diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index b5b2765e43844f..d3e842d9f31d01 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,4 +1,4 @@ -trigger: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7'] +trigger: ['main', '3.13', '3.12', '3.11', '3.10', '3.9', '3.8'] jobs: - job: Prebuild diff --git a/.azure-pipelines/posix-deps-apt.sh b/.azure-pipelines/posix-deps-apt.sh deleted file mode 100755 index e0f4ca5d8d8e88..00000000000000 --- a/.azure-pipelines/posix-deps-apt.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -apt-get update - -apt-get -yq install \ - build-essential \ - zlib1g-dev \ - libbz2-dev \ - liblzma-dev \ - libncurses5-dev \ - libreadline6-dev \ - libsqlite3-dev \ - libssl-dev \ - libgdbm-dev \ - tk-dev \ - lzma \ - lzma-dev \ - liblzma-dev \ - libffi-dev \ - uuid-dev \ - xvfb - -if [ ! -z "$1" ] -then - echo ##vso[task.prependpath]$PWD/multissl/openssl/$1 - echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1 - python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux -fi diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml deleted file mode 100644 index e23c7b1dcb55c1..00000000000000 --- a/.azure-pipelines/posix-steps.yml +++ /dev/null @@ -1,26 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 5 - -# Work around a known issue affecting Ubuntu VMs on Pipelines -- script: sudo setfacl -Rb /home/vsts - displayName: 'Workaround ACL issue' - -- script: sudo ./.azure-pipelines/posix-deps-apt.sh $(openssl_version) - displayName: 'Install dependencies' - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - -- script: make -j4 - displayName: 'Build CPython' - -- script: make pythoninfo - displayName: 'Display build info' - -- script: | - git fetch origin - ./python Tools/patchcheck/patchcheck.py --ci true - displayName: 'Run patchcheck.py' - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml deleted file mode 100644 index 335a4b407cb83c..00000000000000 --- a/.azure-pipelines/pr.yml +++ /dev/null @@ -1,28 +0,0 @@ -pr: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7'] - -jobs: -- job: Prebuild - displayName: Pre-build checks - - pool: - vmImage: ubuntu-22.04 - - steps: - - template: ./prebuild-checks.yml - - -- job: Ubuntu_Patchcheck - displayName: Ubuntu patchcheck - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-linux' - testRunPlatform: linux - openssl_version: 1.1.1u - - steps: - - template: ./posix-steps.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 9f808af38e69df..00000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM docker.io/library/fedora:37 - -ENV CC=clang - -ENV WASI_SDK_VERSION=20 -ENV WASI_SDK_PATH=/opt/wasi-sdk - -ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=14.0.4 -ENV WASMTIME_CPU_ARCH=x86_64 - -RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ - dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \ - dnf -y clean all - -RUN mkdir ${WASI_SDK_PATH} && \ - curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-linux.tar.gz | \ - tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip - -RUN mkdir --parents ${WASMTIME_HOME} && \ - curl --location "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_CPU_ARCH}-linux.tar.xz" | \ - xz --decompress | \ - tar --strip-components 1 --directory ${WASMTIME_HOME} -x && \ - ln -s ${WASMTIME_HOME}/wasmtime /usr/local/bin diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0dc303015df5c7..64c85c1101e6e6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,5 @@ { - "build": { - "dockerfile": "Dockerfile" - }, + "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730", "onCreateCommand": [ // Install common tooling. "dnf", diff --git a/.editorconfig b/.editorconfig index 0169eed951cd3f..a6187d64f3ce46 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,6 +1,6 @@ root = true -[*.{py,c,cpp,h,rst,md,yml}] +[*.{py,c,cpp,h,js,rst,md,yml}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space @@ -11,5 +11,5 @@ indent_size = 4 [*.rst] indent_size = 3 -[*.yml] +[*.{js,yml}] indent_size = 2 diff --git a/.gitattributes b/.gitattributes index 2a48df079e1aeb..2f5a030981fb94 100644 --- a/.gitattributes +++ b/.gitattributes @@ -27,8 +27,6 @@ Lib/test/cjkencodings/* noeol Lib/test/tokenizedata/coding20731.py noeol Lib/test/decimaltestdata/*.decTest noeol Lib/test/test_email/data/*.txt noeol -Lib/test/test_importlib/resources/data01/* noeol -Lib/test/test_importlib/resources/namespacedata01/* noeol Lib/test/xmltestdata/* noeol # Shell scripts should have LF even on Windows because of Cygwin @@ -77,11 +75,14 @@ Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/internal/pycore_uop_ids.h generated +Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/keyword.py generated +Lib/test/certdata/*.pem generated +Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated Lib/token.py generated @@ -94,6 +95,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7933d319550576..9162f9c7bb1576 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,9 +13,11 @@ # Build system configure* @erlend-aasland @corona10 +Makefile.pre.in @erlend-aasland +Modules/Setup* @erlend-aasland # asyncio -**/*asyncio* @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc +**/*asyncio* @1st1 @asvetlov @kumaraditya303 @willingc # Core **/*context* @1st1 @@ -29,17 +31,23 @@ Objects/type* @markshannon Objects/codeobject.c @markshannon Objects/frameobject.c @markshannon Objects/call.c @markshannon -Python/ceval*.c @markshannon @gvanrossum -Python/ceval*.h @markshannon @gvanrossum +Python/ceval*.c @markshannon +Python/ceval*.h @markshannon +Python/codegen.c @markshannon @iritkatriel Python/compile.c @markshannon @iritkatriel Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel -Python/ast_opt.c @isidentical -Python/bytecodes.c @markshannon @gvanrossum -Python/optimizer*.c @markshannon @gvanrossum +Python/instruction_sequence.c @iritkatriel +Python/bytecodes.c @markshannon +Python/optimizer*.c @markshannon +Python/optimizer_analysis.c @Fidget-Spinner +Python/optimizer_bytecodes.c @Fidget-Spinner +Python/symtable.c @JelleZijlstra @carljm +Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra -Lib/test/test_capi/test_misc.py @markshannon @gvanrossum +Lib/test/test_capi/test_misc.py @markshannon +Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv Tools/c-analyzer/ @ericsnowcurrently # dbm @@ -64,17 +72,15 @@ Include/internal/pycore_freelist.h @ericsnowcurrently Include/internal/pycore_global_objects.h @ericsnowcurrently Include/internal/pycore_obmalloc.h @ericsnowcurrently Include/internal/pycore_pymem.h @ericsnowcurrently +Include/internal/pycore_stackref.h @Fidget-Spinner Modules/main.c @ericsnowcurrently Programs/_bootstrap_python.c @ericsnowcurrently Programs/python.c @ericsnowcurrently Tools/build/generate_global_objects.py @ericsnowcurrently # Exceptions -Lib/traceback.py @iritkatriel Lib/test/test_except*.py @iritkatriel -Lib/test/test_traceback.py @iritkatriel Objects/exceptions.c @iritkatriel -Python/traceback.c @iritkatriel # Hashing **/*hashlib* @gpshead @tiran @@ -82,7 +88,6 @@ Python/traceback.c @iritkatriel **/sha* @gpshead @tiran Modules/md5* @gpshead @tiran **/*blake* @gpshead @tiran -Modules/_blake2/** @gpshead @tiran Modules/_hacl/** @gpshead # logging @@ -117,7 +122,7 @@ Python/dynload_*.c @ericsnowcurrently Lib/test/test_module/ @ericsnowcurrently Doc/c-api/module.rst @ericsnowcurrently **/*importlib/resources/* @jaraco @warsaw @FFY00 -**/importlib/metadata/* @jaraco @warsaw +**/*importlib/metadata/* @jaraco @warsaw # Dates and times **/*datetime* @pganssle @abalkin @@ -148,13 +153,15 @@ Include/internal/pycore_time.h @pganssle @abalkin /Lib/test/test_tokenize.py @pablogsal @lysnikolaou # Code generator -/Tools/cases_generator/ @gvanrossum +/Tools/cases_generator/ @markshannon # AST -Python/ast.c @isidentical -Parser/asdl.py @isidentical -Parser/asdl_c.py @isidentical -Lib/ast.py @isidentical +Python/ast.c @isidentical @JelleZijlstra @eclips4 +Python/ast_opt.c @isidentical @eclips4 +Parser/asdl.py @isidentical @JelleZijlstra @eclips4 +Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 +Lib/ast.py @isidentical @JelleZijlstra @eclips4 +Lib/test/test_ast/ @eclips4 # Mock /Lib/unittest/mock.py @cjw296 @@ -171,6 +178,10 @@ Lib/ast.py @isidentical /Lib/test/test_subprocess.py @gpshead /Modules/*subprocess* @gpshead +# debugger +**/*pdb* @gaogaotiantian +**/*bdb* @gaogaotiantian + # Limited C API & stable ABI Tools/build/stable_abi.py @encukou Misc/stable_abi.toml @encukou @@ -192,18 +203,19 @@ Doc/c-api/stable.rst @encukou **/*itertools* @rhettinger **/*collections* @rhettinger **/*random* @rhettinger -**/*queue* @rhettinger **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger -**/*decimal* @rhettinger **/*dataclasses* @ericvsmith **/*ensurepip* @pfmoore @pradyunsg +/Doc/library/idle.rst @terryjreedy **/*idlelib* @terryjreedy +**/*turtledemo* @terryjreedy +**/*annotationlib* @JelleZijlstra **/*typing* @JelleZijlstra @AlexWaygood **/*ftplib @giampaolo @@ -238,12 +250,34 @@ Doc/howto/clinic.rst @erlend-aasland **/*interpreteridobject.* @ericsnowcurrently **/*crossinterp* @ericsnowcurrently Lib/test/support/interpreters/ @ericsnowcurrently -Modules/_xx*interp*module.c @ericsnowcurrently +Modules/_interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently +# Android +**/*Android* @mhsmith @freakboy3742 +**/*android* @mhsmith @freakboy3742 + +# iOS (but not termios) +**/iOS* @freakboy3742 +**/ios* @freakboy3742 +**/*_iOS* @freakboy3742 +**/*_ios* @freakboy3742 +**/*-iOS* @freakboy3742 +**/*-ios* @freakboy3742 + # WebAssembly -/Tools/wasm/ @brettcannon +/Tools/wasm/ @brettcannon @freakboy3742 # SBOM +/Misc/externals.spdx.json @sethmlarson /Misc/sbom.spdx.json @sethmlarson /Tools/build/generate_sbom.py @sethmlarson + +# Config Parser +Lib/configparser.py @jaraco +Lib/test/test_configparser.py @jaraco + +# Doc sections +Doc/reference/ @willingc + +**/*weakref* @kumaraditya303 diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 395516f29b037c..7b7810cf6965fd 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -34,12 +34,12 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index c14d7cf2599d4c..58da2dfe0c7354 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -27,11 +27,12 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" + - "3.13" + - "3.14" - "CPython main branch" validations: required: true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0a2f6da50ed8a0..083b07156674df 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,26 +1,15 @@ name: Tests -# gh-84728: "paths-ignore" is not used to skip documentation-only PRs, because -# it prevents to mark a job as mandatory. A PR cannot be merged if a job is -# mandatory but not scheduled because of "paths-ignore". on: workflow_dispatch: push: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' + - '3.*' pull_request: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' + - '3.*' permissions: contents: read @@ -31,86 +20,19 @@ concurrency: jobs: check_source: - name: 'Check for source changes' - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - run-docs: ${{ steps.docs-changes.outputs.run-docs || false }} - run_tests: ${{ steps.check.outputs.run_tests }} - run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} - run_cifuzz: ${{ steps.check.outputs.run_cifuzz }} - config_hash: ${{ steps.config_hash.outputs.hash }} - steps: - - uses: actions/checkout@v4 - - name: Check for source changes - id: check - run: | - if [ -z "$GITHUB_BASE_REF" ]; then - echo "run_tests=true" >> $GITHUB_OUTPUT - else - git fetch origin $GITHUB_BASE_REF --depth=1 - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more - # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), - # but it requires to download more commits (this job uses - # "git fetch --depth=1"). - # - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git - # 2.26, but Git 2.28 is stricter and fails with "no merge base". - # - # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on - # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF - # into the PR branch anyway. - # - # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true - fi - - # Check if we should run hypothesis tests - GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} - echo $GIT_BRANCH - if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then - echo "Branch too old for hypothesis tests" - echo "run_hypothesis=false" >> $GITHUB_OUTPUT - else - echo "Run hypothesis tests" - echo "run_hypothesis=true" >> $GITHUB_OUTPUT - fi - - # oss-fuzz maintains a configuration for fuzzing the main branch of - # CPython, so CIFuzz should be run only for code that is likely to be - # merged into the main branch; compatibility with older branches may - # be broken. - FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' - if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then - # The tests are pretty slow so they are executed only for PRs - # changing relevant files. - echo "Run CIFuzz tests" - echo "run_cifuzz=true" >> $GITHUB_OUTPUT - else - echo "Branch too old for CIFuzz tests; or no C files were changed" - echo "run_cifuzz=false" >> $GITHUB_OUTPUT - fi - - name: Compute hash for config cache key - id: config_hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT - - name: Get a list of the changed documentation-related files - if: github.event_name == 'pull_request' - id: changed-docs-files - uses: Ana06/get-changed-files@v2.2.0 - with: - filter: | - Doc/** - Misc/** - .github/workflows/reusable-docs.yml - format: csv # works for paths with spaces - - name: Check for docs changes - if: >- - github.event_name == 'pull_request' - && steps.changed-docs-files.outputs.added_modified_renamed != '' - id: docs-changes - run: | - echo "run-docs=true" >> "${GITHUB_OUTPUT}" + name: Change detection + # To use boolean outputs from this job, parse them as JSON. + # Here's some examples: + # + # if: fromJSON(needs.check_source.outputs.run-docs) + # + # ${{ + # fromJSON(needs.check_source.outputs.run_tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # + uses: ./.github/workflows/reusable-change-detection.yml check-docs: name: Docs @@ -118,6 +40,50 @@ jobs: if: fromJSON(needs.check_source.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml + check_autoconf_regen: + name: 'Check if Autoconf files are up to date' + # Don't use ubuntu-latest but a specific version to make the job + # reproducible: to get the same tools versions (autoconf, aclocal, ...) + runs-on: ubuntu-24.04 + container: + image: ghcr.io/python/autoconf:2024.10.16.11360930377 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - name: Install Git + run: | + apt install git -yq + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" + - name: Check Autoconf and aclocal versions + run: | + grep "Generated by GNU Autoconf 2.71" configure + grep "aclocal 1.16.5" aclocal.m4 + grep -q "runstatedir" configure + grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 + - name: Regenerate autoconf files + # Same command used by Tools/build/regen-configure.sh ($AUTORECONF) + run: autoreconf -ivf -Werror + - name: Check for changes + run: | + git add -u + changes=$(git status --porcelain) + # Check for changes in regenerated files + if test -n "$changes"; then + echo "Generated files not up to date." + echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)" + echo "configure files must be regenerated with a specific version of autoconf." + echo "$changes" + echo "" + git diff --staged || true + exit 1 + fi + check_generated_files: name: 'Check if generated files are up to date' # Don't use ubuntu-latest but a specific version to make the job @@ -131,36 +97,30 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.x' + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: save: false - - name: Check Autoconf and aclocal versions - run: | - grep "Generated by GNU Autoconf 2.71" configure - grep "aclocal 1.16.5" aclocal.m4 - grep -q "runstatedir" configure - grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 - name: Configure CPython run: | # Build Python with the libpython dynamic library ./configure --config-cache --with-pydebug --enable-shared - - name: Regenerate autoconf files - # Same command used by Tools/build/regen-configure.sh ($AUTORECONF) - run: autoreconf -ivf -Werror - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names + make regen-stdlib-module-names regen-sbom - name: Check for changes run: | git add -u @@ -184,73 +144,101 @@ jobs: run: make check-c-globals build_windows: - name: 'Windows' + name: >- + Windows + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-windows.yml - - build_windows_free_threading: - name: 'Windows (free-threading)' - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: fromJSON(needs.check_source.outputs.run_tests) + strategy: + matrix: + arch: + - Win32 + - x64 + - arm64 + free-threading: + - false + - true uses: ./.github/workflows/reusable-windows.yml with: - free-threading: true + arch: ${{ matrix.arch }} + free-threading: ${{ matrix.free-threading }} - build_macos: - name: 'macOS' + build_windows_msi: + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Windows MSI${{ '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-macos.yml + if: fromJSON(needs.check_source.outputs.run-win-msi) + strategy: + matrix: + arch: + - x86 + - x64 + - arm64 + uses: ./.github/workflows/reusable-windows-msi.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} + arch: ${{ matrix.arch }} - build_macos_free_threading: - name: 'macOS (free-threading)' + build_macos: + name: >- + macOS + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source if: needs.check_source.outputs.run_tests == 'true' + strategy: + fail-fast: false + matrix: + # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. + # macOS 13 only runs tests against the GIL-enabled CPython. + # Cirrus used for upstream, macos-14 for forks. + os: + - ghcr.io/cirruslabs/macos-runner:sonoma + - macos-14 + - macos-13 + is-fork: # only used for the exclusion trick + - ${{ github.repository_owner != 'python' }} + free-threading: + - false + - true + exclude: + - os: ghcr.io/cirruslabs/macos-runner:sonoma + is-fork: true + - os: macos-14 + is-fork: false + - os: macos-13 + free-threading: true uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - free-threading: true + free-threading: ${{ matrix.free-threading }} + os: ${{ matrix.os }} build_ubuntu: - name: 'Ubuntu' - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-ubuntu.yml - with: - config_hash: ${{ needs.check_source.outputs.config_hash }} - options: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR - - build_ubuntu_free_threading: - name: 'Ubuntu (free-threading)' + name: >- + Ubuntu + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source if: needs.check_source.outputs.run_tests == 'true' + strategy: + matrix: + free-threading: + - false + - true uses: ./.github/workflows/reusable-ubuntu.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - options: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR \ - --disable-gil + free-threading: ${{ matrix.free-threading }} build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' strategy: fail-fast: false matrix: - openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1] + os: [ubuntu-22.04] + openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -258,38 +246,40 @@ jobs: LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: save: false - name: Configure CPython - run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR + run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - name: Build CPython run: make -j4 - name: Display build info @@ -297,14 +287,22 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build_wasi: + name: 'WASI' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-wasi.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + test_hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 3.0.13 + OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -314,9 +312,9 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -325,34 +323,37 @@ jobs: key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: save: false - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | ../cpython-ro-srcdir/configure \ --config-cache \ --with-pydebug \ - --with-openssl=$OPENSSL_DIR + --enable-slower-safety \ + --with-openssl="$OPENSSL_DIR" - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 @@ -361,26 +362,26 @@ jobs: run: make pythoninfo - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: "Create hypothesis venv" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | VENV_LOC=$(realpath -m .)/hypovenv VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV - echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV - ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt + echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" + echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" + ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database uses: actions/cache@v4 with: - path: ./hypothesis + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} restore-keys: | - - hypothesis-database- + hypothesis-database- - name: "Run tests" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -405,26 +406,31 @@ jobs: if: always() with: name: hypothesis-example-db - path: .hypothesis/examples/ + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ build_asan: name: 'Address sanitizer' - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' + strategy: + matrix: + os: [ubuntu-22.04] env: - OPENSSL_VER: 3.0.13 + OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -435,21 +441,21 @@ jobs: version: 10 - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -462,7 +468,29 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make test + run: xvfb-run make ci + + build_tsan: + name: 'Thread sanitizer' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: ./configure --config-cache --with-thread-sanitizer --with-pydebug + suppressions_path: Tools/tsan/supressions.txt + tsan_logs_artifact_name: tsan-logs-default + + build_tsan_free_threading: + name: 'Thread sanitizer (free-threading)' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug + suppressions_path: Tools/tsan/suppressions_free_threading.txt + tsan_logs_artifact_name: tsan-logs-free-threading # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: @@ -511,16 +539,18 @@ jobs: needs: - check_source # Transitive dependency, needed to access `run_tests` value - check-docs + - check_autoconf_regen - check_generated_files - build_macos - - build_macos_free_threading - build_ubuntu - - build_ubuntu_free_threading - build_ubuntu_ssltests + - build_wasi - build_windows - - build_windows_free_threading + - build_windows_msi - test_hypothesis - build_asan + - build_tsan + - build_tsan_free_threading - cifuzz runs-on: ubuntu-latest @@ -531,6 +561,7 @@ jobs: with: allowed-failures: >- build_ubuntu_ssltests, + build_windows_msi, cifuzz, test_hypothesis, allowed-skips: >- @@ -544,15 +575,16 @@ jobs: ${{ needs.check_source.outputs.run_tests != 'true' && ' + check_autoconf_regen, check_generated_files, build_macos, - build_macos_free_threading, build_ubuntu, - build_ubuntu_free_threading, build_ubuntu_ssltests, + build_wasi, build_windows, - build_windows_free_threading, build_asan, + build_tsan, + build_tsan_free_threading, ' || '' }} diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml deleted file mode 100644 index 65d32c734e7745..00000000000000 --- a/.github/workflows/build_msi.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: TestsMSI - -on: - workflow_dispatch: - push: - branches: - - 'main' - - '3.*' - paths: - - 'Tools/msi/**' - - '.github/workflows/build_msi.yml' - pull_request: - branches: - - 'main' - - '3.*' - paths: - - 'Tools/msi/**' - - '.github/workflows/build_msi.yml' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build: - name: Windows Installer - runs-on: windows-latest - timeout-minutes: 60 - strategy: - matrix: - type: [x86, x64, arm64] - env: - IncludeFreethreaded: true - steps: - - uses: actions/checkout@v4 - - name: Build CPython installer - run: .\Tools\msi\build.bat --doc -${{ matrix.type }} diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 69648d87947ad6..48f05818a38f96 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -4,27 +4,53 @@ on: paths: - '**jit**' - 'Python/bytecodes.c' + - 'Python/optimizer*.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' push: paths: - '**jit**' - 'Python/bytecodes.c' + - 'Python/optimizer*.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' workflow_dispatch: +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: + interpreter: + name: Interpreter (Debug) + runs-on: ubuntu-22.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - name: Build tier two interpreter + run: | + ./configure --enable-experimental-jit=interpreter --with-pydebug + make all --jobs 4 + - name: Test tier two interpreter + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 jit: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + needs: interpreter runs-on: ${{ matrix.runner }} - timeout-minutes: 60 + timeout-minutes: 90 strategy: fail-fast: false matrix: target: - i686-pc-windows-msvc/msvc - x86_64-pc-windows-msvc/msvc + - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -35,7 +61,7 @@ jobs: - true - false llvm: - - 16 + - 19 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -45,6 +71,10 @@ jobs: architecture: x64 runner: windows-latest compiler: msvc + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-latest + compiler: msvc - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-13 @@ -55,24 +85,20 @@ jobs: compiler: clang - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: gcc - target: x86_64-unknown-linux-gnu/clang architecture: x86_64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: clang - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: gcc - # These fail because of emulation, not because of the JIT: - exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv - target: aarch64-unknown-linux-gnu/clang architecture: aarch64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: clang - # These fail because of emulation, not because of the JIT: - exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv env: CC: ${{ matrix.compiler }} steps: @@ -81,21 +107,34 @@ jobs: with: python-version: '3.11' - - name: Windows - if: runner.os == 'Windows' + - name: Native Windows + if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} - ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + # No PGO or tests (yet): + - name: Emulated Windows + if: runner.os == 'Windows' && matrix.architecture == 'ARM64' + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} - - name: macOS + # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. + # This is a bug in the macOS runner image where the pre-installed Python is installed in the same + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # the symlink to the pre-installed Python so that the Homebrew Python is used instead. + - name: Native macOS if: runner.os == 'macOS' run: | + brew update + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} - export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + SDKROOT="$(xcrun --show-sdk-path)" \ + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 - ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - name: Native Linux if: runner.os == 'Linux' && matrix.architecture == 'x86_64' @@ -104,10 +143,11 @@ jobs: export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - name: Emulated Linux if: runner.os == 'Linux' && matrix.architecture != 'x86_64' + # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux. run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" @@ -118,10 +158,33 @@ jobs: sudo apt install --yes "gcc-$HOST" qemu-user ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} - export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" - export CPP="$CC --preprocess" - export HOSTRUNNER=qemu-${{ matrix.architecture }} export QEMU_LD_PREFIX="/usr/$HOST" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ + CPP="$CC --preprocess" \ + HOSTRUNNER=qemu-${{ matrix.architecture }} \ + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + jit-with-disabled-gil: + name: Free-Threaded (Debug) + needs: interpreter + runs-on: ubuntu-22.04 + strategy: + matrix: + llvm: + - 19 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT enabled and GIL disabled + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit --with-pydebug --disable-gil + make all --jobs 4 + - name: Run tests + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4a70ec6205a05b..ccde03f91983df 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,4 +23,4 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.x" - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index b766785de405d2..e5b05302b5ac27 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,6 +8,7 @@ on: pull_request: paths: - ".github/workflows/mypy.yml" + - "Lib/_pyrepl/**" - "Lib/test/libregrtest/**" - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" @@ -33,10 +34,12 @@ concurrency: jobs: mypy: strategy: + fail-fast: false matrix: target: [ + "Lib/_pyrepl", "Lib/test/libregrtest", - "Tools/build/", + "Tools/build", "Tools/cases_generator", "Tools/clinic", "Tools/jit", @@ -50,7 +53,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.13" cache: pip cache-dependency-path: Tools/requirements-dev.txt - run: pip install -r Tools/requirements-dev.txt diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 0800401f4cd113..bfc5a0874281bd 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -1,11 +1,9 @@ #!/bin/sh apt-get update -# autoconf-archive is needed by autoreconf (check_generated_files job) apt-get -yq install \ build-essential \ pkg-config \ - autoconf-archive \ ccache \ gdb \ lcov \ @@ -15,6 +13,7 @@ apt-get -yq install \ libgdbm-dev \ libgdbm-compat-dev \ liblzma-dev \ + libmpdec-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 7574bfc208ff76..066d8593a70cf6 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -23,7 +23,7 @@ jobs: - { project: 32, label: sprint } steps: - - uses: actions/add-to-project@v0.1.0 + - uses: actions/add-to-project@v1.0.0 with: project-url: https://github.com/orgs/python/projects/${{ matrix.project }} github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index ff5cbdf3eda749..bbedd22cc6d189 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -9,15 +9,53 @@ permissions: pull-requests: write jobs: - label: - name: DO-NOT-MERGE / unresolved review + label-dnm: + name: DO-NOT-MERGE if: github.repository_owner == 'python' runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: mheap/github-action-required-labels@v5 + - name: Check there's no DO-NOT-MERGE + uses: mheap/github-action-required-labels@v5 with: mode: exactly count: 0 - labels: "DO-NOT-MERGE, awaiting changes, awaiting change review" + labels: | + DO-NOT-MERGE + + label-reviews: + name: Unresolved review + if: github.repository_owner == 'python' + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + # Check that the PR is not awaiting changes from the author due to previous review. + - name: Check there's no required changes + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 0 + labels: | + awaiting changes + awaiting change review + - id: is-feature + name: Check whether this PR is a feature (contains a "type-feature" label) + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 1 + labels: | + type-feature + exit_type: success # don't fail the check if the PR is not a feature, just record the result + # In case of a feature PR, check for a complete review (contains an "awaiting merge" label). + - id: awaiting-merge + if: steps.is-feature.outputs.status == 'success' + name: Check for complete review + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 1 + labels: | + awaiting merge diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml new file mode 100644 index 00000000000000..1a6fd33186840c --- /dev/null +++ b/.github/workflows/reusable-change-detection.yml @@ -0,0 +1,156 @@ +name: Reusable change detection + +on: # yamllint disable-line rule:truthy + workflow_call: + outputs: + # Some of the referenced steps set outputs conditionally and there may be + # cases when referencing them evaluates to empty strings. It is nice to + # work with proper booleans so they have to be evaluated through JSON + # conversion in the expressions. However, empty strings used like that + # may trigger all sorts of undefined and hard-to-debug behaviors in + # GitHub Actions CI/CD. To help with this, all of the outputs set here + # that are meant to be used as boolean flags (and not arbitrary strings), + # MUST have fallbacks with default values set. A common pattern would be + # to add ` || false` to all such expressions here, in the output + # definitions. They can then later be safely used through the following + # idiom in job conditionals and other expressions. Here's some examples: + # + # if: fromJSON(needs.change-detection.outputs.run-docs) + # + # ${{ + # fromJSON(needs.change-detection.outputs.run-tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # + config_hash: + description: Config hash value for use in cache keys + value: ${{ jobs.compute-changes.outputs.config-hash }} # str + run-docs: + description: Whether to build the docs + value: ${{ jobs.compute-changes.outputs.run-docs || false }} # bool + run_tests: + description: Whether to run the regular tests + value: ${{ jobs.compute-changes.outputs.run-tests || false }} # bool + run-win-msi: + description: Whether to run the MSI installer smoke tests + value: >- # bool + ${{ jobs.compute-changes.outputs.run-win-msi || false }} + run_hypothesis: + description: Whether to run the Hypothesis tests + value: >- # bool + ${{ jobs.compute-changes.outputs.run-hypothesis || false }} + run_cifuzz: + description: Whether to run the CIFuzz job + value: >- # bool + ${{ jobs.compute-changes.outputs.run-cifuzz || false }} + +jobs: + compute-changes: + name: Compute changed files + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + config-hash: ${{ steps.config-hash.outputs.hash }} + run-cifuzz: ${{ steps.check.outputs.run-cifuzz }} + run-docs: ${{ steps.docs-changes.outputs.run-docs }} + run-hypothesis: ${{ steps.check.outputs.run-hypothesis }} + run-tests: ${{ steps.check.outputs.run-tests }} + run-win-msi: ${{ steps.win-msi-changes.outputs.run-win-msi }} + steps: + - run: >- + echo '${{ github.event_name }}' + - uses: actions/checkout@v4 + - name: Check for source changes + id: check + run: | + if [ -z "$GITHUB_BASE_REF" ]; then + echo "run-tests=true" >> "$GITHUB_OUTPUT" + else + git fetch origin "$GITHUB_BASE_REF" --depth=1 + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more + # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), + # but it requires to download more commits (this job uses + # "git fetch --depth=1"). + # + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git + # 2.26, but Git 2.28 is stricter and fails with "no merge base". + # + # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on + # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF + # into the PR branch anyway. + # + # https://github.com/python/core-workflow/issues/373 + git diff --name-only "origin/$GITHUB_BASE_REF.." | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run-tests=true" >> "$GITHUB_OUTPUT" || true + fi + + # Check if we should run hypothesis tests + GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} + echo "$GIT_BRANCH" + if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then + echo "Branch too old for hypothesis tests" + echo "run-hypothesis=false" >> "$GITHUB_OUTPUT" + else + echo "Run hypothesis tests" + echo "run-hypothesis=true" >> "$GITHUB_OUTPUT" + fi + + # oss-fuzz maintains a configuration for fuzzing the main branch of + # CPython, so CIFuzz should be run only for code that is likely to be + # merged into the main branch; compatibility with older branches may + # be broken. + FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' + if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only "origin/$GITHUB_BASE_REF.." | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then + # The tests are pretty slow so they are executed only for PRs + # changing relevant files. + echo "Run CIFuzz tests" + echo "run-cifuzz=true" >> "$GITHUB_OUTPUT" + else + echo "Branch too old for CIFuzz tests; or no C files were changed" + echo "run-cifuzz=false" >> "$GITHUB_OUTPUT" + fi + - name: Compute hash for config cache key + id: config-hash + run: | + echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" + - name: Get a list of the changed documentation-related files + if: github.event_name == 'pull_request' + id: changed-docs-files + uses: Ana06/get-changed-files@v2.3.0 + with: + filter: | + Doc/** + Misc/** + .github/workflows/reusable-docs.yml + format: csv # works for paths with spaces + - name: Check for docs changes + # We only want to run this on PRs when related files are changed, + # or when user triggers manual workflow run. + if: >- + ( + github.event_name == 'pull_request' + && steps.changed-docs-files.outputs.added_modified_renamed != '' + ) || github.event_name == 'workflow_dispatch' + id: docs-changes + run: | + echo "run-docs=true" >> "${GITHUB_OUTPUT}" + - name: Get a list of the MSI installer-related files + if: github.event_name == 'pull_request' + id: changed-win-msi-files + uses: Ana06/get-changed-files@v2.3.0 + with: + filter: | + Tools/msi/** + .github/workflows/reusable-windows-msi.yml + format: csv # works for paths with spaces + - name: Check for changes in MSI installer-related files + # We only want to run this on PRs when related files are changed, + # or when user triggers manual workflow run. + if: >- + ( + github.event_name == 'pull_request' + && steps.changed-win-msi-files.outputs.added_modified_renamed != '' + ) || github.event_name == 'workflow_dispatch' + id: win-msi-changes + run: | + echo "run-win-msi=true" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index cea8f93d67b29c..39a97392e898aa 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -1,4 +1,4 @@ -name: Docs +name: Reusable Docs on: workflow_call: @@ -11,6 +11,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: build_doc: name: 'Docs' @@ -25,9 +28,15 @@ jobs: - name: 'Check out latest PR branch commit' uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: >- + ${{ + github.event_name == 'pull_request' + && github.event.pull_request.head.sha + || '' + }} # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 - name: 'Fetch commits to get branch diff' + if: github.event_name == 'pull_request' run: | # Fetch enough history to find a common ancestor commit (aka merge-base): git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \ @@ -62,7 +71,8 @@ jobs: python Doc/tools/check-warnings.py \ --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ --fail-if-regression \ - --fail-if-improved + --fail-if-improved \ + --fail-if-new-news-nit # This build doesn't use problem matchers or check annotations build_doc_oldest_supported_sphinx: @@ -74,7 +84,7 @@ jobs: - name: 'Set up Python' uses: actions/setup-python@v5 with: - python-version: '3.11' # known to work with Sphinx 4.2 + python-version: '3.13' # known to work with Sphinx 7.2.6 cache: 'pip' cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt' - name: 'Install build dependencies' @@ -85,7 +95,7 @@ jobs: # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: name: 'Doctest' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index cad619b78ce5f2..915481d0737c7d 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -1,3 +1,5 @@ +name: Reusable macOS + on: workflow_call: inputs: @@ -8,33 +10,34 @@ on: required: false type: boolean default: false + os: + description: OS to run the job + required: true + type: string jobs: build_macos: - name: 'build and test' + name: build and test (${{ inputs.os }}) timeout-minutes: 60 env: HOMEBREW_NO_ANALYTICS: 1 HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 - strategy: - fail-fast: false - matrix: - os: [ - "macos-14", # M1 - "macos-13", # Intel - ] - runs-on: ${{ matrix.os }} + TERM: linux + runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ matrix.os }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make - name: Configure CPython run: | GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ @@ -42,12 +45,28 @@ jobs: ./configure \ --config-cache \ --with-pydebug \ + --enable-slower-safety \ + --enable-safety \ ${{ inputs.free-threading && '--disable-gil' || '' }} \ --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: make -j4 + if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + run: gmake -j8 + - name: Build CPython for compiler warning check + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo + - name: Check compiler warnings + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: >- + python3 Tools/build/check_warnings.py + --compiler-output-file-path=compiler_output_macos.txt + --warning-ignore-file-path=Tools/build/.warningignore_macos + --compiler-output-type=clang + --fail-on-regression + --fail-on-improvement + --path-prefix="./" - name: Tests - run: make test + run: make ci diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml new file mode 100644 index 00000000000000..65072efa8e9023 --- /dev/null +++ b/.github/workflows/reusable-tsan.yml @@ -0,0 +1,78 @@ +name: Reusable Thread Sanitizer + +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + options: + required: true + type: string + suppressions_path: + description: 'A repo relative path to the suppressions file' + required: true + type: string + tsan_logs_artifact_name: + description: 'Name of the TSAN logs artifact. Must be unique for each job.' + required: true + type: string + +jobs: + build_tsan_reusable: + name: 'Thread sanitizer' + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + # Install clang-18 + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 + sudo update-alternatives --set clang /usr/bin/clang-17 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-17 + # Reduce ASLR to avoid TSAN crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + - name: TSAN Option Setup + run: | + echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> "$GITHUB_ENV" + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: ${{ inputs.options }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: ./python -m test --tsan -j4 + - name: Display TSAN logs + if: always() + run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000 + - name: Archive TSAN logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.tsan_logs_artifact_name }} + path: tsan_log.* + if-no-files-found: ignore diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 0cbad57f0c6572..f0ca6a9e7ed793 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -1,21 +1,31 @@ +name: Reusable Ubuntu + on: workflow_call: inputs: config_hash: required: true type: string - options: - required: true - type: string + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false jobs: build_ubuntu_reusable: name: 'build and test' timeout-minutes: 60 - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04] env: - OPENSSL_VER: 3.0.13 + FORCE_COLOR: 1 + OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux steps: - uses: actions/checkout@v4 - name: Register gcc problem matcher @@ -24,21 +34,21 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -46,29 +56,53 @@ jobs: max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: ${{ inputs.options }} + run: >- + ../cpython-ro-srcdir/configure + --config-cache + --with-pydebug + --enable-slower-safety + --enable-safety + --with-openssl="$OPENSSL_DIR" + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Build CPython out-of-tree + if: ${{ inputs.free-threading }} working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 + - name: Build CPython out-of-tree (for compiler warning check) + if: ${{ !inputs.free-threading}} + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: set -o pipefail; make -j4 --output-sync 2>&1 | tee compiler_output_ubuntu.txt - name: Display build info working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo + - name: Check compiler warnings + if: ${{ !inputs.free-threading }} + run: >- + python Tools/build/check_warnings.py + --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt + --warning-ignore-file-path "${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu" + --compiler-output-type=gcc + --fail-on-regression + --fail-on-improvement + --path-prefix="../cpython-ro-srcdir/" - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make test + run: xvfb-run make ci diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml new file mode 100644 index 00000000000000..abc617a317cc0f --- /dev/null +++ b/.github/workflows/reusable-wasi.yml @@ -0,0 +1,77 @@ +name: Reusable WASI + +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + +jobs: + build_wasi_reusable: + name: 'build and test' + timeout-minutes: 60 + runs-on: ubuntu-22.04 + env: + WASMTIME_VERSION: 22.0.0 + WASI_SDK_VERSION: 24 + WASI_SDK_PATH: /opt/wasi-sdk + CROSS_BUILD_PYTHON: cross-build/build + CROSS_BUILD_WASI: cross-build/wasm32-wasi + steps: + - uses: actions/checkout@v4 + # No problem resolver registered as one doesn't currently exist for Clang. + - name: "Install wasmtime" + uses: bytecodealliance/actions/wasmtime/setup@v1 + with: + version: ${{ env.WASMTIME_VERSION }} + - name: "Restore WASI SDK" + id: cache-wasi-sdk + uses: actions/cache@v4 + with: + path: ${{ env.WASI_SDK_PATH }} + key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} + - name: "Install WASI SDK" + if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + run: | + mkdir ${{ env.WASI_SDK_PATH }} && \ + curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-x86_64-linux.tar.gz | \ + tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip + - name: "Configure ccache action" + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: "Add ccache to PATH" + run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: "Install Python" + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: "Restore Python build config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. + # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. + # (Make sure to keep the key in sync with the other config.cache step below.) + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + - name: "Configure build Python" + run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Tools/wasm/wasi.py make-build-python + - name: "Restore host config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_WASI }}/config.cache + # Should be kept in sync with the other config.cache step above. + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + - name: "Configure host" + # `--with-pydebug` inferred from configure-build-python + run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + - name: "Make host" + run: python3 Tools/wasm/wasi.py make-host + - name: "Display build info" + run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo + - name: "Test" + run: make --directory ${{ env.CROSS_BUILD_WASI }} test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml new file mode 100644 index 00000000000000..abdb1a1982fef8 --- /dev/null +++ b/.github/workflows/reusable-windows-msi.yml @@ -0,0 +1,24 @@ +name: Reusable Windows MSI + +on: + workflow_call: + inputs: + arch: + description: CPU architecture + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: installer for ${{ inputs.arch }} + runs-on: windows-latest + timeout-minutes: 60 + env: + IncludeFreethreaded: true + steps: + - uses: actions/checkout@v4 + - name: Build CPython installer + run: .\Tools\msi\build.bat --doc -${{ inputs.arch }} diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index c0209e0e1c92e9..dcfc62d7f5d145 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -1,53 +1,47 @@ +name: Reusable Windows + on: workflow_call: inputs: + arch: + description: CPU architecture + required: true + type: string free-threading: + description: Whether to compile CPython in free-threading mode required: false type: boolean default: false -jobs: - build_win32: - name: 'build and test (x86)' - runs-on: windows-latest - timeout-minutes: 60 - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p Win32 ${{ inputs.free-threading && '--disable-gil' || '' }} - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} +env: + IncludeUwp: >- + true - build_win_amd64: - name: 'build and test (x64)' +jobs: + build: + name: >- + build${{ inputs.arch != 'arm64' && ' and test' || '' }} + (${{ inputs.arch }}) runs-on: windows-latest timeout-minutes: 60 - env: - IncludeUwp: 'true' steps: - uses: actions/checkout@v4 - name: Register MSVC problem matcher + if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p x64 ${{ inputs.free-threading && '--disable-gil' || '' }} + run: >- + .\PCbuild\build.bat + -e -d -v + -p ${{ inputs.arch }} + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Display build info + if: inputs.arch != 'arm64' run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} - - build_win_arm64: - name: 'build (arm64)' - runs-on: windows-latest - timeout-minutes: 60 - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Register MSVC problem matcher - run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p arm64 ${{ inputs.free-threading && '--disable-gil' || '' }} + if: inputs.arch != 'arm64' + run: >- + .\PCbuild\rt.bat + -p ${{ inputs.arch }} + -d -q --fast-ci + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 07608fe91b4dbe..f97587e68cbbe4 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,7 @@ name: Mark stale pull requests on: schedule: - - cron: "0 0 * * *" + - cron: "0 */6 * * *" permissions: pull-requests: write diff --git a/.gitignore b/.gitignore index 6ed7197e3ab626..8872e9d5508ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,17 @@ Lib/test/data/* /_bootstrap_python /Makefile /Makefile.pre +/iOSTestbed.* +iOS/Frameworks/ +iOS/Resources/Info.plist +iOS/testbed/build +iOS/testbed/Python.xcframework/ios-*/bin +iOS/testbed/Python.xcframework/ios-*/include +iOS/testbed/Python.xcframework/ios-*/lib +iOS/testbed/Python.xcframework/ios-*/Python.framework +iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace +iOS/testbed/iOSTestbed.xcodeproj/xcuserdata +iOS/testbed/iOSTestbed.xcodeproj/xcshareddata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -131,7 +142,6 @@ Tools/unicode/data/ /profile-clean-stamp /profile-run-stamp /profile-bolt-stamp -/Python/deepfreeze/*.c /pybuilddir.txt /pyconfig.h /python-config diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69d85238985150..ec769d7ff70314 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,46 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.0 + rev: v0.7.1 hooks: - id: ruff - name: Run Ruff on Lib/test/ + name: Run Ruff (lint) on Doc/ + args: [--exit-non-zero-on-fix] + files: ^Doc/ + - id: ruff + name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ - id: ruff - name: Run Ruff on Argument Clinic + name: Run Ruff (lint) on Tools/build/check_warnings.py + args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] + files: ^Tools/build/check_warnings.py + - id: ruff + name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - id: ruff-format + name: Run Ruff (format) on Doc/ + args: [--check] + files: ^Doc/ + + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.10.0 + hooks: + - id: black + name: Run Black on Tools/build/check_warnings.py + files: ^Tools/build/check_warnings.py + language_version: python3.12 + args: [--line-length=79] + - id: black + name: Run Black on Tools/jit/ + files: ^Tools/jit/ + language_version: python3.12 - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: + - id: check-case-conflict + - id: check-merge-conflict - id: check-toml exclude: ^Lib/test/test_tomllib/ - id: check-yaml @@ -23,13 +50,44 @@ repos: - id: trailing-whitespace types_or: [c, inc, python, rst] + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.29.4 + hooks: + - id: check-dependabot + - id: check-github-workflows + + - repo: https://github.com/rhysd/actionlint + rev: v1.7.3 + hooks: + - id: actionlint + args: [ + -ignore=1st argument of function call is not assignable, + -ignore=SC2(015|038|086|091|097|098|129|155), + ] + - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.9.1 + rev: v1.0.0 hooks: - id: sphinx-lint args: [--enable=default-role] files: ^Doc/|^Misc/NEWS.d/ + - repo: local + hooks: + - id: blurb-no-space-c-api + name: Check C API news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/C_API/ + files: Misc/NEWS.d/next/C API/20.*.rst + + - repo: local + hooks: + - id: blurb-no-space-core-and-builtins + name: Check Core and Builtins news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/Core_and_Builtins/ + files: Misc/NEWS.d/next/Core and Builtins/20.*.rst + - repo: meta hooks: - id: check-hooks-apply diff --git a/.readthedocs.yml b/.readthedocs.yml index 59830c79a404e0..a57de00544e4e3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,7 +8,7 @@ sphinx: configuration: Doc/conf.py build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: python: "3" @@ -26,6 +26,9 @@ build: exit 183; fi + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest - make -C Doc venv html - mkdir _readthedocs - mv Doc/build/html _readthedocs/html diff --git a/Android/README.md b/Android/README.md new file mode 100644 index 00000000000000..3daa545cc93746 --- /dev/null +++ b/Android/README.md @@ -0,0 +1,136 @@ +# Python for Android + +These instructions are only needed if you're planning to compile Python for +Android yourself. Most users should *not* need to do this. Instead, use one of +the tools listed in `Doc/using/android.rst`, which will provide a much easier +experience. + + +## Prerequisites + +First, make sure you have all the usual tools and libraries needed to build +Python for your development machine. + +Second, you'll need an Android SDK. If you already have the SDK installed, +export the `ANDROID_HOME` environment variable to point at its location. +Otherwise, here's how to install it: + +* Download the "Command line tools" from . +* Create a directory `android-sdk/cmdline-tools`, and unzip the command line + tools package into it. +* Rename `android-sdk/cmdline-tools/cmdline-tools` to + `android-sdk/cmdline-tools/latest`. +* `export ANDROID_HOME=/path/to/android-sdk` + +The `android.py` script also requires the following commands to be on the `PATH`: + +* `curl` +* `java` (or set the `JAVA_HOME` environment variable) +* `tar` +* `unzip` + + +## Building + +Python can be built for Android on any POSIX platform supported by the Android +development tools, which currently means Linux or macOS. This involves doing a +cross-build where you use a "build" Python (for your development machine) to +help produce a "host" Python for Android. + +The easiest way to do a build is to use the `android.py` script. You can either +have it perform the entire build process from start to finish in one step, or +you can do it in discrete steps that mirror running `configure` and `make` for +each of the two builds of Python you end up producing. + +The discrete steps for building via `android.py` are: + +```sh +./android.py configure-build +./android.py make-build +./android.py configure-host HOST +./android.py make-host HOST +``` + +`HOST` identifies which architecture to build. To see the possible values, run +`./android.py configure-host --help`. + +To do all steps in a single command, run: + +```sh +./android.py build HOST +``` + +In the end you should have a build Python in `cross-build/build`, and an Android +build in `cross-build/HOST`. + +You can use `--` as a separator for any of the `configure`-related commands – +including `build` itself – to pass arguments to the underlying `configure` +call. For example, if you want a pydebug build that also caches the results from +`configure`, you can do: + +```sh +./android.py build HOST -- -C --with-pydebug +``` + + +## Testing + +The test suite can be run on Linux, macOS, or Windows: + +* On Linux, the emulator needs access to the KVM virtualization interface, and + a DISPLAY environment variable pointing at an X server. +* On Windows, you won't be able to do the build on the same machine, so you'll + have to copy the `cross-build/HOST` directory from somewhere else. + +The test suite can usually be run on a device with 2 GB of RAM, but this is +borderline, so you may need to increase it to 4 GB. As of Android +Studio Koala, 2 GB is the default for all emulators, although the user interface +may indicate otherwise. Locate the emulator's directory under `~/.android/avd`, +and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these +manually to the same value, or use the Android Studio Device Manager, which will +update both files. + +Before running the test suite, follow the instructions in the previous section +to build the architecture you want to test. Then run the test script in one of +the following modes: + +* In `--connected` mode, it runs on a device or emulator you have already + connected to the build machine. List the available devices with + `$ANDROID_HOME/platform-tools/adb devices -l`, then pass a device ID to the + script like this: + + ```sh + ./android.py test --connected emulator-5554 + ``` + +* In `--managed` mode, it uses a temporary headless emulator defined in the + `managedDevices` section of testbed/app/build.gradle.kts. This mode is slower, + but more reproducible. + + We currently define two devices: `minVersion` and `maxVersion`, corresponding + to our minimum and maximum supported Android versions. For example: + + ```sh + ./android.py test --managed maxVersion + ``` + +By default, the only messages the script will show are Python's own stdout and +stderr. Add the `-v` option to also show Gradle output, and non-Python logcat +messages. + +Any other arguments on the `android.py test` command line will be passed through +to `python -m test` – use `--` to separate them from android.py's own options. +See the [Python Developer's +Guide](https://devguide.python.org/testing/run-write-tests/) for common options +– most of them will work on Android, except for those that involve subprocesses, +such as `-j`. + +Every time you run `android.py test`, changes in pure-Python files in the +repository's `Lib` directory will be picked up immediately. Changes in C files, +and architecture-specific files such as sysconfigdata, will not take effect +until you re-run `android.py make-host` or `build`. + + +## Using in your own app + +See `Doc/using/android.rst`. diff --git a/Android/android-env.sh b/Android/android-env.sh new file mode 100644 index 00000000000000..a0f23ef8c9fc52 --- /dev/null +++ b/Android/android-env.sh @@ -0,0 +1,97 @@ +# This script must be sourced with the following variables already set: +: ${ANDROID_HOME:?} # Path to Android SDK +: ${HOST:?} # GNU target triplet + +# You may also override the following: +: ${api_level:=24} # Minimum Android API level the build will run on +: ${PREFIX:-} # Path in which to find required libraries + + +# Print all messages on stderr so they're visible when running within build-wheel. +log() { + echo "$1" >&2 +} + +fail() { + log "$1" + exit 1 +} + +# When moving to a new version of the NDK, carefully review the following: +# +# * https://developer.android.com/ndk/downloads/revision_history +# +# * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md +# where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: +# https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md +ndk_version=27.1.12297006 + +ndk=$ANDROID_HOME/ndk/$ndk_version +if ! [ -e $ndk ]; then + log "Installing NDK - this may take several minutes" + yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version" +fi + +if [ $HOST = "arm-linux-androideabi" ]; then + clang_triplet=armv7a-linux-androideabi +else + clang_triplet=$HOST +fi + +# These variables are based on BuildSystemMaintainers.md above, and +# $ndk/build/cmake/android.toolchain.cmake. +toolchain=$(echo $ndk/toolchains/llvm/prebuilt/*) +export AR="$toolchain/bin/llvm-ar" +export AS="$toolchain/bin/llvm-as" +export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CXX="${CC}++" +export LD="$toolchain/bin/ld" +export NM="$toolchain/bin/llvm-nm" +export RANLIB="$toolchain/bin/llvm-ranlib" +export READELF="$toolchain/bin/llvm-readelf" +export STRIP="$toolchain/bin/llvm-strip" + +# The quotes make sure the wildcard in the `toolchain` assignment has been expanded. +for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do + if ! [ -e "$path" ]; then + fail "$path does not exist" + fi +done + +export CFLAGS="-D__BIONIC_NO_PAGE_SIZE_MACRO" +export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,-z,max-page-size=16384" + +# Unlike Linux, Android does not implicitly use a dlopened library to resolve +# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used +# (https://github.com/android/ndk/issues/1244). So any library that fails to +# build with this flag, would also fail to load at runtime. +LDFLAGS="$LDFLAGS -Wl,--no-undefined" + +# Many packages get away with omitting -lm on Linux, but Android is stricter. +LDFLAGS="$LDFLAGS -lm" + +# -mstackrealign is included where necessary in the clang launcher scripts which are +# pointed to by $CC, so we don't need to include it here. +if [ $HOST = "arm-linux-androideabi" ]; then + CFLAGS="$CFLAGS -march=armv7-a -mthumb" +fi + +if [ -n "${PREFIX:-}" ]; then + abs_prefix=$(realpath $PREFIX) + CFLAGS="$CFLAGS -I$abs_prefix/include" + LDFLAGS="$LDFLAGS -L$abs_prefix/lib" + + export PKG_CONFIG="pkg-config --define-prefix" + export PKG_CONFIG_LIBDIR="$abs_prefix/lib/pkgconfig" +fi + +# When compiling C++, some build systems will combine CFLAGS and CXXFLAGS, and some will +# use CXXFLAGS alone. +export CXXFLAGS=$CFLAGS + +# Use the same variable name as conda-build +if [ $(uname) = "Darwin" ]; then + export CPU_COUNT=$(sysctl -n hw.ncpu) +else + export CPU_COUNT=$(nproc) +fi diff --git a/Android/android.py b/Android/android.py new file mode 100755 index 00000000000000..ae630aa8f4427c --- /dev/null +++ b/Android/android.py @@ -0,0 +1,650 @@ +#!/usr/bin/env python3 + +import asyncio +import argparse +from glob import glob +import os +import re +import shlex +import shutil +import signal +import subprocess +import sys +import sysconfig +from asyncio import wait_for +from contextlib import asynccontextmanager +from os.path import basename, relpath +from pathlib import Path +from subprocess import CalledProcessError +from tempfile import TemporaryDirectory + + +SCRIPT_NAME = Path(__file__).name +CHECKOUT = Path(__file__).resolve().parent.parent +ANDROID_DIR = CHECKOUT / "Android" +TESTBED_DIR = ANDROID_DIR / "testbed" +CROSS_BUILD_DIR = CHECKOUT / "cross-build" + +APP_ID = "org.python.testbed" +DECODE_ARGS = ("UTF-8", "backslashreplace") + + +try: + android_home = Path(os.environ['ANDROID_HOME']) +except KeyError: + sys.exit("The ANDROID_HOME environment variable is required.") + +adb = Path( + f"{android_home}/platform-tools/adb" + + (".exe" if os.name == "nt" else "") +) + +gradlew = Path( + f"{TESTBED_DIR}/gradlew" + + (".bat" if os.name == "nt" else "") +) + +logcat_started = False + + +def delete_glob(pattern): + # Path.glob doesn't accept non-relative patterns. + for path in glob(str(pattern)): + path = Path(path) + print(f"Deleting {path} ...") + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + + +def subdir(name, *, clean=None): + path = CROSS_BUILD_DIR / name + if clean: + delete_glob(path) + if not path.exists(): + if clean is None: + sys.exit( + f"{path} does not exist. Create it by running the appropriate " + f"`configure` subcommand of {SCRIPT_NAME}.") + else: + path.mkdir(parents=True) + return path + + +def run(command, *, host=None, env=None, log=True, **kwargs): + kwargs.setdefault("check", True) + if env is None: + env = os.environ.copy() + original_env = env.copy() + + if host: + env_script = ANDROID_DIR / "android-env.sh" + env_output = subprocess.run( + f"set -eu; " + f"HOST={host}; " + f"PREFIX={subdir(host)}/prefix; " + f". {env_script}; " + f"export", + check=True, shell=True, text=True, stdout=subprocess.PIPE + ).stdout + + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if env.get(key) != value: + print(line) + env[key] = value + + if env == original_env: + raise ValueError(f"Found no variables in {env_script.name} output:\n" + + env_output) + + if log: + print(">", " ".join(map(str, command))) + return subprocess.run(command, env=env, **kwargs) + + +def build_python_path(): + """The path to the build Python binary.""" + build_dir = subdir("build") + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " + f"{build_dir}") + + return binary + + +def configure_build_python(context): + os.chdir(subdir("build", clean=context.clean)) + + command = [relpath(CHECKOUT / "configure")] + if context.args: + command.extend(context.args) + run(command) + + +def make_build_python(context): + os.chdir(subdir("build")) + run(["make", "-j", str(os.cpu_count())]) + + +def unpack_deps(host): + deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" + for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", + "sqlite-3.45.3-3", "xz-5.4.6-1"]: + filename = f"{name_ver}-{host}.tar.gz" + download(f"{deps_url}/{name_ver}/{filename}") + run(["tar", "-xf", filename]) + os.remove(filename) + + +def download(url, target_dir="."): + out_path = f"{target_dir}/{basename(url)}" + run(["curl", "-Lf", "-o", out_path, url]) + return out_path + + +def configure_host_python(context): + host_dir = subdir(context.host, clean=context.clean) + + prefix_dir = host_dir / "prefix" + if not prefix_dir.exists(): + prefix_dir.mkdir() + os.chdir(prefix_dir) + unpack_deps(context.host) + + build_dir = host_dir / "build" + build_dir.mkdir(exist_ok=True) + os.chdir(build_dir) + + command = [ + # Basic cross-compiling configuration + relpath(CHECKOUT / "configure"), + f"--host={context.host}", + f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", + f"--with-build-python={build_python_path()}", + "--without-ensurepip", + + # Android always uses a shared libpython. + "--enable-shared", + "--without-static-libpython", + + # Dependent libraries. The others are found using pkg-config: see + # android-env.sh. + f"--with-openssl={prefix_dir}", + ] + + if context.args: + command.extend(context.args) + run(command, host=context.host) + + +def make_host_python(context): + # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so + # delete any previous Python installation to prevent it being used during + # the build. + host_dir = subdir(context.host) + prefix_dir = host_dir / "prefix" + delete_glob(f"{prefix_dir}/include/python*") + delete_glob(f"{prefix_dir}/lib/libpython*") + delete_glob(f"{prefix_dir}/lib/python*") + + os.chdir(host_dir / "build") + run(["make", "-j", str(os.cpu_count())], host=context.host) + run(["make", "install", f"prefix={prefix_dir}"], host=context.host) + + +def build_all(context): + steps = [configure_build_python, make_build_python, configure_host_python, + make_host_python] + for step in steps: + step(context) + + +def clean_all(context): + delete_glob(CROSS_BUILD_DIR) + + +def setup_sdk(): + sdkmanager = android_home / ( + "cmdline-tools/latest/bin/sdkmanager" + + (".bat" if os.name == "nt" else "") + ) + + # Gradle will fail if it needs to install an SDK package whose license + # hasn't been accepted, so pre-accept all licenses. + if not all((android_home / "licenses" / path).exists() for path in [ + "android-sdk-arm-dbt-license", "android-sdk-license" + ]): + run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + + # Gradle may install this automatically, but we can't rely on that because + # we need to run adb within the logcat task. + if not adb.exists(): + run([sdkmanager, "platform-tools"]) + + +# To avoid distributing compiled artifacts without corresponding source code, +# the Gradle wrapper is not included in the CPython repository. Instead, we +# extract it from the Gradle release. +def setup_testbed(): + if all((TESTBED_DIR / path).exists() for path in [ + "gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar", + ]): + return + + ver_long = "8.7.0" + ver_short = ver_long.removesuffix(".0") + + for filename in ["gradlew", "gradlew.bat"]: + out_path = download( + f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}", + TESTBED_DIR) + os.chmod(out_path, 0o755) + + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + bin_zip = download( + f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip", + temp_dir) + outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar" + run(["unzip", "-d", temp_dir, bin_zip, outer_jar]) + run(["unzip", "-o", "-d", f"{TESTBED_DIR}/gradle/wrapper", + f"{temp_dir}/{outer_jar}", "gradle-wrapper.jar"]) + + +# run_testbed will build the app automatically, but it's useful to have this as +# a separate command to allow running the app outside of this script. +def build_testbed(context): + setup_sdk() + setup_testbed() + run( + [gradlew, "--console", "plain", "packageDebug", "packageDebugAndroidTest"], + cwd=TESTBED_DIR, + ) + + +# Work around a bug involving sys.exit and TaskGroups +# (https://github.com/python/cpython/issues/101515). +def exit(*args): + raise MySystemExit(*args) + + +class MySystemExit(Exception): + pass + + +# The `test` subcommand runs all subprocesses through this context manager so +# that no matter what happens, they can always be cancelled from another task, +# and they will always be cleaned up on exit. +@asynccontextmanager +async def async_process(*args, **kwargs): + process = await asyncio.create_subprocess_exec(*args, **kwargs) + try: + yield process + finally: + if process.returncode is None: + # Allow a reasonably long time for Gradle to clean itself up, + # because we don't want stale emulators left behind. + timeout = 10 + process.terminate() + try: + await wait_for(process.wait(), timeout) + except TimeoutError: + print( + f"Command {args} did not terminate after {timeout} seconds " + f" - sending SIGKILL" + ) + process.kill() + + # Even after killing the process we must still wait for it, + # otherwise we'll get the warning "Exception ignored in __del__". + await wait_for(process.wait(), timeout=1) + + +async def async_check_output(*args, **kwargs): + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs + ) as process: + stdout, stderr = await process.communicate() + if process.returncode == 0: + return stdout.decode(*DECODE_ARGS) + else: + raise CalledProcessError( + process.returncode, args, + stdout.decode(*DECODE_ARGS), stderr.decode(*DECODE_ARGS) + ) + + +# Return a list of the serial numbers of connected devices. Emulators will have +# serials of the form "emulator-5678". +async def list_devices(): + serials = [] + header_found = False + + lines = (await async_check_output(adb, "devices")).splitlines() + for line in lines: + # Ignore blank lines, and all lines before the header. + line = line.strip() + if line == "List of devices attached": + header_found = True + elif header_found and line: + try: + serial, status = line.split() + except ValueError: + raise ValueError(f"failed to parse {line!r}") + if status == "device": + serials.append(serial) + + if not header_found: + raise ValueError(f"failed to parse {lines}") + return serials + + +async def find_device(context, initial_devices): + if context.managed: + print("Waiting for managed device - this may take several minutes") + while True: + new_devices = set(await list_devices()).difference(initial_devices) + if len(new_devices) == 0: + await asyncio.sleep(1) + elif len(new_devices) == 1: + serial = new_devices.pop() + print(f"Serial: {serial}") + return serial + else: + exit(f"Found more than one new device: {new_devices}") + else: + return context.connected + + +# An older version of this script in #121595 filtered the logs by UID instead. +# But logcat can't filter by UID until API level 31. If we ever switch back to +# filtering by UID, we'll also have to filter by time so we only show messages +# produced after the initial call to `stop_app`. +# +# We're more likely to miss the PID because it's shorter-lived, so there's a +# workaround in PythonSuite.kt to stop it being *too* short-lived. +async def find_pid(serial): + print("Waiting for app to start - this may take several minutes") + shown_error = False + while True: + try: + # `pidof` requires API level 24 or higher. The level 23 emulator + # includes it, but it doesn't work (it returns all processes). + pid = (await async_check_output( + adb, "-s", serial, "shell", "pidof", "-s", APP_ID + )).strip() + except CalledProcessError as e: + # If the app isn't running yet, pidof gives no output. So if there + # is output, there must have been some other error. However, this + # sometimes happens transiently, especially when running a managed + # emulator for the first time, so don't make it fatal. + if (e.stdout or e.stderr) and not shown_error: + print_called_process_error(e) + print("This may be transient, so continuing to wait") + shown_error = True + else: + # Some older devices (e.g. Nexus 4) return zero even when no process + # was found, so check whether we actually got any output. + if pid: + print(f"PID: {pid}") + return pid + + # Loop fairly rapidly to avoid missing a short-lived process. + await asyncio.sleep(0.2) + + +async def logcat_task(context, initial_devices): + # Gradle may need to do some large downloads of libraries and emulator + # images. This will happen during find_device in --managed mode, or find_pid + # in --connected mode. + startup_timeout = 600 + serial = await wait_for(find_device(context, initial_devices), startup_timeout) + pid = await wait_for(find_pid(serial), startup_timeout) + + # `--pid` requires API level 24 or higher. + args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] + hidden_output = [] + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + level, message = match.groups() + else: + # If the regex doesn't match, this is probably the second or + # subsequent line of a multi-line message. Python won't produce + # such messages, but other components might. + level, message = None, line + + # Exclude high-volume messages which are rarely useful. + if context.verbose < 2 and "from python test_syslog" in message: + continue + + # Put high-level messages on stderr so they're highlighted in the + # buildbot logs. This will include Python's own stderr. + stream = ( + sys.stderr + if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT) + else sys.stdout + ) + + # To simplify automated processing of the output, e.g. a buildbot + # posting a failure notice on a GitHub PR, we strip the level and + # tag indicators from Python's stdout and stderr. + for prefix in ["python.stdout: ", "python.stderr: "]: + if message.startswith(prefix): + global logcat_started + logcat_started = True + stream.write(message.removeprefix(prefix)) + break + else: + if context.verbose: + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. + stream.write(line) + else: + hidden_output.append(line) + + # If the device disconnects while logcat is running, which always + # happens in --managed mode, some versions of adb return non-zero. + # Distinguish this from a logcat startup error by checking whether we've + # received a message from Python yet. + status = await wait_for(process.wait(), timeout=1) + if status != 0 and not logcat_started: + raise CalledProcessError(status, args, "".join(hidden_output)) + + +def stop_app(serial): + run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) + + +async def gradle_task(context): + env = os.environ.copy() + if context.managed: + task_prefix = context.managed + else: + task_prefix = "connected" + env["ANDROID_SERIAL"] = context.connected + + args = [ + gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", + "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" + + shlex.join(context.args), + ] + hidden_output = [] + try: + async with async_process( + *args, cwd=TESTBED_DIR, env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + # Gradle may take several minutes to install SDK packages, so + # it's worth showing those messages even in non-verbose mode. + if context.verbose or line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + hidden_output.append(line) + + status = await wait_for(process.wait(), timeout=1) + if status == 0: + exit(0) + else: + raise CalledProcessError(status, args) + finally: + # If logcat never started, then something has gone badly wrong, so the + # user probably wants to see the Gradle output even in non-verbose mode. + if hidden_output and not logcat_started: + sys.stdout.write("".join(hidden_output)) + + # Gradle does not stop the tests when interrupted. + if context.connected: + stop_app(context.connected) + + +async def run_testbed(context): + setup_sdk() + setup_testbed() + + if context.managed: + # In this mode, Gradle will create a device with an unpredictable name. + # So we save a list of the running devices before starting Gradle, and + # find_device then waits for a new device to appear. + initial_devices = await list_devices() + else: + # In case the previous shutdown was unclean, make sure the app isn't + # running, otherwise we might show logs from a previous run. This is + # unnecessary in --managed mode, because Gradle creates a new emulator + # every time. + stop_app(context.connected) + initial_devices = None + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(logcat_task(context, initial_devices)) + tg.create_task(gradle_task(context)) + except* MySystemExit as e: + raise SystemExit(*e.exceptions[0].args) from None + except* CalledProcessError as e: + # Extract it from the ExceptionGroup so it can be handled by `main`. + raise e.exceptions[0] + + +# Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated +# by the buildbot worker, we'll make an attempt to clean up our subprocesses. +def install_signal_handler(): + def signal_handler(*args): + os.kill(os.getpid(), signal.SIGINT) + + signal.signal(signal.SIGTERM, signal_handler) + + +def parse_args(): + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser("configure-build", + help="Run `configure` for the " + "build Python") + make_build = subcommands.add_parser("make-build", + help="Run `make` for the build Python") + configure_host = subcommands.add_parser("configure-host", + help="Run `configure` for Android") + make_host = subcommands.add_parser("make-host", + help="Run `make` for Android") + subcommands.add_parser( + "clean", help="Delete the cross-build directory") + + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "--clean", action="store_true", default=False, dest="clean", + help="Delete any relevant directories before building") + for subcommand in build, configure_host, make_host: + subcommand.add_argument( + "host", metavar="HOST", + choices=["aarch64-linux-android", "x86_64-linux-android"], + help="Host triplet: choices=[%(choices)s]") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + + subcommands.add_parser( + "build-testbed", help="Build the testbed app") + test = subcommands.add_parser( + "test", help="Run the test suite") + test.add_argument( + "-v", "--verbose", action="count", default=0, + help="Show Gradle output, and non-Python logcat messages. " + "Use twice to include high-volume messages which are rarely useful.") + device_group = test.add_mutually_exclusive_group(required=True) + device_group.add_argument( + "--connected", metavar="SERIAL", help="Run on a connected device. " + "Connect it yourself, then get its serial from `adb devices`.") + device_group.add_argument( + "--managed", metavar="NAME", help="Run on a Gradle-managed device. " + "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + test.add_argument( + "args", nargs="*", help=f"Arguments for `python -m test`. " + f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + + return parser.parse_args() + + +def main(): + install_signal_handler() + + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + + context = parse_args() + dispatch = {"configure-build": configure_build_python, + "make-build": make_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "build": build_all, + "clean": clean_all, + "build-testbed": build_testbed, + "test": run_testbed} + + try: + result = dispatch[context.subcommand](context) + if asyncio.iscoroutine(result): + asyncio.run(result) + except CalledProcessError as e: + print_called_process_error(e) + sys.exit(1) + + +def print_called_process_error(e): + for stream_name in ["stdout", "stderr"]: + content = getattr(e, stream_name) + stream = getattr(sys, stream_name) + if content: + stream.write(content) + if not content.endswith("\n"): + stream.write("\n") + + # Format the command so it can be copied into a shell. shlex uses single + # quotes, so we surround the whole command with double quotes. + args_joined = ( + e.cmd if isinstance(e.cmd, str) + else " ".join(shlex.quote(str(arg)) for arg in e.cmd) + ) + print( + f'Command "{args_joined}" returned exit status {e.returncode}' + ) + + +if __name__ == "__main__": + main() diff --git a/Android/testbed/.gitignore b/Android/testbed/.gitignore new file mode 100644 index 00000000000000..b9a7d611c943cf --- /dev/null +++ b/Android/testbed/.gitignore @@ -0,0 +1,21 @@ +# The Gradle wrapper should be downloaded by running `../android.py setup-testbed`. +/gradlew +/gradlew.bat +/gradle/wrapper/gradle-wrapper.jar + +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/deploymentTargetDropdown.xml +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/Android/testbed/app/.gitignore b/Android/testbed/app/.gitignore new file mode 100644 index 00000000000000..42afabfd2abebf --- /dev/null +++ b/Android/testbed/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts new file mode 100644 index 00000000000000..211b5bbfadf64d --- /dev/null +++ b/Android/testbed/app/build.gradle.kts @@ -0,0 +1,200 @@ +import com.android.build.api.variant.* +import kotlin.math.max + +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") +} + +val PYTHON_DIR = file("../../..").canonicalPath +val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build" + +val ABIS = mapOf( + "arm64-v8a" to "aarch64-linux-android", + "x86_64" to "x86_64-linux-android", +).filter { file("$PYTHON_CROSS_DIR/${it.value}").exists() } +if (ABIS.isEmpty()) { + throw GradleException( + "No Android ABIs found in $PYTHON_CROSS_DIR: see Android/README.md " + + "for building instructions." + ) +} + +val PYTHON_VERSION = file("$PYTHON_DIR/Include/patchlevel.h").useLines { + for (line in it) { + val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line) + if (match != null) { + return@useLines match.groupValues[1] + } + } + throw GradleException("Failed to find Python version") +} + + +android { + val androidEnvFile = file("../../android-env.sh").absoluteFile + + namespace = "org.python.testbed" + compileSdk = 34 + + defaultConfig { + applicationId = "org.python.testbed" + + minSdk = androidEnvFile.useLines { + for (line in it) { + """api_level:=(\d+)""".toRegex().find(line)?.let { + return@useLines it.groupValues[1].toInt() + } + } + throw GradleException("Failed to find API level in $androidEnvFile") + } + targetSdk = 34 + + versionCode = 1 + versionName = "1.0" + + ndk.abiFilters.addAll(ABIS.keys) + externalNativeBuild.cmake.arguments( + "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR", + "-DPYTHON_VERSION=$PYTHON_VERSION", + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", + ) + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + + ndkVersion = androidEnvFile.useLines { + for (line in it) { + """ndk_version=(\S+)""".toRegex().find(line)?.let { + return@useLines it.groupValues[1] + } + } + throw GradleException("Failed to find NDK version in $androidEnvFile") + } + externalNativeBuild.cmake { + path("src/main/c/CMakeLists.txt") + } + + // Set this property to something non-empty, otherwise it'll use the default + // list, which ignores asset directories beginning with an underscore. + aaptOptions.ignoreAssetsPattern = ".git" + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } + + testOptions { + managedDevices { + localDevices { + create("minVersion") { + device = "Small Phone" + + // Managed devices have a minimum API level of 27. + apiLevel = max(27, defaultConfig.minSdk!!) + + // ATD devices are smaller and faster, but have a minimum + // API level of 30. + systemImageSource = if (apiLevel >= 30) "aosp-atd" else "aosp" + } + + create("maxVersion") { + device = "Small Phone" + apiLevel = defaultConfig.targetSdk!! + systemImageSource = "aosp-atd" + } + } + + // If the previous test run succeeded and nothing has changed, + // Gradle thinks there's no need to run it again. Override that. + afterEvaluate { + (localDevices.names + listOf("connected")).forEach { + tasks.named("${it}DebugAndroidTest") { + outputs.upToDateWhen { false } + } + } + } + } + } +} + +dependencies { + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.11.0") + implementation("androidx.constraintlayout:constraintlayout:2.1.4") + androidTestImplementation("androidx.test.ext:junit:1.1.5") + androidTestImplementation("androidx.test:rules:1.5.0") +} + + +// Create some custom tasks to copy Python and its standard library from +// elsewhere in the repository. +androidComponents.onVariants { variant -> + val pyPlusVer = "python$PYTHON_VERSION" + generateTask(variant, variant.sources.assets!!) { + into("python") { + into("include/$pyPlusVer") { + for (triplet in ABIS.values) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/include/$pyPlusVer") + } + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + into("lib/$pyPlusVer") { + // To aid debugging, the source directory takes priority. + from("$PYTHON_DIR/Lib") + + // The cross-build directory provides ABI-specific files such as + // sysconfigdata. + for (triplet in ABIS.values) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/lib/$pyPlusVer") + } + + into("site-packages") { + from("$projectDir/src/main/python") + } + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + exclude("**/__pycache__") + } + } + } + + generateTask(variant, variant.sources.jniLibs!!) { + for ((abi, triplet) in ABIS.entries) { + into(abi) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/lib") + include("libpython*.*.so") + include("lib*_python.so") + } + } + } +} + + +fun generateTask( + variant: ApplicationVariant, directories: SourceDirectories, + configure: GenerateTask.() -> Unit +) { + val taskName = "generate" + + listOf(variant.name, "Python", directories.name) + .map { it.replaceFirstChar(Char::uppercase) } + .joinToString("") + + directories.addGeneratedSourceDirectory( + tasks.register(taskName) { + into(outputDir) + configure() + }, + GenerateTask::outputDir) +} + + +// addGeneratedSourceDirectory requires the task to have a DirectoryProperty. +abstract class GenerateTask: Sync() { + @get:OutputDirectory + abstract val outputDir: DirectoryProperty +} diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt new file mode 100644 index 00000000000000..0e888ab71d87da --- /dev/null +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -0,0 +1,35 @@ +package org.python.testbed + +import androidx.test.annotation.UiThreadTest +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + + +@RunWith(AndroidJUnit4::class) +class PythonSuite { + @Test + @UiThreadTest + fun testPython() { + val start = System.currentTimeMillis() + try { + val context = + InstrumentationRegistry.getInstrumentation().targetContext + val args = + InstrumentationRegistry.getArguments().getString("pythonArgs", "") + val status = PythonTestRunner(context).run(args) + assertEquals(0, status) + } finally { + // Make sure the process lives long enough for the test script to + // detect it (see `find_pid` in android.py). + val delay = 2000 - (System.currentTimeMillis() - start) + if (delay > 0) { + Thread.sleep(delay) + } + } + } +} diff --git a/Android/testbed/app/src/main/AndroidManifest.xml b/Android/testbed/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000000000..2be8a82d426099 --- /dev/null +++ b/Android/testbed/app/src/main/AndroidManifest.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/c/CMakeLists.txt b/Android/testbed/app/src/main/c/CMakeLists.txt new file mode 100644 index 00000000000000..1d5df9a73465b6 --- /dev/null +++ b/Android/testbed/app/src/main/c/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.4.1) +project(testbed) + +set(PREFIX_DIR ${PYTHON_CROSS_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/prefix) +include_directories(${PREFIX_DIR}/include/python${PYTHON_VERSION}) +link_directories(${PREFIX_DIR}/lib) +link_libraries(log python${PYTHON_VERSION}) + +add_library(main_activity SHARED main_activity.c) diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c new file mode 100644 index 00000000000000..ec7f93a3e5ee13 --- /dev/null +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -0,0 +1,152 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +static void throw_runtime_exception(JNIEnv *env, const char *message) { + (*env)->ThrowNew( + env, + (*env)->FindClass(env, "java/lang/RuntimeException"), + message); +} + + +// --- Stdio redirection ------------------------------------------------------ + +// Most apps won't need this, because the Python-level sys.stdout and sys.stderr +// are redirected to the Android logcat by Python itself. However, in the +// testbed it's useful to redirect the native streams as well, to debug problems +// in the Python startup or redirection process. +// +// Based on +// https://github.com/beeware/briefcase-android-gradle-template/blob/v0.3.11/%7B%7B%20cookiecutter.safe_formal_name%20%7D%7D/app/src/main/cpp/native-lib.cpp + +typedef struct { + FILE *file; + int fd; + android_LogPriority priority; + char *tag; + int pipe[2]; +} StreamInfo; + +// The FILE member can't be initialized here because stdout and stderr are not +// compile-time constants. Instead, it's initialized immediately before the +// redirection. +static StreamInfo STREAMS[] = { + {NULL, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}}, + {NULL, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}}, + {NULL, -1, ANDROID_LOG_UNKNOWN, NULL, {-1, -1}}, +}; + +// The maximum length of a log message in bytes, including the level marker and +// tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +// platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +// longer than this will be be truncated by logcat. This limit has already been +// reduced at least once in the history of Android (from 4076 to 4068 between API +// level 23 and 26), so leave some headroom. +static const int MAX_BYTES_PER_WRITE = 4000; + +static void *redirection_thread(void *arg) { + StreamInfo *si = (StreamInfo*)arg; + ssize_t read_size; + char buf[MAX_BYTES_PER_WRITE]; + while ((read_size = read(si->pipe[0], buf, sizeof buf - 1)) > 0) { + buf[read_size] = '\0'; /* add null-terminator */ + __android_log_write(si->priority, si->tag, buf); + } + return 0; +} + +static char *redirect_stream(StreamInfo *si) { + /* make the FILE unbuffered, to ensure messages are never lost */ + if (setvbuf(si->file, 0, _IONBF, 0)) { + return "setvbuf"; + } + + /* create the pipe and redirect the file descriptor */ + if (pipe(si->pipe)) { + return "pipe"; + } + if (dup2(si->pipe[1], si->fd) == -1) { + return "dup2"; + } + + /* start the logging thread */ + pthread_t thr; + if ((errno = pthread_create(&thr, 0, redirection_thread, si))) { + return "pthread_create"; + } + if ((errno = pthread_detach(thr))) { + return "pthread_detach"; + } + return 0; +} + +JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToLogcat( + JNIEnv *env, jobject obj +) { + STREAMS[0].file = stdout; + STREAMS[1].file = stderr; + for (StreamInfo *si = STREAMS; si->file; si++) { + char *error_prefix; + if ((error_prefix = redirect_stream(si))) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); + return; + } + } +} + + +// --- Python initialization --------------------------------------------------- + +static PyStatus set_config_string( + JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value +) { + const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); + PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); + (*env)->ReleaseStringUTFChars(env, value, value_utf8); + return status; +} + +static void throw_status(JNIEnv *env, PyStatus status) { + throw_runtime_exception(env, status.err_msg ? status.err_msg : ""); +} + +JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython( + JNIEnv *env, jobject obj, jstring home, jstring runModule +) { + PyConfig config; + PyStatus status; + PyConfig_InitIsolatedConfig(&config); + + status = set_config_string(env, &config, &config.home, home); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return 1; + } + + status = set_config_string(env, &config, &config.run_module, runModule); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return 1; + } + + // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. + config.install_signal_handlers = 1; + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return 1; + } + + return Py_RunMain(); +} diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt new file mode 100644 index 00000000000000..c4bf6cbe83d8cd --- /dev/null +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -0,0 +1,79 @@ +package org.python.testbed + +import android.content.Context +import android.os.* +import android.system.Os +import android.widget.TextView +import androidx.appcompat.app.* +import java.io.* + + +// Launching the tests from an activity is OK for a quick check, but for +// anything more complicated it'll be more convenient to use `android.py test` +// to launch the tests via PythonSuite. +class MainActivity : AppCompatActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + val status = PythonTestRunner(this).run("-W -uall") + findViewById(R.id.tvHello).text = "Exit status $status" + } +} + + +class PythonTestRunner(val context: Context) { + /** @param args Extra arguments for `python -m test`. + * @return The Python exit status: zero if the tests passed, nonzero if + * they failed. */ + fun run(args: String = "") : Int { + Os.setenv("PYTHON_ARGS", args, true) + + // Python needs this variable to help it find the temporary directory, + // but Android only sets it on API level 33 and later. + Os.setenv("TMPDIR", context.cacheDir.toString(), false) + + val pythonHome = extractAssets() + System.loadLibrary("main_activity") + redirectStdioToLogcat() + + // The main module is in src/main/python/main.py. + return runPython(pythonHome.toString(), "main") + } + + private fun extractAssets() : File { + val pythonHome = File(context.filesDir, "python") + if (pythonHome.exists() && !pythonHome.deleteRecursively()) { + throw RuntimeException("Failed to delete $pythonHome") + } + extractAssetDir("python", context.filesDir) + return pythonHome + } + + private fun extractAssetDir(path: String, targetDir: File) { + val names = context.assets.list(path) + ?: throw RuntimeException("Failed to list $path") + val targetSubdir = File(targetDir, path) + if (!targetSubdir.mkdirs()) { + throw RuntimeException("Failed to create $targetSubdir") + } + + for (name in names) { + val subPath = "$path/$name" + val input: InputStream + try { + input = context.assets.open(subPath) + } catch (e: FileNotFoundException) { + extractAssetDir(subPath, targetDir) + continue + } + input.use { + File(targetSubdir, name).outputStream().use { output -> + input.copyTo(output) + } + } + } + } + + private external fun redirectStdioToLogcat() + private external fun runPython(home: String, runModule: String) : Int +} diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py new file mode 100644 index 00000000000000..d6941b14412fcc --- /dev/null +++ b/Android/testbed/app/src/main/python/main.py @@ -0,0 +1,32 @@ +import os +import runpy +import shlex +import signal +import sys + +# Some tests use SIGUSR1, but that's blocked by default in an Android app in +# order to make it available to `sigwait` in the Signal Catcher thread. +# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). +# That thread's functionality is only useful for debugging the JVM, so disabling +# it should not weaken the tests. +# +# There's no safe way of stopping the thread completely (#123982), but simply +# unblocking SIGUSR1 is enough to fix most tests. +# +# However, in tests that generate multiple different signals in quick +# succession, it's possible for SIGUSR1 to arrive while the main thread is busy +# running the C-level handler for a different signal. In that case, the SIGUSR1 +# may be sent to the Signal Catcher thread instead, which will generate a log +# message containing the text "reacting to signal". +# +# Such tests may need to be changed in one of the following ways: +# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in +# test_signal.py). +# * Send the signal to a specific thread rather than the whole process (e.g. +# test_signals in test_threadsignals.py. +signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) + +sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) + +# The test module will call sys.exit to indicate whether the tests passed. +runpy.run_module("test") diff --git a/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png new file mode 100644 index 00000000000000..741d6580d60e05 Binary files /dev/null and b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png differ diff --git a/Android/testbed/app/src/main/res/layout/activity_main.xml b/Android/testbed/app/src/main/res/layout/activity_main.xml new file mode 100644 index 00000000000000..21398609ec9c78 --- /dev/null +++ b/Android/testbed/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,19 @@ + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/res/values/strings.xml b/Android/testbed/app/src/main/res/values/strings.xml new file mode 100644 index 00000000000000..352d2f9e885a2a --- /dev/null +++ b/Android/testbed/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + Python testbed + \ No newline at end of file diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts new file mode 100644 index 00000000000000..4d1d6f87594da3 --- /dev/null +++ b/Android/testbed/build.gradle.kts @@ -0,0 +1,5 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id("com.android.application") version "8.6.1" apply false + id("org.jetbrains.kotlin.android") version "1.9.22" apply false +} diff --git a/Android/testbed/gradle.properties b/Android/testbed/gradle.properties new file mode 100644 index 00000000000000..e9f345c8c26250 --- /dev/null +++ b/Android/testbed/gradle.properties @@ -0,0 +1,28 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true + +# By default, the app will be uninstalled after the tests finish (apparently +# after 10 seconds in case of an unclean shutdown). We disable this, because +# when using android.py it can conflict with the installation of the next run. +android.injected.androidTest.leaveApksInstalledAfterRun=true diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000000..36529c896426b0 --- /dev/null +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Feb 19 20:29:06 GMT 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/Android/testbed/settings.gradle.kts b/Android/testbed/settings.gradle.kts new file mode 100644 index 00000000000000..5e08773e02450f --- /dev/null +++ b/Android/testbed/settings.gradle.kts @@ -0,0 +1,18 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "Python testbed" +include(":app") + \ No newline at end of file diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml new file mode 100644 index 00000000000000..111ce03b91df38 --- /dev/null +++ b/Doc/.ruff.toml @@ -0,0 +1,42 @@ +target-version = "py312" # Align with the version in oldest_supported_sphinx +fix = true +output-format = "full" +line-length = 79 +extend-exclude = [ + "includes/*", + # Temporary exclusions: + "tools/extensions/pyspecific.py", +] + +[lint] +preview = true +select = [ + "C4", # flake8-comprehensions + "B", # flake8-bugbear + "E", # pycodestyle + "F", # pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "LOG", # flake8-logging + "N", # pep8-naming + "PERF", # perflint + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "TCH", # flake8-type-checking + "UP", # pyupgrade + "W", # pycodestyle +] +ignore = [ + "E501", # Ignore line length errors (we use auto-formatting) +] + +[format] +preview = true +quote-style = "preserve" +docstring-code-format = true +exclude = [ + "tools/extensions/lexers/*", +] diff --git a/Doc/Makefile b/Doc/Makefile index 38fd60f2ae01d1..a090ee5ba92705 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -6,6 +6,7 @@ # You can set these variables from the command line. PYTHON = python3 VENVDIR = ./venv +UV = uv SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build BLURB = PATH=$(VENVDIR)/bin:$$PATH blurb JOBS = auto @@ -32,6 +33,7 @@ help: @echo " clean to remove build files" @echo " venv to create a venv with necessary tools" @echo " html to make standalone HTML files" + @echo " gettext to generate POT files" @echo " htmlview to open the index page built by the html target in your browser" @echo " htmllive to rebuild and reload HTML files in your browser" @echo " htmlhelp to make HTML files and a HTML help project" @@ -140,14 +142,19 @@ pydoc-topics: build @echo "Building finished; now run this:" \ "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" +.PHONY: gettext +gettext: BUILDER = gettext +gettext: SPHINXOPTS += -d build/doctrees-gettext +gettext: build + .PHONY: htmlview htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" .PHONY: htmllive -htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild +htmllive: SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-autobuild htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 -htmllive: html +htmllive: _ensure-sphinx-autobuild html .PHONY: clean clean: clean-venv @@ -163,70 +170,126 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ - $(PYTHON) -m venv $(VENVDIR); \ - $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ - $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ + echo "Creating venv in $(VENVDIR)"; \ + if $(UV) --version >/dev/null 2>&1; then \ + $(UV) venv $(VENVDIR); \ + VIRTUAL_ENV=$(VENVDIR) $(UV) pip install -r $(REQUIREMENTS); \ + else \ + $(PYTHON) -m venv $(VENVDIR); \ + $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ + $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ + fi; \ echo "The venv has been created in the $(VENVDIR) directory"; \ fi +.PHONY: dist-no-html +dist-no-html: dist-text dist-pdf dist-epub dist-texinfo + .PHONY: dist dist: rm -rf dist mkdir -p dist - + $(MAKE) dist-html + $(MAKE) dist-text + $(MAKE) dist-pdf + $(MAKE) dist-epub + $(MAKE) dist-texinfo + +.PHONY: dist-html +dist-html: # archive the HTML - make html + @echo "Building HTML..." + mkdir -p dist + rm -rf build/html + find dist -name 'python-$(DISTVERSION)-docs-html*' -exec rm -rf {} \; + $(MAKE) html cp -pPR build/html dist/python-$(DISTVERSION)-docs-html tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html) rm -r dist/python-$(DISTVERSION)-docs-html rm dist/python-$(DISTVERSION)-docs-html.tar + @echo "Build finished and archived!" +.PHONY: dist-text +dist-text: # archive the text build - make text + @echo "Building text..." + mkdir -p dist + rm -rf build/text + find dist -name 'python-$(DISTVERSION)-docs-text*' -exec rm -rf {} \; + $(MAKE) text cp -pPR build/text dist/python-$(DISTVERSION)-docs-text tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text bzip2 -9 -k dist/python-$(DISTVERSION)-docs-text.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-text.zip python-$(DISTVERSION)-docs-text) rm -r dist/python-$(DISTVERSION)-docs-text rm dist/python-$(DISTVERSION)-docs-text.tar + @echo "Build finished and archived!" +.PHONY: dist-pdf +dist-pdf: # archive the A4 latex + @echo "Building LaTeX (A4 paper)..." + mkdir -p dist rm -rf build/latex - make latex PAPER=a4 - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) + find dist -name 'python-$(DISTVERSION)-docs-pdf*' -exec rm -rf {} \; + $(MAKE) latex PAPER=a4 + # remove zip & bz2 dependency on all-pdf, + # as otherwise the full latexmk process is run twice. + # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) + -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 + @echo "Build finished and archived!" - # archive the letter latex - rm -rf build/latex - make latex PAPER=letter - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) - cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip - cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 - +.PHONY: dist-epub +dist-epub: # copy the epub build + @echo "Building EPUB..." + mkdir -p dist rm -rf build/epub - make epub + rm -f dist/python-$(DISTVERSION)-docs.epub + $(MAKE) epub cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + @echo "Build finished and archived!" +.PHONY: dist-texinfo +dist-texinfo: # archive the texinfo build + @echo "Building Texinfo..." + mkdir -p dist rm -rf build/texinfo - make texinfo - make info --directory=build/texinfo + find dist -name 'python-$(DISTVERSION)-docs-texinfo*' -exec rm -rf {} \; + $(MAKE) texinfo + $(MAKE) info --directory=build/texinfo cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo) rm -r dist/python-$(DISTVERSION)-docs-texinfo rm dist/python-$(DISTVERSION)-docs-texinfo.tar + @echo "Build finished and archived!" + +.PHONY: _ensure-package +_ensure-package: venv + if $(UV) --version >/dev/null 2>&1; then \ + VIRTUAL_ENV=$(VENVDIR) $(UV) pip install $(PACKAGE); \ + else \ + $(VENVDIR)/bin/python3 -m pip install $(PACKAGE); \ + fi + +.PHONY: _ensure-pre-commit +_ensure-pre-commit: + $(MAKE) _ensure-package PACKAGE=pre-commit + +.PHONY: _ensure-sphinx-autobuild +_ensure-sphinx-autobuild: + $(MAKE) _ensure-package PACKAGE=sphinx-autobuild .PHONY: check -check: venv - $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit +check: _ensure-pre-commit $(VENVDIR)/bin/python3 -m pre_commit run --all-files .PHONY: serve @@ -242,13 +305,15 @@ serve: # for development releases: always build .PHONY: autobuild-dev +autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) -# for quick rebuilds (HTML only) +# for HTML-only rebuilds .PHONY: autobuild-dev-html +autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage @@ -258,7 +323,7 @@ autobuild-stable: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev + @$(MAKE) autobuild-dev .PHONY: autobuild-stable-html autobuild-stable-html: @@ -266,4 +331,4 @@ autobuild-stable-html: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev-html + @$(MAKE) autobuild-dev-html diff --git a/Doc/README.rst b/Doc/README.rst index a3bb5fa5445c23..2d1148753e0c6b 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -28,7 +28,7 @@ install the tools into there. Using make ---------- -To get started on UNIX, you can create a virtual environment and build +To get started on Unix, you can create a virtual environment and build documentation with the commands:: make venv @@ -40,13 +40,13 @@ If you'd like to create the virtual environment in a different location, you can specify it using the ``VENVDIR`` variable. You can also skip creating the virtual environment altogether, in which case -the Makefile will look for instances of ``sphinx-build`` and ``blurb`` +the ``Makefile`` will look for instances of ``sphinx-build`` and ``blurb`` installed on your process ``PATH`` (configurable with the ``SPHINXBUILD`` and ``BLURB`` variables). -On Windows, we try to emulate the Makefile as closely as possible with a +On Windows, we try to emulate the ``Makefile`` as closely as possible with a ``make.bat`` file. If you need to specify the Python interpreter to use, -set the PYTHON environment variable. +set the ``PYTHON`` environment variable. Available make targets are: @@ -62,15 +62,19 @@ Available make targets are: * "htmlview", which re-uses the "html" builder, but then opens the main page in your default web browser. +* "htmllive", which re-uses the "html" builder, rebuilds the docs, + starts a local server, and automatically reloads the page in your browser + when you make changes to reST files (Unix only). + * "htmlhelp", which builds HTML files and a HTML Help project file usable to convert them into a single Compiled HTML (.chm) file -- these are popular under Microsoft Windows, but very handy on every platform. To create the CHM file, you need to run the Microsoft HTML Help Workshop - over the generated project (.hhp) file. The make.bat script does this for + over the generated project (.hhp) file. The ``make.bat`` script does this for you on Windows. -* "latex", which builds LaTeX source files as input to "pdflatex" to produce +* "latex", which builds LaTeX source files as input to ``pdflatex`` to produce PDF documents. * "text", which builds a plain text file for each source file. @@ -95,8 +99,6 @@ Available make targets are: * "check", which checks for frequent markup errors. -* "serve", which serves the build/html directory on port 8000. - * "dist", (Unix only) which creates distributable archives of HTML, text, PDF, and EPUB builds. @@ -131,8 +133,5 @@ Bugs in the content should be reported to the Bugs in the toolset should be reported to the tools themselves. -You can also send a mail to the Python Documentation Team at docs@python.org, -and we will process your request as soon as possible. - -If you want to help the Documentation Team, you are always welcome. Just send -a mail to docs@python.org. +To help with the documentation, or report any problems, please leave a message +on `discuss.python.org `_. diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 908987cf41ff6e..5d0f68ca69675e 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -16,16 +16,15 @@ Documentation bugs ================== If you find a bug in this documentation or would like to propose an improvement, -please submit a bug report on the :ref:`tracker `. If you +please submit a bug report on the :ref:`issue tracker `. If you have a suggestion on how to fix it, include that as well. You can also open a discussion item on our `Documentation Discourse forum `_. -If you're short on time, you can also email documentation bug reports to -docs@python.org (behavioral bugs can be sent to python-list@python.org). -'docs@' is a mailing list run by volunteers; your request will be noticed, -though it may take a while to be processed. +If you find a bug in the theme (HTML / CSS / JavaScript) of the +documentation, please submit a bug report on the `python-doc-theme issue +tracker `_. .. seealso:: diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index b3609c233156b6..0d53b18ea87d5e 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -54,12 +54,7 @@ Allocating Objects on the Heap .. c:function:: void PyObject_Del(void *op) - Releases memory allocated to an object using :c:macro:`PyObject_New` or - :c:macro:`PyObject_NewVar`. This is normally called from the - :c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of - the object should not be accessed after this call as the memory is no - longer a valid Python object. - + Same as :c:func:`PyObject_Free`. .. c:var:: PyObject _Py_NoneStruct diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 834aae9372fe3b..3201bdc82691f4 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -280,10 +280,10 @@ Numbers length 1, to a C :c:expr:`int`. ``f`` (:class:`float`) [float] - Convert a Python floating point number to a C :c:expr:`float`. + Convert a Python floating-point number to a C :c:expr:`float`. ``d`` (:class:`float`) [double] - Convert a Python floating point number to a C :c:expr:`double`. + Convert a Python floating-point number to a C :c:expr:`double`. ``D`` (:class:`complex`) [Py_complex] Convert a Python complex number to a C :c:type:`Py_complex` structure. @@ -642,10 +642,10 @@ Building values object of length 1. ``d`` (:class:`float`) [double] - Convert a C :c:expr:`double` to a Python floating point number. + Convert a C :c:expr:`double` to a Python floating-point number. ``f`` (:class:`float`) [float] - Convert a C :c:expr:`float` to a Python floating point number. + Convert a C :c:expr:`float` to a Python floating-point number. ``D`` (:class:`complex`) [Py_complex \*] Convert a C :c:type:`Py_complex` structure to a Python complex number. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index e572815ffd6259..dc43a3d5fcb094 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -29,7 +29,7 @@ without intermediate copying. Python provides such a facility at the C level in the form of the :ref:`buffer protocol `. This protocol has two sides: -.. index:: single: PyBufferProcs +.. index:: single: PyBufferProcs (C type) - on the producer side, a type can export a "buffer interface" which allows objects of that type to expose information about their underlying buffer. @@ -147,9 +147,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - .. c:member:: const char *format + .. c:member:: char *format - A *NUL* terminated string in :mod:`struct` module style syntax describing + A *NULL* terminated string in :mod:`struct` module style syntax describing the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) is assumed. @@ -244,7 +244,6 @@ The following fields are not influenced by *flags* and must always be filled in with the correct values: :c:member:`~Py_buffer.obj`, :c:member:`~Py_buffer.buf`, :c:member:`~Py_buffer.len`, :c:member:`~Py_buffer.itemsize`, :c:member:`~Py_buffer.ndim`. - readonly, format ~~~~~~~~~~~~~~~~ @@ -253,7 +252,8 @@ readonly, format Controls the :c:member:`~Py_buffer.readonly` field. If set, the exporter MUST provide a writable buffer or else report failure. Otherwise, the exporter MAY provide either a read-only or writable buffer, but the choice - MUST be consistent for all consumers. + MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` + can be used to request a simple writable buffer. .. c:macro:: PyBUF_FORMAT @@ -265,8 +265,9 @@ readonly, format Since :c:macro:`PyBUF_SIMPLE` is defined as 0, :c:macro:`PyBUF_WRITABLE` can be used as a stand-alone flag to request a simple writable buffer. -:c:macro:`PyBUF_FORMAT` can be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`. -The latter already implies format ``B`` (unsigned bytes). +:c:macro:`PyBUF_FORMAT` must be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`, because +the latter already implies format ``B`` (unsigned bytes). :c:macro:`!PyBUF_FORMAT` cannot be +used on its own. shape, strides, suboffsets diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 456f7d89bca03c..9045689a6be567 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -42,17 +42,22 @@ Direct API functions Return a new bytearray object from any object, *o*, that implements the :ref:`buffer protocol `. + On failure, return ``NULL`` with an exception set. + .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) - Create a new bytearray object from *string* and its length, *len*. On - failure, ``NULL`` is returned. + Create a new bytearray object from *string* and its length, *len*. + + On failure, return ``NULL`` with an exception set. .. c:function:: PyObject* PyByteArray_Concat(PyObject *a, PyObject *b) Concat bytearrays *a* and *b* and return a new bytearray with the result. + On failure, return ``NULL`` with an exception set. + .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 4790d3b2da4375..d47beee68eaa33 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -189,12 +189,30 @@ called with a non-bytes parameter. to *newpart* (i.e. decrements its reference count). +.. c:function:: PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) + + Similar to ``sep.join(iterable)`` in Python. + + *sep* must be Python :class:`bytes` object. + (Note that :c:func:`PyUnicode_Join` accepts ``NULL`` separator and treats + it as a space, whereas :c:func:`PyBytes_Join` doesn't accept ``NULL`` + separator.) + + *iterable* must be an iterable object yielding objects that implement the + :ref:`buffer protocol `. + + On success, return a new :class:`bytes` object. + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.14 + + .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) - A way to resize a bytes object even though it is "immutable". Only use this - to build up a brand new bytes object; don't use this if the bytes may already - be known in other parts of the code. It is an error to call this function if - the refcount on the input bytes object is not one. Pass the address of an + Resize a bytes object. *newsize* will be the new length of the bytes object. + You can think of it as creating a new bytes object and destroying the old + one, only more efficiently. + Pass the address of an existing bytes object as an lvalue (it may be written into), and the new size desired. On success, *\*bytes* holds the resized bytes object and ``0`` is returned; the address in *\*bytes* may differ from its input value. If the diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index f8cd0344fdd1c0..61eb994c370946 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -39,7 +39,8 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: PyObject* PyCell_Get(PyObject *cell) - Return the contents of the cell *cell*. + Return the contents of the cell *cell*, which can be ``NULL``. + If *cell* is not a cell object, returns ``NULL`` with an exception set. .. c:function:: PyObject* PyCell_GET(PyObject *cell) @@ -52,8 +53,10 @@ Cell objects are not likely to be useful elsewhere. Set the contents of the cell object *cell* to *value*. This releases the reference to any current content of the cell. *value* may be ``NULL``. *cell* - must be non-``NULL``; if it is not a cell object, ``-1`` will be returned. On - success, ``0`` will be returned. + must be non-``NULL``. + + On success, return ``0``. + If *cell* is not a cell object, set an exception and return ``-1``. .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 11c12e685fcace..6eae24b38fae48 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -30,9 +30,21 @@ bound into a function. Return true if *co* is a :ref:`code object `. This function always succeeds. -.. c:function:: int PyCode_GetNumFree(PyCodeObject *co) +.. c:function:: Py_ssize_t PyCode_GetNumFree(PyCodeObject *co) - Return the number of free variables in *co*. + Return the number of :term:`free (closure) variables ` + in a code object. + +.. c:function:: int PyUnstable_Code_GetFirstFree(PyCodeObject *co) + + Return the position of the first :term:`free (closure) variable ` + in a code object. + + .. versionchanged:: 3.13 + + Renamed from ``PyCode_GetFirstFree`` as part of :ref:`unstable-c-api`. + The old name is deprecated, but will remain available until the + signature changes again. .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) @@ -49,7 +61,7 @@ bound into a function. .. versionchanged:: 3.11 Added ``qualname`` and ``exceptiontable`` parameters. - .. index:: single: PyCode_New + .. index:: single: PyCode_New (C function) .. versionchanged:: 3.12 @@ -62,7 +74,7 @@ bound into a function. Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. - .. index:: single: PyCode_NewWithPosOnlyArgs + .. index:: single: PyCode_NewWithPosOnlyArgs (C function) .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` @@ -86,8 +98,8 @@ bound into a function. Return the line number of the instruction that occurs on or before ``byte_offset`` and ends after it. If you just need the line number of a frame, use :c:func:`PyFrame_GetLineNumber` instead. - For efficiently iterating over the line numbers in a code object, use `the API described in PEP 626 - `_. + For efficiently iterating over the line numbers in a code object, use :pep:`the API described in PEP 626 + <0626#out-of-process-debuggers-and-profilers>`. .. c:function:: int PyCode_Addr2Location(PyObject *co, int byte_offset, int *start_line, int *start_column, int *end_line, int *end_column) @@ -134,7 +146,8 @@ bound into a function. Equivalent to the Python code ``getattr(co, 'co_freevars')``. Returns a new reference to a :c:type:`PyTupleObject` containing the names of - the free variables. On error, ``NULL`` is returned and an exception is raised. + the :term:`free (closure) variables `. On error, ``NULL`` is returned + and an exception is raised. .. versionadded:: 3.11 @@ -221,7 +234,7 @@ may change without deprecation warnings. *free* will be called on non-``NULL`` data stored under the new index. Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. - .. index:: single: _PyEval_RequestCodeExtraIndex + .. index:: single: _PyEval_RequestCodeExtraIndex (C function) .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` @@ -239,7 +252,7 @@ may change without deprecation warnings. If no data was set under the index, set *extra* to ``NULL`` and return 0 without setting an exception. - .. index:: single: _PyCode_GetExtra + .. index:: single: _PyCode_GetExtra (C function) .. versionadded:: 3.6 as ``_PyCode_GetExtra`` @@ -254,7 +267,7 @@ may change without deprecation warnings. Set the extra data stored under the given index to *extra*. Return 0 on success. Set an exception and return -1 on failure. - .. index:: single: _PyCode_SetExtra + .. index:: single: _PyCode_SetExtra (C function) .. versionadded:: 3.6 as ``_PyCode_SetExtra`` diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 5a0474869071d9..16bd79475dc1e6 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -25,12 +25,16 @@ pointers. This is consistent throughout the API. The C structure which corresponds to the value portion of a Python complex number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. It is - defined as:: + use structures of this type as input or output values, as appropriate. + + .. c:member:: double real + double imag + + The structure is defined as:: typedef struct { - double real; - double imag; + double real; + double imag; } Py_complex; @@ -75,6 +79,8 @@ pointers. This is consistent throughout the API. If *num* is null and *exp* is not a positive real number, this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + Complex Numbers as Python Objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -106,11 +112,13 @@ Complex Numbers as Python Objects .. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) Create a new Python complex number object from a C :c:type:`Py_complex` value. + Return ``NULL`` with an exception set on error. .. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) Return a new :c:type:`PyComplexObject` object from *real* and *imag*. + Return ``NULL`` with an exception set on error. .. c:function:: double PyComplex_RealAsDouble(PyObject *op) @@ -121,7 +129,9 @@ Complex Numbers as Python Objects :meth:`~object.__complex__` method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to call :c:func:`PyFloat_AsDouble` and - returns its result. Upon failure, this method returns ``-1.0``, so one + returns its result. + + Upon failure, this method returns ``-1.0`` with an exception set, so one should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.13 @@ -135,8 +145,10 @@ Complex Numbers as Python Objects :meth:`~object.__complex__` method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to call :c:func:`PyFloat_AsDouble` and - returns ``0.0`` on success. Upon failure, this method returns ``-1.0``, so - one should call :c:func:`PyErr_Occurred` to check for errors. + returns ``0.0`` on success. + + Upon failure, this method returns ``-1.0`` with an exception set, so one + should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.13 Use :meth:`~object.__complex__` if available. @@ -149,8 +161,11 @@ Complex Numbers as Python Objects method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back - to :meth:`~object.__index__`. Upon failure, this method returns ``-1.0`` as a real - value. + to :meth:`~object.__index__`. + + Upon failure, this method returns :c:type:`Py_complex` + with :c:member:`~Py_complex.real` set to ``-1.0`` and with an exception set, so one + should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.8 Use :meth:`~object.__index__` if available. diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index d970f5443b1df5..b7c6550ff34aac 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -6,6 +6,8 @@ Context Variables Objects ------------------------- .. _contextvarsobjects_pointertype_change: +.. versionadded:: 3.7 + .. versionchanged:: 3.7.1 .. note:: @@ -24,8 +26,6 @@ Context Variables Objects See :issue:`34762` for more details. -.. versionadded:: 3.7 - This section details the public C API for the :mod:`contextvars` module. .. c:type:: PyContext @@ -101,6 +101,52 @@ Context object management functions: current context for the current thread. Returns ``0`` on success, and ``-1`` on error. +.. c:function:: int PyContext_AddWatcher(PyContext_WatchCallback callback) + + Register *callback* as a context object watcher for the current interpreter. + Return an ID which may be passed to :c:func:`PyContext_ClearWatcher`. + In case of error (e.g. no more watcher IDs available), + return ``-1`` and set an exception. + + .. versionadded:: 3.14 + +.. c:function:: int PyContext_ClearWatcher(int watcher_id) + + Clear watcher identified by *watcher_id* previously returned from + :c:func:`PyContext_AddWatcher` for the current interpreter. + Return ``0`` on success, or ``-1`` and set an exception on error + (e.g. if the given *watcher_id* was never registered.) + + .. versionadded:: 3.14 + +.. c:type:: PyContextEvent + + Enumeration of possible context object watcher events: + + - ``Py_CONTEXT_SWITCHED``: The :term:`current context` has switched to a + different context. The object passed to the watch callback is the + now-current :class:`contextvars.Context` object, or None if no context is + current. + + .. versionadded:: 3.14 + +.. c:type:: int (*PyContext_WatchCallback)(PyContextEvent event, PyObject *obj) + + Context object watcher callback function. The object passed to the callback + is event-specific; see :c:type:`PyContextEvent` for details. + + If the callback returns with an exception set, it must return ``-1``; this + exception will be printed as an unraisable exception using + :c:func:`PyErr_FormatUnraisable`. Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. + + .. versionadded:: 3.14 + Context variable functions: diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 4aaf3905e81c8a..c92ef4c653a675 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_HUGE_VAL`` (with + if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 97522da773477e..d2d4d5309c7098 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -318,10 +318,10 @@ Macros for the convenience of modules implementing the DB API: .. c:function:: PyObject* PyDateTime_FromTimestamp(PyObject *args) Create and return a new :class:`datetime.datetime` object given an argument - tuple suitable for passing to :meth:`datetime.datetime.fromtimestamp()`. + tuple suitable for passing to :meth:`datetime.datetime.fromtimestamp`. .. c:function:: PyObject* PyDate_FromTimestamp(PyObject *args) Create and return a new :class:`datetime.date` object given an argument - tuple suitable for passing to :meth:`datetime.date.fromtimestamp()`. + tuple suitable for passing to :meth:`datetime.date.fromtimestamp`. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 03f3d28187bfe9..ce73fa0cc60ebb 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -156,7 +156,7 @@ Dictionary Objects .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) - Similar than :c:func:`PyDict_GetItemRef`, but *key* is specified as a + Similar to :c:func:`PyDict_GetItemRef`, but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. @@ -191,6 +191,7 @@ Dictionary Objects to both *default_value* and *\*result* (if it's not ``NULL``). These may refer to the same object: in that case you hold two separate references to it. + .. versionadded:: 3.13 @@ -205,7 +206,7 @@ Dictionary Objects ``NULL``, and return ``0``. - On error, raise an exception and return ``-1``. - This is similar to :meth:`dict.pop`, but without the default value and + Similar to :meth:`dict.pop`, but without the default value and not raising :exc:`KeyError` if the key missing. .. versionadded:: 3.13 @@ -289,6 +290,17 @@ Dictionary Objects Py_DECREF(o); } + The function is not thread-safe in the :term:`free-threaded ` + build without external synchronization. You can use + :c:macro:`Py_BEGIN_CRITICAL_SECTION` to lock the dictionary while iterating + over it:: + + Py_BEGIN_CRITICAL_SECTION(self->dict); + while (PyDict_Next(self->dict, &pos, &key, &value)) { + ... + } + Py_END_CRITICAL_SECTION(); + .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c7e3cd9463e5d7..fc2336d120c259 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -34,7 +34,7 @@ propagated, additional calls into the Python/C API may not behave as intended and may fail in mysterious ways. .. note:: - The error indicator is **not** the result of :func:`sys.exc_info()`. + The error indicator is **not** the result of :func:`sys.exc_info`. The former corresponds to an exception that is not yet caught (and is therefore still propagating), while the latter returns an exception after it is caught (and has therefore stopped propagating). @@ -104,8 +104,8 @@ Printing and clearing Similar to :c:func:`PyErr_WriteUnraisable`, but the *format* and subsequent parameters help format the warning message; they have the same meaning and values as in :c:func:`PyUnicode_FromFormat`. - ``PyErr_WriteUnraisable(obj)`` is roughtly equivalent to - ``PyErr_FormatUnraisable("Exception ignored in: %R, obj)``. + ``PyErr_WriteUnraisable(obj)`` is roughly equivalent to + ``PyErr_FormatUnraisable("Exception ignored in: %R", obj)``. If *format* is ``NULL``, only the traceback is printed. .. versionadded:: 3.13 @@ -180,7 +180,7 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) - .. index:: single: strerror() + .. index:: single: strerror (C function) This is a convenience function to raise an exception when a C library function has returned an error and set the C variable :c:data:`errno`. It constructs a @@ -221,13 +221,14 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) - This is a convenience function to raise :exc:`WindowsError`. If called with + This is a convenience function to raise :exc:`OSError`. If called with *ierr* of ``0``, the error code returned by a call to :c:func:`!GetLastError` is used instead. It calls the Win32 function :c:func:`!FormatMessage` to retrieve the Windows description of error code given by *ierr* or :c:func:`!GetLastError`, - then it constructs a tuple object whose first item is the *ierr* value and whose - second item is the corresponding error message (gotten from - :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, + then it constructs a :exc:`OSError` object with the :attr:`~OSError.winerror` + attribute set to the error code, the :attr:`~OSError.strerror` attribute + set to the corresponding error message (gotten from + :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_OSError, object)``. This function always returns ``NULL``. .. availability:: Windows. @@ -396,7 +397,7 @@ an error value). .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. + :exc:`ResourceWarning` and it passes *source* to :class:`!warnings.WarningMessage`. .. versionadded:: 3.6 @@ -635,7 +636,7 @@ Signal Handling .. index:: pair: module; signal - single: SIGINT + single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) This function interacts with Python's signal handling. @@ -666,7 +667,7 @@ Signal Handling .. index:: pair: module; signal - single: SIGINT + single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) Simulate the effect of a :c:macro:`!SIGINT` signal arriving. @@ -732,7 +733,7 @@ Exception Classes This creates a class object derived from :exc:`Exception` (accessible in C as :c:data:`PyExc_Exception`). - The :attr:`__module__` attribute of the new class is set to the first part (up + The :attr:`~type.__module__` attribute of the new class is set to the first part (up to the last dot) of the *name* argument, and the class name is set to the last part (after the last dot). The *base* argument can be used to specify alternate base classes; it can either be only one class or a tuple of classes. The *dict* @@ -904,8 +905,8 @@ because the :ref:`call protocol ` takes care of recursion handling. Marks a point where a recursive C-level call is about to be performed. - If :c:macro:`USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it + If :c:macro:`!USE_STACKCHECK` is defined, this function checks if the OS + stack overflowed using :c:func:`PyOS_CheckStack`. If this is the case, it sets a :exc:`MemoryError` and returns a nonzero value. The function then checks if the recursion limit is reached. If this is the @@ -968,59 +969,60 @@ All standard Python exceptions are available as global variables whose names are the variables: .. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_BufferError - single: PyExc_ChildProcessError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_GeneratorExit - single: PyExc_ImportError - single: PyExc_IndentationError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_LookupError - single: PyExc_MemoryError - single: PyExc_ModuleNotFoundError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_StopAsyncIteration - single: PyExc_StopIteration - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TabError - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_UnboundLocalError - single: PyExc_UnicodeDecodeError - single: PyExc_UnicodeEncodeError - single: PyExc_UnicodeError - single: PyExc_UnicodeTranslateError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError + single: PyExc_BaseException (C var) + single: PyExc_Exception (C var) + single: PyExc_ArithmeticError (C var) + single: PyExc_AssertionError (C var) + single: PyExc_AttributeError (C var) + single: PyExc_BlockingIOError (C var) + single: PyExc_BrokenPipeError (C var) + single: PyExc_BufferError (C var) + single: PyExc_ChildProcessError (C var) + single: PyExc_ConnectionAbortedError (C var) + single: PyExc_ConnectionError (C var) + single: PyExc_ConnectionRefusedError (C var) + single: PyExc_ConnectionResetError (C var) + single: PyExc_EOFError (C var) + single: PyExc_FileExistsError (C var) + single: PyExc_FileNotFoundError (C var) + single: PyExc_FloatingPointError (C var) + single: PyExc_GeneratorExit (C var) + single: PyExc_ImportError (C var) + single: PyExc_IndentationError (C var) + single: PyExc_IndexError (C var) + single: PyExc_InterruptedError (C var) + single: PyExc_IsADirectoryError (C var) + single: PyExc_KeyError (C var) + single: PyExc_KeyboardInterrupt (C var) + single: PyExc_LookupError (C var) + single: PyExc_MemoryError (C var) + single: PyExc_ModuleNotFoundError (C var) + single: PyExc_NameError (C var) + single: PyExc_NotADirectoryError (C var) + single: PyExc_NotImplementedError (C var) + single: PyExc_OSError (C var) + single: PyExc_OverflowError (C var) + single: PyExc_PermissionError (C var) + single: PyExc_ProcessLookupError (C var) + single: PyExc_PythonFinalizationError (C var) + single: PyExc_RecursionError (C var) + single: PyExc_ReferenceError (C var) + single: PyExc_RuntimeError (C var) + single: PyExc_StopAsyncIteration (C var) + single: PyExc_StopIteration (C var) + single: PyExc_SyntaxError (C var) + single: PyExc_SystemError (C var) + single: PyExc_SystemExit (C var) + single: PyExc_TabError (C var) + single: PyExc_TimeoutError (C var) + single: PyExc_TypeError (C var) + single: PyExc_UnboundLocalError (C var) + single: PyExc_UnicodeDecodeError (C var) + single: PyExc_UnicodeEncodeError (C var) + single: PyExc_UnicodeError (C var) + single: PyExc_UnicodeTranslateError (C var) + single: PyExc_ValueError (C var) + single: PyExc_ZeroDivisionError (C var) +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | @@ -1095,6 +1097,8 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | @@ -1151,18 +1155,18 @@ the variables: These are compatibility aliases to :c:data:`PyExc_OSError`: .. index:: - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError + single: PyExc_EnvironmentError (C var) + single: PyExc_IOError (C var) + single: PyExc_WindowsError (C var) +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ -| :c:data:`PyExc_EnvironmentError` | | +| :c:data:`!PyExc_EnvironmentError` | | +-------------------------------------+----------+ -| :c:data:`PyExc_IOError` | | +| :c:data:`!PyExc_IOError` | | +-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | [2]_ | +| :c:data:`!PyExc_WindowsError` | [2]_ | +-------------------------------------+----------+ .. versionchanged:: 3.3 @@ -1188,17 +1192,17 @@ names are ``PyExc_`` followed by the Python exception name. These have the type the variables: .. index:: - single: PyExc_Warning - single: PyExc_BytesWarning - single: PyExc_DeprecationWarning - single: PyExc_FutureWarning - single: PyExc_ImportWarning - single: PyExc_PendingDeprecationWarning - single: PyExc_ResourceWarning - single: PyExc_RuntimeWarning - single: PyExc_SyntaxWarning - single: PyExc_UnicodeWarning - single: PyExc_UserWarning + single: PyExc_Warning (C var) + single: PyExc_BytesWarning (C var) + single: PyExc_DeprecationWarning (C var) + single: PyExc_FutureWarning (C var) + single: PyExc_ImportWarning (C var) + single: PyExc_PendingDeprecationWarning (C var) + single: PyExc_ResourceWarning (C var) + single: PyExc_RuntimeWarning (C var) + single: PyExc_SyntaxWarning (C var) + single: PyExc_UnicodeWarning (C var) + single: PyExc_UserWarning (C var) +------------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index d3a78c588454e8..e9019a0d500f7e 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -96,7 +96,7 @@ the :mod:`io` APIs instead. .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) - .. index:: single: Py_PRINT_RAW + .. index:: single: Py_PRINT_RAW (C macro) Write object *obj* to file object *p*. The only supported flag for *flags* is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 4f6ac0d8175c6b..1da37a5bcaeef9 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -2,20 +2,20 @@ .. _floatobjects: -Floating Point Objects +Floating-Point Objects ====================== -.. index:: pair: object; floating point +.. index:: pair: object; floating-point .. c:type:: PyFloatObject - This subtype of :c:type:`PyObject` represents a Python floating point object. + This subtype of :c:type:`PyObject` represents a Python floating-point object. .. c:var:: PyTypeObject PyFloat_Type - This instance of :c:type:`PyTypeObject` represents the Python floating point + This instance of :c:type:`PyTypeObject` represents the Python floating-point type. This is the same object as :class:`float` in the Python layer. @@ -45,7 +45,7 @@ Floating Point Objects .. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) Return a C :c:expr:`double` representation of the contents of *pyfloat*. If - *pyfloat* is not a Python floating point object but has a :meth:`~object.__float__` + *pyfloat* is not a Python floating-point object but has a :meth:`~object.__float__` method, this method will first be called to convert *pyfloat* into a float. If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. This method returns ``-1.0`` upon failure, so one should call diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 6bb1e9b5803b58..638a740e0c24da 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -120,12 +120,20 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) - Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`). + Get the *frame*'s :attr:`~frame.f_locals` attribute. + If the frame refers to an :term:`optimized scope`, this returns a + write-through proxy object that allows modifying the locals. + In all other cases (classes, modules, :func:`exec`, :func:`eval`) it returns + the mapping representing the frame locals directly (as described for + :func:`locals`). Return a :term:`strong reference`. .. versionadded:: 3.11 + .. versionchanged:: 3.13 + As part of :pep:`667`, return a proxy object for optimized scopes. + .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 6b2494ee4f0ed4..621da3eb069949 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -83,10 +83,15 @@ rules: .. versionadded:: 3.12 -.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) +.. c:macro:: PyObject_GC_Resize(TYPE, op, newsize) - Resize an object allocated by :c:macro:`PyObject_NewVar`. Returns the - resized object or ``NULL`` on failure. *op* must not be tracked by the collector yet. + Resize an object allocated by :c:macro:`PyObject_NewVar`. + Returns the resized object of type ``TYPE*`` (refers to any C type) + or ``NULL`` on failure. + + *op* must be of type :c:expr:`PyVarObject *` + and must not be tracked by the collector yet. + *newsize* must be of type :c:type:`Py_ssize_t`. .. c:function:: void PyObject_GC_Track(PyObject *op) diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index 91d88ae27bc9f4..00f8cb887dc7eb 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -3,7 +3,7 @@ PyHash API ---------- -See also the :c:member:`PyTypeObject.tp_hash` member. +See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. c:type:: Py_hash_t @@ -17,6 +17,35 @@ See also the :c:member:`PyTypeObject.tp_hash` member. .. versionadded:: 3.2 +.. c:macro:: PyHASH_MODULUS + + The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_BITS + + The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_MULTIPLIER + + Prime multiplier used in string and various other hashes. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_INF + + The hash value returned for a positive infinity. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_IMAG + + The multiplier used for the imaginary part of a complex number. + + .. versionadded:: 3.13 .. c:type:: PyHash_FuncDef @@ -59,3 +88,33 @@ See also the :c:member:`PyTypeObject.tp_hash` member. The function cannot fail: it cannot return ``-1``. .. versionadded:: 3.13 + + +.. c:function:: Py_hash_t Py_HashBuffer(const void *ptr, Py_ssize_t len) + + Compute and return the hash value of a buffer of *len* bytes + starting at address *ptr*. The hash is guaranteed to match that of + :class:`bytes`, :class:`memoryview`, and other built-in objects + that implement the :ref:`buffer protocol `. + + Use this function to implement hashing for immutable objects whose + :c:member:`~PyTypeObject.tp_richcompare` function compares to another + object's buffer. + + *len* must be greater than or equal to ``0``. + + This function always succeeds. + + .. versionadded:: 3.14 + + +.. c:function:: Py_hash_t PyObject_GenericHash(PyObject *obj) + + Generic hashing function that is meant to be put into a type + object's ``tp_hash`` slot. + Its result only depends on the object's identity. + + .. impl-detail:: + In CPython, it is equivalent to :c:func:`Py_HashPointer`. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 7c74e9e88678dc..6e48644c8fef8b 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -136,14 +136,14 @@ Importing Modules such modules have no way to know that the module object is an unknown (and probably damaged with respect to the module author's intents) state. - The module's :attr:`__spec__` and :attr:`__loader__` will be set, if - not set already, with the appropriate values. The spec's loader will - be set to the module's ``__loader__`` (if set) and to an instance of - :class:`~importlib.machinery.SourceFileLoader` otherwise. + The module's :attr:`~module.__spec__` and :attr:`~module.__loader__` will be + set, if not set already, with the appropriate values. The spec's loader + will be set to the module's :attr:`!__loader__` (if set) and to an instance + of :class:`~importlib.machinery.SourceFileLoader` otherwise. - The module's :attr:`__file__` attribute will be set to the code object's - :attr:`~codeobject.co_filename`. If applicable, :attr:`__cached__` will also - be set. + The module's :attr:`~module.__file__` attribute will be set to the code + object's :attr:`~codeobject.co_filename`. If applicable, + :attr:`~module.__cached__` will also be set. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -155,29 +155,29 @@ Importing Modules :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. versionchanged:: 3.12 - The setting of :attr:`__cached__` and :attr:`__loader__` is - deprecated. See :class:`~importlib.machinery.ModuleSpec` for + The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__` + is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. .. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) - Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of - the module object is set to *pathname* if it is non-``NULL``. + Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`~module.__file__` + attribute of the module object is set to *pathname* if it is non-``NULL``. See also :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__` + Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__` attribute of the module object is set to *cpathname* if it is non-``NULL``. Of the three functions, this is the preferred one to use. .. versionadded:: 3.3 .. versionchanged:: 3.12 - Setting :attr:`__cached__` is deprecated. See + Setting :attr:`~module.__cached__` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. @@ -190,7 +190,7 @@ Importing Modules .. versionadded:: 3.2 .. versionchanged:: 3.3 - Uses :func:`!imp.source_from_cache()` in calculating the source path if + Uses :func:`!imp.source_from_cache` in calculating the source path if only the bytecode path is provided. .. versionchanged:: 3.12 No longer uses the removed :mod:`!imp` module. @@ -308,7 +308,7 @@ Importing Modules The module name, as an ASCII encoded string. - .. c: member:: PyObject* (*initfunc)(void) + .. c:member:: PyObject* (*initfunc)(void) Initialization function for a module built into the interpreter. diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index 9a8f1507b3f4cc..ba56b03c6ac8e7 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -25,3 +25,4 @@ document the API functions in detail. memory.rst objimpl.rst apiabiversion.rst + monitoring.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index f8fd48e781d6da..6e881590131cab 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,7 +7,8 @@ Initialization, Finalization, and Threads ***************************************** -See also :ref:`Python Initialization Configuration `. +See :ref:`Python Initialization Configuration ` for details +on how to configure the interpreter prior to initialization. .. _pre-init-safe: @@ -21,6 +22,15 @@ a few functions and the :ref:`global configuration variables The following functions can be safely called before Python is initialized: +* Functions that initialize the interpreter: + + * :c:func:`Py_Initialize` + * :c:func:`Py_InitializeEx` + * :c:func:`Py_InitializeFromConfig` + * :c:func:`Py_BytesMain` + * :c:func:`Py_Main` + * the runtime pre-initialization functions covered in :ref:`init-config` + * Configuration functions: * :c:func:`PyImport_AppendInittab` @@ -29,7 +39,10 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_SetAllocator` * :c:func:`PyMem_SetupDebugHooks` * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` * :c:func:`PySys_ResetWarnOptions` + * the configuration functions covered in :ref:`init-config` * Informative functions: @@ -41,10 +54,12 @@ The following functions can be safely called before Python is initialized: * :c:func:`Py_GetCopyright` * :c:func:`Py_GetPlatform` * :c:func:`Py_GetVersion` + * :c:func:`Py_IsInitialized` * Utilities: * :c:func:`Py_DecodeLocale` + * the status reporting and utility functions covered in :ref:`init-config` * Memory allocators: @@ -53,13 +68,20 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_RawCalloc` * :c:func:`PyMem_RawFree` +* Synchronization: + + * :c:func:`PyMutex_Lock` + * :c:func:`PyMutex_Unlock` + .. note:: - The following functions **should not be called** before - :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, + Despite their apparent similarity to some of the functions listed above, + the following functions **should not be called** before the interpreter has + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - and :c:func:`Py_GetProgramName`. + :c:func:`Py_GetProgramName`, :c:func:`PyEval_InitThreads`, and + :c:func:`Py_RunMain`. .. _global-conf-vars: @@ -326,46 +348,55 @@ Initializing and finalizing the interpreter .. c:function:: void Py_Initialize() .. index:: + single: PyEval_InitThreads() single: modules (in module sys) single: path (in module sys) pair: module; builtins pair: module; __main__ pair: module; sys triple: module; search; path - single: Py_FinalizeEx() + single: Py_FinalizeEx (C function) Initialize the Python interpreter. In an application embedding Python, this should be called before using any other Python/C API functions; see :ref:`Before Python Initialization ` for the few exceptions. - This initializes - the table of loaded modules (``sys.modules``), and creates the fundamental - modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes - the module search path (``sys.path``). It does not set ``sys.argv``; use - the new :c:type:`PyConfig` API of the :ref:`Python Initialization - Configuration ` for that. This is a no-op when called for a - second time - (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a - fatal error if the initialization fails. - - Use the :c:func:`Py_InitializeFromConfig` function to customize the + This initializes the table of loaded modules (``sys.modules``), and creates + the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. + It also initializes the module search path (``sys.path``). It does not set + ``sys.argv``; use the :ref:`Python Initialization Configuration ` + API for that. This is a no-op when called for a second time (without calling + :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal + error if the initialization fails. + + Use :c:func:`Py_InitializeFromConfig` to customize the :ref:`Python Initialization Configuration `. .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, which will - also affect non-Python uses of the console using the C Runtime. + On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, + which will also affect non-Python uses of the console using the C Runtime. .. c:function:: void Py_InitializeEx(int initsigs) This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, which - might be useful when Python is embedded. + *initsigs* is ``0``, it skips initialization registration of signal handlers, + which may be useful when CPython is embedded as part of a larger application. - Use the :c:func:`Py_InitializeFromConfig` function to customize the + Use :c:func:`Py_InitializeFromConfig` to customize the :ref:`Python Initialization Configuration `. +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration, as described in + :ref:`init-from-config`. + + See the :ref:`init-config` section for details on pre-initializing the + interpreter, populating the runtime configuration structure, and querying + the returned status structure. + + .. c:function:: int Py_IsInitialized() Return true (nonzero) when the Python interpreter has been initialized, false @@ -386,11 +417,23 @@ Initializing and finalizing the interpreter Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory - allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). Normally the - return value is ``0``. If there were errors during finalization - (flushing buffered data), ``-1`` is returned. + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second + time (without calling :c:func:`Py_Initialize` again first). + + Since this is the reverse of :c:func:`Py_Initialize`, it should be called + in the same thread with the same interpreter active. That means + the main thread and the main interpreter. + This should never be called while :c:func:`Py_RunMain` is running. + + Normally the return value is ``0``. + If there were errors during finalization (flushing buffered data), + ``-1`` is returned. + + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the preveiously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -406,25 +449,157 @@ Initializing and finalizing the interpreter loaded extension modules loaded by Python are not unloaded. Small amounts of memory allocated by the Python interpreter may not be freed (if you find a leak, please report it). Memory tied up in circular references between objects is not - freed. Some memory allocated by extension modules may not be freed. Some - extensions may not work properly if their initialization routine is called more - than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_FinalizeEx` more than once. + freed. Interned strings will all be deallocated regardless of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. :c:func:`Py_FinalizeEx` must not be called recursively from + within itself. Therefore, it must not be called by any code that may be run + as part of the interpreter shutdown process, such as :py:mod:`atexit` + handlers, object finalizers, or any code that may be run while flushing the + stdout and stderr files. .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx .. versionadded:: 3.6 + .. c:function:: void Py_Finalize() This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that disregards the return value. +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, + allowing the calling application to delegate the text decoding step to + the CPython runtime. + + .. versionadded:: 3.8 + + +.. c:function:: int Py_Main(int argc, wchar_t **argv) + + The main program for the standard interpreter, encapsulating a full + initialization/finalization cycle, as well as additional + behaviour to implement reading configurations settings from the environment + and command line, and then executing ``__main__`` in accordance with + :ref:`using-on-cmdline`. + + This is made available for programs which wish to support the full CPython + command line interface, rather than just embedding a Python runtime in a + larger application. + + The *argc* and *argv* parameters are similar to those which are passed to a + C program's :c:func:`main` function, except that the *argv* entries are first + converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also + important to note that the argument list entries may be modified to point to + strings other than those passed in (however, the contents of the strings + pointed to by the argument list are not modified). + + The return value will be ``0`` if the interpreter exits normally (i.e., + without an exception), ``1`` if the interpreter exits due to an exception, + or ``2`` if the argument list does not represent a valid Python command + line. + + Note that if an otherwise unhandled :exc:`SystemExit` is raised, this + function will not return ``1``, but exit the process, as long as + ``Py_InspectFlag`` is not set. If ``Py_InspectFlag`` is set, execution will + drop into the interactive Python prompt, at which point a second otherwise + unhandled :exc:`SystemExit` will still exit the process, while any other + means of exiting will set the return value as described above. + + In terms of the CPython runtime configuration APIs documented in the + :ref:`runtime configuration ` section (and without accounting + for error handling), ``Py_Main`` is approximately equivalent to:: + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetArgv(&config, argc, argv); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + Py_RunMain(); + + In normal usage, an embedding application will call this function + *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or + :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied + as described elsewhere in this documentation. If this function is instead + called *after* a preceding runtime initialization API call, then exactly + which environmental and command line configuration settings will be updated + is version dependent (as it depends on which settings correctly support + being modified after they have already been set once when the runtime was + first initialized). + + +.. c:function:: int Py_RunMain(void) + + Executes the main module in a fully configured CPython runtime. + + Executes the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. If none of these values are set, runs the interactive Python + prompt (REPL) using the ``__main__`` module's global namespace. + + If :c:member:`PyConfig.inspect` is not set (the default), the return value + will be ``0`` if the interpreter exits normally (that is, without raising + an exception), or ``1`` if the interpreter exits due to an exception. If an + otherwise unhandled :exc:`SystemExit` is raised, the function will immediately + exit the process instead of returning ``1``. + + If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option + is used), rather than returning when the interpreter exits, execution will + instead resume in an interactive Python prompt (REPL) using the ``__main__`` + module's global namespace. If the interpreter exited with an exception, it + is immediately raised in the REPL session. The function return value is + then determined by the way the *REPL session* terminates: returning ``0`` + if the session terminates without raising an unhandled exception, exiting + immediately for an unhandled :exc:`SystemExit`, and returning ``1`` for + any other unhandled exception. + + This function always finalizes the Python interpreter regardless of whether + it returns a value or immediately exits the process due to an unhandled + :exc:`SystemExit` exception. + + See :ref:`Python Configuration ` for an example of a + customized Python that always runs in isolated mode using + :c:func:`Py_RunMain`. + + Process-wide parameters ======================= +.. c:function:: void Py_SetProgramName(const wchar_t *name) + + .. index:: + single: Py_Initialize() + single: main() + single: Py_GetPath() + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python + Initialization Configuration `. + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by :c:func:`Py_GetPath` and some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + .. deprecated:: 3.11 + + .. c:function:: wchar_t* Py_GetProgramName() Return the program name set with :c:member:`PyConfig.program_name`, or the default. @@ -450,7 +625,7 @@ Process-wide parameters returned string points into static storage; the caller should not modify its value. This corresponds to the :makevar:`prefix` variable in the top-level :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` - script at build time. The value is available to Python code as ``sys.prefix``. + script at build time. The value is available to Python code as ``sys.base_prefix``. It is only useful on Unix. See also the next function. This function should not be called before :c:func:`Py_Initialize`, otherwise @@ -460,7 +635,8 @@ Process-wide parameters It now returns ``NULL`` if called before :c:func:`Py_Initialize`. .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.prefix` instead. + Get :data:`sys.base_prefix` instead, or :data:`sys.prefix` if + :ref:`virtual environments ` need to be handled. .. c:function:: wchar_t* Py_GetExecPrefix() @@ -473,7 +649,8 @@ Process-wide parameters should not modify its value. This corresponds to the :makevar:`exec_prefix` variable in the top-level :file:`Makefile` and the ``--exec-prefix`` argument to the :program:`configure` script at build time. The value is - available to Python code as ``sys.exec_prefix``. It is only useful on Unix. + available to Python code as ``sys.base_exec_prefix``. It is only useful on + Unix. Background: The exec-prefix differs from the prefix when platform dependent files (such as executables and shared libraries) are installed in a different @@ -504,7 +681,8 @@ Process-wide parameters It now returns ``NULL`` if called before :c:func:`Py_Initialize`. .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.exec_prefix` instead. + Get :data:`sys.base_exec_prefix` instead, or :data:`sys.exec_prefix` if + :ref:`virtual environments ` need to be handled. .. c:function:: wchar_t* Py_GetProgramFullPath() @@ -626,6 +804,106 @@ Process-wide parameters ``sys.version``. +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and + :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass ``0`` as *updatepath*, + and update :data:`sys.path` themselves if desired. + See :cve:`2008-5983`. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; + check w/ Guido. + + .. deprecated:: 3.11 + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used + instead, see :ref:`Python Initialization Configuration `. + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to ``1`` unless the :program:`python` interpreter was started with the + :option:`-I`. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + + .. deprecated:: 3.11 + + +.. c:function:: void Py_SetPythonHome(const wchar_t *home) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.home` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. + + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + .. deprecated:: 3.11 + + .. c:function:: wchar_t* Py_GetPythonHome() Return the default "home", that is, the value set by @@ -661,7 +939,7 @@ operations could cause problems in a multi-threaded program: for example, when two threads simultaneously increment the reference count of the same object, the reference count could end up being incremented only once instead of twice. -.. index:: single: setswitchinterval() (in module sys) +.. index:: single: setswitchinterval (in module sys) Therefore, the rule exists that only the thread that has acquired the :term:`GIL` may operate on Python objects or call Python/C API functions. @@ -671,8 +949,7 @@ released around potentially blocking I/O operations like reading or writing a file, so that other Python threads can run in the meantime. .. index:: - single: PyThreadState - single: PyThreadState + single: PyThreadState (C type) The Python interpreter keeps some thread-specific bookkeeping information inside a data structure called :c:type:`PyThreadState`. There's also one @@ -698,8 +975,8 @@ This is so common that a pair of macros exists to simplify it:: Py_END_ALLOW_THREADS .. index:: - single: Py_BEGIN_ALLOW_THREADS - single: Py_END_ALLOW_THREADS + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the @@ -714,8 +991,8 @@ The block above expands to the following code:: PyEval_RestoreThread(_save); .. index:: - single: PyEval_RestoreThread() - single: PyEval_SaveThread() + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) Here is how these functions work: the global interpreter lock is used to protect the pointer to the current thread state. When releasing the lock and saving the thread state, @@ -812,6 +1089,37 @@ thread, where the CPython global runtime was originally initialized. The only exception is if :c:func:`exec` will be called immediately after. +.. _cautions-regarding-runtime-finalization: + +Cautions regarding runtime finalization +--------------------------------------- + +In the late stage of :term:`interpreter shutdown`, after attempting to wait for +non-daemon threads to exit (though this can be interrupted by +:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime +is marked as *finalizing*: :c:func:`_Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the *finalization +thread* that initiated finalization (typically the main thread) is allowed to +acquire the :term:`GIL`. + +If any thread, other than the finalization thread, attempts to acquire the GIL +during finalization, either explicitly via :c:func:`PyGILState_Ensure`, +:c:macro:`Py_END_ALLOW_THREADS`, :c:func:`PyEval_AcquireThread`, or +:c:func:`PyEval_AcquireLock`, or implicitly when the interpreter attempts to +reacquire it after having yielded it, the thread enters **a permanently blocked +state** where it remains until the program exits. In most cases this is +harmless, but this can result in deadlock if a later stage of finalization +attempts to acquire a lock owned by the blocked thread, or otherwise waits on +the blocked thread. + +Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ +finalizations further up the call stack when such threads were forcibly exited +here in CPython 3.13 and earlier. The CPython runtime GIL acquiring C APIs +have never had any error reporting or handling expectations at GIL acquisition +time that would've allowed for graceful exit from this situation. Changing that +would require new stable C APIs and rewriting the majority of C code in the +CPython ecosystem to use those with error handling. + High-level API -------------- @@ -842,6 +1150,33 @@ code, or when embedding the Python interpreter: This thread's interpreter state. +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + .. c:function:: PyThreadState* PyEval_SaveThread() Release the global interpreter lock (if it has been created) and reset the @@ -858,11 +1193,14 @@ code, or when embedding the Python interpreter: ensues. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. .. c:function:: PyThreadState* PyThreadState_Get() @@ -917,11 +1255,14 @@ with sub-interpreters: to call arbitrary Python code. Failure is a fatal error. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. .. c:function:: void PyGILState_Release(PyGILState_STATE) @@ -1049,7 +1390,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyThreadState_DeleteCurrent(void) Destroy the current thread state and release the global interpreter lock. - Like :c:func:`PyThreadState_Delete`, the global interpreter lock need not + Like :c:func:`PyThreadState_Delete`, the global interpreter lock must be held. The thread state must have been reset with a previous call to :c:func:`PyThreadState_Clear`. @@ -1199,17 +1540,20 @@ All of the following functions must be called after :c:func:`Py_Initialize`. If this thread already has the lock, deadlock ensues. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. .. versionchanged:: 3.8 Updated to be consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, and terminate the current thread if called while the interpreter is finalizing. + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + :c:func:`PyEval_RestoreThread` is a higher-level function which is always available (even when threads have not been initialized). @@ -1399,8 +1743,8 @@ function. You can create and destroy them using the following functions: may be stored internally on the :c:type:`PyInterpreterState`. .. index:: - single: Py_FinalizeEx() - single: Py_Initialize() + single: Py_FinalizeEx (C function) + single: Py_Initialize (C function) Extension modules are shared between (sub-)interpreters as follows: @@ -1428,7 +1772,7 @@ function. You can create and destroy them using the following functions: As with multi-phase initialization, this means that only C-level static and global variables are shared between these modules. - .. index:: single: close() (in module os) + .. index:: single: close (in module os) .. c:function:: PyThreadState* Py_NewInterpreter(void) @@ -1451,7 +1795,7 @@ function. You can create and destroy them using the following functions: .. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - .. index:: single: Py_FinalizeEx() + .. index:: single: Py_FinalizeEx (C function) Destroy the (sub-)interpreter represented by the given thread state. The given thread state must be the current thread state. See the @@ -1543,8 +1887,6 @@ pointer and a void pointer argument. .. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - .. index:: single: Py_AddPendingCall() - Schedule a function to be called from the main interpreter thread. On success, ``0`` is returned and *func* is queued for being called in the main thread. On failure, ``-1`` is returned without setting any exception. @@ -1578,14 +1920,14 @@ pointer and a void pointer argument. function is generally **not** suitable for calling Python code from arbitrary C threads. Instead, use the :ref:`PyGILState API`. + .. versionadded:: 3.1 + .. versionchanged:: 3.9 If this function is called in a subinterpreter, the function *func* is now scheduled to be called from the subinterpreter, rather than being called from the main interpreter. Each subinterpreter now has its own list of scheduled calls. - .. versionadded:: 3.1 - .. _profiling: Profiling and Tracing @@ -1749,6 +2091,58 @@ Python-level trace functions in previous versions. .. versionadded:: 3.12 +Reference tracing +================= + +.. versionadded:: 3.13 + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + +.. versionadded:: 3.13 + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Not that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. The GIL will be held every time + the tracer function is called. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 .. _advanced-debugging: @@ -1945,3 +2339,145 @@ be used in new code. .. c:function:: void PyThread_delete_key_value(int key) .. c:function:: void PyThread_ReInitTLS() +Synchronization Primitives +========================== + +The C-API provides a basic mutual exclusion lock. + +.. c:type:: PyMutex + + A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to + zero to represent the unlocked state. For example:: + + PyMutex mutex = {0}; + + Instances of :c:type:`!PyMutex` should not be copied or moved. Both the + contents and address of a :c:type:`!PyMutex` are meaningful, and it must + remain at a fixed, writable location in memory. + + .. note:: + + A :c:type:`!PyMutex` currently occupies one byte, but the size should be + considered unstable. The size may change in future Python releases + without a deprecation period. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Lock(PyMutex *m) + + Lock mutex *m*. If another thread has already locked it, the calling + thread will block until the mutex is unlocked. While blocked, the thread + will temporarily release the :term:`GIL` if it is held. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Unlock(PyMutex *m) + + Unlock mutex *m*. The mutex must be locked --- otherwise, the function will + issue a fatal error. + + .. versionadded:: 3.13 + +.. _python-critical-section-api: + +Python Critical Section API +--------------------------- + +The critical section API provides a deadlock avoidance layer on top of +per-object locks for :term:`free-threaded ` CPython. They are +intended to replace reliance on the :term:`global interpreter lock`, and are +no-ops in versions of Python with the global interpreter lock. + +Critical sections avoid deadlocks by implicitly suspending active critical +sections and releasing the locks during calls to :c:func:`PyEval_SaveThread`. +When :c:func:`PyEval_RestoreThread` is called, the most recent critical section +is resumed, and its locks reacquired. This means the critical section API +provides weaker guarantees than traditional locks -- they are useful because +their behavior is similar to the :term:`GIL`. + +The functions and structs used by the macros are exposed for cases +where C macros are not available. They should only be used as in the +given macro expansions. Note that the sizes and contents of the structures may +change in future Python versions. + +.. note:: + + Operations that need to lock two objects at once must use + :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical + sections to lock more than one object at once, because the inner critical + section may suspend the outer critical sections. This API does not provide + a way to lock more than two objects at once. + +Example usage:: + + static PyObject * + set_field(MyObject *self, PyObject *value) + { + Py_BEGIN_CRITICAL_SECTION(self); + Py_SETREF(self->field, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; + } + +In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which +can call arbitrary code through an object's deallocation function. The critical +section API avoids potential deadlocks due to reentrancy and lock ordering +by allowing the runtime to temporarily suspend the critical section if the +code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) + + Acquires the per-object lock for the object *op* and begins a + critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_END_CRITICAL_SECTION() + + Ends the critical section and releases the per-object lock. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection_End(&_py_cs); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) + + Acquires the per-objects locks for the objects *a* and *b* and begins a + critical section. The locks are acquired in a consistent order (lowest + address first) to avoid lock ordering deadlocks. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection_Begin2(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_END_CRITICAL_SECTION2() + + Ends the critical section and releases the per-object locks. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection_End2(&_py_cs2); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 47a8fbb2cd9c97..6194d7446c73e4 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -6,6 +6,9 @@ Python Initialization Configuration *********************************** +PyConfig C API +============== + .. versionadded:: 3.8 Python can be initialized with :c:func:`Py_InitializeFromConfig` and the @@ -34,7 +37,7 @@ See also :ref:`Initialization, Finalization, and Threads `. Example -======= +------- Example of customized Python always running in isolated mode:: @@ -73,7 +76,7 @@ Example of customized Python always running in isolated mode:: PyWideStringList -================ +---------------- .. c:type:: PyWideStringList @@ -116,7 +119,7 @@ PyWideStringList List items. PyStatus -======== +-------- .. c:type:: PyStatus @@ -210,7 +213,7 @@ Example:: PyPreConfig -=========== +----------- .. c:type:: PyPreConfig @@ -321,7 +324,7 @@ PyPreConfig * Set :c:member:`PyConfig.filesystem_encoding` to ``"mbcs"``, * Set :c:member:`PyConfig.filesystem_errors` to ``"replace"``. - Initialized the from :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + Initialized from the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable value. Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for @@ -360,7 +363,7 @@ PyPreConfig .. _c-preinit: Preinitialize Python with PyPreConfig -===================================== +------------------------------------- The preinitialization of Python: @@ -440,7 +443,7 @@ the :ref:`Python UTF-8 Mode `:: PyConfig -======== +-------- .. c:type:: PyConfig @@ -509,7 +512,7 @@ PyConfig The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would parse the application options as Python options. :ref:`Preinitialize Python ` if needed. @@ -1041,7 +1044,7 @@ PyConfig The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would parse the application options as Python options. Default: ``1`` in Python mode, ``0`` in isolated mode. @@ -1245,16 +1248,24 @@ PyConfig .. c:member:: int perf_profiling - Enable compatibility mode with the perf profiler? + Enable the Linux ``perf`` profiler support? + + If equals to ``1``, enable support for the Linux ``perf`` profiler. - If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` - for more information. + If equals to ``2``, enable support for the Linux ``perf`` profiler with + DWARF JIT support. - Set by :option:`-X perf <-X>` command line option and by the + Set to ``1`` by :option:`-X perf <-X>` command-line option and the :envvar:`PYTHONPERFSUPPORT` environment variable. + Set to ``2`` by the :option:`-X perf_jit <-X>` command-line option and + the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable. + Default: ``-1``. + .. seealso:: + See :ref:`perf_profiling` for more information. + .. versionadded:: 3.12 .. c:member:: int use_environment @@ -1345,14 +1356,13 @@ the :option:`-X` command line option. The ``show_alloc_count`` field has been removed. -Initialization with PyConfig -============================ - -Function to initialize Python: +.. _init-from-config: -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) +Initialization with PyConfig +---------------------------- - Initialize Python from *config* configuration. +Initializing the interpreter from a populated configuration struct is handled +by calling :c:func:`Py_InitializeFromConfig`. The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. @@ -1458,7 +1468,7 @@ initialization:: .. _init-isolated-conf: Isolated Configuration -====================== +---------------------- :c:func:`PyPreConfig_InitIsolatedConfig` and :c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to @@ -1478,7 +1488,7 @@ to avoid computing the default path configuration. .. _init-python-config: Python Configuration -==================== +-------------------- :c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` functions create a configuration to build a customized Python which behaves as @@ -1496,7 +1506,7 @@ and :ref:`Python UTF-8 Mode ` .. _init-path-config: Python Path Configuration -========================= +------------------------- :c:type:`PyConfig` contains multiple fields for the path configuration: @@ -1582,24 +1592,321 @@ The ``__PYVENV_LAUNCHER__`` environment variable is used to set :c:member:`PyConfig.base_executable`. -Py_RunMain() -============ +PyInitConfig C API +================== + +C API to configure the Python initialization (:pep:`741`). + +.. versionadded:: 3.14 + +Create Config +------------- + +.. c:struct:: PyInitConfig + + Opaque structure to configure the Python initialization. + + +.. c:function:: PyInitConfig* PyInitConfig_Create(void) + + Create a new initialization configuration using :ref:`Isolated Configuration + ` default values. + + It must be freed by :c:func:`PyInitConfig_Free`. + + Return ``NULL`` on memory allocation failure. + + +.. c:function:: void PyInitConfig_Free(PyInitConfig *config) + + Free memory of the initialization configuration *config*. + + If *config* is ``NULL``, no operation is performed. + + +Error Handling +-------------- + +.. c:function:: int PyInitConfig_GetError(PyInitConfig* config, const char **err_msg) + + Get the *config* error message. + + * Set *\*err_msg* and return ``1`` if an error is set. + * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. + + An error message is an UTF-8 encoded string. + + If *config* has an exit code, format the exit code as an error + message. + + The error message remains valid until another ``PyInitConfig`` + function is called with *config*. The caller doesn't have to free the + error message. + + +.. c:function:: int PyInitConfig_GetExitCode(PyInitConfig* config, int *exitcode) + + Get the *config* exit code. + + * Set *\*exitcode* and return ``1`` if *config* has an exit code set. + * Return ``0`` if *config* has no exit code set. + + Only the ``Py_InitializeFromInitConfig()`` function can set an exit + code if the ``parse_argv`` option is non-zero. + + An exit code can be set when parsing the command line failed (exit + code ``2``) or when a command line option asks to display the command + line help (exit code ``0``). + + +Get Options +----------- + +The configuration option *name* parameter must be a non-NULL +null-terminated UTF-8 encoded string. + +.. c:function:: int PyInitConfig_HasOption(PyInitConfig *config, const char *name) + + Test if the configuration has an option called *name*. + + Return ``1`` if the option exists, or return ``0`` otherwise. + + +.. c:function:: int PyInitConfig_GetInt(PyInitConfig *config, const char *name, int64_t *value) + + Get an integer configuration option. + + * Set *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +.. c:function:: int PyInitConfig_GetStr(PyInitConfig *config, const char *name, char **value) + + Get a string configuration option as a null-terminated UTF-8 + encoded string. + + * Set *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + *\*value* can be set to ``NULL`` if the option is an optional string and the + option is unset. + + On success, the string must be released with ``free(value)`` if it's not + ``NULL``. + + +.. c:function:: int PyInitConfig_GetStrList(PyInitConfig *config, const char *name, size_t *length, char ***items) + + Get a string list configuration option as an array of + null-terminated UTF-8 encoded strings. + + * Set *\*length* and *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + On success, the string list must be released with + ``PyInitConfig_FreeStrList(length, items)``. + + +.. c:function:: void PyInitConfig_FreeStrList(size_t length, char **items) + + Free memory of a string list created by + ``PyInitConfig_GetStrList()``. + + +Set Options +----------- + +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. + +Some configuration options have side effects on other options. This logic is +only implemented when ``Py_InitializeFromInitConfig()`` is called, not by the +"Set" functions below. For example, setting ``dev_mode`` to ``1`` does not set +``faulthandler`` to ``1``. + +.. c:function:: int PyInitConfig_SetInt(PyInitConfig *config, const char *name, int64_t value) + + Set an integer configuration option. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +.. c:function:: int PyInitConfig_SetStr(PyInitConfig *config, const char *name, const char *value) + + Set a string configuration option from a null-terminated UTF-8 + encoded string. The string is copied. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +.. c:function:: int PyInitConfig_SetStrList(PyInitConfig *config, const char *name, size_t length, char * const *items) + + Set a string list configuration option from an array of + null-terminated UTF-8 encoded strings. The string list is copied. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +Module +------ + +.. c:function:: int PyInitConfig_AddModule(PyInitConfig *config, const char *name, PyObject* (*initfunc)(void)) + + Add a built-in extension module to the table of built-in modules. + + The new module can be imported by the name *name*, and uses the function + *initfunc* as the initialization function called on the first attempted + import. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + If Python is initialized multiple times, ``PyInitConfig_AddModule()`` must + be called at each Python initialization. + + Similar to the :c:func:`PyImport_AppendInittab` function. + + +Initialize Python +----------------- + +.. c:function:: int Py_InitializeFromInitConfig(PyInitConfig *config) + + Initialize Python from the initialization configuration. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + * Set an exit code in *config* and return ``-1`` if Python wants to + exit. + + See ``PyInitConfig_GetExitcode()`` for the exit code case. + + +Example +------- + +Example initializing Python, set configuration options of various types, +return ``-1`` on error: + +.. code-block:: c + + int init_python(void) + { + PyInitConfig *config = PyInitConfig_Create(); + if (config == NULL) { + printf("PYTHON INIT ERROR: memory allocation failed\n"); + return -1; + } + + // Set an integer (dev mode) + if (PyInitConfig_SetInt(config, "dev_mode", 1) < 0) { + goto error; + } + + // Set a list of UTF-8 strings (argv) + char *argv[] = {"my_program", "-c", "pass"}; + if (PyInitConfig_SetStrList(config, "argv", + Py_ARRAY_LENGTH(argv), argv) < 0) { + goto error; + } + + // Set a UTF-8 string (program name) + if (PyInitConfig_SetStr(config, "program_name", L"my_program") < 0) { + goto error; + } + + // Initialize Python with the configuration + if (Py_InitializeFromInitConfig(config) < 0) { + goto error; + } + PyInitConfig_Free(config); + return 0; + + error: + { + // Display the error message + // This uncommon braces style is used, because you cannot make + // goto targets point to variable declarations. + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("PYTHON INIT ERROR: %s\n", err_msg); + PyInitConfig_Free(config); + + return -1; + } + } + + +Runtime Python configuration API +================================ + +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. + +Some options are read from the :mod:`sys` attributes. For example, the option +``"argv"`` is read from :data:`sys.argv`. + + +.. c:function:: PyObject* PyConfig_Get(const char *name) + + Get the current runtime value of a configuration option as a Python object. + + * Return a new reference on success. + * Set an exception and return ``NULL`` on error. + + The object type depends on the configuration option. It can be: + + * ``bool`` + * ``int`` + * ``str`` + * ``list[str]`` + * ``dict[str, str]`` + + The caller must hold the GIL. The function cannot be called before + Python initialization nor after Python finalization. + + .. versionadded:: 3.14 + + +.. c:function:: int PyConfig_GetInt(const char *name, int *value) + + Similar to :c:func:`PyConfig_Get`, but get the value as a C int. + + * Return ``0`` on success. + * Set an exception and return ``-1`` on error. + + .. versionadded:: 3.14 + + +.. c:function:: PyObject* PyConfig_Names(void) + + Get all configuration option names as a ``frozenset``. + + * Return a new reference on success. + * Set an exception and return ``NULL`` on error. + + The caller must hold the GIL. The function cannot be called before + Python initialization nor after Python finalization. + + .. versionadded:: 3.14 + -.. c:function:: int Py_RunMain(void) +.. c:function:: int PyConfig_Set(const char *name, PyObject *value) - Execute the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. + Set the current runtime value of a configuration option. - By default and when if :option:`-i` option is used, run the REPL. + * Raise a :exc:`ValueError` if there is no option *name*. + * Raise a :exc:`ValueError` if *value* is an invalid value. + * Raise a :exc:`ValueError` if the option is read-only (cannot be set). + * Raise a :exc:`TypeError` if *value* has not the proper type. - Finally, finalizes Python and returns an exit status that can be passed to - the ``exit()`` function. + The caller must hold the GIL. The function cannot be called before + Python initialization nor after Python finalization. -See :ref:`Python Configuration ` for an example of -customized Python always running in isolated mode using -:c:func:`Py_RunMain`. + .. versionadded:: 3.14 Py_GetArgcArgv() diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index dcda1071a58f35..8ef463e3f88ca8 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -325,8 +325,8 @@ objects that reference each other here; for now, the solution is "don't do that.") .. index:: - single: Py_INCREF() - single: Py_DECREF() + single: Py_INCREF (C function) + single: Py_DECREF (C function) Reference counts are always manipulated explicitly. The normal way is to use the macro :c:func:`Py_INCREF` to take a new reference to an @@ -401,8 +401,8 @@ function, that function assumes that it now owns that reference, and you are not responsible for it any longer. .. index:: - single: PyList_SetItem() - single: PyTuple_SetItem() + single: PyList_SetItem (C function) + single: PyTuple_SetItem (C function) Few functions steal references; the two notable exceptions are :c:func:`PyList_SetItem` and :c:func:`PyTuple_SetItem`, which steal a reference @@ -491,8 +491,8 @@ using :c:func:`PySequence_GetItem` (which happens to take exactly the same arguments), you do own a reference to the returned object. .. index:: - single: PyList_GetItem() - single: PySequence_GetItem() + single: PyList_GetItem (C function) + single: PySequence_GetItem (C function) Here is an example of how you could write a function that computes the sum of the items in a list of integers; once using :c:func:`PyList_GetItem`, and once @@ -587,7 +587,7 @@ caller, then to the caller's caller, and so on, until they reach the top-level interpreter, where they are reported to the user accompanied by a stack traceback. -.. index:: single: PyErr_Occurred() +.. index:: single: PyErr_Occurred (C function) For C programmers, however, error checking always has to be explicit. All functions in the Python/C API can raise exceptions, unless an explicit claim is @@ -601,8 +601,8 @@ ambiguous return value, and require explicit testing for errors with :c:func:`PyErr_Occurred`. These exceptions are always explicitly documented. .. index:: - single: PyErr_SetString() - single: PyErr_Clear() + single: PyErr_SetString (C function) + single: PyErr_Clear (C function) Exception state is maintained in per-thread storage (this is equivalent to using global storage in an unthreaded application). A thread can be in one of @@ -624,7 +624,7 @@ an exception is being passed on between C functions until it reaches the Python bytecode interpreter's main loop, which takes care of transferring it to ``sys.exc_info()`` and friends. -.. index:: single: exc_info() (in module sys) +.. index:: single: exc_info (in module sys) Note that starting with Python 1.5, the preferred, thread-safe way to access the exception state from Python code is to call the function :func:`sys.exc_info`, @@ -709,9 +709,9 @@ Here is the corresponding C code, in all its glory:: .. index:: single: incr_item() .. index:: - single: PyErr_ExceptionMatches() - single: PyErr_Clear() - single: Py_XDECREF() + single: PyErr_ExceptionMatches (C function) + single: PyErr_Clear (C function) + single: Py_XDECREF (C function) This example represents an endorsed use of the ``goto`` statement in C! It illustrates the use of :c:func:`PyErr_ExceptionMatches` and @@ -735,7 +735,7 @@ the finalization, of the Python interpreter. Most functionality of the interpreter can only be used after the interpreter has been initialized. .. index:: - single: Py_Initialize() + single: Py_Initialize (C function) pair: module; builtins pair: module; __main__ pair: module; sys @@ -770,10 +770,10 @@ environment variable :envvar:`PYTHONHOME`, or insert additional directories in front of the standard path by setting :envvar:`PYTHONPATH`. .. index:: - single: Py_GetPath() - single: Py_GetPrefix() - single: Py_GetExecPrefix() - single: Py_GetProgramFullPath() + single: Py_GetPath (C function) + single: Py_GetPrefix (C function) + single: Py_GetExecPrefix (C function) + single: Py_GetProgramFullPath (C function) The embedding application can steer the search by setting :c:member:`PyConfig.program_name` *before* calling @@ -784,7 +784,7 @@ control has to provide its own implementation of :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, and :c:func:`Py_GetProgramFullPath` (all defined in :file:`Modules/getpath.c`). -.. index:: single: Py_IsInitialized() +.. index:: single: Py_IsInitialized (C function) Sometimes, it is desirable to "uninitialize" Python. For instance, the application may want to start over (make another call to diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 434d2021cea8e6..bf9df62c6f1706 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -10,7 +10,8 @@ There are two functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) Return non-zero if the object *o* can be safely passed to - :c:func:`PyIter_Next`, and ``0`` otherwise. This function always succeeds. + :c:func:`PyIter_NextItem` and ``0`` otherwise. + This function always succeeds. .. c:function:: int PyAIter_Check(PyObject *o) @@ -19,41 +20,27 @@ There are two functions specifically for working with iterators. .. versionadded:: 3.10 +.. c:function:: int PyIter_NextItem(PyObject *iter, PyObject **item) + + Return ``1`` and set *item* to a :term:`strong reference` of the + next value of the iterator *iter* on success. + Return ``0`` and set *item* to ``NULL`` if there are no remaining values. + Return ``-1``, set *item* to ``NULL`` and set an exception on error. + + .. versionadded:: 3.14 + .. c:function:: PyObject* PyIter_Next(PyObject *o) + This is an older version of :c:func:`!PyIter_NextItem`, + which is retained for backwards compatibility. + Prefer :c:func:`PyIter_NextItem`. + Return the next value from the iterator *o*. The object must be an iterator according to :c:func:`PyIter_Check` (it is up to the caller to check this). If there are no remaining values, returns ``NULL`` with no exception set. If an error occurs while retrieving the item, returns ``NULL`` and passes along the exception. -To write a loop which iterates over an iterator, the C code should look -something like this:: - - PyObject *iterator = PyObject_GetIter(obj); - PyObject *item; - - if (iterator == NULL) { - /* propagate error */ - } - - while ((item = PyIter_Next(iterator))) { - /* do something with item */ - ... - /* release reference when done */ - Py_DECREF(item); - } - - Py_DECREF(iterator); - - if (PyErr_Occurred()) { - /* propagate error */ - } - else { - /* continue doing useful work */ - } - - .. c:type:: PySendResult The enum value used to represent different results of :c:func:`PyIter_Send`. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 53eb54d3e1021a..758415a76e5cb4 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -38,9 +38,12 @@ List Objects .. note:: If *len* is greater than zero, the returned list object's items are - set to ``NULL``. Thus you cannot use abstract API functions such as - :c:func:`PySequence_SetItem` or expose the object to Python code before - setting all items to a real object with :c:func:`PyList_SetItem`. + set to ``NULL``. Thus you cannot use abstract API functions such as + :c:func:`PySequence_SetItem` or expose the object to Python code before + setting all items to a real object with :c:func:`PyList_SetItem` or + :c:func:`PyList_SET_ITEM()`. The following APIs are safe APIs before + the list is fully initialized: :c:func:`PyList_SetItem()` and :c:func:`PyList_SET_ITEM()`. + .. c:function:: Py_ssize_t PyList_Size(PyObject *list) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 045604870d3c84..9ff3e5265004a1 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -69,12 +69,32 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. on failure. +.. c:function:: PyObject* PyLong_FromInt32(int32_t value) + PyObject* PyLong_FromInt64(int64_t value) + + Return a new :c:type:`PyLongObject` object from a signed C + :c:expr:`int32_t` or :c:expr:`int64_t`, or ``NULL`` + with an exception set on failure. + + .. versionadded:: 3.14 + + .. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, or ``NULL`` on failure. +.. c:function:: PyObject* PyLong_FromUInt32(uint32_t value) + PyObject* PyLong_FromUInt64(uint64_t value) + + Return a new :c:type:`PyLongObject` object from an unsigned C + :c:expr:`uint32_t` or :c:expr:`uint64_t`, or ``NULL`` + with an exception set on failure. + + .. versionadded:: 3.14 + + .. c:function:: PyObject* PyLong_FromDouble(double v) Return a new :c:type:`PyLongObject` object from the integer part of *v*, or @@ -94,9 +114,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. ignored. If there are no digits or *str* is not NULL-terminated following the digits and trailing whitespace, :exc:`ValueError` will be raised. - .. seealso:: Python methods :meth:`int.to_bytes` and :meth:`int.from_bytes` - to convert a :c:type:`PyLongObject` to/from an array of bytes in base - ``256``. You can call those from C using :c:func:`PyObject_CallMethod`. + .. seealso:: :c:func:`PyLong_AsNativeBytes()` and + :c:func:`PyLong_FromNativeBytes()` functions can be used to convert + a :c:type:`PyLongObject` to/from an array of bytes in base ``256``. .. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) @@ -113,11 +133,36 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. retrieved from the resulting value using :c:func:`PyLong_AsVoidPtr`. -.. XXX alias PyLong_AS_LONG (for now) +.. c:function:: PyObject* PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, int flags) + + Create a Python integer from the value contained in the first *n_bytes* of + *buffer*, interpreted as a two's-complement signed number. + + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is a sign bit. Passing + ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` will produce the same result as calling + :c:func:`PyLong_FromUnsignedNativeBytes`. Other flags are ignored. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n_bytes, int flags) + + Create a Python integer from the value contained in the first *n_bytes* of + *buffer*, interpreted as an unsigned number. + + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is not a sign bit. Flags other than endian are ignored. + + .. versionadded:: 3.13 + + .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: - single: LONG_MAX + single: LONG_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:expr:`long` representation of *obj*. If *obj* is not an @@ -135,6 +180,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.10 This function will no longer use :meth:`~object.__int__`. + .. c:namespace:: NULL + + .. c:function:: long PyLong_AS_LONG(PyObject *obj) + + A :term:`soft deprecated` alias. + Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, + it can fail with :exc:`OverflowError` or another exception. + + .. deprecated:: 3.14 + The function is soft deprecated. .. c:function:: int PyLong_AsInt(PyObject *obj) @@ -210,7 +265,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) .. index:: - single: PY_SSIZE_T_MAX + single: PY_SSIZE_T_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:type:`Py_ssize_t` representation of *pylong*. *pylong* must @@ -225,7 +280,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) .. index:: - single: ULONG_MAX + single: ULONG_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:expr:`unsigned long` representation of *pylong*. *pylong* @@ -241,7 +296,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: size_t PyLong_AsSize_t(PyObject *pylong) .. index:: - single: SIZE_MAX + single: SIZE_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:type:`size_t` representation of *pylong*. *pylong* must be @@ -311,6 +366,43 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. This function will no longer use :meth:`~object.__int__`. +.. c:function:: int PyLong_AsInt32(PyObject *obj, int32_t *value) + int PyLong_AsInt64(PyObject *obj, int64_t *value) + + Set *\*value* to a signed C :c:expr:`int32_t` or :c:expr:`int64_t` + representation of *obj*. + + If the *obj* value is out of range, raise an :exc:`OverflowError`. + + Set *\*value* and return ``0`` on success. + Set an exception and return ``-1`` on error. + + *value* must not be ``NULL``. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_AsUInt32(PyObject *obj, uint32_t *value) + int PyLong_AsUInt64(PyObject *obj, uint64_t *value) + + Set *\*value* to an unsigned C :c:expr:`uint32_t` or :c:expr:`uint64_t` + representation of *obj*. + + If *obj* is not an instance of :c:type:`PyLongObject`, first call its + :meth:`~object.__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. + + * If *obj* is negative, raise a :exc:`ValueError`. + * If the *obj* value is out of range, raise an :exc:`OverflowError`. + + Set *\*value* and return ``0`` on success. + Set an exception and return ``-1`` on error. + + *value* must not be ``NULL``. + + .. versionadded:: 3.14 + + .. c:function:: double PyLong_AsDouble(PyObject *pylong) Return a C :c:expr:`double` representation of *pylong*. *pylong* must be @@ -332,6 +424,175 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. +.. c:function:: Py_ssize_t PyLong_AsNativeBytes(PyObject *pylong, void* buffer, Py_ssize_t n_bytes, int flags) + + Copy the Python integer value *pylong* to a native *buffer* of size + *n_bytes*. The *flags* can be set to ``-1`` to behave similarly to a C cast, + or to values documented below to control the behavior. + + Returns ``-1`` with an exception raised on error. This may happen if + *pylong* cannot be interpreted as an integer, or if *pylong* was negative + and the ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` flag was set. + + Otherwise, returns the number of bytes required to store the value. + If this is equal to or less than *n_bytes*, the entire value was copied. + All *n_bytes* of the buffer are written: large buffers are padded with + zeroes. + + If the returned value is greater than than *n_bytes*, the value was + truncated: as many of the lowest bits of the value as could fit are written, + and the higher bits are ignored. This matches the typical behavior + of a C-style downcast. + + .. note:: + + Overflow is not considered an error. If the returned value + is larger than *n_bytes*, most significant bits were discarded. + + ``0`` will never be returned. + + Values are always copied as two's-complement. + + Usage example:: + + int32_t value; + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, &value, sizeof(value), -1); + if (bytes < 0) { + // Failed. A Python exception was set with the reason. + return NULL; + } + else if (bytes <= (Py_ssize_t)sizeof(value)) { + // Success! + } + else { + // Overflow occurred, but 'value' contains the truncated + // lowest bits of pylong. + } + + Passing zero to *n_bytes* will return the size of a buffer that would + be large enough to hold the value. This may be larger than technically + necessary, but not unreasonably so. If *n_bytes=0*, *buffer* may be + ``NULL``. + + .. note:: + + Passing *n_bytes=0* to this function is not an accurate way to determine + the bit length of the value. + + To get at the entire Python value of an unknown size, the function can be + called twice: first to determine the buffer size, then to fill it:: + + // Ask how much space we need. + Py_ssize_t expected = PyLong_AsNativeBytes(pylong, NULL, 0, -1); + if (expected < 0) { + // Failed. A Python exception was set with the reason. + return NULL; + } + assert(expected != 0); // Impossible per the API definition. + uint8_t *bignum = malloc(expected); + if (!bignum) { + PyErr_SetString(PyExc_MemoryError, "bignum malloc failed."); + return NULL; + } + // Safely get the entire value. + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, bignum, expected, -1); + if (bytes < 0) { // Exception has been set. + free(bignum); + return NULL; + } + else if (bytes > expected) { // This should not be possible. + PyErr_SetString(PyExc_RuntimeError, + "Unexpected bignum truncation after a size check."); + free(bignum); + return NULL; + } + // The expected success given the above pre-check. + // ... use bignum ... + free(bignum); + + *flags* is either ``-1`` (``Py_ASNATIVEBYTES_DEFAULTS``) to select defaults + that behave most like a C cast, or a combination of the other flags in + the table below. + Note that ``-1`` cannot be combined with other flags. + + Currently, ``-1`` corresponds to + ``Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_UNSIGNED_BUFFER``. + + .. c:namespace:: NULL + + ============================================= ====== + Flag Value + ============================================= ====== + .. c:macro:: Py_ASNATIVEBYTES_DEFAULTS ``-1`` + .. c:macro:: Py_ASNATIVEBYTES_BIG_ENDIAN ``0`` + .. c:macro:: Py_ASNATIVEBYTES_LITTLE_ENDIAN ``1`` + .. c:macro:: Py_ASNATIVEBYTES_NATIVE_ENDIAN ``3`` + .. c:macro:: Py_ASNATIVEBYTES_UNSIGNED_BUFFER ``4`` + .. c:macro:: Py_ASNATIVEBYTES_REJECT_NEGATIVE ``8`` + .. c:macro:: Py_ASNATIVEBYTES_ALLOW_INDEX ``16`` + ============================================= ====== + + Specifying ``Py_ASNATIVEBYTES_NATIVE_ENDIAN`` will override any other endian + flags. Passing ``2`` is reserved. + + By default, sufficient buffer will be requested to include a sign bit. + For example, when converting 128 with *n_bytes=1*, the function will return + 2 (or more) in order to store a zero sign bit. + + If ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` is specified, a zero sign bit + will be omitted from size calculations. This allows, for example, 128 to fit + in a single-byte buffer. If the destination buffer is later treated as + signed, a positive input value may become negative. + Note that the flag does not affect handling of negative values: for those, + space for a sign bit is always requested. + + Specifying ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` causes an exception to be set + if *pylong* is negative. Without this flag, negative values will be copied + provided there is enough space for at least one sign bit, regardless of + whether ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` was specified. + + If ``Py_ASNATIVEBYTES_ALLOW_INDEX`` is specified and a non-integer value is + passed, its :meth:`~object.__index__` method will be called first. This may + result in Python code executing and other threads being allowed to run, which + could cause changes to other objects or values in use. When *flags* is + ``-1``, this option is not set, and non-integer values will raise + :exc:`TypeError`. + + .. note:: + + With the default *flags* (``-1``, or *UNSIGNED_BUFFER* without + *REJECT_NEGATIVE*), multiple Python integers can map to a single value + without overflow. For example, both ``255`` and ``-1`` fit a single-byte + buffer and set all its bits. + This matches typical C cast behavior. + + .. versionadded:: 3.13 + + +.. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) + + Get the sign of the integer object *obj*. + + On success, set *\*sign* to the integer sign (0, -1 or +1 for zero, negative or + positive integer, respectively) and return 0. + + On failure, return -1 with an exception set. This function always succeeds + if *obj* is a :c:type:`PyLongObject` or its subtype. + + .. versionadded:: 3.14 + + +.. c:function:: PyObject* PyLong_GetInfo(void) + + On success, return a read only :term:`named tuple`, that holds + information about Python's internal representation of integers. + See :data:`sys.int_info` for description of individual fields. + + On failure, return ``NULL`` with an exception set. + + .. versionadded:: 3.1 + + .. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op) Return 1 if *op* is compact, 0 otherwise. @@ -340,13 +601,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. a “fast path” for small integers. For compact values use :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a :c:func:`PyLong_As* ` function or - :c:func:`calling ` :meth:`int.to_bytes`. + :c:func:`PyLong_AsNativeBytes`. The speedup is expected to be negligible for most users. Exactly what values are considered compact is an implementation detail and is subject to change. + .. versionadded:: 3.12 + + .. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`, @@ -354,3 +618,5 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, the return value is undefined. + .. versionadded:: 3.12 + diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 489f1580a414b2..b9085ad3ec361d 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -15,7 +15,7 @@ Numeric values are stored with the least significant byte first. The module supports two versions of the data format: version 0 is the historical version, version 1 shares interned strings in the file, and upon -unmarshalling. Version 2 uses a binary format for floating point numbers. +unmarshalling. Version 2 uses a binary format for floating-point numbers. ``Py_MARSHAL_VERSION`` indicates the current file format (currently 2). diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index c05282ffc59521..aa2ef499bddaf3 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -41,10 +41,10 @@ buffers is performed on demand by the Python memory manager through the Python/C API functions listed in this document. .. index:: - single: malloc() - single: calloc() - single: realloc() - single: free() + single: malloc (C function) + single: calloc (C function) + single: realloc (C function) + single: free (C function) To avoid memory corruption, extension writers should never try to operate on Python objects with the functions exported by the C library: :c:func:`malloc`, @@ -102,30 +102,38 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. There is no hard -requirement to use the memory returned by the allocation functions belonging to -a given domain for only the purposes hinted by that domain (although this is the -recommended practice). For example, one could use the memory returned by -:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned -by :c:func:`PyObject_Malloc` for allocating memory for buffers. +table can be found at :ref:`here `. +The APIs used to allocate and free a block of memory must be from the same domain. +For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. The three allocation domains are: * Raw domain: intended for allocating memory for general-purpose memory buffers where the allocation *must* go to the system allocator or where the allocator can operate without the :term:`GIL`. The memory is requested directly - to the system. + from the system. See :ref:`Raw Memory Interface `. * "Mem" domain: intended for allocating memory for Python buffers and general-purpose memory buffers where the allocation must be performed with the :term:`GIL` held. The memory is taken from the Python private heap. + See :ref:`Memory Interface `. -* Object domain: intended for allocating memory belonging to Python objects. The - memory is taken from the Python private heap. +* Object domain: intended for allocating memory for Python objects. The + memory is taken from the Python private heap. See :ref:`Object allocators `. -When freeing memory previously allocated by the allocating functions belonging to a -given domain,the matching specific deallocating functions must be used. For example, -:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. +.. note:: + + The :term:`free-threaded ` build requires that only Python objects are allocated using the "object" domain + and that all Python objects are allocated using that domain. This differs from the prior Python versions, + where this was only a best practice and not a hard requirement. + + For example, buffers (non-Python objects) should be allocated using :c:func:`PyMem_Malloc`, + :c:func:`PyMem_RawMalloc`, or :c:func:`malloc`, but not :c:func:`PyObject_Malloc`. + + See :ref:`Memory Allocation APIs `. + + +.. _raw-memoryinterface: Raw Memory Interface ==================== @@ -299,6 +307,8 @@ versions and is therefore deprecated in extension modules. * ``PyMem_DEL(ptr)`` +.. _objectinterface: + Object allocators ================= @@ -734,7 +744,7 @@ The same code using the type-oriented function set:: return PyErr_NoMemory(); /* ...Do some I/O operation involving buf... */ res = PyBytes_FromString(buf); - PyMem_Del(buf); /* allocated with PyMem_New */ + PyMem_Free(buf); /* allocated with PyMem_New */ return res; Note that in the two examples above, the buffer is always manipulated via @@ -750,11 +760,11 @@ allocators operating on different heaps. :: ... PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ free(buf2); /* Right -- allocated via malloc() */ - free(buf1); /* Fatal -- should be PyMem_Del() */ + free(buf1); /* Fatal -- should be PyMem_Free() */ In addition to the functions aimed at handling raw memory blocks from the Python heap, objects in Python are allocated and released with :c:macro:`PyObject_New`, -:c:macro:`PyObject_NewVar` and :c:func:`PyObject_Del`. +:c:macro:`PyObject_NewVar` and :c:func:`PyObject_Free`. These will be explained in the next chapter on defining and implementing new object types in C. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 979b22261efa3b..f82a050ab75de0 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -37,16 +37,19 @@ Module Objects single: __package__ (module attribute) single: __loader__ (module attribute) - Return a new module object with the :attr:`__name__` attribute set to *name*. - The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and - :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set - to ``None``); the caller is responsible for providing a :attr:`__file__` - attribute. + Return a new module object with :attr:`module.__name__` set to *name*. + The module's :attr:`!__name__`, :attr:`~module.__doc__`, + :attr:`~module.__package__` and :attr:`~module.__loader__` attributes are + filled in (all but :attr:`!__name__` are set to ``None``). The caller is + responsible for setting a :attr:`~module.__file__` attribute. + + Return ``NULL`` with an exception set on error. .. versionadded:: 3.3 .. versionchanged:: 3.4 - :attr:`__package__` and :attr:`__loader__` are set to ``None``. + :attr:`~module.__package__` and :attr:`~module.__loader__` are now set to + ``None``. .. c:function:: PyObject* PyModule_New(const char *name) @@ -75,8 +78,9 @@ Module Objects single: __name__ (module attribute) single: SystemError (built-in exception) - Return *module*'s :attr:`__name__` value. If the module does not provide one, - or if it is not a string, :exc:`SystemError` is raised and ``NULL`` is returned. + Return *module*'s :attr:`~module.__name__` value. If the module does not + provide one, or if it is not a string, :exc:`SystemError` is raised and + ``NULL`` is returned. .. versionadded:: 3.3 @@ -106,8 +110,8 @@ Module Objects single: SystemError (built-in exception) Return the name of the file from which *module* was loaded using *module*'s - :attr:`__file__` attribute. If this is not defined, or if it is not a - unicode string, raise :exc:`SystemError` and return ``NULL``; otherwise return + :attr:`~module.__file__` attribute. If this is not defined, or if it is not a + string, raise :exc:`SystemError` and return ``NULL``; otherwise return a reference to a Unicode object. .. versionadded:: 3.2 @@ -265,6 +269,8 @@ of the following two module creation functions: API version *module_api_version*. If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_Create` @@ -338,7 +344,8 @@ The available slot types are: The *value* pointer of this slot must point to a function of the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) - :noindex: + :no-index-entry: + :no-contents-entry: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. @@ -373,7 +380,8 @@ The available slot types are: The signature of the function is: .. c:function:: int exec_module(PyObject* module) - :noindex: + :no-index-entry: + :no-contents-entry: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. @@ -411,6 +419,33 @@ The available slot types are: .. versionadded:: 3.12 +.. c:macro:: Py_mod_gil + + Specifies one of the following values: + + .. c:namespace:: NULL + + .. c:macro:: Py_MOD_GIL_USED + + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. + + .. c:macro:: Py_MOD_GIL_NOT_USED + + The module is safe to run without an active GIL. + + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :ref:`whatsnew313-free-threaded-cpython` for more detail. + + Multiple ``Py_mod_gil`` slots may not be specified in one module definition. + + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. + + .. versionadded:: 3.13 + See :PEP:`489` for more details on multi-phase initialization. Low-level module creation functions @@ -436,6 +471,8 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` @@ -486,7 +523,7 @@ state: On success, return ``0``. On error, raise an exception and return ``-1``. - Return ``NULL`` if *value* is ``NULL``. It must be called with an exception + Return ``-1`` if *value* is ``NULL``. It must be called with an exception raised in this case. Example usage:: @@ -518,6 +555,14 @@ state: Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in this case, since *obj* can be ``NULL``. + The number of different *name* strings passed to this function + should be kept small, usually by only using statically allocated strings + as *name*. + For names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_SetAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object. + .. versionadded:: 3.10 @@ -576,15 +621,23 @@ state: .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) Add an integer constant to *module* as *name*. This convenience function can be - used from the module's initialization function. Return ``-1`` on error, ``0`` on - success. + used from the module's initialization function. + Return ``-1`` with an exception set on error, ``0`` on success. + + This is a convenience function that calls :c:func:`PyLong_FromLong` and + :c:func:`PyModule_AddObjectRef`; see their documentation for details. .. c:function:: int PyModule_AddStringConstant(PyObject *module, const char *name, const char *value) Add a string constant to *module* as *name*. This convenience function can be used from the module's initialization function. The string *value* must be - ``NULL``-terminated. Return ``-1`` on error, ``0`` on success. + ``NULL``-terminated. + Return ``-1`` with an exception set on error, ``0`` on success. + + This is a convenience function that calls + :c:func:`PyUnicode_InternFromString` and :c:func:`PyModule_AddObjectRef`; + see their documentation for details. .. c:macro:: PyModule_AddIntMacro(module, macro) @@ -592,7 +645,7 @@ state: Add an int constant to *module*. The name and the value are taken from *macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int constant *AF_INET* with the value of *AF_INET* to *module*. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. c:macro:: PyModule_AddStringMacro(module, macro) @@ -605,10 +658,23 @@ state: The type object is finalized by calling internally :c:func:`PyType_Ready`. The name of the type object is taken from the last component of :c:member:`~PyTypeObject.tp_name` after dot. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.9 +.. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) + + Indicate that *module* does or does not support running without the global + interpreter lock (GIL), using one of the values from + :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization + function. If this function is not called during module initialization, the + import machinery assumes the module does not support running without the + GIL. This function is only available in Python builds configured with + :option:`--disable-gil`. + Return ``-1`` with an exception set on error, ``0`` on success. + + .. versionadded:: 3.13 + Module lookup ^^^^^^^^^^^^^ @@ -644,14 +710,14 @@ since multiple such modules can be created from a single definition. The caller must hold the GIL. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.3 .. c:function:: int PyState_RemoveModule(PyModuleDef *def) Removes the module object created from *def* from the interpreter state. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. The caller must hold the GIL. diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst new file mode 100644 index 00000000000000..51d866cfd47469 --- /dev/null +++ b/Doc/c-api/monitoring.rst @@ -0,0 +1,192 @@ +.. highlight:: c + +.. _c-api-monitoring: + +Monitoring C API +================ + +Added in version 3.13. + +An extension may need to interact with the event monitoring system. Subscribing +to events and registering callbacks can be done via the Python API exposed in +:mod:`sys.monitoring`. + +Generating Execution Events +=========================== + +The functions below make it possible for an extension to fire monitoring +events as it emulates the execution of Python code. Each of these functions +accepts a ``PyMonitoringState`` struct which contains concise information +about the activation state of events, as well as the event arguments, which +include a ``PyObject*`` representing the code object, the instruction offset +and sometimes additional, event-specific arguments (see :mod:`sys.monitoring` +for details about the signatures of the different event callbacks). +The ``codelike`` argument should be an instance of :class:`types.CodeType` +or of a type that emulates it. + +The VM disables tracing when firing an event, so there is no need for user +code to do that. + +Monitoring functions should not be called with an exception set, +except those listed below as working with the current exception. + +.. c:type:: PyMonitoringState + + Representation of the state of an event type. It is allocated by the user + while its contents are maintained by the monitoring API functions described below. + + +All of the functions below return 0 on success and -1 (with an exception set) on error. + +See :mod:`sys.monitoring` for descriptions of the events. + +.. c:function:: int PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_START`` event. + + +.. c:function:: int PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_RESUME`` event. + + +.. c:function:: int PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_YIELD`` event. + + +.. c:function:: int PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* callable, PyObject *arg0) + + Fire a ``CALL`` event. + + +.. c:function:: int PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, int lineno) + + Fire a ``LINE`` event. + + +.. c:function:: int PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``JUMP`` event. + + +.. c:function:: int PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH`` event. + + +.. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval) + + Fire a ``C_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_THROW`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``C_RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RERAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire an ``EXCEPTION_HANDLED`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_UNWIND`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) + + Fire a ``STOP_ITERATION`` event. If ``value`` is an instance of :exc:`StopIteration`, it is used. Otherwise, + a new :exc:`StopIteration` instance is created with ``value`` as its argument. + + +Managing the Monitoring State +----------------------------- + +Monitoring states can be managed with the help of monitoring scopes. A scope +would typically correspond to a python function. + +.. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) + + Enter a monitored scope. ``event_types`` is an array of the event IDs for + events that may be fired from the scope. For example, the ID of a ``PY_START`` + event is the value ``PY_MONITORING_EVENT_PY_START``, which is numerically equal + to the base-2 logarithm of ``sys.monitoring.events.PY_START``. + ``state_array`` is an array with a monitoring state entry for each event in + ``event_types``, it is allocated by the user but populated by + :c:func:`!PyMonitoring_EnterScope` with information about the activation state of + the event. The size of ``event_types`` (and hence also of ``state_array``) + is given in ``length``. + + The ``version`` argument is a pointer to a value which should be allocated + by the user together with ``state_array`` and initialized to 0, + and then set only by :c:func:`!PyMonitoring_EnterScope` itself. It allows this + function to determine whether event states have changed since the previous call, + and to return quickly if they have not. + + The scopes referred to here are lexical scopes: a function, class or method. + :c:func:`!PyMonitoring_EnterScope` should be called whenever the lexical scope is + entered. Scopes can be reentered, reusing the same *state_array* and *version*, + in situations like when emulating a recursive Python function. When a code-like's + execution is paused, such as when emulating a generator, the scope needs to + be exited and re-entered. + + The macros for *event_types* are: + + .. c:namespace:: NULL + + .. The table is here to make the docs searchable, and to allow automatic + links to the identifiers. + + ================================================== ===================================== + Macro Event + ================================================== ===================================== + .. c:macro:: PY_MONITORING_EVENT_BRANCH :monitoring-event:`BRANCH` + .. c:macro:: PY_MONITORING_EVENT_CALL :monitoring-event:`CALL` + .. c:macro:: PY_MONITORING_EVENT_C_RAISE :monitoring-event:`C_RAISE` + .. c:macro:: PY_MONITORING_EVENT_C_RETURN :monitoring-event:`C_RETURN` + .. c:macro:: PY_MONITORING_EVENT_EXCEPTION_HANDLED :monitoring-event:`EXCEPTION_HANDLED` + .. c:macro:: PY_MONITORING_EVENT_INSTRUCTION :monitoring-event:`INSTRUCTION` + .. c:macro:: PY_MONITORING_EVENT_JUMP :monitoring-event:`JUMP` + .. c:macro:: PY_MONITORING_EVENT_LINE :monitoring-event:`LINE` + .. c:macro:: PY_MONITORING_EVENT_PY_RESUME :monitoring-event:`PY_RESUME` + .. c:macro:: PY_MONITORING_EVENT_PY_RETURN :monitoring-event:`PY_RETURN` + .. c:macro:: PY_MONITORING_EVENT_PY_START :monitoring-event:`PY_START` + .. c:macro:: PY_MONITORING_EVENT_PY_THROW :monitoring-event:`PY_THROW` + .. c:macro:: PY_MONITORING_EVENT_PY_UNWIND :monitoring-event:`PY_UNWIND` + .. c:macro:: PY_MONITORING_EVENT_PY_YIELD :monitoring-event:`PY_YIELD` + .. c:macro:: PY_MONITORING_EVENT_RAISE :monitoring-event:`RAISE` + .. c:macro:: PY_MONITORING_EVENT_RERAISE :monitoring-event:`RERAISE` + .. c:macro:: PY_MONITORING_EVENT_STOP_ITERATION :monitoring-event:`STOP_ITERATION` + ================================================== ===================================== + +.. c:function:: int PyMonitoring_ExitScope(void) + + Exit the last scope that was entered with :c:func:`!PyMonitoring_EnterScope`. diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 13d3c5af956905..ad8b5935258fa7 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -51,8 +51,8 @@ Number Protocol Return a reasonable approximation for the mathematical value of *o1* divided by *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when + floating-point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating-point value when passed two integers. This is the equivalent of the Python expression ``o1 / o2``. @@ -177,8 +177,8 @@ Number Protocol Return a reasonable approximation for the mathematical value of *o1* divided by *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when + floating-point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating-point value when passed two integers. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 /= o2``. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 12476412799a4f..630114a4339110 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -6,6 +6,56 @@ Object Protocol =============== +.. c:function:: PyObject* Py_GetConstant(unsigned int constant_id) + + Get a :term:`strong reference` to a constant. + + Set an exception and return ``NULL`` if *constant_id* is invalid. + + *constant_id* must be one of these constant identifiers: + + .. c:namespace:: NULL + + ======================================== ===== ========================= + Constant Identifier Value Returned object + ======================================== ===== ========================= + .. c:macro:: Py_CONSTANT_NONE ``0`` :py:data:`None` + .. c:macro:: Py_CONSTANT_FALSE ``1`` :py:data:`False` + .. c:macro:: Py_CONSTANT_TRUE ``2`` :py:data:`True` + .. c:macro:: Py_CONSTANT_ELLIPSIS ``3`` :py:data:`Ellipsis` + .. c:macro:: Py_CONSTANT_NOT_IMPLEMENTED ``4`` :py:data:`NotImplemented` + .. c:macro:: Py_CONSTANT_ZERO ``5`` ``0`` + .. c:macro:: Py_CONSTANT_ONE ``6`` ``1`` + .. c:macro:: Py_CONSTANT_EMPTY_STR ``7`` ``''`` + .. c:macro:: Py_CONSTANT_EMPTY_BYTES ``8`` ``b''`` + .. c:macro:: Py_CONSTANT_EMPTY_TUPLE ``9`` ``()`` + ======================================== ===== ========================= + + Numeric values are only given for projects which cannot use the constant + identifiers. + + + .. versionadded:: 3.13 + + .. impl-detail:: + + In CPython, all of these constants are :term:`immortal`. + + +.. c:function:: PyObject* Py_GetConstantBorrowed(unsigned int constant_id) + + Similar to :c:func:`Py_GetConstant`, but return a :term:`borrowed + reference`. + + This function is primarily intended for backwards compatibility: + using :c:func:`Py_GetConstant` is recommended for new code. + + The reference is borrowed from the interpreter, and is valid until the + interpreter finalization. + + .. versionadded:: 3.13 + + .. c:var:: PyObject* Py_NotImplemented The ``NotImplemented`` singleton, used to signal that an operation is @@ -16,7 +66,7 @@ Object Protocol Properly handle returning :c:data:`Py_NotImplemented` from within a C function (that is, create a new :term:`strong reference` - to NotImplemented and return it). + to :const:`NotImplemented` and return it). .. c:macro:: Py_PRINT_RAW @@ -156,6 +206,13 @@ Object Protocol If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. + The number of different attribute names passed to this function + should be kept small, usually by using a statically allocated string + as *attr_name*. + For attribute names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_SetAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object. .. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) @@ -181,6 +238,14 @@ Object Protocol specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. + The number of different attribute names passed to this function + should be kept small, usually by using a statically allocated string + as *attr_name*. + For attribute names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_DelAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object for lookup. + .. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context) @@ -302,14 +367,14 @@ Object Protocol The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. - If *cls* has a :meth:`~class.__subclasscheck__` method, it will be called to + If *cls* has a :meth:`~type.__subclasscheck__` method, it will be called to determine the subclass status as described in :pep:`3119`. Otherwise, *derived* is a subclass of *cls* if it is a direct or indirect subclass, - i.e. contained in ``cls.__mro__``. + i.e. contained in :attr:`cls.__mro__ `. Normally only class objects, i.e. instances of :class:`type` or a derived class, are considered classes. However, objects can override this by having - a :attr:`~class.__bases__` attribute (which must be a tuple of base classes). + a :attr:`~type.__bases__` attribute (which must be a tuple of base classes). .. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) @@ -321,15 +386,15 @@ Object Protocol The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. - If *cls* has a :meth:`~class.__instancecheck__` method, it will be called to + If *cls* has a :meth:`~type.__instancecheck__` method, it will be called to determine the subclass status as described in :pep:`3119`. Otherwise, *inst* is an instance of *cls* if its class is a subclass of *cls*. An instance *inst* can override what is considered its class by having a - :attr:`~instance.__class__` attribute. + :attr:`~object.__class__` attribute. An object *cls* can override if it is considered a class, and what its base - classes are, by having a :attr:`~class.__bases__` attribute (which must be a tuple + classes are, by having a :attr:`~type.__bases__` attribute (which must be a tuple of base classes). diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 75e1d46474f1e7..d75dad737bc992 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -23,12 +23,12 @@ of Python objects. Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. - .. versionchanged:: 3.11 - The parameter type is no longer :c:expr:`const PyObject*`. - .. versionchanged:: 3.10 :c:func:`Py_REFCNT()` is changed to the inline static function. + .. versionchanged:: 3.11 + The parameter type is no longer :c:expr:`const PyObject*`. + .. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt) @@ -62,7 +62,7 @@ of Python objects. ``NULL``, use :c:func:`Py_XINCREF`. Do not expect this function to actually modify *o* in any way. - For at least `some objects `_, + For at least :pep:`some objects <0683>`, this function has no effect. .. versionchanged:: 3.12 @@ -130,7 +130,7 @@ of Python objects. use :c:func:`Py_XDECREF`. Do not expect this function to actually modify *o* in any way. - For at least `some objects `_, + For at least :pep:`some objects <683>`, this function has no effect. .. warning:: diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 4b1c4770848a30..038e6977104560 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -7,18 +7,48 @@ Reflection .. c:function:: PyObject* PyEval_GetBuiltins(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameBuiltins` instead. + Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing. .. c:function:: PyObject* PyEval_GetLocals(void) - Return a dictionary of the local variables in the current execution frame, + .. deprecated:: 3.13 + + Use either :c:func:`PyEval_GetFrameLocals` to obtain the same behaviour as calling + :func:`locals` in Python code, or else call :c:func:`PyFrame_GetLocals` on the result + of :c:func:`PyEval_GetFrame` to access the :attr:`~frame.f_locals` attribute of the + currently executing frame. + + Return a mapping providing access to the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. + Refer to :func:`locals` for details of the mapping returned at different scopes. + + As this function returns a :term:`borrowed reference`, the dictionary returned for + :term:`optimized scopes ` is cached on the frame object and will remain + alive as long as the frame object does. Unlike :c:func:`PyEval_GetFrameLocals` and + :func:`locals`, subsequent calls to this function in the same frame will update the + contents of the cached dictionary to reflect changes in the state of the local variables + rather than returning a new snapshot. + + .. versionchanged:: 3.13 + As part of :pep:`667`, :c:func:`PyFrame_GetLocals`, :func:`locals`, and + :attr:`FrameType.f_locals ` no longer make use of the shared cache + dictionary. Refer to the :ref:`What's New entry ` for + additional details. + .. c:function:: PyObject* PyEval_GetGlobals(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameGlobals` instead. + Return a dictionary of the global variables in the current execution frame, or ``NULL`` if no frame is currently executing. @@ -31,6 +61,36 @@ Reflection See also :c:func:`PyThreadState_GetFrame`. +.. c:function:: PyObject* PyEval_GetFrameBuiltins(void) + + Return a dictionary of the builtins in the current execution frame, + or the interpreter of the thread state if no frame is currently executing. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameLocals(void) + + Return a dictionary of the local variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`locals` in Python code. + + To access :attr:`~frame.f_locals` on the current frame without making an independent + snapshot in :term:`optimized scopes `, call :c:func:`PyFrame_GetLocals` + on the result of :c:func:`PyEval_GetFrame`. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameGlobals(void) + + Return a dictionary of the global variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`globals` in Python code. + + .. versionadded:: 3.13 + + .. c:function:: const char* PyEval_GetFuncName(PyObject *func) Return the name of *func* if it is a function, class or instance object, else the diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 27a1757c745d8b..8adf6a961378a3 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -23,7 +23,9 @@ Slice Objects Return a new slice object with the given values. The *start*, *stop*, and *step* parameters are used as the values of the slice object attributes of the same names. Any of the values may be ``NULL``, in which case the - ``None`` will be used for the corresponding attribute. Return ``NULL`` if + ``None`` will be used for the corresponding attribute. + + Return ``NULL`` with an exception set if the new object could not be allocated. @@ -52,7 +54,7 @@ Slice Objects of bounds indices are clipped in a manner consistent with the handling of normal slices. - Returns ``0`` on success and ``-1`` on error with exception set. + Return ``0`` on success and ``-1`` on error with an exception set. .. note:: This function is considered not safe for resizable sequences. @@ -95,7 +97,7 @@ Slice Objects ``PY_SSIZE_T_MIN`` to ``PY_SSIZE_T_MIN``, and silently boost the step values less than ``-PY_SSIZE_T_MAX`` to ``-PY_SSIZE_T_MAX``. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.6.1 diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 0032da9659636c..d333df397782e0 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -187,26 +187,26 @@ Implementing functions and methods PyObject *kwargs); -.. c:type:: _PyCFunctionFast +.. c:type:: PyCFunctionFast Type of the functions used to implement Python callables in C with signature :c:macro:`METH_FASTCALL`. The function signature is:: - PyObject *_PyCFunctionFast(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs); + PyObject *PyCFunctionFast(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs); -.. c:type:: _PyCFunctionFastWithKeywords +.. c:type:: PyCFunctionFastWithKeywords Type of the functions used to implement Python callables in C with signature :ref:`METH_FASTCALL | METH_KEYWORDS `. The function signature is:: - PyObject *_PyCFunctionFastWithKeywords(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames); + PyObject *PyCFunctionFastWithKeywords(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames); .. c:type:: PyCMethod @@ -290,7 +290,7 @@ There are these calling conventions: .. c:macro:: METH_FASTCALL Fast calling convention supporting only positional arguments. - The methods have the type :c:type:`_PyCFunctionFast`. + The methods have the type :c:type:`PyCFunctionFast`. The first parameter is *self*, the second parameter is a C array of :c:expr:`PyObject*` values indicating the arguments and the third parameter is the number of arguments (the length of the array). @@ -306,7 +306,7 @@ There are these calling conventions: :c:expr:`METH_FASTCALL | METH_KEYWORDS` Extension of :c:macro:`METH_FASTCALL` supporting also keyword arguments, - with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + with methods of type :c:type:`PyCFunctionFastWithKeywords`. Keyword arguments are passed the same way as in the :ref:`vectorcall protocol `: there is an additional fourth :c:expr:`PyObject*` parameter @@ -485,7 +485,8 @@ Accessing attributes of extension types ``PyMemberDef`` may contain a definition for the special member ``"__vectorcalloffset__"``, corresponding to :c:member:`~PyTypeObject.tp_vectorcall_offset` in type objects. - These must be defined with ``Py_T_PYSSIZET`` and ``Py_READONLY``, for example:: + This member must be defined with ``Py_T_PYSSIZET``, and either + ``Py_READONLY`` or ``Py_READONLY | Py_RELATIVE_OFFSET``. For example:: static PyMemberDef spam_type_members[] = { {"__vectorcalloffset__", Py_T_PYSSIZET, @@ -506,6 +507,12 @@ Accessing attributes of extension types ``PyMemberDef`` is always available. Previously, it required including ``"structmember.h"``. + .. versionchanged:: 3.14 + + :c:macro:`Py_RELATIVE_OFFSET` is now allowed for + ``"__vectorcalloffset__"``, ``"__dictoffset__"`` and + ``"__weaklistoffset__"``. + .. c:function:: PyObject* PyMember_GetOne(const char *obj_addr, struct PyMemberDef *m) Get an attribute belonging to the object at address *obj_addr*. The @@ -561,9 +568,9 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: - single: READ_RESTRICTED - single: WRITE_RESTRICTED - single: RESTRICTED + single: READ_RESTRICTED (C macro) + single: WRITE_RESTRICTED (C macro) + single: RESTRICTED (C macro) .. versionchanged:: 3.10 @@ -574,7 +581,7 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: :c:macro:`Py_AUDIT_READ`; :c:macro:`!WRITE_RESTRICTED` does nothing. .. index:: - single: READONLY + single: READONLY (C macro) .. versionchanged:: 3.12 @@ -637,24 +644,24 @@ Macro name C type Python type Reading a ``NULL`` pointer raises :py:exc:`AttributeError`. .. index:: - single: T_BYTE - single: T_SHORT - single: T_INT - single: T_LONG - single: T_LONGLONG - single: T_UBYTE - single: T_USHORT - single: T_UINT - single: T_ULONG - single: T_ULONGULONG - single: T_PYSSIZET - single: T_FLOAT - single: T_DOUBLE - single: T_BOOL - single: T_CHAR - single: T_STRING - single: T_STRING_INPLACE - single: T_OBJECT_EX + single: T_BYTE (C macro) + single: T_SHORT (C macro) + single: T_INT (C macro) + single: T_LONG (C macro) + single: T_LONGLONG (C macro) + single: T_UBYTE (C macro) + single: T_USHORT (C macro) + single: T_UINT (C macro) + single: T_ULONG (C macro) + single: T_ULONGULONG (C macro) + single: T_PYSSIZET (C macro) + single: T_FLOAT (C macro) + single: T_DOUBLE (C macro) + single: T_BOOL (C macro) + single: T_CHAR (C macro) + single: T_STRING (C macro) + single: T_STRING_INPLACE (C macro) + single: T_OBJECT_EX (C macro) single: structmember.h .. versionadded:: 3.12 @@ -702,12 +709,12 @@ Defining Getters and Setters .. c:member:: void* closure - Optional function pointer, providing additional data for getter and setter. + Optional user data pointer, providing additional data for getter and setter. .. c:type:: PyObject *(*getter)(PyObject *, void *) The ``get`` function takes one :c:expr:`PyObject*` parameter (the - instance) and a function pointer (the associated ``closure``): + instance) and a user data pointer (the associated ``closure``): It should return a new reference on success or ``NULL`` with a set exception on failure. @@ -715,7 +722,7 @@ Defining Getters and Setters .. c:type:: int (*setter)(PyObject *, PyObject *, void *) ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and - the value to be set) and a function pointer (the associated ``closure``): + the value to be set) and a user data pointer (the associated ``closure``): In case the attribute should be deleted the second parameter is ``NULL``. Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index e3c54b075114ff..d6fca1a0b0a219 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -5,6 +5,7 @@ Operating System Utilities ========================== + .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a @@ -97,27 +98,30 @@ Operating System Utilities .. c:function:: int PyOS_CheckStack() + .. index:: single: USE_STACKCHECK (C macro) + Return true when the interpreter runs out of stack space. This is a reliable - check, but is only available when :c:macro:`USE_STACKCHECK` is defined (currently + check, but is only available when :c:macro:`!USE_STACKCHECK` is defined (currently on certain versions of Windows using the Microsoft Visual C++ compiler). - :c:macro:`USE_STACKCHECK` will be defined automatically; you should never + :c:macro:`!USE_STACKCHECK` will be defined automatically; you should never change the definition in your own code. +.. c:type:: void (*PyOS_sighandler_t)(int) + + .. c:function:: PyOS_sighandler_t PyOS_getsig(int i) Return the current signal handler for signal *i*. This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions - directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void - (\*)(int)`. + directly! .. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) Set the signal handler for signal *i* to be *h*; return the old signal handler. This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do - not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef - alias for :c:expr:`void (\*)(int)`. + not call those functions directly! .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) @@ -342,10 +346,8 @@ accessible to C code. They all work with the current interpreter thread's silently abort the operation by raising an error subclassed from :class:`Exception` (other errors will not be silenced). - The hook function is of type :c:expr:`int (*)(const char *event, PyObject - *args, void *userData)`, where *args* is guaranteed to be a - :c:type:`PyTupleObject`. The hook function is always called with the GIL - held by the Python interpreter that raised the event. + The hook function is always called with the GIL held by the Python + interpreter that raised the event. See :pep:`578` for a detailed description of auditing. Functions in the runtime and standard library that raise events are listed in the @@ -354,12 +356,21 @@ accessible to C code. They all work with the current interpreter thread's .. audit-event:: sys.addaudithook "" c.PySys_AddAuditHook - If the interpreter is initialized, this function raises a auditing event + If the interpreter is initialized, this function raises an auditing event ``sys.addaudithook`` with no arguments. If any existing hooks raise an exception derived from :class:`Exception`, the new hook will not be added and the exception is cleared. As a result, callers cannot assume that their hook has been added unless they control all existing hooks. + .. c:namespace:: NULL + .. c:type:: int (*Py_AuditHookFunction) (const char *event, PyObject *args, void *userData) + + The type of the hook function. + *event* is the C string event argument passed to :c:func:`PySys_Audit` or + :c:func:`PySys_AuditTuple`. + *args* is guaranteed to be a :c:type:`PyTupleObject`. + *userData* is the argument passed to PySys_AddAuditHook(). + .. versionadded:: 3.8 @@ -371,7 +382,7 @@ Process Control .. c:function:: void Py_FatalError(const char *message) - .. index:: single: abort() + .. index:: single: abort (C function) Print a fatal error message and kill the process. No cleanup is performed. This function should only be invoked when a condition is detected that would @@ -391,8 +402,8 @@ Process Control .. c:function:: void Py_Exit(int status) .. index:: - single: Py_FinalizeEx() - single: exit() + single: Py_FinalizeEx (C function) + single: exit (C function) Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` @@ -405,7 +416,7 @@ Process Control .. c:function:: int Py_AtExit(void (*func) ()) .. index:: - single: Py_FinalizeEx() + single: Py_FinalizeEx (C function) single: cleanup functions Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup diff --git a/Doc/c-api/time.rst b/Doc/c-api/time.rst new file mode 100644 index 00000000000000..7032cc48aa6913 --- /dev/null +++ b/Doc/c-api/time.rst @@ -0,0 +1,114 @@ +.. highlight:: c + +.. _c-api-time: + +PyTime C API +============ + +.. versionadded:: 3.13 + +The clock C API provides access to system clocks. +It is similar to the Python :mod:`time` module. + +For C API related to the :mod:`datetime` module, see :ref:`datetimeobjects`. + + +Types +----- + +.. c:type:: PyTime_t + + A timestamp or duration in nanoseconds, represented as a signed 64-bit + integer. + + The reference point for timestamps depends on the clock used. For example, + :c:func:`PyTime_Time` returns timestamps relative to the UNIX epoch. + + The supported range is around [-292.3 years; +292.3 years]. + Using the Unix epoch (January 1st, 1970) as reference, the supported date + range is around [1677-09-21; 2262-04-11]. + The exact limits are exposed as constants: + +.. c:var:: PyTime_t PyTime_MIN + + Minimum value of :c:type:`PyTime_t`. + +.. c:var:: PyTime_t PyTime_MAX + + Maximum value of :c:type:`PyTime_t`. + + +Clock Functions +--------------- + +The following functions take a pointer to a :c:expr:`PyTime_t` that they +set to the value of a particular clock. +Details of each clock are given in the documentation of the corresponding +Python function. + +The functions return ``0`` on success, or ``-1`` (with an exception set) +on failure. + +On integer overflow, they set the :c:data:`PyExc_OverflowError` exception and +set ``*result`` to the value clamped to the ``[PyTime_MIN; PyTime_MAX]`` +range. +(On current systems, integer overflows are likely caused by misconfigured +system time.) + +As any other C API (unless otherwise specified), the functions must be called +with the :term:`GIL` held. + +.. c:function:: int PyTime_Monotonic(PyTime_t *result) + + Read the monotonic clock. + See :func:`time.monotonic` for important details on this clock. + +.. c:function:: int PyTime_PerfCounter(PyTime_t *result) + + Read the performance counter. + See :func:`time.perf_counter` for important details on this clock. + +.. c:function:: int PyTime_Time(PyTime_t *result) + + Read the “wall clock” time. + See :func:`time.time` for details important on this clock. + + +Raw Clock Functions +------------------- + +Similar to clock functions, but don't set an exception on error and don't +require the caller to hold the GIL. + +On success, the functions return ``0``. + +On failure, they set ``*result`` to ``0`` and return ``-1``, *without* setting +an exception. To get the cause of the error, acquire the GIL and call the +regular (non-``Raw``) function. Note that the regular function may succeed after +the ``Raw`` one failed. + +.. c:function:: int PyTime_MonotonicRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Monotonic`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_PerfCounterRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_PerfCounter`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_TimeRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Time`, + but don't set an exception on error and don't require holding the GIL. + + +Conversion functions +-------------------- + +.. c:function:: double PyTime_AsSecondsDouble(PyTime_t t) + + Convert a timestamp to a number of seconds as a C :c:expr:`double`. + + The function cannot fail, but note that :c:expr:`double` has limited + accuracy for large values. diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index b3710560ebe7ac..815afddad19df1 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -33,12 +33,14 @@ Tuple Objects .. c:function:: PyObject* PyTuple_New(Py_ssize_t len) - Return a new tuple object of size *len*, or ``NULL`` on failure. + Return a new tuple object of size *len*, + or ``NULL`` with an exception set on failure. .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) - Return a new tuple object of size *n*, or ``NULL`` on failure. The tuple values + Return a new tuple object of size *n*, + or ``NULL`` with an exception set on failure. The tuple values are initialized to the subsequent *n* C arguments pointing to Python objects. ``PyTuple_Pack(2, a, b)`` is equivalent to ``Py_BuildValue("(OO)", a, b)``. @@ -46,12 +48,12 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. + On error, return ``-1`` and with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) - Return the size of the tuple *p*, which must be non-``NULL`` and point to a tuple; - no error checking is performed. + Like :c:func:`PyTuple_Size`, but without error checking. .. c:function:: PyObject* PyTuple_GetItem(PyObject *p, Py_ssize_t pos) @@ -59,6 +61,12 @@ Tuple Objects Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is negative or out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. + The returned reference is borrowed from the tuple *p* + (that is: it is only valid as long as you hold a reference to *p*). + To get a :term:`strong reference`, use + :c:func:`Py_NewRef(PyTuple_GetItem(...)) ` + or :c:func:`PySequence_GetItem`. + .. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) @@ -68,8 +76,10 @@ Tuple Objects .. c:function:: PyObject* PyTuple_GetSlice(PyObject *p, Py_ssize_t low, Py_ssize_t high) Return the slice of the tuple pointed to by *p* between *low* and *high*, - or ``NULL`` on failure. This is the equivalent of the Python expression - ``p[low:high]``. Indexing from the end of the tuple is not supported. + or ``NULL`` with an exception set on failure. + + This is the equivalent of the Python expression ``p[low:high]``. + Indexing from the end of the tuple is not supported. .. c:function:: int PyTuple_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) @@ -99,6 +109,12 @@ Tuple Objects is being replaced; any reference in the tuple at position *pos* will be leaked. + .. warning:: + + This macro should *only* be used on tuples that are newly created. + Using this macro on a tuple that is already in use (or in other words, has + a refcount > 1) could lead to undefined behavior. + .. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) @@ -129,6 +145,8 @@ type. Create a new struct sequence type from the data in *desc*, described below. Instances of the resulting type can be created with :c:func:`PyStructSequence_New`. + Return ``NULL`` with an exception set on failure. + .. c:function:: void PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) @@ -137,8 +155,8 @@ type. .. c:function:: int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) - The same as ``PyStructSequence_InitType``, but returns ``0`` on success and ``-1`` on - failure. + Like :c:func:`PyStructSequence_InitType`, but returns ``0`` on success + and ``-1`` with an exception set on failure. .. versionadded:: 3.4 @@ -149,7 +167,8 @@ type. .. c:member:: const char *name - Name of the struct sequence type. + Fully qualified name of the type; null-terminated UTF-8 encoded. + The name must contain the module name. .. c:member:: const char *doc @@ -195,6 +214,8 @@ type. Creates an instance of *type*, which must have been created with :c:func:`PyStructSequence_NewType`. + Return ``NULL`` with an exception set on failure. + .. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 5aaa8147dd3176..86d3967d9fb577 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -53,7 +53,8 @@ Type Objects .. c:function:: PyObject* PyType_GetDict(PyTypeObject* type) Return the type object's internal namespace, which is otherwise only - exposed via a read-only proxy (``cls.__dict__``). This is a + exposed via a read-only proxy (:attr:`cls.__dict__ `). + This is a replacement for accessing :c:member:`~PyTypeObject.tp_dict` directly. The returned dictionary must be treated as read-only. @@ -140,7 +141,7 @@ Type Objects Return true if *a* is a subtype of *b*. This function only checks for actual subtypes, which means that - :meth:`~class.__subclasscheck__` is not called on *b*. Call + :meth:`~type.__subclasscheck__` is not called on *b*. Call :c:func:`PyObject_IsSubclass` to do the same check that :func:`issubclass` would do. @@ -174,17 +175,33 @@ Type Objects .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) - Return the type's name. Equivalent to getting the type's ``__name__`` attribute. + Return the type's name. Equivalent to getting the type's + :attr:`~type.__name__` attribute. .. versionadded:: 3.11 .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the - type's ``__qualname__`` attribute. + type's :attr:`~type.__qualname__` attribute. .. versionadded:: 3.11 +.. c:function:: PyObject* PyType_GetFullyQualifiedName(PyTypeObject *type) + + Return the type's fully qualified name. Equivalent to + ``f"{type.__module__}.{type.__qualname__}"``, or :attr:`type.__qualname__` + if :attr:`type.__module__` is not a string or is equal to ``"builtins"``. + + .. versionadded:: 3.13 + +.. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) + + Return the type's module name. Equivalent to getting the + :attr:`type.__module__` attribute. + + .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -249,6 +266,24 @@ Type Objects .. versionadded:: 3.11 +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) + + Find the first superclass in *type*'s :term:`method resolution order` whose + :c:macro:`Py_tp_token` token is equal to the given one. + + * If found, set *\*result* to a new :term:`strong reference` + to it and return ``1``. + * If not found, set *\*result* to ``NULL`` and return ``0``. + * On error, set *\*result* to ``NULL`` and return ``-1`` with an + exception set. + + The *result* argument may be ``NULL``, in which case *\*result* is not set. + Use this if you need only the return value. + + The *token* argument may not be ``NULL``. + + .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -330,8 +365,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -347,8 +386,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) @@ -363,8 +406,26 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. + +.. c:function:: int PyType_Freeze(PyTypeObject *type) + + Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. + + All base classes of *type* must be immutable. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + The type must not be used before it's made immutable. For example, type + instances must not be created before the type is made immutable. + + .. versionadded:: 3.14 .. raw:: html @@ -461,6 +522,11 @@ The following functions and structs are used to create * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + An additional slot is supported that does not correspond to a + :c:type:`!PyTypeObject` struct field: + + * :c:data:`Py_tp_token` + The following “offset” fields cannot be set using :c:type:`PyType_Slot`: * :c:member:`~PyTypeObject.tp_weaklistoffset` @@ -477,14 +543,10 @@ The following functions and structs are used to create See :ref:`PyMemberDef documentation ` for details. - The following fields cannot be set at all when creating a heap type: - - * :c:member:`~PyTypeObject.tp_vectorcall` - (use :c:member:`~PyTypeObject.tp_new` and/or - :c:member:`~PyTypeObject.tp_init`) + The following internal fields cannot be set at all when creating a heap + type: - * Internal fields: - :c:member:`~PyTypeObject.tp_dict`, + * :c:member:`~PyTypeObject.tp_dict`, :c:member:`~PyTypeObject.tp_mro`, :c:member:`~PyTypeObject.tp_cache`, :c:member:`~PyTypeObject.tp_subclasses`, and @@ -504,9 +566,58 @@ The following functions and structs are used to create :c:member:`~PyBufferProcs.bf_releasebuffer` are now available under the :ref:`limited API `. + .. versionchanged:: 3.14 + + The field :c:member:`~PyTypeObject.tp_vectorcall` can now set + using ``Py_tp_vectorcall``. See the field's documentation + for details. + .. c:member:: void *pfunc The desired value of the slot. In most cases, this is a pointer to a function. - Slots other than ``Py_tp_doc`` may not be ``NULL``. + *pfunc* values may not be ``NULL``, except for the following slots: + + * ``Py_tp_doc`` + * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` + rather than ``NULL``) + +.. c:macro:: Py_tp_token + + A :c:member:`~PyType_Slot.slot` that records a static memory layout ID + for a class. + + If the :c:type:`PyType_Spec` of the class is statically + allocated, the token can be set to the spec using the special value + :c:data:`Py_TP_USE_SPEC`: + + .. code-block:: c + + static PyType_Slot foo_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + + It can also be set to an arbitrary pointer, but you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + + Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has + a given token -- that is, check whether the memory layout is compatible. + + To get the token for a given class (without considering superclasses), + use :c:func:`PyType_GetSlot` with ``Py_tp_token``. + + .. versionadded:: 3.14 + + .. c:namespace:: NULL + + .. c:macro:: Py_TP_USE_SPEC + + Used as a value with :c:data:`Py_tp_token` to set the token to the + class's :c:type:`PyType_Spec`. + Expands to ``NULL``. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 8a26f237652d12..8a185486fe44f1 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -355,7 +355,7 @@ slot typedefs +-----------------------------+-----------------------------+----------------------+ | :c:type:`newfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | -| | :c:type:`PyObject` * | | +| | :c:type:`PyTypeObject` * | | | | :c:type:`PyObject` * | | | | :c:type:`PyObject` * | | +-----------------------------+-----------------------------+----------------------+ @@ -567,12 +567,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) For :ref:`statically allocated type objects `, the *tp_name* field should contain a dot. - Everything before the last dot is made accessible as the :attr:`__module__` + Everything before the last dot is made accessible as the :attr:`~type.__module__` attribute, and everything after the last dot is made accessible as the - :attr:`~definition.__name__` attribute. + :attr:`~type.__name__` attribute. If no dot is present, the entire :c:member:`~PyTypeObject.tp_name` field is made accessible as the - :attr:`~definition.__name__` attribute, and the :attr:`__module__` attribute is undefined + :attr:`~type.__name__` attribute, and the :attr:`~type.__module__` attribute is undefined (unless explicitly set in the dictionary, as explained above). This means your type will be impossible to pickle. Additionally, it will not be listed in module documentations created with pydoc. @@ -650,7 +650,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is permissible to call the object deallocator directly instead of via :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the - instance; this is normally :c:func:`PyObject_Del` if the instance was allocated + instance; this is normally :c:func:`PyObject_Free` if the instance was allocated using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or :c:func:`PyObject_GC_Del` if the instance was allocated using :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. @@ -682,6 +682,19 @@ and :c:data:`PyType_Type` effectively act as defaults.) Py_DECREF(tp); } + .. warning:: + + In a garbage collected Python, :c:member:`!tp_dealloc` may be called from + any Python thread, not just the thread which created the object (if the + object becomes part of a refcount cycle, that cycle might be collected by + a garbage collection on any thread). This is not a problem for Python + API calls, since the thread on which :c:member:`!tp_dealloc` is called + will own the Global Interpreter Lock (GIL). However, if the object being + destroyed in turn destroys objects from some other C or C++ library, care + should be taken to ensure that destroying those objects on the thread + which called :c:member:`!tp_dealloc` will not violate any assumptions of + the library. + **Inheritance:** @@ -883,6 +896,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash`, when the subtype's :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both ``NULL``. + **Default:** + + :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericHash`. + .. c:member:: ternaryfunc PyTypeObject.tp_call @@ -1030,7 +1047,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) the type, and the type object is INCREF'ed when a new instance is created, and DECREF'ed when an instance is destroyed (this does not apply to instances of subtypes; only the type referenced by the instance's ob_type gets INCREF'ed or - DECREF'ed). + DECREF'ed). Heap types should also :ref:`support garbage collection ` + as they can form a reference cycle with their own module object. **Inheritance:** @@ -1126,7 +1144,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a ``__dict__`` + This bit indicates that instances of the class have a `~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. @@ -1323,15 +1341,15 @@ and :c:data:`PyType_Type` effectively act as defaults.) To indicate that a class has changed call :c:func:`PyType_Modified` .. warning:: - This flag is present in header files, but is an internal feature and should - not be used. It will be removed in a future version of CPython + This flag is present in header files, but is not be used. + It will be removed in a future version of CPython .. c:member:: const char* PyTypeObject.tp_doc An optional pointer to a NUL-terminated C string giving the docstring for this - type object. This is exposed as the :attr:`__doc__` attribute on the type and - instances of the type. + type object. This is exposed as the :attr:`~type.__doc__` attribute on the + type and instances of the type. **Inheritance:** @@ -1376,7 +1394,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) Py_VISIT(Py_TYPE(self)); It is only needed since Python 3.9. To support Python 3.8 and older, this - line must be conditionnal:: + line must be conditional:: #if PY_VERSION_HEX >= 0x03090000 Py_VISIT(Py_TYPE(self)); @@ -1587,7 +1605,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) weak references to the type object itself. It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and - :c:member:`~PyTypeObject.tp_weaklist`. + :c:member:`~PyTypeObject.tp_weaklistoffset`. **Inheritance:** @@ -1599,7 +1617,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) **Default:** If the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit is set in the - :c:member:`~PyTypeObject.tp_dict` field, then + :c:member:`~PyTypeObject.tp_flags` field, then :c:member:`~PyTypeObject.tp_weaklistoffset` will be set to a negative value, to indicate that it is unsafe to use this field. @@ -1949,7 +1967,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) match :c:func:`PyType_GenericAlloc` and the value of the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. - For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Del`. + For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Free`. .. c:member:: inquiry PyTypeObject.tp_is_gc @@ -2031,7 +2049,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) A collection of subclasses. Internal use only. May be an invalid pointer. To get a list of subclasses, call the Python method - :py:meth:`~class.__subclasses__`. + :py:meth:`~type.__subclasses__`. .. versionchanged:: 3.12 @@ -2104,17 +2122,6 @@ and :c:data:`PyType_Type` effectively act as defaults.) PyErr_Restore(error_type, error_value, error_traceback); } - Also, note that, in a garbage collected Python, - :c:member:`~PyTypeObject.tp_dealloc` may be called from - any Python thread, not just the thread which created the object (if the object - becomes part of a refcount cycle, that cycle might be collected by a garbage - collection on any thread). This is not a problem for Python API calls, since - the thread on which tp_dealloc is called will own the Global Interpreter Lock - (GIL). However, if the object being destroyed in turn destroys objects from some - other C or C++ library, care should be taken to ensure that destroying those - objects on the thread which called tp_dealloc will not violate any assumptions - of the library. - **Inheritance:** This field is inherited by subtypes. @@ -2132,11 +2139,40 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall - Vectorcall function to use for calls of this type object. - In other words, it is used to implement - :ref:`vectorcall ` for ``type.__call__``. - If ``tp_vectorcall`` is ``NULL``, the default call implementation - using :meth:`~object.__new__` and :meth:`~object.__init__` is used. + A :ref:`vectorcall function ` to use for calls of this type + object (rather than instances). + In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, + which typically returns a new instance of *type*. + + As with any vectorcall function, if ``tp_vectorcall`` is ``NULL``, + the *tp_call* protocol (``Py_TYPE(type)->tp_call``) is used instead. + + .. note:: + + The :ref:`vectorcall protocol ` requires that the vectorcall + function has the same behavior as the corresponding ``tp_call``. + This means that ``type->tp_vectorcall`` must match the behavior of + ``Py_TYPE(type)->tp_call``. + + Specifically, if *type* uses the default metaclass, + ``type->tp_vectorcall`` must behave the same as + :c:expr:`PyType_Type->tp_call`, which: + + - calls ``type->tp_new``, + + - if the result is a subclass of *type*, calls ``type->tp_init`` + on the result of ``tp_new``, and + + - returns the result of ``tp_new``. + + Typically, ``tp_vectorcall`` is overridden to optimize this process + for specific :c:member:`~PyTypeObject.tp_new` and + :c:member:`~PyTypeObject.tp_init`. + When doing this for user-subclassable types, note that both can be + overridden (using :py:func:`~object.__new__` and + :py:func:`~object.__init__`, respectively). + + **Inheritance:** @@ -2611,7 +2647,7 @@ Slot Type typedefs See :c:member:`~PyTypeObject.tp_free`. -.. c:type:: PyObject *(*newfunc)(PyObject *, PyObject *, PyObject *) +.. c:type:: PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *) See :c:member:`~PyTypeObject.tp_new`. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 5541eaa521803b..59bd7661965d93 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -317,7 +317,7 @@ These APIs can be used to work with surrogates: .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) - Join two surrogate characters and return a single :c:type:`Py_UCS4` value. + Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must be in the range [0xDC00; 0xDFFF]. @@ -338,6 +338,8 @@ APIs: This is the recommended way to allocate a new Unicode object. Objects created using this function are not resizable. + On error, set an exception and return ``NULL``. + .. versionadded:: 3.3 @@ -518,6 +520,26 @@ APIs: - :c:expr:`PyObject*` - The result of calling :c:func:`PyObject_Repr`. + * - ``T`` + - :c:expr:`PyObject*` + - Get the fully qualified name of an object type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``#T`` + - :c:expr:`PyObject*` + - Similar to ``T`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + + * - ``N`` + - :c:expr:`PyTypeObject*` + - Get the fully qualified name of a type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``#N`` + - :c:expr:`PyTypeObject*` + - Similar to ``N`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + .. note:: The width formatter unit is number of characters rather than bytes. The precision formatter unit is number of bytes or :c:type:`wchar_t` @@ -553,6 +575,9 @@ APIs: In previous versions it caused all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. + .. versionchanged:: 3.13 + Support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats added. + .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) @@ -591,6 +616,8 @@ APIs: Return the length of the Unicode object, in code points. + On error, set an exception and return ``-1``. + .. versionadded:: 3.3 @@ -634,6 +661,8 @@ APIs: not out of bounds, and that the object can be modified safely (i.e. that it its reference count is one). + Return ``0`` on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -643,6 +672,8 @@ APIs: Unicode object and the index is not out of bounds, in contrast to :c:func:`PyUnicode_READ_CHAR`, which performs no error checking. + Return character on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -651,6 +682,7 @@ APIs: Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. + On error, set an exception and return ``NULL``. .. versionadded:: 3.3 @@ -854,7 +886,12 @@ wchar_t Support Copy the Unicode object contents into the :c:type:`wchar_t` buffer *wstr*. At most *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing null termination character). Return the number of :c:type:`wchar_t` characters - copied or ``-1`` in case of an error. Note that the resulting :c:expr:`wchar_t*` + copied or ``-1`` in case of an error. + + When *wstr* is ``NULL``, instead return the *size* that would be required + to store all of *unicode* including a terminating null. + + Note that the resulting :c:expr:`wchar_t*` string may or may not be null-terminated. It is the responsibility of the caller to make sure that the :c:expr:`wchar_t*` string is null-terminated in case this is required by the application. Also, note that the :c:expr:`wchar_t*` string @@ -962,6 +999,9 @@ These are the UTF-8 codec APIs: object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + .. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) @@ -974,6 +1014,9 @@ These are the UTF-8 codec APIs: On error, set an exception, set *size* to ``-1`` (if it's not NULL) and return ``NULL``. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not responsible for deallocating the buffer. The buffer is deallocated and @@ -1395,14 +1438,40 @@ They all return ``NULL`` or ``-1`` if an exception occurs. This function returns ``-1`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. + .. seealso:: + + The :c:func:`PyUnicode_Equal` function. + + +.. c:function:: int PyUnicode_Equal(PyObject *a, PyObject *b) + + Test if two strings are equal: + + * Return ``1`` if *a* is equal to *b*. + * Return ``0`` if *a* is not equal to *b*. + * Set a :exc:`TypeError` exception and return ``-1`` if *a* or *b* is not a + :class:`str` object. + + The function always succeeds if *a* and *b* are :class:`str` objects. + + The function works for :class:`str` subclasses, but does not honor custom + ``__eq__()`` method. + + .. seealso:: + + The :c:func:`PyUnicode_Compare` function. + + .. versionadded:: 3.14 + .. c:function:: int PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *string, Py_ssize_t size) Compare a Unicode object with a char buffer which is interpreted as being UTF-8 or ASCII encoded and return true (``1``) if they are equal, or false (``0``) otherwise. - If the Unicode object contains surrogate characters or - the C string is not valid UTF-8, false (``0``) is returned. + If the Unicode object contains surrogate code points + (``U+D800`` - ``U+DFFF``) or the C string is not valid UTF-8, + false (``0``) is returned. This function does not raise exceptions. @@ -1462,15 +1531,164 @@ They all return ``NULL`` or ``-1`` if an exception occurs. existing interned string that is the same as :c:expr:`*p_unicode`, it sets :c:expr:`*p_unicode` to it (releasing the reference to the old string object and creating a new :term:`strong reference` to the interned string object), otherwise it leaves - :c:expr:`*p_unicode` alone and interns it (creating a new :term:`strong reference`). + :c:expr:`*p_unicode` alone and interns it. + (Clarification: even though there is a lot of talk about references, think - of this function as reference-neutral; you own the object after the call - if and only if you owned it before the call.) + of this function as reference-neutral. You must own the object you pass in; + after the call you no longer own the passed-in reference, but you newly own + the result.) + + This function never raises an exception. + On error, it leaves its argument unchanged without interning it. + + Instances of subclasses of :py:class:`str` may not be interned, that is, + :c:expr:`PyUnicode_CheckExact(*p_unicode)` must be true. If it is not, + then -- as with any other error -- the argument is left unchanged. + + Note that interned strings are not “immortal”. + You must keep a reference to the result to benefit from interning. .. c:function:: PyObject* PyUnicode_InternFromString(const char *str) A combination of :c:func:`PyUnicode_FromString` and - :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string - object that has been interned, or a new ("owned") reference to an earlier - interned string object with the same value. + :c:func:`PyUnicode_InternInPlace`, meant for statically allocated strings. + + Return a new ("owned") reference to either a new Unicode string object + that has been interned, or an earlier interned string object with the + same value. + + Python may keep a reference to the result, or make it :term:`immortal`, + preventing it from being garbage-collected promptly. + For interning an unbounded number of different strings, such as ones coming + from user input, prefer calling :c:func:`PyUnicode_FromString` and + :c:func:`PyUnicode_InternInPlace` directly. + + .. impl-detail:: + + Strings interned this way are made :term:`immortal`. + + +PyUnicodeWriter +^^^^^^^^^^^^^^^ + +The :c:type:`PyUnicodeWriter` API can be used to create a Python :class:`str` +object. + +.. versionadded:: 3.14 + +.. c:type:: PyUnicodeWriter + + A Unicode writer instance. + + The instance must be destroyed by :c:func:`PyUnicodeWriter_Finish` on + success, or :c:func:`PyUnicodeWriter_Discard` on error. + +.. c:function:: PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) + + Create a Unicode writer instance. + + Set an exception and return ``NULL`` on error. + +.. c:function:: PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) + + Return the final Python :class:`str` object and destroy the writer instance. + + Set an exception and return ``NULL`` on error. + +.. c:function:: void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) + + Discard the internal Unicode buffer and destroy the writer instance. + + If *writer* is ``NULL``, no operation is performed. + +.. c:function:: int PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) + + Write the single Unicode character *ch* into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, const char *str, Py_ssize_t size) + + Decode the string *str* from UTF-8 in strict mode and write the output into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + See also :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + +.. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) + + Writer the wide string *str* into *writer*. + + *size* is a number of wide characters. If *size* is equal to ``-1``, call + ``wcslen(str)`` to get the string length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, Py_UCS4 *str, Py_ssize_t size) + + Writer the UCS4 string *str* into *writer*. + + *size* is a number of UCS4 characters. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) + + Call :c:func:`PyObject_Str` on *obj* and write the output into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) + + Call :c:func:`PyObject_Repr` on *obj* and write the output into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) + + Write the substring ``str[start:end]`` into *writer*. + + *str* must be Python :class:`str` object. *start* must be greater than or + equal to 0, and less than or equal to *end*. *end* must be less than or + equal to *str* length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) + + Similar to :c:func:`PyUnicode_FromFormat`, but write the output directly into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, const char *string, Py_ssize_t length, const char *errors, Py_ssize_t *consumed) + + Decode the string *str* from UTF-8 with *errors* error handler and write the + output into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + *errors* is an error handler name, such as ``"replace"``. If *errors* is + ``NULL``, use the strict error handler. + + If *consumed* is not ``NULL``, set *\*consumed* to the number of decoded + bytes on success. + If *consumed* is ``NULL``, treat trailing incomplete UTF-8 byte sequences + as an error. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + See also :c:func:`PyUnicodeWriter_WriteUTF8`. diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst index 48ae54acebe887..9d0abf440f791d 100644 --- a/Doc/c-api/utilities.rst +++ b/Doc/c-api/utilities.rst @@ -20,4 +20,5 @@ and parsing function arguments and constructing Python values from C values. hash.rst reflection.rst codec.rst + time.rst perfmaps.rst diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 324518c035096b..9f02bdb5896563 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -25,30 +25,6 @@ are only passed to these functions if it is certain that they were created by the same library that the Python runtime is using. -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter. This is made available for - programs which embed Python. The *argc* and *argv* parameters should be - prepared exactly as those which are passed to a C program's :c:func:`main` - function (converted to wchar_t according to the user's locale). It is - important to note that the argument list may be modified (but the contents of - the strings pointed to by the argument list are not). The return value will - be ``0`` if the interpreter exits normally (i.e., without an exception), - ``1`` if the interpreter exits due to an exception, or ``2`` if the parameter - list does not represent a valid Python command line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - :c:member:`PyConfig.inspect` is zero. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings. - - .. versionadded:: 3.8 - - .. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving @@ -322,7 +298,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_eval_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for isolated expressions; for use with :c:func:`Py_CompileString`. @@ -330,7 +306,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_file_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for sequences of statements as read from a file or other source; for use with :c:func:`Py_CompileString`. This is @@ -339,7 +315,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_single_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for a single statement; for use with :c:func:`Py_CompileString`. This is the symbol used for the interactive diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 038f54a9751fd1..8f233e16fb17cf 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -35,7 +35,7 @@ as much as it can. callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a - weakly referencable object, or if *callback* is not callable, ``None``, or + weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -47,7 +47,7 @@ as much as it can. be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* - is not a weakly referencable object, or if *callback* is not callable, + is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -96,3 +96,19 @@ as much as it can. This iterates through the weak references for *object* and calls callbacks for those references which have one. It returns when all callbacks have been attempted. + + +.. c:function:: void PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *object) + + Clears the weakrefs for *object* without calling the callbacks. + + This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler + for types with finalizers (i.e., :meth:`~object.__del__`). The handler for + those objects first calls :c:func:`PyObject_ClearWeakRefs` to clear weakrefs + and call their callbacks, then the finalizer, and finally this function to + clear any weakrefs that may have been created by the finalizer. + + In most circumstances, it's more appropriate to use + :c:func:`PyObject_ClearWeakRefs` to clear weakrefs instead of this function. + + .. versionadded:: 3.13 diff --git a/Doc/conf.py b/Doc/conf.py index c2d57696aeeaa3..73d7d5db26ff7b 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -6,35 +6,45 @@ # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). +import importlib import os import sys import time + +import sphinx + +# Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) +# Python specific content from Doc/Tools/extensions/pyspecific.py +from pyspecific import SOURCE_URI + # General configuration # --------------------- +# Our custom Sphinx extensions are found in Doc/Tools/extensions/ extensions = [ - 'asdl_highlight', + 'audit_events', + 'availability', 'c_annotations', - 'escape4chm', 'glossary_search', - 'peg_highlight', + 'lexers', 'pyspecific', 'sphinx.ext.coverage', 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', ] # Skip if downstream redistributors haven't installed them try: - import notfound.extension + import notfound.extension # noqa: F401 except ImportError: pass else: extensions.append('notfound.extension') try: - import sphinxext.opengraph + import sphinxext.opengraph # noqa: F401 except ImportError: pass else: @@ -47,7 +57,7 @@ except ImportError: _tkinter = None # Treat warnings as errors, done here to prevent warnings in Sphinx code from -# causing spurious test failures. +# causing spurious CPython test failures. import warnings warnings.simplefilter('error') del warnings @@ -57,31 +67,42 @@ # General substitutions. project = 'Python' -copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" +if sphinx.version_info[:2] >= (8, 1): + copyright = "2001-%Y, Python Software Foundation" +else: + copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. -import patchlevel -version, release = patchlevel.get_version_info() +# See Doc/tools/extensions/patchlevel.py +version, release = importlib.import_module('patchlevel').get_version_info() + +rst_epilog = f""" +.. |python_version_literal| replace:: ``Python {version}`` +.. |python_x_dot_y_literal| replace:: ``python{version}`` +.. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` +""" -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: +# There are two options for replacing |today|. Either, you set today to some +# non-false value and use it. today = '' -# Else, today_fmt is used as the format for a strftime call. +# Or else, today_fmt is used as the format for a strftime call. today_fmt = '%B %d, %Y' # By default, highlight as Python 3. highlight_language = 'python3' # Minimum version of sphinx required -needs_sphinx = '4.2' +needs_sphinx = '7.2.6' # Create table of contents entries for domain objects (e.g. functions, classes, # attributes, etc.). Default is True. -toc_object_entries = False +toc_object_entries = True +# Hide parents to tidy up long entries in sidebar +toc_object_entries_show_parents = 'hide' # Ignore any .rst files in the includes/ directory; -# they're embedded in pages but not rendered individually. +# they're embedded in pages but not rendered as individual pages. # Ignore any .rst files in the venv/ directory. exclude_patterns = ['includes/*.rst', 'venv/*', 'README.rst'] venvdir = os.getenv('VENVDIR') @@ -95,11 +116,13 @@ ('c:func', 'dlopen'), ('c:func', 'exec'), ('c:func', 'fcntl'), + ('c:func', 'flock'), ('c:func', 'fork'), ('c:func', 'free'), ('c:func', 'gettimeofday'), ('c:func', 'gmtime'), ('c:func', 'grantpt'), + ('c:func', 'ioctl'), ('c:func', 'localeconv'), ('c:func', 'localtime'), ('c:func', 'main'), @@ -120,6 +143,9 @@ ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), + ('c:type', 'int8_t'), + ('c:type', 'int16_t'), + ('c:type', 'int32_t'), ('c:type', 'int64_t'), ('c:type', 'intmax_t'), ('c:type', 'off_t'), @@ -128,6 +154,9 @@ ('c:type', 'size_t'), ('c:type', 'ssize_t'), ('c:type', 'time_t'), + ('c:type', 'uint8_t'), + ('c:type', 'uint16_t'), + ('c:type', 'uint32_t'), ('c:type', 'uint64_t'), ('c:type', 'uintmax_t'), ('c:type', 'uintptr_t'), @@ -135,11 +164,14 @@ ('c:type', 'wchar_t'), ('c:type', '__int64'), ('c:type', 'unsigned __int64'), + ('c:type', 'double'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), ('c:struct', 'stat'), ('c:struct', 'statvfs'), + ('c:struct', 'timeval'), + ('c:struct', 'timespec'), # Standard C macros ('c:macro', 'LLONG_MAX'), ('c:macro', 'LLONG_MIN'), @@ -167,6 +199,7 @@ ('envvar', 'LC_TIME'), ('envvar', 'LINES'), ('envvar', 'LOGNAME'), + ('envvar', 'MANPAGER'), ('envvar', 'PAGER'), ('envvar', 'PATH'), ('envvar', 'PATHEXT'), @@ -227,6 +260,7 @@ ('c:data', 'PyExc_OverflowError'), ('c:data', 'PyExc_PermissionError'), ('c:data', 'PyExc_ProcessLookupError'), + ('c:data', 'PyExc_PythonFinalizationError'), ('c:data', 'PyExc_RecursionError'), ('c:data', 'PyExc_ReferenceError'), ('c:data', 'PyExc_RuntimeError'), @@ -257,6 +291,9 @@ ('c:data', 'PyExc_UnicodeWarning'), ('c:data', 'PyExc_UserWarning'), ('c:data', 'PyExc_Warning'), + # Undocumented public C macros + ('c:macro', 'Py_BUILD_ASSERT'), + ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. @@ -266,46 +303,52 @@ ('py:attr', '__annotations__'), ('py:meth', '__missing__'), ('py:attr', '__wrapped__'), + ('py:attr', 'decimal.Context.clamp'), ('py:meth', 'index'), # list.index, tuple.index, etc. ] -# gh-106948: Copy standard C types declared in the "c:type" domain to the -# "c:identifier" domain, since "c:function" markup looks for types in the -# "c:identifier" domain. Use list() to not iterate on items which are being -# added +# gh-106948: Copy standard C types declared in the "c:type" domain and C +# structures declared in the "c:struct" domain to the "c:identifier" domain, +# since "c:function" markup looks for types in the "c:identifier" domain. Use +# list() to not iterate on items which are being added for role, name in list(nitpick_ignore): - if role == 'c:type': + if role in ('c:type', 'c:struct'): nitpick_ignore.append(('c:identifier', name)) del role, name # Disable Docutils smartquotes for several translations smartquotes_excludes = { - 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'], + 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], + 'builders': ['man', 'text'], } -# Avoid a warning with Sphinx >= 2.0 -master_doc = 'contents' +# Avoid a warning with Sphinx >= 4.0 +root_doc = 'contents' # Allow translation of index directives gettext_additional_targets = [ 'index', + 'literal-block', ] # Options for HTML output # ----------------------- -# Use our custom theme. +# Use our custom theme: https://github.com/python/python-docs-theme html_theme = 'python_docs_theme' +# Location of overrides for theme templates and static files html_theme_path = ['tools'] html_theme_options = { 'collapsiblesidebar': True, 'issues_url': '/bugs.html', 'license_url': '/license.html', - 'root_include_title': False # We use the version switcher instead. + 'root_include_title': False, # We use the version switcher instead. } if os.getenv("READTHEDOCS"): - html_theme_options["hosted_on"] = 'Read the Docs' + html_theme_options["hosted_on"] = ( + 'Read the Docs' + ) # Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207 # https://github.com/python/cpython/issues/91207 @@ -319,17 +362,27 @@ # Deployment preview information # (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) -repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL") +is_deployment_preview = os.getenv("READTHEDOCS_VERSION_TYPE") == "external" +repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL", "") +repository_url = repository_url.removesuffix(".git") html_context = { - "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", - "repository_url": repository_url.removesuffix(".git") if repository_url else None, - "pr_id": os.getenv("READTHEDOCS_VERSION") + "is_deployment_preview": is_deployment_preview, + "repository_url": repository_url or None, + "pr_id": os.getenv("READTHEDOCS_VERSION"), + "enable_analytics": os.getenv("PYTHON_DOCS_ENABLE_ANALYTICS"), } # This 'Last updated on:' timestamp is inserted at the bottom of every page. -html_last_updated_fmt = time.strftime('%b %d, %Y (%H:%M UTC)', time.gmtime()) +html_last_updated_fmt = '%b %d, %Y (%H:%M UTC)' +if sphinx.version_info[:2] >= (8, 1): + html_last_updated_use_utc = True +else: + html_time = int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) + html_last_updated_fmt = time.strftime( + html_last_updated_fmt, time.gmtime(html_time) + ) -# Path to find HTML templates. +# Path to find HTML templates to override theme templates_path = ['tools/templates'] # Custom sidebar templates, filenames relative to this file. @@ -358,6 +411,8 @@ # Split the index html_split_index = True +# Split pot files one per reST file +gettext_compact = False # Options for LaTeX output # ------------------------ @@ -375,8 +430,8 @@ \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', - # The paper size ('letter' or 'a4'). - 'papersize': 'a4', + # The paper size ('letterpaper' or 'a4paper'). + 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', } @@ -385,30 +440,70 @@ # (source start file, target name, title, author, document class [howto/manual]). _stdauthor = 'Guido van Rossum and the Python development team' latex_documents = [ - ('c-api/index', 'c-api.tex', - 'The Python/C API', _stdauthor, 'manual'), - ('extending/index', 'extending.tex', - 'Extending and Embedding Python', _stdauthor, 'manual'), - ('installing/index', 'installing.tex', - 'Installing Python Modules', _stdauthor, 'manual'), - ('library/index', 'library.tex', - 'The Python Library Reference', _stdauthor, 'manual'), - ('reference/index', 'reference.tex', - 'The Python Language Reference', _stdauthor, 'manual'), - ('tutorial/index', 'tutorial.tex', - 'Python Tutorial', _stdauthor, 'manual'), - ('using/index', 'using.tex', - 'Python Setup and Usage', _stdauthor, 'manual'), - ('faq/index', 'faq.tex', - 'Python Frequently Asked Questions', _stdauthor, 'manual'), - ('whatsnew/' + version, 'whatsnew.tex', - 'What\'s New in Python', 'A. M. Kuchling', 'howto'), + ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), + ( + 'extending/index', + 'extending.tex', + 'Extending and Embedding Python', + _stdauthor, + 'manual', + ), + ( + 'installing/index', + 'installing.tex', + 'Installing Python Modules', + _stdauthor, + 'manual', + ), + ( + 'library/index', + 'library.tex', + 'The Python Library Reference', + _stdauthor, + 'manual', + ), + ( + 'reference/index', + 'reference.tex', + 'The Python Language Reference', + _stdauthor, + 'manual', + ), + ( + 'tutorial/index', + 'tutorial.tex', + 'Python Tutorial', + _stdauthor, + 'manual', + ), + ( + 'using/index', + 'using.tex', + 'Python Setup and Usage', + _stdauthor, + 'manual', + ), + ( + 'faq/index', + 'faq.tex', + 'Python Frequently Asked Questions', + _stdauthor, + 'manual', + ), + ( + 'whatsnew/' + version, + 'whatsnew.tex', + 'What\'s New in Python', + 'A. M. Kuchling', + 'howto', + ), ] # Collect all HOWTOs individually -latex_documents.extend(('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', - '', _stdauthor, 'howto') - for fn in os.listdir('howto') - if fn.endswith('.rst') and fn != 'index.rst') +latex_documents.extend( + ('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', '', _stdauthor, 'howto') + for fn in os.listdir('howto') + if fn.endswith('.rst') and fn != 'index.rst' +) # Documents to append as an appendix to all manuals. latex_appendices = ['glossary', 'about', 'license', 'copyright'] @@ -419,6 +514,10 @@ epub_author = 'Python Documentation Authors' epub_publisher = 'Python Software Foundation' +# index pages are not valid xhtml +# https://github.com/sphinx-doc/sphinx/issues/12359 +epub_use_index = False + # Options for the coverage checker # -------------------------------- @@ -432,8 +531,7 @@ 'test($|_)', ] -coverage_ignore_classes = [ -] +coverage_ignore_classes = [] # Glob patterns for C source files for C API coverage, relative to this directory. coverage_c_path = [ @@ -450,7 +548,7 @@ # The coverage checker will ignore all C items whose names match these regexes # (using re.match) -- the keys must be the same as in coverage_c_regexes. coverage_ignore_c_items = { -# 'cfunction': [...] + # 'cfunction': [...] } @@ -472,9 +570,15 @@ r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*', r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*', r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*', + # Debian's man page redirects to its current stable version + r'https://manpages.debian.org/\w+\(\d(\w+)?\)': r'https://manpages.debian.org/\w+/[\w/\-\.]*\.\d(\w+)?\.en\.html', # Language redirects r'https://toml.io': 'https://toml.io/en/', r'https://www.redhat.com': 'https://www.redhat.com/en', + # pypi.org project name normalization (upper to lowercase, underscore to hyphen) + r'https://pypi.org/project/[A-Za-z\d_\-\.]+/': r'https://pypi.org/project/[a-z\d\-\.]+/', + # Discourse title name expansion (text changes when title is edited) + r'https://discuss\.python\.org/t/\d+': r'https://discuss\.python\.org/t/.*/\d+', # Other redirects r'https://www.boost.org/libs/.+': r'https://www.boost.org/doc/libs/\d_\d+_\d/.+', r'https://support.microsoft.com/en-us/help/\d+': 'https://support.microsoft.com/en-us/topic/.+', @@ -501,15 +605,38 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] +# Options for sphinx.ext.extlinks +# ------------------------------- + +# This config is a dictionary of external sites, +# mapping unique short aliases to a base URL and a prefix. +# https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html +extlinks = { + "pypi": ("https://pypi.org/project/%s/", "%s"), + "source": (SOURCE_URI, "%s"), +} +extlinks_detect_hardcoded_links = True + +if sphinx.version_info[:2] < (8, 1): + # Sphinx 8.1 has in-built CVE and CWE roles. + extlinks |= { + "cve": ( + "https://www.cve.org/CVERecord?id=CVE-%s", + "CVE-%s", + ), + "cwe": ("https://cwe.mitre.org/data/definitions/%s.html", "CWE-%s"), + } -# Options for extensions -# ---------------------- +# Options for c_annotations extension +# ----------------------------------- # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' -# sphinxext-opengraph config +# Options for sphinxext-opengraph +# ------------------------------- + ogp_site_url = 'https://docs.python.org/3/' ogp_site_name = 'Python documentation' ogp_image = '_static/og-image.png' diff --git a/Doc/constraints.txt b/Doc/constraints.txt index 147de1271eb2b7..26ac1862dbac0b 100644 --- a/Doc/constraints.txt +++ b/Doc/constraints.txt @@ -7,18 +7,20 @@ # Direct dependencies of Sphinx babel<3 colorama<0.5 -imagesize<1.5 -Jinja2<3.2 -packaging<24 -Pygments>=2.16.1,<3 +imagesize<2 +Jinja2<4 +packaging<25 +Pygments<3 requests<3 snowballstemmer<3 -sphinxcontrib-applehelp<1.1 -sphinxcontrib-devhelp<1.1 -sphinxcontrib-htmlhelp<2.1 -sphinxcontrib-jsmath<1.1 -sphinxcontrib-qthelp<1.1 -sphinxcontrib-serializinghtml<1.2 +# keep lower-bounds until Sphinx 8.1 is released +# https://github.com/sphinx-doc/sphinx/pull/12756 +sphinxcontrib-applehelp>=1.0.7,<3 +sphinxcontrib-devhelp>=1.0.6,<3 +sphinxcontrib-htmlhelp>=2.0.6,<3 +sphinxcontrib-jsmath>=1.0.1,<2 +sphinxcontrib-qthelp>=1.0.6,<3 +sphinxcontrib-serializinghtml>=1.1.9,<3 # Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above) -MarkupSafe<2.2 +MarkupSafe<3 diff --git a/Doc/contents.rst b/Doc/contents.rst index 24ceacb0076b5e..b57f4b09a5dcb6 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -14,6 +14,7 @@ installing/index.rst howto/index.rst faq/index.rst + deprecations/index.rst glossary.rst about.rst diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 62a96146d605ff..65d48f8bea7de8 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -364,8 +364,6 @@ PyComplex_RealAsDouble:PyObject*:op:0: PyContext_CheckExact:int::: PyContext_CheckExact:PyObject*:o:0: -PyContext_ClearFreeList:int::: - PyContext_Copy:PyObject*::+1: PyContext_Copy:PyObject*:ctx:0: @@ -790,6 +788,12 @@ PyEval_GetGlobals:PyObject*::0: PyEval_GetFrame:PyObject*::0: +PyEval_GetFrameBuiltins:PyObject*::+1: + +PyEval_GetFrameLocals:PyObject*::+1: + +PyEval_GetFrameGlobals:PyObject*::+1: + PyEval_GetFuncDesc:const char*::: PyEval_GetFuncDesc:PyObject*:func:0: @@ -916,6 +920,32 @@ PyFloat_FromString:PyObject*:str:0: PyFloat_GetInfo:PyObject*::+1: PyFloat_GetInfo::void:: +PyFrame_GetBack:PyObject*::+1: +PyFrame_GetBack:PyFrameObject*:frame:0: + +PyFrame_GetBuiltins:PyObject*::+1: +PyFrame_GetBuiltins:PyFrameObject*:frame:0: + +PyFrame_GetCode:PyObject*::+1: +PyFrame_GetCode:PyFrameObject*:frame:0: + +PyFrame_GetGenerator:PyObject*::+1: +PyFrame_GetGenerator:PyFrameObject*:frame:0: + +PyFrame_GetGlobals:PyObject*::+1: +PyFrame_GetGlobals:PyFrameObject*:frame:0: + +PyFrame_GetLocals:PyObject*::+1: +PyFrame_GetLocals:PyFrameObject*:frame:0: + +PyFrame_GetVar:PyObject*::+1: +PyFrame_GetVar:PyFrameObject*:frame:0: +PyFrame_GetVar:PyObject*:name:0: + +PyFrame_GetVarString:PyObject*::+1: +PyFrame_GetVarString:PyFrameObject*:frame:0: +PyFrame_GetVarString:const char*:name:: + PyFrozenSet_Check:int::: PyFrozenSet_Check:PyObject*:p:0: @@ -998,8 +1028,6 @@ PyImport_AddModule:const char*:name:: PyImport_AddModuleObject:PyObject*::0:reference borrowed from sys.modules PyImport_AddModuleObject:PyObject*:name:0: -PyImport_Cleanup:void::: - PyImport_ExecCodeModule:PyObject*::+1: PyImport_ExecCodeModule:const char*:name:: PyImport_ExecCodeModule:PyObject*:co:0: @@ -1104,6 +1132,10 @@ PyAIter_Check:PyObject*:o:0: PyIter_Next:PyObject*::+1: PyIter_Next:PyObject*:o:0: +PyIter_NextItem:int::: +PyIter_NextItem:PyObject*:iter:0: +PyIter_NextItem:PyObject**:item:+1: + PyIter_Send:int::: PyIter_Send:PyObject*:iter:0: PyIter_Send:PyObject*:arg:0: @@ -2373,12 +2405,6 @@ PyUnicode_DATA:PyObject*:o:0: PyUnicode_GET_LENGTH:Py_ssize_t::: PyUnicode_GET_LENGTH:PyObject*:o:0: -PyUnicode_GET_SIZE:Py_ssize_t::: -PyUnicode_GET_SIZE:PyObject*:o:0: - -PyUnicode_GET_DATA_SIZE:Py_ssize_t::: -PyUnicode_GET_DATA_SIZE:PyObject*:o:0: - PyUnicode_KIND:int::: PyUnicode_KIND:PyObject*:o:0: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index def1903204add7..6f9d27297e8f65 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,873 +1,901 @@ role,name,added,ifdef_note,struct_abi_kind macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, -function,PyAIter_Check,3.10,, -function,PyArg_Parse,3.2,, -function,PyArg_ParseTuple,3.2,, -function,PyArg_ParseTupleAndKeywords,3.2,, -function,PyArg_UnpackTuple,3.2,, -function,PyArg_VaParse,3.2,, -function,PyArg_VaParseTupleAndKeywords,3.2,, -function,PyArg_ValidateKeywordArguments,3.2,, -var,PyBaseObject_Type,3.2,, -function,PyBool_FromLong,3.2,, -var,PyBool_Type,3.2,, -function,PyBuffer_FillContiguousStrides,3.11,, -function,PyBuffer_FillInfo,3.11,, -function,PyBuffer_FromContiguous,3.11,, -function,PyBuffer_GetPointer,3.11,, -function,PyBuffer_IsContiguous,3.11,, -function,PyBuffer_Release,3.11,, -function,PyBuffer_SizeFromFormat,3.11,, -function,PyBuffer_ToContiguous,3.11,, -var,PyByteArrayIter_Type,3.2,, -function,PyByteArray_AsString,3.2,, -function,PyByteArray_Concat,3.2,, -function,PyByteArray_FromObject,3.2,, -function,PyByteArray_FromStringAndSize,3.2,, -function,PyByteArray_Resize,3.2,, -function,PyByteArray_Size,3.2,, -var,PyByteArray_Type,3.2,, -var,PyBytesIter_Type,3.2,, -function,PyBytes_AsString,3.2,, -function,PyBytes_AsStringAndSize,3.2,, -function,PyBytes_Concat,3.2,, -function,PyBytes_ConcatAndDel,3.2,, -function,PyBytes_DecodeEscape,3.2,, -function,PyBytes_FromFormat,3.2,, -function,PyBytes_FromFormatV,3.2,, -function,PyBytes_FromObject,3.2,, -function,PyBytes_FromString,3.2,, -function,PyBytes_FromStringAndSize,3.2,, -function,PyBytes_Repr,3.2,, -function,PyBytes_Size,3.2,, -var,PyBytes_Type,3.2,, +func,PyAIter_Check,3.10,, +func,PyArg_Parse,3.2,, +func,PyArg_ParseTuple,3.2,, +func,PyArg_ParseTupleAndKeywords,3.2,, +func,PyArg_UnpackTuple,3.2,, +func,PyArg_VaParse,3.2,, +func,PyArg_VaParseTupleAndKeywords,3.2,, +func,PyArg_ValidateKeywordArguments,3.2,, +data,PyBaseObject_Type,3.2,, +func,PyBool_FromLong,3.2,, +data,PyBool_Type,3.2,, +func,PyBuffer_FillContiguousStrides,3.11,, +func,PyBuffer_FillInfo,3.11,, +func,PyBuffer_FromContiguous,3.11,, +func,PyBuffer_GetPointer,3.11,, +func,PyBuffer_IsContiguous,3.11,, +func,PyBuffer_Release,3.11,, +func,PyBuffer_SizeFromFormat,3.11,, +func,PyBuffer_ToContiguous,3.11,, +data,PyByteArrayIter_Type,3.2,, +func,PyByteArray_AsString,3.2,, +func,PyByteArray_Concat,3.2,, +func,PyByteArray_FromObject,3.2,, +func,PyByteArray_FromStringAndSize,3.2,, +func,PyByteArray_Resize,3.2,, +func,PyByteArray_Size,3.2,, +data,PyByteArray_Type,3.2,, +data,PyBytesIter_Type,3.2,, +func,PyBytes_AsString,3.2,, +func,PyBytes_AsStringAndSize,3.2,, +func,PyBytes_Concat,3.2,, +func,PyBytes_ConcatAndDel,3.2,, +func,PyBytes_DecodeEscape,3.2,, +func,PyBytes_FromFormat,3.2,, +func,PyBytes_FromFormatV,3.2,, +func,PyBytes_FromObject,3.2,, +func,PyBytes_FromString,3.2,, +func,PyBytes_FromStringAndSize,3.2,, +func,PyBytes_Repr,3.2,, +func,PyBytes_Size,3.2,, +data,PyBytes_Type,3.2,, type,PyCFunction,3.2,, +type,PyCFunctionFast,3.13,, +type,PyCFunctionFastWithKeywords,3.13,, type,PyCFunctionWithKeywords,3.2,, -function,PyCFunction_GetFlags,3.2,, -function,PyCFunction_GetFunction,3.2,, -function,PyCFunction_GetSelf,3.2,, -function,PyCFunction_New,3.4,, -function,PyCFunction_NewEx,3.2,, -var,PyCFunction_Type,3.2,, -function,PyCMethod_New,3.9,, -function,PyCallIter_New,3.2,, -var,PyCallIter_Type,3.2,, -function,PyCallable_Check,3.2,, +func,PyCFunction_GetFlags,3.2,, +func,PyCFunction_GetFunction,3.2,, +func,PyCFunction_GetSelf,3.2,, +func,PyCFunction_New,3.4,, +func,PyCFunction_NewEx,3.2,, +data,PyCFunction_Type,3.2,, +func,PyCMethod_New,3.9,, +func,PyCallIter_New,3.2,, +data,PyCallIter_Type,3.2,, +func,PyCallable_Check,3.2,, type,PyCapsule_Destructor,3.2,, -function,PyCapsule_GetContext,3.2,, -function,PyCapsule_GetDestructor,3.2,, -function,PyCapsule_GetName,3.2,, -function,PyCapsule_GetPointer,3.2,, -function,PyCapsule_Import,3.2,, -function,PyCapsule_IsValid,3.2,, -function,PyCapsule_New,3.2,, -function,PyCapsule_SetContext,3.2,, -function,PyCapsule_SetDestructor,3.2,, -function,PyCapsule_SetName,3.2,, -function,PyCapsule_SetPointer,3.2,, -var,PyCapsule_Type,3.2,, -var,PyClassMethodDescr_Type,3.2,, -function,PyCodec_BackslashReplaceErrors,3.2,, -function,PyCodec_Decode,3.2,, -function,PyCodec_Decoder,3.2,, -function,PyCodec_Encode,3.2,, -function,PyCodec_Encoder,3.2,, -function,PyCodec_IgnoreErrors,3.2,, -function,PyCodec_IncrementalDecoder,3.2,, -function,PyCodec_IncrementalEncoder,3.2,, -function,PyCodec_KnownEncoding,3.2,, -function,PyCodec_LookupError,3.2,, -function,PyCodec_NameReplaceErrors,3.7,, -function,PyCodec_Register,3.2,, -function,PyCodec_RegisterError,3.2,, -function,PyCodec_ReplaceErrors,3.2,, -function,PyCodec_StreamReader,3.2,, -function,PyCodec_StreamWriter,3.2,, -function,PyCodec_StrictErrors,3.2,, -function,PyCodec_Unregister,3.10,, -function,PyCodec_XMLCharRefReplaceErrors,3.2,, -function,PyComplex_FromDoubles,3.2,, -function,PyComplex_ImagAsDouble,3.2,, -function,PyComplex_RealAsDouble,3.2,, -var,PyComplex_Type,3.2,, -function,PyDescr_NewClassMethod,3.2,, -function,PyDescr_NewGetSet,3.2,, -function,PyDescr_NewMember,3.2,, -function,PyDescr_NewMethod,3.2,, -var,PyDictItems_Type,3.2,, -var,PyDictIterItem_Type,3.2,, -var,PyDictIterKey_Type,3.2,, -var,PyDictIterValue_Type,3.2,, -var,PyDictKeys_Type,3.2,, -function,PyDictProxy_New,3.2,, -var,PyDictProxy_Type,3.2,, -var,PyDictRevIterItem_Type,3.8,, -var,PyDictRevIterKey_Type,3.8,, -var,PyDictRevIterValue_Type,3.8,, -var,PyDictValues_Type,3.2,, -function,PyDict_Clear,3.2,, -function,PyDict_Contains,3.2,, -function,PyDict_Copy,3.2,, -function,PyDict_DelItem,3.2,, -function,PyDict_DelItemString,3.2,, -function,PyDict_GetItem,3.2,, -function,PyDict_GetItemRef,3.13,, -function,PyDict_GetItemString,3.2,, -function,PyDict_GetItemStringRef,3.13,, -function,PyDict_GetItemWithError,3.2,, -function,PyDict_Items,3.2,, -function,PyDict_Keys,3.2,, -function,PyDict_Merge,3.2,, -function,PyDict_MergeFromSeq2,3.2,, -function,PyDict_New,3.2,, -function,PyDict_Next,3.2,, -function,PyDict_SetItem,3.2,, -function,PyDict_SetItemString,3.2,, -function,PyDict_Size,3.2,, -var,PyDict_Type,3.2,, -function,PyDict_Update,3.2,, -function,PyDict_Values,3.2,, -var,PyEllipsis_Type,3.2,, -var,PyEnum_Type,3.2,, -function,PyErr_BadArgument,3.2,, -function,PyErr_BadInternalCall,3.2,, -function,PyErr_CheckSignals,3.2,, -function,PyErr_Clear,3.2,, -function,PyErr_Display,3.2,, -function,PyErr_DisplayException,3.12,, -function,PyErr_ExceptionMatches,3.2,, -function,PyErr_Fetch,3.2,, -function,PyErr_Format,3.2,, -function,PyErr_FormatV,3.5,, -function,PyErr_GetExcInfo,3.7,, -function,PyErr_GetHandledException,3.11,, -function,PyErr_GetRaisedException,3.12,, -function,PyErr_GivenExceptionMatches,3.2,, -function,PyErr_NewException,3.2,, -function,PyErr_NewExceptionWithDoc,3.2,, -function,PyErr_NoMemory,3.2,, -function,PyErr_NormalizeException,3.2,, -function,PyErr_Occurred,3.2,, -function,PyErr_Print,3.2,, -function,PyErr_PrintEx,3.2,, -function,PyErr_ProgramText,3.2,, -function,PyErr_ResourceWarning,3.6,, -function,PyErr_Restore,3.2,, -function,PyErr_SetExcFromWindowsErr,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, -function,PyErr_SetExcInfo,3.7,, -function,PyErr_SetFromErrno,3.2,, -function,PyErr_SetFromErrnoWithFilename,3.2,, -function,PyErr_SetFromErrnoWithFilenameObject,3.2,, -function,PyErr_SetFromErrnoWithFilenameObjects,3.7,, -function,PyErr_SetFromWindowsErr,3.7,on Windows, -function,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, -function,PyErr_SetHandledException,3.11,, -function,PyErr_SetImportError,3.7,, -function,PyErr_SetImportErrorSubclass,3.6,, -function,PyErr_SetInterrupt,3.2,, -function,PyErr_SetInterruptEx,3.10,, -function,PyErr_SetNone,3.2,, -function,PyErr_SetObject,3.2,, -function,PyErr_SetRaisedException,3.12,, -function,PyErr_SetString,3.2,, -function,PyErr_SyntaxLocation,3.2,, -function,PyErr_SyntaxLocationEx,3.7,, -function,PyErr_WarnEx,3.2,, -function,PyErr_WarnExplicit,3.2,, -function,PyErr_WarnFormat,3.2,, -function,PyErr_WriteUnraisable,3.2,, -function,PyEval_AcquireThread,3.2,, -function,PyEval_EvalCode,3.2,, -function,PyEval_EvalCodeEx,3.2,, -function,PyEval_EvalFrame,3.2,, -function,PyEval_EvalFrameEx,3.2,, -function,PyEval_GetBuiltins,3.2,, -function,PyEval_GetFrame,3.2,, -function,PyEval_GetFuncDesc,3.2,, -function,PyEval_GetFuncName,3.2,, -function,PyEval_GetGlobals,3.2,, -function,PyEval_GetLocals,3.2,, -function,PyEval_ReleaseThread,3.2,, -function,PyEval_RestoreThread,3.2,, -function,PyEval_SaveThread,3.2,, -var,PyExc_ArithmeticError,3.2,, -var,PyExc_AssertionError,3.2,, -var,PyExc_AttributeError,3.2,, -var,PyExc_BaseException,3.2,, -var,PyExc_BaseExceptionGroup,3.11,, -var,PyExc_BlockingIOError,3.7,, -var,PyExc_BrokenPipeError,3.7,, -var,PyExc_BufferError,3.2,, -var,PyExc_BytesWarning,3.2,, -var,PyExc_ChildProcessError,3.7,, -var,PyExc_ConnectionAbortedError,3.7,, -var,PyExc_ConnectionError,3.7,, -var,PyExc_ConnectionRefusedError,3.7,, -var,PyExc_ConnectionResetError,3.7,, -var,PyExc_DeprecationWarning,3.2,, -var,PyExc_EOFError,3.2,, -var,PyExc_EncodingWarning,3.10,, -var,PyExc_EnvironmentError,3.2,, -var,PyExc_Exception,3.2,, -var,PyExc_FileExistsError,3.7,, -var,PyExc_FileNotFoundError,3.7,, -var,PyExc_FloatingPointError,3.2,, -var,PyExc_FutureWarning,3.2,, -var,PyExc_GeneratorExit,3.2,, -var,PyExc_IOError,3.2,, -var,PyExc_ImportError,3.2,, -var,PyExc_ImportWarning,3.2,, -var,PyExc_IncompleteInputError,3.13,, -var,PyExc_IndentationError,3.2,, -var,PyExc_IndexError,3.2,, -var,PyExc_InterruptedError,3.7,, -var,PyExc_IsADirectoryError,3.7,, -var,PyExc_KeyError,3.2,, -var,PyExc_KeyboardInterrupt,3.2,, -var,PyExc_LookupError,3.2,, -var,PyExc_MemoryError,3.2,, -var,PyExc_ModuleNotFoundError,3.6,, -var,PyExc_NameError,3.2,, -var,PyExc_NotADirectoryError,3.7,, -var,PyExc_NotImplementedError,3.2,, -var,PyExc_OSError,3.2,, -var,PyExc_OverflowError,3.2,, -var,PyExc_PendingDeprecationWarning,3.2,, -var,PyExc_PermissionError,3.7,, -var,PyExc_ProcessLookupError,3.7,, -var,PyExc_RecursionError,3.7,, -var,PyExc_ReferenceError,3.2,, -var,PyExc_ResourceWarning,3.7,, -var,PyExc_RuntimeError,3.2,, -var,PyExc_RuntimeWarning,3.2,, -var,PyExc_StopAsyncIteration,3.7,, -var,PyExc_StopIteration,3.2,, -var,PyExc_SyntaxError,3.2,, -var,PyExc_SyntaxWarning,3.2,, -var,PyExc_SystemError,3.2,, -var,PyExc_SystemExit,3.2,, -var,PyExc_TabError,3.2,, -var,PyExc_TimeoutError,3.7,, -var,PyExc_TypeError,3.2,, -var,PyExc_UnboundLocalError,3.2,, -var,PyExc_UnicodeDecodeError,3.2,, -var,PyExc_UnicodeEncodeError,3.2,, -var,PyExc_UnicodeError,3.2,, -var,PyExc_UnicodeTranslateError,3.2,, -var,PyExc_UnicodeWarning,3.2,, -var,PyExc_UserWarning,3.2,, -var,PyExc_ValueError,3.2,, -var,PyExc_Warning,3.2,, -var,PyExc_WindowsError,3.7,on Windows, -var,PyExc_ZeroDivisionError,3.2,, -function,PyExceptionClass_Name,3.8,, -function,PyException_GetArgs,3.12,, -function,PyException_GetCause,3.2,, -function,PyException_GetContext,3.2,, -function,PyException_GetTraceback,3.2,, -function,PyException_SetArgs,3.12,, -function,PyException_SetCause,3.2,, -function,PyException_SetContext,3.2,, -function,PyException_SetTraceback,3.2,, -function,PyFile_FromFd,3.2,, -function,PyFile_GetLine,3.2,, -function,PyFile_WriteObject,3.2,, -function,PyFile_WriteString,3.2,, -var,PyFilter_Type,3.2,, -function,PyFloat_AsDouble,3.2,, -function,PyFloat_FromDouble,3.2,, -function,PyFloat_FromString,3.2,, -function,PyFloat_GetInfo,3.2,, -function,PyFloat_GetMax,3.2,, -function,PyFloat_GetMin,3.2,, -var,PyFloat_Type,3.2,, +func,PyCapsule_GetContext,3.2,, +func,PyCapsule_GetDestructor,3.2,, +func,PyCapsule_GetName,3.2,, +func,PyCapsule_GetPointer,3.2,, +func,PyCapsule_Import,3.2,, +func,PyCapsule_IsValid,3.2,, +func,PyCapsule_New,3.2,, +func,PyCapsule_SetContext,3.2,, +func,PyCapsule_SetDestructor,3.2,, +func,PyCapsule_SetName,3.2,, +func,PyCapsule_SetPointer,3.2,, +data,PyCapsule_Type,3.2,, +data,PyClassMethodDescr_Type,3.2,, +func,PyCodec_BackslashReplaceErrors,3.2,, +func,PyCodec_Decode,3.2,, +func,PyCodec_Decoder,3.2,, +func,PyCodec_Encode,3.2,, +func,PyCodec_Encoder,3.2,, +func,PyCodec_IgnoreErrors,3.2,, +func,PyCodec_IncrementalDecoder,3.2,, +func,PyCodec_IncrementalEncoder,3.2,, +func,PyCodec_KnownEncoding,3.2,, +func,PyCodec_LookupError,3.2,, +func,PyCodec_NameReplaceErrors,3.7,, +func,PyCodec_Register,3.2,, +func,PyCodec_RegisterError,3.2,, +func,PyCodec_ReplaceErrors,3.2,, +func,PyCodec_StreamReader,3.2,, +func,PyCodec_StreamWriter,3.2,, +func,PyCodec_StrictErrors,3.2,, +func,PyCodec_Unregister,3.10,, +func,PyCodec_XMLCharRefReplaceErrors,3.2,, +func,PyComplex_FromDoubles,3.2,, +func,PyComplex_ImagAsDouble,3.2,, +func,PyComplex_RealAsDouble,3.2,, +data,PyComplex_Type,3.2,, +func,PyDescr_NewClassMethod,3.2,, +func,PyDescr_NewGetSet,3.2,, +func,PyDescr_NewMember,3.2,, +func,PyDescr_NewMethod,3.2,, +data,PyDictItems_Type,3.2,, +data,PyDictIterItem_Type,3.2,, +data,PyDictIterKey_Type,3.2,, +data,PyDictIterValue_Type,3.2,, +data,PyDictKeys_Type,3.2,, +func,PyDictProxy_New,3.2,, +data,PyDictProxy_Type,3.2,, +data,PyDictRevIterItem_Type,3.8,, +data,PyDictRevIterKey_Type,3.8,, +data,PyDictRevIterValue_Type,3.8,, +data,PyDictValues_Type,3.2,, +func,PyDict_Clear,3.2,, +func,PyDict_Contains,3.2,, +func,PyDict_Copy,3.2,, +func,PyDict_DelItem,3.2,, +func,PyDict_DelItemString,3.2,, +func,PyDict_GetItem,3.2,, +func,PyDict_GetItemRef,3.13,, +func,PyDict_GetItemString,3.2,, +func,PyDict_GetItemStringRef,3.13,, +func,PyDict_GetItemWithError,3.2,, +func,PyDict_Items,3.2,, +func,PyDict_Keys,3.2,, +func,PyDict_Merge,3.2,, +func,PyDict_MergeFromSeq2,3.2,, +func,PyDict_New,3.2,, +func,PyDict_Next,3.2,, +func,PyDict_SetItem,3.2,, +func,PyDict_SetItemString,3.2,, +func,PyDict_Size,3.2,, +data,PyDict_Type,3.2,, +func,PyDict_Update,3.2,, +func,PyDict_Values,3.2,, +data,PyEllipsis_Type,3.2,, +data,PyEnum_Type,3.2,, +func,PyErr_BadArgument,3.2,, +func,PyErr_BadInternalCall,3.2,, +func,PyErr_CheckSignals,3.2,, +func,PyErr_Clear,3.2,, +func,PyErr_Display,3.2,, +func,PyErr_DisplayException,3.12,, +func,PyErr_ExceptionMatches,3.2,, +func,PyErr_Fetch,3.2,, +func,PyErr_Format,3.2,, +func,PyErr_FormatV,3.5,, +func,PyErr_GetExcInfo,3.7,, +func,PyErr_GetHandledException,3.11,, +func,PyErr_GetRaisedException,3.12,, +func,PyErr_GivenExceptionMatches,3.2,, +func,PyErr_NewException,3.2,, +func,PyErr_NewExceptionWithDoc,3.2,, +func,PyErr_NoMemory,3.2,, +func,PyErr_NormalizeException,3.2,, +func,PyErr_Occurred,3.2,, +func,PyErr_Print,3.2,, +func,PyErr_PrintEx,3.2,, +func,PyErr_ProgramText,3.2,, +func,PyErr_ResourceWarning,3.6,, +func,PyErr_Restore,3.2,, +func,PyErr_SetExcFromWindowsErr,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, +func,PyErr_SetExcInfo,3.7,, +func,PyErr_SetFromErrno,3.2,, +func,PyErr_SetFromErrnoWithFilename,3.2,, +func,PyErr_SetFromErrnoWithFilenameObject,3.2,, +func,PyErr_SetFromErrnoWithFilenameObjects,3.7,, +func,PyErr_SetFromWindowsErr,3.7,on Windows, +func,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, +func,PyErr_SetHandledException,3.11,, +func,PyErr_SetImportError,3.7,, +func,PyErr_SetImportErrorSubclass,3.6,, +func,PyErr_SetInterrupt,3.2,, +func,PyErr_SetInterruptEx,3.10,, +func,PyErr_SetNone,3.2,, +func,PyErr_SetObject,3.2,, +func,PyErr_SetRaisedException,3.12,, +func,PyErr_SetString,3.2,, +func,PyErr_SyntaxLocation,3.2,, +func,PyErr_SyntaxLocationEx,3.7,, +func,PyErr_WarnEx,3.2,, +func,PyErr_WarnExplicit,3.2,, +func,PyErr_WarnFormat,3.2,, +func,PyErr_WriteUnraisable,3.2,, +func,PyEval_AcquireThread,3.2,, +func,PyEval_EvalCode,3.2,, +func,PyEval_EvalCodeEx,3.2,, +func,PyEval_EvalFrame,3.2,, +func,PyEval_EvalFrameEx,3.2,, +func,PyEval_GetBuiltins,3.2,, +func,PyEval_GetFrame,3.2,, +func,PyEval_GetFrameBuiltins,3.13,, +func,PyEval_GetFrameGlobals,3.13,, +func,PyEval_GetFrameLocals,3.13,, +func,PyEval_GetFuncDesc,3.2,, +func,PyEval_GetFuncName,3.2,, +func,PyEval_GetGlobals,3.2,, +func,PyEval_GetLocals,3.2,, +func,PyEval_InitThreads,3.2,, +func,PyEval_ReleaseThread,3.2,, +func,PyEval_RestoreThread,3.2,, +func,PyEval_SaveThread,3.2,, +data,PyExc_ArithmeticError,3.2,, +data,PyExc_AssertionError,3.2,, +data,PyExc_AttributeError,3.2,, +data,PyExc_BaseException,3.2,, +data,PyExc_BaseExceptionGroup,3.11,, +data,PyExc_BlockingIOError,3.7,, +data,PyExc_BrokenPipeError,3.7,, +data,PyExc_BufferError,3.2,, +data,PyExc_BytesWarning,3.2,, +data,PyExc_ChildProcessError,3.7,, +data,PyExc_ConnectionAbortedError,3.7,, +data,PyExc_ConnectionError,3.7,, +data,PyExc_ConnectionRefusedError,3.7,, +data,PyExc_ConnectionResetError,3.7,, +data,PyExc_DeprecationWarning,3.2,, +data,PyExc_EOFError,3.2,, +data,PyExc_EncodingWarning,3.10,, +data,PyExc_EnvironmentError,3.2,, +data,PyExc_Exception,3.2,, +data,PyExc_FileExistsError,3.7,, +data,PyExc_FileNotFoundError,3.7,, +data,PyExc_FloatingPointError,3.2,, +data,PyExc_FutureWarning,3.2,, +data,PyExc_GeneratorExit,3.2,, +data,PyExc_IOError,3.2,, +data,PyExc_ImportError,3.2,, +data,PyExc_ImportWarning,3.2,, +data,PyExc_IndentationError,3.2,, +data,PyExc_IndexError,3.2,, +data,PyExc_InterruptedError,3.7,, +data,PyExc_IsADirectoryError,3.7,, +data,PyExc_KeyError,3.2,, +data,PyExc_KeyboardInterrupt,3.2,, +data,PyExc_LookupError,3.2,, +data,PyExc_MemoryError,3.2,, +data,PyExc_ModuleNotFoundError,3.6,, +data,PyExc_NameError,3.2,, +data,PyExc_NotADirectoryError,3.7,, +data,PyExc_NotImplementedError,3.2,, +data,PyExc_OSError,3.2,, +data,PyExc_OverflowError,3.2,, +data,PyExc_PendingDeprecationWarning,3.2,, +data,PyExc_PermissionError,3.7,, +data,PyExc_ProcessLookupError,3.7,, +data,PyExc_RecursionError,3.7,, +data,PyExc_ReferenceError,3.2,, +data,PyExc_ResourceWarning,3.7,, +data,PyExc_RuntimeError,3.2,, +data,PyExc_RuntimeWarning,3.2,, +data,PyExc_StopAsyncIteration,3.7,, +data,PyExc_StopIteration,3.2,, +data,PyExc_SyntaxError,3.2,, +data,PyExc_SyntaxWarning,3.2,, +data,PyExc_SystemError,3.2,, +data,PyExc_SystemExit,3.2,, +data,PyExc_TabError,3.2,, +data,PyExc_TimeoutError,3.7,, +data,PyExc_TypeError,3.2,, +data,PyExc_UnboundLocalError,3.2,, +data,PyExc_UnicodeDecodeError,3.2,, +data,PyExc_UnicodeEncodeError,3.2,, +data,PyExc_UnicodeError,3.2,, +data,PyExc_UnicodeTranslateError,3.2,, +data,PyExc_UnicodeWarning,3.2,, +data,PyExc_UserWarning,3.2,, +data,PyExc_ValueError,3.2,, +data,PyExc_Warning,3.2,, +data,PyExc_WindowsError,3.7,on Windows, +data,PyExc_ZeroDivisionError,3.2,, +func,PyExceptionClass_Name,3.8,, +func,PyException_GetArgs,3.12,, +func,PyException_GetCause,3.2,, +func,PyException_GetContext,3.2,, +func,PyException_GetTraceback,3.2,, +func,PyException_SetArgs,3.12,, +func,PyException_SetCause,3.2,, +func,PyException_SetContext,3.2,, +func,PyException_SetTraceback,3.2,, +func,PyFile_FromFd,3.2,, +func,PyFile_GetLine,3.2,, +func,PyFile_WriteObject,3.2,, +func,PyFile_WriteString,3.2,, +data,PyFilter_Type,3.2,, +func,PyFloat_AsDouble,3.2,, +func,PyFloat_FromDouble,3.2,, +func,PyFloat_FromString,3.2,, +func,PyFloat_GetInfo,3.2,, +func,PyFloat_GetMax,3.2,, +func,PyFloat_GetMin,3.2,, +data,PyFloat_Type,3.2,, type,PyFrameObject,3.2,,opaque -function,PyFrame_GetCode,3.10,, -function,PyFrame_GetLineNumber,3.10,, -function,PyFrozenSet_New,3.2,, -var,PyFrozenSet_Type,3.2,, -function,PyGC_Collect,3.2,, -function,PyGC_Disable,3.10,, -function,PyGC_Enable,3.10,, -function,PyGC_IsEnabled,3.10,, -function,PyGILState_Ensure,3.2,, -function,PyGILState_GetThisThreadState,3.2,, -function,PyGILState_Release,3.2,, +func,PyFrame_GetCode,3.10,, +func,PyFrame_GetLineNumber,3.10,, +func,PyFrozenSet_New,3.2,, +data,PyFrozenSet_Type,3.2,, +func,PyGC_Collect,3.2,, +func,PyGC_Disable,3.10,, +func,PyGC_Enable,3.10,, +func,PyGC_IsEnabled,3.10,, +func,PyGILState_Ensure,3.2,, +func,PyGILState_GetThisThreadState,3.2,, +func,PyGILState_Release,3.2,, type,PyGILState_STATE,3.2,, type,PyGetSetDef,3.2,,full-abi -var,PyGetSetDescr_Type,3.2,, -function,PyImport_AddModule,3.2,, -function,PyImport_AddModuleObject,3.7,, -function,PyImport_AddModuleRef,3.13,, -function,PyImport_AppendInittab,3.2,, -function,PyImport_ExecCodeModule,3.2,, -function,PyImport_ExecCodeModuleEx,3.2,, -function,PyImport_ExecCodeModuleObject,3.7,, -function,PyImport_ExecCodeModuleWithPathnames,3.2,, -function,PyImport_GetImporter,3.2,, -function,PyImport_GetMagicNumber,3.2,, -function,PyImport_GetMagicTag,3.2,, -function,PyImport_GetModule,3.8,, -function,PyImport_GetModuleDict,3.2,, -function,PyImport_Import,3.2,, -function,PyImport_ImportFrozenModule,3.2,, -function,PyImport_ImportFrozenModuleObject,3.7,, -function,PyImport_ImportModule,3.2,, -function,PyImport_ImportModuleLevel,3.2,, -function,PyImport_ImportModuleLevelObject,3.7,, -function,PyImport_ImportModuleNoBlock,3.2,, -function,PyImport_ReloadModule,3.2,, -function,PyIndex_Check,3.8,, +data,PyGetSetDescr_Type,3.2,, +func,PyImport_AddModule,3.2,, +func,PyImport_AddModuleObject,3.7,, +func,PyImport_AddModuleRef,3.13,, +func,PyImport_AppendInittab,3.2,, +func,PyImport_ExecCodeModule,3.2,, +func,PyImport_ExecCodeModuleEx,3.2,, +func,PyImport_ExecCodeModuleObject,3.7,, +func,PyImport_ExecCodeModuleWithPathnames,3.2,, +func,PyImport_GetImporter,3.2,, +func,PyImport_GetMagicNumber,3.2,, +func,PyImport_GetMagicTag,3.2,, +func,PyImport_GetModule,3.8,, +func,PyImport_GetModuleDict,3.2,, +func,PyImport_Import,3.2,, +func,PyImport_ImportFrozenModule,3.2,, +func,PyImport_ImportFrozenModuleObject,3.7,, +func,PyImport_ImportModule,3.2,, +func,PyImport_ImportModuleLevel,3.2,, +func,PyImport_ImportModuleLevelObject,3.7,, +func,PyImport_ImportModuleNoBlock,3.2,, +func,PyImport_ReloadModule,3.2,, +func,PyIndex_Check,3.8,, type,PyInterpreterState,3.2,,opaque -function,PyInterpreterState_Clear,3.2,, -function,PyInterpreterState_Delete,3.2,, -function,PyInterpreterState_Get,3.9,, -function,PyInterpreterState_GetDict,3.8,, -function,PyInterpreterState_GetID,3.7,, -function,PyInterpreterState_New,3.2,, -function,PyIter_Check,3.8,, -function,PyIter_Next,3.2,, -function,PyIter_Send,3.10,, -var,PyListIter_Type,3.2,, -var,PyListRevIter_Type,3.2,, -function,PyList_Append,3.2,, -function,PyList_AsTuple,3.2,, -function,PyList_GetItem,3.2,, -function,PyList_GetItemRef,3.13,, -function,PyList_GetSlice,3.2,, -function,PyList_Insert,3.2,, -function,PyList_New,3.2,, -function,PyList_Reverse,3.2,, -function,PyList_SetItem,3.2,, -function,PyList_SetSlice,3.2,, -function,PyList_Size,3.2,, -function,PyList_Sort,3.2,, -var,PyList_Type,3.2,, +func,PyInterpreterState_Clear,3.2,, +func,PyInterpreterState_Delete,3.2,, +func,PyInterpreterState_Get,3.9,, +func,PyInterpreterState_GetDict,3.8,, +func,PyInterpreterState_GetID,3.7,, +func,PyInterpreterState_New,3.2,, +func,PyIter_Check,3.8,, +func,PyIter_Next,3.2,, +func,PyIter_NextItem,3.14,, +func,PyIter_Send,3.10,, +data,PyListIter_Type,3.2,, +data,PyListRevIter_Type,3.2,, +func,PyList_Append,3.2,, +func,PyList_AsTuple,3.2,, +func,PyList_GetItem,3.2,, +func,PyList_GetItemRef,3.13,, +func,PyList_GetSlice,3.2,, +func,PyList_Insert,3.2,, +func,PyList_New,3.2,, +func,PyList_Reverse,3.2,, +func,PyList_SetItem,3.2,, +func,PyList_SetSlice,3.2,, +func,PyList_Size,3.2,, +func,PyList_Sort,3.2,, +data,PyList_Type,3.2,, type,PyLongObject,3.2,,opaque -var,PyLongRangeIter_Type,3.2,, -function,PyLong_AsDouble,3.2,, -function,PyLong_AsInt,3.13,, -function,PyLong_AsLong,3.2,, -function,PyLong_AsLongAndOverflow,3.2,, -function,PyLong_AsLongLong,3.2,, -function,PyLong_AsLongLongAndOverflow,3.2,, -function,PyLong_AsSize_t,3.2,, -function,PyLong_AsSsize_t,3.2,, -function,PyLong_AsUnsignedLong,3.2,, -function,PyLong_AsUnsignedLongLong,3.2,, -function,PyLong_AsUnsignedLongLongMask,3.2,, -function,PyLong_AsUnsignedLongMask,3.2,, -function,PyLong_AsVoidPtr,3.2,, -function,PyLong_FromDouble,3.2,, -function,PyLong_FromLong,3.2,, -function,PyLong_FromLongLong,3.2,, -function,PyLong_FromSize_t,3.2,, -function,PyLong_FromSsize_t,3.2,, -function,PyLong_FromString,3.2,, -function,PyLong_FromUnsignedLong,3.2,, -function,PyLong_FromUnsignedLongLong,3.2,, -function,PyLong_FromVoidPtr,3.2,, -function,PyLong_GetInfo,3.2,, -var,PyLong_Type,3.2,, -var,PyMap_Type,3.2,, -function,PyMapping_Check,3.2,, -function,PyMapping_GetItemString,3.2,, -function,PyMapping_GetOptionalItem,3.13,, -function,PyMapping_GetOptionalItemString,3.13,, -function,PyMapping_HasKey,3.2,, -function,PyMapping_HasKeyString,3.2,, -function,PyMapping_HasKeyStringWithError,3.13,, -function,PyMapping_HasKeyWithError,3.13,, -function,PyMapping_Items,3.2,, -function,PyMapping_Keys,3.2,, -function,PyMapping_Length,3.2,, -function,PyMapping_SetItemString,3.2,, -function,PyMapping_Size,3.2,, -function,PyMapping_Values,3.2,, -function,PyMem_Calloc,3.7,, -function,PyMem_Free,3.2,, -function,PyMem_Malloc,3.2,, -function,PyMem_RawCalloc,3.13,, -function,PyMem_RawFree,3.13,, -function,PyMem_RawMalloc,3.13,, -function,PyMem_RawRealloc,3.13,, -function,PyMem_Realloc,3.2,, +data,PyLongRangeIter_Type,3.2,, +func,PyLong_AsDouble,3.2,, +func,PyLong_AsInt,3.13,, +func,PyLong_AsInt32,3.14,, +func,PyLong_AsInt64,3.14,, +func,PyLong_AsLong,3.2,, +func,PyLong_AsLongAndOverflow,3.2,, +func,PyLong_AsLongLong,3.2,, +func,PyLong_AsLongLongAndOverflow,3.2,, +func,PyLong_AsSize_t,3.2,, +func,PyLong_AsSsize_t,3.2,, +func,PyLong_AsUInt32,3.14,, +func,PyLong_AsUInt64,3.14,, +func,PyLong_AsUnsignedLong,3.2,, +func,PyLong_AsUnsignedLongLong,3.2,, +func,PyLong_AsUnsignedLongLongMask,3.2,, +func,PyLong_AsUnsignedLongMask,3.2,, +func,PyLong_AsVoidPtr,3.2,, +func,PyLong_FromDouble,3.2,, +func,PyLong_FromInt32,3.14,, +func,PyLong_FromInt64,3.14,, +func,PyLong_FromLong,3.2,, +func,PyLong_FromLongLong,3.2,, +func,PyLong_FromSize_t,3.2,, +func,PyLong_FromSsize_t,3.2,, +func,PyLong_FromString,3.2,, +func,PyLong_FromUInt32,3.14,, +func,PyLong_FromUInt64,3.14,, +func,PyLong_FromUnsignedLong,3.2,, +func,PyLong_FromUnsignedLongLong,3.2,, +func,PyLong_FromVoidPtr,3.2,, +func,PyLong_GetInfo,3.2,, +data,PyLong_Type,3.2,, +data,PyMap_Type,3.2,, +func,PyMapping_Check,3.2,, +func,PyMapping_GetItemString,3.2,, +func,PyMapping_GetOptionalItem,3.13,, +func,PyMapping_GetOptionalItemString,3.13,, +func,PyMapping_HasKey,3.2,, +func,PyMapping_HasKeyString,3.2,, +func,PyMapping_HasKeyStringWithError,3.13,, +func,PyMapping_HasKeyWithError,3.13,, +func,PyMapping_Items,3.2,, +func,PyMapping_Keys,3.2,, +func,PyMapping_Length,3.2,, +func,PyMapping_SetItemString,3.2,, +func,PyMapping_Size,3.2,, +func,PyMapping_Values,3.2,, +func,PyMem_Calloc,3.7,, +func,PyMem_Free,3.2,, +func,PyMem_Malloc,3.2,, +func,PyMem_RawCalloc,3.13,, +func,PyMem_RawFree,3.13,, +func,PyMem_RawMalloc,3.13,, +func,PyMem_RawRealloc,3.13,, +func,PyMem_Realloc,3.2,, type,PyMemberDef,3.2,,full-abi -var,PyMemberDescr_Type,3.2,, -function,PyMember_GetOne,3.2,, -function,PyMember_SetOne,3.2,, -function,PyMemoryView_FromBuffer,3.11,, -function,PyMemoryView_FromMemory,3.7,, -function,PyMemoryView_FromObject,3.2,, -function,PyMemoryView_GetContiguous,3.2,, -var,PyMemoryView_Type,3.2,, +data,PyMemberDescr_Type,3.2,, +func,PyMember_GetOne,3.2,, +func,PyMember_SetOne,3.2,, +func,PyMemoryView_FromBuffer,3.11,, +func,PyMemoryView_FromMemory,3.7,, +func,PyMemoryView_FromObject,3.2,, +func,PyMemoryView_GetContiguous,3.2,, +data,PyMemoryView_Type,3.2,, type,PyMethodDef,3.2,,full-abi -var,PyMethodDescr_Type,3.2,, +data,PyMethodDescr_Type,3.2,, type,PyModuleDef,3.2,,full-abi type,PyModuleDef_Base,3.2,,full-abi -function,PyModuleDef_Init,3.5,, -var,PyModuleDef_Type,3.5,, -function,PyModule_Add,3.13,, -function,PyModule_AddFunctions,3.7,, -function,PyModule_AddIntConstant,3.2,, -function,PyModule_AddObject,3.2,, -function,PyModule_AddObjectRef,3.10,, -function,PyModule_AddStringConstant,3.2,, -function,PyModule_AddType,3.10,, -function,PyModule_Create2,3.2,, -function,PyModule_ExecDef,3.7,, -function,PyModule_FromDefAndSpec2,3.7,, -function,PyModule_GetDef,3.2,, -function,PyModule_GetDict,3.2,, -function,PyModule_GetFilename,3.2,, -function,PyModule_GetFilenameObject,3.2,, -function,PyModule_GetName,3.2,, -function,PyModule_GetNameObject,3.7,, -function,PyModule_GetState,3.2,, -function,PyModule_New,3.2,, -function,PyModule_NewObject,3.7,, -function,PyModule_SetDocString,3.7,, -var,PyModule_Type,3.2,, -function,PyNumber_Absolute,3.2,, -function,PyNumber_Add,3.2,, -function,PyNumber_And,3.2,, -function,PyNumber_AsSsize_t,3.2,, -function,PyNumber_Check,3.2,, -function,PyNumber_Divmod,3.2,, -function,PyNumber_Float,3.2,, -function,PyNumber_FloorDivide,3.2,, -function,PyNumber_InPlaceAdd,3.2,, -function,PyNumber_InPlaceAnd,3.2,, -function,PyNumber_InPlaceFloorDivide,3.2,, -function,PyNumber_InPlaceLshift,3.2,, -function,PyNumber_InPlaceMatrixMultiply,3.7,, -function,PyNumber_InPlaceMultiply,3.2,, -function,PyNumber_InPlaceOr,3.2,, -function,PyNumber_InPlacePower,3.2,, -function,PyNumber_InPlaceRemainder,3.2,, -function,PyNumber_InPlaceRshift,3.2,, -function,PyNumber_InPlaceSubtract,3.2,, -function,PyNumber_InPlaceTrueDivide,3.2,, -function,PyNumber_InPlaceXor,3.2,, -function,PyNumber_Index,3.2,, -function,PyNumber_Invert,3.2,, -function,PyNumber_Long,3.2,, -function,PyNumber_Lshift,3.2,, -function,PyNumber_MatrixMultiply,3.7,, -function,PyNumber_Multiply,3.2,, -function,PyNumber_Negative,3.2,, -function,PyNumber_Or,3.2,, -function,PyNumber_Positive,3.2,, -function,PyNumber_Power,3.2,, -function,PyNumber_Remainder,3.2,, -function,PyNumber_Rshift,3.2,, -function,PyNumber_Subtract,3.2,, -function,PyNumber_ToBase,3.2,, -function,PyNumber_TrueDivide,3.2,, -function,PyNumber_Xor,3.2,, -function,PyOS_AfterFork,3.2,on platforms with fork(), -function,PyOS_AfterFork_Child,3.7,on platforms with fork(), -function,PyOS_AfterFork_Parent,3.7,on platforms with fork(), -function,PyOS_BeforeFork,3.7,on platforms with fork(), -function,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, -function,PyOS_FSPath,3.6,, -var,PyOS_InputHook,3.2,, -function,PyOS_InterruptOccurred,3.2,, -function,PyOS_double_to_string,3.2,, -function,PyOS_getsig,3.2,, -function,PyOS_mystricmp,3.2,, -function,PyOS_mystrnicmp,3.2,, -function,PyOS_setsig,3.2,, +func,PyModuleDef_Init,3.5,, +data,PyModuleDef_Type,3.5,, +func,PyModule_Add,3.13,, +func,PyModule_AddFunctions,3.7,, +func,PyModule_AddIntConstant,3.2,, +func,PyModule_AddObject,3.2,, +func,PyModule_AddObjectRef,3.10,, +func,PyModule_AddStringConstant,3.2,, +func,PyModule_AddType,3.10,, +func,PyModule_Create2,3.2,, +func,PyModule_ExecDef,3.7,, +func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_GetDef,3.2,, +func,PyModule_GetDict,3.2,, +func,PyModule_GetFilename,3.2,, +func,PyModule_GetFilenameObject,3.2,, +func,PyModule_GetName,3.2,, +func,PyModule_GetNameObject,3.7,, +func,PyModule_GetState,3.2,, +func,PyModule_New,3.2,, +func,PyModule_NewObject,3.7,, +func,PyModule_SetDocString,3.7,, +data,PyModule_Type,3.2,, +func,PyNumber_Absolute,3.2,, +func,PyNumber_Add,3.2,, +func,PyNumber_And,3.2,, +func,PyNumber_AsSsize_t,3.2,, +func,PyNumber_Check,3.2,, +func,PyNumber_Divmod,3.2,, +func,PyNumber_Float,3.2,, +func,PyNumber_FloorDivide,3.2,, +func,PyNumber_InPlaceAdd,3.2,, +func,PyNumber_InPlaceAnd,3.2,, +func,PyNumber_InPlaceFloorDivide,3.2,, +func,PyNumber_InPlaceLshift,3.2,, +func,PyNumber_InPlaceMatrixMultiply,3.7,, +func,PyNumber_InPlaceMultiply,3.2,, +func,PyNumber_InPlaceOr,3.2,, +func,PyNumber_InPlacePower,3.2,, +func,PyNumber_InPlaceRemainder,3.2,, +func,PyNumber_InPlaceRshift,3.2,, +func,PyNumber_InPlaceSubtract,3.2,, +func,PyNumber_InPlaceTrueDivide,3.2,, +func,PyNumber_InPlaceXor,3.2,, +func,PyNumber_Index,3.2,, +func,PyNumber_Invert,3.2,, +func,PyNumber_Long,3.2,, +func,PyNumber_Lshift,3.2,, +func,PyNumber_MatrixMultiply,3.7,, +func,PyNumber_Multiply,3.2,, +func,PyNumber_Negative,3.2,, +func,PyNumber_Or,3.2,, +func,PyNumber_Positive,3.2,, +func,PyNumber_Power,3.2,, +func,PyNumber_Remainder,3.2,, +func,PyNumber_Rshift,3.2,, +func,PyNumber_Subtract,3.2,, +func,PyNumber_ToBase,3.2,, +func,PyNumber_TrueDivide,3.2,, +func,PyNumber_Xor,3.2,, +func,PyOS_AfterFork,3.2,on platforms with fork(), +func,PyOS_AfterFork_Child,3.7,on platforms with fork(), +func,PyOS_AfterFork_Parent,3.7,on platforms with fork(), +func,PyOS_BeforeFork,3.7,on platforms with fork(), +func,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, +func,PyOS_FSPath,3.6,, +data,PyOS_InputHook,3.2,, +func,PyOS_InterruptOccurred,3.2,, +func,PyOS_double_to_string,3.2,, +func,PyOS_getsig,3.2,, +func,PyOS_mystricmp,3.2,, +func,PyOS_mystrnicmp,3.2,, +func,PyOS_setsig,3.2,, type,PyOS_sighandler_t,3.2,, -function,PyOS_snprintf,3.2,, -function,PyOS_string_to_double,3.2,, -function,PyOS_strtol,3.2,, -function,PyOS_strtoul,3.2,, -function,PyOS_vsnprintf,3.2,, +func,PyOS_snprintf,3.2,, +func,PyOS_string_to_double,3.2,, +func,PyOS_strtol,3.2,, +func,PyOS_strtoul,3.2,, +func,PyOS_vsnprintf,3.2,, type,PyObject,3.2,,members member,PyObject.ob_refcnt,3.2,, member,PyObject.ob_type,3.2,, -function,PyObject_ASCII,3.2,, -function,PyObject_AsFileDescriptor,3.2,, -function,PyObject_Bytes,3.2,, -function,PyObject_Call,3.2,, -function,PyObject_CallFunction,3.2,, -function,PyObject_CallFunctionObjArgs,3.2,, -function,PyObject_CallMethod,3.2,, -function,PyObject_CallMethodObjArgs,3.2,, -function,PyObject_CallNoArgs,3.10,, -function,PyObject_CallObject,3.2,, -function,PyObject_Calloc,3.7,, -function,PyObject_CheckBuffer,3.11,, -function,PyObject_ClearWeakRefs,3.2,, -function,PyObject_CopyData,3.11,, -function,PyObject_DelAttr,3.13,, -function,PyObject_DelAttrString,3.13,, -function,PyObject_DelItem,3.2,, -function,PyObject_DelItemString,3.2,, -function,PyObject_Dir,3.2,, -function,PyObject_Format,3.2,, -function,PyObject_Free,3.2,, -function,PyObject_GC_Del,3.2,, -function,PyObject_GC_IsFinalized,3.9,, -function,PyObject_GC_IsTracked,3.9,, -function,PyObject_GC_Track,3.2,, -function,PyObject_GC_UnTrack,3.2,, -function,PyObject_GenericGetAttr,3.2,, -function,PyObject_GenericGetDict,3.10,, -function,PyObject_GenericSetAttr,3.2,, -function,PyObject_GenericSetDict,3.7,, -function,PyObject_GetAIter,3.10,, -function,PyObject_GetAttr,3.2,, -function,PyObject_GetAttrString,3.2,, -function,PyObject_GetBuffer,3.11,, -function,PyObject_GetItem,3.2,, -function,PyObject_GetIter,3.2,, -function,PyObject_GetOptionalAttr,3.13,, -function,PyObject_GetOptionalAttrString,3.13,, -function,PyObject_GetTypeData,3.12,, -function,PyObject_HasAttr,3.2,, -function,PyObject_HasAttrString,3.2,, -function,PyObject_HasAttrStringWithError,3.13,, -function,PyObject_HasAttrWithError,3.13,, -function,PyObject_Hash,3.2,, -function,PyObject_HashNotImplemented,3.2,, -function,PyObject_Init,3.2,, -function,PyObject_InitVar,3.2,, -function,PyObject_IsInstance,3.2,, -function,PyObject_IsSubclass,3.2,, -function,PyObject_IsTrue,3.2,, -function,PyObject_Length,3.2,, -function,PyObject_Malloc,3.2,, -function,PyObject_Not,3.2,, -function,PyObject_Realloc,3.2,, -function,PyObject_Repr,3.2,, -function,PyObject_RichCompare,3.2,, -function,PyObject_RichCompareBool,3.2,, -function,PyObject_SelfIter,3.2,, -function,PyObject_SetAttr,3.2,, -function,PyObject_SetAttrString,3.2,, -function,PyObject_SetItem,3.2,, -function,PyObject_Size,3.2,, -function,PyObject_Str,3.2,, -function,PyObject_Type,3.2,, -function,PyObject_Vectorcall,3.12,, -function,PyObject_VectorcallMethod,3.12,, -var,PyProperty_Type,3.2,, -var,PyRangeIter_Type,3.2,, -var,PyRange_Type,3.2,, -var,PyReversed_Type,3.2,, -function,PySeqIter_New,3.2,, -var,PySeqIter_Type,3.2,, -function,PySequence_Check,3.2,, -function,PySequence_Concat,3.2,, -function,PySequence_Contains,3.2,, -function,PySequence_Count,3.2,, -function,PySequence_DelItem,3.2,, -function,PySequence_DelSlice,3.2,, -function,PySequence_Fast,3.2,, -function,PySequence_GetItem,3.2,, -function,PySequence_GetSlice,3.2,, -function,PySequence_In,3.2,, -function,PySequence_InPlaceConcat,3.2,, -function,PySequence_InPlaceRepeat,3.2,, -function,PySequence_Index,3.2,, -function,PySequence_Length,3.2,, -function,PySequence_List,3.2,, -function,PySequence_Repeat,3.2,, -function,PySequence_SetItem,3.2,, -function,PySequence_SetSlice,3.2,, -function,PySequence_Size,3.2,, -function,PySequence_Tuple,3.2,, -var,PySetIter_Type,3.2,, -function,PySet_Add,3.2,, -function,PySet_Clear,3.2,, -function,PySet_Contains,3.2,, -function,PySet_Discard,3.2,, -function,PySet_New,3.2,, -function,PySet_Pop,3.2,, -function,PySet_Size,3.2,, -var,PySet_Type,3.2,, -function,PySlice_AdjustIndices,3.7,, -function,PySlice_GetIndices,3.2,, -function,PySlice_GetIndicesEx,3.2,, -function,PySlice_New,3.2,, -var,PySlice_Type,3.2,, -function,PySlice_Unpack,3.7,, -function,PyState_AddModule,3.3,, -function,PyState_FindModule,3.2,, -function,PyState_RemoveModule,3.3,, +func,PyObject_ASCII,3.2,, +func,PyObject_AsFileDescriptor,3.2,, +func,PyObject_Bytes,3.2,, +func,PyObject_Call,3.2,, +func,PyObject_CallFunction,3.2,, +func,PyObject_CallFunctionObjArgs,3.2,, +func,PyObject_CallMethod,3.2,, +func,PyObject_CallMethodObjArgs,3.2,, +func,PyObject_CallNoArgs,3.10,, +func,PyObject_CallObject,3.2,, +func,PyObject_Calloc,3.7,, +func,PyObject_CheckBuffer,3.11,, +func,PyObject_ClearWeakRefs,3.2,, +func,PyObject_CopyData,3.11,, +func,PyObject_DelAttr,3.13,, +func,PyObject_DelAttrString,3.13,, +func,PyObject_DelItem,3.2,, +func,PyObject_DelItemString,3.2,, +func,PyObject_Dir,3.2,, +func,PyObject_Format,3.2,, +func,PyObject_Free,3.2,, +func,PyObject_GC_Del,3.2,, +func,PyObject_GC_IsFinalized,3.9,, +func,PyObject_GC_IsTracked,3.9,, +func,PyObject_GC_Track,3.2,, +func,PyObject_GC_UnTrack,3.2,, +func,PyObject_GenericGetAttr,3.2,, +func,PyObject_GenericGetDict,3.10,, +func,PyObject_GenericSetAttr,3.2,, +func,PyObject_GenericSetDict,3.7,, +func,PyObject_GetAIter,3.10,, +func,PyObject_GetAttr,3.2,, +func,PyObject_GetAttrString,3.2,, +func,PyObject_GetBuffer,3.11,, +func,PyObject_GetItem,3.2,, +func,PyObject_GetIter,3.2,, +func,PyObject_GetOptionalAttr,3.13,, +func,PyObject_GetOptionalAttrString,3.13,, +func,PyObject_GetTypeData,3.12,, +func,PyObject_HasAttr,3.2,, +func,PyObject_HasAttrString,3.2,, +func,PyObject_HasAttrStringWithError,3.13,, +func,PyObject_HasAttrWithError,3.13,, +func,PyObject_Hash,3.2,, +func,PyObject_HashNotImplemented,3.2,, +func,PyObject_Init,3.2,, +func,PyObject_InitVar,3.2,, +func,PyObject_IsInstance,3.2,, +func,PyObject_IsSubclass,3.2,, +func,PyObject_IsTrue,3.2,, +func,PyObject_Length,3.2,, +func,PyObject_Malloc,3.2,, +func,PyObject_Not,3.2,, +func,PyObject_Realloc,3.2,, +func,PyObject_Repr,3.2,, +func,PyObject_RichCompare,3.2,, +func,PyObject_RichCompareBool,3.2,, +func,PyObject_SelfIter,3.2,, +func,PyObject_SetAttr,3.2,, +func,PyObject_SetAttrString,3.2,, +func,PyObject_SetItem,3.2,, +func,PyObject_Size,3.2,, +func,PyObject_Str,3.2,, +func,PyObject_Type,3.2,, +func,PyObject_Vectorcall,3.12,, +func,PyObject_VectorcallMethod,3.12,, +data,PyProperty_Type,3.2,, +data,PyRangeIter_Type,3.2,, +data,PyRange_Type,3.2,, +data,PyReversed_Type,3.2,, +func,PySeqIter_New,3.2,, +data,PySeqIter_Type,3.2,, +func,PySequence_Check,3.2,, +func,PySequence_Concat,3.2,, +func,PySequence_Contains,3.2,, +func,PySequence_Count,3.2,, +func,PySequence_DelItem,3.2,, +func,PySequence_DelSlice,3.2,, +func,PySequence_Fast,3.2,, +func,PySequence_GetItem,3.2,, +func,PySequence_GetSlice,3.2,, +func,PySequence_In,3.2,, +func,PySequence_InPlaceConcat,3.2,, +func,PySequence_InPlaceRepeat,3.2,, +func,PySequence_Index,3.2,, +func,PySequence_Length,3.2,, +func,PySequence_List,3.2,, +func,PySequence_Repeat,3.2,, +func,PySequence_SetItem,3.2,, +func,PySequence_SetSlice,3.2,, +func,PySequence_Size,3.2,, +func,PySequence_Tuple,3.2,, +data,PySetIter_Type,3.2,, +func,PySet_Add,3.2,, +func,PySet_Clear,3.2,, +func,PySet_Contains,3.2,, +func,PySet_Discard,3.2,, +func,PySet_New,3.2,, +func,PySet_Pop,3.2,, +func,PySet_Size,3.2,, +data,PySet_Type,3.2,, +func,PySlice_AdjustIndices,3.7,, +func,PySlice_GetIndices,3.2,, +func,PySlice_GetIndicesEx,3.2,, +func,PySlice_New,3.2,, +data,PySlice_Type,3.2,, +func,PySlice_Unpack,3.7,, +func,PyState_AddModule,3.3,, +func,PyState_FindModule,3.2,, +func,PyState_RemoveModule,3.3,, type,PyStructSequence_Desc,3.2,,full-abi type,PyStructSequence_Field,3.2,,full-abi -function,PyStructSequence_GetItem,3.2,, -function,PyStructSequence_New,3.2,, -function,PyStructSequence_NewType,3.2,, -function,PyStructSequence_SetItem,3.2,, -var,PyStructSequence_UnnamedField,3.11,, -var,PySuper_Type,3.2,, -function,PySys_Audit,3.13,, -function,PySys_AuditTuple,3.13,, -function,PySys_FormatStderr,3.2,, -function,PySys_FormatStdout,3.2,, -function,PySys_GetObject,3.2,, -function,PySys_GetXOptions,3.7,, -function,PySys_ResetWarnOptions,3.2,, -function,PySys_SetObject,3.2,, -function,PySys_WriteStderr,3.2,, -function,PySys_WriteStdout,3.2,, +func,PyStructSequence_GetItem,3.2,, +func,PyStructSequence_New,3.2,, +func,PyStructSequence_NewType,3.2,, +func,PyStructSequence_SetItem,3.2,, +data,PyStructSequence_UnnamedField,3.11,, +data,PySuper_Type,3.2,, +func,PySys_Audit,3.13,, +func,PySys_AuditTuple,3.13,, +func,PySys_FormatStderr,3.2,, +func,PySys_FormatStdout,3.2,, +func,PySys_GetObject,3.2,, +func,PySys_GetXOptions,3.7,, +func,PySys_ResetWarnOptions,3.2,, +func,PySys_SetArgv,3.2,, +func,PySys_SetArgvEx,3.2,, +func,PySys_SetObject,3.2,, +func,PySys_WriteStderr,3.2,, +func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque -function,PyThreadState_Clear,3.2,, -function,PyThreadState_Delete,3.2,, -function,PyThreadState_Get,3.2,, -function,PyThreadState_GetDict,3.2,, -function,PyThreadState_GetFrame,3.10,, -function,PyThreadState_GetID,3.10,, -function,PyThreadState_GetInterpreter,3.10,, -function,PyThreadState_New,3.2,, -function,PyThreadState_SetAsyncExc,3.2,, -function,PyThreadState_Swap,3.2,, -function,PyThread_GetInfo,3.3,, -function,PyThread_ReInitTLS,3.2,, -function,PyThread_acquire_lock,3.2,, -function,PyThread_acquire_lock_timed,3.2,, -function,PyThread_allocate_lock,3.2,, -function,PyThread_create_key,3.2,, -function,PyThread_delete_key,3.2,, -function,PyThread_delete_key_value,3.2,, -function,PyThread_exit_thread,3.2,, -function,PyThread_free_lock,3.2,, -function,PyThread_get_key_value,3.2,, -function,PyThread_get_stacksize,3.2,, -function,PyThread_get_thread_ident,3.2,, -function,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, -function,PyThread_init_thread,3.2,, -function,PyThread_release_lock,3.2,, -function,PyThread_set_key_value,3.2,, -function,PyThread_set_stacksize,3.2,, -function,PyThread_start_new_thread,3.2,, -function,PyThread_tss_alloc,3.7,, -function,PyThread_tss_create,3.7,, -function,PyThread_tss_delete,3.7,, -function,PyThread_tss_free,3.7,, -function,PyThread_tss_get,3.7,, -function,PyThread_tss_is_created,3.7,, -function,PyThread_tss_set,3.7,, -function,PyTraceBack_Here,3.2,, -function,PyTraceBack_Print,3.2,, -var,PyTraceBack_Type,3.2,, -var,PyTupleIter_Type,3.2,, -function,PyTuple_GetItem,3.2,, -function,PyTuple_GetSlice,3.2,, -function,PyTuple_New,3.2,, -function,PyTuple_Pack,3.2,, -function,PyTuple_SetItem,3.2,, -function,PyTuple_Size,3.2,, -var,PyTuple_Type,3.2,, +func,PyThreadState_Clear,3.2,, +func,PyThreadState_Delete,3.2,, +func,PyThreadState_Get,3.2,, +func,PyThreadState_GetDict,3.2,, +func,PyThreadState_GetFrame,3.10,, +func,PyThreadState_GetID,3.10,, +func,PyThreadState_GetInterpreter,3.10,, +func,PyThreadState_New,3.2,, +func,PyThreadState_SetAsyncExc,3.2,, +func,PyThreadState_Swap,3.2,, +func,PyThread_GetInfo,3.3,, +func,PyThread_ReInitTLS,3.2,, +func,PyThread_acquire_lock,3.2,, +func,PyThread_acquire_lock_timed,3.2,, +func,PyThread_allocate_lock,3.2,, +func,PyThread_create_key,3.2,, +func,PyThread_delete_key,3.2,, +func,PyThread_delete_key_value,3.2,, +func,PyThread_exit_thread,3.2,, +func,PyThread_free_lock,3.2,, +func,PyThread_get_key_value,3.2,, +func,PyThread_get_stacksize,3.2,, +func,PyThread_get_thread_ident,3.2,, +func,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, +func,PyThread_init_thread,3.2,, +func,PyThread_release_lock,3.2,, +func,PyThread_set_key_value,3.2,, +func,PyThread_set_stacksize,3.2,, +func,PyThread_start_new_thread,3.2,, +func,PyThread_tss_alloc,3.7,, +func,PyThread_tss_create,3.7,, +func,PyThread_tss_delete,3.7,, +func,PyThread_tss_free,3.7,, +func,PyThread_tss_get,3.7,, +func,PyThread_tss_is_created,3.7,, +func,PyThread_tss_set,3.7,, +func,PyTraceBack_Here,3.2,, +func,PyTraceBack_Print,3.2,, +data,PyTraceBack_Type,3.2,, +data,PyTupleIter_Type,3.2,, +func,PyTuple_GetItem,3.2,, +func,PyTuple_GetSlice,3.2,, +func,PyTuple_New,3.2,, +func,PyTuple_Pack,3.2,, +func,PyTuple_SetItem,3.2,, +func,PyTuple_Size,3.2,, +data,PyTuple_Type,3.2,, type,PyTypeObject,3.2,,opaque -function,PyType_ClearCache,3.2,, -function,PyType_FromMetaclass,3.12,, -function,PyType_FromModuleAndSpec,3.10,, -function,PyType_FromSpec,3.2,, -function,PyType_FromSpecWithBases,3.3,, -function,PyType_GenericAlloc,3.2,, -function,PyType_GenericNew,3.2,, -function,PyType_GetFlags,3.2,, -function,PyType_GetModule,3.10,, -function,PyType_GetModuleState,3.10,, -function,PyType_GetName,3.11,, -function,PyType_GetQualName,3.11,, -function,PyType_GetSlot,3.4,, -function,PyType_GetTypeDataSize,3.12,, -function,PyType_IsSubtype,3.2,, -function,PyType_Modified,3.2,, -function,PyType_Ready,3.2,, +func,PyType_ClearCache,3.2,, +func,PyType_Freeze,3.14,, +func,PyType_FromMetaclass,3.12,, +func,PyType_FromModuleAndSpec,3.10,, +func,PyType_FromSpec,3.2,, +func,PyType_FromSpecWithBases,3.3,, +func,PyType_GenericAlloc,3.2,, +func,PyType_GenericNew,3.2,, +func,PyType_GetBaseByToken,3.14,, +func,PyType_GetFlags,3.2,, +func,PyType_GetFullyQualifiedName,3.13,, +func,PyType_GetModule,3.10,, +func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleName,3.13,, +func,PyType_GetModuleState,3.10,, +func,PyType_GetName,3.11,, +func,PyType_GetQualName,3.11,, +func,PyType_GetSlot,3.4,, +func,PyType_GetTypeDataSize,3.12,, +func,PyType_IsSubtype,3.2,, +func,PyType_Modified,3.2,, +func,PyType_Ready,3.2,, type,PyType_Slot,3.2,,full-abi type,PyType_Spec,3.2,,full-abi -var,PyType_Type,3.2,, -function,PyUnicodeDecodeError_Create,3.2,, -function,PyUnicodeDecodeError_GetEncoding,3.2,, -function,PyUnicodeDecodeError_GetEnd,3.2,, -function,PyUnicodeDecodeError_GetObject,3.2,, -function,PyUnicodeDecodeError_GetReason,3.2,, -function,PyUnicodeDecodeError_GetStart,3.2,, -function,PyUnicodeDecodeError_SetEnd,3.2,, -function,PyUnicodeDecodeError_SetReason,3.2,, -function,PyUnicodeDecodeError_SetStart,3.2,, -function,PyUnicodeEncodeError_GetEncoding,3.2,, -function,PyUnicodeEncodeError_GetEnd,3.2,, -function,PyUnicodeEncodeError_GetObject,3.2,, -function,PyUnicodeEncodeError_GetReason,3.2,, -function,PyUnicodeEncodeError_GetStart,3.2,, -function,PyUnicodeEncodeError_SetEnd,3.2,, -function,PyUnicodeEncodeError_SetReason,3.2,, -function,PyUnicodeEncodeError_SetStart,3.2,, -var,PyUnicodeIter_Type,3.2,, -function,PyUnicodeTranslateError_GetEnd,3.2,, -function,PyUnicodeTranslateError_GetObject,3.2,, -function,PyUnicodeTranslateError_GetReason,3.2,, -function,PyUnicodeTranslateError_GetStart,3.2,, -function,PyUnicodeTranslateError_SetEnd,3.2,, -function,PyUnicodeTranslateError_SetReason,3.2,, -function,PyUnicodeTranslateError_SetStart,3.2,, -function,PyUnicode_Append,3.2,, -function,PyUnicode_AppendAndDel,3.2,, -function,PyUnicode_AsASCIIString,3.2,, -function,PyUnicode_AsCharmapString,3.2,, -function,PyUnicode_AsDecodedObject,3.2,, -function,PyUnicode_AsDecodedUnicode,3.2,, -function,PyUnicode_AsEncodedObject,3.2,, -function,PyUnicode_AsEncodedString,3.2,, -function,PyUnicode_AsEncodedUnicode,3.2,, -function,PyUnicode_AsLatin1String,3.2,, -function,PyUnicode_AsMBCSString,3.7,on Windows, -function,PyUnicode_AsRawUnicodeEscapeString,3.2,, -function,PyUnicode_AsUCS4,3.7,, -function,PyUnicode_AsUCS4Copy,3.7,, -function,PyUnicode_AsUTF16String,3.2,, -function,PyUnicode_AsUTF32String,3.2,, -function,PyUnicode_AsUTF8AndSize,3.10,, -function,PyUnicode_AsUTF8String,3.2,, -function,PyUnicode_AsUnicodeEscapeString,3.2,, -function,PyUnicode_AsWideChar,3.2,, -function,PyUnicode_AsWideCharString,3.7,, -function,PyUnicode_BuildEncodingMap,3.2,, -function,PyUnicode_Compare,3.2,, -function,PyUnicode_CompareWithASCIIString,3.2,, -function,PyUnicode_Concat,3.2,, -function,PyUnicode_Contains,3.2,, -function,PyUnicode_Count,3.2,, -function,PyUnicode_Decode,3.2,, -function,PyUnicode_DecodeASCII,3.2,, -function,PyUnicode_DecodeCharmap,3.2,, -function,PyUnicode_DecodeCodePageStateful,3.7,on Windows, -function,PyUnicode_DecodeFSDefault,3.2,, -function,PyUnicode_DecodeFSDefaultAndSize,3.2,, -function,PyUnicode_DecodeLatin1,3.2,, -function,PyUnicode_DecodeLocale,3.7,, -function,PyUnicode_DecodeLocaleAndSize,3.7,, -function,PyUnicode_DecodeMBCS,3.7,on Windows, -function,PyUnicode_DecodeMBCSStateful,3.7,on Windows, -function,PyUnicode_DecodeRawUnicodeEscape,3.2,, -function,PyUnicode_DecodeUTF16,3.2,, -function,PyUnicode_DecodeUTF16Stateful,3.2,, -function,PyUnicode_DecodeUTF32,3.2,, -function,PyUnicode_DecodeUTF32Stateful,3.2,, -function,PyUnicode_DecodeUTF7,3.2,, -function,PyUnicode_DecodeUTF7Stateful,3.2,, -function,PyUnicode_DecodeUTF8,3.2,, -function,PyUnicode_DecodeUTF8Stateful,3.2,, -function,PyUnicode_DecodeUnicodeEscape,3.2,, -function,PyUnicode_EncodeCodePage,3.7,on Windows, -function,PyUnicode_EncodeFSDefault,3.2,, -function,PyUnicode_EncodeLocale,3.7,, -function,PyUnicode_EqualToUTF8,3.13,, -function,PyUnicode_EqualToUTF8AndSize,3.13,, -function,PyUnicode_FSConverter,3.2,, -function,PyUnicode_FSDecoder,3.2,, -function,PyUnicode_Find,3.2,, -function,PyUnicode_FindChar,3.7,, -function,PyUnicode_Format,3.2,, -function,PyUnicode_FromEncodedObject,3.2,, -function,PyUnicode_FromFormat,3.2,, -function,PyUnicode_FromFormatV,3.2,, -function,PyUnicode_FromObject,3.2,, -function,PyUnicode_FromOrdinal,3.2,, -function,PyUnicode_FromString,3.2,, -function,PyUnicode_FromStringAndSize,3.2,, -function,PyUnicode_FromWideChar,3.2,, -function,PyUnicode_GetDefaultEncoding,3.2,, -function,PyUnicode_GetLength,3.7,, -function,PyUnicode_InternFromString,3.2,, -function,PyUnicode_InternInPlace,3.2,, -function,PyUnicode_IsIdentifier,3.2,, -function,PyUnicode_Join,3.2,, -function,PyUnicode_Partition,3.2,, -function,PyUnicode_RPartition,3.2,, -function,PyUnicode_RSplit,3.2,, -function,PyUnicode_ReadChar,3.7,, -function,PyUnicode_Replace,3.2,, -function,PyUnicode_Resize,3.2,, -function,PyUnicode_RichCompare,3.2,, -function,PyUnicode_Split,3.2,, -function,PyUnicode_Splitlines,3.2,, -function,PyUnicode_Substring,3.7,, -function,PyUnicode_Tailmatch,3.2,, -function,PyUnicode_Translate,3.2,, -var,PyUnicode_Type,3.2,, -function,PyUnicode_WriteChar,3.7,, +data,PyType_Type,3.2,, +func,PyUnicodeDecodeError_Create,3.2,, +func,PyUnicodeDecodeError_GetEncoding,3.2,, +func,PyUnicodeDecodeError_GetEnd,3.2,, +func,PyUnicodeDecodeError_GetObject,3.2,, +func,PyUnicodeDecodeError_GetReason,3.2,, +func,PyUnicodeDecodeError_GetStart,3.2,, +func,PyUnicodeDecodeError_SetEnd,3.2,, +func,PyUnicodeDecodeError_SetReason,3.2,, +func,PyUnicodeDecodeError_SetStart,3.2,, +func,PyUnicodeEncodeError_GetEncoding,3.2,, +func,PyUnicodeEncodeError_GetEnd,3.2,, +func,PyUnicodeEncodeError_GetObject,3.2,, +func,PyUnicodeEncodeError_GetReason,3.2,, +func,PyUnicodeEncodeError_GetStart,3.2,, +func,PyUnicodeEncodeError_SetEnd,3.2,, +func,PyUnicodeEncodeError_SetReason,3.2,, +func,PyUnicodeEncodeError_SetStart,3.2,, +data,PyUnicodeIter_Type,3.2,, +func,PyUnicodeTranslateError_GetEnd,3.2,, +func,PyUnicodeTranslateError_GetObject,3.2,, +func,PyUnicodeTranslateError_GetReason,3.2,, +func,PyUnicodeTranslateError_GetStart,3.2,, +func,PyUnicodeTranslateError_SetEnd,3.2,, +func,PyUnicodeTranslateError_SetReason,3.2,, +func,PyUnicodeTranslateError_SetStart,3.2,, +func,PyUnicode_Append,3.2,, +func,PyUnicode_AppendAndDel,3.2,, +func,PyUnicode_AsASCIIString,3.2,, +func,PyUnicode_AsCharmapString,3.2,, +func,PyUnicode_AsDecodedObject,3.2,, +func,PyUnicode_AsDecodedUnicode,3.2,, +func,PyUnicode_AsEncodedObject,3.2,, +func,PyUnicode_AsEncodedString,3.2,, +func,PyUnicode_AsEncodedUnicode,3.2,, +func,PyUnicode_AsLatin1String,3.2,, +func,PyUnicode_AsMBCSString,3.7,on Windows, +func,PyUnicode_AsRawUnicodeEscapeString,3.2,, +func,PyUnicode_AsUCS4,3.7,, +func,PyUnicode_AsUCS4Copy,3.7,, +func,PyUnicode_AsUTF16String,3.2,, +func,PyUnicode_AsUTF32String,3.2,, +func,PyUnicode_AsUTF8AndSize,3.10,, +func,PyUnicode_AsUTF8String,3.2,, +func,PyUnicode_AsUnicodeEscapeString,3.2,, +func,PyUnicode_AsWideChar,3.2,, +func,PyUnicode_AsWideCharString,3.7,, +func,PyUnicode_BuildEncodingMap,3.2,, +func,PyUnicode_Compare,3.2,, +func,PyUnicode_CompareWithASCIIString,3.2,, +func,PyUnicode_Concat,3.2,, +func,PyUnicode_Contains,3.2,, +func,PyUnicode_Count,3.2,, +func,PyUnicode_Decode,3.2,, +func,PyUnicode_DecodeASCII,3.2,, +func,PyUnicode_DecodeCharmap,3.2,, +func,PyUnicode_DecodeCodePageStateful,3.7,on Windows, +func,PyUnicode_DecodeFSDefault,3.2,, +func,PyUnicode_DecodeFSDefaultAndSize,3.2,, +func,PyUnicode_DecodeLatin1,3.2,, +func,PyUnicode_DecodeLocale,3.7,, +func,PyUnicode_DecodeLocaleAndSize,3.7,, +func,PyUnicode_DecodeMBCS,3.7,on Windows, +func,PyUnicode_DecodeMBCSStateful,3.7,on Windows, +func,PyUnicode_DecodeRawUnicodeEscape,3.2,, +func,PyUnicode_DecodeUTF16,3.2,, +func,PyUnicode_DecodeUTF16Stateful,3.2,, +func,PyUnicode_DecodeUTF32,3.2,, +func,PyUnicode_DecodeUTF32Stateful,3.2,, +func,PyUnicode_DecodeUTF7,3.2,, +func,PyUnicode_DecodeUTF7Stateful,3.2,, +func,PyUnicode_DecodeUTF8,3.2,, +func,PyUnicode_DecodeUTF8Stateful,3.2,, +func,PyUnicode_DecodeUnicodeEscape,3.2,, +func,PyUnicode_EncodeCodePage,3.7,on Windows, +func,PyUnicode_EncodeFSDefault,3.2,, +func,PyUnicode_EncodeLocale,3.7,, +func,PyUnicode_Equal,3.14,, +func,PyUnicode_EqualToUTF8,3.13,, +func,PyUnicode_EqualToUTF8AndSize,3.13,, +func,PyUnicode_FSConverter,3.2,, +func,PyUnicode_FSDecoder,3.2,, +func,PyUnicode_Find,3.2,, +func,PyUnicode_FindChar,3.7,, +func,PyUnicode_Format,3.2,, +func,PyUnicode_FromEncodedObject,3.2,, +func,PyUnicode_FromFormat,3.2,, +func,PyUnicode_FromFormatV,3.2,, +func,PyUnicode_FromObject,3.2,, +func,PyUnicode_FromOrdinal,3.2,, +func,PyUnicode_FromString,3.2,, +func,PyUnicode_FromStringAndSize,3.2,, +func,PyUnicode_FromWideChar,3.2,, +func,PyUnicode_GetDefaultEncoding,3.2,, +func,PyUnicode_GetLength,3.7,, +func,PyUnicode_InternFromString,3.2,, +func,PyUnicode_InternInPlace,3.2,, +func,PyUnicode_IsIdentifier,3.2,, +func,PyUnicode_Join,3.2,, +func,PyUnicode_Partition,3.2,, +func,PyUnicode_RPartition,3.2,, +func,PyUnicode_RSplit,3.2,, +func,PyUnicode_ReadChar,3.7,, +func,PyUnicode_Replace,3.2,, +func,PyUnicode_Resize,3.2,, +func,PyUnicode_RichCompare,3.2,, +func,PyUnicode_Split,3.2,, +func,PyUnicode_Splitlines,3.2,, +func,PyUnicode_Substring,3.7,, +func,PyUnicode_Tailmatch,3.2,, +func,PyUnicode_Translate,3.2,, +data,PyUnicode_Type,3.2,, +func,PyUnicode_WriteChar,3.7,, type,PyVarObject,3.2,,members member,PyVarObject.ob_base,3.2,, member,PyVarObject.ob_size,3.2,, -function,PyVectorcall_Call,3.12,, -function,PyVectorcall_NARGS,3.12,, +func,PyVectorcall_Call,3.12,, +func,PyVectorcall_NARGS,3.12,, type,PyWeakReference,3.2,,opaque -function,PyWeakref_GetObject,3.2,, -function,PyWeakref_GetRef,3.13,, -function,PyWeakref_NewProxy,3.2,, -function,PyWeakref_NewRef,3.2,, -var,PyWrapperDescr_Type,3.2,, -function,PyWrapper_New,3.2,, -var,PyZip_Type,3.2,, -function,Py_AddPendingCall,3.2,, -function,Py_AtExit,3.2,, +func,PyWeakref_GetObject,3.2,, +func,PyWeakref_GetRef,3.13,, +func,PyWeakref_NewProxy,3.2,, +func,PyWeakref_NewRef,3.2,, +data,PyWrapperDescr_Type,3.2,, +func,PyWrapper_New,3.2,, +data,PyZip_Type,3.2,, +func,Py_AddPendingCall,3.2,, +func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, macro,Py_BLOCK_THREADS,3.2,, -function,Py_BuildValue,3.2,, -function,Py_BytesMain,3.8,, -function,Py_CompileString,3.2,, -function,Py_DecRef,3.2,, -function,Py_DecodeLocale,3.7,, +func,Py_BuildValue,3.2,, +func,Py_BytesMain,3.8,, +func,Py_CompileString,3.2,, +func,Py_DecRef,3.2,, +func,Py_DecodeLocale,3.7,, macro,Py_END_ALLOW_THREADS,3.2,, -function,Py_EncodeLocale,3.7,, -function,Py_EndInterpreter,3.2,, -function,Py_EnterRecursiveCall,3.9,, -function,Py_Exit,3.2,, -function,Py_FatalError,3.2,, -var,Py_FileSystemDefaultEncodeErrors,3.10,, -var,Py_FileSystemDefaultEncoding,3.2,, -function,Py_Finalize,3.2,, -function,Py_FinalizeEx,3.6,, -function,Py_GenericAlias,3.9,, -var,Py_GenericAliasType,3.9,, -function,Py_GetBuildInfo,3.2,, -function,Py_GetCompiler,3.2,, -function,Py_GetCopyright,3.2,, -function,Py_GetExecPrefix,3.2,, -function,Py_GetPath,3.2,, -function,Py_GetPlatform,3.2,, -function,Py_GetPrefix,3.2,, -function,Py_GetProgramFullPath,3.2,, -function,Py_GetProgramName,3.2,, -function,Py_GetPythonHome,3.2,, -function,Py_GetRecursionLimit,3.2,, -function,Py_GetVersion,3.2,, -var,Py_HasFileSystemDefaultEncoding,3.2,, -function,Py_IncRef,3.2,, -function,Py_Initialize,3.2,, -function,Py_InitializeEx,3.2,, -function,Py_Is,3.10,, -function,Py_IsFalse,3.10,, -function,Py_IsFinalizing,3.13,, -function,Py_IsInitialized,3.2,, -function,Py_IsNone,3.10,, -function,Py_IsTrue,3.10,, -function,Py_LeaveRecursiveCall,3.9,, -function,Py_Main,3.2,, -function,Py_MakePendingCalls,3.2,, -function,Py_NewInterpreter,3.2,, -function,Py_NewRef,3.10,, -function,Py_ReprEnter,3.2,, -function,Py_ReprLeave,3.2,, -function,Py_SetRecursionLimit,3.2,, +func,Py_EncodeLocale,3.7,, +func,Py_EndInterpreter,3.2,, +func,Py_EnterRecursiveCall,3.9,, +func,Py_Exit,3.2,, +func,Py_FatalError,3.2,, +data,Py_FileSystemDefaultEncodeErrors,3.10,, +data,Py_FileSystemDefaultEncoding,3.2,, +func,Py_Finalize,3.2,, +func,Py_FinalizeEx,3.6,, +func,Py_GenericAlias,3.9,, +data,Py_GenericAliasType,3.9,, +func,Py_GetBuildInfo,3.2,, +func,Py_GetCompiler,3.2,, +func,Py_GetConstant,3.13,, +func,Py_GetConstantBorrowed,3.13,, +func,Py_GetCopyright,3.2,, +func,Py_GetExecPrefix,3.2,, +func,Py_GetPath,3.2,, +func,Py_GetPlatform,3.2,, +func,Py_GetPrefix,3.2,, +func,Py_GetProgramFullPath,3.2,, +func,Py_GetProgramName,3.2,, +func,Py_GetPythonHome,3.2,, +func,Py_GetRecursionLimit,3.2,, +func,Py_GetVersion,3.2,, +data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_IncRef,3.2,, +func,Py_Initialize,3.2,, +func,Py_InitializeEx,3.2,, +func,Py_Is,3.10,, +func,Py_IsFalse,3.10,, +func,Py_IsFinalizing,3.13,, +func,Py_IsInitialized,3.2,, +func,Py_IsNone,3.10,, +func,Py_IsTrue,3.10,, +func,Py_LeaveRecursiveCall,3.9,, +func,Py_Main,3.2,, +func,Py_MakePendingCalls,3.2,, +func,Py_NewInterpreter,3.2,, +func,Py_NewRef,3.10,, +func,Py_REFCNT,3.14,, +func,Py_ReprEnter,3.2,, +func,Py_ReprLeave,3.2,, +func,Py_SetProgramName,3.2,, +func,Py_SetPythonHome,3.2,, +func,Py_SetRecursionLimit,3.2,, +func,Py_TYPE,3.14,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, -var,Py_UTF8Mode,3.8,, -function,Py_VaBuildValue,3.2,, -var,Py_Version,3.11,, -function,Py_XNewRef,3.10,, +data,Py_UTF8Mode,3.8,, +func,Py_VaBuildValue,3.2,, +data,Py_Version,3.11,, +func,Py_XNewRef,3.10,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, type,Py_ssize_t,3.2,, diff --git a/Doc/deprecations/c-api-pending-removal-in-3.14.rst b/Doc/deprecations/c-api-pending-removal-in-3.14.rst new file mode 100644 index 00000000000000..9e10bf2691e5c8 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.14.rst @@ -0,0 +1,72 @@ +Pending removal in Python 3.14 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The ``ma_version_tag`` field in :c:type:`PyDictObject` for extension modules + (:pep:`699`; :gh:`101193`). + +* Creating :c:data:`immutable types ` with mutable + bases (:gh:`95388`). + +* Functions to configure Python's initialization, deprecated in Python 3.11: + + * :c:func:`!PySys_SetArgvEx()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!PySys_SetArgv()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!Py_SetProgramName()`: + Set :c:member:`PyConfig.program_name` instead. + * :c:func:`!Py_SetPythonHome()`: + Set :c:member:`PyConfig.home` instead. + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: + Use :c:member:`PyConfig.parser_debug` instead. + * :c:var:`Py_VerboseFlag`: + Use :c:member:`PyConfig.verbose` instead. + * :c:var:`Py_QuietFlag`: + Use :c:member:`PyConfig.quiet` instead. + * :c:var:`Py_InteractiveFlag`: + Use :c:member:`PyConfig.interactive` instead. + * :c:var:`Py_InspectFlag`: + Use :c:member:`PyConfig.inspect` instead. + * :c:var:`Py_OptimizeFlag`: + Use :c:member:`PyConfig.optimization_level` instead. + * :c:var:`Py_NoSiteFlag`: + Use :c:member:`PyConfig.site_import` instead. + * :c:var:`Py_BytesWarningFlag`: + Use :c:member:`PyConfig.bytes_warning` instead. + * :c:var:`Py_FrozenFlag`: + Use :c:member:`PyConfig.pathconfig_warnings` instead. + * :c:var:`Py_IgnoreEnvironmentFlag`: + Use :c:member:`PyConfig.use_environment` instead. + * :c:var:`Py_DontWriteBytecodeFlag`: + Use :c:member:`PyConfig.write_bytecode` instead. + * :c:var:`Py_NoUserSiteDirectory`: + Use :c:member:`PyConfig.user_site_directory` instead. + * :c:var:`Py_UnbufferedStdioFlag`: + Use :c:member:`PyConfig.buffered_stdio` instead. + * :c:var:`Py_HashRandomizationFlag`: + Use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` instead. + * :c:var:`Py_IsolatedFlag`: + Use :c:member:`PyConfig.isolated` instead. + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` instead. + * :c:var:`Py_LegacyWindowsStdioFlag`: + Use :c:member:`PyConfig.legacy_windows_stdio` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` instead. + * :c:var:`!Py_HasFileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: + Use :c:member:`PyConfig.filesystem_errors` instead. + * :c:var:`!Py_UTF8Mode`: + Use :c:member:`PyPreConfig.utf8_mode` instead. + (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst new file mode 100644 index 00000000000000..0ce0f9c118c094 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -0,0 +1,27 @@ +Pending removal in Python 3.15 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The bundled copy of ``libmpdecimal``. +* The :c:func:`PyImport_ImportModuleNoBlock`: + Use :c:func:`PyImport_ImportModule` instead. +* :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: + Use :c:func:`PyWeakref_GetRef` instead. +* :c:type:`Py_UNICODE` type and the :c:macro:`!Py_UNICODE_WIDE` macro: + Use :c:type:`wchar_t` instead. +* Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: + Get :data:`sys.base_exec_prefix` and :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: + Get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: + Get :data:`sys.base_prefix` and :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: + Get :c:member:`PyConfig.home` + or the :envvar:`PYTHONHOME` environment variable instead. diff --git a/Doc/deprecations/c-api-pending-removal-in-future.rst b/Doc/deprecations/c-api-pending-removal-in-future.rst new file mode 100644 index 00000000000000..8fc1c80c35d092 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-future.rst @@ -0,0 +1,51 @@ +Pending removal in future versions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following APIs are deprecated and will be removed, +although there is currently no date scheduled for their removal. + +* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: + Unneeded since Python 3.8. +* :c:func:`PyErr_Fetch`: + Use :c:func:`PyErr_GetRaisedException` instead. +* :c:func:`PyErr_NormalizeException`: + Use :c:func:`PyErr_GetRaisedException` instead. +* :c:func:`PyErr_Restore`: + Use :c:func:`PyErr_SetRaisedException` instead. +* :c:func:`PyModule_GetFilename`: + Use :c:func:`PyModule_GetFilenameObject` instead. +* :c:func:`PyOS_AfterFork`: + Use :c:func:`PyOS_AfterFork_Child` instead. +* :c:func:`PySlice_GetIndicesEx`: + Use :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` instead. +* :c:func:`!PyUnicode_AsDecodedObject`: + Use :c:func:`PyCodec_Decode` instead. +* :c:func:`!PyUnicode_AsDecodedUnicode`: + Use :c:func:`PyCodec_Decode` instead. +* :c:func:`!PyUnicode_AsEncodedObject`: + Use :c:func:`PyCodec_Encode` instead. +* :c:func:`!PyUnicode_AsEncodedUnicode`: + Use :c:func:`PyCodec_Encode` instead. +* :c:func:`PyUnicode_READY`: + Unneeded since Python 3.12 +* :c:func:`!PyErr_Display`: + Use :c:func:`PyErr_DisplayException` instead. +* :c:func:`!_PyErr_ChainExceptions`: + Use :c:func:`!_PyErr_ChainExceptions1` instead. +* :c:member:`!PyBytesObject.ob_shash` member: + call :c:func:`PyObject_Hash` instead. +* :c:member:`!PyDictObject.ma_version_tag` member. +* Thread Local Storage (TLS) API: + + * :c:func:`PyThread_create_key`: + Use :c:func:`PyThread_tss_alloc` instead. + * :c:func:`PyThread_delete_key`: + Use :c:func:`PyThread_tss_free` instead. + * :c:func:`PyThread_set_key_value`: + Use :c:func:`PyThread_tss_set` instead. + * :c:func:`PyThread_get_key_value`: + Use :c:func:`PyThread_tss_get` instead. + * :c:func:`PyThread_delete_key_value`: + Use :c:func:`PyThread_tss_delete` instead. + * :c:func:`PyThread_ReInitTLS`: + Unneeded since Python 3.7. diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst new file mode 100644 index 00000000000000..bac6e3f18d4594 --- /dev/null +++ b/Doc/deprecations/index.rst @@ -0,0 +1,15 @@ +Deprecations +============ + +.. include:: pending-removal-in-3.15.rst + +.. include:: pending-removal-in-3.16.rst + +.. include:: pending-removal-in-future.rst + +C API deprecations +------------------ + +.. include:: c-api-pending-removal-in-3.15.rst + +.. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst new file mode 100644 index 00000000000000..2fd2f12cc6a2c4 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -0,0 +1,52 @@ +Pending removal in Python 3.13 +------------------------------ + +Modules (see :pep:`594`): + +* :mod:`!aifc` +* :mod:`!audioop` +* :mod:`!cgi` +* :mod:`!cgitb` +* :mod:`!chunk` +* :mod:`!crypt` +* :mod:`!imghdr` +* :mod:`!mailcap` +* :mod:`!msilib` +* :mod:`!nis` +* :mod:`!nntplib` +* :mod:`!ossaudiodev` +* :mod:`!pipes` +* :mod:`!sndhdr` +* :mod:`!spwd` +* :mod:`!sunau` +* :mod:`!telnetlib` +* :mod:`!uu` +* :mod:`!xdrlib` + +Other modules: + +* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) + +APIs: + +* :class:`!configparser.LegacyInterpolation` (:gh:`90765`) +* ``locale.resetlocale()`` (:gh:`90817`) +* :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) +* :func:`!unittest.findTestCases` (:gh:`50096`) +* :func:`!unittest.getTestCaseNames` (:gh:`50096`) +* :func:`!unittest.makeSuite` (:gh:`50096`) +* :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) +* :class:`!webbrowser.MacOSX` (:gh:`86421`) +* :class:`classmethod` descriptor chaining (:gh:`89519`) +* :mod:`importlib.resources` deprecated methods: + + * ``contents()`` + * ``is_resource()`` + * ``open_binary()`` + * ``open_text()`` + * ``path()`` + * ``read_binary()`` + * ``read_text()`` + + Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy + `_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst new file mode 100644 index 00000000000000..23c9bde5c01ba7 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -0,0 +1,122 @@ +Pending removal in Python 3.14 +------------------------------ + +* The import system: + + * Setting :attr:`~module.__loader__` on a module while + failing to set :attr:`__spec__.loader ` + is deprecated. In Python 3.14, :attr:`!__loader__` will cease to be set or + taken into consideration by the import system or the standard library. + +* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` are deprecated + and will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`92248`.) + +* :mod:`ast`: The following features have been deprecated in documentation + since Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at + runtime when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + +* :mod:`asyncio`: + + * The child watcher classes :class:`!asyncio.MultiLoopChildWatcher`, + :class:`!asyncio.FastChildWatcher`, :class:`!asyncio.AbstractChildWatcher` + and :class:`!asyncio.SafeChildWatcher` are deprecated and + will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * :func:`!asyncio.set_child_watcher`, :func:`!asyncio.get_child_watcher`, + :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + and will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * The :meth:`~asyncio.get_event_loop` method of the + default event loop policy now emits a :exc:`DeprecationWarning` if there + is no current event loop set and it decides to create one. + (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) + +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. + Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, + or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. + (Contributed by Alan Williams in :gh:`72346`.) + +* :mod:`importlib.abc` deprecated classes: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + +* :mod:`itertools` had undocumented, inefficient, historically buggy, + and inconsistent support for copy, deepcopy, and pickle operations. + This will be removed in 3.14 for a significant reduction in code + volume and maintenance burden. + (Contributed by Raymond Hettinger in :gh:`101588`.) + +* :mod:`multiprocessing`: The default start method will change to a safer one on + Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently + the default (:gh:`84559`). Adding a runtime warning about this was deemed too + disruptive as the majority of code is not expected to care. Use the + :func:`~multiprocessing.get_context` or + :func:`~multiprocessing.set_start_method` APIs to explicitly specify when + your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. + +* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and + :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is + deprecated. + +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:!pkgutil.get_loader` + now raise :exc:`DeprecationWarning`; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + +* :mod:`pty`: + + * ``master_open()``: use :func:`pty.openpty`. + * ``slave_open()``: use :func:`pty.openpty`. + +* :mod:`sqlite3`: + + * :data:`!version` and :data:`!version_info`. + + * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` + if :ref:`named placeholders ` are used and + *parameters* is a sequence instead of a :class:`dict`. + + * date and datetime adapter, date and timestamp converter: + see the :mod:`sqlite3` documentation for suggested replacement recipes. + +* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + deprecated in :pep:`626` + since 3.10 and was planned to be removed in 3.12, + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`101866`.) + +* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, + now causes a :exc:`DeprecationWarning` to be emitted when it is used. + +* :mod:`urllib`: + :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a + public API. + (Contributed by Gregory P. Smith in :gh:`88168`.) diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst new file mode 100644 index 00000000000000..17029b8d4773bd --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -0,0 +1,80 @@ +Pending removal in Python 3.15 +------------------------------ + +* The import system: + + * Setting :attr:`~module.__cached__` on a module while + failing to set :attr:`__spec__.cached ` + is deprecated. In Python 3.15, :attr:`!__cached__` will cease to be set or + take into consideration by the import system or standard library. (:gh:`97879`) + + * Setting :attr:`~module.__package__` on a module while + failing to set :attr:`__spec__.parent ` + is deprecated. In Python 3.15, :attr:`!__package__` will cease to be set or + take into consideration by the import system or standard library. (:gh:`97879`) + +* :mod:`ctypes`: + + * The undocumented :func:`!ctypes.SetPointerType` function + has been deprecated since Python 3.13. + +* :mod:`http.server`: + + * The obsolete and rarely used :class:`~http.server.CGIHTTPRequestHandler` + has been deprecated since Python 3.13. + No direct replacement exists. + *Anything* is better than CGI to interface + a web server with a request handler. + + * The :option:`!--cgi` flag to the :program:`python -m http.server` + command-line interface has been deprecated since Python 3.13. + +* :class:`locale`: + + * The :func:`~locale.getdefaultlocale` function + has been deprecated since Python 3.11. + Its removal was originally planned for Python 3.13 (:gh:`90817`), + but has been postponed to Python 3.15. + Use :func:`~locale.getlocale`, :func:`~locale.setlocale`, + and :func:`~locale.getencoding` instead. + (Contributed by Hugo van Kemenade in :gh:`111187`.) + +* :mod:`pathlib`: + + * :meth:`.PurePath.is_reserved` + has been deprecated since Python 3.13. + Use :func:`os.path.isreserved` to detect reserved paths on Windows. + +* :mod:`platform`: + + * :func:`~platform.java_ver` has been deprecated since Python 3.13. + This function is only useful for Jython support, has a confusing API, + and is largely untested. + +* :mod:`threading`: + + * :func:`~threading.RLock` will take no arguments in Python 3.15. + Passing any arguments has been deprecated since Python 3.14, + as the Python version does not permit any arguments, + but the C version allows any number of positional or keyword arguments, + ignoring every argument. + +* :mod:`typing`: + + * The undocumented keyword argument syntax for creating + :class:`~typing.NamedTuple` classes + (for example, ``Point = NamedTuple("Point", x=int, y=int)``) + has been deprecated since Python 3.13. + Use the class-based syntax or the functional syntax instead. + + * The :func:`typing.no_type_check_decorator` decorator function + has been deprecated since Python 3.13. + After eight years in the :mod:`typing` module, + it has yet to be supported by any major type checker. + +* :mod:`wave`: + + * The :meth:`~wave.Wave_read.getmark`, :meth:`!setmark`, + and :meth:`~wave.Wave_read.getmarkers` methods of + the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes + have been deprecated since Python 3.13. diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst new file mode 100644 index 00000000000000..fac500d34742ca --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -0,0 +1,49 @@ +Pending removal in Python 3.16 +------------------------------ + +* :mod:`array`: + + * The ``'u'`` format code (:c:type:`wchar_t`) + has been deprecated in documentation since Python 3.3 + and at runtime since Python 3.13. + Use the ``'w'`` format code (:c:type:`Py_UCS4`) + for Unicode characters instead. + +* :mod:`asyncio`: + + * :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + +* :mod:`builtins`: + + * Bitwise inversion on boolean types, ``~True`` or ``~False`` + has been deprecated since Python 3.12, + as it produces surprising and unintuitive results (``-2`` and ``-1``). + Use ``not x`` instead for the logical negation of a Boolean. + In the rare case that you need the bitwise inversion of + the underlying integer, convert to ``int`` explicitly (``~int(x)``). + +* :mod:`shutil`: + + * The :class:`!ExecError` exception + has been deprecated since Python 3.14. + It has not been used by any function in :mod:`!shutil` since Python 3.4, + and is now an alias of :exc:`RuntimeError`. + +* :mod:`symtable`: + + * The :meth:`Class.get_methods ` method + has been deprecated since Python 3.14. + +* :mod:`sys`: + + * The :func:`~sys._enablelegacywindowsfsencoding` function + has been deprecated since Python 3.13. + Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. + +* :mod:`tarfile`: + + * The undocumented and unused :attr:`!TarFile.tarfile` attribute + has been deprecated since Python 3.13. diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst new file mode 100644 index 00000000000000..3be0dabfd1f257 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -0,0 +1,161 @@ +Pending removal in future versions +---------------------------------- + +The following APIs will be removed in the future, +although there is currently no date scheduled for their removal. + +* :mod:`argparse`: + + * Nesting argument groups and nesting mutually exclusive + groups are deprecated. + * Passing the undocumented keyword argument *prefix_chars* to + :meth:`~argparse.ArgumentParser.add_argument_group` is now + deprecated. + * The :class:`argparse.FileType` type converter is deprecated. + +* :mod:`array`'s ``'u'`` format code (:gh:`57281`) + +* :mod:`builtins`: + + * ``bool(NotImplemented)``. + * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` + signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, + the single argument signature. + * Currently Python accepts numeric literals immediately followed by keywords, + for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and + ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as + ``[0x1 for x in y]`` or ``[0x1f or x in y]``). A syntax warning is raised + if the numeric literal is immediately followed by one of keywords + :keyword:`and`, :keyword:`else`, :keyword:`for`, :keyword:`if`, + :keyword:`in`, :keyword:`is` and :keyword:`or`. In a future release it + will be changed to a syntax error. (:gh:`87999`) + * Support for ``__index__()`` and ``__int__()`` method returning non-int type: + these methods will be required to return an instance of a strict subclass of + :class:`int`. + * Support for ``__float__()`` method returning a strict subclass of + :class:`float`: these methods will be required to return an instance of + :class:`float`. + * Support for ``__complex__()`` method returning a strict subclass of + :class:`complex`: these methods will be required to return an instance of + :class:`complex`. + * Delegation of ``int()`` to ``__trunc__()`` method. + * Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; it should only be passed + as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) + +* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are + deprecated and replaced by :data:`calendar.JANUARY` and + :data:`calendar.FEBRUARY`. + (Contributed by Prince Roshan in :gh:`103636`.) + +* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method + instead. + +* :mod:`datetime`: + + * :meth:`~datetime.datetime.utcnow`: + use ``datetime.datetime.now(tz=datetime.UTC)``. + * :meth:`~datetime.datetime.utcfromtimestamp`: + use ``datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)``. + +* :mod:`gettext`: Plural value must be an integer. + +* :mod:`importlib`: + + * ``load_module()`` method: use ``exec_module()`` instead. + * :func:`~importlib.util.cache_from_source` *debug_override* parameter is + deprecated: use the *optimization* parameter instead. + +* :mod:`importlib.metadata`: + + * ``EntryPoints`` tuple interface. + * Implicit ``None`` on return values. + +* :mod:`logging`: the ``warn()`` method has been deprecated + since Python 3.3, use :meth:`~logging.warning` instead. + +* :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use + BytesIO and binary mode instead. + +* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. + +* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is + deprecated, use an exception instance. + +* :mod:`re`: More strict rules are now applied for numerical group references + and group names in regular expressions. Only sequence of ASCII digits is now + accepted as a numerical reference. The group name in bytes patterns and + replacement strings can now only contain ASCII letters and digits and + underscore. + (Contributed by Serhiy Storchaka in :gh:`91760`.) + +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + +* :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in + Python 3.12; use the *onexc* parameter instead. + +* :mod:`ssl` options and protocols: + + * :class:`ssl.SSLContext` without protocol argument is deprecated. + * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and + :meth:`!selected_npn_protocol` are deprecated: use ALPN + instead. + * ``ssl.OP_NO_SSL*`` options + * ``ssl.OP_NO_TLS*`` options + * ``ssl.PROTOCOL_SSLv3`` + * ``ssl.PROTOCOL_TLS`` + * ``ssl.PROTOCOL_TLSv1`` + * ``ssl.PROTOCOL_TLSv1_1`` + * ``ssl.PROTOCOL_TLSv1_2`` + * ``ssl.TLSVersion.SSLv3`` + * ``ssl.TLSVersion.TLSv1`` + * ``ssl.TLSVersion.TLSv1_1`` + +* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and + ignored. + +* :mod:`threading` methods: + + * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. + * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. + * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: + use :attr:`threading.Thread.daemon` attribute. + * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: + use :attr:`threading.Thread.name` attribute. + * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. + * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. + +* :class:`typing.Text` (:gh:`92332`). + +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not ``None`` from a test case. + +* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead + + * ``splitattr()`` + * ``splithost()`` + * ``splitnport()`` + * ``splitpasswd()`` + * ``splitport()`` + * ``splitquery()`` + * ``splittag()`` + * ``splittype()`` + * ``splituser()`` + * ``splitvalue()`` + * ``to_bytes()`` + +* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and + :class:`~urllib.request.FancyURLopener` style of invoking requests is + deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. + +* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial + writes. + +* :mod:`xml.etree.ElementTree`: Testing the truth value of an + :class:`~xml.etree.ElementTree.Element` is deprecated. In a future release it + will always return ``True``. Prefer explicit ``len(elem)`` or + ``elem is not None`` tests instead. + +* :meth:`zipimport.zipimporter.load_module` is deprecated: + use :meth:`~zipimport.zipimporter.exec_module` instead. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 745fc10a22d161..b0493bed75b151 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -547,7 +547,7 @@ reference count of an object and are safe in the presence of ``NULL`` pointers (but note that *temp* will not be ``NULL`` in this context). More info on them in section :ref:`refcounts`. -.. index:: single: PyObject_CallObject() +.. index:: single: PyObject_CallObject (C function) Later, when it is time to call the function, you call the C function :c:func:`PyObject_CallObject`. This function has two arguments, both pointers to @@ -638,7 +638,7 @@ the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. : Extracting Parameters in Extension Functions ============================================ -.. index:: single: PyArg_ParseTuple() +.. index:: single: PyArg_ParseTuple (C function) The :c:func:`PyArg_ParseTuple` function is declared as follows:: @@ -730,7 +730,7 @@ Some example calls:: Keyword Parameters for Extension Functions ========================================== -.. index:: single: PyArg_ParseTupleAndKeywords() +.. index:: single: PyArg_ParseTupleAndKeywords (C function) The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: @@ -868,7 +868,7 @@ It is important to call :c:func:`free` at the right time. If a block's address is forgotten but :c:func:`free` is not called for it, the memory it occupies cannot be reused until the program terminates. This is called a :dfn:`memory leak`. On the other hand, if a program calls :c:func:`free` for a block and then -continues to use the block, it creates a conflict with re-use of the block +continues to use the block, it creates a conflict with reuse of the block through another :c:func:`malloc` call. This is called :dfn:`using freed memory`. It has the same bad consequences as referencing uninitialized data --- core dumps, wrong results, mysterious crashes. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 7a92b3257c6cd3..7f57a3a6aac0ed 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -89,8 +89,8 @@ If your type supports garbage collection, the destructor should call } .. index:: - single: PyErr_Fetch() - single: PyErr_Restore() + single: PyErr_Fetch (C function) + single: PyErr_Restore (C function) One important requirement of the deallocator function is that it leaves any pending exceptions alone. This is important since deallocators are frequently @@ -296,7 +296,7 @@ An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table descriptors that are used at runtime is that any attribute defined this way can have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the -class object, and get the doc string using its :attr:`!__doc__` attribute. +class object, and get the doc string using its :attr:`~type.__doc__` attribute. As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. @@ -545,7 +545,7 @@ performance-critical objects (such as numbers). .. seealso:: Documentation for the :mod:`weakref` module. -For an object to be weakly referencable, the extension type must set the +For an object to be weakly referenceable, the extension type must set the ``Py_TPFLAGS_MANAGED_WEAKREF`` bit of the :c:member:`~PyTypeObject.tp_flags` field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 7eba9759119b3b..bcf938f117d148 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -144,7 +144,7 @@ only used for variable-sized objects and should otherwise be zero. If you want your type to be subclassable from Python, and your type has the same :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple inheritance. A Python subclass of your type will have to list your type first - in its :attr:`~class.__bases__`, or else it will not be able to call your type's + in its :attr:`~type.__bases__`, or else it will not be able to call your type's :meth:`~object.__new__` method without getting an error. You can avoid this problem by ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its base type does. Most of the time, this will be true anyway, because either your @@ -447,7 +447,7 @@ Further, the attributes can be deleted, setting the C pointers to ``NULL``. Eve though we can make sure the members are initialized to non-``NULL`` values, the members can be set to ``NULL`` if the attributes are deleted. -We define a single method, :meth:`!Custom.name()`, that outputs the objects name as the +We define a single method, :meth:`!Custom.name`, that outputs the objects name as the concatenation of the first and last names. :: static PyObject * diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 300e1b6cc40a58..e2710fab9cf800 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -70,7 +70,7 @@ operations. This means that as far as floating-point operations are concerned, Python behaves like many popular languages including C and Java. Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: +exactly in binary floating point. For example, after:: >>> x = 1.2 @@ -87,7 +87,7 @@ which is exactly:: The typical precision of 53 bits provides Python floats with 15--16 decimal digits of accuracy. -For a fuller explanation, please see the :ref:`floating point arithmetic +For a fuller explanation, please see the :ref:`floating-point arithmetic ` chapter in the Python tutorial. @@ -259,9 +259,11 @@ is evaluated in all cases. Why isn't there a switch or case statement in Python? ----------------------------------------------------- -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. +In general, structured switch statements execute one block of code +when an expression has a particular value or set of values. +Since Python 3.10 one can easily match literal values, or constants +within a namespace, with a ``match ... case`` statement. +An older alternative is a sequence of ``if... elif... elif... else``. For cases where you need to choose from a very large number of possibilities, you can create a dictionary mapping case values to functions to call. For @@ -290,6 +292,9 @@ It's suggested that you use a prefix for the method names, such as ``visit_`` in this example. Without such a prefix, if values are coming from an untrusted source, an attacker would be able to call any method on your object. +Imitating switch with fallthrough, as with C's switch-case-default, +is possible, much harder, and less needed. + Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? -------------------------------------------------------------------------------------------------------- @@ -323,7 +328,7 @@ Can Python be compiled to machine code, C or some other language? ----------------------------------------------------------------- `Cython `_ compiles a modified version of Python with -optional annotations into C extensions. `Nuitka `_ is +optional annotations into C extensions. `Nuitka `_ is an up-and-coming compiler of Python into C++ code, aiming to support the full Python language. @@ -340,7 +345,7 @@ to perform a garbage collection, obtain debugging statistics, and tune the collector's parameters. Other implementations (such as `Jython `_ or -`PyPy `_), however, can rely on a different mechanism +`PyPy `_), however, can rely on a different mechanism such as a full-blown garbage collector. This difference can cause some subtle porting problems if your Python code depends on the behavior of the reference counting implementation. diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 2a8b976925d042..3147fda7c37124 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -50,7 +50,7 @@ to learn Python's C API. If you need to interface to some C or C++ library for which no Python extension currently exists, you can try wrapping the library's data types and functions with a tool such as `SWIG `_. `SIP -`__, `CXX +`__, `CXX `_ `Boost `_, or `Weave `_ are also @@ -246,13 +246,12 @@ Then, when you run GDB: I want to compile a Python module on my Linux system, but some files are missing. Why? -------------------------------------------------------------------------------------- -Most packaged versions of Python don't include the -:file:`/usr/lib/python2.{x}/config/` directory, which contains various files +Most packaged versions of Python omit some files required for compiling Python extensions. -For Red Hat, install the python-devel RPM to get the necessary files. +For Red Hat, install the python3-devel RPM to get the necessary files. -For Debian, run ``apt-get install python-dev``. +For Debian, run ``apt-get install python3-dev``. How do I tell "incomplete input" from "invalid input"? ------------------------------------------------------ diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 8727332594bda6..578777d7f23621 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -122,6 +122,8 @@ available. Consult `the Python Package Index `_ to find packages of interest to you. +.. _faq-version-numbering-scheme: + How does the Python version numbering scheme work? -------------------------------------------------- @@ -133,8 +135,6 @@ Python versions are numbered "A.B.C" or "A.B": changes. * *C* is the micro version number -- it is incremented for each bugfix release. -See :pep:`6` for more information about bugfix releases. - Not all releases are bugfix releases. In the run-up to a new feature release, a series of development releases are made, denoted as alpha, beta, or release candidate. Alphas are early releases in which interfaces aren't yet finalized; @@ -157,7 +157,11 @@ unreleased versions, built directly from the CPython development repository. In practice, after a final minor release is made, the version is incremented to the next minor version, which becomes the "a0" version, e.g. "2.4a0". -See also the documentation for :data:`sys.version`, :data:`sys.hexversion`, and +See the `Developer's Guide +`__ +for more information about the development cycle, and +:pep:`387` to learn more about Python's backward compatibility policy. See also +the documentation for :data:`sys.version`, :data:`sys.hexversion`, and :data:`sys.version_info`. @@ -181,8 +185,6 @@ information on getting the source code and compiling it. How do I get documentation on Python? ------------------------------------- -.. XXX mention py3k - The standard documentation for the current stable version of Python is available at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. @@ -307,10 +309,9 @@ guaranteed that interfaces will remain the same throughout a series of bugfix releases. The latest stable releases can always be found on the `Python download page -`_. There are two production-ready versions -of Python: 2.x and 3.x. The recommended version is 3.x, which is supported by -most widely used libraries. Although 2.x is still widely used, `it is not -maintained anymore `_. +`_. +Python 3.x is the recommended version and supported by most widely used libraries. +Python 2.x :pep:`is not maintained anymore <373>`. How many people are using Python? --------------------------------- diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index e2f8004c7e3aea..d8d75ca6f2ec96 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -541,91 +541,12 @@ Thus, to read *n* bytes from a pipe *p* created with :func:`os.popen`, you need use ``p.read(n)``. -.. XXX update to use subprocess. See the :ref:`subprocess-replacements` section. - - How do I run a subprocess with pipes connected to both input and output? - ------------------------------------------------------------------------ - - Use the :mod:`popen2` module. For example:: - - import popen2 - fromchild, tochild = popen2.popen2("command") - tochild.write("input\n") - tochild.flush() - output = fromchild.readline() - - Warning: in general it is unwise to do this because you can easily cause a - deadlock where your process is blocked waiting for output from the child - while the child is blocked waiting for input from you. This can be caused - by the parent expecting the child to output more text than it does or - by data being stuck in stdio buffers due to lack of flushing. - The Python parent can of course explicitly flush the data it sends to the - child before it reads any output, but if the child is a naive C program it - may have been written to never explicitly flush its output, even if it is - interactive, since flushing is normally automatic. - - Note that a deadlock is also possible if you use :func:`popen3` to read - stdout and stderr. If one of the two is too large for the internal buffer - (increasing the buffer size does not help) and you ``read()`` the other one - first, there is a deadlock, too. - - Note on a bug in popen2: unless your program calls ``wait()`` or - ``waitpid()``, finished child processes are never removed, and eventually - calls to popen2 will fail because of a limit on the number of child - processes. Calling :func:`os.waitpid` with the :const:`os.WNOHANG` option can - prevent this; a good place to insert such a call would be before calling - ``popen2`` again. - - In many cases, all you really need is to run some data through a command and - get the result back. Unless the amount of data is very large, the easiest - way to do this is to write it to a temporary file and run the command with - that temporary file as input. The standard module :mod:`tempfile` exports a - :func:`~tempfile.mktemp` function to generate unique temporary file names. :: - - import tempfile - import os - - class Popen3: - """ - This is a deadlock-safe version of popen that returns - an object with errorlevel, out (a string) and err (a string). - (capturestderr may not work under windows.) - Example: print(Popen3('grep spam','\n\nhere spam\n\n').out) - """ - def __init__(self,command,input=None,capturestderr=None): - outfile=tempfile.mktemp() - command="( %s ) > %s" % (command,outfile) - if input: - infile=tempfile.mktemp() - open(infile,"w").write(input) - command=command+" <"+infile - if capturestderr: - errfile=tempfile.mktemp() - command=command+" 2>"+errfile - self.errorlevel=os.system(command) >> 8 - self.out=open(outfile,"r").read() - os.remove(outfile) - if input: - os.remove(infile) - if capturestderr: - self.err=open(errfile,"r").read() - os.remove(errfile) - - Note that many interactive programs (e.g. vi) don't work well with pipes - substituted for standard input and output. You will have to use pseudo ttys - ("ptys") instead of pipes. Or you can use a Python interface to Don Libes' - "expect" library. A Python extension that interfaces to expect is called - "expy" and available from https://expectpy.sourceforge.net. A pure Python - solution that works like expect is `pexpect - `_. - - How do I access the serial (RS232) port? ---------------------------------------- For Win32, OSX, Linux, BSD, Jython, IronPython: - https://pypi.org/project/pyserial/ + :pypi:`pyserial` For Unix, see a Usenet post by Mitch Chapman: @@ -797,12 +718,12 @@ is simple:: import random random.random() -This returns a random floating point number in the range [0, 1). +This returns a random floating-point number in the range [0, 1). There are also many other specialized generators in this module, such as: * ``randrange(a, b)`` chooses an integer in the range [a, b). -* ``uniform(a, b)`` chooses a floating point number in the range [a, b). +* ``uniform(a, b)`` chooses a floating-point number in the range [a, b). * ``normalvariate(mean, sdev)`` samples the normal (Gaussian) distribution. Some higher-level functions operate on sequences directly, such as: diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 0a88c5f6384f2b..fa7b22bde1dc6f 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -869,7 +869,7 @@ How do I convert a string to a number? -------------------------------------- For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') -== 144``. Similarly, :func:`float` converts to floating-point, +== 144``. Similarly, :func:`float` converts to a floating-point number, e.g. ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == @@ -1013,7 +1013,7 @@ Not as such. For simple input parsing, the easiest approach is usually to split the line into whitespace-delimited words using the :meth:`~str.split` method of string objects and then convert decimal strings to numeric values using :func:`int` or -:func:`float`. :meth:`!split()` supports an optional "sep" parameter which is useful +:func:`float`. :meth:`!split` supports an optional "sep" parameter which is useful if the line uses something other than whitespace as a separator. For more complicated input parsing, regular expressions are more powerful @@ -1613,9 +1613,16 @@ method too, and it must do so carefully. The basic implementation of self.__dict__[name] = value ... -Most :meth:`!__setattr__` implementations must modify -:meth:`self.__dict__ ` to store -local state for self without causing an infinite recursion. +Many :meth:`~object.__setattr__` implementations call :meth:`!object.__setattr__` to set +an attribute on self without causing infinite recursion:: + + class X: + def __setattr__(self, name, value): + # Custom logic here... + object.__setattr__(self, name, value) + +Alternatively, it is possible to set attributes by inserting +entries into :attr:`self.__dict__ ` directly. How do I call a method defined in a base class from a derived class that extends it? @@ -1741,11 +1748,31 @@ but effective way to define class private variables. Any identifier of the form is textually replaced with ``_classname__spam``, where ``classname`` is the current class name with any leading underscores stripped. -This doesn't guarantee privacy: an outside user can still deliberately access -the "_classname__spam" attribute, and private values are visible in the object's -``__dict__``. Many Python programmers never bother to use private variable -names at all. +The identifier can be used unchanged within the class, but to access it outside +the class, the mangled name must be used: + +.. code-block:: python + + class A: + def __one(self): + return 1 + def two(self): + return 2 * self.__one() + + class B(A): + def three(self): + return 3 * self._A__one() + + four = 4 * A()._A__one() + +In particular, this does not guarantee privacy since an outside user can still +deliberately access the private attribute; many Python programmers never bother +to use private variable names at all. + +.. seealso:: + The :ref:`private name mangling specifications ` + for details and special cases. My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- diff --git a/Doc/glossary.rst b/Doc/glossary.rst index f656e32514c717..f67f3ecad0bc40 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -9,13 +9,14 @@ Glossary .. glossary:: ``>>>`` - The default Python prompt of the interactive shell. Often seen for code - examples which can be executed interactively in the interpreter. + The default Python prompt of the :term:`interactive` shell. Often + seen for code examples which can be executed interactively in the + interpreter. ``...`` Can refer to: - * The default Python prompt of the interactive shell when entering the + * The default Python prompt of the :term:`interactive` shell when entering the code for an indented code block, when within a pair of matching left and right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. @@ -35,6 +36,12 @@ Glossary and loaders (in the :mod:`importlib.abc` module). You can create your own ABCs with the :mod:`abc` module. + annotate function + A function that can be called to retrieve the :term:`annotations ` + of an object. This function is accessible as the :attr:`~object.__annotate__` + attribute of functions, classes, and modules. Annotate functions are a + subset of :term:`evaluate functions `. + annotation A label associated with a variable, a class attribute or a function parameter or return value, @@ -42,12 +49,11 @@ Glossary Annotations of local variables cannot be accessed at runtime, but annotations of global variables, class attributes, and functions - are stored in the :attr:`__annotations__` - special attribute of modules, classes, and functions, - respectively. + can be retrieved by calling :func:`annotationlib.get_annotations` + on modules, classes, and functions, respectively. - See :term:`variable annotation`, :term:`function annotation`, :pep:`484` - and :pep:`526`, which describe this functionality. + See :term:`variable annotation`, :term:`function annotation`, :pep:`484`, + :pep:`526`, and :pep:`649`, which describe this functionality. Also see :ref:`annotations-howto` for best practices on working with annotations. @@ -225,6 +231,28 @@ Glossary A variable defined in a class and intended to be modified only at class level (i.e., not in an instance of the class). + closure variable + A :term:`free variable` referenced from a :term:`nested scope` that is defined in an outer + scope rather than being resolved at runtime from the globals or builtin namespaces. + May be explicitly defined with the :keyword:`nonlocal` keyword to allow write access, + or implicitly defined if the variable is only being read. + + For example, in the ``inner`` function in the following code, both ``x`` and ``print`` are + :term:`free variables `, but only ``x`` is a *closure variable*:: + + def outer(): + x = 0 + def inner(): + nonlocal x + x += 1 + print(x) + return inner + + Due to the :attr:`codeobject.co_freevars` attribute (which, despite its name, only + includes the names of closure variables rather than listing all referenced free + variables), the more general :term:`free variable` term is sometimes used even + when the intended meaning is to refer specifically to closure variables. + complex number An extension of the familiar real number system in which all numbers are expressed as a sum of a real part and an imaginary part. Imaginary @@ -237,19 +265,33 @@ Glossary advanced mathematical feature. If you're not aware of a need for them, it's almost certain you can safely ignore them. + context + This term has different meanings depending on where and how it is used. + Some common meanings: + + * The temporary state or environment established by a :term:`context + manager` via a :keyword:`with` statement. + * The collection of key­value bindings associated with a particular + :class:`contextvars.Context` object and accessed via + :class:`~contextvars.ContextVar` objects. Also see :term:`context + variable`. + * A :class:`contextvars.Context` object. Also see :term:`current + context`. + + context management protocol + The :meth:`~object.__enter__` and :meth:`~object.__exit__` methods called + by the :keyword:`with` statement. See :pep:`343`. + context manager - An object which controls the environment seen in a :keyword:`with` - statement by defining :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. - See :pep:`343`. + An object which implements the :term:`context management protocol` and + controls the environment seen in a :keyword:`with` statement. See + :pep:`343`. context variable - A variable which can have different values depending on its context. - This is similar to Thread-Local Storage in which each execution - thread may have a different value for a variable. However, with context - variables, there may be several contexts in one execution thread and the - main usage for context variables is to keep track of variables in + A variable whose value depends on which context is the :term:`current + context`. Values are accessed via :class:`contextvars.ContextVar` + objects. Context variables are primarily used to isolate state between concurrent asynchronous tasks. - See :mod:`contextvars`. contiguous .. index:: C-contiguous, Fortran contiguous @@ -283,6 +325,14 @@ Glossary is used when necessary to distinguish this implementation from others such as Jython or IronPython. + current context + The :term:`context` (:class:`contextvars.Context` object) that is + currently used by :class:`~contextvars.ContextVar` objects to access (get + or set) the values of :term:`context variables `. Each + thread has its own current context. Frameworks for executing asynchronous + tasks (see :mod:`asyncio`) associate each task with a context which + becomes the current context whenever the task starts or resumes execution. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for @@ -341,7 +391,7 @@ Glossary docstring A string literal which appears as the first expression in a class, function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`!__doc__` attribute + recognized by the compiler and put into the :attr:`~definition.__doc__` attribute of the enclosing class, function or module. Since it is available via introspection, it is the canonical place for documentation of the object. @@ -365,6 +415,11 @@ Glossary statements. The technique contrasts with the :term:`LBYL` style common to many other languages such as C. + evaluate function + A function that can be called to evaluate a lazily evaluated attribute + of an object, such as the value of type aliases created with the :keyword:`type` + statement. + expression A piece of syntax which can be evaluated to some value. In other words, an expression is an accumulation of expression elements like literals, @@ -424,11 +479,11 @@ Glossary An object that tries to find the :term:`loader` for a module that is being imported. - Since Python 3.3, there are two types of finder: :term:`meta path finders + There are two types of finder: :term:`meta path finders ` for use with :data:`sys.meta_path`, and :term:`path entry finders ` for use with :data:`sys.path_hooks`. - See :pep:`302`, :pep:`420` and :pep:`451` for much more detail. + See :ref:`finders-and-loaders` and :mod:`importlib` for much more detail. floor division Mathematical division that rounds down to nearest integer. The floor @@ -437,6 +492,19 @@ Glossary division. Note that ``(-11) // 4`` is ``-3`` because that is ``-2.75`` rounded *downward*. See :pep:`238`. + free threading + A threading model where multiple threads can run Python bytecode + simultaneously within the same interpreter. This is in contrast to + the :term:`global interpreter lock` which allows only one thread to + execute Python bytecode at a time. See :pep:`703`. + + free variable + Formally, as defined in the :ref:`language execution model `, a free + variable is any variable used in a namespace which is not a local variable in that + namespace. See :term:`closure variable` for an example. + Pragmatically, due to the name of the :attr:`codeobject.co_freevars` attribute, + the term is also sometimes used as a synonym for :term:`closure variable`. + function A series of statements which returns some value to a caller. It can also be passed zero or more :term:`arguments ` which may be used in @@ -547,12 +615,12 @@ Glossary tasks such as compression or hashing. Also, the GIL is always released when doing I/O. - Past efforts to create a "free-threaded" interpreter (one which locks - shared data at a much finer granularity) have not been successful - because performance suffered in the common single-processor case. It - is believed that overcoming this performance issue would make the - implementation much more complicated and therefore costlier to maintain. - + As of Python 3.13, the GIL can be disabled using the :option:`--disable-gil` + build configuration. After building Python with this option, code must be + run with :option:`-X gil=0 <-X>` or after setting the :envvar:`PYTHON_GIL=0 ` + environment variable. This feature enables improved performance for + multi-threaded applications and makes it easier to use multi-core CPUs + efficiently. For more details, see :pep:`703`. hash-based pyc A bytecode cache file that uses the hash rather than the last-modified @@ -583,14 +651,12 @@ Glossary which ships with the standard distribution of Python. immortal - If an object is immortal, its reference count is never modified, and - therefore it is never deallocated. + *Immortal objects* are a CPython implementation detail introduced + in :pep:`683`. - Built-in strings and singletons are immortal objects. For example, - :const:`True` and :const:`None` singletons are immmortal. - - See `PEP 683 – Immortal Objects, Using a Fixed Refcount - `_ for more information. + If an object is immortal, its :term:`reference count` is never modified, + and therefore it is never deallocated while the interpreter is running. + For example, :const:`True` and :const:`None` are immortal in CPython. immutable An object with a fixed value. Immutable objects include numbers, strings and @@ -620,7 +686,8 @@ Glossary execute them and see their results. Just launch ``python`` with no arguments (possibly by selecting it from your computer's main menu). It is a very powerful way to test out new ideas or inspect - modules and packages (remember ``help(x)``). + modules and packages (remember ``help(x)``). For more on interactive + mode, see :ref:`tut-interac`. interpreted Python is an interpreted language, as opposed to a compiled one, @@ -686,6 +753,9 @@ Glossary CPython does not consistently apply the requirement that an iterator define :meth:`~iterator.__iter__`. + And also please note that the free-threading CPython does not guarantee + the thread-safety of iterator operations. + key function A key function or collation function is a callable that returns a value @@ -727,18 +797,6 @@ Glossary thread removes *key* from *mapping* after the test, but before the lookup. This issue can be solved with locks or by using the EAFP approach. - locale encoding - On Unix, it is the encoding of the LC_CTYPE locale. It can be set with - :func:`locale.setlocale(locale.LC_CTYPE, new_locale) `. - - On Windows, it is the ANSI code page (ex: ``"cp1252"``). - - On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. - - ``locale.getencoding()`` can be used to get the locale encoding. - - See also the :term:`filesystem encoding and error handler`. - list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to @@ -755,8 +813,23 @@ Glossary loader An object that loads a module. It must define a method named :meth:`load_module`. A loader is typically returned by a - :term:`finder`. See :pep:`302` for details and - :class:`importlib.abc.Loader` for an :term:`abstract base class`. + :term:`finder`. See also: + + * :ref:`finders-and-loaders` + * :class:`importlib.abc.Loader` + * :pep:`302` + + locale encoding + On Unix, it is the encoding of the LC_CTYPE locale. It can be set with + :func:`locale.setlocale(locale.LC_CTYPE, new_locale) `. + + On Windows, it is the ANSI code page (ex: ``"cp1252"``). + + On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. + + :func:`locale.getencoding` can be used to get the locale encoding. + + See also the :term:`filesystem encoding and error handler`. magic method .. index:: pair: magic; method @@ -800,8 +873,7 @@ Glossary method resolution order Method Resolution Order is the order in which base classes are searched - for a member during lookup. See `The Python 2.3 Method Resolution Order - `_ for details of the + for a member during lookup. See :ref:`python_2.3_mro` for details of the algorithm used by the Python interpreter since the 2.3 release. module @@ -815,6 +887,8 @@ Glossary A namespace containing the import-related information used to load a module. An instance of :class:`importlib.machinery.ModuleSpec`. + See also :ref:`module-specs`. + MRO See :term:`method resolution order`. @@ -841,10 +915,11 @@ Glossary Some named tuples are built-in types (such as the above examples). Alternatively, a named tuple can be created from a regular class definition that inherits from :class:`tuple` and that defines named - fields. Such a class can be written by hand or it can be created with - the factory function :func:`collections.namedtuple`. The latter - technique also adds some extra methods that may not be found in - hand-written or built-in named tuples. + fields. Such a class can be written by hand, or it can be created by + inheriting :class:`typing.NamedTuple`, or with the factory function + :func:`collections.namedtuple`. The latter techniques also add some + extra methods that may not be found in hand-written or built-in named + tuples. namespace The place where a variable is stored. Namespaces are implemented as @@ -887,6 +962,15 @@ Glossary (methods). Also the ultimate base class of any :term:`new-style class`. + optimized scope + A scope where target local variable names are reliably known to the + compiler when the code is compiled, allowing optimization of read and + write access to these names. The local namespaces for functions, + generators, coroutines, comprehensions, and generator expressions are + optimized in this fashion. Note: most interpreter optimizations are + applied to all scopes, only those relying on a known set of local + and nonlocal variable names are restricted to optimized scopes. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a @@ -1084,6 +1168,10 @@ Glossary See also :term:`namespace package`. + REPL + An acronym for the "read–eval–print loop", another name for the + :term:`interactive` interpreter shell. + __slots__ A declaration inside a class that saves memory by pre-declaring space for instance attributes and eliminating instance dictionaries. Though @@ -1099,7 +1187,7 @@ Glossary :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also supports :meth:`~object.__getitem__` and :meth:`!__len__`, but is considered a mapping rather than a sequence because the lookups use arbitrary - :term:`immutable` keys rather than integers. + :term:`hashable` keys rather than integers. The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just @@ -1128,16 +1216,12 @@ Glossary (subscript) notation uses :class:`slice` objects internally. soft deprecated - A soft deprecation can be used when using an API which should no longer - be used to write new code, but it remains safe to continue using it in - existing code. The API remains documented and tested, but will not be - developed further (no enhancement). - - The main difference between a "soft" and a (regular) "hard" deprecation - is that the soft deprecation does not imply scheduling the removal of the - deprecated API. + A soft deprecated API should not be used in new code, + but it is safe for already existing code to use it. + The API remains documented and tested, but will not be enhanced further. - Another difference is that a soft deprecation does not issue a warning. + Soft deprecation, unlike normal deprecation, does not plan on removing the API + and will not emit warnings. See `PEP 387: Soft Deprecation `_. @@ -1209,7 +1293,7 @@ Glossary type The type of a Python object determines what kind of object it is; every object has a type. An object's type is accessible as its - :attr:`~instance.__class__` attribute or can be retrieved with + :attr:`~object.__class__` attribute or can be retrieved with ``type(obj)``. type alias diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index be8c7e6c827f57..78f3704ba5d000 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -34,11 +34,16 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer Python 3.10 adds a new function to the standard library: :func:`inspect.get_annotations`. In Python versions 3.10 -and newer, calling this function is the best practice for +through 3.13, calling this function is the best practice for accessing the annotations dict of any object that supports annotations. This function can also "un-stringize" stringized annotations for you. +In Python 3.14, there is a new :mod:`annotationlib` module +with functionality for working with annotations. This +includes a :func:`annotationlib.get_annotations` function, +which supersedes :func:`inspect.get_annotations`. + If for some reason :func:`inspect.get_annotations` isn't viable for your use case, you may access the ``__annotations__`` data member manually. Best practice @@ -102,9 +107,9 @@ Your code will have to have a separate code path if the object you're examining is a class (``isinstance(o, type)``). In that case, best practice relies on an implementation detail of Python 3.9 and before: if a class has annotations defined, -they are stored in the class's ``__dict__`` dictionary. Since +they are stored in the class's :attr:`~type.__dict__` dictionary. Since the class may or may not have annotations defined, best practice -is to call the ``get`` method on the class dict. +is to call the :meth:`~dict.get` method on the class dict. To put it all together, here is some sample code that safely accesses the ``__annotations__`` attribute on an arbitrary @@ -121,8 +126,8 @@ the type of ``ann`` using :func:`isinstance` before further examination. Note that some exotic or malformed type objects may not have -a ``__dict__`` attribute, so for extra safety you may also wish -to use :func:`getattr` to access ``__dict__``. +a :attr:`~type.__dict__` attribute, so for extra safety you may also wish +to use :func:`getattr` to access :attr:`!__dict__`. Manually Un-Stringizing Stringized Annotations @@ -184,7 +189,11 @@ Best Practices For ``__annotations__`` In Any Python Version * If you do assign directly to the ``__annotations__`` member of an object, you should always set it to a ``dict`` object. -* If you directly access the ``__annotations__`` member +* You should avoid accessing ``__annotations__`` directly on any object. + Instead, use :func:`annotationlib.get_annotations` (Python 3.14+) + or :func:`inspect.get_annotations` (Python 3.10+). + +* If you do directly access the ``__annotations__`` member of an object, you should ensure that it's a dictionary before attempting to examine its contents. @@ -231,3 +240,12 @@ itself be quoted. In effect the annotation is quoted This prints ``{'a': "'str'"}``. This shouldn't really be considered a "quirk"; it's mentioned here simply because it might be surprising. + +If you use a class with a custom metaclass and access ``__annotations__`` +on the class, you may observe unexpected behavior; see +:pep:`749 <749#pep749-metaclasses>` for some examples. You can avoid these +quirks by using :func:`annotationlib.get_annotations` on Python 3.14+ or +:func:`inspect.get_annotations` on Python 3.10+. On earlier versions of +Python, you can avoid these bugs by accessing the annotations from the +class's :attr:`~type.__dict__` +(e.g., ``cls.__dict__.get('__annotations__', None)``). diff --git a/Doc/howto/argparse-optparse.rst b/Doc/howto/argparse-optparse.rst new file mode 100644 index 00000000000000..cef2d893b28a62 --- /dev/null +++ b/Doc/howto/argparse-optparse.rst @@ -0,0 +1,55 @@ +.. currentmodule:: argparse + +.. _upgrading-optparse-code: + +========================== +Upgrading optparse code +========================== + +Originally, the :mod:`argparse` module had attempted to maintain compatibility +with :mod:`optparse`. However, :mod:`optparse` was difficult to extend +transparently, particularly with the changes required to support +``nargs=`` specifiers and better usage messages. When most everything in +:mod:`optparse` had either been copy-pasted over or monkey-patched, it no +longer seemed practical to try to maintain the backwards compatibility. + +The :mod:`argparse` module improves on the :mod:`optparse` +module in a number of ways including: + +* Handling positional arguments. +* Supporting subcommands. +* Allowing alternative option prefixes like ``+`` and ``/``. +* Handling zero-or-more and one-or-more style arguments. +* Producing more informative usage messages. +* Providing a much simpler interface for custom ``type`` and ``action``. + +A partial upgrade path from :mod:`optparse` to :mod:`argparse`: + +* Replace all :meth:`optparse.OptionParser.add_option` calls with + :meth:`ArgumentParser.add_argument` calls. + +* Replace ``(options, args) = parser.parse_args()`` with ``args = + parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` + calls for the positional arguments. Keep in mind that what was previously + called ``options``, now in the :mod:`argparse` context is called ``args``. + +* Replace :meth:`optparse.OptionParser.disable_interspersed_args` + by using :meth:`~ArgumentParser.parse_intermixed_args` instead of + :meth:`~ArgumentParser.parse_args`. + +* Replace callback actions and the ``callback_*`` keyword arguments with + ``type`` or ``action`` arguments. + +* Replace string names for ``type`` keyword arguments with the corresponding + type objects (e.g. int, float, complex, etc). + +* Replace :class:`optparse.Values` with :class:`Namespace` and + :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with + :exc:`ArgumentError`. + +* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with + the standard Python syntax to use dictionaries to format strings, that is, + ``%(default)s`` and ``%(prog)s``. + +* Replace the OptionParser constructor ``version`` argument with a call to + ``parser.add_argument('--version', action='version', version='')``. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index ae5bab90bf8131..1efbee64d60bb3 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -444,7 +444,7 @@ And the output: options: -h, --help show this help message and exit - -v {0,1,2}, --verbosity {0,1,2} + -v, --verbosity {0,1,2} increase output verbosity Note that the change also reflects both in the error message as well as the @@ -841,6 +841,53 @@ translated messages. To translate your own strings in the :mod:`argparse` output, use :mod:`gettext`. +Custom type converters +====================== + +The :mod:`argparse` module allows you to specify custom type converters for +your command-line arguments. This allows you to modify user input before it's +stored in the :class:`argparse.Namespace`. This can be useful when you need to +pre-process the input before it is used in your program. + +When using a custom type converter, you can use any callable that takes a +single string argument (the argument value) and returns the converted value. +However, if you need to handle more complex scenarios, you can use a custom +action class with the **action** parameter instead. + +For example, let's say you want to handle arguments with different prefixes and +process them accordingly:: + + import argparse + + parser = argparse.ArgumentParser(prefix_chars='-+') + + parser.add_argument('-a', metavar='', action='append', + type=lambda x: ('-', x)) + parser.add_argument('+a', metavar='', action='append', + type=lambda x: ('+', x)) + + args = parser.parse_args() + print(args) + +Output: + +.. code-block:: shell-session + + $ python prog.py -a value1 +a value2 + Namespace(a=[('-', 'value1'), ('+', 'value2')]) + +In this example, we: + +* Created a parser with custom prefix characters using the ``prefix_chars`` + parameter. + +* Defined two arguments, ``-a`` and ``+a``, which used the ``type`` parameter to + create custom type converters to store the value in a tuple with the prefix. + +Without the custom type converters, the arguments would have treated the ``-a`` +and ``+a`` as the same argument, which would have been undesirable. By using custom +type converters, we were able to differentiate between the two arguments. + Conclusion ========== diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 4828e2fa29bd24..f9ad81e38f8dc3 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -43,7 +43,7 @@ appearance---and the curses library will figure out what control codes need to be sent to the terminal to produce the right output. curses doesn't provide many user-interface concepts such as buttons, checkboxes, or dialogs; if you need such features, consider a user interface library such as -`Urwid `_. +:pypi:`Urwid`. The curses library was originally written for BSD Unix; the later System V versions of Unix from AT&T added many enhancements and new functions. BSD curses @@ -56,8 +56,7 @@ versions of curses carried by some proprietary Unixes may not support everything, though. The Windows version of Python doesn't include the :mod:`curses` -module. A ported version called `UniCurses -`_ is available. +module. A ported version called :pypi:`UniCurses` is available. The Python curses module @@ -429,8 +428,7 @@ User Input The C curses library offers only very simple input mechanisms. Python's :mod:`curses` module adds a basic text-input widget. (Other libraries -such as `Urwid `_ have more extensive -collections of widgets.) +such as :pypi:`Urwid` have more extensive collections of widgets.) There are two methods for getting input from a window: diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 75346f2c7618c2..01264bfe823746 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1,8 +1,8 @@ .. _descriptorhowto: -====================== -Descriptor HowTo Guide -====================== +================ +Descriptor Guide +================ :Author: Raymond Hettinger :Contact: @@ -389,7 +389,9 @@ Here are three practical data validation utilities: def validate(self, value): if value not in self.options: - raise ValueError(f'Expected {value!r} to be one of {self.options!r}') + raise ValueError( + f'Expected {value!r} to be one of {self.options!r}' + ) class Number(Validator): @@ -469,6 +471,7 @@ The descriptors prevent invalid instances from being created: Traceback (most recent call last): ... ValueError: Expected -5 to be at least 0 + >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number Traceback (most recent call last): ... @@ -513,7 +516,7 @@ were defined. Descriptors are a powerful, general purpose protocol. They are the mechanism behind properties, methods, static methods, class methods, and -:func:`super()`. They are used throughout Python itself. Descriptors +:func:`super`. They are used throughout Python itself. Descriptors simplify the underlying C code and offer a flexible set of new tools for everyday Python programs. @@ -559,8 +562,8 @@ attribute access. The expression ``obj.x`` looks up the attribute ``x`` in the chain of namespaces for ``obj``. If the search finds a descriptor outside of the -instance ``__dict__``, its :meth:`__get__` method is invoked according to the -precedence rules listed below. +instance :attr:`~object.__dict__`, its :meth:`~object.__get__` method is +invoked according to the precedence rules listed below. The details of invocation depend on whether ``obj`` is an object, class, or instance of super. @@ -787,7 +790,7 @@ Invocation from super --------------------- The logic for super's dotted lookup is in the :meth:`__getattribute__` method for -object returned by :class:`super()`. +object returned by :func:`super`. A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` for the base class ``B`` immediately following ``A`` and then returns @@ -803,7 +806,7 @@ The full C implementation can be found in :c:func:`!super_getattro` in Summary of invocation logic --------------------------- -The mechanism for descriptors is embedded in the :meth:`__getattribute__()` +The mechanism for descriptors is embedded in the :meth:`__getattribute__` methods for :class:`object`, :class:`type`, and :func:`super`. The important points to remember are: @@ -990,7 +993,7 @@ The documentation shows a typical use to define a managed attribute ``x``: AttributeError: 'C' object has no attribute '_C__x' To see how :func:`property` is implemented in terms of the descriptor protocol, -here is a pure Python equivalent: +here is a pure Python equivalent that implements most of the core functionality: .. testcode:: @@ -1004,48 +1007,35 @@ here is a pure Python equivalent: if doc is None and fget is not None: doc = fget.__doc__ self.__doc__ = doc - self._name = '' def __set_name__(self, owner, name): - self._name = name + self.__name__ = name def __get__(self, obj, objtype=None): if obj is None: return self if self.fget is None: - raise AttributeError( - f'property {self._name!r} of {type(obj).__name__!r} object has no getter' - ) + raise AttributeError return self.fget(obj) def __set__(self, obj, value): if self.fset is None: - raise AttributeError( - f'property {self._name!r} of {type(obj).__name__!r} object has no setter' - ) + raise AttributeError self.fset(obj, value) def __delete__(self, obj): if self.fdel is None: - raise AttributeError( - f'property {self._name!r} of {type(obj).__name__!r} object has no deleter' - ) + raise AttributeError self.fdel(obj) def getter(self, fget): - prop = type(self)(fget, self.fset, self.fdel, self.__doc__) - prop._name = self._name - return prop + return type(self)(fget, self.fset, self.fdel, self.__doc__) def setter(self, fset): - prop = type(self)(self.fget, fset, self.fdel, self.__doc__) - prop._name = self._name - return prop + return type(self)(self.fget, fset, self.fdel, self.__doc__) def deleter(self, fdel): - prop = type(self)(self.fget, self.fset, fdel, self.__doc__) - prop._name = self._name - return prop + return type(self)(self.fget, self.fset, fdel, self.__doc__) .. testcode:: :hide: @@ -1108,23 +1098,23 @@ here is a pure Python equivalent: >>> try: ... cc.no_getter ... except AttributeError as e: - ... e.args[0] + ... type(e).__name__ ... - "property 'no_getter' of 'CC' object has no getter" + 'AttributeError' >>> try: ... cc.no_setter = 33 ... except AttributeError as e: - ... e.args[0] + ... type(e).__name__ ... - "property 'no_setter' of 'CC' object has no setter" + 'AttributeError' >>> try: ... del cc.no_deleter ... except AttributeError as e: - ... e.args[0] + ... type(e).__name__ ... - "property 'no_deleter' of 'CC' object has no deleter" + 'AttributeError' >>> CC.no_doc.__doc__ is None True @@ -1192,6 +1182,10 @@ roughly equivalent to: "Emulate method_getattro() in Objects/classobject.c" return getattr(self.__func__, name) + def __get__(self, obj, objtype=None): + "Emulate method_descr_get() in Objects/classobject.c" + return self + To support automatic creation of methods, functions include the :meth:`__get__` method for binding methods during attribute access. This means that functions are non-data descriptors that return bound methods @@ -1214,8 +1208,20 @@ descriptor works in practice: .. testcode:: class D: - def f(self, x): - return x + def f(self): + return self + + class D2: + pass + +.. doctest:: + :hide: + + >>> d = D() + >>> d2 = D2() + >>> d2.f = d.f.__get__(d2, D2) + >>> d2.f() is d + True The function has a :term:`qualified name` attribute to support introspection: @@ -1299,8 +1305,8 @@ mean, median, and other descriptive statistics that depend on the data. However, there may be useful functions which are conceptually related but do not depend on the data. For instance, ``erf(x)`` is handy conversion routine that comes up in statistical work but does not directly depend on a particular dataset. -It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or -``Sample.erf(1.5) --> .9332``. +It can be called either from an object or the class: ``s.erf(1.5) --> 0.9332`` +or ``Sample.erf(1.5) --> 0.9332``. Since static methods return the underlying function with no changes, the example calls are unexciting: @@ -1339,11 +1345,15 @@ Using the non-data descriptor protocol, a pure Python version of def __call__(self, *args, **kwds): return self.f(*args, **kwds) + @property + def __annotations__(self): + return self.f.__annotations__ + The :func:`functools.update_wrapper` call adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look like the wrapped -function: :attr:`~function.__name__`, :attr:`~function.__qualname__`, -:attr:`~function.__doc__`, and :attr:`~function.__annotations__`. +function, including :attr:`~function.__name__`, :attr:`~function.__qualname__`, +and :attr:`~function.__doc__`. .. testcode:: :hide: diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 30be15230fc088..66929b4104d8de 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -1,3 +1,5 @@ +.. _enum-howto: + ========== Enum HOWTO ========== @@ -7,7 +9,7 @@ Enum HOWTO .. currentmodule:: enum An :class:`Enum` is a set of symbolic names bound to unique values. They are -similar to global variables, but they offer a more useful :func:`repr()`, +similar to global variables, but they offer a more useful :func:`repr`, grouping, type-safety, and a few other features. They are most useful when you have a variable that can take one of a limited @@ -165,7 +167,7 @@ And a function to display the chores for a given day:: answer SO questions In cases where the actual values of the members do not matter, you can save -yourself some work and use :func:`auto()` for the values:: +yourself some work and use :func:`auto` for the values:: >>> from enum import auto >>> class Weekday(Flag): @@ -606,7 +608,7 @@ The solution is to specify the module name explicitly as follows:: the source, pickling will be disabled. The new pickle protocol 4 also, in some circumstances, relies on -:attr:`~definition.__qualname__` being set to the location where pickle will be able +:attr:`~type.__qualname__` being set to the location where pickle will be able to find the class. For example, if the class was made available in class SomeData in the global scope:: @@ -1150,6 +1152,14 @@ the following are true: >>> (Color.RED | Color.GREEN).name 'RED|GREEN' + >>> class Perm(IntFlag): + ... R = 4 + ... W = 2 + ... X = 1 + ... + >>> (Perm.R & Perm.W).name is None # effectively Perm(0) + True + - multi-bit flags, aka aliases, can be returned from operations:: >>> Color.RED | Color.BLUE diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst new file mode 100644 index 00000000000000..6abe93d71ad529 --- /dev/null +++ b/Doc/howto/free-threading-extensions.rst @@ -0,0 +1,280 @@ +.. highlight:: c + +.. _freethreading-extensions-howto: + +****************************************** +C API Extension Support for Free Threading +****************************************** + +Starting with the 3.13 release, CPython has experimental support for running +with the :term:`global interpreter lock` (GIL) disabled in a configuration +called :term:`free threading`. This document describes how to adapt C API +extensions to support free threading. + + +Identifying the Free-Threaded Build in C +======================================== + +The CPython C API exposes the ``Py_GIL_DISABLED`` macro: in the free-threaded +build it's defined to ``1``, and in the regular build it's not defined. +You can use it to enable code that only runs under the free-threaded build:: + + #ifdef Py_GIL_DISABLED + /* code that only runs in the free-threaded build */ + #endif + +Module Initialization +===================== + +Extension modules need to explicitly indicate that they support running with +the GIL disabled; otherwise importing the extension will raise a warning and +enable the GIL at runtime. + +There are two ways to indicate that an extension module supports running with +the GIL disabled depending on whether the extension uses multi-phase or +single-phase initialization. + +Multi-Phase Initialization +.......................... + +Extensions that use multi-phase initialization (i.e., +:c:func:`PyModuleDef_Init`) should add a :c:data:`Py_mod_gil` slot in the +module definition. If your extension supports older versions of CPython, +you should guard the slot with a :c:data:`PY_VERSION_HEX` check. + +:: + + static struct PyModuleDef_Slot module_slots[] = { + ... + #if PY_VERSION_HEX >= 0x030D0000 + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + {0, NULL} + }; + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + .m_slots = module_slots, + ... + }; + + +Single-Phase Initialization +........................... + +Extensions that use single-phase initialization (i.e., +:c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to +indicate that they support running with the GIL disabled. The function is +only defined in the free-threaded build, so you should guard the call with +``#ifdef Py_GIL_DISABLED`` to avoid compilation errors in the regular build. + +:: + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + ... + }; + + PyMODINIT_FUNC + PyInit_mymodule(void) + { + PyObject *m = PyModule_Create(&moduledef); + if (m == NULL) { + return NULL; + } + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); + #endif + return m; + } + + +General API Guidelines +====================== + +Most of the C API is thread-safe, but there are some exceptions. + +* **Struct Fields**: Accessing fields in Python C API objects or structs + directly is not thread-safe if the field may be concurrently modified. +* **Macros**: Accessor macros like :c:macro:`PyList_GET_ITEM` and + :c:macro:`PyList_SET_ITEM` do not perform any error checking or locking. + These macros are not thread-safe if the container object may be modified + concurrently. +* **Borrowed References**: C API functions that return + :term:`borrowed references ` may not be thread-safe if + the containing object is modified concurrently. See the section on + :ref:`borrowed references ` for more information. + + +Container Thread Safety +....................... + +Containers like :c:struct:`PyListObject`, +:c:struct:`PyDictObject`, and :c:struct:`PySetObject` perform internal locking +in the free-threaded build. For example, the :c:func:`PyList_Append` will +lock the list before appending an item. + +.. _PyDict_Next: + +``PyDict_Next`` +''''''''''''''' + +A notable exception is :c:func:`PyDict_Next`, which does not lock the +dictionary. You should use :c:macro:`Py_BEGIN_CRITICAL_SECTION` to protect +the dictionary while iterating over it if the dictionary may be concurrently +modified:: + + Py_BEGIN_CRITICAL_SECTION(dict); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(dict, &pos, &key, &value)) { + ... + } + Py_END_CRITICAL_SECTION(); + + +Borrowed References +=================== + +.. _borrowed-references: + +Some C API functions return :term:`borrowed references `. +These APIs are not thread-safe if the containing object is modified +concurrently. For example, it's not safe to use :c:func:`PyList_GetItem` +if the list may be modified concurrently. + +The following table lists some borrowed reference APIs and their replacements +that return :term:`strong references `. + ++-----------------------------------+-----------------------------------+ +| Borrowed reference API | Strong reference API | ++===================================+===================================+ +| :c:func:`PyList_GetItem` | :c:func:`PyList_GetItemRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_GetItem` | :c:func:`PyDict_GetItemRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_GetItemWithError` | :c:func:`PyDict_GetItemRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_GetItemString` | :c:func:`PyDict_GetItemStringRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_SetDefault` | :c:func:`PyDict_SetDefaultRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_Next` | none (see :ref:`PyDict_Next`) | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyImport_AddModule` | :c:func:`PyImport_AddModuleRef` | ++-----------------------------------+-----------------------------------+ + +Not all APIs that return borrowed references are problematic. For +example, :c:func:`PyTuple_GetItem` is safe because tuples are immutable. +Similarly, not all uses of the above APIs are problematic. For example, +:c:func:`PyDict_GetItem` is often used for parsing keyword argument +dictionaries in function calls; those keyword argument dictionaries are +effectively private (not accessible by other threads), so using borrowed +references in that context is safe. + +Some of these functions were added in Python 3.13. You can use the +`pythoncapi-compat `_ package +to provide implementations of these functions for older Python versions. + + +.. _free-threaded-memory-allocation: + +Memory Allocation APIs +====================== + +Python's memory management C API provides functions in three different +:ref:`allocation domains `: "raw", "mem", and "object". +For thread-safety, the free-threaded build requires that only Python objects +are allocated using the object domain, and that all Python object are +allocated using that domain. This differs from the prior Python versions, +where this was only a best practice and not a hard requirement. + +.. note:: + + Search for uses of :c:func:`PyObject_Malloc` in your + extension and check that the allocated memory is used for Python objects. + Use :c:func:`PyMem_Malloc` to allocate buffers instead of + :c:func:`PyObject_Malloc`. + + +Thread State and GIL APIs +========================= + +Python provides a set of functions and macros to manage thread state and the +GIL, such as: + +* :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` +* :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread` +* :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` + +These functions should still be used in the free-threaded build to manage +thread state even when the :term:`GIL` is disabled. For example, if you +create a thread outside of Python, you must call :c:func:`PyGILState_Ensure` +before calling into the Python API to ensure that the thread has a valid +Python thread state. + +You should continue to call :c:func:`PyEval_SaveThread` or +:c:macro:`Py_BEGIN_ALLOW_THREADS` around blocking operations, such as I/O or +lock acquisitions, to allow other threads to run the +:term:`cyclic garbage collector `. + + +Protecting Internal Extension State +=================================== + +Your extension may have internal state that was previously protected by the +GIL. You may need to add locking to protect this state. The approach will +depend on your extension, but some common patterns include: + +* **Caches**: global caches are a common source of shared state. Consider + using a lock to protect the cache or disabling it in the free-threaded build + if the cache is not critical for performance. +* **Global State**: global state may need to be protected by a lock or moved + to thread local storage. C11 and C++11 provide the ``thread_local`` or + ``_Thread_local`` for + `thread-local storage `_. + + +Building Extensions for the Free-Threaded Build +=============================================== + +C API extensions need to be built specifically for the free-threaded build. +The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. + +* `pypa/manylinux `_ supports the + free-threaded build, with the ``t`` suffix, such as ``python3.13t``. +* `pypa/cibuildwheel `_ supports the + free-threaded build if you set + `CIBW_FREE_THREADED_SUPPORT `_. + +Limited C API and Stable ABI +............................ + +The free-threaded build does not currently support the +:ref:`Limited C API ` or the stable ABI. If you use +`setuptools `_ to build +your extension and currently set ``py_limited_api=True`` you can use +``py_limited_api=not sysconfig.get_config_var("Py_GIL_DISABLED")`` to opt out +of the limited API when building with the free-threaded build. + +.. note:: + You will need to build separate wheels specifically for the free-threaded + build. If you currently use the stable ABI, you can continue to build a + single wheel for multiple non-free-threaded Python versions. + + +Windows +....... + +Due to a limitation of the official Windows installer, you will need to +manually define ``Py_GIL_DISABLED=1`` when building extensions from source. + +.. seealso:: + + `Porting Extension Modules to Support Free-Threading + `_: + A community-maintained porting guide for extension authors. diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst new file mode 100644 index 00000000000000..b21e3287ecaa3f --- /dev/null +++ b/Doc/howto/free-threading-python.rst @@ -0,0 +1,154 @@ +.. _freethreading-python-howto: + +********************************************** +Python experimental support for free threading +********************************************** + +Starting with the 3.13 release, CPython has experimental support for a build of +Python called :term:`free threading` where the :term:`global interpreter lock` +(GIL) is disabled. Free-threaded execution allows for full utilization of the +available processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multi-core hardware. + +**The free-threaded mode is experimental** and work is ongoing to improve it: +expect some bugs and a substantial single-threaded performance hit. + +This document describes the implications of free threading +for Python code. See :ref:`freethreading-extensions-howto` for information on +how to write C extensions that support the free-threaded build. + +.. seealso:: + + :pep:`703` – Making the Global Interpreter Lock Optional in CPython for an + overall description of free-threaded Python. + + +Installation +============ + +Starting with Python 3.13, the official macOS and Windows installers +optionally support installing free-threaded Python binaries. The installers +are available at https://www.python.org/downloads/. + +For information on other platforms, see the `Installing a Free-Threaded Python +`_, a +community-maintained installation guide for installing free-threaded Python. + +When building CPython from source, the :option:`--disable-gil` configure option +should be used to build a free-threaded Python interpreter. + + +Identifying free-threaded Python +================================ + +To check if the current interpreter supports free-threading, :option:`python -VV <-V>` +and :attr:`sys.version` contain "experimental free-threading build". +The new :func:`sys._is_gil_enabled` function can be used to check whether +the GIL is actually disabled in the running process. + +The ``sysconfig.get_config_var("Py_GIL_DISABLED")`` configuration variable can +be used to determine whether the build supports free threading. If the variable +is set to ``1``, then the build supports free threading. This is the recommended +mechanism for decisions related to the build configuration. + + +The global interpreter lock in free-threaded Python +=================================================== + +Free-threaded builds of CPython support optionally running with the GIL enabled +at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command-line option :option:`-X gil`. + +The GIL may also automatically be enabled when importing a C-API extension +module that is not explicitly marked as supporting free threading. A warning +will be printed in this case. + +In addition to individual package documentation, the following websites track +the status of popular packages support for free threading: + +* https://py-free-threading.github.io/tracking/ +* https://hugovk.github.io/free-threaded-wheels/ + + +Thread safety +============= + +The free-threaded build of CPython aims to provide similar thread-safety +behavior at the Python level to the default GIL-enabled build. Built-in +types like :class:`dict`, :class:`list`, and :class:`set` use internal locks +to protect against concurrent modifications in ways that behave similarly to +the GIL. However, Python has not historically guaranteed specific behavior for +concurrent modifications to these built-in types, so this should be treated +as a description of the current implementation, not a guarantee of current or +future behavior. + +.. note:: + + It's recommended to use the :class:`threading.Lock` or other synchronization + primitives instead of relying on the internal locks of built-in types, when + possible. + + +Known limitations +================= + +This section describes known limitations of the free-threaded CPython build. + +Immortalization +--------------- + +The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +Immortal objects are not deallocated and have reference counts that are +never modified. This is done to avoid reference count contention that would +prevent efficient multi-threaded scaling. + +An object will be made immortal when a new thread is started for the first time +after the main thread is running. The following objects are immortalized: + +* :ref:`function ` objects declared at the module level +* :ref:`method ` descriptors +* :ref:`code ` objects +* :term:`module` objects and their dictionaries +* :ref:`classes ` (type objects) + +Because immortal objects are never deallocated, applications that create many +objects of these types may see increased memory usage. This is expected to be +addressed in the 3.14 release. + +Additionally, numeric and string literals in the code as well as strings +returned by :func:`sys.intern` are also immortalized. This behavior is +expected to remain in the 3.14 free-threaded build. + + +Frame objects +------------- + +It is not safe to access :ref:`frame ` objects from other +threads and doing so may cause your program to crash . This means that +:func:`sys._current_frames` is generally not safe to use in a free-threaded +build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` +are generally safe as long as the resulting frame object is not passed to +another thread. + +Iterators +--------- + +Sharing the same iterator object between multiple threads is generally not +safe and threads may see duplicate or missing elements when iterating or crash +the interpreter. + + +Single-threaded performance +--------------------------- + +The free-threaded build has additional overhead when executing Python code +compared to the default GIL-enabled build. In 3.13, this overhead is about +40% on the `pyperformance `_ suite. +Programs that spend most of their time in C extensions or I/O will see +less of an impact. The largest impact is because the specializing adaptive +interpreter (:pep:`659`) is disabled in the free-threaded build. We expect +to re-enable it in a thread-safe way in the 3.14 release. This overhead is +expected to be reduced in upcoming Python release. We are aiming for an +overhead of 10% or less on the pyperformance suite compared to the default +GIL-enabled build. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index b0f9d22d74f0e3..1f0608fb0fc53f 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1,3 +1,5 @@ +.. _functional-howto: + ******************************** Functional Programming HOWTO ******************************** diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst new file mode 100644 index 00000000000000..53bbf7ddaa2ab9 --- /dev/null +++ b/Doc/howto/gdb_helpers.rst @@ -0,0 +1,449 @@ +.. _gdb: + +========================================================= +Debugging C API extensions and CPython Internals with GDB +========================================================= + +.. highlight:: none + +This document explains how the Python GDB extension, ``python-gdb.py``, can +be used with the GDB debugger to debug CPython extensions and the +CPython interpreter itself. + +When debugging low-level problems such as crashes or deadlocks, a low-level +debugger, such as GDB, is useful to diagnose and correct the issue. +By default, GDB (or any of its front-ends) doesn't support high-level +information specific to the CPython interpreter. + +The ``python-gdb.py`` extension adds CPython interpreter information to GDB. +The extension helps introspect the stack of currently executing Python functions. +Given a Python object represented by a :c:expr:`PyObject *` pointer, +the extension surfaces the type and value of the object. + +Developers who are working on CPython extensions or tinkering with parts +of CPython that are written in C can use this document to learn how to use the +``python-gdb.py`` extension with GDB. + +.. note:: + + This document assumes that you are familiar with the basics of GDB and the + CPython C API. It consolidates guidance from the + `devguide `_ and the + `Python wiki `_. + + +Prerequisites +============= + +You need to have: + +- GDB 7 or later. (For earlier versions of GDB, see ``Misc/gdbinit`` in the + sources of Python 3.11 or earlier.) +- GDB-compatible debugging information for Python and any extension you are + debugging. +- The ``python-gdb.py`` extension. + +The extension is built with Python, but might be distributed separately or +not at all. Below, we include tips for a few common systems as examples. +Note that even if the instructions match your system, they might be outdated. + + +Setup with Python built from source +----------------------------------- + +When you build CPython from source, debugging information should be available, +and the build should add a ``python-gdb.py`` file to the root directory of +your repository. + +To activate support, you must add the directory containing ``python-gdb.py`` +to GDB's "auto-load-safe-path". +If you haven't done this, recent versions of GDB will print out a warning +with instructions on how to do this. + +.. note:: + + If you do not see instructions for your version of GDB, put this in your + configuration file (``~/.gdbinit`` or ``~/.config/gdb/gdbinit``):: + + add-auto-load-safe-path /path/to/cpython + + You can also add multiple paths, separated by ``:``. + + +Setup for Python from a Linux distro +------------------------------------ + +Most Linux systems provide debug information for the system Python +in a package called ``python-debuginfo``, ``python-dbg`` or similar. +For example: + +- Fedora: + + .. code-block:: shell + + sudo dnf install gdb + sudo dnf debuginfo-install python3 + +- Ubuntu: + + .. code-block:: shell + + sudo apt install gdb python3-dbg + +On several recent Linux systems, GDB can download debugging symbols +automatically using *debuginfod*. +However, this will not install the ``python-gdb.py`` extension; +you generally do need to install the debug info package separately. + + +Using the Debug build and Development mode +========================================== + +For easier debugging, you might want to: + +- Use a :ref:`debug build ` of Python. (When building from source, + use ``configure --with-pydebug``. On Linux distros, install and run a package + like ``python-debug`` or ``python-dbg``, if available.) +- Use the runtime :ref:`development mode ` (``-X dev``). + +Both enable extra assertions and disable some optimizations. +Sometimes this hides the bug you are trying to find, but in most cases they +make the process easier. + + +Using the ``python-gdb`` extension +================================== + +When the extension is loaded, it provides two main features: +pretty printers for Python values, and additional commands. + +Pretty-printers +--------------- + +This is what a GDB backtrace looks like (truncated) when this extension is +enabled:: + + #0 0x000000000041a6b1 in PyObject_Malloc (nbytes=Cannot access memory at address 0x7fffff7fefe8 + ) at Objects/obmalloc.c:748 + #1 0x000000000041b7c0 in _PyObject_DebugMallocApi (id=111 'o', nbytes=24) at Objects/obmalloc.c:1445 + #2 0x000000000041b717 in _PyObject_DebugMalloc (nbytes=24) at Objects/obmalloc.c:1412 + #3 0x000000000044060a in _PyUnicode_New (length=11) at Objects/unicodeobject.c:346 + #4 0x00000000004466aa in PyUnicodeUCS2_DecodeUTF8Stateful (s=0x5c2b8d "__lltrace__", size=11, errors=0x0, consumed= + 0x0) at Objects/unicodeobject.c:2531 + #5 0x0000000000446647 in PyUnicodeUCS2_DecodeUTF8 (s=0x5c2b8d "__lltrace__", size=11, errors=0x0) + at Objects/unicodeobject.c:2495 + #6 0x0000000000440d1b in PyUnicodeUCS2_FromStringAndSize (u=0x5c2b8d "__lltrace__", size=11) + at Objects/unicodeobject.c:551 + #7 0x0000000000440d94 in PyUnicodeUCS2_FromString (u=0x5c2b8d "__lltrace__") at Objects/unicodeobject.c:569 + #8 0x0000000000584abd in PyDict_GetItemString (v= + {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__cached__': None, '__name__': '__main__', 'z': , '__doc__': None}, key= + 0x5c2b8d "__lltrace__") at Objects/dictobject.c:2171 + +Notice how the dictionary argument to ``PyDict_GetItemString`` is displayed +as its ``repr()``, rather than an opaque ``PyObject *`` pointer. + +The extension works by supplying a custom printing routine for values of type +``PyObject *``. If you need to access lower-level details of an object, then +cast the value to a pointer of the appropriate type. For example:: + + (gdb) p globals + $1 = {'__builtins__': , '__name__': + '__main__', 'ctypes': , '__doc__': None, + '__package__': None} + + (gdb) p *(PyDictObject*)globals + $2 = {ob_refcnt = 3, ob_type = 0x3dbdf85820, ma_fill = 5, ma_used = 5, + ma_mask = 7, ma_table = 0x63d0f8, ma_lookup = 0x3dbdc7ea70 + , ma_smalltable = {{me_hash = 7065186196740147912, + me_key = '__builtins__', me_value = }, + {me_hash = -368181376027291943, me_key = '__name__', + me_value ='__main__'}, {me_hash = 0, me_key = 0x0, me_value = 0x0}, + {me_hash = 0, me_key = 0x0, me_value = 0x0}, + {me_hash = -9177857982131165996, me_key = 'ctypes', + me_value = }, + {me_hash = -8518757509529533123, me_key = '__doc__', me_value = None}, + {me_hash = 0, me_key = 0x0, me_value = 0x0}, { + me_hash = 6614918939584953775, me_key = '__package__', me_value = None}}} + +Note that the pretty-printers do not actually call ``repr()``. +For basic types, they try to match its result closely. + +An area that can be confusing is that the custom printer for some types look a +lot like GDB's built-in printer for standard types. For example, the +pretty-printer for a Python ``int`` (:c:expr:`PyLongObject *`) +gives a representation that is not distinguishable from one of a +regular machine-level integer:: + + (gdb) p some_machine_integer + $3 = 42 + + (gdb) p some_python_integer + $4 = 42 + +The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`: + + (gdb) p *(PyLongObject*)some_python_integer + $5 = {ob_base = {ob_base = {ob_refcnt = 8, ob_type = 0x3dad39f5e0}, ob_size = 1}, + ob_digit = {42}} + +A similar confusion can arise with the ``str`` type, where the output looks a +lot like gdb's built-in printer for ``char *``:: + + (gdb) p ptr_to_python_str + $6 = '__builtins__' + +The pretty-printer for ``str`` instances defaults to using single-quotes (as +does Python's ``repr`` for strings) whereas the standard printer for ``char *`` +values uses double-quotes and contains a hexadecimal address:: + + (gdb) p ptr_to_char_star + $7 = 0x6d72c0 "hello world" + +Again, the implementation details can be revealed with a cast to +:c:expr:`PyUnicodeObject *`:: + + (gdb) p *(PyUnicodeObject*)$6 + $8 = {ob_base = {ob_refcnt = 33, ob_type = 0x3dad3a95a0}, length = 12, + str = 0x7ffff2128500, hash = 7065186196740147912, state = 1, defenc = 0x0} + +``py-list`` +----------- + + The extension adds a ``py-list`` command, which + lists the Python source code (if any) for the current frame in the selected + thread. The current line is marked with a ">":: + + (gdb) py-list + 901 if options.profile: + 902 options.profile = False + 903 profile_me() + 904 return + 905 + >906 u = UI() + 907 if not u.quit: + 908 try: + 909 gtk.main() + 910 except KeyboardInterrupt: + 911 # properly quit on a keyboard interrupt... + + Use ``py-list START`` to list at a different line number within the Python + source, and ``py-list START,END`` to list a specific range of lines within + the Python source. + +``py-up`` and ``py-down`` +------------------------- + + The ``py-up`` and ``py-down`` commands are analogous to GDB's regular ``up`` + and ``down`` commands, but try to move at the level of CPython frames, rather + than C frames. + + GDB is not always able to read the relevant frame information, depending on + the optimization level with which CPython was compiled. Internally, the + commands look for C frames that are executing the default frame evaluation + function (that is, the core bytecode interpreter loop within CPython) and + look up the value of the related ``PyFrameObject *``. + + They emit the frame number (at the C level) within the thread. + + For example:: + + (gdb) py-up + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/ + gnome_sudoku/main.py, line 906, in start_game () + u = UI() + (gdb) py-up + #40 Frame 0x948e82c, for file /usr/lib/python2.6/site-packages/ + gnome_sudoku/gnome_sudoku.py, line 22, in start_game(main=) + main.start_game() + (gdb) py-up + Unable to find an older python frame + + so we're at the top of the Python stack. + + The frame numbers correspond to those displayed by GDB's standard + ``backtrace`` command. + The command skips C frames which are not executing Python code. + + Going back down:: + + (gdb) py-down + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/gnome_sudoku/main.py, line 906, in start_game () + u = UI() + (gdb) py-down + #34 (unable to read python frame information) + (gdb) py-down + #23 (unable to read python frame information) + (gdb) py-down + #19 (unable to read python frame information) + (gdb) py-down + #14 Frame 0x99262ac, for file /usr/lib/python2.6/site-packages/gnome_sudoku/game_selector.py, line 201, in run_swallowed_dialog (self=, puzzle=None, saved_games=[{'gsd.auto_fills': 0, 'tracking': {}, 'trackers': {}, 'notes': [], 'saved_at': 1270084485, 'game': '7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 0 0 0 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5\n7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 1 8 3 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5', 'gsd.impossible_hints': 0, 'timer.__absolute_start_time__': , 'gsd.hints': 0, 'timer.active_time': , 'timer.total_time': }], dialog=, saved_game_model=, sudoku_maker=, main_page=0) at remote 0x98fa6e4>, d=) + gtk.main() + (gdb) py-down + #8 (unable to read python frame information) + (gdb) py-down + Unable to find a newer python frame + + and we're at the bottom of the Python stack. + + Note that in Python 3.12 and newer, the same C stack frame can be used for + multiple Python stack frames. This means that ``py-up`` and ``py-down`` + may move multiple Python frames at once. For example:: + + (gdb) py-up + #6 Frame 0x7ffff7fb62b0, for file /tmp/rec.py, line 5, in recursive_function (n=0) + time.sleep(5) + #6 Frame 0x7ffff7fb6240, for file /tmp/rec.py, line 7, in recursive_function (n=1) + recursive_function(n-1) + #6 Frame 0x7ffff7fb61d0, for file /tmp/rec.py, line 7, in recursive_function (n=2) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6160, for file /tmp/rec.py, line 7, in recursive_function (n=3) + recursive_function(n-1) + #6 Frame 0x7ffff7fb60f0, for file /tmp/rec.py, line 7, in recursive_function (n=4) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6080, for file /tmp/rec.py, line 7, in recursive_function (n=5) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6020, for file /tmp/rec.py, line 9, in () + recursive_function(5) + (gdb) py-up + Unable to find an older python frame + + +``py-bt`` +--------- + + The ``py-bt`` command attempts to display a Python-level backtrace of the + current thread. + + For example:: + + (gdb) py-bt + #8 (unable to read python frame information) + #11 Frame 0x9aead74, for file /usr/lib/python2.6/site-packages/gnome_sudoku/dialog_swallower.py, line 48, in run_dialog (self=, main_page=0) at remote 0x98fa6e4>, d=) + gtk.main() + #14 Frame 0x99262ac, for file /usr/lib/python2.6/site-packages/gnome_sudoku/game_selector.py, line 201, in run_swallowed_dialog (self=, puzzle=None, saved_games=[{'gsd.auto_fills': 0, 'tracking': {}, 'trackers': {}, 'notes': [], 'saved_at': 1270084485, 'game': '7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 0 0 0 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5\n7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 1 8 3 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5', 'gsd.impossible_hints': 0, 'timer.__absolute_start_time__': , 'gsd.hints': 0, 'timer.active_time': , 'timer.total_time': }], dialog=, saved_game_model=, sudoku_maker=) + main.start_game() + + The frame numbers correspond to those displayed by GDB's standard + ``backtrace`` command. + +``py-print`` +------------ + + The ``py-print`` command looks up a Python name and tries to print it. + It looks in locals within the current thread, then globals, then finally + builtins:: + + (gdb) py-print self + local 'self' = , + main_page=0) at remote 0x98fa6e4> + (gdb) py-print __name__ + global '__name__' = 'gnome_sudoku.dialog_swallower' + (gdb) py-print len + builtin 'len' = + (gdb) py-print scarlet_pimpernel + 'scarlet_pimpernel' not found + + If the current C frame corresponds to multiple Python frames, ``py-print`` + only considers the first one. + +``py-locals`` +------------- + + The ``py-locals`` command looks up all Python locals within the current + Python frame in the selected thread, and prints their representations:: + + (gdb) py-locals + self = , + main_page=0) at remote 0x98fa6e4> + d = + + If the current C frame corresponds to multiple Python frames, locals from + all of them will be shown:: + + (gdb) py-locals + Locals for recursive_function + n = 0 + Locals for recursive_function + n = 1 + Locals for recursive_function + n = 2 + Locals for recursive_function + n = 3 + Locals for recursive_function + n = 4 + Locals for recursive_function + n = 5 + Locals for + + +Use with GDB commands +===================== + +The extension commands complement GDB's built-in commands. +For example, you can use a frame numbers shown by ``py-bt`` with the ``frame`` +command to go a specific frame within the selected thread, like this:: + + (gdb) py-bt + (output snipped) + #68 Frame 0xaa4560, for file Lib/test/regrtest.py, line 1548, in () + main() + (gdb) frame 68 + #68 0x00000000004cd1e6 in PyEval_EvalFrameEx (f=Frame 0xaa4560, for file Lib/test/regrtest.py, line 1548, in (), throwflag=0) at Python/ceval.c:2665 + 2665 x = call_function(&sp, oparg); + (gdb) py-list + 1543 # Run the tests in a context manager that temporary changes the CWD to a + 1544 # temporary and writable directory. If it's not possible to create or + 1545 # change the CWD, the original CWD will be used. The original CWD is + 1546 # available from test_support.SAVEDCWD. + 1547 with test_support.temp_cwd(TESTCWD, quiet=True): + >1548 main() + +The ``info threads`` command will give you a list of the threads within the +process, and you can use the ``thread`` command to select a different one:: + + (gdb) info threads + 105 Thread 0x7fffefa18710 (LWP 10260) sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:86 + 104 Thread 0x7fffdf5fe710 (LWP 10259) sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:86 + * 1 Thread 0x7ffff7fe2700 (LWP 10145) 0x00000038e46d73e3 in select () at ../sysdeps/unix/syscall-template.S:82 + +You can use ``thread apply all COMMAND`` or (``t a a COMMAND`` for short) to run +a command on all threads. With ``py-bt``, this lets you see what every +thread is doing at the Python level:: + + (gdb) t a a py-bt + + Thread 105 (Thread 0x7fffefa18710 (LWP 10260)): + #5 Frame 0x7fffd00019d0, for file /home/david/coding/python-svn/Lib/threading.py, line 155, in _acquire_restore (self=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=, _RLock__count=1) at remote 0xd7ff40>, count_owner=(1, 140737213728528), count=1, owner=140737213728528) + self.__block.acquire() + #8 Frame 0x7fffac001640, for file /home/david/coding/python-svn/Lib/threading.py, line 269, in wait (self=<_Condition(_Condition__lock=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=, _RLock__count=1) at remote 0xd7ff40>, acquire=, _is_owned=, _release_save=, release=, _acquire_restore=, _Verbose__verbose=False, _Condition__waiters=[]) at remote 0xd7fd10>, timeout=None, waiter=, saved_state=(1, 140737213728528)) + self._acquire_restore(saved_state) + #12 Frame 0x7fffb8001a10, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 348, in f () + cond.wait() + #16 Frame 0x7fffb8001c40, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 37, in task (tid=140737213728528) + f() + + Thread 104 (Thread 0x7fffdf5fe710 (LWP 10259)): + #5 Frame 0x7fffe4001580, for file /home/david/coding/python-svn/Lib/threading.py, line 155, in _acquire_restore (self=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=, _RLock__count=1) at remote 0xd7ff40>, count_owner=(1, 140736940992272), count=1, owner=140736940992272) + self.__block.acquire() + #8 Frame 0x7fffc8002090, for file /home/david/coding/python-svn/Lib/threading.py, line 269, in wait (self=<_Condition(_Condition__lock=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=, _RLock__count=1) at remote 0xd7ff40>, acquire=, _is_owned=, _release_save=, release=, _acquire_restore=, _Verbose__verbose=False, _Condition__waiters=[]) at remote 0xd7fd10>, timeout=None, waiter=, saved_state=(1, 140736940992272)) + self._acquire_restore(saved_state) + #12 Frame 0x7fffac001c90, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 348, in f () + cond.wait() + #16 Frame 0x7fffac0011c0, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 37, in task (tid=140736940992272) + f() + + Thread 1 (Thread 0x7ffff7fe2700 (LWP 10145)): + #5 Frame 0xcb5380, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 16, in _wait () + time.sleep(0.01) + #8 Frame 0x7fffd00024a0, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 378, in _check_notify (self=, skipped=[], _mirrorOutput=False, testsRun=39, buffer=False, _original_stderr=, _stdout_buffer=, _stderr_buffer=, _moduleSetUpFailed=False, expectedFailures=[], errors=[], _previousTestClass=, unexpectedSuccesses=[], failures=[], shouldStop=False, failfast=False) at remote 0xc185a0>, _threads=(0,), _cleanups=[], _type_equality_funcs={: , : , : , : , `_, billed as a spiritual successor to ZeroMQ. +The following snippets illustrate -- you can test them in an environment which has +``pynng`` installed. Just for variety, we present the listener first. - Module :mod:`logging.config` - Configuration API for the logging module. - Module :mod:`logging.handlers` - Useful handlers included with the logging module. +Subclass ``QueueListener`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ - :ref:`A basic logging tutorial ` +.. code-block:: python - :ref:`A more advanced logging tutorial ` + # listener.py + import json + import logging + import logging.handlers + import pynng + + DEFAULT_ADDR = "tcp://localhost:13232" + + interrupted = False + + class NNGSocketListener(logging.handlers.QueueListener): + + def __init__(self, uri, /, *handlers, **kwargs): + # Have a timeout for interruptability, and open a + # subscriber socket + socket = pynng.Sub0(listen=uri, recv_timeout=500) + # The b'' subscription matches all topics + topics = kwargs.pop('topics', None) or b'' + socket.subscribe(topics) + # We treat the socket as a queue + super().__init__(socket, *handlers, **kwargs) + + def dequeue(self, block): + data = None + # Keep looping while not interrupted and no data received over the + # socket + while not interrupted: + try: + data = self.queue.recv(block=block) + break + except pynng.Timeout: + pass + except pynng.Closed: # sometimes happens when you hit Ctrl-C + break + if data is None: + return None + # Get the logging event sent from a publisher + event = json.loads(data.decode('utf-8')) + return logging.makeLogRecord(event) + + def enqueue_sentinel(self): + # Not used in this implementation, as the socket isn't really a + # queue + pass + + logging.getLogger('pynng').propagate = False + listener = NNGSocketListener(DEFAULT_ADDR, logging.StreamHandler(), topics=b'') + listener.start() + print('Press Ctrl-C to stop.') + try: + while True: + pass + except KeyboardInterrupt: + interrupted = True + finally: + listener.stop() + + +Subclass ``QueueHandler`` +^^^^^^^^^^^^^^^^^^^^^^^^^ .. currentmodule:: logging +.. code-block:: python + + # sender.py + import json + import logging + import logging.handlers + import time + import random + + import pynng + + DEFAULT_ADDR = "tcp://localhost:13232" + + class NNGSocketHandler(logging.handlers.QueueHandler): + + def __init__(self, uri): + socket = pynng.Pub0(dial=uri, send_timeout=500) + super().__init__(socket) + + def enqueue(self, record): + # Send the record as UTF-8 encoded JSON + d = dict(record.__dict__) + data = json.dumps(d) + self.queue.send(data.encode('utf-8')) + + def close(self): + self.queue.close() + + logging.getLogger('pynng').propagate = False + handler = NNGSocketHandler(DEFAULT_ADDR) + # Make sure the process ID is in the output + logging.basicConfig(level=logging.DEBUG, + handlers=[logging.StreamHandler(), handler], + format='%(levelname)-8s %(name)10s %(process)6s %(message)s') + levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, + logging.CRITICAL) + logger_names = ('myapp', 'myapp.lib1', 'myapp.lib2') + msgno = 1 + while True: + # Just randomly select some loggers and levels and log away + level = random.choice(levels) + logger = logging.getLogger(random.choice(logger_names)) + logger.log(level, 'Message no. %5d' % msgno) + msgno += 1 + delay = random.random() * 2 + 0.5 + time.sleep(delay) + +You can run the above two snippets in separate command shells. If we run the +listener in one shell and run the sender in two separate shells, we should see +something like the following. In the first sender shell: + +.. code-block:: console + + $ python sender.py + DEBUG myapp 613 Message no. 1 + WARNING myapp.lib2 613 Message no. 2 + CRITICAL myapp.lib2 613 Message no. 3 + WARNING myapp.lib2 613 Message no. 4 + CRITICAL myapp.lib1 613 Message no. 5 + DEBUG myapp 613 Message no. 6 + CRITICAL myapp.lib1 613 Message no. 7 + INFO myapp.lib1 613 Message no. 8 + (and so on) + +In the second sender shell: + +.. code-block:: console + + $ python sender.py + INFO myapp.lib2 657 Message no. 1 + CRITICAL myapp.lib2 657 Message no. 2 + CRITICAL myapp 657 Message no. 3 + CRITICAL myapp.lib1 657 Message no. 4 + INFO myapp.lib1 657 Message no. 5 + WARNING myapp.lib2 657 Message no. 6 + CRITICAL myapp 657 Message no. 7 + DEBUG myapp.lib1 657 Message no. 8 + (and so on) + +In the listener shell: + +.. code-block:: console + + $ python listener.py + Press Ctrl-C to stop. + DEBUG myapp 613 Message no. 1 + WARNING myapp.lib2 613 Message no. 2 + INFO myapp.lib2 657 Message no. 1 + CRITICAL myapp.lib2 613 Message no. 3 + CRITICAL myapp.lib2 657 Message no. 2 + CRITICAL myapp 657 Message no. 3 + WARNING myapp.lib2 613 Message no. 4 + CRITICAL myapp.lib1 613 Message no. 5 + CRITICAL myapp.lib1 657 Message no. 4 + INFO myapp.lib1 657 Message no. 5 + DEBUG myapp 613 Message no. 6 + WARNING myapp.lib2 657 Message no. 6 + CRITICAL myapp 657 Message no. 7 + CRITICAL myapp.lib1 613 Message no. 7 + INFO myapp.lib1 613 Message no. 8 + DEBUG myapp.lib1 657 Message no. 8 + (and so on) + +As you can see, the logging from the two sender processes is interleaved in the +listener's output. + + An example dictionary-based configuration ----------------------------------------- @@ -2780,7 +2950,7 @@ When run, this produces a file with exactly two lines: .. code-block:: none 28/01/2015 07:21:23|INFO|Sample message| - 28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'| + 28/01/2015 07:21:23|ERROR|ZeroDivisionError: division by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: division by zero'| While the above treatment is simplistic, it points the way to how exception information can be formatted to your liking. The :mod:`traceback` module may be @@ -3405,9 +3575,8 @@ A Qt GUI for logging A question that comes up from time to time is about how to log to a GUI application. The `Qt `_ framework is a popular -cross-platform UI framework with Python bindings using `PySide2 -`_ or `PyQt5 -`_ libraries. +cross-platform UI framework with Python bindings using :pypi:`PySide2` +or :pypi:`PyQt5` libraries. The following example shows how to log to a Qt GUI. This introduces a simple ``QtHandler`` class which takes a callable, which should be a slot in the main @@ -3420,9 +3589,10 @@ The worker thread is implemented using Qt's ``QThread`` class rather than the :mod:`threading` module, as there are circumstances where one has to use ``QThread``, which offers better integration with other ``Qt`` components. -The code should work with recent releases of either ``PySide2`` or ``PyQt5``. -You should be able to adapt the approach to earlier versions of Qt. Please -refer to the comments in the code snippet for more detailed information. +The code should work with recent releases of any of ``PySide6``, ``PyQt6``, +``PySide2`` or ``PyQt5``. You should be able to adapt the approach to earlier +versions of Qt. Please refer to the comments in the code snippet for more +detailed information. .. code-block:: python3 @@ -3432,16 +3602,25 @@ refer to the comments in the code snippet for more detailed information. import sys import time - # Deal with minor differences between PySide2 and PyQt5 + # Deal with minor differences between different Qt packages try: - from PySide2 import QtCore, QtGui, QtWidgets + from PySide6 import QtCore, QtGui, QtWidgets Signal = QtCore.Signal Slot = QtCore.Slot except ImportError: - from PyQt5 import QtCore, QtGui, QtWidgets - Signal = QtCore.pyqtSignal - Slot = QtCore.pyqtSlot - + try: + from PyQt6 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot + except ImportError: + try: + from PySide2 import QtCore, QtGui, QtWidgets + Signal = QtCore.Signal + Slot = QtCore.Slot + except ImportError: + from PyQt5 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot logger = logging.getLogger(__name__) @@ -3513,8 +3692,14 @@ refer to the comments in the code snippet for more detailed information. while not QtCore.QThread.currentThread().isInterruptionRequested(): delay = 0.5 + random.random() * 2 time.sleep(delay) - level = random.choice(LEVELS) - logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + try: + if random.random() < 0.1: + raise ValueError('Exception raised: %d' % i) + else: + level = random.choice(LEVELS) + logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + except ValueError as e: + logger.exception('Failed: %s', e, extra=extra) i += 1 # @@ -3541,7 +3726,10 @@ refer to the comments in the code snippet for more detailed information. self.textedit = te = QtWidgets.QPlainTextEdit(self) # Set whatever the default monospace font is for the platform f = QtGui.QFont('nosuchfont') - f.setStyleHint(f.Monospace) + if hasattr(f, 'Monospace'): + f.setStyleHint(f.Monospace) + else: + f.setStyleHint(f.StyleHint.Monospace) # for Qt6 te.setFont(f) te.setReadOnly(True) PB = QtWidgets.QPushButton @@ -3628,7 +3816,11 @@ refer to the comments in the code snippet for more detailed information. app = QtWidgets.QApplication(sys.argv) example = Window(app) example.show() - sys.exit(app.exec_()) + if hasattr(app, 'exec'): + rc = app.exec() + else: + rc = app.exec_() + sys.exit(rc) if __name__=='__main__': main() @@ -3830,7 +4022,7 @@ As you can see, this output isn't ideal. That's because the underlying code which writes to ``sys.stderr`` makes multiple writes, each of which results in a separate logged line (for example, the last three lines above). To get around this problem, you need to buffer things and only output log lines when newlines -are seen. Let's use a slghtly better implementation of ``LoggerWriter``: +are seen. Let's use a slightly better implementation of ``LoggerWriter``: .. code-block:: python diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 347330e98dd00c..3182d5664ab6ec 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -1,3 +1,5 @@ +.. _logging-howto: + ============= Logging HOWTO ============= @@ -25,10 +27,12 @@ or *severity*. When to use logging ^^^^^^^^^^^^^^^^^^^ -Logging provides a set of convenience functions for simple logging usage. These -are :func:`debug`, :func:`info`, :func:`warning`, :func:`error` and -:func:`critical`. To determine when to use logging, see the table below, which -states, for each of a set of common tasks, the best tool to use for it. +You can access logging functionality by creating a logger via ``logger = +getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +:meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and +:meth:`~Logger.critical` methods. To determine when to use logging, and to see +which logger methods to use when, see the table below. It states, for each of a +set of common tasks, the best tool to use for that task. +-------------------------------------+--------------------------------------+ | Task you want to perform | The best tool for the task | @@ -37,8 +41,8 @@ states, for each of a set of common tasks, the best tool to use for it. | usage of a command line script or | | | program | | +-------------------------------------+--------------------------------------+ -| Report events that occur during | :func:`logging.info` (or | -| normal operation of a program (e.g. | :func:`logging.debug` for very | +| Report events that occur during | A logger's :meth:`~Logger.info` (or | +| normal operation of a program (e.g. | :meth:`~Logger.debug` method for very| | for status monitoring or fault | detailed output for diagnostic | | investigation) | purposes) | +-------------------------------------+--------------------------------------+ @@ -47,22 +51,23 @@ states, for each of a set of common tasks, the best tool to use for it. | | the client application should be | | | modified to eliminate the warning | | | | -| | :func:`logging.warning` if there is | -| | nothing the client application can do| -| | about the situation, but the event | -| | should still be noted | +| | A logger's :meth:`~Logger.warning` | +| | method if there is nothing the client| +| | application can do about the | +| | situation, but the event should still| +| | be noted | +-------------------------------------+--------------------------------------+ | Report an error regarding a | Raise an exception | | particular runtime event | | +-------------------------------------+--------------------------------------+ -| Report suppression of an error | :func:`logging.error`, | -| without raising an exception (e.g. | :func:`logging.exception` or | -| error handler in a long-running | :func:`logging.critical` as | +| Report suppression of an error | A logger's :meth:`~Logger.error`, | +| without raising an exception (e.g. | :meth:`~Logger.exception` or | +| error handler in a long-running | :meth:`~Logger.critical` method as | | server process) | appropriate for the specific error | | | and application domain | +-------------------------------------+--------------------------------------+ -The logging functions are named after the level or severity of the events +The logger methods are named after the level or severity of the events they are used to track. The standard levels and their applicability are described below (in increasing order of severity): @@ -115,12 +120,18 @@ If you type these lines into a script and run it, you'll see: WARNING:root:Watch out! printed out on the console. The ``INFO`` message doesn't appear because the -default level is ``WARNING``. The printed message includes the indication of -the level and the description of the event provided in the logging call, i.e. -'Watch out!'. Don't worry about the 'root' part for now: it will be explained -later. The actual output can be formatted quite flexibly if you need that; -formatting options will also be explained later. - +default level is ``WARNING``. The printed message includes the indication of the +level and the description of the event provided in the logging call, i.e. +'Watch out!'. The actual output can be formatted quite flexibly if you need +that; formatting options will also be explained later. + +Notice that in this example, we use functions directly on the ``logging`` +module, like ``logging.debug``, rather than creating a logger and calling +functions on it. These functions operation on the root logger, but can be useful +as they will call :func:`~logging.basicConfig` for you if it has not been called yet, like in +this example. In larger programs you'll usually want to control the logging +configuration explicitly however - so for that reason as well as others, it's +better to create loggers and call their methods. Logging to a file ^^^^^^^^^^^^^^^^^ @@ -130,11 +141,12 @@ look at that next. Be sure to try the following in a newly started Python interpreter, and don't just continue from the session described above:: import logging + logger = logging.getLogger(__name__) logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.DEBUG) - logging.debug('This message should go to the log file') - logging.info('So should this') - logging.warning('And this, too') - logging.error('And non-ASCII stuff, too, like Øresund and Malmö') + logger.debug('This message should go to the log file') + logger.info('So should this') + logger.warning('And this, too') + logger.error('And non-ASCII stuff, too, like Øresund and Malmö') .. versionchanged:: 3.9 The *encoding* argument was added. In earlier Python versions, or if not @@ -148,10 +160,10 @@ messages: .. code-block:: none - DEBUG:root:This message should go to the log file - INFO:root:So should this - WARNING:root:And this, too - ERROR:root:And non-ASCII stuff, too, like Øresund and Malmö + DEBUG:__main__:This message should go to the log file + INFO:__main__:So should this + WARNING:__main__:And this, too + ERROR:__main__:And non-ASCII stuff, too, like Øresund and Malmö This example also shows how you can set the logging level which acts as the threshold for tracking. In this case, because we set the threshold to @@ -180,11 +192,9 @@ following example:: raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, ...) -The call to :func:`basicConfig` should come *before* any calls to -:func:`debug`, :func:`info`, etc. Otherwise, those functions will call -:func:`basicConfig` for you with the default options. As it's intended as a -one-off simple configuration facility, only the first call will actually do -anything: subsequent calls are effectively no-ops. +The call to :func:`basicConfig` should come *before* any calls to a logger's +methods such as :meth:`~Logger.debug`, :meth:`~Logger.info`, etc. Otherwise, +that logging event may not be handled in the desired manner. If you run the above script several times, the messages from successive runs are appended to the file *example.log*. If you want each run to start afresh, @@ -197,50 +207,6 @@ The output will be the same as before, but the log file is no longer appended to, so the messages from earlier runs are lost. -Logging from multiple modules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your program consists of multiple modules, here's an example of how you -could organize logging in it:: - - # myapp.py - import logging - import mylib - - def main(): - logging.basicConfig(filename='myapp.log', level=logging.INFO) - logging.info('Started') - mylib.do_something() - logging.info('Finished') - - if __name__ == '__main__': - main() - -:: - - # mylib.py - import logging - - def do_something(): - logging.info('Doing something') - -If you run *myapp.py*, you should see this in *myapp.log*: - -.. code-block:: none - - INFO:root:Started - INFO:root:Doing something - INFO:root:Finished - -which is hopefully what you were expecting to see. You can generalize this to -multiple modules, using the pattern in *mylib.py*. Note that for this simple -usage pattern, you won't know, by looking in the log file, *where* in your -application your messages came from, apart from looking at the event -description. If you want to track the location of your messages, you'll need -to refer to the documentation beyond the tutorial level -- see -:ref:`logging-advanced-tutorial`. - - Logging variable data ^^^^^^^^^^^^^^^^^^^^^ @@ -416,8 +382,52 @@ Logging Flow The flow of log event information in loggers and handlers is illustrated in the following diagram. -.. image:: logging_flow.png - :class: invert-in-dark-mode +.. only:: not html + + .. image:: logging_flow.* + +.. raw:: html + :file: logging_flow.svg + +.. raw:: html + + Loggers ^^^^^^^ diff --git a/Doc/howto/logging_flow.png b/Doc/howto/logging_flow.png index d65e597f811db5..d60ed7c031585a 100644 Binary files a/Doc/howto/logging_flow.png and b/Doc/howto/logging_flow.png differ diff --git a/Doc/howto/logging_flow.svg b/Doc/howto/logging_flow.svg new file mode 100644 index 00000000000000..4974994ac6b400 --- /dev/null +++ b/Doc/howto/logging_flow.svg @@ -0,0 +1,327 @@ + + + + + + + + + + + Logger flow + + + + + Create + LogRecord + + + + + + + + + + + + Logging call in user + code, e.g. + + + logger.info(...) + + + + + + + + + Stop + + + + + + Does a filter attached + to logger reject the + record? + + + + + + + + + + Pass record to + handlers of + current logger + + + + + + Is propagate true for + current logger? + + + + + + Is there a parent + logger? + + + + + + Set current + logger to parent + + + + + + At least one handler + in hierarchy? + + + + + + Use + lastResort + handler + + + + + + Handler enabled for + level of record? + + + + + + Does a filter attached + to handler reject the + record? + + + + + + Stop + + + + + + Emit (includes formatting) + + + + Handler flow + + + + + Logger enabled for + level of call? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No + + + Yes + + + Yes + + + No + + + No + + + Yes + + + Yes + + + No + + + No + + + Yes + + + + + + No + + + + + + + + + Yes + + + No + + + + + + Yes + + + Record passed + to handler + + + + + + + + diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst new file mode 100644 index 00000000000000..46db516e16dae4 --- /dev/null +++ b/Doc/howto/mro.rst @@ -0,0 +1,671 @@ +.. _python_2.3_mro: + +The Python 2.3 Method Resolution Order +====================================== + +.. note:: + + This is a historical document, provided as an appendix to the official + documentation. + The Method Resolution Order discussed here was *introduced* in Python 2.3, + but it is still used in later versions -- including Python 3. + +By `Michele Simionato `__. + +:Abstract: + + *This document is intended for Python programmers who want to + understand the C3 Method Resolution Order used in Python 2.3. + Although it is not intended for newbies, it is quite pedagogical with + many worked out examples. I am not aware of other publicly available + documents with the same scope, therefore it should be useful.* + +Disclaimer: + + *I donate this document to the Python Software Foundation, under the + Python 2.3 license. As usual in these circumstances, I warn the + reader that what follows* should *be correct, but I don't give any + warranty. Use it at your own risk and peril!* + +Acknowledgments: + + *All the people of the Python mailing list who sent me their support. + Paul Foley who pointed out various imprecisions and made me to add the + part on local precedence ordering. David Goodger for help with the + formatting in reStructuredText. David Mertz for help with the editing. + Finally, Guido van Rossum who enthusiastically added this document to + the official Python 2.3 home-page.* + +The beginning +------------- + + *Felix qui potuit rerum cognoscere causas* -- Virgilius + +Everything started with a post by Samuele Pedroni to the Python +development mailing list [#]_. In his post, Samuele showed that the +Python 2.2 method resolution order is not monotonic and he proposed to +replace it with the C3 method resolution order. Guido agreed with his +arguments and therefore now Python 2.3 uses C3. The C3 method itself +has nothing to do with Python, since it was invented by people working +on Dylan and it is described in a paper intended for lispers [#]_. The +present paper gives a (hopefully) readable discussion of the C3 +algorithm for Pythonistas who want to understand the reasons for the +change. + +First of all, let me point out that what I am going to say only applies +to the *new style classes* introduced in Python 2.2: *classic classes* +maintain their old method resolution order, depth first and then left to +right. Therefore, there is no breaking of old code for classic classes; +and even if in principle there could be breaking of code for Python 2.2 +new style classes, in practice the cases in which the C3 resolution +order differs from the Python 2.2 method resolution order are so rare +that no real breaking of code is expected. Therefore: + + *Don't be scared!* + +Moreover, unless you make strong use of multiple inheritance and you +have non-trivial hierarchies, you don't need to understand the C3 +algorithm, and you can easily skip this paper. On the other hand, if +you really want to know how multiple inheritance works, then this paper +is for you. The good news is that things are not as complicated as you +might expect. + +Let me begin with some basic definitions. + +1) Given a class C in a complicated multiple inheritance hierarchy, it + is a non-trivial task to specify the order in which methods are + overridden, i.e. to specify the order of the ancestors of C. + +2) The list of the ancestors of a class C, including the class itself, + ordered from the nearest ancestor to the furthest, is called the + class precedence list or the *linearization* of C. + +3) The *Method Resolution Order* (MRO) is the set of rules that + construct the linearization. In the Python literature, the idiom + "the MRO of C" is also used as a synonymous for the linearization of + the class C. + +4) For instance, in the case of single inheritance hierarchy, if C is a + subclass of C1, and C1 is a subclass of C2, then the linearization of + C is simply the list [C, C1 , C2]. However, with multiple + inheritance hierarchies, the construction of the linearization is + more cumbersome, since it is more difficult to construct a + linearization that respects *local precedence ordering* and + *monotonicity*. + +5) I will discuss the local precedence ordering later, but I can give + the definition of monotonicity here. A MRO is monotonic when the + following is true: *if C1 precedes C2 in the linearization of C, + then C1 precedes C2 in the linearization of any subclass of C*. + Otherwise, the innocuous operation of deriving a new class could + change the resolution order of methods, potentially introducing very + subtle bugs. Examples where this happens will be shown later. + +6) Not all classes admit a linearization. There are cases, in + complicated hierarchies, where it is not possible to derive a class + such that its linearization respects all the desired properties. + +Here I give an example of this situation. Consider the hierarchy + + >>> O = object + >>> class X(O): pass + >>> class Y(O): pass + >>> class A(X,Y): pass + >>> class B(Y,X): pass + +which can be represented with the following inheritance graph, where I +have denoted with O the ``object`` class, which is the beginning of any +hierarchy for new style classes: + + .. code-block:: text + + ----------- + | | + | O | + | / \ | + - X Y / + | / | / + | / |/ + A B + \ / + ? + +In this case, it is not possible to derive a new class C from A and B, +since X precedes Y in A, but Y precedes X in B, therefore the method +resolution order would be ambiguous in C. + +Python 2.3 raises an exception in this situation (TypeError: MRO +conflict among bases Y, X) forbidding the naive programmer from creating +ambiguous hierarchies. Python 2.2 instead does not raise an exception, +but chooses an *ad hoc* ordering (CABXYO in this case). + +The C3 Method Resolution Order +------------------------------ + +Let me introduce a few simple notations which will be useful for the +following discussion. I will use the shortcut notation:: + + C1 C2 ... CN + +to indicate the list of classes [C1, C2, ... , CN]. + +The *head* of the list is its first element:: + + head = C1 + +whereas the *tail* is the rest of the list:: + + tail = C2 ... CN. + +I shall also use the notation:: + + C + (C1 C2 ... CN) = C C1 C2 ... CN + +to denote the sum of the lists [C] + [C1, C2, ... ,CN]. + +Now I can explain how the MRO works in Python 2.3. + +Consider a class C in a multiple inheritance hierarchy, with C +inheriting from the base classes B1, B2, ... , BN. We want to +compute the linearization L[C] of the class C. The rule is the +following: + + *the linearization of C is the sum of C plus the merge of the + linearizations of the parents and the list of the parents.* + +In symbolic notation:: + + L[C(B1 ... BN)] = C + merge(L[B1] ... L[BN], B1 ... BN) + +In particular, if C is the ``object`` class, which has no parents, the +linearization is trivial:: + + L[object] = object. + +However, in general one has to compute the merge according to the following +prescription: + + *take the head of the first list, i.e L[B1][0]; if this head is not in + the tail of any of the other lists, then add it to the linearization + of C and remove it from the lists in the merge, otherwise look at the + head of the next list and take it, if it is a good head. Then repeat + the operation until all the class are removed or it is impossible to + find good heads. In this case, it is impossible to construct the + merge, Python 2.3 will refuse to create the class C and will raise an + exception.* + +This prescription ensures that the merge operation *preserves* the +ordering, if the ordering can be preserved. On the other hand, if the +order cannot be preserved (as in the example of serious order +disagreement discussed above) then the merge cannot be computed. + +The computation of the merge is trivial if C has only one parent +(single inheritance); in this case:: + + L[C(B)] = C + merge(L[B],B) = C + L[B] + +However, in the case of multiple inheritance things are more cumbersome +and I don't expect you can understand the rule without a couple of +examples ;-) + +Examples +-------- + +First example. Consider the following hierarchy: + + >>> O = object + >>> class F(O): pass + >>> class E(O): pass + >>> class D(O): pass + >>> class C(D,F): pass + >>> class B(D,E): pass + >>> class A(B,C): pass + +In this case the inheritance graph can be drawn as: + + .. code-block:: text + + 6 + --- + Level 3 | O | (more general) + / --- \ + / | \ | + / | \ | + / | \ | + --- --- --- | + Level 2 3 | D | 4| E | | F | 5 | + --- --- --- | + \ \ _ / | | + \ / \ _ | | + \ / \ | | + --- --- | + Level 1 1 | B | | C | 2 | + --- --- | + \ / | + \ / \ / + --- + Level 0 0 | A | (more specialized) + --- + + +The linearizations of O,D,E and F are trivial:: + + L[O] = O + L[D] = D O + L[E] = E O + L[F] = F O + +The linearization of B can be computed as:: + + L[B] = B + merge(DO, EO, DE) + +We see that D is a good head, therefore we take it and we are reduced to +compute ``merge(O,EO,E)``. Now O is not a good head, since it is in the +tail of the sequence EO. In this case the rule says that we have to +skip to the next sequence. Then we see that E is a good head; we take +it and we are reduced to compute ``merge(O,O)`` which gives O. Therefore:: + + L[B] = B D E O + +Using the same procedure one finds:: + + L[C] = C + merge(DO,FO,DF) + = C + D + merge(O,FO,F) + = C + D + F + merge(O,O) + = C D F O + +Now we can compute:: + + L[A] = A + merge(BDEO,CDFO,BC) + = A + B + merge(DEO,CDFO,C) + = A + B + C + merge(DEO,DFO) + = A + B + C + D + merge(EO,FO) + = A + B + C + D + E + merge(O,FO) + = A + B + C + D + E + F + merge(O,O) + = A B C D E F O + +In this example, the linearization is ordered in a pretty nice way +according to the inheritance level, in the sense that lower levels (i.e. +more specialized classes) have higher precedence (see the inheritance +graph). However, this is not the general case. + +I leave as an exercise for the reader to compute the linearization for +my second example: + + >>> O = object + >>> class F(O): pass + >>> class E(O): pass + >>> class D(O): pass + >>> class C(D,F): pass + >>> class B(E,D): pass + >>> class A(B,C): pass + +The only difference with the previous example is the change B(D,E) --> +B(E,D); however even such a little modification completely changes the +ordering of the hierarchy: + + .. code-block:: text + + 6 + --- + Level 3 | O | + / --- \ + / | \ + / | \ + / | \ + --- --- --- + Level 2 2 | E | 4 | D | | F | 5 + --- --- --- + \ / \ / + \ / \ / + \ / \ / + --- --- + Level 1 1 | B | | C | 3 + --- --- + \ / + \ / + --- + Level 0 0 | A | + --- + + +Notice that the class E, which is in the second level of the hierarchy, +precedes the class C, which is in the first level of the hierarchy, i.e. +E is more specialized than C, even if it is in a higher level. + +A lazy programmer can obtain the MRO directly from Python 2.2, since in +this case it coincides with the Python 2.3 linearization. It is enough +to invoke the :meth:`~type.mro` method of class A: + + >>> A.mro() # doctest: +NORMALIZE_WHITESPACE + [, , , + , , , + ] + +Finally, let me consider the example discussed in the first section, +involving a serious order disagreement. In this case, it is +straightforward to compute the linearizations of O, X, Y, A and B: + + .. code-block:: text + + L[O] = 0 + L[X] = X O + L[Y] = Y O + L[A] = A X Y O + L[B] = B Y X O + +However, it is impossible to compute the linearization for a class C +that inherits from A and B:: + + L[C] = C + merge(AXYO, BYXO, AB) + = C + A + merge(XYO, BYXO, B) + = C + A + B + merge(XYO, YXO) + +At this point we cannot merge the lists XYO and YXO, since X is in the +tail of YXO whereas Y is in the tail of XYO: therefore there are no +good heads and the C3 algorithm stops. Python 2.3 raises an error and +refuses to create the class C. + +Bad Method Resolution Orders +---------------------------- + +A MRO is *bad* when it breaks such fundamental properties as local +precedence ordering and monotonicity. In this section, I will show +that both the MRO for classic classes and the MRO for new style classes +in Python 2.2 are bad. + +It is easier to start with the local precedence ordering. Consider the +following example: + + >>> F=type('Food',(),{'remember2buy':'spam'}) + >>> E=type('Eggs',(F,),{'remember2buy':'eggs'}) + >>> G=type('GoodFood',(F,E),{}) # under Python 2.3 this is an error! # doctest: +SKIP + +with inheritance diagram + + .. code-block:: text + + O + | + (buy spam) F + | \ + | E (buy eggs) + | / + G + + (buy eggs or spam ?) + + +We see that class G inherits from F and E, with F *before* E: therefore +we would expect the attribute *G.remember2buy* to be inherited by +*F.rembermer2buy* and not by *E.remember2buy*: nevertheless Python 2.2 +gives + + >>> G.remember2buy # doctest: +SKIP + 'eggs' + +This is a breaking of local precedence ordering since the order in the +local precedence list, i.e. the list of the parents of G, is not +preserved in the Python 2.2 linearization of G:: + + L[G,P22]= G E F object # F *follows* E + +One could argue that the reason why F follows E in the Python 2.2 +linearization is that F is less specialized than E, since F is the +superclass of E; nevertheless the breaking of local precedence ordering +is quite non-intuitive and error prone. This is particularly true since +it is a different from old style classes: + + >>> class F: remember2buy='spam' + >>> class E(F): remember2buy='eggs' + >>> class G(F,E): pass # doctest: +SKIP + >>> G.remember2buy # doctest: +SKIP + 'spam' + +In this case the MRO is GFEF and the local precedence ordering is +preserved. + +As a general rule, hierarchies such as the previous one should be +avoided, since it is unclear if F should override E or vice-versa. +Python 2.3 solves the ambiguity by raising an exception in the creation +of class G, effectively stopping the programmer from generating +ambiguous hierarchies. The reason for that is that the C3 algorithm +fails when the merge:: + + merge(FO,EFO,FE) + +cannot be computed, because F is in the tail of EFO and E is in the tail +of FE. + +The real solution is to design a non-ambiguous hierarchy, i.e. to derive +G from E and F (the more specific first) and not from F and E; in this +case the MRO is GEF without any doubt. + + .. code-block:: text + + O + | + F (spam) + / | + (eggs) E | + \ | + G + (eggs, no doubt) + + +Python 2.3 forces the programmer to write good hierarchies (or, at +least, less error-prone ones). + +On a related note, let me point out that the Python 2.3 algorithm is +smart enough to recognize obvious mistakes, as the duplication of +classes in the list of parents: + + >>> class A(object): pass + >>> class C(A,A): pass # error + Traceback (most recent call last): + File "", line 1, in ? + TypeError: duplicate base class A + +Python 2.2 (both for classic classes and new style classes) in this +situation, would not raise any exception. + +Finally, I would like to point out two lessons we have learned from this +example: + +1. despite the name, the MRO determines the resolution order of + attributes, not only of methods; + +2. the default food for Pythonistas is spam ! (but you already knew + that ;-) + +Having discussed the issue of local precedence ordering, let me now +consider the issue of monotonicity. My goal is to show that neither the +MRO for classic classes nor that for Python 2.2 new style classes is +monotonic. + +To prove that the MRO for classic classes is non-monotonic is rather +trivial, it is enough to look at the diamond diagram: + + .. code-block:: text + + + C + / \ + / \ + A B + \ / + \ / + D + +One easily discerns the inconsistency:: + + L[B,P21] = B C # B precedes C : B's methods win + L[D,P21] = D A C B C # B follows C : C's methods win! + +On the other hand, there are no problems with the Python 2.2 and 2.3 +MROs, they give both:: + + L[D] = D A B C + +Guido points out in his essay [#]_ that the classic MRO is not so bad in +practice, since one can typically avoids diamonds for classic classes. +But all new style classes inherit from ``object``, therefore diamonds are +unavoidable and inconsistencies shows up in every multiple inheritance +graph. + +The MRO of Python 2.2 makes breaking monotonicity difficult, but not +impossible. The following example, originally provided by Samuele +Pedroni, shows that the MRO of Python 2.2 is non-monotonic: + + >>> class A(object): pass + >>> class B(object): pass + >>> class C(object): pass + >>> class D(object): pass + >>> class E(object): pass + >>> class K1(A,B,C): pass + >>> class K2(D,B,E): pass + >>> class K3(D,A): pass + >>> class Z(K1,K2,K3): pass + +Here are the linearizations according to the C3 MRO (the reader should +verify these linearizations as an exercise and draw the inheritance +diagram ;-) :: + + L[A] = A O + L[B] = B O + L[C] = C O + L[D] = D O + L[E] = E O + L[K1]= K1 A B C O + L[K2]= K2 D B E O + L[K3]= K3 D A O + L[Z] = Z K1 K2 K3 D A B C E O + +Python 2.2 gives exactly the same linearizations for A, B, C, D, E, K1, +K2 and K3, but a different linearization for Z:: + + L[Z,P22] = Z K1 K3 A K2 D B C E O + +It is clear that this linearization is *wrong*, since A comes before D +whereas in the linearization of K3 A comes *after* D. In other words, in +K3 methods derived by D override methods derived by A, but in Z, which +still is a subclass of K3, methods derived by A override methods derived +by D! This is a violation of monotonicity. Moreover, the Python 2.2 +linearization of Z is also inconsistent with local precedence ordering, +since the local precedence list of the class Z is [K1, K2, K3] (K2 +precedes K3), whereas in the linearization of Z K2 *follows* K3. These +problems explain why the 2.2 rule has been dismissed in favor of the C3 +rule. + +The end +------- + +This section is for the impatient reader, who skipped all the previous +sections and jumped immediately to the end. This section is for the +lazy programmer too, who didn't want to exercise her/his brain. +Finally, it is for the programmer with some hubris, otherwise s/he would +not be reading a paper on the C3 method resolution order in multiple +inheritance hierarchies ;-) These three virtues taken all together (and +*not* separately) deserve a prize: the prize is a short Python 2.2 +script that allows you to compute the 2.3 MRO without risk to your +brain. Simply change the last line to play with the various examples I +have discussed in this paper.:: + + # + + """C3 algorithm by Samuele Pedroni (with readability enhanced by me).""" + + class __metaclass__(type): + "All classes are metamagically modified to be nicely printed" + __repr__ = lambda cls: cls.__name__ + + class ex_2: + "Serious order disagreement" #From Guido + class O: pass + class X(O): pass + class Y(O): pass + class A(X,Y): pass + class B(Y,X): pass + try: + class Z(A,B): pass #creates Z(A,B) in Python 2.2 + except TypeError: + pass # Z(A,B) cannot be created in Python 2.3 + + class ex_5: + "My first example" + class O: pass + class F(O): pass + class E(O): pass + class D(O): pass + class C(D,F): pass + class B(D,E): pass + class A(B,C): pass + + class ex_6: + "My second example" + class O: pass + class F(O): pass + class E(O): pass + class D(O): pass + class C(D,F): pass + class B(E,D): pass + class A(B,C): pass + + class ex_9: + "Difference between Python 2.2 MRO and C3" #From Samuele + class O: pass + class A(O): pass + class B(O): pass + class C(O): pass + class D(O): pass + class E(O): pass + class K1(A,B,C): pass + class K2(D,B,E): pass + class K3(D,A): pass + class Z(K1,K2,K3): pass + + def merge(seqs): + print '\n\nCPL[%s]=%s' % (seqs[0][0],seqs), + res = []; i=0 + while 1: + nonemptyseqs=[seq for seq in seqs if seq] + if not nonemptyseqs: return res + i+=1; print '\n',i,'round: candidates...', + for seq in nonemptyseqs: # find merge candidates among seq heads + cand = seq[0]; print ' ',cand, + nothead=[s for s in nonemptyseqs if cand in s[1:]] + if nothead: cand=None #reject candidate + else: break + if not cand: raise "Inconsistent hierarchy" + res.append(cand) + for seq in nonemptyseqs: # remove cand + if seq[0] == cand: del seq[0] + + def mro(C): + "Compute the class precedence list (mro) according to C3" + return merge([[C]]+map(mro,C.__bases__)+[list(C.__bases__)]) + + def print_mro(C): + print '\nMRO[%s]=%s' % (C,mro(C)) + print '\nP22 MRO[%s]=%s' % (C,C.mro()) + + print_mro(ex_9.Z) + + # + +That's all folks, + + enjoy ! + + +Resources +--------- + +.. [#] The thread on python-dev started by Samuele Pedroni: + https://mail.python.org/pipermail/python-dev/2002-October/029035.html + +.. [#] The paper *A Monotonic Superclass Linearization for Dylan*: + https://doi.org/10.1145/236337.236343 + +.. [#] Guido van Rossum's essay, *Unifying types and classes in Python 2.2*: + https://web.archive.org/web/20140210194412/http://www.python.org/download/releases/2.2.2/descrintro diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index bb1c00e0aa51d5..06459d1b222964 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -162,12 +162,12 @@ the :option:`!-X` option takes precedence over the environment variable. Example, using the environment variable:: - $ PYTHONPERFSUPPORT=1 python script.py + $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python script.py $ perf report -g -i perf.data Example, using the :option:`!-X` option:: - $ python -X perf script.py + $ perf record -F 9999 -g -o perf.data python -X perf script.py $ perf report -g -i perf.data Example, using the :mod:`sys` APIs in file :file:`example.py`: @@ -184,7 +184,7 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: ...then:: - $ python ./example.py + $ perf record -F 9999 -g -o perf.data python ./example.py $ perf report -g -i perf.data @@ -205,3 +205,62 @@ You can check if your system has been compiled with this flag by running:: If you don't see any output it means that your interpreter has not been compiled with frame pointers and therefore it may not be able to show Python functions in the output of ``perf``. + + +How to work without frame pointers +---------------------------------- + +If you are working with a Python interpreter that has been compiled without +frame pointers, you can still use the ``perf`` profiler, but the overhead will be +a bit higher because Python needs to generate unwinding information for every +Python function call on the fly. Additionally, ``perf`` will take more time to +process the data because it will need to use the DWARF debugging information to +unwind the stack and this is a slow process. + +To enable this mode, you can use the environment variable +:envvar:`PYTHON_PERF_JIT_SUPPORT` or the :option:`-X perf_jit <-X>` option, +which will enable the JIT mode for the ``perf`` profiler. + +.. note:: + + Due to a bug in the ``perf`` tool, only ``perf`` versions higher than v6.8 + will work with the JIT mode. The fix was also backported to the v6.7.2 + version of the tool. + + Note that when checking the version of the ``perf`` tool (which can be done + by running ``perf version``) you must take into account that some distros + add some custom version numbers including a ``-`` character. This means + that ``perf 6.7-3`` is not necessarily ``perf 6.7.3``. + +When using the perf JIT mode, you need an extra step before you can run ``perf +report``. You need to call the ``perf inject`` command to inject the JIT +information into the ``perf.data`` file.:: + + $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperf_jit my_script.py + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data + +or using the environment variable:: + + $ PYTHON_PERF_JIT_SUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data + +``perf inject --jit`` command will read ``perf.data``, +automatically pick up the perf dump file that Python creates (in +``/tmp/perf-$PID.dump``), and then create ``perf.jit.data`` which merges all the +JIT information together. It should also create a lot of ``jitted-XXXX-N.so`` +files in the current directory which are ELF images for all the JIT trampolines +that were created by Python. + +.. warning:: + Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take + snapshots of the stack of the process being profiled and save the + information in the ``perf.data`` file. By default the size of the stack dump + is 8192 bytes but the user can change the size by passing the size after + comma like ``--call-graph dwarf,4096``. The size of the stack dump is + important because if the size is too small ``perf`` will not be able to + unwind the stack and the output will be incomplete. On the other hand, if + the size is too big, then ``perf`` won't be able to sample the process as + frequently as it would like as the overhead will be higher. + diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 501b16d82d4d6f..9f73c811cfcbc0 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -1,3 +1,5 @@ +:orphan: + .. _pyporting-howto: ************************************* @@ -6,423 +8,30 @@ How to port Python 2 Code to Python 3 :author: Brett Cannon -.. topic:: Abstract - - Python 2 reached its official end-of-life at the start of 2020. This means - that no new bug reports, fixes, or changes will be made to Python 2 - it's - no longer supported. - - This guide is intended to provide you with a path to Python 3 for your - code, that includes compatibility with Python 2 as a first step. - - If you are looking to port an extension module instead of pure Python code, - please see :ref:`cporting-howto`. - - The archived python-porting_ mailing list may contain some useful guidance. - - -The Short Explanation -===================== - -To achieve Python 2/3 compatibility in a single code base, the basic steps -are: - -#. Only worry about supporting Python 2.7 -#. Make sure you have good test coverage (coverage.py_ can help; - ``python -m pip install coverage``) -#. Learn the differences between Python 2 and 3 -#. Use Futurize_ (or Modernize_) to update your code (e.g. ``python -m pip install future``) -#. Use Pylint_ to help make sure you don't regress on your Python 3 support - (``python -m pip install pylint``) -#. Use caniusepython3_ to find out which of your dependencies are blocking your - use of Python 3 (``python -m pip install caniusepython3``) -#. Once your dependencies are no longer blocking you, use continuous integration - to make sure you stay compatible with Python 2 and 3 (tox_ can help test - against multiple versions of Python; ``python -m pip install tox``) -#. Consider using optional :term:`static type checking ` - to make sure your type usage - works in both Python 2 and 3 (e.g. use mypy_ to check your typing under both - Python 2 and Python 3; ``python -m pip install mypy``). - -.. note:: - - Note: Using ``python -m pip install`` guarantees that the ``pip`` you invoke - is the one installed for the Python currently in use, whether it be - a system-wide ``pip`` or one installed within a - :ref:`virtual environment `. - -Details -======= - -Even if other factors - say, dependencies over which you have no control - -still require you to support Python 2, that does not prevent you taking the -step of including Python 3 support. - -Most changes required to support Python 3 lead to cleaner code using newer -practices even in Python 2 code. - - -Different versions of Python 2 ------------------------------- - -Ideally, your code should be compatible with Python 2.7, which was the -last supported version of Python 2. - -Some of the tools mentioned in this guide will not work with Python 2.6. - -If absolutely necessary, the six_ project can help you support Python 2.5 and -3 simultaneously. Do realize, though, that nearly all the projects listed in -this guide will not be available to you. - -If you are able to skip Python 2.5 and older, the required changes to your -code will be minimal. At worst you will have to use a function instead of a -method in some instances or have to import a function instead of using a -built-in one. - - -Make sure you specify the proper version support in your ``setup.py`` file --------------------------------------------------------------------------- - -In your ``setup.py`` file you should have the proper `trove classifier`_ -specifying what versions of Python you support. As your project does not support -Python 3 yet you should at least have -``Programming Language :: Python :: 2 :: Only`` specified. Ideally you should -also specify each major/minor version of Python that you do support, e.g. -``Programming Language :: Python :: 2.7``. - - -Have good test coverage ------------------------ - -Once you have your code supporting the oldest version of Python 2 you want it -to, you will want to make sure your test suite has good coverage. A good rule of -thumb is that if you want to be confident enough in your test suite that any -failures that appear after having tools rewrite your code are actual bugs in the -tools and not in your code. If you want a number to aim for, try to get over 80% -coverage (and don't feel bad if you find it hard to get better than 90% -coverage). If you don't already have a tool to measure test coverage then -coverage.py_ is recommended. - - -Be aware of the differences between Python 2 and 3 --------------------------------------------------- - -Once you have your code well-tested you are ready to begin porting your code to -Python 3! But to fully understand how your code is going to change and what -you want to look out for while you code, you will want to learn what changes -Python 3 makes in terms of Python 2. - -Some resources for understanding the differences and their implications for you -code: - -* the :ref:`"What's New" ` doc for each release of Python 3 -* the `Porting to Python 3`_ book (which is free online) -* the handy `cheat sheet`_ from the Python-Future project. - - -Update your code ----------------- - -There are tools available that can port your code automatically. - -Futurize_ does its best to make Python 3 idioms and practices exist in Python -2, e.g. backporting the ``bytes`` type from Python 3 so that you have -semantic parity between the major versions of Python. This is the better -approach for most cases. - -Modernize_, on the other hand, is more conservative and targets a Python 2/3 -subset of Python, directly relying on six_ to help provide compatibility. - -A good approach is to run the tool over your test suite first and visually -inspect the diff to make sure the transformation is accurate. After you have -transformed your test suite and verified that all the tests still pass as -expected, then you can transform your application code knowing that any tests -which fail is a translation failure. - -Unfortunately the tools can't automate everything to make your code work under -Python 3, and you will also need to read the tools' documentation in case some -options you need are turned off by default. - -Key issues to be aware of and check for: - -Division -++++++++ - -In Python 3, ``5 / 2 == 2.5`` and not ``2`` as it was in Python 2; all -division between ``int`` values result in a ``float``. This change has -actually been planned since Python 2.2 which was released in 2002. Since then -users have been encouraged to add ``from __future__ import division`` to any -and all files which use the ``/`` and ``//`` operators or to be running the -interpreter with the ``-Q`` flag. If you have not been doing this then you -will need to go through your code and do two things: - -#. Add ``from __future__ import division`` to your files -#. Update any division operator as necessary to either use ``//`` to use floor - division or continue using ``/`` and expect a float - -The reason that ``/`` isn't simply translated to ``//`` automatically is that if -an object defines a ``__truediv__`` method but not ``__floordiv__`` then your -code would begin to fail (e.g. a user-defined class that uses ``/`` to -signify some operation but not ``//`` for the same thing or at all). +Python 2 reached its official end-of-life at the start of 2020. This means +that no new bug reports, fixes, or changes will be made to Python 2 - it's +no longer supported: see :pep:`373` and +`status of Python versions `_. +If you are looking to port an extension module instead of pure Python code, +please see :ref:`cporting-howto`. -Text versus binary data -+++++++++++++++++++++++ +The archived python-porting_ mailing list may contain some useful guidance. -In Python 2 you could use the ``str`` type for both text and binary data. -Unfortunately this confluence of two different concepts could lead to brittle -code which sometimes worked for either kind of data, sometimes not. It also -could lead to confusing APIs if people didn't explicitly state that something -that accepted ``str`` accepted either text or binary data instead of one -specific type. This complicated the situation especially for anyone supporting -multiple languages as APIs wouldn't bother explicitly supporting ``unicode`` -when they claimed text data support. +Since Python 3.11 the original porting guide was discontinued. +You can find the old guide in the +`archive `_. -Python 3 made text and binary data distinct types that cannot simply be mixed -together. For any code that deals only with text or only binary data, this -separation doesn't pose an issue. But for code that has to deal with both, it -does mean you might have to now care about when you are using text compared -to binary data, which is why this cannot be entirely automated. -Decide which APIs take text and which take binary (it is **highly** recommended -you don't design APIs that can take both due to the difficulty of keeping the -code working; as stated earlier it is difficult to do well). In Python 2 this -means making sure the APIs that take text can work with ``unicode`` and those -that work with binary data work with the ``bytes`` type from Python 3 -(which is a subset of ``str`` in Python 2 and acts as an alias for ``bytes`` -type in Python 2). Usually the biggest issue is realizing which methods exist -on which types in Python 2 and 3 simultaneously (for text that's ``unicode`` -in Python 2 and ``str`` in Python 3, for binary that's ``str``/``bytes`` in -Python 2 and ``bytes`` in Python 3). +Third-party guides +================== -The following table lists the **unique** methods of each data type across -Python 2 and 3 (e.g., the ``decode()`` method is usable on the equivalent binary -data type in either Python 2 or 3, but it can't be used by the textual data -type consistently between Python 2 and 3 because ``str`` in Python 3 doesn't -have the method). Do note that as of Python 3.5 the ``__mod__`` method was -added to the bytes type. +There are also multiple third-party guides that might be useful: -======================== ===================== -**Text data** **Binary data** ------------------------- --------------------- -\ decode ------------------------- --------------------- -encode ------------------------- --------------------- -format ------------------------- --------------------- -isdecimal ------------------------- --------------------- -isnumeric -======================== ===================== +- `Guide by Fedora `_ +- `PyCon 2020 tutorial `_ +- `Guide by DigitalOcean `_ +- `Guide by ActiveState `_ -Making the distinction easier to handle can be accomplished by encoding and -decoding between binary data and text at the edge of your code. This means that -when you receive text in binary data, you should immediately decode it. And if -your code needs to send text as binary data then encode it as late as possible. -This allows your code to work with only text internally and thus eliminates -having to keep track of what type of data you are working with. -The next issue is making sure you know whether the string literals in your code -represent text or binary data. You should add a ``b`` prefix to any -literal that presents binary data. For text you should add a ``u`` prefix to -the text literal. (There is a :mod:`__future__` import to force all unspecified -literals to be Unicode, but usage has shown it isn't as effective as adding a -``b`` or ``u`` prefix to all literals explicitly) - -You also need to be careful about opening files. Possibly you have not always -bothered to add the ``b`` mode when opening a binary file (e.g., ``rb`` for -binary reading). Under Python 3, binary files and text files are clearly -distinct and mutually incompatible; see the :mod:`io` module for details. -Therefore, you **must** make a decision of whether a file will be used for -binary access (allowing binary data to be read and/or written) or textual access -(allowing text data to be read and/or written). You should also use :func:`io.open` -for opening files instead of the built-in :func:`open` function as the :mod:`io` -module is consistent from Python 2 to 3 while the built-in :func:`open` function -is not (in Python 3 it's actually :func:`io.open`). Do not bother with the -outdated practice of using :func:`codecs.open` as that's only necessary for -keeping compatibility with Python 2.5. - -The constructors of both ``str`` and ``bytes`` have different semantics for the -same arguments between Python 2 and 3. Passing an integer to ``bytes`` in Python 2 -will give you the string representation of the integer: ``bytes(3) == '3'``. -But in Python 3, an integer argument to ``bytes`` will give you a bytes object -as long as the integer specified, filled with null bytes: -``bytes(3) == b'\x00\x00\x00'``. A similar worry is necessary when passing a -bytes object to ``str``. In Python 2 you just get the bytes object back: -``str(b'3') == b'3'``. But in Python 3 you get the string representation of the -bytes object: ``str(b'3') == "b'3'"``. - -Finally, the indexing of binary data requires careful handling (slicing does -**not** require any special handling). In Python 2, -``b'123'[1] == b'2'`` while in Python 3 ``b'123'[1] == 50``. Because binary data -is simply a collection of binary numbers, Python 3 returns the integer value for -the byte you index on. But in Python 2 because ``bytes == str``, indexing -returns a one-item slice of bytes. The six_ project has a function -named ``six.indexbytes()`` which will return an integer like in Python 3: -``six.indexbytes(b'123', 1)``. - -To summarize: - -#. Decide which of your APIs take text and which take binary data -#. Make sure that your code that works with text also works with ``unicode`` and - code for binary data works with ``bytes`` in Python 2 (see the table above - for what methods you cannot use for each type) -#. Mark all binary literals with a ``b`` prefix, textual literals with a ``u`` - prefix -#. Decode binary data to text as soon as possible, encode text as binary data as - late as possible -#. Open files using :func:`io.open` and make sure to specify the ``b`` mode when - appropriate -#. Be careful when indexing into binary data - - -Use feature detection instead of version detection -++++++++++++++++++++++++++++++++++++++++++++++++++ - -Inevitably you will have code that has to choose what to do based on what -version of Python is running. The best way to do this is with feature detection -of whether the version of Python you're running under supports what you need. -If for some reason that doesn't work then you should make the version check be -against Python 2 and not Python 3. To help explain this, let's look at an -example. - -Let's pretend that you need access to a feature of :mod:`importlib` that -is available in Python's standard library since Python 3.3 and available for -Python 2 through importlib2_ on PyPI. You might be tempted to write code to -access e.g. the :mod:`importlib.abc` module by doing the following:: - - import sys - - if sys.version_info[0] == 3: - from importlib import abc - else: - from importlib2 import abc - -The problem with this code is what happens when Python 4 comes out? It would -be better to treat Python 2 as the exceptional case instead of Python 3 and -assume that future Python versions will be more compatible with Python 3 than -Python 2:: - - import sys - - if sys.version_info[0] > 2: - from importlib import abc - else: - from importlib2 import abc - -The best solution, though, is to do no version detection at all and instead rely -on feature detection. That avoids any potential issues of getting the version -detection wrong and helps keep you future-compatible:: - - try: - from importlib import abc - except ImportError: - from importlib2 import abc - - -Prevent compatibility regressions ---------------------------------- - -Once you have fully translated your code to be compatible with Python 3, you -will want to make sure your code doesn't regress and stop working under -Python 3. This is especially true if you have a dependency which is blocking you -from actually running under Python 3 at the moment. - -To help with staying compatible, any new modules you create should have -at least the following block of code at the top of it:: - - from __future__ import absolute_import - from __future__ import division - from __future__ import print_function - -You can also run Python 2 with the ``-3`` flag to be warned about various -compatibility issues your code triggers during execution. If you turn warnings -into errors with ``-Werror`` then you can make sure that you don't accidentally -miss a warning. - -You can also use the Pylint_ project and its ``--py3k`` flag to lint your code -to receive warnings when your code begins to deviate from Python 3 -compatibility. This also prevents you from having to run Modernize_ or Futurize_ -over your code regularly to catch compatibility regressions. This does require -you only support Python 2.7 and Python 3.4 or newer as that is Pylint's -minimum Python version support. - - -Check which dependencies block your transition ----------------------------------------------- - -**After** you have made your code compatible with Python 3 you should begin to -care about whether your dependencies have also been ported. The caniusepython3_ -project was created to help you determine which projects --- directly or indirectly -- are blocking you from supporting Python 3. There -is both a command-line tool as well as a web interface at -https://caniusepython3.com. - -The project also provides code which you can integrate into your test suite so -that you will have a failing test when you no longer have dependencies blocking -you from using Python 3. This allows you to avoid having to manually check your -dependencies and to be notified quickly when you can start running on Python 3. - - -Update your ``setup.py`` file to denote Python 3 compatibility --------------------------------------------------------------- - -Once your code works under Python 3, you should update the classifiers in -your ``setup.py`` to contain ``Programming Language :: Python :: 3`` and to not -specify sole Python 2 support. This will tell anyone using your code that you -support Python 2 **and** 3. Ideally you will also want to add classifiers for -each major/minor version of Python you now support. - - -Use continuous integration to stay compatible ---------------------------------------------- - -Once you are able to fully run under Python 3 you will want to make sure your -code always works under both Python 2 and 3. Probably the best tool for running -your tests under multiple Python interpreters is tox_. You can then integrate -tox with your continuous integration system so that you never accidentally break -Python 2 or 3 support. - -You may also want to use the ``-bb`` flag with the Python 3 interpreter to -trigger an exception when you are comparing bytes to strings or bytes to an int -(the latter is available starting in Python 3.5). By default type-differing -comparisons simply return ``False``, but if you made a mistake in your -separation of text/binary data handling or indexing on bytes you wouldn't easily -find the mistake. This flag will raise an exception when these kinds of -comparisons occur, making the mistake much easier to track down. - - -Consider using optional static type checking --------------------------------------------- - -Another way to help port your code is to use a :term:`static type checker` like -mypy_ or pytype_ on your code. These tools can be used to analyze your code as -if it's being run under Python 2, then you can run the tool a second time as if -your code is running under Python 3. By running a static type checker twice like -this you can discover if you're e.g. misusing binary data type in one version -of Python compared to another. If you add optional type hints to your code you -can also explicitly state whether your APIs use textual or binary data, helping -to make sure everything functions as expected in both versions of Python. - - -.. _caniusepython3: https://pypi.org/project/caniusepython3 -.. _cheat sheet: https://python-future.org/compatible_idioms.html -.. _coverage.py: https://pypi.org/project/coverage -.. _Futurize: https://python-future.org/automatic_conversion.html -.. _importlib2: https://pypi.org/project/importlib2 -.. _Modernize: https://python-modernize.readthedocs.io/ -.. _mypy: https://mypy-lang.org/ -.. _Porting to Python 3: http://python3porting.com/ -.. _Pylint: https://pypi.org/project/pylint - -.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.io/en/latest/python3/questions_and_answers.html - -.. _pytype: https://github.com/google/pytype -.. _python-future: https://python-future.org/ .. _python-porting: https://mail.python.org/pipermail/python-porting/ -.. _six: https://pypi.org/project/six -.. _tox: https://pypi.org/project/tox -.. _trove classifier: https://pypi.org/classifiers - -.. _Why Python 3 exists: https://snarky.ca/why-python-3-exists diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index 0bbf97da39768d..cbc49d15a0771b 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -100,8 +100,8 @@ mainloop of the web server:: (clientsocket, address) = serversocket.accept() # now do something with the clientsocket # in this case, we'll pretend this is a threaded server - ct = client_thread(clientsocket) - ct.run() + ct = make_client_thread(clientsocket) + ct.start() There's actually 3 general ways in which this loop could work - dispatching a thread to handle ``clientsocket``, create a new process to handle diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 38dd09f0a721d2..70c34cde8a0659 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -1,10 +1,9 @@ .. _sortinghowto: -Sorting HOW TO -************** +Sorting Techniques +****************** :Author: Andrew Dalke and Raymond Hettinger -:Release: 0.1 Python lists have a built-in :meth:`list.sort` method that modifies the list @@ -48,15 +47,18 @@ lists. In contrast, the :func:`sorted` function accepts any iterable. Key Functions ============= -Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a -function (or other callable) to be called on each list element prior to making +The :meth:`list.sort` method and the functions :func:`sorted`, +:func:`min`, :func:`max`, :func:`heapq.nsmallest`, and +:func:`heapq.nlargest` have a *key* parameter to specify a function (or +other callable) to be called on each list element prior to making comparisons. -For example, here's a case-insensitive string comparison: +For example, here's a case-insensitive string comparison using +:meth:`str.casefold`: .. doctest:: - >>> sorted("This is a test string from Andrew".split(), key=str.lower) + >>> sorted("This is a test string from Andrew".split(), key=str.casefold) ['a', 'Andrew', 'from', 'is', 'string', 'test', 'This'] The value of the *key* parameter should be a function (or other callable) that @@ -97,10 +99,14 @@ The same technique works for objects with named attributes. For example: >>> sorted(student_objects, key=lambda student: student.age) # sort by age [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] -Operator Module Functions -========================= +Objects with named attributes can be made by a regular class as shown +above, or they can be instances of :class:`~dataclasses.dataclass` or +a :term:`named tuple`. -The key-function patterns shown above are very common, so Python provides +Operator Module Functions and Partial Function Evaluation +========================================================= + +The :term:`key function` patterns shown above are very common, so Python provides convenience functions to make accessor functions easier and faster. The :mod:`operator` module has :func:`~operator.itemgetter`, :func:`~operator.attrgetter`, and a :func:`~operator.methodcaller` function. @@ -128,6 +134,24 @@ sort by *grade* then by *age*: >>> sorted(student_objects, key=attrgetter('grade', 'age')) [('john', 'A', 15), ('dave', 'B', 10), ('jane', 'B', 12)] +The :mod:`functools` module provides another helpful tool for making +key-functions. The :func:`~functools.partial` function can reduce the +`arity `_ of a multi-argument +function making it suitable for use as a key-function. + +.. doctest:: + + >>> from functools import partial + >>> from unicodedata import normalize + + >>> names = 'Zoë Åbjørn Núñez Élana Zeke Abe Nubia Eloise'.split() + + >>> sorted(names, key=partial(normalize, 'NFD')) + ['Abe', 'Åbjørn', 'Eloise', 'Élana', 'Nubia', 'Núñez', 'Zeke', 'Zoë'] + + >>> sorted(names, key=partial(normalize, 'NFC')) + ['Abe', 'Eloise', 'Nubia', 'Núñez', 'Zeke', 'Zoë', 'Åbjørn', 'Élana'] + Ascending and Descending ======================== @@ -200,6 +224,8 @@ This idiom is called Decorate-Sort-Undecorate after its three steps: For example, to sort the student data by *grade* using the DSU approach: +.. doctest:: + >>> decorated = [(student.grade, i, student) for i, student in enumerate(student_objects)] >>> decorated.sort() >>> [student for grade, i, student in decorated] # undecorate @@ -249,6 +275,70 @@ to make it usable as a key function:: sorted(words, key=cmp_to_key(strcoll)) # locale-aware sort order +Strategies For Unorderable Types and Values +=========================================== + +A number of type and value issues can arise when sorting. +Here are some strategies that can help: + +* Convert non-comparable input types to strings prior to sorting: + +.. doctest:: + + >>> data = ['twelve', '11', 10] + >>> sorted(map(str, data)) + ['10', '11', 'twelve'] + +This is needed because most cross-type comparisons raise a +:exc:`TypeError`. + +* Remove special values prior to sorting: + +.. doctest:: + + >>> from math import isnan + >>> from itertools import filterfalse + >>> data = [3.3, float('nan'), 1.1, 2.2] + >>> sorted(filterfalse(isnan, data)) + [1.1, 2.2, 3.3] + +This is needed because the `IEEE-754 standard +`_ specifies that, "Every NaN +shall compare unordered with everything, including itself." + +Likewise, ``None`` can be stripped from datasets as well: + +.. doctest:: + + >>> data = [3.3, None, 1.1, 2.2] + >>> sorted(x for x in data if x is not None) + [1.1, 2.2, 3.3] + +This is needed because ``None`` is not comparable to other types. + +* Convert mapping types into sorted item lists before sorting: + +.. doctest:: + + >>> data = [{'a': 1}, {'b': 2}] + >>> sorted(data, key=lambda d: sorted(d.items())) + [{'a': 1}, {'b': 2}] + +This is needed because dict-to-dict comparisons raise a +:exc:`TypeError`. + +* Convert set types into sorted lists before sorting: + +.. doctest:: + + >>> data = [{'a', 'b', 'c'}, {'b', 'c', 'd'}] + >>> sorted(map(sorted, data)) + [['a', 'b', 'c'], ['b', 'c', 'd']] + +This is needed because the elements contained in set types do not have a +deterministic order. For example, ``list({'a', 'b'})`` may produce +either ``['a', 'b']`` or ``['b', 'a']``. + Odds and Ends ============= @@ -282,7 +372,11 @@ Odds and Ends [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] However, note that ``<`` can fall back to using :meth:`~object.__gt__` if - :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__`). + :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__` + for details on the mechanics). To avoid surprises, :pep:`8` + recommends that all six comparison methods be implemented. + The :func:`~functools.total_ordering` decorator is provided to make that + task easier. * Key functions need not depend directly on the objects being sorted. A key function can also access external resources. For instance, if the student grades @@ -295,3 +389,24 @@ Odds and Ends >>> newgrades = {'john': 'F', 'jane':'A', 'dave': 'C'} >>> sorted(students, key=newgrades.__getitem__) ['jane', 'dave', 'john'] + +Partial Sorts +============= + +Some applications require only some of the data to be ordered. The standard +library provides several tools that do less work than a full sort: + +* :func:`min` and :func:`max` return the smallest and largest values, + respectively. These functions make a single pass over the input data and + require almost no auxiliary memory. + +* :func:`heapq.nsmallest` and :func:`heapq.nlargest` return + the *n* smallest and largest values, respectively. These functions + make a single pass over the data keeping only *n* elements in memory + at a time. For values of *n* that are small relative to the number of + inputs, these functions make far fewer comparisons than a full sort. + +* :func:`heapq.heappush` and :func:`heapq.heappop` create and maintain a + partially sorted arrangement of data that keeps the smallest element + at position ``0``. These functions are suitable for implementing + priority queues which are commonly used for task scheduling. diff --git a/Doc/howto/timerfd.rst b/Doc/howto/timerfd.rst index 98f0294f9d082d..b5fc06ae8c6810 100644 --- a/Doc/howto/timerfd.rst +++ b/Doc/howto/timerfd.rst @@ -108,7 +108,7 @@ descriptors to wait until the file descriptor is ready for reading: # In 1.5 seconds, 1st timer, 2nd timer and 3rd timer fires at once. # # If a timer file descriptor is signaled more than once since - # the last os.read() call, os.read() returns the nubmer of signaled + # the last os.read() call, os.read() returns the number of signaled # as host order of class bytes. print(f"Signaled events={events}") for fd, event in events: diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 7f54a410881514..33a2a7ea89ea07 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -594,5 +594,5 @@ This document was reviewed and revised by John Lee. scripts with a localhost server, I have to prevent urllib from using the proxy. .. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe - `_. + `_. diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py index 2fc1570e654db6..aa2a5c7cda52aa 100644 --- a/Doc/includes/email-dir.py +++ b/Doc/includes/email-dir.py @@ -53,7 +53,7 @@ def main(): # Guess the content type based on the file's extension. Encoding # will be ignored, although we should check for simple things like # gzip'd or compressed files. - ctype, encoding = mimetypes.guess_type(path) + ctype, encoding = mimetypes.guess_file_type(path) if ctype is None or encoding is not None: # No guess could be made, or the file is encoded (compressed), so # use a generic bag-of-bits type. diff --git a/Doc/includes/newtypes/custom2.c b/Doc/includes/newtypes/custom2.c index a0222b1795209b..768ce29fab9ff0 100644 --- a/Doc/includes/newtypes/custom2.c +++ b/Doc/includes/newtypes/custom2.c @@ -23,12 +23,12 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) CustomObject *self; self = (CustomObject *) type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->last == NULL) { Py_DECREF(self); return NULL; diff --git a/Doc/includes/newtypes/custom3.c b/Doc/includes/newtypes/custom3.c index 4aeebe0a7507d1..7d969adfa7c9cc 100644 --- a/Doc/includes/newtypes/custom3.c +++ b/Doc/includes/newtypes/custom3.c @@ -23,12 +23,12 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) CustomObject *self; self = (CustomObject *) type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->last == NULL) { Py_DECREF(self); return NULL; diff --git a/Doc/includes/newtypes/custom4.c b/Doc/includes/newtypes/custom4.c index 3998918f68301e..a7b8de44a57c90 100644 --- a/Doc/includes/newtypes/custom4.c +++ b/Doc/includes/newtypes/custom4.c @@ -39,12 +39,12 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) CustomObject *self; self = (CustomObject *) type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->last == NULL) { Py_DECREF(self); return NULL; diff --git a/Doc/includes/wasm-mobile-notavail.rst b/Doc/includes/wasm-mobile-notavail.rst new file mode 100644 index 00000000000000..725b0f7a86d17d --- /dev/null +++ b/Doc/includes/wasm-mobile-notavail.rst @@ -0,0 +1,6 @@ +.. include for modules that don't work on WASM or mobile platforms + +.. availability:: not Android, not iOS, not WASI. + + This module is not supported on :ref:`mobile platforms ` + or :ref:`WebAssembly platforms `. diff --git a/Doc/includes/wasm-notavail.rst b/Doc/includes/wasm-notavail.rst index e680e1f9b43807..c1b79d2a4a0508 100644 --- a/Doc/includes/wasm-notavail.rst +++ b/Doc/includes/wasm-notavail.rst @@ -1,7 +1,6 @@ .. include for modules that don't work on WASM -.. availability:: not Emscripten, not WASI. +.. availability:: not WASI. - This module does not work or is not available on WebAssembly platforms - ``wasm32-emscripten`` and ``wasm32-wasi``. See + This module does not work or is not available on WebAssembly. See :ref:`wasm-availability` for more information. diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 762f8b4695b3dd..4f3b663006fb28 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -1,5 +1,5 @@ -:mod:`__future__` --- Future statement definitions -================================================== +:mod:`!__future__` --- Future statement definitions +=================================================== .. module:: __future__ :synopsis: Future statement definitions @@ -64,8 +64,10 @@ language using this mechanism: | generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | | | | | *StopIteration handling inside generators* | +------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations* | +| annotations | 3.7.0b1 | Never [1]_ | :pep:`563`: | +| | | | *Postponed evaluation of annotations*, | +| | | | :pep:`649`: *Deferred evaluation of | +| | | | annotations using descriptors* | +------------------+-------------+--------------+---------------------------------------------+ .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. @@ -115,11 +117,9 @@ language using this mechanism: .. [1] ``from __future__ import annotations`` was previously scheduled to - become mandatory in Python 3.10, but the Python Steering Council - twice decided to delay the change - (`announcement for Python 3.10 `__; - `announcement for Python 3.11 `__). - No final decision has been made yet. See also :pep:`563` and :pep:`649`. + become mandatory in Python 3.10, but the change was delayed and ultimately + canceled. This feature will eventually be deprecated and removed. See + :pep:`649` and :pep:`749`. .. seealso:: diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index c999253f781b10..647ff9da04d10d 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -1,5 +1,5 @@ -:mod:`__main__` --- Top-level code environment -============================================== +:mod:`!__main__` --- Top-level code environment +=============================================== .. module:: __main__ :synopsis: The environment where top-level code is run. Covers command-line @@ -251,9 +251,9 @@ attribute will include the package's path if imported:: >>> asyncio.__main__.__name__ 'asyncio.__main__' -This won't work for ``__main__.py`` files in the root directory of a .zip file -though. Hence, for consistency, minimal ``__main__.py`` like the :mod:`venv` -one mentioned below are preferred. +This won't work for ``__main__.py`` files in the root directory of a +``.zip`` file though. Hence, for consistency, a minimal ``__main__.py`` +without a ``__name__`` check is preferred. .. seealso:: diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 297f50a46e0692..ed29ac70035597 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -1,5 +1,5 @@ -:mod:`_thread` --- Low-level threading API -========================================== +:mod:`!_thread` --- Low-level threading API +=========================================== .. module:: _thread :synopsis: Low-level threading API. @@ -169,14 +169,14 @@ Lock objects have the following methods: time can acquire a lock --- that's their reason for existence). If the *blocking* argument is present, the action depends on its - value: if it is False, the lock is only acquired if it can be acquired - immediately without waiting, while if it is True, the lock is acquired + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired unconditionally as above. If the floating-point *timeout* argument is present and positive, it specifies the maximum wait time in seconds before returning. A negative *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is False. + a *timeout* if *blocking* is false. The return value is ``True`` if the lock is acquired successfully, ``False`` if not. @@ -187,6 +187,9 @@ Lock objects have the following methods: .. versionchanged:: 3.2 Lock acquires can now be interrupted by signals on POSIX. + .. versionchanged:: 3.14 + Lock acquires can now be interrupted by signals on Windows. + .. method:: lock.release() @@ -213,23 +216,14 @@ In addition to these methods, lock objects can also be used via the .. index:: pair: module; signal -* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` - exception will be received by an arbitrary thread. (When the :mod:`signal` - module is available, interrupts always go to the main thread.) +* Interrupts always go to the main thread (the :exc:`KeyboardInterrupt` + exception will be received by that thread.) * Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is equivalent to calling :func:`_thread.exit`. -* It is not possible to interrupt the :meth:`~threading.Lock.acquire` method on - a lock --- the :exc:`KeyboardInterrupt` exception will happen after the lock - has been acquired. - * When the main thread exits, it is system defined whether the other threads survive. On most systems, they are killed without executing :keyword:`try` ... :keyword:`finally` clauses or executing object destructors. -* When the main thread exits, it does not do any of its usual cleanup (except - that :keyword:`try` ... :keyword:`finally` clauses are honored), and the - standard I/O files are not flushed. - diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index c073ea955abaa4..38d744e97d087d 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -1,5 +1,5 @@ -:mod:`abc` --- Abstract Base Classes -==================================== +:mod:`!abc` --- Abstract Base Classes +===================================== .. module:: abc :synopsis: Abstract base classes according to :pep:`3119`. @@ -99,13 +99,13 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: that you can customize the behavior of :func:`issubclass` further without the need to call :meth:`register` on every class you want to consider a subclass of the ABC. (This class method is called from the - :meth:`~class.__subclasscheck__` method of the ABC.) + :meth:`~type.__subclasscheck__` method of the ABC.) - This method should return ``True``, ``False`` or ``NotImplemented``. If + This method should return ``True``, ``False`` or :data:`NotImplemented`. If it returns ``True``, the *subclass* is considered a subclass of this ABC. If it returns ``False``, the *subclass* is not considered a subclass of this ABC, even if it would normally be one. If it returns - ``NotImplemented``, the subclass check is continued with the usual + :data:`!NotImplemented`, the subclass check is continued with the usual mechanism. .. XXX explain the "usual mechanism" @@ -149,7 +149,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: The :meth:`__subclasshook__` class method defined here says that any class that has an :meth:`~iterator.__iter__` method in its :attr:`~object.__dict__` (or in that of one of its base classes, accessed - via the :attr:`~class.__mro__` list) is considered a ``MyIterable`` too. + via the :attr:`~type.__mro__` list) is considered a ``MyIterable`` too. Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, even though it does not define an :meth:`~iterator.__iter__` method (it uses diff --git a/Doc/library/allos.rst b/Doc/library/allos.rst index f7105d8af8e28b..0223c1054ea5d8 100644 --- a/Doc/library/allos.rst +++ b/Doc/library/allos.rst @@ -16,7 +16,6 @@ but they are available on most other systems as well. Here's an overview: io.rst time.rst argparse.rst - getopt.rst logging.rst logging.config.rst logging.handlers.rst diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst new file mode 100644 index 00000000000000..37490456d13312 --- /dev/null +++ b/Doc/library/annotationlib.rst @@ -0,0 +1,385 @@ +:mod:`!annotationlib` --- Functionality for introspecting annotations +===================================================================== + +.. module:: annotationlib + :synopsis: Functionality for introspecting annotations + + +**Source code:** :source:`Lib/annotationlib.py` + +.. testsetup:: default + + import annotationlib + from annotationlib import * + +-------------- + +The :mod:`!annotationlib` module provides tools for introspecting +:term:`annotations ` on modules, classes, and functions. + +Annotations are :ref:`lazily evaluated ` and often contain +forward references to objects that are not yet defined when the annotation +is created. This module provides a set of low-level tools that can be used to retrieve annotations in a reliable way, even +in the presence of forward references and other edge cases. + +This module supports retrieving annotations in three main formats +(see :class:`Format`), each of which works best for different use cases: + +* :attr:`~Format.VALUE` evaluates the annotations and returns their value. + This is most straightforward to work with, but it may raise errors, + for example if the annotations contain references to undefined names. +* :attr:`~Format.FORWARDREF` returns :class:`ForwardRef` objects + for annotations that cannot be resolved, allowing you to inspect the + annotations without evaluating them. This is useful when you need to + work with annotations that may contain unresolved forward references. +* :attr:`~Format.STRING` returns the annotations as a string, similar + to how it would appear in the source file. This is useful for documentation + generators that want to display annotations in a readable way. + +The :func:`get_annotations` function is the main entry point for +retrieving annotations. Given a function, class, or module, it returns +an annotations dictionary in the requested format. This module also provides +functionality for working directly with the :term:`annotate function` +that is used to evaluate annotations, such as :func:`get_annotate_function` +and :func:`call_annotate_function`, as well as the +:func:`call_evaluate_function` function for working with +:term:`evaluate functions `. + + +.. seealso:: + + :pep:`649` proposed the current model for how annotations work in Python. + + :pep:`749` expanded on various aspects of :pep:`649` and introduced the + :mod:`!annotationlib` module. + + :ref:`annotations-howto` provides best practices for working with + annotations. + + :pypi:`typing-extensions` provides a backport of :func:`get_annotations` + that works on earlier versions of Python. + +Annotation semantics +-------------------- + +The way annotations are evaluated has changed over the history of Python 3, +and currently still depends on a :ref:`future import `. +There have been execution models for annotations: + +* *Stock semantics* (default in Python 3.0 through 3.13; see :pep:`3107` + and :pep:`526`): Annotations are evaluated eagerly, as they are + encountered in the source code. +* *Stringified annotations* (used with ``from __future__ import annotations`` + in Python 3.7 and newer; see :pep:`563`): Annotations are stored as + strings only. +* *Deferred evaluation* (default in Python 3.14 and newer; see :pep:`649` and + :pep:`749`): Annotations are evaluated lazily, only when they are accessed. + +As an example, consider the following program:: + + def func(a: Cls) -> None: + print(a) + + class Cls: pass + + print(func.__annotations__) + +This will behave as follows: + +* Under stock semantics (Python 3.13 and earlier), it will throw a + :exc:`NameError` at the line where ``func`` is defined, + because ``Cls`` is an undefined name at that point. +* Under stringified annotations (if ``from __future__ import annotations`` + is used), it will print ``{'a': 'Cls', 'return': 'None'}``. +* Under deferred evaluation (Python 3.14 and later), it will print + ``{'a': , 'return': None}``. + +Stock semantics were used when function annotations were first introduced +in Python 3.0 (by :pep:`3107`) because this was the simplest, most obvious +way to implement annotations. The same execution model was used when variable +annotations were introduced in Python 3.6 (by :pep:`526`). However, +stock semantics caused problems when using annotations as type hints, +such as a need to refer to names that are not yet defined when the +annotation is encountered. In addition, there were performance problems +with executing annotations at module import time. Therefore, in Python 3.7, +:pep:`563` introduced the ability to store annotations as strings using the +``from __future__ import annotations`` syntax. The plan at the time was to +eventually make this behavior the default, but a problem appeared: +stringified annotations are more difficult to process for those who +introspect annotations at runtime. An alternative proposal, :pep:`649`, +introduced the third execution model, deferred evaluation, and was implemented +in Python 3.14. Stringified annotations are still used if +``from __future__ import annotations`` is present, but this behavior will +eventually be removed. + +Classes +------- + +.. class:: Format + + An :class:`~enum.IntEnum` describing the formats in which annotations + can be returned. Members of the enum, or their equivalent integer values, + can be passed to :func:`get_annotations` and other functions in this + module, as well as to :attr:`~object.__annotate__` functions. + + .. attribute:: VALUE + :value: 1 + + Values are the result of evaluating the annotation expressions. + + .. attribute:: FORWARDREF + :value: 2 + + Values are real annotation values (as per :attr:`Format.VALUE` format) + for defined values, and :class:`ForwardRef` proxies for undefined + values. Real objects may contain references to, :class:`ForwardRef` + proxy objects. + + .. attribute:: STRING + :value: 3 + + Values are the text string of the annotation as it appears in the + source code, up to modifications including, but not restricted to, + whitespace normalizations and constant values optimizations. + + The exact values of these strings may change in future versions of Python. + + .. versionadded:: 3.14 + +.. class:: ForwardRef + + A proxy object for forward references in annotations. + + Instances of this class are returned when the :attr:`~Format.FORWARDREF` + format is used and annotations contain a name that cannot be resolved. + This can happen when a forward reference is used in an annotation, such as + when a class is referenced before it is defined. + + .. attribute:: __forward_arg__ + + A string containing the code that was evaluated to produce the + :class:`~ForwardRef`. The string may not be exactly equivalent + to the original source. + + .. method:: evaluate(*, globals=None, locals=None, type_params=None, owner=None) + + Evaluate the forward reference, returning its value. + + This may throw an exception, such as :exc:`NameError`, if the forward + reference refers to names that do not exist. The arguments to this + method can be used to provide bindings for names that would otherwise + be undefined. + + :class:`~ForwardRef` instances returned by :func:`get_annotations` + retain references to information about the scope they originated from, + so calling this method with no further arguments may be sufficient to + evaluate such objects. :class:`~ForwardRef` instances created by other + means may not have any information about their scope, so passing + arguments to this method may be necessary to evaluate them successfully. + + *globals* and *locals* are passed to :func:`eval`, representing + the global and local namespaces in which the name is evaluated. + *type_params*, if given, must be a tuple of + :ref:`type parameters ` that are in scope while the forward + reference is being evaluated. *owner* is the object that owns the + annotation from which the forward reference derives, usually a function, + class, or module. + + .. important:: + + Once a :class:`~ForwardRef` instance has been evaluated, it caches + the evaluated value, and future calls to :meth:`evaluate` will return + the cached value, regardless of the parameters passed in. + + .. versionadded:: 3.14 + + +Functions +--------- + +.. function:: annotations_to_string(annotations) + + Convert an annotations dict containing runtime values to a + dict containing only strings. If the values are not already strings, + they are converted using :func:`value_to_string`. + This is meant as a helper for user-provided + annotate functions that support the :attr:`~Format.STRING` format but + do not have access to the code creating the annotations. + + For example, this is used to implement the :attr:`~Format.STRING` for + :class:`typing.TypedDict` classes created through the functional syntax: + + .. doctest:: + + >>> from typing import TypedDict + >>> Movie = TypedDict("movie", {"name": str, "year": int}) + >>> get_annotations(Movie, format=Format.STRING) + {'name': 'str', 'year': 'int'} + + .. versionadded:: 3.14 + +.. function:: call_annotate_function(annotate, format, *, owner=None) + + Call the :term:`annotate function` *annotate* with the given *format*, + a member of the :class:`Format` enum, and return the annotations + dictionary produced by the function. + + This helper function is required because annotate functions generated by + the compiler for functions, classes, and modules only support + the :attr:`~Format.VALUE` format when called directly. + To support other formats, this function calls the annotate function + in a special environment that allows it to produce annotations in the + other formats. This is a useful building block when implementing + functionality that needs to partially evaluate annotations while a class + is being constructed. + + *owner* is the object that owns the annotation function, usually + a function, class, or module. If provided, it is used in the + :attr:`~Format.FORWARDREF` format to produce a :class:`ForwardRef` + object that carries more information. + + .. seealso:: + + :PEP:`PEP 649 <649#the-stringizer-and-the-fake-globals-environment>` + contains an explanation of the implementation technique used by this + function. + + .. versionadded:: 3.14 + +.. function:: call_evaluate_function(evaluate, format, *, owner=None) + + Call the :term:`evaluate function` *evaluate* with the given *format*, + a member of the :class:`Format` enum, and return the value produced by + the function. This is similar to :func:`call_annotate_function`, + but the latter always returns a dictionary mapping strings to annotations, + while this function returns a single value. + + This is intended for use with the evaluate functions generated for lazily + evaluated elements related to type aliases and type parameters: + + * :meth:`typing.TypeAliasType.evaluate_value`, the value of type aliases + * :meth:`typing.TypeVar.evaluate_bound`, the bound of type variables + * :meth:`typing.TypeVar.evaluate_constraints`, the constraints of + type variables + * :meth:`typing.TypeVar.evaluate_default`, the default value of + type variables + * :meth:`typing.ParamSpec.evaluate_default`, the default value of + parameter specifications + * :meth:`typing.TypeVarTuple.evaluate_default`, the default value of + type variable tuples + + *owner* is the object that owns the evaluate function, such as the type + alias or type variable object. + + *format* can be used to control the format in which the value is returned: + + .. doctest:: + + >>> type Alias = undefined + >>> call_evaluate_function(Alias.evaluate_value, Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'undefined' is not defined + >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) + ForwardRef('undefined') + >>> call_evaluate_function(Alias.evaluate_value, Format.STRING) + 'undefined' + + .. versionadded:: 3.14 + +.. function:: get_annotate_function(obj) + + Retrieve the :term:`annotate function` for *obj*. Return :const:`!None` + if *obj* does not have an annotate function. + + This is usually equivalent to accessing the :attr:`~object.__annotate__` + attribute of *obj*, but direct access to the attribute may return the wrong + object in certain situations involving metaclasses. This function should be + used instead of accessing the attribute directly. + + .. versionadded:: 3.14 + +.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE) + + Compute the annotations dict for an object. + + *obj* may be a callable, class, module, or other object with + :attr:`~object.__annotate__` and :attr:`~object.__annotations__` attributes. + Passing in an object of any other type raises :exc:`TypeError`. + + The *format* parameter controls the format in which annotations are returned, + and must be a member of the :class:`Format` enum or its integer equivalent. + + Returns a dict. :func:`!get_annotations` returns a new dict every time + it's called; calling it twice on the same object will return two + different but equivalent dicts. + + This function handles several details for you: + + * If *eval_str* is true, values of type :class:`!str` will + be un-stringized using :func:`eval`. This is intended + for use with stringized annotations + (``from __future__ import annotations``). It is an error + to set *eval_str* to true with formats other than :attr:`Format.VALUE`. + * If *obj* doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes, as well as annotations + on metaclasses. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using ``getattr()`` and ``dict.get()`` for safety. + + *eval_str* controls whether or not values of type :class:`!str` are + replaced with the result of calling :func:`eval` on those values: + + * If eval_str is true, :func:`eval` is called on values of type + :class:`!str`. (Note that :func:`!get_annotations` doesn't catch + exceptions; if :func:`eval` raises an exception, it will unwind + the stack past the :func:`!get_annotations` call.) + * If *eval_str* is false (the default), values of type :class:`!str` are + unchanged. + + *globals* and *locals* are passed in to :func:`eval`; see the documentation + for :func:`eval` for more information. If *globals* or *locals* + is :const:`!None`, this function may replace that value with a + context-specific default, contingent on ``type(obj)``: + + * If *obj* is a module, *globals* defaults to ``obj.__dict__``. + * If *obj* is a class, *globals* defaults to + ``sys.modules[obj.__module__].__dict__`` and *locals* defaults + to the *obj* class namespace. + * If *obj* is a callable, *globals* defaults to + :attr:`obj.__globals__ `, + although if *obj* is a wrapped function (using + :func:`functools.update_wrapper`) or a :class:`functools.partial` object, + it is unwrapped until a non-wrapped function is found. + + Calling :func:`!get_annotations` is best practice for accessing the + annotations dict of any object. See :ref:`annotations-howto` for + more information on annotations best practices. + + .. doctest:: + + >>> def f(a: int, b: str) -> float: + ... pass + >>> get_annotations(f) + {'a': , 'b': , 'return': } + + .. versionadded:: 3.14 + +.. function:: value_to_string(value) + + Convert an arbitrary Python value to a format suitable for use by the + :attr:`~Format.STRING` format. This calls :func:`repr` for most + objects, but has special handling for some objects, such as type objects. + + This is meant as a helper for user-provided + annotate functions that support the :attr:`~Format.STRING` format but + do not have access to the code creating the annotations. It can also + be used to provide a user-friendly string representation for other + objects that contain values that are commonly encountered in annotations. + + .. versionadded:: 3.14 + diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 952643a46416d2..7638798ca2552f 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1,5 +1,5 @@ -:mod:`argparse` --- Parser for command-line options, arguments and sub-commands -=============================================================================== +:mod:`!argparse` --- Parser for command-line options, arguments and subcommands +================================================================================ .. module:: argparse :synopsis: Command-line option and argument parsing library. @@ -19,17 +19,13 @@ introduction to Python command-line parsing, have a look at the :ref:`argparse tutorial `. -The :mod:`argparse` module makes it easy to write user-friendly command-line -interfaces. The program defines what arguments it requires, and :mod:`argparse` -will figure out how to parse those out of :data:`sys.argv`. The :mod:`argparse` +The :mod:`!argparse` module makes it easy to write user-friendly command-line +interfaces. The program defines what arguments it requires, and :mod:`!argparse` +will figure out how to parse those out of :data:`sys.argv`. The :mod:`!argparse` module also automatically generates help and usage messages. The module will also issue errors when users give the program invalid arguments. - -Core Functionality ------------------- - -The :mod:`argparse` module's support for command-line interfaces is built +The :mod:`!argparse` module's support for command-line interfaces is built around an instance of :class:`argparse.ArgumentParser`. It is a container for argument specifications and has options that apply to the parser as whole:: @@ -53,133 +49,9 @@ the extracted data in a :class:`argparse.Namespace` object:: args = parser.parse_args() print(args.filename, args.count, args.verbose) - -Quick Links for add_argument() ------------------------------- - -============================ =========================================================== ========================================================================================================================== -Name Description Values -============================ =========================================================== ========================================================================================================================== -action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'`` -choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance -const_ Store a constant value -default_ Default value used when an argument is not provided Defaults to ``None`` -dest_ Specify the attribute name used in the result namespace -help_ Help message for an argument -metavar_ Alternate display name for the argument as shown in help -nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, or ``'+'`` -required_ Indicate whether an argument is required or optional ``True`` or ``False`` -:ref:`type ` Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function -============================ =========================================================== ========================================================================================================================== - - -Example -------- - -The following code is a Python program that takes a list of integers and -produces either the sum or the max:: - - import argparse - - parser = argparse.ArgumentParser(description='Process some integers.') - parser.add_argument('integers', metavar='N', type=int, nargs='+', - help='an integer for the accumulator') - parser.add_argument('--sum', dest='accumulate', action='store_const', - const=sum, default=max, - help='sum the integers (default: find the max)') - - args = parser.parse_args() - print(args.accumulate(args.integers)) - -Assuming the above Python code is saved into a file called ``prog.py``, it can -be run at the command line and it provides useful help messages: - -.. code-block:: shell-session - - $ python prog.py -h - usage: prog.py [-h] [--sum] N [N ...] - - Process some integers. - - positional arguments: - N an integer for the accumulator - - options: - -h, --help show this help message and exit - --sum sum the integers (default: find the max) - -When run with the appropriate arguments, it prints either the sum or the max of -the command-line integers: - -.. code-block:: shell-session - - $ python prog.py 1 2 3 4 - 4 - - $ python prog.py 1 2 3 4 --sum - 10 - -If invalid arguments are passed in, an error will be displayed: - -.. code-block:: shell-session - - $ python prog.py a b c - usage: prog.py [-h] [--sum] N [N ...] - prog.py: error: argument N: invalid int value: 'a' - -The following sections walk you through this example. - - -Creating a parser -^^^^^^^^^^^^^^^^^ - -The first step in using the :mod:`argparse` is creating an -:class:`ArgumentParser` object:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.') - -The :class:`ArgumentParser` object will hold all the information necessary to -parse the command line into Python data types. - - -Adding arguments -^^^^^^^^^^^^^^^^ - -Filling an :class:`ArgumentParser` with information about program arguments is -done by making calls to the :meth:`~ArgumentParser.add_argument` method. -Generally, these calls tell the :class:`ArgumentParser` how to take the strings -on the command line and turn them into objects. This information is stored and -used when :meth:`~ArgumentParser.parse_args` is called. For example:: - - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.add_argument('--sum', dest='accumulate', action='store_const', - ... const=sum, default=max, - ... help='sum the integers (default: find the max)') - -Later, calling :meth:`~ArgumentParser.parse_args` will return an object with -two attributes, ``integers`` and ``accumulate``. The ``integers`` attribute -will be a list of one or more integers, and the ``accumulate`` attribute will be -either the :func:`sum` function, if ``--sum`` was specified at the command line, -or the :func:`max` function if it was not. - - -Parsing arguments -^^^^^^^^^^^^^^^^^ - -:class:`ArgumentParser` parses arguments through the -:meth:`~ArgumentParser.parse_args` method. This will inspect the command line, -convert each argument to the appropriate type and then invoke the appropriate action. -In most cases, this means a simple :class:`Namespace` object will be built up from -attributes parsed out of the command line:: - - >>> parser.parse_args(['--sum', '7', '-1', '42']) - Namespace(accumulate=, integers=[7, -1, 42]) - -In a script, :meth:`~ArgumentParser.parse_args` will typically be called with no -arguments, and the :class:`ArgumentParser` will automatically determine the -command-line arguments from :data:`sys.argv`. - +.. note:: + If you're looking for a guide about how to upgrade :mod:`optparse` code + to :mod:`!argparse`, see :ref:`Upgrading Optparse Code `. ArgumentParser objects ---------------------- @@ -189,14 +61,15 @@ ArgumentParser objects formatter_class=argparse.HelpFormatter, \ prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ - add_help=True, allow_abbrev=True, exit_on_error=True) + add_help=True, allow_abbrev=True, exit_on_error=True, \ + suggest_on_error=False) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description below, but in short they are: - * prog_ - The name of the program (default: - ``os.path.basename(sys.argv[0])``) + * prog_ - The name of the program (default: generated from the ``__main__`` + module attributes and ``sys.argv[0]``) * usage_ - The string describing the program usage (default: generated from arguments added to parser) @@ -228,9 +101,13 @@ ArgumentParser objects * allow_abbrev_ - Allows long options to be abbreviated if the abbreviation is unambiguous. (default: ``True``) - * exit_on_error_ - Determines whether or not ArgumentParser exits with + * exit_on_error_ - Determines whether or not :class:`!ArgumentParser` exits with error info when an error occurs. (default: ``True``) + * suggest_on_error_ - Enables suggestions for mistyped argument choices + and subparser names (default: ``False``) + + .. versionchanged:: 3.5 *allow_abbrev* parameter was added. @@ -249,38 +126,21 @@ The following sections describe how each of these are used. prog ^^^^ -By default, :class:`ArgumentParser` objects use ``sys.argv[0]`` to determine -how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match how the program was -invoked on the command line. For example, consider a file named -``myprogram.py`` with the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() -The help for this program will display ``myprogram.py`` as the program name -(regardless of where the program was invoked from): +By default, :class:`ArgumentParser` calculates the name of the program +to display in help messages depending on the way the Python interpreter was run: -.. code-block:: shell-session +* The :func:`base name ` of ``sys.argv[0]`` if a file was + passed as argument. +* The Python interpreter name followed by ``sys.argv[0]`` if a directory or + a zipfile was passed as argument. +* The Python interpreter name followed by ``-m`` followed by the + module or package name if the :option:`-m` option was used. - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help - $ cd .. - $ python subdir/myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help - -To change this default behavior, another value can be supplied using the -``prog=`` argument to :class:`ArgumentParser`:: +This default is almost always desirable because it will make the help messages +match the string that was used to invoke the program on the command line. +However, to change this default behavior, another value can be supplied using +the ``prog=`` argument to :class:`ArgumentParser`:: >>> parser = argparse.ArgumentParser(prog='myprogram') >>> parser.print_help() @@ -289,7 +149,8 @@ To change this default behavior, another value can be supplied using the options: -h, --help show this help message and exit -Note that the program name, whether determined from ``sys.argv[0]`` or from the +Note that the program name, whether determined from ``sys.argv[0]``, +from the ``__main__`` module attributes or from the ``prog=`` argument, is available to help messages using the ``%(prog)s`` format specifier. @@ -304,27 +165,16 @@ specifier. -h, --help show this help message and exit --foo FOO foo of the myprogram program +.. versionchanged:: 3.14 + The default ``prog`` value now reflects how ``__main__`` was actually executed, + rather than always being ``os.path.basename(sys.argv[0])``. usage ^^^^^ By default, :class:`ArgumentParser` calculates the usage message from the -arguments it contains:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') - >>> parser.print_help() - usage: PROG [-h] [--foo [FOO]] bar [bar ...] - - positional arguments: - bar bar help - - options: - -h, --help show this help message and exit - --foo [FOO] foo help - -The default message can be overridden with the ``usage=`` keyword argument:: +arguments it contains. The default message can be overridden with the +``usage=`` keyword argument:: >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') >>> parser.add_argument('--foo', nargs='?', help='foo help') @@ -352,16 +202,7 @@ Most calls to the :class:`ArgumentParser` constructor will use the ``description=`` keyword argument. This argument gives a brief description of what the program does and how it works. In help messages, the description is displayed between the command-line usage string and the help messages for the -various arguments:: - - >>> parser = argparse.ArgumentParser(description='A foo that bars') - >>> parser.print_help() - usage: argparse.py [-h] - - A foo that bars - - options: - -h, --help show this help message and exit +various arguments. By default, the description will be line-wrapped so that it fits within the given space. To change this behavior, see the formatter_class_ argument. @@ -491,7 +332,7 @@ should not be line-wrapped:: -h, --help show this help message and exit :class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text, -including argument descriptions. However, multiple new lines are replaced with +including argument descriptions. However, multiple newlines are replaced with one. If you wish to preserve multiple blank lines, add spaces between the newlines. @@ -540,7 +381,7 @@ Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. Parsers that need to support different or additional prefix characters, e.g. for options like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument -to the ArgumentParser constructor:: +to the :class:`ArgumentParser` constructor:: >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') >>> parser.add_argument('+f') @@ -585,8 +426,8 @@ arguments will never be treated as file references. .. versionchanged:: 3.12 :class:`ArgumentParser` changed encoding and errors to read arguments files - from default (e.g. :func:`locale.getpreferredencoding(False) ` and - ``"strict"``) to :term:`filesystem encoding and error handler`. + from default (e.g. :func:`locale.getpreferredencoding(False) ` + and ``"strict"``) to the :term:`filesystem encoding and error handler`. Arguments file should be encoded in UTF-8 instead of ANSI Codepage on Windows. @@ -671,26 +512,9 @@ string was overridden. add_help ^^^^^^^^ -By default, ArgumentParser objects add an option which simply displays -the parser's help message. For example, consider a file named -``myprogram.py`` containing the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser -help will be printed: - -.. code-block:: shell-session - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help +By default, :class:`ArgumentParser` objects add an option which simply displays +the parser's help message. If ``-h`` or ``--help`` is supplied at the command +line, the :class:`!ArgumentParser` help will be printed. Occasionally, it may be useful to disable the addition of this help option. This can be achieved by passing ``False`` as the ``add_help=`` argument to @@ -722,7 +546,8 @@ exit_on_error ^^^^^^^^^^^^^ Normally, when you pass an invalid argument list to the :meth:`~ArgumentParser.parse_args` -method of an :class:`ArgumentParser`, it will exit with error info. +method of an :class:`ArgumentParser`, it will print a *message* to :data:`sys.stderr` and exit with a status +code of 2. If the user would like to catch errors manually, the feature can be enabled by setting ``exit_on_error`` to ``False``:: @@ -739,19 +564,40 @@ If the user would like to catch errors manually, the feature can be enabled by s .. versionadded:: 3.9 +suggest_on_error +^^^^^^^^^^^^^^^^ + +By default, when a user passes an invalid argument choice or subparser name, +:class:`ArgumentParser` will exit with error info and list the permissible +argument choices (if specified) or subparser names as part of the error message. + +If the user would like to enable suggestions for mistyped argument choices and +subparser names, the feature can be enabled by setting ``suggest_on_error`` to +``True``. Note that this only applies for arguments when the choices specified +are strings:: + + >>> parser = argparse.ArgumentParser(description='Process some integers.', suggest_on_error=True) + >>> parser.add_argument('--action', choices=['sum', 'max']) + >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', + ... help='an integer for the accumulator') + >>> parser.parse_args(['--action', 'sumn', 1, 2, 3]) + tester.py: error: argument --action: invalid choice: 'sumn', maybe you meant 'sum'? (choose from 'sum', 'max') + +.. versionadded:: 3.14 + The add_argument() method ------------------------- -.. method:: ArgumentParser.add_argument(name or flags..., [action], [nargs], \ +.. method:: ArgumentParser.add_argument(name or flags..., *, [action], [nargs], \ [const], [default], [type], [choices], [required], \ - [help], [metavar], [dest]) + [help], [metavar], [dest], [deprecated]) Define how a single command-line argument should be parsed. Each parameter has its own more detailed description below, but in short they are: - * `name or flags`_ - Either a name or a list of option strings, e.g. ``foo`` - or ``-f, --foo``. + * `name or flags`_ - Either a name or a list of option strings, e.g. ``'foo'`` + or ``'-f', '--foo'``. * action_ - The basic type of action to be taken when this argument is encountered at the command line. @@ -782,7 +628,7 @@ The add_argument() method The following sections describe how each of these are used. -.. _name_or_flags: +.. _`name or flags`: name or flags ^^^^^^^^^^^^^ @@ -816,6 +662,25 @@ be positional:: usage: PROG [-h] [-f FOO] bar PROG: error: the following arguments are required: bar +By default, :mod:`!argparse` automatically handles the internal naming and +display names of arguments, simplifying the process without requiring +additional configuration. +As such, you do not need to specify the dest_ and metavar_ parameters. +The dest_ parameter defaults to the argument name with underscores ``_`` +replacing hyphens ``-`` . The metavar_ parameter defaults to the +upper-cased name. For example:: + + >>> parser = argparse.ArgumentParser(prog='PROG') + >>> parser.add_argument('--foo-bar') + >>> parser.parse_args(['--foo-bar', 'FOO-BAR'] + Namespace(foo_bar='FOO-BAR') + >>> parser.print_help() + usage: [-h] [--foo-bar FOO-BAR] + + optional arguments: + -h, --help show this help message and exit + --foo-bar FOO-BAR + .. _action: @@ -829,12 +694,7 @@ them, though most actions simply add an attribute to the object returned by how the command-line arguments should be handled. The supplied actions are: * ``'store'`` - This just stores the argument's value. This is the default - action. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args('--foo 1'.split()) - Namespace(foo='1') + action. * ``'store_const'`` - This stores the value specified by the const_ keyword argument; note that the const_ keyword argument defaults to ``None``. The @@ -849,7 +709,7 @@ how the command-line arguments should be handled. The supplied actions are: * ``'store_true'`` and ``'store_false'`` - These are special cases of ``'store_const'`` used for storing the values ``True`` and ``False`` respectively. In addition, they create default values of ``False`` and - ``True`` respectively. For example:: + ``True`` respectively:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', action='store_true') @@ -881,6 +741,21 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--str --int'.split()) Namespace(types=[, ]) +* ``'extend'`` - This stores a list and appends each item from the multi-value + argument list to it. + The ``'extend'`` action is typically used with the nargs_ keyword argument + value ``'+'`` or ``'*'``. + Note that when nargs_ is ``None`` (the default) or ``'?'``, each + character of the argument string will be appended to the list. + Example usage:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument("--foo", action="extend", nargs="+", type=str) + >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"]) + Namespace(foo=['f1', 'f2', 'f3', 'f4']) + + .. versionadded:: 3.8 + * ``'count'`` - This counts the number of times a keyword argument occurs. For example, this is useful for increasing verbosity levels:: @@ -906,33 +781,27 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 -* ``'extend'`` - This stores a list, and extends each argument value to the - list. - Example usage:: +Only actions that consume command-line arguments (e.g. ``'store'``, +``'append'`` or ``'extend'``) can be used with positional arguments. - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument("--foo", action="extend", nargs="+", type=str) - >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"]) - Namespace(foo=['f1', 'f2', 'f3', 'f4']) +.. class:: BooleanOptionalAction - .. versionadded:: 3.8 + You may also specify an arbitrary action by passing an :class:`Action` subclass or + other object that implements the same interface. The :class:`!BooleanOptionalAction` + is available in :mod:`!argparse` and adds support for boolean actions such as + ``--foo`` and ``--no-foo``:: -You may also specify an arbitrary action by passing an Action subclass or -other object that implements the same interface. The ``BooleanOptionalAction`` -is available in ``argparse`` and adds support for boolean actions such as -``--foo`` and ``--no-foo``:: + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) + >>> parser.parse_args(['--no-foo']) + Namespace(foo=False) - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) - >>> parser.parse_args(['--no-foo']) - Namespace(foo=False) - -.. versionadded:: 3.9 + .. versionadded:: 3.9 The recommended way to create a custom action is to extend :class:`Action`, -overriding the ``__call__`` method and optionally the ``__init__`` and -``format_usage`` methods. +overriding the :meth:`!__call__` method and optionally the :meth:`!__init__` and +:meth:`!format_usage` methods. An example of a custom action:: @@ -962,7 +831,7 @@ For more details, see :class:`Action`. nargs ^^^^^ -ArgumentParser objects usually associate a single command-line argument with a +:class:`ArgumentParser` objects usually associate a single command-line argument with a single action to be taken. The ``nargs`` keyword argument associates a different number of command-line arguments with a single action. See also :ref:`specifying-ambiguous-arguments`. The supported values are: @@ -1002,16 +871,14 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are: output files:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), - ... default=sys.stdin) - >>> parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), - ... default=sys.stdout) + >>> parser.add_argument('infile', nargs='?') + >>> parser.add_argument('outfile', nargs='?') >>> parser.parse_args(['input.txt', 'output.txt']) - Namespace(infile=<_io.TextIOWrapper name='input.txt' encoding='UTF-8'>, - outfile=<_io.TextIOWrapper name='output.txt' encoding='UTF-8'>) + Namespace(infile='input.txt', outfile='output.txt') + >>> parser.parse_args(['input.txt']) + Namespace(infile='input.txt', outfile=None) >>> parser.parse_args([]) - Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>, - outfile=<_io.TextIOWrapper name='' encoding='UTF-8'>) + Namespace(infile=None, outfile=None) .. index:: single: * (asterisk); in argparse module @@ -1044,6 +911,8 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are: If the ``nargs`` keyword argument is not provided, the number of arguments consumed is determined by the action_. Generally this means a single command-line argument will be consumed and a single item (not a list) will be produced. +Actions that do not consume command-line arguments (e.g. +``'store_const'``) set ``nargs=0``. .. _const: @@ -1094,7 +963,7 @@ was not present at the command line:: Namespace(foo=42) If the target namespace already has an attribute set, the action *default* -will not over write it:: +will not overwrite it:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default=42) @@ -1122,6 +991,9 @@ is used when no command-line argument was present:: >>> parser.parse_args([]) Namespace(foo=42) +For required_ arguments, the ``default`` value is ignored. For example, this +applies to positional arguments with nargs_ values other than ``?`` or ``*``, +or optional arguments marked as ``required=True``. Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the command-line argument was not present:: @@ -1165,8 +1037,6 @@ Common built-in types and functions can be used as type converters: parser.add_argument('distance', type=float) parser.add_argument('street', type=ascii) parser.add_argument('code_point', type=ord) - parser.add_argument('source_file', type=open) - parser.add_argument('dest_file', type=argparse.FileType('w', encoding='latin-1')) parser.add_argument('datapath', type=pathlib.Path) User defined functions can be used as well: @@ -1196,10 +1066,11 @@ better reporting than can be given by the ``type`` keyword. A :exc:`FileNotFoundError` exception would not be handled at all. Even :class:`~argparse.FileType` has its limitations for use with the ``type`` -keyword. If one argument uses *FileType* and then a subsequent argument fails, -an error is reported but the file is not automatically closed. In this case, it -would be better to wait until after the parser has run and then use the -:keyword:`with`-statement to manage the files. +keyword. If one argument uses :class:`~argparse.FileType` and then a +subsequent argument fails, an error is reported but the file is not +automatically closed. In this case, it would be better to wait until after +the parser has run and then use the :keyword:`with`-statement to manage the +files. For type checkers that simply check against a fixed set of values, consider using the choices_ keyword instead. @@ -1227,15 +1098,7 @@ if the argument was not one of the acceptable values:: Note that inclusion in the *choices* sequence is checked after any type_ conversions have been performed, so the type of the objects in the *choices* -sequence should match the type_ specified:: - - >>> parser = argparse.ArgumentParser(prog='doors.py') - >>> parser.add_argument('door', type=int, choices=range(1, 4)) - >>> print(parser.parse_args(['3'])) - Namespace(door=3) - >>> parser.parse_args(['4']) - usage: doors.py [-h] {1,2,3} - doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) +sequence should match the type_ specified. Any sequence can be passed as the *choices* value, so :class:`list` objects, :class:`tuple` objects, and custom sequences are all supported. @@ -1254,7 +1117,7 @@ many choices), just specify an explicit metavar_. required ^^^^^^^^ -In general, the :mod:`argparse` module assumes that flags like ``-f`` and ``--bar`` +In general, the :mod:`!argparse` module assumes that flags like ``-f`` and ``--bar`` indicate *optional* arguments, which can always be omitted at the command line. To make an option *required*, ``True`` can be specified for the ``required=`` keyword argument to :meth:`~ArgumentParser.add_argument`:: @@ -1285,22 +1148,7 @@ help The ``help`` value is a string containing a brief description of the argument. When a user requests help (usually by using ``-h`` or ``--help`` at the command line), these ``help`` descriptions will be displayed with each -argument:: - - >>> parser = argparse.ArgumentParser(prog='frobble') - >>> parser.add_argument('--foo', action='store_true', - ... help='foo the bars before frobbling') - >>> parser.add_argument('bar', nargs='+', - ... help='one of the bars to be frobbled') - >>> parser.parse_args(['-h']) - usage: frobble [-h] [--foo] bar [bar ...] - - positional arguments: - bar one of the bars to be frobbled - - options: - -h, --help show this help message and exit - --foo foo the bars before frobbling +argument. The ``help`` strings can include various format specifiers to avoid repetition of things like the program name or the argument default_. The available @@ -1322,7 +1170,7 @@ specifiers include the program name, ``%(prog)s`` and most keyword arguments to As the help string supports %-formatting, if you want a literal ``%`` to appear in the help string, you must escape it as ``%%``. -:mod:`argparse` supports silencing the help entry for certain options, by +:mod:`!argparse` supports silencing the help entry for certain options, by setting the ``help`` value to ``argparse.SUPPRESS``:: >>> parser = argparse.ArgumentParser(prog='frobble') @@ -1340,7 +1188,7 @@ metavar ^^^^^^^ When :class:`ArgumentParser` generates help messages, it needs some way to refer -to each expected argument. By default, ArgumentParser objects use the dest_ +to each expected argument. By default, :class:`!ArgumentParser` objects use the dest_ value as the "name" of each object. By default, for positional argument actions, the dest_ value is used directly, and for optional argument actions, the dest_ value is uppercased. So, a single positional argument with @@ -1455,7 +1303,7 @@ The ``deprecated`` keyword argument of specifies if the argument is deprecated and will be removed in the future. For arguments, if ``deprecated`` is ``True``, then a warning will be -printed to standard error when the argument is used:: +printed to :data:`sys.stderr` when the argument is used:: >>> import argparse >>> parser = argparse.ArgumentParser(prog='snake.py') @@ -1466,13 +1314,13 @@ printed to standard error when the argument is used:: snake.py: warning: option '--legs' is deprecated Namespace(legs=4) -.. versionchanged:: 3.13 +.. versionadded:: 3.13 Action classes ^^^^^^^^^^^^^^ -Action classes implement the Action API, a callable which returns a callable +:class:`!Action` classes implement the Action API, a callable which returns a callable which processes arguments from the command-line. Any object which follows this API may be passed as the ``action`` parameter to :meth:`~ArgumentParser.add_argument`. @@ -1481,40 +1329,46 @@ this API may be passed as the ``action`` parameter to type=None, choices=None, required=False, help=None, \ metavar=None) -Action objects are used by an ArgumentParser to represent the information -needed to parse a single argument from one or more strings from the -command line. The Action class must accept the two positional arguments -plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` -except for the ``action`` itself. + :class:`!Action` objects are used by an :class:`ArgumentParser` to represent the information + needed to parse a single argument from one or more strings from the + command line. The :class:`!Action` class must accept the two positional arguments + plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` + except for the ``action`` itself. + + Instances of :class:`!Action` (or return value of any callable to the + ``action`` parameter) should have attributes :attr:`!dest`, + :attr:`!option_strings`, :attr:`!default`, :attr:`!type`, :attr:`!required`, + :attr:`!help`, etc. defined. The easiest way to ensure these attributes + are defined is to call :meth:`!Action.__init__`. -Instances of Action (or return value of any callable to the ``action`` -parameter) should have attributes "dest", "option_strings", "default", "type", -"required", "help", etc. defined. The easiest way to ensure these attributes -are defined is to call ``Action.__init__``. + .. method:: __call__(parser, namespace, values, option_string=None) -Action instances should be callable, so subclasses must override the -``__call__`` method, which should accept four parameters: + :class:`!Action` instances should be callable, so subclasses must override the + :meth:`!__call__` method, which should accept four parameters: -* ``parser`` - The ArgumentParser object which contains this action. + * *parser* - The :class:`ArgumentParser` object which contains this action. -* ``namespace`` - The :class:`Namespace` object that will be returned by - :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this - object using :func:`setattr`. + * *namespace* - The :class:`Namespace` object that will be returned by + :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this + object using :func:`setattr`. -* ``values`` - The associated command-line arguments, with any type conversions - applied. Type conversions are specified with the type_ keyword argument to - :meth:`~ArgumentParser.add_argument`. + * *values* - The associated command-line arguments, with any type conversions + applied. Type conversions are specified with the type_ keyword argument to + :meth:`~ArgumentParser.add_argument`. -* ``option_string`` - The option string that was used to invoke this action. - The ``option_string`` argument is optional, and will be absent if the action - is associated with a positional argument. + * *option_string* - The option string that was used to invoke this action. + The ``option_string`` argument is optional, and will be absent if the action + is associated with a positional argument. -The ``__call__`` method may perform arbitrary actions, but will typically set -attributes on the ``namespace`` based on ``dest`` and ``values``. + The :meth:`!__call__` method may perform arbitrary actions, but will typically set + attributes on the ``namespace`` based on ``dest`` and ``values``. + + .. method:: format_usage() + + :class:`!Action` subclasses can define a :meth:`!format_usage` method that takes no argument + and return a string which will be used when printing the usage of the program. + If such method is not provided, a sensible default will be used. -Action subclasses can define a ``format_usage`` method that takes no argument -and return a string which will be used when printing the usage of the program. -If such method is not provided, a sensible default will be used. The parse_args() method ----------------------- @@ -1526,7 +1380,7 @@ The parse_args() method Previous calls to :meth:`add_argument` determine exactly what objects are created and how they are assigned. See the documentation for - :meth:`add_argument` for details. + :meth:`!add_argument` for details. * args_ - List of strings to parse. The default is taken from :data:`sys.argv`. @@ -1682,7 +1536,7 @@ This feature can be disabled by setting :ref:`allow_abbrev` to ``False``. Beyond ``sys.argv`` ^^^^^^^^^^^^^^^^^^^ -Sometimes it may be useful to have an ArgumentParser parse arguments other than those +Sometimes it may be useful to have an :class:`ArgumentParser` parse arguments other than those of :data:`sys.argv`. This can be accomplished by passing a list of strings to :meth:`~ArgumentParser.parse_args`. This is useful for testing at the interactive prompt:: @@ -1709,29 +1563,29 @@ The Namespace object Simple class used by default by :meth:`~ArgumentParser.parse_args` to create an object holding attributes and return it. -This class is deliberately simple, just an :class:`object` subclass with a -readable string representation. If you prefer to have dict-like view of the -attributes, you can use the standard Python idiom, :func:`vars`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> args = parser.parse_args(['--foo', 'BAR']) - >>> vars(args) - {'foo': 'BAR'} - -It may also be useful to have an :class:`ArgumentParser` assign attributes to an -already existing object, rather than a new :class:`Namespace` object. This can -be achieved by specifying the ``namespace=`` keyword argument:: + This class is deliberately simple, just an :class:`object` subclass with a + readable string representation. If you prefer to have dict-like view of the + attributes, you can use the standard Python idiom, :func:`vars`:: - >>> class C: - ... pass - ... - >>> c = C() - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c) - >>> c.foo - 'BAR' + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> args = parser.parse_args(['--foo', 'BAR']) + >>> vars(args) + {'foo': 'BAR'} + + It may also be useful to have an :class:`ArgumentParser` assign attributes to an + already existing object, rather than a new :class:`Namespace` object. This can + be achieved by specifying the ``namespace=`` keyword argument:: + + >>> class C: + ... pass + ... + >>> c = C() + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c) + >>> c.foo + 'BAR' Other utilities @@ -1740,38 +1594,38 @@ Other utilities Sub-commands ^^^^^^^^^^^^ -.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ +.. method:: ArgumentParser.add_subparsers(*, [title], [description], [prog], \ [parser_class], [action], \ - [option_strings], [dest], [required], \ + [dest], [required], \ [help], [metavar]) - Many programs split up their functionality into a number of sub-commands, - for example, the ``svn`` program can invoke sub-commands like ``svn + Many programs split up their functionality into a number of subcommands, + for example, the ``svn`` program can invoke subcommands like ``svn checkout``, ``svn update``, and ``svn commit``. Splitting up functionality this way can be a particularly good idea when a program performs several different functions which require different kinds of command-line arguments. - :class:`ArgumentParser` supports the creation of such sub-commands with the - :meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally + :class:`ArgumentParser` supports the creation of such subcommands with the + :meth:`!add_subparsers` method. The :meth:`!add_subparsers` method is normally called with no arguments and returns a special action object. This object has a single method, :meth:`~_SubParsersAction.add_parser`, which takes a - command name and any :class:`ArgumentParser` constructor arguments, and - returns an :class:`ArgumentParser` object that can be modified as usual. + command name and any :class:`!ArgumentParser` constructor arguments, and + returns an :class:`!ArgumentParser` object that can be modified as usual. Description of parameters: - * title - title for the sub-parser group in help output; by default + * *title* - title for the sub-parser group in help output; by default "subcommands" if description is provided, otherwise uses title for positional arguments - * description - description for the sub-parser group in help output, by + * *description* - description for the sub-parser group in help output, by default ``None`` - * prog - usage information that will be displayed with sub-command help, + * *prog* - usage information that will be displayed with sub-command help, by default the name of the program and any positional arguments before the subparser argument - * parser_class - class which will be used to create sub-parser instances, by - default the class of the current parser (e.g. ArgumentParser) + * *parser_class* - class which will be used to create sub-parser instances, by + default the class of the current parser (e.g. :class:`ArgumentParser`) * action_ - the basic type of action to be taken when this argument is encountered at the command line @@ -1784,15 +1638,15 @@ Sub-commands * help_ - help for sub-parser group in help output, by default ``None`` - * metavar_ - string presenting available sub-commands in help; by default it - is ``None`` and presents sub-commands in form {cmd1, cmd2, ..} + * metavar_ - string presenting available subcommands in help; by default it + is ``None`` and presents subcommands in form {cmd1, cmd2, ..} Some example usage:: >>> # create the top-level parser >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('--foo', action='store_true', help='foo help') - >>> subparsers = parser.add_subparsers(help='sub-command help') + >>> subparsers = parser.add_subparsers(help='subcommand help') >>> >>> # create the parser for the "a" command >>> parser_a = subparsers.add_parser('a', help='a help') @@ -1800,7 +1654,7 @@ Sub-commands >>> >>> # create the parser for the "b" command >>> parser_b = subparsers.add_parser('b', help='b help') - >>> parser_b.add_argument('--baz', choices='XYZ', help='baz help') + >>> parser_b.add_argument('--baz', choices=('X', 'Y', 'Z'), help='baz help') >>> >>> # parse some argument lists >>> parser.parse_args(['a', '12']) @@ -1827,7 +1681,7 @@ Sub-commands usage: PROG [-h] [--foo] {a,b} ... positional arguments: - {a,b} sub-command help + {a,b} subcommand help a a help b b help @@ -1898,12 +1752,12 @@ Sub-commands .. versionadded:: 3.13 - One particularly effective way of handling sub-commands is to combine the use + One particularly effective way of handling subcommands is to combine the use of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so that each subparser knows which Python function it should execute. For example:: - >>> # sub-command functions + >>> # subcommand functions >>> def foo(args): ... print(args.x * args.y) ... @@ -1952,7 +1806,7 @@ Sub-commands Namespace(subparser_name='2', y='frobble') .. versionchanged:: 3.7 - New *required* keyword argument. + New *required* keyword-only parameter. FileType objects @@ -1981,20 +1835,31 @@ FileType objects >>> parser.parse_args(['-']) Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>) + .. note:: + + If one argument uses *FileType* and then a subsequent argument fails, + an error is reported but the file is not automatically closed. + This can also clobber the output files. + In this case, it would be better to wait until after the parser has + run and then use the :keyword:`with`-statement to manage the files. + .. versionchanged:: 3.4 Added the *encodings* and *errors* parameters. + .. deprecated:: 3.14 + Argument groups ^^^^^^^^^^^^^^^ -.. method:: ArgumentParser.add_argument_group(title=None, description=None) +.. method:: ArgumentParser.add_argument_group(title=None, description=None, *, \ + [argument_default], [conflict_handler]) By default, :class:`ArgumentParser` groups command-line arguments into "positional arguments" and "options" when displaying help messages. When there is a better conceptual grouping of arguments than this default one, appropriate groups can be created using the - :meth:`add_argument_group` method:: + :meth:`!add_argument_group` method:: >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) >>> group = parser.add_argument_group('group') @@ -2011,7 +1876,7 @@ Argument groups has an :meth:`~ArgumentParser.add_argument` method just like a regular :class:`ArgumentParser`. When an argument is added to the group, the parser treats it just like a normal argument, but displays the argument in a - separate group for help messages. The :meth:`add_argument_group` method + separate group for help messages. The :meth:`!add_argument_group` method accepts *title* and *description* arguments which can be used to customize this display:: @@ -2033,6 +1898,11 @@ Argument groups --bar BAR bar help + The optional, keyword-only parameters argument_default_ and conflict_handler_ + allow for finer-grained control of the behavior of the argument group. These + parameters have the same meaning as in the :class:`ArgumentParser` constructor, + but apply specifically to the argument group rather than the entire parser. + Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. @@ -2042,13 +1912,17 @@ Argument groups The function exists on the API by accident through inheritance and will be removed in the future. + .. deprecated:: 3.14 + Passing prefix_chars_ to :meth:`add_argument_group` + is now deprecated. + Mutual exclusion ^^^^^^^^^^^^^^^^ .. method:: ArgumentParser.add_mutually_exclusive_group(required=False) - Create a mutually exclusive group. :mod:`argparse` will make sure that only + Create a mutually exclusive group. :mod:`!argparse` will make sure that only one of the arguments in the mutually exclusive group was present on the command line:: @@ -2185,20 +2059,20 @@ Partial parsing .. method:: ArgumentParser.parse_known_args(args=None, namespace=None) -Sometimes a script may only parse a few of the command-line arguments, passing -the remaining arguments on to another script or program. In these cases, the -:meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like -:meth:`~ArgumentParser.parse_args` except that it does not produce an error when -extra arguments are present. Instead, it returns a two item tuple containing -the populated namespace and the list of remaining argument strings. + Sometimes a script may only parse a few of the command-line arguments, passing + the remaining arguments on to another script or program. In these cases, the + :meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like + :meth:`~ArgumentParser.parse_args` except that it does not produce an error when + extra arguments are present. Instead, it returns a two item tuple containing + the populated namespace and the list of remaining argument strings. -:: + :: - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='store_true') - >>> parser.add_argument('bar') - >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) - (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action='store_true') + >>> parser.add_argument('bar') + >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) + (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) .. warning:: :ref:`Prefix matching ` rules apply to @@ -2235,8 +2109,8 @@ Exiting methods .. method:: ArgumentParser.exit(status=0, message=None) This method terminates the program, exiting with the specified *status* - and, if given, it prints a *message* before that. The user can override - this method to handle these steps differently:: + and, if given, it prints a *message* to :data:`sys.stderr` before that. + The user can override this method to handle these steps differently:: class ErrorCatchingArgumentParser(argparse.ArgumentParser): def exit(self, status=0, message=None): @@ -2246,8 +2120,8 @@ Exiting methods .. method:: ArgumentParser.error(message) - This method prints a usage message including the *message* to the - standard error and terminates the program with a status code of 2. + This method prints a usage message, including the *message*, to + :data:`sys.stderr` and terminates the program with a status code of 2. Intermixed parsing @@ -2256,90 +2130,38 @@ Intermixed parsing .. method:: ArgumentParser.parse_intermixed_args(args=None, namespace=None) .. method:: ArgumentParser.parse_known_intermixed_args(args=None, namespace=None) -A number of Unix commands allow the user to intermix optional arguments with -positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args` -and :meth:`~ArgumentParser.parse_known_intermixed_args` methods -support this parsing style. - -These parsers do not support all the argparse features, and will raise -exceptions if unsupported features are used. In particular, subparsers, -and mutually exclusive groups that include both -optionals and positionals are not supported. - -The following example shows the difference between -:meth:`~ArgumentParser.parse_known_args` and -:meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2', -'3']`` as unparsed arguments, while the latter collects all the positionals -into ``rest``. :: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.add_argument('cmd') - >>> parser.add_argument('rest', nargs='*', type=int) - >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split()) - (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3']) - >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split()) - Namespace(cmd='doit', foo='bar', rest=[1, 2, 3]) - -:meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple -containing the populated namespace and the list of remaining argument strings. -:meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any -remaining unparsed argument strings. - -.. versionadded:: 3.7 - -.. _upgrading-optparse-code: - -Upgrading optparse code ------------------------ - -Originally, the :mod:`argparse` module had attempted to maintain compatibility -with :mod:`optparse`. However, :mod:`optparse` was difficult to extend -transparently, particularly with the changes required to support the new -``nargs=`` specifiers and better usage messages. When most everything in -:mod:`optparse` had either been copy-pasted over or monkey-patched, it no -longer seemed practical to try to maintain the backwards compatibility. - -The :mod:`argparse` module improves on the standard library :mod:`optparse` -module in a number of ways including: - -* Handling positional arguments. -* Supporting sub-commands. -* Allowing alternative option prefixes like ``+`` and ``/``. -* Handling zero-or-more and one-or-more style arguments. -* Producing more informative usage messages. -* Providing a much simpler interface for custom ``type`` and ``action``. - -A partial upgrade path from :mod:`optparse` to :mod:`argparse`: - -* Replace all :meth:`optparse.OptionParser.add_option` calls with - :meth:`ArgumentParser.add_argument` calls. - -* Replace ``(options, args) = parser.parse_args()`` with ``args = - parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` - calls for the positional arguments. Keep in mind that what was previously - called ``options``, now in the :mod:`argparse` context is called ``args``. + A number of Unix commands allow the user to intermix optional arguments with + positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args` + and :meth:`~ArgumentParser.parse_known_intermixed_args` methods + support this parsing style. -* Replace :meth:`optparse.OptionParser.disable_interspersed_args` - by using :meth:`~ArgumentParser.parse_intermixed_args` instead of - :meth:`~ArgumentParser.parse_args`. + These parsers do not support all the :mod:`!argparse` features, and will raise + exceptions if unsupported features are used. In particular, subparsers, + and mutually exclusive groups that include both + optionals and positionals are not supported. -* Replace callback actions and the ``callback_*`` keyword arguments with - ``type`` or ``action`` arguments. + The following example shows the difference between + :meth:`~ArgumentParser.parse_known_args` and + :meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2', + '3']`` as unparsed arguments, while the latter collects all the positionals + into ``rest``. :: -* Replace string names for ``type`` keyword arguments with the corresponding - type objects (e.g. int, float, complex, etc). + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> parser.add_argument('cmd') + >>> parser.add_argument('rest', nargs='*', type=int) + >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split()) + (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3']) + >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split()) + Namespace(cmd='doit', foo='bar', rest=[1, 2, 3]) -* Replace :class:`optparse.Values` with :class:`Namespace` and - :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with - :exc:`ArgumentError`. + :meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple + containing the populated namespace and the list of remaining argument strings. + :meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any + remaining unparsed argument strings. -* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with - the standard Python syntax to use dictionaries to format strings, that is, - ``%(default)s`` and ``%(prog)s``. + .. versionadded:: 3.7 -* Replace the OptionParser constructor ``version`` argument with a call to - ``parser.add_argument('--version', action='version', version='')``. Exceptions ---------- @@ -2354,3 +2176,12 @@ Exceptions .. exception:: ArgumentTypeError Raised when something goes wrong converting a command line string to a type. + + +.. rubric:: Guides and Tutorials + +.. toctree:: + :maxdepth: 1 + + ../howto/argparse.rst + ../howto/argparse-optparse.rst diff --git a/Doc/library/array.rst b/Doc/library/array.rst index a0e8bb20a098fd..e0b1eb89cf6c05 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -1,5 +1,5 @@ -:mod:`array` --- Efficient arrays of numeric values -=================================================== +:mod:`!array` --- Efficient arrays of numeric values +==================================================== .. module:: array :synopsis: Space efficient arrays of uniformly typed numeric values. @@ -9,7 +9,7 @@ -------------- This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating point numbers. Arrays are sequence +basic values: characters, integers, floating-point numbers. Arrays are sequence types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a :dfn:`type code`, which is a single character. The following type codes are @@ -215,6 +215,13 @@ The module defines the following type: Remove the first occurrence of *x* from the array. + .. method:: clear() + + Remove all elements from the array. + + .. versionadded:: 3.13 + + .. method:: reverse() Reverse the order of the items in the array. @@ -256,7 +263,7 @@ The string representation is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the :class:`~array.array` class has been imported using ``from array import array``. Variables ``inf`` and ``nan`` must also be defined if it contains -corresponding floating point values. +corresponding floating-point values. Examples:: array('l') @@ -272,4 +279,3 @@ Examples:: `NumPy `_ The NumPy package defines another array type. - diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index c943c2f498173e..22d8c87cb58e78 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1,5 +1,5 @@ -:mod:`ast` --- Abstract Syntax Trees -==================================== +:mod:`!ast` --- Abstract Syntax Trees +===================================== .. module:: ast :synopsis: Abstract Syntax Tree classes and manipulation. @@ -61,7 +61,7 @@ Node classes .. attribute:: _fields - Each concrete class has an attribute :attr:`_fields` which gives the names + Each concrete class has an attribute :attr:`!_fields` which gives the names of all child nodes. Each instance of a concrete class has one attribute for each child node, @@ -74,6 +74,18 @@ Node classes as Python lists. All possible attributes must be present and have valid values when compiling an AST with :func:`compile`. + .. attribute:: _field_types + + The :attr:`!_field_types` attribute on each concrete class is a dictionary + mapping field names (as also listed in :attr:`_fields`) to their types. + + .. doctest:: + + >>> ast.TypeVar._field_types + {'name': , 'bound': ast.expr | None, 'default_value': ast.expr | None} + + .. versionadded:: 3.13 + .. attribute:: lineno col_offset end_lineno @@ -103,20 +115,16 @@ Node classes For example, to create and populate an :class:`ast.UnaryOp` node, you could use :: - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Constant() - node.operand.value = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), lineno=0, col_offset=0) + If a field that is optional in the grammar is omitted from the constructor, + it defaults to ``None``. If a list field is omitted, it defaults to the empty + list. If a field of type :class:`!ast.expr_context` is omitted, it defaults to + :class:`Load() `. If any other field is omitted, a :exc:`DeprecationWarning` is raised + and the AST node will not have this field. In Python 3.15, this condition will + raise an error. + .. versionchanged:: 3.8 Class :class:`ast.Constant` is now used for all constants. @@ -126,6 +134,11 @@ Node classes Simple indices are represented by their value, extended slices are represented as tuples. +.. versionchanged:: 3.14 + + The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes + the values of the node fields. + .. deprecated:: 3.8 Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, @@ -140,6 +153,14 @@ Node classes In the meantime, instantiating them will return an instance of a different class. +.. deprecated-removed:: 3.13 3.15 + + Previous versions of Python allowed the creation of AST nodes that were missing + required fields. Similarly, AST node constructors allowed arbitrary keyword + arguments that were set as attributes of the AST node, even if they did not + match any of the fields of the AST node. This behavior is deprecated and will + be removed in Python 3.15. + .. note:: The descriptions of the specific node classes displayed here were initially adapted from the fantastic `Green Tree @@ -157,9 +178,9 @@ Root nodes A Python module, as with :ref:`file input `. Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*. - *body* is a :class:`list` of the module's :ref:`ast-statements`. + ``body`` is a :class:`list` of the module's :ref:`ast-statements`. - *type_ignores* is a :class:`list` of the module's type ignore comments; + ``type_ignores`` is a :class:`list` of the module's type ignore comments; see :func:`ast.parse` for more details. .. doctest:: @@ -170,8 +191,7 @@ Root nodes Assign( targets=[ Name(id='x', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) + value=Constant(value=1))]) .. class:: Expression(body) @@ -179,7 +199,7 @@ Root nodes A single Python :ref:`expression input `. Node type generated by :func:`ast.parse` when *mode* is ``"eval"``. - *body* is a single node, + ``body`` is a single node, one of the :ref:`expression types `. .. doctest:: @@ -194,7 +214,7 @@ Root nodes A single :ref:`interactive input `, like in :ref:`tut-interac`. Node type generated by :func:`ast.parse` when *mode* is ``"single"``. - *body* is a :class:`list` of :ref:`statement nodes `. + ``body`` is a :class:`list` of :ref:`statement nodes `. .. doctest:: @@ -223,9 +243,9 @@ Root nodes # type: (int, int) -> int return a + b - *argtypes* is a :class:`list` of :ref:`expression nodes `. + ``argtypes`` is a :class:`list` of :ref:`expression nodes `. - *returns* is a single :ref:`expression node `. + ``returns`` is a single :ref:`expression node `. .. doctest:: @@ -299,8 +319,7 @@ Literals value=Call( func=Name(id='sin', ctx=Load()), args=[ - Name(id='a', ctx=Load())], - keywords=[]), + Name(id='a', ctx=Load())]), conversion=-1, format_spec=JoinedStr( values=[ @@ -395,8 +414,7 @@ Variables Module( body=[ Expr( - value=Name(id='a', ctx=Load()))], - type_ignores=[]) + value=Name(id='a', ctx=Load()))]) >>> print(ast.dump(ast.parse('a = 1'), indent=4)) Module( @@ -404,16 +422,14 @@ Variables Assign( targets=[ Name(id='a', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) + value=Constant(value=1))]) >>> print(ast.dump(ast.parse('del a'), indent=4)) Module( body=[ Delete( targets=[ - Name(id='a', ctx=Del())])], - type_ignores=[]) + Name(id='a', ctx=Del())])]) .. class:: Starred(value, ctx) @@ -436,8 +452,7 @@ Variables value=Name(id='b', ctx=Store()), ctx=Store())], ctx=Store())], - value=Name(id='it', ctx=Load()))], - type_ignores=[]) + value=Name(id='it', ctx=Load()))]) .. _ast-expressions: @@ -460,8 +475,7 @@ Expressions Expr( value=UnaryOp( op=USub(), - operand=Name(id='a', ctx=Load())))], - type_ignores=[]) + operand=Name(id='a', ctx=Load())))]) .. class:: UnaryOp(op, operand) @@ -588,8 +602,7 @@ Expressions * ``keywords`` holds a list of :class:`.keyword` objects representing arguments passed by keyword. - When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. + The ``args`` and ``keywords`` arguments are optional and default to empty lists. .. doctest:: @@ -726,7 +739,10 @@ Comprehensions .. doctest:: - >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) + >>> print(ast.dump( + ... ast.parse('[x for x in numbers]', mode='eval'), + ... indent=4, + ... )) Expression( body=ListComp( elt=Name(id='x', ctx=Load()), @@ -734,9 +750,11 @@ Comprehensions comprehension( target=Name(id='x', ctx=Store()), iter=Name(id='numbers', ctx=Load()), - ifs=[], is_async=0)])) - >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) + >>> print(ast.dump( + ... ast.parse('{x: x**2 for x in numbers}', mode='eval'), + ... indent=4, + ... )) Expression( body=DictComp( key=Name(id='x', ctx=Load()), @@ -748,9 +766,11 @@ Comprehensions comprehension( target=Name(id='x', ctx=Store()), iter=Name(id='numbers', ctx=Load()), - ifs=[], is_async=0)])) - >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) + >>> print(ast.dump( + ... ast.parse('{x for x in numbers}', mode='eval'), + ... indent=4, + ... )) Expression( body=SetComp( elt=Name(id='x', ctx=Load()), @@ -758,7 +778,6 @@ Comprehensions comprehension( target=Name(id='x', ctx=Store()), iter=Name(id='numbers', ctx=Load()), - ifs=[], is_async=0)])) @@ -781,18 +800,15 @@ Comprehensions elt=Call( func=Name(id='ord', ctx=Load()), args=[ - Name(id='c', ctx=Load())], - keywords=[]), + Name(id='c', ctx=Load())]), generators=[ comprehension( target=Name(id='line', ctx=Store()), iter=Name(id='file', ctx=Load()), - ifs=[], is_async=0), comprehension( target=Name(id='c', ctx=Store()), iter=Name(id='line', ctx=Load()), - ifs=[], is_async=0)])) >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), @@ -831,7 +847,6 @@ Comprehensions comprehension( target=Name(id='i', ctx=Store()), iter=Name(id='soc', ctx=Load()), - ifs=[], is_async=1)])) @@ -861,8 +876,7 @@ Statements targets=[ Name(id='a', ctx=Store()), Name(id='b', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) + value=Constant(value=1))]) >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking Module( @@ -874,18 +888,21 @@ Statements Name(id='a', ctx=Store()), Name(id='b', ctx=Store())], ctx=Store())], - value=Name(id='c', ctx=Load()))], - type_ignores=[]) + value=Name(id='c', ctx=Load()))]) .. class:: AnnAssign(target, annotation, value, simple) An assignment with a type annotation. ``target`` is a single node and can - be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. + be a :class:`Name`, an :class:`Attribute` or a :class:`Subscript`. ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` - node. ``value`` is a single optional node. ``simple`` is a boolean integer - set to True for a :class:`Name` node in ``target`` that do not appear in - between parenthesis and are hence pure names and not expressions. + node. ``value`` is a single optional node. + + ``simple`` is always either 0 (indicating a "complex" target) or 1 + (indicating a "simple" target). A "simple" target consists solely of a + :class:`Name` node that does not appear between parentheses; all other + targets are considered complex. Only simple targets appear in + the :attr:`~object.__annotations__` dictionary of modules and classes. .. doctest:: @@ -895,8 +912,7 @@ Statements AnnAssign( target=Name(id='c', ctx=Store()), annotation=Name(id='int', ctx=Load()), - simple=1)], - type_ignores=[]) + simple=1)]) >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis Module( @@ -905,8 +921,7 @@ Statements target=Name(id='a', ctx=Store()), annotation=Name(id='int', ctx=Load()), value=Constant(value=1), - simple=0)], - type_ignores=[]) + simple=0)]) >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation Module( @@ -917,8 +932,7 @@ Statements attr='b', ctx=Store()), annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) + simple=0)]) >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation Module( @@ -929,8 +943,7 @@ Statements slice=Constant(value=1), ctx=Store()), annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) + simple=0)]) .. class:: AugAssign(target, op, value) @@ -951,8 +964,7 @@ Statements AugAssign( target=Name(id='x', ctx=Store()), op=Add(), - value=Constant(value=2))], - type_ignores=[]) + value=Constant(value=2))]) .. class:: Raise(exc, cause) @@ -968,8 +980,7 @@ Statements body=[ Raise( exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))], - type_ignores=[]) + cause=Name(id='y', ctx=Load()))]) .. class:: Assert(test, msg) @@ -984,8 +995,7 @@ Statements body=[ Assert( test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))], - type_ignores=[]) + msg=Name(id='y', ctx=Load()))]) .. class:: Delete(targets) @@ -1002,8 +1012,7 @@ Statements targets=[ Name(id='x', ctx=Del()), Name(id='y', ctx=Del()), - Name(id='z', ctx=Del())])], - type_ignores=[]) + Name(id='z', ctx=Del())])]) .. class:: Pass() @@ -1015,8 +1024,7 @@ Statements >>> print(ast.dump(ast.parse('pass'), indent=4)) Module( body=[ - Pass()], - type_ignores=[]) + Pass()]) .. class:: TypeAlias(name, type_params, value) @@ -1033,9 +1041,7 @@ Statements body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), - type_params=[], - value=Name(id='int', ctx=Load()))], - type_ignores=[]) + value=Name(id='int', ctx=Load()))]) .. versionadded:: 3.12 @@ -1058,8 +1064,7 @@ Imports names=[ alias(name='x'), alias(name='y'), - alias(name='z')])], - type_ignores=[]) + alias(name='z')])]) .. class:: ImportFrom(module, names, level) @@ -1080,8 +1085,7 @@ Imports alias(name='x'), alias(name='y'), alias(name='z')], - level=0)], - type_ignores=[]) + level=0)]) .. class:: alias(name, asname) @@ -1099,8 +1103,7 @@ Imports names=[ alias(name='a', asname='b'), alias(name='c')], - level=2)], - type_ignores=[]) + level=2)]) Control flow ^^^^^^^^^^^^ @@ -1143,8 +1146,7 @@ Control flow value=Constant(value=Ellipsis))], orelse=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. class:: For(target, iter, body, orelse, type_comment) @@ -1178,8 +1180,7 @@ Control flow value=Constant(value=Ellipsis))], orelse=[ Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) + value=Constant(value=Ellipsis))])]) .. class:: While(test, body, orelse) @@ -1204,8 +1205,7 @@ Control flow value=Constant(value=Ellipsis))], orelse=[ Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) + value=Constant(value=Ellipsis))])]) .. class:: Break @@ -1239,9 +1239,7 @@ Control flow body=[ Break()], orelse=[ - Continue()])], - orelse=[])], - type_ignores=[]) + Continue()])])]) .. class:: Try(body, handlers, orelse, finalbody) @@ -1286,8 +1284,7 @@ Control flow value=Constant(value=Ellipsis))], finalbody=[ Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) + value=Constant(value=Ellipsis))])]) .. class:: TryStar(body, handlers, orelse, finalbody) @@ -1315,10 +1312,7 @@ Control flow type=Name(id='Exception', ctx=Load()), body=[ Expr( - value=Constant(value=Ellipsis))])], - orelse=[], - finalbody=[])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.11 @@ -1350,10 +1344,7 @@ Control flow ExceptHandler( type=Name(id='TypeError', ctx=Load()), body=[ - Pass()])], - orelse=[], - finalbody=[])], - type_ignores=[]) + Pass()])])]) .. class:: With(items, body, type_comment) @@ -1395,9 +1386,7 @@ Control flow func=Name(id='something', ctx=Load()), args=[ Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())], - keywords=[]))])], - type_ignores=[]) + Name(id='d', ctx=Load())]))])]) Pattern matching @@ -1454,14 +1443,10 @@ Pattern matching value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( - cls=Name(id='tuple', ctx=Load()), - patterns=[], - kwd_attrs=[], - kwd_patterns=[]), + cls=Name(id='tuple', ctx=Load())), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1489,8 +1474,7 @@ Pattern matching value=Constant(value='Relevant')), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1516,8 +1500,7 @@ Pattern matching pattern=MatchSingleton(value=None), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1549,8 +1532,7 @@ Pattern matching value=Constant(value=2))]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1591,8 +1573,7 @@ Pattern matching MatchStar()]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1636,11 +1617,10 @@ Pattern matching Expr( value=Constant(value=Ellipsis))]), match_case( - pattern=MatchMapping(keys=[], patterns=[], rest='rest'), + pattern=MatchMapping(rest='rest'), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1682,16 +1662,13 @@ Pattern matching MatchValue( value=Constant(value=0)), MatchValue( - value=Constant(value=0))], - kwd_attrs=[], - kwd_patterns=[]), + value=Constant(value=0))]), body=[ Expr( value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( cls=Name(id='Point3D', ctx=Load()), - patterns=[], kwd_attrs=[ 'x', 'y', @@ -1705,8 +1682,7 @@ Pattern matching value=Constant(value=0))]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1748,8 +1724,7 @@ Pattern matching pattern=MatchAs(), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1782,8 +1757,7 @@ Pattern matching MatchAs(name='y')]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1795,15 +1769,17 @@ Type parameters :ref:`Type parameters ` can exist on classes, functions, and type aliases. -.. class:: TypeVar(name, bound) +.. class:: TypeVar(name, bound, default_value) A :class:`typing.TypeVar`. ``name`` is the name of the type variable. ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, - it represents constraints; otherwise it represents the bound. + it represents constraints; otherwise it represents the bound. ``default_value`` + is the default value; if the :class:`!TypeVar` has no default, this + attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[T: int] = list[T]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[T: int = bool] = list[T]"), indent=4)) Module( body=[ TypeAlias( @@ -1811,28 +1787,39 @@ aliases. type_params=[ TypeVar( name='T', - bound=Name(id='int', ctx=Load()))], + bound=Name(id='int', ctx=Load()), + default_value=Name(id='bool', ctx=Load()))], value=Subscript( value=Name(id='list', ctx=Load()), slice=Name(id='T', ctx=Load()), - ctx=Load()))], - type_ignores=[]) + ctx=Load()))]) .. versionadded:: 3.12 -.. class:: ParamSpec(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: ParamSpec(name, default_value) A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + ``default_value`` is the default value; if the :class:`!ParamSpec` has no default, + this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[**P] = Callable[P, int]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[**P = (int, str)] = Callable[P, int]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - ParamSpec(name='P')], + ParamSpec( + name='P', + default_value=Tuple( + elts=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + ctx=Load()))], value=Subscript( value=Name(id='Callable', ctx=Load()), slice=Tuple( @@ -1840,24 +1827,30 @@ aliases. Name(id='P', ctx=Load()), Name(id='int', ctx=Load())], ctx=Load()), - ctx=Load()))], - type_ignores=[]) + ctx=Load()))]) .. versionadded:: 3.12 -.. class:: TypeVarTuple(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: TypeVarTuple(name, default_value) A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + ``default_value`` is the default value; if the :class:`!TypeVarTuple` has no + default, this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[*Ts] = tuple[*Ts]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[*Ts = ()] = tuple[*Ts]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - TypeVarTuple(name='Ts')], + TypeVarTuple( + name='Ts', + default_value=Tuple(ctx=Load()))], value=Subscript( value=Name(id='tuple', ctx=Load()), slice=Tuple( @@ -1866,11 +1859,13 @@ aliases. value=Name(id='Ts', ctx=Load()), ctx=Load())], ctx=Load()), - ctx=Load()))], - type_ignores=[]) + ctx=Load()))]) .. versionadded:: 3.12 + .. versionchanged:: 3.13 + Added the *default_value* parameter. + Function and class definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1907,15 +1902,10 @@ Function and class definitions Expr( value=Lambda( args=arguments( - posonlyargs=[], args=[ arg(arg='x'), - arg(arg='y')], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=Constant(value=Ellipsis)))], - type_ignores=[]) + arg(arg='y')]), + body=Constant(value=Ellipsis)))]) .. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) @@ -1954,7 +1944,6 @@ Function and class definitions FunctionDef( name='f', args=arguments( - posonlyargs=[], args=[ arg( arg='a', @@ -1977,9 +1966,7 @@ Function and class definitions decorator_list=[ Name(id='decorator1', ctx=Load()), Name(id='decorator2', ctx=Load())], - returns=Constant(value='return annotation'), - type_params=[])], - type_ignores=[]) + returns=Constant(value='return annotation'))]) .. class:: Return(value) @@ -1992,15 +1979,14 @@ Function and class definitions Module( body=[ Return( - value=Constant(value=4))], - type_ignores=[]) + value=Constant(value=4))]) .. class:: Yield(value) YieldFrom(value) A ``yield`` or ``yield from`` expression. Because these are expressions, they - must be wrapped in a :class:`Expr` node if the value sent back is not used. + must be wrapped in an :class:`Expr` node if the value sent back is not used. .. doctest:: @@ -2009,16 +1995,14 @@ Function and class definitions body=[ Expr( value=Yield( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) + value=Name(id='x', ctx=Load())))]) >>> print(ast.dump(ast.parse('yield from x'), indent=4)) Module( body=[ Expr( value=YieldFrom( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) + value=Name(id='x', ctx=Load())))]) .. class:: Global(names) @@ -2035,8 +2019,7 @@ Function and class definitions names=[ 'x', 'y', - 'z'])], - type_ignores=[]) + 'z'])]) >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) Module( @@ -2045,8 +2028,7 @@ Function and class definitions names=[ 'x', 'y', - 'z'])], - type_ignores=[]) + 'z'])]) .. class:: ClassDef(name, bases, keywords, body, decorator_list, type_params) @@ -2056,8 +2038,7 @@ Function and class definitions * ``name`` is a raw string for the class name * ``bases`` is a list of nodes for explicitly specified base classes. * ``keywords`` is a list of :class:`.keyword` nodes, principally for 'metaclass'. - Other keywords will be passed to the metaclass, as per `PEP-3115 - `_. + Other keywords will be passed to the metaclass, as per :pep:`3115`. * ``body`` is a list of nodes representing the code within the class definition. * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. @@ -2086,9 +2067,7 @@ Function and class definitions Pass()], decorator_list=[ Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], - type_params=[])], - type_ignores=[]) + Name(id='decorator2', ctx=Load())])]) .. versionchanged:: 3.12 Added ``type_params``. @@ -2120,22 +2099,12 @@ Async and await body=[ AsyncFunctionDef( name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), + args=arguments(), body=[ Expr( value=Await( value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[], - type_params=[])], - type_ignores=[]) + func=Name(id='other_func', ctx=Load()))))])]) .. class:: AsyncFor(target, iter, body, orelse, type_comment) @@ -2169,7 +2138,7 @@ and classes for traversing abstract syntax trees: If ``type_comments=True`` is given, the parser is modified to check and return type comments as specified by :pep:`484` and :pep:`526`. This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the - flags passed to :func:`compile()`. This will report syntax errors + flags passed to :func:`compile`. This will report syntax errors for misplaced type comments. Without this flag, type comments will be ignored, and the ``type_comment`` field on selected AST nodes will always be ``None``. In addition, the locations of ``# type: @@ -2180,14 +2149,17 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. - Also, setting ``feature_version`` to a tuple ``(major, minor)`` - will attempt to parse using that Python version's grammar. - Currently ``major`` must equal to ``3``. For example, setting - ``feature_version=(3, 4)`` will allow the use of ``async`` and - ``await`` as variable names. The lowest supported version is - ``(3, 7)``; the highest is ``sys.version_info[0:2]``. + Setting ``feature_version`` to a tuple ``(major, minor)`` will result in + a "best-effort" attempt to parse using that Python version's grammar. + For example, setting ``feature_version=(3, 9)`` will attempt to disallow + parsing of :keyword:`match` statements. + Currently ``major`` must equal to ``3``. The lowest supported version is + ``(3, 7)`` (and this may increase in future Python versions); + the highest is ``sys.version_info[0:2]``. "Best-effort" attempt means there + is no guarantee that the parse (or success of the parse) is the same as + when run on the Python version corresponding to ``feature_version``. - If source contains a null character ('\0'), :exc:`ValueError` is raised. + If source contains a null character (``\0``), :exc:`ValueError` is raised. .. warning:: Note that successfully parsing source code into an AST object doesn't @@ -2419,7 +2391,7 @@ and classes for traversing abstract syntax trees: node = YourTransformer().visit(node) -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None, show_empty=False) Return a formatted dump of the tree in *node*. This is mainly useful for debugging purposes. If *annotate_fields* is true (by default), @@ -2436,9 +2408,42 @@ and classes for traversing abstract syntax trees: indents that many spaces per level. If *indent* is a string (such as ``"\t"``), that string is used to indent each level. + If *show_empty* is ``False`` (the default), empty lists and fields that are ``None`` + will be omitted from the output. + .. versionchanged:: 3.9 Added the *indent* option. + .. versionchanged:: 3.13 + Added the *show_empty* option. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... async def f(): + ... await other_func() + ... """), indent=4, show_empty=True)) + Module( + body=[ + AsyncFunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=[ + Expr( + value=Await( + value=Call( + func=Name(id='other_func', ctx=Load()), + args=[], + keywords=[])))], + decorator_list=[], + type_params=[])], + type_ignores=[]) + .. _ast-compiler-flags: @@ -2475,6 +2480,20 @@ effects on the compilation of a program: .. versionadded:: 3.8 +.. function:: compare(a, b, /, *, compare_attributes=False) + + Recursively compares two ASTs. + + *compare_attributes* affects whether AST attributes are considered + in the comparison. If *compare_attributes* is ``False`` (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + differ in whitespace or similar details. Attributes include line numbers + and column offsets. + + .. versionadded:: 3.14 + + .. _ast-cli: Command-Line Usage @@ -2530,7 +2549,8 @@ to stdout. Otherwise, the content is read from stdin. code that generated them. This is helpful for tools that make source code transformations. - `leoAst.py `_ unifies the + `leoAst.py `_ + unifies the token-based and parse-tree-based views of python programs by inserting two-way links between tokens and ast nodes. @@ -2542,4 +2562,4 @@ to stdout. Otherwise, the content is read from stdin. `Parso `_ is a Python parser that supports error recovery and round-trip parsing for different Python versions (in multiple Python versions). Parso is also able to list multiple syntax errors - in your python file. + in your Python file. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index a9c3a0183bb72d..44b507a9811116 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -103,7 +103,8 @@ To handle signals the event loop must be run in the main thread. The :meth:`loop.run_in_executor` method can be used with a -:class:`concurrent.futures.ThreadPoolExecutor` to execute +:class:`concurrent.futures.ThreadPoolExecutor` or +:class:`~concurrent.futures.InterpreterPoolExecutor` to execute blocking code in a different OS thread without blocking the OS thread that the event loop runs in. @@ -128,7 +129,8 @@ if a function performs a CPU-intensive calculation for 1 second, all concurrent asyncio Tasks and IO operations would be delayed by 1 second. -An executor can be used to run a task in a different thread or even in +An executor can be used to run a task in a different thread, +including in a different interpreter, or even in a different process to avoid blocking the OS thread with the event loop. See the :meth:`loop.run_in_executor` method for more details. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 828e506a72c937..14fd153f640f05 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -126,7 +126,7 @@ Running and stopping the loop Run the event loop until :meth:`stop` is called. - If :meth:`stop` is called before :meth:`run_forever()` is called, + If :meth:`stop` is called before :meth:`run_forever` is called, the loop will poll the I/O selector once with a timeout of zero, run all callbacks scheduled in response to I/O events (and those that were already scheduled), and then exit. @@ -165,7 +165,7 @@ Running and stopping the loop .. coroutinemethod:: loop.shutdown_asyncgens() Schedule all currently open :term:`asynchronous generator` objects to - close with an :meth:`~agen.aclose()` call. After calling this method, + close with an :meth:`~agen.aclose` call. After calling this method, the event loop will issue a warning if a new asynchronous generator is iterated. This should be used to reliably finalize all scheduled asynchronous generators. @@ -605,6 +605,9 @@ Opening network connections The *family*, *proto*, *flags*, *reuse_address*, *reuse_port*, *allow_broadcast*, and *sock* parameters were added. + .. versionchanged:: 3.8 + Added support for Windows. + .. versionchanged:: 3.8.1 The *reuse_address* parameter is no longer supported, as using :ref:`socket.SO_REUSEADDR ` @@ -622,11 +625,8 @@ Opening network connections prevents processes with differing UIDs from assigning sockets to the same socket address. - .. versionchanged:: 3.8 - Added support for Windows. - .. versionchanged:: 3.11 - The *reuse_address* parameter, disabled since Python 3.9.0, 3.8.1, + The *reuse_address* parameter, disabled since Python 3.8.1, 3.7.6 and 3.6.10, has been entirely removed. .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ @@ -796,7 +796,7 @@ Creating network servers :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths are supported. - If *cleanup_socket* is True then the Unix socket will automatically + If *cleanup_socket* is true then the Unix socket will automatically be removed from the filesystem when the server is closed, unless the socket has been replaced after the server has been created. @@ -1155,6 +1155,14 @@ DNS Asynchronous version of :meth:`socket.getnameinfo`. +.. note:: + Both *getaddrinfo* and *getnameinfo* internally utilize their synchronous + versions through the loop's default thread pool executor. + When this executor is saturated, these methods may experience delays, + which higher-level networking libraries may report as increased timeouts. + To mitigate this, consider using a custom executor for other user tasks, + or setting a default executor with a larger number of workers. + .. versionchanged:: 3.7 Both *getaddrinfo* and *getnameinfo* methods were always documented to return a coroutine, but prior to Python 3.7 they were, in fact, @@ -1254,6 +1262,9 @@ Executing code in thread or process pools The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. + The default executor can be set by :meth:`loop.set_default_executor`, + otherwise, a :class:`concurrent.futures.ThreadPoolExecutor` will be + lazy-initialized and used by :func:`run_in_executor` if needed. Example:: @@ -1294,6 +1305,12 @@ Executing code in thread or process pools pool, cpu_bound) print('custom process pool', result) + # 4. Run in a custom interpreter pool: + with concurrent.futures.InterpreterPoolExecutor() as pool: + result = await loop.run_in_executor( + pool, cpu_bound) + print('custom interpreter pool', result) + if __name__ == '__main__': asyncio.run(main()) @@ -1318,7 +1335,8 @@ Executing code in thread or process pools Set *executor* as the default executor used by :meth:`run_in_executor`. *executor* must be an instance of - :class:`~concurrent.futures.ThreadPoolExecutor`. + :class:`~concurrent.futures.ThreadPoolExecutor`, which includes + :class:`~concurrent.futures.InterpreterPoolExecutor`. .. versionchanged:: 3.11 *executor* must be an instance of @@ -1391,7 +1409,7 @@ Allows customizing how exceptions are handled in the event loop. This method should not be overloaded in subclassed event loops. For custom exception handling, use - the :meth:`set_exception_handler()` method. + the :meth:`set_exception_handler` method. Enabling debug mode ^^^^^^^^^^^^^^^^^^^ @@ -1474,7 +1492,7 @@ async/await code consider using the high-level * *stdin* can be any of these: * a file-like object - * an existing file descriptor (a positive integer), for example those created with :meth:`os.pipe()` + * an existing file descriptor (a positive integer), for example those created with :meth:`os.pipe` * the :const:`subprocess.PIPE` constant (default) which will create a new pipe and connect it, * the value ``None`` which will make the subprocess inherit the file @@ -1641,6 +1659,31 @@ Do not instantiate the :class:`Server` class directly. coroutine to wait until the server is closed (and no more connections are active). + .. method:: close_clients() + + Close all existing incoming client connections. + + Calls :meth:`~asyncio.BaseTransport.close` on all associated + transports. + + :meth:`close` should be called before :meth:`close_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + + .. method:: abort_clients() + + Close all existing incoming client connections immediately, + without waiting for pending operations to complete. + + Calls :meth:`~asyncio.WriteTransport.abort` on all associated + transports. + + :meth:`close` should be called before :meth:`abort_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + .. method:: get_loop() Return the event loop associated with the server object. diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 893ae5518f757d..9dce0731411940 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -120,20 +120,20 @@ Future Object a :exc:`CancelledError` exception. If the Future's result isn't yet available, this method raises - a :exc:`InvalidStateError` exception. + an :exc:`InvalidStateError` exception. .. method:: set_result(result) Mark the Future as *done* and set its result. - Raises a :exc:`InvalidStateError` error if the Future is + Raises an :exc:`InvalidStateError` error if the Future is already *done*. .. method:: set_exception(exception) Mark the Future as *done* and set an exception. - Raises a :exc:`InvalidStateError` error if the Future is + Raises an :exc:`InvalidStateError` error if the Future is already *done*. .. method:: done() diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index 67136ba69ec875..f5af888f31f186 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -56,10 +56,10 @@ See also the main documentation section about the * - :meth:`loop.close` - Close the event loop. - * - :meth:`loop.is_running()` + * - :meth:`loop.is_running` - Return ``True`` if the event loop is running. - * - :meth:`loop.is_closed()` + * - :meth:`loop.is_closed` - Return ``True`` if the event loop is closed. * - ``await`` :meth:`loop.shutdown_asyncgens` @@ -96,7 +96,7 @@ See also the main documentation section about the - Invoke a callback *at* the given time. -.. rubric:: Thread/Process Pool +.. rubric:: Thread/Interpreter/Process Pool .. list-table:: :widths: 50 50 :class: full-width-table diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 19ec726c1be060..a2a3114ad6e4c5 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -77,11 +77,6 @@ Subprocess Support on Windows On Windows, the default event loop :class:`ProactorEventLoop` supports subprocesses, whereas :class:`SelectorEventLoop` does not. -The :meth:`policy.set_child_watcher() -` function is also -not supported, as :class:`ProactorEventLoop` has a different mechanism -to watch child processes. - macOS ===== diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 346b740a8f757a..837ccc6606786e 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -79,25 +79,6 @@ The abstract event loop policy base class is defined as follows: This method should never return ``None``. - .. method:: get_child_watcher() - - Get a child process watcher object. - - Return a watcher object implementing the - :class:`AbstractChildWatcher` interface. - - This function is Unix specific. - - .. deprecated:: 3.12 - - .. method:: set_child_watcher(watcher) - - Set the current child process watcher to *watcher*. - - This function is Unix specific. - - .. deprecated:: 3.12 - .. _asyncio-policy-builtin: @@ -139,172 +120,6 @@ asyncio ships with the following built-in policies: .. availability:: Windows. -.. _asyncio-watchers: - -Process Watchers -================ - -A process watcher allows customization of how an event loop monitors -child processes on Unix. Specifically, the event loop needs to know -when a child process has exited. - -In asyncio, child processes are created with -:func:`create_subprocess_exec` and :meth:`loop.subprocess_exec` -functions. - -asyncio defines the :class:`AbstractChildWatcher` abstract base class, which child -watchers should implement, and has four different implementations: -:class:`ThreadedChildWatcher` (configured to be used by default), -:class:`MultiLoopChildWatcher`, :class:`SafeChildWatcher`, and -:class:`FastChildWatcher`. - -See also the :ref:`Subprocess and Threads ` -section. - -The following two functions can be used to customize the child process watcher -implementation used by the asyncio event loop: - -.. function:: get_child_watcher() - - Return the current child watcher for the current policy. - - .. deprecated:: 3.12 - -.. function:: set_child_watcher(watcher) - - Set the current child watcher to *watcher* for the current - policy. *watcher* must implement methods defined in the - :class:`AbstractChildWatcher` base class. - - .. deprecated:: 3.12 - -.. note:: - Third-party event loops implementations might not support - custom child watchers. For such event loops, using - :func:`set_child_watcher` might be prohibited or have no effect. - -.. class:: AbstractChildWatcher - - .. method:: add_child_handler(pid, callback, *args) - - Register a new child handler. - - Arrange for ``callback(pid, returncode, *args)`` to be called - when a process with PID equal to *pid* terminates. Specifying - another callback for the same process replaces the previous - handler. - - The *callback* callable must be thread-safe. - - .. method:: remove_child_handler(pid) - - Removes the handler for process with PID equal to *pid*. - - The function returns ``True`` if the handler was successfully - removed, ``False`` if there was nothing to remove. - - .. method:: attach_loop(loop) - - Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then - it is first detached before attaching to the new loop. - - Note: loop may be ``None``. - - .. method:: is_active() - - Return ``True`` if the watcher is ready to use. - - Spawning a subprocess with *inactive* current child watcher raises - :exc:`RuntimeError`. - - .. versionadded:: 3.8 - - .. method:: close() - - Close the watcher. - - This method has to be called to ensure that underlying - resources are cleaned-up. - - .. deprecated:: 3.12 - - -.. class:: ThreadedChildWatcher - - This implementation starts a new waiting thread for every subprocess spawn. - - It works reliably even when the asyncio event loop is run in a non-main OS thread. - - There is no noticeable overhead when handling a big number of children (*O*\ (1) each - time a child terminates), but starting a thread per process requires extra memory. - - This watcher is used by default. - - .. versionadded:: 3.8 - -.. class:: MultiLoopChildWatcher - - This implementation registers a :py:data:`SIGCHLD` signal handler on - instantiation. That can break third-party code that installs a custom handler for - :py:data:`SIGCHLD` signal. - - The watcher avoids disrupting other code spawning processes - by polling every process explicitly on a :py:data:`SIGCHLD` signal. - - There is no limitation for running subprocesses from different threads once the - watcher is installed. - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O*\ (*n*) each time a - :py:data:`SIGCHLD` is received). - - .. versionadded:: 3.8 - - .. deprecated:: 3.12 - -.. class:: SafeChildWatcher - - This implementation uses active event loop from the main thread to handle - :py:data:`SIGCHLD` signal. If the main thread has no running event loop another - thread cannot spawn a subprocess (:exc:`RuntimeError` is raised). - - The watcher avoids disrupting other code spawning processes - by polling every process explicitly on a :py:data:`SIGCHLD` signal. - - This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O*\ (*n*) - complexity but requires a running event loop in the main thread to work. - - .. deprecated:: 3.12 - -.. class:: FastChildWatcher - - This implementation reaps every terminated processes by calling - ``os.waitpid(-1)`` directly, possibly breaking other code spawning - processes and waiting for their termination. - - There is no noticeable overhead when handling a big number of - children (*O*\ (1) each time a child terminates). - - This solution requires a running event loop in the main thread to work, as - :class:`SafeChildWatcher`. - - .. deprecated:: 3.12 - -.. class:: PidfdChildWatcher - - This implementation polls process file descriptors (pidfds) to await child - process termination. In some respects, :class:`PidfdChildWatcher` is a - "Goldilocks" child watcher implementation. It doesn't require signals or - threads, doesn't interfere with any processes launched outside the event - loop, and scales linearly with the number of subprocesses launched by the - event loop. The main disadvantage is that pidfds are specific to Linux, and - only work on recent (5.3+) kernels. - - .. versionadded:: 3.9 - - .. _asyncio-custom-policies: Custom Policies diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3f734f544afe21..7c08d65f26bc27 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -362,6 +362,11 @@ Datagram Transports This method does not block; it buffers the data and arranges for it to be sent out asynchronously. + .. versionchanged:: 3.13 + This method can be called with an empty bytes object to send a + zero-length datagram. The buffer size calculation used for flow + control is also updated to account for the datagram header. + .. method:: DatagramTransport.abort() Close the transport immediately, without waiting for pending @@ -417,8 +422,8 @@ Subprocess Transports Stop the subprocess. - On POSIX systems, this method sends SIGTERM to the subprocess. - On Windows, the Windows API function TerminateProcess() is called to + On POSIX systems, this method sends :py:const:`~signal.SIGTERM` to the subprocess. + On Windows, the Windows API function :c:func:`!TerminateProcess` is called to stop the subprocess. See also :meth:`subprocess.Popen.terminate`. diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d86fbc21351e2d..61991bf2f4ed1d 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -55,13 +55,16 @@ Queue Return ``True`` if there are :attr:`maxsize` items in the queue. If the queue was initialized with ``maxsize=0`` (the default), - then :meth:`full()` never returns ``True``. + then :meth:`full` never returns ``True``. .. coroutinemethod:: get() Remove and return an item from the queue. If queue is empty, wait until an item is available. + Raises :exc:`QueueShutDown` if the queue has been shut down and + is empty, or if the queue has been shut down immediately. + .. method:: get_nowait() Return an item if one is immediately available, else raise @@ -82,6 +85,8 @@ Queue Put an item into the queue. If the queue is full, wait until a free slot is available before adding the item. + Raises :exc:`QueueShutDown` if the queue has been shut down. + .. method:: put_nowait(item) Put an item into the queue without blocking. @@ -92,6 +97,22 @@ Queue Return the number of items in the queue. + .. method:: shutdown(immediate=False) + + Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` + raise :exc:`QueueShutDown`. + + By default, :meth:`~Queue.get` on a shut down queue will only + raise once the queue is empty. Set *immediate* to true to make + :meth:`~Queue.get` raise immediately instead. + + All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` + will be unblocked. If *immediate* is true, a task will be marked + as done for each remaining item in the queue, which may unblock + callers of :meth:`~Queue.join`. + + .. versionadded:: 3.13 + .. method:: task_done() Indicate that a formerly enqueued task is complete. @@ -105,6 +126,9 @@ Queue call was received for every item that had been :meth:`~Queue.put` into the queue). + ``shutdown(immediate=True)`` calls :meth:`task_done` for each + remaining item in the queue. + Raises :exc:`ValueError` if called more times than there were items placed in the queue. @@ -145,6 +169,14 @@ Exceptions on a queue that has reached its *maxsize*. +.. exception:: QueueShutDown + + Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is + called on a queue which has been shut down. + + .. versionadded:: 3.13 + + Examples ======== diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index ec170dfde9e9aa..28d5aaf3692baa 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -24,11 +24,13 @@ Running an asyncio Program .. function:: run(coro, *, debug=None, loop_factory=None) - Execute the :term:`coroutine` *coro* and return the result. + Execute *coro* in an asyncio event loop and return the result. - This function runs the passed coroutine, taking care of - managing the asyncio event loop, *finalizing asynchronous - generators*, and closing the executor. + The argument can be any awaitable object. + + This function runs the awaitable, taking care of managing the + asyncio event loop, *finalizing asynchronous generators*, and + closing the executor. This function cannot be called when another asyncio event loop is running in the same thread. @@ -70,6 +72,10 @@ Running an asyncio Program Added *loop_factory* parameter. + .. versionchanged:: 3.14 + + *coro* can be any awaitable object. + Runner context manager ====================== @@ -91,7 +97,7 @@ Runner context manager current one. By default :func:`asyncio.new_event_loop` is used and set as current event loop with :func:`asyncio.set_event_loop` if *loop_factory* is ``None``. - Basically, :func:`asyncio.run()` example can be rewritten with the runner usage:: + Basically, :func:`asyncio.run` example can be rewritten with the runner usage:: async def main(): await asyncio.sleep(1) @@ -104,17 +110,25 @@ Runner context manager .. method:: run(coro, *, context=None) - Run a :term:`coroutine ` *coro* in the embedded loop. + Execute *coro* in the embedded event loop. + + The argument can be any awaitable object. - Return the coroutine's result or raise its exception. + If the argument is a coroutine, it is wrapped in a Task. An optional keyword-only *context* argument allows specifying a - custom :class:`contextvars.Context` for the *coro* to run in. - The runner's default context is used if ``None``. + custom :class:`contextvars.Context` for the code to run in. + The runner's default context is used if context is ``None``. + + Returns the awaitable's result or raises an exception. This function cannot be called when another asyncio event loop is running in the same thread. + .. versionchanged:: 3.14 + + *coro* can be any awaitable object. + .. method:: close() Close the runner. diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 3427da1b43caef..3fdc79b3c6896c 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -260,8 +260,19 @@ StreamReader buffer is reset. The :attr:`IncompleteReadError.partial` attribute may contain a portion of the separator. + The *separator* may also be a tuple of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of :exc:`LimitOverrunError`, + the shortest possible separator is considered to be the one that + matched. + .. versionadded:: 3.5.2 + .. versionchanged:: 3.13 + + The *separator* parameter may now be a :class:`tuple` of + separators. + .. method:: at_eof() Return ``True`` if the buffer is empty and :meth:`feed_eof` @@ -347,7 +358,7 @@ StreamWriter be resumed. When there is nothing to wait for, the :meth:`drain` returns immediately. - .. coroutinemethod:: start_tls(sslcontext, \*, server_hostname=None, \ + .. coroutinemethod:: start_tls(sslcontext, *, server_hostname=None, \ ssl_handshake_timeout=None, ssl_shutdown_timeout=None) Upgrade an existing stream-based connection to TLS. diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index bf35b1cb798aee..b477a7fa2d37ef 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -240,7 +240,7 @@ their completion. .. note:: - On Windows, :py:data:`SIGTERM` is an alias for :meth:`terminate`. + On Windows, :py:const:`~signal.SIGTERM` is an alias for :meth:`terminate`. ``CTRL_C_EVENT`` and ``CTRL_BREAK_EVENT`` can be sent to processes started with a *creationflags* parameter which includes ``CREATE_NEW_PROCESS_GROUP``. @@ -249,10 +249,10 @@ their completion. Stop the child process. - On POSIX systems this method sends :py:const:`signal.SIGTERM` to the + On POSIX systems this method sends :py:const:`~signal.SIGTERM` to the child process. - On Windows the Win32 API function :c:func:`TerminateProcess` is + On Windows the Win32 API function :c:func:`!TerminateProcess` is called to stop the child process. .. method:: kill() @@ -316,18 +316,6 @@ default. On Windows subprocesses are provided by :class:`ProactorEventLoop` only (default), :class:`SelectorEventLoop` has no subprocess support. -On UNIX *child watchers* are used for subprocess finish waiting, see -:ref:`asyncio-watchers` for more info. - - -.. versionchanged:: 3.8 - - UNIX switched to use :class:`ThreadedChildWatcher` for spawning subprocesses from - different threads without any limitation. - - Spawning a subprocess with *inactive* current child watcher raises - :exc:`RuntimeError`. - Note that alternative event loop implementations might have own limitations; please refer to their documentation. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 24bd36e6431b4f..f27e858cf420f4 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -158,7 +158,7 @@ other coroutines:: # Nothing happens if we just call "nested()". # A coroutine object is created but not awaited, # so it *won't run at all*. - nested() + nested() # will raise a "RuntimeWarning". # Let's do it differently now and await it: print(await nested()) # will print "42". @@ -334,6 +334,13 @@ and reliable way to wait for all tasks in the group to finish. Create a task in this task group. The signature matches that of :func:`asyncio.create_task`. + If the task group is inactive (e.g. not yet entered, + already finished, or in the process of shutting down), + we will close the given ``coro``. + + .. versionchanged:: 3.13 + + Close the given coroutine if the task group is not active. Example:: @@ -385,6 +392,74 @@ is also included in the exception group. The same special case is made for :exc:`KeyboardInterrupt` and :exc:`SystemExit` as in the previous paragraph. +Task groups are careful not to mix up the internal cancellation used to +"wake up" their :meth:`~object.__aexit__` with cancellation requests +for the task in which they are running made by other parties. +In particular, when one task group is syntactically nested in another, +and both experience an exception in one of their child tasks simultaneously, +the inner task group will process its exceptions, and then the outer task group +will receive another cancellation and process its own exceptions. + +In the case where a task group is cancelled externally and also must +raise an :exc:`ExceptionGroup`, it will call the parent task's +:meth:`~asyncio.Task.cancel` method. This ensures that a +:exc:`asyncio.CancelledError` will be raised at the next +:keyword:`await`, so the cancellation is not lost. + +Task groups preserve the cancellation count +reported by :meth:`asyncio.Task.cancelling`. + +.. versionchanged:: 3.13 + + Improved handling of simultaneous internal and external cancellations + and correct preservation of cancellation counts. + +Terminating a Task Group +------------------------ + +While terminating a task group is not natively supported by the standard +library, termination can be achieved by adding an exception-raising task +to the task group and ignoring the raised exception: + +.. code-block:: python + + import asyncio + from asyncio import TaskGroup + + class TerminateTaskGroup(Exception): + """Exception raised to terminate a task group.""" + + async def force_terminate_task_group(): + """Used to force termination of a task group.""" + raise TerminateTaskGroup() + + async def job(task_id, sleep_time): + print(f'Task {task_id}: start') + await asyncio.sleep(sleep_time) + print(f'Task {task_id}: done') + + async def main(): + try: + async with TaskGroup() as group: + # spawn some tasks + group.create_task(job(1, 0.5)) + group.create_task(job(2, 1.5)) + # sleep for 1 second + await asyncio.sleep(1) + # add an exception-raising task to force the group to terminate + group.create_task(force_terminate_task_group()) + except* TerminateTaskGroup: + pass + + asyncio.run(main()) + +Expected output: + +.. code-block:: text + + Task 1: start + Task 2: start + Task 1: done Sleeping ======== @@ -510,7 +585,7 @@ Running Tasks Concurrently # [2, 6, 24] .. note:: - If *return_exceptions* is False, cancelling gather() after it + If *return_exceptions* is false, cancelling gather() after it has been marked done won't cancel any submitted awaitables. For instance, gather can be marked done after propagating an exception to the caller, therefore, calling ``gather.cancel()`` @@ -860,19 +935,50 @@ Waiting Primitives .. function:: as_completed(aws, *, timeout=None) - Run :ref:`awaitable objects ` in the *aws* - iterable concurrently. Return an iterator of coroutines. - Each coroutine returned can be awaited to get the earliest next - result from the iterable of the remaining awaitables. + Run :ref:`awaitable objects ` in the *aws* iterable + concurrently. The returned object can be iterated to obtain the results + of the awaitables as they finish. - Raises :exc:`TimeoutError` if the timeout occurs before - all Futures are done. + The object returned by ``as_completed()`` can be iterated as an + :term:`asynchronous iterator` or a plain :term:`iterator`. When asynchronous + iteration is used, the originally-supplied awaitables are yielded if they + are tasks or futures. This makes it easy to correlate previously-scheduled + tasks with their results. Example:: - Example:: + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] + + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect + + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") + + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. - for coro in as_completed(aws): - earliest_result = await coro - # ... + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13:: + + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] + + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect + + A :exc:`TimeoutError` is raised if the timeout occurs before all awaitables + are done. This is raised by the ``async for`` loop during asynchronous + iteration or by the coroutines yielded during plain iteration. .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -884,6 +990,10 @@ Waiting Primitives .. versionchanged:: 3.12 Added support for generators yielding tasks. + .. versionchanged:: 3.13 + The result can now be used as either an :term:`asynchronous iterator` + or as a plain :term:`iterator` (previously it was only a plain iterator). + Running in Threads ================== @@ -1107,7 +1217,7 @@ Task Object a :exc:`CancelledError` exception. If the Task's result isn't yet available, this method raises - a :exc:`InvalidStateError` exception. + an :exc:`InvalidStateError` exception. .. method:: exception() @@ -1323,10 +1433,19 @@ Task Object with :meth:`uncancel`. :class:`TaskGroup` context managers use :func:`uncancel` in a similar fashion. - If end-user code is, for some reason, suppresing cancellation by + If end-user code is, for some reason, suppressing cancellation by catching :exc:`CancelledError`, it needs to call this method to remove the cancellation state. + When this method decrements the cancellation count to zero, + the method checks if a previous :meth:`cancel` call had arranged + for :exc:`CancelledError` to be thrown into the task. + If it hasn't been thrown yet, that arrangement will be + rescinded (by resetting the internal ``_must_cancel`` flag). + + .. versionchanged:: 3.13 + Changed to rescind pending cancellation requests upon reaching zero. + .. method:: cancelling() Return the number of pending cancellation requests to this Task, i.e., diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 5f33c6813e74c0..5f83b3a2658da4 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -1,5 +1,5 @@ -:mod:`asyncio` --- Asynchronous I/O -=================================== +:mod:`!asyncio` --- Asynchronous I/O +==================================== .. module:: asyncio :synopsis: Asynchronous I/O. @@ -56,9 +56,13 @@ Additionally, there are **low-level** APIs for * :ref:`bridge ` callback-based libraries and code with async/await syntax. +.. include:: ../includes/wasm-notavail.rst + .. _asyncio-cli: -You can experiment with an ``asyncio`` concurrent context in the REPL: +.. rubric:: asyncio REPL + +You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: .. code-block:: pycon @@ -70,7 +74,14 @@ You can experiment with an ``asyncio`` concurrent context in the REPL: >>> await asyncio.sleep(10, result='hello') 'hello' -.. include:: ../includes/wasm-notavail.rst +.. audit-event:: cpython.run_stdin "" "" + +.. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) + Emits audit events. + +.. versionchanged:: 3.13 + Uses PyREPL if possible, in which case :envvar:`PYTHONSTARTUP` is + also executed. Emits audit events. .. We use the "rubric" directive here to avoid creating the "Reference" subsection in the TOC. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 43a8bd2d7cd133..02d2f0807df8f6 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -1,5 +1,5 @@ -:mod:`atexit` --- Exit handlers -=============================== +:mod:`!atexit` --- Exit handlers +================================ .. module:: atexit :synopsis: Register and execute cleanup functions. diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst index 8227a7955bef81..a2a90a00d0cfca 100644 --- a/Doc/library/audit_events.rst +++ b/Doc/library/audit_events.rst @@ -7,7 +7,7 @@ Audit events table This table contains all events raised by :func:`sys.audit` or :c:func:`PySys_Audit` calls throughout the CPython runtime and the -standard library. These calls were added in 3.8.0 or later (see :pep:`578`). +standard library. These calls were added in 3.8 or later (see :pep:`578`). See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for information on handling these events. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index d5b6af8c1928ef..834ab2536e6c14 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -1,5 +1,5 @@ -:mod:`base64` --- Base16, Base32, Base64, Base85 Data Encodings -=============================================================== +:mod:`!base64` --- Base16, Base32, Base64, Base85 Data Encodings +================================================================ .. module:: base64 :synopsis: RFC 4648: Base16, Base32, Base64 Data Encodings; @@ -193,7 +193,7 @@ The modern interface provides: *wrapcol* controls whether the output should have newline (``b'\n'``) characters added to it. If this is non-zero, each output line will be - at most this many characters long. + at most this many characters long, excluding the trailing newline. *pad* controls whether the input is padded to a multiple of 4 before encoding. Note that the ``btoa`` implementation always pads. @@ -244,6 +244,24 @@ The modern interface provides: .. versionadded:: 3.4 +.. function:: z85encode(s) + + Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) + and return the encoded :class:`bytes`. See `Z85 specification + `_ for more information. + + .. versionadded:: 3.13 + + +.. function:: z85decode(s) + + Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and + return the decoded :class:`bytes`. See `Z85 specification + `_ for more information. + + .. versionadded:: 3.13 + + The legacy interface: .. function:: decode(input, output) diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 52f0ca7c013482..85df7914a9a014 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -1,5 +1,5 @@ -:mod:`bdb` --- Debugger framework -================================= +:mod:`!bdb` --- Debugger framework +================================== .. module:: bdb :synopsis: Debugger framework. @@ -86,7 +86,7 @@ The :mod:`bdb` module also defines two classes: .. attribute:: temporary - True if a :class:`Breakpoint` at (file, line) is temporary. + ``True`` if a :class:`Breakpoint` at (file, line) is temporary. .. attribute:: cond @@ -99,7 +99,7 @@ The :mod:`bdb` module also defines two classes: .. attribute:: enabled - True if :class:`Breakpoint` is enabled. + ``True`` if :class:`Breakpoint` is enabled. .. attribute:: bpbynumber @@ -148,8 +148,8 @@ The :mod:`bdb` module also defines two classes: .. method:: reset() - Set the :attr:`botframe`, :attr:`stopframe`, :attr:`returnframe` and - :attr:`quitting` attributes with values ready to start debugging. + Set the :attr:`!botframe`, :attr:`!stopframe`, :attr:`!returnframe` and + :attr:`quitting ` attributes with values ready to start debugging. .. method:: trace_dispatch(frame, event, arg) @@ -182,7 +182,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop on the current line, invoke the :meth:`user_line` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting ` flag is set (which can be set from :meth:`user_line`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -190,7 +190,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop on this function call, invoke the :meth:`user_call` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting ` flag is set (which can be set from :meth:`user_call`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -198,7 +198,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop on this function return, invoke the :meth:`user_return` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting ` flag is set (which can be set from :meth:`user_return`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -206,7 +206,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop at this exception, invokes the :meth:`user_exception` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting ` flag is set (which can be set from :meth:`user_exception`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -215,22 +215,22 @@ The :mod:`bdb` module also defines two classes: .. method:: is_skipped_line(module_name) - Return True if *module_name* matches any skip pattern. + Return ``True`` if *module_name* matches any skip pattern. .. method:: stop_here(frame) - Return True if *frame* is below the starting frame in the stack. + Return ``True`` if *frame* is below the starting frame in the stack. .. method:: break_here(frame) - Return True if there is an effective breakpoint for this line. + Return ``True`` if there is an effective breakpoint for this line. Check whether a line or function breakpoint exists and is in effect. Delete temporary breakpoints based on information from :func:`effective`. .. method:: break_anywhere(frame) - Return True if any breakpoint exists for *frame*'s filename. + Return ``True`` if any breakpoint exists for *frame*'s filename. Derived classes should override these methods to gain control over debugger operation. @@ -240,6 +240,9 @@ The :mod:`bdb` module also defines two classes: Called from :meth:`dispatch_call` if a break might stop inside the called function. + *argument_list* is not used anymore and will always be ``None``. + The argument is kept for backwards compatibility. + .. method:: user_line(frame) Called from :meth:`dispatch_line` when either :meth:`stop_here` or @@ -286,6 +289,10 @@ The :mod:`bdb` module also defines two classes: Start debugging from *frame*. If *frame* is not specified, debugging starts from caller's frame. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. + .. method:: set_continue() Stop only at breakpoints or when finished. If there are no breakpoints, @@ -293,7 +300,9 @@ The :mod:`bdb` module also defines two classes: .. method:: set_quit() - Set the :attr:`quitting` attribute to ``True``. This raises :exc:`BdbQuit` in + .. index:: single: quitting (bdb.Bdb attribute) + + Set the :attr:`!quitting` attribute to ``True``. This raises :exc:`BdbQuit` in the next call to one of the :meth:`!dispatch_\*` methods. @@ -339,7 +348,7 @@ The :mod:`bdb` module also defines two classes: .. method:: get_break(filename, lineno) - Return True if there is a breakpoint for *lineno* in *filename*. + Return ``True`` if there is a breakpoint for *lineno* in *filename*. .. method:: get_breaks(filename, lineno) @@ -383,7 +392,7 @@ The :mod:`bdb` module also defines two classes: .. method:: run(cmd, globals=None, locals=None) Debug a statement executed via the :func:`exec` function. *globals* - defaults to :attr:`__main__.__dict__`, *locals* defaults to *globals*. + defaults to :attr:`!__main__.__dict__`, *locals* defaults to *globals*. .. method:: runeval(expr, globals=None, locals=None) @@ -403,7 +412,7 @@ Finally, the module defines the following functions: .. function:: checkfuncname(b, frame) - Return True if we should break here, depending on the way the + Return ``True`` if we should break here, depending on the way the :class:`Breakpoint` *b* was set. If it was set via line number, it checks if @@ -422,14 +431,14 @@ Finally, the module defines the following functions: :attr:`bplist ` for the (:attr:`file `, :attr:`line `) (which must exist) that is :attr:`enabled `, for - which :func:`checkfuncname` is True, and that has neither a False + which :func:`checkfuncname` is true, and that has neither a false :attr:`condition ` nor positive :attr:`ignore ` count. The *flag*, meaning that a - temporary breakpoint should be deleted, is False only when the + temporary breakpoint should be deleted, is ``False`` only when the :attr:`cond ` cannot be evaluated (in which case, :attr:`ignore ` count is ignored). - If no such entry exists, then (None, None) is returned. + If no such entry exists, then ``(None, None)`` is returned. .. function:: set_trace() diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 39fabb59bb1984..1bab785684bbab 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -1,5 +1,5 @@ -:mod:`binascii` --- Convert between binary and ASCII -==================================================== +:mod:`!binascii` --- Convert between binary and ASCII +===================================================== .. module:: binascii :synopsis: Tools for converting between binary and various ASCII-encoded binary diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 31c79b91061591..78da563397b625 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -1,5 +1,5 @@ -:mod:`bisect` --- Array bisection algorithm -=========================================== +:mod:`!bisect` --- Array bisection algorithm +============================================ .. module:: bisect :synopsis: Array bisection algorithms for binary searching. diff --git a/Doc/library/builtins.rst b/Doc/library/builtins.rst index 7e4f8fe0531567..c4979db52d2aed 100644 --- a/Doc/library/builtins.rst +++ b/Doc/library/builtins.rst @@ -1,5 +1,5 @@ -:mod:`builtins` --- Built-in objects -==================================== +:mod:`!builtins` --- Built-in objects +===================================== .. module:: builtins :synopsis: The module that provides the built-in namespace. @@ -7,10 +7,7 @@ -------------- This module provides direct access to all 'built-in' identifiers of Python; for -example, ``builtins.open`` is the full name for the built-in function -:func:`open`. See :ref:`built-in-funcs` and :ref:`built-in-consts` for -documentation. - +example, ``builtins.open`` is the full name for the built-in function :func:`open`. This module is not normally accessed explicitly by most applications, but can be useful in modules that provide objects with the same name as a built-in value, @@ -40,3 +37,10 @@ available as part of their globals. The value of ``__builtins__`` is normally either this module or the value of this module's :attr:`~object.__dict__` attribute. Since this is an implementation detail, it may not be used by alternate implementations of Python. + +.. seealso:: + + * :ref:`built-in-consts` + * :ref:`bltin-exceptions` + * :ref:`built-in-funcs` + * :ref:`bltin-types` diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 6a95a4a6e8d183..ebe2e43febaefa 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -1,5 +1,5 @@ -:mod:`bz2` --- Support for :program:`bzip2` compression -======================================================= +:mod:`!bz2` --- Support for :program:`bzip2` compression +======================================================== .. module:: bz2 :synopsis: Interfaces for bzip2 compression and decompression. @@ -91,7 +91,7 @@ The :mod:`bz2` module contains: and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - :class:`BZ2File` also provides the following methods: + :class:`BZ2File` also provides the following methods and attributes: .. method:: peek([n]) @@ -148,6 +148,19 @@ The :mod:`bz2` module contains: .. versionadded:: 3.3 + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionadded:: 3.13 + + .. attribute:: name + + The bzip2 file name. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + .. versionadded:: 3.13 + .. versionchanged:: 3.1 Support for the :keyword:`with` statement was added. @@ -156,7 +169,6 @@ The :mod:`bz2` module contains: Support was added for *filename* being a :term:`file object` instead of an actual filename. - .. versionchanged:: 3.3 The ``'a'`` (append) mode was added, along with support for reading multi-stream files. diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index e699a7284ac802..eafc038d6cb722 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -1,5 +1,5 @@ -:mod:`calendar` --- General calendar-related functions -====================================================== +:mod:`!calendar` --- General calendar-related functions +======================================================= .. module:: calendar :synopsis: Functions for working with calendars, including some emulation @@ -393,13 +393,22 @@ The :mod:`calendar` module exports the following data attributes: .. data:: day_name - An array that represents the days of the week in the current locale. + A sequence that represents the days of the week in the current locale, + where Monday is day number 0. + + >>> import calendar + >>> list(calendar.day_name) + ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] .. data:: day_abbr - An array that represents the abbreviated days of the week in the current locale. + A sequence that represents the abbreviated days of the week in the current locale, + where Mon is day number 0. + >>> import calendar + >>> list(calendar.day_abbr) + ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] .. data:: MONDAY TUESDAY @@ -426,17 +435,24 @@ The :mod:`calendar` module exports the following data attributes: .. data:: month_name - An array that represents the months of the year in the current locale. This + A sequence that represents the months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_name[0]`` is the empty string. + >>> import calendar + >>> list(calendar.month_name) + ['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + .. data:: month_abbr - An array that represents the abbreviated months of the year in the current + A sequence that represents the abbreviated months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_abbr[0]`` is the empty string. + >>> import calendar + >>> list(calendar.month_abbr) + ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] .. data:: JANUARY FEBRUARY diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index fdac51d9603ceb..381a8332f4b187 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -1,5 +1,5 @@ -:mod:`cmath` --- Mathematical functions for complex numbers -=========================================================== +:mod:`!cmath` --- Mathematical functions for complex numbers +============================================================ .. module:: cmath :synopsis: Mathematical functions for complex numbers. @@ -43,10 +43,7 @@ Conversions to and from polar coordinates A Python complex number ``z`` is stored internally using *rectangular* or *Cartesian* coordinates. It is completely determined by its *real -part* ``z.real`` and its *imaginary part* ``z.imag``. In other -words:: - - z == z.real + z.imag*1j +part* ``z.real`` and its *imaginary part* ``z.imag``. *Polar coordinates* give an alternative way to represent a complex number. In polar coordinates, a complex number *z* is defined by the @@ -90,7 +87,7 @@ rectangular coordinates to polar coordinates and back. .. function:: rect(r, phi) Return the complex number *x* with polar coordinates *r* and *phi*. - Equivalent to ``r * (math.cos(phi) + math.sin(phi)*1j)``. + Equivalent to ``complex(r * math.cos(phi), r * math.sin(phi))``. Power and logarithmic functions diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 39ef4b481478d1..66544f82f6ff3f 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -1,5 +1,5 @@ -:mod:`cmd` --- Support for line-oriented command interpreters -============================================================= +:mod:`!cmd` --- Support for line-oriented command interpreters +============================================================== .. module:: cmd :synopsis: Build line-oriented command interpreters. diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index b2379befeffcba..78fe95a014ff7c 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -23,7 +23,7 @@ The following modules have a command-line interface. * :ref:`http.server ` * :mod:`!idlelib` * :ref:`inspect ` -* :ref:`json.tool ` +* :ref:`json ` * :mod:`mimetypes` * :mod:`pdb` * :mod:`pickle` @@ -36,9 +36,11 @@ The following modules have a command-line interface. * :mod:`pyclbr` * :mod:`pydoc` * :mod:`quopri` +* :ref:`random ` * :mod:`runpy` * :ref:`site ` * :ref:`sqlite3 ` +* :ref:`symtable ` * :ref:`sysconfig ` * :mod:`tabnanny` * :ref:`tarfile ` diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 091840781bd235..8f7692df9fb22d 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -1,5 +1,5 @@ -:mod:`code` --- Interpreter base classes -======================================== +:mod:`!code` --- Interpreter base classes +========================================= .. module:: code :synopsis: Facilities to implement read-eval-print loops. @@ -18,16 +18,16 @@ build applications which provide an interactive interpreter prompt. This class deals with parsing and interpreter state (the user's namespace); it does not deal with input buffering or prompting or input file naming (the filename is always passed in explicitly). The optional *locals* argument - specifies the dictionary in which code will be executed; it defaults to a newly - created dictionary with key ``'__name__'`` set to ``'__console__'`` and key - ``'__doc__'`` set to ``None``. + specifies a mapping to use as the namespace in which code will be executed; + it defaults to a newly created dictionary with key ``'__name__'`` set to + ``'__console__'`` and key ``'__doc__'`` set to ``None``. .. class:: InteractiveConsole(locals=None, filename="", local_exit=False) Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar - ``sys.ps1`` and ``sys.ps2``, and input buffering. If *local_exit* is True, + ``sys.ps1`` and ``sys.ps2``, and input buffering. If *local_exit* is true, ``exit()`` and ``quit()`` in the console will not raise :exc:`SystemExit`, but instead return to the calling code. @@ -41,7 +41,7 @@ build applications which provide an interactive interpreter prompt. the :meth:`InteractiveConsole.raw_input` method, if provided. If *local* is provided, it is passed to the :class:`InteractiveConsole` constructor for use as the default namespace for the interpreter loop. If *local_exit* is provided, - it is passed to the :class:`InteractiveConsole` constructor. The :meth:`interact` + it is passed to the :class:`InteractiveConsole` constructor. The :meth:`~InteractiveConsole.interact` method of the instance is then run with *banner* and *exitmsg* passed as the banner and exit message to use, if provided. The console object is discarded after use. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 4617624686b1f3..2cfd8a1eaee806 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1,5 +1,5 @@ -:mod:`codecs` --- Codec registry and base classes -================================================= +:mod:`!codecs` --- Codec registry and base classes +================================================== .. module:: codecs :synopsis: Encode and decode data and streams. @@ -1132,7 +1132,8 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp875 | | Greek | +-----------------+--------------------------------+--------------------------------+ -| cp932 | 932, ms932, mskanji, ms-kanji | Japanese | +| cp932 | 932, ms932, mskanji, ms-kanji, | Japanese | +| | windows-31j | | +-----------------+--------------------------------+--------------------------------+ | cp949 | 949, ms949, uhc | Korean | +-----------------+--------------------------------+--------------------------------+ @@ -1477,7 +1478,7 @@ Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding and :mod:`stringprep`. If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the -third-party `idna module `_. +third-party :pypi:`idna` module. These RFCs together define a protocol to support non-ASCII characters in domain names. A domain name containing non-ASCII characters (such as @@ -1540,13 +1541,13 @@ This module implements the ANSI codepage (CP_ACP). .. availability:: Windows. -.. versionchanged:: 3.3 - Support any error handler. - .. versionchanged:: 3.2 Before 3.2, the *errors* argument was ignored; ``'replace'`` was always used to encode, and ``'ignore'`` to decode. +.. versionchanged:: 3.3 + Support any error handler. + :mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature ------------------------------------------------------------- diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 55606e1c5f09ac..16f674adb4b22b 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -1,5 +1,5 @@ -:mod:`codeop` --- Compile Python code -===================================== +:mod:`!codeop` --- Compile Python code +====================================== .. module:: codeop :synopsis: Compile (possibly incomplete) Python code. diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 7bcaba60c6ddbd..42e40de152148c 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -1,5 +1,5 @@ -:mod:`collections.abc` --- Abstract Base Classes for Containers -=============================================================== +:mod:`!collections.abc` --- Abstract Base Classes for Containers +================================================================ .. module:: collections.abc :synopsis: Abstract base classes for containers @@ -141,12 +141,10 @@ ABC Inherits from Abstract Methods Mi ``__len__``, ``insert`` -:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods - ``__len__`` - :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, - ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` + ``__len__`` ``__sub__``, ``__rsub__``, ``__xor__``, ``__rxor__`` + and ``isdisjoint`` :class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, @@ -165,7 +163,7 @@ ABC Inherits from Abstract Methods Mi ``__len__`` -:class:`MappingView` :class:`Sized` ``__len__`` +:class:`MappingView` :class:`Sized` ``__init__``, ``__len__`` and ``__repr__`` :class:`ItemsView` :class:`MappingView`, ``__contains__``, :class:`Set` ``__iter__`` :class:`KeysView` :class:`MappingView`, ``__contains__``, @@ -216,6 +214,9 @@ Collections Abstract Base Classes -- Detailed Descriptions ABC for classes that provide the :meth:`~object.__call__` method. + See :ref:`annotating-callables` for details on how to use + :class:`!Callable` in type annotations. + .. class:: Iterable ABC for classes that provide the :meth:`~container.__iter__` method. @@ -253,11 +254,13 @@ Collections Abstract Base Classes -- Detailed Descriptions :meth:`~generator.send`, :meth:`~generator.throw` and :meth:`~generator.close` methods. + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`!Generator` in type annotations. + .. versionadded:: 3.5 .. class:: Sequence MutableSequence - ByteString ABCs for read-only and mutable :term:`sequences `. @@ -274,12 +277,6 @@ Collections Abstract Base Classes -- Detailed Descriptions The index() method added support for *stop* and *start* arguments. - .. deprecated-removed:: 3.12 3.14 - The :class:`ByteString` ABC has been deprecated. - For use in typing, prefer a union, like ``bytes | bytearray``, or - :class:`collections.abc.Buffer`. - For use as an ABC, prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - .. class:: Set MutableSet @@ -331,6 +328,11 @@ Collections Abstract Base Classes -- Detailed Descriptions Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`!Coroutine` in type annotations. + The variance and order of type parameters correspond to those of + :class:`Generator`. + .. versionadded:: 3.5 .. class:: AsyncIterable @@ -352,6 +354,9 @@ Collections Abstract Base Classes -- Detailed Descriptions ABC for :term:`asynchronous generator` classes that implement the protocol defined in :pep:`525` and :pep:`492`. + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`!AsyncGenerator` in type annotations. + .. versionadded:: 3.6 .. class:: Buffer diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index c246173c1bbf53..0cc9063f153aba 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -1,5 +1,5 @@ -:mod:`collections` --- Container datatypes -========================================== +:mod:`!collections` --- Container datatypes +=========================================== .. module:: collections :synopsis: Container datatypes @@ -99,7 +99,7 @@ The class can be used to simulate nested scopes and is useful in templating. :func:`super` function. A reference to ``d.parents`` is equivalent to: ``ChainMap(*d.maps[1:])``. - Note, the iteration order of a :class:`ChainMap()` is determined by + Note, the iteration order of a :class:`ChainMap` is determined by scanning the mappings last to first:: >>> baseline = {'music': 'bach', 'art': 'rembrandt'} @@ -134,7 +134,7 @@ The class can be used to simulate nested scopes and is useful in templating. :attr:`~collections.ChainMap.parents` property. * The `Nested Contexts recipe - `_ has options to control + `_ has options to control whether writes and other mutations apply only to the first mapping or to any mapping in the chain. @@ -343,7 +343,7 @@ superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. All of those tests treat missing elements as having zero counts so that ``Counter(a=1) == Counter(a=1, b=0)`` returns true. -.. versionadded:: 3.10 +.. versionchanged:: 3.10 Rich comparison operations were added. .. versionchanged:: 3.10 @@ -874,8 +874,8 @@ they add the ability to access fields by name instead of position index. ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to ``1``, and ``z`` will default to ``2``. - If *module* is defined, the ``__module__`` attribute of the named tuple is - set to that value. + If *module* is defined, the :attr:`~type.__module__` attribute of the + named tuple is set to that value. Named tuple instances do not have per-instance dictionaries, so they are lightweight and require no more memory than regular tuples. @@ -1169,8 +1169,11 @@ Some differences from :class:`dict` still remain: In addition to the usual mapping methods, ordered dictionaries also support reverse iteration using :func:`reversed`. +.. _collections_OrderedDict__eq__: + Equality tests between :class:`OrderedDict` objects are order-sensitive -and are implemented as ``list(od1.items())==list(od2.items())``. +and are roughly equivalent to ``list(od1.items())==list(od2.items())``. + Equality tests between :class:`OrderedDict` objects and other :class:`~collections.abc.Mapping` objects are order-insensitive like regular dictionaries. This allows :class:`OrderedDict` objects to be substituted @@ -1186,7 +1189,7 @@ anywhere a regular dictionary is used. method. .. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. + Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. :class:`OrderedDict` Examples and Recipes diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index b672a05b39145d..ffebf4e40dd609 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -1,5 +1,5 @@ -:mod:`colorsys` --- Conversions between color systems -===================================================== +:mod:`!colorsys` --- Conversions between color systems +====================================================== .. module:: colorsys :synopsis: Conversion functions between RGB and other color systems. @@ -14,7 +14,7 @@ The :mod:`colorsys` module defines bidirectional conversions of color values between colors expressed in the RGB (Red Green Blue) color space used in computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color -spaces are floating point values. In the YIQ space, the Y coordinate is between +spaces are floating-point values. In the YIQ space, the Y coordinate is between 0 and 1, but the I and Q coordinates can be positive or negative. In all other spaces, the coordinates are all between 0 and 1. diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index df1eefab839cc1..c42288419c4d2d 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -1,5 +1,5 @@ -:mod:`compileall` --- Byte-compile Python libraries -=================================================== +:mod:`!compileall` --- Byte-compile Python libraries +==================================================== .. module:: compileall :synopsis: Tools for byte-compiling all Python source files in a directory tree. @@ -90,7 +90,7 @@ compile Python sources. .. option:: -j N Use *N* workers to compile the files within the given directory. - If ``0`` is used, then the result of :func:`os.process_cpu_count()` + If ``0`` is used, then the result of :func:`os.process_cpu_count` will be used. .. option:: --invalidation-mode [timestamp|checked-hash|unchecked-hash] @@ -226,7 +226,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. .. versionchanged:: 3.8 Setting *workers* to 0 now chooses the optimal number of cores. @@ -289,7 +289,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. .. versionchanged:: 3.9 Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. @@ -318,7 +318,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index d3c7a40aa9d390..48e027152a9851 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -1,5 +1,5 @@ -:mod:`concurrent.futures` --- Launching parallel tasks -====================================================== +:mod:`!concurrent.futures` --- Launching parallel tasks +======================================================= .. module:: concurrent.futures :synopsis: Execute computations concurrently using threads or processes. @@ -15,9 +15,10 @@ The :mod:`concurrent.futures` module provides a high-level interface for asynchronously executing callables. The asynchronous execution can be performed with threads, using -:class:`ThreadPoolExecutor`, or separate processes, using -:class:`ProcessPoolExecutor`. Both implement the same interface, which is -defined by the abstract :class:`Executor` class. +:class:`ThreadPoolExecutor` or :class:`InterpreterPoolExecutor`, +or separate processes, using :class:`ProcessPoolExecutor`. +Each implements the same interface, which is defined +by the abstract :class:`Executor` class. .. include:: ../includes/wasm-notavail.rst @@ -63,7 +64,8 @@ Executor Objects setting *chunksize* to a positive integer. For very long iterables, using a large value for *chunksize* can significantly improve performance compared to the default size of 1. With - :class:`ThreadPoolExecutor`, *chunksize* has no effect. + :class:`ThreadPoolExecutor` and :class:`InterpreterPoolExecutor`, + *chunksize* has no effect. .. versionchanged:: 3.5 Added the *chunksize* argument. @@ -206,7 +208,7 @@ ThreadPoolExecutor Example 'http://www.cnn.com/', 'http://europe.wsj.com/', 'http://www.bbc.co.uk/', - 'http://nonexistant-subdomain.python.org/'] + 'http://nonexistent-subdomain.python.org/'] # Retrieve a single page and report the URL and contents def load_url(url, timeout): @@ -227,6 +229,111 @@ ThreadPoolExecutor Example print('%r page is %d bytes' % (url, len(data))) +InterpreterPoolExecutor +----------------------- + +The :class:`InterpreterPoolExecutor` class uses a pool of interpreters +to execute calls asynchronously. It is a :class:`ThreadPoolExecutor` +subclass, which means each worker is running in its own thread. +The difference here is that each worker has its own interpreter, +and runs each task using that interpreter. + +The biggest benefit to using interpreters instead of only threads +is true multi-core parallelism. Each interpreter has its own +:term:`Global Interpreter Lock `, so code +running in one interpreter can run on one CPU core, while code in +another interpreter runs unblocked on a different core. + +The tradeoff is that writing concurrent code for use with multiple +interpreters can take extra effort. However, this is because it +forces you to be deliberate about how and when interpreters interact, +and to be explicit about what data is shared between interpreters. +This results in several benefits that help balance the extra effort, +including true multi-core parallelism, For example, code written +this way can make it easier to reason about concurrency. Another +major benefit is that you don't have to deal with several of the +big pain points of using threads, like nrace conditions. + +Each worker's interpreter is isolated from all the other interpreters. +"Isolated" means each interpreter has its own runtime state and +operates completely independently. For example, if you redirect +:data:`sys.stdout` in one interpreter, it will not be automatically +redirected any other interpreter. If you import a module in one +interpreter, it is not automatically imported in any other. You +would need to import the module separately in interpreter where +you need it. In fact, each module imported in an interpreter is +a completely separate object from the same module in a different +interpreter, including :mod:`sys`, :mod:`builtins`, +and even ``__main__``. + +Isolation means a mutable object, or other data, cannot be used +by more than one interpreter at the same time. That effectively means +interpreters cannot actually share such objects or data. Instead, +each interpreter must have its own copy, and you will have to +synchronize any changes between the copies manually. Immutable +objects and data, like the builtin singletons, strings, and tuples +of immutable objects, don't have these limitations. + +Communicating and synchronizing between interpreters is most effectively +done using dedicated tools, like those proposed in :pep:`734`. One less +efficient alternative is to serialize with :mod:`pickle` and then send +the bytes over a shared :mod:`socket ` or +:func:`pipe `. + +.. class:: InterpreterPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=(), shared=None) + + A :class:`ThreadPoolExecutor` subclass that executes calls asynchronously + using a pool of at most *max_workers* threads. Each thread runs + tasks in its own interpreter. The worker interpreters are isolated + from each other, which means each has its own runtime state and that + they can't share any mutable objects or other data. Each interpreter + has its own :term:`Global Interpreter Lock `, + which means code run with this executor has true multi-core parallelism. + + The optional *initializer* and *initargs* arguments have the same + meaning as for :class:`!ThreadPoolExecutor`: the initializer is run + when each worker is created, though in this case it is run.in + the worker's interpreter. The executor serializes the *initializer* + and *initargs* using :mod:`pickle` when sending them to the worker's + interpreter. + + .. note:: + Functions defined in the ``__main__`` module cannot be pickled + and thus cannot be used. + + .. note:: + The executor may replace uncaught exceptions from *initializer* + with :class:`~concurrent.futures.interpreter.ExecutionFailed`. + + The optional *shared* argument is a :class:`dict` of objects that all + interpreters in the pool share. The *shared* items are added to each + interpreter's ``__main__`` module. Not all objects are shareable. + Shareable objects include the builtin singletons, :class:`str` + and :class:`bytes`, and :class:`memoryview`. See :pep:`734` + for more info. + + Other caveats from parent :class:`ThreadPoolExecutor` apply here. + +:meth:`~Executor.submit` and :meth:`~Executor.map` work like normal, +except the worker serializes the callable and arguments using +:mod:`pickle` when sending them to its interpreter. The worker +likewise serializes the return value when sending it back. + +.. note:: + Functions defined in the ``__main__`` module cannot be pickled + and thus cannot be used. + +When a worker's current task raises an uncaught exception, the worker +always tries to preserve the exception as-is. If that is successful +then it also sets the ``__cause__`` to a corresponding +:class:`~concurrent.futures.interpreter.ExecutionFailed` +instance, which contains a summary of the original exception. +In the uncommon case that the worker is not able to preserve the +original as-is then it directly preserves the corresponding +:class:`~concurrent.futures.interpreter.ExecutionFailed` +instance instead. + + ProcessPoolExecutor ------------------- @@ -286,14 +393,6 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. Added the *initializer* and *initargs* arguments. - .. note:: - The default :mod:`multiprocessing` start method - (see :ref:`multiprocessing-start-methods`) will change away from - *fork* in Python 3.14. Code that requires *fork* be used for their - :class:`ProcessPoolExecutor` should explicitly specify that by - passing a ``mp_context=multiprocessing.get_context("fork")`` - parameter. - .. versionchanged:: 3.11 The *max_tasks_per_child* argument was added to allow users to control the lifetime of workers in the pool. @@ -310,6 +409,12 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. *max_workers* uses :func:`os.process_cpu_count` by default, instead of :func:`os.cpu_count`. + .. versionchanged:: 3.14 + The default process start method (see + :ref:`multiprocessing-start-methods`) changed away from *fork*. If you + require the *fork* start method for :class:`ProcessPoolExecutor` you must + explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. _processpoolexecutor-example: ProcessPoolExecutor Example @@ -576,6 +681,26 @@ Exception classes .. versionadded:: 3.7 +.. currentmodule:: concurrent.futures.interpreter + +.. exception:: BrokenInterpreterPool + + Derived from :exc:`~concurrent.futures.thread.BrokenThreadPool`, + this exception class is raised when one of the workers + of a :class:`~concurrent.futures.InterpreterPoolExecutor` + has failed initializing. + + .. versionadded:: next + +.. exception:: ExecutionFailed + + Raised from :class:`~concurrent.futures.InterpreterPoolExecutor` when + the given initializer fails or from + :meth:`~concurrent.futures.Executor.submit` when there's an uncaught + exception from the submitted task. + + .. versionadded:: next + .. currentmodule:: concurrent.futures.process .. exception:: BrokenProcessPool diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 18e5bc20f3f690..3aad6f7b5d2d20 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -1,5 +1,5 @@ -:mod:`configparser` --- Configuration file parser -================================================= +:mod:`!configparser` --- Configuration file parser +================================================== .. module:: configparser :synopsis: Configuration file parser. @@ -54,6 +54,7 @@ can be customized by end users easily. import os os.remove("example.ini") + os.remove("override.ini") Quick Start @@ -147,23 +148,28 @@ case-insensitive and stored in lowercase [1]_. It is possible to read several configurations into a single :class:`ConfigParser`, where the most recently added configuration has the highest priority. Any conflicting keys are taken from the more recent -configuration while the previously existing keys are retained. +configuration while the previously existing keys are retained. The example +below reads in an ``override.ini`` file, which will override any conflicting +keys from the ``example.ini`` file. + +.. code-block:: ini + + [DEFAULT] + ServerAliveInterval = -1 .. doctest:: - >>> another_config = configparser.ConfigParser() - >>> another_config.read('example.ini') - ['example.ini'] - >>> another_config['topsecret.server.example']['Port'] - '50022' - >>> another_config.read_string("[topsecret.server.example]\nPort=48484") - >>> another_config['topsecret.server.example']['Port'] - '48484' - >>> another_config.read_dict({"topsecret.server.example": {"Port": 21212}}) - >>> another_config['topsecret.server.example']['Port'] - '21212' - >>> another_config['topsecret.server.example']['ForwardX11'] - 'no' + >>> config_override = configparser.ConfigParser() + >>> config_override['DEFAULT'] = {'ServerAliveInterval': '-1'} + >>> with open('override.ini', 'w') as configfile: + ... config_override.write(configfile) + ... + >>> config_override = configparser.ConfigParser() + >>> config_override.read(['example.ini', 'override.ini']) + ['example.ini', 'override.ini'] + >>> print(config_override.get('DEFAULT', 'ServerAliveInterval')) + -1 + This behaviour is equivalent to a :meth:`ConfigParser.read` call with several files passed to the *filenames* parameter. @@ -274,6 +280,11 @@ may be treated as parts of multiline values or ignored. By default, a valid section name can be any string that does not contain '\\n'. To change this, see :attr:`ConfigParser.SECTCRE`. +The first section name may be omitted if the parser is configured to allow an +unnamed top level section with ``allow_unnamed_section=True``. In this case, +the keys/values may be retrieved by :const:`UNNAMED_SECTION` as in +``config[UNNAMED_SECTION]``. + Configuration files may include comments, prefixed by specific characters (``#`` and ``;`` by default [1]_). Comments may appear on their own on an otherwise empty line, possibly indented. [1]_ @@ -325,6 +336,27 @@ For example: # Did I mention we can indent comments, too? +.. _unnamed-sections: + +Unnamed Sections +---------------- + +The name of the first section (or unique) may be omitted and values +retrieved by the :const:`UNNAMED_SECTION` attribute. + +.. doctest:: + + >>> config = """ + ... option = value + ... + ... [ Section 2 ] + ... another = val + ... """ + >>> unnamed = configparser.ConfigParser(allow_unnamed_section=True) + >>> unnamed.read_string(config) + >>> unnamed.get(configparser.UNNAMED_SECTION, 'option') + 'value' + Interpolation of values ----------------------- @@ -955,9 +987,34 @@ ConfigParser Objects When *converters* is given, it should be a dictionary where each key represents the name of a type converter and each value is a callable implementing the conversion from string to the desired datatype. Every - converter gets its own corresponding :meth:`!get*()` method on the parser + converter gets its own corresponding :meth:`!get*` method on the parser object and section proxies. + It is possible to read several configurations into a single + :class:`ConfigParser`, where the most recently added configuration has the + highest priority. Any conflicting keys are taken from the more recent + configuration while the previously existing keys are retained. The example + below reads in an ``override.ini`` file, which will override any conflicting + keys from the ``example.ini`` file. + + .. code-block:: ini + + [DEFAULT] + ServerAliveInterval = -1 + + .. doctest:: + + >>> config_override = configparser.ConfigParser() + >>> config_override['DEFAULT'] = {'ServerAliveInterval': '-1'} + >>> with open('override.ini', 'w') as configfile: + ... config_override.write(configfile) + ... + >>> config_override = configparser.ConfigParser() + >>> config_override.read(['example.ini', 'override.ini']) + ['example.ini', 'override.ini'] + >>> print(config_override.get('DEFAULT', 'ServerAliveInterval')) + -1 + .. versionchanged:: 3.1 The default *dict_type* is :class:`collections.OrderedDict`. @@ -970,7 +1027,7 @@ ConfigParser Objects The *converters* argument was added. .. versionchanged:: 3.7 - The *defaults* argument is read with :meth:`read_dict()`, + The *defaults* argument is read with :meth:`read_dict`, providing consistent behavior across the parser: non-string keys and values are implicitly converted to strings. @@ -978,6 +1035,10 @@ ConfigParser Objects The default *dict_type* is :class:`dict`, since it now preserves insertion order. + .. versionchanged:: 3.13 + Raise a :exc:`MultilineContinuationError` when *allow_no_value* is + ``True``, and a key without a value is continued with an indented line. + .. method:: defaults() Return a dictionary containing the instance-wide defaults. @@ -1123,7 +1184,7 @@ ConfigParser Objects .. method:: getfloat(section, option, *, raw=False, vars=None[, fallback]) A convenience method which coerces the *option* in the specified *section* - to a floating point number. See :meth:`get` for explanation of *raw*, + to a floating-point number. See :meth:`get` for explanation of *raw*, *vars* and *fallback*. @@ -1212,6 +1273,11 @@ ConfigParser Objects names is stripped before :meth:`optionxform` is called. +.. data:: UNNAMED_SECTION + + A special object representing a section name used to reference the unnamed section (see :ref:`unnamed-sections`). + + .. data:: MAX_INTERPOLATION_DEPTH The maximum depth for recursive interpolation for :meth:`~configparser.ConfigParser.get` when the *raw* @@ -1249,13 +1315,19 @@ RawConfigParser Objects .. method:: add_section(section) - Add a section named *section* to the instance. If a section by the given - name already exists, :exc:`DuplicateSectionError` is raised. If the - *default section* name is passed, :exc:`ValueError` is raised. + Add a section named *section* or :const:`UNNAMED_SECTION` to the instance. + + If the given section already exists, :exc:`DuplicateSectionError` is + raised. If the *default section* name is passed, :exc:`ValueError` is + raised. If :const:`UNNAMED_SECTION` is passed and support is disabled, + :exc:`UnnamedSectionDisabledError` is raised. Type of *section* is not checked which lets users create non-string named sections. This behaviour is unsupported and may cause internal errors. + .. versionchanged:: 3.14 + Added support for :const:`UNNAMED_SECTION`. + .. method:: set(section, option, value) @@ -1340,7 +1412,6 @@ Exceptions Exception raised when attempting to parse a file which has no section headers. - .. exception:: ParsingError Exception raised when errors occur attempting to parse a file. @@ -1349,6 +1420,20 @@ Exceptions The ``filename`` attribute and :meth:`!__init__` constructor argument were removed. They have been available using the name ``source`` since 3.2. +.. exception:: MultilineContinuationError + + Exception raised when a key without a corresponding value is continued with + an indented line. + + .. versionadded:: 3.13 + +.. exception:: UnnamedSectionDisabledError + + Exception raised when attempting to use the + :const:`UNNAMED_SECTION` without enabling it. + + .. versionadded:: 3.14 + .. rubric:: Footnotes .. [1] Config parsers allow for heavy customization. If you are interested in diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 401dc9a320c5e0..04bb8c51a3b197 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -33,29 +33,32 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - ``NotImplemented`` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. .. note:: - When a binary (or in-place) method returns ``NotImplemented`` the + When a binary (or in-place) method returns :data:`!NotImplemented` the interpreter will try the reflected operation on the other type (or some other fallback, depending on the operator). If all attempts return - ``NotImplemented``, the interpreter will raise an appropriate exception. - Incorrectly returning ``NotImplemented`` will result in a misleading - error message or the ``NotImplemented`` value being returned to Python code. + :data:`!NotImplemented`, the interpreter will raise an appropriate exception. + Incorrectly returning :data:`!NotImplemented` will result in a misleading + error message or the :data:`!NotImplemented` value being returned to Python code. See :ref:`implementing-the-arithmetic-operations` for examples. .. note:: - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and :data:`!NotImplemented` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. + Evaluating :data:`!NotImplemented` in a boolean context was deprecated. + + .. versionchanged:: 3.14 + Evaluating :data:`!NotImplemented` in a boolean context now raises a :exc:`TypeError`. + It previously evaluated to :const:`True` and emitted a :exc:`DeprecationWarning` + since Python 3.9. .. index:: single: ...; ellipsis literal @@ -79,6 +82,8 @@ A small number of constants live in the built-in namespace. They are: :exc:`SyntaxError`), so they can be considered "true" constants. +.. _site-consts: + Constants added by the :mod:`site` module ----------------------------------------- @@ -94,6 +99,13 @@ should not be used in programs. (i.e. EOF) to exit", and when called, raise :exc:`SystemExit` with the specified exit code. +.. data:: help + :noindex: + + Object that when printed, prints the message "Type help() for interactive + help, or help(object) for help about object.", and when called, + acts as described :func:`elsewhere `. + .. data:: copyright credits diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 73e53aec9cbf1c..f5b349441bcfee 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -314,13 +314,15 @@ Functions and classes provided: If the code within the :keyword:`!with` block raises a :exc:`BaseExceptionGroup`, suppressed exceptions are removed from the - group. If any exceptions in the group are not suppressed, a group containing them is re-raised. + group. Any exceptions of the group which are not suppressed are re-raised in + a new group which is created using the original group's :meth:`~BaseExceptionGroup.derive` + method. .. versionadded:: 3.4 .. versionchanged:: 3.12 ``suppress`` now supports suppressing exceptions raised as - part of an :exc:`BaseExceptionGroup`. + part of a :exc:`BaseExceptionGroup`. .. function:: redirect_stdout(new_target) @@ -796,7 +798,7 @@ executing that callback:: if result: stack.pop_all() -This allows the intended cleanup up behaviour to be made explicit up front, +This allows the intended cleanup behaviour to be made explicit up front, rather than requiring a separate flag variable. If a particular application uses this pattern a lot, it can be simplified diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 647832447de946..2b1fb9fdd29cd8 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -1,5 +1,5 @@ -:mod:`contextvars` --- Context Variables -======================================== +:mod:`!contextvars` --- Context Variables +========================================= .. module:: contextvars :synopsis: Context Variables @@ -15,7 +15,7 @@ function and the :class:`~contextvars.Context` class should be used to manage the current context in asynchronous frameworks. Context managers that have state should use Context Variables -instead of :func:`threading.local()` to prevent their state from +instead of :func:`threading.local` to prevent their state from bleeding to other code unexpectedly, when used in concurrent code. See also :pep:`567` for additional details. @@ -144,51 +144,89 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. - Every thread will have a different top-level :class:`~contextvars.Context` - object. This means that a :class:`ContextVar` object behaves in a similar - fashion to :func:`threading.local()` when values are assigned in different - threads. + Each thread has its own effective stack of :class:`!Context` objects. The + :term:`current context` is the :class:`!Context` object at the top of the + current thread's stack. All :class:`!Context` objects in the stacks are + considered to be *entered*. + + *Entering* a context, which can be done by calling its :meth:`~Context.run` + method, makes the context the current context by pushing it onto the top of + the current thread's context stack. + + *Exiting* from the current context, which can be done by returning from the + callback passed to the :meth:`~Context.run` method, restores the current + context to what it was before the context was entered by popping the context + off the top of the context stack. + + Since each thread has its own context stack, :class:`ContextVar` objects + behave in a similar fashion to :func:`threading.local` when values are + assigned in different threads. + + Attempting to enter an already entered context, including contexts entered in + other threads, raises a :exc:`RuntimeError`. + + After exiting a context, it can later be re-entered (from any thread). + + Any changes to :class:`ContextVar` values via the :meth:`ContextVar.set` + method are recorded in the current context. The :meth:`ContextVar.get` + method returns the value associated with the current context. Exiting a + context effectively reverts any changes made to context variables while the + context was entered (if needed, the values can be restored by re-entering the + context). Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) - Execute ``callable(*args, **kwargs)`` code in the context object - the *run* method is called on. Return the result of the execution - or propagate an exception if one occurred. + Enters the Context, executes ``callable(*args, **kwargs)``, then exits the + Context. Returns *callable*'s return value, or propagates an exception if + one occurred. + + Example: + + .. testcode:: + + import contextvars - Any changes to any context variables that *callable* makes will - be contained in the context object:: + var = contextvars.ContextVar('var') + var.set('spam') + print(var.get()) # 'spam' - var = ContextVar('var') - var.set('spam') + ctx = contextvars.copy_context() - def main(): - # 'var' was set to 'spam' before - # calling 'copy_context()' and 'ctx.run(main)', so: - # var.get() == ctx[var] == 'spam' + def main(): + # 'var' was set to 'spam' before + # calling 'copy_context()' and 'ctx.run(main)', so: + print(var.get()) # 'spam' + print(ctx[var]) # 'spam' - var.set('ham') + var.set('ham') - # Now, after setting 'var' to 'ham': - # var.get() == ctx[var] == 'ham' + # Now, after setting 'var' to 'ham': + print(var.get()) # 'ham' + print(ctx[var]) # 'ham' - ctx = copy_context() + # Any changes that the 'main' function makes to 'var' + # will be contained in 'ctx'. + ctx.run(main) - # Any changes that the 'main' function makes to 'var' - # will be contained in 'ctx'. - ctx.run(main) + # The 'main()' function was run in the 'ctx' context, + # so changes to 'var' are contained in it: + print(ctx[var]) # 'ham' - # The 'main()' function was run in the 'ctx' context, - # so changes to 'var' are contained in it: - # ctx[var] == 'ham' + # However, outside of 'ctx', 'var' is still set to 'spam': + print(var.get()) # 'spam' - # However, outside of 'ctx', 'var' is still set to 'spam': - # var.get() == 'spam' + .. testoutput:: + :hide: - The method raises a :exc:`RuntimeError` when called on the same - context object from more than one OS thread, or when called - recursively. + spam + spam + spam + ham + ham + ham + spam .. method:: copy() @@ -254,7 +292,7 @@ client:: # without passing it explicitly to this function. client_addr = client_addr_var.get() - return f'Good bye, client @ {client_addr}\n'.encode() + return f'Good bye, client @ {client_addr}\r\n'.encode() async def handle_request(reader, writer): addr = writer.transport.get_extra_info('socket').getpeername() @@ -268,9 +306,10 @@ client:: print(line) if not line.strip(): break - writer.write(line) - writer.write(render_goodbye()) + writer.write(b'HTTP/1.1 200 OK\r\n') # status line + writer.write(b'\r\n') # headers + writer.write(render_goodbye()) # body writer.close() async def main(): @@ -282,5 +321,6 @@ client:: asyncio.run(main()) - # To test it you can use telnet: + # To test it you can use telnet or curl: # telnet 127.0.0.1 8081 + # curl 127.0.0.1:8081 diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 74333b2e934814..95b41f988a035b 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -1,5 +1,5 @@ -:mod:`copy` --- Shallow and deep copy operations -================================================ +:mod:`!copy` --- Shallow and deep copy operations +================================================= .. module:: copy :synopsis: Shallow and deep copy operations. diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 2a28c043f80723..6e3144824ebe91 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -1,5 +1,5 @@ -:mod:`copyreg` --- Register :mod:`pickle` support functions -=========================================================== +:mod:`!copyreg` --- Register :mod:`!pickle` support functions +============================================================= .. module:: copyreg :synopsis: Register pickle support functions. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 4ee7820585d3a2..533cdf13974be6 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -1,5 +1,5 @@ -:mod:`csv` --- CSV File Reading and Writing -=========================================== +:mod:`!csv` --- CSV File Reading and Writing +============================================ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. @@ -156,8 +156,10 @@ The :mod:`csv` module defines the following classes: The *fieldnames* parameter is a :term:`sequence`. If *fieldnames* is omitted, the values in the first row of file *f* will be used as the - fieldnames. Regardless of how the fieldnames are determined, the - dictionary preserves their original ordering. + fieldnames and will be omitted from the results. If + *fieldnames* is provided, they will be used and the first row will be + included in the results. Regardless of how the fieldnames are determined, + the dictionary preserves their original ordering. If a row has more fields than fieldnames, the remaining data is put in a list and stored with the fieldname specified by *restkey* (which defaults @@ -347,8 +349,8 @@ The :mod:`csv` module defines the following constants: ``None``. This is similar to :data:`QUOTE_ALL`, except that if a field value is ``None`` an empty (unquoted) string is written. - Instructs :class:`reader` objects to interpret an empty (unquoted) field as None and - to otherwise behave as :data:`QUOTE_ALL`. + Instructs :class:`reader` objects to interpret an empty (unquoted) field + as ``None`` and to otherwise behave as :data:`QUOTE_ALL`. .. versionadded:: 3.12 diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index ef3a9a0f5898af..99909de20ef439 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1,5 +1,5 @@ -:mod:`ctypes` --- A foreign function library for Python -======================================================= +:mod:`!ctypes` --- A foreign function library for Python +======================================================== .. module:: ctypes :synopsis: A foreign function library for Python. @@ -51,7 +51,7 @@ function call fails. Here are some examples for Windows. Note that ``msvcrt`` is the MS standard C -library containing most standard C functions, and uses the cdecl calling +library containing most standard C functions, and uses the ``cdecl`` calling convention:: >>> from ctypes import * @@ -93,7 +93,6 @@ Accessing functions from loaded dlls Functions are accessed as attributes of dll objects:: - >>> from ctypes import * >>> libc.printf <_FuncPtr object at 0x...> >>> print(windll.kernel32.GetModuleHandleA) # doctest: +WINDOWS @@ -108,7 +107,7 @@ Functions are accessed as attributes of dll objects:: Note that win32 system dlls like ``kernel32`` and ``user32`` often export ANSI as well as UNICODE versions of a function. The UNICODE version is exported with -an ``W`` appended to the name, while the ANSI version is exported with an ``A`` +a ``W`` appended to the name, while the ANSI version is exported with an ``A`` appended to the name. The win32 ``GetModuleHandle`` function, which returns a *module handle* for a given module name, has the following C prototype, and a macro is used to expose one of them as ``GetModuleHandle`` depending on whether @@ -200,7 +199,7 @@ calls). Python objects that can directly be used as parameters in these function calls. ``None`` is passed as a C ``NULL`` pointer, bytes objects and strings are passed as pointer to the memory block that contains their data (:c:expr:`char *` or -:c:expr:`wchar_t *`). Python integers are passed as the platforms default C +:c:expr:`wchar_t *`). Python integers are passed as the platform's default C :c:expr:`int` type, their value is masked to fit into the C type. Before we move on calling functions with other parameter types, we have to learn @@ -267,6 +266,20 @@ Fundamental data types (1) The constructor accepts any object with a truth value. +Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported, the following +complex types are available: + ++----------------------------------+---------------------------------+-----------------+ +| ctypes type | C type | Python type | ++==================================+=================================+=================+ +| :class:`c_float_complex` | :c:expr:`float complex` | complex | ++----------------------------------+---------------------------------+-----------------+ +| :class:`c_double_complex` | :c:expr:`double complex` | complex | ++----------------------------------+---------------------------------+-----------------+ +| :class:`c_longdouble_complex` | :c:expr:`long double complex` | complex | ++----------------------------------+---------------------------------+-----------------+ + + All these types can be created by calling them with an optional initializer of the correct type and value:: @@ -384,7 +397,7 @@ as calling functions with a fixed number of parameters. On some platforms, and i particular ARM64 for Apple Platforms, the calling convention for variadic functions is different than that for regular functions. -On those platforms it is required to specify the :attr:`~_FuncPtr.argtypes` +On those platforms it is required to specify the :attr:`~_CFuncPtr.argtypes` attribute for the regular, non-variadic, function arguments: .. code-block:: python3 @@ -392,7 +405,7 @@ attribute for the regular, non-variadic, function arguments: libc.printf.argtypes = [ctypes.c_char_p] Because specifying the attribute does not inhibit portability it is advised to always -specify :attr:`~_FuncPtr.argtypes` for all variadic functions. +specify :attr:`~_CFuncPtr.argtypes` for all variadic functions. .. _ctypes-calling-functions-with-own-custom-data-types: @@ -427,9 +440,9 @@ Specifying the required argument types (function prototypes) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to specify the required argument types of functions exported from -DLLs by setting the :attr:`~_FuncPtr.argtypes` attribute. +DLLs by setting the :attr:`~_CFuncPtr.argtypes` attribute. -:attr:`~_FuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is +:attr:`~_CFuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is probably not a good example here, because it takes a variable number and different types of parameters depending on the format string, on the other hand this is quite handy to experiment with this feature):: @@ -454,7 +467,7 @@ prototype for a C function), and tries to convert the arguments to valid types:: If you have defined your own classes which you pass to function calls, you have to implement a :meth:`~_CData.from_param` class method for them to be able to use them -in the :attr:`~_FuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives +in the :attr:`~_CFuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to @@ -477,7 +490,7 @@ Return types By default functions are assumed to return the C :c:expr:`int` type. Other -return types can be specified by setting the :attr:`~_FuncPtr.restype` attribute of the +return types can be specified by setting the :attr:`~_CFuncPtr.restype` attribute of the function object. The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` @@ -486,7 +499,7 @@ specify the :attr:`!restype` attribute:: >>> libc.time.restype = c_time_t -The argument types can be specified using :attr:`~_FuncPtr.argtypes`:: +The argument types can be specified using :attr:`~_CFuncPtr.argtypes`:: >>> libc.time.argtypes = (POINTER(c_time_t),) @@ -509,7 +522,7 @@ a string pointer and a char, and returns a pointer to a string:: >>> If you want to avoid the :func:`ord("x") ` calls above, you can set the -:attr:`~_FuncPtr.argtypes` attribute, and the second argument will be converted from a +:attr:`~_CFuncPtr.argtypes` attribute, and the second argument will be converted from a single character Python bytes object into a C char: .. doctest:: @@ -528,7 +541,7 @@ single character Python bytes object into a C char: >>> You can also use a callable Python object (a function or a class for example) as -the :attr:`~_FuncPtr.restype` attribute, if the foreign function returns an integer. The +the :attr:`~_CFuncPtr.restype` attribute, if the foreign function returns an integer. The callable will be called with the *integer* the C function returns, and the result of this call will be used as the result of your function call. This is useful to check for error return values and automatically raise an exception:: @@ -556,7 +569,7 @@ get the string representation of an error code, and *returns* an exception. :func:`GetLastError` to retrieve it. Please note that a much more powerful error checking mechanism is available -through the :attr:`~_FuncPtr.errcheck` attribute; +through the :attr:`~_CFuncPtr.errcheck` attribute; see the reference manual for details. @@ -662,14 +675,22 @@ for debugging because they can provide useful information:: guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. -Structure/union alignment and byte order -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Structure/union layout, alignment and byte order +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, Structure and Union fields are laid out in the same way the C +compiler does it. It is possible to override this behavior entirely by specifying a +:attr:`~Structure._layout_` class attribute in the subclass definition; see +the attribute documentation for details. -By default, Structure and Union fields are aligned in the same way the C -compiler does it. It is possible to override this behavior by specifying a -:attr:`~Structure._pack_` class attribute in the subclass definition. -This must be set to a positive integer and specifies the maximum alignment for the fields. -This is what ``#pragma pack(n)`` also does in MSVC. +It is possible to specify the maximum alignment for the fields by setting +the :attr:`~Structure._pack_` class attribute to a positive integer. +This matches what ``#pragma pack(n)`` does in MSVC. + +It is also possible to set a minimum alignment for how the subclass itself is packed in the +same way ``#pragma align(n)`` works in MSVC. +This can be achieved by specifying a ::attr:`~Structure._align_` class attribute +in the subclass definition. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -856,7 +877,7 @@ Type conversions ^^^^^^^^^^^^^^^^ Usually, ctypes does strict type checking. This means, if you have -``POINTER(c_int)`` in the :attr:`~_FuncPtr.argtypes` list of a function or as the type of +``POINTER(c_int)`` in the :attr:`~_CFuncPtr.argtypes` list of a function or as the type of a member field in a structure definition, only instances of exactly the same type are accepted. There are some exceptions to this rule, where ctypes accepts other objects. For example, you can pass compatible array instances instead of @@ -877,7 +898,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`~_FuncPtr.argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`~_CFuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -1113,10 +1134,6 @@ api:: >>> print(hex(version.value)) 0x30c00a0 -If the interpreter would have been started with :option:`-O`, the sample would -have printed ``c_long(1)``, or ``c_long(2)`` if :option:`-OO` would have been -specified. - An extended example which also demonstrates the use of pointers accesses the :c:data:`PyImport_FrozenModules` pointer exported by Python. @@ -1335,8 +1352,9 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On macOS, :func:`~ctypes.util.find_library` tries several predefined naming schemes and paths -to locate the library, and returns a full pathname if successful:: +On macOS and Android, :func:`~ctypes.util.find_library` uses the system's +standard naming schemes and paths to locate the library, and returns a full +pathname if successful:: >>> from ctypes.util import find_library >>> find_library("c") @@ -1441,7 +1459,7 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms :c:func:`!dlopen` or +parameter, otherwise the underlying platform's :c:func:`!dlopen` or :c:func:`!LoadLibrary` function is used to load the library into the process, and to get a handle to it. @@ -1452,7 +1470,7 @@ configurable. The *use_errno* parameter, when set to true, enables a ctypes mechanism that allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the systems :data:`errno` +:mod:`ctypes` maintains a thread-local copy of the system's :data:`errno` variable; if you call foreign functions created with ``use_errno=True`` then the :data:`errno` value before the function call is swapped with the ctypes private copy, the same happens immediately after the function call. @@ -1609,10 +1627,20 @@ As explained in the previous section, foreign functions can be accessed as attributes of loaded shared libraries. The function objects created in this way by default accept any number of arguments, accept any ctypes data instances as arguments, and return the default result type specified by the library loader. -They are instances of a private class: +They are instances of a private local class :class:`!_FuncPtr` (not exposed +in :mod:`!ctypes`) which inherits from the private :class:`_CFuncPtr` class: + +.. doctest:: + + >>> import ctypes + >>> lib = ctypes.CDLL(None) + >>> issubclass(lib._FuncPtr, ctypes._CFuncPtr) + True + >>> lib._FuncPtr is ctypes._CFuncPtr + False -.. class:: _FuncPtr +.. class:: _CFuncPtr Base class for C callable foreign functions. @@ -1778,7 +1806,7 @@ different ways, depending on the type and number of the parameters in the call: The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. -*paramflags* must be a tuple of the same length as :attr:`~_FuncPtr.argtypes`. +*paramflags* must be a tuple of the same length as :attr:`~_CFuncPtr.argtypes`. Each item in this tuple contains further information about a parameter, it must be a tuple containing one, two, or three items. @@ -1849,7 +1877,7 @@ value if there is a single one, or a tuple containing the output parameter values when there are more than one, so the GetWindowRect function now returns a RECT instance, when called. -Output parameters can be combined with the :attr:`~_FuncPtr.errcheck` protocol to do +Output parameters can be combined with the :attr:`~_CFuncPtr.errcheck` protocol to do further output processing and error checking. The win32 ``GetWindowRect`` api function returns a ``BOOL`` to signal success or failure, so this function could do the error checking, and raises an exception when the api call failed:: @@ -1862,7 +1890,7 @@ do the error checking, and raises an exception when the api call failed:: >>> GetWindowRect.errcheck = errcheck >>> -If the :attr:`~_FuncPtr.errcheck` function returns the argument tuple it receives +If the :attr:`~_CFuncPtr.errcheck` function returns the argument tuple it receives unchanged, :mod:`ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them @@ -2077,13 +2105,13 @@ Utility functions Does the same as the C ``sizeof`` operator. -.. function:: string_at(address, size=-1) +.. function:: string_at(ptr, size=-1) - This function returns the C string starting at memory address *address* as a bytes - object. If size is specified, it is used as size, otherwise the string is assumed + Return the byte string at *void \*ptr*. + If *size* is specified, it is used as size, otherwise the string is assumed to be zero-terminated. - .. audit-event:: ctypes.string_at address,size ctypes.string_at + .. audit-event:: ctypes.string_at ptr,size ctypes.string_at .. function:: WinError(code=None, descr=None) @@ -2099,14 +2127,14 @@ Utility functions alias of :exc:`OSError`. -.. function:: wstring_at(address, size=-1) +.. function:: wstring_at(ptr, size=-1) - This function returns the wide character string starting at memory address - *address* as a string. If *size* is specified, it is used as the number of + Return the wide-character string at *void \*ptr*. + If *size* is specified, it is used as the number of characters of the string, otherwise the string is assumed to be zero-terminated. - .. audit-event:: ctypes.wstring_at address,size ctypes.wstring_at + .. audit-event:: ctypes.wstring_at ptr,size ctypes.wstring_at .. _ctypes-data-types: @@ -2162,7 +2190,7 @@ Data types This method adapts *obj* to a ctypes type. It is called with the actual object used in a foreign function call when the type is present in the - foreign function's :attr:`~_FuncPtr.argtypes` tuple; + foreign function's :attr:`~_CFuncPtr.argtypes` tuple; it must return an object that can be used as a function call parameter. All ctypes data types have a default implementation of this classmethod @@ -2228,7 +2256,7 @@ Fundamental data types Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently converted to native Python types. In other words, if a foreign function has a -:attr:`~_FuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes +:attr:`~_CFuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes object, *not* a :class:`c_char_p` instance. .. XXX above is false, it actually returns a Unicode string @@ -2280,6 +2308,30 @@ These are the fundamental ctypes data types: optional float initializer. +.. class:: c_double_complex + + Represents the C :c:expr:`double complex` datatype, if available. The + constructor accepts an optional :class:`complex` initializer. + + .. versionadded:: 3.14 + + +.. class:: c_float_complex + + Represents the C :c:expr:`float complex` datatype, if available. The + constructor accepts an optional :class:`complex` initializer. + + .. versionadded:: 3.14 + + +.. class:: c_longdouble_complex + + Represents the C :c:expr:`long double complex` datatype, if available. The + constructor accepts an optional :class:`complex` initializer. + + .. versionadded:: 3.14 + + .. class:: c_int Represents the C :c:expr:`signed int` datatype. The constructor accepts an @@ -2457,6 +2509,8 @@ Structured data types Abstract base class for unions in native byte order. + Unions share common attributes and behavior with structures; + see :class:`Structure` documentation for details. .. class:: BigEndianUnion(*args, **kw) @@ -2516,14 +2570,19 @@ fields, or any other data types containing pointer type fields. ... ] - The :attr:`_fields_` class variable must, however, be defined before the - type is first used (an instance is created, :func:`sizeof` is called on it, - and so on). Later assignments to the :attr:`_fields_` class variable will - raise an AttributeError. + The :attr:`!_fields_` class variable can only be set once. + Later assignments will raise an :exc:`AttributeError`. + + Additionally, the :attr:`!_fields_` class variable must be defined before + the structure or union type is first used: an instance or subclass is + created, :func:`sizeof` is called on it, and so on. + Later assignments to :attr:`!_fields_` will raise an :exc:`AttributeError`. + If :attr:`!_fields_` has not been set before such use, + the structure or union will have no own fields, as if :attr:`!_fields_` + was empty. - It is possible to define sub-subclasses of structure types, they inherit - the fields of the base class plus the :attr:`_fields_` defined in the - sub-subclass, if any. + Sub-subclasses of structure types inherit the fields of the base class + plus the :attr:`_fields_` defined in the sub-subclass, if any. .. attribute:: _pack_ @@ -2534,6 +2593,39 @@ fields, or any other data types containing pointer type fields. Setting this attribute to 0 is the same as not setting it at all. + .. attribute:: _align_ + + An optional small integer that allows overriding the alignment of + the structure when being packed or unpacked to/from memory. + Setting this attribute to 0 is the same as not setting it at all. + + .. versionadded:: 3.13 + + .. attribute:: _layout_ + + An optional string naming the struct/union layout. It can currently + be set to: + + - ``"ms"``: the layout used by the Microsoft compiler (MSVC). + On GCC and Clang, this layout can be selected with + ``__attribute__((ms_struct))``. + - ``"gcc-sysv"``: the layout used by GCC with the System V or “SysV-like” + data model, as used on Linux and macOS. + With this layout, :attr:`~Structure._pack_` must be unset or zero. + + If not set explicitly, ``ctypes`` will use a default that + matches the platform conventions. This default may change in future + Python releases (for example, when a new platform gains official support, + or when a difference between similar platforms is found). + Currently the default will be: + + - On Windows: ``"ms"`` + - When :attr:`~Structure._pack_` is specified: ``"ms"`` + - Otherwise: ``"gcc-sysv"`` + + :attr:`!_layout_` must already be defined when + :attr:`~Structure._fields_` is assigned, otherwise it will have no effect. + .. attribute:: _anonymous_ An optional sequence that lists the names of unnamed (anonymous) fields. @@ -2615,6 +2707,15 @@ Arrays and pointers Array subclass constructors accept positional arguments, used to initialize the elements in order. +.. function:: ARRAY(type, length) + + Create an array. + Equivalent to ``type * length``, where *type* is a + :mod:`ctypes` data type and *length* an integer. + + This function is :term:`soft deprecated` in favor of multiplication. + There are no plans to remove it. + .. class:: _Pointer diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index 410b76e77c025b..cb895664ff1b11 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -1,5 +1,5 @@ -:mod:`curses.ascii` --- Utilities for ASCII characters -====================================================== +:mod:`!curses.ascii` --- Utilities for ASCII characters +======================================================= .. module:: curses.ascii :synopsis: Constants and set-membership functions for ASCII characters. diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index d770c03c8375f4..11fd841d381f69 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -1,5 +1,5 @@ -:mod:`curses.panel` --- A panel stack extension for curses -========================================================== +:mod:`!curses.panel` --- A panel stack extension for curses +=========================================================== .. module:: curses.panel :synopsis: A panel stack extension that adds depth to curses windows. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 9b8a98f05f7cbb..6c7fc721a3e0fb 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1,5 +1,5 @@ -:mod:`curses` --- Terminal handling for character-cell displays -=============================================================== +:mod:`!curses` --- Terminal handling for character-cell displays +================================================================ .. module:: curses :synopsis: An interface to the curses library, providing portable @@ -21,6 +21,8 @@ for Windows, DOS, and possibly other systems as well. This extension module is designed to match the API of ncurses, an open-source curses library hosted on Linux and the BSD variants of Unix. +.. include:: ../includes/wasm-mobile-notavail.rst + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -922,7 +924,7 @@ the following methods and attributes: .. method:: window.getbegyx() - Return a tuple ``(y, x)`` of co-ordinates of upper-left corner. + Return a tuple ``(y, x)`` of coordinates of upper-left corner. .. method:: window.getbkgd() diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 4ada69d63abada..e34b2db0210960 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -1,5 +1,5 @@ -:mod:`dataclasses` --- Data Classes -=================================== +:mod:`!dataclasses` --- Data Classes +==================================== .. module:: dataclasses :synopsis: Generate special methods on user-defined classes. @@ -12,7 +12,7 @@ -------------- This module provides a decorator and functions for automatically -adding generated :term:`special method`\s such as :meth:`~object.__init__` and +adding generated :term:`special methods ` such as :meth:`~object.__init__` and :meth:`~object.__repr__` to user-defined classes. It was originally described in :pep:`557`. @@ -31,7 +31,7 @@ using :pep:`526` type annotations. For example, this code:: def total_cost(self) -> float: return self.unit_price * self.quantity_on_hand -will add, among other things, a :meth:`~object.__init__` that looks like:: +will add, among other things, a :meth:`!__init__` that looks like:: def __init__(self, name: str, unit_price: float, quantity_on_hand: int = 0): self.name = name @@ -39,7 +39,7 @@ will add, among other things, a :meth:`~object.__init__` that looks like:: self.quantity_on_hand = quantity_on_hand Note that this method is automatically added to the class: it is not -directly specified in the ``InventoryItem`` definition shown above. +directly specified in the :class:`!InventoryItem` definition shown above. .. versionadded:: 3.7 @@ -49,26 +49,26 @@ Module contents .. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) This function is a :term:`decorator` that is used to add generated - :term:`special method`\s to classes, as described below. + :term:`special methods ` to classes, as described below. - The :func:`dataclass` decorator examines the class to find + The ``@dataclass`` decorator examines the class to find ``field``\s. A ``field`` is defined as a class variable that has a :term:`type annotation `. With two - exceptions described below, nothing in :func:`dataclass` + exceptions described below, nothing in ``@dataclass`` examines the type specified in the variable annotation. The order of the fields in all of the generated methods is the order in which they appear in the class definition. - The :func:`dataclass` decorator will add various "dunder" methods to + The ``@dataclass`` decorator will add various "dunder" methods to the class, described below. If any of the added methods already exist in the class, the behavior depends on the parameter, as documented below. The decorator returns the same class that it is called on; no new class is created. - If :func:`dataclass` is used just as a simple decorator with no parameters, + If ``@dataclass`` is used just as a simple decorator with no parameters, it acts as if it has the default values documented in this - signature. That is, these three uses of :func:`dataclass` are + signature. That is, these three uses of ``@dataclass`` are equivalent:: @dataclass @@ -84,124 +84,132 @@ Module contents class C: ... - The parameters to :func:`dataclass` are: + The parameters to ``@dataclass`` are: - - ``init``: If true (the default), a :meth:`~object.__init__` method will be + - *init*: If true (the default), a :meth:`~object.__init__` method will be generated. - If the class already defines :meth:`~object.__init__`, this parameter is + If the class already defines :meth:`!__init__`, this parameter is ignored. - - ``repr``: If true (the default), a :meth:`~object.__repr__` method will be + - *repr*: If true (the default), a :meth:`~object.__repr__` method will be generated. The generated repr string will have the class name and the name and repr of each field, in the order they are defined in the class. Fields that are marked as being excluded from the repr are not included. For example: ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. - If the class already defines :meth:`~object.__repr__`, this parameter is + If the class already defines :meth:`!__repr__`, this parameter is ignored. - - ``eq``: If true (the default), an :meth:`~object.__eq__` method will be + - *eq*: If true (the default), an :meth:`~object.__eq__` method will be generated. This method compares the class as if it were a tuple of its fields, in order. Both instances in the comparison must be of the identical type. - If the class already defines :meth:`~object.__eq__`, this parameter is + If the class already defines :meth:`!__eq__`, this parameter is ignored. - - ``order``: If true (the default is ``False``), :meth:`~object.__lt__`, + - *order*: If true (the default is ``False``), :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be generated. These compare the class as if it were a tuple of its fields, in order. Both instances in the comparison must be of the - identical type. If ``order`` is true and ``eq`` is false, a + identical type. If *order* is true and *eq* is false, a :exc:`ValueError` is raised. - If the class already defines any of :meth:`~object.__lt__`, - :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, then + If the class already defines any of :meth:`!__lt__`, + :meth:`!__le__`, :meth:`!__gt__`, or :meth:`!__ge__`, then :exc:`TypeError` is raised. - - ``unsafe_hash``: If ``False`` (the default), a :meth:`~object.__hash__` method - is generated according to how ``eq`` and ``frozen`` are set. + - *unsafe_hash*: If ``False`` (the default), a :meth:`~object.__hash__` method + is generated according to how *eq* and *frozen* are set. - :meth:`~object.__hash__` is used by built-in :meth:`hash()`, and when objects are + :meth:`!__hash__` is used by built-in :meth:`hash`, and when objects are added to hashed collections such as dictionaries and sets. Having a - :meth:`~object.__hash__` implies that instances of the class are immutable. + :meth:`!__hash__` implies that instances of the class are immutable. Mutability is a complicated property that depends on the programmer's - intent, the existence and behavior of :meth:`~object.__eq__`, and the values of - the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. + intent, the existence and behavior of :meth:`!__eq__`, and the values of + the *eq* and *frozen* flags in the ``@dataclass`` decorator. - By default, :func:`dataclass` will not implicitly add a :meth:`~object.__hash__` + By default, ``@dataclass`` will not implicitly add a :meth:`~object.__hash__` method unless it is safe to do so. Neither will it add or change an - existing explicitly defined :meth:`~object.__hash__` method. Setting the class + existing explicitly defined :meth:`!__hash__` method. Setting the class attribute ``__hash__ = None`` has a specific meaning to Python, as - described in the :meth:`~object.__hash__` documentation. + described in the :meth:`!__hash__` documentation. - If :meth:`~object.__hash__` is not explicitly defined, or if it is set to ``None``, - then :func:`dataclass` *may* add an implicit :meth:`~object.__hash__` method. - Although not recommended, you can force :func:`dataclass` to create a - :meth:`~object.__hash__` method with ``unsafe_hash=True``. This might be the case + If :meth:`!__hash__` is not explicitly defined, or if it is set to ``None``, + then ``@dataclass`` *may* add an implicit :meth:`!__hash__` method. + Although not recommended, you can force ``@dataclass`` to create a + :meth:`!__hash__` method with ``unsafe_hash=True``. This might be the case if your class is logically immutable but can still be mutated. This is a specialized use case and should be considered carefully. - Here are the rules governing implicit creation of a :meth:`~object.__hash__` - method. Note that you cannot both have an explicit :meth:`~object.__hash__` + Here are the rules governing implicit creation of a :meth:`!__hash__` + method. Note that you cannot both have an explicit :meth:`!__hash__` method in your dataclass and set ``unsafe_hash=True``; this will result in a :exc:`TypeError`. - If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will - generate a :meth:`~object.__hash__` method for you. If ``eq`` is true and - ``frozen`` is false, :meth:`~object.__hash__` will be set to ``None``, marking it - unhashable (which it is, since it is mutable). If ``eq`` is false, - :meth:`~object.__hash__` will be left untouched meaning the :meth:`~object.__hash__` + If *eq* and *frozen* are both true, by default ``@dataclass`` will + generate a :meth:`!__hash__` method for you. If *eq* is true and + *frozen* is false, :meth:`!__hash__` will be set to ``None``, marking it + unhashable (which it is, since it is mutable). If *eq* is false, + :meth:`!__hash__` will be left untouched meaning the :meth:`!__hash__` method of the superclass will be used (if the superclass is :class:`object`, this means it will fall back to id-based hashing). - - ``frozen``: If true (the default is ``False``), assigning to fields will + - *frozen*: If true (the default is ``False``), assigning to fields will generate an exception. This emulates read-only frozen instances. If :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class, then :exc:`TypeError` is raised. See the discussion below. - - ``match_args``: If true (the default is ``True``), the - ``__match_args__`` tuple will be created from the list of + - *match_args*: If true (the default is ``True``), the + :attr:`~object.__match_args__` tuple will be created from the list of parameters to the generated :meth:`~object.__init__` method (even if - :meth:`~object.__init__` is not generated, see above). If false, or if - ``__match_args__`` is already defined in the class, then - ``__match_args__`` will not be generated. + :meth:`!__init__` is not generated, see above). If false, or if + :attr:`!__match_args__` is already defined in the class, then + :attr:`!__match_args__` will not be generated. .. versionadded:: 3.10 - - ``kw_only``: If true (the default value is ``False``), then all + - *kw_only*: If true (the default value is ``False``), then all fields will be marked as keyword-only. If a field is marked as keyword-only, then the only effect is that the :meth:`~object.__init__` parameter generated from a keyword-only field must be specified - with a keyword when :meth:`~object.__init__` is called. There is no + with a keyword when :meth:`!__init__` is called. There is no effect on any other aspect of dataclasses. See the :term:`parameter` glossary entry for details. Also see the :const:`KW_ONLY` section. .. versionadded:: 3.10 - - ``slots``: If true (the default is ``False``), :attr:`~object.__slots__` attribute + - *slots*: If true (the default is ``False``), :attr:`~object.__slots__` attribute will be generated and new class will be returned instead of the original one. - If :attr:`~object.__slots__` is already defined in the class, then :exc:`TypeError` + If :attr:`!__slots__` is already defined in the class, then :exc:`TypeError` is raised. + .. warning:: + Passing parameters to a base class :meth:`~object.__init_subclass__` + when using ``slots=True`` will result in a :exc:`TypeError`. + Either use ``__init_subclass__`` with no parameters + or use default values as a workaround. + See :gh:`91126` for full details. + .. versionadded:: 3.10 .. versionchanged:: 3.11 - If a field name is already included in the ``__slots__`` - of a base class, it will not be included in the generated ``__slots__`` + If a field name is already included in the :attr:`!__slots__` + of a base class, it will not be included in the generated :attr:`!__slots__` to prevent :ref:`overriding them `. - Therefore, do not use ``__slots__`` to retrieve the field names of a + Therefore, do not use :attr:`!__slots__` to retrieve the field names of a dataclass. Use :func:`fields` instead. To be able to determine inherited slots, - base class ``__slots__`` may be any iterable, but *not* an iterator. + base class :attr:`!__slots__` may be any iterable, but *not* an iterator. - - ``weakref_slot``: If true (the default is ``False``), add a slot + - *weakref_slot*: If true (the default is ``False``), add a slot named "__weakref__", which is required to make an instance - weakref-able. It is an error to specify ``weakref_slot=True`` + :func:`weakref-able `. + It is an error to specify ``weakref_slot=True`` without also specifying ``slots=True``. .. versionadded:: 3.11 @@ -214,7 +222,7 @@ Module contents a: int # 'a' has no default value b: int = 0 # assign a default value for 'b' - In this example, both ``a`` and ``b`` will be included in the added + In this example, both :attr:`!a` and :attr:`!b` will be included in the added :meth:`~object.__init__` method, which will be defined as:: def __init__(self, a: int, b: int = 0): @@ -223,13 +231,13 @@ Module contents follows a field with a default value. This is true whether this occurs in a single class, or as a result of class inheritance. -.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING) +.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None) For common and simple use cases, no other functionality is required. There are, however, some dataclass features that require additional per-field information. To satisfy this need for additional information, you can replace the default field value - with a call to the provided :func:`field` function. For example:: + with a call to the provided :func:`!field` function. For example:: @dataclass class C: @@ -243,27 +251,27 @@ Module contents used because ``None`` is a valid value for some parameters with a distinct meaning. No code should directly use the :const:`MISSING` value. - The parameters to :func:`field` are: + The parameters to :func:`!field` are: - - ``default``: If provided, this will be the default value for this - field. This is needed because the :meth:`field` call itself + - *default*: If provided, this will be the default value for this + field. This is needed because the :func:`!field` call itself replaces the normal position of the default value. - - ``default_factory``: If provided, it must be a zero-argument + - *default_factory*: If provided, it must be a zero-argument callable that will be called when a default value is needed for this field. Among other purposes, this can be used to specify fields with mutable default values, as discussed below. It is an - error to specify both ``default`` and ``default_factory``. + error to specify both *default* and *default_factory*. - - ``init``: If true (the default), this field is included as a + - *init*: If true (the default), this field is included as a parameter to the generated :meth:`~object.__init__` method. - - ``repr``: If true (the default), this field is included in the + - *repr*: If true (the default), this field is included in the string returned by the generated :meth:`~object.__repr__` method. - - ``hash``: This can be a bool or ``None``. If true, this field is + - *hash*: This can be a bool or ``None``. If true, this field is included in the generated :meth:`~object.__hash__` method. If ``None`` (the - default), use the value of ``compare``: this would normally be + default), use the value of *compare*: this would normally be the expected behavior. A field should be considered in the hash if it's used for comparisons. Setting this value to anything other than ``None`` is discouraged. @@ -274,11 +282,11 @@ Module contents fields that contribute to the type's hash value. Even if a field is excluded from the hash, it will still be used for comparisons. - - ``compare``: If true (the default), this field is included in the + - *compare*: If true (the default), this field is included in the generated equality and comparison methods (:meth:`~object.__eq__`, :meth:`~object.__gt__`, et al.). - - ``metadata``: This can be a mapping or None. None is treated as + - *metadata*: This can be a mapping or ``None``. ``None`` is treated as an empty dict. This value is wrapped in :func:`~types.MappingProxyType` to make it read-only, and exposed on the :class:`Field` object. It is not used at all by Data @@ -286,17 +294,21 @@ Module contents Multiple third-parties can each have their own key, to use as a namespace in the metadata. - - ``kw_only``: If true, this field will be marked as keyword-only. + - *kw_only*: If true, this field will be marked as keyword-only. This is used when the generated :meth:`~object.__init__` method's parameters are computed. .. versionadded:: 3.10 + - ``doc``: optional docstring for this field. + + .. versionadded:: 3.13 + If the default value of a field is specified by a call to - :func:`field()`, then the class attribute for this field will be - replaced by the specified ``default`` value. If no ``default`` is + :func:`!field`, then the class attribute for this field will be + replaced by the specified *default* value. If *default* is not provided, then the class attribute will be deleted. The intent is - that after the :func:`dataclass` decorator runs, the class + that after the :func:`@dataclass ` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -308,21 +320,21 @@ Module contents z: int = field(repr=False, default=10) t: int = 20 - The class attribute ``C.z`` will be ``10``, the class attribute - ``C.t`` will be ``20``, and the class attributes ``C.x`` and - ``C.y`` will not be set. + The class attribute :attr:`!C.z` will be ``10``, the class attribute + :attr:`!C.t` will be ``20``, and the class attributes :attr:`!C.x` and + :attr:`!C.y` will not be set. .. class:: Field - :class:`Field` objects describe each defined field. These objects + :class:`!Field` objects describe each defined field. These objects are created internally, and are returned by the :func:`fields` module-level method (see below). Users should never instantiate a - :class:`Field` object directly. Its documented attributes are: + :class:`!Field` object directly. Its documented attributes are: - - ``name``: The name of the field. - - ``type``: The type of the field. - - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, - ``compare``, ``metadata``, and ``kw_only`` have the identical + - :attr:`!name`: The name of the field. + - :attr:`!type`: The type of the field. + - :attr:`!default`, :attr:`!default_factory`, :attr:`!init`, :attr:`!repr`, :attr:`!hash`, + :attr:`!compare`, :attr:`!metadata`, and :attr:`!kw_only` have the identical meaning and values as they do in the :func:`field` function. Other attributes may exist, but they are private and must not be @@ -337,13 +349,13 @@ Module contents .. function:: asdict(obj, *, dict_factory=dict) - Converts the dataclass ``obj`` to a dict (by using the - factory function ``dict_factory``). Each dataclass is converted + Converts the dataclass *obj* to a dict (by using the + factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, lists, and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. - Example of using :func:`asdict` on nested dataclasses:: + Example of using :func:`!asdict` on nested dataclasses:: @dataclass class Point: @@ -362,15 +374,15 @@ Module contents To create a shallow copy, the following workaround may be used:: - dict((field.name, getattr(obj, field.name)) for field in fields(obj)) + {field.name: getattr(obj, field.name) for field in fields(obj)} - :func:`asdict` raises :exc:`TypeError` if ``obj`` is not a dataclass + :func:`!asdict` raises :exc:`TypeError` if *obj* is not a dataclass instance. .. function:: astuple(obj, *, tuple_factory=tuple) - Converts the dataclass ``obj`` to a tuple (by using the - factory function ``tuple_factory``). Each dataclass is converted + Converts the dataclass *obj* to a tuple (by using the + factory function *tuple_factory*). Each dataclass is converted to a tuple of its field values. dataclasses, dicts, lists, and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. @@ -384,28 +396,33 @@ Module contents tuple(getattr(obj, field.name) for field in dataclasses.fields(obj)) - :func:`astuple` raises :exc:`TypeError` if ``obj`` is not a dataclass + :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, decorator=dataclass) - Creates a new dataclass with name ``cls_name``, fields as defined - in ``fields``, base classes as given in ``bases``, and initialized - with a namespace as given in ``namespace``. ``fields`` is an + Creates a new dataclass with name *cls_name*, fields as defined + in *fields*, base classes as given in *bases*, and initialized + with a namespace as given in *namespace*. *fields* is an iterable whose elements are each either ``name``, ``(name, type)``, or ``(name, type, Field)``. If just ``name`` is supplied, - ``typing.Any`` is used for ``type``. The values of ``init``, - ``repr``, ``eq``, ``order``, ``unsafe_hash``, ``frozen``, - ``match_args``, ``kw_only``, ``slots``, and ``weakref_slot`` have - the same meaning as they do in :func:`dataclass`. + :data:`typing.Any` is used for ``type``. The values of *init*, + *repr*, *eq*, *order*, *unsafe_hash*, *frozen*, + *match_args*, *kw_only*, *slots*, and *weakref_slot* have + the same meaning as they do in :func:`@dataclass `. - If ``module`` is defined, the ``__module__`` attribute + If *module* is defined, the :attr:`!__module__` attribute of the dataclass is set to that value. By default, it is set to the module name of the caller. + The *decorator* parameter is a callable that will be used to create the dataclass. + It should take the class object as a first argument and the same keyword arguments + as :func:`@dataclass `. By default, the :func:`@dataclass ` + function is used. + This function is not strictly required, because any Python - mechanism for creating a new class with ``__annotations__`` can - then apply the :func:`dataclass` function to convert that class to + mechanism for creating a new class with :attr:`!__annotations__` can + then apply the :func:`@dataclass ` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -426,40 +443,43 @@ Module contents def add_one(self): return self.x + 1 + .. versionadded:: 3.14 + Added the *decorator* parameter. + .. function:: replace(obj, /, **changes) - Creates a new object of the same type as ``obj``, replacing - fields with values from ``changes``. If ``obj`` is not a Data - Class, raises :exc:`TypeError`. If values in ``changes`` do not - specify fields, raises :exc:`TypeError`. + Creates a new object of the same type as *obj*, replacing + fields with values from *changes*. If *obj* is not a Data + Class, raises :exc:`TypeError`. If keys in *changes* are not + field names of the given dataclass, raises :exc:`TypeError`. The newly returned object is created by calling the :meth:`~object.__init__` method of the dataclass. This ensures that :meth:`__post_init__`, if present, is also called. Init-only variables without default values, if any exist, must be - specified on the call to :func:`replace` so that they can be passed to - :meth:`~object.__init__` and :meth:`__post_init__`. + specified on the call to :func:`!replace` so that they can be passed to + :meth:`!__init__` and :meth:`__post_init__`. - It is an error for ``changes`` to contain any fields that are + It is an error for *changes* to contain any fields that are defined as having ``init=False``. A :exc:`ValueError` will be raised in this case. Be forewarned about how ``init=False`` fields work during a call to - :func:`replace`. They are not copied from the source object, but + :func:`!replace`. They are not copied from the source object, but rather are initialized in :meth:`__post_init__`, if they're initialized at all. It is expected that ``init=False`` fields will be rarely and judiciously used. If they are used, it might be wise to have alternate class constructors, or perhaps a custom - ``replace()`` (or similarly named) method which handles instance + :func:`!replace` (or similarly named) method which handles instance copying. Dataclass instances are also supported by generic function :func:`copy.replace`. .. function:: is_dataclass(obj) - Return ``True`` if its parameter is a dataclass or an instance of one, - otherwise return ``False``. + Return ``True`` if its parameter is a dataclass (including subclasses of a + dataclass) or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not @@ -475,11 +495,11 @@ Module contents .. data:: KW_ONLY A sentinel value used as a type annotation. Any fields after a - pseudo-field with the type of :const:`KW_ONLY` are marked as + pseudo-field with the type of :const:`!KW_ONLY` are marked as keyword-only fields. Note that a pseudo-field of type - :const:`KW_ONLY` is otherwise completely ignored. This includes the + :const:`!KW_ONLY` is otherwise completely ignored. This includes the name of such a field. By convention, a name of ``_`` is used for a - :const:`KW_ONLY` field. Keyword-only fields signify + :const:`!KW_ONLY` field. Keyword-only fields signify :meth:`~object.__init__` parameters that must be specified as keywords when the class is instantiated. @@ -495,7 +515,7 @@ Module contents p = Point(0, y=1.5, z=2.0) In a single dataclass, it is an error to specify more than one - field whose type is :const:`KW_ONLY`. + field whose type is :const:`!KW_ONLY`. .. versionadded:: 3.10 @@ -513,11 +533,11 @@ Post-init processing .. function:: __post_init__() When defined on the class, it will be called by the generated - :meth:`~object.__init__`, normally as ``self.__post_init__()``. + :meth:`~object.__init__`, normally as :meth:`!self.__post_init__`. However, if any ``InitVar`` fields are defined, they will also be - passed to :meth:`__post_init__` in the order they were defined in the - class. If no :meth:`~object.__init__` method is generated, then - :meth:`__post_init__` will not automatically be called. + passed to :meth:`!__post_init__` in the order they were defined in the + class. If no :meth:`!__init__` method is generated, then + :meth:`!__post_init__` will not automatically be called. Among other uses, this allows for initializing field values that depend on one or more other fields. For example:: @@ -531,8 +551,8 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`dataclass` does not call base -class :meth:`~object.__init__` methods. If the base class has an :meth:`~object.__init__` method +The :meth:`~object.__init__` method generated by :func:`@dataclass ` does not call base +class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: @@ -548,29 +568,33 @@ that has to be called, it is common to call this method in a def __post_init__(self): super().__init__(self.side, self.side) -Note, however, that in general the dataclass-generated :meth:`~object.__init__` methods +Note, however, that in general the dataclass-generated :meth:`!__init__` methods don't need to be called, since the derived dataclass will take care of initializing all fields of any base class that is a dataclass itself. See the section below on init-only variables for ways to pass -parameters to :meth:`__post_init__`. Also see the warning about how +parameters to :meth:`!__post_init__`. Also see the warning about how :func:`replace` handles ``init=False`` fields. +.. _dataclasses-class-variables: + Class variables --------------- -One of the few places where :func:`dataclass` actually inspects the type +One of the few places where :func:`@dataclass ` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is -``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded +:data:`typing.ClassVar`. If a field is a ``ClassVar``, it is excluded from consideration as a field and is ignored by the dataclass mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the module-level :func:`fields` function. +.. _dataclasses-init-only-variables: + Init-only variables ------------------- -Another place where :func:`dataclass` inspects a type annotation is to +Another place where :func:`@dataclass ` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing if the type of a field is of type ``dataclasses.InitVar``. If a field is an ``InitVar``, it is considered a pseudo-field called an init-only @@ -595,14 +619,16 @@ value is not provided when creating the class:: c = C(10, database=my_database) -In this case, :func:`fields` will return :class:`Field` objects for ``i`` and -``j``, but not for ``database``. +In this case, :func:`fields` will return :class:`Field` objects for :attr:`!i` and +:attr:`!j`, but not for :attr:`!database`. + +.. _dataclasses-frozen: Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :meth:`dataclass` decorator you can +by passing ``frozen=True`` to the :func:`@dataclass ` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. @@ -611,10 +637,14 @@ There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and must use :meth:`!object.__setattr__`. +.. Make sure to not remove "object" from "object.__setattr__" in the above markup! + +.. _dataclasses-inheritance: + Inheritance ----------- -When the dataclass is being created by the :meth:`dataclass` decorator, +When the dataclass is being created by the :func:`@dataclass ` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -634,15 +664,15 @@ example:: z: int = 10 x: int = 15 -The final list of fields is, in order, ``x``, ``y``, ``z``. The final -type of ``x`` is ``int``, as specified in class ``C``. +The final list of fields is, in order, :attr:`!x`, :attr:`!y`, :attr:`!z`. The final +type of :attr:`!x` is :class:`int`, as specified in class :class:`!C`. -The generated :meth:`~object.__init__` method for ``C`` will look like:: +The generated :meth:`~object.__init__` method for :class:`!C` will look like:: def __init__(self, x: int = 15, y: int = 0, z: int = 10): -Re-ordering of keyword-only parameters in :meth:`~object.__init__` ------------------------------------------------------------------- +Re-ordering of keyword-only parameters in :meth:`!__init__` +----------------------------------------------------------- After the parameters needed for :meth:`~object.__init__` are computed, any keyword-only parameters are moved to come after all regular @@ -650,8 +680,8 @@ keyword-only parameters are moved to come after all regular keyword-only parameters are implemented in Python: they must come after non-keyword-only parameters. -In this example, ``Base.y``, ``Base.w``, and ``D.t`` are keyword-only -fields, and ``Base.x`` and ``D.z`` are regular fields:: +In this example, :attr:`!Base.y`, :attr:`!Base.w`, and :attr:`!D.t` are keyword-only +fields, and :attr:`!Base.x` and :attr:`!D.z` are regular fields:: @dataclass class Base: @@ -665,7 +695,7 @@ fields, and ``Base.x`` and ``D.z`` are regular fields:: z: int = 10 t: int = field(kw_only=True, default=0) -The generated :meth:`~object.__init__` method for ``D`` will look like:: +The generated :meth:`!__init__` method for :class:`!D` will look like:: def __init__(self, x: Any = 15.0, z: int = 10, *, y: int = 0, w: int = 1, t: int = 0): @@ -674,22 +704,22 @@ the list of fields: parameters derived from regular fields are followed by parameters derived from keyword-only fields. The relative ordering of keyword-only parameters is maintained in the -re-ordered :meth:`~object.__init__` parameter list. +re-ordered :meth:`!__init__` parameter list. Default factory functions ------------------------- -If a :func:`field` specifies a ``default_factory``, it is called with +If a :func:`field` specifies a *default_factory*, it is called with zero arguments when a default value for the field is needed. For example, to create a new instance of a list, use:: mylist: list = field(default_factory=list) If a field is excluded from :meth:`~object.__init__` (using ``init=False``) -and the field also specifies ``default_factory``, then the default +and the field also specifies *default_factory*, then the default factory function will always be called from the generated -:meth:`~object.__init__` function. This happens because there is no other +:meth:`!__init__` function. This happens because there is no other way to give the field an initial value. Mutable default values @@ -710,8 +740,8 @@ Consider this example, not using dataclasses:: assert o1.x == [1, 2] assert o1.x is o2.x -Note that the two instances of class ``C`` share the same class -variable ``x``, as expected. +Note that the two instances of class :class:`!C` share the same class +variable :attr:`!x`, as expected. Using dataclasses, *if* this code was valid:: @@ -719,7 +749,7 @@ Using dataclasses, *if* this code was valid:: class D: x: list = [] # This code raises ValueError def add(self, element): - self.x += element + self.x.append(element) it would generate code similar to:: @@ -728,17 +758,17 @@ it would generate code similar to:: def __init__(self, x=x): self.x = x def add(self, element): - self.x += element + self.x.append(element) assert D().x is D().x -This has the same issue as the original example using class ``C``. -That is, two instances of class ``D`` that do not specify a value -for ``x`` when creating a class instance will share the same copy -of ``x``. Because dataclasses just use normal Python class +This has the same issue as the original example using class :class:`!C`. +That is, two instances of class :class:`!D` that do not specify a value +for :attr:`!x` when creating a class instance will share the same copy +of :attr:`!x`. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`dataclass` decorator will raise a :exc:`ValueError` if it +:func:`@dataclass ` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -753,8 +783,8 @@ mutable types as default values for fields:: assert D().x is not D().x .. versionchanged:: 3.11 - Instead of looking for and disallowing objects of type ``list``, - ``dict``, or ``set``, unhashable objects are now not allowed as + Instead of looking for and disallowing objects of type :class:`list`, + :class:`dict`, or :class:`set`, unhashable objects are now not allowed as default values. Unhashability is used to approximate mutability. @@ -764,15 +794,17 @@ Descriptor-typed fields Fields that are assigned :ref:`descriptor objects ` as their default value have the following special behaviors: -* The value for the field passed to the dataclass's ``__init__`` method is - passed to the descriptor's ``__set__`` method rather than overwriting the +* The value for the field passed to the dataclass's :meth:`~object.__init__` method is + passed to the descriptor's :meth:`~object.__set__` method rather than overwriting the descriptor object. + * Similarly, when getting or setting the field, the descriptor's - ``__get__`` or ``__set__`` method is called rather than returning or + :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, ``dataclasses`` - will call the descriptor's ``__get__`` method using its class access - form (i.e. ``descriptor.__get__(obj=None, type=cls)``. If the + +* To determine whether a field contains a default value, :func:`@dataclass ` + will call the descriptor's :meth:`!__get__` method using its class access + form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the field's default. On the other hand, if the descriptor raises :exc:`AttributeError` in this situation, no default value will be diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 930af6cbbe9e8d..2f81080d525f86 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1,5 +1,5 @@ -:mod:`datetime` --- Basic date and time types -============================================= +:mod:`!datetime` --- Basic date and time types +============================================== .. module:: datetime :synopsis: Basic date and time types. @@ -37,7 +37,7 @@ on efficient attribute extraction for output formatting and manipulation. Package `dateutil `_ Third-party library with expanded time zone and parsing support. - Package `DateType `_ + Package :pypi:`DateType` Third-party library that introduces distinct static types to e.g. allow :term:`static type checkers ` to differentiate between naive and aware datetimes. @@ -48,7 +48,7 @@ Aware and Naive Objects ----------------------- Date and time objects may be categorized as "aware" or "naive" depending on -whether or not they include timezone information. +whether or not they include time zone information. With sufficient knowledge of applicable algorithmic and political time adjustments, such as time zone and daylight saving time information, @@ -58,7 +58,7 @@ interpretation. [#]_ A **naive** object does not contain enough information to unambiguously locate itself relative to other date/time objects. Whether a naive object represents -Coordinated Universal Time (UTC), local time, or time in some other timezone is +Coordinated Universal Time (UTC), local time, or time in some other time zone is purely up to the program, just like it is up to the program whether a particular number represents metres, miles, or mass. Naive objects are easy to understand and to work with, at the cost of ignoring some aspects of reality. @@ -70,9 +70,9 @@ These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`!datetime` module. The :class:`timezone` class can -represent simple timezones with fixed offsets from UTC, such as UTC itself or -North American EST and EDT timezones. Supporting timezones at deeper levels of +supplied by the :mod:`!datetime` module. The :class:`!timezone` class can +represent simple time zones with fixed offsets from UTC, such as UTC itself or +North American EST and EDT time zones. Supporting time zones at deeper levels of detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. @@ -85,17 +85,17 @@ The :mod:`!datetime` module exports the following constants: .. data:: MINYEAR The smallest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MINYEAR` is ``1``. + :const:`MINYEAR` is 1. .. data:: MAXYEAR The largest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MAXYEAR` is ``9999``. + :const:`MAXYEAR` is 9999. .. attribute:: UTC - Alias for the UTC timezone singleton :attr:`datetime.timezone.utc`. + Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. .. versionadded:: 3.11 @@ -180,19 +180,19 @@ Objects of the :class:`date` type are always naive. An object of type :class:`.time` or :class:`.datetime` may be aware or naive. -A :class:`.datetime` object *d* is aware if both of the following hold: +A :class:`.datetime` object ``d`` is aware if both of the following hold: 1. ``d.tzinfo`` is not ``None`` 2. ``d.tzinfo.utcoffset(d)`` does not return ``None`` -Otherwise, *d* is naive. +Otherwise, ``d`` is naive. -A :class:`.time` object *t* is aware if both of the following hold: +A :class:`.time` object ``t`` is aware if both of the following hold: 1. ``t.tzinfo`` is not ``None`` 2. ``t.tzinfo.utcoffset(None)`` does not return ``None``. -Otherwise, *t* is naive. +Otherwise, ``t`` is naive. The distinction between aware and naive doesn't apply to :class:`timedelta` objects. @@ -207,7 +207,7 @@ A :class:`timedelta` object represents a duration, the difference between two .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) - All arguments are optional and default to ``0``. Arguments may be integers + All arguments are optional and default to 0. Arguments may be integers or floats, and may be positive or negative. Only *days*, *seconds* and *microseconds* are stored internally. @@ -280,20 +280,40 @@ Class attributes: The smallest possible difference between non-equal :class:`timedelta` objects, ``timedelta(microseconds=1)``. -Note that, because of normalization, ``timedelta.max`` > ``-timedelta.min``. +Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. + Instance attributes (read-only): -+------------------+--------------------------------------------+ -| Attribute | Value | -+==================+============================================+ -| ``days`` | Between -999999999 and 999999999 inclusive | -+------------------+--------------------------------------------+ -| ``seconds`` | Between 0 and 86399 inclusive | -+------------------+--------------------------------------------+ -| ``microseconds`` | Between 0 and 999999 inclusive | -+------------------+--------------------------------------------+ +.. attribute:: timedelta.days + + Between -999,999,999 and 999,999,999 inclusive. + + +.. attribute:: timedelta.seconds + + Between 0 and 86,399 inclusive. + + .. caution:: + + It is a somewhat common bug for code to unintentionally use this attribute + when it is actually intended to get a :meth:`~timedelta.total_seconds` + value instead: + + .. doctest:: + + >>> from datetime import timedelta + >>> duration = timedelta(seconds=11235813) + >>> duration.days, duration.seconds + (130, 3813) + >>> duration.total_seconds() + 11235813.0 + +.. attribute:: timedelta.microseconds + + Between 0 and 999,999 inclusive. + Supported operations: @@ -302,26 +322,27 @@ Supported operations: +--------------------------------+-----------------------------------------------+ | Operation | Result | +================================+===============================================+ -| ``t1 = t2 + t3`` | Sum of *t2* and *t3*. Afterwards *t1*-*t2* == | -| | *t3* and *t1*-*t3* == *t2* are true. (1) | +| ``t1 = t2 + t3`` | Sum of ``t2`` and ``t3``. | +| | Afterwards ``t1 - t2 == t3`` and | +| | ``t1 - t3 == t2`` are true. (1) | +--------------------------------+-----------------------------------------------+ -| ``t1 = t2 - t3`` | Difference of *t2* and *t3*. Afterwards *t1* | -| | == *t2* - *t3* and *t2* == *t1* + *t3* are | +| ``t1 = t2 - t3`` | Difference of ``t2`` and ``t3``. Afterwards | +| | ``t1 == t2 - t3`` and ``t2 == t1 + t3`` are | | | true. (1)(6) | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 * i or t1 = i * t2`` | Delta multiplied by an integer. | -| | Afterwards *t1* // i == *t2* is true, | +| | Afterwards ``t1 // i == t2`` is true, | | | provided ``i != 0``. | +--------------------------------+-----------------------------------------------+ -| | In general, *t1* \* i == *t1* \* (i-1) + *t1* | +| | In general, ``t1 * i == t1 * (i-1) + t1`` | | | is true. (1) | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 * f or t1 = f * t2`` | Delta multiplied by a float. The result is | | | rounded to the nearest multiple of | | | timedelta.resolution using round-half-to-even.| +--------------------------------+-----------------------------------------------+ -| ``f = t2 / t3`` | Division (3) of overall duration *t2* by | -| | interval unit *t3*. Returns a :class:`float` | +| ``f = t2 / t3`` | Division (3) of overall duration ``t2`` by | +| | interval unit ``t3``. Returns a :class:`float`| | | object. | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 / f or t1 = t2 / i`` | Delta divided by a float or an int. The result| @@ -337,19 +358,18 @@ Supported operations: +--------------------------------+-----------------------------------------------+ | ``q, r = divmod(t1, t2)`` | Computes the quotient and the remainder: | | | ``q = t1 // t2`` (3) and ``r = t1 % t2``. | -| | q is an integer and r is a :class:`timedelta` | -| | object. | +| | ``q`` is an integer and ``r`` is a | +| | :class:`timedelta` object. | +--------------------------------+-----------------------------------------------+ | ``+t1`` | Returns a :class:`timedelta` object with the | | | same value. (2) | +--------------------------------+-----------------------------------------------+ -| ``-t1`` | equivalent to | -| | :class:`timedelta`\ (-*t1.days*, | -| | -*t1.seconds*, -*t1.microseconds*), | -| | and to *t1*\* -1. (1)(4) | +| ``-t1`` | Equivalent to ``timedelta(-t1.days, | +| | -t1.seconds, -t1.microseconds)``, | +| | and to ``t1 * -1``. (1)(4) | +--------------------------------+-----------------------------------------------+ -| ``abs(t)`` | equivalent to +\ *t* when ``t.days >= 0``, | -| | and to -*t* when ``t.days < 0``. (2) | +| ``abs(t)`` | Equivalent to ``+t`` when ``t.days >= 0``, | +| | and to ``-t`` when ``t.days < 0``. (2) | +--------------------------------+-----------------------------------------------+ | ``str(t)`` | Returns a string in the form | | | ``[D day[s], ][H]H:MM:SS[.UUUUUU]``, where D | @@ -370,10 +390,10 @@ Notes: This is exact and cannot overflow. (3) - Division by 0 raises :exc:`ZeroDivisionError`. + Division by zero raises :exc:`ZeroDivisionError`. (4) - -*timedelta.max* is not representable as a :class:`timedelta` object. + ``-timedelta.max`` is not representable as a :class:`timedelta` object. (5) String representations of :class:`timedelta` objects are normalized @@ -506,7 +526,7 @@ Other constructors, all class methods: January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= - date.max.toordinal()``. For any date *d*, + date.max.toordinal()``. For any date ``d``, ``date.fromordinal(d.toordinal()) == d``. @@ -542,6 +562,39 @@ Other constructors, all class methods: .. versionadded:: 3.8 +.. classmethod:: date.strptime(date_string, format) + + Return a :class:`.date` corresponding to *date_string*, parsed according to + *format*. This is equivalent to:: + + date(*(time.strptime(date_string, format)[0:3])) + + :exc:`ValueError` is raised if the date_string and format + can't be parsed by :func:`time.strptime` or if it returns a value which isn't a + time tuple. See also :ref:`strftime-strptime-behavior` and + :meth:`date.fromisoformat`. + + .. note:: + + If *format* specifies a day of month without a year a + :exc:`DeprecationWarning` is emitted. This is to avoid a quadrennial + leap year bug in code seeking to parse only a month and day as the + default year used in absence of one in the format is not a leap year. + Such *format* values may raise an error as of Python 3.15. The + workaround is to always include a year in your *format*. If parsing + *date_string* values that do not have a year, explicitly add a year that + is a leap year before parsing: + + .. doctest:: + + >>> from datetime import date + >>> date_string = "02/29" + >>> when = date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when.strftime("%B %d") # doctest: +SKIP + 'February 29' + + .. versionadded:: 3.14 + Class attributes: @@ -583,10 +636,10 @@ Supported operations: +-------------------------------+----------------------------------------------+ | Operation | Result | +===============================+==============================================+ -| ``date2 = date1 + timedelta`` | *date2* will be ``timedelta.days`` days | -| | after *date1*. (1) | +| ``date2 = date1 + timedelta`` | ``date2`` will be ``timedelta.days`` days | +| | after ``date1``. (1) | +-------------------------------+----------------------------------------------+ -| ``date2 = date1 - timedelta`` | Computes *date2* such that ``date2 + | +| ``date2 = date1 - timedelta`` | Computes ``date2`` such that ``date2 + | | | timedelta == date1``. (2) | +-------------------------------+----------------------------------------------+ | ``timedelta = date1 - date2`` | \(3) | @@ -613,17 +666,33 @@ Notes: ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. (3) - This is exact, and cannot overflow. timedelta.seconds and - timedelta.microseconds are 0, and date2 + timedelta == date1 after. + This is exact, and cannot overflow. ``timedelta.seconds`` and + ``timedelta.microseconds`` are 0, and ``date2 + timedelta == date1`` after. (4) :class:`date` objects are equal if they represent the same date. + :class:`!date` objects that are not also :class:`.datetime` instances + are never equal to :class:`!datetime` objects, even if they represent + the same date. + (5) *date1* is considered less than *date2* when *date1* precedes *date2* in time. In other words, ``date1 < date2`` if and only if ``date1.toordinal() < date2.toordinal()``. + Order comparison between a :class:`!date` object that is not also a + :class:`.datetime` instance and a :class:`!datetime` object raises + :exc:`TypeError`. + +.. versionchanged:: 3.13 + Comparison between :class:`.datetime` object and an instance of + the :class:`date` subclass that is not a :class:`!datetime` subclass + no longer converts the latter to :class:`!date`, ignoring the time part + and the time zone. + The default behavior can be changed by overriding the special comparison + methods in subclasses. + In Boolean contexts, all :class:`date` objects are considered to be true. Instance methods: @@ -655,13 +724,13 @@ Instance methods: time.struct_time((d.year, d.month, d.day, 0, 0, 0, d.weekday(), yday, -1)) where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` - is the day number within the current year starting with ``1`` for January 1st. + is the day number within the current year starting with 1 for January 1st. .. method:: date.toordinal() Return the proleptic Gregorian ordinal of the date, where January 1 of year 1 - has ordinal 1. For any :class:`date` object *d*, + has ordinal 1. For any :class:`date` object ``d``, ``date.fromordinal(d.toordinal()) == d``. @@ -713,7 +782,7 @@ Instance methods: .. method:: date.__str__() - For a date *d*, ``str(d)`` is equivalent to ``d.isoformat()``. + For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``. .. method:: date.ctime() @@ -853,7 +922,7 @@ Other constructors, all class methods: .. classmethod:: datetime.today() - Return the current local datetime, with :attr:`.tzinfo` ``None``. + Return the current local date and time, with :attr:`.tzinfo` ``None``. Equivalent to:: @@ -879,6 +948,10 @@ Other constructors, all class methods: This function is preferred over :meth:`today` and :meth:`utcnow`. + .. note:: + + Subsequent calls to :meth:`!datetime.now` may return the same + instant depending on the precision of the underlying clock. .. classmethod:: datetime.utcnow() @@ -975,8 +1048,8 @@ Other constructors, all class methods: .. classmethod:: datetime.fromordinal(ordinal) Return the :class:`.datetime` corresponding to the proleptic Gregorian ordinal, - where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 - <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and + where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless + ``1 <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and microsecond of the result are all 0, and :attr:`.tzinfo` is ``None``. @@ -990,7 +1063,7 @@ Other constructors, all class methods: is used. If the *date* argument is a :class:`.datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. - For any :class:`.datetime` object *d*, + For any :class:`.datetime` object ``d``, ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. .. versionchanged:: 3.6 @@ -1037,7 +1110,7 @@ Other constructors, all class methods: .. versionadded:: 3.7 .. versionchanged:: 3.11 Previously, this method only supported formats that could be emitted by - :meth:`date.isoformat()` or :meth:`datetime.isoformat()`. + :meth:`date.isoformat` or :meth:`datetime.isoformat`. .. classmethod:: datetime.fromisocalendar(year, week, day) @@ -1054,7 +1127,7 @@ Other constructors, all class methods: Return a :class:`.datetime` corresponding to *date_string*, parsed according to *format*. - If *format* does not contain microseconds or timezone information, this is equivalent to:: + If *format* does not contain microseconds or time zone information, this is equivalent to:: datetime(*(time.strptime(date_string, format)[0:6])) @@ -1063,6 +1136,24 @@ Other constructors, all class methods: time tuple. See also :ref:`strftime-strptime-behavior` and :meth:`datetime.fromisoformat`. + .. versionchanged:: 3.13 + + If *format* specifies a day of month without a year a + :exc:`DeprecationWarning` is now emitted. This is to avoid a quadrennial + leap year bug in code seeking to parse only a month and day as the + default year used in absence of one in the format is not a leap year. + Such *format* values may raise an error as of Python 3.15. The + workaround is to always include a year in your *format*. If parsing + *date_string* values that do not have a year, explicitly add a year that + is a leap year before parsing: + + .. doctest:: + + >>> from datetime import datetime + >>> date_string = "02/29" + >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when.strftime("%B %d") # doctest: +SKIP + 'February 29' Class attributes: @@ -1133,8 +1224,8 @@ Instance attributes (read-only): In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. + The values 0 and 1 represent, respectively, the earlier and later of the two + moments with the same wall time representation. .. versionadded:: 3.6 @@ -1159,16 +1250,16 @@ Supported operations: +---------------------------------------+--------------------------------+ (1) - datetime2 is a duration of timedelta removed from datetime1, moving forward in - time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The + ``datetime2`` is a duration of ``timedelta`` removed from ``datetime1``, moving forward in + time if ``timedelta.days > 0``, or backward if ``timedelta.days < 0``. The result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and - datetime2 - datetime1 == timedelta after. :exc:`OverflowError` is raised if - datetime2.year would be smaller than :const:`MINYEAR` or larger than + ``datetime2 - datetime1 == timedelta`` after. :exc:`OverflowError` is raised if + ``datetime2.year`` would be smaller than :const:`MINYEAR` or larger than :const:`MAXYEAR`. Note that no time zone adjustments are done even if the input is an aware object. (2) - Computes the datetime2 such that datetime2 + timedelta == datetime1. As for + Computes the ``datetime2`` such that ``datetime2 + timedelta == datetime1``. As for addition, the result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and no time zone adjustments are done even if the input is aware. @@ -1179,11 +1270,11 @@ Supported operations: If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute, the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta` - object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments + object ``t`` such that ``datetime2 + t == datetime1``. No time zone adjustments are done in this case. If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes. The + as if ``a`` and ``b`` were first converted to naive UTC datetimes. The result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - b.utcoffset())`` except that the implementation never overflows. @@ -1192,10 +1283,10 @@ Supported operations: and time, taking into account the time zone. Naive and aware :class:`!datetime` objects are never equal. - :class:`!datetime` objects are never equal to :class:`date` objects - that are not also :class:`!datetime` instances, even if they represent - the same date. + If both comparands are aware, and have the same :attr:`!tzinfo` attribute, + the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and + the base datetimes are compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparison acts as comparands were first converted to UTC datetimes except that the implementation never overflows. @@ -1206,10 +1297,12 @@ Supported operations: *datetime1* is considered less than *datetime2* when *datetime1* precedes *datetime2* in time, taking into account the time zone. - Order comparison between naive and aware :class:`.datetime` objects, - as well as a :class:`!datetime` object and a :class:`!date` object - that is not also a :class:`!datetime` instance, raises :exc:`TypeError`. + Order comparison between naive and aware :class:`.datetime` objects + raises :exc:`TypeError`. + If both comparands are aware, and have the same :attr:`!tzinfo` attribute, + the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and + the base datetimes are compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparison acts as comparands were first converted to UTC datetimes except that the implementation never overflows. @@ -1218,6 +1311,14 @@ Supported operations: Equality comparisons between aware and naive :class:`.datetime` instances don't raise :exc:`TypeError`. +.. versionchanged:: 3.13 + Comparison between :class:`.datetime` object and an instance of + the :class:`date` subclass that is not a :class:`!datetime` subclass + no longer converts the latter to :class:`!date`, ignoring the time part + and the time zone. + The default behavior can be changed by overriding the special comparison + methods in subclasses. + Instance methods: .. method:: datetime.date() @@ -1267,22 +1368,22 @@ Instance methods: If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self* - is naive, it is presumed to represent time in the system timezone. + is naive, it is presumed to represent time in the system time zone. If called without arguments (or with ``tz=None``) the system local - timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted + time zone is assumed for the target time zone. The ``.tzinfo`` attribute of the converted datetime instance will be set to an instance of :class:`timezone` with the zone name and offset obtained from the OS. If ``self.tzinfo`` is *tz*, ``self.astimezone(tz)`` is equal to *self*: no adjustment of date or time data is performed. Else the result is local - time in the timezone *tz*, representing the same UTC time as *self*: after + time in the time zone *tz*, representing the same UTC time as *self*: after ``astz = dt.astimezone(tz)``, ``astz - astz.utcoffset()`` will have the same date and time data as ``dt - dt.utcoffset()``. - If you merely want to attach a time zone object *tz* to a datetime *dt* without + If you merely want to attach a :class:`timezone` object *tz* to a datetime *dt* without adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you - merely want to remove the time zone object from an aware datetime *dt* without + merely want to remove the :class:`!timezone` object from an aware datetime *dt* without conversion of date and time data, use ``dt.replace(tzinfo=None)``. Note that the default :meth:`tzinfo.fromutc` method can be overridden in a @@ -1292,7 +1393,7 @@ Instance methods: def astimezone(self, tz): if self.tzinfo is tz: return self - # Convert self to UTC, and attach the new time zone object. + # Convert self to UTC, and attach the new timezone object. utc = (self - self.utcoffset()).replace(tzinfo=tz) # Convert from UTC to tz's local time. return tz.fromutc(utc) @@ -1343,24 +1444,24 @@ Instance methods: d.weekday(), yday, dst)) where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` - is the day number within the current year starting with ``1`` for January + is the day number within the current year starting with 1 for January 1st. The :attr:`~time.struct_time.tm_isdst` flag of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns ``None``, :attr:`!tm_isdst` is set to ``-1``; else if :meth:`dst` returns a - non-zero value, :attr:`!tm_isdst` is set to ``1``; else :attr:`!tm_isdst` is - set to ``0``. + non-zero value, :attr:`!tm_isdst` is set to 1; else :attr:`!tm_isdst` is + set to 0. .. method:: datetime.utctimetuple() - If :class:`.datetime` instance *d* is naive, this is the same as + If :class:`.datetime` instance ``d`` is naive, this is the same as ``d.timetuple()`` except that :attr:`~.time.struct_time.tm_isdst` is forced to 0 regardless of what ``d.dst()`` returns. DST is never in effect for a UTC time. - If *d* is aware, *d* is normalized to UTC time, by subtracting + If ``d`` is aware, ``d`` is normalized to UTC time, by subtracting ``d.utcoffset()``, and a :class:`time.struct_time` for the normalized time is returned. :attr:`!tm_isdst` is forced to 0. Note - that an :exc:`OverflowError` may be raised if *d*.year was + that an :exc:`OverflowError` may be raised if ``d.year`` was ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year boundary. @@ -1406,7 +1507,7 @@ Instance methods: There is no method to obtain the POSIX timestamp directly from a naive :class:`.datetime` instance representing UTC time. If your - application uses this convention and your system timezone is not + application uses this convention and your system time zone is not set to UTC, you can obtain the POSIX timestamp by supplying ``tzinfo=timezone.utc``:: @@ -1505,7 +1606,7 @@ Instance methods: .. method:: datetime.__str__() - For a :class:`.datetime` instance *d*, ``str(d)`` is equivalent to + For a :class:`.datetime` instance ``d``, ``str(d)`` is equivalent to ``d.isoformat(' ')``. @@ -1691,7 +1792,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. * ``fold in [0, 1]``. If an argument outside those ranges is given, :exc:`ValueError` is raised. All - default to ``0`` except *tzinfo*, which defaults to :const:`None`. + default to 0 except *tzinfo*, which defaults to ``None``. Class attributes: @@ -1746,20 +1847,20 @@ Instance attributes (read-only): In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. + The values 0 and 1 represent, respectively, the earlier and later of the two + moments with the same wall time representation. .. versionadded:: 3.6 :class:`.time` objects support equality and order comparisons, -where *a* is considered less than *b* when *a* precedes *b* in time. +where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time. Naive and aware :class:`!time` objects are never equal. Order comparison between naive and aware :class:`!time` objects raises :exc:`TypeError`. -If both comparands are aware, and have -the same :attr:`~.time.tzinfo` attribute, the common :attr:`!tzinfo` attribute is +If both comparands are aware, and have the same :attr:`~.time.tzinfo` +attribute, the :attr:`!tzinfo` and :attr:`!fold` attributes are ignored and the base times are compared. If both comparands are aware and have different :attr:`!tzinfo` attributes, the comparands are first adjusted by subtracting their UTC offsets (obtained from ``self.utcoffset()``). @@ -1777,7 +1878,7 @@ In Boolean contexts, a :class:`.time` object is always considered to be true. details. -Other constructor: +Other constructors: .. classmethod:: time.fromisoformat(time_string) @@ -1791,7 +1892,7 @@ Other constructor: be truncated). 4. Fractional hours and minutes are not supported. - Examples:: + Examples: .. doctest:: @@ -1817,7 +1918,23 @@ Other constructor: .. versionadded:: 3.7 .. versionchanged:: 3.11 Previously, this method only supported formats that could be emitted by - :meth:`time.isoformat()`. + :meth:`time.isoformat`. + +.. classmethod:: time.strptime(date_string, format) + + Return a :class:`.time` corresponding to *date_string*, parsed according to + *format*. + + If *format* does not contain microseconds or timezone information, this is equivalent to:: + + time(*(time.strptime(date_string, format)[3:6])) + + :exc:`ValueError` is raised if the *date_string* and *format* + cannot be parsed by :func:`time.strptime` or if it returns a value which is not a + time tuple. See also :ref:`strftime-strptime-behavior` and + :meth:`time.fromisoformat`. + + .. versionadded:: 3.14 Instance methods: @@ -1883,7 +2000,7 @@ Instance methods: .. method:: time.__str__() - For a time *t*, ``str(t)`` is equivalent to ``t.isoformat()``. + For a time ``t``, ``str(t)`` is equivalent to ``t.isoformat()``. .. method:: time.strftime(format) @@ -1977,7 +2094,7 @@ Examples of working with a :class:`.time` object:: supply implementations of the standard :class:`tzinfo` methods needed by the :class:`.datetime` methods you use. The :mod:`!datetime` module provides :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can - represent timezones with fixed offset from UTC such as UTC itself or North + represent time zones with fixed offset from UTC such as UTC itself or North American EST and EDT. Special requirement for pickling: A :class:`tzinfo` subclass must have an @@ -2039,7 +2156,7 @@ Examples of working with a :class:`.time` object:: ``tz.utcoffset(dt) - tz.dst(dt)`` must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz`` For sane :class:`tzinfo` subclasses, this expression yields the time + tz``. For sane :class:`tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's @@ -2076,7 +2193,7 @@ Examples of working with a :class:`.time` object:: Return the time zone name corresponding to the :class:`.datetime` object *dt*, as a string. Nothing about string names is defined by the :mod:`!datetime` module, and there's no requirement that it mean anything in particular. For example, - "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all + ``"GMT"``, ``"UTC"``, ``"-500"``, ``"-5:00"``, ``"EDT"``, ``"US/Eastern"``, ``"America/New York"`` are all valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value @@ -2102,14 +2219,14 @@ When a :class:`.datetime` object is passed in response to a :class:`.datetime` method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can rely on this, unless user code calls :class:`tzinfo` methods directly. The intent is that the :class:`tzinfo` methods interpret *dt* as being in local -time, and not need worry about objects in other timezones. +time, and not need worry about objects in other time zones. There is one more :class:`tzinfo` method that a subclass may wish to override: .. method:: tzinfo.fromutc(dt) - This is called from the default :class:`datetime.astimezone()` + This is called from the default :meth:`datetime.astimezone` implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s date and time data are to be viewed as expressing a UTC time. The purpose of :meth:`fromutc` is to adjust the date and time data, returning an @@ -2219,12 +2336,12 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). :mod:`zoneinfo` The :mod:`!datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` - attribute (a UTC timezone instance). + attribute (a UTC :class:`!timezone` instance). - ``zoneinfo`` brings the *IANA timezone database* (also known as the Olson + ``zoneinfo`` brings the *IANA time zone database* (also known as the Olson database) to Python, and its usage is recommended. - `IANA timezone database `_ + `IANA time zone database `_ The Time Zone Database (often called tz, tzdata or zoneinfo) contains code and data that represent the history of local time for many representative locations around the globe. It is updated periodically to reflect changes @@ -2238,10 +2355,10 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). ------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each -instance of which represents a timezone defined by a fixed offset from +instance of which represents a time zone defined by a fixed offset from UTC. -Objects of this class cannot be used to represent timezone information in the +Objects of this class cannot be used to represent time zone information in the locations where different offsets are used in different days of the year or where historical changes have been made to civil time. @@ -2302,7 +2419,7 @@ Class attributes: .. attribute:: timezone.utc - The UTC timezone, ``timezone(timedelta(0))``. + The UTC time zone, ``timezone(timedelta(0))``. .. index:: @@ -2317,24 +2434,22 @@ Class attributes: ``strftime(format)`` method, to create a string representing the time under the control of an explicit format string. -Conversely, the :meth:`datetime.strptime` class method creates a -:class:`.datetime` object from a string representing a date and time and a -corresponding format string. +Conversely, the :meth:`date.strptime`, :meth:`datetime.strptime` and +:meth:`time.strptime` class methods create an object from a string +representing the time and a corresponding format string. The table below provides a high-level comparison of :meth:`~.datetime.strftime` versus :meth:`~.datetime.strptime`: -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| | ``strftime`` | ``strptime`` | -+================+========================================================+==============================================================================+ -| Usage | Convert object to a string according to a given format | Parse a string into a :class:`.datetime` object given a corresponding format | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Type of method | Instance method | Class method | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Method of | :class:`date`; :class:`.datetime`; :class:`.time` | :class:`.datetime` | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Signature | ``strftime(format)`` | ``strptime(date_string, format)`` | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ ++----------------+--------------------------------------------------------+------------------------------------------------------------+ +| | ``strftime`` | ``strptime`` | ++================+========================================================+============================================================+ +| Usage | Convert object to a string according to a given format | Parse a string into an object given a corresponding format | ++----------------+--------------------------------------------------------+------------------------------------------------------------+ +| Type of method | Instance method | Class method | ++----------------+--------------------------------------------------------+------------------------------------------------------------+ +| Signature | ``strftime(format)`` | ``strptime(date_string, format)`` | ++----------------+--------------------------------------------------------+------------------------------------------------------------+ .. _format-codes: @@ -2511,17 +2626,17 @@ Using ``datetime.strptime(date_string, format)`` is equivalent to:: datetime(*(time.strptime(date_string, format)[0:6])) -except when the format includes sub-second components or timezone offset +except when the format includes sub-second components or time zone offset information, which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. For :class:`.time` objects, the format codes for year, month, and day should not be used, as :class:`!time` objects have no such values. If they're used anyway, -``1900`` is substituted for the year, and ``1`` for the month and day. +1900 is substituted for the year, and 1 for the month and day. For :class:`date` objects, the format codes for hours, minutes, seconds, and microseconds should not be used, as :class:`date` objects have no such -values. If they're used anyway, ``0`` is substituted for them. +values. If they're used anyway, 0 is substituted for them. For the same reason, handling of format strings containing Unicode code points that can't be represented in the charset of the current locale is also @@ -2631,6 +2746,25 @@ Notes: for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. +(10) + When parsing a month and day using :meth:`~.datetime.strptime`, always + include a year in the format. If the value you need to parse lacks a year, + append an explicit dummy leap year. Otherwise your code will raise an + exception when it encounters leap day because the default year used by the + parser is not a leap year. Users run into this bug every four years... + + .. doctest:: + + >>> month_day = "02/29" + >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. + datetime.datetime(1984, 2, 29, 0, 0) + + .. deprecated-removed:: 3.13 3.15 + :meth:`~.datetime.strptime` calls using a format string containing + a day of month without a year now emit a + :exc:`DeprecationWarning`. In 3.15 or later we may change this into + an error or change the default year to a leap year. See :gh:`70647`. + .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity @@ -2645,4 +2779,4 @@ Notes: `_ for a good explanation. -.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year. +.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since 1900 is not a leap year. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 9bb5e5f8950956..6c659ea52ad821 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -1,5 +1,5 @@ -:mod:`dbm` --- Interfaces to Unix "databases" -============================================= +:mod:`!dbm` --- Interfaces to Unix "databases" +============================================== .. module:: dbm :synopsis: Interfaces to various Unix "database" formats. @@ -8,13 +8,17 @@ -------------- -:mod:`dbm` is a generic interface to variants of the DBM database --- -:mod:`dbm.gnu` or :mod:`dbm.ndbm`. If none of these modules is installed, the +:mod:`dbm` is a generic interface to variants of the DBM database: + +* :mod:`dbm.sqlite3` +* :mod:`dbm.gnu` +* :mod:`dbm.ndbm` + +If none of these modules are installed, the slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There is a `third party interface `_ to the Oracle Berkeley DB. - .. exception:: error A tuple containing the exceptions that can be raised by each of the supported @@ -25,8 +29,8 @@ the Oracle Berkeley DB. .. function:: whichdb(filename) This function attempts to guess which of the several simple database modules - available --- :mod:`dbm.gnu`, :mod:`dbm.ndbm` or :mod:`dbm.dumb` --- should - be used to open a given file. + available --- :mod:`dbm.sqlite3`, :mod:`dbm.gnu`, :mod:`dbm.ndbm`, + or :mod:`dbm.dumb` --- should be used to open a given file. Return one of the following values: @@ -56,10 +60,6 @@ the Oracle Berkeley DB. The Unix file access mode of the file (default: octal ``0o666``), used only when the database has to be created. -.. |incompat_note| replace:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible - and can not be used interchangeably. - .. function:: open(file, flag='r', mode=0o666) Open a database and return the corresponding database object. @@ -144,6 +144,48 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. +:mod:`dbm.sqlite3` --- SQLite backend for dbm +--------------------------------------------- + +.. module:: dbm.sqlite3 + :platform: All + :synopsis: SQLite backend for dbm + +.. versionadded:: 3.13 + +**Source code:** :source:`Lib/dbm/sqlite3.py` + +-------------- + +This module uses the standard library :mod:`sqlite3` module to provide an +SQLite backend for the :mod:`dbm` module. +The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, +or any other SQLite browser, including the SQLite CLI. + +.. include:: ../includes/wasm-notavail.rst + +.. function:: open(filename, /, flag="r", mode=0o666) + + Open an SQLite database. + The returned object behaves like a :term:`mapping`, + implements a :meth:`!close` method, + and supports a "closing" context manager via the :keyword:`with` keyword. + + :param filename: + The path to the database to be opened. + :type filename: :term:`path-like object` + + :param str flag: + + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| + + :param mode: + The Unix file access mode of the file (default: octal ``0o666``), + used only when the database has to be created. + :mod:`dbm.gnu` --- GNU database manager --------------------------------------- @@ -160,11 +202,12 @@ The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` library, similar to the :mod:`dbm.ndbm` module, but with additional functionality like crash tolerance. -:class:`!gdbm` objects behave similar to :term:`mappings `, -except that keys and values are always converted to :class:`bytes` before storing, -and the :meth:`!items` and :meth:`!values` methods are not supported. +.. note:: -.. note:: |incompat_note| + The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + and can not be used interchangeably. + +.. include:: ../includes/wasm-mobile-notavail.rst .. exception:: error @@ -211,8 +254,9 @@ and the :meth:`!items` and :meth:`!values` methods are not supported. A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. - In addition to the dictionary-like methods, :class:`gdbm` objects have the - following methods and attributes: + :class:`!gdbm` objects behave similar to :term:`mappings `, + but :meth:`!items` and :meth:`!values` methods are not supported. + The following methods are also provided: .. method:: gdbm.firstkey() @@ -269,14 +313,13 @@ and the :meth:`!items` and :meth:`!values` methods are not supported. The :mod:`dbm.ndbm` module provides an interface to the :abbr:`NDBM (New Database Manager)` library. -:class:`!ndbm` objects behave similar to :term:`mappings `, -except that keys and values are always stored as :class:`bytes`, -and the :meth:`!items` and :meth:`!values` methods are not supported. - This module can be used with the "classic" NDBM interface or the :abbr:`GDBM (GNU dbm)` compatibility interface. -.. note:: |incompat_note| +.. note:: + + The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + and can not be used interchangeably. .. warning:: @@ -285,6 +328,8 @@ This module can be used with the "classic" NDBM interface or the when storing values larger than this limit. Reading such corrupted files can result in a hard crash (segmentation fault). +.. include:: ../includes/wasm-mobile-notavail.rst + .. exception:: error Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised @@ -314,8 +359,9 @@ This module can be used with the "classic" NDBM interface or the :param int mode: |mode_param_doc| - In addition to the dictionary-like methods, :class:`!ndbm` objects - provide the following method: + :class:`!ndbm` objects behave similar to :term:`mappings `, + but :meth:`!items` and :meth:`!values` methods are not supported. + The following methods are also provided: .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. @@ -354,8 +400,6 @@ The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like interface which is written entirely in Python. Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no external library is required. -As with other :mod:`dbm` backends, -the keys and values are always stored as :class:`bytes`. The :mod:`!dbm.dumb` module defines the following: @@ -416,4 +460,3 @@ The :mod:`!dbm.dumb` module defines the following: .. method:: dumbdbm.close() Close the database. - diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index fb8bbf72adf268..c9a3e448cad063 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1,5 +1,5 @@ -:mod:`decimal` --- Decimal fixed point and floating point arithmetic -==================================================================== +:mod:`!decimal` --- Decimal fixed-point and floating-point arithmetic +===================================================================== .. module:: decimal :synopsis: Implementation of the General Decimal Arithmetic Specification. @@ -31,7 +31,7 @@ -------------- The :mod:`decimal` module provides support for fast correctly rounded -decimal floating point arithmetic. It offers several advantages over the +decimal floating-point arithmetic. It offers several advantages over the :class:`float` datatype: * Decimal "is based on a floating-point model which was designed with people @@ -207,7 +207,7 @@ a decimal raises :class:`InvalidOperation`:: .. versionchanged:: 3.3 Decimals interact well with much of the rest of Python. Here is a small decimal -floating point flying circus: +floating-point flying circus: .. doctest:: :options: +NORMALIZE_WHITESPACE @@ -373,7 +373,7 @@ Decimal objects digits, and an integer exponent. For example, ``Decimal((0, (1, 4, 1, 4), -3))`` returns ``Decimal('1.414')``. - If *value* is a :class:`float`, the binary floating point value is losslessly + If *value* is a :class:`float`, the binary floating-point value is losslessly converted to its exact decimal equivalent. This conversion can often require 53 or more digits of precision. For example, ``Decimal(float('1.1'))`` converts to @@ -403,7 +403,7 @@ Decimal objects Underscores are allowed for grouping, as with integral and floating-point literals in code. - Decimal floating point objects share many properties with the other built-in + Decimal floating-point objects share many properties with the other built-in numeric types such as :class:`float` and :class:`int`. All of the usual math operations and special methods apply. Likewise, decimal objects can be copied, pickled, printed, used as dictionary keys, used as set elements, @@ -445,7 +445,7 @@ Decimal objects Mixed-type comparisons between :class:`Decimal` instances and other numeric types are now fully supported. - In addition to the standard numeric properties, decimal floating point + In addition to the standard numeric properties, decimal floating-point objects also have a number of specialized methods: @@ -598,6 +598,23 @@ Decimal objects .. versionadded:: 3.1 + .. classmethod:: from_number(number) + + Alternative constructor that only accepts instances of + :class:`float`, :class:`int` or :class:`Decimal`, but not strings + or tuples. + + .. doctest:: + + >>> Decimal.from_number(314) + Decimal('314') + >>> Decimal.from_number(0.1) + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) + Decimal('3.14') + + .. versionadded:: 3.14 + .. method:: fma(other, third, context=None) Fused multiply-add. Return self*other+third with no rounding of the @@ -897,6 +914,48 @@ Decimal objects :const:`Rounded`. If given, applies *rounding*; otherwise, uses the rounding method in either the supplied *context* or the current context. + Decimal numbers can be rounded using the :func:`.round` function: + + .. describe:: round(number) + .. describe:: round(number, ndigits) + + If *ndigits* is not given or ``None``, + returns the nearest :class:`int` to *number*, + rounding ties to even, and ignoring the rounding mode of the + :class:`Decimal` context. Raises :exc:`OverflowError` if *number* is an + infinity or :exc:`ValueError` if it is a (quiet or signaling) NaN. + + If *ndigits* is an :class:`int`, the context's rounding mode is respected + and a :class:`Decimal` representing *number* rounded to the nearest + multiple of ``Decimal('1E-ndigits')`` is returned; in this case, + ``round(number, ndigits)`` is equivalent to + ``self.quantize(Decimal('1E-ndigits'))``. Returns ``Decimal('NaN')`` if + *number* is a quiet NaN. Raises :class:`InvalidOperation` if *number* + is an infinity, a signaling NaN, or if the length of the coefficient after + the quantize operation would be greater than the current context's + precision. In other words, for the non-corner cases: + + * if *ndigits* is positive, return *number* rounded to *ndigits* decimal + places; + * if *ndigits* is zero, return *number* rounded to the nearest integer; + * if *ndigits* is negative, return *number* rounded to the nearest + multiple of ``10**abs(ndigits)``. + + For example:: + + >>> from decimal import Decimal, getcontext, ROUND_DOWN + >>> getcontext().rounding = ROUND_DOWN + >>> round(Decimal('3.75')) # context rounding ignored + 4 + >>> round(Decimal('3.5')) # round-ties-to-even + 4 + >>> round(Decimal('3.75'), 0) # uses the context rounding + Decimal('3') + >>> round(Decimal('3.75'), 1) + Decimal('3.7') + >>> round(Decimal('3.75'), -1) + Decimal('0E+1') + .. _logical_operands_label: @@ -1517,7 +1576,7 @@ are also included in the pure Python version for compatibility. the C version uses a thread-local rather than a coroutine-local context and the value is ``False``. This is slightly faster in some nested context scenarios. -.. versionadded:: 3.9 backported to 3.7 and 3.8. + .. versionadded:: 3.8.3 Rounding modes @@ -1699,7 +1758,7 @@ The following table summarizes the hierarchy of signals:: .. _decimal-notes: -Floating Point Notes +Floating-Point Notes -------------------- @@ -1712,7 +1771,7 @@ can still incur round-off error when non-zero digits exceed the fixed precision. The effects of round-off error can be amplified by the addition or subtraction of nearly offsetting quantities resulting in loss of significance. Knuth -provides two instructive examples where rounded floating point arithmetic with +provides two instructive examples where rounded floating-point arithmetic with insufficient precision causes the breakdown of the associative and distributive properties of addition: @@ -1802,7 +1861,7 @@ treated as equal and their sign is informational. In addition to the two signed zeros which are distinct yet equal, there are various representations of zero with differing precisions yet equivalent in value. This takes a bit of getting used to. For an eye accustomed to -normalized floating point representations, it is not immediately obvious that +normalized floating-point representations, it is not immediately obvious that the following calculation returns a value equal to zero: >>> 1 / Decimal('Infinity') @@ -2129,7 +2188,7 @@ value unchanged: Q. Is there a way to convert a regular float to a :class:`Decimal`? -A. Yes, any binary floating point number can be exactly expressed as a +A. Yes, any binary floating-point number can be exactly expressed as a Decimal though an exact conversion may take more precision than intuition would suggest: @@ -2183,7 +2242,7 @@ Q. Is the CPython implementation fast for large numbers? A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for -arbitrary precision correctly rounded decimal floating point arithmetic [#]_. +arbitrary precision correctly rounded decimal floating-point arithmetic [#]_. ``libmpdec`` uses `Karatsuba multiplication `_ for medium-sized numbers and the `Number Theoretic Transform diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index d45e46448207a4..ce948a6860f02c 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -1,5 +1,5 @@ -:mod:`difflib` --- Helpers for computing deltas -=============================================== +:mod:`!difflib` --- Helpers for computing deltas +================================================ .. module:: difflib :synopsis: Helpers for computing differences between objects. @@ -631,7 +631,7 @@ If you want to know how to change the first sequence into the second, use work. * `Simple version control recipe - `_ for a small application + `_ for a small application built with :class:`SequenceMatcher`. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index e654760fb91c65..cf203a714ba126 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1,5 +1,5 @@ -:mod:`dis` --- Disassembler for Python bytecode -=============================================== +:mod:`!dis` --- Disassembler for Python bytecode +================================================ .. module:: dis :synopsis: Disassembler for Python bytecode. @@ -56,6 +56,10 @@ interpreter. for jump targets and exception handlers. The ``-O`` command line option and the ``show_offsets`` argument were added. + .. versionchanged:: 3.14 + The :option:`-P ` command-line option + and the ``show_positions`` argument were added. + Example: Given the function :func:`!myfunc`:: def myfunc(alist): @@ -85,7 +89,7 @@ The :mod:`dis` module can be invoked as a script from the command line: .. code-block:: sh - python -m dis [-h] [-C] [-O] [infile] + python -m dis [-h] [-C] [-O] [-P] [infile] The following options are accepted: @@ -103,8 +107,12 @@ The following options are accepted: Show offsets of instructions. +.. cmdoption:: -P, --show-positions + + Show positions of instructions in the source code. + If :file:`infile` is specified, its disassembled code will be written to stdout. -Otherwise, disassembly is performed on compiled source code recieved from stdin. +Otherwise, disassembly is performed on compiled source code received from stdin. Bytecode analysis ----------------- @@ -116,7 +124,8 @@ The bytecode analysis API allows pieces of Python code to be wrapped in a code. .. class:: Bytecode(x, *, first_line=None, current_offset=None,\ - show_caches=False, adaptive=False, show_offsets=False) + show_caches=False, adaptive=False, show_offsets=False,\ + show_positions=False) Analyse the bytecode corresponding to a function, generator, asynchronous generator, coroutine, method, string of source code, or a code object (as @@ -144,6 +153,9 @@ code. If *show_offsets* is ``True``, :meth:`.dis` will include instruction offsets in the output. + If *show_positions* is ``True``, :meth:`.dis` will include instruction + source code positions in the output. + .. classmethod:: from_traceback(tb, *, show_caches=False) Construct a :class:`Bytecode` instance from the given traceback, setting @@ -173,6 +185,12 @@ code. .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter + + .. versionchanged:: 3.14 + Added the *show_positions* parameter. + Example: .. doctest:: @@ -226,7 +244,8 @@ operation is being performed, so the intermediate analysis object isn't useful: Added *file* parameter. -.. function:: dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False) +.. function:: dis(x=None, *, file=None, depth=None, show_caches=False,\ + adaptive=False, show_offsets=False, show_positions=False) Disassemble the *x* object. *x* can denote either a module, a class, a method, a function, a generator, an asynchronous generator, a coroutine, @@ -265,9 +284,14 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter. + + .. versionchanged:: 3.14 + Added the *show_positions* parameter. -.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False, - show_offset=False) +.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False,\ + show_offset=False, show_positions=False) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is @@ -285,14 +309,20 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.13 Added the *show_offsets* parameter. -.. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) - disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False, - show_offsets=False) + .. versionchanged:: 3.14 + Added the *show_positions* parameter. + +.. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False,\ + adaptive=False, show_offsets=False, show_positions=False) + disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False,\ + show_offsets=False, show_positions=False) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: - #. the line number, for the first instruction of each line + #. the source code location of the instruction. Complete location information + is shown if *show_positions* is true. Otherwise (the default) only the + line number is displayed. #. the current instruction, indicated as ``-->``, #. a labelled instruction, indicated with ``>>``, #. the address of the instruction, @@ -315,6 +345,9 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.13 Added the *show_offsets* parameter. + .. versionchanged:: 3.14 + Added the *show_positions* parameter. + .. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) Return an iterator over the instructions in the supplied function, method, @@ -336,9 +369,10 @@ operation is being performed, so the intermediate analysis object isn't useful: Added the *show_caches* and *adaptive* parameters. .. versionchanged:: 3.13 - The *show_caches* parameter is deprecated and has no effect. The *cache_info* - field of each instruction is populated regardless of its value. - + The *show_caches* parameter is deprecated and has no effect. The iterator + generates the :class:`Instruction` instances with the *cache_info* + field populated (regardless of the value of *show_caches*) and it no longer + generates separate items for the cache entries. .. function:: findlinestarts(code) @@ -779,16 +813,6 @@ not have to be) the original ``STACK[-2]``. .. versionadded:: 3.12 -.. opcode:: BEFORE_ASYNC_WITH - - Resolves ``__aenter__`` and ``__aexit__`` from ``STACK[-1]``. - Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack:: - - STACK.extend((__aexit__, __aenter__()) - - .. versionadded:: 3.5 - - **Miscellaneous opcodes** @@ -838,13 +862,6 @@ iterations of the loop. Returns with ``STACK[-1]`` to the caller of the function. -.. opcode:: RETURN_CONST (consti) - - Returns with ``co_consts[consti]`` to the caller of the function. - - .. versionadded:: 3.12 - - .. opcode:: YIELD_VALUE Yields ``STACK.pop()`` from a :term:`generator`. @@ -929,12 +946,13 @@ iterations of the loop. Exception representation on the stack now consist of one, not three, items. -.. opcode:: LOAD_ASSERTION_ERROR +.. opcode:: LOAD_COMMON_CONSTANT - Pushes :exc:`AssertionError` onto the stack. Used by the :keyword:`assert` - statement. + Pushes a common constant onto the stack. The interpreter contains a hardcoded + list of constants supported by this instruction. Used by the :keyword:`assert` + statement to load :exc:`AssertionError`. - .. versionadded:: 3.9 + .. versionadded:: 3.14 .. opcode:: LOAD_BUILD_CLASS @@ -942,21 +960,10 @@ iterations of the loop. Pushes :func:`!builtins.__build_class__` onto the stack. It is later called to construct a class. - -.. opcode:: BEFORE_WITH - - This opcode performs several operations before a with block starts. First, - it loads :meth:`~object.__exit__` from the context manager and pushes it onto - the stack for later use by :opcode:`WITH_EXCEPT_START`. Then, - :meth:`~object.__enter__` is called. Finally, the result of calling the - ``__enter__()`` method is pushed onto the stack. - - .. versionadded:: 3.11 - - .. opcode:: GET_LEN - Perform ``STACK.append(len(STACK[-1]))``. + Perform ``STACK.append(len(STACK[-1]))``. Used in :keyword:`match` statements where + comparison with structure of pattern is needed. .. versionadded:: 3.10 @@ -1072,6 +1079,22 @@ iterations of the loop. Pushes ``co_consts[consti]`` onto the stack. +.. opcode:: LOAD_SMALL_INT (i) + + Pushes the integer ``i`` onto the stack. + ``i`` must be in ``range(256)`` + + .. versionadded:: 3.14 + + +.. opcode:: LOAD_CONST_IMMORTAL (consti) + + Pushes ``co_consts[consti]`` onto the stack. + Can be used when the constant value is known to be immortal. + + .. versionadded:: 3.14 + + .. opcode:: LOAD_NAME (namei) Pushes the value associated with ``co_names[namei]`` onto the stack. @@ -1101,11 +1124,15 @@ iterations of the loop. .. opcode:: BUILD_TUPLE (count) Creates a tuple consuming *count* items from the stack, and pushes the - resulting tuple onto the stack.:: + resulting tuple onto the stack:: + + if count == 0: + value = () + else: + value = tuple(STACK[-count:]) + STACK = STACK[:-count] - assert count > 0 - STACK, values = STACK[:-count], STACK[-count:] - STACK.append(tuple(values)) + STACK.append(value) .. opcode:: BUILD_LIST (count) @@ -1129,15 +1156,6 @@ iterations of the loop. empty dictionary pre-sized to hold *count* items. -.. opcode:: BUILD_CONST_KEY_MAP (count) - - The version of :opcode:`BUILD_MAP` specialized for constant keys. Pops the - top element on the stack which contains a tuple of keys, then starting from - ``STACK[-2]``, pops *count* values to form values in the built dictionary. - - .. versionadded:: 3.6 - - .. opcode:: BUILD_STRING (count) Concatenates *count* strings from the stack and pushes the resulting string @@ -1224,7 +1242,7 @@ iterations of the loop. except that ``namei`` is shifted left by 2 bits instead of 1. The low bit of ``namei`` signals to attempt a method load, as with - :opcode:`LOAD_ATTR`, which results in pushing ``None`` and the loaded method. + :opcode:`LOAD_ATTR`, which results in pushing ``NULL`` and the loaded method. When it is unset a single value is pushed to the stack. The second-low bit of ``namei``, if set, means that this was a two-argument @@ -1426,7 +1444,7 @@ iterations of the loop. slot ``i`` of the "fast locals" storage in this mapping. If the name is not found there, loads it from the cell contained in slot ``i``, similar to :opcode:`LOAD_DEREF`. This is used for loading - free variables in class bodies (which previously used + :term:`closure variables ` in class bodies (which previously used :opcode:`!LOAD_CLASSDEREF`) and in :ref:`annotation scopes ` within class bodies. @@ -1455,8 +1473,8 @@ iterations of the loop. .. opcode:: COPY_FREE_VARS (n) - Copies the ``n`` free variables from the closure into the frame. - Removes the need for special code on the caller's side when calling + Copies the ``n`` :term:`free (closure) variables ` from the closure + into the frame. Removes the need for special code on the caller's side when calling closures. .. versionadded:: 3.11 @@ -1580,7 +1598,7 @@ iterations of the loop. end = STACK.pop() start = STACK.pop() - STACK.append(slice(start, stop)) + STACK.append(slice(start, end)) if it is 3, implements:: @@ -1606,7 +1624,7 @@ iterations of the loop. value = STACK.pop() result = func(value) - STACK.push(result) + STACK.append(result) * ``oparg == 1``: call :func:`str` on *value* * ``oparg == 2``: call :func:`repr` on *value* @@ -1623,20 +1641,20 @@ iterations of the loop. value = STACK.pop() result = value.__format__("") - STACK.push(result) + STACK.append(result) Used for implementing formatted literal strings (f-strings). .. versionadded:: 3.13 -.. opcode:: FORMAT_SPEC +.. opcode:: FORMAT_WITH_SPEC Formats the given value with the given format spec:: spec = STACK.pop() value = STACK.pop() result = value.__format__(spec) - STACK.push(result) + STACK.append(result) Used for implementing formatted literal strings (f-strings). @@ -1665,7 +1683,7 @@ iterations of the loop. A no-op. Performs internal tracing, debugging and optimization checks. - The ``context`` oparand consists of two parts. The lowest two bits + The ``context`` operand consists of two parts. The lowest two bits indicate where the ``RESUME`` occurs: * ``0`` The start of a function, which is neither a generator, coroutine @@ -1747,7 +1765,7 @@ iterations of the loop. | ``INTRINSIC_STOPITERATION_ERROR`` | Extracts the return value from a | | | ``StopIteration`` exception. | +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an aync generator value | + | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an async generator value | +-----------------------------------+-----------------------------------+ | ``INTRINSIC_UNARY_POSITIVE`` | Performs the unary ``+`` | | | operation | @@ -1783,7 +1801,7 @@ iterations of the loop. arg2 = STACK.pop() arg1 = STACK.pop() result = intrinsic2(arg1, arg2) - STACK.push(result) + STACK.append(result) The operand determines which intrinsic function is called: @@ -1810,6 +1828,17 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: LOAD_SPECIAL + + Performs special method lookup on ``STACK[-1]``. + If ``type(STACK[-1]).__xxx__`` is a method, leave + ``type(STACK[-1]).__xxx__; STACK[-1]`` on the stack. + If ``type(STACK[-1]).__xxx__`` is not a method, leave + ``STACK[-1].__xxx__; NULL`` on the stack. + + .. versionadded:: 3.14 + + **Pseudo-instructions** These opcodes do not appear in Python bytecode. They are used by the compiler @@ -1853,6 +1882,12 @@ but are replaced by real opcodes or removed before bytecode is generated. Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. +.. opcode:: JUMP_IF_TRUE +.. opcode:: JUMP_IF_FALSE + + Conditional jumps which do not impact the stack. Replaced by the sequence + ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. + .. opcode:: LOAD_CLOSURE (i) Pushes a reference to the cell contained in slot ``i`` of the "fast locals" @@ -1912,10 +1947,10 @@ instructions: .. data:: hasfree - Sequence of bytecodes that access a free variable. 'free' in this - context refers to names in the current scope that are referenced by inner - scopes or names in outer scopes that are referenced from this scope. It does - *not* include references to global or builtin scopes. + Sequence of bytecodes that access a :term:`free (closure) variable `. + 'free' in this context refers to names in the current scope that are + referenced by inner scopes or names in outer scopes that are referenced + from this scope. It does *not* include references to global or builtin scopes. .. data:: hasname diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 1bfcd69f72df2e..6b0282eed49566 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1,5 +1,5 @@ -:mod:`doctest` --- Test interactive Python examples -=================================================== +:mod:`!doctest` --- Test interactive Python examples +==================================================== .. module:: doctest :synopsis: Test pieces of code within docstrings. @@ -123,10 +123,10 @@ And so on, eventually ending with: OverflowError: n too large ok 2 items passed all tests: - 1 tests in __main__ - 8 tests in __main__.factorial - 9 tests in 2 items. - 9 passed and 0 failed. + 1 test in __main__ + 6 tests in __main__.factorial + 7 tests in 2 items. + 7 passed. Test passed. $ @@ -430,10 +430,10 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): File "", line 1, in - ValueError: 42 is not in list + ValueError: list.remove(x): x not in list -That doctest succeeds if :exc:`ValueError` is raised, with the ``42 is not in list`` -detail as shown. +That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): +x not in list`` detail as shown. The expected output for an exception must start with a traceback header, which may be either of the following two lines, indented the same as the first line of @@ -800,18 +800,18 @@ guarantee about output. For example, when printing a set, Python doesn't guarantee that the element is printed in any particular order, so a test like :: >>> foo() - {"Hermione", "Harry"} + {"spam", "eggs"} is vulnerable! One workaround is to do :: - >>> foo() == {"Hermione", "Harry"} + >>> foo() == {"spam", "eggs"} True instead. Another is to do :: >>> d = sorted(foo()) >>> d - ['Harry', 'Hermione'] + ['eggs', 'spam'] There are others, but you get the idea. @@ -1021,7 +1021,8 @@ from text files and modules with doctests: and runs the interactive examples in each file. If an example in any file fails, then the synthesized unit test fails, and a :exc:`failureException` exception is raised showing the name of the file containing the test and a - (sometimes approximate) line number. + (sometimes approximate) line number. If all the examples in a file are + skipped, then the synthesized unit test is also marked as skipped. Pass one or more paths (as strings) to text files to be examined. @@ -1087,7 +1088,8 @@ from text files and modules with doctests: and runs each doctest in the module. If any of the doctests fail, then the synthesized unit test fails, and a :exc:`failureException` exception is raised showing the name of the file containing the test and a (sometimes approximate) - line number. + line number. If all the examples in a docstring are skipped, then the + synthesized unit test is also marked as skipped. Optional argument *module* provides the module to be tested. It can be a module object or a (possibly dotted) module name. If not specified, the module calling @@ -1933,7 +1935,7 @@ such a test runner:: optionflags=flags) else: fail, total = doctest.testmod(optionflags=flags) - print("{} failures out of {} tests".format(fail, total)) + print(f"{fail} failures out of {total} tests") .. rubric:: Footnotes diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index aa0134412f3a60..6875af2be49d7a 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -1,5 +1,5 @@ -:mod:`email.charset`: Representing character sets -------------------------------------------------- +:mod:`!email.charset`: Representing character sets +-------------------------------------------------- .. module:: email.charset :synopsis: Character Sets diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index c4c322a82e1f44..4285c436e8da80 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -7,6 +7,7 @@ :synopsis: The base class representing email messages in a fashion backward compatible with Python 3.2 :noindex: + :no-index: The :class:`Message` class is very similar to the @@ -104,7 +105,7 @@ Here are the methods of the :class:`Message` class: .. method:: __str__() - Equivalent to :meth:`.as_string()`. Allows ``str(msg)`` to produce a + Equivalent to :meth:`.as_string`. Allows ``str(msg)`` to produce a string containing the formatted message. @@ -142,7 +143,7 @@ Here are the methods of the :class:`Message` class: .. method:: __bytes__() - Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a + Equivalent to :meth:`.as_bytes`. Allows ``bytes(msg)`` to produce a bytes object containing the formatted message. .. versionadded:: 3.4 diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 5b49339650f0e9..a86e227429b06d 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -1,5 +1,5 @@ -:mod:`email.contentmanager`: Managing MIME Content --------------------------------------------------- +:mod:`!email.contentmanager`: Managing MIME Content +--------------------------------------------------- .. module:: email.contentmanager :synopsis: Storing and Retrieving Content from MIME Parts @@ -58,11 +58,12 @@ * the type itself (``typ``) * the type's fully qualified name (``typ.__module__ + '.' + typ.__qualname__``). - * the type's qualname (``typ.__qualname__``) - * the type's name (``typ.__name__``). + * the type's :attr:`qualname ` (``typ.__qualname__``) + * the type's :attr:`name ` (``typ.__name__``). If none of the above match, repeat all of the checks above for each of - the types in the :term:`MRO` (``typ.__mro__``). Finally, if no other key + the types in the :term:`MRO` (:attr:`typ.__mro__ `). + Finally, if no other key yields a handler, check for a handler for the key ``None``. If there is no handler for ``None``, raise a :exc:`KeyError` for the fully qualified name of the type. diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 3bd377e33f6c15..9c8c8c9234ed7a 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -1,5 +1,5 @@ -:mod:`email.encoders`: Encoders -------------------------------- +:mod:`!email.encoders`: Encoders +-------------------------------- .. module:: email.encoders :synopsis: Encoders for email message payloads. diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 56aea6598b8615..f8f43d82a3df2e 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -1,5 +1,5 @@ -:mod:`email.errors`: Exception and Defect classes -------------------------------------------------- +:mod:`!email.errors`: Exception and Defect classes +-------------------------------------------------- .. module:: email.errors :synopsis: The exception classes used by the email package. @@ -58,6 +58,13 @@ The following exception classes are defined in the :mod:`email.errors` module: :class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g. :class:`~email.mime.image.MIMEImage`). + +.. exception:: HeaderWriteError() + + Raised when an error occurs when the :mod:`~email.generator` outputs + headers. + + .. exception:: MessageDefect() This is the base class for all defects found when parsing email messages. diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index afa0038ea2d6c4..a3132d02687bc9 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -1,5 +1,5 @@ -:mod:`email.generator`: Generating MIME documents -------------------------------------------------- +:mod:`!email.generator`: Generating MIME documents +-------------------------------------------------- .. module:: email.generator :synopsis: Generate flat text email messages from a message structure. diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index e093f138936b36..219fad0d2f6745 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -1,5 +1,5 @@ -:mod:`email.header`: Internationalized headers ----------------------------------------------- +:mod:`!email.header`: Internationalized headers +----------------------------------------------- .. module:: email.header :synopsis: Representing non-ASCII headers @@ -77,7 +77,7 @@ Here is the :class:`Header` class description: The maximum line length can be specified explicitly via *maxlinelen*. For splitting the first line to a shorter value (to account for the field header which isn't included in *s*, e.g. :mailheader:`Subject`) pass in the name of the - field in *header_name*. The default *maxlinelen* is 76, and the default value + field in *header_name*. The default *maxlinelen* is 78, and the default value for *header_name* is ``None``, meaning it is not taken into account for the first line of a long, split header. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 00a954e0307ea6..7f8044932fae99 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -1,5 +1,5 @@ -:mod:`email.headerregistry`: Custom Header Objects --------------------------------------------------- +:mod:`!email.headerregistry`: Custom Header Objects +--------------------------------------------------- .. module:: email.headerregistry :synopsis: Automatic Parsing of headers based on the field name @@ -317,7 +317,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. class. When *use_default_map* is ``True`` (the default), the standard mapping of header names to classes is copied in to the registry during initialization. *base_class* is always the last class in the generated - class's ``__bases__`` list. + class's :class:`~type.__bases__` list. The default mappings are: diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index d53ab33b8904a7..090981d84b4de3 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -1,5 +1,5 @@ -:mod:`email.iterators`: Iterators ---------------------------------- +:mod:`!email.iterators`: Iterators +---------------------------------- .. module:: email.iterators :synopsis: Iterate over a message object tree. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index adea067e082615..71d6e321f387bc 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -1,5 +1,5 @@ -:mod:`email.message`: Representing an email message ---------------------------------------------------- +:mod:`!email.message`: Representing an email message +---------------------------------------------------- .. module:: email.message :synopsis: The base class representing email messages. @@ -41,7 +41,7 @@ The :class:`EmailMessage` dictionary-like interface is indexed by the header names, which must be ASCII values. The values of the dictionary are strings with some extra methods. Headers are stored and returned in case-preserving form, but field names are matched case-insensitively. The keys are ordered, -but unlike a real dict, there can be duplicates. Addtional methods are +but unlike a real dict, there can be duplicates. Additional methods are provided for working with headers that have duplicate keys. The *payload* is either a string or bytes object, in the case of simple message @@ -124,7 +124,7 @@ message objects. .. method:: __bytes__() - Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a + Equivalent to :meth:`.as_bytes`. Allows ``bytes(msg)`` to produce a bytes object containing the serialized message. diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index dc0dd3b9eebde6..b85673a4acd0d0 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -1,5 +1,5 @@ -:mod:`email.mime`: Creating email and MIME objects from scratch ---------------------------------------------------------------- +:mod:`!email.mime`: Creating email and MIME objects from scratch +---------------------------------------------------------------- .. module:: email.mime :synopsis: Build MIME messages. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index dda0466a6afa7d..439b5c8f34b65a 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -1,5 +1,5 @@ -:mod:`email.parser`: Parsing email messages -------------------------------------------- +:mod:`!email.parser`: Parsing email messages +-------------------------------------------- .. module:: email.parser :synopsis: Parse flat text email messages to produce a message object structure. diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index f4777bb2462138..314767d0802a08 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -1,5 +1,5 @@ -:mod:`email.policy`: Policy Objects ------------------------------------ +:mod:`!email.policy`: Policy Objects +------------------------------------ .. module:: email.policy :synopsis: Controlling the parsing and generating of messages @@ -229,6 +229,24 @@ added matters. To illustrate:: .. versionadded:: 3.6 + + .. attribute:: verify_generated_headers + + If ``True`` (the default), the generator will raise + :exc:`~email.errors.HeaderWriteError` instead of writing a header + that is improperly folded or delimited, such that it would + be parsed as multiple headers or joined with adjacent data. + Such headers can be generated by custom header classes or bugs + in the ``email`` module. + + As it's a security feature, this defaults to ``True`` even in the + :class:`~email.policy.Compat32` policy. + For backwards compatible, but unsafe, behavior, it must be set to + ``False`` explicitly. + + .. versionadded:: 3.13 + + The following :class:`Policy` method is intended to be called by code using the email library to create policy instances with custom settings: diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 3a6039004fcaae..66c42e4a5008ee 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -1,5 +1,5 @@ -:mod:`email` --- An email and MIME handling package -=================================================== +:mod:`!email` --- An email and MIME handling package +==================================================== .. module:: email :synopsis: Package supporting the parsing, manipulating, and generating diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index d693a9bc3933b5..611549604fda15 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -1,5 +1,5 @@ -:mod:`email.utils`: Miscellaneous utilities -------------------------------------------- +:mod:`!email.utils`: Miscellaneous utilities +-------------------------------------------- .. module:: email.utils :synopsis: Miscellaneous email package utilities. @@ -17,8 +17,7 @@ module: arguments, return current time. Otherwise *dt* argument should be a :class:`~datetime.datetime` instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that - is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. The - *isdst* parameter is ignored. + is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. .. versionadded:: 3.3 @@ -159,7 +158,7 @@ of the new API. Fri, 09 Nov 2001 01:08:47 -0000 - Optional *timeval* if given is a floating point time value as accepted by + Optional *timeval* if given is a floating-point time value as accepted by :func:`time.gmtime` and :func:`time.localtime`, otherwise the current time is used. diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index de3b93f5e61073..8dfb7ad9c95c3e 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -1,5 +1,5 @@ -:mod:`ensurepip` --- Bootstrapping the ``pip`` installer -======================================================== +:mod:`!ensurepip` --- Bootstrapping the ``pip`` installer +========================================================= .. module:: ensurepip :synopsis: Bootstrapping the "pip" installer into an existing Python @@ -38,7 +38,7 @@ when creating a virtual environment) or after explicitly uninstalling :pep:`453`: Explicit bootstrapping of pip in Python installations The original rationale and specification for this module. -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-mobile-notavail.rst Command line interface ---------------------- diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 30d80ce8d488cc..16a9b0326e9f3d 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -1,5 +1,5 @@ -:mod:`enum` --- Support for enumerations -======================================== +:mod:`!enum` --- Support for enumerations +========================================= .. module:: enum :synopsis: Implementation of an enumeration class. @@ -44,7 +44,7 @@ using function-call syntax:: ... BLUE = 3 >>> # functional syntax - >>> Color = Enum('Color', ['RED', 'GREEN', 'BLUE']) + >>> Color = Enum('Color', [('RED', 1), ('GREEN', 2), ('BLUE', 3)]) Even though we can use :keyword:`class` syntax to create Enums, Enums are not normal Python classes. See @@ -170,7 +170,7 @@ Data Types final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. - .. method:: EnumType.__call__(cls, value, names=None, \*, module=None, qualname=None, type=None, start=1, boundary=None) + .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -279,6 +279,8 @@ Data Types >>> Color.RED.value 1 + Value of the member, can be set in :meth:`~Enum.__new__`. + .. note:: Enum member values Member values can be anything: :class:`int`, :class:`str`, etc. If @@ -286,13 +288,18 @@ Data Types appropriate value will be chosen for you. See :class:`auto` for the details. + While mutable/unhashable values, such as :class:`dict`, :class:`list` or + a mutable :class:`~dataclasses.dataclass`, can be used, they will have a + quadratic performance impact during creation relative to the + total number of mutable/unhashable values in the enum. + .. attribute:: Enum._name_ Name of the member. .. attribute:: Enum._value_ - Value of the member, can be set in :meth:`~object.__new__`. + Value of the member, can be set in :meth:`~Enum.__new__`. .. attribute:: Enum._order_ @@ -350,7 +357,7 @@ Data Types >>> PowersOfThree.SECOND.value 9 - .. method:: Enum.__init__(self, \*args, \**kwds) + .. method:: Enum.__init__(self, *args, **kwds) By default, does nothing. If multiple values are given in the member assignment, those values become separate arguments to ``__init__``; e.g. @@ -361,7 +368,7 @@ Data Types ``Weekday.__init__()`` would be called as ``Weekday.__init__(self, 1, 'Mon')`` - .. method:: Enum.__init_subclass__(cls, \**kwds) + .. method:: Enum.__init_subclass__(cls, **kwds) A *classmethod* that is used to further configure subsequent subclasses. By default, does nothing. @@ -388,17 +395,22 @@ Data Types >>> Build('deBUG') - .. method:: Enum.__new__(cls, \*args, \**kwds) + .. method:: Enum.__new__(cls, *args, **kwds) By default, doesn't exist. If specified, either in the enum class definition or in a mixin class (such as ``int``), all values given in the member assignment will be passed; e.g. >>> from enum import Enum - >>> class MyIntEnum(Enum): - ... SEVENTEEN = '1a', 16 + >>> class MyIntEnum(int, Enum): + ... TWENTYSIX = '1a', 16 + + results in the call ``int('1a', 16)`` and a value of ``26`` for the member. + + .. note:: - results in the call ``int('1a', 16)`` and a value of ``17`` for the member. + When writing a custom ``__new__``, do not use ``super().__new__`` -- + call the appropriate ``__new__`` instead. .. method:: Enum.__repr__(self) @@ -515,7 +527,7 @@ Data Types ``Flag`` is the same as :class:`Enum`, but its members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), ``^`` (*XOR*), and ``~`` (*INVERT*); - the results of those operators are members of the enumeration. + the results of those operations are (aliases of) members of the enumeration. .. method:: __contains__(self, value) @@ -558,6 +570,8 @@ Data Types >>> len(white) 3 + .. versionadded:: 3.11 + .. method:: __bool__(self): Returns *True* if any members in flag, *False* otherwise:: @@ -617,7 +631,7 @@ Data Types of two, starting with ``1``. .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It - is now:: + is now: >>> Color(0) # doctest: +SKIP @@ -649,7 +663,7 @@ Data Types * the result is a valid *IntFlag*: an *IntFlag* is returned * the result is not a valid *IntFlag*: the result depends on the :class:`FlagBoundary` setting - The :func:`repr()` of unnamed zero-valued flags has changed. It is now: + The :func:`repr` of unnamed zero-valued flags has changed. It is now: >>> Color(0) @@ -817,7 +831,7 @@ Supported ``__dunder__`` names :attr:`~EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` items. It is only available on the class. -:meth:`~object.__new__`, if specified, must create and return the enum members; +:meth:`~Enum.__new__`, if specified, must create and return the enum members; it is also a very good idea to set the member's :attr:`!_value_` appropriately. Once all the members are created it is no longer used. @@ -849,9 +863,15 @@ Supported ``_sunder_`` names For :class:`Flag` classes the next value chosen will be the next highest power-of-two. +- While ``_sunder_`` names are generally reserved for the further development + of the :class:`Enum` class and can not be used, some are explicitly allowed: + + - ``_repr_*`` (e.g. ``_repr_html_``), as used in `IPython's rich display`_ + .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` -.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_``, ``_repr_*`` +.. _`IPython's rich display`: https://ipython.readthedocs.io/en/stable/config/integrating.html#rich-display --------------- diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 283e8b013265d9..4983b8961b1c3f 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -1,5 +1,5 @@ -:mod:`errno` --- Standard errno system symbols -============================================== +:mod:`!errno` --- Standard errno system symbols +=============================================== .. module:: errno :synopsis: Standard errno system symbols. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 3191315049ad5a..b5ba86f1b19223 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -335,9 +335,9 @@ The following exceptions are the exceptions that are usually raised. .. note:: - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and :data:`NotImplemented` are not interchangeable, even though they have similar names and purposes. See - :data:`NotImplemented` for details on when to use it. + :data:`!NotImplemented` for details on when to use it. .. exception:: OSError([arg]) OSError(errno, strerror[, filename[, winerror[, filename2]]]) @@ -412,8 +412,26 @@ The following exceptions are the exceptions that are usually raised. represented. This cannot occur for integers (which would rather raise :exc:`MemoryError` than give up). However, for historical reasons, OverflowError is sometimes raised for integers that are outside a required - range. Because of the lack of standardization of floating point exception - handling in C, most floating point operations are not checked. + range. Because of the lack of standardization of floating-point exception + handling in C, most floating-point operations are not checked. + + +.. exception:: PythonFinalizationError + + This exception is derived from :exc:`RuntimeError`. It is raised when + an operation is blocked during interpreter shutdown also known as + :term:`Python finalization `. + + Examples of operations which can be blocked with a + :exc:`PythonFinalizationError` during the Python finalization: + + * Creating a new Python thread. + * :func:`os.fork`. + + See also the :func:`sys.is_finalizing` function. + + .. versionadded:: 3.13 + Previously, a plain :exc:`RuntimeError` was raised. .. exception:: RecursionError @@ -971,7 +989,8 @@ their subgroups based on the types of the contained exceptions. Returns an exception group with the same :attr:`message`, but which wraps the exceptions in ``excs``. - This method is used by :meth:`subgroup` and :meth:`split`. A + This method is used by :meth:`subgroup` and :meth:`split`, which + are used in various contexts to break up an exception group. A subclass needs to override it in order to make :meth:`subgroup` and :meth:`split` return instances of the subclass rather than :exc:`ExceptionGroup`. diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index f64dfeb5e081c7..4067d7912b88b2 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -1,5 +1,5 @@ -:mod:`faulthandler` --- Dump the Python traceback -================================================= +:mod:`!faulthandler` --- Dump the Python traceback +================================================== .. module:: faulthandler :synopsis: Dump the Python traceback. @@ -10,14 +10,15 @@ This module contains functions to dump Python tracebacks explicitly, on a fault, after a timeout, or on a user signal. Call :func:`faulthandler.enable` to -install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`, -:const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also +install fault handlers for the :const:`~signal.SIGSEGV`, +:const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS`, and +:const:`~signal.SIGILL` signals. You can also enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment variable or by using the :option:`-X` ``faulthandler`` command line option. The fault handler is compatible with system fault handlers like Apport or the Windows fault handler. The module uses an alternative stack for signal handlers -if the :c:func:`sigaltstack` function is available. This allows it to dump the +if the :c:func:`!sigaltstack` function is available. This allows it to dump the traceback even on a stack overflow. The fault handler is called on catastrophic cases and therefore can only use @@ -70,8 +71,9 @@ Fault handler state .. function:: enable(file=sys.stderr, all_threads=True) - Enable the fault handler: install handlers for the :const:`SIGSEGV`, - :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` + Enable the fault handler: install handlers for the :const:`~signal.SIGSEGV`, + :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` + and :const:`~signal.SIGILL` signals to dump the Python traceback. If *all_threads* is ``True``, produce tracebacks for every running thread. Otherwise, dump only the current thread. @@ -106,8 +108,8 @@ Dumping the tracebacks after a timeout Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call - :c:func:`_exit` with status=1 after dumping the tracebacks. (Note - :c:func:`_exit` exits the process immediately, which means it doesn't do any + :c:func:`!_exit` with status=1 after dumping the tracebacks. (Note + :c:func:`!_exit` exits the process immediately, which means it doesn't do any cleanup like flushing file buffers.) If the function is called twice, the new call replaces previous parameters and resets the timeout. The timer has a sub-second resolution. @@ -118,12 +120,12 @@ Dumping the tracebacks after a timeout This function is implemented using a watchdog thread. - .. versionchanged:: 3.7 - This function is now always available. - .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.7 + This function is now always available. + .. function:: cancel_dump_traceback_later() Cancel the last call to :func:`dump_traceback_later`. diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 13ad2dd7da5090..7bd64e43dd5bfe 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -1,5 +1,5 @@ -:mod:`fcntl` --- The ``fcntl`` and ``ioctl`` system calls -========================================================= +:mod:`!fcntl` --- The ``fcntl`` and ``ioctl`` system calls +========================================================== .. module:: fcntl :platform: Unix @@ -13,12 +13,12 @@ ---------------- -This module performs file control and I/O control on file descriptors. It is an -interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. For a -complete description of these calls, see :manpage:`fcntl(2)` and -:manpage:`ioctl(2)` Unix manual pages. +This module performs file and I/O control on file descriptors. It is an +interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. +See the :manpage:`fcntl(2)` and :manpage:`ioctl(2)` Unix manual pages +for full details. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI. All functions in this module take a file descriptor *fd* as their first argument. This can be an integer file descriptor, such as returned by @@ -101,7 +101,7 @@ The module defines the following functions: most likely to result in a segmentation violation or a more subtle data corruption. - If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. + If the :c:func:`fcntl` call fails, an :exc:`OSError` is raised. .. audit-event:: fcntl.fcntl fd,cmd,arg fcntl.fcntl @@ -139,7 +139,7 @@ The module defines the following functions: buffer 1024 bytes long which is then passed to :func:`ioctl` and copied back into the supplied buffer. - If the :c:func:`ioctl` fails, an :exc:`OSError` exception is raised. + If the :c:func:`ioctl` call fails, an :exc:`OSError` exception is raised. An example:: @@ -164,7 +164,7 @@ The module defines the following functions: :manpage:`flock(2)` for details. (On some systems, this function is emulated using :c:func:`fcntl`.) - If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. + If the :c:func:`flock` call fails, an :exc:`OSError` exception is raised. .. audit-event:: fcntl.flock fd,operation fcntl.flock @@ -176,17 +176,28 @@ The module defines the following functions: method are accepted as well) of the file to lock or unlock, and *cmd* is one of the following values: - * :const:`LOCK_UN` -- unlock - * :const:`LOCK_SH` -- acquire a shared lock - * :const:`LOCK_EX` -- acquire an exclusive lock + .. data:: LOCK_UN - When *cmd* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be - bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition. - If :const:`LOCK_NB` is used and the lock cannot be acquired, an + Release an existing lock. + + .. data:: LOCK_SH + + Acquire a shared lock. + + .. data:: LOCK_EX + + Acquire an exclusive lock. + + .. data:: LOCK_NB + + Bitwise OR with any of the other three ``LOCK_*`` constants to make + the request non-blocking. + + If :const:`!LOCK_NB` is used and the lock cannot be acquired, an :exc:`OSError` will be raised and the exception will have an *errno* - attribute set to :const:`EACCES` or :const:`EAGAIN` (depending on the + attribute set to :const:`~errno.EACCES` or :const:`~errno.EAGAIN` (depending on the operating system; for portability, check for both values). On at least some - systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a + systems, :const:`!LOCK_EX` can only be used if the file descriptor refers to a file opened for writing. *len* is the number of bytes to lock, *start* is the byte offset at diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index dfe4b7c59fd578..282d0e0d8db5cf 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -1,5 +1,5 @@ -:mod:`filecmp` --- File and Directory Comparisons -================================================= +:mod:`!filecmp` --- File and Directory Comparisons +================================================== .. module:: filecmp :synopsis: Compare files efficiently. @@ -70,7 +70,7 @@ The :mod:`filecmp` module defines the following functions: The :class:`dircmp` class ------------------------- -.. class:: dircmp(a, b, ignore=None, hide=None) +.. class:: dircmp(a, b, ignore=None, hide=None, *, shallow=True) Construct a new directory comparison object, to compare the directories *a* and *b*. *ignore* is a list of names to ignore, and defaults to @@ -78,7 +78,12 @@ The :class:`dircmp` class defaults to ``[os.curdir, os.pardir]``. The :class:`dircmp` class compares files by doing *shallow* comparisons - as described for :func:`filecmp.cmp`. + as described for :func:`filecmp.cmp` by default using the *shallow* + parameter. + + .. versionchanged:: 3.13 + + Added the *shallow* parameter. The :class:`dircmp` class provides the following methods: diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index f93e9a58791eeb..8f32b11e565365 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -1,5 +1,5 @@ -:mod:`fileinput` --- Iterate over lines from multiple input streams -=================================================================== +:mod:`!fileinput` --- Iterate over lines from multiple input streams +==================================================================== .. module:: fileinput :synopsis: Loop over standard input or a list of files. @@ -47,7 +47,7 @@ Lines are returned with any newlines intact, which means that the last line in a file may not have one. You can control how files are opened by providing an opening hook via the -*openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The +*openhook* parameter to :func:`fileinput.input` or :func:`FileInput`. The hook must be a function that takes two arguments, *filename* and *mode*, and returns an accordingly opened file-like object. If *encoding* and/or *errors* are specified, they will be passed to the hook as additional keyword arguments. diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 7cddecd5e80887..fda44923f204fc 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -1,5 +1,5 @@ -:mod:`fnmatch` --- Unix filename pattern matching -================================================= +:mod:`!fnmatch` --- Unix filename pattern matching +================================================== .. module:: fnmatch :synopsis: Unix shell style filename pattern matching. diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 887c3844d20faa..fc7f9a6301a915 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -1,5 +1,5 @@ -:mod:`fractions` --- Rational numbers -===================================== +:mod:`!fractions` --- Rational numbers +====================================== .. module:: fractions :synopsis: Rational numbers. @@ -17,25 +17,30 @@ The :mod:`fractions` module provides support for rational number arithmetic. A Fraction instance can be constructed from a pair of integers, from another rational number, or from a string. +.. index:: single: as_integer_ratio() + .. class:: Fraction(numerator=0, denominator=1) - Fraction(other_fraction) - Fraction(float) - Fraction(decimal) + Fraction(number) Fraction(string) The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance with value ``numerator/denominator``. If *denominator* is ``0``, it - raises a :exc:`ZeroDivisionError`. The second version requires that - *other_fraction* is an instance of :class:`numbers.Rational` and returns a - :class:`Fraction` instance with the same value. The next two versions accept - either a :class:`float` or a :class:`decimal.Decimal` instance, and return a - :class:`Fraction` instance with exactly the same value. Note that due to the - usual issues with binary floating-point (see :ref:`tut-fp-issues`), the + raises a :exc:`ZeroDivisionError`. + + The second version requires that *number* is an instance of + :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method + (this includes :class:`float` and :class:`decimal.Decimal`). + It returns a :class:`Fraction` instance with exactly the same value. + Assumed, that the :meth:`!as_integer_ratio` method returns a pair + of coprime integers and last one is positive. + Note that due to the + usual issues with binary point (see :ref:`tut-fp-issues`), the argument to ``Fraction(1.1)`` is not exactly equal to 11/10, and so ``Fraction(1.1)`` does *not* return ``Fraction(11, 10)`` as one might expect. (But see the documentation for the :meth:`limit_denominator` method below.) - The last version of the constructor expects a string or unicode instance. + + The last version of the constructor expects a string. The usual form for this instance is:: [sign] numerator ['/' denominator] @@ -87,7 +92,7 @@ another rational number, or from a string. .. versionchanged:: 3.9 The :func:`math.gcd` function is now used to normalize the *numerator* - and *denominator*. :func:`math.gcd` always return a :class:`int` type. + and *denominator*. :func:`math.gcd` always returns an :class:`int` type. Previously, the GCD type depended on *numerator* and *denominator*. .. versionchanged:: 3.11 @@ -110,6 +115,10 @@ another rational number, or from a string. Formatting of :class:`Fraction` instances without a presentation type now supports fill, alignment, sign handling, minimum width and grouping. + .. versionchanged:: 3.14 + The :class:`Fraction` constructor now accepts any objects with the + :meth:`!as_integer_ratio` method. + .. attribute:: numerator Numerator of the Fraction in lowest term. @@ -157,6 +166,16 @@ another rational number, or from a string. instance. + .. classmethod:: from_number(number) + + Alternative constructor which only accepts instances of + :class:`numbers.Integral`, :class:`numbers.Rational`, + :class:`float` or :class:`decimal.Decimal`, and objects with + the :meth:`!as_integer_ratio` method, but not strings. + + .. versionadded:: 3.14 + + .. method:: limit_denominator(max_denominator=1000000) Finds and returns the closest :class:`Fraction` to ``self`` that has diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 2f98a272c297ae..bb15322067245e 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -1,5 +1,5 @@ -:mod:`ftplib` --- FTP protocol client -===================================== +:mod:`!ftplib` --- FTP protocol client +====================================== .. module:: ftplib :synopsis: FTP protocol client (requires sockets). @@ -104,7 +104,7 @@ FTP objects :param timeout: A timeout in seconds for blocking operations like :meth:`connect` (default: the global default timeout setting). - :type timeout: int | None + :type timeout: float | None :param source_address: |param_doc_source_address| @@ -178,7 +178,7 @@ FTP objects :param timeout: A timeout in seconds for the connection attempt (default: the global default timeout setting). - :type timeout: int | None + :type timeout: float | None :param source_address: |param_doc_source_address| @@ -232,8 +232,8 @@ FTP objects .. method:: FTP.voidcmd(cmd) Send a simple command string to the server and handle the response. Return - nothing if a response code corresponding to success (codes in the range - 200--299) is received. Raise :exc:`error_reply` otherwise. + the response string if the response code corresponds to success (codes in + the range 200--299). Raise :exc:`error_reply` otherwise. .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd @@ -243,7 +243,7 @@ FTP objects Retrieve a file in binary transfer mode. :param str cmd: - An appropriate ``STOR`` command: :samp:`"STOR {filename}"`. + An appropriate ``RETR`` command: :samp:`"RETR {filename}"`. :param callback: A single parameter callable that is called @@ -483,7 +483,7 @@ FTP_TLS objects :param timeout: A timeout in seconds for blocking operations like :meth:`~FTP.connect` (default: the global default timeout setting). - :type timeout: int | None + :type timeout: float | None :param source_address: |param_doc_source_address| diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 27fce5aa0f1a63..03fc41fa793977 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -57,7 +57,7 @@ are always available. They are listed here in alphabetical order. .. function:: abs(x) Return the absolute value of a number. The argument may be an - integer, a floating point number, or an object implementing + integer, a floating-point number, or an object implementing :meth:`~object.__abs__`. If the argument is a complex number, its magnitude is returned. @@ -141,10 +141,11 @@ are always available. They are listed here in alphabetical order. See also :func:`format` for more information. -.. class:: bool(x=False) +.. class:: bool(object=False, /) - Return a Boolean value, i.e. one of ``True`` or ``False``. *x* is converted - using the standard :ref:`truth testing procedure `. If *x* is false + Return a Boolean value, i.e. one of ``True`` or ``False``. The argument + is converted using the standard :ref:`truth testing procedure `. + If the argument is false or omitted, this returns ``False``; otherwise, it returns ``True``. The :class:`bool` class is a subclass of :class:`int` (see :ref:`typesnumeric`). It cannot be subclassed further. Its only instances are ``False`` and @@ -153,14 +154,14 @@ are always available. They are listed here in alphabetical order. .. index:: pair: Boolean; type .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The parameter is now positional-only. .. function:: breakpoint(*args, **kws) This function drops you into the debugger at the call site. Specifically, it calls :func:`sys.breakpointhook`, passing ``args`` and ``kws`` straight through. By default, ``sys.breakpointhook()`` calls - :func:`pdb.set_trace()` expecting no arguments. In this case, it is + :func:`pdb.set_trace` expecting no arguments. In this case, it is purely a convenience function so you don't have to explicitly import :mod:`pdb` or type as much code to enter the debugger. However, :func:`sys.breakpointhook` can be set to some other function and @@ -282,9 +283,11 @@ are always available. They are listed here in alphabetical order. :func:`property`. .. versionchanged:: 3.10 - Class methods now inherit the method attributes (``__module__``, - ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``) and - have a new ``__wrapped__`` attribute. + Class methods now inherit the method attributes + (:attr:`~function.__module__`, :attr:`~function.__name__`, + :attr:`~function.__qualname__`, :attr:`~function.__doc__` and + :attr:`~function.__annotations__`) and have a new ``__wrapped__`` + attribute. .. deprecated-removed:: 3.11 3.13 Class methods can no longer wrap other :term:`descriptors ` such as @@ -371,29 +374,75 @@ are always available. They are listed here in alphabetical order. support for top-level ``await``, ``async for``, and ``async with``. -.. class:: complex(real=0, imag=0) - complex(string) +.. class:: complex(number=0, /) + complex(string, /) + complex(real=0, imag=0) + + Convert a single string or number to a complex number, or create a + complex number from real and imaginary parts. + + Examples: + + .. doctest:: + + >>> complex('+1.23') + (1.23+0j) + >>> complex('-4.5j') + -4.5j + >>> complex('-1.23+4.5j') + (-1.23+4.5j) + >>> complex('\t( -1.23+4.5J )\n') + (-1.23+4.5j) + >>> complex('-Infinity+NaNj') + (-inf+nanj) + >>> complex(1.23) + (1.23+0j) + >>> complex(imag=-4.5) + -4.5j + >>> complex(-1.23, 4.5) + (-1.23+4.5j) + + If the argument is a string, it must contain either a real part (in the + same format as for :func:`float`) or an imaginary part (in the same + format but with a ``'j'`` or ``'J'`` suffix), or both real and imaginary + parts (the sign of the imaginary part is mandatory in this case). + The string can optionally be surrounded by whitespaces and the round + parentheses ``'('`` and ``')'``, which are ignored. + The string must not contain whitespace between ``'+'``, ``'-'``, the + ``'j'`` or ``'J'`` suffix, and the decimal number. + For example, ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises + :exc:`ValueError`. + More precisely, the input must conform to the :token:`~float:complexvalue` + production rule in the following grammar, after parentheses and leading and + trailing whitespace characters are removed: - Return a complex number with the value *real* + *imag*\*1j or convert a string - or number to a complex number. If the first parameter is a string, it will - be interpreted as a complex number and the function must be called without a - second parameter. The second parameter can never be a string. Each argument - may be any numeric type (including complex). If *imag* is omitted, it - defaults to zero and the constructor serves as a numeric conversion like - :class:`int` and :class:`float`. If both arguments are omitted, returns - ``0j``. + .. productionlist:: float + complexvalue: `floatvalue` | + : `floatvalue` ("j" | "J") | + : `floatvalue` `sign` `absfloatvalue` ("j" | "J") + If the argument is a number, the constructor serves as a numeric + conversion like :class:`int` and :class:`float`. For a general Python object ``x``, ``complex(x)`` delegates to - ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back - to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + ``x.__complex__()``. + If :meth:`~object.__complex__` is not defined then it falls back + to :meth:`~object.__float__`. + If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. - .. note:: + If two arguments are provided or keyword arguments are used, each argument + may be any numeric type (including complex). + If both arguments are real numbers, return a complex number with the real + component *real* and the imaginary component *imag*. + If both arguments are complex numbers, return a complex number with the real + component ``real.real-imag.imag`` and the imaginary component + ``real.imag+imag.real``. + If one of arguments is a real number, only its real component is used in + the above expressions. + + See also :meth:`complex.from_number` which only accepts a single numeric argument. - When converting from a string, the string must not contain whitespace - around the central ``+`` or ``-`` operator. For example, - ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises - :exc:`ValueError`. + If all arguments are omitted, returns ``0j``. The complex type is described in :ref:`typesnumeric`. @@ -404,6 +453,10 @@ are always available. They are listed here in alphabetical order. Falls back to :meth:`~object.__index__` if :meth:`~object.__complex__` and :meth:`~object.__float__` are not defined. + .. deprecated:: 3.14 + Passing a complex number as the *real* or *imag* argument is now + deprecated; it should only be passed as a single positional argument. + .. function:: delattr(object, name) @@ -493,7 +546,7 @@ are always available. They are listed here in alphabetical order. Take two (non-complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using integer division. With mixed operand types, the rules for binary arithmetic operators apply. For - integers, the result is the same as ``(a // b, a % b)``. For floating point + integers, the result is the same as ``(a // b, a % b)``. For floating-point numbers the result is ``(q, a % b)``, where *q* is usually ``math.floor(a / b)`` but may be 1 less than that. In any case ``q * b + a % b`` is very close to *a*, if ``a % b`` is non-zero it has the same sign as *b*, and ``0 @@ -524,29 +577,45 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression, globals=None, locals=None) +.. function:: eval(source, /, globals=None, locals=None) + + :param source: + A Python expression. + :type source: :class:`str` | :ref:`code object ` + + :param globals: + The global namespace (default: ``None``). + :type globals: :class:`dict` | ``None`` + + :param locals: + The local namespace (default: ``None``). + :type locals: :term:`mapping` | ``None`` + + :returns: The result of the evaluated expression. + :raises: Syntax errors are reported as exceptions. + + .. warning:: - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. + This function executes arbitrary code. Calling it with + user-supplied input may lead to security vulnerabilities. The *expression* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* - dictionaries as global and local namespace. If the *globals* dictionary is + mappings as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is inserted under that key before *expression* is parsed. That way you can control what builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* dictionary is omitted it defaults to the - *globals* dictionary. If both dictionaries are omitted, the expression is + :func:`eval`. If the *locals* mapping is omitted it defaults to the + *globals* dictionary. If both mappings are omitted, the expression is executed with the *globals* and *locals* in the environment where - :func:`eval` is called. Note, *eval()* does not have access to the + :func:`eval` is called. Note, *eval()* will only have access to the :term:`nested scopes ` (non-locals) in the enclosing - environment. + environment if they are already referenced in the scope that is calling + :func:`eval` (e.g. via a :keyword:`nonlocal` statement). - The return value is the result of - the evaluated expression. Syntax errors are reported as exceptions. Example: + Example: >>> x = 1 >>> eval('x+1') @@ -573,11 +642,25 @@ are always available. They are listed here in alphabetical order. Raises an :ref:`auditing event ` ``exec`` with the code object as the argument. Code compilation events may also be raised. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. index:: pair: built-in function; exec -.. function:: exec(object, globals=None, locals=None, /, *, closure=None) +.. function:: exec(source, /, globals=None, locals=None, *, closure=None) - This function supports dynamic execution of Python code. *object* must be + .. warning:: + + This function executes arbitrary code. Calling it with + user-supplied input may lead to security vulnerabilities. + + This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as a suite of Python statements which is then executed (unless a syntax error occurs). [#]_ If it is a code object, it is simply executed. In all cases, @@ -594,9 +677,15 @@ are always available. They are listed here in alphabetical order. will be used for both the global and the local variables. If *globals* and *locals* are given, they are used for the global and local variables, respectively. If provided, *locals* can be any mapping object. Remember - that at the module level, globals and locals are the same dictionary. If exec - gets two separate objects as *globals* and *locals*, the code will be - executed as if it were embedded in a class definition. + that at the module level, globals and locals are the same dictionary. + + .. note:: + + When ``exec`` gets two separate objects as *globals* and *locals*, the + code will be executed as if it were embedded in a class definition. This + means functions and classes defined in the executed code will not be able + to access variables assigned at the top level (as the "top level" + variables are treated as class variables in a class definition). If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module @@ -605,9 +694,10 @@ are always available. They are listed here in alphabetical order. ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. The *closure* argument specifies a closure--a tuple of cellvars. - It's only valid when the *object* is a code object containing free variables. - The length of the tuple must exactly match the number of free variables - referenced by the code object. + It's only valid when the *object* is a code object containing + :term:`free (closure) variables `. + The length of the tuple must exactly match the length of the code object's + :attr:`~codeobject.co_freevars` attribute. .. audit-event:: exec code_object exec @@ -617,19 +707,27 @@ are always available. They are listed here in alphabetical order. .. note:: The built-in functions :func:`globals` and :func:`locals` return the current - global and local dictionary, respectively, which may be useful to pass around + global and local namespace, respectively, which may be useful to pass around for use as the second and third argument to :func:`exec`. .. note:: - The default *locals* act as described for function :func:`locals` below: - modifications to the default *locals* dictionary should not be attempted. + The default *locals* act as described for function :func:`locals` below. Pass an explicit *locals* dictionary if you need to see effects of the code on *locals* after function :func:`exec` returns. .. versionchanged:: 3.11 Added the *closure* parameter. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. function:: filter(function, iterable) @@ -648,21 +746,38 @@ are always available. They are listed here in alphabetical order. elements of *iterable* for which *function* is false. -.. class:: float(x=0.0) +.. class:: float(number=0.0, /) + float(string, /) .. index:: single: NaN single: Infinity - Return a floating point number constructed from a number or string *x*. + Return a floating-point number constructed from a number or a string. + + Examples: + + .. doctest:: + + >>> float('+1.23') + 1.23 + >>> float(' -12345\n') + -12345.0 + >>> float('1e-003') + 0.001 + >>> float('+1E6') + 1000000.0 + >>> float('-Infinity') + -inf If the argument is a string, it should contain a decimal number, optionally preceded by a sign, and optionally embedded in whitespace. The optional sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN - (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the ``floatvalue`` production rule in the following - grammar, after leading and trailing whitespace characters are removed: + (not-a-number), or positive or negative infinity. + More precisely, the input must conform to the :token:`~float:floatvalue` + production rule in the following grammar, after leading and trailing + whitespace characters are removed: .. productionlist:: float sign: "+" | "-" @@ -671,15 +786,16 @@ are always available. They are listed here in alphabetical order. digit: digitpart: `digit` (["_"] `digit`)* number: [`digitpart`] "." `digitpart` | `digitpart` ["."] - exponent: ("e" | "E") ["+" | "-"] `digitpart` - floatnumber: number [`exponent`] - floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + exponent: ("e" | "E") [`sign`] `digitpart` + floatnumber: `number` [`exponent`] + absfloatvalue: `floatnumber` | `infinity` | `nan` + floatvalue: [`sign`] `absfloatvalue` Case is not significant, so, for example, "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for positive infinity. - Otherwise, if the argument is an integer or a floating point number, a - floating point number with the same value (within Python's floating point + Otherwise, if the argument is an integer or a floating-point number, a + floating-point number with the same value (within Python's floating-point precision) is returned. If the argument is outside the range of a Python float, an :exc:`OverflowError` will be raised. @@ -687,20 +803,9 @@ are always available. They are listed here in alphabetical order. ``x.__float__()``. If :meth:`~object.__float__` is not defined then it falls back to :meth:`~object.__index__`. - If no argument is given, ``0.0`` is returned. - - Examples:: + See also :meth:`float.from_number` which only accepts a numeric argument. - >>> float('+1.23') - 1.23 - >>> float(' -12345\n') - -12345.0 - >>> float('1e-003') - 0.001 - >>> float('+1E6') - 1000000.0 - >>> float('-Infinity') - -inf + If no argument is given, ``0.0`` is returned. The float type is described in :ref:`typesnumeric`. @@ -708,7 +813,7 @@ are always available. They are listed here in alphabetical order. Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The parameter is now positional-only. .. versionchanged:: 3.8 Falls back to :meth:`~object.__index__` if :meth:`~object.__float__` is not defined. @@ -892,17 +997,35 @@ are always available. They are listed here in alphabetical order. with the result after successfully reading input. -.. class:: int(x=0) - int(x, base=10) +.. class:: int(number=0, /) + int(string, /, base=10) + + Return an integer object constructed from a number or a string, or return + ``0`` if no arguments are given. + + Examples: - Return an integer object constructed from a number or string *x*, or return - ``0`` if no arguments are given. If *x* defines :meth:`~object.__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`~object.__index__`, - it returns ``x.__index__()``. If *x* defines :meth:`~object.__trunc__`, - it returns ``x.__trunc__()``. - For floating point numbers, this truncates towards zero. + .. doctest:: - If *x* is not a number or if *base* is given, then *x* must be a string, + >>> int(123.45) + 123 + >>> int('123') + 123 + >>> int(' -12_345\n') + -12345 + >>> int('FACE', 16) + 64206 + >>> int('0xface', 0) + 64206 + >>> int('01110011', base=2) + 115 + + If the argument defines :meth:`~object.__int__`, + ``int(x)`` returns ``x.__int__()``. If the argument defines + :meth:`~object.__index__`, it returns ``x.__index__()``. + For floating-point numbers, this truncates towards zero. + + If the argument is not a number or if *base* is given, then it must be a string, :class:`bytes`, or :class:`bytearray` instance representing an integer in radix *base*. Optionally, the string can be preceded by ``+`` or ``-`` (with no space in between), have leading zeros, be surrounded by whitespace, @@ -932,22 +1055,22 @@ are always available. They are listed here in alphabetical order. Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The first parameter is now positional-only. .. versionchanged:: 3.8 Falls back to :meth:`~object.__index__` if :meth:`~object.__int__` is not defined. - .. versionchanged:: 3.11 - The delegation to :meth:`~object.__trunc__` is deprecated. - .. versionchanged:: 3.11 :class:`int` string inputs and string representations can be limited to help avoid denial of service attacks. A :exc:`ValueError` is raised when - the limit is exceeded while converting a string *x* to an :class:`int` or + the limit is exceeded while converting a string to an :class:`int` or when converting an :class:`int` into a string would exceed the limit. See the :ref:`integer string conversion length limitation ` documentation. + .. versionchanged:: 3.14 + :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. + .. function:: isinstance(object, classinfo) Return ``True`` if the *object* argument is an instance of the *classinfo* @@ -1031,14 +1154,56 @@ are always available. They are listed here in alphabetical order. .. function:: locals() - Update and return a dictionary representing the current local symbol table. - Free variables are returned by :func:`locals` when it is called in function - blocks, but not in class blocks. Note that at the module level, :func:`locals` - and :func:`globals` are the same dictionary. + Return a mapping object representing the current local symbol table, with + variable names as the keys, and their currently bound references as the + values. + + At module scope, as well as when using :func:`exec` or :func:`eval` with + a single namespace, this function returns the same namespace as + :func:`globals`. + + At class scope, it returns the namespace that will be passed to the + metaclass constructor. + + When using ``exec()`` or ``eval()`` with separate local and global + arguments, it returns the local namespace passed in to the function call. + + In all of the above cases, each call to ``locals()`` in a given frame of + execution will return the *same* mapping object. Changes made through + the mapping object returned from ``locals()`` will be visible as assigned, + reassigned, or deleted local variables, and assigning, reassigning, or + deleting local variables will immediately affect the contents of the + returned mapping object. + + In an :term:`optimized scope` (including functions, generators, and + coroutines), each call to ``locals()`` instead returns a fresh dictionary + containing the current bindings of the function's local variables and any + nonlocal cell references. In this case, name binding changes made via the + returned dict are *not* written back to the corresponding local variables + or nonlocal cell references, and assigning, reassigning, or deleting local + variables and nonlocal cell references does *not* affect the contents + of previously returned dictionaries. + + Calling ``locals()`` as part of a comprehension in a function, generator, or + coroutine is equivalent to calling it in the containing scope, except that + the comprehension's initialised iteration variables will be included. In + other scopes, it behaves as if the comprehension were running as a nested + function. + + Calling ``locals()`` as part of a generator expression is equivalent to + calling it in a nested generator function. + + .. versionchanged:: 3.12 + The behaviour of ``locals()`` in a comprehension has been updated as + described in :pep:`709`. + + .. versionchanged:: 3.13 + As part of :pep:`667`, the semantics of mutating the mapping objects + returned from this function are now defined. The behavior in + :term:`optimized scopes ` is now as described above. + Aside from being defined, the behaviour in other scopes remains + unchanged from previous versions. - .. note:: - The contents of this dictionary should not be modified; changes may not - affect the values of local and free variables used by the interpreter. .. function:: map(function, iterable, *iterables) @@ -1128,14 +1293,16 @@ are always available. They are listed here in alphabetical order. .. class:: object() - Return a new featureless object. :class:`object` is a base for all classes. - It has methods that are common to all instances of Python classes. This - function does not accept any arguments. + This is the ultimate base class of all other classes. It has methods + that are common to all instances of Python classes. When the constructor + is called, it returns a new featureless object. The constructor does not + accept any arguments. .. note:: - :class:`object` does *not* have a :attr:`~object.__dict__`, so you can't - assign arbitrary attributes to an instance of the :class:`object` class. + :class:`object` instances do *not* have :attr:`~object.__dict__` + attributes, so you can't assign arbitrary attributes to an instance of + :class:`object`. .. function:: oct(x) @@ -1184,7 +1351,7 @@ are always available. They are listed here in alphabetical order. (which on *some* Unix systems, means that *all* writes append to the end of the file regardless of the current seek position). In text mode, if *encoding* is not specified the encoding used is platform-dependent: - :func:`locale.getencoding()` is called to get the current locale encoding. + :func:`locale.getencoding` is called to get the current locale encoding. (For reading and writing raw bytes use binary mode and leave *encoding* unspecified.) The available modes are: @@ -1357,7 +1524,7 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. - .. audit-event:: open file,mode,flags open + .. audit-event:: open path,mode,flags open The ``mode`` and ``flags`` arguments may have been modified or inferred from the original call. @@ -1413,7 +1580,9 @@ are always available. They are listed here in alphabetical order. returns ``100``, but ``pow(10, -2)`` returns ``0.01``. For a negative base of type :class:`int` or :class:`float` and a non-integral exponent, a complex result is delivered. For example, ``pow(-9, 0.5)`` returns a value close - to ``3j``. + to ``3j``. Whereas, for a negative base of type :class:`int` or :class:`float` + with an integral exponent, a float result is delivered. For example, + ``pow(-9, 2.0)`` returns ``81.0``. For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must also be of integer type and *mod* must be nonzero. If *mod* is present and @@ -1546,6 +1715,13 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.5 The docstrings of property objects are now writeable. + .. attribute:: __name__ + + Attribute holding the name of the property. The name of the property + can be changed at runtime. + + .. versionadded:: 3.13 + .. _func-range: .. class:: range(stop) @@ -1569,6 +1745,16 @@ are always available. They are listed here in alphabetical order. If :func:`sys.displayhook` is not accessible, this function will raise :exc:`RuntimeError`. + This class has a custom representation that can be evaluated:: + + class Person: + def __init__(self, name, age): + self.name = name + self.age = age + + def __repr__(self): + return f"Person('{self.name}', {self.age})" + .. function:: reversed(seq) @@ -1713,8 +1899,9 @@ are always available. They are listed here in alphabetical order. :ref:`function` for details. A static method can be called either on the class (such as ``C.f()``) or on - an instance (such as ``C().f()``). Moreover, they can be called as regular - functions (such as ``f()``). + an instance (such as ``C().f()``). + Moreover, the static method :term:`descriptor` is also callable, so it can + be used in the class definition (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also, see :func:`classmethod` for a variant that is useful for creating alternate class @@ -1735,10 +1922,11 @@ are always available. They are listed here in alphabetical order. For more information on static methods, see :ref:`types`. .. versionchanged:: 3.10 - Static methods now inherit the method attributes (``__module__``, - ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``), - have a new ``__wrapped__`` attribute, and are now callable as regular - functions. + Static methods now inherit the method attributes + (:attr:`~function.__module__`, :attr:`~function.__name__`, + :attr:`~function.__qualname__`, :attr:`~function.__doc__` and + :attr:`~function.__annotations__`), have a new ``__wrapped__`` attribute, + and are now callable as regular functions. .. index:: @@ -1763,7 +1951,7 @@ are always available. They are listed here in alphabetical order. For some use cases, there are good alternatives to :func:`sum`. The preferred, fast way to concatenate a sequence of strings is by calling - ``''.join(sequence)``. To add floating point values with extended precision, + ``''.join(sequence)``. To add floating-point values with extended precision, see :func:`math.fsum`\. To concatenate a series of iterables, consider using :func:`itertools.chain`. @@ -1773,6 +1961,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.12 Summation of floats switched to an algorithm that gives higher accuracy and better commutativity on most builds. + .. versionchanged:: 3.14 + Added specialization for summation of complexes, + using same algorithm as for summation of floats. + .. class:: super() super(type, object_or_type=None) @@ -1785,14 +1977,14 @@ are always available. They are listed here in alphabetical order. to be searched. The search starts from the class right after the *type*. - For example, if :attr:`~class.__mro__` of *object_or_type* is + For example, if :attr:`~type.__mro__` of *object_or_type* is ``D -> B -> C -> A -> object`` and the value of *type* is ``B``, then :func:`super` searches ``C -> A -> object``. - The :attr:`~class.__mro__` attribute of the *object_or_type* lists the method - resolution search order used by both :func:`getattr` and :func:`super`. The - attribute is dynamic and can change whenever the inheritance hierarchy is - updated. + The :attr:`~type.__mro__` attribute of the class corresponding to + *object_or_type* lists the method resolution search order used by both + :func:`getattr` and :func:`super`. The attribute is dynamic and can change + whenever the inheritance hierarchy is updated. If the second argument is omitted, the super object returned is unbound. If the second argument is an object, ``isinstance(obj, type)`` must be true. If @@ -1851,6 +2043,10 @@ are always available. They are listed here in alphabetical order. :func:`super`, see `guide to using super() `_. + .. versionchanged:: 3.14 + :class:`super` objects are now :mod:`pickleable ` and + :mod:`copyable `. + .. _func-tuple: .. class:: tuple() @@ -1868,28 +2064,30 @@ are always available. They are listed here in alphabetical order. With one argument, return the type of an *object*. The return value is a type object and generally the same object as returned by - :attr:`object.__class__ `. + :attr:`object.__class__`. The :func:`isinstance` built-in function is recommended for testing the type of an object, because it takes subclasses into account. - With three arguments, return a new type object. This is essentially a dynamic form of the :keyword:`class` statement. The *name* string is - the class name and becomes the :attr:`~definition.__name__` attribute. + the class name and becomes the :attr:`~type.__name__` attribute. The *bases* tuple contains the base classes and becomes the - :attr:`~class.__bases__` attribute; if empty, :class:`object`, the + :attr:`~type.__bases__` attribute; if empty, :class:`object`, the ultimate base of all classes, is added. The *dict* dictionary contains attribute and method definitions for the class body; it may be copied - or wrapped before becoming the :attr:`~object.__dict__` attribute. - The following two statements create identical :class:`type` objects: + or wrapped before becoming the :attr:`~type.__dict__` attribute. + The following two statements create identical :class:`!type` objects: >>> class X: ... a = 1 ... >>> X = type('X', (), dict(a=1)) - See also :ref:`bltin-type-objects`. + See also: + + * :ref:`Documentation on attributes and methods on classes `. + * :ref:`bltin-type-objects` Keyword arguments provided to the three argument form are passed to the appropriate metaclass machinery (usually :meth:`~object.__init_subclass__`) @@ -1899,28 +2097,32 @@ are always available. They are listed here in alphabetical order. See also :ref:`class-customization`. .. versionchanged:: 3.6 - Subclasses of :class:`type` which don't override ``type.__new__`` may no + Subclasses of :class:`!type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. .. function:: vars() vars(object) Return the :attr:`~object.__dict__` attribute for a module, class, instance, - or any other object with a :attr:`~object.__dict__` attribute. + or any other object with a :attr:`!__dict__` attribute. Objects such as modules and instances have an updateable :attr:`~object.__dict__` attribute; however, other objects may have write restrictions on their - :attr:`~object.__dict__` attributes (for example, classes use a + :attr:`!__dict__` attributes (for example, classes use a :class:`types.MappingProxyType` to prevent direct dictionary updates). - Without an argument, :func:`vars` acts like :func:`locals`. Note, the - locals dictionary is only useful for reads since updates to the locals - dictionary are ignored. + Without an argument, :func:`vars` acts like :func:`locals`. A :exc:`TypeError` exception is raised if an object is specified but it doesn't have a :attr:`~object.__dict__` attribute (for example, if its class defines the :attr:`~object.__slots__` attribute). + .. versionchanged:: 3.13 + + The result of calling this function without an argument has been + updated as described for the :func:`locals` builtin. + + .. function:: zip(*iterables, strict=False) Iterate over several iterables in parallel, producing tuples with an item diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 20fcbe76c36985..e26a2226aa947a 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -1,5 +1,5 @@ -:mod:`functools` --- Higher-order functions and operations on callable objects -============================================================================== +:mod:`!functools` --- Higher-order functions and operations on callable objects +=============================================================================== .. module:: functools :synopsis: Higher-order functions and operations on callable objects. @@ -34,7 +34,7 @@ The :mod:`functools` module defines the following functions: Returns the same as ``lru_cache(maxsize=None)``, creating a thin wrapper around a dictionary lookup for the function arguments. Because it never needs to evict old values, this is smaller and faster than - :func:`lru_cache()` with a size limit. + :func:`lru_cache` with a size limit. For example:: @@ -218,7 +218,7 @@ The :mod:`functools` module defines the following functions: cache. See :ref:`faq-cache-method-calls` An `LRU (least recently used) cache - `_ + `_ works best when the most recent calls are the best predictors of upcoming calls (for example, the most popular articles on a news server tend to change each day). The cache's size limit assures that the cache does not @@ -325,9 +325,17 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.2 .. versionchanged:: 3.4 - Returning NotImplemented from the underlying comparison function for + Returning ``NotImplemented`` from the underlying comparison function for unrecognised types is now supported. +.. data:: Placeholder + + A singleton object used as a sentinel to reserve a place + for positional arguments when calling :func:`partial` + and :func:`partialmethod`. + + .. versionadded:: 3.14 + .. function:: partial(func, /, *args, **keywords) Return a new :ref:`partial object` which when called @@ -338,26 +346,68 @@ The :mod:`functools` module defines the following functions: Roughly equivalent to:: def partial(func, /, *args, **keywords): - def newfunc(*fargs, **fkeywords): - newkeywords = {**keywords, **fkeywords} - return func(*args, *fargs, **newkeywords) + def newfunc(*more_args, **more_keywords): + return func(*args, *more_args, **(keywords | more_keywords)) newfunc.func = func newfunc.args = args newfunc.keywords = keywords return newfunc - The :func:`partial` is used for partial function application which "freezes" + The :func:`!partial` function is used for partial function application which "freezes" some portion of a function's arguments and/or keywords resulting in a new object with a simplified signature. For example, :func:`partial` can be used to create a callable that behaves like the :func:`int` function where the *base* argument - defaults to two: + defaults to ``2``: + + .. doctest:: - >>> from functools import partial >>> basetwo = partial(int, base=2) >>> basetwo.__doc__ = 'Convert base 2 string to an int.' >>> basetwo('10010') 18 + If :data:`Placeholder` sentinels are present in *args*, they will be filled first + when :func:`!partial` is called. This makes it possible to pre-fill any positional + argument with a call to :func:`!partial`; without :data:`!Placeholder`, only the + first positional argument can be pre-filled. + + If any :data:`!Placeholder` sentinels are present, all must be filled at call time: + + .. doctest:: + + >>> say_to_world = partial(print, Placeholder, Placeholder, "world!") + >>> say_to_world('Hello', 'dear') + Hello dear world! + + Calling ``say_to_world('Hello')`` raises a :exc:`TypeError`, because + only one positional argument is provided, but there are two placeholders + that must be filled in. + + If :func:`!partial` is applied to an existing :func:`!partial` object, + :data:`!Placeholder` sentinels of the input object are filled in with + new positional arguments. + A placeholder can be retained by inserting a new + :data:`!Placeholder` sentinel to the place held by a previous :data:`!Placeholder`: + + .. doctest:: + + >>> from functools import partial, Placeholder as _ + >>> remove = partial(str.replace, _, _, '') + >>> message = 'Hello, dear dear world!' + >>> remove(message, ' dear') + 'Hello, world!' + >>> remove_dear = partial(remove, _, ' dear') + >>> remove_dear(message) + 'Hello, world!' + >>> remove_first_dear = partial(remove_dear, _, 1) + >>> remove_first_dear(message) + 'Hello, dear world!' + + :data:`!Placeholder` has no special treatment when used in a keyword + argument to :func:`!partial`. + + .. versionchanged:: 3.14 + Added support for :data:`Placeholder` in positional arguments. .. class:: partialmethod(func, /, *args, **keywords) @@ -492,6 +542,25 @@ The :mod:`functools` module defines the following functions: ... print(arg.real, arg.imag) ... + For code that dispatches on a collections type (e.g., ``list``), but wants + to typehint the items of the collection (e.g., ``list[int]``), the + dispatch type should be passed explicitly to the decorator itself with the + typehint going into the function definition:: + + >>> @fun.register(list) + ... def _(arg: list[int], verbose=False): + ... if verbose: + ... print("Enumerate this:") + ... for i, elem in enumerate(arg): + ... print(i, elem) + + .. note:: + + At runtime the function will dispatch on an instance of a list regardless + of the type contained within the list i.e. ``[1,2,3]`` will be + dispatched the same as ``["foo", "bar", "baz"]``. The annotation + provided in this example is for static type checkers only and has no + runtime impact. To enable registering :term:`lambdas` and pre-existing functions, the :func:`register` attribute can also be used in a functional form:: @@ -646,9 +715,11 @@ The :mod:`functools` module defines the following functions: attributes of the wrapper function are updated with the corresponding attributes from the original function. The default values for these arguments are the module level constants ``WRAPPER_ASSIGNMENTS`` (which assigns to the wrapper - function's ``__module__``, ``__name__``, ``__qualname__``, ``__annotations__`` - and ``__doc__``, the documentation string) and ``WRAPPER_UPDATES`` (which - updates the wrapper function's ``__dict__``, i.e. the instance dictionary). + function's :attr:`~function.__module__`, :attr:`~function.__name__`, + :attr:`~function.__qualname__`, :attr:`~function.__annotations__`, + :attr:`~function.__type_params__`, and :attr:`~function.__doc__`, the + documentation string) and ``WRAPPER_UPDATES`` (which updates the wrapper + function's :attr:`~function.__dict__`, i.e. the instance dictionary). To allow access to the original function for introspection and other purposes (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function @@ -667,13 +738,9 @@ The :mod:`functools` module defines the following functions: on the wrapper function). :exc:`AttributeError` is still raised if the wrapper function itself is missing any attributes named in *updated*. - .. versionadded:: 3.2 - Automatic addition of the ``__wrapped__`` attribute. - - .. versionadded:: 3.2 - Copying of the ``__annotations__`` attribute by default. - .. versionchanged:: 3.2 + The ``__wrapped__`` attribute is now automatically added. + The :attr:`~function.__annotations__` attribute is now copied by default. Missing attributes no longer trigger an :exc:`AttributeError`. .. versionchanged:: 3.4 @@ -681,6 +748,9 @@ The :mod:`functools` module defines the following functions: function, even if that function defined a ``__wrapped__`` attribute. (see :issue:`17482`) + .. versionchanged:: 3.12 + The :attr:`~function.__type_params__` attribute is now copied by default. + .. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) @@ -741,9 +811,7 @@ have three read-only attributes: The keyword arguments that will be supplied when the :class:`partial` object is called. -:class:`partial` objects are like :class:`function` objects in that they are +:class:`partial` objects are like :ref:`function objects ` in that they are callable, weak referenceable, and can have attributes. There are some important -differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes -are not created automatically. Also, :class:`partial` objects defined in -classes behave like static methods and do not transform into bound methods -during instance attribute look-up. +differences. For instance, the :attr:`~definition.__name__` and :attr:`~definition.__doc__` attributes +are not created automatically. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index e36a71af2b64ab..1065ec30802841 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -1,5 +1,5 @@ -:mod:`gc` --- Garbage Collector interface -========================================= +:mod:`!gc` --- Garbage Collector interface +========================================== .. module:: gc :synopsis: Interface to the cycle-detecting garbage collector. @@ -40,11 +40,18 @@ The :mod:`gc` module provides the following functions: .. function:: collect(generation=2) - With no arguments, run a full collection. The optional argument *generation* + Perform a collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The sum of + :exc:`ValueError` is raised if the generation number is invalid. The sum of collected objects and uncollectable objects is returned. + Calling ``gc.collect(0)`` will perform a GC collection on the young generation. + + Calling ``gc.collect(1)`` will perform a GC collection on the young generation + and an increment of the old generation. + + Calling ``gc.collect(2)`` or ``gc.collect()`` performs a full collection + The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) is run. Not all items in some free lists may be freed due to the @@ -53,6 +60,9 @@ The :mod:`gc` module provides the following functions: The effect of calling ``gc.collect()`` while the interpreter is already performing a collection is undefined. + .. versionchanged:: 3.13 + ``generation=1`` performs an increment of collection. + .. function:: set_debug(flags) @@ -68,13 +78,20 @@ The :mod:`gc` module provides the following functions: .. function:: get_objects(generation=None) + Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not None, return only the objects tracked by - the collector that are in that generation. + returned. If *generation* is not ``None``, return only the objects as follows: + + * 0: All objects in the young generation + * 1: No objects, as there is no generation 1 (as of Python 3.13) + * 2: All objects in the old generation .. versionchanged:: 3.8 New *generation* parameter. + .. versionchanged:: 3.13 + Generation 1 is removed + .. audit-event:: gc.get_objects generation gc.get_objects .. function:: get_stats() @@ -101,19 +118,27 @@ The :mod:`gc` module provides the following functions: Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. - The GC classifies objects into three generations depending on how many - collection sweeps they have survived. New objects are placed in the youngest - generation (generation ``0``). If an object survives a collection it is moved - into the next older generation. Since generation ``2`` is the oldest - generation, objects in that generation remain there after a collection. In - order to decide when to run, the collector keeps track of the number object + The GC classifies objects into two generations depending on whether they have + survived a collection. New objects are placed in the young generation. If an + object survives a collection it is moved into the old generation. + + In order to decide when to run, the collector keeps track of the number of object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds *threshold0*, collection - starts. Initially only generation ``0`` is examined. If generation ``0`` has - been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. - With the third generation, things are a bit more complicated, - see `Collecting the oldest generation `_ for more information. + starts. For each collection, all the objects in the young generation and some + fraction of the old generation is collected. + + The fraction of the old generation that is collected is **inversely** proportional + to *threshold1*. The larger *threshold1* is, the slower objects in the old generation + are collected. + For the default value of 10, 1% of the old generation is scanned during each collection. + + *threshold2* is ignored. + + See `Garbage collector design `_ for more information. + + .. versionchanged:: 3.13 + *threshold2* is ignored .. function:: get_count() diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index ada68b240143e8..d43d3250732306 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -1,5 +1,5 @@ -:mod:`getopt` --- C-style parser for command line options -========================================================= +:mod:`!getopt` --- C-style parser for command line options +========================================================== .. module:: getopt :synopsis: Portable parser for command line options; support both short and diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 54c84d45a59856..3b5296f9ec66fa 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -1,5 +1,5 @@ -:mod:`getpass` --- Portable password input -========================================== +:mod:`!getpass` --- Portable password input +=========================================== .. module:: getpass :synopsis: Portable reading of passwords and retrieval of the userid. @@ -49,7 +49,7 @@ The :mod:`getpass` module provides two functions: systems which support the :mod:`pwd` module, otherwise, an :exc:`OSError` is raised. - In general, this function should be preferred over :func:`os.getlogin()`. + In general, this function should be preferred over :func:`os.getlogin`. .. versionchanged:: 3.13 Previously, various exceptions beyond just :exc:`OSError` were raised. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 41beac3e0c7396..d0de83907eb297 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -1,5 +1,5 @@ -:mod:`gettext` --- Multilingual internationalization services -============================================================= +:mod:`!gettext` --- Multilingual internationalization services +============================================================== .. module:: gettext :synopsis: Multilingual internationalization services. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 19a0bbba8966ba..684466d354aef8 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -1,5 +1,5 @@ -:mod:`glob` --- Unix style pathname pattern expansion -===================================================== +:mod:`!glob` --- Unix style pathname pattern expansion +====================================================== .. module:: glob :synopsis: Unix shell style pathname pattern expansion. @@ -75,6 +75,10 @@ The :mod:`glob` module defines the following functions: Using the "``**``" pattern in large directory trees may consume an inordinate amount of time. + .. note:: + This function may return duplicate path names if *pathname* + contains multiple "``**``" patterns and *recursive* is true. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -94,6 +98,10 @@ The :mod:`glob` module defines the following functions: .. audit-event:: glob.glob pathname,recursive glob.iglob .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.iglob + .. note:: + This function may return duplicate path names if *pathname* + contains multiple "``**``" patterns and *recursive* is true. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -136,8 +144,7 @@ The :mod:`glob` module defines the following functions: separators, and ``*`` pattern segments match precisely one path segment. If *recursive* is true, the pattern segment "``**``" will match any number - of path segments. If "``**``" occurs in any position other than a full - pattern segment, :exc:`ValueError` is raised. + of path segments. If *include_hidden* is true, wildcards can match path segments that start with a dot (``.``). diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 5414d6370b78ce..a0b16576fad219 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -1,5 +1,5 @@ -:mod:`graphlib` --- Functionality to operate with graph-like structures -========================================================================= +:mod:`!graphlib` --- Functionality to operate with graph-like structures +======================================================================== .. module:: graphlib :synopsis: Functionality to operate with graph-like structures diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index 274a353103b488..d1c7f22a209780 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -1,5 +1,5 @@ -:mod:`grp` --- The group database -================================= +:mod:`!grp` --- The group database +================================== .. module:: grp :platform: Unix @@ -10,7 +10,7 @@ This module provides access to the Unix group database. It is available on all Unix versions. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI, not Android, not iOS. Group database entries are reported as a tuple-like object, whose attributes correspond to the members of the ``group`` structure (Attribute field below, see diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 79be215a766045..f24e73517e5767 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -1,5 +1,5 @@ -:mod:`gzip` --- Support for :program:`gzip` files -================================================= +:mod:`!gzip` --- Support for :program:`gzip` files +================================================== .. module:: gzip :synopsis: Interfaces for gzip compression and decompression using file objects. @@ -100,10 +100,12 @@ The module defines the following items: compression, and ``9`` is slowest and produces the most compression. ``0`` is no compression. The default is ``9``. - The *mtime* argument is an optional numeric timestamp to be written to - the last modification time field in the stream when compressing. It - should only be provided in compression mode. If omitted or ``None``, the - current time is used. See the :attr:`mtime` attribute for more details. + The optional *mtime* argument is the timestamp requested by gzip. The time + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. + If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 + to generate a compressed stream that does not depend on creation time. + + See below for the :attr:`mtime` attribute that is set when decompressing. Calling a :class:`GzipFile` object's :meth:`!close` method does not close *fileobj*, since you might wish to append more material after the compressed @@ -131,17 +133,19 @@ The module defines the following items: .. versionadded:: 3.2 - .. attribute:: mtime + .. attribute:: mode - When decompressing, the value of the last modification time field in - the most recently read header may be read from this attribute, as an - integer. The initial value before reading any headers is ``None``. + ``'rb'`` for reading and ``'wb'`` for writing. - All :program:`gzip` compressed streams are required to contain this - timestamp field. Some programs, such as :program:`gunzip`\ , make use - of the timestamp. The format is the same as the return value of - :func:`time.time` and the :attr:`~os.stat_result.st_mtime` attribute of - the object returned by :func:`os.stat`. + .. versionchanged:: 3.13 + In previous versions it was an integer ``1`` or ``2``. + + .. attribute:: mtime + + When decompressing, this attribute is set to the last timestamp in the most + recently read header. It is an integer, holding the number of seconds + since the Unix epoch (00:00:00 UTC, January 1, 1970). + The initial value before reading any headers is ``None``. .. attribute:: name @@ -171,22 +175,21 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: 3.12 - Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` - attribute instead. - .. deprecated:: 3.9 Opening :class:`GzipFile` for writing without specifying the *mode* argument is deprecated. + .. versionchanged:: 3.12 + Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` + attribute instead. + -.. function:: compress(data, compresslevel=9, *, mtime=None) +.. function:: compress(data, compresslevel=9, *, mtime=0) Compress the *data*, returning a :class:`bytes` object containing the compressed data. *compresslevel* and *mtime* have the same meaning as in - the :class:`GzipFile` constructor above. When *mtime* is set to ``0``, this - function is equivalent to :func:`zlib.compress` with *wbits* set to ``31``. - The zlib function is faster. + the :class:`GzipFile` constructor above, + but *mtime* defaults to 0 for reproducible output. .. versionadded:: 3.2 .. versionchanged:: 3.8 @@ -194,7 +197,17 @@ The module defines the following items: .. versionchanged:: 3.11 Speed is improved by compressing all data at once instead of in a streamed fashion. Calls with *mtime* set to ``0`` are delegated to - :func:`zlib.compress` for better speed. + :func:`zlib.compress` for better speed. In this situation the + output may contain a gzip header "OS" byte value other than 255 + "unknown" as supplied by the underlying zlib implementation. + + .. versionchanged:: 3.13 + The gzip header OS byte is guaranteed to be set to 255 when this function + is used as was the case in 3.10 and earlier. + .. versionchanged:: 3.14 + The *mtime* parameter now defaults to 0 for reproducible output. + For the previous behaviour of using the current time, + pass ``None`` to *mtime*. .. function:: decompress(data) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 761dd84edee299..dffb167c74771f 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -1,5 +1,5 @@ -:mod:`hashlib` --- Secure hashes and message digests -==================================================== +:mod:`!hashlib` --- Secure hashes and message digests +===================================================== .. module:: hashlib :synopsis: Secure hash and message digest algorithms. @@ -77,8 +77,6 @@ accessible by name via :func:`new`. See :data:`algorithms_available`. SHA3 (Keccak) and SHAKE constructors :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256` were added. - -.. versionadded:: 3.6 :func:`blake2b` and :func:`blake2s` were added. .. _hashlib-usedforsecurity: @@ -121,7 +119,7 @@ More condensed: Constructors ------------ -.. function:: new(name[, data], \*, usedforsecurity=True) +.. function:: new(name[, data], *, usedforsecurity=True) Is a generic constructor that takes the string *name* of the desired algorithm as its first parameter. It also exists to allow access to the @@ -330,7 +328,7 @@ include a `salt `_. your application, read *Appendix A.2.2* of NIST-SP-800-132_. The answers on the `stackexchange pbkdf2 iterations question`_ explain in detail. - *dklen* is the length of the derived key. If *dklen* is ``None`` then the + *dklen* is the length of the derived key in bytes. If *dklen* is ``None`` then the digest size of the hash algorithm *hash_name* is used, e.g. 64 for SHA-512. >>> from hashlib import pbkdf2_hmac @@ -359,7 +357,7 @@ include a `salt `_. *n* is the CPU/Memory cost factor, *r* the block size, *p* parallelization factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MiB). - *dklen* is the length of the derived key. + *dklen* is the length of the derived key in bytes. .. versionadded:: 3.6 @@ -657,7 +655,7 @@ on the hash function used in digital signatures. by the signer. (`NIST SP-800-106 "Randomized Hashing for Digital Signatures" - `_) + `_) In BLAKE2 the salt is processed as a one-time input to the hash function during initialization, rather than as an input to each compression function. @@ -811,8 +809,8 @@ Domain Dedication 1.0 Universal: .. _NIST-SP-800-132: https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-132.pdf .. _stackexchange pbkdf2 iterations question: https://security.stackexchange.com/questions/3959/recommended-of-iterations-when-using-pbkdf2-sha256/ .. _Attacks on cryptographic hash algorithms: https://en.wikipedia.org/wiki/Cryptographic_hash_function#Attacks_on_cryptographic_hash_algorithms -.. _the FIPS 180-4 standard: https://csrc.nist.gov/publications/detail/fips/180/4/final -.. _the FIPS 202 standard: https://csrc.nist.gov/publications/detail/fips/202/final +.. _the FIPS 180-4 standard: https://csrc.nist.gov/pubs/fips/180-4/upd1/final +.. _the FIPS 202 standard: https://csrc.nist.gov/pubs/fips/202/final .. _HACL\* project: https://github.com/hacl-star/hacl-star @@ -829,7 +827,7 @@ Domain Dedication 1.0 Universal: https://nvlpubs.nist.gov/nistpubs/fips/nist.fips.180-4.pdf The FIPS 180-4 publication on Secure Hash Algorithms. - https://csrc.nist.gov/publications/detail/fips/202/final + https://csrc.nist.gov/pubs/fips/202/final The FIPS 202 publication on the SHA-3 Standard. https://www.blake2.net/ diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index ddbada13bddf5b..d3c4b920ba500a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -1,5 +1,5 @@ -:mod:`heapq` --- Heap queue algorithm -===================================== +:mod:`!heapq` --- Heap queue algorithm +====================================== .. module:: heapq :synopsis: Heap queue algorithm (a.k.a. priority queue). @@ -17,7 +17,9 @@ This module provides an implementation of the heap queue algorithm, also known as the priority queue algorithm. Heaps are binary trees for which every parent node has a value less than or -equal to any of its children. This implementation uses arrays for which +equal to any of its children. We refer to this condition as the heap invariant. + +This implementation uses arrays for which ``heap[k] <= heap[2*k+1]`` and ``heap[k] <= heap[2*k+2]`` for all *k*, counting elements from zero. For the sake of comparison, non-existing elements are considered to be infinite. The interesting property of a heap is that its @@ -319,4 +321,3 @@ applications, and I think it is good to keep a 'heap' module around. :-) backwards, and this was also used to avoid the rewinding time. Believe me, real good tape sorts were quite spectacular to watch! From all times, sorting has always been a Great Art! :-) - diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index 43012e03c580e8..d6692033b2d4c3 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -1,5 +1,5 @@ -:mod:`hmac` --- Keyed-Hashing for Message Authentication -======================================================== +:mod:`!hmac` --- Keyed-Hashing for Message Authentication +========================================================= .. module:: hmac :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index 10529561a92cd0..add18e4c87d220 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -1,5 +1,5 @@ -:mod:`html.entities` --- Definitions of HTML general entities -============================================================= +:mod:`!html.entities` --- Definitions of HTML general entities +============================================================== .. module:: html.entities :synopsis: Definitions of HTML general entities. diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index d35090111e0822..6d433b5a04fc4a 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -1,5 +1,5 @@ -:mod:`html.parser` --- Simple HTML and XHTML parser -=================================================== +:mod:`!html.parser` --- Simple HTML and XHTML parser +==================================================== .. module:: html.parser :synopsis: A simple parser that can handle HTML and XHTML. diff --git a/Doc/library/html.rst b/Doc/library/html.rst index c2b01e14ea7555..9aa39ba9a42b0f 100644 --- a/Doc/library/html.rst +++ b/Doc/library/html.rst @@ -1,5 +1,5 @@ -:mod:`html` --- HyperText Markup Language support -================================================= +:mod:`!html` --- HyperText Markup Language support +================================================== .. module:: html :synopsis: Helpers for manipulating HTML. diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 7e4502064f22a1..2835c8d0eb711e 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -1,5 +1,5 @@ -:mod:`http.client` --- HTTP protocol client -=========================================== +:mod:`!http.client` --- HTTP protocol client +============================================ .. module:: http.client :synopsis: HTTP and HTTPS protocol client (requires sockets). diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 12a6d768437ea5..23ddecf873876d 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -1,5 +1,5 @@ -:mod:`http.cookiejar` --- Cookie handling for HTTP clients -========================================================== +:mod:`!http.cookiejar` --- Cookie handling for HTTP clients +=========================================================== .. module:: http.cookiejar :synopsis: Classes for automatic handling of HTTP cookies. @@ -137,7 +137,7 @@ The following classes are provided: The Netscape protocol with the bugs fixed. Uses :mailheader:`Set-Cookie2` in place of :mailheader:`Set-Cookie`. Not widely used. - http://kristol.org/cookie/errata.html + https://kristol.org/cookie/errata.html Unfinished errata to :rfc:`2965`. :rfc:`2964` - Use of HTTP State Management @@ -649,6 +649,11 @@ internal consistency, so you should know what you're doing if you do that. :const:`None`. +.. attribute:: Cookie.domain + + Cookie domain (a string). + + .. attribute:: Cookie.path Cookie path (a string, eg. ``'/acme/rocket_launchers'``). diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index e91972fe621a48..4ce2e3c4f4cb42 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -1,5 +1,5 @@ -:mod:`http.cookies` --- HTTP state management -============================================= +:mod:`!http.cookies` --- HTTP state management +============================================== .. module:: http.cookies :synopsis: Support for HTTP state management (cookies). diff --git a/Doc/library/http.rst b/Doc/library/http.rst index 5e1912716e5319..ce3fb9f8120502 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -1,5 +1,5 @@ -:mod:`http` --- HTTP modules -============================ +:mod:`!http` --- HTTP modules +============================= .. module:: http :synopsis: HTTP status codes and messages @@ -59,63 +59,63 @@ available in :class:`http.HTTPStatus` are: ======= =================================== ================================================================== Code Enum Name Details ======= =================================== ================================================================== -``100`` ``CONTINUE`` HTTP/1.1 :rfc:`7231`, Section 6.2.1 -``101`` ``SWITCHING_PROTOCOLS`` HTTP/1.1 :rfc:`7231`, Section 6.2.2 +``100`` ``CONTINUE`` HTTP Semantics :rfc:`9110`, Section 15.2.1 +``101`` ``SWITCHING_PROTOCOLS`` HTTP Semantics :rfc:`9110`, Section 15.2.2 ``102`` ``PROCESSING`` WebDAV :rfc:`2518`, Section 10.1 ``103`` ``EARLY_HINTS`` An HTTP Status Code for Indicating Hints :rfc:`8297` -``200`` ``OK`` HTTP/1.1 :rfc:`7231`, Section 6.3.1 -``201`` ``CREATED`` HTTP/1.1 :rfc:`7231`, Section 6.3.2 -``202`` ``ACCEPTED`` HTTP/1.1 :rfc:`7231`, Section 6.3.3 -``203`` ``NON_AUTHORITATIVE_INFORMATION`` HTTP/1.1 :rfc:`7231`, Section 6.3.4 -``204`` ``NO_CONTENT`` HTTP/1.1 :rfc:`7231`, Section 6.3.5 -``205`` ``RESET_CONTENT`` HTTP/1.1 :rfc:`7231`, Section 6.3.6 -``206`` ``PARTIAL_CONTENT`` HTTP/1.1 :rfc:`7233`, Section 4.1 +``200`` ``OK`` HTTP Semantics :rfc:`9110`, Section 15.3.1 +``201`` ``CREATED`` HTTP Semantics :rfc:`9110`, Section 15.3.2 +``202`` ``ACCEPTED`` HTTP Semantics :rfc:`9110`, Section 15.3.3 +``203`` ``NON_AUTHORITATIVE_INFORMATION`` HTTP Semantics :rfc:`9110`, Section 15.3.4 +``204`` ``NO_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.3.5 +``205`` ``RESET_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.3.6 +``206`` ``PARTIAL_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.3.7 ``207`` ``MULTI_STATUS`` WebDAV :rfc:`4918`, Section 11.1 ``208`` ``ALREADY_REPORTED`` WebDAV Binding Extensions :rfc:`5842`, Section 7.1 (Experimental) ``226`` ``IM_USED`` Delta Encoding in HTTP :rfc:`3229`, Section 10.4.1 -``300`` ``MULTIPLE_CHOICES`` HTTP/1.1 :rfc:`7231`, Section 6.4.1 -``301`` ``MOVED_PERMANENTLY`` HTTP/1.1 :rfc:`7231`, Section 6.4.2 -``302`` ``FOUND`` HTTP/1.1 :rfc:`7231`, Section 6.4.3 -``303`` ``SEE_OTHER`` HTTP/1.1 :rfc:`7231`, Section 6.4.4 -``304`` ``NOT_MODIFIED`` HTTP/1.1 :rfc:`7232`, Section 4.1 -``305`` ``USE_PROXY`` HTTP/1.1 :rfc:`7231`, Section 6.4.5 -``307`` ``TEMPORARY_REDIRECT`` HTTP/1.1 :rfc:`7231`, Section 6.4.7 -``308`` ``PERMANENT_REDIRECT`` Permanent Redirect :rfc:`7238`, Section 3 (Experimental) -``400`` ``BAD_REQUEST`` HTTP/1.1 :rfc:`7231`, Section 6.5.1 -``401`` ``UNAUTHORIZED`` HTTP/1.1 Authentication :rfc:`7235`, Section 3.1 -``402`` ``PAYMENT_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.2 -``403`` ``FORBIDDEN`` HTTP/1.1 :rfc:`7231`, Section 6.5.3 -``404`` ``NOT_FOUND`` HTTP/1.1 :rfc:`7231`, Section 6.5.4 -``405`` ``METHOD_NOT_ALLOWED`` HTTP/1.1 :rfc:`7231`, Section 6.5.5 -``406`` ``NOT_ACCEPTABLE`` HTTP/1.1 :rfc:`7231`, Section 6.5.6 -``407`` ``PROXY_AUTHENTICATION_REQUIRED`` HTTP/1.1 Authentication :rfc:`7235`, Section 3.2 -``408`` ``REQUEST_TIMEOUT`` HTTP/1.1 :rfc:`7231`, Section 6.5.7 -``409`` ``CONFLICT`` HTTP/1.1 :rfc:`7231`, Section 6.5.8 -``410`` ``GONE`` HTTP/1.1 :rfc:`7231`, Section 6.5.9 -``411`` ``LENGTH_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.10 -``412`` ``PRECONDITION_FAILED`` HTTP/1.1 :rfc:`7232`, Section 4.2 -``413`` ``REQUEST_ENTITY_TOO_LARGE`` HTTP/1.1 :rfc:`7231`, Section 6.5.11 -``414`` ``REQUEST_URI_TOO_LONG`` HTTP/1.1 :rfc:`7231`, Section 6.5.12 -``415`` ``UNSUPPORTED_MEDIA_TYPE`` HTTP/1.1 :rfc:`7231`, Section 6.5.13 -``416`` ``REQUESTED_RANGE_NOT_SATISFIABLE`` HTTP/1.1 Range Requests :rfc:`7233`, Section 4.4 -``417`` ``EXPECTATION_FAILED`` HTTP/1.1 :rfc:`7231`, Section 6.5.14 +``300`` ``MULTIPLE_CHOICES`` HTTP Semantics :rfc:`9110`, Section 15.4.1 +``301`` ``MOVED_PERMANENTLY`` HTTP Semantics :rfc:`9110`, Section 15.4.2 +``302`` ``FOUND`` HTTP Semantics :rfc:`9110`, Section 15.4.3 +``303`` ``SEE_OTHER`` HTTP Semantics :rfc:`9110`, Section 15.4.4 +``304`` ``NOT_MODIFIED`` HTTP Semantics :rfc:`9110`, Section 15.4.5 +``305`` ``USE_PROXY`` HTTP Semantics :rfc:`9110`, Section 15.4.6 +``307`` ``TEMPORARY_REDIRECT`` HTTP Semantics :rfc:`9110`, Section 15.4.8 +``308`` ``PERMANENT_REDIRECT`` HTTP Semantics :rfc:`9110`, Section 15.4.9 +``400`` ``BAD_REQUEST`` HTTP Semantics :rfc:`9110`, Section 15.5.1 +``401`` ``UNAUTHORIZED`` HTTP Semantics :rfc:`9110`, Section 15.5.2 +``402`` ``PAYMENT_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.3 +``403`` ``FORBIDDEN`` HTTP Semantics :rfc:`9110`, Section 15.5.4 +``404`` ``NOT_FOUND`` HTTP Semantics :rfc:`9110`, Section 15.5.5 +``405`` ``METHOD_NOT_ALLOWED`` HTTP Semantics :rfc:`9110`, Section 15.5.6 +``406`` ``NOT_ACCEPTABLE`` HTTP Semantics :rfc:`9110`, Section 15.5.7 +``407`` ``PROXY_AUTHENTICATION_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.8 +``408`` ``REQUEST_TIMEOUT`` HTTP Semantics :rfc:`9110`, Section 15.5.9 +``409`` ``CONFLICT`` HTTP Semantics :rfc:`9110`, Section 15.5.10 +``410`` ``GONE`` HTTP Semantics :rfc:`9110`, Section 15.5.11 +``411`` ``LENGTH_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.12 +``412`` ``PRECONDITION_FAILED`` HTTP Semantics :rfc:`9110`, Section 15.5.13 +``413`` ``CONTENT_TOO_LARGE`` HTTP Semantics :rfc:`9110`, Section 15.5.14 +``414`` ``URI_TOO_LONG`` HTTP Semantics :rfc:`9110`, Section 15.5.15 +``415`` ``UNSUPPORTED_MEDIA_TYPE`` HTTP Semantics :rfc:`9110`, Section 15.5.16 +``416`` ``RANGE_NOT_SATISFIABLE`` HTTP Semantics :rfc:`9110`, Section 15.5.17 +``417`` ``EXPECTATION_FAILED`` HTTP Semantics :rfc:`9110`, Section 15.5.18 ``418`` ``IM_A_TEAPOT`` HTCPCP/1.0 :rfc:`2324`, Section 2.3.2 -``421`` ``MISDIRECTED_REQUEST`` HTTP/2 :rfc:`7540`, Section 9.1.2 -``422`` ``UNPROCESSABLE_ENTITY`` WebDAV :rfc:`4918`, Section 11.2 +``421`` ``MISDIRECTED_REQUEST`` HTTP Semantics :rfc:`9110`, Section 15.5.20 +``422`` ``UNPROCESSABLE_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.5.21 ``423`` ``LOCKED`` WebDAV :rfc:`4918`, Section 11.3 ``424`` ``FAILED_DEPENDENCY`` WebDAV :rfc:`4918`, Section 11.4 ``425`` ``TOO_EARLY`` Using Early Data in HTTP :rfc:`8470` -``426`` ``UPGRADE_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.15 +``426`` ``UPGRADE_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.22 ``428`` ``PRECONDITION_REQUIRED`` Additional HTTP Status Codes :rfc:`6585` ``429`` ``TOO_MANY_REQUESTS`` Additional HTTP Status Codes :rfc:`6585` ``431`` ``REQUEST_HEADER_FIELDS_TOO_LARGE`` Additional HTTP Status Codes :rfc:`6585` ``451`` ``UNAVAILABLE_FOR_LEGAL_REASONS`` An HTTP Status Code to Report Legal Obstacles :rfc:`7725` -``500`` ``INTERNAL_SERVER_ERROR`` HTTP/1.1 :rfc:`7231`, Section 6.6.1 -``501`` ``NOT_IMPLEMENTED`` HTTP/1.1 :rfc:`7231`, Section 6.6.2 -``502`` ``BAD_GATEWAY`` HTTP/1.1 :rfc:`7231`, Section 6.6.3 -``503`` ``SERVICE_UNAVAILABLE`` HTTP/1.1 :rfc:`7231`, Section 6.6.4 -``504`` ``GATEWAY_TIMEOUT`` HTTP/1.1 :rfc:`7231`, Section 6.6.5 -``505`` ``HTTP_VERSION_NOT_SUPPORTED`` HTTP/1.1 :rfc:`7231`, Section 6.6.6 +``500`` ``INTERNAL_SERVER_ERROR`` HTTP Semantics :rfc:`9110`, Section 15.6.1 +``501`` ``NOT_IMPLEMENTED`` HTTP Semantics :rfc:`9110`, Section 15.6.2 +``502`` ``BAD_GATEWAY`` HTTP Semantics :rfc:`9110`, Section 15.6.3 +``503`` ``SERVICE_UNAVAILABLE`` HTTP Semantics :rfc:`9110`, Section 15.6.4 +``504`` ``GATEWAY_TIMEOUT`` HTTP Semantics :rfc:`9110`, Section 15.6.5 +``505`` ``HTTP_VERSION_NOT_SUPPORTED`` HTTP Semantics :rfc:`9110`, Section 15.6.6 ``506`` ``VARIANT_ALSO_NEGOTIATES`` Transparent Content Negotiation in HTTP :rfc:`2295`, Section 8.1 (Experimental) ``507`` ``INSUFFICIENT_STORAGE`` WebDAV :rfc:`4918`, Section 11.5 ``508`` ``LOOP_DETECTED`` WebDAV Binding Extensions :rfc:`5842`, Section 7.2 (Experimental) @@ -137,6 +137,10 @@ equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as .. versionadded:: 3.9 Added ``103 EARLY_HINTS``, ``418 IM_A_TEAPOT`` and ``425 TOO_EARLY`` status codes. +.. versionchanged:: 3.13 + Implemented RFC9110 naming for status constants. Old constant names are preserved for + backwards compatibility. + HTTP status category -------------------- @@ -144,15 +148,15 @@ HTTP status category The enum values have several properties to indicate the HTTP status category: -==================== ======================== =============================== +==================== ======================== ====================================== Property Indicates that Details -==================== ======================== =============================== -``is_informational`` ``100 <= status <= 199`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_success`` ``200 <= status <= 299`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_redirection`` ``300 <= status <= 399`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_client_error`` ``400 <= status <= 499`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_server_error`` ``500 <= status <= 599`` HTTP/1.1 :rfc:`7231`, Section 6 -==================== ======================== =============================== +==================== ======================== ====================================== +``is_informational`` ``100 <= status <= 199`` HTTP Semantics :rfc:`9110`, Section 15 +``is_success`` ``200 <= status <= 299`` HTTP Semantics :rfc:`9110`, Section 15 +``is_redirection`` ``300 <= status <= 399`` HTTP Semantics :rfc:`9110`, Section 15 +``is_client_error`` ``400 <= status <= 499`` HTTP Semantics :rfc:`9110`, Section 15 +``is_server_error`` ``500 <= status <= 599`` HTTP Semantics :rfc:`9110`, Section 15 +==================== ======================== ====================================== Usage:: @@ -203,13 +207,13 @@ available in :class:`http.HTTPMethod` are: =========== =================================== ================================================================== Method Enum Name Details =========== =================================== ================================================================== -``GET`` ``GET`` HTTP/1.1 :rfc:`7231`, Section 4.3.1 -``HEAD`` ``HEAD`` HTTP/1.1 :rfc:`7231`, Section 4.3.2 -``POST`` ``POST`` HTTP/1.1 :rfc:`7231`, Section 4.3.3 -``PUT`` ``PUT`` HTTP/1.1 :rfc:`7231`, Section 4.3.4 -``DELETE`` ``DELETE`` HTTP/1.1 :rfc:`7231`, Section 4.3.5 -``CONNECT`` ``CONNECT`` HTTP/1.1 :rfc:`7231`, Section 4.3.6 -``OPTIONS`` ``OPTIONS`` HTTP/1.1 :rfc:`7231`, Section 4.3.7 -``TRACE`` ``TRACE`` HTTP/1.1 :rfc:`7231`, Section 4.3.8 +``GET`` ``GET`` HTTP Semantics :rfc:`9110`, Section 9.3.1 +``HEAD`` ``HEAD`` HTTP Semantics :rfc:`9110`, Section 9.3.2 +``POST`` ``POST`` HTTP Semantics :rfc:`9110`, Section 9.3.3 +``PUT`` ``PUT`` HTTP Semantics :rfc:`9110`, Section 9.3.4 +``DELETE`` ``DELETE`` HTTP Semantics :rfc:`9110`, Section 9.3.5 +``CONNECT`` ``CONNECT`` HTTP Semantics :rfc:`9110`, Section 9.3.6 +``OPTIONS`` ``OPTIONS`` HTTP Semantics :rfc:`9110`, Section 9.3.7 +``TRACE`` ``TRACE`` HTTP Semantics :rfc:`9110`, Section 9.3.8 ``PATCH`` ``PATCH`` HTTP/1.1 :rfc:`5789` =========== =================================== ================================================================== diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index bc59d3d17912fd..1197b575c00455 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -1,5 +1,5 @@ -:mod:`http.server` --- HTTP servers -=================================== +:mod:`!http.server` --- HTTP servers +==================================== .. module:: http.server :synopsis: HTTP server and request handlers. @@ -263,7 +263,7 @@ provides three different variants: Adds a blank line (indicating the end of the HTTP headers in the response) - to the headers buffer and calls :meth:`flush_headers()`. + to the headers buffer and calls :meth:`flush_headers`. .. versionchanged:: 3.2 The buffered headers are written to the output stream. @@ -378,7 +378,7 @@ provides three different variants: If the request was mapped to a file, it is opened. Any :exc:`OSError` exception in opening the requested file is mapped to a ``404``, - ``'File not found'`` error. If there was a ``'If-Modified-Since'`` + ``'File not found'`` error. If there was an ``'If-Modified-Since'`` header in the request, and the file was not modified after this time, a ``304``, ``'Not Modified'`` response is sent. Otherwise, the content type is guessed by calling the :meth:`guess_type` method, which in turn @@ -520,6 +520,12 @@ the ``--cgi`` option:: :mod:`http.server` command line ``--cgi`` support is being removed because :class:`CGIHTTPRequestHandler` is being removed. +.. warning:: + + :class:`CGIHTTPRequestHandler` and the ``--cgi`` command line option + are not intended for use by untrusted clients and may be vulnerable + to exploitation. Always use within a secure environment. + .. _http.server-security: Security Considerations diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 249dc0ea6ba735..59b181aab3e484 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -429,7 +429,7 @@ Several non-character keys move the cursor and possibly delete characters. Deletion does not puts text on the clipboard, but IDLE has an undo list. Wherever this doc discusses keys, 'C' refers to the :kbd:`Control` key on Windows and -Unix and the :kbd:`Command` key on macOS. (And all such dicussions +Unix and the :kbd:`Command` key on macOS. (And all such discussions assume that the keys have not been re-bound to something else.) * Arrow keys move the cursor one character or line. @@ -604,7 +604,7 @@ in an editor window. The editing features described in previous subsections work when entering code interactively. IDLE's Shell window also responds to the following: -* :kbd:`C-c` attemps to interrupt statement execution (but may fail). +* :kbd:`C-c` attempts to interrupt statement execution (but may fail). * :kbd:`C-d` closes Shell if typed at a ``>>>`` prompt. diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index d5c868def3b64f..a2dad58b00b9fa 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -1,5 +1,5 @@ -:mod:`imaplib` --- IMAP4 protocol client -======================================== +:mod:`!imaplib` --- IMAP4 protocol client +========================================= .. module:: imaplib :synopsis: IMAP4 protocol client (requires sockets). @@ -39,7 +39,7 @@ base class: initialized. If *host* is not specified, ``''`` (the local host) is used. If *port* is omitted, the standard IMAP4 port (143) is used. The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. - If timeout is not given or is None, the global default socket timeout is used. + If timeout is not given or is ``None``, the global default socket timeout is used. The :class:`IMAP4` class supports the :keyword:`with` statement. When used like this, the IMAP4 ``LOGOUT`` command is issued automatically when the @@ -97,7 +97,7 @@ There's also a subclass for secure connections: best practices. The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt. If timeout is not given or is None, the global default + connection attempt. If timeout is not given or is ``None``, the global default socket timeout is used. .. versionchanged:: 3.3 @@ -360,7 +360,7 @@ An :class:`IMAP4` instance has the following methods: Opens socket to *port* at *host*. The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. - If timeout is not given or is None, the global default socket timeout + If timeout is not given or is ``None``, the global default socket timeout is used. Also note that if the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to reject creating a non-blocking socket. This method is implicitly called by the :class:`IMAP4` constructor. @@ -622,7 +622,7 @@ retrieves and prints all messages:: import getpass, imaplib - M = imaplib.IMAP4() + M = imaplib.IMAP4(host='example.org') M.login(getpass.getuser(), getpass.getpass()) M.select() typ, data = M.search(None, 'ALL') diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index cc4a0da92da60a..37cd237357aa4b 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -19,14 +19,14 @@ such as its entry points or its top-level names (`Import Package `_\s, modules, if any). Built in part on Python's import system, this library intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with +API`_ and `metadata API`_ of ``pkg_resources``. Along with :mod:`importlib.resources`, this package can eliminate the need to use the older and less efficient ``pkg_resources`` package. ``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as -`pip `_. +:pypi:`pip`. Specifically, it works with distributions with discoverable ``dist-info`` or ``egg-info`` directories, and metadata defined by the `Core metadata specifications `_. @@ -46,7 +46,7 @@ and metadata defined by the `Core metadata specifications `_ you've installed -using ``pip``. We start by creating a virtual environment and installing +using ``pip``. We start by creating a virtual environment and installing something into it: .. code-block:: shell-session @@ -87,7 +87,7 @@ You can get the version string for ``wheel`` by running the following: '0.32.3' You can also get a collection of entry points selectable by properties of the EntryPoint (typically 'group' or 'name'), such as -``console_scripts``, ``distutils.commands`` and others. Each group contains a +``console_scripts``, ``distutils.commands`` and others. Each group contains a collection of :ref:`EntryPoint ` objects. You can get the :ref:`metadata for a distribution `:: @@ -100,6 +100,13 @@ You can also get a :ref:`distribution's version number `, list its :ref:`requirements`. +.. exception:: PackageNotFoundError + + Subclass of :class:`ModuleNotFoundError` raised by several functions in this + module when queried for a distribution package which is not installed in the + current Python environment. + + Functional API ============== @@ -111,31 +118,53 @@ This package provides the following functionality via its public API. Entry points ------------ -The ``entry_points()`` function returns a collection of entry points. -Entry points are represented by ``EntryPoint`` instances; -each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. There are also ``.module``, -``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute. +.. function:: entry_points(**select_params) + + Returns a :class:`EntryPoints` instance describing entry points for the + current environment. Any given keyword parameters are passed to the + :meth:`!select` method for comparison to the attributes of + the individual entry point definitions. + + Note: it is not currently possible to query for entry points based on + their :attr:`!EntryPoint.dist` attribute (as different :class:`!Distribution` + instances do not currently compare equal, even if they have the same attributes) + +.. class:: EntryPoints + + Details of a collection of installed entry points. + + Also provides a ``.groups`` attribute that reports all identifed entry + point groups, and a ``.names`` attribute that reports all identified entry + point names. + +.. class:: EntryPoint + + Details of an installed entry point. + + Each :class:`!EntryPoint` instance has ``.name``, ``.group``, and ``.value`` + attributes and a ``.load()`` method to resolve the value. There are also + ``.module``, ``.attr``, and ``.extras`` attributes for getting the + components of the ``.value`` attribute, and ``.dist`` for obtaining + information regarding the distribution package that provides the entry point. Query all entry points:: >>> eps = entry_points() # doctest: +SKIP -The ``entry_points()`` function returns an ``EntryPoints`` object, -a collection of all ``EntryPoint`` objects with ``names`` and ``groups`` +The :func:`!entry_points` function returns a :class:`!EntryPoints` object, +a collection of all :class:`!EntryPoint` objects with ``names`` and ``groups`` attributes for convenience:: >>> sorted(eps.groups) # doctest: +SKIP ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] -``EntryPoints`` has a ``select`` method to select entry points +:class:`!EntryPoints` has a :meth:`!select` method to select entry points matching specific properties. Select entry points in the ``console_scripts`` group:: >>> scripts = eps.select(group='console_scripts') # doctest: +SKIP -Equivalently, since ``entry_points`` passes keyword arguments +Equivalently, since :func:`!entry_points` passes keyword arguments through to select:: >>> scripts = entry_points(group='console_scripts') # doctest: +SKIP @@ -167,7 +196,7 @@ Inspect the resolved entry point:: The ``group`` and ``name`` are arbitrary values defined by the package author and usually a client will wish to resolve all entry points for a particular -group. Read `the setuptools docs +group. Read `the setuptools docs `_ for more information on entry points, their definition, and usage. @@ -177,7 +206,7 @@ for more information on entry points, their definition, and usage. no parameters and always returned a dictionary of entry points, keyed by group. With ``importlib_metadata`` 5.0 and Python 3.12, ``entry_points`` always returns an ``EntryPoints`` object. See - `backports.entry_points_selectable `_ + :pypi:`backports.entry_points_selectable` for compatibility options. .. versionchanged:: 3.13 @@ -189,37 +218,46 @@ for more information on entry points, their definition, and usage. Distribution metadata --------------------- -Every `Distribution Package `_ includes some metadata, -which you can extract using the -``metadata()`` function:: +.. function:: metadata(distribution_name) + + Return the distribution metadata corresponding to the named + distribution package as a :class:`PackageMetadata` instance. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + +.. class:: PackageMetadata + + A concrete implementation of the + `PackageMetadata protocol `_. + + In addition to providing the defined protocol methods and attributes, subscripting + the instance is equivalent to calling the :meth:`!get` method. + +Every `Distribution Package `_ +includes some metadata, which you can extract using the :func:`!metadata` function:: >>> wheel_metadata = metadata('wheel') # doctest: +SKIP -The keys of the returned data structure, a ``PackageMetadata``, -name the metadata keywords, and +The keys of the returned data structure name the metadata keywords, and the values are returned unparsed from the distribution metadata:: >>> wheel_metadata['Requires-Python'] # doctest: +SKIP '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' -``PackageMetadata`` also presents a ``json`` attribute that returns +:class:`PackageMetadata` also presents a :attr:`!json` attribute that returns all the metadata in a JSON-compatible form per :PEP:`566`:: >>> wheel_metadata.json['requires_python'] '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' -.. note:: - - The actual type of the object returned by ``metadata()`` is an - implementation detail and should be accessed only through the interface - described by the - `PackageMetadata protocol `_. +The full set of available metadata is not described here. +See the PyPA `Core metadata specification `_ for additional details. .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. -.. versionadded:: 3.10 The ``json`` attribute was added. @@ -228,7 +266,15 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: Distribution versions --------------------- -The ``version()`` function is the quickest way to get a +.. function:: version(distribution_name) + + Return the installed distribution package version for the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + +The :func:`!version` function is the quickest way to get a `Distribution Package `_'s version number, as a string:: @@ -241,12 +287,28 @@ number, as a string:: Distribution files ------------------ -You can also get the full set of files contained within a distribution. The -``files()`` function takes a `Distribution Package `_ name -and returns all of the -files installed by this distribution. Each file object returned is a -``PackagePath``, a :class:`pathlib.PurePath` derived object with additional ``dist``, -``size``, and ``hash`` properties as indicated by the metadata. For example:: +.. function:: files(distribution_name) + + Return the full set of files contained within the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + + Returns :const:`None` if the distribution is found but the installation + database records reporting the files associated with the distribuion package + are missing. + +.. class:: PackagePath + + A :class:`pathlib.PurePath` derived object with additional ``dist``, + ``size``, and ``hash`` properties corresponding to the distribution + package's installation metadata for that file. + +The :func:`!files` function takes a +`Distribution Package `_ +name and returns all of the files installed by this distribution. Each file is reported +as a :class:`PackagePath` instance. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP >>> util # doctest: +SKIP @@ -269,16 +331,16 @@ Once you have the file, you can also read its contents:: return s.encode('utf-8') return s -You can also use the ``locate`` method to get a the absolute path to the -file:: +You can also use the :meth:`!locate` method to get the absolute +path to the file:: >>> util.locate() # doctest: +SKIP PosixPath('/home/gustav/example/lib/site-packages/wheel/util.py') In the case where the metadata file listing files -(RECORD or SOURCES.txt) is missing, ``files()`` will -return ``None``. The caller may wish to wrap calls to -``files()`` in `always_iterable +(``RECORD`` or ``SOURCES.txt``) is missing, :func:`!files` will +return :const:`None`. The caller may wish to wrap calls to +:func:`!files` in `always_iterable `_ or otherwise guard against this condition if the target distribution is not known to have the metadata present. @@ -288,8 +350,16 @@ distribution is not known to have the metadata present. Distribution requirements ------------------------- +.. function:: requires(distribution_name) + + Return the declared dependency specifiers for the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + To get the full set of requirements for a `Distribution Package `_, -use the ``requires()`` +use the :func:`!requires` function:: >>> requires('wheel') # doctest: +SKIP @@ -302,6 +372,16 @@ function:: Mapping import to distribution packages --------------------------------------- +.. function:: packages_distributions() + + Return a mapping from the top level module and import package + names found via :attr:`sys.meta_path` to the names of the distribution + packages (if any) that provide the corresponding files. + + To allow for namespace packages (which may have members provided by + multiple distribution packages), each top level import name maps to a + list of distribution names rather than mapping directly to a single name. + A convenience method to resolve the `Distribution Package `_ name (or names, in the case of a namespace package) that provide each importable top-level @@ -321,37 +401,56 @@ function is not reliable with such installs. Distributions ============= -While the above API is the most common and convenient usage, you can get all -of that information from the ``Distribution`` class. A ``Distribution`` is an -abstract object that represents the metadata for -a Python `Distribution Package `_. You can -get the ``Distribution`` instance:: +.. function:: distribution(distribution_name) + + Return a :class:`Distribution` instance describing the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + +.. class:: Distribution + + Details of an installed distribution package. + + Note: different :class:`!Distribution` instances do not currently compare + equal, even if they relate to the same installed distribution and + accordingly have the same attributes. + +While the module level API described above is the most common and convenient usage, +you can get all of that information from the :class:`!Distribution` class. +:class:`!Distribution` is an abstract object that represents the metadata for +a Python `Distribution Package `_. +You can get the concreate :class:`!Distribution` subclass instance for an installed +distribution package by calling the :func:`distribution` function:: >>> from importlib.metadata import distribution # doctest: +SKIP >>> dist = distribution('wheel') # doctest: +SKIP + >>> type(dist) # doctest: +SKIP + Thus, an alternative way to get the version number is through the -``Distribution`` instance:: +:class:`!Distribution` instance:: >>> dist.version # doctest: +SKIP '0.32.3' -There are all kinds of additional metadata available on the ``Distribution`` -instance:: +There are all kinds of additional metadata available on :class:`!Distribution` +instances:: >>> dist.metadata['Requires-Python'] # doctest: +SKIP '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' >>> dist.metadata['License'] # doctest: +SKIP 'MIT' -For editable packages, an origin property may present :pep:`610` +For editable packages, an ``origin`` property may present :pep:`610` metadata:: >>> dist.origin.url 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' The full set of available metadata is not described here. -See the `Core metadata specifications `_ for additional details. +See the PyPA `Core metadata specification `_ for additional details. .. versionadded:: 3.13 The ``.origin`` property was added. @@ -367,21 +466,26 @@ This metadata finder search defaults to ``sys.path``, but varies slightly in how - ``importlib.metadata`` will incidentally honor :py:class:`pathlib.Path` objects on ``sys.path`` even though such values will be ignored for imports. -Extending the search algorithm -============================== +Implementing Custom Providers +============================= + +``importlib.metadata`` address two API surfaces, one for *consumers* +and another for *providers*. Most users are consumers, consuming +metadata provided by the packages. There are other use-cases, however, +where users wish to expose metadata through some other mechanism, +such as alongside a custom importer. Such a use case calls for a +*custom provider*. Because `Distribution Package `_ metadata is not available through :data:`sys.path` searches, or package loaders directly, the metadata for a distribution is found through import -system :ref:`finders `. To find a distribution package's metadata, +system :ref:`finders `. To find a distribution package's metadata, ``importlib.metadata`` queries the list of :term:`meta path finders ` on :data:`sys.meta_path`. -By default ``importlib.metadata`` installs a finder for distribution packages -found on the file system. -This finder doesn't actually find any *distributions*, -but it can find their metadata. +The implementation has hooks integrated into the ``PathFinder``, +serving metadata for distribution packages found on the file system. The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the interface expected of finders by Python's import system. @@ -392,21 +496,98 @@ interface expected of finders by Python's import system. method:: @abc.abstractmethod - def find_distributions(context=DistributionFinder.Context()): + def find_distributions(context=DistributionFinder.Context()) -> Iterable[Distribution]: """Return an iterable of all Distribution instances capable of loading the metadata for packages for the indicated ``context``. """ The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` properties indicating the path to search and name to match and may -supply other relevant context. +supply other relevant context sought by the consumer. -What this means in practice is that to support finding distribution package +In practice, to support finding distribution package metadata in locations other than the file system, subclass ``Distribution`` and implement the abstract methods. Then from a custom finder, return instances of this derived ``Distribution`` in the ``find_distributions()`` method. +Example +------- + +Imagine a custom finder that loads Python modules from a database:: + + class DatabaseImporter(importlib.abc.MetaPathFinder): + def __init__(self, db): + self.db = db + + def find_spec(self, fullname, target=None) -> ModuleSpec: + return self.db.spec_from_name(fullname) + + sys.meta_path.append(DatabaseImporter(connect_db(...))) + +That importer now presumably provides importable modules from a +database, but it provides no metadata or entry points. For this +custom importer to provide metadata, it would also need to implement +``DistributionFinder``:: + + from importlib.metadata import DistributionFinder + + class DatabaseImporter(DistributionFinder): + ... + + def find_distributions(self, context=DistributionFinder.Context()): + query = dict(name=context.name) if context.name else {} + for dist_record in self.db.query_distributions(query): + yield DatabaseDistribution(dist_record) + +In this way, ``query_distributions`` would return records for +each distribution served by the database matching the query. For +example, if ``requests-1.0`` is in the database, ``find_distributions`` +would yield a ``DatabaseDistribution`` for ``Context(name='requests')`` +or ``Context(name=None)``. + +For the sake of simplicity, this example ignores ``context.path``\. The +``path`` attribute defaults to ``sys.path`` and is the set of import paths to +be considered in the search. A ``DatabaseImporter`` could potentially function +without any concern for a search path. Assuming the importer does no +partitioning, the "path" would be irrelevant. In order to illustrate the +purpose of ``path``, the example would need to illustrate a more complex +``DatabaseImporter`` whose behavior varied depending on +``sys.path``/``PYTHONPATH``. In that case, the ``find_distributions`` should +honor the ``context.path`` and only yield ``Distribution``\ s pertinent to that +path. + +``DatabaseDistribution``, then, would look something like:: + + class DatabaseDistribution(importlib.metadata.Distribution): + def __init__(self, record): + self.record = record + + def read_text(self, filename): + """ + Read a file like "METADATA" for the current distribution. + """ + if filename == "METADATA": + return f"""Name: {self.record.name} + Version: {self.record.version} + """ + if filename == "entry_points.txt": + return "\n".join( + f"""[{ep.group}]\n{ep.name}={ep.value}""" + for ep in self.record.entry_points) + + def locate_file(self, path): + raise RuntimeError("This distribution has no file system") + +This basic implementation should provide metadata and entry points for +packages served by the ``DatabaseImporter``, assuming that the +``record`` supplies suitable ``.name``, ``.version``, and +``.entry_points`` attributes. + +The ``DatabaseDistribution`` may also provide other metadata files, like +``RECORD`` (required for ``Distribution.files``) or override the +implementation of ``Distribution.files``. See the source for more inspiration. + .. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points .. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index c508b6ba965cc0..54995ddbfbca12 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources.abc` -- Abstract base classes for resources ---------------------------------------------------------------------- +:mod:`!importlib.resources.abc` -- Abstract base classes for resources +---------------------------------------------------------------------- .. module:: importlib.resources.abc :synopsis: Abstract base classes for resources @@ -22,7 +22,7 @@ something like a data file that lives next to the ``__init__.py`` file of the package. The purpose of this class is to help abstract out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored in a e.g. zip file + the package and its data file(s) are stored e.g. in a zip file versus on the file system. For any of methods of this class, a *resource* argument is @@ -103,19 +103,41 @@ .. abstractmethod:: is_dir() - Return True if self is a directory. + Return ``True`` if self is a directory. .. abstractmethod:: is_file() - Return True if self is a file. + Return ``True`` if self is a file. - .. abstractmethod:: joinpath(child) + .. abstractmethod:: joinpath(*pathsegments) - Return Traversable child in self. + Traverse directories according to *pathsegments* and return + the result as :class:`!Traversable`. + + Each *pathsegments* argument may contain multiple names separated by + forward slashes (``/``, ``posixpath.sep`` ). + For example, the following are equivalent:: + + files.joinpath('subdir', 'subsuddir', 'file.txt') + files.joinpath('subdir/subsuddir/file.txt') + + Note that some :class:`!Traversable` implementations + might not be updated to the latest version of the protocol. + For compatibility with such implementations, provide a single argument + without path separators to each call to ``joinpath``. For example:: + + files.joinpath('subdir').joinpath('subsubdir').joinpath('file.txt') + + .. versionchanged:: 3.11 + + ``joinpath`` accepts multiple *pathsegments*, and these segments + may contain forward slashes as path separators. + Previously, only a single *child* argument was accepted. .. abstractmethod:: __truediv__(child) Return Traversable child in self. + Equivalent to ``joinpath(child)``. .. abstractmethod:: open(mode='r', *args, **kwargs) diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index a5adf0b8546dbf..e002198899c8b8 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources` -- Package resource reading, opening and access --------------------------------------------------------------------------- +:mod:`!importlib.resources` -- Package resource reading, opening and access +--------------------------------------------------------------------------- .. module:: importlib.resources :synopsis: Package resource reading, opening, and access @@ -97,3 +97,181 @@ for example, a package and its resources can be imported from a zip file using .. versionchanged:: 3.12 Added support for *traversable* representing a directory. + + +.. _importlib_resources_functional: + +Functional API +^^^^^^^^^^^^^^ + +A set of simplified, backwards-compatible helpers is available. +These allow common operations in a single function call. + +For all the following functions: + +- *anchor* is an :class:`~importlib.resources.Anchor`, + as in :func:`~importlib.resources.files`. + Unlike in ``files``, it may not be omitted. + +- *path_names* are components of a resource's path name, relative to + the anchor. + For example, to get the text of resource named ``info.txt``, use:: + + importlib.resources.read_text(my_module, "info.txt") + + Like :meth:`Traversable.joinpath `, + The individual components should use forward slashes (``/``) + as path separators. + For example, the following are equivalent:: + + importlib.resources.read_binary(my_module, "pics/painting.png") + importlib.resources.read_binary(my_module, "pics", "painting.png") + + For backward compatibility reasons, functions that read text require + an explicit *encoding* argument if multiple *path_names* are given. + For example, to get the text of ``info/chapter1.txt``, use:: + + importlib.resources.read_text(my_module, "info", "chapter1.txt", + encoding='utf-8') + +.. function:: open_binary(anchor, *path_names) + + Open the named resource for binary reading. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function returns a :class:`~typing.BinaryIO` object, + that is, a binary stream open for reading. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).open('rb') + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: open_text(anchor, *path_names, encoding='utf-8', errors='strict') + + Open the named resource for text reading. + By default, the contents are read as strict UTF-8. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + *encoding* and *errors* have the same meaning as in built-in :func:`open`. + + For backward compatibility reasons, the *encoding* argument must be given + explicitly if there are multiple *path_names*. + This limitation is scheduled to be removed in Python 3.15. + + This function returns a :class:`~typing.TextIO` object, + that is, a text stream open for reading. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).open('r', encoding=encoding) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: read_binary(anchor, *path_names) + + Read and return the contents of the named resource as :class:`bytes`. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).read_bytes() + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: read_text(anchor, *path_names, encoding='utf-8', errors='strict') + + Read and return the contents of the named resource as :class:`str`. + By default, the contents are read as strict UTF-8. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + *encoding* and *errors* have the same meaning as in built-in :func:`open`. + + For backward compatibility reasons, the *encoding* argument must be given + explicitly if there are multiple *path_names*. + This limitation is scheduled to be removed in Python 3.15. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).read_text(encoding=encoding) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: path(anchor, *path_names) + + Provides the path to the *resource* as an actual file system path. This + function returns a context manager for use in a :keyword:`with` statement. + The context manager provides a :class:`pathlib.Path` object. + + Exiting the context manager cleans up any temporary files created, e.g. + when the resource needs to be extracted from a zip file. + + For example, the :meth:`~pathlib.Path.stat` method requires + an actual file system path; it can be used like this:: + + with importlib.resources.path(anchor, "resource.txt") as fspath: + result = fspath.stat() + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + as_file(files(anchor).joinpath(*path_names)) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: is_resource(anchor, *path_names) + + Return ``True`` if the named resource exists, otherwise ``False``. + This function does not consider directories to be resources. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).is_file() + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: contents(anchor, *path_names) + + Return an iterable over the named items within the package or path. + The iterable returns names of resources (e.g. files) and non-resources + (e.g. directories) as :class:`str`. + The iterable does not recurse into subdirectories. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + for resource in files(anchor).joinpath(*path_names).iterdir(): + yield resource.name + + .. deprecated:: 3.11 + Prefer ``iterdir()`` as above, which offers more control over the + results and richer functionality. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 2402bc5cd3ee2c..9e088a598a6c08 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -249,7 +249,7 @@ ABC hierarchy:: An abstract method for finding a :term:`spec ` for the specified module. If this is a top-level import, *path* will be ``None``. Otherwise, this is a search for a subpackage or - module and *path* will be the value of :attr:`__path__` from the + module and *path* will be the value of :attr:`~module.__path__` from the parent package. If a spec cannot be found, ``None`` is returned. When passed in, ``target`` is a module object that the finder may use to make a more educated guess about what spec to return. @@ -265,7 +265,7 @@ ABC hierarchy:: when invalidating the caches of all finders on :data:`sys.meta_path`. .. versionchanged:: 3.4 - Returns ``None`` when called instead of ``NotImplemented``. + Returns ``None`` when called instead of :data:`NotImplemented`. .. class:: PathEntryFinder @@ -355,34 +355,12 @@ ABC hierarchy:: (note that some of these attributes can change when a module is reloaded): - - :attr:`__name__` - The module's fully qualified name. - It is ``'__main__'`` for an executed module. - - - :attr:`__file__` - The location the :term:`loader` used to load the module. - For example, for modules loaded from a .py file this is the filename. - It is not set on all modules (e.g. built-in modules). - - - :attr:`__cached__` - The filename of a compiled version of the module's code. - It is not set on all modules (e.g. built-in modules). - - - :attr:`__path__` - The list of locations where the package's submodules will be found. - Most of the time this is a single directory. - The import system passes this attribute to ``__import__()`` and to finders - in the same way as :data:`sys.path` but just for the package. - It is not set on non-package modules so it can be used - as an indicator that the module is a package. - - - :attr:`__package__` - The fully qualified name of the package the module is in (or the - empty string for a top-level module). - If the module is a package then this is the same as :attr:`__name__`. - - - :attr:`__loader__` - The :term:`loader` used to load the module. + - :attr:`module.__name__` + - :attr:`module.__file__` + - :attr:`module.__cached__` *(deprecated)* + - :attr:`module.__path__` + - :attr:`module.__package__` *(deprecated)* + - :attr:`module.__loader__` *(deprecated)* When :meth:`exec_module` is available then backwards-compatible functionality is provided. @@ -418,7 +396,8 @@ ABC hierarchy:: can implement this abstract method to give direct access to the data stored. :exc:`OSError` is to be raised if the *path* cannot be found. The *path* is expected to be constructed using a module's - :attr:`__file__` attribute or an item from a package's :attr:`__path__`. + :attr:`~module.__file__` attribute or an item from a package's + :attr:`~module.__path__`. .. versionchanged:: 3.4 Raises :exc:`OSError` instead of :exc:`NotImplementedError`. @@ -505,9 +484,9 @@ ABC hierarchy:: .. abstractmethod:: get_filename(fullname) - An abstract method that is to return the value of :attr:`__file__` for - the specified module. If no path is available, :exc:`ImportError` is - raised. + An abstract method that is to return the value of + :attr:`~module.__file__` for the specified module. If no path is + available, :exc:`ImportError` is raised. If source code is available, then the method should return the path to the source file, regardless of whether a bytecode was used to load the @@ -657,7 +636,7 @@ ABC hierarchy:: something like a data file that lives next to the ``__init__.py`` file of the package. The purpose of this class is to help abstract out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored in a e.g. zip file + the package and its data file(s) are stored e.g. in a zip file versus on the file system. For any of methods of this class, a *resource* argument is @@ -1166,79 +1145,137 @@ find and load modules. .. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) A specification for a module's import-system-related state. This is - typically exposed as the module's :attr:`__spec__` attribute. In the - descriptions below, the names in parentheses give the corresponding - attribute available directly on the module object, - e.g. ``module.__spec__.origin == module.__file__``. Note, however, that + typically exposed as the module's :attr:`~module.__spec__` attribute. Many + of these attributes are also available directly on a module: for example, + ``module.__spec__.origin == module.__file__``. Note, however, that while the *values* are usually equivalent, they can differ since there is - no synchronization between the two objects. For example, it is possible to update - the module's :attr:`__file__` at runtime and this will not be automatically - reflected in the module's :attr:`__spec__.origin`, and vice versa. + no synchronization between the two objects. For example, it is possible to + update the module's :attr:`~module.__file__` at runtime and this will not be + automatically reflected in the module's + :attr:`__spec__.origin `, and vice versa. .. versionadded:: 3.4 .. attribute:: name - (:attr:`__name__`) - - The module's fully qualified name. - The :term:`finder` should always set this attribute to a non-empty string. + The module's fully qualified name (see :attr:`module.__name__`). + The :term:`finder` should always set this attribute to a non-empty string. .. attribute:: loader - (:attr:`__loader__`) - - The :term:`loader` used to load the module. - The :term:`finder` should always set this attribute. + The :term:`loader` used to load the module (see :attr:`module.__loader__`). + The :term:`finder` should always set this attribute. .. attribute:: origin - (:attr:`__file__`) - - The location the :term:`loader` should use to load the module. - For example, for modules loaded from a .py file this is the filename. - The :term:`finder` should always set this attribute to a meaningful value - for the :term:`loader` to use. In the uncommon case that there is not one - (like for namespace packages), it should be set to ``None``. + The location the :term:`loader` should use to load the module + (see :attr:`module.__file__`). + For example, for modules loaded from a ``.py`` file this is the filename. + The :term:`finder` should always set this attribute to a meaningful value + for the :term:`loader` to use. In the uncommon case that there is not one + (like for namespace packages), it should be set to ``None``. .. attribute:: submodule_search_locations - (:attr:`__path__`) + A (possibly empty) :term:`sequence` of strings enumerating the locations + in which a package's submodules will be found + (see :attr:`module.__path__`). Most of the time there will only be a + single directory in this list. - The list of locations where the package's submodules will be found. - Most of the time this is a single directory. - The :term:`finder` should set this attribute to a list, even an empty one, to indicate - to the import system that the module is a package. It should be set to ``None`` for - non-package modules. It is set automatically later to a special object for - namespace packages. + The :term:`finder` should set this attribute to a sequence, even an empty + one, to indicate + to the import system that the module is a package. It should be set to ``None`` for + non-package modules. It is set automatically later to a special object for + namespace packages. .. attribute:: loader_state - The :term:`finder` may set this attribute to an object containing additional, - module-specific data to use when loading the module. Otherwise it should be - set to ``None``. + The :term:`finder` may set this attribute to an object containing additional, + module-specific data to use when loading the module. Otherwise it should be + set to ``None``. .. attribute:: cached - (:attr:`__cached__`) - - The filename of a compiled version of the module's code. - The :term:`finder` should always set this attribute but it may be ``None`` - for modules that do not need compiled code stored. + The filename of a compiled version of the module's code + (see :attr:`module.__cached__`). + The :term:`finder` should always set this attribute but it may be ``None`` + for modules that do not need compiled code stored. .. attribute:: parent - (:attr:`__package__`) - - (Read-only) The fully qualified name of the package the module is in (or the - empty string for a top-level module). - If the module is a package then this is the same as :attr:`name`. + (Read-only) The fully qualified name of the package the module is in (or the + empty string for a top-level module). + See :attr:`module.__package__`. + If the module is a package then this is the same as :attr:`name`. .. attribute:: has_location - ``True`` if the spec's :attr:`origin` refers to a loadable location, - ``False`` otherwise. This value impacts how :attr:`origin` is interpreted - and how the module's :attr:`__file__` is populated. + ``True`` if the spec's :attr:`origin` refers to a loadable location, + ``False`` otherwise. This value impacts how :attr:`!origin` is interpreted + and how the module's :attr:`~module.__file__` is populated. + + +.. class:: AppleFrameworkLoader(name, path) + + A specialization of :class:`importlib.machinery.ExtensionFileLoader` that + is able to load extension modules in Framework format. + + For compatibility with the iOS App Store, *all* binary modules in an iOS app + must be dynamic libraries, contained in a framework with appropriate + metadata, stored in the ``Frameworks`` folder of the packaged app. There can + be only a single binary per framework, and there can be no executable binary + material outside the Frameworks folder. + + To accommodate this requirement, when running on iOS, extension module + binaries are *not* packaged as ``.so`` files on ``sys.path``, but as + individual standalone frameworks. To discover those frameworks, this loader + is be registered against the ``.fwork`` file extension, with a ``.fwork`` + file acting as a placeholder in the original location of the binary on + ``sys.path``. The ``.fwork`` file contains the path of the actual binary in + the ``Frameworks`` folder, relative to the app bundle. To allow for + resolving a framework-packaged binary back to the original location, the + framework is expected to contain a ``.origin`` file that contains the + location of the ``.fwork`` file, relative to the app bundle. + + For example, consider the case of an import ``from foo.bar import _whiz``, + where ``_whiz`` is implemented with the binary module + ``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location + registered on ``sys.path``, relative to the application bundle. This module + *must* be distributed as + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` (creating the framework + name from the full import path of the module), with an ``Info.plist`` file + in the ``.framework`` directory identifying the binary as a framework. The + ``foo.bar._whiz`` module would be represented in the original location with + a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing the path + ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also contain + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing the + path to the ``.fwork`` file. + + When a module is loaded with this loader, the ``__file__`` for the module + will report as the location of the ``.fwork`` file. This allows code to use + the ``__file__`` of a module as an anchor for file system traversal. + However, the spec origin will reference the location of the *actual* binary + in the ``.framework`` folder. + + The Xcode project building the app is responsible for converting any ``.so`` + files from wherever they exist in the ``PYTHONPATH`` into frameworks in the + ``Frameworks`` folder (including stripping extensions from the module file, + the addition of framework metadata, and signing the resulting framework), + and creating the ``.fwork`` and ``.origin`` files. This will usually be done + with a build step in the Xcode project; see the iOS documentation for + details on how to construct this build step. + + .. versionadded:: 3.13 + + .. availability:: iOS. + + .. attribute:: name + + Name of the module the loader supports. + + .. attribute:: path + + Path to the ``.fwork`` file for the extension module. :mod:`importlib.util` -- Utility code for importers @@ -1360,8 +1397,8 @@ an :term:`importer`. .. versionchanged:: 3.7 Raises :exc:`ModuleNotFoundError` instead of :exc:`AttributeError` if - **package** is in fact not a package (i.e. lacks a :attr:`__path__` - attribute). + **package** is in fact not a package (i.e. lacks a + :attr:`~module.__path__` attribute). .. function:: module_from_spec(spec) @@ -1521,20 +1558,34 @@ Note that if ``name`` is a submodule (contains a dot), Importing a source file directly '''''''''''''''''''''''''''''''' -To import a Python source file directly, use the following recipe:: +This recipe should be used with caution: it is an approximation of an import +statement where the file path is specified directly, rather than +:data:`sys.path` being searched. Alternatives should first be considered first, +such as modifying :data:`sys.path` when a proper module is required, or using +:func:`runpy.run_path` when the global namespace resulting from running a Python +file is appropriate. - import importlib.util - import sys +To import a Python source file directly from a path, use the following recipe:: - # For illustrative purposes. - import tokenize - file_path = tokenize.__file__ - module_name = tokenize.__name__ + import importlib.util + import sys + + + def import_from_path(module_name, file_path): + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module - spec = importlib.util.spec_from_file_location(module_name, file_path) - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) + + # For illustrative purposes only (use of `json` is arbitrary). + import json + file_path = json.__file__ + module_name = json.__name__ + + # Similar outcome as `import json`. + json = import_from_path(module_name, file_path) Implementing lazy imports @@ -1560,7 +1611,6 @@ The example below shows how to implement lazy imports:: False - Setting up an importer '''''''''''''''''''''' diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index f8b3e39c4f54f0..ca5dac87aff2b4 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1,5 +1,5 @@ -:mod:`inspect` --- Inspect live objects -======================================= +:mod:`!inspect` --- Inspect live objects +======================================== .. testsetup:: * @@ -42,210 +42,258 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. this function name is too big to fit in the ascii-art table below .. |coroutine-origin-link| replace:: :func:`sys.set_coroutine_origin_tracking_depth` -+-----------+-------------------+---------------------------+ -| Type | Attribute | Description | -+===========+===================+===========================+ -| class | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | name with which this | -| | | class was defined | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __module__ | name of module in which | -| | | this class was defined | -+-----------+-------------------+---------------------------+ -| method | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | name with which this | -| | | method was defined | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __func__ | function object | -| | | containing implementation | -| | | of method | -+-----------+-------------------+---------------------------+ -| | __self__ | instance to which this | -| | | method is bound, or | -| | | ``None`` | -+-----------+-------------------+---------------------------+ -| | __module__ | name of module in which | -| | | this method was defined | -+-----------+-------------------+---------------------------+ -| function | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | name with which this | -| | | function was defined | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __code__ | code object containing | -| | | compiled function | -| | | :term:`bytecode` | -+-----------+-------------------+---------------------------+ -| | __defaults__ | tuple of any default | -| | | values for positional or | -| | | keyword parameters | -+-----------+-------------------+---------------------------+ -| | __kwdefaults__ | mapping of any default | -| | | values for keyword-only | -| | | parameters | -+-----------+-------------------+---------------------------+ -| | __globals__ | global namespace in which | -| | | this function was defined | -+-----------+-------------------+---------------------------+ -| | __builtins__ | builtins namespace | -+-----------+-------------------+---------------------------+ -| | __annotations__ | mapping of parameters | -| | | names to annotations; | -| | | ``"return"`` key is | -| | | reserved for return | -| | | annotations. | -+-----------+-------------------+---------------------------+ -| | __module__ | name of module in which | -| | | this function was defined | -+-----------+-------------------+---------------------------+ -| traceback | tb_frame | frame object at this | -| | | level | -+-----------+-------------------+---------------------------+ -| | tb_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-------------------+---------------------------+ -| | tb_lineno | current line number in | -| | | Python source code | -+-----------+-------------------+---------------------------+ -| | tb_next | next inner traceback | -| | | object (called by this | -| | | level) | -+-----------+-------------------+---------------------------+ -| frame | f_back | next outer frame object | -| | | (this frame's caller) | -+-----------+-------------------+---------------------------+ -| | f_builtins | builtins namespace seen | -| | | by this frame | -+-----------+-------------------+---------------------------+ -| | f_code | code object being | -| | | executed in this frame | -+-----------+-------------------+---------------------------+ -| | f_globals | global namespace seen by | -| | | this frame | -+-----------+-------------------+---------------------------+ -| | f_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-------------------+---------------------------+ -| | f_lineno | current line number in | -| | | Python source code | -+-----------+-------------------+---------------------------+ -| | f_locals | local namespace seen by | -| | | this frame | -+-----------+-------------------+---------------------------+ -| | f_trace | tracing function for this | -| | | frame, or ``None`` | -+-----------+-------------------+---------------------------+ -| code | co_argcount | number of arguments (not | -| | | including keyword only | -| | | arguments, \* or \*\* | -| | | args) | -+-----------+-------------------+---------------------------+ -| | co_code | string of raw compiled | -| | | bytecode | -+-----------+-------------------+---------------------------+ -| | co_cellvars | tuple of names of cell | -| | | variables (referenced by | -| | | containing scopes) | -+-----------+-------------------+---------------------------+ -| | co_consts | tuple of constants used | -| | | in the bytecode | -+-----------+-------------------+---------------------------+ -| | co_filename | name of file in which | -| | | this code object was | -| | | created | -+-----------+-------------------+---------------------------+ -| | co_firstlineno | number of first line in | -| | | Python source code | -+-----------+-------------------+---------------------------+ -| | co_flags | bitmap of ``CO_*`` flags, | -| | | read more :ref:`here | -| | | `| -+-----------+-------------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------+-------------------+---------------------------+ -| | co_freevars | tuple of names of free | -| | | variables (referenced via | -| | | a function's closure) | -+-----------+-------------------+---------------------------+ -| | co_posonlyargcount| number of positional only | -| | | arguments | -+-----------+-------------------+---------------------------+ -| | co_kwonlyargcount | number of keyword only | -| | | arguments (not including | -| | | \*\* arg) | -+-----------+-------------------+---------------------------+ -| | co_name | name with which this code | -| | | object was defined | -+-----------+-------------------+---------------------------+ -| | co_qualname | fully qualified name with | -| | | which this code object | -| | | was defined | -+-----------+-------------------+---------------------------+ -| | co_names | tuple of names other | -| | | than arguments and | -| | | function locals | -+-----------+-------------------+---------------------------+ -| | co_nlocals | number of local variables | -+-----------+-------------------+---------------------------+ -| | co_stacksize | virtual machine stack | -| | | space required | -+-----------+-------------------+---------------------------+ -| | co_varnames | tuple of names of | -| | | arguments and local | -| | | variables | -+-----------+-------------------+---------------------------+ -| generator | __name__ | name | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | gi_frame | frame | -+-----------+-------------------+---------------------------+ -| | gi_running | is the generator running? | -+-----------+-------------------+---------------------------+ -| | gi_code | code | -+-----------+-------------------+---------------------------+ -| | gi_yieldfrom | object being iterated by | -| | | ``yield from``, or | -| | | ``None`` | -+-----------+-------------------+---------------------------+ -| coroutine | __name__ | name | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | cr_await | object being awaited on, | -| | | or ``None`` | -+-----------+-------------------+---------------------------+ -| | cr_frame | frame | -+-----------+-------------------+---------------------------+ -| | cr_running | is the coroutine running? | -+-----------+-------------------+---------------------------+ -| | cr_code | code | -+-----------+-------------------+---------------------------+ -| | cr_origin | where coroutine was | -| | | created, or ``None``. See | -| | | |coroutine-origin-link| | -+-----------+-------------------+---------------------------+ -| builtin | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | original name of this | -| | | function or method | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __self__ | instance to which a | -| | | method is bound, or | -| | | ``None`` | -+-----------+-------------------+---------------------------+ ++-----------------+-------------------+---------------------------+ +| Type | Attribute | Description | ++=================+===================+===========================+ +| class | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | class was defined | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this class was defined | ++-----------------+-------------------+---------------------------+ +| | __type_params__ | A tuple containing the | +| | | :ref:`type parameters | +| | | ` of | +| | | a generic class | ++-----------------+-------------------+---------------------------+ +| method | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | method was defined | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __func__ | function object | +| | | containing implementation | +| | | of method | ++-----------------+-------------------+---------------------------+ +| | __self__ | instance to which this | +| | | method is bound, or | +| | | ``None`` | ++-----------------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this method was defined | ++-----------------+-------------------+---------------------------+ +| function | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | function was defined | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __code__ | code object containing | +| | | compiled function | +| | | :term:`bytecode` | ++-----------------+-------------------+---------------------------+ +| | __defaults__ | tuple of any default | +| | | values for positional or | +| | | keyword parameters | ++-----------------+-------------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | ++-----------------+-------------------+---------------------------+ +| | __globals__ | global namespace in which | +| | | this function was defined | ++-----------------+-------------------+---------------------------+ +| | __builtins__ | builtins namespace | ++-----------------+-------------------+---------------------------+ +| | __annotations__ | mapping of parameters | +| | | names to annotations; | +| | | ``"return"`` key is | +| | | reserved for return | +| | | annotations. | ++-----------------+-------------------+---------------------------+ +| | __type_params__ | A tuple containing the | +| | | :ref:`type parameters | +| | | ` of | +| | | a generic function | ++-----------------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this function was defined | ++-----------------+-------------------+---------------------------+ +| traceback | tb_frame | frame object at this | +| | | level | ++-----------------+-------------------+---------------------------+ +| | tb_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------------+-------------------+---------------------------+ +| | tb_lineno | current line number in | +| | | Python source code | ++-----------------+-------------------+---------------------------+ +| | tb_next | next inner traceback | +| | | object (called by this | +| | | level) | ++-----------------+-------------------+---------------------------+ +| frame | f_back | next outer frame object | +| | | (this frame's caller) | ++-----------------+-------------------+---------------------------+ +| | f_builtins | builtins namespace seen | +| | | by this frame | ++-----------------+-------------------+---------------------------+ +| | f_code | code object being | +| | | executed in this frame | ++-----------------+-------------------+---------------------------+ +| | f_globals | global namespace seen by | +| | | this frame | ++-----------------+-------------------+---------------------------+ +| | f_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------------+-------------------+---------------------------+ +| | f_lineno | current line number in | +| | | Python source code | ++-----------------+-------------------+---------------------------+ +| | f_locals | local namespace seen by | +| | | this frame | ++-----------------+-------------------+---------------------------+ +| | f_trace | tracing function for this | +| | | frame, or ``None`` | ++-----------------+-------------------+---------------------------+ +| | f_trace_lines | indicate whether a | +| | | tracing event is | +| | | triggered for each source | +| | | source line | ++-----------------+-------------------+---------------------------+ +| | f_trace_opcodes | indicate whether | +| | | per-opcode events are | +| | | requested | ++-----------------+-------------------+---------------------------+ +| | clear() | used to clear all | +| | | references to local | +| | | variables | ++-----------------+-------------------+---------------------------+ +| code | co_argcount | number of arguments (not | +| | | including keyword only | +| | | arguments, \* or \*\* | +| | | args) | ++-----------------+-------------------+---------------------------+ +| | co_code | string of raw compiled | +| | | bytecode | ++-----------------+-------------------+---------------------------+ +| | co_cellvars | tuple of names of cell | +| | | variables (referenced by | +| | | containing scopes) | ++-----------------+-------------------+---------------------------+ +| | co_consts | tuple of constants used | +| | | in the bytecode | ++-----------------+-------------------+---------------------------+ +| | co_filename | name of file in which | +| | | this code object was | +| | | created | ++-----------------+-------------------+---------------------------+ +| | co_firstlineno | number of first line in | +| | | Python source code | ++-----------------+-------------------+---------------------------+ +| | co_flags | bitmap of ``CO_*`` flags, | +| | | read more :ref:`here | +| | | `| ++-----------------+-------------------+---------------------------+ +| | co_lnotab | encoded mapping of line | +| | | numbers to bytecode | +| | | indices | ++-----------------+-------------------+---------------------------+ +| | co_freevars | tuple of names of free | +| | | variables (referenced via | +| | | a function's closure) | ++-----------------+-------------------+---------------------------+ +| | co_posonlyargcount| number of positional only | +| | | arguments | ++-----------------+-------------------+---------------------------+ +| | co_kwonlyargcount | number of keyword only | +| | | arguments (not including | +| | | \*\* arg) | ++-----------------+-------------------+---------------------------+ +| | co_name | name with which this code | +| | | object was defined | ++-----------------+-------------------+---------------------------+ +| | co_qualname | fully qualified name with | +| | | which this code object | +| | | was defined | ++-----------------+-------------------+---------------------------+ +| | co_names | tuple of names other | +| | | than arguments and | +| | | function locals | ++-----------------+-------------------+---------------------------+ +| | co_nlocals | number of local variables | ++-----------------+-------------------+---------------------------+ +| | co_stacksize | virtual machine stack | +| | | space required | ++-----------------+-------------------+---------------------------+ +| | co_varnames | tuple of names of | +| | | arguments and local | +| | | variables | ++-----------------+-------------------+---------------------------+ +| | co_lines() | returns an iterator that | +| | | yields successive | +| | | bytecode ranges | ++-----------------+-------------------+---------------------------+ +| | co_positions() | returns an iterator of | +| | | source code positions for | +| | | each bytecode instruction | ++-----------------+-------------------+---------------------------+ +| | replace() | returns a copy of the | +| | | code object with new | +| | | values | ++-----------------+-------------------+---------------------------+ +| generator | __name__ | name | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | gi_frame | frame | ++-----------------+-------------------+---------------------------+ +| | gi_running | is the generator running? | ++-----------------+-------------------+---------------------------+ +| | gi_code | code | ++-----------------+-------------------+---------------------------+ +| | gi_yieldfrom | object being iterated by | +| | | ``yield from``, or | +| | | ``None`` | ++-----------------+-------------------+---------------------------+ +| async generator | __name__ | name | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | ag_await | object being awaited on, | +| | | or ``None`` | ++-----------------+-------------------+---------------------------+ +| | ag_frame | frame | ++-----------------+-------------------+---------------------------+ +| | ag_running | is the generator running? | ++-----------------+-------------------+---------------------------+ +| | ag_code | code | ++-----------------+-------------------+---------------------------+ +| coroutine | __name__ | name | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | cr_await | object being awaited on, | +| | | or ``None`` | ++-----------------+-------------------+---------------------------+ +| | cr_frame | frame | ++-----------------+-------------------+---------------------------+ +| | cr_running | is the coroutine running? | ++-----------------+-------------------+---------------------------+ +| | cr_code | code | ++-----------------+-------------------+---------------------------+ +| | cr_origin | where coroutine was | +| | | created, or ``None``. See | +| | | |coroutine-origin-link| | ++-----------------+-------------------+---------------------------+ +| builtin | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | original name of this | +| | | function or method | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __self__ | instance to which a | +| | | method is bound, or | +| | | ``None`` | ++-----------------+-------------------+---------------------------+ .. versionchanged:: 3.5 @@ -326,6 +374,13 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a bound method written in Python. +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + + .. function:: isfunction(object) Return ``True`` if the object is a Python function, which includes functions @@ -340,6 +395,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. + .. versionchanged:: 3.13 + Functions wrapped in :func:`functools.partialmethod` now return ``True`` + if the wrapped function is a Python generator function. .. function:: isgenerator(object) @@ -363,6 +421,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Sync functions marked with :func:`markcoroutinefunction` now return ``True``. + .. versionchanged:: 3.13 + Functions wrapped in :func:`functools.partialmethod` now return ``True`` + if the wrapped function is a :term:`coroutine function`. + .. function:: markcoroutinefunction(func) @@ -427,8 +489,11 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the - wrapped function is a :term:`asynchronous generator` function. + wrapped function is an :term:`asynchronous generator` function. + .. versionchanged:: 3.13 + Functions wrapped in :func:`functools.partialmethod` now return ``True`` + if the wrapped function is a :term:`coroutine function`. .. function:: isasyncgen(object) @@ -484,10 +549,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): are true. This, for example, is true of ``int.__add__``. An object passing this test - has a :meth:`~object.__get__` method but not a :meth:`~object.__set__` - method, but beyond that the set of attributes varies. A - :attr:`~definition.__name__` attribute is usually - sensible, and :attr:`!__doc__` often is. + has a :meth:`~object.__get__` method, but not a :meth:`~object.__set__` + method or a :meth:`~object.__delete__` method. Beyond that, the set of + attributes varies. A :attr:`~definition.__name__` attribute is usually + sensible, and :attr:`~definition.__doc__` often is. Methods implemented via descriptors that also pass one of the other tests return ``False`` from the :func:`ismethoddescriptor` test, simply because the @@ -495,6 +560,11 @@ attributes (see :ref:`import-mod-attrs` for module attributes): :attr:`~method.__func__` attribute (etc) when an object passes :func:`ismethod`. + .. versionchanged:: 3.13 + This function no longer incorrectly reports objects with :meth:`~object.__get__` + and :meth:`~object.__delete__`, but not :meth:`~object.__set__`, as being method + descriptors (such objects are data descriptors, not method descriptors). + .. function:: isdatadescriptor(object) @@ -631,7 +701,7 @@ and its return annotation. To retrieve a :class:`!Signature` object, use the :func:`!signature` function. -.. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) +.. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False, annotation_format=Format.VALUE) Return a :class:`Signature` object for the given *callable*: @@ -655,25 +725,27 @@ function. Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. - If the passed object has a ``__signature__`` attribute, this function - returns it without further computations. - - For objects defined in modules using stringized annotations - (``from __future__ import annotations``), :func:`signature` will + If some of the annotations are strings (e.g., because + ``from __future__ import annotations`` was used), :func:`signature` will attempt to automatically un-stringize the annotations using - :func:`get_annotations`. The + :func:`annotationlib.get_annotations`. The *globals*, *locals*, and *eval_str* parameters are passed - into :func:`get_annotations` when resolving the - annotations; see the documentation for :func:`get_annotations` - for instructions on how to use these parameters. + into :func:`!annotationlib.get_annotations` when resolving the + annotations; see the documentation for :func:`!annotationlib.get_annotations` + for instructions on how to use these parameters. A member of the + :class:`annotationlib.Format` enum can be passed to the + *annotation_format* parameter to control the format of the returned + annotations. For example, use + ``annotation_format=annotationlib.Format.STRING`` to return annotations in string + format. Raises :exc:`ValueError` if no signature can be provided, and :exc:`TypeError` if that type of object is not supported. Also, if the annotations are stringized, and *eval_str* is not false, - the ``eval()`` call(s) to un-stringize the annotations in :func:`get_annotations` + the ``eval()`` call(s) to un-stringize the annotations in :func:`annotationlib.get_annotations` could potentially raise any kind of exception. - A slash(/) in the signature of a function denotes that the parameters prior + A slash (/) in the signature of a function denotes that the parameters prior to it are positional-only. For more info, see :ref:`the FAQ entry on positional-only parameters `. @@ -686,12 +758,22 @@ function. .. versionchanged:: 3.10 The *globals*, *locals*, and *eval_str* parameters were added. + .. versionchanged:: 3.14 + The *annotation_format* parameter was added. + .. note:: Some callables may not be introspectable in certain implementations of Python. For example, in CPython, some built-in functions defined in C provide no metadata about their arguments. + .. impl-detail:: + + If the passed object has a :attr:`!__signature__` attribute, + we may use it to create the signature. + The exact semantics are an implementation detail and are subject to + unannounced changes. Consult the source code for current semantics. + .. class:: Signature(parameters=None, *, return_annotation=Signature.empty) @@ -771,7 +853,7 @@ function. :class:`Signature` objects are also supported by the generic function :func:`copy.replace`. - .. method:: format(*, max_width=None) + .. method:: format(*, max_width=None, quote_annotation_strings=True) Create a string representation of the :class:`Signature` object. @@ -780,8 +862,17 @@ function. If the signature is longer than *max_width*, all parameters will be on separate lines. + If *quote_annotation_strings* is False, :term:`annotations ` + in the signature are displayed without opening and closing quotation + marks if they are strings. This is useful if the signature was created with the + :attr:`~annotationlib.Format.STRING` format or if + ``from __future__ import annotations`` was used. + .. versionadded:: 3.13 + .. versionchanged:: 3.14 + The *unquote_annotations* parameter was added. + .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) Return a :class:`Signature` (or its subclass) object for a given callable @@ -896,7 +987,7 @@ function. .. attribute:: Parameter.kind.description - Describes a enum value of :attr:`Parameter.kind`. + Describes an enum value of :attr:`Parameter.kind`. .. versionadded:: 3.8 @@ -976,7 +1067,8 @@ function. .. attribute:: BoundArguments.kwargs A dict of keyword arguments values. Dynamically computed from the - :attr:`arguments` attribute. + :attr:`arguments` attribute. Arguments that can be passed positionally + are included in :attr:`args` instead. .. attribute:: BoundArguments.signature @@ -1180,62 +1272,19 @@ Classes and functions .. versionadded:: 3.4 -.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False) +.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False, format=annotationlib.Format.VALUE) Compute the annotations dict for an object. - ``obj`` may be a callable, class, or module. - Passing in an object of any other type raises :exc:`TypeError`. - - Returns a dict. ``get_annotations()`` returns a new dict every time - it's called; calling it twice on the same object will return two - different but equivalent dicts. - - This function handles several details for you: - - * If ``eval_str`` is true, values of type ``str`` will - be un-stringized using :func:`eval()`. This is intended - for use with stringized annotations - (``from __future__ import annotations``). - * If ``obj`` doesn't have an annotations dict, returns an - empty dict. (Functions and methods always have an - annotations dict; classes, modules, and other types of - callables may not.) - * Ignores inherited annotations on classes. If a class - doesn't have its own annotations dict, returns an empty dict. - * All accesses to object members and dict values are done - using ``getattr()`` and ``dict.get()`` for safety. - * Always, always, always returns a freshly created dict. - - ``eval_str`` controls whether or not values of type ``str`` are replaced - with the result of calling :func:`eval()` on those values: - - * If eval_str is true, :func:`eval()` is called on values of type ``str``. - (Note that ``get_annotations`` doesn't catch exceptions; if :func:`eval()` - raises an exception, it will unwind the stack past the ``get_annotations`` - call.) - * If eval_str is false (the default), values of type ``str`` are unchanged. - - ``globals`` and ``locals`` are passed in to :func:`eval()`; see the documentation - for :func:`eval()` for more information. If ``globals`` or ``locals`` - is ``None``, this function may replace that value with a context-specific - default, contingent on ``type(obj)``: - - * If ``obj`` is a module, ``globals`` defaults to ``obj.__dict__``. - * If ``obj`` is a class, ``globals`` defaults to - ``sys.modules[obj.__module__].__dict__`` and ``locals`` defaults - to the ``obj`` class namespace. - * If ``obj`` is a callable, ``globals`` defaults to - :attr:`obj.__globals__ `, - although if ``obj`` is a wrapped function (using - :func:`functools.update_wrapper`) it is first unwrapped. - - Calling ``get_annotations`` is best practice for accessing the - annotations dict of any object. See :ref:`annotations-howto` for - more information on annotations best practices. + This is an alias for :func:`annotationlib.get_annotations`; see the documentation + of that function for more information. .. versionadded:: 3.10 + .. versionchanged:: 3.14 + This function is now an alias for :func:`annotationlib.get_annotations`. + Calling it as ``inspect.get_annotations`` will continue to work. + .. _inspect-stack: @@ -1651,6 +1700,14 @@ which is a bitmap of the following flags: .. versionadded:: 3.6 +.. data:: CO_HAS_DOCSTRING + + The flag is set when there is a docstring for the code object in + the source code. If set, it will be the first item in + :attr:`~codeobject.co_consts`. + + .. versionadded:: 3.14 + .. note:: The flags are specific to CPython, and may not be defined in other Python implementations. Furthermore, the flags are an implementation diff --git a/Doc/library/intro.rst b/Doc/library/intro.rst index 5a4c9b8b16ab3b..8f76044be488cd 100644 --- a/Doc/library/intro.rst +++ b/Doc/library/intro.rst @@ -58,7 +58,7 @@ Notes on availability operating system. * If not separately noted, all functions that claim "Availability: Unix" are - supported on macOS, which builds on a Unix core. + supported on macOS, iOS and Android, all of which build on a Unix core. * If an availability note contains both a minimum Kernel version and a minimum libc version, then both conditions must hold. For example a feature with note @@ -119,3 +119,58 @@ DOM APIs as well as limited networking capabilities with JavaScript's .. _wasmtime: https://wasmtime.dev/ .. _Pyodide: https://pyodide.org/ .. _PyScript: https://pyscript.net/ + +.. _mobile-availability: +.. _iOS-availability: + +Mobile platforms +---------------- + +Android and iOS are, in most respects, POSIX operating systems. File I/O, socket handling, +and threading all behave as they would on any POSIX operating system. However, +there are several major differences: + +* Mobile platforms can only use Python in "embedded" mode. There is no Python + REPL, and no ability to use separate executables such as :program:`python` or + :program:`pip`. To add Python code to your mobile app, you must use + the :ref:`Python embedding API `. For more details, see + :ref:`using-android` and :ref:`using-ios`. + +* Subprocesses: + + * On Android, creating subprocesses is possible but `officially unsupported + `__. + In particular, Android does not support any part of the System V IPC API, + so :mod:`multiprocessing` is not available. + + * An iOS app cannot use any form of subprocessing, multiprocessing, or + inter-process communication. If an iOS app attempts to create a subprocess, + the process creating the subprocess will either lock up, or crash. An iOS app + has no visibility of other applications that are running, nor any ability to + communicate with other running applications, outside of the iOS-specific APIs + that exist for this purpose. + +* Mobile apps have limited access to modify system resources (such as the system + clock). These resources will often be *readable*, but attempts to modify + those resources will usually fail. + +* Console input and output: + + * On Android, the native ``stdout`` and ``stderr`` are not connected to + anything, so Python installs its own streams which redirect messages to the + system log. These can be seen under the tags ``python.stdout`` and + ``python.stderr`` respectively. + + * iOS apps have a limited concept of console output. ``stdout`` and + ``stderr`` *exist*, and content written to ``stdout`` and ``stderr`` will be + visible in logs when running in Xcode, but this content *won't* be recorded + in the system log. If a user who has installed your app provides their app + logs as a diagnostic aid, they will not include any detail written to + ``stdout`` or ``stderr``. + + * Mobile apps have no usable ``stdin`` at all. While apps can display an on-screen + keyboard, this is a software feature, not something that is attached to + ``stdin``. + + As a result, Python modules that involve console manipulation (such as + :mod:`curses` and :mod:`readline`) are not available on mobile platforms. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 8eb531aa4ea248..f793d7a7ef9a84 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -1,5 +1,5 @@ -:mod:`io` --- Core tools for working with streams -================================================= +:mod:`!io` --- Core tools for working with streams +================================================== .. module:: io :synopsis: Core tools for working with streams. @@ -55,7 +55,7 @@ the backing store is natively made of bytes (such as in the case of a file), encoding and decoding of data is made transparently as well as optional translation of platform-specific newline characters. -The easiest way to create a text stream is with :meth:`open()`, optionally +The easiest way to create a text stream is with :meth:`open`, optionally specifying an encoding:: f = open("myfile.txt", "r", encoding="utf-8") @@ -77,7 +77,7 @@ objects. No encoding, decoding, or newline translation is performed. This category of streams can be used for all kinds of non-text data, and also when manual control over the handling of text data is desired. -The easiest way to create a binary stream is with :meth:`open()` with ``'b'`` in +The easiest way to create a binary stream is with :meth:`open` with ``'b'`` in the mode string:: f = open("myfile.jpg", "rb") @@ -950,7 +950,7 @@ Text I/O :class:`TextIOBase`. *encoding* gives the name of the encoding that the stream will be decoded or - encoded with. It defaults to :func:`locale.getencoding()`. + encoded with. It defaults to :func:`locale.getencoding`. ``encoding="locale"`` can be used to specify the current locale's encoding explicitly. See :ref:`io-text-encoding` for more information. @@ -1182,7 +1182,7 @@ re-enter a buffered object which it is already accessing, a :exc:`RuntimeError` is raised. Note this doesn't prohibit a different thread from entering the buffered object. -The above implicitly extends to text files, since the :func:`open()` function +The above implicitly extends to text files, since the :func:`open` function will wrap a buffered object inside a :class:`TextIOWrapper`. This includes -standard streams and therefore affects the built-in :func:`print()` function as +standard streams and therefore affects the built-in :func:`print` function as well. diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 1de36b643c4dca..e5bdfbb144b65a 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -1,5 +1,5 @@ -:mod:`ipaddress` --- IPv4/IPv6 manipulation library -=================================================== +:mod:`!ipaddress` --- IPv4/IPv6 manipulation library +==================================================== .. module:: ipaddress :synopsis: IPv4/IPv6 manipulation library. @@ -121,26 +121,20 @@ write code that handles both IP versions correctly. Address objects are Leading zeros are tolerated, even in ambiguous cases that look like octal notation. - .. versionchanged:: 3.10 + .. versionchanged:: 3.9.5 Leading zeros are no longer tolerated and are treated as an error. IPv4 address strings are now parsed as strict as glibc :func:`~socket.inet_pton`. - .. versionchanged:: 3.9.5 - - The above change was also included in Python 3.9 starting with - version 3.9.5. - - .. versionchanged:: 3.8.12 - - The above change was also included in Python 3.8 starting with - version 3.8.12. - .. attribute:: version The appropriate version number: ``4`` for IPv4, ``6`` for IPv6. + .. versionchanged:: 3.14 + + Made available on the class. + .. attribute:: max_prefixlen The total number of bits in the address representation for this @@ -150,6 +144,10 @@ write code that handles both IP versions correctly. Address objects are are compared to determine whether or not an address is part of a network. + .. versionchanged:: 3.14 + + Made available on the class. + .. attribute:: compressed .. attribute:: exploded @@ -188,18 +186,53 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_private - ``True`` if the address is allocated for private networks. See + ``True`` if the address is defined as not globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exceptions: + + * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``) + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + + ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. + + .. versionchanged:: 3.13 + + Fixed some false positives and false negatives. + + * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and + ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private). + * ``64:ff9b:1::/48`` is considered private. + * ``2002::/16`` is considered private. + * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``, + ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``. + The exceptions are not considered private. .. attribute:: is_global - ``True`` if the address is allocated for public networks. See + ``True`` if the address is defined as globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. .. versionadded:: 3.4 + .. versionchanged:: 3.13 + + Fixed some false positives and false negatives, see :attr:`is_private` for details. + .. attribute:: is_unspecified ``True`` if the address is unspecified. See :RFC:`5735` (for IPv4) @@ -309,14 +342,14 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_multicast .. attribute:: is_private .. attribute:: is_global + + .. versionadded:: 3.4 + .. attribute:: is_unspecified .. attribute:: is_reserved .. attribute:: is_loopback .. attribute:: is_link_local - .. versionadded:: 3.4 - is_global - .. attribute:: is_site_local ``True`` if the address is reserved for site-local usage. Note that @@ -479,7 +512,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 32-bits integer, a 4-bytes packed - integer, or an existing IPv4Address object; and the netmask is either + integer, or an existing :class:`IPv4Address` object; and the netmask is either an integer representing the prefix length (e.g. ``24``) or a string representing the prefix mask (e.g. ``255.255.255.0``). @@ -700,7 +733,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 128-bits integer, a 16-bytes packed - integer, or an existing IPv6Address object; and the netmask is an + integer, or an existing :class:`IPv6Address` object; and the netmask is an integer representing the prefix length. An :exc:`AddressValueError` is raised if *address* is not a valid IPv6 @@ -756,7 +789,7 @@ dictionaries. .. attribute:: is_site_local - These attribute is true for the network as a whole if it is true + This attribute is true for the network as a whole if it is true for both the network address and the broadcast address. @@ -965,7 +998,7 @@ The module also provides the following module level functions: .. function:: collapse_addresses(addresses) Return an iterator of the collapsed :class:`IPv4Network` or - :class:`IPv6Network` objects. *addresses* is an iterator of + :class:`IPv6Network` objects. *addresses* is an :term:`iterable` of :class:`IPv4Network` or :class:`IPv6Network` objects. A :exc:`TypeError` is raised if *addresses* contains mixed version objects. @@ -985,7 +1018,7 @@ The module also provides the following module level functions: doesn't make sense. There are some times however, where you may wish to have :mod:`ipaddress` sort these anyway. If you need to do this, you can use - this function as the *key* argument to :func:`sorted()`. + this function as the *key* argument to :func:`sorted`. *obj* is either a network or address object. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 338a5f9615aae3..c138e903fa5a0f 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1,5 +1,5 @@ -:mod:`itertools` --- Functions creating iterators for efficient looping -======================================================================= +:mod:`!itertools` --- Functions creating iterators for efficient looping +======================================================================== .. module:: itertools :synopsis: Functions creating iterators for efficient looping. @@ -41,9 +41,9 @@ operator can be mapped across two vectors to form an efficient dot-product: ================== ================= ================================================= ========================================= Iterator Arguments Results Example ================== ================= ================================================= ========================================= -:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) --> 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') --> A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) --> 10 10 10`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` +:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` +:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` ================== ================= ================================================= ========================================= **Iterators terminating on the shortest input sequence:** @@ -51,20 +51,20 @@ Iterator Arguments Results ============================ ============================ ================================================= ============================================================= Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= -:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) --> ABC DEF G`` -:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` -:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` -:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` -:func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` -:func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` -:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) -:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` -:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') --> AB BC CD DE EF FG`` -:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` -:func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` -:func:`tee` it, n it1, it2, ... itn splits one iterator into n -:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` +:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) → 1 3 6 10 15`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) → ABC DEF G`` +:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` +:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` +:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` +:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` +:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` +:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` +:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` +:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` +:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` +:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` +:func:`tee` it, n it1, it2, ... itn splits one iterator into n ``tee('ABC', 2) → A B C, A B C`` +:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D-`` ============================ ============================ ================================================= ============================================================= **Combinatoric iterators:** @@ -84,72 +84,71 @@ Examples Results ``product('ABCD', repeat=2)`` ``AA AB AC AD BA BB BC BD CA CB CC CD DA DB DC DD`` ``permutations('ABCD', 2)`` ``AB AC AD BA BC BD CA CB CD DA DB DC`` ``combinations('ABCD', 2)`` ``AB AC AD BC BD CD`` -``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` +``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` ============================================== ============================================================= .. _itertools-functions: -Itertool functions +Itertool Functions ------------------ -The following module functions all construct and return iterators. Some provide +The following functions all construct and return iterators. Some provide streams of infinite length, so they should only be accessed by functions or loops that truncate the stream. -.. function:: accumulate(iterable[, func, *, initial=None]) - Make an iterator that returns accumulated sums, or accumulated - results of other binary functions (specified via the optional - *func* argument). +.. function:: accumulate(iterable[, function, *, initial=None]) - If *func* is supplied, it should be a function - of two arguments. Elements of the input *iterable* may be any type - that can be accepted as arguments to *func*. (For example, with - the default operation of addition, elements may be any addable - type including :class:`~decimal.Decimal` or - :class:`~fractions.Fraction`.) + Make an iterator that returns accumulated sums or accumulated + results from other binary functions. - Usually, the number of elements output matches the input iterable. - However, if the keyword argument *initial* is provided, the - accumulation leads off with the *initial* value so that the output - has one more element than the input iterable. + The *function* defaults to addition. The *function* should accept + two arguments, an accumulated total and a value from the *iterable*. + + If an *initial* value is provided, the accumulation will start with + that value and the output will have one more element than the input + iterable. Roughly equivalent to:: - def accumulate(iterable, func=operator.add, *, initial=None): + def accumulate(iterable, function=operator.add, *, initial=None): 'Return running totals' - # accumulate([1,2,3,4,5]) --> 1 3 6 10 15 - # accumulate([1,2,3,4,5], initial=100) --> 100 101 103 106 110 115 - # accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120 - it = iter(iterable) + # accumulate([1,2,3,4,5]) → 1 3 6 10 15 + # accumulate([1,2,3,4,5], initial=100) → 100 101 103 106 110 115 + # accumulate([1,2,3,4,5], operator.mul) → 1 2 6 24 120 + + iterator = iter(iterable) total = initial if initial is None: try: - total = next(it) + total = next(iterator) except StopIteration: return + yield total - for element in it: - total = func(total, element) + for element in iterator: + total = function(total, element) yield total - There are a number of uses for the *func* argument. It can be set to - :func:`min` for a running minimum, :func:`max` for a running maximum, or - :func:`operator.mul` for a running product. Amortization tables can be - built by accumulating interest and applying payments: + To compute a running minimum, set *function* to :func:`min`. + For a running maximum, set *function* to :func:`max`. + Or for a running product, set *function* to :func:`operator.mul`. + To build an `amortization table + `_, + accumulate the interest and apply payments: .. doctest:: >>> data = [3, 4, 6, 2, 1, 9, 0, 7, 5, 8] - >>> list(accumulate(data, operator.mul)) # running product - [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] >>> list(accumulate(data, max)) # running maximum [3, 4, 6, 6, 6, 9, 9, 9, 9, 9] + >>> list(accumulate(data, operator.mul)) # running product + [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] # Amortize a 5% loan of 1000 with 10 annual payments of 90 - >>> account_update = lambda bal, pmt: round(bal * 1.05) + pmt - >>> list(accumulate(repeat(-90, 10), account_update, initial=1_000)) + >>> update = lambda balance, payment: round(balance * 1.05) - payment + >>> list(accumulate(repeat(90, 10), update, initial=1_000)) [1000, 960, 918, 874, 828, 779, 728, 674, 618, 559, 497] See :func:`functools.reduce` for a similar function that returns only the @@ -158,7 +157,7 @@ loops that truncate the stream. .. versionadded:: 3.2 .. versionchanged:: 3.3 - Added the optional *func* parameter. + Added the optional *function* parameter. .. versionchanged:: 3.8 Added the optional *initial* parameter. @@ -184,21 +183,14 @@ loops that truncate the stream. >>> unflattened [('roses', 'red'), ('violets', 'blue'), ('sugar', 'sweet')] - >>> for batch in batched('ABCDEFG', 3): - ... print(batch) - ... - ('A', 'B', 'C') - ('D', 'E', 'F') - ('G',) - Roughly equivalent to:: def batched(iterable, n, *, strict=False): - # batched('ABCDEFG', 3) --> ABC DEF G + # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') - it = iter(iterable) - while batch := tuple(islice(it, n)): + iterator = iter(iterable) + while batch := tuple(islice(iterator, n)): if strict and len(batch) != n: raise ValueError('batched(): incomplete batch') yield batch @@ -211,16 +203,15 @@ loops that truncate the stream. .. function:: chain(*iterables) - Make an iterator that returns elements from the first iterable until it is - exhausted, then proceeds to the next iterable, until all of the iterables are - exhausted. Used for treating consecutive sequences as a single sequence. - Roughly equivalent to:: + Make an iterator that returns elements from the first iterable until + it is exhausted, then proceeds to the next iterable, until all of the + iterables are exhausted. This combines multiple data sources into a + single iterator. Roughly equivalent to:: def chain(*iterables): - # chain('ABC', 'DEF') --> A B C D E F - for it in iterables: - for element in it: - yield element + # chain('ABC', 'DEF') → A B C D E F + for iterable in iterables: + yield from iterable .. classmethod:: chain.from_iterable(iterable) @@ -229,34 +220,40 @@ loops that truncate the stream. single iterable argument that is evaluated lazily. Roughly equivalent to:: def from_iterable(iterables): - # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F - for it in iterables: - for element in it: - yield element + # chain.from_iterable(['ABC', 'DEF']) → A B C D E F + for iterable in iterables: + yield from iterable .. function:: combinations(iterable, r) Return *r* length subsequences of elements from the input *iterable*. - The combination tuples are emitted in lexicographic ordering according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + The output is a subsequence of :func:`product` keeping only entries that + are subsequences of the *iterable*. The length of the output is given + by :func:`math.comb` which computes ``n! / r! / (n - r)!`` when ``0 ≤ r + ≤ n`` or zero when ``r > n``. + + The combination tuples are emitted in lexicographic order according to + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated - values in each combination. + value. If the input elements are unique, there will be no repeated + values within each combination. Roughly equivalent to:: def combinations(iterable, r): - # combinations('ABCD', 2) --> AB AC AD BC BD CD - # combinations(range(4), 3) --> 012 013 023 123 + # combinations('ABCD', 2) → AB AC AD BC BD CD + # combinations(range(4), 3) → 012 013 023 123 + pool = tuple(iterable) n = len(pool) if r > n: return indices = list(range(r)) + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -269,42 +266,36 @@ loops that truncate the stream. indices[j] = indices[j-1] + 1 yield tuple(pool[i] for i in indices) - The code for :func:`combinations` can be also expressed as a subsequence - of :func:`permutations` after filtering entries where the elements are not - in sorted order (according to their position in the input pool):: - - def combinations(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in permutations(range(n), r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / r! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: combinations_with_replacement(iterable, r) Return *r* length subsequences of elements from the input *iterable* allowing individual elements to be repeated more than once. - The combination tuples are emitted in lexicographic ordering according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + The output is a subsequence of :func:`product` that keeps only entries + that are subsequences (with possible repeated elements) of the + *iterable*. The number of subsequence returned is ``(n + r - 1)! / r! / + (n - 1)!`` when ``n > 0``. + + The combination tuples are emitted in lexicographic order according to + the order of the input *iterable*. if the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, the generated combinations + value. If the input elements are unique, the generated combinations will also be unique. Roughly equivalent to:: def combinations_with_replacement(iterable, r): - # combinations_with_replacement('ABC', 2) --> AA AB AC BB BC CC + # combinations_with_replacement('ABC', 2) → AA AB AC BB BC CC + pool = tuple(iterable) n = len(pool) if not n and r: return indices = [0] * r + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -315,104 +306,103 @@ loops that truncate the stream. indices[i:] = [indices[i] + 1] * (r - i) yield tuple(pool[i] for i in indices) - The code for :func:`combinations_with_replacement` can be also expressed as - a subsequence of :func:`product` after filtering entries where the elements - are not in sorted order (according to their position in the input pool):: - - def combinations_with_replacement(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in product(range(n), repeat=r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``(n+r-1)! / r! / (n-1)!`` when ``n > 0``. - .. versionadded:: 3.1 .. function:: compress(data, selectors) - Make an iterator that filters elements from *data* returning only those that - have a corresponding element in *selectors* that evaluates to ``True``. - Stops when either the *data* or *selectors* iterables has been exhausted. - Roughly equivalent to:: + Make an iterator that returns elements from *data* where the + corresponding element in *selectors* is true. Stops when either the + *data* or *selectors* iterables have been exhausted. Roughly + equivalent to:: def compress(data, selectors): - # compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F - return (d for d, s in zip(data, selectors) if s) + # compress('ABCDEF', [1,0,1,0,1,1]) → A C E F + return (datum for datum, selector in zip(data, selectors) if selector) .. versionadded:: 3.1 .. function:: count(start=0, step=1) - Make an iterator that returns evenly spaced values starting with number *start*. Often - used as an argument to :func:`map` to generate consecutive data points. - Also, used with :func:`zip` to add sequence numbers. Roughly equivalent to:: + Make an iterator that returns evenly spaced values beginning with + *start*. Can be used with :func:`map` to generate consecutive data + points or with :func:`zip` to add sequence numbers. Roughly + equivalent to:: def count(start=0, step=1): - # count(10) --> 10 11 12 13 14 ... - # count(2.5, 0.5) --> 2.5 3.0 3.5 ... + # count(10) → 10 11 12 13 14 ... + # count(2.5, 0.5) → 2.5 3.0 3.5 ... n = start while True: yield n n += step - When counting with floating point numbers, better accuracy can sometimes be + When counting with floating-point numbers, better accuracy can sometimes be achieved by substituting multiplicative code such as: ``(start + step * i for i in count())``. .. versionchanged:: 3.1 Added *step* argument and allowed non-integer arguments. + .. function:: cycle(iterable) - Make an iterator returning elements from the iterable and saving a copy of each. - When the iterable is exhausted, return elements from the saved copy. Repeats - indefinitely. Roughly equivalent to:: + Make an iterator returning elements from the *iterable* and saving a + copy of each. When the iterable is exhausted, return elements from + the saved copy. Repeats indefinitely. Roughly equivalent to:: def cycle(iterable): - # cycle('ABCD') --> A B C D A B C D A B C D ... + # cycle('ABCD') → A B C D A B C D A B C D ... + saved = [] for element in iterable: yield element saved.append(element) + while saved: for element in saved: - yield element + yield element - Note, this member of the toolkit may require significant auxiliary storage - (depending on the length of the iterable). + This itertool may require significant auxiliary storage (depending on + the length of the iterable). .. function:: dropwhile(predicate, iterable) - Make an iterator that drops elements from the iterable as long as the predicate - is true; afterwards, returns every element. Note, the iterator does not produce - *any* output until the predicate first becomes false, so it may have a lengthy - start-up time. Roughly equivalent to:: + Make an iterator that drops elements from the *iterable* while the + *predicate* is true and afterwards returns every element. Roughly + equivalent to:: def dropwhile(predicate, iterable): - # dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1 - iterable = iter(iterable) - for x in iterable: + # dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8 + + iterator = iter(iterable) + for x in iterator: if not predicate(x): yield x break - for x in iterable: + + for x in iterator: yield x + Note this does not produce *any* output until the predicate first + becomes false, so this itertool may have a lengthy start-up time. + + .. function:: filterfalse(predicate, iterable) - Make an iterator that filters elements from iterable returning only those for - which the predicate is false. If *predicate* is ``None``, return the items - that are false. Roughly equivalent to:: + Make an iterator that filters elements from the *iterable* returning + only those for which the *predicate* returns a false value. If + *predicate* is ``None``, returns the items that are false. Roughly + equivalent to:: def filterfalse(predicate, iterable): - # filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8 + # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 + if predicate is None: predicate = bool + for x in iterable: if not predicate(x): yield x @@ -446,82 +436,81 @@ loops that truncate the stream. :func:`groupby` is roughly equivalent to:: - class groupby: - # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B - # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D - - def __init__(self, iterable, key=None): - if key is None: - key = lambda x: x - self.keyfunc = key - self.it = iter(iterable) - self.tgtkey = self.currkey = self.currvalue = object() - - def __iter__(self): - return self - - def __next__(self): - self.id = object() - while self.currkey == self.tgtkey: - self.currvalue = next(self.it) # Exit on StopIteration - self.currkey = self.keyfunc(self.currvalue) - self.tgtkey = self.currkey - return (self.currkey, self._grouper(self.tgtkey, self.id)) - - def _grouper(self, tgtkey, id): - while self.id is id and self.currkey == tgtkey: - yield self.currvalue - try: - self.currvalue = next(self.it) - except StopIteration: + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: return - self.currkey = self.keyfunc(self.currvalue) + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass .. function:: islice(iterable, stop) islice(iterable, start, stop[, step]) - Make an iterator that returns selected elements from the iterable. If *start* is - non-zero, then elements from the iterable are skipped until start is reached. - Afterward, elements are returned consecutively unless *step* is set higher than - one which results in items being skipped. If *stop* is ``None``, then iteration - continues until the iterator is exhausted, if at all; otherwise, it stops at the - specified position. + Make an iterator that returns selected elements from the iterable. + Works like sequence slicing but does not support negative values for + *start*, *stop*, or *step*. + + If *start* is zero or ``None``, iteration starts at zero. Otherwise, + elements from the iterable are skipped until *start* is reached. - If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, - then the step defaults to one. + If *stop* is ``None``, iteration continues until the input is + exhausted, if at all. Otherwise, it stops at the specified position. - Unlike regular slicing, :func:`islice` does not support negative values for - *start*, *stop*, or *step*. Can be used to extract related fields from - data where the internal structure has been flattened (for example, a - multi-line report may list a name field on every third line). + If *step* is ``None``, the step defaults to one. Elements are returned + consecutively unless *step* is set higher than one which results in + items being skipped. Roughly equivalent to:: def islice(iterable, *args): - # islice('ABCDEFG', 2) --> A B - # islice('ABCDEFG', 2, 4) --> C D - # islice('ABCDEFG', 2, None) --> C D E F G - # islice('ABCDEFG', 0, None, 2) --> A C E G + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step + + If the input is an iterator, then fully consuming the *islice* + advances the input iterator by ``max(start, stop)`` steps regardless + of the *step* value. .. function:: pairwise(iterable) @@ -535,43 +524,56 @@ loops that truncate the stream. Roughly equivalent to:: def pairwise(iterable): - # pairwise('ABCDEFG') --> AB BC CD DE EF FG - a, b = tee(iterable) - next(b, None) - return zip(a, b) + # pairwise('ABCDEFG') → AB BC CD DE EF FG + + iterator = iter(iterable) + a = next(iterator, None) + + for b in iterator: + yield a, b + a = b .. versionadded:: 3.10 .. function:: permutations(iterable, r=None) - Return successive *r* length permutations of elements in the *iterable*. + Return successive *r* length `permutations of elements + `_ from the *iterable*. If *r* is not specified or is ``None``, then *r* defaults to the length of the *iterable* and all possible full-length permutations are generated. + The output is a subsequence of :func:`product` where entries with + repeated elements have been filtered out. The length of the output is + given by :func:`math.perm` which computes ``n! / (n - r)!`` when + ``0 ≤ r ≤ n`` or zero when ``r > n``. + The permutation tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated + value. If the input elements are unique, there will be no repeated values within a permutation. Roughly equivalent to:: def permutations(iterable, r=None): - # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC - # permutations(range(3)) --> 012 021 102 120 201 210 + # permutations('ABCD', 2) → AB AC AD BA BC BD CA CB CD DA DB DC + # permutations(range(3)) → 012 021 102 120 201 210 + pool = tuple(iterable) n = len(pool) r = n if r is None else r if r > n: return + indices = list(range(n)) cycles = list(range(n, n-r, -1)) yield tuple(pool[i] for i in indices[:r]) + while n: for i in reversed(range(r)): cycles[i] -= 1 @@ -586,24 +588,11 @@ loops that truncate the stream. else: return - The code for :func:`permutations` can be also expressed as a subsequence of - :func:`product`, filtered to exclude entries with repeated elements (those - from the same position in the input pool):: - - def permutations(iterable, r=None): - pool = tuple(iterable) - n = len(pool) - r = n if r is None else r - for indices in product(range(n), repeat=r): - if len(set(indices)) == r: - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: product(*iterables, repeat=1) - Cartesian product of input iterables. + `Cartesian product `_ + of the input iterables. Roughly equivalent to nested for-loops in a generator expression. For example, ``product(A, B)`` returns the same as ``((x,y) for x in A for y in B)``. @@ -620,13 +609,18 @@ loops that truncate the stream. This function is roughly equivalent to the following code, except that the actual implementation does not build up intermediate results in memory:: - def product(*args, repeat=1): - # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy - # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 - pools = [tuple(pool) for pool in args] * repeat + def product(*iterables, repeat=1): + # product('ABCD', 'xy') → Ax Ay Bx By Cx Cy Dx Dy + # product(range(2), repeat=3) → 000 001 010 011 100 101 110 111 + + if repeat < 0: + raise ValueError('repeat argument cannot be negative') + pools = [tuple(pool) for pool in iterables] * repeat + result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] + for prod in result: yield tuple(prod) @@ -634,6 +628,7 @@ loops that truncate the stream. keeping pools of values in memory to generate the products. Accordingly, it is only useful with finite inputs. + .. function:: repeat(object[, times]) Make an iterator that returns *object* over and over again. Runs indefinitely @@ -642,7 +637,7 @@ loops that truncate the stream. Roughly equivalent to:: def repeat(object, times=None): - # repeat(10, 3) --> 10 10 10 + # repeat(10, 3) → 10 10 10 if times is None: while True: yield object @@ -658,63 +653,108 @@ loops that truncate the stream. >>> list(map(pow, range(10), repeat(2))) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] + .. function:: starmap(function, iterable) - Make an iterator that computes the function using arguments obtained from - the iterable. Used instead of :func:`map` when argument parameters are already - grouped in tuples from a single iterable (when the data has been - "pre-zipped"). + Make an iterator that computes the *function* using arguments obtained + from the *iterable*. Used instead of :func:`map` when argument + parameters have already been "pre-zipped" into tuples. The difference between :func:`map` and :func:`starmap` parallels the distinction between ``function(a,b)`` and ``function(*c)``. Roughly equivalent to:: def starmap(function, iterable): - # starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000 + # starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000 for args in iterable: yield function(*args) .. function:: takewhile(predicate, iterable) - Make an iterator that returns elements from the iterable as long as the - predicate is true. Roughly equivalent to:: + Make an iterator that returns elements from the *iterable* as long as + the *predicate* is true. Roughly equivalent to:: def takewhile(predicate, iterable): - # takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4 + # takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4 for x in iterable: - if predicate(x): - yield x - else: + if not predicate(x): break + yield x + + Note, the element that first fails the predicate condition is + consumed from the input iterator and there is no way to access it. + This could be an issue if an application wants to further consume the + input iterator after *takewhile* has been run to exhaustion. To work + around this problem, consider using `more-iterools before_and_after() + `_ + instead. .. function:: tee(iterable, n=2) Return *n* independent iterators from a single iterable. - The following Python code helps explain what *tee* does (although the actual - implementation is more complex and uses only a single underlying - :abbr:`FIFO (first-in, first-out)` queue):: + Roughly equivalent to:: def tee(iterable, n=2): - it = iter(iterable) - deques = [collections.deque() for i in range(n)] - def gen(mydeque): - while True: - if not mydeque: # when the local deque is empty - try: - newval = next(it) # fetch a new value and - except StopIteration: - return - for d in deques: # load it to all the deques - d.append(newval) - yield mydeque.popleft() - return tuple(gen(d) for d in deques) - - Once a :func:`tee` has been created, the original *iterable* should not be - used anywhere else; otherwise, the *iterable* could get advanced without - the tee objects being informed. + if n < 0: + raise ValueError + if n == 0: + return () + iterator = _tee(iterable) + result = [iterator] + for _ in range(n - 1): + result.append(_tee(iterator)) + return tuple(result) + + class _tee: + + def __init__(self, iterable): + it = iter(iterable) + if isinstance(it, _tee): + self.iterator = it.iterator + self.link = it.link + else: + self.iterator = it + self.link = [None, None] + + def __iter__(self): + return self + + def __next__(self): + link = self.link + if link[1] is None: + link[0] = next(self.iterator) + link[1] = [None, None] + value, self.link = link + return value + + When the input *iterable* is already a tee iterator object, all + members of the return tuple are constructed as if they had been + produced by the upstream :func:`tee` call. This "flattening step" + allows nested :func:`tee` calls to share the same underlying data + chain and to have a single update step rather than a chain of calls. + + The flattening property makes tee iterators efficiently peekable: + + .. testcode:: + + def lookahead(tee_iterator): + "Return the next value without moving the input forward" + [forked_iterator] = tee(tee_iterator, 1) + return next(forked_iterator) + + .. doctest:: + + >>> iterator = iter('abcdef') + >>> [iterator] = tee(iterator, 1) # Make the input peekable + >>> next(iterator) # Move the iterator forward + 'a' + >>> lookahead(iterator) # Check next value + 'b' + >>> next(iterator) # Continue moving forward + 'b' ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be raised when simultaneously using iterators returned by the same :func:`tee` @@ -728,21 +768,29 @@ loops that truncate the stream. .. function:: zip_longest(*iterables, fillvalue=None) - Make an iterator that aggregates elements from each of the iterables. If the - iterables are of uneven length, missing values are filled-in with *fillvalue*. - Iteration continues until the longest iterable is exhausted. Roughly equivalent to:: + Make an iterator that aggregates elements from each of the + *iterables*. + + If the iterables are of uneven length, missing values are filled-in + with *fillvalue*. If not specified, *fillvalue* defaults to ``None``. - def zip_longest(*args, fillvalue=None): - # zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D- - iterators = [iter(it) for it in args] + Iteration continues until the longest iterable is exhausted. + + Roughly equivalent to:: + + def zip_longest(*iterables, fillvalue=None): + # zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D- + + iterators = list(map(iter, iterables)) num_active = len(iterators) if not num_active: return + while True: values = [] - for i, it in enumerate(iterators): + for i, iterator in enumerate(iterators): try: - value = next(it) + value = next(iterator) except StopIteration: num_active -= 1 if not num_active: @@ -754,8 +802,7 @@ loops that truncate the stream. If one of the iterables is potentially infinite, then the :func:`zip_longest` function should be wrapped with something that limits the number of calls - (for example :func:`islice` or :func:`takewhile`). If not specified, - *fillvalue* defaults to ``None``. + (for example :func:`islice` or :func:`takewhile`). .. _itertools-recipes: @@ -770,33 +817,34 @@ The primary purpose of the itertools recipes is educational. The recipes show various ways of thinking about individual tools — for example, that ``chain.from_iterable`` is related to the concept of flattening. The recipes also give ideas about ways that the tools can be combined — for example, how -``compress()`` and ``range()`` can work together. The recipes also show patterns +``starmap()`` and ``repeat()`` can work together. The recipes also show patterns for using itertools with the :mod:`operator` and :mod:`collections` modules as well as with the built-in itertools such as ``map()``, ``filter()``, ``reversed()``, and ``enumerate()``. A secondary purpose of the recipes is to serve as an incubator. The ``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``sliding_window()`` and ``iter_index()`` recipes -are being tested to see whether they prove their worth. +recipes. Currently, the ``sliding_window()``, ``iter_index()``, and ``sieve()`` +recipes are being tested to see whether they prove their worth. Substantially all of these recipes and many, many others can be installed from -the `more-itertools project `_ found +the :pypi:`more-itertools` project found on the Python Package Index:: python -m pip install more-itertools Many of the recipes offer the same high performance as the underlying toolset. -Superior memory performance is kept by processing elements one at a time -rather than bringing the whole iterable into memory all at once. Code volume is -kept small by linking the tools together in a functional style which helps -eliminate temporary variables. High speed is retained by preferring -"vectorized" building blocks over the use of for-loops and :term:`generator`\s -which incur interpreter overhead. +Superior memory performance is kept by processing elements one at a time rather +than bringing the whole iterable into memory all at once. Code volume is kept +small by linking the tools together in a `functional style +`_. High speed +is retained by preferring "vectorized" building blocks over the use of for-loops +and :term:`generators ` which incur interpreter overhead. .. testcode:: import collections + import contextlib import functools import math import operator @@ -808,7 +856,7 @@ which incur interpreter overhead. def prepend(value, iterable): "Prepend a single value in front of an iterable." - # prepend(1, [2, 3, 4]) --> 1 2 3 4 + # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) def tabulate(function, start=0): @@ -816,10 +864,7 @@ which incur interpreter overhead. return map(function, count(start)) def repeatfunc(func, times=None, *args): - """Repeat calls to func with specified arguments. - - Example: repeatfunc(random.random) - """ + "Repeat calls to func with specified arguments." if times is None: return starmap(func, repeat(args)) return starmap(func, repeat(args, times)) @@ -834,49 +879,48 @@ which incur interpreter overhead. def tail(n, iterable): "Return an iterator over the last n items." - # tail(3, 'ABCDEFG') --> E F G + # tail(3, 'ABCDEFG') → E F G return iter(collections.deque(iterable, maxlen=n)) def consume(iterator, n=None): "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: - # feed the entire iterator into a zero-length deque collections.deque(iterator, maxlen=0) else: - # advance to the empty slice starting at position n next(islice(iterator, n, n), None) def nth(iterable, n, default=None): "Returns the nth item or a default value." return next(islice(iterable, n, None), default) - def quantify(iterable, pred=bool): + def quantify(iterable, predicate=bool): "Given a predicate that returns True or False, count the True results." - return sum(map(pred, iterable)) - - def all_equal(iterable): - "Returns True if all the elements are equal to each other." - g = groupby(iterable) - return next(g, True) and not next(g, False) - - def first_true(iterable, default=False, pred=None): - """Returns the first true value in the iterable. + return sum(map(predicate, iterable)) - If no true value is found, returns *default* + def first_true(iterable, default=False, predicate=None): + "Returns the first true value or the *default* if there is no true value." + # first_true([a,b,c], x) → a or b or c or x + # first_true([a,b], x, f) → a if f(a) else b if f(b) else x + return next(filter(predicate, iterable), default) - If *pred* is not None, returns the first item - for which pred(item) is true. + def all_equal(iterable, key=None): + "Returns True if all the elements are equal to each other." + # all_equal('4٤௪౪໔', key=int) → True + return len(take(2, groupby(iterable, key))) <= 1 - """ - # first_true([a,b,c], x) --> a or b or c or x - # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x - return next(filter(pred, iterable), default) + def unique_justseen(iterable, key=None): + "Yield unique elements, preserving order. Remember only the element just seen." + # unique_justseen('AAAABBBCCDAABBB') → A B C D A B + # unique_justseen('ABBcCAD', str.casefold) → A B c A D + if key is None: + return map(operator.itemgetter(0), groupby(iterable)) + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.casefold) --> A B c D + "Yield unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') → A B C D + # unique_everseen('ABBcCAD', str.casefold) → A B c D seen = set() if key is None: for element in filterfalse(seen.__contains__, iterable): @@ -889,147 +933,76 @@ which incur interpreter overhead. seen.add(k) yield element - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.casefold) --> A B c A D - if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - - def iter_index(iterable, value, start=0, stop=None): - "Return indices where a value occurs in a sequence or iterable." - # iter_index('AABCADEAF', 'A') --> 0 1 4 7 - seq_index = getattr(iterable, 'index', None) - if seq_index is None: - # Slow path for general iterables - it = islice(iterable, start, stop) - for i, element in enumerate(it, start): - if element is value or element == value: - yield i - else: - # Fast path for sequences - stop = len(iterable) if stop is None else stop - i = start - 1 - try: - while True: - yield (i := seq_index(value, i+1, stop)) - except ValueError: - pass + def unique(iterable, key=None, reverse=False): + "Yield unique elements in sorted order. Supports unhashable inputs." + # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] + return unique_justseen(sorted(iterable, key=key, reverse=reverse), key=key) def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." - # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG - it = iter(iterable) - window = collections.deque(islice(it, n-1), maxlen=n) - for x in it: + # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG + iterator = iter(iterable) + window = collections.deque(islice(iterator, n - 1), maxlen=n) + for x in iterator: window.append(x) yield tuple(window) def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks." - # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx - # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError - # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF - args = [iter(iterable)] * n + # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx + # grouper('ABCDEFG', 3, incomplete='strict') → ABC DEF ValueError + # grouper('ABCDEFG', 3, incomplete='ignore') → ABC DEF + iterators = [iter(iterable)] * n match incomplete: case 'fill': - return zip_longest(*args, fillvalue=fillvalue) + return zip_longest(*iterators, fillvalue=fillvalue) case 'strict': - return zip(*args, strict=True) + return zip(*iterators, strict=True) case 'ignore': - return zip(*args) + return zip(*iterators) case _: raise ValueError('Expected fill, strict, or ignore') def roundrobin(*iterables): "Visit input iterables in a cycle until each is exhausted." - # roundrobin('ABC', 'D', 'EF') --> A D E B F C - # Recipe credited to George Sakkis - num_active = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while num_active: - try: - for next in nexts: - yield next() - except StopIteration: - # Remove the iterator we just exhausted from the cycle. - num_active -= 1 - nexts = cycle(islice(nexts, num_active)) - - def partition(pred, iterable): - """Partition entries into false entries and true entries. - - If *pred* is slow, consider wrapping it with functools.lru_cache(). - """ - # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 - t1, t2 = tee(iterable) - return filterfalse(pred, t1), filter(pred, t2) + # roundrobin('ABC', 'D', 'EF') → A D E B F C + # Algorithm credited to George Sakkis + iterators = map(iter, iterables) + for num_active in range(len(iterables), 0, -1): + iterators = cycle(islice(iterators, num_active)) + yield from map(next, iterators) def subslices(seq): "Return all contiguous non-empty subslices of a sequence." - # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D + # subslices('ABCD') → A AB ABC ABCD B BC BCD C CD D slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Priority queue iterator: - iter_except(functools.partial(heappop, h), IndexError) - - Non-blocking dictionary iterator: - iter_except(d.popitem, KeyError) - - Non-blocking deque iterator: - iter_except(d.popleft, IndexError) - - Non-blocking iterator over a producer Queue: - iter_except(q.get_nowait, Queue.Empty) - - Non-blocking set iterator: - iter_except(s.pop, KeyError) + def iter_index(iterable, value, start=0, stop=None): + "Return indices where a value occurs in a sequence or iterable." + # iter_index('AABCADEAF', 'A') → 0 1 4 7 + seq_index = getattr(iterable, 'index', None) + if seq_index is None: + iterator = islice(iterable, start, stop) + for i, element in enumerate(iterator, start): + if element is value or element == value: + yield i + else: + stop = len(iterable) if stop is None else stop + i = start + with contextlib.suppress(ValueError): + while True: + yield (i := seq_index(value, i, stop)) + i += 1 - """ - try: + def iter_except(func, exception, first=None): + "Convert a call-until-exception interface to an iterator interface." + # iter_except(d.popitem, KeyError) → non-blocking dictionary iterator + with contextlib.suppress(exception): if first is not None: - # For database APIs needing an initial call to db.first() yield first() while True: yield func() - except exception: - pass - - def before_and_after(predicate, it): - """ Variant of takewhile() that allows complete - access to the remainder of the iterator. - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) # takewhile() would lose the 'd' - 'dEfGhI' - - Note that the true iterator must be fully consumed - before the remainder iterator can generate valid results. - """ - it = iter(it) - transition = [] - - def true_iterator(): - for elem in it: - if predicate(elem): - yield elem - else: - transition.append(elem) - return - - return true_iterator(), chain(transition, it) The following recipes have a more mathematical flavor: @@ -1037,47 +1010,46 @@ The following recipes have a more mathematical flavor: .. testcode:: def powerset(iterable): - "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" + "powerset([1,2,3]) → () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) - def sum_of_squares(it): + def sum_of_squares(iterable): "Add up the squares of the input values." - # sum_of_squares([10, 20, 30]) -> 1400 - return math.sumprod(*tee(it)) + # sum_of_squares([10, 20, 30]) → 1400 + return math.sumprod(*tee(iterable)) def reshape(matrix, cols): "Reshape a 2-D matrix to have a given number of columns." - # reshape([(0, 1), (2, 3), (4, 5)], 3) --> (0, 1, 2), (3, 4, 5) + # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2), (3, 4, 5) return batched(chain.from_iterable(matrix), cols, strict=True) def transpose(matrix): "Swap the rows and columns of a 2-D matrix." - # transpose([(1, 2, 3), (11, 22, 33)]) --> (1, 11) (2, 22) (3, 33) + # transpose([(1, 2, 3), (11, 22, 33)]) → (1, 11) (2, 22) (3, 33) return zip(*matrix, strict=True) def matmul(m1, m2): "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) --> (49, 80), (41, 60) + # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80), (41, 60) n = len(m2[0]) return batched(starmap(math.sumprod, product(m1, transpose(m2))), n) def convolve(signal, kernel): """Discrete linear convolution of two iterables. + Equivalent to polynomial multiplication. - The kernel is fully consumed before the calculations begin. - The signal is consumed lazily and can be infinite. - - Convolutions are mathematically commutative. - If the signal and kernel are swapped, - the output will be the same. + Convolutions are mathematically commutative; however, the inputs are + evaluated differently. The signal is consumed lazily and can be + infinite. The kernel is fully consumed before the calculations begin. Article: https://betterexplained.com/articles/intuitive-convolution/ Video: https://www.youtube.com/watch?v=KuXjwB4LzSA """ - # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) - # convolve(data, [1/2, 0, -1/2]) --> 1st derivative estimate - # convolve(data, [1, -2, 1]) --> 2nd derivative estimate + # convolve([1, -1, -20], [1, -3]) → 1 -4 -17 60 + # convolve(data, [0.25, 0.25, 0.25, 0.25]) → Moving average (blur) + # convolve(data, [1/2, 0, -1/2]) → 1st derivative estimate + # convolve(data, [1, -2, 1]) → 2nd derivative estimate kernel = tuple(kernel)[::-1] n = len(kernel) padded_signal = chain(repeat(0, n-1), signal, repeat(0, n-1)) @@ -1089,7 +1061,7 @@ The following recipes have a more mathematical flavor: (x - 5) (x + 4) (x - 3) expands to: x³ -4x² -17x + 60 """ - # polynomial_from_roots([5, -4, 3]) --> [1, -4, -17, 60] + # polynomial_from_roots([5, -4, 3]) → [1, -4, -17, 60] factors = zip(repeat(1), map(operator.neg, roots)) return list(functools.reduce(convolve, factors, [1])) @@ -1098,8 +1070,8 @@ The following recipes have a more mathematical flavor: Computes with better numeric stability than Horner's method. """ - # Evaluate x³ -4x² -17x + 60 at x = 2.5 - # polynomial_eval([1, -4, -17, 60], x=2.5) --> 8.125 + # Evaluate x³ -4x² -17x + 60 at x = 5 + # polynomial_eval([1, -4, -17, 60], x=5) → 0 n = len(coefficients) if not n: return type(x)(0) @@ -1112,30 +1084,26 @@ The following recipes have a more mathematical flavor: f(x) = x³ -4x² -17x + 60 f'(x) = 3x² -8x -17 """ - # polynomial_derivative([1, -4, -17, 60]) -> [3, -8, -17] + # polynomial_derivative([1, -4, -17, 60]) → [3, -8, -17] n = len(coefficients) powers = reversed(range(1, n)) return list(map(operator.mul, coefficients, powers)) def sieve(n): "Primes less than n." - # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + # sieve(30) → 2 3 5 7 11 13 17 19 23 29 if n > 2: yield 2 - start = 3 data = bytearray((0, 1)) * (n // 2) - limit = math.isqrt(n) + 1 - for p in iter_index(data, 1, start, limit): - yield from iter_index(data, 1, start, p*p) + for p in iter_index(data, 1, start=3, stop=math.isqrt(n) + 1): data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - start = p*p - yield from iter_index(data, 1, start) + yield from iter_index(data, 1, start=3) def factor(n): "Prime factors of n." - # factor(99) --> 3 3 11 - # factor(1_000_000_000_000_007) --> 47 59 360620266859 - # factor(1_000_000_000_000_403) --> 1000000000000403 + # factor(99) → 3 3 11 + # factor(1_000_000_000_000_007) → 47 59 360620266859 + # factor(1_000_000_000_000_403) → 1000000000000403 for prime in sieve(math.isqrt(n) + 1): while not n % prime: yield prime @@ -1148,9 +1116,9 @@ The following recipes have a more mathematical flavor: def totient(n): "Count of natural numbers up to n that are coprime to n." # https://mathworld.wolfram.com/TotientFunction.html - # totient(12) --> 4 because len([1, 5, 7, 11]) == 4 - for p in unique_justseen(factor(n)): - n -= n // p + # totient(12) → 4 because len([1, 5, 7, 11]) == 4 + for prime in set(factor(n)): + n -= n // prime return n @@ -1214,61 +1182,98 @@ The following recipes have a more mathematical flavor: >>> take(10, count()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> # Verify that the input is consumed lazily + >>> it = iter('abcdef') + >>> take(3, it) + ['a', 'b', 'c'] + >>> list(it) + ['d', 'e', 'f'] + >>> list(prepend(1, [2, 3, 4])) [1, 2, 3, 4] + >>> list(enumerate('abc')) [(0, 'a'), (1, 'b'), (2, 'c')] + >>> list(islice(tabulate(lambda x: 2*x), 4)) [0, 2, 4, 6] + >>> list(tail(3, 'ABCDEFG')) ['E', 'F', 'G'] + >>> # Verify the input is consumed greedily + >>> input_iterator = iter('ABCDEFG') + >>> output_iterator = tail(3, input_iterator) + >>> list(input_iterator) + [] + >>> it = iter(range(10)) >>> consume(it, 3) + >>> # Verify the input is consumed lazily >>> next(it) 3 + >>> # Verify the input is consumed completely >>> consume(it) >>> next(it, 'Done') 'Done' + >>> nth('abcde', 3) 'd' - >>> nth('abcde', 9) is None True + >>> # Verify that the input is consumed lazily + >>> it = iter('abcde') + >>> nth(it, 2) + 'c' + >>> list(it) + ['d', 'e'] + >>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] [True, True, True, False, False] + >>> [all_equal(s, key=str.casefold) for s in ('', 'A', 'AaAa', 'AAAB', 'AAABA')] + [True, True, True, False, False] + >>> # Verify that the input is consumed lazily and that only + >>> # one element of a second equivalence class is used to disprove + >>> # the assertion that all elements are equal. + >>> it = iter('aaabbbccc') + >>> all_equal(it) + False + >>> ''.join(it) + 'bbccc' + >>> quantify(range(99), lambda x: x%2==0) 50 - >>> quantify([True, False, False, True, True]) 3 - - >>> quantify(range(12), pred=lambda x: x%2==1) + >>> quantify(range(12), predicate=lambda x: x%2==1) 6 + >>> a = [[1, 2, 3], [4, 5, 6]] >>> list(flatten(a)) [1, 2, 3, 4, 5, 6] - >>> list(repeatfunc(pow, 5, 2, 3)) - [8, 8, 8, 8, 8] - - >>> take(5, map(int, repeatfunc(random.random))) - [0, 0, 0, 0, 0] >>> list(ncycles('abc', 3)) ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] + >>> # Verify greedy consumption of input iterator + >>> input_iterator = iter('abc') + >>> output_iterator = ncycles(input_iterator, 3) + >>> list(input_iterator) + [] + >>> sum_of_squares([10, 20, 30]) 1400 + >>> list(reshape([(0, 1), (2, 3), (4, 5)], 3)) [(0, 1, 2), (3, 4, 5)] >>> M = [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] @@ -1289,14 +1294,27 @@ The following recipes have a more mathematical flavor: >>> list(reshape(M, 12)) [(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)] + >>> list(transpose([(1, 2, 3), (11, 22, 33)])) [(1, 11), (2, 22), (3, 33)] + >>> # Verify that the inputs are consumed lazily + >>> input1 = iter([1, 2, 3]) + >>> input2 = iter([11, 22, 33]) + >>> output_iterator = transpose([input1, input2]) + >>> next(output_iterator) + (1, 11) + >>> list(zip(input1, input2)) + [(2, 22), (3, 33)] + >>> list(matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]])) [(49, 80), (41, 60)] >>> list(matmul([[2, 5], [7, 9], [3, 4]], [[7, 11, 5, 4, 9], [3, 5, 2, 6, 3]])) [(29, 47, 20, 38, 33), (76, 122, 53, 82, 90), (33, 53, 23, 36, 39)] + + >>> list(convolve([1, -1, -20], [1, -3])) == [1, -4, -17, 60] + True >>> data = [20, 40, 24, 32, 20, 28, 16] >>> list(convolve(data, [0.25, 0.25, 0.25, 0.25])) [5.0, 15.0, 21.0, 29.0, 29.0, 26.0, 24.0, 16.0, 11.0, 4.0] @@ -1304,13 +1322,26 @@ The following recipes have a more mathematical flavor: [20, 20, -16, 8, -12, 8, -12, -16] >>> list(convolve(data, [1, -2, 1])) [20, 0, -36, 24, -20, 20, -20, -4, 16] + >>> # Verify signal is consumed lazily and the kernel greedily + >>> signal_iterator = iter([10, 20, 30, 40, 50]) + >>> kernel_iterator = iter([1, 2, 3]) + >>> output_iterator = convolve(signal_iterator, kernel_iterator) + >>> list(kernel_iterator) + [] + >>> next(output_iterator) + 10 + >>> next(output_iterator) + 40 + >>> list(signal_iterator) + [30, 40, 50] + >>> from fractions import Fraction >>> from decimal import Decimal - >>> polynomial_eval([1, -4, -17, 60], x=2) - 18 - >>> x = 2; x**3 - 4*x**2 -17*x + 60 - 18 + >>> polynomial_eval([1, -4, -17, 60], x=5) + 0 + >>> x = 5; x**3 - 4*x**2 -17*x + 60 + 0 >>> polynomial_eval([1, -4, -17, 60], x=2.5) 8.125 >>> x = 2.5; x**3 - 4*x**2 -17*x + 60 @@ -1336,6 +1367,7 @@ The following recipes have a more mathematical flavor: >>> polynomial_eval([11, 2], 7) == 11 * 7 + 2 True + >>> polynomial_from_roots([5, -4, 3]) [1, -4, -17, 60] >>> factored = lambda x: (x - 5) * (x + 4) * (x - 3) @@ -1343,9 +1375,11 @@ The following recipes have a more mathematical flavor: >>> all(factored(x) == expanded(x) for x in range(-10, 11)) True + >>> polynomial_derivative([1, -4, -17, 60]) [3, -8, -17] + >>> list(iter_index('AABCADEAF', 'A')) [0, 1, 4, 7] >>> list(iter_index('AABCADEAF', 'B')) @@ -1391,6 +1425,35 @@ The following recipes have a more mathematical flavor: >>> # Test list input. Lists do not support None for the stop argument >>> list(iter_index(list('AABCADEAF'), 'A')) [0, 1, 4, 7] + >>> # Verify that input is consumed lazily + >>> input_iterator = iter('AABCADEAF') + >>> output_iterator = iter_index(input_iterator, 'A') + >>> next(output_iterator) + 0 + >>> next(output_iterator) + 1 + >>> next(output_iterator) + 4 + >>> ''.join(input_iterator) + 'DEAF' + + + >>> # Verify that the target value can be a sequence. + >>> seq = [[10, 20], [30, 40], 30, 40, [30, 40], 50] + >>> target = [30, 40] + >>> list(iter_index(seq, target)) + [1, 4] + + + >>> # Verify faithfulness to type specific index() method behaviors. + >>> # For example, bytes and str perform continuous-subsequence searches + >>> # that do not match the general behavior specified + >>> # in collections.abc.Sequence.index(). + >>> seq = 'abracadabra' + >>> target = 'ab' + >>> list(iter_index(seq, target)) + [0, 7] + >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] @@ -1411,6 +1474,7 @@ The following recipes have a more mathematical flavor: >>> set(sieve(10_000)).isdisjoint(carmichael) True + >>> list(factor(99)) # Code example 1 [3, 3, 11] >>> list(factor(1_000_000_000_000_007)) # Code example 2 @@ -1456,6 +1520,7 @@ The following recipes have a more mathematical flavor: >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True + >>> totient(0) # https://www.wolframalpha.com/input?i=totient+0 0 >>> first_totients = [1, 1, 2, 2, 4, 2, 6, 4, 6, 4, 10, 4, 12, 6, 8, 8, 16, 6, @@ -1475,9 +1540,15 @@ The following recipes have a more mathematical flavor: >>> totient(6 ** 20) == 1 * 2**19 * 2 * 3**19 # repeated primes True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] + + >>> list(repeatfunc(pow, 5, 2, 3)) + [8, 8, 8, 8, 8] + >>> take(5, map(int, repeatfunc(random.random))) + [0, 0, 0, 0, 0] >>> random.seed(85753098575309) >>> list(repeatfunc(random.random, 3)) [0.16370491282496968, 0.45889608687313455, 0.3747076837820118] @@ -1486,9 +1557,11 @@ The following recipes have a more mathematical flavor: >>> list(repeatfunc(pow, 3, 2, 5)) [32, 32, 32] + >>> list(grouper('abcdefg', 3, fillvalue='x')) [('a', 'b', 'c'), ('d', 'e', 'f'), ('g', 'x', 'x')] + >>> it = grouper('abcdefg', 3, incomplete='strict') >>> next(it) ('a', 'b', 'c') @@ -1502,6 +1575,7 @@ The following recipes have a more mathematical flavor: >>> list(grouper('abcdefg', n=3, incomplete='ignore')) [('a', 'b', 'c'), ('d', 'e', 'f')] + >>> list(sliding_window('ABCDEFG', 1)) [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)] >>> list(sliding_window('ABCDEFG', 2)) @@ -1531,43 +1605,47 @@ The following recipes have a more mathematical flavor: ... 'zero or negative n not supported' + >>> list(roundrobin('abc', 'd', 'ef')) ['a', 'd', 'e', 'b', 'f', 'c'] + >>> ranges = [range(5, 1000), range(4, 3000), range(0), range(3, 2000), range(2, 5000), range(1, 3500)] + >>> collections.Counter(roundrobin(*ranges)) == collections.Counter(chain(*ranges)) + True + >>> # Verify that the inputs are consumed lazily + >>> input_iterators = list(map(iter, ['abcd', 'ef', '', 'ghijk', 'l', 'mnopqr'])) + >>> output_iterator = roundrobin(*input_iterators) + >>> ''.join(islice(output_iterator, 10)) + 'aeglmbfhnc' + >>> ''.join(chain(*input_iterators)) + 'dijkopqr' - >>> def is_odd(x): - ... return x % 2 == 1 - - >>> evens, odds = partition(is_odd, range(10)) - >>> list(evens) - [0, 2, 4, 6, 8] - >>> list(odds) - [1, 3, 5, 7, 9] - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) - 'dEfGhI' >>> list(subslices('ABCD')) ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] + >>> list(powerset([1,2,3])) [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] - >>> all(len(list(powerset(range(n)))) == 2**n for n in range(18)) True - >>> list(powerset('abcde')) == sorted(sorted(set(powerset('abcde'))), key=len) True + >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] >>> list(unique_everseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'D'] >>> list(unique_everseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'D'] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter('AAAABBBCCDAABBB') + >>> output_iterator = unique_everseen(input_iterator) + >>> next(output_iterator) + 'A' + >>> ''.join(input_iterator) + 'AAABBBCCDAABBB' + >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] @@ -1575,6 +1653,22 @@ The following recipes have a more mathematical flavor: ['A', 'B', 'C', 'A', 'D'] >>> list(unique_justseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'A', 'D'] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter('AAAABBBCCDAABBB') + >>> output_iterator = unique_justseen(input_iterator) + >>> next(output_iterator) + 'A' + >>> ''.join(input_iterator) + 'AAABBBCCDAABBB' + + + >>> list(unique([[1, 2], [3, 4], [1, 2]])) + [[1, 2], [3, 4]] + >>> list(unique('ABBcCAD', str.casefold)) + ['A', 'B', 'c', 'D'] + >>> list(unique('ABBcCAD', str.casefold, reverse=True)) + ['D', 'c', 'B', 'A'] + >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) @@ -1593,8 +1687,15 @@ The following recipes have a more mathematical flavor: >>> next(it, 'empty') 'empty' + >>> first_true('ABC0DEF1', '9', str.isdigit) '0' + >>> # Verify that inputs are consumed lazily + >>> it = iter('ABC0DEF1') + >>> first_true(it, predicate=str.isdigit) + '0' + >>> ''.join(it) + 'DEF1' .. testcode:: @@ -1618,7 +1719,7 @@ The following recipes have a more mathematical flavor: def triplewise(iterable): "Return overlapping triplets from an iterable" - # triplewise('ABCDEFG') --> ABC BCD CDE DEF EFG + # triplewise('ABCDEFG') → ABC BCD CDE DEF EFG for (a, _), (b, c) in pairwise(pairwise(iterable)): yield a, b, c @@ -1640,6 +1741,43 @@ The following recipes have a more mathematical flavor: result.append(pool[-1-n]) return tuple(result) + def before_and_after(predicate, it): + """ Variant of takewhile() that allows complete + access to the remainder of the iterator. + + >>> it = iter('ABCdEfGhI') + >>> all_upper, remainder = before_and_after(str.isupper, it) + >>> ''.join(all_upper) + 'ABC' + >>> ''.join(remainder) # takewhile() would lose the 'd' + 'dEfGhI' + + Note that the true iterator must be fully consumed + before the remainder iterator can generate valid results. + """ + it = iter(it) + transition = [] + + def true_iterator(): + for elem in it: + if predicate(elem): + yield elem + else: + transition.append(elem) + return + + return true_iterator(), chain(transition, it) + + def partition(predicate, iterable): + """Partition entries into false entries and true entries. + + If *predicate* is slow, consider wrapping it with functools.lru_cache(). + """ + # partition(is_odd, range(10)) → 0 2 4 6 8 and 1 3 5 7 9 + t1, t2 = tee(iterable) + return filterfalse(predicate, t1), filter(predicate, t2) + + .. doctest:: :hide: @@ -1647,15 +1785,19 @@ The following recipes have a more mathematical flavor: >>> dotproduct([1,2,3], [4,5,6]) 32 + >>> sumprod([1,2,3], [4,5,6]) 32 + >>> list(islice(pad_none('abc'), 0, 6)) ['a', 'b', 'c', None, None, None] + >>> list(triplewise('ABCDEFG')) [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] + >>> population = 'ABCDEFGH' >>> for r in range(len(population) + 1): ... seq = list(combinations(population, r)) @@ -1663,9 +1805,38 @@ The following recipes have a more mathematical flavor: ... assert nth_combination(population, r, i) == seq[i] ... for i in range(-len(seq), 0): ... assert nth_combination(population, r, i) == seq[i] - + ... >>> iterable = 'abcde' >>> r = 3 >>> combos = list(combinations(iterable, r)) >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) True + + + >>> it = iter('ABCdEfGhI') + >>> all_upper, remainder = before_and_after(str.isupper, it) + >>> ''.join(all_upper) + 'ABC' + >>> ''.join(remainder) + 'dEfGhI' + + + >>> def is_odd(x): + ... return x % 2 == 1 + ... + >>> evens, odds = partition(is_odd, range(10)) + >>> list(evens) + [0, 2, 4, 6, 8] + >>> list(odds) + [1, 3, 5, 7, 9] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter(range(10)) + >>> evens, odds = partition(is_odd, input_iterator) + >>> next(odds) + 1 + >>> next(odds) + 3 + >>> next(evens) + 0 + >>> list(input_iterator) + [4, 5, 6, 7, 8, 9] diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 0ce4b697145cb3..758d47462b6e12 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -1,5 +1,5 @@ -:mod:`json` --- JSON encoder and decoder -======================================== +:mod:`!json` --- JSON encoder and decoder +========================================= .. module:: json :synopsis: Encode and decode the JSON format. @@ -13,7 +13,7 @@ `JSON (JavaScript Object Notation) `_, specified by :rfc:`7159` (which obsoletes :rfc:`4627`) and by -`ECMA-404 `_, +`ECMA-404 `_, is a lightweight data interchange format inspired by `JavaScript `_ object literal syntax (although it is not a strict subset of JavaScript [#rfc-errata]_ ). @@ -106,7 +106,7 @@ Extending :class:`JSONEncoder`:: ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... # Let the base class default method raise the TypeError - ... return json.JSONEncoder.default(self, obj) + ... return super().default(obj) ... >>> json.dumps(2 + 1j, cls=ComplexEncoder) '[2.0, 1.0]' @@ -116,15 +116,15 @@ Extending :class:`JSONEncoder`:: ['[2.0', ', 1.0', ']'] -Using :mod:`json.tool` from the shell to validate and pretty-print: +Using :mod:`json` from the shell to validate and pretty-print: .. code-block:: shell-session - $ echo '{"json":"obj"}' | python -m json.tool + $ echo '{"json":"obj"}' | python -m json { "json": "obj" } - $ echo '{1.2:3.4}' | python -m json.tool + $ echo '{1.2:3.4}' | python -m json Expecting property name enclosed in double quotes: line 1 column 2 (char 1) See :ref:`json-commandline` for detailed documentation. @@ -241,28 +241,28 @@ Basic Usage *object_hook* is an optional function that will be called with the result of any object literal decoded (a :class:`dict`). The return value of - *object_hook* will be used instead of the :class:`dict`. This feature can be used - to implement custom decoders (e.g. `JSON-RPC `_ - class hinting). + *object_hook* will be used instead of the :class:`dict`. This feature can + be used to implement custom decoders (e.g. `JSON-RPC + `_ class hinting). *object_pairs_hook* is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders. - If *object_hook* is also defined, the *object_pairs_hook* takes priority. + :class:`dict`. This feature can be used to implement custom decoders. If + *object_hook* is also defined, the *object_pairs_hook* takes priority. .. versionchanged:: 3.1 Added support for *object_pairs_hook*. - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). + *parse_float* is an optional function that will be called with the string of + every JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser for + JSON floats (e.g. :class:`decimal.Decimal`). - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). + *parse_int* is an optional function that will be called with the string of + every JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser for + JSON integers (e.g. :class:`float`). .. versionchanged:: 3.11 The default *parse_int* of :func:`int` now limits the maximum length of @@ -270,10 +270,9 @@ Basic Usage conversion length limitation ` to help avoid denial of service attacks. - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. + *parse_constant* is an optional function that will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This can be + used to raise an exception if invalid JSON numbers are encountered. .. versionchanged:: 3.1 *parse_constant* doesn't get called on 'null', 'true', 'false' anymore. @@ -345,34 +344,33 @@ Encoders and Decoders It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their corresponding ``float`` values, which is outside the JSON spec. - *object_hook*, if specified, will be called with the result of every JSON - object decoded and its return value will be used in place of the given - :class:`dict`. This can be used to provide custom deserializations (e.g. to - support `JSON-RPC `_ class hinting). + *object_hook* is an optional function that will be called with the result of + every JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom deserializations + (e.g. to support `JSON-RPC `_ class hinting). - *object_pairs_hook*, if specified will be called with the result of every - JSON object decoded with an ordered list of pairs. The return value of - *object_pairs_hook* will be used instead of the :class:`dict`. This - feature can be used to implement custom decoders. If *object_hook* is also - defined, the *object_pairs_hook* takes priority. + *object_pairs_hook* is an optional function that will be called with the + result of every JSON object decoded with an ordered list of pairs. The + return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders. If + *object_hook* is also defined, the *object_pairs_hook* takes priority. .. versionchanged:: 3.1 Added support for *object_pairs_hook*. - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). + *parse_float* is an optional function that will be called with the string of + every JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser for + JSON floats (e.g. :class:`decimal.Decimal`). - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). + *parse_int* is an optional function that will be called with the string of + every JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser for + JSON integers (e.g. :class:`float`). - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. + *parse_constant* is an optional function that will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This can be + used to raise an exception if invalid JSON numbers are encountered. If *strict* is false (``True`` is the default), then control characters will be allowed inside strings. Control characters in this context are @@ -504,7 +502,7 @@ Encoders and Decoders else: return list(iterable) # Let the base class default method raise the TypeError - return json.JSONEncoder.default(self, o) + return super().default(o) .. method:: encode(o) @@ -559,7 +557,7 @@ Standard Compliance and Interoperability ---------------------------------------- The JSON format is specified by :rfc:`7159` and by -`ECMA-404 `_. +`ECMA-404 `_. This section details this module's level of compliance with the RFC. For simplicity, :class:`JSONEncoder` and :class:`JSONDecoder` subclasses, and parameters other than those explicitly mentioned, are not considered. @@ -678,31 +676,32 @@ when serializing instances of "exotic" numerical types such as .. _json-commandline: -.. program:: json.tool +.. program:: json -Command Line Interface +Command-line interface ---------------------- .. module:: json.tool - :synopsis: A command line to validate and pretty-print JSON. + :synopsis: A command-line interface to validate and pretty-print JSON. **Source code:** :source:`Lib/json/tool.py` -------------- -The :mod:`json.tool` module provides a simple command line interface to validate -and pretty-print JSON objects. +The :mod:`json` module can be invoked as a script via ``python -m json`` +to validate and pretty-print JSON objects. The :mod:`json.tool` submodule +implements this interface. If the optional ``infile`` and ``outfile`` arguments are not specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: .. code-block:: shell-session - $ echo '{"json": "obj"}' | python -m json.tool + $ echo '{"json": "obj"}' | python -m json { "json": "obj" } - $ echo '{1.2:3.4}' | python -m json.tool + $ echo '{1.2:3.4}' | python -m json Expecting property name enclosed in double quotes: line 1 column 2 (char 1) .. versionchanged:: 3.5 @@ -710,8 +709,13 @@ specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: :option:`--sort-keys` option to sort the output of dictionaries alphabetically by key. +.. versionchanged:: 3.14 + The :mod:`json` module may now be directly executed as + ``python -m json``. For backwards compatibility, invoking + the CLI as ``python -m json.tool`` remains supported. + -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: infile @@ -720,7 +724,7 @@ Command line options .. code-block:: shell-session - $ python -m json.tool mp_films.json + $ python -m json mp_films.json [ { "title": "And Now for Something Completely Different", diff --git a/Doc/library/keyword.rst b/Doc/library/keyword.rst index c3b4699cb05af6..ac57140f888024 100644 --- a/Doc/library/keyword.rst +++ b/Doc/library/keyword.rst @@ -1,5 +1,5 @@ -:mod:`keyword` --- Testing for Python keywords -============================================== +:mod:`!keyword` --- Testing for Python keywords +=============================================== .. module:: keyword :synopsis: Test whether a string is a keyword in Python. diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index dd9f4ee45ba82e..88c6079a05b7fa 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -1,5 +1,5 @@ -:mod:`linecache` --- Random access to text lines -================================================ +:mod:`!linecache` --- Random access to text lines +================================================= .. module:: linecache :synopsis: Provides random access to individual lines from text files. diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 414979524e57b6..f172a55080efc9 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -1,5 +1,5 @@ -:mod:`locale` --- Internationalization services -=============================================== +:mod:`!locale` --- Internationalization services +================================================ .. module:: locale :synopsis: Internationalization services. @@ -311,8 +311,18 @@ The :mod:`locale` module defines the following exception and functions: .. data:: ALT_DIGITS - Get a representation of up to 100 values used to represent the values - 0 to 99. + Get a string consisting of up to 100 semicolon-separated symbols used + to represent the values 0 to 99 in a locale-specific way. + In most locales this is an empty string. + + The function temporarily sets the ``LC_CTYPE`` locale to the locale + of the category that determines the requested value (``LC_TIME``, + ``LC_NUMERIC``, ``LC_MONETARY`` or ``LC_MESSAGES``) if locales are + different and the resulting string is non-ASCII. + This temporary change affects other threads. + + .. versionchanged:: 3.14 + The function now temporarily sets the ``LC_CTYPE`` locale in some cases. .. function:: getdefaultlocale([envvars]) @@ -424,7 +434,7 @@ The :mod:`locale` module defines the following exception and functions: .. function:: format_string(format, val, grouping=False, monetary=False) Formats a number *val* according to the current :const:`LC_NUMERIC` setting. - The format follows the conventions of the ``%`` operator. For floating point + The format follows the conventions of the ``%`` operator. For floating-point values, the decimal point is modified if appropriate. If *grouping* is ``True``, also takes the grouping into account. @@ -455,7 +465,7 @@ The :mod:`locale` module defines the following exception and functions: .. function:: str(float) - Formats a floating point number using the same format as the built-in function + Formats a floating-point number using the same format as the built-in function ``str(float)``, but takes the decimal point into account. diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 13850c91446da5..0e9dc33ae2123a 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -1,5 +1,5 @@ -:mod:`logging.config` --- Logging configuration -=============================================== +:mod:`!logging.config` --- Logging configuration +================================================ .. module:: logging.config :synopsis: Configuration of the logging module. @@ -69,7 +69,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in dictConfigClass(config).configure() For example, a subclass of :class:`DictConfigurator` could call - ``DictConfigurator.__init__()`` in its own :meth:`__init__()`, then + ``DictConfigurator.__init__()`` in its own :meth:`__init__`, then set up custom prefixes which would be usable in the subsequent :meth:`configure` call. :attr:`dictConfigClass` would be bound to this new subclass, and then :func:`dictConfig` could be called exactly as @@ -753,13 +753,17 @@ The ``queue`` and ``listener`` keys are optional. If the ``queue`` key is present, the corresponding value can be one of the following: -* An actual instance of :class:`queue.Queue` or a subclass thereof. This is of course - only possible if you are constructing or modifying the configuration dictionary in - code. +* An object implementing the :meth:`Queue.put_nowait ` + and :meth:`Queue.get ` public API. For instance, this may be + an actual instance of :class:`queue.Queue` or a subclass thereof, or a proxy + obtained by :meth:`multiprocessing.managers.SyncManager.Queue`. + + This is of course only possible if you are constructing or modifying + the configuration dictionary in code. * A string that resolves to a callable which, when called with no arguments, returns - the :class:`queue.Queue` instance to use. That callable could be a - :class:`queue.Queue` subclass or a function which returns a suitable queue instance, + the queue instance to use. That callable could be a :class:`queue.Queue` subclass + or a function which returns a suitable queue instance, such as ``my.module.queue_factory()``. * A dict with a ``'()'`` key which is constructed in the usual way as discussed in diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 2fe9370333beaf..5a081f9e7add99 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -1,5 +1,5 @@ -:mod:`logging.handlers` --- Logging handlers -============================================ +:mod:`!logging.handlers` --- Logging handlers +============================================= .. module:: logging.handlers :synopsis: Handlers for the logging module. @@ -66,7 +66,7 @@ and :meth:`flush` methods). :param stream: The stream that the handler should use. - :return: the old stream, if the stream was changed, or *None* if it wasn't. + :return: the old stream, if the stream was changed, or ``None`` if it wasn't. .. versionadded:: 3.7 diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 39eb41ce1f1670..235bcc281ac8f8 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1,5 +1,5 @@ -:mod:`logging` --- Logging facility for Python -============================================== +:mod:`!logging` --- Logging facility for Python +=============================================== .. module:: logging :synopsis: Flexible event logging system for applications. @@ -30,20 +30,61 @@ is that all Python modules can participate in logging, so your application log can include your own messages integrated with messages from third-party modules. -The simplest example: +Here's a simple example of idiomatic usage: :: + + # myapp.py + import logging + import mylib + logger = logging.getLogger(__name__) + + def main(): + logging.basicConfig(filename='myapp.log', level=logging.INFO) + logger.info('Started') + mylib.do_something() + logger.info('Finished') + + if __name__ == '__main__': + main() + +:: + + # mylib.py + import logging + logger = logging.getLogger(__name__) + + def do_something(): + logger.info('Doing something') + +If you run *myapp.py*, you should see this in *myapp.log*: .. code-block:: none - >>> import logging - >>> logging.warning('Watch out!') - WARNING:root:Watch out! + INFO:__main__:Started + INFO:mylib:Doing something + INFO:__main__:Finished + +The key feature of this idiomatic usage is that the majority of code is simply +creating a module level logger with ``getLogger(__name__)``, and using that +logger to do any needed logging. This is concise, while allowing downstream +code fine-grained control if needed. Logged messages to the module-level logger +get forwarded to handlers of loggers in higher-level modules, all the way up to +the highest-level logger known as the root logger; this approach is known as +hierarchical logging. + +For logging to be useful, it needs to be configured: setting the levels and +destinations for each logger, potentially changing how specific modules log, +often based on command-line arguments or application configuration. In most +cases, like the one above, only the root logger needs to be so configured, since +all the lower level loggers at module level eventually forward their messages to +its handlers. :func:`~logging.basicConfig` provides a quick way to configure +the root logger that handles many use cases. The module provides a lot of functionality and flexibility. If you are unfamiliar with logging, the best way to get to grips with it is to view the tutorials (**see the links above and on the right**). -The basic classes defined by the module, together with their functions, are -listed below. +The basic classes defined by the module, together with their attributes and +methods, are listed in the sections below. * Loggers expose the interface that application code directly uses. * Handlers send the log records (created by loggers) to the appropriate @@ -68,15 +109,36 @@ The ``name`` is potentially a period-separated hierarchical value, like Loggers that are further down in the hierarchical list are children of loggers higher up in the list. For example, given a logger with a name of ``foo``, loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. +descendants of ``foo``. In addition, all loggers are descendants of the root +logger. The logger name hierarchy is analogous to the Python package hierarchy, +and identical to it if you organise your loggers on a per-module basis using +the recommended construction ``logging.getLogger(__name__)``. That's because +in a module, ``__name__`` is the module's name in the Python package namespace. .. class:: Logger + .. attribute:: Logger.name + + This is the logger's name, and is the value that was passed to :func:`getLogger` + to obtain the logger. + + .. note:: This attribute should be treated as read-only. + + .. attribute:: Logger.level + + The threshold of this logger, as set by the :meth:`setLevel` method. + + .. note:: Do not set this attribute directly - always use :meth:`setLevel`, + which has checks for the level passed to it. + + .. attribute:: Logger.parent + + The parent logger of this logger. It may change based on later instantiation + of loggers which are higher up in the namespace hierarchy. + + .. note:: This value should be treated as read-only. + .. attribute:: Logger.propagate If this attribute evaluates to true, events logged to this logger will be @@ -108,6 +170,21 @@ is the module's name in the Python package namespace. scenario is to attach handlers only to the root logger, and to let propagation take care of the rest. + .. attribute:: Logger.handlers + + The list of handlers directly attached to this logger instance. + + .. note:: This attribute should be treated as read-only; it is normally changed via + the :meth:`addHandler` and :meth:`removeHandler` methods, which use locks to ensure + thread-safe operation. + + .. attribute:: Logger.disabled + + This attribute disables handling of any events. It is set to ``False`` in the + initializer, and only changed by logging configuration code. + + .. note:: This attribute should be treated as read-only. + .. method:: Logger.setLevel(level) Sets the threshold for this logger to *level*. Logging messages which are less @@ -227,7 +304,8 @@ is the module's name in the Python package namespace. parameter mirrors the equivalent one in the :mod:`warnings` module. The fourth keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the :class:`LogRecord` + dictionary which is used to populate the :attr:`~object.__dict__` of the + :class:`LogRecord` created for the logging event with user-defined attributes. These custom attributes can then be used as you like. For example, they could be incorporated into logged messages. For example:: @@ -275,10 +353,6 @@ is the module's name in the Python package namespace. .. versionchanged:: 3.8 The *stacklevel* parameter was added. - .. versionchanged:: 3.13 - Remove the undocumented ``warn()`` method which was an alias to the - :meth:`warning` method. - .. method:: Logger.info(msg, *args, **kwargs) @@ -291,6 +365,10 @@ is the module's name in the Python package namespace. Logs a message with level :const:`WARNING` on this logger. The arguments are interpreted as for :meth:`debug`. + .. note:: There is an obsolete method ``warn`` which is functionally + identical to ``warning``. As ``warn`` is deprecated, please do not use + it - use ``warning`` instead. + .. method:: Logger.error(msg, *args, **kwargs) Logs a message with level :const:`ERROR` on this logger. The arguments are @@ -926,7 +1004,7 @@ the options available to you. | | | portion of the time). | +----------------+-------------------------+-----------------------------------------------+ | created | ``%(created)f`` | Time when the :class:`LogRecord` was created | -| | | (as returned by :func:`time.time`). | +| | | (as returned by :func:`time.time_ns` / 1e9). | +----------------+-------------------------+-----------------------------------------------+ | exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | | | format this yourself. | if no exception has occurred, ``None``. | @@ -1021,11 +1099,11 @@ information into logging calls. For a usage example, see the section on .. attribute:: manager - Delegates to the underlying :attr:`!manager`` on *logger*. + Delegates to the underlying :attr:`!manager` on *logger*. .. attribute:: _log - Delegates to the underlying :meth:`!_log`` method on *logger*. + Delegates to the underlying :meth:`!_log` method on *logger*. In addition to the above, :class:`LoggerAdapter` supports the following methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, @@ -1047,11 +1125,6 @@ information into logging calls. For a usage example, see the section on Attribute :attr:`!manager` and method :meth:`!_log` were added, which delegate to the underlying logger and allow adapters to be nested. - .. versionchanged:: 3.13 - - Remove the undocumented :meth:`!warn`` method which was an alias to the - :meth:`!warning` method. - .. versionchanged:: 3.13 The *merge_extra* argument was added. @@ -1080,10 +1153,12 @@ functions. .. function:: getLogger(name=None) - Return a logger with the specified name or, if name is ``None``, return a - logger which is the root logger of the hierarchy. If specified, the name is - typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. - Choice of these names is entirely up to the developer who is using logging. + Return a logger with the specified name or, if name is ``None``, return the + root logger of the hierarchy. If specified, the name is typically a + dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. Choice + of these names is entirely up to the developer who is using logging, though + it is recommended that ``__name__`` be used unless you have a specific + reason for not doing that, as mentioned in :ref:`logger`. All calls to this function with a given name return the same logger instance. This means that logger instances never need to be passed between different parts @@ -1115,121 +1190,59 @@ functions. .. function:: debug(msg, *args, **kwargs) - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. + This is a convenience function that calls :meth:`Logger.debug`, on the root + logger. The handling of the arguments is in every way identical + to what is described in that method. - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: + The only difference is that if the root logger has no handlers, then + :func:`basicConfig` is called, prior to calling ``debug`` on the root logger. - FORMAT = '%(asctime)s %(clientip)-15s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) + For very short scripts or quick demonstrations of ``logging`` facilities, + ``debug`` and the other module-level functions may be convenient. However, + most programs will want to carefully and explicitly control the logging + configuration, and should therefore prefer creating a module-level logger and + calling :meth:`Logger.debug` (or other level-specific methods) on it, as + described at the beginning of this documentation. - would print something like: - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - This function (as well as :func:`info`, :func:`warning`, :func:`error` and - :func:`critical`) will call :func:`basicConfig` if the root logger doesn't - have any handler attached. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. .. function:: info(msg, *args, **kwargs) - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`INFO` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: warning(msg, *args, **kwargs) - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`WARNING` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. note:: There is an obsolete function ``warn`` which is functionally identical to ``warning``. As ``warn`` is deprecated, please do not use it - use ``warning`` instead. - .. versionchanged:: 3.13 - Remove the undocumented ``warn()`` function which was an alias to the - :func:`warning` function. - .. function:: error(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: critical(msg, *args, **kwargs) - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`CRITICAL` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: exception(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. Exception info is added to the logging message. This function should only be called from an exception handler. .. function:: log(level, msg, *args, **kwargs) - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. + Logs a message with level *level* on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: disable(level=CRITICAL) diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 0d69c3bc01d1e2..69f7cb8d48d7ae 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -1,5 +1,5 @@ -:mod:`lzma` --- Compression using the LZMA algorithm -==================================================== +:mod:`!lzma` --- Compression using the LZMA algorithm +===================================================== .. module:: lzma :synopsis: A Python wrapper for the liblzma compression library. @@ -104,7 +104,7 @@ Reading and writing compressed files and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - The following method is also provided: + The following method and attributes are also provided: .. method:: peek(size=-1) @@ -117,6 +117,20 @@ Reading and writing compressed files file object (e.g. if the :class:`LZMAFile` was constructed by passing a file object for *filename*). + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionadded:: 3.13 + + .. attribute:: name + + The lzma file name. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + .. versionadded:: 3.13 + + .. versionchanged:: 3.4 Added support for the ``"x"`` and ``"xb"`` modes. diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index a613548c9e518e..abb32f9bf3457f 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -1,5 +1,5 @@ -:mod:`mailbox` --- Manipulate mailboxes in various formats -========================================================== +:mod:`!mailbox` --- Manipulate mailboxes in various formats +=========================================================== .. module:: mailbox :synopsis: Manipulate mailboxes in various formats @@ -1387,7 +1387,7 @@ When an :class:`!MHMessage` instance is created based upon a .. method:: get_visible() - Return an :class:`Message` instance whose headers are the message's + Return a :class:`Message` instance whose headers are the message's visible headers and whose body is empty. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index c6a006b7b4028a..9e4606df0f774e 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -1,5 +1,5 @@ -:mod:`marshal` --- Internal Python object serialization -======================================================= +:mod:`!marshal` --- Internal Python object serialization +======================================================== .. module:: marshal :synopsis: Convert Python objects to streams of bytes and back (with different @@ -10,7 +10,7 @@ This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, -transport the file to a Sun, and read it back there). Details of the format are +transport the file to a Mac, and read it back there). Details of the format are undocumented on purpose; it may change between Python versions (although it rarely does). [#]_ @@ -42,8 +42,8 @@ supports a substantially wider range of objects than marshal. Not all Python object types are supported; in general, only objects whose value is independent from a particular invocation of Python can be written and read by -this module. The following types are supported: booleans, integers, floating -point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, +this module. The following types are supported: booleans, integers, floating-point +numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, frozensets, dictionaries, and code objects (if *allow_code* is true), where it should be understood that tuples, lists, sets, frozensets and dictionaries are only supported as long as @@ -142,7 +142,7 @@ In addition, the following constants are defined: Indicates the format that the module uses. Version 0 is the historical format, version 1 shares interned strings and version 2 uses a binary format - for floating point numbers. + for floating-point numbers. Version 3 adds support for object instancing and recursion. The current version is 4. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 9caf7230eed0aa..6be61c99274eb7 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -1,5 +1,5 @@ -:mod:`math` --- Mathematical functions -====================================== +:mod:`!math` --- Mathematical functions +======================================= .. module:: math :synopsis: Mathematical functions (sin() etc.). @@ -26,6 +26,92 @@ The following functions are provided by this module. Except when explicitly noted otherwise, all return values are floats. +==================================================== ============================================ +**Number-theoretic and representation functions** +-------------------------------------------------------------------------------------------------- +:func:`ceil(x) ` Ceiling of *x*, the smallest integer greater than or equal to *x* +:func:`comb(n, k) ` Number of ways to choose *k* items from *n* items without repetition and without order +:func:`copysign(x, y) ` Magnitude (absolute value) of *x* with the sign of *y* +:func:`fabs(x) ` Absolute value of *x* +:func:`factorial(n) ` *n* factorial +:func:`floor (x) ` Floor of *x*, the largest integer less than or equal to *x* +:func:`fma(x, y, z) ` Fused multiply-add operation: ``(x * y) + z`` +:func:`fmod(x, y) ` Remainder of division ``x / y`` +:func:`frexp(x) ` Mantissa and exponent of *x* +:func:`fsum(iterable) ` Sum of values in the input *iterable* +:func:`gcd(*integers) ` Greatest common divisor of the integer arguments +:func:`isclose(a, b, rel_tol, abs_tol) ` Check if the values *a* and *b* are close to each other +:func:`isfinite(x) ` Check if *x* is neither an infinity nor a NaN +:func:`isinf(x) ` Check if *x* is a positive or negative infinity +:func:`isnan(x) ` Check if *x* is a NaN (not a number) +:func:`isqrt(n) ` Integer square root of a nonnegative integer *n* +:func:`lcm(*integers) ` Least common multiple of the integer arguments +:func:`ldexp(x, i) ` ``x * (2**i)``, inverse of function :func:`frexp` +:func:`modf(x) ` Fractional and integer parts of *x* +:func:`nextafter(x, y, steps) ` Floating-point value *steps* steps after *x* towards *y* +:func:`perm(n, k) ` Number of ways to choose *k* items from *n* items without repetition and with order +:func:`prod(iterable, start) ` Product of elements in the input *iterable* with a *start* value +:func:`remainder(x, y) ` Remainder of *x* with respect to *y* +:func:`sumprod(p, q) ` Sum of products from two iterables *p* and *q* +:func:`trunc(x) ` Integer part of *x* +:func:`ulp(x) ` Value of the least significant bit of *x* + +**Power and logarithmic functions** +-------------------------------------------------------------------------------------------------- +:func:`cbrt(x) ` Cube root of *x* +:func:`exp(x) ` *e* raised to the power *x* +:func:`exp2(x) ` *2* raised to the power *x* +:func:`expm1(x) ` *e* raised to the power *x*, minus 1 +:func:`log(x, base) ` Logarithm of *x* to the given base (*e* by default) +:func:`log1p(x) ` Natural logarithm of *1+x* (base *e*) +:func:`log2(x) ` Base-2 logarithm of *x* +:func:`log10(x) ` Base-10 logarithm of *x* +:func:`pow(x, y) ` *x* raised to the power *y* +:func:`sqrt(x) ` Square root of *x* + +**Trigonometric functions** +-------------------------------------------------------------------------------------------------- +:func:`acos(x) ` Arc cosine of *x* +:func:`asin(x) ` Arc sine of *x* +:func:`atan(x) ` Arc tangent of *x* +:func:`atan2(y, x) ` ``atan(y / x)`` +:func:`cos(x) ` Cosine of *x* +:func:`dist(p, q) ` Euclidean distance between two points *p* and *q* given as an iterable of coordinates +:func:`hypot(*coordinates) ` Euclidean norm of an iterable of coordinates +:func:`sin(x) ` Sine of *x* +:func:`tan(x) ` Tangent of *x* + +**Angular conversion** +-------------------------------------------------------------------------------------------------- +:func:`degrees(x) ` Convert angle *x* from radians to degrees +:func:`radians(x) ` Convert angle *x* from degrees to radians + +**Hyperbolic functions** +-------------------------------------------------------------------------------------------------- +:func:`acosh(x) ` Inverse hyperbolic cosine of *x* +:func:`asinh(x) ` Inverse hyperbolic sine of *x* +:func:`atanh(x) ` Inverse hyperbolic tangent of *x* +:func:`cosh(x) ` Hyperbolic cosine of *x* +:func:`sinh(x) ` Hyperbolic sine of *x* +:func:`tanh(x) ` Hyperbolic tangent of *x* + +**Special functions** +-------------------------------------------------------------------------------------------------- +:func:`erf(x) ` `Error function `_ at *x* +:func:`erfc(x) ` `Complementary error function `_ at *x* +:func:`gamma(x) ` `Gamma function `_ at *x* +:func:`lgamma(x) ` Natural logarithm of the absolute value of the `Gamma function `_ at *x* + +**Constants** +-------------------------------------------------------------------------------------------------- +:data:`pi` *π* = 3.141592... +:data:`e` *e* = 2.718281... +:data:`tau` *τ* = 2\ *π* = 6.283185... +:data:`inf` Positive infinity +:data:`nan` "Not a number" (NaN) +==================================================== ============================================ + + Number-theoretic and representation functions --------------------------------------------- @@ -82,6 +168,22 @@ Number-theoretic and representation functions should return an :class:`~numbers.Integral` value. +.. function:: fma(x, y, z) + + Fused multiply-add operation. Return ``(x * y) + z``, computed as though with + infinite precision and range followed by a single round to the ``float`` + format. This operation often provides better accuracy than the direct + expression ``(x * y) + z``. + + This function follows the specification of the fusedMultiplyAdd operation + described in the IEEE 754 standard. The standard leaves one case + implementation-defined, namely the result of ``fma(0, inf, nan)`` + and ``fma(inf, 0, nan)``. In these cases, ``math.fma`` returns a NaN, + and does not raise any exception. + + .. versionadded:: 3.13 + + .. function:: fmod(x, y) Return ``fmod(x, y)``, as defined by the platform C library. Note that the @@ -107,7 +209,7 @@ Number-theoretic and representation functions .. function:: fsum(iterable) - Return an accurate floating point sum of values in the iterable. Avoids + Return an accurate floating-point sum of values in the iterable. Avoids loss of precision by tracking multiple intermediate partial sums. The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the @@ -117,8 +219,8 @@ Number-theoretic and representation functions least significant bit. For further discussion and two alternative approaches, see the `ASPN cookbook - recipes for accurate floating point summation - `_\. + recipes for accurate floating-point summation + `_\. .. function:: gcd(*integers) @@ -239,11 +341,11 @@ Number-theoretic and representation functions See also :func:`math.ulp`. + .. versionadded:: 3.9 + .. versionchanged:: 3.12 Added the *steps* argument. - .. versionadded:: 3.9 - .. function:: perm(n, k=None) Return the number of ways to choose *k* items from *n* items @@ -252,7 +354,7 @@ Number-theoretic and representation functions Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates to zero when ``k > n``. - If *k* is not specified or is None, then *k* defaults to *n* + If *k* is not specified or is ``None``, then *k* defaults to *n* and the function returns ``n!``. Raises :exc:`TypeError` if either of the arguments are not integers. @@ -288,7 +390,7 @@ Number-theoretic and representation functions If the result of the remainder operation is zero, that zero will have the same sign as *x*. - On platforms using IEEE 754 binary floating-point, the result of this + On platforms using IEEE 754 binary floating point, the result of this operation is always exactly representable: no rounding error is introduced. .. versionadded:: 3.7 @@ -431,11 +533,11 @@ Power and logarithmic functions .. function:: pow(x, y) - Return ``x`` raised to the power ``y``. Exceptional cases follow + Return *x* raised to the power *y*. Exceptional cases follow the IEEE 754 standard as far as possible. In particular, ``pow(1.0, x)`` and ``pow(x, 0.0)`` always return ``1.0``, even - when ``x`` is a zero or a NaN. If both ``x`` and ``y`` are finite, - ``x`` is negative, and ``y`` is not an integer then ``pow(x, y)`` + when *x* is a zero or a NaN. If both *x* and *y* are finite, + *x* is negative, and *y* is not an integer then ``pow(x, y)`` is undefined, and raises :exc:`ValueError`. Unlike the built-in ``**`` operator, :func:`math.pow` converts both @@ -592,7 +694,7 @@ Special functions The :func:`erf` function can be used to compute traditional statistical functions such as the `cumulative standard normal distribution - `_:: + `_:: def phi(x): 'Cumulative distribution function for the standard normal distribution' @@ -680,11 +782,11 @@ Constants >>> math.isnan(float('nan')) True + .. versionadded:: 3.5 + .. versionchanged:: 3.11 It is now always available. - .. versionadded:: 3.5 - .. impl-detail:: diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index f610032acbe417..8ad4850584a7e1 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -1,5 +1,5 @@ -:mod:`mimetypes` --- Map filenames to MIME types -================================================ +:mod:`!mimetypes` --- Map filenames to MIME types +================================================= .. module:: mimetypes :synopsis: Mapping of filename extensions to MIME types. @@ -52,7 +52,22 @@ the information :func:`init` sets up. are also recognized. .. versionchanged:: 3.8 - Added support for url being a :term:`path-like object`. + Added support for *url* being a :term:`path-like object`. + + .. deprecated:: 3.13 + Passing a file path instead of URL is :term:`soft deprecated`. + Use :func:`guess_file_type` for this. + + +.. function:: guess_file_type(path, *, strict=True) + + .. index:: pair: MIME; headers + + Guess the type of a file based on its path, given by *path*. + Similar to the :func:`guess_type` function, but accepts a path instead of URL. + Path can be a string, a bytes object or a :term:`path-like object`. + + .. versionadded:: 3.13 .. function:: guess_all_extensions(type, strict=True) @@ -61,7 +76,7 @@ the information :func:`init` sets up. return value is a list of strings giving all possible filename extensions, including the leading dot (``'.'``). The extensions are not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME - type *type* by :func:`guess_type`. + type *type* by :func:`guess_type` and :func:`guess_file_type`. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -72,8 +87,8 @@ the information :func:`init` sets up. return value is a string giving a filename extension, including the leading dot (``'.'``). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME type *type* by - :func:`guess_type`. If no extension can be guessed for *type*, ``None`` is - returned. + :func:`guess_type` and :func:`guess_file_type`. + If no extension can be guessed for *type*, ``None`` is returned. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -238,6 +253,14 @@ than one MIME-type database; it provides an interface similar to the one of the the object. + .. method:: MimeTypes.guess_file_type(path, *, strict=True) + + Similar to the :func:`guess_file_type` function, using the tables stored + as part of the object. + + .. versionadded:: 3.13 + + .. method:: MimeTypes.guess_all_extensions(type, strict=True) Similar to the :func:`guess_all_extensions` function, using the tables stored @@ -272,3 +295,13 @@ than one MIME-type database; it provides an interface similar to the one of the types, else to the list of non-standard types. .. versionadded:: 3.2 + + + .. method:: MimeTypes.add_type(type, ext, strict=True) + + Add a mapping from the MIME type *type* to the extension *ext*. When the + extension is already known, the new type will replace the old one. When the type + is already known the extension will be added to the list of known extensions. + + When *strict* is ``True`` (the default), the mapping will be added to the + official MIME types, otherwise to the non-standard ones. diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 758721433f77de..4e20c07331a220 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -1,5 +1,5 @@ -:mod:`mmap` --- Memory-mapped file support -========================================== +:mod:`!mmap` --- Memory-mapped file support +=========================================== .. module:: mmap :synopsis: Interface to memory-mapped files for Unix and Windows. diff --git a/Doc/library/modulefinder.rst b/Doc/library/modulefinder.rst index 526f0ff868c2b7..823d853f1ed8eb 100644 --- a/Doc/library/modulefinder.rst +++ b/Doc/library/modulefinder.rst @@ -1,5 +1,5 @@ -:mod:`modulefinder` --- Find modules used by a script -===================================================== +:mod:`!modulefinder` --- Find modules used by a script +====================================================== .. module:: modulefinder :synopsis: Find modules used by a script. diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 2a6d980ab78a60..327cc3602b1a77 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -1,5 +1,5 @@ -:mod:`msvcrt` --- Useful routines from the MS VC++ runtime -========================================================== +:mod:`!msvcrt` --- Useful routines from the MS VC++ runtime +=========================================================== .. module:: msvcrt :platform: Windows @@ -211,7 +211,7 @@ Other Functions After you use :func:`CrtSetReportMode` to specify :const:`CRTDBG_MODE_FILE`, you can specify the file handle to receive the message text. *type* must be - one of the :const:`!CRT_\*` constants listed below. *file* shuld be the file + one of the :const:`!CRT_\*` constants listed below. *file* should be the file handle your want specified. Only available in :ref:`debug build of Python `. @@ -252,3 +252,18 @@ Other Functions .. data:: CRTDBG_REPORT_MODE Returns current *mode* for the specified *type*. + + +.. data:: CRT_ASSEMBLY_VERSION + + The CRT Assembly version, from the :file:`crtassem.h` header file. + + +.. data:: VC_ASSEMBLY_PUBLICKEYTOKEN + + The VC Assembly public key token, from the :file:`crtassem.h` header file. + + +.. data:: LIBRARIES_ASSEMBLY_NAME_PREFIX + + The Libraries Assembly name prefix, from the :file:`crtassem.h` header file. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index b104a6483b70e6..036b8f44b9ff3b 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1,5 +1,5 @@ -:mod:`multiprocessing` --- Process-based parallelism -==================================================== +:mod:`!multiprocessing` --- Process-based parallelism +===================================================== .. module:: multiprocessing :synopsis: Process-based parallelism. @@ -8,7 +8,7 @@ -------------- -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-mobile-notavail.rst Introduction ------------ @@ -124,11 +124,11 @@ to start a process. These *start methods* are inherited by the child process. Note that safely forking a multithreaded process is problematic. - Available on POSIX systems. Currently the default on POSIX except macOS. + Available on POSIX systems. - .. note:: - The default start method will change away from *fork* in Python 3.14. - Code that requires *fork* should explicitly specify that via + .. versionchanged:: 3.14 + This is no longer the default start method on any platform. + Code that requires *fork* must explicitly specify that via :func:`get_context` or :func:`set_start_method`. .. versionchanged:: 3.12 @@ -146,9 +146,17 @@ to start a process. These *start methods* are side-effect so it is generally safe for it to use :func:`os.fork`. No unnecessary resources are inherited. - Available on POSIX platforms which support passing file descriptors - over Unix pipes such as Linux. + Available on POSIX platforms which support passing file descriptors over + Unix pipes such as Linux. The default on those. + .. versionchanged:: 3.14 + This became the default start method on POSIX platforms. + +.. versionchanged:: 3.4 + *spawn* added on all POSIX platforms, and *forkserver* added for + some POSIX platforms. + Child processes no longer inherit all of the parents inheritable + handles on Windows. .. versionchanged:: 3.8 @@ -156,11 +164,12 @@ to start a process. These *start methods* are method should be considered unsafe as it can lead to crashes of the subprocess as macOS system libraries may start threads. See :issue:`33725`. -.. versionchanged:: 3.4 - *spawn* added on all POSIX platforms, and *forkserver* added for - some POSIX platforms. - Child processes no longer inherit all of the parents inheritable - handles on Windows. +.. versionchanged:: 3.14 + + On POSIX platforms the default start method was changed from *fork* to + *forkserver* to retain the performance but avoid common multithreaded + process incompatibilities. See :gh:`84559`. + On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named @@ -254,6 +263,7 @@ processes: p.join() Queues are thread and process safe. + Any object put into a :mod:`~multiprocessing` queue will be serialized. **Pipes** @@ -281,6 +291,8 @@ processes: of corruption from processes using different ends of the pipe at the same time. + The :meth:`~Connection.send` method serializes the the object and + :meth:`~Connection.recv` re-creates the object. Synchronization between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -502,7 +514,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the The constructor should always be called with keyword arguments. *group* should always be ``None``; it exists solely for compatibility with :class:`threading.Thread`. *target* is the callable object to be invoked by - the :meth:`run()` method. It defaults to ``None``, meaning nothing is + the :meth:`run` method. It defaults to ``None``, meaning nothing is called. *name* is the process name (see :attr:`name` for more details). *args* is the argument tuple for the target invocation. *kwargs* is a dictionary of keyword arguments for the target invocation. If provided, @@ -519,7 +531,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the to the process. .. versionchanged:: 3.3 - Added the *daemon* argument. + Added the *daemon* parameter. .. method:: run() @@ -639,7 +651,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the You can use this value if you want to wait on several events at once using :func:`multiprocessing.connection.wait`. Otherwise - calling :meth:`join()` is simpler. + calling :meth:`join` is simpler. On Windows, this is an OS handle usable with the ``WaitForSingleObject`` and ``WaitForMultipleObjects`` family of API calls. On POSIX, this is @@ -649,8 +661,8 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. method:: terminate() - Terminate the process. On POSIX this is done using the ``SIGTERM`` signal; - on Windows :c:func:`TerminateProcess` is used. Note that exit handlers and + Terminate the process. On POSIX this is done using the :py:const:`~signal.SIGTERM` signal; + on Windows :c:func:`!TerminateProcess` is used. Note that exit handlers and finally clauses, etc., will not be executed. Note that descendant processes of the process will *not* be terminated -- @@ -666,7 +678,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. method:: kill() - Same as :meth:`terminate()` but using the ``SIGKILL`` signal on POSIX. + Same as :meth:`terminate` but using the ``SIGKILL`` signal on POSIX. .. versionadded:: 3.7 @@ -709,7 +721,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. exception:: BufferTooShort - Exception raised by :meth:`Connection.recv_bytes_into()` when the supplied + Exception raised by :meth:`Connection.recv_bytes_into` when the supplied buffer object is too small for the message read. If ``e`` is an instance of :exc:`BufferTooShort` then ``e.args[0]`` will give @@ -745,6 +757,11 @@ If you use :class:`JoinableQueue` then you **must** call semaphore used to count the number of unfinished tasks may eventually overflow, raising an exception. +One difference from other Python queue implementations, is that :mod:`multiprocessing` +queues serializes all objects that are put into them using :mod:`pickle`. +The object return by the get method is a re-created object that does not share memory +with the original object. + Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. @@ -811,6 +828,8 @@ For an example of the usage of queues for interprocess communication see used for receiving messages and ``conn2`` can only be used for sending messages. + The :meth:`~multiprocessing.Connection.send` method serializes the the object using + :mod:`pickle` and the :meth:`~multiprocessing.Connection.recv` re-creates the object. .. class:: Queue([maxsize]) @@ -837,6 +856,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. Because of multithreading/multiprocessing semantics, this is not reliable. + May raise an :exc:`OSError` on closed queues. (not guaranteed) + .. method:: full() Return ``True`` if the queue is full, ``False`` otherwise. Because of @@ -940,6 +961,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. + Always raises an :exc:`OSError` if the SimpleQueue is closed. + .. method:: get() Remove and return an item from the queue. @@ -1084,13 +1107,13 @@ Miscellaneous The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. -.. versionchanged:: 3.8 + .. versionadded:: 3.4 - On macOS, the *spawn* start method is now the default. The *fork* start - method should be considered unsafe as it can lead to crashes of the - subprocess. See :issue:`33725`. + .. versionchanged:: 3.8 - .. versionadded:: 3.4 + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess. See :issue:`33725`. .. function:: set_executable(executable) @@ -1245,8 +1268,7 @@ Connection objects are usually created using Connection objects themselves can now be transferred between processes using :meth:`Connection.send` and :meth:`Connection.recv`. - .. versionadded:: 3.3 - Connection objects now support the context management protocol -- see + Connection objects also now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the connection object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -1460,17 +1482,6 @@ object -- see :ref:`multiprocessing-managers`. On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with a timeout will emulate that function's behavior using a sleeping loop. -.. note:: - - If the SIGINT signal generated by :kbd:`Ctrl-C` arrives while the main thread is - blocked by a call to :meth:`BoundedSemaphore.acquire`, :meth:`Lock.acquire`, - :meth:`RLock.acquire`, :meth:`Semaphore.acquire`, :meth:`Condition.acquire` - or :meth:`Condition.wait` then the call will be immediately interrupted and - :exc:`KeyboardInterrupt` will be raised. - - This differs from the behaviour of :mod:`threading` where SIGINT will be - ignored while the equivalent blocking calls are in progress. - .. note:: Some of this package's functionality requires a functioning shared semaphore @@ -2250,11 +2261,11 @@ with the :class:`Pool` class. as CPython does not assure that the finalizer of the pool will be called (see :meth:`object.__del__` for more information). - .. versionadded:: 3.2 - *maxtasksperchild* + .. versionchanged:: 3.2 + Added the *maxtasksperchild* parameter. - .. versionadded:: 3.4 - *context* + .. versionchanged:: 3.4 + Added the *context* parameter. .. versionchanged:: 3.13 *processes* uses :func:`os.process_cpu_count` by default, instead of @@ -2380,7 +2391,7 @@ with the :class:`Pool` class. Wait for the worker processes to exit. One must call :meth:`close` or :meth:`terminate` before using :meth:`join`. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Pool objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. @@ -2484,9 +2495,9 @@ multiple connections at the same time. generally be omitted since it can usually be inferred from the format of *address*. (See :ref:`multiprocessing-address-formats`) - If *authkey* is given and not None, it should be a byte string and will be + If *authkey* is given and not ``None``, it should be a byte string and will be used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. + authentication is done if *authkey* is ``None``. :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. See :ref:`multiprocessing-auth-keys`. @@ -2519,9 +2530,9 @@ multiple connections at the same time. to the :meth:`~socket.socket.listen` method of the socket once it has been bound. - If *authkey* is given and not None, it should be a byte string and will be + If *authkey* is given and not ``None``, it should be a byte string and will be used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. + authentication is done if *authkey* is ``None``. :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. See :ref:`multiprocessing-auth-keys`. @@ -2549,7 +2560,7 @@ multiple connections at the same time. The address from which the last accepted connection came. If this is unavailable then it is ``None``. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Listener objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the listener object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -2959,7 +2970,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" resulting in a bad file descriptor error, but introduces a potential danger to applications which replace :func:`sys.stdin` with a "file-like object" with output buffering. This danger is that if multiple processes call - :meth:`~io.IOBase.close()` on this file-like object, it could result in the same + :meth:`~io.IOBase.close` on this file-like object, it could result in the same data being flushed to the object multiple times, resulting in corruption. If you write a file-like object and implement your own caching, you can @@ -2979,7 +2990,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" The *spawn* and *forkserver* start methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are a few extra restriction which don't apply to the *fork* +There are a few extra restrictions which don't apply to the *fork* start method. More picklability diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 933fd07d62418a..e8f04a6ac7b95d 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -1,5 +1,5 @@ -:mod:`multiprocessing.shared_memory` --- Shared memory for direct access across processes -========================================================================================= +:mod:`!multiprocessing.shared_memory` --- Shared memory for direct access across processes +========================================================================================== .. module:: multiprocessing.shared_memory :synopsis: Provides shared memory for direct access across processes. diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index c36e5cfecfc6a8..f6260383b2b057 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -1,6 +1,5 @@ - -:mod:`netrc` --- netrc file processing -====================================== +:mod:`!netrc` --- netrc file processing +======================================= .. module:: netrc :synopsis: Loading of .netrc files. diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 17d1a275f04c9b..d0ae79c7a3df76 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -1,5 +1,5 @@ -:mod:`numbers` --- Numeric abstract base classes -================================================ +:mod:`!numbers` --- Numeric abstract base classes +================================================= .. module:: numbers :synopsis: Numeric abstract base classes (Complex, Real, Integral, etc.). @@ -84,10 +84,10 @@ The numeric tower ``~``. -Notes for type implementors +Notes for type implementers --------------------------- -Implementors should be careful to make equal numbers equal and hash +Implementers should be careful to make equal numbers equal and hash them to the same values. This may be subtle if there are two different extensions of the real numbers. For example, :class:`fractions.Fraction` implements :func:`hash` as follows:: @@ -166,7 +166,7 @@ Complex``. I'll consider ``a + b``: 2. If ``A`` falls back to the boilerplate code, and it were to return a value from :meth:`~object.__add__`, we'd miss the possibility that ``B`` defines a more intelligent :meth:`~object.__radd__`, so the - boilerplate should return :const:`NotImplemented` from + boilerplate should return :data:`NotImplemented` from :meth:`!__add__`. (Or ``A`` may not implement :meth:`!__add__` at all.) 3. Then ``B``'s :meth:`~object.__radd__` gets a chance. If it accepts diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 96f2c287875d41..e8e71068dd99eb 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -1,5 +1,5 @@ -:mod:`operator` --- Standard operators as functions -=================================================== +:mod:`!operator` --- Standard operators as functions +==================================================== .. module:: operator :synopsis: Functions corresponding to the standard operators. @@ -80,6 +80,20 @@ truth tests, identity tests, and boolean operations: Return ``a is not b``. Tests object identity. +.. function:: is_none(a) + + Return ``a is None``. Tests object identity. + + .. versionadded:: 3.14 + + +.. function:: is_not_none(a) + + Return ``a is not None``. Tests object identity. + + .. versionadded:: 3.14 + + The mathematical and bitwise operations are the most numerous: @@ -405,6 +419,10 @@ Python syntax and the functions in the :mod:`operator` module. +-----------------------+-------------------------+---------------------------------------+ | Identity | ``a is not b`` | ``is_not(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ +| Identity | ``a is None`` | ``is_none(a)`` | ++-----------------------+-------------------------+---------------------------------------+ +| Identity | ``a is not None`` | ``is_not_none(a)`` | ++-----------------------+-------------------------+---------------------------------------+ | Indexed Assignment | ``obj[k] = v`` | ``setitem(obj, k, v)`` | +-----------------------+-------------------------+---------------------------------------+ | Indexed Deletion | ``del obj[k]`` | ``delitem(obj, k)`` | diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 015e83ed2ce5f7..74a49a8fb33666 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -1,5 +1,5 @@ -:mod:`optparse` --- Parser for command line options -=================================================== +:mod:`!optparse` --- Parser for command line options +==================================================== .. module:: optparse :synopsis: Command-line option parsing library. @@ -1352,7 +1352,7 @@ The whole point of creating and populating an OptionParser is to call its the list of arguments to process (default: ``sys.argv[1:]``) ``values`` - an :class:`Values` object to store option arguments in (default: a + a :class:`Values` object to store option arguments in (default: a new instance of :class:`Values`) -- if you give an existing object, the option defaults will not be initialized on it @@ -1739,7 +1739,7 @@ seen, but blow up if it comes after ``-b`` in the command-line. :: Callback example 3: check option order (generalized) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you want to re-use this callback for several similar options (set a flag, but +If you want to reuse this callback for several similar options (set a flag, but blow up if ``-b`` has already been seen), it needs a bit of work: the error message and the flag that it sets must be generalized. :: diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 34bc76b231de92..ecbbc1d7605f9f 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -1,10 +1,10 @@ -:mod:`os.path` --- Common pathname manipulations -================================================ +:mod:`!os.path` --- Common pathname manipulations +================================================= .. module:: os.path :synopsis: Operations on pathnames. -**Source code:** :source:`Lib/posixpath.py` (for POSIX) and +**Source code:** :source:`Lib/genericpath.py`, :source:`Lib/posixpath.py` (for POSIX) and :source:`Lib/ntpath.py` (for Windows). .. index:: single: path; operations @@ -79,19 +79,20 @@ the :mod:`glob` module.) .. function:: commonpath(paths) - Return the longest common sub-path of each pathname in the sequence + Return the longest common sub-path of each pathname in the iterable *paths*. Raise :exc:`ValueError` if *paths* contain both absolute - and relative pathnames, the *paths* are on the different drives or + and relative pathnames, if *paths* are on different drives, or if *paths* is empty. Unlike :func:`commonprefix`, this returns a valid path. - .. availability:: Unix, Windows. - .. versionadded:: 3.5 .. versionchanged:: 3.6 Accepts a sequence of :term:`path-like objects `. + .. versionchanged:: 3.13 + Any iterable can now be passed, rather than just sequences. + .. function:: commonprefix(list) @@ -144,7 +145,7 @@ the :mod:`glob` module.) .. function:: lexists(path) - Return ``True`` if *path* refers to an existing path. Returns ``True`` for + Return ``True`` if *path* refers to an existing path, including broken symbolic links. Equivalent to :func:`exists` on platforms lacking :func:`os.lstat`. @@ -200,14 +201,14 @@ the :mod:`glob` module.) .. function:: getatime(path) - Return the time of last access of *path*. The return value is a floating point number giving + Return the time of last access of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. .. function:: getmtime(path) - Return the time of last modification of *path*. The return value is a floating point number + Return the time of last modification of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. @@ -301,8 +302,8 @@ the :mod:`glob` module.) always mount points, and for any other path ``GetVolumePathName`` is called to see if it is different from the input path. - .. versionadded:: 3.4 - Support for detecting non-root mount points on Windows. + .. versionchanged:: 3.4 + Added support for detecting non-root mount points on Windows. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -321,10 +322,11 @@ the :mod:`glob` module.) Dev Drives. See `the Windows documentation `_ for information on enabling and creating Dev Drives. - .. availability:: Windows. - .. versionadded:: 3.12 + .. versionchanged:: 3.13 + The function is now available on all platforms, and will always return ``False`` on those that have no support for Dev Drives + .. function:: isreserved(path) @@ -387,7 +389,7 @@ the :mod:`glob` module.) that contains symbolic links. On Windows, it converts forward slashes to backward slashes. To normalize case, use :func:`normcase`. - .. note:: + .. note:: On POSIX systems, in accordance with `IEEE Std 1003.1 2013 Edition; 4.13 Pathname Resolution `_, if a pathname begins with exactly two slashes, the first component @@ -407,9 +409,8 @@ the :mod:`glob` module.) style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is - resolved as far as possible and any remainder is appended without checking - whether it exists. + ``True``, :exc:`OSError` is raised. If *strict* is ``False`` these errors + are ignored, and so the result might be missing or otherwise inaccessible. .. note:: This function emulates the operating system's procedure for making a path @@ -439,8 +440,6 @@ the :mod:`glob` module.) *start* defaults to :data:`os.curdir`. - .. availability:: Unix, Windows. - .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -451,8 +450,6 @@ the :mod:`glob` module.) This is determined by the device number and i-node number and raises an exception if an :func:`os.stat` call on either pathname fails. - .. availability:: Unix, Windows. - .. versionchanged:: 3.2 Added Windows support. @@ -467,8 +464,6 @@ the :mod:`glob` module.) Return ``True`` if the file descriptors *fp1* and *fp2* refer to the same file. - .. availability:: Unix, Windows. - .. versionchanged:: 3.2 Added Windows support. @@ -483,8 +478,6 @@ the :mod:`glob` module.) :func:`os.lstat`, or :func:`os.stat`. This function implements the underlying comparison used by :func:`samefile` and :func:`sameopenfile`. - .. availability:: Unix, Windows. - .. versionchanged:: 3.4 Added Windows support. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index cc9f3e75a80c51..f9cded40c2c755 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1,5 +1,5 @@ -:mod:`os` --- Miscellaneous operating system interfaces -======================================================= +:mod:`!os` --- Miscellaneous operating system interfaces +======================================================== .. module:: os :synopsis: Miscellaneous operating system interfaces. @@ -34,12 +34,12 @@ Notes on the availability of these functions: * On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. -* On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, large - parts of the :mod:`os` module are not available or behave differently. API - related to processes (e.g. :func:`~os.fork`, :func:`~os.execve`), signals - (e.g. :func:`~os.kill`, :func:`~os.wait`), and resources - (e.g. :func:`~os.nice`) are not available. Others like :func:`~os.getuid` - and :func:`~os.getpid` are emulated or stubs. +* On WebAssembly platforms, Android and iOS, large parts of the :mod:`os` module are + not available or behave differently. APIs related to processes (e.g. + :func:`~os.fork`, :func:`~os.execve`) and resources (e.g. :func:`~os.nice`) + are not available. Others like :func:`~os.getuid` and :func:`~os.getpid` are + emulated or stubs. WebAssembly platforms also lack support for signals (e.g. + :func:`~os.kill`, :func:`~os.wait`). .. note:: @@ -113,8 +113,8 @@ of the UTF-8 encoding: * Use UTF-8 as the :term:`filesystem encoding `. -* :func:`sys.getfilesystemencoding()` returns ``'utf-8'``. -* :func:`locale.getpreferredencoding()` returns ``'utf-8'`` (the *do_setlocale* +* :func:`sys.getfilesystemencoding` returns ``'utf-8'``. +* :func:`locale.getpreferredencoding` returns ``'utf-8'`` (the *do_setlocale* argument has no effect). * :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` all use UTF-8 as their text encoding, with the ``surrogateescape`` @@ -133,8 +133,8 @@ level APIs also exhibit different default behaviours: * Command line arguments, environment variables and filenames are decoded to text using the UTF-8 encoding. -* :func:`os.fsdecode()` and :func:`os.fsencode()` use the UTF-8 encoding. -* :func:`open()`, :func:`io.open()`, and :func:`codecs.open()` use the UTF-8 +* :func:`os.fsdecode` and :func:`os.fsencode` use the UTF-8 encoding. +* :func:`open`, :func:`io.open`, and :func:`codecs.open` use the UTF-8 encoding by default. However, they still use the strict error handler by default so that attempting to open a binary file in text mode is likely to raise an exception rather than producing nonsense data. @@ -178,7 +178,7 @@ process and user. Return the filename corresponding to the controlling terminal of the process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: environ @@ -193,6 +193,10 @@ process and user. to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. + The :meth:`!os.environ.refresh` method updates :data:`os.environ` with + changes to the environment made by :func:`os.putenv`, by + :func:`os.unsetenv`, or made outside Python in the same process. + This mapping may be used to modify the environment as well as query the environment. :func:`putenv` will be called automatically when the mapping is modified. @@ -225,6 +229,9 @@ process and user. .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. versionchanged:: 3.14 + Added the :meth:`!os.environ.refresh` method. + .. data:: environb @@ -355,7 +362,7 @@ process and user. Return the effective group id of the current process. This corresponds to the "set id" bit on the file being executed in the current process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: geteuid() @@ -364,7 +371,7 @@ process and user. Return the current process's effective user id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getgid() @@ -375,8 +382,8 @@ process and user. .. availability:: Unix. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: getgrouplist(user, group, /) @@ -386,7 +393,7 @@ process and user. field from the password record for *user*, because that group ID will otherwise be potentially omitted. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -395,7 +402,7 @@ process and user. Return list of supplemental group ids associated with the current process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. note:: @@ -423,7 +430,7 @@ process and user. falls back to ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the current real user id. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. function:: getpgid(pid) @@ -431,7 +438,7 @@ process and user. Return the process group id of the process with process id *pid*. If *pid* is 0, the process group id of the current process is returned. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getpgrp() @@ -439,7 +446,7 @@ process and user. Return the id of the current process group. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getpid() @@ -448,8 +455,8 @@ process and user. Return the current process id. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: getppid() @@ -459,7 +466,7 @@ process and user. the id returned is the one of the init process (1), on Windows it is still the same id, which may be already reused by another process. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionchanged:: 3.2 Added support for Windows. @@ -477,7 +484,7 @@ process and user. (respectively) the calling process, the process group of the calling process, or the real user ID of the calling process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -488,7 +495,7 @@ process and user. Parameters for the :func:`getpriority` and :func:`setpriority` functions. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -509,7 +516,7 @@ process and user. Return a tuple (ruid, euid, suid) denoting the current process's real, effective, and saved user ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.2 @@ -519,7 +526,7 @@ process and user. Return a tuple (rgid, egid, sgid) denoting the current process's real, effective, and saved group ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.2 @@ -532,8 +539,8 @@ process and user. .. availability:: Unix. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: initgroups(username, gid, /) @@ -542,7 +549,7 @@ process and user. the groups of which the specified username is a member, plus the specified group id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. versionadded:: 3.2 @@ -561,6 +568,8 @@ process and user. of :data:`os.environ`. This also applies to :func:`getenv` and :func:`getenvb`, which respectively use :data:`os.environ` and :data:`os.environb` in their implementations. + See also the :data:`os.environ.refresh() ` method. + .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may @@ -576,21 +585,21 @@ process and user. Set the current process's effective group id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: seteuid(euid, /) Set the current process's effective user id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: setgid(gid, /) Set the current process' group id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: setgroups(groups, /) @@ -599,7 +608,7 @@ process and user. *groups*. *groups* must be a sequence, and each element must be an integer identifying a group. This operation is typically available only to the superuser. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. note:: On macOS, the length of *groups* may not exceed the system-defined maximum number of effective group ids, typically 16. @@ -649,7 +658,7 @@ process and user. Call the system call :c:func:`!setpgrp` or ``setpgrp(0, 0)`` depending on which version is implemented (if any). See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setpgid(pid, pgrp, /) @@ -658,7 +667,7 @@ process and user. process with id *pid* to the process group with id *pgrp*. See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setpriority(which, who, priority) @@ -675,7 +684,7 @@ process and user. *priority* is a value in the range -20 to 19. The default priority is 0; lower priorities cause more favorable scheduling. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -684,14 +693,14 @@ process and user. Set the current process's real and effective group ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: setresgid(rgid, egid, sgid, /) Set the current process's real, effective, and saved group ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. versionadded:: 3.2 @@ -700,7 +709,7 @@ process and user. Set the current process's real, effective, and saved user ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. versionadded:: 3.2 @@ -709,21 +718,21 @@ process and user. Set the current process's real and effective user ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: getsid(pid, /) Call the system call :c:func:`!getsid`. See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setsid() Call the system call :c:func:`!setsid`. See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setuid(uid, /) @@ -732,7 +741,7 @@ process and user. Set the current process's user id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. placed in this section since it relates to errno.... a little weak @@ -755,8 +764,8 @@ process and user. Set the current numeric umask and return the previous umask. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: uname() @@ -784,6 +793,11 @@ process and user. :func:`socket.gethostname` or even ``socket.gethostbyaddr(socket.gethostname())``. + On macOS, iOS and Android, this returns the *kernel* name and version (i.e., + ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname` + can be used to get the user-facing operating system name and version on iOS and + Android. + .. availability:: Unix. .. versionchanged:: 3.3 @@ -804,6 +818,8 @@ process and user. don't update :data:`os.environ`, so it is actually preferable to delete items of :data:`os.environ`. + See also the :data:`os.environ.refresh() ` method. + .. audit-event:: os.unsetenv key os.unsetenv .. versionchanged:: 3.9 @@ -918,10 +934,10 @@ as internal buffering of data. Copy *count* bytes from file descriptor *src*, starting from offset *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. - If *offset_src* is None, then *src* is read from the current position; + If *offset_src* is ``None``, then *src* is read from the current position; respectively for *offset_dst*. - In Linux kernel older than 5.3, the files pointed by *src* and *dst* + In Linux kernel older than 5.3, the files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. @@ -1003,8 +1019,8 @@ as internal buffering of data. .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. versionchanged:: 3.13 Added support on Windows. @@ -1021,8 +1037,8 @@ as internal buffering of data. .. availability:: Unix. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. function:: fdatasync(fd) @@ -1112,8 +1128,8 @@ as internal buffering of data. .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. On Windows, this function is limited to pipes. @@ -1131,7 +1147,7 @@ as internal buffering of data. Calls the C standard library function :c:func:`grantpt`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.13 @@ -1175,7 +1191,7 @@ as internal buffering of data. Make the calling process a session leader; make the tty the controlling tty, the stdin, the stdout, and the stderr of the calling process; close fd. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.11 @@ -1359,7 +1375,7 @@ or `the MSDN `_ on Windo descriptors are :ref:`non-inheritable `. For a (slightly) more portable approach, use the :mod:`pty` module. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.4 The new file descriptors are now non-inheritable. @@ -1385,7 +1401,7 @@ or `the MSDN `_ on Windo Return a pair of file descriptors ``(r, w)`` usable for reading and writing, respectively. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -1395,7 +1411,7 @@ or `the MSDN `_ on Windo Ensures that enough disk space is allocated for the file specified by *fd* starting from *offset* and continuing for *len* bytes. - .. availability:: Unix, not Emscripten. + .. availability:: Unix. .. versionadded:: 3.3 @@ -1455,7 +1471,7 @@ or `the MSDN `_ on Windo If the value :data:`O_CLOEXEC` is available on the system, it is added to *oflag*. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.13 @@ -1527,7 +1543,7 @@ or `the MSDN `_ on Windo it is available; otherwise, the C standard library function :c:func:`ptsname`, which is not guaranteed to be thread-safe, is called. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.13 @@ -1546,7 +1562,7 @@ or `the MSDN `_ on Windo .. function:: pwritev(fd, buffers, offset, flags=0, /) - Write the *buffers* contents to file descriptor *fd* at a offset *offset*, + Write the *buffers* contents to file descriptor *fd* at an offset *offset*, leaving the file offset unchanged. *buffers* must be a sequence of :term:`bytes-like objects `. Buffers are processed in array order. Entire contents of the first buffer is written before @@ -1654,7 +1670,7 @@ or `the MSDN `_ on Windo Cross-platform applications should not use *headers*, *trailers* and *flags* arguments. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. note:: @@ -1674,7 +1690,7 @@ or `the MSDN `_ on Windo Parameters to the :func:`sendfile` function, if the implementation supports them. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -1683,7 +1699,7 @@ or `the MSDN `_ on Windo Parameter to the :func:`sendfile` function, if the implementation supports it. The data won't be cached in the virtual memory and will be freed afterwards. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.11 @@ -1697,8 +1713,8 @@ or `the MSDN `_ on Windo .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. On Windows, this function is limited to pipes. @@ -1708,14 +1724,31 @@ or `the MSDN `_ on Windo Added support for pipes on Windows. -.. function:: splice(src, dst, count, offset_src=None, offset_dst=None) +.. function:: splice(src, dst, count, offset_src=None, offset_dst=None, flags=0) Transfer *count* bytes from file descriptor *src*, starting from offset *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. + + The splicing behaviour can be modified by specifying a *flags* value. + Any of the following variables may used, combined using bitwise OR + (the ``|`` operator): + + * If :const:`SPLICE_F_MOVE` is specified, + the kernel is asked to move pages instead of copying, + but pages may still be copied if the kernel cannot move the pages from the pipe. + + * If :const:`SPLICE_F_NONBLOCK` is specified, + the kernel is asked to not block on I/O. + This makes the splice pipe operations nonblocking, + but splice may nevertheless block because the spliced file descriptors may block. + + * If :const:`SPLICE_F_MORE` is specified, + it hints to the kernel that more data will be coming in a subsequent splice. + At least one of the file descriptors must refer to a pipe. If *offset_src* - is None, then *src* is read from the current position; respectively for + is ``None``, then *src* is read from the current position; respectively for *offset_dst*. The offset associated to the file descriptor that refers to a - pipe must be ``None``. The files pointed by *src* and *dst* must reside in + pipe must be ``None``. The files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. @@ -1730,6 +1763,8 @@ or `the MSDN `_ on Windo make sense to block because there are no writers connected to the write end of the pipe. + .. seealso:: The :manpage:`splice(2)` man page. + .. availability:: Linux >= 2.6.17 with glibc >= 2.5 .. versionadded:: 3.10 @@ -1792,7 +1827,7 @@ or `the MSDN `_ on Windo Calls the C standard library function :c:func:`unlockpt`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.13 @@ -1893,8 +1928,7 @@ Using the :mod:`subprocess` module, all file descriptors except standard streams are closed, and inheritable handles are only inherited if the *close_fds* parameter is ``False``. -On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, the file -descriptor cannot be modified. +On WebAssembly platforms, the file descriptor cannot be modified. .. function:: get_inheritable(fd, /) @@ -2080,7 +2114,7 @@ features: .. audit-event:: os.chflags path,flags os.chflags - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.3 Added the *follow_symlinks* parameter. @@ -2126,12 +2160,12 @@ features: constants or a corresponding integer value). All other bits are ignored. The default value of *follow_symlinks* is ``False`` on Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. audit-event:: os.chmod path,mode,dir_fd os.chmod - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. @@ -2159,10 +2193,10 @@ features: .. availability:: Unix. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. @@ -2174,7 +2208,7 @@ features: Change the root directory of the current process to *path*. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2214,7 +2248,7 @@ features: .. audit-event:: os.chflags path,flags os.lchflags - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2264,7 +2298,7 @@ features: .. audit-event:: os.link src,dst,src_dir_fd,dst_dir_fd os.link - .. availability:: Unix, Windows, not Emscripten. + .. availability:: Unix, Windows. .. versionchanged:: 3.2 Added Windows support. @@ -2306,7 +2340,7 @@ features: .. versionchanged:: 3.2 The *path* parameter became optional. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. .. versionchanged:: 3.6 @@ -2426,6 +2460,10 @@ features: platform-dependent. On some platforms, they are ignored and you should call :func:`chmod` explicitly to set them. + On Windows, a *mode* of ``0o700`` is specifically handled to apply access + control to the new directory such that only the current user and + administrators have access. Other values of *mode* are ignored. + This function can also support :ref:`paths relative to directory descriptors `. @@ -2440,6 +2478,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Windows now handles a *mode* of ``0o700``. + .. function:: makedirs(name, mode=0o777, exist_ok=False) @@ -2500,7 +2541,7 @@ features: FIFO for reading, and the client opens it for writing. Note that :func:`mkfifo` doesn't open the FIFO --- it just creates the rendezvous point. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.3 Added the *dir_fd* parameter. @@ -2522,7 +2563,7 @@ features: This function can also support :ref:`paths relative to directory descriptors `. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.3 Added the *dir_fd* parameter. @@ -2614,7 +2655,6 @@ features: .. versionchanged:: 3.8 Accepts a :term:`path-like object` and a bytes object on Windows. - .. versionchanged:: 3.8 Added support for directory junctions, and changed to return the substitution path (which typically includes ``\\?\`` prefix) rather than the optional "print name" field that was previously returned. @@ -2820,7 +2860,7 @@ features: .. versionchanged:: 3.6 Added support for the :term:`context manager` protocol and the - :func:`~scandir.close()` method. If a :func:`scandir` iterator is neither + :func:`~scandir.close` method. If a :func:`scandir` iterator is neither exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted in its destructor. @@ -3115,21 +3155,21 @@ features: Time of most recent access expressed in nanoseconds as an integer. - .. versionadded: 3.3 + .. versionadded:: 3.3 .. attribute:: st_mtime_ns Time of most recent content modification expressed in nanoseconds as an integer. - .. versionadded: 3.3 + .. versionadded:: 3.3 .. attribute:: st_ctime_ns Time of most recent metadata change expressed in nanoseconds as an integer. - .. versionadded: 3.3 + .. versionadded:: 3.3 .. versionchanged:: 3.12 ``st_ctime_ns`` is deprecated on Windows. Use ``st_birthtime_ns`` @@ -3257,10 +3297,10 @@ features: Windows now returns the file index as :attr:`st_ino` when available. - .. versionadded:: 3.7 + .. versionchanged:: 3.7 Added the :attr:`st_fstype` member to Solaris/derivatives. - .. versionadded:: 3.8 + .. versionchanged:: 3.8 Added the :attr:`st_reparse_tag` member on Windows. .. versionchanged:: 3.8 @@ -3274,16 +3314,13 @@ features: platforms, but for now still contains creation time. Use :attr:`st_birthtime` for the creation time. - .. versionchanged:: 3.12 On Windows, :attr:`st_ino` may now be up to 128 bits, depending on the file system. Previously it would not be above 64 bits, and larger file identifiers would be arbitrarily packed. - .. versionchanged:: 3.12 On Windows, :attr:`st_rdev` no longer returns a value. Previously it would contain the same as :attr:`st_dev`, which was incorrect. - .. versionadded:: 3.12 Added the :attr:`st_birthtime` member on Windows. @@ -3448,8 +3485,8 @@ features: .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. versionchanged:: 3.2 Added support for Windows 6.0 (Vista) symbolic links. @@ -3638,6 +3675,7 @@ features: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) + os.rmdir(top) .. audit-event:: os.walk top,topdown,onerror,followlinks os.walk @@ -3768,7 +3806,7 @@ features: new file descriptor is :ref:`non-inheritable `. *initval* is the initial value of the event counter. The initial value - must be an 32 bit unsigned integer. Please note that the initial value is + must be a 32 bit unsigned integer. Please note that the initial value is limited to a 32 bit unsigned int although the event counter is an unsigned 64 bit integer with a maximum value of 2\ :sup:`64`\ -\ 2. @@ -3847,7 +3885,7 @@ features: .. data:: EFD_SEMAPHORE - Provide semaphore-like semantics for reads from a :func:`eventfd` file + Provide semaphore-like semantics for reads from an :func:`eventfd` file descriptor. On read the internal counter is decremented by one. .. availability:: Linux >= 2.6.30 @@ -3855,6 +3893,8 @@ features: .. versionadded:: 3.10 +.. _os-timerfd: + Timer File Descriptors ~~~~~~~~~~~~~~~~~~~~~~ @@ -4275,7 +4315,7 @@ to be ignored. .. audit-event:: os.exec path,args,env os.execl - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor @@ -4318,49 +4358,49 @@ written in Python, such as a mail server's external command delivery program. Exit code that means the command was used incorrectly, such as when the wrong number of arguments are given. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_DATAERR Exit code that means the input data was incorrect. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOINPUT Exit code that means an input file did not exist or was not readable. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOUSER Exit code that means a specified user did not exist. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOHOST Exit code that means a specified host did not exist. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_UNAVAILABLE Exit code that means that a required service is unavailable. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_SOFTWARE Exit code that means an internal software error was detected. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_OSERR @@ -4368,7 +4408,7 @@ written in Python, such as a mail server's external command delivery program. Exit code that means an operating system error was detected, such as the inability to fork or create a pipe. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_OSFILE @@ -4376,21 +4416,21 @@ written in Python, such as a mail server's external command delivery program. Exit code that means some system file did not exist, could not be opened, or had some other kind of error. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_CANTCREAT Exit code that means a user specified output file could not be created. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_IOERR Exit code that means that an error occurred while doing I/O on some file. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_TEMPFAIL @@ -4399,7 +4439,7 @@ written in Python, such as a mail server's external command delivery program. that may not really be an error, such as a network connection that couldn't be made during a retryable operation. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_PROTOCOL @@ -4407,7 +4447,7 @@ written in Python, such as a mail server's external command delivery program. Exit code that means that a protocol exchange was illegal, invalid, or not understood. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOPERM @@ -4415,21 +4455,21 @@ written in Python, such as a mail server's external command delivery program. Exit code that means that there were insufficient permissions to perform the operation (but not intended for file system problems). - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_CONFIG Exit code that means that some kind of configuration error occurred. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOTFOUND Exit code that means something like "an entry was not found". - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: fork() @@ -4478,7 +4518,7 @@ written in Python, such as a mail server's external command delivery program. for technical details of why we're surfacing this longstanding platform compatibility problem to developers. - .. availability:: POSIX, not Emscripten, not WASI. + .. availability:: POSIX, not WASI, not Android, not iOS. .. function:: forkpty() @@ -4496,16 +4536,16 @@ written in Python, such as a mail server's external command delivery program. On macOS the use of this function is unsafe when mixed with using higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.8 + Calling ``forkpty()`` in a subinterpreter is no longer supported + (:exc:`RuntimeError` is raised). + .. versionchanged:: 3.12 If Python is able to detect that your process has multiple threads, this now raises a :exc:`DeprecationWarning`. See the longer explanation on :func:`os.fork`. - .. versionchanged:: 3.8 - Calling ``forkpty()`` in a subinterpreter is no longer supported - (:exc:`RuntimeError` is raised). - - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: kill(pid, sig, /) @@ -4529,7 +4569,7 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.kill pid,sig os.kill - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not iOS. .. versionchanged:: 3.2 Added Windows support. @@ -4545,14 +4585,14 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.killpg pgid,sig os.killpg - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: nice(increment, /) Add *increment* to the process's "niceness". Return the new niceness. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: pidfd_open(pid, flags=0) @@ -4563,7 +4603,7 @@ written in Python, such as a mail server's external command delivery program. See the :manpage:`pidfd_open(2)` man page for more details. - .. availability:: Linux >= 5.3 + .. availability:: Linux >= 5.3, Android >= :func:`build-time ` API level 31 .. versionadded:: 3.9 .. data:: PIDFD_NONBLOCK @@ -4582,7 +4622,7 @@ written in Python, such as a mail server's external command delivery program. Lock program segments into memory. The value of *op* (defined in ````) determines which segments are locked. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: popen(cmd, mode='r', buffering=-1) @@ -4614,7 +4654,7 @@ written in Python, such as a mail server's external command delivery program. documentation for more powerful ways to manage and communicate with subprocesses. - .. availability:: not Emscripten, not WASI. + .. availability:: not WASI, not Android, not iOS. .. note:: The :ref:`Python UTF-8 Mode ` affects encodings used @@ -4624,6 +4664,10 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. + .. deprecated:: 3.14 + The function is :term:`soft deprecated` and should no longer be used to + write new code. The :mod:`subprocess` module is recommended instead. + .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ @@ -4719,12 +4763,10 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.13 *env* parameter accepts ``None``. - - .. versionchanged:: 3.13 ``os.POSIX_SPAWN_CLOSEFROM`` is available on platforms where :c:func:`!posix_spawn_file_actions_addclosefrom_np` exists. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: posix_spawnp(path, argv, env, *, file_actions=None, \ setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ @@ -4740,7 +4782,7 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.8 - .. availability:: POSIX, not Emscripten, not WASI. + .. availability:: POSIX, not WASI, not Android, not iOS. See :func:`posix_spawn` documentation. @@ -4773,7 +4815,7 @@ written in Python, such as a mail server's external command delivery program. There is no way to unregister a function. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.7 @@ -4842,7 +4884,7 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.spawn mode,path,args,env os.spawnl - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. :func:`spawnlp`, :func:`spawnlpe`, :func:`spawnvp` and :func:`spawnvpe` are not available on Windows. :func:`spawnle` and @@ -4852,6 +4894,10 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. deprecated:: 3.14 + These functions are :term:`soft deprecated` and should no longer be used + to write new code. The :mod:`subprocess` module is recommended instead. + .. data:: P_NOWAIT P_NOWAITO @@ -4956,7 +5002,7 @@ written in Python, such as a mail server's external command delivery program. shell documentation. The :mod:`subprocess` module provides more powerful facilities for spawning - new processes and retrieving their results; using that module is preferable + new processes and retrieving their results; using that module is recommended to using this function. See the :ref:`subprocess-replacements` section in the :mod:`subprocess` documentation for some helpful recipes. @@ -4966,7 +5012,7 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.system command os.system - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. .. function:: times() @@ -5010,7 +5056,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. seealso:: @@ -5044,7 +5090,7 @@ written in Python, such as a mail server's external command delivery program. Otherwise, if there are no matching children that could be waited for, :exc:`ChildProcessError` is raised. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.3 @@ -5085,7 +5131,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -5105,7 +5151,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: wait4(pid, options) @@ -5119,7 +5165,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: P_PID @@ -5136,7 +5182,7 @@ written in Python, such as a mail server's external command delivery program. * :data:`!P_PIDFD` - wait for the child identified by the file descriptor *id* (a process file descriptor created with :func:`pidfd_open`). - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. note:: :data:`!P_PIDFD` is only available on Linux >= 5.4. @@ -5151,7 +5197,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitid` causes child processes to be reported if they have been continued from a job control stop since they were last reported. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: WEXITED @@ -5162,7 +5208,7 @@ written in Python, such as a mail server's external command delivery program. The other ``wait*`` functions always report children that have terminated, so this option is not available for them. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.3 @@ -5174,7 +5220,7 @@ written in Python, such as a mail server's external command delivery program. This option is not available for the other ``wait*`` functions. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.3 @@ -5187,7 +5233,7 @@ written in Python, such as a mail server's external command delivery program. This option is not available for :func:`waitid`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: WNOHANG @@ -5196,7 +5242,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitid` to return right away if no child process status is available immediately. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: WNOWAIT @@ -5206,7 +5252,7 @@ written in Python, such as a mail server's external command delivery program. This option is not available for the other ``wait*`` functions. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: CLD_EXITED @@ -5219,7 +5265,7 @@ written in Python, such as a mail server's external command delivery program. These are the possible values for :attr:`!si_code` in the result returned by :func:`waitid`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.3 @@ -5254,7 +5300,7 @@ written in Python, such as a mail server's external command delivery program. :func:`WIFEXITED`, :func:`WEXITSTATUS`, :func:`WIFSIGNALED`, :func:`WTERMSIG`, :func:`WIFSTOPPED`, :func:`WSTOPSIG` functions. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. .. versionadded:: 3.9 @@ -5270,7 +5316,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFSIGNALED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WIFCONTINUED(status) @@ -5281,7 +5327,7 @@ used to determine the disposition of a process. See :data:`WCONTINUED` option. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WIFSTOPPED(status) @@ -5293,14 +5339,14 @@ used to determine the disposition of a process. done using :data:`WUNTRACED` option or when the process is being traced (see :manpage:`ptrace(2)`). - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WIFSIGNALED(status) Return ``True`` if the process was terminated by a signal, otherwise return ``False``. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WIFEXITED(status) @@ -5309,7 +5355,7 @@ used to determine the disposition of a process. by calling ``exit()`` or ``_exit()``, or by returning from ``main()``; otherwise return ``False``. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WEXITSTATUS(status) @@ -5318,7 +5364,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFEXITED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WSTOPSIG(status) @@ -5327,7 +5373,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFSTOPPED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WTERMSIG(status) @@ -5336,7 +5382,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFSIGNALED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. Interface to the scheduler @@ -5534,7 +5580,7 @@ Miscellaneous System Information If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, :func:`process_cpu_count` returns the overridden value *n*. - See also the :func:`sched_getaffinity` functions. + See also the :func:`sched_getaffinity` function. .. versionadded:: 3.13 @@ -5708,20 +5754,20 @@ Random numbers easy-to-use interface to the random number generator provided by your platform, please see :class:`random.SystemRandom`. - .. versionchanged:: 3.6.0 - On Linux, ``getrandom()`` is now used in blocking mode to increase the - security. - - .. versionchanged:: 3.5.2 - On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool - is not initialized yet), fall back on reading ``/dev/urandom``. - .. versionchanged:: 3.5 On Linux 3.17 and newer, the ``getrandom()`` syscall is now used when available. On OpenBSD 5.6 and newer, the C ``getentropy()`` function is now used. These functions avoid the usage of an internal file descriptor. + .. versionchanged:: 3.5.2 + On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool + is not initialized yet), fall back on reading ``/dev/urandom``. + + .. versionchanged:: 3.6 + On Linux, ``getrandom()`` is now used in blocking mode to increase the + security. + .. versionchanged:: 3.11 On Windows, ``BCryptGenRandom()`` is used instead of ``CryptGenRandom()`` which is deprecated. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index f94b6fb3805684..b6fb36554f7cec 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1,13 +1,12 @@ - -:mod:`pathlib` --- Object-oriented filesystem paths -=================================================== +:mod:`!pathlib` --- Object-oriented filesystem paths +==================================================== .. module:: pathlib :synopsis: Object-oriented filesystem paths .. versionadded:: 3.4 -**Source code:** :source:`Lib/pathlib.py` +**Source code:** :source:`Lib/pathlib/` .. index:: single: path; operations @@ -22,6 +21,12 @@ inherit from pure paths but also provide I/O operations. .. image:: pathlib-inheritance.png :align: center :class: invert-in-dark-mode + :alt: Inheritance diagram showing the classes available in pathlib. The + most basic class is PurePath, which has three direct subclasses: + PurePosixPath, PureWindowsPath, and Path. Further to these four + classes, there are two classes that use multiple inheritance: + PosixPath subclasses PurePosixPath and Path, and WindowsPath + subclasses PureWindowsPath and Path. If you've never used this module before or just aren't sure which class is right for your task, :class:`Path` is most likely what you need. It instantiates @@ -173,8 +178,8 @@ we also call *flavours*: A subclass of :class:`PurePath`, this path flavour represents non-Windows filesystem paths:: - >>> PurePosixPath('/etc') - PurePosixPath('/etc') + >>> PurePosixPath('/etc/hosts') + PurePosixPath('/etc/hosts') *pathsegments* is specified similarly to :class:`PurePath`. @@ -183,8 +188,8 @@ we also call *flavours*: A subclass of :class:`PurePath`, this path flavour represents Windows filesystem paths, including `UNC paths`_:: - >>> PureWindowsPath('c:/Program Files/') - PureWindowsPath('c:/Program Files') + >>> PureWindowsPath('c:/', 'Users', 'Ximénez') + PureWindowsPath('c:/Users/Ximénez') >>> PureWindowsPath('//server/share/file') PureWindowsPath('//server/share/file') @@ -303,10 +308,10 @@ Methods and properties Pure paths provide the following methods and properties: -.. attribute:: PurePath.pathmod +.. attribute:: PurePath.parser The implementation of the :mod:`os.path` module used for low-level path - operations: either :mod:`posixpath` or :mod:`ntpath`. + parsing and joining: either :mod:`posixpath` or :mod:`ntpath`. .. versionadded:: 3.13 @@ -450,6 +455,10 @@ Pure paths provide the following methods and properties: This is commonly called the file extension. + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.suffixes A list of the path's suffixes, often called file extensions:: @@ -461,6 +470,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').suffixes [] + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.stem @@ -572,6 +585,9 @@ Pure paths provide the following methods and properties: >>> PurePath('/a/b/c.py').full_match('**/*.py') True + .. seealso:: + :ref:`pathlib-pattern-language` documentation. + As with other methods, case-sensitivity follows platform defaults:: >>> PurePosixPath('b.py').full_match('*.PY') @@ -625,8 +641,8 @@ Pure paths provide the following methods and properties: raise ValueError(error_message.format(str(self), str(formatted))) ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other is absolute. - When *walk_up* is False (the default), the path must start with *other*. - When the argument is True, ``..`` entries may be added to form the + When *walk_up* is false (the default), the path must start with *other*. + When the argument is true, ``..`` entries may be added to form the relative path. In all other cases, such as the paths referencing different drives, :exc:`ValueError` is raised.:: @@ -711,6 +727,11 @@ Pure paths provide the following methods and properties: >>> p.with_suffix('') PureWindowsPath('README') + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. In previous + versions, :exc:`ValueError` is raised if a single dot is supplied. + .. method:: PurePath.with_segments(*pathsegments) @@ -762,8 +783,8 @@ calls on path objects. There are three ways to instantiate concrete paths: A subclass of :class:`Path` and :class:`PurePosixPath`, this class represents concrete non-Windows filesystem paths:: - >>> PosixPath('/etc') - PosixPath('/etc') + >>> PosixPath('/etc/hosts') + PosixPath('/etc/hosts') *pathsegments* is specified similarly to :class:`PurePath`. @@ -777,8 +798,8 @@ calls on path objects. There are three ways to instantiate concrete paths: A subclass of :class:`Path` and :class:`PureWindowsPath`, this class represents concrete Windows filesystem paths:: - >>> WindowsPath('c:/Program Files/') - WindowsPath('c:/Program Files') + >>> WindowsPath('c:/', 'Users', 'Ximénez') + WindowsPath('c:/Users/Ximénez') *pathsegments* is specified similarly to :class:`PurePath`. @@ -805,9 +826,12 @@ bugs or failures in your application):: % (cls.__name__,)) UnsupportedOperation: cannot instantiate 'WindowsPath' on your system +Some concrete path methods can raise an :exc:`OSError` if a system call fails +(for example because the path doesn't exist). -File URIs -^^^^^^^^^ + +Parsing and generating URIs +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Concrete path objects can be created from, and represented as, 'file' URIs conforming to :rfc:`8089`. @@ -867,21 +891,36 @@ conforming to :rfc:`8089`. it strictly impure. -Methods -^^^^^^^ +Expanding and resolving paths +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Concrete paths provide the following methods in addition to pure paths -methods. Many of these methods can raise an :exc:`OSError` if a system -call fails (for example because the path doesn't exist). +.. classmethod:: Path.home() -.. versionchanged:: 3.8 + Return a new path object representing the user's home directory (as + returned by :func:`os.path.expanduser` with ``~`` construct). If the home + directory can't be resolved, :exc:`RuntimeError` is raised. - :meth:`~Path.exists()`, :meth:`~Path.is_dir()`, :meth:`~Path.is_file()`, - :meth:`~Path.is_mount()`, :meth:`~Path.is_symlink()`, - :meth:`~Path.is_block_device()`, :meth:`~Path.is_char_device()`, - :meth:`~Path.is_fifo()`, :meth:`~Path.is_socket()` now return ``False`` - instead of raising an exception for paths that contain characters - unrepresentable at the OS level. + :: + + >>> Path.home() + PosixPath('/home/antoine') + + .. versionadded:: 3.5 + + +.. method:: Path.expanduser() + + Return a new path with expanded ``~`` and ``~user`` constructs, + as returned by :meth:`os.path.expanduser`. If a home directory can't be + resolved, :exc:`RuntimeError` is raised. + + :: + + >>> p = PosixPath('~/films/Monty Python') + >>> p.expanduser() + PosixPath('/home/eric/films/Monty Python') + + .. versionadded:: 3.5 .. classmethod:: Path.cwd() @@ -893,23 +932,91 @@ call fails (for example because the path doesn't exist). PosixPath('/home/antoine/pathlib') -.. classmethod:: Path.home() +.. method:: Path.absolute() - Return a new path object representing the user's home directory (as - returned by :func:`os.path.expanduser` with ``~`` construct). If the home - directory can't be resolved, :exc:`RuntimeError` is raised. + Make the path absolute, without normalization or resolving symlinks. + Returns a new path object:: - :: + >>> p = Path('tests') + >>> p + PosixPath('tests') + >>> p.absolute() + PosixPath('/home/antoine/pathlib/tests') - >>> Path.home() - PosixPath('/home/antoine') - .. versionadded:: 3.5 +.. method:: Path.resolve(strict=False) + + Make the path absolute, resolving any symlinks. A new path object is + returned:: + + >>> p = Path() + >>> p + PosixPath('.') + >>> p.resolve() + PosixPath('/home/antoine/pathlib') + + "``..``" components are also eliminated (this is the only method to do so):: + + >>> p = Path('docs/../setup.py') + >>> p.resolve() + PosixPath('/home/antoine/pathlib/setup.py') + + If a path doesn't exist or a symlink loop is encountered, and *strict* is + ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is + resolved as far as possible and any remainder is appended without checking + whether it exists. + + .. versionchanged:: 3.6 + The *strict* parameter was added (pre-3.6 behavior is strict). + + .. versionchanged:: 3.13 + Symlink loops are treated like other errors: :exc:`OSError` is raised in + strict mode, and no exception is raised in non-strict mode. In previous + versions, :exc:`RuntimeError` is raised no matter the value of *strict*. + + +.. method:: Path.readlink() + + Return the path to which the symbolic link points (as returned by + :func:`os.readlink`):: + + >>> p = Path('mylink') + >>> p.symlink_to('setup.py') + >>> p.readlink() + PosixPath('setup.py') + + .. versionadded:: 3.9 + + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.readlink` is not + available. In previous versions, :exc:`NotImplementedError` was raised. + + +Querying file type and status +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionchanged:: 3.8 + + :meth:`~Path.exists`, :meth:`~Path.is_dir`, :meth:`~Path.is_file`, + :meth:`~Path.is_mount`, :meth:`~Path.is_symlink`, + :meth:`~Path.is_block_device`, :meth:`~Path.is_char_device`, + :meth:`~Path.is_fifo`, :meth:`~Path.is_socket` now return ``False`` + instead of raising an exception for paths that contain characters + unrepresentable at the OS level. + +.. versionchanged:: 3.14 + + The methods given above now return ``False`` instead of raising any + :exc:`OSError` exception from the operating system. In previous versions, + some kinds of :exc:`OSError` exception are raised, and others suppressed. + The new behaviour is consistent with :func:`os.path.exists`, + :func:`os.path.isdir`, etc. Use :meth:`~Path.stat` to retrieve the file + status without suppressing exceptions. .. method:: Path.stat(*, follow_symlinks=True) - Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. + Return an :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. The result is looked up at each call to this method. This method normally follows symlinks; to stat a symlink add the argument @@ -926,29 +1033,18 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.10 The *follow_symlinks* parameter was added. -.. method:: Path.chmod(mode, *, follow_symlinks=True) - - Change the file mode and permissions, like :func:`os.chmod`. - - This method normally follows symlinks. Some Unix flavours support changing - permissions on the symlink itself; on these platforms you may add the - argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. - :: +.. method:: Path.lstat() - >>> p = Path('setup.py') - >>> p.stat().st_mode - 33277 - >>> p.chmod(0o444) - >>> p.stat().st_mode - 33060 + Like :meth:`Path.stat` but, if the path points to a symbolic link, return + the symbolic link's information rather than its target's. - .. versionchanged:: 3.10 - The *follow_symlinks* parameter was added. .. method:: Path.exists(*, follow_symlinks=True) Return ``True`` if the path points to an existing file or directory. + ``False`` will be returned if the path is invalid, inaccessible or missing. + Use :meth:`Path.stat` to distinguish between these cases. This method normally follows symlinks; to check if a symlink exists, add the argument ``follow_symlinks=False``. @@ -967,94 +1063,16 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.12 The *follow_symlinks* parameter was added. -.. method:: Path.expanduser() - - Return a new path with expanded ``~`` and ``~user`` constructs, - as returned by :meth:`os.path.expanduser`. If a home directory can't be - resolved, :exc:`RuntimeError` is raised. - - :: - - >>> p = PosixPath('~/films/Monty Python') - >>> p.expanduser() - PosixPath('/home/eric/films/Monty Python') - - .. versionadded:: 3.5 - - -.. method:: Path.glob(pattern, *, case_sensitive=None, follow_symlinks=None) - - Glob the given relative *pattern* in the directory represented by this path, - yielding all matching files (of any kind):: - - >>> sorted(Path('.').glob('*.py')) - [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] - >>> sorted(Path('.').glob('*/*.py')) - [PosixPath('docs/conf.py')] - - Patterns are the same as for :mod:`fnmatch`, with the addition of "``**``" - which means "this directory and all subdirectories, recursively". In other - words, it enables recursive globbing:: - - >>> sorted(Path('.').glob('**/*.py')) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. - - .. tip:: - Set *follow_symlinks* to ``True`` or ``False`` to improve performance - of recursive globbing. - - This method calls :meth:`Path.is_dir` on the top-level directory and - propagates any :exc:`OSError` exception that is raised. Subsequent - :exc:`OSError` exceptions from scanning directories are suppressed. - - By default, or when the *case_sensitive* keyword-only argument is set to - ``None``, this method matches paths using platform-specific casing rules: - typically, case-sensitive on POSIX, and case-insensitive on Windows. - Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - - By default, or when the *follow_symlinks* keyword-only argument is set to - ``None``, this method follows symlinks except when expanding "``**``" - wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or - ``False`` to treat all symlinks as files. - - .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob - - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). - - .. versionchanged:: 3.12 - The *case_sensitive* parameter was added. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - .. versionchanged:: 3.13 - Return files and directories if *pattern* ends with "``**``". In - previous versions, only directories were returned. - - .. versionchanged:: 3.13 - The *pattern* parameter accepts a :term:`path-like object`. - -.. method:: Path.group(*, follow_symlinks=True) - Return the name of the group owning the file. :exc:`KeyError` is raised - if the file's gid isn't found in the system database. +.. method:: Path.is_file(*, follow_symlinks=True) - This method normally follows symlinks; to get the group of the symlink, add - the argument ``follow_symlinks=False``. + Return ``True`` if the path points to a regular file. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a regular file. Use :meth:`Path.stat` to + distinguish between these cases. - .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not - available. In previous versions, :exc:`NotImplementedError` was raised. + This method normally follows symlinks; to exclude symlinks, add the + argument ``follow_symlinks=False``. .. versionchanged:: 3.13 The *follow_symlinks* parameter was added. @@ -1062,11 +1080,10 @@ call fails (for example because the path doesn't exist). .. method:: Path.is_dir(*, follow_symlinks=True) - Return ``True`` if the path points to a directory, ``False`` if it points - to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a directory. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a directory. Use :meth:`Path.stat` to distinguish + between these cases. This method normally follows symlinks; to exclude symlinks to directories, add the argument ``follow_symlinks=False``. @@ -1075,19 +1092,12 @@ call fails (for example because the path doesn't exist). The *follow_symlinks* parameter was added. -.. method:: Path.is_file(*, follow_symlinks=True) - - Return ``True`` if the path points to a regular file, ``False`` if it - points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - This method normally follows symlinks; to exclude symlinks, add the - argument ``follow_symlinks=False``. +.. method:: Path.is_symlink() - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. + Return ``True`` if the path points to a symbolic link, even if that symlink + is broken. ``False`` will be returned if the path is invalid, inaccessible + or missing, or if it points to something other than a symbolic link. Use + :meth:`Path.stat` to distinguish between these cases. .. method:: Path.is_junction() @@ -1114,78 +1124,271 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.12 Windows support was added. +.. method:: Path.is_socket() -.. method:: Path.is_symlink() + Return ``True`` if the path points to a Unix socket. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a Unix socket. Use :meth:`Path.stat` to + distinguish between these cases. - Return ``True`` if the path points to a symbolic link, ``False`` otherwise. - ``False`` is also returned if the path doesn't exist; other errors (such - as permission errors) are propagated. +.. method:: Path.is_fifo() + Return ``True`` if the path points to a FIFO. ``False`` will be returned if + the path is invalid, inaccessible or missing, or if it points to something + other than a FIFO. Use :meth:`Path.stat` to distinguish between these + cases. -.. method:: Path.is_socket() - Return ``True`` if the path points to a Unix socket (or a symbolic link - pointing to a Unix socket), ``False`` if it points to another kind of file. +.. method:: Path.is_block_device() - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a block device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a block device. Use :meth:`Path.stat` to + distinguish between these cases. -.. method:: Path.is_fifo() +.. method:: Path.is_char_device() - Return ``True`` if the path points to a FIFO (or a symbolic link - pointing to a FIFO), ``False`` if it points to another kind of file. + Return ``True`` if the path points to a character device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a character device. Use :meth:`Path.stat` to + distinguish between these cases. - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. +.. method:: Path.samefile(other_path) -.. method:: Path.is_block_device() + Return whether this path points to the same file as *other_path*, which + can be either a Path object, or a string. The semantics are similar + to :func:`os.path.samefile` and :func:`os.path.samestat`. - Return ``True`` if the path points to a block device (or a symbolic link - pointing to a block device), ``False`` if it points to another kind of file. + An :exc:`OSError` can be raised if either file cannot be accessed for some + reason. - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + :: + >>> p = Path('spam') + >>> q = Path('eggs') + >>> p.samefile(q) + False + >>> p.samefile('spam') + True -.. method:: Path.is_char_device() + .. versionadded:: 3.5 - Return ``True`` if the path points to a character device (or a symbolic link - pointing to a character device), ``False`` if it points to another kind of file. - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. +Reading and writing files +^^^^^^^^^^^^^^^^^^^^^^^^^ -.. method:: Path.iterdir() +.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) - When the path points to a directory, yield path objects of the directory - contents:: + Open the file pointed to by the path, like the built-in :func:`open` + function does:: - >>> p = Path('docs') - >>> for child in p.iterdir(): child + >>> p = Path('setup.py') + >>> with p.open() as f: + ... f.readline() ... - PosixPath('docs/conf.py') - PosixPath('docs/_templates') - PosixPath('docs/make.bat') - PosixPath('docs/index.rst') - PosixPath('docs/_build') - PosixPath('docs/_static') - PosixPath('docs/Makefile') + '#!/usr/bin/env python3\n' - The children are yielded in arbitrary order, and the special entries - ``'.'`` and ``'..'`` are not included. If a file is removed from or added - to the directory after creating the iterator, whether a path object for - that file be included is unspecified. -.. method:: Path.walk(top_down=True, on_error=None, follow_symlinks=False) +.. method:: Path.read_text(encoding=None, errors=None, newline=None) - Generate the file names in a directory tree by walking the tree - either top-down or bottom-up. + Return the decoded contents of the pointed-to file as a string:: - For each directory in the directory tree rooted at *self* (including + >>> p = Path('my_text_file') + >>> p.write_text('Text file contents') + 18 + >>> p.read_text() + 'Text file contents' + + The file is opened and then closed. The optional parameters have the same + meaning as in :func:`open`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.13 + The *newline* parameter was added. + + +.. method:: Path.read_bytes() + + Return the binary contents of the pointed-to file as a bytes object:: + + >>> p = Path('my_binary_file') + >>> p.write_bytes(b'Binary file contents') + 20 + >>> p.read_bytes() + b'Binary file contents' + + .. versionadded:: 3.5 + + +.. method:: Path.write_text(data, encoding=None, errors=None, newline=None) + + Open the file pointed to in text mode, write *data* to it, and close the + file:: + + >>> p = Path('my_text_file') + >>> p.write_text('Text file contents') + 18 + >>> p.read_text() + 'Text file contents' + + An existing file of the same name is overwritten. The optional parameters + have the same meaning as in :func:`open`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.10 + The *newline* parameter was added. + + +.. method:: Path.write_bytes(data) + + Open the file pointed to in bytes mode, write *data* to it, and close the + file:: + + >>> p = Path('my_binary_file') + >>> p.write_bytes(b'Binary file contents') + 20 + >>> p.read_bytes() + b'Binary file contents' + + An existing file of the same name is overwritten. + + .. versionadded:: 3.5 + + +Reading directories +^^^^^^^^^^^^^^^^^^^ + +.. method:: Path.iterdir() + + When the path points to a directory, yield path objects of the directory + contents:: + + >>> p = Path('docs') + >>> for child in p.iterdir(): child + ... + PosixPath('docs/conf.py') + PosixPath('docs/_templates') + PosixPath('docs/make.bat') + PosixPath('docs/index.rst') + PosixPath('docs/_build') + PosixPath('docs/_static') + PosixPath('docs/Makefile') + + The children are yielded in arbitrary order, and the special entries + ``'.'`` and ``'..'`` are not included. If a file is removed from or added + to the directory after creating the iterator, it is unspecified whether + a path object for that file is included. + + If the path is not a directory or otherwise inaccessible, :exc:`OSError` is + raised. + + +.. method:: Path.scandir() + + When the path points to a directory, return an iterator of + :class:`os.DirEntry` objects corresponding to entries in the directory. The + returned iterator supports the :term:`context manager` protocol. It is + implemented using :func:`os.scandir` and gives the same guarantees. + + Using :meth:`~Path.scandir` instead of :meth:`~Path.iterdir` can + significantly increase the performance of code that also needs file type or + file attribute information, because :class:`os.DirEntry` objects expose + this information if the operating system provides it when scanning a + directory. + + The following example displays the names of subdirectories. The + ``entry.is_dir()`` check will generally not make an additional system call:: + + >>> p = Path('docs') + >>> with p.scandir() as entries: + ... for entry in entries: + ... if entry.is_dir(): + ... entry.name + ... + '_templates' + '_build' + '_static' + + .. versionadded:: 3.14 + + +.. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) + + Glob the given relative *pattern* in the directory represented by this path, + yielding all matching files (of any kind):: + + >>> sorted(Path('.').glob('*.py')) + [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] + >>> sorted(Path('.').glob('*/*.py')) + [PosixPath('docs/conf.py')] + >>> sorted(Path('.').glob('**/*.py')) + [PosixPath('build/lib/pathlib.py'), + PosixPath('docs/conf.py'), + PosixPath('pathlib.py'), + PosixPath('setup.py'), + PosixPath('test_pathlib.py')] + + .. seealso:: + :ref:`pathlib-pattern-language` documentation. + + By default, or when the *case_sensitive* keyword-only argument is set to + ``None``, this method matches paths using platform-specific casing rules: + typically, case-sensitive on POSIX, and case-insensitive on Windows. + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + + By default, or when the *recurse_symlinks* keyword-only argument is set to + ``False``, this method follows symlinks except when expanding "``**``" + wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. + + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob + + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *recurse_symlinks* parameter was added. + + .. versionchanged:: 3.13 + The *pattern* parameter accepts a :term:`path-like object`. + + .. versionchanged:: 3.13 + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. In previous versions, such exceptions are suppressed in many + cases, but not all. + + +.. method:: Path.rglob(pattern, *, case_sensitive=None, recurse_symlinks=False) + + Glob the given relative *pattern* recursively. This is like calling + :func:`Path.glob` with "``**/``" added in front of the *pattern*. + + .. seealso:: + :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. + + .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob + + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *recurse_symlinks* parameter was added. + + .. versionchanged:: 3.13 + The *pattern* parameter accepts a :term:`path-like object`. + + +.. method:: Path.walk(top_down=True, on_error=None, follow_symlinks=False) + + Generate the file names in a directory tree by walking the tree + either top-down or bottom-up. + + For each directory in the directory tree rooted at *self* (including *self* but excluding '.' and '..'), the method yields a 3-tuple of ``(dirpath, dirnames, filenames)``. @@ -1211,7 +1414,7 @@ call fails (for example because the path doesn't exist). This can be used to prune the search, or to impose a specific order of visiting, or even to inform :meth:`Path.walk` about directories the caller creates or renames before it resumes :meth:`Path.walk` again. Modifying *dirnames* when - *top_down* is false has no effect on the behavior of :meth:`Path.walk()` since the + *top_down* is false has no effect on the behavior of :meth:`Path.walk` since the directories in *dirnames* have already been generated by the time *dirnames* is yielded to the caller. @@ -1275,22 +1478,27 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.12 -.. method:: Path.lchmod(mode) - Like :meth:`Path.chmod` but, if the path points to a symbolic link, the - symbolic link's mode is changed rather than its target's. +Creating files and directories +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. method:: Path.touch(mode=0o666, exist_ok=True) -.. method:: Path.lstat() + Create a file at this given path. If *mode* is given, it is combined + with the process's ``umask`` value to determine the file mode and access + flags. If the file already exists, the function succeeds when *exist_ok* + is true (and its modification time is updated to the current time), + otherwise :exc:`FileExistsError` is raised. - Like :meth:`Path.stat` but, if the path points to a symbolic link, return - the symbolic link's information rather than its target's. + .. seealso:: + The :meth:`~Path.open`, :meth:`~Path.write_text` and + :meth:`~Path.write_bytes` methods are often used to create files. .. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) Create a new directory at this given path. If *mode* is given, it is - combined with the process' ``umask`` value to determine the file mode + combined with the process's ``umask`` value to determine the file mode and access flags. If the path already exists, :exc:`FileExistsError` is raised. @@ -1312,87 +1520,97 @@ call fails (for example because the path doesn't exist). The *exist_ok* parameter was added. -.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) - - Open the file pointed to by the path, like the built-in :func:`open` - function does:: +.. method:: Path.symlink_to(target, target_is_directory=False) - >>> p = Path('setup.py') - >>> with p.open() as f: - ... f.readline() - ... - '#!/usr/bin/env python3\n' + Make this path a symbolic link pointing to *target*. + On Windows, a symlink represents either a file or a directory, and does not + morph to the target dynamically. If the target is present, the type of the + symlink will be created to match. Otherwise, the symlink will be created + as a directory if *target_is_directory* is true or a file symlink (the + default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. -.. method:: Path.owner(*, follow_symlinks=True) + :: - Return the name of the user owning the file. :exc:`KeyError` is raised - if the file's uid isn't found in the system database. + >>> p = Path('mylink') + >>> p.symlink_to('setup.py') + >>> p.resolve() + PosixPath('/home/antoine/pathlib/setup.py') + >>> p.stat().st_size + 956 + >>> p.lstat().st_size + 8 - This method normally follows symlinks; to get the owner of the symlink, add - the argument ``follow_symlinks=False``. + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.symlink`'s. .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if the :mod:`pwd` module is not + Raises :exc:`UnsupportedOperation` if :func:`os.symlink` is not available. In previous versions, :exc:`NotImplementedError` was raised. - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. +.. method:: Path.hardlink_to(target) -.. method:: Path.read_bytes() + Make this path a hard link to the same file as *target*. - Return the binary contents of the pointed-to file as a bytes object:: + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.link`'s. - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' + .. versionadded:: 3.10 - .. versionadded:: 3.5 + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.link` is not + available. In previous versions, :exc:`NotImplementedError` was raised. -.. method:: Path.read_text(encoding=None, errors=None, newline=None) +Copying, moving and deleting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Return the decoded contents of the pointed-to file as a string:: +.. method:: Path.copy(target, *, follow_symlinks=True, dirs_exist_ok=False, \ + preserve_metadata=False) - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' + Copy this file or directory tree to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. - The file is opened and then closed. The optional parameters have the same - meaning as in :func:`open`. + If the source is a file, the target will be replaced if it is an existing + file. If the source is a symlink and *follow_symlinks* is true (the + default), the symlink's target is copied. Otherwise, the symlink is + recreated at the destination. - .. versionadded:: 3.5 + If the source is a directory and *dirs_exist_ok* is false (the default), a + :exc:`FileExistsError` is raised if the target is an existing directory. + If *dirs_exists_ok* is true, the copying operation will overwrite + existing files within the destination tree with corresponding files + from the source tree. - .. versionchanged:: 3.13 - The *newline* parameter was added. + If *preserve_metadata* is false (the default), only directory structures + and file data are guaranteed to be copied. Set *preserve_metadata* to true + to ensure that file and directory permissions, flags, last access and + modification times, and extended attributes are copied where supported. + This argument has no effect when copying files on Windows (where + metadata is always preserved). -.. method:: Path.readlink() + .. versionadded:: 3.14 - Return the path to which the symbolic link points (as returned by - :func:`os.readlink`):: - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.readlink() - PosixPath('setup.py') +.. method:: Path.copy_into(target_dir, *, follow_symlinks=True, \ + dirs_exist_ok=False, preserve_metadata=False) - .. versionadded:: 3.9 + Copy this file or directory tree into the given *target_dir*, which should + be an existing directory. Other arguments are handled identically to + :meth:`Path.copy`. Returns a new :class:`!Path` instance pointing to the + copy. - .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.readlink` is not - available. In previous versions, :exc:`NotImplementedError` was raised. + .. versionadded:: 3.14 .. method:: Path.rename(target) - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. On Unix, if *target* exists and is a file, - it will be replaced silently if the user has permission. + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. On Unix, if *target* exists + and is a file, it will be replaced silently if the user has permission. On Windows, if *target* exists, :exc:`FileExistsError` will be raised. *target* can be either a string or another path object:: @@ -1406,290 +1624,306 @@ call fails (for example because the path doesn't exist). 'some text' The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. + relative to the current working directory, *not* the directory of the + :class:`!Path` object. It is implemented in terms of :func:`os.rename` and gives the same guarantees. .. versionchanged:: 3.8 - Added return value, return the new Path instance. + Added return value, return the new :class:`!Path` instance. .. method:: Path.replace(target) - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. If *target* points to an existing file or - empty directory, it will be unconditionally replaced. + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. If *target* points to an + existing file or empty directory, it will be unconditionally replaced. The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. + relative to the current working directory, *not* the directory of the + :class:`!Path` object. .. versionchanged:: 3.8 - Added return value, return the new Path instance. + Added return value, return the new :class:`!Path` instance. -.. method:: Path.absolute() +.. method:: Path.move(target) - Make the path absolute, without normalization or resolving symlinks. - Returns a new path object:: - - >>> p = Path('tests') - >>> p - PosixPath('tests') - >>> p.absolute() - PosixPath('/home/antoine/pathlib/tests') - - -.. method:: Path.resolve(strict=False) - - Make the path absolute, resolving any symlinks. A new path object is - returned:: + Move this file or directory tree to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. - >>> p = Path() - >>> p - PosixPath('.') - >>> p.resolve() - PosixPath('/home/antoine/pathlib') - - "``..``" components are also eliminated (this is the only method to do so):: + If the *target* doesn't exist it will be created. If both this path and the + *target* are existing files, then the target is overwritten. If both paths + point to the same file or directory, or the *target* is a non-empty + directory, then :exc:`OSError` is raised. - >>> p = Path('docs/../setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - - If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is - resolved as far as possible and any remainder is appended without checking - whether it exists. + If both paths are on the same filesystem, the move is performed with + :func:`os.replace`. Otherwise, this path is copied (preserving metadata and + symlinks) and then deleted. - .. versionchanged:: 3.6 - The *strict* parameter was added (pre-3.6 behavior is strict). + .. versionadded:: 3.14 - .. versionchanged:: 3.13 - Symlink loops are treated like other errors: :exc:`OSError` is raised in - strict mode, and no exception is raised in non-strict mode. In previous - versions, :exc:`RuntimeError` is raised no matter the value of *strict*. -.. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) +.. method:: Path.move_into(target_dir) - Glob the given relative *pattern* recursively. This is like calling - :func:`Path.glob` with "``**/``" added in front of the *pattern*, where - *patterns* are the same as for :mod:`fnmatch`:: + Move this file or directory tree into the given *target_dir*, which should + be an existing directory. Returns a new :class:`!Path` instance pointing to + the moved path. - >>> sorted(Path().rglob("*.py")) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] + .. versionadded:: 3.14 - By default, or when the *case_sensitive* keyword-only argument is set to - ``None``, this method matches paths using platform-specific casing rules: - typically, case-sensitive on POSIX, and case-insensitive on Windows. - Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - By default, or when the *follow_symlinks* keyword-only argument is set to - ``None``, this method follows symlinks except when expanding "``**``" - wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or - ``False`` to treat all symlinks as files. +.. method:: Path.unlink(missing_ok=False) - .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob + Remove this file or symbolic link. If the path points to a directory, + use :func:`Path.rmdir` instead. - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). + If *missing_ok* is false (the default), :exc:`FileNotFoundError` is + raised if the path does not exist. - .. versionchanged:: 3.12 - The *case_sensitive* parameter was added. + If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be + ignored (same behavior as the POSIX ``rm -f`` command). - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. + .. versionchanged:: 3.8 + The *missing_ok* parameter was added. - .. versionchanged:: 3.13 - The *pattern* parameter accepts a :term:`path-like object`. .. method:: Path.rmdir() Remove this directory. The directory must be empty. -.. method:: Path.samefile(other_path) - - Return whether this path points to the same file as *other_path*, which - can be either a Path object, or a string. The semantics are similar - to :func:`os.path.samefile` and :func:`os.path.samestat`. - - An :exc:`OSError` can be raised if either file cannot be accessed for some - reason. - - :: - - >>> p = Path('spam') - >>> q = Path('eggs') - >>> p.samefile(q) - False - >>> p.samefile('spam') - True - - .. versionadded:: 3.5 +Permissions and ownership +^^^^^^^^^^^^^^^^^^^^^^^^^ +.. method:: Path.owner(*, follow_symlinks=True) -.. method:: Path.symlink_to(target, target_is_directory=False) - - Make this path a symbolic link pointing to *target*. - - On Windows, a symlink represents either a file or a directory, and does not - morph to the target dynamically. If the target is present, the type of the - symlink will be created to match. Otherwise, the symlink will be created - as a directory if *target_is_directory* is ``True`` or a file symlink (the - default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. - - :: - - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - >>> p.stat().st_size - 956 - >>> p.lstat().st_size - 8 + Return the name of the user owning the file. :exc:`KeyError` is raised + if the file's user identifier (UID) isn't found in the system database. - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.symlink`'s. + This method normally follows symlinks; to get the owner of the symlink, add + the argument ``follow_symlinks=False``. .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.symlink` is not - available. In previous versions, :exc:`NotImplementedError` was raised. + Raises :exc:`UnsupportedOperation` if the :mod:`pwd` module is not + available. In earlier versions, :exc:`NotImplementedError` was raised. + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. -.. method:: Path.hardlink_to(target) - Make this path a hard link to the same file as *target*. +.. method:: Path.group(*, follow_symlinks=True) - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.link`'s. + Return the name of the group owning the file. :exc:`KeyError` is raised + if the file's group identifier (GID) isn't found in the system database. - .. versionadded:: 3.10 + This method normally follows symlinks; to get the group of the symlink, add + the argument ``follow_symlinks=False``. .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.link` is not - available. In previous versions, :exc:`NotImplementedError` was raised. - - -.. method:: Path.touch(mode=0o666, exist_ok=True) + Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not + available. In earlier versions, :exc:`NotImplementedError` was raised. - Create a file at this given path. If *mode* is given, it is combined - with the process' ``umask`` value to determine the file mode and access - flags. If the file already exists, the function succeeds if *exist_ok* - is true (and its modification time is updated to the current time), - otherwise :exc:`FileExistsError` is raised. + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. -.. method:: Path.unlink(missing_ok=False) - - Remove this file or symbolic link. If the path points to a directory, - use :func:`Path.rmdir` instead. +.. method:: Path.chmod(mode, *, follow_symlinks=True) - If *missing_ok* is false (the default), :exc:`FileNotFoundError` is - raised if the path does not exist. + Change the file mode and permissions, like :func:`os.chmod`. - If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be - ignored (same behavior as the POSIX ``rm -f`` command). + This method normally follows symlinks. Some Unix flavours support changing + permissions on the symlink itself; on these platforms you may add the + argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. - .. versionchanged:: 3.8 - The *missing_ok* parameter was added. + :: + >>> p = Path('setup.py') + >>> p.stat().st_mode + 33277 + >>> p.chmod(0o444) + >>> p.stat().st_mode + 33060 -.. method:: Path.write_bytes(data) + .. versionchanged:: 3.10 + The *follow_symlinks* parameter was added. - Open the file pointed to in bytes mode, write *data* to it, and close the - file:: - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' +.. method:: Path.lchmod(mode) - An existing file of the same name is overwritten. + Like :meth:`Path.chmod` but, if the path points to a symbolic link, the + symbolic link's mode is changed rather than its target's. - .. versionadded:: 3.5 +.. _pathlib-pattern-language: -.. method:: Path.write_text(data, encoding=None, errors=None, newline=None) +Pattern language +---------------- - Open the file pointed to in text mode, write *data* to it, and close the - file:: +The following wildcards are supported in patterns for +:meth:`~PurePath.full_match`, :meth:`~Path.glob` and :meth:`~Path.rglob`: - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' +``**`` (entire segment) + Matches any number of file or directory segments, including zero. +``*`` (entire segment) + Matches one file or directory segment. +``*`` (part of a segment) + Matches any number of non-separator characters, including zero. +``?`` + Matches one non-separator character. +``[seq]`` + Matches one character in *seq*. +``[!seq]`` + Matches one character not in *seq*. - An existing file of the same name is overwritten. The optional parameters - have the same meaning as in :func:`open`. +For a literal match, wrap the meta-characters in brackets. +For example, ``"[?]"`` matches the character ``"?"``. - .. versionadded:: 3.5 +The "``**``" wildcard enables recursive globbing. A few examples: - .. versionchanged:: 3.10 - The *newline* parameter was added. +========================= =========================================== +Pattern Meaning +========================= =========================================== +"``**/*``" Any path with at least one segment. +"``**/*.py``" Any path with a final segment ending "``.py``". +"``assets/**``" Any path starting with "``assets/``". +"``assets/**/*``" Any path starting with "``assets/``", excluding "``assets/``" itself. +========================= =========================================== -Correspondence to tools in the :mod:`os` module ------------------------------------------------ +.. note:: + Globbing with the "``**``" wildcard visits every directory in the tree. + Large directory trees may take a long time to search. + +.. versionchanged:: 3.13 + Globbing with a pattern that ends with "``**``" returns both files and + directories. In previous versions, only directories were returned. + +In :meth:`Path.glob` and :meth:`~Path.rglob`, a trailing slash may be added to +the pattern to match only directories. + +.. versionchanged:: 3.11 + Globbing with a pattern that ends with a pathname components separator + (:data:`~os.sep` or :data:`~os.altsep`) returns only directories. + + +Comparison to the :mod:`glob` module +------------------------------------ + +The patterns accepted and results generated by :meth:`Path.glob` and +:meth:`Path.rglob` differ slightly from those by the :mod:`glob` module: + +1. Files beginning with a dot are not special in pathlib. This is + like passing ``include_hidden=True`` to :func:`glob.glob`. +2. "``**``" pattern components are always recursive in pathlib. This is like + passing ``recursive=True`` to :func:`glob.glob`. +3. "``**``" pattern components do not follow symlinks by default in pathlib. + This behaviour has no equivalent in :func:`glob.glob`, but you can pass + ``recurse_symlinks=True`` to :meth:`Path.glob` for compatible behaviour. +4. Like all :class:`PurePath` and :class:`Path` objects, the values returned + from :meth:`Path.glob` and :meth:`Path.rglob` don't include trailing + slashes. +5. The values returned from pathlib's ``path.glob()`` and ``path.rglob()`` + include the *path* as a prefix, unlike the results of + ``glob.glob(root_dir=path)``. +6. The values returned from pathlib's ``path.glob()`` and ``path.rglob()`` + may include *path* itself, for example when globbing "``**``", whereas the + results of ``glob.glob(root_dir=path)`` never include an empty string that + would correspond to *path*. + + +Comparison to the :mod:`os` and :mod:`os.path` modules +------------------------------------------------------ + +pathlib implements path operations using :class:`PurePath` and :class:`Path` +objects, and so it's said to be *object-oriented*. On the other hand, the +:mod:`os` and :mod:`os.path` modules supply functions that work with low-level +``str`` and ``bytes`` objects, which is a more *procedural* approach. Some +users consider the object-oriented style to be more readable. + +Many functions in :mod:`os` and :mod:`os.path` support ``bytes`` paths and +:ref:`paths relative to directory descriptors `. These features aren't +available in pathlib. + +Python's ``str`` and ``bytes`` types, and portions of the :mod:`os` and +:mod:`os.path` modules, are written in C and are very speedy. pathlib is +written in pure Python and is often slower, but rarely slow enough to matter. + +pathlib's path normalization is slightly more opinionated and consistent than +:mod:`os.path`. For example, whereas :func:`os.path.abspath` eliminates +"``..``" segments from a path, which may change its meaning if symlinks are +involved, :meth:`Path.absolute` preserves these segments for greater safety. + +pathlib's path normalization may render it unsuitable for some applications: + +1. pathlib normalizes ``Path("my_folder/")`` to ``Path("my_folder")``, which + changes a path's meaning when supplied to various operating system APIs and + command-line utilities. Specifically, the absence of a trailing separator + may allow the path to be resolved as either a file or directory, rather + than a directory only. +2. pathlib normalizes ``Path("./my_program")`` to ``Path("my_program")``, + which changes a path's meaning when used as an executable search path, such + as in a shell or when spawning a child process. Specifically, the absence + of a separator in the path may force it to be looked up in :envvar:`PATH` + rather than the current directory. + +As a consequence of these differences, pathlib is not a drop-in replacement +for :mod:`os.path`. + + +Corresponding tools +^^^^^^^^^^^^^^^^^^^ Below is a table mapping various :mod:`os` functions to their corresponding :class:`PurePath`/:class:`Path` equivalent. -.. note:: - - Not all pairs of functions/methods below are equivalent. Some of them, - despite having some overlapping use-cases, have different semantics. They - include :func:`os.path.abspath` and :meth:`Path.absolute`, - :func:`os.path.relpath` and :meth:`PurePath.relative_to`. - -==================================== ============================== -:mod:`os` and :mod:`os.path` :mod:`pathlib` -==================================== ============================== -:func:`os.path.abspath` :meth:`Path.absolute` [#]_ -:func:`os.path.realpath` :meth:`Path.resolve` -:func:`os.chmod` :meth:`Path.chmod` -:func:`os.mkdir` :meth:`Path.mkdir` -:func:`os.makedirs` :meth:`Path.mkdir` -:func:`os.rename` :meth:`Path.rename` -:func:`os.replace` :meth:`Path.replace` -:func:`os.rmdir` :meth:`Path.rmdir` -:func:`os.remove`, :func:`os.unlink` :meth:`Path.unlink` -:func:`os.getcwd` :func:`Path.cwd` -:func:`os.path.exists` :meth:`Path.exists` -:func:`os.path.expanduser` :meth:`Path.expanduser` and - :meth:`Path.home` -:func:`os.listdir` :meth:`Path.iterdir` -:func:`os.walk` :meth:`Path.walk` -:func:`os.path.isdir` :meth:`Path.is_dir` -:func:`os.path.isfile` :meth:`Path.is_file` -:func:`os.path.islink` :meth:`Path.is_symlink` -:func:`os.link` :meth:`Path.hardlink_to` -:func:`os.symlink` :meth:`Path.symlink_to` -:func:`os.readlink` :meth:`Path.readlink` -:func:`os.path.relpath` :meth:`PurePath.relative_to` [#]_ -:func:`os.stat` :meth:`Path.stat`, - :meth:`Path.owner`, - :meth:`Path.group` -:func:`os.path.isabs` :meth:`PurePath.is_absolute` -:func:`os.path.join` :func:`PurePath.joinpath` -:func:`os.path.basename` :attr:`PurePath.name` -:func:`os.path.dirname` :attr:`PurePath.parent` -:func:`os.path.samefile` :meth:`Path.samefile` -:func:`os.path.splitext` :attr:`PurePath.stem` and - :attr:`PurePath.suffix` -==================================== ============================== +===================================== ============================================== +:mod:`os` and :mod:`os.path` :mod:`pathlib` +===================================== ============================================== +:func:`os.path.dirname` :attr:`PurePath.parent` +:func:`os.path.basename` :attr:`PurePath.name` +:func:`os.path.splitext` :attr:`PurePath.stem`, :attr:`PurePath.suffix` +:func:`os.path.join` :meth:`PurePath.joinpath` +:func:`os.path.isabs` :meth:`PurePath.is_absolute` +:func:`os.path.relpath` :meth:`PurePath.relative_to` [1]_ +:func:`os.path.expanduser` :meth:`Path.expanduser` [2]_ +:func:`os.path.realpath` :meth:`Path.resolve` +:func:`os.path.abspath` :meth:`Path.absolute` [3]_ +:func:`os.path.exists` :meth:`Path.exists` +:func:`os.path.isfile` :meth:`Path.is_file` +:func:`os.path.isdir` :meth:`Path.is_dir` +:func:`os.path.islink` :meth:`Path.is_symlink` +:func:`os.path.isjunction` :meth:`Path.is_junction` +:func:`os.path.ismount` :meth:`Path.is_mount` +:func:`os.path.samefile` :meth:`Path.samefile` +:func:`os.getcwd` :meth:`Path.cwd` +:func:`os.stat` :meth:`Path.stat` +:func:`os.lstat` :meth:`Path.lstat` +:func:`os.listdir` :meth:`Path.iterdir` +:func:`os.walk` :meth:`Path.walk` [4]_ +:func:`os.mkdir`, :func:`os.makedirs` :meth:`Path.mkdir` +:func:`os.link` :meth:`Path.hardlink_to` +:func:`os.symlink` :meth:`Path.symlink_to` +:func:`os.readlink` :meth:`Path.readlink` +:func:`os.rename` :meth:`Path.rename` +:func:`os.replace` :meth:`Path.replace` +:func:`os.remove`, :func:`os.unlink` :meth:`Path.unlink` +:func:`os.rmdir` :meth:`Path.rmdir` +:func:`os.chmod` :meth:`Path.chmod` +:func:`os.lchmod` :meth:`Path.lchmod` +===================================== ============================================== .. rubric:: Footnotes -.. [#] :func:`os.path.abspath` normalizes the resulting path, which may change its meaning in the presence of symlinks, while :meth:`Path.absolute` does not. -.. [#] :meth:`PurePath.relative_to` requires ``self`` to be the subpath of the argument, but :func:`os.path.relpath` does not. +.. [1] :func:`os.path.relpath` calls :func:`~os.path.abspath` to make paths + absolute and remove "``..``" parts, whereas :meth:`PurePath.relative_to` + is a lexical operation that raises :exc:`ValueError` when its inputs' + anchors differ (e.g. if one path is absolute and the other relative.) +.. [2] :func:`os.path.expanduser` returns the path unchanged if the home + directory can't be resolved, whereas :meth:`Path.expanduser` raises + :exc:`RuntimeError`. +.. [3] :func:`os.path.abspath` removes "``..``" components without resolving + symlinks, which may change the meaning of the path, whereas + :meth:`Path.absolute` leaves any "``..``" components in the path. +.. [4] :func:`os.walk` always follows symlinks when categorizing paths into + *dirnames* and *filenames*, whereas :meth:`Path.walk` categorizes all + symlinks into *filenames* when *follow_symlinks* is false (the default.) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index cb17acfb367619..6c099b22b38c21 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -49,7 +49,7 @@ You can then step through the code following this statement, and continue running without the debugger using the :pdbcmd:`continue` command. .. versionchanged:: 3.7 - The built-in :func:`breakpoint()`, when called with defaults, can be used + The built-in :func:`breakpoint`, when called with defaults, can be used instead of ``import pdb; pdb.set_trace()``. :: @@ -62,8 +62,8 @@ running without the debugger using the :pdbcmd:`continue` command. The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug mode:: - > ...(3)double() - -> return x * 2 + > ...(2)double() + -> breakpoint() (Pdb) p x 3 (Pdb) continue @@ -123,6 +123,11 @@ The typical usage to inspect a crashed program is:: 0 (Pdb) +.. versionchanged:: 3.13 + The implementation of :pep:`667` means that name assignments made via ``pdb`` + will immediately affect the active scope, even when running inside an + :term:`optimized scope`. + The module defines the following functions; each enters the debugger in a slightly different way: @@ -154,16 +159,25 @@ slightly different way: is entered. -.. function:: set_trace(*, header=None) +.. function:: set_trace(*, header=None, commands=None) Enter the debugger at the calling stack frame. This is useful to hard-code a breakpoint at a given point in a program, even if the code is not otherwise being debugged (e.g. when an assertion fails). If given, *header* is printed to the console just before debugging begins. + The *commands* argument, if given, is a list of commands to execute + when the debugger starts. + .. versionchanged:: 3.7 The keyword-only argument *header*. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. + + .. versionadded:: 3.14 + The *commands* argument. .. function:: post_mortem(traceback=None) @@ -184,7 +198,7 @@ The ``run*`` functions and :func:`set_trace` are aliases for instantiating the access further features, you have to do this yourself: .. class:: Pdb(completekey='tab', stdin=None, stdout=None, skip=None, \ - nosigint=False, readrc=True) + nosigint=False, readrc=True, mode=None) :class:`Pdb` is the debugger class. @@ -203,6 +217,13 @@ access further features, you have to do this yourself: The *readrc* argument defaults to true and controls whether Pdb will load .pdbrc files from the filesystem. + The *mode* argument specifies how the debugger was invoked. + It impacts the workings of some debugger commands. + Valid values are ``'inline'`` (used by the breakpoint() builtin), + ``'cli'`` (used by the command line invocation) + or ``None`` (for backwards compatible behaviour, as before the *mode* + argument was added). + Example call to enable tracing with *skip*:: import pdb; pdb.Pdb(skip=['django.*']).set_trace() @@ -219,6 +240,9 @@ access further features, you have to do this yourself: .. versionchanged:: 3.6 The *readrc* argument. + .. versionadded:: 3.14 + Added the *mode* argument. + .. method:: run(statement, globals=None, locals=None) runeval(expression, globals=None, locals=None) runcall(function, *args, **kwds) @@ -282,25 +306,28 @@ There are three preset *convenience variables*: .. versionadded:: 3.12 + Added the *convenience variable* feature. + .. index:: pair: .pdbrc; file triple: debugger; configuration; file If a file :file:`.pdbrc` exists in the user's home directory or in the current directory, it is read with ``'utf-8'`` encoding and executed as if it had been -typed at the debugger prompt. This is particularly useful for aliases. If both +typed at the debugger prompt, with the exception that empty lines and lines +starting with ``#`` are ignored. This is particularly useful for aliases. If both files exist, the one in the home directory is read first and aliases defined there can be overridden by the local file. -.. versionchanged:: 3.11 - :file:`.pdbrc` is now read with ``'utf-8'`` encoding. Previously, it was read - with the system locale encoding. - .. versionchanged:: 3.2 :file:`.pdbrc` can now contain commands that continue debugging, such as :pdbcmd:`continue` or :pdbcmd:`next`. Previously, these commands had no effect. +.. versionchanged:: 3.11 + :file:`.pdbrc` is now read with ``'utf-8'`` encoding. Previously, it was read + with the system locale encoding. + .. pdbcommand:: h(elp) [command] @@ -310,11 +337,17 @@ can be overridden by the local file. argument must be an identifier, ``help exec`` must be entered to get help on the ``!`` command. -.. pdbcommand:: w(here) +.. pdbcommand:: w(here) [count] - Print a stack trace, with the most recent frame at the bottom. An arrow (``>``) + Print a stack trace, with the most recent frame at the bottom. if *count* + is 0, print the current frame entry. If *count* is negative, print the least + recent - *count* frames. If *count* is positive, print the most recent + *count* frames. An arrow (``>``) indicates the current frame, which determines the context of most commands. + .. versionchanged:: 3.14 + *count* argument is added. + .. pdbcommand:: d(own) [count] Move the current frame *count* (default one) levels down in the stack trace @@ -327,12 +360,16 @@ can be overridden by the local file. .. pdbcommand:: b(reak) [([filename:]lineno | function) [, condition]] - With a *lineno* argument, set a break there in the current file. With a - *function* argument, set a break at the first executable statement within - that function. The line number may be prefixed with a filename and a colon, - to specify a breakpoint in another file (probably one that hasn't been loaded - yet). The file is searched on :data:`sys.path`. Note that each breakpoint - is assigned a number to which all the other breakpoint commands refer. + With a *lineno* argument, set a break at line *lineno* in the current file. + The line number may be prefixed with a *filename* and a colon, + to specify a breakpoint in another file (possibly one that hasn't been loaded + yet). The file is searched on :data:`sys.path`. Acceptable forms of *filename* + are ``/abspath/to/file.py``, ``relpath/file.py``, ``module`` and + ``package.module``. + + With a *function* argument, set a break at the first executable statement within + that function. *function* can be any expression that evaluates to a function + in the current namespace. If a second argument is present, it is an expression which must evaluate to true before the breakpoint is honored. @@ -341,6 +378,9 @@ can be overridden by the local file. of times that breakpoint has been hit, the current ignore count, and the associated condition if any. + Each breakpoint is assigned a number to which all the other + breakpoint commands refer. + .. pdbcommand:: tbreak [([filename:]lineno | function) [, condition]] Temporary breakpoint, which is removed automatically when it is first hit. @@ -399,17 +439,20 @@ can be overridden by the local file. Specifying any command resuming execution (currently :pdbcmd:`continue`, :pdbcmd:`step`, :pdbcmd:`next`, - :pdbcmd:`return`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) + :pdbcmd:`return`, :pdbcmd:`until`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) terminates the command list (as if that command was immediately followed by end). This is because any time you resume execution (even with a simple next or step), you may encounter another breakpoint—which could have its own command list, leading to ambiguities about which list to execute. - If you use the ``silent`` command in the command list, the usual message about - stopping at a breakpoint is not printed. This may be desirable for breakpoints - that are to print a specific message and then continue. If none of the other - commands print anything, you see no sign that the breakpoint was reached. + If the list of commands contains the ``silent`` command, or a command that + resumes execution, then the breakpoint message containing information about + the frame is not displayed. + + .. versionchanged:: 3.14 + Frame information will not be displayed if a command that resumes execution + is present in the command list. .. pdbcommand:: s(tep) @@ -568,18 +611,17 @@ can be overridden by the local file. .. pdbcommand:: interact - Start an interactive interpreter (using the :mod:`code` module) whose global - namespace contains all the (global and local) names found in the current - scope. Use ``exit()`` or ``quit()`` to exit the interpreter and return to - the debugger. + Start an interactive interpreter (using the :mod:`code` module) in a new + global namespace initialised from the local and global namespaces for the + current scope. Use ``exit()`` or ``quit()`` to exit the interpreter and + return to the debugger. .. note:: - Because interact creates a new global namespace with the current global - and local namespace for execution, assignment to variables will not - affect the original namespaces. - However, modification to the mutable objects will be reflected in the - original namespaces. + As ``interact`` creates a new dedicated namespace for code execution, + assignments to variables will not affect the original namespaces. + However, modifications to any referenced mutable objects will be reflected + in the original namespaces as usual. .. versionadded:: 3.2 @@ -646,6 +688,10 @@ can be overridden by the local file. History, breakpoints, actions and debugger options are preserved. :pdbcmd:`restart` is an alias for :pdbcmd:`run`. + .. versionchanged:: 3.14 + :pdbcmd:`run` and :pdbcmd:`restart` commands are disabled when the + debugger is invoked in ``'inline'`` mode. + .. pdbcommand:: q(uit) Quit from the debugger. The program being executed is aborted. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index acada092afb679..71fe3743c5968d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -1,5 +1,5 @@ -:mod:`pickle` --- Python object serialization -============================================= +:mod:`!pickle` --- Python object serialization +============================================== .. module:: pickle :synopsis: Convert Python objects to streams of bytes and back. @@ -156,13 +156,14 @@ to read the pickle produced. * Protocol version 4 was added in Python 3.4. It adds support for very large objects, pickling more kinds of objects, and some data format - optimizations. It is the default protocol starting with Python 3.8. + optimizations. This was the default protocol in Python 3.8--3.13. Refer to :pep:`3154` for information about improvements brought by protocol 4. * Protocol version 5 was added in Python 3.8. It adds support for out-of-band - data and speedup for in-band data. Refer to :pep:`574` for information about - improvements brought by protocol 5. + data and speedup for in-band data. It is the default protocol starting with + Python 3.14. Refer to :pep:`574` for information about improvements brought + by protocol 5. .. note:: Serialization is a more primitive notion than persistence; although @@ -199,8 +200,10 @@ The :mod:`pickle` module provides the following constants: An integer, the default :ref:`protocol version ` used for pickling. May be less than :data:`HIGHEST_PROTOCOL`. Currently the - default protocol is 4, first introduced in Python 3.4 and incompatible - with previous versions. + default protocol is 5, introduced in Python 3.8 and incompatible + with previous versions. This version introduces support for out-of-band + buffers, where :pep:`3118`-compatible data can be transmitted separately + from the main pickle stream. .. versionchanged:: 3.0 @@ -210,6 +213,10 @@ The :mod:`pickle` module provides the following constants: The default protocol is 4. + .. versionchanged:: 3.14 + + The default protocol is 5. + The :mod:`pickle` module provides the following functions to make the pickling process more convenient: @@ -314,16 +321,16 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, map the new Python 3 names to the old module names used in Python 2, so that the pickle data stream is readable with Python 2. - If *buffer_callback* is None (the default), buffer views are + If *buffer_callback* is ``None`` (the default), buffer views are serialized into *file* as part of the pickle stream. - If *buffer_callback* is not None, then it can be called any number + If *buffer_callback* is not ``None``, then it can be called any number of times with a buffer view. If the callback returns a false value - (such as None), the given buffer is :ref:`out-of-band `; + (such as ``None``), the given buffer is :ref:`out-of-band `; otherwise the buffer is serialized in-band, i.e. inside the pickle stream. - It is an error if *buffer_callback* is not None and *protocol* is - None or smaller than 5. + It is an error if *buffer_callback* is not ``None`` and *protocol* is + ``None`` or smaller than 5. .. versionchanged:: 3.8 The *buffer_callback* argument was added. @@ -377,7 +384,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, Special reducer that can be defined in :class:`Pickler` subclasses. This method has priority over any reducer in the :attr:`dispatch_table`. It should conform to the same interface as a :meth:`~object.__reduce__` method, and - can optionally return ``NotImplemented`` to fallback on + can optionally return :data:`NotImplemented` to fallback on :attr:`dispatch_table`-registered reducers to pickle ``obj``. For a detailed example, see :ref:`reducer_override`. @@ -420,12 +427,12 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, instances of :class:`~datetime.datetime`, :class:`~datetime.date` and :class:`~datetime.time` pickled by Python 2. - If *buffers* is None (the default), then all data necessary for + If *buffers* is ``None`` (the default), then all data necessary for deserialization must be contained in the pickle stream. This means - that the *buffer_callback* argument was None when a :class:`Pickler` + that the *buffer_callback* argument was ``None`` when a :class:`Pickler` was instantiated (or when :func:`dump` or :func:`dumps` was called). - If *buffers* is not None, it should be an iterable of buffer-enabled + If *buffers* is not ``None``, it should be an iterable of buffer-enabled objects that is consumed each time the pickle stream references an :ref:`out-of-band ` buffer view. Such buffers have been given in order to the *buffer_callback* of a Pickler object. @@ -503,7 +510,7 @@ What can be pickled and unpickled? The following types can be pickled: * built-in constants (``None``, ``True``, ``False``, ``Ellipsis``, and - ``NotImplemented``); + :data:`NotImplemented`); * integers, floating-point numbers, complex numbers; @@ -653,8 +660,8 @@ methods: .. note:: - If :meth:`__getstate__` returns a false value, the :meth:`__setstate__` - method will not be called upon unpickling. + If :meth:`__reduce__` returns a state with value ``None`` at pickling, + the :meth:`__setstate__` method will not be called upon unpickling. Refer to the section :ref:`pickle-state` for more information about how to use @@ -905,7 +912,7 @@ functions and classes. For those cases, it is possible to subclass from the :class:`Pickler` class and implement a :meth:`~Pickler.reducer_override` method. This method can return an arbitrary reduction tuple (see :meth:`~object.__reduce__`). It can alternatively return -``NotImplemented`` to fallback to the traditional behavior. +:data:`NotImplemented` to fallback to the traditional behavior. If both the :attr:`~Pickler.dispatch_table` and :meth:`~Pickler.reducer_override` are defined, then diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 9739207a224431..e072605974f6c2 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -1,5 +1,5 @@ -:mod:`pickletools` --- Tools for pickle developers -================================================== +:mod:`!pickletools` --- Tools for pickle developers +=================================================== .. module:: pickletools :synopsis: Contains extensive comments about the pickle protocols and diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 891a867d1ceb68..20b8f6bcf19117 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -1,5 +1,5 @@ -:mod:`pkgutil` --- Package extension utility -============================================ +:mod:`!pkgutil` --- Package extension utility +============================================= .. module:: pkgutil :synopsis: Utilities for the import system. @@ -26,7 +26,8 @@ support. __path__ = extend_path(__path__, __name__) For each directory on :data:`sys.path` that has a subdirectory that matches the - package name, add the subdirectory to the package's :attr:`__path__`. This is useful + package name, add the subdirectory to the package's + :attr:`~module.__path__`. This is useful if one wants to distribute different parts of a single logical package as multiple directories. @@ -34,9 +35,9 @@ support. *name* argument. This feature is similar to :file:`\*.pth` files (see the :mod:`site` module for more information), except that it doesn't special-case lines starting with ``import``. A :file:`\*.pkg` file is trusted at face - value: apart from checking for duplicates, all entries found in a - :file:`\*.pkg` file are added to the path, regardless of whether they exist - on the filesystem. (This is a feature.) + value: apart from skipping blank lines and ignoring comments, all entries + found in a :file:`\*.pkg` file are added to the path, regardless of whether + they exist on the filesystem (this is a feature). If the input path is not a list (as is the case for frozen packages) it is returned unchanged. The input path is not modified; an extended copy is @@ -48,25 +49,6 @@ support. this function to raise an exception (in line with :func:`os.path.isdir` behavior). -.. function:: find_loader(fullname) - - Retrieve a module :term:`loader` for the given *fullname*. - - This is a backwards compatibility wrapper around - :func:`importlib.util.find_spec` that converts most failures to - :exc:`ImportError` and only returns the loader rather than the full - :class:`importlib.machinery.ModuleSpec`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - .. deprecated-removed:: 3.12 3.14 - Use :func:`importlib.util.find_spec` instead. - .. function:: get_importer(path_item) @@ -83,27 +65,6 @@ support. on the package internal :pep:`302` import emulation. -.. function:: get_loader(module_or_name) - - Get a :term:`loader` object for *module_or_name*. - - If the module or package is accessible via the normal import mechanism, a - wrapper around the relevant part of that machinery is returned. Returns - ``None`` if the module cannot be found or imported. If the named module is - not already imported, its containing package (if any) is imported, in order - to establish the package ``__path__``. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - .. deprecated-removed:: 3.12 3.14 - Use :func:`importlib.util.find_spec` instead. - - .. function:: iter_importers(fullname='') Yield :term:`finder` objects for the given module name. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index ec2a7ebd5d6e0b..1beb3b9eb89d22 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -1,5 +1,5 @@ -:mod:`platform` --- Access to underlying platform's identifying data -===================================================================== +:mod:`!platform` --- Access to underlying platform's identifying data +====================================================================== .. module:: platform :synopsis: Retrieves as much platform identifying data as possible. @@ -148,6 +148,9 @@ Cross Platform Returns the system/OS name, such as ``'Linux'``, ``'Darwin'``, ``'Java'``, ``'Windows'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this returns the user-facing OS name (i.e, ``'iOS``, + ``'iPadOS'`` or ``'Android'``). To obtain the kernel name (``'Darwin'`` or + ``'Linux'``), use :func:`os.uname`. .. function:: system_alias(system, release, version) @@ -161,6 +164,8 @@ Cross Platform Returns the system's release version, e.g. ``'#3 on degas'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this is the user-facing OS version. To obtain the + Darwin or Linux kernel version, use :func:`os.uname`. .. function:: uname() @@ -196,6 +201,10 @@ Java Platform ``(os_name, os_version, os_arch)``. Values which cannot be determined are set to the defaults given as parameters (which all default to ``''``). + .. deprecated-removed:: 3.13 3.15 + It was largely untested, had a confusing API, + and was only useful for Jython support. + Windows Platform ---------------- @@ -210,8 +219,8 @@ Windows Platform default to an empty string). As a hint: *ptype* is ``'Uniprocessor Free'`` on single processor NT machines - and ``'Multiprocessor Free'`` on multi processor machines. The *'Free'* refers - to the OS version being free of debugging code. It could also state *'Checked'* + and ``'Multiprocessor Free'`` on multi processor machines. The ``'Free'`` refers + to the OS version being free of debugging code. It could also state ``'Checked'`` which means the OS version uses debugging code, i.e. code that checks arguments, ranges, etc. @@ -234,7 +243,6 @@ Windows Platform macOS Platform -------------- - .. function:: mac_ver(release='', versioninfo=('','',''), machine='') Get macOS version information and return it as tuple ``(release, versioninfo, @@ -244,6 +252,24 @@ macOS Platform Entries which cannot be determined are set to ``''``. All tuple entries are strings. +iOS Platform +------------ + +.. function:: ios_ver(system='', release='', model='', is_simulator=False) + + Get iOS version information and return it as a + :func:`~collections.namedtuple` with the following attributes: + + * ``system`` is the OS name; either ``'iOS'`` or ``'iPadOS'``. + * ``release`` is the iOS version number as a string (e.g., ``'17.2'``). + * ``model`` is the device model identifier; this will be a string like + ``'iPhone13,2'`` for a physical device, or ``'iPhone'`` on a simulator. + * ``is_simulator`` is a boolean describing if the app is running on a + simulator or a physical device. + + Entries which cannot be determined are set to the defaults given as + parameters. + Unix Platforms -------------- @@ -297,3 +323,39 @@ Linux Platforms return ids .. versionadded:: 3.10 + + +Android Platform +---------------- + +.. function:: android_ver(release="", api_level=0, manufacturer="", \ + model="", device="", is_emulator=False) + + Get Android device information. Returns a :func:`~collections.namedtuple` + with the following attributes. Values which cannot be determined are set to + the defaults given as parameters. + + * ``release`` - Android version, as a string (e.g. ``"14"``). + + * ``api_level`` - API level of the running device, as an integer (e.g. ``34`` + for Android 14). To get the API level which Python was built against, see + :func:`sys.getandroidapilevel`. + + * ``manufacturer`` - `Manufacturer name + `__. + + * ``model`` - `Model name + `__ – + typically the marketing name or model number. + + * ``device`` - `Device name + `__ – + typically the model number or a codename. + + * ``is_emulator`` - ``True`` if the device is an emulator; ``False`` if it's + a physical device. + + Google maintains a `list of known model and device names + `__. + + .. versionadded:: 3.13 diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 7416ca2650bab4..2906ebe7822f52 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -1,5 +1,5 @@ -:mod:`plistlib` --- Generate and parse Apple ``.plist`` files -============================================================= +:mod:`!plistlib` --- Generate and parse Apple ``.plist`` files +============================================================== .. module:: plistlib :synopsis: Generate and parse Apple plist files. @@ -117,7 +117,7 @@ This module defines the following functions: when a key of a dictionary is not a string, otherwise such keys are skipped. When *aware_datetime* is true and any field with type ``datetime.datetime`` - is set as a :ref:`aware object `, it will convert to + is set as an :ref:`aware object `, it will convert to UTC timezone before writing it. A :exc:`TypeError` will be raised if the object is of an unsupported type or diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 943eb21f6eec02..23f20b00e6dc6d 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -1,5 +1,5 @@ -:mod:`poplib` --- POP3 protocol client -====================================== +:mod:`!poplib` --- POP3 protocol client +======================================= .. module:: poplib :synopsis: POP3 protocol client (requires sockets). diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 5871574b442667..14ab3e91e8a8e4 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -1,5 +1,5 @@ -:mod:`posix` --- The most common POSIX system calls -=================================================== +:mod:`!posix` --- The most common POSIX system calls +==================================================== .. module:: posix :platform: Unix diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index e883acd67d6c72..1b3498e51f766d 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -1,5 +1,5 @@ -:mod:`pprint` --- Data pretty printer -===================================== +:mod:`!pprint` --- Data pretty printer +====================================== .. module:: pprint :synopsis: Data pretty printer. @@ -19,9 +19,8 @@ such as files, sockets or classes are included, as well as many other objects which are not representable as Python literals. The formatted representation keeps objects on a single line if it can, and -breaks them onto multiple lines if they don't fit within the allowed width. -Construct :class:`PrettyPrinter` objects explicitly if you need to adjust the -width constraint. +breaks them onto multiple lines if they don't fit within the allowed width, +adjustable by the *width* parameter defaulting to 80 characters. Dictionaries are sorted by key before the display is computed. @@ -31,86 +30,82 @@ Dictionaries are sorted by key before the display is computed. .. versionchanged:: 3.10 Added support for pretty-printing :class:`dataclasses.dataclass`. -The :mod:`pprint` module defines one class: +.. _pprint-functions: + +Functions +--------- + +.. function:: pp(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=False, underscore_numbers=False) + + Prints the formatted representation of *object*, followed by a newline. + This function may be used in the interactive interpreter + instead of the :func:`print` function for inspecting values. + Tip: you can reassign ``print = pprint.pp`` for use within a scope. + + :param object: + The object to be printed. + + :param stream: + A file-like object to which the output will be written + by calling its :meth:`!write` method. + If ``None`` (the default), :data:`sys.stdout` is used. + :type stream: :term:`file-like object` | None + + :param int indent: + The amount of indentation added for each nesting level. + + :param int width: + The desired maximum number of characters per line in the output. + If a structure cannot be formatted within the width constraint, + a best effort will be made. + + :param depth: + The number of nesting levels which may be printed. + If the data structure being printed is too deep, + the next contained level is replaced by ``...``. + If ``None`` (the default), there is no constraint + on the depth of the objects being formatted. + :type depth: int | None + + :param bool compact: + Control the way long :term:`sequences ` are formatted. + If ``False`` (the default), + each item of a sequence will be formatted on a separate line, + otherwise as many items as will fit within the *width* + will be formatted on each output line. + + :param bool sort_dicts: + If ``True``, dictionaries will be formatted with + their keys sorted, otherwise + they will be displayed in insertion order (the default). + + :param bool underscore_numbers: + If ``True``, + integers will be formatted with the ``_`` character for a thousands separator, + otherwise underscores are not displayed (the default). -.. First the implementation class: - - -.. index:: single: ...; placeholder - -.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Construct a :class:`PrettyPrinter` instance. This constructor understands - several keyword parameters. - - *stream* (default ``sys.stdout``) is a :term:`file-like object` to - which the output will be written by calling its :meth:`!write` method. - If both *stream* and ``sys.stdout`` are ``None``, then - :meth:`~PrettyPrinter.pprint` silently returns. - - Other values configure the manner in which nesting of complex data - structures is displayed. - - *indent* (default 1) specifies the amount of indentation added for - each nesting level. - - *depth* controls the number of nesting levels which may be printed; if - the data structure being printed is too deep, the next contained level - is replaced by ``...``. By default, there is no constraint on the - depth of the objects being formatted. - - *width* (default 80) specifies the desired maximum number of characters per - line in the output. If a structure cannot be formatted within the width - constraint, a best effort will be made. - - *compact* impacts the way that long sequences (lists, tuples, sets, etc) - are formatted. If *compact* is false (the default) then each item of a - sequence will be formatted on a separate line. If *compact* is true, as - many items as will fit within the *width* will be formatted on each output - line. - - If *sort_dicts* is true (the default), dictionaries will be formatted with - their keys sorted, otherwise they will display in insertion order. + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff) + >>> pprint.pp(stuff) + [, + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] - If *underscore_numbers* is true, integers will be formatted with the - ``_`` character for a thousands separator, otherwise underscores are not - displayed (the default). + .. versionadded:: 3.8 - .. versionchanged:: 3.4 - Added the *compact* parameter. - .. versionchanged:: 3.8 - Added the *sort_dicts* parameter. +.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) - .. versionchanged:: 3.10 - Added the *underscore_numbers* parameter. + Alias for :func:`~pprint.pp` with *sort_dicts* set to ``True`` by default, + which would automatically sort the dictionaries' keys, + you might want to use :func:`~pprint.pp` instead where it is ``False`` by default. - .. versionchanged:: 3.11 - No longer attempts to write to ``sys.stdout`` if it is ``None``. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff[:]) - >>> pp = pprint.PrettyPrinter(indent=4) - >>> pp.pprint(stuff) - [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - >>> pp = pprint.PrettyPrinter(width=41, compact=True) - >>> pp.pprint(stuff) - [['spam', 'eggs', 'lumberjack', - 'knights', 'ni'], - 'spam', 'eggs', 'lumberjack', 'knights', - 'ni'] - >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', - ... ('parrot', ('fresh fruit',)))))))) - >>> pp = pprint.PrettyPrinter(depth=6) - >>> pp.pprint(tup) - ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) .. function:: pformat(object, indent=1, width=80, depth=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) @@ -118,44 +113,8 @@ The :mod:`pprint` module defines one class: Return the formatted representation of *object* as a string. *indent*, *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are passed to the :class:`PrettyPrinter` constructor as formatting parameters - and their meanings are as described in its documentation above. - + and their meanings are as described in the documentation above. -.. function:: pp(object, *args, sort_dicts=False, **kwargs) - - Prints the formatted representation of *object* followed by a newline. - If *sort_dicts* is false (the default), dictionaries will be displayed with - their keys in insertion order, otherwise the dict keys will be sorted. - *args* and *kwargs* will be passed to :func:`pprint` as formatting - parameters. - - .. versionadded:: 3.8 - - -.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). - - The configuration parameters *stream*, *indent*, *width*, *depth*, - *compact*, *sort_dicts* and *underscore_numbers* are passed to the - :class:`PrettyPrinter` constructor and their meanings are as - described in its documentation above. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pprint(stuff) - [, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] .. function:: isreadable(object) @@ -176,8 +135,6 @@ The :mod:`pprint` module defines one class: :exc:`RecursionError` if it fails to detect a recursive object. -One more support function is also defined: - .. function:: saferepr(object) Return a string representation of *object*, protected against recursion in @@ -190,12 +147,58 @@ One more support function is also defined: >>> pprint.saferepr(stuff) "[, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" - .. _prettyprinter-objects: PrettyPrinter Objects --------------------- +.. index:: single: ...; placeholder + +.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) + + Construct a :class:`PrettyPrinter` instance. + + Arguments have the same meaning as for :func:`~pprint.pp`. + Note that they are in a different order, and that *sort_dicts* defaults to ``True``. + + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff[:]) + >>> pp = pprint.PrettyPrinter(indent=4) + >>> pp.pprint(stuff) + [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, compact=True) + >>> pp.pprint(stuff) + [['spam', 'eggs', 'lumberjack', + 'knights', 'ni'], + 'spam', 'eggs', 'lumberjack', 'knights', + 'ni'] + >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', + ... ('parrot', ('fresh fruit',)))))))) + >>> pp = pprint.PrettyPrinter(depth=6) + >>> pp.pprint(tup) + ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) + + + .. versionchanged:: 3.4 + Added the *compact* parameter. + + .. versionchanged:: 3.8 + Added the *sort_dicts* parameter. + + .. versionchanged:: 3.10 + Added the *underscore_numbers* parameter. + + .. versionchanged:: 3.11 + No longer attempts to write to :data:`!sys.stdout` if it is ``None``. + + :class:`PrettyPrinter` instances have the following methods: @@ -258,7 +261,7 @@ are converted to strings. The default implementation uses the internals of the Example ------- -To demonstrate several uses of the :func:`pprint` function and its parameters, +To demonstrate several uses of the :func:`~pprint.pp` function and its parameters, let's fetch information about a project from `PyPI `_:: >>> import json @@ -267,9 +270,9 @@ let's fetch information about a project from `PyPI `_:: >>> with urlopen('https://pypi.org/pypi/sampleproject/json') as resp: ... project_info = json.load(resp)['info'] -In its basic form, :func:`pprint` shows the whole object:: +In its basic form, :func:`~pprint.pp` shows the whole object:: - >>> pprint.pprint(project_info) + >>> pprint.pp(project_info) {'author': 'The Python Packaging Authority', 'author_email': 'pypa-dev@googlegroups.com', 'bugtrack_url': None, @@ -326,7 +329,7 @@ In its basic form, :func:`pprint` shows the whole object:: The result can be limited to a certain *depth* (ellipsis is used for deeper contents):: - >>> pprint.pprint(project_info, depth=1) + >>> pprint.pp(project_info, depth=1) {'author': 'The Python Packaging Authority', 'author_email': 'pypa-dev@googlegroups.com', 'bugtrack_url': None, @@ -372,7 +375,7 @@ contents):: Additionally, maximum character *width* can be suggested. If a long object cannot be split, the specified width will be exceeded:: - >>> pprint.pprint(project_info, depth=1, width=60) + >>> pprint.pp(project_info, depth=1, width=60) {'author': 'The Python Packaging Authority', 'author_email': 'pypa-dev@googlegroups.com', 'bugtrack_url': None, diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index cc059b66fcb84b..3334833eba6b8c 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -234,7 +234,7 @@ functions: .. function:: runctx(command, globals, locals, filename=None, sort=-1) This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. This routine + globals and locals mappings for the *command* string. This routine executes:: exec(command, globals, locals) @@ -299,6 +299,13 @@ functions: Create a :class:`~pstats.Stats` object based on the current profile and print the results to stdout. + The *sort* parameter specifies the sorting order of the displayed + statistics. It accepts a single key or a tuple of keys to enable + multi-level sorting, as in :func:`Stats.sort_stats `. + + .. versionadded:: 3.13 + :meth:`~Profile.print_stats` now accepts a tuple of keys. + .. method:: dump_stats(filename) Write the results of the current profile to *filename*. @@ -675,7 +682,7 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, that you choose (see :ref:`profile-calibration`). For most machines, a timer that returns a lone integer value will provide the best results in terms of low overhead during profiling. (:func:`os.times` is *pretty* bad, as it - returns a tuple of floating point values). If you want to substitute a + returns a tuple of floating-point values). If you want to substitute a better timer in the cleanest fashion, derive a class and hardwire a replacement dispatch method that best handles your timer call, along with the appropriate calibration constant. @@ -692,7 +699,7 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, As the :class:`cProfile.Profile` class cannot be calibrated, custom timer functions should be used with care and should be as fast as possible. For the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`_lsprof` module. + in the C source of the internal :mod:`!_lsprof` module. Python 3.3 adds several new functions in :mod:`time` that can be used to make precise measurements of process or wall-clock time. For example, see diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index bd2f5ed45cb8b4..1a44bb13a841de 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -1,5 +1,5 @@ -:mod:`pty` --- Pseudo-terminal utilities -======================================== +:mod:`!pty` --- Pseudo-terminal utilities +========================================= .. module:: pty :platform: Unix diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index dbe68cd14ec4d4..e1ff32912132f7 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -1,5 +1,5 @@ -:mod:`pwd` --- The password database -==================================== +:mod:`!pwd` --- The password database +===================================== .. module:: pwd :platform: Unix @@ -10,7 +10,7 @@ This module provides access to the Unix user account and password database. It is available on all Unix versions. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI, not iOS. Password database entries are reported as a tuple-like object, whose attributes correspond to the members of the ``passwd`` structure (Attribute field below, diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 38c416f9ad0305..75aa739d1003b8 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -1,5 +1,5 @@ -:mod:`py_compile` --- Compile Python source files -================================================= +:mod:`!py_compile` --- Compile Python source files +================================================== .. module:: py_compile :synopsis: Generate byte-code files from Python source files. @@ -96,7 +96,7 @@ byte-code cache files in the directory containing the source code. .. class:: PycInvalidationMode - A enumeration of possible methods the interpreter can use to determine + An enumeration of possible methods the interpreter can use to determine whether a bytecode file is up to date with a source file. The ``.pyc`` file indicates the desired invalidation mode in its header. See :ref:`pyc-invalidation` for more information on how Python invalidates diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 1e9876849b02f3..5efb11d89dd143 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -1,5 +1,5 @@ -:mod:`pyclbr` --- Python module browser support -=============================================== +:mod:`!pyclbr` --- Python module browser support +================================================ .. module:: pyclbr :synopsis: Supports information extraction for a Python module browser. @@ -142,7 +142,7 @@ Class Objects .. attribute:: parent - For top-level classes, None. For nested classes, the parent. + For top-level classes, ``None``. For nested classes, the parent. .. versionadded:: 3.7 diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 03e0915bf6d135..e8f153ee1b35ce 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -1,5 +1,5 @@ -:mod:`pydoc` --- Documentation generator and online help system -=============================================================== +:mod:`!pydoc` --- Documentation generator and online help system +================================================================ .. module:: pydoc :synopsis: Documentation generator and online help system. @@ -16,19 +16,19 @@ -------------- -The :mod:`pydoc` module automatically generates documentation from Python +The :mod:`!pydoc` module automatically generates documentation from Python modules. The documentation can be presented as pages of text on the console, served to a web browser, or saved to HTML files. For modules, classes, functions and methods, the displayed documentation is -derived from the docstring (i.e. the :attr:`__doc__` attribute) of the object, +derived from the docstring (i.e. the :attr:`~definition.__doc__` attribute) of the object, and recursively of its documentable members. If there is no docstring, -:mod:`pydoc` tries to obtain a description from the block of comment lines just +:mod:`!pydoc` tries to obtain a description from the block of comment lines just above the definition of the class, function or method in the source file, or at the top of the module (see :func:`inspect.getcomments`). The built-in function :func:`help` invokes the online help system in the -interactive interpreter, which uses :mod:`pydoc` to generate its documentation +interactive interpreter, which uses :mod:`!pydoc` to generate its documentation as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: @@ -46,14 +46,15 @@ produced for that file. .. note:: - In order to find objects and their documentation, :mod:`pydoc` imports the + In order to find objects and their documentation, :mod:`!pydoc` imports the module(s) to be documented. Therefore, any code on module level will be executed on that occasion. Use an ``if __name__ == '__main__':`` guard to only execute code when a file is invoked as a script and not just imported. When printing output to the console, :program:`pydoc` attempts to paginate the -output for easier reading. If the :envvar:`PAGER` environment variable is set, -:program:`pydoc` will use its value as a pagination program. +output for easier reading. If either the :envvar:`MANPAGER` or the +:envvar:`PAGER` environment variable is set, :program:`pydoc` will use its +value as a pagination program. When both are set, :envvar:`MANPAGER` is used. Specifying a ``-w`` flag before the argument will cause HTML documentation to be written out to a file in the current directory, instead of displaying text @@ -90,7 +91,7 @@ Python interpreter and typed ``import spam``. Module docs for core modules are assumed to reside in ``https://docs.python.org/X.Y/library/`` where ``X`` and ``Y`` are the major and minor version numbers of the Python interpreter. This can -be overridden by setting the :envvar:`PYTHONDOCS` environment variable +be overridden by setting the :envvar:`!PYTHONDOCS` environment variable to a different URL or to a local directory containing the Library Reference Manual pages. @@ -101,7 +102,7 @@ Reference Manual pages. The ``-g`` command line option was removed. .. versionchanged:: 3.4 - :mod:`pydoc` now uses :func:`inspect.signature` rather than + :mod:`!pydoc` now uses :func:`inspect.signature` rather than :func:`inspect.getfullargspec` to extract signature information from callables. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 935e872480efda..c0e9999f4b1270 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -1,5 +1,5 @@ -:mod:`xml.parsers.expat` --- Fast XML parsing using Expat -========================================================= +:mod:`!xml.parsers.expat` --- Fast XML parsing using Expat +========================================================== .. module:: xml.parsers.expat :synopsis: An interface to the Expat non-validating XML parser. @@ -196,6 +196,42 @@ XMLParser Objects :exc:`ExpatError` to be raised with the :attr:`code` attribute set to ``errors.codes[errors.XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING]``. +.. method:: xmlparser.SetReparseDeferralEnabled(enabled) + + .. warning:: + + Calling ``SetReparseDeferralEnabled(False)`` has security implications, + as detailed below; please make sure to understand these consequences + prior to using the ``SetReparseDeferralEnabled`` method. + + Expat 2.6.0 introduced a security mechanism called "reparse deferral" + where instead of causing denial of service through quadratic runtime + from reparsing large tokens, reparsing of unfinished tokens is now delayed + by default until a sufficient amount of input is reached. + Due to this delay, registered handlers may — depending of the sizing of + input chunks pushed to Expat — no longer be called right after pushing new + input to the parser. Where immediate feedback and taking over responsibility + of protecting against denial of service from large tokens are both wanted, + calling ``SetReparseDeferralEnabled(False)`` disables reparse deferral + for the current Expat parser instance, temporarily or altogether. + Calling ``SetReparseDeferralEnabled(True)`` allows re-enabling reparse + deferral. + + Note that :meth:`SetReparseDeferralEnabled` has been backported to some + prior releases of CPython as a security fix. Check for availability of + :meth:`SetReparseDeferralEnabled` using :func:`hasattr` if used in code + running across a variety of Python versions. + + .. versionadded:: 3.13 + +.. method:: xmlparser.GetReparseDeferralEnabled() + + Returns whether reparse deferral is currently enabled for the given + Expat parser instance. + + .. versionadded:: 3.13 + + :class:`xmlparser` objects have the following attributes: @@ -214,7 +250,8 @@ XMLParser Objects :meth:`CharacterDataHandler` callback whenever possible. This can improve performance substantially since Expat normally breaks character data into chunks at every line ending. This attribute is false by default, and may be changed at - any time. + any time. Note that when it is false, data that does not contain newlines + may be chunked too. .. attribute:: xmlparser.buffer_used @@ -372,7 +409,10 @@ otherwise stated. marked content, and ignorable whitespace. Applications which must distinguish these cases can use the :attr:`StartCdataSectionHandler`, :attr:`EndCdataSectionHandler`, and :attr:`ElementDeclHandler` callbacks to - collect the required information. + collect the required information. Note that the character data may be + chunked even if it is short and so you may receive more than one call to + :meth:`CharacterDataHandler`. Set the :attr:`buffer_text` instance attribute + to ``True`` to avoid that. .. method:: xmlparser.UnparsedEntityDeclHandler(entityName, base, systemId, publicId, notationName) diff --git a/Doc/library/python.rst b/Doc/library/python.rst index 610435999d9f48..c2c231af7c3033 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -25,4 +25,5 @@ overview: __future__.rst gc.rst inspect.rst + annotationlib.rst site.rst diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 1421fc2e552f0e..fbbebcf4ed8f92 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -1,5 +1,5 @@ -:mod:`queue` --- A synchronized queue class -=========================================== +:mod:`!queue` --- A synchronized queue class +============================================ .. module:: queue :synopsis: A synchronized queue class. @@ -187,11 +187,12 @@ fully processed by daemon consumer threads. processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put` into the queue). + ``shutdown(immediate=True)`` calls :meth:`task_done` for each remaining item + in the queue. + Raises a :exc:`ValueError` if called more times than there were items placed in the queue. - Raises :exc:`ShutDown` if the queue has been shut down immediately. - .. method:: Queue.join() @@ -202,8 +203,6 @@ fully processed by daemon consumer threads. indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. - Raises :exc:`ShutDown` if the queue has been shut down immediately. - Example of how to wait for enqueued tasks to be completed:: @@ -246,8 +245,10 @@ them down. queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise immediately instead. - All blocked callers of :meth:`~Queue.put` will be unblocked. If *immediate* - is true, also unblock callers of :meth:`~Queue.get` and :meth:`~Queue.join`. + All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` will be + unblocked. If *immediate* is true, a task will be marked as done for each + remaining item in the queue, which may unblock callers of + :meth:`~Queue.join`. .. versionadded:: 3.13 diff --git a/Doc/library/quopri.rst b/Doc/library/quopri.rst index 86717c00c3c136..977cb08d836afe 100644 --- a/Doc/library/quopri.rst +++ b/Doc/library/quopri.rst @@ -1,5 +1,5 @@ -:mod:`quopri` --- Encode and decode MIME quoted-printable data -============================================================== +:mod:`!quopri` --- Encode and decode MIME quoted-printable data +=============================================================== .. module:: quopri :synopsis: Encode and decode files using the MIME quoted-printable encoding. diff --git a/Doc/library/random.rst b/Doc/library/random.rst index d0ced2416c9578..ef0cfb0e76cef6 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -1,5 +1,5 @@ -:mod:`random` --- Generate pseudo-random numbers -================================================ +:mod:`!random` --- Generate pseudo-random numbers +================================================= .. module:: random :synopsis: Generate pseudo-random numbers with various common distributions. @@ -55,10 +55,16 @@ from sources provided by the operating system. `Complementary-Multiply-with-Carry recipe - `_ for a compatible alternative + `_ for a compatible alternative random number generator with a long period and comparatively simple update operations. +.. note:: + The global random number generator and instances of :class:`Random` are thread-safe. + However, in the free-threaded build, concurrent calls to the global generator or + to the same instance of :class:`Random` may encounter contention and poor performance. + Consider using separate instances of :class:`Random` per thread instead. + Bookkeeping functions --------------------- @@ -194,8 +200,8 @@ Functions for sequences For a given seed, the :func:`choices` function with equal weighting typically produces a different sequence than repeated calls to - :func:`choice`. The algorithm used by :func:`choices` uses floating - point arithmetic for internal consistency and speed. The algorithm used + :func:`choice`. The algorithm used by :func:`choices` uses floating-point + arithmetic for internal consistency and speed. The algorithm used by :func:`choice` defaults to integer arithmetic with repeated selections to avoid small biases from round-off error. @@ -292,21 +298,22 @@ be found in any statistics text. .. function:: random() - Return the next random floating point number in the range ``0.0 <= X < 1.0`` + Return the next random floating-point number in the range ``0.0 <= X < 1.0`` .. function:: uniform(a, b) - Return a random floating point number *N* such that ``a <= N <= b`` for + Return a random floating-point number *N* such that ``a <= N <= b`` for ``a <= b`` and ``b <= N <= a`` for ``b < a``. The end-point value ``b`` may or may not be included in the range - depending on floating-point rounding in the equation ``a + (b-a) * random()``. + depending on floating-point rounding in the expression + ``a + (b-a) * random()``. .. function:: triangular(low, high, mode) - Return a random floating point number *N* such that ``low <= N <= high`` and + Return a random floating-point number *N* such that ``low <= N <= high`` and with the specified *mode* between those bounds. The *low* and *high* bounds default to zero and one. The *mode* argument defaults to the midpoint between the bounds, giving a symmetric distribution. @@ -699,3 +706,83 @@ positive unnormalized float and is equal to ``math.ulp(0.0)``.) `_ a paper by Allen B. Downey describing ways to generate more fine-grained floats than normally generated by :func:`.random`. + +.. _random-cli: + +Command-line usage +------------------ + +.. versionadded:: 3.13 + +The :mod:`!random` module can be executed from the command line. + +.. code-block:: sh + + python -m random [-h] [-c CHOICE [CHOICE ...] | -i N | -f N] [input ...] + +The following options are accepted: + +.. program:: random + +.. option:: -h, --help + + Show the help message and exit. + +.. option:: -c CHOICE [CHOICE ...] + --choice CHOICE [CHOICE ...] + + Print a random choice, using :meth:`choice`. + +.. option:: -i + --integer + + Print a random integer between 1 and N inclusive, using :meth:`randint`. + +.. option:: -f + --float + + Print a random floating-point number between 0 and N inclusive, + using :meth:`uniform`. + +If no options are given, the output depends on the input: + +* String or multiple: same as :option:`--choice`. +* Integer: same as :option:`--integer`. +* Float: same as :option:`--float`. + +.. _random-cli-example: + +Command-line example +-------------------- + +Here are some examples of the :mod:`!random` command-line interface: + +.. code-block:: console + + $ # Choose one at random + $ python -m random egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + Lobster Thermidor aux crevettes with a Mornay sauce + + $ # Random integer + $ python -m random 6 + 6 + + $ # Random floating-point number + $ python -m random 1.8 + 1.7080016272295635 + + $ # With explicit arguments + $ python -m random --choice egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + egg + + $ python -m random --integer 6 + 3 + + $ python -m random --float 1.8 + 1.5666339105010318 + + $ python -m random --integer 6 + 5 + + $ python -m random --float 6 + 3.1942323316565915 diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 0a8c88b50cdeec..9db6f1da3be4db 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1,5 +1,5 @@ -:mod:`re` --- Regular expression operations -=========================================== +:mod:`!re` --- Regular expression operations +============================================ .. module:: re :synopsis: Regular expression operations. @@ -48,7 +48,7 @@ fine-tuning parameters. .. seealso:: - The third-party `regex `_ module, + The third-party :pypi:`regex` module, which has an API compatible with the standard library :mod:`re` module, but offers additional functionality and a more thorough Unicode support. @@ -101,7 +101,7 @@ The special characters are: ``.`` (Dot.) In the default mode, this matches any character except a newline. If the :const:`DOTALL` flag has been specified, this matches any character - including a newline. + including a newline. ``(?s:.)`` matches any character regardless of flags. .. index:: single: ^ (caret); in regular expressions @@ -572,6 +572,12 @@ character ``'$'``. Word boundaries are determined by the current locale if the :py:const:`~re.LOCALE` flag is used. + .. note:: + + Note that ``\B`` does not match an empty string, which differs from + RE implementations in other programming languages such as Perl. + This behavior is kept for compatibility reasons. + .. index:: single: \d; in regular expressions ``\d`` @@ -600,10 +606,9 @@ character ``'$'``. ``\s`` For Unicode (str) patterns: - Matches Unicode whitespace characters (which includes - ``[ \t\n\r\f\v]``, and also many other characters, for example the - non-breaking spaces mandated by typography rules in many - languages). + Matches Unicode whitespace characters (as defined by :py:meth:`str.isspace`). + This includes ``[ \t\n\r\f\v]``, and also many other characters, for example the + non-breaking spaces mandated by typography rules in many languages. Matches ``[ \t\n\r\f\v]`` if the :py:const:`~re.ASCII` flag is used. @@ -911,6 +916,10 @@ Functions ``None`` if no position in the string matches the pattern; note that this is different from finding a zero-length match at some point in the string. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. function:: match(pattern, string, flags=0) @@ -925,6 +934,10 @@ Functions If you want to locate a match anywhere in *string*, use :func:`search` instead (see also :ref:`search-vs-match`). + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. function:: fullmatch(pattern, string, flags=0) @@ -932,6 +945,10 @@ Functions corresponding :class:`~re.Match`. Return ``None`` if the string does not match the pattern; note that this is different from a zero-length match. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionadded:: 3.4 @@ -974,6 +991,10 @@ Functions >>> re.split(r'(\W*)', '...words...') ['', '...', '', '', 'w', '', 'o', '', 'r', '', 'd', '', 's', '...', '', '', ''] + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.1 Added the optional flags argument. @@ -1004,6 +1025,10 @@ Functions >>> re.findall(r'(\w+)=(\d+)', 'set width=20 and height=10') [('width', '20'), ('height', '10')] + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.7 Non-empty matches can now start just after a previous empty match. @@ -1015,6 +1040,10 @@ Functions is scanned left-to-right, and matches are returned in the order found. Empty matches are included in the result. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.7 Non-empty matches can now start just after a previous empty match. @@ -1070,6 +1099,10 @@ Functions character ``'0'``. The backreference ``\g<0>`` substitutes in the entire substring matched by the RE. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.1 Added the optional flags argument. @@ -1102,6 +1135,10 @@ Functions Perform the same operation as :func:`sub`, but return a tuple ``(new_string, number_of_subs_made)``. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. function:: escape(pattern) @@ -1344,7 +1381,8 @@ when there is no match, you can test whether there was a match with a simple Escapes such as ``\n`` are converted to the appropriate characters, and numeric backreferences (``\1``, ``\2``) and named backreferences (``\g<1>``, ``\g``) are replaced by the contents of the - corresponding group. + corresponding group. The backreference ``\g<0>`` will be + replaced by the entire match. .. versionchanged:: 3.5 Unmatched groups are replaced with an empty string. @@ -1597,7 +1635,7 @@ To find out what card the pair consists of, one could use the Simulating scanf() ^^^^^^^^^^^^^^^^^^ -.. index:: single: scanf() +.. index:: single: scanf (C function) Python does not currently have an equivalent to :c:func:`!scanf`. Regular expressions are generally more powerful, though also more verbose, than diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 54c6d9f3b32b1a..4a04205663258c 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -1,5 +1,5 @@ -:mod:`readline` --- GNU readline interface -========================================== +:mod:`!readline` --- GNU readline interface +=========================================== .. module:: readline :platform: Unix @@ -24,6 +24,8 @@ in the GNU Readline manual for information about the format and allowable constructs of that file, and the capabilities of the Readline library in general. +.. include:: ../includes/wasm-mobile-notavail.rst + .. note:: The underlying Readline library API may be implemented by @@ -43,6 +45,10 @@ Readline library in general. python:bind -v python:bind ^I rl_complete + Also note that different libraries may use different history file formats. + When switching the underlying library, existing history files may become + unusable. + .. data:: backend The name of the underlying Readline library being used, either diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 678a11c6f45490..28c7855dfeeef3 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -1,5 +1,5 @@ -:mod:`reprlib` --- Alternate :func:`repr` implementation -======================================================== +:mod:`!reprlib` --- Alternate :func:`repr` implementation +========================================================= .. module:: reprlib :synopsis: Alternate repr() implementation with size limits. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 4e58b043f1da31..0515d205bbca0b 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -1,5 +1,5 @@ -:mod:`resource` --- Resource usage information -============================================== +:mod:`!resource` --- Resource usage information +=============================================== .. module:: resource :platform: Unix @@ -13,7 +13,7 @@ This module provides basic mechanisms for measuring and controlling system resources utilized by a program. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI. Symbolic constants are used to specify particular system resources and to request usage information about either the current process or its children. @@ -177,6 +177,8 @@ platform. The largest area of mapped memory which the process may occupy. + .. availability:: FreeBSD >= 11. + .. data:: RLIMIT_AS @@ -303,7 +305,7 @@ These functions are used to retrieve resource usage information: elements. The fields :attr:`ru_utime` and :attr:`ru_stime` of the return value are - floating point values representing the amount of time spent executing in user + floating-point values representing the amount of time spent executing in user mode and the amount of time spent executing in system mode, respectively. The remaining values are integers. Consult the :manpage:`getrusage(2)` man page for detailed information about these values. A brief summary is presented here: diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 8287699c5f013e..91779feb525013 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -1,5 +1,5 @@ -:mod:`rlcompleter` --- Completion function for GNU readline -=========================================================== +:mod:`!rlcompleter` --- Completion function for GNU readline +============================================================ .. module:: rlcompleter :synopsis: Python identifier completion, suitable for the GNU readline library. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index f2cb595f495f6b..b07ec6e93f80ab 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -1,5 +1,5 @@ -:mod:`runpy` --- Locating and executing Python modules -====================================================== +:mod:`!runpy` --- Locating and executing Python modules +======================================================= .. module:: runpy :synopsis: Locate and run Python modules without importing them first. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 01bac5afd0b9b3..517dbe8c321898 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -1,5 +1,5 @@ -:mod:`sched` --- Event scheduler -================================ +:mod:`!sched` --- Event scheduler +================================= .. module:: sched :synopsis: General purpose event scheduler. @@ -36,7 +36,7 @@ scheduler: Example:: >>> import sched, time - >>> s = sched.scheduler(time.monotonic, time.sleep) + >>> s = sched.scheduler(time.time, time.sleep) >>> def print_time(a='default'): ... print("From print_time", time.time(), a) ... diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 4405dfc0535973..75dafc54d40ca5 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -1,5 +1,5 @@ -:mod:`secrets` --- Generate secure random numbers for managing secrets -====================================================================== +:mod:`!secrets` --- Generate secure random numbers for managing secrets +======================================================================= .. module:: secrets :synopsis: Generate secure random numbers for managing secrets. @@ -42,17 +42,17 @@ randomness that your operating system provides. sources provided by the operating system. See :class:`random.SystemRandom` for additional details. -.. function:: choice(sequence) +.. function:: choice(seq) Return a randomly chosen element from a non-empty sequence. -.. function:: randbelow(n) +.. function:: randbelow(exclusive_upper_bound) - Return a random int in the range [0, *n*). + Return a random int in the range [0, *exclusive_upper_bound*). .. function:: randbits(k) - Return an int with *k* random bits. + Return a non-negative int with *k* random bits. Generating tokens @@ -155,7 +155,7 @@ Generate an eight-character alphanumeric password: .. note:: Applications should not - `store passwords in a recoverable format `_, + :cwe:`store passwords in a recoverable format <257>`, whether plain text or encrypted. They should be salted and hashed using a cryptographically strong one-way (irreversible) hash function. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index a0058046d0ce4c..f23a249f44b485 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -1,5 +1,5 @@ -:mod:`select` --- Waiting for I/O completion -============================================ +:mod:`!select` --- Waiting for I/O completion +============================================= .. module:: select :synopsis: Wait for I/O completion on multiple streams. @@ -129,7 +129,7 @@ The module defines the following: Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The - optional *timeout* argument specifies a time-out as a floating point number + optional *timeout* argument specifies a time-out as a floating-point number in seconds. When the *timeout* argument is omitted the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index 76cbf91412f763..de8c3ef0ea2275 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -1,5 +1,5 @@ -:mod:`selectors` --- High-level I/O multiplexing -================================================ +:mod:`!selectors` --- High-level I/O multiplexing +================================================= .. module:: selectors :synopsis: High-level I/O multiplexing. diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 95c54991887022..6e74a59b82b8ec 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -1,5 +1,5 @@ -:mod:`shelve` --- Python object persistence -=========================================== +:mod:`!shelve` --- Python object persistence +============================================ .. module:: shelve :synopsis: Python object persistence. @@ -86,7 +86,7 @@ Two additional methods are supported: .. seealso:: - `Persistent dictionary recipe `_ + `Persistent dictionary recipe `_ with widely supported storage formats and having the speed of native dictionaries. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index f94833ad5331a9..a96f0864dc1260 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -1,5 +1,5 @@ -:mod:`shlex` --- Simple lexical analysis -======================================== +:mod:`!shlex` --- Simple lexical analysis +========================================= .. module:: shlex :synopsis: Simple lexical analysis for Unix shell-like languages. @@ -412,17 +412,17 @@ otherwise. To illustrate, you can see the difference in the following snippet: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> import shlex - >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" - >>> s = shlex.shlex(text, posix=True) - >>> s.whitespace_split = True - >>> list(s) - ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] - >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) - >>> s.whitespace_split = True - >>> list(s) - ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', - '(', 'def', 'ghi', ')'] + >>> import shlex + >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" + >>> s = shlex.shlex(text, posix=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] + >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', + '(', 'def', 'ghi', ')'] Of course, tokens will be returned which are not valid for shells, and you'll need to implement your own error checks on the returned tokens. @@ -431,10 +431,10 @@ Instead of passing ``True`` as the value for the punctuation_chars parameter, you can pass a string with specific characters, which will be used to determine which characters constitute punctuation. For example:: - >>> import shlex - >>> s = shlex.shlex("a && b || c", punctuation_chars="|") - >>> list(s) - ['a', '&', '&', 'b', '||', 'c'] + >>> import shlex + >>> s = shlex.shlex("a && b || c", punctuation_chars="|") + >>> list(s) + ['a', '&', '&', 'b', '||', 'c'] .. note:: When ``punctuation_chars`` is specified, the :attr:`~shlex.wordchars` attribute is augmented with the characters ``~-./*?=``. That is because these diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index ff8c9a189ab3de..d25701c087ed07 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -1,5 +1,5 @@ -:mod:`shutil` --- High-level file operations -============================================ +:mod:`!shutil` --- High-level file operations +============================================= .. module:: shutil :synopsis: High-level file operations, including copying. @@ -39,7 +39,7 @@ Directory and files operations .. function:: copyfileobj(fsrc, fdst[, length]) - Copy the contents of the file-like object *fsrc* to the file-like object *fdst*. + Copy the contents of the :term:`file-like object ` *fsrc* to the file-like object *fdst*. The integer *length*, if given, is the buffer size. In particular, a negative *length* value means to copy the data without looping over the source data in chunks; by default the data is read in chunks to avoid uncontrolled memory @@ -52,7 +52,7 @@ Directory and files operations Copy the contents (no metadata) of the file named *src* to a file named *dst* and return *dst* in the most efficient way possible. - *src* and *dst* are path-like objects or path names given as strings. + *src* and *dst* are :term:`path-like objects ` or path names given as strings. *dst* must be the complete target file name; look at :func:`~shutil.copy` for a copy that accepts a target directory path. If *src* and *dst* @@ -94,7 +94,7 @@ Directory and files operations .. function:: copymode(src, dst, *, follow_symlinks=True) Copy the permission bits from *src* to *dst*. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path names + group are unaffected. *src* and *dst* are :term:`path-like objects ` or path names given as strings. If *follow_symlinks* is false, and both *src* and *dst* are symbolic links, :func:`copymode` will attempt to modify the mode of *dst* itself (rather @@ -113,7 +113,7 @@ Directory and files operations Copy the permission bits, last access time, last modification time, and flags from *src* to *dst*. On Linux, :func:`copystat` also copies the "extended attributes" where possible. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path + group are unaffected. *src* and *dst* are :term:`path-like objects ` or path names given as strings. If *follow_symlinks* is false, and *src* and *dst* both @@ -242,7 +242,7 @@ Directory and files operations be copied as far as the platform allows; if false or omitted, the contents and metadata of the linked files are copied to the new tree. - When *symlinks* is false, if the file pointed by the symlink doesn't + When *symlinks* is false, if the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an :exc:`Error` exception at the end of the copy process. You can set the optional *ignore_dangling_symlinks* flag to true if you @@ -274,16 +274,16 @@ Directory and files operations .. audit-event:: shutil.copytree src,dst shutil.copytree - .. versionchanged:: 3.3 - Copy metadata when *symlinks* is false. - Now returns *dst*. - .. versionchanged:: 3.2 Added the *copy_function* argument to be able to provide a custom copy function. Added the *ignore_dangling_symlinks* argument to silence dangling symlinks errors when *symlinks* is false. + .. versionchanged:: 3.3 + Copy metadata when *symlinks* is false. + Now returns *dst*. + .. versionchanged:: 3.8 Platform-specific fast-copy syscalls may be used internally in order to copy the file more efficiently. See @@ -338,7 +338,7 @@ Directory and files operations before removing the junction. .. versionchanged:: 3.11 - The *dir_fd* parameter. + Added the *dir_fd* parameter. .. versionchanged:: 3.12 Added the *onexc* parameter, deprecated *onerror*. @@ -421,7 +421,8 @@ Directory and files operations .. availability:: Unix, Windows. -.. function:: chown(path, user=None, group=None) +.. function:: chown(path, user=None, group=None, *, dir_fd=None, \ + follow_symlinks=True) Change owner *user* and/or *group* of the given *path*. @@ -436,6 +437,9 @@ Directory and files operations .. versionadded:: 3.3 + .. versionchanged:: 3.13 + Added *dir_fd* and *follow_symlinks* parameters. + .. function:: which(cmd, mode=os.F_OK | os.X_OK, path=None) @@ -443,10 +447,12 @@ Directory and files operations called. If no *cmd* would be called, return ``None``. *mode* is a permission mask passed to :func:`os.access`, by default - determining if the file exists and executable. + determining if the file exists and is executable. - When no *path* is specified, the results of :func:`os.environ` are used, - returning either the "PATH" value or a fallback of :data:`os.defpath`. + *path* is a "``PATH`` string" specifying the directories to look in, + delimited by :data:`os.pathsep`. When no *path* is specified, the + :envvar:`PATH` environment variable is read from :data:`os.environ`, + falling back to :data:`os.defpath` if it is not set. On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the @@ -455,9 +461,9 @@ Directory and files operations consulting the current working directory for executables: set the environment variable ``NoDefaultCurrentDirectoryInExePath``. - Also on Windows, the ``PATHEXT`` variable is used to resolve commands - that may not already include an extension. For example, if you call - ``shutil.which("python")``, :func:`which` will search ``PATHEXT`` + Also on Windows, the :envvar:`PATHEXT` environment variable is used to + resolve commands that may not already include an extension. For example, + if you call ``shutil.which("python")``, :func:`which` will search ``PATHEXT`` to know that it should look for ``python.exe`` within the *path* directories. For example, on Windows:: @@ -512,7 +518,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``". On macOS `fcopyfile`_ is used to copy the file content (not metadata). -On Linux :func:`os.sendfile` is used. +On Linux and Solaris :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of @@ -524,6 +530,9 @@ file then shutil will silently fallback on using less efficient .. versionchanged:: 3.8 +.. versionchanged:: 3.14 + Solaris now uses :func:`os.sendfile`. + .. _shutil-copytree-example: copytree example @@ -701,11 +710,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. The keyword-only *filter* argument is passed to the underlying unpacking function. For zip files, *filter* is not accepted. - For tar files, it is recommended to set it to ``'data'``, - unless using features specific to tar and UNIX-like filesystems. + For tar files, it is recommended to use ``'data'`` (default since Python + 3.14), unless using features specific to tar and UNIX-like filesystems. (See :ref:`tarfile-extraction-filter` for details.) - The ``'data'`` filter will become the default for tar files - in Python 3.14. .. audit-event:: shutil.unpack_archive filename,extract_dir,format shutil.unpack_archive @@ -716,6 +723,12 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. the *extract_dir* argument, e.g. members that have absolute filenames starting with "/" or filenames with two dots "..". + Since Python 3.14, the defaults for both built-in formats (zip and tar + files) will prevent the most dangerous of such security issues, + but will not prevent *all* unintended behavior. + Read the :ref:`tarfile-further-verification` + section for tar-specific details. + .. versionchanged:: 3.7 Accepts a :term:`path-like object` for *filename* and *extract_dir*. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 85a073aad233ac..17fcb2b3707978 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -1,5 +1,5 @@ -:mod:`signal` --- Set handlers for asynchronous events -====================================================== +:mod:`!signal` --- Set handlers for asynchronous events +======================================================= .. module:: signal :synopsis: Set handlers for asynchronous events. @@ -26,9 +26,9 @@ explicitly reset (Python emulates the BSD style interface regardless of the underlying implementation), with the exception of the handler for :const:`SIGCHLD`, which follows the underlying implementation. -On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, signals -are emulated and therefore behave differently. Several functions and signals -are not available on these platforms. +On WebAssembly platforms, signals are emulated and therefore behave +differently. Several functions and signals are not available on these +platforms. Execution of Python signal handlers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -411,7 +411,7 @@ The :mod:`signal` module defines the following functions: See the :manpage:`pidfd_send_signal(2)` man page for more information. - .. availability:: Linux >= 5.1 + .. availability:: Linux >= 5.1, Android >= :func:`build-time ` API level 31 .. versionadded:: 3.9 @@ -425,7 +425,7 @@ The :mod:`signal` module defines the following functions: signal to a particular Python thread would be to force a running system call to fail with :exc:`InterruptedError`. - Use :func:`threading.get_ident()` or the :attr:`~threading.Thread.ident` + Use :func:`threading.get_ident` or the :attr:`~threading.Thread.ident` attribute of :class:`threading.Thread` objects to get a suitable value for *thread_id*. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 2dc9fb09d727e2..4508091f679dc7 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -1,5 +1,5 @@ -:mod:`site` --- Site-specific configuration hook -================================================ +:mod:`!site` --- Site-specific configuration hook +================================================= .. module:: site :synopsis: Module responsible for site-specific configuration. @@ -15,8 +15,9 @@ import can be suppressed using the interpreter's :option:`-S` option. .. index:: triple: module; search; path -Importing this module will append site-specific paths to the module search path -and add a few builtins, unless :option:`-S` was used. In that case, this module +Importing this module normally appends site-specific paths to the module search path +and adds :ref:`callables `, including :func:`help` to the built-in +namespace. However, Python startup option :option:`-S` blocks this and this module can be safely imported with no automatic modifications to the module search path or additions to the builtins. To explicitly trigger the usual site-specific additions, call the :func:`main` function. @@ -32,7 +33,10 @@ It starts by constructing up to four directories from a head and a tail part. For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or -:file:`lib/python{X.Y}/site-packages` (on Unix and macOS). For each +:file:`lib/python{X.Y[t]}/site-packages` (on Unix and macOS). (The +optional suffix "t" indicates the :term:`free threading` build, and is +appended if ``"t"`` is present in the :attr:`sys.abiflags` constant.) +For each of the distinct head-tail combinations, it sees if it refers to an existing directory, and if so, adds it to ``sys.path`` and also inspects the newly added path for configuration files. @@ -40,6 +44,11 @@ added path for configuration files. .. versionchanged:: 3.5 Support for the "site-python" directory has been removed. +.. versionchanged:: 3.13 + On Unix, :term:`Free threading ` Python installations are + identified by the "t" suffix in the version-specific directory name, such as + :file:`lib/python3.13t/`. + If a file named "pyvenv.cfg" exists one directory above sys.executable, sys.prefix and sys.exec_prefix are set to that directory and it is also checked for site-packages (sys.base_prefix and @@ -74,6 +83,10 @@ with ``import`` (followed by space or tab) are executed. Limiting a code chunk to a single line is a deliberate measure to discourage putting anything more complex here. +.. versionchanged:: 3.13 + The :file:`.pth` files are now decoded by UTF-8 at first and then by the + :term:`locale encoding` if it fails. + .. index:: single: package triple: path; configuration; file @@ -184,11 +197,12 @@ Module contents Path to the user site-packages for the running Python. Can be ``None`` if :func:`getusersitepackages` hasn't been called yet. Default value is - :file:`~/.local/lib/python{X.Y}/site-packages` for UNIX and non-framework + :file:`~/.local/lib/python{X.Y}[t]/site-packages` for UNIX and non-framework macOS builds, :file:`~/Library/Python/{X.Y}/lib/python/site-packages` for macOS framework builds, and :file:`{%APPDATA%}\\Python\\Python{XY}\\site-packages` - on Windows. This directory is a site directory, which means that - :file:`.pth` files in it will be processed. + on Windows. The optional "t" indicates the free-threaded build. This + directory is a site directory, which means that :file:`.pth` files in it + will be processed. .. data:: USER_BASE diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index aaec2aa1ef1dbe..7cd530a5fd6438 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -1,5 +1,5 @@ -:mod:`smtplib` --- SMTP protocol client -======================================= +:mod:`!smtplib` --- SMTP protocol client +======================================== .. module:: smtplib :synopsis: SMTP protocol client (requires sockets). @@ -556,34 +556,33 @@ This example prompts the user for addresses needed in the message envelope ('To' and 'From' addresses), and the message to be delivered. Note that the headers to be included with the message must be included in the message as entered; this example doesn't do any processing of the :rfc:`822` headers. In particular, the -'To' and 'From' addresses must be included in the message headers explicitly. :: +'To' and 'From' addresses must be included in the message headers explicitly:: import smtplib - def prompt(prompt): - return input(prompt).strip() + def prompt(title): + return input(title).strip() - fromaddr = prompt("From: ") - toaddrs = prompt("To: ").split() + from_addr = prompt("From: ") + to_addrs = prompt("To: ").split() print("Enter message, end with ^D (Unix) or ^Z (Windows):") # Add the From: and To: headers at the start! - msg = ("From: %s\r\nTo: %s\r\n\r\n" - % (fromaddr, ", ".join(toaddrs))) + lines = [f"From: {from_addr}", f"To: {', '.join(to_addrs)}", ""] while True: try: line = input() except EOFError: break - if not line: - break - msg = msg + line + else: + lines.append(line) + msg = "\r\n".join(lines) print("Message length is", len(msg)) - server = smtplib.SMTP('localhost') + server = smtplib.SMTP("localhost") server.set_debuglevel(1) - server.sendmail(fromaddr, toaddrs, msg) + server.sendmail(from_addr, to_addrs, msg) server.quit() .. note:: diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 4bfb0d8c2cfeac..0c7b9328648f66 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1,5 +1,5 @@ -:mod:`socket` --- Low-level networking interface -================================================ +:mod:`!socket` --- Low-level networking interface +================================================= .. module:: socket :synopsis: Low-level networking interface. @@ -413,14 +413,14 @@ Constants ``TCP_USER_TIMEOUT``, ``TCP_CONGESTION`` were added. .. versionchanged:: 3.6.5 - On Windows, ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` appear if run-time Windows - supports. + Added support for ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` on Windows platforms + when available. .. versionchanged:: 3.7 ``TCP_NOTSENT_LOWAT`` was added. - On Windows, ``TCP_KEEPIDLE``, ``TCP_KEEPINTVL`` appear if run-time Windows - supports. + Added support for ``TCP_KEEPIDLE``, ``TCP_KEEPINTVL`` on Windows platforms + when available. .. versionchanged:: 3.10 ``IP_RECVTOS`` was added. @@ -445,6 +445,19 @@ Constants Added ``IP_PKTINFO``, ``IP_UNBLOCK_SOURCE``, ``IP_BLOCK_SOURCE``, ``IP_ADD_SOURCE_MEMBERSHIP``, ``IP_DROP_SOURCE_MEMBERSHIP``. + .. versionchanged:: 3.13 + Added ``SO_BINDTOIFINDEX``. On Linux this constant can be used in the + same way that ``SO_BINDTODEVICE`` is used, but with the index of a + network interface instead of its name. + + .. versionchanged:: 3.14 + Added missing ``IP_RECVERR``, ``IPV6_RECVERR``, ``IP_RECVTTL``, and + ``IP_RECVORIGDSTADDR`` on Linux. + + .. versionchanged:: 3.14 + Added support for ``TCP_QUICKACK`` on Windows platforms when available. + + .. data:: AF_CAN PF_CAN SOL_CAN_* @@ -695,6 +708,13 @@ Constants .. versionadded:: 3.12 +.. data:: SHUT_RD + SHUT_WR + SHUT_RDWR + + These constants are used by the :meth:`~socket.socket.shutdown` method of socket objects. + + .. availability:: not WASI. Functions ^^^^^^^^^ @@ -724,7 +744,7 @@ The following functions all create :ref:`socket objects `. of :meth:`socket.getpeername` but not the actual OS resource. Unlike :func:`socket.fromfd`, *fileno* will return the same socket and not a duplicate. This may help close a detached socket using - :meth:`socket.close()`. + :meth:`socket.close`. The newly created socket is :ref:`non-inheritable `. @@ -1208,7 +1228,7 @@ The :mod:`socket` module also offers various network-related services: buffer. Raises :exc:`OverflowError` if *length* is outside the permissible range of values. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. Most Unix platforms. @@ -1231,7 +1251,7 @@ The :mod:`socket` module also offers various network-related services: amount of ancillary data that can be received, since additional data may be able to fit into the padding area. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. most Unix platforms. @@ -1260,7 +1280,7 @@ The :mod:`socket` module also offers various network-related services: .. audit-event:: socket.sethostname name socket.sethostname - .. availability:: Unix. + .. availability:: Unix, not Android. .. versionadded:: 3.3 @@ -1271,7 +1291,7 @@ The :mod:`socket` module also offers various network-related services: (index int, name string) tuples. :exc:`OSError` if the system call fails. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionadded:: 3.3 @@ -1298,7 +1318,7 @@ The :mod:`socket` module also offers various network-related services: interface name. :exc:`OSError` if no interface with the given name exists. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionadded:: 3.3 @@ -1315,7 +1335,7 @@ The :mod:`socket` module also offers various network-related services: interface index number. :exc:`OSError` if no interface with the given index exists. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionadded:: 3.3 @@ -1332,7 +1352,7 @@ The :mod:`socket` module also offers various network-related services: The *fds* parameter is a sequence of file descriptors. Consult :meth:`~socket.sendmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. Unix platforms supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. @@ -1346,7 +1366,7 @@ The :mod:`socket` module also offers various network-related services: Return ``(msg, list(fds), flags, addr)``. Consult :meth:`~socket.recvmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. Unix platforms supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. @@ -1403,7 +1423,7 @@ to sockets. .. method:: socket.close() Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile()` + descriptor) is also closed when all file objects from :meth:`makefile` are closed. Once that happens, all future operations on the socket object will fail. The remote end will receive no more data (after queued data is flushed). @@ -1418,10 +1438,10 @@ to sockets. .. note:: - :meth:`close()` releases the resource associated with a connection but + :meth:`close` releases the resource associated with a connection but does not necessarily close the connection immediately. If you want - to close the connection in a timely fashion, call :meth:`shutdown()` - before :meth:`close()`. + to close the connection in a timely fashion, call :meth:`shutdown` + before :meth:`close`. .. method:: socket.connect(address) @@ -1584,7 +1604,8 @@ to sockets. Return a :term:`file object` associated with the socket. The exact returned type depends on the arguments given to :meth:`makefile`. These arguments are interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'`` and ``'b'``. + the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or + a combination of those. The socket must be in blocking mode; it can have a timeout, but the file object's internal buffer may end up in an inconsistent state if a timeout @@ -1605,8 +1626,9 @@ to sockets. Receive data from the socket. The return value is a bytes object representing the data received. The maximum amount of data to be received at once is specified - by *bufsize*. See the Unix manual page :manpage:`recv(2)` for the meaning of - the optional argument *flags*; it defaults to zero. + by *bufsize*. A returned empty bytes object indicates that the client has disconnected. + See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument + *flags*; it defaults to zero. .. note:: @@ -1915,7 +1937,7 @@ to sockets. .. method:: socket.settimeout(value) Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative floating point number expressing seconds, or ``None``. + nonnegative floating-point number expressing seconds, or ``None``. If a non-zero value is given, subsequent socket operations will raise a :exc:`timeout` exception if the timeout period *value* has elapsed before the operation has completed. If zero is given, the socket is put in @@ -2028,7 +2050,7 @@ can be changed by calling :func:`setdefaulttimeout`. in non-blocking mode. Also, the blocking and timeout modes are shared between file descriptors and socket objects that refer to the same network endpoint. This implementation detail can have visible consequences if e.g. you decide - to use the :meth:`~socket.fileno()` of a socket. + to use the :meth:`~socket.fileno` of a socket. Timeouts and the ``connect`` method ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 864b1dadb78562..69f06e6cf4d923 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -1,5 +1,5 @@ -:mod:`socketserver` --- A framework for network servers -======================================================= +:mod:`!socketserver` --- A framework for network servers +======================================================== .. module:: socketserver :synopsis: A framework for network servers. @@ -126,6 +126,12 @@ server is the address family. waits until all non-daemon threads complete, except if :attr:`block_on_close` attribute is ``False``. + .. attribute:: max_children + + Specify how many child processes will exist to handle requests at a time + for :class:`ForkingMixIn`. If the limit is reached, + new requests will wait until one child process has finished. + .. attribute:: daemon_threads For :class:`ThreadingMixIn` use daemonic threads by setting diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 87d5ef1e42ca3a..096892b605b99c 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1,5 +1,5 @@ -:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases -============================================================ +:mod:`!sqlite3` --- DB-API 2.0 interface for SQLite databases +============================================================= .. module:: sqlite3 :synopsis: A DB-API 2.0 implementation using SQLite 3.x. @@ -16,6 +16,8 @@ src = sqlite3.connect(":memory:", isolation_level=None) dst = sqlite3.connect("tutorial.db", isolation_level=None) src.backup(dst) + src.close() + dst.close() del src, dst .. _sqlite3-intro: @@ -125,7 +127,7 @@ and call :meth:`res.fetchone() ` to fetch the resulting row: We can see that the table has been created, as the query returns a :class:`tuple` containing the table's name. If we query ``sqlite_master`` for a non-existent table ``spam``, -:meth:`!res.fetchone()` will return ``None``: +:meth:`!res.fetchone` will return ``None``: .. doctest:: @@ -220,6 +222,7 @@ creating a new cursor, then querying the database: >>> title, year = res.fetchone() >>> print(f'The highest scoring Monty Python movie is {title!r}, released in {year}') The highest scoring Monty Python movie is 'Monty Python and the Holy Grail', released in 1975 + >>> new_con.close() You've now created an SQLite database using the :mod:`!sqlite3` module, inserted data and retrieved values from it in multiple ways. @@ -394,29 +397,11 @@ Module functions will get tracebacks from callbacks on :data:`sys.stderr`. Use ``False`` to disable the feature again. - Register an :func:`unraisable hook handler ` for an - improved debug experience: - - .. testsetup:: sqlite3.trace - - import sqlite3 + .. note:: - .. doctest:: sqlite3.trace - - >>> sqlite3.enable_callback_tracebacks(True) - >>> con = sqlite3.connect(":memory:") - >>> def evil_trace(stmt): - ... 5/0 - ... - >>> con.set_trace_callback(evil_trace) - >>> def debug(unraisable): - ... print(f"{unraisable.exc_value!r} in callback {unraisable.object.__name__}") - ... print(f"Error message: {unraisable.err_msg}") - >>> import sys - >>> sys.unraisablehook = debug - >>> cur = con.execute("SELECT 1") - ZeroDivisionError('division by zero') in callback evil_trace - Error message: None + Errors in user-defined function callbacks are logged as unraisable exceptions. + Use an :func:`unraisable hook handler ` for + introspection of the failed callback. .. function:: register_adapter(type, adapter, /) @@ -540,46 +525,25 @@ Module constants The mappings from SQLite threading modes to DB-API 2.0 threadsafety levels are as follows: - +------------------+-----------------+----------------------+-------------------------------+ - | SQLite threading | `threadsafety`_ | `SQLITE_THREADSAFE`_ | DB-API 2.0 meaning | - | mode | | | | - +==================+=================+======================+===============================+ - | single-thread | 0 | 0 | Threads may not share the | - | | | | module | - +------------------+-----------------+----------------------+-------------------------------+ - | multi-thread | 1 | 2 | Threads may share the module, | - | | | | but not connections | - +------------------+-----------------+----------------------+-------------------------------+ - | serialized | 3 | 1 | Threads may share the module, | - | | | | connections and cursors | - +------------------+-----------------+----------------------+-------------------------------+ - - .. _threadsafety: https://peps.python.org/pep-0249/#threadsafety + +------------------+----------------------+----------------------+-------------------------------+ + | SQLite threading | :pep:`threadsafety | `SQLITE_THREADSAFE`_ | DB-API 2.0 meaning | + | mode | <0249#threadsafety>` | | | + +==================+======================+======================+===============================+ + | single-thread | 0 | 0 | Threads may not share the | + | | | | module | + +------------------+----------------------+----------------------+-------------------------------+ + | multi-thread | 1 | 2 | Threads may share the module, | + | | | | but not connections | + +------------------+----------------------+----------------------+-------------------------------+ + | serialized | 3 | 1 | Threads may share the module, | + | | | | connections and cursors | + +------------------+----------------------+----------------------+-------------------------------+ + .. _SQLITE_THREADSAFE: https://sqlite.org/compile.html#threadsafe .. versionchanged:: 3.11 Set *threadsafety* dynamically instead of hard-coding it to ``1``. -.. data:: version - - Version number of this module as a :class:`string `. - This is not the version of the SQLite library. - - .. deprecated-removed:: 3.12 3.14 - This constant used to reflect the version number of the ``pysqlite`` - package, a third-party library which used to upstream changes to - :mod:`!sqlite3`. Today, it carries no meaning or practical value. - -.. data:: version_info - - Version number of this module as a :class:`tuple` of :class:`integers `. - This is not the version of the SQLite library. - - .. deprecated-removed:: 3.12 3.14 - This constant used to reflect the version number of the ``pysqlite`` - package, a third-party library which used to upstream changes to - :mod:`!sqlite3`. Today, it carries no meaning or practical value. - .. _sqlite3-dbconfig-constants: .. data:: SQLITE_DBCONFIG_DEFENSIVE @@ -612,6 +576,8 @@ Module constants https://www.sqlite.org/c3ref/c_dbconfig_defensive.html SQLite docs: Database Connection Configuration Options +.. deprecated-removed:: 3.12 3.14 + The :data:`!version` and :data:`!version_info` constants. .. _sqlite3-connection-objects: @@ -762,6 +728,7 @@ Connection objects >>> for row in con.execute("SELECT md5(?)", (b"foo",)): ... print(row) ('acbd18db4cc2f85cedef654fccc4a4d8',) + >>> con.close() .. versionchanged:: 3.13 @@ -908,6 +875,7 @@ Connection objects FROM test ORDER BY x """) print(cur.fetchall()) + con.close() .. testoutput:: :hide: @@ -1068,13 +1036,10 @@ Connection objects .. versionchanged:: 3.10 Added the ``sqlite3.enable_load_extension`` auditing event. - .. testsetup:: sqlite3.loadext - - import sqlite3 - con = sqlite3.connect(":memory:") + .. We cannot doctest the load extension API, since there is no convenient + way to skip it. - .. testcode:: sqlite3.loadext - :skipif: True # not testable at the moment + .. code-block:: con.enable_load_extension(True) @@ -1098,14 +1063,6 @@ Connection objects for row in con.execute("SELECT rowid, name, ingredients FROM recipe WHERE name MATCH 'pie'"): print(row) - con.close() - - .. testoutput:: sqlite3.loadext - :hide: - - (2, 'broccoli pie', 'broccoli cheese onions flour') - (3, 'pumpkin pie', 'pumpkin sugar flour butter') - .. method:: load_extension(path, /, *, entrypoint=None) Load an SQLite extension from a shared library. @@ -1135,7 +1092,7 @@ Connection objects .. versionchanged:: 3.12 Added the *entrypoint* parameter. - .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_ + .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension .. method:: iterdump(*, filter=None) @@ -1230,6 +1187,8 @@ Connection objects src = sqlite3.connect('example.db') dst = sqlite3.connect(':memory:') src.backup(dst) + dst.close() + src.close() .. versionadded:: 3.7 @@ -1296,6 +1255,10 @@ Connection objects >>> con.getlimit(sqlite3.SQLITE_LIMIT_ATTACHED) 1 + .. testcleanup:: sqlite3.limits + + con.close() + .. versionadded:: 3.11 .. _SQLite limit category: https://www.sqlite.org/c3ref/c_limit_attached.html @@ -1577,6 +1540,10 @@ Cursor objects # cur is an sqlite3.Cursor object cur.executemany("INSERT INTO data VALUES(?)", rows) + .. testcleanup:: sqlite3.cursor + + con.close() + .. note:: Any resulting rows are discarded, @@ -1682,6 +1649,7 @@ Cursor objects >>> cur = con.cursor() >>> cur.connection == con True + >>> con.close() .. attribute:: description @@ -1802,6 +1770,7 @@ Blob objects greeting = blob.read() print(greeting) # outputs "b'Hello, world!'" + con.close() .. testoutput:: :hide: @@ -2114,6 +2083,7 @@ Here's an example of both styles: params = (1972,) cur.execute("SELECT * FROM lang WHERE first_appeared = ?", params) print(cur.fetchall()) + con.close() .. testoutput:: :hide: @@ -2172,6 +2142,7 @@ The object passed to *protocol* will be of type :class:`PrepareProtocol`. cur.execute("SELECT ?", (Point(4.0, -3.2),)) print(cur.fetchone()[0]) + con.close() .. testoutput:: :hide: @@ -2202,6 +2173,7 @@ This function can then be registered using :func:`register_adapter`. cur.execute("SELECT ?", (Point(1.0, 2.5),)) print(cur.fetchone()[0]) + con.close() .. testoutput:: :hide: @@ -2286,6 +2258,8 @@ The following example illustrates the implicit and explicit approaches: cur.execute("INSERT INTO test(p) VALUES(?)", (p,)) cur.execute('SELECT p AS "p [point]" FROM test') print("with column names:", cur.fetchone()[0]) + cur.close() + con.close() .. testoutput:: :hide: @@ -2468,6 +2442,7 @@ Some useful URI tricks include: >>> con.execute("CREATE TABLE readonly(data)") Traceback (most recent call last): OperationalError: attempt to write a readonly database + >>> con.close() * Do not implicitly create a new database file if it does not already exist; will raise :exc:`~sqlite3.OperationalError` if unable to create a new file: @@ -2492,6 +2467,8 @@ Some useful URI tricks include: res = con2.execute("SELECT data FROM shared") assert res.fetchone() == (28,) + con1.close() + con2.close() More information about this feature, including a list of parameters, can be found in the `SQLite URI documentation`_. @@ -2538,6 +2515,7 @@ Queries now return :class:`!Row` objects: 'Earth' >>> row["RADIUS"] # Column names are case-insensitive. 6378 + >>> con.close() .. note:: @@ -2564,6 +2542,7 @@ Using it, queries now return a :class:`!dict` instead of a :class:`!tuple`: >>> for row in con.execute("SELECT 1 AS a, 2 AS b"): ... print(row) {'a': 1, 'b': 2} + >>> con.close() The following row factory returns a :term:`named tuple`: @@ -2590,6 +2569,7 @@ The following row factory returns a :term:`named tuple`: 1 >>> row.b # Attribute access. 2 + >>> con.close() With some adjustments, the above recipe can be adapted to use a :class:`~dataclasses.dataclass`, or any other custom class, @@ -2747,3 +2727,11 @@ regardless of the value of :attr:`~Connection.isolation_level`. .. _SQLite transaction behaviour: https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions + +.. testcleanup:: + + import os + os.remove("backup.db") + os.remove("dump.sql") + os.remove("example.db") + os.remove("tutorial.db") diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index f9648fa6744bdc..b7fb1fc07d199f 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1,5 +1,5 @@ -:mod:`ssl` --- TLS/SSL wrapper for socket objects -================================================= +:mod:`!ssl` --- TLS/SSL wrapper for socket objects +================================================== .. module:: ssl :synopsis: TLS/SSL wrapper for socket objects @@ -151,6 +151,12 @@ purposes. variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` enables key logging. + The default settings for this context include + :data:`VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT`. + These make the underlying OpenSSL implementation behave more like + a conforming implementation of :rfc:`5280`, in exchange for a small + amount of incompatibility with older X.509 certificates. + .. note:: The protocol, options, cipher and other settings may change to more restrictive values anytime without prior deprecation. The values @@ -172,6 +178,15 @@ purposes. ctx = ssl.create_default_context(Purpose.CLIENT_AUTH) ctx.options &= ~ssl.OP_NO_SSLv3 + .. note:: + This context enables :data:`VERIFY_X509_STRICT` by default, which + may reject pre-:rfc:`5280` or malformed certificates that the + underlying OpenSSL implementation otherwise would accept. While disabling + this is not recommended, you can do so using:: + + ctx = ssl.create_default_context() + ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + .. versionadded:: 3.4 .. versionchanged:: 3.4.4 @@ -194,6 +209,11 @@ purposes. :data:`PROTOCOL_TLS_SERVER` protocol instead of generic :data:`PROTOCOL_TLS`. + .. versionchanged:: 3.13 + + The context now uses :data:`VERIFY_X509_PARTIAL_CHAIN` and + :data:`VERIFY_X509_STRICT` in its default verify flags. + Exceptions ^^^^^^^^^^ @@ -737,11 +757,11 @@ Constants When Python has been compiled against an older version of OpenSSL, the flag defaults to *0*. - .. versionadded:: 3.7 + .. versionadded:: 3.6.3 .. deprecated:: 3.7 - The option is deprecated since OpenSSL 1.1.0. It was added to 2.7.15, - 3.6.3 and 3.7.0 for backwards compatibility with OpenSSL 1.0.2. + The option is deprecated since OpenSSL 1.1.0. It was added to 2.7.15 and + 3.6.3 for backwards compatibility with OpenSSL 1.0.2. .. data:: OP_NO_RENEGOTIATION @@ -761,7 +781,7 @@ Constants .. data:: OP_SINGLE_DH_USE - Prevents re-use of the same DH key for distinct SSL sessions. This + Prevents reuse of the same DH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. @@ -769,7 +789,7 @@ Constants .. data:: OP_SINGLE_ECDH_USE - Prevents re-use of the same ECDH key for distinct SSL sessions. This + Prevents reuse of the same ECDH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. @@ -1029,25 +1049,25 @@ SSL Sockets SSL sockets provide the following methods of :ref:`socket-objects`: - - :meth:`~socket.socket.accept()` - - :meth:`~socket.socket.bind()` - - :meth:`~socket.socket.close()` - - :meth:`~socket.socket.connect()` - - :meth:`~socket.socket.detach()` - - :meth:`~socket.socket.fileno()` - - :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` - - :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` - - :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, - :meth:`~socket.socket.setblocking()` - - :meth:`~socket.socket.listen()` - - :meth:`~socket.socket.makefile()` - - :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` + - :meth:`~socket.socket.accept` + - :meth:`~socket.socket.bind` + - :meth:`~socket.socket.close` + - :meth:`~socket.socket.connect` + - :meth:`~socket.socket.detach` + - :meth:`~socket.socket.fileno` + - :meth:`~socket.socket.getpeername`, :meth:`~socket.socket.getsockname` + - :meth:`~socket.socket.getsockopt`, :meth:`~socket.socket.setsockopt` + - :meth:`~socket.socket.gettimeout`, :meth:`~socket.socket.settimeout`, + :meth:`~socket.socket.setblocking` + - :meth:`~socket.socket.listen` + - :meth:`~socket.socket.makefile` + - :meth:`~socket.socket.recv`, :meth:`~socket.socket.recv_into` (but passing a non-zero ``flags`` argument is not allowed) - - :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with + - :meth:`~socket.socket.send`, :meth:`~socket.socket.sendall` (with the same limitation) - - :meth:`~socket.socket.sendfile()` (but :mod:`os.sendfile` will be used - for plain-text sockets only, else :meth:`~socket.socket.send()` will be used) - - :meth:`~socket.socket.shutdown()` + - :meth:`~socket.socket.sendfile` (but :mod:`os.sendfile` will be used + for plain-text sockets only, else :meth:`~socket.socket.send` will be used) + - :meth:`~socket.socket.shutdown` However, since the SSL (and TLS) protocol has its own framing atop of TCP, the SSL sockets abstraction can, in certain respects, diverge from @@ -1452,6 +1472,19 @@ to speed up repeated connections from the same clients. :data:`PROTOCOL_TLS`, :data:`PROTOCOL_TLS_CLIENT`, and :data:`PROTOCOL_TLS_SERVER` use TLS 1.2 as minimum TLS version. + .. note:: + + :class:`SSLContext` only supports limited mutation once it has been used + by a connection. Adding new certificates to the internal trust store is + allowed, but changing ciphers, verification settings, or mTLS + certificates may result in surprising behavior. + + .. note:: + + :class:`SSLContext` is designed to be shared and used by multiple + connections. + Thus, it is thread-safe as long as it is not reconfigured after being + used by a connection. :class:`SSLContext` objects have the following methods and attributes: @@ -1533,7 +1566,7 @@ to speed up repeated connections from the same clients. The *capath* string, if present, is the path to a directory containing several CA certificates in PEM format, following an `OpenSSL specific layout - `_. + `_. The *cadata* object, if present, is either an ASCII string of one or more PEM-encoded certificates or a :term:`bytes-like object` of DER-encoded @@ -1608,7 +1641,7 @@ to speed up repeated connections from the same clients. Set the available ciphers for sockets created with this context. It should be a string in the `OpenSSL cipher list format - `_. + `_. If no cipher can be selected (because compile-time options or other configuration forbids use of all the specified ciphers), an :class:`SSLError` will be raised. @@ -1709,7 +1742,7 @@ to speed up repeated connections from the same clients. IDN-encoded internationalized domain name, the *server_name_callback* receives a decoded U-label (``"pythön.org"``). - If there is an decoding error on the server name, the TLS connection will + If there is a decoding error on the server name, the TLS connection will terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS alert message to the client. @@ -1790,6 +1823,9 @@ to speed up repeated connections from the same clients. *session*, see :attr:`~SSLSocket.session`. + To wrap an :class:`SSLSocket` in another :class:`SSLSocket`, use + :meth:`SSLContext.wrap_bio`. + .. versionchanged:: 3.5 Always allow a server_hostname to be passed, even if OpenSSL does not have SNI. @@ -1797,7 +1833,7 @@ to speed up repeated connections from the same clients. .. versionchanged:: 3.6 *session* argument was added. - .. versionchanged:: 3.7 + .. versionchanged:: 3.7 The method returns an instance of :attr:`SSLContext.sslsocket_class` instead of hard-coded :class:`SSLSocket`. @@ -1838,7 +1874,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.session_stats() Get statistics about the SSL sessions created or managed by this context. - A dictionary is returned which maps the names of each `piece of information `_ to their + A dictionary is returned which maps the names of each `piece of information `_ to their numeric values. For example, here is the total number of hits and misses in the session cache since the context was created:: @@ -1975,13 +2011,13 @@ to speed up repeated connections from the same clients. .. versionchanged:: 3.10 - The flag had no effect with OpenSSL before version 1.1.1k. Python 3.8.9, + The flag had no effect with OpenSSL before version 1.1.1l. Python 3.8.9, 3.9.3, and 3.10 include workarounds for previous versions. .. attribute:: SSLContext.security_level An integer representing the `security level - `_ + `_ for the context. This attribute is read-only. .. versionadded:: 3.10 @@ -2674,7 +2710,7 @@ Verifying certificates When calling the :class:`SSLContext` constructor directly, :const:`CERT_NONE` is the default. Since it does not authenticate the other -peer, it can be insecure, especially in client mode where most of time you +peer, it can be insecure, especially in client mode where most of the time you would like to ensure the authenticity of the server you're talking to. Therefore, when in client mode, it is highly recommended to use :const:`CERT_REQUIRED`. However, it is in itself not sufficient; you also @@ -2723,7 +2759,7 @@ enabled when negotiating a SSL session is possible through the :meth:`SSLContext.set_ciphers` method. Starting from Python 3.2.3, the ssl module disables certain weak ciphers by default, but you may want to further restrict the cipher choice. Be sure to read OpenSSL's documentation -about the `cipher list format `_. +about the `cipher list format `_. If you want to check which ciphers are enabled by a given cipher list, use :meth:`SSLContext.get_ciphers` or the ``openssl ciphers`` command on your system. diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index c941d5557e31b5..8434b2e8c75cf4 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -1,5 +1,5 @@ -:mod:`stat` --- Interpreting :func:`~os.stat` results -===================================================== +:mod:`!stat` --- Interpreting :func:`~os.stat` results +====================================================== .. module:: stat :synopsis: Utilities for interpreting the results of os.stat(), @@ -354,7 +354,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: All user settable flags. - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: UF_NODUMP @@ -384,13 +384,13 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: Used for handling document IDs (macOS) - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: UF_DATAVAULT The file needs an entitlement for reading or writing (macOS 10.13+) - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: UF_HIDDEN @@ -400,7 +400,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: All super-user changeable flags - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_SUPPORTED @@ -408,7 +408,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: .. availability:: macOS - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_SYNTHETIC @@ -416,7 +416,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: .. availability:: macOS - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_ARCHIVED @@ -434,7 +434,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file needs an entitlement to write to (macOS 10.13+) - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_NOUNLINK @@ -448,13 +448,13 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file is a firmlink (macOS 10.15+) - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_DATALESS The file is a dataless object (macOS 10.15+) - .. versionadded: 3.13 + .. versionadded:: 3.13 See the \*BSD or macOS systems man page :manpage:`chflags(2)` for more information. diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 0417b3f38a9807..614f5b905a4a2e 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -1,5 +1,5 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= +:mod:`!statistics` --- Mathematical statistics functions +======================================================== .. module:: statistics :synopsis: Mathematical statistics functions @@ -73,13 +73,15 @@ or sample. ======================= =============================================================== :func:`mean` Arithmetic mean ("average") of data. -:func:`fmean` Fast, floating point arithmetic mean, with optional weighting. +:func:`fmean` Fast, floating-point arithmetic mean, with optional weighting. :func:`geometric_mean` Geometric mean of data. :func:`harmonic_mean` Harmonic mean of data. +:func:`kde` Estimate the probability density distribution of the data. +:func:`kde_random` Random sampling from the PDF generated by kde(). :func:`median` Median (middle value) of data. :func:`median_low` Low median of data. :func:`median_high` High median of data. -:func:`median_grouped` Median, or 50th percentile, of grouped data. +:func:`median_grouped` Median (50th percentile) of grouped data. :func:`mode` Single mode (most common value) of discrete or nominal data. :func:`multimode` List of modes (most common values) of discrete or nominal data. :func:`quantiles` Divide data into intervals with equal probability. @@ -218,7 +220,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: harmonic_mean(data, weights=None) Return the harmonic mean of *data*, a sequence or iterable of - real-valued numbers. If *weights* is omitted or *None*, then + real-valued numbers. If *weights* is omitted or ``None``, then equal weighting is assumed. The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the @@ -259,6 +261,81 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionchanged:: 3.10 Added support for *weights*. + +.. function:: kde(data, h, kernel='normal', *, cumulative=False) + + `Kernel Density Estimation (KDE) + `_: + Create a continuous probability density function or cumulative + distribution function from discrete samples. + + The basic idea is to smooth the data using `a kernel function + `_. + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter *h* + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The *kernel* determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point include + *normal* (*gauss*), *logistic*, and *sigmoid*. + + Kernels that only give weight to sample points within the bandwidth + include *rectangular* (*uniform*), *triangular*, *parabolic* + (*epanechnikov*), *quartic* (*biweight*), *triweight*, and *cosine*. + + If *cumulative* is true, will return a cumulative distribution function. + + A :exc:`StatisticsError` will be raised if the *data* sequence is empty. + + `Wikipedia has an example + `_ + where we can use :func:`kde` to generate and plot a probability + density function estimated from a small sample: + + .. doctest:: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + >>> xarr = [i/100 for i in range(-750, 1100)] + >>> yarr = [f_hat(x) for x in xarr] + + The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: + + .. image:: kde_example.png + :alt: Scatter plot of the estimated probability density function. + + .. versionadded:: 3.13 + + +.. function:: kde_random(data, h, kernel='normal', *, seed=None) + + Return a function that makes a random selection from the estimated + probability density function produced by ``kde(data, h, kernel)``. + + Providing a *seed* allows reproducible selections. In the future, the + values may change slightly as more accurate kernel inverse CDF estimates + are implemented. The seed may be an integer, float, str, or bytes. + + A :exc:`StatisticsError` will be raised if the *data* sequence is empty. + + Continuing the example for :func:`kde`, we can use + :func:`kde_random` to generate new random selections from an + estimated probability density function: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + .. versionadded:: 3.13 + + .. function:: median(data) Return the median (middle value) of numeric data, using the common "mean of @@ -329,55 +406,56 @@ However, for reading convenience, most of the examples show sorted sequences. be an actual data point rather than interpolated. -.. function:: median_grouped(data, interval=1) +.. function:: median_grouped(data, interval=1.0) - Return the median of grouped continuous data, calculated as the 50th - percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. + Estimates the median for numeric data that has been `grouped or binned + `_ around the midpoints + of consecutive, fixed-width intervals. - .. doctest:: + The *data* can be any iterable of numeric data with each value being + exactly the midpoint of a bin. At least one value must be present. - >>> median_grouped([52, 52, 53, 54]) - 52.5 + The *interval* is the width of each bin. - In the following example, the data are rounded, so that each value represents - the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 - is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data - given, the middle value falls somewhere in the class 3.5--4.5, and - interpolation is used to estimate it: + For example, demographic information may have been summarized into + consecutive ten-year age groups with each group being represented + by the 5-year midpoints of the intervals: .. doctest:: - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - - Optional argument *interval* represents the class interval, and defaults - to 1. Changing the class interval naturally will change the interpolation: + >>> from collections import Counter + >>> demographics = Counter({ + ... 25: 172, # 20 to 30 years old + ... 35: 484, # 30 to 40 years old + ... 45: 387, # 40 to 50 years old + ... 55: 22, # 50 to 60 years old + ... 65: 6, # 60 to 70 years old + ... }) + ... + + The 50th percentile (median) is the 536th person out of the 1071 + member cohort. That person is in the 30 to 40 year old age group. + + The regular :func:`median` function would assume that everyone in the + tricenarian age group was exactly 35 years old. A more tenable + assumption is that the 484 members of that age group are evenly + distributed between 30 and 40. For that, we use + :func:`median_grouped`: .. doctest:: - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 - - This function does not check whether the data points are at least - *interval* apart. + >>> data = list(demographics.elements()) + >>> median(data) + 35 + >>> round(median_grouped(data, interval=10), 1) + 37.5 - .. impl-detail:: + The caller is responsible for making sure the data points are separated + by exact multiples of *interval*. This is essential for getting a + correct result. The function does not check this precondition. - Under some circumstances, :func:`median_grouped` may coerce data points to - floats. This behaviour is likely to change in the future. - - .. seealso:: - - * "Statistics for the Behavioral Sciences", Frederick J Gravetter and - Larry B Wallnau (8th Edition). - - * The `SSMEDIAN - `_ - function in the Gnome Gnumeric spreadsheet, including `this discussion - `_. + Inputs may be any numeric type that can be coerced to a float during + the interpolation step. .. function:: mode(data) @@ -407,6 +485,12 @@ However, for reading convenience, most of the examples show sorted sequences. >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) 'red' + Only hashable inputs are supported. To handle type :class:`set`, + consider casting to :class:`frozenset`. To handle type :class:`list`, + consider casting to :class:`tuple`. For mixed or nested inputs, consider + using this slower quadratic algorithm that only depends on equality tests: + ``max(data, key=data.count)``. + .. versionchanged:: 3.8 Now handles multimodal datasets by returning the first mode encountered. Formerly, it raised :exc:`StatisticsError` when more than one mode was @@ -448,9 +532,9 @@ However, for reading convenience, most of the examples show sorted sequences. variance indicates that the data is spread out; a small variance indicates it is clustered closely around the mean. - If the optional second argument *mu* is given, it is typically the mean of - the *data*. It can also be used to compute the second moment around a - point that is not the mean. If it is missing or ``None`` (the default), + If the optional second argument *mu* is given, it should be the *population* + mean of the *data*. It can also be used to compute the second moment around + a point that is not the mean. If it is missing or ``None`` (the default), the arithmetic mean is automatically calculated. Use this function to calculate the variance from the entire population. To @@ -520,8 +604,8 @@ However, for reading convenience, most of the examples show sorted sequences. the data is spread out; a small variance indicates it is clustered closely around the mean. - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is + If the optional second argument *xbar* is given, it should be the *sample* + mean of *data*. If it is missing or ``None`` (the default), the mean is automatically calculated. Use this function when your data is a sample from a population. To calculate @@ -537,8 +621,8 @@ However, for reading convenience, most of the examples show sorted sequences. >>> variance(data) 1.3720238095238095 - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: + If you have already calculated the sample mean of your data, you can pass it + as the optional second argument *xbar* to avoid recalculation: .. doctest:: @@ -948,8 +1032,8 @@ of applications in statistics. .. versionadded:: 3.8 -:class:`NormalDist` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Examples and Recipes +-------------------- Classic probability problems @@ -984,7 +1068,7 @@ Find the `quartiles `_ and `deciles Monte Carlo inputs for simulations ********************************** -To estimate the distribution for a model than isn't easy to solve +To estimate the distribution for a model that isn't easy to solve analytically, :class:`NormalDist` can generate input samples for a `Monte Carlo simulation `_: @@ -1095,46 +1179,6 @@ The final prediction goes to the largest posterior. This is known as the 'female' -Kernel density estimation -************************* - -It is possible to estimate a continuous probability density function -from a fixed number of discrete samples. - -The basic idea is to smooth the data using `a kernel function such as a -normal distribution, triangular distribution, or uniform distribution -`_. -The degree of smoothing is controlled by a scaling parameter, ``h``, -which is called the *bandwidth*. - -.. testcode:: - - def kde_normal(sample, h): - "Create a continuous probability density function from a sample." - # Smooth the sample with a normal distribution kernel scaled by h. - kernel_h = NormalDist(0.0, h).pdf - n = len(sample) - def pdf(x): - return sum(kernel_h(x - x_i) for x_i in sample) / n - return pdf - -`Wikipedia has an example -`_ -where we can use the ``kde_normal()`` recipe to generate and plot -a probability density function estimated from a small sample: - -.. doctest:: - - >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde_normal(sample, h=1.5) - >>> xarr = [i/100 for i in range(-750, 1100)] - >>> yarr = [f_hat(x) for x in xarr] - -The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: - -.. image:: kde_example.png - :alt: Scatter plot of the estimated probability density function. - .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 1a4c12590c1018..a9b7662dcb212b 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -209,18 +209,18 @@ Numeric Types --- :class:`int`, :class:`float`, :class:`complex` pair: object; numeric pair: object; Boolean pair: object; integer - pair: object; floating point + pair: object; floating-point pair: object; complex number pair: C; language -There are three distinct numeric types: :dfn:`integers`, :dfn:`floating -point numbers`, and :dfn:`complex numbers`. In addition, Booleans are a -subtype of integers. Integers have unlimited precision. Floating point +There are three distinct numeric types: :dfn:`integers`, :dfn:`floating-point +numbers`, and :dfn:`complex numbers`. In addition, Booleans are a +subtype of integers. Integers have unlimited precision. Floating-point numbers are usually implemented using :c:expr:`double` in C; information -about the precision and internal representation of floating point +about the precision and internal representation of floating-point numbers for the machine on which your program is running is available in :data:`sys.float_info`. Complex numbers have a real and imaginary -part, which are each a floating point number. To extract these parts +part, which are each a floating-point number. To extract these parts from a complex number *z*, use ``z.real`` and ``z.imag``. (The standard library includes the additional numeric types :mod:`fractions.Fraction`, for rationals, and :mod:`decimal.Decimal`, for floating-point numbers with @@ -229,7 +229,7 @@ user-definable precision.) .. index:: pair: numeric; literals pair: integer; literals - pair: floating point; literals + pair: floating-point; literals pair: complex number; literals pair: hexadecimal; literals pair: octal; literals @@ -238,7 +238,7 @@ user-definable precision.) Numbers are created by numeric literals or as the result of built-in functions and operators. Unadorned integer literals (including hex, octal and binary numbers) yield integers. Numeric literals containing a decimal point or an -exponent sign yield floating point numbers. Appending ``'j'`` or ``'J'`` to a +exponent sign yield floating-point numbers. Appending ``'j'`` or ``'J'`` to a numeric literal yields an imaginary number (a complex number with a zero real part) which you can add to an integer or float to get a complex number with real and imaginary parts. @@ -625,6 +625,23 @@ Additional Methods on Float The float type implements the :class:`numbers.Real` :term:`abstract base class`. float also has the following additional methods. +.. classmethod:: float.from_number(x) + + Class method to return a floating-point number constructed from a number *x*. + + If the argument is an integer or a floating-point number, a + floating-point number with the same value (within Python's floating-point + precision) is returned. If the argument is outside the range of a Python + float, an :exc:`OverflowError` will be raised. + + For a general Python object ``x``, ``float.from_number(x)`` delegates to + ``x.__float__()``. + If :meth:`~object.__float__` is not defined then it falls back + to :meth:`~object.__index__`. + + .. versionadded:: 3.14 + + .. method:: float.as_integer_ratio() Return a pair of integers whose ratio is exactly equal to the @@ -703,6 +720,25 @@ hexadecimal string representing the same number:: '0x1.d380000000000p+11' +Additional Methods on Complex +----------------------------- + +The :class:`!complex` type implements the :class:`numbers.Complex` +:term:`abstract base class`. +:class:`!complex` also has the following additional methods. + +.. classmethod:: complex.from_number(x) + + Class method to convert a number to a complex number. + + For a general Python object ``x``, ``complex.from_number(x)`` delegates to + ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back + to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + to :meth:`~object.__index__`. + + .. versionadded:: 3.14 + + .. _numeric-hash: Hashing of numeric types @@ -832,7 +868,7 @@ over ``&``, ``|`` and ``^``. .. deprecated:: 3.12 The use of the bitwise inversion operator ``~`` is deprecated and will - raise an error in Python 3.14. + raise an error in Python 3.16. :class:`bool` is a subclass of :class:`int` (see :ref:`typesnumeric`). In many numeric contexts, ``False`` and ``True`` behave like the integers 0 and 1, respectively. @@ -1209,8 +1245,9 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | ``s.pop()`` or ``s.pop(i)`` | retrieves the item at *i* and | \(2) | | | also removes it from *s* | | +------------------------------+--------------------------------+---------------------+ -| ``s.remove(x)`` | remove the first item from *s* | \(3) | -| | where ``s[i]`` is equal to *x* | | +| ``s.remove(x)`` | removes the first item from | \(3) | +| | *s* where ``s[i]`` is equal to | | +| | *x* | | +------------------------------+--------------------------------+---------------------+ | ``s.reverse()`` | reverses the items of *s* in | \(4) | | | place | | @@ -1220,7 +1257,7 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). Notes: (1) - *t* must have the same length as the slice it is replacing. + If *k* is not equal to ``1``, *t* must have the same length as the slice it is replacing. (2) The optional argument *i* defaults to ``-1``, so that by default the last @@ -1491,15 +1528,14 @@ objects that compare equal might have different :attr:`~range.start`, sequence of values they define (instead of comparing based on object identity). -.. versionadded:: 3.3 - The :attr:`~range.start`, :attr:`~range.stop` and :attr:`~range.step` + Added the :attr:`~range.start`, :attr:`~range.stop` and :attr:`~range.step` attributes. .. seealso:: - * The `linspace recipe `_ - shows how to implement a lazy version of range suitable for floating - point applications. + * The `linspace recipe `_ + shows how to implement a lazy version of range suitable for floating-point + applications. .. index:: single: string; text sequence type @@ -1643,7 +1679,7 @@ expression support in the :mod:`re` module). The casefolding algorithm is `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + `__. .. versionadded:: 3.3 @@ -1769,7 +1805,7 @@ expression support in the :mod:`re` module). cases. -.. method:: str.format_map(mapping) +.. method:: str.format_map(mapping, /) Similar to ``str.format(**mapping)``, except that ``mapping`` is used directly and not copied to a :class:`dict`. This is useful @@ -1807,7 +1843,7 @@ expression support in the :mod:`re` module). property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard - `_. + `_. .. method:: str.isascii() @@ -1943,7 +1979,7 @@ expression support in the :mod:`re` module). The lowercasing algorithm used is `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + `__. .. method:: str.lstrip([chars]) @@ -2096,8 +2132,9 @@ expression support in the :mod:`re` module). If *sep* is given, consecutive delimiters are not grouped together and are deemed to delimit empty strings (for example, ``'1,,2'.split(',')`` returns ``['1', '', '2']``). The *sep* argument may consist of multiple characters - (for example, ``'1<>2<>3'.split('<>')`` returns ``['1', '2', '3']``). - Splitting an empty string with a specified separator returns ``['']``. + as a single delimiter (to split with multiple delimiters, use + :func:`re.split`). Splitting an empty string with a specified separator + returns ``['']``. For example:: @@ -2107,6 +2144,8 @@ expression support in the :mod:`re` module). ['1', '2,3'] >>> '1,2,,3,'.split(',') ['1', '2', '', '3', ''] + >>> '1<>2<>3<4'.split('<>') + ['1', '2', '3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm is applied: runs of consecutive whitespace are regarded as a single separator, @@ -2292,7 +2331,7 @@ expression support in the :mod:`re` module). The uppercasing algorithm used is `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + `__. .. method:: str.zfill(width) @@ -2340,7 +2379,13 @@ String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. Given ``format % values`` (where *format* is a string), ``%`` conversion specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. +The effect is similar to using the :c:func:`sprintf` function in the C language. +For example: + +.. doctest:: + + >>> print('%s has %d quote types.' % ('Python', 2)) + Python has 2 quote types. If *format* requires a single argument, *values* may be a single non-tuple object. [5]_ Otherwise, *values* must be a tuple with exactly the number of @@ -2434,19 +2479,19 @@ The conversion types are: +------------+-----------------------------------------------------+-------+ | ``'X'`` | Signed hexadecimal (uppercase). | \(2) | +------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | +| ``'e'`` | Floating-point exponential format (lowercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | +| ``'E'`` | Floating-point exponential format (uppercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | +| ``'f'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | +| ``'F'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | +| ``'g'`` | Floating-point format. Uses lowercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | +| ``'G'`` | Floating-point format. Uses uppercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ @@ -3144,10 +3189,9 @@ produce new objects. If *sep* is given, consecutive delimiters are not grouped together and are deemed to delimit empty subsequences (for example, ``b'1,,2'.split(b',')`` returns ``[b'1', b'', b'2']``). The *sep* argument may consist of a - multibyte sequence (for example, ``b'1<>2<>3'.split(b'<>')`` returns - ``[b'1', b'2', b'3']``). Splitting an empty sequence with a specified - separator returns ``[b'']`` or ``[bytearray(b'')]`` depending on the type - of object being split. The *sep* argument may be any + multibyte sequence as a single delimiter. Splitting an empty sequence with + a specified separator returns ``[b'']`` or ``[bytearray(b'')]`` depending + on the type of object being split. The *sep* argument may be any :term:`bytes-like object`. For example:: @@ -3158,6 +3202,8 @@ produce new objects. [b'1', b'2,3'] >>> b'1,2,,3,'.split(b',') [b'1', b'2', b'', b'3', b''] + >>> b'1<>2<>3<4'.split(b'<>') + [b'1', b'2', b'3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm is applied: runs of consecutive ASCII whitespace are regarded as a single @@ -3431,7 +3477,7 @@ place, and instead produce new objects. ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - Unlike :func:`str.swapcase()`, it is always the case that + Unlike :func:`str.swapcase`, it is always the case that ``bin.swapcase().swapcase() == bin`` for the binary versions. Case conversions are symmetrical in ASCII, even though that is not generally true for arbitrary Unicode code points. @@ -3652,19 +3698,19 @@ The conversion types are: +------------+-----------------------------------------------------+-------+ | ``'X'`` | Signed hexadecimal (uppercase). | \(2) | +------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | +| ``'e'`` | Floating-point exponential format (lowercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | +| ``'E'`` | Floating-point exponential format (uppercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | +| ``'f'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | +| ``'F'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | +| ``'g'`` | Floating-point format. Uses lowercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | +| ``'G'`` | Floating-point format. Uses uppercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ @@ -3843,6 +3889,9 @@ copying. .. versionchanged:: 3.5 memoryviews can now be indexed with tuple of integers. + .. versionchanged:: next + memoryview is now a :term:`generic type`. + :class:`memoryview` has several methods: .. method:: __eq__(exporter) @@ -3886,7 +3935,7 @@ copying. >>> a == b False - Note that, as with floating point numbers, ``v is w`` does *not* imply + Note that, as with floating-point numbers, ``v is w`` does *not* imply ``v == w`` for memoryview objects. .. versionchanged:: 3.3 @@ -3977,7 +4026,7 @@ copying. dangling resources) as soon as possible. After this method has been called, any further operation on the view - raises a :class:`ValueError` (except :meth:`release()` itself which can + raises a :class:`ValueError` (except :meth:`release` itself which can be called multiple times):: >>> m = memoryview(b'abc') @@ -4459,14 +4508,14 @@ can be used interchangeably to index the same dictionary entry. ``dict([('foo', 100), ('bar', 200)])``, ``dict(foo=100, bar=200)`` If no positional argument is given, an empty dictionary is created. - If a positional argument is given and it is a mapping object, a dictionary - is created with the same key-value pairs as the mapping object. Otherwise, - the positional argument must be an :term:`iterable` object. Each item in - the iterable must itself be an iterable with exactly two objects. The - first object of each item becomes a key in the new dictionary, and the - second object the corresponding value. If a key occurs more than once, the - last value for that key becomes the corresponding value in the new - dictionary. + If a positional argument is given and it defines a ``keys()`` method, a + dictionary is created by calling :meth:`~object.__getitem__` on the argument with + each returned key from the method. Otherwise, the positional argument must be an + :term:`iterable` object. Each item in the iterable must itself be an iterable + with exactly two elements. The first element of each item becomes a key in the + new dictionary, and the second element the corresponding value. If a key occurs + more than once, the last value for that key becomes the corresponding value in + the new dictionary. If keyword arguments are given, the keyword arguments and their values are added to the dictionary created from the positional argument. If a key @@ -4560,7 +4609,7 @@ can be used interchangeably to index the same dictionary entry. Return a shallow copy of the dictionary. - .. classmethod:: fromkeys(iterable[, value]) + .. classmethod:: fromkeys(iterable, value=None, /) Create a new dictionary with keys from *iterable* and values set to *value*. @@ -4570,7 +4619,7 @@ can be used interchangeably to index the same dictionary entry. such as an empty list. To get distinct values, use a :ref:`dict comprehension ` instead. - .. method:: get(key[, default]) + .. method:: get(key, default=None) Return the value for *key* if *key* is in the dictionary, else *default*. If *default* is not given, it defaults to ``None``, so that this method @@ -4612,7 +4661,7 @@ can be used interchangeably to index the same dictionary entry. .. versionadded:: 3.8 - .. method:: setdefault(key[, default]) + .. method:: setdefault(key, default=None) If *key* is in the dictionary, return its value. If not, insert *key* with a value of *default* and return *default*. *default* defaults to @@ -4623,10 +4672,11 @@ can be used interchangeably to index the same dictionary entry. Update the dictionary with the key/value pairs from *other*, overwriting existing keys. Return ``None``. - :meth:`update` accepts either another dictionary object or an iterable of - key/value pairs (as tuples or other iterables of length two). If keyword - arguments are specified, the dictionary is then updated with those - key/value pairs: ``d.update(red=1, blue=2)``. + :meth:`update` accepts either another object with a ``keys()`` method (in + which case :meth:`~object.__getitem__` is called with every key returned from + the method) or an iterable of key/value pairs (as tuples or other iterables + of length two). If keyword arguments are specified, the dictionary is then + updated with those key/value pairs: ``d.update(red=1, blue=2)``. .. method:: values() @@ -5056,7 +5106,6 @@ list is non-exhaustive. * :class:`collections.abc.MutableMapping` * :class:`collections.abc.Sequence` * :class:`collections.abc.MutableSequence` -* :class:`collections.abc.ByteString` * :class:`collections.abc.MappingView` * :class:`collections.abc.KeysView` * :class:`collections.abc.ItemsView` @@ -5450,10 +5499,10 @@ The NotImplemented Object This object is returned from comparisons and binary operations when they are asked to operate on types they don't support. See :ref:`comparisons` for more -information. There is exactly one ``NotImplemented`` object. -``type(NotImplemented)()`` produces the singleton instance. +information. There is exactly one :data:`NotImplemented` object. +:code:`type(NotImplemented)()` produces the singleton instance. -It is written as ``NotImplemented``. +It is written as :code:`NotImplemented`. .. _typesinternal: @@ -5476,22 +5525,6 @@ types, where they are relevant. Some of these are not reported by the :func:`dir` built-in function. -.. attribute:: object.__dict__ - - A dictionary or other mapping object used to store an object's (writable) - attributes. - - -.. attribute:: instance.__class__ - - The class to which a class instance belongs. - - -.. attribute:: class.__bases__ - - The tuple of base classes of a class object. - - .. attribute:: definition.__name__ The name of the class, function, method, descriptor, or @@ -5506,35 +5539,23 @@ types, where they are relevant. Some of these are not reported by the .. versionadded:: 3.3 -.. attribute:: definition.__type_params__ +.. attribute:: definition.__module__ - The :ref:`type parameters ` of generic classes, functions, - and :ref:`type aliases `. - - .. versionadded:: 3.12 - - -.. attribute:: class.__mro__ + The name of the module in which a class or function was defined. - This attribute is a tuple of classes that are considered when looking for - base classes during method resolution. +.. attribute:: definition.__doc__ -.. method:: class.mro() + The documentation string of a class or function, or ``None`` if undefined. - This method can be overridden by a metaclass to customize the method - resolution order for its instances. It is called at class instantiation, and - its result is stored in :attr:`~class.__mro__`. +.. attribute:: definition.__type_params__ -.. method:: class.__subclasses__ - - Each class keeps a list of weak references to its immediate subclasses. This - method returns a list of all those references still alive. The list is in - definition order. Example:: + The :ref:`type parameters ` of generic classes, functions, + and :ref:`type aliases `. For classes and functions that + are not generic, this will be an empty tuple. - >>> int.__subclasses__() - [, , , ] + .. versionadded:: 3.12 .. _int_max_str_digits: @@ -5554,8 +5575,7 @@ a string to a binary integer or a binary integer to a string in linear time, have sub-quadratic complexity. Converting a large value such as ``int('1' * 500_000)`` can take over a second on a fast CPU. -Limiting conversion size offers a practical way to avoid `CVE-2020-10735 -`_. +Limiting conversion size offers a practical way to avoid :cve:`2020-10735`. The limit is applied to the number of digit characters in the input or output string when a non-linear conversion algorithm would be involved. Underscores diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 1867678b2077fc..9e8e44a8abe770 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -1,5 +1,5 @@ -:mod:`string` --- Common string operations -========================================== +:mod:`!string` --- Common string operations +=========================================== .. module:: string :synopsis: Common string operations. @@ -350,8 +350,9 @@ The meaning of the various alignment options is as follows: | ``'='`` | Forces the padding to be placed after the sign (if any) | | | but before the digits. This is used for printing fields | | | in the form '+000000120'. This alignment option is only | -| | valid for numeric types. It becomes the default for | -| | numbers when '0' immediately precedes the field width. | +| | valid for numeric types, excluding :class:`complex`. | +| | It becomes the default for numbers when '0' immediately | +| | precedes the field width. | +---------+----------------------------------------------------------+ | ``'^'`` | Forces the field to be centered within the available | | | space. | @@ -418,7 +419,7 @@ instead. .. index:: single: _ (underscore); in string formatting The ``'_'`` option signals the use of an underscore for a thousands -separator for floating point presentation types and for integer +separator for floating-point presentation types and for integer presentation type ``'d'``. For integer presentation types ``'b'``, ``'o'``, ``'x'``, and ``'X'``, underscores will be inserted every 4 digits. For other presentation types, specifying this option is an @@ -432,9 +433,9 @@ including any prefixes, separators, and other formatting characters. If not specified, then the field width will be determined by the content. When no explicit alignment is given, preceding the *width* field by a zero -(``'0'``) character enables -sign-aware zero-padding for numeric types. This is equivalent to a *fill* -character of ``'0'`` with an *alignment* type of ``'='``. +(``'0'``) character enables sign-aware zero-padding for numeric types, +excluding :class:`complex`. This is equivalent to a *fill* character of +``'0'`` with an *alignment* type of ``'='``. .. versionchanged:: 3.10 Preceding the *width* field by ``'0'`` no longer affects the default @@ -491,9 +492,9 @@ The available integer presentation types are: +---------+----------------------------------------------------------+ In addition to the above presentation types, integers can be formatted -with the floating point presentation types listed below (except +with the floating-point presentation types listed below (except ``'n'`` and ``None``). When doing so, :func:`float` is used to convert the -integer to a floating point number before formatting. +integer to a floating-point number before formatting. The available presentation types for :class:`float` and :class:`~decimal.Decimal` values are: @@ -509,9 +510,8 @@ The available presentation types for :class:`float` and | | significant digits. With no precision given, uses a | | | precision of ``6`` digits after the decimal point for | | | :class:`float`, and shows all coefficient digits | - | | for :class:`~decimal.Decimal`. If no digits follow the | - | | decimal point, the decimal point is also removed unless | - | | the ``#`` option is used. | + | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | + | | point is omitted unless the ``#`` option is used. | +---------+----------------------------------------------------------+ | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | | | an upper case 'E' as the separator character. | @@ -522,9 +522,8 @@ The available presentation types for :class:`float` and | | precision given, uses a precision of ``6`` digits after | | | the decimal point for :class:`float`, and uses a | | | precision large enough to show all coefficient digits | - | | for :class:`~decimal.Decimal`. If no digits follow the | - | | decimal point, the decimal point is also removed unless | - | | the ``#`` option is used. | + | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | + | | point is omitted unless the ``#`` option is used. | +---------+----------------------------------------------------------+ | ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts | | | ``nan`` to ``NAN`` and ``inf`` to ``INF``. | @@ -574,11 +573,13 @@ The available presentation types for :class:`float` and | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | +---------+----------------------------------------------------------+ - | None | For :class:`float` this is the same as ``'g'``, except | + | None | For :class:`float` this is like the ``'g'`` type, except | | | that when fixed-point notation is used to format the | | | result, it always includes at least one digit past the | - | | decimal point. The precision used is as large as needed | - | | to represent the given value faithfully. | + | | decimal point, and switches to the scientific notation | + | | when ``exp >= p - 1``. When the precision is not | + | | specified, the latter will be as large as needed to | + | | represent the given value faithfully. | | | | | | For :class:`~decimal.Decimal`, this is the same as | | | either ``'g'`` or ``'G'`` depending on the value of | @@ -588,6 +589,15 @@ The available presentation types for :class:`float` and | | as altered by the other format modifiers. | +---------+----------------------------------------------------------+ +The available presentation types for :class:`complex` are the same as those for +:class:`float` (``'%'`` is not allowed). Both the real and imaginary components +of a complex number are formatted as floating-point numbers, according to the +specified presentation type. They are separated by the mandatory sign of the +imaginary part, the latter being terminated by a ``j`` suffix. If the presentation +type is missing, the result will match the output of :func:`str` (complex numbers with +a non-zero real part are also surrounded by parentheses), possibly altered by +other format modifiers. + .. _formatexamples: diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index c6d78a356d97bc..37d5adf0fa9541 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -1,5 +1,5 @@ -:mod:`stringprep` --- Internet String Preparation -================================================= +:mod:`!stringprep` --- Internet String Preparation +================================================== .. module:: stringprep :synopsis: String preparation, as per RFC 3453 diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index e2e6fc542e3e67..3ea9e5ba071289 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -1,5 +1,5 @@ -:mod:`struct` --- Interpret bytes as packed binary data -======================================================= +:mod:`!struct` --- Interpret bytes as packed binary data +======================================================== .. testsetup:: * @@ -160,6 +160,21 @@ following table: If the first character is not one of these, ``'@'`` is assumed. +.. note:: + + The number 1023 (``0x3ff`` in hexadecimal) has the following byte representations: + + * ``03 ff`` in big-endian (``>``) + * ``ff 03`` in little-endian (``<``) + + Python example: + + >>> import struct + >>> struct.pack('>h', 1023) + b'\x03\xff' + >>> struct.pack('`, it is possible to use :func:`shlex.quote` for this escaping. +On Windows, batch files (:file:`*.bat` or :file:`*.cmd`) may be launched by the +operating system in a system shell regardless of the arguments passed to this +library. This could result in arguments being parsed according to shell rules, +but without any escaping added by Python. If you are intentionally launching a +batch file with arguments from untrusted sources, consider passing +``shell=True`` to allow Python to escape special characters. See :gh:`114539` +for additional discussion. + Popen Objects ------------- @@ -857,8 +868,8 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.terminate() - Stop the child. On POSIX OSs the method sends SIGTERM to the - child. On Windows the Win32 API function :c:func:`TerminateProcess` is called + Stop the child. On POSIX OSs the method sends :py:const:`~signal.SIGTERM` to the + child. On Windows the Win32 API function :c:func:`!TerminateProcess` is called to stop the child. @@ -1055,6 +1066,22 @@ The :mod:`subprocess` module exposes the following constants. Specifies that the :attr:`STARTUPINFO.wShowWindow` attribute contains additional information. +.. data:: STARTF_FORCEONFEEDBACK + + A :attr:`STARTUPINFO.dwFlags` parameter to specify that the + *Working in Background* mouse cursor will be displayed while a + process is launching. This is the default behavior for GUI + processes. + + .. versionadded:: 3.13 + +.. data:: STARTF_FORCEOFFFEEDBACK + + A :attr:`STARTUPINFO.dwFlags` parameter to specify that the mouse + cursor will not be changed when launching a process. + + .. versionadded:: 3.13 + .. data:: CREATE_NEW_CONSOLE The new process has a new console, instead of inheriting its parent's @@ -1099,7 +1126,7 @@ The :mod:`subprocess` module exposes the following constants. .. data:: NORMAL_PRIORITY_CLASS A :class:`Popen` ``creationflags`` parameter to specify that a new process - will have an normal priority. (default) + will have a normal priority. (default) .. versionadded:: 3.7 @@ -1416,36 +1443,8 @@ Environment example:: -Replacing :func:`os.popen`, :func:`os.popen2`, :func:`os.popen3` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdin, child_stdout) = (p.stdin, p.stdout) - -:: - - (child_stdin, - child_stdout, - child_stderr) = os.popen3(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) - (child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - -:: - - (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) - (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) +Replacing :func:`os.popen` +^^^^^^^^^^^^^^^^^^^^^^^^^^ Return code handling translates as follows:: @@ -1462,44 +1461,6 @@ Return code handling translates as follows:: print("There were some errors") -Replacing functions from the :mod:`!popen2` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. note:: - - If the cmd argument to popen2 functions is a string, the command is executed - through /bin/sh. If it is a list, the command is directly executed. - -:: - - (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) - ==> - p = Popen("somestring", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:: - - (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode) - ==> - p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:class:`popen2.Popen3` and :class:`popen2.Popen4` basically work as -:class:`subprocess.Popen`, except that: - -* :class:`Popen` raises an exception if the execution fails. - -* The *capturestderr* argument is replaced with the *stderr* argument. - -* ``stdin=PIPE`` and ``stdout=PIPE`` must be specified. - -* popen2 closes all file descriptors by default, but you have to specify - ``close_fds=True`` with :class:`Popen` to guarantee this behavior on - all platforms or past Python versions. - - Legacy Shell Invocation Functions --------------------------------- @@ -1600,36 +1561,22 @@ runtime): Module which provides function to parse and escape command lines. -.. _disable_vfork: .. _disable_posix_spawn: -Disabling use of ``vfork()`` or ``posix_spawn()`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Disable use of ``posix_spawn()`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ On Linux, :mod:`subprocess` defaults to using the ``vfork()`` system call internally when it is safe to do so rather than ``fork()``. This greatly improves performance. -If you ever encounter a presumed highly unusual situation where you need to -prevent ``vfork()`` from being used by Python, you can set the -:const:`subprocess._USE_VFORK` attribute to a false value. - -:: - - subprocess._USE_VFORK = False # See CPython issue gh-NNNNNN. - -Setting this has no impact on use of ``posix_spawn()`` which could use -``vfork()`` internally within its libc implementation. There is a similar -:const:`subprocess._USE_POSIX_SPAWN` attribute if you need to prevent use of -that. - :: subprocess._USE_POSIX_SPAWN = False # See CPython issue gh-NNNNNN. -It is safe to set these to false on any Python version. They will have no -effect on older versions when unsupported. Do not assume the attributes are -available to read. Despite their names, a true value does not indicate that the +It is safe to set this to false on any Python version. It will have no +effect on older or newer versions where unsupported. Do not assume the attribute +is available to read. Despite the name, a true value does not indicate the corresponding function will be used, only that it may be. Please file issues any time you have to use these private knobs with a way to @@ -1637,4 +1584,3 @@ reproduce the issue you were seeing. Link to that issue from a comment in your code. .. versionadded:: 3.8 ``_USE_POSIX_SPAWN`` -.. versionadded:: 3.11 ``_USE_VFORK`` diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 47568387f9a7ce..54e19af4bd69a6 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -1,5 +1,5 @@ -:mod:`symtable` --- Access to the compiler's symbol tables -========================================================== +:mod:`!symtable` --- Access to the compiler's symbol tables +=========================================================== .. module:: symtable :synopsis: Interface to the compiler's internal symbol tables. @@ -31,21 +31,74 @@ Generating Symbol Tables Examining Symbol Tables ----------------------- +.. class:: SymbolTableType + + An enumeration indicating the type of a :class:`SymbolTable` object. + + .. attribute:: MODULE + :value: "module" + + Used for the symbol table of a module. + + .. attribute:: FUNCTION + :value: "function" + + Used for the symbol table of a function. + + .. attribute:: CLASS + :value: "class" + + Used for the symbol table of a class. + + The following members refer to different flavors of + :ref:`annotation scopes `. + + .. attribute:: ANNOTATION + :value: "annotation" + + Used for annotations if ``from __future__ import annotations`` is active. + + .. attribute:: TYPE_ALIAS + :value: "type alias" + + Used for the symbol table of :keyword:`type` constructions. + + .. attribute:: TYPE_PARAMETERS + :value: "type parameters" + + Used for the symbol table of :ref:`generic functions ` + or :ref:`generic classes `. + + .. attribute:: TYPE_VARIABLE + :value: "type variable" + + Used for the symbol table of the bound, the constraint tuple or the + default value of a single type variable in the formal sense, i.e., + a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two do + not support a bound or a constraint tuple). + + .. versionadded:: 3.13 + .. class:: SymbolTable A namespace table for a block. The constructor is not public. .. method:: get_type() - Return the type of the symbol table. Possible values are ``'class'``, - ``'module'``, ``'function'``, ``'annotation'``, ``'TypeVar bound'``, - ``'type alias'``, and ``'type parameter'``. The latter four refer to - different flavors of :ref:`annotation scopes `. + Return the type of the symbol table. Possible values are members + of the :class:`SymbolTableType` enumeration. .. versionchanged:: 3.12 Added ``'annotation'``, ``'TypeVar bound'``, ``'type alias'``, and ``'type parameter'`` as possible return values. + .. versionchanged:: 3.13 + Return values are members of the :class:`SymbolTableType` enumeration. + + The exact values of the returned string may change in the future, + and thus, it is recommended to use :class:`SymbolTableType` members + instead of hard-coded strings. + .. method:: get_id() Return the table's identifier. @@ -114,11 +167,12 @@ Examining Symbol Tables .. method:: get_nonlocals() - Return a tuple containing names of nonlocals in this function. + Return a tuple containing names of explicitly declared nonlocals in this function. .. method:: get_frees() - Return a tuple containing names of free variables in this function. + Return a tuple containing names of :term:`free (closure) variables ` + in this function. .. class:: Class @@ -127,7 +181,54 @@ Examining Symbol Tables .. method:: get_methods() - Return a tuple containing the names of methods declared in the class. + Return a tuple containing the names of method-like functions declared + in the class. + + Here, the term 'method' designates *any* function defined in the class + body via :keyword:`def` or :keyword:`async def`. + + Functions defined in a deeper scope (e.g., in an inner class) are not + picked up by :meth:`get_methods`. + + For example: + + .. testsetup:: symtable.Class.get_methods + + import warnings + context = warnings.catch_warnings() + context.__enter__() + warnings.simplefilter("ignore", category=DeprecationWarning) + + .. testcleanup:: symtable.Class.get_methods + + context.__exit__() + + .. doctest:: symtable.Class.get_methods + + >>> import symtable + >>> st = symtable.symtable(''' + ... def outer(): pass + ... + ... class A: + ... def f(): + ... def w(): pass + ... + ... def g(self): pass + ... + ... @classmethod + ... async def h(cls): pass + ... + ... global outer + ... def outer(self): pass + ... ''', 'test', 'exec') + >>> class_A = st.get_children()[2] + >>> class_A.get_methods() + ('f', 'g', 'h') + + Although ``A().f()`` raises :exc:`TypeError` at runtime, ``A.f`` is still + considered as a method-like function. + + .. deprecated-removed:: 3.14 3.16 .. class:: Symbol @@ -151,6 +252,12 @@ Examining Symbol Tables Return ``True`` if the symbol is a parameter. + .. method:: is_type_parameter() + + Return ``True`` if the symbol is a type parameter. + + .. versionadded:: 3.14 + .. method:: is_global() Return ``True`` if the symbol is global. @@ -178,10 +285,42 @@ Examining Symbol Tables Return ``True`` if the symbol is referenced in its block, but not assigned to. + .. method:: is_free_class() + + Return *True* if a class-scoped symbol is free from + the perspective of a method. + + Consider the following example:: + + def f(): + x = 1 # function-scoped + class C: + x = 2 # class-scoped + def method(self): + return x + + In this example, the class-scoped symbol ``x`` is considered to + be free from the perspective of ``C.method``, thereby allowing + the latter to return *1* at runtime and not *2*. + + .. versionadded:: 3.14 + .. method:: is_assigned() Return ``True`` if the symbol is assigned to in its block. + .. method:: is_comp_iter() + + Return ``True`` if the symbol is a comprehension iteration variable. + + .. versionadded:: 3.14 + + .. method:: is_comp_cell() + + Return ``True`` if the symbol is a cell in an inlined comprehension. + + .. versionadded:: 3.14 + .. method:: is_namespace() Return ``True`` if name binding introduces new namespace. diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 4980227c60b21e..f7140af2494898 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -1,5 +1,5 @@ -:mod:`sys.monitoring` --- Execution event monitoring -==================================================== +:mod:`!sys.monitoring` --- Execution event monitoring +===================================================== .. module:: sys.monitoring :synopsis: Access and control event monitoring @@ -50,16 +50,14 @@ Registering and using tools *tool_id* must be in the range 0 to 5 inclusive. Raises a :exc:`ValueError` if *tool_id* is in use. -.. function:: free_tool_id(tool_id: int, /) -> None +.. function:: clear_tool_id(tool_id: int, /) -> None - Should be called once a tool no longer requires *tool_id*. + Unregister all events and callback functions associated with *tool_id*. -.. note:: +.. function:: free_tool_id(tool_id: int, /) -> None - :func:`free_tool_id` will not disable global or local events associated - with *tool_id*, nor will it unregister any callback functions. This - function is only intended to be used to notify the VM that the - particular *tool_id* is no longer in use. + Should be called once a tool no longer requires *tool_id*. + Will call :func:`clear_tool_id` before releasing *tool_id*. .. function:: get_tool(tool_id: int, /) -> str | None @@ -160,7 +158,7 @@ events, use the expression ``PY_RETURN | PY_START``. .. monitoring-event:: NO_EVENTS - An alias for ``0`` so users can do explict comparisions like:: + An alias for ``0`` so users can do explicit comparisons like:: if get_events(DEBUGGER_ID) == NO_EVENTS: ... @@ -226,6 +224,10 @@ To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. :monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. +Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` +event for a :exc:`StopIteration` exception are equivalent, and are treated as interchangeable +when generating events. Implementations will favor :monitoring-event:`STOP_ITERATION` for +performance reasons, but may generate a :monitoring-event:`RAISE` event with a :exc:`StopIteration`. Turning events on and off ------------------------- @@ -255,7 +257,10 @@ No events are active by default. Per code object events '''''''''''''''''''''' -Events can also be controlled on a per code object basis. +Events can also be controlled on a per code object basis. The functions +defined below which accept a :class:`types.CodeType` should be prepared +to accept a look-alike object from functions which are not defined +in Python (see :ref:`c-api-monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a97a369b77b88a..d83816ec1502ca 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1,5 +1,5 @@ -:mod:`sys` --- System-specific parameters and functions -======================================================= +:mod:`!sys` --- System-specific parameters and functions +======================================================== .. module:: sys :synopsis: Access system-specific parameters and functions. @@ -16,12 +16,12 @@ always available. On POSIX systems where Python was built with the standard ``configure`` script, this contains the ABI flags as specified by :pep:`3149`. + .. versionadded:: 3.2 + .. versionchanged:: 3.8 Default flags became an empty string (``m`` flag for pymalloc has been removed). - .. versionadded:: 3.2 - .. availability:: Unix. @@ -195,6 +195,17 @@ always available. This function should be used for internal and specialized purposes only. + .. deprecated:: 3.13 + Use the more general :func:`_clear_internal_caches` function instead. + + +.. function:: _clear_internal_caches() + + Clear all internal performance-related caches. Use this function *only* to + release unnecessary references and memory blocks when hunting for leaks. + + .. versionadded:: 3.13 + .. function:: _current_frames() @@ -724,11 +735,11 @@ always available. regardless of their size. This function is mainly useful for tracking and debugging memory leaks. Because of the interpreter's internal caches, the result can vary from call to call; you may have to call - :func:`_clear_type_cache()` and :func:`gc.collect()` to get more + :func:`_clear_internal_caches` and :func:`gc.collect` to get more predictable results. If a Python build or implementation cannot reasonably compute this - information, :func:`getallocatedblocks()` is allowed to return 0 instead. + information, :func:`getallocatedblocks` is allowed to return 0 instead. .. versionadded:: 3.4 @@ -742,7 +753,9 @@ always available. .. function:: getandroidapilevel() - Return the build time API version of Android as an integer. + Return the build-time API level of Android as an integer. This represents the + minimum version of Android this build of Python can run on. For runtime + version information, see :func:`platform.android_ver`. .. availability:: Android. @@ -864,7 +877,7 @@ always available. additional garbage collector overhead if the object is managed by the garbage collector. - See `recursive sizeof recipe `_ + See `recursive sizeof recipe `_ for an example of using :func:`getsizeof` recursively to find the size of containers and all their contents. @@ -907,6 +920,35 @@ always available. It is not guaranteed to exist in all implementations of Python. +.. function:: getobjects(limit[, type]) + + This function only exists if CPython was built using the + specialized configure option :option:`--with-trace-refs`. + It is intended only for debugging garbage-collection issues. + + Return a list of up to *limit* dynamically allocated Python objects. + If *type* is given, only objects of that exact type (not subtypes) + are included. + + Objects from the list are not safe to use. + Specifically, the result will include objects from all interpreters that + share their object allocator state (that is, ones created with + :c:member:`PyInterpreterConfig.use_main_obmalloc` set to 1 + or using :c:func:`Py_NewInterpreter`, and the + :ref:`main interpreter `). + Mixing objects from different interpreters may lead to crashes + or other unexpected behavior. + + .. impl-detail:: + + This function should be used for specialized purposes only. + It is not guaranteed to exist in all implementations of Python. + + .. versionchanged:: next + + The result may include objects from other interpreters. + + .. function:: getprofile() .. index:: @@ -1186,11 +1228,21 @@ always available. return value of :func:`intern` around to benefit from it. +.. function:: _is_gil_enabled() + + Return :const:`True` if the :term:`GIL` is enabled and :const:`False` if + it is disabled. + + .. versionadded:: 3.13 + + .. function:: is_finalizing() Return :const:`True` if the main Python interpreter is :term:`shutting down `. Return :const:`False` otherwise. + See also the :exc:`PythonFinalizationError` exception. + .. versionadded:: 3.5 .. data:: last_exc @@ -1251,7 +1303,8 @@ always available. that implement Python's default import semantics. The :meth:`~importlib.abc.MetaPathFinder.find_spec` method is called with at least the absolute name of the module being imported. If the module to be - imported is contained in a package, then the parent package's :attr:`__path__` + imported is contained in a package, then the parent package's + :attr:`~module.__path__` attribute is passed in as a second argument. The method returns a :term:`module spec`, or ``None`` if the module cannot be found. @@ -1354,47 +1407,42 @@ always available. .. data:: platform - This string contains a platform identifier that can be used to append - platform-specific components to :data:`sys.path`, for instance. - - For Unix systems, except on Linux and AIX, this is the lowercased OS name as - returned by ``uname -s`` with the first part of the version as returned by - ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time - when Python was built*. Unless you want to test for a specific system - version, it is therefore recommended to use the following idiom:: - - if sys.platform.startswith('freebsd'): - # FreeBSD-specific code here... - elif sys.platform.startswith('linux'): - # Linux-specific code here... - elif sys.platform.startswith('aix'): - # AIX-specific code here... - - For other systems, the values are: + A string containing a platform identifier. Known values are: ================ =========================== System ``platform`` value ================ =========================== AIX ``'aix'`` + Android ``'android'`` Emscripten ``'emscripten'`` + iOS ``'ios'`` Linux ``'linux'`` - WASI ``'wasi'`` + macOS ``'darwin'`` Windows ``'win32'`` Windows/Cygwin ``'cygwin'`` - macOS ``'darwin'`` + WASI ``'wasi'`` ================ =========================== + On Unix systems not listed in the table, the value is the lowercased OS name + as returned by ``uname -s``, with the first part of the version as returned by + ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time + when Python was built*. Unless you want to test for a specific system + version, it is therefore recommended to use the following idiom:: + + if sys.platform.startswith('freebsd'): + # FreeBSD-specific code here... + .. versionchanged:: 3.3 On Linux, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. .. versionchanged:: 3.8 On AIX, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. + + .. versionchanged:: 3.13 + On Android, :data:`sys.platform` now returns ``'android'`` rather than + ``'linux'``. .. seealso:: @@ -1690,11 +1738,11 @@ always available. contain a tuple of (filename, line number, function name) tuples describing the traceback where the coroutine object was created, with the most recent call first. When disabled, ``cr_origin`` will - be None. + be ``None``. To enable, pass a *depth* value greater than zero; this sets the number of frames whose information will be captured. To disable, - pass set *depth* to zero. + set *depth* to zero. This setting is thread-specific. @@ -1709,6 +1757,8 @@ always available. Activate the stack profiler trampoline *backend*. The only supported backend is ``"perf"``. + Stack trampolines cannot be activated if the JIT is active. + .. availability:: Linux. .. versionadded:: 3.12 diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 2faab212e46eff..3921908b7c7bfc 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -1,5 +1,5 @@ -:mod:`sysconfig` --- Provide access to Python's configuration information -========================================================================= +:mod:`!sysconfig` --- Provide access to Python's configuration information +========================================================================== .. module:: sysconfig :synopsis: Python's configuration information @@ -305,7 +305,7 @@ Installation path functions mix with those by the other. End users should not use this function, but :func:`get_default_scheme` and - :func:`get_preferred_scheme()` instead. + :func:`get_preferred_scheme` instead. .. versionadded:: 3.10 @@ -376,7 +376,7 @@ Other functions This is used mainly to distinguish platform-specific build directories and platform-specific built distributions. Typically includes the OS name and - version and the architecture (as supplied by 'os.uname()'), although the + version and the architecture (as supplied by :func:`os.uname`), although the exact information included depends on the OS; e.g., on Linux, the kernel version isn't particularly important. diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 7b27fc7e85b62d..548898a37bc6ea 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -1,5 +1,5 @@ -:mod:`syslog` --- Unix syslog library routines -============================================== +:mod:`!syslog` --- Unix syslog library routines +=============================================== .. module:: syslog :platform: Unix @@ -11,7 +11,7 @@ This module provides an interface to the Unix ``syslog`` library routines. Refer to the Unix manual pages for a detailed description of the ``syslog`` facility. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI, not iOS. This module wraps the system ``syslog`` family of routines. A pure Python library that can speak to a syslog server is available in the diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index dfe688a2f93e0c..4f61b3dd761400 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -1,5 +1,5 @@ -:mod:`tabnanny` --- Detection of ambiguous indentation -====================================================== +:mod:`!tabnanny` --- Detection of ambiguous indentation +======================================================= .. module:: tabnanny :synopsis: Tool for detecting white space related problems in Python diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 2134293a0bb0de..c9d69cf5094095 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1,5 +1,5 @@ -:mod:`tarfile` --- Read and write tar archive files -=================================================== +:mod:`!tarfile` --- Read and write tar archive files +==================================================== .. module:: tarfile :synopsis: Read and write tar-format archive files. @@ -40,9 +40,12 @@ Some facts and figures: Archives are extracted using a :ref:`filter `, which makes it possible to either limit surprising/dangerous features, or to acknowledge that they are expected and the archive is fully trusted. - By default, archives are fully trusted, but this default is deprecated - and slated to change in Python 3.14. +.. versionchanged:: 3.14 + Set the default extraction filter to :func:`data `, + which disallows some dangerous features such as links to absolute paths + or paths outside of the destination. Previously, the filter strategy + was equivalent to :func:`fully_trusted `. .. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) @@ -495,18 +498,18 @@ be finalized; only the internally used file object will be closed. See the The *filter* argument specifies how ``members`` are modified or rejected before extraction. See :ref:`tarfile-extraction-filter` for details. - It is recommended to set this explicitly depending on which *tar* features - you need to support. + It is recommended to set this explicitly only if specific *tar* features + are required, or as ``filter='data'`` to support Python versions with a less + secure default (3.13 and lower). .. warning:: Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of *path*, e.g. members - that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. - Set ``filter='data'`` to prevent the most dangerous security issues, - and read the :ref:`tarfile-extraction-filter` section for details. + Since Python 3.14, the default (:func:`data `) will prevent + the most dangerous security issues. + However, it will not prevent *all* unintended or insecure behavior. + Read the :ref:`tarfile-extraction-filter` section for details. .. versionchanged:: 3.5 Added the *numeric_owner* parameter. @@ -517,6 +520,9 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.12 Added the *filter* parameter. + .. versionchanged:: 3.14 + The *filter* parameter now defaults to ``'data'``. + .. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False, filter=None) @@ -536,10 +542,8 @@ be finalized; only the internally used file object will be closed. See the .. warning:: - See the warning for :meth:`extractall`. - - Set ``filter='data'`` to prevent the most dangerous security issues, - and read the :ref:`tarfile-extraction-filter` section for details. + Never extract archives from untrusted sources without prior inspection. + See the warning for :meth:`extractall` for details. .. versionchanged:: 3.2 Added the *set_attrs* parameter. @@ -565,6 +569,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.3 Return an :class:`io.BufferedReader` object. + .. versionchanged:: 3.13 + The returned :class:`io.BufferedReader` object has the :attr:`!mode` + attribute which is always equal to ``'rb'``. + .. attribute:: TarFile.errorlevel :type: int @@ -598,14 +606,8 @@ be finalized; only the internally used file object will be closed. See the String names are not allowed for this attribute, unlike the *filter* argument to :meth:`~TarFile.extract`. - If ``extraction_filter`` is ``None`` (the default), - calling an extraction method without a *filter* argument will raise a - ``DeprecationWarning``, - and fall back to the :func:`fully_trusted ` filter, - whose dangerous behavior matches previous versions of Python. - - In Python 3.14+, leaving ``extraction_filter=None`` will cause - extraction methods to use the :func:`data ` filter by default. + If ``extraction_filter`` is ``None`` (the default), extraction methods + will use the :func:`data ` filter by default. The attribute may be set on instances or overridden in subclasses. It also is possible to set it on the ``TarFile`` class itself to set a @@ -613,7 +615,15 @@ be finalized; only the internally used file object will be closed. See the it is best practice to only do so in top-level applications or :mod:`site configuration `. To set a global default this way, a filter function needs to be wrapped in - :func:`staticmethod()` to prevent injection of a ``self`` argument. + :func:`staticmethod` to prevent injection of a ``self`` argument. + + .. versionchanged:: 3.14 + + The default filter is set to :func:`data `, + which disallows some dangerous features such as links to absolute paths + or paths outside of the destination. + Previously, the default was equivalent to + :func:`fully_trusted `. .. method:: TarFile.add(name, arcname=None, recursive=True, *, filter=None) @@ -637,11 +647,15 @@ be finalized; only the internally used file object will be closed. See the .. method:: TarFile.addfile(tarinfo, fileobj=None) - Add the :class:`TarInfo` object *tarinfo* to the archive. If *fileobj* is given, - it should be a :term:`binary file`, and - ``tarinfo.size`` bytes are read from it and added to the archive. You can + Add the :class:`TarInfo` object *tarinfo* to the archive. If *tarinfo* represents + a non zero-size regular file, the *fileobj* argument should be a :term:`binary file`, + and ``tarinfo.size`` bytes are read from it and added to the archive. You can create :class:`TarInfo` objects directly, or by using :meth:`gettarinfo`. + .. versionchanged:: 3.13 + + *fileobj* must be given for non-zero-sized regular files. + .. method:: TarFile.gettarinfo(name=None, arcname=None, fileobj=None) @@ -961,6 +975,12 @@ In most cases, the full functionality is not needed. Therefore, *tarfile* supports extraction filters: a mechanism to limit functionality, and thus mitigate some of the security issues. +.. warning:: + + None of the available filters blocks *all* dangerous archive features. + Never extract archives from untrusted sources without prior inspection. + See also :ref:`tarfile-further-verification`. + .. seealso:: :pep:`706` @@ -984,12 +1004,13 @@ can be: * ``None`` (default): Use :attr:`TarFile.extraction_filter`. - If that is also ``None`` (the default), raise a ``DeprecationWarning``, - and fall back to the ``'fully_trusted'`` filter, whose dangerous behavior - matches previous versions of Python. + If that is also ``None`` (the default), the ``'data'`` filter will be used. + + .. versionchanged:: 3.14 - In Python 3.14, the ``'data'`` filter will become the default instead. - It's possible to switch earlier; see :attr:`TarFile.extraction_filter`. + The default filter is set to :func:`data `. + Previously, the default was equivalent to + :func:`fully_trusted `. * A callable which will be called for each extracted member with a :ref:`TarInfo ` describing the member and the destination @@ -1072,6 +1093,9 @@ reused in custom filters: Return the modified ``TarInfo`` member. + Note that this filter does not block *all* dangerous archive features. + See :ref:`tarfile-further-verification` for details. + .. _tarfile-extraction-refuse: @@ -1085,6 +1109,8 @@ With ``errorlevel=0`` the error will be logged and the member will be skipped, but extraction will continue. +.. _tarfile-further-verification: + Hints for further verification ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1102,9 +1128,10 @@ Here is an incomplete list of things to consider: disk, memory and CPU usage. * Check filenames against an allow-list of characters (to filter out control characters, confusables, foreign path separators, - etc.). + and so on). * Check that filenames have expected extensions (discouraging files that - execute when you “click on them”, or extension-less files like Windows special device names). + execute when you “click on them”, or extension-less files like Windows + special device names). * Limit the number of extracted files, total size of extracted data, filename length (including symlink length), and size of individual files. * Check for files that would be shadowed on case-insensitive filesystems. diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 9add8500c7788c..f0a81a093b435b 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -1,5 +1,5 @@ -:mod:`tempfile` --- Generate temporary files and directories -============================================================ +:mod:`!tempfile` --- Generate temporary files and directories +============================================================= .. module:: tempfile :synopsis: Generate temporary files and directories. diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index 57705ddc4e6470..0c6f3059fe71d1 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -1,5 +1,5 @@ -:mod:`termios` --- POSIX style tty control -========================================== +:mod:`!termios` --- POSIX style tty control +=========================================== .. module:: termios :platform: Unix diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 7d28f625345726..04d28aee0f8672 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1,5 +1,5 @@ -:mod:`test` --- Regression tests package for Python -=================================================== +:mod:`!test` --- Regression tests package for Python +==================================================== .. module:: test :synopsis: Regression tests package containing the testing suite for Python. @@ -324,7 +324,7 @@ The :mod:`test.support` module defines the following constants: .. data:: Py_DEBUG - True if Python was built with the :c:macro:`Py_DEBUG` macro + ``True`` if Python was built with the :c:macro:`Py_DEBUG` macro defined, that is, if Python was :ref:`built in debug mode `. @@ -731,6 +731,12 @@ The :mod:`test.support` module defines the following functions: macOS version is less than the minimum, the test is skipped. +.. decorator:: requires_gil_enabled + + Decorator for skipping tests on the free-threaded build. If the + :term:`GIL` is disabled, the test is skipped. + + .. decorator:: requires_IEEE_754 Decorator for skipping tests on non-IEEE 754 platforms. @@ -940,7 +946,7 @@ The :mod:`test.support` module defines the following functions: other modules, possibly a C backend (like ``csv`` and its ``_csv``). The *extra* argument can be a set of names that wouldn't otherwise be automatically - detected as "public", like objects without a proper ``__module__`` + detected as "public", like objects without a proper :attr:`~definition.__module__` attribute. If provided, it will be added to the automatically detected ones. The *not_exported* argument can be a set of names that must not be treated @@ -1695,7 +1701,7 @@ The :mod:`test.support.warnings_helper` module provides support for warnings tes .. function:: check_warnings(*filters, quiet=True) - A convenience wrapper for :func:`warnings.catch_warnings()` that makes it + A convenience wrapper for :func:`warnings.catch_warnings` that makes it easier to test that a warning was correctly raised. It is approximately equivalent to calling ``warnings.catch_warnings(record=True)`` with :meth:`warnings.simplefilter` set to ``always`` and with the option to diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index 7445410f91808c..a58b460fef409c 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -1,5 +1,5 @@ -:mod:`textwrap` --- Text wrapping and filling -============================================= +:mod:`!textwrap` --- Text wrapping and filling +============================================== .. module:: textwrap :synopsis: Text wrapping and filling @@ -154,7 +154,7 @@ hyphenated words; only then will long words be broken if necessary, unless wrapper = TextWrapper() wrapper.initial_indent = "* " - You can re-use the same :class:`TextWrapper` object many times, and you can + You can reuse the same :class:`TextWrapper` object many times, and you can change any of its options through direct assignment to instance attributes between uses. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 5fbf9379b8202c..d4b343db36efb3 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -1,5 +1,5 @@ -:mod:`threading` --- Thread-based parallelism -============================================= +:mod:`!threading` --- Thread-based parallelism +============================================== .. module:: threading :synopsis: Thread-based parallelism. @@ -363,12 +363,12 @@ since it is impossible to detect the termination of alien threads. base class constructor (``Thread.__init__()``) before doing anything else to the thread. + .. versionchanged:: 3.3 + Added the *daemon* parameter. + .. versionchanged:: 3.10 Use the *target* name if *name* argument is omitted. - .. versionchanged:: 3.3 - Added the *daemon* argument. - .. method:: start() Start the thread's activity. @@ -412,7 +412,7 @@ since it is impossible to detect the termination of alien threads. timeout occurs. When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). As :meth:`~Thread.join` always returns ``None``, you must call :meth:`~Thread.is_alive` after :meth:`~Thread.join` to decide whether a timeout happened -- if the thread is still alive, the @@ -567,6 +567,9 @@ All methods are executed atomically. Lock acquisition can now be interrupted by signals on POSIX if the underlying threading implementation supports it. + .. versionchanged:: 3.14 + Lock acquisition can now be interrupted by signals on Windows. + .. method:: release() @@ -598,14 +601,25 @@ and "recursion level" in addition to the locked/unlocked state used by primitive locks. In the locked state, some thread owns the lock; in the unlocked state, no thread owns it. -To lock the lock, a thread calls its :meth:`~RLock.acquire` method; this -returns once the thread owns the lock. To unlock the lock, a thread calls -its :meth:`~Lock.release` method. :meth:`~Lock.acquire`/:meth:`~Lock.release` -call pairs may be nested; only the final :meth:`~Lock.release` (the -:meth:`~Lock.release` of the outermost pair) resets the lock to unlocked and -allows another thread blocked in :meth:`~Lock.acquire` to proceed. +Threads call a lock's :meth:`~RLock.acquire` method to lock it, +and its :meth:`~Lock.release` method to unlock it. -Reentrant locks also support the :ref:`context management protocol `. +.. note:: + + Reentrant locks support the :ref:`context management protocol `, + so it is recommended to use :keyword:`with` instead of manually calling + :meth:`~RLock.acquire` and :meth:`~RLock.release` + to handle acquiring and releasing the lock for a block of code. + +RLock's :meth:`~RLock.acquire`/:meth:`~RLock.release` call pairs may be nested, +unlike Lock's :meth:`~Lock.acquire`/:meth:`~Lock.release`. Only the final +:meth:`~RLock.release` (the :meth:`~Lock.release` of the outermost pair) resets +the lock to an unlocked state and allows another thread blocked in +:meth:`~RLock.acquire` to proceed. + +:meth:`~RLock.acquire`/:meth:`~RLock.release` must be used in pairs: each acquire +must have a release in the thread that has acquired the lock. Failing to +call release as many times the lock has been acquired can lead to deadlock. .. class:: RLock() @@ -624,25 +638,41 @@ Reentrant locks also support the :ref:`context management protocol ` Acquire a lock, blocking or non-blocking. - When invoked without arguments: if this thread already owns the lock, increment - the recursion level by one, and return immediately. Otherwise, if another - thread owns the lock, block until the lock is unlocked. Once the lock is - unlocked (not owned by any thread), then grab ownership, set the recursion level - to one, and return. If more than one thread is blocked waiting until the lock - is unlocked, only one at a time will be able to grab ownership of the lock. - There is no return value in this case. + .. seealso:: - When invoked with the *blocking* argument set to ``True``, do the same thing as when - called without arguments, and return ``True``. + :ref:`Using RLock as a context manager ` + Recommended over manual :meth:`!acquire` and :meth:`release` calls + whenever practical. - When invoked with the *blocking* argument set to ``False``, do not block. If a call - without an argument would block, return ``False`` immediately; otherwise, do the - same thing as when called without arguments, and return ``True``. - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. Return ``True`` if the lock has - been acquired, ``False`` if the timeout has elapsed. + When invoked with the *blocking* argument set to ``True`` (the default): + + * If no thread owns the lock, acquire the lock and return immediately. + + * If another thread owns the lock, block until we are able to acquire + lock, or *timeout*, if set to a positive float value. + + * If the same thread owns the lock, acquire the lock again, and + return immediately. This is the difference between :class:`Lock` and + :class:`!RLock`; :class:`Lock` handles this case the same as the previous, + blocking until the lock can be acquired. + + When invoked with the *blocking* argument set to ``False``: + + * If no thread owns the lock, acquire the lock and return immediately. + + * If another thread owns the lock, return immediately. + + * If the same thread owns the lock, acquire the lock again and return + immediately. + + In all cases, if the thread was able to acquire the lock, return ``True``. + If the thread was unable to acquire the lock (i.e. if not blocking or + the timeout was reached) return ``False``. + + If called multiple times, failing to call :meth:`~RLock.release` as many times + may lead to deadlock. Consider using :class:`!RLock` as a context manager rather than + calling acquire/release directly. .. versionchanged:: 3.2 The *timeout* parameter is new. @@ -658,7 +688,7 @@ Reentrant locks also support the :ref:`context management protocol ` Only call this method when the calling thread owns the lock. A :exc:`RuntimeError` is raised if this method is called when the lock is - unlocked. + not acquired. There is no return value. @@ -767,7 +797,7 @@ item to the buffer only needs to wake up one consumer thread. occurs. Once awakened or timed out, it re-acquires the lock and returns. When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an :class:`RLock`, it is not released using @@ -987,18 +1017,15 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. .. method:: wait(timeout=None) - Block until the internal flag is true. If the internal flag is true on - entry, return immediately. Otherwise, block until another thread calls - :meth:`.set` to set the flag to true, or until the optional timeout occurs. + Block as long as the internal flag is false and the timeout, if given, + has not expired. The return value represents the + reason that this blocking method returned; ``True`` if returning because + the internal flag is set to true, or ``False`` if a timeout is given and + the internal flag did not become true within the given wait time. When the timeout argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). - - This method returns ``True`` if and only if the internal flag has been set to - true, either before the wait call or after the wait starts, so it will - always return ``True`` except if a timeout is given and the operation - times out. + floating-point number specifying a timeout for the operation in seconds, + or fractions thereof. .. versionchanged:: 3.1 Previously, the method always returned ``None``. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 2782a961363666..9cd5db768e9853 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -1,5 +1,5 @@ -:mod:`time` --- Time access and conversions -=========================================== +:mod:`!time` --- Time access and conversions +============================================ .. module:: time :synopsis: Time access and conversions. @@ -69,7 +69,7 @@ An explanation of some terminology and conventions is in order. systems, the clock "ticks" only 50 or 100 times a second. * On the other hand, the precision of :func:`.time` and :func:`sleep` is better - than their Unix equivalents: times are expressed as floating point numbers, + than their Unix equivalents: times are expressed as floating-point numbers, :func:`.time` returns the most accurate time available (using Unix :c:func:`!gettimeofday` where available), and :func:`sleep` will accept a time with a nonzero fraction (Unix :c:func:`!select` is used to implement this, where @@ -193,7 +193,7 @@ Functions Use :func:`clock_settime_ns` to avoid the precision loss caused by the :class:`float` type. - .. availability:: Unix. + .. availability:: Unix, not Android, not iOS. .. versionadded:: 3.3 @@ -202,7 +202,7 @@ Functions Similar to :func:`clock_settime` but set time with nanoseconds. - .. availability:: Unix. + .. availability:: Unix, not Android, not iOS. .. versionadded:: 3.7 @@ -273,7 +273,7 @@ Functions This is the inverse function of :func:`localtime`. Its argument is the :class:`struct_time` or full 9-tuple (since the dst flag is needed; use ``-1`` as the dst flag if it is unknown) which expresses the time in *local* time, not - UTC. It returns a floating point number, for compatibility with :func:`.time`. + UTC. It returns a floating-point number, for compatibility with :func:`.time`. If the input value cannot be represented as a valid time, either :exc:`OverflowError` or :exc:`ValueError` will be raised (which depends on whether the invalid value is caught by Python or the underlying C libraries). @@ -287,6 +287,15 @@ Functions The reference point of the returned value is undefined, so that only the difference between the results of two calls is valid. + Clock: + + * On Windows, call ``QueryPerformanceCounter()`` and + ``QueryPerformanceFrequency()``. + * On macOS, call ``mach_absolute_time()`` and ``mach_timebase_info()``. + * On HP-UX, call ``gethrtime()``. + * Call ``clock_gettime(CLOCK_HIGHRES)`` if available. + * Otherwise, call ``clock_gettime(CLOCK_MONOTONIC)``. + Use :func:`monotonic_ns` to avoid the precision loss caused by the :class:`float` type. @@ -316,6 +325,11 @@ Functions point of the returned value is undefined, so that only the difference between the results of two calls is valid. + .. impl-detail:: + + On CPython, use the same clock as :func:`time.monotonic` and is a + monotonic clock, i.e. a clock that cannot go backwards. + Use :func:`perf_counter_ns` to avoid the precision loss caused by the :class:`float` type. @@ -324,6 +338,10 @@ Functions .. versionchanged:: 3.10 On Windows, the function is now system-wide. + .. versionchanged:: 3.13 + Use the same clock as :func:`time.monotonic`. + + .. function:: perf_counter_ns() -> int Similar to :func:`perf_counter`, but return time as nanoseconds. @@ -358,7 +376,7 @@ Functions .. function:: sleep(secs) Suspend execution of the calling thread for the given number of seconds. - The argument may be a floating point number to indicate a more precise sleep + The argument may be a floating-point number to indicate a more precise sleep time. If the sleep is interrupted by a signal and no exception is raised by the @@ -383,15 +401,14 @@ Functions .. audit-event:: time.sleep secs - .. versionchanged:: 3.11 - On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now - used if available. On Windows, a waitable timer is now used. - .. versionchanged:: 3.5 The function now sleeps at least *secs* even if the sleep is interrupted by a signal, except if the signal handler raises an exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.11 + On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now + used if available. On Windows, a waitable timer is now used. .. versionchanged:: 3.13 Raises an auditing event. @@ -433,6 +450,10 @@ Functions | ``%d`` | Day of the month as a decimal number [01,31]. | | | | | | +-----------+------------------------------------------------+-------+ + | ``%f`` | Microseconds as a decimal number | \(1) | + | | [000000,999999]. | | + | | | | + +-----------+------------------------------------------------+-------+ | ``%H`` | Hour (24-hour clock) as a decimal number | | | | [00,23]. | | +-----------+------------------------------------------------+-------+ @@ -448,13 +469,13 @@ Functions | ``%M`` | Minute as a decimal number [00,59]. | | | | | | +-----------+------------------------------------------------+-------+ - | ``%p`` | Locale's equivalent of either AM or PM. | \(1) | + | ``%p`` | Locale's equivalent of either AM or PM. | \(2) | | | | | +-----------+------------------------------------------------+-------+ - | ``%S`` | Second as a decimal number [00,61]. | \(2) | + | ``%S`` | Second as a decimal number [00,61]. | \(3) | | | | | +-----------+------------------------------------------------+-------+ - | ``%U`` | Week number of the year (Sunday as the first | \(3) | + | ``%U`` | Week number of the year (Sunday as the first | \(4) | | | day of the week) as a decimal number [00,53]. | | | | All days in a new year preceding the first | | | | Sunday are considered to be in week 0. | | @@ -462,10 +483,13 @@ Functions | | | | | | | | +-----------+------------------------------------------------+-------+ + | ``%u`` | Day of the week (Monday is 1; Sunday is 7) | | + | | as a decimal number [1, 7]. | | + +-----------+------------------------------------------------+-------+ | ``%w`` | Weekday as a decimal number [0(Sunday),6]. | | | | | | +-----------+------------------------------------------------+-------+ - | ``%W`` | Week number of the year (Monday as the first | \(3) | + | ``%W`` | Week number of the year (Monday as the first | \(4) | | | day of the week) as a decimal number [00,53]. | | | | All days in a new year preceding the first | | | | Monday are considered to be in week 0. | | @@ -494,23 +518,39 @@ Functions | ``%Z`` | Time zone name (no characters if no time zone | | | | exists). Deprecated. [1]_ | | +-----------+------------------------------------------------+-------+ + | ``%G`` | ISO 8601 year (similar to ``%Y`` but follows | | + | | the rules for the ISO 8601 calendar year). | | + | | The year starts with the week that contains | | + | | the first Thursday of the calendar year. | | + +-----------+------------------------------------------------+-------+ + | ``%V`` | ISO 8601 week number (as a decimal number | | + | | [01,53]). The first week of the year is the | | + | | one that contains the first Thursday of the | | + | | year. Weeks start on Monday. | | + +-----------+------------------------------------------------+-------+ | ``%%`` | A literal ``'%'`` character. | | +-----------+------------------------------------------------+-------+ Notes: (1) + The ``%f`` format directive only applies to :func:`strptime`, + not to :func:`strftime`. However, see also :meth:`datetime.datetime.strptime` and + :meth:`datetime.datetime.strftime` where the ``%f`` format directive + :ref:`applies to microseconds `. + + (2) When used with the :func:`strptime` function, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. .. _leap-second: - (2) + (3) The range really is ``0`` to ``61``; value ``60`` is valid in timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. - (3) + (4) When used with the :func:`strptime` function, ``%U`` and ``%W`` are only used in calculations when the day of the week and the year are specified. @@ -590,7 +630,7 @@ Functions - range [1, 12] * - 2 - - .. attribute:: tm_day + - .. attribute:: tm_mday - range [1, 31] * - 3 @@ -638,13 +678,13 @@ Functions .. function:: time() -> float - Return the time in seconds since the epoch_ as a floating point + Return the time in seconds since the epoch_ as a floating-point number. The handling of `leap seconds`_ is platform dependent. On Windows and most Unix systems, the leap seconds are not counted towards the time in seconds since the epoch_. This is commonly referred to as `Unix time `_. - Note that even though the time is always returned as a floating point + Note that even though the time is always returned as a floating-point number, not all systems provide time with a better precision than 1 second. While this function normally returns non-decreasing values, it can return a lower value than a previous call if the system clock has been set back @@ -657,6 +697,12 @@ Functions :class:`struct_time` object is returned, from which the components of the calendar date may be accessed as attributes. + Clock: + + * On Windows, call ``GetSystemTimeAsFileTime()``. + * Call ``clock_gettime(CLOCK_REALTIME)`` if available. + * Otherwise, call ``gettimeofday()``. + Use :func:`time_ns` to avoid the precision loss caused by the :class:`float` type. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 616f8365b80f6c..548a3ee0540506 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -1,5 +1,5 @@ -:mod:`timeit` --- Measure execution time of small code snippets -=============================================================== +:mod:`!timeit` --- Measure execution time of small code snippets +================================================================ .. module:: timeit :synopsis: Measure the execution time of small code snippets. diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst index 6e8479c1dea1e2..df2b324fd5d3a7 100644 --- a/Doc/library/tkinter.colorchooser.rst +++ b/Doc/library/tkinter.colorchooser.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.colorchooser` --- Color choosing dialog -===================================================== +:mod:`!tkinter.colorchooser` --- Color choosing dialog +====================================================== .. module:: tkinter.colorchooser :platform: Tk diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst index 02de0fd331958d..62298d96c26459 100644 --- a/Doc/library/tkinter.dnd.rst +++ b/Doc/library/tkinter.dnd.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.dnd` --- Drag and drop support -============================================ +:mod:`!tkinter.dnd` --- Drag and drop support +============================================= .. module:: tkinter.dnd :platform: Tk @@ -25,8 +25,8 @@ Selection of a target object occurs as follows: #. Top-down search of area under mouse for target widget * Target widget should have a callable *dnd_accept* attribute - * If *dnd_accept* is not present or returns None, search moves to parent widget - * If no target widget is found, then the target object is None + * If *dnd_accept* is not present or returns ``None``, search moves to parent widget + * If no target widget is found, then the target object is ``None`` 2. Call to *.dnd_leave(source, event)* #. Call to *.dnd_enter(source, event)* diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index c7c2b7b566cf8f..ed01bd5f483943 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.font` --- Tkinter font wrapper -============================================ +:mod:`!tkinter.font` --- Tkinter font wrapper +============================================= .. module:: tkinter.font :platform: Tk diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 56090a0a0e424b..0dc9632ca73304 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.messagebox` --- Tkinter message prompts -===================================================== +:mod:`!tkinter.messagebox` --- Tkinter message prompts +====================================================== .. module:: tkinter.messagebox :platform: Tk diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index e084d8554c7c09..f284988daf2d4e 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -1,5 +1,5 @@ -:mod:`tkinter` --- Python interface to Tcl/Tk -============================================= +:mod:`!tkinter` --- Python interface to Tcl/Tk +============================================== .. module:: tkinter :synopsis: Interface to Tcl/Tk for graphical user interfaces @@ -58,7 +58,7 @@ details that are unchanged. * `Modern Tkinter for Busy Python Developers `_ By Mark Roseman. (ISBN 978-1999149567) - * `Python GUI programming with Tkinter `_ + * `Python GUI programming with Tkinter `_ By Alan D. Moore. (ISBN 978-1788835886) * `Programming Python `_ @@ -979,6 +979,15 @@ of :class:`tkinter.Image`: Either type of image is created through either the ``file`` or the ``data`` option (other options are available as well). +.. versionchanged:: 3.13 + Added the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region + from one image to other image, possibly with pixel zooming and/or + subsampling. + Add *from_coords* parameter to :class:`!PhotoImage` methods :meth:`!copy`, + :meth:`!zoom` and :meth:`!subsample`. + Add *zoom* and *subsample* parameters to :class:`!PhotoImage` method + :meth:`!copy`. + The image object can then be used wherever an ``image`` option is supported by some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index d20365baa38690..763e24929d74b5 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.scrolledtext` --- Scrolled Text Widget -==================================================== +:mod:`!tkinter.scrolledtext` --- Scrolled Text Widget +===================================================== .. module:: tkinter.scrolledtext :platform: Tk diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index bd0d8b3799a0f1..628e9f945ac365 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.ttk` --- Tk themed widgets -======================================== +:mod:`!tkinter.ttk` --- Tk themed widgets +========================================= .. module:: tkinter.ttk :synopsis: Tk themed widget set diff --git a/Doc/library/token.rst b/Doc/library/token.rst index e6dc37d7ad852c..0cc9dddd91ed6b 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -1,5 +1,5 @@ -:mod:`token` --- Constants used with Python parse trees -======================================================= +:mod:`!token` --- Constants used with Python parse trees +======================================================== .. module:: token :synopsis: Constants representing terminal nodes of the parse tree. @@ -75,7 +75,7 @@ the :mod:`tokenize` module. :noindex: Token value indicating that a type comment was recognized. Such - tokens are only produced when :func:`ast.parse()` is invoked with + tokens are only produced when :func:`ast.parse` is invoked with ``type_comments=True``. diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 92bdb052267a68..f719319a302a23 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -1,5 +1,5 @@ -:mod:`tokenize` --- Tokenizer for Python source -=============================================== +:mod:`!tokenize` --- Tokenizer for Python source +================================================ .. module:: tokenize :synopsis: Lexical scanner for Python source code. diff --git a/Doc/library/tomllib.rst b/Doc/library/tomllib.rst index f9e2dfeb13dc87..521a7a17fb3e8b 100644 --- a/Doc/library/tomllib.rst +++ b/Doc/library/tomllib.rst @@ -1,5 +1,5 @@ -:mod:`tomllib` --- Parse TOML files -=================================== +:mod:`!tomllib` --- Parse TOML files +==================================== .. module:: tomllib :synopsis: Parse TOML files. @@ -13,20 +13,20 @@ -------------- -This module provides an interface for parsing TOML (Tom's Obvious Minimal +This module provides an interface for parsing TOML 1.0.0 (Tom's Obvious Minimal Language, `https://toml.io `_). This module does not support writing TOML. .. seealso:: - The `Tomli-W package `__ + The :pypi:`Tomli-W package ` is a TOML writer that can be used in conjunction with this module, providing a write API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. .. seealso:: - The `TOML Kit package `__ + The :pypi:`TOML Kit package ` is a style-preserving TOML library with both read and write capability. It is a recommended replacement for this module for editing already existing TOML files. diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 8854905e192b45..cae94ea08e17e5 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -1,5 +1,5 @@ -:mod:`trace` --- Trace or track Python statement execution -========================================================== +:mod:`!trace` --- Trace or track Python statement execution +=========================================================== .. module:: trace :synopsis: Trace or track Python statement execution. diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index ab83e0df10b709..100a92b73d5497 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -1,5 +1,5 @@ -:mod:`traceback` --- Print or retrieve a stack traceback -======================================================== +:mod:`!traceback` --- Print or retrieve a stack traceback +========================================================= .. module:: traceback :synopsis: Print or retrieve a stack traceback. @@ -8,11 +8,15 @@ -------------- -This module provides a standard interface to extract, format and print stack -traces of Python programs. It exactly mimics the behavior of the Python -interpreter when it prints a stack trace. This is useful when you want to print -stack traces under program control, such as in a "wrapper" around the -interpreter. +This module provides a standard interface to extract, format and print +stack traces of Python programs. It is more flexible than the +interpreter's default traceback display, and therefore makes it +possible to configure certain aspects of the output. Finally, +it contains a utility for capturing enough information about an +exception to print it later, without the need to save a reference +to the actual exception. Since exceptions can be the roots of large +objects graph, this utility can significantly improve +memory management. .. index:: pair: object; traceback @@ -29,7 +33,20 @@ which are assigned to the :attr:`~BaseException.__traceback__` field of Module :mod:`pdb` Interactive source code debugger for Python programs. -The module defines the following functions: +The module's API can be divided into two parts: + +* Module-level functions offering basic functionality, which are useful for interactive + inspection of exceptions and tracebacks. + +* :class:`TracebackException` class and its helper classes + :class:`StackSummary` and :class:`FrameSummary`. These offer both more + flexibility in the output generated and the ability to store the information + necessary for later formatting without holding references to actual exception + and traceback objects. + + +Module-Level Functions +---------------------- .. function:: print_tb(tb, limit=None, file=None) @@ -42,6 +59,14 @@ The module defines the following functions: :term:`file ` or :term:`file-like object` to receive the output. + .. note:: + + The meaning of the *limit* parameter is different than the meaning + of :const:`sys.tracebacklimit`. A negative *limit* value corresponds to + a positive value of :const:`!sys.tracebacklimit`, whereas the behaviour of + a positive *limit* value cannot be achieved with + :const:`!sys.tracebacklimit`. + .. versionchanged:: 3.5 Added negative *limit* support. @@ -229,7 +254,6 @@ The module defines the following functions: .. versionadded:: 3.5 -The module also defines the following classes: :class:`!TracebackException` Objects ------------------------------------ @@ -237,12 +261,17 @@ The module also defines the following classes: .. versionadded:: 3.5 :class:`!TracebackException` objects are created from actual exceptions to -capture data for later printing in a lightweight fashion. +capture data for later printing. They offer a more lightweight method of +storing this information by avoiding holding references to +:ref:`traceback` and :ref:`frame` objects +In addition, they expose more options to configure the output compared to +the module-level functions described above. .. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) - Capture an exception for later rendering. *limit*, *lookup_lines* and - *capture_locals* are as for the :class:`StackSummary` class. + Capture an exception for later rendering. The meaning of *limit*, + *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` + class. If *compact* is true, only data that is required by :class:`!TracebackException`'s :meth:`format` method @@ -473,7 +502,7 @@ in a :ref:`traceback `. attribute accessed (which also happens when casting it to a :class:`tuple`). :attr:`~FrameSummary.line` may be directly provided, and will prevent line lookups happening at all. *locals* is an optional local variable - dictionary, and if supplied the variable representations are stored in the + mapping, and if supplied the variable representations are stored in the summary for later display. :class:`!FrameSummary` instances have the following attributes: @@ -501,8 +530,8 @@ in a :ref:`traceback `. .. _traceback-example: -Traceback Examples ------------------- +Examples of Using the Module-Level Functions +-------------------------------------------- This simple example implements a basic read-eval-print loop, similar to (but less useful than) the standard Python interactive interpreter loop. For a more @@ -541,8 +570,7 @@ exception and traceback: try: lumberjack() - except IndexError: - exc = sys.exception() + except IndexError as exc: print("*** print_tb:") traceback.print_tb(exc.__traceback__, limit=1, file=sys.stdout) print("*** print_exception:") @@ -645,5 +673,88 @@ This last example demonstrates the final few formatting functions: [' File "spam.py", line 3, in \n spam.eggs()\n', ' File "eggs.py", line 42, in eggs\n return "bacon"\n'] >>> an_error = IndexError('tuple index out of range') - >>> traceback.format_exception_only(type(an_error), an_error) + >>> traceback.format_exception_only(an_error) ['IndexError: tuple index out of range\n'] + + +Examples of Using :class:`TracebackException` +--------------------------------------------- + +With the helper class, we have more options:: + + >>> import sys + >>> from traceback import TracebackException + >>> + >>> def lumberjack(): + ... bright_side_of_life() + ... + >>> def bright_side_of_life(): + ... t = "bright", "side", "of", "life" + ... return t[5] + ... + >>> try: + ... lumberjack() + ... except IndexError as e: + ... exc = e + ... + >>> try: + ... try: + ... lumberjack() + ... except: + ... 1/0 + ... except Exception as e: + ... chained_exc = e + ... + >>> # limit works as with the module-level functions + >>> TracebackException.from_exception(exc, limit=-2).print() + Traceback (most recent call last): + File "", line 6, in lumberjack + bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ + File "", line 10, in bright_side_of_life + return t[5] + ~^^^ + IndexError: tuple index out of range + + >>> # capture_locals adds local variables in frames + >>> TracebackException.from_exception(exc, limit=-2, capture_locals=True).print() + Traceback (most recent call last): + File "", line 6, in lumberjack + bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ + File "", line 10, in bright_side_of_life + return t[5] + ~^^^ + t = ("bright", "side", "of", "life") + IndexError: tuple index out of range + + >>> # The *chain* kwarg to print() controls whether chained + >>> # exceptions are displayed + >>> TracebackException.from_exception(chained_exc).print() + Traceback (most recent call last): + File "", line 4, in + lumberjack() + ~~~~~~~~~~^^ + File "", line 7, in lumberjack + bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ + File "", line 11, in bright_side_of_life + return t[5] + ~^^^ + IndexError: tuple index out of range + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "", line 6, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> TracebackException.from_exception(chained_exc).print(chain=False) + Traceback (most recent call last): + File "", line 6, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 68432aeaecbcc1..2370d927292eb0 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -1,5 +1,5 @@ -:mod:`tracemalloc` --- Trace memory allocations -=============================================== +:mod:`!tracemalloc` --- Trace memory allocations +================================================ .. module:: tracemalloc :synopsis: Trace memory allocations. diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index ed63561c40de24..37778bf20bdcc7 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -1,5 +1,5 @@ -:mod:`tty` --- Terminal control functions -========================================= +:mod:`!tty` --- Terminal control functions +========================================== .. module:: tty :platform: Unix @@ -53,7 +53,7 @@ The :mod:`tty` module defines the following functions: is saved before setting *fd* to raw mode; this value is returned. .. versionchanged:: 3.12 - The return value is now the original tty attributes, instead of None. + The return value is now the original tty attributes, instead of ``None``. .. function:: setcbreak(fd, when=termios.TCSAFLUSH) @@ -67,7 +67,7 @@ The :mod:`tty` module defines the following functions: the minimum input to 1 byte with no delay. .. versionchanged:: 3.12 - The return value is now the original tty attributes, instead of None. + The return value is now the original tty attributes, instead of ``None``. .. versionchanged:: 3.12.2 The ``ICRNL`` flag is no longer cleared. This restores the behavior diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 88b1f09eb3c8b7..8eb4f8271fcfae 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -14,6 +14,11 @@ from turtle import * turtle = Turtle() +.. testcleanup:: + + import os + os.remove("my_drawing.ps") + -------------- Introduction @@ -120,7 +125,7 @@ off-screen):: home() The home position is at the center of the turtle's screen. If you ever need to -know them, get the turtle's x-y co-ordinates with:: +know them, get the turtle's x-y coordinates with:: pos() @@ -427,6 +432,7 @@ Input methods Methods specific to Screen | :func:`bye` | :func:`exitonclick` + | :func:`save` | :func:`setup` | :func:`title` @@ -606,7 +612,7 @@ Turtle motion >>> turtle.pos() (20.00,30.00) - .. versionadded: 3.12 + .. versionadded:: 3.12 .. function:: setx(x) @@ -2269,6 +2275,24 @@ Methods specific to Screen, not inherited from TurtleScreen client script. +.. function:: save(filename, overwrite=False) + + Save the current turtle drawing (and turtles) as a PostScript file. + + :param filename: the path of the saved PostScript file + :param overwrite: if ``False`` and there already exists a file with the given + filename, then the function will raise a + ``FileExistsError``. If it is ``True``, the file will be + overwritten. + + .. doctest:: + :skipif: _tkinter is None + + >>> screen.save("my_drawing.ps") + >>> screen.save("my_drawing.ps", overwrite=True) + + .. versionadded:: 3.14 + .. function:: setup(width=_CFG["width"], height=_CFG["height"], startx=_CFG["leftright"], starty=_CFG["topbottom"]) Set the size and position of the main window. Default values of arguments @@ -2754,9 +2778,6 @@ Changes since Python 3.0 :func:`Screen.numinput `. These pop up input dialogs and return strings and numbers respectively. -- Two example scripts :file:`tdemo_nim.py` and :file:`tdemo_round_dance.py` - have been added to the :file:`Lib/turtledemo` directory. - .. doctest:: :skipif: _tkinter is None diff --git a/Doc/library/types.rst b/Doc/library/types.rst index c8c981024c1aeb..439e119461f798 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -1,5 +1,5 @@ -:mod:`types` --- Dynamic type creation and names for built-in types -=================================================================== +:mod:`!types` --- Dynamic type creation and names for built-in types +==================================================================== .. module:: types :synopsis: Names for built-in types. @@ -91,8 +91,8 @@ Dynamic Type Creation For classes that have an ``__orig_bases__`` attribute, this function returns the value of ``cls.__orig_bases__``. - For classes without the ``__orig_bases__`` attribute, ``cls.__bases__`` is - returned. + For classes without the ``__orig_bases__`` attribute, + :attr:`cls.__bases__ ` is returned. Examples:: @@ -188,7 +188,7 @@ Standard names are defined for the following types: .. index:: pair: built-in function; compile - The type for code objects such as returned by :func:`compile`. + The type of :ref:`code objects ` such as returned by :func:`compile`. .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags types.CodeType @@ -196,18 +196,10 @@ Standard names are defined for the following types: required by the initializer. The audit event only occurs for direct instantiation of code objects, and is not raised for normal compilation. - .. method:: CodeType.replace(**kwargs) - - Return a copy of the code object with new values for the specified fields. - - Code objects are also supported by generic function :func:`copy.replace`. - - .. versionadded:: 3.8 - .. data:: CellType The type for cell objects: such objects are used as containers for - a function's free variables. + a function's :term:`closure variables `. .. versionadded:: 3.8 @@ -268,63 +260,18 @@ Standard names are defined for the following types: The type of :term:`modules `. The constructor takes the name of the module to be created and optionally its :term:`docstring`. - .. note:: - Use :func:`importlib.util.module_from_spec` to create a new module if you - wish to set the various import-controlled attributes. - - .. attribute:: __doc__ - - The :term:`docstring` of the module. Defaults to ``None``. - - .. attribute:: __loader__ - - The :term:`loader` which loaded the module. Defaults to ``None``. - - This attribute is to match :attr:`importlib.machinery.ModuleSpec.loader` - as stored in the :attr:`__spec__` object. - - .. note:: - A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferably read from the - :attr:`__spec__` attribute instead or use - ``getattr(module, "__loader__", None)`` if you explicitly need to use - this attribute. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __name__ - - The name of the module. Expected to match - :attr:`importlib.machinery.ModuleSpec.name`. - - .. attribute:: __package__ - - Which :term:`package` a module belongs to. If the module is top-level - (i.e. not a part of any specific package) then the attribute should be set - to ``''``, else it should be set to the name of the package (which can be - :attr:`__name__` if the module is a package itself). Defaults to ``None``. - - This attribute is to match :attr:`importlib.machinery.ModuleSpec.parent` - as stored in the :attr:`__spec__` object. - - .. note:: - A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferably read from the - :attr:`__spec__` attribute instead or use - ``getattr(module, "__package__", None)`` if you explicitly need to use - this attribute. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __spec__ - - A record of the module's import-system-related state. Expected to be an - instance of :class:`importlib.machinery.ModuleSpec`. + .. seealso:: - .. versionadded:: 3.4 + :ref:`Documentation on module objects ` + Provides details on the special attributes that can be found on + instances of :class:`!ModuleType`. + :func:`importlib.util.module_from_spec` + Modules created using the :class:`!ModuleType` constructor are + created with many of their special attributes unset or set to default + values. :func:`!module_from_spec` provides a more robust way of + creating :class:`!ModuleType` instances which ensures the various + attributes are set appropriately. .. data:: EllipsisType @@ -400,7 +347,7 @@ Standard names are defined for the following types: In addition, when a class is defined with a :attr:`~object.__slots__` attribute, then for each slot, an instance of :class:`!MemberDescriptorType` will be added as an attribute - on the class. This allows the slot to appear in the class's :attr:`~object.__dict__`. + on the class. This allows the slot to appear in the class's :attr:`~type.__dict__`. .. impl-detail:: @@ -489,14 +436,25 @@ Additional Utility Classes and Functions A simple :class:`object` subclass that provides attribute access to its namespace, as well as a meaningful repr. - Unlike :class:`object`, with ``SimpleNamespace`` you can add and remove - attributes. If a ``SimpleNamespace`` object is initialized with keyword - arguments, those are directly added to the underlying namespace. + Unlike :class:`object`, with :class:`!SimpleNamespace` you can add and remove + attributes. + + :py:class:`SimpleNamespace` objects may be initialized + in the same way as :class:`dict`: either with keyword arguments, + with a single positional argument, or with both. + When initialized with keyword arguments, + those are directly added to the underlying namespace. + Alternatively, when initialized with a positional argument, + the underlying namespace will be updated with key-value pairs + from that argument (either a mapping object or + an :term:`iterable` object producing key-value pairs). + All such keys must be strings. The type is roughly equivalent to the following code:: class SimpleNamespace: - def __init__(self, /, **kwargs): + def __init__(self, mapping_or_iterable=(), /, **kwargs): + self.__dict__.update(mapping_or_iterable) self.__dict__.update(kwargs) def __repr__(self): @@ -520,6 +478,9 @@ Additional Utility Classes and Functions Attribute order in the repr changed from alphabetical to insertion (like ``dict``). + .. versionchanged:: 3.13 + Added support for an optional positional argument. + .. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) Route attribute access on a class to __getattr__. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 63bd62d1f6679b..cd8b90854b0e94 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -23,27 +23,26 @@ -------------- -This module provides runtime support for type hints. For the original -specification of the typing system, see :pep:`484`. For a simplified -introduction to type hints, see :pep:`483`. +This module provides runtime support for type hints. +Consider the function below:: -The function below takes and returns a string and is annotated as follows:: + def surface_area_of_cube(edge_length: float) -> str: + return f"The surface area of the cube is {6 * edge_length ** 2}." - def greeting(name: str) -> str: - return 'Hello ' + name +The function ``surface_area_of_cube`` takes an argument expected to +be an instance of :class:`float`, as indicated by the :term:`type hint` +``edge_length: float``. The function is expected to return an instance +of :class:`str`, as indicated by the ``-> str`` hint. -In the function ``greeting``, the argument ``name`` is expected to be of type -:class:`str` and the return type :class:`str`. Subtypes are accepted as -arguments. +While type hints can be simple classes like :class:`float` or :class:`str`, +they can also be more complex. The :mod:`typing` module provides a vocabulary of +more advanced type hints. New features are frequently added to the ``typing`` module. -The `typing_extensions `_ package +The :pypi:`typing_extensions` package provides backports of these new features to older versions of Python. -For a summary of deprecated features and a deprecation timeline, please see -`Deprecation Timeline of Major Features`_. - .. seealso:: `"Typing cheat sheet" `_ @@ -61,67 +60,11 @@ For a summary of deprecated features and a deprecation timeline, please see .. _relevant-peps: -Relevant PEPs -============= - -Since the initial introduction of type hints in :pep:`484` and :pep:`483`, a -number of PEPs have modified and enhanced Python's framework for type -annotations: - -.. raw:: html - -
- The full list of PEPs - -* :pep:`526`: Syntax for Variable Annotations - *Introducing* syntax for annotating variables outside of function - definitions, and :data:`ClassVar` -* :pep:`544`: Protocols: Structural subtyping (static duck typing) - *Introducing* :class:`Protocol` and the - :func:`@runtime_checkable` decorator -* :pep:`585`: Type Hinting Generics In Standard Collections - *Introducing* :class:`types.GenericAlias` and the ability to use standard - library classes as :ref:`generic types` -* :pep:`586`: Literal Types - *Introducing* :data:`Literal` -* :pep:`589`: TypedDict: Type Hints for Dictionaries with a Fixed Set of Keys - *Introducing* :class:`TypedDict` -* :pep:`591`: Adding a final qualifier to typing - *Introducing* :data:`Final` and the :func:`@final` decorator -* :pep:`593`: Flexible function and variable annotations - *Introducing* :data:`Annotated` -* :pep:`604`: Allow writing union types as ``X | Y`` - *Introducing* :data:`types.UnionType` and the ability to use - the binary-or operator ``|`` to signify a - :ref:`union of types` -* :pep:`612`: Parameter Specification Variables - *Introducing* :class:`ParamSpec` and :data:`Concatenate` -* :pep:`613`: Explicit Type Aliases - *Introducing* :data:`TypeAlias` -* :pep:`646`: Variadic Generics - *Introducing* :data:`TypeVarTuple` -* :pep:`647`: User-Defined Type Guards - *Introducing* :data:`TypeGuard` -* :pep:`655`: Marking individual TypedDict items as required or potentially missing - *Introducing* :data:`Required` and :data:`NotRequired` -* :pep:`673`: Self type - *Introducing* :data:`Self` -* :pep:`675`: Arbitrary Literal String Type - *Introducing* :data:`LiteralString` -* :pep:`681`: Data Class Transforms - *Introducing* the :func:`@dataclass_transform` decorator -* :pep:`692`: Using ``TypedDict`` for more precise ``**kwargs`` typing - *Introducing* a new way of typing ``**kwargs`` with :data:`Unpack` and - :data:`TypedDict` -* :pep:`695`: Type Parameter Syntax - *Introducing* builtin syntax for creating generic functions, classes, and type aliases. -* :pep:`698`: Adding an override decorator to typing - *Introducing* the :func:`@override` decorator - -.. raw:: html - -
-
+Specification for the Python Type System +======================================== + +The canonical, up-to-date specification of the Python type system can be +found at `"Specification for the Python type system" `_. .. _type-aliases: @@ -155,8 +98,9 @@ Type aliases are useful for simplifying complex type signatures. For example:: # The static type checker will treat the previous type signature as # being exactly equivalent to this one. def broadcast_message( - message: str, - servers: Sequence[tuple[tuple[str, int], dict[str, str]]]) -> None: + message: str, + servers: Sequence[tuple[tuple[str, int], dict[str, str]]] + ) -> None: ... The :keyword:`type` statement is new in Python 3.12. For backwards @@ -264,7 +208,7 @@ Annotating callable objects =========================== Functions -- or other :term:`callable` objects -- can be annotated using -:class:`collections.abc.Callable` or :data:`typing.Callable`. +:class:`collections.abc.Callable` or deprecated :data:`typing.Callable`. ``Callable[[int], str]`` signifies a function that takes a single parameter of type :class:`int` and returns a :class:`str`. @@ -457,7 +401,7 @@ The type of class objects ========================= A variable annotated with ``C`` may accept a value of type ``C``. In -contrast, a variable annotated with ``type[C]`` (or +contrast, a variable annotated with ``type[C]`` (or deprecated :class:`typing.Type[C] `) may accept values that are classes themselves -- specifically, it will accept the *class object* of ``C``. For example:: @@ -497,6 +441,87 @@ For example:: ``type[Any]`` is equivalent to :class:`type`, which is the root of Python's :ref:`metaclass hierarchy `. + +.. _annotating-generators-and-coroutines: + +Annotating generators and coroutines +==================================== + +A generator can be annotated using the generic type +:class:`Generator[YieldType, SendType, ReturnType] `. +For example:: + + def echo_round() -> Generator[int, float, str]: + sent = yield 0 + while sent >= 0: + sent = yield round(sent) + return 'Done' + +Note that unlike many other generic classes in the standard library, +the ``SendType`` of :class:`~collections.abc.Generator` behaves +contravariantly, not covariantly or invariantly. + +The ``SendType`` and ``ReturnType`` parameters default to :const:`!None`:: + + def infinite_stream(start: int) -> Generator[int]: + while True: + yield start + start += 1 + +It is also possible to set these types explicitly:: + + def infinite_stream(start: int) -> Generator[int, None, None]: + while True: + yield start + start += 1 + +Simple generators that only ever yield values can also be annotated +as having a return type of either +:class:`Iterable[YieldType] ` +or :class:`Iterator[YieldType] `:: + + def infinite_stream(start: int) -> Iterator[int]: + while True: + yield start + start += 1 + +Async generators are handled in a similar fashion, but don't +expect a ``ReturnType`` type argument +(:class:`AsyncGenerator[YieldType, SendType] `). +The ``SendType`` argument defaults to :const:`!None`, so the following definitions +are equivalent:: + + async def infinite_stream(start: int) -> AsyncGenerator[int]: + while True: + yield start + start = await increment(start) + + async def infinite_stream(start: int) -> AsyncGenerator[int, None]: + while True: + yield start + start = await increment(start) + +As in the synchronous case, +:class:`AsyncIterable[YieldType] ` +and :class:`AsyncIterator[YieldType] ` are +available as well:: + + async def infinite_stream(start: int) -> AsyncIterator[int]: + while True: + yield start + start = await increment(start) + +Coroutines can be annotated using +:class:`Coroutine[YieldType, SendType, ReturnType] `. +Generic arguments correspond to those of :class:`~collections.abc.Generator`, +for example:: + + from collections.abc import Coroutine + c: Coroutine[list[str], str, int] # Some coroutine defined elsewhere + x = c.send('hi') # Inferred type of 'x' is list[str] + async def bar() -> None: + y = await c # Inferred type of 'y' is int + .. _user-defined-generics: User-defined generic types @@ -910,14 +935,25 @@ using ``[]``. .. versionadded:: 3.11 .. data:: Never + NoReturn - The `bottom type `_, + :data:`!Never` and :data:`!NoReturn` represent the + `bottom type `_, a type that has no members. - This can be used to define a function that should never be - called, or a function that never returns:: + They can be used to indicate that a function never returns, + such as :func:`sys.exit`:: + + from typing import Never # or NoReturn + + def stop() -> Never: + raise RuntimeError('no way') + + Or to define a function that should never be + called, as there are no valid arguments, such as + :func:`assert_never`:: - from typing import Never + from typing import Never # or NoReturn def never_call_me(arg: Never) -> None: pass @@ -930,32 +966,18 @@ using ``[]``. case str(): print("It's a str") case _: - never_call_me(arg) # OK, arg is of type Never + never_call_me(arg) # OK, arg is of type Never (or NoReturn) - .. versionadded:: 3.11 - - On older Python versions, :data:`NoReturn` may be used to express the - same concept. ``Never`` was added to make the intended meaning more explicit. - -.. data:: NoReturn - - Special type indicating that a function never returns. - - For example:: + :data:`!Never` and :data:`!NoReturn` have the same meaning in the type system + and static type checkers treat both equivalently. - from typing import NoReturn + .. versionadded:: 3.6.2 - def stop() -> NoReturn: - raise RuntimeError('no way') + Added :data:`NoReturn`. - ``NoReturn`` can also be used as a - `bottom type `_, a type that - has no values. Starting in Python 3.11, the :data:`Never` type should - be used for this concept instead. Type checkers should treat the two - equivalently. + .. versionadded:: 3.11 - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.2 + Added :data:`Never`. .. data:: Self @@ -1234,6 +1256,10 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.5.3 + .. versionchanged:: 3.13 + + :data:`ClassVar` can now be nested in :data:`Final` and vice versa. + .. data:: Final Special typing construct to indicate final names to type checkers. @@ -1257,6 +1283,10 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.8 + .. versionchanged:: 3.13 + + :data:`Final` can now be nested in :data:`ClassVar` and vice versa. + .. data:: Required Special typing construct to mark a :class:`TypedDict` key as required. @@ -1275,6 +1305,26 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.11 +.. data:: ReadOnly + + A special typing construct to mark an item of a :class:`TypedDict` as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1999 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + + See :class:`TypedDict` and :pep:`705` for more details. + + .. versionadded:: 3.13 + .. data:: Annotated Special typing form to add context-specific metadata to an annotation. @@ -1408,6 +1458,23 @@ These can be used as types in annotations. They all support subscription using >>> X.__metadata__ ('very', 'important', 'metadata') + * At runtime, if you want to retrieve the original + type wrapped by ``Annotated``, use the :attr:`!__origin__` attribute: + + .. doctest:: + + >>> from typing import Annotated, get_origin + >>> Password = Annotated[str, "secret"] + >>> Password.__origin__ + + + Note that using :func:`get_origin` will return ``Annotated`` itself: + + .. doctest:: + + >>> get_origin(Password) + typing.Annotated + .. seealso:: :pep:`593` - Flexible function and variable annotations @@ -1416,22 +1483,23 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.9 -.. data:: TypeGuard +.. data:: TypeIs - Special typing construct for marking user-defined type guard functions. + Special typing construct for marking user-defined type predicate functions. - ``TypeGuard`` can be used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. - At runtime, functions marked this way should return a boolean. + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and take at + least one positional argument. - ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard":: + conditional expression here is sometimes referred to as a "type predicate":: def is_str(val: str | float): - # "isinstance" type guard + # "isinstance" type predicate if isinstance(val, str): # Type of ``val`` is narrowed to ``str`` ... @@ -1440,8 +1508,73 @@ These can be used as types in annotations. They all support subscription using ... Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeIs[...]`` or + :data:`TypeGuard` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. + + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for a given + function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. + + For example:: + + from typing import assert_type, final, TypeIs + + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass + + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) + + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + If a ``TypeIs`` function is a class or instance method, then the type in + ``TypeIs`` maps to the type of the second parameter (after ``cls`` or + ``self``). + + In short, the form ``def foo(arg: TypeA) -> TypeIs[TypeB]: ...``, + means that if ``foo(arg)`` returns ``True``, then ``arg`` is an instance + of ``TypeB``, and if it returns ``False``, it is not an instance of ``TypeB``. + + ``TypeIs`` also works with type variables. For more information, see + :pep:`742` (Narrowing types with ``TypeIs``). + + .. versionadded:: 3.13 + + +.. data:: TypeGuard + + Special typing construct for marking user-defined type predicate functions. + + Type predicate functions are user-defined functions that return whether their + argument is an instance of a particular type. + ``TypeGuard`` works similarly to :data:`TypeIs`, but has subtly different + effects on type checking behavior (see below). Using ``-> TypeGuard`` tells the static type checker that for a given function: @@ -1450,6 +1583,8 @@ These can be used as types in annotations. They all support subscription using 2. If the return value is ``True``, the type of its argument is the type inside ``TypeGuard``. + ``TypeGuard`` also works with type variables. See :pep:`647` for more details. + For example:: def is_str_list(val: list[object]) -> TypeGuard[list[str]]: @@ -1464,23 +1599,19 @@ These can be used as types in annotations. They all support subscription using # Type of ``val`` remains as ``list[object]``. print("Not a list of strings!") - If ``is_str_list`` is a class or instance method, then the type in - ``TypeGuard`` maps to the type of the second parameter after ``cls`` or - ``self``. - - In short, the form ``def foo(arg: TypeA) -> TypeGuard[TypeB]: ...``, - means that if ``foo(arg)`` returns ``True``, then ``arg`` narrows from - ``TypeA`` to ``TypeB``. - - .. note:: - - ``TypeB`` need not be a narrower form of ``TypeA`` -- it can even be a - wider form. The main reason is to allow for things like - narrowing ``list[object]`` to ``list[str]`` even though the latter - is not a subtype of the former, since ``list`` is invariant. - The responsibility of writing type-safe type guards is left to the user. + ``TypeIs`` and ``TypeGuard`` differ in the following ways: - ``TypeGuard`` also works with type variables. See :pep:`647` for more details. + * ``TypeIs`` requires the narrowed type to be a subtype of the input type, while + ``TypeGuard`` does not. The main reason is to allow for things like + narrowing ``list[object]`` to ``list[str]`` even though the latter + is not a subtype of the former, since ``list`` is invariant. + * When a ``TypeGuard`` function returns ``True``, type checkers narrow the type of the + variable to exactly the ``TypeGuard`` type. When a ``TypeIs`` function returns ``True``, + type checkers can infer a more precise type combining the previously known type of the + variable with the ``TypeIs`` type. (Technically, this is known as an intersection type.) + * When a ``TypeGuard`` function returns ``False``, type checkers cannot narrow the type of + the variable at all. When a ``TypeIs`` function returns ``False``, type checkers can narrow + the type of the variable to exclude the ``TypeIs`` type. .. versionadded:: 3.10 @@ -1583,7 +1714,7 @@ without the dedicated syntax, as documented below. .. _typevar: -.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False) +.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) Type variable. @@ -1711,6 +1842,16 @@ without the dedicated syntax, as documented below. the bound is evaluated only when the attribute is accessed, not when the type variable is created (see :ref:`lazy-evaluation`). + .. method:: evaluate_bound + + An :term:`evaluate function` corresponding to the :attr:`~TypeVar.__bound__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVar.__bound__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + .. attribute:: __constraints__ A tuple containing the constraints of the type variable, if any. @@ -1721,15 +1862,55 @@ without the dedicated syntax, as documented below. the constraints are evaluated only when the attribute is accessed, not when the type variable is created (see :ref:`lazy-evaluation`). + .. method:: evaluate_constraints + + An :term:`evaluate function` corresponding to the :attr:`~TypeVar.__constraints__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVar.__constraints__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + + .. attribute:: __default__ + + The default value of the type variable, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: evaluate_default + + An :term:`evaluate function` corresponding to the :attr:`~TypeVar.__default__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVar.__default__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + + .. method:: has_default() + + Return whether or not the type variable has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + .. versionchanged:: 3.12 Type variables can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. The ``infer_variance`` parameter was added. + .. versionchanged:: 3.13 + + Support for default values was added. + .. _typevartuple: -.. class:: TypeVarTuple(name) +.. class:: TypeVarTuple(name, *, default=typing.NoDefault) Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. @@ -1820,8 +2001,8 @@ without the dedicated syntax, as documented below. of ``*args``:: def call_soon[*Ts]( - callback: Callable[[*Ts], None], - *args: *Ts + callback: Callable[[*Ts], None], + *args: *Ts ) -> None: ... callback(*args) @@ -1839,6 +2020,32 @@ without the dedicated syntax, as documented below. The name of the type variable tuple. + .. attribute:: __default__ + + The default value of the type variable tuple, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: evaluate_default + + An :term:`evaluate function` corresponding to the :attr:`~TypeVarTuple.__default__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVarTuple.__default__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + + .. method:: has_default() + + Return whether or not the type variable tuple has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -1846,7 +2053,11 @@ without the dedicated syntax, as documented below. Type variable tuples can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. -.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False) + .. versionchanged:: 3.13 + + Support for default values was added. + +.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of :ref:`type variables `. @@ -1915,6 +2126,32 @@ without the dedicated syntax, as documented below. The name of the parameter specification. + .. attribute:: __default__ + + The default value of the parameter specification, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: evaluate_default + + An :term:`evaluate function` corresponding to the :attr:`~ParamSpec.__default__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~ParamSpec.__default__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + + .. method:: has_default() + + Return whether or not the parameter specification has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + Parameter specification variables created with ``covariant=True`` or ``contravariant=True`` can be used to declare covariant or contravariant generic types. The ``bound`` argument is also accepted, similar to @@ -1928,6 +2165,10 @@ without the dedicated syntax, as documented below. Parameter specifications can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. + .. versionchanged:: 3.13 + + Support for default values was added. + .. note:: Only parameter specification variables defined in global scope can be pickled. @@ -1939,7 +2180,7 @@ without the dedicated syntax, as documented below. * :ref:`annotating-callables` .. data:: ParamSpecArgs -.. data:: ParamSpecKwargs + ParamSpecKwargs Arguments and keyword arguments attributes of a :class:`ParamSpec`. The ``P.args`` attribute of a ``ParamSpec`` is an instance of ``ParamSpecArgs``, @@ -2026,6 +2267,32 @@ without the dedicated syntax, as documented below. >>> Recursive.__value__ Mutually + .. method:: evaluate_value + + An :term:`evaluate function` corresponding to the :attr:`__value__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`__value__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format: + + .. doctest:: + + >>> type Alias = undefined + >>> Alias.__value__ + Traceback (most recent call last): + ... + NameError: name 'undefined' is not defined + >>> from annotationlib import Format, call_evaluate_function + >>> Alias.evaluate_value(Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'undefined' is not defined + >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) + ForwardRef('undefined') + + .. versionadded:: 3.14 + + Other special directives """""""""""""""""""""""" @@ -2082,7 +2349,9 @@ types. Backward-compatible usage:: - # For creating a generic NamedTuple on Python 3.11 or lower + # For creating a generic NamedTuple on Python 3.11 + T = TypeVar("T") + class Group(NamedTuple, Generic[T]): key: T group: list[T] @@ -2407,7 +2676,7 @@ types. This attribute reflects *only* the value of the ``total`` argument to the current ``TypedDict`` class, not whether the class is semantically - total. For example, a ``TypedDict`` with ``__total__`` set to True may + total. For example, a ``TypedDict`` with ``__total__`` set to ``True`` may have keys marked with :data:`NotRequired`, or it may inherit from another ``TypedDict`` with ``total=False``. Therefore, it is generally better to use :attr:`__required_keys__` and :attr:`__optional_keys__` for introspection. @@ -2455,6 +2724,22 @@ types. ``__required_keys__`` and ``__optional_keys__`` rely on may not work properly, and the values of the attributes may be incorrect. + Support for :data:`ReadOnly` is reflected in the following attributes: + + .. attribute:: __readonly_keys__ + + A :class:`frozenset` containing the names of all read-only keys. Keys + are read-only if they carry the :data:`ReadOnly` qualifier. + + .. versionadded:: 3.13 + + .. attribute:: __mutable_keys__ + + A :class:`frozenset` containing the names of all mutable keys. Keys + are mutable if they do not carry the :data:`ReadOnly` qualifier. + + .. versionadded:: 3.13 + See :pep:`589` for more examples and detailed rules of using ``TypedDict``. .. versionadded:: 3.8 @@ -2469,6 +2754,9 @@ types. .. versionchanged:: 3.13 Removed support for the keyword-argument method of creating ``TypedDict``\ s. + .. versionchanged:: 3.13 + Support for the :data:`ReadOnly` qualifier was added. + .. deprecated-removed:: 3.13 3.15 When using the functional syntax to create a TypedDict class, failing to pass a value to the 'fields' parameter (``TD = TypedDict("TD")``) is @@ -2597,7 +2885,7 @@ Functions and decorators .. seealso:: `Unreachable Code and Exhaustiveness Checking - `__ has more + `__ has more information about exhaustiveness checking with static typing. .. versionadded:: 3.11 @@ -2970,33 +3258,38 @@ Introspection helpers Return a dictionary containing type hints for a function, method, module or class object. - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. - - The function recursively replaces all ``Annotated[T, ...]`` with ``T``, - unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for - more information). For example: - - .. testcode:: - - class Student(NamedTuple): - name: Annotated[str, 'some marker'] - - assert get_type_hints(Student) == {'name': str} - assert get_type_hints(Student, include_extras=False) == {'name': str} - assert get_type_hints(Student, include_extras=True) == { - 'name': Annotated[str, 'some marker'] - } + This is often the same as ``obj.__annotations__``, but this function makes + the following changes to the annotations dictionary: + + * Forward references encoded as string literals or :class:`ForwardRef` + objects are handled by evaluating them in *globalns*, *localns*, and + (where applicable) *obj*'s :ref:`type parameter ` namespace. + If *globalns* or *localns* is not given, appropriate namespace + dictionaries are inferred from *obj*. + * ``None`` is replaced with :class:`types.NoneType`. + * If :func:`@no_type_check ` has been applied to *obj*, an + empty dictionary is returned. + * If *obj* is a class ``C``, the function returns a dictionary that merges + annotations from ``C``'s base classes with those on ``C`` directly. This + is done by traversing :attr:`C.__mro__ ` and iteratively + combining + ``__annotations__`` dictionaries. Annotations on classes appearing + earlier in the :term:`method resolution order` always take precedence over + annotations on classes appearing later in the method resolution order. + * The function recursively replaces all occurrences of ``Annotated[T, ...]`` + with ``T``, unless *include_extras* is set to ``True`` (see + :class:`Annotated` for more information). + + See also :func:`inspect.get_annotations`, a lower-level function that + returns annotations more directly. .. note:: - :func:`get_type_hints` does not work with imported - :ref:`type aliases ` that include forward references. - Enabling postponed evaluation of annotations (:pep:`563`) may remove - the need for most forward references. + If any forward references in the annotations of *obj* are not resolvable + or are not valid Python code, this function will raise an exception + such as :exc:`NameError`. For example, this can happen with imported + :ref:`type aliases ` that include forward references, + or with names imported under :data:`if TYPE_CHECKING `. .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. @@ -3025,6 +3318,7 @@ Introspection helpers assert get_origin(str) is None assert get_origin(Dict[str, int]) is dict assert get_origin(Union[int, str]) is Union + assert get_origin(Annotated[str, "metadata"]) is Annotated P = ParamSpec('P') assert get_origin(P.args) is P assert get_origin(P.kwargs) is P @@ -3109,7 +3403,7 @@ Introspection helpers Class used for internal typing representation of string forward references. For example, ``List["SomeClass"]`` is implicitly transformed into - ``List[ForwardRef("SomeClass")]``. ``ForwardRef`` should not be instantiated by + ``List[ForwardRef("SomeClass")]``. :class:`!ForwardRef` should not be instantiated by a user, but may be used by introspection tools. .. note:: @@ -3119,6 +3413,55 @@ Introspection helpers .. versionadded:: 3.7.4 + .. versionchanged:: 3.14 + This is now an alias for :class:`annotationlib.ForwardRef`. + +.. function:: evaluate_forward_ref(forward_ref, *, owner=None, globals=None, locals=None, type_params=None, format=annotationlib.Format.VALUE) + + Evaluate an :class:`annotationlib.ForwardRef` as a :term:`type hint`. + + This is similar to calling :meth:`annotationlib.ForwardRef.evaluate`, + but unlike that method, :func:`!evaluate_forward_ref` also: + + * Recursively evaluates forward references nested within the type hint. + * Raises :exc:`TypeError` when it encounters certain objects that are + not valid type hints. + * Replaces type hints that evaluate to :const:`!None` with + :class:`types.NoneType`. + * Supports the :attr:`~annotationlib.Format.FORWARDREF` and + :attr:`~annotationlib.Format.STRING` formats. + + *forward_ref* must be an instance of :class:`~annotationlib.ForwardRef`. + *owner*, if given, should be the object that holds the annotations that + the forward reference derived from, such as a module, class object, or function. + It is used to infer the namespaces to use for looking up names. + *globals* and *locals* can also be explicitly given to provide + the global and local namespaces. + *type_params* is a tuple of :ref:`type parameters ` that + are in scope when evaluating the forward reference. + This parameter must be provided (though it may be an empty tuple) if *owner* + is not given and the forward reference does not already have an owner set. + *format* specifies the format of the annotation and is a member of + the :class:`annotationlib.Format` enum. + + .. versionadded:: 3.14 + +.. data:: NoDefault + + A sentinel object used to indicate that a type parameter has no default + value. For example: + + .. doctest:: + + >>> T = TypeVar("T") + >>> T.__default__ is typing.NoDefault + True + >>> S = TypeVar("S", default=None) + >>> S.__default__ is None + True + + .. versionadded:: 3.13 + Constant -------- @@ -3186,14 +3529,9 @@ Aliases to built-in types Deprecated alias to :class:`dict`. Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`Mapping` + to use an abstract collection type such as :class:`~collections.abc.Mapping` rather than to use :class:`dict` or :class:`!typing.Dict`. - This type can be used as follows:: - - def count_words(text: str) -> Dict[str, int]: - ... - .. deprecated:: 3.9 :class:`builtins.dict ` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. @@ -3203,16 +3541,9 @@ Aliases to built-in types Deprecated alias to :class:`list`. Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`Sequence` or - :class:`Iterable` rather than to use :class:`list` or :class:`!typing.List`. - - This type may be used as follows:: - - def vec2[T: (int, float)](x: T, y: T) -> List[T]: - return [x, y] - - def keep_positives[T: (int, float)](vector: Sequence[T]) -> List[T]: - return [item for item in vector if item > 0] + to use an abstract collection type such as + :class:`~collections.abc.Sequence` or :class:`~collections.abc.Iterable` + rather than to use :class:`list` or :class:`!typing.List`. .. deprecated:: 3.9 :class:`builtins.list ` now supports subscripting (``[]``). @@ -3223,8 +3554,8 @@ Aliases to built-in types Deprecated alias to :class:`builtins.set `. Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`AbstractSet` - rather than to use :class:`set` or :class:`!typing.Set`. + to use an abstract collection type such as :class:`collections.abc.Set` + rather than to use :class:`set` or :class:`typing.Set`. .. deprecated:: 3.9 :class:`builtins.set ` now supports subscripting (``[]``). @@ -3292,7 +3623,6 @@ Aliases to types in :mod:`collections` Deprecated alias to :class:`collections.ChainMap`. - .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. deprecated:: 3.9 @@ -3303,7 +3633,6 @@ Aliases to types in :mod:`collections` Deprecated alias to :class:`collections.Counter`. - .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. deprecated:: 3.9 @@ -3314,7 +3643,6 @@ Aliases to types in :mod:`collections` Deprecated alias to :class:`collections.deque`. - .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. deprecated:: 3.9 @@ -3377,19 +3705,11 @@ Aliases to container ABCs in :mod:`collections.abc` :class:`collections.abc.Set` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. class:: ByteString(Sequence[int]) - - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview` of byte sequences. - - .. deprecated-removed:: 3.9 3.14 - Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. - .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) Deprecated alias to :class:`collections.abc.Collection`. - .. versionadded:: 3.6.0 + .. versionadded:: 3.6 .. deprecated:: 3.9 :class:`collections.abc.Collection` now supports subscripting (``[]``). @@ -3423,11 +3743,6 @@ Aliases to container ABCs in :mod:`collections.abc` Deprecated alias to :class:`collections.abc.Mapping`. - This type can be used as follows:: - - def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: - return word_list[word] - .. deprecated:: 3.9 :class:`collections.abc.Mapping` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. @@ -3491,14 +3806,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` Deprecated alias to :class:`collections.abc.Coroutine`. - The variance and order of type variables - correspond to those of :class:`Generator`, for example:: - - from collections.abc import Coroutine - c: Coroutine[list[str], str, int] # Some coroutine defined elsewhere - x = c.send('hi') # Inferred type of 'x' is list[str] - async def bar() -> None: - y = await c # Inferred type of 'y' is int + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`collections.abc.Coroutine` + and ``typing.Coroutine`` in type annotations. .. versionadded:: 3.5.3 @@ -3510,34 +3820,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` Deprecated alias to :class:`collections.abc.AsyncGenerator`. - An async generator can be annotated by the generic type - ``AsyncGenerator[YieldType, SendType]``. For example:: - - async def echo_round() -> AsyncGenerator[int, float]: - sent = yield 0 - while sent >= 0.0: - rounded = await round(sent) - sent = yield rounded - - Unlike normal generators, async generators cannot return a value, so there - is no ``ReturnType`` type parameter. As with :class:`Generator`, the - ``SendType`` behaves contravariantly. - - If your generator will only yield values, set the ``SendType`` to - ``None``:: - - async def infinite_stream(start: int) -> AsyncGenerator[int, None]: - while True: - yield start - start = await increment(start) - - Alternatively, annotate your generator as having a return type of - either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: - - async def infinite_stream(start: int) -> AsyncIterator[int]: - while True: - yield start - start = await increment(start) + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`collections.abc.AsyncGenerator` + and ``typing.AsyncGenerator`` in type annotations. .. versionadded:: 3.6.1 @@ -3546,6 +3831,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + The ``SendType`` parameter now has a default. + .. class:: AsyncIterable(Generic[T_co]) Deprecated alias to :class:`collections.abc.AsyncIterable`. @@ -3616,39 +3904,17 @@ Aliases to other ABCs in :mod:`collections.abc` Deprecated alias to :class:`collections.abc.Generator`. - A generator can be annotated by the generic type - ``Generator[YieldType, SendType, ReturnType]``. For example:: - - def echo_round() -> Generator[int, float, str]: - sent = yield 0 - while sent >= 0: - sent = yield round(sent) - return 'Done' - - Note that unlike many other generics in the typing module, the ``SendType`` - of :class:`Generator` behaves contravariantly, not covariantly or - invariantly. - - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: - - def infinite_stream(start: int) -> Generator[int, None, None]: - while True: - yield start - start += 1 - - Alternatively, annotate your generator as having a return type of - either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: - - def infinite_stream(start: int) -> Iterator[int]: - while True: - yield start - start += 1 + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`collections.abc.Generator` + and ``typing.Generator`` in type annotations. .. deprecated:: 3.9 :class:`collections.abc.Generator` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Default values for the send and return types were added. + .. class:: Hashable Deprecated alias to :class:`collections.abc.Hashable`. @@ -3676,23 +3942,34 @@ Aliases to other ABCs in :mod:`collections.abc` Aliases to :mod:`contextlib` ABCs """"""""""""""""""""""""""""""""" -.. class:: ContextManager(Generic[T_co]) +.. class:: ContextManager(Generic[T_co, ExitT_co]) Deprecated alias to :class:`contextlib.AbstractContextManager`. + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__enter__` method. The optional second type parameter, ``ExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__exit__` method. + .. versionadded:: 3.5.4 - .. versionadded:: 3.6.0 .. deprecated:: 3.9 :class:`contextlib.AbstractContextManager` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. class:: AsyncContextManager(Generic[T_co]) + .. versionchanged:: 3.13 + Added the optional second type parameter, ``ExitT_co``. + +.. class:: AsyncContextManager(Generic[T_co, AExitT_co]) Deprecated alias to :class:`contextlib.AbstractAsyncContextManager`. - .. versionadded:: 3.5.4 + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__aenter__` method. The optional second type parameter, ``AExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__aexit__` method. + .. versionadded:: 3.6.2 .. deprecated:: 3.9 @@ -3700,6 +3977,9 @@ Aliases to :mod:`contextlib` ABCs now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Added the optional second type parameter, ``AExitT_co``. + Deprecation Timeline of Major Features ====================================== @@ -3718,10 +3998,6 @@ convenience. This is subject to change, and not all deprecations are listed. - 3.9 - Undecided (see :ref:`deprecated-aliases` for more information) - :pep:`585` - * - :class:`typing.ByteString` - - 3.9 - - 3.14 - - :gh:`91896` * - :data:`typing.Text` - 3.11 - Undecided diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 7db47d48022a0e..0aef597d064e0e 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -1,5 +1,5 @@ -:mod:`unicodedata` --- Unicode Database -======================================= +:mod:`!unicodedata` --- Unicode Database +======================================== .. module:: unicodedata :synopsis: Access the Unicode Database. @@ -17,8 +17,8 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 15.1.0 -`_. +this database is compiled from the `UCD version 16.0.0 +`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" @@ -175,6 +175,6 @@ Examples: .. rubric:: Footnotes -.. [#] https://www.unicode.org/Public/15.1.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/16.0.0/ucd/NameAliases.txt -.. [#] https://www.unicode.org/Public/15.1.0/ucd/NamedSequences.txt +.. [#] https://www.unicode.org/Public/16.0.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index f2bdde80bdbd64..00cc9bfc0a5f2b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -1,5 +1,5 @@ -:mod:`unittest.mock` --- getting started -======================================== +:mod:`!unittest.mock` --- getting started +========================================= .. moduleauthor:: Michael Foord .. currentmodule:: unittest.mock diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index eca20b94ec8e74..eae3ef2888eae0 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -1,6 +1,5 @@ - -:mod:`unittest.mock` --- mock object library -============================================ +:mod:`!unittest.mock` --- mock object library +============================================= .. module:: unittest.mock :synopsis: Mock object library. @@ -35,7 +34,7 @@ is based on the 'action -> assertion' pattern instead of 'record -> replay' used by many mocking frameworks. There is a backport of :mod:`unittest.mock` for earlier versions of Python, -available as `mock on PyPI `_. +available as :pypi:`mock` on PyPI. Quick Guide @@ -69,7 +68,7 @@ available, and then make assertions about how they have been used: 3 >>> thing.method.assert_called_with(3, 4, 5, key='value') -:attr:`side_effect` allows you to perform side effects, including raising an +:attr:`~Mock.side_effect` allows you to perform side effects, including raising an exception when a mock is called: >>> from unittest.mock import Mock @@ -240,7 +239,7 @@ the *new_callable* argument to :func:`patch`. Accessing any attribute not in this list will raise an :exc:`AttributeError`. If *spec* is an object (rather than a list of strings) then - :attr:`~instance.__class__` returns the class of the spec object. This + :attr:`~object.__class__` returns the class of the spec object. This allows mocks to pass :func:`isinstance` tests. * *spec_set*: A stricter variant of *spec*. If used, attempting to *set* @@ -402,6 +401,8 @@ the *new_callable* argument to :func:`patch`. The reset_mock method resets all the call attributes on a mock object: + .. doctest:: + >>> mock = Mock(return_value=None) >>> mock('hello') >>> mock.called @@ -410,20 +411,41 @@ the *new_callable* argument to :func:`patch`. >>> mock.called False - .. versionchanged:: 3.6 - Added two keyword-only arguments to the reset_mock function. - This can be useful where you want to make a series of assertions that - reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the - return value, :attr:`side_effect` or any child attributes you have - set using normal assignment by default. In case you want to reset - *return_value* or :attr:`side_effect`, then pass the corresponding - parameter as ``True``. Child mocks and the return value mock - (if any) are reset as well. + reuse the same object. + + *return_value* parameter when set to ``True`` resets :attr:`return_value`: + + .. doctest:: + + >>> mock = Mock(return_value=5) + >>> mock('hello') + 5 + >>> mock.reset_mock(return_value=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + *side_effect* parameter when set to ``True`` resets :attr:`side_effect`: + + .. doctest:: + + >>> mock = Mock(side_effect=ValueError) + >>> mock('hello') + Traceback (most recent call last): + ... + ValueError + >>> mock.reset_mock(side_effect=True) + >>> mock('hello') # doctest: +ELLIPSIS + - .. note:: *return_value*, and :attr:`side_effect` are keyword-only - arguments. + Note that :meth:`reset_mock` *doesn't* clear the + :attr:`return_value`, :attr:`side_effect` or any child attributes you have + set using normal assignment by default. + Child mocks are reset as well. + + .. versionchanged:: 3.6 + Added two keyword-only arguments to the reset_mock function. .. method:: mock_add_spec(spec, spec_set=False) @@ -738,8 +760,8 @@ the *new_callable* argument to :func:`patch`. .. attribute:: __class__ - Normally the :attr:`__class__` attribute of an object will return its type. - For a mock object with a :attr:`spec`, ``__class__`` returns the spec class + Normally the :attr:`!__class__` attribute of an object will return its type. + For a mock object with a :attr:`!spec`, :attr:`!__class__` returns the spec class instead. This allows mock objects to pass :func:`isinstance` tests for the object they are replacing / masquerading as: @@ -747,7 +769,7 @@ the *new_callable* argument to :func:`patch`. >>> isinstance(mock, int) True - :attr:`__class__` is assignable to, this allows a mock to pass an + :attr:`!__class__` is assignable to, this allows a mock to pass an :func:`isinstance` check without forcing you to use a spec: >>> mock = Mock() @@ -761,8 +783,8 @@ the *new_callable* argument to :func:`patch`. meaning of :class:`Mock`, with the exception of *return_value* and *side_effect* which have no meaning on a non-callable mock. -Mock objects that use a class or an instance as a :attr:`spec` or -:attr:`spec_set` are able to pass :func:`isinstance` tests: +Mock objects that use a class or an instance as a :attr:`!spec` or +:attr:`!spec_set` are able to pass :func:`isinstance` tests: >>> mock = Mock(spec=SomeClass) >>> isinstance(mock, SomeClass) @@ -861,6 +883,20 @@ object:: 3 >>> p.assert_called_once_with() +.. caution:: + + If an :exc:`AttributeError` is raised by :class:`PropertyMock`, + it will be interpreted as a missing descriptor and + :meth:`~object.__getattr__` will be called on the parent mock:: + + >>> m = MagicMock() + >>> no_attribute = PropertyMock(side_effect=AttributeError) + >>> type(m).my_property = no_attribute + >>> m.my_property + + + See :meth:`~object.__getattr__` for details. + .. class:: AsyncMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) @@ -869,7 +905,7 @@ object:: call is an awaitable. >>> mock = AsyncMock() - >>> asyncio.iscoroutinefunction(mock) + >>> inspect.iscoroutinefunction(mock) True >>> inspect.isawaitable(mock()) # doctest: +SKIP True @@ -1162,7 +1198,7 @@ Calls made to the object will be recorded in the attributes like :attr:`~Mock.call_args` and :attr:`~Mock.call_args_list`. If :attr:`~Mock.side_effect` is set then it will be called after the call has -been recorded, so if :attr:`side_effect` raises an exception the call is still +been recorded, so if :attr:`!side_effect` raises an exception the call is still recorded. The simplest way to make a mock raise an exception when called is to make @@ -1183,8 +1219,8 @@ The simplest way to make a mock raise an exception when called is to make >>> m.mock_calls [call(1, 2, 3), call('two', 'three', 'four')] -If :attr:`side_effect` is a function then whatever that function returns is what -calls to the mock return. The :attr:`side_effect` function is called with the +If :attr:`~Mock.side_effect` is a function then whatever that function returns is what +calls to the mock return. The :attr:`!side_effect` function is called with the same arguments as the mock. This allows you to vary the return value of the call dynamically, based on the input: @@ -1201,7 +1237,7 @@ call dynamically, based on the input: If you want the mock to still return the default return value (a new mock), or any set return value, then there are two ways of doing this. Either return -:attr:`mock.return_value` from inside :attr:`side_effect`, or return :data:`DEFAULT`: +:attr:`~Mock.return_value` from inside :attr:`~Mock.side_effect`, or return :data:`DEFAULT`: >>> m = MagicMock() >>> def side_effect(*args, **kwargs): @@ -1218,8 +1254,8 @@ any set return value, then there are two ways of doing this. Either return >>> m() 3 -To remove a :attr:`side_effect`, and return to the default behaviour, set the -:attr:`side_effect` to ``None``: +To remove a :attr:`~Mock.side_effect`, and return to the default behaviour, set the +:attr:`!side_effect` to ``None``: >>> m = MagicMock(return_value=6) >>> def side_effect(*args, **kwargs): @@ -1232,7 +1268,7 @@ To remove a :attr:`side_effect`, and return to the default behaviour, set the >>> m() 6 -The :attr:`side_effect` can also be any iterable object. Repeated calls to the mock +The :attr:`~Mock.side_effect` can also be any iterable object. Repeated calls to the mock will return values from the iterable (until the iterable is exhausted and a :exc:`StopIteration` is raised): @@ -1273,7 +1309,7 @@ objects of any type. You may want a mock object to return ``False`` to a :func:`hasattr` call, or raise an :exc:`AttributeError` when an attribute is fetched. You can do this by providing -an object as a :attr:`spec` for a mock, but that isn't always convenient. +an object as a :attr:`!spec` for a mock, but that isn't always convenient. You "block" attributes by deleting them. Once deleted, accessing an attribute will raise an :exc:`AttributeError`. @@ -1442,7 +1478,7 @@ patch If you are patching builtins in a module then you don't need to pass ``create=True``, it will be added by default. - Patch can be used as a :class:`TestCase` class decorator. It works by + Patch can be used as a :class:`~unittest.TestCase` class decorator. It works by decorating each test method in the class. This reduces the boilerplate code when your test methods share a common patchings set. :func:`patch` finds tests by looking for method names that start with ``patch.TEST_PREFIX``. @@ -1480,7 +1516,7 @@ If the class is instantiated multiple times you could use can set the *return_value* to be anything you want. To configure return values on methods of *instances* on the patched class -you must do this on the :attr:`return_value`. For example:: +you must do this on the :attr:`~Mock.return_value`. For example:: >>> class Class: ... def method(self): @@ -1625,7 +1661,8 @@ patch.dict .. function:: patch.dict(in_dict, values=(), clear=False, **kwargs) Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is a + copy of the dictionary as it was before the test. *in_dict* can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items @@ -1801,13 +1838,13 @@ context manager is a dictionary where created mocks are keyed by name:: patch methods: start and stop ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -All the patchers have :meth:`start` and :meth:`stop` methods. These make it simpler to do +All the patchers have :meth:`!start` and :meth:`!stop` methods. These make it simpler to do patching in ``setUp`` methods or where you want to do multiple patches without nesting decorators or with statements. To use them call :func:`patch`, :func:`patch.object` or :func:`patch.dict` as normal and keep a reference to the returned ``patcher`` object. You can then -call :meth:`start` to put the patch in place and :meth:`stop` to undo it. +call :meth:`!start` to put the patch in place and :meth:`!stop` to undo it. If you are using :func:`patch` to create a mock for you then it will be returned by the call to ``patcher.start``. :: @@ -1824,7 +1861,7 @@ the call to ``patcher.start``. :: A typical use case for this might be for doing multiple patches in the ``setUp`` -method of a :class:`TestCase`:: +method of a :class:`~unittest.TestCase`:: >>> class MyTest(unittest.TestCase): ... def setUp(self): @@ -2143,10 +2180,10 @@ to change the default. Methods and their defaults: -* ``__lt__``: ``NotImplemented`` -* ``__gt__``: ``NotImplemented`` -* ``__le__``: ``NotImplemented`` -* ``__ge__``: ``NotImplemented`` +* ``__lt__``: :data:`NotImplemented` +* ``__gt__``: :data:`!NotImplemented` +* ``__le__``: :data:`!NotImplemented` +* ``__ge__``: :data:`!NotImplemented` * ``__int__``: ``1`` * ``__contains__``: ``False`` * ``__len__``: ``0`` @@ -2426,6 +2463,14 @@ passed in. >>> m.mock_calls == [call(1), call(1, 2), ANY] True +:data:`ANY` is not limited to comparisons with call objects and so +can also be used in test assertions:: + + class TestStringMethods(unittest.TestCase): + + def test_split(self): + s = 'hello world' + self.assertEqual(s.split(), ['hello', ANY]) FILTER_DIR @@ -2489,7 +2534,7 @@ behaviour you can switch it off by setting the module level switch Alternatively you can just use ``vars(my_mock)`` (instance members) and ``dir(type(my_mock))`` (type members) to bypass the filtering irrespective of -:const:`mock.FILTER_DIR`. +:const:`FILTER_DIR`. mock_open @@ -2504,7 +2549,7 @@ mock_open default) then a :class:`MagicMock` will be created for you, with the API limited to methods or attributes available on standard file handles. - *read_data* is a string for the :meth:`~io.IOBase.read`, + *read_data* is a string for the :meth:`~io.RawIOBase.read`, :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods of the file handle to return. Calls to those methods will take data from *read_data* until it is depleted. The mock of these methods is pretty @@ -2516,7 +2561,7 @@ mock_open .. versionchanged:: 3.4 Added :meth:`~io.IOBase.readline` and :meth:`~io.IOBase.readlines` support. - The mock of :meth:`~io.IOBase.read` changed to consume *read_data* rather + The mock of :meth:`~io.RawIOBase.read` changed to consume *read_data* rather than returning it on each call. .. versionchanged:: 3.5 @@ -2568,7 +2613,7 @@ And for reading files:: Autospeccing ~~~~~~~~~~~~ -Autospeccing is based on the existing :attr:`spec` feature of mock. It limits the +Autospeccing is based on the existing :attr:`!spec` feature of mock. It limits the api of mocks to the api of an original object (the spec), but it is recursive (implemented lazily) so that attributes of mocks only have the same api as the attributes of the spec. In addition mocked functions / methods have the @@ -2577,48 +2622,24 @@ called incorrectly. Before I explain how auto-speccing works, here's why it is needed. -:class:`Mock` is a very powerful and flexible object, but it suffers from two flaws -when used to mock out objects from a system under test. One of these flaws is -specific to the :class:`Mock` api and the other is a more general problem with using -mock objects. +:class:`Mock` is a very powerful and flexible object, but it suffers from a flaw which +is general to mocking. If you refactor some of your code, rename members and so on, any +tests for code that is still using the *old api* but uses mocks instead of the real +objects will still pass. This means your tests can all pass even though your code is +broken. -First the problem specific to :class:`Mock`. :class:`Mock` has two assert methods that are -extremely handy: :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with`. +.. versionchanged:: 3.5 - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - Traceback (most recent call last): - ... - AssertionError: Expected 'mock' to be called once. Called 2 times. - -Because mocks auto-create attributes on demand, and allow you to call them -with arbitrary arguments, if you misspell one of these assert methods then -your assertion is gone: - -.. code-block:: pycon - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assret_called_once_with(4, 5, 6) # Intentional typo! - -Your tests can pass silently and incorrectly because of the typo. - -The second issue is more general to mocking. If you refactor some of your -code, rename members and so on, any tests for code that is still using the -*old api* but uses mocks instead of the real objects will still pass. This -means your tests can all pass even though your code is broken. + Before 3.5, tests with a typo in the word assert would silently pass when they should + raise an error. You can still achieve this behavior by passing ``unsafe=True`` to Mock. Note that this is another reason why you need integration tests as well as unit tests. Testing everything in isolation is all fine and dandy, but if you don't test how your units are "wired together" there is still lots of room for bugs that tests might have caught. -:mod:`mock` already provides a feature to help with this, called speccing. If you -use a class or instance as the :attr:`spec` for a mock then you can only access +:mod:`unittest.mock` already provides a feature to help with this, called speccing. If you +use a class or instance as the :attr:`!spec` for a mock then you can only access attributes on the mock that exist on the real class: >>> from urllib import request @@ -2656,7 +2677,7 @@ Here's an example of it in use:: >>> mock_request.Request -You can see that :class:`request.Request` has a spec. :class:`request.Request` takes two +You can see that :class:`!request.Request` has a spec. :class:`!request.Request` takes two arguments in the constructor (one of which is *self*). Here's what happens if we try to call it incorrectly:: @@ -2672,8 +2693,8 @@ specced mocks):: >>> req -:class:`Request` objects are not callable, so the return value of instantiating our -mocked out :class:`request.Request` is a non-callable mock. With the spec in place +:class:`!Request` objects are not callable, so the return value of instantiating our +mocked out :class:`!request.Request` is a non-callable mock. With the spec in place any typos in our asserts will raise the correct error:: >>> req.add_header('spam', 'eggs') @@ -2823,3 +2844,123 @@ Sealing mocks >>> mock.not_submock.attribute2 # This won't raise. .. versionadded:: 3.7 + + +Order of precedence of :attr:`!side_effect`, :attr:`!return_value` and *wraps* +------------------------------------------------------------------------------ + +The order of their precedence is: + +1. :attr:`~Mock.side_effect` +2. :attr:`~Mock.return_value` +3. *wraps* + +If all three are set, mock will return the value from :attr:`~Mock.side_effect`, +ignoring :attr:`~Mock.return_value` and the wrapped object altogether. If any +two are set, the one with the higher precedence will return the value. +Regardless of the order of which was set first, the order of precedence +remains unchanged. + + >>> from unittest.mock import Mock + >>> class Order: + ... @staticmethod + ... def get_value(): + ... return "third" + ... + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.get_value.side_effect = ["first"] + >>> order_mock.get_value.return_value = "second" + >>> order_mock.get_value() + 'first' + +As ``None`` is the default value of :attr:`~Mock.side_effect`, if you reassign +its value back to ``None``, the order of precedence will be checked between +:attr:`~Mock.return_value` and the wrapped object, ignoring +:attr:`~Mock.side_effect`. + + >>> order_mock.get_value.side_effect = None + >>> order_mock.get_value() + 'second' + +If the value being returned by :attr:`~Mock.side_effect` is :data:`DEFAULT`, +it is ignored and the order of precedence moves to the successor to obtain the +value to return. + + >>> from unittest.mock import DEFAULT + >>> order_mock.get_value.side_effect = [DEFAULT] + >>> order_mock.get_value() + 'second' + +When :class:`Mock` wraps an object, the default value of +:attr:`~Mock.return_value` will be :data:`DEFAULT`. + + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.return_value + sentinel.DEFAULT + >>> order_mock.get_value.return_value + sentinel.DEFAULT + +The order of precedence will ignore this value and it will move to the last +successor which is the wrapped object. + +As the real call is being made to the wrapped object, creating an instance of +this mock will return the real instance of the class. The positional arguments, +if any, required by the wrapped object must be passed. + + >>> order_mock_instance = order_mock() + >>> isinstance(order_mock_instance, Order) + True + >>> order_mock_instance.get_value() + 'third' + + >>> order_mock.get_value.return_value = DEFAULT + >>> order_mock.get_value() + 'third' + + >>> order_mock.get_value.return_value = "second" + >>> order_mock.get_value() + 'second' + +But if you assign ``None`` to it, this will not be ignored as it is an +explicit assignment. So, the order of precedence will not move to the wrapped +object. + + >>> order_mock.get_value.return_value = None + >>> order_mock.get_value() is None + True + +Even if you set all three at once when initializing the mock, the order of +precedence remains the same: + + >>> order_mock = Mock(spec=Order, wraps=Order, + ... **{"get_value.side_effect": ["first"], + ... "get_value.return_value": "second"} + ... ) + ... + >>> order_mock.get_value() + 'first' + >>> order_mock.get_value.side_effect = None + >>> order_mock.get_value() + 'second' + >>> order_mock.get_value.return_value = DEFAULT + >>> order_mock.get_value() + 'third' + +If :attr:`~Mock.side_effect` is exhausted, the order of precedence will not +cause a value to be obtained from the successors. Instead, ``StopIteration`` +exception is raised. + + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.get_value.side_effect = ["first side effect value", + ... "another side effect value"] + >>> order_mock.get_value.return_value = "second" + + >>> order_mock.get_value() + 'first side effect value' + >>> order_mock.get_value() + 'another side effect value' + + >>> order_mock.get_value() + Traceback (most recent call last): + ... + StopIteration diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index e6140ac70eb87a..38bad9405597dd 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1,5 +1,5 @@ -:mod:`unittest` --- Unit testing framework -========================================== +:mod:`!unittest` --- Unit testing framework +=========================================== .. module:: unittest :synopsis: Unit testing framework for Python. @@ -340,28 +340,21 @@ Test modules and packages can customize test loading and discovery by through the `load_tests protocol`_. .. versionchanged:: 3.4 - Test discovery supports :term:`namespace packages ` - for the start directory. Note that you need to specify the top level - directory too (e.g. - ``python -m unittest discover -s root/namespace -t root``). + Test discovery supports :term:`namespace packages `. .. versionchanged:: 3.11 - :mod:`unittest` dropped the :term:`namespace packages ` - support in Python 3.11. It has been broken since Python 3.7. Start directory and - subdirectories containing tests must be regular package that have - ``__init__.py`` file. + Test discovery dropped the :term:`namespace packages ` + support. It has been broken since Python 3.7. + Start directory and its subdirectories containing tests must be regular + package that have ``__init__.py`` file. - Directories containing start directory still can be a namespace package. - In this case, you need to specify start directory as dotted package name, - and target directory explicitly. For example:: + If the start directory is the dotted name of the package, the ancestor packages + can be namespace packages. - # proj/ <-- current directory - # namespace/ - # mypkg/ - # __init__.py - # test_mypkg.py - - python -m unittest discover -s namespace.mypkg -t . +.. versionchanged:: 3.14 + Test discovery supports :term:`namespace package` as start directory again. + To avoid scanning directories unrelated to Python, + tests are not searched in subdirectories that do not contain ``__init__.py``. .. _organizing-tests: @@ -1880,8 +1873,8 @@ Loading and running tests Python identifiers) will be loaded. All test modules must be importable from the top level of the project. If - the start directory is not the top level directory then the top level - directory must be specified separately. + the start directory is not the top level directory then *top_level_dir* + must be specified separately. If importing a module fails, for example due to a syntax error, then this will be recorded as a single error and discovery will continue. If @@ -1901,9 +1894,11 @@ Loading and running tests package. The pattern is deliberately not stored as a loader attribute so that - packages can continue discovery themselves. *top_level_dir* is stored so - ``load_tests`` does not need to pass this argument in to - ``loader.discover()``. + packages can continue discovery themselves. + + *top_level_dir* is stored internally, and used as a default to any + nested calls to ``discover()``. That is, if a package's ``load_tests`` + calls ``loader.discover()``, it does not need to pass this argument. *start_dir* can be a dotted module name as well as a directory. @@ -1913,10 +1908,8 @@ Loading and running tests Modules that raise :exc:`SkipTest` on import are recorded as skips, not errors. - .. versionchanged:: 3.4 *start_dir* can be a :term:`namespace packages `. - .. versionchanged:: 3.4 Paths are sorted before being imported so that execution order is the same even if the underlying file system's ordering is not dependent on file name. @@ -1928,8 +1921,13 @@ Loading and running tests .. versionchanged:: 3.11 *start_dir* can not be a :term:`namespace packages `. - It has been broken since Python 3.7 and Python 3.11 officially remove it. + It has been broken since Python 3.7, and Python 3.11 officially removes it. + + .. versionchanged:: 3.13 + *top_level_dir* is only stored for the duration of *discover* call. + .. versionchanged:: 3.14 + *start_dir* can once again be a :term:`namespace package`. The following attributes of a :class:`TestLoader` can be configured either by subclassing or assignment on an instance: @@ -2311,8 +2309,8 @@ Loading and running tests (see :ref:`Warning control `), otherwise it will be set to ``'default'``. - Calling ``main`` actually returns an instance of the ``TestProgram`` class. - This stores the result of the tests run as the ``result`` attribute. + Calling ``main`` returns an object with the ``result`` attribute that contains + the result of the tests run as a :class:`unittest.TestResult`. .. versionchanged:: 3.1 The *exit* parameter was added. @@ -2524,7 +2522,7 @@ Signal Handling .. versionadded:: 3.2 The :option:`-c/--catch ` command-line option to unittest, -along with the ``catchbreak`` parameter to :func:`unittest.main()`, provide +along with the ``catchbreak`` parameter to :func:`unittest.main`, provide more friendly handling of control-C during a test run. With catch break behavior enabled control-C will allow the currently running test to complete, and the test run will then end and report all the results so far. A second diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index facb11f42a40c5..1686ddd09caa48 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -1,5 +1,5 @@ -:mod:`urllib.error` --- Exception classes raised by urllib.request -================================================================== +:mod:`!urllib.error` --- Exception classes raised by urllib.request +=================================================================== .. module:: urllib.error :synopsis: Exception classes raised by urllib.request. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 3c898c3e826304..fb5353e1895bf9 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -1,5 +1,5 @@ -:mod:`urllib.parse` --- Parse URLs into components -================================================== +:mod:`!urllib.parse` --- Parse URLs into components +=================================================== .. module:: urllib.parse :synopsis: Parse URLs into or assemble them from components. @@ -22,15 +22,28 @@ to an absolute URL given a "base URL." The module has been designed to match the internet RFC on Relative Uniform Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, -``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, +``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``itms-services``, ``mailto``, ``mms``, ``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtsps``, ``rtspu``, ``sftp``, ``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, ``wais``, ``ws``, ``wss``. +.. impl-detail:: + + The inclusion of the ``itms-services`` URL scheme can prevent an app from + passing Apple's App Store review process for the macOS and iOS App Stores. + Handling for the ``itms-services`` scheme is always removed on iOS; on + macOS, it *may* be removed if CPython has been built with the + :option:`--with-app-store-compliance` option. + The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in the following sections. +This module's functions use the deprecated term ``netloc`` (or ``net_loc``), +which was introduced in :rfc:`1808`. However, this term has been obsoleted by +:rfc:`3986`, which introduced the term ``authority`` as its replacement. +The use of ``netloc`` is continued for backward compatibility. + URL Parsing ----------- @@ -168,7 +181,7 @@ or on combining URL components into a URL string. Added IPv6 URL parsing capabilities. .. versionchanged:: 3.3 - The fragment is now parsed for all URL schemes (unless *allow_fragment* is + The fragment is now parsed for all URL schemes (unless *allow_fragments* is false), in accordance with :rfc:`3986`. Previously, an allowlist of schemes that support fragments existed. diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index affdce144cd5fc..ce82552a3ae4be 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -1,5 +1,5 @@ -:mod:`urllib.request` --- Extensible library for opening URLs -============================================================= +:mod:`!urllib.request` --- Extensible library for opening URLs +============================================================== .. module:: urllib.request :synopsis: Extensible library for opening URLs. @@ -97,11 +97,9 @@ The :mod:`urllib.request` module defines the following functions: .. versionchanged:: 3.2 *cafile* and *capath* were added. - .. versionchanged:: 3.2 HTTPS virtual hosts are now supported if possible (that is, if :const:`ssl.HAS_SNI` is true). - .. versionadded:: 3.2 *data* can be an iterable object. .. versionchanged:: 3.3 @@ -115,9 +113,9 @@ The :mod:`urllib.request` module defines the following functions: ``http/1.1`` when no *context* is given. Custom *context* should set ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocols`. - .. versionchanged:: 3.13 - Remove *cafile*, *capath* and *cadefault* parameters: use the *context* - parameter instead. + .. versionchanged:: 3.13 + Remove *cafile*, *capath* and *cadefault* parameters: use the *context* + parameter instead. .. function:: install_opener(opener) @@ -220,7 +218,7 @@ The following classes are provided: An appropriate ``Content-Type`` header should be included if the *data* argument is present. If this header has not been provided and *data* - is not None, ``Content-Type: application/x-www-form-urlencoded`` will + is not ``None``, ``Content-Type: application/x-www-form-urlencoded`` will be added as a default. The next two arguments are only of interest for correct handling @@ -243,7 +241,7 @@ The following classes are provided: *method* should be a string that indicates the HTTP request method that will be used (e.g. ``'HEAD'``). If provided, its value is stored in the - :attr:`~Request.method` attribute and is used by :meth:`get_method()`. + :attr:`~Request.method` attribute and is used by :meth:`get_method`. The default is ``'GET'`` if *data* is ``None`` or ``'POST'`` otherwise. Subclasses may indicate a different default method by setting the :attr:`~Request.method` attribute in the class itself. @@ -1094,7 +1092,7 @@ FileHandler Objects .. versionchanged:: 3.2 This method is applicable only for local hostnames. When a remote - hostname is given, an :exc:`~urllib.error.URLError` is raised. + hostname is given, a :exc:`~urllib.error.URLError` is raised. .. _data-handler-objects: @@ -1109,7 +1107,7 @@ DataHandler Objects ignores white spaces in base64 encoded data URLs so the URL may be wrapped in whatever source file it comes from. But even though some browsers don't mind about a missing padding at the end of a base64 encoded data URL, this - implementation will raise an :exc:`ValueError` in that case. + implementation will raise a :exc:`ValueError` in that case. .. _ftp-handler-objects: diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index b5a49d9c592387..016fcdc75da67a 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -1,5 +1,5 @@ -:mod:`urllib.robotparser` --- Parser for robots.txt -==================================================== +:mod:`!urllib.robotparser` --- Parser for robots.txt +===================================================== .. module:: urllib.robotparser :synopsis: Load a robots.txt file and answer questions about diff --git a/Doc/library/urllib.rst b/Doc/library/urllib.rst index 624e164625556a..7d9f39ef070fc3 100644 --- a/Doc/library/urllib.rst +++ b/Doc/library/urllib.rst @@ -1,5 +1,5 @@ -:mod:`urllib` --- URL handling modules -====================================== +:mod:`!urllib` --- URL handling modules +======================================= .. module:: urllib diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index e2d231da38fd9a..0f2d7820cb25c8 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -1,5 +1,5 @@ -:mod:`uuid` --- UUID objects according to :rfc:`4122` -===================================================== +:mod:`!uuid` --- UUID objects according to :rfc:`4122` +====================================================== .. module:: uuid :synopsis: UUID objects (universally unique identifiers) according to RFC 4122 diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index aa18873f223a6b..5205c6c211d9bf 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -1,5 +1,5 @@ -:mod:`venv` --- Creation of virtual environments -================================================ +:mod:`!venv` --- Creation of virtual environments +================================================= .. module:: venv :synopsis: Creation of virtual environments. @@ -27,7 +27,7 @@ optionally be isolated from the packages in the base environment, so only those explicitly installed in the virtual environment are available. When used from within a virtual environment, common installation tools such as -`pip`_ will install Python packages into a virtual environment +:pypi:`pip` will install Python packages into a virtual environment without needing to be told to do so explicitly. A virtual environment is (amongst other things): @@ -37,14 +37,14 @@ A virtual environment is (amongst other things): are by default isolated from software in other virtual environments and Python interpreters and libraries installed in the operating system. -* Contained in a directory, conventionally either named ``venv`` or ``.venv`` in +* Contained in a directory, conventionally named ``.venv`` or ``venv`` in the project directory, or under a container directory for lots of virtual environments, such as ``~/.virtualenvs``. * Not checked into source control systems such as Git. * Considered as disposable -- it should be simple to delete and recreate it from - scratch. You don't place any project code in the environment + scratch. You don't place any project code in the environment. * Not considered as movable or copyable -- you just recreate the same environment in the target location. @@ -54,14 +54,134 @@ See :pep:`405` for more background on Python virtual environments. .. seealso:: `Python Packaging User Guide: Creating and using virtual environments - `__ + `__ -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-mobile-notavail.rst Creating virtual environments ----------------------------- -.. include:: /using/venv-create.inc +:ref:`Virtual environments ` are created by executing the ``venv`` +module: + +.. code-block:: shell + + python -m venv /path/to/new/virtual/environment + +This creates the target directory (including parent directories as needed) +and places a :file:`pyvenv.cfg` file in it with a ``home`` key +pointing to the Python installation from which the command was run. +It also creates a :file:`bin` (or :file:`Scripts` on Windows) subdirectory +containing a copy or symlink of the Python executable +(as appropriate for the platform or arguments used at environment creation time). +It also creates a :file:`lib/pythonX.Y/site-packages` subdirectory +(on Windows, this is :file:`Lib\site-packages`). +If an existing directory is specified, it will be re-used. + +.. versionchanged:: 3.5 + The use of ``venv`` is now recommended for creating virtual environments. + +.. deprecated-removed:: 3.6 3.8 + :program:`pyvenv` was the recommended tool for creating virtual environments + for Python 3.3 and 3.4, and replaced in 3.5 by executing ``venv`` directly. + +.. highlight:: none + +On Windows, invoke the ``venv`` command as follows: + +.. code-block:: ps1con + + PS> python -m venv C:\path\to\new\virtual\environment + +The command, if run with ``-h``, will show the available options:: + + usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] + [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] + [--without-scm-ignore-files] + ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + options: + -h, --help show this help message and exit + --system-site-packages + Give the virtual environment access to the system + site-packages dir. + --symlinks Try to use symlinks rather than copies, when + symlinks are not the default for the platform. + --copies Try to use copies rather than symlinks, even when + symlinks are the default for the platform. + --clear Delete the contents of the environment directory + if it already exists, before environment creation. + --upgrade Upgrade the environment directory to use this + version of Python, assuming Python has been + upgraded in-place. + --without-pip Skips installing or upgrading pip in the virtual + environment (pip is bootstrapped by default) + --prompt PROMPT Provides an alternative prompt prefix for this + environment. + --upgrade-deps Upgrade core dependencies (pip) to the latest + version in PyPI + --without-scm-ignore-files + Skips adding SCM ignore files to the environment + directory (Git is supported by default). + + Once an environment has been created, you may wish to activate it, e.g. by + sourcing an activate script in its bin directory. + + +.. versionchanged:: 3.4 + Installs pip by default, added the ``--without-pip`` and ``--copies`` + options. + +.. versionchanged:: 3.4 + In earlier versions, if the target directory already existed, an error was + raised, unless the ``--clear`` or ``--upgrade`` option was provided. + +.. versionchanged:: 3.9 + Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI. + +.. versionchanged:: 3.12 + + ``setuptools`` is no longer a core venv dependency. + +.. versionchanged:: 3.13 + + Added the ``--without-scm-ignore-files`` option. +.. versionchanged:: 3.13 + ``venv`` now creates a :file:`.gitignore` file for Git by default. + +.. note:: + While symlinks are supported on Windows, they are not recommended. Of + particular note is that double-clicking ``python.exe`` in File Explorer + will resolve the symlink eagerly and ignore the virtual environment. + +.. note:: + On Microsoft Windows, it may be required to enable the ``Activate.ps1`` + script by setting the execution policy for the user. You can do this by + issuing the following PowerShell command: + + .. code-block:: powershell + + PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + + See `About Execution Policies + `_ + for more information. + +The created :file:`pyvenv.cfg` file also includes the +``include-system-site-packages`` key, set to ``true`` if ``venv`` is +run with the ``--system-site-packages`` option, ``false`` otherwise. + +Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be +invoked to bootstrap ``pip`` into the virtual environment. + +Multiple paths can be given to ``venv``, in which case an identical virtual +environment will be created, according to the given options, at each provided +path. .. _venv-explanation: @@ -95,7 +215,7 @@ containing the virtual environment): | +------------+--------------------------------------------------+ | | csh/tcsh | :samp:`$ source {}/bin/activate.csh` | | +------------+--------------------------------------------------+ -| | PowerShell | :samp:`$ {}/bin/Activate.ps1` | +| | pwsh | :samp:`$ {}/bin/Activate.ps1` | +-------------+------------+--------------------------------------------------+ | Windows | cmd.exe | :samp:`C:\\> {}\\Scripts\\activate.bat` | | +------------+--------------------------------------------------+ @@ -117,7 +237,7 @@ should be runnable without activating it. In order to achieve this, scripts installed into virtual environments have a "shebang" line which points to the environment's Python interpreter, -i.e. :samp:`#!/{}/bin/python`. +:samp:`#!/{}/bin/python`. This means that the script will run with that interpreter regardless of the value of :envvar:`PATH`. On Windows, "shebang" line processing is supported if you have the :ref:`launcher` installed. Thus, double-clicking an installed @@ -168,31 +288,31 @@ creation according to their needs, the :class:`EnvBuilder` class. The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: - * ``system_site_packages`` -- a Boolean value indicating that the system Python + * *system_site_packages* -- a boolean value indicating that the system Python site-packages should be available to the environment (defaults to ``False``). - * ``clear`` -- a Boolean value which, if true, will delete the contents of + * *clear* -- a boolean value which, if true, will delete the contents of any existing target directory, before creating the environment. - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the + * *symlinks* -- a boolean value indicating whether to attempt to symlink the Python binary rather than copying. - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing + * *upgrade* -- a boolean value which, if true, will upgrade an existing environment with the running Python - for use when that Python has been upgraded in-place (defaults to ``False``). - * ``with_pip`` -- a Boolean value which, if true, ensures pip is + * *with_pip* -- a boolean value which, if true, ensures pip is installed in the virtual environment. This uses :mod:`ensurepip` with the ``--default-pip`` option. - * ``prompt`` -- a String to be used after virtual environment is activated + * *prompt* -- a string to be used after virtual environment is activated (defaults to ``None`` which means directory name of the environment would be used). If the special string ``"."`` is provided, the basename of the current directory is used as the prompt. - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + * *upgrade_deps* -- Update the base venv modules to the latest on PyPI - * ``scm_ignore_files`` -- Create ignore files based for the specified source + * *scm_ignore_files* -- Create ignore files based for the specified source control managers (SCM) in the iterable. Support is defined by having a method named ``create_{scm}_ignore_file``. The only value supported by default is ``"git"`` via :meth:`create_git_ignore_file`. @@ -210,10 +330,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionchanged:: 3.13 Added the ``scm_ignore_files`` parameter - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: + :class:`EnvBuilder` may be used as a base class. .. method:: create(env_dir) @@ -286,15 +403,15 @@ creation according to their needs, the :class:`EnvBuilder` class. the virtual environment. - .. versionchanged:: 3.12 - The attribute ``lib_path`` was added to the context, and the context - object was documented. - .. versionchanged:: 3.11 The *venv* :ref:`sysconfig installation scheme ` is used to construct the paths of the created directories. + .. versionchanged:: 3.12 + The attribute ``lib_path`` was added to the context, and the context + object was documented. + .. method:: create_configuration(context) Creates the ``pyvenv.cfg`` configuration file in the environment. @@ -313,14 +430,14 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: upgrade_dependencies(context) - Upgrades the core venv dependency packages (currently ``pip``) + Upgrades the core venv dependency packages (currently :pypi:`pip`) in the environment. This is done by shelling out to the ``pip`` executable in the environment. .. versionadded:: 3.9 .. versionchanged:: 3.12 - ``setuptools`` is no longer a core venv dependency. + :pypi:`setuptools` is no longer a core venv dependency. .. method:: post_setup(context) @@ -328,25 +445,15 @@ creation according to their needs, the :class:`EnvBuilder` class. implementations to pre-install packages in the virtual environment or perform other post-creation steps. - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - .. method:: install_scripts(context, path) + This method can be + called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to + assist in installing custom scripts into the virtual environment. + *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the + ``common``, ``posix``, ``nt``; each containing scripts destined for the + ``bin`` directory in the environment. The contents of ``common`` and the directory corresponding to :data:`os.name` are copied after some text replacement of placeholders: @@ -371,10 +478,20 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: create_git_ignore_file(context) Creates a ``.gitignore`` file within the virtual environment that causes - the entire directory to be ignored by the ``git`` source control manager. + the entire directory to be ignored by the Git source control manager. .. versionadded:: 3.13 + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. + There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ @@ -387,16 +504,16 @@ There is also a module-level convenience function: .. versionadded:: 3.3 .. versionchanged:: 3.4 - Added the ``with_pip`` parameter + Added the *with_pip* parameter .. versionchanged:: 3.6 - Added the ``prompt`` parameter + Added the *prompt* parameter .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter + Added the *upgrade_deps* parameter .. versionchanged:: 3.13 - Added the ``scm_ignore_files`` parameter + Added the *scm_ignore_files* parameter An example of extending ``EnvBuilder`` -------------------------------------- @@ -614,7 +731,3 @@ subclass which installs setuptools and pip into a created virtual environment:: This script is also available for download `online `_. - - -.. _setuptools: https://pypi.org/project/setuptools/ -.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 500398636e11ae..1e7c4c8b915e7e 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -1,5 +1,5 @@ -:mod:`warnings` --- Warning control -=================================== +:mod:`!warnings` --- Warning control +==================================== .. module:: warnings :synopsis: Issue warning messages and control their disposition. @@ -145,6 +145,8 @@ the disposition of the match. Each entry is a tuple of the form (*action*, +---------------+----------------------------------------------+ | ``"always"`` | always print matching warnings | +---------------+----------------------------------------------+ + | ``"all"`` | alias to "always" | + +---------------+----------------------------------------------+ | ``"module"`` | print the first occurrence of matching | | | warnings for each module where the warning | | | is issued (regardless of line number) | @@ -396,7 +398,7 @@ Available Functions ------------------- -.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=None) +.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=()) Issue a warning, or maybe ignore it or raise an exception. The *category* argument, if given, must be a :ref:`warning category class `; it @@ -595,6 +597,9 @@ Available Context Managers passed to :func:`simplefilter` as if it were called immediately on entering the context. + See :ref:`warning-filter` for the meaning of the *category* and *lineno* + parameters. + .. note:: The :class:`catch_warnings` manager works by replacing and diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index 55b029bc742b24..36c2bde87fb8fb 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -1,5 +1,5 @@ -:mod:`wave` --- Read and write WAV files -======================================== +:mod:`!wave` --- Read and write WAV files +========================================= .. module:: wave :synopsis: Provide an interface to the WAV sound format. @@ -46,8 +46,8 @@ The :mod:`wave` module defines the following function and exception: the file object. The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`!with` block completes, the :meth:`Wave_read.close()` or - :meth:`Wave_write.close()` method is called. + the :keyword:`!with` block completes, the :meth:`Wave_read.close` or + :meth:`Wave_write.close` method is called. .. versionchanged:: 3.4 Added support for unseekable files. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index d6e062df945c64..2a25ed045c68bd 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -197,7 +197,7 @@ See :ref:`__slots__ documentation ` for details. >>> del k1 # d = {k2: 2} .. versionchanged:: 3.9 - Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. + Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`. :class:`WeakKeyDictionary` objects have an additional method that exposes the internal references directly. The references are not guaranteed to diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 4667b81e38ada2..2d19c514ce43b6 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -1,5 +1,5 @@ -:mod:`webbrowser` --- Convenient web-browser controller -======================================================= +:mod:`!webbrowser` --- Convenient web-browser controller +======================================================== .. module:: webbrowser :synopsis: Easy-to-use controller for web browsers. @@ -33,15 +33,25 @@ allow the remote browser to maintain its own windows on the display. If remote browsers are not available on Unix, the controlling process will launch a new browser and wait. +On iOS, the :envvar:`BROWSER` environment variable, as well as any arguments +controlling autoraise, browser preference, and new tab/window creation will be +ignored. Web pages will *always* be opened in the user's preferred browser, in +a new tab, with the browser being brought to the foreground. The use of the +:mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If +:mod:`ctypes` isn't available, calls to :func:`.open` will fail. + The script :program:`webbrowser` can be used as a command-line interface for the module. It accepts a URL as the argument. It accepts the following optional -parameters: ``-n`` opens the URL in a new browser window, if possible; -``-t`` opens the URL in a new browser page ("tab"). The options are, -naturally, mutually exclusive. Usage example:: +parameters: + +* ``-n``/``--new-window`` opens the URL in a new browser window, if possible. +* ``-t``/``--new-tab`` opens the URL in a new browser page ("tab"). + +The options are, naturally, mutually exclusive. Usage example:: python -m webbrowser -t "https://www.python.org" -.. include:: ../includes/wasm-notavail.rst +.. availability:: not WASI, not Android. The following exception is defined: @@ -62,6 +72,8 @@ The following functions are defined: (note that under many window managers this will occur regardless of the setting of this variable). + Returns ``True`` if a browser was successfully launched, ``False`` otherwise. + Note that on some platforms, trying to open a filename using this function, may work and start the operating system's associated program. However, this is neither supported nor portable. @@ -74,11 +86,16 @@ The following functions are defined: Open *url* in a new window of the default browser, if possible, otherwise, open *url* in the only browser window. + Returns ``True`` if a browser was successfully launched, ``False`` otherwise. + + .. function:: open_new_tab(url) Open *url* in a new page ("tab") of the default browser, if possible, otherwise equivalent to :func:`open_new`. + Returns ``True`` if a browser was successfully launched, ``False`` otherwise. + .. function:: get(using=None) @@ -147,6 +164,8 @@ for the controller classes, all defined in this module. +------------------------+-----------------------------------------+-------+ | ``'chromium-browser'`` | ``Chromium('chromium-browser')`` | | +------------------------+-----------------------------------------+-------+ +| ``'iosbrowser'`` | ``IOSBrowser`` | \(4) | ++------------------------+-----------------------------------------+-------+ Notes: @@ -161,7 +180,10 @@ Notes: Only on Windows platforms. (3) - Only on macOS platform. + Only on macOS. + +(4) + Only on iOS. .. versionadded:: 3.2 A new :class:`!MacOSXOSAScript` class has been added @@ -176,6 +198,9 @@ Notes: Removed browsers include Grail, Mosaic, Netscape, Galeon, Skipstone, Iceape, and Firefox versions 35 and below. +.. versionchanged:: 3.13 + Support for iOS has been added. + Here are some simple examples:: url = 'https://docs.python.org/' diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index 06bd4d87eb03c6..b3a824fb69a49f 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -1,5 +1,5 @@ -:mod:`winreg` --- Windows registry access -========================================= +:mod:`!winreg` --- Windows registry access +========================================== .. module:: winreg :platform: Windows diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 370c5216652ba7..f7ca9dc57bbe28 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -1,5 +1,5 @@ -:mod:`winsound` --- Sound-playing interface for Windows -======================================================= +:mod:`!winsound` --- Sound-playing interface for Windows +======================================================== .. module:: winsound :platform: Windows diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index c2b0ba7046967e..8d4c5eb6600e8b 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -1,5 +1,5 @@ -:mod:`wsgiref` --- WSGI Utilities and Reference Implementation -============================================================== +:mod:`!wsgiref` --- WSGI Utilities and Reference Implementation +=============================================================== .. module:: wsgiref :synopsis: WSGI Utilities and Reference Implementation. @@ -783,8 +783,8 @@ in :pep:`3333`. .. class:: StartResponse() - A :class:`typing.Protocol` describing `start_response() - `_ + A :class:`typing.Protocol` describing :pep:`start_response() + <3333#the-start-response-callable>` callables (:pep:`3333`). .. data:: WSGIEnvironment @@ -797,18 +797,18 @@ in :pep:`3333`. .. class:: InputStream() - A :class:`typing.Protocol` describing a `WSGI Input Stream - `_. + A :class:`typing.Protocol` describing a :pep:`WSGI Input Stream + <3333#input-and-error-streams>`. .. class:: ErrorStream() - A :class:`typing.Protocol` describing a `WSGI Error Stream - `_. + A :class:`typing.Protocol` describing a :pep:`WSGI Error Stream + <3333#input-and-error-streams>`. .. class:: FileWrapper() - A :class:`typing.Protocol` describing a `file wrapper - `_. + A :class:`typing.Protocol` describing a :pep:`file wrapper + <3333#optional-platform-specific-file-handling>`. See :class:`wsgiref.util.FileWrapper` for a concrete implementation of this protocol. @@ -865,7 +865,7 @@ directory and port number (default: 8000) on the command line:: fn = os.path.join(path, environ["PATH_INFO"][1:]) if "." not in fn.split(os.path.sep)[-1]: fn = os.path.join(fn, "index.html") - mime_type = mimetypes.guess_type(fn)[0] + mime_type = mimetypes.guess_file_type(fn)[0] # Return 200 OK if file exists, otherwise 404 Not Found if os.path.exists(fn): diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 72a7a98c2ac4f2..00a18751207e7a 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom.minidom` --- Minimal DOM implementation -===================================================== +:mod:`!xml.dom.minidom` --- Minimal DOM implementation +====================================================== .. module:: xml.dom.minidom :synopsis: Minimal Document Object Model (DOM) implementation. diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index 843c2fd7fdb937..fd96765cbe3c96 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom.pulldom` --- Support for building partial DOM trees -================================================================= +:mod:`!xml.dom.pulldom` --- Support for building partial DOM trees +================================================================== .. module:: xml.dom.pulldom :synopsis: Support for building partial DOM trees from SAX events. diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index d0e1b248d595d1..f33b19bc2724d0 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom` --- The Document Object Model API -================================================ +:mod:`!xml.dom` --- The Document Object Model API +================================================= .. module:: xml.dom :synopsis: Document Object Model API for Python. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 75a7915c15240d..1daf6628013bf0 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1,5 +1,5 @@ -:mod:`xml.etree.ElementTree` --- The ElementTree XML API -======================================================== +:mod:`!xml.etree.ElementTree` --- The ElementTree XML API +========================================================= .. module:: xml.etree.ElementTree :synopsis: Implementation of the ElementTree API. @@ -49,7 +49,7 @@ and its sub-elements are done on the :class:`Element` level. Parsing XML ^^^^^^^^^^^ -We'll be using the following XML document as the sample data for this section: +We'll be using the fictive :file:`country_data.xml` XML document as the sample data for this section: .. code-block:: xml @@ -166,6 +166,11 @@ data but would still like to have incremental parsing capabilities, take a look at :func:`iterparse`. It can be useful when you're reading a large XML document and don't want to hold it wholly in memory. +Where *immediate* feedback through events is wanted, calling method +:meth:`XMLPullParser.flush` can help reduce delay; +please make sure to study the related security notes. + + Finding interesting elements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -503,7 +508,7 @@ Functions `C14N 2.0 `_ transformation function. Canonicalization is a way to normalise XML output in a way that allows - byte-by-byte comparisons and digital signatures. It reduced the freedom + byte-by-byte comparisons and digital signatures. It reduces the freedom that XML serializers have and instead generates a more constrained XML representation. The main restrictions regard the placement of namespace declarations, the ordering of attributes, and ignorable whitespace. @@ -835,33 +840,28 @@ Functions .. module:: xml.etree.ElementInclude -.. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None) - :module: +.. function:: default_loader(href, parse, encoding=None) - Default loader. This default loader reads an included resource from disk. *href* is a URL. - *parse* is for parse mode either "xml" or "text". *encoding* - is an optional text encoding. If not given, encoding is ``utf-8``. Returns the - expanded resource. If the parse mode is ``"xml"``, this is an ElementTree - instance. If the parse mode is "text", this is a Unicode string. If the - loader fails, it can return None or raise an exception. + Default loader. This default loader reads an included resource from disk. + *href* is a URL. *parse* is for parse mode either "xml" or "text". + *encoding* is an optional text encoding. If not given, encoding is ``utf-8``. + Returns the expanded resource. + If the parse mode is ``"xml"``, this is an :class:`~xml.etree.ElementTree.Element` instance. + If the parse mode is ``"text"``, this is a string. + If the loader fails, it can return ``None`` or raise an exception. -.. function:: xml.etree.ElementInclude.include( elem, loader=None, base_url=None, \ - max_depth=6) - :module: +.. function:: include(elem, loader=None, base_url=None, max_depth=6) - This function expands XInclude directives. *elem* is the root element. *loader* is - an optional resource loader. If omitted, it defaults to :func:`default_loader`. + This function expands XInclude directives in-place in tree pointed by *elem*. + *elem* is either the root :class:`~xml.etree.ElementTree.Element` or an + :class:`~xml.etree.ElementTree.ElementTree` instance to find such element. + *loader* is an optional resource loader. If omitted, it defaults to :func:`default_loader`. If given, it should be a callable that implements the same interface as :func:`default_loader`. *base_url* is base URL of the original file, to resolve relative include file references. *max_depth* is the maximum number of recursive - inclusions. Limited to reduce the risk of malicious content explosion. Pass a - negative value to disable the limitation. - - Returns the expanded resource. If the parse mode is - ``"xml"``, this is an ElementTree instance. If the parse mode is "text", - this is a Unicode string. If the loader fails, it can return None or - raise an exception. + inclusions. Limited to reduce the risk of malicious content explosion. + Pass ``None`` to disable the limitation. .. versionchanged:: 3.9 Added the *base_url* and *max_depth* parameters. @@ -874,6 +874,7 @@ Element Objects .. module:: xml.etree.ElementTree :noindex: + :no-index: .. class:: Element(tag, attrib={}, **extra) @@ -970,7 +971,7 @@ Element Objects .. method:: extend(subelements) - Appends *subelements* from a sequence object with zero or more elements. + Appends *subelements* from an iterable of elements. Raises :exc:`TypeError` if a subelement is not an :class:`Element`. .. versionadded:: 3.2 @@ -1058,9 +1059,10 @@ Element Objects :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__len__`. - Caution: Elements with no subelements will test as ``False``. Testing the - truth value of an Element is deprecated and will raise an exception in - Python 3.14. Use specific ``len(elem)`` or ``elem is None`` test instead.:: + Caution: Elements with no subelements will test as ``False``. In a future + release of Python, all elements will test as ``True`` regardless of whether + subelements exist. Instead, prefer explicit ``len(elem)`` or + ``elem is not None`` tests.:: element = root.find('foo') @@ -1373,7 +1375,7 @@ XMLParser Objects .. versionchanged:: 3.8 Parameters are now :ref:`keyword-only `. - The *html* argument no longer supported. + The *html* argument is no longer supported. .. method:: close() @@ -1387,6 +1389,24 @@ XMLParser Objects Feeds data to the parser. *data* is encoded data. + + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + Note that :meth:`flush` has been backported to some prior releases of + CPython as a security fix. Check for availability of :meth:`flush` + using :func:`hasattr` if used in code running across a variety of Python + versions. + + .. versionadded:: 3.13 + + :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method for each opening tag, its ``end(tag)`` method for each closing tag, and data is processed by method ``data(data)``. For further supported callback @@ -1448,6 +1468,22 @@ XMLPullParser Objects Feed the given bytes data to the parser. + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + Note that :meth:`flush` has been backported to some prior releases of + CPython as a security fix. Check for availability of :meth:`flush` + using :func:`hasattr` if used in code running across a variety of Python + versions. + + .. versionadded:: 3.13 + .. method:: close() Signal the parser that the data stream is terminated. Unlike diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 909022ea4ba6a4..d495995398959d 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -68,6 +68,7 @@ quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable* external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) `DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe decompression bomb Safe Safe Safe Safe **Vulnerable** +large tokens **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) ========================= ================== ================== ================== ================== ================== 1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and @@ -81,6 +82,11 @@ decompression bomb Safe Safe Safe 4. :mod:`xmlrpc.client` doesn't expand external entities and omits them. 5. Since Python 3.7.1, external general entities are no longer processed by default. +6. Expat 2.6.0 and newer is not vulnerable to denial of service + through quadratic runtime caused by parsing large tokens. + Items still listed as vulnerable due to + potential reliance on system-provided libraries. Check + :const:`!pyexpat.EXPAT_VERSION`. billion laughs / exponential entity expansion @@ -114,7 +120,13 @@ decompression bomb files. For an attacker it can reduce the amount of transmitted data by three magnitudes or more. -The documentation for `defusedxml`_ on PyPI has further information about +large tokens + Expat needs to re-parse unfinished tokens; without the protection + introduced in Expat 2.6.0, this can lead to quadratic runtime that can + be used to cause denial of service in the application parsing XML. + The issue is known as :cve:`2023-52425`. + +The documentation for :pypi:`defusedxml` on PyPI has further information about all known attack vectors with examples and references. .. _defusedxml-package: @@ -122,14 +134,13 @@ all known attack vectors with examples and references. The :mod:`!defusedxml` Package ------------------------------ -`defusedxml`_ is a pure Python package with modified subclasses of all stdlib +:pypi:`defusedxml` is a pure Python package with modified subclasses of all stdlib XML parsers that prevent any potentially malicious operation. Use of this package is recommended for any server code that parses untrusted XML data. The package also ships with example exploits and extended documentation on more XML exploits such as XPath injection. -.. _defusedxml: https://pypi.org/project/defusedxml/ .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb .. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index e2f28e3244cb09..c2c9d6424b5072 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.handler` --- Base classes for SAX handlers -======================================================== +:mod:`!xml.sax.handler` --- Base classes for SAX handlers +========================================================= .. module:: xml.sax.handler :synopsis: Base classes for SAX event handlers. diff --git a/Doc/library/xml.sax.reader.rst b/Doc/library/xml.sax.reader.rst index 113e9e93fb04ff..b0bc84062e0719 100644 --- a/Doc/library/xml.sax.reader.rst +++ b/Doc/library/xml.sax.reader.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.xmlreader` --- Interface for XML parsers -====================================================== +:mod:`!xml.sax.xmlreader` --- Interface for XML parsers +======================================================= .. module:: xml.sax.xmlreader :synopsis: Interface which SAX-compliant XML parsers must implement. diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index 6d351dfb4d7072..c60e9e505f7544 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax` --- Support for SAX2 parsers -=========================================== +:mod:`!xml.sax` --- Support for SAX2 parsers +============================================ .. module:: xml.sax :synopsis: Package containing SAX2 base classes and convenience functions. diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index 3a524c9c0d5a9f..5ee11d58c3dd26 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.saxutils` --- SAX Utilities -========================================= +:mod:`!xml.sax.saxutils` --- SAX Utilities +========================================== .. module:: xml.sax.saxutils :synopsis: Convenience functions and classes for use with SAX. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index f7f23007fb0522..c57f433e6efd98 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -1,5 +1,5 @@ -:mod:`xmlrpc.client` --- XML-RPC client access -============================================== +:mod:`!xmlrpc.client` --- XML-RPC client access +=============================================== .. module:: xmlrpc.client :synopsis: XML-RPC client access. @@ -165,7 +165,7 @@ between conformable Python objects and XML on the wire. A good description of XML-RPC operation and client software in several languages. Contains pretty much everything an XML-RPC client developer needs to know. - `XML-RPC Introspection `_ + `XML-RPC Introspection `_ Describes the XML-RPC protocol extension for introspection. `XML-RPC Specification `_ diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index ca1ea455f0acfc..06169c7eca8b0c 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -1,5 +1,5 @@ -:mod:`xmlrpc.server` --- Basic XML-RPC servers -============================================== +:mod:`!xmlrpc.server` --- Basic XML-RPC servers +=============================================== .. module:: xmlrpc.server :synopsis: Basic XML-RPC server implementations. diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index c8a059bdb1cb93..cdaba07ab46c8f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -1,5 +1,5 @@ -:mod:`zipapp` --- Manage executable Python zip archives -======================================================= +:mod:`!zipapp` --- Manage executable Python zip archives +======================================================== .. module:: zipapp :synopsis: Manage executable Python zip archives @@ -332,7 +332,7 @@ Formally, the Python zip application format is therefore: interpreter name, and then a newline (``b'\n'``) character. The interpreter name can be anything acceptable to the OS "shebang" processing, or the Python launcher on Windows. The interpreter should be encoded in UTF-8 on Windows, - and in :func:`sys.getfilesystemencoding()` on POSIX. + and in :func:`sys.getfilesystemencoding` on POSIX. 2. Standard zipfile data, as generated by the :mod:`zipfile` module. The zipfile content *must* include a file called ``__main__.py`` (which must be in the "root" of the zipfile - i.e., it cannot be in a subdirectory). The diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index b6f881fd2dfd70..5583c6b24be5c6 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -1,5 +1,5 @@ -:mod:`zipfile` --- Work with ZIP archives -========================================= +:mod:`!zipfile` --- Work with ZIP archives +========================================== .. module:: zipfile :synopsis: Read and write ZIP-format archive files. @@ -301,6 +301,10 @@ ZipFile Objects attempting to read or write other files in the ZIP file will raise a :exc:`ValueError`. + In both cases the file-like object has also attributes :attr:`!name`, + which is equivalent to the name of a file within the archive, and + :attr:`!mode`, which is ``'rb'`` or ``'wb'`` depending on the input mode. + When writing a file, if the file size is not known in advance but may exceed 2 GiB, pass ``force_zip64=True`` to ensure that the header format is capable of supporting large files. If the file size is known in advance, @@ -325,6 +329,12 @@ ZipFile Objects Calling :meth:`.open` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.13 + Added attributes :attr:`!name` and :attr:`!mode` for the writeable + file-like object. + The value of the :attr:`!mode` attribute for the readable file-like + object was changed from ``'r'`` to ``'rb'``. + .. method:: ZipFile.extract(member, path=None, pwd=None) @@ -575,6 +585,15 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. Return ``True`` if the current context references a file. +.. method:: Path.is_symlink() + + Return ``True`` if the current context references a symbolic link. + + .. versionadded:: 3.12 + + .. versionchanged:: 3.13 + Previously, ``is_symlink`` would unconditionally return ``False``. + .. method:: Path.exists() Return ``True`` if the current context references a file or @@ -632,7 +651,7 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. Prior to 3.10, ``joinpath`` was undocumented and accepted exactly one parameter. -The `zipp `_ project provides backports +The :pypi:`zipp` project provides backports of the latest path object functionality to older Pythons. Use ``zipp.Path`` in place of ``zipfile.Path`` for early access to changes. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 47c81f0e63603d..9353a45bdcecba 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -1,5 +1,5 @@ -:mod:`zipimport` --- Import modules from Zip archives -===================================================== +:mod:`!zipimport` --- Import modules from Zip archives +====================================================== .. module:: zipimport :synopsis: Support for importing Python modules from ZIP archives. @@ -30,6 +30,9 @@ Any files may be present in the ZIP archive, but importers are only invoked for corresponding :file:`.pyc` file, meaning that if a ZIP archive doesn't contain :file:`.pyc` files, importing may be rather slow. +.. versionchanged:: 3.13 + ZIP64 is supported + .. versionchanged:: 3.8 Previously, ZIP archives with an archive comment were not supported. diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index ac179722dee2be..965b82a3daffb9 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -1,5 +1,5 @@ -:mod:`zlib` --- Compression compatible with :program:`gzip` -=========================================================== +:mod:`!zlib` --- Compression compatible with :program:`gzip` +============================================================ .. module:: zlib :synopsis: Low-level interface to compression and decompression routines diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index f8624da6e51dbb..a57f3b8b3e858c 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -1,5 +1,5 @@ -:mod:`zoneinfo` --- IANA time zone support -========================================== +:mod:`!zoneinfo` --- IANA time zone support +=========================================== .. module:: zoneinfo :synopsis: IANA time zone support @@ -17,7 +17,7 @@ The :mod:`zoneinfo` module provides a concrete time zone implementation to support the IANA time zone database as originally specified in :pep:`615`. By default, :mod:`zoneinfo` uses the system's time zone data if available; if no system time zone data is available, the library will fall back to using the -first-party `tzdata`_ package available on PyPI. +first-party :pypi:`tzdata` package available on PyPI. .. seealso:: @@ -25,7 +25,7 @@ first-party `tzdata`_ package available on PyPI. Provides the :class:`~datetime.time` and :class:`~datetime.datetime` types with which the :class:`ZoneInfo` class is designed to be used. - Package `tzdata`_ + Package :pypi:`tzdata` First-party package maintained by the CPython core developers to supply time zone data via PyPI. @@ -93,7 +93,7 @@ Data sources The ``zoneinfo`` module does not directly provide time zone data, and instead pulls time zone information from the system time zone database or the -first-party PyPI package `tzdata`_, if available. Some systems, including +first-party PyPI package :pypi:`tzdata`, if available. Some systems, including notably Windows systems, do not have an IANA database available, and so for projects targeting cross-platform compatibility that require time zone data, it is recommended to declare a dependency on tzdata. If neither system data nor @@ -413,5 +413,3 @@ Exceptions and warnings be filtered out, such as a relative path. .. Links and references: - -.. _tzdata: https://pypi.org/project/tzdata/ diff --git a/Doc/license.rst b/Doc/license.rst index 9fc0ff7161a591..674ac5f56e6f97 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -1042,30 +1042,32 @@ https://www.w3.org/TR/xml-c14n2-testcases/ and is distributed under the OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +.. _mimalloc-license: + mimalloc -------- -MIT License +MIT License:: -Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen + Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. asyncio @@ -1095,3 +1097,35 @@ which is distributed under the MIT license:: LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Global Unbounded Sequences (GUS) +-------------------------------- + +The file :file:`Python/qsbr.c` is adapted from FreeBSD's "Global Unbounded +Sequences" safe memory reclamation scheme in +`subr_smr.c `_. +The file is distributed under the 2-Clause BSD License:: + + Copyright (c) 2019,2020 Jeffrey Roberson + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice unmodified, this list of conditions, and the following + disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 374404bf33abbe..1b1e9f479cbe08 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -245,13 +245,12 @@ handler is started. This search inspects the :keyword:`!except` clauses in turn until one is found that matches the exception. An expression-less :keyword:`!except` clause, if present, must be last; it matches any exception. -For an :keyword:`!except` clause with an expression, -that expression is evaluated, and the clause matches the exception -if the resulting object is "compatible" with the exception. An object is -compatible with an exception if the object is the class or a -:term:`non-virtual base class ` of the exception object, -or a tuple containing an item that is the class or a non-virtual base class -of the exception object. + +For an :keyword:`!except` clause with an expression, the +expression must evaluate to an exception type or a tuple of exception types. +The raised exception matches an :keyword:`!except` clause whose expression evaluates +to the class or a :term:`non-virtual base class ` of the exception object, +or to a tuple that contains such a class. If no :keyword:`!except` clause matches the exception, the search for an exception handler @@ -378,8 +377,10 @@ exception group with an empty message string. :: ... ExceptionGroup('', (BlockingIOError())) -An :keyword:`!except*` clause must have a matching type, -and this type cannot be a subclass of :exc:`BaseExceptionGroup`. +An :keyword:`!except*` clause must have a matching expression; it cannot be ``except*:``. +Furthermore, this expression cannot contain exception group types, because that would +have ambiguous semantics. + It is not possible to mix :keyword:`except` and :keyword:`!except*` in the same :keyword:`try`. :keyword:`break`, :keyword:`continue` and :keyword:`return` @@ -840,7 +841,7 @@ A literal pattern corresponds to most : | "None" : | "True" : | "False" - : | `signed_number`: NUMBER | "-" NUMBER + signed_number: ["-"] NUMBER The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are @@ -1216,9 +1217,10 @@ A function definition defines a user-defined function object (see section : | `parameter_list_no_posonly` parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] : | `parameter_list_starargs` - parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] + parameter_list_starargs: "*" [`star_parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] + star_parameter: `identifier` [":" ["*"] `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` @@ -1325,16 +1327,15 @@ and may only be passed by positional arguments. Parameters may have an :term:`annotation ` of the form "``: expression``" following the parameter name. Any parameter may have an annotation, even those of the form -``*identifier`` or ``**identifier``. Functions may have "return" annotation of +``*identifier`` or ``**identifier``. (As a special case, parameters of the form +``*identifier`` may have an annotation "``: *expression``".) Functions may have "return" annotation of the form "``-> expression``" after the parameter list. These annotations can be any valid Python expression. The presence of annotations does not change the -semantics of a function. The annotation values are available as values of -a dictionary keyed by the parameters' names in the :attr:`__annotations__` -attribute of the function object. If the ``annotations`` import from -:mod:`__future__` is used, annotations are preserved as strings at runtime which -enables postponed evaluation. Otherwise, they are evaluated when the function -definition is executed. In this case annotations may be evaluated in -a different order than they appear in the source code. +semantics of a function. See :ref:`annotations` for more information on annotations. + +.. versionchanged:: 3.11 + Parameters of the form "``*identifier``" may have an annotation + "``: *expression``". See :pep:`646`. .. index:: pair: lambda; expression @@ -1421,7 +1422,7 @@ dictionary. The class name is bound to this class object in the original local namespace. The order in which attributes are defined in the class body is preserved -in the new class's ``__dict__``. Note that this is reliable only right +in the new class's :attr:`~type.__dict__`. Note that this is reliable only right after the class is created and only for classes that were defined using the definition syntax. @@ -1452,8 +1453,8 @@ decorators. The result is then bound to the class name. A list of :ref:`type parameters ` may be given in square brackets immediately after the class's name. This indicates to static type checkers that the class is generic. At runtime, -the type parameters can be retrieved from the class's ``__type_params__`` -attribute. See :ref:`generic-classes` for more. +the type parameters can be retrieved from the class's +:attr:`~type.__type_params__` attribute. See :ref:`generic-classes` for more. .. versionchanged:: 3.12 Type parameter lists are new in Python 3.12. @@ -1620,15 +1621,18 @@ Type parameter lists .. versionadded:: 3.12 +.. versionchanged:: 3.13 + Support for default values was added (see :pep:`696`). + .. index:: single: type parameters .. productionlist:: python-grammar type_params: "[" `type_param` ("," `type_param`)* "]" type_param: `typevar` | `typevartuple` | `paramspec` - typevar: `identifier` (":" `expression`)? - typevartuple: "*" `identifier` - paramspec: "**" `identifier` + typevar: `identifier` (":" `expression`)? ("=" `expression`)? + typevartuple: "*" `identifier` ("=" `expression`)? + paramspec: "**" `identifier` ("=" `expression`)? :ref:`Functions ` (including :ref:`coroutines `), :ref:`classes ` and :ref:`type aliases ` may @@ -1663,8 +1667,8 @@ with more precision. The scope of type parameters is modeled with a special function (technically, an :ref:`annotation scope `) that wraps the creation of the generic object. -Generic functions, classes, and type aliases have a :attr:`!__type_params__` -attribute listing their type parameters. +Generic functions, classes, and type aliases have a +:attr:`~definition.__type_params__` attribute listing their type parameters. Type parameters come in three kinds: @@ -1694,19 +1698,31 @@ evaluated in a separate :ref:`annotation scope `. :data:`typing.TypeVarTuple`\ s and :data:`typing.ParamSpec`\ s cannot have bounds or constraints. +All three flavors of type parameters can also have a *default value*, which is used +when the type parameter is not explicitly provided. This is added by appending +a single equals sign (``=``) followed by an expression. Like the bounds and +constraints of type variables, the default value is not evaluated when the +object is created, but only when the type parameter's ``__default__`` attribute +is accessed. To this end, the default value is evaluated in a separate +:ref:`annotation scope `. If no default value is specified +for a type parameter, the ``__default__`` attribute is set to the special +sentinel object :data:`typing.NoDefault`. + The following example indicates the full set of allowed type parameter declarations:: def overly_generic[ SimpleTypeVar, + TypeVarWithDefault = int, TypeVarWithBound: int, TypeVarWithConstraints: (str, bytes), - *SimpleTypeVarTuple, - **SimpleParamSpec, + *SimpleTypeVarTuple = (int, float), + **SimpleParamSpec = (str, bytearray), ]( a: SimpleTypeVar, - b: TypeVarWithBound, - c: Callable[SimpleParamSpec, TypeVarWithConstraints], - *d: SimpleTypeVarTuple, + b: TypeVarWithDefault, + c: TypeVarWithBound, + d: Callable[SimpleParamSpec, TypeVarWithConstraints], + *e: SimpleTypeVarTuple, ): ... .. _generic-functions: @@ -1836,6 +1852,44 @@ Here, ``annotation-def`` (not a real keyword) indicates an :ref:`annotation scope `. The capitalized names like ``TYPE_PARAMS_OF_ListOrSet`` are not actually bound at runtime. +.. _annotations: + +Annotations +=========== + +.. versionchanged:: 3.14 + Annotations are now lazily evaluated by default. + +Variables and function parameters may carry :term:`annotations `, +created by adding a colon after the name, followed by an expression:: + + x: annotation = 1 + def f(param: annotation): ... + +Functions may also carry a return annotation following an arrow:: + + def f() -> annotation: ... + +Annotations are conventionally used for :term:`type hints `, but this +is not enforced by the language, and in general annotations may contain arbitrary +expressions. The presence of annotations does not change the runtime semantics of +the code, except if some mechanism is used that introspects and uses the annotations +(such as :mod:`dataclasses` or :func:`functools.singledispatch`). + +By default, annotations are lazily evaluated in a :ref:`annotation scope `. +This means that they are not evaluated when the code containing the annotation is evaluated. +Instead, the interpreter saves information that can be used to evaluate the annotation later +if requested. The :mod:`annotationlib` module provides tools for evaluating annotations. + +If the :ref:`future statement ` ``from __future__ import annotations`` is present, +all annotations are instead stored as strings:: + + >>> from __future__ import annotations + >>> def f(param: annotation): ... + >>> f.__annotations__ + {'param': 'annotation'} + + .. rubric:: Footnotes .. [#] The exception is propagated to the invocation stack unless @@ -1876,5 +1930,5 @@ like ``TYPE_PARAMS_OF_ListOrSet`` are not actually bound at runtime. therefore the function's :term:`docstring`. .. [#] A string literal appearing as the first statement in the class body is - transformed into the namespace's ``__doc__`` item and therefore the class's - :term:`docstring`. + transformed into the namespace's :attr:`~type.__doc__` item and therefore + the class's :term:`docstring`. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 885ee825c12296..dfd1addf656a85 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -34,7 +34,7 @@ represented by objects.) Every object has an identity, a type and a value. An object's *identity* never changes once it has been created; you may think of it as the object's address in -memory. The ':keyword:`is`' operator compares the identity of two objects; the +memory. The :keyword:`is` operator compares the identity of two objects; the :func:`id` function returns an integer representing its identity. .. impl-detail:: @@ -81,7 +81,7 @@ are still reachable. Note that the use of the implementation's tracing or debugging facilities may keep objects alive that would normally be collectable. Also note that catching -an exception with a ':keyword:`try`...\ :keyword:`except`' statement may keep +an exception with a :keyword:`try`...\ :keyword:`except` statement may keep objects alive. Some objects contain references to "external" resources such as open files or @@ -89,8 +89,8 @@ windows. It is understood that these resources are freed when the object is garbage-collected, but since garbage collection is not guaranteed to happen, such objects also provide an explicit way to release the external resource, usually a :meth:`!close` method. Programs are strongly recommended to explicitly -close such objects. The ':keyword:`try`...\ :keyword:`finally`' statement -and the ':keyword:`with`' statement provide convenient ways to do this. +close such objects. The :keyword:`try`...\ :keyword:`finally` statement +and the :keyword:`with` statement provide convenient ways to do this. .. index:: single: container @@ -106,12 +106,16 @@ that mutable object is changed. Types affect almost all aspects of object behavior. Even the importance of object identity is affected in some sense: for immutable types, operations that compute new values may actually return a reference to any existing object with -the same type and value, while for mutable objects this is not allowed. E.g., -after ``a = 1; b = 1``, ``a`` and ``b`` may or may not refer to the same object -with the value one, depending on the implementation, but after ``c = []; d = -[]``, ``c`` and ``d`` are guaranteed to refer to two different, unique, newly -created empty lists. (Note that ``c = d = []`` assigns the same object to both -``c`` and ``d``.) +the same type and value, while for mutable objects this is not allowed. +For example, after ``a = 1; b = 1``, *a* and *b* may or may not refer to +the same object with the value one, depending on the implementation. +This is because :class:`int` is an immutable type, so the reference to ``1`` +can be reused. This behaviour depends on the implementation used, so should +not be relied upon, but is something to be aware of when making use of object +identity tests. +However, after ``c = []; d = []``, *c* and *d* are guaranteed to refer to two +different, unique, newly created empty lists. (Note that ``e = f = []`` assigns +the *same* object to both *e* and *f*.) .. _types: @@ -159,7 +163,7 @@ NotImplemented .. index:: pair: object; NotImplemented This type has a single value. There is a single object with this value. This -object is accessed through the built-in name ``NotImplemented``. Numeric methods +object is accessed through the built-in name :data:`NotImplemented`. Numeric methods and rich comparison methods should return this value if they do not implement the operation for the operands provided. (The interpreter will then try the reflected operation, or some other fallback, depending on the operator.) It @@ -170,9 +174,12 @@ See for more details. .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. + Evaluating :data:`NotImplemented` in a boolean context was deprecated. + +.. versionchanged:: 3.14 + Evaluating :data:`NotImplemented` in a boolean context now raises a :exc:`TypeError`. + It previously evaluated to :const:`True` and emitted a :exc:`DeprecationWarning` + since Python 3.9. Ellipsis @@ -215,7 +222,7 @@ properties: * A sign is shown only when the number is negative. -Python distinguishes between integers, floating point numbers, and complex +Python distinguishes between integers, floating-point numbers, and complex numbers: @@ -259,18 +266,18 @@ Booleans (:class:`bool`) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. index:: - pair: object; floating point - pair: floating point; number + pair: object; floating-point + pair: floating-point; number pair: C; language pair: Java; language -These represent machine-level double precision floating point numbers. You are +These represent machine-level double precision floating-point numbers. You are at the mercy of the underlying machine architecture (and C or Java implementation) for the accepted range and handling of overflow. Python does not -support single-precision floating point numbers; the savings in processor and +support single-precision floating-point numbers; the savings in processor and memory usage that are usually the reason for using these are dwarfed by the overhead of using objects in Python, so there is no reason to complicate the -language with two kinds of floating point numbers. +language with two kinds of floating-point numbers. :class:`numbers.Complex` (:class:`complex`) @@ -281,7 +288,7 @@ language with two kinds of floating point numbers. pair: complex; number These represent complex numbers as a pair of machine-level double precision -floating point numbers. The same caveats apply as for floating point numbers. +floating-point numbers. The same caveats apply as for floating-point numbers. The real and imaginary parts of a complex number ``z`` can be retrieved through the read-only attributes ``z.real`` and ``z.imag``. @@ -299,14 +306,17 @@ Sequences These represent finite ordered sets indexed by non-negative numbers. The built-in function :func:`len` returns the number of items of a sequence. When the length of a sequence is *n*, the index set contains the numbers 0, 1, -..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. +..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. Some sequences, +including built-in sequences, interpret negative subscripts by adding the +sequence length. For example, ``a[-2]`` equals ``a[n-2]``, the second to last +item of sequence a with length ``n``. .. index:: single: slicing Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a -sequence of the same type. This implies that the index set is renumbered so -that it starts at 0. +sequence of the same type. The comment above about negative indexes also applies +to negative slice positions. Some sequences also support "extended slicing" with a third "step" parameter: ``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* @@ -370,7 +380,7 @@ Bytes A bytes object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in :func:`bytes()` constructor + (like ``b'abc'``) and the built-in :func:`bytes` constructor can be used to create bytes objects. Also, bytes objects can be decoded to strings via the :meth:`~bytes.decode` method. @@ -489,7 +499,7 @@ in the same order they were added sequentially over the dictionary. Replacing an existing key does not change the order, however removing a key and re-inserting it will add it to the end instead of keeping its old place. -Dictionaries are mutable; they can be created by the ``{...}`` notation (see +Dictionaries are mutable; they can be created by the ``{}`` notation (see section :ref:`dict`). .. index:: @@ -554,8 +564,9 @@ Special read-only attributes in which the function was defined. * - .. attribute:: function.__closure__ - - ``None`` or a :class:`tuple` of cells that contain bindings for the - function's free variables. + - ``None`` or a :class:`tuple` of cells that contain bindings for the names specified + in the :attr:`~codeobject.co_freevars` attribute of the function's + :attr:`code object `. A cell object has the attribute ``cell_contents``. This can be used to get the value of the cell, as well as set the value. @@ -571,6 +582,7 @@ Special writable attributes single: __defaults__ (function attribute) single: __code__ (function attribute) single: __annotations__ (function attribute) + single: __annotate__ (function attribute) single: __kwdefaults__ (function attribute) single: __type_params__ (function attribute) @@ -584,7 +596,6 @@ Most of these attributes check the type of the assigned value: * - .. attribute:: function.__doc__ - The function's documentation string, or ``None`` if unavailable. - Not inherited by subclasses. * - .. attribute:: function.__name__ - The function's name. @@ -618,7 +629,17 @@ Most of these attributes check the type of the assigned value: :term:`parameters `. The keys of the dictionary are the parameter names, and ``'return'`` for the return annotation, if provided. - See also: :ref:`annotations-howto`. + See also: :attr:`object.__annotations__`. + + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. + + * - .. attribute:: function.__annotate__ + - The :term:`annotate function` for this function, or ``None`` + if the function has no annotations. See :attr:`object.__annotate__`. + + .. versionadded:: 3.14 * - .. attribute:: function.__kwdefaults__ - A :class:`dictionary ` containing defaults for keyword-only @@ -724,14 +745,7 @@ When an instance method object is derived from a :class:`classmethod` object, th itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is the underlying function. -Note that the transformation from :ref:`function object ` -to instance method -object happens each time the attribute is retrieved from the instance. In -some cases, a fruitful optimization is to assign the attribute to a local -variable and call that local variable. Also notice that this -transformation only happens for user-defined functions; other callable -objects (and all non-callable objects) are retrieved without -transformation. It is also important to note that user-defined functions +It is important to note that user-defined functions which are attributes of a class instance are not converted to bound methods; this *only* happens when the function is an attribute of the class. @@ -832,6 +846,7 @@ this case, the special read-only attribute :attr:`!__self__` is set to the objec denoted by *alist*. (The attribute has the same semantics as it does with :attr:`other instance methods `.) +.. _classes: Classes ^^^^^^^ @@ -850,6 +865,8 @@ Instances of arbitrary classes can be made callable by defining a :meth:`~object.__call__` method in their class. +.. _module-objects: + Modules ------- @@ -875,48 +892,247 @@ Attribute assignment updates the module's namespace dictionary, e.g., .. index:: single: __name__ (module attribute) - single: __doc__ (module attribute) + single: __spec__ (module attribute) + single: __package__ (module attribute) + single: __loader__ (module attribute) + single: __path__ (module attribute) single: __file__ (module attribute) + single: __cached__ (module attribute) + single: __doc__ (module attribute) single: __annotations__ (module attribute) + single: __annotate__ (module attribute) pair: module; namespace -Predefined (writable) attributes: +.. _import-mod-attrs: + +Import-related attributes on module objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Module objects have the following attributes that relate to the +:ref:`import system `. When a module is created using the machinery associated +with the import system, these attributes are filled in based on the module's +:term:`spec `, before the :term:`loader` executes and loads the +module. + +To create a module dynamically rather than using the import system, +it's recommended to use :func:`importlib.util.module_from_spec`, +which will set the various import-controlled attributes to appropriate values. +It's also possible to use the :class:`types.ModuleType` constructor to create +modules directly, but this technique is more error-prone, as most attributes +must be manually set on the module object after it has been created when using +this approach. + +.. caution:: + + With the exception of :attr:`~module.__name__`, it is **strongly** + recommended that you rely on :attr:`~module.__spec__` and its attributes + instead of any of the other individual attributes listed in this subsection. + Note that updating an attribute on :attr:`!__spec__` will not update the + corresponding attribute on the module itself: + + .. doctest:: + + >>> import typing + >>> typing.__name__, typing.__spec__.name + ('typing', 'typing') + >>> typing.__spec__.name = 'spelling' + >>> typing.__name__, typing.__spec__.name + ('typing', 'spelling') + >>> typing.__name__ = 'keyboard_smashing' + >>> typing.__name__, typing.__spec__.name + ('keyboard_smashing', 'spelling') + +.. attribute:: module.__name__ + + The name used to uniquely identify the module in the import system. + For a directly executed module, this will be set to ``"__main__"``. + + This attribute must be set to the fully qualified name of the module. + It is expected to match the value of + :attr:`module.__spec__.name `. + +.. attribute:: module.__spec__ + + A record of the module's import-system-related state. + + Set to the :class:`module spec ` that was + used when importing the module. See :ref:`module-specs` for more details. + + .. versionadded:: 3.4 + +.. attribute:: module.__package__ + + The :term:`package` a module belongs to. + + If the module is top-level (that is, not a part of any specific package) + then the attribute should be set to ``''`` (the empty string). Otherwise, + it should be set to the name of the module's package (which can be equal to + :attr:`module.__name__` if the module itself is a package). See :pep:`366` + for further details. + + This attribute is used instead of :attr:`~module.__name__` to calculate + explicit relative imports for main modules. It defaults to ``None`` for + modules created dynamically using the :class:`types.ModuleType` constructor; + use :func:`importlib.util.module_from_spec` instead to ensure the attribute + is set to a :class:`str`. + + It is **strongly** recommended that you use + :attr:`module.__spec__.parent ` + instead of :attr:`!module.__package__`. :attr:`__package__` is now only used + as a fallback if :attr:`!__spec__.parent` is not set, and this fallback + path is deprecated. + + .. versionchanged:: 3.4 + This attribute now defaults to ``None`` for modules created dynamically + using the :class:`types.ModuleType` constructor. + Previously the attribute was optional. + + .. versionchanged:: 3.6 + The value of :attr:`!__package__` is expected to be the same as + :attr:`__spec__.parent `. + :attr:`__package__` is now only used as a fallback during import + resolution if :attr:`!__spec__.parent` is not defined. + + .. versionchanged:: 3.10 + :exc:`ImportWarning` is raised if an import resolution falls back to + :attr:`!__package__` instead of + :attr:`__spec__.parent `. + + .. versionchanged:: 3.12 + Raise :exc:`DeprecationWarning` instead of :exc:`ImportWarning` when + falling back to :attr:`!__package__` during import resolution. + + .. deprecated-removed:: 3.13 3.15 + :attr:`!__package__` will cease to be set or taken into consideration + by the import system or standard library. + +.. attribute:: module.__loader__ + + The :term:`loader` object that the import machinery used to load the module. + + This attribute is mostly useful for introspection, but can be used for + additional loader-specific functionality, for example getting data + associated with a loader. + + :attr:`!__loader__` defaults to ``None`` for modules created dynamically + using the :class:`types.ModuleType` constructor; + use :func:`importlib.util.module_from_spec` instead to ensure the attribute + is set to a :term:`loader` object. + + It is **strongly** recommended that you use + :attr:`module.__spec__.loader ` + instead of :attr:`!module.__loader__`. + + .. versionchanged:: 3.4 + This attribute now defaults to ``None`` for modules created dynamically + using the :class:`types.ModuleType` constructor. + Previously the attribute was optional. + + .. deprecated-removed:: 3.12 3.14 + Setting :attr:`!__loader__` on a module while failing to set + :attr:`!__spec__.loader` is deprecated. In Python 3.14, + :attr:`!__loader__` will cease to be set or taken into consideration by + the import system or the standard library. + +.. attribute:: module.__path__ + + A (possibly empty) :term:`sequence` of strings enumerating the locations + where the package's submodules will be found. Non-package modules should + not have a :attr:`!__path__` attribute. See :ref:`package-path-rules` for + more details. + + It is **strongly** recommended that you use + :attr:`module.__spec__.submodule_search_locations ` + instead of :attr:`!module.__path__`. + +.. attribute:: module.__file__ +.. attribute:: module.__cached__ + + :attr:`!__file__` and :attr:`!__cached__` are both optional attributes that + may or may not be set. Both attributes should be a :class:`str` when they + are available. + + :attr:`!__file__` indicates the pathname of the file from which the module + was loaded (if loaded from a file), or the pathname of the shared library + file for extension modules loaded dynamically from a shared library. + It might be missing for certain types of modules, such as C modules that are + statically linked into the interpreter, and the + :ref:`import system ` may opt to leave it unset if it + has no semantic meaning (for example, a module loaded from a database). + + If :attr:`!__file__` is set then the :attr:`!__cached__` attribute might + also be set, which is the path to any compiled version of + the code (for example, a byte-compiled file). The file does not need to exist + to set this attribute; the path can simply point to where the + compiled file *would* exist (see :pep:`3147`). + + Note that :attr:`!__cached__` may be set even if :attr:`!__file__` is not + set. However, that scenario is quite atypical. Ultimately, the + :term:`loader` is what makes use of the module spec provided by the + :term:`finder` (from which :attr:`!__file__` and :attr:`!__cached__` are + derived). So if a loader can load from a cached module but otherwise does + not load from a file, that atypical scenario may be appropriate. + + It is **strongly** recommended that you use + :attr:`module.__spec__.cached ` + instead of :attr:`!module.__cached__`. + + .. deprecated-removed:: 3.13 3.15 + Setting :attr:`!__cached__` on a module while failing to set + :attr:`!__spec__.cached` is deprecated. In Python 3.15, + :attr:`!__cached__` will cease to be set or taken into consideration by + the import system or standard library. + +Other writable attributes on module objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As well as the import-related attributes listed above, module objects also have +the following writable attributes: + +.. attribute:: module.__doc__ + + The module's documentation string, or ``None`` if unavailable. + See also: :attr:`__doc__ attributes `. - :attr:`__name__` - The module's name. +.. attribute:: module.__annotations__ - :attr:`__doc__` - The module's documentation string, or ``None`` if - unavailable. + A dictionary containing :term:`variable annotations ` + collected during module body execution. For best practices on working with + :attr:`!__annotations__`, see :mod:`annotationlib`. - :attr:`__file__` - The pathname of the file from which the - module was loaded, if it was loaded from a file. - The :attr:`__file__` - attribute may be missing for certain types of modules, such as C modules - that are statically linked into the interpreter. For extension modules - loaded dynamically from a shared library, it's the pathname of the shared - library file. + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` collected during - module body execution. For best practices on working - with :attr:`__annotations__`, please see :ref:`annotations-howto`. +.. attribute:: module.__annotate__ + + The :term:`annotate function` for this module, or ``None`` if the module has + no annotations. See also: :attr:`~object.__annotate__` attributes. + + .. versionadded:: 3.14 + +Module dictionaries +^^^^^^^^^^^^^^^^^^^ + +Module objects also have the following special read-only attribute: .. index:: single: __dict__ (module attribute) +.. attribute:: module.__dict__ -Special read-only attribute: :attr:`~object.__dict__` is the module's -namespace as a dictionary object. + The module's namespace as a dictionary object. Uniquely among the attributes + listed here, :attr:`!__dict__` cannot be accessed as a global variable from + within a module; it can only be accessed as an attribute on module objects. -.. impl-detail:: + .. impl-detail:: - Because of the way CPython clears module dictionaries, the module - dictionary will be cleared when the module falls out of scope even if the - dictionary still has live references. To avoid this, copy the dictionary - or keep the module around while using its dictionary directly. + Because of the way CPython clears module dictionaries, the module + dictionary will be cleared when the module falls out of scope even if the + dictionary still has live references. To avoid this, copy the dictionary + or keep the module around while using its dictionary directly. +.. _class-attrs-and-methods: + Custom classes -------------- @@ -929,11 +1145,8 @@ name is not found there, the attribute search continues in the base classes. This search of the base classes uses the C3 method resolution order which behaves correctly even in the presence of 'diamond' inheritance structures where there are multiple inheritance paths leading back to a common ancestor. -Additional details on the C3 MRO used by Python can be found in the -documentation accompanying the 2.3 release at -https://www.python.org/download/releases/2.3/mro/. - -.. XXX: Could we add that MRO doc as an appendix to the language ref? +Additional details on the C3 MRO used by Python can be found at +:ref:`python_2.3_mro`. .. index:: pair: object; class @@ -962,6 +1175,9 @@ of a base class. A class object can be called (see above) to yield a class instance (see below). +Special attributes +^^^^^^^^^^^^^^^^^^ + .. index:: single: __name__ (class attribute) single: __module__ (class attribute) @@ -969,37 +1185,126 @@ A class object can be called (see above) to yield a class instance (see below). single: __bases__ (class attribute) single: __doc__ (class attribute) single: __annotations__ (class attribute) + single: __annotate__ (class attribute) single: __type_params__ (class attribute) + single: __static_attributes__ (class attribute) + single: __firstlineno__ (class attribute) -Special attributes: +.. list-table:: + :header-rows: 1 - :attr:`~definition.__name__` - The class name. + * - Attribute + - Meaning - :attr:`__module__` - The name of the module in which the class was defined. + * - .. attribute:: type.__name__ + - The class's name. + See also: :attr:`__name__ attributes `. - :attr:`~object.__dict__` - The dictionary containing the class's namespace. + * - .. attribute:: type.__qualname__ + - The class's :term:`qualified name`. + See also: :attr:`__qualname__ attributes `. + + * - .. attribute:: type.__module__ + - The name of the module in which the class was defined. + + * - .. attribute:: type.__dict__ + - A :class:`mapping proxy ` + providing a read-only view of the class's namespace. + See also: :attr:`__dict__ attributes `. + + * - .. attribute:: type.__bases__ + - A :class:`tuple` containing the class's bases. + In most cases, for a class defined as ``class X(A, B, C)``, + ``X.__bases__`` will be exactly equal to ``(A, B, C)``. + + * - .. attribute:: type.__doc__ + - The class's documentation string, or ``None`` if undefined. + Not inherited by subclasses. + + * - .. attribute:: type.__annotations__ + - A dictionary containing + :term:`variable annotations ` + collected during class body execution. See also: + :attr:`__annotations__ attributes `. + + For best practices on working with :attr:`~object.__annotations__`, + please see :mod:`annotationlib`. - :attr:`~class.__bases__` - A tuple containing the base classes, in the order of - their occurrence in the base class list. + .. caution:: - :attr:`__doc__` - The class's documentation string, or ``None`` if undefined. + Accessing the :attr:`!__annotations__` attribute of a class + object directly may yield incorrect results in the presence of + metaclasses. In addition, the attribute may not exist for + some classes. Use :func:`annotationlib.get_annotations` to + retrieve class annotations safely. + + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. + + * - .. method:: type.__annotate__ + - The :term:`annotate function` for this class, or ``None`` + if the class has no annotations. + See also: :attr:`__annotate__ attributes `. + + .. caution:: + + Accessing the :attr:`!__annotate__` attribute of a class + object directly may yield incorrect results in the presence of + metaclasses. Use :func:`annotationlib.get_annotate_function` to + retrieve the annotate function safely. + + .. versionadded:: 3.14 + + * - .. attribute:: type.__type_params__ + - A :class:`tuple` containing the :ref:`type parameters ` of + a :ref:`generic class `. + + .. versionadded:: 3.12 - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` - collected during class body execution. For best practices on - working with :attr:`__annotations__`, please see - :ref:`annotations-howto`. + * - .. attribute:: type.__static_attributes__ + - A :class:`tuple` containing names of attributes of this class which are + assigned through ``self.X`` from any function in its body. - :attr:`__type_params__` - A tuple containing the :ref:`type parameters ` of - a :ref:`generic class `. + .. versionadded:: 3.13 + * - .. attribute:: type.__firstlineno__ + - The line number of the first line of the class definition, + including decorators. + Setting the :attr:`__module__` attribute removes the + :attr:`!__firstlineno__` item from the type's dictionary. + + .. versionadded:: 3.13 + + * - .. attribute:: type.__mro__ + - The :class:`tuple` of classes that are considered when looking for + base classes during method resolution. + + +Special methods +^^^^^^^^^^^^^^^ + +In addition to the special attributes described above, all Python classes also +have the following two methods available: + +.. method:: type.mro + + This method can be overridden by a metaclass to customize the method + resolution order for its instances. It is called at class instantiation, + and its result is stored in :attr:`~type.__mro__`. + +.. method:: type.__subclasses__ + + Each class keeps a list of weak references to its immediate subclasses. This + method returns a list of all those references still alive. The list is in + definition order. Example: + + .. doctest:: + + >>> class A: pass + >>> class B(A): pass + >>> A.__subclasses__() + [] Class instances --------------- @@ -1039,12 +1344,22 @@ dictionary directly. Class instances can pretend to be numbers, sequences, or mappings if they have methods with certain special names. See section :ref:`specialnames`. +Special attributes +^^^^^^^^^^^^^^^^^^ + .. index:: single: __dict__ (instance attribute) single: __class__ (instance attribute) -Special attributes: :attr:`~object.__dict__` is the attribute dictionary; -:attr:`~instance.__class__` is the instance's class. +.. attribute:: object.__class__ + + The class to which a class instance belongs. + +.. attribute:: object.__dict__ + + A dictionary or other mapping object used to store an object's (writable) + attributes. Not all instances have a :attr:`!__dict__` attribute; see the + section on :ref:`slots` for more details. I/O objects (also known as file objects) @@ -1134,6 +1449,8 @@ Special read-only attributes * - .. attribute:: codeobject.co_qualname - The fully qualified function name + .. versionadded:: 3.11 + * - .. attribute:: codeobject.co_argcount - The total number of positional :term:`parameters ` (including positional-only parameters and parameters with default values) @@ -1157,10 +1474,14 @@ Special read-only attributes * - .. attribute:: codeobject.co_cellvars - A :class:`tuple` containing the names of :ref:`local variables ` - that are referenced by nested functions inside the function + that are referenced from at least one :term:`nested scope` inside the function * - .. attribute:: codeobject.co_freevars - - A :class:`tuple` containing the names of free variables in the function + - A :class:`tuple` containing the names of + :term:`free (closure) variables ` that a :term:`nested scope` + references in an outer scope. See also :attr:`function.__closure__`. + + Note: references to global and builtin names are *not* included. * - .. attribute:: codeobject.co_code - A string representing the sequence of :term:`bytecode` instructions in @@ -1215,9 +1536,9 @@ Other bits in :attr:`~codeobject.co_flags` are reserved for internal use. .. index:: single: documentation string -If a code object represents a function, the first item in -:attr:`~codeobject.co_consts` is -the documentation string of the function, or ``None`` if undefined. +If a code object represents a function and has a docstring, +the first item in :attr:`~codeobject.co_consts` is +the docstring of the function. Methods on code objects ~~~~~~~~~~~~~~~~~~~~~~~ @@ -1229,7 +1550,7 @@ Methods on code objects The iterator returns :class:`tuple`\s containing the ``(start_line, end_line, start_column, end_column)``. The *i-th* tuple corresponds to the - position of the source code that compiled to the *i-th* instruction. + position of the source code that compiled to the *i-th* code unit. Column information is 0-indexed utf-8 byte offsets on the given source line. @@ -1290,6 +1611,14 @@ Methods on code objects :pep:`626` - Precise line numbers for debugging and other tools. The PEP that introduced the :meth:`!co_lines` method. +.. method:: codeobject.replace(**kwargs) + + Return a copy of the code object with new values for the specified fields. + + Code objects are also supported by the generic function :func:`copy.replace`. + + .. versionadded:: 3.8 + .. _frame-objects: @@ -1325,8 +1654,13 @@ Special read-only attributes ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. * - .. attribute:: frame.f_locals - - The dictionary used by the frame to look up - :ref:`local variables ` + - The mapping used by the frame to look up + :ref:`local variables `. + If the frame refers to an :term:`optimized scope`, + this may return a write-through proxy object. + + .. versionchanged:: 3.13 + Return a proxy for optimized scopes. * - .. attribute:: frame.f_globals - The dictionary used by the frame to look up @@ -1640,6 +1974,8 @@ Basic customization It is not guaranteed that :meth:`__del__` methods are called for objects that still exist when the interpreter exits. + :class:`weakref.finalize` provides a straightforward way to register + a cleanup function to be called when an object is garbage collected. .. note:: @@ -1697,7 +2033,8 @@ Basic customization "informal" string representation of instances of that class is required. This is typically used for debugging, so it is important that the representation - is information-rich and unambiguous. + is information-rich and unambiguous. A default implementation is provided by the + :class:`object` class itself. .. index:: single: string; __str__() (object method) @@ -1707,10 +2044,10 @@ Basic customization .. method:: object.__str__(self) - Called by :func:`str(object) ` and the built-in functions - :func:`format` and :func:`print` to compute the "informal" or nicely + Called by :func:`str(object) `, the default :meth:`__format__` implementation, + and the built-in function :func:`print`, to compute the "informal" or nicely printable string representation of an object. The return value must be a - :ref:`string ` object. + :ref:`str ` object. This method differs from :meth:`object.__repr__` in that there is no expectation that :meth:`__str__` return a valid Python expression: a more @@ -1727,7 +2064,8 @@ Basic customization .. index:: pair: built-in function; bytes Called by :ref:`bytes ` to compute a byte-string representation - of an object. This should return a :class:`bytes` object. + of an object. This should return a :class:`bytes` object. The :class:`object` + class itself does not provide this method. .. index:: single: string; __format__() (object method) @@ -1751,6 +2089,9 @@ Basic customization The return value must be a string object. + The default implementation by the :class:`object` class should be given + an empty *format_spec* string. It delegates to :meth:`__str__`. + .. versionchanged:: 3.4 The __format__ method of ``object`` itself raises a :exc:`TypeError` if passed any non-empty string. @@ -1777,7 +2118,7 @@ Basic customization ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and ``x>=y`` calls ``x.__ge__(y)``. - A rich comparison method may return the singleton ``NotImplemented`` if it does + A rich comparison method may return the singleton :data:`NotImplemented` if it does not implement the operation for a given pair of arguments. By convention, ``False`` and ``True`` are returned for a successful comparison. However, these methods can return any value, so if the comparison operator is used in a Boolean @@ -1785,14 +2126,20 @@ Basic customization :func:`bool` on the value to determine if the result is true or false. By default, ``object`` implements :meth:`__eq__` by using ``is``, returning - ``NotImplemented`` in the case of a false comparison: + :data:`NotImplemented` in the case of a false comparison: ``True if x is y else NotImplemented``. For :meth:`__ne__`, by default it delegates to :meth:`__eq__` and inverts the result unless it is - ``NotImplemented``. There are no other implied relationships among the + :data:`!NotImplemented`. There are no other implied relationships among the comparison operators or default implementations; for example, the truth of ``(x` with multiple slotted parent @@ -2541,7 +2892,7 @@ in the local namespace as the defined class. When a new class is created by ``type.__new__``, the object provided as the namespace parameter is copied to a new ordered mapping and the original object is discarded. The new copy is wrapped in a read-only proxy, which -becomes the :attr:`~object.__dict__` attribute of the class object. +becomes the :attr:`~type.__dict__` attribute of the class object. .. seealso:: @@ -2569,14 +2920,14 @@ order to allow the addition of Abstract Base Classes (ABCs) as "virtual base classes" to any class or type (including built-in types), including other ABCs. -.. method:: class.__instancecheck__(self, instance) +.. method:: type.__instancecheck__(self, instance) Return true if *instance* should be considered a (direct or indirect) instance of *class*. If defined, called to implement ``isinstance(instance, class)``. -.. method:: class.__subclasscheck__(self, subclass) +.. method:: type.__subclasscheck__(self, subclass) Return true if *subclass* should be considered a (direct or indirect) subclass of *class*. If defined, called to implement ``issubclass(subclass, @@ -2592,8 +2943,8 @@ case the instance is itself a class. :pep:`3119` - Introducing Abstract Base Classes Includes the specification for customizing :func:`isinstance` and - :func:`issubclass` behavior through :meth:`~class.__instancecheck__` and - :meth:`~class.__subclasscheck__`, with motivation for this functionality + :func:`issubclass` behavior through :meth:`~type.__instancecheck__` and + :meth:`~type.__subclasscheck__`, with motivation for this functionality in the context of adding Abstract Base Classes (see the :mod:`abc` module) to the language. @@ -2751,6 +3102,7 @@ Emulating callable objects Called when the instance is "called" as a function; if this method is defined, ``x(arg1, arg2, ...)`` roughly translates to ``type(x).__call__(x, arg1, ...)``. + The :class:`object` class itself does not provide this method. .. _sequence-types: @@ -2758,10 +3110,11 @@ Emulating callable objects Emulating container types ------------------------- -The following methods can be defined to implement container objects. Containers -usually are :term:`sequences ` (such as :class:`lists ` or +The following methods can be defined to implement container objects. None of them +are provided by the :class:`object` class itself. Containers usually are +:term:`sequences ` (such as :class:`lists ` or :class:`tuples `) or :term:`mappings ` (like -:class:`dictionaries `), +:term:`dictionaries `), but can represent other containers as well. The first set of methods is used either to emulate a sequence or to emulate a mapping; the difference is that for a sequence, the allowable keys should be the integers *k* for which ``0 <= k < @@ -2818,7 +3171,7 @@ through the object's keys; for sequences, it should iterate through the values. Called to implement :func:`operator.length_hint`. Should return an estimated length for the object (which may be greater or less than the actual length). The length must be an integer ``>=`` 0. The return value may also be - :const:`NotImplemented`, which is treated the same as if the + :data:`NotImplemented`, which is treated the same as if the ``__length_hint__`` method didn't exist at all. This method is purely an optimization and is never required for correctness. @@ -2970,7 +3323,7 @@ left undefined. function is to be supported. If one of those methods does not support the operation with the supplied - arguments, it should return ``NotImplemented``. + arguments, it should return :data:`NotImplemented`. .. method:: object.__radd__(self, other) @@ -2995,12 +3348,13 @@ left undefined. These methods are called to implement the binary arithmetic operations (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with reflected - (swapped) operands. These functions are only called if the left operand does - not support the corresponding operation [#]_ and the operands are of different - types. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is - an instance of a class that has an :meth:`__rsub__` method, - ``type(y).__rsub__(y, x)`` is called if ``type(x).__sub__(x, y)`` returns - *NotImplemented*. + (swapped) operands. These functions are only called if the operands + are of different types, when the left operand does not support the corresponding + operation [#]_, or the right operand's class is derived from the left operand's + class. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is + an instance of a class that has an :meth:`__rsub__` method, ``type(y).__rsub__(y, x)`` + is called if ``type(x).__sub__(x, y)`` returns :data:`NotImplemented` or ``type(y)`` + is a subclass of ``type(x)``. [#]_ .. index:: pair: built-in function; pow @@ -3015,7 +3369,6 @@ left undefined. non-reflected method. This behavior allows subclasses to override their ancestors' operations. - .. method:: object.__iadd__(self, other) object.__isub__(self, other) object.__imul__(self, other) @@ -3034,10 +3387,12 @@ left undefined. (``+=``, ``-=``, ``*=``, ``@=``, ``/=``, ``//=``, ``%=``, ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods should attempt to do the operation in-place (modifying *self*) and return the result (which could be, - but does not have to be, *self*). If a specific method is not defined, the + but does not have to be, *self*). If a specific method is not defined, or if + that method returns :data:`NotImplemented`, the augmented assignment falls back to the normal methods. For instance, if *x* is an instance of a class with an :meth:`__iadd__` method, ``x += y`` is - equivalent to ``x = x.__iadd__(y)`` . Otherwise, ``x.__add__(y)`` and + equivalent to ``x = x.__iadd__(y)`` . If :meth:`__iadd__` does not exist, or if ``x.__iadd__(y)`` + returns :data:`!NotImplemented`, ``x.__add__(y)`` and ``y.__radd__(x)`` are considered, as with the evaluation of ``x + y``. In certain situations, augmented assignment can result in unexpected errors (see :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in fact @@ -3095,11 +3450,8 @@ left undefined. return the value of the object truncated to an :class:`~numbers.Integral` (typically an :class:`int`). - The built-in function :func:`int` falls back to :meth:`__trunc__` if neither - :meth:`__int__` nor :meth:`__index__` is defined. - - .. versionchanged:: 3.11 - The delegation of :func:`int` to :meth:`__trunc__` is deprecated. + .. versionchanged:: 3.14 + :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. .. _context-managers: @@ -3122,6 +3474,7 @@ Typical uses of context managers include saving and restoring various kinds of global state, locking and unlocking resources, closing opened files, etc. For more information on context managers, see :ref:`typecontextmanager`. +The :class:`object` class itself does not provide the context manager methods. .. method:: object.__enter__(self) @@ -3225,6 +3578,51 @@ implement the protocol in Python. :class:`collections.abc.Buffer` ABC for buffer types. +Annotations +----------- + +Functions, classes, and modules may contain :term:`annotations `, +which are a way to associate information (usually :term:`type hints `) +with a symbol. + +.. attribute:: object.__annotations__ + + This attribute contains the annotations for an object. It is + :ref:`lazily evaluated `, so accessing the attribute may + execute arbitrary code and raise exceptions. If evaluation is successful, the + attribute is set to a dictionary mapping from variable names to annotations. + + .. versionchanged:: 3.14 + Annotations are now lazily evaluated. + +.. method:: object.__annotate__(format) + + An :term:`annotate function`. + Returns a new dictionary object mapping attribute/parameter names to their annotation values. + + Takes a format parameter specifying the format in which annotations values should be provided. + It must be a member of the :class:`annotationlib.Format` enum, or an integer with + a value corresponding to a member of the enum. + + If an annotate function doesn't support the requested format, it must raise + :exc:`NotImplementedError`. Annotate functions must always support + :attr:`~annotationlib.Format.VALUE` format; they must not raise + :exc:`NotImplementedError()` when called with this format. + + When called with :attr:`~annotationlib.Format.VALUE` format, an annotate function may raise + :exc:`NameError`; it must not raise :exc:`!NameError` when called requesting any other format. + + If an object does not have any annotations, :attr:`~object.__annotate__` should preferably be set + to ``None`` (it can’t be deleted), rather than set to a function that returns an empty dict. + + .. versionadded:: 3.14 + +.. seealso:: + + :pep:`649` --- Deferred evaluation of annotation using descriptors + Introduces lazy evaluation of annotations and the ``__annotate__`` function. + + .. _special-lookup: Special method lookup @@ -3326,6 +3724,8 @@ are awaitable. Must return an :term:`iterator`. Should be used to implement :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements this method to be compatible with the :keyword:`await` expression. + The :class:`object` class itself is not awaitable and does not provide + this method. .. note:: @@ -3411,6 +3811,9 @@ its ``__anext__`` method. Asynchronous iterators can be used in an :keyword:`async for` statement. +The :class:`object` class itself does not provide these methods. + + .. method:: object.__aiter__(self) Must return an *asynchronous iterator* object. @@ -3457,6 +3860,8 @@ suspend execution in its ``__aenter__`` and ``__aexit__`` methods. Asynchronous context managers can be used in an :keyword:`async with` statement. +The :class:`object` class itself does not provide these methods. + .. method:: object.__aenter__(self) Semantically similar to :meth:`~object.__enter__`, the only @@ -3493,12 +3898,15 @@ An example of an asynchronous context manager class:: the behavior that ``None`` is not callable. .. [#] "Does not support" here means that the class has no such method, or - the method returns ``NotImplemented``. Do not set the method to + the method returns :data:`NotImplemented`. Do not set the method to ``None`` if you want to force fallback to the right operand's reflected method—that will instead have the opposite effect of explicitly *blocking* such fallback. -.. [#] For operands of the same type, it is assumed that if the non-reflected - method -- such as :meth:`~object.__add__` -- fails then the overall - operation is not - supported, which is why the reflected method is not called. +.. [#] For operands of the same type, it is assumed that if the non-reflected method + (such as :meth:`~object.__add__`) fails then the operation is not supported, which is why the + reflected method is not called. + +.. [#] If the right operand's type is a subclass of the left operand's type, the + reflected method having precedence allows subclasses to override their ancestors' + operations. diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index cea3a56ba51644..cb6c524dd97a30 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -90,7 +90,7 @@ If a name is bound in a block, it is a local variable of that block, unless declared as :keyword:`nonlocal` or :keyword:`global`. If a name is bound at the module level, it is a global variable. (The variables of the module code block are local and global.) If a variable is used in a code block but not -defined there, it is a :dfn:`free variable`. +defined there, it is a :term:`free variable`. Each occurrence of a name in the program text refers to the :dfn:`binding` of that name established by the following name resolution rules. @@ -139,8 +139,9 @@ namespace. Names are resolved in the top-level namespace by searching the global namespace, i.e. the namespace of the module containing the code block, and the builtins namespace, the namespace of the module :mod:`builtins`. The global namespace is searched first. If the names are not found there, the -builtins namespace is searched. The :keyword:`!global` statement must precede -all uses of the listed names. +builtins namespace is searched next. If the names are also not found in the +builtins namespace, new variables are created in the global namespace. +The global statement must precede all uses of the listed names. The :keyword:`global` statement has the same scope as a name binding operation in the same block. If the nearest enclosing scope for a free variable contains @@ -189,14 +190,15 @@ However, the following will succeed:: Annotation scopes ----------------- -:ref:`Type parameter lists ` and :keyword:`type` statements +:term:`Annotations `, :ref:`type parameter lists ` +and :keyword:`type` statements introduce *annotation scopes*, which behave mostly like function scopes, -but with some exceptions discussed below. :term:`Annotations ` -currently do not use annotation scopes, but they are expected to use -annotation scopes in Python 3.13 when :pep:`649` is implemented. +but with some exceptions discussed below. Annotation scopes are used in the following contexts: +* :term:`Function annotations `. +* :term:`Variable annotations `. * Type parameter lists for :ref:`generic type aliases `. * Type parameter lists for :ref:`generic functions `. A generic function's annotations are @@ -204,7 +206,7 @@ Annotation scopes are used in the following contexts: * Type parameter lists for :ref:`generic classes `. A generic class's base classes and keyword arguments are executed within the annotation scope, but its decorators are not. -* The bounds and constraints for type variables +* The bounds, constraints, and default values for type parameters (:ref:`lazily evaluated `). * The value of type aliases (:ref:`lazily evaluated `). @@ -224,24 +226,34 @@ Annotation scopes differ from function scopes in the following ways: statements in inner scopes. This includes only type parameters, as no other syntactic elements that can appear within annotation scopes can introduce new names. * While annotation scopes have an internal name, that name is not reflected in the - :term:`__qualname__ ` of objects defined within the scope. - Instead, the :attr:`!__qualname__` + :term:`qualified name` of objects defined within the scope. + Instead, the :attr:`~definition.__qualname__` of such objects is as if the object were defined in the enclosing scope. .. versionadded:: 3.12 Annotation scopes were introduced in Python 3.12 as part of :pep:`695`. +.. versionchanged:: 3.13 + Annotation scopes are also used for type parameter defaults, as + introduced by :pep:`696`. + +.. versionchanged:: 3.14 + Annotation scopes are now also used for annotations, as specified in + :pep:`649` and :pep:`749`. + .. _lazy-evaluation: Lazy evaluation --------------- -The values of type aliases created through the :keyword:`type` statement are -*lazily evaluated*. The same applies to the bounds and constraints of type +Most annotation scopes are *lazily evaluated*. This includes annotations, +the values of type aliases created through the :keyword:`type` statement, and +the bounds, constraints, and default values of type variables created through the :ref:`type parameter syntax `. This means that they are not evaluated when the type alias or type variable is -created. Instead, they are only evaluated when doing so is necessary to resolve -an attribute access. +created, or when the object carrying annotations is created. Instead, they +are only evaluated when necessary, for example when the ``__value__`` +attribute on a type alias is accessed. Example: @@ -325,6 +337,9 @@ enclosing namespace, but in the global namespace. [#]_ The :func:`exec` and :func:`eval` functions have optional arguments to override the global and local namespace. If only one namespace is specified, it is used for both. +.. XXX(ncoghlan) above is only accurate for string execution. When executing code objects, + closure cells may now be passed explicitly to resolve co_freevars references. + Docs issue: https://github.com/python/cpython/issues/122826 .. _exceptions: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 50e0f97a6534af..decde0d297cf59 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -33,7 +33,7 @@ implementation for built-in types works as follows: * If either argument is a complex number, the other is converted to complex; -* otherwise, if either argument is a floating point number, the other is +* otherwise, if either argument is a floating-point number, the other is converted to floating point; * otherwise, both must be integers and no conversion is necessary. @@ -83,18 +83,47 @@ exception. pair: name; mangling pair: private; names -**Private name mangling:** When an identifier that textually occurs in a class -definition begins with two or more underscore characters and does not end in two -or more underscores, it is considered a :dfn:`private name` of that class. -Private names are transformed to a longer form before code is generated for -them. The transformation inserts the class name, with leading underscores -removed and a single underscore inserted, in front of the name. For example, -the identifier ``__spam`` occurring in a class named ``Ham`` will be transformed -to ``_Ham__spam``. This transformation is independent of the syntactical -context in which the identifier is used. If the transformed name is extremely -long (longer than 255 characters), implementation defined truncation may happen. -If the class name consists only of underscores, no transformation is done. +Private name mangling +^^^^^^^^^^^^^^^^^^^^^ +When an identifier that textually occurs in a class definition begins with two +or more underscore characters and does not end in two or more underscores, it +is considered a :dfn:`private name` of that class. + +.. seealso:: + + The :ref:`class specifications `. + +More precisely, private names are transformed to a longer form before code is +generated for them. If the transformed name is longer than 255 characters, +implementation-defined truncation may happen. + +The transformation is independent of the syntactical context in which the +identifier is used but only the following private identifiers are mangled: + +- Any name used as the name of a variable that is assigned or read or any + name of an attribute being accessed. + + The :attr:`~definition.__name__` attribute of nested functions, classes, and + type aliases is however not mangled. + +- The name of imported modules, e.g., ``__spam`` in ``import __spam``. + If the module is part of a package (i.e., its name contains a dot), + the name is *not* mangled, e.g., the ``__foo`` in ``import __foo.bar`` + is not mangled. + +- The name of an imported member, e.g., ``__f`` in ``from spam import __f``. + +The transformation rule is defined as follows: + +- The class name, with leading underscores removed and a single leading + underscore inserted, is inserted in front of the identifier, e.g., the + identifier ``__spam`` occurring in a class named ``Foo``, ``_Foo`` or + ``__Foo`` is transformed to ``_Foo__spam``. + +- If the class name consists only of underscores, the transformation is the + identity, e.g., the identifier ``__spam`` occurring in a class named ``_`` + or ``__`` is left as is. .. _atom-literals: @@ -110,8 +139,8 @@ Python supports string and bytes literals and various numeric literals: : | `integer` | `floatnumber` | `imagnumber` Evaluation of a literal yields an object of the given type (string, bytes, -integer, floating point number, complex number) with the given value. The value -may be approximated in the case of floating point and imaginary (complex) +integer, floating-point number, complex number) with the given value. The value +may be approximated in the case of floating-point and imaginary (complex) literals. See section :ref:`literals` for details. .. index:: @@ -218,10 +247,12 @@ A comprehension in an :keyword:`!async def` function may consist of either a :keyword:`!for` or :keyword:`!async for` clause following the leading expression, may contain additional :keyword:`!for` or :keyword:`!async for` clauses, and may also use :keyword:`await` expressions. -If a comprehension contains either :keyword:`!async for` clauses or -:keyword:`!await` expressions or other asynchronous comprehensions it is called -an :dfn:`asynchronous comprehension`. An asynchronous comprehension may -suspend the execution of the coroutine function in which it appears. + +If a comprehension contains :keyword:`!async for` clauses, or if it contains +:keyword:`!await` expressions or other asynchronous comprehensions anywhere except +the iterable expression in the leftmost :keyword:`!for` clause, it is called an +:dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the +execution of the coroutine function in which it appears. See also :pep:`530`. .. versionadded:: 3.6 @@ -253,7 +284,7 @@ A list display is a possibly empty series of expressions enclosed in square brackets: .. productionlist:: python-grammar - list_display: "[" [`starred_list` | `comprehension`] "]" + list_display: "[" [`flexible_expression_list` | `comprehension`] "]" A list display yields a new list object, the contents being specified by either a list of expressions or a comprehension. When a comma-separated list of @@ -278,7 +309,7 @@ A set display is denoted by curly braces and distinguishable from dictionary displays by the lack of colons separating keys and values: .. productionlist:: python-grammar - set_display: "{" (`starred_list` | `comprehension`) "}" + set_display: "{" (`flexible_expression_list` | `comprehension`) "}" A set display yields a new mutable set object, the contents being specified by either a sequence of expressions or a comprehension. When a comma-separated @@ -423,7 +454,7 @@ Yield expressions .. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" yield_from: "yield" "from" `expression` - yield_expression: "yield" `expression_list` | `yield_from` + yield_expression: "yield" `yield_list` | `yield_from` The yield expression is used when defining a :term:`generator` function or an :term:`asynchronous generator` function and @@ -454,9 +485,9 @@ When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of the generator function. The execution starts when one of the generator's methods is called. At that time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`~python-grammar:expression_list` +suspended again, returning the value of :token:`~python-grammar:yield_list` to the generator's caller, -or ``None`` if :token:`~python-grammar:expression_list` is omitted. +or ``None`` if :token:`~python-grammar:yield_list` is omitted. By suspended, we mean that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -545,7 +576,7 @@ is already executing raises a :exc:`ValueError` exception. :meth:`~generator.__next__` method, the current yield expression always evaluates to :const:`None`. The execution then continues to the next yield expression, where the generator is suspended again, and the value of the - :token:`~python-grammar:expression_list` is returned to :meth:`__next__`'s + :token:`~python-grammar:yield_list` is returned to :meth:`__next__`'s caller. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. @@ -664,7 +695,7 @@ how a generator object would be used in a :keyword:`for` statement. Calling one of the asynchronous generator's methods returns an :term:`awaitable` object, and the execution starts when this object is awaited on. At that time, the execution proceeds to the first yield expression, where it is suspended -again, returning the value of :token:`~python-grammar:expression_list` to the +again, returning the value of :token:`~python-grammar:yield_list` to the awaiting coroutine. As with a generator, suspension means that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -728,7 +759,7 @@ which are used to control the execution of a generator function. asynchronous generator function is resumed with an :meth:`~agen.__anext__` method, the current yield expression always evaluates to :const:`None` in the returned awaitable, which when run will continue to the next yield - expression. The value of the :token:`~python-grammar:expression_list` of the + expression. The value of the :token:`~python-grammar:yield_list` of the yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without yielding another value, the awaitable instead raises a @@ -741,7 +772,7 @@ which are used to control the execution of a generator function. .. coroutinemethod:: agen.asend(value) Returns an awaitable which when run resumes the execution of the - asynchronous generator. As with the :meth:`~generator.send()` method for a + asynchronous generator. As with the :meth:`~generator.send` method for a generator, this "sends" a value into the asynchronous generator function, and the *value* argument becomes the result of the current yield expression. The awaitable returned by the :meth:`asend` method will return the next @@ -861,7 +892,7 @@ will generally select an element from the container. The subscription of a :ref:`GenericAlias ` object. .. productionlist:: python-grammar - subscription: `primary` "[" `expression_list` "]" + subscription: `primary` "[" `flexible_expression_list` "]" When an object is subscripted, the interpreter will evaluate the primary and the expression list. @@ -873,9 +904,13 @@ primary is subscripted, the evaluated result of the expression list will be passed to one of these methods. For more details on when ``__class_getitem__`` is called instead of ``__getitem__``, see :ref:`classgetitem-versus-getitem`. -If the expression list contains at least one comma, it will evaluate to a -:class:`tuple` containing the items of the expression list. Otherwise, the -expression list will evaluate to the value of the list's sole member. +If the expression list contains at least one comma, or if any of the expressions +are starred, the expression list will evaluate to a :class:`tuple` containing +the items of the expression list. Otherwise, the expression list will evaluate +to the value of the list's sole member. + +.. versionchanged:: 3.11 + Expressions in an expression list may be starred. See :pep:`646`. For built-in objects, there are two types of objects that support subscription via :meth:`~object.__getitem__`: @@ -1211,7 +1246,8 @@ Raising ``0.0`` to a negative power results in a :exc:`ZeroDivisionError`. Raising a negative number to a fractional power results in a :class:`complex` number. (In earlier versions it raised a :exc:`ValueError`.) -This operation can be customized using the special :meth:`~object.__pow__` method. +This operation can be customized using the special :meth:`~object.__pow__` and +:meth:`~object.__rpow__` methods. .. _unary: @@ -1299,6 +1335,9 @@ This operation can be customized using the special :meth:`~object.__mul__` and The ``@`` (at) operator is intended to be used for matrix multiplication. No builtin Python types implement this operator. +This operation can be customized using the special :meth:`~object.__matmul__` and +:meth:`~object.__rmatmul__` methods. + .. versionadded:: 3.5 .. index:: @@ -1314,8 +1353,10 @@ integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` exception. -This operation can be customized using the special :meth:`~object.__truediv__` and -:meth:`~object.__floordiv__` methods. +The division operation can be customized using the special :meth:`~object.__truediv__` +and :meth:`~object.__rtruediv__` methods. +The floor division operation can be customized using the special +:meth:`~object.__floordiv__` and :meth:`~object.__rfloordiv__` methods. .. index:: single: modulo @@ -1324,7 +1365,7 @@ This operation can be customized using the special :meth:`~object.__truediv__` a The ``%`` (modulo) operator yields the remainder from the division of the first argument by the second. The numeric arguments are first converted to a common type. A zero right argument raises the :exc:`ZeroDivisionError` exception. The -arguments may be floating point numbers, e.g., ``3.14%0.7`` equals ``0.34`` +arguments may be floating-point numbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals ``4*0.7 + 0.34``.) The modulo operator always yields a result with the same sign as its second operand (or zero); the absolute value of the result is strictly smaller than the absolute value of the second operand @@ -1340,11 +1381,12 @@ also overloaded by string objects to perform old-style string formatting (also known as interpolation). The syntax for string formatting is described in the Python Library Reference, section :ref:`old-string-formatting`. -The *modulo* operation can be customized using the special :meth:`~object.__mod__` method. +The *modulo* operation can be customized using the special :meth:`~object.__mod__` +and :meth:`~object.__rmod__` methods. The floor division operator, the modulo operator, and the :func:`divmod` -function are not defined for complex numbers. Instead, convert to a floating -point number using the :func:`abs` function if appropriate. +function are not defined for complex numbers. Instead, convert to a +floating-point number using the :func:`abs` function if appropriate. .. index:: single: addition @@ -1367,7 +1409,8 @@ This operation can be customized using the special :meth:`~object.__add__` and The ``-`` (subtraction) operator yields the difference of its arguments. The numeric arguments are first converted to a common type. -This operation can be customized using the special :meth:`~object.__sub__` method. +This operation can be customized using the special :meth:`~object.__sub__` and +:meth:`~object.__rsub__` methods. .. _shifting: @@ -1388,8 +1431,10 @@ The shifting operations have lower priority than the arithmetic operations: These operators accept integers as arguments. They shift the first argument to the left or right by the number of bits given by the second argument. -This operation can be customized using the special :meth:`~object.__lshift__` and -:meth:`~object.__rshift__` methods. +The left shift operation can be customized using the special :meth:`~object.__lshift__` +and :meth:`~object.__rlshift__` methods. +The right shift operation can be customized using the special :meth:`~object.__rshift__` +and :meth:`~object.__rrshift__` methods. .. index:: pair: exception; ValueError @@ -1539,7 +1584,7 @@ built-in types. ``x == x`` are all false, while ``x != x`` is true. This behavior is compliant with IEEE 754. -* ``None`` and ``NotImplemented`` are singletons. :PEP:`8` advises that +* ``None`` and :data:`NotImplemented` are singletons. :PEP:`8` advises that comparisons for singletons should always be done with ``is`` or ``is not``, never the equality operators. @@ -1762,6 +1807,9 @@ returns a boolean value regardless of the type of its argument single: assignment expression single: walrus operator single: named expression + pair: assignment; expression + +.. _assignment-expressions: Assignment expressions ====================== @@ -1864,10 +1912,12 @@ Expression lists single: , (comma); expression list .. productionlist:: python-grammar + starred_expression: ["*"] `or_expr` + flexible_expression: `assignment_expression` | `starred_expression` + flexible_expression_list: `flexible_expression` ("," `flexible_expression`)* [","] + starred_expression_list: `starred_expression` ("," `starred_expression`)* [","] expression_list: `expression` ("," `expression`)* [","] - starred_list: `starred_item` ("," `starred_item`)* [","] - starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `assignment_expression` | "*" `or_expr` + yield_list: `expression_list` | `starred_expression` "," [`starred_expression_list`] .. index:: pair: object; tuple @@ -1888,6 +1938,9 @@ the unpacking. .. versionadded:: 3.5 Iterable unpacking in expression lists, originally proposed by :pep:`448`. +.. versionadded:: 3.11 + Any item in an expression list may be starred. See :pep:`646`. + .. index:: pair: trailing; comma A trailing comma is required only to create a one-item tuple, diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index f8c9724114da9e..ac363e8cfa00dc 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -281,7 +281,7 @@ When the named module is not found in :data:`sys.modules`, Python next searches :data:`sys.meta_path`, which contains a list of meta path finder objects. These finders are queried in order to see if they know how to handle the named module. Meta path finders must implement a method called -:meth:`~importlib.abc.MetaPathFinder.find_spec()` which takes three arguments: +:meth:`~importlib.abc.MetaPathFinder.find_spec` which takes three arguments: a name, an import path, and (optionally) a target module. The meta path finder can use any strategy it wants to determine whether it can handle the named module or not. @@ -292,7 +292,7 @@ spec object. If it cannot handle the named module, it returns ``None``. If a spec, then a :exc:`ModuleNotFoundError` is raised. Any other exceptions raised are simply propagated up, aborting the import process. -The :meth:`~importlib.abc.MetaPathFinder.find_spec()` method of meta path +The :meth:`~importlib.abc.MetaPathFinder.find_spec` method of meta path finders is called with two or three arguments. The first is the fully qualified name of the module being imported, for example ``foo.bar.baz``. The second argument is the path entries to use for the module search. For @@ -513,8 +513,10 @@ holding is that if you have ``sys.modules['spam']`` and ``sys.modules['spam.foo']`` (as you would after the above import), the latter must appear as the ``foo`` attribute of the former. -Module spec ------------ +.. _module-specs: + +Module specs +------------ The import machinery uses a variety of information about each module during import, especially before loading. Most of the information is @@ -527,155 +529,44 @@ and the loader that executes it. Most importantly, it allows the import machinery to perform the boilerplate operations of loading, whereas without a module spec the loader had that responsibility. -The module's spec is exposed as the ``__spec__`` attribute on a module object. +The module's spec is exposed as :attr:`module.__spec__`. Setting +:attr:`!__spec__` appropriately applies equally to +:ref:`modules initialized during interpreter startup `. +The one exception is ``__main__``, where :attr:`!__spec__` is +:ref:`set to None in some cases `. + See :class:`~importlib.machinery.ModuleSpec` for details on the contents of the module spec. .. versionadded:: 3.4 -.. _import-mod-attrs: - -Import-related module attributes --------------------------------- - -The import machinery fills in these attributes on each module object -during loading, based on the module's spec, before the loader executes -the module. - -It is **strongly** recommended that you rely on :attr:`__spec__` and -its attributes instead of any of the other individual attributes -listed below. - -.. attribute:: __name__ - - The ``__name__`` attribute must be set to the fully qualified name of - the module. This name is used to uniquely identify the module in - the import system. - -.. attribute:: __loader__ - - The ``__loader__`` attribute must be set to the loader object that - the import machinery used when loading the module. This is mostly - for introspection, but can be used for additional loader-specific - functionality, for example getting data associated with a loader. - - It is **strongly** recommended that you rely on :attr:`__spec__` - instead of this attribute. - - .. versionchanged:: 3.12 - The value of ``__loader__`` is expected to be the same as - ``__spec__.loader``. The use of ``__loader__`` is deprecated and slated - for removal in Python 3.14. - -.. attribute:: __package__ - - The module's ``__package__`` attribute may be set. Its value must - be a string, but it can be the same value as its ``__name__``. When - the module is a package, its ``__package__`` value should be set to - its ``__name__``. When the module is not a package, ``__package__`` - should be set to the empty string for top-level modules, or for - submodules, to the parent package's name. See :pep:`366` for further - details. - - This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules, as defined in :pep:`366`. - - It is **strongly** recommended that you rely on :attr:`__spec__` - instead of this attribute. - - .. versionchanged:: 3.6 - The value of ``__package__`` is expected to be the same as - ``__spec__.parent``. - - .. versionchanged:: 3.10 - :exc:`ImportWarning` is raised if import falls back to - ``__package__`` instead of - :attr:`~importlib.machinery.ModuleSpec.parent`. - - .. versionchanged:: 3.12 - Raise :exc:`DeprecationWarning` instead of :exc:`ImportWarning` - when falling back to ``__package__``. - - -.. attribute:: __spec__ - - The ``__spec__`` attribute must be set to the module spec that was - used when importing the module. Setting ``__spec__`` - appropriately applies equally to :ref:`modules initialized during - interpreter startup `. The one exception is ``__main__``, - where ``__spec__`` is :ref:`set to None in some cases `. - - When ``__spec__.parent`` is not set, ``__package__`` is used as - a fallback. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.6 - ``__spec__.parent`` is used as a fallback when ``__package__`` is - not defined. - -.. attribute:: __path__ - - If the module is a package (either regular or namespace), the module - object's ``__path__`` attribute must be set. The value must be - iterable, but may be empty if ``__path__`` has no further significance. - If ``__path__`` is not empty, it must produce strings when iterated - over. More details on the semantics of ``__path__`` are given - :ref:`below `. - - Non-package modules should not have a ``__path__`` attribute. - -.. attribute:: __file__ -.. attribute:: __cached__ - - ``__file__`` is optional (if set, value must be a string). It indicates - the pathname of the file from which the module was loaded (if - loaded from a file), or the pathname of the shared library file - for extension modules loaded dynamically from a shared library. - It might be missing for certain types of modules, such as C - modules that are statically linked into the interpreter, and the - import system may opt to leave it unset if it has no semantic - meaning (e.g. a module loaded from a database). - - If ``__file__`` is set then the ``__cached__`` attribute might also - be set, which is the path to any compiled version of - the code (e.g. byte-compiled file). The file does not need to exist - to set this attribute; the path can simply point to where the - compiled file would exist (see :pep:`3147`). - - Note that ``__cached__`` may be set even if ``__file__`` is not - set. However, that scenario is quite atypical. Ultimately, the - loader is what makes use of the module spec provided by the finder - (from which ``__file__`` and ``__cached__`` are derived). So - if a loader can load from a cached module but otherwise does not load - from a file, that atypical scenario may be appropriate. - - It is **strongly** recommended that you rely on :attr:`__spec__` - instead of ``__cached__``. - .. _package-path-rules: -module.__path__ ---------------- +__path__ attributes on modules +------------------------------ -By definition, if a module has a ``__path__`` attribute, it is a package. +The :attr:`~module.__path__` attribute should be a (possibly empty) +:term:`sequence` of strings enumerating the locations where the package's +submodules will be found. By definition, if a module has a :attr:`!__path__` +attribute, it is a :term:`package`. -A package's ``__path__`` attribute is used during imports of its subpackages. +A package's :attr:`~module.__path__` attribute is used during imports of its +subpackages. Within the import machinery, it functions much the same as :data:`sys.path`, i.e. providing a list of locations to search for modules during import. -However, ``__path__`` is typically much more constrained than -:data:`sys.path`. +However, :attr:`!__path__` is typically much more constrained than +:data:`!sys.path`. -``__path__`` must be an iterable of strings, but it may be empty. The same rules used for :data:`sys.path` also apply to a package's -``__path__``, and :data:`sys.path_hooks` (described below) are -consulted when traversing a package's ``__path__``. +:attr:`!__path__`. :data:`sys.path_hooks` (described below) are +consulted when traversing a package's :attr:`!__path__`. -A package's ``__init__.py`` file may set or alter the package's ``__path__`` +A package's ``__init__.py`` file may set or alter the package's +:attr:`~module.__path__` attribute, and this was typically the way namespace packages were implemented prior to :pep:`420`. With the adoption of :pep:`420`, namespace packages no -longer need to supply ``__init__.py`` files containing only ``__path__`` -manipulation code; the import machinery automatically sets ``__path__`` +longer need to supply ``__init__.py`` files containing only :attr:`!__path__` +manipulation code; the import machinery automatically sets :attr:`!__path__` correctly for the namespace package. Module reprs diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index cf186705e6e987..b7b70e6be5a5b7 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -74,7 +74,7 @@ PyPy and a Just in Time compiler. One of the goals of the project is to encourage experimentation with the language itself by making it easier to modify the interpreter (since it is written in Python). Additional information is - available on `the PyPy project's home page `_. + available on `the PyPy project's home page `_. Each of these implementations varies in some way from the language as documented in this manual, or introduces specific information beyond what's covered in the diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 0adfb0365934e4..f7167032ad7df9 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -96,10 +96,9 @@ which is recognized also by GNU Emacs, and :: which is recognized by Bram Moolenaar's VIM. -If no encoding declaration is found, the default encoding is UTF-8. In -addition, if the first bytes of the file are the UTF-8 byte-order mark -(``b'\xef\xbb\xbf'``), the declared file encoding is UTF-8 (this is supported, -among others, by Microsoft's :program:`notepad`). +If no encoding declaration is found, the default encoding is UTF-8. If the +implicit or explicit encoding of a file is UTF-8, an initial UTF-8 byte-order +mark (b'\xef\xbb\xbf') is ignored rather than being a syntax error. If an encoding is declared, the encoding name must be recognized by Python (see :ref:`standard-encodings`). The @@ -285,11 +284,10 @@ UAX-31, with elaboration and changes as defined below; see also :pep:`3131` for further details. Within the ASCII range (U+0001..U+007F), the valid characters for identifiers -are the same as in Python 2.x: the uppercase and lowercase letters ``A`` through +include the uppercase and lowercase letters ``A`` through ``Z``, the underscore ``_`` and, except for the first character, the digits ``0`` through ``9``. - -Python 3.0 introduces additional characters from outside the ASCII range (see +Python 3.0 introduced additional characters from outside the ASCII range (see :pep:`3131`). For these characters, the classification uses the version of the Unicode Character Database as included in the :mod:`unicodedata` module. @@ -315,7 +313,7 @@ The Unicode category codes mentioned above stand for: * *Nd* - decimal numbers * *Pc* - connector punctuations * *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards + `_ to support backwards compatibility * *Other_ID_Continue* - likewise @@ -323,8 +321,8 @@ All identifiers are converted into the normal form NFKC while parsing; compariso of identifiers is based on NFKC. A non-normative HTML file listing all valid identifier characters for Unicode -15.1.0 can be found at -https://www.unicode.org/Public/15.1.0/ucd/DerivedCoreProperties.txt +16.0.0 can be found at +https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt .. _keywords: @@ -504,17 +502,15 @@ must be expressed with escapes. single: r"; raw string literal Both string and bytes literals may optionally be prefixed with a letter ``'r'`` -or ``'R'``; such strings are called :dfn:`raw strings` and treat backslashes as -literal characters. As a result, in string literals, ``'\U'`` and ``'\u'`` -escapes in raw strings are not treated specially. Given that Python 2.x's raw -unicode literals behave differently than Python 3.x's the ``'ur'`` syntax -is not supported. +or ``'R'``; such constructs are called :dfn:`raw string literals` +and :dfn:`raw bytes literals` respectively and treat backslashes as +literal characters. As a result, in raw string literals, ``'\U'`` and ``'\u'`` +escapes are not treated specially. .. versionadded:: 3.3 The ``'rb'`` prefix of raw bytes literals has been added as a synonym of ``'br'``. -.. versionadded:: 3.3 Support for the unicode legacy literal (``u'value'``) was reintroduced to simplify the maintenance of dual Python 2.x and 3.x codebases. See :pep:`414` for more information. @@ -734,7 +730,7 @@ for the contents of the string is: : ("," `conditional_expression` | "," "*" `or_expr`)* [","] : | `yield_expression` conversion: "s" | "r" | "a" - format_spec: (`literal_char` | NULL | `replacement_field`)* + format_spec: (`literal_char` | `replacement_field`)* literal_char: The parts of the string outside curly braces are treated literally, @@ -881,10 +877,10 @@ Numeric literals ---------------- .. index:: number, numeric literal, integer literal - floating point literal, hexadecimal literal + floating-point literal, hexadecimal literal octal literal, binary literal, decimal literal, imaginary literal, complex literal -There are three types of numeric literals: integers, floating point numbers, and +There are three types of numeric literals: integers, floating-point numbers, and imaginary numbers. There are no complex literals (complex numbers can be formed by adding a real number and an imaginary number). @@ -945,10 +941,10 @@ Some examples of integer literals:: single: _ (underscore); in numeric literal .. _floating: -Floating point literals +Floating-point literals ----------------------- -Floating point literals are described by the following lexical definitions: +Floating-point literals are described by the following lexical definitions: .. productionlist:: python-grammar floatnumber: `pointfloat` | `exponentfloat` @@ -960,10 +956,10 @@ Floating point literals are described by the following lexical definitions: Note that the integer and exponent parts are always interpreted using radix 10. For example, ``077e010`` is legal, and denotes the same number as ``77e10``. The -allowed range of floating point literals is implementation-dependent. As in +allowed range of floating-point literals is implementation-dependent. As in integer literals, underscores are supported for digit grouping. -Some examples of floating point literals:: +Some examples of floating-point literals:: 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 @@ -984,9 +980,9 @@ Imaginary literals are described by the following lexical definitions: imagnumber: (`floatnumber` | `digitpart`) ("j" | "J") An imaginary literal yields a complex number with a real part of 0.0. Complex -numbers are represented as a pair of floating point numbers and have the same +numbers are represented as a pair of floating-point numbers and have the same restrictions on their range. To create a complex number with a nonzero real -part, add a floating point number to it, e.g., ``(3+4j)``. Some examples of +part, add a floating-point number to it, e.g., ``(3+4j)``. Some examples of imaginary literals:: 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j @@ -1021,9 +1017,9 @@ The following tokens serve as delimiters in the grammar: .. code-block:: none ( ) [ ] { } - , : . ; @ = -> - += -= *= /= //= %= @= - &= |= ^= >>= <<= **= + , : ! . ; @ = + -> += -= *= /= //= %= + @= &= |= ^= >>= <<= **= The period can also occur in floating-point and imaginary literals. A sequence of three periods has a special meaning as an ellipsis literal. The second half @@ -1047,4 +1043,4 @@ occurrence outside string literals and comments is an unconditional error: .. rubric:: Footnotes -.. [#] https://www.unicode.org/Public/15.1.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/16.0.0/ucd/NameAliases.txt diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 04132c78ce77a6..24df4a6ba7b678 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -293,7 +293,7 @@ statements, cannot be an unpacking) and the expression list, performs the binary operation specific to the type of assignment on the two operands, and assigns the result to the original target. The target is only evaluated once. -An augmented assignment expression like ``x += 1`` can be rewritten as ``x = x + +An augmented assignment statement like ``x += 1`` can be rewritten as ``x = x + 1`` to achieve a similar, but not exactly equal effect. In the augmented version, ``x`` is only evaluated once. Also, when possible, the actual operation is performed *in-place*, meaning that rather than creating a new object and @@ -333,23 +333,24 @@ statement, of a variable or attribute annotation and an optional assignment stat The difference from normal :ref:`assignment` is that only a single target is allowed. -For simple names as assignment targets, if in class or module scope, -the annotations are evaluated and stored in a special class or module -attribute :attr:`__annotations__` -that is a dictionary mapping from variable names (mangled if private) to -evaluated annotations. This attribute is writable and is automatically -created at the start of class or module body execution, if annotations -are found statically. +The assignment target is considered "simple" if it consists of a single +name that is not enclosed in parentheses. +For simple assignment targets, if in class or module scope, +the annotations are gathered in a lazily evaluated +:ref:`annotation scope `. The annotations can be +evaluated using the :attr:`~object.__annotations__` attribute of a +class or module, or using the facilities in the :mod:`annotationlib` +module. -For expressions as assignment targets, the annotations are evaluated if -in class or module scope, but not stored. +If the assignment target is not simple (an attribute, subscript node, or +parenthesized name), the annotation is never evaluated. If a name is annotated in a function scope, then this name is local for that scope. Annotations are never evaluated and stored in function scopes. If the right hand side is present, an annotated -assignment performs the actual assignment before evaluating annotations -(where applicable). If the right hand side is not present for an expression +assignment performs the actual assignment as if there was no annotation +present. If the right hand side is not present for an expression target, then the interpreter evaluates the target except for the last :meth:`~object.__setitem__` or :meth:`~object.__setattr__` call. @@ -370,6 +371,10 @@ target, then the interpreter evaluates the target except for the last regular assignments. Previously, some expressions (like un-parenthesized tuple expressions) caused a syntax error. +.. versionchanged:: 3.14 + Annotations are now lazily evaluated in a separate :ref:`annotation scope `. + If the assignment target is not simple, annotations are never evaluated. + .. _assert: @@ -664,8 +669,7 @@ and information about handling exceptions is in section :ref:`try`. .. versionchanged:: 3.3 :const:`None` is now permitted as ``Y`` in ``raise X from Y``. -.. versionadded:: 3.3 - The :attr:`~BaseException.__suppress_context__` attribute to suppress + Added the :attr:`~BaseException.__suppress_context__` attribute to suppress automatic display of the exception context. .. versionchanged:: 3.11 @@ -973,8 +977,8 @@ block textually preceding that :keyword:`!global` statement. Names listed in a :keyword:`global` statement must not be defined as formal parameters, or as targets in :keyword:`with` statements or :keyword:`except` clauses, or in a :keyword:`for` target list, :keyword:`class` -definition, function definition, :keyword:`import` statement, or variable -annotation. +definition, function definition, :keyword:`import` statement, or +:term:`variable annotations `. .. impl-detail:: @@ -1007,25 +1011,29 @@ The :keyword:`!nonlocal` statement .. productionlist:: python-grammar nonlocal_stmt: "nonlocal" `identifier` ("," `identifier`)* -The :keyword:`nonlocal` statement causes the listed identifiers to refer to -previously bound variables in the nearest enclosing scope excluding globals. -This is important because the default behavior for binding is to search the -local namespace first. The statement allows encapsulated code to rebind -variables outside of the local scope besides the global (module) scope. - -Names listed in a :keyword:`nonlocal` statement, unlike those listed in a -:keyword:`global` statement, must refer to pre-existing bindings in an -enclosing scope (the scope in which a new binding should be created cannot -be determined unambiguously). +When the definition of a function or class is nested (enclosed) within +the definitions of other functions, its nonlocal scopes are the local +scopes of the enclosing functions. The :keyword:`nonlocal` statement +causes the listed identifiers to refer to names previously bound in +nonlocal scopes. It allows encapsulated code to rebind such nonlocal +identifiers. If a name is bound in more than one nonlocal scope, the +nearest binding is used. If a name is not bound in any nonlocal scope, +or if there is no nonlocal scope, a :exc:`SyntaxError` is raised. -Names listed in a :keyword:`nonlocal` statement must not collide with -pre-existing bindings in the local scope. +The nonlocal statement applies to the entire scope of a function or +class body. A :exc:`SyntaxError` is raised if a variable is used or +assigned to prior to its nonlocal declaration in the scope. .. seealso:: :pep:`3104` - Access to Names in Outer Scopes The specification for the :keyword:`nonlocal` statement. +**Programmer's note:** :keyword:`nonlocal` is a directive to the parser +and applies only to code parsed along with it. See the note for the +:keyword:`global` statement. + + .. _type: The :keyword:`!type` statement diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index 597341d99ffeaa..3483faea6b56cb 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -7,29 +7,29 @@ blurb python-docs-theme>=2022.1 # Generated from: -# pip install "Sphinx~=4.2.0" +# pip install "Sphinx~=7.2.6" # pip freeze # -# Sphinx 4.2 comes from ``needs_sphinx = '4.2'`` in ``Doc/conf.py``. +# Sphinx 7.2.6 comes from ``needs_sphinx = '7.2.6'`` in ``Doc/conf.py``. -alabaster==0.7.13 -Babel==2.13.0 -certifi==2023.7.22 -charset-normalizer==3.3.0 -docutils==0.17.1 -idna==3.4 +alabaster==0.7.16 +Babel==2.16.0 +certifi==2024.8.30 +charset-normalizer==3.4.0 +docutils==0.20.1 +idna==3.10 imagesize==1.4.1 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -packaging==23.2 -Pygments==2.16.1 -requests==2.31.0 +Jinja2==3.1.4 +MarkupSafe==3.0.1 +packaging==24.1 +Pygments==2.18.0 +requests==2.32.3 snowballstemmer==2.2.0 -Sphinx==4.2.0 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 +Sphinx==7.2.6 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -urllib3==2.0.7 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 +urllib3==2.2.3 diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 118e6c322b4be2..5105786ccf283c 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -6,13 +6,12 @@ # Sphinx version is pinned so that new versions that introduce new warnings # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. -sphinx~=7.2.0 +sphinx~=8.1.0 blurb -sphinx-autobuild -sphinxext-opengraph==0.7.5 -sphinx-notfound-page==1.0.0 +sphinxext-opengraph~=0.9.0 +sphinx-notfound-page~=1.0.0 # The theme used by the documentation is stored separately, so we need # to install that as well. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 9db02c5c3c73c9..66914f79f3d4ec 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -3,15 +3,12 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. Doc/c-api/descriptor.rst -Doc/c-api/exceptions.rst Doc/c-api/float.rst -Doc/c-api/gcsupport.rst Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst Doc/c-api/module.rst Doc/c-api/stable.rst -Doc/c-api/sys.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst Doc/extending/extending.rst @@ -20,9 +17,7 @@ Doc/howto/descriptor.rst Doc/howto/enum.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst -Doc/library/asyncio-policy.rst Doc/library/asyncio-subprocess.rst -Doc/library/bdb.rst Doc/library/collections.rst Doc/library/dbm.rst Doc/library/decimal.rst @@ -32,8 +27,6 @@ Doc/library/email.errors.rst Doc/library/email.parser.rst Doc/library/email.policy.rst Doc/library/exceptions.rst -Doc/library/faulthandler.rst -Doc/library/fcntl.rst Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst @@ -49,7 +42,6 @@ Doc/library/pickletools.rst Doc/library/platform.rst Doc/library/plistlib.rst Doc/library/profile.rst -Doc/library/pydoc.rst Doc/library/pyexpat.rst Doc/library/readline.rst Doc/library/resource.rst @@ -83,15 +75,11 @@ Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/tutorial/datastructures.rst Doc/using/windows.rst -Doc/whatsnew/2.0.rst -Doc/whatsnew/2.1.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst Doc/whatsnew/2.7.rst Doc/whatsnew/3.0.rst -Doc/whatsnew/3.1.rst -Doc/whatsnew/3.2.rst Doc/whatsnew/3.3.rst Doc/whatsnew/3.4.rst Doc/whatsnew/3.5.rst diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index 809a8d63087e12..c686eecf8d9271 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -2,6 +2,7 @@ """ Check the output of running Sphinx in nit-picky mode (missing references). """ + from __future__ import annotations import argparse @@ -13,6 +14,9 @@ from pathlib import Path from typing import TextIO +# Fail if NEWS nit found before this line number +NEWS_NIT_THRESHOLD = 1700 + # Exclude these whether they're dirty or clean, # because they trigger a rebuild of dirty files. EXCLUDE_FILES = { @@ -203,7 +207,9 @@ def annotate_diff( def fail_if_regression( - warnings: list[str], files_with_expected_nits: set[str], files_with_nits: set[str] + warnings: list[str], + files_with_expected_nits: set[str], + files_with_nits: set[str], ) -> int: """ Ensure some files always pass Sphinx nit-picky mode (no missing references). @@ -245,6 +251,26 @@ def fail_if_improved( return 0 +def fail_if_new_news_nit(warnings: list[str], threshold: int) -> int: + """ + Ensure no warnings are found in the NEWS file before a given line number. + """ + news_nits = (warning for warning in warnings if "/build/NEWS:" in warning) + + # Nits found before the threshold line + new_news_nits = [ + nit for nit in news_nits if int(nit.split(":")[1]) <= threshold + ] + + if new_news_nits: + print("\nError: new NEWS nits:\n") + for warning in new_news_nits: + print(warning) + return -1 + + return 0 + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( @@ -264,6 +290,14 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Fail if new files with no nits are found", ) + parser.add_argument( + "--fail-if-new-news-nit", + metavar="threshold", + type=int, + nargs="?", + const=NEWS_NIT_THRESHOLD, + help="Fail if new NEWS nit found before threshold line number", + ) args = parser.parse_args(argv) if args.annotate_diff is not None and len(args.annotate_diff) > 2: @@ -274,7 +308,8 @@ def main(argv: list[str] | None = None) -> int: exit_code = 0 wrong_directory_msg = "Must run this script from the repo root" - assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg + if not Path("Doc").exists() or not Path("Doc").is_dir(): + raise RuntimeError(wrong_directory_msg) with Path("Doc/sphinx-warnings.txt").open(encoding="UTF-8") as f: warnings = f.read().splitlines() @@ -302,7 +337,12 @@ def main(argv: list[str] | None = None) -> int: ) if args.fail_if_improved: - exit_code += fail_if_improved(files_with_expected_nits, files_with_nits) + exit_code += fail_if_improved( + files_with_expected_nits, files_with_nits + ) + + if args.fail_if_new_news_nit: + exit_code += fail_if_new_news_nit(warnings, args.fail_if_new_news_nit) return exit_code diff --git a/Doc/tools/extensions/audit_events.py b/Doc/tools/extensions/audit_events.py new file mode 100644 index 00000000000000..23d82c0f4414bf --- /dev/null +++ b/Doc/tools/extensions/audit_events.py @@ -0,0 +1,264 @@ +"""Support for documenting audit events.""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from docutils import nodes +from sphinx.errors import NoUri +from sphinx.locale import _ as sphinx_gettext +from sphinx.transforms.post_transforms import SphinxPostTransform +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sphinx.application import Sphinx + from sphinx.builders import Builder + from sphinx.environment import BuildEnvironment + +logger = logging.getLogger(__name__) + +# This list of sets are allowable synonyms for event argument names. +# If two names are in the same set, they are treated as equal for the +# purposes of warning. This won't help if the number of arguments is +# different! +_SYNONYMS = [ + frozenset({"file", "path", "fd"}), +] + + +class AuditEvents: + def __init__(self) -> None: + self.events: dict[str, list[str]] = {} + self.sources: dict[str, list[tuple[str, str]]] = {} + + def __iter__(self) -> Iterator[tuple[str, list[str], tuple[str, str]]]: + for name, args in self.events.items(): + for source in self.sources[name]: + yield name, args, source + + def add_event( + self, name, args: list[str], source: tuple[str, str] + ) -> None: + if name in self.events: + self._check_args_match(name, args) + else: + self.events[name] = args + self.sources.setdefault(name, []).append(source) + + def _check_args_match(self, name: str, args: list[str]) -> None: + current_args = self.events[name] + msg = ( + f"Mismatched arguments for audit-event {name}: " + f"{current_args!r} != {args!r}" + ) + if current_args == args: + return + if len(current_args) != len(args): + logger.warning(msg) + return + for a1, a2 in zip(current_args, args, strict=False): + if a1 == a2: + continue + if any(a1 in s and a2 in s for s in _SYNONYMS): + continue + logger.warning(msg) + return + + def id_for(self, name) -> str: + source_count = len(self.sources.get(name, ())) + name_clean = re.sub(r"\W", "_", name) + return f"audit_event_{name_clean}_{source_count}" + + def rows(self) -> Iterator[tuple[str, list[str], list[tuple[str, str]]]]: + for name in sorted(self.events.keys()): + yield name, self.events[name], self.sources[name] + + +def initialise_audit_events(app: Sphinx) -> None: + """Initialise the audit_events attribute on the environment.""" + if not hasattr(app.env, "audit_events"): + app.env.audit_events = AuditEvents() + + +def audit_events_purge( + app: Sphinx, env: BuildEnvironment, docname: str +) -> None: + """This is to remove traces of removed documents from env.audit_events.""" + fresh_audit_events = AuditEvents() + for name, args, (doc, target) in env.audit_events: + if doc != docname: + fresh_audit_events.add_event(name, args, (doc, target)) + + +def audit_events_merge( + app: Sphinx, + env: BuildEnvironment, + docnames: list[str], + other: BuildEnvironment, +) -> None: + """In Sphinx parallel builds, this merges audit_events from subprocesses.""" + for name, args, source in other.audit_events: + env.audit_events.add_event(name, args, source) + + +class AuditEvent(SphinxDirective): + has_content = True + required_arguments = 1 + optional_arguments = 2 + final_argument_whitespace = True + + _label = [ + sphinx_gettext( + "Raises an :ref:`auditing event ` " + "{name} with no arguments." + ), + sphinx_gettext( + "Raises an :ref:`auditing event ` " + "{name} with argument {args}." + ), + sphinx_gettext( + "Raises an :ref:`auditing event ` " + "{name} with arguments {args}." + ), + ] + + def run(self) -> list[nodes.paragraph]: + name = self.arguments[0] + if len(self.arguments) >= 2 and self.arguments[1]: + args = [ + arg + for argument in self.arguments[1].strip("'\"").split(",") + if (arg := argument.strip()) + ] + else: + args = [] + ids = [] + try: + target = self.arguments[2].strip("\"'") + except (IndexError, TypeError): + target = None + if not target: + target = self.env.audit_events.id_for(name) + ids.append(target) + self.env.audit_events.add_event(name, args, (self.env.docname, target)) + + node = nodes.paragraph("", classes=["audit-hook"], ids=ids) + self.set_source_info(node) + if self.content: + node.rawsource = '\n'.join(self.content) # for gettext + self.state.nested_parse(self.content, self.content_offset, node) + else: + num_args = min(2, len(args)) + text = self._label[num_args].format( + name=f"``{name}``", + args=", ".join(f"``{a}``" for a in args), + ) + node.rawsource = text # for gettext + parsed, messages = self.state.inline_text(text, self.lineno) + node += parsed + node += messages + return [node] + + +class audit_event_list(nodes.General, nodes.Element): # noqa: N801 + pass + + +class AuditEventListDirective(SphinxDirective): + def run(self) -> list[audit_event_list]: + return [audit_event_list()] + + +class AuditEventListTransform(SphinxPostTransform): + default_priority = 500 + + def run(self) -> None: + if self.document.next_node(audit_event_list) is None: + return + + table = self._make_table(self.app.builder, self.env.docname) + for node in self.document.findall(audit_event_list): + node.replace_self(table) + + def _make_table(self, builder: Builder, docname: str) -> nodes.table: + table = nodes.table(cols=3) + group = nodes.tgroup( + "", + nodes.colspec(colwidth=30), + nodes.colspec(colwidth=55), + nodes.colspec(colwidth=15), + cols=3, + ) + head = nodes.thead() + body = nodes.tbody() + + table += group + group += head + group += body + + head += nodes.row( + "", + nodes.entry("", nodes.paragraph("", "Audit event")), + nodes.entry("", nodes.paragraph("", "Arguments")), + nodes.entry("", nodes.paragraph("", "References")), + ) + + for name, args, sources in builder.env.audit_events.rows(): + body += self._make_row(builder, docname, name, args, sources) + + return table + + @staticmethod + def _make_row( + builder: Builder, + docname: str, + name: str, + args: list[str], + sources: list[tuple[str, str]], + ) -> nodes.row: + row = nodes.row() + name_node = nodes.paragraph("", nodes.Text(name)) + row += nodes.entry("", name_node) + + args_node = nodes.paragraph() + for arg in args: + args_node += nodes.literal(arg, arg) + args_node += nodes.Text(", ") + if len(args_node.children) > 0: + args_node.children.pop() # remove trailing comma + row += nodes.entry("", args_node) + + backlinks_node = nodes.paragraph() + backlinks = enumerate(sorted(set(sources)), start=1) + for i, (doc, label) in backlinks: + if isinstance(label, str): + ref = nodes.reference("", f"[{i}]", internal=True) + try: + target = ( + f"{builder.get_relative_uri(docname, doc)}#{label}" + ) + except NoUri: + continue + else: + ref["refuri"] = target + backlinks_node += ref + row += nodes.entry("", backlinks_node) + return row + + +def setup(app: Sphinx): + app.add_directive("audit-event", AuditEvent) + app.add_directive("audit-event-table", AuditEventListDirective) + app.add_post_transform(AuditEventListTransform) + app.connect("builder-inited", initialise_audit_events) + app.connect("env-purge-doc", audit_events_purge) + app.connect("env-merge-info", audit_events_merge) + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/availability.py b/Doc/tools/extensions/availability.py new file mode 100644 index 00000000000000..47833fdcb87590 --- /dev/null +++ b/Doc/tools/extensions/availability.py @@ -0,0 +1,125 @@ +"""Support for documenting platform availability""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from docutils import nodes +from sphinx import addnodes +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective + +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata + +logger = logging.getLogger("availability") + +# known platform, libc, and threading implementations +_PLATFORMS = frozenset({ + "AIX", + "Android", + "BSD", + "DragonFlyBSD", + "Emscripten", + "FreeBSD", + "GNU/kFreeBSD", + "iOS", + "Linux", + "macOS", + "NetBSD", + "OpenBSD", + "POSIX", + "Solaris", + "Unix", + "VxWorks", + "WASI", + "Windows", +}) +_LIBC = frozenset({ + "BSD libc", + "glibc", + "musl", +}) +_THREADING = frozenset({ + # POSIX platforms with pthreads + "pthreads", +}) +KNOWN_PLATFORMS = _PLATFORMS | _LIBC | _THREADING + + +class Availability(SphinxDirective): + has_content = True + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + + def run(self) -> list[nodes.container]: + title = "Availability" + refnode = addnodes.pending_xref( + title, + nodes.inline(title, title, classes=["xref", "std", "std-ref"]), + refdoc=self.env.docname, + refdomain="std", + refexplicit=True, + reftarget="availability", + reftype="ref", + refwarn=True, + ) + sep = nodes.Text(": ") + parsed, msgs = self.state.inline_text(self.arguments[0], self.lineno) + pnode = nodes.paragraph(title, "", refnode, sep, *parsed, *msgs) + self.set_source_info(pnode) + cnode = nodes.container("", pnode, classes=["availability"]) + self.set_source_info(cnode) + if self.content: + self.state.nested_parse(self.content, self.content_offset, cnode) + self.parse_platforms() + + return [cnode] + + def parse_platforms(self) -> dict[str, str | bool]: + """Parse platform information from arguments + + Arguments is a comma-separated string of platforms. A platform may + be prefixed with "not " to indicate that a feature is not available. + + Example:: + + .. availability:: Windows, Linux >= 4.2, not WASI + + Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not + parsed into separate tokens. + """ + platforms = {} + for arg in self.arguments[0].rstrip(".").split(","): + arg = arg.strip() + platform, _, version = arg.partition(" >= ") + if platform.startswith("not "): + version = False + platform = platform.removeprefix("not ") + elif not version: + version = True + platforms[platform] = version + + if unknown := set(platforms).difference(KNOWN_PLATFORMS): + logger.warning( + "Unknown platform%s or syntax '%s' in '.. availability:: %s', " + "see %s:KNOWN_PLATFORMS for a set of known platforms.", + "s" if len(platforms) != 1 else "", + " ".join(sorted(unknown)), + self.arguments[0], + __file__, + ) + + return platforms + + +def setup(app: Sphinx) -> ExtensionMetadata: + app.add_directive("availability", Availability) + + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index a8b6d8995e3f40..50065d34a2c27a 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -1,235 +1,302 @@ -# -*- coding: utf-8 -*- -""" - c_annotations.py - ~~~~~~~~~~~~~~~~ - - Supports annotations for C API elements: +"""Support annotations for C API elements. - * reference count annotations for C API functions. Based on - refcount.py and anno-api.py in the old Python documentation tools. +* Reference count annotations for C API functions. +* Stable ABI annotations +* Limited API annotations - * stable API annotations +Configuration: +* Set ``refcount_file`` to the path to the reference count data file. +* Set ``stable_abi_file`` to the path to stable ABI list. +""" - Usage: - * Set the `refcount_file` config value to the path to the reference - count data file. - * Set the `stable_abi_file` config value to the path to stable ABI list. +from __future__ import annotations - :copyright: Copyright 2007-2014 by Georg Brandl. - :license: Python license. -""" +import csv +import dataclasses +from pathlib import Path +from typing import TYPE_CHECKING -from os import path -import docutils +import sphinx from docutils import nodes -from docutils.parsers.rst import directives -from docutils.parsers.rst import Directive from docutils.statemachine import StringList -from sphinx.locale import _ as sphinx_gettext -import csv - from sphinx import addnodes -from sphinx.domains.c import CObject - - -REST_ROLE_MAP = { - 'function': 'func', - 'var': 'data', - 'type': 'type', - 'macro': 'macro', - 'type': 'type', - 'member': 'member', +from sphinx.locale import _ as sphinx_gettext +from sphinx.util.docutils import SphinxDirective + +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata + +ROLE_TO_OBJECT_TYPE = { + "func": "function", + "macro": "macro", + "member": "member", + "type": "type", + "data": "var", } -# Monkeypatch nodes.Node.findall for forwards compatibility -# This patch can be dropped when the minimum Sphinx version is 4.4.0 -# or the minimum Docutils version is 0.18.1. -if docutils.__version_info__ < (0, 18, 1): - def findall(self, *args, **kwargs): - return iter(self.traverse(*args, **kwargs)) - - nodes.Node.findall = findall - - -class RCEntry: - def __init__(self, name): - self.name = name - self.args = [] - self.result_type = '' - self.result_refs = None - - -class Annotations: - def __init__(self, refcount_filename, stable_abi_file): - self.refcount_data = {} - with open(refcount_filename, 'r') as fp: - for line in fp: - line = line.strip() - if line[:1] in ("", "#"): - # blank lines and comments - continue - parts = line.split(":", 4) - if len(parts) != 5: - raise ValueError("Wrong field count in %r" % line) - function, type, arg, refcount, comment = parts - # Get the entry, creating it if needed: - try: - entry = self.refcount_data[function] - except KeyError: - entry = self.refcount_data[function] = RCEntry(function) - if not refcount or refcount == "null": - refcount = None - else: - refcount = int(refcount) - # Update the entry with the new parameter or the result - # information. - if arg: - entry.args.append((arg, type, refcount)) - else: - entry.result_type = type - entry.result_refs = refcount - - self.stable_abi_data = {} - with open(stable_abi_file, 'r') as fp: - for record in csv.DictReader(fp): - role = record['role'] - name = record['name'] - self.stable_abi_data[name] = record - - def add_annotations(self, app, doctree): - for node in doctree.findall(addnodes.desc_content): - par = node.parent - if par['domain'] != 'c': - continue - if not par[0].has_key('ids') or not par[0]['ids']: - continue - name = par[0]['ids'][0] - if name.startswith("c."): - name = name[2:] - - objtype = par['objtype'] - - # Stable ABI annotation. These have two forms: - # Part of the [Stable ABI](link). - # Part of the [Stable ABI](link) since version X.Y. - # For structs, there's some more info in the message: - # Part of the [Limited API](link) (as an opaque struct). - # Part of the [Stable ABI](link) (including all members). - # Part of the [Limited API](link) (Only some members are part - # of the stable ABI.). - # ... all of which can have "since version X.Y" appended. - record = self.stable_abi_data.get(name) - if record: - if record['role'] != objtype: - raise ValueError( - f"Object type mismatch in limited API annotation " - f"for {name}: {record['role']!r} != {objtype!r}") - stable_added = record['added'] - message = sphinx_gettext('Part of the') - message = message.center(len(message) + 2) - emph_node = nodes.emphasis(message, message, - classes=['stableabi']) - ref_node = addnodes.pending_xref( - 'Stable ABI', refdomain="std", reftarget='stable', - reftype='ref', refexplicit="False") - struct_abi_kind = record['struct_abi_kind'] - if struct_abi_kind in {'opaque', 'members'}: - ref_node += nodes.Text(sphinx_gettext('Limited API')) - else: - ref_node += nodes.Text(sphinx_gettext('Stable ABI')) - emph_node += ref_node - if struct_abi_kind == 'opaque': - emph_node += nodes.Text(' ' + sphinx_gettext('(as an opaque struct)')) - elif struct_abi_kind == 'full-abi': - emph_node += nodes.Text(' ' + sphinx_gettext('(including all members)')) - if record['ifdef_note']: - emph_node += nodes.Text(' ' + record['ifdef_note']) - if stable_added == '3.2': - # Stable ABI was introduced in 3.2. - pass - else: - emph_node += nodes.Text(' ' + sphinx_gettext('since version %s') % stable_added) - emph_node += nodes.Text('.') - if struct_abi_kind == 'members': - emph_node += nodes.Text( - ' ' + sphinx_gettext('(Only some members are part of the stable ABI.)')) - node.insert(0, emph_node) - - # Unstable API annotation. - if name.startswith('PyUnstable'): - warn_node = nodes.admonition( - classes=['unstable-c-api', 'warning']) - message = sphinx_gettext('This is') + ' ' - emph_node = nodes.emphasis(message, message) - ref_node = addnodes.pending_xref( - 'Unstable API', refdomain="std", - reftarget='unstable-c-api', - reftype='ref', refexplicit="False") - ref_node += nodes.Text(sphinx_gettext('Unstable API')) - emph_node += ref_node - emph_node += nodes.Text(sphinx_gettext('. It may change without warning in minor releases.')) - warn_node += emph_node - node.insert(0, warn_node) - - # Return value annotation - if objtype != 'function': - continue - entry = self.refcount_data.get(name) - if not entry: - continue - elif not entry.result_type.endswith("Object*"): - continue - if entry.result_refs is None: - rc = sphinx_gettext('Return value: Always NULL.') - elif entry.result_refs: - rc = sphinx_gettext('Return value: New reference.') - else: - rc = sphinx_gettext('Return value: Borrowed reference.') - node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) - - -def init_annotations(app): - annotations = Annotations( - path.join(app.srcdir, app.config.refcount_file), - path.join(app.srcdir, app.config.stable_abi_file), +@dataclasses.dataclass(slots=True) +class RefCountEntry: + # Name of the function. + name: str + # List of (argument name, type, refcount effect) tuples. + # (Currently not used. If it was, a dataclass might work better.) + args: list = dataclasses.field(default_factory=list) + # Return type of the function. + result_type: str = "" + # Reference count effect for the return value. + result_refs: int | None = None + + +@dataclasses.dataclass(frozen=True, slots=True) +class StableABIEntry: + # Role of the object. + # Source: Each [item_kind] in stable_abi.toml is mapped to a C Domain role. + role: str + # Name of the object. + # Source: [.*] in stable_abi.toml. + name: str + # Version when the object was added to the stable ABI. + # (Source: [.*.added] in stable_abi.toml. + added: str + # An explananatory blurb for the ifdef. + # Source: ``feature_macro.*.doc`` in stable_abi.toml. + ifdef_note: str + # Defines how much of the struct is exposed. Only relevant for structs. + # Source: [.*.struct_abi_kind] in stable_abi.toml. + struct_abi_kind: str + + +def read_refcount_data(refcount_filename: Path) -> dict[str, RefCountEntry]: + refcount_data = {} + refcounts = refcount_filename.read_text(encoding="utf8") + for line in refcounts.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + # blank lines and comments + continue + + # Each line is of the form + # function ':' type ':' [param name] ':' [refcount effect] ':' [comment] + parts = line.split(":", 4) + if len(parts) != 5: + raise ValueError(f"Wrong field count in {line!r}") + function, type, arg, refcount, _comment = parts + + # Get the entry, creating it if needed: + try: + entry = refcount_data[function] + except KeyError: + entry = refcount_data[function] = RefCountEntry(function) + if not refcount or refcount == "null": + refcount = None + else: + refcount = int(refcount) + # Update the entry with the new parameter + # or the result information. + if arg: + entry.args.append((arg, type, refcount)) + else: + entry.result_type = type + entry.result_refs = refcount + + return refcount_data + + +def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]: + stable_abi_data = {} + with open(stable_abi_file, encoding="utf8") as fp: + for record in csv.DictReader(fp): + name = record["name"] + stable_abi_data[name] = StableABIEntry(**record) + + return stable_abi_data + + +def add_annotations(app: Sphinx, doctree: nodes.document) -> None: + state = app.env.domaindata["c_annotations"] + refcount_data = state["refcount_data"] + stable_abi_data = state["stable_abi_data"] + for node in doctree.findall(addnodes.desc_content): + par = node.parent + if par["domain"] != "c": + continue + if not par[0].get("ids", None): + continue + name = par[0]["ids"][0].removeprefix("c.") + objtype = par["objtype"] + + # Stable ABI annotation. + if record := stable_abi_data.get(name): + if ROLE_TO_OBJECT_TYPE[record.role] != objtype: + msg = ( + f"Object type mismatch in limited API annotation for {name}: " + f"{ROLE_TO_OBJECT_TYPE[record.role]!r} != {objtype!r}" + ) + raise ValueError(msg) + annotation = _stable_abi_annotation(record) + node.insert(0, annotation) + + # Unstable API annotation. + if name.startswith("PyUnstable"): + annotation = _unstable_api_annotation() + node.insert(0, annotation) + + # Return value annotation + if objtype != "function": + continue + if name not in refcount_data: + continue + entry = refcount_data[name] + if not entry.result_type.endswith("Object*"): + continue + annotation = _return_value_annotation(entry.result_refs) + node.insert(0, annotation) + + +def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: + """Create the Stable ABI annotation. + + These have two forms: + Part of the `Stable ABI `_. + Part of the `Stable ABI `_ since version X.Y. + For structs, there's some more info in the message: + Part of the `Limited API `_ (as an opaque struct). + Part of the `Stable ABI `_ (including all members). + Part of the `Limited API `_ (Only some members are part + of the stable ABI.). + ... all of which can have "since version X.Y" appended. + """ + stable_added = record.added + message = sphinx_gettext("Part of the") + message = message.center(len(message) + 2) + emph_node = nodes.emphasis(message, message, classes=["stableabi"]) + ref_node = addnodes.pending_xref( + "Stable ABI", + refdomain="std", + reftarget="stable", + reftype="ref", + refexplicit="False", + ) + struct_abi_kind = record.struct_abi_kind + if struct_abi_kind in {"opaque", "members"}: + ref_node += nodes.Text(sphinx_gettext("Limited API")) + else: + ref_node += nodes.Text(sphinx_gettext("Stable ABI")) + emph_node += ref_node + if struct_abi_kind == "opaque": + emph_node += nodes.Text(" " + sphinx_gettext("(as an opaque struct)")) + elif struct_abi_kind == "full-abi": + emph_node += nodes.Text( + " " + sphinx_gettext("(including all members)") + ) + if record.ifdef_note: + emph_node += nodes.Text(f" {record.ifdef_note}") + if stable_added == "3.2": + # Stable ABI was introduced in 3.2. + pass + else: + emph_node += nodes.Text( + " " + sphinx_gettext("since version %s") % stable_added + ) + emph_node += nodes.Text(".") + if struct_abi_kind == "members": + msg = " " + sphinx_gettext( + "(Only some members are part of the stable ABI.)" + ) + emph_node += nodes.Text(msg) + return emph_node + + +def _unstable_api_annotation() -> nodes.admonition: + ref_node = addnodes.pending_xref( + "Unstable API", + nodes.Text(sphinx_gettext("Unstable API")), + refdomain="std", + reftarget="unstable-c-api", + reftype="ref", + refexplicit="False", + ) + emph_node = nodes.emphasis( + "This is ", + sphinx_gettext("This is") + " ", + ref_node, + nodes.Text( + sphinx_gettext( + ". It may change without warning in minor releases." + ) + ), + ) + return nodes.admonition( + "", + emph_node, + classes=["unstable-c-api", "warning"], ) - app.connect('doctree-read', annotations.add_annotations) - - class LimitedAPIList(Directive): - has_content = False - required_arguments = 0 - optional_arguments = 0 - final_argument_whitespace = True - def run(self): - content = [] - for record in annotations.stable_abi_data.values(): - role = REST_ROLE_MAP[record['role']] - name = record['name'] - content.append(f'* :c:{role}:`{name}`') +def _return_value_annotation(result_refs: int | None) -> nodes.emphasis: + classes = ["refcount"] + if result_refs is None: + rc = sphinx_gettext("Return value: Always NULL.") + classes.append("return_null") + elif result_refs: + rc = sphinx_gettext("Return value: New reference.") + classes.append("return_new_ref") + else: + rc = sphinx_gettext("Return value: Borrowed reference.") + classes.append("return_borrowed_ref") + return nodes.emphasis(rc, rc, classes=classes) + + +class LimitedAPIList(SphinxDirective): + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = True + + def run(self) -> list[nodes.Node]: + state = self.env.domaindata["c_annotations"] + content = [ + f"* :c:{record.role}:`{record.name}`" + for record in state["stable_abi_data"].values() + ] + node = nodes.paragraph() + self.state.nested_parse(StringList(content), 0, node) + return [node] + + +def init_annotations(app: Sphinx) -> None: + # Using domaindata is a bit hack-ish, + # but allows storing state without a global variable or closure. + app.env.domaindata["c_annotations"] = state = {} + state["refcount_data"] = read_refcount_data( + Path(app.srcdir, app.config.refcount_file) + ) + state["stable_abi_data"] = read_stable_abi_data( + Path(app.srcdir, app.config.stable_abi_file) + ) - pnode = nodes.paragraph() - self.state.nested_parse(StringList(content), 0, pnode) - return [pnode] - app.add_directive('limited-api-list', LimitedAPIList) +def setup(app: Sphinx) -> ExtensionMetadata: + app.add_config_value("refcount_file", "", "env", types={str}) + app.add_config_value("stable_abi_file", "", "env", types={str}) + app.add_directive("limited-api-list", LimitedAPIList) + app.connect("builder-inited", init_annotations) + app.connect("doctree-read", add_annotations) + if sphinx.version_info[:2] < (7, 2): + from docutils.parsers.rst import directives + from sphinx.domains.c import CObject -def setup(app): - app.add_config_value('refcount_file', '', True) - app.add_config_value('stable_abi_file', '', True) - app.connect('builder-inited', init_annotations) + # monkey-patch C object... + CObject.option_spec |= { + "no-index-entry": directives.flag, + "no-contents-entry": directives.flag, + } - # monkey-patch C object... - CObject.option_spec = { - 'noindex': directives.flag, - 'stableabi': directives.flag, + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, } - old_handle_signature = CObject.handle_signature - def new_handle_signature(self, sig, signode): - signode.parent['stableabi'] = 'stableabi' in self.options - return old_handle_signature(self, sig, signode) - CObject.handle_signature = new_handle_signature - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/escape4chm.py b/Doc/tools/extensions/escape4chm.py deleted file mode 100644 index 89970975b9032b..00000000000000 --- a/Doc/tools/extensions/escape4chm.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Escape the `body` part of .chm source file to 7-bit ASCII, to fix visual -effect on some MBCS Windows systems. - -https://bugs.python.org/issue32174 -""" - -import pathlib -import re -from html.entities import codepoint2name - -from sphinx.util.logging import getLogger - -# escape the characters which codepoint > 0x7F -def _process(string): - def escape(matchobj): - codepoint = ord(matchobj.group(0)) - - name = codepoint2name.get(codepoint) - if name is None: - return '&#%d;' % codepoint - else: - return '&%s;' % name - - return re.sub(r'[^\x00-\x7F]', escape, string) - -def escape_for_chm(app, pagename, templatename, context, doctree): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp': - return - - # escape the `body` part to 7-bit ASCII - body = context.get('body') - if body is not None: - context['body'] = _process(body) - -def fixup_keywords(app, exception): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp' or exception: - return - - getLogger(__name__).info('fixing HTML escapes in keywords file...') - outdir = pathlib.Path(app.builder.outdir) - outname = app.builder.config.htmlhelp_basename - with open(outdir / (outname + '.hhk'), 'rb') as f: - index = f.read() - with open(outdir / (outname + '.hhk'), 'wb') as f: - f.write(index.replace(b''', b''')) - -def setup(app): - # `html-page-context` event emitted when the HTML builder has - # created a context dictionary to render a template with. - app.connect('html-page-context', escape_for_chm) - # `build-finished` event emitted when all the files have been - # output. - app.connect('build-finished', fixup_keywords) - - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py index 59a6862ea3d3f4..502b6cd95bcb94 100644 --- a/Doc/tools/extensions/glossary_search.py +++ b/Doc/tools/extensions/glossary_search.py @@ -1,63 +1,63 @@ -# -*- coding: utf-8 -*- -""" - glossary_search.py - ~~~~~~~~~~~~~~~~ +"""Feature search results for glossary items prominently.""" - Feature search results for glossary items prominently. +from __future__ import annotations - :license: Python license. -""" import json -import os.path -from docutils.nodes import definition_list_item +from pathlib import Path +from typing import TYPE_CHECKING + +from docutils import nodes from sphinx.addnodes import glossary from sphinx.util import logging +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata logger = logging.getLogger(__name__) -STATIC_DIR = '_static' -JSON = 'glossary.json' -def process_glossary_nodes(app, doctree, fromdocname): - if app.builder.format != 'html': +def process_glossary_nodes( + app: Sphinx, + doctree: nodes.document, + _docname: str, +) -> None: + if app.builder.format != 'html' or app.builder.embedded: return - terms = {} + if hasattr(app.env, 'glossary_terms'): + terms = app.env.glossary_terms + else: + terms = app.env.glossary_terms = {} - for node in doctree.traverse(glossary): - for glossary_item in node.traverse(definition_list_item): - term = glossary_item[0].astext().lower() - definition = glossary_item[1] + for node in doctree.findall(glossary): + for glossary_item in node.findall(nodes.definition_list_item): + term = glossary_item[0].astext() + definition = glossary_item[-1] rendered = app.builder.render_partial(definition) - terms[term] = { - 'title': glossary_item[0].astext(), - 'body': rendered['html_body'] + terms[term.lower()] = { + 'title': term, + 'body': rendered['html_body'], } - if hasattr(app.env, 'glossary_terms'): - app.env.glossary_terms.update(terms) - else: - app.env.glossary_terms = terms -def on_build_finish(app, exc): - if not hasattr(app.env, 'glossary_terms'): - return - if not app.env.glossary_terms: +def write_glossary_json(app: Sphinx, _exc: Exception) -> None: + if not getattr(app.env, 'glossary_terms', None): return - logger.info(f'Writing {JSON}', color='green') - - dest_dir = os.path.join(app.outdir, STATIC_DIR) - os.makedirs(dest_dir, exist_ok=True) - - with open(os.path.join(dest_dir, JSON), 'w') as f: - json.dump(app.env.glossary_terms, f) + logger.info('Writing glossary.json', color='green') + dest = Path(app.outdir, '_static', 'glossary.json') + dest.parent.mkdir(exist_ok=True) + dest.write_text(json.dumps(app.env.glossary_terms), encoding='utf-8') -def setup(app): +def setup(app: Sphinx) -> ExtensionMetadata: app.connect('doctree-resolved', process_glossary_nodes) - app.connect('build-finished', on_build_finish) + app.connect('build-finished', write_glossary_json) - return {'version': '0.1', 'parallel_read_safe': True} + return { + 'version': '1.0', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/Doc/tools/extensions/lexers/__init__.py b/Doc/tools/extensions/lexers/__init__.py new file mode 100644 index 00000000000000..e12ac5be8139cc --- /dev/null +++ b/Doc/tools/extensions/lexers/__init__.py @@ -0,0 +1,15 @@ +from .asdl_lexer import ASDLLexer +from .peg_lexer import PEGLexer + + +def setup(app): + # Used for highlighting Parser/Python.asdl in library/ast.rst + app.add_lexer("asdl", ASDLLexer) + # Used for highlighting Grammar/python.gram in reference/grammar.rst + app.add_lexer("peg", PEGLexer) + + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/asdl_highlight.py b/Doc/tools/extensions/lexers/asdl_lexer.py similarity index 62% rename from Doc/tools/extensions/asdl_highlight.py rename to Doc/tools/extensions/lexers/asdl_lexer.py index 42863a4b3bcd6a..3a74174a1f7dfb 100644 --- a/Doc/tools/extensions/asdl_highlight.py +++ b/Doc/tools/extensions/lexers/asdl_lexer.py @@ -1,15 +1,6 @@ -import sys -from pathlib import Path +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text -CPYTHON_ROOT = Path(__file__).resolve().parent.parent.parent.parent -sys.path.append(str(CPYTHON_ROOT / "Parser")) - -from pygments.lexer import RegexLexer, bygroups, include, words -from pygments.token import (Comment, Keyword, Name, Operator, - Punctuation, Text) - -from asdl import builtin_types -from sphinx.highlighting import lexers class ASDLLexer(RegexLexer): name = "ASDL" @@ -34,7 +25,10 @@ class ASDLLexer(RegexLexer): r"(\w+)(\*\s|\?\s|\s)(\w+)", bygroups(Name.Builtin.Pseudo, Operator, Name), ), - (words(builtin_types), Name.Builtin), + # Keep in line with ``builtin_types`` from Parser/asdl.py. + # ASDL's 4 builtin types are + # constant, identifier, int, string + ("constant|identifier|int|string", Name.Builtin), (r"attributes", Name.Builtin), ( _name + _text_ws + "(=)", @@ -46,8 +40,3 @@ class ASDLLexer(RegexLexer): (r".", Text), ], } - - -def setup(app): - lexers["asdl"] = ASDLLexer() - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/lexers/peg_lexer.py similarity index 94% rename from Doc/tools/extensions/peg_highlight.py rename to Doc/tools/extensions/lexers/peg_lexer.py index 4bdc2ee1861334..827af205583f61 100644 --- a/Doc/tools/extensions/peg_highlight.py +++ b/Doc/tools/extensions/lexers/peg_lexer.py @@ -1,8 +1,6 @@ from pygments.lexer import RegexLexer, bygroups, include from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text -from sphinx.highlighting import lexers - class PEGLexer(RegexLexer): """Pygments Lexer for PEG grammar (.gram) files @@ -79,8 +77,3 @@ class PEGLexer(RegexLexer): (r".", Text), ], } - - -def setup(app): - lexers["peg"] = PEGLexer() - return {"version": "1.0", "parallel_read_safe": True} diff --git a/Doc/tools/extensions/patchlevel.py b/Doc/tools/extensions/patchlevel.py index 617f28c2527ddf..9ccaec3dd5ce0f 100644 --- a/Doc/tools/extensions/patchlevel.py +++ b/Doc/tools/extensions/patchlevel.py @@ -1,68 +1,81 @@ -# -*- coding: utf-8 -*- -""" - patchlevel.py - ~~~~~~~~~~~~~ +"""Extract version information from Include/patchlevel.h.""" - Extract version info from Include/patchlevel.h. - Adapted from Doc/tools/getversioninfo. +import re +import sys +from pathlib import Path +from typing import Literal, NamedTuple - :copyright: 2007-2008 by Georg Brandl. - :license: Python license. -""" +CPYTHON_ROOT = Path( + __file__, # cpython/Doc/tools/extensions/patchlevel.py + "..", # cpython/Doc/tools/extensions + "..", # cpython/Doc/tools + "..", # cpython/Doc + "..", # cpython +).resolve() +PATCHLEVEL_H = CPYTHON_ROOT / "Include" / "patchlevel.h" -from __future__ import print_function +RELEASE_LEVELS = { + "PY_RELEASE_LEVEL_ALPHA": "alpha", + "PY_RELEASE_LEVEL_BETA": "beta", + "PY_RELEASE_LEVEL_GAMMA": "candidate", + "PY_RELEASE_LEVEL_FINAL": "final", +} -import os -import re -import sys -def get_header_version_info(srcdir): - patchlevel_h = os.path.join(srcdir, '..', 'Include', 'patchlevel.h') +class version_info(NamedTuple): # noqa: N801 + major: int #: Major release number + minor: int #: Minor release number + micro: int #: Patch release number + releaselevel: Literal["alpha", "beta", "candidate", "final"] + serial: int #: Serial release number - # This won't pick out all #defines, but it will pick up the ones we - # care about. - rx = re.compile(r'\s*#define\s+([a-zA-Z][a-zA-Z_0-9]*)\s+([a-zA-Z_0-9]+)') - d = {} - with open(patchlevel_h) as f: - for line in f: - m = rx.match(line) - if m is not None: - name, value = m.group(1, 2) - d[name] = value +def get_header_version_info() -> version_info: + # Capture PY_ prefixed #defines. + pat = re.compile(r"\s*#define\s+(PY_\w*)\s+(\w+)", re.ASCII) - release = version = '%s.%s' % (d['PY_MAJOR_VERSION'], d['PY_MINOR_VERSION']) - micro = int(d['PY_MICRO_VERSION']) - release += '.' + str(micro) + defines = {} + patchlevel_h = PATCHLEVEL_H.read_text(encoding="utf-8") + for line in patchlevel_h.splitlines(): + if (m := pat.match(line)) is not None: + name, value = m.groups() + defines[name] = value - level = d['PY_RELEASE_LEVEL'] - suffixes = { - 'PY_RELEASE_LEVEL_ALPHA': 'a', - 'PY_RELEASE_LEVEL_BETA': 'b', - 'PY_RELEASE_LEVEL_GAMMA': 'rc', - } - if level != 'PY_RELEASE_LEVEL_FINAL': - release += suffixes[level] + str(int(d['PY_RELEASE_SERIAL'])) - return version, release + return version_info( + major=int(defines["PY_MAJOR_VERSION"]), + minor=int(defines["PY_MINOR_VERSION"]), + micro=int(defines["PY_MICRO_VERSION"]), + releaselevel=RELEASE_LEVELS[defines["PY_RELEASE_LEVEL"]], + serial=int(defines["PY_RELEASE_SERIAL"]), + ) -def get_sys_version_info(): - major, minor, micro, level, serial = sys.version_info - release = version = '%s.%s' % (major, minor) - release += '.%s' % micro - if level != 'final': - release += '%s%s' % (level[0], serial) +def format_version_info(info: version_info) -> tuple[str, str]: + version = f"{info.major}.{info.minor}" + release = f"{info.major}.{info.minor}.{info.micro}" + if info.releaselevel != "final": + suffix = {"alpha": "a", "beta": "b", "candidate": "rc"} + release += f"{suffix[info.releaselevel]}{info.serial}" return version, release def get_version_info(): try: - return get_header_version_info('.') - except (IOError, OSError): - version, release = get_sys_version_info() - print('Can\'t get version info from Include/patchlevel.h, ' \ - 'using version of this interpreter (%s).' % release, file=sys.stderr) + info = get_header_version_info() + return format_version_info(info) + except OSError: + version, release = format_version_info(sys.version_info) + print( + f"Failed to get version info from Include/patchlevel.h, " + f"using version of this interpreter ({release}).", + file=sys.stderr, + ) return version, release -if __name__ == '__main__': - print(get_header_version_info('.')[1]) + +if __name__ == "__main__": + short_ver, full_ver = format_version_info(get_header_version_info()) + if sys.argv[1:2] == ["--short"]: + print(short_ver) + else: + print(full_ver) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index cd441836f62bde..f4df7ec0839339 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -15,30 +15,23 @@ from time import asctime from pprint import pformat -from docutils import nodes, utils +from docutils import nodes from docutils.io import StringOutput -from docutils.parsers.rst import Directive -from docutils.utils import new_document +from docutils.parsers.rst import directives +from docutils.utils import new_document, unescape from sphinx import addnodes from sphinx.builders import Builder -from sphinx.domains.python import PyFunction, PyMethod -from sphinx.errors import NoUri +from sphinx.domains.changeset import VersionChange, versionlabels, versionlabel_classes +from sphinx.domains.python import PyFunction, PyMethod, PyModule from sphinx.locale import _ as sphinx_gettext -from sphinx.util import logging from sphinx.util.docutils import SphinxDirective -from sphinx.util.nodes import split_explicit_title from sphinx.writers.text import TextWriter, TextTranslator - -try: - # Sphinx 6+ - from sphinx.util.display import status_iterator -except ImportError: - # Deprecated in Sphinx 6.1, will be removed in Sphinx 8 - from sphinx.util import status_iterator +from sphinx.util.display import status_iterator ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' GH_ISSUE_URI = 'https://github.com/python/cpython/issues/%s' +# Used in conf.py and updated here by python/release-tools/run_release.py SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists @@ -54,10 +47,14 @@ std.token_re = re.compile(r'`((~?[\w-]*:)?\w+)`') +# backport :no-index: +PyModule.option_spec['no-index'] = directives.flag + + # Support for marking up and linking to bugs.python.org issues def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) + issue = unescape(text) # sanity check: there are no bpo issues within these two values if 47261 < int(issue) < 400000: msg = inliner.reporter.error(f'The BPO ID {text!r} seems too high -- ' @@ -72,7 +69,7 @@ def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): # Support for marking up and linking to GitHub issues def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) + issue = unescape(text) # sanity check: all GitHub issues have ID >= 32426 # even though some of them are also valid BPO IDs if int(issue) < 32426: @@ -85,19 +82,9 @@ def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): return [refnode], [] -# Support for linking to Python source files easily - -def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - has_t, title, target = split_explicit_title(text) - title = utils.unescape(title) - target = utils.unescape(target) - refnode = nodes.reference(title, title, refuri=SOURCE_URI % target) - return [refnode], [] - - # Support for marking up implementation details -class ImplementationDetail(Directive): +class ImplementationDetail(SphinxDirective): has_content = True final_argument_whitespace = True @@ -120,218 +107,6 @@ def run(self): return [pnode] -# Support for documenting platform availability - -class Availability(SphinxDirective): - - has_content = True - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = True - - # known platform, libc, and threading implementations - known_platforms = frozenset({ - "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD", - "GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", - "Unix", "VxWorks", "WASI", "Windows", "macOS", - # libc - "BSD libc", "glibc", "musl", - # POSIX platforms with pthreads - "pthreads", - }) - - def run(self): - availability_ref = ':ref:`Availability `: ' - avail_nodes, avail_msgs = self.state.inline_text( - availability_ref + self.arguments[0], - self.lineno) - pnode = nodes.paragraph(availability_ref + self.arguments[0], - '', *avail_nodes, *avail_msgs) - self.set_source_info(pnode) - cnode = nodes.container("", pnode, classes=["availability"]) - self.set_source_info(cnode) - if self.content: - self.state.nested_parse(self.content, self.content_offset, cnode) - self.parse_platforms() - - return [cnode] - - def parse_platforms(self): - """Parse platform information from arguments - - Arguments is a comma-separated string of platforms. A platform may - be prefixed with "not " to indicate that a feature is not available. - - Example:: - - .. availability:: Windows, Linux >= 4.2, not Emscripten, not WASI - - Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not - parsed into separate tokens. - """ - platforms = {} - for arg in self.arguments[0].rstrip(".").split(","): - arg = arg.strip() - platform, _, version = arg.partition(" >= ") - if platform.startswith("not "): - version = False - platform = platform[4:] - elif not version: - version = True - platforms[platform] = version - - unknown = set(platforms).difference(self.known_platforms) - if unknown: - cls = type(self) - logger = logging.getLogger(cls.__qualname__) - logger.warning( - f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' " - f"in '.. availability:: {self.arguments[0]}', see " - f"{__file__}:{cls.__qualname__}.known_platforms for a set " - "known platforms." - ) - - return platforms - - - -# Support for documenting audit event - -def audit_events_purge(app, env, docname): - """This is to remove from env.all_audit_events old traces of removed - documents. - """ - if not hasattr(env, 'all_audit_events'): - return - fresh_all_audit_events = {} - for name, event in env.all_audit_events.items(): - event["source"] = [(d, t) for d, t in event["source"] if d != docname] - if event["source"]: - # Only keep audit_events that have at least one source. - fresh_all_audit_events[name] = event - env.all_audit_events = fresh_all_audit_events - - -def audit_events_merge(app, env, docnames, other): - """In Sphinx parallel builds, this merges env.all_audit_events from - subprocesses. - - all_audit_events is a dict of names, with values like: - {'source': [(docname, target), ...], 'args': args} - """ - if not hasattr(other, 'all_audit_events'): - return - if not hasattr(env, 'all_audit_events'): - env.all_audit_events = {} - for name, value in other.all_audit_events.items(): - if name in env.all_audit_events: - env.all_audit_events[name]["source"].extend(value["source"]) - else: - env.all_audit_events[name] = value - - -class AuditEvent(Directive): - - has_content = True - required_arguments = 1 - optional_arguments = 2 - final_argument_whitespace = True - - _label = [ - sphinx_gettext("Raises an :ref:`auditing event ` {name} with no arguments."), - sphinx_gettext("Raises an :ref:`auditing event ` {name} with argument {args}."), - sphinx_gettext("Raises an :ref:`auditing event ` {name} with arguments {args}."), - ] - - @property - def logger(self): - cls = type(self) - return logging.getLogger(cls.__module__ + "." + cls.__name__) - - def run(self): - name = self.arguments[0] - if len(self.arguments) >= 2 and self.arguments[1]: - args = (a.strip() for a in self.arguments[1].strip("'\"").split(",")) - args = [a for a in args if a] - else: - args = [] - - label = self._label[min(2, len(args))] - text = label.format(name="``{}``".format(name), - args=", ".join("``{}``".format(a) for a in args if a)) - - env = self.state.document.settings.env - if not hasattr(env, 'all_audit_events'): - env.all_audit_events = {} - - new_info = { - 'source': [], - 'args': args - } - info = env.all_audit_events.setdefault(name, new_info) - if info is not new_info: - if not self._do_args_match(info['args'], new_info['args']): - self.logger.warning( - "Mismatched arguments for audit-event {}: {!r} != {!r}" - .format(name, info['args'], new_info['args']) - ) - - ids = [] - try: - target = self.arguments[2].strip("\"'") - except (IndexError, TypeError): - target = None - if not target: - target = "audit_event_{}_{}".format( - re.sub(r'\W', '_', name), - len(info['source']), - ) - ids.append(target) - - info['source'].append((env.docname, target)) - - pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) - pnode.line = self.lineno - if self.content: - self.state.nested_parse(self.content, self.content_offset, pnode) - else: - n, m = self.state.inline_text(text, self.lineno) - pnode.extend(n + m) - - return [pnode] - - # This list of sets are allowable synonyms for event argument names. - # If two names are in the same set, they are treated as equal for the - # purposes of warning. This won't help if number of arguments is - # different! - _SYNONYMS = [ - {"file", "path", "fd"}, - ] - - def _do_args_match(self, args1, args2): - if args1 == args2: - return True - if len(args1) != len(args2): - return False - for a1, a2 in zip(args1, args2): - if a1 == a2: - continue - if any(a1 in s and a2 in s for s in self._SYNONYMS): - continue - return False - return True - - -class audit_event_list(nodes.General, nodes.Element): - pass - - -class AuditEventListDirective(Directive): - - def run(self): - return [audit_event_list('')] - - # Support for documenting decorators class PyDecoratorMixin(object): @@ -409,60 +184,55 @@ def run(self): return PyMethod.run(self) -# Support for documenting version of removal in deprecations +# Support for documenting version of changes, additions, deprecations -class DeprecatedRemoved(Directive): - has_content = True +def expand_version_arg(argument, release): + """Expand "next" to the current version""" + if argument == 'next': + return sphinx_gettext('{} (unreleased)').format(release) + return argument + + +class PyVersionChange(VersionChange): + def run(self): + # Replace the 'next' special token with the current development version + self.arguments[0] = expand_version_arg(self.arguments[0], + self.config.release) + return super().run() + + +class DeprecatedRemoved(VersionChange): required_arguments = 2 - optional_arguments = 1 - final_argument_whitespace = True - option_spec = {} - _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') - _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') + _deprecated_label = sphinx_gettext('Deprecated since version %s, will be removed in version %s') + _removed_label = sphinx_gettext('Deprecated since version %s, removed in version %s') def run(self): - node = addnodes.versionmodified() - node.document = self.state.document - node['type'] = 'deprecated-removed' - version = (self.arguments[0], self.arguments[1]) - node['version'] = version - env = self.state.document.settings.env - current_version = tuple(int(e) for e in env.config.version.split('.')) - removed_version = tuple(int(e) for e in self.arguments[1].split('.')) + # Replace the first two arguments (deprecated version and removed version) + # with a single tuple of both versions. + version_deprecated = expand_version_arg(self.arguments[0], + self.config.release) + version_removed = self.arguments.pop(1) + if version_removed == 'next': + raise ValueError( + 'deprecated-removed:: second argument cannot be `next`') + self.arguments[0] = version_deprecated, version_removed + + # Set the label based on if we have reached the removal version + current_version = tuple(map(int, self.config.version.split('.'))) + removed_version = tuple(map(int, version_removed.split('.'))) if current_version < removed_version: - label = self._deprecated_label - else: - label = self._removed_label - - text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) - if len(self.arguments) == 3: - inodes, messages = self.state.inline_text(self.arguments[2], - self.lineno+1) - para = nodes.paragraph(self.arguments[2], '', *inodes, translatable=False) - node.append(para) + versionlabels[self.name] = self._deprecated_label + versionlabel_classes[self.name] = 'deprecated' else: - messages = [] - if self.content: - self.state.nested_parse(self.content, self.content_offset, node) - if len(node): - if isinstance(node[0], nodes.paragraph) and node[0].rawsource: - content = nodes.inline(node[0].rawsource, translatable=True) - content.source = node[0].source - content.line = node[0].line - content += node[0].children - node[0].replace_self(nodes.paragraph('', '', content, translatable=False)) - node[0].insert(0, nodes.inline('', '%s: ' % text, - classes=['versionmodified'])) - else: - para = nodes.paragraph('', '', - nodes.inline('', '%s.' % text, - classes=['versionmodified']), - translatable=False) - node.append(para) - env = self.state.document.settings.env - env.get_domain('changeset').note_changeset(node) - return [node] + messages + versionlabels[self.name] = self._removed_label + versionlabel_classes[self.name] = 'removed' + try: + return super().run() + finally: + # reset versionlabels and versionlabel_classes + versionlabels[self.name] = '' + versionlabel_classes[self.name] = '' # Support for including Misc/NEWS @@ -472,7 +242,7 @@ def run(self): whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") -class MiscNews(Directive): +class MiscNews(SphinxDirective): has_content = False required_arguments = 1 optional_arguments = 0 @@ -487,7 +257,7 @@ def run(self): if not source_dir: source_dir = path.dirname(path.abspath(source)) fpath = path.join(source_dir, fname) - self.state.document.settings.record_dependencies.add(fpath) + self.env.note_dependency(path.abspath(fpath)) try: with io.open(fpath, encoding='utf-8') as fp: content = fp.read() @@ -508,8 +278,8 @@ def run(self): # Support for building "topic help" for pydoc pydoc_topic_labels = [ - 'assert', 'assignment', 'async', 'atom-identifiers', 'atom-literals', - 'attribute-access', 'attribute-references', 'augassign', 'await', + 'assert', 'assignment', 'assignment-expressions', 'async', 'atom-identifiers', + 'atom-literals', 'attribute-access', 'attribute-references', 'augassign', 'await', 'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object', 'bltin-null-object', 'bltin-type-objects', 'booleans', 'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound', @@ -619,70 +389,6 @@ def parse_monitoring_event(env, sig, signode): return sig -def process_audit_events(app, doctree, fromdocname): - for node in doctree.traverse(audit_event_list): - break - else: - return - - env = app.builder.env - - table = nodes.table(cols=3) - group = nodes.tgroup( - '', - nodes.colspec(colwidth=30), - nodes.colspec(colwidth=55), - nodes.colspec(colwidth=15), - cols=3, - ) - head = nodes.thead() - body = nodes.tbody() - - table += group - group += head - group += body - - row = nodes.row() - row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event'))) - row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments'))) - row += nodes.entry('', nodes.paragraph('', nodes.Text('References'))) - head += row - - for name in sorted(getattr(env, "all_audit_events", ())): - audit_event = env.all_audit_events[name] - - row = nodes.row() - node = nodes.paragraph('', nodes.Text(name)) - row += nodes.entry('', node) - - node = nodes.paragraph() - for i, a in enumerate(audit_event['args']): - if i: - node += nodes.Text(", ") - node += nodes.literal(a, nodes.Text(a)) - row += nodes.entry('', node) - - node = nodes.paragraph() - backlinks = enumerate(sorted(set(audit_event['source'])), start=1) - for i, (doc, label) in backlinks: - if isinstance(label, str): - ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True) - try: - ref['refuri'] = "{}#{}".format( - app.builder.get_relative_uri(fromdocname, doc), - label, - ) - except NoUri: - continue - node += ref - row += nodes.entry('', node) - - body += row - - for node in doctree.traverse(audit_event_list): - node.replace_self(table) - - def patch_pairindextypes(app, _env) -> None: """Remove all entries from ``pairindextypes`` before writing POT files. @@ -710,11 +416,11 @@ def patch_pairindextypes(app, _env) -> None: def setup(app): app.add_role('issue', issue_role) app.add_role('gh', gh_issue_role) - app.add_role('source', source_role) app.add_directive('impl-detail', ImplementationDetail) - app.add_directive('availability', Availability) - app.add_directive('audit-event', AuditEvent) - app.add_directive('audit-event-table', AuditEventListDirective) + app.add_directive('versionadded', PyVersionChange, override=True) + app.add_directive('versionchanged', PyVersionChange, override=True) + app.add_directive('versionremoved', PyVersionChange, override=True) + app.add_directive('deprecated', PyVersionChange, override=True) app.add_directive('deprecated-removed', DeprecatedRemoved) app.add_builder(PydocTopicsBuilder) app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) @@ -728,8 +434,6 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) + app.add_css_file('sidebar-wrap.css') app.connect('env-check-consistency', patch_pairindextypes) - app.connect('doctree-resolved', process_audit_events) - app.connect('env-merge-info', audit_events_merge) - app.connect('env-purge-doc', audit_events_purge) return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/static/changelog_search.js b/Doc/tools/static/changelog_search.js index c881a9bd4c84a7..0a77c0d71ae937 100644 --- a/Doc/tools/static/changelog_search.js +++ b/Doc/tools/static/changelog_search.js @@ -1,53 +1,59 @@ -$(document).ready(function() { - // add the search form and bind the events - $('h1').after([ - '

Filter entries by content:', - '', - '

' - ].join('\n')); +document.addEventListener("DOMContentLoaded", function () { + // add the search form and bind the events + document + .querySelector("h1") + .insertAdjacentHTML( + "afterend", + [ + "

Filter entries by content:", + '', + '

', + ].join("\n"), + ); - function dofilter() { - try { - var query = new RegExp($('#searchbox').val(), 'i'); + function doFilter() { + let query; + try { + query = new RegExp(document.querySelector("#searchbox").value, "i"); + } catch (e) { + return; // not a valid regex (yet) + } + // find headers for the versions (What's new in Python X.Y.Z?) + const h2s = document.querySelectorAll("#changelog h2"); + for (const h2 of h2s) { + let sections_found = 0; + // find headers for the sections (Core, Library, etc.) + const h3s = h2.parentNode.querySelectorAll("h3"); + for (const h3 of h3s) { + let entries_found = 0; + // find all the entries + const lis = h3.parentNode.querySelectorAll("li"); + for (let li of lis) { + // check if the query matches the entry + if (query.test(li.textContent)) { + li.style.display = "block"; + entries_found++; + } else { + li.style.display = "none"; + } } - catch (e) { - return; // not a valid regex (yet) + // if there are entries, show the section, otherwise hide it + if (entries_found > 0) { + h3.parentNode.style.display = "block"; + sections_found++; + } else { + h3.parentNode.style.display = "none"; } - // find headers for the versions (What's new in Python X.Y.Z?) - $('#changelog h2').each(function(index1, h2) { - var h2_parent = $(h2).parent(); - var sections_found = 0; - // find headers for the sections (Core, Library, etc.) - h2_parent.find('h3').each(function(index2, h3) { - var h3_parent = $(h3).parent(); - var entries_found = 0; - // find all the entries - h3_parent.find('li').each(function(index3, li) { - var li = $(li); - // check if the query matches the entry - if (query.test(li.text())) { - li.show(); - entries_found++; - } - else { - li.hide(); - } - }); - // if there are entries, show the section, otherwise hide it - if (entries_found > 0) { - h3_parent.show(); - sections_found++; - } - else { - h3_parent.hide(); - } - }); - if (sections_found > 0) - h2_parent.show(); - else - h2_parent.hide(); - }); + } + if (sections_found > 0) { + h2.parentNode.style.display = "block"; + } else { + h2.parentNode.style.display = "none"; + } } - $('#searchbox').keyup(dofilter); - $('#searchbox-submit').click(dofilter); + } + document.querySelector("#searchbox").addEventListener("keyup", doFilter); + document + .querySelector("#searchbox-submit") + .addEventListener("click", doFilter); }); diff --git a/Doc/tools/static/glossary_search.js b/Doc/tools/static/glossary_search.js new file mode 100644 index 00000000000000..13d728dc027f1d --- /dev/null +++ b/Doc/tools/static/glossary_search.js @@ -0,0 +1,47 @@ +"use strict"; + +const GLOSSARY_PAGE = "glossary.html"; + +const glossary_search = async () => { + const response = await fetch("_static/glossary.json"); + if (!response.ok) { + throw new Error("Failed to fetch glossary.json"); + } + const glossary = await response.json(); + + const params = new URLSearchParams(document.location.search).get("q"); + if (!params) { + return; + } + + const searchParam = params.toLowerCase(); + const glossaryItem = glossary[searchParam]; + if (!glossaryItem) { + return; + } + + // set up the title text with a link to the glossary page + const glossaryTitle = document.getElementById("glossary-title"); + glossaryTitle.textContent = "Glossary: " + glossaryItem.title; + const linkTarget = searchParam.replace(/ /g, "-"); + glossaryTitle.href = GLOSSARY_PAGE + "#term-" + linkTarget; + + // rewrite any anchor links (to other glossary terms) + // to have a full reference to the glossary page + const glossaryBody = document.getElementById("glossary-body"); + glossaryBody.innerHTML = glossaryItem.body; + const anchorLinks = glossaryBody.querySelectorAll('a[href^="#"]'); + anchorLinks.forEach(function (link) { + const currentUrl = link.getAttribute("href"); + link.href = GLOSSARY_PAGE + currentUrl; + }); + + const glossaryResult = document.getElementById("glossary-result"); + glossaryResult.style.display = ""; +}; + +if (document.readyState !== "loading") { + glossary_search().catch(console.error); +} else { + document.addEventListener("DOMContentLoaded", glossary_search); +} diff --git a/Doc/tools/static/rtd_switcher.js b/Doc/tools/static/rtd_switcher.js new file mode 100644 index 00000000000000..2bf01a002db90c --- /dev/null +++ b/Doc/tools/static/rtd_switcher.js @@ -0,0 +1,55 @@ + function onSwitch(event) { + const option = event.target.selectedIndex; + const item = event.target.options[option]; + window.location.href = item.dataset.url; + } + + document.addEventListener("readthedocs-addons-data-ready", function(event) { + const config = event.detail.data() + const versionSelect = ` + + `; + + // Prepend the current language to the options on the selector + let languages = config.projects.translations.concat(config.projects.current); + languages = languages.sort((a, b) => a.language.name.localeCompare(b.language.name)); + + const languageSelect = ` + + `; + + // Query all the placeholders because there are different ones for Desktop/Mobile + const versionPlaceholders = document.querySelectorAll(".version_switcher_placeholder"); + for (placeholder of versionPlaceholders) { + placeholder.innerHTML = versionSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + + const languagePlaceholders = document.querySelectorAll(".language_switcher_placeholder"); + for (placeholder of languagePlaceholders) { + placeholder.innerHTML = languageSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + }); diff --git a/Doc/tools/static/sidebar-wrap.css b/Doc/tools/static/sidebar-wrap.css new file mode 100644 index 00000000000000..0a80f516f28349 --- /dev/null +++ b/Doc/tools/static/sidebar-wrap.css @@ -0,0 +1,6 @@ +div.sphinxsidebarwrapper { + overflow-x: scroll; +} +div.sphinxsidebarwrapper li code { + overflow-wrap: normal; +} diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index b5353d6fb77ab4..45ec436fee72d7 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -1,64 +1,75 @@ {% extends "layout.html" %} -{% set title = 'Download' %} +{% set title = _('Download') %} {% if daily is defined %} - {% set dlbase = pathto('archives', 1) %} + {% set dl_base = pathto('archives', resource=True) %} + {% set dl_version = version %} {% else %} {# The link below returns HTTP 404 until the first related alpha release. This is expected; use daily documentation builds for CPython development. #} - {% set dlbase = 'https://docs.python.org/ftp/python/doc/' + release %} + {% set dl_base = 'https://www.python.org/ftp/python/doc/' + release %} + {% set dl_version = release %} {% endif %} {% block body %} -

Download Python {{ release }} Documentation

+

{% trans %}Download Python {{ dl_version }} Documentation{% endtrans %}

-{% if last_updated %}

Last updated on: {{ last_updated }}.

{% endif %} +{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} -

To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table.

+

{% trans %}To download an archive containing all the documents for this version of +Python in one of various formats, follow one of links in this table.{% endtrans %}

- - - - + + + + - - - + + + + - - - + + + + - - - + + + + - - + + + + + + + +
FormatPacked as .zipPacked as .tar.bz2
PDF (US-Letter paper size)Download (ca. 13 MiB)Download (ca. 13 MiB)
{% trans %}Format{% endtrans %}{% trans %}Packed as .zip{% endtrans %}{% trans %}Packed as .tar.bz2{% endtrans %}
PDF (A4 paper size)Download (ca. 13 MiB)Download (ca. 13 MiB)
{% trans %}PDF{% endtrans %}{% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %}
HTMLDownload (ca. 9 MiB)Download (ca. 6 MiB)
{% trans %}HTML{% endtrans %}{% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %}
Plain TextDownload (ca. 3 MiB)Download (ca. 2 MiB)
{% trans %}Plain text{% endtrans %}{% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %}
EPUBDownload (ca. 5 MiB)
{% trans %}Texinfo{% endtrans %}{% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %}
{% trans %}EPUB{% endtrans %}{% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %}
-

These archives contain all the content in the documentation.

+

{% trans %}These archives contain all the content in the documentation.{% endtrans %}

-

Unpacking

+

{% trans %}Unpacking{% endtrans %}

-

Unix users should download the .tar.bz2 archives; these are bzipped tar +

{% trans %}Unix users should download the .tar.bz2 archives; these are bzipped tar archives and can be handled in the usual way using tar and the bzip2 program. The Info-ZIP unzip program can be used to handle the ZIP archives if desired. The .tar.bz2 archives provide the -best compression and fastest download times.

+best compression and fastest download times.{% endtrans %}

-

Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the Info-ZIP zip program.

+

{% trans %}Windows users can use the ZIP archives since those are customary on that +platform. These are created on Unix using the Info-ZIP zip program.{% endtrans %}

-

Problems

+

{% trans %}Problems{% endtrans %}

-

If you have comments or suggestions for the Python documentation, please send -email to docs@python.org.

+

{% trans %}If you have comments or suggestions for the Python documentation, please send +email to docs@python.org.{% endtrans %}

{% endblock %} diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 1e3ab7cfe02fee..2686f48dad2a95 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -7,62 +7,64 @@

{{ docstitle|e }}

{% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %}

-

{% trans %}Parts of the documentation:{% endtrans %}

+

{% trans %}Documentation sections:{% endtrans %}

+ {% trans whatsnew_index=pathto("whatsnew/index") %}Or all "What's new" documents since Python 2.0{% endtrans %}

- - - + {% trans %}Start here: a tour of Python's syntax and features{% endtrans %}

+ + + + {% trans %}In-depth topic manuals{% endtrans %}

- - - - + + + + + {% trans %}Frequently asked questions (with answers!){% endtrans %}

+
-

{% trans %}Indices and tables:{% endtrans %}

+

{% trans %}Indices, glossary, and search:{% endtrans %}

- - + + + {% trans %}Terms explained{% endtrans %}

- + {% trans %}Search this documentation{% endtrans %}

+
-

{% trans %}Meta information:{% endtrans %}

+

{% trans %}Project information:{% endtrans %}

- - - + + + - + - +
{% endblock %} diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 9498b2ccc5af92..b09fd21a8ddcc9 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,6 +26,9 @@ {% endblock %} {% block extrahead %} + {% if builder == "html" and enable_analytics %} + + {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} @@ -41,4 +44,9 @@ {{ "}" }} {{ super() }} + +{%- if not embedded %} + + +{%- endif %} {% endblock %} diff --git a/Doc/tools/templates/search.html b/Doc/tools/templates/search.html index 852974461380f2..6ddac5f828bab1 100644 --- a/Doc/tools/templates/search.html +++ b/Doc/tools/templates/search.html @@ -2,61 +2,16 @@ {% block extrahead %} {{ super() }} - + +{% endblock %} +{% block searchresults %} +
+ {# For glossary_search.js #} + +
{% endblock %} diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 4bea0d8a49ce20..6a1611afadb57c 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -10,6 +10,28 @@ Appendix Interactive Mode ================ +There are two variants of the interactive :term:`REPL`. The classic +basic interpreter is supported on all platforms with minimal line +control capabilities. + +On Windows, or Unix-like systems with :mod:`curses` support, +a new interactive shell is used by default. +This one supports color, multiline editing, history browsing, and +paste mode. To disable color, see :ref:`using-on-controlling-color` for +details. Function keys provide some additional functionality. +:kbd:`F1` enters the interactive help browser :mod:`pydoc`. +:kbd:`F2` allows for browsing command-line history with neither output nor the +:term:`>>>` and :term:`...` prompts. :kbd:`F3` enters "paste mode", which +makes pasting larger blocks of code easier. Press :kbd:`F3` to return to +the regular prompt. + +When using the new interactive shell, exit the shell by typing :kbd:`exit` +or :kbd:`quit`. Adding call parentheses after those commands is not +required. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + .. _tut-error: Error Handling @@ -40,7 +62,7 @@ Executable Python Scripts On BSD'ish Unix systems, Python scripts can be made directly executable, like shell scripts, by putting the line :: - #!/usr/bin/env python3.5 + #!/usr/bin/env python3 (assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning of the script and giving the file an executable mode. The ``#!`` must be the @@ -107,7 +129,7 @@ of your user site-packages directory. Start Python and run this code:: >>> import site >>> site.getusersitepackages() - '/home/user/.local/lib/python3.5/site-packages' + '/home/user/.local/lib/python3.x/site-packages' Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index d1c303ef037027..492568961d8a51 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -276,8 +276,8 @@ definition looked like this:: then ``MyClass.i`` and ``MyClass.f`` are valid attribute references, returning an integer and a function object, respectively. Class attributes can also be assigned to, so you can change the value of ``MyClass.i`` by assignment. -:attr:`!__doc__` is also a valid attribute, returning the docstring belonging to -the class: ``"A simple example class"``. +:attr:`~type.__doc__` is also a valid attribute, returning the docstring +belonging to the class: ``"A simple example class"``. Class *instantiation* uses function notation. Just pretend that the class object is a parameterless function that returns a new instance of the class. @@ -338,11 +338,7 @@ code will print the value ``16``, without leaving a trace:: del x.counter The other kind of instance attribute reference is a *method*. A method is a -function that "belongs to" an object. (In Python, the term method is not unique -to class instances: other object types can have methods as well. For example, -list objects have methods called append, insert, remove, sort, and so on. -However, in the following discussion, we'll use the term method exclusively to -mean methods of class instance objects, unless explicitly stated otherwise.) +function that "belongs to" an object. .. index:: pair: object; method @@ -665,7 +661,7 @@ class, that calls each parent only once, and that is monotonic (meaning that a class can be subclassed without affecting the precedence order of its parents). Taken together, these properties make it possible to design reliable and extensible classes with multiple inheritance. For more detail, see -https://www.python.org/download/releases/2.3/mro/. +:ref:`python_2.3_mro`. .. _tut-private: @@ -692,6 +688,11 @@ current class name with leading underscore(s) stripped. This mangling is done without regard to the syntactic position of the identifier, as long as it occurs within the definition of a class. +.. seealso:: + + The :ref:`private name mangling specifications ` + for details and special cases. + Name mangling is helpful for letting subclasses override methods without breaking intraclass method calls. For example:: @@ -931,6 +932,6 @@ Examples:: .. [#] Except for one thing. Module objects have a secret read-only attribute called :attr:`~object.__dict__` which returns the dictionary used to implement the module's - namespace; the name :attr:`~object.__dict__` is an attribute but not a global name. + namespace; the name ``__dict__`` is an attribute but not a global name. Obviously, using this violates the abstraction of namespace implementation, and should be restricted to things like post-mortem debuggers. diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 77444f9cb8358d..8261bbdbfb7a01 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -61,7 +61,7 @@ they appear in the sequence. For example (no pun intended): :: >>> # Measure some strings: - ... words = ['cat', 'window', 'defenestrate'] + >>> words = ['cat', 'window', 'defenestrate'] >>> for w in words: ... print(w, len(w)) ... @@ -160,21 +160,60 @@ arguments. In chapter :ref:`tut-structures`, we will discuss in more detail abo .. _tut-break: -:keyword:`!break` and :keyword:`!continue` Statements, and :keyword:`!else` Clauses on Loops -============================================================================================ +:keyword:`!break` and :keyword:`!continue` Statements +===================================================== The :keyword:`break` statement breaks out of the innermost enclosing -:keyword:`for` or :keyword:`while` loop. +:keyword:`for` or :keyword:`while` loop:: -A :keyword:`!for` or :keyword:`!while` loop can include an :keyword:`!else` clause. + >>> for n in range(2, 10): + ... for x in range(2, n): + ... if n % x == 0: + ... print(f"{n} equals {x} * {n//x}") + ... break + ... + 4 equals 2 * 2 + 6 equals 2 * 3 + 8 equals 2 * 4 + 9 equals 3 * 3 + +The :keyword:`continue` statement continues with the next +iteration of the loop:: + + >>> for num in range(2, 10): + ... if num % 2 == 0: + ... print(f"Found an even number {num}") + ... continue + ... print(f"Found an odd number {num}") + ... + Found an even number 2 + Found an odd number 3 + Found an even number 4 + Found an odd number 5 + Found an even number 6 + Found an odd number 7 + Found an even number 8 + Found an odd number 9 + +.. _tut-for-else: +.. _break-and-continue-statements-and-else-clauses-on-loops: + +:keyword:`!else` Clauses on Loops +================================= + +In a :keyword:`!for` or :keyword:`!while` loop the :keyword:`!break` statement +may be paired with an :keyword:`!else` clause. If the loop finishes without +executing the :keyword:`!break`, the :keyword:`!else` clause executes. In a :keyword:`for` loop, the :keyword:`!else` clause is executed -after the loop reaches its final iteration. +after the loop finishes its final iteration, that is, if no break occurred. In a :keyword:`while` loop, it's executed after the loop's condition becomes false. -In either kind of loop, the :keyword:`!else` clause is **not** executed -if the loop was terminated by a :keyword:`break`. +In either kind of loop, the :keyword:`!else` clause is **not** executed if the +loop was terminated by a :keyword:`break`. Of course, other ways of ending the +loop early, such as a :keyword:`return` or a raised exception, will also skip +execution of the :keyword:`else` clause. This is exemplified in the following :keyword:`!for` loop, which searches for prime numbers:: @@ -198,32 +237,19 @@ which searches for prime numbers:: 9 equals 3 * 3 (Yes, this is the correct code. Look closely: the ``else`` clause belongs to -the :keyword:`for` loop, **not** the :keyword:`if` statement.) +the ``for`` loop, **not** the ``if`` statement.) -When used with a loop, the ``else`` clause has more in common with the -``else`` clause of a :keyword:`try` statement than it does with that of -:keyword:`if` statements: a :keyword:`try` statement's ``else`` clause runs -when no exception occurs, and a loop's ``else`` clause runs when no ``break`` -occurs. For more on the :keyword:`!try` statement and exceptions, see -:ref:`tut-handling`. - -The :keyword:`continue` statement, also borrowed from C, continues with the next -iteration of the loop:: +One way to think of the else clause is to imagine it paired with the ``if`` +inside the loop. As the loop executes, it will run a sequence like +if/if/if/else. The ``if`` is inside the loop, encountered a number of times. If +the condition is ever true, a ``break`` will happen. If the condition is never +true, the ``else`` clause outside the loop will execute. - >>> for num in range(2, 10): - ... if num % 2 == 0: - ... print("Found an even number", num) - ... continue - ... print("Found an odd number", num) - ... - Found an even number 2 - Found an odd number 3 - Found an even number 4 - Found an odd number 5 - Found an even number 6 - Found an odd number 7 - Found an even number 8 - Found an odd number 9 +When used with a loop, the ``else`` clause has more in common with the ``else`` +clause of a :keyword:`try` statement than it does with that of ``if`` +statements: a ``try`` statement's ``else`` clause runs when no exception +occurs, and a loop's ``else`` clause runs when no ``break`` occurs. For more on +the ``try`` statement and exceptions, see :ref:`tut-handling`. .. _tut-pass: @@ -436,8 +462,8 @@ Defining Functions We can create a function that writes the Fibonacci series to an arbitrary boundary:: - >>> def fib(n): # write Fibonacci series up to n - ... """Print a Fibonacci series up to n.""" + >>> def fib(n): # write Fibonacci series less than n + ... """Print a Fibonacci series less than n.""" ... a, b = 0, 1 ... while a < n: ... print(a, end=' ') @@ -445,7 +471,7 @@ boundary:: ... print() ... >>> # Now call the function we just defined: - ... fib(2000) + >>> fib(2000) 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987 1597 .. index:: @@ -807,7 +833,7 @@ parameters as there is a ``/`` in the function definition:: File "", line 1, in TypeError: pos_only_arg() got some positional-only arguments passed as keyword arguments: 'arg' -The third function ``kwd_only_args`` only allows keyword arguments as indicated +The third function ``kwd_only_arg`` only allows keyword arguments as indicated by a ``*`` in the function definition:: >>> kwd_only_arg(3) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index de2827461e2f24..31941bc112a135 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -19,13 +19,13 @@ objects: .. method:: list.append(x) :noindex: - Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``. + Add an item to the end of the list. Similar to ``a[len(a):] = [x]``. .. method:: list.extend(iterable) :noindex: - Extend the list by appending all the items from the iterable. Equivalent to + Extend the list by appending all the items from the iterable. Similar to ``a[len(a):] = iterable``. @@ -56,7 +56,7 @@ objects: .. method:: list.clear() :noindex: - Remove all items from the list. Equivalent to ``del a[:]``. + Remove all items from the list. Similar to ``del a[:]``. .. method:: list.index(x[, start[, end]]) @@ -93,7 +93,7 @@ objects: .. method:: list.copy() :noindex: - Return a shallow copy of the list. Equivalent to ``a[:]``. + Return a shallow copy of the list. Similar to ``a[:]``. An example that uses most of the list methods:: @@ -126,7 +126,7 @@ Python. Another thing you might notice is that not all data can be sorted or compared. For instance, ``[None, 'hello', 10]`` doesn't sort because -integers can't be compared to strings and *None* can't be compared to +integers can't be compared to strings and ``None`` can't be compared to other types. Also, there are some types that don't have a defined ordering relation. For example, ``3+4j < 5+7j`` isn't a valid comparison. @@ -383,16 +383,16 @@ A tuple consists of a number of values separated by commas, for instance:: >>> t (12345, 54321, 'hello!') >>> # Tuples may be nested: - ... u = t, (1, 2, 3, 4, 5) + >>> u = t, (1, 2, 3, 4, 5) >>> u ((12345, 54321, 'hello!'), (1, 2, 3, 4, 5)) >>> # Tuples are immutable: - ... t[0] = 88888 + >>> t[0] = 88888 Traceback (most recent call last): File "", line 1, in TypeError: 'tuple' object does not support item assignment >>> # but they can contain mutable objects: - ... v = ([1, 2, 3], [3, 2, 1]) + >>> v = ([1, 2, 3], [3, 2, 1]) >>> v ([1, 2, 3], [3, 2, 1]) @@ -465,7 +465,7 @@ Here is a brief demonstration:: False >>> # Demonstrate set operations on unique letters from two words - ... + >>> >>> a = set('abracadabra') >>> b = set('alacazam') >>> a # unique letters in a diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 4058ebe8efdb42..4c61cbb2b5bc3d 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -20,12 +20,12 @@ complaint you get while you are still learning Python:: >>> while True print('Hello world') File "", line 1 while True print('Hello world') - ^ + ^^^^^ SyntaxError: invalid syntax -The parser repeats the offending line and displays a little 'arrow' pointing at -the earliest point in the line where the error was detected. The error is -caused by (or at least detected at) the token *preceding* the arrow: in the +The parser repeats the offending line and displays little 'arrow's pointing +at the token in the line where the error was detected. The error may be +caused by the absence of a token *before* the indicated token. In the example, the error is detected at the function :func:`print`, since a colon (``':'``) is missing before it. File name and line number are printed so you know where to look in case the input came from a script. @@ -45,14 +45,20 @@ programs, however, and result in error messages as shown here:: >>> 10 * (1/0) Traceback (most recent call last): File "", line 1, in + 10 * (1/0) + ~^~ ZeroDivisionError: division by zero >>> 4 + spam*3 Traceback (most recent call last): File "", line 1, in + 4 + spam*3 + ^^^^ NameError: name 'spam' is not defined >>> '2' + 2 Traceback (most recent call last): File "", line 1, in + '2' + 2 + ~~~~^~~ TypeError: can only concatenate str (not "int") to str The last line of the error message indicates what happened. Exceptions come in @@ -119,9 +125,9 @@ may name multiple exceptions as a parenthesized tuple, for example:: ... except (RuntimeError, TypeError, NameError): ... pass -A class in an :keyword:`except` clause is compatible with an exception if it is -the same class or a base class thereof (but not the other way around --- an -*except clause* listing a derived class is not compatible with a base class). +A class in an :keyword:`except` clause matches exceptions which are instances of the +class itself or one of its derived classes (but not the other way around --- an +*except clause* listing a derived class does not match instances of its base classes). For example, the following code will print B, C, D in that order:: class B(Exception): @@ -252,6 +258,7 @@ exception to occur. For example:: >>> raise NameError('HiThere') Traceback (most recent call last): File "", line 1, in + raise NameError('HiThere') NameError: HiThere The sole argument to :keyword:`raise` indicates the exception to be raised. @@ -275,6 +282,7 @@ re-raise the exception:: An exception flew by! Traceback (most recent call last): File "", line 2, in + raise NameError('HiThere') NameError: HiThere @@ -294,12 +302,15 @@ message:: ... Traceback (most recent call last): File "", line 2, in + open("database.sqlite") + ~~~~^^^^^^^^^^^^^^^^^^^ FileNotFoundError: [Errno 2] No such file or directory: 'database.sqlite' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "", line 4, in + raise RuntimeError("unable to handle error") RuntimeError: unable to handle error To indicate that an exception is a direct consequence of another, the @@ -320,6 +331,8 @@ This can be useful when you are transforming exceptions. For example:: ... Traceback (most recent call last): File "", line 2, in + func() + ~~~~^^ File "", line 2, in func ConnectionError @@ -327,6 +340,7 @@ This can be useful when you are transforming exceptions. For example:: Traceback (most recent call last): File "", line 4, in + raise RuntimeError('Failed to open database') from exc RuntimeError: Failed to open database It also allows disabling automatic exception chaining using the ``from None`` @@ -339,6 +353,7 @@ idiom:: ... Traceback (most recent call last): File "", line 4, in + raise RuntimeError from None RuntimeError For more information about chaining mechanics, see :ref:`bltin-exceptions`. @@ -381,6 +396,7 @@ example:: Goodbye, world! Traceback (most recent call last): File "", line 2, in + raise KeyboardInterrupt KeyboardInterrupt If a :keyword:`finally` clause is present, the :keyword:`!finally` @@ -448,7 +464,11 @@ A more complicated example:: executing finally clause Traceback (most recent call last): File "", line 1, in + divide("2", "1") + ~~~~~~^^^^^^^^^^ File "", line 3, in divide + result = x / y + ~~^~~ TypeError: unsupported operand type(s) for /: 'str' and 'str' As you can see, the :keyword:`finally` clause is executed in any event. The @@ -511,8 +531,11 @@ caught like any other exception. :: >>> f() + Exception Group Traceback (most recent call last): | File "", line 1, in + | f() + | ~^^ | File "", line 3, in f - | ExceptionGroup: there were problems + | raise ExceptionGroup('there were problems', excs) + | ExceptionGroup: there were problems (2 sub-exceptions) +-+---------------- 1 ---------------- | OSError: error 1 +---------------- 2 ---------------- @@ -560,10 +583,15 @@ other clauses and eventually to be reraised. :: There were SystemErrors + Exception Group Traceback (most recent call last): | File "", line 2, in + | f() + | ~^^ | File "", line 2, in f - | ExceptionGroup: group1 + | raise ExceptionGroup( + | ...<12 lines>... + | ) + | ExceptionGroup: group1 (1 sub-exception) +-+---------------- 1 ---------------- - | ExceptionGroup: group2 + | ExceptionGroup: group2 (1 sub-exception) +-+---------------- 1 ---------------- | RecursionError: 4 +------------------------------------ @@ -607,6 +635,7 @@ includes all notes, in the order they were added, after the exception. :: ... Traceback (most recent call last): File "", line 2, in + raise TypeError('bad type') TypeError: bad type Add some information Add some more information @@ -630,23 +659,33 @@ exception in the group has a note indicating when this error has occurred. :: >>> raise ExceptionGroup('We have some problems', excs) + Exception Group Traceback (most recent call last): | File "", line 1, in + | raise ExceptionGroup('We have some problems', excs) | ExceptionGroup: We have some problems (3 sub-exceptions) +-+---------------- 1 ---------------- | Traceback (most recent call last): | File "", line 3, in + | f() + | ~^^ | File "", line 2, in f + | raise OSError('operation failed') | OSError: operation failed | Happened in Iteration 1 +---------------- 2 ---------------- | Traceback (most recent call last): | File "", line 3, in + | f() + | ~^^ | File "", line 2, in f + | raise OSError('operation failed') | OSError: operation failed | Happened in Iteration 2 +---------------- 3 ---------------- | Traceback (most recent call last): | File "", line 3, in + | f() + | ~^^ | File "", line 2, in f + | raise OSError('operation failed') | OSError: operation failed | Happened in Iteration 3 +------------------------------------ diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index 0795e2fef98830..dfe2d1d3a8378f 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -6,7 +6,7 @@ .. _tut-fp-issues: ************************************************** -Floating Point Arithmetic: Issues and Limitations +Floating-Point Arithmetic: Issues and Limitations ************************************************** .. sectionauthor:: Tim Peters @@ -88,7 +88,7 @@ the one with 17 significant digits, ``0.10000000000000001``. Starting with Python 3.1, Python (on most systems) is now able to choose the shortest of these and simply display ``0.1``. -Note that this is in the very nature of binary floating-point: this is not a bug +Note that this is in the very nature of binary floating point: this is not a bug in Python, and it is not a bug in your code either. You'll see the same kind of thing in all languages that support your hardware's floating-point arithmetic (although some languages may not *display* the difference by default, or in all @@ -148,13 +148,13 @@ Binary floating-point arithmetic holds many surprises like this. The problem with "0.1" is explained in precise detail below, in the "Representation Error" section. See `Examples of Floating Point Problems `_ for -a pleasant summary of how binary floating-point works and the kinds of +a pleasant summary of how binary floating point works and the kinds of problems commonly encountered in practice. Also see `The Perils of Floating Point `_ for a more complete account of other common surprises. As that says near the end, "there are no easy answers." Still, don't be unduly -wary of floating-point! The errors in Python float operations are inherited +wary of floating point! The errors in Python float operations are inherited from the floating-point hardware, and on most machines are on the order of no more than 1 part in 2\*\*53 per operation. That's more than adequate for most tasks, but you do need to keep in mind that it's not decimal arithmetic and @@ -230,7 +230,7 @@ accumulate to the point where they affect the final total: >>> sum([0.1] * 10) == 1.0 True -The :func:`math.fsum()` goes further and tracks all of the "lost digits" +The :func:`math.fsum` goes further and tracks all of the "lost digits" as values are added onto a running total so that the result has only a single rounding. This is slower than :func:`sum` but will be more accurate in uncommon cases where large magnitude inputs mostly cancel diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index fe9ca9ccb9c7e0..35b8c7cd8eb049 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -37,16 +37,23 @@ printing space-separated values. There are several ways to format output. * The :meth:`str.format` method of strings requires more manual effort. You'll still use ``{`` and ``}`` to mark where a variable will be substituted and can provide detailed formatting directives, - but you'll also need to provide the information to be formatted. + but you'll also need to provide the information to be formatted. In the following code + block there are two examples of how to format variables: + :: >>> yes_votes = 42_572_654 - >>> no_votes = 43_132_495 - >>> percentage = yes_votes / (yes_votes + no_votes) + >>> total_votes = 85_705_149 + >>> percentage = yes_votes / total_votes >>> '{:-9} YES votes {:2.2%}'.format(yes_votes, percentage) ' 42572654 YES votes 49.67%' + Notice how the ``yes_votes`` are padded with spaces and a negative sign only for negative numbers. + The example also prints ``percentage`` multiplied by 100, with 2 decimal + places and followed by a percent sign (see :ref:`formatspec` for details). + + * Finally, you can do all the string handling yourself by using string slicing and concatenation operations to create any layout you can imagine. The string type has some methods that perform useful operations for padding @@ -80,12 +87,12 @@ Some examples:: >>> print(s) The value of x is 32.5, and y is 40000... >>> # The repr() of a string adds string quotes and backslashes: - ... hello = 'hello, world\n' + >>> hello = 'hello, world\n' >>> hellos = repr(hello) >>> print(hellos) 'hello, world\n' >>> # The argument to repr() may be any Python object: - ... repr((x, y, ('spam', 'eggs'))) + >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" The :mod:`string` module contains a :class:`~string.Template` class that offers @@ -93,6 +100,13 @@ yet another way to substitute values into strings, using placeholders like ``$x`` and replacing them with values from a dictionary, but offers much less control of the formatting. +.. index:: + single: formatted string literal + single: interpolated string literal + single: string; formatted literal + single: string; interpolated literal + single: f-string + single: fstring .. _tut-f-strings: @@ -197,7 +211,12 @@ notation. :: Jack: 4098; Sjoerd: 4127; Dcab: 8637678 This is particularly useful in combination with the built-in function -:func:`vars`, which returns a dictionary containing all local variables. +:func:`vars`, which returns a dictionary containing all local variables:: + + >>> table = {k: str(v) for k, v in vars().items()} + >>> message = " ".join([f'{k}: ' + '{' + k +'};' for k in table.keys()]) + >>> print(message.format(**table)) + __name__: __main__; __doc__: None; __package__: None; __loader__: ... As an example, the following lines produce a tidily aligned set of columns giving integers and their squares and cubes:: @@ -267,9 +286,11 @@ left with zeros. It understands about plus and minus signs:: Old string formatting --------------------- -The % operator (modulo) can also be used for string formatting. Given ``'string' -% values``, instances of ``%`` in ``string`` are replaced with zero or more -elements of ``values``. This operation is commonly known as string +The % operator (modulo) can also be used for string formatting. +Given ``format % values`` (where *format* is a string), +``%`` conversion specifications in *format* are replaced with +zero or more elements of *values*. +This operation is commonly known as string interpolation. For example:: >>> import math diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 42ebf2b3d294a8..02e7de77322e99 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,13 +10,13 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.13` +The Python interpreter is usually installed as |usr_local_bin_python_x_dot_y_literal| on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.13 + python3.14 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -24,7 +24,7 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) On Windows machines where you have installed Python from the :ref:`Microsoft Store -`, the :file:`python3.13` command will be available. If you have +`, the |python_x_dot_y_literal| command will be available. If you have the :ref:`py.exe launcher ` installed, you can use the :file:`py` command. See :ref:`setting-envvars` for other ways to launch Python. @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.13 - Python 3.13 (default, April 4 2023, 09:25:04) + $ python3.14 + Python 3.14 (default, April 4 2024, 09:25:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 4536ab9486d39c..65e3b1938bca9c 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -62,7 +62,7 @@ For example:: 20 >>> (50 - 5*6) / 4 5.0 - >>> 8 / 5 # division always returns a floating point number + >>> 8 / 5 # division always returns a floating-point number 1.6 The integer numbers (e.g. ``2``, ``4``, ``20``) have type :class:`int`, @@ -197,21 +197,19 @@ and workarounds. String literals can span multiple lines. One way is using triple-quotes: ``"""..."""`` or ``'''...'''``. End of lines are automatically included in the string, but it's possible to prevent this by adding a ``\`` at -the end of the line. The following example:: - - print("""\ +the end of the line. In the following example, the initial newline is not +included:: + + >>> print("""\ + ... Usage: thingy [OPTIONS] + ... -h Display this usage message + ... -H hostname Hostname to connect to + ... """) Usage: thingy [OPTIONS] -h Display this usage message -H hostname Hostname to connect to - """) - -produces the following output (note that the initial newline is not included): - -.. code-block:: text - Usage: thingy [OPTIONS] - -h Display this usage message - -H hostname Hostname to connect to + >>> Strings can be concatenated (glued together) with the ``+`` operator, and repeated with ``*``:: @@ -405,13 +403,6 @@ indexed and sliced:: >>> squares[-3:] # slicing returns a new list [9, 16, 25] -All slice operations return a new list containing the requested elements. This -means that the following slice returns a -:ref:`shallow copy ` of the list:: - - >>> squares[:] - [1, 4, 9, 16, 25] - Lists also support operations like concatenation:: >>> squares + [36, 49, 64, 81, 100] @@ -435,6 +426,30 @@ the :meth:`!list.append` *method* (we will see more about methods later):: >>> cubes [1, 8, 27, 64, 125, 216, 343] +Simple assignment in Python never copies data. When you assign a list +to a variable, the variable refers to the *existing list*. +Any changes you make to the list through one variable will be seen +through all other variables that refer to it.:: + + >>> rgb = ["Red", "Green", "Blue"] + >>> rgba = rgb + >>> id(rgb) == id(rgba) # they reference the same object + True + >>> rgba.append("Alph") + >>> rgb + ["Red", "Green", "Blue", "Alph"] + +All slice operations return a new list containing the requested elements. This +means that the following slice returns a +:ref:`shallow copy ` of the list:: + + >>> correct_rgba = rgba[:] + >>> correct_rgba[-1] = "Alpha" + >>> correct_rgba + ["Red", "Green", "Blue", "Alpha"] + >>> rgba + ["Red", "Green", "Blue", "Alph"] + Assignment to slices is also possible, and this can even change the size of the list or clear it entirely:: @@ -484,8 +499,8 @@ together. For instance, we can write an initial sub-sequence of the as follows:: >>> # Fibonacci series: - ... # the sum of two elements defines the next - ... a, b = 0, 1 + >>> # the sum of two elements defines the next + >>> a, b = 0, 1 >>> while a < 10: ... print(a) ... a, b = b, a+b @@ -527,7 +542,7 @@ This example introduces several new features. * The :func:`print` function writes the value of the argument(s) it is given. It differs from just writing the expression you want to write (as we did earlier in the calculator examples) in the way it handles multiple arguments, - floating point quantities, and strings. Strings are printed without quotes, + floating-point quantities, and strings. Strings are printed without quotes, and a space is inserted between items, so you can format things nicely, like this:: diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 0316239e776a95..de7aa0e2342946 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -585,8 +585,9 @@ as the main module of a Python application must always use absolute imports. Packages in Multiple Directories -------------------------------- -Packages support one more special attribute, :attr:`__path__`. This is -initialized to be a list containing the name of the directory holding the +Packages support one more special attribute, :attr:`~module.__path__`. This is +initialized to be a :term:`sequence` of strings containing the name of the +directory holding the package's :file:`__init__.py` before the code in that file is executed. This variable can be modified; doing so affects future searches for modules and subpackages contained in the package. diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 63f4b5e1ce0207..4b3eef313e76d7 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python312' + 'C:\\Python314' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -138,7 +138,7 @@ Mathematics =========== The :mod:`math` module gives access to the underlying C library functions for -floating point math:: +floating-point math:: >>> import math >>> math.cos(math.pi / 4) diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 33f311db3a24d2..a2f96b34b2dead 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -279,7 +279,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python312/lib/weakref.py", line 46, in __getitem__ + File "C:/python314/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' @@ -293,7 +293,7 @@ Many data structure needs can be met with the built-in list type. However, sometimes there is a need for alternative implementations with different performance trade-offs. -The :mod:`array` module provides an :class:`~array.array()` object that is like +The :mod:`array` module provides an :class:`~array.array` object that is like a list that stores only homogeneous data and stores it more compactly. The following example shows an array of numbers stored as two byte unsigned binary numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular @@ -306,7 +306,7 @@ lists of Python int objects:: >>> a[1:3] array('H', [10, 700]) -The :mod:`collections` module provides a :class:`~collections.deque()` object +The :mod:`collections` module provides a :class:`~collections.deque` object that is like a list with faster appends and pops from the left side but slower lookups in the middle. These objects are well suited for implementing queues and breadth first tree searches:: @@ -352,11 +352,11 @@ not want to run a full list sort:: .. _tut-decimal-fp: -Decimal Floating Point Arithmetic +Decimal Floating-Point Arithmetic ================================= The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for -decimal floating point arithmetic. Compared to the built-in :class:`float` +decimal floating-point arithmetic. Compared to the built-in :class:`float` implementation of binary floating point, the class is especially helpful for * financial applications and other uses which require exact decimal diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index a6dead2eac11f6..f362e1943b666f 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -36,10 +36,10 @@ Creating Virtual Environments ============================= The module used to create and manage virtual environments is called -:mod:`venv`. :mod:`venv` will usually install the most recent version of -Python that you have available. If you have multiple versions of Python on your -system, you can select a specific Python version by running ``python3`` or -whichever version you want. +:mod:`venv`. :mod:`venv` will install the Python version from which +the command was run (as reported by the :option:`--version` option). +For instance, executing the command with ``python3.12`` will install +version 3.12. To create a virtual environment, decide upon a directory where you want to place it, and run the :mod:`venv` module as a script with the directory path:: @@ -76,7 +76,7 @@ virtual environment you're using, and modify the environment so that running ``python`` will get you that particular version and installation of Python. For example: -.. code-block:: bash +.. code-block:: console $ source ~/envs/tutorial-env/bin/activate (tutorial-env) $ python @@ -108,7 +108,7 @@ complete documentation for ``pip``.) You can install the latest version of a package by specifying a package's name: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install novas Collecting novas @@ -120,7 +120,7 @@ You can install the latest version of a package by specifying a package's name: You can also install a specific version of a package by giving the package name followed by ``==`` and the version number: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install requests==2.6.0 Collecting requests==2.6.0 @@ -133,7 +133,7 @@ version is already installed and do nothing. You can supply a different version number to get that version, or you can run ``python -m pip install --upgrade`` to upgrade the package to the latest version: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install --upgrade requests Collecting requests @@ -148,7 +148,7 @@ remove the packages from the virtual environment. ``python -m pip show`` will display information about a particular package: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip show requests --- @@ -166,7 +166,7 @@ remove the packages from the virtual environment. ``python -m pip list`` will display all of the packages installed in the virtual environment: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip list novas (3.1.1.3) @@ -179,7 +179,7 @@ the virtual environment: but the output uses the format that ``python -m pip install`` expects. A common convention is to put this list in a ``requirements.txt`` file: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip freeze > requirements.txt (tutorial-env) $ cat requirements.txt @@ -191,7 +191,7 @@ The ``requirements.txt`` can then be committed to version control and shipped as part of an application. Users can then install all the necessary packages with ``install -r``: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install -r requirements.txt Collecting novas==3.1.1.3 (from -r requirements.txt (line 1)) diff --git a/Doc/using/android.rst b/Doc/using/android.rst new file mode 100644 index 00000000000000..957705f7f5e189 --- /dev/null +++ b/Doc/using/android.rst @@ -0,0 +1,65 @@ +.. _using-android: + +======================= +Using Python on Android +======================= + +Python on Android is unlike Python on desktop platforms. On a desktop platform, +Python is generally installed as a system resource that can be used by any user +of that computer. Users then interact with Python by running a :program:`python` +executable and entering commands at an interactive prompt, or by running a +Python script. + +On Android, there is no concept of installing as a system resource. The only unit +of software distribution is an "app". There is also no console where you could +run a :program:`python` executable, or interact with a Python REPL. + +As a result, the only way you can use Python on Android is in embedded mode – that +is, by writing a native Android application, embedding a Python interpreter +using ``libpython``, and invoking Python code using the :ref:`Python embedding +API `. The full Python interpreter, the standard library, and all +your Python code is then packaged into your app for its own private use. + +The Python standard library has some notable omissions and restrictions on +Android. See the :ref:`API availability guide ` for +details. + +Adding Python to an Android app +------------------------------- + +These instructions are only needed if you're planning to compile Python for +Android yourself. Most users should *not* need to do this. Instead, use one of +the following tools, which will provide a much easier experience: + +* `Briefcase `__, from the BeeWare project +* `Buildozer `__, from the Kivy project +* `Chaquopy `__ +* `pyqtdeploy `__ +* `Termux `__ + +If you're sure you want to do all of this manually, read on. You can use the +:source:`testbed app ` as a guide; each step below contains a +link to the relevant file. + +* Build Python by following the instructions in :source:`Android/README.md`. + +* Add code to your :source:`build.gradle ` + file to copy the following items into your project. All except your own Python + code can be copied from ``cross-build/HOST/prefix/lib``: + + * In your JNI libraries: + + * ``libpython*.*.so`` + * ``lib*_python.so`` (external libraries such as OpenSSL) + + * In your assets: + + * ``python*.*`` (the Python standard library) + * ``python*.*/site-packages`` (your own Python code) + +* Add code to your app to :source:`extract the assets to the filesystem + `. + +* Add code to your app to :source:`start Python in embedded mode + `. This will need to be C code + called via JNI. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 53c95ca1a05c9b..6cf42b27718022 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -24,7 +24,7 @@ Command line When invoking Python, you may specify any of these options:: - python [-bBdEhiIOqsSuvVWx?] [-c command | -m module-name | script | - ] [args] + python [-bBdEhiIOPqRsSuvVWx?] [-c command | -m module-name | script | - ] [args] The most common use case is, of course, a simple invocation of a script:: @@ -42,6 +42,7 @@ additional methods of invocation: * When called with standard input connected to a tty device, it prompts for commands and executes them until an EOF (an end-of-file character, you can produce that with :kbd:`Ctrl-D` on UNIX or :kbd:`Ctrl-Z, Enter` on Windows) is read. + For more on interactive mode, see :ref:`tut-interac`. * When called with a file name argument or with a file as standard input, it reads and executes a script from that file. * When called with a directory name argument, it reads and executes an @@ -242,12 +243,13 @@ Miscellaneous options .. option:: -b - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error when the - option is given twice (:option:`!-bb`). + Issue a warning when converting :class:`bytes` or :class:`bytearray` to + :class:`str` without specifying encoding or comparing :class:`!bytes` or + :class:`!bytearray` with :class:`!str` or :class:`!bytes` with :class:`int`. + Issue an error when the option is given twice (:option:`!-bb`). .. versionchanged:: 3.5 - Affects comparisons of :class:`bytes` with :class:`int`. + Affects also comparisons of :class:`bytes` with :class:`int`. .. option:: -B @@ -288,9 +290,15 @@ Miscellaneous options .. option:: -i - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. The + Enter interactive mode after execution. + + Using the :option:`-i` option will enter interactive mode in any of the following circumstances\: + + * When a script is passed as first argument + * When the :option:`-c` option is used + * When the :option:`-m` option is used + + Interactive mode will start even when :data:`sys.stdin` does not appear to be a terminal. The :envvar:`PYTHONSTARTUP` file is not read. This can be useful to inspect global variables or a stack trace when a script @@ -375,17 +383,19 @@ Miscellaneous options :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. + .. versionadded:: 3.2.3 + .. versionchanged:: 3.7 The option is no longer ignored. - .. versionadded:: 3.2.3 - .. option:: -s Don't add the :data:`user site-packages directory ` to :data:`sys.path`. + See also :envvar:`PYTHONNOUSERSITE`. + .. seealso:: :pep:`370` -- Per user site-packages directory @@ -437,6 +447,7 @@ Miscellaneous options -Wdefault # Warn once per call location -Werror # Convert to exceptions -Walways # Warn every time + -Wall # Same as -Walways -Wmodule # Warn once per calling module -Wonce # Warn once per Python process -Wignore # Never warn @@ -497,43 +508,73 @@ Miscellaneous options * ``-X faulthandler`` to enable :mod:`faulthandler`. See also :envvar:`PYTHONFAULTHANDLER`. + + .. versionadded:: 3.3 + * ``-X showrefcount`` to output the total reference count and number of used memory blocks when the program finishes or after each statement in the interactive interpreter. This only works on :ref:`debug builds `. + + .. versionadded:: 3.4 + * ``-X tracemalloc`` to start tracing Python memory allocations using the :mod:`tracemalloc` module. By default, only the most recent frame is stored in a traceback of a trace. Use ``-X tracemalloc=NFRAME`` to start tracing with a traceback limit of *NFRAME* frames. See :func:`tracemalloc.start` and :envvar:`PYTHONTRACEMALLOC` for more information. + + .. versionadded:: 3.4 + * ``-X int_max_str_digits`` configures the :ref:`integer string conversion length limitation `. See also :envvar:`PYTHONINTMAXSTRDIGITS`. + + .. versionadded:: 3.11 + * ``-X importtime`` to show how long each import takes. It shows module name, cumulative time (including nested imports) and self time (excluding nested imports). Note that its output may be broken in multi-threaded application. Typical usage is ``python3 -X importtime -c 'import asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. + + .. versionadded:: 3.7 + * ``-X dev``: enable :ref:`Python Development Mode `, introducing additional runtime checks that are too expensive to be enabled by - default. + default. See also :envvar:`PYTHONDEVMODE`. + + .. versionadded:: 3.7 + * ``-X utf8`` enables the :ref:`Python UTF-8 Mode `. ``-X utf8=0`` explicitly disables :ref:`Python UTF-8 Mode ` (even when it would otherwise activate automatically). See also :envvar:`PYTHONUTF8`. + + .. versionadded:: 3.7 + * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel tree rooted at the given directory instead of to the code tree. See also :envvar:`PYTHONPYCACHEPREFIX`. + + .. versionadded:: 3.8 + * ``-X warn_default_encoding`` issues a :class:`EncodingWarning` when the locale-specific default encoding is used for opening files. See also :envvar:`PYTHONWARNDEFAULTENCODING`. + + .. versionadded:: 3.10 + * ``-X no_debug_ranges`` disables the inclusion of the tables mapping extra location information (end line, start column offset and end column offset) to every instruction in code objects. This is useful when smaller code objects and pyc files are desired as well as suppressing the extra visual location indicators when the interpreter displays tracebacks. See also :envvar:`PYTHONNODEBUGRANGES`. + + .. versionadded:: 3.11 + * ``-X frozen_modules`` determines whether or not frozen modules are ignored by the import machinery. A value of ``on`` means they get imported and ``off`` means they are ignored. The default is ``on`` @@ -542,17 +583,35 @@ Miscellaneous options Note that the :mod:`!importlib_bootstrap` and :mod:`!importlib_bootstrap_external` frozen modules are always used, even if this flag is set to ``off``. See also :envvar:`PYTHON_FROZEN_MODULES`. + + .. versionadded:: 3.11 + * ``-X perf`` enables support for the Linux ``perf`` profiler. When this option is provided, the ``perf`` profiler will be able to report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. + + .. versionadded:: 3.12 + + * ``-X perf_jit`` enables support for the Linux ``perf`` profiler with DWARF + support. When this option is provided, the ``perf`` profiler will be able + to report Python calls using DWARF information. This option is only available on + some platforms and will do nothing if is not supported on the current + system. The default value is "off". See also :envvar:`PYTHON_PERF_JIT_SUPPORT` + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`, :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. *n* must be greater than or equal to 1. This option may be useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHON_CPU_COUNT`. If *n* is ``default``, nothing is overridden. + + .. versionadded:: 3.13 + * :samp:`-X presite={package.module}` specifies a module that should be imported before the :mod:`site` module is executed and before the :mod:`__main__` module exists. Therefore, the imported module isn't @@ -560,56 +619,26 @@ Miscellaneous options initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. - It also allows passing arbitrary values and retrieving them through the - :data:`sys._xoptions` dictionary. - - .. versionchanged:: 3.2 - The :option:`-X` option was added. + .. versionadded:: 3.13 - .. versionadded:: 3.3 - The ``-X faulthandler`` option. + * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, + respectively. Setting to ``0`` is only available in builds configured with + :option:`--disable-gil`. See also :envvar:`PYTHON_GIL` and + :ref:`whatsnew313-free-threaded-cpython`. - .. versionadded:: 3.4 - The ``-X showrefcount`` and ``-X tracemalloc`` options. + .. versionadded:: 3.13 - .. versionadded:: 3.6 - The ``-X showalloccount`` option. - - .. versionadded:: 3.7 - The ``-X importtime``, ``-X dev`` and ``-X utf8`` options. + It also allows passing arbitrary values and retrieving them through the + :data:`sys._xoptions` dictionary. - .. versionadded:: 3.8 - The ``-X pycache_prefix`` option. The ``-X dev`` option now logs - ``close()`` exceptions in :class:`io.IOBase` destructor. + .. versionadded:: 3.2 .. versionchanged:: 3.9 - Using ``-X dev`` option, check *encoding* and *errors* arguments on - string encoding and decoding operations. - - The ``-X showalloccount`` option has been removed. + Removed the ``-X showalloccount`` option. - .. versionadded:: 3.10 - The ``-X warn_default_encoding`` option. + .. versionchanged:: 3.10 Removed the ``-X oldparser`` option. - .. versionadded:: 3.11 - The ``-X no_debug_ranges`` option. - - .. versionadded:: 3.11 - The ``-X frozen_modules`` option. - - .. versionadded:: 3.11 - The ``-X int_max_str_digits`` option. - - .. versionadded:: 3.12 - The ``-X perf`` option. - - .. versionadded:: 3.13 - The ``-X cpu_count`` option. - - .. versionadded:: 3.13 - The ``-X presite`` option. - .. _using-on-controlling-color: Controlling color @@ -621,11 +650,11 @@ behavior can be controlled by setting different environment variables. Setting the environment variable ``TERM`` to ``dumb`` will disable color. -If the environment variable ``FORCE_COLOR`` is set, then color will be +If the |FORCE_COLOR|_ environment variable is set, then color will be enabled regardless of the value of TERM. This is useful on CI systems which aren’t terminals but can still display ANSI escape sequences. -If the environment variable ``NO_COLOR`` is set, Python will disable all color +If the |NO_COLOR|_ environment variable is set, Python will disable all color in the output. This takes precedence over ``FORCE_COLOR``. All these environment variables are used also by other tools to control color @@ -634,6 +663,14 @@ output. To control the color output only in the Python interpreter, the precedence over ``NO_COLOR``, which in turn takes precedence over ``FORCE_COLOR``. +.. Apparently this how you hack together a formatted link: + +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ + +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ + Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -756,6 +793,15 @@ conflict. This variable can also be modified by Python code using :data:`os.environ` to force inspect mode on program termination. + .. audit-event:: cpython.run_stdin "" "" + + .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) + Emits audit events. + + .. versionchanged:: 3.13 + Uses PyREPL if possible, in which case :envvar:`PYTHONSTARTUP` is + also executed. Emits audit events. + .. envvar:: PYTHONUNBUFFERED @@ -879,6 +925,7 @@ conflict. PYTHONWARNINGS=default # Warn once per call location PYTHONWARNINGS=error # Convert to exceptions PYTHONWARNINGS=always # Warn every time + PYTHONWARNINGS=all # Same as PYTHONWARNINGS=always PYTHONWARNINGS=module # Warn once per calling module PYTHONWARNINGS=once # Warn once per Python process PYTHONWARNINGS=ignore # Never warn @@ -955,11 +1002,11 @@ conflict. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. * ``mimalloc_debug``: same as ``mimalloc`` but also install debug hooks. + .. versionadded:: 3.6 + .. versionchanged:: 3.7 Added the ``"default"`` allocator. - .. versionadded:: 3.6 - .. envvar:: PYTHONMALLOCSTATS @@ -984,7 +1031,7 @@ conflict. 'surrogatepass' are used. This may also be enabled at runtime with - :func:`sys._enablelegacywindowsfsencoding()`. + :func:`sys._enablelegacywindowsfsencoding`. .. availability:: Windows. @@ -1108,6 +1155,21 @@ conflict. .. versionadded:: 3.12 +.. envvar:: PYTHON_PERF_JIT_SUPPORT + + If this variable is set to a nonzero value, it enables support for + the Linux ``perf`` profiler so Python calls can be detected by it + using DWARF information. + + If set to ``0``, disable Linux ``perf`` profiler support. + + See also the :option:`-X perf_jit <-X>` command-line option + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + + + .. envvar:: PYTHON_CPU_COUNT If this variable is set to a positive integer, it overrides the return @@ -1139,6 +1201,15 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_REPL + + If this variable is set to ``1``, the interpreter will not attempt to + load the Python-based :term:`REPL` that requires :mod:`curses` and + :mod:`readline`, and will instead use the traditional parser-based + :term:`REPL`. + + .. versionadded:: 3.13 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a @@ -1147,6 +1218,17 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_GIL + + If this variable is set to ``1``, the global interpreter lock (GIL) will be + forced on. Setting it to ``0`` forces the GIL off (needs Python configured with + the :option:`--disable-gil` build option). + + See also the :option:`-X gil <-X>` command-line option, which takes + precedence over this variable, and :ref:`whatsnew313-free-threaded-cpython`. + + .. versionadded:: 3.13 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index aab9469b44828a..83994af795e3fc 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -2,6 +2,8 @@ Configure Python **************** +.. highlight:: sh + Build Requirements ================== @@ -14,8 +16,8 @@ Features and minimum versions required to build CPython: * On Windows, Microsoft Visual Studio 2017 or later is required. -* Support for `IEEE 754 `_ floating - point numbers and `floating point Not-a-Number (NaN) +* Support for `IEEE 754 `_ + floating-point numbers and `floating-point Not-a-Number (NaN) `_. * Support for threads. @@ -27,34 +29,34 @@ Features and minimum versions required to build CPython: * Tcl/Tk 8.5.12 for the :mod:`tkinter` module. -* Autoconf 2.71 and aclocal 1.16.4 are required to regenerate the +* Autoconf 2.71 and aclocal 1.16.5 are required to regenerate the :file:`configure` script. -.. versionchanged:: 3.13: - Autoconf 2.71, aclocal 1.16.4 and SQLite 3.15.2 are now required. +.. versionchanged:: 3.1 + Tcl/Tk version 8.3.1 is now required. -.. versionchanged:: 3.11 - C11 compiler, IEEE 754 and NaN support are now required. - On Windows, Visual Studio 2017 or later is required. - Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. +.. versionchanged:: 3.5 + On Windows, Visual Studio 2015 or later is now required. + Tcl/Tk version 8.4 is now required. -.. versionchanged:: 3.10 - OpenSSL 1.1.1 is now required. - Require SQLite 3.7.15. +.. versionchanged:: 3.6 + Selected C99 features are now required, like ```` and ``static + inline`` functions. .. versionchanged:: 3.7 Thread support and OpenSSL 1.0.2 are now required. -.. versionchanged:: 3.6 - Selected C99 features are now required, like ```` and ``static - inline`` functions. +.. versionchanged:: 3.10 + OpenSSL 1.1.1 is now required. + Require SQLite 3.7.15. -.. versionchanged:: 3.5 - On Windows, Visual Studio 2015 or later is now required. - Tcl/Tk version 8.4 is now required. +.. versionchanged:: 3.11 + C11 compiler, IEEE 754 and NaN support are now required. + On Windows, Visual Studio 2017 or later is required. + Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. -.. versionchanged:: 3.1 - Tcl/Tk version 8.3.1 is now required. +.. versionchanged:: 3.13 + Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". @@ -181,12 +183,6 @@ General Options See :envvar:`PYTHONCOERCECLOCALE` and the :pep:`538`. -.. option:: --without-freelists - - Disable all freelists except the empty tuple singleton. - - .. versionadded:: 3.11 - .. option:: --with-platlibdir=DIRNAME Python library directory name (default is ``lib``). @@ -275,7 +271,7 @@ General Options * to/from free lists; * dictionary materialized/dematerialized; * type cache; - * optimization attemps; + * optimization attempts; * optimization traces created/executed; * uops executed. @@ -297,7 +293,7 @@ General Options Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to :data:`sys.abiflags`. - See :pep:`703` "Making the Global Interpreter Lock Optional in CPython". + See :ref:`whatsnew313-free-threaded-cpython` for more detail. .. versionadded:: 3.13 @@ -387,6 +383,17 @@ Options for third-party dependencies C compiler and linker flags for ``libffi``, used by :mod:`ctypes` module, overriding ``pkg-config``. +.. option:: LIBMPDEC_CFLAGS +.. option:: LIBMPDEC_LIBS + + C compiler and linker flags for ``libmpdec``, used by :mod:`decimal` module, + overriding ``pkg-config``. + + .. note:: + + These environment variables have no effect unless + :option:`--with-system-libmpdec` is specified. + .. option:: LIBLZMA_CFLAGS .. option:: LIBLZMA_LIBS @@ -414,7 +421,7 @@ Options for third-party dependencies .. option:: PANEL_CFLAGS .. option:: PANEL_LIBS - C compiler and Linker flags for PANEL, overriding ``pkg-config``. + C compiler and linker flags for PANEL, overriding ``pkg-config``. C compiler and linker flags for ``libpanel`` or ``libpanelw``, used by :mod:`curses.panel` module, overriding ``pkg-config``. @@ -516,6 +523,15 @@ also be used to improve performance. GCC is used: add ``-fno-semantic-interposition`` to the compiler and linker flags. + .. note:: + + During the build, you may encounter compiler warnings about + profile data not being available for some source files. + These warnings are harmless, as only a subset of the code is exercised + during profile data acquisition. + To disable these warnings on Clang, manually suppress them by adding + ``-Wno-profile-instr-unprofiled`` to :envvar:`CFLAGS`. + .. versionadded:: 3.6 .. versionchanged:: 3.10 @@ -593,7 +609,7 @@ also be used to improve performance. .. option:: --without-mimalloc - Disable the fast mimalloc allocator :ref:`mimalloc ` + Disable the fast :ref:`mimalloc ` allocator (enabled by default). See also :envvar:`PYTHONMALLOC` environment variable. @@ -686,7 +702,7 @@ Debug options Effects: * Define the ``Py_TRACE_REFS`` macro. - * Add :func:`!sys.getobjects` function. + * Add :func:`sys.getobjects` function. * Add :envvar:`PYTHONDUMPREFS` environment variable. The :envvar:`PYTHONDUMPREFS` environment variable can be used to dump @@ -694,12 +710,12 @@ Debug options :ref:`Statically allocated objects ` are not traced. + .. versionadded:: 3.8 + .. versionchanged:: 3.13 This build is now ABI compatible with release build and :ref:`debug build `. - .. versionadded:: 3.8 - .. option:: --with-assertions Build with C assertions enabled (default is no): ``assert(...);`` and @@ -782,11 +798,20 @@ Libraries options .. option:: --with-system-libmpdec - Build the ``_decimal`` extension module using an installed ``mpdec`` - library, see the :mod:`decimal` module (default is no). + Build the ``_decimal`` extension module using an installed ``mpdecimal`` + library, see the :mod:`decimal` module (default is yes). .. versionadded:: 3.3 + .. versionchanged:: 3.13 + Default to using the installed ``mpdecimal`` library. + + .. deprecated-removed:: 3.13 3.15 + A copy of the ``mpdecimal`` library sources will no longer be distributed + with Python 3.15. + + .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. + .. option:: --with-readline=readline|editline Designate a backend library for the :mod:`readline` module. @@ -876,10 +901,42 @@ Security Options The settings ``python`` and *STRING* also set TLS 1.2 as minimum protocol version. +.. option:: --disable-safety + + Disable compiler options that are `recommended by OpenSSF`_ for security reasons with no performance overhead. + If this option is not enabled, CPython will be built based on safety compiler options with no slow down. + When this option is enabled, CPython will not be built with the compiler options listed below. + + The following compiler options are disabled with :option:`!--disable-safety`: + + * `-fstack-protector-strong`_: Enable run-time checks for stack-based buffer overflows. + * `-Wtrampolines`_: Enable warnings about trampolines that require executable stacks. + + .. _recommended by OpenSSF: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md + .. _-fstack-protector-strong: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md#enable-run-time-checks-for-stack-based-buffer-overflows + .. _-Wtrampolines: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md#enable-warning-about-trampolines-that-require-executable-stacks + + .. versionadded:: 3.14 + +.. option:: --enable-slower-safety + + Enable compiler options that are `recommended by OpenSSF`_ for security reasons which require overhead. + If this option is not enabled, CPython will not be built based on safety compiler options which performance impact. + When this option is enabled, CPython will be built with the compiler options listed below. + + The following compiler options are enabled with :option:`!--enable-slower-safety`: + + * `-D_FORTIFY_SOURCE=3`_: Fortify sources with compile- and run-time checks for unsafe libc usage and buffer overflows. + + .. _-D_FORTIFY_SOURCE=3: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md#fortify-sources-for-unsafe-libc-usage-and-buffer-overflows + + .. versionadded:: 3.14 + + macOS Options ------------- -See ``Mac/README.rst``. +See :source:`Mac/README.rst`. .. option:: --enable-universalsdk .. option:: --enable-universalsdk=SDKDIR @@ -914,6 +971,31 @@ See ``Mac/README.rst``. Specify the name for the python framework on macOS only valid when :option:`--enable-framework` is set (default: ``Python``). +.. option:: --with-app-store-compliance +.. option:: --with-app-store-compliance=PATCH-FILE + + The Python standard library contains strings that are known to trigger + automated inspection tool errors when submitted for distribution by + the macOS and iOS App Stores. If enabled, this option will apply the list of + patches that are known to correct app store compliance. A custom patch + file can also be specified. This option is disabled by default. + + .. versionadded:: 3.13 + +iOS Options +----------- + +See :source:`iOS/README.rst`. + +.. option:: --enable-framework=INSTALLDIR + + Create a Python.framework. Unlike macOS, the *INSTALLDIR* argument + specifying the installation path is mandatory. + +.. option:: --with-framework-name=FRAMEWORK + + Specify the name for the framework (default: ``Python``). + Cross Compiling Options ----------------------- @@ -941,7 +1023,9 @@ the version of the cross compiled host Python. An environment variable that points to a file with configure overrides. - Example *config.site* file:: + Example *config.site* file: + + .. code-block:: ini # config.site-aarch64 ac_cv_buggy_getaddrinfo=no @@ -987,39 +1071,116 @@ Main build steps Main Makefile targets --------------------- -* ``make``: Build Python with the standard library. -* ``make platform:``: build the ``python`` program, but don't build the - standard library extension modules. -* ``make profile-opt``: build Python using Profile Guided Optimization (PGO). - You can use the configure :option:`--enable-optimizations` option to make - this the default target of the ``make`` command (``make all`` or just - ``make``). - -* ``make test``: Build Python and run the Python test suite with ``--fast-ci`` - option. Variables: - - * ``TESTOPTS``: additional regrtest command line options. - * ``TESTPYTHONOPTS``: additional Python command line options. - * ``TESTTIMEOUT``: timeout in seconds (default: 20 minutes). - -* ``make buildbottest``: Similar to ``make test``, but use ``--slow-ci`` - option and default timeout of 20 minutes, instead of ``--fast-ci`` option - and a default timeout of 10 minutes. - -* ``make install``: Build and install Python. -* ``make regen-all``: Regenerate (almost) all generated files; - ``make regen-stdlib-module-names`` and ``autoconf`` must be run separately - for the remaining generated files. -* ``make clean``: Remove built files. -* ``make distclean``: Same than ``make clean``, but remove also files created - by the configure script. +make +^^^^ + +For the most part, when rebuilding after editing some code or +refreshing your checkout from upstream, all you need to do is execute +``make``, which (per Make's semantics) builds the default target, the +first one defined in the Makefile. By tradition (including in the +CPython project) this is usually the ``all`` target. The +``configure`` script expands an ``autoconf`` variable, +``@DEF_MAKE_ALL_RULE@`` to describe precisely which targets ``make +all`` will build. The three choices are: + +* ``profile-opt`` (configured with ``--enable-optimizations``) +* ``build_wasm`` (configured with ``--with-emscripten-target``) +* ``build_all`` (configured without explicitly using either of the others) + +Depending on the most recent source file changes, Make will rebuild +any targets (object files and executables) deemed out-of-date, +including running ``configure`` again if necessary. Source/target +dependencies are many and maintained manually however, so Make +sometimes doesn't have all the information necessary to correctly +detect all targets which need to be rebuilt. Depending on which +targets aren't rebuilt, you might experience a number of problems. If +you have build or test problems which you can't otherwise explain, +``make clean && make`` should work around most dependency problems, at +the expense of longer build times. + + +make platform +^^^^^^^^^^^^^ + +Build the ``python`` program, but don't build the standard library +extension modules. This generates a file named ``platform`` which +contains a single line describing the details of the build platform, +e.g., ``macosx-14.3-arm64-3.12`` or ``linux-x86_64-3.13``. + + +make profile-opt +^^^^^^^^^^^^^^^^ + +Build Python using profile-guided optimization (PGO). You can use the +configure :option:`--enable-optimizations` option to make this the +default target of the ``make`` command (``make all`` or just +``make``). + + + +make clean +^^^^^^^^^^ + +Remove built files. + + +make distclean +^^^^^^^^^^^^^^ + +In addition to the work done by ``make clean``, remove files +created by the configure script. ``configure`` will have to be run +before building again. [#]_ + + +make install +^^^^^^^^^^^^ + +Build the ``all`` target and install Python. + + +make test +^^^^^^^^^ + +Build the ``all`` target and run the Python test suite with the +``--fast-ci`` option without GUI tests. Variables: + +* ``TESTOPTS``: additional regrtest command-line options. +* ``TESTPYTHONOPTS``: additional Python command-line options. +* ``TESTTIMEOUT``: timeout in seconds (default: 10 minutes). + + +make ci +^^^^^^^ + +This is similar to ``make test``, but uses the ``-ugui`` to also run GUI tests. + +.. versionadded:: 3.14 + + +make buildbottest +^^^^^^^^^^^^^^^^^ + +This is similar to ``make test``, but uses the ``--slow-ci`` +option and default timeout of 20 minutes, instead of ``--fast-ci`` option. + + +make regen-all +^^^^^^^^^^^^^^ + +Regenerate (almost) all generated files. These include (but are not +limited to) bytecode cases, and parser generator file. +``make regen-stdlib-module-names`` and ``autoconf`` must be run +separately for the remaining `generated files <#generated-files>`_. + C extensions ------------ Some C extensions are built as built-in modules, like the ``sys`` module. They are built with the ``Py_BUILD_CORE_BUILTIN`` macro defined. -Built-in modules have no ``__file__`` attribute:: +Built-in modules have no ``__file__`` attribute: + +.. code-block:: pycon >>> import sys >>> sys @@ -1031,7 +1192,9 @@ Built-in modules have no ``__file__`` attribute:: Other C extensions are built as dynamic libraries, like the ``_asyncio`` module. They are built with the ``Py_BUILD_CORE_MODULE`` macro defined. -Example on Linux x86-64:: +Example on Linux x86-64: + +.. code-block:: pycon >>> import _asyncio >>> _asyncio @@ -1303,3 +1466,14 @@ Linker flags Linker flags used for building the interpreter object files. .. versionadded:: 3.8 + + +.. rubric:: Footnotes + +.. [#] ``git clean -fdx`` is an even more extreme way to "clean" your + checkout. It removes all files not known to Git. + When bug hunting using ``git bisect``, this is + `recommended between probes `_ + to guarantee a completely clean build. **Use with care**, as it + will delete all files not checked into Git, including your + new, uncommitted work. diff --git a/Doc/using/index.rst b/Doc/using/index.rst index e1a3111f36a44f..90fdfc0bec0583 100644 --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -12,10 +12,13 @@ interpreter and things that make working with Python easier. .. toctree:: :numbered: + :maxdepth: 2 cmdline.rst unix.rst configure.rst windows.rst mac.rst + android.rst + ios.rst editors.rst diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst new file mode 100644 index 00000000000000..4d4eb2031ee980 --- /dev/null +++ b/Doc/using/ios.rst @@ -0,0 +1,343 @@ +.. _using-ios: + +=================== +Using Python on iOS +=================== + +:Authors: + Russell Keith-Magee (2024-03) + +Python on iOS is unlike Python on desktop platforms. On a desktop platform, +Python is generally installed as a system resource that can be used by any user +of that computer. Users then interact with Python by running a :program:`python` +executable and entering commands at an interactive prompt, or by running a +Python script. + +On iOS, there is no concept of installing as a system resource. The only unit +of software distribution is an "app". There is also no console where you could +run a :program:`python` executable, or interact with a Python REPL. + +As a result, the only way you can use Python on iOS is in embedded mode - that +is, by writing a native iOS application, and embedding a Python interpreter +using ``libPython``, and invoking Python code using the :ref:`Python embedding +API `. The full Python interpreter, the standard library, and all +your Python code is then packaged as a standalone bundle that can be +distributed via the iOS App Store. + +If you're looking to experiment for the first time with writing an iOS app in +Python, projects such as `BeeWare `__ and `Kivy +`__ will provide a much more approachable user experience. +These projects manage the complexities associated with getting an iOS project +running, so you only need to deal with the Python code itself. + +Python at runtime on iOS +======================== + +iOS version compatibility +------------------------- + +The minimum supported iOS version is specified at compile time, using the +:option:`--host` option to ``configure``. By default, when compiled for iOS, +Python will be compiled with a minimum supported iOS version of 13.0. To use a +different minimum iOS version, provide the version number as part of the +:option:`!--host` argument - for example, +``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 simulator build +with a deployment target of 15.4. + +Platform identification +----------------------- + +When executing on iOS, ``sys.platform`` will report as ``ios``. This value will +be returned on an iPhone or iPad, regardless of whether the app is running on +the simulator or a physical device. + +Information about the specific runtime environment, including the iOS version, +device model, and whether the device is a simulator, can be obtained using +:func:`platform.ios_ver`. :func:`platform.system` will report ``iOS`` or +``iPadOS``, depending on the device. + +:func:`os.uname` reports kernel-level details; it will report a name of +``Darwin``. + +Standard library availability +----------------------------- + +The Python standard library has some notable omissions and restrictions on +iOS. See the :ref:`API availability guide for iOS ` for +details. + +Binary extension modules +------------------------ + +One notable difference about iOS as a platform is that App Store distribution +imposes hard requirements on the packaging of an application. One of these +requirements governs how binary extension modules are distributed. + +The iOS App Store requires that *all* binary modules in an iOS app must be +dynamic libraries, contained in a framework with appropriate metadata, stored +in the ``Frameworks`` folder of the packaged app. There can be only a single +binary per framework, and there can be no executable binary material outside +the ``Frameworks`` folder. + +This conflicts with the usual Python approach for distributing binaries, which +allows a binary extension module to be loaded from any location on +``sys.path``. To ensure compliance with App Store policies, an iOS project must +post-process any Python packages, converting ``.so`` binary modules into +individual standalone frameworks with appropriate metadata and signing. For +details on how to perform this post-processing, see the guide for :ref:`adding +Python to your project `. + +To help Python discover binaries in their new location, the original ``.so`` +file on ``sys.path`` is replaced with a ``.fwork`` file. This file is a text +file containing the location of the framework binary, relative to the app +bundle. To allow the framework to resolve back to the original location, the +framework must contain a ``.origin`` file that contains the location of the +``.fwork`` file, relative to the app bundle. + +For example, consider the case of an import ``from foo.bar import _whiz``, +where ``_whiz`` is implemented with the binary module +``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location +registered on ``sys.path``, relative to the application bundle. This module +*must* be distributed as ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` +(creating the framework name from the full import path of the module), with an +``Info.plist`` file in the ``.framework`` directory identifying the binary as a +framework. The ``foo.bar._whiz`` module would be represented in the original +location with a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing +the path ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also +contain ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing +the path to the ``.fwork`` file. + +When running on iOS, the Python interpreter will install an +:class:`~importlib.machinery.AppleFrameworkLoader` that is able to read and +import ``.fwork`` files. Once imported, the ``__file__`` attribute of the +binary module will report as the location of the ``.fwork`` file. However, the +:class:`~importlib.machinery.ModuleSpec` for the loaded module will report the +``origin`` as the location of the binary in the framework folder. + +Compiler stub binaries +---------------------- + +Xcode doesn't expose explicit compilers for iOS; instead, it uses an ``xcrun`` +script that resolves to a full compiler path (e.g., ``xcrun --sdk iphoneos +clang`` to get the ``clang`` for an iPhone device). However, using this script +poses two problems: + +* The output of ``xcrun`` includes paths that are machine specific, resulting + in a sysconfig module that cannot be shared between users; and + +* It results in ``CC``/``CPP``/``LD``/``AR`` definitions that include spaces. + There is a lot of C ecosystem tooling that assumes that you can split a + command line at the first space to get the path to the compiler executable; + this isn't the case when using ``xcrun``. + +To avoid these problems, Python provided stubs for these tools. These stubs are +shell script wrappers around the underingly ``xcrun`` tools, distributed in a +``bin`` folder distributed alongside the compiled iOS framework. These scripts +are relocatable, and will always resolve to the appropriate local system paths. +By including these scripts in the bin folder that accompanies a framework, the +contents of the ``sysconfig`` module becomes useful for end-users to compile +their own modules. When compiling third-party Python modules for iOS, you +should ensure these stub binaries are on your path. + +Installing Python on iOS +======================== + +Tools for building iOS apps +--------------------------- + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting from the Platforms tab of the Xcode Settings panel. + +.. _adding-ios: + +Adding Python to an iOS project +------------------------------- + +Python can be added to any iOS project, using either Swift or Objective C. The +following examples will use Objective C; if you are using Swift, you may find a +library like `PythonKit `__ to be +helpful. + +To add Python to an iOS Xcode project: + +1. Build or obtain a Python ``XCFramework``. See the instructions in + :source:`iOS/README.rst` (in the CPython source distribution) for details on + how to build a Python ``XCFramework``. At a minimum, you will need a build + that supports ``arm64-apple-ios``, plus one of either + ``arm64-apple-ios-simulator`` or ``x86_64-apple-ios-simulator``. + +2. Drag the ``XCframework`` into your iOS project. In the following + instructions, we'll assume you've dropped the ``XCframework`` into the root + of your project; however, you can use any other location that you want by + adjusting paths as needed. + +3. Drag the ``iOS/Resources/dylib-Info-template.plist`` file into your project, + and ensure it is associated with the app target. + +4. Add your application code as a folder in your Xcode project. In the + following instructions, we'll assume that your user code is in a folder + named ``app`` in the root of your project; you can use any other location by + adjusting paths as needed. Ensure that this folder is associated with your + app target. + +5. Select the app target by selecting the root node of your Xcode project, then + the target name in the sidebar that appears. + +6. In the "General" settings, under "Frameworks, Libraries and Embedded + Content", add ``Python.xcframework``, with "Embed & Sign" selected. + +7. In the "Build Settings" tab, modify the following: + + - Build Options + + * User Script Sandboxing: No + * Enable Testability: Yes + + - Search Paths + + * Framework Search Paths: ``$(PROJECT_DIR)`` + * Header Search Paths: ``"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers"`` + + - Apple Clang - Warnings - All languages + + * Quoted Include In Framework Header: No + +8. Add a build step that copies the Python standard library into your app. In + the "Build Phases" tab, add a new "Run Script" build step *before* the + "Embed Frameworks" step, but *after* the "Copy Bundle Resources" step. Name + the step "Install Target Specific Python Standard Library", disable the + "Based on dependency analysis" checkbox, and set the script content to: + + .. code-block:: bash + + set -e + + mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" + if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then + echo "Installing Python modules for iOS Simulator" + rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + else + echo "Installing Python modules for iOS Device" + rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + fi + + Note that the name of the simulator "slice" in the XCframework may be + different, depending the CPU architectures your ``XCFramework`` supports. + +9. Add a second build step that processes the binary extension modules in the + standard library into "Framework" format. Add a "Run Script" build step + *directly after* the one you added in step 8, named "Prepare Python Binary + Modules". It should also have "Based on dependency analysis" unchecked, with + the following script content: + + .. code-block:: bash + + set -e + + install_dylib () { + INSTALL_BASE=$1 + FULL_EXT=$2 + + # The name of the extension file + EXT=$(basename "$FULL_EXT") + # The location of the extension file, relative to the bundle + RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} + # The path to the extension file, relative to the install base + PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} + # The full dotted name of the extension module, constructed from the file path. + FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); + # A bundle identifier; not actually used, but required by Xcode framework packaging + FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") + # The name of the framework folder. + FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" + + # If the framework folder doesn't exist, create it. + if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then + echo "Creating framework for $RELATIVE_EXT" + mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" + cp "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + fi + + echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + # Create a placeholder .fwork file where the .so was + echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork + # Create a back reference to the .so file location in the framework + echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" + } + + PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") + echo "Install Python $PYTHON_VER standard library extension modules..." + find "$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload" -name "*.so" | while read FULL_EXT; do + install_dylib python/lib/$PYTHON_VER/lib-dynload/ "$FULL_EXT" + done + + # Clean up dylib template + rm -f "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" + + echo "Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." + find "$CODESIGNING_FOLDER_PATH/Frameworks" -name "*.framework" -exec /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "{}" \; + +10. Add Objective C code to initialize and use a Python interpreter in embedded + mode. You should ensure that: + + * :c:member:`UTF-8 mode ` is *enabled*; + * :c:member:`Buffered stdio ` is *disabled*; + * :c:member:`Writing bytecode ` is *disabled*; + * :c:member:`Signal handlers ` are *enabled*; + * ``PYTHONHOME`` for the interpreter is configured to point at the + ``python`` subfolder of your app's bundle; and + * The ``PYTHONPATH`` for the interpreter includes: + + - the ``python/lib/python3.X`` subfolder of your app's bundle, + - the ``python/lib/python3.X/lib-dynload`` subfolder of your app's bundle, and + - the ``app`` subfolder of your app's bundle + + Your app's bundle location can be determined using ``[[NSBundle mainBundle] + resourcePath]``. + +Steps 8, 9 and 10 of these instructions assume that you have a single folder of +pure Python application code, named ``app``. If you have third-party binary +modules in your app, some additional steps will be required: + +* You need to ensure that any folders containing third-party binaries are + either associated with the app target, or copied in as part of step 8. Step 8 + should also purge any binaries that are not appropriate for the platform a + specific build is targeting (i.e., delete any device binaries if you're + building an app targeting the simulator). + +* Any folders that contain third-party binaries must be processed into + framework form by step 9. The invocation of ``install_dylib`` that processes + the ``lib-dynload`` folder can be copied and adapted for this purpose. + +* If you're using a separate folder for third-party packages, ensure that folder + is included as part of the ``PYTHONPATH`` configuration in step 10. + +App Store Compliance +==================== + +The only mechanism for distributing apps to third-party iOS devices is to +submit the app to the iOS App Store; apps submitted for distribution must pass +Apple's app review process. This process includes a set of automated validation +rules that inspect the submitted application bundle for problematic code. + +The Python standard library contains some code that is known to violate these +automated rules. While these violations appear to be false positives, Apple's +review rules cannot be challenged; so, it is necessary to modify the Python +standard library for an app to pass App Store review. + +The Python source tree contains +:source:`a patch file ` that will remove +all code that is known to cause issues with the App Store review process. This +patch is applied automatically when building for iOS. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index eb1413af2cbc3d..4b6c884f3d4f25 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -2,173 +2,473 @@ .. _using-on-mac: ********************* -Using Python on a Mac +Using Python on macOS ********************* -:Author: Bob Savage +.. sectionauthor:: Bob Savage +.. sectionauthor:: Ned Deily +This document aims to give an overview of macOS-specific behavior you should +know about to get started with Python on Mac computers. +Python on a Mac running macOS is very similar to Python on other Unix-derived platforms, +but there are some differences in installation and some features. -Python on a Mac running macOS is in principle very similar to Python on -any other Unix platform, but there are a number of additional features such as -the IDE and the Package Manager that are worth pointing out. +There are various ways to obtain and install Python for macOS. +Pre-built versions of the most recent versions of Python are available +from a number of distributors. Much of this document describes use of +the Pythons provided by the CPython release team for download from +the `python.org website `_. See +:ref:`alternative_bundles` for some other options. + +.. |usemac_x_dot_y| replace:: 3.13 +.. |usemac_python_x_dot_y_literal| replace:: ``python3.13`` +.. |usemac_python_x_dot_y_t_literal| replace:: ``python3.13t`` +.. |usemac_python_x_dot_y_t_literal_config| replace:: ``python3.13t-config`` +.. |usemac_applications_folder_name| replace:: ``Python 3.13`` +.. |usemac_applications_folder_version| replace:: ``/Applications/Python 3.13/`` .. _getting-osx: +.. _getting-and-installing-macpython: -Getting and Installing MacPython -================================ +Using Python for macOS from ``python.org`` +========================================== -macOS used to come with Python 2.7 pre-installed between versions -10.8 and `12.3 `_. -You are invited to install the most recent version of Python 3 from the Python -website (https://www.python.org). A current "universal binary" build of Python, -which runs natively on the Mac's new Intel and legacy PPC CPU's, is available -there. +Installation steps +------------------ -What you get after installing is a number of things: +For `current Python versions `_ +(other than those in ``security`` status), the release team produces a +**Python for macOS** installer package for each new release. +A list of available installers +is available `here `_. +We recommend using the most recent supported Python version where possible. +Current installers provide a +`universal2 binary `_ build +of Python which runs natively on all Macs (Apple Silicon and Intel) that are +supported by a wide range of macOS versions, +currently typically from at least **macOS 10.13 High Sierra** on. -* A :file:`Python 3.12` folder in your :file:`Applications` folder. In here - you find IDLE, the development environment that is a standard part of official - Python distributions; and PythonLauncher, which handles double-clicking Python - scripts from the Finder. +The downloaded file is a standard macOS installer package file (``.pkg``). +File integrity information (checksum, size, sigstore signature, etc) for each file is included +on the release download page. Installer packages and their contents are signed and notarized +with ``Python Software Foundation`` Apple Developer ID certificates +to meet `macOS Gatekeeper requirements `_. -* A framework :file:`/Library/Frameworks/Python.framework`, which includes the - Python executable and libraries. The installer adds this location to your shell - path. To uninstall MacPython, you can simply remove these three things. A - symlink to the Python executable is placed in /usr/local/bin/. - -The Apple-provided build of Python is installed in -:file:`/System/Library/Frameworks/Python.framework` and :file:`/usr/bin/python`, -respectively. You should never modify or delete these, as they are -Apple-controlled and are used by Apple- or third-party software. Remember that -if you choose to install a newer Python version from python.org, you will have -two different but functional Python installations on your computer, so it will -be important that your paths and usages are consistent with what you want to do. - -IDLE includes a help menu that allows you to access Python documentation. If you -are completely new to Python you should start reading the tutorial introduction -in that document. +For a default installation, double-click on the downloaded installer package file. +This should launch the standard macOS Installer app and display the first of several +installer windows steps. -If you are familiar with Python on other Unix platforms you should read the -section on running Python scripts from the Unix shell. +.. image:: mac_installer_01_introduction.png +Clicking on the **Continue** button brings up the **Read Me** for this installer. +Besides other important information, the **Read Me** documents which Python version is +going to be installed and on what versions of macOS it is supported. You may need +to scroll through to read the whole file. By default, this **Read Me** will also be +installed in |usemac_applications_folder_version| and available to read anytime. -How to run a Python script --------------------------- +.. image:: mac_installer_02_readme.png -Your best way to get started with Python on macOS is through the IDLE -integrated development environment, see section :ref:`ide` and use the Help menu -when the IDE is running. +Clicking on **Continue** proceeds to display the license for Python and for +other included software. You will then need to **Agree** to the license terms +before proceeding to the next step. This license file will also be installed +and available to be read later. -If you want to run Python scripts from the Terminal window command line or from -the Finder you first need an editor to create your script. macOS comes with a -number of standard Unix command line editors, :program:`vim` and -:program:`emacs` among them. If you want a more Mac-like editor, -:program:`BBEdit` or :program:`TextWrangler` from Bare Bones Software (see -http://www.barebones.com/products/bbedit/index.html) are good choices, as is -:program:`TextMate` (see https://macromates.com/). Other editors include -:program:`Gvim` (https://macvim.org/macvim/) and :program:`Aquamacs` -(http://aquamacs.org/). +.. image:: mac_installer_03_license.png -To run your script from the Terminal window you must make sure that -:file:`/usr/local/bin` is in your shell search path. +After the license terms are accepted, the next step is the **Installation Type** +display. For most uses, the standard set of installation operations is appropriate. -To run your script from the Finder you have two options: +.. image:: mac_installer_04_installation_type.png -* Drag it to :program:`PythonLauncher` +By pressing the **Customize** button, you can choose to omit or select certain package +components of the installer. Click on each package name to see a description of +what it installs. +To also install support for the optional experimental free-threaded feature, +see :ref:`install-freethreaded-macos`. -* Select :program:`PythonLauncher` as the default application to open your - script (or any .py script) through the finder Info window and double-click it. - :program:`PythonLauncher` has various preferences to control how your script is - launched. Option-dragging allows you to change these for one invocation, or use - its Preferences menu to change things globally. +.. image:: mac_installer_05_custom_install.png +In either case, clicking **Install** will begin the install process by asking +permission to install new software. A macOS user name with ``Administrator`` privilege +is needed as the installed Python will be available to all users of the Mac. -.. _osx-gui-scripts: +When the installation is complete, the **Summary** window will appear. + +.. image:: mac_installer_06_summary.png + +Double-click on the :command:`Install Certificates.command` +icon or file in the |usemac_applications_folder_version| window to complete the +installation. + +.. image:: mac_installer_07_applications.png + +This will open a temporary :program:`Terminal` shell window that +will use the new Python to download and install SSL root certificates +for its use. + +.. image:: mac_installer_08_install_certificates.png + +If ``Successfully installed certifi`` and ``update complete`` appears +in the terminal window, the installation is complete. +Close this terminal window and the installer window. + +A default install will include: -Running scripts with a GUI +* A |usemac_applications_folder_name| folder in your :file:`Applications` folder. In here + you find :program:`IDLE`, the development environment that is a standard part of official + Python distributions; and :program:`Python Launcher`, which handles double-clicking Python + scripts from the macOS `Finder `_. + +* A framework :file:`/Library/Frameworks/Python.framework`, which includes the + Python executable and libraries. The installer adds this location to your shell + path. To uninstall Python, you can remove these three things. + Symlinks to the Python executable are placed in :file:`/usr/local/bin/`. + +.. note:: + + Recent versions of macOS include a :command:`python3` command in :file:`/usr/bin/python3` + that links to a usually older and incomplete version of Python provided by and for use by + the Apple development tools, :program:`Xcode` or the :program:`Command Line Tools for Xcode`. + You should never modify or attempt to delete this installation, as it is + Apple-controlled and is used by Apple-provided or third-party software. If + you choose to install a newer Python version from ``python.org``, you will have + two different but functional Python installations on your computer that + can co-exist. The default installer options should ensure that its :command:`python3` + will be used instead of the system :command:`python3`. + +How to run a Python script -------------------------- -With older versions of Python, there is one macOS quirk that you need to be -aware of: programs that talk to the Aqua window manager (in other words, -anything that has a GUI) need to be run in a special way. Use :program:`pythonw` -instead of :program:`python` to start such scripts. +There are two ways to invoke the Python interpreter. +If you are familiar with using a Unix shell in a terminal +window, you can invoke |usemac_python_x_dot_y_literal| or ``python3`` optionally +followed by one or more command line options (described in :ref:`using-on-general`). +The Python tutorial also has a useful section on +:ref:`using Python interactively from a shell `. + +You can also invoke the interpreter through an integrated +development environment. +:ref:`idle` is a basic editor and interpreter environment +which is included with the standard distribution of Python. +:program:`IDLE` includes a Help menu that allows you to access Python documentation. If you +are completely new to Python, you can read the tutorial introduction +in that document. + +There are many other editors and IDEs available, see :ref:`editors` +for more information. + +To run a Python script file from the terminal window, you can +invoke the interpreter with the name of the script file: + + |usemac_python_x_dot_y_literal| ``myscript.py`` -With Python 3.9, you can use either :program:`python` or :program:`pythonw`. +To run your script from the Finder, you can either: + +* Drag it to :program:`Python Launcher`. + +* Select :program:`Python Launcher` as the default application to open your + script (or any ``.py`` script) through the Finder Info window and double-click it. + :program:`Python Launcher` has various preferences to control how your script is + launched. Option-dragging allows you to change these for one invocation, or use + its ``Preferences`` menu to change things globally. +Be aware that running the script directly from the macOS Finder might +produce different results than when running from a terminal window as +the script will not be run in the usual shell environment including +any setting of environment variables in shell profiles. +And, as with any other script or program, +be certain of what you are about to run. -Configuration -------------- +.. _alternative_bundles: -Python on macOS honors all standard Unix environment variables such as -:envvar:`PYTHONPATH`, but setting these variables for programs started from the -Finder is non-standard as the Finder does not read your :file:`.profile` or -:file:`.cshrc` at startup. You need to create a file -:file:`~/.MacOSX/environment.plist`. See Apple's Technical Document QA1067 for -details. +Alternative Distributions +========================= -For more information on installation Python packages in MacPython, see section -:ref:`mac-package-manager`. +Besides the standard ``python.org`` for macOS installer, there are third-party +distributions for macOS that may include additional functionality. +Some popular distributions and their key features: +`ActivePython `_ + Installer with multi-platform compatibility, documentation -.. _ide: +`Anaconda `_ + Popular scientific modules (such as numpy, scipy, and pandas) and the + ``conda`` package manager. -The IDE -======= +`Homebrew `_ + Package manager for macOS including multiple versions of Python and many + third-party Python-based packages (including numpy, scipy, and pandas). -MacPython ships with the standard IDLE development environment. A good -introduction to using IDLE can be found at -http://www.hashcollision.org/hkn/python/idle_intro/index.html. +`MacPorts `_ + Another package manager for macOS including multiple versions of Python and many + third-party Python-based packages. May include pre-built versions of Python and + many packages for older versions of macOS. +Note that distributions might not include the latest versions of Python or +other libraries, and are not maintained or supported by the core Python team. .. _mac-package-manager: Installing Additional Python Packages ===================================== -This section has moved to the `Python Packaging User Guide`_. +Refer to the `Python Packaging User Guide`_ for more information. .. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ -GUI Programming on the Mac -========================== +.. _osx-gui-scripts: -There are several options for building GUI applications on the Mac with Python. +.. _gui-programming-on-the-mac: + +GUI Programming +=============== -*PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework, which is -the foundation of most modern Mac development. Information on PyObjC is -available from https://pypi.org/project/pyobjc/. +There are several options for building GUI applications on the Mac with Python. The standard Python GUI toolkit is :mod:`tkinter`, based on the cross-platform -Tk toolkit (https://www.tcl.tk). An Aqua-native version of Tk is bundled with -macOS by Apple, and the latest version can be downloaded and installed from -https://www.activestate.com; it can also be built from source. +Tk toolkit (https://www.tcl.tk). A macOS-native version of Tk is included with +the installer. -*wxPython* is another popular cross-platform GUI toolkit that runs natively on -macOS. Packages and documentation are available from https://www.wxpython.org. +*PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework. +Information on PyObjC is available from :pypi:`pyobjc`. -*PyQt* is another popular cross-platform GUI toolkit that runs natively on -macOS. More information can be found at -https://riverbankcomputing.com/software/pyqt/intro. +A number of alternative macOS GUI toolkits are available including: +* `PySide `_: Official Python bindings to the + `Qt GUI toolkit `_. -Distributing Python Applications on the Mac -=========================================== +* `PyQt `_: Alternative + Python bindings to Qt. -The standard tool for deploying standalone Python applications on the Mac is -:program:`py2app`. More information on installing and using py2app can be found -at https://pypi.org/project/py2app/. +* `Kivy `_: A cross-platform GUI toolkit that supports + desktop and mobile platforms. +* `Toga `_: Part of the `BeeWare Project + `_; supports desktop, mobile, web and console apps. -Other Resources +* `wxPython `_: A cross-platform toolkit that + supports desktop operating systems. + + +Advanced Topics =============== -The MacPython mailing list is an excellent support resource for Python users and -developers on the Mac: +.. _install-freethreaded-macos: + +Installing Free-threaded Binaries +--------------------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + + Everything described in this section is considered experimental, + and should be expected to change in future releases. + +The ``python.org`` :ref:`Python for macOS ` +installer package can optionally install an additional build of +Python |usemac_x_dot_y| that supports :pep:`703`, the experimental free-threading feature +(running with the :term:`global interpreter lock` disabled). +Check the release page on ``python.org`` for possible updated information. + +Because this feature is still considered experimental, the support for it +is not installed by default. It is packaged as a separate install option, +available by clicking the **Customize** button on the **Installation Type** +step of the installer as described above. + +.. image:: mac_installer_09_custom_install_free_threaded.png -https://www.python.org/community/sigs/current/pythonmac-sig/ +If the box next to the **Free-threaded Python** package name is checked, +a separate :file:`PythonT.framework` will also be installed +alongside the normal :file:`Python.framework` in :file:`/Library/Frameworks`. +This configuration allows a free-threaded Python |usemac_x_dot_y| build to co-exist +on your system with a traditional (GIL only) Python |usemac_x_dot_y| build with +minimal risk while installing or testing. This installation layout is itself +experimental and is subject to change in future releases. -Another useful resource is the MacPython wiki: +Known cautions and limitations: -https://wiki.python.org/moin/MacPython +- The **UNIX command-line tools** package, which is selected by default, + will install links in :file:`/usr/local/bin` for |usemac_python_x_dot_y_t_literal|, + the free-threaded interpreter, and |usemac_python_x_dot_y_t_literal_config|, + a configuration utility which may be useful for package builders. + Since :file:`/usr/local/bin` is typically included in your shell ``PATH``, + in most cases no changes to your ``PATH`` environment variables should + be needed to use |usemac_python_x_dot_y_t_literal|. + +- For this release, the **Shell profile updater** package and the + :file:`Update Shell Profile.command` in |usemac_applications_folder_version| + do not support the free-threaded package. + +- The free-threaded build and the traditional build have separate search + paths and separate :file:`site-packages` directories so, by default, + if you need a package available in both builds, it may need to be installed in both. + The free-threaded package will install a separate instance of :program:`pip` for use + with |usemac_python_x_dot_y_t_literal|. + + - To install a package using :command:`pip` without a :command:`venv`: + + |usemac_python_x_dot_y_t_literal| ``-m pip install `` + +- When working with multiple Python environments, it is usually safest and easiest + to :ref:`create and use virtual environments `. + This can avoid possible command name conflicts and confusion about which Python is in use: + + |usemac_python_x_dot_y_t_literal| ``-m venv `` + + then :command:`activate`. + +- To run a free-threaded version of IDLE: + + |usemac_python_x_dot_y_t_literal| ``-m idlelib`` + +- The interpreters in both builds respond to the same + :ref:`PYTHON environment variables ` + which may have unexpected results, for example, if you have ``PYTHONPATH`` + set in a shell profile. If necessary, there are + :ref:`command line options ` like ``-E`` + to ignore these environment variables. + +- The free-threaded build links to the third-party shared libraries, + such as ``OpenSSL`` and ``Tk``, installed in the traditional framework. + This means that both builds also share one set of trust certificates + as installed by the :command:`Install Certificates.command` script, + thus it only needs to be run once. + +- If you cannot depend on the link in ``/usr/local/bin`` pointing to the + ``python.org`` free-threaded |usemac_python_x_dot_y_t_literal| (for example, if you want + to install your own version there or some other distribution does), + you can explicitly set your shell ``PATH`` environment variable to + include the ``PythonT`` framework ``bin`` directory: + + .. code-block:: sh + + export PATH="/Library/Frameworks/PythonT.framework/Versions/3.13/bin":"$PATH" + + The traditional framework installation by default does something similar, + except for :file:`Python.framework`. Be aware that having both framework ``bin`` + directories in ``PATH`` can lead to confusion if there are duplicate names + like ``python3.13`` in both; which one is actually used depends on the order + they appear in ``PATH``. The ``which python3.x`` or ``which python3.xt`` + commands can show which path is being used. Using virtual environments + can help avoid such ambiguities. Another option might be to create + a shell :command:`alias` to the desired interpreter, like: + + .. code-block:: sh + + alias py3.13="/Library/Frameworks/Python.framework/Versions/3.13/bin/python3.13" + alias py3.13t="/Library/Frameworks/PythonT.framework/Versions/3.13/bin/python3.13t" + +Installing using the command line +--------------------------------- + +If you want to use automation to install the ``python.org`` installer package +(rather than by using the familiar macOS :program:`Installer` GUI app), +the macOS command line :command:`installer` utility lets you select non-default +options, too. If you are not familiar with :command:`installer`, it can be +somewhat cryptic (see :command:`man installer` for more information). +As an example, the following shell snippet shows one way to do it, +using the ``3.13.0b2`` release and selecting the free-threaded interpreter +option: + +.. code-block:: sh + + RELEASE="python-3.13.0b2-macos11.pkg" + + # download installer pkg + curl -O https://www.python.org/ftp/python/3.13.0/${RELEASE} + + # create installer choicechanges to customize the install: + # enable the PythonTFramework-3.13 package + # while accepting the other defaults (install all other packages) + cat > ./choicechanges.plist < + + + + + attributeSetting + 1 + choiceAttribute + selected + choiceIdentifier + org.python.Python.PythonTFramework-3.13 + + + + EOF + + sudo installer -pkg ./${RELEASE} -applyChoiceChangesXML ./choicechanges.plist -target / + + +You can then test that both installer builds are now available with something like: + +.. code-block:: console + + $ # test that the free-threaded interpreter was installed if the Unix Command Tools package was enabled + $ /usr/local/bin/python3.13t -VV + Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ # and the traditional interpreter + $ /usr/local/bin/python3.13 -VV + Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + $ # test that they are also available without the prefix if /usr/local/bin is on $PATH + $ python3.13t -VV + Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ python3.13 -VV + Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + +.. note:: + + Current ``python.org`` installers only install to fixed locations like + :file:`/Library/Frameworks/`, :file:`/Applications`, and :file:`/usr/local/bin`. + You cannot use the :command:`installer` ``-domain`` option to install to + other locations. + +.. _distributing-python-applications-on-the-mac: + +Distributing Python Applications +-------------------------------- + +A range of tools exist for converting your Python code into a standalone +distributable application: + +* :pypi:`py2app`: Supports creating macOS ``.app`` + bundles from a Python project. + +* `Briefcase `_: Part of the `BeeWare Project + `_; a cross-platform packaging tool that supports + creation of ``.app`` bundles on macOS, as well as managing signing and + notarization. + +* `PyInstaller `_: A cross-platform packaging tool that creates + a single file or folder as a distributable artifact. + +App Store Compliance +-------------------- + +Apps submitted for distribution through the macOS App Store must pass Apple's +app review process. This process includes a set of automated validation rules +that inspect the submitted application bundle for problematic code. + +The Python standard library contains some code that is known to violate these +automated rules. While these violations appear to be false positives, Apple's +review rules cannot be challenged. Therefore, it is necessary to modify the +Python standard library for an app to pass App Store review. + +The Python source tree contains +:source:`a patch file ` that will remove +all code that is known to cause issues with the App Store review process. This +patch is applied automatically when CPython is configured with the +:option:`--with-app-store-compliance` option. + +This patch is not normally required to use CPython on a Mac; nor is it required +if you are distributing an app *outside* the macOS App Store. It is *only* +required if you are using the macOS App Store as a distribution channel. + +Other Resources +=============== +The `python.org Help page `_ has links to many useful resources. +The `Pythonmac-SIG mailing list `_ +is another support resource specifically for Python users and developers on the Mac. diff --git a/Doc/using/mac_installer_01_introduction.png b/Doc/using/mac_installer_01_introduction.png new file mode 100644 index 00000000000000..1999f3a3759093 Binary files /dev/null and b/Doc/using/mac_installer_01_introduction.png differ diff --git a/Doc/using/mac_installer_02_readme.png b/Doc/using/mac_installer_02_readme.png new file mode 100644 index 00000000000000..a36efaf7d50fd6 Binary files /dev/null and b/Doc/using/mac_installer_02_readme.png differ diff --git a/Doc/using/mac_installer_03_license.png b/Doc/using/mac_installer_03_license.png new file mode 100644 index 00000000000000..598c22a13d9e62 Binary files /dev/null and b/Doc/using/mac_installer_03_license.png differ diff --git a/Doc/using/mac_installer_04_installation_type.png b/Doc/using/mac_installer_04_installation_type.png new file mode 100644 index 00000000000000..9498fd06240a4e Binary files /dev/null and b/Doc/using/mac_installer_04_installation_type.png differ diff --git a/Doc/using/mac_installer_05_custom_install.png b/Doc/using/mac_installer_05_custom_install.png new file mode 100644 index 00000000000000..3a201d2f44655a Binary files /dev/null and b/Doc/using/mac_installer_05_custom_install.png differ diff --git a/Doc/using/mac_installer_06_summary.png b/Doc/using/mac_installer_06_summary.png new file mode 100644 index 00000000000000..1af6eee2c668cd Binary files /dev/null and b/Doc/using/mac_installer_06_summary.png differ diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png new file mode 100644 index 00000000000000..940219cad6f61c Binary files /dev/null and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/mac_installer_08_install_certificates.png b/Doc/using/mac_installer_08_install_certificates.png new file mode 100644 index 00000000000000..c125eeb18aa0c1 Binary files /dev/null and b/Doc/using/mac_installer_08_install_certificates.png differ diff --git a/Doc/using/mac_installer_09_custom_install_free_threaded.png b/Doc/using/mac_installer_09_custom_install_free_threaded.png new file mode 100644 index 00000000000000..0f69c55eddb228 Binary files /dev/null and b/Doc/using/mac_installer_09_custom_install_free_threaded.png differ diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 58838c28e6eb86..a2bcdab09a9282 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -17,12 +17,12 @@ On Linux Python comes preinstalled on most Linux distributions, and is available as a package on all others. However there are certain features you might want to use -that are not available on your distro's package. You can easily compile the +that are not available on your distro's package. You can compile the latest version of Python from source. -In the event that Python doesn't come preinstalled and isn't in the repositories as -well, you can easily make packages for your own distro. Have a look at the -following links: +In the event that the latest version of Python doesn't come preinstalled and isn't +in the repositories as well, you can make packages for your own distro. Have a +look at the following links: .. seealso:: diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc deleted file mode 100644 index 1cf438b198a9af..00000000000000 --- a/Doc/using/venv-create.inc +++ /dev/null @@ -1,121 +0,0 @@ -Creation of :ref:`virtual environments ` is done by executing the -command ``venv``:: - - python -m venv /path/to/new/virtual/environment - -Running this command creates the target directory (creating any parent -directories that don't exist already) and places a ``pyvenv.cfg`` file in it -with a ``home`` key pointing to the Python installation from which the command -was run (a common name for the target directory is ``.venv``). It also creates -a ``bin`` (or ``Scripts`` on Windows) subdirectory containing a copy/symlink -of the Python binary/binaries (as appropriate for the platform or arguments -used at environment creation time). It also creates an (initially empty) -``lib/pythonX.Y/site-packages`` subdirectory (on Windows, this is -``Lib\site-packages``). If an existing directory is specified, it will be -re-used. - -.. deprecated:: 3.6 - ``pyvenv`` was the recommended tool for creating virtual environments for - Python 3.3 and 3.4, and is - :ref:`deprecated in Python 3.6 `. - -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - -.. highlight:: none - -On Windows, invoke the ``venv`` command as follows:: - - c:\>Python35\python -m venv c:\path\to\myenv - -Alternatively, if you configured the ``PATH`` and ``PATHEXT`` variables for -your :ref:`Python installation `:: - - c:\>python -m venv c:\path\to\myenv - -The command, if run with ``-h``, will show the available options:: - - usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] - [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] - [--without-scm-ignore-file] - ENV_DIR [ENV_DIR ...] - - Creates virtual Python environments in one or more target directories. - - positional arguments: - ENV_DIR A directory to create the environment in. - - options: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when - symlinks are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory if - it already exists, before environment creation. - --upgrade Upgrade the environment directory to use this - version of Python, assuming Python has been upgraded - in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the latest - version in PyPI - --without-scm-ignore-file - Skips adding the default SCM ignore file to the - environment directory (the default is a .gitignore - file). - - Once an environment has been created, you may wish to activate it, e.g. by - sourcing an activate script in its bin directory. - -.. versionchanged:: 3.13 - - ``--without-scm-ignore-file`` was added along with creating an ignore file - for ``git`` by default. - -.. versionchanged:: 3.12 - - ``setuptools`` is no longer a core venv dependency. - -.. versionchanged:: 3.9 - Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI - -.. versionchanged:: 3.4 - Installs pip by default, added the ``--without-pip`` and ``--copies`` - options - -.. versionchanged:: 3.4 - In earlier versions, if the target directory already existed, an error was - raised, unless the ``--clear`` or ``--upgrade`` option was provided. - -.. note:: - While symlinks are supported on Windows, they are not recommended. Of - particular note is that double-clicking ``python.exe`` in File Explorer - will resolve the symlink eagerly and ignore the virtual environment. - -.. note:: - On Microsoft Windows, it may be required to enable the ``Activate.ps1`` - script by setting the execution policy for the user. You can do this by - issuing the following PowerShell command: - - PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - - See `About Execution Policies - `_ - for more information. - -The created ``pyvenv.cfg`` file also includes the -``include-system-site-packages`` key, set to ``true`` if ``venv`` is -run with the ``--system-site-packages`` option, ``false`` otherwise. - -Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be -invoked to bootstrap ``pip`` into the virtual environment. - -Multiple paths can be given to ``venv``, in which case an identical virtual -environment will be created, according to the given options, at each provided -path. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 2a0e7b4b06f586..daaf8822af1161 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -14,8 +14,8 @@ know about when using Python on Microsoft Windows. Unlike most Unix systems and services, Windows does not include a system supported installation of Python. To make Python available, the CPython team -has compiled Windows installers (MSI packages) with every `release -`_ for many years. These installers +has compiled Windows installers with every `release +`_ for many years. These installers are primarily intended to add a per-user installation of Python, with the core interpreter and library being used by a single user. The installer is also able to install for all users of a single machine, and a separate ZIP file is @@ -23,8 +23,9 @@ available for application-local distributions. As specified in :pep:`11`, a Python release only supports a Windows platform while Microsoft considers the platform under extended support. This means that -Python |version| supports Windows 8.1 and newer. If you require Windows 7 -support, please install Python 3.8. +Python |version| supports Windows 10 and newer. If you require Windows 7 +support, please install Python 3.8. If you require Windows 8.1 support, +please install Python 3.12. There are a number of different installers available for Windows, each with certain benefits and downsides. @@ -608,7 +609,7 @@ key features: Popular scientific modules (such as numpy, scipy and pandas) and the ``conda`` package manager. -`Enthought Deployment Manager `_ +`Enthought Deployment Manager `_ "The Next Generation Python Environment and Package Manager". Previously Enthought provided Canopy, but it `reached end of life in 2016 @@ -837,8 +838,8 @@ The short form of the argument (``-3``) only ever selects from core Python releases, and not other distributions. However, the longer form (``-V:3``) will select from any. -The Company is matched on the full string, case-insenitive. The Tag is matched -oneither the full string, or a prefix, provided the next character is a dot or a +The Company is matched on the full string, case-insensitive. The Tag is matched +on either the full string, or a prefix, provided the next character is a dot or a hyphen. This allows ``-V:3.1`` to match ``3.1-32``, but not ``3.10``. Tags are sorted using numerical ordering (``3.10`` is newer than ``3.1``), but are compared using text (``-V:3.01`` does not match ``3.1``). @@ -1285,7 +1286,7 @@ The Windows-specific standard modules are documented in PyWin32 ------- -The `PyWin32 `_ module by Mark Hammond +The :pypi:`PyWin32` module by Mark Hammond is a collection of modules for advanced Windows-specific support. This includes utilities for: @@ -1305,7 +1306,7 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. .. seealso:: - `Win32 How Do I...? `_ + `Win32 How Do I...? `_ by Tim Golden `Python and COM `_ diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index af8171487fbcfa..1a949ec4035807 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -217,13 +217,13 @@ often use the ``codecs.lookup(encoding)`` function, which returns a was consumed. * *stream_reader* is a class that supports decoding input from a stream. - *stream_reader(file_obj)* returns an object that supports the :meth:`read`, - :meth:`readline`, and :meth:`readlines` methods. These methods will all + *stream_reader(file_obj)* returns an object that supports the :meth:`!read`, + :meth:`!readline`, and :meth:`!readlines` methods. These methods will all translate from the given encoding and return Unicode strings. * *stream_writer*, similarly, is a class that supports encoding output to a stream. *stream_writer(file_obj)* returns an object that supports the - :meth:`write` and :meth:`writelines` methods. These methods expect Unicode + :meth:`!write` and :meth:`!writelines` methods. These methods expect Unicode strings, translating them to the given encoding on output. For example, the following code writes a Unicode string into a file, encoding @@ -356,8 +356,8 @@ variable ``a`` by 2, equivalent to the slightly lengthier ``a = a + 2``. The full list of supported assignment operators is ``+=``, ``-=``, ``*=``, ``/=``, ``%=``, ``**=``, ``&=``, ``|=``, ``^=``, ``>>=``, and ``<<=``. Python classes can override the augmented assignment operators by defining methods -named :meth:`__iadd__`, :meth:`__isub__`, etc. For example, the following -:class:`Number` class stores a number and supports using += to create a new +named :meth:`!__iadd__`, :meth:`!__isub__`, etc. For example, the following +:class:`!Number` class stores a number and supports using += to create a new instance with an incremented value. .. The empty groups below prevent conversion to guillemets. @@ -374,7 +374,7 @@ instance with an incremented value. n += 3 print n.value -The :meth:`__iadd__` special method is called with the value of the increment, +The :meth:`!__iadd__` special method is called with the value of the increment, and should return a new instance with an appropriately modified value; this return value is bound as the new value of the variable on the left-hand side. @@ -390,10 +390,10 @@ String Methods ============== Until now string-manipulation functionality was in the :mod:`string` module, -which was usually a front-end for the :mod:`strop` module written in C. The -addition of Unicode posed a difficulty for the :mod:`strop` module, because the +which was usually a front-end for the :mod:`!strop` module written in C. The +addition of Unicode posed a difficulty for the :mod:`!strop` module, because the functions would all need to be rewritten in order to accept either 8-bit or -Unicode strings. For functions such as :func:`string.replace`, which takes 3 +Unicode strings. For functions such as :func:`!string.replace`, which takes 3 string arguments, that means eight possible permutations, and correspondingly complicated code. @@ -416,13 +416,13 @@ The old :mod:`string` module is still around for backwards compatibility, but it mostly acts as a front-end to the new string methods. Two methods which have no parallel in pre-2.0 versions, although they did exist -in JPython for quite some time, are :meth:`startswith` and :meth:`endswith`. +in JPython for quite some time, are :meth:`!startswith` and :meth:`!endswith`. ``s.startswith(t)`` is equivalent to ``s[:len(t)] == t``, while ``s.endswith(t)`` is equivalent to ``s[-len(t):] == t``. -One other method which deserves special mention is :meth:`join`. The -:meth:`join` method of a string receives one parameter, a sequence of strings, -and is equivalent to the :func:`string.join` function from the old :mod:`string` +One other method which deserves special mention is :meth:`!join`. The +:meth:`!join` method of a string receives one parameter, a sequence of strings, +and is equivalent to the :func:`!string.join` function from the old :mod:`string` module, with the arguments reversed. In other words, ``s.join(seq)`` is equivalent to the old ``string.join(seq, s)``. @@ -503,9 +503,9 @@ Minor Language Changes A new syntax makes it more convenient to call a given function with a tuple of arguments and/or a dictionary of keyword arguments. In Python 1.5 and earlier, -you'd use the :func:`apply` built-in function: ``apply(f, args, kw)`` calls the -function :func:`f` with the argument tuple *args* and the keyword arguments in -the dictionary *kw*. :func:`apply` is the same in 2.0, but thanks to a patch +you'd use the :func:`!apply` built-in function: ``apply(f, args, kw)`` calls the +function :func:`!f` with the argument tuple *args* and the keyword arguments in +the dictionary *kw*. :func:`!apply` is the same in 2.0, but thanks to a patch from Greg Ewing, ``f(*args, **kw)`` is a shorter and clearer way to achieve the same effect. This syntax is symmetrical with the syntax for defining functions:: @@ -518,7 +518,7 @@ functions:: The ``print`` statement can now have its output directed to a file-like object by following the ``print`` with ``>> file``, similar to the redirection operator in Unix shells. Previously you'd either have to use the -:meth:`write` method of the file-like object, which lacks the convenience and +:meth:`!write` method of the file-like object, which lacks the convenience and simplicity of ``print``, or you could assign a new value to ``sys.stdout`` and then restore the old value. For sending output to standard error, it's much easier to write this:: @@ -540,7 +540,7 @@ Previously there was no way to implement a class that overrode Python's built-in true if *obj* is present in the sequence *seq*; Python computes this by simply trying every index of the sequence until either *obj* is found or an :exc:`IndexError` is encountered. Moshe Zadka contributed a patch which adds a -:meth:`__contains__` magic method for providing a custom implementation for +:meth:`!__contains__` magic method for providing a custom implementation for :keyword:`!in`. Additionally, new built-in objects written in C can define what :keyword:`!in` means for them via a new slot in the sequence protocol. @@ -562,7 +562,7 @@ the python-dev mailing list for the discussion leading up to this implementation, and some useful relevant links. Note that comparisons can now also raise exceptions. In earlier versions of Python, a comparison operation such as ``cmp(a,b)`` would always produce an answer, even if a user-defined -:meth:`__cmp__` method encountered an error, since the resulting exception would +:meth:`!__cmp__` method encountered an error, since the resulting exception would simply be silently swallowed. .. Starting URL: @@ -607,7 +607,7 @@ seq1, seq2)`` is that :func:`map` pads the sequences with ``None`` if the sequences aren't all of the same length, while :func:`zip` truncates the returned list to the length of the shortest argument sequence. -The :func:`int` and :func:`long` functions now accept an optional "base" +The :func:`int` and :func:`!long` functions now accept an optional "base" parameter when the first argument is a string. ``int('123', 10)`` returns 123, while ``int('123', 16)`` returns 291. ``int(123, 16)`` raises a :exc:`TypeError` exception with the message "can't convert non-string with @@ -620,8 +620,8 @@ would be ``(2, 0, 1, 'beta', 1)``. *level* is a string such as ``"alpha"``, ``"beta"``, or ``"final"`` for a final release. Dictionaries have an odd new method, ``setdefault(key, default)``, which -behaves similarly to the existing :meth:`get` method. However, if the key is -missing, :meth:`setdefault` both returns the value of *default* as :meth:`get` +behaves similarly to the existing :meth:`!get` method. However, if the key is +missing, :meth:`!setdefault` both returns the value of *default* as :meth:`!get` would do, and also inserts it into the dictionary as the value for *key*. Thus, the following lines of code:: @@ -656,7 +656,7 @@ break. The change which will probably break the most code is tightening up the arguments accepted by some methods. Some methods would take multiple arguments and treat them as a tuple, particularly various list methods such as -:meth:`append` and :meth:`insert`. In earlier versions of Python, if ``L`` is +:meth:`!append` and :meth:`!insert`. In earlier versions of Python, if ``L`` is a list, ``L.append( 1,2 )`` appends the tuple ``(1,2)`` to the list. In Python 2.0 this causes a :exc:`TypeError` exception to be raised, with the message: 'append requires exactly 1 argument; 2 given'. The fix is to simply add an @@ -693,7 +693,7 @@ advantage of this fact will break in 2.0. Some work has been done to make integers and long integers a bit more interchangeable. In 1.5.2, large-file support was added for Solaris, to allow -reading files larger than 2 GiB; this made the :meth:`tell` method of file +reading files larger than 2 GiB; this made the :meth:`!tell` method of file objects return a long integer instead of a regular integer. Some code would subtract two file offsets and attempt to use the result to multiply a sequence or slice a string, but this raised a :exc:`TypeError`. In 2.0, long integers @@ -701,7 +701,7 @@ can be used to multiply or slice a sequence, and it'll behave as you'd intuitively expect it to; ``3L * 'abc'`` produces 'abcabcabc', and ``(0,1,2,3)[2L:4L]`` produces (2,3). Long integers can also be used in various contexts where previously only integers were accepted, such as in the -:meth:`seek` method of file objects, and in the formats supported by the ``%`` +:meth:`!seek` method of file objects, and in the formats supported by the ``%`` operator (``%d``, ``%i``, ``%x``, etc.). For example, ``"%d" % 2L**64`` will produce the string ``18446744073709551616``. @@ -715,7 +715,7 @@ digit. Taking the :func:`repr` of a float now uses a different formatting precision than :func:`str`. :func:`repr` uses ``%.17g`` format string for C's -:func:`sprintf`, while :func:`str` uses ``%.12g`` as before. The effect is that +:func:`!sprintf`, while :func:`str` uses ``%.12g`` as before. The effect is that :func:`repr` may occasionally show more decimal places than :func:`str`, for certain numbers. For example, the number 8.1 can't be represented exactly in binary, so ``repr(8.1)`` is ``'8.0999999999999996'``, while str(8.1) is @@ -723,7 +723,7 @@ binary, so ``repr(8.1)`` is ``'8.0999999999999996'``, while str(8.1) is The ``-X`` command-line option, which turned all standard exceptions into strings instead of classes, has been removed; the standard exceptions will now -always be classes. The :mod:`exceptions` module containing the standard +always be classes. The :mod:`!exceptions` module containing the standard exceptions was translated from Python to a built-in C module, written by Barry Warsaw and Fredrik Lundh. @@ -879,11 +879,11 @@ joins the basic set of Python documentation. XML Modules =========== -Python 1.5.2 included a simple XML parser in the form of the :mod:`xmllib` +Python 1.5.2 included a simple XML parser in the form of the :mod:`!xmllib` module, contributed by Sjoerd Mullender. Since 1.5.2's release, two different interfaces for processing XML have become common: SAX2 (version 2 of the Simple API for XML) provides an event-driven interface with some similarities to -:mod:`xmllib`, and the DOM (Document Object Model) provides a tree-based +:mod:`!xmllib`, and the DOM (Document Object Model) provides a tree-based interface, transforming an XML document into a tree of nodes that can be traversed and modified. Python 2.0 includes a SAX2 interface and a stripped-down DOM interface as part of the :mod:`xml` package. Here we will give a brief @@ -898,9 +898,9 @@ SAX2 Support SAX defines an event-driven interface for parsing XML. To use SAX, you must write a SAX handler class. Handler classes inherit from various classes provided by SAX, and override various methods that will then be called by the -XML parser. For example, the :meth:`startElement` and :meth:`endElement` +XML parser. For example, the :meth:`~xml.sax.handler.ContentHandler.startElement` and :meth:`~xml.sax.handler.ContentHandler.endElement` methods are called for every starting and end tag encountered by the parser, the -:meth:`characters` method is called for every chunk of character data, and so +:meth:`~xml.sax.handler.ContentHandler.characters` method is called for every chunk of character data, and so forth. The advantage of the event-driven approach is that the whole document doesn't @@ -940,8 +940,8 @@ DOM Support ----------- The Document Object Model is a tree-based representation for an XML document. A -top-level :class:`Document` instance is the root of the tree, and has a single -child which is the top-level :class:`Element` instance. This :class:`Element` +top-level :class:`!Document` instance is the root of the tree, and has a single +child which is the top-level :class:`!Element` instance. This :class:`!Element` has children nodes representing character data and any sub-elements, which may have further children of their own, and so forth. Using the DOM you can traverse the resulting tree any way you like, access element and attribute @@ -955,18 +955,18 @@ simply writing ````...\ ```` to a file. The DOM implementation included with Python lives in the :mod:`xml.dom.minidom` module. It's a lightweight implementation of the Level 1 DOM with support for -XML namespaces. The :func:`parse` and :func:`parseString` convenience +XML namespaces. The :func:`!parse` and :func:`!parseString` convenience functions are provided for generating a DOM tree:: from xml.dom import minidom doc = minidom.parse('hamlet.xml') -``doc`` is a :class:`Document` instance. :class:`Document`, like all the other -DOM classes such as :class:`Element` and :class:`Text`, is a subclass of the -:class:`Node` base class. All the nodes in a DOM tree therefore support certain -common methods, such as :meth:`toxml` which returns a string containing the XML +``doc`` is a :class:`!Document` instance. :class:`!Document`, like all the other +DOM classes such as :class:`!Element` and :class:`Text`, is a subclass of the +:class:`!Node` base class. All the nodes in a DOM tree therefore support certain +common methods, such as :meth:`!toxml` which returns a string containing the XML representation of the node and its children. Each class also has special -methods of its own; for example, :class:`Element` and :class:`Document` +methods of its own; for example, :class:`!Element` and :class:`!Document` instances have a method to find all child elements with a given tag name. Continuing from the previous 2-line example:: @@ -995,7 +995,7 @@ its children can be easily modified by deleting, adding, or removing nodes:: root.insertBefore( root.childNodes[0], root.childNodes[20] ) Again, I will refer you to the Python documentation for a complete listing of -the different :class:`Node` classes and their various methods. +the different :class:`!Node` classes and their various methods. Relationship to PyXML @@ -1020,7 +1020,7 @@ features in PyXML include: * The xmlproc validating parser, written by Lars Marius Garshol. -* The :mod:`sgmlop` parser accelerator module, written by Fredrik Lundh. +* The :mod:`!sgmlop` parser accelerator module, written by Fredrik Lundh. .. ====================================================================== @@ -1031,7 +1031,7 @@ Module changes Lots of improvements and bugfixes were made to Python's extensive standard library; some of the affected modules include :mod:`readline`, :mod:`ConfigParser `, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, -:mod:`xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, +:mod:`!xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, and :mod:`!nntplib`. Consult the CVS logs for the exact patch-by-patch details. Brian Gallew contributed OpenSSL support for the :mod:`socket` module. OpenSSL @@ -1044,11 +1044,12 @@ were also changed to support ``https://`` URLs, though no one has implemented FTP or SMTP over SSL. The :mod:`httplib ` module has been rewritten by Greg Stein to support HTTP/1.1. + Backward compatibility with the 1.5 version of :mod:`!httplib` is provided, though using HTTP/1.1 features such as pipelining will require rewriting code to use a different set of interfaces. -The :mod:`Tkinter` module now supports Tcl/Tk version 8.1, 8.2, or 8.3, and +The :mod:`!Tkinter` module now supports Tcl/Tk version 8.1, 8.2, or 8.3, and support for the older 7.x versions has been dropped. The Tkinter module now supports displaying Unicode strings in Tk widgets. Also, Fredrik Lundh contributed an optimization which makes operations like ``create_line`` and @@ -1083,11 +1084,11 @@ module. calling :func:`atexit.register` with the function to be called on exit. (Contributed by Skip Montanaro.) -* :mod:`codecs`, :mod:`encodings`, :mod:`unicodedata`: Added as part of the new +* :mod:`codecs`, :mod:`!encodings`, :mod:`unicodedata`: Added as part of the new Unicode support. -* :mod:`filecmp`: Supersedes the old :mod:`cmp`, :mod:`cmpcache` and - :mod:`dircmp` modules, which have now become deprecated. (Contributed by Gordon +* :mod:`filecmp`: Supersedes the old :mod:`!cmp`, :mod:`!cmpcache` and + :mod:`!dircmp` modules, which have now become deprecated. (Contributed by Gordon MacMillan and Moshe Zadka.) * :mod:`gettext`: This module provides internationalization (I18N) and @@ -1105,7 +1106,7 @@ module. be passed to functions that expect ordinary strings, such as the :mod:`re` module. (Contributed by Sam Rushing, with some extensions by A.M. Kuchling.) -* :mod:`pyexpat`: An interface to the Expat XML parser. (Contributed by Paul +* :mod:`!pyexpat`: An interface to the Expat XML parser. (Contributed by Paul Prescod.) * :mod:`robotparser `: Parse a :file:`robots.txt` file, which is used for writing @@ -1117,7 +1118,7 @@ module. * :mod:`tabnanny`: A module/script to check Python source code for ambiguous indentation. (Contributed by Tim Peters.) -* :mod:`UserString`: A base class useful for deriving objects that behave like +* :mod:`!UserString`: A base class useful for deriving objects that behave like strings. * :mod:`webbrowser`: A module that provides a platform independent way to launch @@ -1184,13 +1185,13 @@ Deleted and Deprecated Modules ============================== A few modules have been dropped because they're obsolete, or because there are -now better ways to do the same thing. The :mod:`stdwin` module is gone; it was +now better ways to do the same thing. The :mod:`!stdwin` module is gone; it was for a platform-independent windowing toolkit that's no longer developed. A number of modules have been moved to the :file:`lib-old` subdirectory: -:mod:`cmp`, :mod:`cmpcache`, :mod:`dircmp`, :mod:`dump`, :mod:`find`, -:mod:`grep`, :mod:`packmail`, :mod:`poly`, :mod:`util`, :mod:`whatsound`, -:mod:`zmod`. If you have code which relies on a module that's been moved to +:mod:`!cmp`, :mod:`!cmpcache`, :mod:`!dircmp`, :mod:`!dump`, :mod:`!find`, +:mod:`!grep`, :mod:`!packmail`, :mod:`!poly`, :mod:`!util`, :mod:`!whatsound`, +:mod:`!zmod`. If you have code which relies on a module that's been moved to :file:`lib-old`, you can simply add that directory to ``sys.path`` to get them back, but you're encouraged to update any code that uses these modules. diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index 6d2d3cc02b8768..f23f27c994d717 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -48,7 +48,7 @@ nested recursive function definition doesn't work:: return g(value-1) + 1 ... -The function :func:`g` will always raise a :exc:`NameError` exception, because +The function :func:`!g` will always raise a :exc:`NameError` exception, because the binding of the name ``g`` isn't in either its local namespace or in the module-level namespace. This isn't much of a problem in practice (how often do you recursively define interior functions like this?), but this also made using @@ -104,7 +104,7 @@ To make the preceding explanation a bit clearer, here's an example:: Line 4 containing the ``exec`` statement is a syntax error, since ``exec`` would define a new local variable named ``x`` whose value should -be accessed by :func:`g`. +be accessed by :func:`!g`. This shouldn't be much of a limitation, since ``exec`` is rarely used in most Python code (and when it is used, it's often a sign of a poor design @@ -161,7 +161,7 @@ PEP 207: Rich Comparisons In earlier versions, Python's support for implementing comparisons on user-defined classes and extension types was quite simple. Classes could implement a -:meth:`__cmp__` method that was given two instances of a class, and could only +:meth:`!__cmp__` method that was given two instances of a class, and could only return 0 if they were equal or +1 or -1 if they weren't; the method couldn't raise an exception or return anything other than a Boolean value. Users of Numeric Python often found this model too weak and restrictive, because in the @@ -175,21 +175,21 @@ In Python 2.1, rich comparisons were added in order to support this need. Python classes can now individually overload each of the ``<``, ``<=``, ``>``, ``>=``, ``==``, and ``!=`` operations. The new magic method names are: -+-----------+----------------+ -| Operation | Method name | -+===========+================+ -| ``<`` | :meth:`__lt__` | -+-----------+----------------+ -| ``<=`` | :meth:`__le__` | -+-----------+----------------+ -| ``>`` | :meth:`__gt__` | -+-----------+----------------+ -| ``>=`` | :meth:`__ge__` | -+-----------+----------------+ -| ``==`` | :meth:`__eq__` | -+-----------+----------------+ -| ``!=`` | :meth:`__ne__` | -+-----------+----------------+ ++-----------+------------------------+ +| Operation | Method name | ++===========+========================+ +| ``<`` | :meth:`~object.__lt__` | ++-----------+------------------------+ +| ``<=`` | :meth:`~object.__le__` | ++-----------+------------------------+ +| ``>`` | :meth:`~object.__gt__` | ++-----------+------------------------+ +| ``>=`` | :meth:`~object.__ge__` | ++-----------+------------------------+ +| ``==`` | :meth:`~object.__eq__` | ++-----------+------------------------+ +| ``!=`` | :meth:`~object.__ne__` | ++-----------+------------------------+ (The magic methods are named after the corresponding Fortran operators ``.LT.``. ``.LE.``, &c. Numeric programmers are almost certainly quite familiar with @@ -208,7 +208,7 @@ The built-in ``cmp(A,B)`` function can use the rich comparison machinery, and now accepts an optional argument specifying which comparison operation to use; this is given as one of the strings ``"<"``, ``"<="``, ``">"``, ``">="``, ``"=="``, or ``"!="``. If called without the optional third argument, -:func:`cmp` will only return -1, 0, or +1 as in previous versions of Python; +:func:`!cmp` will only return -1, 0, or +1 as in previous versions of Python; otherwise it will call the appropriate method and can return any Python object. There are also corresponding changes of interest to C programmers; there's a new @@ -245,7 +245,7 @@ out warnings that you don't want to be displayed. Third-party modules can also use this framework to deprecate old features that they no longer wish to support. -For example, in Python 2.1 the :mod:`regex` module is deprecated, so importing +For example, in Python 2.1 the :mod:`!regex` module is deprecated, so importing it causes a warning to be printed:: >>> import regex @@ -262,7 +262,7 @@ can be used to specify a particular warning category. Filters can be added to disable certain warnings; a regular expression pattern can be applied to the message or to the module name in order to suppress a -warning. For example, you may have a program that uses the :mod:`regex` module +warning. For example, you may have a program that uses the :mod:`!regex` module and not want to spare the time to convert it to use the :mod:`re` module right now. The warning can be suppressed by calling :: @@ -274,7 +274,7 @@ now. The warning can be suppressed by calling :: This adds a filter that will apply only to warnings of the class :class:`DeprecationWarning` triggered in the :mod:`__main__` module, and applies -a regular expression to only match the message about the :mod:`regex` module +a regular expression to only match the message about the :mod:`!regex` module being deprecated, and will cause such warnings to be ignored. Warnings can also be printed only once, printed every time the offending code is executed, or turned into exceptions that will cause the program to stop (unless the @@ -368,7 +368,7 @@ dictionary:: This version works for simple things such as integers, but it has a side effect; the ``_cache`` dictionary holds a reference to the return values, so they'll never be deallocated until the Python process exits and cleans up. This isn't -very noticeable for integers, but if :func:`f` returns an object, or a data +very noticeable for integers, but if :func:`!f` returns an object, or a data structure that takes up a lot of memory, this can be a problem. Weak references provide a way to implement a cache that won't keep objects alive @@ -379,7 +379,7 @@ created by calling ``wr = weakref.ref(obj)``. The object being referred to is returned by calling the weak reference as if it were a function: ``wr()``. It will return the referenced object, or ``None`` if the object no longer exists. -This makes it possible to write a :func:`memoize` function whose cache doesn't +This makes it possible to write a :func:`!memoize` function whose cache doesn't keep objects alive, by storing weak references in the cache. :: _cache = {} @@ -402,7 +402,7 @@ weak references --- an object referenced only by proxy objects is deallocated -- but instead of requiring an explicit call to retrieve the object, the proxy transparently forwards all operations to the object as long as the object still exists. If the object is deallocated, attempting to use a proxy will cause a -:exc:`weakref.ReferenceError` exception to be raised. :: +:exc:`!weakref.ReferenceError` exception to be raised. :: proxy = weakref.proxy(obj) proxy.attr # Equivalent to obj.attr @@ -443,10 +443,10 @@ Python syntax:: f.grammar = "A ::= B (C D)*" The dictionary containing attributes can be accessed as the function's -:attr:`~object.__dict__`. Unlike the :attr:`~object.__dict__` attribute of class instances, in -functions you can actually assign a new dictionary to :attr:`~object.__dict__`, though +:attr:`~function.__dict__`. Unlike the :attr:`~type.__dict__` attribute of class instances, in +functions you can actually assign a new dictionary to :attr:`~function.__dict__`, though the new value is restricted to a regular Python dictionary; you *can't* be -tricky and set it to a :class:`UserDict` instance, or any other random object +tricky and set it to a :class:`!UserDict` instance, or any other random object that behaves like a mapping. @@ -584,11 +584,11 @@ available from the Distutils SIG at https://www.python.org/community/sigs/curren New and Improved Modules ======================== -* Ka-Ping Yee contributed two new modules: :mod:`inspect.py`, a module for - getting information about live Python code, and :mod:`pydoc.py`, a module for +* Ka-Ping Yee contributed two new modules: :mod:`!inspect.py`, a module for + getting information about live Python code, and :mod:`!pydoc.py`, a module for interactively converting docstrings to HTML or text. As a bonus, :file:`Tools/scripts/pydoc`, which is now automatically installed, uses - :mod:`pydoc.py` to display documentation given a Python module, package, or + :mod:`!pydoc.py` to display documentation given a Python module, package, or class name. For example, ``pydoc xml.dom`` displays the following:: Python Library Documentation: package xml.dom in xml @@ -617,7 +617,7 @@ New and Improved Modules Kent Beck's Smalltalk testing framework. See https://pyunit.sourceforge.net/ for more information about PyUnit. -* The :mod:`difflib` module contains a class, :class:`SequenceMatcher`, which +* The :mod:`difflib` module contains a class, :class:`~difflib.SequenceMatcher`, which compares two sequences and computes the changes required to transform one sequence into the other. For example, this module can be used to write a tool similar to the Unix :program:`diff` program, and in fact the sample program @@ -633,7 +633,7 @@ New and Improved Modules 2.1 includes an updated version of the :mod:`xml` package. Some of the noteworthy changes include support for Expat 1.2 and later versions, the ability for Expat parsers to handle files in any encoding supported by Python, and - various bugfixes for SAX, DOM, and the :mod:`minidom` module. + various bugfixes for SAX, DOM, and the :mod:`!minidom` module. * Ping also contributed another hook for handling uncaught exceptions. :func:`sys.excepthook` can be set to a callable object. When an exception isn't @@ -643,10 +643,10 @@ New and Improved Modules printing an extended traceback that not only lists the stack frames, but also lists the function arguments and the local variables for each frame. -* Various functions in the :mod:`time` module, such as :func:`asctime` and - :func:`localtime`, require a floating point argument containing the time in +* Various functions in the :mod:`time` module, such as :func:`~time.asctime` and + :func:`~time.localtime`, require a floating-point argument containing the time in seconds since the epoch. The most common use of these functions is to work with - the current time, so the floating point argument has been made optional; when a + the current time, so the floating-point argument has been made optional; when a value isn't provided, the current time will be used. For example, log file entries usually need a string containing the current time; in Python 2.1, ``time.asctime()`` can be used, instead of the lengthier @@ -724,10 +724,10 @@ of the more notable changes are: a discussion in comp.lang.python. A new module and method for file objects was also added, contributed by Jeff - Epler. The new method, :meth:`xreadlines`, is similar to the existing - :func:`xrange` built-in. :func:`xreadlines` returns an opaque sequence object + Epler. The new method, :meth:`!xreadlines`, is similar to the existing + :func:`!xrange` built-in. :func:`!xreadlines` returns an opaque sequence object that only supports being iterated over, reading a line on every iteration but - not reading the entire file into memory as the existing :meth:`readlines` method + not reading the entire file into memory as the existing :meth:`!readlines` method does. You'd use it like this:: for line in sys.stdin.xreadlines(): @@ -737,7 +737,7 @@ of the more notable changes are: For a fuller discussion of the line I/O changes, see the python-dev summary for January 1--15, 2001 at https://mail.python.org/pipermail/python-dev/2001-January/. -* A new method, :meth:`popitem`, was added to dictionaries to enable +* A new method, :meth:`~dict.popitem`, was added to dictionaries to enable destructively iterating through the contents of a dictionary; this can be faster for large dictionaries because there's no need to construct a list containing all the keys or values. ``D.popitem()`` removes a random ``(key, value)`` pair diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index e6c13f957b8d54..856be5ecfa56ad 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -171,7 +171,7 @@ attributes of their own: * :attr:`~definition.__name__` is the attribute's name. -* :attr:`!__doc__` is the attribute's docstring. +* :attr:`~definition.__doc__` is the attribute's docstring. * ``__get__(object)`` is a method that retrieves the attribute value from *object*. @@ -186,7 +186,8 @@ are:: descriptor = obj.__class__.x descriptor.__get__(obj) -For methods, :meth:`!descriptor.__get__` returns a temporary object that's +For methods, :meth:`descriptor.__get__ ` returns a temporary +object that's callable, and wraps up the instance and the method to be called on it. This is also why static methods and class methods are now possible; they have descriptors that wrap up just the method, or the method and the class. As a @@ -1062,7 +1063,7 @@ code, none of the changes described here will affect you very much. simply been changed to use the new C-level interface. (Contributed by Fred L. Drake, Jr.) -* Another low-level API, primarily of interest to implementors of Python +* Another low-level API, primarily of interest to implementers of Python debuggers and development tools, was added. :c:func:`PyInterpreterState_Head` and :c:func:`PyInterpreterState_Next` let a caller walk through all the existing interpreter objects; :c:func:`PyInterpreterState_ThreadHead` and @@ -1249,7 +1250,7 @@ Some of the more notable changes are: * The :func:`pow` built-in function no longer supports 3 arguments when floating-point numbers are supplied. ``pow(x, y, z)`` returns ``(x**y) % z``, - but this is never useful for floating point numbers, and the final result varies + but this is never useful for floating-point numbers, and the final result varies unpredictably depending on the platform. A call such as ``pow(2.0, 8.0, 7.0)`` will now raise a :exc:`TypeError` exception. diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 37cd41add8132c..ac463f82cfb8ca 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1084,7 +1084,7 @@ Here are all of the changes that Python 2.3 makes to the core Python language. C3 algorithm as described in the paper `"A Monotonic Superclass Linearization for Dylan" `_. To understand the motivation for this change, read Michele Simionato's article - `"Python 2.3 Method Resolution Order" `_, or + :ref:`python_2.3_mro`, or read the thread on python-dev starting with the message at https://mail.python.org/pipermail/python-dev/2002-October/029035.html. Samuele Pedroni first pointed out the problem and also implemented the fix by coding the @@ -1113,10 +1113,10 @@ Here are all of the changes that Python 2.3 makes to the core Python language. * One of the noted incompatibilities between old- and new-style classes has been - removed: you can now assign to the :attr:`~definition.__name__` and :attr:`~class.__bases__` + removed: you can now assign to the :attr:`~type.__name__` and :attr:`~type.__bases__` attributes of new-style classes. There are some restrictions on what can be - assigned to :attr:`~class.__bases__` along the lines of those relating to assigning to - an instance's :attr:`~instance.__class__` attribute. + assigned to :attr:`!__bases__` along the lines of those relating to assigning to + an instance's :attr:`~object.__class__` attribute. .. ====================================================================== @@ -1382,7 +1382,7 @@ complete list of changes, or look through the CVS logs for all the details. In Python 2.4, the default will change to always returning floats. Application developers should enable this feature only if all their libraries - work properly when confronted with floating point time stamps, or if they use + work properly when confronted with floating-point time stamps, or if they use the tuple API. If used, the feature should be activated on an application level instead of trying to enable it on a per-use basis. @@ -1925,8 +1925,8 @@ Changes to Python's build process and to the C API include: dependence on a system version or local installation of Expat. * If you dynamically allocate type objects in your extension, you should be - aware of a change in the rules relating to the :attr:`!__module__` and - :attr:`~definition.__name__` attributes. In summary, you will want to ensure the type's + aware of a change in the rules relating to the :attr:`~type.__module__` and + :attr:`~type.__name__` attributes. In summary, you will want to ensure the type's dictionary contains a ``'__module__'`` key; making the module name the part of the type name leading up to the final period will no longer have the desired effect. For more detail, read the API reference documentation or the source. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 7e235d4370edaa..7628cfefe0ec96 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -684,11 +684,11 @@ includes a quick-start tutorial and a reference. Written by Facundo Batista and implemented by Facundo Batista, Eric Price, Raymond Hettinger, Aahz, and Tim Peters. - http://www.lahey.com/float.htm + `http://www.lahey.com/float.htm `__ The article uses Fortran code to illustrate many of the problems that floating-point inaccuracy can cause. - http://speleotrove.com/decimal/ + https://speleotrove.com/decimal/ A description of a decimal-based representation. This representation is being proposed as a standard, and underlies the new Python decimal type. Much of this material was written by Mike Cowlishaw, designer of the Rexx language. @@ -757,7 +757,7 @@ API that perform ASCII-only conversions, ignoring the locale setting: :c:expr:`double` to an ASCII string. The code for these functions came from the GLib library -(https://developer-old.gnome.org/glib/2.26/), whose developers kindly +(`https://developer-old.gnome.org/glib/2.26/ `__), whose developers kindly relicensed the relevant functions and donated them to the Python Software Foundation. The :mod:`locale` module can now change the numeric locale, letting extensions such as GTK+ produce the correct results. diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 2ae26e7a106a0b..3430ac8668e280 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1724,7 +1724,7 @@ attribute of the function object to change this:: :mod:`ctypes` also provides a wrapper for Python's C API as the ``ctypes.pythonapi`` object. This object does *not* release the global interpreter lock before calling a function, because the lock must be held when -calling into the interpreter's code. There's a :class:`py_object()` type +calling into the interpreter's code. There's a :class:`~ctypes.py_object` type constructor that will create a :c:expr:`PyObject *` pointer. A simple usage:: import ctypes @@ -1734,7 +1734,7 @@ constructor that will create a :c:expr:`PyObject *` pointer. A simple usage:: ctypes.py_object("abc"), ctypes.py_object(1)) # d is now {'abc', 1}. -Don't forget to use :class:`py_object()`; if it's omitted you end up with a +Don't forget to use :func:`~ctypes.py_object`; if it's omitted you end up with a segmentation fault. :mod:`ctypes` has been around for a while, but people still write and diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 7d3769a22286e2..fdccfb7deb1ed7 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -502,12 +502,12 @@ Python's :option:`-m` switch allows running a module as a script. When you ran a module that was located inside a package, relative imports didn't work correctly. -The fix for Python 2.6 adds a :attr:`__package__` attribute to -modules. When this attribute is present, relative imports will be +The fix for Python 2.6 adds a :attr:`module.__package__` attribute. +When this attribute is present, relative imports will be relative to the value of this attribute instead of the -:attr:`__name__` attribute. +:attr:`~module.__name__` attribute. -PEP 302-style importers can then set :attr:`__package__` as necessary. +PEP 302-style importers can then set :attr:`~module.__package__` as necessary. The :mod:`runpy` module that implements the :option:`-m` switch now does this, so relative imports will now work correctly in scripts running from inside a package. @@ -1453,7 +1453,7 @@ that will be the numerator and denominator of the resulting fraction. :: Fraction(5, 3) For converting floating-point numbers to rationals, -the float type now has an :meth:`as_integer_ratio()` method that returns +the float type now has an :meth:`as_integer_ratio` method that returns the numerator and denominator for a fraction that evaluates to the same floating-point value:: @@ -2273,7 +2273,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Guido van Rossum from work for Google App Engine; :issue:`3487`.) -* The :mod:`rlcompleter` module's :meth:`Completer.complete()` method +* The :mod:`rlcompleter` module's :meth:`Completer.complete` method will now ignore exceptions triggered while evaluating a name. (Fixed by Lorenz Quack; :issue:`2250`.) @@ -2388,11 +2388,11 @@ changes, or look through the Subversion logs for all the details. using the format character ``'?'``. (Contributed by David Remahl.) -* The :class:`Popen` objects provided by the :mod:`subprocess` module - now have :meth:`terminate`, :meth:`kill`, and :meth:`send_signal` methods. - On Windows, :meth:`send_signal` only supports the :const:`SIGTERM` +* The :class:`~subprocess.Popen` objects provided by the :mod:`subprocess` module + now have :meth:`~subprocess.Popen.terminate`, :meth:`~subprocess.Popen.kill`, and :meth:`~subprocess.Popen.send_signal` methods. + On Windows, :meth:`!send_signal` only supports the :py:const:`~signal.SIGTERM` signal, and all these methods are aliases for the Win32 API function - :c:func:`TerminateProcess`. + :c:func:`!TerminateProcess`. (Contributed by Christian Heimes.) * A new variable in the :mod:`sys` module, :attr:`float_info`, is an @@ -2566,7 +2566,7 @@ changes, or look through the Subversion logs for all the details. :meth:`tracer`, and :meth:`speed` methods. * The ability to set new shapes for the turtle, and to define a new coordinate system. - * Turtles now have an :meth:`undo()` method that can roll back actions. + * Turtles now have an :meth:`undo` method that can roll back actions. * Simple support for reacting to input events such as mouse and keyboard activity, making it possible to write simple games. * A :file:`turtle.cfg` file can be used to customize the starting appearance @@ -2992,6 +2992,32 @@ Changes to Python's build process and to the C API include: architectures (x86, PowerPC), 64-bit (x86-64 and PPC-64), or both. (Contributed by Ronald Oussoren.) +* A new function added in Python 2.6.6, :c:func:`!PySys_SetArgvEx`, sets + the value of ``sys.argv`` and can optionally update ``sys.path`` to + include the directory containing the script named by ``sys.argv[0]`` + depending on the value of an *updatepath* parameter. + + This function was added to close a security hole for applications + that embed Python. The old function, :c:func:`!PySys_SetArgv`, would + always update ``sys.path``, and sometimes it would add the current + directory. This meant that, if you ran an application embedding + Python in a directory controlled by someone else, attackers could + put a Trojan-horse module in the directory (say, a file named + :file:`os.py`) that your application would then import and run. + + If you maintain a C/C++ application that embeds Python, check + whether you're calling :c:func:`!PySys_SetArgv` and carefully consider + whether the application should be using :c:func:`!PySys_SetArgvEx` + with *updatepath* set to false. Note that using this function will + break compatibility with Python versions 2.6.5 and earlier; if you + have to continue working with earlier versions, you can leave + the call to :c:func:`!PySys_SetArgv` alone and call + ``PyRun_SimpleString("sys.path.pop(0)\n")`` afterwards to discard + the first ``sys.path`` component. + + Security issue reported as :cve:`2008-5983`; + discussed in :gh:`50003`, and fixed by Antoine Pitrou. + * The BerkeleyDB module now has a C API object, available as ``bsddb.db.api``. This object can be used by other C extensions that wish to use the :mod:`bsddb` module for their own purposes. @@ -3025,7 +3051,7 @@ Changes to Python's build process and to the C API include: * Several functions return information about the platform's floating-point support. :c:func:`PyFloat_GetMax` returns - the maximum representable floating point value, + the maximum representable floating-point value, and :c:func:`PyFloat_GetMin` returns the minimum positive value. :c:func:`PyFloat_GetInfo` returns an object containing more information from the :file:`float.h` file, such as @@ -3294,6 +3320,15 @@ that may require changes to your code: scoping rules, also cause warnings because such comparisons are forbidden entirely in 3.0. +For applications that embed Python: + +* The :c:func:`!PySys_SetArgvEx` function was added in Python 2.6.6, + letting applications close a security hole when the existing + :c:func:`!PySys_SetArgv` function was used. Check whether you're + calling :c:func:`!PySys_SetArgv` and carefully consider whether the + application should be using :c:func:`!PySys_SetArgvEx` with + *updatepath* set to false. + .. ====================================================================== diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index ada05aa22b46f6..0e4dee0bd24fb2 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -196,7 +196,7 @@ A partial list of 3.1 features that were backported to 2.7: Other new Python3-mode warnings include: -* :func:`operator.isCallable` and :func:`operator.sequenceIncludes`, +* :func:`!operator.isCallable` and :func:`!operator.sequenceIncludes`, which are not supported in 3.x, now trigger warnings. * The :option:`!-3` switch now automatically enables the :option:`!-Qwarn` switch that causes warnings @@ -291,7 +291,7 @@ modules. configuration files can now be read, modified, and then written back in their original order. -* The :meth:`~collections.somenamedtuple._asdict()` method for +* The :meth:`~collections.somenamedtuple._asdict` method for :func:`collections.namedtuple` now returns an ordered dictionary with the values appearing in the same order as the underlying tuple indices. @@ -455,11 +455,11 @@ a varying number of handlers. All this flexibility can require a lot of configuration. You can write Python statements to create objects and set their properties, but a complex set-up requires verbose but boring code. -:mod:`logging` also supports a :func:`~logging.fileConfig` +:mod:`logging` also supports a :func:`~logging.config.fileConfig` function that parses a file, but the file format doesn't support configuring filters, and it's messier to generate programmatically. -Python 2.7 adds a :func:`~logging.dictConfig` function that +Python 2.7 adds a :func:`~logging.config.dictConfig` function that uses a dictionary to configure logging. There are many ways to produce a dictionary from different sources: construct one with code; parse a file containing JSON; or use a YAML parsing library if one is @@ -533,7 +533,7 @@ implemented by Vinay Sajip, are: ``getLogger('app.network.listen')``. * The :class:`~logging.LoggerAdapter` class gained an - :meth:`~logging.LoggerAdapter.isEnabledFor` method that takes a + :meth:`~logging.Logger.isEnabledFor` method that takes a *level* and returns whether the underlying logger would process a message of that level of importance. @@ -554,8 +554,8 @@ called a :dfn:`view` instead of a fully materialized list. It's not possible to change the return values of :meth:`~dict.keys`, :meth:`~dict.values`, and :meth:`~dict.items` in Python 2.7 because too much code would break. Instead the 3.x versions were added -under the new names :meth:`~dict.viewkeys`, :meth:`~dict.viewvalues`, -and :meth:`~dict.viewitems`. +under the new names :meth:`!viewkeys`, :meth:`!viewvalues`, +and :meth:`!viewitems`. :: @@ -720,7 +720,7 @@ Some smaller changes made to the core Python language are: with B() as b: ... suite of statements ... - The :func:`contextlib.nested` function provides a very similar + The :func:`!contextlib.nested` function provides a very similar function, so it's no longer necessary and has been deprecated. (Proposed in https://codereview.appspot.com/53094; implemented by @@ -785,7 +785,7 @@ Some smaller changes made to the core Python language are: implemented by Mark Dickinson; :issue:`1811`.) * Implicit coercion for complex numbers has been removed; the interpreter - will no longer ever attempt to call a :meth:`__coerce__` method on complex + will no longer ever attempt to call a :meth:`!__coerce__` method on complex objects. (Removed by Meador Inge and Mark Dickinson; :issue:`5211`.) * The :meth:`str.format` method now supports automatic numbering of the replacement @@ -817,7 +817,7 @@ Some smaller changes made to the core Python language are: A low-level change: the :meth:`object.__format__` method now triggers a :exc:`PendingDeprecationWarning` if it's passed a format string, - because the :meth:`__format__` method for :class:`object` converts + because the :meth:`!__format__` method for :class:`object` converts the object to a string representation and formats that. Previously the method silently applied the format string to the string representation, but that could hide mistakes in Python code. If @@ -825,7 +825,7 @@ Some smaller changes made to the core Python language are: precision, presumably you're expecting the formatting to be applied in some object-specific way. (Fixed by Eric Smith; :issue:`7994`.) -* The :func:`int` and :func:`long` types gained a ``bit_length`` +* The :func:`int` and :func:`!long` types gained a ``bit_length`` method that returns the number of bits necessary to represent its argument in binary:: @@ -848,8 +848,8 @@ Some smaller changes made to the core Python language are: statements that were only working by accident. (Fixed by Meador Inge; :issue:`7902`.) -* It's now possible for a subclass of the built-in :class:`unicode` type - to override the :meth:`__unicode__` method. (Implemented by +* It's now possible for a subclass of the built-in :class:`!unicode` type + to override the :meth:`!__unicode__` method. (Implemented by Victor Stinner; :issue:`1583863`.) * The :class:`bytearray` type's :meth:`~bytearray.translate` method now accepts @@ -876,7 +876,7 @@ Some smaller changes made to the core Python language are: Forgeot d'Arc in :issue:`1616979`; CP858 contributed by Tim Hatch in :issue:`8016`.) -* The :class:`file` object will now set the :attr:`filename` attribute +* The :class:`!file` object will now set the :attr:`!filename` attribute on the :exc:`IOError` exception when trying to open a directory on POSIX platforms (noted by Jan Kaliszewski; :issue:`4764`), and now explicitly checks for and forbids writing to read-only file objects @@ -966,7 +966,7 @@ Several performance enhancements have been added: Apart from the performance improvements this change should be invisible to end users, with one exception: for testing and - debugging purposes there's a new structseq :data:`sys.long_info` that + debugging purposes there's a new structseq :data:`!sys.long_info` that provides information about the internal format, giving the number of bits per digit and the size in bytes of the C type used to store each digit:: @@ -1005,8 +1005,8 @@ Several performance enhancements have been added: conversion function that supports arbitrary bases. (Patch by Gawain Bolton; :issue:`6713`.) -* The :meth:`split`, :meth:`replace`, :meth:`rindex`, - :meth:`rpartition`, and :meth:`rsplit` methods of string-like types +* The :meth:`!split`, :meth:`!replace`, :meth:`!rindex`, + :meth:`!rpartition`, and :meth:`!rsplit` methods of string-like types (strings, Unicode strings, and :class:`bytearray` objects) now use a fast reverse-search algorithm instead of a character-by-character scan. This is sometimes faster by a factor of 10. (Added by @@ -1044,7 +1044,7 @@ changes, or look through the Subversion logs for all the details. used with :class:`memoryview` instances and other similar buffer objects. (Backported from 3.x by Florent Xicluna; :issue:`7703`.) -* Updated module: the :mod:`bsddb` module has been updated from 4.7.2devel9 +* Updated module: the :mod:`!bsddb` module has been updated from 4.7.2devel9 to version 4.8.4 of `the pybsddb package `__. The new version features better Python 3.x compatibility, various bug fixes, @@ -1129,8 +1129,8 @@ changes, or look through the Subversion logs for all the details. (Added by Raymond Hettinger; :issue:`1818`.) - Finally, the :class:`~collections.Mapping` abstract base class now - returns :const:`NotImplemented` if a mapping is compared to + Finally, the :class:`~collections.abc.Mapping` abstract base class now + returns :data:`NotImplemented` if a mapping is compared to another type that isn't a :class:`Mapping`. (Fixed by Daniel Stutzbach; :issue:`8729`.) @@ -1158,7 +1158,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Mats Kindahl; :issue:`7005`.) -* Deprecated function: :func:`contextlib.nested`, which allows +* Deprecated function: :func:`!contextlib.nested`, which allows handling more than one context manager with a single :keyword:`with` statement, has been deprecated, because the :keyword:`!with` statement now supports multiple context managers. @@ -1184,7 +1184,7 @@ changes, or look through the Subversion logs for all the details. * New method: the :class:`~decimal.Decimal` class gained a :meth:`~decimal.Decimal.from_float` class method that performs an exact - conversion of a floating-point number to a :class:`~decimal.Decimal`. + conversion of a floating-point number to a :class:`!Decimal`. This exact conversion strives for the closest decimal approximation to the floating-point representation's value; the resulting decimal value will therefore still include the inaccuracy, @@ -1198,9 +1198,9 @@ changes, or look through the Subversion logs for all the details. of the operands. Previously such comparisons would fall back to Python's default rules for comparing objects, which produced arbitrary results based on their type. Note that you still cannot combine - :class:`Decimal` and floating-point in other operations such as addition, + :class:`!Decimal` and floating point in other operations such as addition, since you should be explicitly choosing how to convert between float and - :class:`~decimal.Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) + :class:`!Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) The constructor for :class:`~decimal.Decimal` now accepts floating-point numbers (added by Raymond Hettinger; :issue:`8257`) @@ -1218,7 +1218,7 @@ changes, or look through the Subversion logs for all the details. more sensible for numeric types. (Changed by Mark Dickinson; :issue:`6857`.) Comparisons involving a signaling NaN value (or ``sNAN``) now signal - :const:`InvalidOperation` instead of silently returning a true or + :const:`~decimal.InvalidOperation` instead of silently returning a true or false value depending on the comparison operator. Quiet NaN values (or ``NaN``) are now hashable. (Fixed by Mark Dickinson; :issue:`7279`.) @@ -1235,13 +1235,13 @@ changes, or look through the Subversion logs for all the details. created some new files that should be included. (Fixed by Tarek Ziadé; :issue:`8688`.) -* The :mod:`doctest` module's :const:`IGNORE_EXCEPTION_DETAIL` flag +* The :mod:`doctest` module's :const:`~doctest.IGNORE_EXCEPTION_DETAIL` flag will now ignore the name of the module containing the exception being tested. (Patch by Lennart Regebro; :issue:`7490`.) * The :mod:`email` module's :class:`~email.message.Message` class will now accept a Unicode-valued payload, automatically converting the - payload to the encoding specified by :attr:`output_charset`. + payload to the encoding specified by :attr:`!output_charset`. (Added by R. David Murray; :issue:`1368247`.) * The :class:`~fractions.Fraction` class now accepts a single float or @@ -1268,10 +1268,10 @@ changes, or look through the Subversion logs for all the details. :issue:`6845`.) * New class decorator: :func:`~functools.total_ordering` in the :mod:`functools` - module takes a class that defines an :meth:`__eq__` method and one of - :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, + module takes a class that defines an :meth:`~object.__eq__` method and one of + :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, and generates the missing comparison methods. Since the - :meth:`__cmp__` method is being deprecated in Python 3.x, + :meth:`!__cmp__` method is being deprecated in Python 3.x, this decorator makes it easier to define ordered classes. (Added by Raymond Hettinger; :issue:`5479`.) @@ -1300,7 +1300,7 @@ changes, or look through the Subversion logs for all the details. :mod:`gzip` module will now consume these trailing bytes. (Fixed by Tadek Pietraszek and Brian Curtin; :issue:`2846`.) -* New attribute: the :mod:`hashlib` module now has an :attr:`~hashlib.hashlib.algorithms` +* New attribute: the :mod:`hashlib` module now has an :attr:`!algorithms` attribute containing a tuple naming the supported algorithms. In Python 2.7, ``hashlib.algorithms`` contains ``('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')``. @@ -1348,10 +1348,10 @@ changes, or look through the Subversion logs for all the details. * Updated module: The :mod:`io` library has been upgraded to the version shipped with Python 3.1. For 3.1, the I/O library was entirely rewritten in C and is 2 to 20 times faster depending on the task being performed. The - original Python version was renamed to the :mod:`_pyio` module. + original Python version was renamed to the :mod:`!_pyio` module. One minor resulting change: the :class:`io.TextIOBase` class now - has an :attr:`errors` attribute giving the error setting + has an :attr:`~io.TextIOBase.errors` attribute giving the error setting used for encoding and decoding errors (one of ``'strict'``, ``'replace'``, ``'ignore'``). @@ -1423,10 +1423,10 @@ changes, or look through the Subversion logs for all the details. passed to the callable. (Contributed by lekma; :issue:`5585`.) - The :class:`~multiprocessing.Pool` class, which controls a pool of worker processes, + The :class:`~multiprocessing.pool.Pool` class, which controls a pool of worker processes, now has an optional *maxtasksperchild* parameter. Worker processes will perform the specified number of tasks and then exit, causing the - :class:`~multiprocessing.Pool` to start a new worker. This is useful if tasks may leak + :class:`!Pool` to start a new worker. This is useful if tasks may leak memory or other resources, or if some tasks will cause the worker to become very large. (Contributed by Charles Cazabon; :issue:`6963`.) @@ -1498,7 +1498,7 @@ changes, or look through the Subversion logs for all the details. global site-packages directories, :func:`~site.getusersitepackages` returns the path of the user's site-packages directory, and - :func:`~site.getuserbase` returns the value of the :envvar:`USER_BASE` + :func:`~site.getuserbase` returns the value of the :data:`~site.USER_BASE` environment variable, giving the path to a directory that can be used to store data. (Contributed by Tarek Ziadé; :issue:`6693`.) @@ -1540,15 +1540,15 @@ changes, or look through the Subversion logs for all the details. * The :mod:`ssl` module's :class:`~ssl.SSLSocket` objects now support the buffer API, which fixed a test suite failure (fix by Antoine Pitrou; :issue:`7133`) and automatically set - OpenSSL's :c:macro:`SSL_MODE_AUTO_RETRY`, which will prevent an error + OpenSSL's :c:macro:`!SSL_MODE_AUTO_RETRY`, which will prevent an error code being returned from :meth:`recv` operations that trigger an SSL renegotiation (fix by Antoine Pitrou; :issue:`8222`). - The :func:`ssl.wrap_socket` constructor function now takes a + The :func:`~ssl.SSLContext.wrap_socket` constructor function now takes a *ciphers* argument that's a string listing the encryption algorithms to be allowed; the format of the string is described `in the OpenSSL documentation - `__. + `__. (Added by Antoine Pitrou; :issue:`8322`.) Another change makes the extension load all of OpenSSL's ciphers and @@ -1568,8 +1568,8 @@ changes, or look through the Subversion logs for all the details. code (one of ``bBhHiIlLqQ``); it now always raises a :exc:`struct.error` exception. (Changed by Mark Dickinson; :issue:`1523`.) The :func:`~struct.pack` function will also - attempt to use :meth:`__index__` to convert and pack non-integers - before trying the :meth:`__int__` method or reporting an error. + attempt to use :meth:`~object.__index__` to convert and pack non-integers + before trying the :meth:`~object.__int__` method or reporting an error. (Changed by Mark Dickinson; :issue:`8300`.) * New function: the :mod:`subprocess` module's @@ -1590,7 +1590,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Gregory P. Smith.) The :mod:`subprocess` module will now retry its internal system calls - on receiving an :const:`EINTR` signal. (Reported by several people; final + on receiving an :const:`~errno.EINTR` signal. (Reported by several people; final patch by Gregory P. Smith in :issue:`1068268`.) * New function: :func:`~symtable.Symbol.is_declared_global` in the :mod:`symtable` module @@ -1602,16 +1602,16 @@ changes, or look through the Subversion logs for all the details. identifier instead of the previous default value of ``'python'``. (Changed by Sean Reifschneider; :issue:`8451`.) -* The ``sys.version_info`` value is now a named tuple, with attributes - named :attr:`major`, :attr:`minor`, :attr:`micro`, - :attr:`releaselevel`, and :attr:`serial`. (Contributed by Ross +* The :attr:`sys.version_info` value is now a named tuple, with attributes + named :attr:`!major`, :attr:`!minor`, :attr:`!micro`, + :attr:`!releaselevel`, and :attr:`!serial`. (Contributed by Ross Light; :issue:`4285`.) :func:`sys.getwindowsversion` also returns a named tuple, - with attributes named :attr:`major`, :attr:`minor`, :attr:`build`, - :attr:`platform`, :attr:`service_pack`, :attr:`service_pack_major`, - :attr:`service_pack_minor`, :attr:`suite_mask`, and - :attr:`product_type`. (Contributed by Brian Curtin; :issue:`7766`.) + with attributes named :attr:`!major`, :attr:`!minor`, :attr:`!build`, + :attr:`!platform`, :attr:`!service_pack`, :attr:`!service_pack_major`, + :attr:`!service_pack_minor`, :attr:`!suite_mask`, and + :attr:`!product_type`. (Contributed by Brian Curtin; :issue:`7766`.) * The :mod:`tarfile` module's default error handling has changed, to no longer suppress fatal errors. The default error level was previously 0, @@ -1691,7 +1691,7 @@ changes, or look through the Subversion logs for all the details. (Originally implemented in Python 3.x by Raymond Hettinger, and backported to 2.7 by Michael Foord.) -* The ElementTree library, :mod:`xml.etree`, no longer escapes +* The :mod:`xml.etree.ElementTree` library, no longer escapes ampersands and angle brackets when outputting an XML processing instruction (which looks like ````) or comment (which looks like ````). @@ -1701,8 +1701,8 @@ changes, or look through the Subversion logs for all the details. :mod:`SimpleXMLRPCServer ` modules, have improved performance by supporting HTTP/1.1 keep-alive and by optionally using gzip encoding to compress the XML being exchanged. The gzip compression is - controlled by the :attr:`encode_threshold` attribute of - :class:`SimpleXMLRPCRequestHandler`, which contains a size in bytes; + controlled by the :attr:`!encode_threshold` attribute of + :class:`~xmlrpc.server.SimpleXMLRPCRequestHandler`, which contains a size in bytes; responses larger than this will be compressed. (Contributed by Kristján Valur Jónsson; :issue:`6267`.) @@ -1713,7 +1713,8 @@ changes, or look through the Subversion logs for all the details. :mod:`zipfile` now also supports archiving empty directories and extracts them correctly. (Fixed by Kuba Wieczorek; :issue:`4710`.) Reading files out of an archive is faster, and interleaving - :meth:`~zipfile.ZipFile.read` and :meth:`~zipfile.ZipFile.readline` now works correctly. + :meth:`read() ` and + :meth:`readline() ` now works correctly. (Contributed by Nir Aides; :issue:`7610`.) The :func:`~zipfile.is_zipfile` function now @@ -1737,7 +1738,7 @@ New module: importlib Python 3.1 includes the :mod:`importlib` package, a re-implementation of the logic underlying Python's :keyword:`import` statement. -:mod:`importlib` is useful for implementors of Python interpreters and +:mod:`importlib` is useful for implementers of Python interpreters and to users who wish to write new importers that can participate in the import process. Python 2.7 doesn't contain the complete :mod:`importlib` package, but instead has a tiny subset that contains @@ -1807,14 +1808,14 @@ closely resemble the native platform's widgets. This widget set was originally called Tile, but was renamed to Ttk (for "themed Tk") on being added to Tcl/Tck release 8.5. -To learn more, read the :mod:`ttk` module documentation. You may also +To learn more, read the :mod:`~tkinter.ttk` module documentation. You may also wish to read the Tcl/Tk manual page describing the Ttk theme engine, available at -https://www.tcl.tk/man/tcl8.5/TkCmd/ttk_intro.htm. Some +https://www.tcl.tk/man/tcl8.5/TkCmd/ttk_intro.html. Some screenshots of the Python/Ttk code in use are at https://code.google.com/archive/p/python-ttk/wikis/Screenshots.wiki. -The :mod:`ttk` module was written by Guilherme Polo and added in +The :mod:`tkinter.ttk` module was written by Guilherme Polo and added in :issue:`2983`. An alternate version called ``Tile.py``, written by Martin Franklin and maintained by Kevin Walzer, was proposed for inclusion in :issue:`2618`, but the authors argued that Guilherme @@ -1830,8 +1831,7 @@ The :mod:`unittest` module was greatly enhanced; many new features were added. Most of these features were implemented by Michael Foord, unless otherwise noted. The enhanced version of the module is downloadable separately for use with Python versions 2.4 to 2.6, -packaged as the :mod:`unittest2` package, from -https://pypi.org/project/unittest2. +packaged as the :mod:`!unittest2` package, from :pypi:`unittest2`. When used from the command line, the module can automatically discover tests. It's not as fancy as `py.test `__ or @@ -1938,19 +1938,20 @@ GvR worked on merging them into Python's version of :mod:`unittest`. differences in the two strings. This comparison is now used by default when Unicode strings are compared with :meth:`~unittest.TestCase.assertEqual`. -* :meth:`~unittest.TestCase.assertRegexpMatches` and - :meth:`~unittest.TestCase.assertNotRegexpMatches` checks whether the +* :meth:`assertRegexpMatches() ` and + :meth:`assertNotRegexpMatches() ` checks whether the first argument is a string matching or not matching the regular expression provided as the second argument (:issue:`8038`). -* :meth:`~unittest.TestCase.assertRaisesRegexp` checks whether a particular exception +* :meth:`assertRaisesRegexp() ` checks + whether a particular exception is raised, and then also checks that the string representation of the exception matches the provided regular expression. * :meth:`~unittest.TestCase.assertIn` and :meth:`~unittest.TestCase.assertNotIn` tests whether *first* is or is not in *second*. -* :meth:`~unittest.TestCase.assertItemsEqual` tests whether two provided sequences +* :meth:`assertItemsEqual() ` tests whether two provided sequences contain the same elements. * :meth:`~unittest.TestCase.assertSetEqual` compares whether two sets are equal, and @@ -1966,7 +1967,7 @@ GvR worked on merging them into Python's version of :mod:`unittest`. * :meth:`~unittest.TestCase.assertDictEqual` compares two dictionaries and reports the differences; it's now used by default when you compare two dictionaries - using :meth:`~unittest.TestCase.assertEqual`. :meth:`~unittest.TestCase.assertDictContainsSubset` checks whether + using :meth:`~unittest.TestCase.assertEqual`. :meth:`!assertDictContainsSubset` checks whether all of the key/value pairs in *first* are found in *second*. * :meth:`~unittest.TestCase.assertAlmostEqual` and :meth:`~unittest.TestCase.assertNotAlmostEqual` test @@ -2023,8 +2024,8 @@ version 1.3. Some of the new features are: p = ET.XMLParser(encoding='utf-8') t = ET.XML("""""", parser=p) - Errors in parsing XML now raise a :exc:`ParseError` exception, whose - instances have a :attr:`position` attribute + Errors in parsing XML now raise a :exc:`~xml.etree.ElementTree.ParseError` exception, whose + instances have a :attr:`!position` attribute containing a (*line*, *column*) tuple giving the location of the problem. * ElementTree's code for converting trees to a string has been @@ -2034,7 +2035,8 @@ version 1.3. Some of the new features are: "xml" (the default), "html", or "text". HTML mode will output empty elements as ```` instead of ````, and text mode will skip over elements and only output the text chunks. If - you set the :attr:`tag` attribute of an element to ``None`` but + you set the :attr:`~xml.etree.ElementTree.Element.tag` attribute of an + element to ``None`` but leave its children in place, the element will be omitted when the tree is written out, so you don't need to do more extensive rearrangement to remove a single element. @@ -2064,14 +2066,14 @@ version 1.3. Some of the new features are: # Outputs 1... print ET.tostring(new) -* New :class:`Element` method: +* New :class:`~xml.etree.ElementTree.Element` method: :meth:`~xml.etree.ElementTree.Element.iter` yields the children of the element as a generator. It's also possible to write ``for child in elem:`` to loop over an element's children. The existing method - :meth:`getiterator` is now deprecated, as is :meth:`getchildren` + :meth:`!getiterator` is now deprecated, as is :meth:`!getchildren` which constructs and returns a list of children. -* New :class:`Element` method: +* New :class:`~xml.etree.ElementTree.Element` method: :meth:`~xml.etree.ElementTree.Element.itertext` yields all chunks of text that are descendants of the element. For example:: @@ -2175,8 +2177,7 @@ Changes to Python's build process and to the C API include: whether the application should be using :c:func:`!PySys_SetArgvEx` with *updatepath* set to false. - Security issue reported as `CVE-2008-5983 - `_; + Security issue reported as :cve:`2008-5983`; discussed in :issue:`5753`, and fixed by Antoine Pitrou. * New macros: the Python header files now define the following macros: @@ -2227,7 +2228,7 @@ Changes to Python's build process and to the C API include: (Fixed by Thomas Wouters; :issue:`1590864`.) * The :c:func:`Py_Finalize` function now calls the internal - :func:`threading._shutdown` function; this prevents some exceptions from + :func:`!threading._shutdown` function; this prevents some exceptions from being raised when an interpreter shuts down. (Patch by Adam Olsen; :issue:`1722344`.) @@ -2242,7 +2243,7 @@ Changes to Python's build process and to the C API include: Heller; :issue:`3102`.) * New configure option: the :option:`!--with-system-expat` switch allows - building the :mod:`pyexpat` module to use the system Expat library. + building the :mod:`pyexpat ` module to use the system Expat library. (Contributed by Arfrever Frehtes Taifersar Arahesis; :issue:`7609`.) * New configure option: the @@ -2329,9 +2330,9 @@ Port-Specific Changes: Windows * The :mod:`msvcrt` module now contains some constants from the :file:`crtassem.h` header file: - :data:`CRT_ASSEMBLY_VERSION`, - :data:`VC_ASSEMBLY_PUBLICKEYTOKEN`, - and :data:`LIBRARIES_ASSEMBLY_NAME_PREFIX`. + :data:`~msvcrt.CRT_ASSEMBLY_VERSION`, + :data:`~msvcrt.VC_ASSEMBLY_PUBLICKEYTOKEN`, + and :data:`~msvcrt.LIBRARIES_ASSEMBLY_NAME_PREFIX`. (Contributed by David Cournapeau; :issue:`4365`.) * The :mod:`_winreg ` module for accessing the registry now implements @@ -2342,21 +2343,21 @@ Port-Specific Changes: Windows were also tested and documented. (Implemented by Brian Curtin: :issue:`7347`.) -* The new :c:func:`_beginthreadex` API is used to start threads, and +* The new :c:func:`!_beginthreadex` API is used to start threads, and the native thread-local storage functions are now used. (Contributed by Kristján Valur Jónsson; :issue:`3582`.) * The :func:`os.kill` function now works on Windows. The signal value - can be the constants :const:`CTRL_C_EVENT`, - :const:`CTRL_BREAK_EVENT`, or any integer. The first two constants + can be the constants :const:`~signal.CTRL_C_EVENT`, + :const:`~signal.CTRL_BREAK_EVENT`, or any integer. The first two constants will send :kbd:`Control-C` and :kbd:`Control-Break` keystroke events to - subprocesses; any other value will use the :c:func:`TerminateProcess` + subprocesses; any other value will use the :c:func:`!TerminateProcess` API. (Contributed by Miki Tebeka; :issue:`1220212`.) * The :func:`os.listdir` function now correctly fails for an empty path. (Fixed by Hirokazu Yamamoto; :issue:`5913`.) -* The :mod:`mimelib` module will now read the MIME database from +* The :mod:`mimetypes` module will now read the MIME database from the Windows registry when initializing. (Patch by Gabriel Genellina; :issue:`4969`.) @@ -2385,7 +2386,7 @@ Port-Specific Changes: Mac OS X Port-Specific Changes: FreeBSD ----------------------------------- -* FreeBSD 7.1's :const:`SO_SETFIB` constant, used with the :func:`~socket.socket` methods +* FreeBSD 7.1's :const:`!SO_SETFIB` constant, used with the :func:`~socket.socket` methods :func:`~socket.socket.getsockopt`/:func:`~socket.socket.setsockopt` to select an alternate routing table, is now available in the :mod:`socket` module. (Added by Kyle VanderBeek; :issue:`8235`.) @@ -2441,7 +2442,7 @@ This section lists previously described changes and other bugfixes that may require changes to your code: * The :func:`range` function processes its arguments more - consistently; it will now call :meth:`__int__` on non-float, + consistently; it will now call :meth:`~object.__int__` on non-float, non-integer arguments that are supplied to it. (Fixed by Alexander Belopolsky; :issue:`1533`.) @@ -2486,13 +2487,13 @@ In the standard library: (or ``NaN``) are now hashable. (Fixed by Mark Dickinson; :issue:`7279`.) -* The ElementTree library, :mod:`xml.etree`, no longer escapes +* The :mod:`xml.etree.ElementTree` library no longer escapes ampersands and angle brackets when outputting an XML processing instruction (which looks like ````) or comment (which looks like ````). (Patch by Neil Muller; :issue:`2746`.) -* The :meth:`~StringIO.StringIO.readline` method of :class:`~StringIO.StringIO` objects now does +* The :meth:`!readline` method of :class:`~io.StringIO` objects now does nothing when a negative length is requested, as other file-like objects do. (:issue:`7348`). @@ -2577,11 +2578,11 @@ Two new environment variables for debug mode -------------------------------------------- In debug mode, the ``[xxx refs]`` statistic is not written by default, the -:envvar:`PYTHONSHOWREFCOUNT` environment variable now must also be set. +:envvar:`!PYTHONSHOWREFCOUNT` environment variable now must also be set. (Contributed by Victor Stinner; :issue:`31733`.) When Python is compiled with ``COUNT_ALLOC`` defined, allocation counts are no -longer dumped by default anymore: the :envvar:`PYTHONSHOWALLOCCOUNT` environment +longer dumped by default anymore: the :envvar:`!PYTHONSHOWALLOCCOUNT` environment variable must now also be set. Moreover, allocation counts are now dumped into stderr, rather than stdout. (Contributed by Victor Stinner; :issue:`31692`.) @@ -2623,7 +2624,7 @@ with the first of those changes appearing in the Python 2.7.7 release. 2 applications. (Contributed by Alex Gaynor; :issue:`21304`.) * OpenSSL 1.0.1h was upgraded for the official Windows installers published on - python.org. (contributed by Zachary Ware in :issue:`21671` for CVE-2014-0224) + python.org. (Contributed by Zachary Ware in :issue:`21671` for :cve:`2014-0224`.) :pep:`466` related features added in Python 2.7.9: @@ -2679,14 +2680,12 @@ automatic ``PATH`` modifications to have ``pip`` available from the command line by default, otherwise it can still be accessed through the Python launcher for Windows as ``py -m pip``. -As `discussed in the PEP`__, platform packagers may choose not to install +As :pep:`discussed in the PEP <0477#disabling-ensurepip-by-downstream-distributors>`, +platform packagers may choose not to install these commands by default, as long as, when invoked, they provide clear and simple directions on how to install them on that platform (usually using the system package manager). -__ https://peps.python.org/pep-0477/#disabling-ensurepip-by-downstream-distributors - - Documentation Changes ~~~~~~~~~~~~~~~~~~~~~ @@ -2712,7 +2711,8 @@ PEP 476: Enabling certificate verification by default for stdlib http clients ----------------------------------------------------------------------------- :pep:`476` updated :mod:`httplib ` and modules which use it, such as -:mod:`urllib2 ` and :mod:`xmlrpclib`, to now verify that the server +:mod:`urllib2 ` and :mod:`xmlrpclib `, to now +verify that the server presents a certificate which is signed by a Certificate Authority in the platform trust store and whose hostname matches the hostname being requested by default, significantly improving security for many applications. This @@ -2753,7 +2753,7 @@ entire Python process back to the default permissive behaviour of Python 2.7.8 and earlier. For cases where the connection establishment code can't be modified, but the -overall application can be, the new :func:`ssl._https_verify_certificates` +overall application can be, the new :func:`!ssl._https_verify_certificates` function can be used to adjust the default behaviour at runtime. diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 888e6279754fc2..d97f5fdd9eaa4a 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -357,8 +357,8 @@ New Syntax provides a standardized way of annotating a function's parameters and return value. There are no semantics attached to such annotations except that they can be introspected at runtime using - the :attr:`__annotations__` attribute. The intent is to encourage - experimentation through metaclasses, decorators or frameworks. + the :attr:`~object.__annotations__` attribute. The intent is to + encourage experimentation through metaclasses, decorators or frameworks. * :pep:`3102`: Keyword-only arguments. Named parameters occurring after ``*args`` in the parameter list *must* be specified using diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index e237179f4b1829..b9606beb5f9ef9 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -80,6 +80,28 @@ Support was also added for third-party tools like `PyYAML ` PEP written by Armin Ronacher and Raymond Hettinger. Implementation written by Raymond Hettinger. +Since an ordered dictionary remembers its insertion order, it can be used +in conjunction with sorting to make a sorted dictionary:: + + >>> # regular unsorted dictionary + >>> d = {'banana': 3, 'apple':4, 'pear': 1, 'orange': 2} + + >>> # dictionary sorted by key + >>> OrderedDict(sorted(d.items(), key=lambda t: t[0])) + OrderedDict([('apple', 4), ('banana', 3), ('orange', 2), ('pear', 1)]) + + >>> # dictionary sorted by value + >>> OrderedDict(sorted(d.items(), key=lambda t: t[1])) + OrderedDict([('pear', 1), ('orange', 2), ('banana', 3), ('apple', 4)]) + + >>> # dictionary sorted by length of the key string + >>> OrderedDict(sorted(d.items(), key=lambda t: len(t[0]))) + OrderedDict([('pear', 1), ('apple', 4), ('orange', 2), ('banana', 3)]) + +The new sorted dictionaries maintain their sort order when entries +are deleted. But when new keys are added, the keys are appended +to the end and the sort is not maintained. + PEP 378: Format Specifier for Thousands Separator ================================================= @@ -152,7 +174,7 @@ Some smaller changes made to the core Python language are: (Contributed by Eric Smith; :issue:`5237`.) -* The :func:`string.maketrans` function is deprecated and is replaced by new +* The :func:`!string.maketrans` function is deprecated and is replaced by new static methods, :meth:`bytes.maketrans` and :meth:`bytearray.maketrans`. This change solves the confusion around which types were supported by the :mod:`string` module. Now, :class:`str`, :class:`bytes`, and @@ -169,7 +191,7 @@ Some smaller changes made to the core Python language are: ... if '' in line: ... outfile.write(line) - With the new syntax, the :func:`contextlib.nested` function is no longer + With the new syntax, the :func:`!contextlib.nested` function is no longer needed and is now deprecated. (Contributed by Georg Brandl and Mattias Brändström; @@ -183,9 +205,9 @@ Some smaller changes made to the core Python language are: (Contributed by Mark Dickinson; :issue:`4707`.) -* Python now uses David Gay's algorithm for finding the shortest floating - point representation that doesn't change its value. This should help - mitigate some of the confusion surrounding binary floating point +* Python now uses David Gay's algorithm for finding the shortest floating-point + representation that doesn't change its value. This should help + mitigate some of the confusion surrounding binary floating-point numbers. The significance is easily seen with a number like ``1.1`` which does not @@ -193,7 +215,7 @@ Some smaller changes made to the core Python language are: equivalent, an expression like ``float('1.1')`` evaluates to the nearest representable value which is ``0x1.199999999999ap+0`` in hex or ``1.100000000000000088817841970012523233890533447265625`` in decimal. That - nearest value was and still is used in subsequent floating point + nearest value was and still is used in subsequent floating-point calculations. What is new is how the number gets displayed. Formerly, Python used a @@ -202,7 +224,7 @@ Some smaller changes made to the core Python language are: using 17 digits was that it relied on IEEE-754 guarantees to assure that ``eval(repr(1.1))`` would round-trip exactly to its original value. The disadvantage is that many people found the output to be confusing (mistaking - intrinsic limitations of binary floating point representation as being a + intrinsic limitations of binary floating-point representation as being a problem with Python itself). The new algorithm for ``repr(1.1)`` is smarter and returns ``'1.1'``. @@ -214,8 +236,8 @@ Some smaller changes made to the core Python language are: it does not change the underlying values. So, it is still the case that ``1.1 + 2.2 != 3.3`` even though the representations may suggest otherwise. - The new algorithm depends on certain features in the underlying floating - point implementation. If the required features are not found, the old + The new algorithm depends on certain features in the underlying floating-point + implementation. If the required features are not found, the old algorithm will continue to be used. Also, the text pickle protocols assure cross-platform portability by using the old algorithm. @@ -359,16 +381,20 @@ New, Improved, and Deprecated Modules x / 0 In addition, several new assertion methods were added including - :func:`assertSetEqual`, :func:`assertDictEqual`, - :func:`assertDictContainsSubset`, :func:`assertListEqual`, - :func:`assertTupleEqual`, :func:`assertSequenceEqual`, - :func:`assertRaisesRegexp`, :func:`assertIsNone`, - and :func:`assertIsNotNone`. + :meth:`~unittest.TestCase.assertSetEqual`, + :meth:`~unittest.TestCase.assertDictEqual`, + :meth:`!assertDictContainsSubset`, + :meth:`~unittest.TestCase.assertListEqual`, + :meth:`~unittest.TestCase.assertTupleEqual`, + :meth:`~unittest.TestCase.assertSequenceEqual`, + :meth:`assertRaisesRegexp() `, + :meth:`~unittest.TestCase.assertIsNone`, + and :meth:`~unittest.TestCase.assertIsNotNone`. (Contributed by Benjamin Peterson and Antoine Pitrou.) -* The :mod:`io` module has three new constants for the :meth:`seek` - method :data:`SEEK_SET`, :data:`SEEK_CUR`, and :data:`SEEK_END`. +* The :mod:`io` module has three new constants for the :meth:`~io.IOBase.seek` + method: :data:`~os.SEEK_SET`, :data:`~os.SEEK_CUR`, and :data:`~os.SEEK_END`. * The :data:`sys.version_info` tuple is now a named tuple:: @@ -524,7 +550,7 @@ Porting to Python 3.1 This section lists previously described changes and other bugfixes that may require changes to your code: -* The new floating point string representations can break existing doctests. +* The new floating-point string representations can break existing doctests. For example:: def e(): diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 7dc06e9af694d9..e4699fbf8edaf7 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -352,7 +352,7 @@ was expecting an indentation, including the location of the statement: AttributeErrors ~~~~~~~~~~~~~~~ -When printing :exc:`AttributeError`, :c:func:`PyErr_Display` will offer +When printing :exc:`AttributeError`, :c:func:`!PyErr_Display` will offer suggestions of similar attribute names in the object that the exception was raised from: @@ -366,14 +366,14 @@ raised from: (Contributed by Pablo Galindo in :issue:`38530`.) .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error + Notice this won't work if :c:func:`!PyErr_Display` is not called to display the error which can happen if some other custom error display function is used. This is a common scenario in some REPLs like IPython. NameErrors ~~~~~~~~~~ -When printing :exc:`NameError` raised by the interpreter, :c:func:`PyErr_Display` +When printing :exc:`NameError` raised by the interpreter, :c:func:`!PyErr_Display` will offer suggestions of similar variable names in the function that the exception was raised from: @@ -388,7 +388,7 @@ was raised from: (Contributed by Pablo Galindo in :issue:`38530`.) .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error, + Notice this won't work if :c:func:`!PyErr_Display` is not called to display the error, which can happen if some other custom error display function is used. This is a common scenario in some REPLs like IPython. @@ -690,7 +690,7 @@ are in :pep:`635`, and a longer tutorial is in :pep:`636`. Optional ``EncodingWarning`` and ``encoding="locale"`` option ------------------------------------------------------------- -The default encoding of :class:`TextIOWrapper` and :func:`open` is +The default encoding of :class:`~io.TextIOWrapper` and :func:`open` is platform and locale dependent. Since UTF-8 is used on most Unix platforms, omitting ``encoding`` option when opening UTF-8 files (e.g. JSON, YAML, TOML, Markdown) is a very common bug. For example:: @@ -785,7 +785,7 @@ especially when forward references or invalid types were involved. Compare:: StrCache = 'Cache[str]' # a type alias LOG_PREFIX = 'LOG[DEBUG]' # a module constant -Now the :mod:`typing` module has a special value :data:`TypeAlias` +Now the :mod:`typing` module has a special value :data:`~typing.TypeAlias` which lets you declare type aliases more explicitly:: StrCache: TypeAlias = 'Cache[str]' # a type alias @@ -798,10 +798,10 @@ See :pep:`613` for more details. PEP 647: User-Defined Type Guards --------------------------------- -:data:`TypeGuard` has been added to the :mod:`typing` module to annotate +:data:`~typing.TypeGuard` has been added to the :mod:`typing` module to annotate type guard functions and improve information provided to static type checkers -during type narrowing. For more information, please see :data:`TypeGuard`\ 's -documentation, and :pep:`647`. +during type narrowing. For more information, please see +:data:`~typing.TypeGuard`\ 's documentation, and :pep:`647`. (Contributed by Ken Jin and Guido van Rossum in :issue:`43766`. PEP written by Eric Traut.) @@ -828,7 +828,7 @@ Other Language Changes :meth:`~object.__index__` method). (Contributed by Serhiy Storchaka in :issue:`37999`.) -* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will +* If :func:`object.__ipow__` returns :data:`NotImplemented`, the operator will correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. (Contributed by Alex Shkop in :issue:`38302`.) @@ -972,8 +972,8 @@ and objects representing asynchronously released resources. Add asynchronous context manager support to :func:`contextlib.nullcontext`. (Contributed by Tom Gringauz in :issue:`41543`.) -Add :class:`AsyncContextDecorator`, for supporting usage of async context managers -as decorators. +Add :class:`~contextlib.AsyncContextDecorator`, for supporting usage of async +context managers as decorators. curses ------ @@ -1089,8 +1089,8 @@ encodings enum ---- -:class:`Enum` :func:`__repr__` now returns ``enum_name.member_name`` and -:func:`__str__` now returns ``member_name``. Stdlib enums available as +:class:`~enum.Enum` :func:`~object.__repr__` now returns ``enum_name.member_name`` and +:func:`~object.__str__` now returns ``member_name``. Stdlib enums available as module constants have a :func:`repr` of ``module_name.member_name``. (Contributed by Ethan Furman in :issue:`40066`.) @@ -1104,7 +1104,7 @@ Add *encoding* and *errors* parameters in :func:`fileinput.input` and :class:`fileinput.FileInput`. (Contributed by Inada Naoki in :issue:`43712`.) -:func:`fileinput.hook_compressed` now returns :class:`TextIOWrapper` object +:func:`fileinput.hook_compressed` now returns :class:`~io.TextIOWrapper` object when *mode* is "r" and file is compressed, like uncompressed files. (Contributed by Inada Naoki in :issue:`5758`.) @@ -1202,12 +1202,12 @@ Feature parity with ``importlib_metadata`` 4.6 :ref:`importlib.metadata entry points ` now provide a nicer experience for selecting entry points by group and name through a new -:class:`importlib.metadata.EntryPoints` class. See the Compatibility +:ref:`importlib.metadata.EntryPoints ` class. See the Compatibility Note in the docs for more info on the deprecation and usage. -Added :func:`importlib.metadata.packages_distributions` for resolving -top-level Python modules and packages to their -:class:`importlib.metadata.Distribution`. +Added :ref:`importlib.metadata.packages_distributions() ` +for resolving top-level Python modules and packages to their +:ref:`importlib.metadata.Distribution `. inspect ------- @@ -1224,7 +1224,7 @@ best practice for accessing the annotations dict defined on any Python object; for more information on best practices for working with annotations, please see :ref:`annotations-howto`. Relatedly, :func:`inspect.signature`, -:func:`inspect.Signature.from_callable`, and :func:`inspect.Signature.from_function` +:func:`inspect.Signature.from_callable`, and :func:`!inspect.Signature.from_function` now call :func:`inspect.get_annotations` to retrieve annotations. This means :func:`inspect.signature` and :func:`inspect.Signature.from_callable` can also now un-stringize stringized annotations. @@ -1233,7 +1233,7 @@ also now un-stringize stringized annotations. itertools --------- -Add :func:`itertools.pairwise()`. +Add :func:`itertools.pairwise`. (Contributed by Raymond Hettinger in :issue:`38200`.) linecache @@ -1245,14 +1245,14 @@ When a module does not define ``__loader__``, fall back to ``__spec__.loader``. os -- -Add :func:`os.cpu_count()` support for VxWorks RTOS. +Add :func:`os.cpu_count` support for VxWorks RTOS. (Contributed by Peixing Xin in :issue:`41440`.) Add a new function :func:`os.eventfd` and related helpers to wrap the ``eventfd2`` syscall on Linux. (Contributed by Christian Heimes in :issue:`41001`.) -Add :func:`os.splice()` that allows to move data between two file +Add :func:`os.splice` that allows to move data between two file descriptors without copying between kernel address space and user address space, where one of the file descriptors must refer to a pipe. (Contributed by Pablo Galindo in :issue:`41625`.) @@ -1292,7 +1292,7 @@ functions in the :mod:`os` module. platform -------- -Add :func:`platform.freedesktop_os_release()` to retrieve operation system +Add :func:`platform.freedesktop_os_release` to retrieve operation system identification from `freedesktop.org os-release `_ standard file. (Contributed by Christian Heimes in :issue:`28468`.) @@ -1484,9 +1484,9 @@ is a :class:`typing.TypedDict`. Subclasses of ``typing.Protocol`` which only have data variables declared will now raise a ``TypeError`` when checked with ``isinstance`` unless they -are decorated with :func:`runtime_checkable`. Previously, these checks +are decorated with :func:`~typing.runtime_checkable`. Previously, these checks passed silently. Users should decorate their -subclasses with the :func:`runtime_checkable` decorator +subclasses with the :func:`!runtime_checkable` decorator if they want runtime protocols. (Contributed by Yurii Karabas in :issue:`38908`.) @@ -1517,6 +1517,13 @@ functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) + xml --- @@ -1588,8 +1595,8 @@ Optimizations :func:`map`, :func:`filter`, :func:`reversed`, :func:`bool` and :func:`float`. (Contributed by Donghee Na and Jeroen Demeyer in :issue:`43575`, :issue:`43287`, :issue:`41922`, :issue:`41873` and :issue:`41870`.) -* :class:`BZ2File` performance is improved by removing internal ``RLock``. - This makes :class:`BZ2File` thread unsafe in the face of multiple simultaneous +* :class:`~bz2.BZ2File` performance is improved by removing internal ``RLock``. + This makes :class:`!BZ2File` thread unsafe in the face of multiple simultaneous readers or writers, just like its equivalent classes in :mod:`gzip` and :mod:`lzma` have always been. (Contributed by Inada Naoki in :issue:`43785`.) @@ -1613,7 +1620,7 @@ Deprecated cleaning up old import semantics that were kept for Python 2.7 compatibility. Specifically, :meth:`!find_loader`/:meth:`!find_module` - (superseded by :meth:`~importlib.abc.Finder.find_spec`), + (superseded by :meth:`~importlib.abc.MetaPathFinder.find_spec`), :meth:`~importlib.abc.Loader.load_module` (superseded by :meth:`~importlib.abc.Loader.exec_module`), :meth:`!module_repr` (which the import system @@ -1640,7 +1647,7 @@ Deprecated :meth:`~importlib.abc.Loader.exec_module` instead. (Contributed by Brett Cannon in :issue:`26131`.) -* :meth:`zimport.zipimporter.load_module` has been deprecated in +* :meth:`!zimport.zipimporter.load_module` has been deprecated in preference for :meth:`~zipimport.zipimporter.exec_module`. (Contributed by Brett Cannon in :issue:`26131`.) @@ -1752,23 +1759,23 @@ Deprecated * The following :mod:`ssl` features have been deprecated since Python 3.6, Python 3.7, or OpenSSL 1.1.0 and will be removed in 3.11: - * :data:`~ssl.OP_NO_SSLv2`, :data:`~ssl.OP_NO_SSLv3`, :data:`~ssl.OP_NO_TLSv1`, - :data:`~ssl.OP_NO_TLSv1_1`, :data:`~ssl.OP_NO_TLSv1_2`, and - :data:`~ssl.OP_NO_TLSv1_3` are replaced by - :attr:`sslSSLContext.minimum_version` and - :attr:`sslSSLContext.maximum_version`. + * :data:`!OP_NO_SSLv2`, :data:`!OP_NO_SSLv3`, :data:`!OP_NO_TLSv1`, + :data:`!OP_NO_TLSv1_1`, :data:`!OP_NO_TLSv1_2`, and + :data:`!OP_NO_TLSv1_3` are replaced by + :attr:`~ssl.SSLContext.minimum_version` and + :attr:`~ssl.SSLContext.maximum_version`. - * :data:`~ssl.PROTOCOL_SSLv2`, :data:`~ssl.PROTOCOL_SSLv3`, - :data:`~ssl.PROTOCOL_SSLv23`, :data:`~ssl.PROTOCOL_TLSv1`, - :data:`~ssl.PROTOCOL_TLSv1_1`, :data:`~ssl.PROTOCOL_TLSv1_2`, and - :const:`~ssl.PROTOCOL_TLS` are deprecated in favor of + * :data:`!PROTOCOL_SSLv2`, :data:`!PROTOCOL_SSLv3`, + :data:`!PROTOCOL_SSLv23`, :data:`!PROTOCOL_TLSv1`, + :data:`!PROTOCOL_TLSv1_1`, :data:`!PROTOCOL_TLSv1_2`, and + :const:`!PROTOCOL_TLS` are deprecated in favor of :const:`~ssl.PROTOCOL_TLS_CLIENT` and :const:`~ssl.PROTOCOL_TLS_SERVER` - * :func:`~ssl.wrap_socket` is replaced by :meth:`ssl.SSLContext.wrap_socket` + * :func:`!wrap_socket` is replaced by :meth:`ssl.SSLContext.wrap_socket` - * :func:`~ssl.match_hostname` + * :func:`!match_hostname` - * :func:`~ssl.RAND_pseudo_bytes`, :func:`~ssl.RAND_egd` + * :func:`!RAND_pseudo_bytes`, :func:`!RAND_egd` * NPN features like :meth:`ssl.SSLSocket.selected_npn_protocol` and :meth:`ssl.SSLContext.set_npn_protocols` are replaced by ALPN. @@ -2315,3 +2322,42 @@ Removed * The ``PyThreadState.use_tracing`` member has been removed to optimize Python. (Contributed by Mark Shannon in :issue:`43760`.) + + +Notable security feature in 3.10.7 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation ` documentation. The default limit +is 4300 digits in string form. + +Notable security feature in 3.10.8 +================================== + +The deprecated :mod:`!mailcap` module now refuses to inject unsafe text +(filenames, MIME types, parameters) into shell commands. Instead of using such +text, it will warn and act as if a match was not found (or for test commands, +as if the test failed). +(Contributed by Petr Viktorin in :gh:`98966`.) + +Notable changes in 3.10.12 +========================== + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 4f4c1de8d8d596..e5c6d7cd308504 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -544,8 +544,7 @@ Other CPython Implementation Changes (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) now raises a :exc:`ValueError` if the number of digits in string form is above a limit to avoid potential denial of service attacks due to the - algorithmic complexity. This is a mitigation for `CVE-2020-10735 - `_. + algorithmic complexity. This is a mitigation for :cve:`2020-10735`. This limit can be configured or disabled by environment variable, command line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion length limitation ` documentation. The default limit @@ -769,6 +768,21 @@ functools (Contributed by Yurii Karabas in :issue:`46014`.) +.. _whatsnew311-gzip: + +gzip +---- + +* The :func:`gzip.compress` function is now faster when used with the + **mtime=0** argument as it delegates the compression entirely to a single + :func:`zlib.compress` operation. There is one side effect of this change: The + gzip file header contains an "OS" byte in its header. That was traditionally + always set to a value of 255 representing "unknown" by the :mod:`gzip` + module. Now, when using :func:`~gzip.compress` with **mtime=0**, it may be + set to a different value by the underlying zlib C library Python was linked + against. + (See :gh:`112346` for details on the side effect.) + .. _whatsnew311-hashlib: hashlib @@ -2018,8 +2032,8 @@ Removed C APIs are :ref:`listed separately `. It was introduced in Python 3.4 but has been broken since Python 3.7. (Contributed by Inada Naoki in :issue:`23882`.) -* Removed the undocumented private :meth:`!float.__set_format__()` method, - previously known as :meth:`!float.__setformat__()` in Python 3.7. +* Removed the undocumented private :meth:`!float.__set_format__` method, + previously known as :meth:`!float.__setformat__` in Python 3.7. Its docstring said: "You probably don't want to use this function. It exists mainly to be used in Python's test suite." (Contributed by Victor Stinner in :issue:`46852`.) @@ -2028,7 +2042,7 @@ Removed C APIs are :ref:`listed separately `. (and corresponding :c:macro:`!EXPERIMENTAL_ISOLATED_SUBINTERPRETERS` macro) have been removed. -* `Pynche `_ +* :pypi:`Pynche` --- The Pythonically Natural Color and Hue Editor --- has been moved out of ``Tools/scripts`` and is `being developed independently `_ from the Python source tree. @@ -2124,7 +2138,7 @@ Build Changes :issue:`45440` and :issue:`46640`.) * Support for `IEEE 754 `_ - floating point numbers. + floating-point numbers. (Contributed by Victor Stinner in :issue:`46917`.) * The :c:macro:`!Py_NO_NAN` macro has been removed. @@ -2154,7 +2168,7 @@ Build Changes (Contributed by Donghee Na and Brett Holman in :issue:`44340`.) * Freelists for object structs can now be disabled. A new :program:`configure` - option :option:`--without-freelists` can be used to disable all freelists + option ``--without-freelists`` can be used to disable all freelists except empty tuple singleton. (Contributed by Christian Heimes in :issue:`45522`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 100312a5940b79..3640095acbaa2b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -59,7 +59,7 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.12. Brevity is key. -Python 3.12 is the latest stable release of the Python programming language, +Python 3.12 is a stable release of the Python programming language, with a mix of changes to the language and the standard library. The library changes focus on cleaning up deprecated APIs, usability, and correctness. Of note, the :mod:`!distutils` package has been removed from the standard library. @@ -154,7 +154,7 @@ Important deprecations, removals or restrictions: reducing the size of every :class:`str` object by at least 8 bytes. * :pep:`632`: Remove the :mod:`!distutils` package. - See `the migration guide `_ + See :pep:`the migration guide <0632#migration-advice>` for advice replacing the APIs it provided. The third-party `Setuptools `__ package continues to provide :mod:`!distutils`, @@ -359,7 +359,7 @@ create an interpreter with its own GIL: /* The new interpreter is now active in the current thread. */ For further examples how to use the C-API for sub-interpreters with a -per-interpreter GIL, see :source:`Modules/_xxsubinterpretersmodule.c`. +per-interpreter GIL, see ``Modules/_xxsubinterpretersmodule.c``. (Contributed by Eric Snow in :gh:`104210`, etc.) @@ -652,14 +652,14 @@ asyncio making some use-cases 2x to 5x faster. (Contributed by Jacob Bower & Itamar Oren in :gh:`102853`, :gh:`104140`, and :gh:`104138`) -* On Linux, :mod:`asyncio` uses :class:`asyncio.PidfdChildWatcher` by default +* On Linux, :mod:`asyncio` uses :class:`!asyncio.PidfdChildWatcher` by default if :func:`os.pidfd_open` is available and functional instead of - :class:`asyncio.ThreadedChildWatcher`. + :class:`!asyncio.ThreadedChildWatcher`. (Contributed by Kumar Aditya in :gh:`98024`.) * The event loop now uses the best available child watcher for each platform - (:class:`asyncio.PidfdChildWatcher` if supported and - :class:`asyncio.ThreadedChildWatcher` otherwise), so manually + (:class:`!asyncio.PidfdChildWatcher` if supported and + :class:`!asyncio.ThreadedChildWatcher` otherwise), so manually configuring a child watcher is not recommended. (Contributed by Kumar Aditya in :gh:`94597`.) @@ -726,7 +726,7 @@ inspect * Add :func:`inspect.markcoroutinefunction` to mark sync functions that return a :term:`coroutine` for use with :func:`inspect.iscoroutinefunction`. - (Contributed Carlton Gibson in :gh:`99247`.) + (Contributed by Carlton Gibson in :gh:`99247`.) * Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals` for determining the current state of asynchronous generators. @@ -734,13 +734,12 @@ inspect * The performance of :func:`inspect.getattr_static` has been considerably improved. Most calls to the function should be at least 2x faster than they - were in Python 3.11, and some may be 6x faster or more. (Contributed by Alex - Waygood in :gh:`103193`.) + were in Python 3.11. (Contributed by Alex Waygood in :gh:`103193`.) itertools --------- -* Add :class:`itertools.batched()` for collecting into even-sized +* Add :func:`itertools.batched` for collecting into even-sized tuples where the last batch may be shorter than the rest. (Contributed by Raymond Hettinger in :gh:`98363`.) @@ -751,8 +750,8 @@ math (Contributed by Raymond Hettinger in :gh:`100485`.) * Extend :func:`math.nextafter` to include a *steps* argument - for moving up or down multiple steps at a time. - (By Matthias Goergens, Mark Dickinson, and Raymond Hettinger in :gh:`94906`.) + for moving up or down multiple steps at a time. (Contributed by + Matthias Goergens, Mark Dickinson, and Raymond Hettinger in :gh:`94906`.) os -- @@ -927,8 +926,6 @@ tempfile * :func:`tempfile.mkdtemp` now always returns an absolute path, even if the argument provided to the *dir* parameter is a relative path. -.. _whatsnew-typing-py312: - threading --------- @@ -963,6 +960,8 @@ types :ref:`user-defined-generics` when subclassed. (Contributed by James Hilton-Balfe and Alex Waygood in :gh:`101827`.) +.. _whatsnew-typing-py312: + typing ------ @@ -1006,8 +1005,8 @@ typing :func:`runtime-checkable protocols ` has changed significantly. Most ``isinstance()`` checks against protocols with only a few members should be at least 2x faster than in 3.11, and some may be 20x - faster or more. However, ``isinstance()`` checks against protocols with fourteen - or more members may be slower than in Python 3.11. (Contributed by Alex + faster or more. However, ``isinstance()`` checks against protocols with many + members may be slower than in Python 3.11. (Contributed by Alex Waygood in :gh:`74690` and :gh:`103193`.) * All :data:`typing.TypedDict` and :data:`typing.NamedTuple` classes now have the @@ -1124,7 +1123,7 @@ CPython bytecode changes * Add the :opcode:`LOAD_SUPER_ATTR` instruction. (Contributed by Carl Meyer and Vladimir Matveev in :gh:`103497`.) -* Add the :opcode:`RETURN_CONST` instruction. (Contributed by Wenyang Wang in :gh:`101632`.) +* Add the ``RETURN_CONST`` instruction. (Contributed by Wenyang Wang in :gh:`101632`.) Demos and Tools =============== @@ -1163,15 +1162,15 @@ Deprecated * :mod:`asyncio`: - * The child watcher classes :class:`asyncio.MultiLoopChildWatcher`, - :class:`asyncio.FastChildWatcher`, :class:`asyncio.AbstractChildWatcher` - and :class:`asyncio.SafeChildWatcher` are deprecated and + * The child watcher classes :class:`!asyncio.MultiLoopChildWatcher`, + :class:`!asyncio.FastChildWatcher`, :class:`!asyncio.AbstractChildWatcher` + and :class:`!asyncio.SafeChildWatcher` are deprecated and will be removed in Python 3.14. (Contributed by Kumar Aditya in :gh:`94597`.) - * :func:`asyncio.set_child_watcher`, :func:`asyncio.get_child_watcher`, - :meth:`asyncio.AbstractEventLoopPolicy.set_child_watcher` and - :meth:`asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + * :func:`!asyncio.set_child_watcher`, :func:`!asyncio.get_child_watcher`, + :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated and will be removed in Python 3.14. (Contributed by Kumar Aditya in :gh:`94597`.) @@ -1184,7 +1183,7 @@ Deprecated replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) -* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. (Contributed by Shantanu Jain in :gh:`91896`.) @@ -1230,7 +1229,7 @@ Deprecated your code *requires* ``'fork'``. See :ref:`contexts and start methods `. -* :mod:`pkgutil`: :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` are deprecated and will be removed in Python 3.14; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -1252,10 +1251,10 @@ Deprecated :exc:`DeprecationWarning` when it can detect being called from a multithreaded process. There has always been a fundamental incompatibility with the POSIX platform when doing so. Even if such code *appeared* to work. - We added the warning to to raise awareness as issues encounted by code doing + We added the warning to raise awareness as issues encountered by code doing this are becoming more frequent. See the :func:`os.fork` documentation for more details along with `this discussion on fork being incompatible with threads - `_ for *why* we're now surfacing this + `_ for *why* we're now surfacing this longstanding platform compatibility problem to developers. When this warning appears due to usage of :mod:`multiprocessing` or @@ -1294,7 +1293,7 @@ Deprecated :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are deprecated. (:gh:`94309`.) - * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a + * :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. (Contributed by Alex Waygood in :gh:`91896`.) @@ -1310,17 +1309,18 @@ Deprecated may be removed in a future version of Python. Use the single-arg versions of these functions instead. (Contributed by Ofey Chan in :gh:`89874`.) -* :exc:`DeprecationWarning` is now raised when ``__package__`` on a - module differs from ``__spec__.parent`` (previously it was - :exc:`ImportWarning`). +* :exc:`DeprecationWarning` is now raised when :attr:`~module.__package__` on a + module differs from + :attr:`__spec__.parent ` (previously + it was :exc:`ImportWarning`). (Contributed by Brett Cannon in :gh:`65961`.) -* Setting ``__package__`` or ``__cached__`` on a module is deprecated, - and will cease to be set or taken into consideration by the import system in Python 3.14. - (Contributed by Brett Cannon in :gh:`65961`.) +* Setting :attr:`~module.__package__` or :attr:`~module.__cached__` on a + module is deprecated, and will cease to be set or taken into consideration by + the import system in Python 3.14. (Contributed by Brett Cannon in :gh:`65961`.) * The bitwise inversion operator (``~``) on bool is deprecated. It will throw an - error in Python 3.14. Use ``not`` for logical negation of bools instead. + error in Python 3.16. Use ``not`` for logical negation of bools instead. In the rare case that you really need the bitwise inversion of the underlying ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) @@ -1331,152 +1331,15 @@ Deprecated therefore it will be removed in 3.14. (Contributed by Nikita Sobolev in :gh:`101866`.) -Pending Removal in Python 3.13 ------------------------------- - -The following modules and APIs have been deprecated in earlier Python releases, -and will be removed in Python 3.13. - -Modules (see :pep:`594`): - -* :mod:`!aifc` -* :mod:`!audioop` -* :mod:`!cgi` -* :mod:`!cgitb` -* :mod:`!chunk` -* :mod:`!crypt` -* :mod:`!imghdr` -* :mod:`!mailcap` -* :mod:`!msilib` -* :mod:`!nis` -* :mod:`!nntplib` -* :mod:`!ossaudiodev` -* :mod:`!pipes` -* :mod:`!sndhdr` -* :mod:`!spwd` -* :mod:`!sunau` -* :mod:`!telnetlib` -* :mod:`!uu` -* :mod:`!xdrlib` - -Other modules: - -* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) - -APIs: - -* :class:`!configparser.LegacyInterpolation` (:gh:`90765`) -* ``locale.resetlocale()`` (:gh:`90817`) -* :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) -* :func:`!unittest.findTestCases` (:gh:`50096`) -* :func:`!unittest.getTestCaseNames` (:gh:`50096`) -* :func:`!unittest.makeSuite` (:gh:`50096`) -* :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) -* :class:`!webbrowser.MacOSX` (:gh:`86421`) -* :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) - -Pending Removal in Python 3.14 ------------------------------- - -The following APIs have been deprecated -and will be removed in Python 3.14. - -* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction` - -* :mod:`ast`: - - * :class:`!ast.Num` - * :class:`!ast.Str` - * :class:`!ast.Bytes` - * :class:`!ast.NameConstant` - * :class:`!ast.Ellipsis` - -* :mod:`asyncio`: - - * :class:`!asyncio.MultiLoopChildWatcher` - * :class:`!asyncio.FastChildWatcher` - * :class:`!asyncio.AbstractChildWatcher` - * :class:`!asyncio.SafeChildWatcher` - * :func:`!asyncio.set_child_watcher` - * :func:`!asyncio.get_child_watcher`, - * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` - -* :mod:`collections.abc`: :class:`!collections.abc.ByteString`. - -* :mod:`email`: the *isdst* parameter in :func:`email.utils.localtime`. - -* :mod:`importlib.abc`: - - * :class:`!importlib.abc.ResourceReader` - * :class:`!importlib.abc.Traversable` - * :class:`!importlib.abc.TraversableResources` - -* :mod:`itertools`: Support for copy, deepcopy, and pickle operations. +.. include:: ../deprecations/pending-removal-in-3.13.rst -* :mod:`pkgutil`: +.. include:: ../deprecations/pending-removal-in-3.14.rst - * :func:`!pkgutil.find_loader` - * :func:`!pkgutil.get_loader`. - -* :mod:`pty`: - - * :func:`!pty.master_open` - * :func:`!pty.slave_open` - -* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` - -* :mod:`typing`: :class:`!typing.ByteString` - -* :mod:`xml.etree.ElementTree`: Testing the truth value of an :class:`xml.etree.ElementTree.Element`. - -* The ``__package__`` and ``__cached__`` attributes on module objects. - -* The :attr:`~codeobject.co_lnotab` attribute of code objects. - -Pending Removal in Python 3.15 ------------------------------- +.. include:: ../deprecations/pending-removal-in-3.15.rst -The following APIs have been deprecated -and will be removed in Python 3.15. - -APIs: - -* :func:`locale.getdefaultlocale` (:gh:`90817`) - - -Pending Removal in Future Versions ----------------------------------- - -The following APIs were deprecated in earlier Python versions and will be removed, -although there is currently no date scheduled for their removal. - -* :mod:`array`'s ``'u'`` format code (:gh:`57281`) - -* :class:`typing.Text` (:gh:`92332`) - -* Currently Python accepts numeric literals immediately followed by keywords, - for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing - and ambiguous expressions like ``[0x1for x in y]`` (which can be - interpreted as ``[0x1 for x in y]`` or ``[0x1f or x in y]``). - A syntax warning is raised if the numeric literal is - immediately followed by one of keywords :keyword:`and`, :keyword:`else`, - :keyword:`for`, :keyword:`if`, :keyword:`in`, :keyword:`is` and :keyword:`or`. - In a future release it will be changed to a syntax error. (:gh:`87999`) +.. include:: ../deprecations/pending-removal-in-3.16.rst +.. include:: ../deprecations/pending-removal-in-future.rst Removed ======= @@ -1560,9 +1423,9 @@ hashlib ------- * Remove the pure Python implementation of :mod:`hashlib`'s - :func:`hashlib.pbkdf2_hmac()`, deprecated in Python 3.10. Python 3.10 and + :func:`hashlib.pbkdf2_hmac`, deprecated in Python 3.10. Python 3.10 and newer requires OpenSSL 1.1.1 (:pep:`644`): this OpenSSL version provides - a C implementation of :func:`~hashlib.pbkdf2_hmac()` which is faster. + a C implementation of :func:`~hashlib.pbkdf2_hmac` which is faster. (Contributed by Victor Stinner in :gh:`94199`.) importlib @@ -1571,7 +1434,7 @@ importlib * Many previously deprecated cleanups in :mod:`importlib` have now been completed: - * References to, and support for :meth:`!module_repr()` has been removed. + * References to, and support for :meth:`!module_repr` has been removed. (Contributed by Barry Warsaw in :gh:`97850`.) * ``importlib.util.set_package``, ``importlib.util.set_loader`` and @@ -1659,12 +1522,10 @@ smtpd * The ``smtpd`` module has been removed according to the schedule in :pep:`594`, having been deprecated in Python 3.4.7 and 3.5.4. - Use aiosmtpd_ PyPI module or any other + Use the :pypi:`aiosmtpd` PyPI module or any other :mod:`asyncio`-based server instead. (Contributed by Oleg Iarygin in :gh:`93243`.) -.. _aiosmtpd: https://pypi.org/project/aiosmtpd/ - sqlite3 ------- @@ -1701,9 +1562,8 @@ ssl instead, create a :class:`ssl.SSLContext` object and call its :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses :func:`!ssl.wrap_socket` is broken and insecure. The function neither sends a - SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 - `_: Improper Certificate - Validation. + SNI TLS extension nor validates the server hostname. Code is subject to :cwe:`295` + (Improper Certificate Validation). (Contributed by Victor Stinner in :gh:`94199`.) unittest @@ -1743,7 +1603,7 @@ unittest * Undocumented :meth:`TestLoader.loadTestsFromModule ` parameter *use_load_tests* - (deprecated and ignored since Python 3.2). + (deprecated and ignored since Python 3.5). * An alias of the :class:`~unittest.TextTestResult` class: ``_TextTestResult`` (deprecated in Python 3.2). @@ -1832,7 +1692,7 @@ Changes in the Python API * Remove the ``asyncore``-based ``smtpd`` module deprecated in Python 3.4.7 and 3.5.4. A recommended replacement is the - :mod:`asyncio`-based aiosmtpd_ PyPI module. + :mod:`asyncio`-based :pypi:`aiosmtpd` PyPI module. * :func:`shlex.split`: Passing ``None`` for *s* argument now raises an exception, rather than reading :data:`sys.stdin`. The feature was deprecated @@ -1956,6 +1816,8 @@ Build Changes :file:`!configure`. (Contributed by Christian Heimes in :gh:`89886`.) +* Windows builds and macOS installers from python.org now use OpenSSL 3.0. + C API Changes ============= @@ -2026,7 +1888,7 @@ New Features The :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` flags have been added. This allows extensions classes to support object - ``__dict__`` and weakrefs with less bookkeeping, + :attr:`~object.__dict__` and weakrefs with less bookkeeping, using less memory and with faster access. * API for performing calls using @@ -2145,7 +2007,7 @@ Porting to Python 3.12 internal-only field directly. To get a list of subclasses, call the Python method - :py:meth:`~class.__subclasses__` (using :c:func:`PyObject_CallMethod`, + :py:meth:`~type.__subclasses__` (using :c:func:`PyObject_CallMethod`, for example). * Add support of more formatting options (left aligning, octals, uppercase @@ -2164,7 +2026,7 @@ Porting to Python 3.12 :c:func:`PyUnicode_FromFormatV`. (Contributed by Philip Georgi in :gh:`95504`.) -* Extension classes wanting to add a ``__dict__`` or weak reference slot +* Extension classes wanting to add a :attr:`~object.__dict__` or weak reference slot should use :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and ``tp_weaklistoffset``, respectively. @@ -2349,92 +2211,13 @@ Deprecated overrides :c:member:`~PyTypeObject.tp_new` is deprecated. Call the metaclass instead. -Pending Removal in Python 3.14 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. Add deprecations above alphabetically, not here at the end. -* The ``ma_version_tag`` field in :c:type:`PyDictObject` for extension modules - (:pep:`699`; :gh:`101193`). +.. include:: ../deprecations/c-api-pending-removal-in-3.14.rst -* Global configuration variables: +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst - * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` - * :c:var:`Py_VerboseFlag`: use :c:member:`PyConfig.verbose` - * :c:var:`Py_QuietFlag`: use :c:member:`PyConfig.quiet` - * :c:var:`Py_InteractiveFlag`: use :c:member:`PyConfig.interactive` - * :c:var:`Py_InspectFlag`: use :c:member:`PyConfig.inspect` - * :c:var:`Py_OptimizeFlag`: use :c:member:`PyConfig.optimization_level` - * :c:var:`Py_NoSiteFlag`: use :c:member:`PyConfig.site_import` - * :c:var:`Py_BytesWarningFlag`: use :c:member:`PyConfig.bytes_warning` - * :c:var:`Py_FrozenFlag`: use :c:member:`PyConfig.pathconfig_warnings` - * :c:var:`Py_IgnoreEnvironmentFlag`: use :c:member:`PyConfig.use_environment` - * :c:var:`Py_DontWriteBytecodeFlag`: use :c:member:`PyConfig.write_bytecode` - * :c:var:`Py_NoUserSiteDirectory`: use :c:member:`PyConfig.user_site_directory` - * :c:var:`Py_UnbufferedStdioFlag`: use :c:member:`PyConfig.buffered_stdio` - * :c:var:`Py_HashRandomizationFlag`: use :c:member:`PyConfig.use_hash_seed` - and :c:member:`PyConfig.hash_seed` - * :c:var:`Py_IsolatedFlag`: use :c:member:`PyConfig.isolated` - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` - * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` - * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` - * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` - * :c:var:`!Py_UTF8Mode`: use :c:member:`PyPreConfig.utf8_mode` (see :c:func:`Py_PreInitialize`) - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. - -* Creating :c:data:`immutable types ` with mutable - bases (:gh:`95388`). - -Pending Removal in Python 3.15 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule` -* :c:type:`!Py_UNICODE_WIDE` type: use :c:type:`wchar_t` -* :c:type:`Py_UNICODE` type: use :c:type:`wchar_t` -* Python initialization functions: - - * :c:func:`PySys_ResetWarnOptions`: clear :data:`sys.warnoptions` and - :data:`!warnings.filters` - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` - * :c:func:`Py_GetPath`: get :data:`sys.path` - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - the :envvar:`PYTHONHOME` environment variable - -Pending Removal in Future Versions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following APIs are deprecated and will be removed, -although there is currently no date scheduled for their removal. - -* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: unneeded since Python 3.8 -* :c:func:`PyErr_Fetch`: use :c:func:`PyErr_GetRaisedException` -* :c:func:`PyErr_NormalizeException`: use :c:func:`PyErr_GetRaisedException` -* :c:func:`PyErr_Restore`: use :c:func:`PyErr_SetRaisedException` -* :c:func:`PyModule_GetFilename`: use :c:func:`PyModule_GetFilenameObject` -* :c:func:`PyOS_AfterFork`: use :c:func:`PyOS_AfterFork_Child` -* :c:func:`PySlice_GetIndicesEx`: use :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` -* :c:func:`!PyUnicode_AsDecodedObject`: use :c:func:`PyCodec_Decode` -* :c:func:`!PyUnicode_AsDecodedUnicode`: use :c:func:`PyCodec_Decode` -* :c:func:`!PyUnicode_AsEncodedObject`: use :c:func:`PyCodec_Encode` -* :c:func:`!PyUnicode_AsEncodedUnicode`: use :c:func:`PyCodec_Encode` -* :c:func:`PyUnicode_READY`: unneeded since Python 3.12 -* :c:func:`!PyErr_Display`: use :c:func:`PyErr_DisplayException` -* :c:func:`!_PyErr_ChainExceptions`: use ``_PyErr_ChainExceptions1`` -* :c:member:`!PyBytesObject.ob_shash` member: - call :c:func:`PyObject_Hash` instead -* :c:member:`!PyDictObject.ma_version_tag` member -* Thread Local Storage (TLS) API: - - * :c:func:`PyThread_create_key`: use :c:func:`PyThread_tss_alloc` - * :c:func:`PyThread_delete_key`: use :c:func:`PyThread_tss_free` - * :c:func:`PyThread_set_key_value`: use :c:func:`PyThread_tss_set` - * :c:func:`PyThread_get_key_value`: use :c:func:`PyThread_tss_get` - * :c:func:`PyThread_delete_key_value`: use :c:func:`PyThread_tss_delete` - * :c:func:`PyThread_ReInitTLS`: unneeded since Python 3.7 +.. include:: ../deprecations/c-api-pending-removal-in-future.rst Removed ------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index de79bd979aff80..de4c7fd4c0486b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -3,7 +3,7 @@ What's New In Python 3.13 **************************** -:Editor: TBD +:Editors: Adam Turner and Thomas Wouters .. Rules for maintenance: @@ -46,408 +46,1253 @@ when researching a change. This article explains the new features in Python 3.13, compared to 3.12. - +Python 3.13 was released on October 7, 2024. For full details, see the :ref:`changelog `. -.. note:: +.. seealso:: - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.13 moves towards release, - so it's worth checking back even after reading earlier versions. + :pep:`719` -- Python 3.13 Release Schedule -Summary -- Release highlights +Summary -- Release Highlights ============================= .. This section singles out the most important changes in Python 3.13. Brevity is key. +Python 3.13 is the latest stable release of the Python programming +language, with a mix of changes to the language, the implementation +and the standard library. +The biggest changes include a new `interactive interpreter +`_, +experimental support for running in a `free-threaded mode +`_ (:pep:`703`), +and a `Just-In-Time compiler `_ (:pep:`744`). + +Error messages continue to improve, with tracebacks now highlighted in color +by default. The :func:`locals` builtin now has :ref:`defined semantics +` for changing the returned mapping, +and type parameters now support default values. + +The library changes contain removal of deprecated APIs and modules, +as well as the usual improvements in user-friendliness and correctness. +Several legacy standard library modules have now `been removed +`_ following their deprecation in Python 3.11 (:pep:`594`). + +This article doesn't attempt to provide a complete specification +of all new features, but instead gives a convenient overview. +For full details refer to the documentation, +such as the :ref:`Library Reference ` +and :ref:`Language Reference `. +To understand the complete implementation and design rationale for a change, +refer to the PEP for a particular new feature; +but note that PEPs usually are not kept up-to-date +once a feature has been fully implemented. +See `Porting to Python 3.13`_ for guidance on upgrading from +earlier versions of Python. + +-------------- .. PEP-sized items next. -Important deprecations, removals or restrictions: +Interpreter improvements: + +* A greatly improved :ref:`interactive interpreter + ` and + :ref:`improved error messages `. +* :pep:`667`: The :func:`locals` builtin now has + :ref:`defined semantics ` when mutating the + returned mapping. Python debuggers and similar tools may now more reliably + update local variables in optimized scopes even during concurrent code + execution. +* :pep:`703`: CPython 3.13 has experimental support for running with the + :term:`global interpreter lock` disabled. See :ref:`Free-threaded CPython + ` for more details. +* :pep:`744`: A basic :ref:`JIT compiler ` was added. + It is currently disabled by default (though we may turn it on later). + Performance improvements are modest -- we expect to improve this + over the next few releases. +* Color support in the new :ref:`interactive interpreter + `, + as well as in :ref:`tracebacks ` + and :ref:`doctest ` output. + This can be disabled through the :envvar:`PYTHON_COLORS` and |NO_COLOR|_ + environment variables. + +Python data model improvements: + +* :attr:`~type.__static_attributes__` stores the names of attributes accessed + through ``self.X`` in any function in a class body. +* :attr:`~type.__firstlineno__` records the first line number of a class + definition. + +Significant improvements in the standard library: + +* Add a new :exc:`PythonFinalizationError` exception, raised when an operation + is blocked during :term:`finalization `. +* The :mod:`argparse` module now supports deprecating command-line options, + positional arguments, and subcommands. +* The new functions :func:`base64.z85encode` and :func:`base64.z85decode` + support encoding and decoding `Z85 data`_. +* The :mod:`copy` module now has a :func:`copy.replace` function, + with support for many builtin types and any class defining + the :func:`~object.__replace__` method. +* The new :mod:`dbm.sqlite3` module is now the default :mod:`dbm` backend. +* The :mod:`os` module has a :ref:`suite of new functions ` + for working with Linux's timer notification file descriptors. +* The :mod:`random` module now has a :ref:`command-line interface `. + +Security improvements: + +* :func:`ssl.create_default_context` sets :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` + and :data:`ssl.VERIFY_X509_STRICT` as default flags. + +C API improvements: + +* The :c:data:`Py_mod_gil` slot is now used to indicate that + an extension module supports running with the :term:`GIL` disabled. +* The :doc:`PyTime C API ` has been added, + providing access to system clocks. +* :c:type:`PyMutex` is a new lightweight mutex that occupies a single byte. +* There is a new :ref:`suite of functions ` + for generating :pep:`669` monitoring events in the C API. + +New typing features: + +* :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`, + and :data:`typing.TypeVarTuple`) now support defaults. +* :pep:`702`: The new :func:`warnings.deprecated` decorator adds support + for marking deprecations in the type system and at runtime. +* :pep:`705`: :data:`typing.ReadOnly` can be used to mark an item of a + :class:`typing.TypedDict` as read-only for type checkers. +* :pep:`742`: :data:`typing.TypeIs` provides more intuitive + type narrowing behavior, as an alternative to :data:`typing.TypeGuard`. + +Platform support: + +* :pep:`730`: Apple's iOS is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. +* :pep:`738`: Android is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. +* ``wasm32-wasi`` is now supported as a :pep:`tier 2 <11#tier-2>` platform. +* ``wasm32-emscripten`` is no longer an officially supported platform. + +Important removals: * :ref:`PEP 594 `: The remaining 19 "dead batteries" - have been removed from the standard library: + (legacy stdlib modules) have been removed from the standard library: :mod:`!aifc`, :mod:`!audioop`, :mod:`!cgi`, :mod:`!cgitb`, :mod:`!chunk`, :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, - :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, - :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. - -* :pep:`602` ("Annual Release Cycle for Python") has been updated: + :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, + :mod:`!spwd`, :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. +* Remove the :program:`2to3` tool and :mod:`!lib2to3` module + (deprecated in Python 3.11). +* Remove the :mod:`!tkinter.tix` module (deprecated in Python 3.6). +* Remove the :func:`!locale.resetlocale` function. +* Remove the :mod:`!typing.io` and :mod:`!typing.re` namespaces. +* Remove chained :class:`classmethod` descriptors. - * Python 3.9 - 3.12 have one and a half years of full support, - followed by three and a half years of security fixes. - * Python 3.13 and later have two years of full support, - followed by three years of security fixes. +Release schedule changes: -Interpreter improvements: +:pep:`602` ("Annual Release Cycle for Python") has been updated +to extend the full support ('bugfix') period for new releases to two years. +This updated policy means that: -* A basic :ref:`JIT compiler ` was added. - It is currently disabled by default (though we may turn it on later). - Performance improvements are modest -- we expect to be improving this - over the next few releases. +* Python 3.9--3.12 have one and a half years of full support, + followed by three and a half years of security fixes. +* Python 3.13 and later have two years of full support, + followed by three years of security fixes. New Features ============ -Improved Error Messages + +.. _whatsnew313-better-interactive-interpreter: + +A better interactive interpreter +-------------------------------- + +Python now uses a new :term:`interactive` shell by default, based on code +from the `PyPy project`_. +When the user starts the :term:`REPL` from an interactive terminal, +the following new features are now supported: + +* Multiline editing with history preservation. +* Direct support for REPL-specific commands like :kbd:`help`, :kbd:`exit`, + and :kbd:`quit`, without the need to call them as functions. +* Prompts and tracebacks with :ref:`color enabled by default + `. +* Interactive help browsing using :kbd:`F1` with a separate command + history. +* History browsing using :kbd:`F2` that skips output as well as the + :term:`>>>` and :term:`...` prompts. +* "Paste mode" with :kbd:`F3` that makes pasting larger blocks of code + easier (press :kbd:`F3` again to return to the regular prompt). + +To disable the new interactive shell, +set the :envvar:`PYTHON_BASIC_REPL` environment variable. +For more on interactive mode, see :ref:`tut-interac`. + +(Contributed by Pablo Galindo Salgado, Łukasz Langa, and +Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project. +Windows support contributed by Dino Viehland and Anthony Shaw.) + +.. _`PyPy project`: https://pypy.org/ + + +.. _whatsnew313-improved-error-messages: + +Improved error messages ----------------------- -* The interpreter now colorizes error messages when displaying tracebacks by default. - This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment - variable as well as the canonical ``NO_COLOR`` and ``FORCE_COLOR`` environment - variables. See also :ref:`using-on-controlling-color`. +* The interpreter now uses color by default when displaying tracebacks in the + terminal. This feature :ref:`can be controlled ` + via the new :envvar:`PYTHON_COLORS` environment variable as well as + the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) -* When an incorrect keyword argument is passed to a function, the error message - now potentially suggests the correct keyword argument. +.. Apparently this how you hack together a formatted link: + (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) + +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ + +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ + +* A common mistake is to write a script with the same name as a + standard library module. When this results in errors, we now + display a more helpful error message: + + .. code-block:: pytb + + $ python random.py + Traceback (most recent call last): + File "/home/me/random.py", line 1, in + import random + File "/home/me/random.py", line 3, in + print(random.randint(5)) + ^^^^^^^^^^^^^^ + AttributeError: module 'random' has no attribute 'randint' (consider renaming '/home/me/random.py' since it has the same name as the standard library module named 'random' and prevents importing that standard library module) + + Similarly, if a script has the same name as a third-party + module that it attempts to import and this results in errors, + we also display a more helpful error message: + + .. code-block:: pytb + + $ python numpy.py + Traceback (most recent call last): + File "/home/me/numpy.py", line 1, in + import numpy as np + File "/home/me/numpy.py", line 3, in + np.array([1, 2, 3]) + ^^^^^^^^ + AttributeError: module 'numpy' has no attribute 'array' (consider renaming '/home/me/numpy.py' if it has the same name as a library you intended to import) + + (Contributed by Shantanu Jain in :gh:`95754`.) + +* The error message now tries to suggest the correct keyword argument + when an incorrect keyword argument is passed to a function. + + .. code-block:: pycon + + >>> "Better error messages!".split(max_split=1) + Traceback (most recent call last): + File "", line 1, in + "Better error messages!".split(max_split=1) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ + TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? + (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) - >>> "better error messages!".split(max_split=1) - Traceback (most recent call last): - File "", line 1, in - "better error messages!".split(max_split=1) - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ - TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? + +.. _whatsnew313-free-threaded-cpython: + +Free-threaded CPython +--------------------- + +CPython now has experimental support for running in a free-threaded mode, +with the :term:`global interpreter lock` (GIL) disabled. +This is an experimental feature and therefore is not enabled by default. +The free-threaded mode requires a different executable, +usually called ``python3.13t`` or ``python3.13t.exe``. +Pre-built binaries marked as *free-threaded* can be installed as part of +the official :ref:`Windows ` +and :ref:`macOS ` installers, +or CPython can be built from source with the :option:`--disable-gil` option. + +Free-threaded execution allows for full utilization of the available +processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multi-core hardware. +**The free-threaded mode is experimental** and work is ongoing to improve it: +expect some bugs and a substantial single-threaded performance hit. +Free-threaded builds of CPython support optionally running with the GIL +enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command-line option :option:`-X gil=1`. + +To check if the current interpreter supports free-threading, :option:`python -VV <-V>` +and :attr:`sys.version` contain "experimental free-threading build". +The new :func:`!sys._is_gil_enabled` function can be used to check whether +the GIL is actually disabled in the running process. + +C-API extension modules need to be built specifically for the free-threaded +build. Extensions that support running with the :term:`GIL` disabled should +use the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should +use :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support +running with the GIL disabled. Importing C extensions that don't use these +mechanisms will cause the GIL to be enabled, unless the GIL was explicitly +disabled with the :envvar:`PYTHON_GIL` environment variable or the +:option:`-X gil=0` option. +pip 24.1 or newer is required to install packages with C extensions in the +free-threaded build. + +This work was made possible thanks to many individuals and +organizations, including the large community of contributors to Python +and third-party projects to test and enable free-threading support. +Notable contributors include: +Sam Gross, Ken Jin, Donghee Na, Itamar Oren, Matt Page, Brett Simmers, +Dino Viehland, Carl Meyer, Nathan Goldbaum, Ralf Gommers, +Lysandros Nikolaou, and many others. +Many of these contributors are employed by Meta, which has +provided significant engineering resources to support this project. + +.. seealso:: + + :pep:`703` "Making the Global Interpreter Lock Optional in CPython" + contains rationale and information surrounding this work. + + `Porting Extension Modules to Support Free-Threading + `_: A community-maintained + porting guide for extension authors. + + +.. _whatsnew313-jit-compiler: + +An experimental just-in-time (JIT) compiler +------------------------------------------- + +When CPython is configured and built using +the :option:`!--enable-experimental-jit` option, +a just-in-time (JIT) compiler is added which may speed up some Python programs. +On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT +or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter. +Build requirements and further supporting information `are contained at`__ +:file:`Tools/jit/README.md`. + +__ https://github.com/python/cpython/blob/main/Tools/jit/README.md + +The :option:`!--enable-experimental-jit` option takes these (optional) values, +defaulting to ``yes`` if :option:`!--enable-experimental-jit` is present +without the optional value. + +* ``no``: Disable the entire Tier 2 and JIT pipeline. +* ``yes``: Enable the JIT. + To disable the JIT at runtime, pass the environment variable ``PYTHON_JIT=0``. +* ``yes-off``: Build the JIT but disable it by default. + To enable the JIT at runtime, pass the environment variable ``PYTHON_JIT=1``. +* ``interpreter``: Enable the Tier 2 interpreter but disable the JIT. + The interpreter can be disabled by running with ``PYTHON_JIT=0``. + +The internal architecture is roughly as follows: + +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 ` for details. +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new purely internal intermediate representation (IR), + called the *Tier 2 IR*, and sometimes referred to as micro-ops ("uops"). +* The Tier 2 IR uses the same stack-based virtual machine as Tier 1, + but the instruction format is better suited to translation to machine code. +* We have several optimization passes for Tier 2 IR, which are applied + before it is interpreted or translated to machine code. +* There is a Tier 2 interpreter, but it is mostly intended for debugging + the earlier stages of the optimization pipeline. + The Tier 2 interpreter can be enabled by configuring Python + with ``--enable-experimental-jit=interpreter``. +* When the JIT is enabled, the optimized + Tier 2 IR is translated to machine code, which is then executed. +* The machine code translation process uses a technique called + *copy-and-patch*. It has no runtime dependencies, but there is a new + build-time dependency on LLVM. + +.. seealso:: :pep:`744` + +(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. +Tier 2 IR by Mark Shannon and Guido van Rossum. +Tier 2 optimizer by Ken Jin.) + + +.. _whatsnew313-locals-semantics: + +Defined mutation semantics for :py:func:`locals` +------------------------------------------------ + +Historically, the expected result of mutating the return value of +:func:`locals` has been left to individual Python implementations to define. +Starting from Python 3.13, :pep:`667` standardises +the historical behavior of CPython for most code execution scopes, +but changes :term:`optimized scopes ` +(functions, generators, coroutines, comprehensions, and generator expressions) +to explicitly return independent snapshots of the currently assigned local +variables, including locally referenced nonlocal variables captured in closures. + +This change to the semantics of :func:`locals` in optimized scopes also +affects the default behavior of code execution functions that implicitly +target :func:`!locals` if no explicit namespace is provided +(such as :func:`exec` and :func:`eval`). +In previous versions, whether or not changes could be accessed by calling +:func:`!locals` after calling the code execution function was +implementation-dependent. In CPython specifically, such code would typically +appear to work as desired, but could sometimes fail in optimized scopes based +on other code (including debuggers and code execution tracing tools) +potentially resetting the shared snapshot in that scope. +Now, the code will always run against an independent snapshot of +the local variables in optimized scopes, and hence the changes will never +be visible in subsequent calls to :func:`!locals`. +To access the changes made in these cases, an explicit namespace reference +must now be passed to the relevant function. +Alternatively, it may make sense to update affected code to use a higher level +code execution API that returns the resulting code execution namespace +(e.g. :func:`runpy.run_path` when executing Python files from disk). + +To ensure debuggers and similar tools can reliably update local variables in +scopes affected by this change, :attr:`FrameType.f_locals ` now +returns a write-through proxy to the frame's local and locally referenced +nonlocal variables in these scopes, rather than returning an inconsistently +updated shared ``dict`` instance with undefined runtime semantics. + +See :pep:`667` for more details, including related C API changes +and deprecations. Porting notes are also provided below for the affected +:ref:`Python APIs ` and :ref:`C APIs +`. + +(PEP and implementation contributed by Mark Shannon and Tian Gao in +:gh:`74929`. Documentation updates provided by Guido van Rossum and +Alyssa Coghlan.) + + +.. _whatsnew313-platform-support: + +Support for mobile platforms +---------------------------- + +:pep:`730`: iOS is now a :pep:`11` supported platform, with the +``arm64-apple-ios`` and ``arm64-apple-ios-simulator`` targets at tier 3 +(iPhone and iPad devices released after 2013 and the Xcode iOS simulator +running on Apple silicon hardware, respectively). +``x86_64-apple-ios-simulator`` +(the Xcode iOS simulator running on older ``x86_64`` hardware) +is not a tier 3 supported platform, but will have best-effort support. +(PEP written and implementation contributed by Russell Keith-Magee in +:gh:`114099`.) + +:pep:`738`: Android is now a :pep:`11` supported platform, with the +``aarch64-linux-android`` and ``x86_64-linux-android`` targets at tier 3. +The 32-bit targets ``arm-linux-androideabi`` and ``i686-linux-android`` +are not tier 3 supported platforms, but will have best-effort support. +(PEP written and implementation contributed by Malcolm Smith in +:gh:`116622`.) + +.. seealso:: :pep:`730`, :pep:`738` + Other Language Changes ====================== -* Allow the *count* argument of :meth:`str.replace` to be a keyword. - (Contributed by Hugo van Kemenade in :gh:`106487`.) +* The compiler now strips common leading whitespace + from every line in a docstring. + This reduces the size of the :term:`bytecode cache ` + (such as ``.pyc`` files), with reductions in file size of around 5%, + for example in :mod:`!sqlalchemy.orm.session` from SQLAlchemy 2.0. + This change affects tools that use docstrings, such as :mod:`doctest`. + + .. doctest:: + + >>> def spam(): + ... """ + ... This is a docstring with + ... leading whitespace. + ... + ... It even has multiple paragraphs! + ... """ + ... + >>> spam.__doc__ + '\nThis is a docstring with\n leading whitespace.\n\nIt even has multiple paragraphs!\n' -* Compiler now strip indents from docstrings. - This will reduce the size of :term:`bytecode cache ` (e.g. ``.pyc`` file). - For example, cache file size for ``sqlalchemy.orm.session`` in SQLAlchemy 2.0 - is reduced by about 5%. - This change will affect tools using docstrings, like :mod:`doctest`. (Contributed by Inada Naoki in :gh:`81283`.) -* The :func:`compile` built-in can now accept a new flag, +* :ref:`Annotation scopes ` within class scopes + can now contain lambdas and comprehensions. + Comprehensions that are located within class scopes + are not inlined into their parent scope. + + .. code-block:: python + + class C[T]: + type Alias = lambda: T + + (Contributed by Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) + +* :ref:`Future statements ` are no longer triggered by + relative imports of the :mod:`__future__` module, + meaning that statements of the form ``from .__future__ import ...`` + are now simply standard relative imports, with no special features activated. + (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) + +* :keyword:`global` declarations are now permitted in :keyword:`except` blocks + when that global is used in the :keyword:`else` block. + Previously this raised an erroneous :exc:`SyntaxError`. + (Contributed by Irit Katriel in :gh:`111123`.) + +* Add :envvar:`PYTHON_FROZEN_MODULES`, a new environment variable that + determines whether frozen modules are ignored by the import machinery, + equivalent to the :option:`-X frozen_modules <-X>` command-line option. + (Contributed by Yilei Yang in :gh:`111374`.) + +* Add :ref:`support for the perf profiler ` working + without `frame pointers `_ through + the new environment variable :envvar:`PYTHON_PERF_JIT_SUPPORT` + and command-line option :option:`-X perf_jit <-X>`. + (Contributed by Pablo Galindo in :gh:`118518`.) + +* The location of a :file:`.python_history` file can be changed via the + new :envvar:`PYTHON_HISTORY` environment variable. + (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade + in :gh:`73965`.) + +* Classes have a new :attr:`~type.__static_attributes__` attribute. + This is populated by the compiler with a tuple of the class's attribute names + which are assigned through ``self.`` from any function in its body. + (Contributed by Irit Katriel in :gh:`115775`.) + +* The compiler now creates a :attr:`!__firstlineno__` attribute on classes + with the line number of the first line of the class definition. + (Contributed by Serhiy Storchaka in :gh:`118465`.) + +* The :func:`exec` and :func:`eval` builtins now accept + the *globals* and *locals* arguments as keywords. + (Contributed by Raphael Gaschignard in :gh:`105879`) + +* The :func:`compile` builtin now accepts a new flag, ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` - except that the returned ``AST`` is optimized according to the value - of the ``optimize`` argument. + except that the returned AST is optimized according to + the value of the *optimize* argument. (Contributed by Irit Katriel in :gh:`108113`). -* :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`: - Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the - default number of worker threads and processes. Get the CPU affinity - if supported. - (Contributed by Victor Stinner in :gh:`109649`.) +* Add a :attr:`~property.__name__` attribute on :class:`property` objects. + (Contributed by Eugene Toder in :gh:`101860`.) -* :func:`os.path.realpath` now resolves MS-DOS style file names even if - the file is not accessible. - (Contributed by Moonsik Park in :gh:`82367`.) +* Add :exc:`PythonFinalizationError`, a new exception derived from + :exc:`RuntimeError` and used to signal when operations are blocked + during :term:`finalization `. + The following callables now raise :exc:`!PythonFinalizationError`, + instead of :exc:`RuntimeError`: -* Fixed a bug where a :keyword:`global` declaration in an :keyword:`except` block - is rejected when the global is used in the :keyword:`else` block. - (Contributed by Irit Katriel in :gh:`111123`.) + * :func:`_thread.start_new_thread` + * :func:`os.fork` + * :func:`os.forkpty` + * :class:`subprocess.Popen` + + (Contributed by Victor Stinner in :gh:`114570`.) + +* Allow the *count* argument of :meth:`str.replace` to be a keyword. + (Contributed by Hugo van Kemenade in :gh:`106487`.) * Many functions now emit a warning if a boolean value is passed as a file descriptor argument. This can help catch some errors earlier. (Contributed by Serhiy Storchaka in :gh:`82626`.) -* Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It - determines whether or not frozen modules are ignored by the import machinery, - equivalent of the :option:`-X frozen_modules <-X>` command-line option. - (Contributed by Yilei Yang in :gh:`111374`.) +* Added :attr:`!name` and :attr:`!mode` attributes + for compressed and archived file-like objects in + the :mod:`bz2`, :mod:`lzma`, :mod:`tarfile`, and :mod:`zipfile` modules. + (Contributed by Serhiy Storchaka in :gh:`115961`.) -* The new :envvar:`PYTHON_HISTORY` environment variable can be used to change - the location of a ``.python_history`` file. - (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in - :gh:`73965`.) New Modules =========== -* None yet. +* :mod:`dbm.sqlite3`: An SQLite backend for :mod:`dbm`. + (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) Improved Modules ================ + argparse -------- -* Add parameter *deprecated* in methods - :meth:`~argparse.ArgumentParser.add_argument` and :meth:`!add_parser` - which allows to deprecate command-line options, positional arguments and - subcommands. - (Contributed by Serhiy Storchaka in :gh:`83648`). +* Add the *deprecated* parameter to the + :meth:`~argparse.ArgumentParser.add_argument` + and :meth:`!add_parser` methods, to enable deprecating + command-line options, positional arguments, and subcommands. + (Contributed by Serhiy Storchaka in :gh:`83648`.) + array ----- -* Add ``'w'`` type code (``Py_UCS4``) that can be used for Unicode strings. - It can be used instead of ``'u'`` type code, which is deprecated. +* Add the ``'w'`` type code (``Py_UCS4``) for Unicode characters. + It should be used instead of the deprecated ``'u'`` type code. (Contributed by Inada Naoki in :gh:`80480`.) +* Register :class:`array.array` as a :class:`~collections.abc.MutableSequence` + by implementing the :meth:`~array.array.clear` method. + (Contributed by Mike Zimin in :gh:`114894`.) + + ast --- -* :func:`ast.parse` now accepts an optional argument ``optimize`` - which is passed on to the :func:`compile` built-in. This makes it - possible to obtain an optimized ``AST``. +* The constructors of node types in the :mod:`ast` module are now + stricter in the arguments they accept, + with more intuitive behavior when arguments are omitted. + + If an optional field on an AST node is not included as an argument when + constructing an instance, the field will now be set to ``None``. Similarly, + if a list field is omitted, that field will now be set to an empty list, + and if an :class:`!expr_context` field is omitted, it defaults to + :class:`Load() `. + (Previously, in all cases, the attribute would be missing on the newly + constructed AST node instance.) + + In all other cases, where a required argument is omitted, + the node constructor will emit a :exc:`DeprecationWarning`. + This will raise an exception in Python 3.15. + Similarly, passing a keyword argument to the constructor + that does not map to a field on the AST node is now deprecated, + and will raise an exception in Python 3.15. + + These changes do not apply to user-defined subclasses of :class:`ast.AST` + unless the class opts in to the new behavior + by defining the :attr:`.AST._field_types` mapping. + + (Contributed by Jelle Zijlstra in :gh:`105858`, :gh:`117486`, and :gh:`118851`.) + +* :func:`ast.parse` now accepts an optional argument *optimize* + which is passed on to :func:`compile`. + This makes it possible to obtain an optimized AST. (Contributed by Irit Katriel in :gh:`108113`.) + asyncio ------- +* :func:`asyncio.as_completed` now returns an object that is both an + :term:`asynchronous iterator` and a plain :term:`iterator` + of :term:`awaitables `. + The awaitables yielded by asynchronous iteration include original task + or future objects that were passed in, + making it easier to associate results with the tasks being completed. + (Contributed by Justin Arthur in :gh:`77714`.) + * :meth:`asyncio.loop.create_unix_server` will now automatically remove the Unix socket when the server is closed. (Contributed by Pierre Ossman in :gh:`111246`.) +* :meth:`.DatagramTransport.sendto` will now send zero-length + datagrams if called with an empty bytes object. + The transport flow control also now accounts for the datagram header + when calculating the buffer size. + (Contributed by Jamie Phan in :gh:`115199`.) + +* Add :meth:`Queue.shutdown ` + and :exc:`~asyncio.QueueShutDown` to manage queue termination. + (Contributed by Laurie Opperman and Yves Duprat in :gh:`104228`.) + +* Add the :meth:`.Server.close_clients` and :meth:`.Server.abort_clients` + methods, which more forcefully close an asyncio server. + (Contributed by Pierre Ossman in :gh:`113538`.) + +* Accept a tuple of separators in :meth:`.StreamReader.readuntil`, + stopping when any one of them is encountered. + (Contributed by Bruce Merry in :gh:`81322`.) + +* Improve the behavior of :class:`~asyncio.TaskGroup` when + an external cancellation collides with an internal cancellation. + For example, when two task groups are nested + and both experience an exception in a child task simultaneously, + it was possible that the outer task group would hang, + because its internal cancellation was swallowed by the inner task group. + + In the case where a task group is cancelled externally + and also must raise an :exc:`ExceptionGroup`, + it will now call the parent task's :meth:`~asyncio.Task.cancel` method. + This ensures that a :exc:`~asyncio.CancelledError` will be raised + at the next :keyword:`await`, so the cancellation is not lost. + + An added benefit of these changes is that task groups now preserve + the cancellation count (:meth:`~asyncio.Task.cancelling`). + + In order to handle some corner cases, :meth:`~asyncio.Task.uncancel` may now + reset the undocumented ``_must_cancel`` flag + when the cancellation count reaches zero. + + (Inspired by an issue reported by Arthur Tacca in :gh:`116720`.) + +* When :meth:`.TaskGroup.create_task` is called on an inactive + :class:`~asyncio.TaskGroup`, the given coroutine will be closed (which + prevents a :exc:`RuntimeWarning` about the given coroutine being + never awaited). + (Contributed by Arthur Tacca and Jason Zhang in :gh:`115957`.) + + +base64 +------ + +* Add :func:`~base64.z85encode` and :func:`~base64.z85decode` functions + for encoding :class:`bytes` as `Z85 data`_ + and decoding Z85-encoded data to :class:`!bytes`. + (Contributed by Matan Perelman in :gh:`75299`.) + + .. _Z85 data: https://rfc.zeromq.org/spec/32/ + + +compileall +---------- + +* The default number of worker threads and processes is now selected using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + +concurrent.futures +------------------ + +* The default number of worker threads and processes is now selected using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + +configparser +------------ + +* :class:`~configparser.ConfigParser` now has support for unnamed sections, + which allows for top-level key-value pairs. + This can be enabled with the new *allow_unnamed_section* parameter. + (Contributed by Pedro Sousa Lacerda in :gh:`66449`.) + + copy ---- -* Add :func:`copy.replace` function which allows to create a modified copy of - an object, which is especially useful for immutable objects. - It supports named tuples created with the factory function - :func:`collections.namedtuple`, :class:`~dataclasses.dataclass` instances, - various :mod:`datetime` objects, :class:`~inspect.Signature` objects, - :class:`~inspect.Parameter` objects, :ref:`code object `, and - any user classes which define the :meth:`!__replace__` method. +* The new :func:`~copy.replace` function and the :meth:`replace protocol + ` make creating modified copies of objects much simpler. + This is especially useful when working with immutable objects. + The following types support the :func:`~copy.replace` function + and implement the replace protocol: + + * :func:`collections.namedtuple` + * :class:`dataclasses.dataclass` + * :class:`datetime.datetime`, :class:`datetime.date`, :class:`datetime.time` + * :class:`inspect.Signature`, :class:`inspect.Parameter` + * :class:`types.SimpleNamespace` + * :ref:`code objects ` + + Any user-defined class can also support :func:`copy.replace` by defining + the :meth:`~object.__replace__` method. (Contributed by Serhiy Storchaka in :gh:`108751`.) + +ctypes +------ + +* As a consequence of necessary internal refactoring, initialization of + internal metaclasses now happens in ``__init__`` rather + than in ``__new__``. This affects projects that subclass these internal + metaclasses to provide custom initialization. + Generally: + + - Custom logic that was done in ``__new__`` after calling ``super().__new__`` + should be moved to ``__init__``. + - To create a class, call the metaclass, not only the metaclass's + ``__new__`` method. + + See :gh:`124520` for discussion and links to changes in some affected + projects. + +* :class:`ctypes.Structure` objects have a new :attr:`~ctypes.Structure._align_` + attribute which allows the alignment of the structure being packed to/from + memory to be specified explicitly. + (Contributed by Matt Sanderson in :gh:`112433`) + dbm --- -* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items - from the database. +* Add :mod:`dbm.sqlite3`, a new module which implements an SQLite backend, + and make it the default :mod:`!dbm` backend. + (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) + +* Allow removing all items from the database through + the new :meth:`.gdbm.clear` and :meth:`.ndbm.clear` methods. (Contributed by Donghee Na in :gh:`107122`.) + dis --- * Change the output of :mod:`dis` module functions to show logical labels for jump targets and exception handlers, rather than offsets. - The offsets can be added with the new ``-O`` command line option or - the ``show_offsets`` parameter. + The offsets can be added with the new + :option:`-O ` command-line option + or the *show_offsets* argument. (Contributed by Irit Katriel in :gh:`112137`.) +* :meth:`~dis.get_instructions` no longer represents cache entries + as separate instructions. + Instead, it returns them as part of the :class:`~dis.Instruction`, + in the new *cache_info* field. + The *show_caches* argument to :meth:`~dis.get_instructions` is deprecated + and no longer has any effect. + (Contributed by Irit Katriel in :gh:`112962`.) + + +.. _whatsnew313-doctest: + doctest ------- -* The :meth:`doctest.DocTestRunner.run` method now counts the number of skipped - tests. Add :attr:`doctest.DocTestRunner.skips` and - :attr:`doctest.TestResults.skipped` attributes. +* :mod:`doctest` output is now colored by default. + This can be controlled via the new :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ + and |FORCE_COLOR|_ environment variables. + See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`117225`.) + +* The :meth:`.DocTestRunner.run` method now counts the number of skipped tests. + Add the :attr:`.DocTestRunner.skips` and :attr:`.TestResults.skipped` attributes. (Contributed by Victor Stinner in :gh:`108794`.) + email ----- -* :func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now return - ``('', '')`` 2-tuples in more situations where invalid email addresses are - encountered instead of potentially inaccurate values. Add optional *strict* - parameter to these two functions: use ``strict=False`` to get the old - behavior, accept malformed inputs. - ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to - check if the *strict* parameter is available. +* Headers with embedded newlines are now quoted on output. + The :mod:`~email.generator` will now refuse to serialize (write) headers + that are improperly folded or delimited, such that they would be parsed as + multiple headers or joined with adjacent data. + If you need to turn this safety feature off, + set :attr:`~email.policy.Policy.verify_generated_headers`. + (Contributed by Bas Bloemsaat and Petr Viktorin in :gh:`121650`.) + +* :func:`~email.utils.getaddresses` and :func:`~email.utils.parseaddr` now + return ``('', '')`` pairs in more situations where invalid email addresses + are encountered instead of potentially inaccurate values. + The two functions have a new optional *strict* parameter (default ``True``). + To get the old behavior (accepting malformed input), use ``strict=False``. + ``getattr(email.utils, 'supports_strict_parsing', False)`` can be used + to check if the *strict* parameter is available. (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve - the CVE-2023-27043 fix.) + the :cve:`2023-27043` fix.) + fractions --------- -* Formatting for objects of type :class:`fractions.Fraction` now supports - the standard format specification mini-language rules for fill, alignment, - sign handling, minimum width and grouping. (Contributed by Mark Dickinson - in :gh:`111320`.) +* :class:`~fractions.Fraction` objects now support the standard + :ref:`format specification mini-language ` rules + for fill, alignment, sign handling, minimum width, and grouping. + (Contributed by Mark Dickinson in :gh:`111320`.) + glob ---- -* Add :func:`glob.translate` function that converts a path specification with - shell-style wildcards to a regular expression. +* Add :func:`~glob.translate`, a function to convert a path specification + with shell-style wildcards to a regular expression. (Contributed by Barney Gale in :gh:`72904`.) + +importlib +--------- + +* The following functions in :mod:`importlib.resources` now allow accessing + a directory (or tree) of resources, using multiple positional arguments + (the *encoding* and *errors* arguments in the text-reading functions + are now keyword-only): + + * :func:`~importlib.resources.is_resource` + * :func:`~importlib.resources.open_binary` + * :func:`~importlib.resources.open_text` + * :func:`~importlib.resources.path` + * :func:`~importlib.resources.read_binary` + * :func:`~importlib.resources.read_text` + + These functions are no longer deprecated and are not scheduled for removal. + (Contributed by Petr Viktorin in :gh:`116608`.) + +* :func:`~importlib.resources.contents` remains deprecated in favor of + the fully-featured :class:`~importlib.resources.abc.Traversable` API. + However, there is now no plan to remove it. + (Contributed by Petr Viktorin in :gh:`116608`.) + + io -- -The :class:`io.IOBase` finalizer now logs the ``close()`` method errors with -:data:`sys.unraisablehook`. Previously, errors were ignored silently by default, -and only logged in :ref:`Python Development Mode ` or on :ref:`Python -built on debug mode `. -(Contributed by Victor Stinner in :gh:`62948`.) +* The :class:`~io.IOBase` finalizer now logs any errors raised by + the :meth:`~io.IOBase.close` method with :data:`sys.unraisablehook`. + Previously, errors were ignored silently by default, + and only logged in :ref:`Python Development Mode ` + or when using a :ref:`Python debug build `. + (Contributed by Victor Stinner in :gh:`62948`.) + ipaddress --------- -* Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. +* Add the :attr:`.IPv4Address.ipv6_mapped` property, + which returns the IPv4-mapped IPv6 address. (Contributed by Charles Machalow in :gh:`109466`.) +* Fix ``is_global`` and ``is_private`` behavior in + :class:`~ipaddress.IPv4Address`, :class:`~ipaddress.IPv6Address`, + :class:`~ipaddress.IPv4Network`, and :class:`~ipaddress.IPv6Network`. + (Contributed by Jakub Stasiak in :gh:`113171`.) + + +itertools +--------- + +* :func:`~itertools.batched` has a new *strict* parameter, + which raises a :exc:`ValueError` if the final batch is shorter + than the specified batch size. + (Contributed by Raymond Hettinger in :gh:`113202`.) + + marshal ------- * Add the *allow_code* parameter in module functions. - Passing ``allow_code=False`` prevents serialization and de-serialization of - code objects which are incompatible between Python versions. + Passing ``allow_code=False`` prevents serialization and de-serialization + of code objects which are incompatible between Python versions. (Contributed by Serhiy Storchaka in :gh:`113626`.) + +math +---- + +* The new function :func:`~math.fma` performs fused multiply-add operations. + This computes ``x * y + z`` with only a single round, + and so avoids any intermediate loss of precision. + It wraps the ``fma()`` function provided by C99, + and follows the specification of the IEEE 754 "fusedMultiplyAdd" operation + for special cases. + (Contributed by Mark Dickinson and Victor Stinner in :gh:`73468`.) + + +mimetypes +--------- + +* Add the :func:`~mimetypes.guess_file_type` function to guess a MIME type + from a filesystem path. + Using paths with :func:`~mimetypes.guess_type` is now :term:`soft deprecated`. + (Contributed by Serhiy Storchaka in :gh:`66543`.) + + mmap ---- -* The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method +* :class:`~mmap.mmap` is now protected from crashing on Windows when the + mapped memory is inaccessible due to file system errors or access violations. + (Contributed by Jannis Weigend in :gh:`118209`.) + +* :class:`~mmap.mmap` has a new :meth:`~mmap.mmap.seekable` method that can be used when a seekable file-like object is required. The :meth:`~mmap.mmap.seek` method now returns the new absolute position. (Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.) -* :class:`mmap.mmap` now has a *trackfd* parameter on Unix; if it is ``False``, - the file descriptor specified by *fileno* will not be duplicated. + +* The new UNIX-only *trackfd* parameter for :class:`~mmap.mmap` controls + file descriptor duplication; + if false, the file descriptor specified by *fileno* will not be duplicated. (Contributed by Zackery Spytz and Petr Viktorin in :gh:`78502`.) -opcode ------- -* Move ``opcode.ENABLE_SPECIALIZATION`` to ``_opcode.ENABLE_SPECIALIZATION``. - This field was added in 3.12, it was never documented and is not intended for - external usage. (Contributed by Irit Katriel in :gh:`105481`.) +multiprocessing +--------------- + +* The default number of worker threads and processes is now selected using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) -* Removed ``opcode.is_pseudo``, ``opcode.MIN_PSEUDO_OPCODE`` and - ``opcode.MAX_PSEUDO_OPCODE``, which were added in 3.12, were never - documented or exposed through ``dis``, and were not intended to be - used externally. os -- -* Add :func:`os.process_cpu_count` function to get the number of logical CPUs - usable by the calling thread of the current process. +* Add :func:`~os.process_cpu_count` function to get the number + of logical CPU cores usable by the calling thread of the current process. (Contributed by Victor Stinner in :gh:`109649`.) -* Add a low level interface for Linux's timer notification file descriptors - via :func:`os.timerfd_create`, - :func:`os.timerfd_settime`, :func:`os.timerfd_settime_ns`, - :func:`os.timerfd_gettime`, and :func:`os.timerfd_gettime_ns`, - :const:`os.TFD_NONBLOCK`, :const:`os.TFD_CLOEXEC`, - :const:`os.TFD_TIMER_ABSTIME`, and :const:`os.TFD_TIMER_CANCEL_ON_SET` - (Contributed by Masaru Tsuchiyama in :gh:`108277`.) - -* :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overridden through - the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option - :option:`-X cpu_count <-X>`. This option is useful for users who need to limit - CPU resources of a container system without having to modify the container (application code). +* :func:`~os.cpu_count` and :func:`~os.process_cpu_count` can be overridden + through the new environment variable :envvar:`PYTHON_CPU_COUNT` + or the new command-line option :option:`-X cpu_count <-X>`. + This option is useful for users who need to limit CPU resources + of a container system without having to modify application code + or the container itself. (Contributed by Donghee Na in :gh:`109595`.) -* Add support of :func:`os.lchmod` and the *follow_symlinks* argument - in :func:`os.chmod` on Windows. - Note that the default value of *follow_symlinks* in :func:`!os.lchmod` is - ``False`` on Windows. +* Add a :ref:`low level interface ` to Linux's + :manpage:`timer file descriptors ` + via :func:`~os.timerfd_create`, + :func:`~os.timerfd_settime`, :func:`~os.timerfd_settime_ns`, + :func:`~os.timerfd_gettime`, :func:`~os.timerfd_gettime_ns`, + :const:`~os.TFD_NONBLOCK`, :const:`~os.TFD_CLOEXEC`, + :const:`~os.TFD_TIMER_ABSTIME`, and :const:`~os.TFD_TIMER_CANCEL_ON_SET` + (Contributed by Masaru Tsuchiyama in :gh:`108277`.) + +* :func:`~os.lchmod` and the *follow_symlinks* argument of :func:`~os.chmod` + are both now available on Windows. + Note that the default value of *follow_symlinks* + in :func:`!lchmod` is ``False`` on Windows. (Contributed by Serhiy Storchaka in :gh:`59616`.) -* Add support of :func:`os.fchmod` and a file descriptor - in :func:`os.chmod` on Windows. +* :func:`~os.fchmod` and support for file descriptors in :func:`~os.chmod` + are both now available on Windows. (Contributed by Serhiy Storchaka in :gh:`113191`.) -* :func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned - process use the current process environment. +* On Windows, :func:`~os.mkdir` and :func:`~os.makedirs` now support passing + a *mode* value of ``0o700`` to apply access control to the new directory. + This implicitly affects :func:`tempfile.mkdtemp` + and is a mitigation for :cve:`2024-4030`. + Other values for *mode* continue to be ignored. + (Contributed by Steve Dower in :gh:`118486`.) + +* :func:`~os.posix_spawn` now accepts ``None`` for the *env* argument, + which makes the newly spawned process use the current process environment. (Contributed by Jakub Kulik in :gh:`113119`.) -* :func:`os.posix_spawn` gains an :attr:`os.POSIX_SPAWN_CLOSEFROM` attribute for - use in ``file_actions=`` on platforms that support +* :func:`~os.posix_spawn` can now use the :attr:`~os.POSIX_SPAWN_CLOSEFROM` + attribute in the *file_actions* parameter on platforms that support :c:func:`!posix_spawn_file_actions_addclosefrom_np`. (Contributed by Jakub Kulik in :gh:`113117`.) + os.path ------- -* Add :func:`os.path.isreserved` to check if a path is reserved on the current - system. This function is only available on Windows. +* Add :func:`~os.path.isreserved` to check if a path is reserved + on the current system. + This function is only available on Windows. (Contributed by Barney Gale in :gh:`88569`.) -* On Windows, :func:`os.path.isabs` no longer considers paths starting with - exactly one (back)slash to be absolute. + +* On Windows, :func:`~os.path.isabs` no longer considers paths + starting with exactly one slash (``\`` or ``/``) to be absolute. (Contributed by Barney Gale and Jon Foster in :gh:`44626`.) +* :func:`~os.path.realpath` now resolves MS-DOS style file names + even if the file is not accessible. + (Contributed by Moonsik Park in :gh:`82367`.) + + pathlib ------- -* Add :exc:`pathlib.UnsupportedOperation`, which is raised instead of +* Add :exc:`~pathlib.UnsupportedOperation`, which is raised instead of :exc:`NotImplementedError` when a path operation isn't supported. (Contributed by Barney Gale in :gh:`89812`.) -* Add :meth:`pathlib.Path.from_uri`, a new constructor to create a :class:`pathlib.Path` - object from a 'file' URI (``file://``). +* Add a new constructor for creating :class:`~pathlib.Path` objects + from 'file' URIs (``file:///``), :meth:`.Path.from_uri`. (Contributed by Barney Gale in :gh:`107465`.) -* Add :meth:`pathlib.PurePath.full_match` for matching paths with +* Add :meth:`.PurePath.full_match` for matching paths with shell-style wildcards, including the recursive wildcard "``**``". (Contributed by Barney Gale in :gh:`73435`.) -* Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.glob`, - :meth:`~pathlib.Path.rglob`, :meth:`~pathlib.Path.is_file`, - :meth:`~pathlib.Path.is_dir`, :meth:`~pathlib.Path.owner`, - :meth:`~pathlib.Path.group`. - (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`, and - Kamil Turek in :gh:`107962`). +* Add the :attr:`.PurePath.parser` class attribute to store the + implementation of :mod:`os.path` used + for low-level path parsing and joining. + This will be either :mod:`!posixpath` or :mod:`!ntpath`. + +* Add *recurse_symlinks* keyword-only argument to + :meth:`.Path.glob` and :meth:`~pathlib.Path.rglob`. + (Contributed by Barney Gale in :gh:`77609`.) + +* :meth:`.Path.glob` and :meth:`~pathlib.Path.rglob` + now return files and directories when given a pattern that ends with "``**``". + Previously, only directories were returned. + (Contributed by Barney Gale in :gh:`70303`.) + +* Add the *follow_symlinks* keyword-only argument to + :meth:`Path.is_file `, + :meth:`Path.is_dir `, + :meth:`.Path.owner`, and :meth:`.Path.group`. + (Contributed by Barney Gale in :gh:`105793` and Kamil Turek in :gh:`107962`.) -* Return files and directories from :meth:`pathlib.Path.glob` and - :meth:`~pathlib.Path.rglob` when given a pattern that ends with "``**``". In - earlier versions, only directories were returned. - (Contributed by Barney Gale in :gh:`70303`). pdb --- -* Add ability to move between chained exceptions during post mortem debugging in :func:`~pdb.pm` using - the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias - Bussonnier in :gh:`106676`.) +* :func:`breakpoint` and :func:`~pdb.set_trace` now enter the debugger immediately + rather than on the next line of code to be executed. This change prevents the + debugger from breaking outside of the context when :func:`!breakpoint` is positioned + at the end of the context. + (Contributed by Tian Gao in :gh:`118579`.) + +* ``sys.path[0]`` is no longer replaced by the directory of the script + being debugged when :attr:`sys.flags.safe_path` is set. + (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) + +* :mod:`zipapp` is now supported as a debugging target. + (Contributed by Tian Gao in :gh:`118501`.) -* Expressions/statements whose prefix is a pdb command are now correctly +* Add ability to move between chained exceptions during + post-mortem debugging in :func:`~pdb.pm` using + the new :pdbcmd:`exceptions [exc_number] ` command for Pdb. + (Contributed by Matthias Bussonnier in :gh:`106676`.) + +* Expressions and statements whose prefix is a pdb command are now correctly identified and executed. (Contributed by Tian Gao in :gh:`108464`.) -* ``sys.path[0]`` will no longer be replaced by the directory of the script - being debugged when ``sys.flags.safe_path`` is set (via the :option:`-P` - command line option or :envvar:`PYTHONSAFEPATH` environment variable). - (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) queue ----- -* Add :meth:`queue.Queue.shutdown` (along with :exc:`queue.ShutDown`) for queue - termination. +* Add :meth:`Queue.shutdown ` and :exc:`~queue.ShutDown` + to manage queue termination. (Contributed by Laurie Opperman and Yves Duprat in :gh:`104750`.) + +random +------ + +* Add a :ref:`command-line interface `. + (Contributed by Hugo van Kemenade in :gh:`118131`.) + + re -- -* Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. + +* Rename :exc:`!re.error` to :exc:`~re.PatternError` for improved clarity. :exc:`!re.error` is kept for backward compatibility. + +shutil +------ + +* Support the *dir_fd* and *follow_symlinks* keyword arguments + in :func:`~shutil.chown`. + (Contributed by Berker Peksag and Tahia K in :gh:`62308`) + + +site +---- + +* :file:`.pth` files are now decoded using UTF-8 first, + and then with the :term:`locale encoding` if UTF-8 decoding fails. + (Contributed by Inada Naoki in :gh:`117802`.) + + sqlite3 ------- -* A :exc:`ResourceWarning` is now emitted if a :class:`sqlite3.Connection` +* A :exc:`ResourceWarning` is now emitted if a :class:`~sqlite3.Connection` object is not :meth:`closed ` explicitly. (Contributed by Erlend E. Aasland in :gh:`105539`.) -* Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump` +* Add the *filter* keyword-only parameter to :meth:`.Connection.iterdump` for filtering database objects to dump. (Contributed by Mariusz Felisiak in :gh:`91602`.) + +ssl +--- + +* The :func:`~ssl.create_default_context` API now includes + :data:`~ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`~ssl.VERIFY_X509_STRICT` + in its default flags. + + .. note:: + + :data:`~ssl.VERIFY_X509_STRICT` may reject pre-:rfc:`5280` + or malformed certificates that the underlying OpenSSL implementation + might otherwise accept. + Whilst disabling this is not recommended, you can do so using: + + .. code-block:: python + + import ssl + + ctx = ssl.create_default_context() + ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + + (Contributed by William Woodruff in :gh:`112389`.) + + +statistics +---------- + +* Add :func:`~statistics.kde` for kernel density estimation. + This makes it possible to estimate a continuous probability density function + from a fixed number of discrete samples. + (Contributed by Raymond Hettinger in :gh:`115863`.) + +* Add :func:`~statistics.kde_random` for sampling from an + estimated probability density function created by :func:`~statistics.kde`. + (Contributed by Raymond Hettinger in :gh:`115863`.) + + +.. _whatsnew313-subprocess: + subprocess ---------- -* The :mod:`subprocess` module now uses the :func:`os.posix_spawn` function in - more situations. Notably in the default case of ``close_fds=True`` on more - recent versions of platforms including Linux, FreeBSD, and Solaris where the - C library provides :c:func:`!posix_spawn_file_actions_addclosefrom_np`. - On Linux this should perform similar to our existing Linux :c:func:`!vfork` - based code. A private control knob :attr:`!subprocess._USE_POSIX_SPAWN` can - be set to ``False`` if you need to force :mod:`subprocess` not to ever use - :func:`os.posix_spawn`. Please report your reason and platform details in - the CPython issue tracker if you set this so that we can improve our API - selection logic for everyone. +* The :mod:`subprocess` module now uses the :func:`~os.posix_spawn` function in + more situations. + + Notably, when *close_fds* is ``True`` (the default), + :func:`~os.posix_spawn` will be used when the C library provides + :c:func:`!posix_spawn_file_actions_addclosefrom_np`, + which includes recent versions of Linux, FreeBSD, and Solaris. + On Linux, this should perform similarly to the existing + Linux :c:func:`!vfork` based code. + + A private control knob :attr:`!subprocess._USE_POSIX_SPAWN` can + be set to ``False`` if you need to force :mod:`subprocess` + to never use :func:`~os.posix_spawn`. + Please report your reason and platform details in + the :ref:`issue tracker ` if you set this + so that we can improve our API selection logic for everyone. (Contributed by Jakub Kulik in :gh:`113117`.) + sys --- -* Add the :func:`sys._is_interned` function to test if the string was interned. +* Add the :func:`~sys._is_interned` function to test if a string was interned. This function is not guaranteed to exist in all implementations of Python. (Contributed by Serhiy Storchaka in :gh:`78573`.) + +tempfile +-------- + +* On Windows, the default mode ``0o700`` used by :func:`tempfile.mkdtemp` now + limits access to the new directory due to changes to :func:`os.mkdir`. + This is a mitigation for :cve:`2024-4030`. + (Contributed by Steve Dower in :gh:`118486`.) + + +time +---- + +* On Windows, :func:`~time.monotonic` now uses the + ``QueryPerformanceCounter()`` clock for a resolution of 1 microsecond, + instead of the ``GetTickCount64()`` clock which has + a resolution of 15.6 milliseconds. + (Contributed by Victor Stinner in :gh:`88494`.) + +* On Windows, :func:`~time.time` now uses the + ``GetSystemTimePreciseAsFileTime()`` clock for a resolution of 1 microsecond, + instead of the ``GetSystemTimeAsFileTime()`` clock which has + a resolution of 15.6 milliseconds. + (Contributed by Victor Stinner in :gh:`63207`.) + + tkinter ------- @@ -459,1077 +1304,1328 @@ tkinter * The :mod:`tkinter` widget method :meth:`!wm_attributes` now accepts the attribute name without the minus prefix to get window attributes, - e.g. ``w.wm_attributes('alpha')`` and allows to specify attributes and - values to set as keyword arguments, e.g. ``w.wm_attributes(alpha=0.5)``. - Add new optional keyword-only parameter *return_python_dict*: calling - ``w.wm_attributes(return_python_dict=True)`` returns the attributes as - a dict instead of a tuple. + for example ``w.wm_attributes('alpha')`` + and allows specifying attributes and values to set as keyword arguments, + for example ``w.wm_attributes(alpha=0.5)``. + (Contributed by Serhiy Storchaka in :gh:`43457`.) + +* :meth:`!wm_attributes` can now return attributes as a :class:`dict`, + by using the new optional keyword-only parameter *return_python_dict*. (Contributed by Serhiy Storchaka in :gh:`43457`.) -* Add support of the "vsapi" element type in +* :meth:`!Text.count` can now return a simple :class:`int` + when the new optional keyword-only parameter *return_ints* is used. + Otherwise, the single count is returned as a 1-tuple or ``None``. + (Contributed by Serhiy Storchaka in :gh:`97928`.) + +* Support the "vsapi" element type in the :meth:`~tkinter.ttk.Style.element_create` method of :class:`tkinter.ttk.Style`. (Contributed by Serhiy Storchaka in :gh:`68166`.) +* Add the :meth:`!after_info` method for Tkinter widgets. + (Contributed by Cheryl Sabella in :gh:`77020`.) + +* Add a new :meth:`!copy_replace` method to :class:`!PhotoImage` + to copy a region from one image to another, + possibly with pixel zooming, subsampling, or both. + (Contributed by Serhiy Storchaka in :gh:`118225`.) + +* Add *from_coords* parameter to the :class:`!PhotoImage` methods + :meth:`!copy`, :meth:`!zoom` and :meth:`!subsample`. + Add *zoom* and *subsample* parameters to the :class:`!PhotoImage` method + :meth:`!copy`. + (Contributed by Serhiy Storchaka in :gh:`118225`.) + +* Add the :class:`!PhotoImage` methods + :meth:`!read` to read an image from a file + and :meth:`!data` to get the image data. + Add *background* and *grayscale* parameters to the :meth:`!write` method. + (Contributed by Serhiy Storchaka in :gh:`118271`.) + + traceback --------- -* Add *show_group* parameter to :func:`traceback.TracebackException.format_exception_only` - to format the nested exceptions of a :exc:`BaseExceptionGroup` instance, recursively. +* Add the :attr:`~traceback.TracebackException.exc_type_str` attribute + to :class:`~traceback.TracebackException`, + which holds a string display of the *exc_type*. + Deprecate the :attr:`~traceback.TracebackException.exc_type` attribute, + which holds the type object itself. + Add parameter *save_exc_type* (default ``True``) + to indicate whether ``exc_type`` should be saved. + (Contributed by Irit Katriel in :gh:`112332`.) + +* Add a new *show_group* keyword-only parameter to + :meth:`.TracebackException.format_exception_only` to (recursively) format + the nested exceptions of a :exc:`BaseExceptionGroup` instance. (Contributed by Irit Katriel in :gh:`105292`.) -* Add the field *exc_type_str* to :class:`~traceback.TracebackException`, which - holds a string display of the *exc_type*. Deprecate the field *exc_type* - which holds the type object itself. Add parameter *save_exc_type* (default - ``True``) to indicate whether ``exc_type`` should be saved. - (Contributed by Irit Katriel in :gh:`112332`.) + +types +----- + +* :class:`~types.SimpleNamespace` can now take a single positional argument + to initialise the namespace's arguments. + This argument must either be a mapping or an iterable of key-value pairs. + (Contributed by Serhiy Storchaka in :gh:`108191`.) + typing ------ -* Add :func:`typing.get_protocol_members` to return the set of members - defining a :class:`typing.Protocol`. Add :func:`typing.is_protocol` to - check whether a class is a :class:`typing.Protocol`. (Contributed by Jelle Zijlstra in - :gh:`104873`.) +* :pep:`705`: Add :data:`~typing.ReadOnly`, a special typing construct + to mark a :class:`~typing.TypedDict` item as read-only for type checkers. + +* :pep:`742`: Add :data:`~typing.TypeIs`, a typing construct + that can be used to instruct a type checker how to narrow a type. + +* Add :data:`~typing.NoDefault`, a sentinel object used to represent + the defaults of some parameters in the :mod:`typing` module. + (Contributed by Jelle Zijlstra in :gh:`116126`.) + +* Add :func:`~typing.get_protocol_members` to return the set of members + defining a :class:`typing.Protocol`. + (Contributed by Jelle Zijlstra in :gh:`104873`.) + +* Add :func:`~typing.is_protocol` to check whether a class + is a :class:`~typing.Protocol`. + (Contributed by Jelle Zijlstra in :gh:`104873`.) + +* :data:`~typing.ClassVar` can now be nested in :data:`~typing.Final`, + and vice versa. + (Contributed by Mehdi Drissi in :gh:`89547`.) + unicodedata ----------- -* The Unicode database has been updated to version 15.1.0. (Contributed by - James Gerity in :gh:`109559`.) +* Update the Unicode database to `version 15.1.0`__. + (Contributed by James Gerity in :gh:`109559`.) + + __ https://www.unicode.org/versions/Unicode15.1.0/ + venv ---- -* Add support for adding source control management (SCM) ignore files to a - virtual environment's directory. By default, Git is supported. This is - implemented as opt-in via the API which can be extended to support other SCMs - (:class:`venv.EnvBuilder` and :func:`venv.create`), and opt-out via the CLI - (using ``--without-scm-ignore-files``). (Contributed by Brett Cannon in - :gh:`108125`.) +* Add support for creating source control management (SCM) ignore files + in a virtual environment's directory. + By default, Git is supported. + This is implemented as opt-in via the API, + which can be extended to support other SCMs + (:class:`~venv.EnvBuilder` and :func:`~venv.create`), + and opt-out via the CLI, using :option:`!--without-scm-ignore-files`. + (Contributed by Brett Cannon in :gh:`108125`.) + warnings -------- -* The new :func:`warnings.deprecated` decorator provides a way to communicate - deprecations to :term:`static type checkers ` and - to warn on usage of deprecated classes and functions. A runtime deprecation - warning may also be emitted when a decorated function or class is used at runtime. - See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) +* :pep:`702`: The new :func:`warnings.deprecated` decorator provides a way to + communicate deprecations to a :term:`static type checker` + and to warn on usage of deprecated classes and functions. + A :exc:`DeprecationWarning` may also be emitted when + a decorated function or class is used at runtime. + (Contributed by Jelle Zijlstra in :gh:`104003`.) -xml.etree.ElementTree ---------------------- + +xml +--- + +* Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) + by adding five new methods: + + * :meth:`xml.etree.ElementTree.XMLParser.flush` + * :meth:`xml.etree.ElementTree.XMLPullParser.flush` + * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` + * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` + * :meth:`!xml.sax.expatreader.ExpatParser.flush` + + (Contributed by Sebastian Pipping in :gh:`115623`.) * Add the :meth:`!close` method for the iterator returned by - :func:`~xml.etree.ElementTree.iterparse` for explicit cleaning up. + :func:`~xml.etree.ElementTree.iterparse` for explicit cleanup. (Contributed by Serhiy Storchaka in :gh:`69893`.) +zipimport +--------- + +* Add support for ZIP64_ format files. + Everybody loves huge data, right? + (Contributed by Tim Hatch in :gh:`94146`.) + + .. _ZIP64: https://en.wikipedia.org/wiki/Zip_(file_format)#ZIP64 + + Optimizations ============= -* :func:`textwrap.indent` is now ~30% faster than before for large input. +* Several standard library modules have had + their import times significantly improved. + For example, the import time of the :mod:`typing` module + has been reduced by around a third by removing dependencies + on :mod:`re` and :mod:`contextlib`. + Other modules to enjoy import-time speedups include + :mod:`email.utils`, :mod:`enum`, :mod:`functools`, + :mod:`importlib.metadata`, and :mod:`threading`. + (Contributed by Alex Waygood, Shantanu Jain, Adam Turner, Daniel Hollas, + and others in :gh:`109653`.) + +* :func:`textwrap.indent` is now around 30% faster than before for large input. (Contributed by Inada Naoki in :gh:`107369`.) -* The :mod:`subprocess` module uses :func:`os.posix_spawn` in more situations - including the default where ``close_fds=True`` on many modern platforms. This - should provide a noteworthy performance increase launching processes on - FreeBSD and Solaris. See the ``subprocess`` section above for details. +* The :mod:`subprocess` module now uses the :func:`~os.posix_spawn` function in + more situations, including when *close_fds* is ``True`` (the default) + on many modern platforms. + This should provide a notable performance increase + when launching processes on FreeBSD and Solaris. + See the :ref:`subprocess ` section above for details. (Contributed by Jakub Kulik in :gh:`113117`.) -.. _whatsnew313-jit-compiler: -Experimental JIT Compiler -========================= +Removed Modules And APIs +======================== -When CPython is configured using the ``--enable-experimental-jit`` option, -a just-in-time compiler is added which can speed up some Python programs. -The internal architecture is roughly as follows. +.. _whatsnew313-pep594: -* We start with specialized *Tier 1 bytecode*. - See :ref:`What's new in 3.11 ` for details. +PEP 594: Remove "dead batteries" from the standard library +---------------------------------------------------------- + +:pep:`594` proposed removing 19 modules from the standard library, +colloquially referred to as 'dead batteries' due to their +historic, obsolete, or insecure status. +All of the following modules were deprecated in Python 3.11, +and are now removed: + +* :mod:`!aifc` +* :mod:`!audioop` +* :mod:`!chunk` +* :mod:`!cgi` and :mod:`!cgitb` + + * :class:`!cgi.FieldStorage` can typically be replaced with + :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, + and the :mod:`email.message` module or the :pypi:`multipart` library + for ``POST`` and ``PUT`` requests. + + * :func:`!cgi.parse` can be replaced by calling + :func:`urllib.parse.parse_qs` directly on the desired query string, + unless the input is ``multipart/form-data``, + which should be replaced as described below for :func:`!cgi.parse_multipart`. + + * :func:`!cgi.parse_header` can be replaced with the functionality + in the :mod:`email` package, which implements the same MIME RFCs. + For example, with :class:`email.message.EmailMessage`: + + .. code-block:: python + + from email.message import EmailMessage + + msg = EmailMessage() + msg['content-type'] = 'application/json; charset="utf8"' + main, params = msg.get_content_type(), msg['content-type'].params + + * :func:`!cgi.parse_multipart` can be replaced with the functionality + in the :mod:`email` package, which implements the same MIME RFCs, + or with the :pypi:`multipart` library. + For example, the :class:`email.message.EmailMessage` + and :class:`email.message.Message` classes. + +* :mod:`!crypt` and the private :mod:`!_crypt` extension. + The :mod:`hashlib` module may be an appropriate replacement + when simply hashing a value is required. + Otherwise, various third-party libraries on PyPI are available: + + * :pypi:`bcrypt`: + Modern password hashing for your software and your servers. + * :pypi:`passlib`: + Comprehensive password hashing framework supporting over 30 schemes. + * :pypi:`argon2-cffi`: + The secure Argon2 password hashing algorithm. + * :pypi:`legacycrypt`: + :mod:`ctypes` wrapper to the POSIX crypt library call + and associated functionality. + * :pypi:`crypt_r`: + Fork of the :mod:`!crypt` module, + wrapper to the :manpage:`crypt_r(3)` library call + and associated functionality. + +* :mod:`!imghdr`: + The :pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic` libraries + should be used as replacements. + For example, the :func:`!puremagic.what` function can be used + to replace the :func:`!imghdr.what` function for all file formats + that were supported by :mod:`!imghdr`. +* :mod:`!mailcap`: + Use the :mod:`mimetypes` module instead. +* :mod:`!msilib` +* :mod:`!nis` +* :mod:`!nntplib`: + Use the :pypi:`pynntp` library from PyPI instead. +* :mod:`!ossaudiodev`: + For audio playback, use the :pypi:`pygame` library from PyPI instead. +* :mod:`!pipes`: + Use the :mod:`subprocess` module instead. +* :mod:`!sndhdr`: + The :pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic` libraries + should be used as replacements. +* :mod:`!spwd`: + Use the :pypi:`python-pam` library from PyPI instead. +* :mod:`!sunau` +* :mod:`!telnetlib`, + Use the :pypi:`telnetlib3` or :pypi:`Exscript` libraries from PyPI instead. +* :mod:`!uu`: + Use the :mod:`base64` module instead, as a modern alternative. +* :mod:`!xdrlib` + +(Contributed by Victor Stinner and Zachary Ware in :gh:`104773` and :gh:`104780`.) -* When the Tier 1 bytecode gets hot enough, it gets translated - to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). -* The Tier 2 IR uses the same stack-based VM as Tier 1, but the - instruction format is better suited to translation to machine code. +2to3 +---- -* We have several optimization passes for Tier 2 IR, which are applied - before it is interpreted or translated to machine code. +* Remove the :program:`2to3` program and the :mod:`!lib2to3` module, + previously deprecated in Python 3.11. + (Contributed by Victor Stinner in :gh:`104780`.) -* There is a Tier 2 interpreter, but it is mostly intended for debugging - the earlier stages of the optimization pipeline. If the JIT is not - enabled, the Tier 2 interpreter can be invoked by passing Python the - ``-X uops`` option or by setting the ``PYTHON_UOPS`` environment - variable to ``1``. -* When the ``--enable-experimental-jit`` option is used, the optimized - Tier 2 IR is translated to machine code, which is then executed. - This does not require additional runtime options. +builtins +-------- -* The machine code translation process uses an architecture called - *copy-and-patch*. It has no runtime dependencies, but there is a new - build-time dependency on LLVM. +* Remove support for chained :class:`classmethod` descriptors + (introduced in :gh:`63272`). + These can no longer be used to wrap other descriptors, + such as :class:`property`. + The core design of this feature was flawed and led to several problems. + To "pass-through" a :class:`classmethod`, consider using + the :attr:`!__wrapped__` attribute that was added in Python 3.10. + (Contributed by Raymond Hettinger in :gh:`89519`.) + +* Raise a :exc:`RuntimeError` when calling :meth:`frame.clear` + on a suspended frame (as has always been the case for an executing frame). + (Contributed by Irit Katriel in :gh:`79932`.) -(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. -Tier 2 IR by Mark Shannon and Guido van Rossum. -Tier 2 optimizer by Ken Jin.) +configparser +------------ -Deprecated -========== - -* :mod:`array`: :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, - emits :exc:`DeprecationWarning` since 3.13 - and will be removed in Python 3.16. - Use the ``'w'`` format code instead. - (contributed by Hugo van Kemenade in :gh:`80480`) - -* :mod:`ctypes`: Deprecate undocumented :func:`!ctypes.SetPointerType` - and :func:`!ctypes.ARRAY` functions. - Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. - (Contributed by Victor Stinner in :gh:`105733`.) - -* :mod:`decimal`: Deprecate non-standard format specifier "N" for - :class:`decimal.Decimal`. - It was not documented and only supported in the C implementation. - (Contributed by Serhiy Storchaka in :gh:`89902`.) - -* :mod:`dis`: The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check - membership in :data:`~dis.hasarg` instead. - (Contributed by Irit Katriel in :gh:`109319`.) - -* :mod:`getopt` and :mod:`optparse` modules: They are now - :term:`soft deprecated`: the :mod:`argparse` module should be used for new projects. - Previously, the :mod:`optparse` module was already deprecated, its removal - was not scheduled, and no warnings was emitted: so there is no change in - practice. - (Contributed by Victor Stinner in :gh:`106535`.) - -* :mod:`gettext`: Emit deprecation warning for non-integer numbers in - :mod:`gettext` functions and methods that consider plural forms even if the - translation was not found. - (Contributed by Serhiy Storchaka in :gh:`88434`.) - -* :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a - :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI - HTTP servers have been out of favor for a very long time. This code was - outdated, unmaintained, and rarely used. It has a high potential for both - security and functionality bugs. This includes removal of the ``--cgi`` - flag to the ``python -m http.server`` command line in 3.15. +* Remove the undocumented :class:`!LegacyInterpolation` class, + deprecated in the docstring since Python 3.2, + and at runtime since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104886`.) -* :mod:`pathlib`: - :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for - removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved - paths on Windows. -* :mod:`pydoc`: Deprecate undocumented :func:`!pydoc.ispackage` function. - (Contributed by Zackery Spytz in :gh:`64020`.) +importlib.metadata +------------------ -* :mod:`sqlite3`: Passing more than one positional argument to - :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is - deprecated. The remaining parameters will become keyword-only in Python 3.15. +* Remove deprecated subscript (:meth:`~object.__getitem__`) access for + :ref:`EntryPoint ` objects. + (Contributed by Jason R. Coombs in :gh:`113175`.) - Deprecate passing name, number of arguments, and the callable as keyword - arguments for the following :class:`sqlite3.Connection` APIs: - * :meth:`~sqlite3.Connection.create_function` - * :meth:`~sqlite3.Connection.create_aggregate` +locale +------ - Deprecate passing the callback callable by keyword for the following - :class:`sqlite3.Connection` APIs: +* Remove the :func:`!locale.resetlocale` function, deprecated in Python 3.11. + Use ``locale.setlocale(locale.LC_ALL, "")`` instead. + (Contributed by Victor Stinner in :gh:`104783`.) - * :meth:`~sqlite3.Connection.set_authorizer` - * :meth:`~sqlite3.Connection.set_progress_handler` - * :meth:`~sqlite3.Connection.set_trace_callback` - The affected parameters will become positional-only in Python 3.15. +opcode +------ - (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) +* Move :attr:`!opcode.ENABLE_SPECIALIZATION` to :attr:`!_opcode.ENABLE_SPECIALIZATION`. + This field was added in 3.12, it was never documented, + and is not intended for external use. + (Contributed by Irit Katriel in :gh:`105481`.) -* :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function. - Replace it with the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. - (Contributed by Inada Naoki in :gh:`73427`.) +* Remove :func:`!opcode.is_pseudo`, :attr:`!opcode.MIN_PSEUDO_OPCODE`, + and :attr:`!opcode.MAX_PSEUDO_OPCODE`, which were added in Python 3.12, + but were neither documented nor exposed through :mod:`dis`, + and were not intended to be used externally. + (Contributed by Irit Katriel in :gh:`105481`.) -* :mod:`traceback`: The field *exc_type* of :class:`traceback.TracebackException` - is deprecated. Use *exc_type_str* instead. -* :mod:`typing`: +pathlib +------- - * Creating a :class:`typing.NamedTuple` class using keyword arguments to denote - the fields (``NT = NamedTuple("NT", x=int, y=int)``) is deprecated, and will - be disallowed in Python 3.15. Use the class-based syntax or the functional - syntax instead. (Contributed by Alex Waygood in :gh:`105566`.) - - * When using the functional syntax to create a :class:`typing.NamedTuple` - class or a :class:`typing.TypedDict` class, failing to pass a value to the - 'fields' parameter (``NT = NamedTuple("NT")`` or ``TD = TypedDict("TD")``) is - deprecated. Passing ``None`` to the 'fields' parameter - (``NT = NamedTuple("NT", None)`` or ``TD = TypedDict("TD", None)``) is also - deprecated. Both will be disallowed in Python 3.15. To create a NamedTuple - class with 0 fields, use ``class NT(NamedTuple): pass`` or - ``NT = NamedTuple("NT", [])``. To create a TypedDict class with 0 fields, use - ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. - (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) +* Remove the ability to use :class:`~pathlib.Path` objects as context managers. + This functionality was deprecated and has had no effect since Python 3.9. + (Contributed by Barney Gale in :gh:`83863`.) - * :func:`typing.no_type_check_decorator` is deprecated, and scheduled for - removal in Python 3.15. After eight years in the :mod:`typing` module, it - has yet to be supported by any major type checkers. - (Contributed by Alex Waygood in :gh:`106309`.) - * :data:`typing.AnyStr` is deprecated. In Python 3.16, it will be removed from - ``typing.__all__``, and a :exc:`DeprecationWarning` will be emitted when it - is imported or accessed. It will be removed entirely in Python 3.18. Use - the new :ref:`type parameter syntax ` instead. - (Contributed by Michael The in :gh:`107116`.) +re +-- -* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` - methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. - They will be removed in Python 3.15. - (Contributed by Victor Stinner in :gh:`105096`.) +* Remove the undocumented, deprecated, and broken + :func:`!re.template` function and :attr:`!re.TEMPLATE` / :attr:`!re.T` flag. + (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) -* Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError` - (as has always been the case for an executing frame). - (Contributed by Irit Katriel in :gh:`79932`.) -* Assignment to a function's :attr:`~function.__code__` attribute where the new code - object's type does not match the function's type, is deprecated. The - different types are: plain function, generator, async generator and - coroutine. - (Contributed by Irit Katriel in :gh:`81137`.) +tkinter.tix +----------- +* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. + The third-party Tix library which the module wrapped is unmaintained. + (Contributed by Zachary Ware in :gh:`75552`.) -Pending Removal in Python 3.14 ------------------------------- -* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction` are deprecated - and will be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`92248`.) +turtle +------ -* :mod:`ast`: The following features have been deprecated in documentation - since Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at - runtime when they are accessed or used, and will be removed in Python 3.14: +* Remove the :meth:`!RawTurtle.settiltangle` method, + deprecated in the documentation since Python 3.1 + and at runtime since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104876`.) - * :class:`!ast.Num` - * :class:`!ast.Str` - * :class:`!ast.Bytes` - * :class:`!ast.NameConstant` - * :class:`!ast.Ellipsis` - Use :class:`ast.Constant` instead. - (Contributed by Serhiy Storchaka in :gh:`90953`.) +typing +------ -* :mod:`collections.abc`: Deprecated :class:`~collections.abc.ByteString`. - Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, - or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) +* Remove the :mod:`!typing.io` and :mod:`!typing.re` namespaces, + deprecated since Python 3.8. + The items in those namespaces can be imported directly + from the :mod:`typing` module. + (Contributed by Sebastian Rittau in :gh:`92871`.) -* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. - (Contributed by Alan Williams in :gh:`72346`.) +* Remove the keyword-argument method of creating + :class:`~typing.TypedDict` types, deprecated in Python 3.11. + (Contributed by Tomas Roun in :gh:`104786`.) -* :mod:`importlib`: ``__package__`` and ``__cached__`` will cease to be set or - taken into consideration by the import system (:gh:`97879`). -* :mod:`importlib.abc` deprecated classes: +unittest +-------- - * :class:`!importlib.abc.ResourceReader` - * :class:`!importlib.abc.Traversable` - * :class:`!importlib.abc.TraversableResources` +* Remove the following :mod:`unittest` functions, deprecated in Python 3.11: - Use :mod:`importlib.resources.abc` classes instead: + * :func:`!unittest.findTestCases` + * :func:`!unittest.makeSuite` + * :func:`!unittest.getTestCaseNames` - * :class:`importlib.resources.abc.Traversable` - * :class:`importlib.resources.abc.TraversableResources` + Use :class:`~unittest.TestLoader` methods instead: - (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + * :meth:`~unittest.TestLoader.loadTestsFromModule` + * :meth:`~unittest.TestLoader.loadTestsFromTestCase` + * :meth:`~unittest.TestLoader.getTestCaseNames` -* :mod:`itertools` had undocumented, inefficient, historically buggy, - and inconsistent support for copy, deepcopy, and pickle operations. - This will be removed in 3.14 for a significant reduction in code - volume and maintenance burden. - (Contributed by Raymond Hettinger in :gh:`101588`.) + (Contributed by Hugo van Kemenade in :gh:`104835`.) -* :mod:`multiprocessing`: The default start method will change to a safer one on - Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently - the default (:gh:`84559`). Adding a runtime warning about this was deemed too - disruptive as the majority of code is not expected to care. Use the - :func:`~multiprocessing.get_context` or - :func:`~multiprocessing.set_start_method` APIs to explicitly specify when - your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. +* Remove the untested and undocumented :meth:`!TestProgram.usageExit` + method, deprecated in Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104992`.) -* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and - :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is - deprecated. -* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` - now raise :exc:`DeprecationWarning`; - use :func:`importlib.util.find_spec` instead. - (Contributed by Nikita Sobolev in :gh:`97850`.) +urllib +------ -* :mod:`pty`: +* Remove the *cafile*, *capath*, and *cadefault* parameters of the + :func:`urllib.request.urlopen` function, deprecated in Python 3.6. + Use the *context* parameter instead with an :class:`~ssl.SSLContext` instance. + The :meth:`ssl.SSLContext.load_cert_chain` function + can be used to load specific certificates, + or let :func:`ssl.create_default_context` select + the operating system's trusted certificate authority (CA) certificates. + (Contributed by Victor Stinner in :gh:`105382`.) - * ``master_open()``: use :func:`pty.openpty`. - * ``slave_open()``: use :func:`pty.openpty`. -* :func:`shutil.rmtree` *onerror* parameter is deprecated in 3.12, - and will be removed in 3.14: use the *onexc* parameter instead. +webbrowser +---------- -* :mod:`sqlite3`: +* Remove the untested and undocumented :class:`!MacOSX` class, + deprecated in Python 3.11. + Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. + (Contributed by Hugo van Kemenade in :gh:`104804`.) - * :data:`~sqlite3.version` and :data:`~sqlite3.version_info`. +* Remove the deprecated :attr:`!MacOSXOSAScript._name` attribute. + Use the :attr:`MacOSXOSAScript.name ` + attribute instead. + (Contributed by Nikita Sobolev in :gh:`105546`.) - * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` - if :ref:`named placeholders ` are used and - *parameters* is a sequence instead of a :class:`dict`. - * date and datetime adapter, date and timestamp converter: - see the :mod:`sqlite3` documentation for suggested replacement recipes. +New Deprecations +================ -* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was - deprecated in :pep:`626` - since 3.10 and was planned to be removed in 3.12, - but it only got a proper :exc:`DeprecationWarning` in 3.12. - May be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`101866`.) +* :ref:`User-defined functions `: -* :mod:`typing`: :class:`~typing.ByteString`, deprecated since Python 3.9, - now causes a :exc:`DeprecationWarning` to be emitted when it is used. + * Deprecate assignment to a function's :attr:`~function.__code__` attribute, + where the new code object's type does not match the function's type. + The different types are: + plain function, generator, async generator, and coroutine. + (Contributed by Irit Katriel in :gh:`81137`.) -* :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a - public API. - (Contributed by Gregory P. Smith in :gh:`88168`.) +* :mod:`array`: -* :mod:`xml.etree.ElementTree`: Testing the truth value of an - :class:`~xml.etree.ElementTree.Element` is deprecated and will raise an - exception in Python 3.14. + * Deprecate the ``'u'`` format code (:c:type:`wchar_t`) at runtime. + This format code has been deprecated in documentation since Python 3.3, + and will be removed in Python 3.16. + Use the ``'w'`` format code (:c:type:`Py_UCS4`) + for Unicode characters instead. + (Contributed by Hugo van Kemenade in :gh:`80480`.) -Pending Removal in Python 3.15 ------------------------------- +* :mod:`ctypes`: -* :class:`http.server.CGIHTTPRequestHandler` will be removed along with its - related ``--cgi`` flag to ``python -m http.server``. It was obsolete and - rarely used. No direct replacement exists. *Anything* is better than CGI - to interface a web server with a request handler. + * Deprecate the undocumented :func:`!SetPointerType` function, + to be removed in Python 3.15. + (Contributed by Victor Stinner in :gh:`105733`.) -* :class:`locale`: :func:`locale.getdefaultlocale` was deprecated in Python 3.11 - and originally planned for removal in Python 3.13 (:gh:`90817`), - but removal has been postponed to Python 3.15. - Use :func:`locale.setlocale()`, :func:`locale.getencoding()` and - :func:`locale.getlocale()` instead. - (Contributed by Hugo van Kemenade in :gh:`111187`.) + * :term:`Soft-deprecate ` the :func:`~ctypes.ARRAY` + function in favour of ``type * length`` multiplication. + (Contributed by Victor Stinner in :gh:`105733`.) -* :mod:`pathlib`: - :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for - removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved - paths on Windows. - -* :mod:`threading`: - Passing any arguments to :func:`threading.RLock` is now deprecated. - C version allows any numbers of args and kwargs, - but they are just ignored. Python version does not allow any arguments. - All arguments will be removed from :func:`threading.RLock` in Python 3.15. - (Contributed by Nikita Sobolev in :gh:`102029`.) - -* :class:`typing.NamedTuple`: - - * The undocumented keyword argument syntax for creating NamedTuple classes - (``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed in - 3.15. Use the class-based syntax or the functional syntax instead. - - * When using the functional syntax to create a NamedTuple class, failing to - pass a value to the 'fields' parameter (``NT = NamedTuple("NT")``) is - deprecated. Passing ``None`` to the 'fields' parameter - (``NT = NamedTuple("NT", None)``) is also deprecated. Both will be - disallowed in Python 3.15. To create a NamedTuple class with 0 fields, use - ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", [])``. - -* :class:`typing.TypedDict`: When using the functional syntax to create a - TypedDict class, failing to pass a value to the 'fields' parameter (``TD = - TypedDict("TD")``) is deprecated. Passing ``None`` to the 'fields' parameter - (``TD = TypedDict("TD", None)``) is also deprecated. Both will be disallowed - in Python 3.15. To create a TypedDict class with 0 fields, use ``class - TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. - -* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` - methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. - They will be removed in Python 3.15. - (Contributed by Victor Stinner in :gh:`105096`.) - -Pending Removal in Python 3.16 ------------------------------- - -* :class:`array.array` ``'u'`` type (:c:type:`wchar_t`): - use the ``'w'`` type instead (``Py_UCS4``). - -Pending Removal in Future Versions ----------------------------------- - -The following APIs were deprecated in earlier Python versions and will be removed, -although there is currently no date scheduled for their removal. - -* :mod:`argparse`: Nesting argument groups and nesting mutually exclusive - groups are deprecated. - -* :mod:`builtins`: - - * ``~bool``, bitwise inversion on bool. - * ``bool(NotImplemented)``. - * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` - signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, - the single argument signature. - * Currently Python accepts numeric literals immediately followed by keywords, - for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and - ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as - ``[0x1 for x in y]`` or ``[0x1f or x in y]``). A syntax warning is raised - if the numeric literal is immediately followed by one of keywords - :keyword:`and`, :keyword:`else`, :keyword:`for`, :keyword:`if`, - :keyword:`in`, :keyword:`is` and :keyword:`or`. In a future release it - will be changed to a syntax error. (:gh:`87999`) - * Support for ``__index__()`` and ``__int__()`` method returning non-int type: - these methods will be required to return an instance of a strict subclass of - :class:`int`. - * Support for ``__float__()`` method returning a strict subclass of - :class:`float`: these methods will be required to return an instance of - :class:`float`. - * Support for ``__complex__()`` method returning a strict subclass of - :class:`complex`: these methods will be required to return an instance of - :class:`complex`. - * Delegation of ``int()`` to ``__trunc__()`` method. - -* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are - deprecated and replaced by :data:`calendar.JANUARY` and - :data:`calendar.FEBRUARY`. - (Contributed by Prince Roshan in :gh:`103636`.) - -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method - instead. - -* :mod:`datetime`: - - * :meth:`~datetime.datetime.utcnow`: - use ``datetime.datetime.now(tz=datetime.UTC)``. - * :meth:`~datetime.datetime.utcfromtimestamp`: - use ``datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)``. - -* :mod:`gettext`: Plural value must be an integer. - -* :mod:`importlib`: - - * ``load_module()`` method: use ``exec_module()`` instead. - * :func:`~importlib.util.cache_from_source` *debug_override* parameter is - deprecated: use the *optimization* parameter instead. - -* :mod:`importlib.metadata`: - - * ``EntryPoints`` tuple interface. - * Implicit ``None`` on return values. - -* :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use - BytesIO and binary mode instead. - -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. - -* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is - deprecated, use an exception instance. - -* :mod:`re`: More strict rules are now applied for numerical group references - and group names in regular expressions. Only sequence of ASCII digits is now - accepted as a numerical reference. The group name in bytes patterns and - replacement strings can now only contain ASCII letters and digits and - underscore. - (Contributed by Serhiy Storchaka in :gh:`91760`.) - -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - -* :mod:`ssl` options and protocols: - - * :class:`ssl.SSLContext` without protocol argument is deprecated. - * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and - :meth:`!selected_npn_protocol` are deprecated: use ALPN - instead. - * ``ssl.OP_NO_SSL*`` options - * ``ssl.OP_NO_TLS*`` options - * ``ssl.PROTOCOL_SSLv3`` - * ``ssl.PROTOCOL_TLS`` - * ``ssl.PROTOCOL_TLSv1`` - * ``ssl.PROTOCOL_TLSv1_1`` - * ``ssl.PROTOCOL_TLSv1_2`` - * ``ssl.TLSVersion.SSLv3`` - * ``ssl.TLSVersion.TLSv1`` - * ``ssl.TLSVersion.TLSv1_1`` - -* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and - ignored. - -* :mod:`threading` methods: - - * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. - * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. - * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: - use :attr:`threading.Thread.daemon` attribute. - * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: - use :attr:`threading.Thread.name` attribute. - * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. - * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. - -* :class:`typing.Text` (:gh:`92332`). - -* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value - that is not ``None`` from a test case. - -* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead - - * ``splitattr()`` - * ``splithost()`` - * ``splitnport()`` - * ``splitpasswd()`` - * ``splitport()`` - * ``splitquery()`` - * ``splittag()`` - * ``splittype()`` - * ``splituser()`` - * ``splitvalue()`` - * ``to_bytes()`` +* :mod:`decimal`: -* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and - :class:`~urllib.request.FancyURLopener` style of invoking requests is - deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. + * Deprecate the non-standard and undocumented :class:`~decimal.Decimal` + format specifier ``'N'``, + which is only supported in the :mod:`!decimal` module's C implementation. + (Contributed by Serhiy Storchaka in :gh:`89902`.) -* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial - writes. +* :mod:`dis`: -* :meth:`zipimport.zipimporter.load_module` is deprecated: - use :meth:`~zipimport.zipimporter.exec_module` instead. + * Deprecate the :attr:`!HAVE_ARGUMENT` separator. + Check membership in :data:`~dis.hasarg` instead. + (Contributed by Irit Katriel in :gh:`109319`.) +* :mod:`getopt` and :mod:`optparse`: -Removed -======= + * Both modules are now :term:`soft deprecated`, + with :mod:`argparse` preferred for new projects. + This is a new soft-deprecation for the :mod:`!getopt` module, + whereas the :mod:`!optparse` module was already *de facto* soft deprecated. + (Contributed by Victor Stinner in :gh:`106535`.) -.. _whatsnew313-pep594: +* :mod:`gettext`: -PEP 594: dead batteries ------------------------ + * Deprecate non-integer numbers as arguments to functions and methods + that consider plural forms in the :mod:`!gettext` module, + even if no translation was found. + (Contributed by Serhiy Storchaka in :gh:`88434`.) -* :pep:`594` removed 19 modules from the standard library, - deprecated in Python 3.11: +* :mod:`glob`: - * :mod:`!aifc`. - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate the undocumented :func:`!glob0` and :func:`!glob1` functions. + Use :func:`~glob.glob` and pass a :term:`path-like object` specifying + the root directory to the *root_dir* parameter instead. + (Contributed by Barney Gale in :gh:`117337`.) - * :mod:`!audioop`. - (Contributed by Victor Stinner in :gh:`104773`.) +* :mod:`http.server`: - * :mod:`!chunk`. - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate :class:`~http.server.CGIHTTPRequestHandler`, + to be removed in Python 3.15. + Process-based CGI HTTP servers have been out of favor for a very long time. + This code was outdated, unmaintained, and rarely used. + It has a high potential for both security and functionality bugs. + (Contributed by Gregory P. Smith in :gh:`109096`.) - * :mod:`!cgi` and :mod:`!cgitb`. + * Deprecate the :option:`!--cgi` flag to + the :program:`python -m http.server` command-line interface, + to be removed in Python 3.15. + (Contributed by Gregory P. Smith in :gh:`109096`.) - * ``cgi.FieldStorage`` can typically be replaced with - :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, - and the :mod:`email.message` module or `multipart - `__ PyPI project for ``POST`` and - ``PUT``. +* :mod:`mimetypes`: - * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` - directly on the desired query string, except for ``multipart/form-data`` - input, which can be handled as described for ``cgi.parse_multipart()``. + * :term:`Soft-deprecate ` file path arguments + to :func:`~mimetypes.guess_type`, + use :func:`~mimetypes.guess_file_type` instead. + (Contributed by Serhiy Storchaka in :gh:`66543`.) - * ``cgi.parse_header()`` can be replaced with the functionality in the - :mod:`email` package, which implements the same MIME RFCs. For example, - with :class:`email.message.EmailMessage`:: +* :mod:`re`: - from email.message import EmailMessage - msg = EmailMessage() - msg['content-type'] = 'application/json; charset="utf8"' - main, params = msg.get_content_type(), msg['content-type'].params + * Deprecate passing the optional *maxsplit*, *count*, or *flags* arguments + as positional arguments to the module-level + :func:`~re.split`, :func:`~re.sub`, and :func:`~re.subn` functions. + These parameters will become :ref:`keyword-only ` + in a future version of Python. + (Contributed by Serhiy Storchaka in :gh:`56166`.) - * ``cgi.parse_multipart()`` can be replaced with the functionality in the - :mod:`email` package (e.g. :class:`email.message.EmailMessage` and - :class:`email.message.Message`) which implements the same MIME RFCs, or - with the `multipart `__ PyPI project. +* :mod:`pathlib`: - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate :meth:`.PurePath.is_reserved`, + to be removed in Python 3.15. + Use :func:`os.path.isreserved` to detect reserved paths on Windows. + (Contributed by Barney Gale in :gh:`88569`.) - * :mod:`!crypt` module and its private :mod:`!_crypt` extension. - The :mod:`hashlib` module is a potential replacement for certain use cases. - Otherwise, the following PyPI projects can be used: +* :mod:`platform`: - * `bcrypt `_: - Modern password hashing for your software and your servers. - * `passlib `_: - Comprehensive password hashing framework supporting over 30 schemes. - * `argon2-cffi `_: - The secure Argon2 password hashing algorithm. - * `legacycrypt `_: - Wrapper to the POSIX crypt library call and associated functionality. + * Deprecate :func:`~platform.java_ver`, + to be removed in Python 3.15. + This function is only useful for Jython support, has a confusing API, + and is largely untested. + (Contributed by Nikita Sobolev in :gh:`116349`.) - (Contributed by Victor Stinner in :gh:`104773`.) +* :mod:`pydoc`: - * :mod:`!imghdr`: use the projects - `filetype `_, - `puremagic `_, - or `python-magic `_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate the undocumented :func:`!ispackage` function. + (Contributed by Zackery Spytz in :gh:`64020`.) - * :mod:`!mailcap`. - The :mod:`mimetypes` module provides an alternative. - (Contributed by Victor Stinner in :gh:`104773`.) +* :mod:`sqlite3`: - * :mod:`!msilib`. - (Contributed by Zachary Ware in :gh:`104773`.) + * Deprecate passing more than one positional argument to + the :func:`~sqlite3.connect` function + and the :class:`~sqlite3.Connection` constructor. + The remaining parameters will become keyword-only in Python 3.15. + (Contributed by Erlend E. Aasland in :gh:`107948`.) - * :mod:`!nis`. - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate passing name, number of arguments, and the callable as keyword + arguments for :meth:`.Connection.create_function` + and :meth:`.Connection.create_aggregate` + These parameters will become positional-only in Python 3.15. + (Contributed by Erlend E. Aasland in :gh:`108278`.) - * :mod:`!nntplib`: - the `PyPI nntplib project `_ - can be used instead. - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate passing the callback callable by keyword for the + :meth:`~sqlite3.Connection.set_authorizer`, + :meth:`~sqlite3.Connection.set_progress_handler`, and + :meth:`~sqlite3.Connection.set_trace_callback` + :class:`~sqlite3.Connection` methods. + The callback callables will become positional-only in Python 3.15. + (Contributed by Erlend E. Aasland in :gh:`108278`.) - * :mod:`!ossaudiodev`: use the - `pygame project `_ for audio playback. - (Contributed by Victor Stinner in :gh:`104780`.) +* :mod:`sys`: - * :mod:`!pipes`: use the :mod:`subprocess` module instead. - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate the :func:`~sys._enablelegacywindowsfsencoding` function, + to be removed in Python 3.16. + Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. + (Contributed by Inada Naoki in :gh:`73427`.) - * :mod:`!sndhdr`: use the projects - `filetype `_, - `puremagic `_, or - `python-magic `_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) +* :mod:`tarfile`: - * :mod:`!spwd`: - the `python-pam project `_ - can be used instead. - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate the undocumented and unused :attr:`!TarFile.tarfile` attribute, + to be removed in Python 3.16. + (Contributed in :gh:`115256`.) - * :mod:`!sunau`. - (Contributed by Victor Stinner in :gh:`104773`.) +* :mod:`traceback`: - * :mod:`!telnetlib`, use the projects - `telnetlib3 `_ or - `Exscript `_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate the :attr:`.TracebackException.exc_type` attribute. + Use :attr:`.TracebackException.exc_type_str` instead. + (Contributed by Irit Katriel in :gh:`112332`.) - * :mod:`!uu`: the :mod:`base64` module is a modern alternative. - (Contributed by Victor Stinner in :gh:`104773`.) +* :mod:`typing`: - * :mod:`!xdrlib`. - (Contributed by Victor Stinner in :gh:`104773`.) + * Deprecate the undocumented keyword argument syntax for creating + :class:`~typing.NamedTuple` classes + (e.g. ``Point = NamedTuple("Point", x=int, y=int)``), + to be removed in Python 3.15. + Use the class-based syntax or the functional syntax instead. + (Contributed by Alex Waygood in :gh:`105566`.) + + * Deprecate omitting the *fields* parameter when creating + a :class:`~typing.NamedTuple` or :class:`typing.TypedDict` class, + and deprecate passing ``None`` to the *fields* parameter of both types. + Python 3.15 will require a valid sequence for the *fields* parameter. + To create a NamedTuple class with zero fields, + use ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", ())``. + To create a TypedDict class with zero fields, + use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. + (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) -2to3 ----- + * Deprecate the :func:`typing.no_type_check_decorator` decorator function, + to be removed in in Python 3.15. + After eight years in the :mod:`typing` module, + it has yet to be supported by any major type checker. + (Contributed by Alex Waygood in :gh:`106309`.) -* Remove the ``2to3`` program and the :mod:`!lib2to3` module, - deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104780`.) + * Deprecate :data:`typing.AnyStr`. + In Python 3.16, it will be removed from ``typing.__all__``, + and a :exc:`DeprecationWarning` will be emitted at runtime + when it is imported or accessed. + It will be removed entirely in Python 3.18. + Use the new :ref:`type parameter syntax ` instead. + (Contributed by Michael The in :gh:`107116`.) -configparser ------------- +* :mod:`wave`: -* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, - deprecated in the docstring since Python 3.2, - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104886`.) + * Deprecate the :meth:`~wave.Wave_read.getmark`, :meth:`!setmark`, + and :meth:`~wave.Wave_read.getmarkers` methods of + the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes, + to be removed in Python 3.15. + (Contributed by Victor Stinner in :gh:`105096`.) -importlib ---------- +.. Add deprecations above alphabetically, not here at the end. -* Remove :mod:`importlib.resources` deprecated methods: +.. include:: ../deprecations/pending-removal-in-3.14.rst - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` +.. include:: ../deprecations/pending-removal-in-3.15.rst - Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy - `_ - for migration advice. - (Contributed by Jason R. Coombs in :gh:`106532`.) +.. include:: ../deprecations/pending-removal-in-3.16.rst -* Remove deprecated :meth:`~object.__getitem__` access for - :class:`!importlib.metadata.EntryPoint` objects. - (Contributed by Jason R. Coombs in :gh:`113175`.) +.. include:: ../deprecations/pending-removal-in-future.rst -locale ------- +CPython Bytecode Changes +======================== -* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: - use ``locale.setlocale(locale.LC_ALL, "")`` instead. - (Contributed by Victor Stinner in :gh:`104783`.) +* The oparg of :opcode:`YIELD_VALUE` is now + ``1`` if the yield is part of a yield-from or await, and ``0`` otherwise. + The oparg of :opcode:`RESUME` was changed to add a bit indicating + if the except-depth is 1, which is needed to optimize closing of generators. + (Contributed by Irit Katriel in :gh:`111354`.) -logging -------- -* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and - ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated - since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` - method, :meth:`!logging.LoggerAdapter.warning` method and - :func:`logging.warning` function. - (Contributed by Victor Stinner in :gh:`105376`.) +C API Changes +============= -pathlib -------- +New Features +------------ -* Remove support for using :class:`pathlib.Path` objects as context managers. - This functionality was deprecated and made a no-op in Python 3.9. +* Add the :ref:`PyMonitoring C API ` + for generating :pep:`669` monitoring events: + + * :c:type:`PyMonitoringState` + * :c:func:`PyMonitoring_FirePyStartEvent` + * :c:func:`PyMonitoring_FirePyResumeEvent` + * :c:func:`PyMonitoring_FirePyReturnEvent` + * :c:func:`PyMonitoring_FirePyYieldEvent` + * :c:func:`PyMonitoring_FireCallEvent` + * :c:func:`PyMonitoring_FireLineEvent` + * :c:func:`PyMonitoring_FireJumpEvent` + * :c:func:`PyMonitoring_FireBranchEvent` + * :c:func:`PyMonitoring_FireCReturnEvent` + * :c:func:`PyMonitoring_FirePyThrowEvent` + * :c:func:`PyMonitoring_FireRaiseEvent` + * :c:func:`PyMonitoring_FireCRaiseEvent` + * :c:func:`PyMonitoring_FireReraiseEvent` + * :c:func:`PyMonitoring_FireExceptionHandledEvent` + * :c:func:`PyMonitoring_FirePyUnwindEvent` + * :c:func:`PyMonitoring_FireStopIterationEvent` + * :c:func:`PyMonitoring_EnterScope` + * :c:func:`PyMonitoring_ExitScope` + + (Contributed by Irit Katriel in :gh:`111997`). + +* Add :c:type:`PyMutex`, a lightweight mutex that occupies a single byte, + and the new :c:func:`PyMutex_Lock` and :c:func:`PyMutex_Unlock` functions. + :c:func:`!PyMutex_Lock` will release the :term:`GIL` (if currently held) + if the operation needs to block. + (Contributed by Sam Gross in :gh:`108724`.) + +* Add the :ref:`PyTime C API ` to provide access to system clocks: + + * :c:type:`PyTime_t`. + * :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX`. + * :c:func:`PyTime_AsSecondsDouble`. + * :c:func:`PyTime_Monotonic`. + * :c:func:`PyTime_MonotonicRaw`. + * :c:func:`PyTime_PerfCounter`. + * :c:func:`PyTime_PerfCounterRaw`. + * :c:func:`PyTime_Time`. + * :c:func:`PyTime_TimeRaw`. + + (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) + +* Add the :c:func:`PyDict_ContainsString` function + with the same behavior as :c:func:`PyDict_Contains`, + but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + (Contributed by Victor Stinner in :gh:`108314`.) -re --- +* Add the :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` + functions, + which behave similarly to :c:func:`PyDict_GetItemWithError`, + but return a :term:`strong reference` instead of a :term:`borrowed reference`. + Moreover, these functions return ``-1`` on error, + removing the need to check :c:func:`!PyErr_Occurred`. + (Contributed by Victor Stinner in :gh:`106004`.) -* Remove undocumented, never working, and deprecated ``re.template`` function - and ``re.TEMPLATE`` flag (and ``re.T`` alias). - (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) +* Add the :c:func:`PyDict_SetDefaultRef` function, + which behaves similarly to :c:func:`PyDict_SetDefault`, + but returns a :term:`strong reference` instead of a :term:`borrowed reference`. + This function returns ``-1`` on error, + ``0`` on insertion, + and ``1`` if the key was already present in the dictionary. + (Contributed by Sam Gross in :gh:`112066`.) -tkinter -------- +* Add the :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions + to remove a key from a dictionary and optionally return the removed value. + This is similar to :meth:`dict.pop`, + though there is no default value, + and :exc:`KeyError` is not raised for missing keys. + (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) -* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The - third-party Tix library which the module wrapped is unmaintained. - (Contributed by Zachary Ware in :gh:`75552`.) +* Add the :c:func:`PyMapping_GetOptionalItem` + and :c:func:`PyMapping_GetOptionalItemString` functions + as alternatives to :c:func:`PyObject_GetItem` + and :c:func:`PyMapping_GetItemString` respectively. + The new functions do not raise :exc:`KeyError` + if the requested key is missing from the mapping. + These variants are more convenient and faster + if a missing key should not be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106307`.) -turtle ------- +* Add the :c:func:`PyObject_GetOptionalAttr` + and :c:func:`PyObject_GetOptionalAttrString` functions + as alternatives to :c:func:`PyObject_GetAttr` + and :c:func:`PyObject_GetAttrString` respectively. + The new functions do not raise :exc:`AttributeError` + if the requested attribute is not found on the object. + These variants are more convenient and faster + if the missing attribute should not be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106521`.) -* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, - deprecated in docs since Python 3.1 - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104876`.) +* Add the :c:func:`PyErr_FormatUnraisable` function + as an extension to :c:func:`PyErr_WriteUnraisable` + that allows customizing the warning message. + (Contributed by Serhiy Storchaka in :gh:`108082`.) -typing ------- +* Add new functions that return a :term:`strong reference` instead of + a :term:`borrowed reference` for frame locals, globals, and builtins, + as part of :ref:`PEP 667 `: -* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, - are now removed. The items in those namespaces can be imported directly - from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) + * :c:func:`PyEval_GetFrameBuiltins` replaces :c:func:`PyEval_GetBuiltins` + * :c:func:`PyEval_GetFrameGlobals` replaces :c:func:`PyEval_GetGlobals` + * :c:func:`PyEval_GetFrameLocals` replaces :c:func:`PyEval_GetLocals` -* Remove support for the keyword-argument method of creating - :class:`typing.TypedDict` types, deprecated in Python 3.11. - (Contributed by Tomas Roun in :gh:`104786`.) + (Contributed by Mark Shannon and Tian Gao in :gh:`74929`.) -unittest --------- +* Add the :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` + functions to get :term:`strong ` + or :term:`borrowed ` references to constants. + For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns a strong reference + to the constant zero. + (Contributed by Victor Stinner in :gh:`115754`.) -* Removed the following :mod:`unittest` functions, deprecated in Python 3.11: +* Add the :c:func:`PyImport_AddModuleRef` function + as a replacement for :c:func:`PyImport_AddModule` + that returns a :term:`strong reference` instead of a :term:`borrowed reference`. + (Contributed by Victor Stinner in :gh:`105922`.) - * :func:`!unittest.findTestCases` - * :func:`!unittest.makeSuite` - * :func:`!unittest.getTestCaseNames` +* Add the :c:func:`Py_IsFinalizing` function to check + whether the main Python interpreter is + :term:`shutting down `. + (Contributed by Victor Stinner in :gh:`108014`.) - Use :class:`~unittest.TestLoader` methods instead: +* Add the :c:func:`PyList_GetItemRef` function + as a replacement for :c:func:`PyList_GetItem` + that returns a :term:`strong reference` instead of a :term:`borrowed reference`. + (Contributed by Sam Gross in :gh:`114329`.) - * :meth:`unittest.TestLoader.loadTestsFromModule` - * :meth:`unittest.TestLoader.loadTestsFromTestCase` - * :meth:`unittest.TestLoader.getTestCaseNames` +* Add the :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions, + mirroring the Python :meth:`!list.extend` and :meth:`!list.clear` methods. + (Contributed by Victor Stinner in :gh:`111138`.) - (Contributed by Hugo van Kemenade in :gh:`104835`.) +* Add the :c:func:`PyLong_AsInt` function. + It behaves similarly to :c:func:`PyLong_AsLong`, + but stores the result in a C :c:expr:`int` instead of a C :c:expr:`long`. + (Contributed by Victor Stinner in :gh:`108014`.) -* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` - method, deprecated in Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104992`.) +* Add the :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes`, + and :c:func:`PyLong_FromUnsignedNativeBytes` functions + to simplify converting between native integer types + and Python :class:`int` objects. + (Contributed by Steve Dower in :gh:`111140`.) -urllib ------- +* Add :c:func:`PyModule_Add` function, which is similar to + :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject`, + but always steals a reference to the value. + (Contributed by Serhiy Storchaka in :gh:`86493`.) -* Remove *cafile*, *capath* and *cadefault* parameters of the - :func:`urllib.request.urlopen` function, deprecated in Python 3.6: use the - *context* parameter instead. Please use - :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - (Contributed by Victor Stinner in :gh:`105382`.) +* Add the :c:func:`PyObject_GenericHash` function + that implements the default hashing function of a Python object. + (Contributed by Serhiy Storchaka in :gh:`113024`.) -webbrowser ----------- +* Add the :c:func:`Py_HashPointer` function to hash a raw pointer. + (Contributed by Victor Stinner in :gh:`111545`.) -* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, - deprecated in Python 3.11. - Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. - (Contributed by Hugo van Kemenade in :gh:`104804`.) +* Add the :c:func:`PyObject_VisitManagedDict` and + :c:func:`PyObject_ClearManagedDict` functions. + which must be called by the traverse and clear functions of a type using + the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. + The `pythoncapi-compat project`_ can be used to + use these functions with Python 3.11 and 3.12. + (Contributed by Victor Stinner in :gh:`107073`.) -* Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. - Use :attr:`webbrowser.MacOSXOSAScript.name ` - attribute instead. - (Contributed by Nikita Sobolev in :gh:`105546`.) +* Add the :c:func:`PyRefTracer_SetTracer` + and :c:func:`PyRefTracer_GetTracer` functions, + which enable tracking object creation and destruction + in the same way that the :mod:`tracemalloc` module does. + (Contributed by Pablo Galindo in :gh:`93502`.) -Others ------- +* Add the :c:func:`PySys_AuditTuple` function + as an alternative to :c:func:`PySys_Audit` + that takes event arguments as a Python :class:`tuple` object. + (Contributed by Victor Stinner in :gh:`85283`.) -* None yet +* Add the :c:func:`PyThreadState_GetUnchecked()` function + as an alternative to :c:func:`PyThreadState_Get()` + that doesn't kill the process with a fatal error if it is ``NULL``. + The caller is responsible for checking if the result is ``NULL``. + (Contributed by Victor Stinner in :gh:`108867`.) -CPython bytecode changes -======================== +* Add the :c:func:`PyType_GetFullyQualifiedName` function + to get the type's fully qualified name. + The module name is prepended if :attr:`type.__module__` is + a string and is not equal to either ``'builtins'`` or ``'__main__'``. + (Contributed by Victor Stinner in :gh:`111696`.) + +* Add the :c:func:`PyType_GetModuleName` function + to get the type's module name. This is equivalent to getting the + :attr:`type.__module__` attribute. + (Contributed by Eric Snow and Victor Stinner in :gh:`111696`.) + +* Add the :c:func:`PyUnicode_EqualToUTF8AndSize` + and :c:func:`PyUnicode_EqualToUTF8` functions + to compare a Unicode object with a :c:expr:`const char*` UTF-8 encoded string + and ``1`` if they are equal or ``0`` otherwise. + These functions do not raise exceptions. + (Contributed by Serhiy Storchaka in :gh:`110289`.) -* The oparg of ``YIELD_VALUE`` is now ``1`` if the yield is part of a - yield-from or await, and ``0`` otherwise. The oparg of ``RESUME`` was - changed to add a bit indicating whether the except-depth is 1, which - is needed to optimize closing of generators. - (Contributed by Irit Katriel in :gh:`111354`.) +* Add the :c:func:`PyWeakref_GetRef` function + as an alternative to :c:func:`PyWeakref_GetObject` + that returns a :term:`strong reference` + or ``NULL`` if the referent is no longer live. + (Contributed by Victor Stinner in :gh:`105927`.) -Porting to Python 3.13 -====================== +* Add fixed variants of functions which silently ignore errors: -This section lists previously described changes and other bugfixes -that may require changes to your code. + * :c:func:`PyObject_HasAttrWithError` replaces :c:func:`PyObject_HasAttr`. + * :c:func:`PyObject_HasAttrStringWithError` + replaces :c:func:`PyObject_HasAttrString`. + * :c:func:`PyMapping_HasKeyWithError` replaces :c:func:`PyMapping_HasKey`. + * :c:func:`PyMapping_HasKeyStringWithError` + replaces :c:func:`PyMapping_HasKeyString`. -Changes in the Python API -------------------------- + The new functions return ``-1`` for errors + and the standard ``1`` for true and ``0`` for false. -* :meth:`!tkinter.Text.count` now always returns an integer if one or less - counting options are specified. - Previously it could return a single count as a 1-tuple, an integer (only if - option ``"update"`` was specified) or ``None`` if no items found. - The result is now the same if ``wantobjects`` is set to ``0``. - (Contributed by Serhiy Storchaka in :gh:`97928`.) + (Contributed by Serhiy Storchaka in :gh:`108511`.) + + +Changed C APIs +-------------- + +* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` + and :c:func:`PyArg_VaParseTupleAndKeywords` + now has type :c:expr:`char * const *` in C + and :c:expr:`const char * const *` in C++, + instead of :c:expr:`char **`. + In C++, this makes these functions compatible with arguments + of type :c:expr:`const char * const *`, :c:expr:`const char **`, + or :c:expr:`char * const *` without an explicit type cast. + In C, the functions only support arguments of type :c:expr:`char * const *`. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + (Contributed by Serhiy Storchaka in :gh:`65210`.) + +* :c:func:`PyArg_ParseTupleAndKeywords` now supports + non-ASCII keyword parameter names. + (Contributed by Serhiy Storchaka in :gh:`110815`.) + +* The :c:func:`!PyCode_GetFirstFree` function is now unstable API + and is now named :c:func:`PyUnstable_Code_GetFirstFree`. + (Contributed by Bogdan Romanyuk in :gh:`115781`.) -* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, +* The :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, - :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and - :c:func:`PySys_GetObject`, which clear all errors which occurred when calling - them, now report them using :func:`sys.unraisablehook`. - You may replace them with other functions as - recommended in the documentation. + :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, + and :c:func:`PySys_GetObject` functions, + each of which clears all errors which occurred when calling them + now reports these errors using :func:`sys.unraisablehook`. + You may replace them with other functions as recommended in the documentation. (Contributed by Serhiy Storchaka in :gh:`106672`.) -* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to - retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or - :exc:`KeyError` on Unix platforms where the password database is empty. +* Add support for the ``%T``, ``%#T``, ``%N`` and ``%#N`` formats + to :c:func:`PyUnicode_FromFormat`: -* The :mod:`threading` module now expects the :mod:`!_thread` module to have - an ``_is_main_interpreter`` attribute. It is a function with no - arguments that return ``True`` if the current interpreter is the - main interpreter. + * ``%T``: Get the fully qualified name of an object type + * ``%#T``: As above, but use a colon as the separator + * ``%N``: Get the fully qualified name of a type + * ``%#N``: As above, but use a colon as the separator - Any library or application that provides a custom ``_thread`` module - must provide ``_is_main_interpreter()``, just like the module's - other "private" attributes. - (See :gh:`112826`.) + See :pep:`737` for more information. + (Contributed by Victor Stinner in :gh:`111696`.) -* :class:`mailbox.Maildir` now ignores files with a leading dot. - (Contributed by Zackery Spytz in :gh:`65559`.) +* You no longer have to define the ``PY_SSIZE_T_CLEAN`` macro before + including :file:`Python.h` when using ``#`` formats in + :ref:`format codes `. + APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. + (Contributed by Inada Naoki in :gh:`104922`.) -* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both - files and directories if a pattern that ends with "``**``" is given, rather - than directories only. Users may add a trailing slash to match only - directories. +* If Python is built in :ref:`debug mode ` + or :option:`with assertions <--with-assertions>`, + :c:func:`PyTuple_SET_ITEM` and :c:func:`PyList_SET_ITEM` + now check the index argument with an assertion. + (Contributed by Victor Stinner in :gh:`106168`.) -Build Changes -============= +Limited C API Changes +--------------------- -* Autoconf 2.71 and aclocal 1.16.4 is now required to regenerate - the :file:`configure` script. - (Contributed by Christian Heimes in :gh:`89886`.) +* The following functions are now included in the Limited C API: -* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. - (Contributed by Erlend Aasland in :gh:`105875`.) + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawFree` + * :c:func:`PySys_Audit` + * :c:func:`PySys_AuditTuple` + * :c:func:`PyType_GetModuleByDef` + + (Contributed by Victor Stinner in :gh:`85283`, :gh:`85283`, and :gh:`116936`.) -* Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) is now ABI compatible with the Python release build and - :ref:`debug build `. +* Python built with :option:`--with-trace-refs` (tracing references) + now supports the :ref:`Limited API `. (Contributed by Victor Stinner in :gh:`108634`.) -* Building CPython now requires a compiler with support for the C11 atomic - library, GCC built-in atomic functions, or MSVC interlocked intrinsics. -* The ``errno``, ``md5``, ``resource``, ``winsound``, ``_ctypes_test``, - ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, - ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the - :ref:`limited C API `. - (Contributed by Victor Stinner in :gh:`85283`.) +Removed C APIs +-------------- -* ``wasm32-wasi`` is now a tier 2 platform. - (Contributed by Brett Cannon in :gh:`115192`.) +* Remove several functions, macros, variables, etc + with names prefixed by ``_Py`` or ``_PY`` (which are considered private). + If your project is affected by one of these removals + and you believe that the removed API should remain available, + please :ref:`open a new issue ` to request a public C API + and add ``cc: @vstinner`` to the issue to notify Victor Stinner. + (Contributed by Victor Stinner in :gh:`106320`.) -* ``wasm32-emscripten`` is no longer a supported platform. - (Contributed by Brett Cannon in :gh:`115192`.) +* Remove old buffer protocols deprecated in Python 3.0. + Use :ref:`bufferobjects` instead. + * :c:func:`!PyObject_CheckReadBuffer`: + Use :c:func:`PyObject_CheckBuffer` to test + whether the object supports the buffer protocol. + Note that :c:func:`PyObject_CheckBuffer` doesn't guarantee + that :c:func:`PyObject_GetBuffer` will succeed. + To test if the object is actually readable, + see the next example of :c:func:`PyObject_GetBuffer`. -C API Changes -============= + * :c:func:`!PyObject_AsCharBuffer`, :c:func:`!PyObject_AsReadBuffer`: + Use :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: -New Features ------------- + .. code-block:: c -* You no longer have to define the ``PY_SSIZE_T_CLEAN`` macro before including - :file:`Python.h` when using ``#`` formats in - :ref:`format codes `. - APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. - (Contributed by Inada Naoki in :gh:`104922`.) + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) < 0) { + return NULL; + } + // Use `view.buf` and `view.len` to read from the buffer. + // You may need to cast buf as `(const char*)view.buf`. + PyBuffer_Release(&view); -* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and - :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *` - in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. - It makes these functions compatible with arguments of type - :c:expr:`const char * const *`, :c:expr:`const char **` or - :c:expr:`char * const *` in C++ and :c:expr:`char * const *` in C - without an explicit type cast. - This can be overridden with the :c:macro:`PY_CXX_CONST` macro. - (Contributed by Serhiy Storchaka in :gh:`65210`.) + * :c:func:`!PyObject_AsWriteBuffer`: + Use :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: -* Add :c:func:`PyImport_AddModuleRef`: similar to - :c:func:`PyImport_AddModule`, but return a :term:`strong reference` instead - of a :term:`borrowed reference`. - (Contributed by Victor Stinner in :gh:`105922`.) + .. code-block:: c -* Add :c:func:`PyWeakref_GetRef` function: similar to - :c:func:`PyWeakref_GetObject` but returns a :term:`strong reference`, or - ``NULL`` if the referent is no longer live. - (Contributed by Victor Stinner in :gh:`105927`.) + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_WRITABLE) < 0) { + return NULL; + } + // Use `view.buf` and `view.len` to write to the buffer. + PyBuffer_Release(&view); -* Add :c:func:`PyObject_GetOptionalAttr` and - :c:func:`PyObject_GetOptionalAttrString`, variants of - :c:func:`PyObject_GetAttr` and :c:func:`PyObject_GetAttrString` which - don't raise :exc:`AttributeError` if the attribute is not found. - These variants are more convenient and faster if the missing attribute - should not be treated as a failure. - (Contributed by Serhiy Storchaka in :gh:`106521`.) + (Contributed by Inada Naoki in :gh:`85275`.) -* Add :c:func:`PyMapping_GetOptionalItem` and - :c:func:`PyMapping_GetOptionalItemString`: variants of - :c:func:`PyObject_GetItem` and :c:func:`PyMapping_GetItemString` which don't - raise :exc:`KeyError` if the key is not found. - These variants are more convenient and faster if the missing key should not - be treated as a failure. - (Contributed by Serhiy Storchaka in :gh:`106307`.) +* Remove various functions deprecated in Python 3.9: -* Add fixed variants of functions which silently ignore errors: + * :c:func:`!PyEval_CallObject`, :c:func:`!PyEval_CallObjectWithKeywords`: + Use :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. - - :c:func:`PyObject_HasAttrWithError` replaces :c:func:`PyObject_HasAttr`. - - :c:func:`PyObject_HasAttrStringWithError` replaces :c:func:`PyObject_HasAttrString`. - - :c:func:`PyMapping_HasKeyWithError` replaces :c:func:`PyMapping_HasKey`. - - :c:func:`PyMapping_HasKeyStringWithError` replaces :c:func:`PyMapping_HasKeyString`. + .. warning:: - New functions return not only ``1`` for true and ``0`` for false, but also - ``-1`` for error. + In :c:func:`PyObject_Call`, positional arguments must be a :class:`tuple` + and must not be ``NULL``, + and keyword arguments must be a :class:`dict` or ``NULL``, + whereas the removed functions checked argument types + and accepted ``NULL`` positional and keyword arguments. + To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with + :c:func:`PyObject_Call`, + pass an empty tuple as positional arguments using + :c:func:`PyTuple_New(0) `. - (Contributed by Serhiy Storchaka in :gh:`108511`.) + * :c:func:`!PyEval_CallFunction`: + Use :c:func:`PyObject_CallFunction` instead. + * :c:func:`!PyEval_CallMethod`: + Use :c:func:`PyObject_CallMethod` instead. + * :c:func:`!PyCFunction_Call`: + Use :c:func:`PyObject_Call` instead. -* If Python is built in :ref:`debug mode ` or :option:`with - assertions <--with-assertions>`, :c:func:`PyTuple_SET_ITEM` and - :c:func:`PyList_SET_ITEM` now check the index argument with an assertion. - (Contributed by Victor Stinner in :gh:`106168`.) + (Contributed by Victor Stinner in :gh:`105107`.) + +* Remove the following old functions to configure the Python initialization, + deprecated in Python 3.11: + + * :c:func:`!PySys_AddWarnOptionUnicode`: + Use :c:member:`PyConfig.warnoptions` instead. + * :c:func:`!PySys_AddWarnOption`: + Use :c:member:`PyConfig.warnoptions` instead. + * :c:func:`!PySys_AddXOption`: + Use :c:member:`PyConfig.xoptions` instead. + * :c:func:`!PySys_HasWarnOptions`: + Use :c:member:`PyConfig.xoptions` instead. + * :c:func:`!PySys_SetPath`: + Set :c:member:`PyConfig.module_search_paths` instead. + * :c:func:`!Py_SetPath`: + Set :c:member:`PyConfig.module_search_paths` instead. + * :c:func:`!Py_SetStandardStreamEncoding`: + Set :c:member:`PyConfig.stdio_encoding` instead, + and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on Windows). + * :c:func:`!_Py_SetProgramFullPath`: + Set :c:member:`PyConfig.executable` instead. + + Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization + Configuration ` instead (:pep:`587`), added to Python 3.8. + (Contributed by Victor Stinner in :gh:`105145`.) + +* Remove :c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_ReleaseLock` functions, + deprecated in Python 3.2. + They didn't update the current thread state. + They can be replaced with: + + * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread`; + * low-level :c:func:`PyEval_AcquireThread` and :c:func:`PyEval_RestoreThread`; + * or :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`. + + (Contributed by Victor Stinner in :gh:`105182`.) + +* Remove the :c:func:`!PyEval_ThreadsInitialized` function, + deprecated in Python 3.9. + Since Python 3.7, :c:func:`!Py_Initialize` always creates the GIL: + calling :c:func:`!PyEval_InitThreads` does nothing and + :c:func:`!PyEval_ThreadsInitialized` always returns non-zero. + (Contributed by Victor Stinner in :gh:`105182`.) + +* Remove the :c:func:`!_PyInterpreterState_Get` alias to + :c:func:`PyInterpreterState_Get()` + which was kept for backward compatibility with Python 3.8. + The `pythoncapi-compat project`_ can be used to get + :c:func:`PyInterpreterState_Get()` on Python 3.8 and older. + (Contributed by Victor Stinner in :gh:`106320`.) + +* Remove the private :c:func:`!_PyObject_FastCall` function: + use :c:func:`!PyObject_Vectorcall` which is available since Python 3.8 + (:pep:`590`). + (Contributed by Victor Stinner in :gh:`106023`.) + +* Remove the ``cpython/pytime.h`` header file, + which only contained private functions. + (Contributed by Victor Stinner in :gh:`106316`.) + +* Remove the undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. + (Contributed by Victor Stinner in :gh:`110014`.) + +* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` + and ``Py_TRASHCAN_SAFE_END``. + Replace both with the new macros ``Py_TRASHCAN_BEGIN`` + and ``Py_TRASHCAN_END``. + (Contributed by Irit Katriel in :gh:`105111`.) + +Deprecated C APIs +----------------- + +* Deprecate old Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: + Get :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: + Get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: + Get :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: + Get :c:member:`PyConfig.home` + or the :envvar:`PYTHONHOME` environment variable instead. + + (Contributed by Victor Stinner in :gh:`105145`.) + +* :term:`Soft deprecate ` the + :c:func:`PyEval_GetBuiltins`, :c:func:`PyEval_GetGlobals`, + and :c:func:`PyEval_GetLocals` functions, + which return a :term:`borrowed reference`. + (Soft deprecated as part of :pep:`667`.) + +* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function, + which is just an alias to :c:func:`PyImport_ImportModule` since Python 3.3. + (Contributed by Victor Stinner in :gh:`105396`.) -* Add :c:func:`PyModule_Add` function: similar to - :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject` but - always steals a reference to the value. +* :term:`Soft deprecate ` the + :c:func:`PyModule_AddObject` function. + It should be replaced with :c:func:`PyModule_Add` + or :c:func:`PyModule_AddObjectRef`. (Contributed by Serhiy Storchaka in :gh:`86493`.) -* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` - functions: similar to :c:func:`PyDict_GetItemWithError` but returning a - :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, - these functions return -1 on error and so checking ``PyErr_Occurred()`` is - not needed. - (Contributed by Victor Stinner in :gh:`106004`.) +* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types + and the :c:macro:`!Py_UNICODE_WIDE` define. + Use the :c:type:`wchar_t` type directly instead. + Since Python 3.3, ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` + are just aliases to :c:type:`!wchar_t`. + (Contributed by Victor Stinner in :gh:`105156`.) -* Added :c:func:`PyDict_SetDefaultRef`, which is similar to - :c:func:`PyDict_SetDefault` but returns a :term:`strong reference` instead of - a :term:`borrowed reference`. This function returns ``-1`` on error, ``0`` on - insertion, and ``1`` if the key was already present in the dictionary. - (Contributed by Sam Gross in :gh:`112066`.) +* Deprecate the :c:func:`PyWeakref_GetObject` and + :c:func:`PyWeakref_GET_OBJECT` functions, + which return a :term:`borrowed reference`. + Replace them with the new :c:func:`PyWeakref_GetRef` function, + which returns a :term:`strong reference`. + The `pythoncapi-compat project`_ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Victor Stinner in :gh:`105927`.) -* Add :c:func:`PyDict_ContainsString` function: same as - :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` - UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. - (Contributed by Victor Stinner in :gh:`108314`.) +.. Add deprecations above alphabetically, not here at the end. -* Added :c:func:`PyList_GetItemRef` function: similar to - :c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of - a :term:`borrowed reference`. +.. include:: ../deprecations/c-api-pending-removal-in-3.14.rst -* Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is - :term:`shutting down `. - (Contributed by Victor Stinner in :gh:`108014`.) +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst -* Add :c:func:`PyLong_AsInt` function: similar to :c:func:`PyLong_AsLong`, but - store the result in a C :c:expr:`int` instead of a C :c:expr:`long`. - Previously, it was known as the private function :c:func:`!_PyLong_AsInt` - (with an underscore prefix). - (Contributed by Victor Stinner in :gh:`108014`.) +.. include:: ../deprecations/c-api-pending-removal-in-future.rst -* Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) now supports the :ref:`Limited API `. - (Contributed by Victor Stinner in :gh:`108634`.) +.. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ -* Add :c:func:`PyObject_VisitManagedDict` and - :c:func:`PyObject_ClearManagedDict` functions which must be called by the - traverse and clear functions of a type using - :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. The `pythoncapi-compat project - `__ can be used to get these - functions on Python 3.11 and 3.12. - (Contributed by Victor Stinner in :gh:`107073`.) +Build Changes +============= -* Add :c:func:`PyUnicode_EqualToUTF8AndSize` and :c:func:`PyUnicode_EqualToUTF8` - functions: compare Unicode object with a :c:expr:`const char*` UTF-8 encoded - string and return true (``1``) if they are equal, or false (``0``) otherwise. - These functions do not raise exceptions. - (Contributed by Serhiy Storchaka in :gh:`110289`.) +* ``arm64-apple-ios`` and ``arm64-apple-ios-simulator`` are both + now :pep:`11` tier 3 platforms. + (:ref:`PEP 730 ` written + and implementation contributed by Russell Keith-Magee in :gh:`114099`.) -* Add :c:func:`PyThreadState_GetUnchecked()` function: similar to - :c:func:`PyThreadState_Get()`, but don't kill the process with a fatal error - if it is NULL. The caller is responsible to check if the result is NULL. - Previously, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - (Contributed by Victor Stinner in :gh:`108867`.) +* ``aarch64-linux-android`` and ``x86_64-linux-android`` are both + now :pep:`11` tier 3 platforms. + (:ref:`PEP 738 ` written + and implementation contributed by Malcolm Smith in :gh:`116622`.) -* Add :c:func:`PySys_AuditTuple` function: similar to :c:func:`PySys_Audit`, - but pass event arguments as a Python :class:`tuple` object. - (Contributed by Victor Stinner in :gh:`85283`.) +* ``wasm32-wasi`` is now a :pep:`11` tier 2 platform. + (Contributed by Brett Cannon in :gh:`115192`.) -* :c:func:`PyArg_ParseTupleAndKeywords` now supports non-ASCII keyword - parameter names. - (Contributed by Serhiy Storchaka in :gh:`110815`.) +* ``wasm32-emscripten`` is no longer a :pep:`11` supported platform. + (Contributed by Brett Cannon in :gh:`115192`.) -* Add :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawCalloc`, - :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree` to the limited C API - (version 3.13). - (Contributed by Victor Stinner in :gh:`85283`.) +* Building CPython now requires a compiler with support for the C11 atomic + library, GCC built-in atomic functions, or MSVC interlocked intrinsics. -* Add :c:func:`PySys_Audit` and :c:func:`PySys_AuditTuple` functions to the - limited C API. - (Contributed by Victor Stinner in :gh:`85283`.) +* Autoconf 2.71 and aclocal 1.16.5 are now required to regenerate + the :file:`configure` script. + (Contributed by Christian Heimes in :gh:`89886` and by Victor Stinner in :gh:`112090`.) -* Add :c:func:`PyErr_FormatUnraisable` function: similar to - :c:func:`PyErr_WriteUnraisable`, but allow customizing the warning message. - (Contributed by Serhiy Storchaka in :gh:`108082`.) +* SQLite 3.15.2 or newer is required to build + the :mod:`sqlite3` extension module. + (Contributed by Erlend Aasland in :gh:`105875`.) -* Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to - Python ``list.extend()`` and ``list.clear()`` methods. - (Contributed by Victor Stinner in :gh:`111138`.) +* CPython now bundles the `mimalloc library`_ by default. + It is licensed under the MIT license; + see :ref:`mimalloc license `. + The bundled mimalloc has custom changes, see :gh:`113141` for details. + (Contributed by Dino Viehland in :gh:`109914`.) -* Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a - key from a dictionary and optionally return the removed value. This is - similar to :meth:`dict.pop`, but without the default value and not raising - :exc:`KeyError` if the key is missing. - (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) + .. _mimalloc library: https://github.com/microsoft/mimalloc/ -* Add :c:func:`Py_HashPointer` function to hash a pointer. - (Contributed by Victor Stinner in :gh:`111545`.) +* The :file:`configure` option :option:`--with-system-libmpdec` + now defaults to ``yes``. + The bundled copy of ``libmpdecimal`` will be removed in Python 3.15. + +* Python built with :file:`configure` :option:`--with-trace-refs` + (tracing references) is now ABI compatible with the Python release build + and :ref:`debug build `. + (Contributed by Victor Stinner in :gh:`108634`.) + +* On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI + flags. For example, the free-threaded build generates ``python-3.13t.pc`` + and the debug build generates ``python-3.13d.pc``. + +* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, + ``termios``, ``winsound``, + ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, + ``_statistics``, ``_testconsole``, ``_testimportmultiple`` and ``_uuid`` + C extensions are now built with the :ref:`limited C API `. + (Contributed by Victor Stinner in :gh:`85283`.) Porting to Python 3.13 ----------------------- +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +Changes in the Python API +------------------------- + +.. _pep667-porting-notes-py: + +* :ref:`PEP 667 ` introduces several changes + to the semantics of :func:`locals` and :attr:`f_locals `: + + * Calling :func:`locals` in an :term:`optimized scope` now produces an + independent snapshot on each call, and hence no longer implicitly updates + previously returned references. Obtaining the legacy CPython behavior now + requires explicit calls to update the initially returned dictionary with the + results of subsequent calls to :func:`!locals`. Code execution functions that + implicitly target :func:`!locals` (such as ``exec`` and ``eval``) must be + passed an explicit namespace to access their results in an optimized scope. + (Changed as part of :pep:`667`.) + + * Calling :func:`locals` from a comprehension at module or class scope + (including via ``exec`` or ``eval``) once more behaves as if the comprehension + were running as an independent nested function (i.e. the local variables from + the containing scope are not included). In Python 3.12, this had changed + to include the local variables from the containing scope when implementing + :pep:`709`. (Changed as part of :pep:`667`.) + + * Accessing :attr:`FrameType.f_locals ` in an + :term:`optimized scope` now returns a write-through proxy rather than a + snapshot that gets updated at ill-specified times. If a snapshot is desired, + it must be created explicitly with ``dict`` or the proxy's ``.copy()`` method. + (Changed as part of :pep:`667`.) + +* :class:`functools.partial` now emits a :exc:`FutureWarning` + when used as a method. + The behavior will change in future Python versions. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka in :gh:`121027`.) + +* An :exc:`OSError` is now raised by :func:`getpass.getuser` + for any failure to retrieve a username, + instead of :exc:`ImportError` on non-Unix platforms + or :exc:`KeyError` on Unix platforms where the password database is empty. + +* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` + is now a string (``'rb'`` or ``'wb'``) instead of an integer (``1`` or ``2``). + The value of the :attr:`!mode` attribute of the readable file-like object + returned by :meth:`zipfile.ZipFile.open` is now ``'rb'`` instead of ``'r'``. + (Contributed by Serhiy Storchaka in :gh:`115961`.) + +* :class:`mailbox.Maildir` now ignores files with a leading dot (``.``). + (Contributed by Zackery Spytz in :gh:`65559`.) + +* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both + files and directories if a pattern that ends with "``**``" is given, + rather than directories only. + Add a trailing slash to keep the previous behavior and only match directories. + +* The :mod:`threading` module now expects the :mod:`!_thread` module + to have an :func:`!_is_main_interpreter` function. + This function takes no arguments and returns ``True`` + if the current interpreter is the main interpreter. + + Any library or application that provides a custom :mod:`!_thread` module + must provide :func:`!_is_main_interpreter`, + just like the module's other "private" attributes. + (:gh:`112826`.) + + +Changes in the C API +-------------------- * ``Python.h`` no longer includes the ```` standard header. It was - included for the ``finite()`` function which is now provided by the + included for the :c:func:`!finite` function which is now provided by the ```` header. It should now be included explicitly if needed. Remove also the ``HAVE_IEEEFP_H`` macro. (Contributed by Victor Stinner in :gh:`108765`.) * ``Python.h`` no longer includes these standard header files: ````, ```` and ````. If needed, they should now be - included explicitly. For example, ```` provides the ``clock()`` and - ``gmtime()`` functions, ```` provides the ``select()`` - function, and ```` provides the ``futimes()``, ``gettimeofday()`` - and ``setitimer()`` functions. + included explicitly. For example, ```` provides the :c:func:`!clock` and + :c:func:`!gmtime` functions, ```` provides the :c:func:`!select` + function, and ```` provides the :c:func:`!futimes`, :c:func:`!gettimeofday` + and :c:func:`!setitimer` functions. + (Contributed by Victor Stinner in :gh:`108765`.) + +* On Windows, ``Python.h`` no longer includes the ```` standard + header file. If needed, it should now be included explicitly. For example, it + provides :c:func:`!offsetof` function, and ``size_t`` and ``ptrdiff_t`` types. + Including ```` explicitly was already needed by all other + platforms, the ``HAVE_STDDEF_H`` macro is only defined on Windows. (Contributed by Victor Stinner in :gh:`108765`.) * If the :c:macro:`Py_LIMITED_API` macro is defined, :c:macro:`!Py_BUILD_CORE`, @@ -1541,7 +2637,7 @@ Porting to Python 3.13 were removed. They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. - A tp_dealloc function that has the old macros, such as:: + A ``tp_dealloc`` function that has the old macros, such as:: static void mytype_dealloc(mytype *p) @@ -1564,281 +2660,43 @@ Porting to Python 3.13 } Note that ``Py_TRASHCAN_BEGIN`` has a second argument which - should be the deallocation function it is in. - -* On Windows, ``Python.h`` no longer includes the ```` standard - header file. If needed, it should now be included explicitly. For example, it - provides ``offsetof()`` function, and ``size_t`` and ``ptrdiff_t`` types. - Including ```` explicitly was already needed by all other - platforms, the ``HAVE_STDDEF_H`` macro is only defined on Windows. - (Contributed by Victor Stinner in :gh:`108765`.) - - -Deprecated ----------- - -* Passing optional arguments *maxsplit*, *count* and *flags* in module-level - functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional - arguments is now deprecated. - In future Python versions these parameters will be - :ref:`keyword-only `. - (Contributed by Serhiy Storchaka in :gh:`56166`.) - -* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly - the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and - ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. - (Contributed by Victor Stinner in :gh:`105156`.) - -* Deprecate old Python initialization functions: - - * :c:func:`PySys_ResetWarnOptions`: - clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - :envvar:`PYTHONHOME` environment variable instead. - - Functions scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105145`.) - -* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just - an alias to :c:func:`PyImport_ImportModule` since Python 3.3. - Scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105396`.) - -* Deprecate the :c:func:`PyWeakref_GetObject` and - :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed - reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it - returns a :term:`strong reference`. The `pythoncapi-compat project - `__ can be used to get - :c:func:`PyWeakref_GetRef` on Python 3.12 and older. - (Contributed by Victor Stinner in :gh:`105927`.) - -Removed -------- - -* Removed chained :class:`classmethod` descriptors (introduced in - :gh:`63272`). This can no longer be used to wrap other descriptors - such as :class:`property`. The core design of this feature was flawed - and caused a number of downstream problems. To "pass-through" a - :class:`classmethod`, consider using the :attr:`!__wrapped__` - attribute that was added in Python 3.10. (Contributed by Raymond - Hettinger in :gh:`89519`.) - -* Remove many APIs (functions, macros, variables) with names prefixed by - ``_Py`` or ``_PY`` (considered as private API). If your project is affected - by one of these removals and you consider that the removed API should remain - available, please open a new issue to request a public C API and - add ``cc @vstinner`` to the issue to notify Victor Stinner. - (Contributed by Victor Stinner in :gh:`106320`.) - -* Remove functions deprecated in Python 3.9: - - * ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use - :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. - Warning: :c:func:`PyObject_Call` positional arguments must be a - :class:`tuple` and must not be ``NULL``, keyword arguments must be a - :class:`dict` or ``NULL``, whereas removed functions checked arguments type - and accepted ``NULL`` positional and keyword arguments. - To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with - :c:func:`PyObject_Call`, pass an empty tuple as positional arguments using - :c:func:`PyTuple_New(0) `. - * ``PyEval_CallFunction()``: use :c:func:`PyObject_CallFunction` instead. - * ``PyEval_CallMethod()``: use :c:func:`PyObject_CallMethod` instead. - * ``PyCFunction_Call()``: use :c:func:`PyObject_Call` instead. - - (Contributed by Victor Stinner in :gh:`105107`.) - -* Remove old buffer protocols deprecated in Python 3.0. Use :ref:`bufferobjects` instead. - - * :c:func:`!PyObject_CheckReadBuffer`: Use :c:func:`PyObject_CheckBuffer` to - test if the object supports the buffer protocol. - Note that :c:func:`PyObject_CheckBuffer` doesn't guarantee that - :c:func:`PyObject_GetBuffer` will succeed. - To test if the object is actually readable, see the next example - of :c:func:`PyObject_GetBuffer`. - - * :c:func:`!PyObject_AsCharBuffer`, :c:func:`!PyObject_AsReadBuffer`: - :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: - - .. code-block:: c - - Py_buffer view; - if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) < 0) { - return NULL; - } - // Use `view.buf` and `view.len` to read from the buffer. - // You may need to cast buf as `(const char*)view.buf`. - PyBuffer_Release(&view); - - * :c:func:`!PyObject_AsWriteBuffer`: Use - :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: - - .. code-block:: c - - Py_buffer view; - if (PyObject_GetBuffer(obj, &view, PyBUF_WRITABLE) < 0) { - return NULL; - } - // Use `view.buf` and `view.len` to write to the buffer. - PyBuffer_Release(&view); - - (Contributed by Inada Naoki in :gh:`85275`.) - -* Remove the following old functions to configure the Python initialization, - deprecated in Python 3.11: - - * ``PySys_AddWarnOptionUnicode()``: use :c:member:`PyConfig.warnoptions` instead. - * ``PySys_AddWarnOption()``: use :c:member:`PyConfig.warnoptions` instead. - * ``PySys_AddXOption()``: use :c:member:`PyConfig.xoptions` instead. - * ``PySys_HasWarnOptions()``: use :c:member:`PyConfig.xoptions` instead. - * ``PySys_SetArgvEx()``: set :c:member:`PyConfig.argv` instead. - * ``PySys_SetArgv()``: set :c:member:`PyConfig.argv` instead. - * ``PySys_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. - * ``Py_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. - * ``Py_SetProgramName()``: set :c:member:`PyConfig.program_name` instead. - * ``Py_SetPythonHome()``: set :c:member:`PyConfig.home` instead. - * ``Py_SetStandardStreamEncoding()``: set :c:member:`PyConfig.stdio_encoding` - instead, and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on - Windows). - * ``_Py_SetProgramFullPath()``: set :c:member:`PyConfig.executable` instead. - - Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization - Configuration ` instead (:pep:`587`), added to Python 3.8. - (Contributed by Victor Stinner in :gh:`105145`.) - -* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and - ``Py_TRASHCAN_SAFE_END``. They should be replaced by the new macros - ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. The new macros were + should be the deallocation function it is in. The new macros were added in Python 3.8 and the old macros were deprecated in Python 3.11. (Contributed by Irit Katriel in :gh:`105111`.) -* Remove ``PyEval_InitThreads()`` and ``PyEval_ThreadsInitialized()`` - functions, deprecated in Python 3.9. Since Python 3.7, ``Py_Initialize()`` - always creates the GIL: calling ``PyEval_InitThreads()`` did nothing and - ``PyEval_ThreadsInitialized()`` always returned non-zero. - (Contributed by Victor Stinner in :gh:`105182`.) - -* Remove ``PyEval_AcquireLock()`` and ``PyEval_ReleaseLock()`` functions, - deprecated in Python 3.2. They didn't update the current thread state. - They can be replaced with: - - * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread`; - * low-level :c:func:`PyEval_AcquireThread` and :c:func:`PyEval_RestoreThread`; - * or :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`. - - (Contributed by Victor Stinner in :gh:`105182`.) - -* Remove private ``_PyObject_FastCall()`` function: - use ``PyObject_Vectorcall()`` which is available since Python 3.8 - (:pep:`590`). - (Contributed by Victor Stinner in :gh:`106023`.) - -* Remove ``cpython/pytime.h`` header file: it only contained private functions. - (Contributed by Victor Stinner in :gh:`106316`.) - -* Remove ``_PyInterpreterState_Get()`` alias to - :c:func:`PyInterpreterState_Get()` which was kept for backward compatibility - with Python 3.8. The `pythoncapi-compat project - `__ can be used to get - :c:func:`PyInterpreterState_Get()` on Python 3.8 and older. - (Contributed by Victor Stinner in :gh:`106320`.) - -* The :c:func:`PyModule_AddObject` function is now :term:`soft deprecated`: - :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` functions should - be used instead. - (Contributed by Serhiy Storchaka in :gh:`86493`.) - -Pending Removal in Python 3.14 ------------------------------- - -* Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable - bases using the C API. -* Global configuration variables: - - * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` - * :c:var:`Py_VerboseFlag`: use :c:member:`PyConfig.verbose` - * :c:var:`Py_QuietFlag`: use :c:member:`PyConfig.quiet` - * :c:var:`Py_InteractiveFlag`: use :c:member:`PyConfig.interactive` - * :c:var:`Py_InspectFlag`: use :c:member:`PyConfig.inspect` - * :c:var:`Py_OptimizeFlag`: use :c:member:`PyConfig.optimization_level` - * :c:var:`Py_NoSiteFlag`: use :c:member:`PyConfig.site_import` - * :c:var:`Py_BytesWarningFlag`: use :c:member:`PyConfig.bytes_warning` - * :c:var:`Py_FrozenFlag`: use :c:member:`PyConfig.pathconfig_warnings` - * :c:var:`Py_IgnoreEnvironmentFlag`: use :c:member:`PyConfig.use_environment` - * :c:var:`Py_DontWriteBytecodeFlag`: use :c:member:`PyConfig.write_bytecode` - * :c:var:`Py_NoUserSiteDirectory`: use :c:member:`PyConfig.user_site_directory` - * :c:var:`Py_UnbufferedStdioFlag`: use :c:member:`PyConfig.buffered_stdio` - * :c:var:`Py_HashRandomizationFlag`: use :c:member:`PyConfig.use_hash_seed` - and :c:member:`PyConfig.hash_seed` - * :c:var:`Py_IsolatedFlag`: use :c:member:`PyConfig.isolated` - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` - * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` - * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` - * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` - * :c:var:`!Py_UTF8Mode`: use :c:member:`PyPreConfig.utf8_mode` (see :c:func:`Py_PreInitialize`) - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. - -Pending Removal in Python 3.15 ------------------------------- - -* :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule`. -* :c:func:`PyWeakref_GET_OBJECT`: use :c:func:`PyWeakref_GetRef` instead. -* :c:func:`PyWeakref_GetObject`: use :c:func:`PyWeakref_GetRef` instead. -* :c:type:`!Py_UNICODE_WIDE` type: use :c:type:`wchar_t` instead. -* :c:type:`Py_UNICODE` type: use :c:type:`wchar_t` instead. -* Python initialization functions: - - * :c:func:`PySys_ResetWarnOptions`: clear :data:`sys.warnoptions` and - :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - :envvar:`PYTHONHOME` environment variable instead. - -Pending Removal in Future Versions ----------------------------------- - -The following APIs were deprecated in earlier Python versions and will be -removed, although there is currently no date scheduled for their removal. - -* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: no needed since Python 3.8. -* :c:func:`PyErr_Fetch`: use :c:func:`PyErr_GetRaisedException`. -* :c:func:`PyErr_NormalizeException`: use :c:func:`PyErr_GetRaisedException`. -* :c:func:`PyErr_Restore`: use :c:func:`PyErr_SetRaisedException`. -* :c:func:`PyModule_GetFilename`: use :c:func:`PyModule_GetFilenameObject`. -* :c:func:`PyOS_AfterFork`: use :c:func:`PyOS_AfterFork_Child()`. -* :c:func:`PySlice_GetIndicesEx`. -* :c:func:`!PyUnicode_AsDecodedObject`. -* :c:func:`!PyUnicode_AsDecodedUnicode`. -* :c:func:`!PyUnicode_AsEncodedObject`. -* :c:func:`!PyUnicode_AsEncodedUnicode`. -* :c:func:`PyUnicode_READY`: not needed since Python 3.12. -* :c:func:`!_PyErr_ChainExceptions`. -* :c:member:`!PyBytesObject.ob_shash` member: - call :c:func:`PyObject_Hash` instead. -* :c:member:`!PyDictObject.ma_version_tag` member. -* TLS API: - - * :c:func:`PyThread_create_key`: use :c:func:`PyThread_tss_alloc`. - * :c:func:`PyThread_delete_key`: use :c:func:`PyThread_tss_free`. - * :c:func:`PyThread_set_key_value`: use :c:func:`PyThread_tss_set`. - * :c:func:`PyThread_get_key_value`: use :c:func:`PyThread_tss_get`. - * :c:func:`PyThread_delete_key_value`: use :c:func:`PyThread_tss_delete`. - * :c:func:`PyThread_ReInitTLS`: no longer needed. - -* Remove undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. - (Contributed by Victor Stinner in :gh:`110014`.) - +.. _pep667-porting-notes-c: + +* :ref:`PEP 667 ` introduces several changes + to frame-related functions: + + * The effects of mutating the dictionary returned from + :c:func:`PyEval_GetLocals` in an :term:`optimized scope` have changed. + New dict entries added this way will now *only* be visible to + subsequent :c:func:`PyEval_GetLocals` calls in that frame, + as :c:func:`PyFrame_GetLocals`, :func:`locals`, + and :attr:`FrameType.f_locals ` no longer access + the same underlying cached dictionary. + Changes made to entries for actual variable names and names added via + the write-through proxy interfaces will be overwritten on subsequent calls + to :c:func:`PyEval_GetLocals` in that frame. + The recommended code update depends on how the function was being used, + so refer to the deprecation notice on the function for details. + + * Calling :c:func:`PyFrame_GetLocals` in an :term:`optimized scope` + now returns a write-through proxy rather than a snapshot + that gets updated at ill-specified times. + If a snapshot is desired, it must be created explicitly + (e.g. with :c:func:`PyDict_Copy`), + or by calling the new :c:func:`PyEval_GetFrameLocals` API. + + * :c:func:`!PyFrame_FastToLocals` and :c:func:`!PyFrame_FastToLocalsWithError` + no longer have any effect. + Calling these functions has been redundant since Python 3.11, + when :c:func:`PyFrame_GetLocals` was first introduced. + + * :c:func:`!PyFrame_LocalsToFast` no longer has any effect. + Calling this function is redundant now that :c:func:`PyFrame_GetLocals` + returns a write-through proxy for :term:`optimized scopes `. Regression Test Changes ======================= diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst new file mode 100644 index 00000000000000..21bc289c2be5d8 --- /dev/null +++ b/Doc/whatsnew/3.14.rst @@ -0,0 +1,835 @@ + +**************************** + What's new in Python 3.14 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.14, compared to 3.13. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.14 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- release highlights +============================= + +.. This section singles out the most important changes in Python 3.14. + Brevity is key. + + +.. PEP-sized items next. + + + +New features +============ + +.. _whatsnew314-pep649: + +PEP 649: deferred evaluation of annotations +------------------------------------------- + +The :term:`annotations ` on functions, classes, and modules are no +longer evaluated eagerly. Instead, annotations are stored in special-purpose +:term:`annotate functions ` and evaluated only when +necessary. This is specified in :pep:`649` and :pep:`749`. + +This change is designed to make annotations in Python more performant and more +usable in most circumstances. The runtime cost for defining annotations is +minimized, but it remains possible to introspect annotations at runtime. +It is usually no longer necessary to enclose annotations in strings if they +contain forward references. + +The new :mod:`annotationlib` module provides tools for inspecting deferred +annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VALUE` +format (which evaluates annotations to runtime values, similar to the behavior in +earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format +(which replaces undefined names with special markers), and the +:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). + +This example shows how these formats behave: + +.. doctest:: + + >>> from annotationlib import get_annotations, Format + >>> def func(arg: Undefined): + ... pass + >>> get_annotations(func, format=Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'Undefined' is not defined + >>> get_annotations(func, format=Format.FORWARDREF) + {'arg': ForwardRef('Undefined')} + >>> get_annotations(func, format=Format.STRING) + {'arg': 'Undefined'} + +Implications for annotated code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you define annotations in your code (for example, for use with a static type +checker), then this change probably does not affect you: you can keep +writing annotations the same way you did with previous versions of Python. + +You will likely be able to remove quoted strings in annotations, which are frequently +used for forward references. Similarly, if you use ``from __future__ import annotations`` +to avoid having to write strings in annotations, you may well be able to +remove that import. However, if you rely on third-party libraries that read annotations, +those libraries may need changes to support unquoted annotations before they +work as expected. + +Implications for readers of ``__annotations__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If your code reads the ``__annotations__`` attribute on objects, you may want +to make changes in order to support code that relies on deferred evaluation of +annotations. For example, you may want to use :func:`annotationlib.get_annotations` +with the :attr:`~annotationlib.Format.FORWARDREF` format, as the :mod:`dataclasses` +module now does. + +Related changes +^^^^^^^^^^^^^^^ + +The changes in Python 3.14 are designed to rework how ``__annotations__`` +works at runtime while minimizing breakage to code that contains +annotations in source code and to code that reads ``__annotations__``. However, +if you rely on undocumented details of the annotation behavior or on private +functions in the standard library, there are many ways in which your code may +not work in Python 3.14. To safeguard your code against future changes, +use only the documented functionality of the :mod:`annotationlib` module. + +``from __future__ import annotations`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` +directive, which turns all annotations into strings. This directive is now +considered deprecated and it is expected to be removed in a future version of Python. +However, this removal will not happen until after Python 3.13, the last version of +Python without deferred evaluation of annotations, reaches its end of life in 2029. +In Python 3.14, the behavior of code using ``from __future__ import annotations`` +is unchanged. + + +Improved error messages +----------------------- + +* When unpacking assignment fails due to incorrect number of variables, the + error message prints the received number of values in more cases than before. + (Contributed by Tushar Sadhwani in :gh:`122239`.) + + .. code-block:: pycon + + >>> x, y, z = 1, 2, 3, 4 + Traceback (most recent call last): + File "", line 1, in + x, y, z = 1, 2, 3, 4 + ^^^^^^^ + ValueError: too many values to unpack (expected 3, got 4) + + +Other language changes +====================== + +* Incorrect usage of :keyword:`await` and asynchronous comprehensions + is now detected even if the code is optimized away by the :option:`-O` + command-line option. For example, ``python -O -c 'assert await 1'`` + now produces a :exc:`SyntaxError`. (Contributed by Jelle Zijlstra in :gh:`121637`.) + +* Writes to ``__debug__`` are now detected even if the code is optimized + away by the :option:`-O` command-line option. For example, + ``python -O -c 'assert (__debug__ := 1)'`` now produces a + :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) + +* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` + to convert a number to :class:`float` or :class:`complex` type correspondingly. + They raise an error if the argument is a string. + (Contributed by Serhiy Storchaka in :gh:`84978`.) + +* :class:`super` objects are now :mod:`pickleable ` and + :mod:`copyable `. + (Contributed by Serhiy Storchaka in :gh:`125767`.) + +* The :class:`memoryview` type now supports subscription, + making it a :term:`generic type`. + (Contributed by Brian Schubert in :gh:`126012`.) + + +New modules +=========== + +* :mod:`annotationlib`: For introspecting :term:`annotations `. + See :pep:`749` for more details. + (Contributed by Jelle Zijlstra in :gh:`119180`.) + + +Improved modules +================ + +argparse +-------- + +* The default value of the :ref:`program name ` for + :class:`argparse.ArgumentParser` now reflects the way the Python + interpreter was instructed to find the ``__main__`` module code. + (Contributed by Serhiy Storchaka and Alyssa Coghlan in :gh:`66436`.) + +* Introduced the optional *suggest_on_error* parameter to + :class:`argparse.ArgumentParser`, enabling suggestions for argument choices + and subparser names if mistyped by the user. + (Contributed by Savannah Ostrowski in :gh:`124456`) + +ast +--- + +* Add :func:`ast.compare` for comparing two ASTs. + (Contributed by Batuhan Taskaya and Jeremy Hylton in :gh:`60191`.) + +* Add support for :func:`copy.replace` for AST nodes. + (Contributed by Bénédikt Tran in :gh:`121141`.) + +* Docstrings are now removed from an optimized AST in optimization level 2. + (Contributed by Irit Katriel in :gh:`123958`.) + +* The ``repr()`` output for AST nodes now includes more information. + (Contributed by Tomas R in :gh:`116022`.) + +concurrent.futures +------------------ + +* Add :class:`~concurrent.futures.InterpreterPoolExecutor`, + which exposes "subinterpreters (multiple Python interpreters in the + same process) to Python code. This is separate from the proposed API + in :pep:`734`. + (Contributed by Eric Snow in :gh:`124548`.) + +ctypes +------ + +* The layout of :ref:`bit fields ` + in :class:`~ctypes.Structure` and :class:`~ctypes.Union` + now matches platform defaults (GCC/Clang or MVSC) more closely. + In particular, fields no longer overlap. + (Contributed by Matthias Görgens in :gh:`97702`.) + +* The :attr:`.Structure._layout_` class attribute can now be set + to help match a non-default ABI. + (Contributed by Petr Viktorin in :gh:`97702`.) + +decimal +------- + +* Add alternative :class:`~decimal.Decimal` constructor + :meth:`Decimal.from_number() `. + (Contributed by Serhiy Storchaka in :gh:`121798`.) + +datetime +-------- + +* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. + (Contributed by Wannes Boeykens in :gh:`41431`.) + +dis +--- + +* Add support for rendering full source location information of + :class:`instructions `, rather than only the line number. + This feature is added to the following interfaces via the *show_positions* + keyword argument: + + - :class:`dis.Bytecode` + - :func:`dis.dis` + - :func:`dis.distb` + - :func:`dis.disassemble` + + This feature is also exposed via :option:`dis --show-positions`. + (Contributed by Bénédikt Tran in :gh:`123165`.) + + +fractions +--------- + +* Add support for converting any objects that have the + :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. + (Contributed by Serhiy Storchaka in :gh:`82017`.) + +* Add alternative :class:`~fractions.Fraction` constructor + :meth:`Fraction.from_number() `. + (Contributed by Serhiy Storchaka in :gh:`121797`.) + + +functools +--------- + +* Add support to :func:`functools.partial` and + :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels + to reserve a place for positional arguments. + (Contributed by Dominykas Grigonis in :gh:`119127`.) + + +http +---- + +* Directory lists and error pages generated by the :mod:`http.server` + module allow the browser to apply its default dark mode. + (Contributed by Yorik Hansen in :gh:`123430`.) + + +inspect +------- + +* :func:`inspect.signature` takes a new argument *annotation_format* to control + the :class:`annotationlib.Format` used for representing annotations. + (Contributed by Jelle Zijlstra in :gh:`101552`.) + +* :meth:`inspect.Signature.format` takes a new argument *unquote_annotations*. + If true, string :term:`annotations ` are displayed without surrounding quotes. + (Contributed by Jelle Zijlstra in :gh:`101552`.) + +* Add function :func:`inspect.ispackage` to determine whether an object is a + :term:`package` or not. + (Contributed by Zhikang Yan in :gh:`125634`.) + + +json +---- + +* Add notes for JSON serialization errors that allow to identify the source + of the error. + (Contributed by Serhiy Storchaka in :gh:`122163`.) + +* Enable the :mod:`json` module to work as a script using the :option:`-m` + switch: :program:`python -m json`. + See the :ref:`JSON command-line interface ` documentation. + (Contributed by Trey Hunner in :gh:`122873`.) + + +operator +-------- + +* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` + have been added, such that ``operator.is_none(obj)`` is equivalent + to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent + to ``obj is not None``. + (Contributed by Raymond Hettinger and Nico Mexis in :gh:`115808`.) + + +os +-- + +* Add the :data:`os.environ.refresh() ` method to update + :data:`os.environ` with changes to the environment made by :func:`os.putenv`, + by :func:`os.unsetenv`, or made outside Python in the same process. + (Contributed by Victor Stinner in :gh:`120057`.) + + +pathlib +------- + +* Add methods to :class:`pathlib.Path` to recursively copy or move files and + directories: + + * :meth:`~pathlib.Path.copy` copies a file or directory tree to a destination. + * :meth:`~pathlib.Path.copy_into` copies *into* a destination directory. + * :meth:`~pathlib.Path.move` moves a file or directory tree to a destination. + * :meth:`~pathlib.Path.move_into` moves *into* a destination directory. + + (Contributed by Barney Gale in :gh:`73991`.) + +* Add :meth:`pathlib.Path.scandir` to scan a directory and return an iterator + of :class:`os.DirEntry` objects. This is exactly equivalent to calling + :func:`os.scandir` on a path object. + + (Contributed by Barney Gale in :gh:`125413`.) + + +pdb +--- + +* Hardcoded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now + reuse the most recent :class:`~pdb.Pdb` instance that calls + :meth:`~pdb.Pdb.set_trace`, instead of creating a new one each time. + As a result, all the instance specific data like :pdbcmd:`display` and + :pdbcmd:`commands` are preserved across hardcoded breakpoints. + (Contributed by Tian Gao in :gh:`121450`.) + +* Add a new argument *mode* to :class:`pdb.Pdb`. Disable the ``restart`` + command when :mod:`pdb` is in ``inline`` mode. + (Contributed by Tian Gao in :gh:`123757`.) + +pickle +------ + +* Set the default protocol version on the :mod:`pickle` module to 5. + For more details, see :ref:`pickle protocols `. + +* Add notes for pickle serialization errors that allow to identify the source + of the error. + (Contributed by Serhiy Storchaka in :gh:`122213`.) + +pydoc +----- + +* :term:`Annotations ` in help output are now usually + displayed in a format closer to that in the original source. + (Contributed by Jelle Zijlstra in :gh:`101552`.) + + +symtable +-------- + +* Expose the following :class:`symtable.Symbol` methods: + + * :meth:`~symtable.Symbol.is_comp_cell` + * :meth:`~symtable.Symbol.is_comp_iter` + * :meth:`~symtable.Symbol.is_free_class` + + (Contributed by Bénédikt Tran in :gh:`120029`.) + + +sys +--- + +* The previously undocumented special function :func:`sys.getobjects`, + which only exists in specialized builds of Python, may now return objects + from other interpreters than the one it's called in. + + +unicodedata +----------- + +* The Unicode database has been updated to Unicode 16.0.0. + + +unittest +-------- + +* unittest discovery supports :term:`namespace package` as start + directory again. It was removed in Python 3.11. + (Contributed by Jacob Walls in :gh:`80958`.) + + +.. Add improved modules above alphabetically, not here at the end. + +Optimizations +============= + +asyncio +------- + +* :mod:`asyncio` now uses double linked list implementation for native tasks + which speeds up execution by 10% on standard pyperformance benchmarks and + reduces memory usage. + (Contributed by Kumar Aditya in :gh:`107803`.) + +Deprecated +========== + +* :mod:`argparse`: + + * Passing the undocumented keyword argument *prefix_chars* to + :meth:`~argparse.ArgumentParser.add_argument_group` is now + deprecated. + (Contributed by Savannah Ostrowski in :gh:`125563`.) + * Deprecated the :class:`argparse.FileType` type converter. + Anything with resource management should be done downstream after the + arguments are parsed. + (Contributed by Serhiy Storchaka in :gh:`58032`.) + +* :mod:`asyncio`: + :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + +* :mod:`builtins`: + Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; it should only be passed + as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) + +* :mod:`multiprocessing` and :mod:`concurrent.futures`: + The default start method (see :ref:`multiprocessing-start-methods`) changed + away from *fork* to *forkserver* on platforms where it was not already + *spawn* (Windows & macOS). If you require the threading incompatible *fork* + start method you must explicitly specify it when using :mod:`multiprocessing` + or :mod:`concurrent.futures` APIs. + (Contributed by Gregory P. Smith in :gh:`84559`.) + +* :mod:`os`: + :term:`Soft deprecate ` :func:`os.popen` and + :func:`os.spawn* ` functions. They should no longer be used to + write new code. The :mod:`subprocess` module is recommended instead. + (Contributed by Victor Stinner in :gh:`120743`.) + +* :mod:`symtable`: + Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. + (Contributed by Bénédikt Tran in :gh:`119698`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.15.rst + +.. include:: ../deprecations/pending-removal-in-3.16.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + +Removed +======= + +argparse +-------- + +* Remove the *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction`. + They were deprecated since 3.12. + +ast +--- + +* Remove the following classes. They were all deprecated since Python 3.8, + and have emitted deprecation warnings since Python 3.12: + + * :class:`!ast.Bytes` + * :class:`!ast.Ellipsis` + * :class:`!ast.NameConstant` + * :class:`!ast.Num` + * :class:`!ast.Str` + + Use :class:`ast.Constant` instead. As a consequence of these removals, + user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, + ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom + :class:`ast.NodeVisitor` subclasses will no longer be called when the + :class:`!NodeVisitor` subclass is visiting an AST. Define a ``visit_Constant`` + method instead. + + Also, remove the following deprecated properties on :class:`ast.Constant`, + which were present for compatibility with the now-removed AST classes: + + * :attr:`!ast.Constant.n` + * :attr:`!ast.Constant.s` + + Use :attr:`!ast.Constant.value` instead. + + (Contributed by Alex Waygood in :gh:`119562`.) + +asyncio +------- + +* Remove the following classes and functions. They were all deprecated and + emitted deprecation warnings since Python 3.12: + + * :func:`!asyncio.get_child_watcher` + * :func:`!asyncio.set_child_watcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` + * :class:`!asyncio.AbstractChildWatcher` + * :class:`!asyncio.FastChildWatcher` + * :class:`!asyncio.MultiLoopChildWatcher` + * :class:`!asyncio.PidfdChildWatcher` + * :class:`!asyncio.SafeChildWatcher` + * :class:`!asyncio.ThreadedChildWatcher` + + (Contributed by Kumar Aditya in :gh:`120804`.) + +collections.abc +--------------- + +* Remove :class:`!collections.abc.ByteString`. It had previously raised a + :exc:`DeprecationWarning` since Python 3.12. + +email +----- + +* Remove the *isdst* parameter from :func:`email.utils.localtime`. + (Contributed by Hugo van Kemenade in :gh:`118798`.) + +importlib +--------- + +* Remove deprecated :mod:`importlib.abc` classes: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + +itertools +--------- + +* Remove :mod:`itertools` support for copy, deepcopy, and pickle operations. + These had previously raised a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Raymond Hettinger in :gh:`101588`.) + +pathlib +------- + +* Remove support for passing additional keyword arguments to + :class:`pathlib.Path`. In previous versions, any such arguments are ignored. +* Remove support for passing additional positional arguments to + :meth:`pathlib.PurePath.relative_to` and + :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such + arguments are joined onto *other*. + +pkgutil +------- + +* Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. + These had previously raised a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Bénédikt Tran in :gh:`97850`.) + +pty +--- + +* Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. + They had previously raised a :exc:`DeprecationWarning` since Python 3.12. + Use :func:`pty.openpty` instead. + (Contributed by Nikita Sobolev in :gh:`118824`.) + +sqlite3 +------- + +* Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. + (Contributed by Hugo van Kemenade in :gh:`118924`.) + +* Disallow using a sequence of parameters with named placeholders. + This had previously raised a :exc:`DeprecationWarning` since Python 3.12; + it will now raise a :exc:`sqlite3.ProgrammingError`. + (Contributed by Erlend E. Aasland in :gh:`118928` and :gh:`101693`.) + +typing +------ + +* Remove :class:`!typing.ByteString`. It had previously raised a + :exc:`DeprecationWarning` since Python 3.12. + +urllib +------ + +* Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. + It had previously raised a :exc:`DeprecationWarning` since Python 3.11. + (Contributed by Nikita Sobolev in :gh:`118827`.) + +Others +------ + +* Using :data:`NotImplemented` in a boolean context will now raise a :exc:`TypeError`. + It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed + by Jelle Zijlstra in :gh:`118767`.) + +* The :func:`int` built-in no longer delegates to + :meth:`~object.__trunc__`. Classes that want to support conversion to + integer must implement either :meth:`~object.__int__` or + :meth:`~object.__index__`. (Contributed by Mark Dickinson in :gh:`119743`.) + + +Porting to Python 3.14 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +Changes in the Python API +------------------------- + +* :class:`functools.partial` is now a method descriptor. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) + +* The :func:`locale.nl_langinfo` function now sets temporarily the ``LC_CTYPE`` + locale in some cases. + This temporary change affects other threads. + (Contributed by Serhiy Storchaka in :gh:`69998`.) + + +Build changes +============= + + +C API changes +============= + +New features +------------ + +* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. + (Contributed by Sergey B Kirpichev in :gh:`116560`.) + +* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` + object: + + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteWideChar` + + (Contributed by Victor Stinner in :gh:`119182`.) + +* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, + which has an ambiguous return value. + (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) + +* :c:func:`Py_Finalize` now deletes all interned strings. This + is backwards incompatible to any C-Extension that holds onto an interned + string after a call to :c:func:`Py_Finalize` and is then reused after a + call to :c:func:`Py_Initialize`. Any issues arising from this behavior will + normally result in crashes during the execution of the subsequent call to + :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use + an address sanitizer to identify any use-after-free coming from + an interned string and deallocate it during module shutdown. + (Contributed by Eddie Elizondo in :gh:`113601`.) + +* Add new functions to convert C ```` numbers from/to Python + :class:`int`: + + * :c:func:`PyLong_AsInt32` + * :c:func:`PyLong_AsInt64` + * :c:func:`PyLong_AsUInt32` + * :c:func:`PyLong_AsUInt64` + * :c:func:`PyLong_FromInt32` + * :c:func:`PyLong_FromInt64` + * :c:func:`PyLong_FromUInt32` + * :c:func:`PyLong_FromUInt64` + + (Contributed by Victor Stinner in :gh:`120389`.) + +* Add :c:func:`PyBytes_Join(sep, iterable) ` function, + similar to ``sep.join(iterable)`` in Python. + (Contributed by Victor Stinner in :gh:`121645`.) + +* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. + (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) + +* Add functions to get and set the current runtime Python configuration + (:pep:`741`): + + * :c:func:`PyConfig_Get` + * :c:func:`PyConfig_GetInt` + * :c:func:`PyConfig_Set` + * :c:func:`PyConfig_Names` + + (Contributed by Victor Stinner in :gh:`107954`.) + +* Add functions to configure the Python initialization (:pep:`741`): + + * :c:func:`Py_InitializeFromInitConfig` + * :c:func:`PyInitConfig_AddModule` + * :c:func:`PyInitConfig_Create` + * :c:func:`PyInitConfig_Free` + * :c:func:`PyInitConfig_FreeStrList` + * :c:func:`PyInitConfig_GetError` + * :c:func:`PyInitConfig_GetExitCode` + * :c:func:`PyInitConfig_GetInt` + * :c:func:`PyInitConfig_GetStr` + * :c:func:`PyInitConfig_GetStrList` + * :c:func:`PyInitConfig_HasOption` + * :c:func:`PyInitConfig_SetInt` + * :c:func:`PyInitConfig_SetStr` + * :c:func:`PyInitConfig_SetStrList` + + (Contributed by Victor Stinner in :gh:`107954`.) + +* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for easier + superclass identification, which attempts to resolve the `type checking issue + `__ mentioned in :pep:`630` + (:gh:`124153`). + +* Add :c:func:`PyUnicode_Equal` function to the limited C API: + test if two strings are equal. + (Contributed by Victor Stinner in :gh:`124502`.) + + +* Add :c:func:`PyType_Freeze` function to make a type immutable. + (Contributed by Victor Stinner in :gh:`121654`.) + +Porting to Python 3.14 +---------------------- + +* In the limited C API 3.14 and newer, :c:func:`Py_TYPE` and + :c:func:`Py_REFCNT` are now implemented as an opaque function call to hide + implementation details. + (Contributed by Victor Stinner in :gh:`120600` and :gh:`124127`.) + + +Deprecated +---------- + +* The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`, + use :c:macro:`!Py_INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`120026`.) + +* Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` + and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, + use instead :c:macro:`!isnan`, :c:macro:`!isinf` and + :c:macro:`!isfinite` available from :file:`math.h` + since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) + +.. Add C API deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + +Removed +------- + +* Creating :c:data:`immutable types ` with mutable + bases was deprecated since 3.12 and now raises a :exc:`TypeError`. diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 9834bc03dc4b74..7104904c956a7a 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -312,8 +312,8 @@ cluttering source directories, the *pyc* files are now collected in a Aside from the filenames and target directories, the new scheme has a few aspects that are visible to the programmer: -* Imported modules now have a :attr:`__cached__` attribute which stores the name - of the actual file that was imported: +* Imported modules now have a :attr:`~module.__cached__` attribute which stores + the name of the actual file that was imported: >>> import collections >>> collections.__cached__ # doctest: +SKIP @@ -344,8 +344,8 @@ aspects that are visible to the programmer: * The :mod:`importlib.abc` module has been updated with new :term:`abstract base classes ` for loading bytecode files. The obsolete - ABCs, :class:`~importlib.abc.PyLoader` and - :class:`~importlib.abc.PyPycLoader`, have been deprecated (instructions on how + ABCs, :class:`!PyLoader` and + :class:`!PyPycLoader`, have been deprecated (instructions on how to stay Python 3.1 compatible are included with the documentation). .. seealso:: @@ -401,7 +401,7 @@ The *native strings* are always of type :class:`str` but are restricted to code points between *U+0000* through *U+00FF* which are translatable to bytes using *Latin-1* encoding. These strings are used for the keys and values in the environment dictionary and for response headers and statuses in the -:func:`start_response` function. They must follow :rfc:`2616` with respect to +:func:`!start_response` function. They must follow :rfc:`2616` with respect to encoding. That is, they must either be *ISO-8859-1* characters or use :rfc:`2047` MIME encoding. @@ -415,8 +415,8 @@ points: encoded in utf-8 was using ``h.encode('utf-8')`` now needs to convert from bytes to native strings using ``h.encode('utf-8').decode('latin-1')``. -* Values yielded by an application or sent using the :meth:`write` method - must be byte strings. The :func:`start_response` function and environ +* Values yielded by an application or sent using the :meth:`!write` method + must be byte strings. The :func:`!start_response` function and environ must use native strings. The two cannot be mixed. For server implementers writing CGI-to-WSGI pathways or other CGI-style @@ -499,7 +499,7 @@ Some smaller changes made to the core Python language are: * The :func:`hasattr` function works by calling :func:`getattr` and detecting whether an exception is raised. This technique allows it to detect methods - created dynamically by :meth:`__getattr__` or :meth:`__getattribute__` which + created dynamically by :meth:`~object.__getattr__` or :meth:`~object.__getattribute__` which would otherwise be absent from the class dictionary. Formerly, *hasattr* would catch any exception, possibly masking genuine errors. Now, *hasattr* has been tightened to only catch :exc:`AttributeError` and let other @@ -531,7 +531,7 @@ Some smaller changes made to the core Python language are: (Proposed and implemented by Mark Dickinson; :issue:`9337`.) -* :class:`memoryview` objects now have a :meth:`~memoryview.release()` method +* :class:`memoryview` objects now have a :meth:`~memoryview.release` method and they also now support the context management protocol. This allows timely release of any resources that were acquired when requesting a buffer from the original object. @@ -620,7 +620,7 @@ Some smaller changes made to the core Python language are: * :class:`range` objects now support *index* and *count* methods. This is part of an effort to make more objects fully implement the - :class:`collections.Sequence` :term:`abstract base class`. As a result, the + :class:`collections.Sequence ` :term:`abstract base class`. As a result, the language will have a more uniform API. In addition, :class:`range` objects now support slicing and negative indices, even with values larger than :data:`sys.maxsize`. This makes *range* more interoperable with lists:: @@ -720,7 +720,7 @@ format. elementtree ----------- -The :mod:`xml.etree.ElementTree` package and its :mod:`xml.etree.cElementTree` +The :mod:`xml.etree.ElementTree` package and its :mod:`!xml.etree.cElementTree` counterpart have been updated to version 1.3. Several new and useful functions and methods have been added: @@ -743,8 +743,8 @@ Several new and useful functions and methods have been added: Two methods have been deprecated: -* :meth:`xml.etree.ElementTree.getchildren` use ``list(elem)`` instead. -* :meth:`xml.etree.ElementTree.getiterator` use ``Element.iter`` instead. +* :meth:`!xml.etree.ElementTree.getchildren` use ``list(elem)`` instead. +* :meth:`!xml.etree.ElementTree.getiterator` use ``Element.iter`` instead. For details of the update, see `Introducing ElementTree `_ @@ -785,8 +785,8 @@ functools (Contributed by Raymond Hettinger and incorporating design ideas from Jim Baker, Miki Tebeka, and Nick Coghlan; see `recipe 498245 - `_\, `recipe 577479 - `_\, :issue:`10586`, and + `_\, `recipe 577479 + `_\, :issue:`10586`, and :issue:`10593`.) * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute @@ -1008,13 +1008,13 @@ datetime and time after 1900. The new supported year range is from 1000 to 9999 inclusive. * Whenever a two-digit year is used in a time tuple, the interpretation has been - governed by :data:`time.accept2dyear`. The default is ``True`` which means that + governed by :data:`!time.accept2dyear`. The default is ``True`` which means that for a two-digit year, the century is guessed according to the POSIX rules governing the ``%y`` strptime format. Starting with Py3.2, use of the century guessing heuristic will emit a :exc:`DeprecationWarning`. Instead, it is recommended that - :data:`time.accept2dyear` be set to ``False`` so that large date ranges + :data:`!time.accept2dyear` be set to ``False`` so that large date ranges can be used without guesswork:: >>> import time, warnings @@ -1032,7 +1032,7 @@ datetime and time 'Fri Jan 1 12:34:56 11' Several functions now have significantly expanded date ranges. When - :data:`time.accept2dyear` is false, the :func:`time.asctime` function will + :data:`!time.accept2dyear` is false, the :func:`time.asctime` function will accept any year that fits in a C int, while the :func:`time.mktime` and :func:`time.strftime` functions will accept the full range supported by the corresponding operating system functions. @@ -1148,15 +1148,15 @@ for slice notation are well-suited to in-place editing:: reprlib ------- -When writing a :meth:`__repr__` method for a custom container, it is easy to +When writing a :meth:`~object.__repr__` method for a custom container, it is easy to forget to handle the case where a member refers back to the container itself. Python's builtin objects such as :class:`list` and :class:`set` handle self-reference by displaying "..." in the recursive part of the representation string. -To help write such :meth:`__repr__` methods, the :mod:`reprlib` module has a new +To help write such :meth:`~object.__repr__` methods, the :mod:`reprlib` module has a new decorator, :func:`~reprlib.recursive_repr`, for detecting recursive calls to -:meth:`__repr__` and substituting a placeholder string instead:: +:meth:`!__repr__` and substituting a placeholder string instead:: >>> class MyList(list): ... @recursive_repr() @@ -1308,11 +1308,11 @@ used for the imaginary part of a number: >>> sys.hash_info # doctest: +SKIP sys.hash_info(width=64, modulus=2305843009213693951, inf=314159, nan=0, imag=1000003) -An early decision to limit the inter-operability of various numeric types has +An early decision to limit the interoperability of various numeric types has been relaxed. It is still unsupported (and ill-advised) to have implicit mixing in arithmetic expressions such as ``Decimal('1.1') + float('1.1')`` because the latter loses information in the process of constructing the binary -float. However, since existing floating point value can be converted losslessly +float. However, since existing floating-point value can be converted losslessly to either a decimal or rational representation, it makes sense to add them to the constructor and to support mixed-type comparisons. @@ -1325,7 +1325,7 @@ the constructor and to support mixed-type comparisons. and :class:`fractions.Fraction` (:issue:`2531` and :issue:`8188`). Similar changes were made to :class:`fractions.Fraction` so that the -:meth:`~fractions.Fraction.from_float()` and :meth:`~fractions.Fraction.from_decimal` +:meth:`~fractions.Fraction.from_float` and :meth:`~fractions.Fraction.from_decimal` methods are no longer needed (:issue:`8294`): >>> from decimal import Decimal @@ -1336,7 +1336,7 @@ Decimal('1.100000000000000088817841970012523233890533447265625') Fraction(2476979795053773, 2251799813685248) Another useful change for the :mod:`decimal` module is that the -:attr:`Context.clamp` attribute is now public. This is useful in creating +:attr:`Context.clamp ` attribute is now public. This is useful in creating contexts that correspond to the decimal interchange formats specified in IEEE 754 (see :issue:`8540`). @@ -1428,7 +1428,7 @@ before compressing and decompressing: Aides and Brian Curtin in :issue:`9962`, :issue:`1675951`, :issue:`7471` and :issue:`2846`.) -Also, the :class:`zipfile.ZipExtFile` class was reworked internally to represent +Also, the :class:`zipfile.ZipExtFile ` class was reworked internally to represent files stored inside an archive. The new implementation is significantly faster and can be wrapped in an :class:`io.BufferedReader` object for more speedups. It also solves an issue where interleaved calls to *read* and *readline* gave the @@ -1596,7 +1596,7 @@ sqlite3 The :mod:`sqlite3` module was updated to pysqlite version 2.6.0. It has two new capabilities. -* The :attr:`sqlite3.Connection.in_transit` attribute is true if there is an +* The :attr:`!sqlite3.Connection.in_transit` attribute is true if there is an active transaction for uncommitted changes. * The :meth:`sqlite3.Connection.enable_load_extension` and @@ -1622,7 +1622,7 @@ socket The :mod:`socket` module has two new improvements. -* Socket objects now have a :meth:`~socket.socket.detach()` method which puts +* Socket objects now have a :meth:`~socket.socket.detach` method which puts the socket into closed state without actually closing the underlying file descriptor. The latter can then be reused for other purposes. (Added by Antoine Pitrou; :issue:`8524`.) @@ -1643,14 +1643,14 @@ for secure (encrypted, authenticated) internet connections: other options. It includes a :meth:`~ssl.SSLContext.wrap_socket` for creating an SSL socket from an SSL context. -* A new function, :func:`ssl.match_hostname`, supports server identity +* A new function, :func:`!ssl.match_hostname`, supports server identity verification for higher-level protocols by implementing the rules of HTTPS (from :rfc:`2818`) which are also suitable for other protocols. -* The :func:`ssl.wrap_socket` constructor function now takes a *ciphers* +* The :func:`ssl.wrap_socket() ` constructor function now takes a *ciphers* argument. The *ciphers* string lists the allowed encryption algorithms using the format described in the `OpenSSL documentation - `__. + `__. * When linked against recent versions of OpenSSL, the :mod:`ssl` module now supports the Server Name Indication extension to the TLS protocol, allowing @@ -1759,7 +1759,7 @@ names. (Contributed by Michael Foord.) * Experimentation at the interactive prompt is now easier because the - :class:`unittest.case.TestCase` class can now be instantiated without + :class:`unittest.TestCase` class can now be instantiated without arguments: >>> from unittest import TestCase @@ -1797,7 +1797,7 @@ names. * In addition, the method names in the module have undergone a number of clean-ups. For example, :meth:`~unittest.TestCase.assertRegex` is the new name for - :meth:`~unittest.TestCase.assertRegexpMatches` which was misnamed because the + :meth:`!assertRegexpMatches` which was misnamed because the test uses :func:`re.search`, not :func:`re.match`. Other methods using regular expressions are now named using short form "Regex" in preference to "Regexp" -- this matches the names used in other unittest implementations, @@ -1812,11 +1812,11 @@ names. =============================== ============================== Old Name Preferred Name =============================== ============================== - :meth:`assert_` :meth:`.assertTrue` - :meth:`assertEquals` :meth:`.assertEqual` - :meth:`assertNotEquals` :meth:`.assertNotEqual` - :meth:`assertAlmostEquals` :meth:`.assertAlmostEqual` - :meth:`assertNotAlmostEquals` :meth:`.assertNotAlmostEqual` + :meth:`!assert_` :meth:`.assertTrue` + :meth:`!assertEquals` :meth:`.assertEqual` + :meth:`!assertNotEquals` :meth:`.assertNotEqual` + :meth:`!assertAlmostEquals` :meth:`.assertAlmostEqual` + :meth:`!assertNotAlmostEquals` :meth:`.assertNotAlmostEqual` =============================== ============================== Likewise, the ``TestCase.fail*`` methods deprecated in Python 3.1 are expected @@ -1824,7 +1824,7 @@ names. (Contributed by Ezio Melotti; :issue:`9424`.) -* The :meth:`~unittest.TestCase.assertDictContainsSubset` method was deprecated +* The :meth:`!assertDictContainsSubset` method was deprecated because it was misimplemented with the arguments in the wrong order. This created hard-to-debug optical illusions where tests like ``TestCase().assertDictContainsSubset({'a':1, 'b':2}, {'a':1})`` would fail. @@ -1859,11 +1859,11 @@ asyncore -------- :class:`!asyncore.dispatcher` now provides a -:meth:`!handle_accepted()` method +:meth:`!handle_accepted` method returning a ``(sock, addr)`` pair which is called when a connection has actually been established with a new remote endpoint. This is supposed to be used as a -replacement for old :meth:`!handle_accept()` and avoids -the user to call :meth:`!accept()` directly. +replacement for old :meth:`!handle_accept` and avoids +the user to call :meth:`!accept` directly. (Contributed by Giampaolo Rodolà; :issue:`6706`.) @@ -1997,7 +1997,7 @@ under-the-hood. dbm --- -All database modules now support the :meth:`get` and :meth:`setdefault` methods. +All database modules now support the :meth:`!get` and :meth:`!setdefault` methods. (Suggested by Ray Allen in :issue:`9523`.) @@ -2118,7 +2118,7 @@ The :mod:`pdb` debugger module gained a number of usability improvements: :file:`.pdbrc` script file. * A :file:`.pdbrc` script file can contain ``continue`` and ``next`` commands that continue debugging. -* The :class:`Pdb` class constructor now accepts a *nosigint* argument. +* The :class:`~pdb.Pdb` class constructor now accepts a *nosigint* argument. * New commands: ``l(list)``, ``ll(long list)`` and ``source`` for listing source code. * New commands: ``display`` and ``undisplay`` for showing or hiding @@ -2321,7 +2321,7 @@ Multi-threading intervals and reduced overhead due to lock contention and the number of ensuing system calls. The notion of a "check interval" to allow thread switches has been abandoned and replaced by an absolute duration expressed in - seconds. This parameter is tunable through :func:`sys.setswitchinterval()`. + seconds. This parameter is tunable through :func:`sys.setswitchinterval`. It currently defaults to 5 milliseconds. Additional details about the implementation can be read from a `python-dev @@ -2394,11 +2394,11 @@ A number of small performance enhancements have been added: (Contributed by Antoine Pitrou; :issue:`3001`.) -* The fast-search algorithm in stringlib is now used by the :meth:`split`, - :meth:`rsplit`, :meth:`splitlines` and :meth:`replace` methods on +* The fast-search algorithm in stringlib is now used by the :meth:`~str.split`, + :meth:`~str.rsplit`, :meth:`~str.splitlines` and :meth:`~str.replace` methods on :class:`bytes`, :class:`bytearray` and :class:`str` objects. Likewise, the - algorithm is also used by :meth:`rfind`, :meth:`rindex`, :meth:`rsplit` and - :meth:`rpartition`. + algorithm is also used by :meth:`~str.rfind`, :meth:`~str.rindex`, :meth:`~str.rsplit` and + :meth:`~str.rpartition`. (Patch by Florent Xicluna in :issue:`7622` and :issue:`7462`.) @@ -2410,8 +2410,8 @@ A number of small performance enhancements have been added: There were several other minor optimizations. Set differencing now runs faster when one operand is much larger than the other (patch by Andress Bennetts in -:issue:`8685`). The :meth:`array.repeat` method has a faster implementation -(:issue:`1569291` by Alexander Belopolsky). The :class:`BaseHTTPRequestHandler` +:issue:`8685`). The :meth:`!array.repeat` method has a faster implementation +(:issue:`1569291` by Alexander Belopolsky). The :class:`~http.server.BaseHTTPRequestHandler` has more efficient buffering (:issue:`3709` by Andrew Schaaf). The :func:`operator.attrgetter` function has been sped-up (:issue:`10160` by Christos Georgiou). And :class:`~configparser.ConfigParser` loads multi-line arguments a bit @@ -2562,11 +2562,11 @@ Changes to Python's build process and to the C API include: (Suggested by Raymond Hettinger and implemented by Benjamin Peterson; :issue:`9778`.) -* A new macro :c:macro:`Py_VA_COPY` copies the state of the variable argument +* A new macro :c:macro:`!Py_VA_COPY` copies the state of the variable argument list. It is equivalent to C99 *va_copy* but available on all Python platforms (:issue:`2443`). -* A new C API function :c:func:`PySys_SetArgvEx` allows an embedded interpreter +* A new C API function :c:func:`!PySys_SetArgvEx` allows an embedded interpreter to set :data:`sys.argv` without also modifying :data:`sys.path` (:issue:`5753`). @@ -2650,8 +2650,9 @@ require changes to your code: * :class:`bytearray` objects can no longer be used as filenames; instead, they should be converted to :class:`bytes`. -* The :meth:`array.tostring` and :meth:`array.fromstring` have been renamed to - :meth:`array.tobytes` and :meth:`array.frombytes` for clarity. The old names +* The :meth:`!array.tostring` and :meth:`!array.fromstring` have been renamed to + :meth:`array.tobytes() ` and + :meth:`array.frombytes() ` for clarity. The old names have been deprecated. (See :issue:`8990`.) * ``PyArg_Parse*()`` functions: @@ -2664,7 +2665,7 @@ require changes to your code: instead; the new type has a well-defined interface for passing typing safety information and a less complicated signature for calling a destructor. -* The :func:`sys.setfilesystemencoding` function was removed because +* The :func:`!sys.setfilesystemencoding` function was removed because it had a flawed design. * The :func:`random.seed` function and method now salt string seeds with an @@ -2672,7 +2673,7 @@ require changes to your code: reproduce Python 3.1 sequences, set the *version* argument to *1*, ``random.seed(s, version=1)``. -* The previously deprecated :func:`string.maketrans` function has been removed +* The previously deprecated :func:`!string.maketrans` function has been removed in favor of the static methods :meth:`bytes.maketrans` and :meth:`bytearray.maketrans`. This change solves the confusion around which types were supported by the :mod:`string` module. Now, :class:`str`, @@ -2682,7 +2683,7 @@ require changes to your code: (Contributed by Georg Brandl; :issue:`5675`.) -* The previously deprecated :func:`contextlib.nested` function has been removed +* The previously deprecated :func:`!contextlib.nested` function has been removed in favor of a plain :keyword:`with` statement which can accept multiple context managers. The latter technique is faster (because it is built-in), and it does a better job finalizing multiple context managers when one of them diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 29b4034e32821c..f814c4e90d5719 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -549,9 +549,11 @@ separation of binary and text data). PEP 3155: Qualified name for classes and functions ================================================== -Functions and class objects have a new ``__qualname__`` attribute representing +Functions and class objects have a new :attr:`~definition.__qualname__` +attribute representing the "path" from the module top-level to their definition. For global functions -and classes, this is the same as ``__name__``. For other functions and classes, +and classes, this is the same as :attr:`~definition.__name__`. +For other functions and classes, it provides better information about where they were actually defined, and how they might be accessible from the global scope. @@ -779,8 +781,8 @@ Other Language Changes Some smaller changes made to the core Python language are: * Added support for Unicode name aliases and named sequences. - Both :func:`unicodedata.lookup()` and ``'\N{...}'`` now resolve name aliases, - and :func:`unicodedata.lookup()` resolves named sequences too. + Both :func:`unicodedata.lookup` and ``'\N{...}'`` now resolve name aliases, + and :func:`unicodedata.lookup` resolves named sequences too. (Contributed by Ezio Melotti in :issue:`12753`.) @@ -1097,12 +1099,12 @@ decimal C-module and libmpdec written by Stefan Krah. The new C version of the decimal module integrates the high speed libmpdec -library for arbitrary precision correctly rounded decimal floating point +library for arbitrary precision correctly rounded decimal floating-point arithmetic. libmpdec conforms to IBM's General Decimal Arithmetic Specification. Performance gains range from 10x for database applications to 100x for numerically intensive applications. These numbers are expected gains -for standard precisions used in decimal floating point arithmetic. Since +for standard precisions used in decimal floating-point arithmetic. Since the precision is user configurable, the exact figures may vary. For example, in integer bignum arithmetic the differences can be significantly higher. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index e07eda985d9bad..9d746b378995c3 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -215,13 +215,12 @@ automatic ``PATH`` modifications to have ``pip`` available from the command line by default, otherwise it can still be accessed through the Python launcher for Windows as ``py -m pip``. -As `discussed in the PEP`__, platform packagers may choose not to install +As :pep:`discussed in the PEP <0453#recommendations-for-downstream-distributors>` +platform packagers may choose not to install these commands by default, as long as, when invoked, they provide clear and simple directions on how to install them on that platform (usually using the system package manager). -__ https://peps.python.org/pep-0453/#recommendations-for-downstream-distributors - .. note:: To avoid conflicts between parallel Python 2 and Python 3 installations, @@ -872,7 +871,7 @@ multiple implementations of an operation that allows it to work with PEP written and implemented by Łukasz Langa. :func:`~functools.total_ordering` now supports a return value of -:const:`NotImplemented` from the underlying comparison function. (Contributed +:data:`NotImplemented` from the underlying comparison function. (Contributed by Katie Miller in :issue:`10042`.) A pure-python version of the :func:`~functools.partial` function is now in the @@ -1495,7 +1494,7 @@ The dictionary returned by :meth:`.SSLSocket.getpeercert` contains additional stat ---- -The :mod:`stat` module is now backed by a C implementation in :mod:`_stat`. A C +The :mod:`stat` module is now backed by a C implementation in :mod:`!_stat`. A C implementation is required as most of the values aren't standardized and are platform-dependent. (Contributed by Christian Heimes in :issue:`11016`.) @@ -1963,11 +1962,11 @@ Other Improvements `_ will build python, run the test suite, and generate an HTML coverage report for the C codebase using ``gcov`` and `lcov - `_. + `_. * The ``-R`` option to the :ref:`python regression test suite ` now also checks for memory allocation leaks, using - :func:`sys.getallocatedblocks()`. (Contributed by Antoine Pitrou in + :func:`sys.getallocatedblocks`. (Contributed by Antoine Pitrou in :issue:`13390`.) * ``python -m`` now works with namespace packages. @@ -2272,7 +2271,8 @@ Changes in the Python API :func:`super` and falling through all the way to the ABCs. For compatibility, catch both :exc:`NotImplementedError` or the appropriate exception as needed. -* The module type now initializes the :attr:`__package__` and :attr:`__loader__` +* The module type now initializes the :attr:`~module.__package__` and + :attr:`~module.__loader__` attributes to ``None`` by default. To determine if these attributes were set in a backwards-compatible fashion, use e.g. ``getattr(module, '__loader__', None) is not None``. (:issue:`17115`.) @@ -2413,7 +2413,7 @@ Changes in the Python API formal public interface the naming has been made consistent (:issue:`18532`). * Because :mod:`unittest.TestSuite` now drops references to tests after they - are run, test harnesses that re-use a :class:`~unittest.TestSuite` to re-run + are run, test harnesses that reuse a :class:`~unittest.TestSuite` to re-run a set of tests may fail. Test suites should not be re-used in this fashion since it means state is retained between test runs, breaking the test isolation that :mod:`unittest` is designed to provide. However, if the lack diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 5c2ec230441b42..3f3f634171dab6 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -423,8 +423,8 @@ are declared in the annotations:: return 'Hello ' + name While these annotations are available at runtime through the usual -:attr:`__annotations__` attribute, *no automatic type checking happens at -runtime*. Instead, it is assumed that a separate off-line type checker +:attr:`~object.__annotations__` attribute, *no automatic type checking happens +at runtime*. Instead, it is assumed that a separate off-line type checker (e.g. `mypy `_) will be used for on-demand source code analysis. @@ -951,7 +951,7 @@ New :class:`~collections.abc.Awaitable`, :class:`~collections.abc.Coroutine`, (Contributed by Yury Selivanov in :issue:`24184`.) For earlier Python versions, a backport of the new ABCs is available in an -external `PyPI package `_. +external :pypi:`PyPI package `. compileall @@ -1667,7 +1667,7 @@ Both the :class:`!SMTPServer` and :class:`!SMTPChannel` classes now accept a *decode_data* keyword argument to determine if the ``DATA`` portion of the SMTP transaction is decoded using the ``"utf-8"`` codec or is instead provided to the -:meth:`!SMTPServer.process_message()` +:meth:`!SMTPServer.process_message` method as a byte string. The default is ``True`` for backward compatibility reasons, but will change to ``False`` in Python 3.6. If *decode_data* is set to ``False``, the ``process_message`` method must be prepared to accept keyword @@ -1677,14 +1677,14 @@ arguments. The :class:`!SMTPServer` class now advertises the ``8BITMIME`` extension (:rfc:`6152`) if *decode_data* has been set ``True``. If the client specifies ``BODY=8BITMIME`` on the ``MAIL`` command, it is passed to -:meth:`!SMTPServer.process_message()` +:meth:`!SMTPServer.process_message` via the *mail_options* keyword. (Contributed by Milan Oberkirch and R. David Murray in :issue:`21795`.) The :class:`!SMTPServer` class now also supports the ``SMTPUTF8`` extension (:rfc:`6531`: Internationalized Email). If the client specified ``SMTPUTF8 BODY=8BITMIME`` on the ``MAIL`` command, they are passed to -:meth:`!SMTPServer.process_message()` +:meth:`!SMTPServer.process_message` via the *mail_options* keyword. It is the responsibility of the ``process_message`` method to correctly handle the ``SMTPUTF8`` data. (Contributed by Milan Oberkirch in :issue:`21725`.) @@ -1935,8 +1935,8 @@ specifying the namespace in which the code will be running. tkinter ------- -The :mod:`tkinter._fix` module used for setting up the Tcl/Tk environment -on Windows has been replaced by a private function in the :mod:`_tkinter` +The :mod:`!tkinter._fix` module used for setting up the Tcl/Tk environment +on Windows has been replaced by a private function in the :mod:`!_tkinter` module which makes no permanent changes to environment variables. (Contributed by Zachary Ware in :issue:`20035`.) @@ -2405,7 +2405,7 @@ Changes in the Python API error-prone and has been removed in Python 3.5. See :issue:`13936` for full details. -* The :meth:`ssl.SSLSocket.send()` method now raises either +* The :meth:`ssl.SSLSocket.send` method now raises either :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` on a non-blocking socket if the operation would block. Previously, it would return ``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) @@ -2526,9 +2526,9 @@ Changes in the C API to format the :func:`repr` of the object. (Contributed by Serhiy Storchaka in :issue:`22453`.) -* Because the lack of the :attr:`__module__` attribute breaks pickling and +* Because the lack of the :attr:`~type.__module__` attribute breaks pickling and introspection, a deprecation warning is now raised for builtin types without - the :attr:`__module__` attribute. This would be an AttributeError in + the :attr:`~type.__module__` attribute. This will be an :exc:`AttributeError` in the future. (Contributed by Serhiy Storchaka in :issue:`20204`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 11e1d73232a96d..2276fed60c8db3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -261,7 +261,7 @@ allowed. The :ref:`string formatting ` language also now has support for the ``'_'`` option to signal the use of an underscore for a thousands -separator for floating point presentation types and for integer +separator for floating-point presentation types and for integer presentation type ``'d'``. For integer presentation types ``'b'``, ``'o'``, ``'x'``, and ``'X'``, underscores will be inserted every 4 digits:: @@ -511,10 +511,10 @@ correct. Prior to Python 3.6, data loss could result when using bytes paths on Windows. With this change, using bytes to represent paths is now supported on Windows, provided those bytes are encoded with the encoding returned by -:func:`sys.getfilesystemencoding()`, which now defaults to ``'utf-8'``. +:func:`sys.getfilesystemencoding`, which now defaults to ``'utf-8'``. Applications that do not use str to represent paths should use -:func:`os.fsencode()` and :func:`os.fsdecode()` to ensure their bytes are +:func:`os.fsencode` and :func:`os.fsdecode` to ensure their bytes are correctly encoded. To revert to the previous behaviour, set :envvar:`PYTHONLEGACYWINDOWSFSENCODING` or call :func:`sys._enablelegacywindowsfsencoding`. @@ -549,7 +549,7 @@ PEP 520: Preserving Class Attribute Definition Order Attributes in a class definition body have a natural ordering: the same order in which the names appear in the source. This order is now -preserved in the new class's :attr:`~object.__dict__` attribute. +preserved in the new class's :attr:`~type.__dict__` attribute. Also, the effective default class *execution* namespace (returned from :ref:`type.__prepare__() `) is now an insertion-order-preserving @@ -780,7 +780,7 @@ for managing secrets, such as account authentication, tokens, and similar. Note that the pseudo-random generators in the :mod:`random` module should *NOT* be used for security purposes. Use :mod:`secrets` - on Python 3.6+ and :func:`os.urandom()` on Python 3.5 and earlier. + on Python 3.6+ and :func:`os.urandom` on Python 3.5 and earlier. .. seealso:: @@ -934,7 +934,7 @@ asynchronous generators. The :func:`~collections.namedtuple` function now accepts an optional keyword argument *module*, which, when specified, is used for -the ``__module__`` attribute of the returned named tuple class. +the :attr:`~type.__module__` attribute of the returned named tuple class. (Contributed by Raymond Hettinger in :issue:`17941`.) The *verbose* and *rename* arguments for @@ -1316,7 +1316,7 @@ Storchaka in :issue:`24164`.) pickletools ----------- -:func:`pickletools.dis()` now outputs the implicit memo index for the +:func:`pickletools.dis` now outputs the implicit memo index for the ``MEMOIZE`` opcode. (Contributed by Serhiy Storchaka in :issue:`25382`.) @@ -1472,6 +1472,10 @@ Server and client-side specific TLS protocols for :class:`~ssl.SSLContext` were added. (Contributed by Christian Heimes in :issue:`28085`.) +Added :attr:`ssl.SSLContext.post_handshake_auth` to enable and +:meth:`ssl.SSLSocket.verify_client_post_handshake` to initiate TLS 1.3 +post-handshake authentication. +(Contributed by Christian Heimes in :gh:`78851`.) statistics ---------- @@ -2063,6 +2067,15 @@ connected to and thus what Python interpreter will be used by the virtual environment. (Contributed by Brett Cannon in :issue:`25154`.) +xml +--- + +* As mitigation against DTD and external entity retrieval, the + :mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process + external entities by default. + (Contributed by Christian Heimes in :gh:`61441`.) + + Deprecated functions and types of the C API ------------------------------------------- @@ -2323,10 +2336,10 @@ Changes in the Python API * With the introduction of :exc:`ModuleNotFoundError`, import system consumers may start expecting import system replacements to raise that more specific exception when appropriate, rather than the less-specific :exc:`ImportError`. - To provide future compatibility with such consumers, implementors of + To provide future compatibility with such consumers, implementers of alternative import systems that completely replace :func:`__import__` will need to update their implementations to raise the new subclass when a module - can't be found at all. Implementors of compliant plugins to the default + can't be found at all. Implementers of compliant plugins to the default import system shouldn't need to make any changes, as the default import system will raise the new subclass when appropriate. @@ -2430,9 +2443,13 @@ The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some cases. (Contributed by Victor Stinner in :issue:`31900`.) + Notable changes in Python 3.6.7 =============================== +:mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process +external entities by default. See also :gh:`61441`. + In 3.6.7 the :mod:`tokenize` module now implicitly emits a ``NEWLINE`` token when provided with input that does not have a trailing new line. This behavior now matches what the C tokenizer does internally. @@ -2460,3 +2477,19 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.6.14 +================================ + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates RFC 3986, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser :func:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 402b15a277e53d..2d433ef4759d52 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -339,7 +339,7 @@ PEP 564: New Time Functions With Nanosecond Resolution ------------------------------------------------------ The resolution of clocks in modern systems can exceed the limited precision -of a floating point number returned by the :func:`time.time` function +of a floating-point number returned by the :func:`time.time` function and its variants. To avoid loss of precision, :pep:`564` adds six new "nanosecond" variants of the existing timer functions to the :mod:`time` module: @@ -353,7 +353,7 @@ module: The new functions return the number of nanoseconds as an integer value. -`Measurements `_ +:pep:`Measurements <0564#annex-clocks-resolution-in-python>` show that on Linux and Windows the resolution of :func:`time.time_ns` is approximately 3 times better than that of :func:`time.time`. @@ -603,7 +603,7 @@ The new :mod:`importlib.resources` module provides several new APIs and one new ABC for access to, opening, and reading *resources* inside packages. Resources are roughly similar to files inside packages, but they needn't be actual files on the physical file system. Module loaders can provide a -:meth:`get_resource_reader()` function which returns +:meth:`get_resource_reader` function which returns a :class:`importlib.abc.ResourceReader` instance to support this new API. Built-in file path loaders and zip file loaders both support this. @@ -669,8 +669,8 @@ include: * The new :func:`asyncio.current_task` function returns the currently running :class:`~asyncio.Task` instance, and the new :func:`asyncio.all_tasks` function returns a set of all existing ``Task`` instances in a given loop. - The :meth:`Task.current_task() ` and - :meth:`Task.all_tasks() ` methods have been deprecated. + The :meth:`!Task.current_task` and + :meth:`!Task.all_tasks` methods have been deprecated. (Contributed by Andrew Svetlov in :issue:`32250`.) * The new *provisional* :class:`~asyncio.BufferedProtocol` class allows @@ -1133,7 +1133,7 @@ The MIME type of .bmp has been changed from ``'image/x-ms-bmp'`` to msilib ------ -The new :meth:`!Database.Close()` method can be used +The new :meth:`!Database.Close` method can be used to close the :abbr:`MSI` database. (Contributed by Berker Peksag in :issue:`20486`.) @@ -1380,6 +1380,10 @@ Supported protocols are indicated by several new flags, such as :data:`~ssl.HAS_TLSv1_1`. (Contributed by Christian Heimes in :issue:`32609`.) +Added :attr:`ssl.SSLContext.post_handshake_auth` to enable and +:meth:`ssl.SSLSocket.verify_client_post_handshake` to initiate TLS 1.3 +post-handshake authentication. +(Contributed by Christian Heimes in :gh:`78851`.) string ------ @@ -1599,6 +1603,15 @@ at the interactive prompt. See :ref:`whatsnew37-pep565` for details. (Contributed by Nick Coghlan in :issue:`31975`.) +xml +--- + +As mitigation against DTD and external entity retrieval, the +:mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process +external entities by default. +(Contributed by Christian Heimes in :gh:`61441`.) + + xml.etree --------- @@ -1956,7 +1969,7 @@ asynchronous context manager must be used in order to acquire and release the synchronization resource. (Contributed by Andrew Svetlov in :issue:`32253`.) -The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` +The :meth:`!asyncio.Task.current_task` and :meth:`!asyncio.Task.all_tasks` methods have been deprecated. (Contributed by Andrew Svetlov in :issue:`32250`.) @@ -2004,11 +2017,11 @@ importlib --------- Methods -:meth:`!MetaPathFinder.find_module()` +:meth:`!MetaPathFinder.find_module` (replaced by :meth:`MetaPathFinder.find_spec() `) and -:meth:`!PathEntryFinder.find_loader()` +:meth:`!PathEntryFinder.find_loader` (replaced by :meth:`PathEntryFinder.find_spec() `) both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. @@ -2035,7 +2048,7 @@ The :mod:`macpath` is now deprecated and will be removed in Python 3.8. threading --------- -:mod:`dummy_threading` and :mod:`_dummy_thread` have been deprecated. It is +:mod:`!dummy_threading` and :mod:`!_dummy_thread` have been deprecated. It is no longer possible to build Python with threading disabled. Use :mod:`threading` instead. (Contributed by Antoine Pitrou in :issue:`31370`.) @@ -2171,7 +2184,7 @@ The following features and APIs have been removed from Python 3.7: ``socket.socketpair`` on Python 3.5 and newer. * :mod:`asyncio` no longer exports the :mod:`selectors` and - :mod:`_overlapped` modules as ``asyncio.selectors`` and + :mod:`!_overlapped` modules as ``asyncio.selectors`` and ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with ``import selectors``. @@ -2353,7 +2366,7 @@ Changes in the Python API positions 2--3. To match only blank lines, the pattern should be rewritten as ``r'(?m)^[^\S\n]*$'``. - :func:`re.sub()` now replaces empty matches adjacent to a previous + :func:`re.sub` now replaces empty matches adjacent to a previous non-empty match. For example ``re.sub('x*', '-', 'abxd')`` returns now ``'-a-b--d-'`` instead of ``'-a-b-d-'`` (the first minus between 'b' and 'd' replaces 'x', and the second minus replaces an empty string between @@ -2412,7 +2425,7 @@ Changes in the Python API to :meth:`ArgumentParser.add_subparsers() `. (Contributed by Anthony Sottile in :issue:`26510`.) -* :meth:`ast.literal_eval()` is now stricter. Addition and subtraction of +* :meth:`ast.literal_eval` is now stricter. Addition and subtraction of arbitrary numbers are no longer allowed. (Contributed by Serhiy Storchaka in :issue:`31778`.) @@ -2571,3 +2584,33 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.7.11 +================================ + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates RFC 3986, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser :func:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) + +Notable security feature in 3.7.14 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation ` documentation. The default limit +is 4300 digits in string form. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index b041e592d61ed1..fc9f49e65af847 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -428,9 +428,9 @@ Other Language Changes normal assignment syntax:: >>> def parse(family): - lastname, *members = family.split() - return lastname.upper(), *members - + ... lastname, *members = family.split() + ... return lastname.upper(), *members + ... >>> parse('simpsons homer marge bart lisa maggie') ('SIMPSONS', 'homer', 'marge', 'bart', 'lisa', 'maggie') @@ -936,7 +936,7 @@ Add option ``--json-lines`` to parse every input line as a separate JSON object. logging ------- -Added a *force* keyword argument to :func:`logging.basicConfig()` +Added a *force* keyword argument to :func:`logging.basicConfig` When set to true, any existing handlers attached to the root logger are removed and closed before carrying out the configuration specified by the other arguments. @@ -1077,16 +1077,16 @@ pathlib ------- :mod:`pathlib.Path` methods that return a boolean result like -:meth:`~pathlib.Path.exists()`, :meth:`~pathlib.Path.is_dir()`, -:meth:`~pathlib.Path.is_file()`, :meth:`~pathlib.Path.is_mount()`, -:meth:`~pathlib.Path.is_symlink()`, :meth:`~pathlib.Path.is_block_device()`, -:meth:`~pathlib.Path.is_char_device()`, :meth:`~pathlib.Path.is_fifo()`, -:meth:`~pathlib.Path.is_socket()` now return ``False`` instead of raising +:meth:`~pathlib.Path.exists`, :meth:`~pathlib.Path.is_dir`, +:meth:`~pathlib.Path.is_file`, :meth:`~pathlib.Path.is_mount`, +:meth:`~pathlib.Path.is_symlink`, :meth:`~pathlib.Path.is_block_device`, +:meth:`~pathlib.Path.is_char_device`, :meth:`~pathlib.Path.is_fifo`, +:meth:`~pathlib.Path.is_socket` now return ``False`` instead of raising :exc:`ValueError` or its subclass :exc:`UnicodeEncodeError` for paths that contain characters unrepresentable at the OS level. (Contributed by Serhiy Storchaka in :issue:`33721`.) -Added :meth:`!pathlib.Path.link_to()` which creates a hard link pointing +Added :meth:`!pathlib.Path.link_to` which creates a hard link pointing to a path. (Contributed by Joannah Nanjekye in :issue:`26978`) Note that ``link_to`` was deprecated in 3.10 and removed in 3.12 in @@ -1170,13 +1170,13 @@ recursively removing their contents first. socket ------ -Added :meth:`~socket.create_server()` and :meth:`~socket.has_dualstack_ipv6()` +Added :meth:`~socket.create_server` and :meth:`~socket.has_dualstack_ipv6` convenience functions to automate the necessary tasks usually involved when creating a server socket, including accepting both IPv4 and IPv6 connections on the same socket. (Contributed by Giampaolo Rodolà in :issue:`17561`.) -The :func:`socket.if_nameindex()`, :func:`socket.if_nametoindex()`, and -:func:`socket.if_indextoname()` functions have been implemented on Windows. +The :func:`socket.if_nameindex`, :func:`socket.if_nametoindex`, and +:func:`socket.if_indextoname` functions have been implemented on Windows. (Contributed by Zackery Spytz in :issue:`37007`.) @@ -1192,11 +1192,11 @@ post-handshake authentication. statistics ---------- -Added :func:`statistics.fmean` as a faster, floating point variant of -:func:`statistics.mean()`. (Contributed by Raymond Hettinger and +Added :func:`statistics.fmean` as a faster, floating-point variant of +:func:`statistics.mean`. (Contributed by Raymond Hettinger and Steven D'Aprano in :issue:`35904`.) -Added :func:`statistics.geometric_mean()` +Added :func:`statistics.geometric_mean` (Contributed by Raymond Hettinger in :issue:`27181`.) Added :func:`statistics.multimode` that returns a list of the most @@ -1367,10 +1367,10 @@ Added :class:`~unittest.mock.AsyncMock` to support an asynchronous version of have been added as well. (Contributed by Lisa Roach in :issue:`26467`). -Added :func:`~unittest.addModuleCleanup()` and -:meth:`~unittest.TestCase.addClassCleanup()` to unittest to support -cleanups for :func:`~unittest.setUpModule()` and -:meth:`~unittest.TestCase.setUpClass()`. +Added :func:`~unittest.addModuleCleanup` and +:meth:`~unittest.TestCase.addClassCleanup` to unittest to support +cleanups for :func:`~unittest.setUpModule` and +:meth:`~unittest.TestCase.setUpClass`. (Contributed by Lisa Roach in :issue:`24412`.) Several mock assert functions now also print a list of actual calls upon @@ -1432,7 +1432,7 @@ and ``{namespace}*`` which returns all tags in the given namespace. (Contributed by Stefan Behnel in :issue:`28238`.) The :mod:`xml.etree.ElementTree` module provides a new function -:func:`–xml.etree.ElementTree.canonicalize()` that implements C14N 2.0. +:func:`–xml.etree.ElementTree.canonicalize` that implements C14N 2.0. (Contributed by Stefan Behnel in :issue:`13611`.) The target object of :class:`xml.etree.ElementTree.XMLParser` can @@ -1712,7 +1712,7 @@ Deprecated the ``l*gettext()`` functions. (Contributed by Serhiy Storchaka in :issue:`33710`.) -* The :meth:`~threading.Thread.isAlive()` method of :class:`threading.Thread` +* The :meth:`~threading.Thread.isAlive` method of :class:`threading.Thread` has been deprecated. (Contributed by Donghee Na in :issue:`35283`.) @@ -1755,7 +1755,7 @@ The following features and APIs have been removed from Python 3.8: * Starting with Python 3.3, importing ABCs from :mod:`collections` was deprecated, and importing should be done from :mod:`collections.abc`. Being able to import from collections was marked for removal in 3.8, but has been - delayed to 3.9. (See :issue:`36952`.) + delayed to 3.9. (See :gh:`81134`.) * The :mod:`macpath` module, deprecated in Python 3.7, has been removed. (Contributed by Victor Stinner in :issue:`35471`.) @@ -2243,6 +2243,21 @@ details, see the documentation for ``loop.create_datagram_endpoint()``. (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in :issue:`37228`.) +Notable changes in Python 3.8.2 +=============================== + +Fixed a regression with the ``ignore`` callback of :func:`shutil.copytree`. +The argument types are now str and List[str] again. +(Contributed by Manuel Barkhau and Giampaolo Rodola in :gh:`83571`.) + +Notable changes in Python 3.8.3 +=============================== + +The constant values of future flags in the :mod:`__future__` module +are updated in order to prevent collision with compiler flags. Previously +``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``. +(Contributed by Batuhan Taskaya in :gh:`83743`) + Notable changes in Python 3.8.8 =============================== @@ -2256,9 +2271,55 @@ functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) +Notable changes in Python 3.8.9 +=============================== + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + +Notable changes in Python 3.8.10 +================================ + +macOS 11.0 (Big Sur) and Apple Silicon Mac support +-------------------------------------------------- + +As of 3.8.10, Python now supports building and running on macOS 11 +(Big Sur) and on Apple Silicon Macs (based on the ``ARM64`` architecture). +A new universal build variant, ``universal2``, is now available to natively +support both ``ARM64`` and ``Intel 64`` in one set of executables. +Note that support for "weaklinking", building binaries targeted for newer +versions of macOS that will also run correctly on older versions by +testing at runtime for missing features, is not included in this backport +from Python 3.9; to support a range of macOS versions, continue to target +for and build on the oldest version in the range. + +(Originally contributed by Ronald Oussoren and Lawrence D'Anna in :gh:`85272`, +with fixes by FX Coudert and Eli Rykoff, and backported to 3.8 by Maxime Bélanger +and Ned Deily) + +Notable changes in Python 3.8.10 +================================ + +urllib.parse +------------ + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :issue:`43882`) + + Notable changes in Python 3.8.12 ================================ +Changes in the Python API +------------------------- + Starting with Python 3.8.12 the :mod:`ipaddress` module no longer accepts any leading zeros in IPv4 address strings. Leading zeros are ambiguous and interpreted as octal notation by some libraries. For example the legacy @@ -2268,3 +2329,32 @@ any leading zeros. (Originally contributed by Christian Heimes in :issue:`36384`, and backported to 3.8 by Achraf Merzouki.) + +Notable security feature in 3.8.14 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation ` documentation. The default limit +is 4300 digits in string form. + +Notable changes in 3.8.17 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index f7ad4372325ccb..6118b02dd9bd48 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -81,13 +81,13 @@ Interpreter improvements: * a number of Python builtins (range, tuple, set, frozenset, list, dict) are now sped up using :pep:`590` vectorcall; * garbage collection does not block on resurrected objects; -* a number of Python modules (:mod:`_abc`, :mod:`!audioop`, :mod:`_bz2`, - :mod:`_codecs`, :mod:`_contextvars`, :mod:`!_crypt`, :mod:`_functools`, - :mod:`_json`, :mod:`_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, - :mod:`time`, :mod:`_weakref`) now use multiphase initialization as defined +* a number of Python modules (:mod:`!_abc`, :mod:`!audioop`, :mod:`!_bz2`, + :mod:`!_codecs`, :mod:`!_contextvars`, :mod:`!_crypt`, :mod:`!_functools`, + :mod:`!_json`, :mod:`!_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, + :mod:`time`, :mod:`!_weakref`) now use multiphase initialization as defined by PEP 489; * a number of standard library modules (:mod:`!audioop`, :mod:`ast`, :mod:`grp`, - :mod:`_hashlib`, :mod:`pwd`, :mod:`_posixsubprocess`, :mod:`random`, + :mod:`!_hashlib`, :mod:`pwd`, :mod:`!_posixsubprocess`, :mod:`random`, :mod:`select`, :mod:`struct`, :mod:`termios`, :mod:`zlib`) are now using the stable ABI defined by PEP 384. @@ -203,7 +203,7 @@ The :mod:`ast` module uses the new parser and produces the same AST as the old parser. In Python 3.10, the old parser will be deleted and so will all -functionality that depends on it (primarily the :mod:`parser` module, +functionality that depends on it (primarily the :mod:`!parser` module, which has long been deprecated). In Python 3.9 *only*, you can switch back to the LL(1) parser using a command line switch (``-X oldparser``) or an environment variable (``PYTHONOLDPARSER=1``). @@ -300,12 +300,9 @@ Example:: As a fall-back source of data for platforms that don't ship the IANA database, -the |tzdata|_ module was released as a first-party package -- distributed via +the :pypi:`tzdata` module was released as a first-party package -- distributed via PyPI and maintained by the CPython core team. -.. |tzdata| replace:: ``tzdata`` -.. _tzdata: https://pypi.org/project/tzdata/ - .. seealso:: :pep:`615` -- Support for the IANA Time Zone Database in the Standard Library @@ -355,7 +352,7 @@ that schedules a shutdown for the default executor that waits on the :func:`asyncio.run` has been updated to use the new :term:`coroutine`. (Contributed by Kyle Stanley in :issue:`34037`.) -Added :class:`asyncio.PidfdChildWatcher`, a Linux-specific child watcher +Added :class:`!asyncio.PidfdChildWatcher`, a Linux-specific child watcher implementation that polls process file descriptors. (:issue:`38692`) Added a new :term:`coroutine` :func:`asyncio.to_thread`. It is mainly used for @@ -369,7 +366,7 @@ wait until the cancellation is complete also in the case when *timeout* is <= 0, like it does with positive timeouts. (Contributed by Elvis Pranskevichus in :issue:`32751`.) -:mod:`asyncio` now raises :exc:`TyperError` when calling incompatible +:mod:`asyncio` now raises :exc:`TypeError` when calling incompatible methods with an :class:`ssl.SSLSocket` socket. (Contributed by Ido Michael in :issue:`37404`.) @@ -411,8 +408,8 @@ Added :func:`curses.get_escdelay`, :func:`curses.set_escdelay`, datetime -------- -The :meth:`~datetime.date.isocalendar()` of :class:`datetime.date` -and :meth:`~datetime.datetime.isocalendar()` of :class:`datetime.datetime` +The :meth:`~datetime.date.isocalendar` of :class:`datetime.date` +and :meth:`~datetime.datetime.isocalendar` of :class:`datetime.datetime` methods now returns a :func:`~collections.namedtuple` instead of a :class:`tuple`. (Contributed by Donghee Na in :issue:`24416`.) @@ -592,7 +589,7 @@ a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) os -- -Added :const:`~os.CLD_KILLED` and :const:`~os.CLD_STOPPED` for :attr:`si_code`. +Added :const:`~os.CLD_KILLED` and :const:`~os.CLD_STOPPED` for :attr:`!si_code`. (Contributed by Donghee Na in :issue:`38493`.) Exposed the Linux-specific :func:`os.pidfd_open` (:issue:`38692`) and @@ -613,7 +610,7 @@ convert a wait status to an exit code. pathlib ------- -Added :meth:`pathlib.Path.readlink()` which acts similarly to +Added :meth:`pathlib.Path.readlink` which acts similarly to :func:`os.readlink`. (Contributed by Girts Folkmanis in :issue:`30618`) @@ -640,7 +637,8 @@ pydoc ----- The documentation string is now shown not only for class, function, -method etc, but for any object that has its own ``__doc__`` attribute. +method etc, but for any object that has its own :attr:`~definition.__doc__` +attribute. (Contributed by Serhiy Storchaka in :issue:`40257`.) random @@ -864,7 +862,7 @@ Deprecated Python versions it will raise a :exc:`TypeError` for all floats. (Contributed by Serhiy Storchaka in :issue:`37315`.) -* The :mod:`parser` and :mod:`symbol` modules are deprecated and will be +* The :mod:`!parser` and :mod:`!symbol` modules are deprecated and will be removed in future versions of Python. For the majority of use cases, users can leverage the Abstract Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` module. @@ -892,20 +890,20 @@ Deprecated it for writing and silencing a warning. (Contributed by Serhiy Storchaka in :issue:`28286`.) -* Deprecated the ``split()`` method of :class:`_tkinter.TkappType` in +* Deprecated the ``split()`` method of :class:`!_tkinter.TkappType` in favour of the ``splitlist()`` method which has more consistent and - predicable behavior. + predictable behavior. (Contributed by Serhiy Storchaka in :issue:`38371`.) * The explicit passing of coroutine objects to :func:`asyncio.wait` has been deprecated and will be removed in version 3.11. (Contributed by Yury Selivanov and Kyle Stanley in :issue:`34790`.) -* binhex4 and hexbin4 standards are now deprecated. The :mod:`binhex` module +* binhex4 and hexbin4 standards are now deprecated. The :mod:`!binhex` module and the following :mod:`binascii` functions are now deprecated: - * :func:`~binascii.b2a_hqx`, :func:`~binascii.a2b_hqx` - * :func:`~binascii.rlecode_hqx`, :func:`~binascii.rledecode_hqx` + * :func:`!b2a_hqx`, :func:`!a2b_hqx` + * :func:`!rlecode_hqx`, :func:`!rledecode_hqx` (Contributed by Victor Stinner in :issue:`39353`.) @@ -953,7 +951,7 @@ Deprecated Removed ======= -* The erroneous version at :data:`unittest.mock.__version__` has been removed. +* The erroneous version at :data:`!unittest.mock.__version__` has been removed. * :class:`!nntplib.NNTP`: ``xpath()`` and ``xgtitle()`` methods have been removed. These methods are deprecated since Python 3.3. Generally, these extensions @@ -986,13 +984,13 @@ Removed (Contributed by Victor Stinner in :issue:`37312`.) * ``aifc.openfp()`` alias to ``aifc.open()``, ``sunau.openfp()`` alias to - ``sunau.open()``, and ``wave.openfp()`` alias to :func:`wave.open()` have been + ``sunau.open()``, and ``wave.openfp()`` alias to :func:`wave.open` have been removed. They were deprecated since Python 3.7. (Contributed by Victor Stinner in :issue:`37320`.) -* The :meth:`~threading.Thread.isAlive()` method of :class:`threading.Thread` +* The :meth:`!isAlive` method of :class:`threading.Thread` has been removed. It was deprecated since Python 3.8. - Use :meth:`~threading.Thread.is_alive()` instead. + Use :meth:`~threading.Thread.is_alive` instead. (Contributed by Donghee Na in :issue:`37804`.) * Methods ``getchildren()`` and ``getiterator()`` of classes @@ -1038,7 +1036,7 @@ Removed ``asyncio.Condition`` and ``asyncio.Semaphore``. (Contributed by Andrew Svetlov in :issue:`34793`.) -* The :func:`sys.getcounts` function, the ``-X showalloccount`` command line +* The :func:`!sys.getcounts` function, the ``-X showalloccount`` command line option and the ``show_alloc_count`` field of the C structure :c:type:`PyConfig` have been removed. They required a special Python build by defining ``COUNT_ALLOCS`` macro. @@ -1049,11 +1047,11 @@ Removed the ``__annotations__`` attribute instead. (Contributed by Serhiy Storchaka in :issue:`40182`.) -* The :meth:`symtable.SymbolTable.has_exec` method has been removed. It was +* The :meth:`!symtable.SymbolTable.has_exec` method has been removed. It was deprecated since 2006, and only returning ``False`` when it's called. (Contributed by Batuhan Taskaya in :issue:`40208`) -* The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` +* The :meth:`!asyncio.Task.current_task` and :meth:`!asyncio.Task.all_tasks` have been removed. They were deprecated since Python 3.7 and you can use :func:`asyncio.current_task` and :func:`asyncio.all_tasks` instead. (Contributed by Rémi Lapeyre in :issue:`40967`) @@ -1126,7 +1124,7 @@ Changes in the Python API ``logging.getLogger(__name__)`` in some top-level module called ``'root.py'``. (Contributed by Vinay Sajip in :issue:`37742`.) -* Division handling of :class:`~pathlib.PurePath` now returns ``NotImplemented`` +* Division handling of :class:`~pathlib.PurePath` now returns :data:`NotImplemented` instead of raising a :exc:`TypeError` when passed something other than an instance of ``str`` or :class:`~pathlib.PurePath`. This allows creating compatible classes that don't inherit from those mentioned types. @@ -1233,7 +1231,7 @@ Build Changes * The ``COUNT_ALLOCS`` special build macro has been removed. (Contributed by Victor Stinner in :issue:`39489`.) -* On non-Windows platforms, the :c:func:`setenv` and :c:func:`unsetenv` +* On non-Windows platforms, the :c:func:`!setenv` and :c:func:`!unsetenv` functions are now required to build Python. (Contributed by Victor Stinner in :issue:`39395`.) @@ -1322,7 +1320,7 @@ New Features the garbage collector respectively. (Contributed by Pablo Galindo Salgado in :issue:`40241`.) -* Added :c:func:`_PyObject_FunctionStr` to get a user-friendly string +* Added :c:func:`!_PyObject_FunctionStr` to get a user-friendly string representation of a function-like object. (Patch by Jeroen Demeyer in :issue:`37645`.) @@ -1364,7 +1362,7 @@ Porting to Python 3.9 and refers to a constant string. (Contributed by Serhiy Storchaka in :issue:`38650`.) -* The :c:type:`PyGC_Head` structure is now opaque. It is only defined in the +* The :c:type:`!PyGC_Head` structure is now opaque. It is only defined in the internal C API (``pycore_gc.h``). (Contributed by Victor Stinner in :issue:`40241`.) @@ -1387,12 +1385,12 @@ Porting to Python 3.9 * :c:func:`PyObject_IS_GC` macro was converted to a function. - * The :c:func:`PyObject_NEW` macro becomes an alias to the - :c:macro:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro + * The :c:func:`!PyObject_NEW` macro becomes an alias to the + :c:macro:`PyObject_New` macro, and the :c:func:`!PyObject_NEW_VAR` macro becomes an alias to the :c:macro:`PyObject_NewVar` macro. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. - * :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: + * :c:func:`!PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: the macro accessed directly the :c:member:`PyTypeObject.tp_weaklistoffset` member. @@ -1562,3 +1560,54 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.9.3 +=============================== + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + +Notable changes in Python 3.9.5 +=============================== + +urllib.parse +------------ + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) + +Notable security feature in 3.9.14 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation ` documentation. The default limit +is 4300 digits in string form. + +Notable changes in 3.9.17 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index b9c19602653219..6ff722a1894585 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.14.rst 3.13.rst 3.12.rst 3.11.rst @@ -34,8 +35,8 @@ anyone wishing to stay up-to-date after a new release. 2.1.rst 2.0.rst -The "Changelog" is an HTML version of the `file built -`_ from the contents of the +The "Changelog" is an HTML version of the :pypi:`file built` +from the contents of the :source:`Misc/NEWS.d` directory tree, which contains *all* nontrivial changes to Python for the current version. diff --git a/Grammar/python.gram b/Grammar/python.gram index 174b4dbb6f7842..b47028460b94f4 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -78,6 +78,9 @@ _PyPegen_parse(Parser *p) # Fail if e can be parsed, without consuming any input. # ~ # Commit to the current alternative, even if it fails to parse. +# &&e +# Eager parse e. The parser will not backtrack and will immediately +# fail with SyntaxError if e cannot be parsed. # # STARTING RULES @@ -124,7 +127,6 @@ simple_stmt[stmt_ty] (memo): | &'nonlocal' nonlocal_stmt compound_stmt[stmt_ty]: - | invalid_compound_stmt | &('def' | '@' | 'async') function_def | &'if' if_stmt | &('class' | '@') class_def @@ -149,9 +151,9 @@ assignment[stmt_ty]: | a=('(' b=single_target ')' { b } | single_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] { CHECK_VERSION(stmt_ty, 6, "Variable annotations syntax is", _PyAST_AnnAssign(a, b, c, 0, EXTRA)) } - | a[asdl_expr_seq*]=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) !'=' tc=[TYPE_COMMENT] { + | a[asdl_expr_seq*]=(z=star_targets '=' { z })+ b=annotated_rhs !'=' tc=[TYPE_COMMENT] { _PyAST_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | a=single_target b=augassign ~ c=(yield_expr | star_expressions) { + | a=single_target b=augassign ~ c=annotated_rhs { _PyAST_AugAssign(a, b->kind, c, EXTRA) } | invalid_assignment @@ -266,11 +268,11 @@ function_def[stmt_ty]: function_def_raw[stmt_ty]: | invalid_def_raw - | 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { _PyAST_FunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), t, EXTRA) } - | 'async' 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'async' 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { CHECK_VERSION( stmt_ty, 5, @@ -393,7 +395,7 @@ for_stmt[stmt_ty]: with_stmt[stmt_ty]: | invalid_with_stmt_indent | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' tc=[TYPE_COMMENT] b=block { - CHECK_VERSION(stmt_ty, 9, "Parenthesized context managers are", _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } + _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'async' 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { @@ -432,7 +434,7 @@ except_star_block[excepthandler_ty]: | invalid_except_star_stmt_indent | 'except' '*' e=expression t=['as' z=NAME { z }] ':' b=block { _PyAST_ExceptHandler(e, (t) ? ((expr_ty) t)->v.Name.id : NULL, b, EXTRA) } - | invalid_except_stmt + | invalid_except_star_stmt finally_block[asdl_stmt_seq*]: | invalid_finally_stmt | 'finally' &&':' a=block { a } @@ -638,27 +640,24 @@ type_alias[stmt_ty]: # Type parameter declaration # -------------------------- -type_params[asdl_type_param_seq*]: '[' t=type_param_seq ']' { +type_params[asdl_type_param_seq*]: + | invalid_type_params + | '[' t=type_param_seq ']' { CHECK_VERSION(asdl_type_param_seq *, 12, "Type parameter lists are", t) } type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [','] { a } type_param[type_param_ty] (memo): - | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) } - | '*' a=NAME colon=':' e=expression { - RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind - ? "cannot use constraints with TypeVarTuple" - : "cannot use bound with TypeVarTuple") - } - | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) } - | '**' a=NAME colon=':' e=expression { - RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind - ? "cannot use constraints with ParamSpec" - : "cannot use bound with ParamSpec") - } - | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) } + | a=NAME b=[type_param_bound] c=[type_param_default] { _PyAST_TypeVar(a->v.Name.id, b, c, EXTRA) } + | invalid_type_param + | '*' a=NAME b=[type_param_starred_default] { _PyAST_TypeVarTuple(a->v.Name.id, b, EXTRA) } + | '**' a=NAME b=[type_param_default] { _PyAST_ParamSpec(a->v.Name.id, b, EXTRA) } type_param_bound[expr_ty]: ':' e=expression { e } +type_param_default[expr_ty]: '=' e=expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } +type_param_starred_default[expr_ty]: '=' e=star_expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } # EXPRESSIONS # ----------- @@ -778,6 +777,7 @@ bitwise_and[expr_ty]: shift_expr[expr_ty]: | a=shift_expr '<<' b=sum { _PyAST_BinOp(a, LShift, b, EXTRA) } | a=shift_expr '>>' b=sum { _PyAST_BinOp(a, RShift, b, EXTRA) } + | invalid_arithmetic | sum # Arithmetic operators @@ -794,6 +794,7 @@ term[expr_ty]: | a=term '//' b=factor { _PyAST_BinOp(a, FloorDiv, b, EXTRA) } | a=term '%' b=factor { _PyAST_BinOp(a, Mod, b, EXTRA) } | a=term '@' b=factor { CHECK_VERSION(expr_ty, 5, "The '@' operator is", _PyAST_BinOp(a, MatMult, b, EXTRA)) } + | invalid_factor | factor factor[expr_ty] (memo): @@ -913,7 +914,7 @@ fstring_middle[expr_ty]: | fstring_replacement_field | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) } fstring_replacement_field[expr_ty]: - | '{' a=(yield_expr | star_expressions) debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' { + | '{' a=annotated_rhs debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' { _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) } | invalid_replacement_field fstring_conversion[ResultTokenWithMetadata*]: @@ -968,8 +969,7 @@ for_if_clause[comprehension_ty]: CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) } | 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { _PyAST_comprehension(a, b, c, 0, p->arena) } - | 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' { - RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") } + | invalid_for_if_clause | invalid_for_target listcomp[expr_ty]: @@ -1009,8 +1009,9 @@ kwargs[asdl_seq*]: | ','.kwarg_or_double_starred+ starred_expression[expr_ty]: - | invalid_starred_expression + | invalid_starred_expression_unpacking | '*' a=expression { _PyAST_Starred(a, Load, EXTRA) } + | invalid_starred_expression kwarg_or_starred[KeywordOrStarred*]: | invalid_kwarg @@ -1131,8 +1132,8 @@ func_type_comment[Token*]: # From here on, there are rules for invalid syntax with specialised error messages invalid_arguments: - | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' b='*' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "iterable argument unpacking follows keyword argument unpacking") } + | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) a=',' ','.(starred_expression !'=')+ { + RAISE_SYNTAX_ERROR_STARTING_FROM(a, "iterable argument unpacking follows keyword argument unpacking") } | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") } | a=NAME b='=' expression for_if_clauses { @@ -1164,6 +1165,18 @@ invalid_legacy_expression: _PyPegen_check_legacy_stmt(p, a) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Missing parentheses in call to '%U'. Did you mean %U(...)?", a->v.Name.id, a->v.Name.id) : NULL} +invalid_type_param: + | '*' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with TypeVarTuple" + : "cannot use bound with TypeVarTuple") + } + | '**' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with ParamSpec" + : "cannot use bound with ParamSpec") + } + invalid_expression: # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" # Soft keywords need to also be ignored because they can be parsed as NAME NAME @@ -1198,7 +1211,7 @@ invalid_assignment: | (star_targets '=')* a=star_expressions '=' { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } | (star_targets '=')* a=yield_expr '=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "assignment to yield expression not possible") } - | a=star_expressions augassign (yield_expr | star_expressions) { + | a=star_expressions augassign annotated_rhs { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "'%s' is an illegal expression for augmented assignment", @@ -1284,6 +1297,10 @@ invalid_with_item: | expression 'as' a=expression &(',' | ')' | ':') { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } +invalid_for_if_clause: + | 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' { + RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") } + invalid_for_target: | 'async'? 'for' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) } @@ -1296,14 +1313,14 @@ invalid_group: invalid_import: | a='import' ','.dotted_name+ 'from' dotted_name { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "Did you mean to use 'from ... import ...' instead?") } + | 'import' token=NEWLINE { + RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_import_from_targets: | import_from_as_names ',' NEWLINE { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } - -invalid_compound_stmt: - | a='elif' named_expression ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "'elif' must match an if-statement here") } - | a='else' ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "'else' must match a valid statement here") } + | token=NEWLINE { + RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_with_stmt: | ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } @@ -1323,11 +1340,21 @@ invalid_try_stmt: | 'try' ':' block* except_star_block+ a='except' [expression ['as' NAME]] ':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot have both 'except' and 'except*' on the same 'try'") } invalid_except_stmt: - | 'except' '*'? a=expression ',' expressions ['as' NAME ] ':' { + | 'except' a=expression ',' expressions ['as' NAME ] ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized") } - | a='except' '*'? expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | a='except' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | 'except' expression 'as' a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, "cannot use except statement with %s", _PyPegen_get_expr_name(a)) } +invalid_except_star_stmt: + | 'except' '*' a=expression ',' expressions ['as' NAME ] ':' { + RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized") } + | a='except' '*' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' '*' (NEWLINE | ':') { RAISE_SYNTAX_ERROR("expected one or more exception types") } + | 'except' '*' expression 'as' a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, "cannot use except* statement with %s", _PyPegen_get_expr_name(a)) } invalid_finally_stmt: | a='finally' ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'finally' statement on line %d", a->lineno) } @@ -1348,7 +1375,9 @@ invalid_case_block: RAISE_INDENTATION_ERROR("expected an indented block after 'case' statement on line %d", a->lineno) } invalid_as_pattern: | or_pattern 'as' a="_" { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use '_' as a target") } - | or_pattern 'as' !NAME a=expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "invalid pattern target") } + | or_pattern 'as' a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, "cannot use %s as pattern target", _PyPegen_get_expr_name(a)) } invalid_class_pattern: | name_or_attr '(' a=invalid_class_argument_pattern { RAISE_SYNTAX_ERROR_KNOWN_RANGE( PyPegen_first_item(a, pattern_ty), @@ -1378,6 +1407,7 @@ invalid_for_stmt: invalid_def_raw: | ['async'] a='def' NAME [type_params] '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) } + | ['async'] 'def' NAME [type_params] &&'(' [params] ')' ['->' expression] &&':' [func_type_comment] block invalid_class_def_raw: | 'class' NAME [type_params] ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='class' NAME [type_params] ['(' [arguments] ')'] ':' NEWLINE !INDENT { @@ -1392,26 +1422,40 @@ invalid_kvpair: RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") } | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } -invalid_starred_expression: +invalid_starred_expression_unpacking: | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") } +invalid_starred_expression: + | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") } + invalid_replacement_field: | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") } | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") } | '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before ':'") } | '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '}'") } - | '{' !(yield_expr | star_expressions) { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'")} - | '{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}') { + | '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'")} + | '{' annotated_rhs !('=' | '!' | ':' | '}') { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '=', or '!', or ':', or '}'") } - | '{' (yield_expr | star_expressions) '=' !('!' | ':' | '}') { + | '{' annotated_rhs '=' !('!' | ':' | '}') { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '!', or ':', or '}'") } - | '{' (yield_expr | star_expressions) '='? invalid_conversion_character - | '{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}') { + | '{' annotated_rhs '='? invalid_conversion_character + | '{' annotated_rhs '='? ['!' NAME] !(':' | '}') { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting ':' or '}'") } - | '{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}' { + | '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}', or format specs") } - | '{' (yield_expr | star_expressions) '='? ['!' NAME] !'}' { + | '{' annotated_rhs '='? ['!' NAME] !'}' { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}'") } invalid_conversion_character: | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: missing conversion character") } | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: invalid conversion character") } + +invalid_arithmetic: + | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } +invalid_factor: + | ('+' | '-' | '~') a='not' b=factor { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } + +invalid_type_params: + | '[' token=']' { + RAISE_SYNTAX_ERROR_STARTING_FROM( + token, + "Type parameter list cannot be empty")} diff --git a/Include/Python.h b/Include/Python.h index 196751c3201e62..717e27feab62db 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -45,6 +45,19 @@ # endif #endif +// gh-111506: The free-threaded build is not compatible with the limited API +// or the stable ABI. +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +# error "The limited API is not currently supported in the free-threaded build" +#endif + +#if defined(Py_GIL_DISABLED) && defined(_MSC_VER) +# include // __readgsqword() +#endif + +#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) +# include // __readgsqword() +#endif // Include Python header files #include "pyport.h" @@ -55,7 +68,9 @@ #include "pybuffer.h" #include "pystats.h" #include "pyatomic.h" +#include "lock.h" #include "object.h" +#include "refcount.h" #include "objimpl.h" #include "typeslots.h" #include "pyhash.h" @@ -63,6 +78,7 @@ #include "bytearrayobject.h" #include "bytesobject.h" #include "unicodeobject.h" +#include "pyerrors.h" #include "longobject.h" #include "cpython/longintrepr.h" #include "boolobject.h" @@ -78,6 +94,7 @@ #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" +#include "monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" @@ -97,8 +114,8 @@ #include "weakrefobject.h" #include "structseq.h" #include "cpython/picklebufobject.h" +#include "cpython/pytime.h" #include "codecs.h" -#include "pyerrors.h" #include "pythread.h" #include "cpython/context.h" #include "modsupport.h" @@ -107,17 +124,18 @@ #include "pylifecycle.h" #include "ceval.h" #include "sysmodule.h" +#include "audit.h" #include "osmodule.h" #include "intrcheck.h" #include "import.h" #include "abstract.h" #include "bltinmodule.h" +#include "critical_section.h" #include "cpython/pyctype.h" #include "pystrtod.h" #include "pystrcmp.h" #include "fileutils.h" #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" -#include "cpython/optimizer.h" #endif /* !Py_PYTHON_H */ diff --git a/Include/abstract.h b/Include/abstract.h index bd12a54963c13f..7cfee1332ccaa4 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -397,13 +397,23 @@ PyAPI_FUNC(int) PyIter_Check(PyObject *); This function always succeeds. */ PyAPI_FUNC(int) PyAIter_Check(PyObject *); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +/* Return 1 and set 'item' to the next item of 'iter' on success. + * Return 0 and set 'item' to NULL when there are no remaining values. + * Return -1, set 'item' to NULL and set an exception on error. + */ +PyAPI_FUNC(int) PyIter_NextItem(PyObject *iter, PyObject **item); +#endif + /* Takes an iterator object and calls its tp_iternext slot, returning the next value. If the iterator is exhausted, this returns NULL without setting an exception. - NULL with an exception means an error occurred. */ + NULL with an exception means an error occurred. + + Prefer PyIter_NextItem() instead. */ PyAPI_FUNC(PyObject *) PyIter_Next(PyObject *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 @@ -852,15 +862,15 @@ PyAPI_FUNC(int) PyMapping_HasKeyWithError(PyObject *o, PyObject *key); PyAPI_FUNC(int) PyMapping_HasKeyStringWithError(PyObject *o, const char *key); -/* On success, return a list or tuple of the keys in mapping object 'o'. +/* On success, return a list of the keys in mapping object 'o'. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Keys(PyObject *o); -/* On success, return a list or tuple of the values in mapping object 'o'. +/* On success, return a list of the values in mapping object 'o'. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Values(PyObject *o); -/* On success, return a list or tuple of the items in mapping object 'o', +/* On success, return a list of the items in mapping object 'o', where each item is a tuple containing a key-value pair. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Items(PyObject *o); diff --git a/Include/audit.h b/Include/audit.h new file mode 100644 index 00000000000000..793b7077e1027b --- /dev/null +++ b/Include/audit.h @@ -0,0 +1,30 @@ +#ifndef Py_AUDIT_H +#define Py_AUDIT_H +#ifdef __cplusplus +extern "C" { +#endif + + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PySys_Audit( + const char *event, + const char *argFormat, + ...); + +PyAPI_FUNC(int) PySys_AuditTuple( + const char *event, + PyObject *args); +#endif + + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_AUDIT_H +# include "cpython/audit.h" +# undef Py_CPYTHON_AUDIT_H +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_AUDIT_H */ diff --git a/Include/boolobject.h b/Include/boolobject.h index 19aef5b1b87c6a..3037e61bbf6d0c 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -18,8 +18,13 @@ PyAPI_DATA(PyLongObject) _Py_FalseStruct; PyAPI_DATA(PyLongObject) _Py_TrueStruct; /* Use these macros */ -#define Py_False _PyObject_CAST(&_Py_FalseStruct) -#define Py_True _PyObject_CAST(&_Py_TrueStruct) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_False Py_GetConstantBorrowed(Py_CONSTANT_FALSE) +# define Py_True Py_GetConstantBorrowed(Py_CONSTANT_TRUE) +#else +# define Py_False _PyObject_CAST(&_Py_FalseStruct) +# define Py_True _PyObject_CAST(&_Py_TrueStruct) +#endif // Test if an object is the True singleton, the same as "x is True" in Python. PyAPI_FUNC(int) Py_IsTrue(PyObject *x); diff --git a/Include/ceval.h b/Include/ceval.h index 9885bdb7febc21..32ab38972e548f 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -22,6 +22,10 @@ PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void); PyAPI_FUNC(PyObject *) PyEval_GetLocals(void); PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameBuiltins(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameGlobals(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameLocals(void); + PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); PyAPI_FUNC(int) Py_MakePendingCalls(void); @@ -38,7 +42,7 @@ PyAPI_FUNC(int) Py_MakePendingCalls(void); level exceeds "current recursion limit + 50". By construction, this protection can only be triggered when the "overflowed" flag is set. It means the cleanup code has itself gone into an infinite loop, or the - RecursionError has been mistakingly ignored. When this protection is + RecursionError has been mistakenly ignored. When this protection is triggered, the interpreter aborts with a Fatal Error. In addition, the "overflowed" flag is automatically reset when the @@ -107,6 +111,8 @@ PyAPI_FUNC(PyObject *) PyEval_EvalFrameEx(PyFrameObject *f, int exc); PyAPI_FUNC(PyThreadState *) PyEval_SaveThread(void); PyAPI_FUNC(void) PyEval_RestoreThread(PyThreadState *); +Py_DEPRECATED(3.9) PyAPI_FUNC(void) PyEval_InitThreads(void); + PyAPI_FUNC(void) PyEval_AcquireThread(PyThreadState *tstate); PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); @@ -127,6 +133,13 @@ PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); #define FVS_MASK 0x4 #define FVS_HAVE_SPEC 0x4 +/* Special methods used by LOAD_SPECIAL */ +#define SPECIAL___ENTER__ 0 +#define SPECIAL___EXIT__ 1 +#define SPECIAL___AENTER__ 2 +#define SPECIAL___AEXIT__ 3 +#define SPECIAL_MAX 3 + #ifndef Py_LIMITED_API # define Py_CPYTHON_CEVAL_H # include "cpython/ceval.h" diff --git a/Include/cpython/audit.h b/Include/cpython/audit.h new file mode 100644 index 00000000000000..3c5c7a8c06091d --- /dev/null +++ b/Include/cpython/audit.h @@ -0,0 +1,8 @@ +#ifndef Py_CPYTHON_AUDIT_H +# error "this header file must not be included directly" +#endif + + +typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); + +PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index 816823716e9a6f..cf3f0387ecf323 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -31,3 +31,8 @@ static inline Py_ssize_t PyBytes_GET_SIZE(PyObject *op) { return Py_SIZE(self); } #define PyBytes_GET_SIZE(self) PyBytes_GET_SIZE(_PyObject_CAST(self)) + +PyAPI_FUNC(PyObject*) PyBytes_Join(PyObject *sep, PyObject *iterable); + +// Alias kept for backward compatibility +#define _PyBytes_Join PyBytes_Join diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 78f7405661662f..ca8109e3248a8d 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -23,3 +23,21 @@ _PyEval_RequestCodeExtraIndex(freefunc f) { PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); + + +// Trampoline API + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; +} PerfMapState; + +PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); +PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( + const void *code_addr, + unsigned int code_size, + const char *entry_name); +PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); +PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 1f47d99fb60443..2561b2b88baacc 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -8,6 +8,8 @@ extern "C" { #endif +/* Total tool ids available */ +#define _PY_MONITORING_TOOL_IDS 8 /* Count of all local monitoring events */ #define _PY_MONITORING_LOCAL_EVENTS 10 /* Count of all "real" monitoring events (not derived from other events) */ @@ -24,47 +26,6 @@ typedef struct _Py_GlobalMonitors { uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_GlobalMonitors; -/* Each instruction in a code object is a fixed-width value, - * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG - * opcode allows for larger values but the current limit is 3 uses - * of EXTENDED_ARG (see Python/compile.c), for a maximum - * 32-bit value. This aligns with the note in Python/compile.c - * (compiler_addop_i_line) indicating that the max oparg value is - * 2**32 - 1, rather than INT_MAX. - */ - -typedef union { - uint16_t cache; - struct { - uint8_t code; - uint8_t arg; - } op; -} _Py_CODEUNIT; - - -/* These macros only remain defined for compatibility. */ -#define _Py_OPCODE(word) ((word).op.code) -#define _Py_OPARG(word) ((word).op.arg) - -static inline _Py_CODEUNIT -_py_make_codeunit(uint8_t opcode, uint8_t oparg) -{ - // No designated initialisers because of C++ compat - _Py_CODEUNIT word; - word.op.code = opcode; - word.op.arg = oparg; - return word; -} - -static inline void -_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) -{ - word->op.code = opcode; -} - -#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg)) -#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode)) - typedef struct { PyObject *_co_code; @@ -73,7 +34,7 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; -/* Ancilliary data structure used for instrumentation. +/* Ancillary data structure used for instrumentation. Line instrumentation creates an array of these. One entry per code unit.*/ typedef struct { @@ -98,6 +59,8 @@ typedef struct { _Py_LocalMonitors active_monitors; /* The tools that are to be notified for events for the matching code unit */ uint8_t *tools; + /* The version of tools when they instrument the code */ + uintptr_t tool_versions[_PY_MONITORING_TOOL_IDS]; /* Information to support line events */ _PyCoLineInstrumentationData *lines; /* The tools that are to be notified for line events for the matching code unit */ @@ -169,6 +132,7 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ uintptr_t _co_instrumentation_version; /* current instrumentation version */ \ _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ + Py_ssize_t _co_unique_id; /* ID used for per-thread refcounting */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ @@ -210,12 +174,17 @@ struct PyCodeObject _PyCode_DEF(1); #define CO_NO_MONITORING_EVENTS 0x2000000 +/* Whether the code object has a docstring, + If so, it will be the first item in co_consts +*/ +#define CO_HAS_DOCSTRING 0x4000000 + /* This should be defined if a future statement modifies the syntax. For example, when a keyword is added. */ #define PY_PARSER_REQUIRES_FUTURE_KEYWORD -#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */ +#define CO_MAXBLOCKS 21 /* Max static block nesting within a function */ PyAPI_DATA(PyTypeObject) PyCode_Type; @@ -226,13 +195,14 @@ static inline Py_ssize_t PyCode_GetNumFree(PyCodeObject *op) { return op->co_nfreevars; } -static inline int PyCode_GetFirstFree(PyCodeObject *op) { +static inline int PyUnstable_Code_GetFirstFree(PyCodeObject *op) { assert(PyCode_Check(op)); return op->co_nlocalsplus - op->co_nfreevars; } -#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) -#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) +Py_DEPRECATED(3.13) static inline int PyCode_GetFirstFree(PyCodeObject *op) { + return PyUnstable_Code_GetFirstFree(op); +} /* Unstable public interface */ PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_New( diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index 0d587505ef7f85..cfdb7080d45f2b 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -32,28 +32,8 @@ typedef struct { #define _PyCompilerFlags_INIT \ (PyCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION} -/* source location information */ -typedef struct { - int lineno; - int end_lineno; - int col_offset; - int end_col_offset; -} _PyCompilerSrcLocation; - -#define SRC_LOCATION_FROM_AST(n) \ - (_PyCompilerSrcLocation){ \ - .lineno = (n)->lineno, \ - .end_lineno = (n)->end_lineno, \ - .col_offset = (n)->col_offset, \ - .end_col_offset = (n)->end_col_offset } - /* Future feature support */ -typedef struct { - int ff_features; /* flags set by future statements */ - _PyCompilerSrcLocation ff_location; /* location of last future statement */ -} PyFutureFeatures; - #define FUTURE_NESTED_SCOPES "nested_scopes" #define FUTURE_GENERATORS "generators" #define FUTURE_DIVISION "division" diff --git a/Include/cpython/context.h b/Include/cpython/context.h index a3249fc29b082e..3a7a4b459c09ad 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -27,6 +27,39 @@ PyAPI_FUNC(PyObject *) PyContext_CopyCurrent(void); PyAPI_FUNC(int) PyContext_Enter(PyObject *); PyAPI_FUNC(int) PyContext_Exit(PyObject *); +typedef enum { + /* + * The current context has switched to a different context. The object + * passed to the watch callback is the now-current contextvars.Context + * object, or None if no context is current. + */ + Py_CONTEXT_SWITCHED = 1, +} PyContextEvent; + +/* + * Context object watcher callback function. The object passed to the callback + * is event-specific; see PyContextEvent for details. + * + * if the callback returns with an exception set, it must return -1. Otherwise + * it should return 0 + */ +typedef int (*PyContext_WatchCallback)(PyContextEvent, PyObject *); + +/* + * Register a per-interpreter callback that will be invoked for context object + * enter/exit events. + * + * Returns a handle that may be passed to PyContext_ClearWatcher on success, + * or -1 and sets and error if no more handles are available. + */ +PyAPI_FUNC(int) PyContext_AddWatcher(PyContext_WatchCallback callback); + +/* + * Clear the watcher associated with the watcher_id handle. + * + * Returns 0 on success or -1 if no watcher exists for the provided id. + */ +PyAPI_FUNC(int) PyContext_ClearWatcher(int watcher_id); /* Create a new context variable. diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h new file mode 100644 index 00000000000000..35db3fb6a59ce6 --- /dev/null +++ b/Include/cpython/critical_section.h @@ -0,0 +1,134 @@ +#ifndef Py_CPYTHON_CRITICAL_SECTION_H +# error "this header file must not be included directly" +#endif + +// Python critical sections +// +// Conceptually, critical sections are a deadlock avoidance layer on top of +// per-object locks. These helpers, in combination with those locks, replace +// our usage of the global interpreter lock to provide thread-safety for +// otherwise thread-unsafe objects, such as dict. +// +// NOTE: These APIs are no-ops in non-free-threaded builds. +// +// Straightforward per-object locking could introduce deadlocks that were not +// present when running with the GIL. Threads may hold locks for multiple +// objects simultaneously because Python operations can nest. If threads were +// to acquire the same locks in different orders, they would deadlock. +// +// One way to avoid deadlocks is to allow threads to hold only the lock (or +// locks) for a single operation at a time (typically a single lock, but some +// operations involve two locks). When a thread begins a nested operation it +// could suspend the locks for any outer operation: before beginning the nested +// operation, the locks for the outer operation are released and when the +// nested operation completes, the locks for the outer operation are +// reacquired. +// +// To improve performance, this API uses a variation of the above scheme. +// Instead of immediately suspending locks any time a nested operation begins, +// locks are only suspended if the thread would block. This reduces the number +// of lock acquisitions and releases for nested operations, while still +// avoiding deadlocks. +// +// Additionally, the locks for any active operation are suspended around +// other potentially blocking operations, such as I/O. This is because the +// interaction between locks and blocking operations can lead to deadlocks in +// the same way as the interaction between multiple locks. +// +// Each thread's critical sections and their corresponding locks are tracked in +// a stack in `PyThreadState.critical_section`. When a thread calls +// `_PyThreadState_Detach()`, such as before a blocking I/O operation or when +// waiting to acquire a lock, the thread suspends all of its active critical +// sections, temporarily releasing the associated locks. When the thread calls +// `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) +// critical section by reacquiring the associated lock or locks. See +// `_PyCriticalSection_Resume()`. +// +// NOTE: Only the top-most critical section is guaranteed to be active. +// Operations that need to lock two objects at once must use +// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections +// to lock more than one object at once, because the inner critical section +// may suspend the outer critical sections. This API does not provide a way +// to lock more than two objects at once (though it could be added later +// if actually needed). +// +// NOTE: Critical sections implicitly behave like reentrant locks because +// attempting to acquire the same lock will suspend any outer (earlier) +// critical sections. However, they are less efficient for this use case than +// purposefully designed reentrant locks. +// +// Example usage: +// Py_BEGIN_CRITICAL_SECTION(op); +// ... +// Py_END_CRITICAL_SECTION(); +// +// To lock two objects at once: +// Py_BEGIN_CRITICAL_SECTION2(op1, op2); +// ... +// Py_END_CRITICAL_SECTION2(); + +typedef struct PyCriticalSection PyCriticalSection; +typedef struct PyCriticalSection2 PyCriticalSection2; + +PyAPI_FUNC(void) +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); + +PyAPI_FUNC(void) +PyCriticalSection_End(PyCriticalSection *c); + +PyAPI_FUNC(void) +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); + +PyAPI_FUNC(void) +PyCriticalSection2_End(PyCriticalSection2 *c); + +#ifndef Py_GIL_DISABLED +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { +# define Py_END_CRITICAL_SECTION() \ + } +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { +# define Py_END_CRITICAL_SECTION2() \ + } +#else /* !Py_GIL_DISABLED */ + +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +struct PyCriticalSection { + // Tagged pointer to an outer active critical section (or 0). + uintptr_t _cs_prev; + + // Mutex used to protect critical section + PyMutex *_cs_mutex; +}; + +// A critical section protected by two mutexes. Use +// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +struct PyCriticalSection2 { + PyCriticalSection _cs_base; + + PyMutex *_cs_mutex2; +}; + +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) + +# define Py_END_CRITICAL_SECTION() \ + PyCriticalSection_End(&_py_cs); \ + } + +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +# define Py_END_CRITICAL_SECTION2() \ + PyCriticalSection2_End(&_py_cs2); \ + } + +#endif diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 35b6a822a0dfff..78473e54898fa5 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -14,16 +14,13 @@ typedef struct { /* Number of items in the dictionary */ Py_ssize_t ma_used; - /* Dictionary version: globally unique, value change each time - the dictionary is modified */ -#ifdef Py_BUILD_CORE - /* Bits 0-7 are for dict watchers. + /* This is a private field for CPython's internal use. + * Bits 0-7 are for dict watchers. * Bits 8-11 are for the watched mutation counter (used by tier2 optimization) - * The remaining bits (12-63) are the actual version tag. */ - uint64_t ma_version_tag; -#else - Py_DEPRECATED(3.12) uint64_t ma_version_tag; -#endif + * Bits 12-31 are currently unused + * Bits 32-63 are a unique id in the free threading build (used for per-thread refcounting) + */ + uint64_t _ma_watcher_tag; PyDictKeysObject *ma_keys; @@ -37,7 +34,8 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); -PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); +// PyDict_GetItemStringRef() can be used instead +Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); @@ -56,7 +54,11 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; assert(PyDict_Check(op)); mp = _Py_CAST(PyDictObject*, op); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&mp->ma_used); +#else return mp->ma_used; +#endif } #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2c..dbbfbb5105ba7a 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -27,3 +27,9 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + + +typedef struct { + PyObject_HEAD + PyFrameObject* frame; +} PyFrameLocalsProxyObject; diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index de2013323d2c72..598cd330bc9ca9 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -8,7 +8,7 @@ extern "C" { #endif -#define COMMON_FIELDS(PREFIX) \ +#define _Py_COMMON_FIELDS(PREFIX) \ PyObject *PREFIX ## globals; \ PyObject *PREFIX ## builtins; \ PyObject *PREFIX ## name; \ @@ -19,7 +19,7 @@ extern "C" { PyObject *PREFIX ## closure; /* NULL or a tuple of cell objects */ typedef struct { - COMMON_FIELDS(fc_) + _Py_COMMON_FIELDS(fc_) } PyFrameConstructor; /* Function objects and code objects should not be confused with each other: @@ -35,12 +35,13 @@ typedef struct { typedef struct { PyObject_HEAD - COMMON_FIELDS(func_) + _Py_COMMON_FIELDS(func_) PyObject *func_doc; /* The __doc__ attribute, can be anything */ PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ PyObject *func_weakreflist; /* List of weak references */ PyObject *func_module; /* The __module__ attribute, can be anything */ PyObject *func_annotations; /* Annotations, a dict or NULL */ + PyObject *func_annotate; /* Callable to fill the annotations dictionary */ PyObject *func_typeparams; /* Tuple of active type variables or NULL */ vectorcallfunc vectorcall; /* Version number for use by specializer. @@ -60,6 +61,8 @@ typedef struct { */ } PyFunctionObject; +#undef _Py_COMMON_FIELDS + PyAPI_DATA(PyTypeObject) PyFunction_Type; #define PyFunction_Check(op) Py_IS_TYPE((op), &PyFunction_Type) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index 49e46c277d75ae..f75884e597e2c2 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -9,29 +9,7 @@ extern "C" { /* --- Generators --------------------------------------------------------- */ -/* _PyGenObject_HEAD defines the initial segment of generator - and coroutine objects. */ -#define _PyGenObject_HEAD(prefix) \ - PyObject_HEAD \ - /* List of weak reference. */ \ - PyObject *prefix##_weakreflist; \ - /* Name of the generator. */ \ - PyObject *prefix##_name; \ - /* Qualified name of the generator. */ \ - PyObject *prefix##_qualname; \ - _PyErr_StackItem prefix##_exc_state; \ - PyObject *prefix##_origin_or_finalizer; \ - char prefix##_hooks_inited; \ - char prefix##_closed; \ - char prefix##_running_async; \ - /* The frame */ \ - int8_t prefix##_frame_state; \ - PyObject *prefix##_iframe[1]; \ - -typedef struct { - /* The gi_ prefix is intended to remind of generator-iterator. */ - _PyGenObject_HEAD(gi) -} PyGenObject; +typedef struct _PyGenObject PyGenObject; PyAPI_DATA(PyTypeObject) PyGen_Type; @@ -46,9 +24,7 @@ PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); /* --- PyCoroObject ------------------------------------------------------- */ -typedef struct { - _PyGenObject_HEAD(cr) -} PyCoroObject; +typedef struct _PyCoroObject PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; @@ -59,9 +35,7 @@ PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, /* --- Asynchronous Generators -------------------------------------------- */ -typedef struct { - _PyGenObject_HEAD(ag) -} PyAsyncGenObject; +typedef struct _PyAsyncGenObject PyAsyncGenObject; PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; @@ -73,7 +47,6 @@ PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, #define PyAsyncGenASend_CheckExact(op) Py_IS_TYPE((op), &_PyAsyncGenASend_Type) - #undef _PyGenObject_HEAD #ifdef __cplusplus diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 87c059c521cbc9..c2cb4e3cdd92fb 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -181,6 +181,9 @@ typedef struct PyConfig { int int_max_str_digits; int cpu_count; +#ifdef Py_GIL_DISABLED + int enable_gil; +#endif /* --- Path configuration inputs ------------ */ int pathconfig_warnings; @@ -257,6 +260,14 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, Py_ssize_t length, wchar_t **items); +/* --- PyConfig_Get() ----------------------------------------- */ + +PyAPI_FUNC(PyObject*) PyConfig_Get(const char *name); +PyAPI_FUNC(int) PyConfig_GetInt(const char *name, int *value); +PyAPI_FUNC(PyObject*) PyConfig_Names(void); +PyAPI_FUNC(int) PyConfig_Set(const char *name, PyObject *value); + + /* --- Helper functions --------------------------------------- */ /* Get the original command line arguments, before Python modified them. @@ -264,6 +275,51 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, See also PyConfig.orig_argv. */ PyAPI_FUNC(void) Py_GetArgcArgv(int *argc, wchar_t ***argv); + +// --- PyInitConfig --------------------------------------------------------- + +typedef struct PyInitConfig PyInitConfig; + +PyAPI_FUNC(PyInitConfig*) PyInitConfig_Create(void); +PyAPI_FUNC(void) PyInitConfig_Free(PyInitConfig *config); + +PyAPI_FUNC(int) PyInitConfig_GetError(PyInitConfig* config, + const char **err_msg); +PyAPI_FUNC(int) PyInitConfig_GetExitCode(PyInitConfig* config, + int *exitcode); + +PyAPI_FUNC(int) PyInitConfig_HasOption(PyInitConfig *config, + const char *name); +PyAPI_FUNC(int) PyInitConfig_GetInt(PyInitConfig *config, + const char *name, + int64_t *value); +PyAPI_FUNC(int) PyInitConfig_GetStr(PyInitConfig *config, + const char *name, + char **value); +PyAPI_FUNC(int) PyInitConfig_GetStrList(PyInitConfig *config, + const char *name, + size_t *length, + char ***items); +PyAPI_FUNC(void) PyInitConfig_FreeStrList(size_t length, char **items); + +PyAPI_FUNC(int) PyInitConfig_SetInt(PyInitConfig *config, + const char *name, + int64_t value); +PyAPI_FUNC(int) PyInitConfig_SetStr(PyInitConfig *config, + const char *name, + const char *value); +PyAPI_FUNC(int) PyInitConfig_SetStrList(PyInitConfig *config, + const char *name, + size_t length, + char * const *items); + +PyAPI_FUNC(int) PyInitConfig_AddModule(PyInitConfig *config, + const char *name, + PyObject* (*initfunc)(void)); + +PyAPI_FUNC(int) Py_InitializeFromInitConfig(PyInitConfig *config); + + #ifdef __cplusplus } #endif diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h deleted file mode 100644 index 4ab9ad5d315f80..00000000000000 --- a/Include/cpython/interpreteridobject.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H -# error "this header file must not be included directly" -#endif - -/* Interpreter ID Object */ - -PyAPI_DATA(PyTypeObject) PyInterpreterID_Type; - -PyAPI_FUNC(PyObject *) PyInterpreterID_New(int64_t); -PyAPI_FUNC(PyObject *) PyInterpreterState_GetIDObject(PyInterpreterState *); -PyAPI_FUNC(PyInterpreterState *) PyInterpreterID_LookUp(PyObject *); diff --git a/Include/cpython/lock.h b/Include/cpython/lock.h new file mode 100644 index 00000000000000..8ee03e82f74dfd --- /dev/null +++ b/Include/cpython/lock.h @@ -0,0 +1,63 @@ +#ifndef Py_CPYTHON_LOCK_H +# error "this header file must not be included directly" +#endif + +#define _Py_UNLOCKED 0 +#define _Py_LOCKED 1 + +// A mutex that occupies one byte. The lock can be zero initialized to +// represent the unlocked state. +// +// Typical initialization: +// PyMutex m = (PyMutex){0}; +// +// Or initialize as global variables: +// static PyMutex m; +// +// Typical usage: +// PyMutex_Lock(&m); +// ... +// PyMutex_Unlock(&m); +// +// The contents of the PyMutex are not part of the public API, but are +// described to aid in understanding the implementation and debugging. Only +// the two least significant bits are used. The remaining bits are always zero: +// 0b00: unlocked +// 0b01: locked +// 0b10: unlocked and has parked threads +// 0b11: locked and has parked threads +typedef struct PyMutex { + uint8_t _bits; // (private) +} PyMutex; + +// exported function for locking the mutex +PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m); + +// exported function for unlocking the mutex +PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m); + +// Locks the mutex. +// +// If the mutex is currently locked, the calling thread will be parked until +// the mutex is unlocked. If the current thread holds the GIL, then the GIL +// will be released while the thread is parked. +static inline void +_PyMutex_Lock(PyMutex *m) +{ + uint8_t expected = _Py_UNLOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_LOCKED)) { + PyMutex_Lock(m); + } +} +#define PyMutex_Lock _PyMutex_Lock + +// Unlocks the mutex. +static inline void +_PyMutex_Unlock(PyMutex *m) +{ + uint8_t expected = _Py_LOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_UNLOCKED)) { + PyMutex_Unlock(m); + } +} +#define PyMutex_Unlock _PyMutex_Unlock diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index f5ccbb704e8bb8..c60ccc463653f9 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -62,21 +62,32 @@ typedef long stwodigits; /* signed variant of twodigits */ #define PyLong_MASK ((digit)(PyLong_BASE - 1)) /* Long integer representation. + + Long integers are made up of a number of 30- or 15-bit digits, depending on + the platform. The number of digits (ndigits) is stored in the high bits of + the lv_tag field (lvtag >> _PyLong_NON_SIZE_BITS). + The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. - In a normalized number, ob_digit[abs(ob_size)-1] (the most significant + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + + The sign of the value is stored in the lower 2 bits of lv_tag. + + - 0: Positive + - 1: Zero + - 2: Negative + + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. + + In a normalized number, ob_digit[ndigits-1] (the most significant digit) is never zero. Also, in all cases, for all valid i, - 0 <= ob_digit[i] <= MASK. + 0 <= ob_digit[i] <= PyLong_MASK. + The allocation function takes care of allocating extra memory - so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. + so that ob_digit[0] ... ob_digit[ndigits-1] are actually available. We always allocate memory for at least one digit, so accessing ob_digit[0] - is always safe. However, in the case ob_size == 0, the contents of + is always safe. However, in the case ndigits == 0, the contents of ob_digit[0] may be undefined. - - CAUTION: Generic code manipulating subtypes of PyVarObject has to - aware that ints abuse ob_size's sign bit. */ typedef struct _PyLongValue { @@ -109,7 +120,7 @@ PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( static inline int _PyLong_IsCompact(const PyLongObject* op) { - assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); + assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS); } @@ -118,9 +129,10 @@ _PyLong_IsCompact(const PyLongObject* op) { static inline Py_ssize_t _PyLong_CompactValue(const PyLongObject *op) { - assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); + Py_ssize_t sign; + assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); - Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); return sign * (Py_ssize_t)op->long_value.ob_digit[0]; } diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index fd1be29ed397d1..c1214d5e3714ea 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -2,16 +2,82 @@ # error "this header file must not be included directly" #endif +#define _PyLong_CAST(op) \ + (assert(PyLong_Check(op)), _Py_CAST(PyLongObject*, (op))) + PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); +#define Py_ASNATIVEBYTES_DEFAULTS -1 +#define Py_ASNATIVEBYTES_BIG_ENDIAN 0 +#define Py_ASNATIVEBYTES_LITTLE_ENDIAN 1 +#define Py_ASNATIVEBYTES_NATIVE_ENDIAN 3 +#define Py_ASNATIVEBYTES_UNSIGNED_BUFFER 4 +#define Py_ASNATIVEBYTES_REJECT_NEGATIVE 8 +#define Py_ASNATIVEBYTES_ALLOW_INDEX 16 + +/* PyLong_AsNativeBytes: Copy the integer value to a native variable. + buffer points to the first byte of the variable. + n_bytes is the number of bytes available in the buffer. Pass 0 to request + the required size for the value. + flags is a bitfield of the following flags: + * 1 - little endian + * 2 - native endian + * 4 - unsigned destination (e.g. don't reject copying 255 into one byte) + * 8 - raise an exception for negative inputs + * 16 - call __index__ on non-int types + If flags is -1 (all bits set), native endian is used, value truncation + behaves most like C (allows negative inputs and allow MSB set), and non-int + objects will raise a TypeError. + Big endian mode will write the most significant byte into the address + directly referenced by buffer; little endian will write the least significant + byte into that address. + + If an exception is raised, returns a negative value. + Otherwise, returns the number of bytes that are required to store the value. + To check that the full value is represented, ensure that the return value is + equal or less than n_bytes. + All n_bytes are guaranteed to be written (unless an exception occurs), and + so ignoring a positive return value is the equivalent of a downcast in C. + In cases where the full value could not be represented, the returned value + may be larger than necessary - this function is not an accurate way to + calculate the bit length of an integer object. + */ +PyAPI_FUNC(Py_ssize_t) PyLong_AsNativeBytes(PyObject* v, void* buffer, + Py_ssize_t n_bytes, int flags); + +/* PyLong_FromNativeBytes: Create an int value from a native integer + n_bytes is the number of bytes to read from the buffer. Passing 0 will + always produce the zero int. + PyLong_FromUnsignedNativeBytes always produces a non-negative int. + flags is the same as for PyLong_AsNativeBytes, but only supports selecting + the endianness or forcing an unsigned buffer. + + Returns the int object, or NULL with an exception set. */ +PyAPI_FUNC(PyObject*) PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, + int flags); +PyAPI_FUNC(PyObject*) PyLong_FromUnsignedNativeBytes(const void* buffer, + size_t n_bytes, int flags); + PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); -// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. -// v must not be NULL, and must be a normalized long. -// There are no error cases. +/* PyLong_GetSign. Get the sign of an integer object: + 0, -1 or +1 for zero, negative or positive integer, respectively. + + - On success, set '*sign' to the integer sign, and return 0. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_GetSign(PyObject *v, int *sign); + PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); +/* _PyLong_NumBits. Return the number of bits needed to represent the + absolute value of a long. For example, this returns 1 for 1 and -1, 2 + for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. + v must not be NULL, and must be a normalized long. + Always successful. +*/ +PyAPI_FUNC(int64_t) _PyLong_NumBits(PyObject *v); + /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. If n is 0, the integer is 0. Else: @@ -50,7 +116,7 @@ PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( */ PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, - int little_endian, int is_signed); + int little_endian, int is_signed, int with_exceptions); /* For use by the gcd function in mathmodule.c */ PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h new file mode 100644 index 00000000000000..d3b88f58c82ca3 --- /dev/null +++ b/Include/cpython/modsupport.h @@ -0,0 +1,26 @@ +#ifndef Py_CPYTHON_MODSUPPORT_H +# error "this header file must not be included directly" +#endif + +// A data structure that can be used to run initialization code once in a +// thread-safe manner. The C++11 equivalent is std::call_once. +typedef struct { + uint8_t v; +} _PyOnceFlag; + +typedef struct _PyArg_Parser { + const char *format; + const char * const *keywords; + const char *fname; + const char *custom_msg; + _PyOnceFlag once; /* atomic one-time initialization flag */ + int is_kwtuple_owned; /* does this parser own the kwtuple object? */ + int pos; /* number of positional-only arguments */ + int min; /* minimal number of arguments */ + int max; /* maximal number of positional arguments */ + PyObject *kwtuple; /* tuple of keyword parameter names */ + struct _PyArg_Parser *next; +} _PyArg_Parser; + +PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, + struct _PyArg_Parser *, ...); diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h new file mode 100644 index 00000000000000..797ba51246b1c6 --- /dev/null +++ b/Include/cpython/monitoring.h @@ -0,0 +1,250 @@ +#ifndef Py_CPYTHON_MONITORING_H +# error "this header file must not be included directly" +#endif + +/* Local events. + * These require bytecode instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_EVENT_STOP_ITERATION 9 + +#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ + ((ev) < _PY_MONITORING_LOCAL_EVENTS) + +/* Other events, mainly exceptions */ + +#define PY_MONITORING_EVENT_RAISE 10 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 +#define PY_MONITORING_EVENT_PY_UNWIND 12 +#define PY_MONITORING_EVENT_PY_THROW 13 +#define PY_MONITORING_EVENT_RERAISE 14 + + +/* Ancillary events */ + +#define PY_MONITORING_EVENT_C_RETURN 15 +#define PY_MONITORING_EVENT_C_RAISE 16 + + +typedef struct _PyMonitoringState { + uint8_t active; + uint8_t opaque; +} PyMonitoringState; + + +PyAPI_FUNC(int) +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length); + +PyAPI_FUNC(int) +PyMonitoring_ExitScope(void); + + +PyAPI_FUNC(int) +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0); + +PyAPI_FUNC(int) +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno); + +PyAPI_FUNC(int) +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value); + + +#define _PYMONITORING_IF_ACTIVE(STATE, X) \ + if ((STATE)->active) { \ + return (X); \ + } \ + else { \ + return 0; \ + } + +static inline int +PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyStartEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyResumeEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0)); +} + +static inline int +PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireLineEvent(state, codelike, offset, lineno)); +} + +static inline int +PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyThrowEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireReraiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireExceptionHandledEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyUnwindEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireStopIterationEvent(state, codelike, offset, value)); +} + +#undef _PYMONITORING_IF_ACTIVE diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 7512bb70c760fd..f0f61796cd3ec8 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -269,13 +269,17 @@ typedef struct _heaptypeobject { struct _dictkeysobject *ht_cached_keys; PyObject *ht_module; char *_ht_tpname; // Storage for "tp_name"; see PyType_FromModuleAndSpec + void *ht_token; // Storage for the "Py_tp_token" slot struct _specialization_cache _spec_cache; // For use by the specializer. +#ifdef Py_GIL_DISABLED + Py_ssize_t unique_id; // ID used for per-thread refcounting +#endif /* here are optional user slots, followed by the members. */ } PyHeapTypeObject; PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); +PyAPI_FUNC(PyObject *) _PyType_LookupRef(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); @@ -288,6 +292,8 @@ PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); +PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *); + /* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes dict as the last parameter. */ PyAPI_FUNC(PyObject *) @@ -313,7 +319,7 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); * triggered as a side-effect of `dst` getting torn down no longer believes * `dst` points to a valid object. * - * Temporary variables are used to only evalutate macro arguments once and so + * Temporary variables are used to only evaluate macro arguments once and so * avoid the duplication of side effects. _Py_TYPEOF() or memcpy() is used to * avoid a miscompilation caused by type punning. See Py_CLEAR() comment for * implementation details about type punning. @@ -449,8 +455,8 @@ without deallocating anything (and so unbounded call-stack depth is avoided). When the call stack finishes unwinding again, code generated by the END macro notices this, and calls another routine to deallocate all the objects that may have been added to the list of deferred deallocations. In effect, a -chain of N deallocations is broken into (N-1)/(_PyTrash_UNWIND_LEVEL-1) pieces, -with the call stack never exceeding a depth of _PyTrash_UNWIND_LEVEL. +chain of N deallocations is broken into (N-1)/(Py_TRASHCAN_HEADROOM-1) pieces, +with the call stack never exceeding a depth of Py_TRASHCAN_HEADROOM. Since the tp_dealloc of a subclass typically calls the tp_dealloc of the base class, we need to ensure that the trashcan is only triggered on the tp_dealloc @@ -462,35 +468,39 @@ passed as second argument to Py_TRASHCAN_BEGIN(). /* Python 3.9 private API, invoked by the macros below. */ PyAPI_FUNC(int) _PyTrash_begin(PyThreadState *tstate, PyObject *op); PyAPI_FUNC(void) _PyTrash_end(PyThreadState *tstate); + +PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op); +PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate); + + /* Python 3.10 private API, invoked by the Py_TRASHCAN_BEGIN(). */ -PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); -#define Py_TRASHCAN_BEGIN_CONDITION(op, cond) \ - do { \ - PyThreadState *_tstate = NULL; \ - /* If "cond" is false, then _tstate remains NULL and the deallocator \ - * is run normally without involving the trashcan */ \ - if (cond) { \ - _tstate = PyThreadState_GetUnchecked(); \ - if (_PyTrash_begin(_tstate, _PyObject_CAST(op))) { \ - break; \ - } \ - } - /* The body of the deallocator is here. */ -#define Py_TRASHCAN_END \ - if (_tstate) { \ - _PyTrash_end(_tstate); \ - } \ - } while (0); +/* To avoid raising recursion errors during dealloc trigger trashcan before we reach + * recursion limit. To avoid trashing, we don't attempt to empty the trashcan until + * we have headroom above the trigger limit */ +#define Py_TRASHCAN_HEADROOM 50 #define Py_TRASHCAN_BEGIN(op, dealloc) \ - Py_TRASHCAN_BEGIN_CONDITION((op), \ - _PyTrash_cond(_PyObject_CAST(op), (destructor)(dealloc))) +do { \ + PyThreadState *tstate = PyThreadState_Get(); \ + if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \ + _PyTrash_thread_deposit_object(tstate, (PyObject *)op); \ + break; \ + } \ + tstate->c_recursion_remaining--; + /* The body of the deallocator is here. */ +#define Py_TRASHCAN_END \ + tstate->c_recursion_remaining++; \ + if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \ + _PyTrash_thread_destroy_chain(tstate); \ + } \ +} while (0); PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); +PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); #define TYPE_MAX_WATCHERS 8 @@ -507,3 +517,13 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type); * assigned, or 0 if a new tag could not be assigned. */ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); + + +typedef enum { + PyRefTracer_CREATE = 0, + PyRefTracer_DESTROY = 1, +} PyRefTracerEvent; + +typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); +PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data); +PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**); diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 58a30aeea6ac64..e0c2ce286f13ce 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -85,3 +85,20 @@ PyAPI_FUNC(PyObject **) PyObject_GET_WEAKREFS_LISTPTR(PyObject *op); PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *, size_t); + + +/* Visit all live GC-capable objects, similar to gc.get_objects(None). The + * supplied callback is called on every such object with the void* arg set + * to the supplied arg. Returning 0 from the callback ends iteration, returning + * 1 allows iteration to continue. Returning any other value may result in + * undefined behaviour. + * + * If new objects are (de)allocated by the callback it is undefined if they + * will be visited. + + * Garbage collection is disabled during operation. Explicitly running a + * collection in the callback may lead to undefined behaviour e.g. visiting the + * same objects multiple times or not at all. + */ +typedef int (*gcvisitobjects_t)(PyObject*, void*); +PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h deleted file mode 100644 index 5a9ccaea3b2209..00000000000000 --- a/Include/cpython/optimizer.h +++ /dev/null @@ -1,107 +0,0 @@ - -#ifndef Py_LIMITED_API -#ifndef Py_OPTIMIZER_H -#define Py_OPTIMIZER_H -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct _PyExecutorLinkListNode { - struct _PyExecutorObject *next; - struct _PyExecutorObject *previous; -} _PyExecutorLinkListNode; - - -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define BLOOM_FILTER_WORDS 8 - -typedef struct _bloom_filter { - uint32_t bits[BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - -typedef struct { - uint8_t opcode; - uint8_t oparg; - uint8_t valid; - uint8_t linked; - _PyBloomFilter bloom; - _PyExecutorLinkListNode links; -} _PyVMData; - -typedef struct { - uint16_t opcode; - uint16_t oparg; - uint32_t target; - uint64_t operand; // A cache entry -} _PyUOpInstruction; - -typedef struct _PyExecutorObject { - PyObject_VAR_HEAD - _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ - void *jit_code; - size_t jit_size; - _PyUOpInstruction trace[1]; -} _PyExecutorObject; - -typedef struct _PyOptimizerObject _PyOptimizerObject; - -/* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ -typedef int (*optimize_func)( - _PyOptimizerObject* self, struct _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, - int curr_stackentries); - -typedef struct _PyOptimizerObject { - PyObject_HEAD - optimize_func optimize; - /* These thresholds are treated as signed so do not exceed INT16_MAX - * Use INT16_MAX to indicate that the optimizer should never be called */ - uint16_t resume_threshold; - uint16_t backedge_threshold; - /* Data needed by the optimizer goes here, but is opaque to the VM */ -} _PyOptimizerObject; - -/** Test support **/ -typedef struct { - _PyOptimizerObject base; - int64_t count; -} _PyCounterOptimizerObject; - -PyAPI_FUNC(int) PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject *executor); - -PyAPI_FUNC(void) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); - -PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); - -PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); - -int -_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer); - -extern _PyOptimizerObject _PyOptimizer_Default; - -void _Py_ExecutorInit(_PyExecutorObject *, _PyBloomFilter *); -void _Py_ExecutorClear(_PyExecutorObject *); -void _Py_BloomFilter_Init(_PyBloomFilter *); -void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); -PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); -PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj); -extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp); - -/* For testing */ -PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); -PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); - -#define OPTIMIZER_BITS_IN_COUNTER 4 -/* Minimum of 16 additional executions before retry */ -#define MINIMUM_TIER2_BACKOFF 4 - -#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 -#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPTIMIZER_H */ -#endif /* Py_LIMITED_API */ diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 5314a70436bfc3..4ecef4f56edf42 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -360,6 +360,8 @@ _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj); static inline void * _Py_atomic_load_ptr_relaxed(const void *obj); +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj); // --- _Py_atomic_store ------------------------------------------------------ // Atomically performs `*obj = value` (sequential consistency) @@ -452,6 +454,10 @@ _Py_atomic_store_ptr_relaxed(void *obj, void *value); static inline void _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value); + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -459,16 +465,42 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); static inline void * _Py_atomic_load_ptr_acquire(const void *obj); +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj); + // Stores `*obj = value` (release operation) static inline void _Py_atomic_store_ptr_release(void *obj, void *value); +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); + static inline void _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); + + + // --- _Py_atomic_fence ------------------------------------------------------ @@ -478,6 +510,9 @@ _Py_atomic_load_int_acquire(const int *obj); // See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence static inline void _Py_atomic_fence_seq_cst(void); +// Acquire fence +static inline void _Py_atomic_fence_acquire(void); + // Release fence static inline void _Py_atomic_fence_release(void); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 70f2b7e1b5706a..ef09954d53ac1d 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -297,7 +297,7 @@ _Py_atomic_load_ssize(const Py_ssize_t *obj) static inline void * _Py_atomic_load_ptr(const void *obj) -{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_SEQ_CST); } +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_SEQ_CST); } // --- _Py_atomic_load_relaxed ----------------------------------------------- @@ -356,7 +356,11 @@ _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) static inline void * _Py_atomic_load_ptr_relaxed(const void *obj) -{ return (void *)__atomic_load_n((const void **)obj, __ATOMIC_RELAXED); } +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_RELAXED); } + +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } // --- _Py_atomic_store ------------------------------------------------------ @@ -476,25 +480,61 @@ static inline void _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELAXED); } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ static inline void * _Py_atomic_load_ptr_acquire(const void *obj) -{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_ACQUIRE); } +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); } + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ return (uintptr_t)__atomic_load_n(obj, __ATOMIC_ACQUIRE); } static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } // --- _Py_atomic_fence ------------------------------------------------------ @@ -502,6 +542,10 @@ static inline void _Py_atomic_fence_seq_cst(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); } + static inline void +_Py_atomic_fence_acquire(void) +{ __atomic_thread_fence(__ATOMIC_ACQUIRE); } + static inline void _Py_atomic_fence_release(void) { __atomic_thread_fence(__ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 601a0cf65afc1c..84da21bdcbff4f 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -712,6 +712,12 @@ _Py_atomic_load_ptr_relaxed(const void *obj) return *(void * volatile *)obj; } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + return *(volatile unsigned long long *)obj; +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -886,6 +892,14 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) *(volatile Py_ssize_t *)obj = value; } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + *(volatile unsigned long long *)obj = value; +} + + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ static inline void * @@ -900,6 +914,18 @@ _Py_atomic_load_ptr_acquire(const void *obj) #endif } +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uintptr_t volatile *)obj; +#elif defined(_M_ARM64) + return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uintptr_acquire" +#endif +} + static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { @@ -912,6 +938,19 @@ _Py_atomic_store_ptr_release(void *obj, void *value) #endif } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uintptr_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uintptr_release" +#endif +} + static inline void _Py_atomic_store_int_release(int *obj, int value) { @@ -925,6 +964,18 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(Py_ssize_t volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_ssize_release" +#endif +} + static inline int _Py_atomic_load_int_acquire(const int *obj) { @@ -938,6 +989,68 @@ _Py_atomic_load_int_acquire(const int *obj) #endif } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint32_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint32_release" +#endif +} + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint64_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uint64_release" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint64_t volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64_acquire" +#endif +} + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint32_t volatile *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((uint32_t volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint32_acquire" +#endif +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(Py_ssize_t volatile *)obj; +#elif defined(_M_ARM64) + return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ssize_acquire" +#endif +} // --- _Py_atomic_fence ------------------------------------------------------ @@ -953,6 +1066,18 @@ _Py_atomic_fence_seq_cst(void) #else # error "no implementation of _Py_atomic_fence_seq_cst" #endif +} + + static inline void +_Py_atomic_fence_acquire(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISHLD); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadBarrier(); +#else +# error "no implementation of _Py_atomic_fence_acquire" +#endif } static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index a05bfaec47e89d..7c71e94c68f8e6 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -619,6 +619,14 @@ _Py_atomic_load_ptr_relaxed(const void *obj) memory_order_relaxed); } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned long long)*)obj, + memory_order_relaxed); +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -835,6 +843,15 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) memory_order_relaxed); } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned long long)*)obj, value, + memory_order_relaxed); +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -846,6 +863,14 @@ _Py_atomic_load_ptr_acquire(const void *obj) memory_order_acquire); } +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_acquire); +} + static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { @@ -854,6 +879,14 @@ _Py_atomic_store_ptr_release(void *obj, void *value) memory_order_release); } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_int_release(int *obj, int value) { @@ -862,6 +895,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_release); +} + static inline int _Py_atomic_load_int_acquire(const int *obj) { @@ -870,6 +911,45 @@ _Py_atomic_load_int_acquire(const int *obj) memory_order_acquire); } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_release); +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint64_t)*)obj, + memory_order_acquire); +} + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint32_t)*)obj, + memory_order_acquire); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, + memory_order_acquire); +} // --- _Py_atomic_fence ------------------------------------------------------ @@ -881,6 +961,13 @@ _Py_atomic_fence_seq_cst(void) atomic_thread_fence(memory_order_seq_cst); } + static inline void +_Py_atomic_fence_acquire(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_acquire); +} + static inline void _Py_atomic_fence_release(void) { diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 479b908fb7058a..b36b4681f5dddb 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -88,6 +88,10 @@ typedef PyOSErrorObject PyEnvironmentErrorObject; typedef PyOSErrorObject PyWindowsErrorObject; #endif +/* Context manipulation (PEP 3134) */ + +PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); + /* In exceptions.c */ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( @@ -96,7 +100,7 @@ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( /* In signalmodule.c */ -int PySignal_SetWakeupFd(int fd); +PyAPI_FUNC(int) PySignal_SetWakeupFd(int fd); /* Support for adding program text to SyntaxErrors */ @@ -122,4 +126,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFunc( PyAPI_FUNC(void) PyErr_FormatUnraisable(const char *, ...); +PyAPI_DATA(PyObject *) PyExc_PythonFinalizationError; + #define Py_FatalError(message) _Py_FatalErrorFunc(__func__, (message)) diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index c5adbbe4868f69..51529763923ec3 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -3,8 +3,10 @@ #endif PyAPI_DATA(PyTypeObject) PyFrame_Type; +PyAPI_DATA(PyTypeObject) PyFrameLocalsProxy_Type; #define PyFrame_Check(op) Py_IS_TYPE((op), &PyFrame_Type) +#define PyFrameLocalsProxy_Check(op) Py_IS_TYPE((op), &PyFrameLocalsProxy_Type) PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *frame); @@ -26,7 +28,7 @@ struct _PyInterpreterFrame; * Does not raise an exception. */ PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); -/* Returns a byte ofsset into the last executed instruction. +/* Returns a byte offset into the last executed instruction. * Does not raise an exception. */ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index 396c208e1b106a..876a7f0ea44f4d 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -3,21 +3,28 @@ #endif /* Prime multiplier used in string and various other hashes. */ -#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ +#define PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ /* Parameters used for the numeric hash implementation. See notes for _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on reduction modulo the prime 2**_PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 -# define _PyHASH_BITS 61 +# define PyHASH_BITS 61 #else -# define _PyHASH_BITS 31 +# define PyHASH_BITS 31 #endif -#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) -#define _PyHASH_INF 314159 -#define _PyHASH_IMAG _PyHASH_MULTIPLIER +#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_INF 314159 +#define PyHASH_IMAG PyHASH_MULTIPLIER + +/* Aliases kept for backward compatibility with Python 3.12 */ +#define _PyHASH_MULTIPLIER PyHASH_MULTIPLIER +#define _PyHASH_BITS PyHASH_BITS +#define _PyHASH_MODULUS PyHASH_MODULUS +#define _PyHASH_INF PyHASH_INF +#define _PyHASH_IMAG PyHASH_IMAG /* Helpers for hash functions */ PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); @@ -37,3 +44,6 @@ typedef struct { PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); PyAPI_FUNC(Py_hash_t) Py_HashPointer(const void *ptr); +PyAPI_FUNC(Py_hash_t) PyObject_GenericHash(PyObject *); + +PyAPI_FUNC(Py_hash_t) Py_HashBuffer(const void *ptr, Py_ssize_t len); diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index d425a233f71000..e46dfe59ec4630 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -63,6 +63,15 @@ typedef struct { .gil = PyInterpreterConfig_OWN_GIL, \ } +// gh-117649: The free-threaded build does not currently support single-phase +// init extensions in subinterpreters. For now, we ensure that +// `check_multi_interp_extensions` is always `1`, even in the legacy config. +#ifdef Py_GIL_DISABLED +# define _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS 1 +#else +# define _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS 0 +#endif + #define _PyInterpreterConfig_LEGACY_INIT \ { \ .use_main_obmalloc = 1, \ @@ -70,7 +79,7 @@ typedef struct { .allow_exec = 1, \ .allow_threads = 1, \ .allow_daemon_threads = 1, \ - .check_multi_interp_extensions = 0, \ + .check_multi_interp_extensions = _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS, \ .gil = PyInterpreterConfig_SHARED_GIL, \ } diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 9bc8758e72bd8f..32f68378ea5d72 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -56,11 +56,6 @@ typedef struct _stack_chunk { PyObject * data[1]; /* Variable sized */ } _PyStackChunk; -struct _py_trashcan { - int delete_nesting; - PyObject *delete_later; -}; - struct _ts { /* See Python/ceval.c for comments explaining most fields */ @@ -68,6 +63,11 @@ struct _ts { PyThreadState *next; PyInterpreterState *interp; + /* The global instrumentation version in high bits, plus flags indicating + when to break out of the interpreter loop in lower bits. See details in + pycore_ceval.h. */ + uintptr_t eval_breaker; + struct { /* Has been initialized to a safe state. @@ -83,6 +83,8 @@ struct _ts { unsigned int bound_gilstate:1; /* Currently in use (maybe holds the GIL). */ unsigned int active:1; + /* Currently holds the GIL. */ + unsigned int holds_gil:1; /* various stages of finalization */ unsigned int finalizing:1; @@ -90,15 +92,16 @@ struct _ts { unsigned int finalized:1; /* padding to align to 4 bytes */ - unsigned int :24; + unsigned int :23; } _status; #ifdef Py_BUILD_CORE # define _PyThreadState_WHENCE_NOTSET -1 # define _PyThreadState_WHENCE_UNKNOWN 0 -# define _PyThreadState_WHENCE_INTERP 1 -# define _PyThreadState_WHENCE_THREADING 2 -# define _PyThreadState_WHENCE_GILSTATE 3 -# define _PyThreadState_WHENCE_EXEC 4 +# define _PyThreadState_WHENCE_INIT 1 +# define _PyThreadState_WHENCE_FINI 2 +# define _PyThreadState_WHENCE_THREADING 3 +# define _PyThreadState_WHENCE_GILSTATE 4 +# define _PyThreadState_WHENCE_EXEC 5 #endif int _whence; @@ -147,7 +150,7 @@ struct _ts { */ unsigned long native_thread_id; - struct _py_trashcan trash; + PyObject *delete_later; /* Tagged pointer to top-most critical section, or zero if there is no * active critical section. Critical sections are only used in @@ -156,32 +159,6 @@ struct _ts { */ uintptr_t critical_section; - /* Called when a thread state is deleted normally, but not when it - * is destroyed after fork(). - * Pain: to prevent rare but fatal shutdown errors (issue 18808), - * Thread.join() must wait for the join'ed thread's tstate to be unlinked - * from the tstate chain. That happens at the end of a thread's life, - * in pystate.c. - * The obvious way doesn't quite work: create a lock which the tstate - * unlinking code releases, and have Thread.join() wait to acquire that - * lock. The problem is that we _are_ at the end of the thread's life: - * if the thread holds the last reference to the lock, decref'ing the - * lock will delete the lock, and that may trigger arbitrary Python code - * if there's a weakref, with a callback, to the lock. But by this time - * _PyRuntime.gilstate.tstate_current is already NULL, so only the simplest - * of C code can be allowed to run (in particular it must not be possible to - * release the GIL). - * So instead of holding the lock directly, the tstate holds a weakref to - * the lock: that's the value of on_delete_data below. Decref'ing a - * weakref is harmless. - * on_delete points to _threadmodule.c's static release_sentinel() function. - * After the tstate is unlinked, release_sentinel is called with the - * weakref-to-lock (on_delete_data) argument, and release_sentinel releases - * the indirectly held lock. - */ - void (*on_delete)(void *); - void *on_delete_data; - int coroutine_origin_tracking_depth; PyObject *async_gen_firstiter; @@ -212,26 +189,44 @@ struct _ts { /* The thread's exception stack entry. (Always the last entry.) */ _PyErr_StackItem exc_state; + PyObject *previous_executor; + + uint64_t dict_global_version; + + /* Used to store/retrieve `threading.local` keys/values for this thread */ + PyObject *threading_local_key; + + /* Used by `threading.local`s to be remove keys/values for dying threads. + The PyThreadObject must hold the only reference to this value. + */ + PyObject *threading_local_sentinel; }; #ifdef Py_DEBUG // A debug build is likely built with low optimization level which implies // higher stack memory usage than a release build: use a lower limit. -# if defined(__wasi__) - // Based on wasmtime 16. -# define Py_C_RECURSION_LIMIT 150 -# else -# define Py_C_RECURSION_LIMIT 500 -# endif -#elif defined(__wasi__) - // Based on wasmtime 16. # define Py_C_RECURSION_LIMIT 500 #elif defined(__s390x__) # define Py_C_RECURSION_LIMIT 800 +#elif defined(_WIN32) && defined(_M_ARM64) +# define Py_C_RECURSION_LIMIT 1000 #elif defined(_WIN32) # define Py_C_RECURSION_LIMIT 3000 +#elif defined(__ANDROID__) + // On an ARM64 emulator, API level 34 was OK with 10000, but API level 21 + // crashed in test_compiler_recursion_limit. +# define Py_C_RECURSION_LIMIT 3000 #elif defined(_Py_ADDRESS_SANITIZER) # define Py_C_RECURSION_LIMIT 4000 +#elif defined(__sparc__) + // test_descr crashed on sparc64 with >7000 but let's keep a margin of error. +# define Py_C_RECURSION_LIMIT 4000 +#elif defined(__wasi__) + // Based on wasmtime 16. +# define Py_C_RECURSION_LIMIT 5000 +#elif defined(__hppa__) || defined(__powerpc64__) + // test_descr crashed with >8000 but let's keep a margin of error. +# define Py_C_RECURSION_LIMIT 5000 #else // This value is duplicated in Lib/test/support/__init__.py # define Py_C_RECURSION_LIMIT 10000 diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index bf0cfe4cb695b4..f1ca54839fbc38 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -23,6 +23,8 @@ # error "this header file must not be included directly" #endif +#define PYSTATS_MAX_UOP_ID 512 + #define SPECIALIZATION_FAILURE_KINDS 36 /* Stats for determining who is calling PyEval_EvalFrame */ @@ -68,6 +70,10 @@ typedef struct _object_stats { uint64_t decrefs; uint64_t interpreter_increfs; uint64_t interpreter_decrefs; + uint64_t immortal_increfs; + uint64_t immortal_decrefs; + uint64_t interpreter_immortal_increfs; + uint64_t interpreter_immortal_decrefs; uint64_t allocations; uint64_t allocations512; uint64_t allocations4k; @@ -75,12 +81,11 @@ typedef struct _object_stats { uint64_t frees; uint64_t to_freelist; uint64_t from_freelist; - uint64_t new_values; + uint64_t inline_values; uint64_t dict_materialized_on_request; uint64_t dict_materialized_new_key; uint64_t dict_materialized_too_big; uint64_t dict_materialized_str_subclass; - uint64_t dict_dematerialized; uint64_t type_cache_hits; uint64_t type_cache_misses; uint64_t type_cache_dunder_hits; @@ -99,6 +104,7 @@ typedef struct _gc_stats { typedef struct _uop_stats { uint64_t execution_count; uint64_t miss; + uint64_t pair_count[PYSTATS_MAX_UOP_ID + 1]; } UOpStats; #define _Py_UOP_HIST_SIZE 32 @@ -115,11 +121,18 @@ typedef struct _optimization_stats { uint64_t inner_loop; uint64_t recursive_call; uint64_t low_confidence; - UOpStats opcode[512]; + uint64_t executors_invalidated; + UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t optimizer_attempts; + uint64_t optimizer_successes; + uint64_t optimizer_failure_reason_no_memory; + uint64_t remove_globals_builtins_changed; + uint64_t remove_globals_incorrect_keys; + uint64_t error_in_opcode[PYSTATS_MAX_UOP_ID + 1]; } OptimizationStats; typedef struct _rare_event_stats { @@ -133,6 +146,9 @@ typedef struct _rare_event_stats { uint64_t builtin_dict; /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ uint64_t func_modification; + /* Modifying a dict that is being watched */ + uint64_t watched_dict_modification; + uint64_t watched_globals_modification; } RareEventStats; typedef struct _stats { @@ -151,7 +167,11 @@ PyAPI_DATA(PyStats*) _Py_stats; #ifdef _PY_INTERPRETER # define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) # define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) +# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0) +# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0) #else # define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) # define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) +# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0) +# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0) #endif diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h new file mode 100644 index 00000000000000..5c68110aeedb86 --- /dev/null +++ b/Include/cpython/pytime.h @@ -0,0 +1,27 @@ +// PyTime_t C API: see Doc/c-api/time.rst for the documentation. + +#ifndef Py_LIMITED_API +#ifndef Py_PYTIME_H +#define Py_PYTIME_H +#ifdef __cplusplus +extern "C" { +#endif + +typedef int64_t PyTime_t; +#define PyTime_MIN INT64_MIN +#define PyTime_MAX INT64_MAX + +PyAPI_FUNC(double) PyTime_AsSecondsDouble(PyTime_t t); +PyAPI_FUNC(int) PyTime_Monotonic(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounter(PyTime_t *result); +PyAPI_FUNC(int) PyTime_Time(PyTime_t *result); + +PyAPI_FUNC(int) PyTime_MonotonicRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounterRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_TimeRaw(PyTime_t *result); + +#ifdef __cplusplus +} +#endif +#endif /* Py_PYTIME_H */ +#endif /* Py_LIMITED_API */ diff --git a/Include/cpython/setobject.h b/Include/cpython/setobject.h index 1778c778a05324..89565cb29212fc 100644 --- a/Include/cpython/setobject.h +++ b/Include/cpython/setobject.h @@ -62,6 +62,10 @@ typedef struct { (assert(PyAnySet_Check(so)), _Py_CAST(PySetObject*, so)) static inline Py_ssize_t PySet_GET_SIZE(PyObject *so) { +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PySet_CAST(so)->used)); +#else return _PySet_CAST(so)->used; +#endif } #define PySet_GET_SIZE(so) PySet_GET_SIZE(_PyObject_CAST(so)) diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h deleted file mode 100644 index a3ac07f538a94f..00000000000000 --- a/Include/cpython/sysmodule.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef Py_CPYTHON_SYSMODULE_H -# error "this header file must not be included directly" -#endif - -typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); - -PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); - -typedef struct { - FILE* perf_map; - PyThread_type_lock map_lock; -} PerfMapState; - -PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); -PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( - const void *code_addr, - unsigned int code_size, - const char *entry_name); -PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); -PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); -PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); -PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index d9b54bce83202d..91799137101280 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -444,7 +444,54 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( Py_ssize_t size); -/* --- _PyUnicodeWriter API ----------------------------------------------- */ +/* --- Public PyUnicodeWriter API ----------------------------------------- */ + +typedef struct PyUnicodeWriter PyUnicodeWriter; + +PyAPI_FUNC(PyUnicodeWriter*) PyUnicodeWriter_Create(Py_ssize_t length); +PyAPI_FUNC(void) PyUnicodeWriter_Discard(PyUnicodeWriter *writer); +PyAPI_FUNC(PyObject*) PyUnicodeWriter_Finish(PyUnicodeWriter *writer); + +PyAPI_FUNC(int) PyUnicodeWriter_WriteChar( + PyUnicodeWriter *writer, + Py_UCS4 ch); +PyAPI_FUNC(int) PyUnicodeWriter_WriteUTF8( + PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size); +PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( + PyUnicodeWriter *writer, + const wchar_t *str, + Py_ssize_t size); +PyAPI_FUNC(int) PyUnicodeWriter_WriteUCS4( + PyUnicodeWriter *writer, + Py_UCS4 *str, + Py_ssize_t size); + +PyAPI_FUNC(int) PyUnicodeWriter_WriteStr( + PyUnicodeWriter *writer, + PyObject *obj); +PyAPI_FUNC(int) PyUnicodeWriter_WriteRepr( + PyUnicodeWriter *writer, + PyObject *obj); +PyAPI_FUNC(int) PyUnicodeWriter_WriteSubstring( + PyUnicodeWriter *writer, + PyObject *str, + Py_ssize_t start, + Py_ssize_t end); +PyAPI_FUNC(int) PyUnicodeWriter_Format( + PyUnicodeWriter *writer, + const char *format, + ...); +PyAPI_FUNC(int) PyUnicodeWriter_DecodeUTF8Stateful( + PyUnicodeWriter *writer, + const char *string, /* UTF-8 encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed); /* bytes consumed */ + + +/* --- Private _PyUnicodeWriter API --------------------------------------- */ typedef struct { PyObject *buffer; @@ -466,7 +513,7 @@ typedef struct { /* If readonly is 1, buffer is a shared string (cannot be modified) and size is set to 0. */ unsigned char readonly; -} _PyUnicodeWriter ; +} _PyUnicodeWriter; // Initialize a Unicode writer. // diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 1559e2def61260..9aa1a92c413fe9 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -30,15 +30,25 @@ struct _PyWeakReference { PyWeakReference *wr_prev; PyWeakReference *wr_next; vectorcallfunc vectorcall; + +#ifdef Py_GIL_DISABLED + /* Pointer to the lock used when clearing in free-threaded builds. + * Normally this can be derived from wr_object, but in some cases we need + * to lock after wr_object has been set to Py_None. + */ + PyMutex *weakrefs_lock; +#endif }; +PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); + +#define _PyWeakref_CAST(op) \ + (assert(PyWeakref_Check(op)), _Py_CAST(PyWeakReference*, (op))) + Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { - PyWeakReference *ref; - PyObject *obj; - assert(PyWeakref_Check(ref_obj)); - ref = _Py_CAST(PyWeakReference*, ref_obj); - obj = ref->wr_object; + PyWeakReference *ref = _PyWeakref_CAST(ref_obj); + PyObject *obj = ref->wr_object; // Explanation for the Py_REFCNT() check: when a weakref's target is part // of a long chain of deallocations which triggers the trashcan mechanism, // clearing the weakrefs can be delayed long after the target's refcount diff --git a/Include/critical_section.h b/Include/critical_section.h new file mode 100644 index 00000000000000..3b37615a8b17e2 --- /dev/null +++ b/Include/critical_section.h @@ -0,0 +1,16 @@ +#ifndef Py_CRITICAL_SECTION_H +#define Py_CRITICAL_SECTION_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_CRITICAL_SECTION_H +# include "cpython/critical_section.h" +# undef Py_CPYTHON_CRITICAL_SECTION_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CRITICAL_SECTION_H */ diff --git a/Include/exports.h b/Include/exports.h index ce601216f17156..0c646d5beb6ad6 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -41,11 +41,8 @@ * we may still need to support gcc >= 4, as some Ubuntu LTS and Centos versions * have 4 < gcc < 5. */ - #ifndef __has_attribute - #define __has_attribute(x) 0 // Compatibility with non-clang compilers. - #endif #if (defined(__GNUC__) && (__GNUC__ >= 4)) ||\ - (defined(__clang__) && __has_attribute(visibility)) + (defined(__clang__) && _Py__has_attribute(visibility)) #define Py_IMPORTED_SYMBOL __attribute__ ((visibility ("default"))) #define Py_EXPORTED_SYMBOL __attribute__ ((visibility ("default"))) #define Py_LOCAL_SYMBOL __attribute__ ((visibility ("hidden"))) diff --git a/Include/floatobject.h b/Include/floatobject.h index 999441ac536e1d..4d24a76edd5de1 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -2,7 +2,7 @@ /* Float object interface */ /* -PyFloatObject represents a (double precision) floating point number. +PyFloatObject represents a (double precision) floating-point number. */ #ifndef Py_FLOATOBJECT_H @@ -21,10 +21,10 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_INF(sign) \ do { \ if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_HUGE_VAL); \ + return PyFloat_FromDouble(Py_INFINITY); \ } \ else { \ - return PyFloat_FromDouble(-Py_HUGE_VAL); \ + return PyFloat_FromDouble(-Py_INFINITY); \ } \ } while(0) diff --git a/Include/internal/mimalloc/mimalloc/atomic.h b/Include/internal/mimalloc/mimalloc/atomic.h index eb8478ceed6adf..cdd9c372beafd5 100644 --- a/Include/internal/mimalloc/mimalloc/atomic.h +++ b/Include/internal/mimalloc/mimalloc/atomic.h @@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------------------------------------------- // Atomics // We need to be portable between C, C++, and MSVC. -// We base the primitives on the C/C++ atomics and create a mimimal wrapper for MSVC in C compilation mode. +// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode. // This is why we try to use only `uintptr_t` and `*` as atomic types. // To gain better insight in the range of used atomics, we use explicitly named memory order operations // instead of passing the memory order as a parameter. @@ -23,7 +23,9 @@ terms of the MIT license. A copy of the license can be found in the file #define _Atomic(tp) std::atomic #define mi_atomic(name) std::atomic_##name #define mi_memory_order(name) std::memory_order_##name -#if !defined(ATOMIC_VAR_INIT) || (__cplusplus >= 202002L) // c++20, see issue #571 +#if (__cplusplus >= 202002L) // c++20, see issue #571 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) #define MI_ATOMIC_VAR_INIT(x) x #else #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) @@ -39,7 +41,9 @@ terms of the MIT license. A copy of the license can be found in the file #include #define mi_atomic(name) atomic_##name #define mi_memory_order(name) memory_order_##name -#if !defined(ATOMIC_VAR_INIT) || (__STDC_VERSION__ >= 201710L) // c17, see issue #735 +#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) #define MI_ATOMIC_VAR_INIT(x) x #else #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index 8af841cfdffc01..d97f51b8eefbe5 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -10,12 +10,12 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------------------------- -// This file contains the interal API's of mimalloc and various utility +// This file contains the internal API's of mimalloc and various utility // functions and macros. // -------------------------------------------------------------------------- -#include "mimalloc/types.h" -#include "mimalloc/track.h" +#include "types.h" +#include "track.h" #if (MI_DEBUG>0) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) diff --git a/Include/internal/mimalloc/mimalloc/prim.h b/Include/internal/mimalloc/mimalloc/prim.h index 4b9e4dc4194d77..322ab29e6b41c2 100644 --- a/Include/internal/mimalloc/mimalloc/prim.h +++ b/Include/internal/mimalloc/mimalloc/prim.h @@ -131,7 +131,13 @@ extern bool _mi_process_is_initialized; // has mi_process_init been static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; -#if defined(_WIN32) +#ifdef MI_PRIM_THREAD_ID + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + return MI_PRIM_THREAD_ID(); +} + +#elif defined(_WIN32) #define WIN32_LEAN_AND_MEAN #include @@ -145,9 +151,9 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { // If you test on another platform and it works please send a PR :-) // see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. #elif defined(__GNUC__) && ( \ - (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \ - || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ ) diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index b8cae24507fc5e..354839ba955b36 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -21,7 +21,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include "mimalloc/atomic.h" // _Atomic +#include "atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int @@ -311,7 +311,9 @@ typedef struct mi_page_s { uint32_t slice_offset; // distance from the actual page data slice (0 if a page) uint8_t is_committed : 1; // `true` if the page virtual memory is committed uint8_t is_zero_init : 1; // `true` if the page was initially zero initialized + uint8_t use_qsbr : 1; // delay page freeing using qsbr uint8_t tag : 4; // tag from the owning heap + uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` @@ -335,8 +337,13 @@ typedef struct mi_page_s { struct mi_page_s* next; // next page owned by this thread with the same `block_size` struct mi_page_s* prev; // previous page owned by this thread with the same `block_size` +#ifdef Py_GIL_DISABLED + struct llist_node qsbr_node; + uint64_t qsbr_goal; +#endif + // 64-bit 9 words, 32-bit 12 words, (+2 for secure) - #if MI_INTPTR_SIZE==8 + #if MI_INTPTR_SIZE==8 && !defined(Py_GIL_DISABLED) uintptr_t padding[1]; #endif } mi_page_t; @@ -553,6 +560,8 @@ struct mi_heap_s { mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages uint8_t tag; // custom identifier for this heap + uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory + bool page_use_qsbr; // should freeing pages be delayed using QSBR }; diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index f222d485e0b54b..f5bf1205a82be9 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -657,14 +657,17 @@ struct _type_param { struct { identifier name; expr_ty bound; + expr_ty default_value; } TypeVar; struct { identifier name; + expr_ty default_value; } ParamSpec; struct { identifier name; + expr_ty default_value; } TypeVarTuple; } v; @@ -892,14 +895,15 @@ pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); -type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena - *arena); +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, expr_ty + default_value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +type_param_ty _PyAST_ParamSpec(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +type_param_ty _PyAST_TypeVarTuple(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); PyObject* PyAST_mod2obj(mod_ty t); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index f1b1786264803b..09ae95465495c0 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -184,6 +184,7 @@ struct ast_state { PyObject *conversion; PyObject *ctx; PyObject *decorator_list; + PyObject *default_value; PyObject *defaults; PyObject *elt; PyObject *elts; diff --git a/Include/internal/pycore_atexit.h b/Include/internal/pycore_atexit.h index 4dcda8f517c787..507a5c03cbc792 100644 --- a/Include/internal/pycore_atexit.h +++ b/Include/internal/pycore_atexit.h @@ -54,7 +54,7 @@ struct atexit_state { int callback_len; }; -// Export for '_xxinterpchannels' shared extension +// Export for '_interpchannels' shared extension PyAPI_FUNC(int) _Py_AtExit( PyInterpreterState *interp, atexit_datacallbackfunc func, diff --git a/Include/internal/pycore_audit.h b/Include/internal/pycore_audit.h new file mode 100644 index 00000000000000..2811aaa6236123 --- /dev/null +++ b/Include/internal/pycore_audit.h @@ -0,0 +1,35 @@ +#ifndef Py_INTERNAL_AUDIT_H +#define Py_INTERNAL_AUDIT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* Runtime audit hook state */ + +typedef struct _Py_AuditHookEntry { + struct _Py_AuditHookEntry *next; + Py_AuditHookFunction hookCFunction; + void *userData; +} _Py_AuditHookEntry; + + +extern int _PySys_Audit( + PyThreadState *tstate, + const char *event, + const char *argFormat, + ...); + +// _PySys_ClearAuditHooks() must not be exported: use extern rather than +// PyAPI_FUNC(). We want minimal exposure of this function. +extern void _PySys_ClearAuditHooks(PyThreadState *tstate); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_AUDIT_H */ diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h new file mode 100644 index 00000000000000..20436a68b69677 --- /dev/null +++ b/Include/internal/pycore_backoff.h @@ -0,0 +1,141 @@ + +#ifndef Py_INTERNAL_BACKOFF_H +#define Py_INTERNAL_BACKOFF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include +#include + + +typedef struct { + uint16_t value_and_backoff; +} _Py_BackoffCounter; + + +/* 16-bit countdown counters using exponential backoff. + + These are used by the adaptive specializer to count down until + it is time to specialize an instruction. If specialization fails + the counter is reset using exponential backoff. + + Another use is for the Tier 2 optimizer to decide when to create + a new Tier 2 trace (executor). Again, exponential backoff is used. + + The 16-bit counter is structured as a 12-bit unsigned 'value' + and a 4-bit 'backoff' field. When resetting the counter, the + backoff field is incremented (until it reaches a limit) and the + value is set to a bit mask representing the value 2**backoff - 1. + The maximum backoff is 12 (the number of bits in the value). + + There is an exceptional value which must not be updated, 0xFFFF. +*/ + +#define BACKOFF_BITS 4 +#define MAX_BACKOFF 12 +#define UNREACHABLE_BACKOFF 15 + +static inline bool +is_unreachable_backoff_counter(_Py_BackoffCounter counter) +{ + return counter.value_and_backoff == UNREACHABLE_BACKOFF; +} + +static inline _Py_BackoffCounter +make_backoff_counter(uint16_t value, uint16_t backoff) +{ + assert(backoff <= 15); + assert(value <= 0xFFF); + _Py_BackoffCounter result; + result.value_and_backoff = (value << BACKOFF_BITS) | backoff; + return result; +} + +static inline _Py_BackoffCounter +forge_backoff_counter(uint16_t counter) +{ + _Py_BackoffCounter result; + result.value_and_backoff = counter; + return result; +} + +static inline _Py_BackoffCounter +restart_backoff_counter(_Py_BackoffCounter counter) +{ + assert(!is_unreachable_backoff_counter(counter)); + int backoff = counter.value_and_backoff & 15; + if (backoff < MAX_BACKOFF) { + return make_backoff_counter((1 << (backoff + 1)) - 1, backoff + 1); + } + else { + return make_backoff_counter((1 << MAX_BACKOFF) - 1, MAX_BACKOFF); + } +} + +static inline _Py_BackoffCounter +pause_backoff_counter(_Py_BackoffCounter counter) +{ + _Py_BackoffCounter result; + result.value_and_backoff = counter.value_and_backoff | (1 << BACKOFF_BITS); + return result; +} + +static inline _Py_BackoffCounter +advance_backoff_counter(_Py_BackoffCounter counter) +{ + _Py_BackoffCounter result; + result.value_and_backoff = counter.value_and_backoff - (1 << BACKOFF_BITS); + return result; +} + +static inline bool +backoff_counter_triggers(_Py_BackoffCounter counter) +{ + /* Test whether the value is zero and the backoff is not UNREACHABLE_BACKOFF */ + return counter.value_and_backoff < UNREACHABLE_BACKOFF; +} + +/* Initial JUMP_BACKWARD counter. + * This determines when we create a trace for a loop. +* Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */ +#define JUMP_BACKWARD_INITIAL_VALUE 15 +#define JUMP_BACKWARD_INITIAL_BACKOFF 4 +static inline _Py_BackoffCounter +initial_jump_backoff_counter(void) +{ + return make_backoff_counter(JUMP_BACKWARD_INITIAL_VALUE, + JUMP_BACKWARD_INITIAL_BACKOFF); +} + +/* Initial exit temperature. + * Must be larger than ADAPTIVE_COOLDOWN_VALUE, + * otherwise when a side exit warms up we may construct + * a new trace before the Tier 1 code has properly re-specialized. + * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */ +#define SIDE_EXIT_INITIAL_VALUE 63 +#define SIDE_EXIT_INITIAL_BACKOFF 6 + +static inline _Py_BackoffCounter +initial_temperature_backoff_counter(void) +{ + return make_backoff_counter(SIDE_EXIT_INITIAL_VALUE, + SIDE_EXIT_INITIAL_BACKOFF); +} + +/* Unreachable backoff counter. */ +static inline _Py_BackoffCounter +initial_unreachable_backoff_counter(void) +{ + return make_backoff_counter(0, UNREACHABLE_BACKOFF); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_BACKOFF_H */ diff --git a/Include/internal/pycore_bytes_methods.h b/Include/internal/pycore_bytes_methods.h index 11e8ab20e91367..059dc2599bbd77 100644 --- a/Include/internal/pycore_bytes_methods.h +++ b/Include/internal/pycore_bytes_methods.h @@ -26,14 +26,23 @@ extern void _Py_bytes_title(char *result, const char *s, Py_ssize_t len); extern void _Py_bytes_capitalize(char *result, const char *s, Py_ssize_t len); extern void _Py_bytes_swapcase(char *result, const char *s, Py_ssize_t len); -extern PyObject *_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); extern int _Py_bytes_contains(const char *str, Py_ssize_t len, PyObject *arg); -extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, + PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); +extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, + PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 94d421a9eb742a..300e7f4896a39e 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -23,10 +23,6 @@ extern PyObject* _PyBytes_FromHex( PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, const char **); -/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, - x must be an iterable object. */ -extern PyObject* _PyBytes_Join(PyObject *sep, PyObject *x); - // Substring Search. // diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index c92028a01299e2..49f5c3322de267 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_identifier.h" // _Py_Identifier #include "pycore_pystate.h" // _PyThreadState_GET() /* Suggested size (number of positional arguments) for arrays of PyObject* diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h new file mode 100644 index 00000000000000..27f67d57b2fb79 --- /dev/null +++ b/Include/internal/pycore_cell.h @@ -0,0 +1,48 @@ +#ifndef Py_INTERNAL_CELL_H +#define Py_INTERNAL_CELL_H + +#include "pycore_critical_section.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Sets the cell contents to `value` and return previous contents. Steals a +// reference to `value`. +static inline PyObject * +PyCell_SwapTakeRef(PyCellObject *cell, PyObject *value) +{ + PyObject *old_value; + Py_BEGIN_CRITICAL_SECTION(cell); + old_value = cell->ob_ref; + cell->ob_ref = value; + Py_END_CRITICAL_SECTION(); + return old_value; +} + +static inline void +PyCell_SetTakeRef(PyCellObject *cell, PyObject *value) +{ + PyObject *old_value = PyCell_SwapTakeRef(cell, value); + Py_XDECREF(old_value); +} + +// Gets the cell contents. Returns a new reference. +static inline PyObject * +PyCell_GetRef(PyCellObject *cell) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(cell); + res = Py_XNewRef(cell->ob_ref); + Py_END_CRITICAL_SECTION(); + return res; +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CELL_H */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index b158fc9ff5ebc1..411bbff106dd69 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -42,20 +42,23 @@ PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); extern void _Py_FinishPendingCalls(PyThreadState *tstate); extern void _PyEval_InitState(PyInterpreterState *); -extern void _PyEval_SignalReceived(PyInterpreterState *interp); +extern void _PyEval_SignalReceived(void); // bitwise flags: #define _Py_PENDING_MAINTHREADONLY 1 #define _Py_PENDING_RAWFREE 2 +typedef int _Py_add_pending_call_result; +#define _Py_ADD_PENDING_SUCCESS 0 +#define _Py_ADD_PENDING_FULL -1 + // Export for '_testinternalcapi' shared extension -PyAPI_FUNC(int) _PyEval_AddPendingCall( +PyAPI_FUNC(_Py_add_pending_call_result) _PyEval_AddPendingCall( PyInterpreterState *interp, _Py_pending_call_func func, void *arg, int flags); -extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); #endif @@ -80,9 +83,6 @@ extern void _PyEval_Fini(void); extern PyObject* _PyEval_GetBuiltins(PyThreadState *tstate); -extern PyObject* _PyEval_BuiltinsFromGlobals( - PyThreadState *tstate, - PyObject *globals); // Trampoline API @@ -105,6 +105,7 @@ extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; +extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; #endif static inline PyObject* @@ -128,7 +129,52 @@ extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_AcquireLock(PyThreadState *tstate); -extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *); + +extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *, + int final_release); + +#ifdef Py_GIL_DISABLED +// Returns 0 or 1 if the GIL for the given thread's interpreter is disabled or +// enabled, respectively. +// +// The enabled state of the GIL will not change while one or more threads are +// attached. +static inline int +_PyEval_IsGILEnabled(PyThreadState *tstate) +{ + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + return _Py_atomic_load_int_relaxed(&gil->enabled) != 0; +} + +// Enable or disable the GIL used by the interpreter that owns tstate, which +// must be the current thread. This may affect other interpreters, if the GIL +// is shared. All three functions will be no-ops (and return 0) if the +// interpreter's `enable_gil' config is not _PyConfig_GIL_DEFAULT. +// +// Every call to _PyEval_EnableGILTransient() must be paired with exactly one +// call to either _PyEval_EnableGILPermanent() or +// _PyEval_DisableGIL(). _PyEval_EnableGILPermanent() and _PyEval_DisableGIL() +// must only be called while the GIL is enabled from a call to +// _PyEval_EnableGILTransient(). +// +// _PyEval_EnableGILTransient() returns 1 if it enabled the GIL, or 0 if the +// GIL was already enabled, whether transiently or permanently. The caller will +// hold the GIL upon return. +// +// _PyEval_EnableGILPermanent() returns 1 if it permanently enabled the GIL +// (which must already be enabled), or 0 if it was already permanently +// enabled. Once _PyEval_EnableGILPermanent() has been called once, all +// subsequent calls to any of the three functions will be no-ops. +// +// _PyEval_DisableGIL() returns 1 if it disabled the GIL, or 0 if the GIL was +// kept enabled because of another request, whether transient or permanent. +// +// All three functions must be called by an attached thread (this implies that +// if the GIL is enabled, the current thread must hold it). +extern int _PyEval_EnableGILTransient(PyThreadState *tstate); +extern int _PyEval_EnableGILPermanent(PyThreadState *tstate); +extern int _PyEval_DisableGIL(PyThreadState *state); +#endif extern void _PyEval_DeactivateOpCache(void); @@ -139,12 +185,12 @@ extern void _PyEval_DeactivateOpCache(void); /* With USE_STACKCHECK macro defined, trigger stack checks in _Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */ static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (tstate->c_recursion_remaining-- <= 0 + return (tstate->c_recursion_remaining-- < 0 || (tstate->c_recursion_remaining & 63) == 0); } #else static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return tstate->c_recursion_remaining-- <= 0; + return tstate->c_recursion_remaining-- < 0; } #endif @@ -162,6 +208,11 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); } +static inline void _Py_EnterRecursiveCallTstateUnchecked(PyThreadState *tstate) { + assert(tstate->c_recursion_remaining > 0); + tstate->c_recursion_remaining--; +} + static inline int _Py_EnterRecursiveCall(const char *where) { PyThreadState *tstate = _PyThreadState_GET(); return _Py_EnterRecursiveCallTstate(tstate, where); @@ -178,62 +229,92 @@ static inline void _Py_LeaveRecursiveCall(void) { extern struct _PyInterpreterFrame* _PyEval_GetFrame(void); -extern PyObject* _Py_MakeCoro(PyFunctionObject *func); +PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ -extern int _Py_HandlePending(PyThreadState *tstate); +PyAPI_FUNC(int) _Py_HandlePending(PyThreadState *tstate); extern PyObject * _PyEval_GetFrameLocals(void); -extern const binaryfunc _PyEval_BinaryOps[]; -int _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); -int _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); -int _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); -void _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); -void _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); -void _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); -void _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); -PyObject *_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); -PyObject *_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); -int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); -void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); - - -#define _PY_GIL_DROP_REQUEST_BIT 0 -#define _PY_SIGNALS_PENDING_BIT 1 -#define _PY_CALLS_TO_DO_BIT 2 -#define _PY_ASYNC_EXCEPTION_BIT 3 -#define _PY_GC_SCHEDULED_BIT 4 -#define _PY_EVAL_PLEASE_STOP_BIT 5 -#define _PY_EVAL_EXPLICIT_MERGE_BIT 6 +typedef PyObject *(*conversion_func)(PyObject *); + +PyAPI_DATA(const binaryfunc) _PyEval_BinaryOps[]; +PyAPI_DATA(const conversion_func) _PyEval_ConversionFuncs[]; + +typedef struct _special_method { + PyObject *name; + const char *error; +} _Py_SpecialMethod; + +PyAPI_DATA(const _Py_SpecialMethod) _Py_SpecialMethods[]; +PyAPI_DATA(const size_t) _Py_FunctionAttributeOffsets[]; + +PyAPI_FUNC(int) _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); +PyAPI_FUNC(int) _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); +PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); +PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); +PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); +PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); +PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); +PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); +PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); +PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v, int argcnt, int argcntafter, _PyStackRef *sp); +PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); + +PyAPI_FUNC(void) _PyObjectArray_Free(PyObject **array, PyObject **scratch); + +PyAPI_FUNC(PyObject *) _PyEval_GetANext(PyObject *aiter); +PyAPI_FUNC(void) _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name, _PyStackRef *writeto); +PyAPI_FUNC(PyObject *) _PyEval_GetAwaitable(PyObject *iterable, int oparg); +PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name); +PyAPI_FUNC(int) +_Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args); + +/* Bits that can be set in PyThreadState.eval_breaker */ +#define _PY_GIL_DROP_REQUEST_BIT (1U << 0) +#define _PY_SIGNALS_PENDING_BIT (1U << 1) +#define _PY_CALLS_TO_DO_BIT (1U << 2) +#define _PY_ASYNC_EXCEPTION_BIT (1U << 3) +#define _PY_GC_SCHEDULED_BIT (1U << 4) +#define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) +#define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) +#define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 #define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) static inline void -_Py_set_eval_breaker_bit(PyInterpreterState *interp, uint32_t bit, uint32_t set) +_Py_set_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit) { - assert(set == 0 || set == 1); - uintptr_t to_set = set << bit; - uintptr_t mask = ((uintptr_t)1) << bit; - uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); - if ((old & mask) == to_set) { - return; - } - uintptr_t new; - do { - new = (old & ~mask) | to_set; - } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); + _Py_atomic_or_uintptr(&tstate->eval_breaker, bit); +} + +static inline void +_Py_unset_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit) +{ + _Py_atomic_and_uintptr(&tstate->eval_breaker, ~bit); } -static inline bool -_Py_eval_breaker_bit_is_set(PyInterpreterState *interp, int32_t bit) +static inline int +_Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit) { - return _Py_atomic_load_uintptr_relaxed(&interp->ceval.eval_breaker) & (((uintptr_t)1) << bit); + uintptr_t b = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + return (b & bit) != 0; } +// Free-threaded builds use these functions to set or unset a bit on all +// threads in the given interpreter. +void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); +void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); + +PyAPI_FUNC(PyObject *) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 28738980eb49be..009a1ea41eb985 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -14,28 +14,58 @@ extern "C" { typedef int (*_Py_pending_call_func)(void *); +struct _pending_call { + _Py_pending_call_func func; + void *arg; + int flags; +}; + +#define PENDINGCALLSARRAYSIZE 300 + +#define MAXPENDINGCALLS PENDINGCALLSARRAYSIZE +/* For interpreter-level pending calls, we want to avoid spending too + much time on pending calls in any one thread, so we apply a limit. */ +#if MAXPENDINGCALLS > 100 +# define MAXPENDINGCALLSLOOP 100 +#else +# define MAXPENDINGCALLSLOOP MAXPENDINGCALLS +#endif + +/* We keep the number small to preserve as much compatibility + as possible with earlier versions. */ +#define MAXPENDINGCALLS_MAIN 32 +/* For the main thread, we want to make sure all pending calls are + run at once, for the sake of prompt signal handling. This is + unlikely to cause any problems since there should be very few + pending calls for the main thread. */ +#define MAXPENDINGCALLSLOOP_MAIN 0 + struct _pending_calls { - int busy; + PyThreadState *handling_thread; PyMutex mutex; /* Request for running pending calls. */ - int32_t calls_to_do; -#define NPENDINGCALLS 32 - struct _pending_call { - _Py_pending_call_func func; - void *arg; - int flags; - } calls[NPENDINGCALLS]; + int32_t npending; + /* The maximum allowed number of pending calls. + If the queue fills up to this point then _PyEval_AddPendingCall() + will return _Py_ADD_PENDING_FULL. */ + int32_t max; + /* We don't want a flood of pending calls to interrupt any one thread + for too long, so we keep a limit on the number handled per pass. + A value of 0 means there is no limit (other than the maximum + size of the list of pending calls). */ + int32_t maxloop; + struct _pending_call calls[PENDINGCALLSARRAYSIZE]; int first; - int last; + int next; }; + typedef enum { PERF_STATUS_FAILED = -1, // Perf trampoline is in an invalid state PERF_STATUS_NO_INIT = 0, // Perf trampoline is not initialized PERF_STATUS_OK = 1, // Perf trampoline is ready to be executed } perf_status_t; - #ifdef PY_HAVE_PERF_TRAMPOLINE struct code_arena_st; @@ -45,13 +75,16 @@ struct trampoline_api_st { unsigned int code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; + Py_ssize_t code_padding; }; #endif + struct _ceval_runtime_state { struct { #ifdef PY_HAVE_PERF_TRAMPOLINE perf_status_t status; + int perf_trampoline_type; Py_ssize_t extra_code_index; struct code_arena_st *code_arena; struct trampoline_api_st trampoline_api; @@ -62,9 +95,15 @@ struct _ceval_runtime_state { #endif } perf; /* Pending calls to be made only on the main thread. */ + // The signal machinery falls back on this + // so it must be especially stable and efficient. + // For example, we use a preallocated array + // for the list of pending calls. struct _pending_calls pending_mainthread; + PyMutex sys_trace_profile_mutex; }; + #ifdef PY_HAVE_PERF_TRAMPOLINE # define _PyEval_RUNTIME_PERF_INIT \ { \ @@ -78,13 +117,10 @@ struct _ceval_runtime_state { struct _ceval_state { - /* This single variable consolidates all requests to break out of - * the fast path in the eval loop. - * It is by far the hottest field in this struct and - * should be placed at the beginning. */ - uintptr_t eval_breaker; - /* Avoid false sharing */ - int64_t padding[7]; + /* This variable holds the global instrumentation version. When a thread is + running, this value is overlaid onto PyThreadState.eval_breaker so that + changes in the instrumentation version will trigger the eval breaker. */ + uintptr_t instrumentation_version; int recursion_limit; struct _gil_runtime_state *gil; int own_gil; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index fdd5918228455d..57e0a14bb9b5bd 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -8,6 +8,73 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_stackref.h" // _PyStackRef +#include "pycore_lock.h" // PyMutex +#include "pycore_backoff.h" // _Py_BackoffCounter + + +/* Each instruction in a code object is a fixed-width value, + * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG + * opcode allows for larger values but the current limit is 3 uses + * of EXTENDED_ARG (see Python/compile.c), for a maximum + * 32-bit value. This aligns with the note in Python/compile.c + * (compiler_addop_i_line) indicating that the max oparg value is + * 2**32 - 1, rather than INT_MAX. + */ + +typedef union { + uint16_t cache; + struct { + uint8_t code; + uint8_t arg; + } op; + _Py_BackoffCounter counter; // First cache entry of specializable op +} _Py_CODEUNIT; + +#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) +#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) + + +/* These macros only remain defined for compatibility. */ +#define _Py_OPCODE(word) ((word).op.code) +#define _Py_OPARG(word) ((word).op.arg) + +static inline _Py_CODEUNIT +_py_make_codeunit(uint8_t opcode, uint8_t oparg) +{ + // No designated initialisers because of C++ compat + _Py_CODEUNIT word; + word.op.code = opcode; + word.op.arg = oparg; + return word; +} + +static inline void +_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) +{ + word->op.code = opcode; +} + +#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg)) +#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode)) + + +// We hide some of the newer PyCodeObject fields behind macros. +// This helps with backporting certain changes to 3.12. +#define _PyCode_HAS_EXECUTORS(CODE) \ + (CODE->co_executors != NULL) +#define _PyCode_HAS_INSTRUMENTATION(CODE) \ + (CODE->_co_instrumentation_version > 0) + +struct _py_code_state { + PyMutex mutex; + // Interned constants from code objects. Used by the free-threaded build. + struct _Py_hashtable_t *constants; +}; + +extern PyStatus _PyCode_Init(PyInterpreterState *interp); +extern void _PyCode_Fini(PyInterpreterState *interp); + #define CODE_MAX_WATCHERS 8 /* PEP 659 @@ -22,7 +89,7 @@ extern "C" { #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT)) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t module_keys_version; uint16_t builtin_keys_version; uint16_t index; @@ -31,46 +98,49 @@ typedef struct { #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyBinaryOpCache; #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyUnpackSequenceCache; #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \ CACHE_ENTRIES(_PyUnpackSequenceCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyCompareOpCache; #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyBinarySubscrCache; #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PySuperAttrCache; #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t version[2]; uint16_t index; } _PyAttrCache; typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t type_version[2]; - uint16_t keys_version[2]; + union { + uint16_t keys_version[2]; + uint16_t dict_offset; + }; uint16_t descr[4]; } _PyLoadMethodCache; @@ -81,37 +151,44 @@ typedef struct { #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t func_version[2]; } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) +#define INLINE_CACHE_ENTRIES_CALL_KW CACHE_ENTRIES(_PyCallCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyStoreSubscrCache; #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyForIterCache; #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PySendCache; #define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t version[2]; } _PyToBoolCache; #define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyContainsOpCache; + +#define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache) + // Borrowed references to common callables: struct callable_cache { PyObject *isinstance; @@ -236,38 +313,41 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); +#ifdef Py_GIL_DISABLED +// gh-115999 tracks progress on addressing this. +#define ENABLE_SPECIALIZATION 0 +#else #define ENABLE_SPECIALIZATION 1 +#endif /* Specialization functions */ -extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, +extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, _Py_CODEUNIT *instr, int load_method); -extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, +extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, +extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, +extern void _Py_Specialize_BinarySubscr(_PyStackRef sub, _PyStackRef container, _Py_CODEUNIT *instr); -extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, +extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, +extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); -extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, - int oparg, PyObject **locals); -extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, +extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, + int nargs); +extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, + int oparg, _PyStackRef *locals); +extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, +extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr); - -/* Finalizer function for static codeobjects used in deepfreeze.py */ -extern void _PyStaticCode_Fini(PyCodeObject *co); -/* Function to intern strings of codeobjects and quicken the bytecode */ -extern int _PyStaticCode_Init(PyCodeObject *co); +extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); +extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); #ifdef Py_STATS @@ -286,7 +366,15 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) #define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) #define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) +#define UOP_PAIR_INC(uopcode, lastuop) \ + do { \ + if (lastuop && _Py_stats) { \ + _Py_stats->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ + } \ + lastuop = uopcode; \ + } while (0) #define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) +#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0) #define OPT_HIST(length, name) \ do { \ if (_Py_stats) { \ @@ -296,6 +384,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); } \ } while (0) #define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) +#define OPCODE_DEFERRED_INC(opname) do { if (_Py_stats && opcode == opname) _Py_stats->opcode_stats[opname].specialization.deferred++; } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -312,9 +401,12 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define GC_STAT_ADD(gen, name, n) ((void)0) #define OPT_STAT_INC(name) ((void)0) #define UOP_STAT_INC(opname, name) ((void)0) +#define UOP_PAIR_INC(uopcode, lastuop) ((void)0) #define OPT_UNSUPPORTED_OPCODE(opname) ((void)0) +#define OPT_ERROR_IN_OPCODE(opname) ((void)0) #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) +#define OPCODE_DEFERRED_INC(opname) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. @@ -425,18 +517,14 @@ write_location_entry_start(uint8_t *ptr, int code, int length) /** Counters * The first 16-bit value in each inline cache is a counter. - * When counting misses, the counter is treated as a simple unsigned value. * * When counting executions until the next specialization attempt, * exponential backoff is used to reduce the number of specialization failures. - * The high 12 bits store the counter, the low 4 bits store the backoff exponent. - * On a specialization failure, the backoff exponent is incremented and the - * counter set to (2**backoff - 1). - * Backoff == 6 -> starting counter == 63, backoff == 10 -> starting counter == 1023. + * See pycore_backoff.h for more details. + * On a specialization failure, the backoff counter is restarted. */ -/* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */ -#define ADAPTIVE_BACKOFF_BITS 4 +#include "pycore_backoff.h" // A value of 1 means that we attempt to specialize the *second* time each // instruction is executed. Executing twice is a much better indicator of @@ -454,36 +542,31 @@ write_location_entry_start(uint8_t *ptr, int code, int length) #define ADAPTIVE_COOLDOWN_VALUE 52 #define ADAPTIVE_COOLDOWN_BACKOFF 0 -#define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS) - +// Can't assert this in pycore_backoff.h because of header order dependencies +#if SIDE_EXIT_INITIAL_VALUE <= ADAPTIVE_COOLDOWN_VALUE +# error "Cold exit value should be larger than adaptive cooldown value" +#endif -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_bits(uint16_t value, uint16_t backoff) { - return ((value << ADAPTIVE_BACKOFF_BITS) - | (backoff & ((1 << ADAPTIVE_BACKOFF_BITS) - 1))); + return make_backoff_counter(value, backoff); } -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_warmup(void) { return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE, ADAPTIVE_WARMUP_BACKOFF); } -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_cooldown(void) { return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE, ADAPTIVE_COOLDOWN_BACKOFF); } -static inline uint16_t -adaptive_counter_backoff(uint16_t counter) { - uint16_t backoff = counter & ((1 << ADAPTIVE_BACKOFF_BITS) - 1); - backoff++; - if (backoff > MAX_BACKOFF_VALUE) { - backoff = MAX_BACKOFF_VALUE; - } - uint16_t value = (uint16_t)(1 << backoff) - 1; - return adaptive_counter_bits(value, backoff); +static inline _Py_BackoffCounter +adaptive_counter_backoff(_Py_BackoffCounter counter) { + return restart_backoff_counter(counter); } @@ -509,10 +592,14 @@ adaptive_counter_backoff(uint16_t counter) { extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); -extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset); +extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); +struct _PyCode8 _PyCode_DEF(8); + +PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index a2a7151d50ade7..4400be8b33dee7 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -8,8 +8,30 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex + +/* Initialize codecs-related state for the given interpreter, including + registering the first codec search function. Must be called before any other + PyCodec-related functions, and while only one thread is active. */ +extern PyStatus _PyCodec_InitRegistry(PyInterpreterState *interp); + +/* Finalize codecs-related state for the given interpreter. No PyCodec-related + functions other than PyCodec_Unregister() may be called after this. */ +extern void _PyCodec_Fini(PyInterpreterState *interp); + extern PyObject* _PyCodec_Lookup(const char *encoding); +/* + * Un-register the error handling callback function registered under + * the given 'name'. Only custom error handlers can be un-registered. + * + * - Return -1 and set an exception if 'name' refers to a built-in + * error handling name (e.g., 'strict'), or if an error occurred. + * - Return 0 if no custom error handler can be found for 'name'. + * - Return 1 if the custom error handler was successfully removed. + */ +extern int _PyCodec_UnregisterError(const char *name); + /* Text codec specific encoding and decoding API. Checks the encoding against a list of codecs which do not @@ -48,6 +70,26 @@ extern PyObject* _PyCodecInfo_GetIncrementalEncoder( PyObject *codec_info, const char *errors); +// Per-interpreter state used by codecs.c. +struct codecs_state { + // A list of callable objects used to search for codecs. + PyObject *search_path; + + // A dict mapping codec names to codecs returned from a callable in + // search_path. + PyObject *search_cache; + + // A dict mapping error handling strategies to functions to implement them. + PyObject *error_registry; + +#ifdef Py_GIL_DISABLED + // Used to safely delete a specific item from search_path. + PyMutex search_path_mutex; +#endif + + // Whether or not the rest of the state is initialized. + int initialized; +}; #ifdef __cplusplus } diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index e5870759ba74f1..9f0ca33892a43b 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -8,6 +8,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include + +#include "pycore_ast.h" // mod_ty +#include "pycore_symtable.h" // _Py_SourceLocation +#include "pycore_instruction_sequence.h" + struct _arena; // Type defined in pycore_pyarena.h struct _mod; // Type defined in pycore_ast.h @@ -27,45 +33,12 @@ extern int _PyCompile_AstOptimize( int optimize, struct _arena *arena); -static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; - extern int _PyAST_Optimize( struct _mod *, struct _arena *arena, int optimize, int ff_features); -typedef struct { - int h_label; - int h_startdepth; - int h_preserve_lasti; -} _PyCompile_ExceptHandlerInfo; - -typedef struct { - int i_opcode; - int i_oparg; - _PyCompilerSrcLocation i_loc; - _PyCompile_ExceptHandlerInfo i_except_handler_info; - - /* Used by the assembler */ - int i_target; - int i_offset; -} _PyCompile_Instruction; - -typedef struct { - _PyCompile_Instruction *s_instrs; - int s_allocated; - int s_used; - - int *s_labelmap; /* label id --> instr offset */ - int s_labelmap_size; - int s_next_free_label; /* next free label id */ -} _PyCompile_InstructionSequence; - -int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl); -int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq, - int opcode, int oparg, - _PyCompilerSrcLocation loc); typedef struct { PyObject *u_name; @@ -91,6 +64,115 @@ typedef struct { int u_firstlineno; /* the first lineno of the block */ } _PyCompile_CodeUnitMetadata; +struct _PyCompiler; + +typedef enum { + COMPILE_OP_FAST, + COMPILE_OP_GLOBAL, + COMPILE_OP_DEREF, + COMPILE_OP_NAME, +} _PyCompile_optype; + +/* _PyCompile_FBlockInfo tracks the current frame block. + * + * A frame block is used to handle loops, try/except, and try/finally. + * It's called a frame block to distinguish it from a basic block in the + * compiler IR. + */ + +enum _PyCompile_FBlockType { + COMPILE_FBLOCK_WHILE_LOOP, + COMPILE_FBLOCK_FOR_LOOP, + COMPILE_FBLOCK_TRY_EXCEPT, + COMPILE_FBLOCK_FINALLY_TRY, + COMPILE_FBLOCK_FINALLY_END, + COMPILE_FBLOCK_WITH, + COMPILE_FBLOCK_ASYNC_WITH, + COMPILE_FBLOCK_HANDLER_CLEANUP, + COMPILE_FBLOCK_POP_VALUE, + COMPILE_FBLOCK_EXCEPTION_HANDLER, + COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER, + COMPILE_FBLOCK_ASYNC_COMPREHENSION_GENERATOR, + COMPILE_FBLOCK_STOP_ITERATION, +}; + +typedef struct { + enum _PyCompile_FBlockType fb_type; + _PyJumpTargetLabel fb_block; + _Py_SourceLocation fb_loc; + /* (optional) type-specific exit or cleanup block */ + _PyJumpTargetLabel fb_exit; + /* (optional) additional information required for unwinding */ + void *fb_datum; +} _PyCompile_FBlockInfo; + + +int _PyCompile_PushFBlock(struct _PyCompiler *c, _Py_SourceLocation loc, + enum _PyCompile_FBlockType t, + _PyJumpTargetLabel block_label, + _PyJumpTargetLabel exit, void *datum); +void _PyCompile_PopFBlock(struct _PyCompiler *c, enum _PyCompile_FBlockType t, + _PyJumpTargetLabel block_label); +_PyCompile_FBlockInfo *_PyCompile_TopFBlock(struct _PyCompiler *c); + +int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type, + void *key, int lineno, PyObject *private, + _PyCompile_CodeUnitMetadata *umd); +void _PyCompile_ExitScope(struct _PyCompiler *c); +Py_ssize_t _PyCompile_AddConst(struct _PyCompiler *c, PyObject *o); +_PyInstructionSequence *_PyCompile_InstrSequence(struct _PyCompiler *c); +int _PyCompile_FutureFeatures(struct _PyCompiler *c); +PyObject *_PyCompile_DeferredAnnotations(struct _PyCompiler *c); +PyObject *_PyCompile_Mangle(struct _PyCompiler *c, PyObject *name); +PyObject *_PyCompile_MaybeMangle(struct _PyCompiler *c, PyObject *name); +int _PyCompile_MaybeAddStaticAttributeToClass(struct _PyCompiler *c, expr_ty e); +int _PyCompile_GetRefType(struct _PyCompiler *c, PyObject *name); +int _PyCompile_LookupCellvar(struct _PyCompiler *c, PyObject *name); +int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled, int scope, + _PyCompile_optype *optype, Py_ssize_t *arg); + +int _PyCompile_IsInteractiveTopLevel(struct _PyCompiler *c); +int _PyCompile_IsInInlinedComp(struct _PyCompiler *c); +int _PyCompile_ScopeType(struct _PyCompiler *c); +int _PyCompile_OptimizationLevel(struct _PyCompiler *c); +int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name); +PyObject *_PyCompile_Qualname(struct _PyCompiler *c); +_PyCompile_CodeUnitMetadata *_PyCompile_Metadata(struct _PyCompiler *c); +PyObject *_PyCompile_StaticAttributesAsTuple(struct _PyCompiler *c); + +struct symtable *_PyCompile_Symtable(struct _PyCompiler *c); +PySTEntryObject *_PyCompile_SymtableEntry(struct _PyCompiler *c); + +enum { + COMPILE_SCOPE_MODULE, + COMPILE_SCOPE_CLASS, + COMPILE_SCOPE_FUNCTION, + COMPILE_SCOPE_ASYNC_FUNCTION, + COMPILE_SCOPE_LAMBDA, + COMPILE_SCOPE_COMPREHENSION, + COMPILE_SCOPE_ANNOTATIONS, +}; + + +typedef struct { + PyObject *pushed_locals; + PyObject *temp_symbols; + PyObject *fast_hidden; + _PyJumpTargetLabel cleanup; +} _PyCompile_InlinedComprehensionState; + +int _PyCompile_TweakInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, + PySTEntryObject *entry, + _PyCompile_InlinedComprehensionState *state); +int _PyCompile_RevertInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, + _PyCompile_InlinedComprehensionState *state); +int _PyCompile_AddDeferredAnnotaion(struct _PyCompiler *c, stmt_ty s); + +int _PyCodegen_AddReturnAtEnd(struct _PyCompiler *c, int addNone); +int _PyCodegen_EnterAnonymousScope(struct _PyCompiler* c, mod_ty mod); +int _PyCodegen_Expression(struct _PyCompiler *c, expr_ty e); +int _PyCodegen_Body(struct _PyCompiler *c, _Py_SourceLocation loc, asdl_stmt_seq *stmts, + bool is_interactive); /* Utility for a number of growing arrays used in the compiler */ int _PyCompile_EnsureArrayLargeEnough( @@ -102,17 +184,13 @@ int _PyCompile_EnsureArrayLargeEnough( int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj); +PyCodeObject *_PyCompile_OptimizeAndAssemble(struct _PyCompiler *c, int addNone); -// Export for '_opcode' extention module -PyAPI_FUNC(int) _PyCompile_OpcodeIsValid(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasArg(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasConst(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasName(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasJump(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasFree(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasLocal(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasExc(int opcode); +Py_ssize_t _PyCompile_DictAddObj(PyObject *dict, PyObject *o); +int _PyCompile_Error(struct _PyCompiler *c, _Py_SourceLocation loc, const char *format, ...); +int _PyCompile_Warn(struct _PyCompiler *c, _Py_SourceLocation loc, const char *format, ...); +// Export for '_opcode' extension module PyAPI_FUNC(PyObject*) _PyCompile_GetUnaryIntrinsicName(int index); PyAPI_FUNC(PyObject*) _PyCompile_GetBinaryIntrinsicName(int index); @@ -129,12 +207,6 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( int optimize, int compile_mode); -// Export for '_testinternalcapi' shared extension -PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( - PyObject *instructions, - PyObject *consts, - int nlocals); - // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyCodeObject*) _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index ae5c47f195eb7f..c2b98d15da68fa 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -5,9 +5,9 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_hamt.h" // PyHamtObject +#define CONTEXT_MAX_WATCHERS 8 extern PyTypeObject _PyContextTokenMissing_Type; @@ -35,9 +35,11 @@ struct _pycontextvarobject { PyObject_HEAD PyObject *var_name; PyObject *var_default; +#ifndef Py_GIL_DISABLED PyObject *var_cached; uint64_t var_cached_tsid; uint64_t var_cached_tsver; +#endif Py_hash_t var_hash; }; diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 38ed8cd69804ba..78cd0d54972660 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -13,102 +13,48 @@ extern "C" { #endif -// Implementation of Python critical sections -// -// Conceptually, critical sections are a deadlock avoidance layer on top of -// per-object locks. These helpers, in combination with those locks, replace -// our usage of the global interpreter lock to provide thread-safety for -// otherwise thread-unsafe objects, such as dict. -// -// NOTE: These APIs are no-ops in non-free-threaded builds. -// -// Straightforward per-object locking could introduce deadlocks that were not -// present when running with the GIL. Threads may hold locks for multiple -// objects simultaneously because Python operations can nest. If threads were -// to acquire the same locks in different orders, they would deadlock. -// -// One way to avoid deadlocks is to allow threads to hold only the lock (or -// locks) for a single operation at a time (typically a single lock, but some -// operations involve two locks). When a thread begins a nested operation it -// could suspend the locks for any outer operation: before beginning the nested -// operation, the locks for the outer operation are released and when the -// nested operation completes, the locks for the outer operation are -// reacquired. -// -// To improve performance, this API uses a variation of the above scheme. -// Instead of immediately suspending locks any time a nested operation begins, -// locks are only suspended if the thread would block. This reduces the number -// of lock acquisitions and releases for nested operations, while still -// avoiding deadlocks. -// -// Additionally, the locks for any active operation are suspended around -// other potentially blocking operations, such as I/O. This is because the -// interaction between locks and blocking operations can lead to deadlocks in -// the same way as the interaction between multiple locks. -// -// Each thread's critical sections and their corresponding locks are tracked in -// a stack in `PyThreadState.critical_section`. When a thread calls -// `_PyThreadState_Detach()`, such as before a blocking I/O operation or when -// waiting to acquire a lock, the thread suspends all of its active critical -// sections, temporarily releasing the associated locks. When the thread calls -// `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) -// critical section by reacquiring the associated lock or locks. See -// `_PyCriticalSection_Resume()`. -// -// NOTE: Only the top-most critical section is guaranteed to be active. -// Operations that need to lock two objects at once must use -// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections -// to lock more than one object at once, because the inner critical section -// may suspend the outer critical sections. This API does not provide a way -// to lock more than two objects at once (though it could be added later -// if actually needed). -// -// NOTE: Critical sections implicitly behave like reentrant locks because -// attempting to acquire the same lock will suspend any outer (earlier) -// critical sections. However, they are less efficient for this use case than -// purposefully designed reentrant locks. -// -// Example usage: -// Py_BEGIN_CRITICAL_SECTION(op); -// ... -// Py_END_CRITICAL_SECTION(); -// -// To lock two objects at once: -// Py_BEGIN_CRITICAL_SECTION2(op1, op2); -// ... -// Py_END_CRITICAL_SECTION2(); - - // Tagged pointers to critical sections use the two least significant bits to // mark if the pointed-to critical section is inactive and whether it is a -// _PyCriticalSection2 object. +// PyCriticalSection2 object. #define _Py_CRITICAL_SECTION_INACTIVE 0x1 #define _Py_CRITICAL_SECTION_TWO_MUTEXES 0x2 #define _Py_CRITICAL_SECTION_MASK 0x3 #ifdef Py_GIL_DISABLED -# define Py_BEGIN_CRITICAL_SECTION(op) \ +# define Py_BEGIN_CRITICAL_SECTION_MUT(mutex) \ { \ - _PyCriticalSection _cs; \ - _PyCriticalSection_Begin(&_cs, &_PyObject_CAST(op)->ob_mutex) - -# define Py_END_CRITICAL_SECTION() \ - _PyCriticalSection_End(&_cs); \ - } + PyCriticalSection _py_cs; \ + _PyCriticalSection_BeginMutex(&_py_cs, mutex) -# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ +# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + _PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + +// Specialized version of critical section locking to safely use +// PySequence_Fast APIs without the GIL. For performance, the argument *to* +// PySequence_Fast() is provided to the macro, not the *result* of +// PySequence_Fast(), which would require an extra test to determine if the +// lock must be acquired. +# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) \ { \ - _PyCriticalSection2 _cs2; \ - _PyCriticalSection2_Begin(&_cs2, &_PyObject_CAST(a)->ob_mutex, &_PyObject_CAST(b)->ob_mutex) + PyObject *_orig_seq = _PyObject_CAST(original); \ + const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \ + PyCriticalSection _cs; \ + if (_should_lock_cs) { \ + _PyCriticalSection_Begin(&_cs, _orig_seq); \ + } -# define Py_END_CRITICAL_SECTION2() \ - _PyCriticalSection2_End(&_cs2); \ +# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \ + if (_should_lock_cs) { \ + PyCriticalSection_End(&_cs); \ + } \ } // Asserts that the mutex is locked. The mutex must be held by the // top-most critical section otherwise there's the possibility // that the mutex would be swalled out in some code paths. -#define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \ +#define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \ _PyCriticalSection_AssertHeld(mutex) // Asserts that the mutex for the given object is locked. The mutex must @@ -116,70 +62,57 @@ extern "C" { // possibility that the mutex would be swalled out in some code paths. #ifdef Py_DEBUG -#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ if (Py_REFCNT(op) != 1) { \ _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyObject_CAST(op)->ob_mutex); \ } #else /* Py_DEBUG */ -#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* Py_DEBUG */ #else /* !Py_GIL_DISABLED */ // The critical section APIs are no-ops with the GIL. -# define Py_BEGIN_CRITICAL_SECTION(op) -# define Py_END_CRITICAL_SECTION() -# define Py_BEGIN_CRITICAL_SECTION2(a, b) -# define Py_END_CRITICAL_SECTION2() +# define Py_BEGIN_CRITICAL_SECTION_MUT(mut) { +# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) { +# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) { +# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() } # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* !Py_GIL_DISABLED */ -typedef struct { - // Tagged pointer to an outer active critical section (or 0). - // The two least-significant-bits indicate whether the pointed-to critical - // section is inactive and whether it is a _PyCriticalSection2 object. - uintptr_t prev; - - // Mutex used to protect critical section - PyMutex *mutex; -} _PyCriticalSection; - -// A critical section protected by two mutexes. Use -// _PyCriticalSection2_Begin and _PyCriticalSection2_End. -typedef struct { - _PyCriticalSection base; - - PyMutex *mutex2; -} _PyCriticalSection2; - -static inline int -_PyCriticalSection_IsActive(uintptr_t tag) -{ - return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0; -} - // Resumes the top-most critical section. PyAPI_FUNC(void) _PyCriticalSection_Resume(PyThreadState *tstate); // (private) slow path for locking the mutex PyAPI_FUNC(void) -_PyCriticalSection_BeginSlow(_PyCriticalSection *c, PyMutex *m); +_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m); PyAPI_FUNC(void) -_PyCriticalSection2_BeginSlow(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, +_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, int is_m1_locked); +PyAPI_FUNC(void) +_PyCriticalSection_SuspendAll(PyThreadState *tstate); + +#ifdef Py_GIL_DISABLED + +static inline int +_PyCriticalSection_IsActive(uintptr_t tag) +{ + return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0; +} + static inline void -_PyCriticalSection_Begin(_PyCriticalSection *c, PyMutex *m) +_PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) { - if (PyMutex_LockFast(&m->v)) { + if (PyMutex_LockFast(&m->_bits)) { PyThreadState *tstate = _PyThreadState_GET(); - c->mutex = m; - c->prev = tstate->critical_section; + c->_cs_mutex = m; + c->_cs_prev = tstate->critical_section; tstate->critical_section = (uintptr_t)c; } else { @@ -187,14 +120,21 @@ _PyCriticalSection_Begin(_PyCriticalSection *c, PyMutex *m) } } +static inline void +_PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) +{ + _PyCriticalSection_BeginMutex(c, &op->ob_mutex); +} +#define PyCriticalSection_Begin _PyCriticalSection_Begin + // Removes the top-most critical section from the thread's stack of critical // sections. If the new top-most critical section is inactive, then it is // resumed. static inline void -_PyCriticalSection_Pop(_PyCriticalSection *c) +_PyCriticalSection_Pop(PyCriticalSection *c) { PyThreadState *tstate = _PyThreadState_GET(); - uintptr_t prev = c->prev; + uintptr_t prev = c->_cs_prev; tstate->critical_section = prev; if ((prev & _Py_CRITICAL_SECTION_INACTIVE) != 0) { @@ -203,20 +143,21 @@ _PyCriticalSection_Pop(_PyCriticalSection *c) } static inline void -_PyCriticalSection_End(_PyCriticalSection *c) +_PyCriticalSection_End(PyCriticalSection *c) { - PyMutex_Unlock(c->mutex); + PyMutex_Unlock(c->_cs_mutex); _PyCriticalSection_Pop(c); } +#define PyCriticalSection_End _PyCriticalSection_End static inline void -_PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +_PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) { if (m1 == m2) { // If the two mutex arguments are the same, treat this as a critical // section with a single mutex. - c->mutex2 = NULL; - _PyCriticalSection_Begin(&c->base, m1); + c->_cs_mutex2 = NULL; + _PyCriticalSection_BeginMutex(&c->_cs_base, m1); return; } @@ -229,12 +170,12 @@ _PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) m2 = tmp; } - if (PyMutex_LockFast(&m1->v)) { - if (PyMutex_LockFast(&m2->v)) { + if (PyMutex_LockFast(&m1->_bits)) { + if (PyMutex_LockFast(&m2->_bits)) { PyThreadState *tstate = _PyThreadState_GET(); - c->base.mutex = m1; - c->mutex2 = m2; - c->base.prev = tstate->critical_section; + c->_cs_base._cs_mutex = m1; + c->_cs_mutex2 = m2; + c->_cs_base._cs_prev = tstate->critical_section; uintptr_t p = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES; tstate->critical_section = p; @@ -249,38 +190,42 @@ _PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) } static inline void -_PyCriticalSection2_End(_PyCriticalSection2 *c) +_PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) { - if (c->mutex2) { - PyMutex_Unlock(c->mutex2); - } - PyMutex_Unlock(c->base.mutex); - _PyCriticalSection_Pop(&c->base); + _PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex); } +#define PyCriticalSection2_Begin _PyCriticalSection2_Begin -PyAPI_FUNC(void) -_PyCriticalSection_SuspendAll(PyThreadState *tstate); - -#ifdef Py_GIL_DISABLED +static inline void +_PyCriticalSection2_End(PyCriticalSection2 *c) +{ + if (c->_cs_mutex2) { + PyMutex_Unlock(c->_cs_mutex2); + } + PyMutex_Unlock(c->_cs_base._cs_mutex); + _PyCriticalSection_Pop(&c->_cs_base); +} +#define PyCriticalSection2_End _PyCriticalSection2_End static inline void -_PyCriticalSection_AssertHeld(PyMutex *mutex) { +_PyCriticalSection_AssertHeld(PyMutex *mutex) +{ #ifdef Py_DEBUG PyThreadState *tstate = _PyThreadState_GET(); uintptr_t prev = tstate->critical_section; if (prev & _Py_CRITICAL_SECTION_TWO_MUTEXES) { - _PyCriticalSection2 *cs = (_PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK); - assert(cs != NULL && (cs->base.mutex == mutex || cs->mutex2 == mutex)); + PyCriticalSection2 *cs = (PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && (cs->_cs_base._cs_mutex == mutex || cs->_cs_mutex2 == mutex)); } else { - _PyCriticalSection *cs = (_PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK); - assert(cs != NULL && cs->mutex == mutex); + PyCriticalSection *cs = (PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && cs->_cs_mutex == mutex); } #endif } -#endif +#endif /* Py_GIL_DISABLED */ #ifdef __cplusplus } diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index d6e297a7e8e6db..2dd165eae74850 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -87,6 +87,11 @@ struct _xid { PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); +#define _PyCrossInterpreterData_DATA(DATA) ((DATA)->data) +#define _PyCrossInterpreterData_OBJ(DATA) ((DATA)->obj) +#define _PyCrossInterpreterData_INTERPID(DATA) ((DATA)->interpid) +// Users should not need getters for "new_object" or "free". + /* defining cross-interpreter data */ @@ -101,6 +106,25 @@ PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( PyInterpreterState *, _PyCrossInterpreterData *); +// Normally the Init* functions are sufficient. The only time +// additional initialization might be needed is to set the "free" func, +// though that should be infrequent. +#define _PyCrossInterpreterData_SET_FREE(DATA, FUNC) \ + do { \ + (DATA)->free = (FUNC); \ + } while (0) +// Additionally, some shareable types are essentially light wrappers +// around other shareable types. The crossinterpdatafunc of the wrapper +// can often be implemented by calling the wrapped object's +// crossinterpdatafunc and then changing the "new_object" function. +// We have _PyCrossInterpreterData_SET_NEW_OBJECT() here for that, +// but might be better to have a function like +// _PyCrossInterpreterData_AdaptToWrapper() instead. +#define _PyCrossInterpreterData_SET_NEW_OBJECT(DATA, FUNC) \ + do { \ + (DATA)->new_object = (FUNC); \ + } while (0) + /* using cross-interpreter data */ @@ -170,6 +194,8 @@ extern void _PyXI_Fini(PyInterpreterState *interp); extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp); extern void _PyXI_FiniTypes(PyInterpreterState *interp); +#define _PyInterpreterState_GetXIState(interp) (&(interp)->xi) + /***************************/ /* short-term data sharing */ @@ -191,6 +217,11 @@ typedef struct _excinfo { const char *errdisplay; } _PyXI_excinfo; +PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc); +PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info); +PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info); +PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info); + typedef enum error_code { _PyXI_ERR_NO_ERROR = 0, @@ -287,6 +318,22 @@ PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); +/*************/ +/* other API */ +/*************/ + +// Export for _testinternalcapi shared extension +PyAPI_FUNC(PyInterpreterState *) _PyXI_NewInterpreter( + PyInterpreterConfig *config, + long *maybe_whence, + PyThreadState **p_tstate, + PyThreadState **p_save_tstate); +PyAPI_FUNC(void) _PyXI_EndInterpreter( + PyInterpreterState *interp, + PyThreadState *tstate, + PyThreadState **p_save_tstate); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h new file mode 100644 index 00000000000000..184f4b9360b6d3 --- /dev/null +++ b/Include/internal/pycore_debug_offsets.h @@ -0,0 +1,269 @@ +#ifndef Py_INTERNAL_DEBUG_OFFSETS_H +#define Py_INTERNAL_DEBUG_OFFSETS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#define _Py_Debug_Cookie "xdebugpy" + +#ifdef Py_GIL_DISABLED +# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) +# define _Py_Debug_Free_Threaded 1 +#else +# define _Py_Debug_gilruntimestate_enabled 0 +# define _Py_Debug_Free_Threaded 0 +#endif + + +typedef struct _Py_DebugOffsets { + char cookie[8]; + uint64_t version; + uint64_t free_threaded; + // Runtime state offset; + struct _runtime_state { + uint64_t size; + uint64_t finalizing; + uint64_t interpreters_head; + } runtime_state; + + // Interpreter state offset; + struct _interpreter_state { + uint64_t size; + uint64_t id; + uint64_t next; + uint64_t threads_head; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state; + uint64_t gil_runtime_state_enabled; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; + } interpreter_state; + + // Thread state offset; + struct _thread_state{ + uint64_t size; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; + uint64_t datastack_chunk; + uint64_t status; + } thread_state; + + // InterpreterFrame offset; + struct _interpreter_frame { + uint64_t size; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; + } interpreter_frame; + + // Code object offset; + struct _code_object { + uint64_t size; + uint64_t filename; + uint64_t name; + uint64_t qualname; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; + } code_object; + + // PyObject offset; + struct _pyobject { + uint64_t size; + uint64_t ob_type; + } pyobject; + + // PyTypeObject object offset; + struct _type_object { + uint64_t size; + uint64_t tp_name; + uint64_t tp_repr; + uint64_t tp_flags; + } type_object; + + // PyTuple object offset; + struct _tuple_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } tuple_object; + + // PyList object offset; + struct _list_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } list_object; + + // PyDict object offset; + struct _dict_object { + uint64_t size; + uint64_t ma_keys; + uint64_t ma_values; + } dict_object; + + // PyFloat object offset; + struct _float_object { + uint64_t size; + uint64_t ob_fval; + } float_object; + + // PyLong object offset; + struct _long_object { + uint64_t size; + uint64_t lv_tag; + uint64_t ob_digit; + } long_object; + + // PyBytes object offset; + struct _bytes_object { + uint64_t size; + uint64_t ob_size; + uint64_t ob_sval; + } bytes_object; + + // Unicode object offset; + struct _unicode_object { + uint64_t size; + uint64_t state; + uint64_t length; + uint64_t asciiobject_size; + } unicode_object; + + // GC runtime state offset; + struct _gc { + uint64_t size; + uint64_t collecting; + } gc; +} _Py_DebugOffsets; + + +#define _Py_DebugOffsets_INIT(debug_cookie) { \ + .cookie = debug_cookie, \ + .version = PY_VERSION_HEX, \ + .free_threaded = _Py_Debug_Free_Threaded, \ + .runtime_state = { \ + .size = sizeof(_PyRuntimeState), \ + .finalizing = offsetof(_PyRuntimeState, _finalizing), \ + .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ + }, \ + .interpreter_state = { \ + .size = sizeof(PyInterpreterState), \ + .id = offsetof(PyInterpreterState, id), \ + .next = offsetof(PyInterpreterState, next), \ + .threads_head = offsetof(PyInterpreterState, threads.head), \ + .gc = offsetof(PyInterpreterState, gc), \ + .imports_modules = offsetof(PyInterpreterState, imports.modules), \ + .sysdict = offsetof(PyInterpreterState, sysdict), \ + .builtins = offsetof(PyInterpreterState, builtins), \ + .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + .gil_runtime_state = offsetof(PyInterpreterState, _gil), \ + .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ + .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ + .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ + }, \ + .thread_state = { \ + .size = sizeof(PyThreadState), \ + .prev = offsetof(PyThreadState, prev), \ + .next = offsetof(PyThreadState, next), \ + .interp = offsetof(PyThreadState, interp), \ + .current_frame = offsetof(PyThreadState, current_frame), \ + .thread_id = offsetof(PyThreadState, thread_id), \ + .native_thread_id = offsetof(PyThreadState, native_thread_id), \ + .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ + .status = offsetof(PyThreadState, _status), \ + }, \ + .interpreter_frame = { \ + .size = sizeof(_PyInterpreterFrame), \ + .previous = offsetof(_PyInterpreterFrame, previous), \ + .executable = offsetof(_PyInterpreterFrame, f_executable), \ + .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ + .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ + .owner = offsetof(_PyInterpreterFrame, owner), \ + }, \ + .code_object = { \ + .size = sizeof(PyCodeObject), \ + .filename = offsetof(PyCodeObject, co_filename), \ + .name = offsetof(PyCodeObject, co_name), \ + .qualname = offsetof(PyCodeObject, co_qualname), \ + .linetable = offsetof(PyCodeObject, co_linetable), \ + .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ + .argcount = offsetof(PyCodeObject, co_argcount), \ + .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ + .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ + .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + }, \ + .pyobject = { \ + .size = sizeof(PyObject), \ + .ob_type = offsetof(PyObject, ob_type), \ + }, \ + .type_object = { \ + .size = sizeof(PyTypeObject), \ + .tp_name = offsetof(PyTypeObject, tp_name), \ + .tp_repr = offsetof(PyTypeObject, tp_repr), \ + .tp_flags = offsetof(PyTypeObject, tp_flags), \ + }, \ + .tuple_object = { \ + .size = sizeof(PyTupleObject), \ + .ob_item = offsetof(PyTupleObject, ob_item), \ + .ob_size = offsetof(PyTupleObject, ob_base.ob_size), \ + }, \ + .list_object = { \ + .size = sizeof(PyListObject), \ + .ob_item = offsetof(PyListObject, ob_item), \ + .ob_size = offsetof(PyListObject, ob_base.ob_size), \ + }, \ + .dict_object = { \ + .size = sizeof(PyDictObject), \ + .ma_keys = offsetof(PyDictObject, ma_keys), \ + .ma_values = offsetof(PyDictObject, ma_values), \ + }, \ + .float_object = { \ + .size = sizeof(PyFloatObject), \ + .ob_fval = offsetof(PyFloatObject, ob_fval), \ + }, \ + .long_object = { \ + .size = sizeof(PyLongObject), \ + .lv_tag = offsetof(PyLongObject, long_value.lv_tag), \ + .ob_digit = offsetof(PyLongObject, long_value.ob_digit), \ + }, \ + .bytes_object = { \ + .size = sizeof(PyBytesObject), \ + .ob_size = offsetof(PyBytesObject, ob_base.ob_size), \ + .ob_sval = offsetof(PyBytesObject, ob_sval), \ + }, \ + .unicode_object = { \ + .size = sizeof(PyUnicodeObject), \ + .state = offsetof(PyUnicodeObject, _base._base.state), \ + .length = offsetof(PyUnicodeObject, _base._base.length), \ + .asciiobject_size = sizeof(PyASCIIObject), \ + }, \ + .gc = { \ + .size = sizeof(struct _gc_runtime_state), \ + .collecting = offsetof(struct _gc_runtime_state, collecting), \ + }, \ +} + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_DEBUG_OFFSETS_H */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 233da058f464d1..c5399ad8e0497f 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -1,4 +1,3 @@ - #ifndef Py_INTERNAL_DICT_H #define Py_INTERNAL_DICT_H #ifdef __cplusplus @@ -9,15 +8,19 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState -#include "pycore_identifier.h" // _Py_Identifier -#include "pycore_object.h" // PyDictOrValues +#include "pycore_object.h" // PyManagedDictPointer +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE +#include "pycore_stackref.h" // _PyStackRef // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); -extern int _PyDict_DelItemIf(PyObject *mp, PyObject *key, - int (*predicate)(PyObject *value)); +// Delete an item from a dict if a predicate is true +// Returns -1 on error, 1 if the item was deleted, 0 otherwise +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key, + int (*predicate)(PyObject *value, void *arg), + void *arg); // "KnownHash" variants // Export for '_asyncio' shared extension @@ -52,7 +55,7 @@ PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); of a key wins, if override is 2, a KeyError with conflicting key as argument is raised. */ -extern int _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); +PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); extern void _PyDict_DebugMallocStats(FILE *out); @@ -67,12 +70,6 @@ typedef struct { extern PyObject* _PyDictView_New(PyObject *, PyTypeObject *); extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other); - -/* runtime lifecycle */ - -extern void _PyDict_Fini(PyInterpreterState *state); - - /* other API */ typedef struct { @@ -87,7 +84,7 @@ typedef struct { PyObject *me_value; /* This field is only meaningful for combined tables */ } PyDictUnicodeEntry; -extern PyDictKeysObject *_PyDict_NewKeysForClass(void); +extern PyDictKeysObject *_PyDict_NewKeysForClass(PyHeapTypeObject *); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); /* Gets a version number unique to the current state of the keys of dict, if possible. @@ -103,14 +100,28 @@ extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); * -1 when no entry found, -3 when compare raises error. */ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr); extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); -extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); +PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); +PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); + +// Loads the __builtins__ object from the globals dict. Returns a new reference. +extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); /* Consumes references to key and value */ -extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); -extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); +PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); +extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, + PyObject *value, Py_hash_t hash); +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); +extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_Pop_KnownHash( PyDictObject *dict, @@ -142,6 +153,11 @@ struct _dictkeysobject { /* Kind of keys */ uint8_t dk_kind; +#ifdef Py_GIL_DISABLED + /* Lock used to protect shared keys */ + PyMutex dk_mutex; +#endif + /* Version number -- Reset to 0 by any modification to keys */ uint32_t dk_version; @@ -151,6 +167,7 @@ struct _dictkeysobject { /* Number of used entries in dk_entries. */ Py_ssize_t dk_nentries; + /* Actual hash table of dk_size entries. It holds indices in dk_entries, or DKIX_EMPTY(-1) or DKIX_DUMMY(-2). @@ -167,7 +184,7 @@ struct _dictkeysobject { char dk_indices[]; /* char is required to avoid strict aliasing. */ /* "PyDictKeyEntry or PyDictUnicodeEntry dk_entries[USABLE_FRACTION(DK_SIZE(dk))];" array follows: - see the DK_ENTRIES() macro */ + see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */ }; /* This must be no more than 250, for the prefix size to fit in one byte. */ @@ -181,6 +198,10 @@ struct _dictkeysobject { * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order. */ struct _dictvalues { + uint8_t capacity; + uint8_t size; + uint8_t embedded; + uint8_t valid; PyObject *values[1]; }; @@ -196,6 +217,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) { size_t index = (size_t)1 << dk->dk_log2_index_bytes; return (&indices[index]); } + static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) { assert(dk->dk_kind == DICT_KEYS_GENERAL); return (PyDictKeyEntry*)_DK_ENTRIES(dk); @@ -209,24 +231,19 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_VERSION_INCREMENT (1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) +#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) +#define DICT_UNIQUE_ID_SHIFT (32) +#define DICT_UNIQUE_ID_MAX ((UINT64_C(1) << (64 - DICT_UNIQUE_ID_SHIFT)) - 1) -#ifdef Py_GIL_DISABLED -#define DICT_NEXT_VERSION(INTERP) \ - (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) -#else -#define DICT_NEXT_VERSION(INTERP) \ - ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) -#endif - -void +PyAPI_FUNC(void) _PyDict_SendEvent(int watcher_bits, PyDict_WatchEvent event, PyDictObject *mp, PyObject *key, PyObject *value); -static inline uint64_t +static inline void _PyDict_NotifyEvent(PyInterpreterState *interp, PyDict_WatchEvent event, PyDictObject *mp, @@ -234,33 +251,127 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; + int watcher_bits = mp->_ma_watcher_tag & DICT_WATCHER_MASK; if (watcher_bits) { + RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); - return DICT_NEXT_VERSION(interp) | watcher_bits; } - return DICT_NEXT_VERSION(interp); } -extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); -extern bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); -extern PyObject *_PyDict_FromItems( +extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj); + +PyAPI_FUNC(PyObject *)_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, Py_ssize_t length); +static inline uint8_t * +get_insertion_order_array(PyDictValues *values) +{ + return (uint8_t *)&values->values[values->capacity]; +} + static inline void _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) { assert(ix < SHARED_KEYS_MAX_SIZE); - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; - assert(size+2 < ((uint8_t *)values)[-1]); - size++; - size_ptr[-size] = (uint8_t)ix; - *size_ptr = size; + int size = values->size; + uint8_t *array = get_insertion_order_array(values); + assert(size < values->capacity); + assert(((uint8_t)ix) == ix); + array[size] = (uint8_t)ix; + values->size = size+1; +} + +static inline size_t +shared_keys_usable_size(PyDictKeysObject *keys) +{ + // dk_usable will decrease for each instance that is created and each + // value that is added. dk_nentries will increase for each value that + // is added. We want to always return the right value or larger. + // We therefore increase dk_nentries first and we decrease dk_usable + // second, and conversely here we read dk_usable first and dk_entries + // second (to avoid the case where we read entries before the increment + // and read usable after the decrement) + Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable); + Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries); + return dk_nentries + dk_usable; +} + +static inline size_t +_PyInlineValuesSize(PyTypeObject *tp) +{ + PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys; + assert(keys != NULL); + size_t size = shared_keys_usable_size(keys); + size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *)); + assert(prefix_size < 256); + return prefix_size + (size + 1) * sizeof(PyObject *); } +int +_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj); + +// Enables per-thread ref counting on this dict in the free threading build +extern void _PyDict_EnablePerThreadRefcounting(PyObject *op); + +PyDictObject *_PyObject_MaterializeManagedDict_LockHeld(PyObject *); + +// See `_Py_INCREF_TYPE()` in pycore_object.h +#ifndef Py_GIL_DISABLED +# define _Py_INCREF_DICT Py_INCREF +# define _Py_DECREF_DICT Py_DECREF +# define _Py_INCREF_BUILTINS Py_INCREF +# define _Py_DECREF_BUILTINS Py_DECREF +#else +static inline Py_ssize_t +_PyDict_UniqueId(PyDictObject *mp) +{ + // Offset by one so that _ma_watcher_tag=0 represents an unassigned id + return (Py_ssize_t)(mp->_ma_watcher_tag >> DICT_UNIQUE_ID_SHIFT) - 1; +} + +static inline void +_Py_INCREF_DICT(PyObject *op) +{ + assert(PyDict_Check(op)); + Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); + _Py_THREAD_INCREF_OBJECT(op, id); +} + +static inline void +_Py_DECREF_DICT(PyObject *op) +{ + assert(PyDict_Check(op)); + Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); + _Py_THREAD_DECREF_OBJECT(op, id); +} + +// Like `_Py_INCREF_DICT`, but also handles non-dict objects because builtins +// may not be a dict. +static inline void +_Py_INCREF_BUILTINS(PyObject *op) +{ + if (PyDict_CheckExact(op)) { + _Py_INCREF_DICT(op); + } + else { + Py_INCREF(op); + } +} + +static inline void +_Py_DECREF_BUILTINS(PyObject *op) +{ + if (PyDict_CheckExact(op)) { + _Py_DECREF_DICT(op); + } + else { + Py_DECREF(op); + } +} +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 1a44755c7a01a3..11932b8d1e1ab6 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -12,10 +12,6 @@ extern "C" { #define DICT_WATCHED_MUTATION_BITS 4 struct _Py_dict_state { - /*Global counter used to set ma_version_tag field of dictionary. - * It is incremented each time that a dictionary is created and each - * time that a dictionary is modified. */ - uint64_t global_version; uint32_t next_keys_version; PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index c5cfdf4ce8f823..e4222c5267d6be 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -11,8 +11,6 @@ extern "C" { #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR -#if _PY_SHORT_FLOAT_REPR == 1 - typedef uint32_t ULong; struct @@ -22,15 +20,15 @@ Bigint { ULong x[1]; }; -#ifdef Py_USING_MEMORY_DEBUGGER +#if defined(Py_USING_MEMORY_DEBUGGER) || _PY_SHORT_FLOAT_REPR == 0 struct _dtoa_state { int _not_used; }; -#define _dtoa_interp_state_INIT(INTERP) \ +#define _dtoa_state_INIT(INTERP) \ {0} -#else // !Py_USING_MEMORY_DEBUGGER +#else // !Py_USING_MEMORY_DEBUGGER && _PY_SHORT_FLOAT_REPR != 0 /* The size of the Bigint freelist */ #define Bigint_Kmax 7 @@ -66,8 +64,6 @@ extern char* _Py_dg_dtoa(double d, int mode, int ndigits, int *decpt, int *sign, char **rve); extern void _Py_dg_freedtoa(char *s); -#endif // _PY_SHORT_FLOAT_REPR == 1 - extern PyStatus _PyDtoa_Init(PyInterpreterState *interp); extern void _PyDtoa_Fini(PyInterpreterState *interp); diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 5c55282fa39e6f..13f86b01bbfe8f 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -290,6 +290,8 @@ extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ +extern void _Py_skiproot(const wchar_t *path, Py_ssize_t size, Py_ssize_t *drvsize, Py_ssize_t *rootsize); + // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. // Usage: @@ -324,6 +326,9 @@ extern int _PyFile_Flush(PyObject *); extern int _Py_GetTicksPerSecond(long *ticks_per_second); #endif +// Export for '_testcapi' shared extension +PyAPI_FUNC(int) _Py_IsValidFD(int fd); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 3767df5506d43f..be1c6cc97720d2 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_unicodeobject.h" // _PyUnicodeWriter /* runtime lifecycle */ @@ -34,7 +33,7 @@ struct _Py_float_runtime_state { -void _PyFloat_ExactDealloc(PyObject *op); +PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyObject *op); extern void _PyFloat_DebugMallocStats(FILE* out); diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 58fed46886ea45..5043260d2fd99f 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -8,17 +8,14 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode_utils.h" #include "pycore_compile.h" - -typedef struct { - int id; -} _PyCfgJumpTargetLabel; +#include "pycore_instruction_sequence.h" +#include "pycore_opcode_utils.h" struct _PyCfgBuilder; -int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); -int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); +int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyJumpTargetLabel lbl); +int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _Py_SourceLocation loc); struct _PyCfgBuilder* _PyCfgBuilder_New(void); void _PyCfgBuilder_Free(struct _PyCfgBuilder *g); @@ -27,16 +24,23 @@ int _PyCfgBuilder_CheckSize(struct _PyCfgBuilder* g); int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, int nlocals, int nparams, int firstlineno); -int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_InstructionSequence *seq); +struct _PyCfgBuilder* _PyCfg_FromInstructionSequence(_PyInstructionSequence *seq); +int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyInstructionSequence *seq); int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, int code_flags, int *stackdepth, int *nlocalsplus, - _PyCompile_InstructionSequence *seq); + _PyInstructionSequence *seq); PyCodeObject * _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache, - PyObject *consts, int maxdepth, _PyCompile_InstructionSequence *instrs, + PyObject *consts, int maxdepth, _PyInstructionSequence *instrs, int nlocalsplus, int code_flags, PyObject *filename); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( + PyObject *instructions, + PyObject *consts, + int nlocals); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 0f9e7333cf1e1c..c9ac3819d0390b 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -11,8 +11,9 @@ extern "C" { #include #include // offsetof() #include "pycore_code.h" // STATS +#include "pycore_stackref.h" // _PyStackRef -/* See Objects/frame_layout.md for an explanation of the frame stack +/* See InternalDocs/frames.md for an explanation of the frame stack * including explanation of the PyFrameObject and _PyInterpreterFrame * structs. */ @@ -25,7 +26,11 @@ struct _frame { int f_lineno; /* Current line number. Only valid if non-zero */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ - char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ + PyObject *f_extra_locals; /* Dict for locals set by users using f_locals, could be NULL */ + /* This is purely for backwards compatibility for PyEval_GetLocals. + PyEval_GetLocals requires a borrowed reference so the actual reference + is stored here */ + PyObject *f_locals_cache; /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; }; @@ -55,48 +60,55 @@ enum _frameowner { }; typedef struct _PyInterpreterFrame { - PyObject *f_executable; /* Strong reference */ + _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */ struct _PyInterpreterFrame *previous; - PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ + _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ - int stacktop; /* Offset of TOS from localsplus */ + _PyStackRef *stackpointer; uint16_t return_offset; /* Only relevant during a function call */ char owner; /* Locals and stack */ - PyObject *localsplus[1]; + _PyStackRef localsplus[1]; } _PyInterpreterFrame; #define _PyInterpreterFrame_LASTI(IF) \ ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF)))) static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { - assert(PyCode_Check(f->f_executable)); - return (PyCodeObject *)f->f_executable; + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + assert(PyCode_Check(executable)); + return (PyCodeObject *)executable; } -static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) { - return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus; +static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { + PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); + assert(PyFunction_Check(func)); + return (PyFunctionObject *)func; } -static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - assert(f->localsplus[f->stacktop-1] != NULL); - return f->localsplus[f->stacktop-1]; +static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { + return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } -static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - f->stacktop--; - return f->localsplus[f->stacktop]; +static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + assert(!PyStackRef_IsNull(f->stackpointer[-1])); + return f->stackpointer[-1]; } -static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { - f->localsplus[f->stacktop] = value; - f->stacktop++; +static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + f->stackpointer--; + return *f->stackpointer; +} + +static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, _PyStackRef value) { + *f->stackpointer = value; + f->stackpointer++; } #define FRAME_SPECIALS_SIZE ((int)((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *))) @@ -110,7 +122,27 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) return code->co_framesize - FRAME_SPECIALS_SIZE; } -void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); +static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) +{ + *dest = *src; + assert(src->stackpointer != NULL); + int stacktop = (int)(src->stackpointer - src->localsplus); + assert(stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); + dest->stackpointer = dest->localsplus + stacktop; + for (int i = 1; i < stacktop; i++) { + dest->localsplus[i] = src->localsplus[i]; + } + // Don't leave a dangling pointer to the old frame when creating generators + // and coroutines: + dest->previous = NULL; + +#ifdef Py_GIL_DISABLED + PyCodeObject *co = _PyFrame_GetCode(dest); + for (int i = stacktop; i < co->co_nlocalsplus + co->co_stacksize; i++) { + dest->localsplus[i] = PyStackRef_NULL; + } +#endif +} /* Consumes reference to func and locals. Does not initialize frame->previous, which happens @@ -118,50 +150,63 @@ void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); */ static inline void _PyFrame_Initialize( - _PyInterpreterFrame *frame, PyFunctionObject *func, - PyObject *locals, PyCodeObject *code, int null_locals_from) + _PyInterpreterFrame *frame, _PyStackRef func, + PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous) { - frame->f_funcobj = (PyObject *)func; - frame->f_executable = Py_NewRef(code); - frame->f_builtins = func->func_builtins; - frame->f_globals = func->func_globals; + frame->previous = previous; + frame->f_funcobj = func; + frame->f_executable = PyStackRef_FromPyObjectNew(code); + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + frame->f_builtins = func_obj->func_builtins; + frame->f_globals = func_obj->func_globals; frame->f_locals = locals; - frame->stacktop = code->co_nlocalsplus; + frame->stackpointer = frame->localsplus + code->co_nlocalsplus; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { - frame->localsplus[i] = NULL; + frame->localsplus[i] = PyStackRef_NULL; + } + +#ifdef Py_GIL_DISABLED + // On GIL disabled, we walk the entire stack in GC. Since stacktop + // is not always in sync with the real stack pointer, we have + // no choice but to traverse the entire stack. + // This just makes sure we don't pass the GC invalid stack values. + for (int i = code->co_nlocalsplus; i < code->co_nlocalsplus + code->co_stacksize; i++) { + frame->localsplus[i] = PyStackRef_NULL; } +#endif } /* Gets the pointer to the locals array * that precedes this frame. */ -static inline PyObject** +static inline _PyStackRef* _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) { return frame->localsplus; } -/* Fetches the stack pointer, and sets stacktop to -1. - Having stacktop <= 0 ensures that invalid - values are not visible to the cycle GC. - We choose -1 rather than 0 to assist debugging. */ -static inline PyObject** +/* Fetches the stack pointer, and sets stackpointer to NULL. + Having stackpointer == NULL ensures that invalid + values are not visible to the cycle GC. */ +static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - PyObject **sp = frame->localsplus + frame->stacktop; - frame->stacktop = -1; + assert(frame->stackpointer != NULL); + _PyStackRef *sp = frame->stackpointer; + frame->stackpointer = NULL; return sp; } static inline void -_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) +_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - frame->stacktop = (int)(stack_pointer - frame->localsplus); + assert(frame->stackpointer == NULL); + frame->stackpointer = stack_pointer; } /* Determine whether a frame is incomplete. @@ -204,7 +249,7 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); /* Gets the PyFrameObject for this frame, lazily * creating it if necessary. - * Returns a borrowed referennce */ + * Returns a borrowed reference */ static inline PyFrameObject * _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) { @@ -217,6 +262,9 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) return _PyFrame_MakeAndSetFrameObject(frame); } +void +_PyFrame_ClearLocals(_PyInterpreterFrame *frame); + /* Clears all references in the frame. * If take is non-zero, then the _PyInterpreterFrame frame * may be transferred to the frame object it references @@ -232,14 +280,11 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden); - -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame); +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame); -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear); +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame); static inline bool _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) @@ -256,54 +301,62 @@ _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) extern _PyInterpreterFrame * _PyThreadState_PushFrame(PyThreadState *tstate, size_t size); -void _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); /* Pushes a frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() * Consumes reference to func. */ static inline _PyInterpreterFrame * -_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from) +_PyFrame_PushUnchecked(PyThreadState *tstate, _PyStackRef func, int null_locals_from, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); - PyCodeObject *code = (PyCodeObject *)func->func_code; + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + PyCodeObject *code = (PyCodeObject *)func_obj->func_code; _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from); + _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from, previous); return new_frame; } /* Pushes a trampoline frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() */ static inline _PyInterpreterFrame * -_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth) +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - frame->f_funcobj = Py_None; - frame->f_executable = Py_NewRef(code); + frame->previous = previous; + frame->f_funcobj = PyStackRef_None; + frame->f_executable = PyStackRef_FromPyObjectNew(code); #ifdef Py_DEBUG frame->f_builtins = NULL; frame->f_globals = NULL; #endif frame->f_locals = NULL; - frame->stacktop = code->co_nlocalsplus + stackdepth; + assert(stackdepth <= code->co_stacksize); + frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; frame->return_offset = 0; + +#ifdef Py_GIL_DISABLED + assert(code->co_nlocalsplus == 0); + for (int i = 0; i < code->co_stacksize; i++) { + frame->localsplus[i] = PyStackRef_NULL; + } +#endif return frame; } -static inline -PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) -{ - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); - size_t offset_in_gen = offsetof(PyGenObject, gi_iframe); - return (PyGenObject *)(((char *)frame) - offset_in_gen); -} +PyAPI_FUNC(_PyInterpreterFrame *) +_PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, + PyObject *locals, _PyStackRef const* args, + size_t argcount, PyObject *kwnames, + _PyInterpreterFrame *previous); #ifdef __cplusplus } diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index 1bc551914794f0..da2d7bf6ae1393 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -8,132 +8,101 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -// PyTuple_MAXSAVESIZE - largest tuple to save on free list -// PyTuple_MAXFREELIST - maximum number of tuples of each size to save - -#ifdef WITH_FREELISTS -// with freelists -# define PyTuple_MAXSAVESIZE 20 -# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE -# define PyTuple_MAXFREELIST 2000 -# define PyList_MAXFREELIST 80 -# define PyDict_MAXFREELIST 80 -# define PyFloat_MAXFREELIST 100 -# define PyContext_MAXFREELIST 255 -# define _PyAsyncGen_MAXFREELIST 80 -# define _PyObjectStackChunk_MAXFREELIST 4 -#else -# define PyTuple_NFREELISTS 0 -# define PyTuple_MAXFREELIST 0 -# define PyList_MAXFREELIST 0 -# define PyDict_MAXFREELIST 0 -# define PyFloat_MAXFREELIST 0 -# define PyContext_MAXFREELIST 0 -# define _PyAsyncGen_MAXFREELIST 0 -# define _PyObjectStackChunk_MAXFREELIST 0 +#include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_object.h" // _PyObject_IS_GC +#include "pycore_pystate.h" // _PyThreadState_GET +#include "pycore_code.h" // OBJECT_STAT_INC + +static inline struct _Py_freelists * +_Py_freelists_GET(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + _Py_EnsureTstateNotNULL(tstate); #endif -struct _Py_list_state { -#ifdef WITH_FREELISTS - PyListObject *free_list[PyList_MAXFREELIST]; - int numfree; -#endif -}; - -struct _Py_tuple_state { -#if WITH_FREELISTS - /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE. - The empty tuple is handled separately. - - Each tuple stored in the array is the head of the linked list - (and the next available tuple) for that size. The actual tuple - object is used as the linked list node, with its first item - (ob_item[0]) pointing to the next node (i.e. the previous head). - Each linked list is initially NULL. */ - PyTupleObject *free_list[PyTuple_NFREELISTS]; - int numfree[PyTuple_NFREELISTS]; +#ifdef Py_GIL_DISABLED + return &((_PyThreadStateImpl*)tstate)->freelists; #else - char _unused; // Empty structs are not allowed. -#endif -}; - -struct _Py_float_state { -#ifdef WITH_FREELISTS - /* Special free list - free_list is a singly-linked list of available PyFloatObjects, - linked via abuse of their ob_type members. */ - int numfree; - PyFloatObject *free_list; -#endif -}; - -struct _Py_dict_freelist { -#ifdef WITH_FREELISTS - /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *free_list[PyDict_MAXFREELIST]; - PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; - int numfree; - int keys_numfree; -#endif -}; - -struct _Py_slice_state { -#ifdef WITH_FREELISTS - /* Using a cache is very effective since typically only a single slice is - created and then deleted again. */ - PySliceObject *slice_cache; -#endif -}; - -struct _Py_context_state { -#ifdef WITH_FREELISTS - // List of free PyContext objects - PyContext *freelist; - int numfree; + return &tstate->interp->object_state.freelists; #endif -}; +} -struct _Py_async_gen_state { -#ifdef WITH_FREELISTS - /* Freelists boost performance 6-10%; they also reduce memory - fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend - are short-living objects that are instantiated for every - __anext__() call. */ - struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST]; - int value_numfree; +// Pushes `op` to the freelist, calls `freefunc` if the freelist is full +#define _Py_FREELIST_FREE(NAME, op, freefunc) \ + _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \ + Py_ ## NAME ## _MAXFREELIST, freefunc) +// Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full +#define _Py_FREELIST_PUSH(NAME, op, limit) \ + _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit) + +// Pops a PyObject from the freelist, returns NULL if the freelist is empty. +#define _Py_FREELIST_POP(TYPE, NAME) \ + _Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME)) + +// Pops a non-PyObject data structure from the freelist, returns NULL if the +// freelist is empty. +#define _Py_FREELIST_POP_MEM(NAME) \ + _PyFreeList_PopMem(&_Py_freelists_GET()->NAME) + +#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size) + +static inline int +_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize) +{ + if (fl->size < maxsize && fl->size >= 0) { + *(void **)obj = fl->freelist; + fl->freelist = obj; + fl->size++; + OBJECT_STAT_INC(to_freelist); + return 1; + } + return 0; +} - struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST]; - int asend_numfree; -#endif -}; +static inline void +_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize, + freefunc dofree) +{ + if (!_PyFreeList_Push(fl, obj, maxsize)) { + dofree(obj); + } +} -struct _PyObjectStackChunk; +static inline void * +_PyFreeList_PopNoStats(struct _Py_freelist *fl) +{ + void *obj = fl->freelist; + if (obj != NULL) { + assert(fl->size > 0); + fl->freelist = *(void **)obj; + fl->size--; + } + return obj; +} -struct _Py_object_stack_state { - struct _PyObjectStackChunk *free_list; - Py_ssize_t numfree; -}; +static inline PyObject * +_PyFreeList_Pop(struct _Py_freelist *fl) +{ + PyObject *op = _PyFreeList_PopNoStats(fl); + if (op != NULL) { + OBJECT_STAT_INC(from_freelist); + _Py_NewReference(op); + } + return op; +} -typedef struct _Py_freelist_state { - struct _Py_float_state floats; - struct _Py_tuple_state tuples; - struct _Py_list_state lists; - struct _Py_dict_freelist dicts; - struct _Py_slice_state slices; - struct _Py_context_state contexts; - struct _Py_async_gen_state async_gens; - struct _Py_object_stack_state object_stacks; -} _PyFreeListState; +static inline void * +_PyFreeList_PopMem(struct _Py_freelist *fl) +{ + void *op = _PyFreeList_PopNoStats(fl); + if (op != NULL) { + OBJECT_STAT_INC(from_freelist); + } + return op; +} -extern void _PyObject_ClearFreeLists(_PyFreeListState *state, int is_finalization); -extern void _PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PyFloat_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PySlice_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PyDict_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PyAsyncGen_ClearFreeLists(_PyFreeListState *state, int is_finalization); -extern void _PyContext_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PyObjectStackChunk_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization); #ifdef __cplusplus } diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h new file mode 100644 index 00000000000000..4e04cf431e0b31 --- /dev/null +++ b/Include/internal/pycore_freelist_state.h @@ -0,0 +1,54 @@ +#ifndef Py_INTERNAL_FREELIST_STATE_H +#define Py_INTERNAL_FREELIST_STATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +# define PyTuple_MAXSAVESIZE 20 // Largest tuple to save on freelist +# define Py_tuple_MAXFREELIST 2000 // Maximum number of tuples of each size to save +# define Py_lists_MAXFREELIST 80 +# define Py_dicts_MAXFREELIST 80 +# define Py_dictkeys_MAXFREELIST 80 +# define Py_floats_MAXFREELIST 100 +# define Py_slices_MAXFREELIST 1 +# define Py_contexts_MAXFREELIST 255 +# define Py_async_gens_MAXFREELIST 80 +# define Py_async_gen_asends_MAXFREELIST 80 +# define Py_futureiters_MAXFREELIST 255 +# define Py_object_stack_chunks_MAXFREELIST 4 +# define Py_unicode_writers_MAXFREELIST 1 + +// A generic freelist of either PyObjects or other data structures. +struct _Py_freelist { + // Entries are linked together using the first word of the object. + // For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid` + // field. + void *freelist; + + // The number of items in the free list or -1 if the free list is disabled + Py_ssize_t size; +}; + +struct _Py_freelists { + struct _Py_freelist floats; + struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; + struct _Py_freelist lists; + struct _Py_freelist dicts; + struct _Py_freelist dictkeys; + struct _Py_freelist slices; + struct _Py_freelist contexts; + struct _Py_freelist async_gens; + struct _Py_freelist async_gen_asends; + struct _Py_freelist futureiters; + struct _Py_freelist object_stack_chunks; + struct _Py_freelist unicode_writers; +}; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FREELIST_STATE_H */ diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 3f3da8a44b77e4..c45d281125febb 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -4,6 +4,8 @@ extern "C" { #endif +#include "pycore_lock.h" + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif @@ -16,21 +18,43 @@ extern PyObject* _PyFunction_Vectorcall( #define FUNC_MAX_WATCHERS 8 +#define FUNC_VERSION_UNSET 0 +#define FUNC_VERSION_CLEARED 1 +#define FUNC_VERSION_FIRST_VALID 2 + #define FUNC_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ + +struct _func_version_cache_item { + PyFunctionObject *func; + PyObject *code; +}; + struct _py_func_state { +#ifdef Py_GIL_DISABLED + // Protects next_version + PyMutex mutex; +#endif + uint32_t next_version; - // Borrowed references to function objects whose + // Borrowed references to function and code objects whose // func_version % FUNC_VERSION_CACHE_SIZE // once was equal to the index in the table. - // They are cleared when the function is deallocated. - PyFunctionObject *func_version_cache[FUNC_VERSION_CACHE_SIZE]; + // They are cleared when the function or code object is deallocated. + struct _func_version_cache_item func_version_cache[FUNC_VERSION_CACHE_SIZE]; }; extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); +static inline int +_PyFunction_IsVersionValid(uint32_t version) +{ + return version >= FUNC_VERSION_FIRST_VALID; +} + extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); -extern void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); -PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version); +PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); +void _PyFunction_ClearCodeByVersion(uint32_t version); +PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code); extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 582a16bf5218ce..b85957df5a6b9f 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState - /* GC information is stored BEFORE the object structure. */ typedef struct { // Pointer to next object in the list. @@ -37,19 +35,53 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { } -/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) */ +/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) + * + * Setting the bits requires a relaxed store. The per-object lock must also be + * held, except when the object is only visible to a single thread (e.g. during + * object initialization or destruction). + * + * Reading the bits requires using a relaxed load, but does not require holding + * the per-object lock. + */ #ifdef Py_GIL_DISABLED -# define _PyGC_BITS_TRACKED (1) -# define _PyGC_BITS_FINALIZED (2) +# define _PyGC_BITS_TRACKED (1) // Tracked by the GC +# define _PyGC_BITS_FINALIZED (2) // tp_finalize was called # define _PyGC_BITS_UNREACHABLE (4) # define _PyGC_BITS_FROZEN (8) # define _PyGC_BITS_SHARED (16) +# define _PyGC_BITS_SHARED_INLINE (32) +# define _PyGC_BITS_DEFERRED (64) // Use deferred reference counting +#endif + +#ifdef Py_GIL_DISABLED + +static inline void +_PyObject_SET_GC_BITS(PyObject *op, uint8_t new_bits) +{ + uint8_t bits = _Py_atomic_load_uint8_relaxed(&op->ob_gc_bits); + _Py_atomic_store_uint8_relaxed(&op->ob_gc_bits, bits | new_bits); +} + +static inline int +_PyObject_HAS_GC_BITS(PyObject *op, uint8_t bits) +{ + return (_Py_atomic_load_uint8_relaxed(&op->ob_gc_bits) & bits) != 0; +} + +static inline void +_PyObject_CLEAR_GC_BITS(PyObject *op, uint8_t bits_to_clear) +{ + uint8_t bits = _Py_atomic_load_uint8_relaxed(&op->ob_gc_bits); + _Py_atomic_store_uint8_relaxed(&op->ob_gc_bits, bits & ~bits_to_clear); +} + #endif /* True if the object is currently tracked by the GC. */ static inline int _PyObject_GC_IS_TRACKED(PyObject *op) { #ifdef Py_GIL_DISABLED - return (op->ob_gc_bits & _PyGC_BITS_TRACKED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_TRACKED); #else PyGC_Head *gc = _Py_AS_GC(op); return (gc->_gc_next != 0); @@ -71,28 +103,63 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { #ifdef Py_GIL_DISABLED -/* True if an object is shared between multiple threads and - * needs special purpose when freeing to do the possibility - * of in-flight lock-free reads occuring */ +/* True if memory the object references is shared between + * multiple threads and needs special purpose when freeing + * those references due to the possibility of in-flight + * lock-free reads occurring. The object is responsible + * for calling _PyMem_FreeDelayed on the referenced + * memory. */ static inline int _PyObject_GC_IS_SHARED(PyObject *op) { - return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_SHARED); } #define _PyObject_GC_IS_SHARED(op) _PyObject_GC_IS_SHARED(_Py_CAST(PyObject*, op)) static inline void _PyObject_GC_SET_SHARED(PyObject *op) { - op->ob_gc_bits |= _PyGC_BITS_SHARED; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_SHARED); } #define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) +/* True if the memory of the object is shared between multiple + * threads and needs special purpose when freeing due to + * the possibility of in-flight lock-free reads occurring. + * Objects with this bit that are GC objects will automatically + * delay-freed by PyObject_GC_Del. */ +static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) { + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_SHARED_INLINE); +} +#define _PyObject_GC_IS_SHARED_INLINE(op) \ + _PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op)) + +static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) { + _PyObject_SET_GC_BITS(op, _PyGC_BITS_SHARED_INLINE); +} +#define _PyObject_GC_SET_SHARED_INLINE(op) \ + _PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op)) + #endif /* Bit flags for _gc_prev */ /* Bit 0 is set when tp_finalize is called */ -#define _PyGC_PREV_MASK_FINALIZED (1) +#define _PyGC_PREV_MASK_FINALIZED ((uintptr_t)1) /* Bit 1 is set when the object is in generation which is GCed currently. */ -#define _PyGC_PREV_MASK_COLLECTING (2) -/* The (N-2) most significant bits contain the real address. */ -#define _PyGC_PREV_SHIFT (2) +#define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2) + +/* Bit 0 in _gc_next is the old space bit. + * It is set as follows: + * Young: gcstate->visited_space + * old[0]: 0 + * old[1]: 1 + * permanent: 0 + * + * During a collection all objects handled should have the bit set to + * gcstate->visited_space, as objects are moved from the young gen + * and the increment into old[gcstate->visited_space]. + * When object are moved from the pending space, old[gcstate->visited_space^1] + * into the increment, the old space bit is flipped. +*/ +#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 + +#define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) /* set for debugging information */ @@ -118,11 +185,13 @@ typedef enum { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next; + uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - gc->_gc_next = (uintptr_t)next; + uintptr_t unext = (uintptr_t)next; + assert((unext & ~_PyGC_PREV_MASK) == 0); + gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -130,6 +199,7 @@ static inline PyGC_Head* _PyGCHead_PREV(PyGC_Head *gc) { uintptr_t prev = (gc->_gc_prev & _PyGC_PREV_MASK); return (PyGC_Head*)prev; } + static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { uintptr_t uprev = (uintptr_t)prev; assert((uprev & ~_PyGC_PREV_MASK) == 0); @@ -138,7 +208,7 @@ static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { static inline int _PyGC_FINALIZED(PyObject *op) { #ifdef Py_GIL_DISABLED - return (op->ob_gc_bits & _PyGC_BITS_FINALIZED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_FINALIZED); #else PyGC_Head *gc = _Py_AS_GC(op); return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0); @@ -146,7 +216,7 @@ static inline int _PyGC_FINALIZED(PyObject *op) { } static inline void _PyGC_SET_FINALIZED(PyObject *op) { #ifdef Py_GIL_DISABLED - op->ob_gc_bits |= _PyGC_BITS_FINALIZED; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_FINALIZED); #else PyGC_Head *gc = _Py_AS_GC(op); gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED; @@ -154,7 +224,7 @@ static inline void _PyGC_SET_FINALIZED(PyObject *op) { } static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { #ifdef Py_GIL_DISABLED - op->ob_gc_bits &= ~_PyGC_BITS_FINALIZED; + _PyObject_CLEAR_GC_BITS(op, _PyGC_BITS_FINALIZED); #else PyGC_Head *gc = _Py_AS_GC(op); gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED; @@ -215,6 +285,13 @@ struct gc_generation { generations */ }; +struct gc_collection_stats { + /* number of collected objects */ + Py_ssize_t collected; + /* total number of uncollectable objects (put into gc.garbage) */ + Py_ssize_t uncollectable; +}; + /* Running stats per generation */ struct gc_generation_stats { /* total number of collections */ @@ -236,8 +313,8 @@ struct _gc_runtime_state { int enabled; int debug; /* linked lists of container objects */ - struct gc_generation generations[NUM_GENERATIONS]; - PyGC_Head *generation0; + struct gc_generation young; + struct gc_generation old[2]; /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; struct gc_generation_stats generation_stats[NUM_GENERATIONS]; @@ -247,6 +324,13 @@ struct _gc_runtime_state { PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + + Py_ssize_t heap_size; + Py_ssize_t work_to_do; + /* Which of the old spaces is the visited space */ + int visited_space; + +#ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -258,14 +342,21 @@ struct _gc_runtime_state { collections, and are awaiting to undergo a full collection for the first time. */ Py_ssize_t long_lived_pending; +#endif }; +#ifdef Py_GIL_DISABLED +struct _gc_thread_state { + /* Thread-local allocation count. */ + Py_ssize_t alloc_count; +}; +#endif + extern void _PyGC_InitState(struct _gc_runtime_state *); -extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, - _PyGC_Reason reason); -extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); +extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason); +extern void _PyGC_CollectNoFail(PyThreadState *tstate); /* Freeze objects tracked by the GC and ignore them in future collections. */ extern void _PyGC_Freeze(PyInterpreterState *interp); @@ -274,14 +365,31 @@ extern void _PyGC_Unfreeze(PyInterpreterState *interp); /* Number of frozen objects */ extern Py_ssize_t _PyGC_GetFreezeCount(PyInterpreterState *interp); -extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation); +extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, int generation); extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ScheduleGC(PyInterpreterState *interp); +extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); +union _PyStackRef; + +// GC visit callback for tracked interpreter frames +extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg); +extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); + +// Like Py_VISIT but for _PyStackRef fields +#define _Py_VISIT_STACKREF(ref) \ + do { \ + if (!PyStackRef_IsNull(ref)) { \ + int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \ + if (vret) \ + return vret; \ + } \ + } while (0) + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index b2aa017598409f..f6d7e6d367177b 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -8,9 +8,51 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" +#include "pycore_frame.h" -extern PyObject *_PyGen_yf(PyGenObject *); +/* _PyGenObject_HEAD defines the initial segment of generator + and coroutine objects. */ +#define _PyGenObject_HEAD(prefix) \ + PyObject_HEAD \ + /* List of weak reference. */ \ + PyObject *prefix##_weakreflist; \ + /* Name of the generator. */ \ + PyObject *prefix##_name; \ + /* Qualified name of the generator. */ \ + PyObject *prefix##_qualname; \ + _PyErr_StackItem prefix##_exc_state; \ + PyObject *prefix##_origin_or_finalizer; \ + char prefix##_hooks_inited; \ + char prefix##_closed; \ + char prefix##_running_async; \ + /* The frame */ \ + int8_t prefix##_frame_state; \ + struct _PyInterpreterFrame prefix##_iframe; \ + +struct _PyGenObject { + /* The gi_ prefix is intended to remind of generator-iterator. */ + _PyGenObject_HEAD(gi) +}; + +struct _PyCoroObject { + _PyGenObject_HEAD(cr) +}; + +struct _PyAsyncGenObject { + _PyGenObject_HEAD(ag) +}; + +#undef _PyGenObject_HEAD + +static inline +PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + size_t offset_in_gen = offsetof(PyGenObject, gi_iframe); + return (PyGenObject *)(((char *)frame) - offset_in_gen); +} + +PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *); extern void _PyGen_Finalize(PyObject *self); // Export for '_asyncio' shared extension @@ -19,7 +61,7 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); -extern PyObject *_PyCoro_GetAwaitableIter(PyObject *o); +PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); extern PyTypeObject _PyCoroWrapper_Type; diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index 19b0d23a68568a..a2de5077371eba 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -20,6 +20,23 @@ extern "C" { #define FORCE_SWITCHING struct _gil_runtime_state { +#ifdef Py_GIL_DISABLED + /* If this GIL is disabled, enabled == 0. + + If this GIL is enabled transiently (most likely to initialize a module + of unknown safety), enabled indicates the number of active transient + requests. + + If this GIL is enabled permanently, enabled == INT_MAX. + + It must not be modified directly; use _PyEval_EnableGILTransiently(), + _PyEval_EnableGILPermanently(), and _PyEval_DisableGIL() + + It is always read and written atomically, but a thread can assume its + value will be stable as long as that thread is attached or knows that no + other threads are attached (e.g., during a stop-the-world.). */ + int enabled; +#endif /* microseconds (the Python API uses seconds, though) */ unsigned long interval; /* Last PyThreadState holding / having held the GIL. This helps us diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 327fcc24cb29f1..e3f7ac707f0c37 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -66,9 +66,6 @@ struct _Py_static_objects { struct _Py_interp_cached_objects { PyObject *interned_strings; - /* AST */ - PyObject *str_replace_inf; - /* object.__reduce__ */ PyObject *objreduce; PyObject *type_slots_pname; @@ -81,6 +78,7 @@ struct _Py_interp_cached_objects { PyTypeObject *paramspec_type; PyTypeObject *paramspecargs_type; PyTypeObject *paramspeckwargs_type; + PyTypeObject *constevaluator_type; }; #define _Py_INTERP_STATIC_OBJECT(interp, NAME) \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 932738c3049882..2fd7d5d13a98b2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -11,7 +11,7 @@ extern "C" { #ifdef Py_DEBUG static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { - if (Py_REFCNT(obj) < _Py_IMMORTAL_REFCNT) { + if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); _PyObject_Dump(obj); } @@ -551,21 +551,18 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_setcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_string)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_unknown)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(close_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_close_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_open_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_percent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(defaults)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(newline)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(open_br)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(percent)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); @@ -577,7 +574,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(TextIOWrapper)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(WarningMessage)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_WindowsConsoleIO)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__IOBase_closed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abc_tpflags__)); @@ -590,9 +586,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__all__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__and__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__anext__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__asyncio_running_event_loop__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bases__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bool__)); @@ -609,7 +605,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__classdictcell__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__complex__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__contains__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__copy__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__ctypes_from_outparam__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__del__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__delattr__)); @@ -624,6 +619,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__eq__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__exit__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__file__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__firstlineno__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__float__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__floordiv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__format__)); @@ -724,12 +720,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__slotnames__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__slots__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__spec__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__static_attributes__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__str__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__sub__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasscheck__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasshook__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__truediv__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__trunc__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__type_params__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_is_unpacked_typevartuple__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_prepare_subst__)); @@ -742,7 +738,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abc_impl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); @@ -752,6 +747,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_dealloc_warn)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_feature_version)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_field_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fields_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); @@ -762,23 +758,25 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_isatty_open_only)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_pack_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_only_immortal)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_restype_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_showwarnmsg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_shutdown)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_slotnames)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_swappedbytes_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime_date)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime_datetime)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime_time)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_type_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_uninitialized_submodules)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_warn_unawaited_coroutine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_xoptions)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(a)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(abs_tol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(access)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aclose)); @@ -788,8 +786,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); @@ -801,12 +801,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attribute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(authorizer_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(autocommit)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(b)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(backtick)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(before)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(big)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(binary_form)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(block)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bound)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer)); @@ -819,10 +819,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(byteorder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes_per_sep)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cafile)); @@ -860,6 +860,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_stacksize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_varnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); @@ -873,7 +874,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(count)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(covariant)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cwd)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); @@ -883,6 +883,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(defaultaction)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(delete)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(depth)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(desired_access)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(detect_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device)); @@ -898,16 +899,17 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(displayhook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dklen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(doc)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(done)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dont_inherit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(e)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_lineno)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(endpos)); @@ -924,7 +926,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(f)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false)); @@ -934,6 +935,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fd2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fdel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fget)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fields)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(file)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(file_actions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filename)); @@ -950,6 +952,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format_spec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromtimestamp)); @@ -957,7 +960,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(g)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); @@ -971,15 +973,15 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groupindex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groups)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(h)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle_seq)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(has_location)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hash_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(header)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(headers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); @@ -987,12 +989,16 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(in_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(incoming)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(index)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(infer_variance)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inherit_handle)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inheritable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_bytes)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_owner)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_state)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initval)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inner_size)); @@ -1004,6 +1010,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intersection)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(interval)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_running)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_struct)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isatty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isinstance)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isoformat)); @@ -1025,6 +1032,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(label)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_exc)); @@ -1048,6 +1056,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(logoption)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(loop)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(manual_reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mapping)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); @@ -1064,6 +1073,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); @@ -1073,8 +1083,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(month)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mutex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_arg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_fields)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_sequence_fields)); @@ -1085,6 +1095,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(narg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_limit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(newline)); @@ -1119,7 +1130,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(owner)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(p)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(password)); @@ -1147,7 +1157,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(query)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(quotetabs)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(r)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(raw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read1)); @@ -1171,11 +1180,11 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(return)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reverse)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reversed)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(s)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(security_attributes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(selectors)); @@ -1203,6 +1212,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sort)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source_traceback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(spam)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src_dir_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stacklevel)); @@ -1263,6 +1273,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(volume)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wait_all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warn_on_full_buffer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnings)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnoptions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); @@ -1274,7 +1286,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(writable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write_through)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(x)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(year)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zdict)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[0]); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index da62b4f0a951ff..fc3871570cc49d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -37,21 +37,18 @@ struct _Py_global_strings { STRUCT_FOR_STR(anon_setcomp, "") STRUCT_FOR_STR(anon_string, "") STRUCT_FOR_STR(anon_unknown, "") - STRUCT_FOR_STR(close_br, "}") STRUCT_FOR_STR(dbl_close_br, "}}") STRUCT_FOR_STR(dbl_open_br, "{{") STRUCT_FOR_STR(dbl_percent, "%%") STRUCT_FOR_STR(defaults, ".defaults") - STRUCT_FOR_STR(dot, ".") STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") + STRUCT_FOR_STR(format, ".format") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") - STRUCT_FOR_STR(newline, "\n") - STRUCT_FOR_STR(open_br, "{") - STRUCT_FOR_STR(percent, "%") + STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; @@ -66,7 +63,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(TextIOWrapper) STRUCT_FOR_ID(True) STRUCT_FOR_ID(WarningMessage) - STRUCT_FOR_ID(_) STRUCT_FOR_ID(_WindowsConsoleIO) STRUCT_FOR_ID(__IOBase_closed) STRUCT_FOR_ID(__abc_tpflags__) @@ -79,9 +75,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(__all__) STRUCT_FOR_ID(__and__) STRUCT_FOR_ID(__anext__) + STRUCT_FOR_ID(__annotate__) STRUCT_FOR_ID(__annotations__) STRUCT_FOR_ID(__args__) - STRUCT_FOR_ID(__asyncio_running_event_loop__) STRUCT_FOR_ID(__await__) STRUCT_FOR_ID(__bases__) STRUCT_FOR_ID(__bool__) @@ -98,7 +94,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(__classdictcell__) STRUCT_FOR_ID(__complex__) STRUCT_FOR_ID(__contains__) - STRUCT_FOR_ID(__copy__) STRUCT_FOR_ID(__ctypes_from_outparam__) STRUCT_FOR_ID(__del__) STRUCT_FOR_ID(__delattr__) @@ -113,6 +108,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__eq__) STRUCT_FOR_ID(__exit__) STRUCT_FOR_ID(__file__) + STRUCT_FOR_ID(__firstlineno__) STRUCT_FOR_ID(__float__) STRUCT_FOR_ID(__floordiv__) STRUCT_FOR_ID(__format__) @@ -213,12 +209,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(__slotnames__) STRUCT_FOR_ID(__slots__) STRUCT_FOR_ID(__spec__) + STRUCT_FOR_ID(__static_attributes__) STRUCT_FOR_ID(__str__) STRUCT_FOR_ID(__sub__) STRUCT_FOR_ID(__subclasscheck__) STRUCT_FOR_ID(__subclasshook__) STRUCT_FOR_ID(__truediv__) - STRUCT_FOR_ID(__trunc__) STRUCT_FOR_ID(__type_params__) STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__) STRUCT_FOR_ID(__typing_prepare_subst__) @@ -231,7 +227,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_abc_impl) STRUCT_FOR_ID(_abstract_) STRUCT_FOR_ID(_active) - STRUCT_FOR_ID(_annotation) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) @@ -241,6 +236,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_check_retval_) STRUCT_FOR_ID(_dealloc_warn) STRUCT_FOR_ID(_feature_version) + STRUCT_FOR_ID(_field_types) STRUCT_FOR_ID(_fields_) STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) @@ -251,23 +247,25 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) + STRUCT_FOR_ID(_isatty_open_only) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) STRUCT_FOR_ID(_loop) STRUCT_FOR_ID(_needs_com_addref_) - STRUCT_FOR_ID(_pack_) + STRUCT_FOR_ID(_only_immortal) STRUCT_FOR_ID(_restype_) STRUCT_FOR_ID(_showwarnmsg) STRUCT_FOR_ID(_shutdown) STRUCT_FOR_ID(_slotnames) - STRUCT_FOR_ID(_strptime_datetime) - STRUCT_FOR_ID(_swappedbytes_) + STRUCT_FOR_ID(_strptime) + STRUCT_FOR_ID(_strptime_datetime_date) + STRUCT_FOR_ID(_strptime_datetime_datetime) + STRUCT_FOR_ID(_strptime_datetime_time) STRUCT_FOR_ID(_type_) STRUCT_FOR_ID(_uninitialized_submodules) STRUCT_FOR_ID(_warn_unawaited_coroutine) STRUCT_FOR_ID(_xoptions) - STRUCT_FOR_ID(a) STRUCT_FOR_ID(abs_tol) STRUCT_FOR_ID(access) STRUCT_FOR_ID(aclose) @@ -277,8 +275,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) STRUCT_FOR_ID(alias) + STRUCT_FOR_ID(align) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(append) + STRUCT_FOR_ID(arg) STRUCT_FOR_ID(argdefs) STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) @@ -290,12 +290,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(attribute) STRUCT_FOR_ID(authorizer_callback) STRUCT_FOR_ID(autocommit) - STRUCT_FOR_ID(b) STRUCT_FOR_ID(backtick) STRUCT_FOR_ID(base) STRUCT_FOR_ID(before) STRUCT_FOR_ID(big) STRUCT_FOR_ID(binary_form) + STRUCT_FOR_ID(bit_size) STRUCT_FOR_ID(block) STRUCT_FOR_ID(bound) STRUCT_FOR_ID(buffer) @@ -308,10 +308,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(byteorder) STRUCT_FOR_ID(bytes) STRUCT_FOR_ID(bytes_per_sep) - STRUCT_FOR_ID(c) STRUCT_FOR_ID(c_call) STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_return) + STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) STRUCT_FOR_ID(cafile) @@ -349,6 +349,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(co_stacksize) STRUCT_FOR_ID(co_varnames) STRUCT_FOR_ID(code) + STRUCT_FOR_ID(col_offset) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) @@ -362,7 +363,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(count) STRUCT_FOR_ID(covariant) STRUCT_FOR_ID(cwd) - STRUCT_FOR_ID(d) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) STRUCT_FOR_ID(day) @@ -372,6 +372,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(defaultaction) STRUCT_FOR_ID(delete) STRUCT_FOR_ID(depth) + STRUCT_FOR_ID(desired_access) STRUCT_FOR_ID(detect_types) STRUCT_FOR_ID(deterministic) STRUCT_FOR_ID(device) @@ -387,16 +388,17 @@ struct _Py_global_strings { STRUCT_FOR_ID(displayhook) STRUCT_FOR_ID(dklen) STRUCT_FOR_ID(doc) + STRUCT_FOR_ID(done) STRUCT_FOR_ID(dont_inherit) STRUCT_FOR_ID(dst) STRUCT_FOR_ID(dst_dir_fd) - STRUCT_FOR_ID(e) STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) STRUCT_FOR_ID(encode) STRUCT_FOR_ID(encoding) STRUCT_FOR_ID(end) + STRUCT_FOR_ID(end_col_offset) STRUCT_FOR_ID(end_lineno) STRUCT_FOR_ID(end_offset) STRUCT_FOR_ID(endpos) @@ -413,7 +415,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(exp) STRUCT_FOR_ID(extend) STRUCT_FOR_ID(extra_tokens) - STRUCT_FOR_ID(f) STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) STRUCT_FOR_ID(false) @@ -423,6 +424,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fd2) STRUCT_FOR_ID(fdel) STRUCT_FOR_ID(fget) + STRUCT_FOR_ID(fields) STRUCT_FOR_ID(file) STRUCT_FOR_ID(file_actions) STRUCT_FOR_ID(filename) @@ -439,6 +441,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fold) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) + STRUCT_FOR_ID(format_spec) STRUCT_FOR_ID(from_param) STRUCT_FOR_ID(fromlist) STRUCT_FOR_ID(fromtimestamp) @@ -446,7 +449,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(fset) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) - STRUCT_FOR_ID(g) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(genexpr) STRUCT_FOR_ID(get) @@ -460,15 +462,15 @@ struct _Py_global_strings { STRUCT_FOR_ID(globals) STRUCT_FOR_ID(groupindex) STRUCT_FOR_ID(groups) - STRUCT_FOR_ID(h) STRUCT_FOR_ID(handle) + STRUCT_FOR_ID(handle_seq) + STRUCT_FOR_ID(has_location) STRUCT_FOR_ID(hash_name) STRUCT_FOR_ID(header) STRUCT_FOR_ID(headers) STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) - STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) @@ -476,12 +478,16 @@ struct _Py_global_strings { STRUCT_FOR_ID(importlib) STRUCT_FOR_ID(in_fd) STRUCT_FOR_ID(incoming) + STRUCT_FOR_ID(index) STRUCT_FOR_ID(indexgroup) STRUCT_FOR_ID(inf) STRUCT_FOR_ID(infer_variance) + STRUCT_FOR_ID(inherit_handle) STRUCT_FOR_ID(inheritable) STRUCT_FOR_ID(initial) STRUCT_FOR_ID(initial_bytes) + STRUCT_FOR_ID(initial_owner) + STRUCT_FOR_ID(initial_state) STRUCT_FOR_ID(initial_value) STRUCT_FOR_ID(initval) STRUCT_FOR_ID(inner_size) @@ -493,6 +499,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(intersection) STRUCT_FOR_ID(interval) STRUCT_FOR_ID(is_running) + STRUCT_FOR_ID(is_struct) STRUCT_FOR_ID(isatty) STRUCT_FOR_ID(isinstance) STRUCT_FOR_ID(isoformat) @@ -514,6 +521,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw1) STRUCT_FOR_ID(kw2) STRUCT_FOR_ID(kwdefaults) + STRUCT_FOR_ID(label) STRUCT_FOR_ID(lambda) STRUCT_FOR_ID(last) STRUCT_FOR_ID(last_exc) @@ -537,6 +545,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(locals) STRUCT_FOR_ID(logoption) STRUCT_FOR_ID(loop) + STRUCT_FOR_ID(manual_reset) STRUCT_FOR_ID(mapping) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) @@ -553,6 +562,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) STRUCT_FOR_ID(microsecond) + STRUCT_FOR_ID(milliseconds) STRUCT_FOR_ID(minute) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) @@ -562,8 +572,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(month) STRUCT_FOR_ID(mro) STRUCT_FOR_ID(msg) + STRUCT_FOR_ID(mutex) STRUCT_FOR_ID(mycmp) - STRUCT_FOR_ID(n) STRUCT_FOR_ID(n_arg) STRUCT_FOR_ID(n_fields) STRUCT_FOR_ID(n_sequence_fields) @@ -574,6 +584,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(namespaces) STRUCT_FOR_ID(narg) STRUCT_FOR_ID(ndigits) + STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) STRUCT_FOR_ID(new_limit) STRUCT_FOR_ID(newline) @@ -608,7 +619,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(overlapped) STRUCT_FOR_ID(owner) - STRUCT_FOR_ID(p) STRUCT_FOR_ID(pages) STRUCT_FOR_ID(parent) STRUCT_FOR_ID(password) @@ -636,7 +646,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(ps2) STRUCT_FOR_ID(query) STRUCT_FOR_ID(quotetabs) - STRUCT_FOR_ID(r) STRUCT_FOR_ID(raw) STRUCT_FOR_ID(read) STRUCT_FOR_ID(read1) @@ -660,11 +669,11 @@ struct _Py_global_strings { STRUCT_FOR_ID(return) STRUCT_FOR_ID(reverse) STRUCT_FOR_ID(reversed) - STRUCT_FOR_ID(s) STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) STRUCT_FOR_ID(second) + STRUCT_FOR_ID(security_attributes) STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) STRUCT_FOR_ID(selectors) @@ -692,6 +701,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(sort) STRUCT_FOR_ID(source) STRUCT_FOR_ID(source_traceback) + STRUCT_FOR_ID(spam) STRUCT_FOR_ID(src) STRUCT_FOR_ID(src_dir_fd) STRUCT_FOR_ID(stacklevel) @@ -752,6 +762,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) STRUCT_FOR_ID(volume) + STRUCT_FOR_ID(wait_all) + STRUCT_FOR_ID(warn_on_full_buffer) STRUCT_FOR_ID(warnings) STRUCT_FOR_ID(warnoptions) STRUCT_FOR_ID(wbits) @@ -763,7 +775,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(writable) STRUCT_FOR_ID(write) STRUCT_FOR_ID(write_through) - STRUCT_FOR_ID(x) STRUCT_FOR_ID(year) STRUCT_FOR_ID(zdict) } identifiers; @@ -786,6 +797,10 @@ struct _Py_global_strings { (_Py_SINGLETON(strings.identifiers._py_ ## NAME._ascii.ob_base)) #define _Py_STR(NAME) \ (_Py_SINGLETON(strings.literals._py_ ## NAME._ascii.ob_base)) +#define _Py_LATIN1_CHR(CH) \ + ((CH) < 128 \ + ? (PyObject*)&_Py_SINGLETON(strings).ascii[(CH)] \ + : (PyObject*)&_Py_SINGLETON(strings).latin1[(CH) - 128]) /* _Py_DECLARE_STR() should precede all uses of _Py_STR() in a function. diff --git a/Include/internal/pycore_identifier.h b/Include/internal/pycore_identifier.h deleted file mode 100644 index cda28810a48196..00000000000000 --- a/Include/internal/pycore_identifier.h +++ /dev/null @@ -1,20 +0,0 @@ -/* String Literals: _Py_Identifier API */ - -#ifndef Py_INTERNAL_IDENTIFIER_H -#define Py_INTERNAL_IDENTIFIER_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -extern PyObject* _PyType_LookupId(PyTypeObject *, _Py_Identifier *); -extern PyObject* _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); -extern int _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); - -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_IDENTIFIER_H diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index c84f87a831bf38..290ba95e1a0ad7 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -11,7 +11,6 @@ extern "C" { #include "pycore_lock.h" // PyMutex #include "pycore_hashtable.h" // _Py_hashtable_t -#include "pycore_time.h" // _PyTime_t extern int _PyImport_IsInitialized(PyInterpreterState *); @@ -21,15 +20,15 @@ PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); extern int _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); -extern int _PyImport_ReleaseLock(PyInterpreterState *interp); +extern void _PyImport_ReleaseLock(PyInterpreterState *interp); +// This is used exclusively for the sys and builtins modules: extern int _PyImport_FixupBuiltin( + PyThreadState *tstate, PyObject *mod, const char *name, /* UTF-8 encoded string */ PyObject *modules ); -extern int _PyImport_FixupExtensionObject(PyObject*, PyObject *, - PyObject *, PyObject *); // Export for many shared extensions, like '_json' PyAPI_FUNC(PyObject*) _PyImport_GetModuleAttr(PyObject *, PyObject *); @@ -51,9 +50,9 @@ struct _import_runtime_state { PyMutex mutex; /* The actual cache of (filename, name, PyModuleDef) for modules. Only legacy (single-phase init) extension modules are added - and only if they support multiple initialization (m_size >- 0) + and only if they support multiple initialization (m_size >= 0) or are imported in the main interpreter. - This is initialized lazily in _PyImport_FixupExtensionObject(). + This is initialized lazily in fix_up_extension() in import.c. Modules are added there and looked up in _imp.find_extension(). */ _Py_hashtable_t *hashtable; } extensions; @@ -95,15 +94,11 @@ struct _import_state { #endif PyObject *import_func; /* The global import lock. */ - struct { - PyThread_type_lock mutex; - unsigned long thread; - int level; - } lock; + _PyRecursiveMutex lock; /* diagnostic info in PyImport_ImportModuleLevelObject() */ struct { int import_level; - _PyTime_t accumulated; + PyTime_t accumulated; int header; } find_and_load; }; @@ -124,11 +119,6 @@ struct _import_state { #define IMPORTS_INIT \ { \ DLOPENFLAGS_INIT \ - .lock = { \ - .mutex = NULL, \ - .thread = PYTHREAD_INVALID_THREAD_ID, \ - .level = 0, \ - }, \ .find_and_load = { \ .header = 1, \ }, \ @@ -181,11 +171,6 @@ extern void _PyImport_FiniCore(PyInterpreterState *interp); extern void _PyImport_FiniExternal(PyInterpreterState *interp); -#ifdef HAVE_FORK -extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp); -#endif - - extern PyObject* _PyImport_GetBuiltinModuleNames(void); struct _module_alias { @@ -207,6 +192,19 @@ extern int _PyImport_CheckSubinterpIncompatibleExtensionAllowed( // Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); +#ifdef Py_GIL_DISABLED +// Assuming that the GIL is enabled from a call to +// _PyEval_EnableGILTransient(), resolve the transient request depending on the +// state of the module argument: +// - If module is NULL or a PyModuleObject with md_gil == Py_MOD_GIL_NOT_USED, +// call _PyEval_DisableGIL(). +// - Otherwise, call _PyEval_EnableGILPermanent(). If the GIL was not already +// enabled permanently, issue a warning referencing the module's name. +// +// This function may raise an exception. +extern int _PyImport_CheckGILForModule(PyObject *module, PyObject *module_name); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index c8583582b358ac..525a16f6b97274 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,9 +14,91 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; -extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); +typedef enum ext_module_kind { + _Py_ext_module_kind_UNKNOWN = 0, + _Py_ext_module_kind_SINGLEPHASE = 1, + _Py_ext_module_kind_MULTIPHASE = 2, + _Py_ext_module_kind_INVALID = 3, +} _Py_ext_module_kind; + +typedef enum ext_module_origin { + _Py_ext_module_origin_CORE = 1, + _Py_ext_module_origin_BUILTIN = 2, + _Py_ext_module_origin_DYNAMIC = 3, +} _Py_ext_module_origin; + +/* Input for loading an extension module. */ +struct _Py_ext_module_loader_info { + PyObject *filename; +#ifndef MS_WINDOWS + PyObject *filename_encoded; +#endif + PyObject *name; + PyObject *name_encoded; + /* path is always a borrowed ref of name or filename, + * depending on if it's builtin or not. */ + PyObject *path; + _Py_ext_module_origin origin; + const char *hook_prefix; + const char *newcontext; +}; +extern void _Py_ext_module_loader_info_clear( + struct _Py_ext_module_loader_info *info); +extern int _Py_ext_module_loader_info_init( + struct _Py_ext_module_loader_info *info, + PyObject *name, + PyObject *filename, + _Py_ext_module_origin origin); +extern int _Py_ext_module_loader_info_init_for_core( + struct _Py_ext_module_loader_info *p_info, + PyObject *name); +extern int _Py_ext_module_loader_info_init_for_builtin( + struct _Py_ext_module_loader_info *p_info, + PyObject *name); +#ifdef HAVE_DYNAMIC_LOADING +extern int _Py_ext_module_loader_info_init_from_spec( + struct _Py_ext_module_loader_info *info, + PyObject *spec); +#endif + +/* The result from running an extension module's init function. */ +struct _Py_ext_module_loader_result { + PyModuleDef *def; + PyObject *module; + _Py_ext_module_kind kind; + struct _Py_ext_module_loader_result_error *err; + struct _Py_ext_module_loader_result_error { + enum _Py_ext_module_loader_result_error_kind { + _Py_ext_module_loader_result_EXCEPTION = 0, + _Py_ext_module_loader_result_ERR_MISSING = 1, + _Py_ext_module_loader_result_ERR_UNREPORTED_EXC = 2, + _Py_ext_module_loader_result_ERR_UNINITIALIZED = 3, + _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE = 4, + _Py_ext_module_loader_result_ERR_NOT_MODULE = 5, + _Py_ext_module_loader_result_ERR_MISSING_DEF = 6, + } kind; + PyObject *exc; + } _err; +}; +extern void _Py_ext_module_loader_result_clear( + struct _Py_ext_module_loader_result *res); +extern void _Py_ext_module_loader_result_apply_error( + struct _Py_ext_module_loader_result *res, + const char *name); + +/* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); +#ifdef HAVE_DYNAMIC_LOADING +extern PyModInitFunction _PyImport_GetModInitFunc( + struct _Py_ext_module_loader_info *info, + FILE *fp); +#endif +extern int _PyImport_RunModInitFunc( + PyModInitFunction p0, + struct _Py_ext_module_loader_info *info, + struct _Py_ext_module_loader_result *p_res); + /* Max length of module suffix searched for -- accommodates "module.slb" */ #define MAXSUFFIXSIZE 12 diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index c86988234f6a05..25ec8cec3bd249 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,6 +153,18 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; +typedef enum { + /* For now, this means the GIL is enabled. + + gh-116329: This will eventually change to "the GIL is disabled but can + be re-enabled by loading an incompatible extension module." */ + _PyConfig_GIL_DEFAULT = -1, + + /* The GIL has been forced off or on, and will not be affected by module loading. */ + _PyConfig_GIL_DISABLE = 0, + _PyConfig_GIL_ENABLE = 1, +} _PyConfigGILEnum; + // Export for '_testembed' program PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); @@ -169,7 +181,7 @@ extern PyStatus _PyConfig_Write(const PyConfig *config, extern PyStatus _PyConfig_SetPyArgv( PyConfig *config, const _PyArgv *args); - +extern PyObject* _PyConfig_CreateXOptionsDict(const PyConfig *config); extern void _Py_DumpPathConfig(PyThreadState *tstate); diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h new file mode 100644 index 00000000000000..099f2fd124007a --- /dev/null +++ b/Include/internal/pycore_instruction_sequence.h @@ -0,0 +1,78 @@ +#ifndef Py_INTERNAL_INSTRUCTION_SEQUENCE_H +#define Py_INTERNAL_INSTRUCTION_SEQUENCE_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_symtable.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef struct { + int h_label; + int h_startdepth; + int h_preserve_lasti; +} _PyExceptHandlerInfo; + +typedef struct { + int i_opcode; + int i_oparg; + _Py_SourceLocation i_loc; + _PyExceptHandlerInfo i_except_handler_info; + + /* Temporary fields, used by the assembler and in instr_sequence_to_cfg */ + int i_target; + int i_offset; +} _PyInstruction; + +typedef struct instruction_sequence { + PyObject_HEAD + _PyInstruction *s_instrs; + int s_allocated; + int s_used; + + int s_next_free_label; /* next free label id */ + + /* Map of a label id to instruction offset (index into s_instrs). + * If s_labelmap is NULL, then each label id is the offset itself. + */ + int *s_labelmap; + int s_labelmap_size; + + /* PyList of instruction sequences of nested functions */ + PyObject *s_nested; +} _PyInstructionSequence; + +typedef struct { + int id; +} _PyJumpTargetLabel; + +#define NO_LABEL ((const _PyJumpTargetLabel){-1}) + +#define SAME_JUMP_TARGET_LABEL(L1, L2) ((L1).id == (L2).id) +#define IS_JUMP_TARGET_LABEL(L) (!SAME_JUMP_TARGET_LABEL((L), (NO_LABEL))) + +PyAPI_FUNC(PyObject*)_PyInstructionSequence_New(void); + +int _PyInstructionSequence_UseLabel(_PyInstructionSequence *seq, int lbl); +int _PyInstructionSequence_Addop(_PyInstructionSequence *seq, + int opcode, int oparg, + _Py_SourceLocation loc); +_PyJumpTargetLabel _PyInstructionSequence_NewLabel(_PyInstructionSequence *seq); +int _PyInstructionSequence_ApplyLabelMap(_PyInstructionSequence *seq); +int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int pos, + int opcode, int oparg, _Py_SourceLocation loc); +int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); +void PyInstructionSequence_Fini(_PyInstructionSequence *seq); + +extern PyTypeObject _PyInstructionSequence_Type; +#define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &_PyInstructionSequence_Type) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_INSTRUCTION_SEQUENCE_H */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index eae8371ef7f9b8..4e5b374968ea98 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -13,38 +13,6 @@ extern "C" { #define PY_MONITORING_TOOL_IDS 8 -/* Local events. - * These require bytecode instrumentation */ - -#define PY_MONITORING_EVENT_PY_START 0 -#define PY_MONITORING_EVENT_PY_RESUME 1 -#define PY_MONITORING_EVENT_PY_RETURN 2 -#define PY_MONITORING_EVENT_PY_YIELD 3 -#define PY_MONITORING_EVENT_CALL 4 -#define PY_MONITORING_EVENT_LINE 5 -#define PY_MONITORING_EVENT_INSTRUCTION 6 -#define PY_MONITORING_EVENT_JUMP 7 -#define PY_MONITORING_EVENT_BRANCH 8 -#define PY_MONITORING_EVENT_STOP_ITERATION 9 - -#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) - -/* Other events, mainly exceptions */ - -#define PY_MONITORING_EVENT_RAISE 10 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 -#define PY_MONITORING_EVENT_PY_UNWIND 12 -#define PY_MONITORING_EVENT_PY_THROW 13 -#define PY_MONITORING_EVENT_RERAISE 14 - - -/* Ancilliary events */ - -#define PY_MONITORING_EVENT_C_RETURN 15 -#define PY_MONITORING_EVENT_C_RAISE 16 - - typedef uint32_t _PyMonitoringEventSet; /* Tool IDs */ @@ -55,7 +23,7 @@ typedef uint32_t _PyMonitoringEventSet; #define PY_MONITORING_PROFILER_ID 2 #define PY_MONITORING_OPTIMIZER_ID 5 -/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */ +/* Internal IDs used to support sys.setprofile() and sys.settrace() */ #define PY_MONITORING_SYS_PROFILE_ID 6 #define PY_MONITORING_SYS_TRACE_ID 7 diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 31d88071e19d0c..36cd71e5a007d5 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -14,25 +14,28 @@ extern "C" { #include "pycore_atexit.h" // struct atexit_state #include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache +#include "pycore_codecs.h" // struct codecs_state #include "pycore_context.h" // struct _Py_context_state #include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_dict_state.h" // struct _Py_dict_state #include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_floatobject.h" // struct _Py_float_state -#include "pycore_freelist.h" // struct _Py_freelist_state #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_gc.h" // struct _gc_runtime_state -#include "pycore_genobject.h" // struct _Py_async_gen_state +#include "pycore_genobject.h" // _PyGen_FetchStopIterationValue #include "pycore_global_objects.h"// struct _Py_interp_cached_objects #include "pycore_import.h" // struct _import_state #include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_mimalloc.h" // struct _mimalloc_interp_state #include "pycore_object_state.h" // struct _py_object_state +#include "pycore_optimizer.h" // _PyOptimizerObject #include "pycore_obmalloc.h" // struct _obmalloc_state +#include "pycore_qsbr.h" // struct _qsbr_state #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state +#include "pycore_uniqueid.h" // struct _Py_unique_id_pool #include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state @@ -59,6 +62,12 @@ struct _stoptheworld_state { PyThreadState *requester; // Thread that requested the pause (may be NULL). }; +#ifdef Py_GIL_DISABLED +// This should be prime but otherwise the choice is arbitrary. A larger value +// increases concurrency at the expense of memory. +# define NUM_WEAKREF_LIST_LOCKS 127 +#endif + /* cross-interpreter data registry */ /* Tracks some rare events per-interpreter, used by the optimizer to turn on/off @@ -85,7 +94,7 @@ typedef struct _rare_events { */ struct _is { - /* This struct countains the eval_breaker, + /* This struct contains the eval_breaker, * which is by far the hottest field in this struct * and should be placed at the beginning. */ struct _ceval_state ceval; @@ -93,15 +102,26 @@ struct _is { PyInterpreterState *next; int64_t id; - int64_t id_refcount; + Py_ssize_t id_refcount; int requires_idref; - PyThread_type_lock id_mutex; + +#define _PyInterpreterState_WHENCE_NOTSET -1 +#define _PyInterpreterState_WHENCE_UNKNOWN 0 +#define _PyInterpreterState_WHENCE_RUNTIME 1 +#define _PyInterpreterState_WHENCE_LEGACY_CAPI 2 +#define _PyInterpreterState_WHENCE_CAPI 3 +#define _PyInterpreterState_WHENCE_XI 4 +#define _PyInterpreterState_WHENCE_STDLIB 5 +#define _PyInterpreterState_WHENCE_MAX 5 + long _whence; /* Has been initialized to a safe state. In order to be effective, this must be set to 0 during or right after allocation. */ int _initialized; + /* Has been fully initialized via pylifecycle.c. */ + int _ready; int finalizing; uintptr_t last_restart_version; @@ -112,7 +132,7 @@ struct _is { /* The thread currently executing in the __main__ module, if any. */ PyThreadState *main; /* Used in Modules/_threadmodule.c. */ - long count; + Py_ssize_t count; /* Support for runtime thread stack size tuning. A value of 0 means using the platform's default stack size or the size specified by the THREAD_STACK_SIZE macro. */ @@ -164,10 +184,7 @@ struct _is { possible to facilitate out-of-process observability tools. */ - PyObject *codec_search_path; - PyObject *codec_search_cache; - PyObject *codec_error_registry; - int codecs_initialized; + struct codecs_state codecs; PyConfig config; unsigned long feature_flags; @@ -198,10 +215,13 @@ struct _is { struct _warnings_runtime_state warnings; struct atexit_state atexit; struct _stoptheworld_state stoptheworld; + struct _qsbr_shared qsbr; #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; struct _brc_state brc; // biased reference counting state + struct _Py_unique_id_pool unique_ids; // object ids for per-thread refcounts + PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; #endif // Per-interpreter state for the obmalloc allocator. For the main @@ -219,29 +239,28 @@ struct _is { PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; PyCode_WatchCallback code_watchers[CODE_MAX_WATCHERS]; + PyContext_WatchCallback context_watchers[CONTEXT_MAX_WATCHERS]; // One bit is set for each non-NULL entry in code_watchers uint8_t active_code_watchers; + uint8_t active_context_watchers; -#if !defined(Py_GIL_DISABLED) - struct _Py_freelist_state freelist_state; -#endif struct _py_object_state object_state; struct _Py_unicode_state unicode; struct _Py_long_state long_state; struct _dtoa_state dtoa; struct _py_func_state func_state; + struct _py_code_state code_state; struct _Py_dict_state dict_state; struct _Py_exc_state exc_state; + struct _Py_mem_interp_free_queue mem_free_queue; struct ast_state ast; struct types_state types; struct callable_cache callable_cache; _PyOptimizerObject *optimizer; _PyExecutorObject *executor_list_head; - uint16_t optimizer_resume_threshold; - uint16_t optimizer_backedge_threshold; - uint32_t next_func_version; + size_t trace_run_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; @@ -252,6 +271,7 @@ struct _is { Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; + uintptr_t monitoring_tool_versions[PY_MONITORING_TOOL_IDS]; struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; @@ -292,13 +312,21 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst } -// Export for the _xxinterpchannels module. -PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); -PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); -PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); +// Exports for the _testinternalcapi module. +PyAPI_FUNC(int64_t) _PyInterpreterState_ObjectToID(PyObject *); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpIDObject(PyObject *); +PyAPI_FUNC(void) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IsReady(PyInterpreterState *interp); + +PyAPI_FUNC(long) _PyInterpreterState_GetWhence(PyInterpreterState *interp); +extern void _PyInterpreterState_SetWhence( + PyInterpreterState *interp, + long whence); + extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); // Get a copy of the current interpreter configuration. @@ -377,7 +405,10 @@ PyAPI_FUNC(PyStatus) _PyInterpreterState_New( #define RARE_EVENT_INTERP_INC(interp, name) \ do { \ /* saturating add */ \ - if (interp->rare_events.name < UINT8_MAX) interp->rare_events.name++; \ + int val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ + if (val < UINT8_MAX) { \ + FT_ATOMIC_STORE_UINT8(interp->rare_events.name, val + 1); \ + } \ RARE_EVENT_STAT_INC(name); \ } while (0); \ diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 3a8dd95cff8e5d..39c2a30f6e979d 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -28,8 +28,9 @@ #define INTRINSIC_TYPEVAR_WITH_BOUND 2 #define INTRINSIC_TYPEVAR_WITH_CONSTRAINTS 3 #define INTRINSIC_SET_FUNCTION_TYPE_PARAMS 4 +#define INTRINSIC_SET_TYPEPARAM_DEFAULT 5 -#define MAX_INTRINSIC_2 4 +#define MAX_INTRINSIC_2 5 typedef PyObject *(*intrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*intrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); @@ -44,7 +45,7 @@ typedef struct { const char *name; } intrinsic_func2_info; -extern const intrinsic_func1_info _PyIntrinsics_UnaryFunctions[]; -extern const intrinsic_func2_info _PyIntrinsics_BinaryFunctions[]; +PyAPI_DATA(const intrinsic_func1_info) _PyIntrinsics_UnaryFunctions[]; +PyAPI_DATA(const intrinsic_func2_info) _PyIntrinsics_BinaryFunctions[]; #endif // !Py_INTERNAL_INTRINSIC_H diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 0b71eb6f758ac6..4d6cc35a7a3de7 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -11,9 +11,9 @@ extern "C" { #ifdef _Py_JIT -typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate); +typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); -int _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length); +int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); #endif // _Py_JIT diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 50dc13c4da4487..2c666f9be4bd79 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,14 +8,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState - -extern PyObject* _PyList_Extend(PyListObject *, PyObject *); +PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) -extern int +PyAPI_FUNC(int) _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem); // In free-threaded build: self should be locked by the caller, if it should be thread-safe. @@ -28,7 +26,11 @@ _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) Py_ssize_t allocated = self->allocated; assert((size_t)len + 1 < PY_SSIZE_T_MAX); if (allocated > len) { +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ptr_release(&self->ob_item[len], newitem); +#else PyList_SET_ITEM(self, len, newitem); +#endif Py_SET_SIZE(self, len + 1); return 0; } @@ -43,7 +45,7 @@ _Py_memory_repeat(char* dest, Py_ssize_t len_dest, Py_ssize_t len_src) Py_ssize_t copied = len_src; while (copied < len_dest) { Py_ssize_t bytes_to_copy = Py_MIN(copied, len_dest - copied); - memcpy(dest + copied, dest, bytes_to_copy); + memcpy(dest + copied, dest, (size_t)bytes_to_copy); copied += bytes_to_copy; } } @@ -54,7 +56,10 @@ typedef struct { PyListObject *it_seq; /* Set to NULL when iterator is exhausted */ } _PyListIterObject; -extern PyObject *_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n); +union _PyStackRef; + +PyAPI_FUNC(PyObject *)_PyList_FromStackRefSteal(const union _PyStackRef *src, Py_ssize_t n); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 18a8896d97a548..57cbce8f126aca 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -13,47 +13,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t - - -// A mutex that occupies one byte. The lock can be zero initialized. -// -// Only the two least significant bits are used. The remaining bits should be -// zero: -// 0b00: unlocked -// 0b01: locked -// 0b10: unlocked and has parked threads -// 0b11: locked and has parked threads -// -// Typical initialization: -// PyMutex m = (PyMutex){0}; -// -// Typical usage: -// PyMutex_Lock(&m); -// ... -// PyMutex_Unlock(&m); - -// NOTE: In Py_GIL_DISABLED builds, `struct _PyMutex` is defined in Include/object.h. -// The Py_GIL_DISABLED builds need the definition in Include/object.h for the -// `ob_mutex` field in PyObject. For the default (non-free-threaded) build, -// we define the struct here to avoid exposing it in the public API. -#ifndef Py_GIL_DISABLED -struct _PyMutex { uint8_t v; }; -#endif - -typedef struct _PyMutex PyMutex; - -#define _Py_UNLOCKED 0 -#define _Py_LOCKED 1 +//_Py_UNLOCKED is defined as 0 and _Py_LOCKED as 1 in Include/cpython/lock.h #define _Py_HAS_PARKED 2 #define _Py_ONCE_INITIALIZED 4 -// (private) slow path for locking the mutex -PyAPI_FUNC(void) _PyMutex_LockSlow(PyMutex *m); - -// (private) slow path for unlocking the mutex -PyAPI_FUNC(void) _PyMutex_UnlockSlow(PyMutex *m); - static inline int PyMutex_LockFast(uint8_t *lock_bits) { @@ -61,35 +24,11 @@ PyMutex_LockFast(uint8_t *lock_bits) return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); } -// Locks the mutex. -// -// If the mutex is currently locked, the calling thread will be parked until -// the mutex is unlocked. If the current thread holds the GIL, then the GIL -// will be released while the thread is parked. -static inline void -PyMutex_Lock(PyMutex *m) -{ - uint8_t expected = _Py_UNLOCKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_LOCKED)) { - _PyMutex_LockSlow(m); - } -} - -// Unlocks the mutex. -static inline void -PyMutex_Unlock(PyMutex *m) -{ - uint8_t expected = _Py_LOCKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_UNLOCKED)) { - _PyMutex_UnlockSlow(m); - } -} - // Checks if the mutex is currently locked. static inline int PyMutex_IsLocked(PyMutex *m) { - return (_Py_atomic_load_uint8(&m->v) & _Py_LOCKED) != 0; + return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; } // Re-initializes the mutex after a fork to the unlocked state. @@ -113,20 +52,20 @@ typedef enum _PyLockFlags { // Lock a mutex with an optional timeout and additional options. See // _PyLockFlags for details. extern PyLockStatus -_PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout_ns, _PyLockFlags flags); +_PyMutex_LockTimed(PyMutex *m, PyTime_t timeout_ns, _PyLockFlags flags); -// Lock a mutex with aditional options. See _PyLockFlags for details. +// Lock a mutex with additional options. See _PyLockFlags for details. static inline void PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) { uint8_t expected = _Py_UNLOCKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_LOCKED)) { + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_LOCKED)) { _PyMutex_LockTimed(m, -1, flags); } } -// Unlock a mutex, returns 0 if the mutex is not locked (used for improved -// error messages). +// Unlock a mutex, returns -1 if the mutex is not locked (used for improved +// error messages) otherwise returns 0. extern int _PyMutex_TryUnlock(PyMutex *m); @@ -135,6 +74,10 @@ typedef struct { uint8_t v; } PyEvent; +// Check if the event is set without blocking. Returns 1 if the event is set or +// 0 otherwise. +PyAPI_FUNC(int) _PyEvent_IsSet(PyEvent *evt); + // Set the event and notify any waiting threads. // Export for '_testinternalcapi' shared extension PyAPI_FUNC(void) _PyEvent_Notify(PyEvent *evt); @@ -145,9 +88,10 @@ PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); // Wait for the event to be set, or until the timeout expires. If the event is // already set, then this returns immediately. Returns 1 if the event was set, -// and 0 if the timeout expired or thread was interrupted. -PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns); - +// and 0 if the timeout expired or thread was interrupted. If `detach` is +// true, then the thread will detach/release the GIL while waiting. +PyAPI_FUNC(int) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach); // _PyRawMutex implements a word-sized mutex that that does not depend on the // parking lot API, and therefore can be used in the parking lot @@ -184,12 +128,6 @@ _PyRawMutex_Unlock(_PyRawMutex *m) _PyRawMutex_UnlockSlow(m); } -// A data structure that can be used to run initialization code once in a -// thread-safe manner. The C++11 equivalent is std::call_once. -typedef struct { - uint8_t v; -} _PyOnceFlag; - // Type signature for one-time initialization functions. The function should // return 0 on success and -1 on failure. typedef int _Py_once_fn_t(void *arg); @@ -213,6 +151,19 @@ _PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) return _PyOnceFlag_CallOnceSlow(flag, fn, arg); } +// A recursive mutex. The mutex should zero-initialized. +typedef struct { + PyMutex mutex; + unsigned long long thread; // i.e., PyThread_get_thread_ident_ex() + size_t level; +} _PyRecursiveMutex; + +PyAPI_FUNC(int) _PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m); +PyAPI_FUNC(void) _PyRecursiveMutex_Lock(_PyRecursiveMutex *m); +extern PyLockStatus _PyRecursiveMutex_LockTimed(_PyRecursiveMutex *m, PyTime_t timeout, _PyLockFlags flags); +PyAPI_FUNC(void) _PyRecursiveMutex_Unlock(_PyRecursiveMutex *m); +extern int _PyRecursiveMutex_TryUnlock(_PyRecursiveMutex *m); + // A readers-writer (RW) lock. The lock supports multiple concurrent readers or // a single writer. The lock is write-preferring: if a writer is waiting while // the lock is read-locked then, new readers will be blocked. This avoids @@ -251,6 +202,39 @@ PyAPI_FUNC(void) _PyRWMutex_RUnlock(_PyRWMutex *rwmutex); PyAPI_FUNC(void) _PyRWMutex_Lock(_PyRWMutex *rwmutex); PyAPI_FUNC(void) _PyRWMutex_Unlock(_PyRWMutex *rwmutex); +// Similar to linux seqlock: https://en.wikipedia.org/wiki/Seqlock +// We use a sequence number to lock the writer, an even sequence means we're unlocked, an odd +// sequence means we're locked. Readers will read the sequence before attempting to read the +// underlying data and then read the sequence number again after reading the data. If the +// sequence has not changed the data is valid. +// +// Differs a little bit in that we use CAS on sequence as the lock, instead of a separate spin lock. +// The writer can also detect that the undelering data has not changed and abandon the write +// and restore the previous sequence. +typedef struct { + uint32_t sequence; +} _PySeqLock; + +// Lock the sequence lock for the writer +PyAPI_FUNC(void) _PySeqLock_LockWrite(_PySeqLock *seqlock); + +// Unlock the sequence lock and move to the next sequence number. +PyAPI_FUNC(void) _PySeqLock_UnlockWrite(_PySeqLock *seqlock); + +// Abandon the current update indicating that no mutations have occurred +// and restore the previous sequence value. +PyAPI_FUNC(void) _PySeqLock_AbandonWrite(_PySeqLock *seqlock); + +// Begin a read operation and return the current sequence number. +PyAPI_FUNC(uint32_t) _PySeqLock_BeginRead(_PySeqLock *seqlock); + +// End the read operation and confirm that the sequence number has not changed. +// Returns 1 if the read was successful or 0 if the read should be retried. +PyAPI_FUNC(int) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous); + +// Check if the lock was held during a fork and clear the lock. Returns 1 +// if the lock was held and any associated data should be cleared. +PyAPI_FUNC(int) _PySeqLock_AfterFork(_PySeqLock *seqlock); #ifdef __cplusplus } diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index ec27df9e416c58..196b4152280a35 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -47,17 +47,6 @@ extern "C" { # error "_PY_LONG_DEFAULT_MAX_STR_DIGITS smaller than threshold." #endif -// _PyLong_NumBits. Return the number of bits needed to represent the -// absolute value of a long. For example, this returns 1 for 1 and -1, 2 -// for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. -// v must not be NULL, and must be a normalized long. -// (size_t)-1 is returned and OverflowError set if the true result doesn't -// fit in a size_t. -// -// Export for 'math' shared extension. -PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); - - /* runtime lifecycle */ extern PyStatus _PyLong_InitTypes(PyInterpreterState *); @@ -90,14 +79,13 @@ static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) } // _PyLong_Frexp returns a double x and an exponent e such that the -// true value is approximately equal to x * 2**e. e is >= 0. x is +// true value is approximately equal to x * 2**e. x is // 0.0 if and only if the input is 0 (in which case, e and x are both -// zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is -// possible if the number of bits doesn't fit into a Py_ssize_t, sets -// OverflowError and returns -1.0 for x, 0 for e. +// zeroes); otherwise, 0.5 <= abs(x) < 1.0. +// Always successful. // // Export for 'math' shared extension -PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); +PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, int64_t *e); extern PyObject* _PyLong_FromBytes(const char *, Py_ssize_t, int); @@ -116,14 +104,14 @@ PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t); +PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, size_t); +PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); -extern PyObject* _PyLong_Add(PyLongObject *left, PyLongObject *right); -extern PyObject* _PyLong_Multiply(PyLongObject *left, PyLongObject *right); -extern PyObject* _PyLong_Subtract(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Subtract(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; @@ -189,8 +177,12 @@ PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); * we define them to the numbers in both places and then assert that * they're the same. */ -static_assert(SIGN_MASK == _PyLong_SIGN_MASK, "SIGN_MASK does not match _PyLong_SIGN_MASK"); -static_assert(NON_SIZE_BITS == _PyLong_NON_SIZE_BITS, "NON_SIZE_BITS does not match _PyLong_NON_SIZE_BITS"); +#if SIGN_MASK != _PyLong_SIGN_MASK +# error "SIGN_MASK does not match _PyLong_SIGN_MASK" +#endif +#if NON_SIZE_BITS != _PyLong_NON_SIZE_BITS +# error "NON_SIZE_BITS does not match _PyLong_NON_SIZE_BITS" +#endif /* All *compact" values are guaranteed to fit into * a Py_ssize_t with at least one bit to spare. @@ -235,7 +227,7 @@ static inline Py_ssize_t _PyLong_DigitCount(const PyLongObject *op) { assert(PyLong_Check(op)); - return op->long_value.lv_tag >> NON_SIZE_BITS; + return (Py_ssize_t)(op->long_value.lv_tag >> NON_SIZE_BITS); } /* Equivalent to _PyLong_DigitCount(op) * _PyLong_NonCompactSign(op) */ @@ -251,7 +243,7 @@ static inline int _PyLong_CompactSign(const PyLongObject *op) { assert(PyLong_Check(op)); - assert(_PyLong_IsCompact(op)); + assert(_PyLong_IsCompact((PyLongObject *)op)); return 1 - (op->long_value.lv_tag & SIGN_MASK); } @@ -259,7 +251,7 @@ static inline int _PyLong_NonCompactSign(const PyLongObject *op) { assert(PyLong_Check(op)); - assert(!_PyLong_IsCompact(op)); + assert(!_PyLong_IsCompact((PyLongObject *)op)); return 1 - (op->long_value.lv_tag & SIGN_MASK); } @@ -270,7 +262,8 @@ _PyLong_SameSign(const PyLongObject *a, const PyLongObject *b) return (a->long_value.lv_tag & SIGN_MASK) == (b->long_value.lv_tag & SIGN_MASK); } -#define TAG_FROM_SIGN_AND_SIZE(sign, size) ((1 - (sign)) | ((size) << NON_SIZE_BITS)) +#define TAG_FROM_SIGN_AND_SIZE(sign, size) \ + ((uintptr_t)(1 - (sign)) | ((uintptr_t)(size) << NON_SIZE_BITS)) static inline void _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) @@ -278,7 +271,7 @@ _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) assert(size >= 0); assert(-1 <= sign && sign <= 1); assert(sign != 0 || size == 0); - op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, (size_t)size); + op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, size); } static inline void @@ -288,7 +281,7 @@ _PyLong_SetDigitCount(PyLongObject *op, Py_ssize_t size) op->long_value.lv_tag = (((size_t)size) << NON_SIZE_BITS) | (op->long_value.lv_tag & SIGN_MASK); } -#define NON_SIZE_MASK ~((1 << NON_SIZE_BITS) - 1) +#define NON_SIZE_MASK ~(uintptr_t)((1 << NON_SIZE_BITS) - 1) static inline void _PyLong_FlipSign(PyLongObject *op) { diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h new file mode 100644 index 00000000000000..14e29576875c6e --- /dev/null +++ b/Include/internal/pycore_magic_number.h @@ -0,0 +1,287 @@ +#ifndef Py_INTERNAL_MAGIC_NUMBER_H +#define Py_INTERNAL_MAGIC_NUMBER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +/* + +Magic number to reject .pyc files generated by other Python versions. +It should change for each incompatible change to the bytecode. + +PYC_MAGIC_NUMBER must change whenever the bytecode emitted by the compiler may +no longer be understood by older implementations of the eval loop (usually due +to the addition of new opcodes). + +The value of CR and LF is incorporated so if you ever read or write +a .pyc file in text mode the magic number will be wrong; also, the +Apple MPW compiler swaps their values, botching string constants. + +There were a variety of old schemes for setting the magic number. Starting with +Python 3.11, Python 3.n starts with magic number 2900+50n. Within each minor +version, the magic number is incremented by 1 each time the file format changes. + +Known values: + Python 1.5: 20121 + Python 1.5.1: 20121 + Python 1.5.2: 20121 + Python 1.6: 50428 + Python 2.0: 50823 + Python 2.0.1: 50823 + Python 2.1: 60202 + Python 2.1.1: 60202 + Python 2.1.2: 60202 + Python 2.2: 60717 + Python 2.3a0: 62011 + Python 2.3a0: 62021 + Python 2.3a0: 62011 (!) + Python 2.4a0: 62041 + Python 2.4a3: 62051 + Python 2.4b1: 62061 + Python 2.5a0: 62071 + Python 2.5a0: 62081 (ast-branch) + Python 2.5a0: 62091 (with) + Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) + Python 2.5b3: 62101 (fix wrong code: for x, in ...) + Python 2.5b3: 62111 (fix wrong code: x += yield) + Python 2.5c1: 62121 (fix wrong lnotab with for loops and + storing constants that should have been removed) + Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) + Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) + Python 2.6a1: 62161 (WITH_CLEANUP optimization) + Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) + Python 2.7a0: 62181 (optimize conditional branches: + introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) + Python 2.7a0 62191 (introduce SETUP_WITH) + Python 2.7a0 62201 (introduce BUILD_SET) + Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) + Python 3000: 3000 + 3010 (removed UNARY_CONVERT) + 3020 (added BUILD_SET) + 3030 (added keyword-only parameters) + 3040 (added signature annotations) + 3050 (print becomes a function) + 3060 (PEP 3115 metaclass syntax) + 3061 (string literals become unicode) + 3071 (PEP 3109 raise changes) + 3081 (PEP 3137 make __file__ and __name__ unicode) + 3091 (kill str8 interning) + 3101 (merge from 2.6a0, see 62151) + 3103 (__file__ points to source file) + Python 3.0a4: 3111 (WITH_CLEANUP optimization). + Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT + #3021) + Python 3.1a1: 3141 (optimize list, set and dict comprehensions: + change LIST_APPEND and SET_ADD, add MAP_ADD #2183) + Python 3.1a1: 3151 (optimize conditional branches: + introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE + #4715) + Python 3.2a1: 3160 (add SETUP_WITH #6101) + Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) + Python 3.2a3 3180 (add DELETE_DEREF #4617) + Python 3.3a1 3190 (__class__ super closure changed) + Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) + Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) + Python 3.3a2 3220 (changed PEP 380 implementation #14230) + Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) + Python 3.4a1 3250 (evaluate positional default arguments before + keyword-only defaults #16967) + Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override + free vars #17853) + Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) + Python 3.4a1 3280 (remove implicit class argument) + Python 3.4a4 3290 (changes to __qualname__ computation #19301) + Python 3.4a4 3300 (more changes to __qualname__ computation #19301) + Python 3.4rc2 3310 (alter __qualname__ computation #20625) + Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) + Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) + Python 3.5b2 3340 (fix dictionary display evaluation order #11205) + Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) + Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) + Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) + Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) + Python 3.6a2 3370 (16 bit wordcode #26647) + Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) + Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE + #27095) + Python 3.6b1 3373 (add BUILD_STRING opcode #27078) + Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes + #27985) + Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL + #27213) + Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) + Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) + Python 3.6rc1 3379 (more thorough __class__ validation #23722) + Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) + Python 3.7a2 3391 (update GET_AITER #31709) + Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) + Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) + Python 3.7b5 3394 (restored docstring as the first stmt in the body; + this might affected the first line number #32911) + Python 3.8a1 3400 (move frame block handling to compiler #17611) + Python 3.8a1 3401 (add END_ASYNC_FOR #33041) + Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) + Python 3.8b2 3411 (Reverse evaluation order of key: value in dict + comprehensions #35224) + Python 3.8b2 3412 (Swap the position of positional args and positional + only args in ast.arguments #37593) + Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) + Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) + Python 3.9a0 3421 (simplified bytecode for with blocks #32949) + Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) + Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) + Python 3.9a2 3424 (simplify bytecodes for *value unpacking) + Python 3.9a2 3425 (simplify bytecodes for **value unpacking) + Python 3.10a1 3430 (Make 'annotations' future by default) + Python 3.10a1 3431 (New line number table format -- PEP 626) + Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) + Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) + Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) + Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). + Python 3.10b1 3436 (Add GEN_START bytecode #43683) + Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) + Python 3.10b1 3438 Safer line number table handling. + Python 3.10b1 3439 (Add ROT_N) + Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) + Python 3.11a1 3451 (Add CALL_METHOD_KW) + Python 3.11a1 3452 (drop nlocals from marshaled code objects) + Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) + Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) + Python 3.11a1 3455 (add MAKE_CELL bpo-43693) + Python 3.11a1 3456 (interleave cell args bpo-43693) + Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) + Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) + Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) + Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) + Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) + Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change + MATCH_CLASS and MATCH_KEYS, and add COPY) + Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the + active exception) + Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*INPLACE_* into + BINARY_OP) + Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) + Python 3.11a4 3466 (bpo-45292: PEP-654 except*) + Python 3.11a4 3467 (Change CALL_xxx opcodes) + Python 3.11a4 3468 (Add SEND opcode) + Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) + Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) + Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) + Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) + Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) + Python 3.11a4 3474 (Add RESUME opcode) + Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) + Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) + Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and + ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) + Python 3.11a5 3478 (New CALL opcodes) + Python 3.11a5 3479 (Add PUSH_NULL opcode) + Python 3.11a5 3480 (New CALL opcodes, second iteration) + Python 3.11a5 3481 (Use inline cache for BINARY_OP) + Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) + Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) + Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and + STORE_ATTR) + Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) + Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) + Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) + Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) + Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) + Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) + Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, + add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) + Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) + Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) + Python 3.11a7 3494 (New location info table) + Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) + Python 3.12a1 3500 (Remove PRECALL opcode) + Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) + Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) + Python 3.12a1 3503 (Shrink LOAD_METHOD cache) + Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) + Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) + Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) + Python 3.12a1 3507 (Set lineno of module's RESUME to 0) + Python 3.12a1 3508 (Add CLEANUP_THROW) + Python 3.12a1 3509 (Conditional jumps only jump forward) + Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) + Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) + Python 3.12a2 3512 (Remove all unused consts from code objects) + Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) + Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) + Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) + Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) + Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) + Python 3.12a6 3518 (Add RETURN_CONST instruction) + Python 3.12a6 3519 (Modify SEND instruction) + Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) + Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) + Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) + Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) + Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) + Python 3.12b1 3525 (Shrink the CALL caches) + Python 3.12b1 3526 (Add instrumentation support) + Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) + Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) + Python 3.12b1 3529 (Inline list/dict/set comprehensions) + Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) + Python 3.12b1 3531 (Add PEP 695 changes) + Python 3.13a1 3550 (Plugin optimizer support) + Python 3.13a1 3551 (Compact superinstructions) + Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) + Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) + Python 3.13a1 3554 (more efficient bytecodes for f-strings) + Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) + Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) + Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) + Python 3.13a1 3558 (Reorder the stack items for CALL) + Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) + Python 3.13a1 3560 (Add RESUME_CHECK instruction) + Python 3.13a1 3561 (Add cache entry to branch instructions) + Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) + Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) + Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) + Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) + Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) + Python 3.13a1 3567 (Reimplement line number propagation by the compiler) + Python 3.13a1 3568 (Change semantics of END_FOR) + Python 3.13a5 3569 (Specialize CONTAINS_OP) + Python 3.13a6 3570 (Add __firstlineno__ class attribute) + Python 3.13b1 3571 (Fix miscompilation of private names in generic classes) + Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT) + Python 3.14a1 3601 (Fix miscompilation of private names in generic classes) + Python 3.14a1 3602 (Add LOAD_SPECIAL. Remove BEFORE_WITH and BEFORE_ASYNC_WITH) + Python 3.14a1 3603 (Remove BUILD_CONST_KEY_MAP) + Python 3.14a1 3604 (Do not duplicate test at end of while statements) + Python 3.14a1 3605 (Move ENTER_EXECUTOR to opcode 255) + Python 3.14a1 3606 (Specialize CALL_KW) + Python 3.14a1 3607 (Add pseudo instructions JUMP_IF_TRUE/FALSE) + Python 3.14a1 3608 (Add support for slices) + Python 3.14a2 3609 (Add LOAD_SMALL_INT and LOAD_CONST_IMMORTAL instructions, remove RETURN_CONST) + + Python 3.15 will start with 3650 + + Please don't copy-paste the same pre-release tag for new entries above!!! + You should always use the *upcoming* tag. For example, if 3.12a6 came out + a week ago, I should put "Python 3.12a7" next to my new magic number. + +Whenever PYC_MAGIC_NUMBER is changed, the ranges in the magic_values array in +PC/launcher.c must also be updated. + +*/ + +#define PYC_MAGIC_NUMBER 3609 +/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes + (little-endian) and then appending b'\r\n'. */ +#define PYC_MAGIC_NUMBER_TOKEN \ + ((uint32_t)PYC_MAGIC_NUMBER | ((uint32_t)'\r' << 16) | ((uint32_t)'\n' << 24)) + + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_MAGIC_NUMBER_H diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index 1e7ed5a4ca62e2..d870d01beb702c 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -20,18 +20,34 @@ typedef enum { #include "pycore_pymem.h" #ifdef WITH_MIMALLOC -#define MI_DEBUG_UNINIT PYMEM_CLEANBYTE -#define MI_DEBUG_FREED PYMEM_DEADBYTE -#define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE +# ifdef Py_GIL_DISABLED +# define MI_PRIM_THREAD_ID _Py_ThreadId +# endif +# define MI_DEBUG_UNINIT PYMEM_CLEANBYTE +# define MI_DEBUG_FREED PYMEM_DEADBYTE +# define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE #ifdef Py_DEBUG -# define MI_DEBUG 1 +# define MI_DEBUG 2 #else # define MI_DEBUG 0 #endif -#include "mimalloc.h" -#include "mimalloc/types.h" -#include "mimalloc/internal.h" +#ifdef _Py_THREAD_SANITIZER +# define MI_TSAN 1 +#endif + +#ifdef __cplusplus +extern "C++" { +#endif + +#include "mimalloc/mimalloc.h" +#include "mimalloc/mimalloc/types.h" +#include "mimalloc/mimalloc/internal.h" + +#ifdef __cplusplus +} +#endif + #endif #ifdef Py_GIL_DISABLED @@ -45,6 +61,8 @@ struct _mimalloc_thread_state { mi_heap_t *current_object_heap; mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT]; mi_tld_t tld; + int initialized; + struct llist_node page_list; }; #endif diff --git a/Include/internal/pycore_modsupport.h b/Include/internal/pycore_modsupport.h index 3d3cd6722528e9..11fde814875938 100644 --- a/Include/internal/pycore_modsupport.h +++ b/Include/internal/pycore_modsupport.h @@ -67,24 +67,6 @@ PyAPI_FUNC(void) _PyArg_BadArgument( // --- _PyArg_Parser API --------------------------------------------------- -typedef struct _PyArg_Parser { - const char *format; - const char * const *keywords; - const char *fname; - const char *custom_msg; - _PyOnceFlag once; /* atomic one-time initialization flag */ - int is_kwtuple_owned; /* does this parser own the kwtuple object? */ - int pos; /* number of positional-only arguments */ - int min; /* minimal number of arguments */ - int max; /* maximal number of positional arguments */ - PyObject *kwtuple; /* tuple of keyword parameter names */ - struct _PyArg_Parser *next; -} _PyArg_Parser; - -// Export for '_testclinic' shared extension -PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, - struct _PyArg_Parser *, ...); - // Export for '_dbm' shared extension PyAPI_FUNC(int) _PyArg_ParseStackAndKeywords( PyObject *const *args, diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 5644bbe5e0552b..9bb282a13a9659 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -11,6 +11,8 @@ extern "C" { extern void _PyModule_Clear(PyObject *); extern void _PyModule_ClearDict(PyObject *); extern int _PyModuleSpec_IsInitializing(PyObject *); +extern int _PyModuleSpec_GetFileOrigin(PyObject *, PyObject **); +extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); @@ -22,6 +24,9 @@ typedef struct { PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; +#ifdef Py_GIL_DISABLED + void *md_gil; +#endif } PyModuleObject; static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { @@ -43,7 +48,7 @@ static inline PyObject* _PyModule_GetDict(PyObject *mod) { } PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress); -PyObject* _Py_module_getattro(PyModuleObject *m, PyObject *name); +PyObject* _Py_module_getattro(PyObject *m, PyObject *name); #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 34a83ea228e8b1..c7af720b1ce43d 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -12,7 +12,21 @@ extern "C" { #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall() #include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_uniqueid.h" // _PyObject_ThreadIncrefSlow() + +// This value is added to `ob_ref_shared` for objects that use deferred +// reference counting so that they are not immediately deallocated when the +// non-deferred reference count drops to zero. +// +// The value is half the maximum shared refcount because the low two bits of +// `ob_ref_shared` are used for flags. +#define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8) + +/* For backwards compatibility -- Do not use this */ +#define _Py_IsImmortalLoose(op) _Py_IsImmortal + /* Check if an object is consistent. For example, ensure that the reference counter is greater than or equal to 1, and ensure that ob_type is not NULL. @@ -63,7 +77,7 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); #else #define _PyObject_HEAD_INIT(type) \ { \ - .ob_refcnt = _Py_IMMORTAL_REFCNT, \ + .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ .ob_type = (type) \ } #endif @@ -73,7 +87,7 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); .ob_size = size \ } -extern void _Py_NO_RETURN _Py_FatalRefcountErrorFunc( +PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( const char *func, const char *message); @@ -86,9 +100,9 @@ extern void _Py_NO_RETURN _Py_FatalRefcountErrorFunc( built against the pre-3.12 stable ABI. */ PyAPI_DATA(Py_ssize_t) _Py_RefTotal; -extern void _Py_AddRefTotal(PyInterpreterState *, Py_ssize_t); -extern void _Py_IncRefTotal(PyInterpreterState *); -extern void _Py_DecRefTotal(PyInterpreterState *); +extern void _Py_AddRefTotal(PyThreadState *, Py_ssize_t); +extern void _Py_IncRefTotal(PyThreadState *); +extern void _Py_DecRefTotal(PyThreadState *); # define _Py_DEC_REFTOTAL(interp) \ interp->object_state.reftotal-- @@ -101,7 +115,7 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) return; } #ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyInterpreterState_GET(), n); + _Py_AddRefTotal(_PyThreadState_GET(), n); #endif #if !defined(Py_GIL_DISABLED) op->ob_refcnt += n; @@ -125,19 +139,25 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) } #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) -static inline void _Py_SetImmortal(PyObject *op) +// Checks if an object has a single, unique reference. If the caller holds a +// unique reference, it may be able to safely modify the object in-place. +static inline int +_PyObject_IsUniquelyReferenced(PyObject *ob) { - if (op) { -#ifdef Py_GIL_DISABLED - op->ob_tid = _Py_UNOWNED_TID; - op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - op->ob_ref_shared = 0; +#if !defined(Py_GIL_DISABLED) + return Py_REFCNT(ob) == 1; #else - op->ob_refcnt = _Py_IMMORTAL_REFCNT; + // NOTE: the entire ob_ref_shared field must be zero, including flags, to + // ensure that other threads cannot concurrently create new references to + // this object. + return (_Py_IsOwnedByCurrentThread(ob) && + _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 && + _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0); #endif - } } -#define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op)) + +PyAPI_FUNC(void) _Py_SetImmortal(PyObject *op); +PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op); // Makes an immortal object mortal again with the specified refcnt. Should only // be used during runtime finalization. @@ -174,6 +194,7 @@ static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); @@ -187,6 +208,11 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_DESTROY, data); + } destruct(op); } } @@ -195,6 +221,7 @@ static inline void _Py_DECREF_NO_DEALLOC(PyObject *op) { if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); @@ -252,12 +279,12 @@ extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); when a memory block is reused from a free list. Internal function called by _Py_NewReference(). */ -extern int _PyTraceMalloc_NewReference(PyObject *op); +extern int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void*); // Fast inlined version of PyType_HasFeature() static inline int _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { - return ((type->tp_flags & feature) != 0); + return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0); } extern void _PyType_InitCache(PyInterpreterState *interp); @@ -266,6 +293,117 @@ extern PyStatus _PyObject_InitState(PyInterpreterState *interp); extern void _PyObject_FiniState(PyInterpreterState *interp); extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); +// Macros used for per-thread reference counting in the free threading build. +// They resolve to normal Py_INCREF/DECREF calls in the default build. +// +// The macros are used for only a few references that would otherwise cause +// scaling bottlenecks in the free threading build: +// - The reference from an object to `ob_type`. +// - The reference from a function to `func_code`. +// - The reference from a function to `func_globals` and `func_builtins`. +// +// It's safe, but not performant or necessary, to use these macros for other +// references to code, type, or dict objects. It's also safe to mix their +// usage with normal Py_INCREF/DECREF calls. +// +// See also Include/internal/pycore_dict.h for _Py_INCREF_DICT/_Py_DECREF_DICT. +#ifndef Py_GIL_DISABLED +# define _Py_INCREF_TYPE Py_INCREF +# define _Py_DECREF_TYPE Py_DECREF +# define _Py_INCREF_CODE Py_INCREF +# define _Py_DECREF_CODE Py_DECREF +#else +static inline void +_Py_THREAD_INCREF_OBJECT(PyObject *obj, Py_ssize_t unique_id) +{ + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + + // Unsigned comparison so that `unique_id=-1`, which indicates that + // per-thread refcounting has been disabled on this object, is handled by + // the "else". + if ((size_t)unique_id < (size_t)tstate->refcounts.size) { +# ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +# endif + _Py_INCREF_STAT_INC(); + tstate->refcounts.values[unique_id]++; + } + else { + // The slow path resizes the per-thread refcount array if necessary. + // It handles the unique_id=-1 case to keep the inlinable function smaller. + _PyObject_ThreadIncrefSlow(obj, unique_id); + } +} + +static inline void +_Py_INCREF_TYPE(PyTypeObject *type) +{ + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + assert(_Py_IsImmortal(type)); + _Py_INCREF_IMMORTAL_STAT_INC(); + return; + } + + // gh-122974: GCC 11 warns about the access to PyHeapTypeObject fields when + // _Py_INCREF_TYPE() is called on a statically allocated type. The + // _PyType_HasFeature check above ensures that the type is a heap type. +#if defined(__GNUC__) && __GNUC__ >= 11 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Warray-bounds" +#endif + _Py_THREAD_INCREF_OBJECT((PyObject *)type, ((PyHeapTypeObject *)type)->unique_id); +#if defined(__GNUC__) && __GNUC__ >= 11 +# pragma GCC diagnostic pop +#endif +} + +static inline void +_Py_INCREF_CODE(PyCodeObject *co) +{ + _Py_THREAD_INCREF_OBJECT((PyObject *)co, co->_co_unique_id); +} + +static inline void +_Py_THREAD_DECREF_OBJECT(PyObject *obj, Py_ssize_t unique_id) +{ + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + + // Unsigned comparison so that `unique_id=-1`, which indicates that + // per-thread refcounting has been disabled on this object, is handled by + // the "else". + if ((size_t)unique_id < (size_t)tstate->refcounts.size) { +# ifdef Py_REF_DEBUG + _Py_DECREF_DecRefTotal(); +# endif + _Py_DECREF_STAT_INC(); + tstate->refcounts.values[unique_id]--; + } + else { + // Directly decref the object if the id is not assigned or if + // per-thread refcounting has been disabled on this object. + Py_DECREF(obj); + } +} + +static inline void +_Py_DECREF_TYPE(PyTypeObject *type) +{ + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + assert(_Py_IsImmortal(type)); + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)type; + _Py_THREAD_DECREF_OBJECT((PyObject *)type, ht->unique_id); +} + +static inline void +_Py_DECREF_CODE(PyCodeObject *co) +{ + _Py_THREAD_DECREF_OBJECT((PyObject *)co, co->_co_unique_id); +} +#endif + /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and _PyObject_InitVar() is the fast version of PyObject_InitVar(). @@ -276,9 +414,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) { assert(op != NULL); Py_SET_TYPE(op, typeobj); - if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) { - Py_INCREF(typeobj); - } + assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); + _Py_INCREF_TYPE(typeobj); _Py_NewReference(op); } @@ -316,7 +453,7 @@ static inline void _PyObject_GC_TRACK( "object already tracked by the garbage collector", filename, lineno, __func__); #ifdef Py_GIL_DISABLED - op->ob_gc_bits |= _PyGC_BITS_TRACKED; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_TRACKED); #else PyGC_Head *gc = _Py_AS_GC(op); _PyObject_ASSERT_FROM(op, @@ -325,11 +462,12 @@ static inline void _PyObject_GC_TRACK( filename, lineno, __func__); PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = interp->gc.generation0; + PyGC_Head *generation0 = &interp->gc.young.head; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - _PyGCHead_SET_NEXT(gc, generation0); + /* Young objects will be moved into the visited space during GC, so set the bit here */ + gc->_gc_next = ((uintptr_t)generation0) | (uintptr_t)interp->gc.visited_space; generation0->_gc_prev = (uintptr_t)gc; #endif } @@ -356,7 +494,7 @@ static inline void _PyObject_GC_UNTRACK( filename, lineno, __func__); #ifdef Py_GIL_DISABLED - op->ob_gc_bits &= ~_PyGC_BITS_TRACKED; + _PyObject_CLEAR_GC_BITS(op, _PyGC_BITS_TRACKED); #else PyGC_Head *gc = _Py_AS_GC(op); PyGC_Head *prev = _PyGCHead_PREV(gc); @@ -398,13 +536,14 @@ _Py_TryIncrefFast(PyObject *op) { local += 1; if (local == 0) { // immortal + _Py_INCREF_IMMORTAL_STAT_INC(); return 1; } if (_Py_IsOwnedByCurrentThread(op)) { _Py_INCREF_STAT_INC(); _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif return 1; } @@ -427,7 +566,7 @@ _Py_TryIncRefShared(PyObject *op) &shared, shared + (1 << _Py_REF_SHARED_SHIFT))) { #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif _Py_INCREF_STAT_INC(); return 1; @@ -437,7 +576,7 @@ _Py_TryIncRefShared(PyObject *op) /* Tries to incref the object op and ensures that *src still points to it. */ static inline int -_Py_TryIncref(PyObject **src, PyObject *op) +_Py_TryIncrefCompare(PyObject **src, PyObject *op) { if (_Py_TryIncrefFast(op)) { return 1; @@ -463,7 +602,7 @@ _Py_XGetRef(PyObject **ptr) if (value == NULL) { return value; } - if (_Py_TryIncref(ptr, value)) { + if (_Py_TryIncrefCompare(ptr, value)) { return value; } } @@ -478,7 +617,7 @@ _Py_TryXGetRef(PyObject **ptr) if (value == NULL) { return value; } - if (_Py_TryIncref(ptr, value)) { + if (_Py_TryIncrefCompare(ptr, value)) { return value; } return NULL; @@ -492,6 +631,9 @@ _Py_NewRefWithLock(PyObject *op) if (_Py_TryIncrefFast(op)) { return op; } +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif _Py_INCREF_STAT_INC(); for (;;) { Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); @@ -517,7 +659,41 @@ _Py_XNewRefWithLock(PyObject *obj) return _Py_NewRefWithLock(obj); } +static inline void +_PyObject_SetMaybeWeakref(PyObject *op) +{ + if (_Py_IsImmortal(op)) { + return; + } + for (;;) { + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) { + // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states. + return; + } + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) { + return; + } + } +} + +#endif + +/* Tries to incref op and returns 1 if successful or 0 otherwise. */ +static inline int +_Py_TryIncref(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return _Py_TryIncrefFast(op) || _Py_TryIncRefShared(op); +#else + if (Py_REFCNT(op) > 0) { + Py_INCREF(op); + return 1; + } + return 0; #endif +} #ifdef Py_REF_DEBUG extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *); @@ -547,7 +723,7 @@ _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) if (PyType_Check(op) && ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState( + managed_static_type_state *state = _PyStaticType_GetState( interp, (PyTypeObject *)op); return _PyStaticType_GET_WEAKREFS_LISTPTR(state); } @@ -577,27 +753,40 @@ _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op) return (PyWeakReference **)((char *)op + offset); } +// Fast inlined version of PyType_IS_GC() +#define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) // Fast inlined version of PyObject_IS_GC() static inline int _PyObject_IS_GC(PyObject *obj) { PyTypeObject *type = Py_TYPE(obj); - return (PyType_IS_GC(type) + return (_PyType_IS_GC(type) && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); } -// Fast inlined version of PyType_IS_GC() -#define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) +// Fast inlined version of PyObject_Hash() +static inline Py_hash_t +_PyObject_HashFast(PyObject *op) +{ + if (PyUnicode_CheckExact(op)) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( + _PyASCIIObject_CAST(op)->hash); + if (hash != -1) { + return hash; + } + } + return PyObject_Hash(op); +} static inline size_t _PyType_PreHeaderSize(PyTypeObject *tp) { return ( #ifndef Py_GIL_DISABLED - _PyType_IS_GC(tp) * sizeof(PyGC_Head) + + (size_t)_PyType_IS_GC(tp) * sizeof(PyGC_Head) + #endif - _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) + (size_t)_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) ); } @@ -615,18 +804,18 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); -extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); +PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); +extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); -extern int _PyObject_InitializeDict(PyObject *obj); -int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); -extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name, PyObject *value); -PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name); +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); +extern int _PyObject_StoreInstanceAttribute(PyObject *obj, + PyObject *name, PyObject *value); +extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, + PyObject **attr); #ifdef Py_GIL_DISABLED # define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) @@ -637,54 +826,43 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, #endif typedef union { - PyObject *dict; - /* Use a char* to generate a warning if directly assigning a PyDictValues */ - char *values; -} PyDictOrValues; + PyDictObject *dict; +} PyManagedDictPointer; -static inline PyDictOrValues * -_PyObject_DictOrValuesPointer(PyObject *obj) +static inline PyManagedDictPointer * +_PyObject_ManagedDictPointer(PyObject *obj) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET); + return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET); } -static inline int -_PyDictOrValues_IsValues(PyDictOrValues dorv) +static inline PyDictObject * +_PyObject_GetManagedDict(PyObject *obj) { - return ((uintptr_t)dorv.values) & 1; + PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj); + return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict); } static inline PyDictValues * -_PyDictOrValues_GetValues(PyDictOrValues dorv) +_PyObject_InlineValues(PyObject *obj) { - assert(_PyDictOrValues_IsValues(dorv)); - return (PyDictValues *)(dorv.values + 1); -} - -static inline PyObject * -_PyDictOrValues_GetDict(PyDictOrValues dorv) -{ - assert(!_PyDictOrValues_IsValues(dorv)); - return dorv.dict; -} - -static inline void -_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) -{ - ptr->values = ((char *)values) - 1; + PyTypeObject *tp = Py_TYPE(obj); + assert(tp->tp_basicsize > 0 && (size_t)tp->tp_basicsize % sizeof(PyObject *) == 0); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + return (PyDictValues *)((char *)obj + tp->tp_basicsize); } extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); -extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); // Export for 'math' shared extension PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *); +PyAPI_FUNC(PyObject*) _PyObject_LookupSpecialMethod(PyObject *self, PyObject *attr, PyObject **self_or_null); extern int _PyObject_IsAbstract(PyObject *); -extern int _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); extern PyObject* _PyObject_NextNotImplemented(PyObject *); // Pickle support. @@ -705,13 +883,7 @@ PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *); * Third party code unintentionally rely on problematic fpcasts. The call * trampoline mitigates common occurrences of bad fpcasts on Emscripten. */ -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -#define _PyCFunction_TrampolineCall(meth, self, args) \ - _PyCFunctionWithKeywords_TrampolineCall( \ - (*(PyCFunctionWithKeywords)(void(*)(void))(meth)), (self), (args), NULL) -extern PyObject* _PyCFunctionWithKeywords_TrampolineCall( - PyCFunctionWithKeywords meth, PyObject *, PyObject *, PyObject *); -#else +#if !(defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE)) #define _PyCFunction_TrampolineCall(meth, self, args) \ (meth)((self), (args)) #define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \ @@ -726,6 +898,8 @@ PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; // Export for the stable ABI. PyAPI_DATA(int) _Py_SwappedOp[]; +extern void _Py_GetConstant_Init(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_object_deferred.h b/Include/internal/pycore_object_deferred.h new file mode 100644 index 00000000000000..c070d768b7d771 --- /dev/null +++ b/Include/internal/pycore_object_deferred.h @@ -0,0 +1,32 @@ +#ifndef Py_INTERNAL_OBJECT_DEFERRED_H +#define Py_INTERNAL_OBJECT_DEFERRED_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "pycore_gc.h" + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Mark an object as supporting deferred reference counting. This is a no-op +// in the default (with GIL) build. Objects that use deferred reference +// counting should be tracked by the GC so that they are eventually collected. +extern void _PyObject_SetDeferredRefcount(PyObject *op); + +static inline int +_PyObject_HasDeferredRefcount(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED); +#else + return 0; +#endif +} + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_OBJECT_DEFERRED_H diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h index fc130b1e9920b4..39e69b7cde52a1 100644 --- a/Include/internal/pycore_object_stack.h +++ b/Include/internal/pycore_object_stack.h @@ -1,8 +1,6 @@ #ifndef Py_INTERNAL_OBJECT_STACK_H #define Py_INTERNAL_OBJECT_STACK_H -#include "pycore_freelist.h" // _PyFreeListState - #ifdef __cplusplus extern "C" { #endif @@ -73,6 +71,16 @@ _PyObjectStack_Pop(_PyObjectStack *stack) return obj; } +static inline Py_ssize_t +_PyObjectStack_Size(_PyObjectStack *stack) +{ + Py_ssize_t size = 0; + for (_PyObjectStackChunk *buf = stack->head; buf != NULL; buf = buf->prev) { + size += buf->n; + } + return size; +} + // Merge src into dst, leaving src empty extern void _PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src); diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 9eac27b1a9a4e3..8a47a6d9e6eb0d 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -8,7 +8,16 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_freelist_state.h" // _Py_freelists +#include "pycore_hashtable.h" // _Py_hashtable_t + + +/* Reference tracer state */ +struct _reftracer_runtime_state { + PyRefTracer tracer_func; + void* tracer_data; +}; + struct _py_object_runtime_state { #ifdef Py_REF_DEBUG @@ -18,6 +27,9 @@ struct _py_object_runtime_state { }; struct _py_object_state { +#if !defined(Py_GIL_DISABLED) + struct _Py_freelists freelists; +#endif #ifdef Py_REF_DEBUG Py_ssize_t reftotal; #endif diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 9140d8f08f0af1..a7ba8f340737aa 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -255,8 +255,8 @@ struct pool_header { union { pymem_block *_padding; uint count; } ref; /* number of allocated blocks */ pymem_block *freeblock; /* pool's free list head */ - struct pool_header *nextpool; /* next pool of this size class */ - struct pool_header *prevpool; /* previous pool "" */ + struct pool_header *nextpool; /* see "Pool table" for meaning */ + struct pool_header *prevpool; /* " */ uint arenaindex; /* index into arenas of base adr */ uint szidx; /* block size class index */ uint nextoffset; /* bytes to virgin block */ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 75d7f44025328e..58e583eabbcc46 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -20,12 +20,10 @@ extern "C" { #define IS_PSEUDO_INSTR(OP) ( \ ((OP) == LOAD_CLOSURE) || \ ((OP) == STORE_FAST_MAYBE_NULL) || \ - ((OP) == LOAD_SUPER_METHOD) || \ - ((OP) == LOAD_ZERO_SUPER_METHOD) || \ - ((OP) == LOAD_ZERO_SUPER_ATTR) || \ - ((OP) == LOAD_METHOD) || \ ((OP) == JUMP) || \ ((OP) == JUMP_NO_INTERRUPT) || \ + ((OP) == JUMP_IF_FALSE) || \ + ((OP) == JUMP_IF_TRUE) || \ ((OP) == SETUP_FINALLY) || \ ((OP) == SETUP_CLEANUP) || \ ((OP) == SETUP_WITH) || \ @@ -37,10 +35,6 @@ extern int _PyOpcode_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA int _PyOpcode_num_popped(int opcode, int oparg) { switch(opcode) { - case BEFORE_ASYNC_WITH: - return 1; - case BEFORE_WITH: - return 1; case BINARY_OP: return 2; case BINARY_OP_ADD_FLOAT: @@ -73,8 +67,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case BINARY_SUBSCR_TUPLE_INT: return 2; - case BUILD_CONST_KEY_MAP: - return 1 + oparg; case BUILD_LIST: return oparg; case BUILD_MAP: @@ -95,6 +87,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_BOUND_METHOD_EXACT_ARGS: return 2 + oparg; + case CALL_BOUND_METHOD_GENERAL: + return 2 + oparg; case CALL_BUILTIN_CLASS: return 2 + oparg; case CALL_BUILTIN_FAST: @@ -113,10 +107,16 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_KW: return 3 + oparg; + case CALL_KW_BOUND_METHOD: + return 3 + oparg; + case CALL_KW_NON_PY: + return 3 + oparg; + case CALL_KW_PY: + return 3 + oparg; case CALL_LEN: return 2 + oparg; case CALL_LIST_APPEND: - return 2 + oparg; + return 3; case CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -125,16 +125,18 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; + case CALL_NON_PY_GENERAL: + return 2 + oparg; case CALL_PY_EXACT_ARGS: return 2 + oparg; - case CALL_PY_WITH_DEFAULTS: + case CALL_PY_GENERAL: return 2 + oparg; case CALL_STR_1: - return 2 + oparg; + return 3; case CALL_TUPLE_1: - return 2 + oparg; + return 3; case CALL_TYPE_1: - return 2 + oparg; + return 3; case CHECK_EG_MATCH: return 2; case CHECK_EXC_MATCH: @@ -151,6 +153,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case CONTAINS_OP: return 2; + case CONTAINS_OP_DICT: + return 2; + case CONTAINS_OP_SET: + return 2; case CONVERT_VALUE: return 1; case COPY: @@ -216,7 +222,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case IMPORT_NAME: return 2; case INSTRUMENTED_CALL: - return 0; + return 2 + oparg; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -233,8 +239,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case INSTRUMENTED_JUMP_FORWARD: return 0; + case INSTRUMENTED_LINE: + return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: - return 3; + return 0; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -245,8 +253,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case INSTRUMENTED_RESUME: return 0; - case INSTRUMENTED_RETURN_CONST: - return 0; case INSTRUMENTED_RETURN_VALUE: return 1; case INSTRUMENTED_YIELD_VALUE: @@ -255,22 +261,30 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case IS_OP: return 2; + case JUMP: + return 0; case JUMP_BACKWARD: return 0; case JUMP_BACKWARD_NO_INTERRUPT: return 0; case JUMP_FORWARD: return 0; + case JUMP_IF_FALSE: + return 1; + case JUMP_IF_TRUE: + return 1; + case JUMP_NO_INTERRUPT: + return 0; case LIST_APPEND: return 2 + (oparg-1); case LIST_EXTEND: return 2 + (oparg-1); - case LOAD_ASSERTION_ERROR: - return 0; case LOAD_ATTR: return 1; case LOAD_ATTR_CLASS: return 1; + case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: + return 1; case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: return 1; case LOAD_ATTR_INSTANCE_VALUE: @@ -295,8 +309,14 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case LOAD_BUILD_CLASS: return 0; + case LOAD_CLOSURE: + return 0; + case LOAD_COMMON_CONSTANT: + return 0; case LOAD_CONST: return 0; + case LOAD_CONST_IMMORTAL: + return 0; case LOAD_DEREF: return 0; case LOAD_FAST: @@ -321,6 +341,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case LOAD_NAME: return 0; + case LOAD_SMALL_INT: + return 0; + case LOAD_SPECIAL: + return 1; case LOAD_SUPER_ATTR: return 3; case LOAD_SUPER_ATTR_ATTR: @@ -343,6 +367,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case NOP: return 0; + case POP_BLOCK: + return 0; case POP_EXCEPT: return 1; case POP_JUMP_IF_FALSE: @@ -369,8 +395,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; - case RETURN_CONST: - return 0; case RETURN_GENERATOR: return 0; case RETURN_VALUE: @@ -381,6 +405,12 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case SETUP_ANNOTATIONS: return 0; + case SETUP_CLEANUP: + return 0; + case SETUP_FINALLY: + return 0; + case SETUP_WITH: + return 0; case SET_ADD: return 2 + (oparg-1); case SET_FUNCTION_ATTRIBUTE: @@ -401,6 +431,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case STORE_FAST_LOAD_FAST: return 1; + case STORE_FAST_MAYBE_NULL: + return 1; case STORE_FAST_STORE_FAST: return 2; case STORE_GLOBAL: @@ -448,7 +480,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case UNPACK_SEQUENCE_TWO_TUPLE: return 1; case WITH_EXCEPT_START: - return 4; + return 5; case YIELD_VALUE: return 1; default: @@ -462,10 +494,6 @@ extern int _PyOpcode_num_pushed(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA int _PyOpcode_num_pushed(int opcode, int oparg) { switch(opcode) { - case BEFORE_ASYNC_WITH: - return 2; - case BEFORE_WITH: - return 2; case BINARY_OP: return 1; case BINARY_OP_ADD_FLOAT: @@ -491,15 +519,13 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case BINARY_SUBSCR_DICT: return 1; case BINARY_SUBSCR_GETITEM: - return 1; + return 0; case BINARY_SUBSCR_LIST_INT: return 1; case BINARY_SUBSCR_STR_INT: return 1; case BINARY_SUBSCR_TUPLE_INT: return 1; - case BUILD_CONST_KEY_MAP: - return 1; case BUILD_LIST: return 1; case BUILD_MAP: @@ -517,9 +543,11 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL: return 1; case CALL_ALLOC_AND_ENTER_INIT: - return 1; + return 0; case CALL_BOUND_METHOD_EXACT_ARGS: - return ((0) ? 1 : 0); + return 0; + case CALL_BOUND_METHOD_GENERAL: + return 0; case CALL_BUILTIN_CLASS: return 1; case CALL_BUILTIN_FAST: @@ -538,10 +566,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_KW: return 1; + case CALL_KW_BOUND_METHOD: + return 0; + case CALL_KW_NON_PY: + return 1; + case CALL_KW_PY: + return 0; case CALL_LEN: return 1; case CALL_LIST_APPEND: - return 1; + return 0; case CALL_METHOD_DESCRIPTOR_FAST: return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -550,10 +584,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_METHOD_DESCRIPTOR_O: return 1; - case CALL_PY_EXACT_ARGS: - return ((0) ? 1 : 0); - case CALL_PY_WITH_DEFAULTS: + case CALL_NON_PY_GENERAL: return 1; + case CALL_PY_EXACT_ARGS: + return 0; + case CALL_PY_GENERAL: + return 0; case CALL_STR_1: return 1; case CALL_TUPLE_1: @@ -576,6 +612,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CONTAINS_OP: return 1; + case CONTAINS_OP_DICT: + return 1; + case CONTAINS_OP_SET: + return 1; case CONVERT_VALUE: return 1; case COPY: @@ -617,7 +657,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case FOR_ITER: return 2; case FOR_ITER_GEN: - return 2; + return 1; case FOR_ITER_LIST: return 2; case FOR_ITER_RANGE: @@ -641,7 +681,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case IMPORT_NAME: return 1; case INSTRUMENTED_CALL: - return 0; + return 1; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -658,8 +698,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case INSTRUMENTED_JUMP_FORWARD: return 0; + case INSTRUMENTED_LINE: + return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: - return 1 + (oparg & 1); + return 0; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -670,58 +712,70 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case INSTRUMENTED_RESUME: return 0; - case INSTRUMENTED_RETURN_CONST: - return 0; case INSTRUMENTED_RETURN_VALUE: - return 0; + return 1; case INSTRUMENTED_YIELD_VALUE: return 1; case INTERPRETER_EXIT: return 0; case IS_OP: return 1; + case JUMP: + return 0; case JUMP_BACKWARD: return 0; case JUMP_BACKWARD_NO_INTERRUPT: return 0; case JUMP_FORWARD: return 0; + case JUMP_IF_FALSE: + return 1; + case JUMP_IF_TRUE: + return 1; + case JUMP_NO_INTERRUPT: + return 0; case LIST_APPEND: return 1 + (oparg-1); case LIST_EXTEND: return 1 + (oparg-1); - case LOAD_ASSERTION_ERROR: - return 1; case LOAD_ATTR: return 1 + (oparg & 1); case LOAD_ATTR_CLASS: return 1 + (oparg & 1); + case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: + return 1 + (oparg & 1); case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_INSTANCE_VALUE: return 1 + (oparg & 1); case LOAD_ATTR_METHOD_LAZY_DICT: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_METHOD_NO_DICT: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_METHOD_WITH_VALUES: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_MODULE: return 1 + (oparg & 1); case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_PROPERTY: - return 1 + ((0) ? 1 : 0); + return 0; case LOAD_ATTR_SLOT: return 1 + (oparg & 1); case LOAD_ATTR_WITH_HINT: return 1 + (oparg & 1); case LOAD_BUILD_CLASS: return 1; + case LOAD_CLOSURE: + return 1; + case LOAD_COMMON_CONSTANT: + return 1; case LOAD_CONST: return 1; + case LOAD_CONST_IMMORTAL: + return 1; case LOAD_DEREF: return 1; case LOAD_FAST: @@ -746,10 +800,14 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case LOAD_NAME: return 1; + case LOAD_SMALL_INT: + return 1; + case LOAD_SPECIAL: + return 2; case LOAD_SUPER_ATTR: return 1 + (oparg & 1); case LOAD_SUPER_ATTR_ATTR: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_SUPER_ATTR_METHOD: return 2; case MAKE_CELL: @@ -768,6 +826,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case NOP: return 0; + case POP_BLOCK: + return 0; case POP_EXCEPT: return 0; case POP_JUMP_IF_FALSE: @@ -794,18 +854,22 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; - case RETURN_CONST: - return 0; case RETURN_GENERATOR: - return 0; + return 1; case RETURN_VALUE: - return 0; + return 1; case SEND: return 2; case SEND_GEN: - return 2; + return 1; case SETUP_ANNOTATIONS: return 0; + case SETUP_CLEANUP: + return 2; + case SETUP_FINALLY: + return 1; + case SETUP_WITH: + return 1; case SET_ADD: return 1 + (oparg-1); case SET_FUNCTION_ATTRIBUTE: @@ -826,6 +890,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case STORE_FAST_LOAD_FAST: return 1; + case STORE_FAST_MAYBE_NULL: + return 0; case STORE_FAST_STORE_FAST: return 0; case STORE_GLOBAL: @@ -863,7 +929,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case UNARY_NOT: return 1; case UNPACK_EX: - return 1 + (oparg >> 8) + (oparg & 0xFF); + return 1 + (oparg & 0xFF) + (oparg >> 8); case UNPACK_SEQUENCE: return oparg; case UNPACK_SEQUENCE_LIST: @@ -871,9 +937,9 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case UNPACK_SEQUENCE_TUPLE: return oparg; case UNPACK_SEQUENCE_TWO_TUPLE: - return oparg; + return 2; case WITH_EXCEPT_START: - return 5; + return 6; case YIELD_VALUE: return 1; default: @@ -896,7 +962,7 @@ enum InstructionFormat { }; #define IS_VALID_OPCODE(OP) \ - (((OP) >= 0) && ((OP) < 268) && \ + (((OP) >= 0) && ((OP) < 266) && \ (_PyOpcode_opcode_metadata[(OP)].valid_entry)) #define HAS_ARG_FLAG (1) @@ -909,8 +975,11 @@ enum InstructionFormat { #define HAS_DEOPT_FLAG (128) #define HAS_ERROR_FLAG (256) #define HAS_ESCAPES_FLAG (512) -#define HAS_PURE_FLAG (1024) -#define HAS_PASSTHROUGH_FLAG (2048) +#define HAS_EXIT_FLAG (1024) +#define HAS_PURE_FLAG (2048) +#define HAS_PASSTHROUGH_FLAG (4096) +#define HAS_OPARG_AND_1_FLAG (8192) +#define HAS_ERROR_NO_POP_FLAG (16384) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -921,8 +990,11 @@ enum InstructionFormat { #define OPCODE_HAS_DEOPT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_DEOPT_FLAG)) #define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG)) #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) +#define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) +#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) +#define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -939,162 +1011,170 @@ struct opcode_metadata { int16_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[268]; +extern const struct opcode_metadata _PyOpcode_opcode_metadata[266]; #ifdef NEED_OPCODE_METADATA -const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { - [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, +const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [CACHE] = { true, INSTR_FMT_IX, 0 }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IXC, 0 }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, 0 }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, + [LOAD_CONST_IMMORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [NOP] = { true, INSTR_FMT_IX, 0 }, + [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -1103,24 +1183,23 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RESERVED] = { true, INSTR_FMT_IX, 0 }, + [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1130,15 +1209,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1148,23 +1227,21 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_IF_TRUE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, - [LOAD_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_ATTR] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [POP_BLOCK] = { true, -1, 0 }, - [SETUP_CLEANUP] = { true, -1, HAS_ARG_FLAG }, - [SETUP_FINALLY] = { true, -1, HAS_ARG_FLAG }, - [SETUP_WITH] = { true, -1, HAS_ARG_FLAG }, + [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, + [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, }; #endif -#define MAX_UOP_PER_EXPANSION 8 +#define MAX_UOP_PER_EXPANSION 9 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1174,12 +1251,11 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { - [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { _BEFORE_ASYNC_WITH, 0, 0 } } }, - [BEFORE_WITH] = { .nuops = 1, .uops = { { _BEFORE_WITH, 0, 0 } } }, [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, 0, 0 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, 0, 0 } } }, [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } }, @@ -1187,40 +1263,50 @@ _PyOpcode_macro_expansion[256] = { [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, 0, 0 } } }, [BINARY_SUBSCR] = { .nuops = 1, .uops = { { _BINARY_SUBSCR, 0, 0 } } }, [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_DICT, 0, 0 } } }, + [BINARY_SUBSCR_GETITEM] = { .nuops = 4, .uops = { { _CHECK_PEP_523, 0, 0 }, { _BINARY_SUBSCR_CHECK_FUNC, 0, 0 }, { _BINARY_SUBSCR_INIT_CALL, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_LIST_INT, 0, 0 } } }, [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_STR_INT, 0, 0 } } }, [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, - [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } }, [BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } }, [BUILD_SET] = { .nuops = 1, .uops = { { _BUILD_SET, 0, 0 } } }, [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST, 0, 0 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { _CALL_BUILTIN_O, 0, 0 } } }, + [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, 0, 0 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, 0, 0 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, 0, 0 }, { _CALL_KW_NON_PY, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_STR_1] = { .nuops = 1, .uops = { { _CALL_STR_1, 0, 0 } } }, - [CALL_TUPLE_1] = { .nuops = 1, .uops = { { _CALL_TUPLE_1, 0, 0 } } }, + [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, 0, 0 }, { _CALL_NON_PY_GENERAL, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 7, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_GENERAL] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_STR_1] = { .nuops = 2, .uops = { { _CALL_STR_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_TUPLE_1] = { .nuops = 2, .uops = { { _CALL_TUPLE_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { _CALL_TYPE_1, 0, 0 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, 0, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, 0, 0 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, 0, 0 } } }, - [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { _COMPARE_OP_FLOAT, 0, 0 } } }, - [COMPARE_OP_INT] = { .nuops = 1, .uops = { { _COMPARE_OP_INT, 0, 0 } } }, - [COMPARE_OP_STR] = { .nuops = 1, .uops = { { _COMPARE_OP_STR, 0, 0 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _COMPARE_OP_FLOAT, 0, 0 } } }, + [COMPARE_OP_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _COMPARE_OP_INT, 0, 0 } } }, + [COMPARE_OP_STR] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _COMPARE_OP_STR, 0, 0 } } }, [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, 0, 0 } } }, + [CONTAINS_OP_DICT] = { .nuops = 1, .uops = { { _CONTAINS_OP_DICT, 0, 0 } } }, + [CONTAINS_OP_SET] = { .nuops = 1, .uops = { { _CONTAINS_OP_SET, 0, 0 } } }, [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, 0, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, 0, 0 } } }, [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, 0, 0 } } }, @@ -1238,6 +1324,7 @@ _PyOpcode_macro_expansion[256] = { [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, 0, 0 } } }, [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, 0, 0 } } }, [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, 9, 0 } } }, + [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, 0, 0 }, { _FOR_ITER_GEN_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, 0, 0 }, { _ITER_JUMP_LIST, 9, 1 }, { _ITER_NEXT_LIST, 0, 0 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, 0, 0 }, { _ITER_JUMP_RANGE, 9, 1 }, { _ITER_NEXT_RANGE, 0, 0 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, 0, 0 }, { _ITER_JUMP_TUPLE, 9, 1 }, { _ITER_NEXT_TUPLE, 0, 0 } } }, @@ -1247,35 +1334,41 @@ _PyOpcode_macro_expansion[256] = { [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, 0, 0 } } }, [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, 0, 0 } } }, [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, 0, 0 } } }, + [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, 0, 0 } } }, + [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, 0, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, 0, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, 0, 0 } } }, - [LOAD_ASSERTION_ERROR] = { .nuops = 1, .uops = { { _LOAD_ASSERTION_ERROR, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 3, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _GUARD_TYPE_VERSION, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_PROPERTY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, 7, 9 }, { _PUSH_FRAME, 0, 0 } } }, [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, 0, 0 } } }, + [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, 0, 0 } } }, [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, 0, 0 } } }, + [LOAD_CONST_IMMORTAL] = { .nuops = 1, .uops = { { _LOAD_CONST_IMMORTAL, 0, 0 } } }, [LOAD_DEREF] = { .nuops = 1, .uops = { { _LOAD_DEREF, 0, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { _LOAD_FAST, 0, 0 } } }, [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { _LOAD_FAST_AND_CLEAR, 0, 0 } } }, [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { _LOAD_FAST_CHECK, 0, 0 } } }, [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, - [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, - [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, - [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, + [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION_PUSH_KEYS, 1, 2 }, { _LOAD_GLOBAL_BUILTINS_FROM_KEYS, 1, 3 } } }, + [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION_PUSH_KEYS, 1, 1 }, { _LOAD_GLOBAL_MODULE_FROM_KEYS, 1, 3 } } }, [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } }, + [LOAD_SMALL_INT] = { .nuops = 1, .uops = { { _LOAD_SMALL_INT, 0, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 1, .uops = { { _LOAD_SPECIAL, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, 0, 0 } } }, @@ -1295,15 +1388,17 @@ _PyOpcode_macro_expansion[256] = { [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, 0, 0 } } }, [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, 0, 0 } } }, [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, 0, 0 } } }, - [RETURN_CONST] = { .nuops = 2, .uops = { { _LOAD_CONST, 0, 0 }, { _POP_FRAME, 0, 0 } } }, - [RETURN_VALUE] = { .nuops = 1, .uops = { { _POP_FRAME, 0, 0 } } }, + [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 1, .uops = { { _RETURN_VALUE, 0, 0 } } }, + [SEND_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, 0, 0 }, { _SEND_GEN_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, 0, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, 0, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, + [STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } }, [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } }, [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, @@ -1316,7 +1411,7 @@ _PyOpcode_macro_expansion[256] = { [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_LIST_INT, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, 0, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { _TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, 0, 0 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, 0, 0 } } }, [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, 0, 0 } } }, [TO_BOOL_LIST] = { .nuops = 1, .uops = { { _TO_BOOL_LIST, 0, 0 } } }, @@ -1331,14 +1426,13 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, 0, 0 } } }, + [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, 0, 0 } } }, }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[268]; +extern const char *_PyOpcode_OpName[266]; #ifdef NEED_OPCODE_METADATA -const char *_PyOpcode_OpName[268] = { - [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", - [BEFORE_WITH] = "BEFORE_WITH", +const char *_PyOpcode_OpName[266] = { [BINARY_OP] = "BINARY_OP", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", @@ -1355,7 +1449,6 @@ const char *_PyOpcode_OpName[268] = { [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_LIST] = "BUILD_LIST", [BUILD_MAP] = "BUILD_MAP", [BUILD_SET] = "BUILD_SET", @@ -1366,6 +1459,7 @@ const char *_PyOpcode_OpName[268] = { [CALL] = "CALL", [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BOUND_METHOD_GENERAL] = "CALL_BOUND_METHOD_GENERAL", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", @@ -1375,14 +1469,18 @@ const char *_PyOpcode_OpName[268] = { [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", [CALL_ISINSTANCE] = "CALL_ISINSTANCE", [CALL_KW] = "CALL_KW", + [CALL_KW_BOUND_METHOD] = "CALL_KW_BOUND_METHOD", + [CALL_KW_NON_PY] = "CALL_KW_NON_PY", + [CALL_KW_PY] = "CALL_KW_PY", [CALL_LEN] = "CALL_LEN", [CALL_LIST_APPEND] = "CALL_LIST_APPEND", [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", + [CALL_NON_PY_GENERAL] = "CALL_NON_PY_GENERAL", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [CALL_PY_GENERAL] = "CALL_PY_GENERAL", [CALL_STR_1] = "CALL_STR_1", [CALL_TUPLE_1] = "CALL_TUPLE_1", [CALL_TYPE_1] = "CALL_TYPE_1", @@ -1394,6 +1492,8 @@ const char *_PyOpcode_OpName[268] = { [COMPARE_OP_INT] = "COMPARE_OP_INT", [COMPARE_OP_STR] = "COMPARE_OP_STR", [CONTAINS_OP] = "CONTAINS_OP", + [CONTAINS_OP_DICT] = "CONTAINS_OP_DICT", + [CONTAINS_OP_SET] = "CONTAINS_OP_SET", [CONVERT_VALUE] = "CONVERT_VALUE", [COPY] = "COPY", [COPY_FREE_VARS] = "COPY_FREE_VARS", @@ -1442,7 +1542,6 @@ const char *_PyOpcode_OpName[268] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", [INTERPRETER_EXIT] = "INTERPRETER_EXIT", @@ -1451,12 +1550,14 @@ const char *_PyOpcode_OpName[268] = { [JUMP_BACKWARD] = "JUMP_BACKWARD", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [JUMP_FORWARD] = "JUMP_FORWARD", + [JUMP_IF_FALSE] = "JUMP_IF_FALSE", + [JUMP_IF_TRUE] = "JUMP_IF_TRUE", [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", [LIST_APPEND] = "LIST_APPEND", [LIST_EXTEND] = "LIST_EXTEND", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [LOAD_ATTR] = "LOAD_ATTR", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", @@ -1470,7 +1571,9 @@ const char *_PyOpcode_OpName[268] = { [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [LOAD_CLOSURE] = "LOAD_CLOSURE", + [LOAD_COMMON_CONSTANT] = "LOAD_COMMON_CONSTANT", [LOAD_CONST] = "LOAD_CONST", + [LOAD_CONST_IMMORTAL] = "LOAD_CONST_IMMORTAL", [LOAD_DEREF] = "LOAD_DEREF", [LOAD_FAST] = "LOAD_FAST", [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", @@ -1482,14 +1585,12 @@ const char *_PyOpcode_OpName[268] = { [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [LOAD_LOCALS] = "LOAD_LOCALS", - [LOAD_METHOD] = "LOAD_METHOD", [LOAD_NAME] = "LOAD_NAME", + [LOAD_SMALL_INT] = "LOAD_SMALL_INT", + [LOAD_SPECIAL] = "LOAD_SPECIAL", [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", - [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", - [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", [MAKE_CELL] = "MAKE_CELL", [MAKE_FUNCTION] = "MAKE_FUNCTION", [MAP_ADD] = "MAP_ADD", @@ -1512,7 +1613,6 @@ const char *_PyOpcode_OpName[268] = { [RESERVED] = "RESERVED", [RESUME] = "RESUME", [RESUME_CHECK] = "RESUME_CHECK", - [RETURN_CONST] = "RETURN_CONST", [RETURN_GENERATOR] = "RETURN_GENERATOR", [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", @@ -1563,7 +1663,6 @@ const char *_PyOpcode_OpName[268] = { extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { - [JUMP_BACKWARD] = 1, [TO_BOOL] = 3, [BINARY_SUBSCR] = 1, [STORE_SUBSCR] = 1, @@ -1574,12 +1673,15 @@ const uint8_t _PyOpcode_Caches[256] = { [LOAD_SUPER_ATTR] = 1, [LOAD_ATTR] = 9, [COMPARE_OP] = 1, + [CONTAINS_OP] = 1, + [JUMP_BACKWARD] = 1, [POP_JUMP_IF_TRUE] = 1, [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, [FOR_ITER] = 1, [CALL] = 3, + [CALL_KW] = 3, [BINARY_OP] = 1, }; #endif @@ -1587,8 +1689,6 @@ const uint8_t _PyOpcode_Caches[256] = { extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, - [BEFORE_WITH] = BEFORE_WITH, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -1605,7 +1705,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, - [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, [BUILD_LIST] = BUILD_LIST, [BUILD_MAP] = BUILD_MAP, [BUILD_SET] = BUILD_SET, @@ -1616,6 +1715,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL] = CALL, [CALL_ALLOC_AND_ENTER_INIT] = CALL, [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [CALL_BOUND_METHOD_GENERAL] = CALL, [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, @@ -1625,14 +1725,18 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, [CALL_ISINSTANCE] = CALL, [CALL_KW] = CALL_KW, + [CALL_KW_BOUND_METHOD] = CALL_KW, + [CALL_KW_NON_PY] = CALL_KW, + [CALL_KW_PY] = CALL_KW, [CALL_LEN] = CALL, [CALL_LIST_APPEND] = CALL, [CALL_METHOD_DESCRIPTOR_FAST] = CALL, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_METHOD_DESCRIPTOR_NOARGS] = CALL, [CALL_METHOD_DESCRIPTOR_O] = CALL, + [CALL_NON_PY_GENERAL] = CALL, [CALL_PY_EXACT_ARGS] = CALL, - [CALL_PY_WITH_DEFAULTS] = CALL, + [CALL_PY_GENERAL] = CALL, [CALL_STR_1] = CALL, [CALL_TUPLE_1] = CALL, [CALL_TYPE_1] = CALL, @@ -1644,6 +1748,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [COMPARE_OP_INT] = COMPARE_OP, [COMPARE_OP_STR] = COMPARE_OP, [CONTAINS_OP] = CONTAINS_OP, + [CONTAINS_OP_DICT] = CONTAINS_OP, + [CONTAINS_OP_SET] = CONTAINS_OP, [CONVERT_VALUE] = CONVERT_VALUE, [COPY] = COPY, [COPY_FREE_VARS] = COPY_FREE_VARS, @@ -1692,7 +1798,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INTERPRETER_EXIT] = INTERPRETER_EXIT, @@ -1702,9 +1807,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [JUMP_FORWARD] = JUMP_FORWARD, [LIST_APPEND] = LIST_APPEND, [LIST_EXTEND] = LIST_EXTEND, - [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, [LOAD_ATTR] = LOAD_ATTR, [LOAD_ATTR_CLASS] = LOAD_ATTR, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = LOAD_ATTR, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, @@ -1717,7 +1822,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_ATTR_SLOT] = LOAD_ATTR, [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = LOAD_COMMON_CONSTANT, [LOAD_CONST] = LOAD_CONST, + [LOAD_CONST_IMMORTAL] = LOAD_CONST, [LOAD_DEREF] = LOAD_DEREF, [LOAD_FAST] = LOAD_FAST, [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, @@ -1730,6 +1837,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, [LOAD_LOCALS] = LOAD_LOCALS, [LOAD_NAME] = LOAD_NAME, + [LOAD_SMALL_INT] = LOAD_SMALL_INT, + [LOAD_SPECIAL] = LOAD_SPECIAL, [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, @@ -1754,7 +1863,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [RESERVED] = RESERVED, [RESUME] = RESUME, [RESUME_CHECK] = RESUME, - [RETURN_CONST] = RETURN_CONST, [RETURN_GENERATOR] = RETURN_GENERATOR, [RETURN_VALUE] = RETURN_VALUE, [SEND] = SEND, @@ -1800,6 +1908,9 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 116: \ + case 117: \ + case 118: \ case 119: \ case 120: \ case 121: \ @@ -1830,15 +1941,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ - case 223: \ - case 224: \ - case 225: \ - case 226: \ - case 227: \ case 228: \ case 229: \ case 230: \ @@ -1847,32 +1949,31 @@ const uint8_t _PyOpcode_Deopt[256] = { case 233: \ case 234: \ case 235: \ - case 255: \ + case 236: \ ; struct pseudo_targets { - uint8_t targets[3]; + uint8_t as_sequence; + uint8_t targets[4]; }; -extern const struct pseudo_targets _PyOpcode_PseudoTargets[12]; +extern const struct pseudo_targets _PyOpcode_PseudoTargets[10]; #ifdef NEED_OPCODE_METADATA -const struct pseudo_targets _PyOpcode_PseudoTargets[12] = { - [LOAD_CLOSURE-256] = { { LOAD_FAST, 0, 0 } }, - [STORE_FAST_MAYBE_NULL-256] = { { STORE_FAST, 0, 0 } }, - [LOAD_SUPER_METHOD-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, - [LOAD_ZERO_SUPER_METHOD-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, - [LOAD_ZERO_SUPER_ATTR-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, - [LOAD_METHOD-256] = { { LOAD_ATTR, 0, 0 } }, - [JUMP-256] = { { JUMP_FORWARD, JUMP_BACKWARD, 0 } }, - [JUMP_NO_INTERRUPT-256] = { { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0 } }, - [SETUP_FINALLY-256] = { { NOP, 0, 0 } }, - [SETUP_CLEANUP-256] = { { NOP, 0, 0 } }, - [SETUP_WITH-256] = { { NOP, 0, 0 } }, - [POP_BLOCK-256] = { { NOP, 0, 0 } }, +const struct pseudo_targets _PyOpcode_PseudoTargets[10] = { + [LOAD_CLOSURE-256] = { 0, { LOAD_FAST, 0, 0, 0 } }, + [STORE_FAST_MAYBE_NULL-256] = { 0, { STORE_FAST, 0, 0, 0 } }, + [JUMP-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD, 0, 0 } }, + [JUMP_NO_INTERRUPT-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0, 0 } }, + [JUMP_IF_FALSE-256] = { 1, { COPY, TO_BOOL, POP_JUMP_IF_FALSE, 0 } }, + [JUMP_IF_TRUE-256] = { 1, { COPY, TO_BOOL, POP_JUMP_IF_TRUE, 0 } }, + [SETUP_FINALLY-256] = { 0, { NOP, 0, 0, 0 } }, + [SETUP_CLEANUP-256] = { 0, { NOP, 0, 0, 0 } }, + [SETUP_WITH-256] = { 0, { NOP, 0, 0, 0 } }, + [POP_BLOCK-256] = { 0, { NOP, 0, 0, 0 } }, }; #endif // NEED_OPCODE_METADATA static inline bool is_pseudo_target(int pseudo, int target) { - if (pseudo < 256 || pseudo >= 268) { + if (pseudo < 256 || pseudo >= 266) { return false; } for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) { diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 208bfb2f75308b..c6ce7e65a65d4b 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -47,7 +47,6 @@ extern "C" { #define IS_SCOPE_EXIT_OPCODE(opcode) \ ((opcode) == RETURN_VALUE || \ - (opcode) == RETURN_CONST || \ (opcode) == RAISE_VARARGS || \ (opcode) == RERAISE) @@ -57,6 +56,12 @@ extern "C" { #define MAKE_FUNCTION_KWDEFAULTS 0x02 #define MAKE_FUNCTION_ANNOTATIONS 0x04 #define MAKE_FUNCTION_CLOSURE 0x08 +#define MAKE_FUNCTION_ANNOTATE 0x10 + +/* Values used as the oparg for LOAD_COMMON_CONSTANT */ +#define CONSTANT_ASSERTIONERROR 0 +#define CONSTANT_NOTIMPLEMENTEDERROR 1 +#define NUM_COMMON_CONSTANTS 2 /* Values used in the oparg for RESUME */ #define RESUME_AT_FUNC_START 0 diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index e21412fc815540..f92c0a0cddf906 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -8,7 +8,141 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, +#include "pycore_uop_ids.h" +#include + + +typedef struct _PyExecutorLinkListNode { + struct _PyExecutorObject *next; + struct _PyExecutorObject *previous; +} _PyExecutorLinkListNode; + + +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + +typedef struct { + uint8_t opcode; + uint8_t oparg; + uint8_t valid:1; + uint8_t linked:1; + uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. + bool warm; + int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). + _PyBloomFilter bloom; + _PyExecutorLinkListNode links; + PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). +} _PyVMData; + +/* Depending on the format, + * the 32 bits between the oparg and operand are: + * UOP_FORMAT_TARGET: + * uint32_t target; + * UOP_FORMAT_JUMP + * uint16_t jump_target; + * uint16_t error_target; + */ +typedef struct { + uint16_t opcode:15; + uint16_t format:1; + uint16_t oparg; + union { + uint32_t target; + struct { + uint16_t jump_target; + uint16_t error_target; + }; + }; + uint64_t operand; // A cache entry +} _PyUOpInstruction; + +typedef struct { + uint32_t target; + _Py_BackoffCounter temperature; + const struct _PyExecutorObject *executor; +} _PyExitData; + +typedef struct _PyExecutorObject { + PyObject_VAR_HEAD + const _PyUOpInstruction *trace; + _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ + uint32_t exit_count; + uint32_t code_size; + size_t jit_size; + void *jit_code; + void *jit_side_entry; + _PyExitData exits[1]; +} _PyExecutorObject; + +typedef struct _PyOptimizerObject _PyOptimizerObject; + +/* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ +typedef int (*_Py_optimize_func)( + _PyOptimizerObject* self, struct _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, + int curr_stackentries, bool progress_needed); + +struct _PyOptimizerObject { + PyObject_HEAD + _Py_optimize_func optimize; + /* Data needed by the optimizer goes here, but is opaque to the VM */ +}; + +/** Test support **/ +typedef struct { + _PyOptimizerObject base; + int64_t count; +} _PyCounterOptimizerObject; + +_PyOptimizerObject *_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject* optimizer); + + +// Export for '_opcode' shared extension (JIT compiler). +PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset); + +void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); +void _Py_ExecutorDetach(_PyExecutorObject *); +void _Py_BloomFilter_Init(_PyBloomFilter *); +void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); +PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); + +// For testing +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(_PyOptimizerObject *) _Py_GetOptimizer(void); +PyAPI_FUNC(int) _Py_SetTier2Optimizer(_PyOptimizerObject* optimizer); +PyAPI_FUNC(PyObject *) _PyOptimizer_NewCounter(void); +PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); + +#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 +#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 + +#ifdef _Py_TIER2 +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); + +#else +# define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) +# define _Py_Executors_InvalidateAll(A, B) ((void)0) +# define _Py_Executors_InvalidateCold(A) ((void)0) + +#endif + +// Used as the threshold to trigger executor invalidation when +// trace_run_counter is greater than this value. +#define JIT_CLEANUP_THRESHOLD 100000 + +// This is the length of the trace we project initially. +#define UOP_MAX_TRACE_LENGTH 800 + +#define TRACE_STACK_SIZE 5 + +int _Py_uop_analyze_and_optimize(struct _PyInterpreterFrame *frame, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); @@ -18,6 +152,138 @@ extern PyTypeObject _PyDefaultOptimizer_Type; extern PyTypeObject _PyUOpExecutor_Type; extern PyTypeObject _PyUOpOptimizer_Type; +/* Symbols */ +/* See explanation in optimizer_symbols.c */ + +struct _Py_UopsSymbol { + int flags; // 0 bits: Top; 2 or more bits: Bottom + PyTypeObject *typ; // Borrowed reference + PyObject *const_val; // Owned reference (!) + unsigned int type_version; // currently stores type version +}; + +#define UOP_FORMAT_TARGET 0 +#define UOP_FORMAT_JUMP 1 + +static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_TARGET); + return inst->target; +} + +static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_JUMP); + return inst->jump_target; +} + +static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) +{ + assert(inst->format != UOP_FORMAT_TARGET); + return inst->error_target; +} + +// Holds locals, stack, locals, stack ... co_consts (in that order) +#define MAX_ABSTRACT_INTERP_SIZE 4096 + +#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) + +// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) +#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) + +// The maximum number of side exits that we can take before requiring forward +// progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this +// is the "maximum amount of polymorphism" that an isolated trace tree can +// handle before rejoining the rest of the program. +#define MAX_CHAIN_DEPTH 4 + +typedef struct _Py_UopsSymbol _Py_UopsSymbol; + +struct _Py_UOpsAbstractFrame { + // Max stacklen + int stack_len; + int locals_len; + + _Py_UopsSymbol **stack_pointer; + _Py_UopsSymbol **stack; + _Py_UopsSymbol **locals; +}; + +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +typedef struct ty_arena { + int ty_curr_number; + int ty_max_number; + _Py_UopsSymbol arena[TY_ARENA_SIZE]; +} ty_arena; + +struct _Py_UOpsContext { + char done; + char out_of_space; + bool contradiction; + // The current "executing" frame. + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; + int curr_frame_depth; + + // Arena for the symbolic types. + ty_arena t_arena; + + _Py_UopsSymbol **n_consumed; + _Py_UopsSymbol **limit; + _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; +}; + +typedef struct _Py_UOpsContext _Py_UOpsContext; + +extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym); +extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym); +extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_type( + _Py_UOpsContext *ctx, PyTypeObject *typ); +extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); +extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); +extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version); +extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version); +extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); +extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); +extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsSymbol *sym); + + +extern void _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); +extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); + +extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( + _Py_UOpsContext *ctx, + PyCodeObject *co, + int curr_stackentries, + _Py_UopsSymbol **args, + int arg_len); +extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); + +PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); + +PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyStackRef *stack_pointer, _PyExecutorObject **exec_ptr, int chain_depth); + +static inline int is_terminator(const _PyUOpInstruction *uop) +{ + int opcode = uop->opcode; + return ( + opcode == _EXIT_TRACE || + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT + ); +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_parking_lot.h b/Include/internal/pycore_parking_lot.h index f444da730055e8..8c9260e2636fbc 100644 --- a/Include/internal/pycore_parking_lot.h +++ b/Include/internal/pycore_parking_lot.h @@ -18,8 +18,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t - enum { // The thread was unparked by another thread. @@ -61,7 +59,7 @@ enum { // } PyAPI_FUNC(int) _PyParkingLot_Park(const void *address, const void *expected, - size_t address_size, _PyTime_t timeout_ns, + size_t address_size, PyTime_t timeout_ns, void *park_arg, int detach); // Callback for _PyParkingLot_Unpark: diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 067b34c12c4e7f..b16084aaa15515 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -21,6 +21,9 @@ extern "C" { struct _parser_runtime_state { #ifdef Py_DEBUG long memo_statistics[_PYPEGEN_NSTATISTICS]; +#ifdef Py_GIL_DISABLED + PyMutex mutex; +#endif #else int _not_used; #endif @@ -28,8 +31,10 @@ struct _parser_runtime_state { }; _Py_DECLARE_STR(empty, "") +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) #define _parser_runtime_state_INIT \ { \ + .mutex = {0}, \ .dummy_name = { \ .kind = Name_kind, \ .v.Name.id = &_Py_STR(empty), \ @@ -40,6 +45,20 @@ _Py_DECLARE_STR(empty, "") .end_col_offset = 0, \ }, \ } +#else +#define _parser_runtime_state_INIT \ + { \ + .dummy_name = { \ + .kind = Name_kind, \ + .v.Name.id = &_Py_STR(empty), \ + .v.Name.ctx = Load, \ + .lineno = 1, \ + .col_offset = 0, \ + .end_lineno = 1, \ + .end_col_offset = 0, \ + }, \ + } +#endif extern struct _mod* _PyParser_ASTFromString( const char *str, diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h new file mode 100644 index 00000000000000..a1bb383bcd22e9 --- /dev/null +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -0,0 +1,94 @@ +// This header file provides wrappers around the atomic operations found in +// `pyatomic.h` that are only atomic in free-threaded builds. +// +// These are intended to be used in places where atomics are required in +// free-threaded builds, but not in the default build, and we don't want to +// introduce the potential performance overhead of an atomic operation in the +// default build. +// +// All usages of these macros should be replaced with unconditionally atomic or +// non-atomic versions, and this file should be removed, once the dust settles +// on free threading. +#ifndef Py_ATOMIC_FT_WRAPPERS_H +#define Py_ATOMIC_FT_WRAPPERS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +#error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED +#define FT_ATOMIC_LOAD_PTR(value) _Py_atomic_load_ptr(&value) +#define FT_ATOMIC_STORE_PTR(value, new_value) _Py_atomic_store_ptr(&value, new_value) +#define FT_ATOMIC_LOAD_SSIZE(value) _Py_atomic_load_ssize(&value) +#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) \ + _Py_atomic_load_ssize_acquire(&value) +#define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) \ + _Py_atomic_load_ssize_relaxed(&value) +#define FT_ATOMIC_STORE_PTR(value, new_value) \ + _Py_atomic_store_ptr(&value, new_value) +#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \ + _Py_atomic_load_ptr_acquire(&value) +#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ + _Py_atomic_load_uintptr_acquire(&value) +#define FT_ATOMIC_LOAD_PTR_RELAXED(value) \ + _Py_atomic_load_ptr_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT8(value) \ + _Py_atomic_load_uint8(&value) +#define FT_ATOMIC_STORE_UINT8(value, new_value) \ + _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ + _Py_atomic_load_uint8_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ + _Py_atomic_load_uint16_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ + _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ + _Py_atomic_load_ulong_relaxed(&value) +#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ + _Py_atomic_store_ptr_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ + _Py_atomic_store_ptr_release(&value, new_value) +#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \ + _Py_atomic_store_uintptr_release(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ + _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ + _Py_atomic_store_uint8_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ + _Py_atomic_store_uint16_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ + _Py_atomic_store_uint32_relaxed(&value, new_value) + +#else +#define FT_ATOMIC_LOAD_PTR(value) value +#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_SSIZE(value) value +#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value +#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_PTR_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT8(value) value +#define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value +#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value + +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_ATOMIC_FT_WRAPPERS_H */ diff --git a/Include/internal/pycore_pybuffer.h b/Include/internal/pycore_pybuffer.h index 3cbc290b2ea3ee..9439d2bd770587 100644 --- a/Include/internal/pycore_pybuffer.h +++ b/Include/internal/pycore_pybuffer.h @@ -9,7 +9,7 @@ extern "C" { #endif -// Exported for the _xxinterpchannels module. +// Exported for the _interpchannels module. PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreter( PyInterpreterState *interp, Py_buffer *view); PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreterAndRawFree( diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 0f16fb894d17e1..02945f0e71a145 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -95,7 +95,7 @@ extern void _PyErr_Fetch( extern PyObject* _PyErr_GetRaisedException(PyThreadState *tstate); -extern int _PyErr_ExceptionMatches( +PyAPI_FUNC(int) _PyErr_ExceptionMatches( PyThreadState *tstate, PyObject *exc); @@ -114,18 +114,23 @@ extern void _PyErr_SetObject( extern void _PyErr_ChainStackItem(void); -extern void _PyErr_Clear(PyThreadState *tstate); +PyAPI_FUNC(void) _PyErr_Clear(PyThreadState *tstate); extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); -extern void _PyErr_SetString( +extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, + int end_lineno, int end_col_offset); +extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, + int end_lineno, int end_col_offset); + +PyAPI_FUNC(void) _PyErr_SetString( PyThreadState *tstate, PyObject *exception, const char *string); -extern PyObject* _PyErr_Format( +PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, PyObject *exception, const char *format, @@ -161,14 +166,17 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception); PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, Py_ssize_t max_cost); -void _PyErr_FormatNote(const char *format, ...); +// Export for '_json' shared extension +PyAPI_FUNC(void) _PyErr_FormatNote(const char *format, ...); /* Context manipulation (PEP 3134) */ Py_DEPRECATED(3.12) extern void _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); -// Export for '_zoneinfo' shared extension -PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); +// implementation detail for the codeop module. +// Exported for test.test_peg_generator.test_c_parser +PyAPI_DATA(PyTypeObject) _PyExc_IncompleteInputError; +#define PyExc_IncompleteInputError ((PyObject *)(&_PyExc_IncompleteInputError)) #ifdef __cplusplus } diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 0ce08900e96f0b..9414e7761171d2 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -20,9 +20,6 @@ _Py_HashPointerRaw(const void *ptr) return (Py_hash_t)x; } -// Export for '_datetime' shared extension -PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); - /* Hash secret * * memory layout on 64 bit systems diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index c675098685764c..f426ae0e103b9c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -42,7 +42,6 @@ extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); -extern int _Py_Deepfreeze_Init(void); /* Various internal finalizers */ @@ -58,7 +57,6 @@ extern void _PyWarnings_Fini(PyInterpreterState *interp); extern void _PyAST_Fini(PyInterpreterState *interp); extern void _PyAtExit_Fini(PyInterpreterState *interp); extern void _PyThread_FiniType(PyInterpreterState *interp); -extern void _Py_Deepfreeze_Fini(void); extern void _PyArg_Fini(void); extern void _Py_FinalizeAllocatedBlocks(_PyRuntimeState *); @@ -116,6 +114,22 @@ PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); // Export for special main.c string compiling with source tracebacks int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + +/* interpreter config */ + +// Export for _testinternalcapi shared extension +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromState( + PyInterpreterConfig *, + PyInterpreterState *); +PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromDict( + PyInterpreterConfig *, + PyObject *); +PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict( + PyInterpreterConfig *, + PyObject *); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index 7a4e1c1eb714f7..eea8996ba68ca0 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL) { + if (x == Py_INFINITY || x == -Py_INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL || - y == Py_HUGE_VAL || y == -Py_HUGE_VAL) + if (x == Py_INFINITY || x == -Py_INFINITY || + y == Py_INFINITY || y == -Py_INFINITY) { if (errno == 0) { errno = ERANGE; diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 1a72d07b50b738..dd6b0762370c92 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -1,6 +1,7 @@ #ifndef Py_INTERNAL_PYMEM_H #define Py_INTERNAL_PYMEM_H +#include "pycore_llist.h" // struct llist_node #include "pycore_lock.h" // PyMutex #ifdef __cplusplus @@ -48,6 +49,11 @@ struct _pymem_allocators { PyObjectArenaAllocator obj_arena; }; +struct _Py_mem_interp_free_queue { + int has_work; // true if the queue is not empty + PyMutex mutex; // protects the queue + struct llist_node head; // queue of _mem_work_chunk items +}; /* Set the memory allocator of the specified domain to the default. Save the old allocator into *old_alloc if it's non-NULL. @@ -110,6 +116,22 @@ extern int _PyMem_SetupAllocators(PyMemAllocatorName allocator); /* Is the debug allocator enabled? */ extern int _PyMem_DebugEnabled(void); +// Enqueue a pointer to be freed possibly after some delay. +extern void _PyMem_FreeDelayed(void *ptr); + +// Enqueue an object to be freed possibly after some delay +extern void _PyObject_FreeDelayed(void *ptr); + +// Periodically process delayed free requests. +extern void _PyMem_ProcessDelayed(PyThreadState *tstate); + +// Abandon all thread-local delayed free requests and push them to the +// interpreter's queue. +extern void _PyMem_AbandonDelayed(PyThreadState *tstate); + +// On interpreter shutdown, frees all delayed free requests. +extern void _PyMem_FiniDelayed(PyInterpreterState *interp); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 96c49ed7338d6d..c593edc86d9952 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -92,6 +92,11 @@ extern void _PyMem_ArenaFree(void *, void *, size_t); { NULL, _PyMem_ArenaAlloc, _PyMem_ArenaFree } +#define _Py_mem_free_queue_INIT(queue) \ + { \ + .head = LLIST_INIT(queue.head), \ + } + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 289ef28f0dd9a9..fade55945b7dbf 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,11 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_runtime.h" // _PyRuntime #include "pycore_tstate.h" // _PyThreadStateImpl - // Values for PyThreadState.state. A thread must be in the "attached" state // before calling most Python APIs. If the GIL is enabled, then "attached" // implies that the thread holds the GIL and "detached" implies that the @@ -77,12 +75,18 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) interp == &_PyRuntime._main_interpreter); } -// Export for _xxsubinterpreters module. +// Export for _interpreters module. +PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); + +// Export for _interpreters module. PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_FailIfRunningMain(PyInterpreterState *); +extern int _PyThreadState_IsRunningMain(PyThreadState *); +extern void _PyInterpreterState_ReinitRunningMain(PyThreadState *); + static inline const PyConfig * _Py_GetMainConfig(void) @@ -211,11 +215,16 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) { // PyThreadState functions -extern PyThreadState * _PyThreadState_New( +// Export for _testinternalcapi +PyAPI_FUNC(PyThreadState *) _PyThreadState_New( PyInterpreterState *interp, int whence); extern void _PyThreadState_Bind(PyThreadState *tstate); -extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); +PyAPI_FUNC(PyThreadState *) _PyThreadState_NewBound( + PyInterpreterState *interp, + int whence); +extern PyThreadState * _PyThreadState_RemoveExcept(PyThreadState *tstate); +extern void _PyThreadState_DeleteList(PyThreadState *list); extern void _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate); // Export for '_testinternalcapi' shared extension @@ -268,20 +277,6 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // See also PyInterpreterState_Get() and _PyInterpreterState_GET(). extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); -static inline _PyFreeListState* _PyFreeListState_GET(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); -#ifdef Py_DEBUG - _Py_EnsureTstateNotNULL(tstate); -#endif - -#ifdef Py_GIL_DISABLED - return &((_PyThreadStateImpl*)tstate)->freelist_state; -#else - return &tstate->interp->freelist_state; -#endif -} - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 265299d7574838..a1e084cf67d58d 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -46,7 +46,8 @@ extern "C" { #if defined(HAVE_PTHREAD_STUBS) -#include // bool +#include "cpython/pthread_stubs.h" // PTHREAD_KEYS_MAX +#include // bool // pthread_key struct py_stub_tls_entry { @@ -77,7 +78,7 @@ struct _pythread_runtime_state { } stubs; #endif - // Linked list of ThreadHandleObjects + // Linked list of ThreadHandles struct llist_node handles; }; @@ -96,9 +97,9 @@ extern void _PyThread_AfterFork(struct _pythread_runtime_state *state); // unset: -1 seconds, in nanoseconds -#define PyThread_UNSET_TIMEOUT ((_PyTime_t)(-1 * 1000 * 1000 * 1000)) +#define PyThread_UNSET_TIMEOUT ((PyTime_t)(-1 * 1000 * 1000 * 1000)) -// Exported for the _xxinterpchannels module. +// Exported for the _interpchannels module. PyAPI_FUNC(int) PyThread_ParseTimeoutArg( PyObject *arg, int blocking, @@ -110,7 +111,7 @@ PyAPI_FUNC(int) PyThread_ParseTimeoutArg( * are returned, depending on whether the lock can be acquired within the * timeout. */ -// Exported for the _xxinterpchannels module. +// Exported for the _interpchannels module. PyAPI_FUNC(PyLockStatus) PyThread_acquire_lock_timed_with_retries( PyThread_type_lock, PY_TIMEOUT_T microseconds); @@ -146,11 +147,24 @@ PyAPI_FUNC(int) PyThread_start_joinable_thread(void (*func)(void *), PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); /* * Detach a thread started with `PyThread_start_joinable_thread`, such - * that its resources are relased as soon as it exits. + * that its resources are released as soon as it exits. * This function cannot be interrupted. It returns 0 on success, * a non-zero value on failure. */ PyAPI_FUNC(int) PyThread_detach_thread(PyThread_handle_t); +/* + * Hangs the thread indefinitely without exiting it. + * + * gh-87135: There is no safe way to exit a thread other than returning + * normally from its start function. This is used during finalization in lieu + * of actually exiting the thread. Since the program is expected to terminate + * soon anyway, it does not matter if the thread stack stays around until then. + * + * This is unfortunate for embedders who may not be terminating their process + * when they're done with the interpreter, but our C API design does not allow + * for safely exiting threads attempting to re-enter Python post finalization. + */ +void _Py_NO_RETURN PyThread_hang_thread(void); #ifdef __cplusplus } diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h new file mode 100644 index 00000000000000..20e643e172b38d --- /dev/null +++ b/Include/internal/pycore_qsbr.h @@ -0,0 +1,154 @@ +// The QSBR APIs (quiescent state-based reclamation) provide a mechanism for +// the free-threaded build to safely reclaim memory when there may be +// concurrent accesses. +// +// Many operations in the free-threaded build are protected by locks. However, +// in some cases, we want to allow reads to happen concurrently with updates. +// In this case, we need to delay freeing ("reclaiming") any memory that may be +// concurrently accessed by a reader. The QSBR APIs provide a way to do this. +#ifndef Py_INTERNAL_QSBR_H +#define Py_INTERNAL_QSBR_H + +#include +#include +#include "pycore_lock.h" // PyMutex + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// The shared write sequence is always odd and incremented by two. Detached +// threads are indicated by a read sequence of zero. This avoids collisions +// between the offline state and any valid sequence number even if the +// sequences numbers wrap around. +#define QSBR_OFFLINE 0 +#define QSBR_INITIAL 1 +#define QSBR_INCR 2 + +// Wrap-around safe comparison. This is a holdover from the FreeBSD +// implementation, which uses 32-bit sequence numbers. We currently use 64-bit +// sequence numbers, so wrap-around is unlikely. +#define QSBR_LT(a, b) ((int64_t)((a)-(b)) < 0) +#define QSBR_LEQ(a, b) ((int64_t)((a)-(b)) <= 0) + +struct _qsbr_shared; +struct _PyThreadStateImpl; // forward declare to avoid circular dependency + +// Per-thread state +struct _qsbr_thread_state { + // Last observed write sequence (or 0 if detached) + uint64_t seq; + + // Shared (per-interpreter) QSBR state + struct _qsbr_shared *shared; + + // Thread state (or NULL) + PyThreadState *tstate; + + // Used to defer advancing write sequence a fixed number of times + int deferrals; + + // Is this thread state allocated? + bool allocated; + struct _qsbr_thread_state *freelist_next; +}; + +// Padding to avoid false sharing +struct _qsbr_pad { + struct _qsbr_thread_state qsbr; + char __padding[64 - sizeof(struct _qsbr_thread_state)]; +}; + +// Per-interpreter state +struct _qsbr_shared { + // Write sequence: always odd, incremented by two + uint64_t wr_seq; + + // Minimum observed read sequence of all QSBR thread states + uint64_t rd_seq; + + // Array of QSBR thread states. + struct _qsbr_pad *array; + Py_ssize_t size; + + // Freelist of unused _qsbr_thread_states (protected by mutex) + PyMutex mutex; + struct _qsbr_thread_state *freelist; +}; + +static inline uint64_t +_Py_qsbr_shared_current(struct _qsbr_shared *shared) +{ + return _Py_atomic_load_uint64_acquire(&shared->wr_seq); +} + +// Reports a quiescent state: the caller no longer holds any pointer to shared +// data not protected by locks or reference counts. +static inline void +_Py_qsbr_quiescent_state(struct _qsbr_thread_state *qsbr) +{ + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); + _Py_atomic_store_uint64_release(&qsbr->seq, seq); +} + +// Have the read sequences advanced to the given goal? Like `_Py_qsbr_poll()`, +// but does not perform a scan of threads. +static inline bool +_Py_qbsr_goal_reached(struct _qsbr_thread_state *qsbr, uint64_t goal) +{ + uint64_t rd_seq = _Py_atomic_load_uint64(&qsbr->shared->rd_seq); + return QSBR_LEQ(goal, rd_seq); +} + +// Advance the write sequence and return the new goal. This should be called +// after data is removed. The returned goal is used with `_Py_qsbr_poll()` to +// determine when it is safe to reclaim (free) the memory. +extern uint64_t +_Py_qsbr_advance(struct _qsbr_shared *shared); + +// Batches requests to advance the write sequence. This advances the write +// sequence every N calls, which reduces overhead but increases time to +// reclamation. Returns the new goal. +extern uint64_t +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr); + +// Have the read sequences advanced to the given goal? If this returns true, +// it safe to reclaim any memory tagged with the goal (or earlier goal). +extern bool +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal); + +// Called when thread attaches to interpreter +extern void +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr); + +// Called when thread detaches from interpreter +extern void +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr); + +// Reserves (allocates) a QSBR state and returns its index. +extern Py_ssize_t +_Py_qsbr_reserve(PyInterpreterState *interp); + +// Associates a PyThreadState with the QSBR state at the given index +extern void +_Py_qsbr_register(struct _PyThreadStateImpl *tstate, + PyInterpreterState *interp, Py_ssize_t index); + +// Disassociates a PyThreadState from the QSBR state and frees the QSBR state. +extern void +_Py_qsbr_unregister(PyThreadState *tstate); + +extern void +_Py_qsbr_fini(PyInterpreterState *interp); + +extern void +_Py_qsbr_after_fork(struct _PyThreadStateImpl *tstate); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_QSBR_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 7c705d1224f915..7f592aa6cf9f05 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -9,8 +9,10 @@ extern "C" { #endif #include "pycore_atexit.h" // struct _atexit_runtime_state +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_debug_offsets.h" // _Py_DebugOffsets #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state @@ -25,110 +27,12 @@ extern "C" { #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state -struct _getargs_runtime_state { - struct _PyArg_Parser *static_parsers; -}; - -/* GIL state */ - -struct _gilstate_runtime_state { - /* bpo-26558: Flag to disable PyGILState_Check(). - If set to non-zero, PyGILState_Check() always return 1. */ - int check_enabled; - /* The single PyInterpreterState used by this process' - GILState implementation - */ - /* TODO: Given interp_main, it may be possible to kill this ref */ - PyInterpreterState *autoInterpreterState; -}; - -/* Runtime audit hook state */ - -typedef struct _Py_AuditHookEntry { - struct _Py_AuditHookEntry *next; - Py_AuditHookFunction hookCFunction; - void *userData; -} _Py_AuditHookEntry; - -typedef struct _Py_DebugOffsets { - char cookie[8]; - uint64_t version; - // Runtime state offset; - struct _runtime_state { - off_t finalizing; - off_t interpreters_head; - } runtime_state; - - // Interpreter state offset; - struct _interpreter_state { - off_t next; - off_t threads_head; - off_t gc; - off_t imports_modules; - off_t sysdict; - off_t builtins; - off_t ceval_gil; - off_t gil_runtime_state_locked; - off_t gil_runtime_state_holder; - } interpreter_state; - - // Thread state offset; - struct _thread_state{ - off_t prev; - off_t next; - off_t interp; - off_t current_frame; - off_t thread_id; - off_t native_thread_id; - } thread_state; - - // InterpreterFrame offset; - struct _interpreter_frame { - off_t previous; - off_t executable; - off_t instr_ptr; - off_t localsplus; - off_t owner; - } interpreter_frame; - - // CFrame offset; - struct _cframe { - off_t current_frame; - off_t previous; - } cframe; - - // Code object offset; - struct _code_object { - off_t filename; - off_t name; - off_t linetable; - off_t firstlineno; - off_t argcount; - off_t localsplusnames; - off_t localspluskinds; - off_t co_code_adaptive; - } code_object; - - // PyObject offset; - struct _pyobject { - off_t ob_type; - } pyobject; - - // PyTypeObject object offset; - struct _type_object { - off_t tp_name; - } type_object; - - // PyTuple object offset; - struct _tuple_object { - off_t ob_item; - } tuple_object; -} _Py_DebugOffsets; /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. - That data is exposed in the internal API as a static variable (_PyRuntime). + That data is exported by the internal API as a global variable + (_PyRuntime, defined near the top of pylifecycle.c). */ typedef struct pyruntimestate { /* This field must be first to facilitate locating it by out of process @@ -191,7 +95,10 @@ typedef struct pyruntimestate { int64_t next_id; } interpreters; + /* Platform-specific identifier and PyThreadState, respectively, for the + main thread in the main interpreter. */ unsigned long main_thread; + PyThreadState *main_tstate; /* ---------- IMPORTANT --------------------------- The fields above this line are declared as early as @@ -221,11 +128,23 @@ typedef struct pyruntimestate { struct _import_runtime_state imports; struct _ceval_runtime_state ceval; - struct _gilstate_runtime_state gilstate; - struct _getargs_runtime_state getargs; + struct _gilstate_runtime_state { + /* bpo-26558: Flag to disable PyGILState_Check(). + If set to non-zero, PyGILState_Check() always return 1. */ + int check_enabled; + /* The single PyInterpreterState used by this process' + GILState implementation + */ + /* TODO: Given interp_main, it may be possible to kill this ref */ + PyInterpreterState *autoInterpreterState; + } gilstate; + struct _getargs_runtime_state { + struct _PyArg_Parser *static_parsers; + } getargs; struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; + struct _reftracer_runtime_state ref_tracer; // The rwmutex is used to prevent overlapping global and per-interpreter // stop-the-world events. Global stop-the-world events lock the mutex @@ -325,6 +244,7 @@ _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { } } + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 571a7d612c94e2..e99febab2f3d57 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -9,14 +9,17 @@ extern "C" { #endif #include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT +#include "pycore_debug_offsets.h" // _Py_DebugOffsets_INIT() #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT #include "pycore_floatobject.h" // _py_float_format_unknown +#include "pycore_function.h" #include "pycore_object.h" // _PyObject_HEAD_INIT #include "pycore_obmalloc_init.h" // _obmalloc_global_state_INIT #include "pycore_parser.h" // _parser_runtime_state_INIT #include "pycore_pyhash.h" // pyhash_state_INIT #include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT #include "pycore_pythread.h" // _pythread_RUNTIME_INIT +#include "pycore_qsbr.h" // QSBR_INITIAL #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT @@ -28,61 +31,9 @@ extern PyTypeObject _PyExc_MemoryError; /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ -#define _PyRuntimeState_INIT(runtime) \ +#define _PyRuntimeState_INIT(runtime, debug_cookie) \ { \ - .debug_offsets = { \ - .cookie = "xdebugpy", \ - .version = PY_VERSION_HEX, \ - .runtime_state = { \ - .finalizing = offsetof(_PyRuntimeState, _finalizing), \ - .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ - }, \ - .interpreter_state = { \ - .next = offsetof(PyInterpreterState, next), \ - .threads_head = offsetof(PyInterpreterState, threads.head), \ - .gc = offsetof(PyInterpreterState, gc), \ - .imports_modules = offsetof(PyInterpreterState, imports.modules), \ - .sysdict = offsetof(PyInterpreterState, sysdict), \ - .builtins = offsetof(PyInterpreterState, builtins), \ - .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ - .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ - .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ - }, \ - .thread_state = { \ - .prev = offsetof(PyThreadState, prev), \ - .next = offsetof(PyThreadState, next), \ - .interp = offsetof(PyThreadState, interp), \ - .current_frame = offsetof(PyThreadState, current_frame), \ - .thread_id = offsetof(PyThreadState, thread_id), \ - .native_thread_id = offsetof(PyThreadState, native_thread_id), \ - }, \ - .interpreter_frame = { \ - .previous = offsetof(_PyInterpreterFrame, previous), \ - .executable = offsetof(_PyInterpreterFrame, f_executable), \ - .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ - .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ - .owner = offsetof(_PyInterpreterFrame, owner), \ - }, \ - .code_object = { \ - .filename = offsetof(PyCodeObject, co_filename), \ - .name = offsetof(PyCodeObject, co_name), \ - .linetable = offsetof(PyCodeObject, co_linetable), \ - .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ - .argcount = offsetof(PyCodeObject, co_argcount), \ - .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ - .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ - .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ - }, \ - .pyobject = { \ - .ob_type = offsetof(PyObject, ob_type), \ - }, \ - .type_object = { \ - .tp_name = offsetof(PyTypeObject, tp_name), \ - }, \ - .tuple_object = { \ - .ob_item = offsetof(PyTupleObject, ob_item), \ - }, \ - }, \ + .debug_offsets = _Py_DebugOffsets_INIT(debug_cookie), \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ .debug = _pymem_allocators_debug_INIT, \ @@ -108,6 +59,10 @@ extern PyTypeObject _PyExc_MemoryError; .autoTSSkey = Py_tss_NEEDS_INIT, \ .parser = _parser_runtime_state_INIT, \ .ceval = { \ + .pending_mainthread = { \ + .max = MAXPENDINGCALLS_MAIN, \ + .maxloop = MAXPENDINGCALLSLOOP_MAIN, \ + }, \ .perf = _PyEval_RUNTIME_PERF_INIT, \ }, \ .gilstate = { \ @@ -118,6 +73,10 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ + .ref_tracer = { \ + .tracer_func = NULL, \ + .tracer_data = NULL, \ + }, \ .stoptheworld = { \ .is_global = 1, \ }, \ @@ -156,24 +115,33 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyInterpreterState_INIT(INTERP) \ { \ .id_refcount = -1, \ + ._whence = _PyInterpreterState_WHENCE_NOTSET, \ .imports = IMPORTS_INIT, \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ + .pending = { \ + .max = MAXPENDINGCALLS, \ + .maxloop = MAXPENDINGCALLSLOOP, \ + }, \ }, \ .gc = { \ .enabled = 1, \ - .generations = { \ - /* .head is set in _PyGC_InitState(). */ \ - { .threshold = 700, }, \ - { .threshold = 10, }, \ + .young = { .threshold = 2000, }, \ + .old = { \ { .threshold = 10, }, \ + { .threshold = 0, }, \ }, \ + .work_to_do = -5000, \ + }, \ + .qsbr = { \ + .wr_seq = QSBR_INITIAL, \ + .rd_seq = QSBR_INITIAL, \ }, \ - .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ + .mem_free_queue = _Py_mem_free_queue_INIT(INTERP.mem_free_queue), \ .func_state = { \ - .next_version = 1, \ + .next_version = FUNC_VERSION_FIRST_VALID, \ }, \ .types = { \ .next_version_tag = _Py_TYPE_BASE_VERSION_TAG, \ @@ -206,16 +174,6 @@ extern PyTypeObject _PyExc_MemoryError; .context_ver = 1, \ } -#ifdef Py_TRACE_REFS -# define _py_object_state_INIT(INTERP) \ - { \ - .refchain = NULL, \ - } -#else -# define _py_object_state_INIT(INTERP) \ - { 0 } -#endif - // global objects diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 68fbbcb4378e17..3b80e265b0ca50 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -546,21 +546,18 @@ extern "C" { INIT_STR(anon_setcomp, ""), \ INIT_STR(anon_string, ""), \ INIT_STR(anon_unknown, ""), \ - INIT_STR(close_br, "}"), \ INIT_STR(dbl_close_br, "}}"), \ INIT_STR(dbl_open_br, "{{"), \ INIT_STR(dbl_percent, "%%"), \ INIT_STR(defaults, ".defaults"), \ - INIT_STR(dot, "."), \ INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ + INIT_STR(format, ".format"), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ - INIT_STR(newline, "\n"), \ - INIT_STR(open_br, "{"), \ - INIT_STR(percent, "%"), \ + INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } @@ -575,7 +572,6 @@ extern "C" { INIT_ID(TextIOWrapper), \ INIT_ID(True), \ INIT_ID(WarningMessage), \ - INIT_ID(_), \ INIT_ID(_WindowsConsoleIO), \ INIT_ID(__IOBase_closed), \ INIT_ID(__abc_tpflags__), \ @@ -588,9 +584,9 @@ extern "C" { INIT_ID(__all__), \ INIT_ID(__and__), \ INIT_ID(__anext__), \ + INIT_ID(__annotate__), \ INIT_ID(__annotations__), \ INIT_ID(__args__), \ - INIT_ID(__asyncio_running_event_loop__), \ INIT_ID(__await__), \ INIT_ID(__bases__), \ INIT_ID(__bool__), \ @@ -607,7 +603,6 @@ extern "C" { INIT_ID(__classdictcell__), \ INIT_ID(__complex__), \ INIT_ID(__contains__), \ - INIT_ID(__copy__), \ INIT_ID(__ctypes_from_outparam__), \ INIT_ID(__del__), \ INIT_ID(__delattr__), \ @@ -622,6 +617,7 @@ extern "C" { INIT_ID(__eq__), \ INIT_ID(__exit__), \ INIT_ID(__file__), \ + INIT_ID(__firstlineno__), \ INIT_ID(__float__), \ INIT_ID(__floordiv__), \ INIT_ID(__format__), \ @@ -722,12 +718,12 @@ extern "C" { INIT_ID(__slotnames__), \ INIT_ID(__slots__), \ INIT_ID(__spec__), \ + INIT_ID(__static_attributes__), \ INIT_ID(__str__), \ INIT_ID(__sub__), \ INIT_ID(__subclasscheck__), \ INIT_ID(__subclasshook__), \ INIT_ID(__truediv__), \ - INIT_ID(__trunc__), \ INIT_ID(__type_params__), \ INIT_ID(__typing_is_unpacked_typevartuple__), \ INIT_ID(__typing_prepare_subst__), \ @@ -740,7 +736,6 @@ extern "C" { INIT_ID(_abc_impl), \ INIT_ID(_abstract_), \ INIT_ID(_active), \ - INIT_ID(_annotation), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ @@ -750,6 +745,7 @@ extern "C" { INIT_ID(_check_retval_), \ INIT_ID(_dealloc_warn), \ INIT_ID(_feature_version), \ + INIT_ID(_field_types), \ INIT_ID(_fields_), \ INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ @@ -760,23 +756,25 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ + INIT_ID(_isatty_open_only), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ INIT_ID(_loop), \ INIT_ID(_needs_com_addref_), \ - INIT_ID(_pack_), \ + INIT_ID(_only_immortal), \ INIT_ID(_restype_), \ INIT_ID(_showwarnmsg), \ INIT_ID(_shutdown), \ INIT_ID(_slotnames), \ - INIT_ID(_strptime_datetime), \ - INIT_ID(_swappedbytes_), \ + INIT_ID(_strptime), \ + INIT_ID(_strptime_datetime_date), \ + INIT_ID(_strptime_datetime_datetime), \ + INIT_ID(_strptime_datetime_time), \ INIT_ID(_type_), \ INIT_ID(_uninitialized_submodules), \ INIT_ID(_warn_unawaited_coroutine), \ INIT_ID(_xoptions), \ - INIT_ID(a), \ INIT_ID(abs_tol), \ INIT_ID(access), \ INIT_ID(aclose), \ @@ -786,8 +784,10 @@ extern "C" { INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ INIT_ID(alias), \ + INIT_ID(align), \ INIT_ID(allow_code), \ INIT_ID(append), \ + INIT_ID(arg), \ INIT_ID(argdefs), \ INIT_ID(args), \ INIT_ID(arguments), \ @@ -799,12 +799,12 @@ extern "C" { INIT_ID(attribute), \ INIT_ID(authorizer_callback), \ INIT_ID(autocommit), \ - INIT_ID(b), \ INIT_ID(backtick), \ INIT_ID(base), \ INIT_ID(before), \ INIT_ID(big), \ INIT_ID(binary_form), \ + INIT_ID(bit_size), \ INIT_ID(block), \ INIT_ID(bound), \ INIT_ID(buffer), \ @@ -817,10 +817,10 @@ extern "C" { INIT_ID(byteorder), \ INIT_ID(bytes), \ INIT_ID(bytes_per_sep), \ - INIT_ID(c), \ INIT_ID(c_call), \ INIT_ID(c_exception), \ INIT_ID(c_return), \ + INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ INIT_ID(cafile), \ @@ -858,6 +858,7 @@ extern "C" { INIT_ID(co_stacksize), \ INIT_ID(co_varnames), \ INIT_ID(code), \ + INIT_ID(col_offset), \ INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ @@ -871,7 +872,6 @@ extern "C" { INIT_ID(count), \ INIT_ID(covariant), \ INIT_ID(cwd), \ - INIT_ID(d), \ INIT_ID(data), \ INIT_ID(database), \ INIT_ID(day), \ @@ -881,6 +881,7 @@ extern "C" { INIT_ID(defaultaction), \ INIT_ID(delete), \ INIT_ID(depth), \ + INIT_ID(desired_access), \ INIT_ID(detect_types), \ INIT_ID(deterministic), \ INIT_ID(device), \ @@ -896,16 +897,17 @@ extern "C" { INIT_ID(displayhook), \ INIT_ID(dklen), \ INIT_ID(doc), \ + INIT_ID(done), \ INIT_ID(dont_inherit), \ INIT_ID(dst), \ INIT_ID(dst_dir_fd), \ - INIT_ID(e), \ INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ INIT_ID(encode), \ INIT_ID(encoding), \ INIT_ID(end), \ + INIT_ID(end_col_offset), \ INIT_ID(end_lineno), \ INIT_ID(end_offset), \ INIT_ID(endpos), \ @@ -922,7 +924,6 @@ extern "C" { INIT_ID(exp), \ INIT_ID(extend), \ INIT_ID(extra_tokens), \ - INIT_ID(f), \ INIT_ID(facility), \ INIT_ID(factory), \ INIT_ID(false), \ @@ -932,6 +933,7 @@ extern "C" { INIT_ID(fd2), \ INIT_ID(fdel), \ INIT_ID(fget), \ + INIT_ID(fields), \ INIT_ID(file), \ INIT_ID(file_actions), \ INIT_ID(filename), \ @@ -948,6 +950,7 @@ extern "C" { INIT_ID(fold), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ + INIT_ID(format_spec), \ INIT_ID(from_param), \ INIT_ID(fromlist), \ INIT_ID(fromtimestamp), \ @@ -955,7 +958,6 @@ extern "C" { INIT_ID(fset), \ INIT_ID(func), \ INIT_ID(future), \ - INIT_ID(g), \ INIT_ID(generation), \ INIT_ID(genexpr), \ INIT_ID(get), \ @@ -969,15 +971,15 @@ extern "C" { INIT_ID(globals), \ INIT_ID(groupindex), \ INIT_ID(groups), \ - INIT_ID(h), \ INIT_ID(handle), \ + INIT_ID(handle_seq), \ + INIT_ID(has_location), \ INIT_ID(hash_name), \ INIT_ID(header), \ INIT_ID(headers), \ INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ - INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ @@ -985,12 +987,16 @@ extern "C" { INIT_ID(importlib), \ INIT_ID(in_fd), \ INIT_ID(incoming), \ + INIT_ID(index), \ INIT_ID(indexgroup), \ INIT_ID(inf), \ INIT_ID(infer_variance), \ + INIT_ID(inherit_handle), \ INIT_ID(inheritable), \ INIT_ID(initial), \ INIT_ID(initial_bytes), \ + INIT_ID(initial_owner), \ + INIT_ID(initial_state), \ INIT_ID(initial_value), \ INIT_ID(initval), \ INIT_ID(inner_size), \ @@ -1002,6 +1008,7 @@ extern "C" { INIT_ID(intersection), \ INIT_ID(interval), \ INIT_ID(is_running), \ + INIT_ID(is_struct), \ INIT_ID(isatty), \ INIT_ID(isinstance), \ INIT_ID(isoformat), \ @@ -1023,6 +1030,7 @@ extern "C" { INIT_ID(kw1), \ INIT_ID(kw2), \ INIT_ID(kwdefaults), \ + INIT_ID(label), \ INIT_ID(lambda), \ INIT_ID(last), \ INIT_ID(last_exc), \ @@ -1046,6 +1054,7 @@ extern "C" { INIT_ID(locals), \ INIT_ID(logoption), \ INIT_ID(loop), \ + INIT_ID(manual_reset), \ INIT_ID(mapping), \ INIT_ID(match), \ INIT_ID(max_length), \ @@ -1062,6 +1071,7 @@ extern "C" { INIT_ID(metadata), \ INIT_ID(method), \ INIT_ID(microsecond), \ + INIT_ID(milliseconds), \ INIT_ID(minute), \ INIT_ID(mod), \ INIT_ID(mode), \ @@ -1071,8 +1081,8 @@ extern "C" { INIT_ID(month), \ INIT_ID(mro), \ INIT_ID(msg), \ + INIT_ID(mutex), \ INIT_ID(mycmp), \ - INIT_ID(n), \ INIT_ID(n_arg), \ INIT_ID(n_fields), \ INIT_ID(n_sequence_fields), \ @@ -1083,6 +1093,7 @@ extern "C" { INIT_ID(namespaces), \ INIT_ID(narg), \ INIT_ID(ndigits), \ + INIT_ID(nested), \ INIT_ID(new_file_name), \ INIT_ID(new_limit), \ INIT_ID(newline), \ @@ -1117,7 +1128,6 @@ extern "C" { INIT_ID(outgoing), \ INIT_ID(overlapped), \ INIT_ID(owner), \ - INIT_ID(p), \ INIT_ID(pages), \ INIT_ID(parent), \ INIT_ID(password), \ @@ -1145,7 +1155,6 @@ extern "C" { INIT_ID(ps2), \ INIT_ID(query), \ INIT_ID(quotetabs), \ - INIT_ID(r), \ INIT_ID(raw), \ INIT_ID(read), \ INIT_ID(read1), \ @@ -1169,11 +1178,11 @@ extern "C" { INIT_ID(return), \ INIT_ID(reverse), \ INIT_ID(reversed), \ - INIT_ID(s), \ INIT_ID(salt), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ INIT_ID(second), \ + INIT_ID(security_attributes), \ INIT_ID(seek), \ INIT_ID(seekable), \ INIT_ID(selectors), \ @@ -1201,6 +1210,7 @@ extern "C" { INIT_ID(sort), \ INIT_ID(source), \ INIT_ID(source_traceback), \ + INIT_ID(spam), \ INIT_ID(src), \ INIT_ID(src_dir_fd), \ INIT_ID(stacklevel), \ @@ -1261,6 +1271,8 @@ extern "C" { INIT_ID(values), \ INIT_ID(version), \ INIT_ID(volume), \ + INIT_ID(wait_all), \ + INIT_ID(warn_on_full_buffer), \ INIT_ID(warnings), \ INIT_ID(warnoptions), \ INIT_ID(wbits), \ @@ -1272,7 +1284,6 @@ extern "C" { INIT_ID(writable), \ INIT_ID(write), \ INIT_ID(write_through), \ - INIT_ID(x), \ INIT_ID(year), \ INIT_ID(zdict), \ } diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 4c37df7b39a48a..ffcc6d80344d6e 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -8,7 +8,6 @@ #endif #include "pycore_pythread.h" // _POSIX_SEMAPHORES -#include "pycore_time.h" // _PyTime_t #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN @@ -48,7 +47,7 @@ typedef struct _PySemaphore { // If `detach` is true, then the thread will detach/release the GIL while // sleeping. PyAPI_FUNC(int) -_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout_ns, int detach); +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns, int detach); // Wakes up a single thread waiting on sema. Note that _PySemaphore_Wakeup() // can be called before _PySemaphore_Wait(). diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 34a00e6d45fe69..0494c07fe1869d 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -8,19 +8,31 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -// Export for '_pickle' shared extension +// Export for '_abc' shared extension PyAPI_FUNC(int) _PySet_NextEntry( PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash); +// Export for '_pickle' shared extension +PyAPI_FUNC(int) _PySet_NextEntryRef( + PyObject *set, + Py_ssize_t *pos, + PyObject **key, + Py_hash_t *hash); + // Export for '_pickle' shared extension PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); // Export for the gdb plugin's (python-gdb.py) benefit PyAPI_DATA(PyObject *) _PySet_Dummy; +PyAPI_FUNC(int) _PySet_Contains(PySetObject *so, PyObject *key); + +// Clears the set without acquiring locks. Used by _PyCode_Fini. +extern void _PySet_ClearInternal(PySetObject *so); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index 89086f67683a2f..ba8b1f1cb27dee 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -11,7 +11,7 @@ extern "C" { /* runtime lifecycle */ -extern PyObject * +PyAPI_FUNC(PyObject *) _PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); #ifdef __cplusplus diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h new file mode 100644 index 00000000000000..588e57f6cd97e0 --- /dev/null +++ b/Include/internal/pycore_stackref.h @@ -0,0 +1,264 @@ +#ifndef Py_INTERNAL_STACKREF_H +#define Py_INTERNAL_STACKREF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_object_deferred.h" + +#include +#include + +/* + This file introduces a new API for handling references on the stack, called + _PyStackRef. This API is inspired by HPy. + + There are 3 main operations, that convert _PyStackRef to PyObject* and + vice versa: + + 1. Borrow (discouraged) + 2. Steal + 3. New + + Borrow means that the reference is converted without any change in ownership. + This is discouraged because it makes verification much harder. It also makes + unboxed integers harder in the future. + + Steal means that ownership is transferred to something else. The total + number of references to the object stays the same. + + New creates a new reference from the old reference. The old reference + is still valid. + + With these 3 API, a strict stack discipline must be maintained. All + _PyStackRef must be operated on by the new reference operations: + + 1. DUP + 2. CLOSE + + DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old + reference. The old reference remains unchanged. + + CLOSE is roughly equivalent to Py_DECREF. It destroys a reference. + + Note that it is unsafe to borrow a _PyStackRef and then do normal + CPython refcounting operations on it! +*/ + +typedef union _PyStackRef { + uintptr_t bits; +} _PyStackRef; + + +#define Py_TAG_DEFERRED (1) + +#define Py_TAG_PTR ((uintptr_t)0) +#define Py_TAG_BITS ((uintptr_t)1) + +#ifdef Py_GIL_DISABLED + +static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; +#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) +#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) +#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) +#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) + +static inline PyObject * +PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) +{ + PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); + return cleared; +} + +#define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) + +static inline PyObject * +PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref) +{ + assert(!PyStackRef_IsDeferred(stackref)); + return (PyObject *)stackref.bits; +} + +static inline PyObject * +PyStackRef_AsPyObjectSteal(_PyStackRef stackref) +{ + assert(!PyStackRef_IsNull(stackref)); + if (PyStackRef_IsDeferred(stackref)) { + return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); + } + return PyStackRef_AsPyObjectBorrow(stackref); +} + +static inline _PyStackRef +_PyStackRef_FromPyObjectSteal(PyObject *obj) +{ + assert(obj != NULL); + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + unsigned int tag = _Py_IsImmortal(obj) ? (Py_TAG_DEFERRED) : Py_TAG_PTR; + return ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag}); +} +# define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) + +static inline _PyStackRef +PyStackRef_FromPyObjectNew(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + assert(obj != NULL); + if (_Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; + } + else { + return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; + } +} +#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) + +static inline _PyStackRef +PyStackRef_FromPyObjectImmortal(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + assert(obj != NULL); + assert(_Py_IsImmortal(obj)); + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; +} +#define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj)) + +#define PyStackRef_CLOSE(REF) \ + do { \ + _PyStackRef _close_tmp = (REF); \ + assert(!PyStackRef_IsNull(_close_tmp)); \ + if (!PyStackRef_IsDeferred(_close_tmp)) { \ + Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ + } \ + } while (0) + +static inline _PyStackRef +PyStackRef_DUP(_PyStackRef stackref) +{ + assert(!PyStackRef_IsNull(stackref)); + if (PyStackRef_IsDeferred(stackref)) { + assert(_Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) || + _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref)) + ); + return stackref; + } + Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); + return stackref; +} + +// Convert a possibly deferred reference to a strong reference. +static inline _PyStackRef +PyStackRef_AsStrongReference(_PyStackRef stackref) +{ + return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); +} + +#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref) + + +#else // Py_GIL_DISABLED + +// With GIL +static const _PyStackRef PyStackRef_NULL = { .bits = 0 }; +#define PyStackRef_IsNull(stackref) ((stackref).bits == 0) +#define PyStackRef_True ((_PyStackRef){.bits = (uintptr_t)&_Py_TrueStruct }) +#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) }) +#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) }) + +#define PyStackRef_AsPyObjectBorrow(stackref) ((PyObject *)(stackref).bits) + +#define PyStackRef_AsPyObjectSteal(stackref) PyStackRef_AsPyObjectBorrow(stackref) + +#define PyStackRef_FromPyObjectSteal(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))}) + +#define PyStackRef_FromPyObjectNew(obj) ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) }) + +#define PyStackRef_FromPyObjectImmortal(obj) ((_PyStackRef){ .bits = (uintptr_t)(obj) }) + +#define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref)) + +#define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))) + +#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) _Py_DECREF_SPECIALIZED(PyStackRef_AsPyObjectBorrow(stackref), dealloc) + +#endif // Py_GIL_DISABLED + +// Note: this is a macro because MSVC (Windows) has trouble inlining it. + +#define PyStackRef_Is(a, b) ((a).bits == (b).bits) + +// Converts a PyStackRef back to a PyObject *, converting the +// stackref to a new reference. +#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)) + +#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) + +#define PyStackRef_CLEAR(op) \ + do { \ + _PyStackRef *_tmp_op_ptr = &(op); \ + _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ + if (!PyStackRef_IsNull(_tmp_old_op)) { \ + *_tmp_op_ptr = PyStackRef_NULL; \ + PyStackRef_CLOSE(_tmp_old_op); \ + } \ + } while (0) + +#define PyStackRef_XCLOSE(stackref) \ + do { \ + _PyStackRef _tmp = (stackref); \ + if (!PyStackRef_IsNull(_tmp)) { \ + PyStackRef_CLOSE(_tmp); \ + } \ + } while (0); + + + +// StackRef type checks + +static inline bool +PyStackRef_GenCheck(_PyStackRef stackref) +{ + return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_BoolCheck(_PyStackRef stackref) +{ + return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_LongCheck(_PyStackRef stackref) +{ + return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) +{ + return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_CodeCheck(_PyStackRef stackref) +{ + return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_FunctionCheck(_PyStackRef stackref) +{ + return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_STACKREF_H */ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index b44393b5644673..91dac767d5885b 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -12,14 +12,27 @@ struct _mod; // Type defined in pycore_ast.h typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, - // Used for annotations if 'from __future__ import annotations' is active. - // Annotation blocks cannot bind names and are not evaluated. + // Used for annotations. If 'from __future__ import annotations' is active, + // annotation blocks cannot bind names and are not evaluated. Otherwise, they + // are lazily evaluated (see PEP 649). AnnotationBlock, - // Used for generics and type aliases. These work mostly like functions - // (see PEP 695 for details). The three different blocks function identically; - // they are different enum entries only so that error messages can be more - // precise. - TypeVarBoundBlock, TypeAliasBlock, TypeParamBlock + + // The following blocks are used for generics and type aliases. These work + // mostly like functions (see PEP 695 for details). The three different + // blocks function identically; they are different enum entries only so + // that error messages can be more precise. + + // The block to enter when processing a "type" (PEP 695) construction, + // e.g., "type MyGeneric[T] = list[T]". + TypeAliasBlock, + // The block to enter when processing a "generic" (PEP 695) object, + // e.g., "def foo[T](): pass" or "class A[T]: pass". + TypeParametersBlock, + // The block to enter when processing the bound, the constraint tuple + // or the default value of a single "type variable" in the formal sense, + // i.e., a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two + // do not support a bound or a constraint tuple). + TypeVariableBlock, } _Py_block_ty; typedef enum _comprehension_type { @@ -29,6 +42,29 @@ typedef enum _comprehension_type { SetComprehension = 3, GeneratorExpression = 4 } _Py_comprehension_ty; +/* source location information */ +typedef struct { + int lineno; + int end_lineno; + int col_offset; + int end_col_offset; +} _Py_SourceLocation; + +#define SRC_LOCATION_FROM_AST(n) \ + (_Py_SourceLocation){ \ + .lineno = (n)->lineno, \ + .end_lineno = (n)->end_lineno, \ + .col_offset = (n)->col_offset, \ + .end_col_offset = (n)->end_col_offset } + +static const _Py_SourceLocation NO_LOCATION = {-1, -1, -1, -1}; + +/* __future__ information */ +typedef struct { + int ff_features; /* flags set by future statements */ + _Py_SourceLocation ff_location; /* location of last future statement */ +} _PyFutureFeatures; + struct _symtable_entry; struct symtable { @@ -44,7 +80,7 @@ struct symtable { consistency with the corresponding compiler structure */ PyObject *st_private; /* name of current class or NULL */ - PyFutureFeatures *st_future; /* module's future features that affect + _PyFutureFeatures *st_future; /* module's future features that affect the symbol table */ int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ @@ -58,13 +94,21 @@ typedef struct _symtable_entry { PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ + PyObject *ste_mangled_names; /* set of names for which mangling should be applied */ + _Py_block_ty ste_type; + // Optional string set by symtable.c and used when reporting errors. + // The content of that string is a description of the current "context". + // + // For instance, if we are processing the default value of the type + // variable "T" in "def foo[T = int](): pass", `ste_scope_info` is + // set to "a TypeVar default". + const char *ste_scope_info; + int ste_nested; /* true if block is nested */ - unsigned ste_free : 1; /* true if block has free variables */ - unsigned ste_child_free : 1; /* true if a child block has free vars, - including free refs to globals */ unsigned ste_generator : 1; /* true if namespace is a generator */ unsigned ste_coroutine : 1; /* true if namespace is a coroutine */ + unsigned ste_annotations_used : 1; /* true if there are any annotations in this scope */ _Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */ unsigned ste_varargs : 1; /* true if block has varargs */ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ @@ -79,13 +123,10 @@ typedef struct _symtable_entry { unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */ unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an enclosing class scope */ + unsigned ste_has_docstring : 1; /* true if docstring present */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ - int ste_lineno; /* first line of block */ - int ste_col_offset; /* offset of first line of block */ - int ste_end_lineno; /* end line of block */ - int ste_end_col_offset; /* end offset of first line of block */ - int ste_opt_lineno; /* lineno of last exec or import * */ - int ste_opt_col_offset; /* offset of last exec or import * */ + _Py_SourceLocation ste_loc; /* source location of block */ + struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ struct symtable *ste_table; } PySTEntryObject; @@ -100,11 +141,13 @@ extern int _PyST_IsFunctionLike(PySTEntryObject *); extern struct symtable* _PySymtable_Build( struct _mod *mod, PyObject *filename, - PyFutureFeatures *future); + _PyFutureFeatures *future); extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); +extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **); extern void _PySymtable_Free(struct symtable *); +extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name); extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); /* Flags for def-use information */ @@ -114,7 +157,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define DEF_PARAM (2<<1) /* formal parameter */ #define DEF_NONLOCAL (2<<2) /* nonlocal stmt */ #define USE (2<<3) /* name is used */ -#define DEF_FREE (2<<4) /* name used but not defined in nested block */ #define DEF_FREE_CLASS (2<<5) /* free variable from class's method */ #define DEF_IMPORT (2<<6) /* assignment occurred via import */ #define DEF_ANNOT (2<<7) /* this name is annotated */ @@ -125,11 +167,12 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) /* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol - table. GLOBAL is returned from PyST_GetScope() for either of them. + table. GLOBAL is returned from _PyST_GetScope() for either of them. It is stored in ste_symbols at bits 13-16. */ #define SCOPE_OFFSET 12 #define SCOPE_MASK (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL) +#define SYMBOL_TO_SCOPE(S) (((S) >> SCOPE_OFFSET) & SCOPE_MASK) #define LOCAL 1 #define GLOBAL_EXPLICIT 2 @@ -137,9 +180,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define FREE 4 #define CELL 5 -#define GENERATOR 1 -#define GENERATOR_EXPRESSION 2 - // Used by symtablemodule.c extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, @@ -150,7 +190,7 @@ extern struct symtable* _Py_SymtableStringObjectFlags( int _PyFuture_FromAST( struct _mod * mod, PyObject *filename, - PyFutureFeatures* futures); + _PyFutureFeatures* futures); #ifdef __cplusplus } diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 9b8eafd3d6cfd3..99968df54a45f6 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -14,21 +14,14 @@ PyAPI_FUNC(PyObject*) _PySys_GetAttr(PyThreadState *tstate, PyObject *name); // Export for '_pickle' shared extension PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); -extern int _PySys_Audit( - PyThreadState *tstate, - const char *event, - const char *argFormat, - ...); - -// _PySys_ClearAuditHooks() must not be exported: use extern rather than -// PyAPI_FUNC(). We want minimal exposure of this function. -extern void _PySys_ClearAuditHooks(PyThreadState *tstate); - extern int _PySys_SetAttr(PyObject *, PyObject *); extern int _PySys_ClearAttrString(PyInterpreterState *interp, const char *name, int verbose); +extern int _PySys_SetFlagObj(Py_ssize_t pos, PyObject *new_value); +extern int _PySys_SetIntMaxStrDigits(int maxdigits); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index dabbd7b41556cd..205ac5d3781ddd 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -1,34 +1,39 @@ -// The _PyTime_t API is written to use timestamp and timeout values stored in -// various formats and to read clocks. +// Internal PyTime_t C API: see Doc/c-api/time.rst for the documentation. // -// The _PyTime_t type is an integer to support directly common arithmetic -// operations like t1 + t2. +// The PyTime_t type is an integer to support directly common arithmetic +// operations such as t1 + t2. // -// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type -// is signed to support negative timestamps. The supported range is around -// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the -// supported date range is around [1677-09-21; 2262-04-11]. +// Time formats: // -// Formats: +// * Seconds. +// * Seconds as a floating-point number (C double). +// * Milliseconds (10^-3 seconds). +// * Microseconds (10^-6 seconds). +// * 100 nanoseconds (10^-7 seconds), used on Windows. +// * Nanoseconds (10^-9 seconds). +// * timeval structure, 1 microsecond (10^-6 seconds). +// * timespec structure, 1 nanosecond (10^-9 seconds). // -// * seconds -// * seconds as a floating pointer number (C double) -// * milliseconds (10^-3 seconds) -// * microseconds (10^-6 seconds) -// * 100 nanoseconds (10^-7 seconds) -// * nanoseconds (10^-9 seconds) -// * timeval structure, 1 microsecond resolution (10^-6 seconds) -// * timespec structure, 1 nanosecond resolution (10^-9 seconds) +// Note that PyTime_t is now specified as int64_t, in nanoseconds. +// (If we need to change this, we'll need new public API with new names.) +// Previously, PyTime_t was configurable (in theory); some comments and code +// might still allude to that. // // Integer overflows are detected and raise OverflowError. Conversion to a -// resolution worse than 1 nanosecond is rounded correctly with the requested -// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling -// (towards +inf), half even and up (away from zero). +// resolution larger than 1 nanosecond is rounded correctly with the requested +// rounding mode. Available rounding modes: // -// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so -// the caller doesn't have to handle errors and doesn't need to hold the GIL. -// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on -// overflow. +// * Round towards minus infinity (-inf). For example, used to read a clock. +// * Round towards infinity (+inf). For example, used for timeout to wait "at +// least" N seconds. +// * Round to nearest with ties going to nearest even integer. For example, used +// to round from a Python float. +// * Round away from zero. For example, used for timeout. +// +// Some functions clamp the result in the range [PyTime_MIN; PyTime_MAX]. The +// caller doesn't have to handle errors and so doesn't need to hold the GIL to +// handle exceptions. For example, _PyTime_Add(t1, t2) computes t1+t2 and +// clamps the result on overflow. // // Clocks: // @@ -36,10 +41,11 @@ // * Monotonic clock // * Performance counter // -// Operations like (t * k / q) with integers are implemented in a way to reduce -// the risk of integer overflow. Such operation is used to convert a clock -// value expressed in ticks with a frequency to _PyTime_t, like -// QueryPerformanceCounter() with QueryPerformanceFrequency(). +// Internally, operations like (t * k / q) with integers are implemented in a +// way to reduce the risk of integer overflow. Such operation is used to convert a +// clock value expressed in ticks with a frequency to PyTime_t, like +// QueryPerformanceCounter() with QueryPerformanceFrequency() on Windows. + #ifndef Py_INTERNAL_TIME_H #define Py_INTERNAL_TIME_H @@ -56,14 +62,6 @@ extern "C" { struct timeval; #endif -// _PyTime_t: Python timestamp with subsecond precision. It can be used to -// store a duration, and so indirectly a date (related to another date, like -// UNIX epoch). -typedef int64_t _PyTime_t; -// _PyTime_MIN nanoseconds is around -292.3 years -#define _PyTime_MIN INT64_MIN -// _PyTime_MAX nanoseconds is around +292.3 years -#define _PyTime_MAX INT64_MAX #define _SIZEOF_PYTIME_T 8 typedef enum { @@ -131,76 +129,66 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( // Create a timestamp from a number of seconds. // Export for '_socket' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); +PyAPI_FUNC(PyTime_t) _PyTime_FromSeconds(int seconds); // Create a timestamp from a number of seconds in double. -// Export for '_socket' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); +extern int _PyTime_FromSecondsDouble( + double seconds, + _PyTime_round_t round, + PyTime_t *result); // Macro to create a timestamp from a number of seconds, no integer overflow. // Only use the macro for small values, prefer _PyTime_FromSeconds(). #define _PYTIME_FROMSECONDS(seconds) \ - ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) - -// Create a timestamp from a number of nanoseconds. -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); + ((PyTime_t)(seconds) * (1000 * 1000 * 1000)) // Create a timestamp from a number of microseconds. -// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -extern _PyTime_t _PyTime_FromMicrosecondsClamp(_PyTime_t us); +// Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTime_FromMicrosecondsClamp(PyTime_t us); -// Create a timestamp from nanoseconds (Python int). +// Create a timestamp from a Python int object (number of nanoseconds). // Export for '_lsprof' shared extension. -PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromLong(PyTime_t *t, PyObject *obj); // Convert a number of seconds (Python float or int) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for '_socket' shared extension. -PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromSecondsObject(PyTime_t *t, PyObject *obj, _PyTime_round_t round); // Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for 'select' shared extension. -PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(PyTime_t *t, PyObject *obj, _PyTime_round_t round); -// Convert a timestamp to a number of seconds as a C double. -// Export for '_socket' shared extension. -PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); - // Convert timestamp to a number of milliseconds (10^-3 seconds). // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, +PyAPI_FUNC(PyTime_t) _PyTime_AsMilliseconds(PyTime_t t, _PyTime_round_t round); // Convert timestamp to a number of microseconds (10^-6 seconds). // Export for '_queue' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, +PyAPI_FUNC(PyTime_t) _PyTime_AsMicroseconds(PyTime_t t, _PyTime_round_t round); -// Convert timestamp to a number of nanoseconds (10^-9 seconds). -extern _PyTime_t _PyTime_AsNanoseconds(_PyTime_t t); - #ifdef MS_WINDOWS // Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). -extern _PyTime_t _PyTime_As100Nanoseconds(_PyTime_t t, +extern PyTime_t _PyTime_As100Nanoseconds(PyTime_t t, _PyTime_round_t round); #endif -// Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int -// object. +// Convert a timestamp (number of nanoseconds) as a Python int object. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(PyObject*) _PyTime_AsNanosecondsObject(_PyTime_t t); +PyAPI_FUNC(PyObject*) _PyTime_AsLong(PyTime_t t); #ifndef MS_WINDOWS // Create a timestamp from a timeval structure. // Raise an exception and return -1 on overflow, return 0 on success. -extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); +extern int _PyTime_FromTimeval(PyTime_t *tp, struct timeval *tv); #endif // Convert a timestamp to a timeval structure (microsecond resolution). @@ -208,14 +196,14 @@ extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); // Raise an exception and return -1 if the conversion overflowed, // return 0 on success. // Export for 'select' shared extension. -PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, +PyAPI_FUNC(int) _PyTime_AsTimeval(PyTime_t t, struct timeval *tv, _PyTime_round_t round); // Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. -// On overflow, clamp tv_sec to _PyTime_t min/max. +// On overflow, clamp tv_sec to PyTime_t min/max. // Export for 'select' shared extension. -PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, +PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(PyTime_t t, struct timeval *tv, _PyTime_round_t round); @@ -227,7 +215,7 @@ PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, // return 0 on success. // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( - _PyTime_t t, + PyTime_t t, time_t *secs, int *us, _PyTime_round_t round); @@ -235,23 +223,23 @@ PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) // Create a timestamp from a timespec structure. // Raise an exception and return -1 on overflow, return 0 on success. -extern int _PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts); +extern int _PyTime_FromTimespec(PyTime_t *tp, const struct timespec *ts); // Convert a timestamp to a timespec structure (nanosecond resolution). // tv_nsec is always positive. // Raise an exception and return -1 on error, return 0 on success. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); +PyAPI_FUNC(int) _PyTime_AsTimespec(PyTime_t t, struct timespec *ts); // Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. -// On overflow, clamp tv_sec to _PyTime_t min/max. +// On overflow, clamp tv_sec to PyTime_t min/max. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); +PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(PyTime_t t, struct timespec *ts); #endif -// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -extern _PyTime_t _PyTime_Add(_PyTime_t t1, _PyTime_t t2); +// Compute t1 + t2. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTime_Add(PyTime_t t1, PyTime_t t2); // Structure used by time.get_clock_info() typedef struct { @@ -261,36 +249,13 @@ typedef struct { double resolution; } _Py_clock_info_t; -// Get the current time from the system clock. -// -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. -// -// Use _PyTime_GetSystemClockWithInfo() to check for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); - // Get the current time from the system clock. // On success, set *t and *info (if not NULL), and return 0. // On error, raise an exception and return -1. -extern int _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, +extern int _PyTime_TimeWithInfo( + PyTime_t *t, _Py_clock_info_t *info); -// Get the time of a monotonic clock, i.e. a clock that cannot go backwards. -// The clock is not affected by system clock updates. The reference point of -// the returned value is undefined, so that only the difference between the -// results of consecutive calls is valid. -// -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. -// -// Use _PyTime_GetMonotonicClockWithInfo() to check for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); - // Get the time of a monotonic clock, i.e. a clock that cannot go backwards. // The clock is not affected by system clock updates. The reference point of // the returned value is undefined, so that only the difference between the @@ -300,8 +265,8 @@ PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); // // Return 0 on success, raise an exception and return -1 on error. // Export for '_testsinglephase' shared extension. -PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( - _PyTime_t *t, +PyAPI_FUNC(int) _PyTime_MonotonicWithInfo( + PyTime_t *t, _Py_clock_info_t *info); @@ -315,16 +280,6 @@ PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); -// Get the performance counter: clock with the highest available resolution to -// measure a short duration. -// -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. -// -// Use _PyTime_GetPerfCounterWithInfo() to check for failure. -// Export for '_lsprof' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); // Get the performance counter: clock with the highest available resolution to // measure a short duration. @@ -332,27 +287,29 @@ PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); // Fill info (if set) with information of the function used to get the time. // // Return 0 on success, raise an exception and return -1 on error. -extern int _PyTime_GetPerfCounterWithInfo( - _PyTime_t *t, +extern int _PyTime_PerfCounterWithInfo( + PyTime_t *t, _Py_clock_info_t *info); +// --- _PyDeadline ----------------------------------------------------------- + // Create a deadline. -// Pseudo code: _PyTime_GetMonotonicClock() + timeout. +// Pseudo code: return PyTime_MonotonicRaw() + timeout // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); +PyAPI_FUNC(PyTime_t) _PyDeadline_Init(PyTime_t timeout); // Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_GetMonotonicClock(). +// Pseudo code: return deadline - PyTime_MonotonicRaw() // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); +PyAPI_FUNC(PyTime_t) _PyDeadline_Get(PyTime_t deadline); // --- _PyTimeFraction ------------------------------------------------------- typedef struct { - _PyTime_t numer; - _PyTime_t denom; + PyTime_t numer; + PyTime_t denom; } _PyTimeFraction; // Set a fraction. @@ -360,13 +317,13 @@ typedef struct { // Return -1 if the fraction is invalid. extern int _PyTimeFraction_Set( _PyTimeFraction *frac, - _PyTime_t numer, - _PyTime_t denom); + PyTime_t numer, + PyTime_t denom); // Compute ticks * frac.numer / frac.denom. -// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -extern _PyTime_t _PyTimeFraction_Mul( - _PyTime_t ticks, +// Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTimeFraction_Mul( + PyTime_t ticks, const _PyTimeFraction *frac); // Compute a clock resolution: frac.numer / frac.denom / 1e9. diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 77a1dc59163d21..e0e7d5ebf0912c 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -8,9 +8,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // struct _Py_freelist_state -#include "pycore_mimalloc.h" // struct _mimalloc_thread_state -#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_mimalloc.h" // struct _mimalloc_thread_state +#include "pycore_qsbr.h" // struct qsbr // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The @@ -20,10 +21,33 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + PyObject *asyncio_running_loop; // Strong reference + + struct _qsbr_thread_state *qsbr; // only used by free-threaded build + struct llist_node mem_free_queue; // delayed free queue + #ifdef Py_GIL_DISABLED + struct _gc_thread_state gc; struct _mimalloc_thread_state mimalloc; - struct _Py_freelist_state freelist_state; + struct _Py_freelists freelists; struct _brc_thread_state brc; + struct { + // The per-thread refcounts + Py_ssize_t *values; + + // Size of the refcounts array. + Py_ssize_t size; + + // If set, don't use per-thread refcounts + int is_finalized; + } refcounts; + + // When >1, code objects do not immortalize their non-string constants. + int suppress_co_const_immortalization; +#endif + +#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) + Py_ssize_t reftotal; // this thread's total refcount operations #endif } _PyThreadStateImpl; diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 4605f355ccbc38..dfbbd6fd0c7de5 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -21,7 +21,8 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); -extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); +PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefSteal(const union _PyStackRef *, Py_ssize_t); +PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); typedef struct { PyObject_HEAD diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index c03c3d766bef61..118bc98b35d5e3 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_lock.h" // PyMutex /* state */ @@ -16,11 +17,25 @@ extern "C" { #define _Py_TYPE_BASE_VERSION_TAG (2<<16) #define _Py_MAX_GLOBAL_TYPE_VERSION_TAG (_Py_TYPE_BASE_VERSION_TAG - 1) +/* For now we hard-code this to a value for which we are confident + all the static builtin types will fit (for all builds). */ +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 +#define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 +#define _Py_MAX_MANAGED_STATIC_TYPES \ + (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) + struct _types_runtime_state { /* Used to set PyTypeObject.tp_version_tag for core static types. */ // bpo-42745: next_version_tag remains shared by all interpreters // because of static types. unsigned int next_version_tag; + + struct { + struct { + PyTypeObject *type; + int64_t interp_count; + } types[_Py_MAX_MANAGED_STATIC_TYPES]; + } managed_static; }; @@ -28,6 +43,9 @@ struct _types_runtime_state { // see _PyType_Lookup(). struct type_cache_entry { unsigned int version; // initialized from type->tp_version_tag +#ifdef Py_GIL_DISABLED + _PySeqLock sequence; +#endif PyObject *name; // reference to exactly a str or None PyObject *value; // borrowed reference or NULL }; @@ -38,12 +56,9 @@ struct type_cache { struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; }; -/* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_STATIC_BUILTIN_TYPES 200 - typedef struct { PyTypeObject *type; + int isbuiltin; int readying; int ready; // XXX tp_dict can probably be statically allocated, @@ -55,7 +70,9 @@ typedef struct { are also some diagnostic uses for the list of weakrefs, so we still keep it. */ PyObject *tp_weaklist; -} static_builtin_state; +} managed_static_type_state; + +#define TYPE_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ struct types_state { /* Used to set PyTypeObject.tp_version_tag. @@ -64,8 +81,60 @@ struct types_state { unsigned int next_version_tag; struct type_cache type_cache; - size_t num_builtins_initialized; - static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES]; + + /* Every static builtin type is initialized for each interpreter + during its own initialization, including for the main interpreter + during global runtime initialization. This is done by calling + _PyStaticType_InitBuiltin(). + + The first time a static builtin type is initialized, all the + normal PyType_Ready() stuff happens. The only difference from + normal is that there are three PyTypeObject fields holding + objects which are stored here (on PyInterpreterState) rather + than in the corresponding PyTypeObject fields. Those are: + tp_dict (cls.__dict__), tp_subclasses (cls.__subclasses__), + and tp_weaklist. + + When a subinterpreter is initialized, each static builtin type + is still initialized, but only the interpreter-specific portion, + namely those three objects. + + Those objects are stored in the PyInterpreterState.types.builtins + array, at the index corresponding to each specific static builtin + type. That index (a size_t value) is stored in the tp_subclasses + field. For static builtin types, we re-purposed the now-unused + tp_subclasses to avoid adding another field to PyTypeObject. + In all other cases tp_subclasses holds a dict like before. + (The field was previously defined as PyObject*, but is now void* + to reflect its dual use.) + + The index for each static builtin type isn't statically assigned. + Instead it is calculated the first time a type is initialized + (by the main interpreter). The index matches the order in which + the type was initialized relative to the others. The actual + value comes from the current value of num_builtins_initialized, + as each type is initialized for the main interpreter. + + num_builtins_initialized is incremented once for each static + builtin type. Once initialization is over for a subinterpreter, + the value will be the same as for all other interpreters. */ + struct { + size_t num_initialized; + managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES]; + } builtins; + /* We apply a similar strategy for managed extension modules. */ + struct { + size_t num_initialized; + size_t next_index; + managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_EXT_TYPES]; + } for_extensions; + PyMutex mutex; + + // Borrowed references to type objects whose + // tp_version_tag % TYPE_VERSION_CACHE_SIZE + // once was equal to the index in the table. + // They are cleared when the type object is deallocated. + PyTypeObject *type_version_cache[TYPE_VERSION_CACHE_SIZE]; }; @@ -73,8 +142,9 @@ struct types_state { extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); extern void _PyTypes_FiniTypes(PyInterpreterState *); +extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); - +extern void _PyTypes_AfterFork(void); /* other API */ @@ -88,12 +158,34 @@ typedef struct wrapperbase pytype_slotdef; static inline PyObject ** -_PyStaticType_GET_WEAKREFS_LISTPTR(static_builtin_state *state) +_PyStaticType_GET_WEAKREFS_LISTPTR(managed_static_type_state *state) { assert(state != NULL); return &state->tp_weaklist; } +extern int _PyStaticType_InitBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); +extern void _PyStaticType_FiniBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); +extern void _PyStaticType_ClearWeakRefs( + PyInterpreterState *interp, + PyTypeObject *type); +extern managed_static_type_state * _PyStaticType_GetState( + PyInterpreterState *interp, + PyTypeObject *type); + +// Export for '_datetime' shared extension. +PyAPI_FUNC(int) _PyStaticType_InitForExtension( + PyInterpreterState *interp, + PyTypeObject *self); + +// Export for _testinternalcapi extension. +PyAPI_FUNC(PyObject *) _PyStaticType_GetBuiltins(void); + + /* Like PyType_GetModuleState, but skips verification * that type is a heap type with an associated module */ static inline void * @@ -109,11 +201,6 @@ _PyType_GetModuleState(PyTypeObject *type) } -extern int _PyStaticType_InitBuiltin(PyInterpreterState *, PyTypeObject *type); -extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTypeObject *); -extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); -extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); - // Export for 'math' shared extension, used via _PyType_IsReady() static inline // function PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); @@ -123,6 +210,9 @@ extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); +// Export for _testinternalcapi extension. +PyAPI_FUNC(PyObject *) _PyType_GetSlotWrapperNames(void); + // PyType_Ready() must be called if _PyType_IsReady() is false. // See also the Py_TPFLAGS_READY flag. static inline int @@ -142,13 +232,26 @@ extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); extern PyTypeObject _PyBufferWrapper_Type; -extern PyObject* _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, +PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); +extern PyObject* _PyType_GetFullyQualifiedName(PyTypeObject *type, char sep); + +// Perform the following operation, in a thread-safe way when required by the +// build mode. +// +// self->tp_flags = (self->tp_flags & ~mask) | flags; +extern void _PyType_SetFlags(PyTypeObject *self, unsigned long mask, + unsigned long flags); -// This is exported for the _testinternalcapi module. -PyAPI_FUNC(PyObject *) _PyType_GetModuleName(PyTypeObject *); +// Like _PyType_SetFlags(), but apply the operation to self and any of its +// subclasses without Py_TPFLAGS_IMMUTABLETYPE set. +extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, + unsigned long flags); +extern unsigned int _PyType_GetVersionForCurrentState(PyTypeObject *tp); +PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version); +PyTypeObject *_PyType_LookupByVersion(unsigned int version); #ifdef __cplusplus } diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index c9fa97d6820757..4d7556e68cdaee 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -13,10 +13,14 @@ extern PyObject *_Py_make_paramspec(PyThreadState *, PyObject *); extern PyObject *_Py_make_typevartuple(PyThreadState *, PyObject *); extern PyObject *_Py_make_typealias(PyThreadState *, PyObject *); extern PyObject *_Py_subscript_generic(PyThreadState *, PyObject *); +extern PyObject *_Py_set_typeparam_default(PyThreadState *, PyObject *, PyObject *); extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); +extern int _Py_typing_type_repr(PyUnicodeWriter *, PyObject *); extern PyTypeObject _PyTypeAlias_Type; +extern PyTypeObject _PyNoDefault_Type; +extern PyObject _Py_NoDefaultStruct; #ifdef __cplusplus } diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 7ee540154b23d8..a60372f58295a9 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -10,8 +10,8 @@ extern "C" { #include "pycore_lock.h" // PyMutex #include "pycore_fileutils.h" // _Py_error_handler -#include "pycore_identifier.h" // _Py_Identifier #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +#include "pycore_global_objects.h" // _Py_SINGLETON /* --- Characters Type APIs ----------------------------------------------- */ @@ -31,8 +31,9 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); -extern void _PyUnicode_ExactDealloc(PyObject *op); +PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op); extern Py_ssize_t _PyUnicode_InternedSize(void); +extern Py_ssize_t _PyUnicode_InternedSize_Immortal(void); // Get a copy of a Unicode string. // Export for '_datetime' shared extension. @@ -189,7 +190,7 @@ extern PyObject* _PyUnicode_EncodeCharmap( /* --- Decimal Encoder ---------------------------------------------------- */ -// Coverts a Unicode object holding a decimal value to an ASCII string +// Converts a Unicode object holding a decimal value to an ASCII string // for using in int, float and complex parsers. // Transforms code points that have decimal digit property to the // corresponding ASCII digit code points. Transforms spaces to ASCII. @@ -202,7 +203,7 @@ PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( /* --- Methods & Slots ---------------------------------------------------- */ -extern PyObject* _PyUnicode_JoinArray( +PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( PyObject *separator, PyObject *const *items, Py_ssize_t seqlen @@ -251,11 +252,7 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int); -/* Fast equality check when the inputs are known to be exact unicode types - and where the hash values are equal (i.e. a very probable match) */ -extern int _PyUnicode_EQ(PyObject *, PyObject *); - -// Equality check. +// Fast equality check when the inputs are known to be exact unicode types. // Export for '_pickle' shared extension. PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *); @@ -275,6 +272,18 @@ extern void _PyUnicode_FiniTypes(PyInterpreterState *); extern PyTypeObject _PyUnicodeASCIIIter_Type; +/* --- Interning ---------------------------------------------------------- */ + +// All these are "ref-neutral", like the public PyUnicode_InternInPlace. + +// Explicit interning routines: +PyAPI_FUNC(void) _PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **); +PyAPI_FUNC(void) _PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **); +// Left here to help backporting: +PyAPI_FUNC(void) _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p); +// Only for singletons in the _PyRuntime struct: +extern void _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **); + /* --- Other API ---------------------------------------------------------- */ struct _Py_unicode_runtime_ids { @@ -311,7 +320,6 @@ struct _Py_unicode_state { struct _Py_unicode_ids ids; }; -extern void _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p); extern void _PyUnicode_ClearInterned(PyInterpreterState *interp); // Like PyUnicode_AsUTF8(), but check for embedded null characters. diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index c8458b4e36ccc9..eb2eca06ec4d4f 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -13,2132 +13,2989 @@ static inline void _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { PyObject *string; string = &_Py_ID(CANCELLED); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(FINISHED); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(False); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(JSONDecodeError); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(PENDING); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(Py_Repr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(TextIOWrapper); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(True); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(WarningMessage); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_WindowsConsoleIO); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__IOBase_closed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__abc_tpflags__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__abs__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__abstractmethods__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__add__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__aenter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__aexit__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__aiter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__all__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__and__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__anext__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__annotate__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__annotations__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__args__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__asyncio_running_event_loop__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__await__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__bases__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__bool__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__buffer__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__build_class__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__builtins__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__bytes__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__call__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__cantrace__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__class__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__class_getitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__classcell__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__classdict__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__classdictcell__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__complex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__contains__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__copy__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ctypes_from_outparam__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__del__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__delattr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__delete__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__delitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__dict__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__dictoffset__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__dir__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__divmod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__doc__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__enter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__eq__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__exit__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__file__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__firstlineno__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__float__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__floordiv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__format__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__fspath__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ge__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__get__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getattr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getattribute__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getinitargs__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getnewargs__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getnewargs_ex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getstate__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__gt__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__hash__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__iadd__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__iand__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ifloordiv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ilshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__imatmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__imod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__import__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__imul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__index__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__init__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__init_subclass__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__instancecheck__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__int__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__invert__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ior__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ipow__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__irshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__isabstractmethod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__isub__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__iter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__itruediv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ixor__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__le__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__len__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__length_hint__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__lltrace__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__loader__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__lshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__lt__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__main__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__match_args__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__matmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__missing__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__mod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__module__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__mro_entries__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__mul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__name__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ne__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__neg__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__new__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__newobj__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__newobj_ex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__next__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__notes__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__or__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__orig_class__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__origin__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__package__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__parameters__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__path__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__pos__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__pow__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__prepare__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__qualname__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__radd__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rand__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rdivmod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__reduce__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__reduce_ex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__release_buffer__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__repr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__reversed__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rfloordiv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rlshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rmatmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rmod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ror__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__round__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rpow__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rrshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rsub__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rtruediv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rxor__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__set__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__set_name__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__setattr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__setitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__setstate__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__sizeof__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__slotnames__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__slots__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__spec__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__static_attributes__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__str__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__sub__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__subclasscheck__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__subclasshook__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__truediv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__trunc__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__type_params__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_is_unpacked_typevartuple__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_prepare_subst__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_subst__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_unpacked_tuple_args__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__warningregistry__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__weaklistoffset__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__weakref__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__xor__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_abc_impl); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_abstract_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_active); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_annotation); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_anonymous_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_argtypes_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_as_parameter_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_asyncio_future_blocking); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_blksize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_bootstrap); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_check_retval_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_dealloc_warn); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_feature_version); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_field_types); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fields_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_finalizing); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_find_and_load); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fix_up_module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_flags_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_get_sourcefile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_handle_fromlist); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_initializing); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_io); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_is_text_encoding); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_isatty_open_only); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_length_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_limbo); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_lock_unlock_module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_needs_com_addref_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_pack_); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_only_immortal); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_restype_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_showwarnmsg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_shutdown); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_slotnames); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_strptime_datetime); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_swappedbytes_); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime_datetime_date); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime_datetime_datetime); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime_datetime_time); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_type_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_uninitialized_submodules); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_warn_unawaited_coroutine); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_xoptions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(a); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(abs_tol); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(access); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(aclose); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(add); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(add_done_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(after_in_child); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(after_in_parent); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(aggregate_class); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(alias); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(align); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(allow_code); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(append); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(arg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(argdefs); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(args); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(arguments); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(argv); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(as_integer_ratio); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(asend); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ast); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(athrow); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(attribute); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(authorizer_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(autocommit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(b); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(backtick); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(base); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(before); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(big); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(binary_form); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(bit_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(block); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bound); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffer); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffer_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffer_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffering); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffers); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bufsize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(builtins); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(byteorder); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bytes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bytes_per_sep); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(c); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(c_call); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(c_exception); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(c_return); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cached_datetime_module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_statements); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cadata); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cafile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(call); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(call_exception_handler); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(call_soon); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cancel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(capath); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(category); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cb_type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(certfile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(check_same_thread); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(clear); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(close); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(closed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(closefd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(closure); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_argcount); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_cellvars); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_code); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_consts); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_exceptiontable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_filename); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_firstlineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_flags); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_freevars); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_kwonlyargcount); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_linetable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_names); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_nlocals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_posonlyargcount); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_qualname); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_stacksize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_varnames); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(code); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(col_offset); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(command); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(comment_factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(compile_mode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(consts); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(context); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(contravariant); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cookie); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(copy); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(copyreg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(coro); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(count); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(covariant); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cwd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(d); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(data); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(database); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(day); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(decode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(decoder); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(default); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(defaultaction); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(delete); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(depth); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(desired_access); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(detect_types); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(deterministic); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(device); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dict); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dictcomp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(difference_update); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(digest); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(digest_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(digestmod); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dir_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(discard); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dispatch_table); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(displayhook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dklen); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(doc); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(done); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dont_inherit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dst); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dst_dir_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(e); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(eager_start); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(effective_ids); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(element_factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(encode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(encoding); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(end); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(end_col_offset); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(end_lineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(end_offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(endpos); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(entrypoint); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(env); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(errors); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(event); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(eventmask); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(excepthook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exception); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(existing_file_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(extend); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(extra_tokens); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(f); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(facility); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(false); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(family); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fanout); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fd2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fdel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fget); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(file); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(file_actions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filename); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fileno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filepath); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fillvalue); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filter); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filters); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(final); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(find_class); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fix_imports); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(flags); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(flush); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fold); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(follow_symlinks); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(format); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(format_spec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(from_param); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fromlist); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fromtimestamp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fromutc); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(func); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(future); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(g); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(generation); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(genexpr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_debug); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_event_loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_source); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(getattr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(getstate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(gid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(globals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(groupindex); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(groups); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(h); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(handle); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(handle_seq); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(has_location); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hash_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(header); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(headers); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hi); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hour); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(id); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ident); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(identity_hint); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ignore); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(imag); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(importlib); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(in_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(incoming); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(index); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(indexgroup); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inf); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(infer_variance); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(inherit_handle); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inheritable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial_bytes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(initial_owner); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(initial_state); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial_value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initval); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inner_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(input); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(insert_comments); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(insert_pis); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(instructions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(intern); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(intersection); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(interval); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(is_running); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(is_struct); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isatty); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isinstance); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isoformat); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isolation_level); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(istext); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(item); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(items); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(iter); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(iterable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(iterations); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(join); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(jump); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(keepends); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(key); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(keyfile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(keys); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kind); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kw); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kw1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kw2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kwdefaults); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(label); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(lambda); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_exc); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_node); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_traceback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(latin1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(leaf_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(len); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(length); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(level); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(limit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(line); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(line_buffering); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(lineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(listcomp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(little); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(lo); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(locale); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(locals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(logoption); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(loop); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(manual_reset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mapping); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(match); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(max_length); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxdigits); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxevents); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxlen); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxmem); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxsplit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxvalue); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(memLevel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(memlimit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(message); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(metaclass); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(metadata); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(method); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(microsecond); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(milliseconds); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(minute); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mod); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(module_globals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(modules); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(month); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mro); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(msg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(mycmp); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(mutex); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(n); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(mycmp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_arg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_sequence_fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_unnamed_fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(name_from); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(namespace_separator); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(namespaces); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(narg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(nested); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(new_file_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(new_limit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(newline); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(newlines); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(next); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(nlocals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(node_depth); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(node_offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ns); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(nstype); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(nt); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(null); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(number); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(obj); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(object); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(offset_dst); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(offset_src); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(on_type_read); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(onceregistry); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(only_keys); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(oparg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(opcode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(open); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(opener); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(operation); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(optimize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(options); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(order); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(origin); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(out_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(outgoing); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(overlapped); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(owner); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(p); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pages); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(parent); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(password); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(path); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pattern); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(peek); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(persistent_id); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(persistent_load); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(person); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pi_factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(policy); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(posix); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(print_file_and_line); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(priority); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_handler); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_routine); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(proto); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(protocol); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ps1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ps2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(query); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(quotetabs); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(r); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(raw); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(read); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(read1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readall); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readinto); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readinto1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readline); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readonly); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(real); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reducer_override); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(registry); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(rel_tol); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(release); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reload); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(repl); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(replace); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reserved); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(resetids); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(return); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reverse); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reversed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(s); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(salt); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sched_priority); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(scheduler); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(second); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(security_attributes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(seek); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(seekable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(selectors); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(self); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(send); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sep); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sequence); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(server_hostname); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(server_side); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(session); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setcomp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setpgroup); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setsid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setsigdef); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setsigmask); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setstate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(shape); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(show_cmd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(signed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sizehint); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(skip_file_prefixes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sleep); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sock); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sort); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(source); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(source_traceback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(spam); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(src); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(src_dir_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stacklevel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(start); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(statement); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(status); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stderr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stdin); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stdout); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(step); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(steps); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(store_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strategy); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strftime); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strict); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strict_mode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(string); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sub_key); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(symmetric_difference_update); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tabsize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tag); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target_is_directory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(task); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_frame); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_lasti); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_lineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_next); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tell); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(template); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(term); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(text); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(threading); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(throw); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timeout); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(times); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timetuple); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(top); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(trace_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(traceback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(trailers); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(translate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(true); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(truncate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(twice); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(txt); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(type_params); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tz); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tzinfo); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tzname); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(uid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(unlink); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(unraisablehook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(uri); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(usedforsecurity); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(values); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(version); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(volume); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(wait_all); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(warn_on_full_buffer); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(warnings); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(warnoptions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(wbits); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(week); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(weekday); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(which); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(who); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(withdata); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(writable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(write); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(write_through); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(x); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(year); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(zdict); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(empty); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dbl_percent); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dot_locals); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(defaults); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(format); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(generic_base); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(kwdefaults); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(type_params); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(str_replace_inf); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_null); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_dictcomp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_genexpr); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_lambda); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_listcomp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_module); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_setcomp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_string); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_unknown); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(json_decoder); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(list_err); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(utf_8); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dbl_open_br); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dbl_close_br); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); } /* End auto-generated code */ #ifdef __cplusplus diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h index 87264635b6e1cf..6ece7134cdeca0 100644 --- a/Include/internal/pycore_unionobject.h +++ b/Include/internal/pycore_unionobject.h @@ -8,9 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyTypeObject _PyUnion_Type; +// For extensions created by test_peg_generator +PyAPI_DATA(PyTypeObject) _PyUnion_Type; +PyAPI_FUNC(PyObject *) _Py_union_type_or(PyObject *, PyObject *); + #define _PyUnion_Check(op) Py_IS_TYPE((op), &_PyUnion_Type) -extern PyObject *_Py_union_type_or(PyObject *, PyObject *); #define _PyGenericAlias_Check(op) PyObject_TypeCheck((op), &Py_GenericAliasType) extern PyObject *_Py_subs_parameters(PyObject *, PyObject *, PyObject *, PyObject *); diff --git a/Include/internal/pycore_uniqueid.h b/Include/internal/pycore_uniqueid.h new file mode 100644 index 00000000000000..d3db49ddb78103 --- /dev/null +++ b/Include/internal/pycore_uniqueid.h @@ -0,0 +1,75 @@ +#ifndef Py_INTERNAL_UNIQUEID_H +#define Py_INTERNAL_UNIQUEID_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// This contains code for allocating unique ids to objects for per-thread +// reference counting. +// +// Per-thread reference counting is used along with deferred reference +// counting to avoid scaling bottlenecks due to reference count contention. +// +// An id of -1 is used to indicate that an object doesn't use per-thread +// refcounting. This value is used when the object is finalized by the GC +// and during interpreter shutdown to allow the object to be +// deallocated promptly when the object's refcount reaches zero. +// +// Each entry implicitly represents a unique id based on its offset in the +// table. Non-allocated entries form a free-list via the 'next' pointer. +// Allocated entries store the corresponding PyObject. +typedef union _Py_unique_id_entry { + // Points to the next free type id, when part of the freelist + union _Py_unique_id_entry *next; + + // Stores the object when the id is assigned + PyObject *obj; +} _Py_unique_id_entry; + +struct _Py_unique_id_pool { + PyMutex mutex; + + // combined table of object with allocated unique ids and unallocated ids. + _Py_unique_id_entry *table; + + // Next entry to allocate inside 'table' or NULL + _Py_unique_id_entry *freelist; + + // size of 'table' + Py_ssize_t size; +}; + +// Assigns the next id from the pool of ids. +extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj); + +// Releases the allocated id back to the pool. +extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id); + +// Releases the allocated id back to the pool. +extern void _PyObject_DisablePerThreadRefcounting(PyObject *obj); + +// Merges the per-thread reference counts into the corresponding objects. +extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Like _PyObject_MergePerThreadRefcounts, but also frees the per-thread +// array of refcounts. +extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Frees the interpreter's pool of type ids. +extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp); + +// Increfs the object, resizing the thread-local refcount array if necessary. +PyAPI_FUNC(void) _PyObject_ThreadIncrefSlow(PyObject *obj, Py_ssize_t unique_id); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNIQUEID_H */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index b2476e1c6e5c4b..de628d240d1c07 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -11,233 +11,304 @@ extern "C" { #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _NOP NOP -#define _RESUME_CHECK RESUME_CHECK -#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME -#define _LOAD_FAST_CHECK LOAD_FAST_CHECK -#define _LOAD_FAST LOAD_FAST -#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST -#define _LOAD_CONST LOAD_CONST -#define _STORE_FAST STORE_FAST -#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST -#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST -#define _POP_TOP POP_TOP -#define _PUSH_NULL PUSH_NULL -#define _END_SEND END_SEND -#define _UNARY_NEGATIVE UNARY_NEGATIVE -#define _UNARY_NOT UNARY_NOT -#define _TO_BOOL 302 -#define _TO_BOOL_BOOL TO_BOOL_BOOL -#define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST TO_BOOL_LIST -#define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR TO_BOOL_STR -#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE -#define _UNARY_INVERT UNARY_INVERT -#define _GUARD_BOTH_INT 303 -#define _BINARY_OP_MULTIPLY_INT 304 -#define _BINARY_OP_ADD_INT 305 -#define _BINARY_OP_SUBTRACT_INT 306 -#define _GUARD_BOTH_FLOAT 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_ADD_FLOAT 309 -#define _BINARY_OP_SUBTRACT_FLOAT 310 -#define _GUARD_BOTH_UNICODE 311 -#define _BINARY_OP_ADD_UNICODE 312 -#define _BINARY_SUBSCR 313 -#define _BINARY_SLICE BINARY_SLICE -#define _STORE_SLICE STORE_SLICE +#define _BINARY_OP 302 +#define _BINARY_OP_ADD_FLOAT 303 +#define _BINARY_OP_ADD_INT 304 +#define _BINARY_OP_ADD_UNICODE 305 +#define _BINARY_OP_INPLACE_ADD_UNICODE 306 +#define _BINARY_OP_MULTIPLY_FLOAT 307 +#define _BINARY_OP_MULTIPLY_INT 308 +#define _BINARY_OP_SUBTRACT_FLOAT 309 +#define _BINARY_OP_SUBTRACT_INT 310 +#define _BINARY_SLICE 311 +#define _BINARY_SUBSCR 312 +#define _BINARY_SUBSCR_CHECK_FUNC 313 +#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT +#define _BINARY_SUBSCR_INIT_CALL 314 #define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT #define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT #define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT -#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT -#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM -#define _LIST_APPEND LIST_APPEND -#define _SET_ADD SET_ADD -#define _STORE_SUBSCR 314 -#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT -#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT -#define _DELETE_SUBSCR DELETE_SUBSCR +#define _BUILD_LIST BUILD_LIST +#define _BUILD_MAP BUILD_MAP +#define _BUILD_SET BUILD_SET +#define _BUILD_SLICE BUILD_SLICE +#define _BUILD_STRING BUILD_STRING +#define _BUILD_TUPLE BUILD_TUPLE +#define _CALL_BUILTIN_CLASS 315 +#define _CALL_BUILTIN_FAST 316 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 317 +#define _CALL_BUILTIN_O 318 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _POP_FRAME 315 -#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE -#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST +#define _CALL_ISINSTANCE CALL_ISINSTANCE +#define _CALL_KW_NON_PY 319 +#define _CALL_LEN CALL_LEN +#define _CALL_LIST_APPEND CALL_LIST_APPEND +#define _CALL_METHOD_DESCRIPTOR_FAST 320 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 321 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 322 +#define _CALL_METHOD_DESCRIPTOR_O 323 +#define _CALL_NON_PY_GENERAL 324 +#define _CALL_STR_1 325 +#define _CALL_TUPLE_1 326 +#define _CALL_TYPE_1 CALL_TYPE_1 +#define _CHECK_AND_ALLOCATE_OBJECT 327 +#define _CHECK_ATTR_CLASS 328 +#define _CHECK_ATTR_METHOD_LAZY_DICT 329 +#define _CHECK_ATTR_MODULE 330 +#define _CHECK_ATTR_WITH_HINT 331 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 332 +#define _CHECK_EG_MATCH CHECK_EG_MATCH +#define _CHECK_EXC_MATCH CHECK_EXC_MATCH +#define _CHECK_FUNCTION 333 +#define _CHECK_FUNCTION_EXACT_ARGS 334 +#define _CHECK_FUNCTION_VERSION 335 +#define _CHECK_FUNCTION_VERSION_KW 336 +#define _CHECK_IS_NOT_PY_CALLABLE 337 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 338 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 339 +#define _CHECK_METHOD_VERSION 340 +#define _CHECK_METHOD_VERSION_KW 341 +#define _CHECK_PEP_523 342 +#define _CHECK_PERIODIC 343 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 344 +#define _CHECK_STACK_SPACE 345 +#define _CHECK_STACK_SPACE_OPERAND 346 +#define _CHECK_VALIDITY 347 +#define _CHECK_VALIDITY_AND_SET_IP 348 +#define _COMPARE_OP 349 +#define _COMPARE_OP_FLOAT 350 +#define _COMPARE_OP_INT 351 +#define _COMPARE_OP_STR 352 +#define _CONTAINS_OP 353 +#define _CONTAINS_OP_DICT CONTAINS_OP_DICT +#define _CONTAINS_OP_SET CONTAINS_OP_SET +#define _CONVERT_VALUE CONVERT_VALUE +#define _COPY COPY +#define _COPY_FREE_VARS COPY_FREE_VARS +#define _CREATE_INIT_FRAME 354 +#define _DELETE_ATTR DELETE_ATTR +#define _DELETE_DEREF DELETE_DEREF +#define _DELETE_FAST DELETE_FAST +#define _DELETE_GLOBAL DELETE_GLOBAL +#define _DELETE_NAME DELETE_NAME +#define _DELETE_SUBSCR DELETE_SUBSCR +#define _DEOPT 355 +#define _DICT_MERGE DICT_MERGE +#define _DICT_UPDATE DICT_UPDATE +#define _DO_CALL 356 +#define _DO_CALL_FUNCTION_EX 357 +#define _DO_CALL_KW 358 +#define _DYNAMIC_EXIT 359 +#define _END_SEND END_SEND +#define _ERROR_POP_N 360 +#define _EXIT_INIT_CHECK EXIT_INIT_CHECK +#define _EXPAND_METHOD 361 +#define _EXPAND_METHOD_KW 362 +#define _FATAL_ERROR 363 +#define _FORMAT_SIMPLE FORMAT_SIMPLE +#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC +#define _FOR_ITER 364 +#define _FOR_ITER_GEN_FRAME 365 +#define _FOR_ITER_TIER_TWO 366 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE -#define _SEND 316 -#define _SEND_GEN SEND_GEN -#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _POP_EXCEPT POP_EXCEPT -#define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR +#define _GET_ITER GET_ITER +#define _GET_LEN GET_LEN +#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER +#define _GUARD_BOTH_FLOAT 367 +#define _GUARD_BOTH_INT 368 +#define _GUARD_BOTH_UNICODE 369 +#define _GUARD_BUILTINS_VERSION_PUSH_KEYS 370 +#define _GUARD_DORV_NO_DICT 371 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 372 +#define _GUARD_GLOBALS_VERSION 373 +#define _GUARD_GLOBALS_VERSION_PUSH_KEYS 374 +#define _GUARD_IS_FALSE_POP 375 +#define _GUARD_IS_NONE_POP 376 +#define _GUARD_IS_NOT_NONE_POP 377 +#define _GUARD_IS_TRUE_POP 378 +#define _GUARD_KEYS_VERSION 379 +#define _GUARD_NOS_FLOAT 380 +#define _GUARD_NOS_INT 381 +#define _GUARD_NOT_EXHAUSTED_LIST 382 +#define _GUARD_NOT_EXHAUSTED_RANGE 383 +#define _GUARD_NOT_EXHAUSTED_TUPLE 384 +#define _GUARD_TOS_FLOAT 385 +#define _GUARD_TOS_INT 386 +#define _GUARD_TYPE_VERSION 387 +#define _IMPORT_FROM IMPORT_FROM +#define _IMPORT_NAME IMPORT_NAME +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 388 +#define _INIT_CALL_PY_EXACT_ARGS 389 +#define _INIT_CALL_PY_EXACT_ARGS_0 390 +#define _INIT_CALL_PY_EXACT_ARGS_1 391 +#define _INIT_CALL_PY_EXACT_ARGS_2 392 +#define _INIT_CALL_PY_EXACT_ARGS_3 393 +#define _INIT_CALL_PY_EXACT_ARGS_4 394 +#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX +#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW +#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER +#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION +#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD +#define _INSTRUMENTED_LINE INSTRUMENTED_LINE +#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE +#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE +#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE +#define _INTERNAL_INCREMENT_OPT_COUNTER 395 +#define _IS_NONE 396 +#define _IS_OP IS_OP +#define _ITER_CHECK_LIST 397 +#define _ITER_CHECK_RANGE 398 +#define _ITER_CHECK_TUPLE 399 +#define _ITER_JUMP_LIST 400 +#define _ITER_JUMP_RANGE 401 +#define _ITER_JUMP_TUPLE 402 +#define _ITER_NEXT_LIST 403 +#define _ITER_NEXT_RANGE 404 +#define _ITER_NEXT_TUPLE 405 +#define _JUMP_TO_TOP 406 +#define _LIST_APPEND LIST_APPEND +#define _LIST_EXTEND LIST_EXTEND +#define _LOAD_ATTR 407 +#define _LOAD_ATTR_CLASS 408 +#define _LOAD_ATTR_CLASS_0 409 +#define _LOAD_ATTR_CLASS_1 410 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN +#define _LOAD_ATTR_INSTANCE_VALUE 411 +#define _LOAD_ATTR_INSTANCE_VALUE_0 412 +#define _LOAD_ATTR_INSTANCE_VALUE_1 413 +#define _LOAD_ATTR_METHOD_LAZY_DICT 414 +#define _LOAD_ATTR_METHOD_NO_DICT 415 +#define _LOAD_ATTR_METHOD_WITH_VALUES 416 +#define _LOAD_ATTR_MODULE 417 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 418 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 419 +#define _LOAD_ATTR_PROPERTY_FRAME 420 +#define _LOAD_ATTR_SLOT 421 +#define _LOAD_ATTR_SLOT_0 422 +#define _LOAD_ATTR_SLOT_1 423 +#define _LOAD_ATTR_WITH_HINT 424 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _STORE_NAME STORE_NAME -#define _DELETE_NAME DELETE_NAME -#define _UNPACK_SEQUENCE 317 -#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE -#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE -#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST -#define _UNPACK_EX UNPACK_EX -#define _STORE_ATTR 318 -#define _DELETE_ATTR DELETE_ATTR -#define _STORE_GLOBAL STORE_GLOBAL -#define _DELETE_GLOBAL DELETE_GLOBAL -#define _LOAD_LOCALS LOAD_LOCALS +#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT +#define _LOAD_CONST LOAD_CONST +#define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL +#define _LOAD_CONST_INLINE 425 +#define _LOAD_CONST_INLINE_BORROW 426 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 427 +#define _LOAD_CONST_INLINE_WITH_NULL 428 +#define _LOAD_DEREF LOAD_DEREF +#define _LOAD_FAST 429 +#define _LOAD_FAST_0 430 +#define _LOAD_FAST_1 431 +#define _LOAD_FAST_2 432 +#define _LOAD_FAST_3 433 +#define _LOAD_FAST_4 434 +#define _LOAD_FAST_5 435 +#define _LOAD_FAST_6 436 +#define _LOAD_FAST_7 437 +#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR +#define _LOAD_FAST_CHECK LOAD_FAST_CHECK +#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST +#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS +#define _LOAD_GLOBAL 438 +#define _LOAD_GLOBAL_BUILTINS 439 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 440 +#define _LOAD_GLOBAL_MODULE 441 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 442 +#define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_GLOBAL 319 -#define _GUARD_GLOBALS_VERSION 320 -#define _GUARD_BUILTINS_VERSION 321 -#define _LOAD_GLOBAL_MODULE 322 -#define _LOAD_GLOBAL_BUILTINS 323 -#define _DELETE_FAST DELETE_FAST -#define _MAKE_CELL MAKE_CELL -#define _DELETE_DEREF DELETE_DEREF -#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF -#define _LOAD_DEREF LOAD_DEREF -#define _STORE_DEREF STORE_DEREF -#define _COPY_FREE_VARS COPY_FREE_VARS -#define _BUILD_STRING BUILD_STRING -#define _BUILD_TUPLE BUILD_TUPLE -#define _BUILD_LIST BUILD_LIST -#define _LIST_EXTEND LIST_EXTEND -#define _SET_UPDATE SET_UPDATE -#define _BUILD_SET BUILD_SET -#define _BUILD_MAP BUILD_MAP -#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS -#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP -#define _DICT_UPDATE DICT_UPDATE -#define _DICT_MERGE DICT_MERGE -#define _MAP_ADD MAP_ADD -#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _LOAD_SMALL_INT 443 +#define _LOAD_SMALL_INT_0 444 +#define _LOAD_SMALL_INT_1 445 +#define _LOAD_SMALL_INT_2 446 +#define _LOAD_SMALL_INT_3 447 +#define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _LOAD_ATTR 324 -#define _GUARD_TYPE_VERSION 325 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 326 -#define _LOAD_ATTR_INSTANCE_VALUE 327 -#define _CHECK_ATTR_MODULE 328 -#define _LOAD_ATTR_MODULE 329 -#define _CHECK_ATTR_WITH_HINT 330 -#define _LOAD_ATTR_WITH_HINT 331 -#define _LOAD_ATTR_SLOT 332 -#define _CHECK_ATTR_CLASS 333 -#define _LOAD_ATTR_CLASS 334 -#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _GUARD_DORV_VALUES 335 -#define _STORE_ATTR_INSTANCE_VALUE 336 -#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT -#define _STORE_ATTR_SLOT 337 -#define _COMPARE_OP 338 -#define _COMPARE_OP_FLOAT COMPARE_OP_FLOAT -#define _COMPARE_OP_INT COMPARE_OP_INT -#define _COMPARE_OP_STR COMPARE_OP_STR -#define _IS_OP IS_OP -#define _CONTAINS_OP CONTAINS_OP -#define _CHECK_EG_MATCH CHECK_EG_MATCH -#define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _JUMP_BACKWARD JUMP_BACKWARD -#define _POP_JUMP_IF_FALSE 339 -#define _POP_JUMP_IF_TRUE 340 -#define _IS_NONE 341 -#define _GET_LEN GET_LEN +#define _MAKE_CALLARGS_A_TUPLE 448 +#define _MAKE_CELL MAKE_CELL +#define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAKE_WARM 449 +#define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS +#define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MATCH_KEYS MATCH_KEYS -#define _GET_ITER GET_ITER -#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _FOR_ITER 342 -#define _FOR_ITER_TIER_TWO 343 -#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER -#define _ITER_CHECK_LIST 344 -#define _ITER_JUMP_LIST 345 -#define _GUARD_NOT_EXHAUSTED_LIST 346 -#define _ITER_NEXT_LIST 347 -#define _ITER_CHECK_TUPLE 348 -#define _ITER_JUMP_TUPLE 349 -#define _GUARD_NOT_EXHAUSTED_TUPLE 350 -#define _ITER_NEXT_TUPLE 351 -#define _ITER_CHECK_RANGE 352 -#define _ITER_JUMP_RANGE 353 -#define _GUARD_NOT_EXHAUSTED_RANGE 354 -#define _ITER_NEXT_RANGE 355 -#define _FOR_ITER_GEN FOR_ITER_GEN -#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH -#define _BEFORE_WITH BEFORE_WITH -#define _WITH_EXCEPT_START WITH_EXCEPT_START +#define _MAYBE_EXPAND_METHOD 450 +#define _MAYBE_EXPAND_METHOD_KW 451 +#define _MONITOR_CALL 452 +#define _MONITOR_JUMP_BACKWARD 453 +#define _MONITOR_RESUME 454 +#define _NOP NOP +#define _POP_EXCEPT POP_EXCEPT +#define _POP_JUMP_IF_FALSE 455 +#define _POP_JUMP_IF_TRUE 456 +#define _POP_TOP POP_TOP +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 457 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 356 -#define _GUARD_KEYS_VERSION 357 -#define _LOAD_ATTR_METHOD_WITH_VALUES 358 -#define _LOAD_ATTR_METHOD_NO_DICT 359 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 360 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 361 -#define _CHECK_ATTR_METHOD_LAZY_DICT 362 -#define _LOAD_ATTR_METHOD_LAZY_DICT 363 -#define _INSTRUMENTED_CALL INSTRUMENTED_CALL -#define _CALL 364 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 365 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 366 -#define _CHECK_PEP_523 367 -#define _CHECK_FUNCTION_EXACT_ARGS 368 -#define _CHECK_STACK_SPACE 369 -#define _INIT_CALL_PY_EXACT_ARGS 370 -#define _PUSH_FRAME 371 -#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS -#define _CALL_TYPE_1 CALL_TYPE_1 -#define _CALL_STR_1 CALL_STR_1 -#define _CALL_TUPLE_1 CALL_TUPLE_1 -#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT -#define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS -#define _CALL_BUILTIN_O CALL_BUILTIN_O -#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS -#define _CALL_LEN CALL_LEN -#define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS -#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS -#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST -#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW -#define _CALL_KW CALL_KW -#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX -#define _CALL_FUNCTION_EX CALL_FUNCTION_EX -#define _MAKE_FUNCTION MAKE_FUNCTION +#define _PUSH_FRAME 458 +#define _PUSH_NULL PUSH_NULL +#define _PY_FRAME_GENERAL 459 +#define _PY_FRAME_KW 460 +#define _QUICKEN_RESUME 461 +#define _REPLACE_WITH_TRUE 462 +#define _RESUME_CHECK RESUME_CHECK +#define _RETURN_GENERATOR RETURN_GENERATOR +#define _RETURN_VALUE RETURN_VALUE +#define _SAVE_RETURN_OFFSET 463 +#define _SEND 464 +#define _SEND_GEN_FRAME 465 +#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS +#define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _BUILD_SLICE BUILD_SLICE -#define _CONVERT_VALUE CONVERT_VALUE -#define _FORMAT_SIMPLE FORMAT_SIMPLE -#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _COPY COPY -#define _BINARY_OP 372 +#define _SET_UPDATE SET_UPDATE +#define _START_EXECUTOR 466 +#define _STORE_ATTR 467 +#define _STORE_ATTR_INSTANCE_VALUE 468 +#define _STORE_ATTR_SLOT 469 +#define _STORE_ATTR_WITH_HINT 470 +#define _STORE_DEREF STORE_DEREF +#define _STORE_FAST 471 +#define _STORE_FAST_0 472 +#define _STORE_FAST_1 473 +#define _STORE_FAST_2 474 +#define _STORE_FAST_3 475 +#define _STORE_FAST_4 476 +#define _STORE_FAST_5 477 +#define _STORE_FAST_6 478 +#define _STORE_FAST_7 479 +#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST +#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST +#define _STORE_GLOBAL STORE_GLOBAL +#define _STORE_NAME STORE_NAME +#define _STORE_SLICE 480 +#define _STORE_SUBSCR 481 +#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT +#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION -#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD -#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD -#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE -#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE -#define _GUARD_IS_TRUE_POP 373 -#define _GUARD_IS_FALSE_POP 374 -#define _GUARD_IS_NONE_POP 375 -#define _GUARD_IS_NOT_NONE_POP 376 -#define _JUMP_TO_TOP 377 -#define _SAVE_RETURN_OFFSET 378 -#define _CHECK_VALIDITY 379 -#define _LOAD_CONST_INLINE 380 -#define _LOAD_CONST_INLINE_BORROW 381 -#define _LOAD_CONST_INLINE_WITH_NULL 382 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 383 -#define _CHECK_GLOBALS 384 -#define _CHECK_BUILTINS 385 -#define _INTERNAL_INCREMENT_OPT_COUNTER 386 -#define MAX_UOP_ID 386 +#define _TIER2_RESUME_CHECK 482 +#define _TO_BOOL 483 +#define _TO_BOOL_BOOL TO_BOOL_BOOL +#define _TO_BOOL_INT TO_BOOL_INT +#define _TO_BOOL_LIST TO_BOOL_LIST +#define _TO_BOOL_NONE TO_BOOL_NONE +#define _TO_BOOL_STR TO_BOOL_STR +#define _UNARY_INVERT UNARY_INVERT +#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _UNARY_NOT UNARY_NOT +#define _UNPACK_EX UNPACK_EX +#define _UNPACK_SEQUENCE 484 +#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST +#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE +#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE +#define _WITH_EXCEPT_START WITH_EXCEPT_START +#define _YIELD_VALUE YIELD_VALUE +#define MAX_UOP_ID 484 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 2b5b37e6b8d6a4..4cfdecec78b0db 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -12,17 +12,44 @@ extern "C" { #include #include "pycore_uop_ids.h" extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; +extern int _PyUop_num_popped(int opcode, int oparg); + #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { - [_NOP] = 0, + [_NOP] = HAS_PURE_FLAG, + [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, - [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, + [_LOAD_CONST_IMMORTAL] = HAS_ARG_FLAG | HAS_CONST_FLAG, + [_LOAD_SMALL_INT_0] = 0, + [_LOAD_SMALL_INT_1] = 0, + [_LOAD_SMALL_INT_2] = 0, + [_LOAD_SMALL_INT_3] = 0, + [_LOAD_SMALL_INT] = HAS_ARG_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG, [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, @@ -32,23 +59,28 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_TO_BOOL_INT] = HAS_DEOPT_FLAG, - [_TO_BOOL_LIST] = HAS_DEOPT_FLAG, - [_TO_BOOL_NONE] = HAS_DEOPT_FLAG, - [_TO_BOOL_STR] = HAS_DEOPT_FLAG, - [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG, + [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, + [_TO_BOOL_INT] = HAS_EXIT_FLAG, + [_TO_BOOL_LIST] = HAS_EXIT_FLAG, + [_TO_BOOL_NONE] = HAS_EXIT_FLAG, + [_TO_BOOL_STR] = HAS_EXIT_FLAG, + [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_INT] = HAS_EXIT_FLAG, + [_GUARD_NOS_INT] = HAS_EXIT_FLAG, + [_GUARD_TOS_INT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, - [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -56,23 +88,27 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_INIT_CALL] = 0, [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG, [_STORE_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_POP_FRAME] = HAS_ESCAPES_FLAG, + [_RETURN_VALUE] = 0, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, - [_LOAD_ASSERTION_ERROR] = 0, + [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -81,58 +117,69 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, - [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, - [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, - [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_GLOBALS_VERSION_PUSH_KEYS] = HAS_DEOPT_FLAG, + [_GUARD_BUILTINS_VERSION_PUSH_KEYS] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_MODULE_FROM_KEYS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_BUILTINS_FROM_KEYS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, - [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, + [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG, [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG, - [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, - [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, + [_CHECK_ATTR_WITH_HINT] = HAS_EXIT_FLAG, + [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, + [_CHECK_ATTR_CLASS] = HAS_EXIT_FLAG, + [_LOAD_ATTR_CLASS_0] = 0, + [_LOAD_ATTR_CLASS_1] = 0, + [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, + [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, + [_STORE_ATTR_INSTANCE_VALUE] = 0, + [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_ATTR_SLOT] = 0, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_STR] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, + [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_COMPARE_OP_STR] = HAS_ARG_FLAG, [_IS_OP] = HAS_ARG_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IS_NONE] = 0, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -140,95 +187,137 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, + [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST] = 0, - [_ITER_CHECK_TUPLE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, + [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, - [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_ITER_CHECK_RANGE] = HAS_EXIT_FLAG, + [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, + [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, + [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, + [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, + [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, - [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, + [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD] = HAS_ARG_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_NON_PY_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, - [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, - [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD_KW] = HAS_ARG_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CALLARGS_A_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG, - [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, - [_SET_IP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG, + [_JUMP_TO_TOP] = 0, + [_SET_IP] = 0, + [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_DEOPT_FLAG, + [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, - [_LOAD_CONST_INLINE] = 0, - [_LOAD_CONST_INLINE_BORROW] = 0, - [_LOAD_CONST_INLINE_WITH_NULL] = 0, - [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = 0, - [_CHECK_GLOBALS] = HAS_DEOPT_FLAG, - [_CHECK_BUILTINS] = HAS_DEOPT_FLAG, + [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, + [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, + [_START_EXECUTOR] = 0, + [_MAKE_WARM] = 0, + [_FATAL_ERROR] = 0, + [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, + [_DEOPT] = 0, + [_ERROR_POP_N] = HAS_ARG_FLAG, + [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, +}; + +const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { + [_LOAD_FAST] = 8, + [_LOAD_SMALL_INT] = 4, + [_STORE_FAST] = 8, + [_INIT_CALL_PY_EXACT_ARGS] = 5, }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { - [_BEFORE_ASYNC_WITH] = "_BEFORE_ASYNC_WITH", - [_BEFORE_WITH] = "_BEFORE_WITH", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_SLICE] = "_BINARY_SLICE", [_BINARY_SUBSCR] = "_BINARY_SUBSCR", + [_BINARY_SUBSCR_CHECK_FUNC] = "_BINARY_SUBSCR_CHECK_FUNC", [_BINARY_SUBSCR_DICT] = "_BINARY_SUBSCR_DICT", + [_BINARY_SUBSCR_INIT_CALL] = "_BINARY_SUBSCR_INIT_CALL", [_BINARY_SUBSCR_LIST_INT] = "_BINARY_SUBSCR_LIST_INT", [_BINARY_SUBSCR_STR_INT] = "_BINARY_SUBSCR_STR_INT", [_BINARY_SUBSCR_TUPLE_INT] = "_BINARY_SUBSCR_TUPLE_INT", - [_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP", [_BUILD_LIST] = "_BUILD_LIST", [_BUILD_MAP] = "_BUILD_MAP", [_BUILD_SET] = "_BUILD_SET", @@ -242,49 +331,72 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", [_CALL_LEN] = "_CALL_LEN", + [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", [_CALL_STR_1] = "_CALL_STR_1", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", [_CALL_TYPE_1] = "_CALL_TYPE_1", + [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", - [_CHECK_BUILTINS] = "_CHECK_BUILTINS", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", + [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", - [_CHECK_GLOBALS] = "_CHECK_GLOBALS", + [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", + [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PERIODIC] = "_CHECK_PERIODIC", + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", [_COMPARE_OP_STR] = "_COMPARE_OP_STR", [_CONTAINS_OP] = "_CONTAINS_OP", + [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", + [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", [_CONVERT_VALUE] = "_CONVERT_VALUE", [_COPY] = "_COPY", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", [_DELETE_ATTR] = "_DELETE_ATTR", [_DELETE_DEREF] = "_DELETE_DEREF", [_DELETE_FAST] = "_DELETE_FAST", [_DELETE_GLOBAL] = "_DELETE_GLOBAL", [_DELETE_NAME] = "_DELETE_NAME", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_SEND] = "_END_SEND", + [_ERROR_POP_N] = "_ERROR_POP_N", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", + [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", + [_FOR_ITER_GEN_FRAME] = "_FOR_ITER_GEN_FRAME", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", [_GET_AITER] = "_GET_AITER", [_GET_ANEXT] = "_GET_ANEXT", @@ -295,21 +407,33 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", - [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", - [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", + [_GUARD_BUILTINS_VERSION_PUSH_KEYS] = "_GUARD_BUILTINS_VERSION_PUSH_KEYS", + [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_GLOBALS_VERSION_PUSH_KEYS] = "_GUARD_GLOBALS_VERSION_PUSH_KEYS", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", + [_GUARD_NOS_INT] = "_GUARD_NOS_INT", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", + [_GUARD_TOS_INT] = "_GUARD_TOS_INT", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_IMPORT_FROM] = "_IMPORT_FROM", + [_IMPORT_NAME] = "_IMPORT_NAME", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", [_INTERNAL_INCREMENT_OPT_COUNTER] = "_INTERNAL_INCREMENT_OPT_COUNTER", [_IS_NONE] = "_IS_NONE", [_IS_OP] = "_IS_OP", @@ -322,64 +446,107 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_JUMP_TO_TOP] = "_JUMP_TO_TOP", [_LIST_APPEND] = "_LIST_APPEND", [_LIST_EXTEND] = "_LIST_EXTEND", - [_LOAD_ASSERTION_ERROR] = "_LOAD_ASSERTION_ERROR", [_LOAD_ATTR] = "_LOAD_ATTR", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_0] = "_LOAD_ATTR_CLASS_0", + [_LOAD_ATTR_CLASS_1] = "_LOAD_ATTR_CLASS_1", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_0] = "_LOAD_ATTR_INSTANCE_VALUE_0", + [_LOAD_ATTR_INSTANCE_VALUE_1] = "_LOAD_ATTR_INSTANCE_VALUE_1", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_0] = "_LOAD_ATTR_SLOT_0", + [_LOAD_ATTR_SLOT_1] = "_LOAD_ATTR_SLOT_1", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", + [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_IMMORTAL] = "_LOAD_CONST_IMMORTAL", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = "_LOAD_CONST_INLINE_BORROW_WITH_NULL", [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_7] = "_LOAD_FAST_7", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", - [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_BUILTINS_FROM_KEYS] = "_LOAD_GLOBAL_BUILTINS_FROM_KEYS", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_MODULE_FROM_KEYS] = "_LOAD_GLOBAL_MODULE_FROM_KEYS", [_LOAD_LOCALS] = "_LOAD_LOCALS", [_LOAD_NAME] = "_LOAD_NAME", + [_LOAD_SMALL_INT] = "_LOAD_SMALL_INT", + [_LOAD_SMALL_INT_0] = "_LOAD_SMALL_INT_0", + [_LOAD_SMALL_INT_1] = "_LOAD_SMALL_INT_1", + [_LOAD_SMALL_INT_2] = "_LOAD_SMALL_INT_2", + [_LOAD_SMALL_INT_3] = "_LOAD_SMALL_INT_3", + [_LOAD_SPECIAL] = "_LOAD_SPECIAL", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_MAKE_CALLARGS_A_TUPLE] = "_MAKE_CALLARGS_A_TUPLE", [_MAKE_CELL] = "_MAKE_CELL", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_WARM] = "_MAKE_WARM", [_MAP_ADD] = "_MAP_ADD", [_MATCH_CLASS] = "_MATCH_CLASS", [_MATCH_KEYS] = "_MATCH_KEYS", [_MATCH_MAPPING] = "_MATCH_MAPPING", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", + [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", [_NOP] = "_NOP", [_POP_EXCEPT] = "_POP_EXCEPT", - [_POP_FRAME] = "_POP_FRAME", [_POP_TOP] = "_POP_TOP", + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", + [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_KW] = "_PY_FRAME_KW", + [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", + [_RETURN_GENERATOR] = "_RETURN_GENERATOR", + [_RETURN_VALUE] = "_RETURN_VALUE", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", [_SET_ADD] = "_SET_ADD", [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", + [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_ATTR_WITH_HINT] = "_STORE_ATTR_WITH_HINT", [_STORE_DEREF] = "_STORE_DEREF", [_STORE_FAST] = "_STORE_FAST", + [_STORE_FAST_0] = "_STORE_FAST_0", + [_STORE_FAST_1] = "_STORE_FAST_1", + [_STORE_FAST_2] = "_STORE_FAST_2", + [_STORE_FAST_3] = "_STORE_FAST_3", + [_STORE_FAST_4] = "_STORE_FAST_4", + [_STORE_FAST_5] = "_STORE_FAST_5", + [_STORE_FAST_6] = "_STORE_FAST_6", + [_STORE_FAST_7] = "_STORE_FAST_7", [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", [_STORE_GLOBAL] = "_STORE_GLOBAL", @@ -389,8 +556,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", [_SWAP] = "_SWAP", + [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", - [_TO_BOOL_ALWAYS_TRUE] = "_TO_BOOL_ALWAYS_TRUE", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", [_TO_BOOL_INT] = "_TO_BOOL_INT", [_TO_BOOL_LIST] = "_TO_BOOL_LIST", @@ -405,7 +572,560 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_YIELD_VALUE] = "_YIELD_VALUE", }; +int _PyUop_num_popped(int opcode, int oparg) +{ + switch(opcode) { + case _NOP: + return 0; + case _CHECK_PERIODIC: + return 0; + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: + return 0; + case _RESUME_CHECK: + return 0; + case _LOAD_FAST_CHECK: + return 0; + case _LOAD_FAST_0: + return 0; + case _LOAD_FAST_1: + return 0; + case _LOAD_FAST_2: + return 0; + case _LOAD_FAST_3: + return 0; + case _LOAD_FAST_4: + return 0; + case _LOAD_FAST_5: + return 0; + case _LOAD_FAST_6: + return 0; + case _LOAD_FAST_7: + return 0; + case _LOAD_FAST: + return 0; + case _LOAD_FAST_AND_CLEAR: + return 0; + case _LOAD_FAST_LOAD_FAST: + return 0; + case _LOAD_CONST: + return 0; + case _LOAD_CONST_IMMORTAL: + return 0; + case _LOAD_SMALL_INT_0: + return 0; + case _LOAD_SMALL_INT_1: + return 0; + case _LOAD_SMALL_INT_2: + return 0; + case _LOAD_SMALL_INT_3: + return 0; + case _LOAD_SMALL_INT: + return 0; + case _STORE_FAST_0: + return 1; + case _STORE_FAST_1: + return 1; + case _STORE_FAST_2: + return 1; + case _STORE_FAST_3: + return 1; + case _STORE_FAST_4: + return 1; + case _STORE_FAST_5: + return 1; + case _STORE_FAST_6: + return 1; + case _STORE_FAST_7: + return 1; + case _STORE_FAST: + return 1; + case _STORE_FAST_LOAD_FAST: + return 1; + case _STORE_FAST_STORE_FAST: + return 2; + case _POP_TOP: + return 1; + case _PUSH_NULL: + return 0; + case _END_SEND: + return 2; + case _UNARY_NEGATIVE: + return 1; + case _UNARY_NOT: + return 1; + case _TO_BOOL: + return 1; + case _TO_BOOL_BOOL: + return 1; + case _TO_BOOL_INT: + return 1; + case _TO_BOOL_LIST: + return 1; + case _TO_BOOL_NONE: + return 1; + case _TO_BOOL_STR: + return 1; + case _REPLACE_WITH_TRUE: + return 1; + case _UNARY_INVERT: + return 1; + case _GUARD_BOTH_INT: + return 2; + case _GUARD_NOS_INT: + return 2; + case _GUARD_TOS_INT: + return 1; + case _BINARY_OP_MULTIPLY_INT: + return 2; + case _BINARY_OP_ADD_INT: + return 2; + case _BINARY_OP_SUBTRACT_INT: + return 2; + case _GUARD_BOTH_FLOAT: + return 2; + case _GUARD_NOS_FLOAT: + return 2; + case _GUARD_TOS_FLOAT: + return 1; + case _BINARY_OP_MULTIPLY_FLOAT: + return 2; + case _BINARY_OP_ADD_FLOAT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT: + return 2; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 2; + case _BINARY_OP_INPLACE_ADD_UNICODE: + return 2; + case _BINARY_SUBSCR: + return 2; + case _BINARY_SLICE: + return 3; + case _STORE_SLICE: + return 4; + case _BINARY_SUBSCR_LIST_INT: + return 2; + case _BINARY_SUBSCR_STR_INT: + return 2; + case _BINARY_SUBSCR_TUPLE_INT: + return 2; + case _BINARY_SUBSCR_DICT: + return 2; + case _BINARY_SUBSCR_CHECK_FUNC: + return 2; + case _BINARY_SUBSCR_INIT_CALL: + return 2; + case _LIST_APPEND: + return 2 + (oparg-1); + case _SET_ADD: + return 2 + (oparg-1); + case _STORE_SUBSCR: + return 3; + case _STORE_SUBSCR_LIST_INT: + return 3; + case _STORE_SUBSCR_DICT: + return 3; + case _DELETE_SUBSCR: + return 2; + case _CALL_INTRINSIC_1: + return 1; + case _CALL_INTRINSIC_2: + return 2; + case _RETURN_VALUE: + return 1; + case _GET_AITER: + return 1; + case _GET_ANEXT: + return 1; + case _GET_AWAITABLE: + return 1; + case _SEND_GEN_FRAME: + return 2; + case _YIELD_VALUE: + return 1; + case _POP_EXCEPT: + return 1; + case _LOAD_COMMON_CONSTANT: + return 0; + case _LOAD_BUILD_CLASS: + return 0; + case _STORE_NAME: + return 1; + case _DELETE_NAME: + return 0; + case _UNPACK_SEQUENCE: + return 1; + case _UNPACK_SEQUENCE_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_TUPLE: + return 1; + case _UNPACK_SEQUENCE_LIST: + return 1; + case _UNPACK_EX: + return 1; + case _STORE_ATTR: + return 2; + case _DELETE_ATTR: + return 1; + case _STORE_GLOBAL: + return 1; + case _DELETE_GLOBAL: + return 0; + case _LOAD_LOCALS: + return 0; + case _LOAD_NAME: + return 0; + case _LOAD_GLOBAL: + return 0; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: + return 0; + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: + return 0; + case _LOAD_GLOBAL_MODULE_FROM_KEYS: + return 1; + case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: + return 1; + case _DELETE_FAST: + return 0; + case _MAKE_CELL: + return 0; + case _DELETE_DEREF: + return 0; + case _LOAD_FROM_DICT_OR_DEREF: + return 1; + case _LOAD_DEREF: + return 0; + case _STORE_DEREF: + return 1; + case _COPY_FREE_VARS: + return 0; + case _BUILD_STRING: + return oparg; + case _BUILD_TUPLE: + return oparg; + case _BUILD_LIST: + return oparg; + case _LIST_EXTEND: + return 2 + (oparg-1); + case _SET_UPDATE: + return 2 + (oparg-1); + case _BUILD_SET: + return oparg; + case _BUILD_MAP: + return oparg*2; + case _SETUP_ANNOTATIONS: + return 0; + case _DICT_UPDATE: + return 2 + (oparg - 1); + case _DICT_MERGE: + return 5 + (oparg - 1); + case _MAP_ADD: + return 3 + (oparg - 1); + case _LOAD_SUPER_ATTR_ATTR: + return 3; + case _LOAD_SUPER_ATTR_METHOD: + return 3; + case _LOAD_ATTR: + return 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE_0: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE_1: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return 1; + case _CHECK_ATTR_MODULE: + return 1; + case _LOAD_ATTR_MODULE: + return 1; + case _CHECK_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_SLOT_0: + return 1; + case _LOAD_ATTR_SLOT_1: + return 1; + case _LOAD_ATTR_SLOT: + return 1; + case _CHECK_ATTR_CLASS: + return 1; + case _LOAD_ATTR_CLASS_0: + return 1; + case _LOAD_ATTR_CLASS_1: + return 1; + case _LOAD_ATTR_CLASS: + return 1; + case _LOAD_ATTR_PROPERTY_FRAME: + return 1; + case _GUARD_DORV_NO_DICT: + return 1; + case _STORE_ATTR_INSTANCE_VALUE: + return 2; + case _STORE_ATTR_WITH_HINT: + return 2; + case _STORE_ATTR_SLOT: + return 2; + case _COMPARE_OP: + return 2; + case _COMPARE_OP_FLOAT: + return 2; + case _COMPARE_OP_INT: + return 2; + case _COMPARE_OP_STR: + return 2; + case _IS_OP: + return 2; + case _CONTAINS_OP: + return 2; + case _CONTAINS_OP_SET: + return 2; + case _CONTAINS_OP_DICT: + return 2; + case _CHECK_EG_MATCH: + return 2; + case _CHECK_EXC_MATCH: + return 2; + case _IMPORT_NAME: + return 2; + case _IMPORT_FROM: + return 1; + case _IS_NONE: + return 1; + case _GET_LEN: + return 1; + case _MATCH_CLASS: + return 3; + case _MATCH_MAPPING: + return 1; + case _MATCH_SEQUENCE: + return 1; + case _MATCH_KEYS: + return 2; + case _GET_ITER: + return 1; + case _GET_YIELD_FROM_ITER: + return 1; + case _FOR_ITER_TIER_TWO: + return 1; + case _ITER_CHECK_LIST: + return 1; + case _GUARD_NOT_EXHAUSTED_LIST: + return 1; + case _ITER_NEXT_LIST: + return 1; + case _ITER_CHECK_TUPLE: + return 1; + case _GUARD_NOT_EXHAUSTED_TUPLE: + return 1; + case _ITER_NEXT_TUPLE: + return 1; + case _ITER_CHECK_RANGE: + return 1; + case _GUARD_NOT_EXHAUSTED_RANGE: + return 1; + case _ITER_NEXT_RANGE: + return 1; + case _FOR_ITER_GEN_FRAME: + return 1; + case _LOAD_SPECIAL: + return 1; + case _WITH_EXCEPT_START: + return 5; + case _PUSH_EXC_INFO: + return 1; + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 1; + case _GUARD_KEYS_VERSION: + return 1; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 1; + case _LOAD_ATTR_METHOD_NO_DICT: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 1; + case _LOAD_ATTR_METHOD_LAZY_DICT: + return 1; + case _MAYBE_EXPAND_METHOD: + return 2 + oparg; + case _PY_FRAME_GENERAL: + return 2 + oparg; + case _CHECK_FUNCTION_VERSION: + return 2 + oparg; + case _CHECK_METHOD_VERSION: + return 2 + oparg; + case _EXPAND_METHOD: + return 2 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE: + return 2 + oparg; + case _CALL_NON_PY_GENERAL: + return 2 + oparg; + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return 2 + oparg; + case _CHECK_STACK_SPACE: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_0: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_1: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_2: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_3: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_4: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS: + return 2 + oparg; + case _PUSH_FRAME: + return 1; + case _CALL_TYPE_1: + return 3; + case _CALL_STR_1: + return 3; + case _CALL_TUPLE_1: + return 3; + case _CHECK_AND_ALLOCATE_OBJECT: + return 2 + oparg; + case _CREATE_INIT_FRAME: + return 2 + oparg; + case _EXIT_INIT_CHECK: + return 1; + case _CALL_BUILTIN_CLASS: + return 2 + oparg; + case _CALL_BUILTIN_O: + return 2 + oparg; + case _CALL_BUILTIN_FAST: + return 2 + oparg; + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_LEN: + return 2 + oparg; + case _CALL_ISINSTANCE: + return 2 + oparg; + case _CALL_LIST_APPEND: + return 3; + case _CALL_METHOD_DESCRIPTOR_O: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_NOARGS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST: + return 2 + oparg; + case _MAYBE_EXPAND_METHOD_KW: + return 3 + oparg; + case _PY_FRAME_KW: + return 3 + oparg; + case _CHECK_FUNCTION_VERSION_KW: + return 3 + oparg; + case _CHECK_METHOD_VERSION_KW: + return 3 + oparg; + case _EXPAND_METHOD_KW: + return 3 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE_KW: + return 3 + oparg; + case _CALL_KW_NON_PY: + return 3 + oparg; + case _MAKE_CALLARGS_A_TUPLE: + return 3 + (oparg & 1); + case _MAKE_FUNCTION: + return 1; + case _SET_FUNCTION_ATTRIBUTE: + return 2; + case _RETURN_GENERATOR: + return 0; + case _BUILD_SLICE: + return 2 + ((oparg == 3) ? 1 : 0); + case _CONVERT_VALUE: + return 1; + case _FORMAT_SIMPLE: + return 1; + case _FORMAT_WITH_SPEC: + return 2; + case _COPY: + return 1 + (oparg-1); + case _BINARY_OP: + return 2; + case _SWAP: + return 2 + (oparg-2); + case _GUARD_IS_TRUE_POP: + return 1; + case _GUARD_IS_FALSE_POP: + return 1; + case _GUARD_IS_NONE_POP: + return 1; + case _GUARD_IS_NOT_NONE_POP: + return 1; + case _JUMP_TO_TOP: + return 0; + case _SET_IP: + return 0; + case _CHECK_STACK_SPACE_OPERAND: + return 0; + case _SAVE_RETURN_OFFSET: + return 0; + case _EXIT_TRACE: + return 0; + case _CHECK_VALIDITY: + return 0; + case _LOAD_CONST_INLINE: + return 0; + case _LOAD_CONST_INLINE_BORROW: + return 0; + case _POP_TOP_LOAD_CONST_INLINE_BORROW: + return 1; + case _LOAD_CONST_INLINE_WITH_NULL: + return 0; + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: + return 0; + case _CHECK_FUNCTION: + return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; + case _INTERNAL_INCREMENT_OPT_COUNTER: + return 1; + case _DYNAMIC_EXIT: + return 0; + case _START_EXECUTOR: + return 0; + case _MAKE_WARM: + return 0; + case _FATAL_ERROR: + return 0; + case _CHECK_VALIDITY_AND_SET_IP: + return 0; + case _DEOPT: + return 0; + case _ERROR_POP_N: + return oparg; + case _TIER2_RESUME_CHECK: + return 0; + default: + return -1; + } +} + #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_warnings.h b/Include/internal/pycore_warnings.h index 9785d7cc467de2..f9f6559312f4ef 100644 --- a/Include/internal/pycore_warnings.h +++ b/Include/internal/pycore_warnings.h @@ -14,6 +14,7 @@ struct _warnings_runtime_state { PyObject *filters; /* List */ PyObject *once_registry; /* Dict */ PyObject *default_action; /* String */ + PyMutex mutex; long filters_version; }; diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index dea267b49039e7..94aadb2c1547dd 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -9,7 +9,35 @@ extern "C" { #endif #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_lock.h" #include "pycore_object.h" // _Py_REF_IS_MERGED() +#include "pycore_pyatomic_ft_wrappers.h" + +#ifdef Py_GIL_DISABLED + +#define WEAKREF_LIST_LOCK(obj) \ + _PyInterpreterState_GET() \ + ->weakref_locks[((uintptr_t)obj) % NUM_WEAKREF_LIST_LOCKS] + +// Lock using the referenced object +#define LOCK_WEAKREFS(obj) \ + PyMutex_LockFlags(&WEAKREF_LIST_LOCK(obj), _Py_LOCK_DONT_DETACH) +#define UNLOCK_WEAKREFS(obj) PyMutex_Unlock(&WEAKREF_LIST_LOCK(obj)) + +// Lock using a weakref +#define LOCK_WEAKREFS_FOR_WR(wr) \ + PyMutex_LockFlags(wr->weakrefs_lock, _Py_LOCK_DONT_DETACH) +#define UNLOCK_WEAKREFS_FOR_WR(wr) PyMutex_Unlock(wr->weakrefs_lock) + +#else + +#define LOCK_WEAKREFS(obj) +#define UNLOCK_WEAKREFS(obj) + +#define LOCK_WEAKREFS_FOR_WR(wr) +#define UNLOCK_WEAKREFS_FOR_WR(wr) + +#endif static inline int _is_dead(PyObject *obj) { @@ -30,53 +58,62 @@ static inline int _is_dead(PyObject *obj) static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) { assert(PyWeakref_Check(ref_obj)); - PyObject *ret = NULL; - Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); - PyObject *obj = ref->wr_object; + PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); if (obj == Py_None) { // clear_weakref() was called - goto end; + return NULL; } - if (_is_dead(obj)) { - goto end; + LOCK_WEAKREFS(obj); +#ifdef Py_GIL_DISABLED + if (ref->wr_object == Py_None) { + // clear_weakref() was called + UNLOCK_WEAKREFS(obj); + return NULL; } -#if !defined(Py_GIL_DISABLED) - assert(Py_REFCNT(obj) > 0); #endif - ret = Py_NewRef(obj); -end: - Py_END_CRITICAL_SECTION(); - return ret; + if (_Py_TryIncref(obj)) { + UNLOCK_WEAKREFS(obj); + return obj; + } + UNLOCK_WEAKREFS(obj); + return NULL; } static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) { assert(PyWeakref_Check(ref_obj)); int ret = 0; - Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); - PyObject *obj = ref->wr_object; + PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); if (obj == Py_None) { // clear_weakref() was called ret = 1; } else { + LOCK_WEAKREFS(obj); // See _PyWeakref_GET_REF() for the rationale of this test +#ifdef Py_GIL_DISABLED + ret = (ref->wr_object == Py_None) || _is_dead(obj); +#else ret = _is_dead(obj); +#endif + UNLOCK_WEAKREFS(obj); } - Py_END_CRITICAL_SECTION(); return ret; } -extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyWeakReference *head); +extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyObject *obj); -extern void _PyWeakref_ClearRef(PyWeakReference *self); +// Clear all the weak references to obj but leave their callbacks uncalled and +// intact. +extern void _PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj); + +PyAPI_FUNC(int) _PyWeakref_IsDead(PyObject *weakref); #ifdef __cplusplus } #endif #endif /* !Py_INTERNAL_WEAKREF_H */ - diff --git a/Include/interpreteridobject.h b/Include/interpreteridobject.h deleted file mode 100644 index 8432632f339e92..00000000000000 --- a/Include/interpreteridobject.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef Py_INTERPRETERIDOBJECT_H -#define Py_INTERPRETERIDOBJECT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_INTERPRETERIDOBJECT_H -# include "cpython/interpreteridobject.h" -# undef Py_CPYTHON_INTERPRETERIDOBJECT_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERPRETERIDOBJECT_H */ diff --git a/Include/listobject.h b/Include/listobject.h index 4e4084b43483a2..e1e059b0ba7466 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -29,7 +29,9 @@ PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size); PyAPI_FUNC(Py_ssize_t) PyList_Size(PyObject *); PyAPI_FUNC(PyObject *) PyList_GetItem(PyObject *, Py_ssize_t); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 PyAPI_FUNC(PyObject *) PyList_GetItemRef(PyObject *, Py_ssize_t); +#endif PyAPI_FUNC(int) PyList_SetItem(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Insert(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Append(PyObject *, PyObject *); diff --git a/Include/lock.h b/Include/lock.h new file mode 100644 index 00000000000000..782b9dbc70d056 --- /dev/null +++ b/Include/lock.h @@ -0,0 +1,16 @@ +#ifndef Py_LOCK_H +#define Py_LOCK_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_LOCK_H +# include "cpython/lock.h" +# undef Py_CPYTHON_LOCK_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_LOCK_H */ diff --git a/Include/longobject.h b/Include/longobject.h index 51005efff636fa..45c0d218c13f2f 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -30,6 +30,18 @@ PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLongMask(PyObject *); PyAPI_FUNC(int) PyLong_AsInt(PyObject *); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +PyAPI_FUNC(PyObject*) PyLong_FromInt32(int32_t value); +PyAPI_FUNC(PyObject*) PyLong_FromUInt32(uint32_t value); +PyAPI_FUNC(PyObject*) PyLong_FromInt64(int64_t value); +PyAPI_FUNC(PyObject*) PyLong_FromUInt64(uint64_t value); + +PyAPI_FUNC(int) PyLong_AsInt32(PyObject *obj, int32_t *value); +PyAPI_FUNC(int) PyLong_AsUInt32(PyObject *obj, uint32_t *value); +PyAPI_FUNC(int) PyLong_AsInt64(PyObject *obj, int64_t *value); +PyAPI_FUNC(int) PyLong_AsUInt64(PyObject *obj, uint64_t *value); +#endif + PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); /* It may be useful in the future. I've added it in the PyInt -> PyLong @@ -40,7 +52,24 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); #if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT #define _Py_PARSE_PID "i" #define PyLong_FromPid PyLong_FromLong -#define PyLong_AsPid PyLong_AsLong +# if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define PyLong_AsPid PyLong_AsInt +# elif SIZEOF_INT == SIZEOF_LONG +# define PyLong_AsPid PyLong_AsLong +# else +static inline int +PyLong_AsPid(PyObject *obj) +{ + int overflow; + long result = PyLong_AsLongAndOverflow(obj, &overflow); + if (overflow || result > INT_MAX || result < INT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)result; +} +# endif #elif SIZEOF_PID_T == SIZEOF_LONG #define _Py_PARSE_PID "l" #define PyLong_FromPid PyLong_FromLong diff --git a/Include/methodobject.h b/Include/methodobject.h index 2381e8482b82a8..39272815b127f4 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -17,15 +17,22 @@ PyAPI_DATA(PyTypeObject) PyCFunction_Type; #define PyCFunction_Check(op) PyObject_TypeCheck((op), &PyCFunction_Type) typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFast) (PyObject *, PyObject *const *, Py_ssize_t); +typedef PyObject *(*PyCFunctionFast) (PyObject *, PyObject *const *, Py_ssize_t); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFastWithKeywords) (PyObject *, - PyObject *const *, Py_ssize_t, - PyObject *); +typedef PyObject *(*PyCFunctionFastWithKeywords) (PyObject *, + PyObject *const *, Py_ssize_t, + PyObject *); typedef PyObject *(*PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +// For backwards compatibility. `METH_FASTCALL` was added to the stable API in +// 3.10 alongside `_PyCFunctionFastWithKeywords` and `_PyCFunctionFast`. +// Note that the underscore-prefixed names were documented in public docs; +// people may be using them. +typedef PyCFunctionFast _PyCFunctionFast; +typedef PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords; + // Cast an function to the PyCFunction type to use it with PyMethodDef. // // This macro can be used to prevent compiler warnings if the first parameter diff --git a/Include/modsupport.h b/Include/modsupport.h index ea4c0fce9f4562..af995f567b004c 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -134,6 +134,12 @@ PyAPI_FUNC(PyObject *) PyModule_FromDefAndSpec2(PyModuleDef *def, #endif /* New in 3.5 */ +#ifndef Py_LIMITED_API +# define Py_CPYTHON_MODSUPPORT_H +# include "cpython/modsupport.h" +# undef Py_CPYTHON_MODSUPPORT_H +#endif + #ifdef __cplusplus } #endif diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 42b87cc4e91012..2a17c891dda811 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -53,7 +53,7 @@ typedef struct PyModuleDef_Base { /* A copy of the module's __dict__ after the first time it was loaded. This is only set/used for legacy modules that do not support multiple initializations. - It is set by _PyImport_FixupExtensionObject(). */ + It is set by fix_up_extension() in import.c. */ PyObject* m_copy; } PyModuleDef_Base; @@ -76,9 +76,13 @@ struct PyModuleDef_Slot { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 # define Py_mod_multiple_interpreters 3 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_mod_gil 4 +#endif + #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 3 +#define _Py_mod_LAST_SLOT 4 #endif #endif /* New in 3.5 */ @@ -90,6 +94,16 @@ struct PyModuleDef_Slot { # define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) #endif +/* for Py_mod_gil: */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_MOD_GIL_USED ((void *)0) +# define Py_MOD_GIL_NOT_USED ((void *)1) +#endif + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); +#endif + struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; diff --git a/Include/monitoring.h b/Include/monitoring.h new file mode 100644 index 00000000000000..985f7f230e44e3 --- /dev/null +++ b/Include/monitoring.h @@ -0,0 +1,18 @@ +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifdef __cplusplus +extern "C" { +#endif + +// There is currently no limited API for monitoring + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_MONITORING_H +# include "cpython/monitoring.h" +# undef Py_CPYTHON_MONITORING_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_MONITORING_H */ diff --git a/Include/object.h b/Include/object.h index 05187fe5dc4f20..3876d8449afbe2 100644 --- a/Include/object.h +++ b/Include/object.h @@ -59,64 +59,14 @@ whose size is determined when the object is allocated. /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; -/* -Immortalization: - -The following indicates the immortalization strategy depending on the amount -of available bits in the reference count field. All strategies are backwards -compatible but the specific reference count value or immortalization check -might change depending on the specializations for the underlying system. - -Proper deallocation of immortal instances requires distinguishing between -statically allocated immortal instances vs those promoted by the runtime to be -immortal. The latter should be the only instances that require -cleanup during runtime finalization. -*/ - -#if SIZEOF_VOID_P > 4 -/* -In 64+ bit systems, an object will be marked as immortal by setting all of the -lower 32 bits of the reference count field, which is equal to: 0xFFFFFFFF - -Using the lower 32 bits makes the value backwards compatible by allowing -C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely -increase and decrease the objects reference count. The object would lose its -immortality, but the execution would still be correct. - -Reference count increases will use saturated arithmetic, taking advantage of -having all the lower 32 bits set, which will avoid the reference count to go -beyond the refcount limit. Immortality checks for reference count decreases will -be done by checking the bit sign flag in the lower 32 bits. -*/ -#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX) - -#else -/* -In 32 bit systems, an object will be marked as immortal by setting all of the -lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF - -Using the lower 30 bits makes the value backwards compatible by allowing -C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely -increase and decrease the objects reference count. The object would lose its -immortality, but the execution would still be correct. - -Reference count increases and decreases will first go through an immortality -check by comparing the reference count field to the immortality reference count. -*/ -#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX >> 2) -#endif - -// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is -// always 32-bits. -#ifdef Py_GIL_DISABLED -#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX -#endif - // Kept for backward compatibility. It was needed by Py_TRACE_REFS build. #define _PyObject_EXTRA_INIT -// Make all internal uses of PyObject_HEAD_INIT immortal while preserving the -// C-API expectation that the refcnt will be set to 1. +/* Make all uses of PyObject_HEAD_INIT immortal. + * + * Statically allocated objects might be shared between + * interpreters, so must be marked as immortal. + */ #if defined(Py_GIL_DISABLED) #define PyObject_HEAD_INIT(type) \ { \ @@ -128,19 +78,13 @@ check by comparing the reference count field to the immortality reference count. 0, \ (type), \ }, -#elif defined(Py_BUILD_CORE) +#else #define PyObject_HEAD_INIT(type) \ { \ - { _Py_IMMORTAL_REFCNT }, \ + { _Py_IMMORTAL_INITIAL_REFCNT }, \ (type) \ }, -#else -#define PyObject_HEAD_INIT(type) \ - { \ - { 1 }, \ - (type) \ - }, -#endif /* Py_BUILD_CORE */ +#endif #define PyVarObject_HEAD_INIT(type, size) \ { \ @@ -193,31 +137,13 @@ struct _object { // fields have been merged. #define _Py_UNOWNED_TID 0 -// The shared reference count uses the two least-significant bits to store -// flags. The remaining bits are used to store the reference count. -#define _Py_REF_SHARED_SHIFT 2 -#define _Py_REF_SHARED_FLAG_MASK 0x3 - -// The shared flags are initialized to zero. -#define _Py_REF_SHARED_INIT 0x0 -#define _Py_REF_MAYBE_WEAKREF 0x1 -#define _Py_REF_QUEUED 0x2 -#define _Py_REF_MERGED 0x3 - -// Create a shared field from a refcnt and desired flags -#define _Py_REF_SHARED(refcnt, flags) (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) - -// NOTE: In non-free-threaded builds, `struct _PyMutex` is defined in -// pycore_lock.h. See pycore_lock.h for more details. -struct _PyMutex { uint8_t v; }; - struct _object { // ob_tid stores the thread id (or zero). It is also used by the GC and the // trashcan mechanism as a linked list pointer and by the GC to store the // computed "gc_refs" refcount. uintptr_t ob_tid; uint16_t _padding; - struct _PyMutex ob_mutex; // per-object lock + PyMutex ob_mutex; // per-object lock uint8_t ob_gc_bits; // gc-related state uint32_t ob_ref_local; // local reference count Py_ssize_t ob_ref_shared; // shared (atomic) reference count @@ -254,13 +180,19 @@ _Py_ThreadId(void) tid = __readfsdword(24); #elif defined(_MSC_VER) && defined(_M_ARM64) tid = __getReg(18); +#elif defined(__MINGW32__) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(__MINGW32__) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(__MINGW32__) && defined(_M_ARM64) + tid = __getReg(18); #elif defined(__i386__) __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) __asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS #elif defined(__x86_64__) __asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS -#elif defined(__arm__) +#elif defined(__arm__) && __ARM_ARCH >= 7 __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); #elif defined(__aarch64__) && defined(__APPLE__) __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); @@ -306,34 +238,30 @@ _Py_ThreadId(void) static inline Py_ALWAYS_INLINE int _Py_IsOwnedByCurrentThread(PyObject *ob) { - return ob->ob_tid == _Py_ThreadId(); -} -#endif - -static inline Py_ssize_t Py_REFCNT(PyObject *ob) { -#if !defined(Py_GIL_DISABLED) - return ob->ob_refcnt; +#ifdef _Py_THREAD_SANITIZER + return _Py_atomic_load_uintptr_relaxed(&ob->ob_tid) == _Py_ThreadId(); #else - uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return _Py_IMMORTAL_REFCNT; - } - Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); - return _Py_STATIC_CAST(Py_ssize_t, local) + - Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); + return ob->ob_tid == _Py_ThreadId(); #endif } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) #endif +// Py_TYPE() implementation for the stable ABI +PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); -// bpo-39573: The Py_SET_TYPE() function must be used to set an object type. -static inline PyTypeObject* Py_TYPE(PyObject *ob) { - return ob->ob_type; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob)) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 + // Stable ABI implements Py_TYPE() as a function call + // on limited C API version 3.14 and newer. +#else + static inline PyTypeObject* _Py_TYPE(PyObject *ob) + { + return ob->ob_type; + } + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 + # define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST(ob)) + #else + # define Py_TYPE(ob) _Py_TYPE(ob) + #endif #endif PyAPI_DATA(PyTypeObject) PyLong_Type; @@ -341,27 +269,14 @@ PyAPI_DATA(PyTypeObject) PyBool_Type; // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. static inline Py_ssize_t Py_SIZE(PyObject *ob) { - assert(ob->ob_type != &PyLong_Type); - assert(ob->ob_type != &PyBool_Type); - PyVarObject *var_ob = _PyVarObject_CAST(ob); - return var_ob->ob_size; + assert(Py_TYPE(ob) != &PyLong_Type); + assert(Py_TYPE(ob) != &PyBool_Type); + return _PyVarObject_CAST(ob)->ob_size; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif -static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) -{ -#if defined(Py_GIL_DISABLED) - return (op->ob_ref_local == _Py_IMMORTAL_REFCNT_LOCAL); -#elif SIZEOF_VOID_P > 4 - return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0); -#else - return (op->ob_refcnt == _Py_IMMORTAL_REFCNT); -#endif -} -#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) - static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; } @@ -370,55 +285,6 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { #endif -// Py_SET_REFCNT() implementation for stable ABI -PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); - -static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 - // Stable ABI implements Py_SET_REFCNT() as a function call - // on limited C API version 3.13 and newer. - _Py_SetRefcnt(ob, refcnt); -#else - // This immortal check is for code that is unaware of immortal objects. - // The runtime tracks these objects and we should avoid as much - // as possible having extensions inadvertently change the refcnt - // of an immortalized object. - if (_Py_IsImmortal(ob)) { - return; - } - -#ifndef Py_GIL_DISABLED - ob->ob_refcnt = refcnt; -#else - if (_Py_IsOwnedByCurrentThread(ob)) { - if ((size_t)refcnt > (size_t)UINT32_MAX) { - // On overflow, make the object immortal - ob->ob_tid = _Py_UNOWNED_TID; - ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - ob->ob_ref_shared = 0; - } - else { - // Set local refcount to desired refcount and shared refcount - // to zero, but preserve the shared refcount flags. - ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); - ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; - } - } - else { - // Set local refcount to zero and shared refcount to desired refcount. - // Mark the object as merged. - ob->ob_tid = _Py_UNOWNED_TID; - ob->ob_ref_local = 0; - ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); - } -#endif // Py_GIL_DISABLED -#endif // Py_LIMITED_API+0 < 0x030d0000 -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) -#endif - - static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } @@ -427,8 +293,8 @@ static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { #endif static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { - assert(ob->ob_base.ob_type != &PyLong_Type); - assert(ob->ob_base.ob_type != &PyBool_Type); + assert(Py_TYPE(_PyObject_CAST(ob)) != &PyLong_Type); + assert(Py_TYPE(_PyObject_CAST(ob)) != &PyBool_Type); #ifdef Py_GIL_DISABLED _Py_atomic_store_ssize_relaxed(&ob->ob_size, size); #else @@ -522,11 +388,19 @@ PyAPI_FUNC(void *) PyType_GetModuleState(PyTypeObject *); PyAPI_FUNC(PyObject *) PyType_GetName(PyTypeObject *); PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 +PyAPI_FUNC(PyObject *) PyType_GetFullyQualifiedName(PyTypeObject *type); +PyAPI_FUNC(PyObject *) PyType_GetModuleName(PyTypeObject *type); +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*); PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls); PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 +PyAPI_FUNC(int) PyType_GetBaseByToken(PyTypeObject *, void *, PyTypeObject **); +#define Py_TP_USE_SPEC NULL +#endif /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); @@ -626,13 +500,18 @@ given type object has a specified feature. /* Track types initialized using _PyStaticType_InitBuiltin(). */ #define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1) +/* The values array is placed inline directly after the rest of + * the object. Implies Py_TPFLAGS_HAVE_GC. + */ +#define Py_TPFLAGS_INLINE_VALUES (1 << 2) + /* Placement of weakref pointers are managed by the VM, not by the type. * The VM will automatically set tp_weaklistoffset. */ #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3) /* Placement of dict (and values) pointers are managed by the VM, not by the type. - * The VM will automatically set tp_dictoffset. + * The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC. */ #define Py_TPFLAGS_MANAGED_DICT (1 << 4) @@ -685,7 +564,7 @@ given type object has a specified feature. /* Objects behave like an unbound method */ #define Py_TPFLAGS_METHOD_DESCRIPTOR (1UL << 17) -/* Object has up-to-date type attribute cache */ +/* Unused. Legacy flag */ #define Py_TPFLAGS_VALID_VERSION_TAG (1UL << 19) /* Type is abstract and cannot be instantiated */ @@ -730,338 +609,20 @@ given type object has a specified feature. #define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18) -/* -The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement -reference counts. Py_DECREF calls the object's deallocator function when -the refcount falls to 0; for -objects that don't contain references to other objects or heap memory -this can be the standard function free(). Both macros can be used -wherever a void expression is allowed. The argument must not be a -NULL pointer. If it may be NULL, use Py_XINCREF/Py_XDECREF instead. -The macro _Py_NewReference(op) initialize reference counts to 1, and -in special builds (Py_REF_DEBUG, Py_TRACE_REFS) performs additional -bookkeeping appropriate to the special build. - -We assume that the reference count field can never overflow; this can -be proven when the size of the field is the same as the pointer size, so -we ignore the possibility. Provided a C int is at least 32 bits (which -is implicitly assumed in many parts of this code), that's enough for -about 2**31 references to an object. - -XXX The following became out of date in Python 2.2, but I'm not sure -XXX what the full truth is now. Certainly, heap-allocated type objects -XXX can and should be deallocated. -Type objects should never be deallocated; the type pointer in an object -is not considered to be a reference to the type object, to save -complications in the deallocation function. (This is actually a -decision that's up to the implementer of each new type so if you want, -you can count such references to the type object.) -*/ - -#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) -PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, - PyObject *op); -PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void); -PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void); -#endif // Py_REF_DEBUG && !Py_LIMITED_API - -PyAPI_FUNC(void) _Py_Dealloc(PyObject *); +#define Py_CONSTANT_NONE 0 +#define Py_CONSTANT_FALSE 1 +#define Py_CONSTANT_TRUE 2 +#define Py_CONSTANT_ELLIPSIS 3 +#define Py_CONSTANT_NOT_IMPLEMENTED 4 +#define Py_CONSTANT_ZERO 5 +#define Py_CONSTANT_ONE 6 +#define Py_CONSTANT_EMPTY_STR 7 +#define Py_CONSTANT_EMPTY_BYTES 8 +#define Py_CONSTANT_EMPTY_TUPLE 9 -/* -These are provided as conveniences to Python runtime embedders, so that -they can have object code that is not dependent on Python compilation flags. -*/ -PyAPI_FUNC(void) Py_IncRef(PyObject *); -PyAPI_FUNC(void) Py_DecRef(PyObject *); - -// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL. -// Private functions used by Py_INCREF() and Py_DECREF(). -PyAPI_FUNC(void) _Py_IncRef(PyObject *); -PyAPI_FUNC(void) _Py_DecRef(PyObject *); - -static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) -{ -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) - // Stable ABI implements Py_INCREF() as a function call on limited C API - // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() - // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. - // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. -# if Py_LIMITED_API+0 >= 0x030a00A7 - _Py_IncRef(op); -# else - Py_IncRef(op); -# endif -#else - // Non-limited C API and limited C API for Python 3.9 and older access - // directly PyObject.ob_refcnt. -#if defined(Py_GIL_DISABLED) - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - uint32_t new_local = local + 1; - if (new_local == 0) { - // local is equal to _Py_IMMORTAL_REFCNT: do nothing - return; - } - if (_Py_IsOwnedByCurrentThread(op)) { - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local); - } - else { - _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); - } -#elif SIZEOF_VOID_P > 4 - // Portable saturated add, branching on the carry flag and set low bits - PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; - PY_UINT32_T new_refcnt = cur_refcnt + 1; - if (new_refcnt == 0) { - // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal, - // do nothing - return; - } - op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; -#else - // Explicitly check immortality against the immortal value - if (_Py_IsImmortal(op)) { - return; - } - op->ob_refcnt++; -#endif - _Py_INCREF_STAT_INC(); -#ifdef Py_REF_DEBUG - _Py_INCREF_IncRefTotal(); -#endif -#endif -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) -#endif - - -#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) -// Implements Py_DECREF on objects not owned by the current thread. -PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); -PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); - -// Called from Py_DECREF by the owning thread when the local refcount reaches -// zero. The call will deallocate the object if the shared refcount is also -// zero. Otherwise, the thread gives up ownership and merges the reference -// count fields. -PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); -#endif - -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) -// Stable ABI implements Py_DECREF() as a function call on limited C API -// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was -// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. -// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. -static inline void Py_DECREF(PyObject *op) { -# if Py_LIMITED_API+0 >= 0x030a00A7 - _Py_DecRef(op); -# else - Py_DecRef(op); -# endif -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) - -#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG) -static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) -{ - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return; - } - _Py_DECREF_STAT_INC(); - _Py_DECREF_DecRefTotal(); - if (_Py_IsOwnedByCurrentThread(op)) { - if (local == 0) { - _Py_NegativeRefcount(filename, lineno, op); - } - local--; - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); - if (local == 0) { - _Py_MergeZeroLocalRefcount(op); - } - } - else { - _Py_DecRefSharedDebug(op, filename, lineno); - } -} -#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) - -#elif defined(Py_GIL_DISABLED) -static inline void Py_DECREF(PyObject *op) -{ - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return; - } - _Py_DECREF_STAT_INC(); - if (_Py_IsOwnedByCurrentThread(op)) { - local--; - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); - if (local == 0) { - _Py_MergeZeroLocalRefcount(op); - } - } - else { - _Py_DecRefShared(op); - } -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) - -#elif defined(Py_REF_DEBUG) -static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) -{ - if (op->ob_refcnt <= 0) { - _Py_NegativeRefcount(filename, lineno, op); - } - if (_Py_IsImmortal(op)) { - return; - } - _Py_DECREF_STAT_INC(); - _Py_DECREF_DecRefTotal(); - if (--op->ob_refcnt == 0) { - _Py_Dealloc(op); - } -} -#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) - -#else -static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) -{ - // Non-limited C API and limited C API for Python 3.9 and older access - // directly PyObject.ob_refcnt. - if (_Py_IsImmortal(op)) { - return; - } - _Py_DECREF_STAT_INC(); - if (--op->ob_refcnt == 0) { - _Py_Dealloc(op); - } -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) -#endif - - -/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear - * and tp_dealloc implementations. - * - * Note that "the obvious" code can be deadly: - * - * Py_XDECREF(op); - * op = NULL; - * - * Typically, `op` is something like self->containee, and `self` is done - * using its `containee` member. In the code sequence above, suppose - * `containee` is non-NULL with a refcount of 1. Its refcount falls to - * 0 on the first line, which can trigger an arbitrary amount of code, - * possibly including finalizers (like __del__ methods or weakref callbacks) - * coded in Python, which in turn can release the GIL and allow other threads - * to run, etc. Such code may even invoke methods of `self` again, or cause - * cyclic gc to trigger, but-- oops! --self->containee still points to the - * object being torn down, and it may be in an insane state while being torn - * down. This has in fact been a rich historic source of miserable (rare & - * hard-to-diagnose) segfaulting (and other) bugs. - * - * The safe way is: - * - * Py_CLEAR(op); - * - * That arranges to set `op` to NULL _before_ decref'ing, so that any code - * triggered as a side-effect of `op` getting torn down no longer believes - * `op` points to a valid object. - * - * There are cases where it's safe to use the naive code, but they're brittle. - * For example, if `op` points to a Python integer, you know that destroying - * one of those can't cause problems -- but in part that relies on that - * Python integers aren't currently weakly referencable. Best practice is - * to use Py_CLEAR() even if you can't think of a reason for why you need to. - * - * gh-98724: Use a temporary variable to only evaluate the macro argument once, - * to avoid the duplication of side effects if the argument has side effects. - * - * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, - * the code can be miscompiled with strict aliasing because of type punning. - * With strict aliasing, a compiler considers that two pointers of different - * types cannot read or write the same memory which enables optimization - * opportunities. - * - * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, - * and so avoid type punning. Otherwise, use memcpy() which causes type erasure - * and so prevents the compiler to reuse an old cached 'op' value after - * Py_CLEAR(). - */ -#ifdef _Py_TYPEOF -#define Py_CLEAR(op) \ - do { \ - _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ - _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ - if (_tmp_old_op != NULL) { \ - *_tmp_op_ptr = _Py_NULL; \ - Py_DECREF(_tmp_old_op); \ - } \ - } while (0) -#else -#define Py_CLEAR(op) \ - do { \ - PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ - PyObject *_tmp_old_op = (*_tmp_op_ptr); \ - if (_tmp_old_op != NULL) { \ - PyObject *_null_ptr = _Py_NULL; \ - memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ - Py_DECREF(_tmp_old_op); \ - } \ - } while (0) -#endif - - -/* Function to use in case the object pointer can be NULL: */ -static inline void Py_XINCREF(PyObject *op) -{ - if (op != _Py_NULL) { - Py_INCREF(op); - } -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) -#endif - -static inline void Py_XDECREF(PyObject *op) -{ - if (op != _Py_NULL) { - Py_DECREF(op); - } -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) -#endif - -// Create a new strong reference to an object: -// increment the reference count of the object and return the object. -PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); - -// Similar to Py_NewRef(), but the object can be NULL. -PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); - -static inline PyObject* _Py_NewRef(PyObject *obj) -{ - Py_INCREF(obj); - return obj; -} - -static inline PyObject* _Py_XNewRef(PyObject *obj) -{ - Py_XINCREF(obj); - return obj; -} - -// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. -// Names overridden with macros by static inline functions for best -// performances. -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) -# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) -#else -# define Py_NewRef(obj) _Py_NewRef(obj) -# define Py_XNewRef(obj) _Py_XNewRef(obj) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(PyObject*) Py_GetConstant(unsigned int constant_id); +PyAPI_FUNC(PyObject*) Py_GetConstantBorrowed(unsigned int constant_id); #endif @@ -1070,7 +631,12 @@ _Py_NoneStruct is an object of undefined type which can be used in contexts where NULL (nil) is not suitable (since NULL often means 'error'). */ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ -#define Py_None (&_Py_NoneStruct) + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_None Py_GetConstantBorrowed(Py_CONSTANT_NONE) +#else +# define Py_None (&_Py_NoneStruct) +#endif // Test if an object is the None singleton, the same as "x is None" in Python. PyAPI_FUNC(int) Py_IsNone(PyObject *x); @@ -1084,7 +650,12 @@ Py_NotImplemented is a singleton used to signal that an operation is not implemented for a given type combination. */ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ -#define Py_NotImplemented (&_Py_NotImplementedStruct) + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_NotImplemented Py_GetConstantBorrowed(Py_CONSTANT_NOT_IMPLEMENTED) +#else +# define Py_NotImplemented (&_Py_NotImplementedStruct) +#endif /* Macro for returning Py_NotImplemented from a function */ #define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented @@ -1193,7 +764,11 @@ PyType_HasFeature(PyTypeObject *type, unsigned long feature) // PyTypeObject is opaque in the limited C API flags = PyType_GetFlags(type); #else - flags = type->tp_flags; +# ifdef Py_GIL_DISABLED + flags = _Py_atomic_load_ulong_relaxed(&type->tp_flags); +# else + flags = type->tp_flags; +# endif #endif return ((flags & feature) != 0); } @@ -1217,6 +792,14 @@ static inline int PyType_CheckExact(PyObject *op) { # define PyType_CheckExact(op) PyType_CheckExact(_PyObject_CAST(op)) #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); +#endif + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/objimpl.h b/Include/objimpl.h index ff5fa7a8c1d3d8..56472a72e42d34 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -153,25 +153,6 @@ PyAPI_FUNC(int) PyGC_Enable(void); PyAPI_FUNC(int) PyGC_Disable(void); PyAPI_FUNC(int) PyGC_IsEnabled(void); - -#if !defined(Py_LIMITED_API) -/* Visit all live GC-capable objects, similar to gc.get_objects(None). The - * supplied callback is called on every such object with the void* arg set - * to the supplied arg. Returning 0 from the callback ends iteration, returning - * 1 allows iteration to continue. Returning any other value may result in - * undefined behaviour. - * - * If new objects are (de)allocated by the callback it is undefined if they - * will be visited. - - * Garbage collection is disabled during operation. Explicitly running a - * collection in the callback may lead to undefined behaviour e.g. visiting the - * same objects multiple times or not at all. - */ -typedef int (*gcvisitobjects_t)(PyObject*, void*); -PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); -#endif - /* Test if a type has a GC head */ #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index fe969342ee79e7..ce3d23eaa6d56d 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -11,124 +11,121 @@ extern "C" { /* Instruction opcodes for compiled code */ #define CACHE 0 -#define BEFORE_ASYNC_WITH 1 -#define BEFORE_WITH 2 +#define BINARY_SLICE 1 +#define BINARY_SUBSCR 2 #define BINARY_OP_INPLACE_ADD_UNICODE 3 -#define BINARY_SLICE 4 -#define BINARY_SUBSCR 5 -#define CHECK_EG_MATCH 6 -#define CHECK_EXC_MATCH 7 -#define CLEANUP_THROW 8 -#define DELETE_SUBSCR 9 -#define END_ASYNC_FOR 10 -#define END_FOR 11 -#define END_SEND 12 -#define EXIT_INIT_CHECK 13 -#define FORMAT_SIMPLE 14 -#define FORMAT_WITH_SPEC 15 -#define GET_AITER 16 +#define CHECK_EG_MATCH 4 +#define CHECK_EXC_MATCH 5 +#define CLEANUP_THROW 6 +#define DELETE_SUBSCR 7 +#define END_ASYNC_FOR 8 +#define END_FOR 9 +#define END_SEND 10 +#define EXIT_INIT_CHECK 11 +#define FORMAT_SIMPLE 12 +#define FORMAT_WITH_SPEC 13 +#define GET_AITER 14 +#define GET_ANEXT 15 +#define GET_ITER 16 #define RESERVED 17 -#define GET_ANEXT 18 -#define GET_ITER 19 -#define GET_LEN 20 -#define GET_YIELD_FROM_ITER 21 -#define INTERPRETER_EXIT 22 -#define LOAD_ASSERTION_ERROR 23 -#define LOAD_BUILD_CLASS 24 -#define LOAD_LOCALS 25 -#define MAKE_FUNCTION 26 -#define MATCH_KEYS 27 -#define MATCH_MAPPING 28 -#define MATCH_SEQUENCE 29 -#define NOP 30 -#define POP_EXCEPT 31 -#define POP_TOP 32 -#define PUSH_EXC_INFO 33 -#define PUSH_NULL 34 -#define RETURN_GENERATOR 35 -#define RETURN_VALUE 36 -#define SETUP_ANNOTATIONS 37 -#define STORE_SLICE 38 -#define STORE_SUBSCR 39 -#define TO_BOOL 40 -#define UNARY_INVERT 41 -#define UNARY_NEGATIVE 42 -#define UNARY_NOT 43 -#define WITH_EXCEPT_START 44 -#define BINARY_OP 45 -#define BUILD_CONST_KEY_MAP 46 -#define BUILD_LIST 47 -#define BUILD_MAP 48 -#define BUILD_SET 49 -#define BUILD_SLICE 50 -#define BUILD_STRING 51 -#define BUILD_TUPLE 52 -#define CALL 53 -#define CALL_FUNCTION_EX 54 -#define CALL_INTRINSIC_1 55 -#define CALL_INTRINSIC_2 56 -#define CALL_KW 57 -#define COMPARE_OP 58 -#define CONTAINS_OP 59 -#define CONVERT_VALUE 60 -#define COPY 61 -#define COPY_FREE_VARS 62 -#define DELETE_ATTR 63 -#define DELETE_DEREF 64 -#define DELETE_FAST 65 -#define DELETE_GLOBAL 66 -#define DELETE_NAME 67 -#define DICT_MERGE 68 -#define DICT_UPDATE 69 -#define ENTER_EXECUTOR 70 -#define EXTENDED_ARG 71 -#define FOR_ITER 72 -#define GET_AWAITABLE 73 -#define IMPORT_FROM 74 -#define IMPORT_NAME 75 -#define IS_OP 76 -#define JUMP_BACKWARD 77 -#define JUMP_BACKWARD_NO_INTERRUPT 78 -#define JUMP_FORWARD 79 -#define LIST_APPEND 80 -#define LIST_EXTEND 81 -#define LOAD_ATTR 82 -#define LOAD_CONST 83 -#define LOAD_DEREF 84 -#define LOAD_FAST 85 -#define LOAD_FAST_AND_CLEAR 86 -#define LOAD_FAST_CHECK 87 -#define LOAD_FAST_LOAD_FAST 88 -#define LOAD_FROM_DICT_OR_DEREF 89 -#define LOAD_FROM_DICT_OR_GLOBALS 90 -#define LOAD_GLOBAL 91 -#define LOAD_NAME 92 -#define LOAD_SUPER_ATTR 93 -#define MAKE_CELL 94 -#define MAP_ADD 95 -#define MATCH_CLASS 96 -#define POP_JUMP_IF_FALSE 97 -#define POP_JUMP_IF_NONE 98 -#define POP_JUMP_IF_NOT_NONE 99 -#define POP_JUMP_IF_TRUE 100 -#define RAISE_VARARGS 101 -#define RERAISE 102 -#define RETURN_CONST 103 -#define SEND 104 -#define SET_ADD 105 -#define SET_FUNCTION_ATTRIBUTE 106 -#define SET_UPDATE 107 -#define STORE_ATTR 108 -#define STORE_DEREF 109 -#define STORE_FAST 110 -#define STORE_FAST_LOAD_FAST 111 -#define STORE_FAST_STORE_FAST 112 -#define STORE_GLOBAL 113 -#define STORE_NAME 114 -#define SWAP 115 -#define UNPACK_EX 116 -#define UNPACK_SEQUENCE 117 -#define YIELD_VALUE 118 +#define GET_LEN 18 +#define GET_YIELD_FROM_ITER 19 +#define INTERPRETER_EXIT 20 +#define LOAD_BUILD_CLASS 21 +#define LOAD_LOCALS 22 +#define MAKE_FUNCTION 23 +#define MATCH_KEYS 24 +#define MATCH_MAPPING 25 +#define MATCH_SEQUENCE 26 +#define NOP 27 +#define POP_EXCEPT 28 +#define POP_TOP 29 +#define PUSH_EXC_INFO 30 +#define PUSH_NULL 31 +#define RETURN_GENERATOR 32 +#define RETURN_VALUE 33 +#define SETUP_ANNOTATIONS 34 +#define STORE_SLICE 35 +#define STORE_SUBSCR 36 +#define TO_BOOL 37 +#define UNARY_INVERT 38 +#define UNARY_NEGATIVE 39 +#define UNARY_NOT 40 +#define WITH_EXCEPT_START 41 +#define BINARY_OP 42 +#define BUILD_LIST 43 +#define BUILD_MAP 44 +#define BUILD_SET 45 +#define BUILD_SLICE 46 +#define BUILD_STRING 47 +#define BUILD_TUPLE 48 +#define CALL 49 +#define CALL_FUNCTION_EX 50 +#define CALL_INTRINSIC_1 51 +#define CALL_INTRINSIC_2 52 +#define CALL_KW 53 +#define COMPARE_OP 54 +#define CONTAINS_OP 55 +#define CONVERT_VALUE 56 +#define COPY 57 +#define COPY_FREE_VARS 58 +#define DELETE_ATTR 59 +#define DELETE_DEREF 60 +#define DELETE_FAST 61 +#define DELETE_GLOBAL 62 +#define DELETE_NAME 63 +#define DICT_MERGE 64 +#define DICT_UPDATE 65 +#define EXTENDED_ARG 66 +#define FOR_ITER 67 +#define GET_AWAITABLE 68 +#define IMPORT_FROM 69 +#define IMPORT_NAME 70 +#define IS_OP 71 +#define JUMP_BACKWARD 72 +#define JUMP_BACKWARD_NO_INTERRUPT 73 +#define JUMP_FORWARD 74 +#define LIST_APPEND 75 +#define LIST_EXTEND 76 +#define LOAD_ATTR 77 +#define LOAD_COMMON_CONSTANT 78 +#define LOAD_CONST 79 +#define LOAD_DEREF 80 +#define LOAD_FAST 81 +#define LOAD_FAST_AND_CLEAR 82 +#define LOAD_FAST_CHECK 83 +#define LOAD_FAST_LOAD_FAST 84 +#define LOAD_FROM_DICT_OR_DEREF 85 +#define LOAD_FROM_DICT_OR_GLOBALS 86 +#define LOAD_GLOBAL 87 +#define LOAD_NAME 88 +#define LOAD_SMALL_INT 89 +#define LOAD_SPECIAL 90 +#define LOAD_SUPER_ATTR 91 +#define MAKE_CELL 92 +#define MAP_ADD 93 +#define MATCH_CLASS 94 +#define POP_JUMP_IF_FALSE 95 +#define POP_JUMP_IF_NONE 96 +#define POP_JUMP_IF_NOT_NONE 97 +#define POP_JUMP_IF_TRUE 98 +#define RAISE_VARARGS 99 +#define RERAISE 100 +#define SEND 101 +#define SET_ADD 102 +#define SET_FUNCTION_ATTRIBUTE 103 +#define SET_UPDATE 104 +#define STORE_ATTR 105 +#define STORE_DEREF 106 +#define STORE_FAST 107 +#define STORE_FAST_LOAD_FAST 108 +#define STORE_FAST_STORE_FAST 109 +#define STORE_GLOBAL 110 +#define STORE_NAME 111 +#define SWAP 112 +#define UNPACK_EX 113 +#define UNPACK_SEQUENCE 114 +#define YIELD_VALUE 115 #define RESUME 149 #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 @@ -144,95 +141,103 @@ extern "C" { #define BINARY_SUBSCR_TUPLE_INT 161 #define CALL_ALLOC_AND_ENTER_INIT 162 #define CALL_BOUND_METHOD_EXACT_ARGS 163 -#define CALL_BUILTIN_CLASS 164 -#define CALL_BUILTIN_FAST 165 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 166 -#define CALL_BUILTIN_O 167 -#define CALL_ISINSTANCE 168 -#define CALL_LEN 169 -#define CALL_LIST_APPEND 170 -#define CALL_METHOD_DESCRIPTOR_FAST 171 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 172 -#define CALL_METHOD_DESCRIPTOR_NOARGS 173 -#define CALL_METHOD_DESCRIPTOR_O 174 -#define CALL_PY_EXACT_ARGS 175 -#define CALL_PY_WITH_DEFAULTS 176 -#define CALL_STR_1 177 -#define CALL_TUPLE_1 178 -#define CALL_TYPE_1 179 -#define COMPARE_OP_FLOAT 180 -#define COMPARE_OP_INT 181 -#define COMPARE_OP_STR 182 -#define FOR_ITER_GEN 183 -#define FOR_ITER_LIST 184 -#define FOR_ITER_RANGE 185 -#define FOR_ITER_TUPLE 186 -#define LOAD_ATTR_CLASS 187 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 188 -#define LOAD_ATTR_INSTANCE_VALUE 189 -#define LOAD_ATTR_METHOD_LAZY_DICT 190 -#define LOAD_ATTR_METHOD_NO_DICT 191 -#define LOAD_ATTR_METHOD_WITH_VALUES 192 -#define LOAD_ATTR_MODULE 193 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 194 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 195 -#define LOAD_ATTR_PROPERTY 196 -#define LOAD_ATTR_SLOT 197 -#define LOAD_ATTR_WITH_HINT 198 -#define LOAD_GLOBAL_BUILTIN 199 -#define LOAD_GLOBAL_MODULE 200 -#define LOAD_SUPER_ATTR_ATTR 201 -#define LOAD_SUPER_ATTR_METHOD 202 -#define RESUME_CHECK 203 -#define SEND_GEN 204 -#define STORE_ATTR_INSTANCE_VALUE 205 -#define STORE_ATTR_SLOT 206 -#define STORE_ATTR_WITH_HINT 207 -#define STORE_SUBSCR_DICT 208 -#define STORE_SUBSCR_LIST_INT 209 -#define TO_BOOL_ALWAYS_TRUE 210 -#define TO_BOOL_BOOL 211 -#define TO_BOOL_INT 212 -#define TO_BOOL_LIST 213 -#define TO_BOOL_NONE 214 -#define TO_BOOL_STR 215 -#define UNPACK_SEQUENCE_LIST 216 -#define UNPACK_SEQUENCE_TUPLE 217 -#define UNPACK_SEQUENCE_TWO_TUPLE 218 -#define INSTRUMENTED_RESUME 236 +#define CALL_BOUND_METHOD_GENERAL 164 +#define CALL_BUILTIN_CLASS 165 +#define CALL_BUILTIN_FAST 166 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 +#define CALL_BUILTIN_O 168 +#define CALL_ISINSTANCE 169 +#define CALL_KW_BOUND_METHOD 170 +#define CALL_KW_NON_PY 171 +#define CALL_KW_PY 172 +#define CALL_LEN 173 +#define CALL_LIST_APPEND 174 +#define CALL_METHOD_DESCRIPTOR_FAST 175 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 176 +#define CALL_METHOD_DESCRIPTOR_NOARGS 177 +#define CALL_METHOD_DESCRIPTOR_O 178 +#define CALL_NON_PY_GENERAL 179 +#define CALL_PY_EXACT_ARGS 180 +#define CALL_PY_GENERAL 181 +#define CALL_STR_1 182 +#define CALL_TUPLE_1 183 +#define CALL_TYPE_1 184 +#define COMPARE_OP_FLOAT 185 +#define COMPARE_OP_INT 186 +#define COMPARE_OP_STR 187 +#define CONTAINS_OP_DICT 188 +#define CONTAINS_OP_SET 189 +#define FOR_ITER_GEN 190 +#define FOR_ITER_LIST 191 +#define FOR_ITER_RANGE 192 +#define FOR_ITER_TUPLE 193 +#define LOAD_ATTR_CLASS 194 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 195 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 196 +#define LOAD_ATTR_INSTANCE_VALUE 197 +#define LOAD_ATTR_METHOD_LAZY_DICT 198 +#define LOAD_ATTR_METHOD_NO_DICT 199 +#define LOAD_ATTR_METHOD_WITH_VALUES 200 +#define LOAD_ATTR_MODULE 201 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 202 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 203 +#define LOAD_ATTR_PROPERTY 204 +#define LOAD_ATTR_SLOT 205 +#define LOAD_ATTR_WITH_HINT 206 +#define LOAD_CONST_IMMORTAL 207 +#define LOAD_GLOBAL_BUILTIN 208 +#define LOAD_GLOBAL_MODULE 209 +#define LOAD_SUPER_ATTR_ATTR 210 +#define LOAD_SUPER_ATTR_METHOD 211 +#define RESUME_CHECK 212 +#define SEND_GEN 213 +#define STORE_ATTR_INSTANCE_VALUE 214 +#define STORE_ATTR_SLOT 215 +#define STORE_ATTR_WITH_HINT 216 +#define STORE_SUBSCR_DICT 217 +#define STORE_SUBSCR_LIST_INT 218 +#define TO_BOOL_ALWAYS_TRUE 219 +#define TO_BOOL_BOOL 220 +#define TO_BOOL_INT 221 +#define TO_BOOL_LIST 222 +#define TO_BOOL_NONE 223 +#define TO_BOOL_STR 224 +#define UNPACK_SEQUENCE_LIST 225 +#define UNPACK_SEQUENCE_TUPLE 226 +#define UNPACK_SEQUENCE_TWO_TUPLE 227 #define INSTRUMENTED_END_FOR 237 #define INSTRUMENTED_END_SEND 238 -#define INSTRUMENTED_RETURN_VALUE 239 -#define INSTRUMENTED_RETURN_CONST 240 -#define INSTRUMENTED_YIELD_VALUE 241 -#define INSTRUMENTED_LOAD_SUPER_ATTR 242 -#define INSTRUMENTED_FOR_ITER 243 -#define INSTRUMENTED_CALL 244 -#define INSTRUMENTED_CALL_KW 245 -#define INSTRUMENTED_CALL_FUNCTION_EX 246 -#define INSTRUMENTED_INSTRUCTION 247 -#define INSTRUMENTED_JUMP_FORWARD 248 -#define INSTRUMENTED_JUMP_BACKWARD 249 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 251 -#define INSTRUMENTED_POP_JUMP_IF_NONE 252 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 253 +#define INSTRUMENTED_LOAD_SUPER_ATTR 239 +#define INSTRUMENTED_FOR_ITER 240 +#define INSTRUMENTED_CALL_KW 241 +#define INSTRUMENTED_CALL_FUNCTION_EX 242 +#define INSTRUMENTED_INSTRUCTION 243 +#define INSTRUMENTED_JUMP_FORWARD 244 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 245 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 246 +#define INSTRUMENTED_POP_JUMP_IF_NONE 247 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 248 +#define INSTRUMENTED_RESUME 249 +#define INSTRUMENTED_RETURN_VALUE 250 +#define INSTRUMENTED_YIELD_VALUE 251 +#define INSTRUMENTED_CALL 252 +#define INSTRUMENTED_JUMP_BACKWARD 253 #define INSTRUMENTED_LINE 254 +#define ENTER_EXECUTOR 255 #define JUMP 256 -#define JUMP_NO_INTERRUPT 257 -#define LOAD_CLOSURE 258 -#define LOAD_METHOD 259 -#define LOAD_SUPER_METHOD 260 -#define LOAD_ZERO_SUPER_ATTR 261 -#define LOAD_ZERO_SUPER_METHOD 262 -#define POP_BLOCK 263 -#define SETUP_CLEANUP 264 -#define SETUP_FINALLY 265 -#define SETUP_WITH 266 -#define STORE_FAST_MAYBE_NULL 267 +#define JUMP_IF_FALSE 257 +#define JUMP_IF_TRUE 258 +#define JUMP_NO_INTERRUPT 259 +#define LOAD_CLOSURE 260 +#define POP_BLOCK 261 +#define SETUP_CLEANUP 262 +#define SETUP_FINALLY 263 +#define SETUP_WITH 264 +#define STORE_FAST_MAYBE_NULL 265 -#define HAVE_ARGUMENT 44 -#define MIN_INSTRUMENTED_OPCODE 236 +#define HAVE_ARGUMENT 41 +#define MIN_SPECIALIZED_OPCODE 150 +#define MIN_INSTRUMENTED_OPCODE 237 #ifdef __cplusplus } diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 5f9b720f8b1671..f7b33a8038570b 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -17,13 +17,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 13 +#define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 3 +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.13.0a3+" +#define PY_VERSION "3.14.0a1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/py_curses.h b/Include/py_curses.h index e46b08e9cc414e..e11bfedb17d205 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -23,23 +23,40 @@ # endif #endif -#if !defined(HAVE_CURSES_IS_PAD) && defined(WINDOW_HAS_FLAGS) -/* The following definition is necessary for ncurses 5.7; without it, - some of [n]curses.h set NCURSES_OPAQUE to 1, and then Python - can't get at the WINDOW flags field. */ +#if defined(WINDOW_HAS_FLAGS) && defined(__APPLE__) +/* gh-109617, gh-115383: we can rely on the default value for NCURSES_OPAQUE on + most platforms, but not on macOS. This is because, starting with Xcode 15, + Apple-provided ncurses.h comes from ncurses 6 (which defaults to opaque + structs) but can still be linked to older versions of ncurses dynamic + libraries which don't provide functions such as is_pad() to deal with opaque + structs. Setting NCURSES_OPAQUE to 0 is harmless in all ncurses releases to + this date (provided that a thread-safe implementation is not required), but + this might change in the future. This fix might become irrelevant once + support for macOS 13 or earlier is dropped. */ #define NCURSES_OPAQUE 0 #endif -#ifdef HAVE_NCURSES_H -#include -#else -#include +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include #endif -#ifdef HAVE_NCURSES_H +#ifdef NCURSES_VERSION /* configure was checking , but we will use , which has some or all these features. */ -#if !defined(WINDOW_HAS_FLAGS) && !(NCURSES_OPAQUE+0) +#if !defined(WINDOW_HAS_FLAGS) && \ + (NCURSES_VERSION_PATCH+0 < 20070303 || !(NCURSES_OPAQUE+0)) +/* the WINDOW flags field was always accessible in ncurses prior to 20070303; + after that, it depends on the value of NCURSES_OPAQUE. */ #define WINDOW_HAS_FLAGS 1 #endif #if !defined(HAVE_CURSES_IS_PAD) && NCURSES_VERSION_PATCH+0 >= 20090906 @@ -64,8 +81,6 @@ typedef struct { char *encoding; } PyCursesWindowObject; -#define PyCursesWindow_Check(v) Py_IS_TYPE((v), &PyCursesWindow_Type) - #define PyCurses_CAPSULE_NAME "_curses._C_API" @@ -82,6 +97,8 @@ static void **PyCurses_API; #define PyCursesInitialised {if (! ((int (*)(void))PyCurses_API[2]) () ) return NULL;} #define PyCursesInitialisedColor {if (! ((int (*)(void))PyCurses_API[3]) () ) return NULL;} +#define PyCursesWindow_Check(v) Py_IS_TYPE((v), &PyCursesWindow_Type) + #define import_curses() \ PyCurses_API = (void **)PyCapsule_Import(PyCurses_CAPSULE_NAME, 1); diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 68d7985dac8876..5d0028c116e2d8 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -108,7 +108,6 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError; PyAPI_DATA(PyObject *) PyExc_SyntaxError; PyAPI_DATA(PyObject *) PyExc_IndentationError; PyAPI_DATA(PyObject *) PyExc_TabError; -PyAPI_DATA(PyObject *) PyExc_IncompleteInputError; PyAPI_DATA(PyObject *) PyExc_ReferenceError; PyAPI_DATA(PyObject *) PyExc_SystemError; PyAPI_DATA(PyObject *) PyExc_SystemExit; diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 07020b5dc964cb..9824d099c3df7d 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -48,8 +48,10 @@ struct PyExpat_CAPI enum XML_Status (*SetEncoding)(XML_Parser parser, const XML_Char *encoding); int (*DefaultUnknownEncodingHandler)( void *encodingHandlerData, const XML_Char *name, XML_Encoding *info); - /* might be none for expat < 2.1.0 */ + /* might be NULL for expat < 2.1.0 */ int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); + /* might be NULL for expat < 2.6.0 */ + XML_Bool (*SetReparseDeferralEnabled)(XML_Parser parser, XML_Bool enabled); /* always add new stuff to the end! */ }; diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index c1e2bc5e323358..de1bcb1d2cb632 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -34,8 +34,12 @@ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv); /* In pathconfig.c */ +Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); + +Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); + Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); diff --git a/Include/pymacro.h b/Include/pymacro.h index cd6fc4eba9c2ed..e0378f9d27a048 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -15,11 +15,11 @@ // MSVC makes static_assert a keyword in C11-17, contrary to the standards. // // In C++11 and C2x, static_assert is a keyword, redefining is undefined -// behaviour. So only define if building as C (if __STDC_VERSION__ is defined), -// not C++, and only for C11-17. +// behaviour. So only define if building as C, not C++ (if __cplusplus is +// not defined), and only for C11-17. #if !defined(static_assert) && (defined(__GNUC__) || defined(__clang__)) \ - && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ - && __STDC_VERSION__ <= 201710L + && !defined(__cplusplus) && defined(__STDC_VERSION__) \ + && __STDC_VERSION__ >= 201112L && __STDC_VERSION__ <= 201710L # define static_assert _Static_assert #endif @@ -46,24 +46,42 @@ /* Argument must be a char or an int in [-128, 127] or [0, 255]. */ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) -/* Assert a build-time dependency, as an expression. - - Your compile will fail if the condition isn't true, or can't be evaluated - by the compiler. This can be used in an expression: its value is 0. - - Example: - - #define foo_to_char(foo) \ - ((char *)(foo) \ - + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) - - Written by Rusty Russell, public domain, http://ccodearchive.net/ */ -#define Py_BUILD_ASSERT_EXPR(cond) \ +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ + && !defined(__cplusplus) && !defined(_MSC_VER)) +# define Py_BUILD_ASSERT_EXPR(cond) \ + ((void)sizeof(struct { int dummy; _Static_assert(cond, #cond); }), \ + 0) +#else + /* Assert a build-time dependency, as an expression. + * + * Your compile will fail if the condition isn't true, or can't be evaluated + * by the compiler. This can be used in an expression: its value is 0. + * + * Example: + * + * #define foo_to_char(foo) \ + * ((char *)(foo) \ + * + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + * + * Written by Rusty Russell, public domain, http://ccodearchive.net/ + */ +# define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) +#endif -#define Py_BUILD_ASSERT(cond) do { \ - (void)Py_BUILD_ASSERT_EXPR(cond); \ - } while(0) +#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || (defined(__cplusplus) && __cplusplus >= 201103L)) + // Use static_assert() on C11 and newer +# define Py_BUILD_ASSERT(cond) \ + do { \ + static_assert((cond), #cond); \ + } while (0) +#else +# define Py_BUILD_ASSERT(cond) \ + do { \ + (void)Py_BUILD_ASSERT_EXPR(cond); \ + } while(0) +#endif /* Get the number of elements in a visible array diff --git a/Include/pymath.h b/Include/pymath.h index 4c1e3d9984894b..0ead1f95670fde 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -29,14 +29,17 @@ // Py_IS_NAN(X) // Return 1 if float or double arg is a NaN, else 0. +// Soft deprecated since Python 3.14, use isnan() instead. #define Py_IS_NAN(X) isnan(X) // Py_IS_INFINITY(X) // Return 1 if float or double arg is an infinity, else 0. +// Soft deprecated since Python 3.14, use isinf() instead. #define Py_IS_INFINITY(X) isinf(X) // Py_IS_FINITE(X) // Return 1 if float or double arg is neither infinite nor NAN, else 0. +// Soft deprecated since Python 3.14, use isfinite() instead. #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. @@ -46,7 +49,7 @@ /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. Prefer Py_INFINITY for new code. + * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL diff --git a/Include/pyport.h b/Include/pyport.h index 9d7ef0061806ad..2b6bd4c21110e5 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -9,6 +9,24 @@ #endif +// Preprocessor check for a builtin preprocessor function. Always return 0 +// if __has_builtin() macro is not defined. +// +// __has_builtin() is available on clang and GCC 10. +#ifdef __has_builtin +# define _Py__has_builtin(x) __has_builtin(x) +#else +# define _Py__has_builtin(x) 0 +#endif + +// Preprocessor check for a compiler __attribute__. Always return 0 +// if __has_attribute() macro is not defined. +#ifdef __has_attribute +# define _Py__has_attribute(x) __has_attribute(x) +#else +# define _Py__has_attribute(x) 0 +#endif + // Macro to use C++ static_cast<> in the Python C API. #ifdef __cplusplus # define _Py_STATIC_CAST(type, expr) static_cast(expr) @@ -180,6 +198,7 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_LOCAL_INLINE(type) static inline type #endif +// Soft deprecated since Python 3.14, use memcpy() instead. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_MEMCPY memcpy #endif @@ -531,16 +550,6 @@ extern "C" { #endif -// Preprocessor check for a builtin preprocessor function. Always return 0 -// if __has_builtin() macro is not defined. -// -// __has_builtin() is available on clang and GCC 10. -#ifdef __has_builtin -# define _Py__has_builtin(x) __has_builtin(x) -#else -# define _Py__has_builtin(x) 0 -#endif - // _Py_TYPEOF(expr) gets the type of an expression. // // Example: _Py_TYPEOF(x) x_copy = (x); @@ -572,6 +581,9 @@ extern "C" { # if defined(__SANITIZE_ADDRESS__) # define _Py_ADDRESS_SANITIZER # endif +# if defined(__SANITIZE_THREAD__) +# define _Py_THREAD_SANITIZER +# endif #endif @@ -603,4 +615,21 @@ extern "C" { # define _SGI_MP_SOURCE #endif +// Explicit fallthrough in switch case to avoid warnings +// with compiler flag -Wimplicit-fallthrough. +// +// Usage example: +// +// switch (value) { +// case 1: _Py_FALLTHROUGH; +// case 2: code; break; +// } +// +// __attribute__((fallthrough)) was introduced in GCC 7. +#if _Py__has_attribute(fallthrough) +# define _Py_FALLTHROUGH __attribute__((fallthrough)) +#else +# define _Py_FALLTHROUGH do { } while (0) +#endif + #endif /* Py_PYPORT_H */ diff --git a/Include/pystats.h b/Include/pystats.h index acfa32201711e0..a515570d1bb3bc 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -18,6 +18,8 @@ extern "C" { #else # define _Py_INCREF_STAT_INC() ((void)0) # define _Py_DECREF_STAT_INC() ((void)0) +# define _Py_INCREF_IMMORTAL_STAT_INC() ((void)0) +# define _Py_DECREF_IMMORTAL_STAT_INC() ((void)0) #endif // !Py_STATS #ifdef __cplusplus diff --git a/Include/pythread.h b/Include/pythread.h index a3216c51d66165..82247daf8e0aa0 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -17,7 +17,26 @@ typedef enum PyLockStatus { PyAPI_FUNC(void) PyThread_init_thread(void); PyAPI_FUNC(unsigned long) PyThread_start_new_thread(void (*)(void *), void *); -PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); +/* Terminates the current thread. Considered unsafe. + * + * WARNING: This function is only safe to call if all functions in the full call + * stack are written to safely allow it. Additionally, the behavior is + * platform-dependent. This function should be avoided, and is no longer called + * by Python itself. It is retained only for compatibility with existing C + * extension code. + * + * With pthreads, calls `pthread_exit` causes some libcs (glibc?) to attempt to + * unwind the stack and call C++ destructors; if a `noexcept` function is + * reached, they may terminate the process. Others (macOS) do unwinding. + * + * On Windows, calls `_endthreadex` which kills the thread without calling C++ + * destructors. + * + * In either case there is a risk of invalid references remaining to data on the + * thread stack. + */ +Py_DEPRECATED(3.14) PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); + PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void); #if (defined(__APPLE__) || defined(__linux__) || defined(_WIN32) \ diff --git a/Include/refcount.h b/Include/refcount.h new file mode 100644 index 00000000000000..141cbd34dd72e6 --- /dev/null +++ b/Include/refcount.h @@ -0,0 +1,517 @@ +#ifndef Py_REFCOUNT_H +#define Py_REFCOUNT_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* +Immortalization: + +The following indicates the immortalization strategy depending on the amount +of available bits in the reference count field. All strategies are backwards +compatible but the specific reference count value or immortalization check +might change depending on the specializations for the underlying system. + +Proper deallocation of immortal instances requires distinguishing between +statically allocated immortal instances vs those promoted by the runtime to be +immortal. The latter should be the only instances that require +cleanup during runtime finalization. +*/ + +#if SIZEOF_VOID_P > 4 +/* +In 64+ bit systems, any object whose 32 bit reference count is >= 2**31 +will be treated as immortal. + +Using the lower 32 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. + +In order to offer sufficient resilience to C extensions using the stable ABI +compiled against 3.11 or earlier, we set the initial value near the +middle of the range (2**31, 2**32). That way the the refcount can be +off by ~1 billion without affecting immortality. + +Reference count increases will use saturated arithmetic, taking advantage of +having all the lower 32 bits set, which will avoid the reference count to go +beyond the refcount limit. Immortality checks for reference count decreases will +be done by checking the bit sign flag in the lower 32 bits. + +*/ +#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3UL << 30)) + +#else +/* +In 32 bit systems, an object will be treated as immortal if its reference +count equals or exceeds _Py_IMMORTAL_MINIMUM_REFCNT (2**30). + +Using the lower 30 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases and decreases will first go through an immortality +check by comparing the reference count field to the minimum immortality refcount. +*/ +#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3L << 29)) +#define _Py_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(1L << 30)) +#endif + +// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is +// always 32-bits. +#ifdef Py_GIL_DISABLED +#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX +#endif + + +#ifdef Py_GIL_DISABLED + // The shared reference count uses the two least-significant bits to store + // flags. The remaining bits are used to store the reference count. +# define _Py_REF_SHARED_SHIFT 2 +# define _Py_REF_SHARED_FLAG_MASK 0x3 + + // The shared flags are initialized to zero. +# define _Py_REF_SHARED_INIT 0x0 +# define _Py_REF_MAYBE_WEAKREF 0x1 +# define _Py_REF_QUEUED 0x2 +# define _Py_REF_MERGED 0x3 + + // Create a shared field from a refcnt and desired flags +# define _Py_REF_SHARED(refcnt, flags) \ + (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) +#endif // Py_GIL_DISABLED + + +// Py_REFCNT() implementation for the stable ABI +PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 + // Stable ABI implements Py_REFCNT() as a function call + // on limited C API version 3.14 and newer. +#else + static inline Py_ssize_t _Py_REFCNT(PyObject *ob) { + #if !defined(Py_GIL_DISABLED) + return ob->ob_refcnt; + #else + uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return _Py_IMMORTAL_INITIAL_REFCNT; + } + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); + return _Py_STATIC_CAST(Py_ssize_t, local) + + Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); + #endif + } + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 + # define Py_REFCNT(ob) _Py_REFCNT(_PyObject_CAST(ob)) + #endif +#endif + + +static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) +{ +#if defined(Py_GIL_DISABLED) + return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == + _Py_IMMORTAL_REFCNT_LOCAL); +#elif SIZEOF_VOID_P > 4 + return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; +#else + return op->ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT; +#endif +} +#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) + + +// Py_SET_REFCNT() implementation for stable ABI +PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); + +static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 + // Stable ABI implements Py_SET_REFCNT() as a function call + // on limited C API version 3.13 and newer. + _Py_SetRefcnt(ob, refcnt); +#else + // This immortal check is for code that is unaware of immortal objects. + // The runtime tracks these objects and we should avoid as much + // as possible having extensions inadvertently change the refcnt + // of an immortalized object. + if (_Py_IsImmortal(ob)) { + return; + } + +#ifndef Py_GIL_DISABLED + ob->ob_refcnt = refcnt; +#else + if (_Py_IsOwnedByCurrentThread(ob)) { + if ((size_t)refcnt > (size_t)UINT32_MAX) { + // On overflow, make the object immortal + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + ob->ob_ref_shared = 0; + } + else { + // Set local refcount to desired refcount and shared refcount + // to zero, but preserve the shared refcount flags. + ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); + ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; + } + } + else { + // Set local refcount to zero and shared refcount to desired refcount. + // Mark the object as merged. + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = 0; + ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); + } +#endif // Py_GIL_DISABLED +#endif // Py_LIMITED_API+0 < 0x030d0000 +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) +#endif + + +/* +The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement +reference counts. Py_DECREF calls the object's deallocator function when +the refcount falls to 0; for +objects that don't contain references to other objects or heap memory +this can be the standard function free(). Both macros can be used +wherever a void expression is allowed. The argument must not be a +NULL pointer. If it may be NULL, use Py_XINCREF/Py_XDECREF instead. +The macro _Py_NewReference(op) initialize reference counts to 1, and +in special builds (Py_REF_DEBUG, Py_TRACE_REFS) performs additional +bookkeeping appropriate to the special build. + +We assume that the reference count field can never overflow; this can +be proven when the size of the field is the same as the pointer size, so +we ignore the possibility. Provided a C int is at least 32 bits (which +is implicitly assumed in many parts of this code), that's enough for +about 2**31 references to an object. + +XXX The following became out of date in Python 2.2, but I'm not sure +XXX what the full truth is now. Certainly, heap-allocated type objects +XXX can and should be deallocated. +Type objects should never be deallocated; the type pointer in an object +is not considered to be a reference to the type object, to save +complications in the deallocation function. (This is actually a +decision that's up to the implementer of each new type so if you want, +you can count such references to the type object.) +*/ + +#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) +PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, + PyObject *op); +PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void); +PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void); +#endif // Py_REF_DEBUG && !Py_LIMITED_API + +PyAPI_FUNC(void) _Py_Dealloc(PyObject *); + + +/* +These are provided as conveniences to Python runtime embedders, so that +they can have object code that is not dependent on Python compilation flags. +*/ +PyAPI_FUNC(void) Py_IncRef(PyObject *); +PyAPI_FUNC(void) Py_DecRef(PyObject *); + +// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL. +// Private functions used by Py_INCREF() and Py_DECREF(). +PyAPI_FUNC(void) _Py_IncRef(PyObject *); +PyAPI_FUNC(void) _Py_DecRef(PyObject *); + +static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) +{ +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) + // Stable ABI implements Py_INCREF() as a function call on limited C API + // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() + // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. + // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. +# if Py_LIMITED_API+0 >= 0x030a00A7 + _Py_IncRef(op); +# else + Py_IncRef(op); +# endif +#else + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. +#if defined(Py_GIL_DISABLED) + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + uint32_t new_local = local + 1; + if (new_local == 0) { + _Py_INCREF_IMMORTAL_STAT_INC(); + // local is equal to _Py_IMMORTAL_REFCNT_LOCAL: do nothing + return; + } + if (_Py_IsOwnedByCurrentThread(op)) { + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local); + } + else { + _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); + } +#elif SIZEOF_VOID_P > 4 + PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; + if (((int32_t)cur_refcnt) < 0) { + // the object is immortal + _Py_INCREF_IMMORTAL_STAT_INC(); + return; + } + op->ob_refcnt_split[PY_BIG_ENDIAN] = cur_refcnt + 1; +#else + if (_Py_IsImmortal(op)) { + _Py_INCREF_IMMORTAL_STAT_INC(); + return; + } + op->ob_refcnt++; +#endif + _Py_INCREF_STAT_INC(); +#ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +#endif +#endif +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) +#endif + + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +// Implements Py_DECREF on objects not owned by the current thread. +PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); +PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); + +// Called from Py_DECREF by the owning thread when the local refcount reaches +// zero. The call will deallocate the object if the shared refcount is also +// zero. Otherwise, the thread gives up ownership and merges the reference +// count fields. +PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); +#endif + +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +// Stable ABI implements Py_DECREF() as a function call on limited C API +// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was +// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. +// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. +static inline void Py_DECREF(PyObject *op) { +# if Py_LIMITED_API+0 >= 0x030a00A7 + _Py_DecRef(op); +# else + Py_DecRef(op); +# endif +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (_Py_IsOwnedByCurrentThread(op)) { + if (local == 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } + } + else { + _Py_DecRefSharedDebug(op, filename, lineno); + } +} +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +#elif defined(Py_GIL_DISABLED) +static inline void Py_DECREF(PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + _Py_DECREF_STAT_INC(); + if (_Py_IsOwnedByCurrentThread(op)) { + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } + } + else { + _Py_DecRefShared(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ + if (op->ob_refcnt <= 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (--op->ob_refcnt == 0) { + _Py_Dealloc(op); + } +} +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +#else +static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) +{ + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. + if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + _Py_DECREF_STAT_INC(); + if (--op->ob_refcnt == 0) { + _Py_Dealloc(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) +#endif + + +/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear + * and tp_dealloc implementations. + * + * Note that "the obvious" code can be deadly: + * + * Py_XDECREF(op); + * op = NULL; + * + * Typically, `op` is something like self->containee, and `self` is done + * using its `containee` member. In the code sequence above, suppose + * `containee` is non-NULL with a refcount of 1. Its refcount falls to + * 0 on the first line, which can trigger an arbitrary amount of code, + * possibly including finalizers (like __del__ methods or weakref callbacks) + * coded in Python, which in turn can release the GIL and allow other threads + * to run, etc. Such code may even invoke methods of `self` again, or cause + * cyclic gc to trigger, but-- oops! --self->containee still points to the + * object being torn down, and it may be in an insane state while being torn + * down. This has in fact been a rich historic source of miserable (rare & + * hard-to-diagnose) segfaulting (and other) bugs. + * + * The safe way is: + * + * Py_CLEAR(op); + * + * That arranges to set `op` to NULL _before_ decref'ing, so that any code + * triggered as a side-effect of `op` getting torn down no longer believes + * `op` points to a valid object. + * + * There are cases where it's safe to use the naive code, but they're brittle. + * For example, if `op` points to a Python integer, you know that destroying + * one of those can't cause problems -- but in part that relies on that + * Python integers aren't currently weakly referencable. Best practice is + * to use Py_CLEAR() even if you can't think of a reason for why you need to. + * + * gh-98724: Use a temporary variable to only evaluate the macro argument once, + * to avoid the duplication of side effects if the argument has side effects. + * + * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, + * the code can be miscompiled with strict aliasing because of type punning. + * With strict aliasing, a compiler considers that two pointers of different + * types cannot read or write the same memory which enables optimization + * opportunities. + * + * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, + * and so avoid type punning. Otherwise, use memcpy() which causes type erasure + * and so prevents the compiler to reuse an old cached 'op' value after + * Py_CLEAR(). + */ +#ifdef _Py_TYPEOF +#define Py_CLEAR(op) \ + do { \ + _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ + _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + *_tmp_op_ptr = _Py_NULL; \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#else +#define Py_CLEAR(op) \ + do { \ + PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ + PyObject *_tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + PyObject *_null_ptr = _Py_NULL; \ + memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#endif + + +/* Function to use in case the object pointer can be NULL: */ +static inline void Py_XINCREF(PyObject *op) +{ + if (op != _Py_NULL) { + Py_INCREF(op); + } +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) +#endif + +static inline void Py_XDECREF(PyObject *op) +{ + if (op != _Py_NULL) { + Py_DECREF(op); + } +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) +#endif + +// Create a new strong reference to an object: +// increment the reference count of the object and return the object. +PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); + +// Similar to Py_NewRef(), but the object can be NULL. +PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); + +static inline PyObject* _Py_NewRef(PyObject *obj) +{ + Py_INCREF(obj); + return obj; +} + +static inline PyObject* _Py_XNewRef(PyObject *obj) +{ + Py_XINCREF(obj); + return obj; +} + +// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. +// Names overridden with macros by static inline functions for best +// performances. +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) +# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) +#else +# define Py_NewRef(obj) _Py_NewRef(obj) +# define Py_XNewRef(obj) _Py_XNewRef(obj) +#endif + + +#ifdef __cplusplus +} +#endif +#endif // !Py_REFCOUNT_H diff --git a/Include/sliceobject.h b/Include/sliceobject.h index c13863f27c2e63..35e2ea254ca80a 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -8,7 +8,11 @@ extern "C" { PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ -#define Py_Ellipsis (&_Py_EllipsisObject) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_Ellipsis Py_GetConstantBorrowed(Py_CONSTANT_ELLIPSIS) +#else +# define Py_Ellipsis (&_Py_EllipsisObject) +#endif /* Slice object interface */ diff --git a/Include/structmember.h b/Include/structmember.h index f6e8fd829892f4..5f29fbcfed99e3 100644 --- a/Include/structmember.h +++ b/Include/structmember.h @@ -11,7 +11,7 @@ extern "C" { * New definitions are in descrobject.h. * * However, there's nothing wrong with old code continuing to use it, - * and there's not much mainenance overhead in maintaining a few aliases. + * and there's not much maintenance overhead in maintaining a few aliases. * So, don't be too eager to convert old code. * * It uses names not prefixed with Py_. diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 7b14f72ee2e494..c1d5f610fe08a5 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -7,6 +7,9 @@ extern "C" { PyAPI_FUNC(PyObject *) PySys_GetObject(const char *); PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *); +Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **); +Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int); + PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...) Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) @@ -18,23 +21,6 @@ Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 -PyAPI_FUNC(int) PySys_Audit( - const char *event, - const char *argFormat, - ...); - -PyAPI_FUNC(int) PySys_AuditTuple( - const char *event, - PyObject *args); -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_SYSMODULE_H -# include "cpython/sysmodule.h" -# undef Py_CPYTHON_SYSMODULE_H -#endif - #ifdef __cplusplus } #endif diff --git a/Include/typeslots.h b/Include/typeslots.h index 506b05580de146..a7f3017ec02e92 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -86,3 +86,11 @@ /* New in 3.10 */ #define Py_am_send 81 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 +/* New in 3.14 */ +#define Py_tp_vectorcall 82 +#endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 +/* New in 3.14 */ +#define Py_tp_token 83 +#endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index dee00715b3c51d..2ce3a008b7129e 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -966,6 +966,10 @@ PyAPI_FUNC(int) PyUnicode_EqualToUTF8(PyObject *, const char *); PyAPI_FUNC(int) PyUnicode_EqualToUTF8AndSize(PyObject *, const char *, Py_ssize_t); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +PyAPI_FUNC(int) PyUnicode_Equal(PyObject *str1, PyObject *str2); +#endif + /* Rich compare two strings and return one of the following: - NULL in case an exception was raised diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index 727ba6934bbacb..a6e71eb178b124 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -28,7 +28,10 @@ PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, PyObject *callback); Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 PyAPI_FUNC(int) PyWeakref_GetRef(PyObject *ref, PyObject **pobj); +#endif #ifndef Py_LIMITED_API diff --git a/InternalDocs/README.md b/InternalDocs/README.md new file mode 100644 index 00000000000000..2ef6e653ac19d4 --- /dev/null +++ b/InternalDocs/README.md @@ -0,0 +1,44 @@ + +# CPython Internals Documentation + +The documentation in this folder is intended for CPython maintainers. +It describes implementation details of CPython, which should not be +assumed to be part of the Python language specification. These details +can change between any two CPython versions and should not be assumed +to hold for other implementations of the Python language. + +The core dev team attempts to keep this documentation up to date. If +it is not, please report that through the +[issue tracker](https://github.com/python/cpython/issues). + + +Compiling Python Source Code +--- + +- [Guide to the parser](parser.md) + +- [Compiler Design](compiler.md) + +- [Changing Python's Grammar](changing_grammar.md) + +Runtime Objects +--- + +- [Code Objects (coming soon)](code_objects.md) + +- [The Source Code Locations Table](locations.md) + +- [Generators (coming soon)](generators.md) + +- [Frames](frames.md) + +Program Execution +--- + +- [The Interpreter](interpreter.md) + +- [Adaptive Instruction Families](adaptive.md) + +- [Garbage Collector Design](garbage_collector.md) + +- [Exception Handling](exception_handling.md) diff --git a/Python/adaptive.md b/InternalDocs/adaptive.md similarity index 94% rename from Python/adaptive.md rename to InternalDocs/adaptive.md index d978c089b237e0..4ae9e85b387f39 100644 --- a/Python/adaptive.md +++ b/InternalDocs/adaptive.md @@ -2,8 +2,9 @@ ## Families of instructions -The core part of PEP 659 (specializing adaptive interpreter) is the families -of instructions that perform the adaptive specialization. +The core part of [PEP 659](https://peps.python.org/pep-0659/) +(specializing adaptive interpreter) is the families of +instructions that perform the adaptive specialization. A family of instructions has the following fundamental properties: @@ -30,8 +31,8 @@ although these are not fundamental and may change: ## Example family -The `LOAD_GLOBAL` instruction (in Python/bytecodes.c) already has an adaptive -family that serves as a relatively simple example. +The `LOAD_GLOBAL` instruction (in [Python/bytecodes.c](../Python/bytecodes.c)) +already has an adaptive family that serves as a relatively simple example. The `LOAD_GLOBAL` instruction performs adaptive specialization, calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. diff --git a/InternalDocs/changing_grammar.md b/InternalDocs/changing_grammar.md new file mode 100644 index 00000000000000..1a5eebdc1418dc --- /dev/null +++ b/InternalDocs/changing_grammar.md @@ -0,0 +1,63 @@ +# Changing CPython's grammar + +There's more to changing Python's grammar than editing +[`Grammar/python.gram`](../Grammar/python.gram). +Below is a checklist of things that may need to change. + +> [!NOTE] +> +> Many of these changes require re-generating some of the derived +> files. If things mysteriously don't work, it may help to run +> ``make clean``. + +## Checklist + +* [`Grammar/python.gram`](../Grammar/python.gram): The grammar definition, + with actions that build AST nodes. + After changing it, run ``make regen-pegen`` (or ``build.bat --regen`` on Windows), + to regenerate [`Parser/parser.c`](../Parser/parser.c). + (This runs Python's parser generator, [`Tools/peg_generator`](../Tools/peg_generator)). + +* [`Grammar/Tokens`](../Grammar/Tokens) is a place for adding new token types. After + changing it, run ``make regen-token`` to regenerate + [`Include/internal/pycore_token.h`](../Include/internal/pycore_token.h), + [`Parser/token.c`](../Parser/token.c), [`Lib/token.py`](../Lib/token.py) + and [`Doc/library/token-list.inc`](../Doc/library/token-list.inc). + If you change both ``python.gram`` and ``Tokens``, run ``make regen-token`` + before ``make regen-pegen``. + On Windows, ``build.bat --regen`` will regenerate both at the same time. + +* [`Parser/Python.asdl`](../Parser/Python.asdl) may need changes to match the grammar. + Then run ``make regen-ast`` to regenerate + [`Include/internal/pycore_ast.h`](../Include/internal/pycore_ast.h) and + [`Python/Python-ast.c`](../Python/Python-ast.c). + +* [`Parser/lexer/`](../Parser/lexer/) contains the tokenization code. + This is where you would add a new type of comment or string literal, for example. + +* [`Python/ast.c`](../Python/ast.c) will need changes to validate AST objects + involved with the grammar change. + +* [`Python/ast_unparse.c`](../Python/ast_unparse.c) will need changes to unparse + AST involved with the grammar change ("unparsing" is used to turn annotations + into strings per [PEP 563](https://peps.python.org/pep-0563/). + +* The [`compiler`](compiler.md) may need to change when there are changes + to the `AST`. + +* ``_Unparser`` in the [`Lib/ast.py`](../Lib/ast.py) file may need changes + to accommodate any modifications in the AST nodes. + +* [`Doc/library/ast.rst`](../Doc/library/ast.rst) may need to be updated + to reflect changes to AST nodes. + +* Add some usage of your new syntax to ``test_grammar.py``. + +* Certain changes may require tweaks to the library module + [`pyclbr`](https://docs.python.org/3/library/pyclbr.html#module-pyclbr). + +* [`Lib/tokenize.py`](../Lib/tokenize.py) needs changes to match changes + to the tokenizer. + +* Documentation must be written! Specifically, one or more of the pages in + [`Doc/reference/`](../Doc/reference/) will need to be updated. diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md new file mode 100644 index 00000000000000..284a8b7aee5765 --- /dev/null +++ b/InternalDocs/code_objects.md @@ -0,0 +1,5 @@ + +Code objects +============ + +Coming soon. diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md new file mode 100644 index 00000000000000..0da4670c792cb5 --- /dev/null +++ b/InternalDocs/compiler.md @@ -0,0 +1,621 @@ + +Compiler design +=============== + +Abstract +-------- + +In CPython, the compilation from source code to bytecode involves several steps: + +1. Tokenize the source code [Parser/lexer/](../Parser/lexer/) + and [Parser/tokenizer/](../Parser/tokenizer/). +2. Parse the stream of tokens into an Abstract Syntax Tree + [Parser/parser.c](../Parser/parser.c). +3. Transform AST into an instruction sequence + [Python/compile.c](../Python/compile.c). +4. Construct a Control Flow Graph and apply optimizations to it + [Python/flowgraph.c](../Python/flowgraph.c). +5. Emit bytecode based on the Control Flow Graph + [Python/assemble.c](../Python/assemble.c). + +This document outlines how these steps of the process work. + +This document only describes parsing in enough depth to explain what is needed +for understanding compilation. This document provides a detailed, though not +exhaustive, view of the how the entire system works. You will most likely need +to read some source code to have an exact understanding of all details. + + +Parsing +======= + +As of Python 3.9, Python's parser is a PEG parser of a somewhat +unusual design. It is unusual in the sense that the parser's input is a stream +of tokens rather than a stream of characters which is more common with PEG +parsers. + +The grammar file for Python can be found in +[Grammar/python.gram](../Grammar/python.gram). +The definitions for literal tokens (such as `:`, numbers, etc.) can be found in +[Grammar/Tokens](../Grammar/Tokens). Various C files, including +[Parser/parser.c](../Parser/parser.c) are generated from these. + +See Also: + +* [Guide to the parser](https://devguide.python.org/internals/parser/index.html) + for a detailed description of the parser. + +* [Changing CPython’s grammar](https://devguide.python.org/developer-workflow/grammar/#grammar) + for a detailed description of the grammar. + + +Abstract syntax trees (AST) +=========================== + + +The abstract syntax tree (AST) is a high-level representation of the +program structure without the necessity of containing the source code; +it can be thought of as an abstract representation of the source code. The +specification of the AST nodes is specified using the Zephyr Abstract +Syntax Definition Language (ASDL) [^1], [^2]. + +The definition of the AST nodes for Python is found in the file +[Parser/Python.asdl](../Parser/Python.asdl). + +Each AST node (representing statements, expressions, and several +specialized types, like list comprehensions and exception handlers) is +defined by the ASDL. Most definitions in the AST correspond to a +particular source construct, such as an 'if' statement or an attribute +lookup. The definition is independent of its realization in any +particular programming language. + +The following fragment of the Python ASDL construct demonstrates the +approach and syntax: + +``` + module Python + { + stmt = FunctionDef(identifier name, arguments args, stmt* body, + expr* decorators) + | Return(expr? value) | Yield(expr? value) + attributes (int lineno) + } +``` + +The preceding example describes two different kinds of statements and an +expression: function definitions, return statements, and yield expressions. +All three kinds are considered of type `stmt` as shown by `|` separating +the various kinds. They all take arguments of various kinds and amounts. + +Modifiers on the argument type specify the number of values needed; `?` +means it is optional, `*` means 0 or more, while no modifier means only one +value for the argument and it is required. `FunctionDef`, for instance, +takes an `identifier` for the *name*, `arguments` for *args*, zero or more +`stmt` arguments for *body*, and zero or more `expr` arguments for +*decorators*. + +Do notice that something like 'arguments', which is a node type, is +represented as a single AST node and not as a sequence of nodes as with +stmt as one might expect. + +All three kinds also have an 'attributes' argument; this is shown by the +fact that 'attributes' lacks a '|' before it. + +The statement definitions above generate the following C structure type: + + +``` + typedef struct _stmt *stmt_ty; + + struct _stmt { + enum { FunctionDef_kind=1, Return_kind=2, Yield_kind=3 } kind; + union { + struct { + identifier name; + arguments_ty args; + asdl_seq *body; + } FunctionDef; + + struct { + expr_ty value; + } Return; + + struct { + expr_ty value; + } Yield; + } v; + int lineno; + } +``` + +Also generated are a series of constructor functions that allocate (in +this case) a `stmt_ty` struct with the appropriate initialization. The +`kind` field specifies which component of the union is initialized. The +`FunctionDef()` constructor function sets 'kind' to `FunctionDef_kind` and +initializes the *name*, *args*, *body*, and *attributes* fields. + +See also +[Green Tree Snakes - The missing Python AST docs](https://greentreesnakes.readthedocs.io/en/latest) + by Thomas Kluyver. + +Memory management +================= + +Before discussing the actual implementation of the compiler, a discussion of +how memory is handled is in order. To make memory management simple, an **arena** +is used that pools memory in a single location for easy +allocation and removal. This enables the removal of explicit memory +deallocation. Because memory allocation for all needed memory in the compiler +registers that memory with the arena, a single call to free the arena is all +that is needed to completely free all memory used by the compiler. + +In general, unless you are working on the critical core of the compiler, memory +management can be completely ignored. But if you are working at either the +very beginning of the compiler or the end, you need to care about how the arena +works. All code relating to the arena is in either +[Include/internal/pycore_pyarena.h](../Include/internal/pycore_pyarena.h) +or [Python/pyarena.c](../Python/pyarena.c). + +`PyArena_New()` will create a new arena. The returned `PyArena` structure +will store pointers to all memory given to it. This does the bookkeeping of +what memory needs to be freed when the compiler is finished with the memory it +used. That freeing is done with `PyArena_Free()`. This only needs to be +called in strategic areas where the compiler exits. + +As stated above, in general you should not have to worry about memory +management when working on the compiler. The technical details of memory +management have been designed to be hidden from you for most cases. + +The only exception comes about when managing a PyObject. Since the rest +of Python uses reference counting, there is extra support added +to the arena to cleanup each PyObject that was allocated. These cases +are very rare. However, if you've allocated a PyObject, you must tell +the arena about it by calling `PyArena_AddPyObject()`. + + +Source code to AST +================== + +The AST is generated from source code using the function +`_PyParser_ASTFromString()` or `_PyParser_ASTFromFile()` +[Parser/peg_api.c](../Parser/peg_api.c). + +After some checks, a helper function in +[Parser/parser.c](../Parser/parser.c) +begins applying production rules on the source code it receives; converting source +code to tokens and matching these tokens recursively to their corresponding rule. The +production rule's corresponding rule function is called on every match. These rule +functions follow the format `xx_rule`. Where *xx* is the grammar rule +that the function handles and is automatically derived from +[Grammar/python.gram](../Grammar/python.gram) by +[Tools/peg_generator/pegen/c_generator.py](../Tools/peg_generator/pegen/c_generator.py). + +Each rule function in turn creates an AST node as it goes along. It does this +by allocating all the new nodes it needs, calling the proper AST node creation +functions for any required supporting functions and connecting them as needed. +This continues until all nonterminal symbols are replaced with terminals. If an +error occurs, the rule functions backtrack and try another rule function. If +there are no more rules, an error is set and the parsing ends. + +The AST node creation helper functions have the name `_PyAST_{xx}` +where *xx* is the AST node that the function creates. These are defined by the +ASDL grammar and contained in [Python/Python-ast.c](../Python/Python-ast.c) +(which is generated by [Parser/asdl_c.py](../Parser/asdl_c.py) +from [Parser/Python.asdl](../Parser/Python.asdl)). +This all leads to a sequence of AST nodes stored in `asdl_seq` structs. + +To demonstrate everything explained so far, here's the +rule function responsible for a simple named import statement such as +`import sys`. Note that error-checking and debugging code has been +omitted. Removed parts are represented by `...`. +Furthermore, some comments have been added for explanation. These comments +may not be present in the actual code. + + +``` + // This is the production rule (from python.gram) the rule function + // corresponds to: + // import_name: 'import' dotted_as_names + static stmt_ty + import_name_rule(Parser *p) + { + ... + stmt_ty _res = NULL; + { // 'import' dotted_as_names + ... + Token * _keyword; + asdl_alias_seq* a; + // The tokenizing steps. + if ( + (_keyword = _PyPegen_expect_token(p, 513)) // token='import' + && + (a = dotted_as_names_rule(p)) // dotted_as_names + ) + { + ... + // Generate an AST for the import statement. + _res = _PyAST_Import ( a , ...); + ... + goto done; + } + ... + } + _res = NULL; + done: + ... + return _res; + } +``` + + +To improve backtracking performance, some rules (chosen by applying a +`(memo)` flag in the grammar file) are memoized. Each rule function checks if +a memoized version exists and returns that if so, else it continues in the +manner stated in the previous paragraphs. + +There are macros for creating and using `asdl_xx_seq *` types, where *xx* is +a type of the ASDL sequence. Three main types are defined +manually -- `generic`, `identifier` and `int`. These types are found in +[Python/asdl.c](../Python/asdl.c) and its corresponding header file +[Include/internal/pycore_asdl.h](../Include/internal/pycore_asdl.h). +Functions and macros for creating `asdl_xx_seq *` types are as follows: + +`_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)` + Allocate memory for an `asdl_generic_seq` of the specified length +`_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)` + Allocate memory for an `asdl_identifier_seq` of the specified length +`_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)` + Allocate memory for an `asdl_int_seq` of the specified length + +In addition to the three types mentioned above, some ASDL sequence types are +automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py) and found in +[Include/internal/pycore_ast.h](../Include/internal/pycore_ast.h). +Macros for using both manually defined and automatically generated ASDL +sequence types are as follows: + +`asdl_seq_GET(asdl_xx_seq *, int)` + Get item held at a specific position in an `asdl_xx_seq` +`asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)` + Set a specific index in an `asdl_xx_seq` to the specified value + +Untyped counterparts exist for some of the typed macros. These are useful +when a function needs to manipulate a generic ASDL sequence: + +`asdl_seq_GET_UNTYPED(asdl_seq *, int)` + Get item held at a specific position in an `asdl_seq` +`asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)` + Set a specific index in an `asdl_seq` to the specified value +`asdl_seq_LEN(asdl_seq *)` + Return the length of an `asdl_seq` or `asdl_xx_seq` + +Note that typed macros and functions are recommended over their untyped +counterparts. Typed macros carry out checks in debug mode and aid +debugging errors caused by incorrectly casting from `void *`. + +If you are working with statements, you must also worry about keeping +track of what line number generated the statement. Currently the line +number is passed as the last parameter to each `stmt_ty` function. + +See also [PEP 617: New PEG parser for CPython](https://peps.python.org/pep-0617/). + + +Control flow graphs +=================== + +A **control flow graph** (often referenced by its acronym, **CFG**) is a +directed graph that models the flow of a program. A node of a CFG is +not an individual bytecode instruction, but instead represents a +sequence of bytecode instructions that always execute sequentially. +Each node is called a *basic block* and must always execute from +start to finish, with a single entry point at the beginning and a +single exit point at the end. If some bytecode instruction *a* needs +to jump to some other bytecode instruction *b*, then *a* must occur at +the end of its basic block, and *b* must occur at the start of its +basic block. + +As an example, consider the following code snippet: + +```python +if x < 10: + f1() + f2() +else: + g() +end() +``` + +The `x < 10` guard is represented by its own basic block that +compares `x` with `10` and then ends in a conditional jump based on +the result of the comparison. This conditional jump allows the block +to point to both the body of the `if` and the body of the `else`. The +`if` basic block contains the `f1()` and `f2()` calls and points to +the `end()` basic block. The `else` basic block contains the `g()` +call and similarly points to the `end()` block. + +Note that more complex code in the guard, the `if` body, or the `else` +body may be represented by multiple basic blocks. For instance, +short-circuiting boolean logic in a guard like `if x or y:` +will produce one basic block that tests the truth value of `x` +and then points both (1) to the start of the `if` body and (2) to +a different basic block that tests the truth value of y. + +CFGs are useful as an intermediate representation of the code because +they are a convenient data structure for optimizations. + +AST to CFG to bytecode +====================== + +The conversion of an `AST` to bytecode is initiated by a call to the function +`_PyAST_Compile()` in [Python/compile.c](../Python/compile.c). + +The first step is to construct the symbol table. This is implemented by +`_PySymtable_Build()` in [Python/symtable.c](../Python/symtable.c). +This function begins by entering the starting code block for the AST (passed-in) +and then calling the proper `symtable_visit_{xx}` function (with *xx* being the +AST node type). Next, the AST tree is walked with the various code blocks that +delineate the reach of a local variable as blocks are entered and exited using +`symtable_enter_block()` and `symtable_exit_block()`, respectively. + +Once the symbol table is created, the `AST` is transformed by `compiler_codegen()` +in [Python/compile.c](../Python/compile.c) into a sequence of pseudo instructions. +These are similar to bytecode, but in some cases they are more abstract, and are +resolved later into actual bytecode. The construction of this instruction sequence +is handled by several functions that break the task down by various AST node types. +The functions are all named `compiler_visit_{xx}` where *xx* is the name of the node +type (such as `stmt`, `expr`, etc.). Each function receives a `struct compiler *` +and `{xx}_ty` where *xx* is the AST node type. Typically these functions +consist of a large 'switch' statement, branching based on the kind of +node type passed to it. Simple things are handled inline in the +'switch' statement with more complex transformations farmed out to other +functions named `compiler_{xx}` with *xx* being a descriptive name of what is +being handled. + +When transforming an arbitrary AST node, use the `VISIT()` macro. +The appropriate `compiler_visit_{xx}` function is called, based on the value +passed in for (so `VISIT({c}, expr, {node})` calls +`compiler_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, +but is called on AST node sequences (those values that were created as +arguments to a node that used the '*' modifier). + +Emission of bytecode is handled by the following macros: + +* `ADDOP(struct compiler *, location, int)` + add a specified opcode +* `ADDOP_IN_SCOPE(struct compiler *, location, int)` + like `ADDOP`, but also exits current scope; used for adding return value + opcodes in lambdas and closures +* `ADDOP_I(struct compiler *, location, int, Py_ssize_t)` + add an opcode that takes an integer argument +* `ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)` + add an opcode with the proper argument based on the position of the + specified PyObject in PyObject sequence object, but with no handling of + mangled names; used for when you + need to do named lookups of objects such as globals, consts, or + parameters where name mangling is not possible and the scope of the + name is known; *TYPE* is the name of PyObject sequence + (`names` or `varnames`) +* `ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)` + just like `ADDOP_O`, but steals a reference to PyObject +* `ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)` + just like `ADDOP_O`, but name mangling is also handled; used for + attribute loading or importing based on name +* `ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)` + add the `LOAD_CONST` opcode with the proper argument based on the + position of the specified PyObject in the consts table. +* `ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)` + just like `ADDOP_LOAD_CONST_NEW`, but steals a reference to PyObject +* `ADDOP_JUMP(struct compiler *, location, int, basicblock *)` + create a jump to a basic block + +The `location` argument is a struct with the source location to be +associated with this instruction. It is typically extracted from an +`AST` node with the `LOC` macro. The `NO_LOCATION` can be used +for *synthetic* instructions, which we do not associate with a line +number at this stage. For example, the implicit `return None` +which is added at the end of a function is not associated with any +line in the source code. + +There are several helper functions that will emit pseudo-instructions +and are named `compiler_{xx}()` where *xx* is what the function helps +with (`list`, `boolop`, etc.). A rather useful one is `compiler_nameop()`. +This function looks up the scope of a variable and, based on the +expression context, emits the proper opcode to load, store, or delete +the variable. + +Once the instruction sequence is created, it is transformed into a CFG +by `_PyCfg_FromInstructionSequence()`. Then `_PyCfg_OptimizeCodeUnit()` +applies various peephole optimizations, and +`_PyCfg_OptimizedCfgToInstructionSequence()` converts the optimized `CFG` +back into an instruction sequence. These conversions and optimizations are +implemented in [Python/flowgraph.c](../Python/flowgraph.c). + +Finally, the sequence of pseudo-instructions is converted into actual +bytecode. This includes transforming pseudo instructions into actual instructions, +converting jump targets from logical labels to relative offsets, and +construction of the [exception table](exception_handling.md) and +[locations table](locations.md). +The bytecode and tables are then wrapped into a `PyCodeObject` along with additional +metadata, including the `consts` and `names` arrays, information about function +reference to the source code (filename, etc). All of this is implemented by +`_PyAssemble_MakeCodeObject()` in [Python/assemble.c](../Python/assemble.c). + + +Code objects +============ + +The result of `PyAST_CompileObject()` is a `PyCodeObject` which is defined in +[Include/cpython/code.h](../Include/cpython/code.h). +And with that you now have executable Python bytecode! + +The code objects (byte code) are executed in [Python/ceval.c](../Python/ceval.c). +This file will also need a new case statement for the new opcode in the big switch +statement in `_PyEval_EvalFrameDefault()`. + + +Important files +=============== + +* [Parser/](../Parser/) + + * [Parser/Python.asdl](../Parser/Python.asdl): + ASDL syntax file. + + * [Parser/asdl.py](../Parser/asdl.py): + Parser for ASDL definition files. + Reads in an ASDL description and parses it into an AST that describes it. + + * [Parser/asdl_c.py](../Parser/asdl_c.py): + Generate C code from an ASDL description. Generates + [Python/Python-ast.c](../Python/Python-ast.c) and + [Include/internal/pycore_ast.h](../Include/internal/pycore_ast.h). + + * [Parser/parser.c](../Parser/parser.c): + The new PEG parser introduced in Python 3.9. Generated by + [Tools/peg_generator/pegen/c_generator.py](../Tools/peg_generator/pegen/c_generator.py) + from the grammar [Grammar/python.gram](../Grammar/python.gram). + Creates the AST from source code. Rule functions for their corresponding production + rules are found here. + + * [Parser/peg_api.c](../Parser/peg_api.c): + Contains high-level functions which are used by the interpreter to create + an AST from source code. + + * [Parser/pegen.c](../Parser/pegen.c): + Contains helper functions which are used by functions in + [Parser/parser.c](../Parser/parser.c) to construct the AST. Also contains + helper functions which help raise better error messages when parsing source code. + + * [Parser/pegen.h](../Parser/pegen.h): + Header file for the corresponding [Parser/pegen.c](../Parser/pegen.c). + Also contains definitions of the `Parser` and `Token` structs. + +* [Python/](../Python) + + * [Python/Python-ast.c](../Python/Python-ast.c): + Creates C structs corresponding to the ASDL types. Also contains code for + marshalling AST nodes (core ASDL types have marshalling code in + [Python/asdl.c](../Python/asdl.c)). + File automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py). + This file must be committed separately after every grammar change + is committed since the `__version__` value is set to the latest + grammar change revision number. + + * [Python/asdl.c](../Python/asdl.c): + Contains code to handle the ASDL sequence type. + Also has code to handle marshalling the core ASDL types, such as number + and identifier. Used by [Python/Python-ast.c](../Python/Python-ast.c) + for marshalling AST nodes. + + * [Python/ast.c](../Python/ast.c): + Used for validating the AST. + + * [Python/ast_opt.c](../Python/ast_opt.c): + Optimizes the AST. + + * [Python/ast_unparse.c](../Python/ast_unparse.c): + Converts the AST expression node back into a string (for string annotations). + + * [Python/ceval.c](../Python/ceval.c): + Executes byte code (aka, eval loop). + + * [Python/symtable.c](../Python/symtable.c): + Generates a symbol table from AST. + + * [Python/pyarena.c](../Python/pyarena.c): + Implementation of the arena memory manager. + + * [Python/compile.c](../Python/compile.c): + Emits pseudo bytecode based on the AST. + + * [Python/flowgraph.c](../Python/flowgraph.c): + Implements peephole optimizations. + + * [Python/assemble.c](../Python/assemble.c): + Constructs a code object from a sequence of pseudo instructions. + + * [Python/instruction_sequence.c](../Python/instruction_sequence.c): + A data structure representing a sequence of bytecode-like pseudo-instructions. + +* [Include/](../Include/) + + * [Include/cpython/code.h](../Include/cpython/code.h) + : Header file for [Objects/codeobject.c](../Objects/codeobject.c); + contains definition of `PyCodeObject`. + + * [Include/opcode.h](../Include/opcode.h) + : One of the files that must be modified whenever + [Lib/opcode.py](../Lib/opcode.py) is. + + * [Include/internal/pycore_ast.h](../Include/internal/pycore_ast.h) + : Contains the actual definitions of the C structs as generated by + [Python/Python-ast.c](../Python/Python-ast.c). + Automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py). + + * [Include/internal/pycore_asdl.h](../Include/internal/pycore_asdl.h) + : Header for the corresponding [Python/ast.c](../Python/ast.c). + + * [Include/internal/pycore_ast.h](../Include/internal/pycore_ast.h) + : Declares `_PyAST_Validate()` external (from [Python/ast.c](../Python/ast.c)). + + * [Include/internal/pycore_symtable.h](../Include/internal/pycore_symtable.h) + : Header for [Python/symtable.c](../Python/symtable.c). + `struct symtable` and `PySTEntryObject` are defined here. + + * [Include/internal/pycore_parser.h](../Include/internal/pycore_parser.h) + : Header for the corresponding [Parser/peg_api.c](../Parser/peg_api.c). + + * [Include/internal/pycore_pyarena.h](../Include/internal/pycore_pyarena.h) + : Header file for the corresponding [Python/pyarena.c](../Python/pyarena.c). + + * [Include/opcode_ids.h](../Include/opcode_ids.h) + : List of opcodes. Generated from [Python/bytecodes.c](../Python/bytecodes.c) + by + [Tools/cases_generator/opcode_id_generator.py](../Tools/cases_generator/opcode_id_generator.py). + +* [Objects/](../Objects/) + + * [Objects/codeobject.c](../Objects/codeobject.c) + : Contains PyCodeObject-related code. + + * [Objects/frameobject.c](../Objects/frameobject.c) + : Contains the `frame_setlineno()` function which should determine whether it is allowed + to make a jump between two points in a bytecode. + +* [Lib/](../Lib/) + + * [Lib/opcode.py](../Lib/opcode.py) + : opcode utilities exposed to Python. + + * [Include/core/pycore_magic_number.h](../Include/internal/pycore_magic_number.h) + : Home of the magic number (named `MAGIC_NUMBER`) for bytecode versioning. + + +Objects +======= + +* [Locations](locations.md): Describes the location table +* [Frames](frames.md): Describes frames and the frame stack +* [Objects/object_layout.md](../Objects/object_layout.md): Describes object layout for 3.11 and later +* [Exception Handling](exception_handling.md): Describes the exception table + + +Specializing Adaptive Interpreter +================================= + +Adding a specializing, adaptive interpreter to CPython will bring significant +performance improvements. These documents provide more information: + +* [PEP 659: Specializing Adaptive Interpreter](https://peps.python.org/pep-0659/). +* [Adding or extending a family of adaptive instructions](adaptive.md) + + +References +========== + +[^1]: Daniel C. Wang, Andrew W. Appel, Jeff L. Korn, and Chris + S. Serra. `The Zephyr Abstract Syntax Description Language.`_ + In Proceedings of the Conference on Domain-Specific Languages, + pp. 213--227, 1997. + +[^2]: The Zephyr Abstract Syntax Description Language.: + https://www.cs.princeton.edu/research/techreps/TR-554-97 diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md new file mode 100644 index 00000000000000..14066a5864b4da --- /dev/null +++ b/InternalDocs/exception_handling.md @@ -0,0 +1,191 @@ +Description of exception handling +--------------------------------- + +Python uses a technique known as "zero-cost" exception handling, which +minimizes the cost of supporting exceptions. In the common case (where +no exception is raised) the cost is reduced to zero (or close to zero). +The cost of raising an exception is increased, but not by much. + +The following code: + +``` +try: + g(0) +except: + res = "fail" + +``` + +compiles into intermediate code like the following: + +``` + RESUME 0 + + 1 SETUP_FINALLY 8 (to L1) + + 2 LOAD_NAME 0 (g) + PUSH_NULL + LOAD_CONST 0 (0) + CALL 1 + POP_TOP + POP_BLOCK + + -- L1: PUSH_EXC_INFO + + 3 POP_TOP + + 4 LOAD_CONST 1 ('fail') + STORE_NAME 1 (res) +``` + +`SETUP_FINALLY` and `POP_BLOCK` are pseudo-instructions. This means +that they can appear in intermediate code but they are not bytecode +instructions. `SETUP_FINALLY` specifies that henceforth, exceptions +are handled by the code at label L1. The `POP_BLOCK` instruction +reverses the effect of the last `SETUP` instruction, so that the +active exception handler reverts to what it was before. + +`SETUP_FINALLY` and `POP_BLOCK` have no effect when no exceptions +are raised. The idea of zero-cost exception handling is to replace +these pseudo-instructions by metadata which is stored alongside the +bytecode, and which is inspected only when an exception occurs. +This metadata is the exception table, and it is stored in the code +object's `co_exceptiontable` field. + +When the pseudo-instructions are translated into bytecode, +`SETUP_FINALLY` and `POP_BLOCK` are removed, and the exception +table is constructed, mapping each instruction to the exception +handler that covers it, if any. Instructions which are not +covered by any exception handler within the same code object's +bytecode, do not appear in the exception table at all. + +For the code object in our example above, the table has a single +entry specifying that all instructions that were between the +`SETUP_FINALLY` and the `POP_BLOCK` are covered by the exception +handler located at label `L1`. + +Handling Exceptions +------------------- + +At runtime, when an exception occurs, the interpreter calls +`get_exception_handler()` in [Python/ceval.c](../Python/ceval.c) +to look up the offset of the current instruction in the exception +table. If it finds a handler, control flow transfers to it. Otherwise, the +exception bubbles up to the caller, and the caller's frame is +checked for a handler covering the `CALL` instruction. This +repeats until a handler is found or the topmost frame is reached. +If no handler is found, then the interpreter function +(`_PyEval_EvalFrameDefault()`) returns NULL. During unwinding, +the traceback is constructed as each frame is added to it by +`PyTraceBack_Here()`, which is in [Python/traceback.c](../Python/traceback.c). + +Along with the location of an exception handler, each entry of the +exception table also contains the stack depth of the `try` instruction +and a boolean `lasti` value, which indicates whether the instruction +offset of the raising instruction should be pushed to the stack. + +Handling an exception, once an exception table entry is found, consists +of the following steps: + + 1. pop values from the stack until it matches the stack depth for the handler. + 2. if `lasti` is true, then push the offset that the exception was raised at. + 3. push the exception to the stack. + 4. jump to the target offset and resume execution. + + +Reraising Exceptions and `lasti` +-------------------------------- + +The purpose of pushing `lasti` to the stack is for cases where an exception +needs to be re-raised, and be associated with the original instruction that +raised it. This happens, for example, at the end of a `finally` block, when +any in-flight exception needs to be propagated on. As the frame's instruction +pointer now points into the finally block, a `RERAISE` instruction +(with `oparg > 0`) sets it to the `lasti` value from the stack. + +Format of the exception table +----------------------------- + +Conceptually, the exception table consists of a sequence of 5-tuples: +``` + 1. `start-offset` (inclusive) + 2. `end-offset` (exclusive) + 3. `target` + 4. `stack-depth` + 5. `push-lasti` (boolean) +``` + +All offsets and lengths are in code units, not bytes. + +We want the format to be compact, but quickly searchable. +For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. +For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. +Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. + +It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: + `start, size, target, depth, push-lasti`. + +Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each code unit takes 2 bytes. +It also happens that depth is generally quite small. + +So, we need to encode: +``` + `start` (up to 30 bits) + `size` (up to 30 bits) + `target` (up to 30 bits) + `depth` (up to ~8 bits) + `lasti` (1 bit) +``` + +We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. +Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. +Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. +The 8 bits of a byte are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. + +In addition, we combine `depth` and `lasti` into a single value, `((depth<<1)+lasti)`, before encoding. + +For example, the exception entry: +``` + `start`: 20 + `end`: 28 + `target`: 100 + `depth`: 3 + `lasti`: False +``` + +is encoded by first converting to the more compact four value form: +``` + `start`: 20 + `size`: 8 + `target`: 100 + `depth<<1+lasti`: 6 +``` + +which is then encoded as: +``` + 148 (MSB + 20 for start) + 8 (size) + 65 (Extend bit + 1) + 36 (Remainder of target, 100 == (1<<6)+36) + 6 +``` + +for a total of five bytes. + +The code to construct the exception table is in `assemble_exception_table()` +in [Python/assemble.c](../Python/assemble.c). + +The interpreter's function to lookup the table by instruction offset is +`get_exception_handler()` in [Python/ceval.c](../Python/ceval.c). +The Python function `_parse_exception_table()` in [Lib/dis.py](../Lib/dis.py) +returns the exception table content as a list of namedtuple instances. + +Exception Chaining Implementation +--------------------------------- + +[Exception chaining](https://docs.python.org/dev/tutorial/errors.html#exception-chaining) +refers to setting the `__context__` and `__cause__` fields of an exception as it is +being raised. The `__context__` field is set by `_PyErr_SetObject()` in +[Python/errors.c](../Python/errors.c) (which is ultimately called by all +`PyErr_Set*()` functions). The `__cause__` field (explicit chaining) is set by +the `RAISE_VARARGS` bytecode. diff --git a/Objects/frame_layout.md b/InternalDocs/frames.md similarity index 64% rename from Objects/frame_layout.md rename to InternalDocs/frames.md index b348e85689f507..06dc8f0702c3d9 100644 --- a/Objects/frame_layout.md +++ b/InternalDocs/frames.md @@ -1,51 +1,46 @@ -# The Frame Stack - -Each call to a Python function has an activation record, -commonly known as a "frame". -Python semantics allows frames to outlive the activation, -so they have (before 3.11) been allocated on the heap. -This is expensive as it requires many allocations and -results in poor locality of reference. - -In 3.11, rather than have these frames scattered about memory, -as happens for heap-allocated objects, frames are allocated -contiguously in a per-thread stack. -This improves performance significantly for two reasons: -* It reduces allocation overhead to a pointer comparison and increment. -* Stack allocated data has the best possible locality and will always be in - CPU cache. - -Generator and coroutines still need heap allocated activation records, but -can be linked into the per-thread stack so as to not impact performance too much. +# Frames -## Layout - -Each activation record consists of four conceptual sections: +Each call to a Python function has an activation record, commonly known as a +"frame". It contains information about the function being executed, consisting +of three conceptual sections: * Local variables (including arguments, cells and free variables) * Evaluation stack -* Specials: The per-frame object references needed by the VM: globals dict, - code object, etc. -* Linkage: Pointer to the previous activation record, stack depth, etc. +* Specials: The per-frame object references needed by the VM, including + globals dict, code object, instruction pointer, stack depth, the + previous frame, etc. + +The definition of the `_PyInterpreterFrame` struct is in +[Include/internal/pycore_frame.h](../Include/internal/pycore_frame.h). -### Layout +# Allocation -The specials and linkage sections are a fixed size, so are grouped together. +Python semantics allows frames to outlive the activation, so they need to +be allocated outside the C call stack. To reduce overhead and improve locality +of reference, most frames are allocated contiguously in a per-thread stack +(see `_PyThreadState_PushFrame` in [Python/pystate.c](../Python/pystate.c)). + +Frames of generators and coroutines are embedded in the generator and coroutine +objects, so are not allocated in the per-thread stack. See `PyGenObject` in +[Include/internal/pycore_genobject.h](../Include/internal/pycore_genobject.h). + +## Layout Each activation record is laid out as: -* Specials and linkage +* Specials * Locals * Stack This seems to provide the best performance without excessive complexity. -It needs the interpreter to hold two pointers, a frame pointer and a stack pointer. +The specials have a fixed size, so the offset of the locals is know. The +interpreter needs to hold two pointers, a frame pointer and a stack pointer. #### Alternative layout An alternative layout that was used for part of 3.11 alpha was: * Locals -* Specials and linkage +* Specials * Stack This has the advantage that no copying is required when making a call, @@ -53,19 +48,6 @@ as the arguments on the stack are (usually) already in the correct location for the parameters. However, it requires the VM to maintain an extra pointer for the locals, which can hurt performance. -A variant that only needs the need two pointers is to reverse the numbering -of the locals, so that the last one is numbered `0`, and the first in memory -is numbered `N-1`. -This allows the locals, specials and linkage to accessed from the frame pointer. -We may implement this in the future. - -#### Note: - -> In a contiguous stack, we would need to save one fewer registers, as the -> top of the caller's activation record would be the same at the base of the -> callee's. However, since some activation records are kept on the heap we -> cannot do this. - ### Generators and Coroutines Generators and coroutines contain a `_PyInterpreterFrame` @@ -92,25 +74,22 @@ section. The `frame_obj` field is initially `NULL`. The `PyFrameObject` may outlive a stack-allocated `_PyInterpreterFrame`. If it does then `_PyInterpreterFrame` is copied into the `PyFrameObject`, except the evaluation stack which must be empty at this point. -The linkage section is updated to reflect the new location of the frame. +The previous frame link is updated to reflect the new location of the frame. This mechanism provides the appearance of persistent, heap-allocated frames for each activation, but with low runtime overhead. ### Generators and Coroutines - -Generator objects have a `_PyInterpreterFrame` embedded in them. -This means that creating a generator requires only a single allocation, -reducing allocation overhead and improving locality of reference. -The embedded frame is linked into the per-thread frame when iterated or -awaited. +Generators (objects of type `PyGen_Type`, `PyCoro_Type` or +`PyAsyncGen_Type`) have a `_PyInterpreterFrame` embedded in them, so +that they can be created with a single memory allocation. +When such an embedded frame is iterated or awaited, it can be linked with +frames on the per-thread stack via the linkage fields. If a frame object associated with a generator outlives the generator, then -the embedded `_PyInterpreterFrame` is copied into the frame object. - - -All the above applies to coroutines and async generators as well. +the embedded `_PyInterpreterFrame` is copied into the frame object (see +`take_ownership()` in [Python/frame.c](../Python/frame.c)). ### Field names @@ -119,7 +98,7 @@ Thus, some of the field names may be a bit misleading. For example the `f_globals` field has a `f_` prefix implying it belongs to the `PyFrameObject` struct, although it belongs to the `_PyInterpreterFrame` struct. -We may rationalize this naming scheme for 3.12. +We may rationalize this naming scheme for a later version. ### Shim frames diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md new file mode 100644 index 00000000000000..d624cf4befd31a --- /dev/null +++ b/InternalDocs/garbage_collector.md @@ -0,0 +1,596 @@ + +Garbage collector design +======================== + +Abstract +======== + +The main garbage collection algorithm used by CPython is reference counting. The basic idea is +that CPython counts how many different places there are that have a reference to an +object. Such a place could be another object, or a global (or static) C variable, or +a local variable in some C function. When an object’s reference count becomes zero, +the object is deallocated. If it contains references to other objects, their +reference counts are decremented. Those other objects may be deallocated in turn, if +this decrement makes their reference count become zero, and so on. The reference +count field can be examined using the `sys.getrefcount()` function (notice that the +value returned by this function is always 1 more as the function also has a reference +to the object when called): + +```pycon + >>> x = object() + >>> sys.getrefcount(x) + 2 + >>> y = x + >>> sys.getrefcount(x) + 3 + >>> del y + >>> sys.getrefcount(x) + 2 +``` + +The main problem with the reference counting scheme is that it does not handle reference +cycles. For instance, consider this code: + +```pycon + >>> container = [] + >>> container.append(container) + >>> sys.getrefcount(container) + 3 + >>> del container +``` + +In this example, `container` holds a reference to itself, so even when we remove +our reference to it (the variable "container") the reference count never falls to 0 +because it still has its own internal reference. Therefore it would never be +cleaned just by simple reference counting. For this reason some additional machinery +is needed to clean these reference cycles between objects once they become +unreachable. This is the cyclic garbage collector, usually called just Garbage +Collector (GC), even though reference counting is also a form of garbage collection. + +Starting in version 3.13, CPython contains two GC implementations: + +- The default build implementation relies on the + [global interpreter lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) + for thread safety. +- The free-threaded build implementation pauses other executing threads when + performing a collection for thread safety. + +Both implementations use the same basic algorithms, but operate on different +data structures. See the section on +[Differences between GC implementations](#Differences-between-GC-implementations) +for the details. + + +Memory layout and object structure +================================== + +The garbage collector requires additional fields in Python objects to support +garbage collection. These extra fields are different in the default and the +free-threaded builds. + + +GC for the default build +------------------------ + +Normally the C structure supporting a regular Python object looks as follows: + +``` + object -----> +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ \ + | ob_refcnt | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyObject_HEAD + | *ob_type | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ... | +``` + +In order to support the garbage collector, the memory layout of objects is altered +to accommodate extra information **before** the normal layout: + +``` + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ \ + | *_gc_next | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyGC_Head + | *_gc_prev | | + object -----> +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ob_refcnt | \ + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyObject_HEAD + | *ob_type | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ... | +``` + + +In this way the object can be treated as a normal python object and when the extra +information associated to the GC is needed the previous fields can be accessed by a +simple type cast from the original object: `((PyGC_Head *)(the_object)-1)`. + +As is explained later in the +[Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) +section, these two extra fields are normally used to keep doubly linked lists of all the +objects tracked by the garbage collector (these lists are the GC generations, more on +that in the [Optimization: generations](#Optimization-generations) section), but +they are also reused to fulfill other purposes when the full doubly linked list +structure is not needed as a memory optimization. + +Doubly linked lists are used because they efficiently support the most frequently required operations. In +general, the collection of all objects tracked by GC is partitioned into disjoint sets, each in its own +doubly linked list. Between collections, objects are partitioned into "generations", reflecting how +often they've survived collection attempts. During collections, the generation(s) being collected +are further partitioned into, for example, sets of reachable and unreachable objects. Doubly linked lists +support moving an object from one partition to another, adding a new object, removing an object +entirely (objects tracked by GC are most often reclaimed by the refcounting system when GC +isn't running at all!), and merging partitions, all with a small constant number of pointer updates. +With care, they also support iterating over a partition while objects are being added to - and +removed from - it, which is frequently required while GC is running. + +GC for the free-threaded build +------------------------------ + +In the free-threaded build, Python objects contain a 1-byte field +`ob_gc_bits` that is used to track garbage collection related state. The +field exists in all objects, including ones that do not support cyclic +garbage collection. The field is used to identify objects that are tracked +by the collector, ensure that finalizers are called only once per object, +and, during garbage collection, differentiate reachable vs. unreachable objects. + +``` + object -----> +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ \ + | ob_tid | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | + | pad | ob_mutex | ob_gc_bits | ob_ref_local | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyObject_HEAD + | ob_ref_shared | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | + | *ob_type | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ... | +``` + +Note that not all fields are to scale. `pad` is two bytes, `ob_mutex` and +`ob_gc_bits` are each one byte, and `ob_ref_local` is four bytes. The +other fields, `ob_tid`, `ob_ref_shared`, and `ob_type`, are all +pointer-sized (that is, eight bytes on a 64-bit platform). + + +The garbage collector also temporarily repurposes the `ob_tid` (thread ID) +and `ob_ref_local` (local reference count) fields for other purposes during +collections. + + +C APIs +------ + +Specific APIs are offered to allocate, deallocate, initialize, track, and untrack +objects with GC support. These APIs can be found in the +[Garbage Collector C API documentation](https://docs.python.org/3/c-api/gcsupport.html). + +Apart from this object structure, the type object for objects supporting garbage +collection must include the `Py_TPFLAGS_HAVE_GC` in its `tp_flags` slot and +provide an implementation of the `tp_traverse` handler. Unless it can be proven +that the objects cannot form reference cycles with only objects of its type or unless +the type is immutable, a `tp_clear` implementation must also be provided. + + +Identifying reference cycles +============================ + +The algorithm that CPython uses to detect those reference cycles is +implemented in the `gc` module. The garbage collector **only focuses** +on cleaning container objects (that is, objects that can contain a reference +to one or more objects). These can be arrays, dictionaries, lists, custom +class instances, classes in extension modules, etc. One could think that +cycles are uncommon but the truth is that many internal references needed by +the interpreter create cycles everywhere. Some notable examples: + +- Exceptions contain traceback objects that contain a list of frames that + contain the exception itself. +- Module-level functions reference the module's dict (which is needed to resolve globals), + which in turn contains entries for the module-level functions. +- Instances have references to their class which itself references its module, and the module + contains references to everything that is inside (and maybe other modules) + and this can lead back to the original instance. +- When representing data structures like graphs, it is very typical for them to + have internal links to themselves. + +To correctly dispose of these objects once they become unreachable, they need +to be identified first. To understand how the algorithm works, let’s take +the case of a circular linked list which has one link referenced by a +variable `A`, and one self-referencing object which is completely +unreachable: + +```pycon + >>> import gc + + >>> class Link: + ... def __init__(self, next_link=None): + ... self.next_link = next_link + + >>> link_3 = Link() + >>> link_2 = Link(link_3) + >>> link_1 = Link(link_2) + >>> link_3.next_link = link_1 + >>> A = link_1 + >>> del link_1, link_2, link_3 + + >>> link_4 = Link() + >>> link_4.next_link = link_4 + >>> del link_4 + + # Collect the unreachable Link object (and its .__dict__ dict). + >>> gc.collect() + 2 +``` + +The GC starts with a set of candidate objects it wants to scan. In the +default build, these "objects to scan" might be all container objects or a +smaller subset (or "generation"). In the free-threaded build, the collector +always scans all container objects. + +The objective is to identify all the unreachable objects. The collector does +this by identifying reachable objects; the remaining objects must be +unreachable. The first step is to identify all of the "to scan" objects that +are **directly** reachable from outside the set of candidate objects. These +objects have a refcount larger than the number of incoming references from +within the candidate set. + +Every object that supports garbage collection will have an extra reference +count field initialized to the reference count (`gc_ref` in the figures) +of that object when the algorithm starts. This is because the algorithm needs +to modify the reference count to do the computations and in this way the +interpreter will not modify the real reference count field. + +![gc-image1](images/python-cyclic-gc-1-new-page.png) + +The GC then iterates over all containers in the first list and decrements by one the +`gc_ref` field of any other object that container is referencing. Doing +this makes use of the `tp_traverse` slot in the container class (implemented +using the C API or inherited by a superclass) to know what objects are referenced by +each container. After all the objects have been scanned, only the objects that have +references from outside the “objects to scan” list will have `gc_ref > 0`. + +![gc-image2](images/python-cyclic-gc-2-new-page.png) + +Notice that having `gc_ref == 0` does not imply that the object is unreachable. +This is because another object that is reachable from the outside (`gc_ref > 0`) +can still have references to it. For instance, the `link_2` object in our example +ended having `gc_ref == 0` but is referenced still by the `link_1` object that +is reachable from the outside. To obtain the set of objects that are really +unreachable, the garbage collector re-scans the container objects using the +`tp_traverse` slot; this time with a different traverse function that marks objects with +`gc_ref == 0` as "tentatively unreachable" and then moves them to the +tentatively unreachable list. The following image depicts the state of the lists in a +moment when the GC processed the `link_3` and `link_4` objects but has not +processed `link_1` and `link_2` yet. + +![gc-image3](images/python-cyclic-gc-3-new-page.png) + +Then the GC scans the next `link_1` object. Because it has `gc_ref == 1`, +the gc does not do anything special because it knows it has to be reachable (and is +already in what will become the reachable list): + +![gc-image4](images/python-cyclic-gc-4-new-page.png) + +When the GC encounters an object which is reachable (`gc_ref > 0`), it traverses +its references using the `tp_traverse` slot to find all the objects that are +reachable from it, moving them to the end of the list of reachable objects (where +they started originally) and setting its `gc_ref` field to 1. This is what happens +to `link_2` and `link_3` below as they are reachable from `link_1`. From the +state in the previous image and after examining the objects referred to by `link_1` +the GC knows that `link_3` is reachable after all, so it is moved back to the +original list and its `gc_ref` field is set to 1 so that if the GC visits it again, +it will know that it's reachable. To avoid visiting an object twice, the GC marks all +objects that have already been visited once (by unsetting the `PREV_MASK_COLLECTING` +flag) so that if an object that has already been processed is referenced by some other +object, the GC does not process it twice. + +![gc-image5](images/python-cyclic-gc-5-new-page.png) + +Notice that an object that was marked as "tentatively unreachable" and was later +moved back to the reachable list will be visited again by the garbage collector +as now all the references that that object has need to be processed as well. This +process is really a breadth first search over the object graph. Once all the objects +are scanned, the GC knows that all container objects in the tentatively unreachable +list are really unreachable and can thus be garbage collected. + +Pragmatically, it's important to note that no recursion is required by any of this, +and neither does it in any other way require additional memory proportional to the +number of objects, number of pointers, or the lengths of pointer chains. Apart from +`O(1)` storage for internal C needs, the objects themselves contain all the storage +the GC algorithms require. + +Why moving unreachable objects is better +---------------------------------------- + +It sounds logical to move the unreachable objects under the premise that most objects +are usually reachable, until you think about it: the reason it pays isn't actually +obvious. + +Suppose we create objects A, B, C in that order. They appear in the young generation +in the same order. If B points to A, and C to B, and C is reachable from outside, +then the adjusted refcounts after the first step of the algorithm runs will be 0, 0, +and 1 respectively because the only reachable object from the outside is C. + +When the next step of the algorithm finds A, A is moved to the unreachable list. The +same for B when it's first encountered. Then C is traversed, B is moved *back* to +the reachable list. B is eventually traversed, and then A is moved back to the reachable +list. + +So instead of not moving at all, the reachable objects B and A are each moved twice. +Why is this a win? A straightforward algorithm to move the reachable objects instead +would move A, B, and C once each. The key is that this dance leaves the objects in +order C, B, A - it's reversed from the original order. On all *subsequent* scans, +none of them will move. Since most objects aren't in cycles, this can save an +unbounded number of moves across an unbounded number of later collections. The only +time the cost can be higher is the first time the chain is scanned. + +Destroying unreachable objects +============================== + +Once the GC knows the list of unreachable objects, a very delicate process starts +with the objective of completely destroying these objects. Roughly, the process +follows these steps in order: + +1. Handle and clear weak references (if any). Weak references to unreachable objects + are set to `None`. If the weak reference has an associated callback, the callback + is enqueued to be called once the clearing of weak references is finished. We only + invoke callbacks for weak references that are themselves reachable. If both the weak + reference and the pointed-to object are unreachable we do not execute the callback. + This is partly for historical reasons: the callback could resurrect an unreachable + object and support for weak references predates support for object resurrection. + Ignoring the weak reference's callback is fine because both the object and the weakref + are going away, so it's legitimate to say the weak reference is going away first. +2. If an object has legacy finalizers (`tp_del` slot) move it to the + `gc.garbage` list. +3. Call the finalizers (`tp_finalize` slot) and mark the objects as already + finalized to avoid calling finalizers twice if the objects are resurrected or + if other finalizers have removed the object first. +4. Deal with resurrected objects. If some objects have been resurrected, the GC + finds the new subset of objects that are still unreachable by running the cycle + detection algorithm again and continues with them. +5. Call the `tp_clear` slot of every object so all internal links are broken and + the reference counts fall to 0, triggering the destruction of all unreachable + objects. + +Optimization: generations +========================= + +In order to limit the time each garbage collection takes, the GC +implementation for the default build uses a popular optimization: +generations. The main idea behind this concept is the assumption that most +objects have a very short lifespan and can thus be collected soon after their +creation. This has proven to be very close to the reality of many Python +programs as many temporary objects are created and destroyed very quickly. + +To take advantage of this fact, all container objects are segregated into +three spaces/generations. Every new +object starts in the first generation (generation 0). The previous algorithm is +executed only over the objects of a particular generation and if an object +survives a collection of its generation it will be moved to the next one +(generation 1), where it will be surveyed for collection less often. If +the same object survives another GC round in this new generation (generation 1) +it will be moved to the last generation (generation 2) where it will be +surveyed the least often. + +The GC implementation for the free-threaded build does not use multiple +generations. Every collection operates on the entire heap. + +In order to decide when to run, the collector keeps track of the number of object +allocations and deallocations since the last collection. When the number of +allocations minus the number of deallocations exceeds `threshold_0`, +collection starts. Initially only generation 0 is examined. If generation 0 has +been examined more than `threshold_1` times since generation 1 has been +examined, then generation 1 is examined as well. With generation 2, +things are a bit more complicated; see +[Collecting the oldest generation](#Collecting-the-oldest-generation) for +more information. These thresholds can be examined using the +[`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) +function: + +```pycon + >>> import gc + >>> gc.get_threshold() + (700, 10, 10) +``` + +The content of these generations can be examined using the +`gc.get_objects(generation=NUM)` function and collections can be triggered +specifically in a generation by calling `gc.collect(generation=NUM)`. + +```pycon + >>> import gc + >>> class MyObj: + ... pass + ... + + # Move everything to the last generation so it's easier to inspect + # the younger generations. + + >>> gc.collect() + 0 + + # Create a reference cycle. + + >>> x = MyObj() + >>> x.self = x + + # Initially the object is in the youngest generation. + + >>> gc.get_objects(generation=0) + [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] + + # After a collection of the youngest generation the object + # moves to the next generation. + + >>> gc.collect(generation=0) + 0 + >>> gc.get_objects(generation=0) + [] + >>> gc.get_objects(generation=1) + [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] +``` + +Collecting the oldest generation +-------------------------------- + +In addition to the various configurable thresholds, the GC only triggers a full +collection of the oldest generation if the ratio `long_lived_pending / long_lived_total` +is above a given value (hardwired to 25%). The reason is that, while "non-full" +collections (that is, collections of the young and middle generations) will always +examine roughly the same number of objects (determined by the aforementioned +thresholds) the cost of a full collection is proportional to the total +number of long-lived objects, which is virtually unbounded. Indeed, it has +been remarked that doing a full collection every of object +creations entails a dramatic performance degradation in workloads which consist +of creating and storing lots of long-lived objects (for example, building a large list +of GC-tracked objects would show quadratic performance, instead of linear as +expected). Using the above ratio, instead, yields amortized linear performance +in the total number of objects (the effect of which can be summarized thusly: +"each full garbage collection is more and more costly as the number of objects +grows, but we do fewer and fewer of them"). + +Optimization: reusing fields to save memory +=========================================== + +In order to save memory, the two linked list pointers in every object with GC +support are reused for several purposes. This is a common optimization known +as "fat pointers" or "tagged pointers": pointers that carry additional data, +"folded" into the pointer, meaning stored inline in the data representing the +address, taking advantage of certain properties of memory addressing. This is +possible as most architectures align certain types of data +to the size of the data, often a word or multiple thereof. This discrepancy +leaves a few of the least significant bits of the pointer unused, which can be +used for tags or to keep other information – most often as a bit field (each +bit a separate tag) – as long as code that uses the pointer masks out these +bits before accessing memory. For example, on a 32-bit architecture (for both +addresses and word size), a word is 32 bits = 4 bytes, so word-aligned +addresses are always a multiple of 4, hence end in `00`, leaving the last 2 bits +available; while on a 64-bit architecture, a word is 64 bits = 8 bytes, so +word-aligned addresses end in `000`, leaving the last 3 bits available. + +The CPython GC makes use of two fat pointers that correspond to the extra fields +of `PyGC_Head` discussed in the `Memory layout and object structure`_ section: + +> [!WARNING] +> Because the presence of extra information, "tagged" or "fat" pointers cannot be +> dereferenced directly and the extra information must be stripped off before +> obtaining the real memory address. Special care needs to be taken with +> functions that directly manipulate the linked lists, as these functions +> normally assume the pointers inside the lists are in a consistent state. + + +- The `_gc_prev` field is normally used as the "previous" pointer to maintain the + doubly linked list but its lowest two bits are used to keep the flags + `PREV_MASK_COLLECTING` and `_PyGC_PREV_MASK_FINALIZED`. Between collections, + the only flag that can be present is `_PyGC_PREV_MASK_FINALIZED` that indicates + if an object has been already finalized. During collections `_gc_prev` is + temporarily used for storing a copy of the reference count (`gc_ref`), in + addition to two flags, and the GC linked list becomes a singly linked list until + `_gc_prev` is restored. + +- The `_gc_next` field is used as the "next" pointer to maintain the doubly linked + list but during collection its lowest bit is used to keep the + `NEXT_MASK_UNREACHABLE` flag that indicates if an object is tentatively + unreachable during the cycle detection algorithm. This is a drawback to using only + doubly linked lists to implement partitions: while most needed operations are + constant-time, there is no efficient way to determine which partition an object is + currently in. Instead, when that's needed, ad hoc tricks (like the + `NEXT_MASK_UNREACHABLE` flag) are employed. + +Optimization: delay tracking containers +======================================= + +Certain types of containers cannot participate in a reference cycle, and so do +not need to be tracked by the garbage collector. Untracking these objects +reduces the cost of garbage collection. However, determining which objects may +be untracked is not free, and the costs must be weighed against the benefits +for garbage collection. There are two possible strategies for when to untrack +a container: + +1. When the container is created. +2. When the container is examined by the garbage collector. + +As a general rule, instances of atomic types aren't tracked and instances of +non-atomic types (containers, user-defined objects...) are. However, some +type-specific optimizations can be present in order to suppress the garbage +collector footprint of simple instances. Some examples of native types that +benefit from delayed tracking: + +- Tuples containing only immutable objects (integers, strings etc, + and recursively, tuples of immutable objects) do not need to be tracked. The + interpreter creates a large number of tuples, many of which will not survive + until garbage collection. It is therefore not worthwhile to untrack eligible + tuples at creation time. Instead, all tuples except the empty tuple are tracked + when created. During garbage collection it is determined whether any surviving + tuples can be untracked. A tuple can be untracked if all of its contents are + already not tracked. Tuples are examined for untracking in all garbage collection + cycles. It may take more than one cycle to untrack a tuple. + +- Dictionaries containing only immutable objects also do not need to be tracked. + Dictionaries are untracked when created. If a tracked item is inserted into a + dictionary (either as a key or value), the dictionary becomes tracked. During a + full garbage collection (all generations), the collector will untrack any dictionaries + whose contents are not tracked. + +The garbage collector module provides the Python function `is_tracked(obj)`, which returns +the current tracking status of the object. Subsequent garbage collections may change the +tracking status of the object. + +```pycon + >>> gc.is_tracked(0) + False + >>> gc.is_tracked("a") + False + >>> gc.is_tracked([]) + True + >>> gc.is_tracked({}) + False + >>> gc.is_tracked({"a": 1}) + False + >>> gc.is_tracked({"a": []}) + True +``` + +Differences between GC implementations +====================================== + +This section summarizes the differences between the GC implementation in the +default build and the implementation in the free-threaded build. + +The default build implementation makes extensive use of the `PyGC_Head` data +structure, while the free-threaded build implementation does not use that +data structure. + +- The default build implementation stores all tracked objects in a doubly + linked list using `PyGC_Head`. The free-threaded build implementation + instead relies on the embedded mimalloc memory allocator to scan the heap + for tracked objects. +- The default build implementation uses `PyGC_Head` for the unreachable + object list. The free-threaded build implementation repurposes the + `ob_tid` field to store a unreachable objects linked list. +- The default build implementation stores flags in the `_gc_prev` field of + `PyGC_Head`. The free-threaded build implementation stores these flags + in `ob_gc_bits`. + + +The default build implementation relies on the +[global interpreter lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) +for thread safety. The free-threaded build implementation has two "stop the +world" pauses, in which all other executing threads are temporarily paused so +that the GC can safely access reference counts and object attributes. + +The default build implementation is a generational collector. The +free-threaded build is non-generational; each collection scans the entire +heap. + +- Keeping track of object generations is simple and inexpensive in the default + build. The free-threaded build relies on mimalloc for finding tracked + objects; identifying "young" objects without scanning the entire heap would + be more difficult. + + +> [!NOTE] +> **Document history** +> +> Pablo Galindo Salgado - Original author +> +> Irit Katriel - Convert to Markdown diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md new file mode 100644 index 00000000000000..d53f0f9bdff4e4 --- /dev/null +++ b/InternalDocs/generators.md @@ -0,0 +1,9 @@ + +Generators +========== + +Coming soon. + + diff --git a/InternalDocs/images/python-cyclic-gc-1-new-page.png b/InternalDocs/images/python-cyclic-gc-1-new-page.png new file mode 100644 index 00000000000000..2ddac50f4b5575 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-1-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-2-new-page.png b/InternalDocs/images/python-cyclic-gc-2-new-page.png new file mode 100644 index 00000000000000..159aeeb05024a3 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-2-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-3-new-page.png b/InternalDocs/images/python-cyclic-gc-3-new-page.png new file mode 100644 index 00000000000000..29fab0498e5b10 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-3-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-4-new-page.png b/InternalDocs/images/python-cyclic-gc-4-new-page.png new file mode 100644 index 00000000000000..51a2b1065ea64e Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-4-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-5-new-page.png b/InternalDocs/images/python-cyclic-gc-5-new-page.png new file mode 100644 index 00000000000000..fe67a6896fe4b0 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-5-new-page.png differ diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md new file mode 100644 index 00000000000000..dcfddc99370c0e --- /dev/null +++ b/InternalDocs/interpreter.md @@ -0,0 +1,364 @@ + +The bytecode interpreter +======================== + +Overview +-------- + +This document describes the workings and implementation of the bytecode +interpreter, the part of python that executes compiled Python code. Its +entry point is in [Python/ceval.c](../Python/ceval.c). + +At a high level, the interpreter consists of a loop that iterates over the +bytecode instructions, executing each of them via a switch statement that +has a case implementing each opcode. This switch statement is generated +from the instruction definitions in [Python/bytecodes.c](../Python/bytecodes.c) +which are written in [a DSL](../Tools/cases_generator/interpreter_definition.md) +developed for this purpose. + +Recall that the [Python Compiler](compiler.md) produces a [`CodeObject`](code_object.md), +which contains the bytecode instructions along with static data that is required to execute them, +such as the consts list, variable names, +[exception table](exception_handling.md#format-of-the-exception-table), and so on. + +When the interpreter's +[`PyEval_EvalCode()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) +function is called to execute a `CodeObject`, it constructs a [`Frame`](frames.md) and calls +[`_PyEval_EvalFrame()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) +to execute the code object in this frame. The frame hold the dynamic state of the +`CodeObject`'s execution, including the instruction pointer, the globals and builtins. +It also has a reference to the `CodeObject` itself. + +In addition to the frame, `_PyEval_EvalFrame()` also receives a +[`Thread State`](https://docs.python.org/3/c-api/init.html#c.PyThreadState) +object, `tstate`, which includes things like the exception state and the +recursion depth. The thread state also provides access to the per-interpreter +state (`tstate->interp`), which has a pointer to the per-runtime (that is, +truly global) state (`tstate->interp->runtime`). + +Finally, `_PyEval_EvalFrame()` receives an integer argument `throwflag` +which, when nonzero, indicates that the interpreter should just raise the current exception +(this is used in the implementation of +[`gen.throw`](https://docs.python.org/3.14/reference/expressions.html#generator.throw). + +By default, [`_PyEval_EvalFrame()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) +simply calls [`_PyEval_EvalFrameDefault()`] to execute the frame. However, as per +[`PEP 523`](https://peps.python.org/pep-0523/) this is configurable by setting +`interp->eval_frame`. In the following, we describe the default function, +`_PyEval_EvalFrameDefault()`. + + +Instruction decoding +-------------------- + +The first task of the interpreter is to decode the bytecode instructions. +Bytecode is stored as an array of 16-bit code units (`_Py_CODEUNIT`). +Each code unit contains an 8-bit `opcode` and an 8-bit argument (`oparg`), both unsigned. +In order to make the bytecode format independent of the machine byte order when stored on disk, +`opcode` is always the first byte and `oparg` is always the second byte. +Macros are used to extract the `opcode` and `oparg` from a code unit +(`_Py_OPCODE(word)` and `_Py_OPARG(word)`). +Some instructions (for example, `NOP` or `POP_TOP`) have no argument -- in this case +we ignore `oparg`. + +A simplified version of the interpreter's main loop looks like this: + +```c + _Py_CODEUNIT *first_instr = code->co_code_adaptive; + _Py_CODEUNIT *next_instr = first_instr; + while (1) { + _Py_CODEUNIT word = *next_instr++; + unsigned char opcode = _Py_OPCODE(word); + unsigned int oparg = _Py_OPARG(word); + switch (opcode) { + // ... A case for each opcode ... + } + } +``` + +This loop iterates over the instructions, decoding each into its `opcode` +and `oparg`, and then executes the switch case that implements this `opcode`. + +The instruction format supports 256 different opcodes, which is sufficient. +However, it also limits `oparg` to 8-bit values, which is too restrictive. +To overcome this, the `EXTENDED_ARG` opcode allows us to prefix any instruction +with one or more additional data bytes, which combine into a larger oparg. +For example, this sequence of code units: + + EXTENDED_ARG 1 + EXTENDED_ARG 0 + LOAD_CONST 2 + +would set `opcode` to `LOAD_CONST` and `oparg` to `65538` (that is, `0x1_00_02`). +The compiler should limit itself to at most three `EXTENDED_ARG` prefixes, to allow the +resulting `oparg` to fit in 32 bits, but the interpreter does not check this. + +In the following, a `code unit` is always two bytes, while an `instruction` is a +sequence of code units consisting of zero to three `EXTENDED_ARG` opcodes followed by +a primary opcode. + +The following loop, to be inserted just above the `switch` statement, will make the above +snippet decode a complete instruction: + +```c + while (opcode == EXTENDED_ARG) { + word = *next_instr++; + opcode = _Py_OPCODE(word); + oparg = (oparg << 8) | _Py_OPARG(word); + } +``` + +For various reasons we'll get to later (mostly efficiency, given that `EXTENDED_ARG` +is rare) the actual code is different. + +Jumps +===== + +Note that when the `switch` statement is reached, `next_instr` (the "instruction offset") +already points to the next instruction. +Thus, jump instructions can be implemented by manipulating `next_instr`: + +- A jump forward (`JUMP_FORWARD`) sets `next_instr += oparg`. +- A jump backward sets `next_instr -= oparg`. + +Inline cache entries +==================== + +Some (specialized or specializable) instructions have an associated "inline cache". +The inline cache consists of one or more two-byte entries included in the bytecode +array as additional words following the `opcode`/`oparg` pair. +The size of the inline cache for a particular instruction is fixed by its `opcode`. +Moreover, the inline cache size for all instructions in a +[family of specialized/specializable instructions](adaptive.md) +(for example, `LOAD_ATTR`, `LOAD_ATTR_SLOT`, `LOAD_ATTR_MODULE`) must all be +the same. Cache entries are reserved by the compiler and initialized with zeros. +Although they are represented by code units, cache entries do not conform to the +`opcode` / `oparg` format. + +If an instruction has an inline cache, the layout of its cache is described by +a `struct` definition in (`pycore_code.h`)[../Include/internal/pycore_code.h]. +This allows us to access the cache by casting `next_instr` to a pointer to this `struct`. +The size of such a `struct` must be independent of the machine architecture, word size +and alignment requirements. For a 32-bit field, the `struct` should use `_Py_CODEUNIT field[2]`. + +The instruction implementation is responsible for advancing `next_instr` past the inline cache. +For example, if an instruction's inline cache is four bytes (that is, two code units) in size, +the code for the instruction must contain `next_instr += 2;`. +This is equivalent to a relative forward jump by that many code units. +(In the interpreter definition DSL, this is coded as `JUMPBY(n)`, where `n` is the number +of code units to jump, typically given as a named constant.) + +Serializing non-zero cache entries would present a problem because the serialization +(:mod:`marshal`) format must be independent of the machine byte order. + +More information about the use of inline caches can be found in +[PEP 659](https://peps.python.org/pep-0659/#ancillary-data). + +The evaluation stack +-------------------- + +Most instructions read or write some data in the form of object references (`PyObject *`). +The CPython bytecode interpreter is a stack machine, meaning that its instructions operate +by pushing data onto and popping it off the stack. +The stack is forms part of the frame for the code object. Its maximum depth is calculated +by the compiler and stored in the `co_stacksize` field of the code object, so that the +stack can be pre-allocated is a contiguous array of `PyObject*` pointers, when the frame +is created. + +The stack effects of each instruction are also exposed through the +[opcode metadata](../Include/internal/pycore_opcode_metadata.h) through two +functions that report how many stack elements the instructions consumes, +and how many it produces (`_PyOpcode_num_popped` and `_PyOpcode_num_pushed`). +For example, the `BINARY_OP` instruction pops two objects from the stack and pushes the +result back onto the stack. + +The stack grows up in memory; the operation `PUSH(x)` is equivalent to `*stack_pointer++ = x`, +whereas `x = POP()` means `x = *--stack_pointer`. +Overflow and underflow checks are active in debug mode, but are otherwise optimized away. + +At any point during execution, the stack level is knowable based on the instruction pointer +alone, and some properties of each item on the stack are also known. +In particular, only a few instructions may push a `NULL` onto the stack, and the positions +that may be `NULL` are known. +A few other instructions (`GET_ITER`, `FOR_ITER`) push or pop an object that is known to +be an iterator. + +Instruction sequences that do not allow statically knowing the stack depth are deemed illegal; +the bytecode compiler never generates such sequences. +For example, the following sequence is illegal, because it keeps pushing items on the stack: + + LOAD_FAST 0 + JUMP_BACKWARD 2 + +> [!NOTE] +> Do not confuse the evaluation stack with the call stack, which is used to implement calling +> and returning from functions. + +Error handling +-------------- + +When the implementation of an opcode raises an exception, it jumps to the +`exception_unwind` label in [Python/ceval.c](../Python/ceval.c). +The exception is then handled as described in the +[`exception handling documentation`](exception_handling.md#handling-exceptions). + +Python-to-Python calls +---------------------- + +The `_PyEval_EvalFrameDefault()` function is recursive, because sometimes +the interpreter calls some C function that calls back into the interpreter. +In 3.10 and before, this was the case even when a Python function called +another Python function: +The `CALL` opcode would call the `tp_call` dispatch function of the +callee, which would extract the code object, create a new frame for the call +stack, and then call back into the interpreter. This approach is very general +but consumes several C stack frames for each nested Python call, thereby +increasing the risk of an (unrecoverable) C stack overflow. + +Since 3.11, the `CALL` instruction special-cases function objects to "inline" +the call. When a call gets inlined, a new frame gets pushed onto the call +stack and the interpreter "jumps" to the start of the callee's bytecode. +When an inlined callee executes a `RETURN_VALUE` instruction, the frame is +popped off the call stack and the interpreter returns to its caller, +by popping a frame off the call stack and "jumping" to the return address. +There is a flag in the frame (`frame->is_entry`) that indicates whether +the frame was inlined (set if it wasn't). +If `RETURN_VALUE` finds this flag set, it performs the usual cleanup and +returns from `_PyEval_EvalFrameDefault()` altogether, to a C caller. + +A similar check is performed when an unhandled exception occurs. + +The call stack +-------------- + +Up through 3.10, the call stack was implemented as a singly-linked list of +[frame objects](frames.md). This was expensive because each call would require a +heap allocation for the stack frame. + +Since 3.11, frames are no longer fully-fledged objects. Instead, a leaner internal +`_PyInterpreterFrame` structure is used, which is allocated using a custom allocator +function (`_PyThreadState_BumpFramePointer()`), which allocates and initializes a +frame structure. Usually a frame allocation is just a pointer bump, which improves +memory locality. + +Sometimes an actual `PyFrameObject` is needed, such as when Python code calls +`sys._getframe()` or an extension module calls +[`PyEval_GetFrame()`](https://docs.python.org/3/c-api/reflection.html#c.PyEval_GetFrame). +In this case we allocate a proper `PyFrameObject` and initialize it from the +`_PyInterpreterFrame`. + +Things get more complicated when generators are involved, since those do not +follow the push/pop model. This includes async functions, which are based on +the same mechanism. A generator object has space for a `_PyInterpreterFrame` +structure, including the variable-size part (used for locals and the eval stack). +When a generator (or async) function is first called, a special opcode +`RETURN_GENERATOR` is executed, which is responsible for creating the +generator object. The generator object's `_PyInterpreterFrame` is initialized +with a copy of the current stack frame. The current stack frame is then popped +off the frame stack and the generator object is returned. +(Details differ depending on the `is_entry` flag.) +When the generator is resumed, the interpreter pushes its `_PyInterpreterFrame` +onto the frame stack and resumes execution. +See also the [generators](generators.md) section. + + + + + +Introducing a new bytecode instruction +-------------------------------------- + +It is occasionally necessary to add a new opcode in order to implement +a new feature or change the way that existing features are compiled. +This section describes the changes required to do this. + +First, you must choose a name for the bytecode, implement it in +[`Python/bytecodes.c`](../Python/bytecodes.c) and add a documentation +entry in [`Doc/library/dis.rst`](../Doc/library/dis.rst). +Then run `make regen-cases` to assign a number for it (see +[`Include/opcode_ids.h`](../Include/opcode_ids.h)) and regenerate a +number of files with the actual implementation of the bytecode in +[`Python/generated_cases.c.h`](../Python/generated_cases.c.h) and +metadata about it in additional files. + +With a new bytecode you must also change what is called the "magic number" for +.pyc files: bump the value of the variable `MAGIC_NUMBER` in +[`Lib/importlib/_bootstrap_external.py`](../Lib/importlib/_bootstrap_external.py). +Changing this number will lead to all .pyc files with the old `MAGIC_NUMBER` +to be recompiled by the interpreter on import. Whenever `MAGIC_NUMBER` is +changed, the ranges in the `magic_values` array in +[`PC/launcher.c`](../PC/launcher.c) may also need to be updated. Changes to +[`Lib/importlib/_bootstrap_external.py`](../Lib/importlib/_bootstrap_external.py) +will take effect only after running `make regen-importlib`. + +> [!NOTE] +> Running `make regen-importlib` before adding the new bytecode target to +> [`Python/bytecodes.c`](../Python/bytecodes.c) +> (followed by `make regen-cases`) will result in an error. You should only run +> `make regen-importlib` after the new bytecode target has been added. + +> [!NOTE] +> On Windows, running the `./build.bat` script will automatically +> regenerate the required files without requiring additional arguments. + +Finally, you need to introduce the use of the new bytecode. Update +[`Python/codegen.c`](../Python/codegen.c) to emit code with this bytecode. +Optimizations in [`Python/flowgraph.c`](../Python/flowgraph.c) may also +need to be updated. If the new opcode affects a control flow or the block +stack, you may have to update the `frame_setlineno()` function in +[`Objects/frameobject.c`](../Objects/frameobject.c). It may also be necessary +to update [`Lib/dis.py`](../Lib/dis.py) if the new opcode interprets its +argument in a special way (like `FORMAT_VALUE` or `MAKE_FUNCTION`). + +If you make a change here that can affect the output of bytecode that +is already in existence and you do not change the magic number, make +sure to delete your old .py(c|o) files! Even though you will end up changing +the magic number if you change the bytecode, while you are debugging your work +you may be changing the bytecode output without constantly bumping up the +magic number. This can leave you with stale .pyc files that will not be +recreated. +Running `find . -name '*.py[co]' -exec rm -f '{}' +` should delete all .pyc +files you have, forcing new ones to be created and thus allow you test out your +new bytecode properly. Run `make regen-importlib` for updating the +bytecode of frozen importlib files. You have to run `make` again after this +to recompile the generated C files. + +Additional resources +-------------------- + +* Brandt Bucher's talk about the specializing interpreter at PyCon US 2023. + [Slides](https://github.com/brandtbucher/brandtbucher/blob/master/2023/04/21/inside_cpython_311s_new_specializing_adaptive_interpreter.pdf) + [Video](https://www.youtube.com/watch?v=PGZPSWZSkJI&t=1470s) diff --git a/Objects/locations.md b/InternalDocs/locations.md similarity index 88% rename from Objects/locations.md rename to InternalDocs/locations.md index 18a338a95978a9..91a7824e2a8e4d 100644 --- a/Objects/locations.md +++ b/InternalDocs/locations.md @@ -1,10 +1,10 @@ # Locations table -For versions up to 3.10 see ./lnotab_notes.txt +The `co_linetable` bytes object of code objects contains a compact +representation of the source code positions of instructions, which are +returned by the `co_positions()` iterator. -In version 3.11 the `co_linetable` bytes object of code objects contains a compact representation of the positions returned by the `co_positions()` iterator. - -The `co_linetable` consists of a sequence of location entries. +`co_linetable` consists of a sequence of location entries. Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with most significant bit unset. Each entry contains the following information: diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md new file mode 100644 index 00000000000000..6398ba6cd2838f --- /dev/null +++ b/InternalDocs/parser.md @@ -0,0 +1,882 @@ + +Guide to the parser +=================== + +Abstract +-------- + +Python's Parser is currently a +[`PEG` (Parser Expression Grammar)](https://en.wikipedia.org/wiki/Parsing_expression_grammar) +parser. It was introduced in +[PEP 617: New PEG parser for CPython](https://peps.python.org/pep-0617/) to replace +the original [`LL(1)`](https://en.wikipedia.org/wiki/LL_parser) parser. + +The code implementing the parser is generated from a grammar definition by a +[parser generator](https://en.wikipedia.org/wiki/Compiler-compiler). +Therefore, changes to the Python language are made by modifying the +[grammar file](../Grammar/python.gram). +Developers rarely need to modify the generator itself. + +See the devguide's [Changing CPython's grammar](https://devguide.python.org/developer-workflow/grammar/#grammar) +for a detailed description of the grammar and the process for changing it. + +How PEG parsers work +==================== + +A PEG (Parsing Expression Grammar) grammar differs from a +[context-free grammar](https://en.wikipedia.org/wiki/Context-free_grammar) +in that the way it is written more closely reflects how the parser will operate +when parsing. The fundamental technical difference is that the choice operator +is ordered. This means that when writing: + +``` + rule: A | B | C +``` + +a parser that implements a context-free-grammar (such as an `LL(1)` parser) will +generate constructions that, given an input string, *deduce* which alternative +(`A`, `B` or `C`) must be expanded. On the other hand, a PEG parser will +check each alternative, in the order in which they are specified, and select +that first one that succeeds. + +This means that in a PEG grammar, the choice operator is not commutative. +Furthermore, unlike context-free grammars, the derivation according to a +PEG grammar cannot be ambiguous: if a string parses, it has exactly one +valid parse tree. + +PEG parsers are usually constructed as a recursive descent parser in which every +rule in the grammar corresponds to a function in the program implementing the +parser, and the parsing expression (the "expansion" or "definition" of the rule) +represents the "code" in said function. Each parsing function conceptually takes +an input string as its argument, and yields one of the following results: + +* A "success" result. This result indicates that the expression can be parsed by + that rule and the function may optionally move forward or consume one or more + characters of the input string supplied to it. +* A "failure" result, in which case no input is consumed. + +Note that "failure" results do not imply that the program is incorrect, nor do +they necessarily mean that the parsing has failed. Since the choice operator is +ordered, a failure very often merely indicates "try the following option". A +direct implementation of a PEG parser as a recursive descent parser will present +exponential time performance in the worst case, because PEG parsers have +infinite lookahead (this means that they can consider an arbitrary number of +tokens before deciding for a rule). Usually, PEG parsers avoid this exponential +time complexity with a technique called +["packrat parsing"](https://pdos.csail.mit.edu/~baford/packrat/thesis/) +which not only loads the entire program in memory before parsing it but also +allows the parser to backtrack arbitrarily. This is made efficient by memoizing +the rules already matched for each position. The cost of the memoization cache +is that the parser will naturally use more memory than a simple `LL(1)` parser, +which normally are table-based. + + +Key ideas +--------- + +- Alternatives are ordered ( `A | B` is not the same as `B | A` ). +- If a rule returns a failure, it doesn't mean that the parsing has failed, + it just means "try something else". +- By default PEG parsers run in exponential time, which can be optimized to linear by + using memoization. +- If parsing fails completely (no rule succeeds in parsing all the input text), the + PEG parser doesn't have a concept of "where the + [`SyntaxError`](https://docs.python.org/3/library/exceptions.html#SyntaxError) is". + + +> [!IMPORTANT] +> Don't try to reason about a PEG grammar in the same way you would to with an +> [EBNF](https://en.wikipedia.org/wiki/Extended_Backus–Naur_form) +> or context free grammar. PEG is optimized to describe **how** input strings will +> be parsed, while context-free grammars are optimized to generate strings of the +> language they describe (in EBNF, to know whether a given string is in the +> language, you need to do work to find out as it is not immediately obvious from +> the grammar). + + +Consequences of the ordered choice operator +------------------------------------------- + +Although PEG may look like EBNF, its meaning is quite different. The fact +that the alternatives are ordered in a PEG grammer (which is at the core of +how PEG parsers work) has deep consequences, other than removing ambiguity. + +If a rule has two alternatives and the first of them succeeds, the second one is +**not** attempted even if the caller rule fails to parse the rest of the input. +Thus the parser is said to be "eager". To illustrate this, consider +the following two rules (in these examples, a token is an individual character): + +``` + first_rule: ( 'a' | 'aa' ) 'a' + second_rule: ('aa' | 'a' ) 'a' +``` + +In a regular EBNF grammar, both rules specify the language `{aa, aaa}` but +in PEG, one of these two rules accepts the string `aaa` but not the string +`aa`. The other does the opposite -- it accepts the string `aa` +but not the string `aaa`. The rule `('a'|'aa')'a'` does +not accept `aaa` because `'a'|'aa'` consumes the first `a`, letting the +final `a` in the rule consume the second, and leaving out the third `a`. +As the rule has succeeded, no attempt is ever made to go back and let +`'a'|'aa'` try the second alternative. The expression `('aa'|'a')'a'` does +not accept `aa` because `'aa'|'a'` accepts all of `aa`, leaving nothing +for the final `a`. Again, the second alternative of `'aa'|'a'` is not +tried. + +> [!CAUTION] +> The effects of ordered choice, such as the ones illustrated above, may be +> hidden by many levels of rules. + +For this reason, writing rules where an alternative is contained in the next +one is in almost all cases a mistake, for example: + +``` + my_rule: + | 'if' expression 'then' block + | 'if' expression 'then' block 'else' block +``` + +In this example, the second alternative will never be tried because the first one will +succeed first (even if the input string has an `'else' block` that follows). To correctly +write this rule you can simply alter the order: + +``` + my_rule: + | 'if' expression 'then' block 'else' block + | 'if' expression 'then' block +``` + +In this case, if the input string doesn't have an `'else' block`, the first alternative +will fail and the second will be attempted. + +Grammar Syntax +============== + +The grammar consists of a sequence of rules of the form: + +``` + rule_name: expression +``` + +Optionally, a type can be included right after the rule name, which +specifies the return type of the C or Python function corresponding to +the rule: + +``` + rule_name[return_type]: expression +``` + +If the return type is omitted, then a `void *` is returned in C and an +`Any` in Python. + +Grammar expressions +------------------- + +| Expression | Description and Example | +|-----------------|-----------------------------------------------------------------------------------------------------------------------| +| `# comment` | Python-style comments. | +| `e1 e2` | Match `e1`, then match `e2`.
`rule_name: first_rule second_rule` | +| `e1 \| e2` | Match `e1` or `e2`.
`rule_name[return_type]:`
` \| first_alt`
` \| second_alt` | +| `( e )` | Grouping operator: Match `e`.
`rule_name: (e)`
`rule_name: (e1 e2)*` | +| `[ e ]` or `e?` | Optionally match `e`.
`rule_name: [e]`
`rule_name: e (',' e)* [',']` | +| `e*` | Match zero or more occurrences of `e`.
`rule_name: (e1 e2)*` | +| `e+` | Match one or more occurrences of `e`.
`rule_name: (e1 e2)+` | +| `s.e+` | Match one or more occurrences of `e`, separated by `s`.
`rule_name: ','.e+` | +| `&e` | Positive lookahead: Succeed if `e` can be parsed, without consuming input. | +| `!e` | Negative lookahead: Fail if `e` can be parsed, without consuming input.
`primary: atom !'.' !'(' !'['` | +| `~` | Commit to the current alternative, even if it fails to parse (cut).
`rule_name: '(' ~ some_rule ')' \| some_alt` | + + +Left recursion +-------------- + +PEG parsers normally do not support left recursion, but CPython's parser +generator implements a technique similar to the one described in +[Medeiros et al.](https://arxiv.org/pdf/1207.0443) but using the memoization +cache instead of static variables. This approach is closer to the one described +in [Warth et al.](http://web.cs.ucla.edu/~todd/research/pepm08.pdf). This +allows us to write not only simple left-recursive rules but also more +complicated rules that involve indirect left-recursion like: + +``` + rule1: rule2 | 'a' + rule2: rule3 | 'b' + rule3: rule1 | 'c' +``` + +and "hidden left-recursion" like: + +``` + rule: 'optional'? rule '@' some_other_rule +``` + +Variables in the grammar +------------------------ + +A sub-expression can be named by preceding it with an identifier and an +`=` sign. The name can then be used in the action (see below), like this: + +``` + rule_name[return_type]: '(' a=some_other_rule ')' { a } +``` + +Grammar actions +--------------- + +To avoid the intermediate steps that obscure the relationship between the +grammar and the AST generation, the PEG parser allows directly generating AST +nodes for a rule via grammar actions. Grammar actions are language-specific +expressions that are evaluated when a grammar rule is successfully parsed. These +expressions can be written in Python or C depending on the desired output of the +parser generator. This means that if one would want to generate a parser in +Python and another in C, two grammar files should be written, each one with a +different set of actions, keeping everything else apart from said actions +identical in both files. As an example of a grammar with Python actions, the +piece of the parser generator that parses grammar files is bootstrapped from a +meta-grammar file with Python actions that generate the grammar tree as a result +of the parsing. + +In the specific case of the PEG grammar for Python, having actions allows +directly describing how the AST is composed in the grammar itself, making it +more clear and maintainable. This AST generation process is supported by the use +of some helper functions that factor out common AST object manipulations and +some other required operations that are not directly related to the grammar. + +To indicate these actions each alternative can be followed by the action code +inside curly-braces, which specifies the return value of the alternative: + +``` + rule_name[return_type]: + | first_alt1 first_alt2 { first_alt1 } + | second_alt1 second_alt2 { second_alt1 } +``` + +If the action is omitted, a default action is generated: + +- If there is a single name in the rule, it gets returned. +- If there multiple names in the rule, a collection with all parsed + expressions gets returned (the type of the collection will be different + in C and Python). + +This default behaviour is primarily made for very simple situations and for +debugging purposes. + +> [!WARNING] +> It's important that the actions don't mutate any AST nodes that are passed +> into them via variables referring to other rules. The reason for mutation +> being not allowed is that the AST nodes are cached by memoization and could +> potentially be reused in a different context, where the mutation would be +> invalid. If an action needs to change an AST node, it should instead make a +> new copy of the node and change that. + +The full meta-grammar for the grammars supported by the PEG generator is: + +``` + start[Grammar]: grammar ENDMARKER { grammar } + + grammar[Grammar]: + | metas rules { Grammar(rules, metas) } + | rules { Grammar(rules, []) } + + metas[MetaList]: + | meta metas { [meta] + metas } + | meta { [meta] } + + meta[MetaTuple]: + | "@" NAME NEWLINE { (name.string, None) } + | "@" a=NAME b=NAME NEWLINE { (a.string, b.string) } + | "@" NAME STRING NEWLINE { (name.string, literal_eval(string.string)) } + + rules[RuleList]: + | rule rules { [rule] + rules } + | rule { [rule] } + + rule[Rule]: + | rulename ":" alts NEWLINE INDENT more_alts DEDENT { + Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts)) } + | rulename ":" NEWLINE INDENT more_alts DEDENT { Rule(rulename[0], rulename[1], more_alts) } + | rulename ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts) } + + rulename[RuleName]: + | NAME '[' type=NAME '*' ']' {(name.string, type.string+"*")} + | NAME '[' type=NAME ']' {(name.string, type.string)} + | NAME {(name.string, None)} + + alts[Rhs]: + | alt "|" alts { Rhs([alt] + alts.alts)} + | alt { Rhs([alt]) } + + more_alts[Rhs]: + | "|" alts NEWLINE more_alts { Rhs(alts.alts + more_alts.alts) } + | "|" alts NEWLINE { Rhs(alts.alts) } + + alt[Alt]: + | items '$' action { Alt(items + [NamedItem(None, NameLeaf('ENDMARKER'))], action=action) } + | items '$' { Alt(items + [NamedItem(None, NameLeaf('ENDMARKER'))], action=None) } + | items action { Alt(items, action=action) } + | items { Alt(items, action=None) } + + items[NamedItemList]: + | named_item items { [named_item] + items } + | named_item { [named_item] } + + named_item[NamedItem]: + | NAME '=' ~ item {NamedItem(name.string, item)} + | item {NamedItem(None, item)} + | it=lookahead {NamedItem(None, it)} + + lookahead[LookaheadOrCut]: + | '&' ~ atom {PositiveLookahead(atom)} + | '!' ~ atom {NegativeLookahead(atom)} + | '~' {Cut()} + + item[Item]: + | '[' ~ alts ']' {Opt(alts)} + | atom '?' {Opt(atom)} + | atom '*' {Repeat0(atom)} + | atom '+' {Repeat1(atom)} + | sep=atom '.' node=atom '+' {Gather(sep, node)} + | atom {atom} + + atom[Plain]: + | '(' ~ alts ')' {Group(alts)} + | NAME {NameLeaf(name.string) } + | STRING {StringLeaf(string.string)} + + # Mini-grammar for the actions + + action[str]: "{" ~ target_atoms "}" { target_atoms } + + target_atoms[str]: + | target_atom target_atoms { target_atom + " " + target_atoms } + | target_atom { target_atom } + + target_atom[str]: + | "{" ~ target_atoms "}" { "{" + target_atoms + "}" } + | NAME { name.string } + | NUMBER { number.string } + | STRING { string.string } + | "?" { "?" } + | ":" { ":" } +``` + +As an illustrative example this simple grammar file allows directly +generating a full parser that can parse simple arithmetic expressions and that +returns a valid C-based Python AST: + +``` + start[mod_ty]: a=expr_stmt* ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + expr_stmt[stmt_ty]: a=expr NEWLINE { _PyAST_Expr(a, EXTRA) } + + expr[expr_ty]: + | l=expr '+' r=term { _PyAST_BinOp(l, Add, r, EXTRA) } + | l=expr '-' r=term { _PyAST_BinOp(l, Sub, r, EXTRA) } + | term + + term[expr_ty]: + | l=term '*' r=factor { _PyAST_BinOp(l, Mult, r, EXTRA) } + | l=term '/' r=factor { _PyAST_BinOp(l, Div, r, EXTRA) } + | factor + + factor[expr_ty]: + | '(' e=expr ')' { e } + | atom + + atom[expr_ty]: + | NAME + | NUMBER +``` + +Here `EXTRA` is a macro that expands to `start_lineno, start_col_offset, +end_lineno, end_col_offset, p->arena`, those being variables automatically +injected by the parser; `p` points to an object that holds on to all state +for the parser. + +A similar grammar written to target Python AST objects: + +``` + start[ast.Module]: a=expr_stmt* ENDMARKER { ast.Module(body=a or [] } + expr_stmt: a=expr NEWLINE { ast.Expr(value=a, EXTRA) } + + expr: + | l=expr '+' r=term { ast.BinOp(left=l, op=ast.Add(), right=r, EXTRA) } + | l=expr '-' r=term { ast.BinOp(left=l, op=ast.Sub(), right=r, EXTRA) } + | term + + term: + | l=term '*' r=factor { ast.BinOp(left=l, op=ast.Mult(), right=r, EXTRA) } + | l=term '/' r=factor { ast.BinOp(left=l, op=ast.Div(), right=r, EXTRA) } + | factor + + factor: + | '(' e=expr ')' { e } + | atom + + atom: + | NAME + | NUMBER +``` + +Pegen +===== + +Pegen is the parser generator used in CPython to produce the final PEG parser +used by the interpreter. It is the program that can be used to read the python +grammar located in [`Grammar/python.gram`](../Grammar/python.gram) and produce +the final C parser. It contains the following pieces: + +- A parser generator that can read a grammar file and produce a PEG parser + written in Python or C that can parse said grammar. The generator is located at + [`Tools/peg_generator/pegen`](../Tools/peg_generator/pegen). +- A PEG meta-grammar that automatically generates a Python parser which is used + for the parser generator itself (this means that there are no manually-written + parsers). The meta-grammar is located at + [`Tools/peg_generator/pegen/metagrammar.gram`](../Tools/peg_generator/pegen/metagrammar.gram). +- A generated parser (using the parser generator) that can directly produce C and Python AST objects. + +The source code for Pegen lives at [`Tools/peg_generator/pegen`](../Tools/peg_generator/pegen) +but normally all typical commands to interact with the parser generator are executed from +the main makefile. + +How to regenerate the parser +---------------------------- + +Once you have made the changes to the grammar files, to regenerate the `C` +parser (the one used by the interpreter) just execute: + +``` + make regen-pegen +``` + +using the `Makefile` in the main directory. If you are on Windows you can +use the Visual Studio project files to regenerate the parser or to execute: + +``` + ./PCbuild/build.bat --regen +``` + +The generated parser file is located at [`Parser/parser.c`](../Parser/parser.c). + +How to regenerate the meta-parser +--------------------------------- + +The meta-grammar (the grammar that describes the grammar for the grammar files +themselves) is located at +[`Tools/peg_generator/pegen/metagrammar.gram`](../Tools/peg_generator/pegen/metagrammar.gram). +Although it is very unlikely that you will ever need to modify it, if you make +any modifications to this file (in order to implement new Pegen features) you will +need to regenerate the meta-parser (the parser that parses the grammar files). +To do so just execute: + +``` + make regen-pegen-metaparser +``` + +If you are on Windows you can use the Visual Studio project files +to regenerate the parser or to execute: + +``` + ./PCbuild/build.bat --regen +``` + + +Grammatical elements and rules +------------------------------ + +Pegen has some special grammatical elements and rules: + +- Strings with single quotes (') (for example, `'class'`) denote KEYWORDS. +- Strings with double quotes (") (for example, `"match"`) denote SOFT KEYWORDS. +- Uppercase names (for example, `NAME`) denote tokens in the + [`Grammar/Tokens`](../Grammar/Tokens) file. +- Rule names starting with `invalid_` are used for specialized syntax errors. + + - These rules are NOT used in the first pass of the parser. + - Only if the first pass fails to parse, a second pass including the invalid + rules will be executed. + - If the parser fails in the second phase with a generic syntax error, the + location of the generic failure of the first pass will be used (this avoids + reporting incorrect locations due to the invalid rules). + - The order of the alternatives involving invalid rules matter + (like any rule in PEG). + +Tokenization +------------ + +It is common among PEG parser frameworks that the parser does both the parsing +and the tokenization, but this does not happen in Pegen. The reason is that the +Python language needs a custom tokenizer to handle things like indentation +boundaries, some special keywords like `ASYNC` and `AWAIT` (for +compatibility purposes), backtracking errors (such as unclosed parenthesis), +dealing with encoding, interactive mode and much more. Some of these reasons +are also there for historical purposes, and some others are useful even today. + +The list of tokens (all uppercase names in the grammar) that you can use can +be found in the [`Grammar/Tokens`](../Grammar/Tokens) +file. If you change this file to add new tokens, make sure to regenerate the +files by executing: + +``` + make regen-token +``` + +If you are on Windows you can use the Visual Studio project files to regenerate +the tokens or to execute: + +``` + ./PCbuild/build.bat --regen +``` + +How tokens are generated and the rules governing this are completely up to the tokenizer +([`Parser/lexer`](../Parser/lexer) and [`Parser/tokenizer`](../Parser/tokenizer)); +the parser just receives tokens from it. + +Memoization +----------- + +As described previously, to avoid exponential time complexity in the parser, +memoization is used. + +The C parser used by Python is highly optimized and memoization can be expensive +both in memory and time. Although the memory cost is obvious (the parser needs +memory for storing previous results in the cache) the execution time cost comes +for continuously checking if the given rule has a cache hit or not. In many +situations, just parsing it again can be faster. Pegen **disables memoization +by default** except for rules with the special marker `memo` after the rule +name (and type, if present): + +``` + rule_name[typr] (memo): + ... +``` + +By selectively turning on memoization for a handful of rules, the parser becomes +faster and uses less memory. + +> [!NOTE] +> Left-recursive rules always use memoization, since the implementation of +> left-recursion depends on it. + +To determine whether a new rule needs memoization or not, benchmarking is required +(comparing execution times and memory usage of some considerably large files with +and without memoization). There is a very simple instrumentation API available +in the generated C parse code that allows to measure how much each rule uses +memoization (check the [`Parser/pegen.c`](../Parser/pegen.c) +file for more information) but it needs to be manually activated. + +Automatic variables +------------------- + +To make writing actions easier, Pegen injects some automatic variables in the +namespace available when writing actions. In the C parser, some of these +automatic variable names are: + +- `p`: The parser structure. +- `EXTRA`: This is a macro that expands to + `(_start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena)`, + which is normally used to create AST nodes as almost all constructors need these + attributes to be provided. All of the location variables are taken from the + location information of the current token. + +Hard and soft keywords +---------------------- + +> [!NOTE] +> In the grammar files, keywords are defined using **single quotes** (for example, +> `'class'`) while soft keywords are defined using **double quotes** (for example, +> `"match"`). + +There are two kinds of keywords allowed in pegen grammars: *hard* and *soft* +keywords. The difference between hard and soft keywords is that hard keywords +are always reserved words, even in positions where they make no sense +(for example, `x = class + 1`), while soft keywords only get a special +meaning in context. Trying to use a hard keyword as a variable will always +fail: + +``` + >>> class = 3 + File "", line 1 + class = 3 + ^ + SyntaxError: invalid syntax + >>> foo(class=3) + File "", line 1 + foo(class=3) + ^^^^^ + SyntaxError: invalid syntax +``` + +While soft keywords don't have this limitation if used in a context other the +one where they are defined as keywords: + +``` + >>> match = 45 + >>> foo(match="Yeah!") +``` + +The `match` and `case` keywords are soft keywords, so that they are +recognized as keywords at the beginning of a match statement or case block +respectively, but are allowed to be used in other places as variable or +argument names. + +You can get a list of all keywords defined in the grammar from Python: + +``` + >>> import keyword + >>> keyword.kwlist + ['False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', + 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', + 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', + 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'] +``` + +as well as soft keywords: + +``` + >>> import keyword + >>> keyword.softkwlist + ['_', 'case', 'match'] +``` + +> [!CAUTION] +> Soft keywords can be a bit challenging to manage as they can be accepted in +> places you don't intend, given how the order alternatives behave in PEG +> parsers (see the +> [consequences of ordered choice](#consequences-of-the-ordered-choice-operator) +> section for some background on this). In general, try to define them in places +> where there are not many alternatives. + +Error handling +-------------- + +When a pegen-generated parser detects that an exception is raised, it will +**automatically stop parsing**, no matter what the current state of the parser +is, and it will unwind the stack and report the exception. This means that if a +[rule action](#grammar-actions) raises an exception, all parsing will +stop at that exact point. This is done to allow to correctly propagate any +exception set by calling Python's C API functions. This also includes +[`SyntaxError`](https://docs.python.org/3/library/exceptions.html#SyntaxError) +exceptions and it is the main mechanism the parser uses to report custom syntax +error messages. + +> [!NOTE] +> Tokenizer errors are normally reported by raising exceptions but some special +> tokenizer errors such as unclosed parenthesis will be reported only after the +> parser finishes without returning anything. + +How syntax errors are reported +------------------------------ + +As described previously in the [how PEG parsers work](#how-peg-parsers-work) +section, PEG parsers don't have a defined concept of where errors happened +in the grammar, because a rule failure doesn't imply a parsing failure like +in context free grammars. This means that a heuristic has to be used to report +generic errors unless something is explicitly declared as an error in the +grammar. + +To report generic syntax errors, pegen uses a common heuristic in PEG parsers: +the location of *generic* syntax errors is reported to be the furthest token that +was attempted to be matched but failed. This is only done if parsing has failed +(the parser returns `NULL` in C or `None` in Python) but no exception has +been raised. + +As the Python grammar was primordially written as an `LL(1)` grammar, this heuristic +has an extremely high success rate, but some PEG features, such as lookaheads, +can impact this. + +> [!CAUTION] +> Positive and negative lookaheads will try to match a token so they will affect +> the location of generic syntax errors. Use them carefully at boundaries +> between rules. + +To generate more precise syntax errors, custom rules are used. This is a common +practice also in context free grammars: the parser will try to accept some +construct that is known to be incorrect just to report a specific syntax error +for that construct. In pegen grammars, these rules start with the `invalid_` +prefix. This is because trying to match these rules normally has a performance +impact on parsing (and can also affect the 'correct' grammar itself in some +tricky cases, depending on the ordering of the rules) so the generated parser +acts in two phases: + +1. The first phase will try to parse the input stream without taking into + account rules that start with the `invalid_` prefix. If the parsing + succeeds it will return the generated AST and the second phase will be + skipped. + +2. If the first phase failed, a second parsing attempt is done including the + rules that start with an `invalid_` prefix. By design this attempt + **cannot succeed** and is only executed to give to the invalid rules a + chance to detect specific situations where custom, more precise, syntax + errors can be raised. This also allows to trade a bit of performance for + precision reporting errors: given that we know that the input text is + invalid, there is typically no need to be fast because execution is going + to stop anyway. + +> [!IMPORTANT] +> When defining invalid rules: +> +> - Make sure all custom invalid rules raise +> [`SyntaxError`](https://docs.python.org/3/library/exceptions.html#SyntaxError) +> exceptions (or a subclass of it). +> - Make sure **all** invalid rules start with the `invalid_` prefix to not +> impact performance of parsing correct Python code. +> - Make sure the parser doesn't behave differently for regular rules when you introduce invalid rules +> (see the [how PEG parsers work](#how-peg-parsers-work) section for more information). + +You can find a collection of macros to raise specialized syntax errors in the +[`Parser/pegen.h`](../Parser/pegen.h) +header file. These macros allow also to report ranges for +the custom errors, which will be highlighted in the tracebacks that will be +displayed when the error is reported. + + +> [!TIP] +> A good way to test whether an invalid rule will be triggered when you expect +> is to test if introducing a syntax error **after** valid code triggers the +> rule or not. For example: + +``` + $ 42 +``` + +should trigger the syntax error in the `$` character. If your rule is not correctly defined this +won't happen. As another example, suppose that you try to define a rule to match Python 2 style +`print` statements in order to create a better error message and you define it as: + +``` + invalid_print: "print" expression +``` + +This will **seem** to work because the parser will correctly parse `print(something)` because it is valid +code and the second phase will never execute but if you try to parse `print(something) $ 3` the first pass +of the parser will fail (because of the `$`) and in the second phase, the rule will match the +`print(something)` as `print` followed by the variable `something` between parentheses and the error +will be reported there instead of the `$` character. + +Generating AST objects +---------------------- + +The output of the C parser used by CPython, which is generated from the +[grammar file](../Grammar/python.gram), is a Python AST object (using C +structures). This means that the actions in the grammar file generate AST +objects when they succeed. Constructing these objects can be quite cumbersome +(see the [AST compiler section](compiler.md#abstract-syntax-trees-ast) +for more information on how these objects are constructed and how they are used +by the compiler), so special helper functions are used. These functions are +declared in the [`Parser/pegen.h`](../Parser/pegen.h) header file and defined +in the [`Parser/action_helpers.c`](../Parser/action_helpers.c) file. The +helpers include functions that join AST sequences, get specific elements +from them or to perform extra processing on the generated tree. + + +> [!CAUTION] +> Actions must **never** be used to accept or reject rules. It may be tempting +> in some situations to write a very generic rule and then check the generated +> AST to decide whether it is valid or not, but this will render the +> (official grammar)[https://docs.python.org/3/reference/grammar.html] partially +> incorrect (because it does not include actions) and will make it more difficult +> for other Python implementations to adapt the grammar to their own needs. + +As a general rule, if an action spawns multiple lines or requires something more +complicated than a single expression of C code, is normally better to create a +custom helper in [`Parser/action_helpers.c`](../Parser/action_helpers.c) +and expose it in the [`Parser/pegen.h`](../Parser/pegen.h) header file so that +it can be used from the grammar. + +When parsing succeeds, the parser **must** return a **valid** AST object. + +Testing +======= + +There are three files that contain tests for the grammar and the parser: + +- [test_grammar.py](../Lib/test/test_grammar.py) +- [test_syntax.py](../Lib/test/test_syntax.py) +- [test_exceptions.py](../Lib/test/test_exceptions.py) + +Check the contents of these files to know which is the best place for new +tests, depending on the nature of the new feature you are adding. + +Tests for the parser generator itself can be found in the +[test_peg_generator](../Lib/test_peg_generator) directory. + + +Debugging generated parsers +=========================== + +Making experiments +------------------ + +As the generated C parser is the one used by Python, this means that if +something goes wrong when adding some new rules to the grammar, you cannot +correctly compile and execute Python anymore. This makes it a bit challenging +to debug when something goes wrong, especially when experimenting. + +For this reason it is a good idea to experiment first by generating a Python +parser. To do this, you can go to the [Tools/peg_generator](../Tools/peg_generator) +directory on the CPython repository and manually call the parser generator by executing: + +``` + $ python -m pegen python +``` + +This will generate a file called `parse.py` in the same directory that you +can use to parse some input: + +``` + $ python parse.py file_with_source_code_to_test.py +``` + +As the generated `parse.py` file is just Python code, you can modify it +and add breakpoints to debug or better understand some complex situations. + + +Verbose mode +------------ + +When Python is compiled in debug mode (by adding `--with-pydebug` when +running the configure step in Linux or by adding `-d` when calling the +[PCbuild/build.bat](../PCbuild/build.bat)), it is possible to activate a +**very** verbose mode in the generated parser. This is very useful to +debug the generated parser and to understand how it works, but it +can be a bit hard to understand at first. + +> [!NOTE] +> When activating verbose mode in the Python parser, it is better to not use +> interactive mode as it can be much harder to understand, because interactive +> mode involves some special steps compared to regular parsing. + +To activate verbose mode you can add the `-d` flag when executing Python: + +``` + $ python -d file_to_test.py +``` + +This will print **a lot** of output to `stderr` so it is probably better to dump +it to a file for further analysis. The output consists of trace lines with the +following structure:: + +``` + ('>'|'-'|'+'|'!') []: ... +``` + +Every line is indented by a different amount (``) depending on how +deep the call stack is. The next character marks the type of the trace: + +- `>` indicates that a rule is going to be attempted to be parsed. +- `-` indicates that a rule has failed to be parsed. +- `+` indicates that a rule has been parsed correctly. +- `!` indicates that an exception or an error has been detected and the parser is unwinding. + +The `` part indicates the current index in the token array, +the `` part indicates what rule is being parsed and +the `` part indicates what alternative within that rule +is being attempted. + + +> [!NOTE] +> **Document history** +> +> Pablo Galindo Salgado - Original author +> +> Irit Katriel and Jacob Coffee - Convert to Markdown diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md new file mode 100644 index 00000000000000..e0d20632516142 --- /dev/null +++ b/InternalDocs/string_interning.md @@ -0,0 +1,118 @@ +# String interning + +*Interned* strings are conceptually part of an interpreter-global +*set* of interned strings, meaning that: +- no two interned strings have the same content (across an interpreter); +- two interned strings can be safely compared using pointer equality + (Python `is`). + +This is used to optimize dict and attribute lookups, among other things. + +Python uses two different mechanisms to intern strings: singletons and +dynamic interning. + +## Singletons + +The 256 possible one-character latin-1 strings, which can be retrieved with +`_Py_LATIN1_CHR(c)`, are stored in statically allocated arrays, +`_PyRuntime.static_objects.strings.ascii` and +`_PyRuntime.static_objects.strings.latin1`. + +Longer singleton strings are marked in C source with `_Py_ID` (if the string +is a valid C identifier fragment) or `_Py_STR` (if it needs a separate +C-compatible name.) +These are also stored in statically allocated arrays. +They are collected from CPython sources using `make regen-global-objects` +(`Tools/build/generate_global_objects.py`), which generates code +for declaration, initialization and finalization. + +The empty string is one of the singletons: `_Py_STR(empty)`. + +The three sets of singletons (`_Py_LATIN1_CHR`, `_Py_ID`, `_Py_STR`) +are disjoint. +If you have such a singleton, it (and no other copy) will be interned. + +These singletons are interned in a runtime-global lookup table, +`_PyRuntime.cached_objects.interned_strings` (`INTERNED_STRINGS`), +at runtime initialization, and immutable until it's torn down +at runtime finalization. +It is shared across threads and interpreters without any synchronization. + + +## Dynamically allocated strings + +All other strings are allocated dynamically, and have their +`_PyUnicode_STATE(s).statically_allocated` flag set to zero. +When interned, such strings are added to an interpreter-wide dict, +`PyInterpreterState.cached_objects.interned_strings`. + +The key and value of each entry in this dict reference the same object. + + +## Immortality and reference counting + +Invariant: Every immortal string is interned. + +In practice, this means that you must not use `_Py_SetImmortal` on +a string. (If you know it's already immortal, don't immortalize it; +if you know it's not interned you might be immortalizing a redundant copy; +if it's interned and mortal it needs extra processing in +`_PyUnicode_InternImmortal`.) + +The converse is not true: interned strings can be mortal. +For mortal interned strings: +- the 2 references from the interned dict (key & value) are excluded from + their refcount +- the deallocator (`unicode_dealloc`) removes the string from the interned dict +- at shutdown, when the interned dict is cleared, the references are added back + +As with any type, you should only immortalize strings that will live until +interpreter shutdown. +We currently also immortalize strings contained in code objects and similar, +specifically in the compiler and in `marshal`. +These are “close enough” to immortal: even in use cases like hot reloading +or `eval`-ing user input, the number of distinct identifiers and string +constants is expected to stay low. + + +## Internal API + +We have the following *internal* API for interning: + +- `_PyUnicode_InternMortal`: just intern the string +- `_PyUnicode_InternImmortal`: intern, and immortalize the result +- `_PyUnicode_InternStatic`: intern a static singleton (`_Py_STR`, `_Py_ID` + or one-byte). Not for general use. + +All take an interpreter state, and a pointer to a `PyObject*` which they +modify in place. + +The functions take ownership of (“steal”) the reference to their argument, +and update the argument with a *new* reference. +This means: +- They're “reference neutral”. +- They must not be called with a borrowed reference. + + +## State + +The intern state (retrieved by `PyUnicode_CHECK_INTERNED(s)`; +stored in `_PyUnicode_STATE(s).interned`) can be: + +- `SSTATE_NOT_INTERNED` (defined as 0, which is useful in a boolean context) +- `SSTATE_INTERNED_MORTAL` (1) +- `SSTATE_INTERNED_IMMORTAL` (2) +- `SSTATE_INTERNED_IMMORTAL_STATIC` (3) + +The valid transitions between these states are: + +- For dynamically allocated strings: + + - 0 -> 1 (`_PyUnicode_InternMortal`) + - 1 -> 2 or 0 -> 2 (`_PyUnicode_InternImmortal`) + + Using `_PyUnicode_InternStatic` on these is an error; the other cases + don't change the state. + +- Singletons are interned (0 -> 3) at runtime init; + after that all interning functions don't change the state. diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 00000000000000..7572745c851847 --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,181 @@ +import io +import sys +from threading import RLock +from time import sleep, time + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. +# Messages longer than this will be be truncated by logcat. This limit has already +# been reduced at least once in the history of Android (from 4076 to 4068 between +# API level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then we may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + global logcat + logcat = Logcat(android_log_write) + + sys.stdout = TextLogStream( + stdout_prio, "python.stdout", sys.stdout.fileno()) + sys.stderr = TextLogStream( + stderr_prio, "python.stderr", sys.stderr.fileno()) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, prio, tag, fileno=None, **kwargs): + # The default is surrogateescape for stdout and backslashreplace for + # stderr, but in the context of an Android log, readability is more + # important than reversibility. + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("errors", "backslashreplace") + + super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs) + self._lock = RLock() + self._pending_bytes = [] + self._pending_bytes_count = 0 + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string into lines first. Note that "".splitlines() == [], so + # nothing will be logged for an empty string. + with self._lock: + for line in s.splitlines(keepends=True): + while line: + chunk = line[:MAX_CHARS_PER_WRITE] + line = line[MAX_CHARS_PER_WRITE:] + self._write_chunk(chunk) + + return len(s) + + # The size and behavior of TextIOWrapper's buffer is not part of its public + # API, so we handle buffering ourselves to avoid truncation. + def _write_chunk(self, s): + b = s.encode(self.encoding, self.errors) + if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE: + self.flush() + + self._pending_bytes.append(b) + self._pending_bytes_count += len(b) + if ( + self.write_through + or b.endswith(b"\n") + or self._pending_bytes_count > MAX_BYTES_PER_WRITE + ): + self.flush() + + def flush(self): + with self._lock: + self.buffer.write(b"".join(self._pending_bytes)) + self._pending_bytes.clear() + self._pending_bytes_count = 0 + + # Since this is a line-based logging system, line buffering cannot be turned + # off, i.e. a newline always causes a flush. + @property + def line_buffering(self): + return True + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, prio, tag, fileno=None): + self.prio = prio + self.tag = tag + self._fileno = fileno + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + logcat.write(self.prio, self.tag, b) + return len(b) + + # This is needed by the test suite --timeout option, which uses faulthandler. + def fileno(self): + if self._fileno is None: + raise io.UnsupportedOperation("fileno") + return self._fileno + + +# When a large volume of data is written to logcat at once, e.g. when a test +# module fails in --verbose3 mode, there's a risk of overflowing logcat's own +# buffer and losing messages. We avoid this by imposing a rate limit using the +# token bucket algorithm, based on a conservative estimate of how fast `adb +# logcat` can consume data. +MAX_BYTES_PER_SECOND = 1024 * 1024 + +# The logcat buffer size of a device can be determined by running `logcat -g`. +# We set the token bucket size to half of the buffer size of our current minimum +# API level, because other things on the system will be producing messages as +# well. +BUCKET_SIZE = 128 * 1024 + +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 +PER_MESSAGE_OVERHEAD = 28 + + +class Logcat: + def __init__(self, android_log_write): + self.android_log_write = android_log_write + self._lock = RLock() + self._bucket_level = 0 + self._prev_write_time = time() + + def write(self, prio, tag, message): + # Encode null bytes using "modified UTF-8" to avoid them truncating the + # message. + message = message.replace(b"\x00", b"\xc0\x80") + + with self._lock: + now = time() + self._bucket_level += ( + (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) + + # If the bucket level is still below zero, the clock must have gone + # backwards, so reset it to zero and continue. + self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE)) + self._prev_write_time = now + + self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) + if self._bucket_level < 0: + sleep(-self._bucket_level / MAX_BYTES_PER_SECOND) + + self.android_log_write(prio, tag, message) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 601107d2d86771..06667b7434ccef 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", "Buffer", + "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -85,6 +85,10 @@ def _f(): pass dict_items = type({}.items()) ## misc ## mappingproxy = type(type.__dict__) +def _get_framelocalsproxy(): + return type(sys._getframe().f_locals) +framelocalsproxy = _get_framelocalsproxy() +del _get_framelocalsproxy generator = type((lambda: (yield))()) ## coroutine ## async def _coro(): pass @@ -481,9 +485,10 @@ def __new__(cls, origin, args): def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() + from annotationlib import value_to_string return (f'collections.abc.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' - f'{_type_repr(self.__args__[-1])}]') + f'[[{", ".join([value_to_string(a) for a in self.__args__[:-1]])}], ' + f'{value_to_string(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ @@ -520,23 +525,6 @@ def _is_param_expr(obj): names = ('ParamSpec', '_ConcatenateGenericAlias') return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names) -def _type_repr(obj): - """Return the repr() of an object, special-casing types (internal helper). - - Copied from :mod:`typing` since collections.abc - shouldn't depend on that module. - (Keep this roughly in sync with the typing version.) - """ - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is Ellipsis: - return '...' - if isinstance(obj, FunctionType): - return obj.__name__ - return repr(obj) - class Callable(metaclass=ABCMeta): @@ -836,6 +824,7 @@ def __eq__(self, other): __reversed__ = None Mapping.register(mappingproxy) +Mapping.register(framelocalsproxy) class MappingView(Sized): @@ -973,7 +962,7 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E: D[k] = E[k] + If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v In either case, this is followed by: for k, v in F.items(): D[k] = v ''' @@ -1068,40 +1057,10 @@ def count(self, value): Sequence.register(tuple) Sequence.register(str) +Sequence.register(bytes) Sequence.register(range) Sequence.register(memoryview) -class _DeprecateByteStringMeta(ABCMeta): - def __new__(cls, name, bases, namespace, **kwargs): - if name != "ByteString": - import warnings - - warnings._deprecated( - "collections.abc.ByteString", - remove=(3, 14), - ) - return super().__new__(cls, name, bases, namespace, **kwargs) - - def __instancecheck__(cls, instance): - import warnings - - warnings._deprecated( - "collections.abc.ByteString", - remove=(3, 14), - ) - return super().__instancecheck__(instance) - -class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): - """This unifies bytes and bytearray. - - XXX Should add all their methods. - """ - - __slots__ = () - -ByteString.register(bytes) -ByteString.register(bytearray) - class MutableSequence(Sequence): """All the operations on a read-write sequence. @@ -1170,4 +1129,4 @@ def __iadd__(self, values): MutableSequence.register(list) -MutableSequence.register(bytearray) # Multiply inheriting, see ByteString +MutableSequence.register(bytearray) diff --git a/Lib/_colorize.py b/Lib/_colorize.py new file mode 100644 index 00000000000000..845fb57a90abb8 --- /dev/null +++ b/Lib/_colorize.py @@ -0,0 +1,64 @@ +import io +import os +import sys + +COLORIZE = True + + +class ANSIColors: + BOLD_GREEN = "\x1b[1;32m" + BOLD_MAGENTA = "\x1b[1;35m" + BOLD_RED = "\x1b[1;31m" + GREEN = "\x1b[32m" + GREY = "\x1b[90m" + MAGENTA = "\x1b[35m" + RED = "\x1b[31m" + RESET = "\x1b[0m" + YELLOW = "\x1b[33m" + + +NoColors = ANSIColors() + +for attr in dir(NoColors): + if not attr.startswith("__"): + setattr(NoColors, attr, "") + + +def get_colors(colorize: bool = False) -> ANSIColors: + if colorize or can_colorize(): + return ANSIColors() + else: + return NoColors + + +def can_colorize() -> bool: + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + if not sys.flags.ignore_environment: + if os.environ.get("PYTHON_COLORS") == "0": + return False + if os.environ.get("PYTHON_COLORS") == "1": + return True + if "NO_COLOR" in os.environ: + return False + if not COLORIZE: + return False + if not sys.flags.ignore_environment: + if "FORCE_COLOR" in os.environ: + return True + if os.environ.get("TERM") == "dumb": + return False + + if not hasattr(sys.stderr, "fileno"): + return False + + try: + return os.isatty(sys.stderr.fileno()) + except io.UnsupportedOperation: + return sys.stderr.isatty() diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py new file mode 100644 index 00000000000000..20467a7c2bcaeb --- /dev/null +++ b/Lib/_ios_support.py @@ -0,0 +1,71 @@ +import sys +try: + from ctypes import cdll, c_void_p, c_char_p, util +except ImportError: + # ctypes is an optional module. If it's not present, we're limited in what + # we can tell about the system, but we don't want to prevent the module + # from working. + print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr) + objc = None +else: + # ctypes is available. Load the ObjC library, and wrap the objc_getClass, + # sel_registerName methods + lib = util.find_library("objc") + if lib is None: + # Failed to load the objc library + raise ImportError("ObjC runtime library couldn't be loaded") + + objc = cdll.LoadLibrary(lib) + objc.objc_getClass.restype = c_void_p + objc.objc_getClass.argtypes = [c_char_p] + objc.sel_registerName.restype = c_void_p + objc.sel_registerName.argtypes = [c_char_p] + + +def get_platform_ios(): + # Determine if this is a simulator using the multiarch value + is_simulator = sys.implementation._multiarch.endswith("simulator") + + # We can't use ctypes; abort + if not objc: + return None + + # Most of the methods return ObjC objects + objc.objc_msgSend.restype = c_void_p + # All the methods used have no arguments. + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + + # Equivalent of: + # device = [UIDevice currentDevice] + UIDevice = objc.objc_getClass(b"UIDevice") + SEL_currentDevice = objc.sel_registerName(b"currentDevice") + device = objc.objc_msgSend(UIDevice, SEL_currentDevice) + + # Equivalent of: + # device_systemVersion = [device systemVersion] + SEL_systemVersion = objc.sel_registerName(b"systemVersion") + device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion) + + # Equivalent of: + # device_systemName = [device systemName] + SEL_systemName = objc.sel_registerName(b"systemName") + device_systemName = objc.objc_msgSend(device, SEL_systemName) + + # Equivalent of: + # device_model = [device model] + SEL_model = objc.sel_registerName(b"model") + device_model = objc.objc_msgSend(device, SEL_model) + + # UTF8String returns a const char*; + SEL_UTF8String = objc.sel_registerName(b"UTF8String") + objc.objc_msgSend.restype = c_char_p + + # Equivalent of: + # system = [device_systemName UTF8String] + # release = [device_systemVersion UTF8String] + # model = [device_model UTF8String] + system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode() + release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode() + model = objc.objc_msgSend(device_model, SEL_UTF8String).decode() + + return system, release, model, is_simulator diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index fdb099bd0c2ecf..cda3c340c322f3 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -6,6 +6,9 @@ "RESUME": [ "RESUME_CHECK", ], + "LOAD_CONST": [ + "LOAD_CONST_IMMORTAL", + ], "TO_BOOL": [ "TO_BOOL_ALWAYS_TRUE", "TO_BOOL_BOOL", @@ -62,6 +65,7 @@ "LOAD_ATTR_WITH_HINT", "LOAD_ATTR_SLOT", "LOAD_ATTR_CLASS", + "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", "LOAD_ATTR_PROPERTY", "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", "LOAD_ATTR_METHOD_WITH_VALUES", @@ -75,6 +79,10 @@ "COMPARE_OP_INT", "COMPARE_OP_STR", ], + "CONTAINS_OP": [ + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ], "FOR_ITER": [ "FOR_ITER_LIST", "FOR_ITER_TUPLE", @@ -84,7 +92,6 @@ "CALL": [ "CALL_BOUND_METHOD_EXACT_ARGS", "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", "CALL_TYPE_1", "CALL_STR_1", "CALL_TUPLE_1", @@ -100,6 +107,14 @@ "CALL_METHOD_DESCRIPTOR_NOARGS", "CALL_METHOD_DESCRIPTOR_FAST", "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", + ], + "CALL_KW": [ + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", ], } @@ -119,61 +134,70 @@ 'BINARY_SUBSCR_TUPLE_INT': 161, 'CALL_ALLOC_AND_ENTER_INIT': 162, 'CALL_BOUND_METHOD_EXACT_ARGS': 163, - 'CALL_BUILTIN_CLASS': 164, - 'CALL_BUILTIN_FAST': 165, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 166, - 'CALL_BUILTIN_O': 167, - 'CALL_ISINSTANCE': 168, - 'CALL_LEN': 169, - 'CALL_LIST_APPEND': 170, - 'CALL_METHOD_DESCRIPTOR_FAST': 171, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 172, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 173, - 'CALL_METHOD_DESCRIPTOR_O': 174, - 'CALL_PY_EXACT_ARGS': 175, - 'CALL_PY_WITH_DEFAULTS': 176, - 'CALL_STR_1': 177, - 'CALL_TUPLE_1': 178, - 'CALL_TYPE_1': 179, - 'COMPARE_OP_FLOAT': 180, - 'COMPARE_OP_INT': 181, - 'COMPARE_OP_STR': 182, - 'FOR_ITER_GEN': 183, - 'FOR_ITER_LIST': 184, - 'FOR_ITER_RANGE': 185, - 'FOR_ITER_TUPLE': 186, - 'LOAD_ATTR_CLASS': 187, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 188, - 'LOAD_ATTR_INSTANCE_VALUE': 189, - 'LOAD_ATTR_METHOD_LAZY_DICT': 190, - 'LOAD_ATTR_METHOD_NO_DICT': 191, - 'LOAD_ATTR_METHOD_WITH_VALUES': 192, - 'LOAD_ATTR_MODULE': 193, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 194, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 195, - 'LOAD_ATTR_PROPERTY': 196, - 'LOAD_ATTR_SLOT': 197, - 'LOAD_ATTR_WITH_HINT': 198, - 'LOAD_GLOBAL_BUILTIN': 199, - 'LOAD_GLOBAL_MODULE': 200, - 'LOAD_SUPER_ATTR_ATTR': 201, - 'LOAD_SUPER_ATTR_METHOD': 202, - 'RESUME_CHECK': 203, - 'SEND_GEN': 204, - 'STORE_ATTR_INSTANCE_VALUE': 205, - 'STORE_ATTR_SLOT': 206, - 'STORE_ATTR_WITH_HINT': 207, - 'STORE_SUBSCR_DICT': 208, - 'STORE_SUBSCR_LIST_INT': 209, - 'TO_BOOL_ALWAYS_TRUE': 210, - 'TO_BOOL_BOOL': 211, - 'TO_BOOL_INT': 212, - 'TO_BOOL_LIST': 213, - 'TO_BOOL_NONE': 214, - 'TO_BOOL_STR': 215, - 'UNPACK_SEQUENCE_LIST': 216, - 'UNPACK_SEQUENCE_TUPLE': 217, - 'UNPACK_SEQUENCE_TWO_TUPLE': 218, + 'CALL_BOUND_METHOD_GENERAL': 164, + 'CALL_BUILTIN_CLASS': 165, + 'CALL_BUILTIN_FAST': 166, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, + 'CALL_BUILTIN_O': 168, + 'CALL_ISINSTANCE': 169, + 'CALL_KW_BOUND_METHOD': 170, + 'CALL_KW_NON_PY': 171, + 'CALL_KW_PY': 172, + 'CALL_LEN': 173, + 'CALL_LIST_APPEND': 174, + 'CALL_METHOD_DESCRIPTOR_FAST': 175, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 176, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 177, + 'CALL_METHOD_DESCRIPTOR_O': 178, + 'CALL_NON_PY_GENERAL': 179, + 'CALL_PY_EXACT_ARGS': 180, + 'CALL_PY_GENERAL': 181, + 'CALL_STR_1': 182, + 'CALL_TUPLE_1': 183, + 'CALL_TYPE_1': 184, + 'COMPARE_OP_FLOAT': 185, + 'COMPARE_OP_INT': 186, + 'COMPARE_OP_STR': 187, + 'CONTAINS_OP_DICT': 188, + 'CONTAINS_OP_SET': 189, + 'FOR_ITER_GEN': 190, + 'FOR_ITER_LIST': 191, + 'FOR_ITER_RANGE': 192, + 'FOR_ITER_TUPLE': 193, + 'LOAD_ATTR_CLASS': 194, + 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 195, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 196, + 'LOAD_ATTR_INSTANCE_VALUE': 197, + 'LOAD_ATTR_METHOD_LAZY_DICT': 198, + 'LOAD_ATTR_METHOD_NO_DICT': 199, + 'LOAD_ATTR_METHOD_WITH_VALUES': 200, + 'LOAD_ATTR_MODULE': 201, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 202, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 203, + 'LOAD_ATTR_PROPERTY': 204, + 'LOAD_ATTR_SLOT': 205, + 'LOAD_ATTR_WITH_HINT': 206, + 'LOAD_CONST_IMMORTAL': 207, + 'LOAD_GLOBAL_BUILTIN': 208, + 'LOAD_GLOBAL_MODULE': 209, + 'LOAD_SUPER_ATTR_ATTR': 210, + 'LOAD_SUPER_ATTR_METHOD': 211, + 'RESUME_CHECK': 212, + 'SEND_GEN': 213, + 'STORE_ATTR_INSTANCE_VALUE': 214, + 'STORE_ATTR_SLOT': 215, + 'STORE_ATTR_WITH_HINT': 216, + 'STORE_SUBSCR_DICT': 217, + 'STORE_SUBSCR_LIST_INT': 218, + 'TO_BOOL_ALWAYS_TRUE': 219, + 'TO_BOOL_BOOL': 220, + 'TO_BOOL_INT': 221, + 'TO_BOOL_LIST': 222, + 'TO_BOOL_NONE': 223, + 'TO_BOOL_STR': 224, + 'UNPACK_SEQUENCE_LIST': 225, + 'UNPACK_SEQUENCE_TUPLE': 226, + 'UNPACK_SEQUENCE_TWO_TUPLE': 227, } opmap = { @@ -181,153 +205,148 @@ 'RESERVED': 17, 'RESUME': 149, 'INSTRUMENTED_LINE': 254, - 'BEFORE_ASYNC_WITH': 1, - 'BEFORE_WITH': 2, - 'BINARY_SLICE': 4, - 'BINARY_SUBSCR': 5, - 'CHECK_EG_MATCH': 6, - 'CHECK_EXC_MATCH': 7, - 'CLEANUP_THROW': 8, - 'DELETE_SUBSCR': 9, - 'END_ASYNC_FOR': 10, - 'END_FOR': 11, - 'END_SEND': 12, - 'EXIT_INIT_CHECK': 13, - 'FORMAT_SIMPLE': 14, - 'FORMAT_WITH_SPEC': 15, - 'GET_AITER': 16, - 'GET_ANEXT': 18, - 'GET_ITER': 19, - 'GET_LEN': 20, - 'GET_YIELD_FROM_ITER': 21, - 'INTERPRETER_EXIT': 22, - 'LOAD_ASSERTION_ERROR': 23, - 'LOAD_BUILD_CLASS': 24, - 'LOAD_LOCALS': 25, - 'MAKE_FUNCTION': 26, - 'MATCH_KEYS': 27, - 'MATCH_MAPPING': 28, - 'MATCH_SEQUENCE': 29, - 'NOP': 30, - 'POP_EXCEPT': 31, - 'POP_TOP': 32, - 'PUSH_EXC_INFO': 33, - 'PUSH_NULL': 34, - 'RETURN_GENERATOR': 35, - 'RETURN_VALUE': 36, - 'SETUP_ANNOTATIONS': 37, - 'STORE_SLICE': 38, - 'STORE_SUBSCR': 39, - 'TO_BOOL': 40, - 'UNARY_INVERT': 41, - 'UNARY_NEGATIVE': 42, - 'UNARY_NOT': 43, - 'WITH_EXCEPT_START': 44, - 'BINARY_OP': 45, - 'BUILD_CONST_KEY_MAP': 46, - 'BUILD_LIST': 47, - 'BUILD_MAP': 48, - 'BUILD_SET': 49, - 'BUILD_SLICE': 50, - 'BUILD_STRING': 51, - 'BUILD_TUPLE': 52, - 'CALL': 53, - 'CALL_FUNCTION_EX': 54, - 'CALL_INTRINSIC_1': 55, - 'CALL_INTRINSIC_2': 56, - 'CALL_KW': 57, - 'COMPARE_OP': 58, - 'CONTAINS_OP': 59, - 'CONVERT_VALUE': 60, - 'COPY': 61, - 'COPY_FREE_VARS': 62, - 'DELETE_ATTR': 63, - 'DELETE_DEREF': 64, - 'DELETE_FAST': 65, - 'DELETE_GLOBAL': 66, - 'DELETE_NAME': 67, - 'DICT_MERGE': 68, - 'DICT_UPDATE': 69, - 'ENTER_EXECUTOR': 70, - 'EXTENDED_ARG': 71, - 'FOR_ITER': 72, - 'GET_AWAITABLE': 73, - 'IMPORT_FROM': 74, - 'IMPORT_NAME': 75, - 'IS_OP': 76, - 'JUMP_BACKWARD': 77, - 'JUMP_BACKWARD_NO_INTERRUPT': 78, - 'JUMP_FORWARD': 79, - 'LIST_APPEND': 80, - 'LIST_EXTEND': 81, - 'LOAD_ATTR': 82, - 'LOAD_CONST': 83, - 'LOAD_DEREF': 84, - 'LOAD_FAST': 85, - 'LOAD_FAST_AND_CLEAR': 86, - 'LOAD_FAST_CHECK': 87, - 'LOAD_FAST_LOAD_FAST': 88, - 'LOAD_FROM_DICT_OR_DEREF': 89, - 'LOAD_FROM_DICT_OR_GLOBALS': 90, - 'LOAD_GLOBAL': 91, - 'LOAD_NAME': 92, - 'LOAD_SUPER_ATTR': 93, - 'MAKE_CELL': 94, - 'MAP_ADD': 95, - 'MATCH_CLASS': 96, - 'POP_JUMP_IF_FALSE': 97, - 'POP_JUMP_IF_NONE': 98, - 'POP_JUMP_IF_NOT_NONE': 99, - 'POP_JUMP_IF_TRUE': 100, - 'RAISE_VARARGS': 101, - 'RERAISE': 102, - 'RETURN_CONST': 103, - 'SEND': 104, - 'SET_ADD': 105, - 'SET_FUNCTION_ATTRIBUTE': 106, - 'SET_UPDATE': 107, - 'STORE_ATTR': 108, - 'STORE_DEREF': 109, - 'STORE_FAST': 110, - 'STORE_FAST_LOAD_FAST': 111, - 'STORE_FAST_STORE_FAST': 112, - 'STORE_GLOBAL': 113, - 'STORE_NAME': 114, - 'SWAP': 115, - 'UNPACK_EX': 116, - 'UNPACK_SEQUENCE': 117, - 'YIELD_VALUE': 118, - 'INSTRUMENTED_RESUME': 236, + 'ENTER_EXECUTOR': 255, + 'BINARY_SLICE': 1, + 'BINARY_SUBSCR': 2, + 'CHECK_EG_MATCH': 4, + 'CHECK_EXC_MATCH': 5, + 'CLEANUP_THROW': 6, + 'DELETE_SUBSCR': 7, + 'END_ASYNC_FOR': 8, + 'END_FOR': 9, + 'END_SEND': 10, + 'EXIT_INIT_CHECK': 11, + 'FORMAT_SIMPLE': 12, + 'FORMAT_WITH_SPEC': 13, + 'GET_AITER': 14, + 'GET_ANEXT': 15, + 'GET_ITER': 16, + 'GET_LEN': 18, + 'GET_YIELD_FROM_ITER': 19, + 'INTERPRETER_EXIT': 20, + 'LOAD_BUILD_CLASS': 21, + 'LOAD_LOCALS': 22, + 'MAKE_FUNCTION': 23, + 'MATCH_KEYS': 24, + 'MATCH_MAPPING': 25, + 'MATCH_SEQUENCE': 26, + 'NOP': 27, + 'POP_EXCEPT': 28, + 'POP_TOP': 29, + 'PUSH_EXC_INFO': 30, + 'PUSH_NULL': 31, + 'RETURN_GENERATOR': 32, + 'RETURN_VALUE': 33, + 'SETUP_ANNOTATIONS': 34, + 'STORE_SLICE': 35, + 'STORE_SUBSCR': 36, + 'TO_BOOL': 37, + 'UNARY_INVERT': 38, + 'UNARY_NEGATIVE': 39, + 'UNARY_NOT': 40, + 'WITH_EXCEPT_START': 41, + 'BINARY_OP': 42, + 'BUILD_LIST': 43, + 'BUILD_MAP': 44, + 'BUILD_SET': 45, + 'BUILD_SLICE': 46, + 'BUILD_STRING': 47, + 'BUILD_TUPLE': 48, + 'CALL': 49, + 'CALL_FUNCTION_EX': 50, + 'CALL_INTRINSIC_1': 51, + 'CALL_INTRINSIC_2': 52, + 'CALL_KW': 53, + 'COMPARE_OP': 54, + 'CONTAINS_OP': 55, + 'CONVERT_VALUE': 56, + 'COPY': 57, + 'COPY_FREE_VARS': 58, + 'DELETE_ATTR': 59, + 'DELETE_DEREF': 60, + 'DELETE_FAST': 61, + 'DELETE_GLOBAL': 62, + 'DELETE_NAME': 63, + 'DICT_MERGE': 64, + 'DICT_UPDATE': 65, + 'EXTENDED_ARG': 66, + 'FOR_ITER': 67, + 'GET_AWAITABLE': 68, + 'IMPORT_FROM': 69, + 'IMPORT_NAME': 70, + 'IS_OP': 71, + 'JUMP_BACKWARD': 72, + 'JUMP_BACKWARD_NO_INTERRUPT': 73, + 'JUMP_FORWARD': 74, + 'LIST_APPEND': 75, + 'LIST_EXTEND': 76, + 'LOAD_ATTR': 77, + 'LOAD_COMMON_CONSTANT': 78, + 'LOAD_CONST': 79, + 'LOAD_DEREF': 80, + 'LOAD_FAST': 81, + 'LOAD_FAST_AND_CLEAR': 82, + 'LOAD_FAST_CHECK': 83, + 'LOAD_FAST_LOAD_FAST': 84, + 'LOAD_FROM_DICT_OR_DEREF': 85, + 'LOAD_FROM_DICT_OR_GLOBALS': 86, + 'LOAD_GLOBAL': 87, + 'LOAD_NAME': 88, + 'LOAD_SMALL_INT': 89, + 'LOAD_SPECIAL': 90, + 'LOAD_SUPER_ATTR': 91, + 'MAKE_CELL': 92, + 'MAP_ADD': 93, + 'MATCH_CLASS': 94, + 'POP_JUMP_IF_FALSE': 95, + 'POP_JUMP_IF_NONE': 96, + 'POP_JUMP_IF_NOT_NONE': 97, + 'POP_JUMP_IF_TRUE': 98, + 'RAISE_VARARGS': 99, + 'RERAISE': 100, + 'SEND': 101, + 'SET_ADD': 102, + 'SET_FUNCTION_ATTRIBUTE': 103, + 'SET_UPDATE': 104, + 'STORE_ATTR': 105, + 'STORE_DEREF': 106, + 'STORE_FAST': 107, + 'STORE_FAST_LOAD_FAST': 108, + 'STORE_FAST_STORE_FAST': 109, + 'STORE_GLOBAL': 110, + 'STORE_NAME': 111, + 'SWAP': 112, + 'UNPACK_EX': 113, + 'UNPACK_SEQUENCE': 114, + 'YIELD_VALUE': 115, 'INSTRUMENTED_END_FOR': 237, 'INSTRUMENTED_END_SEND': 238, - 'INSTRUMENTED_RETURN_VALUE': 239, - 'INSTRUMENTED_RETURN_CONST': 240, - 'INSTRUMENTED_YIELD_VALUE': 241, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 242, - 'INSTRUMENTED_FOR_ITER': 243, - 'INSTRUMENTED_CALL': 244, - 'INSTRUMENTED_CALL_KW': 245, - 'INSTRUMENTED_CALL_FUNCTION_EX': 246, - 'INSTRUMENTED_INSTRUCTION': 247, - 'INSTRUMENTED_JUMP_FORWARD': 248, - 'INSTRUMENTED_JUMP_BACKWARD': 249, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 239, + 'INSTRUMENTED_FOR_ITER': 240, + 'INSTRUMENTED_CALL_KW': 241, + 'INSTRUMENTED_CALL_FUNCTION_EX': 242, + 'INSTRUMENTED_INSTRUCTION': 243, + 'INSTRUMENTED_JUMP_FORWARD': 244, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 245, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 246, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 247, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 248, + 'INSTRUMENTED_RESUME': 249, + 'INSTRUMENTED_RETURN_VALUE': 250, + 'INSTRUMENTED_YIELD_VALUE': 251, + 'INSTRUMENTED_CALL': 252, + 'INSTRUMENTED_JUMP_BACKWARD': 253, 'JUMP': 256, - 'JUMP_NO_INTERRUPT': 257, - 'LOAD_CLOSURE': 258, - 'LOAD_METHOD': 259, - 'LOAD_SUPER_METHOD': 260, - 'LOAD_ZERO_SUPER_ATTR': 261, - 'LOAD_ZERO_SUPER_METHOD': 262, - 'POP_BLOCK': 263, - 'SETUP_CLEANUP': 264, - 'SETUP_FINALLY': 265, - 'SETUP_WITH': 266, - 'STORE_FAST_MAYBE_NULL': 267, + 'JUMP_IF_FALSE': 257, + 'JUMP_IF_TRUE': 258, + 'JUMP_NO_INTERRUPT': 259, + 'LOAD_CLOSURE': 260, + 'POP_BLOCK': 261, + 'SETUP_CLEANUP': 262, + 'SETUP_FINALLY': 263, + 'SETUP_WITH': 264, + 'STORE_FAST_MAYBE_NULL': 265, } -HAVE_ARGUMENT = 44 -MIN_INSTRUMENTED_OPCODE = 236 +HAVE_ARGUMENT = 41 +MIN_INSTRUMENTED_OPCODE = 237 diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index 54c12d3b2f3f16..78e03e32896740 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1,8 +1,4 @@ -"""Concrete date/time and related types. - -See http://www.iana.org/time-zones/repository/tz-link.html for -time zone and DST data sources. -""" +"""Pure Python implementation of the datetime module.""" __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") @@ -204,6 +200,28 @@ def _format_offset(off, sep=':'): s += '.%06d' % ss.microseconds return s +_normalize_century = None +def _need_normalize_century(): + global _normalize_century + if _normalize_century is None: + try: + _normalize_century = ( + _time.strftime("%Y", (99, 1, 1, 0, 0, 0, 0, 1, 0)) != "0099") + except ValueError: + _normalize_century = True + return _normalize_century + +_supports_c99 = None +def _can_support_c99(): + global _supports_c99 + if _supports_c99 is None: + try: + _supports_c99 = ( + _time.strftime("%F", (1900, 1, 1, 0, 0, 0, 0, 1, 0)) == "1900-01-01") + except ValueError: + _supports_c99 = False + return _supports_c99 + # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. @@ -261,6 +279,20 @@ def _wrap_strftime(object, format, timetuple): # strftime is going to have at this: escape % Zreplace = s.replace('%', '%%') newformat.append(Zreplace) + # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so + # year 1000 for %G can go on the fast path. + elif ((ch in 'YG' or ch in 'FC' and _can_support_c99()) and + object.year < 1000 and _need_normalize_century()): + if ch == 'G': + year = int(_time.strftime("%G", timetuple)) + else: + year = object.year + if ch == 'C': + push('{:02}'.format(year // 100)) + else: + push('{:04}'.format(year)) + if ch == 'F': + push('-{:02}-{:02}'.format(*timetuple[1:3])) else: push('%') push(ch) @@ -431,6 +463,17 @@ def _parse_isoformat_time(tstr): time_comps = _parse_hh_mm_ss_ff(timestr) + hour, minute, second, microsecond = time_comps + became_next_day = False + error_from_components = False + if (hour == 24): + if all(time_comp == 0 for time_comp in time_comps[1:]): + hour = 0 + time_comps[0] = hour + became_next_day = True + else: + error_from_components = True + tzi = None if tz_pos == len_str and tstr[-1] == 'Z': tzi = timezone.utc @@ -463,7 +506,7 @@ def _parse_isoformat_time(tstr): time_comps.append(tzi) - return time_comps + return time_comps, became_next_day, error_from_components # tuple[int, int, int] -> tuple[int, int, int] version of date.fromisocalendar def _isoweek_to_gregorian(year, week, day): @@ -556,10 +599,6 @@ def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): raise TypeError("tzinfo argument must be None or of a tzinfo subclass") -def _cmperror(x, y): - raise TypeError("can't compare '%s' to '%s'" % ( - type(x).__name__, type(y).__name__)) - def _divide_and_round(a, b): """divide a by b and round result to the nearest integer @@ -912,6 +951,7 @@ class date: fromtimestamp() today() fromordinal() + strptime() Operators: @@ -970,6 +1010,8 @@ def __new__(cls, year, month=None, day=None): @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." + if t is None: + raise TypeError("'NoneType' object cannot be interpreted as an integer") y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) @@ -1010,6 +1052,12 @@ def fromisocalendar(cls, year, week, day): This is the inverse of the date.isocalendar() function""" return cls(*_isoweek_to_gregorian(year, week, day)) + @classmethod + def strptime(cls, date_string, format): + """Parse a date string according to the given format (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_date(cls, date_string, format) + # Conversions to string def __repr__(self): @@ -1113,32 +1161,33 @@ def replace(self, year=None, month=None, day=None): # Comparisons of date objects with other. def __eq__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) == 0 return NotImplemented def __le__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) <= 0 return NotImplemented def __lt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) < 0 return NotImplemented def __ge__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) >= 0 return NotImplemented def __gt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) > 0 return NotImplemented def _cmp(self, other): assert isinstance(other, date) + assert not isinstance(other, datetime) y, m, d = self._year, self._month, self._day y2, m2, d2 = other._year, other._month, other._day return _cmp((y, m, d), (y2, m2, d2)) @@ -1329,6 +1378,7 @@ class time: Constructors: __new__() + strptime() Operators: @@ -1387,6 +1437,12 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold self._fold = fold return self + @classmethod + def strptime(cls, date_string, format): + """string, format -> new time parsed from a string (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_time(cls, date_string, format) + # Read-only field accessors @property def hour(self): @@ -1557,7 +1613,7 @@ def fromisoformat(cls, time_string): time_string = time_string.removeprefix('T') try: - return cls(*_parse_isoformat_time(time_string)) + return cls(*_parse_isoformat_time(time_string)[0]) except Exception: raise ValueError(f'Invalid isoformat string: {time_string!r}') @@ -1871,10 +1927,27 @@ def fromisoformat(cls, date_string): if tstr: try: - time_components = _parse_isoformat_time(tstr) + time_components, became_next_day, error_from_components = _parse_isoformat_time(tstr) except ValueError: raise ValueError( f'Invalid isoformat string: {date_string!r}') from None + else: + if error_from_components: + raise ValueError("minute, second, and microsecond must be 0 when hour is 24") + + if became_next_day: + year, month, day = date_components + # Only wrap day/month when it was previously valid + if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + # Calculate midnight of the next day + day += 1 + if day > days_in_month: + day = 1 + month += 1 + if month > 12: + month = 1 + year += 1 + date_components = [year, month, day] else: time_components = [0, 0, 0, 0, None] @@ -2093,7 +2166,7 @@ def __str__(self): def strptime(cls, date_string, format): 'string, format -> new datetime parsed from a string (like time.strptime()).' import _strptime - return _strptime._strptime_datetime(cls, date_string, format) + return _strptime._strptime_datetime_datetime(cls, date_string, format) def utcoffset(self): """Return the timezone offset as timedelta positive east of UTC (negative west of @@ -2137,42 +2210,32 @@ def dst(self): def __eq__(self, other): if isinstance(other, datetime): return self._cmp(other, allow_mixed=True) == 0 - elif not isinstance(other, date): - return NotImplemented else: - return False + return NotImplemented def __le__(self, other): if isinstance(other, datetime): return self._cmp(other) <= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __lt__(self, other): if isinstance(other, datetime): return self._cmp(other) < 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __ge__(self, other): if isinstance(other, datetime): return self._cmp(other) >= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __gt__(self, other): if isinstance(other, datetime): return self._cmp(other) > 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def _cmp(self, other, allow_mixed=False): assert isinstance(other, datetime) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 2692f2fcba45bf..5b60570c6c592a 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -13,104 +13,7 @@ # bug) and will be backported. At this point the spec is stabilizing # and the updates are becoming fewer, smaller, and less significant. -""" -This is an implementation of decimal floating point arithmetic based on -the General Decimal Arithmetic Specification: - - http://speleotrove.com/decimal/decarith.html - -and IEEE standard 854-1987: - - http://en.wikipedia.org/wiki/IEEE_854-1987 - -Decimal floating point has finite precision with arbitrarily large bounds. - -The purpose of this module is to support arithmetic using familiar -"schoolhouse" rules and to avoid some of the tricky representation -issues associated with binary floating point. The package is especially -useful for financial applications or for contexts where users have -expectations that are at odds with binary floating point (for instance, -in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead -of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected -Decimal('0.00')). - -Here are some examples of using the decimal module: - ->>> from decimal import * ->>> setcontext(ExtendedContext) ->>> Decimal(0) -Decimal('0') ->>> Decimal('1') -Decimal('1') ->>> Decimal('-.0123') -Decimal('-0.0123') ->>> Decimal(123456) -Decimal('123456') ->>> Decimal('123.45e12345678') -Decimal('1.2345E+12345680') ->>> Decimal('1.33') + Decimal('1.27') -Decimal('2.60') ->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') -Decimal('-2.20') ->>> dig = Decimal(1) ->>> print(dig / Decimal(3)) -0.333333333 ->>> getcontext().prec = 18 ->>> print(dig / Decimal(3)) -0.333333333333333333 ->>> print(dig.sqrt()) -1 ->>> print(Decimal(3).sqrt()) -1.73205080756887729 ->>> print(Decimal(3) ** 123) -4.85192780976896427E+58 ->>> inf = Decimal(1) / Decimal(0) ->>> print(inf) -Infinity ->>> neginf = Decimal(-1) / Decimal(0) ->>> print(neginf) --Infinity ->>> print(neginf + inf) -NaN ->>> print(neginf * inf) --Infinity ->>> print(dig / 0) -Infinity ->>> getcontext().traps[DivisionByZero] = 1 ->>> print(dig / 0) -Traceback (most recent call last): - ... - ... - ... -decimal.DivisionByZero: x / 0 ->>> c = Context() ->>> c.traps[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> c.divide(Decimal(0), Decimal(0)) -Decimal('NaN') ->>> c.traps[InvalidOperation] = 1 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> print(c.divide(Decimal(0), Decimal(0))) -Traceback (most recent call last): - ... - ... - ... -decimal.InvalidOperation: 0 / 0 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> c.traps[InvalidOperation] = 0 ->>> print(c.divide(Decimal(0), Decimal(0))) -NaN ->>> print(c.flags[InvalidOperation]) -1 ->>> -""" +"""Python decimal arithmetic module""" __all__ = [ # Two major classes @@ -521,7 +424,7 @@ def sin(x): # numbers.py for more detail. class Decimal(object): - """Floating point class for decimal arithmetic.""" + """Floating-point class for decimal arithmetic.""" __slots__ = ('_exp','_int','_sign', '_is_special') # Generally, the value of the Decimal instance is given by @@ -679,6 +582,21 @@ def __new__(cls, value="0", context=None): raise TypeError("Cannot convert %r to Decimal" % value) + @classmethod + def from_number(cls, number): + """Converts a real number to a decimal number, exactly. + + >>> Decimal.from_number(314) # int + Decimal('314') + >>> Decimal.from_number(0.1) # float + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance + Decimal('3.14') + """ + if isinstance(number, (int, Decimal, float)): + return cls(number) + raise TypeError("Cannot convert %r to Decimal" % number) + @classmethod def from_float(cls, f): """Converts a float to a decimal number, exactly. @@ -2228,10 +2146,16 @@ def _power_exact(self, other, p): else: return None - if xc >= 10**p: + # An exact power of 10 is representable, but can convert to a + # string of any length. But an exact power of 10 shouldn't be + # possible at this point. + assert xc > 1, self + assert xc % 10 != 0, self + strxc = str(xc) + if len(strxc) > p: return None xe = -e-xe - return _dec_from_triple(0, str(xc), xe) + return _dec_from_triple(0, strxc, xe) # now y is positive; find m and n such that y = m/n if ye >= 0: @@ -2281,13 +2205,18 @@ def _power_exact(self, other, p): return None xc = xc**m xe *= m - if xc > 10**p: + # An exact power of 10 is representable, but can convert to a string + # of any length. But an exact power of 10 shouldn't be possible at + # this point. + assert xc > 1, self + assert xc % 10 != 0, self + str_xc = str(xc) + if len(str_xc) > p: return None # by this point the result *is* exactly representable # adjust the exponent to get as close as possible to the ideal # exponent, if necessary - str_xc = str(xc) if other._isinteger() and other._sign == 0: ideal_exponent = self._exp*int(other) zeros = min(xe-ideal_exponent, p-len(str_xc)) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 8a0d0dc4b1a0b8..42b0aea4e2eb2e 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -16,7 +16,7 @@ _setmode = None import io -from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) # noqa: F401 valid_seek_flags = {0, 1, 2} # Hardwired values if hasattr(os, 'SEEK_HOLE') : @@ -238,18 +238,11 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, result = raw try: line_buffering = False - if buffering == 1 or buffering < 0 and raw.isatty(): + if buffering == 1 or buffering < 0 and raw._isatty_open_only(): buffering = -1 line_buffering = True if buffering < 0: - buffering = DEFAULT_BUFFER_SIZE - try: - bs = os.fstat(raw.fileno()).st_blksize - except (OSError, AttributeError): - pass - else: - if bs > 1: - buffering = bs + buffering = raw._blksize if buffering < 0: raise ValueError("invalid buffering size") if buffering == 0: @@ -1197,7 +1190,8 @@ def _readinto(self, buf, read1): return written def tell(self): - return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos + # GH-95782: Keep return value non-negative + return max(_BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos, 0) def seek(self, pos, whence=0): if whence not in valid_seek_flags: @@ -1486,6 +1480,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): """ if self._fd >= 0: # Have to close the existing file first. + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1564,18 +1559,15 @@ def __init__(self, file, mode='r', closefd=True, opener=None): os.set_inheritable(fd, False) self._closefd = closefd - fdfstat = os.fstat(fd) + self._stat_atopen = os.fstat(fd) try: - if stat.S_ISDIR(fdfstat.st_mode): + if stat.S_ISDIR(self._stat_atopen.st_mode): raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) except AttributeError: # Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR # don't exist. pass - self._blksize = getattr(fdfstat, 'st_blksize', 0) - if self._blksize <= 1: - self._blksize = DEFAULT_BUFFER_SIZE if _setmode: # don't translate newlines (\r\n <=> \n) @@ -1592,6 +1584,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if e.errno != errno.ESPIPE: raise except: + self._stat_atopen = None if owned_fd is not None: os.close(owned_fd) raise @@ -1621,6 +1614,17 @@ def __repr__(self): return ('<%s name=%r mode=%r closefd=%r>' % (class_name, name, self.mode, self._closefd)) + @property + def _blksize(self): + if self._stat_atopen is None: + return DEFAULT_BUFFER_SIZE + + blksize = getattr(self._stat_atopen, "st_blksize", 0) + # WASI sets blsize to 0 + if not blksize: + return DEFAULT_BUFFER_SIZE + return blksize + def _checkReadable(self): if not self._readable: raise UnsupportedOperation('File not open for reading') @@ -1653,14 +1657,22 @@ def readall(self): """ self._checkClosed() self._checkReadable() - bufsize = DEFAULT_BUFFER_SIZE - try: - pos = os.lseek(self._fd, 0, SEEK_CUR) - end = os.fstat(self._fd).st_size - if end >= pos: - bufsize = end - pos + 1 - except OSError: - pass + if self._stat_atopen is None or self._stat_atopen.st_size <= 0: + bufsize = DEFAULT_BUFFER_SIZE + else: + # In order to detect end of file, need a read() of at least 1 + # byte which returns size 0. Oversize the buffer by 1 byte so the + # I/O can be completed with two read() calls (one for all data, one + # for EOF) without needing to resize the buffer. + bufsize = self._stat_atopen.st_size + 1 + + if self._stat_atopen.st_size > 65536: + try: + pos = os.lseek(self._fd, 0, SEEK_CUR) + if self._stat_atopen.st_size >= pos: + bufsize = self._stat_atopen.st_size - pos + 1 + except OSError: + pass result = bytearray() while True: @@ -1736,6 +1748,7 @@ def truncate(self, size=None): if size is None: size = self.tell() os.ftruncate(self._fd, size) + self._stat_atopen = None return size def close(self): @@ -1745,6 +1758,7 @@ def close(self): called more than once without error. """ if not self.closed: + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1783,6 +1797,21 @@ def isatty(self): self._checkClosed() return os.isatty(self._fd) + def _isatty_open_only(self): + """Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). + """ + if (self._stat_atopen is not None + and not stat.S_ISCHR(self._stat_atopen.st_mode)): + return False + return os.isatty(self._fd) + @property def closefd(self): """True if the file descriptor will be closed by close().""" diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 936346e187ff69..be1acd17ce3592 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -14,7 +14,127 @@ import re import decimal - +try: + import _decimal +except ImportError: + _decimal = None + +# A number of functions have this form, where `w` is a desired number of +# digits in base `base`: +# +# def inner(...w...): +# if w <= LIMIT: +# return something +# lo = w >> 1 +# hi = w - lo +# something involving base**lo, inner(...lo...), j, and inner(...hi...) +# figure out largest w needed +# result = inner(w) +# +# They all had some on-the-fly scheme to cache `base**lo` results for reuse. +# Power is costly. +# +# This routine aims to compute all amd only the needed powers in advance, as +# efficiently as reasonably possible. This isn't trivial, and all the +# on-the-fly methods did needless work in many cases. The driving code above +# changes to: +# +# figure out largest w needed +# mycache = compute_powers(w, base, LIMIT) +# result = inner(w) +# +# and `mycache[lo]` replaces `base**lo` in the inner function. +# +# If an algorithm wants the powers of ceiling(w/2) instead of the floor, +# pass keyword argument `need_hi=True`. +# +# While this does give minor speedups (a few percent at best), the +# primary intent is to simplify the functions using this, by eliminating +# the need for them to craft their own ad-hoc caching schemes. +# +# See code near end of file for a block of code that can be enabled to +# run millions of tests. +def compute_powers(w, base, more_than, *, need_hi=False, show=False): + seen = set() + need = set() + ws = {w} + while ws: + w = ws.pop() # any element is fine to use next + if w in seen or w <= more_than: + continue + seen.add(w) + lo = w >> 1 + hi = w - lo + # only _need_ one here; the other may, or may not, be needed + which = hi if need_hi else lo + need.add(which) + ws.add(which) + if lo != hi: + ws.add(w - which) + + # `need` is the set of exponents needed. To compute them all + # efficiently, possibly add other exponents to `extra`. The goal is + # to ensure that each exponent can be gotten from a smaller one via + # multiplying by the base, squaring it, or squaring and then + # multiplying by the base. + # + # If need_hi is False, this is already the case (w can always be + # gotten from w >> 1 via one of the squaring strategies). But we do + # the work anyway, just in case ;-) + # + # Note that speed is irrelevant. These loops are working on little + # ints (exponents) and go around O(log w) times. The total cost is + # insignificant compared to just one of the bigint multiplies. + cands = need.copy() + extra = set() + while cands: + w = max(cands) + cands.remove(w) + lo = w >> 1 + if lo > more_than and w-1 not in cands and lo not in cands: + extra.add(lo) + cands.add(lo) + assert need_hi or not extra + + d = {} + for n in sorted(need | extra): + lo = n >> 1 + hi = n - lo + if n-1 in d: + if show: + print("* base", end="") + result = d[n-1] * base # cheap! + elif lo in d: + # Multiplying a bigint by itself is about twice as fast + # in CPython provided it's the same object. + if show: + print("square", end="") + result = d[lo] * d[lo] # same object + if hi != lo: + if show: + print(" * base", end="") + assert 2 * lo + 1 == n + result *= base + else: # rare + if show: + print("pow", end='') + result = base ** n + if show: + print(" at", n, "needed" if n in need else "extra") + d[n] = result + + assert need <= d.keys() + if excess := d.keys() - need: + assert need_hi + for n in excess: + del d[n] + return d + +_unbounded_dec_context = decimal.getcontext().copy() +_unbounded_dec_context.prec = decimal.MAX_PREC +_unbounded_dec_context.Emax = decimal.MAX_EMAX +_unbounded_dec_context.Emin = decimal.MIN_EMIN +_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check def int_to_decimal(n): """Asymptotically fast conversion of an 'int' to Decimal.""" @@ -29,61 +149,76 @@ def int_to_decimal(n): # "clever" recursive way. If we want a string representation, we # apply str to _that_. - D = decimal.Decimal - D2 = D(2) - - BITLIM = 128 - - mem = {} - - def w2pow(w): - """Return D(2)**w and store the result. Also possibly save some - intermediate results. In context, these are likely to be reused - across various levels of the conversion to Decimal.""" - if (result := mem.get(w)) is None: - if w <= BITLIM: - result = D2**w - elif w - 1 in mem: - result = (t := mem[w - 1]) + t - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w2pow(w2) * w2pow(w - w2) - mem[w] = result - return result + from decimal import Decimal as D + BITLIM = 200 + # Don't bother caching the "lo" mask in this; the time to compute it is + # tiny compared to the multiply. def inner(n, w): if w <= BITLIM: return D(n) w2 = w >> 1 hi = n >> w2 - lo = n - (hi << w2) - return inner(lo, w2) + inner(hi, w - w2) * w2pow(w2) - - with decimal.localcontext() as ctx: - ctx.prec = decimal.MAX_PREC - ctx.Emax = decimal.MAX_EMAX - ctx.Emin = decimal.MIN_EMIN - ctx.traps[decimal.Inexact] = 1 + lo = n & ((1 << w2) - 1) + return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2] + with decimal.localcontext(_unbounded_dec_context): + nbits = n.bit_length() + w2pow = compute_powers(nbits, D(2), BITLIM) if n < 0: negate = True n = -n else: negate = False - result = inner(n, n.bit_length()) + result = inner(n, nbits) if negate: result = -result return result - def int_to_decimal_string(n): """Asymptotically fast conversion of an 'int' to a decimal string.""" - return str(int_to_decimal(n)) - + w = n.bit_length() + if w > 450_000 and _decimal is not None: + # It is only usable with the C decimal implementation. + # _pydecimal.py calls str() on very large integers, which in its + # turn calls int_to_decimal_string(), causing very deep recursion. + return str(int_to_decimal(n)) + + # Fallback algorithm for the case when the C decimal module isn't + # available. This algorithm is asymptotically worse than the algorithm + # using the decimal module, but better than the quadratic time + # implementation in longobject.c. + + DIGLIM = 1000 + def inner(n, w): + if w <= DIGLIM: + return str(n) + w2 = w >> 1 + hi, lo = divmod(n, pow10[w2]) + return inner(hi, w - w2) + inner(lo, w2).zfill(w2) + + # The estimation of the number of decimal digits. + # There is no harm in small error. If we guess too large, there may + # be leading 0's that need to be stripped. If we guess too small, we + # may need to call str() recursively for the remaining highest digits, + # which can still potentially be a large integer. This is manifested + # only if the number has way more than 10**15 digits, that exceeds + # the 52-bit physical address limit in both Intel64 and AMD64. + w = int(w * 0.3010299956639812 + 1) # log10(2) + pow10 = compute_powers(w, 5, DIGLIM) + for k, v in pow10.items(): + pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k + if n < 0: + n = -n + sign = '-' + else: + sign = '' + s = inner(n, w) + if s[0] == '0' and n: + # If our guess of w is too large, there may be leading 0's that + # need to be stripped. + s = s.lstrip('0') + return sign + s def _str_to_int_inner(s): """Asymptotically fast conversion of a 'str' to an 'int'.""" @@ -100,38 +235,157 @@ def _str_to_int_inner(s): DIGLIM = 2048 - mem = {} - - def w5pow(w): - """Return 5**w and store the result. - Also possibly save some intermediate results. In context, these - are likely to be reused across various levels of the conversion - to 'int'. - """ - if (result := mem.get(w)) is None: - if w <= DIGLIM: - result = 5**w - elif w - 1 in mem: - result = mem[w - 1] * 5 - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w5pow(w2) * w5pow(w - w2) - mem[w] = result - return result - def inner(a, b): if b - a <= DIGLIM: return int(s[a:b]) mid = (a + b + 1) >> 1 - return inner(mid, b) + ((inner(a, mid) * w5pow(b - mid)) << (b - mid)) + return (inner(mid, b) + + ((inner(a, mid) * w5pow[b - mid]) + << (b - mid))) + w5pow = compute_powers(len(s), 5, DIGLIM) return inner(0, len(s)) +# Asymptotically faster version, using the C decimal module. See +# comments at the end of the file. This uses decimal arithmetic to +# convert from base 10 to base 256. The latter is just a string of +# bytes, which CPython can convert very efficiently to a Python int. + +# log of 10 to base 256 with best-possible 53-bit precision. Obtained +# via: +# from mpmath import mp +# mp.prec = 1000 +# print(float(mp.log(10, 256)).hex()) +_LOG_10_BASE_256 = float.fromhex('0x1.a934f0979a371p-2') # about 0.415 + +# _spread is for internal testing. It maps a key to the number of times +# that condition obtained in _dec_str_to_int_inner: +# key 0 - quotient guess was right +# key 1 - quotient had to be boosted by 1, one time +# key 999 - one adjustment wasn't enough, so fell back to divmod +from collections import defaultdict +_spread = defaultdict(int) +del defaultdict + +def _dec_str_to_int_inner(s, *, GUARD=8): + # Yes, BYTELIM is "large". Large enough that CPython will usually + # use the Karatsuba _str_to_int_inner to convert the string. This + # allowed reducing the cutoff for calling _this_ function from 3.5M + # to 2M digits. We could almost certainly do even better by + # fine-tuning this and/or using a larger output base than 256. + BYTELIM = 100_000 + D = decimal.Decimal + result = bytearray() + # See notes at end of file for discussion of GUARD. + assert GUARD > 0 # if 0, `decimal` can blow up - .prec 0 not allowed + + def inner(n, w): + #assert n < D256 ** w # required, but too expensive to check + if w <= BYTELIM: + # XXX Stefan Pochmann discovered that, for 1024-bit ints, + # `int(Decimal)` took 2.5x longer than `int(str(Decimal))`. + # Worse, `int(Decimal) is still quadratic-time for much + # larger ints. So unless/until all that is repaired, the + # seemingly redundant `str(Decimal)` is crucial to speed. + result.extend(int(str(n)).to_bytes(w)) # big-endian default + return + w1 = w >> 1 + w2 = w - w1 + if 0: + # This is maximally clear, but "too slow". `decimal` + # division is asymptotically fast, but we have no way to + # tell it to reuse the high-precision reciprocal it computes + # for pow256[w2], so it has to recompute it over & over & + # over again :-( + hi, lo = divmod(n, pow256[w2][0]) + else: + p256, recip = pow256[w2] + # The integer part will have a number of digits about equal + # to the difference between the log10s of `n` and `pow256` + # (which, since these are integers, is roughly approximated + # by `.adjusted()`). That's the working precision we need, + ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD + hi = +n * +recip # unary `+` chops back to ctx.prec digits + ctx.prec = decimal.MAX_PREC + hi = hi.to_integral_value() # lose the fractional digits + lo = n - hi * p256 + # Because we've been uniformly rounding down, `hi` is a + # lower bound on the correct quotient. + assert lo >= 0 + # Adjust quotient up if needed. It usually isn't. In random + # testing on inputs through 5 billion digit strings, the + # test triggered once in about 200 thousand tries. + count = 0 + if lo >= p256: + count = 1 + lo -= p256 + hi += 1 + if lo >= p256: + # Complete correction via an exact computation. I + # believe it's not possible to get here provided + # GUARD >= 3. It's tested by reducing GUARD below + # that. + count = 999 + hi2, lo = divmod(lo, p256) + hi += hi2 + _spread[count] += 1 + # The assert should always succeed, but way too slow to keep + # enabled. + #assert hi, lo == divmod(n, pow256[w2][0]) + inner(hi, w1) + del hi # at top levels, can free a lot of RAM "early" + inner(lo, w2) + + # How many base 256 digits are needed?. Mathematically, exactly + # floor(log256(int(s))) + 1. There is no cheap way to compute this. + # But we can get an upper bound, and that's necessary for our error + # analysis to make sense. int(s) < 10**len(s), so the log needed is + # < log256(10**len(s)) = len(s) * log256(10). However, using + # finite-precision floating point for this, it's possible that the + # computed value is a little less than the true value. If the true + # value is at - or a little higher than - an integer, we can get an + # off-by-1 error too low. So we add 2 instead of 1 if chopping lost + # a fraction > 0.9. + + # The "WASI" test platform can complain about `len(s)` if it's too + # large to fit in its idea of "an index-sized integer". + lenS = s.__len__() + log_ub = lenS * _LOG_10_BASE_256 + log_ub_as_int = int(log_ub) + w = log_ub_as_int + 1 + (log_ub - log_ub_as_int > 0.9) + # And what if we've plain exhausted the limits of HW floats? We + # could compute the log to any desired precision using `decimal`, + # but it's not plausible that anyone will pass a string requiring + # trillions of bytes (unless they're just trying to "break things"). + if w.bit_length() >= 46: + # "Only" had < 53 - 46 = 7 bits to spare in IEEE-754 double. + raise ValueError(f"cannot convert string of len {lenS} to int") + with decimal.localcontext(_unbounded_dec_context) as ctx: + D256 = D(256) + pow256 = compute_powers(w, D256, BYTELIM, need_hi=True) + rpow256 = compute_powers(w, 1 / D256, BYTELIM, need_hi=True) + # We're going to do inexact, chopped arithmetic, multiplying by + # an approximation to the reciprocal of 256**i. We chop to get a + # lower bound on the true integer quotient. Our approximation is + # a lower bound, the multiplication is chopped too, and + # to_integral_value() is also chopped. + ctx.traps[decimal.Inexact] = 0 + ctx.rounding = decimal.ROUND_DOWN + for k, v in pow256.items(): + # No need to save much more precision in the reciprocal than + # the power of 256 has, plus some guard digits to absorb + # most relevant rounding errors. This is highly significant: + # 1/2**i has the same number of significant decimal digits + # as 5**i, generally over twice the number in 2**i, + ctx.prec = v.adjusted() + GUARD + 1 + # The unary "+" chops the reciprocal back to that precision. + pow256[k] = v, +rpow256[k] + del rpow256 # exact reciprocals no longer needed + ctx.prec = decimal.MAX_PREC + inner(D(s), w) + return int.from_bytes(result) + def int_from_string(s): """Asymptotically fast version of PyLong_FromString(), conversion of a string of decimal digits into an 'int'.""" @@ -140,8 +394,10 @@ def int_from_string(s): # and underscores, and stripped leading whitespace. The input can still # contain underscores and have trailing whitespace. s = s.rstrip().replace('_', '') - return _str_to_int_inner(s) - + func = _str_to_int_inner + if len(s) >= 2_000_000 and _decimal is not None: + func = _dec_str_to_int_inner + return func(s) def str_to_int(s): """Asymptotically fast version of decimal string to 'int' conversion.""" @@ -274,7 +530,7 @@ def int_divmod(a, b): Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b). """ if b == 0: - raise ZeroDivisionError + raise ZeroDivisionError('division by zero') elif b < 0: q, r = int_divmod(-a, -b) return q, -r @@ -283,3 +539,191 @@ def int_divmod(a, b): return ~q, b + ~r else: return _divmod_pos(a, b) + + +# Notes on _dec_str_to_int_inner: +# +# Stefan Pochmann worked up a str->int function that used the decimal +# module to, in effect, convert from base 10 to base 256. This is +# "unnatural", in that it requires multiplying and dividing by large +# powers of 2, which `decimal` isn't naturally suited to. But +# `decimal`'s `*` and `/` are asymptotically superior to CPython's, so +# at _some_ point it could be expected to win. +# +# Alas, the crossover point was too high to be of much real interest. I +# (Tim) then worked on ways to replace its division with multiplication +# by a cached reciprocal approximation instead, fixing up errors +# afterwards. This reduced the crossover point significantly, +# +# I revisited the code, and found ways to improve and simplify it. The +# crossover point is at about 3.4 million digits now. +# +# About .adjusted() +# ----------------- +# Restrict to Decimal values x > 0. We don't use negative numbers in the +# code, and I don't want to have to keep typing, e.g., "absolute value". +# +# For convenience, I'll use `x.a` to mean `x.adjusted()`. x.a doesn't +# look at the digits of x, but instead returns an integer giving x's +# order of magnitude. These are equivalent: +# +# - x.a is the power-of-10 exponent of x's most significant digit. +# - x.a = the infinitely precise floor(log10(x)) +# - x can be written in this form, where f is a real with 1 <= f < 10: +# x = f * 10**x.a +# +# Observation; if x is an integer, len(str(x)) = x.a + 1. +# +# Lemma 1: (x * y).a = x.a + y.a, or one larger +# +# Proof: Write x = f * 10**x.a and y = g * 10**y.a, where f and g are in +# [1, 10). Then x*y = f*g * 10**(x.a + y.a), where 1 <= f*g < 100. If +# f*g < 10, (x*y).a is x.a+y.a. Else divide f*g by 10 to bring it back +# into [1, 10], and add 1 to the exponent to compensate. Then (x*y).a is +# x.a+y.a+1. +# +# Lemma 2: ceiling(log10(x/y)) <= x.a - y.a + 1 +# +# Proof: Express x and y as in Lemma 1. Then x/y = f/g * 10**(x.a - +# y.a), where 1/10 < f/g < 10. If 1 <= f/g, (x/y).a is x.a-y.a. Else +# multiply f/g by 10 to bring it back into [1, 10], and subtract 1 from +# the exponent to compensate. Then (x/y).a is x.a-y.a-1. So the largest +# (x/y).a can be is x.a-y.a. Since that's the floor of log10(x/y). the +# ceiling is at most 1 larger (with equality iff f/g = 1 exactly). +# +# GUARD digits +# ------------ +# We only want the integer part of divisions, so don't need to build +# the full multiplication tree. But using _just_ the number of +# digits expected in the integer part ignores too much. What's left +# out can have a very significant effect on the quotient. So we use +# GUARD additional digits. +# +# The default 8 is more than enough so no more than 1 correction step +# was ever needed for all inputs tried through 2.5 billion digits. In +# fact, I believe 3 guard digits are always enough - but the proof is +# very involved, so better safe than sorry. +# +# Short course: +# +# If prec is the decimal precision in effect, and we're rounding down, +# the result of an operation is exactly equal to the infinitely precise +# result times 1-e for some real e with 0 <= e < 10**(1-prec). In +# +# ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD +# hi = +n * +recip # unary `+` chops to ctx.prec digits +# +# we have 3 visible chopped operations, but there's also a 4th: +# precomputing a truncated `recip` as part of setup. +# +# So the computed product is exactly equal to the true product times +# (1-e1)*(1-e2)*(1-e3)*(1-e4); since the e's are all very small, an +# excellent approximation to the second factor is 1-(e1+e2+e3+e4) (the +# 2nd and higher order terms in the expanded product are too tiny to +# matter). If they're all as large as possible, that's +# +# 1 - 4*10**(1-prec). This, BTW, is all bog-standard FP error analysis. +# +# That implies the computed product is within 1 of the true product +# provided prec >= log10(true_product) + 1.602. +# +# Here are telegraphic details, rephrasing the initial condition in +# equivalent ways, step by step: +# +# prod - prod * (1 - 4*10**(1-prec)) <= 1 +# prod - prod + prod * 4*10**(1-prec)) <= 1 +# prod * 4*10**(1-prec)) <= 1 +# 10**(log10(prod)) * 4*10**(1-prec)) <= 1 +# 4*10**(1-prec+log10(prod))) <= 1 +# 10**(1-prec+log10(prod))) <= 1/4 +# 1-prec+log10(prod) <= log10(1/4) = -0.602 +# -prec <= -1.602 - log10(prod) +# prec >= log10(prod) + 1.602 +# +# The true product is the same as the true ratio n/p256. By Lemma 2 +# above, n.a - p256.a + 1 is an upper bound on the ceiling of +# log10(prod). Then 2 is the ceiling of 1.602. so n.a - p256.a + 3 is an +# upper bound on the right hand side of the inequality. Any prec >= that +# will work. +# +# But since this is just a sketch of a proof ;-), the code uses the +# empirically tested 8 instead of 3. 5 digits more or less makes no +# practical difference to speed - these ints are huge. And while +# increasing GUARD above 3 may not be necessary, every increase cuts the +# percentage of cases that need a correction at all. +# +# On Computing Reciprocals +# ------------------------ +# In general, the exact reciprocals we compute have over twice as many +# significant digits as needed. 1/256**i has the same number of +# significant decimal digits as 5**i. It's a significant waste of RAM +# to store all those unneeded digits. +# +# So we cut exact reciprocals back to the least precision that can +# be needed so that the error analysis above is valid, +# +# [Note: turns out it's very significantly faster to do it this way than +# to compute 1 / 256**i directly to the desired precision, because the +# power method doesn't require division. It's also faster than computing +# (1/256)**i directly to the desired precision - no material division +# there, but `compute_powers()` is much smarter about _how_ to compute +# all the powers needed than repeated applications of `**` - that +# function invokes `**` for at most the few smallest powers needed.] +# +# The hard part is that chopping back to a shorter width occurs +# _outside_ of `inner`. We can't know then what `prec` `inner()` will +# need. We have to pick, for each value of `w2`, the largest possible +# value `prec` can become when `inner()` is working on `w2`. +# +# This is the `prec` inner() uses: +# max(n.a - p256.a, 0) + GUARD +# and what setup uses (renaming its `v` to `p256` - same thing): +# p256.a + GUARD + 1 +# +# We need that the second is always at least as large as the first, +# which is the same as requiring +# +# n.a - 2 * p256.a <= 1 +# +# What's the largest n can be? n < 255**w = 256**(w2 + (w - w2)). The +# worst case in this context is when w ix even. and then w = 2*w2, so +# n < 256**(2*w2) = (256**w2)**2 = p256**2. By Lemma 1, then, n.a +# is at most p256.a + p256.a + 1. +# +# So the most n.a - 2 * p256.a can be is +# p256.a + p256.a + 1 - 2 * p256.a = 1. QED +# +# Note: an earlier version of the code split on floor(e/2) instead of on +# the ceiling. The worst case then is odd `w`, and a more involved proof +# was needed to show that adding 4 (instead of 1) may be necessary. +# Basically because, in that case, n may be up to 256 times larger than +# p256**2. Curiously enough, by splitting on the ceiling instead, +# nothing in any proof here actually depends on the output base (256). + +# Enable for brute-force testing of compute_powers(). This takes about a +# minute, because it tries millions of cases. +if 0: + def consumer(w, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit: + return + if w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + exp = compute_powers(w, 1, limit, need_hi=need_hi) + assert exp.keys() == need + + from itertools import chain + for need_hi in (False, True): + for limit in (0, 1, 10, 100, 1_000, 10_000, 100_000): + for w in chain(range(1, 100_000), + (10**i for i in range(5, 30))): + consumer(w, limit, need_hi) diff --git a/Lib/_pyrepl/__init__.py b/Lib/_pyrepl/__init__.py new file mode 100644 index 00000000000000..1693cbd0b98b74 --- /dev/null +++ b/Lib/_pyrepl/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py new file mode 100644 index 00000000000000..3fa992eee8eeff --- /dev/null +++ b/Lib/_pyrepl/__main__.py @@ -0,0 +1,6 @@ +# Important: don't add things to this module, as they will end up in the REPL's +# default globals. Use _pyrepl.main instead. + +if __name__ == "__main__": + from .main import interactive_console as __pyrepl_interactive_console + __pyrepl_interactive_console() diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py new file mode 100644 index 00000000000000..849617bf7585e4 --- /dev/null +++ b/Lib/_pyrepl/_minimal_curses.py @@ -0,0 +1,68 @@ +"""Minimal '_curses' module, the low-level interface for curses module +which is not meant to be used directly. + +Based on ctypes. It's too incomplete to be really called '_curses', so +to use it, you have to import it and stick it in sys.modules['_curses'] +manually. + +Note that there is also a built-in module _minimal_curses which will +hide this one if compiled in. +""" + +import ctypes +import ctypes.util + + +class error(Exception): + pass + + +def _find_clib() -> str: + trylibs = ["ncursesw", "ncurses", "curses"] + + for lib in trylibs: + path = ctypes.util.find_library(lib) + if path: + return path + raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") + + +_clibpath = _find_clib() +clib = ctypes.cdll.LoadLibrary(_clibpath) + +clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] +clib.setupterm.restype = ctypes.c_int + +clib.tigetstr.argtypes = [ctypes.c_char_p] +clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char) + +clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] +clib.tparm.restype = ctypes.c_char_p + +OK = 0 +ERR = -1 + +# ____________________________________________________________ + + +def setupterm(termstr, fd): + err = ctypes.c_int(0) + result = clib.setupterm(termstr, fd, ctypes.byref(err)) + if result == ERR: + raise error("setupterm() failed (err=%d)" % err.value) + + +def tigetstr(cap): + if not isinstance(cap, bytes): + cap = cap.encode("ascii") + result = clib.tigetstr(cap) + if ctypes.cast(result, ctypes.c_void_p).value == ERR: + return None + return ctypes.cast(result, ctypes.c_char_p).value + + +def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): + result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) + if result is None: + raise error("tparm() returned NULL") + return result diff --git a/Lib/_pyrepl/_threading_handler.py b/Lib/_pyrepl/_threading_handler.py new file mode 100644 index 00000000000000..82f5e8650a2072 --- /dev/null +++ b/Lib/_pyrepl/_threading_handler.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +import traceback + + +TYPE_CHECKING = False +if TYPE_CHECKING: + from threading import Thread + from types import TracebackType + from typing import Protocol + + class ExceptHookArgs(Protocol): + @property + def exc_type(self) -> type[BaseException]: ... + @property + def exc_value(self) -> BaseException | None: ... + @property + def exc_traceback(self) -> TracebackType | None: ... + @property + def thread(self) -> Thread | None: ... + + class ShowExceptions(Protocol): + def __call__(self) -> int: ... + def add(self, s: str) -> None: ... + + from .reader import Reader + + +def install_threading_hook(reader: Reader) -> None: + import threading + + @dataclass + class ExceptHookHandler: + lock: threading.Lock = field(default_factory=threading.Lock) + messages: list[str] = field(default_factory=list) + + def show(self) -> int: + count = 0 + with self.lock: + if not self.messages: + return 0 + reader.restore() + for tb in self.messages: + count += 1 + if tb: + print(tb) + self.messages.clear() + reader.scheduled_commands.append("ctrl-c") + reader.prepare() + return count + + def add(self, s: str) -> None: + with self.lock: + self.messages.append(s) + + def exception(self, args: ExceptHookArgs) -> None: + lines = traceback.format_exception( + args.exc_type, + args.exc_value, + args.exc_traceback, + colorize=reader.can_colorize, + ) # type: ignore[call-overload] + pre = f"\nException in {args.thread.name}:\n" if args.thread else "\n" + tb = pre + "".join(lines) + self.add(tb) + + def __call__(self) -> int: + return self.show() + + + handler = ExceptHookHandler() + reader.threading_hook = handler + threading.excepthook = handler.exception diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py new file mode 100644 index 00000000000000..c3fce91013b001 --- /dev/null +++ b/Lib/_pyrepl/commands.py @@ -0,0 +1,483 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations +import os + +# Categories of actions: +# killing +# yanking +# motion +# editing +# history +# finishing +# [completion] + + +# types +if False: + from .historical_reader import HistoricalReader + + +class Command: + finish: bool = False + kills_digit_arg: bool = True + + def __init__( + self, reader: HistoricalReader, event_name: str, event: list[str] + ) -> None: + # Reader should really be "any reader" but there's too much usage of + # HistoricalReader methods and fields in the code below for us to + # refactor at the moment. + + self.reader = reader + self.event = event + self.event_name = event_name + + def do(self) -> None: + pass + + +class KillCommand(Command): + def kill_range(self, start: int, end: int) -> None: + if start == end: + return + r = self.reader + b = r.buffer + text = b[start:end] + del b[start:end] + if is_kill(r.last_command): + if start < r.pos: + r.kill_ring[-1] = text + r.kill_ring[-1] + else: + r.kill_ring[-1] = r.kill_ring[-1] + text + else: + r.kill_ring.append(text) + r.pos = start + r.dirty = True + + +class YankCommand(Command): + pass + + +class MotionCommand(Command): + pass + + +class EditCommand(Command): + pass + + +class FinishCommand(Command): + finish = True + pass + + +def is_kill(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, KillCommand) + + +def is_yank(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, YankCommand) + + +# etc + + +class digit_arg(Command): + kills_digit_arg = False + + def do(self) -> None: + r = self.reader + c = self.event[-1] + if c == "-": + if r.arg is not None: + r.arg = -r.arg + else: + r.arg = -1 + else: + d = int(c) + if r.arg is None: + r.arg = d + else: + if r.arg < 0: + r.arg = 10 * r.arg - d + else: + r.arg = 10 * r.arg + d + r.dirty = True + + +class clear_screen(Command): + def do(self) -> None: + r = self.reader + r.console.clear() + r.dirty = True + + +class refresh(Command): + def do(self) -> None: + self.reader.dirty = True + + +class repaint(Command): + def do(self) -> None: + self.reader.dirty = True + self.reader.console.repaint() + + +class kill_line(KillCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + eol = r.eol() + for c in b[r.pos : eol]: + if not c.isspace(): + self.kill_range(r.pos, eol) + return + else: + self.kill_range(r.pos, eol + 1) + + +class unix_line_discard(KillCommand): + def do(self) -> None: + r = self.reader + self.kill_range(r.bol(), r.pos) + + +class unix_word_rubout(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.pos, r.eow()) + + +class backward_kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class yank(YankCommand): + def do(self) -> None: + r = self.reader + if not r.kill_ring: + r.error("nothing to yank") + return + r.insert(r.kill_ring[-1]) + + +class yank_pop(YankCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if not r.kill_ring: + r.error("nothing to yank") + return + if not is_yank(r.last_command): + r.error("previous command was not a yank") + return + repl = len(r.kill_ring[-1]) + r.kill_ring.insert(0, r.kill_ring.pop()) + t = r.kill_ring[-1] + b[r.pos - repl : r.pos] = t + r.pos = r.pos - repl + len(t) + r.dirty = True + + +class interrupt(FinishCommand): + def do(self) -> None: + import signal + + self.reader.console.finish() + self.reader.finish() + os.kill(os.getpid(), signal.SIGINT) + + +class ctrl_c(Command): + def do(self) -> None: + self.reader.console.finish() + self.reader.finish() + raise KeyboardInterrupt + + +class suspend(Command): + def do(self) -> None: + import signal + + r = self.reader + p = r.pos + r.console.finish() + os.kill(os.getpid(), signal.SIGSTOP) + ## this should probably be done + ## in a handler for SIGCONT? + r.console.prepare() + r.pos = p + # r.posxy = 0, 0 # XXX this is invalid + r.dirty = True + r.console.screen = [] + + +class up(MotionCommand): + def do(self) -> None: + r = self.reader + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y - 1 + + if r.bol() == 0: + if r.historyi > 0: + r.select_item(r.historyi - 1) + return + r.pos = 0 + r.error("start of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class down(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y + 1 + + if new_y > r.max_row(): + if r.historyi < len(r.history): + r.select_item(r.historyi + 1) + r.pos = r.eol(0) + return + r.pos = len(b) + r.error("end of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class left(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + p = r.pos - 1 + if p >= 0: + r.pos = p + else: + self.reader.error("start of buffer") + + +class right(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + p = r.pos + 1 + if p <= len(b): + r.pos = p + else: + self.reader.error("end of buffer") + + +class beginning_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.bol() + + +class end_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.eol() + + +class home(MotionCommand): + def do(self) -> None: + self.reader.pos = 0 + + +class end(MotionCommand): + def do(self) -> None: + self.reader.pos = len(self.reader.buffer) + + +class forward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.eow() + + +class backward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.bow() + + +class self_insert(EditCommand): + def do(self) -> None: + r = self.reader + text = self.event * r.get_arg() + r.insert(text) + + +class insert_nl(EditCommand): + def do(self) -> None: + r = self.reader + r.insert("\n" * r.get_arg()) + + +class transpose_characters(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + s = r.pos - 1 + if s < 0: + r.error("cannot transpose at start of buffer") + else: + if s == len(b): + s -= 1 + t = min(s + r.get_arg(), len(b) - 1) + c = b[s] + del b[s] + b.insert(t, c) + r.pos = t + r.dirty = True + + +class backspace(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + if r.pos > 0: + r.pos -= 1 + del b[r.pos] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +class delete(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + and self.event[-1] == "\004" + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): + if r.pos != len(b): + del b[r.pos] + r.dirty = True + else: + self.reader.error("end of buffer") + + +class accept(FinishCommand): + def do(self) -> None: + pass + + +class help(Command): + def do(self) -> None: + import _sitebuiltins + + with self.reader.suspend(): + self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] + + +class invalid_key(Command): + def do(self) -> None: + pending = self.reader.console.getpending() + s = "".join(self.event) + pending.data + self.reader.error("`%r' not bound" % s) + + +class invalid_command(Command): + def do(self) -> None: + s = self.event_name + self.reader.error("command `%s' not known" % s) + + +class show_history(Command): + def do(self) -> None: + from .pager import get_pager + from site import gethistoryfile # type: ignore[attr-defined] + + history = os.linesep.join(self.reader.history[:]) + with self.reader.suspend(): + pager = get_pager() + pager(history, gethistoryfile()) + + +class paste_mode(Command): + + def do(self) -> None: + self.reader.paste_mode = not self.reader.paste_mode + self.reader.dirty = True + + +class enable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = True + self.reader.in_bracketed_paste = True + +class disable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = False + self.reader.in_bracketed_paste = False + self.reader.dirty = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py new file mode 100644 index 00000000000000..e856bb9807c7f6 --- /dev/null +++ b/Lib/_pyrepl/completing_reader.py @@ -0,0 +1,290 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from dataclasses import dataclass, field + +import re +from . import commands, console, reader +from .reader import Reader + + +# types +Command = commands.Command +if False: + from .types import KeySpec, CommandName + + +def prefix(wordlist: list[str], j: int = 0) -> str: + d = {} + i = j + try: + while 1: + for word in wordlist: + d[word[i]] = 1 + if len(d) > 1: + return wordlist[0][j:i] + i += 1 + d = {} + except IndexError: + return wordlist[0][j:i] + return "" + + +STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") + +def stripcolor(s: str) -> str: + return STRIPCOLOR_REGEX.sub('', s) + + +def real_len(s: str) -> int: + return len(stripcolor(s)) + + +def left_align(s: str, maxlen: int) -> str: + stripped = stripcolor(s) + if len(stripped) > maxlen: + # too bad, we remove the color + return stripped[:maxlen] + padding = maxlen - len(stripped) + return s + ' '*padding + + +def build_menu( + cons: console.Console, + wordlist: list[str], + start: int, + use_brackets: bool, + sort_in_column: bool, +) -> tuple[list[str], int]: + if use_brackets: + item = "[ %s ]" + padding = 4 + else: + item = "%s " + padding = 2 + maxlen = min(max(map(real_len, wordlist)), cons.width - padding) + cols = int(cons.width / (maxlen + padding)) + rows = int((len(wordlist) - 1)/cols + 1) + + if sort_in_column: + # sort_in_column=False (default) sort_in_column=True + # A B C A D G + # D E F B E + # G C F + # + # "fill" the table with empty words, so we always have the same amount + # of rows for each column + missing = cols*rows - len(wordlist) + wordlist = wordlist + ['']*missing + indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))] + wordlist = [wordlist[i] for i in indexes] + menu = [] + i = start + for r in range(rows): + row = [] + for col in range(cols): + row.append(item % left_align(wordlist[i], maxlen)) + i += 1 + if i >= len(wordlist): + break + menu.append(''.join(row)) + if i >= len(wordlist): + i = 0 + break + if r + 5 > cons.height: + menu.append(" %d more... " % (len(wordlist) - i)) + break + return menu, i + +# this gets somewhat user interface-y, and as a result the logic gets +# very convoluted. +# +# To summarise the summary of the summary:- people are a problem. +# -- The Hitch-Hikers Guide to the Galaxy, Episode 12 + +#### Desired behaviour of the completions commands. +# the considerations are: +# (1) how many completions are possible +# (2) whether the last command was a completion +# (3) if we can assume that the completer is going to return the same set of +# completions: this is controlled by the ``assume_immutable_completions`` +# variable on the reader, which is True by default to match the historical +# behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match +# more closely readline's semantics (this is needed e.g. by +# fancycompleter) +# +# if there's no possible completion, beep at the user and point this out. +# this is easy. +# +# if there's only one possible completion, stick it in. if the last thing +# user did was a completion, point out that he isn't getting anywhere, but +# only if the ``assume_immutable_completions`` is True. +# +# now it gets complicated. +# +# for the first press of a completion key: +# if there's a common prefix, stick it in. + +# irrespective of whether anything got stuck in, if the word is now +# complete, show the "complete but not unique" message + +# if there's no common prefix and if the word is not now complete, +# beep. + +# common prefix -> yes no +# word complete \/ +# yes "cbnu" "cbnu" +# no - beep + +# for the second bang on the completion key +# there will necessarily be no common prefix +# show a menu of the choices. + +# for subsequent bangs, rotate the menu around (if there are sufficient +# choices). + + +class complete(commands.Command): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + last_is_completer = r.last_command_is(self.__class__) + immutable_completions = r.assume_immutable_completions + completions_unchangable = last_is_completer and immutable_completions + stem = r.get_stem() + if not completions_unchangable: + r.cmpltn_menu_choices = r.get_completions(stem) + + completions = r.cmpltn_menu_choices + if not completions: + r.error("no matches") + elif len(completions) == 1: + if completions_unchangable and len(completions[0]) == len(stem): + r.msg = "[ sole completion ]" + r.dirty = True + r.insert(completions[0][len(stem):]) + else: + p = prefix(completions, len(stem)) + if p: + r.insert(p) + if last_is_completer: + r.cmpltn_menu_visible = True + r.cmpltn_message_visible = False + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, r.cmpltn_menu_end, + r.use_brackets, r.sort_in_column) + r.dirty = True + elif not r.cmpltn_menu_visible: + r.cmpltn_message_visible = True + if stem + p in completions: + r.msg = "[ complete but not unique ]" + r.dirty = True + else: + r.msg = "[ not unique ]" + r.dirty = True + + +class self_insert(commands.self_insert): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + + commands.self_insert.do(self) + if r.cmpltn_menu_visible: + stem = r.get_stem() + if len(stem) < 1: + r.cmpltn_reset() + else: + completions = [w for w in r.cmpltn_menu_choices + if w.startswith(stem)] + if completions: + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, 0, + r.use_brackets, r.sort_in_column) + else: + r.cmpltn_reset() + + +@dataclass +class CompletingReader(Reader): + """Adds completion support""" + + ### Class variables + # see the comment for the complete command + assume_immutable_completions = True + use_brackets = True # display completions inside [] + sort_in_column = False + + ### Instance variables + cmpltn_menu: list[str] = field(init=False) + cmpltn_menu_visible: bool = field(init=False) + cmpltn_message_visible: bool = field(init=False) + cmpltn_menu_end: int = field(init=False) + cmpltn_menu_choices: list[str] = field(init=False) + + def __post_init__(self) -> None: + super().__post_init__() + self.cmpltn_reset() + for c in (complete, self_insert): + self.commands[c.__name__] = c + self.commands[c.__name__.replace('_', '-')] = c + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r'\t', 'complete'),) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if not isinstance(cmd, (complete, self_insert)): + self.cmpltn_reset() + + def calc_screen(self) -> list[str]: + screen = super().calc_screen() + if self.cmpltn_menu_visible: + ly = self.lxy[1] + screen[ly:ly] = self.cmpltn_menu + self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) + self.cxy = self.cxy[0], self.cxy[1] + len(self.cmpltn_menu) + return screen + + def finish(self) -> None: + super().finish() + self.cmpltn_reset() + + def cmpltn_reset(self) -> None: + self.cmpltn_menu = [] + self.cmpltn_menu_visible = False + self.cmpltn_message_visible = False + self.cmpltn_menu_end = 0 + self.cmpltn_menu_choices = [] + + def get_stem(self) -> str: + st = self.syntax_table + SW = reader.SYNTAX_WORD + b = self.buffer + p = self.pos - 1 + while p >= 0 and st.get(b[p], SW) == SW: + p -= 1 + return ''.join(b[p+1:self.pos]) + + def get_completions(self, stem: str) -> list[str]: + return [] diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py new file mode 100644 index 00000000000000..03266c4dfc2dd8 --- /dev/null +++ b/Lib/_pyrepl/console.py @@ -0,0 +1,212 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import _colorize # type: ignore[import-not-found] + +from abc import ABC, abstractmethod +import ast +import code +from dataclasses import dataclass, field +import os.path +import sys + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + from typing import Callable + + +@dataclass +class Event: + evt: str + data: str + raw: bytes = b"" + + +@dataclass +class Console(ABC): + screen: list[str] = field(default_factory=list) + height: int = 25 + width: int = 80 + + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + self.encoding = encoding or sys.getdefaultencoding() + + if isinstance(f_in, int): + self.input_fd = f_in + else: + self.input_fd = f_in.fileno() + + if isinstance(f_out, int): + self.output_fd = f_out + else: + self.output_fd = f_out.fileno() + + @abstractmethod + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... + + @abstractmethod + def prepare(self) -> None: ... + + @abstractmethod + def restore(self) -> None: ... + + @abstractmethod + def move_cursor(self, x: int, y: int) -> None: ... + + @abstractmethod + def set_cursor_vis(self, visible: bool) -> None: ... + + @abstractmethod + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + ... + + @abstractmethod + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + ... + + @abstractmethod + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + ... + + @abstractmethod + def beep(self) -> None: ... + + @abstractmethod + def clear(self) -> None: + """Wipe the screen""" + ... + + @abstractmethod + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + ... + + @abstractmethod + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere).""" + ... + + @abstractmethod + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + ... + + @abstractmethod + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + ... + + @abstractmethod + def wait(self, timeout: float | None) -> bool: + """Wait for an event. The return value is True if an event is + available, False if the timeout has been reached. If timeout is + None, wait forever. The timeout is in milliseconds.""" + ... + + @property + def input_hook(self) -> Callable[[], int] | None: + """Returns the current input hook.""" + ... + + @abstractmethod + def repaint(self) -> None: ... + + +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + self.can_colorize = _colorize.can_colorize() + + def showsyntaxerror(self, filename=None, **kwargs): + super().showsyntaxerror(filename=filename, **kwargs) + + def _excepthook(self, typ, value, tb): + import traceback + lines = traceback.format_exception( + typ, value, tb, + colorize=self.can_colorize, + limit=traceback.BUILTIN_EXCEPTION_LIMIT) + self.write(''.join(lines)) + + def runsource(self, source, filename="", symbol="single"): + try: + tree = self.compile.compiler( + source, + filename, + "exec", + ast.PyCF_ONLY_AST, + incomplete_input=False, + ) + except (SyntaxError, OverflowError, ValueError): + self.showsyntaxerror(filename, source=source) + return False + if tree.body: + *_, last_stmt = tree.body + for stmt in tree.body: + wrapper = ast.Interactive if stmt is last_stmt else ast.Module + the_symbol = symbol if stmt is last_stmt else "exec" + item = wrapper([stmt]) + try: + code = self.compile.compiler(item, filename, the_symbol) + except SyntaxError as e: + if e.args[0] == "'await' outside function": + python = os.path.basename(sys.executable) + e.add_note( + f"Try the asyncio REPL ({python} -m asyncio) to use" + f" top-level 'await' and run background asyncio tasks." + ) + self.showsyntaxerror(filename, source=source) + return False + except (OverflowError, ValueError): + self.showsyntaxerror(filename, source=source) + return False + + if code is None: + return True + + self.runcode(code) + return False diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py new file mode 100644 index 00000000000000..3a624d9f6835d1 --- /dev/null +++ b/Lib/_pyrepl/curses.py @@ -0,0 +1,33 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +try: + import _curses +except ImportError: + try: + import curses as _curses # type: ignore[no-redef] + except ImportError: + from . import _minimal_curses as _curses # type: ignore[no-redef] + +setupterm = _curses.setupterm +tigetstr = _curses.tigetstr +tparm = _curses.tparm +error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py new file mode 100644 index 00000000000000..5b85cb0f52521f --- /dev/null +++ b/Lib/_pyrepl/fancy_termios.py @@ -0,0 +1,74 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import termios + + +class TermState: + def __init__(self, tuples): + ( + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ) = tuples + + def as_list(self): + return [ + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ] + + def copy(self): + return self.__class__(self.as_list()) + + +def tcgetattr(fd): + return TermState(termios.tcgetattr(fd)) + + +def tcsetattr(fd, when, attrs): + termios.tcsetattr(fd, when, attrs.as_list()) + + +class Term(TermState): + TS__init__ = TermState.__init__ + + def __init__(self, fd=0): + self.TS__init__(termios.tcgetattr(fd)) + self.fd = fd + self.stack = [] + + def save(self): + self.stack.append(self.as_list()) + + def set(self, when=termios.TCSANOW): + termios.tcsetattr(self.fd, when, self.as_list()) + + def restore(self): + self.TS__init__(self.stack.pop()) + self.set() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py new file mode 100644 index 00000000000000..5d416f336ad5d2 --- /dev/null +++ b/Lib/_pyrepl/historical_reader.py @@ -0,0 +1,415 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field + +from . import commands, input +from .reader import Reader + + +if False: + from .types import SimpleContextManager, KeySpec, CommandName + + +isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"] + + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"] + + [ + ("\\%03o" % c, "isearch-add-character") + for c in range(256) + if chr(c).isalpha() and chr(c) != "\\" + ] + + [ + ("\\\\", "self-insert"), + (r"\C-r", "isearch-backwards"), + (r"\C-s", "isearch-forwards"), + (r"\C-c", "isearch-cancel"), + (r"\C-g", "isearch-cancel"), + (r"\", "isearch-backspace"), + ] +) + +ISEARCH_DIRECTION_NONE = "" +ISEARCH_DIRECTION_BACKWARDS = "r" +ISEARCH_DIRECTION_FORWARDS = "f" + + +class next_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == len(r.history): + r.error("end of history list") + return + r.select_item(r.historyi + 1) + + +class previous_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == 0: + r.error("start of history list") + return + r.select_item(r.historyi - 1) + + +class history_search_backward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=False) + + +class history_search_forward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=True) + + +class restore_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi != len(r.history): + if r.get_unicode() != r.history[r.historyi]: + r.buffer = list(r.history[r.historyi]) + r.pos = len(r.buffer) + r.dirty = True + + +class first_history(commands.Command): + def do(self) -> None: + self.reader.select_item(0) + + +class last_history(commands.Command): + def do(self) -> None: + self.reader.select_item(len(self.reader.history)) + + +class operate_and_get_next(commands.FinishCommand): + def do(self) -> None: + self.reader.next_history = self.reader.historyi + 1 + + +class yank_arg(commands.Command): + def do(self) -> None: + r = self.reader + if r.last_command is self.__class__: + r.yank_arg_i += 1 + else: + r.yank_arg_i = 0 + if r.historyi < r.yank_arg_i: + r.error("beginning of history list") + return + a = r.get_arg(-1) + # XXX how to split? + words = r.get_item(r.historyi - r.yank_arg_i - 1).split() + if a < -len(words) or a >= len(words): + r.error("no such arg") + return + w = words[a] + b = r.buffer + if r.yank_arg_i > 0: + o = len(r.yank_arg_yanked) + else: + o = 0 + b[r.pos - o : r.pos] = list(w) + r.yank_arg_yanked = w + r.pos += len(w) - o + r.dirty = True + + +class forward_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_start = r.historyi, r.pos + r.isearch_term = "" + r.dirty = True + r.push_input_trans(r.isearch_trans) + + +class reverse_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.dirty = True + r.isearch_term = "" + r.push_input_trans(r.isearch_trans) + r.isearch_start = r.historyi, r.pos + + +class isearch_cancel(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.pop_input_trans() + r.select_item(r.isearch_start[0]) + r.pos = r.isearch_start[1] + r.dirty = True + + +class isearch_add_character(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + r.isearch_term += self.event[-1] + r.dirty = True + p = r.pos + len(r.isearch_term) - 1 + if b[p : p + 1] != [r.isearch_term[-1]]: + r.isearch_next() + + +class isearch_backspace(commands.Command): + def do(self) -> None: + r = self.reader + if len(r.isearch_term) > 0: + r.isearch_term = r.isearch_term[:-1] + r.dirty = True + else: + r.error("nothing to rubout") + + +class isearch_forwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_next() + + +class isearch_backwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.isearch_next() + + +class isearch_end(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.console.forgetinput() + r.pop_input_trans() + r.dirty = True + + +@dataclass +class HistoricalReader(Reader): + """Adds history support (with incremental history searching) to the + Reader class. + """ + + history: list[str] = field(default_factory=list) + historyi: int = 0 + next_history: int | None = None + transient_history: dict[int, str] = field(default_factory=dict) + isearch_term: str = "" + isearch_direction: str = ISEARCH_DIRECTION_NONE + isearch_start: tuple[int, int] = field(init=False) + isearch_trans: input.KeymapTranslator = field(init=False) + yank_arg_i: int = 0 + yank_arg_yanked: str = "" + + def __post_init__(self) -> None: + super().__post_init__() + for c in [ + next_history, + previous_history, + restore_history, + first_history, + last_history, + yank_arg, + forward_history_isearch, + reverse_history_isearch, + isearch_end, + isearch_add_character, + isearch_cancel, + isearch_add_character, + isearch_backspace, + isearch_forwards, + isearch_backwards, + operate_and_get_next, + history_search_backward, + history_search_forward, + ]: + self.commands[c.__name__] = c + self.commands[c.__name__.replace("_", "-")] = c + self.isearch_start = self.historyi, self.pos + self.isearch_trans = input.KeymapTranslator( + isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character + ) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\C-n", "next-history"), + (r"\C-p", "previous-history"), + (r"\C-o", "operate-and-get-next"), + (r"\C-r", "reverse-history-isearch"), + (r"\C-s", "forward-history-isearch"), + (r"\M-r", "restore-history"), + (r"\M-.", "yank-arg"), + (r"\", "history-search-forward"), + (r"\x1b[6~", "history-search-forward"), + (r"\", "history-search-backward"), + (r"\x1b[5~", "history-search-backward"), + ) + + def select_item(self, i: int) -> None: + self.transient_history[self.historyi] = self.get_unicode() + buf = self.transient_history.get(i) + if buf is None: + buf = self.history[i].rstrip() + self.buffer = list(buf) + self.historyi = i + self.pos = len(self.buffer) + self.dirty = True + self.last_refresh_cache.invalidated = True + + def get_item(self, i: int) -> str: + if i != len(self.history): + return self.transient_history.get(i, self.history[i]) + else: + return self.transient_history.get(i, self.get_unicode()) + + @contextmanager + def suspend(self) -> SimpleContextManager: + with super().suspend(): + try: + old_history = self.history[:] + del self.history[:] + yield + finally: + self.history[:] = old_history + + def prepare(self) -> None: + super().prepare() + try: + self.transient_history = {} + if self.next_history is not None and self.next_history < len(self.history): + self.historyi = self.next_history + self.buffer[:] = list(self.history[self.next_history]) + self.pos = len(self.buffer) + self.transient_history[len(self.history)] = "" + else: + self.historyi = len(self.history) + self.next_history = None + except: + self.restore() + raise + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE: + d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS] + return "(%s-search `%s') " % (d, self.isearch_term) + else: + return super().get_prompt(lineno, cursor_on_line) + + def search_next(self, *, forwards: bool) -> None: + """Search history for the current line contents up to the cursor. + + Selects the first item found. If nothing is under the cursor, any next + item in history is selected. + """ + pos = self.pos + s = self.get_unicode() + history_index = self.historyi + + # In multiline contexts, we're only interested in the current line. + nl_index = s.rfind('\n', 0, pos) + prefix = s[nl_index + 1:pos] + pos = len(prefix) + + match_prefix = len(prefix) + len_item = 0 + if history_index < len(self.history): + len_item = len(self.get_item(history_index)) + if len_item and pos == len_item: + match_prefix = False + elif not pos: + match_prefix = False + + while 1: + if forwards: + out_of_bounds = history_index >= len(self.history) - 1 + else: + out_of_bounds = history_index == 0 + if out_of_bounds: + if forwards and not match_prefix: + self.pos = 0 + self.buffer = [] + self.dirty = True + else: + self.error("not found") + return + + history_index += 1 if forwards else -1 + s = self.get_item(history_index) + + if not match_prefix: + self.select_item(history_index) + return + + len_acc = 0 + for i, line in enumerate(s.splitlines(keepends=True)): + if line.startswith(prefix): + self.select_item(history_index) + self.pos = pos + len_acc + return + len_acc += len(line) + + def isearch_next(self) -> None: + st = self.isearch_term + p = self.pos + i = self.historyi + s = self.get_unicode() + forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS + while 1: + if forwards: + p = s.find(st, p + 1) + else: + p = s.rfind(st, 0, p + len(st) - 1) + if p != -1: + self.select_item(i) + self.pos = p + return + elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0): + self.error("not found") + return + else: + if forwards: + i += 1 + s = self.get_item(i) + p = -1 + else: + i -= 1 + s = self.get_item(i) + p = len(s) + + def finish(self) -> None: + super().finish() + ret = self.get_unicode() + for i, t in self.transient_history.items(): + if i < len(self.history) and i != self.historyi: + self.history[i] = t + if ret and should_auto_add_history: + self.history.append(ret) + + +should_auto_add_history = True diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py new file mode 100644 index 00000000000000..21c24eb5cde3e3 --- /dev/null +++ b/Lib/_pyrepl/input.py @@ -0,0 +1,114 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# (naming modules after builtin functions is not such a hot idea...) + +# an KeyTrans instance translates Event objects into Command objects + +# hmm, at what level do we want [C-i] and [tab] to be equivalent? +# [meta-a] and [esc a]? obviously, these are going to be equivalent +# for the UnixConsole, but should they be for PygameConsole? + +# it would in any situation seem to be a bad idea to bind, say, [tab] +# and [C-i] to *different* things... but should binding one bind the +# other? + +# executive, temporary decision: [tab] and [C-i] are distinct, but +# [meta-key] is identified with [esc key]. We demand that any console +# class does quite a lot towards emulating a unix terminal. + +from __future__ import annotations + +from abc import ABC, abstractmethod +import unicodedata +from collections import deque + + +# types +if False: + from .types import EventTuple + + +class InputTranslator(ABC): + @abstractmethod + def push(self, evt: EventTuple) -> None: + pass + + @abstractmethod + def get(self) -> EventTuple | None: + return None + + @abstractmethod + def empty(self) -> bool: + return True + + +class KeymapTranslator(InputTranslator): + def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None): + self.verbose = verbose + from .keymap import compile_keymap, parse_keys + + self.keymap = keymap + self.invalid_cls = invalid_cls + self.character_cls = character_cls + d = {} + for keyspec, command in keymap: + keyseq = tuple(parse_keys(keyspec)) + d[keyseq] = command + if self.verbose: + print(d) + self.k = self.ck = compile_keymap(d, ()) + self.results = deque() + self.stack = [] + + def push(self, evt): + if self.verbose: + print("pushed", evt.data, end="") + key = evt.data + d = self.k.get(key) + if isinstance(d, dict): + if self.verbose: + print("transition") + self.stack.append(key) + self.k = d + else: + if d is None: + if self.verbose: + print("invalid") + if self.stack or len(key) > 1 or unicodedata.category(key) == "C": + self.results.append((self.invalid_cls, self.stack + [key])) + else: + # small optimization: + self.k[key] = self.character_cls + self.results.append((self.character_cls, [key])) + else: + if self.verbose: + print("matched", d) + self.results.append((d, self.stack + [key])) + self.stack = [] + self.k = self.ck + + def get(self): + if self.results: + return self.results.popleft() + else: + return None + + def empty(self) -> bool: + return not self.results diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py new file mode 100644 index 00000000000000..d11df4b5164696 --- /dev/null +++ b/Lib/_pyrepl/keymap.py @@ -0,0 +1,213 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +Keymap contains functions for parsing keyspecs and turning keyspecs into +appropriate sequences. + +A keyspec is a string representing a sequence of key presses that can +be bound to a command. All characters other than the backslash represent +themselves. In the traditional manner, a backslash introduces an escape +sequence. + +pyrepl uses its own keyspec format that is meant to be a strict superset of +readline's KEYSEQ format. This means that if a spec is found that readline +accepts that this doesn't, it should be logged as a bug. Note that this means +we're using the '\\C-o' style of readline's keyspec, not the 'Control-o' sort. + +The extension to readline is that the sequence \\ denotes the +sequence of characters produced by hitting KEY. + +Examples: +'a' - what you get when you hit the 'a' key +'\\EOA' - Escape - O - A (up, on my terminal) +'\\' - the up arrow key +'\\' - ditto (keynames are case-insensitive) +'\\C-o', '\\c-o' - control-o +'\\M-.' - meta-period +'\\E.' - ditto (that's how meta works for pyrepl) +'\\', '\\', '\\t', '\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' + - all of these are the tab character. +""" + +_escapes = { + "\\": "\\", + "'": "'", + '"': '"', + "a": "\a", + "b": "\b", + "e": "\033", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", +} + +_keynames = { + "backspace": "backspace", + "delete": "delete", + "down": "down", + "end": "end", + "enter": "\r", + "escape": "\033", + "f1": "f1", + "f2": "f2", + "f3": "f3", + "f4": "f4", + "f5": "f5", + "f6": "f6", + "f7": "f7", + "f8": "f8", + "f9": "f9", + "f10": "f10", + "f11": "f11", + "f12": "f12", + "f13": "f13", + "f14": "f14", + "f15": "f15", + "f16": "f16", + "f17": "f17", + "f18": "f18", + "f19": "f19", + "f20": "f20", + "home": "home", + "insert": "insert", + "left": "left", + "page down": "page down", + "page up": "page up", + "return": "\r", + "right": "right", + "space": " ", + "tab": "\t", + "up": "up", +} + + +class KeySpecError(Exception): + pass + + +def parse_keys(keys: str) -> list[str]: + """Parse keys in keyspec format to a sequence of keys.""" + s = 0 + r: list[str] = [] + while s < len(keys): + k, s = _parse_single_key_sequence(keys, s) + r.extend(k) + return r + + +def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]: + ctrl = 0 + meta = 0 + ret = "" + while not ret and s < len(key): + if key[s] == "\\": + c = key[s + 1].lower() + if c in _escapes: + ret = _escapes[c] + s += 2 + elif c == "c": + if key[s + 2] != "-": + raise KeySpecError( + "\\C must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if ctrl: + raise KeySpecError( + "doubled \\C- (char %d of %s)" % (s + 1, repr(key)) + ) + ctrl = 1 + s += 3 + elif c == "m": + if key[s + 2] != "-": + raise KeySpecError( + "\\M must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if meta: + raise KeySpecError( + "doubled \\M- (char %d of %s)" % (s + 1, repr(key)) + ) + meta = 1 + s += 3 + elif c.isdigit(): + n = key[s + 1 : s + 4] + ret = chr(int(n, 8)) + s += 4 + elif c == "x": + n = key[s + 2 : s + 4] + ret = chr(int(n, 16)) + s += 4 + elif c == "<": + t = key.find(">", s) + if t == -1: + raise KeySpecError( + "unterminated \\< starting at char %d of %s" + % (s + 1, repr(key)) + ) + ret = key[s + 2 : t].lower() + if ret not in _keynames: + raise KeySpecError( + "unrecognised keyname `%s' at char %d of %s" + % (ret, s + 2, repr(key)) + ) + ret = _keynames[ret] + s = t + 1 + else: + raise KeySpecError( + "unknown backslash escape %s at char %d of %s" + % (repr(c), s + 2, repr(key)) + ) + else: + ret = key[s] + s += 1 + if ctrl: + if len(ret) == 1: + ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + elif ret in {"left", "right"}: + ret = f"ctrl {ret}" + else: + raise KeySpecError("\\C- followed by invalid key") + + result = [ret], s + if meta: + result[0].insert(0, "\033") + return result + + +def compile_keymap(keymap, empty=b""): + r = {} + for key, value in keymap.items(): + if isinstance(key, bytes): + first = key[:1] + else: + first = key[0] + r.setdefault(first, {})[key[1:]] = value + for key, value in r.items(): + if empty in value: + if len(value) != 1: + raise KeySpecError("key definitions for %s clash" % (value.values(),)) + else: + r[key] = value[empty] + else: + r[key] = compile_keymap(value, empty) + return r diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py new file mode 100644 index 00000000000000..a6f824dcc4ad14 --- /dev/null +++ b/Lib/_pyrepl/main.py @@ -0,0 +1,59 @@ +import errno +import os +import sys + + +CAN_USE_PYREPL: bool +FAIL_REASON: str +try: + if sys.platform == "win32" and sys.getwindowsversion().build < 10586: + raise RuntimeError("Windows 10 TH2 or later required") + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + from .simple_interact import check + if err := check(): + raise RuntimeError(err) +except Exception as e: + CAN_USE_PYREPL = False + FAIL_REASON = f"warning: can't use pyrepl: {e}" +else: + CAN_USE_PYREPL = True + FAIL_REASON = "" + + +def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): + if not CAN_USE_PYREPL: + if not os.getenv('PYTHON_BASIC_REPL') and FAIL_REASON: + from .trace import trace + trace(FAIL_REASON) + print(FAIL_REASON, file=sys.stderr) + return sys._baserepl() + + if mainmodule: + namespace = mainmodule.__dict__ + else: + import __main__ + namespace = __main__.__dict__ + namespace.pop("__pyrepl_interactive_console", None) + + # sys._baserepl() above does this internally, we do it here + startup_path = os.getenv("PYTHONSTARTUP") + if pythonstartup and startup_path: + sys.audit("cpython.run_startup", startup_path) + + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, namespace) + + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + if not hasattr(sys, "ps2"): + sys.ps2 = "... " + + from .console import InteractiveColoredConsole + from .simple_interact import run_multiline_interactive_console + console = InteractiveColoredConsole(namespace, filename="") + run_multiline_interactive_console(console) diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini new file mode 100644 index 00000000000000..395f5945ab740b --- /dev/null +++ b/Lib/_pyrepl/mypy.ini @@ -0,0 +1,24 @@ +# Config file for running mypy on _pyrepl. +# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/_pyrepl +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code,redundant-expr +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py new file mode 100644 index 00000000000000..1fddc63e3ee3ad --- /dev/null +++ b/Lib/_pyrepl/pager.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import io +import os +import re +import sys + + +# types +if False: + from typing import Protocol + class Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: + ... + + +def get_pager() -> Pager: + """Decide what method to use for paging through text.""" + if not hasattr(sys.stdin, "isatty"): + return plain_pager + if not hasattr(sys.stdout, "isatty"): + return plain_pager + if not sys.stdin.isatty() or not sys.stdout.isatty(): + return plain_pager + if sys.platform == "emscripten": + return plain_pager + use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') + if use_pager: + if sys.platform == 'win32': # pipes completely broken in Windows + return lambda text, title='': tempfile_pager(plain(text), use_pager) + elif os.environ.get('TERM') in ('dumb', 'emacs'): + return lambda text, title='': pipe_pager(plain(text), use_pager, title) + else: + return lambda text, title='': pipe_pager(text, use_pager, title) + if os.environ.get('TERM') in ('dumb', 'emacs'): + return plain_pager + if sys.platform == 'win32': + return lambda text, title='': tempfile_pager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'pager', title) + if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'less', title) + + import tempfile + (fd, filename) = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: + return lambda text, title='': pipe_pager(text, 'more', title) + else: + return tty_pager + finally: + os.unlink(filename) + + +def escape_stdout(text: str) -> str: + # Escape non-encodable characters to avoid encoding errors later + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + return text.encode(encoding, 'backslashreplace').decode(encoding) + + +def escape_less(s: str) -> str: + return re.sub(r'([?:.%\\])', r'\\\1', s) + + +def plain(text: str) -> str: + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + +def tty_pager(text: str, title: str = '') -> None: + """Page through text on a text terminal.""" + lines = plain(escape_stdout(text)).split('\n') + has_tty = False + try: + import tty + import termios + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + tty.setcbreak(fd) + has_tty = True + + def getchar() -> str: + return sys.stdin.read(1) + + except (ImportError, AttributeError, io.UnsupportedOperation): + def getchar() -> str: + return sys.stdin.readline()[:-1][:1] + + try: + try: + h = int(os.environ.get('LINES', 0)) + except ValueError: + h = 0 + if h <= 1: + h = 25 + r = inc = h - 1 + sys.stdout.write('\n'.join(lines[:inc]) + '\n') + while lines[r:]: + sys.stdout.write('-- more --') + sys.stdout.flush() + c = getchar() + + if c in ('q', 'Q'): + sys.stdout.write('\r \r') + break + elif c in ('\r', '\n'): + sys.stdout.write('\r \r' + lines[r] + '\n') + r = r + 1 + continue + if c in ('b', 'B', '\x1b'): + r = r - inc - inc + if r < 0: r = 0 + sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') + r = r + inc + + finally: + if has_tty: + termios.tcsetattr(fd, termios.TCSAFLUSH, old) + + +def plain_pager(text: str, title: str = '') -> None: + """Simply print unformatted text. This is the ultimate fallback.""" + sys.stdout.write(plain(escape_stdout(text))) + + +def pipe_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by feeding it to another program.""" + import subprocess + env = os.environ.copy() + if title: + title += ' ' + esc_title = escape_less(title) + prompt_string = ( + f' {esc_title}' + + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace', env=env) + assert proc.stdin is not None + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + # We've hereby abandoned whatever text hasn't been written, + # but the pager is still in control of the terminal. + pass + except OSError: + pass # Ignore broken pipes caused by quitting the pager program. + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + # Ignore ctl-c like the pager itself does. Otherwise the pager is + # left running and the terminal is in raw mode and unusable. + pass + + +def tempfile_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by invoking a program on a temporary file.""" + import tempfile + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) + os.system(cmd + ' "' + filename + '"') diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py new file mode 100644 index 00000000000000..4b0700d069c621 --- /dev/null +++ b/Lib/_pyrepl/reader.py @@ -0,0 +1,815 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import sys + +from contextlib import contextmanager +from dataclasses import dataclass, field, fields +import unicodedata +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] + + +from . import commands, console, input +from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width +from .trace import trace + + +# types +Command = commands.Command +from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def disp_str(buffer: str) -> tuple[str, list[int]]: + """disp_str(buffer:string) -> (string, [int]) + + Return the string that should be the printed representation of + |buffer| and a list detailing where the characters of |buffer| + get used up. E.g.: + + >>> disp_str(chr(3)) + ('^C', [1, 0]) + + """ + b: list[int] = [] + s: list[str] = [] + for c in buffer: + if c == '\x1a': + s.append(c) + b.append(2) + elif ord(c) < 128: + s.append(c) + b.append(1) + elif unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + s.append(c) + b.extend([0] * (len(c) - 1)) + else: + s.append(c) + b.append(str_width(c)) + return "".join(s), b + + +# syntax classes: + +SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) + + +def make_default_syntax_table() -> dict[str, int]: + # XXX perhaps should use some unicodedata here? + st: dict[str, int] = {} + for c in map(chr, range(256)): + st[c] = SYNTAX_SYMBOL + for c in [a for a in map(chr, range(256)) if a.isalnum()]: + st[c] = SYNTAX_WORD + st["\n"] = st[" "] = SYNTAX_WHITESPACE + return st + + +def make_default_commands() -> dict[CommandName, type[Command]]: + result: dict[CommandName, type[Command]] = {} + for v in vars(commands).values(): + if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): + result[v.__name__] = v + result[v.__name__.replace("_", "-")] = v + return result + + +default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [ + (r"\C-a", "beginning-of-line"), + (r"\C-b", "left"), + (r"\C-c", "interrupt"), + (r"\C-d", "delete"), + (r"\C-e", "end-of-line"), + (r"\C-f", "right"), + (r"\C-g", "cancel"), + (r"\C-h", "backspace"), + (r"\C-j", "accept"), + (r"\", "accept"), + (r"\C-k", "kill-line"), + (r"\C-l", "clear-screen"), + (r"\C-m", "accept"), + (r"\C-t", "transpose-characters"), + (r"\C-u", "unix-line-discard"), + (r"\C-w", "unix-word-rubout"), + (r"\C-x\C-u", "upcase-region"), + (r"\C-y", "yank"), + *(() if sys.platform == "win32" else ((r"\C-z", "suspend"), )), + (r"\M-b", "backward-word"), + (r"\M-c", "capitalize-word"), + (r"\M-d", "kill-word"), + (r"\M-f", "forward-word"), + (r"\M-l", "downcase-word"), + (r"\M-t", "transpose-words"), + (r"\M-u", "upcase-word"), + (r"\M-y", "yank-pop"), + (r"\M--", "digit-arg"), + (r"\M-0", "digit-arg"), + (r"\M-1", "digit-arg"), + (r"\M-2", "digit-arg"), + (r"\M-3", "digit-arg"), + (r"\M-4", "digit-arg"), + (r"\M-5", "digit-arg"), + (r"\M-6", "digit-arg"), + (r"\M-7", "digit-arg"), + (r"\M-8", "digit-arg"), + (r"\M-9", "digit-arg"), + (r"\M-\n", "accept"), + ("\\\\", "self-insert"), + (r"\x1b[200~", "enable_bracketed_paste"), + (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x03", "ctrl-c"), + ] + + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] + + [ + (r"\", "up"), + (r"\", "down"), + (r"\", "left"), + (r"\C-\", "backward-word"), + (r"\", "right"), + (r"\C-\", "forward-word"), + (r"\", "delete"), + (r"\x1b[3~", "delete"), + (r"\", "backspace"), + (r"\M-\", "backward-kill-word"), + (r"\", "end-of-line"), # was 'end' + (r"\", "beginning-of-line"), # was 'home' + (r"\", "help"), + (r"\", "show-history"), + (r"\", "paste-mode"), + (r"\EOF", "end"), # the entries in the terminfo database for xterms + (r"\EOH", "home"), # seem to be wrong. this is a less than ideal + # workaround + ] +) + + +@dataclass(slots=True) +class Reader: + """The Reader class implements the bare bones of a command reader, + handling such details as editing and cursor motion. What it does + not support are such things as completion or history support - + these are implemented elsewhere. + + Instance variables of note include: + + * buffer: + A *list* (*not* a string at the moment :-) containing all the + characters that have been entered. + * console: + Hopefully encapsulates the OS dependent stuff. + * pos: + A 0-based index into 'buffer' for where the insertion point + is. + * screeninfo: + Ahem. This list contains some info needed to move the + insertion point around reasonably efficiently. + * cxy, lxy: + the position of the insertion point in screen ... + * syntax_table: + Dictionary mapping characters to 'syntax class'; read the + emacs docs to see what this means :-) + * commands: + Dictionary mapping command names to command classes. + * arg: + The emacs-style prefix argument. It will be None if no such + argument has been provided. + * dirty: + True if we need to refresh the display. + * kill_ring: + The emacs-style kill-ring; manipulated with yank & yank-pop + * ps1, ps2, ps3, ps4: + prompts. ps1 is the prompt for a one-line input; for a + multiline input it looks like: + ps2> first line of input goes here + ps3> second and further + ps3> lines get ps3 + ... + ps4> and the last one gets ps4 + As with the usual top-level, you can set these to instances if + you like; str() will be called on them (once) at the beginning + of each command. Don't put really long or newline containing + strings here, please! + This is just the default policy; you can change it freely by + overriding get_prompt() (and indeed some standard subclasses + do). + * finished: + handle1 will set this to a true value if a command signals + that we're done. + """ + + console: console.Console + + ## state + buffer: list[str] = field(default_factory=list) + pos: int = 0 + ps1: str = "->> " + ps2: str = "/>> " + ps3: str = "|.. " + ps4: str = R"\__ " + kill_ring: list[list[str]] = field(default_factory=list) + msg: str = "" + arg: int | None = None + dirty: bool = False + finished: bool = False + paste_mode: bool = False + in_bracketed_paste: bool = False + commands: dict[str, type[Command]] = field(default_factory=make_default_commands) + last_command: type[Command] | None = None + syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) + keymap: tuple[tuple[str, str], ...] = () + input_trans: input.KeymapTranslator = field(init=False) + input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + cxy: tuple[int, int] = field(init=False) + lxy: tuple[int, int] = field(init=False) + scheduled_commands: list[str] = field(default_factory=list) + can_colorize: bool = False + threading_hook: Callback | None = None + + ## cached metadata to speed up screen refreshes + @dataclass + class RefreshCache: + in_bracketed_paste: bool = False + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + line_end_offsets: list[int] = field(default_factory=list) + pos: int = field(init=False) + cxy: tuple[int, int] = field(init=False) + dimensions: tuple[int, int] = field(init=False) + invalidated: bool = False + + def update_cache(self, + reader: Reader, + screen: list[str], + screeninfo: list[tuple[int, list[int]]], + ) -> None: + self.in_bracketed_paste = reader.in_bracketed_paste + self.screen = screen.copy() + self.screeninfo = screeninfo.copy() + self.pos = reader.pos + self.cxy = reader.cxy + self.dimensions = reader.console.width, reader.console.height + self.invalidated = False + + def valid(self, reader: Reader) -> bool: + if self.invalidated: + return False + dimensions = reader.console.width, reader.console.height + dimensions_changed = dimensions != self.dimensions + paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste + return not (dimensions_changed or paste_changed) + + def get_cached_location(self, reader: Reader) -> tuple[int, int]: + if self.invalidated: + raise ValueError("Cache is invalidated") + offset = 0 + earliest_common_pos = min(reader.pos, self.pos) + num_common_lines = len(self.line_end_offsets) + while num_common_lines > 0: + offset = self.line_end_offsets[num_common_lines - 1] + if earliest_common_pos > offset: + break + num_common_lines -= 1 + else: + offset = 0 + return offset, num_common_lines + + last_refresh_cache: RefreshCache = field(default_factory=RefreshCache) + + def __post_init__(self) -> None: + # Enable the use of `insert` without a `prepare` call - necessary to + # facilitate the tab completion hack implemented for + # . + self.keymap = self.collect_keymap() + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + self.screeninfo = [(0, [])] + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + self.can_colorize = can_colorize() + + self.last_refresh_cache.screeninfo = self.screeninfo + self.last_refresh_cache.pos = self.pos + self.last_refresh_cache.cxy = self.cxy + self.last_refresh_cache.dimensions = (0, 0) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return default_keymap + + def calc_screen(self) -> list[str]: + """Translate changes in self.buffer into changes in self.console.screen.""" + # Since the last call to calc_screen: + # screen and screeninfo may differ due to a completion menu being shown + # pos and cxy may differ due to edits, cursor movements, or completion menus + + # Lines that are above both the old and new cursor position can't have changed, + # unless the terminal has been resized (which might cause reflowing) or we've + # entered or left paste mode (which changes prompts, causing reflowing). + num_common_lines = 0 + offset = 0 + if self.last_refresh_cache.valid(self): + offset, num_common_lines = self.last_refresh_cache.get_cached_location(self) + + screen = self.last_refresh_cache.screen + del screen[num_common_lines:] + + screeninfo = self.last_refresh_cache.screeninfo + del screeninfo[num_common_lines:] + + last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets + del last_refresh_line_end_offsets[num_common_lines:] + + pos = self.pos + pos -= offset + + prompt_from_cache = (offset and self.buffer[offset - 1] != "\n") + + lines = "".join(self.buffer[offset:]).split("\n") + + cursor_found = False + lines_beyond_cursor = 0 + for ln, line in enumerate(lines, num_common_lines): + ll = len(line) + if 0 <= pos <= ll: + self.lxy = pos, ln + cursor_found = True + elif cursor_found: + lines_beyond_cursor += 1 + if lines_beyond_cursor > self.console.height: + # No need to keep formatting lines. + # The console can't show them. + break + if prompt_from_cache: + # Only the first line's prompt can come from the cache + prompt_from_cache = False + prompt = "" + else: + prompt = self.get_prompt(ln, ll >= pos >= 0) + while "\n" in prompt: + pre_prompt, _, prompt = prompt.partition("\n") + last_refresh_line_end_offsets.append(offset) + screen.append(pre_prompt) + screeninfo.append((0, [])) + pos -= ll + 1 + prompt, lp = self.process_prompt(prompt) + l, l2 = disp_str(line) + wrapcount = (wlen(l) + lp) // self.console.width + if wrapcount == 0: + offset += ll + 1 # Takes all of the line plus the newline + last_refresh_line_end_offsets.append(offset) + screen.append(prompt + l) + screeninfo.append((lp, l2)) + else: + i = 0 + while l: + prelen = lp if i == 0 else 0 + index_to_wrap_before = 0 + column = 0 + for character_width in l2: + if column + character_width >= self.console.width - prelen: + break + index_to_wrap_before += 1 + column += character_width + pre = prompt if i == 0 else "" + if len(l) > index_to_wrap_before: + offset += index_to_wrap_before + post = "\\" + after = [1] + else: + offset += index_to_wrap_before + 1 # Takes the newline + post = "" + after = [] + last_refresh_line_end_offsets.append(offset) + screen.append(pre + l[:index_to_wrap_before] + post) + screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) + l = l[index_to_wrap_before:] + l2 = l2[index_to_wrap_before:] + i += 1 + self.screeninfo = screeninfo + self.cxy = self.pos2xy() + if self.msg: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + + self.last_refresh_cache.update_cache(self, screen, screeninfo) + return screen + + @staticmethod + def process_prompt(prompt: str) -> tuple[str, int]: + """Process the prompt. + + This means calculate the length of the prompt. The character \x01 + and \x02 are used to bracket ANSI control sequences and need to be + excluded from the length calculation. So also a copy of the prompt + is returned with these control characters removed.""" + + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + + # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars, + # which breaks the logic below so we redefine it here. + def wlen(s: str) -> int: + return sum(str_width(i) for i in s) + + out_prompt = "" + l = wlen(prompt) + pos = 0 + while True: + s = prompt.find("\x01", pos) + if s == -1: + break + e = prompt.find("\x02", s) + if e == -1: + break + # Found start and end brackets, subtract from string length + l = l - (e - s + 1) + keep = prompt[pos:s] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + prompt[s + 1 : e] + pos = e + 1 + keep = prompt[pos:] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + return out_prompt, l + + def bow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break preceding p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p -= 1 + return p + 1 + + def eow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break following p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p += 1 + while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p += 1 + return p + + def bol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break preceding p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + p -= 1 + while p >= 0 and b[p] != "\n": + p -= 1 + return p + 1 + + def eol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break following p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + while p < len(b) and b[p] != "\n": + p += 1 + return p + + def max_column(self, y: int) -> int: + """Return the last x-offset for line y""" + return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + + def max_row(self) -> int: + return len(self.screeninfo) - 1 + + def get_arg(self, default: int = 1) -> int: + """Return any prefix argument that the user has supplied, + returning 'default' if there is None. Defaults to 1. + """ + if self.arg is None: + return default + return self.arg + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + """Return what should be in the left-hand margin for line + 'lineno'.""" + if self.arg is not None and cursor_on_line: + prompt = f"(arg: {self.arg}) " + elif self.paste_mode and not self.in_bracketed_paste: + prompt = "(paste) " + elif "\n" in self.buffer: + if lineno == 0: + prompt = self.ps2 + elif self.ps4 and lineno == self.buffer.count("\n"): + prompt = self.ps4 + else: + prompt = self.ps3 + else: + prompt = self.ps1 + + if self.can_colorize: + prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" + return prompt + + def push_input_trans(self, itrans: input.KeymapTranslator) -> None: + self.input_trans_stack.append(self.input_trans) + self.input_trans = itrans + + def pop_input_trans(self) -> None: + self.input_trans = self.input_trans_stack.pop() + + def setpos_from_xy(self, x: int, y: int) -> None: + """Set pos according to coordinates x, y""" + pos = 0 + i = 0 + while i < y: + prompt_len, character_widths = self.screeninfo[i] + offset = len(character_widths) - character_widths.count(0) + in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + if in_wrapped_line: + pos += offset - 1 # -1 cause backslash is not in buffer + else: + pos += offset + 1 # +1 cause newline is in buffer + i += 1 + + j = 0 + cur_x = self.screeninfo[i][0] + while cur_x < x: + if self.screeninfo[i][1][j] == 0: + continue + cur_x += self.screeninfo[i][1][j] + j += 1 + pos += 1 + + self.pos = pos + + def pos2xy(self) -> tuple[int, int]: + """Return the x, y coordinates of position 'pos'.""" + # this *is* incomprehensible, yes. + y = 0 + pos = self.pos + assert 0 <= pos <= len(self.buffer) + if pos == len(self.buffer): + y = len(self.screeninfo) - 1 + p, l2 = self.screeninfo[y] + return p + sum(l2) + l2.count(0), y + + for p, l2 in self.screeninfo: + l = len(l2) - l2.count(0) + in_wrapped_line = p + sum(l2) >= self.console.width + offset = l - 1 if in_wrapped_line else l # need to remove backslash + if offset >= pos: + break + + if p + sum(l2) >= self.console.width: + pos -= l - 1 # -1 cause backslash is not in buffer + else: + pos -= l + 1 # +1 cause newline is in buffer + y += 1 + return p + sum(l2[:pos]), y + + def insert(self, text: str | list[str]) -> None: + """Insert 'text' at the insertion point.""" + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.dirty = True + + def update_cursor(self) -> None: + """Move the cursor to reflect changes in self.pos""" + self.cxy = self.pos2xy() + self.console.move_cursor(*self.cxy) + + def after_command(self, cmd: Command) -> None: + """This function is called to allow post command cleanup.""" + if getattr(cmd, "kills_digit_arg", True): + if self.arg is not None: + self.dirty = True + self.arg = None + + def prepare(self) -> None: + """Get ready to run. Call restore when finished. You must not + write to the console in between the calls to prepare and + restore.""" + try: + self.console.prepare() + self.arg = None + self.finished = False + del self.buffer[:] + self.pos = 0 + self.dirty = True + self.last_command = None + self.calc_screen() + except BaseException: + self.restore() + raise + + while self.scheduled_commands: + cmd = self.scheduled_commands.pop() + self.do_cmd((cmd, [])) + + def last_command_is(self, cls: type) -> bool: + if not self.last_command: + return False + return issubclass(cls, self.last_command) + + def restore(self) -> None: + """Clean up after a run.""" + self.console.restore() + + @contextmanager + def suspend(self) -> SimpleContextManager: + """A context manager to delegate to another reader.""" + prev_state = {f.name: getattr(self, f.name) for f in fields(self)} + try: + self.restore() + yield + finally: + for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"): + setattr(self, arg, prev_state[arg]) + self.prepare() + + def finish(self) -> None: + """Called when a command signals that we're finished.""" + pass + + def error(self, msg: str = "none") -> None: + self.msg = "! " + msg + " " + self.dirty = True + self.console.beep() + + def update_screen(self) -> None: + if self.dirty: + self.refresh() + + def refresh(self) -> None: + """Recalculate and refresh the screen.""" + if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n": + return + + # this call sets up self.cxy, so call it first. + self.screen = self.calc_screen() + self.console.refresh(self.screen, self.cxy) + self.dirty = False + + def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + """`cmd` is a tuple of "event_name" and "event", which in the current + implementation is always just the "buffer" which happens to be a list + of single-character strings.""" + + trace("received command {cmd}", cmd=cmd) + if isinstance(cmd[0], str): + command_type = self.commands.get(cmd[0], commands.invalid_command) + elif isinstance(cmd[0], type): + command_type = cmd[0] + else: + return # nothing to do + + command = command_type(self, *cmd) # type: ignore[arg-type] + command.do() + + self.after_command(command) + + if self.dirty: + self.refresh() + else: + self.update_cursor() + + if not isinstance(cmd, commands.digit_arg): + self.last_command = command_type + + self.finished = bool(command.finish) + if self.finished: + self.console.finish() + self.finish() + + def run_hooks(self) -> None: + threading_hook = self.threading_hook + if threading_hook is None and 'threading' in sys.modules: + from ._threading_handler import install_threading_hook + install_threading_hook(self) + if threading_hook is not None: + try: + threading_hook() + except Exception: + pass + + input_hook = self.console.input_hook + if input_hook: + try: + input_hook() + except Exception: + pass + + def handle1(self, block: bool = True) -> bool: + """Handle a single event. Wait as long as it takes if block + is true (the default), otherwise return False if no event is + pending.""" + + if self.msg: + self.msg = "" + self.dirty = True + + while True: + # We use the same timeout as in readline.c: 100ms + self.run_hooks() + self.console.wait(100) + event = self.console.get_event(block=False) + if not event: + if block: + continue + return False + + translate = True + + if event.evt == "key": + self.input_trans.push(event) + elif event.evt == "scroll": + self.refresh() + elif event.evt == "resize": + self.refresh() + else: + translate = False + + if translate: + cmd = self.input_trans.get() + else: + cmd = [event.evt, event.data] + + if cmd is None: + if block: + continue + return False + + self.do_cmd(cmd) + return True + + def push_char(self, char: int | bytes) -> None: + self.console.push_char(char) + self.handle1(block=False) + + def readline(self, startup_hook: Callback | None = None) -> str: + """Read a line. The implementation of this method also shows + how to drive Reader if you want more control over the event + loop.""" + self.prepare() + try: + if startup_hook is not None: + startup_hook() + self.refresh() + while not self.finished: + self.handle1() + return self.get_unicode() + + finally: + self.restore() + + def bind(self, spec: KeySpec, command: CommandName) -> None: + self.keymap = self.keymap + ((spec, command),) + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + + def get_unicode(self) -> str: + """Return the current buffer as a unicode string.""" + return "".join(self.buffer) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py new file mode 100644 index 00000000000000..5e1d3085874380 --- /dev/null +++ b/Lib/_pyrepl/readline.py @@ -0,0 +1,596 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Alex Gaynor +# Antonio Cuni +# Armin Rigo +# Holger Krekel +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""A compatibility wrapper reimplementing the 'readline' standard module +on top of pyrepl. Not all functionalities are supported. Contains +extensions for multiline input. +""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field + +import os +from site import gethistoryfile # type: ignore[attr-defined] +import sys +from rlcompleter import Completer as RLCompleter + +from . import commands, historical_reader +from .completing_reader import CompletingReader +from .console import Console as ConsoleType + +Console: type[ConsoleType] +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import UnixConsole as Console, _error +except ImportError: + from .windows_console import WindowsConsole as Console, _error + +ENCODING = sys.getdefaultencoding() or "latin1" + + +# types +Command = commands.Command +from collections.abc import Callable, Collection +from .types import Callback, Completer, KeySpec, CommandName + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any, Mapping + + +MoreLinesCallable = Callable[[str], bool] + + +__all__ = [ + "add_history", + "clear_history", + "get_begidx", + "get_completer", + "get_completer_delims", + "get_current_history_length", + "get_endidx", + "get_history_item", + "get_history_length", + "get_line_buffer", + "insert_text", + "parse_and_bind", + "read_history_file", + # "read_init_file", + # "redisplay", + "remove_history_item", + "replace_history_item", + "set_auto_history", + "set_completer", + "set_completer_delims", + "set_history_length", + # "set_pre_input_hook", + "set_startup_hook", + "write_history_file", + # ---- multiline extensions ---- + "multiline_input", +] + +# ____________________________________________________________ + +@dataclass +class ReadlineConfig: + readline_completer: Completer | None = None + completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") + + +@dataclass(kw_only=True) +class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): + # Class fields + assume_immutable_completions = False + use_brackets = False + sort_in_column = True + + # Instance fields + config: ReadlineConfig + more_lines: MoreLinesCallable | None = None + last_used_indentation: str | None = None + + def __post_init__(self) -> None: + super().__post_init__() + self.commands["maybe_accept"] = maybe_accept + self.commands["maybe-accept"] = maybe_accept + self.commands["backspace_dedent"] = backspace_dedent + self.commands["backspace-dedent"] = backspace_dedent + + def error(self, msg: str = "none") -> None: + pass # don't show error messages by default + + def get_stem(self) -> str: + b = self.buffer + p = self.pos - 1 + completer_delims = self.config.completer_delims + while p >= 0 and b[p] not in completer_delims: + p -= 1 + return "".join(b[p + 1 : self.pos]) + + def get_completions(self, stem: str) -> list[str]: + if len(stem) == 0 and self.more_lines is not None: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != "\n": + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [" " * num_spaces] + result = [] + function = self.config.readline_completer + if function is not None: + try: + stem = str(stem) # rlcompleter.py seems to not like unicode + except UnicodeEncodeError: + pass # but feed unicode anyway if we have no choice + state = 0 + while True: + try: + next = function(stem, state) + except Exception: + break + if not isinstance(next, str): + break + result.append(next) + state += 1 + # emulate the behavior of the standard readline that sorts + # the completions before displaying them. + result.sort() + return result + + def get_trimmed_history(self, maxlength: int) -> list[str]: + if maxlength >= 0: + cut = len(self.history) - maxlength + if cut < 0: + cut = 0 + else: + cut = 0 + return self.history[cut:] + + def update_last_used_indentation(self) -> None: + indentation = _get_first_indentation(self.buffer) + if indentation is not None: + self.last_used_indentation = indentation + + # --- simplified support for reading multiline Python statements --- + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\n", "maybe-accept"), + (r"\", "backspace-dedent"), + ) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if self.more_lines is None: + # Force single-line input if we are in raw_input() mode. + # Although there is no direct way to add a \n in this mode, + # multiline buffers can still show up using various + # commands, e.g. navigating the history. + try: + index = self.buffer.index("\n") + except ValueError: + pass + else: + self.buffer = self.buffer[:index] + if self.pos > len(self.buffer): + self.pos = len(self.buffer) + + +def set_auto_history(_should_auto_add_history: bool) -> None: + """Enable or disable automatic history""" + historical_reader.should_auto_add_history = bool(_should_auto_add_history) + + +def _get_this_line_indent(buffer: list[str], pos: int) -> int: + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + + +def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]: + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + + +def _get_first_indentation(buffer: list[str]) -> str | None: + indented_line_start = None + for i in range(len(buffer)): + if (i < len(buffer) - 1 + and buffer[i] == "\n" + and buffer[i + 1] in " \t" + ): + indented_line_start = i + 1 + elif indented_line_start is not None and buffer[i] not in " \t\n": + return ''.join(buffer[indented_line_start : i]) + return None + + +def _should_auto_indent(buffer: list[str], pos: int) -> bool: + # check if last character before "pos" is a colon, ignoring + # whitespaces and comments. + last_char = None + while pos > 0: + pos -= 1 + if last_char is None: + if buffer[pos] not in " \t\n#": # ignore whitespaces and comments + last_char = buffer[pos] + else: + # even if we found a non-whitespace character before + # original pos, we keep going back until newline is reached + # to make sure we ignore comments + if buffer[pos] == "\n": + break + if buffer[pos] == "#": + last_char = None + return last_char == ":" + + +class maybe_accept(commands.Command): + def do(self) -> None: + r: ReadlineAlikeReader + r = self.reader # type: ignore[assignment] + r.dirty = True # this is needed to hide the completion menu, if visible + + if self.reader.in_bracketed_paste: + r.insert("\n") + return + + # if there are already several lines and the cursor + # is not on the last one, always insert a new \n. + text = r.get_unicode() + + if "\n" in r.buffer[r.pos :] or ( + r.more_lines is not None and r.more_lines(text) + ): + def _newline_before_pos(): + before_idx = r.pos - 1 + while before_idx > 0 and text[before_idx].isspace(): + before_idx -= 1 + return text[before_idx : r.pos].count("\n") > 0 + + # if there's already a new line before the cursor then + # even if the cursor is followed by whitespace, we assume + # the user is trying to terminate the block + if _newline_before_pos() and text[r.pos:].isspace(): + self.finish = True + return + + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) + r.insert("\n") + if not self.reader.paste_mode: + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) + r.update_last_used_indentation() + if _should_auto_indent(r.buffer, r.pos): + if r.last_used_indentation is not None: + indentation = r.last_used_indentation + else: + # default + indentation = " " * 4 + r.insert(indentation) + elif not self.reader.paste_mode: + self.finish = True + else: + r.insert("\n") + + +class backspace_dedent(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos : r.pos + repeat] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +# ____________________________________________________________ + + +@dataclass(slots=True) +class _ReadlineWrapper: + f_in: int = -1 + f_out: int = -1 + reader: ReadlineAlikeReader | None = field(default=None, repr=False) + saved_history_length: int = -1 + startup_hook: Callback | None = None + config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False) + + def __post_init__(self) -> None: + if self.f_in == -1: + self.f_in = os.dup(0) + if self.f_out == -1: + self.f_out = os.dup(1) + + def get_reader(self) -> ReadlineAlikeReader: + if self.reader is None: + console = Console(self.f_in, self.f_out, encoding=ENCODING) + self.reader = ReadlineAlikeReader(console=console, config=self.config) + return self.reader + + def input(self, prompt: object = "") -> str: + try: + reader = self.get_reader() + except _error: + assert raw_input is not None + return raw_input(prompt) + prompt_str = str(prompt) + reader.ps1 = prompt_str + sys.audit("builtins.input", prompt_str) + result = reader.readline(startup_hook=self.startup_hook) + sys.audit("builtins.input/result", result) + return result + + def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str: + """Read an input on possibly multiple lines, asking for more + lines as long as 'more_lines(unicodetext)' returns an object whose + boolean value is true. + """ + reader = self.get_reader() + saved = reader.more_lines + try: + reader.more_lines = more_lines + reader.ps1 = ps1 + reader.ps2 = ps1 + reader.ps3 = ps2 + reader.ps4 = "" + with warnings.catch_warnings(action="ignore"): + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + def parse_and_bind(self, string: str) -> None: + pass # XXX we don't support parsing GNU-readline-style init files + + def set_completer(self, function: Completer | None = None) -> None: + self.config.readline_completer = function + + def get_completer(self) -> Completer | None: + return self.config.readline_completer + + def set_completer_delims(self, delimiters: Collection[str]) -> None: + self.config.completer_delims = frozenset(delimiters) + + def get_completer_delims(self) -> str: + return "".join(sorted(self.config.completer_delims)) + + def _histline(self, line: str) -> str: + line = line.rstrip("\n") + return line + + def get_history_length(self) -> int: + return self.saved_history_length + + def set_history_length(self, length: int) -> None: + self.saved_history_length = length + + def get_current_history_length(self) -> int: + return len(self.get_reader().history) + + def read_history_file(self, filename: str = gethistoryfile()) -> None: + # multiline extension (really a hack) for the end of lines that + # are actually continuations inside a single multiline_input() + # history item: we use \r\n instead of just \n. If the history + # file is passed to GNU readline, the extra \r are just ignored. + history = self.get_reader().history + + with open(os.path.expanduser(filename), 'rb') as f: + is_editline = f.readline().startswith(b"_HiStOrY_V2_") + if is_editline: + encoding = "unicode-escape" + else: + f.seek(0) + encoding = "utf-8" + + lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')] + buffer = [] + for line in lines: + if line.endswith("\r"): + buffer.append(line+'\n') + else: + line = self._histline(line) + if buffer: + line = self._histline("".join(buffer).replace("\r", "") + line) + del buffer[:] + if line: + history.append(line) + + def write_history_file(self, filename: str = gethistoryfile()) -> None: + maxlength = self.saved_history_length + history = self.get_reader().get_trimmed_history(maxlength) + with open(os.path.expanduser(filename), "w", encoding="utf-8") as f: + for entry in history: + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + + def clear_history(self) -> None: + del self.get_reader().history[:] + + def get_history_item(self, index: int) -> str | None: + history = self.get_reader().history + if 1 <= index <= len(history): + return history[index - 1] + else: + return None # like readline.c + + def remove_history_item(self, index: int) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + del history[index] + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def replace_history_item(self, index: int, line: str) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + history[index] = self._histline(line) + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def add_history(self, line: str) -> None: + self.get_reader().history.append(self._histline(line)) + + def set_startup_hook(self, function: Callback | None = None) -> None: + self.startup_hook = function + + def get_line_buffer(self) -> str: + return self.get_reader().get_unicode() + + def _get_idxs(self) -> tuple[int, int]: + start = cursor = self.get_reader().pos + buf = self.get_line_buffer() + for i in range(cursor - 1, -1, -1): + if buf[i] in self.get_completer_delims(): + break + start = i + return start, cursor + + def get_begidx(self) -> int: + return self._get_idxs()[0] + + def get_endidx(self) -> int: + return self._get_idxs()[1] + + def insert_text(self, text: str) -> None: + self.get_reader().insert(text) + + +_wrapper = _ReadlineWrapper() + +# ____________________________________________________________ +# Public API + +parse_and_bind = _wrapper.parse_and_bind +set_completer = _wrapper.set_completer +get_completer = _wrapper.get_completer +set_completer_delims = _wrapper.set_completer_delims +get_completer_delims = _wrapper.get_completer_delims +get_history_length = _wrapper.get_history_length +set_history_length = _wrapper.set_history_length +get_current_history_length = _wrapper.get_current_history_length +read_history_file = _wrapper.read_history_file +write_history_file = _wrapper.write_history_file +clear_history = _wrapper.clear_history +get_history_item = _wrapper.get_history_item +remove_history_item = _wrapper.remove_history_item +replace_history_item = _wrapper.replace_history_item +add_history = _wrapper.add_history +set_startup_hook = _wrapper.set_startup_hook +get_line_buffer = _wrapper.get_line_buffer +get_begidx = _wrapper.get_begidx +get_endidx = _wrapper.get_endidx +insert_text = _wrapper.insert_text + +# Extension +multiline_input = _wrapper.multiline_input + +# Internal hook +_get_reader = _wrapper.get_reader + +# ____________________________________________________________ +# Stubs + + +def _make_stub(_name: str, _ret: object) -> None: + def stub(*args: object, **kwds: object) -> None: + import warnings + + warnings.warn("readline.%s() not implemented" % _name, stacklevel=2) + + stub.__name__ = _name + globals()[_name] = stub + + +for _name, _ret in [ + ("read_init_file", None), + ("redisplay", None), + ("set_pre_input_hook", None), +]: + assert _name not in globals(), _name + _make_stub(_name, _ret) + +# ____________________________________________________________ + + +def _setup(namespace: Mapping[str, Any]) -> None: + global raw_input + if raw_input is not None: + return # don't run _setup twice + + try: + f_in = sys.stdin.fileno() + f_out = sys.stdout.fileno() + except (AttributeError, ValueError): + return + if not os.isatty(f_in) or not os.isatty(f_out): + return + + _wrapper.f_in = f_in + _wrapper.f_out = f_out + + # set up namespace in rlcompleter, which requires it to be a bona fide dict + if not isinstance(namespace, dict): + namespace = dict(namespace) + _wrapper.config.readline_completer = RLCompleter(namespace).complete + + # this is not really what readline.c does. Better than nothing I guess + import builtins + raw_input = builtins.input + builtins.input = _wrapper.input + + +raw_input: Callable[[object], str] | None = None diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py new file mode 100644 index 00000000000000..342a4b58bfd0f3 --- /dev/null +++ b/Lib/_pyrepl/simple_interact.py @@ -0,0 +1,175 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""This is an alternative to python_reader which tries to emulate +the CPython prompt as closely as possible, with the exception of +allowing multiline input and multiline history entries. +""" + +from __future__ import annotations + +import _sitebuiltins +import linecache +import functools +import os +import sys +import code + +from .readline import _get_reader, multiline_input + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + + +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import _error +except ModuleNotFoundError: + from .windows_console import _error + +def check() -> str: + """Returns the error message if there is a problem initializing the state.""" + try: + _get_reader() + except _error as e: + if term := os.environ.get("TERM", ""): + term = f"; TERM={term}" + return str(str(e) or repr(e) or "unknown error") + term + return "" + + +def _strip_final_indent(text: str) -> str: + # kill spaces and tabs at the end, but only if they follow '\n'. + # meant to remove the auto-indentation only (although it would of + # course also remove explicitly-added indentation). + short = text.rstrip(" \t") + n = len(short) + if n > 0 and text[n - 1] == "\n": + return short + return text + + +def _clear_screen(): + reader = _get_reader() + reader.scheduled_commands.append("clear_screen") + + +REPL_COMMANDS = { + "exit": _sitebuiltins.Quitter('exit', ''), + "quit": _sitebuiltins.Quitter('quit' ,''), + "copyright": _sitebuiltins._Printer('copyright', sys.copyright), + "help": "help", + "clear": _clear_screen, + "\x1a": _sitebuiltins.Quitter('\x1a', ''), +} + + +def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool: + # ooh, look at the hack: + src = _strip_final_indent(unicodetext) + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + lines = src.splitlines(keepends=True) + if len(lines) == 1: + return False + + last_line = lines[-1] + was_indented = last_line.startswith((" ", "\t")) + not_empty = last_line.strip() != "" + incomplete = not last_line.endswith("\n") + return (was_indented or not_empty) and incomplete + else: + return code is None + + +def run_multiline_interactive_console( + console: code.InteractiveConsole, + *, + future_flags: int = 0, +) -> None: + from .readline import _setup + _setup(console.locals) + if future_flags: + console.compile.compiler.flags |= future_flags + + more_lines = functools.partial(_more_lines, console) + input_n = 0 + + def maybe_run_command(statement: str) -> bool: + statement = statement.strip() + if statement in console.locals or statement not in REPL_COMMANDS: + return False + + reader = _get_reader() + reader.history.pop() # skip internal commands in history + command = REPL_COMMANDS[statement] + if callable(command): + command() + return True + + if isinstance(command, str): + # Internal readline commands require a prepared reader like + # inside multiline_input. + reader.prepare() + reader.refresh() + reader.do_cmd((command, [statement])) + reader.restore() + return True + + return False + + while 1: + try: + try: + sys.stdout.flush() + except Exception: + pass + + ps1 = getattr(sys, "ps1", ">>> ") + ps2 = getattr(sys, "ps2", "... ") + try: + statement = multiline_input(more_lines, ps1, ps2) + except EOFError: + break + + if maybe_run_command(statement): + continue + + input_name = f"" + linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] + assert not more + input_n += 1 + except KeyboardInterrupt: + r = _get_reader() + if r.input_trans is r.isearch_trans: + r.do_cmd(("isearch-end", [""])) + r.pos = len(r.get_unicode()) + r.dirty = True + r.refresh() + r.in_bracketed_paste = False + console.write("\nKeyboardInterrupt\n") + console.resetbuffer() + except MemoryError: + console.write("\nMemoryError\n") + console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py new file mode 100644 index 00000000000000..a8eb2433cd3cce --- /dev/null +++ b/Lib/_pyrepl/trace.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import os + +# types +if False: + from typing import IO + + +trace_file: IO[str] | None = None +if trace_filename := os.environ.get("PYREPL_TRACE"): + trace_file = open(trace_filename, "a") + + +def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py new file mode 100644 index 00000000000000..f9d48b828c720b --- /dev/null +++ b/Lib/_pyrepl/types.py @@ -0,0 +1,8 @@ +from collections.abc import Callable, Iterator + +Callback = Callable[[], object] +SimpleContextManager = Iterator[None] +KeySpec = str # like r"\C-c" +CommandName = str # like "interrupt" +EventTuple = tuple[CommandName, str] +Completer = Callable[[str, int], str | None] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py new file mode 100644 index 00000000000000..2576b938a34c64 --- /dev/null +++ b/Lib/_pyrepl/unix_console.py @@ -0,0 +1,804 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import errno +import os +import re +import select +import signal +import struct +import termios +import time +import platform +from fcntl import ioctl + +from . import curses +from .console import Console, Event +from .fancy_termios import tcgetattr, tcsetattr +from .trace import trace +from .unix_eventqueue import EventQueue +from .utils import wlen + + +TYPE_CHECKING = False + +# types +if TYPE_CHECKING: + from typing import IO, Literal, overload +else: + overload = lambda func: None + + +class InvalidTerminal(RuntimeError): + pass + + +_error = (termios.error, curses.error, InvalidTerminal) + +SIGWINCH_EVENT = "repaint" + +FIONREAD = getattr(termios, "FIONREAD", None) +TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None) + +# ------------ start of baudrate definitions ------------ + +# Add (possibly) missing baudrates (check termios man page) to termios + + +def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: + baudrate_name = "B%d" % rate + if hasattr(termios, baudrate_name): + dictionary[getattr(termios, baudrate_name)] = rate + + +# Check the termios man page (Line speed) to know where these +# values come from. +potential_baudrates = [ + 0, + 110, + 115200, + 1200, + 134, + 150, + 1800, + 19200, + 200, + 230400, + 2400, + 300, + 38400, + 460800, + 4800, + 50, + 57600, + 600, + 75, + 9600, +] + +ratedict: dict[int, int] = {} +for rate in potential_baudrates: + add_baudrate_if_supported(ratedict, rate) + +# Clean up variables to avoid unintended usage +del rate, add_baudrate_if_supported + +# ------------ end of baudrate definitions ------------ + +delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") + +try: + poll: type[select.poll] = select.poll +except AttributeError: + # this is exactly the minimum necessary to support what we + # do with poll objects + class MinimalPoll: + def __init__(self): + pass + + def register(self, fd, flag): + self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* + def poll(self, timeout: float | None = None) -> list[int]: + if timeout is None: + r, w, e = select.select([self.fd], [], []) + else: + r, w, e = select.select([self.fd], [], [], timeout/1000) + return r + + poll = MinimalPoll # type: ignore[assignment] + + +class UnixConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + """ + Initialize the UnixConsole. + + Parameters: + - f_in (int or file-like object): Input file descriptor or object. + - f_out (int or file-like object): Output file descriptor or object. + - term (str): Terminal name. + - encoding (str): Encoding to use for I/O operations. + """ + super().__init__(f_in, f_out, term, encoding) + + self.pollob = poll() + self.pollob.register(self.input_fd, select.POLLIN) + self.input_buffer = b"" + self.input_buffer_pos = 0 + curses.setupterm(term or None, self.output_fd) + self.term = term + + @overload + def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + + @overload + def _my_getstr(cap: str, optional: bool) -> bytes | None: ... + + def _my_getstr(cap: str, optional: bool = False) -> bytes | None: + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + f"terminal doesn't have the required {cap} capability" + ) + return r + + self._bel = _my_getstr("bel") + self._civis = _my_getstr("civis", optional=True) + self._clear = _my_getstr("clear") + self._cnorm = _my_getstr("cnorm", optional=True) + self._cub = _my_getstr("cub", optional=True) + self._cub1 = _my_getstr("cub1", optional=True) + self._cud = _my_getstr("cud", optional=True) + self._cud1 = _my_getstr("cud1", optional=True) + self._cuf = _my_getstr("cuf", optional=True) + self._cuf1 = _my_getstr("cuf1", optional=True) + self._cup = _my_getstr("cup") + self._cuu = _my_getstr("cuu", optional=True) + self._cuu1 = _my_getstr("cuu1", optional=True) + self._dch1 = _my_getstr("dch1", optional=True) + self._dch = _my_getstr("dch", optional=True) + self._el = _my_getstr("el") + self._hpa = _my_getstr("hpa", optional=True) + self._ich = _my_getstr("ich", optional=True) + self._ich1 = _my_getstr("ich1", optional=True) + self._ind = _my_getstr("ind", optional=True) + self._pad = _my_getstr("pad", optional=True) + self._ri = _my_getstr("ri", optional=True) + self._rmkx = _my_getstr("rmkx", optional=True) + self._smkx = _my_getstr("smkx", optional=True) + + self.__setup_movement() + + self.event_queue = EventQueue(self.input_fd, self.encoding) + self.cursor_visible = 1 + + def more_in_buffer(self) -> bool: + return bool( + self.input_buffer + and self.input_buffer_pos < len(self.input_buffer) + ) + + def __read(self, n: int) -> bytes: + if not self.more_in_buffer(): + self.input_buffer = os.read(self.input_fd, 10000) + + ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n] + self.input_buffer_pos += len(ret) + if self.input_buffer_pos >= len(self.input_buffer): + self.input_buffer = b"" + self.input_buffer_pos = 0 + return ret + + + def change_encoding(self, encoding: str) -> None: + """ + Change the encoding used for I/O operations. + + Parameters: + - encoding (str): New encoding to use. + """ + self.encoding = encoding + + def refresh(self, screen, c_xy): + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + if not self.__gone_tall: + while len(self.screen) < min(len(screen), self.height): + self.__hide_cursor() + self.__move(0, len(self.screen) - 1) + self.__write("\n") + self.__posxy = 0, len(self.screen) + self.screen.append("") + else: + while len(self.screen) < len(screen): + self.screen.append("") + + if len(screen) > self.height: + self.__gone_tall = 1 + self.__move = self.__move_tall + + px, py = self.__posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + # use hardware scrolling if we have it. + if old_offset > offset and self._ri: + self.__hide_cursor() + self.__write_code(self._cup, 0, 0) + self.__posxy = 0, old_offset + for i in range(old_offset - offset): + self.__write_code(self._ri) + oldscr.pop(-1) + oldscr.insert(0, "") + elif old_offset < offset and self._ind: + self.__hide_cursor() + self.__write_code(self._cup, self.height - 1, 0) + self.__posxy = 0, old_offset + self.height - 1 + for i in range(offset - old_offset): + self.__write_code(self._ind) + oldscr.pop(0) + oldscr.append("") + + self.__offset = offset + + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self.__hide_cursor() + self.__move(0, y) + self.__posxy = 0, y + self.__write_code(self._el) + y += 1 + + self.__show_cursor() + + self.screen = screen.copy() + self.move_cursor(cx, cy) + self.flushoutput() + + def move_cursor(self, x, y): + """ + Move the cursor to the specified position on the screen. + + Parameters: + - x (int): X coordinate. + - y (int): Y coordinate. + """ + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(Event("scroll", None)) + else: + self.__move(x, y) + self.__posxy = x, y + self.flushoutput() + + def prepare(self): + """ + Prepare the console for input/output operations. + """ + self.__svtermstate = tcgetattr(self.input_fd) + raw = self.__svtermstate.copy() + raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) + raw.oflag &= ~(termios.OPOST) + raw.cflag &= ~(termios.CSIZE | termios.PARENB) + raw.cflag |= termios.CS8 + raw.iflag |= termios.BRKINT + raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN) + raw.lflag |= termios.ISIG + raw.cc[termios.VMIN] = 1 + raw.cc[termios.VTIME] = 0 + tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + + # In macOS terminal we need to deactivate line wrap via ANSI escape code + if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + os.write(self.output_fd, b"\033[?7l") + + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__buffer = [] + + self.__posxy = 0, 0 + self.__gone_tall = 0 + self.__move = self.__move_short + self.__offset = 0 + + self.__maybe_write_code(self._smkx) + + try: + self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) + except ValueError: + pass + + self.__enable_bracketed_paste() + + def restore(self): + """ + Restore the console to the default state + """ + self.__disable_bracketed_paste() + self.__maybe_write_code(self._rmkx) + self.flushoutput() + tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + + if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + os.write(self.output_fd, b"\033[?7h") + + if hasattr(self, "old_sigwinch"): + signal.signal(signal.SIGWINCH, self.old_sigwinch) + del self.old_sigwinch + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + trace("push char {char!r}", char=char) + self.event_queue.push(char) + + def get_event(self, block: bool = True) -> Event | None: + """ + Get an event from the console event queue. + + Parameters: + - block (bool): Whether to block until an event is available. + + Returns: + - Event: Event object from the event queue. + """ + if not block and not self.wait(timeout=0): + return None + + while self.event_queue.empty(): + while True: + try: + self.push_char(self.__read(1)) + except OSError as err: + if err.errno == errno.EINTR: + if not self.event_queue.empty(): + return self.event_queue.get() + else: + continue + else: + raise + else: + break + return self.event_queue.get() + + def wait(self, timeout: float | None = None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.more_in_buffer() + or bool(self.pollob.poll(timeout)) + ) + + def set_cursor_vis(self, visible): + """ + Set the visibility of the cursor. + + Parameters: + - visible (bool): Visibility flag. + """ + if visible: + self.__show_cursor() + else: + self.__hide_cursor() + + if TIOCGWINSZ: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + height, width = struct.unpack( + "hhhh", ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) + )[0:2] + if not height: + return 25, 80 + return height, width + + else: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + return 25, 80 + + def forgetinput(self): + """ + Discard any pending input on the console. + """ + termios.tcflush(self.input_fd, termios.TCIFLUSH) + + def flushoutput(self): + """ + Flush the output buffer. + """ + for text, iscode in self.__buffer: + if iscode: + self.__tputs(text) + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + del self.__buffer[:] + + def finish(self): + """ + Finish console operations and flush the output buffer. + """ + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write("\n\r") + self.flushoutput() + + def beep(self): + """ + Emit a beep sound. + """ + self.__maybe_write_code(self._bel) + self.flushoutput() + + if FIONREAD: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + else: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = 10000 + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + def clear(self): + """ + Clear the console screen. + """ + self.__write_code(self._clear) + self.__gone_tall = 1 + self.__move = self.__move_tall + self.__posxy = 0, 0 + self.screen = [] + + @property + def input_hook(self): + try: + import posix + except ImportError: + return None + if posix._is_inputhook_installed(): + return posix._inputhook + + def __enable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004h") + + def __disable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004l") + + def __setup_movement(self): + """ + Set up the movement functions based on the terminal capabilities. + """ + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 + else: + raise RuntimeError("insufficient terminal (horizontal)") + + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 + else: + raise RuntimeError("insufficient terminal (vertical)") + + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) + else: + self.dch1 = None + + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) + else: + self.ich1 = None + + self.__move = self.__move_short + + def __write_changed_line(self, y, oldline, newline, px_coord): + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequence + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + # if we need to insert a single character right after the first detected change + if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + if ( + y == self.__posxy[1] + and x_coord > self.__posxy[0] + and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + ): + x_pos = px_pos + x_coord = px_coord + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if it's a single character change in the middle of the line + elif ( + x_coord < minlen + and oldline[x_pos + 1 :] == newline[x_pos + 1 :] + and wlen(oldline[x_pos]) == wlen(newline[x_pos]) + ): + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if this is the last character to fit in the line and we edit in the middle of the line + elif ( + self.dch1 + and self.ich1 + and wlen(newline) == self.width + and x_coord < wlen(newline) - 2 + and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + ): + self.__hide_cursor() + self.__move(self.width - 2, y) + self.__posxy = self.width - 2, y + self.__write_code(self.dch1) + + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.__posxy = character_width + 1, y + + else: + self.__hide_cursor() + self.__move(x_coord, y) + if wlen(oldline) > wlen(newline): + self.__write_code(self._el) + self.__write(newline[x_pos:]) + self.__posxy = wlen(newline), y + + if "\x1b" in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def __write(self, text): + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) + + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(dy * self._cud1) + elif dy < 0: + self.__write_code((-dy) * self._cuu1) + + def __move_y_cuu_cud(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(self._cud, dy) + elif dy < 0: + self.__write_code(self._cuu, -dy) + + def __move_x_hpa(self, x: int) -> None: + if x != self.__posxy[0]: + self.__write_code(self._hpa, x) + + def __move_x_cub1_cuf1(self, x: int) -> None: + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf1 * dx) + elif dx < 0: + self.__write_code(self._cub1 * (-dx)) + + def __move_x_cub_cuf(self, x: int) -> None: + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf, dx) + elif dx < 0: + self.__write_code(self._cub, -dx) + + def __move_short(self, x, y): + self.__move_x(x) + self.__move_y(y) + + def __move_tall(self, x, y): + assert 0 <= y - self.__offset < self.height, y - self.__offset + self.__write_code(self._cup, y - self.__offset, x) + + def __sigwinch(self, signum, frame): + self.height, self.width = self.getheightwidth() + self.event_queue.insert(Event("resize", None)) + + def __hide_cursor(self): + if self.cursor_visible: + self.__maybe_write_code(self._civis) + self.cursor_visible = 0 + + def __show_cursor(self): + if not self.cursor_visible: + self.__maybe_write_code(self._cnorm) + self.cursor_visible = 1 + + def repaint(self): + if not self.__gone_tall: + self.__posxy = 0, self.__posxy[1] + self.__write("\r") + ns = len(self.screen) * ["\000" * self.width] + self.screen = ns + else: + self.__posxy = 0, self.__offset + self.__move(0, self.__offset) + ns = self.height * ["\000" * self.width] + self.screen = ns + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py new file mode 100644 index 00000000000000..70cfade26e23b1 --- /dev/null +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -0,0 +1,152 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from collections import deque + +from . import keymap +from .console import Event +from . import curses +from .trace import trace +from termios import tcgetattr, VERASE +import os + + +# Mapping of human-readable key names to their terminal-specific codes +TERMINAL_KEYNAMES = { + "delete": "kdch1", + "down": "kcud1", + "end": "kend", + "enter": "kent", + "home": "khome", + "insert": "kich1", + "left": "kcub1", + "page down": "knp", + "page up": "kpp", + "right": "kcuf1", + "up": "kcuu1", +} + + +# Function keys F1-F20 mapping +TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21)) + +# Known CTRL-arrow keycodes +CTRL_ARROW_KEYCODES= { + # for xterm, gnome-terminal, xfce terminal, etc. + b'\033[1;5D': 'ctrl left', + b'\033[1;5C': 'ctrl right', + # for rxvt + b'\033Od': 'ctrl left', + b'\033Oc': 'ctrl right', +} + +def get_terminal_keycodes() -> dict[bytes, str]: + """ + Generates a dictionary mapping terminal keycodes to human-readable names. + """ + keycodes = {} + for key, terminal_code in TERMINAL_KEYNAMES.items(): + keycode = curses.tigetstr(terminal_code) + trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) + if keycode: + keycodes[keycode] = key + keycodes.update(CTRL_ARROW_KEYCODES) + return keycodes + +class EventQueue: + def __init__(self, fd: int, encoding: str) -> None: + self.keycodes = get_terminal_keycodes() + if os.isatty(fd): + backspace = tcgetattr(fd)[6][VERASE] + self.keycodes[backspace] = "backspace" + self.compiled_keymap = keymap.compile_keymap(self.keycodes) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding + self.events: deque[Event] = deque() + self.buf = bytearray() + + def get(self) -> Event | None: + """ + Retrieves the next event from the queue. + """ + if self.events: + return self.events.popleft() + else: + return None + + def empty(self) -> bool: + """ + Checks if the queue is empty. + """ + return not self.events + + def flush_buf(self) -> bytearray: + """ + Flushes the buffer and returns its contents. + """ + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event: Event) -> None: + """ + Inserts an event into the queue. + """ + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char: int | bytes) -> None: + """ + Processes a character by updating the buffer and handling special key mappings. + """ + ord_char = char if isinstance(char, int) else ord(char) + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) + if char in self.keymap: + if self.keymap is self.compiled_keymap: + #sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.keymap[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.keymap = k + else: + self.insert(Event('key', k, self.flush_buf())) + self.keymap = self.compiled_keymap + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py + trace('unrecognized escape sequence, propagating...') + self.keymap = self.compiled_keymap + self.insert(Event('key', '\033', bytearray(b'\033'))) + for _c in self.flush_buf()[1:]: + self.push(_c) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, self.flush_buf())) + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py new file mode 100644 index 00000000000000..0f36083b6ffa92 --- /dev/null +++ b/Lib/_pyrepl/utils.py @@ -0,0 +1,25 @@ +import re +import unicodedata +import functools + +ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") + + +@functools.cache +def str_width(c: str) -> int: + if ord(c) < 128: + return 1 + w = unicodedata.east_asian_width(c) + if w in ('N', 'Na', 'H', 'A'): + return 1 + return 2 + + +def wlen(s: str) -> int: + if len(s) == 1: + return str_width(s) + length = sum(str_width(i) for i in s) + # remove lengths of any escape sequences + sequence = ANSI_ESCAPE_SEQUENCE.findall(s) + ctrl_z_cnt = s.count('\x1a') + return length - sum(len(i) for i in sequence) + ctrl_z_cnt diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py new file mode 100644 index 00000000000000..d457d2b5a338eb --- /dev/null +++ b/Lib/_pyrepl/windows_console.py @@ -0,0 +1,618 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import io +import os +import sys +import time +import msvcrt + +from collections import deque +import ctypes +from ctypes.wintypes import ( + _COORD, + WORD, + SMALL_RECT, + BOOL, + HANDLE, + CHAR, + DWORD, + WCHAR, + SHORT, +) +from ctypes import Structure, POINTER, Union +from .console import Event, Console +from .trace import trace +from .utils import wlen + +try: + from ctypes import GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] +except: + # Keep MyPy happy off Windows + from ctypes import CDLL as WinDLL, cdll as windll + + def GetLastError() -> int: + return 42 + + class WinError(OSError): # type: ignore[no-redef] + def __init__(self, err: int | None, descr: str | None = None) -> None: + self.err = err + self.descr = descr + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + +# Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes +VK_MAP: dict[int, str] = { + 0x23: "end", # VK_END + 0x24: "home", # VK_HOME + 0x25: "left", # VK_LEFT + 0x26: "up", # VK_UP + 0x27: "right", # VK_RIGHT + 0x28: "down", # VK_DOWN + 0x2E: "delete", # VK_DELETE + 0x70: "f1", # VK_F1 + 0x71: "f2", # VK_F2 + 0x72: "f3", # VK_F3 + 0x73: "f4", # VK_F4 + 0x74: "f5", # VK_F5 + 0x75: "f6", # VK_F6 + 0x76: "f7", # VK_F7 + 0x77: "f8", # VK_F8 + 0x78: "f9", # VK_F9 + 0x79: "f10", # VK_F10 + 0x7A: "f11", # VK_F11 + 0x7B: "f12", # VK_F12 + 0x7C: "f13", # VK_F13 + 0x7D: "f14", # VK_F14 + 0x7E: "f15", # VK_F15 + 0x7F: "f16", # VK_F16 + 0x80: "f17", # VK_F17 + 0x81: "f18", # VK_F18 + 0x82: "f19", # VK_F19 + 0x83: "f20", # VK_F20 +} + +# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +ERASE_IN_LINE = "\x1b[K" +MOVE_LEFT = "\x1b[{}D" +MOVE_RIGHT = "\x1b[{}C" +MOVE_UP = "\x1b[{}A" +MOVE_DOWN = "\x1b[{}B" +CLEAR = "\x1b[H\x1b[J" + + +class _error(Exception): + pass + + +class WindowsConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + super().__init__(f_in, f_out, term, encoding) + + SetConsoleMode( + OutHandle, + ENABLE_WRAP_AT_EOL_OUTPUT + | ENABLE_PROCESSED_OUTPUT + | ENABLE_VIRTUAL_TERMINAL_PROCESSING, + ) + self.screen: list[str] = [] + self.width = 80 + self.height = 25 + self.__offset = 0 + self.event_queue: deque[Event] = deque() + try: + self.out = io._WindowsConsoleIO(self.output_fd, "w") # type: ignore[attr-defined] + except ValueError: + # Console I/O is redirected, fallback... + self.out = None + + def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + + while len(self.screen) < min(len(screen), self.height): + self._hide_cursor() + self._move_relative(0, len(self.screen) - 1) + self.__write("\n") + self.__posxy = 0, len(self.screen) + self.screen.append("") + + px, py = self.__posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + scroll_lines = offset - old_offset + + # Scrolling the buffer as the current input is greater than the visible + # portion of the window. We need to scroll the visible portion and the + # entire history + self._scroll(scroll_lines, self._getscrollbacksize()) + self.__posxy = self.__posxy[0], self.__posxy[1] + scroll_lines + self.__offset += scroll_lines + + for i in range(scroll_lines): + self.screen.append("") + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + self.__offset = offset + + self._hide_cursor() + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self._move_relative(0, y) + self.__posxy = 0, y + self._erase_to_end() + y += 1 + + self._show_cursor() + + self.screen = screen + self.move_cursor(cx, cy) + + @property + def input_hook(self): + try: + import nt + except ImportError: + return None + if nt._is_inputhook_installed(): + return nt._inputhook + + def __write_changed_line( + self, y: int, oldline: str, newline: str, px_coord: int + ) -> None: + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequence + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + self._hide_cursor() + self._move_relative(x_coord, y) + if wlen(oldline) > wlen(newline): + self._erase_to_end() + + self.__write(newline[x_pos:]) + if wlen(newline) == self.width: + # If we wrapped we want to start at the next line + self._move_relative(0, y + 1) + self.__posxy = 0, y + 1 + else: + self.__posxy = wlen(newline), y + + if "\x1b" in newline or y != self.__posxy[1] or '\x1a' in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def _scroll( + self, top: int, bottom: int, left: int | None = None, right: int | None = None + ) -> None: + scroll_rect = SMALL_RECT() + scroll_rect.Top = SHORT(top) + scroll_rect.Bottom = SHORT(bottom) + scroll_rect.Left = SHORT(0 if left is None else left) + scroll_rect.Right = SHORT( + self.getheightwidth()[1] - 1 if right is None else right + ) + destination_origin = _COORD() + fill_info = CHAR_INFO() + fill_info.UnicodeChar = " " + + if not ScrollConsoleScreenBuffer( + OutHandle, scroll_rect, None, destination_origin, fill_info + ): + raise WinError(GetLastError()) + + def _hide_cursor(self): + self.__write("\x1b[?25l") + + def _show_cursor(self): + self.__write("\x1b[?25h") + + def _enable_blinking(self): + self.__write("\x1b[?12h") + + def _disable_blinking(self): + self.__write("\x1b[?12l") + + def __write(self, text: str) -> None: + if "\x1a" in text: + text = ''.join(["^Z" if x == '\x1a' else x for x in text]) + + if self.out is not None: + self.out.write(text.encode(self.encoding, "replace")) + self.out.flush() + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + + @property + def screen_xy(self) -> tuple[int, int]: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return info.dwCursorPosition.X, info.dwCursorPosition.Y + + def _erase_to_end(self) -> None: + self.__write(ERASE_IN_LINE) + + def prepare(self) -> None: + trace("prepare") + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__posxy = 0, 0 + self.__gone_tall = 0 + self.__offset = 0 + + def restore(self) -> None: + pass + + def _move_relative(self, x: int, y: int) -> None: + """Moves relative to the current __posxy""" + dx = x - self.__posxy[0] + dy = y - self.__posxy[1] + if dx < 0: + self.__write(MOVE_LEFT.format(-dx)) + elif dx > 0: + self.__write(MOVE_RIGHT.format(dx)) + + if dy < 0: + self.__write(MOVE_UP.format(-dy)) + elif dy > 0: + self.__write(MOVE_DOWN.format(dy)) + + def move_cursor(self, x: int, y: int) -> None: + if x < 0 or y < 0: + raise ValueError(f"Bad cursor position {x}, {y}") + + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(0, Event("scroll", "")) + else: + self._move_relative(x, y) + self.__posxy = x, y + + def set_cursor_vis(self, visible: bool) -> None: + if visible: + self._show_cursor() + else: + self._hide_cursor() + + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return ( + info.srWindow.Bottom - info.srWindow.Top + 1, + info.srWindow.Right - info.srWindow.Left + 1, + ) + + def _getscrollbacksize(self) -> int: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + + return info.srWindow.Bottom # type: ignore[no-any-return] + + def _read_input(self, block: bool = True) -> INPUT_RECORD | None: + if not block: + events = DWORD() + if not GetNumberOfConsoleInputEvents(InHandle, events): + raise WinError(GetLastError()) + if not events.value: + return None + + rec = INPUT_RECORD() + read = DWORD() + if not ReadConsoleInput(InHandle, rec, 1, read): + raise WinError(GetLastError()) + + return rec + + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + if self.event_queue: + return self.event_queue.pop() + + while True: + rec = self._read_input(block) + if rec is None: + return None + + if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: + return Event("resize", "") + + if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown: + # Only process keys and keydown events + if block: + continue + return None + + key = rec.Event.KeyEvent.uChar.UnicodeChar + + if rec.Event.KeyEvent.uChar.UnicodeChar == "\r": + # Make enter make unix-like + return Event(evt="key", data="\n", raw=b"\n") + elif rec.Event.KeyEvent.wVirtualKeyCode == 8: + # Turn backspace directly into the command + return Event( + evt="key", + data="backspace", + raw=rec.Event.KeyEvent.uChar.UnicodeChar, + ) + elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00": + # Handle special keys like arrow keys and translate them into the appropriate command + code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode) + if code: + return Event( + evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar + ) + if block: + continue + + return None + + return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar) + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + raise NotImplementedError("push_char not supported on Windows") + + def beep(self) -> None: + self.__write("\x07") + + def clear(self) -> None: + """Wipe the screen""" + self.__write(CLEAR) + self.__posxy = 0, 0 + self.screen = [""] + + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self._move_relative(0, min(y, self.height + self.__offset - 1)) + self.__write("\r\n") + + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere). + + All output on Windows is unbuffered so this is a nop""" + pass + + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + if not FlushConsoleInputBuffer(InHandle): + raise WinError(GetLastError()) + + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + return Event("key", "", b"") + + def wait(self, timeout: float | None) -> bool: + """Wait for an event.""" + # Poor man's Windows select loop + start_time = time.time() + while True: + if msvcrt.kbhit(): # type: ignore[attr-defined] + return True + if timeout and time.time() - start_time > timeout / 1000: + return False + time.sleep(0.01) + + def repaint(self) -> None: + raise NotImplementedError("No repaint support") + + +# Windows interop +class CONSOLE_SCREEN_BUFFER_INFO(Structure): + _fields_ = [ + ("dwSize", _COORD), + ("dwCursorPosition", _COORD), + ("wAttributes", WORD), + ("srWindow", SMALL_RECT), + ("dwMaximumWindowSize", _COORD), + ] + + +class CONSOLE_CURSOR_INFO(Structure): + _fields_ = [ + ("dwSize", DWORD), + ("bVisible", BOOL), + ] + + +class CHAR_INFO(Structure): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Attributes", WORD), + ] + + +class Char(Union): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Char", CHAR), + ] + + +class KeyEvent(ctypes.Structure): + _fields_ = [ + ("bKeyDown", BOOL), + ("wRepeatCount", WORD), + ("wVirtualKeyCode", WORD), + ("wVirtualScanCode", WORD), + ("uChar", Char), + ("dwControlKeyState", DWORD), + ] + + +class WindowsBufferSizeEvent(ctypes.Structure): + _fields_ = [("dwSize", _COORD)] + + +class ConsoleEvent(ctypes.Union): + _fields_ = [ + ("KeyEvent", KeyEvent), + ("WindowsBufferSizeEvent", WindowsBufferSizeEvent), + ] + + +class INPUT_RECORD(Structure): + _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)] + + +KEY_EVENT = 0x01 +FOCUS_EVENT = 0x10 +MENU_EVENT = 0x08 +MOUSE_EVENT = 0x02 +WINDOW_BUFFER_SIZE_EVENT = 0x04 + +ENABLE_PROCESSED_OUTPUT = 0x01 +ENABLE_WRAP_AT_EOL_OUTPUT = 0x02 +ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 + +STD_INPUT_HANDLE = -10 +STD_OUTPUT_HANDLE = -11 + +if sys.platform == "win32": + _KERNEL32 = WinDLL("kernel32", use_last_error=True) + + GetStdHandle = windll.kernel32.GetStdHandle + GetStdHandle.argtypes = [DWORD] + GetStdHandle.restype = HANDLE + + GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo + GetConsoleScreenBufferInfo.argtypes = [ + HANDLE, + ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + GetConsoleScreenBufferInfo.restype = BOOL + + ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW + ScrollConsoleScreenBuffer.argtypes = [ + HANDLE, + POINTER(SMALL_RECT), + POINTER(SMALL_RECT), + _COORD, + POINTER(CHAR_INFO), + ] + ScrollConsoleScreenBuffer.restype = BOOL + + SetConsoleMode = _KERNEL32.SetConsoleMode + SetConsoleMode.argtypes = [HANDLE, DWORD] + SetConsoleMode.restype = BOOL + + ReadConsoleInput = _KERNEL32.ReadConsoleInputW + ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] + ReadConsoleInput.restype = BOOL + + GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents + GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] + GetNumberOfConsoleInputEvents.restype = BOOL + + FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer + FlushConsoleInputBuffer.argtypes = [HANDLE] + FlushConsoleInputBuffer.restype = BOOL + + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) + InHandle = GetStdHandle(STD_INPUT_HANDLE) +else: + + def _win_only(*args, **kwargs): + raise NotImplementedError("Windows only") + + GetStdHandle = _win_only + GetConsoleScreenBufferInfo = _win_only + ScrollConsoleScreenBuffer = _win_only + SetConsoleMode = _win_only + ReadConsoleInput = _win_only + GetNumberOfConsoleInputEvents = _win_only + FlushConsoleInputBuffer = _win_only + OutHandle = 0 + InHandle = 0 diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 798cf9f9d3fffe..5f4d2475c0169b 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -10,10 +10,12 @@ strptime -- Calculates the time struct represented by the passed-in string """ +import os import time import locale import calendar from re import compile as re_compile +from re import sub as re_sub from re import IGNORECASE from re import escape as re_escape from datetime import (date as datetime_date, @@ -27,6 +29,18 @@ def _getlang(): # Figure out what the current language is set to. return locale.getlocale(locale.LC_TIME) +def _findall(haystack, needle): + # Find all positions of needle in haystack. + if not needle: + return + i = 0 + while True: + i = haystack.find(needle, i) + if i < 0: + break + yield i + i += len(needle) + class LocaleTime(object): """Stores and handles locale-specific information related to time. @@ -101,7 +115,8 @@ def __calc_am_pm(self): am_pm = [] for hour in (1, 22): time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0)) - am_pm.append(time.strftime("%p", time_tuple).lower()) + # br_FR has AM/PM info (' ',' '). + am_pm.append(time.strftime("%p", time_tuple).lower().strip()) self.am_pm = am_pm def __calc_date_time(self): @@ -113,42 +128,130 @@ def __calc_date_time(self): # values within the format string is very important; it eliminates # possible ambiguity for what something represents. time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) - date_time = [None, None, None] - date_time[0] = time.strftime("%c", time_tuple).lower() - date_time[1] = time.strftime("%x", time_tuple).lower() - date_time[2] = time.strftime("%X", time_tuple).lower() - replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'), - (self.f_month[3], '%B'), (self.a_weekday[2], '%a'), - (self.a_month[3], '%b'), (self.am_pm[1], '%p'), - ('1999', '%Y'), ('99', '%y'), ('22', '%H'), - ('44', '%M'), ('55', '%S'), ('76', '%j'), - ('17', '%d'), ('03', '%m'), ('3', '%m'), - # '3' needed for when no leading zero. - ('2', '%w'), ('10', '%I')] - replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone - for tz in tz_values]) - for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')): - current_format = date_time[offset] - for old, new in replacement_pairs: + time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0)) + replacement_pairs = [ + ('1999', '%Y'), ('99', '%y'), ('22', '%H'), + ('44', '%M'), ('55', '%S'), ('76', '%j'), + ('17', '%d'), ('03', '%m'), ('3', '%m'), + # '3' needed for when no leading zero. + ('2', '%w'), ('10', '%I'), + # Non-ASCII digits + ('\u0661\u0669\u0669\u0669', '%Y'), + ('\u0669\u0669', '%Oy'), + ('\u0662\u0662', '%OH'), + ('\u0664\u0664', '%OM'), + ('\u0665\u0665', '%OS'), + ('\u0661\u0667', '%Od'), + ('\u0660\u0663', '%Om'), + ('\u0663', '%Om'), + ('\u0662', '%Ow'), + ('\u0661\u0660', '%OI'), + ] + date_time = [] + for directive in ('%c', '%x', '%X'): + current_format = time.strftime(directive, time_tuple).lower() + current_format = current_format.replace('%', '%%') + # The month and the day of the week formats are treated specially + # because of a possible ambiguity in some locales where the full + # and abbreviated names are equal or names of different types + # are equal. See doc of __find_month_format for more details. + lst, fmt = self.__find_weekday_format(directive) + if lst: + current_format = current_format.replace(lst[2], fmt, 1) + lst, fmt = self.__find_month_format(directive) + if lst: + current_format = current_format.replace(lst[3], fmt, 1) + if self.am_pm[1]: # Must deal with possible lack of locale info # manifesting itself as the empty string (e.g., Swedish's # lack of AM/PM info) or a platform returning a tuple of empty # strings (e.g., MacOS 9 having timezone as ('','')). - if old: - current_format = current_format.replace(old, new) + current_format = current_format.replace(self.am_pm[1], '%p') + for tz_values in self.timezone: + for tz in tz_values: + if tz: + current_format = current_format.replace(tz, "%Z") + # Transform all non-ASCII digits to digits in range U+0660 to U+0669. + current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", 'f': r"(?P[0-9]{1,6})", 'H': r"(?P2[0-3]|[0-1]\d|\d)", - 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", @@ -210,11 +313,15 @@ def __init__(self, locale_time=None): 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), - '%': '%'}) - base.__setitem__('W', base.__getitem__('U').replace('U', 'W')) - base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) - base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + '%': '%'} + for d in 'dmyHIMS': + mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d + mapping['Ow'] = r'(?P\d)' + mapping['W'] = mapping['U'].replace('U', 'W') + base.__init__(mapping) base.__setitem__('X', self.pattern(self.locale_time.LC_time)) + base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) def __seqToRE(self, to_convert, directive): """Convert a list to a regex string for matching a directive. @@ -242,21 +349,36 @@ def pattern(self, format): regex syntax are escaped. """ - processed_format = '' # The sub() call escapes all characters that might be misconstrued # as regex syntax. Cannot use re.escape since we have to deal with # format directives (%m, etc.). - regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])") - format = regex_chars.sub(r"\\\1", format) - whitespace_replacement = re_compile(r'\s+') - format = whitespace_replacement.sub(r'\\s+', format) - while '%' in format: - directive_index = format.index('%')+1 - processed_format = "%s%s%s" % (processed_format, - format[:directive_index-1], - self[format[directive_index]]) - format = format[directive_index+1:] - return "%s%s" % (processed_format, format) + format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format) + format = re_sub(r'\s+', r'\\s+', format) + format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR + year_in_format = False + day_of_month_in_format = False + def repl(m): + format_char = m[1] + match format_char: + case 'Y' | 'y' | 'G': + nonlocal year_in_format + year_in_format = True + case 'd': + nonlocal day_of_month_in_format + day_of_month_in_format = True + return self[format_char] + format = re_sub(r'%(O?.)', repl, format) + if day_of_month_in_format and not year_in_format: + import warnings + warnings.warn("""\ +Parsing dates involving a day of month without a year specified is ambiguous +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647.""", + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) + return format def compile(self, format): """Return a compiled re object for the format string.""" @@ -548,18 +670,40 @@ def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): tt = _strptime(data_string, format)[0] return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) -def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): - """Return a class cls instance based on the input string and the +def _strptime_datetime_date(cls, data_string, format="%a %b %d %Y"): + """Return a date instance based on the input string and the + format string.""" + tt, _, _ = _strptime(data_string, format) + args = tt[:3] + return cls(*args) + +def _parse_tz(tzname, gmtoff, gmtoff_fraction): + tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) + if tzname: + return datetime_timezone(tzdelta, tzname) + else: + return datetime_timezone(tzdelta) + +def _strptime_datetime_time(cls, data_string, format="%H:%M:%S"): + """Return a time instance based on the input string and the format string.""" tt, fraction, gmtoff_fraction = _strptime(data_string, format) tzname, gmtoff = tt[-2:] - args = tt[:6] + (fraction,) - if gmtoff is not None: - tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) - if tzname: - tz = datetime_timezone(tzdelta, tzname) - else: - tz = datetime_timezone(tzdelta) - args += (tz,) + args = tt[3:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) - return cls(*args) +def _strptime_datetime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a datetime instance based on the input string and the + format string.""" + tt, fraction, gmtoff_fraction = _strptime(data_string, format) + tzname, gmtoff = tt[-2:] + args = tt[:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 489eec714e0d48..d1c7fcaeec9821 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -8,69 +8,29 @@ __all__ = ['WeakSet'] -class _IterationGuard: - # This context manager registers itself in the current iterators of the - # weak container, such as to delay all removals until the context manager - # exits. - # This technique should be relatively thread-safe (since sets are). - - def __init__(self, weakcontainer): - # Don't create cycles - self.weakcontainer = ref(weakcontainer) - - def __enter__(self): - w = self.weakcontainer() - if w is not None: - w._iterating.add(self) - return self - - def __exit__(self, e, t, b): - w = self.weakcontainer() - if w is not None: - s = w._iterating - s.remove(self) - if not s: - w._commit_removals() - - class WeakSet: def __init__(self, data=None): self.data = set() + def _remove(item, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(item) - else: - self.data.discard(item) + self.data.discard(item) + self._remove = _remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() if data is not None: self.update(data) - def _commit_removals(self): - pop = self._pending_removals.pop - discard = self.data.discard - while True: - try: - item = pop() - except IndexError: - return - discard(item) - def __iter__(self): - with _IterationGuard(self): - for itemref in self.data: - item = itemref() - if item is not None: - # Caveat: the iterator will keep a strong reference to - # `item` until it is resumed or closed. - yield item + for itemref in self.data.copy(): + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item def __len__(self): - return len(self.data) - len(self._pending_removals) + return len(self.data) def __contains__(self, item): try: @@ -83,21 +43,15 @@ def __reduce__(self): return self.__class__, (list(self),), self.__getstate__() def add(self, item): - if self._pending_removals: - self._commit_removals() self.data.add(ref(item, self._remove)) def clear(self): - if self._pending_removals: - self._commit_removals() self.data.clear() def copy(self): return self.__class__(self) def pop(self): - if self._pending_removals: - self._commit_removals() while True: try: itemref = self.data.pop() @@ -108,18 +62,12 @@ def pop(self): return item def remove(self, item): - if self._pending_removals: - self._commit_removals() self.data.remove(ref(item)) def discard(self, item): - if self._pending_removals: - self._commit_removals() self.data.discard(ref(item)) def update(self, other): - if self._pending_removals: - self._commit_removals() for element in other: self.add(element) @@ -136,8 +84,6 @@ def difference(self, other): def difference_update(self, other): self.__isub__(other) def __isub__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: @@ -151,8 +97,6 @@ def intersection(self, other): def intersection_update(self, other): self.__iand__(other) def __iand__(self, other): - if self._pending_removals: - self._commit_removals() self.data.intersection_update(ref(item) for item in other) return self @@ -184,8 +128,6 @@ def symmetric_difference(self, other): def symmetric_difference_update(self, other): self.__ixor__(other) def __ixor__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py new file mode 100644 index 00000000000000..732fbfa628cf5f --- /dev/null +++ b/Lib/annotationlib.py @@ -0,0 +1,848 @@ +"""Helpers for introspecting and wrapping annotations.""" + +import ast +import builtins +import enum +import functools +import keyword +import sys +import types + +__all__ = [ + "Format", + "ForwardRef", + "call_annotate_function", + "call_evaluate_function", + "get_annotate_function", + "get_annotations", + "annotations_to_string", + "value_to_string", +] + + +class Format(enum.IntEnum): + VALUE = 1 + FORWARDREF = 2 + STRING = 3 + + +_Union = None +_sentinel = object() + +# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be +# preserved for compatibility with the old typing.ForwardRef class. The remaining +# names are private. +_SLOTS = ( + "__forward_evaluated__", + "__forward_value__", + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + "__weakref__", + "__arg__", + "__ast_node__", + "__code__", + "__globals__", + "__owner__", + "__cell__", + "__stringifier_dict__", +) + + +class ForwardRef: + """Wrapper that holds a forward reference. + + Constructor arguments: + * arg: a string representing the code to be evaluated. + * module: the module where the forward reference was created. + Must be a string, not a module object. + * owner: The owning object (module, class, or function). + * is_argument: Does nothing, retained for compatibility. + * is_class: True if the forward reference was created in class scope. + + """ + + __slots__ = _SLOTS + + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_argument=True, + is_class=False, + ): + if not isinstance(arg, str): + raise TypeError(f"Forward reference must be a string -- got {arg!r}") + + self.__arg__ = arg + self.__forward_evaluated__ = False + self.__forward_value__ = None + self.__forward_is_argument__ = is_argument + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__code__ = None + self.__ast_node__ = None + self.__globals__ = None + self.__cell__ = None + self.__owner__ = owner + + def __init_subclass__(cls, /, *args, **kwds): + raise TypeError("Cannot subclass ForwardRef") + + def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): + """Evaluate the forward reference and return the value. + + If the forward reference cannot be evaluated, raise an exception. + """ + if self.__forward_evaluated__: + return self.__forward_value__ + if self.__cell__ is not None: + try: + value = self.__cell__.cell_contents + except ValueError: + pass + else: + self.__forward_evaluated__ = True + self.__forward_value__ = value + return value + if owner is None: + owner = self.__owner__ + + if globals is None and self.__forward_module__ is not None: + globals = getattr( + sys.modules.get(self.__forward_module__, None), "__dict__", None + ) + if globals is None: + globals = self.__globals__ + if globals is None: + if isinstance(owner, type): + module_name = getattr(owner, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + globals = getattr(module, "__dict__", None) + elif isinstance(owner, types.ModuleType): + globals = getattr(owner, "__dict__", None) + elif callable(owner): + globals = getattr(owner, "__globals__", None) + + # If we pass None to eval() below, the globals of this module are used. + if globals is None: + globals = {} + + if locals is None: + locals = {} + if isinstance(owner, type): + locals.update(vars(owner)) + + if type_params is None and owner is not None: + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + type_params = getattr(owner, "__type_params__", None) + + # type parameters require some special handling, + # as they exist in their own scope + # but `eval()` does not have a dedicated parameter for that scope. + # For classes, names in type parameter scopes should override + # names in the global scope (which here are called `localns`!), + # but should in turn be overridden by names in the class scope + # (which here are called `globalns`!) + if type_params is not None: + globals = dict(globals) + locals = dict(locals) + for param in type_params: + param_name = param.__name__ + if not self.__forward_is_class__ or param_name not in globals: + globals[param_name] = param + locals.pop(param_name, None) + + arg = self.__forward_arg__ + if arg.isidentifier() and not keyword.iskeyword(arg): + if arg in locals: + value = locals[arg] + elif arg in globals: + value = globals[arg] + elif hasattr(builtins, arg): + return getattr(builtins, arg) + else: + raise NameError(arg) + else: + code = self.__forward_code__ + value = eval(code, globals=globals, locals=locals) + self.__forward_evaluated__ = True + self.__forward_value__ = value + return value + + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + import typing + import warnings + + if type_params is _sentinel: + typing._deprecation_warning_for_no_type_params_passed( + "typing.ForwardRef._evaluate" + ) + type_params = () + warnings._deprecated( + "ForwardRef._evaluate", + "{name} is a private API and is retained for compatibility, but will be removed" + " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.", + remove=(3, 16), + ) + return typing.evaluate_forward_ref( + self, + globals=globalns, + locals=localns, + type_params=type_params, + _recursive_guard=recursive_guard, + ) + + @property + def __forward_arg__(self): + if self.__arg__ is not None: + return self.__arg__ + if self.__ast_node__ is not None: + self.__arg__ = ast.unparse(self.__ast_node__) + return self.__arg__ + raise AssertionError( + "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" + ) + + @property + def __forward_code__(self): + if self.__code__ is not None: + return self.__code__ + arg = self.__forward_arg__ + # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. + # Unfortunately, this isn't a valid expression on its own, so we + # do the unpacking manually. + if arg.startswith("*"): + arg_to_compile = f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + arg_to_compile = arg + try: + self.__code__ = compile(arg_to_compile, "", "eval") + except SyntaxError: + raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") + return self.__code__ + + def __eq__(self, other): + if not isinstance(other, ForwardRef): + return NotImplemented + if self.__forward_evaluated__ and other.__forward_evaluated__: + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_value__ == other.__forward_value__ + ) + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + ) + + def __hash__(self): + return hash((self.__forward_arg__, self.__forward_module__)) + + def __or__(self, other): + global _Union + if _Union is None: + from typing import Union as _Union + return _Union[self, other] + + def __ror__(self, other): + global _Union + if _Union is None: + from typing import Union as _Union + return _Union[other, self] + + def __repr__(self): + if self.__forward_module__ is None: + module_repr = "" + else: + module_repr = f", module={self.__forward_module__!r}" + return f"ForwardRef({self.__forward_arg__!r}{module_repr})" + + +class _Stringifier: + # Must match the slots on ForwardRef, so we can turn an instance of one into an + # instance of the other in place. + __slots__ = _SLOTS + + def __init__( + self, + node, + globals=None, + owner=None, + is_class=False, + cell=None, + *, + stringifier_dict, + ): + # Either an AST node or a simple str (for the common case where a ForwardRef + # represent a single name). + assert isinstance(node, (ast.AST, str)) + self.__arg__ = None + self.__forward_evaluated__ = False + self.__forward_value__ = None + self.__forward_is_argument__ = False + self.__forward_is_class__ = is_class + self.__forward_module__ = None + self.__code__ = None + self.__ast_node__ = node + self.__globals__ = globals + self.__cell__ = cell + self.__owner__ = owner + self.__stringifier_dict__ = stringifier_dict + + def __convert_to_ast(self, other): + if isinstance(other, _Stringifier): + if isinstance(other.__ast_node__, str): + return ast.Name(id=other.__ast_node__) + return other.__ast_node__ + elif isinstance(other, slice): + return ast.Slice( + lower=( + self.__convert_to_ast(other.start) + if other.start is not None + else None + ), + upper=( + self.__convert_to_ast(other.stop) + if other.stop is not None + else None + ), + step=( + self.__convert_to_ast(other.step) + if other.step is not None + else None + ), + ) + else: + return ast.Constant(value=other) + + def __get_ast(self): + node = self.__ast_node__ + if isinstance(node, str): + return ast.Name(id=node) + return node + + def __make_new(self, node): + stringifier = _Stringifier( + node, + self.__globals__, + self.__owner__, + self.__forward_is_class__, + stringifier_dict=self.__stringifier_dict__, + ) + self.__stringifier_dict__.stringifiers.append(stringifier) + return stringifier + + # Must implement this since we set __eq__. We hash by identity so that + # stringifiers in dict keys are kept separate. + def __hash__(self): + return id(self) + + def __getitem__(self, other): + # Special case, to avoid stringifying references to class-scoped variables + # as '__classdict__["x"]'. + if self.__ast_node__ == "__classdict__": + raise KeyError + if isinstance(other, tuple): + elts = [self.__convert_to_ast(elt) for elt in other] + other = ast.Tuple(elts) + else: + other = self.__convert_to_ast(other) + assert isinstance(other, ast.AST), repr(other) + return self.__make_new(ast.Subscript(self.__get_ast(), other)) + + def __getattr__(self, attr): + return self.__make_new(ast.Attribute(self.__get_ast(), attr)) + + def __call__(self, *args, **kwargs): + return self.__make_new( + ast.Call( + self.__get_ast(), + [self.__convert_to_ast(arg) for arg in args], + [ + ast.keyword(key, self.__convert_to_ast(value)) + for key, value in kwargs.items() + ], + ) + ) + + def __iter__(self): + yield self.__make_new(ast.Starred(self.__get_ast())) + + def __repr__(self): + if isinstance(self.__ast_node__, str): + return self.__ast_node__ + return ast.unparse(self.__ast_node__) + + def __format__(self, format_spec): + raise TypeError("Cannot stringify annotation containing string formatting") + + def _make_binop(op: ast.AST): + def binop(self, other): + return self.__make_new( + ast.BinOp(self.__get_ast(), op, self.__convert_to_ast(other)) + ) + + return binop + + __add__ = _make_binop(ast.Add()) + __sub__ = _make_binop(ast.Sub()) + __mul__ = _make_binop(ast.Mult()) + __matmul__ = _make_binop(ast.MatMult()) + __truediv__ = _make_binop(ast.Div()) + __mod__ = _make_binop(ast.Mod()) + __lshift__ = _make_binop(ast.LShift()) + __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) + __xor__ = _make_binop(ast.BitXor()) + __and__ = _make_binop(ast.BitAnd()) + __floordiv__ = _make_binop(ast.FloorDiv()) + __pow__ = _make_binop(ast.Pow()) + + del _make_binop + + def _make_rbinop(op: ast.AST): + def rbinop(self, other): + return self.__make_new( + ast.BinOp(self.__convert_to_ast(other), op, self.__get_ast()) + ) + + return rbinop + + __radd__ = _make_rbinop(ast.Add()) + __rsub__ = _make_rbinop(ast.Sub()) + __rmul__ = _make_rbinop(ast.Mult()) + __rmatmul__ = _make_rbinop(ast.MatMult()) + __rtruediv__ = _make_rbinop(ast.Div()) + __rmod__ = _make_rbinop(ast.Mod()) + __rlshift__ = _make_rbinop(ast.LShift()) + __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) + __rxor__ = _make_rbinop(ast.BitXor()) + __rand__ = _make_rbinop(ast.BitAnd()) + __rfloordiv__ = _make_rbinop(ast.FloorDiv()) + __rpow__ = _make_rbinop(ast.Pow()) + + del _make_rbinop + + def _make_compare(op): + def compare(self, other): + return self.__make_new( + ast.Compare( + left=self.__get_ast(), + ops=[op], + comparators=[self.__convert_to_ast(other)], + ) + ) + + return compare + + __lt__ = _make_compare(ast.Lt()) + __le__ = _make_compare(ast.LtE()) + __eq__ = _make_compare(ast.Eq()) + __ne__ = _make_compare(ast.NotEq()) + __gt__ = _make_compare(ast.Gt()) + __ge__ = _make_compare(ast.GtE()) + + del _make_compare + + def _make_unary_op(op): + def unary_op(self): + return self.__make_new(ast.UnaryOp(op, self.__get_ast())) + + return unary_op + + __invert__ = _make_unary_op(ast.Invert()) + __pos__ = _make_unary_op(ast.UAdd()) + __neg__ = _make_unary_op(ast.USub()) + + del _make_unary_op + + +class _StringifierDict(dict): + def __init__(self, namespace, globals=None, owner=None, is_class=False): + super().__init__(namespace) + self.namespace = namespace + self.globals = globals + self.owner = owner + self.is_class = is_class + self.stringifiers = [] + + def __missing__(self, key): + fwdref = _Stringifier( + key, + globals=self.globals, + owner=self.owner, + is_class=self.is_class, + stringifier_dict=self, + ) + self.stringifiers.append(fwdref) + return fwdref + + +def call_evaluate_function(evaluate, format, *, owner=None): + """Call an evaluate function. Evaluate functions are normally generated for + the value of type aliases and the bounds, constraints, and defaults of + type parameter objects. + """ + return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True) + + +def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): + """Call an __annotate__ function. __annotate__ functions are normally + generated by the compiler to defer the evaluation of annotations. They + can be called with any of the format arguments in the Format enum, but + compiler-generated __annotate__ functions only support the VALUE format. + This function provides additional functionality to call __annotate__ + functions with the FORWARDREF and STRING formats. + + *annotate* must be an __annotate__ function, which takes a single argument + and returns a dict of annotations. + + *format* must be a member of the Format enum or one of the corresponding + integer values. + + *owner* can be the object that owns the annotations (i.e., the module, + class, or function that the __annotate__ function derives from). With the + FORWARDREF format, it is used to provide better evaluation capabilities + on the generated ForwardRef objects. + + """ + try: + return annotate(format) + except NotImplementedError: + pass + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special + # environment where every name lookup results in an instance of _Stringifier. + # _Stringifier supports every dunder operation and returns a new _Stringifier. + # At the end, we get a dictionary that mostly contains _Stringifier objects (or + # possibly constants if the annotate function uses them directly). We then + # convert each of those into a string to get an approximation of the + # original source. + globals = _StringifierDict({}) + if annotate.__closure__: + freevars = annotate.__code__.co_freevars + new_closure = [] + for i, cell in enumerate(annotate.__closure__): + if i < len(freevars): + name = freevars[i] + else: + name = "__cell__" + fwdref = _Stringifier(name, stringifier_dict=globals) + new_closure.append(types.CellType(fwdref)) + closure = tuple(new_closure) + else: + closure = None + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + annos = func(Format.VALUE) + if _is_evaluate: + return annos if isinstance(annos, str) else repr(annos) + return { + key: val if isinstance(val, str) else repr(val) + for key, val in annos.items() + } + elif format == Format.FORWARDREF: + # FORWARDREF is implemented similarly to STRING, but there are two changes, + # at the beginning and the end of the process. + # First, while STRING uses an empty dictionary as the namespace, so that all + # name lookups result in _Stringifier objects, FORWARDREF uses the globals + # and builtins, so that defined names map to their real values. + # Second, instead of returning strings, we want to return either real values + # or ForwardRef objects. To do this, we keep track of all _Stringifier objects + # created while the annotation is being evaluated, and at the end we convert + # them all to ForwardRef objects by assigning to __class__. To make this + # technique work, we have to ensure that the _Stringifier and ForwardRef + # classes share the same attributes. + # We use this technique because while the annotations are being evaluated, + # we want to support all operations that the language allows, including even + # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately + # reconstruct the source. But in the dictionary that we eventually return, we + # want to return objects with more user-friendly behavior, such as an __eq__ + # that returns a bool and an defined set of attributes. + namespace = {**annotate.__builtins__, **annotate.__globals__} + is_class = isinstance(owner, type) + globals = _StringifierDict(namespace, annotate.__globals__, owner, is_class) + if annotate.__closure__: + freevars = annotate.__code__.co_freevars + new_closure = [] + for i, cell in enumerate(annotate.__closure__): + try: + cell.cell_contents + except ValueError: + if i < len(freevars): + name = freevars[i] + else: + name = "__cell__" + fwdref = _Stringifier( + name, + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + stringifier_dict=globals, + ) + globals.stringifiers.append(fwdref) + new_closure.append(types.CellType(fwdref)) + else: + new_closure.append(cell) + closure = tuple(new_closure) + else: + closure = None + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + result = func(Format.VALUE) + for obj in globals.stringifiers: + obj.__class__ = ForwardRef + obj.__stringifier_dict__ = None # not needed for ForwardRef + if isinstance(obj.__ast_node__, str): + obj.__arg__ = obj.__ast_node__ + obj.__ast_node__ = None + return result + elif format == Format.VALUE: + # Should be impossible because __annotate__ functions must not raise + # NotImplementedError for this format. + raise RuntimeError("annotate function does not support VALUE format") + else: + raise ValueError(f"Invalid format: {format!r}") + + +# We use the descriptors from builtins.type instead of accessing +# .__annotations__ and .__annotate__ directly on class objects, because +# otherwise we could get wrong results in some cases involving metaclasses. +# See PEP 749. +_BASE_GET_ANNOTATE = type.__dict__["__annotate__"].__get__ +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + +def get_annotate_function(obj): + """Get the __annotate__ function for an object. + + obj may be a function, class, or module, or a user-defined type with + an `__annotate__` attribute. + + Returns the __annotate__ function or None. + """ + if isinstance(obj, type): + try: + return _BASE_GET_ANNOTATE(obj) + except AttributeError: + # AttributeError is raised for static types. + return None + return getattr(obj, "__annotate__", None) + + +def get_annotations( + obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE +): + """Compute the annotations dict for an object. + + obj may be a callable, class, or module. + Passing in an object of any other type raises TypeError. + + Returns a dict. get_annotations() returns a new dict every time + it's called; calling it twice on the same object will return two + different but equivalent dicts. + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if eval_str and format != Format.VALUE: + raise ValueError("eval_str=True is only supported with format=Format.VALUE") + + match format: + case Format.VALUE: + # For VALUE, we only look at __annotations__ + ann = _get_dunder_annotations(obj) + case Format.FORWARDREF: + # For FORWARDREF, we use __annotations__ if it exists + try: + return dict(_get_dunder_annotations(obj)) + except NameError: + pass + + # But if __annotations__ threw a NameError, we try calling __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return ann + + # If that didn't work either, we have a very weird object: evaluating + # __annotations__ threw NameError and there is no __annotate__. In that case, + # we fall back to trying __annotations__ again. + return dict(_get_dunder_annotations(obj)) + case Format.STRING: + # For STRING, we try to call __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return ann + # But if we didn't get it, we use __annotations__ instead. + ann = _get_dunder_annotations(obj) + return annotations_to_string(ann) + case _: + raise ValueError(f"Unsupported format {format!r}") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + elif ann is not None: + obj_globals = obj_locals = unwrap = None + else: + raise TypeError(f"{obj!r} is not a module, class, or callable.") + + if unwrap is not None: + while True: + if hasattr(unwrap, "__wrapped__"): + unwrap = unwrap.__wrapped__ + continue + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + if locals is None: + locals = {} + locals = {param.__name__: param for param in type_params} | locals + + return_value = { + key: value if not isinstance(value, str) else eval(value, globals, locals) + for key, value in ann.items() + } + return return_value + + +def value_to_string(value): + """Convert a Python value to a format suitable for use with the STRING format. + + This is inteded as a helper for tools that support the STRING format but do + not have access to the code that originally produced the annotations. It uses + repr() for most objects. + + """ + if isinstance(value, type): + if value.__module__ == "builtins": + return value.__qualname__ + return f"{value.__module__}.{value.__qualname__}" + if value is ...: + return "..." + if isinstance(value, (types.FunctionType, types.BuiltinFunctionType)): + return value.__name__ + return repr(value) + + +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format.""" + return { + n: t if isinstance(t, str) else value_to_string(t) + for n, t in annotations.items() + } + + +def _get_and_call_annotate(obj, format): + annotate = get_annotate_function(obj) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return dict(ann) + return None + + +def _get_dunder_annotations(obj): + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return {} + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return {} + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + return dict(ann) diff --git a/Lib/argparse.py b/Lib/argparse.py index 04ee3b19aca755..072cd5e7dc0d06 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -18,11 +18,12 @@ 'integers', metavar='int', nargs='+', type=int, help='an integer to be summed') parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), + '--log', help='the file where the sum should be written') args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() + with (open(args.log, 'w') if args.log is not None + else contextlib.nullcontext(sys.stdout)) as log: + log.write('%s' % sum(args.integers)) The module contains the following public classes: @@ -39,7 +40,8 @@ - FileType -- A factory for defining types of files to be created. As the example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. + the type= argument of add_argument() calls. Deprecated since + Python 3.14. - Action -- The base class for parser actions. Typically actions are selected by passing strings like 'store_true' or 'append_const' to @@ -223,7 +225,8 @@ def format_help(self): # add the heading if the section was non-empty if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) + heading_text = _('%(heading)s:') % dict(heading=self.heading) + heading = '%*s%s\n' % (current_indent, '', heading_text) else: heading = '' @@ -260,13 +263,12 @@ def add_argument(self, action): # find all invocations get_invocation = self._format_action_invocation - invocations = [get_invocation(action)] + invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): - invocations.append(get_invocation(subaction)) + invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) # update the maximum item length - invocation_length = max(map(len, invocations)) - action_length = invocation_length + self._current_indent + action_length = max(invocation_lengths) self._action_max_length = max(self._action_max_length, action_length) @@ -327,17 +329,8 @@ def _format_usage(self, usage, actions, groups, prefix): if len(prefix) + len(usage) > text_width: # break usage into wrappable parts - part_regexp = ( - r'\(.*?\)+(?=\s|$)|' - r'\[.*?\]+(?=\s|$)|' - r'\S+' - ) - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage + opt_parts = self._get_actions_usage_parts(optionals, groups) + pos_parts = self._get_actions_usage_parts(positionals, groups) # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -390,6 +383,9 @@ def get_lines(parts, indent, prefix=None): return '%s%s\n\n' % (prefix, usage) def _format_actions_usage(self, actions, groups): + return ' '.join(self._get_actions_usage_parts(actions, groups)) + + def _get_actions_usage_parts(self, actions, groups): # find group indices and identify actions in groups group_actions = set() inserts = {} @@ -397,56 +393,26 @@ def _format_actions_usage(self, actions, groups): if not group._group_actions: raise ValueError(f'empty group {group}') + if all(action.help is SUPPRESS for action in group._group_actions): + continue + try: - start = actions.index(group._group_actions[0]) + start = min(actions.index(item) for item in group._group_actions) except ValueError: continue else: - group_action_count = len(group._group_actions) - end = start + group_action_count - if actions[start:end] == group._group_actions: - - suppressed_actions_count = 0 - for action in group._group_actions: - group_actions.add(action) - if action.help is SUPPRESS: - suppressed_actions_count += 1 - - exposed_actions_count = group_action_count - suppressed_actions_count - - if not group.required: - if start in inserts: - inserts[start] += ' [' - else: - inserts[start] = '[' - if end in inserts: - inserts[end] += ']' - else: - inserts[end] = ']' - elif exposed_actions_count > 1: - if start in inserts: - inserts[start] += ' (' - else: - inserts[start] = '(' - if end in inserts: - inserts[end] += ')' - else: - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' + end = start + len(group._group_actions) + if set(actions[start:end]) == set(group._group_actions): + group_actions.update(group._group_actions) + inserts[start, end] = group # collect all actions format strings parts = [] - for i, action in enumerate(actions): + for action in actions: # suppressed arguments are marked with None - # remove | separators for suppressed arguments if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) + part = None # produce all arg strings elif not action.option_strings: @@ -458,9 +424,6 @@ def _format_actions_usage(self, actions, groups): if part[0] == '[' and part[-1] == ']': part = part[1:-1] - # add the action string to the list - parts.append(part) - # produce the first way to invoke the option in brackets else: option_string = action.option_strings[0] @@ -481,26 +444,32 @@ def _format_actions_usage(self, actions, groups): if not action.required and action not in group_actions: part = '[%s]' % part - # add the action string to the list - parts.append(part) + # add the action string to the list + parts.append(part) - # insert things at the necessary indices - for i in sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = text.strip() - - # return the text - return text + # group mutually exclusive actions + inserted_separators_indices = set() + for start, end in sorted(inserts, reverse=True): + group = inserts[start, end] + group_parts = [item for item in parts[start:end] if item is not None] + group_size = len(group_parts) + if group.required: + open, close = "()" if group_size > 1 else ("", "") + else: + open, close = "[]" + group_parts[0] = open + group_parts[0] + group_parts[-1] = group_parts[-1] + close + for i, part in enumerate(group_parts[:-1], start=start): + # insert a separator if not already done in a nested group + if i not in inserted_separators_indices: + parts[i] = part + ' |' + inserted_separators_indices.add(i) + parts[start + group_size - 1] = group_parts[-1] + for i in range(start + group_size, end): + parts[i] = None + + # return the usage parts + return [item for item in parts if item is not None] def _format_text(self, text): if '%(prog)' in text: @@ -560,8 +529,7 @@ def _format_action(self, action): def _format_action_invocation(self, action): if not action.option_strings: default = self._get_default_metavar_for_positional(action) - metavar, = self._metavar_formatter(action, default)(1) - return metavar + return ' '.join(self._metavar_formatter(action, default)(1)) else: @@ -581,8 +549,7 @@ def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: result = action.metavar elif action.choices is not None: - choice_strs = [str(choice) for choice in action.choices] - result = '{%s}' % ','.join(choice_strs) + result = '{%s}' % ','.join(map(str, action.choices)) else: result = default_metavar @@ -622,17 +589,19 @@ def _format_args(self, action, default_metavar): return result def _expand_help(self, action): + help_string = self._get_help_string(action) + if '%' not in help_string: + return help_string params = dict(vars(action), prog=self._prog) for name in list(params): - if params[name] is SUPPRESS: + value = params[name] + if value is SUPPRESS: del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ + elif hasattr(value, '__name__'): + params[name] = value.__name__ if params.get('choices') is not None: - choices_str = ', '.join([str(c) for c in params['choices']]) - params['choices'] = choices_str - return self._get_help_string(action) % params + params['choices'] = ', '.join(map(str, params['choices'])) + return help_string % params def _iter_indented_subactions(self, action): try: @@ -698,14 +667,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter): """ def _get_help_string(self, action): - """ - Add the default value to the option help message. - - ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't - already present. This code will do that, detecting cornercases to - prevent duplicates or cases where it wouldn't make sense to the end - user. - """ help = action.help if help is None: help = '' @@ -714,7 +675,7 @@ def _get_help_string(self, action): if action.default is not SUPPRESS: defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' + help += _(' (default: %(default)s)') return help @@ -744,11 +705,19 @@ def _get_action_name(argument): elif argument.option_strings: return '/'.join(argument.option_strings) elif argument.metavar not in (None, SUPPRESS): - return argument.metavar + metavar = argument.metavar + if not isinstance(metavar, tuple): + return metavar + if argument.nargs == ZERO_OR_MORE and len(metavar) == 2: + return '%s[, %s]' % metavar + elif argument.nargs == ONE_OR_MORE: + return '%s[, %s]' % metavar + else: + return ', '.join(metavar) elif argument.dest not in (None, SUPPRESS): return argument.dest elif argument.choices: - return '{' + ','.join(argument.choices) + '}' + return '{%s}' % ','.join(map(str, argument.choices)) else: return None @@ -877,22 +846,16 @@ def format_usage(self): return self.option_strings[0] def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - + raise NotImplementedError('.__call__() not defined') -# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. -_deprecated_default = object() class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, default=None, - type=_deprecated_default, - choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default, deprecated=False): _option_strings = [] @@ -903,35 +866,13 @@ def __init__(self, option_string = '--no-' + option_string[2:] _option_strings.append(option_string) - # We need `_deprecated` special value to ban explicit arguments that - # match default value. Like: - # parser.add_argument('-f', action=BooleanOptionalAction, type=int) - for field_name in ('type', 'choices', 'metavar'): - if locals()[field_name] is not _deprecated_default: - import warnings - warnings._deprecated( - field_name, - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - if type is _deprecated_default: - type = None - if choices is _deprecated_default: - choices = None - if metavar is _deprecated_default: - metavar = None - super().__init__( option_strings=_option_strings, dest=dest, nargs=0, default=default, - type=type, - choices=choices, required=required, help=help, - metavar=metavar, deprecated=deprecated) @@ -1165,8 +1106,10 @@ def __init__(self, version=None, dest=SUPPRESS, default=SUPPRESS, - help="show program's version number and exit", + help=None, deprecated=False): + if help is None: + help = _("show program's version number and exit") super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1229,20 +1172,23 @@ def add_parser(self, name, *, deprecated=False, **kwargs): aliases = kwargs.pop('aliases', ()) if name in self._name_parser_map: - raise ArgumentError(self, _('conflicting subparser: %s') % name) + raise ValueError(f'conflicting subparser: {name}') for alias in aliases: if alias in self._name_parser_map: - raise ArgumentError( - self, _('conflicting subparser alias: %s') % alias) + raise ValueError(f'conflicting subparser alias: {alias}') # create a pseudo-action to hold the choice help if 'help' in kwargs: help = kwargs.pop('help') choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) + else: + choice_action = None # create the parser and add it to the map parser = self._parser_class(**kwargs) + if choice_action is not None: + parser._check_help(choice_action) self._name_parser_map[name] = parser # make parser available under aliases also @@ -1291,7 +1237,8 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, key, value) if arg_strings: - vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + if not hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + setattr(namespace, _UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) class _ExtendAction(_AppendAction): @@ -1306,7 +1253,7 @@ def __call__(self, parser, namespace, values, option_string=None): # ============== class FileType(object): - """Factory for creating file object types + """Deprecated factory for creating file object types Instances of FileType are typically passed as type= arguments to the ArgumentParser add_argument() method. @@ -1323,6 +1270,12 @@ class FileType(object): """ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + import warnings + warnings.warn( + "FileType is deprecated. Simply open files after parsing arguments.", + category=PendingDeprecationWarning, + stacklevel=2 + ) self._mode = mode self._bufsize = bufsize self._encoding = encoding @@ -1426,7 +1379,7 @@ def __init__(self, self._defaults = {} # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + self._negative_number_matcher = _re.compile(r'-\.?\d') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited @@ -1476,7 +1429,8 @@ def add_argument(self, *args, **kwargs): chars = self.prefix_chars if not args or len(args) == 1 and args[0][0] not in chars: if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') + raise TypeError('dest supplied twice for positional argument,' + ' did you mean metavar?') kwargs = self._get_positional_kwargs(*args, **kwargs) # otherwise, we're adding an optional argument @@ -1492,27 +1446,34 @@ def add_argument(self, *args, **kwargs): kwargs['default'] = self.argument_default # create the action object, and add it to the parser + action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): - raise ValueError('unknown action "%s"' % (action_class,)) + raise ValueError('unknown action {action_class!r}') action = action_class(**kwargs) + # raise an error if action for positional argument does not + # consume arguments + if not action.option_strings and action.nargs == 0: + raise ValueError(f'action {action_name!r} is not valid for positional arguments') + # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - raise ValueError('%r is not callable' % (type_func,)) + raise TypeError(f'{type_func!r} is not callable') if type_func is FileType: - raise ValueError('%r is a FileType class object, instance of it' - ' must be passed' % (type_func,)) + raise TypeError(f'{type_func!r} is a FileType class object, ' + f'instance of it must be passed') # raise an error if the metavar does not match the type if hasattr(self, "_get_formatter"): + formatter = self._get_formatter() try: - self._get_formatter()._format_args(action, None) + formatter._format_args(action, None) except TypeError: raise ValueError("length of metavar tuple does not match nargs") - + self._check_help(action) return self._add_action(action) def add_argument_group(self, *args, **kwargs): @@ -1556,8 +1517,8 @@ def _add_container_actions(self, container): if group.title in title_group_map: # This branch could happen if a derived class added # groups with duplicated titles in __init__ - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) + msg = f'cannot merge actions - two groups are named {group.title!r}' + raise ValueError(msg) title_group_map[group.title] = group # map each action to its group @@ -1580,7 +1541,11 @@ def _add_container_actions(self, container): # NOTE: if add_mutually_exclusive_group ever gains title= and # description= then this code will need to be expanded as above for group in container._mutually_exclusive_groups: - mutex_group = self.add_mutually_exclusive_group( + if group._container is container: + cont = self + else: + cont = title_group_map[group._container.title] + mutex_group = cont.add_mutually_exclusive_group( required=group.required) # map the actions to their new mutex group @@ -1594,14 +1559,15 @@ def _add_container_actions(self, container): def _get_positional_kwargs(self, dest, **kwargs): # make sure required is not specified if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") + msg = "'required' is an invalid argument for positionals" raise TypeError(msg) # mark positional arguments as required if at least one is # always required - if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: - kwargs['required'] = True - if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + nargs = kwargs.get('nargs') + if nargs == 0: + raise ValueError('nargs for positionals must be != 0') + if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS]: kwargs['required'] = True # return the keyword arguments with no option strings @@ -1614,11 +1580,9 @@ def _get_optional_kwargs(self, *args, **kwargs): for option_string in args: # error on strings that don't start with an appropriate prefix if not option_string[0] in self.prefix_chars: - args = {'option': option_string, - 'prefix_chars': self.prefix_chars} - msg = _('invalid option string %(option)r: ' - 'must start with a character %(prefix_chars)r') - raise ValueError(msg % args) + raise ValueError( + f'invalid option string {option_string!r}: ' + f'must start with a character {self.prefix_chars!r}') # strings starting with two prefix characters are long options option_strings.append(option_string) @@ -1634,8 +1598,8 @@ def _get_optional_kwargs(self, *args, **kwargs): dest_option_string = option_strings[0] dest = dest_option_string.lstrip(self.prefix_chars) if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) + msg = f'dest= is required for options like {option_string!r}' + raise TypeError(msg) dest = dest.replace('-', '_') # return the updated keyword arguments @@ -1651,8 +1615,8 @@ def _get_handler(self): try: return getattr(self, handler_func_name) except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) + msg = f'invalid conflict_resolution value: {self.conflict_handler!r}' + raise ValueError(msg) def _check_conflict(self, action): @@ -1691,10 +1655,26 @@ def _handle_conflict_resolve(self, action, conflicting_actions): if not action.option_strings: action.container._remove_action(action) + def _check_help(self, action): + if action.help and hasattr(self, "_get_formatter"): + formatter = self._get_formatter() + try: + formatter._expand_help(action) + except (ValueError, TypeError, KeyError) as exc: + raise ValueError('badly formed help string') from exc + class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, **kwargs): + if 'prefix_chars' in kwargs: + import warnings + depr_msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + warnings.warn(depr_msg, DeprecationWarning, stacklevel=3) + # add any missing keyword arguments by checking the container update = kwargs.setdefault update('conflict_handler', container.conflict_handler) @@ -1744,7 +1724,7 @@ def __init__(self, container, required=False): def _add_action(self, action): if action.required: - msg = _('mutually exclusive arguments must be optional') + msg = 'mutually exclusive arguments must be optional' raise ValueError(msg) action = self._container._add_action(action) self._group_actions.append(action) @@ -1764,6 +1744,28 @@ def add_mutually_exclusive_group(self, *args, **kwargs): return super().add_mutually_exclusive_group(*args, **kwargs) +def _prog_name(prog=None): + if prog is not None: + return prog + arg0 = _sys.argv[0] + try: + modspec = _sys.modules['__main__'].__spec__ + except (KeyError, AttributeError): + # possibly PYTHONSTARTUP or -X presite or other weird edge case + # no good answer here, so fall back to the default + modspec = None + if modspec is None: + # simple script + return _os.path.basename(arg0) + py = _os.path.basename(_sys.executable) + if modspec.name != '__main__': + # imported module or package + modname = modspec.name.removesuffix('.__main__') + return f'{py} -m {modname}' + # directory or ZIP file + return f'{py} {arg0}' + + class ArgumentParser(_AttributeHolder, _ActionsContainer): """Object for parsing command line strings into Python objects. @@ -1784,6 +1786,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): - allow_abbrev -- Allow long options to be abbreviated unambiguously - exit_on_error -- Determines whether or not ArgumentParser exits with error info when an error occurs + - suggest_on_error - Enables suggestions for mistyped argument choices + and subparser names. (default: ``False``) """ def __init__(self, @@ -1799,7 +1803,8 @@ def __init__(self, conflict_handler='error', add_help=True, allow_abbrev=True, - exit_on_error=True): + exit_on_error=True, + suggest_on_error=False): superinit = super(ArgumentParser, self).__init__ superinit(description=description, @@ -1807,11 +1812,7 @@ def __init__(self, argument_default=argument_default, conflict_handler=conflict_handler) - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog + self.prog = _prog_name(prog) self.usage = usage self.epilog = epilog self.formatter_class = formatter_class @@ -1819,6 +1820,7 @@ def __init__(self, self.add_help = add_help self.allow_abbrev = allow_abbrev self.exit_on_error = exit_on_error + self.suggest_on_error = suggest_on_error add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) @@ -1866,14 +1868,14 @@ def _get_kwargs(self): # ================================== def add_subparsers(self, **kwargs): if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) + raise ValueError('cannot have multiple subparser arguments') # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) if 'title' in kwargs or 'description' in kwargs: - title = _(kwargs.pop('title', 'subcommands')) - description = _(kwargs.pop('description', None)) + title = kwargs.pop('title', _('subcommands')) + description = kwargs.pop('description', None) self._subparsers = self.add_argument_group(title, description) else: self._subparsers = self._positionals @@ -1890,6 +1892,7 @@ def add_subparsers(self, **kwargs): # create the parsers action and add it to the positionals list parsers_class = self._pop_action_class(kwargs, 'parsers') action = parsers_class(option_strings=[], **kwargs) + self._check_help(action) self._subparsers._add_action(action) # return the created parsers action @@ -1918,11 +1921,17 @@ def _get_positional_actions(self): def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_args(self, args=None, namespace=None): + return self._parse_known_args2(args, namespace, intermixed=False) + + def _parse_known_args2(self, args, namespace, intermixed): if args is None: # args default to the system args args = _sys.argv[1:] @@ -1949,18 +1958,18 @@ def parse_known_args(self, args=None, namespace=None): # parse the arguments and exit if there are any errors if self.exit_on_error: try: - namespace, args = self._parse_known_args(args, namespace) + namespace, args = self._parse_known_args(args, namespace, intermixed) except ArgumentError as err: self.error(str(err)) else: - namespace, args = self._parse_known_args(args, namespace) + namespace, args = self._parse_known_args(args, namespace, intermixed) if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) return namespace, args - def _parse_known_args(self, arg_strings, namespace): + def _parse_known_args(self, arg_strings, namespace, intermixed): # replace arg strings that are file references if self.fromfile_prefix_chars is not None: arg_strings = self._read_args_from_files(arg_strings) @@ -1992,11 +2001,11 @@ def _parse_known_args(self, arg_strings, namespace): # otherwise, add the arg to the arg strings # and note the index if it was an option else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: + option_tuples = self._parse_optional(arg_string) + if option_tuples is None: pattern = 'A' else: - option_string_indices[i] = option_tuple + option_string_indices[i] = option_tuples pattern = 'O' arg_string_pattern_parts.append(pattern) @@ -2013,9 +2022,8 @@ def take_action(action, argument_strings, option_string=None): argument_values = self._get_values(action, argument_strings) # error if this argument is not allowed with other previously - # seen arguments, assuming that actions that use the default - # value don't really count as "present" - if argument_values is not action.default: + # seen arguments + if action.option_strings or argument_strings: seen_non_default_actions.add(action) for conflict_action in action_conflicts.get(action, []): if conflict_action in seen_non_default_actions: @@ -2032,8 +2040,16 @@ def take_action(action, argument_strings, option_string=None): def consume_optional(start_index): # get the optional identified at this index - option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple + option_tuples = option_string_indices[start_index] + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, sep, explicit_arg in option_tuples]) + args = {'option': arg_strings[start_index], 'matches': options} + msg = _('ambiguous option: %(option)s could match %(matches)s') + raise ArgumentError(None, msg % args) + + action, option_string, sep, explicit_arg = option_tuples[0] # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) @@ -2044,6 +2060,7 @@ def consume_optional(start_index): # if we found no optional action, skip it if action is None: extras.append(arg_strings[start_index]) + extras_pattern.append('O') return start_index + 1 # if there is an explicit argument, try to match the @@ -2060,18 +2077,28 @@ def consume_optional(start_index): and option_string[1] not in chars and explicit_arg != '' ): + if sep or explicit_arg[0] in chars: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) action_tuples.append((action, [], option_string)) char = option_string[0] option_string = char + explicit_arg[0] - new_explicit_arg = explicit_arg[1:] or None optionals_map = self._option_string_actions if option_string in optionals_map: action = optionals_map[option_string] - explicit_arg = new_explicit_arg + explicit_arg = explicit_arg[1:] + if not explicit_arg: + sep = explicit_arg = None + elif explicit_arg[0] == '=': + sep = '=' + explicit_arg = explicit_arg[1:] + else: + sep = '' else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - + extras.append(char + explicit_arg) + extras_pattern.append('O') + stop = start_index + 1 + break # if the action expect exactly one argument, we've # successfully matched the option; exit the loop elif arg_count == 1: @@ -2124,6 +2151,15 @@ def consume_positionals(start_index): # and add the Positional and its args to the list for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] + # Strip out the first '--' if it is not in REMAINDER arg. + if action.nargs == PARSER: + if arg_strings_pattern[start_index] == '-': + assert args[0] == '--' + args.remove('--') + elif action.nargs != REMAINDER: + if (arg_strings_pattern.find('-', start_index, + start_index + arg_count) >= 0): + args.remove('--') start_index += arg_count if args and action.deprecated and action.dest not in warned: self._warning(_("argument '%(argument_name)s' is deprecated") % @@ -2139,6 +2175,7 @@ def consume_positionals(start_index): # consume Positionals and Optionals alternately, until we have # passed the last option string extras = [] + extras_pattern = [] start_index = 0 if option_string_indices: max_option_string_index = max(option_string_indices) @@ -2147,11 +2184,12 @@ def consume_positionals(start_index): while start_index <= max_option_string_index: # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) - if start_index != next_option_string_index: + next_option_string_index = start_index + while next_option_string_index <= max_option_string_index: + if next_option_string_index in option_string_indices: + break + next_option_string_index += 1 + if not intermixed and start_index != next_option_string_index: positionals_end_index = consume_positionals(start_index) # only try to parse the next optional if we didn't consume @@ -2167,16 +2205,35 @@ def consume_positionals(start_index): if start_index not in option_string_indices: strings = arg_strings[start_index:next_option_string_index] extras.extend(strings) + extras_pattern.extend(arg_strings_pattern[start_index:next_option_string_index]) start_index = next_option_string_index # consume the next optional and any arguments for it start_index = consume_optional(start_index) - # consume any positionals following the last Optional - stop_index = consume_positionals(start_index) + if not intermixed: + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) - # if we didn't consume all the argument strings, there were extras - extras.extend(arg_strings[stop_index:]) + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + else: + extras.extend(arg_strings[start_index:]) + extras_pattern.extend(arg_strings_pattern[start_index:]) + extras_pattern = ''.join(extras_pattern) + assert len(extras_pattern) == len(extras) + # consume all positionals + arg_strings = [s for s, c in zip(extras, extras_pattern) if c != 'O'] + arg_strings_pattern = extras_pattern.replace('O', '') + stop_index = consume_positionals(0) + # leave unknown optionals and non-consumed positionals in extras + for i, c in enumerate(extras_pattern): + if not stop_index: + break + if c != 'O': + stop_index -= 1 + extras[i] = None + extras = [s for s in extras if s is not None] # make sure all required actions were present and also convert # action defaults which were not given as arguments @@ -2198,7 +2255,7 @@ def consume_positionals(start_index): self._get_value(action, action.default)) if required_actions: - self.error(_('the following arguments are required: %s') % + raise ArgumentError(None, _('the following arguments are required: %s') % ', '.join(required_actions)) # make sure all required groups had one option present @@ -2214,7 +2271,7 @@ def consume_positionals(start_index): for action in group._group_actions if action.help is not SUPPRESS] msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) + raise ArgumentError(None, msg % ' '.join(names)) # return the updated namespace and the extra arguments return namespace, extras @@ -2241,7 +2298,7 @@ def _read_args_from_files(self, arg_strings): arg_strings = self._read_args_from_files(arg_strings) new_arg_strings.extend(arg_strings) except OSError as err: - self.error(str(err)) + raise ArgumentError(None, str(err)) # return the modified argument list return new_arg_strings @@ -2274,18 +2331,19 @@ def _match_argument(self, action, arg_strings_pattern): def _match_arguments_partial(self, actions, arg_strings_pattern): # progressively shorten the actions list by slicing off the # final actions until we find a match - result = [] for i in range(len(actions), 0, -1): actions_slice = actions[:i] pattern = ''.join([self._get_nargs_pattern(action) for action in actions_slice]) match = _re.match(pattern, arg_strings_pattern) if match is not None: - result.extend([len(string) for string in match.groups()]) - break - - # return the list of arg string counts - return result + result = [len(string) for string in match.groups()] + if (match.end() < len(arg_strings_pattern) + and arg_strings_pattern[match.end()] == 'O'): + while result and not result[-1]: + del result[-1] + return result + return [] def _parse_optional(self, arg_string): # if it's an empty string, it was meant to be a positional @@ -2299,36 +2357,24 @@ def _parse_optional(self, arg_string): # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] - return action, arg_string, None + return [(action, arg_string, None, None)] # if it's just a single character, it was meant to be positional if len(arg_string) == 1: return None # if the option string before the "=" is present, return the action - if '=' in arg_string: - option_string, explicit_arg = arg_string.split('=', 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg + option_string, sep, explicit_arg = arg_string.partition('=') + if sep and option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return [(action, option_string, sep, explicit_arg)] # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations option_tuples = self._get_option_tuples(arg_string) - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) - args = {'option': arg_string, 'matches': options} - msg = _('ambiguous option: %(option)s could match %(matches)s') - self.error(msg % args) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple + if option_tuples: + return option_tuples # if it was not found as an option, but it looks like a negative # number, it was meant to be positional @@ -2343,7 +2389,7 @@ def _parse_optional(self, arg_string): # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None + return [(None, arg_string, None, None)] def _get_option_tuples(self, option_string): result = [] @@ -2353,39 +2399,38 @@ def _get_option_tuples(self, option_string): chars = self.prefix_chars if option_string[0] in chars and option_string[1] in chars: if self.allow_abbrev: - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None for option_string in self._option_string_actions: if option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, sep, explicit_arg result.append(tup) # single character options can be concatenated with their arguments # but multiple character options always have to have their argument # separate elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string - explicit_arg = None + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] for option_string in self._option_string_actions: if option_string == short_option_prefix: action = self._option_string_actions[option_string] - tup = action, option_string, short_explicit_arg + tup = action, option_string, '', short_explicit_arg result.append(tup) - elif option_string.startswith(option_prefix): + elif self.allow_abbrev and option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, sep, explicit_arg result.append(tup) # shouldn't ever get here else: - self.error(_('unexpected option string: %s') % option_string) + raise ArgumentError(None, _('unexpected option string: %s') % option_string) # return the collected option tuples return result @@ -2394,43 +2439,40 @@ def _get_nargs_pattern(self, action): # in all examples below, we have to allow for '--' args # which are represented as '-' in the pattern nargs = action.nargs + # if this is an optional action, -- is not allowed + option = action.option_strings # the default (None) is assumed to be a single argument if nargs is None: - nargs_pattern = '(-*A-*)' + nargs_pattern = '([A])' if option else '(-*A-*)' # allow zero or one arguments elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' + nargs_pattern = '(A?)' if option else '(-*A?-*)' # allow zero or more arguments elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' + nargs_pattern = '(A*)' if option else '(-*[A-]*)' # allow one or more arguments elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' + nargs_pattern = '(A+)' if option else '(-*A[A-]*)' # allow any number of options or arguments elif nargs == REMAINDER: - nargs_pattern = '([-AO]*)' + nargs_pattern = '([AO]*)' if option else '(.*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: - nargs_pattern = '(-*A[-AO]*)' + nargs_pattern = '(A[AO]*)' if option else '(-*A[-AO]*)' # suppress action, like nargs=0 elif nargs == SUPPRESS: - nargs_pattern = '(-*-*)' + nargs_pattern = '()' if option else '(-*)' # all others should be integers else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') + nargs_pattern = '([AO]{%d})' % nargs if option else '((?:-*A){%d}-*)' % nargs # return the pattern return nargs_pattern @@ -2442,8 +2484,11 @@ def _get_nargs_pattern(self, action): def parse_intermixed_args(self, args=None, namespace=None): args, argv = self.parse_known_intermixed_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_intermixed_args(self, args=None, namespace=None): @@ -2454,10 +2499,6 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # are then parsed. If the parser definition is incompatible with the # intermixed assumptions (e.g. use of REMAINDER, subparsers) a # TypeError is raised. - # - # positionals are 'deactivated' by setting nargs and default to - # SUPPRESS. This blocks the addition of that positional to the - # namespace positionals = self._get_positional_actions() a = [action for action in positionals @@ -2466,80 +2507,20 @@ def parse_known_intermixed_args(self, args=None, namespace=None): raise TypeError('parse_intermixed_args: positional arg' ' with nargs=%s'%a[0].nargs) - if [action.dest for group in self._mutually_exclusive_groups - for action in group._group_actions if action in positionals]: - raise TypeError('parse_intermixed_args: positional in' - ' mutuallyExclusiveGroup') - - try: - save_usage = self.usage - try: - if self.usage is None: - # capture the full usage for use in error messages - self.usage = self.format_usage()[7:] - for action in positionals: - # deactivate positionals - action.save_nargs = action.nargs - # action.nargs = 0 - action.nargs = SUPPRESS - action.save_default = action.default - action.default = SUPPRESS - namespace, remaining_args = self.parse_known_args(args, - namespace) - for action in positionals: - # remove the empty positional values from namespace - if (hasattr(namespace, action.dest) - and getattr(namespace, action.dest)==[]): - from warnings import warn - warn('Do not expect %s in %s' % (action.dest, namespace)) - delattr(namespace, action.dest) - finally: - # restore nargs and usage before exiting - for action in positionals: - action.nargs = action.save_nargs - action.default = action.save_default - optionals = self._get_optional_actions() - try: - # parse positionals. optionals aren't normally required, but - # they could be, so make sure they aren't. - for action in optionals: - action.save_required = action.required - action.required = False - for group in self._mutually_exclusive_groups: - group.save_required = group.required - group.required = False - namespace, extras = self.parse_known_args(remaining_args, - namespace) - finally: - # restore parser values before exiting - for action in optionals: - action.required = action.save_required - for group in self._mutually_exclusive_groups: - group.required = group.save_required - finally: - self.usage = save_usage - return namespace, extras + return self._parse_known_args2(args, namespace, intermixed=True) # ======================== # Value conversion methods # ======================== def _get_values(self, action, arg_strings): - # for everything but PARSER, REMAINDER args, strip out first '--' - if not action.option_strings and action.nargs not in [PARSER, REMAINDER]: - try: - arg_strings.remove('--') - except ValueError: - pass - # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: if action.option_strings: value = action.const else: value = action.default - if isinstance(value, str): + if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2547,11 +2528,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: @@ -2584,8 +2562,7 @@ def _get_values(self, action, arg_strings): def _get_value(self, action, arg_string): type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) + raise TypeError(f'{type_func!r} is not callable') # convert the value to the appropriate type try: @@ -2608,10 +2585,27 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - args = {'value': value, - 'choices': ', '.join(map(repr, action.choices))} + choices = action.choices + if choices is None: + return + + if isinstance(choices, str): + choices = iter(choices) + + if value not in choices: + args = {'value': str(value), + 'choices': ', '.join(map(str, action.choices))} msg = _('invalid choice: %(value)r (choose from %(choices)s)') + + if self.suggest_on_error and isinstance(value, str): + if all(isinstance(choice, str) for choice in action.choices): + import difflib + suggestions = difflib.get_close_matches(value, action.choices, 1) + if suggestions: + args['closest'] = suggestions[0] + msg = _('invalid choice: %(value)r, maybe you meant %(closest)r? ' + '(choose from %(choices)s)') + raise ArgumentError(action, msg % args) # ======================= diff --git a/Lib/ast.py b/Lib/ast.py index 43703a8325cc5e..154d2c8c1f9ebb 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -25,7 +25,6 @@ :license: Python License. """ import sys -import re from _ast import * from contextlib import contextmanager, nullcontext from enum import IntEnum, auto, _simple_enum @@ -114,7 +113,11 @@ def _convert(node): return _convert(node_or_string) -def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): +def dump( + node, annotate_fields=True, include_attributes=False, + *, + indent=None, show_empty=False, +): """ Return a formatted dump of the tree in node. This is mainly useful for debugging purposes. If annotate_fields is true (by default), @@ -125,6 +128,8 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): include_attributes can be set to true. If indent is a non-negative integer or string, then the tree will be pretty-printed with that indent level. None (the default) selects the single line representation. + If show_empty is False, then empty lists and fields that are None + will be omitted from the output for better readability. """ def _format(node, level=0): if indent is not None: @@ -137,6 +142,7 @@ def _format(node, level=0): if isinstance(node, AST): cls = type(node) args = [] + args_buffer = [] allsimple = True keywords = annotate_fields for name in node._fields: @@ -148,6 +154,18 @@ def _format(node, level=0): if value is None and getattr(cls, name, ...) is None: keywords = True continue + if ( + not show_empty + and (value is None or value == []) + # Special cases: + # `Constant(value=None)` and `MatchSingleton(value=None)` + and not isinstance(node, (Constant, MatchSingleton)) + ): + args_buffer.append(repr(value)) + continue + elif not keywords: + args.extend(args_buffer) + args_buffer = [] value, simple = _format(value, level) allsimple = allsimple and simple if keywords: @@ -306,12 +324,18 @@ def get_docstring(node, clean=True): return text -_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") +_line_pattern = None def _splitlines_no_ff(source, maxlines=None): """Split a string into lines ignoring form feed and other chars. This mimics how the Python parser splits source code. """ + global _line_pattern + if _line_pattern is None: + # lazily computed to speedup import time of `ast` + import re + _line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + lines = [] for lineno, match in enumerate(_line_pattern.finditer(source), 1): if maxlines is not None and lineno > maxlines: @@ -382,6 +406,88 @@ def walk(node): yield node +def compare( + a, + b, + /, + *, + compare_attributes=False, +): + """Recursively compares two ASTs. + + compare_attributes affects whether AST attributes are considered + in the comparison. If compare_attributes is False (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + might differ in whitespace or similar details. + """ + + sentinel = object() # handle the possibility of a missing attribute/field + + def _compare(a, b): + # Compare two fields on an AST object, which may themselves be + # AST objects, lists of AST objects, or primitive ASDL types + # like identifiers and constants. + if isinstance(a, AST): + return compare( + a, + b, + compare_attributes=compare_attributes, + ) + elif isinstance(a, list): + # If a field is repeated, then both objects will represent + # the value as a list. + if len(a) != len(b): + return False + for a_item, b_item in zip(a, b): + if not _compare(a_item, b_item): + return False + else: + return True + else: + return type(a) is type(b) and a == b + + def _compare_fields(a, b): + if a._fields != b._fields: + return False + for field in a._fields: + a_field = getattr(a, field, sentinel) + b_field = getattr(b, field, sentinel) + if a_field is sentinel and b_field is sentinel: + # both nodes are missing a field at runtime + continue + if a_field is sentinel or b_field is sentinel: + # one of the node is missing a field + return False + if not _compare(a_field, b_field): + return False + else: + return True + + def _compare_attributes(a, b): + if a._attributes != b._attributes: + return False + # Attributes are always ints. + for attr in a._attributes: + a_attr = getattr(a, attr, sentinel) + b_attr = getattr(b, attr, sentinel) + if a_attr is sentinel and b_attr is sentinel: + # both nodes are missing an attribute at runtime + continue + if a_attr != b_attr: + return False + else: + return True + + if type(a) is not type(b): + return False + if not _compare_fields(a, b): + return False + if compare_attributes and not _compare_attributes(a, b): + return False + return True + + class NodeVisitor(object): """ A node visitor base class that walks the abstract syntax tree and calls a @@ -418,27 +524,6 @@ def generic_visit(self, node): elif isinstance(value, AST): self.visit(value) - def visit_Constant(self, node): - value = node.value - type_name = _const_node_type_names.get(type(value)) - if type_name is None: - for cls, name in _const_node_type_names.items(): - if isinstance(value, cls): - type_name = name - break - if type_name is not None: - method = 'visit_' + type_name - try: - visitor = getattr(self, method) - except AttributeError: - pass - else: - import warnings - warnings.warn(f"{method} is deprecated; add visit_Constant", - DeprecationWarning, 2) - return visitor(node) - return self.generic_visit(node) - class NodeTransformer(NodeVisitor): """ @@ -498,151 +583,6 @@ def generic_visit(self, node): setattr(node, field, new_node) return node - -_DEPRECATED_VALUE_ALIAS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; use value instead" -) -_DEPRECATED_CLASS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; " - "use ast.Constant instead" -) - - -# If the ast module is loaded more than once, only add deprecated methods once -if not hasattr(Constant, 'n'): - # The following code is for backward compatibility. - # It will be removed in future. - - def _n_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _n_setter(self, value): - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - def _s_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _s_setter(self, value): - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - Constant.n = property(_n_getter, _n_setter) - Constant.s = property(_s_getter, _s_setter) - -class _ABC(type): - - def __init__(cls, *args): - cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" - - def __instancecheck__(cls, inst): - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", - message=_DEPRECATED_CLASS_MESSAGE, - remove=(3, 14) - ) - if not isinstance(inst, Constant): - return False - if cls in _const_types: - try: - value = inst.value - except AttributeError: - return False - else: - return ( - isinstance(value, _const_types[cls]) and - not isinstance(value, _const_types_not.get(cls, ())) - ) - return type.__instancecheck__(cls, inst) - -def _new(cls, *args, **kwargs): - for key in kwargs: - if key not in cls._fields: - # arbitrary keyword arguments are accepted - continue - pos = cls._fields.index(key) - if pos < len(args): - raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(*args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -class Num(Constant, metaclass=_ABC): - _fields = ('n',) - __new__ = _new - -class Str(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class Bytes(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class NameConstant(Constant, metaclass=_ABC): - __new__ = _new - -class Ellipsis(Constant, metaclass=_ABC): - _fields = () - - def __new__(cls, *args, **kwargs): - if cls is _ast_Ellipsis: - import warnings - warnings._deprecated( - "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(..., *args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -# Keep another reference to Ellipsis in the global namespace -# so it can be referenced in Ellipsis.__new__ -# (The original "Ellipsis" name is removed from the global namespace later on) -_ast_Ellipsis = Ellipsis - -_const_types = { - Num: (int, float, complex), - Str: (str,), - Bytes: (bytes,), - NameConstant: (type(None), bool), - Ellipsis: (type(...),), -} -_const_types_not = { - Num: (bool,), -} - -_const_node_type_names = { - bool: 'NameConstant', # should be before int - type(None): 'NameConstant', - int: 'Num', - float: 'Num', - complex: 'Num', - str: 'Str', - bytes: 'Bytes', - type(...): 'Ellipsis', -} - class slice(AST): """Deprecated AST node class.""" @@ -1105,12 +1045,21 @@ def visit_TypeVar(self, node): if node.bound: self.write(": ") self.traverse(node.bound) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeVarTuple(self, node): self.write("*" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_ParamSpec(self, node): self.write("**" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeAlias(self, node): self.fill("type ") @@ -1269,14 +1218,18 @@ def visit_JoinedStr(self, node): quote_type = quote_types[0] self.write(f"{quote_type}{value}{quote_type}") - def _write_fstring_inner(self, node, escape_newlines=False): + def _write_fstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): # for both the f-string itself, and format_spec for value in node.values: - self._write_fstring_inner(value, escape_newlines=escape_newlines) + self._write_fstring_inner(value, is_format_spec=is_format_spec) elif isinstance(node, Constant) and isinstance(node.value, str): value = node.value.replace("{", "{{").replace("}", "}}") - if escape_newlines: + + if is_format_spec: + value = value.replace("\\", "\\\\") + value = value.replace("'", "\\'") + value = value.replace('"', '\\"') value = value.replace("\n", "\\n") self.write(value) elif isinstance(node, FormattedValue): @@ -1300,10 +1253,7 @@ def unparse_inner(inner): self.write(f"!{chr(node.conversion)}") if node.format_spec: self.write(":") - self._write_fstring_inner( - node.format_spec, - escape_newlines=True - ) + self._write_fstring_inner(node.format_spec, is_format_spec=True) def visit_Name(self, node): self.write(node.id) @@ -1784,31 +1734,16 @@ def visit_MatchOr(self, node): self.set_precedence(_Precedence.BOR.next(), *node.patterns) self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) + def unparse(ast_obj): unparser = _Unparser() return unparser.visit(ast_obj) -_deprecated_globals = { - name: globals().pop(name) - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') -} - -def __getattr__(name): - if name in _deprecated_globals: - globals()[name] = value = _deprecated_globals[name] - import warnings - warnings._deprecated( - f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return value - raise AttributeError(f"module 'ast' has no attribute '{name}'") - - def main(): import argparse - parser = argparse.ArgumentParser(prog='python -m ast') + parser = argparse.ArgumentParser() parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 18bb87a5bc4ffd..95c636f9e02866 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,41 +1,51 @@ import ast import asyncio -import code import concurrent.futures +import contextvars import inspect +import os +import site import sys import threading import types import warnings +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] +from _pyrepl.console import InteractiveColoredConsole + from . import futures -class AsyncIOInteractiveConsole(code.InteractiveConsole): +class AsyncIOInteractiveConsole(InteractiveColoredConsole): def __init__(self, locals, loop): - super().__init__(locals) + super().__init__(locals, filename="") self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT self.loop = loop + self.context = contextvars.copy_context() def runcode(self, code): + global return_code future = concurrent.futures.Future() def callback(): + global return_code global repl_future - global repl_future_interrupted + global keyboard_interrupted repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False func = types.FunctionType(code, self.locals) try: coro = func() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except KeyboardInterrupt as ex: - repl_future_interrupted = True + keyboard_interrupted = True future.set_exception(ex) return except BaseException as ex: @@ -47,19 +57,21 @@ def callback(): return try: - repl_future = self.loop.create_task(coro) + repl_future = self.loop.create_task(coro, context=self.context) futures._chain_future(repl_future, future) except BaseException as exc: future.set_exception(exc) - loop.call_soon_threadsafe(callback) + loop.call_soon_threadsafe(callback, context=self.context) try: return future.result() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except BaseException: - if repl_future_interrupted: + if keyboard_interrupted: self.write("\nKeyboardInterrupt\n") else: self.showtraceback() @@ -68,18 +80,47 @@ def callback(): class REPLThread(threading.Thread): def run(self): + global return_code + try: banner = ( f'asyncio REPL {sys.version} on {sys.platform}\n' f'Use "await" directly instead of "asyncio.run()".\n' f'Type "help", "copyright", "credits" or "license" ' f'for more information.\n' - f'{getattr(sys, "ps1", ">>> ")}import asyncio' ) - console.interact( - banner=banner, - exitmsg='exiting asyncio REPL...') + console.write(banner) + + if startup_path := os.getenv("PYTHONSTARTUP"): + sys.audit("cpython.run_startup", startup_path) + + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, console.locals) + + ps1 = getattr(sys, "ps1", ">>> ") + if can_colorize() and CAN_USE_PYREPL: + ps1 = f"{ANSIColors.BOLD_MAGENTA}{ps1}{ANSIColors.RESET}" + console.write(f"{ps1}import asyncio\n") + + if CAN_USE_PYREPL: + from _pyrepl.simple_interact import ( + run_multiline_interactive_console, + ) + try: + run_multiline_interactive_console(console) + except SystemExit: + # expected via the `exit` and `quit` commands + pass + except BaseException: + # unexpected issue + console.showtraceback() + console.write("Internal error, ") + return_code = 1 + else: + console.interact(banner="", exitmsg="") finally: warnings.filterwarnings( 'ignore', @@ -88,8 +129,25 @@ def run(self): loop.call_soon_threadsafe(loop.stop) + def interrupt(self) -> None: + if not CAN_USE_PYREPL: + return + + from _pyrepl.simple_interact import _get_reader + r = _get_reader() + if r.threading_hook is not None: + r.threading_hook.add("") # type: ignore + if __name__ == '__main__': + sys.audit("cpython.run_stdin") + + if os.getenv('PYTHON_BASIC_REPL'): + CAN_USE_PYREPL = False + else: + from _pyrepl.main import CAN_USE_PYREPL + + return_code = 0 loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -102,14 +160,31 @@ def run(self): console = AsyncIOInteractiveConsole(repl_locals, loop) repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False try: import readline # NoQA except ImportError: - pass + readline = None + + interactive_hook = getattr(sys, "__interactivehook__", None) - repl_thread = REPLThread() + if interactive_hook is not None: + sys.audit("cpython.run_interactivehook", interactive_hook) + interactive_hook() + + if interactive_hook is site.register_readline: + # Fix the completer function to use the interactive console locals + try: + import rlcompleter + except: + pass + else: + if readline is not None: + completer = rlcompleter.Completer(console.locals) + readline.set_completer(completer.complete) + + repl_thread = REPLThread(name="Interactive thread") repl_thread.daemon = True repl_thread.start() @@ -117,9 +192,13 @@ def run(self): try: loop.run_forever() except KeyboardInterrupt: + keyboard_interrupted = True if repl_future and not repl_future.done(): repl_future.cancel() - repl_future_interrupted = True + repl_thread.interrupt() continue else: break + + console.write('exiting asyncio REPL...\n') + sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index aadc4f478f8b56..5dbe4b28d236d3 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -17,7 +17,6 @@ import collections.abc import concurrent.futures import errno -import functools import heapq import itertools import os @@ -45,6 +44,7 @@ from . import sslproto from . import staggered from . import tasks +from . import timeouts from . import transports from . import trsock from .log import logger @@ -278,7 +278,9 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop self._sockets = sockets - self._active_count = 0 + # Weak references so we don't break Transport's ability to + # detect abandoned transports + self._clients = weakref.WeakSet() self._waiters = [] self._protocol_factory = protocol_factory self._backlog = backlog @@ -291,14 +293,13 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self): + def _attach(self, transport): assert self._sockets is not None - self._active_count += 1 + self._clients.add(transport) - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self._sockets is None: + def _detach(self, transport): + self._clients.discard(transport) + if len(self._clients) == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -347,9 +348,17 @@ def close(self): self._serving_forever_fut.cancel() self._serving_forever_fut = None - if self._active_count == 0: + if len(self._clients) == 0: self._wakeup() + def close_clients(self): + for transport in self._clients.copy(): + transport.close() + + def abort_clients(self): + for transport in self._clients.copy(): + transport.abort() + async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' @@ -598,23 +607,24 @@ async def shutdown_default_executor(self, timeout=None): thread = threading.Thread(target=self._do_shutdown, args=(future,)) thread.start() try: - await future - finally: - thread.join(timeout) - - if thread.is_alive(): + async with timeouts.timeout(timeout): + await future + except TimeoutError: warnings.warn("The executor did not finishing joining " - f"its threads within {timeout} seconds.", - RuntimeWarning, stacklevel=2) + f"its threads within {timeout} seconds.", + RuntimeWarning, stacklevel=2) self._default_executor.shutdown(wait=False) + else: + thread.join() def _do_shutdown(self, future): try: self._default_executor.shutdown(wait=True) if not self.is_closed(): - self.call_soon_threadsafe(future.set_result, None) + self.call_soon_threadsafe(futures._set_result_unless_cancelled, + future, None) except Exception as ex: - if not self.is_closed(): + if not self.is_closed() and not future.cancelled(): self.call_soon_threadsafe(future.set_exception, ex) def _check_running(self): @@ -826,7 +836,7 @@ def call_soon(self, callback, *args, context=None): def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError( f"coroutines cannot be used with {method}()") if not callable(callback): @@ -1017,8 +1027,7 @@ async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None): except OSError as exc: msg = ( f'error while attempting to bind on ' - f'address {laddr!r}: ' - f'{exc.strerror.lower()}' + f'address {laddr!r}: {str(exc).lower()}' ) exc = OSError(exc.errno, msg) my_exceptions.append(exc) @@ -1130,11 +1139,18 @@ async def create_connection( except OSError: continue else: # using happy eyeballs - sock, _, _ = await staggered.staggered_race( - (functools.partial(self._connect_sock, - exceptions, addrinfo, laddr_infos) - for addrinfo in infos), - happy_eyeballs_delay, loop=self) + sock = (await staggered.staggered_race( + ( + # can't use functools.partial as it keeps a reference + # to exceptions + lambda addrinfo=addrinfo: self._connect_sock( + exceptions, addrinfo, laddr_infos + ) + for addrinfo in infos + ), + happy_eyeballs_delay, + loop=self, + ))[0] # can't use sock, _, _ as it keeks a reference to exceptions if sock is None: exceptions = [exc for sub in exceptions for exc in sub] @@ -1588,7 +1604,7 @@ async def create_server( except OSError as err: msg = ('error while attempting ' 'to bind on address %r: %s' - % (sa, err.strerror.lower())) + % (sa, str(err).lower())) if err.errno == errno.EADDRNOTAVAIL: # Assume the family is not enabled (bpo-30945) sockets.pop() diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 6dbde2b696ad1f..9c2ba679ce2bf1 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -1,6 +1,9 @@ import collections import subprocess import warnings +import os +import signal +import sys from . import protocols from . import transports @@ -142,17 +145,31 @@ def _check_proc(self): if self._proc is None: raise ProcessLookupError() - def send_signal(self, signal): - self._check_proc() - self._proc.send_signal(signal) + if sys.platform == 'win32': + def send_signal(self, signal): + self._check_proc() + self._proc.send_signal(signal) + + def terminate(self): + self._check_proc() + self._proc.terminate() + + def kill(self): + self._check_proc() + self._proc.kill() + else: + def send_signal(self, signal): + self._check_proc() + try: + os.kill(self._proc.pid, signal) + except ProcessLookupError: + pass - def terminate(self): - self._check_proc() - self._proc.terminate() + def terminate(self): + self.send_signal(signal.SIGTERM) - def kill(self): - self._check_proc() - self._proc.kill() + def kill(self): + self.send_signal(signal.SIGKILL) async def _connect_pipes(self, waiter): try: diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index ab4f30eb51ba2c..a51319cb72a6a9 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -18,7 +18,16 @@ def _is_debug_mode(): def iscoroutinefunction(func): + import warnings """Return True if func is a decorated coroutine function.""" + warnings._deprecated("asyncio.iscoroutinefunction", + f"{warnings._DEPRECATED_MSG}; " + "use inspect.iscoroutinefunction() instead", + remove=(3,16)) + return _iscoroutinefunction(func) + + +def _iscoroutinefunction(func): return (inspect.iscoroutinefunction(func) or getattr(func, '_is_coroutine', None) is _is_coroutine) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 072a99fee123c3..b63fe6aa79604b 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -10,7 +10,6 @@ 'Handle', 'TimerHandle', 'get_event_loop_policy', 'set_event_loop_policy', 'get_event_loop', 'set_event_loop', 'new_event_loop', - 'get_child_watcher', 'set_child_watcher', '_set_running_loop', 'get_running_loop', '_get_running_loop', ) @@ -54,7 +53,8 @@ def _repr_info(self): info.append('cancelled') if self._callback is not None: info.append(format_helpers._format_callback_source( - self._callback, self._args)) + self._callback, self._args, + debug=self._loop.get_debug())) if self._source_traceback: frame = self._source_traceback[-1] info.append(f'created at {frame[0]}:{frame[1]}') @@ -90,7 +90,8 @@ def _run(self): raise except BaseException as exc: cb = format_helpers._format_callback_source( - self._callback, self._args) + self._callback, self._args, + debug=self._loop.get_debug()) msg = f'Exception in callback {cb}' context = { 'message': msg, @@ -173,6 +174,14 @@ def close(self): """Stop serving. This leaves existing connections open.""" raise NotImplementedError + def close_clients(self): + """Close all active connections.""" + raise NotImplementedError + + def abort_clients(self): + """Close all active connections immediately.""" + raise NotImplementedError + def get_loop(self): """Get the event loop the Server object is attached to.""" raise NotImplementedError @@ -642,17 +651,6 @@ def new_event_loop(self): the current context, set_event_loop must be called explicitly.""" raise NotImplementedError - # Child processes handling (Unix only). - - def get_child_watcher(self): - "Get the watcher for child processes." - raise NotImplementedError - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - raise NotImplementedError - - class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): """Default policy implementation for accessing the event loop. @@ -827,17 +825,6 @@ def new_event_loop(): return get_event_loop_policy().new_event_loop() -def get_child_watcher(): - """Equivalent to calling get_event_loop_policy().get_child_watcher().""" - return get_event_loop_policy().get_child_watcher() - - -def set_child_watcher(watcher): - """Equivalent to calling - get_event_loop_policy().set_child_watcher(watcher).""" - return get_event_loop_policy().set_child_watcher(watcher) - - # Alias pure-Python implementations for testing purposes. _py__get_running_loop = _get_running_loop _py__set_running_loop = _set_running_loop diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py index 27d11fd4fa9553..93737b7708a67b 100644 --- a/Lib/asyncio/format_helpers.py +++ b/Lib/asyncio/format_helpers.py @@ -19,19 +19,26 @@ def _get_function_source(func): return None -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) +def _format_callback_source(func, args, *, debug=False): + func_repr = _format_callback(func, args, None, debug=debug) source = _get_function_source(func) if source: func_repr += f' at {source[0]}:{source[1]}' return func_repr -def _format_args_and_kwargs(args, kwargs): +def _format_args_and_kwargs(args, kwargs, *, debug=False): """Format function arguments and keyword arguments. Special case for a single parameter: ('hello',) is formatted as ('hello'). + + Note that this function only returns argument details when + debug=True is specified, as arguments may contain sensitive + information. """ + if not debug: + return '()' + # use reprlib to limit the length of the output items = [] if args: @@ -41,10 +48,11 @@ def _format_args_and_kwargs(args, kwargs): return '({})'.format(', '.join(items)) -def _format_callback(func, args, kwargs, suffix=''): +def _format_callback(func, args, kwargs, *, debug=False, suffix=''): if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) + suffix = _format_args_and_kwargs(args, kwargs, debug=debug) + suffix + return _format_callback(func.func, func.args, func.keywords, + debug=debug, suffix=suffix) if hasattr(func, '__qualname__') and func.__qualname__: func_repr = func.__qualname__ @@ -53,7 +61,7 @@ def _format_callback(func, args, kwargs, suffix=''): else: func_repr = repr(func) - func_repr += _format_args_and_kwargs(args, kwargs) + func_repr += _format_args_and_kwargs(args, kwargs, debug=debug) if suffix: func_repr += suffix return func_repr diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 5d35321db7943b..c95fce035cd548 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -43,7 +43,6 @@ class Future: - This class is not compatible with the wait() and as_completed() methods in the concurrent.futures package. - (In Python 3.4 or later we may be able to unify the implementations.) """ # Class variables serving as defaults for instance variables. @@ -61,7 +60,7 @@ class Future: # the Future protocol (i.e. is intended to be duck-type compatible). # The value must also be not-None, to enable a subclass to declare # that it is not compatible by setting this to None. - # - It is set by __iter__() below so that Task._step() can tell + # - It is set by __iter__() below so that Task.__step() can tell # the difference between # `await Future()` or`yield from Future()` (correct) vs. # `yield Future()` (incorrect). @@ -191,8 +190,7 @@ def result(self): the future is done and has an exception set, this exception is raised. """ if self._state == _CANCELLED: - exc = self._make_cancelled_error() - raise exc + raise self._make_cancelled_error() if self._state != _FINISHED: raise exceptions.InvalidStateError('Result is not ready.') self.__log_traceback = False @@ -209,8 +207,7 @@ def exception(self): InvalidStateError. """ if self._state == _CANCELLED: - exc = self._make_cancelled_error() - raise exc + raise self._make_cancelled_error() if self._state != _FINISHED: raise exceptions.InvalidStateError('Exception is not set.') self.__log_traceback = False @@ -319,11 +316,9 @@ def _set_result_unless_cancelled(fut, result): def _convert_future_exc(exc): exc_class = type(exc) if exc_class is concurrent.futures.CancelledError: - return exceptions.CancelledError(*exc.args) - elif exc_class is concurrent.futures.TimeoutError: - return exceptions.TimeoutError(*exc.args) + return exceptions.CancelledError(*exc.args).with_traceback(exc.__traceback__) elif exc_class is concurrent.futures.InvalidStateError: - return exceptions.InvalidStateError(*exc.args) + return exceptions.InvalidStateError(*exc.args).with_traceback(exc.__traceback__) else: return exc diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index aaee8ff0702923..f2f8b7ec858096 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -341,9 +341,9 @@ def _notify(self, n): fut.set_result(False) def notify_all(self): - """Wake up all threads waiting on this condition. This method acts - like notify(), but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, + """Wake up all tasks waiting on this condition. This method acts + like notify(), but wakes up all waiting tasks instead of one. If the + calling task has not acquired the lock when this method is called, a RuntimeError is raised. """ self.notify(len(self._waiters)) diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 1e2a730cf368a9..7eb55bd63ddb73 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach() + self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -167,7 +167,7 @@ def _call_connection_lost(self, exc): self._sock = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None self._called_connection_lost = True @@ -487,9 +487,6 @@ def sendto(self, data, addr=None): raise TypeError('data argument must be bytes-like object (%r)', type(data)) - if not data: - return - if self._address is not None and addr not in (None, self._address): raise ValueError( f'Invalid address: must be None or {self._address}') @@ -502,7 +499,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + 8 # include header bytes if self._write_fut is None: # No current write operations are active, kick one off @@ -724,6 +721,8 @@ async def sock_sendto(self, sock, data, address): return await self._proactor.sendto(sock, data, 0, address) async def sock_connect(self, sock, address): + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") return await self._proactor.connect(sock, address) async def sock_accept(self, sock): diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index a9656a6df561ba..2f3865114a84f9 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -1,4 +1,11 @@ -__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty') +__all__ = ( + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'QueueFull', + 'QueueEmpty', + 'QueueShutDown', +) import collections import heapq @@ -18,6 +25,11 @@ class QueueFull(Exception): pass +class QueueShutDown(Exception): + """Raised when putting on to or getting from a shut-down Queue.""" + pass + + class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. @@ -41,6 +53,7 @@ def __init__(self, maxsize=0): self._finished = locks.Event() self._finished.set() self._init(maxsize) + self._is_shutdown = False # These three are overridable in subclasses. @@ -81,6 +94,8 @@ def _format(self): result += f' _putters[{len(self._putters)}]' if self._unfinished_tasks: result += f' tasks={self._unfinished_tasks}' + if self._is_shutdown: + result += ' shutdown' return result def qsize(self): @@ -112,8 +127,12 @@ async def put(self, item): Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. + + Raises QueueShutDown if the queue has been shut down. """ while self.full(): + if self._is_shutdown: + raise QueueShutDown putter = self._get_loop().create_future() self._putters.append(putter) try: @@ -125,7 +144,7 @@ async def put(self, item): self._putters.remove(putter) except ValueError: # The putter could be removed from self._putters by a - # previous get_nowait call. + # previous get_nowait call or a shutdown call. pass if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take @@ -138,7 +157,11 @@ def put_nowait(self, item): """Put an item into the queue without blocking. If no free slot is immediately available, raise QueueFull. + + Raises QueueShutDown if the queue has been shut down. """ + if self._is_shutdown: + raise QueueShutDown if self.full(): raise QueueFull self._put(item) @@ -150,8 +173,13 @@ async def get(self): """Remove and return an item from the queue. If queue is empty, wait until an item is available. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ while self.empty(): + if self._is_shutdown and self.empty(): + raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) try: @@ -163,7 +191,7 @@ async def get(self): self._getters.remove(getter) except ValueError: # The getter could be removed from self._getters by a - # previous put_nowait call. + # previous put_nowait call, or a shutdown call. pass if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take @@ -176,8 +204,13 @@ def get_nowait(self): """Remove and return an item from the queue. Return an item if one is immediately available, else raise QueueEmpty. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ if self.empty(): + if self._is_shutdown: + raise QueueShutDown raise QueueEmpty item = self._get() self._wakeup_next(self._putters) @@ -194,6 +227,9 @@ def task_done(self): been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises ValueError if called more times than there were items placed in the queue. """ @@ -214,6 +250,34 @@ async def join(self): if self._unfinished_tasks > 0: await self._finished.wait() + def shutdown(self, immediate=False): + """Shut-down the queue, making queue gets and puts raise QueueShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). + """ + self._is_shutdown = True + if immediate: + while not self.empty(): + self._get() + if self._unfinished_tasks > 0: + self._unfinished_tasks -= 1 + if self._unfinished_tasks == 0: + self._finished.set() + # All getters need to re-check queue-empty to raise ShutDown + while self._getters: + getter = self._getters.popleft() + if not getter.done(): + getter.set_result(None) + while self._putters: + putter = self._putters.popleft() + if not putter.done(): + putter.set_result(None) + class PriorityQueue(Queue): """A subclass of Queue; retrieves entries in priority order (lowest first). diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 1b89236599aad7..0e63c34f60f4d9 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -3,6 +3,7 @@ import contextvars import enum import functools +import inspect import threading import signal from . import coroutines @@ -84,10 +85,7 @@ def get_loop(self): return self._loop def run(self, coro, *, context=None): - """Run a coroutine inside the embedded event loop.""" - if not coroutines.iscoroutine(coro): - raise ValueError("a coroutine was expected, got {!r}".format(coro)) - + """Run code in the embedded event loop.""" if events._get_running_loop() is not None: # fail fast with short traceback raise RuntimeError( @@ -95,8 +93,19 @@ def run(self, coro, *, context=None): self._lazy_init() + if not coroutines.iscoroutine(coro): + if inspect.isawaitable(coro): + async def _wrap_awaitable(awaitable): + return await awaitable + + coro = _wrap_awaitable(coro) + else: + raise TypeError('An asyncio.Future, a coroutine or an ' + 'awaitable is required') + if context is None: context = self._context + task = self._loop.create_task(coro, context=context) if (threading.current_thread() is threading.main_thread() diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 10fbdd76e93f79..f94bf10b4225e7 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -791,7 +791,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._paused = False # Set when pause_reading() called if self._server is not None: - self._server._attach() + self._server._attach(self) loop._transports[self._sock_fd] = self def __repr__(self): @@ -868,6 +868,8 @@ def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() + if self._server is not None: + self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. @@ -906,7 +908,7 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None def get_write_buffer_size(self): @@ -1241,8 +1243,6 @@ def sendto(self, data, addr=None): if not isinstance(data, (bytes, bytearray, memoryview)): raise TypeError(f'data argument must be a bytes-like object, ' f'not {type(data).__name__!r}') - if not data: - return if self._address: if addr not in (None, self._address): @@ -1278,7 +1278,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + 8 # include header bytes self._maybe_pause_protocol() def _sendto_ready(self): diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index fa99d4533aa0a6..74c5f0d5ca0609 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -101,7 +101,7 @@ def get_protocol(self): return self._ssl_protocol._app_protocol def is_closing(self): - return self._closed + return self._closed or self._ssl_protocol._is_transport_closing() def close(self): """Close the transport. @@ -379,6 +379,9 @@ def _get_app_transport(self): self._app_transport_created = True return self._app_transport + def _is_transport_closing(self): + return self._transport is not None and self._transport.is_closing() + def connection_made(self, transport): """Called when the low-level connection is made. diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index e180cde0243b15..0f4df8855a80b9 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -69,8 +69,11 @@ async def staggered_race(coro_fns, delay, *, loop=None): exceptions = [] running_tasks = [] - async def run_one_coro( - previous_failed: typing.Optional[locks.Event]) -> None: + async def run_one_coro(ok_to_start, previous_failed) -> None: + # in eager tasks this waits for the calling task to append this task + # to running_tasks, in regular tasks this wait is a no-op that does + # not yield a future. See gh-124309. + await ok_to_start.wait() # Wait for the previous task to finish, or for delay seconds if previous_failed is not None: with contextlib.suppress(exceptions_mod.TimeoutError): @@ -86,8 +89,12 @@ async def run_one_coro( return # Start task that will run the next coroutine this_failed = locks.Event() - next_task = loop.create_task(run_one_coro(this_failed)) + next_ok_to_start = locks.Event() + next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) running_tasks.append(next_task) + # next_task has been appended to running_tasks so next_task is ok to + # start. + next_ok_to_start.set() assert len(running_tasks) == this_index + 2 # Prepare place to put this coroutine's exceptions if not won exceptions.append(None) @@ -117,8 +124,11 @@ async def run_one_coro( if i != this_index: t.cancel() - first_task = loop.create_task(run_one_coro(None)) + ok_to_start = locks.Event() + first_task = loop.create_task(run_one_coro(ok_to_start, None)) running_tasks.append(first_task) + # first_task has been appended to running_tasks so first_task is ok to start. + ok_to_start.set() try: # Wait for a growing list of tasks to all finish: poor man's version of # curio's TaskGroup or trio's nursery @@ -134,6 +144,7 @@ async def run_one_coro( raise d.exception() return winner_result, winner_index, exceptions finally: + del exceptions # Make sure no tasks are left running if we leave this function for t in running_tasks: t.cancel() diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index df58b7a799a5ad..64aac4cc50d15a 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -201,7 +201,6 @@ def __init__(self, stream_reader, client_connected_cb=None, loop=None): # is established. self._strong_reader = stream_reader self._reject_connection = False - self._stream_writer = None self._task = None self._transport = None self._client_connected_cb = client_connected_cb @@ -214,10 +213,8 @@ def _stream_reader(self): return None return self._stream_reader_wr() - def _replace_writer(self, writer): + def _replace_transport(self, transport): loop = self._loop - transport = writer.transport - self._stream_writer = writer self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None @@ -239,11 +236,8 @@ def connection_made(self, transport): reader.set_transport(transport) self._over_ssl = transport.get_extra_info('sslcontext') is not None if self._client_connected_cb is not None: - self._stream_writer = StreamWriter(transport, self, - reader, - self._loop) - res = self._client_connected_cb(reader, - self._stream_writer) + writer = StreamWriter(transport, self, reader, self._loop) + res = self._client_connected_cb(reader, writer) if coroutines.iscoroutine(res): def callback(task): if task.cancelled(): @@ -405,7 +399,7 @@ async def start_tls(self, sslcontext, *, ssl_handshake_timeout=ssl_handshake_timeout, ssl_shutdown_timeout=ssl_shutdown_timeout) self._transport = new_transport - protocol._replace_writer(self) + protocol._replace_transport(new_transport) def __del__(self, warnings=warnings): if not self._transport.is_closing(): @@ -596,20 +590,34 @@ async def readuntil(self, separator=b'\n'): If the data cannot be read because of over limit, a LimitOverrunError exception will be raised, and the data will be left in the internal buffer, so it can be read again. + + The ``separator`` may also be a tuple of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of LimitOverrunError, + the shortest possible separator is considered to be the one that + matched. """ - seplen = len(separator) - if seplen == 0: + if isinstance(separator, tuple): + # Makes sure shortest matches wins + separator = sorted(separator, key=len) + else: + separator = [separator] + if not separator: + raise ValueError('Separator should contain at least one element') + min_seplen = len(separator[0]) + max_seplen = len(separator[-1]) + if min_seplen == 0: raise ValueError('Separator should be at least one-byte string') if self._exception is not None: raise self._exception # Consume whole buffer except last bytes, which length is - # one less than seplen. Let's check corner cases with - # separator='SEPARATOR': + # one less than max_seplen. Let's check corner cases with + # separator[-1]='SEPARATOR': # * we have received almost complete separator (without last # byte). i.e buffer='some textSEPARATO'. In this case we - # can safely consume len(separator) - 1 bytes. + # can safely consume max_seplen - 1 bytes. # * last byte of buffer is first byte of separator, i.e. # buffer='abcdefghijklmnopqrS'. We may safely consume # everything except that last byte, but this require to @@ -622,26 +630,35 @@ async def readuntil(self, separator=b'\n'): # messages :) # `offset` is the number of bytes from the beginning of the buffer - # where there is no occurrence of `separator`. + # where there is no occurrence of any `separator`. offset = 0 - # Loop until we find `separator` in the buffer, exceed the buffer size, + # Loop until we find a `separator` in the buffer, exceed the buffer size, # or an EOF has happened. while True: buflen = len(self._buffer) - # Check if we now have enough data in the buffer for `separator` to - # fit. - if buflen - offset >= seplen: - isep = self._buffer.find(separator, offset) - - if isep != -1: - # `separator` is in the buffer. `isep` will be used later - # to retrieve the data. + # Check if we now have enough data in the buffer for shortest + # separator to fit. + if buflen - offset >= min_seplen: + match_start = None + match_end = None + for sep in separator: + isep = self._buffer.find(sep, offset) + + if isep != -1: + # `separator` is in the buffer. `match_start` and + # `match_end` will be used later to retrieve the + # data. + end = isep + len(sep) + if match_end is None or end < match_end: + match_end = end + match_start = isep + if match_end is not None: break # see upper comment for explanation. - offset = buflen + 1 - seplen + offset = max(0, buflen + 1 - max_seplen) if offset > self._limit: raise exceptions.LimitOverrunError( 'Separator is not found, and chunk exceed the limit', @@ -650,7 +667,7 @@ async def readuntil(self, separator=b'\n'): # Complete message (with full separator) may be present in buffer # even when EOF flag is set. This may happen when the last chunk # adds data which makes separator be found. That's why we check for - # EOF *ater* inspecting the buffer. + # EOF *after* inspecting the buffer. if self._eof: chunk = bytes(self._buffer) self._buffer.clear() @@ -659,12 +676,12 @@ async def readuntil(self, separator=b'\n'): # _wait_for_data() will resume reading if stream was paused. await self._wait_for_data('readuntil') - if isep > self._limit: + if match_start > self._limit: raise exceptions.LimitOverrunError( - 'Separator is found, but chunk is longer than limit', isep) + 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:isep + seplen] - del self._buffer[:isep + seplen] + chunk = self._buffer[:match_end] + del self._buffer[:match_end] self._maybe_resume_transport() return bytes(chunk) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index f322b1f6653f6a..9fa772ca9d02cc 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -66,6 +66,20 @@ async def __aenter__(self): return self async def __aexit__(self, et, exc, tb): + tb = None + try: + return await self._aexit(et, exc) + finally: + # Exceptions are heavy objects that can have object + # cycles (bad for GC); let's not keep a reference to + # a bunch of them. It would be nicer to use a try/finally + # in __aexit__ directly but that introduced some diff noise + self._parent_task = None + self._errors = None + self._base_error = None + exc = None + + async def _aexit(self, et, exc): self._exiting = True if (exc is not None and @@ -77,12 +91,6 @@ async def __aexit__(self, et, exc, tb): propagate_cancellation_error = exc else: propagate_cancellation_error = None - if self._parent_cancel_requested: - # If this flag is set we *must* call uncancel(). - if self._parent_task.uncancel() == 0: - # If there are no pending cancellations left, - # don't propagate CancelledError. - propagate_cancellation_error = None if et is not None: if not self._aborting: @@ -128,25 +136,47 @@ async def __aexit__(self, et, exc, tb): assert not self._tasks if self._base_error is not None: - raise self._base_error + try: + raise self._base_error + finally: + exc = None + + if self._parent_cancel_requested: + # If this flag is set we *must* call uncancel(). + if self._parent_task.uncancel() == 0: + # If there are no pending cancellations left, + # don't propagate CancelledError. + propagate_cancellation_error = None # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. - if propagate_cancellation_error is not None and not self._errors: - raise propagate_cancellation_error + try: + if propagate_cancellation_error is not None and not self._errors: + try: + raise propagate_cancellation_error + finally: + exc = None + finally: + propagate_cancellation_error = None if et is not None and not issubclass(et, exceptions.CancelledError): self._errors.append(exc) if self._errors: - # Exceptions are heavy objects that can have object - # cycles (bad for GC); let's not keep a reference to - # a bunch of them. + # If the parent task is being cancelled from the outside + # of the taskgroup, un-cancel and re-cancel the parent task, + # which will keep the cancel count stable. + if self._parent_task.cancelling(): + self._parent_task.uncancel() + self._parent_task.cancel() try: - me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors) - raise me from None + raise BaseExceptionGroup( + 'unhandled errors in a TaskGroup', + self._errors, + ) from None finally: - self._errors = None + exc = None + def create_task(self, coro, *, name=None, context=None): """Create a new task in this group and return it. @@ -154,10 +184,13 @@ def create_task(self, coro, *, name=None, context=None): Similar to `asyncio.create_task`. """ if not self._entered: + coro.close() raise RuntimeError(f"TaskGroup {self!r} has not been entered") if self._exiting and not self._tasks: + coro.close() raise RuntimeError(f"TaskGroup {self!r} is finished") if self._aborting: + coro.close() raise RuntimeError(f"TaskGroup {self!r} is shutting down") if context is None: task = self._loop.create_task(coro, name=name) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index fafee3e738f6aa..2112dd4b99d17f 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -25,6 +25,7 @@ from . import events from . import exceptions from . import futures +from . import queues from . import timeouts # Helper to generate new task names @@ -47,23 +48,8 @@ def all_tasks(loop=None): # capturing the set of eager tasks first, so if an eager task "graduates" # to a regular task in another thread, we don't risk missing it. eager_tasks = list(_eager_tasks) - # Looping over the WeakSet isn't safe as it can be updated from another - # thread, therefore we cast it to list prior to filtering. The list cast - # itself requires iteration, so we repeat it several times ignoring - # RuntimeErrors (which are not very likely to occur). - # See issues 34970 and 36607 for details. - scheduled_tasks = None - i = 0 - while True: - try: - scheduled_tasks = list(_scheduled_tasks) - except RuntimeError: - i += 1 - if i >= 1000: - raise - else: - break - return {t for t in itertools.chain(scheduled_tasks, eager_tasks) + + return {t for t in itertools.chain(_scheduled_tasks, eager_tasks) if futures._get_loop(t) is loop and not t.done()} @@ -254,6 +240,8 @@ def uncancel(self): """ if self._num_cancels_requested > 0: self._num_cancels_requested -= 1 + if self._num_cancels_requested == 0: + self._must_cancel = False return self._num_cancels_requested def __eager_start(self): @@ -278,7 +266,7 @@ def __eager_start(self): def __step(self, exc=None): if self.done(): raise exceptions.InvalidStateError( - f'_step(): already done: {self!r}, {exc!r}') + f'__step(): already done: {self!r}, {exc!r}') if self._must_cancel: if not isinstance(exc, exceptions.CancelledError): exc = self._make_cancelled_error() @@ -376,7 +364,7 @@ def __wakeup(self, future): else: # Don't pass the value of `future.result()` explicitly, # as `Future.__iter__` and `Future.__await__` don't need it. - # If we call `_step(value, None)` instead of `_step()`, + # If we call `__step(value, None)` instead of `__step()`, # Python eval loop would use `.send(value)` method call, # instead of `__next__()`, which is slower for futures # that return non-generator iterators from their `__iter__`. @@ -464,7 +452,7 @@ async def wait_for(fut, timeout): If the wait is cancelled, the task is also cancelled. - If the task supresses the cancellation and returns a value instead, + If the task suppresses the cancellation and returns a value instead, that value is returned. This function is a coroutine. @@ -564,62 +552,125 @@ async def _cancel_and_wait(fut): fut.remove_done_callback(cb) -# This is *not* a @coroutine! It is just an iterator (yielding Futures). +class _AsCompletedIterator: + """Iterator of awaitables representing tasks of asyncio.as_completed. + + As an asynchronous iterator, iteration yields futures as they finish. As a + plain iterator, new coroutines are yielded that will return or raise the + result of the next underlying future to complete. + """ + def __init__(self, aws, timeout): + self._done = queues.Queue() + self._timeout_handle = None + + loop = events.get_event_loop() + todo = {ensure_future(aw, loop=loop) for aw in set(aws)} + for f in todo: + f.add_done_callback(self._handle_completion) + if todo and timeout is not None: + self._timeout_handle = ( + loop.call_later(timeout, self._handle_timeout) + ) + self._todo = todo + self._todo_left = len(todo) + + def __aiter__(self): + return self + + def __iter__(self): + return self + + async def __anext__(self): + if not self._todo_left: + raise StopAsyncIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return await self._wait_for_one() + + def __next__(self): + if not self._todo_left: + raise StopIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return self._wait_for_one(resolve=True) + + def _handle_timeout(self): + for f in self._todo: + f.remove_done_callback(self._handle_completion) + self._done.put_nowait(None) # Sentinel for _wait_for_one(). + self._todo.clear() # Can't do todo.remove(f) in the loop. + + def _handle_completion(self, f): + if not self._todo: + return # _handle_timeout() was here first. + self._todo.remove(f) + self._done.put_nowait(f) + if not self._todo and self._timeout_handle is not None: + self._timeout_handle.cancel() + + async def _wait_for_one(self, resolve=False): + # Wait for the next future to be done and return it unless resolve is + # set, in which case return either the result of the future or raise + # an exception. + f = await self._done.get() + if f is None: + # Dummy value from _handle_timeout(). + raise exceptions.TimeoutError + return f.result() if resolve else f + + def as_completed(fs, *, timeout=None): - """Return an iterator whose values are coroutines. + """Create an iterator of awaitables or their results in completion order. - When waiting for the yielded coroutines you'll get the results (or - exceptions!) of the original Futures (or coroutines), in the order - in which and as soon as they complete. + Run the supplied awaitables concurrently. The returned object can be + iterated to obtain the results of the awaitables as they finish. - This differs from PEP 3148; the proper way to use this is: + The object returned can be iterated as an asynchronous iterator or a plain + iterator. When asynchronous iteration is used, the originally-supplied + awaitables are yielded if they are tasks or futures. This makes it easy to + correlate previously-scheduled tasks with their results: - for f in as_completed(fs): - result = await f # The 'await' may raise. - # Use result. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - If a timeout is specified, the 'await' will raise - TimeoutError when the timeout occurs before all Futures are done. + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect - Note: The futures 'f' are not necessarily members of fs. - """ - if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError(f"expect an iterable of futures, not {type(fs).__name__}") + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") - from .queues import Queue # Import here to avoid circular import problem. - done = Queue() + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. - loop = events.get_event_loop() - todo = {ensure_future(f, loop=loop) for f in set(fs)} - timeout_handle = None + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13: - def _on_timeout(): - for f in todo: - f.remove_done_callback(_on_completion) - done.put_nowait(None) # Queue a dummy value for _wait_for_one(). - todo.clear() # Can't do todo.remove(f) in the loop. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - def _on_completion(f): - if not todo: - return # _on_timeout() was here first. - todo.remove(f) - done.put_nowait(f) - if not todo and timeout_handle is not None: - timeout_handle.cancel() + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect - async def _wait_for_one(): - f = await done.get() - if f is None: - # Dummy value from _on_timeout(). - raise exceptions.TimeoutError - return f.result() # May raise f.exception(). + A TimeoutError is raised if the timeout occurs before all awaitables are + done. This is raised by the async for loop during asynchronous iteration or + by the coroutines yielded during plain iteration. + """ + if inspect.isawaitable(fs): + raise TypeError( + f"expects an iterable of awaitables, not {type(fs).__name__}" + ) - for f in todo: - f.add_done_callback(_on_completion) - if todo and timeout is not None: - timeout_handle = loop.call_later(timeout, _on_timeout) - for _ in range(len(todo)): - yield _wait_for_one() + return _AsCompletedIterator(fs, timeout) @types.coroutine @@ -1031,14 +1082,14 @@ def _unregister_eager_task(task): _py_enter_task = _enter_task _py_leave_task = _leave_task _py_swap_current_task = _swap_current_task - +_py_all_tasks = all_tasks try: from _asyncio import (_register_task, _register_eager_task, _unregister_task, _unregister_eager_task, _enter_task, _leave_task, _swap_current_task, _scheduled_tasks, _eager_tasks, _current_tasks, - current_task) + current_task, all_tasks) except ImportError: pass else: @@ -1050,3 +1101,4 @@ def _unregister_eager_task(task): _c_enter_task = _enter_task _c_leave_task = _leave_task _c_swap_current_task = _swap_current_task + _c_all_tasks = all_tasks diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py index 30fd41d49af71f..34c7ad44ffd8ab 100644 --- a/Lib/asyncio/transports.py +++ b/Lib/asyncio/transports.py @@ -181,6 +181,8 @@ def sendto(self, data, addr=None): to be sent out asynchronously. addr is target socket address. If addr is None use target address pointed on transport creation. + If data is an empty bytes object a zero-length datagram will be + sent. """ raise NotImplementedError diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 41ccf1b78fb93b..0227eb506c6016 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,9 +28,6 @@ __all__ = ( 'SelectorEventLoop', - 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'PidfdChildWatcher', - 'MultiLoopChildWatcher', 'ThreadedChildWatcher', 'DefaultEventLoopPolicy', 'EventLoop', ) @@ -65,6 +62,10 @@ def __init__(self, selector=None): super().__init__(selector) self._signal_handlers = {} self._unix_server_sockets = {} + if can_use_pidfd(): + self._watcher = _PidfdChildWatcher() + else: + self._watcher = _ThreadedChildWatcher() def close(self): super().close() @@ -94,7 +95,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -197,33 +198,22 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, async def _make_subprocess_transport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, extra=None, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = events.get_child_watcher() - - with watcher: - if not watcher.is_active(): - # Check early. - # Raising exception before process creation - # prevents subprocess execution if the watcher - # is not ready to handle it. - raise RuntimeError("asyncio.get_child_watcher() is not activated, " - "subprocess support is not installed.") - waiter = self.create_future() - transp = _UnixSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - watcher.add_child_handler(transp.get_pid(), - self._child_watcher_callback, transp) - try: - await waiter - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - transp.close() - await transp._wait() - raise + watcher = self._watcher + waiter = self.create_future() + transp = _UnixSubprocessTransport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) + watcher.add_child_handler(transp.get_pid(), + self._child_watcher_callback, transp) + try: + await waiter + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: + transp.close() + await transp._wait() + raise return transp @@ -865,93 +855,7 @@ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w.close() -class AbstractChildWatcher: - """Abstract base class for monitoring child processes. - - Objects derived from this class monitor a collection of subprocesses and - report their termination or interruption by a signal. - - New callbacks are registered with .add_child_handler(). Starting a new - process must be done within a 'with' block to allow the watcher to suspend - its activity until the new process if fully registered (this is needed to - prevent a race condition in some implementations). - - Example: - with watcher: - proc = subprocess.Popen("sleep 1") - watcher.add_child_handler(proc.pid, callback) - - Notes: - Implementations of this class must be thread-safe. - - Since child watcher objects may catch the SIGCHLD signal and call - waitpid(-1), there should be only one active object per process. - """ - - def __init_subclass__(cls) -> None: - if cls.__module__ != __name__: - warnings._deprecated("AbstractChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def add_child_handler(self, pid, callback, *args): - """Register a new child handler. - - Arrange for callback(pid, returncode, *args) to be called when - process 'pid' terminates. Specifying another callback for the same - process replaces the previous handler. - - Note: callback() must be thread-safe. - """ - raise NotImplementedError() - - def remove_child_handler(self, pid): - """Removes the handler for process 'pid'. - - The function returns True if the handler was successfully removed, - False if there was nothing to remove.""" - - raise NotImplementedError() - - def attach_loop(self, loop): - """Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then it is - first detached before attaching to the new loop. - - Note: loop may be None. - """ - raise NotImplementedError() - - def close(self): - """Close the watcher. - - This must be called to make sure that any underlying resource is freed. - """ - raise NotImplementedError() - - def is_active(self): - """Return ``True`` if the watcher is active and is used by the event loop. - - Return True if the watcher is installed and ready to handle process exit - notifications. - - """ - raise NotImplementedError() - - def __enter__(self): - """Enter the watcher's context and allow starting new processes - - This function must return self""" - raise NotImplementedError() - - def __exit__(self, a, b, c): - """Exit the watcher's context""" - raise NotImplementedError() - - -class PidfdChildWatcher(AbstractChildWatcher): +class _PidfdChildWatcher: """Child watcher implementation using Linux's pid file descriptors. This child watcher polls process file descriptors (pidfds) to await child @@ -963,21 +867,6 @@ class PidfdChildWatcher(AbstractChildWatcher): recent (5.3+) kernels. """ - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_traceback): - pass - - def is_active(self): - return True - - def close(self): - pass - - def attach_loop(self, loop): - pass - def add_child_handler(self, pid, callback, *args): loop = events.get_running_loop() pidfd = os.pidfd_open(pid) @@ -1002,386 +891,7 @@ def _do_wait(self, pid, pidfd, callback, args): os.close(pidfd) callback(pid, returncode, *args) - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - -class BaseChildWatcher(AbstractChildWatcher): - - def __init__(self): - self._loop = None - self._callbacks = {} - - def close(self): - self.attach_loop(None) - - def is_active(self): - return self._loop is not None and self._loop.is_running() - - def _do_waitpid(self, expected_pid): - raise NotImplementedError() - - def _do_waitpid_all(self): - raise NotImplementedError() - - def attach_loop(self, loop): - assert loop is None or isinstance(loop, events.AbstractEventLoop) - - if self._loop is not None and loop is None and self._callbacks: - warnings.warn( - 'A loop is being detached ' - 'from a child watcher with pending handlers', - RuntimeWarning) - - if self._loop is not None: - self._loop.remove_signal_handler(signal.SIGCHLD) - - self._loop = loop - if loop is not None: - loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) - - # Prevent a race condition in case a child terminated - # during the switch. - self._do_waitpid_all() - - def _sig_chld(self): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException as exc: - # self._loop should always be available here - # as '_sig_chld' is added as a signal handler - # in 'attach_loop' - self._loop.call_exception_handler({ - 'message': 'Unknown exception in SIGCHLD handler', - 'exception': exc, - }) - - -class SafeChildWatcher(BaseChildWatcher): - """'Safe' child watcher implementation. - - This implementation avoids disrupting other code spawning processes by - polling explicitly each process in the SIGCHLD handler instead of calling - os.waitpid(-1). - - This is a safe solution but it has a significant overhead when handling a - big number of children (O(n) each time SIGCHLD is raised) - """ - - def __init__(self): - super().__init__() - warnings._deprecated("SafeChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - super().close() - - def __enter__(self): - return self - - def __exit__(self, a, b, c): - pass - - def add_child_handler(self, pid, callback, *args): - self._callbacks[pid] = (callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - - try: - callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - if self._loop.get_debug(): - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - callback(pid, returncode, *args) - - -class FastChildWatcher(BaseChildWatcher): - """'Fast' child watcher implementation. - - This implementation reaps every terminated processes by calling - os.waitpid(-1) directly, possibly breaking other code spawning processes - and waiting for their termination. - - There is no noticeable overhead when handling a big number of children - (O(1) each time a child terminates). - """ - def __init__(self): - super().__init__() - self._lock = threading.Lock() - self._zombies = {} - self._forks = 0 - warnings._deprecated("FastChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - self._zombies.clear() - super().close() - - def __enter__(self): - with self._lock: - self._forks += 1 - - return self - - def __exit__(self, a, b, c): - with self._lock: - self._forks -= 1 - - if self._forks or not self._zombies: - return - - collateral_victims = str(self._zombies) - self._zombies.clear() - - logger.warning( - "Caught subprocesses termination from unknown pids: %s", - collateral_victims) - - def add_child_handler(self, pid, callback, *args): - assert self._forks, "Must use the context manager" - - with self._lock: - try: - returncode = self._zombies.pop(pid) - except KeyError: - # The child is running. - self._callbacks[pid] = callback, args - return - - # The child is dead already. We can fire the callback. - callback(pid, returncode, *args) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - # Because of signal coalescing, we must keep calling waitpid() as - # long as we're able to reap a child. - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG) - except ChildProcessError: - # No more child processes exist. - return - else: - if pid == 0: - # A child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - - with self._lock: - try: - callback, args = self._callbacks.pop(pid) - except KeyError: - # unknown child - if self._forks: - # It may not be registered yet. - self._zombies[pid] = returncode - if self._loop.get_debug(): - logger.debug('unknown process %s exited ' - 'with returncode %s', - pid, returncode) - continue - callback = None - else: - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - pid, returncode) - - if callback is None: - logger.warning( - "Caught subprocess termination from unknown pid: " - "%d -> %d", pid, returncode) - else: - callback(pid, returncode, *args) - - -class MultiLoopChildWatcher(AbstractChildWatcher): - """A watcher that doesn't require running loop in the main thread. - - This implementation registers a SIGCHLD signal handler on - instantiation (which may conflict with other code that - install own handler for this signal). - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O(n)* each time a - SIGCHLD is received). - """ - - # Implementation note: - # The class keeps compatibility with AbstractChildWatcher ABC - # To achieve this it has empty attach_loop() method - # and doesn't accept explicit loop argument - # for add_child_handler()/remove_child_handler() - # but retrieves the current loop by get_running_loop() - - def __init__(self): - self._callbacks = {} - self._saved_sighandler = None - warnings._deprecated("MultiLoopChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def is_active(self): - return self._saved_sighandler is not None - - def close(self): - self._callbacks.clear() - if self._saved_sighandler is None: - return - - handler = signal.getsignal(signal.SIGCHLD) - if handler != self._sig_chld: - logger.warning("SIGCHLD handler was changed by outside code") - else: - signal.signal(signal.SIGCHLD, self._saved_sighandler) - self._saved_sighandler = None - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def add_child_handler(self, pid, callback, *args): - loop = events.get_running_loop() - self._callbacks[pid] = (loop, callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def attach_loop(self, loop): - # Don't save the loop but initialize itself if called first time - # The reason to do it here is that attach_loop() is called from - # unix policy only for the main thread. - # Main thread is required for subscription on SIGCHLD signal - if self._saved_sighandler is not None: - return - - self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) - if self._saved_sighandler is None: - logger.warning("Previous SIGCHLD handler was set by non-Python code, " - "restore to default handler on watcher close.") - self._saved_sighandler = signal.SIG_DFL - - # Set SA_RESTART to limit EINTR occurrences. - signal.siginterrupt(signal.SIGCHLD, False) - - def _do_waitpid_all(self): - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - debug_log = False - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - debug_log = True - try: - loop, callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - if loop.is_closed(): - logger.warning("Loop %r that handles pid %r is closed", loop, pid) - else: - if debug_log and loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - loop.call_soon_threadsafe(callback, pid, returncode, *args) - - def _sig_chld(self, signum, frame): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - logger.warning('Unknown exception in SIGCHLD handler', exc_info=True) - - -class ThreadedChildWatcher(AbstractChildWatcher): +class _ThreadedChildWatcher: """Threaded child watcher implementation. The watcher uses a thread per process @@ -1398,18 +908,6 @@ def __init__(self): self._pid_counter = itertools.count(0) self._threads = {} - def is_active(self): - return True - - def close(self): - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - def __del__(self, _warn=warnings.warn): threads = [thread for thread in list(self._threads.values()) if thread.is_alive()] @@ -1427,15 +925,6 @@ def add_child_handler(self, pid, callback, *args): self._threads[pid] = thread thread.start() - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - def attach_loop(self, loop): - pass - def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 @@ -1475,61 +964,9 @@ def can_use_pidfd(): class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): - """UNIX event loop policy with a watcher for child processes.""" + """UNIX event loop policy""" _loop_factory = _UnixSelectorEventLoop - def __init__(self): - super().__init__() - self._watcher = None - - def _init_watcher(self): - with events._lock: - if self._watcher is None: # pragma: no branch - if can_use_pidfd(): - self._watcher = PidfdChildWatcher() - else: - self._watcher = ThreadedChildWatcher() - - def set_event_loop(self, loop): - """Set the event loop. - - As a side effect, if a child watcher was set before, then calling - .set_event_loop() from the main thread will call .attach_loop(loop) on - the child watcher. - """ - - super().set_event_loop(loop) - - if (self._watcher is not None and - threading.current_thread() is threading.main_thread()): - self._watcher.attach_loop(loop) - - def get_child_watcher(self): - """Get the watcher for child processes. - - If not yet set, a ThreadedChildWatcher object is automatically created. - """ - if self._watcher is None: - self._init_watcher() - - warnings._deprecated("get_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - return self._watcher - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - - assert watcher is None or isinstance(watcher, AbstractChildWatcher) - - if self._watcher is not None: - self._watcher.close() - - self._watcher = watcher - warnings._deprecated("set_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - SelectorEventLoop = _UnixSelectorEventLoop DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index b62ea75fee3858..bf99bc271c7acd 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -8,6 +8,7 @@ import _overlapped import _winapi import errno +from functools import partial import math import msvcrt import socket @@ -324,13 +325,13 @@ def _run_forever_cleanup(self): if self._self_reading_future is not None: ov = self._self_reading_future._ov self._self_reading_future.cancel() - # self_reading_future was just cancelled so if it hasn't been - # finished yet, it never will be (it's possible that it has - # already finished and its callback is waiting in the queue, - # where it could still happen if the event loop is restarted). - # Unregister it otherwise IocpProactor.close will wait for it - # forever - if ov is not None: + # self_reading_future always uses IOCP, so even though it's + # been cancelled, we need to make sure that the IOCP message + # is received so that the kernel is not holding on to the + # memory, possibly causing memory corruption later. Only + # unregister it if IO is complete in all respects. Otherwise + # we need another _poll() later to complete the IO. + if ov is not None and not ov.pending: self._proactor._unregister(ov) self._self_reading_future = None @@ -467,6 +468,18 @@ def finish_socket_func(trans, key, ov): else: raise + @classmethod + def _finish_recvfrom(cls, trans, key, ov, *, empty_result): + try: + return cls.finish_socket_func(trans, key, ov) + except OSError as exc: + # WSARecvFrom will report ERROR_PORT_UNREACHABLE when the same + # socket is used to send to an address that is not listening. + if exc.winerror == _overlapped.ERROR_PORT_UNREACHABLE: + return empty_result, None + else: + raise + def recv(self, conn, nbytes, flags=0): self._register_with_iocp(conn) ov = _overlapped.Overlapped(NULL) @@ -501,7 +514,8 @@ def recvfrom(self, conn, nbytes, flags=0): except BrokenPipeError: return self._result((b'', None)) - return self._register(ov, conn, self.finish_socket_func) + return self._register(ov, conn, partial(self._finish_recvfrom, + empty_result=b'')) def recvfrom_into(self, conn, buf, flags=0): self._register_with_iocp(conn) @@ -511,17 +525,8 @@ def recvfrom_into(self, conn, buf, flags=0): except BrokenPipeError: return self._result((0, None)) - def finish_recv(trans, key, ov): - try: - return ov.getresult() - except OSError as exc: - if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED, - _overlapped.ERROR_OPERATION_ABORTED): - raise ConnectionResetError(*exc.args) - else: - raise - - return self._register(ov, conn, finish_recv) + return self._register(ov, conn, partial(self._finish_recvfrom, + empty_result=0)) def sendto(self, conn, buf, flags=0, addr=None): self._register_with_iocp(conn) diff --git a/Lib/base64.py b/Lib/base64.py old mode 100755 new mode 100644 index e3e983b3064fe7..61be4fb856e92c --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module @@ -18,7 +16,7 @@ 'b64encode', 'b64decode', 'b32encode', 'b32decode', 'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode', # Base85 and Ascii85 encodings - 'b85encode', 'b85decode', 'a85encode', 'a85decode', + 'b85encode', 'b85decode', 'a85encode', 'a85decode', 'z85encode', 'z85decode', # Standard Base64 encoding 'standard_b64encode', 'standard_b64decode', # Some common Base64 alternatives. As referenced by RFC 3458, see thread @@ -332,7 +330,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): wrapcol controls whether the output should have newline (b'\\n') characters added to it. If this is non-zero, each output line will be at most this - many characters long. + many characters long, excluding the trailing newline. pad controls whether the input is padded to a multiple of 4 before encoding. Note that the btoa implementation always pads. @@ -497,6 +495,33 @@ def b85decode(b): result = result[:-padding] return result +_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') +# Translating b85 valid but z85 invalid chars to b'\x00' is required +# to prevent them from being decoded as b85 valid chars. +_z85_b85_decode_diff = b';_`|~' +_z85_decode_translation = bytes.maketrans( + _z85alphabet + _z85_b85_decode_diff, + _b85alphabet + b'\x00' * len(_z85_b85_decode_diff) +) +_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet) + +def z85encode(s): + """Encode bytes-like object b in z85 format and return a bytes object.""" + return b85encode(s).translate(_z85_encode_translation) + +def z85decode(s): + """Decode the z85-encoded bytes-like object or ASCII string b + + The result is returned as a bytes object. + """ + s = _bytes_from_decode_data(s) + s = s.translate(_z85_decode_translation) + try: + return b85decode(s) + except ValueError as e: + raise ValueError(e.args[0].replace('base85', 'z85')) from None + # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it # though. The files should be opened in binary mode. diff --git a/Lib/bdb.py b/Lib/bdb.py index 1acf7957f0d669..81bba8a130f97c 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -3,6 +3,7 @@ import fnmatch import sys import os +import weakref from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -32,8 +33,11 @@ def __init__(self, skip=None): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} - self.frame_trace_lines = {} + self.frame_trace_lines_opcodes = {} self.frame_returning = None + self.trace_opcodes = False + self.enterframe = None + self.code_linenos = weakref.WeakKeyDictionary() self._load_breaks() @@ -85,6 +89,9 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ + + self.enterframe = frame + if self.quitting: return # None if event == 'line': @@ -101,6 +108,8 @@ def trace_dispatch(self, frame, event, arg): return self.trace_dispatch if event == 'c_return': return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch @@ -148,6 +157,9 @@ def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: # Ignore return events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: + # It's possible to trigger a StopIteration exception in + # the caller so we must set the trace function in the caller + self._set_caller_tracefunc(frame) return self.trace_dispatch try: self.frame_returning = frame @@ -158,6 +170,11 @@ def dispatch_return(self, frame, arg): # The user issued a 'next' or 'until' command. if self.stopframe is frame and self.stoplineno != -1: self._set_stopinfo(None, None) + # The previous frame might not have f_trace set, unless we are + # issuing a command that does not expect to stop, we should set + # f_trace + if self.stoplineno != -1: + self._set_caller_tracefunc(frame) return self.trace_dispatch def dispatch_exception(self, frame, arg): @@ -187,6 +204,17 @@ def dispatch_exception(self, frame, arg): return self.trace_dispatch + def dispatch_opcode(self, frame, arg): + """Invoke user function and return trace function for opcode event. + If the debugger stops on the current opcode, invoke + self.user_opcode(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + if self.stop_here(frame) or self.break_here(frame): + self.user_opcode(frame) + if self.quitting: raise BdbQuit + return self.trace_dispatch + # Normally derived classes don't override the following # methods, but they may if they want to redefine the # definition of stopping and breakpoints. @@ -250,9 +278,25 @@ def do_clear(self, arg): raise NotImplementedError("subclass of bdb must implement do_clear()") def break_anywhere(self, frame): - """Return True if there is any breakpoint for frame's filename. + """Return True if there is any breakpoint in that frame """ - return self.canonic(frame.f_code.co_filename) in self.breaks + filename = self.canonic(frame.f_code.co_filename) + if filename not in self.breaks: + return False + for lineno in self.breaks[filename]: + if self._lineno_in_frame(lineno, frame): + return True + return False + + def _lineno_in_frame(self, lineno, frame): + """Return True if the line number is in the frame's code object. + """ + code = frame.f_code + if lineno < code.co_firstlineno: + return False + if code not in self.code_linenos: + self.code_linenos[code] = set(lineno for _, _, lineno in code.co_lines()) + return lineno in self.code_linenos[code] # Derived classes should override the user_* methods # to gain control. @@ -273,7 +317,21 @@ def user_exception(self, frame, exc_info): """Called when we stop on an exception.""" pass - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): + def user_opcode(self, frame): + """Called when we are about to execute an opcode.""" + pass + + def _set_trace_opcodes(self, trace_opcodes): + if trace_opcodes != self.trace_opcodes: + self.trace_opcodes = trace_opcodes + frame = self.enterframe + while frame is not None: + frame.f_trace_opcodes = trace_opcodes + if frame is self.botframe: + break + frame = frame.f_back + + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -286,6 +344,17 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + self._set_trace_opcodes(opcode) + + def _set_caller_tracefunc(self, current_frame): + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame, unless + # the caller is the botframe. + caller_frame = current_frame.f_back + if caller_frame and not caller_frame.f_trace and caller_frame is not self.botframe: + caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods # to affect the stepping state. @@ -300,16 +369,12 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch self._set_stopinfo(None, None) + def set_stepinstr(self): + """Stop before the next instruction.""" + self._set_stopinfo(None, None, opcode=True) + def set_next(self, frame): """Stop on the next line in or below the given frame.""" self._set_stopinfo(frame, None) @@ -317,7 +382,7 @@ def set_next(self, frame): def set_return(self, frame): """Stop when returning from the given frame.""" if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: - self._set_stopinfo(frame, None, -1) + self._set_stopinfo(frame, frame, -1) else: self._set_stopinfo(frame.f_back, frame) @@ -326,17 +391,19 @@ def set_trace(self, frame=None): If frame is not specified, debugging starts from caller's frame. """ + sys.settrace(None) if frame is None: frame = sys._getframe().f_back self.reset() + self.enterframe = frame while frame: frame.f_trace = self.trace_dispatch self.botframe = frame - # We need f_trace_liens == True for the debugger to work - self.frame_trace_lines[frame] = frame.f_trace_lines + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work frame.f_trace_lines = True frame = frame.f_back - self.set_step() + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): @@ -353,9 +420,9 @@ def set_continue(self): while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back - for frame, prev_trace_lines in self.frame_trace_lines.items(): - frame.f_trace_lines = prev_trace_lines - self.frame_trace_lines = {} + for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): + frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + self.frame_trace_lines_opcodes = {} def set_quit(self): """Set quitting attribute to True. @@ -394,6 +461,14 @@ def set_break(self, filename, lineno, temporary=False, cond=None, return 'Line %s:%d does not exist' % (filename, lineno) self._add_to_breaks(filename, lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) + # After we set a new breakpoint, we need to search through all frames + # and set f_trace to trace_dispatch if there could be a breakpoint in + # that frame. + frame = self.enterframe + while frame: + if self.break_anywhere(frame): + frame.f_trace = self.trace_dispatch + frame = frame.f_back return None def _load_breaks(self): diff --git a/Lib/bz2.py b/Lib/bz2.py index fabe4f73c8d808..2420cd019069b4 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -17,7 +17,7 @@ from _bz2 import BZ2Compressor, BZ2Decompressor -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 @@ -54,7 +54,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9): """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") @@ -100,7 +100,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -115,13 +115,21 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED self._buffer = None @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" diff --git a/Lib/cProfile.py b/Lib/cProfile.py old mode 100755 new mode 100644 index 135a12c3965c00..e7c868b8d55543 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Python interface for the 'lsprof' profiler. Compatible with the 'profile' module. """ @@ -41,7 +39,9 @@ class Profile(_lsprof.Profiler): def print_stats(self, sort=-1): import pstats - pstats.Stats(self).strip_dirs().sort_stats(sort).print_stats() + if not isinstance(sort, tuple): + sort = (sort,) + pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() def dump_stats(self, file): import marshal diff --git a/Lib/calendar.py b/Lib/calendar.py index 833ce331b14a0c..069dd5174112ae 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -159,8 +159,8 @@ def weekday(year, month, day): def monthrange(year, month): - """Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for - year, month.""" + """Return weekday of first day of month (0-6 ~ Mon-Sun) + and number of days (28-31) for year, month.""" if not 1 <= month <= 12: raise IllegalMonthError(month) day1 = weekday(year, month, 1) diff --git a/Lib/cmd.py b/Lib/cmd.py index a37d16cd7bde16..c333e099bd8c9a 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -5,16 +5,16 @@ 1. End of file on input is processed as the command 'EOF'. 2. A command is parsed out of each line by collecting the prefix composed of characters in the identchars member. -3. A command `foo' is dispatched to a method 'do_foo()'; the do_ method +3. A command 'foo' is dispatched to a method 'do_foo()'; the do_ method is passed a single argument consisting of the remainder of the line. 4. Typing an empty line repeats the last command. (Actually, it calls the - method `emptyline', which may be overridden in a subclass.) -5. There is a predefined `help' method. Given an argument `topic', it - calls the command `help_topic'. With no arguments, it lists all topics + method 'emptyline', which may be overridden in a subclass.) +5. There is a predefined 'help' method. Given an argument 'topic', it + calls the command 'help_topic'. With no arguments, it lists all topics with defined help_ functions, broken into up to three topics; documented commands, miscellaneous help topics, and undocumented commands. -6. The command '?' is a synonym for `help'. The command '!' is a synonym - for `shell', if a do_shell method exists. +6. The command '?' is a synonym for 'help'. The command '!' is a synonym + for 'shell', if a do_shell method exists. 7. If completion is enabled, completing commands will be done automatically, and completing of commands args is done by calling complete_foo() with arguments text, line, begidx, endidx. text is string we are matching @@ -23,21 +23,21 @@ indexes of the text being matched, which could be used to provide different completion depending upon which position the argument is in. -The `default' method may be overridden to intercept commands for which there +The 'default' method may be overridden to intercept commands for which there is no do_ method. -The `completedefault' method may be overridden to intercept completions for +The 'completedefault' method may be overridden to intercept completions for commands that have no complete_ method. -The data member `self.ruler' sets the character used to draw separator lines +The data member 'self.ruler' sets the character used to draw separator lines in the help messages. If empty, no ruler line is drawn. It defaults to "=". -If the value of `self.intro' is nonempty when the cmdloop method is called, +If the value of 'self.intro' is nonempty when the cmdloop method is called, it is printed out on interpreter startup. This value may be overridden via an optional argument to the cmdloop() method. -The data members `self.doc_header', `self.misc_header', and -`self.undoc_header' set the headers used for the help function's +The data members 'self.doc_header', 'self.misc_header', and +'self.undoc_header' set the headers used for the help function's listings of documented functions, miscellaneous topics, and undocumented functions respectively. """ diff --git a/Lib/code.py b/Lib/code.py index f4aecddeca7813..1cc2ed8b1dbf28 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -25,10 +25,10 @@ class InteractiveInterpreter: def __init__(self, locals=None): """Constructor. - The optional 'locals' argument specifies the dictionary in - which code will be executed; it defaults to a newly created - dictionary with key "__name__" set to "__console__" and key - "__doc__" set to None. + The optional 'locals' argument specifies a mapping to use as the + namespace in which code will be executed; it defaults to a newly + created dictionary with key "__name__" set to "__console__" and + key "__doc__" set to None. """ if locals is None: @@ -64,7 +64,7 @@ def runsource(self, source, filename="", symbol="single"): code = self.compile(source, filename, symbol) except (OverflowError, SyntaxError, ValueError): # Case 1 - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if code is None: @@ -94,7 +94,7 @@ def runcode(self, code): except: self.showtraceback() - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Display the syntax error that just occurred. This doesn't display a stack trace because there isn't one. @@ -106,29 +106,14 @@ def showsyntaxerror(self, filename=None): The output is written by self.write(), below. """ - type, value, tb = sys.exc_info() - sys.last_exc = value - sys.last_type = type - sys.last_value = value - sys.last_traceback = tb - if filename and type is SyntaxError: - # Work hard to stuff the correct filename in the exception - try: - msg, (dummy_filename, lineno, offset, line) = value.args - except ValueError: - # Not the format we expect; leave it alone - pass - else: - # Stuff in the right filename - value = SyntaxError(msg, (filename, lineno, offset, line)) - sys.last_exc = sys.last_value = value - if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value) - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(type, value, tb) + try: + typ, value, tb = sys.exc_info() + if filename and issubclass(typ, SyntaxError): + value.filename = filename + source = kwargs.pop('source', "") + self._showtraceback(typ, value, None, source) + finally: + typ = value = tb = None def showtraceback(self): """Display the exception that just occurred. @@ -138,19 +123,46 @@ def showtraceback(self): The output is written by self.write(), below. """ - sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() - sys.last_traceback = last_tb - sys.last_exc = ei[1] try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) - if sys.excepthook is sys.__excepthook__: - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(ei[0], ei[1], last_tb) + typ, value, tb = sys.exc_info() + self._showtraceback(typ, value, tb.tb_next, "") finally: - last_tb = ei = None + typ = value = tb = None + + def _showtraceback(self, typ, value, tb, source): + sys.last_type = typ + sys.last_traceback = tb + value = value.with_traceback(tb) + # Set the line of text that the exception refers to + lines = source.splitlines() + if (source and typ is SyntaxError + and not value.text and value.lineno is not None + and len(lines) >= value.lineno): + value.text = lines[value.lineno - 1] + sys.last_exc = sys.last_value = value + if sys.excepthook is sys.__excepthook__: + self._excepthook(typ, value, tb) + else: + # If someone has set sys.excepthook, we let that take precedence + # over self.write + try: + sys.excepthook(typ, value, tb) + except SystemExit: + raise + except BaseException as e: + e.__context__ = None + e = e.with_traceback(e.__traceback__.tb_next) + print('Error in sys.excepthook:', file=sys.stderr) + sys.__excepthook__(type(e), e, e.__traceback__) + print(file=sys.stderr) + print('Original exception was:', file=sys.stderr) + sys.__excepthook__(typ, value, tb) + + def _excepthook(self, typ, value, tb): + # This method is being overwritten in + # _pyrepl.console.InteractiveColoredConsole + lines = traceback.format_exception(typ, value, tb) + self.write(''.join(lines)) def write(self, data): """Write a string. @@ -170,7 +182,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename="", local_exit=False): + def __init__(self, locals=None, filename="", *, local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -280,7 +292,7 @@ def interact(self, banner=None, exitmsg=None): elif exitmsg != '': self.write('%s\n' % exitmsg) - def push(self, line): + def push(self, line, filename=None, _symbol="single"): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -296,7 +308,9 @@ def push(self, line): """ self.buffer.append(line) source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) + if filename is None: + filename = self.filename + more = self.runsource(source, filename, symbol=_symbol) if not more: self.resetbuffer() return more @@ -351,7 +365,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa console.raw_input = readfunc else: try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg) diff --git a/Lib/codecs.py b/Lib/codecs.py index 9b35b6127dd01c..e365e6cf22929f 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -1109,24 +1109,15 @@ def make_encoding_map(decoding_map): ### error handlers -try: - strict_errors = lookup_error("strict") - ignore_errors = lookup_error("ignore") - replace_errors = lookup_error("replace") - xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") - backslashreplace_errors = lookup_error("backslashreplace") - namereplace_errors = lookup_error("namereplace") -except LookupError: - # In --disable-unicode builds, these error handler are missing - strict_errors = None - ignore_errors = None - replace_errors = None - xmlcharrefreplace_errors = None - backslashreplace_errors = None - namereplace_errors = None +strict_errors = lookup_error("strict") +ignore_errors = lookup_error("ignore") +replace_errors = lookup_error("replace") +xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") +backslashreplace_errors = lookup_error("backslashreplace") +namereplace_errors = lookup_error("namereplace") # Tell modulefinder that using codecs probably needs the encodings # package _false = 0 if _false: - import encodings + import encodings # noqa: F401 diff --git a/Lib/codeop.py b/Lib/codeop.py index 6ad60e7f85098d..adf000ba29f88c 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -44,6 +44,7 @@ # Caveat emptor: These flags are undocumented on purpose and depending # on their effect outside the standard library is **unsupported**. PyCF_DONT_IMPLY_DEDENT = 0x200 +PyCF_ONLY_AST = 0x400 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 def _maybe_compile(compiler, source, filename, symbol): @@ -65,7 +66,7 @@ def _maybe_compile(compiler, source, filename, symbol): try: compiler(source + "\n", filename, symbol) return None - except IncompleteInputError as e: + except _IncompleteInputError as e: return None except SyntaxError as e: pass @@ -109,12 +110,14 @@ class Compile: def __init__(self): self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT - def __call__(self, source, filename, symbol, **kwargs): - flags = self.flags + def __call__(self, source, filename, symbol, flags=0, **kwargs): + flags |= self.flags if kwargs.get('incomplete_input', True) is False: flags &= ~PyCF_DONT_IMPLY_DEDENT flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT codeob = compile(source, filename, symbol, flags, True) + if flags & PyCF_ONLY_AST: + return codeob # this is an ast.Module in this case for feature in _features: if codeob.co_flags & feature.compiler_flag: self.flags |= feature.compiler_flag diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 2e527dfd810c43..d688141f9b183d 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -29,6 +29,9 @@ import _collections_abc import sys as _sys +_sys.modules['collections.abc'] = _collections_abc +abc = _collections_abc + from itertools import chain as _chain from itertools import repeat as _repeat from itertools import starmap as _starmap @@ -46,7 +49,8 @@ _collections_abc.MutableSequence.register(deque) try: - from _collections import _deque_iterator + # Expose _deque_iterator to support pickling deque iterators + from _collections import _deque_iterator # noqa: F401 except ImportError: pass @@ -639,7 +643,8 @@ def elements(self): >>> sorted(c.elements()) ['A', 'A', 'B', 'B', 'C', 'C'] - # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + >>> import math >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) >>> math.prod(prime_factors.elements()) @@ -680,7 +685,7 @@ def update(self, iterable=None, /, **kwds): ''' # The regular dict.update() operation makes no sense here because the - # replace behavior results in the some of original untouched counts + # replace behavior results in some of the original untouched counts # being mixed-in with all of the other counts for a mismash that # doesn't have a straight-forward interpretation in most counting # contexts. Instead, we implement straight-addition. Both the inputs @@ -1015,7 +1020,7 @@ def __getitem__(self, key): return self.__missing__(key) # support subclasses that define __missing__ def get(self, key, default=None): - return self[key] if key in self else default + return self[key] if key in self else default # needs to make use of __contains__ def __len__(self): return len(set().union(*self.maps)) # reuses stored hash values if possible @@ -1027,7 +1032,10 @@ def __iter__(self): return iter(d) def __contains__(self, key): - return any(key in m for m in self.maps) + for mapping in self.maps: + if key in mapping: + return True + return False def __bool__(self): return any(self.maps) @@ -1037,9 +1045,9 @@ def __repr__(self): return f'{self.__class__.__name__}({", ".join(map(repr, self.maps))})' @classmethod - def fromkeys(cls, iterable, *args): - 'Create a ChainMap with a single dict created from the iterable.' - return cls(dict.fromkeys(iterable, *args)) + def fromkeys(cls, iterable, value=None, /): + 'Create a new ChainMap with keys from iterable and values set to value.' + return cls(dict.fromkeys(iterable, value)) def copy(self): 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py deleted file mode 100644 index 86ca8b8a8414b3..00000000000000 --- a/Lib/collections/abc.py +++ /dev/null @@ -1,3 +0,0 @@ -from _collections_abc import * -from _collections_abc import __all__ -from _collections_abc import _CallableGenericAlias diff --git a/Lib/colorsys.py b/Lib/colorsys.py index bc897bd0f99298..e97f91718a3a30 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -24,7 +24,7 @@ __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb", "rgb_to_hsv","hsv_to_rgb"] -# Some floating point constants +# Some floating-point constants ONE_THIRD = 1.0/3.0 ONE_SIXTH = 1.0/6.0 diff --git a/Lib/compileall.py b/Lib/compileall.py index 9b53086bf41380..47e2446356e7d7 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -116,7 +116,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, prependdir=prependdir, limit_sl_dest=limit_sl_dest, hardlink_dupes=hardlink_dupes), - files) + files, + chunksize=4) success = min(results, default=True) else: for file in files: diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index 292e886d5a88ac..7ada7431c1ab8c 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -23,11 +23,13 @@ 'ALL_COMPLETED', 'CancelledError', 'TimeoutError', + 'InvalidStateError', 'BrokenExecutor', 'Future', 'Executor', 'wait', 'as_completed', + 'InterpreterPoolExecutor', 'ProcessPoolExecutor', 'ThreadPoolExecutor', ) @@ -38,7 +40,7 @@ def __dir__(): def __getattr__(name): - global ProcessPoolExecutor, ThreadPoolExecutor + global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor if name == 'ProcessPoolExecutor': from .process import ProcessPoolExecutor as pe @@ -50,4 +52,13 @@ def __getattr__(name): ThreadPoolExecutor = te return te + if name == 'InterpreterPoolExecutor': + try: + from .interpreter import InterpreterPoolExecutor as ie + except ModuleNotFoundError: + ie = InterpreterPoolExecutor = None + else: + InterpreterPoolExecutor = ie + return ie + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index 6742a07753c921..707fcdfde79acd 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -23,14 +23,6 @@ CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED' FINISHED = 'FINISHED' -_FUTURE_STATES = [ - PENDING, - RUNNING, - CANCELLED, - CANCELLED_AND_NOTIFIED, - FINISHED -] - _STATE_TO_DESCRIPTION_MAP = { PENDING: "pending", RUNNING: "running", diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py new file mode 100644 index 00000000000000..d17688dc9d7346 --- /dev/null +++ b/Lib/concurrent/futures/interpreter.py @@ -0,0 +1,242 @@ +"""Implements InterpreterPoolExecutor.""" + +import contextlib +import pickle +import textwrap +from . import thread as _thread +import _interpreters +import _interpqueues + + +class ExecutionFailed(_interpreters.InterpreterError): + """An unhandled exception happened during execution.""" + + def __init__(self, excinfo): + msg = excinfo.formatted + if not msg: + if excinfo.type and excinfo.msg: + msg = f'{excinfo.type.__name__}: {excinfo.msg}' + else: + msg = excinfo.type.__name__ or excinfo.msg + super().__init__(msg) + self.excinfo = excinfo + + def __str__(self): + try: + formatted = self.excinfo.errdisplay + except Exception: + return super().__str__() + else: + return textwrap.dedent(f""" +{super().__str__()} + +Uncaught in the interpreter: + +{formatted} + """.strip()) + + +UNBOUND = 2 # error; this should not happen. + + +class WorkerContext(_thread.WorkerContext): + + @classmethod + def prepare(cls, initializer, initargs, shared): + def resolve_task(fn, args, kwargs): + if isinstance(fn, str): + # XXX Circle back to this later. + raise TypeError('scripts not supported') + if args or kwargs: + raise ValueError(f'a script does not take args or kwargs, got {args!r} and {kwargs!r}') + data = textwrap.dedent(fn) + kind = 'script' + # Make sure the script compiles. + # Ideally we wouldn't throw away the resulting code + # object. However, there isn't much to be done until + # code objects are shareable and/or we do a better job + # of supporting code objects in _interpreters.exec(). + compile(data, '', 'exec') + else: + # Functions defined in the __main__ module can't be pickled, + # so they can't be used here. In the future, we could possibly + # borrow from multiprocessing to work around this. + data = pickle.dumps((fn, args, kwargs)) + kind = 'function' + return (data, kind) + + if initializer is not None: + try: + initdata = resolve_task(initializer, initargs, {}) + except ValueError: + if isinstance(initializer, str) and initargs: + raise ValueError(f'an initializer script does not take args, got {initargs!r}') + raise # re-raise + else: + initdata = None + def create_context(): + return cls(initdata, shared) + return create_context, resolve_task + + @classmethod + @contextlib.contextmanager + def _capture_exc(cls, resultsid): + try: + yield + except BaseException as exc: + # Send the captured exception out on the results queue, + # but still leave it unhandled for the interpreter to handle. + err = pickle.dumps(exc) + _interpqueues.put(resultsid, (None, err), 1, UNBOUND) + raise # re-raise + + @classmethod + def _send_script_result(cls, resultsid): + _interpqueues.put(resultsid, (None, None), 0, UNBOUND) + + @classmethod + def _call(cls, func, args, kwargs, resultsid): + with cls._capture_exc(resultsid): + res = func(*args or (), **kwargs or {}) + # Send the result back. + try: + _interpqueues.put(resultsid, (res, None), 0, UNBOUND) + except _interpreters.NotShareableError: + res = pickle.dumps(res) + _interpqueues.put(resultsid, (res, None), 1, UNBOUND) + + @classmethod + def _call_pickled(cls, pickled, resultsid): + with cls._capture_exc(resultsid): + fn, args, kwargs = pickle.loads(pickled) + cls._call(fn, args, kwargs, resultsid) + + def __init__(self, initdata, shared=None): + self.initdata = initdata + self.shared = dict(shared) if shared else None + self.interpid = None + self.resultsid = None + + def __del__(self): + if self.interpid is not None: + self.finalize() + + def _exec(self, script): + assert self.interpid is not None + excinfo = _interpreters.exec(self.interpid, script, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def initialize(self): + assert self.interpid is None, self.interpid + self.interpid = _interpreters.create(reqrefs=True) + try: + _interpreters.incref(self.interpid) + + maxsize = 0 + fmt = 0 + self.resultsid = _interpqueues.create(maxsize, fmt, UNBOUND) + + self._exec(f'from {__name__} import WorkerContext') + + if self.shared: + _interpreters.set___main___attrs( + self.interpid, self.shared, restrict=True) + + if self.initdata: + self.run(self.initdata) + except BaseException: + self.finalize() + raise # re-raise + + def finalize(self): + interpid = self.interpid + resultsid = self.resultsid + self.resultsid = None + self.interpid = None + if resultsid is not None: + try: + _interpqueues.destroy(resultsid) + except _interpqueues.QueueNotFoundError: + pass + if interpid is not None: + try: + _interpreters.decref(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + def run(self, task): + data, kind = task + if kind == 'script': + raise NotImplementedError('script kind disabled') + script = f""" +with WorkerContext._capture_exc({self.resultsid}): +{textwrap.indent(data, ' ')} +WorkerContext._send_script_result({self.resultsid})""" + elif kind == 'function': + script = f'WorkerContext._call_pickled({data!r}, {self.resultsid})' + else: + raise NotImplementedError(kind) + + try: + self._exec(script) + except ExecutionFailed as exc: + exc_wrapper = exc + else: + exc_wrapper = None + + # Return the result, or raise the exception. + while True: + try: + obj = _interpqueues.get(self.resultsid) + except _interpqueues.QueueNotFoundError: + raise # re-raise + except _interpqueues.QueueError: + continue + except ModuleNotFoundError: + # interpreters.queues doesn't exist, which means + # QueueEmpty doesn't. Act as though it does. + continue + else: + break + (res, excdata), pickled, unboundop = obj + assert unboundop is None, unboundop + if excdata is not None: + assert res is None, res + assert pickled + assert exc_wrapper is not None + exc = pickle.loads(excdata) + raise exc from exc_wrapper + return pickle.loads(res) if pickled else res + + +class BrokenInterpreterPool(_thread.BrokenThreadPool): + """ + Raised when a worker thread in an InterpreterPoolExecutor failed initializing. + """ + + +class InterpreterPoolExecutor(_thread.ThreadPoolExecutor): + + BROKEN = BrokenInterpreterPool + + @classmethod + def prepare_context(cls, initializer, initargs, shared): + return WorkerContext.prepare(initializer, initargs, shared) + + def __init__(self, max_workers=None, thread_name_prefix='', + initializer=None, initargs=(), shared=None): + """Initializes a new InterpreterPoolExecutor instance. + + Args: + max_workers: The maximum number of interpreters that can be used to + execute the given calls. + thread_name_prefix: An optional name prefix to give our threads. + initializer: A callable or script used to initialize + each worker interpreter. + initargs: A tuple of arguments to pass to the initializer. + shared: A mapping of shareabled objects to be inserted into + each worker interpreter. + """ + super().__init__(max_workers, thread_name_prefix, + initializer, initargs, shared=shared) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index ca843e11eeb83d..42eee72bc1457f 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -68,27 +68,31 @@ class _ThreadWakeup: def __init__(self): self._closed = False + self._lock = threading.Lock() self._reader, self._writer = mp.Pipe(duplex=False) def close(self): - # Please note that we do not take the shutdown lock when + # Please note that we do not take the self._lock when # calling clear() (to avoid deadlocking) so this method can # only be called safely from the same thread as all calls to - # clear() even if you hold the shutdown lock. Otherwise we + # clear() even if you hold the lock. Otherwise we # might try to read from the closed pipe. - if not self._closed: - self._closed = True - self._writer.close() - self._reader.close() + with self._lock: + if not self._closed: + self._closed = True + self._writer.close() + self._reader.close() def wakeup(self): - if not self._closed: - self._writer.send_bytes(b"") + with self._lock: + if not self._closed: + self._writer.send_bytes(b"") def clear(self): - if not self._closed: - while self._reader.poll(): - self._reader.recv_bytes() + if self._closed: + raise RuntimeError('operation on closed _ThreadWakeup') + while self._reader.poll(): + self._reader.recv_bytes() def _python_exit(): @@ -167,10 +171,8 @@ def __init__(self, work_id, fn, args, kwargs): class _SafeQueue(Queue): """Safe Queue set exception to the future object linked to a job""" - def __init__(self, max_size=0, *, ctx, pending_work_items, shutdown_lock, - thread_wakeup): + def __init__(self, max_size=0, *, ctx, pending_work_items, thread_wakeup): self.pending_work_items = pending_work_items - self.shutdown_lock = shutdown_lock self.thread_wakeup = thread_wakeup super().__init__(max_size, ctx=ctx) @@ -179,8 +181,7 @@ def _on_queue_feeder_error(self, e, obj): tb = format_exception(type(e), e, e.__traceback__) e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb))) work_item = self.pending_work_items.pop(obj.work_id, None) - with self.shutdown_lock: - self.thread_wakeup.wakeup() + self.thread_wakeup.wakeup() # work_item can be None if another process terminated. In this # case, the executor_manager_thread fails all work_items # with BrokenProcessPool @@ -296,11 +297,10 @@ def __init__(self, executor): # if there is no pending work item. def weakref_cb(_, thread_wakeup=self.thread_wakeup, - shutdown_lock=self.shutdown_lock): - mp.util.debug('Executor collected: triggering callback for' + mp_util_debug=mp.util.debug): + mp_util_debug('Executor collected: triggering callback for' ' QueueManager wakeup') - with shutdown_lock: - thread_wakeup.wakeup() + thread_wakeup.wakeup() self.executor_reference = weakref.ref(executor, weakref_cb) @@ -428,11 +428,6 @@ def wait_result_broken_or_wakeup(self): elif wakeup_reader in ready: is_broken = False - # No need to hold the _shutdown_lock here because: - # 1. we're the only thread to use the wakeup reader - # 2. we're also the only thread to call thread_wakeup.close() - # 3. we want to avoid a possible deadlock when both reader and writer - # would block (gh-105829) self.thread_wakeup.clear() return result_item, is_broken, cause @@ -588,7 +583,7 @@ def _check_system_limits(): raise NotImplementedError(_system_limited) _system_limits_checked = True try: - import multiprocessing.synchronize + import multiprocessing.synchronize # noqa: F401 except ImportError: _system_limited = ( "This Python build lacks multiprocessing.synchronize, usually due " @@ -720,10 +715,9 @@ def __init__(self, max_workers=None, mp_context=None, # as it could result in a deadlock if a worker process dies with the # _result_queue write lock still acquired. # - # _shutdown_lock must be locked to access _ThreadWakeup.close() and - # .wakeup(). Care must also be taken to not call clear or close from - # more than one thread since _ThreadWakeup.clear() is not protected by - # the _shutdown_lock + # Care must be taken to only call clear and close from the + # executor_manager_thread, since _ThreadWakeup.clear() is not protected + # by a lock. self._executor_manager_thread_wakeup = _ThreadWakeup() # Create communication channels for the executor @@ -734,7 +728,6 @@ def __init__(self, max_workers=None, mp_context=None, self._call_queue = _SafeQueue( max_size=queue_size, ctx=self._mp_context, pending_work_items=self._pending_work_items, - shutdown_lock=self._shutdown_lock, thread_wakeup=self._executor_manager_thread_wakeup) # Killed worker processes can produce spurious "broken pipe" # tracebacks in the queue's own worker thread. But we detect killed diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index a024033f35fb54..16cc5533d429ef 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -43,19 +43,46 @@ def _python_exit(): after_in_parent=_global_shutdown_lock.release) +class WorkerContext: + + @classmethod + def prepare(cls, initializer, initargs): + if initializer is not None: + if not callable(initializer): + raise TypeError("initializer must be a callable") + def create_context(): + return cls(initializer, initargs) + def resolve_task(fn, args, kwargs): + return (fn, args, kwargs) + return create_context, resolve_task + + def __init__(self, initializer, initargs): + self.initializer = initializer + self.initargs = initargs + + def initialize(self): + if self.initializer is not None: + self.initializer(*self.initargs) + + def finalize(self): + pass + + def run(self, task): + fn, args, kwargs = task + return fn(*args, **kwargs) + + class _WorkItem: - def __init__(self, future, fn, args, kwargs): + def __init__(self, future, task): self.future = future - self.fn = fn - self.args = args - self.kwargs = kwargs + self.task = task - def run(self): + def run(self, ctx): if not self.future.set_running_or_notify_cancel(): return try: - result = self.fn(*self.args, **self.kwargs) + result = ctx.run(self.task) except BaseException as exc: self.future.set_exception(exc) # Break a reference cycle with the exception 'exc' @@ -66,16 +93,15 @@ def run(self): __class_getitem__ = classmethod(types.GenericAlias) -def _worker(executor_reference, work_queue, initializer, initargs): - if initializer is not None: - try: - initializer(*initargs) - except BaseException: - _base.LOGGER.critical('Exception in initializer:', exc_info=True) - executor = executor_reference() - if executor is not None: - executor._initializer_failed() - return +def _worker(executor_reference, ctx, work_queue): + try: + ctx.initialize() + except BaseException: + _base.LOGGER.critical('Exception in initializer:', exc_info=True) + executor = executor_reference() + if executor is not None: + executor._initializer_failed() + return try: while True: try: @@ -89,7 +115,7 @@ def _worker(executor_reference, work_queue, initializer, initargs): work_item = work_queue.get(block=True) if work_item is not None: - work_item.run() + work_item.run(ctx) # Delete references to object. See GH-60488 del work_item continue @@ -110,6 +136,8 @@ def _worker(executor_reference, work_queue, initializer, initargs): del executor except BaseException: _base.LOGGER.critical('Exception in worker', exc_info=True) + finally: + ctx.finalize() class BrokenThreadPool(_base.BrokenExecutor): @@ -120,11 +148,17 @@ class BrokenThreadPool(_base.BrokenExecutor): class ThreadPoolExecutor(_base.Executor): + BROKEN = BrokenThreadPool + # Used to assign unique thread names when thread_name_prefix is not supplied. _counter = itertools.count().__next__ + @classmethod + def prepare_context(cls, initializer, initargs): + return WorkerContext.prepare(initializer, initargs) + def __init__(self, max_workers=None, thread_name_prefix='', - initializer=None, initargs=()): + initializer=None, initargs=(), **ctxkwargs): """Initializes a new ThreadPoolExecutor instance. Args: @@ -133,6 +167,7 @@ def __init__(self, max_workers=None, thread_name_prefix='', thread_name_prefix: An optional name prefix to give our threads. initializer: A callable used to initialize worker threads. initargs: A tuple of arguments to pass to the initializer. + ctxkwargs: Additional arguments to cls.prepare_context(). """ if max_workers is None: # ThreadPoolExecutor is often used to: @@ -146,8 +181,9 @@ def __init__(self, max_workers=None, thread_name_prefix='', if max_workers <= 0: raise ValueError("max_workers must be greater than 0") - if initializer is not None and not callable(initializer): - raise TypeError("initializer must be a callable") + (self._create_worker_context, + self._resolve_work_item_task, + ) = type(self).prepare_context(initializer, initargs, **ctxkwargs) self._max_workers = max_workers self._work_queue = queue.SimpleQueue() @@ -158,13 +194,11 @@ def __init__(self, max_workers=None, thread_name_prefix='', self._shutdown_lock = threading.Lock() self._thread_name_prefix = (thread_name_prefix or ("ThreadPoolExecutor-%d" % self._counter())) - self._initializer = initializer - self._initargs = initargs def submit(self, fn, /, *args, **kwargs): with self._shutdown_lock, _global_shutdown_lock: if self._broken: - raise BrokenThreadPool(self._broken) + raise self.BROKEN(self._broken) if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') @@ -173,7 +207,8 @@ def submit(self, fn, /, *args, **kwargs): 'interpreter shutdown') f = _base.Future() - w = _WorkItem(f, fn, args, kwargs) + task = self._resolve_work_item_task(fn, args, kwargs) + w = _WorkItem(f, task) self._work_queue.put(w) self._adjust_thread_count() @@ -196,9 +231,8 @@ def weakref_cb(_, q=self._work_queue): num_threads) t = threading.Thread(name=thread_name, target=_worker, args=(weakref.ref(self, weakref_cb), - self._work_queue, - self._initializer, - self._initargs)) + self._create_worker_context(), + self._work_queue)) t.start() self._threads.add(t) _threads_queues[t] = self._work_queue @@ -214,7 +248,7 @@ def _initializer_failed(self): except queue.Empty: break if work_item is not None: - work_item.future.set_exception(BrokenThreadPool(self._broken)) + work_item.future.set_exception(self.BROKEN(self._broken)) def shutdown(self, wait=True, *, cancel_futures=False): with self._shutdown_lock: diff --git a/Lib/configparser.py b/Lib/configparser.py index 71362d23ec3757..420dce77c234e1 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -18,8 +18,8 @@ delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', - interpolation=, converters=): - + interpolation=, converters=, + allow_unnamed_section=False): Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. @@ -68,6 +68,10 @@ converter gets its corresponding get*() method on the parser object and section proxies. + When `allow_unnamed_section` is True (default: False), options + without section are accepted: the section for these is + ``configparser.UNNAMED_SECTION``. + sections() Return all the configuration section names, sans DEFAULT. @@ -139,23 +143,28 @@ between keys and values are surrounded by spaces. """ -from collections.abc import MutableMapping +# Do not import dataclasses; overhead is unacceptable (gh-117703) + +from collections.abc import Iterable, MutableMapping from collections import ChainMap as _ChainMap +import contextlib import functools import io import itertools import os import re import sys +import types __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", + "MultilineContinuationError", "UnnamedSectionDisabledError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "SectionProxy", "ConverterMapping", - "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH") + "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") _default_dict = dict DEFAULTSECT = "DEFAULT" @@ -297,15 +306,33 @@ def __init__(self, option, section, rawval): class ParsingError(Error): """Raised when a configuration file does not follow legal syntax.""" - def __init__(self, source): + def __init__(self, source, *args): super().__init__(f'Source contains parsing errors: {source!r}') self.source = source self.errors = [] self.args = (source, ) + if args: + self.append(*args) def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, line) + self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + + def combine(self, others): + for other in others: + for error in other.errors: + self.append(*error) + return self + + @staticmethod + def _raise_all(exceptions: Iterable['ParsingError']): + """ + Combine any number of ParsingErrors into one and raise it. + """ + exceptions = iter(exceptions) + with contextlib.suppress(StopIteration): + raise next(exceptions).combine(exceptions) + class MissingSectionHeaderError(ParsingError): @@ -322,6 +349,36 @@ def __init__(self, filename, lineno, line): self.args = (filename, lineno, line) +class MultilineContinuationError(ParsingError): + """Raised when a key without value is followed by continuation line""" + def __init__(self, filename, lineno, line): + Error.__init__( + self, + "Key without value continued with an indented line.\n" + "file: %r, line: %d\n%r" + %(filename, lineno, line)) + self.source = filename + self.lineno = lineno + self.line = line + self.args = (filename, lineno, line) + + +class UnnamedSectionDisabledError(Error): + """Raised when an attempt to use UNNAMED_SECTION is made with the + feature disabled.""" + def __init__(self): + Error.__init__(self, "Support for UNNAMED_SECTION is disabled.") + + +class _UnnamedSection: + + def __repr__(self): + return "" + + +UNNAMED_SECTION = _UnnamedSection() + + # Used in parser getters to indicate the default behaviour when a specific # option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. @@ -490,6 +547,52 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, "found: %r" % (rest,)) +class _ReadState: + elements_added : set[str] + cursect : dict[str, str] | None = None + sectname : str | None = None + optname : str | None = None + lineno : int = 0 + indent_level : int = 0 + errors : list[ParsingError] + + def __init__(self): + self.elements_added = set() + self.errors = list() + + +class _Line(str): + + def __new__(cls, val, *args, **kwargs): + return super().__new__(cls, val) + + def __init__(self, val, prefixes): + self.prefixes = prefixes + + @functools.cached_property + def clean(self): + return self._strip_full() and self._strip_inline() + + @property + def has_comments(self): + return self.strip() != self.clean + + def _strip_inline(self): + """ + Search for the earliest prefix at the beginning of the line or following a space. + """ + matcher = re.compile( + '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) + # match nothing if no prefixes + or '(?!)' + ) + match = matcher.search(self) + return self[:match.start() if match else None].strip() + + def _strip_full(self): + return '' if any(map(self.strip().startswith, self.prefixes.full)) else True + + class RawConfigParser(MutableMapping): """ConfigParser that does not do interpolation.""" @@ -536,7 +639,8 @@ def __init__(self, defaults=None, dict_type=_default_dict, comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, - interpolation=_UNSET, converters=_UNSET): + interpolation=_UNSET, converters=_UNSET, + allow_unnamed_section=False,): self._dict = dict_type self._sections = self._dict() @@ -555,8 +659,10 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._comment_prefixes = tuple(comment_prefixes or ()) - self._inline_comment_prefixes = tuple(inline_comment_prefixes or ()) + self._prefixes = types.SimpleNamespace( + full=tuple(comment_prefixes or ()), + inline=tuple(inline_comment_prefixes or ()), + ) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values @@ -575,6 +681,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, self._converters.update(converters) if defaults: self._read_defaults(defaults) + self._allow_unnamed_section = allow_unnamed_section def defaults(self): return self._defaults @@ -593,6 +700,10 @@ def add_section(self, section): if section == self.default_section: raise ValueError('Invalid section name: %r' % section) + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + if section in self._sections: raise DuplicateSectionError(section) self._sections[section] = self._dict() @@ -848,13 +959,19 @@ def write(self, fp, space_around_delimiters=True): if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) + if UNNAMED_SECTION in self._sections: + self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True) + for section in self._sections: + if section is UNNAMED_SECTION: + continue self._write_section(fp, section, self._sections[section].items(), d) - def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp`.""" - fp.write("[{}]\n".format(section_name)) + def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): + """Write a single section to the specified 'fp'.""" + if not unnamed: + fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, value) @@ -940,110 +1057,117 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - elements_added = set() - cursect = None # None, or a dictionary - sectname = None - optname = None - lineno = 0 - indent_level = 0 - e = None # None, or an exception - for lineno, line in enumerate(fp, start=1): - comment_start = sys.maxsize - # strip inline comments - inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} - while comment_start == sys.maxsize and inline_prefixes: - next_prefixes = {} - for prefix, index in inline_prefixes.items(): - index = line.find(prefix, index+1) - if index == -1: - continue - next_prefixes[prefix] = index - if index == 0 or (index > 0 and line[index-1].isspace()): - comment_start = min(comment_start, index) - inline_prefixes = next_prefixes - # strip full line comments - for prefix in self._comment_prefixes: - if line.strip().startswith(prefix): - comment_start = 0 - break - if comment_start == sys.maxsize: - comment_start = None - value = line[:comment_start].strip() - if not value: + + try: + ParsingError._raise_all(self._read_inner(fp, fpname)) + finally: + self._join_multiline_values() + + def _read_inner(self, fp, fpname): + st = _ReadState() + + Line = functools.partial(_Line, prefixes=self._prefixes) + for st.lineno, line in enumerate(map(Line, fp), start=1): + if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no # comment on the line - if (comment_start is None and - cursect is not None and - optname and - cursect[optname] is not None): - cursect[optname].append('') # newlines added at join + if (not line.has_comments and + st.cursect is not None and + st.optname and + st.cursect[st.optname] is not None): + st.cursect[st.optname].append('') # newlines added at join else: # empty line marks end of value - indent_level = sys.maxsize + st.indent_level = sys.maxsize continue - # continuation line? + first_nonspace = self.NONSPACECRE.search(line) - cur_indent_level = first_nonspace.start() if first_nonspace else 0 - if (cursect is not None and optname and - cur_indent_level > indent_level): - cursect[optname].append(value) - # a section header or option header? - else: - indent_level = cur_indent_level - # is it a section header? - mo = self.SECTCRE.match(value) - if mo: - sectname = mo.group('header') - if sectname in self._sections: - if self._strict and sectname in elements_added: - raise DuplicateSectionError(sectname, fpname, - lineno) - cursect = self._sections[sectname] - elements_added.add(sectname) - elif sectname == self.default_section: - cursect = self._defaults - else: - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - # So sections can't start with a continuation line - optname = None - # no section header in the file? - elif cursect is None: - raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? - else: - mo = self._optcre.match(value) - if mo: - optname, vi, optval = mo.group('option', 'vi', 'value') - if not optname: - e = self._handle_error(e, fpname, lineno, line) - optname = self.optionxform(optname.rstrip()) - if (self._strict and - (sectname, optname) in elements_added): - raise DuplicateOptionError(sectname, optname, - fpname, lineno) - elements_added.add((sectname, optname)) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - optval = optval.strip() - cursect[optname] = [optval] - else: - # valueless option handling - cursect[optname] = None - else: - # a non-fatal parsing error occurred. set up the - # exception but keep going. the exception will be - # raised at the end of the file and will contain a - # list of all bogus lines - e = self._handle_error(e, fpname, lineno, line) - self._join_multiline_values() - # if any parsing errors occurred, raise an exception - if e: - raise e + st.cur_indent_level = first_nonspace.start() if first_nonspace else 0 + + if self._handle_continuation_line(st, line, fpname): + continue + + self._handle_rest(st, line, fpname) + + return st.errors + + def _handle_continuation_line(self, st, line, fpname): + # continuation line? + is_continue = (st.cursect is not None and st.optname and + st.cur_indent_level > st.indent_level) + if is_continue: + if st.cursect[st.optname] is None: + raise MultilineContinuationError(fpname, st.lineno, line) + st.cursect[st.optname].append(line.clean) + return is_continue + + def _handle_rest(self, st, line, fpname): + # a section header or option header? + if self._allow_unnamed_section and st.cursect is None: + st.sectname = UNNAMED_SECTION + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + + st.indent_level = st.cur_indent_level + # is it a section header? + mo = self.SECTCRE.match(line.clean) + + if not mo and st.cursect is None: + raise MissingSectionHeaderError(fpname, st.lineno, line) + + self._handle_header(st, mo, fpname) if mo else self._handle_option(st, line, fpname) + + def _handle_header(self, st, mo, fpname): + st.sectname = mo.group('header') + if st.sectname in self._sections: + if self._strict and st.sectname in st.elements_added: + raise DuplicateSectionError(st.sectname, fpname, + st.lineno) + st.cursect = self._sections[st.sectname] + st.elements_added.add(st.sectname) + elif st.sectname == self.default_section: + st.cursect = self._defaults + else: + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + # So sections can't start with a continuation line + st.optname = None + + def _handle_option(self, st, line, fpname): + # an option line? + st.indent_level = st.cur_indent_level + + mo = self._optcre.match(line.clean) + if not mo: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + st.errors.append(ParsingError(fpname, st.lineno, line)) + return + + st.optname, vi, optval = mo.group('option', 'vi', 'value') + if not st.optname: + st.errors.append(ParsingError(fpname, st.lineno, line)) + st.optname = self.optionxform(st.optname.rstrip()) + if (self._strict and + (st.sectname, st.optname) in st.elements_added): + raise DuplicateOptionError(st.sectname, st.optname, + fpname, st.lineno) + st.elements_added.add((st.sectname, st.optname)) + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + optval = optval.strip() + st.cursect[st.optname] = [optval] + else: + # valueless option handling + st.cursect[st.optname] = None def _join_multiline_values(self): defaults = self.default_section, self._defaults @@ -1063,12 +1187,6 @@ def _read_defaults(self, defaults): for key, value in defaults.items(): self._defaults[self.optionxform(key)] = value - def _handle_error(self, exc, fpname, lineno, line): - if not exc: - exc = ParsingError(fpname) - exc.append(lineno, repr(line)) - return exc - def _unify_values(self, section, vars): """Create a sequence of lookups with 'vars' taking priority over the 'section' which takes priority over the DEFAULTSECT. @@ -1097,20 +1215,20 @@ def _convert_to_boolean(self, value): return self.BOOLEAN_STATES[value.lower()] def _validate_value_types(self, *, section="", option="", value=""): - """Raises a TypeError for non-string values. + """Raises a TypeError for illegal non-string values. - The only legal non-string value if we allow valueless - options is None, so we need to check if the value is a - string if: - - we do not allow valueless options, or - - we allow valueless options but the value is not None + Legal non-string values are UNNAMED_SECTION and falsey values if + they are allowed. For compatibility reasons this method is not used in classic set() for RawConfigParsers. It is invoked in every case for mapping protocol access and in ConfigParser.set(). """ - if not isinstance(section, str): - raise TypeError("section names must be strings") + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + elif not isinstance(section, str): + raise TypeError("section names must be strings or UNNAMED_SECTION") if not isinstance(option, str): raise TypeError("option keys must be strings") if not self._allow_no_value or value: diff --git a/Lib/copy.py b/Lib/copy.py index a69bc4e78c20b3..f27e109973cfb7 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -4,8 +4,9 @@ import copy - x = copy.copy(y) # make a shallow copy of y - x = copy.deepcopy(y) # make a deep copy of y + x = copy.copy(y) # make a shallow copy of y + x = copy.deepcopy(y) # make a deep copy of y + x = copy.replace(y, a=1, b=2) # new object with fields replaced, as defined by `__replace__` For module specific errors, copy.Error is raised. @@ -56,7 +57,7 @@ class Error(Exception): pass error = Error # backward compatibility -__all__ = ["Error", "copy", "deepcopy"] +__all__ = ["Error", "copy", "deepcopy", "replace"] def copy(x): """Shallow copy operation on arbitrary Python objects. @@ -105,7 +106,7 @@ def _copy_immutable(x): bytes, frozenset, type, range, slice, property, types.BuiltinFunctionType, types.EllipsisType, types.NotImplementedType, types.FunctionType, types.CodeType, - weakref.ref): + weakref.ref, super): d[t] = _copy_immutable d[list] = list.copy @@ -121,6 +122,11 @@ def deepcopy(x, memo=None, _nil=[]): See the module's __doc__ string for more info. """ + cls = type(x) + + if cls in _atomic_types: + return x + d = id(x) if memo is None: memo = {} @@ -129,14 +135,12 @@ def deepcopy(x, memo=None, _nil=[]): if y is not _nil: return y - cls = type(x) - copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: if issubclass(cls, type): - y = _deepcopy_atomic(x, memo) + y = x # atomic copy else: copier = getattr(x, "__deepcopy__", None) if copier is not None: @@ -167,26 +171,12 @@ def deepcopy(x, memo=None, _nil=[]): _keep_alive(x, memo) # Make sure x lives at least as long as d return y +_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType, + int, float, bool, complex, bytes, str, types.CodeType, type, range, + types.BuiltinFunctionType, types.FunctionType, weakref.ref, property} + _deepcopy_dispatch = d = {} -def _deepcopy_atomic(x, memo): - return x -d[types.NoneType] = _deepcopy_atomic -d[types.EllipsisType] = _deepcopy_atomic -d[types.NotImplementedType] = _deepcopy_atomic -d[int] = _deepcopy_atomic -d[float] = _deepcopy_atomic -d[bool] = _deepcopy_atomic -d[complex] = _deepcopy_atomic -d[bytes] = _deepcopy_atomic -d[str] = _deepcopy_atomic -d[types.CodeType] = _deepcopy_atomic -d[type] = _deepcopy_atomic -d[range] = _deepcopy_atomic -d[types.BuiltinFunctionType] = _deepcopy_atomic -d[types.FunctionType] = _deepcopy_atomic -d[weakref.ref] = _deepcopy_atomic -d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] diff --git a/Lib/copyreg.py b/Lib/copyreg.py index 578392409b403c..17c5dde67c887c 100644 --- a/Lib/copyreg.py +++ b/Lib/copyreg.py @@ -36,6 +36,11 @@ def pickle_union(obj): pickle(type(int | str), pickle_union) +def pickle_super(obj): + return super, (obj.__thisclass__, obj.__self__) + +pickle(super, pickle_super) + # Support for pickling new-style objects def _reconstructor(cls, base, state): diff --git a/Lib/csv.py b/Lib/csv.py index 75e35b23236795..cd202659873811 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -47,7 +47,7 @@ class excel: field contains either the quotechar or the delimiter csv.QUOTE_ALL means that quotes are always placed around fields. csv.QUOTE_NONNUMERIC means that quotes are always placed around - fields which do not parse as integers or floating point + fields which do not parse as integers or floating-point numbers. csv.QUOTE_STRINGS means that quotes are always placed around fields which are strings. Note that the Python value None diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 141142a57dcb3e..4a368f02d851c5 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -205,6 +205,18 @@ class c_longdouble(_SimpleCData): if sizeof(c_longdouble) == sizeof(c_double): c_longdouble = c_double +try: + class c_double_complex(_SimpleCData): + _type_ = "C" + _check_size(c_double_complex) + class c_float_complex(_SimpleCData): + _type_ = "E" + _check_size(c_float_complex) + class c_longdouble_complex(_SimpleCData): + _type_ = "F" +except AttributeError: + pass + if _calcsize("l") == _calcsize("q"): # if long and long long have the same size, make c_longlong an alias for c_long c_longlong = c_long @@ -314,8 +326,6 @@ def SetPointerType(pointer, cls): del _pointer_type_cache[id(pointer)] def ARRAY(typ, len): - import warnings - warnings._deprecated("ctypes.ARRAY", remove=(3, 15)) return typ * len ################################################################ @@ -348,6 +358,17 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, winmode=None): if name: name = _os.fspath(name) + + # If the filename that has been provided is an iOS/tvOS/watchOS + # .fwork file, dereference the location to the true origin of the + # binary. + if name.endswith(".fwork"): + with open(name) as f: + name = _os.path.join( + _os.path.dirname(_sys.executable), + f.read().strip() + ) + self._name = name flags = self._func_flags_ if use_errno: @@ -468,6 +489,8 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) +elif _sys.platform == "android": + pythonapi = PyDLL("libpython%d.%d.so" % _sys.version_info[:2]) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) else: @@ -521,9 +544,9 @@ def cast(obj, typ): _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) def string_at(ptr, size=-1): - """string_at(addr[, size]) -> string + """string_at(ptr[, size]) -> string - Return the string at addr.""" + Return the byte string at void *ptr.""" return _string_at(ptr, size) try: @@ -533,9 +556,9 @@ def string_at(ptr, size=-1): else: _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string + """wstring_at(ptr[, size]) -> string - Return the string at addr.""" + Return the wide-character string at void *ptr.""" return _wstring_at(ptr, size) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py new file mode 100644 index 00000000000000..e30db598ab22e1 --- /dev/null +++ b/Lib/ctypes/_layout.py @@ -0,0 +1,337 @@ +"""Python implementation of computing the layout of a struct/union + +This code is internal and tightly coupled to the C part. The interface +may change at any time. +""" + +import sys +import warnings +import struct + +from _ctypes import CField, buffer_info +import ctypes + +def round_down(n, multiple): + assert n >= 0 + assert multiple > 0 + return (n // multiple) * multiple + +def round_up(n, multiple): + assert n >= 0 + assert multiple > 0 + return ((n + multiple - 1) // multiple) * multiple + +def LOW_BIT(offset): + return offset & 0xFFFF + +def NUM_BITS(bitsize): + return bitsize >> 16 + +def BUILD_SIZE(bitsize, offset): + assert 0 <= offset, offset + assert offset <= 0xFFFF, offset + # We don't support zero length bitfields. + # And GET_BITFIELD uses NUM_BITS(size) == 0, + # to figure out whether we are handling a bitfield. + assert bitsize > 0, bitsize + result = (bitsize << 16) + offset + assert bitsize == NUM_BITS(result), (bitsize, result) + assert offset == LOW_BIT(result), (offset, result) + return result + +def build_size(bit_size, bit_offset, big_endian, type_size): + if big_endian: + return BUILD_SIZE(bit_size, 8 * type_size - bit_offset - bit_size) + return BUILD_SIZE(bit_size, bit_offset) + +_INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 + + +class StructUnionLayout: + def __init__(self, fields, size, align, format_spec): + # sequence of CField objects + self.fields = fields + + # total size of the aggregate (rounded up to alignment) + self.size = size + + # total alignment requirement of the aggregate + self.align = align + + # buffer format specification (as a string, UTF-8 but bes + # kept ASCII-only) + self.format_spec = format_spec + + +def get_layout(cls, input_fields, is_struct, base): + """Return a StructUnionLayout for the given class. + + Called by PyCStructUnionType_update_stginfo when _fields_ is assigned + to a class. + """ + # Currently there are two modes, selectable using the '_layout_' attribute: + # + # 'gcc-sysv' mode places fields one after another, bit by bit. + # But "each bit field must fit within a single object of its specified + # type" (GCC manual, section 15.8 "Bit Field Packing"). When it doesn't, + # we insert a few bits of padding to avoid that. + # + # 'ms' mode works similar except for bitfield packing. Adjacent + # bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit + # if the integral types are the same size and if the next bit-field fits + # into the current allocation unit without crossing the boundary imposed + # by the common alignment requirements of the bit-fields. + # + # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields + # for details. + + # We do not support zero length bitfields (we use bitsize != 0 + # elsewhere to indicate a bitfield). Here, non-bitfields have bit_size + # set to size*8. + + # For clarity, variables that count bits have `bit` in their names. + + layout = getattr(cls, '_layout_', None) + if layout is None: + if sys.platform == 'win32' or getattr(cls, '_pack_', None): + gcc_layout = False + else: + gcc_layout = True + elif layout == 'ms': + gcc_layout = False + elif layout == 'gcc-sysv': + gcc_layout = True + else: + raise ValueError(f'unknown _layout_: {layout!r}') + + align = getattr(cls, '_align_', 1) + if align < 0: + raise ValueError('_align_ must be a non-negative integer') + elif align == 0: + # Setting `_align_ = 0` amounts to using the default alignment + align == 1 + + if base: + align = max(ctypes.alignment(base), align) + + swapped_bytes = hasattr(cls, '_swappedbytes_') + if swapped_bytes: + big_endian = sys.byteorder == 'little' + else: + big_endian = sys.byteorder == 'big' + + pack = getattr(cls, '_pack_', None) + if pack is not None: + try: + pack = int(pack) + except (TypeError, ValueError): + raise ValueError("_pack_ must be an integer") + if pack < 0: + raise ValueError("_pack_ must be a non-negative integer") + if pack > _INT_MAX: + raise ValueError("_pack_ too big") + if gcc_layout: + raise ValueError('_pack_ is not compatible with gcc-sysv layout') + + result_fields = [] + + if is_struct: + format_spec_parts = ["T{"] + else: + format_spec_parts = ["B"] + + last_field_bit_size = 0 # used in MS layout only + + # `8 * next_byte_offset + next_bit_offset` points to where the + # next field would start. + next_bit_offset = 0 + next_byte_offset = 0 + + # size if this was a struct (sum of field sizes, plus padding) + struct_size = 0 + # max of field sizes; only meaningful for unions + union_size = 0 + + if base: + struct_size = ctypes.sizeof(base) + if gcc_layout: + next_bit_offset = struct_size * 8 + else: + next_byte_offset = struct_size + + last_size = struct_size + for i, field in enumerate(input_fields): + if not is_struct: + # Unions start fresh each time + last_field_bit_size = 0 + next_bit_offset = 0 + next_byte_offset = 0 + + # Unpack the field + field = tuple(field) + try: + name, ctype = field + except (ValueError, TypeError): + try: + name, ctype, bit_size = field + except (ValueError, TypeError) as exc: + raise ValueError( + '_fields_ must be a sequence of (name, C type) pairs ' + + 'or (name, C type, bit size) triples') from exc + is_bitfield = True + if bit_size <= 0: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + type_size = ctypes.sizeof(ctype) + if bit_size > type_size * 8: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + else: + is_bitfield = False + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 + + type_bit_size = type_size * 8 + type_align = ctypes.alignment(ctype) or 1 + type_bit_align = type_align * 8 + + if gcc_layout: + # We don't use next_byte_offset here + assert pack is None + assert next_byte_offset == 0 + + # Determine whether the bit field, if placed at the next + # free bit, fits within a single object of its specified type. + # That is: determine a "slot", sized & aligned for the + # specified type, which contains the bitfield's beginning: + slot_start_bit = round_down(next_bit_offset, type_bit_align) + slot_end_bit = slot_start_bit + type_bit_size + # And see if it also contains the bitfield's last bit: + field_end_bit = next_bit_offset + bit_size + if field_end_bit > slot_end_bit: + # It doesn't: add padding (bump up to the next + # alignment boundary) + next_bit_offset = round_up(next_bit_offset, type_bit_align) + + offset = round_down(next_bit_offset, type_bit_align) // 8 + if is_bitfield: + effective_bit_offset = next_bit_offset - 8 * offset + size = build_size(bit_size, effective_bit_offset, + big_endian, type_size) + assert effective_bit_offset <= type_bit_size + else: + assert offset == next_bit_offset / 8 + size = type_size + + next_bit_offset += bit_size + struct_size = round_up(next_bit_offset, 8) // 8 + else: + if pack: + type_align = min(pack, type_align) + + # next_byte_offset points to end of current bitfield. + # next_bit_offset is generally non-positive, + # and 8 * next_byte_offset + next_bit_offset points just behind + # the end of the last field we placed. + if ( + (0 < next_bit_offset + bit_size) + or (type_bit_size != last_field_bit_size) + ): + # Close the previous bitfield (if any) + # and start a new bitfield + next_byte_offset = round_up(next_byte_offset, type_align) + + next_byte_offset += type_size + + last_field_bit_size = type_bit_size + # Reminder: 8 * (next_byte_offset) + next_bit_offset + # points to where we would start a new field, namely + # just behind where we placed the last field plus an + # allowance for alignment. + next_bit_offset = -last_field_bit_size + + assert type_bit_size == last_field_bit_size + + offset = next_byte_offset - last_field_bit_size // 8 + if is_bitfield: + assert 0 <= (last_field_bit_size + next_bit_offset) + size = build_size(bit_size, + last_field_bit_size + next_bit_offset, + big_endian, type_size) + else: + size = type_size + if type_bit_size: + assert (last_field_bit_size + next_bit_offset) < type_bit_size + + next_bit_offset += bit_size + struct_size = next_byte_offset + + assert (not is_bitfield) or (LOW_BIT(size) <= size * 8) + + # Add the format spec parts + if is_struct: + padding = offset - last_size + format_spec_parts.append(padding_spec(padding)) + + fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) + + if bf_shape: + format_spec_parts.extend(( + "(", + ','.join(str(n) for n in bf_shape), + ")", + )) + + if fieldfmt is None: + fieldfmt = "B" + if isinstance(name, bytes): + # a bytes name would be rejected later, but we check early + # to avoid a BytesWarning with `python -bb` + raise TypeError( + "field {name!r}: name must be a string, not bytes") + format_spec_parts.append(f"{fieldfmt}:{name}:") + + result_fields.append(CField( + name=name, + type=ctype, + size=size, + offset=offset, + bit_size=bit_size if is_bitfield else None, + index=i, + )) + if is_bitfield and not gcc_layout: + assert type_bit_size > 0 + + align = max(align, type_align) + last_size = struct_size + if not is_struct: + union_size = max(struct_size, union_size) + + if is_struct: + total_size = struct_size + else: + total_size = union_size + + # Adjust the size according to the alignment requirements + aligned_size = round_up(total_size, align) + + # Finish up the format spec + if is_struct: + padding = aligned_size - total_size + format_spec_parts.append(padding_spec(padding)) + format_spec_parts.append("}") + + return StructUnionLayout( + fields=result_fields, + size=aligned_size, + align=align, + format_spec="".join(format_spec_parts), + ) + + +def padding_spec(padding): + if padding <= 0: + return "" + if padding == 1: + return "x" + return f"{padding}x" diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index c550883e7c7d4b..117bf06cb01013 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,7 +67,7 @@ def find_library(name): return fname return None -elif os.name == "posix" and sys.platform == "darwin": +elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): possible = ['lib%s.dylib' % name, @@ -89,6 +89,15 @@ def find_library(name): from ctypes._aix import find_library +elif sys.platform == "android": + def find_library(name): + directory = "/system/lib" + if "64" in os.uname().machine: + directory += "64" + + fname = f"{directory}/lib{name}.so" + return fname if os.path.isfile(fname) else None + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile diff --git a/Lib/ctypes/wintypes.py b/Lib/ctypes/wintypes.py index 9c4e721438aad5..4beba0d19513e2 100644 --- a/Lib/ctypes/wintypes.py +++ b/Lib/ctypes/wintypes.py @@ -63,10 +63,16 @@ def __repr__(self): HBITMAP = HANDLE HBRUSH = HANDLE HCOLORSPACE = HANDLE +HCONV = HANDLE +HCONVLIST = HANDLE +HCURSOR = HANDLE HDC = HANDLE +HDDEDATA = HANDLE HDESK = HANDLE +HDROP = HANDLE HDWP = HANDLE HENHMETAFILE = HANDLE +HFILE = INT HFONT = HANDLE HGDIOBJ = HANDLE HGLOBAL = HANDLE @@ -82,9 +88,11 @@ def __repr__(self): HMONITOR = HANDLE HPALETTE = HANDLE HPEN = HANDLE +HRESULT = LONG HRGN = HANDLE HRSRC = HANDLE HSTR = HANDLE +HSZ = HANDLE HTASK = HANDLE HWINSTA = HANDLE HWND = HANDLE diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py index 69270bfcd2b205..6165fe6c9875c0 100644 --- a/Lib/curses/__init__.py +++ b/Lib/curses/__init__.py @@ -53,7 +53,7 @@ def start_color(): try: has_key except NameError: - from .has_key import has_key + from .has_key import has_key # noqa: F401 # Wrapper for the entire curses-based application. Runs a function which # should be the rest of your curses-based application. If the application diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3335821f3f32e9..7a24f8a9e5ccee 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -4,11 +4,10 @@ import types import inspect import keyword -import functools import itertools +import annotationlib import abc -import _thread -from types import FunctionType, GenericAlias +from reprlib import recursive_repr __all__ = ['dataclass', @@ -245,25 +244,6 @@ def __repr__(self): property, }) -# This function's logic is copied from "recursive_repr" function in -# reprlib module to avoid dependency. -def _recursive_repr(user_function): - # Decorator to make a repr function return "..." for a recursive - # call. - repr_running = set() - - @functools.wraps(user_function) - def wrapper(self): - key = id(self), _thread.get_ident() - if key in repr_running: - return '...' - repr_running.add(key) - try: - result = user_function(self) - finally: - repr_running.discard(key) - return result - return wrapper class InitVar: __slots__ = ('type', ) @@ -303,11 +283,12 @@ class Field: 'compare', 'metadata', 'kw_only', + 'doc', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata, kw_only): + metadata, kw_only, doc): self.name = None self.type = None self.default = default @@ -320,9 +301,10 @@ def __init__(self, default, default_factory, init, repr, hash, compare, if metadata is None else types.MappingProxyType(metadata)) self.kw_only = kw_only + self.doc = doc self._field_type = None - @_recursive_repr + @recursive_repr() def __repr__(self): return ('Field(' f'name={self.name!r},' @@ -335,6 +317,7 @@ def __repr__(self): f'compare={self.compare!r},' f'metadata={self.metadata!r},' f'kw_only={self.kw_only!r},' + f'doc={self.doc!r},' f'_field_type={self._field_type}' ')') @@ -353,7 +336,7 @@ def __set_name__(self, owner, name): # it. func(self.default, owner, name) - __class_getitem__ = classmethod(GenericAlias) + __class_getitem__ = classmethod(types.GenericAlias) class _DataclassParams: @@ -402,7 +385,7 @@ def __repr__(self): # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None, kw_only=MISSING): + hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a @@ -414,7 +397,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, comparison functions. metadata, if specified, must be a mapping which is stored but not otherwise examined by dataclass. If kw_only is true, the field will become a keyword-only parameter to - __init__(). + __init__(). doc is an optional docstring for this field. It is an error to specify both default and default_factory. """ @@ -422,7 +405,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata, kw_only) + metadata, kw_only, doc) def _fields_in_init_order(fields): @@ -446,32 +429,95 @@ def _tuple_str(obj_name, fields): return f'({",".join([f"{obj_name}.{f.name}" for f in fields])},)' -def _create_fn(name, args, body, *, globals=None, locals=None, - return_type=MISSING): - # Note that we may mutate locals. Callers beware! - # The only callers are internal to this module, so no - # worries about external callers. - if locals is None: - locals = {} - return_annotation = '' - if return_type is not MISSING: - locals['__dataclass_return_type__'] = return_type - return_annotation = '->__dataclass_return_type__' - args = ','.join(args) - body = '\n'.join(f' {b}' for b in body) - - # Compute the text of the entire function. - txt = f' def {name}({args}){return_annotation}:\n{body}' - - # Free variables in exec are resolved in the global namespace. - # The global namespace we have is user-provided, so we can't modify it for - # our purposes. So we put the things we need into locals and introduce a - # scope to allow the function we're creating to close over them. - local_vars = ', '.join(locals.keys()) - txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" - ns = {} - exec(txt, globals, ns) - return ns['__create_fn__'](**locals) +class _FuncBuilder: + def __init__(self, globals): + self.names = [] + self.src = [] + self.globals = globals + self.locals = {} + self.overwrite_errors = {} + self.unconditional_adds = {} + + def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, + overwrite_error=False, unconditional_add=False, decorator=None): + if locals is not None: + self.locals.update(locals) + + # Keep track if this method is allowed to be overwritten if it already + # exists in the class. The error is method-specific, so keep it with + # the name. We'll use this when we generate all of the functions in + # the add_fns_to_class call. overwrite_error is either True, in which + # case we'll raise an error, or it's a string, in which case we'll + # raise an error and append this string. + if overwrite_error: + self.overwrite_errors[name] = overwrite_error + + # Should this function always overwrite anything that's already in the + # class? The default is to not overwrite a function that already + # exists. + if unconditional_add: + self.unconditional_adds[name] = True + + self.names.append(name) + + if return_type is not MISSING: + self.locals[f'__dataclass_{name}_return_type__'] = return_type + return_annotation = f'->__dataclass_{name}_return_type__' + else: + return_annotation = '' + args = ','.join(args) + body = '\n'.join(body) + + # Compute the text of the entire function, add it to the text we're generating. + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') + + def add_fns_to_class(self, cls): + # The source to all of the functions we're generating. + fns_src = '\n'.join(self.src) + + # The locals they use. + local_vars = ','.join(self.locals.keys()) + + # The names of all of the functions, used for the return value of the + # outer function. Need to handle the 0-tuple specially. + if len(self.names) == 0: + return_names = '()' + else: + return_names =f'({",".join(self.names)},)' + + # txt is the entire function we're going to execute, including the + # bodies of the functions we're defining. Here's a greatly simplified + # version: + # def __create_fn__(): + # def __init__(self, x, y): + # self.x = x + # self.y = y + # @recursive_repr + # def __repr__(self): + # return f"cls(x={self.x!r},y={self.y!r})" + # return __init__,__repr__ + + txt = f"def __create_fn__({local_vars}):\n{fns_src}\n return {return_names}" + ns = {} + exec(txt, self.globals, ns) + fns = ns['__create_fn__'](**self.locals) + + # Now that we've generated the functions, assign them into cls. + for name, fn in zip(self.names, fns): + fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + if self.unconditional_adds.get(name, False): + setattr(cls, name, fn) + else: + already_exists = _set_new_attribute(cls, name, fn) + + # See if it's an error to overwrite this particular function. + if already_exists and (msg_extra := self.overwrite_errors.get(name)): + error_msg = (f'Cannot overwrite attribute {fn.__name__} ' + f'in class {cls.__name__}') + if not msg_extra is True: + error_msg = f'{error_msg} {msg_extra}' + + raise TypeError(error_msg) def _field_assign(frozen, name, value, self_name): @@ -482,8 +528,8 @@ def _field_assign(frozen, name, value, self_name): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'__dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' - return f'{self_name}.{name}={value}' + return f' __dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' + return f' {self_name}.{name}={value}' def _field_init(f, frozen, globals, self_name, slots): @@ -566,7 +612,7 @@ def _init_param(f): def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, - self_name, globals, slots): + self_name, func_builder, slots): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -585,11 +631,11 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, raise TypeError(f'non-default argument {f.name!r} ' f'follows default argument {seen_default.name!r}') - locals = {f'__dataclass_type_{f.name}__': f.type for f in fields} - locals.update({ - '__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, - '__dataclass_builtins_object__': object, - }) + locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, + **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object, + } + } body_lines = [] for f in fields: @@ -603,89 +649,52 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, if has_post_init: params_str = ','.join(f.name for f in fields if f._field_type is _FIELD_INITVAR) - body_lines.append(f'{self_name}.{_POST_INIT_NAME}({params_str})') + body_lines.append(f' {self_name}.{_POST_INIT_NAME}({params_str})') # If no body lines, use 'pass'. if not body_lines: - body_lines = ['pass'] + body_lines = [' pass'] _init_params = [_init_param(f) for f in std_fields] if kw_only_fields: # Add the keyword-only args. Because the * can only be added if # there's at least one keyword-only arg, there needs to be a test here - # (instead of just concatenting the lists together). + # (instead of just concatenating the lists together). _init_params += ['*'] _init_params += [_init_param(f) for f in kw_only_fields] - return _create_fn('__init__', - [self_name] + _init_params, - body_lines, - locals=locals, - globals=globals, - return_type=None) - - -def _repr_fn(fields, globals): - fn = _create_fn('__repr__', - ('self',), - ['return f"{self.__class__.__qualname__}(' + - ', '.join([f"{f.name}={{self.{f.name}!r}}" - for f in fields]) + - ')"'], - globals=globals) - return _recursive_repr(fn) - - -def _frozen_get_del_attr(cls, fields, globals): + func_builder.add_fn('__init__', + [self_name] + _init_params, + body_lines, + locals=locals, + return_type=None) + + +def _frozen_get_del_attr(cls, fields, func_builder): locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} condition = 'type(self) is cls' if fields: condition += ' or name in {' + ', '.join(repr(f.name) for f in fields) + '}' - return (_create_fn('__setattr__', - ('self', 'name', 'value'), - (f'if {condition}:', - ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', - f'super(cls, self).__setattr__(name, value)'), - locals=locals, - globals=globals), - _create_fn('__delattr__', - ('self', 'name'), - (f'if {condition}:', - ' raise FrozenInstanceError(f"cannot delete field {name!r}")', - f'super(cls, self).__delattr__(name)'), - locals=locals, - globals=globals), - ) - -def _cmp_fn(name, op, self_tuple, other_tuple, globals): - # Create a comparison function. If the fields in the object are - # named 'x' and 'y', then self_tuple is the string - # '(self.x,self.y)' and other_tuple is the string - # '(other.x,other.y)'. - - return _create_fn(name, - ('self', 'other'), - [ 'if other.__class__ is self.__class__:', - f' return {self_tuple}{op}{other_tuple}', - 'return NotImplemented'], - globals=globals) - - -def _hash_fn(fields, globals): - self_tuple = _tuple_str('self', fields) - return _create_fn('__hash__', - ('self',), - [f'return hash({self_tuple})'], - globals=globals) + func_builder.add_fn('__setattr__', + ('self', 'name', 'value'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f' super(cls, self).__setattr__(name, value)'), + locals=locals, + overwrite_error=True) + func_builder.add_fn('__delattr__', + ('self', 'name'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f' super(cls, self).__delattr__(name)'), + locals=locals, + overwrite_error=True) def _is_classvar(a_type, typing): - # This test uses a typing internal class, but it's the best way to - # test if this is a ClassVar. return (a_type is typing.ClassVar - or (type(a_type) is typing._GenericAlias - and a_type.__origin__ is typing.ClassVar)) + or (typing.get_origin(a_type) is typing.ClassVar)) def _is_initvar(a_type, dataclasses): @@ -854,19 +863,11 @@ def _get_field(cls, a_name, a_type, default_kw_only): return f -def _set_qualname(cls, value): - # Ensure that the functions returned from _create_fn uses the proper - # __qualname__ (the class they belong to). - if isinstance(value, FunctionType): - value.__qualname__ = f"{cls.__qualname__}.{value.__name__}" - return value - def _set_new_attribute(cls, name, value): # Never overwrites an existing attribute. Returns True if the # attribute already exists. if name in cls.__dict__: return True - _set_qualname(cls, value) setattr(cls, name, value) return False @@ -876,14 +877,22 @@ def _set_new_attribute(cls, name, value): # take. The common case is to do nothing, so instead of providing a # function that is a no-op, use None to signify that. -def _hash_set_none(cls, fields, globals): - return None +def _hash_set_none(cls, fields, func_builder): + # It's sort of a hack that I'm setting this here, instead of at + # func_builder.add_fns_to_class time, but since this is an exceptional case + # (it's not setting an attribute to a function, but to a scalar value), + # just do it directly here. I might come to regret this. + cls.__hash__ = None -def _hash_add(cls, fields, globals): +def _hash_add(cls, fields, func_builder): flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] - return _set_qualname(cls, _hash_fn(flds, globals)) + self_tuple = _tuple_str('self', flds) + func_builder.add_fn('__hash__', + ('self',), + [f' return hash({self_tuple})'], + unconditional_add=True) -def _hash_exception(cls, fields, globals): +def _hash_exception(cls, fields, func_builder): # Raise an exception. raise TypeError(f'Cannot overwrite attribute __hash__ ' f'in class {cls.__name__}') @@ -973,7 +982,8 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # actual default value. Pseudo-fields ClassVars and InitVars are # included, despite the fact that they're not real fields. That's # dealt with later. - cls_annotations = inspect.get_annotations(cls) + cls_annotations = annotationlib.get_annotations( + cls, format=annotationlib.Format.FORWARDREF) # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) where @@ -1061,24 +1071,26 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, (std_init_fields, kw_only_init_fields) = _fields_in_init_order(all_init_fields) + func_builder = _FuncBuilder(globals) + if init: # Does this class have a post-init function? has_post_init = hasattr(cls, _POST_INIT_NAME) - _set_new_attribute(cls, '__init__', - _init_fn(all_init_fields, - std_init_fields, - kw_only_init_fields, - frozen, - has_post_init, - # The name to use for the "self" - # param in __init__. Use "self" - # if possible. - '__dataclass_self__' if 'self' in fields - else 'self', - globals, - slots, - )) + _init_fn(all_init_fields, + std_init_fields, + kw_only_init_fields, + frozen, + has_post_init, + # The name to use for the "self" + # param in __init__. Use "self" + # if possible. + '__dataclass_self__' if 'self' in fields + else 'self', + func_builder, + slots, + ) + _set_new_attribute(cls, '__replace__', _replace) # Get the fields as a list, and include only real fields. This is @@ -1087,7 +1099,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if repr: flds = [f for f in field_list if f.repr] - _set_new_attribute(cls, '__repr__', _repr_fn(flds, globals)) + func_builder.add_fn('__repr__', + ('self',), + [' return f"{self.__class__.__qualname__}(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in flds]) + ')"'], + locals={'__dataclasses_recursive_repr': recursive_repr}, + decorator="@__dataclasses_recursive_repr()") if eq: # Create __eq__ method. There's no need for a __ne__ method, @@ -1095,14 +1113,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, cmp_fields = (field for field in field_list if field.compare) terms = [f'self.{field.name}==other.{field.name}' for field in cmp_fields] field_comparisons = ' and '.join(terms) or 'True' - body = [f'if other.__class__ is self.__class__:', - f' return {field_comparisons}', - f'return NotImplemented'] - func = _create_fn('__eq__', - ('self', 'other'), - body, - globals=globals) - _set_new_attribute(cls, '__eq__', func) + func_builder.add_fn('__eq__', + ('self', 'other'), + [ ' if self is other:', + ' return True', + ' if other.__class__ is self.__class__:', + f' return {field_comparisons}', + ' return NotImplemented']) if order: # Create and set the ordering methods. @@ -1114,18 +1131,19 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, ('__gt__', '>'), ('__ge__', '>='), ]: - if _set_new_attribute(cls, name, - _cmp_fn(name, op, self_tuple, other_tuple, - globals=globals)): - raise TypeError(f'Cannot overwrite attribute {name} ' - f'in class {cls.__name__}. Consider using ' - 'functools.total_ordering') + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + func_builder.add_fn(name, + ('self', 'other'), + [ ' if other.__class__ is self.__class__:', + f' return {self_tuple}{op}{other_tuple}', + ' return NotImplemented'], + overwrite_error='Consider using functools.total_ordering') if frozen: - for fn in _frozen_get_del_attr(cls, field_list, globals): - if _set_new_attribute(cls, fn.__name__, fn): - raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' - f'in class {cls.__name__}') + _frozen_get_del_attr(cls, field_list, func_builder) # Decide if/how we're going to create a hash function. hash_action = _hash_action[bool(unsafe_hash), @@ -1133,9 +1151,12 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, bool(frozen), has_explicit_hash] if hash_action: - # No need to call _set_new_attribute here, since by the time - # we're here the overwriting is unconditional. - cls.__hash__ = hash_action(cls, field_list, globals) + cls.__hash__ = hash_action(cls, field_list, func_builder) + + # Generate the methods and add them to the class. This needs to be done + # before the __doc__ logic below, since inspect will look at the __init__ + # signature. + func_builder.add_fns_to_class(cls) if not getattr(cls, '__doc__'): # Create a class doc-string. @@ -1148,7 +1169,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, cls.__doc__ = (cls.__name__ + text_sig) if match_args: - # I could probably compute this once + # I could probably compute this once. _set_new_attribute(cls, '__match_args__', tuple(f.name for f in std_init_fields)) @@ -1156,7 +1177,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if weakref_slot and not slots: raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen, weakref_slot) + cls = _add_slots(cls, frozen, weakref_slot, fields) abc.update_abstractmethods(cls) @@ -1179,8 +1200,17 @@ def _dataclass_setstate(self, state): def _get_slots(cls): match cls.__dict__.get('__slots__'): + # `__dictoffset__` and `__weakrefoffset__` can tell us whether + # the base type has dict/weakref slots, in a way that works correctly + # for both Python classes and C extension types. Extension types + # don't use `__slots__` for slot creation case None: - return + slots = [] + if getattr(cls, '__weakrefoffset__', -1) != 0: + slots.append('__weakref__') + if getattr(cls, '__dictoffset__', -1) != 0: + slots.append('__dict__') + yield from slots case str(slot): yield slot # Slots may be any iterable, but we cannot handle an iterator @@ -1191,9 +1221,56 @@ def _get_slots(cls): raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") -def _add_slots(cls, is_frozen, weakref_slot): - # Need to create a new class, since we can't set __slots__ - # after a class has been created. +def _update_func_cell_for__class__(f, oldcls, newcls): + # Returns True if we update a cell, else False. + if f is None: + # f will be None in the case of a property where not all of + # fget, fset, and fdel are used. Nothing to do in that case. + return False + try: + idx = f.__code__.co_freevars.index("__class__") + except ValueError: + # This function doesn't reference __class__, so nothing to do. + return False + # Fix the cell to point to the new class, if it's already pointing + # at the old class. I'm not convinced that the "is oldcls" test + # is needed, but other than performance can't hurt. + closure = f.__closure__[idx] + if closure.cell_contents is oldcls: + closure.cell_contents = newcls + return True + return False + + +def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + seen_docs = False + slots = {} + for slot in itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ): + doc = getattr(defined_fields.get(slot), 'doc', None) + if doc is not None: + seen_docs = True + slots.update({slot: doc}) + + # We only return dict if there's at least one doc member, + # otherwise we return tuple, which is the old default format. + if seen_docs: + return slots + return tuple(slots) + + +def _add_slots(cls, is_frozen, weakref_slot, defined_fields): + # Need to create a new class, since we can't set __slots__ after a + # class has been created, and the @dataclass decorator is called + # after the class is created. # Make sure __slots__ isn't already set. if '__slots__' in cls.__dict__: @@ -1206,17 +1283,9 @@ def _add_slots(cls, is_frozen, weakref_slot): inherited_slots = set( itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) ) - # The slots for our class. Remove slots from our base classes. Add - # '__weakref__' if weakref_slot was given, unless it is already present. - cls_dict["__slots__"] = tuple( - itertools.filterfalse( - inherited_slots.__contains__, - itertools.chain( - # gh-93521: '__weakref__' also needs to be filtered out if - # already present in inherited_slots - field_names, ('__weakref__',) if weakref_slot else () - ) - ), + + cls_dict["__slots__"] = _create_slots( + defined_fields, inherited_slots, field_names, weakref_slot, ) for field_name in field_names: @@ -1232,18 +1301,37 @@ def _add_slots(cls, is_frozen, weakref_slot): # And finally create the class. qualname = getattr(cls, '__qualname__', None) - cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict) if qualname is not None: - cls.__qualname__ = qualname + newcls.__qualname__ = qualname if is_frozen: # Need this for pickling frozen classes with slots. if '__getstate__' not in cls_dict: - cls.__getstate__ = _dataclass_getstate + newcls.__getstate__ = _dataclass_getstate if '__setstate__' not in cls_dict: - cls.__setstate__ = _dataclass_setstate - - return cls + newcls.__setstate__ = _dataclass_setstate + + # Fix up any closures which reference __class__. This is used to + # fix zero argument super so that it points to the correct class + # (the newly created one, which we're returning) and not the + # original class. We can break out of this loop as soon as we + # make an update, since all closures for a class will share a + # given cell. + for member in newcls.__dict__.values(): + # If this is a wrapped function, unwrap it. + member = inspect.unwrap(member) + + if isinstance(member, types.FunctionType): + if _update_func_cell_for__class__(member, cls, newcls): + break + elif isinstance(member, property): + if (_update_func_cell_for__class__(member.fget, cls, newcls) + or _update_func_cell_for__class__(member.fset, cls, newcls) + or _update_func_cell_for__class__(member.fdel, cls, newcls)): + break + + return newcls def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, @@ -1462,7 +1550,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False, module=None): + weakref_slot=False, module=None, decorator=dataclass): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1542,8 +1630,8 @@ def exec_body_callback(ns): if module is not None: cls.__module__ = module - # Apply the normal decorator. - return dataclass(cls, init=init, repr=repr, eq=eq, order=order, + # Apply the normal provided decorator. + return decorator(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, match_args=match_args, kw_only=kw_only, slots=slots, weakref_slot=weakref_slot) diff --git a/Lib/datetime.py b/Lib/datetime.py index a33d2d724cb33d..14f30556584e32 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1,9 +1,13 @@ +"""Specific date/time and related types. + +See https://data.iana.org/time-zones/tz-link.html for +time zone and DST data sources. +""" + try: from _datetime import * - from _datetime import __doc__ except ImportError: from _pydatetime import * - from _pydatetime import __doc__ __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index 8055d3769f9dd0..4fdbc54e74cfb6 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -5,7 +5,7 @@ import dbm d = dbm.open(file, 'w', 0o666) -The returned object is a dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the +The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb database object, dependent on the type of database being opened (determined by the whichdb function) in the case of an existing dbm. If the dbm does not exist and the create or new flag ('c' or 'n') was specified, the dbm type will be determined by the availability of @@ -38,7 +38,7 @@ class error(Exception): pass -_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] +_names = ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] _defaultmod = None _modules = {} @@ -164,6 +164,10 @@ def whichdb(filename): if len(s) != 4: return "" + # Check for SQLite3 header string. + if s16 == b"SQLite format 3\0": + return "dbm.sqlite3" + # Convert to 4-byte int in native byte order -- return "" if impossible try: (magic,) = struct.unpack("=l", s) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py new file mode 100644 index 00000000000000..7e0ae2a29e3a64 --- /dev/null +++ b/Lib/dbm/sqlite3.py @@ -0,0 +1,140 @@ +import os +import sqlite3 +from pathlib import Path +from contextlib import suppress, closing +from collections.abc import MutableMapping + +BUILD_TABLE = """ + CREATE TABLE IF NOT EXISTS Dict ( + key BLOB UNIQUE NOT NULL, + value BLOB NOT NULL + ) +""" +GET_SIZE = "SELECT COUNT (key) FROM Dict" +LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)" +STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))" +DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)" +ITER_KEYS = "SELECT key FROM Dict" + + +class error(OSError): + pass + + +_ERR_CLOSED = "DBM object has already been closed" +_ERR_REINIT = "DBM object does not support reinitialization" + + +def _normalize_uri(path): + path = Path(path) + uri = path.absolute().as_uri() + while "//" in uri: + uri = uri.replace("//", "/") + return uri + + +class _Database(MutableMapping): + + def __init__(self, path, /, *, flag, mode): + if hasattr(self, "_cx"): + raise error(_ERR_REINIT) + + path = os.fsdecode(path) + match flag: + case "r": + flag = "ro" + case "w": + flag = "rw" + case "c": + flag = "rwc" + Path(path).touch(mode=mode, exist_ok=True) + case "n": + flag = "rwc" + Path(path).unlink(missing_ok=True) + Path(path).touch(mode=mode) + case _: + raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', " + f"not {flag!r}") + + # We use the URI format when opening the database. + uri = _normalize_uri(path) + uri = f"{uri}?mode={flag}" + + try: + self._cx = sqlite3.connect(uri, autocommit=True, uri=True) + except sqlite3.Error as exc: + raise error(str(exc)) + + # This is an optimization only; it's ok if it fails. + with suppress(sqlite3.OperationalError): + self._cx.execute("PRAGMA journal_mode = wal") + + if flag == "rwc": + self._execute(BUILD_TABLE) + + def _execute(self, *args, **kwargs): + if not self._cx: + raise error(_ERR_CLOSED) + try: + return closing(self._cx.execute(*args, **kwargs)) + except sqlite3.Error as exc: + raise error(str(exc)) + + def __len__(self): + with self._execute(GET_SIZE) as cu: + row = cu.fetchone() + return row[0] + + def __getitem__(self, key): + with self._execute(LOOKUP_KEY, (key,)) as cu: + row = cu.fetchone() + if not row: + raise KeyError(key) + return row[0] + + def __setitem__(self, key, value): + self._execute(STORE_KV, (key, value)) + + def __delitem__(self, key): + with self._execute(DELETE_KEY, (key,)) as cu: + if not cu.rowcount: + raise KeyError(key) + + def __iter__(self): + try: + with self._execute(ITER_KEYS) as cu: + for row in cu: + yield row[0] + except sqlite3.Error as exc: + raise error(str(exc)) + + def close(self): + if self._cx: + self._cx.close() + self._cx = None + + def keys(self): + return list(super().keys()) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def open(filename, /, flag="r", mode=0o666): + """Open a dbm.sqlite3 database and return the dbm object. + + The 'filename' parameter is the name of the database file. + + The optional 'flag' parameter can be one of ...: + 'r' (default): open an existing database for read only access + 'w': open an existing database for read/write access + 'c': create a database if it does not exist; open for read/write access + 'n': always create a new, empty database; open for read/write access + + The optional 'mode' parameter is the Unix file access mode of the database; + only used when creating a new database. Default: 0o666. + """ + return _Database(filename, flag=flag, mode=mode) diff --git a/Lib/decimal.py b/Lib/decimal.py index 7746ea2601024c..530bdfb38953d9 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1,11 +1,109 @@ +"""Decimal fixed-point and floating-point arithmetic. + +This is an implementation of decimal floating-point arithmetic based on +the General Decimal Arithmetic Specification: + + http://speleotrove.com/decimal/decarith.html + +and IEEE standard 854-1987: + + http://en.wikipedia.org/wiki/IEEE_854-1987 + +Decimal floating point has finite precision with arbitrarily large bounds. + +The purpose of this module is to support arithmetic using familiar +"schoolhouse" rules and to avoid some of the tricky representation +issues associated with binary floating point. The package is especially +useful for financial applications or for contexts where users have +expectations that are at odds with binary floating point (for instance, +in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead +of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected +Decimal('0.00')). + +Here are some examples of using the decimal module: + +>>> from decimal import * +>>> setcontext(ExtendedContext) +>>> Decimal(0) +Decimal('0') +>>> Decimal('1') +Decimal('1') +>>> Decimal('-.0123') +Decimal('-0.0123') +>>> Decimal(123456) +Decimal('123456') +>>> Decimal('123.45e12345678') +Decimal('1.2345E+12345680') +>>> Decimal('1.33') + Decimal('1.27') +Decimal('2.60') +>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') +Decimal('-2.20') +>>> dig = Decimal(1) +>>> print(dig / Decimal(3)) +0.333333333 +>>> getcontext().prec = 18 +>>> print(dig / Decimal(3)) +0.333333333333333333 +>>> print(dig.sqrt()) +1 +>>> print(Decimal(3).sqrt()) +1.73205080756887729 +>>> print(Decimal(3) ** 123) +4.85192780976896427E+58 +>>> inf = Decimal(1) / Decimal(0) +>>> print(inf) +Infinity +>>> neginf = Decimal(-1) / Decimal(0) +>>> print(neginf) +-Infinity +>>> print(neginf + inf) +NaN +>>> print(neginf * inf) +-Infinity +>>> print(dig / 0) +Infinity +>>> getcontext().traps[DivisionByZero] = 1 +>>> print(dig / 0) +Traceback (most recent call last): + ... + ... + ... +decimal.DivisionByZero: x / 0 +>>> c = Context() +>>> c.traps[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> c.divide(Decimal(0), Decimal(0)) +Decimal('NaN') +>>> c.traps[InvalidOperation] = 1 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> print(c.divide(Decimal(0), Decimal(0))) +Traceback (most recent call last): + ... + ... + ... +decimal.InvalidOperation: 0 / 0 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> c.traps[InvalidOperation] = 0 +>>> print(c.divide(Decimal(0), Decimal(0))) +NaN +>>> print(c.flags[InvalidOperation]) +1 +>>> +""" try: from _decimal import * - from _decimal import __doc__ - from _decimal import __version__ - from _decimal import __libmpdec_version__ + from _decimal import __version__ # noqa: F401 + from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: - from _pydecimal import * - from _pydecimal import __doc__ - from _pydecimal import __version__ - from _pydecimal import __libmpdec_version__ + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/Lib/difflib.py b/Lib/difflib.py index ba0b256969ebff..7f595b6c72e641 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -908,87 +908,85 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + abcdefGhijkl ? ^ ^ ^ """ - - # don't synch up unless the lines have a similarity score of at - # least cutoff; best_ratio tracks the best score seen so far - best_ratio, cutoff = 0.74, 0.75 + # Don't synch up unless the lines have a similarity score above + # cutoff. Previously only the smallest pair was handled here, + # and if there are many pairs with the best ratio, recursion + # could grow very deep, and runtime cubic. See: + # https://github.com/python/cpython/issues/119105 + # + # Later, more pathological cases prompted removing recursion + # entirely. + cutoff = 0.74999 cruncher = SequenceMatcher(self.charjunk) - eqi, eqj = None, None # 1st indices of equal lines (if any) + crqr = cruncher.real_quick_ratio + cqr = cruncher.quick_ratio + cr = cruncher.ratio - # search for the pair that matches best without being identical - # (identical lines must be junk lines, & we don't want to synch up - # on junk -- unless we have to) + WINDOW = 10 + best_i = best_j = None + dump_i, dump_j = alo, blo # smallest indices not yet resolved for j in range(blo, bhi): - bj = b[j] - cruncher.set_seq2(bj) - for i in range(alo, ahi): - ai = a[i] - if ai == bj: - if eqi is None: - eqi, eqj = i, j - continue - cruncher.set_seq1(ai) - # computing similarity is expensive, so use the quick - # upper bounds first -- have seen this speed up messy - # compares by a factor of 3. - # note that ratio() is only expensive to compute the first - # time it's called on a sequence pair; the expensive part - # of the computation is cached by cruncher - if cruncher.real_quick_ratio() > best_ratio and \ - cruncher.quick_ratio() > best_ratio and \ - cruncher.ratio() > best_ratio: - best_ratio, best_i, best_j = cruncher.ratio(), i, j - if best_ratio < cutoff: - # no non-identical "pretty close" pair - if eqi is None: - # no identical pair either -- treat it as a straight replace - yield from self._plain_replace(a, alo, ahi, b, blo, bhi) - return - # no close pair, but an identical pair -- synch up on that - best_i, best_j, best_ratio = eqi, eqj, 1.0 - else: - # there's a close pair, so forget the identical pair (if any) - eqi = None - - # a[best_i] very similar to b[best_j]; eqi is None iff they're not - # identical - - # pump out diffs from before the synch point - yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) - - # do intraline marking on the synch pair - aelt, belt = a[best_i], b[best_j] - if eqi is None: - # pump out a '-', '?', '+', '?' quad for the synched lines - atags = btags = "" - cruncher.set_seqs(aelt, belt) - for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): - la, lb = ai2 - ai1, bj2 - bj1 - if tag == 'replace': - atags += '^' * la - btags += '^' * lb - elif tag == 'delete': - atags += '-' * la - elif tag == 'insert': - btags += '+' * lb - elif tag == 'equal': - atags += ' ' * la - btags += ' ' * lb - else: - raise ValueError('unknown tag %r' % (tag,)) - yield from self._qformat(aelt, belt, atags, btags) - else: - # the synch pair is identical - yield ' ' + aelt + cruncher.set_seq2(b[j]) + # Search the corresponding i's within WINDOW for rhe highest + # ratio greater than `cutoff`. + aequiv = alo + (j - blo) + arange = range(max(aequiv - WINDOW, dump_i), + min(aequiv + WINDOW + 1, ahi)) + if not arange: # likely exit if `a` is shorter than `b` + break + best_ratio = cutoff + for i in arange: + cruncher.set_seq1(a[i]) + # Ordering by cheapest to most expensive ratio is very + # valuable, most often getting out early. + if (crqr() > best_ratio + and cqr() > best_ratio + and cr() > best_ratio): + best_i, best_j, best_ratio = i, j, cr() + + if best_i is None: + # found nothing to synch on yet - move to next j + continue - # pump out diffs from after the synch point - yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) + # pump out straight replace from before this synch pair + yield from self._fancy_helper(a, dump_i, best_i, + b, dump_j, best_j) + # do intraline marking on the synch pair + aelt, belt = a[best_i], b[best_j] + if aelt != belt: + # pump out a '-', '?', '+', '?' quad for the synched lines + atags = btags = "" + cruncher.set_seqs(aelt, belt) + for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): + la, lb = ai2 - ai1, bj2 - bj1 + if tag == 'replace': + atags += '^' * la + btags += '^' * lb + elif tag == 'delete': + atags += '-' * la + elif tag == 'insert': + btags += '+' * lb + elif tag == 'equal': + atags += ' ' * la + btags += ' ' * lb + else: + raise ValueError('unknown tag %r' % (tag,)) + yield from self._qformat(aelt, belt, atags, btags) + else: + # the synch pair is identical + yield ' ' + aelt + dump_i, dump_j = best_i + 1, best_j + 1 + best_i = best_j = None + + # pump out straight replace from after the last synch pair + yield from self._fancy_helper(a, dump_i, ahi, + b, dump_j, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: - g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + g = self._plain_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: @@ -1266,6 +1264,12 @@ def _check_types(a, b, *args): if b and not isinstance(b[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(b[0]).__name__, b[0])) + if isinstance(a, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(a).__name__) + if isinstance(b, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(b).__name__) for arg in args: if not isinstance(arg, str): raise TypeError('all arguments must be str, not: %r' % (arg,)) diff --git a/Lib/dis.py b/Lib/dis.py index f05ea1a24f45a7..1718e39cceb638 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -11,12 +11,16 @@ _cache_format, _inline_cache_entries, _nb_ops, + _common_constants, _intrinsic_1_descs, _intrinsic_2_descs, + _special_method_names, _specializations, _specialized_opmap, ) +from _opcode import get_executor + __all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all @@ -28,12 +32,11 @@ CONVERT_VALUE = opmap['CONVERT_VALUE'] SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] -FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') +FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate') ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] -LOAD_CONST = opmap['LOAD_CONST'] -RETURN_CONST = opmap['RETURN_CONST'] LOAD_GLOBAL = opmap['LOAD_GLOBAL'] +LOAD_SMALL_INT = opmap['LOAD_SMALL_INT'] BINARY_OP = opmap['BINARY_OP'] JUMP_BACKWARD = opmap['JUMP_BACKWARD'] FOR_ITER = opmap['FOR_ITER'] @@ -42,9 +45,13 @@ LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] +LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] +LOAD_SPECIAL = opmap['LOAD_SPECIAL'] LOAD_FAST_LOAD_FAST = opmap['LOAD_FAST_LOAD_FAST'] STORE_FAST_LOAD_FAST = opmap['STORE_FAST_LOAD_FAST'] STORE_FAST_STORE_FAST = opmap['STORE_FAST_STORE_FAST'] +IS_OP = opmap['IS_OP'] +CONTAINS_OP = opmap['CONTAINS_OP'] CACHE = opmap["CACHE"] @@ -74,7 +81,7 @@ def _try_compile(source, name): return compile(source, name, 'exec') def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, - show_offsets=False): + show_offsets=False, show_positions=False): """Disassemble classes, methods, functions, and other compiled objects. With no argument, disassemble the last traceback. @@ -85,7 +92,7 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, """ if x is None: distb(file=file, show_caches=show_caches, adaptive=adaptive, - show_offsets=show_offsets) + show_offsets=show_offsets, show_positions=show_positions) return # Extract functions from methods. if hasattr(x, '__func__'): @@ -106,12 +113,12 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, if isinstance(x1, _have_code): print("Disassembly of %s:" % name, file=file) try: - dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) except TypeError as msg: print("Sorry:", msg, file=file) print(file=file) elif hasattr(x, 'co_code'): # Code object - _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) elif isinstance(x, (bytes, bytearray)): # Raw bytecode labels_map = _make_labels_map(x) label_width = 4 + len(str(len(labels_map))) @@ -122,12 +129,12 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, arg_resolver = ArgResolver(labels_map=labels_map) _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) elif isinstance(x, str): # Source code - _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) else: raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) -def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False): +def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): """Disassemble a traceback (default: last traceback).""" if tb is None: try: @@ -138,22 +145,23 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets except AttributeError: raise RuntimeError("no last traceback to disassemble") from None while tb.tb_next: tb = tb.tb_next - disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to # turn the co_flags field into a human readable list. COMPILER_FLAG_NAMES = { - 1: "OPTIMIZED", - 2: "NEWLOCALS", - 4: "VARARGS", - 8: "VARKEYWORDS", - 16: "NESTED", - 32: "GENERATOR", - 64: "NOFREE", - 128: "COROUTINE", - 256: "ITERABLE_COROUTINE", - 512: "ASYNC_GENERATOR", + 1: "OPTIMIZED", + 2: "NEWLOCALS", + 4: "VARARGS", + 8: "VARKEYWORDS", + 16: "NESTED", + 32: "GENERATOR", + 64: "NOFREE", + 128: "COROUTINE", + 256: "ITERABLE_COROUTINE", + 512: "ASYNC_GENERATOR", + 0x4000000: "HAS_DOCSTRING", } def pretty_flags(flags): @@ -205,7 +213,27 @@ def _deoptop(op): return _all_opmap[deoptmap[name]] if name in deoptmap else op def _get_code_array(co, adaptive): - return co._co_code_adaptive if adaptive else co.co_code + if adaptive: + code = co._co_code_adaptive + res = [] + found = False + for i in range(0, len(code), 2): + op, arg = code[i], code[i+1] + if op == ENTER_EXECUTOR: + try: + ex = get_executor(co, i) + except (ValueError, RuntimeError): + ex = None + + if ex: + op, arg = ex.get_opcode(), ex.get_oparg() + found = True + + res.append(op.to_bytes()) + res.append(arg.to_bytes()) + return code if not found else b''.join(res) + else: + return co.co_code def code_info(x): """Formatted details of methods, functions, or code.""" @@ -401,21 +429,25 @@ def __str__(self): class Formatter: def __init__(self, file=None, lineno_width=0, offset_width=0, label_width=0, - line_offset=0, show_caches=False): + line_offset=0, show_caches=False, *, show_positions=False): """Create a Formatter *file* where to write the output - *lineno_width* sets the width of the line number field (0 omits it) + *lineno_width* sets the width of the source location field (0 omits it). + Should be large enough for a line number or full positions (depending + on the value of *show_positions*). *offset_width* sets the width of the instruction offset field *label_width* sets the width of the label field *show_caches* is a boolean indicating whether to display cache lines - + *show_positions* is a boolean indicating whether full positions should + be reported instead of only the line numbers. """ self.file = file self.lineno_width = lineno_width self.offset_width = offset_width self.label_width = label_width self.show_caches = show_caches + self.show_positions = show_positions def print_instruction(self, instr, mark_as_current=False): self.print_instruction_line(instr, mark_as_current) @@ -448,15 +480,27 @@ def print_instruction_line(self, instr, mark_as_current): print(file=self.file) fields = [] - # Column: Source code line number + # Column: Source code locations information if lineno_width: - if instr.starts_line: - lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" - lineno_fmt = lineno_fmt % lineno_width - lineno = _NO_LINENO if instr.line_number is None else instr.line_number - fields.append(lineno_fmt % lineno) + if self.show_positions: + # reporting positions instead of just line numbers + if instr_positions := instr.positions: + if all(p is None for p in instr_positions): + positions_str = _NO_LINENO + else: + ps = tuple('?' if p is None else p for p in instr_positions) + positions_str = f"{ps[0]}:{ps[2]}-{ps[1]}:{ps[3]}" + fields.append(f'{positions_str:{lineno_width}}') + else: + fields.append(' ' * lineno_width) else: - fields.append(' ' * lineno_width) + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" + lineno_fmt = lineno_fmt % lineno_width + lineno = _NO_LINENO if instr.line_number is None else instr.line_number + fields.append(lineno_fmt % lineno) + else: + fields.append(' ' * lineno_width) # Column: Label if instr.label is not None: lbl = f"L{instr.label}:" @@ -505,6 +549,18 @@ def __init__(self, co_consts=None, names=None, varname_from_oparg=None, labels_m self.varname_from_oparg = varname_from_oparg self.labels_map = labels_map or {} + def offset_from_jump_arg(self, op, arg, offset): + deop = _deoptop(op) + if deop in hasjabs: + return arg * 2 + elif deop in hasjrel: + signed_arg = -arg if _is_backward_jump(deop) else arg + argval = offset + 2 + signed_arg*2 + caches = _get_cache_size(_all_opname[deop]) + argval += 2 * caches + return argval + return None + def get_label_for_offset(self, offset): return self.labels_map.get(offset, None) @@ -536,17 +592,11 @@ def get_argval_argrepr(self, op, arg, offset): argrepr = f"{argrepr} + NULL|self" else: argval, argrepr = _get_name_info(arg, get_name) - elif deop in hasjabs: - argval = arg*2 - argrepr = f"to L{self.labels_map[argval]}" - elif deop in hasjrel: - signed_arg = -arg if _is_backward_jump(deop) else arg - argval = offset + 2 + signed_arg*2 - caches = _get_cache_size(_all_opname[deop]) - argval += 2 * caches - if deop == ENTER_EXECUTOR: - argval += 2 - argrepr = f"to L{self.labels_map[argval]}" + elif deop in hasjump or deop in hasexc: + argval = self.offset_from_jump_arg(op, arg, offset) + lbl = self.get_label_for_offset(argval) + assert lbl is not None + argrepr = f"to L{lbl}" elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): arg1 = arg >> 4 arg2 = arg & 15 @@ -573,6 +623,18 @@ def get_argval_argrepr(self, op, arg, offset): argrepr = _intrinsic_1_descs[arg] elif deop == CALL_INTRINSIC_2: argrepr = _intrinsic_2_descs[arg] + elif deop == LOAD_COMMON_CONSTANT: + obj = _common_constants[arg] + if isinstance(obj, type): + argrepr = obj.__name__ + else: + argrepr = repr(obj) + elif deop == LOAD_SPECIAL: + argrepr = _special_method_names[arg] + elif deop == IS_OP: + argrepr = 'is not' if argval else 'is' + elif deop == CONTAINS_OP: + argrepr = 'not in' if argval else 'in' return argval, argrepr def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False): @@ -612,8 +674,10 @@ def _get_const_value(op, arg, co_consts): Otherwise (if it is a LOAD_CONST and co_consts is not provided) returns the dis.UNKNOWN sentinel. """ - assert op in hasconst + assert op in hasconst or op == LOAD_SMALL_INT + if op == LOAD_SMALL_INT: + return arg argval = UNKNOWN if co_consts is not None: argval = co_consts[arg] @@ -672,8 +736,7 @@ def _parse_exception_table(code): def _is_backward_jump(op): return opname[op] in ('JUMP_BACKWARD', - 'JUMP_BACKWARD_NO_INTERRUPT', - 'ENTER_EXECUTOR') + 'JUMP_BACKWARD_NO_INTERRUPT') def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=None, original_code=None, arg_resolver=None): @@ -717,8 +780,10 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N if caches: cache_info = [] + cache_offset = offset for name, size in _cache_format[opname[deop]].items(): - data = code[offset + 2: offset + 2 + 2 * size] + data = code[cache_offset + 2: cache_offset + 2 + 2 * size] + cache_offset += size * 2 cache_info.append((name, size, data)) else: cache_info = None @@ -730,17 +795,22 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, - show_offsets=False): + show_offsets=False, show_positions=False): """Disassemble a code object.""" linestarts = dict(findlinestarts(co)) exception_entries = _parse_exception_table(co) + if show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(linestarts) labels_map = _make_labels_map(co.co_code, exception_entries=exception_entries) label_width = 4 + len(str(len(labels_map))) formatter = Formatter(file=file, - lineno_width=_get_lineno_width(linestarts), + lineno_width=lineno_width, offset_width=len(str(max(len(co.co_code) - 2, 9999))) if show_offsets else 0, label_width=label_width, - show_caches=show_caches) + show_caches=show_caches, + show_positions=show_positions) arg_resolver = ArgResolver(co_consts=co.co_consts, names=co.co_names, varname_from_oparg=co._varname_from_oparg, @@ -749,8 +819,8 @@ def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, exception_entries=exception_entries, co_positions=co.co_positions(), original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter) -def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False): - disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) +def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): + disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) if depth is None or depth > 0: if depth is not None: depth = depth - 1 @@ -760,7 +830,7 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adap print("Disassembly of %r:" % (x,), file=file) _disassemble_recursive( x, file=file, depth=depth, show_caches=show_caches, - adaptive=adaptive, show_offsets=show_offsets + adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions ) @@ -793,6 +863,22 @@ def _get_lineno_width(linestarts): lineno_width = len(_NO_LINENO) return lineno_width +def _get_positions_width(code): + # Positions are formatted as 'LINE:COL-ENDLINE:ENDCOL ' (note trailing space). + # A missing component appears as '?', and when all components are None, we + # render '_NO_LINENO'. thus the minimum width is 1 + len(_NO_LINENO). + # + # If all values are missing, positions are not printed (i.e. positions_width = 0). + has_value = False + values_width = 0 + for positions in code.co_positions(): + has_value |= any(isinstance(p, int) for p in positions) + width = sum(1 if p is None else len(str(p)) for p in positions) + values_width = max(width, values_width) + if has_value: + # 3 = number of separators in a normal format + return 1 + max(len(_NO_LINENO), 3 + values_width) + return 0 def _disassemble_bytes(code, lasti=-1, linestarts=None, *, line_offset=0, exception_entries=(), @@ -910,7 +996,8 @@ def _find_imports(co): if op == IMPORT_NAME and i >= 2: from_op = opargs[i-1] level_op = opargs[i-2] - if (from_op[0] in hasconst and level_op[0] in hasconst): + if (from_op[0] in hasconst and + (level_op[0] in hasconst or level_op[0] == LOAD_SMALL_INT)): level = _get_const_value(level_op[0], level_op[1], consts) fromlist = _get_const_value(from_op[0], from_op[1], consts) yield (names[oparg], level, fromlist) @@ -939,7 +1026,7 @@ class Bytecode: Iterating over this yields the bytecode operations as Instruction instances. """ - def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False): + def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): self.codeobj = co = _get_code_object(x) if first_line is None: self.first_line = co.co_firstlineno @@ -954,6 +1041,7 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False self.show_caches = show_caches self.adaptive = adaptive self.show_offsets = show_offsets + self.show_positions = show_positions def __iter__(self): co = self.codeobj @@ -997,16 +1085,19 @@ def dis(self): with io.StringIO() as output: code = _get_code_array(co, self.adaptive) offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 - - + if self.show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(self._linestarts) labels_map = _make_labels_map(co.co_code, self.exception_entries) label_width = 4 + len(str(len(labels_map))) formatter = Formatter(file=output, - lineno_width=_get_lineno_width(self._linestarts), + lineno_width=lineno_width, offset_width=offset_width, label_width=label_width, line_offset=self._line_offset, - show_caches=self.show_caches) + show_caches=self.show_caches, + show_positions=self.show_positions) arg_resolver = ArgResolver(co_consts=co.co_consts, names=co.co_names, @@ -1032,6 +1123,8 @@ def main(): help='show inline caches') parser.add_argument('-O', '--show-offsets', action='store_true', help='show instruction offsets') + parser.add_argument('-P', '--show-positions', action='store_true', + help='show instruction positions') parser.add_argument('infile', nargs='?', default='-') args = parser.parse_args() if args.infile == '-': @@ -1042,7 +1135,7 @@ def main(): with open(args.infile, 'rb') as infile: source = infile.read() code = compile(source, name, "exec") - dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets) + dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets, show_positions=args.show_positions) if __name__ == "__main__": main() diff --git a/Lib/doctest.py b/Lib/doctest.py index 114aac62a34e95..bb281fc483c41c 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -104,6 +104,8 @@ def _test(): import unittest from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple +import _colorize # Used in doctests +from _colorize import ANSIColors, can_colorize class TestResults(namedtuple('TestResults', 'failed attempted')): @@ -387,11 +389,11 @@ def __init__(self, out): # still use input() to get user input self.use_rawinput = 1 - def set_trace(self, frame=None): + def set_trace(self, frame=None, *, commands=None): self.__debugger_used = True if frame is None: frame = sys._getframe().f_back - pdb.Pdb.set_trace(self, frame) + pdb.Pdb.set_trace(self, frame, commands=commands) def set_continue(self): # Calling set_continue unconditionally would break unit test @@ -1140,7 +1142,14 @@ def _find_lineno(self, obj, source_lines): obj = obj.fget if inspect.isfunction(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. - obj = obj.__code__ + obj = inspect.unwrap(obj) + try: + obj = obj.__code__ + except AttributeError: + # Functions implemented in C don't necessarily + # have a __code__ attribute. + # If there's no code, there's no lineno + return None if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): @@ -1172,6 +1181,9 @@ class DocTestRunner: The `run` method is used to process a single DocTest case. It returns a TestResults instance. + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) >>> tests.sort(key = lambda test: test.name) @@ -1191,9 +1203,9 @@ class DocTestRunner: 2 tests in _TestClass 2 tests in _TestClass.__init__ 2 tests in _TestClass.get - 1 tests in _TestClass.square + 1 test in _TestClass.square 7 tests in 4 items. - 7 passed and 0 failed. + 7 passed. Test passed. TestResults(failed=0, attempted=7) @@ -1215,13 +1227,15 @@ class DocTestRunner: `OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. - First, an output function (`out) can be passed to + First, an output function (`out`) can be passed to `TestRunner.run`; this function will be called with strings that should be displayed. It defaults to `sys.stdout.write`. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods `report_start`, `report_success`, `report_unexpected_exception`, and `report_failure`. + + >>> _colorize.COLORIZE = save_colorize """ # This divider string is used to separate failure messages, and to # separate sections of the summary. @@ -1300,7 +1314,10 @@ def report_unexpected_exception(self, out, test, example, exc_info): 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) def _failure_header(self, test, example): - out = [self.DIVIDER] + red, reset = ( + (ANSIColors.RED, ANSIColors.RESET) if can_colorize() else ("", "") + ) + out = [f"{red}{self.DIVIDER}{reset}"] if test.filename: if test.lineno is not None and example.lineno is not None: lineno = test.lineno + example.lineno + 1 @@ -1540,7 +1557,11 @@ def out(s): # Make sure sys.displayhook just prints the value to stdout save_displayhook = sys.displayhook sys.displayhook = sys.__displayhook__ - + saved_can_colorize = _colorize.can_colorize + _colorize.can_colorize = lambda: False + color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} + for key in color_variables: + color_variables[key] = os.environ.pop(key, None) try: return self.__run(test, compileflags, out) finally: @@ -1549,6 +1570,10 @@ def out(s): sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook + _colorize.can_colorize = saved_can_colorize + for key, value in color_variables.items(): + if value is not None: + os.environ[key] = value if clear_globs: test.globs.clear() import builtins @@ -1568,49 +1593,70 @@ def summarize(self, verbose=None): """ if verbose is None: verbose = self._verbose - notests = [] - passed = [] - failed = [] + + notests, passed, failed = [], [], [] total_tries = total_failures = total_skips = 0 - for item in self._stats.items(): - name, (failures, tries, skips) = item + + for name, (failures, tries, skips) in self._stats.items(): assert failures <= tries total_tries += tries total_failures += failures total_skips += skips + if tries == 0: notests.append(name) elif failures == 0: passed.append((name, tries)) else: - failed.append(item) + failed.append((name, (failures, tries, skips))) + + ansi = _colorize.get_colors() + bold_green = ansi.BOLD_GREEN + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW + if verbose: if notests: - print(f"{len(notests)} items had no tests:") + print(f"{_n_items(notests)} had no tests:") notests.sort() for name in notests: print(f" {name}") + if passed: - print(f"{len(passed)} items passed all tests:") - passed.sort() - for name, count in passed: - print(f" {count:3d} tests in {name}") + print(f"{green}{_n_items(passed)} passed all tests:{reset}") + for name, count in sorted(passed): + s = "" if count == 1 else "s" + print(f" {green}{count:3d} test{s} in {name}{reset}") + if failed: - print(self.DIVIDER) - print(f"{len(failed)} items had failures:") - failed.sort() - for name, (failures, tries, skips) in failed: + print(f"{red}{self.DIVIDER}{reset}") + print(f"{_n_items(failed)} had failures:") + for name, (failures, tries, skips) in sorted(failed): print(f" {failures:3d} of {tries:3d} in {name}") + if verbose: - print(f"{total_tries} tests in {len(self._stats)} items.") - print(f"{total_tries - total_failures} passed and {total_failures} failed.") + s = "" if total_tries == 1 else "s" + print(f"{total_tries} test{s} in {_n_items(self._stats)}.") + + and_f = ( + f" and {red}{total_failures} failed{reset}" + if total_failures else "" + ) + print(f"{green}{total_tries - total_failures} passed{reset}{and_f}.") + if total_failures: - msg = f"***Test Failed*** {total_failures} failures" + s = "" if total_failures == 1 else "s" + msg = f"{bold_red}***Test Failed*** {total_failures} failure{s}{reset}" if total_skips: - msg = f"{msg} and {total_skips} skipped tests" + s = "" if total_skips == 1 else "s" + msg = f"{msg} and {yellow}{total_skips} skipped test{s}{reset}" print(f"{msg}.") elif verbose: - print("Test passed.") + print(f"{bold_green}Test passed.{reset}") + return TestResults(total_failures, total_tries, skipped=total_skips) #///////////////////////////////////////////////////////////////// @@ -1627,9 +1673,18 @@ def merge(self, other): d[name] = (failures, tries, skips) +def _n_items(items: list | dict) -> str: + """ + Helper to pluralise the number of items in a list. + """ + n = len(items) + s = "" if n == 1 else "s" + return f"{n} item{s}" + + class OutputChecker: """ - A class used to check the whether the actual output from a doctest + A class used to check whether the actual output from a doctest example matches the expected output. `OutputChecker` defines two methods: `check_output`, which compares a given pair of outputs, and returns true if they match; and `output_difference`, which @@ -2225,13 +2280,13 @@ def __init__(self, test, optionflags=0, setUp=None, tearDown=None, unittest.TestCase.__init__(self) self._dt_optionflags = optionflags self._dt_checker = checker - self._dt_globs = test.globs.copy() self._dt_test = test self._dt_setUp = setUp self._dt_tearDown = tearDown def setUp(self): test = self._dt_test + self._dt_globs = test.globs.copy() if self._dt_setUp is not None: self._dt_setUp(test) @@ -2262,12 +2317,13 @@ def runTest(self): try: runner.DIVIDER = "-"*70 - failures, tries = runner.run( - test, out=new.write, clear_globs=False) + results = runner.run(test, out=new.write, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") finally: sys.stdout = old - if failures: + if results.failed: raise self.failureException(self.format_failure(new.getvalue())) def format_failure(self, err): @@ -2678,7 +2734,7 @@ def testsource(module, name): return testsrc def debug_src(src, pm=False, globs=None): - """Debug a single doctest docstring, in argument `src`'""" + """Debug a single doctest docstring, in argument `src`""" testsrc = script_from_examples(src) debug_script(testsrc, pm, globs) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 5b653f66c18554..ec2215a5e5f33c 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -92,6 +92,8 @@ ASPECIALS = TSPECIALS | set("*'%") ATTRIBUTE_ENDS = ASPECIALS | WSP EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') +NLSET = {'\n', '\r'} +SPECIALSNL = SPECIALS | NLSET def quote_string(value): return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' @@ -566,12 +568,14 @@ def display_name(self): if res[0].token_type == 'cfws': res.pop(0) else: - if res[0][0].token_type == 'cfws': + if (isinstance(res[0], TokenList) and + res[0][0].token_type == 'cfws'): res[0] = TokenList(res[0][1:]) if res[-1].token_type == 'cfws': res.pop() else: - if res[-1][-1].token_type == 'cfws': + if (isinstance(res[-1], TokenList) and + res[-1][-1].token_type == 'cfws'): res[-1] = TokenList(res[-1][:-1]) return res.value @@ -586,9 +590,13 @@ def value(self): quote = True if len(self) != 0 and quote: pre = post = '' - if self[0].token_type=='cfws' or self[0][0].token_type=='cfws': + if (self[0].token_type == 'cfws' or + isinstance(self[0], TokenList) and + self[0][0].token_type == 'cfws'): pre = ' ' - if self[-1].token_type=='cfws' or self[-1][-1].token_type=='cfws': + if (self[-1].token_type == 'cfws' or + isinstance(self[-1], TokenList) and + self[-1][-1].token_type == 'cfws'): post = ' ' return pre+quote_string(self.display_name)+post else: @@ -949,6 +957,8 @@ class _InvalidEwError(errors.HeaderParseError): # up other parse trees. Maybe should have tests for that, too. DOT = ValueTerminal('.', 'dot') ListSeparator = ValueTerminal(',', 'list-separator') +ListSeparator.as_ew_allowed = False +ListSeparator.syntactic_break = False RouteComponentMarker = ValueTerminal('@', 'route-component-marker') # @@ -1206,7 +1216,7 @@ def get_bare_quoted_string(value): value is the text between the quote marks, with whitespace preserved and quoted pairs decoded. """ - if value[0] != '"': + if not value or value[0] != '"': raise errors.HeaderParseError( "expected '\"' but found '{}'".format(value)) bare_quoted_string = BareQuotedString() @@ -1447,7 +1457,7 @@ def get_local_part(value): """ local_part = LocalPart() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1513,13 +1523,18 @@ def get_obs_local_part(value): raise token, value = get_cfws(value) obs_local_part.append(token) + if not obs_local_part: + raise errors.HeaderParseError( + "expected obs-local-part but found '{}'".format(value)) if (obs_local_part[0].token_type == 'dot' or obs_local_part[0].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[1].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid leading '.' in local part")) if (obs_local_part[-1].token_type == 'dot' or obs_local_part[-1].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[-2].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid trailing '.' in local part")) @@ -1601,7 +1616,7 @@ def get_domain(value): """ domain = Domain() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1677,6 +1692,8 @@ def get_obs_route(value): if value[0] in CFWS_LEADER: token, value = get_cfws(value) obs_route.append(token) + if not value: + break if value[0] == '@': obs_route.append(RouteComponentMarker) token, value = get_domain(value[1:]) @@ -1695,7 +1712,7 @@ def get_angle_addr(value): """ angle_addr = AngleAddr() - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) angle_addr.append(token) if not value or value[0] != '<': @@ -1705,7 +1722,7 @@ def get_angle_addr(value): value = value[1:] # Although it is not legal per RFC5322, SMTP uses '<>' in certain # circumstances. - if value[0] == '>': + if value and value[0] == '>': angle_addr.append(ValueTerminal('>', 'angle-addr-end')) angle_addr.defects.append(errors.InvalidHeaderDefect( "null addr-spec in angle-addr")) @@ -1757,6 +1774,9 @@ def get_name_addr(value): name_addr = NameAddr() # Both the optional display name and the angle-addr can start with cfws. leader = None + if not value: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(value)) if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: @@ -1771,7 +1791,10 @@ def get_name_addr(value): raise errors.HeaderParseError( "expected name-addr but found '{}'".format(token)) if leader is not None: - token[0][:0] = [leader] + if isinstance(token[0], TokenList): + token[0][:0] = [leader] + else: + token[:0] = [leader] leader = None name_addr.append(token) token, value = get_angle_addr(value) @@ -2022,7 +2045,7 @@ def get_address_list(value): address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value: # Must be a , at this point. - address_list.append(ValueTerminal(',', 'list-separator')) + address_list.append(ListSeparator) value = value[1:] return address_list, value @@ -2764,11 +2787,15 @@ def _refold_parse_tree(parse_tree, *, policy): # max_line_length 0/None means no limit, ie: infinitely long. maxlen = policy.max_line_length or sys.maxsize encoding = 'utf-8' if policy.utf8 else 'us-ascii' - lines = [''] - last_ew = None + lines = [''] # Folded lines to be output + leading_whitespace = '' # When we have whitespace between two encoded + # words, we may need to encode the whitespace + # at the beginning of the second word. + last_ew = None # Points to the last encoded character if there's an ew on + # the line last_charset = None wrap_as_ew_blocked = 0 - want_encoding = False + want_encoding = False # This is set to True if we need to encode this part end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') parts = list(parse_tree) while parts: @@ -2777,9 +2804,13 @@ def _refold_parse_tree(parse_tree, *, policy): wrap_as_ew_blocked -= 1 continue tstr = str(part) - if part.token_type == 'ptext' and set(tstr) & SPECIALS: - # Encode if tstr contains special characters. - want_encoding = True + if not want_encoding: + if part.token_type == 'ptext': + # Encode if tstr contains special characters. + want_encoding = not SPECIALSNL.isdisjoint(tstr) + else: + # Encode if tstr contains newlines. + want_encoding = not NLSET.isdisjoint(tstr) try: tstr.encode(encoding) charset = encoding @@ -2792,10 +2823,12 @@ def _refold_parse_tree(parse_tree, *, policy): # 'charset' property on the policy. charset = 'utf-8' want_encoding = True + if part.token_type == 'mime-parameters': # Mime parameter folding (using RFC2231) is extra special. _fold_mime_parameters(part, lines, maxlen, encoding) continue + if want_encoding and not wrap_as_ew_blocked: if not part.as_ew_allowed: want_encoding = False @@ -2818,7 +2851,9 @@ def _refold_parse_tree(parse_tree, *, policy): if not hasattr(part, 'encode'): # It's not a Terminal, do each piece individually. parts = list(part) + parts - else: + want_encoding = False + continue + elif part.as_ew_allowed: # It's a terminal, wrap it as an encoded word, possibly # combining it with previously encoded words if allowed. if (last_ew is not None and @@ -2827,21 +2862,44 @@ def _refold_parse_tree(parse_tree, *, policy): last_charset == 'utf-8' and charset != 'us-ascii')): last_ew = None last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, - part.ew_combine_allowed, charset) + part.ew_combine_allowed, charset, leading_whitespace) + # This whitespace has been added to the lines in _fold_as_ew() + # so clear it now. + leading_whitespace = '' last_charset = charset - want_encoding = False - continue + want_encoding = False + continue + else: + # It's a terminal which should be kept non-encoded + # (e.g. a ListSeparator). + last_ew = None + want_encoding = False + # fall through + if len(tstr) <= maxlen - len(lines[-1]): lines[-1] += tstr continue + # This part is too long to fit. The RFC wants us to break at # "major syntactic breaks", so unless we don't consider this # to be one, check if it will fit on the next line by itself. + leading_whitespace = '' if (part.syntactic_break and len(tstr) + 1 <= maxlen): newline = _steal_trailing_WSP_if_exists(lines) if newline or part.startswith_fws(): + # We're going to fold the data onto a new line here. Due to + # the way encoded strings handle continuation lines, we need to + # be prepared to encode any whitespace if the next line turns + # out to start with an encoded word. lines.append(newline + tstr) + + whitespace_accumulator = [] + for char in lines[-1]: + if char not in WSP: + break + whitespace_accumulator.append(char) + leading_whitespace = ''.join(whitespace_accumulator) last_ew = None continue if not hasattr(part, 'encode'): @@ -2865,9 +2923,10 @@ def _refold_parse_tree(parse_tree, *, policy): else: # We can't fold it onto the next line either... lines[-1] += tstr + return policy.linesep.join(lines) + policy.linesep -def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, leading_whitespace): """Fold string to_encode into lines as encoded word, combining if allowed. Return the new value for last_ew, or None if ew_combine_allowed is False. @@ -2882,7 +2941,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): to_encode = str( get_unstructured(lines[-1][last_ew:] + to_encode)) lines[-1] = lines[-1][:last_ew] - if to_encode[0] in WSP: + elif to_encode[0] in WSP: # We're joining this to non-encoded text, so don't encode # the leading blank. leading_wsp = to_encode[0] @@ -2890,6 +2949,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): if (len(lines[-1]) == maxlen): lines.append(_steal_trailing_WSP_if_exists(lines)) lines[-1] += leading_wsp + trailing_wsp = '' if to_encode[-1] in WSP: # Likewise for the trailing space. @@ -2909,11 +2969,20 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): while to_encode: remaining_space = maxlen - len(lines[-1]) - text_space = remaining_space - chrome_len + text_space = remaining_space - chrome_len - len(leading_whitespace) if text_space <= 0: lines.append(' ') continue + # If we are at the start of a continuation line, prepend whitespace + # (we only want to do this when the line starts with an encoded word + # but if we're folding in this helper function, then we know that we + # are going to be writing out an encoded word.) + if len(lines) > 1 and len(lines[-1]) == 1 and leading_whitespace: + encoded_word = _ew.encode(leading_whitespace, charset=encode_as) + lines[-1] += encoded_word + leading_whitespace = '' + to_encode_word = to_encode[:text_space] encoded_word = _ew.encode(to_encode_word, charset=encode_as) excess = len(encoded_word) - remaining_space @@ -2925,6 +2994,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): excess = len(encoded_word) - remaining_space lines[-1] += encoded_word to_encode = to_encode[len(to_encode_word):] + leading_whitespace = '' if to_encode: lines.append(' ') diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 0f1bf8e4253ec4..36625e35ffb6a7 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -224,7 +224,7 @@ class AddrlistClass: def __init__(self, field): """Initialize a new instance. - `field' is an unparsed address header field, containing + 'field' is an unparsed address header field, containing one or more addresses. """ self.specials = '()<>@,:;.\"[]' @@ -233,7 +233,7 @@ def __init__(self, field): self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it + # Note that RFC 2822 now specifies '.' as obs-phrase, meaning that it # is obsolete syntax. RFC 2822 requires that we recognize obsolete # syntax, so allow dots in phrases. self.phraseends = self.atomends.replace('.', '') @@ -423,14 +423,14 @@ def getdomain(self): def getdelimited(self, beginchar, endchars, allowcomments=True): """Parse a header fragment delimited by special characters. - `beginchar' is the start character for the fragment. - If self is not looking at an instance of `beginchar' then + 'beginchar' is the start character for the fragment. + If self is not looking at an instance of 'beginchar' then getdelimited returns the empty string. - `endchars' is a sequence of allowable end-delimiting characters. + 'endchars' is a sequence of allowable end-delimiting characters. Parsing stops when one of these is encountered. - If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed + If 'allowcomments' is non-zero, embedded RFC 2822 comments are allowed within the parsed fragment. """ if self.field[self.pos] != beginchar: @@ -474,7 +474,7 @@ def getatom(self, atomends=None): Optional atomends specifies a different set of end token delimiters (the default is to use self.atomends). This is used e.g. in - getphraselist() since phrase endings must not include the `.' (which + getphraselist() since phrase endings must not include the '.' (which is legal in phrases).""" atomlist = [''] if atomends is None: diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c9cbadd2a80c48..c7694a44e26639 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -150,13 +150,20 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): wrapping is done. Default is 78. mangle_from_ -- a flag that, when True escapes From_ lines in the - body of the message by putting a `>' in front of + body of the message by putting a '>' in front of them. This is used when the message is being - serialized by a generator. Default: True. + serialized by a generator. Default: False. message_factory -- the class to use to create new message objects. If the value is None, the default is Message. + verify_generated_headers + -- if true, the generator verifies that each header + they are properly folded, so that a parser won't + treat it as multiple headers, start-of-body, or + part of another header. + This is a check against custom Header & fold() + implementations. """ raise_on_defect = False @@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): max_line_length = 78 mangle_from_ = False message_factory = None + verify_generated_headers = True def handle_defect(self, obj, defect): """Based on policy, either raise defect or call register_defect. diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index 4cdf22666e3016..d440de95255bf1 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -15,7 +15,7 @@ with Base64 encoding. RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names +'encoded-word' in a header. This method is commonly used for 8-bit real names in To:, From:, Cc:, etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character conversion diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 043801107b60e5..cfd5a0c456e497 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -175,7 +175,7 @@ class Charset: module expose the following information about a character set: input_charset: The initial character set specified. Common aliases - are converted to their `official' email names (e.g. latin_1 + are converted to their 'official' email names (e.g. latin_1 is converted to iso-8859-1). Defaults to 7-bit us-ascii. header_encoding: If the character set must be encoded before it can be @@ -245,7 +245,7 @@ def __eq__(self, other): def get_body_encoding(self): """Return the content-transfer-encoding used for body encoding. - This is either the string `quoted-printable' or `base64' depending on + This is either the string 'quoted-printable' or 'base64' depending on the encoding used, or it is a function in which case you should call the function with a single argument, the Message object being encoded. The function should then set the Content-Transfer-Encoding diff --git a/Lib/email/errors.py b/Lib/email/errors.py index 3ad00565549968..02aa5eced6ae46 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -29,6 +29,10 @@ class CharsetError(MessageError): """An illegal charset was given.""" +class HeaderWriteError(MessageError): + """Error while writing headers.""" + + # These are parsing defects which the parser was able to work around. class MessageDefect(ValueError): """Base class for a message defect.""" diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 7ccbe10eb76856..205caf0fe9e81d 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -14,12 +14,14 @@ from copy import deepcopy from io import StringIO, BytesIO from email.utils import _has_surrogates +from email.errors import HeaderWriteError UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. NLCRE = re.compile(r'\r\n|\r|\n') fcre = re.compile(r'^From ', re.MULTILINE) +NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') class Generator: @@ -41,7 +43,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, Optional mangle_from_ is a flag that, when True (the default if policy is not set), escapes From_ lines in the body of the message by putting - a `>' in front of them. + a '>' in front of them. Optional maxheaderlen specifies the longest length for a non-continued header. When a header line is longer (in characters, with tabs @@ -74,7 +76,7 @@ def flatten(self, msg, unixfrom=False, linesep=None): unixfrom is a flag that forces the printing of a Unix From_ delimiter before the first object in the message tree. If the original message - has no From_ delimiter, a `standard' one is crafted. By default, this + has no From_ delimiter, a 'standard' one is crafted. By default, this is False to inhibit the printing of any From_ delimiter. Note that for subobjects, no From_ line is printed. @@ -222,7 +224,16 @@ def _dispatch(self, msg): def _write_headers(self, msg): for h, v in msg.raw_items(): - self.write(self.policy.fold(h, v)) + folded = self.policy.fold(h, v) + if self.policy.verify_generated_headers: + linesep = self.policy.linesep + if not folded.endswith(linesep): + raise HeaderWriteError( + f'folded header does not end with {linesep!r}: {folded!r}') + if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)): + raise HeaderWriteError( + f'folded header contains newline: {folded!r}') + self.write(folded) # A blank line always separates headers from body self.write(self._NL) @@ -243,7 +254,7 @@ def _handle_text(self, msg): # existing message. msg = deepcopy(msg) del msg['content-transfer-encoding'] - msg.set_payload(payload, charset) + msg.set_payload(msg._payload, charset) payload = msg.get_payload() self._munge_cte = (msg['content-transfer-encoding'], msg['content-type']) @@ -456,7 +467,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, fmt=None, *, argument is allowed. Walks through all subparts of a message. If the subpart is of main - type `text', then it prints the decoded payload of the subpart. + type 'text', then it prints the decoded payload of the subpart. Otherwise, fmt is a format string that is used instead of the message payload. fmt is expanded with the following keywords (in diff --git a/Lib/email/header.py b/Lib/email/header.py index 984851a7d9a679..66a1d46db50c45 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -192,7 +192,7 @@ def __init__(self, s=None, charset=None, The maximum line length can be specified explicitly via maxlinelen. For splitting the first line to a shorter value (to account for the field - header which isn't included in s, e.g. `Subject') pass in the name of + header which isn't included in s, e.g. 'Subject') pass in the name of the field in header_name. The default maxlinelen is 78 as recommended by RFC 2822. @@ -276,7 +276,7 @@ def append(self, s, charset=None, errors='strict'): output codec of the charset. If the string cannot be encoded to the output codec, a UnicodeError will be raised. - Optional `errors' is passed as the errors argument to the decode + Optional 'errors' is passed as the errors argument to the decode call if s is a byte string. """ if charset is None: @@ -326,7 +326,7 @@ def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'): Optional splitchars is a string containing characters which should be given extra weight by the splitting algorithm during normal header - wrapping. This is in very rough support of RFC 2822's `higher level + wrapping. This is in very rough support of RFC 2822's 'higher level syntactic breaks': split points preceded by a splitchar are preferred during line splitting, with the characters preferred in the order in which they appear in the string. Space and tab may be included in the diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index 3410935e38f476..2f436aefc2300b 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -43,8 +43,8 @@ def body_line_iterator(msg, decode=False): def typed_subpart_iterator(msg, maintype='text', subtype=None): """Iterate over the subparts with a given MIME type. - Use `maintype' as the main MIME type to match against; this defaults to - "text". Optional `subtype' is the MIME subtype to match against; if + Use 'maintype' as the main MIME type to match against; this defaults to + "text". Optional 'subtype' is the MIME subtype to match against; if omitted, only the main type is matched. """ for subpart in msg.walk(): diff --git a/Lib/email/message.py b/Lib/email/message.py index fe769580fed5d0..08192c50a8ff5c 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -21,7 +21,7 @@ SEMISPACE = '; ' -# Regular expression that matches `special' characters in parameters, the +# Regular expression that matches 'special' characters in parameters, the # existence of which force quoting of the parameter value. tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') @@ -141,7 +141,7 @@ class Message: multipart or a message/rfc822), then the payload is a list of Message objects, otherwise it is a string. - Message objects implement part of the `mapping' interface, which assumes + Message objects implement part of the 'mapping' interface, which assumes there is exactly one occurrence of the header per message. Some headers do in fact appear multiple times (e.g. Received) and for those headers, you must use the explicit API to set or get all the headers. Not all of @@ -294,7 +294,7 @@ def get_payload(self, i=None, decode=False): try: bpayload = payload.encode('ascii', 'surrogateescape') try: - payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace') + payload = bpayload.decode(self.get_content_charset('ascii'), 'replace') except LookupError: payload = bpayload.decode('ascii', 'replace') except UnicodeEncodeError: @@ -340,7 +340,7 @@ def set_payload(self, payload, charset=None): return if not isinstance(charset, Charset): charset = Charset(charset) - payload = payload.encode(charset.output_charset) + payload = payload.encode(charset.output_charset, 'surrogateescape') if hasattr(payload, 'decode'): self._payload = payload.decode('ascii', 'surrogateescape') else: @@ -597,7 +597,7 @@ def get_content_type(self): """Return the message's content type. The returned string is coerced to lower case of the form - `maintype/subtype'. If there was no Content-Type header in the + 'maintype/subtype'. If there was no Content-Type header in the message, the default type as given by get_default_type() will be returned. Since according to RFC 2045, messages always have a default type this will always return a value. @@ -620,7 +620,7 @@ def get_content_type(self): def get_content_maintype(self): """Return the message's main content type. - This is the `maintype' part of the string returned by + This is the 'maintype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() @@ -629,14 +629,14 @@ def get_content_maintype(self): def get_content_subtype(self): """Returns the message's sub-content type. - This is the `subtype' part of the string returned by + This is the 'subtype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() return ctype.split('/')[1] def get_default_type(self): - """Return the `default' content type. + """Return the 'default' content type. Most messages have a default content type of text/plain, except for messages that are subparts of multipart/digest containers. Such @@ -645,7 +645,7 @@ def get_default_type(self): return self._default_type def set_default_type(self, ctype): - """Set the `default' content type. + """Set the 'default' content type. ctype should be either "text/plain" or "message/rfc822", although this is not enforced. The default content type is not stored in the @@ -678,8 +678,8 @@ def get_params(self, failobj=None, header='content-type', unquote=True): """Return the message's Content-Type parameters, as a list. The elements of the returned list are 2-tuples of key/value pairs, as - split on the `=' sign. The left hand side of the `=' is the key, - while the right hand side is the value. If there is no `=' sign in + split on the '=' sign. The left hand side of the '=' is the key, + while the right hand side is the value. If there is no '=' sign in the parameter the value is the empty string. The value is as described in the get_param() method. @@ -839,9 +839,9 @@ def get_filename(self, failobj=None): """Return the filename associated with the payload if present. The filename is extracted from the Content-Disposition header's - `filename' parameter, and it is unquoted. If that header is missing - the `filename' parameter, this method falls back to looking for the - `name' parameter. + 'filename' parameter, and it is unquoted. If that header is missing + the 'filename' parameter, this method falls back to looking for the + 'name' parameter. """ missing = object() filename = self.get_param('filename', missing, 'content-disposition') @@ -854,7 +854,7 @@ def get_filename(self, failobj=None): def get_boundary(self, failobj=None): """Return the boundary associated with the payload if present. - The boundary is extracted from the Content-Type header's `boundary' + The boundary is extracted from the Content-Type header's 'boundary' parameter, and it is unquoted. """ missing = object() diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 94d81c771a474e..47fc218e1ae032 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -21,7 +21,7 @@ def __init__(self, _subtype='mixed', boundary=None, _subparts=None, Content-Type and MIME-Version headers. _subtype is the subtype of the multipart content type, defaulting to - `mixed'. + 'mixed'. boundary is the multipart boundary string. By default it is calculated as needed. diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 06d99b17f2f9c4..475aa2b1a66680 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -22,7 +22,7 @@ def __init__(self, _class=None, *, policy=compat32): textual representation of the message. The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the string or by a blank line. @@ -82,7 +82,7 @@ def __init__(self, *args, **kw): textual representation of the message. The input must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the input or by a blank line. diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 8816c84ed175a7..46b7de5bb6d8ae 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -21,7 +21,7 @@ 'HTTP', ] -linesep_splitter = re.compile(r'\n|\r') +linesep_splitter = re.compile(r'\n|\r\n?') @_extend_docstrings class EmailPolicy(Policy): @@ -205,7 +205,8 @@ def _fold(self, name, value, refold_binary=False): if hasattr(value, 'name'): return value.fold(policy=self) maxlen = self.max_line_length if self.max_line_length else sys.maxsize - lines = value.splitlines() + # We can't use splitlines here because it splits on more than \r and \n. + lines = linesep_splitter.split(value) refold = (self.refold_source == 'all' or self.refold_source == 'long' and (lines and len(lines[0])+len(name)+2 > maxlen or diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py index 27fcbb5a26e3ae..500bbc5151769d 100644 --- a/Lib/email/quoprimime.py +++ b/Lib/email/quoprimime.py @@ -5,7 +5,7 @@ """Quoted-printable content transfer encoding per RFCs 2045-2047. This module handles the content transfer encoding method defined in RFC 2045 -to encode US ASCII-like 8-bit data called `quoted-printable'. It is used to +to encode US ASCII-like 8-bit data called 'quoted-printable'. It is used to safely encode text that is in a character set similar to the 7-bit US ASCII character set, but that includes some 8-bit characters that are normally not allowed in email bodies or headers. @@ -17,7 +17,7 @@ with quoted-printable encoding. RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names +'encoded-word' in a header. This method is commonly used for 8-bit real names in To:/From:/Cc: etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character @@ -127,7 +127,7 @@ def quote(c): def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with quoted-printable (like) encoding. - Defined in RFC 2045, this `Q' encoding is similar to quoted-printable, but + Defined in RFC 2045, this 'Q' encoding is similar to quoted-printable, but used specifically for email header fields to allow charsets with mostly 7 bit characters (and some 8 bit) to remain more or less readable in non-RFC 2045 aware mail clients. @@ -290,7 +290,7 @@ def _unquote_match(match): # Header decoding is done a bit differently def header_decode(s): - """Decode a string encoded with RFC 2045 MIME header `Q' encoding. + """Decode a string encoded with RFC 2045 MIME header 'Q' encoding. This function does not parse a full MIME header value encoded with quoted-printable (like =?iso-8859-1?q?Hello_World?=) -- please use diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 103cef61a83538..f276303197396b 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -241,7 +241,7 @@ def formatdate(timeval=None, localtime=False, usegmt=False): Fri, 09 Nov 2001 01:08:47 -0000 - Optional timeval if given is a floating point time value as accepted by + Optional timeval if given is a floating-point time value as accepted by gmtime() and localtime(), otherwise the current time is used. Optional localtime is a flag that when True, interprets timeval, and @@ -466,23 +466,15 @@ def collapse_rfc2231_value(value, errors='replace', # better than not having it. # -def localtime(dt=None, isdst=None): +def localtime(dt=None): """Return local time as an aware datetime object. If called without arguments, return current time. Otherwise *dt* argument should be a datetime instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that is, dt.tzinfo is None), it is assumed to be in local time. - The isdst parameter is ignored. """ - if isdst is not None: - import warnings - warnings._deprecated( - "The 'isdst' parameter to 'localtime'", - message='{name} is deprecated and slated for removal in Python {remove}', - remove=(3, 14), - ) if dt is None: dt = datetime.datetime.now() return dt.astimezone() diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index d85afd6d5cf704..6a5ca046b5eb6c 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -209,6 +209,7 @@ 'ms932' : 'cp932', 'mskanji' : 'cp932', 'ms_kanji' : 'cp932', + 'windows_31j' : 'cp932', # cp949 codec '949' : 'cp949', diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index 5396047a7fb0b8..60a8d5eb227f82 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -11,7 +11,7 @@ sace_prefix = "xn--" # This assumes query strings, so AllowUnassigned is true -def nameprep(label): +def nameprep(label): # type: (str) -> str # Map newlabel = [] for c in label: @@ -25,7 +25,7 @@ def nameprep(label): label = unicodedata.normalize("NFKC", label) # Prohibit - for c in label: + for i, c in enumerate(label): if stringprep.in_table_c12(c) or \ stringprep.in_table_c22(c) or \ stringprep.in_table_c3(c) or \ @@ -35,7 +35,7 @@ def nameprep(label): stringprep.in_table_c7(c) or \ stringprep.in_table_c8(c) or \ stringprep.in_table_c9(c): - raise UnicodeError("Invalid character %r" % c) + raise UnicodeEncodeError("idna", label, i, i+1, f"Invalid character {c!r}") # Check bidi RandAL = [stringprep.in_table_d1(x) for x in label] @@ -46,29 +46,38 @@ def nameprep(label): # This is table C.8, which was already checked # 2) If a string contains any RandALCat character, the string # MUST NOT contain any LCat character. - if any(stringprep.in_table_d2(x) for x in label): - raise UnicodeError("Violation of BIDI requirement 2") + for i, x in enumerate(label): + if stringprep.in_table_d2(x): + raise UnicodeEncodeError("idna", label, i, i+1, + "Violation of BIDI requirement 2") # 3) If a string contains any RandALCat character, a # RandALCat character MUST be the first character of the # string, and a RandALCat character MUST be the last # character of the string. - if not RandAL[0] or not RandAL[-1]: - raise UnicodeError("Violation of BIDI requirement 3") + if not RandAL[0]: + raise UnicodeEncodeError("idna", label, 0, 1, + "Violation of BIDI requirement 3") + if not RandAL[-1]: + raise UnicodeEncodeError("idna", label, len(label)-1, len(label), + "Violation of BIDI requirement 3") return label -def ToASCII(label): +def ToASCII(label): # type: (str) -> bytes try: # Step 1: try ASCII - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 3: UseSTD3ASCIIRules is false, so # Skip to step 8. - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + if 0 < len(label_ascii) < 64: + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 2: nameprep label = nameprep(label) @@ -76,29 +85,34 @@ def ToASCII(label): # Step 3: UseSTD3ASCIIRules is false # Step 4: try ASCII try: - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 8. if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 5: Check ACE prefix - if label.startswith(sace_prefix): - raise UnicodeError("Label starts with ACE prefix") + if label.lower().startswith(sace_prefix): + raise UnicodeEncodeError( + "idna", label, 0, len(sace_prefix), "Label starts with ACE prefix") # Step 6: Encode with PUNYCODE - label = label.encode("punycode") + label_ascii = label.encode("punycode") # Step 7: Prepend ACE prefix - label = ace_prefix + label + label_ascii = ace_prefix + label_ascii # Step 8: Check size - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + # do not check for empty as we prepend ace_prefix. + if len(label_ascii) < 64: + return label_ascii + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") def ToUnicode(label): if len(label) > 1024: @@ -110,7 +124,9 @@ def ToUnicode(label): # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still # preventing us from wasting time decoding a big thing that'll just # hit the actual <= 63 length limit in Step 6. - raise UnicodeError("label way too long") + if isinstance(label, str): + label = label.encode("utf-8", errors="backslashreplace") + raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long") # Step 1: Check for ASCII if isinstance(label, bytes): pure_ascii = True @@ -118,25 +134,32 @@ def ToUnicode(label): try: label = label.encode("ascii") pure_ascii = True - except UnicodeError: + except UnicodeEncodeError: pure_ascii = False if not pure_ascii: + assert isinstance(label, str) # Step 2: Perform nameprep label = nameprep(label) # It doesn't say this, but apparently, it should be ASCII now try: label = label.encode("ascii") - except UnicodeError: - raise UnicodeError("Invalid character in IDN label") + except UnicodeEncodeError as exc: + raise UnicodeEncodeError("idna", label, exc.start, exc.end, + "Invalid character in IDN label") # Step 3: Check for ACE prefix - if not label.startswith(ace_prefix): + assert isinstance(label, bytes) + if not label.lower().startswith(ace_prefix): return str(label, "ascii") # Step 4: Remove ACE prefix label1 = label[len(ace_prefix):] # Step 5: Decode using PUNYCODE - result = label1.decode("punycode") + try: + result = label1.decode("punycode") + except UnicodeDecodeError as exc: + offset = len(ace_prefix) + raise UnicodeDecodeError("idna", label, offset+exc.start, offset+exc.end, exc.reason) # Step 6: Apply ToASCII label2 = ToASCII(result) @@ -144,7 +167,8 @@ def ToUnicode(label): # Step 7: Compare the result of step 6 with the one of step 3 # label2 will already be in lower case. if str(label, "ascii").lower() != str(label2, "ascii"): - raise UnicodeError("IDNA does not round-trip", label, label2) + raise UnicodeDecodeError("idna", label, 0, len(label), + f"IDNA does not round-trip, '{label!r}' != '{label2!r}'") # Step 8: return the result of step 5 return result @@ -156,7 +180,7 @@ def encode(self, input, errors='strict'): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return b'', 0 @@ -168,11 +192,16 @@ def encode(self, input, errors='strict'): else: # ASCII name: fast path labels = result.split(b'.') - for label in labels[:-1]: - if not (0 < len(label) < 64): - raise UnicodeError("label empty or too long") - if len(labels[-1]) >= 64: - raise UnicodeError("label too long") + for i, label in enumerate(labels[:-1]): + if len(label) == 0: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+1, + "label empty") + for i, label in enumerate(labels): + if len(label) >= 64: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+len(label), + "label too long") return result, len(input) result = bytearray() @@ -182,17 +211,27 @@ def encode(self, input, errors='strict'): del labels[-1] else: trailing_dot = b'' - for label in labels: + for i, label in enumerate(labels): if result: # Join with U+002E result.extend(b'.') - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError( + "idna", + input, + offset + exc.start, + offset + exc.end, + exc.reason, + ) return bytes(result+trailing_dot), len(input) def decode(self, input, errors='strict'): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return "", 0 @@ -202,7 +241,7 @@ def decode(self, input, errors='strict'): # XXX obviously wrong, see #3232 input = bytes(input) - if ace_prefix not in input: + if ace_prefix not in input.lower(): # Fast path try: return input.decode('ascii'), len(input) @@ -218,8 +257,15 @@ def decode(self, input, errors='strict'): trailing_dot = '' result = [] - for label in labels: - result.append(ToUnicode(label)) + for i, label in enumerate(labels): + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(x) for x in labels[:i]) + len(labels[:i]) + raise UnicodeDecodeError( + "idna", input, offset+exc.start, offset+exc.end, exc.reason) + else: + result.append(u_label) return ".".join(result)+trailing_dot, len(input) @@ -227,7 +273,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): def _buffer_encode(self, input, errors, final): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return (b'', 0) @@ -251,7 +297,16 @@ def _buffer_encode(self, input, errors, final): # Join with U+002E result.extend(b'.') size += 1 - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeEncodeError( + "idna", + input, + size + exc.start, + size + exc.end, + exc.reason, + ) size += len(label) result += trailing_dot @@ -261,7 +316,7 @@ def _buffer_encode(self, input, errors, final): class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError("Unsupported error handling: {errors}") if not input: return ("", 0) @@ -271,7 +326,11 @@ def _buffer_decode(self, input, errors, final): labels = dots.split(input) else: # Must be ASCII string - input = str(input, "ascii") + try: + input = str(input, "ascii") + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError("idna", input, + exc.start, exc.end, exc.reason) labels = input.split(".") trailing_dot = '' @@ -288,7 +347,18 @@ def _buffer_decode(self, input, errors, final): result = [] size = 0 for label in labels: - result.append(ToUnicode(label)) + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError( + "idna", + input.encode("ascii", errors="backslashreplace"), + size + exc.start, + size + exc.end, + exc.reason, + ) + else: + result.append(u_label) if size: size += 1 size += len(label) diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 1c5726447077b1..4622fc8c9206f3 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -1,4 +1,4 @@ -""" Codec for the Punicode encoding, as specified in RFC 3492 +""" Codec for the Punycode encoding, as specified in RFC 3492 Written by Martin v. Löwis. """ @@ -131,10 +131,11 @@ def decode_generalized_number(extended, extpos, bias, errors): j = 0 while 1: try: - char = ord(extended[extpos]) + char = extended[extpos] except IndexError: if errors == "strict": - raise UnicodeError("incomplete punicode string") + raise UnicodeDecodeError("punycode", extended, extpos, extpos+1, + "incomplete punycode string") return extpos + 1, None extpos += 1 if 0x41 <= char <= 0x5A: # A-Z @@ -142,8 +143,8 @@ def decode_generalized_number(extended, extpos, bias, errors): elif 0x30 <= char <= 0x39: digit = char - 22 # 0x30-26 elif errors == "strict": - raise UnicodeError("Invalid extended code point '%s'" - % extended[extpos-1]) + raise UnicodeDecodeError("punycode", extended, extpos-1, extpos, + f"Invalid extended code point '{extended[extpos-1]}'") else: return extpos, None t = T(j, bias) @@ -155,11 +156,14 @@ def decode_generalized_number(extended, extpos, bias, errors): def insertion_sort(base, extended, errors): - """3.2 Insertion unsort coding""" + """3.2 Insertion sort coding""" + # This function raises UnicodeDecodeError with position in the extended. + # Caller should add the offset. char = 0x80 pos = -1 bias = 72 extpos = 0 + while extpos < len(extended): newpos, delta = decode_generalized_number(extended, extpos, bias, errors) @@ -171,7 +175,9 @@ def insertion_sort(base, extended, errors): char += pos // (len(base) + 1) if char > 0x10FFFF: if errors == "strict": - raise UnicodeError("Invalid character U+%x" % char) + raise UnicodeDecodeError( + "punycode", extended, pos-1, pos, + f"Invalid character U+{char:x}") char = ord('?') pos = pos % (len(base) + 1) base = base[:pos] + chr(char) + base[pos:] @@ -187,11 +193,21 @@ def punycode_decode(text, errors): pos = text.rfind(b"-") if pos == -1: base = "" - extended = str(text, "ascii").upper() + extended = text.upper() else: - base = str(text[:pos], "ascii", errors) - extended = str(text[pos+1:], "ascii").upper() - return insertion_sort(base, extended, errors) + try: + base = str(text[:pos], "ascii", errors) + except UnicodeDecodeError as exc: + raise UnicodeDecodeError("ascii", text, exc.start, exc.end, + exc.reason) from None + extended = text[pos+1:].upper() + try: + return insertion_sort(base, extended, errors) + except UnicodeDecodeError as exc: + offset = pos + 1 + raise UnicodeDecodeError("punycode", text, + offset+exc.start, offset+exc.end, + exc.reason) from None ### Codec APIs @@ -203,7 +219,7 @@ def encode(self, input, errors='strict'): def decode(self, input, errors='strict'): if errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") res = punycode_decode(input, errors) return res, len(input) @@ -214,7 +230,7 @@ def encode(self, input, final=False): class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): if self.errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+self.errors) + raise UnicodeError(f"Unsupported error handling: {self.errors}") return punycode_decode(input, self.errors) class StreamWriter(Codec,codecs.StreamWriter): diff --git a/Lib/encodings/undefined.py b/Lib/encodings/undefined.py index 4690288355c710..082771e1c86677 100644 --- a/Lib/encodings/undefined.py +++ b/Lib/encodings/undefined.py @@ -1,6 +1,6 @@ """ Python 'undefined' Codec - This codec will always raise a ValueError exception when being + This codec will always raise a UnicodeError exception when being used. It is intended for use by the site.py file to switch off automatic string to Unicode coercion. diff --git a/Lib/encodings/utf_16.py b/Lib/encodings/utf_16.py index c61248242be8c7..d3b9980026666f 100644 --- a/Lib/encodings/utf_16.py +++ b/Lib/encodings/utf_16.py @@ -64,7 +64,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_16_be_decode elif consumed >= 2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -138,7 +138,7 @@ def decode(self, input, errors='strict'): elif byteorder == 1: self.decode = codecs.utf_16_be_decode elif consumed>=2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/encodings/utf_32.py b/Lib/encodings/utf_32.py index cdf84d14129a62..1924bedbb74c68 100644 --- a/Lib/encodings/utf_32.py +++ b/Lib/encodings/utf_32.py @@ -59,7 +59,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_32_be_decode elif consumed >= 4: - raise UnicodeError("UTF-32 stream does not start with BOM") + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -132,8 +132,8 @@ def decode(self, input, errors='strict'): self.decode = codecs.utf_32_le_decode elif byteorder == 1: self.decode = codecs.utf_32_be_decode - elif consumed>=4: - raise UnicodeError("UTF-32 stream does not start with BOM") + elif consumed >= 4: + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index e8dd253bb55520..585afc85836c06 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "24.0" +_PIP_VERSION = "24.2" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora @@ -205,7 +205,7 @@ def _uninstall_helper(*, verbosity=0): def _main(argv=None): import argparse - parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl deleted file mode 100644 index 2e6aa9d2cb9923..00000000000000 Binary files a/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-24.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.2-py3-none-any.whl new file mode 100644 index 00000000000000..542cdd1e7284ae Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-24.2-py3-none-any.whl differ diff --git a/Lib/ensurepip/_uninstall.py b/Lib/ensurepip/_uninstall.py index b257904328d2f5..4183c28a809008 100644 --- a/Lib/ensurepip/_uninstall.py +++ b/Lib/ensurepip/_uninstall.py @@ -6,7 +6,7 @@ def _main(argv=None): - parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/enum.py b/Lib/enum.py index 98a8966f5eb159..27be3fb83b2afb 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -162,6 +162,11 @@ def _dedent(text): lines[j] = l[i:] return '\n'.join(lines) +class _not_given: + def __repr__(self): + return('') +_not_given = _not_given() + class _auto_null: def __repr__(self): return '_auto_null' @@ -202,7 +207,7 @@ def __get__(self, instance, ownerclass=None): # use previous enum.property return self.fget(instance) elif self._attr_type == 'attr': - # look up previous attibute + # look up previous attribute return getattr(self._cls_type, self.name) elif self._attr_type == 'desc': # use previous descriptor @@ -279,9 +284,10 @@ def __set_name__(self, enum_class, member_name): enum_member._sort_order_ = len(enum_class._member_names_) if Flag is not None and issubclass(enum_class, Flag): - enum_class._flag_mask_ |= value - if _is_single_bit(value): - enum_class._singles_mask_ |= value + if isinstance(value, int): + enum_class._flag_mask_ |= value + if _is_single_bit(value): + enum_class._singles_mask_ |= value enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1 # If another member with the same value was already defined, the @@ -309,6 +315,7 @@ def __set_name__(self, enum_class, member_name): elif ( Flag is not None and issubclass(enum_class, Flag) + and isinstance(value, int) and _is_single_bit(value) ): # no other instances found, record this member in _member_names_ @@ -320,6 +327,8 @@ def __set_name__(self, enum_class, member_name): # to the map, and by-value lookups for this value will be # linear. enum_class._value2member_map_.setdefault(value, enum_member) + if value not in enum_class._hashable_values_: + enum_class._hashable_values_.append(value) except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) @@ -358,7 +367,10 @@ def __setitem__(self, key, value): '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', '_add_alias_', '_add_value_alias_', - ): + # While not in use internally, those are common for pretty + # printing and thus excluded from Enum's reservation of + # _sunder_ names + ) and not key.startswith('_repr_'): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' % (key, ) @@ -432,7 +444,7 @@ def __setitem__(self, key, value): # accepts iterable as multiple arguments? value = t(auto_valued) except TypeError: - # then pass them in singlely + # then pass them in singly value = t(*auto_valued) self._member_names[key] = None if non_auto_store: @@ -528,7 +540,8 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_member_names_'] = [] classdict['_member_map_'] = {} classdict['_value2member_map_'] = {} - classdict['_unhashable_values_'] = [] + classdict['_hashable_values_'] = [] # for comparing with non-hashable types + classdict['_unhashable_values_'] = [] # e.g. frozenset() with set() classdict['_unhashable_values_map_'] = {} classdict['_member_type_'] = member_type # now set the __repr__ for the value @@ -544,19 +557,16 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_all_bits_'] = 0 classdict['_inverted_'] = None try: - exc = None + classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) + classdict['_%s__in_progress' % cls] = False + delattr(enum_class, '_%s__in_progress' % cls) except Exception as e: - # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." - # is tacked on to the error instead of raising a RuntimeError - # recreate the exception to discard - exc = type(e)(str(e)) - exc.__cause__ = e.__cause__ - exc.__context__ = e.__context__ - tb = e.__traceback__ - if exc is not None: - raise exc.with_traceback(tb) - # + # since 3.12 the note "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError, so discard it + if hasattr(e, '__notes__'): + del e.__notes__ + raise # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__) # @@ -675,7 +685,7 @@ def __bool__(cls): """ return True - def __call__(cls, value, names=None, *values, module=None, qualname=None, type=None, start=1, boundary=None): + def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, type=None, start=1, boundary=None): """ Either returns an existing member, or creates a new enum class. @@ -704,18 +714,18 @@ def __call__(cls, value, names=None, *values, module=None, qualname=None, type=N """ if cls._member_map_: # simple value lookup if members exist - if names: + if names is not _not_given: value = (value, names) + values return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type - if names is None and type is None: + if names is _not_given and type is None: # no body? no data-type? possibly wrong usage raise TypeError( f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" ) return cls._create_( class_name=value, - names=names, + names=None if names is _not_given else names, module=module, qualname=qualname, type=type, @@ -735,7 +745,10 @@ def __contains__(cls, value): try: return value in cls._value2member_map_ except TypeError: - return value in cls._unhashable_values_ + return ( + value in cls._unhashable_values_ # both structures are lists + or value in cls._hashable_values_ + ) def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -1078,8 +1091,21 @@ def _add_member_(cls, name, member): setattr(cls, name, member) # now add to _member_map_ (even aliases) cls._member_map_[name] = member - # - cls._member_map_[name] = member + + @property + def __signature__(cls): + from inspect import Parameter, Signature + if cls._member_names_: + return Signature([Parameter('values', Parameter.VAR_POSITIONAL)]) + else: + return Signature([Parameter('new_class_name', Parameter.POSITIONAL_ONLY), + Parameter('names', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('module', Parameter.KEYWORD_ONLY, default=None), + Parameter('qualname', Parameter.KEYWORD_ONLY, default=None), + Parameter('type', Parameter.KEYWORD_ONLY, default=None), + Parameter('start', Parameter.KEYWORD_ONLY, default=1), + Parameter('boundary', Parameter.KEYWORD_ONLY, default=None)]) + EnumMeta = EnumType # keep EnumMeta name for backwards compatibility @@ -1124,13 +1150,6 @@ class Enum(metaclass=EnumType): attributes -- see the documentation for details. """ - @classmethod - def __signature__(cls): - if cls._member_names_: - return '(*values)' - else: - return '(new_class_name, /, names, *, module=None, qualname=None, type=None, start=1, boundary=None)' - def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' @@ -1147,12 +1166,17 @@ def __new__(cls, value): pass except TypeError: # not there, now do long search -- O(n) behavior - for name, values in cls._unhashable_values_map_.items(): - if value in values: + for name, unhashable_values in cls._unhashable_values_map_.items(): + if value in unhashable_values: + return cls[name] + for name, member in cls._member_map_.items(): + if value == member._value_: return cls[name] # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) if not cls._member_map_: + if getattr(cls, '_%s__in_progress' % cls.__name__, False): + raise TypeError('do not use `super().__new__; call the appropriate __new__ directly') from None raise TypeError("%r has no members defined" % cls) # # still not found -- try _missing_ hook @@ -1212,6 +1236,7 @@ def _add_value_alias_(self, value): # to the map, and by-value lookups for this value will be # linear. cls._value2member_map_.setdefault(value, self) + cls._hashable_values_.append(value) except TypeError: # keep track of the value in a list so containment checks are quick cls._unhashable_values_.append(value) @@ -1558,37 +1583,50 @@ def __str__(self): def __bool__(self): return bool(self._value_) + def _get_value(self, flag): + if isinstance(flag, self.__class__): + return flag._value_ + elif self._member_type_ is not object and isinstance(flag, self._member_type_): + return flag + return NotImplemented + def __or__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with |") value = self._value_ - return self.__class__(value | other) + return self.__class__(value | other_value) def __and__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with &") value = self._value_ - return self.__class__(value & other) + return self.__class__(value & other_value) def __xor__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with ^") value = self._value_ - return self.__class__(value ^ other) + return self.__class__(value ^ other_value) def __invert__(self): + if self._get_value(self) is None: + raise TypeError(f"'{self}' cannot be inverted") + if self._inverted_ is None: if self._boundary_ in (EJECT, KEEP): self._inverted_ = self.__class__(~self._value_) @@ -1655,7 +1693,7 @@ def global_flag_repr(self): cls_name = self.__class__.__name__ if self._name_ is None: return "%s.%s(%r)" % (module, cls_name, self._value_) - if _is_single_bit(self): + if _is_single_bit(self._value_): return '%s.%s' % (module, self._name_) if self._boundary_ is not FlagBoundary.KEEP: return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')]) @@ -1729,6 +1767,7 @@ def convert_class(cls): body['_member_names_'] = member_names = [] body['_member_map_'] = member_map = {} body['_value2member_map_'] = value2member_map = {} + body['_hashable_values_'] = hashable_values = [] body['_unhashable_values_'] = unhashable_values = [] body['_unhashable_values_map_'] = {} body['_member_type_'] = member_type = etype._member_type_ @@ -1777,20 +1816,31 @@ def convert_class(cls): for name, value in attrs.items(): if isinstance(value, auto) and auto.value is _auto_null: value = gnv(name, 1, len(member_names), gnv_last_values) - if value in value2member_map or value in unhashable_values: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member.value in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - enum_class(value)._add_alias_(name) + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) @@ -1801,6 +1851,7 @@ def convert_class(cls): else: enum_class._add_member_(name, member) value2member_map[value] = member + hashable_values.append(value) if _is_single_bit(value): # not a multi-bit alias, record in _member_names_ and _flag_mask_ member_names.append(name) @@ -1822,24 +1873,31 @@ def convert_class(cls): if value.value is _auto_null: value.value = gnv(name, 1, len(member_names), gnv_last_values) value = value.value + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias try: - contained = value in value2member_map + contained = value2member_map.get(member._value_) except TypeError: - contained = value in unhashable_values - if contained: + contained = None + if member._value_ in unhashable_values or member._value_ in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - enum_class(value)._add_alias_(name) + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) @@ -1856,6 +1914,8 @@ def convert_class(cls): # to the map, and by-value lookups for this value will be # linear. enum_class._value2member_map_.setdefault(value, member) + if value not in hashable_values: + hashable_values.append(value) except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) @@ -1993,7 +2053,8 @@ def _test_simple_enum(checked_enum, simple_enum): + list(simple_enum._member_map_.keys()) ) for key in set(checked_keys + simple_keys): - if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__'): + if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', + '__static_attributes__', '__firstlineno__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 30bd900fa805aa..c5b8d854d77de3 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -88,12 +88,15 @@ def _do_cmp(f1, f2): class dircmp: """A class that manages the comparison of 2 directories. - dircmp(a, b, ignore=None, hide=None) + dircmp(a, b, ignore=None, hide=None, *, shallow=True) A and B are directories. IGNORE is a list of names to ignore, defaults to DEFAULT_IGNORES. HIDE is a list of names to hide, defaults to [os.curdir, os.pardir]. + SHALLOW specifies whether to just check the stat signature (do not read + the files). + defaults to True. High level usage: x = dircmp(dir1, dir2) @@ -121,7 +124,7 @@ class dircmp: in common_dirs. """ - def __init__(self, a, b, ignore=None, hide=None): # Initialize + def __init__(self, a, b, ignore=None, hide=None, *, shallow=True): # Initialize self.left = a self.right = b if hide is None: @@ -132,6 +135,7 @@ def __init__(self, a, b, ignore=None, hide=None): # Initialize self.ignore = DEFAULT_IGNORES else: self.ignore = ignore + self.shallow = shallow def phase0(self): # Compare everything except common subdirectories self.left_list = _filter(os.listdir(self.left), @@ -160,12 +164,14 @@ def phase2(self): # Distinguish files, directories, funnies ok = True try: a_stat = os.stat(a_path) - except OSError: + except (OSError, ValueError): + # See https://github.com/python/cpython/issues/122400 + # for the rationale for protecting against ValueError. # print('Can\'t stat', a_path, ':', why.args[1]) ok = False try: b_stat = os.stat(b_path) - except OSError: + except (OSError, ValueError): # print('Can\'t stat', b_path, ':', why.args[1]) ok = False @@ -184,7 +190,7 @@ def phase2(self): # Distinguish files, directories, funnies self.common_funny.append(x) def phase3(self): # Find out differences between common files - xx = cmpfiles(self.left, self.right, self.common_files) + xx = cmpfiles(self.left, self.right, self.common_files, self.shallow) self.same_files, self.diff_files, self.funny_files = xx def phase4(self): # Find out differences between common subdirectories @@ -196,7 +202,8 @@ def phase4(self): # Find out differences between common subdirectories for x in self.common_dirs: a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) - self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide) + self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide, + shallow=self.shallow) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() @@ -280,12 +287,12 @@ def cmpfiles(a, b, common, shallow=True): # Return: # 0 for equal # 1 for different -# 2 for funny cases (can't stat, etc.) +# 2 for funny cases (can't stat, NUL bytes, etc.) # def _cmp(a, b, sh, abs=abs, cmp=cmp): try: return not abs(cmp(a, b, sh)) - except OSError: + except (OSError, ValueError): return 2 diff --git a/Lib/fractions.py b/Lib/fractions.py index 389ab386b6a8a4..f0cbc8c2e6c012 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -3,7 +3,6 @@ """Fraction, infinite-precision, rational numbers.""" -from decimal import Decimal import functools import math import numbers @@ -244,7 +243,9 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = numerator.denominator return self - elif isinstance(numerator, (float, Decimal)): + elif (isinstance(numerator, float) or + (not isinstance(numerator, type) and + hasattr(numerator, 'as_integer_ratio'))): # Exact conversion self._numerator, self._denominator = numerator.as_integer_ratio() return self @@ -278,8 +279,8 @@ def __new__(cls, numerator=0, denominator=None): numerator = -numerator else: - raise TypeError("argument should be a string " - "or a Rational instance") + raise TypeError("argument should be a string or a Rational " + "instance or have the as_integer_ratio() method") elif type(numerator) is int is type(denominator): pass # *very* normal case @@ -305,6 +306,28 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = denominator return self + @classmethod + def from_number(cls, number): + """Converts a finite real number to a rational number, exactly. + + Beware that Fraction.from_number(0.3) != Fraction(3, 10). + + """ + if type(number) is int: + return cls._from_coprime_ints(number, 1) + + elif isinstance(number, numbers.Rational): + return cls._from_coprime_ints(number.numerator, number.denominator) + + elif (isinstance(number, float) or + (not isinstance(number, type) and + hasattr(number, 'as_integer_ratio'))): + return cls._from_coprime_ints(*number.as_integer_ratio()) + + else: + raise TypeError("argument should be a Rational instance or " + "have the as_integer_ratio() method") + @classmethod def from_float(cls, f): """Converts a finite float to a rational number, exactly. @@ -579,7 +602,8 @@ def __format__(self, format_spec, /): f"for object of type {type(self).__name__!r}" ) - def _operator_fallbacks(monomorphic_operator, fallback_operator): + def _operator_fallbacks(monomorphic_operator, fallback_operator, + handle_complex=True): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. @@ -666,8 +690,8 @@ def forward(a, b): return monomorphic_operator(a, Fraction(b)) elif isinstance(b, float): return fallback_operator(float(a), b) - elif isinstance(b, complex): - return fallback_operator(complex(a), b) + elif handle_complex and isinstance(b, complex): + return fallback_operator(float(a), b) else: return NotImplemented forward.__name__ = '__' + fallback_operator.__name__ + '__' @@ -679,8 +703,8 @@ def reverse(b, a): return monomorphic_operator(Fraction(a), b) elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) - elif isinstance(a, numbers.Complex): - return fallback_operator(complex(a), complex(b)) + elif handle_complex and isinstance(a, numbers.Complex): + return fallback_operator(complex(a), float(b)) else: return NotImplemented reverse.__name__ = '__r' + fallback_operator.__name__ + '__' @@ -830,7 +854,7 @@ def _floordiv(a, b): """a // b""" return (a.numerator * b.denominator) // (a.denominator * b.numerator) - __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv) + __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv, False) def _divmod(a, b): """(a // b, a % b)""" @@ -838,16 +862,16 @@ def _divmod(a, b): div, n_mod = divmod(a.numerator * db, da * b.numerator) return div, Fraction(n_mod, da * db) - __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod) + __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod, False) def _mod(a, b): """a % b""" da, db = a.denominator, b.denominator return Fraction((a.numerator * db) % (b.numerator * da), da * db) - __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod) + __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) - def __pow__(a, b): + def __pow__(a, b, modulo=None): """a ** b If b is not an integer, the result will be a float or complex @@ -855,6 +879,8 @@ def __pow__(a, b): result will be rational. """ + if modulo is not None: + return NotImplemented if isinstance(b, numbers.Rational): if b.denominator == 1: power = b.numerator @@ -874,8 +900,10 @@ def __pow__(a, b): # A fractional power will generally produce an # irrational number. return float(a) ** float(b) - else: + elif isinstance(b, (float, complex)): return float(a) ** b + else: + return NotImplemented def __rpow__(b, a): """a ** b""" diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 10c5d1ea08ab11..50771e8c17c250 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -343,7 +343,7 @@ def ntransfercmd(self, cmd, rest=None): connection and the expected size of the transfer. The expected size may be None if it could not be determined. - Optional `rest' argument can be a string that is sent as the + Optional 'rest' argument can be a string that is sent as the argument to a REST command. This is essentially a server marker used to tell the server to skip over any data up to the given marker. diff --git a/Lib/functools.py b/Lib/functools.py index 55990e742bf23f..27abd622a8cff1 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -6,23 +6,22 @@ # Written by Nick Coghlan , # Raymond Hettinger , # and Łukasz Langa . -# Copyright (C) 2006-2013 Python Software Foundation. +# Copyright (C) 2006-2024 Python Software Foundation. # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', - 'cached_property'] + 'cached_property', 'Placeholder'] from abc import get_cache_token from collections import namedtuple -# import types, weakref # Deferred to single_dispatch() +# import weakref # Deferred to single_dispatch() +from operator import itemgetter from reprlib import recursive_repr +from types import GenericAlias, MethodType, MappingProxyType, UnionType from _thread import RLock -# Avoid importing types, so we can speedup import time -GenericAlias = type(list[int]) - ################################################################################ ### update_wrapper() and wraps() decorator ################################################################################ @@ -31,7 +30,7 @@ # wrapper functions that can handle naive introspection WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', - '__annotations__', '__type_params__') + '__annotate__', '__type_params__') WRAPPER_UPDATES = ('__dict__',) def update_wrapper(wrapper, wrapped, @@ -239,12 +238,14 @@ def reduce(function, sequence, initial=_initial_missing): """ reduce(function, iterable[, initial], /) -> value - Apply a function of two arguments cumulatively to the items of a sequence - or iterable, from left to right, so as to reduce the iterable to a single - value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates - ((((1+2)+3)+4)+5). If initial is present, it is placed before the items - of the iterable in the calculation, and serves as a default when the - iterable is empty. + Apply a function of two arguments cumulatively to the items of an iterable, from left to right. + + This effectively reduces the iterable to a single value. If initial is present, + it is placed before the items of the iterable in the calculation, and serves as + a default when the iterable is empty. + + For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) + calculates ((((1 + 2) + 3) + 4) + 5). """ it = iter(sequence) @@ -273,43 +274,130 @@ def reduce(function, sequence, initial=_initial_missing): ### partial() argument application ################################################################################ -# Purely functional, no descriptor behaviour -class partial: - """New function with partial application of the given arguments - and keywords. + +class _PlaceholderType: + """The type of the Placeholder singleton. + + Used as a placeholder for partial arguments. """ + __instance = None + __slots__ = () + + def __init_subclass__(cls, *args, **kwargs): + raise TypeError(f"type '{cls.__name__}' is not an acceptable base type") + + def __new__(cls): + if cls.__instance is None: + cls.__instance = object.__new__(cls) + return cls.__instance - __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" + def __repr__(self): + return 'Placeholder' - def __new__(cls, func, /, *args, **keywords): + def __reduce__(self): + return 'Placeholder' + +Placeholder = _PlaceholderType() + +def _partial_prepare_merger(args): + if not args: + return 0, None + nargs = len(args) + order = [] + j = nargs + for i, a in enumerate(args): + if a is Placeholder: + order.append(j) + j += 1 + else: + order.append(i) + phcount = j - nargs + merger = itemgetter(*order) if phcount else None + return phcount, merger + +def _partial_new(cls, func, /, *args, **keywords): + if issubclass(cls, partial): + base_cls = partial if not callable(func): raise TypeError("the first argument must be callable") + else: + base_cls = partialmethod + # func could be a descriptor like classmethod which isn't callable + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError(f"the first argument {func!r} must be a callable " + "or a descriptor") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + if isinstance(func, base_cls): + pto_phcount = func._phcount + tot_args = func.args + if args: + tot_args += args + if pto_phcount: + # merge args with args of `func` which is `partial` + nargs = len(args) + if nargs < pto_phcount: + tot_args += (Placeholder,) * (pto_phcount - nargs) + tot_args = func._merger(tot_args) + if nargs > pto_phcount: + tot_args += args[pto_phcount:] + phcount, merger = _partial_prepare_merger(tot_args) + else: # works for both pto_phcount == 0 and != 0 + phcount, merger = pto_phcount, func._merger + keywords = {**func.keywords, **keywords} + func = func.func + else: + tot_args = args + phcount, merger = _partial_prepare_merger(tot_args) + + self = object.__new__(cls) + self.func = func + self.args = tot_args + self.keywords = keywords + self._phcount = phcount + self._merger = merger + return self + +def _partial_repr(self): + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" - if hasattr(func, "func"): - args = func.args + args - keywords = {**func.keywords, **keywords} - func = func.func +# Purely functional, no descriptor behaviour +class partial: + """New function with partial application of the given arguments + and keywords. + """ - self = super(partial, cls).__new__(cls) + __slots__ = ("func", "args", "keywords", "_phcount", "_merger", + "__dict__", "__weakref__") - self.func = func - self.args = args - self.keywords = keywords - return self + __new__ = _partial_new + __repr__ = recursive_repr()(_partial_repr) def __call__(self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partial' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(*self.args, *args, **keywords) + return self.func(*pto_args, *args, **keywords) - @recursive_repr() - def __repr__(self): - qualname = type(self).__qualname__ - args = [repr(self.func)] - args.extend(repr(x) for x in self.args) - args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - if type(self).__module__ == "functools": - return f"functools.{qualname}({', '.join(args)})" - return f"{qualname}({', '.join(args)})" + def __get__(self, obj, objtype=None): + if obj is None: + return self + return MethodType(self, obj) def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -326,6 +414,10 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + phcount, merger = _partial_prepare_merger(args) + args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} @@ -338,57 +430,42 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds + self._phcount = phcount + self._merger = merger try: - from _functools import partial + from _functools import partial, Placeholder, _PlaceholderType except ImportError: pass # Descriptor version -class partialmethod(object): +class partialmethod: """Method descriptor with partial application of the given arguments and keywords. Supports wrapping existing descriptors and handles non-descriptor callables as instance methods. """ - - def __init__(self, func, /, *args, **keywords): - if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) - - # func could be a descriptor like classmethod which isn't callable, - # so we can't inherit from partial (it verifies func is callable) - if isinstance(func, partialmethod): - # flattening is mandatory in order to place cls/self before all - # other arguments - # it's also more efficient since only one function will be called - self.func = func.func - self.args = func.args + args - self.keywords = {**func.keywords, **keywords} - else: - self.func = func - self.args = args - self.keywords = keywords - - def __repr__(self): - args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) - format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + __new__ = _partial_new + __repr__ = _partial_repr def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partialmethod' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(cls_or_self, *self.args, *args, **keywords) + return self.func(cls_or_self, *pto_args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ - _method._partialmethod = self + _method.__partialmethod__ = self return _method def __get__(self, obj, cls=None): @@ -424,6 +501,17 @@ def _unwrap_partial(func): func = func.func return func +def _unwrap_partialmethod(func): + prev = None + while func is not prev: + prev = func + while isinstance(getattr(func, "__partialmethod__", None), partialmethod): + func = func.__partialmethod__ + while isinstance(func, partialmethod): + func = getattr(func, 'func') + func = _unwrap_partial(func) + return func + ################################################################################ ### LRU Cache function decorator ################################################################################ @@ -662,7 +750,7 @@ def cache(user_function, /): def _c3_merge(sequences): """Merges MROs in *sequences* to a single MRO using the C3 algorithm. - Adapted from https://www.python.org/download/releases/2.3/mro/. + Adapted from https://docs.python.org/3/howto/mro.html. """ result = [] @@ -811,7 +899,7 @@ def singledispatch(func): # There are many programs that use functools without singledispatch, so we # trade-off making singledispatch marginally slower for the benefit of # making start-up of such applications slightly faster. - import types, weakref + import weakref registry = {} dispatch_cache = weakref.WeakKeyDictionary() @@ -842,7 +930,7 @@ def dispatch(cls): def _is_union_type(cls): from typing import get_origin, Union - return get_origin(cls) in {Union, types.UnionType} + return get_origin(cls) in {Union, UnionType} def _is_valid_dispatch_type(cls): if isinstance(cls, type): @@ -867,8 +955,8 @@ def register(cls, func=None): f"Invalid first argument to `register()`. " f"{cls!r} is not a class or union type." ) - ann = getattr(cls, '__annotations__', {}) - if not ann: + ann = getattr(cls, '__annotate__', None) + if ann is None: raise TypeError( f"Invalid first argument to `register()`: {cls!r}. " f"Use either `@register(some_class)` or plain `@register` " @@ -878,13 +966,19 @@ def register(cls, func=None): # only import typing if annotation parsing is necessary from typing import get_type_hints - argname, cls = next(iter(get_type_hints(func).items())) + from annotationlib import Format, ForwardRef + argname, cls = next(iter(get_type_hints(func, format=Format.FORWARDREF).items())) if not _is_valid_dispatch_type(cls): if _is_union_type(cls): raise TypeError( f"Invalid annotation for {argname!r}. " f"{cls!r} not all arguments are classes." ) + elif isinstance(cls, ForwardRef): + raise TypeError( + f"Invalid annotation for {argname!r}. " + f"{cls!r} is an unresolved forward reference." + ) else: raise TypeError( f"Invalid annotation for {argname!r}. " @@ -907,14 +1001,13 @@ def wrapper(*args, **kw): if not args: raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return dispatch(args[0].__class__)(*args, **kw) funcname = getattr(func, '__name__', 'singledispatch function') registry[object] = func wrapper.register = register wrapper.dispatch = dispatch - wrapper.registry = types.MappingProxyType(registry) + wrapper.registry = MappingProxyType(registry) wrapper._clear_cache = dispatch_cache.clear update_wrapper(wrapper, func) return wrapper @@ -957,7 +1050,11 @@ def __get__(self, obj, cls=None): return _method dispatch = self.dispatcher.dispatch + funcname = getattr(self.func, '__name__', 'singledispatchmethod method') def _method(*args, **kwargs): + if not args: + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) _method.__isabstractmethod__ = self.__isabstractmethod__ diff --git a/Lib/genericpath.py b/Lib/genericpath.py index 1bd5b3897c3af9..ba7b0a13c7f81d 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -7,8 +7,8 @@ import stat __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', - 'getsize', 'isdir', 'isfile', 'islink', 'samefile', 'sameopenfile', - 'samestat'] + 'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink', + 'lexists', 'samefile', 'sameopenfile', 'samestat'] # Does a path exist? @@ -22,6 +22,15 @@ def exists(path): return True +# Being true for dangling symbolic links is also useful. +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + os.lstat(path) + except (OSError, ValueError): + return False + return True + # This follows symbolic links, so both islink() and isdir() can be true # for the same path on systems that support symlinks def isfile(path): @@ -57,6 +66,21 @@ def islink(path): return stat.S_ISLNK(st.st_mode) +# Is a path a junction? +def isjunction(path): + """Test whether a path is a junction + Junctions are not supported on the current platform""" + os.fspath(path) + return False + + +def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive. + Dev Drives are not supported on the current platform""" + os.fspath(path) + return False + + def getsize(filename): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size diff --git a/Lib/getopt.py b/Lib/getopt.py index 5419d77f5d774e..e5fd04fe12a7ee 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -2,8 +2,8 @@ This module helps scripts to parse the command line arguments in sys.argv. It supports the same conventions as the Unix getopt() -function (including the special meanings of arguments of the form `-' -and `--'). Long options similar to those supported by GNU software +function (including the special meanings of arguments of the form '-' +and '--'). Long options similar to those supported by GNU software may be used as well via an optional third argument. This module provides two functions and an exception: @@ -105,7 +105,7 @@ def gnu_getopt(args, shortopts, longopts = []): processing options as soon as a non-option argument is encountered. - If the first character of the option string is `+', or if the + If the first character of the option string is '+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. diff --git a/Lib/gettext.py b/Lib/gettext.py index 62cff81b7b3d49..a0d81cf846a05c 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -648,7 +648,7 @@ def npgettext(context, msgid1, msgid2, n): # import gettext # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR) # _ = cat.gettext -# print _('Hello World') +# print(_('Hello World')) # The resulting catalog object currently don't support access through a # dictionary API, which was supported (but apparently unused) in GNOME diff --git a/Lib/glob.py b/Lib/glob.py index 4a335a10766cf4..ce9b3698888dd9 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -4,11 +4,14 @@ import os import re import fnmatch +import functools import itertools +import operator import stat import sys -__all__ = ["glob", "iglob", "escape"] + +__all__ = ["glob", "iglob", "escape", "translate"] def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): @@ -104,8 +107,8 @@ def _iglob(pathname, root_dir, dir_fd, recursive, dironly, def _glob1(dirname, pattern, dir_fd, dironly, include_hidden=False): names = _listdir(dirname, dir_fd, dironly) - if include_hidden or not _ishidden(pattern): - names = (x for x in names if include_hidden or not _ishidden(x)) + if not (include_hidden or _ishidden(pattern)): + names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): @@ -119,12 +122,19 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -# Following functions are not public but can be used by third-party code. +_deprecated_function_message = ( + "{name} is deprecated and will be removed in Python {remove}. Use " + "glob.glob and pass a directory to its root_dir argument instead." +) def glob0(dirname, pattern): + import warnings + warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) return _glob0(dirname, pattern, None, False) def glob1(dirname, pattern): + import warnings + warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) return _glob1(dirname, pattern, None, False) # This helper function recursively yields relative pathnames inside a literal @@ -132,7 +142,8 @@ def glob1(dirname, pattern): def _glob2(dirname, pattern, dir_fd, dironly, include_hidden=False): assert _isrecursive(pattern) - yield pattern[:0] + if not dirname or _isdir(dirname, dir_fd): + yield pattern[:0] yield from _rlistdir(dirname, dir_fd, dironly, include_hidden=include_hidden) @@ -248,15 +259,16 @@ def escape(pathname): return drive + pathname +_special_parts = ('', '.', '..') _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) +_no_recurse_symlinks = object() def translate(pat, *, recursive=False, include_hidden=False, seps=None): """Translate a pathname with shell wildcards to a regular expression. If `recursive` is true, the pattern segment '**' will match any number of - path segments; if '**' appears outside its own segment, ValueError will be - raised. + path segments. If `include_hidden` is true, wildcards can match path segments beginning with a dot ('.'). @@ -290,22 +302,245 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None): for idx, part in enumerate(parts): if part == '*': results.append(one_segment if idx < last_part_idx else one_last_segment) - continue - if recursive: - if part == '**': - if idx < last_part_idx: - if parts[idx + 1] != '**': - results.append(any_segments) - else: - results.append(any_last_segments) - continue - elif '**' in part: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - if part: - if not include_hidden and part[0] in '*?': - results.append(r'(?!\.)') - results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) - if idx < last_part_idx: - results.append(any_sep) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + if idx < last_part_idx: + results.append(any_sep) res = ''.join(results) return fr'(?s:{res})\Z' + + +@functools.lru_cache(maxsize=512) +def _compile_pattern(pat, sep, case_sensitive, recursive=True): + """Compile given glob pattern to a re.Pattern object (observing case + sensitivity).""" + flags = re.NOFLAG if case_sensitive else re.IGNORECASE + regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) + return re.compile(regex, flags=flags).match + + +class _GlobberBase: + """Abstract class providing shell-style pattern matching and globbing. + """ + + def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): + self.sep = sep + self.case_sensitive = case_sensitive + self.case_pedantic = case_pedantic + self.recursive = recursive + + # Abstract methods + + @staticmethod + def lexists(path): + """Implements os.path.lexists(). + """ + raise NotImplementedError + + @staticmethod + def scandir(path): + """Implements os.scandir(). + """ + raise NotImplementedError + + @staticmethod + def add_slash(path): + """Returns a path with a trailing slash added. + """ + raise NotImplementedError + + @staticmethod + def concat_path(path, text): + """Implements path concatenation. + """ + raise NotImplementedError + + # High-level methods + + def compile(self, pat): + return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) + + def selector(self, parts): + """Returns a function that selects from a given path, walking and + filtering according to the glob-style pattern parts in *parts*. + """ + if not parts: + return self.select_exists + part = parts.pop() + if self.recursive and part == '**': + selector = self.recursive_selector + elif part in _special_parts: + selector = self.special_selector + elif not self.case_pedantic and magic_check.search(part) is None: + selector = self.literal_selector + else: + selector = self.wildcard_selector + return selector(part, parts) + + def special_selector(self, part, parts): + """Returns a function that selects special children of the given path. + """ + select_next = self.selector(parts) + + def select_special(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists) + return select_special + + def literal_selector(self, part, parts): + """Returns a function that selects a literal descendant of a path. + """ + + # Optimization: consume and join any subsequent literal parts here, + # rather than leaving them for the next selector. This reduces the + # number of string concatenation operations and calls to add_slash(). + while parts and magic_check.search(parts[-1]) is None: + part += self.sep + parts.pop() + + select_next = self.selector(parts) + + def select_literal(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists=False) + return select_literal + + def wildcard_selector(self, part, parts): + """Returns a function that selects direct children of a given path, + filtering by pattern. + """ + + match = None if part == '*' else self.compile(part) + dir_only = bool(parts) + if dir_only: + select_next = self.selector(parts) + + def select_wildcard(path, exists=False): + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + prefix = self.add_slash(path) + for entry in entries: + if match is None or match(entry.name): + if dir_only: + try: + if not entry.is_dir(): + continue + except OSError: + continue + entry_path = self.concat_path(prefix, entry.name) + if dir_only: + yield from select_next(entry_path, exists=True) + else: + yield entry_path + return select_wildcard + + def recursive_selector(self, part, parts): + """Returns a function that selects a given path and all its children, + recursively, filtering by pattern. + """ + # Optimization: consume following '**' parts, which have no effect. + while parts and parts[-1] == '**': + parts.pop() + + # Optimization: consume and join any following non-special parts here, + # rather than leaving them for the next selector. They're used to + # build a regular expression, which we use to filter the results of + # the recursive walk. As a result, non-special pattern segments + # following a '**' wildcard don't require additional filesystem access + # to expand. + follow_symlinks = self.recursive is not _no_recurse_symlinks + if follow_symlinks: + while parts and parts[-1] not in _special_parts: + part += self.sep + parts.pop() + + match = None if part == '**' else self.compile(part) + dir_only = bool(parts) + select_next = self.selector(parts) + + def select_recursive(path, exists=False): + path = self.add_slash(path) + match_pos = len(str(path)) + if match is None or match(str(path), match_pos): + yield from select_next(path, exists) + stack = [path] + while stack: + yield from select_recursive_step(stack, match_pos) + + def select_recursive_step(stack, match_pos): + path = stack.pop() + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + prefix = self.add_slash(path) + for entry in entries: + is_dir = False + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + is_dir = True + except OSError: + pass + + if is_dir or not dir_only: + entry_path = self.concat_path(prefix, entry.name) + if match is None or match(str(entry_path), match_pos): + if dir_only: + yield from select_next(entry_path, exists=True) + else: + # Optimization: directly yield the path if this is + # last pattern part. + yield entry_path + if is_dir: + stack.append(entry_path) + + return select_recursive + + def select_exists(self, path, exists=False): + """Yields the given path, if it exists. + """ + if exists: + # Optimization: this path is already known to exist, e.g. because + # it was returned from os.scandir(), so we skip calling lstat(). + yield path + elif self.lexists(path): + yield path + + +class _StringGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for string paths. + """ + lexists = staticmethod(os.path.lexists) + scandir = staticmethod(os.scandir) + concat_path = operator.add + + if os.name == 'nt': + @staticmethod + def add_slash(pathname): + tail = os.path.splitroot(pathname)[2] + if not tail or tail[-1] in '\\/': + return pathname + return f'{pathname}\\' + else: + @staticmethod + def add_slash(pathname): + if not pathname or pathname[-1] == '/': + return pathname + return f'{pathname}/' diff --git a/Lib/graphlib.py b/Lib/graphlib.py index 636545648e12d3..1438a5fc54b9cb 100644 --- a/Lib/graphlib.py +++ b/Lib/graphlib.py @@ -103,7 +103,7 @@ def prepare(self): # nodes as possible before cycles block more progress cycle = self._find_cycle() if cycle: - raise CycleError(f"nodes are in a cycle", cycle) + raise CycleError("nodes are in a cycle", cycle) def get_ready(self): """Return a tuple of all the nodes that are ready. diff --git a/Lib/gzip.py b/Lib/gzip.py index fda93e0261e028..1a3c82ce7e0711 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -15,7 +15,8 @@ FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 -READ, WRITE = 1, 2 +READ = 'rb' +WRITE = 'wb' _COMPRESS_LEVEL_FAST = 1 _COMPRESS_LEVEL_TRADEOFF = 6 @@ -178,9 +179,10 @@ def __init__(self, filename=None, mode=None, and 9 is slowest and produces the most compression. 0 is no compression at all. The default is 9. - The mtime argument is an optional numeric timestamp to be written - to the last modification time field in the stream when compressing. - If omitted or None, the current time is used. + The optional mtime argument is the timestamp requested by gzip. The time + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. + If mtime is omitted or None, the current time is used. Use mtime = 0 + to generate a compressed stream that does not depend on creation time. """ @@ -578,43 +580,21 @@ def _rewind(self): self._new_member = True -def _create_simple_gzip_header(compresslevel: int, - mtime = None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. - """ - if mtime is None: - mtime = time.time() - if compresslevel == _COMPRESS_LEVEL_BEST: - xfl = 2 - elif compresslevel == _COMPRESS_LEVEL_FAST: - xfl = 4 - else: - xfl = 0 - # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra - # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack(" \n # \" --> " # - i = 0 - n = len(str) - res = [] - while 0 <= i < n: - o_match = _OctalPatt.search(str, i) - q_match = _QuotePatt.search(str, i) - if not o_match and not q_match: # Neither matched - res.append(str[i:]) - break - # else: - j = k = -1 - if o_match: - j = o_match.start(0) - if q_match: - k = q_match.start(0) - if q_match and (not o_match or k < j): # QuotePatt matched - res.append(str[i:k]) - res.append(str[k+1]) - i = k + 2 - else: # OctalPatt matched - res.append(str[i:j]) - res.append(chr(int(str[j+1:j+4], 8))) - i = j + 4 - return _nulljoin(res) + return _unquote_sub(_unquote_replace, str) # The _getdate() routine is used to set the expiration time in the cookie's HTTP # header. By default, _getdate() returns the current time in the appropriate # "expires" format for a Set-Cookie header. The one optional argument is an # offset from now, in seconds. For example, an offset of -3600 means "one hour -# ago". The offset may be a floating point number. +# ago". The offset may be a floating-point number. # _weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] @@ -284,6 +266,8 @@ class Morsel(dict): "samesite" : "SameSite", } + _reserved_defaults = dict.fromkeys(_reserved, "") + _flags = {'secure', 'httponly'} def __init__(self): @@ -291,8 +275,7 @@ def __init__(self): self._key = self._value = self._coded_value = None # Set default attributes - for key in self._reserved: - dict.__setitem__(self, key, "") + dict.update(self, self._reserved_defaults) @property def key(self): diff --git a/Lib/http/server.py b/Lib/http/server.py index ee7a9b6aa55b88..a6f7aecc78763f 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -114,6 +114,11 @@ + Error response @@ -129,7 +134,8 @@ class HTTPServer(socketserver.TCPServer): - allow_reuse_address = 1 # Seems to make sense in testing environment + allow_reuse_address = True # Seems to make sense in testing environment + allow_reuse_port = True def server_bind(self): """Override server_bind to store the server name.""" @@ -803,6 +809,7 @@ def list_directory(self, path): r.append('') r.append('') r.append(f'') + r.append('') r.append(f'{title}\n') r.append(f'\n

{title}

') r.append('
\n
    ') @@ -897,7 +904,7 @@ def guess_type(self, path): ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] - guess, _ = mimetypes.guess_type(path) + guess, _ = mimetypes.guess_file_type(path) if guess: return guess return 'application/octet-stream' diff --git a/Lib/idlelib/HISTORY.txt b/Lib/idlelib/HISTORY.txt index 731fabd185fbbf..a601b25b5f838f 100644 --- a/Lib/idlelib/HISTORY.txt +++ b/Lib/idlelib/HISTORY.txt @@ -277,7 +277,7 @@ Command to format a paragraph. Debug menu: JIT (Just-In-Time) stack viewer toggle -- if set, the stack viewer -automaticall pops up when you get a traceback. +automatically pops up when you get a traceback. Windows menu: diff --git a/Lib/idlelib/Icons/README.txt b/Lib/idlelib/Icons/README.txt index d91c4d5d8d8cfa..e245bc0b26e121 100644 --- a/Lib/idlelib/Icons/README.txt +++ b/Lib/idlelib/Icons/README.txt @@ -1,13 +1,51 @@ -The IDLE icons are from https://bugs.python.org/issue1490384 +IDLE-PYTHON LOGOS -Created by Andrew Clover. +These are sent to tk on Windows, *NIX, and non-Aqua macOS +in pyshell following "# set application icon". -The original sources are available from Andrew's website: + +2006?: Andrew Clover made variously sized python icons for win23. https://www.doxdesk.com/software/py/pyicons.html -Various different formats and sizes are available at this GitHub Pull Request: -https://github.com/python/cpython/pull/17473 +2006: 16, 32, and 48 bit .png versions were copied to CPython +as Python application icons, maybe in PC/icons/py.ico. +https://github.com/python/cpython/issues/43372 + +2014: They were copied (perhaps a bit revised) to idlelib/Icons. +https://github.com/python/cpython/issues/64605 +.gif versions were also added. + +2020: Add Clover's 256-bit image. +https://github.com/python/cpython/issues/82620 +Other fixups were done. + +The idle.ico file used for Windows was created with ImageMagick: + $ convert idle_16.png idle_32.png idle_48.png idle_256.png idle.ico +** This needs redoing whenever files are changed. +?? Do Start, Desktop, and Taskbar use idlelib/Icons files? + +Issue added Windows Store PC/icons/idlex44.png and .../idlex150.png. +https://github.com/python/cpython/pull/22817 +?? Should these be updated with major changes? + +2022: Optimize .png images in CPython repository with external program. +https://github.com/python/cpython/pull/21348 +idle.ico (and idlex##) were not updated. + +The idlexx.gif files are only needed for *nix running tcl/tk 8.5. +As of 2022, this was known true for 1 'major' Linux distribution. +(Same would be true for any non-Aqua macOS with 8.5, but now none?) +Can be deleted when we require 8.6 or it is known always used. + +Future: Derivatives of Python logo should be submitted for approval. +PSF Trademark Working Group / Committee psf-trademarks@python.org +https://www.python.org/community/logos/ # Original files +https://www.python.org/psf/trademarks-faq/ +https://www.python.org/psf/trademarks/ # Usage. + + +OTHER GIFS: These are used by browsers using idlelib.tree. +At least some will not be used when tree is replaced by ttk.Treeview. -The idle.ico file was created with ImageMagick: - $ convert idle_16.png idle_32.png idle_48.png idle_256.png idle.ico +Edited 2024 August 26 by TJR. diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index 241b1f48e5c1d8..37ff93f9866e3c 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,18 @@ Released on 2024-10-xx ========================= +gh-120083: Add explicit black IDLE Hovertip foreground color needed for +recent macOS. Fixes Sonoma showing unreadable white on pale yellow. +Patch by John Riggles. + +gh-122482: Change About IDLE to direct users to discuss.python.org +instead of the now unused idle-dev email and mailing list. + +gh-78889: Stop Shell freezes by blocking user access to non-method +sys.stdout.shell attributes, which are all private. + +gh-78955: Use user-selected color theme for Help => IDLE Doc. + gh-96905: In idlelib code, stop redefining built-ins 'dict' and 'object'. gh-72284: Improve the lists of features, editor key bindings, @@ -566,14 +578,14 @@ bpo-33679: Enable theme-specific color configuration for Code Context. color setting, default or custom, on the extensions tab, that applied to all themes.) For built-in themes, the foreground is the same as normal text and the background is a contrasting gray. Context colors for -custom themes are set on the Hightlights tab along with other colors. +custom themes are set on the Highlights tab along with other colors. When one starts IDLE from a console and loads a custom theme without definitions for 'context', one will see a warning message on the console. bpo-33642: Display up to maxlines non-blank lines for Code Context. If there is no current context, show a single blank line. (Previously, -the Code Contex had numlines lines, usually with some blank.) The use +the Code Context had numlines lines, usually with some blank.) The use of a new option, 'maxlines' (default 15), avoids possible interference with user settings of the old option, 'numlines' (default 3). @@ -727,7 +739,7 @@ not affect their keyset-specific customization after 3.6.3. and vice versa. Initial patch by Charles Wohlganger, revised by Terry Jan Reedy. -bpo-31051: Rearrange condigdialog General tab. +bpo-31051: Rearrange configdialog General tab. Sort non-Help options into Window (Shell+Editor) and Editor (only). Leave room for the addition of new options. Patch by Terry Jan Reedy. diff --git a/Lib/idlelib/TODO.txt b/Lib/idlelib/TODO.txt index e2f1ac0f274001..41b86b0c6d5bbd 100644 --- a/Lib/idlelib/TODO.txt +++ b/Lib/idlelib/TODO.txt @@ -179,7 +179,7 @@ it -- i.e. you can only edit the current command, and the cursor can't escape from the command area. (Albert Brandl) - Set X11 class to "idle/Idle", set icon and title to something -beginning with "idle" -- for window manangers. (Randall Hopper) +beginning with "idle" -- for window managers. (Randall Hopper) - Config files editable through a preferences dialog. (me) DONE diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 92992fd9cce9cd..d10c88a43f9231 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -158,8 +158,9 @@ def __init__(self, _utest=False): self.defaultCfg = {} self.userCfg = {} self.cfg = {} # TODO use to select userCfg vs defaultCfg + + # See https://bugs.python.org/issue4630#msg356516 for following. # self.blink_off_time = ['insertofftime'] - # See https:/bugs.python.org/issue4630, msg356516. if not _utest: self.CreateConfigHandlers() @@ -599,7 +600,7 @@ def GetCoreKeys(self, keySetName=None): """ # TODO: = dict(sorted([(v-event, keys), ...]))? keyBindings={ - # vitual-event: list of key events. + # virtual-event: list of key events. '<>': ['', ''], '<>': ['', ''], '<>': ['', ''], diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index eedf97bf74fe6a..4d2adb48570d49 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -111,7 +111,7 @@ def create_widgets(self): load_configs: Load pages except for extensions. activate_config_changes: Tell editors to reload. """ - self.frame = frame = Frame(self, padding="5px") + self.frame = frame = Frame(self, padding=5) self.frame.grid(sticky="nwes") self.note = note = Notebook(frame) self.extpage = ExtPage(note) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 8ee8eba64367a5..c76db20c58792d 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -914,7 +914,7 @@ def RemoveKeybindings(self): def ApplyKeybindings(self): """Apply the virtual, configurable keybindings. - Alse update hotkeys to current keyset. + Also update hotkeys to current keyset. """ # Called from configdialog.activate_config_changes. self.mainmenu.default_keydefs = keydefs = idleConf.GetCurrentKeySet() @@ -1044,7 +1044,9 @@ def open_recent_file(fn_closure=file_name): def saved_change_hook(self): short = self.short_title() long = self.long_title() - if short and long: + if short and long and not macosx.isCocoaTk(): + # Don't use both values on macOS because + # that doesn't match platform conventions. title = short + " - " + long + _py_version elif short: title = short @@ -1059,6 +1061,13 @@ def saved_change_hook(self): self.top.wm_title(title) self.top.wm_iconname(icon) + if macosx.isCocoaTk(): + # Add a proxy icon to the window title + self.top.wm_attributes("-titlepath", long) + + # Maintain the modification status for the window + self.top.wm_attributes("-modified", not self.get_saved()) + def get_saved(self): return self.undo.get_saved() diff --git a/Lib/idlelib/extend.txt b/Lib/idlelib/extend.txt index b482f76c4fb0f7..2522758ceb4f70 100644 --- a/Lib/idlelib/extend.txt +++ b/Lib/idlelib/extend.txt @@ -52,7 +52,7 @@ should probably be refined in the future.) Extensions are not required to define menu entries for all the events they implement. (They are also not required to create keybindings, but in that -case there must be empty bindings in cofig-extensions.def) +case there must be empty bindings in config-extensions.def) Here is a partial example from zzdummy.py: diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py index ef14349960bfa2..42048ff2395fe1 100644 --- a/Lib/idlelib/grep.py +++ b/Lib/idlelib/grep.py @@ -190,7 +190,7 @@ def grep_it(self, prog, path): def _grep_dialog(parent): # htest # - from tkinter import Toplevel, Text, SEL, END + from tkinter import Toplevel, Text, SEL from tkinter.ttk import Frame, Button from idlelib.pyshell import PyShellFileList diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 827d230b54e159..2a4adc6a4d395f 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -5,7 +5,7 @@ - IDLE — Python 3.13.0a2 documentation + IDLE — Python 3.14.0a0 documentation @@ -18,7 +18,7 @@ @@ -26,6 +26,7 @@ + @@ -45,6 +46,8 @@ + + @@ -184,7 +187,7 @@

    Navigation

  • - 3.13.0a2 Documentation » + 3.14.0a0 Documentation »
  • @@ -554,7 +557,7 @@

    Key bindingsControl key on Windows and -Unix and the Command key on macOS. (And all such dicussions +Unix and the Command key on macOS. (And all such discussions assume that the keys have not been re-bound to something else.)

    @@ -694,7 +697,7 @@

    Shell window -
  • C-c attemps to interrupt statement execution (but may fail).

  • +
  • C-c attempts to interrupt statement execution (but may fail).

  • C-d closes Shell if typed at a >>> prompt.

  • Alt-p and Alt-n (C-p and C-n on macOS) retrieve to the current prompt the previous or next previously @@ -1136,7 +1139,7 @@

    Navigation

  • - 3.13.0a2 Documentation » + 3.14.0a0 Documentation »
  • @@ -1180,7 +1183,7 @@

    Navigation



    - Last updated on Jan 17, 2024 (06:57 UTC). + Last updated on Oct 14, 2024 (20:27 UTC). Found a bug?
    diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index bdf4b2b29f11a2..d8613b2eadd6aa 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -33,6 +33,7 @@ from tkinter import font as tkfont from idlelib.config import idleConf +from idlelib.colorizer import color_config ## About IDLE ## @@ -177,14 +178,16 @@ def __init__(self, parent, filename): normalfont = self.findfont(['TkDefaultFont', 'arial', 'helvetica']) fixedfont = self.findfont(['TkFixedFont', 'monaco', 'courier']) + color_config(self) self['font'] = (normalfont, 12) self.tag_configure('em', font=(normalfont, 12, 'italic')) self.tag_configure('h1', font=(normalfont, 20, 'bold')) self.tag_configure('h2', font=(normalfont, 18, 'bold')) self.tag_configure('h3', font=(normalfont, 15, 'bold')) - self.tag_configure('pre', font=(fixedfont, 12), background='#f6f6ff') + self.tag_configure('pre', font=(fixedfont, 12)) + preback = self['selectbackground'] self.tag_configure('preblock', font=(fixedfont, 10), lmargin1=25, - borderwidth=1, relief='solid', background='#eeffcc') + background=preback) self.tag_configure('l1', lmargin1=25, lmargin2=25) self.tag_configure('l2', lmargin1=50, lmargin2=50) self.tag_configure('l3', lmargin1=75, lmargin2=75) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index aa1c352897f9e7..81c65f6264e7b9 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -85,15 +85,18 @@ def create_widgets(self): byline = Label(frame_background, text=byline_text, justify=LEFT, fg=self.fg, bg=self.bg) byline.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) - email = Label(frame_background, text='email: idle-dev@python.org', - justify=LEFT, fg=self.fg, bg=self.bg) - email.grid(row=6, column=0, columnspan=2, sticky=W, padx=10, pady=0) + + forums_url = "https://discuss.python.org" + forums = Label(frame_background, text="Python forums: "+forums_url, + justify=LEFT, fg=self.fg, bg=self.bg) + forums.grid(row=6, column=0, sticky=W, padx=10, pady=0) + forums.bind("", lambda event: webbrowser.open(forums_url)) docs_url = ("https://docs.python.org/%d.%d/library/idle.html" % sys.version_info[:2]) docs = Label(frame_background, text=docs_url, justify=LEFT, fg=self.fg, bg=self.bg) docs.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) - docs.bind("", lambda event: webbrowser.open(docs['text'])) + docs.bind("", lambda event: webbrowser.open(docs_url)) Frame(frame_background, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, @@ -123,9 +126,7 @@ def create_widgets(self): height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - idle = Label(frame_background, - text='IDLE', - fg=self.fg, bg=self.bg) + idle = Label(frame_background, text='IDLE', fg=self.fg, bg=self.bg) idle.grid(row=12, column=0, sticky=W, padx=10, pady=0) idle_buttons = Frame(frame_background, bg=self.bg) idle_buttons.grid(row=13, column=0, columnspan=3, sticky=NSEW) diff --git a/Lib/idlelib/idle_test/example_stub.pyi b/Lib/idlelib/idle_test/example_stub.pyi index 17b58010a9d8de..abcdbc17529974 100644 --- a/Lib/idlelib/idle_test/example_stub.pyi +++ b/Lib/idlelib/idle_test/example_stub.pyi @@ -1,4 +1,4 @@ -" Example to test recognition of .pyi file as Python source code. +# An example file to test recognition of a .pyi file as Python source code. class Example: def method(self, argument1: str, argument2: list[int]) -> None: ... diff --git a/Lib/idlelib/idle_test/test_outwin.py b/Lib/idlelib/idle_test/test_outwin.py index d6e85ad674417c..81f4aad7e95e95 100644 --- a/Lib/idlelib/idle_test/test_outwin.py +++ b/Lib/idlelib/idle_test/test_outwin.py @@ -1,6 +1,7 @@ "Test outwin, coverage 76%." from idlelib import outwin +import sys import unittest from test.support import requires from tkinter import Tk, Text @@ -18,6 +19,10 @@ def setUpClass(cls): root.withdraw() w = cls.window = outwin.OutputWindow(None, None, None, root) cls.text = w.text = Text(root) + if sys.platform == 'darwin': # Issue 112938 + cls.text.update = cls.text.update_idletasks + # Without this, test write, writelines, and goto... fail. + # The reasons and why macOS-specific are unclear. @classmethod def tearDownClass(cls): diff --git a/Lib/idlelib/idle_test/test_run.py b/Lib/idlelib/idle_test/test_run.py index a38e43dcb9d1c4..83ecbffa2a197e 100644 --- a/Lib/idlelib/idle_test/test_run.py +++ b/Lib/idlelib/idle_test/test_run.py @@ -8,6 +8,7 @@ from unittest import mock import idlelib from idlelib.idle_test.mock_idle import Func +from test.support import force_not_colorized idlelib.testing = True # Use {} for executing test user code. @@ -46,6 +47,7 @@ def __eq__(self, other): "Did you mean: 'real'?\n"), ) + @force_not_colorized def test_get_message(self): for code, exc, msg in self.data: with self.subTest(code=code): @@ -57,6 +59,7 @@ def test_get_message(self): expect = f'{exc.__name__}: {msg}' self.assertEqual(actual, expect) + @force_not_colorized @mock.patch.object(run, 'cleanup_traceback', new_callable=lambda: (lambda t, e: None)) def test_get_multiple_message(self, mock): diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index 5ed3f35a7af655..8baa657550de94 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -112,7 +112,7 @@ def write(self, s, tags=(), mark="insert"): assert isinstance(s, str) self.text.insert(mark, s, tags) self.text.see(mark) - self.text.update_idletasks() + self.text.update() return len(s) def writelines(self, lines): diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 1524fccd5d20f8..e882c6cb3b8d19 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -11,15 +11,9 @@ "Your Python may not be configured for Tk. **", file=sys.__stderr__) raise SystemExit(1) -# Valid arguments for the ...Awareness call below are defined in the following. -# https://msdn.microsoft.com/en-us/library/windows/desktop/dn280512(v=vs.85).aspx if sys.platform == 'win32': - try: - import ctypes - PROCESS_SYSTEM_DPI_AWARE = 1 # Int required. - ctypes.OleDLL('shcore').SetProcessDpiAwareness(PROCESS_SYSTEM_DPI_AWARE) - except (ImportError, AttributeError, OSError): - pass + from idlelib.util import fix_win_hidpi + fix_win_hidpi() from tkinter import messagebox @@ -712,7 +706,7 @@ def prepend_syspath(self, filename): del _filename, _sys, _dirname, _dir \n""".format(filename)) - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Override Interactive Interpreter method: Use Colorizing Color the offending position instead of printing it and pointing at it diff --git a/Lib/idlelib/redirector.py b/Lib/idlelib/redirector.py index 08728956abd900..8e2ba68d3815bf 100644 --- a/Lib/idlelib/redirector.py +++ b/Lib/idlelib/redirector.py @@ -106,6 +106,7 @@ def dispatch(self, operation, *args): to *args to accomplish that. For an example, see colorizer.py. ''' + operation = str(operation) # can be a Tcl_Obj m = self._operations.get(operation) try: if m: diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 53e80a9b42801f..a30db99a619a93 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -91,21 +91,28 @@ def capture_warnings(capture): _warnings_showwarning = None capture_warnings(True) -tcl = tkinter.Tcl() -def handle_tk_events(tcl=tcl): - """Process any tk events that are ready to be dispatched if tkinter - has been imported, a tcl interpreter has been created and tk has been - loaded.""" - tcl.eval("update") +if idlelib.testing: + # gh-121008: When testing IDLE, don't create a Tk object to avoid side + # effects such as installing a PyOS_InputHook hook. + def handle_tk_events(): + pass +else: + tcl = tkinter.Tcl() + + def handle_tk_events(tcl=tcl): + """Process any tk events that are ready to be dispatched if tkinter + has been imported, a tcl interpreter has been created and tk has been + loaded.""" + tcl.eval("update") # Thread shared globals: Establish a queue between a subthread (which handles # the socket) and the main thread (which runs user code), plus global -# completion, exit and interruptable (the main thread) flags: +# completion, exit and interruptible (the main thread) flags: exit_now = False quitting = False -interruptable = False +interruptible = False def main(del_exitfunc=False): """Start the Python execution server in a subprocess @@ -436,6 +443,9 @@ class StdioFile(io.TextIOBase): def __init__(self, shell, tags, encoding='utf-8', errors='strict'): self.shell = shell + # GH-78889: accessing unpickleable attributes freezes Shell. + # IDLE only needs methods; allow 'width' for possible use. + self.shell._RPCProxy__attributes = {'width': 1} self.tags = tags self._encoding = encoding self._errors = errors @@ -572,14 +582,14 @@ def __init__(self, rpchandler): self.locals = {} def runcode(self, code): - global interruptable + global interruptible try: self.user_exc_info = None - interruptable = True + interruptible = True try: exec(code, self.locals) finally: - interruptable = False + interruptible = False except SystemExit as e: if e.args: # SystemExit called with an argument. ob = e.args[0] @@ -605,7 +615,7 @@ def runcode(self, code): flush_stdout() def interrupt_the_server(self): - if interruptable: + if interruptible: thread.interrupt_main() def start_the_debugger(self, gui_adap_oid): diff --git a/Lib/idlelib/searchbase.py b/Lib/idlelib/searchbase.py index 64ed50c7364be3..c68a6ca339af04 100644 --- a/Lib/idlelib/searchbase.py +++ b/Lib/idlelib/searchbase.py @@ -86,7 +86,7 @@ def create_widgets(self): top.wm_iconname(self.icon) _setup_dialog(top) self.top = top - self.frame = Frame(top, padding="5px") + self.frame = Frame(top, padding=5) self.frame.grid(sticky="nwes") top.grid_columnconfigure(0, weight=100) top.grid_rowconfigure(0, weight=100) diff --git a/Lib/idlelib/sidebar.py b/Lib/idlelib/sidebar.py index ff77b568a786e0..aa19a24e3edef2 100644 --- a/Lib/idlelib/sidebar.py +++ b/Lib/idlelib/sidebar.py @@ -27,7 +27,7 @@ def get_displaylines(text, index): """Display height, in lines, of a logical line in a Tk text widget.""" return text.count(f"{index} linestart", f"{index} lineend", - "displaylines") + "displaylines", return_ints=True) def get_widget_padding(widget): """Get the total padding of a Tk widget, including its border.""" diff --git a/Lib/idlelib/tooltip.py b/Lib/idlelib/tooltip.py index 3983690dd41177..df5b1fe1dcfb08 100644 --- a/Lib/idlelib/tooltip.py +++ b/Lib/idlelib/tooltip.py @@ -144,7 +144,8 @@ def hidetip(self): class Hovertip(OnHoverTooltipBase): "A tooltip that pops up when a mouse hovers over an anchor widget." - def __init__(self, anchor_widget, text, hover_delay=1000): + def __init__(self, anchor_widget, text, hover_delay=1000, + foreground="#000000", background="#ffffe0"): """Create a text tooltip with a mouse hover delay. anchor_widget: the widget next to which the tooltip will be shown @@ -156,10 +157,13 @@ def __init__(self, anchor_widget, text, hover_delay=1000): """ super().__init__(anchor_widget, hover_delay=hover_delay) self.text = text + self.foreground = foreground + self.background = background def showcontents(self): label = Label(self.tipwindow, text=self.text, justify=LEFT, - background="#ffffe0", relief=SOLID, borderwidth=1) + relief=SOLID, borderwidth=1, + foreground=self.foreground, background=self.background) label.pack() diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 0726d7e23660f6..182ce7189614da 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -83,6 +83,8 @@ def wheel_event(event, widget=None): class TreeNode: + dy = 0 + def __init__(self, canvas, parent, item): self.canvas = canvas self.parent = parent @@ -199,23 +201,22 @@ def update(self): def draw(self, x, y): # XXX This hard-codes too many geometry constants! - dy = 20 self.x, self.y = x, y self.drawicon() self.drawtext() if self.state != 'expanded': - return y + dy + return y + TreeNode.dy # draw children if not self.children: sublist = self.item._GetSubList() if not sublist: # _IsExpandable() was mistaken; that's allowed - return y+17 + return y + TreeNode.dy for item in sublist: child = self.__class__(self.canvas, self, item) self.children.append(child) cx = x+20 - cy = y + dy + cy = y + TreeNode.dy cylast = 0 for child in self.children: cylast = cy @@ -289,6 +290,11 @@ def drawtext(self): self.label.bind("", lambda e: wheel_event(e, self.canvas)) self.label.bind("", lambda e: wheel_event(e, self.canvas)) self.text_id = id + if TreeNode.dy == 0: + # The first row doesn't matter what the dy is, just measure its + # size to get the value of the subsequent dy + coords = self.canvas.bbox(id) + TreeNode.dy = max(20, coords[3] - coords[1] - 3) def select_or_edit(self, event=None): if self.selected and self.item.IsEditable(): diff --git a/Lib/idlelib/util.py b/Lib/idlelib/util.py index a7ae74b0579004..e05604ab4853f6 100644 --- a/Lib/idlelib/util.py +++ b/Lib/idlelib/util.py @@ -12,11 +12,26 @@ * std streams (pyshell, run), * warning stuff (pyshell, run). """ +import sys # .pyw is for Windows; .pyi is for typing stub files. # The extension order is needed for iomenu open/save dialogs. py_extensions = ('.py', '.pyw', '.pyi') + +# Fix for HiDPI screens on Windows. CALL BEFORE ANY TK OPERATIONS! +# URL for arguments for the ...Awareness call below. +# https://msdn.microsoft.com/en-us/library/windows/desktop/dn280512(v=vs.85).aspx +if sys.platform == 'win32': # pragma: no cover + def fix_win_hidpi(): # Called in pyshell and turtledemo. + try: + import ctypes + PROCESS_SYSTEM_DPI_AWARE = 1 # Int required. + ctypes.OleDLL('shcore').SetProcessDpiAwareness(PROCESS_SYSTEM_DPI_AWARE) + except (ImportError, AttributeError, OSError): + pass + + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_util', verbosity=2) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 577b4b9b03a88d..e576c29e67dc0a 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -239,7 +239,7 @@ def _connect(self): if __debug__: self._cmd_log_len = 10 self._cmd_log_idx = 0 - self._cmd_log = {} # Last `_cmd_log_len' interactions + self._cmd_log = {} # Last '_cmd_log_len' interactions if self.debug >= 1: self._mesg('imaplib version %s' % __version__) self._mesg('new IMAP4 connection, tag=%s' % self.tagpre) @@ -396,7 +396,7 @@ def append(self, mailbox, flags, date_time, message): (typ, [data]) = .append(mailbox, flags, date_time, message) - All args except `message' can be None. + All args except 'message' can be None. """ name = 'APPEND' if not mailbox: @@ -927,7 +927,7 @@ def xatom(self, name, *args): (typ, [data]) = .xatom(name, arg, ...) - Returns response appropriate to extension command `name'. + Returns response appropriate to extension command 'name'. """ name = name.upper() #if not name in self.capabilities: # Let the server decide! @@ -1167,7 +1167,7 @@ def _get_tagged_response(self, tag, expect_bye=False): # Some have reported "unexpected response" exceptions. # Note that ignoring them here causes loops. # Instead, send me details of the unexpected response and - # I'll update the code in `_get_response()'. + # I'll update the code in '_get_response()'. try: self._get_response() @@ -1259,7 +1259,7 @@ def _dump_ur(self, untagged_resp_dict): self._mesg('untagged responses dump:' + '\n\t\t'.join(items)) def _log(self, line): - # Keep log of last `_cmd_log_len' interactions for debugging. + # Keep log of last '_cmd_log_len' interactions for debugging. self._cmd_log[self._cmd_log_idx] = (line, time.time()) self._cmd_log_idx += 1 if self._cmd_log_idx >= self._cmd_log_len: diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index f38fe5c1ab461a..a7d57561ead046 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -103,7 +103,7 @@ def reload(module): try: name = module.__name__ except AttributeError: - raise TypeError("reload() argument must be a module") + raise TypeError("reload() argument must be a module") from None if sys.modules.get(name) is not module: raise ImportError(f"module {name} not in sys.modules", name=name) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 6d6292f9559253..b70d09b32abce6 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1134,7 +1134,7 @@ def find_spec(cls, fullname, path=None, target=None): # part of the importer), instead of here (the finder part). # The loader is the usual place to get the data that will # be loaded into the module. (For example, see _LoaderBasics - # in _bootstra_external.py.) Most importantly, this importer + # in _bootstrap_external.py.) Most importantly, this importer # is simpler if we wait to get the data. # However, getting as much data in the finder as possible # to later load the module is okay, and sometimes important. @@ -1241,7 +1241,6 @@ def _find_spec(name, path, target=None): """Find a module's spec.""" meta_path = sys.meta_path if meta_path is None: - # PyImport_Cleanup() is running or has been called. raise ImportError("sys.meta_path is None, Python is likely " "shutting down") diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 2a9aef03179f6f..1b76328429f63a 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -52,7 +52,7 @@ # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', -_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos' _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) @@ -81,6 +81,11 @@ def _pack_uint32(x): return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') +def _unpack_uint64(data): + """Convert 8 bytes in little-endian to an integer.""" + assert len(data) == 8 + return int.from_bytes(data, 'little') + def _unpack_uint32(data): """Convert 4 bytes in little-endian to an integer.""" assert len(data) == 4 @@ -216,274 +221,7 @@ def _write_atomic(path, data, mode=0o666): _code_type = type(_write_atomic.__code__) - -# Finder/loader utility code ############################################### - -# Magic word to reject .pyc files generated by other Python versions. -# It should change for each incompatible change to the bytecode. -# -# The value of CR and LF is incorporated so if you ever read or write -# a .pyc file in text mode the magic number will be wrong; also, the -# Apple MPW compiler swaps their values, botching string constants. -# -# There were a variety of old schemes for setting the magic number. -# The current working scheme is to increment the previous value by -# 10. -# -# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -# number also includes a new "magic tag", i.e. a human readable string used -# to represent the magic number in __pycache__ directories. When you change -# the magic number, you must also set a new unique magic tag. Generally this -# can be named after the Python major version of the magic number bump, but -# it can really be anything, as long as it's different than anything else -# that's come before. The tags are included in the following table, starting -# with Python 3.2a0. -# -# Known values: -# Python 1.5: 20121 -# Python 1.5.1: 20121 -# Python 1.5.2: 20121 -# Python 1.6: 50428 -# Python 2.0: 50823 -# Python 2.0.1: 50823 -# Python 2.1: 60202 -# Python 2.1.1: 60202 -# Python 2.1.2: 60202 -# Python 2.2: 60717 -# Python 2.3a0: 62011 -# Python 2.3a0: 62021 -# Python 2.3a0: 62011 (!) -# Python 2.4a0: 62041 -# Python 2.4a3: 62051 -# Python 2.4b1: 62061 -# Python 2.5a0: 62071 -# Python 2.5a0: 62081 (ast-branch) -# Python 2.5a0: 62091 (with) -# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) -# Python 2.5b3: 62101 (fix wrong code: for x, in ...) -# Python 2.5b3: 62111 (fix wrong code: x += yield) -# Python 2.5c1: 62121 (fix wrong lnotab with for loops and -# storing constants that should have been removed) -# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) -# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) -# Python 2.6a1: 62161 (WITH_CLEANUP optimization) -# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) -# Python 2.7a0: 62181 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -# Python 2.7a0 62191 (introduce SETUP_WITH) -# Python 2.7a0 62201 (introduce BUILD_SET) -# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) -# Python 3000: 3000 -# 3010 (removed UNARY_CONVERT) -# 3020 (added BUILD_SET) -# 3030 (added keyword-only parameters) -# 3040 (added signature annotations) -# 3050 (print becomes a function) -# 3060 (PEP 3115 metaclass syntax) -# 3061 (string literals become unicode) -# 3071 (PEP 3109 raise changes) -# 3081 (PEP 3137 make __file__ and __name__ unicode) -# 3091 (kill str8 interning) -# 3101 (merge from 2.6a0, see 62151) -# 3103 (__file__ points to source file) -# Python 3.0a4: 3111 (WITH_CLEANUP optimization). -# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT - #3021) -# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: -# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) -# Python 3.1a1: 3151 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE - #4715) -# Python 3.2a1: 3160 (add SETUP_WITH #6101) -# tag: cpython-32 -# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) -# tag: cpython-32 -# Python 3.2a3 3180 (add DELETE_DEREF #4617) -# Python 3.3a1 3190 (__class__ super closure changed) -# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) -# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) -# Python 3.3a2 3220 (changed PEP 380 implementation #14230) -# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) -# Python 3.4a1 3250 (evaluate positional default arguments before -# keyword-only defaults #16967) -# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override -# free vars #17853) -# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) -# Python 3.4a1 3280 (remove implicit class argument) -# Python 3.4a4 3290 (changes to __qualname__ computation #19301) -# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) -# Python 3.4rc2 3310 (alter __qualname__ computation #20625) -# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) -# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) -# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) -# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) -# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) -# Python 3.6a2 3370 (16 bit wordcode #26647) -# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) -# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE -# #27095) -# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) -# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes -# #27985) -# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL - #27213) -# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) -# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) -# Python 3.6rc1 3379 (more thorough __class__ validation #23722) -# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) -# Python 3.7a2 3391 (update GET_AITER #31709) -# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) -# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7b5 3394 (restored docstring as the first stmt in the body; -# this might affected the first line number #32911) -# Python 3.8a1 3400 (move frame block handling to compiler #17611) -# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) -# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) -# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict -# comprehensions #35224) -# Python 3.8b2 3412 (Swap the position of positional args and positional -# only args in ast.arguments #37593) -# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) -# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) -# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) -# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) -# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) -# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) -# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) -# Python 3.10a1 3430 (Make 'annotations' future by default) -# Python 3.10a1 3431 (New line number table format -- PEP 626) -# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) -# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) -# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) -# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). -# Python 3.10b1 3436 (Add GEN_START bytecode #43683) -# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) -# Python 3.10b1 3438 Safer line number table handling. -# Python 3.10b1 3439 (Add ROT_N) -# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) -# Python 3.11a1 3451 (Add CALL_METHOD_KW) -# Python 3.11a1 3452 (drop nlocals from marshaled code objects) -# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) -# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) -# Python 3.11a1 3455 (add MAKE_CELL bpo-43693) -# Python 3.11a1 3456 (interleave cell args bpo-43693) -# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) -# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) -# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) -# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) -# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) -# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change -# MATCH_CLASS and MATCH_KEYS, and add COPY) -# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the -# active exception) -# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into -# BINARY_OP) -# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) -# Python 3.11a4 3466 (bpo-45292: PEP-654 except*) -# Python 3.11a4 3467 (Change CALL_xxx opcodes) -# Python 3.11a4 3468 (Add SEND opcode) -# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) -# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) -# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) -# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) -# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) -# Python 3.11a4 3474 (Add RESUME opcode) -# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) -# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) -# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and -# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) -# Python 3.11a5 3478 (New CALL opcodes) -# Python 3.11a5 3479 (Add PUSH_NULL opcode) -# Python 3.11a5 3480 (New CALL opcodes, second iteration) -# Python 3.11a5 3481 (Use inline cache for BINARY_OP) -# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) -# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) -# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and -# STORE_ATTR) -# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) -# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) -# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) -# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) -# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) -# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) -# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, -# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) -# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) -# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) -# Python 3.11a7 3494 (New location info table) -# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) -# Python 3.12a1 3500 (Remove PRECALL opcode) -# Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) -# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) -# Python 3.12a1 3503 (Shrink LOAD_METHOD cache) -# Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) -# Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) -# Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) -# Python 3.12a1 3507 (Set lineno of module's RESUME to 0) -# Python 3.12a1 3508 (Add CLEANUP_THROW) -# Python 3.12a1 3509 (Conditional jumps only jump forward) -# Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) -# Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) -# Python 3.12a2 3512 (Remove all unused consts from code objects) -# Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) -# Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) -# Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) -# Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) -# Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a6 3518 (Add RETURN_CONST instruction) -# Python 3.12a6 3519 (Modify SEND instruction) -# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) -# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) -# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) -# Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) -# Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) -# Python 3.12b1 3525 (Shrink the CALL caches) -# Python 3.12b1 3526 (Add instrumentation support) -# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) -# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) -# Python 3.12b1 3529 (Inline list/dict/set comprehensions) -# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) -# Python 3.12b1 3531 (Add PEP 695 changes) -# Python 3.13a1 3550 (Plugin optimizer support) -# Python 3.13a1 3551 (Compact superinstructions) -# Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) -# Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) -# Python 3.13a1 3554 (more efficient bytecodes for f-strings) -# Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) -# Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) -# Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) -# Python 3.13a1 3558 (Reorder the stack items for CALL) -# Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) -# Python 3.13a1 3560 (Add RESUME_CHECK instruction) -# Python 3.13a1 3561 (Add cache entry to branch instructions) -# Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) -# Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) -# Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) -# Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) -# Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) -# Python 3.13a1 3567 (Reimplement line number propagation by the compiler) -# Python 3.13a1 3568 (Change semantics of END_FOR) - -# Python 3.14 will start with 3600 - -# Please don't copy-paste the same pre-release tag for new entries above!!! -# You should always use the *upcoming* tag. For example, if 3.12a6 came out -# a week ago, I should put "Python 3.12a7" next to my new magic number. - -# MAGIC must change whenever the bytecode emitted by the compiler may no -# longer be understood by older implementations of the eval loop (usually -# due to the addition of new opcodes). -# -# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. -# -# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array -# in PC/launcher.c must also be updated. - -MAGIC_NUMBER = (3568).to_bytes(2, 'little') + b'\r\n' - -_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c +MAGIC_NUMBER = _imp.pyc_magic_number_token.to_bytes(4, 'little') _PYCACHE = '__pycache__' _OPT = 'opt-' @@ -1122,7 +860,7 @@ def get_code(self, fullname): _imp.check_hash_based_pycs == 'always')): source_bytes = self.get_data(source_path) source_hash = _imp.source_hash( - _RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) _validate_hash_pyc(data, source_hash, fullname, @@ -1151,7 +889,7 @@ def get_code(self, fullname): source_mtime is not None): if hash_based: if source_hash is None: - source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, + source_hash = _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) data = _code_to_hash_pyc(code_object, source_hash, check_source) else: @@ -1457,7 +1195,7 @@ class PathFinder: @staticmethod def invalidate_caches(): """Call the invalidate_caches() method on all path entry finders - stored in sys.path_importer_caches (where implemented).""" + stored in sys.path_importer_cache (where implemented).""" for name, finder in list(sys.path_importer_cache.items()): # Drop entry if finder name is a relative path. The current # working directory may have changed. @@ -1469,6 +1207,9 @@ def invalidate_caches(): # https://bugs.python.org/issue45703 _NamespacePath._epoch += 1 + from importlib.metadata import MetadataPathFinder + MetadataPathFinder.invalidate_caches() + @staticmethod def _path_hooks(path): """Search sys.path_hooks for a finder for 'path'.""" @@ -1710,6 +1451,46 @@ def __repr__(self): return f'FileFinder({self.path!r})' +class AppleFrameworkLoader(ExtensionFileLoader): + """A loader for modules that have been packaged as frameworks for + compatibility with Apple's iOS App Store policies. + """ + def create_module(self, spec): + # If the ModuleSpec has been created by the FileFinder, it will have + # been created with an origin pointing to the .fwork file. We need to + # redirect this to the location in the Frameworks folder, using the + # content of the .fwork file. + if spec.origin.endswith(".fwork"): + with _io.FileIO(spec.origin, 'r') as file: + framework_binary = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + spec.origin = _path_join(bundle_path, framework_binary) + + # If the loader is created based on the spec for a loaded module, the + # path will be pointing at the Framework location. If this occurs, + # get the original .fwork location to use as the module's __file__. + if self.path.endswith(".fwork"): + path = self.path + else: + with _io.FileIO(self.path + ".origin", 'r') as file: + origin = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + path = _path_join(bundle_path, origin) + + module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) + + _bootstrap._verbose_message( + "Apple framework extension module {!r} loaded from {!r} (path {!r})", + spec.name, + spec.origin, + path, + ) + + # Ensure that the __file__ points at the .fwork location + module.__file__ = path + + return module + # Import setup ############################################################### def _fix_up_module(ns, name, pathname, cpathname=None): @@ -1742,10 +1523,17 @@ def _get_supported_file_loaders(): Each item is a tuple (loader, suffixes). """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() + extension_loaders = [] + if hasattr(_imp, 'create_dynamic'): + if sys.platform in {"ios", "tvos", "watchos"}: + extension_loaders = [(AppleFrameworkLoader, [ + suffix.replace(".so", ".fwork") + for suffix in _imp.extension_suffixes() + ])] + extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes())) source = SourceFileLoader, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [extensions, source, bytecode] + return extension_loaders + [source, bytecode] def _set_bootstrap_module(_bootstrap_module): diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index b56fa94eb9c135..eea6b38af6fa13 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -13,9 +13,6 @@ _frozen_importlib_external = _bootstrap_external from ._abc import Loader import abc -import warnings - -from .resources import abc as _resources_abc __all__ = [ @@ -25,19 +22,6 @@ ] -def __getattr__(name): - """ - For backwards compatibility, continue to make names - from _resources_abc available through this module. #93963 - """ - if name in _resources_abc.__all__: - obj = getattr(_resources_abc, name) - warnings._deprecated(f"{__name__}.{name}", remove=(3, 14)) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - def _register(abstract_cls, *classes): for cls in classes: abstract_cls.register(cls) @@ -180,7 +164,11 @@ def get_code(self, fullname): else: return self.source_to_code(source, path) -_register(ExecutionLoader, machinery.ExtensionFileLoader) +_register( + ExecutionLoader, + machinery.ExtensionFileLoader, + machinery.AppleFrameworkLoader, +) class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index d9a19a13f7b275..6e294d59bfdcb9 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -12,9 +12,18 @@ from ._bootstrap_external import SourceFileLoader from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import AppleFrameworkLoader from ._bootstrap_external import NamespaceLoader def all_suffixes(): """Returns a list of all recognized module suffixes for this process""" return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES + + +__all__ = ['AppleFrameworkLoader', 'BYTECODE_SUFFIXES', 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', 'FileFinder', 'FrozenImporter', 'ModuleSpec', + 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', + 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', + 'WindowsRegistryFinder', 'all_suffixes'] diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index c612fbefee2e80..8ce62dd864fc27 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import re import abc @@ -16,7 +18,7 @@ import posixpath import collections -from . import _adapters, _meta +from . import _meta from ._collections import FreezableDefaultDict, Pair from ._functools import method_cache, pass_none from ._itertools import always_iterable, unique_everseen @@ -26,7 +28,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import Iterable, List, Mapping, Optional, Set, Union, cast +from typing import Any, Iterable, List, Mapping, Match, Optional, Set, cast __all__ = [ 'Distribution', @@ -163,17 +165,17 @@ class EntryPoint: value: str group: str - dist: Optional['Distribution'] = None + dist: Optional[Distribution] = None def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) - def load(self): + def load(self) -> Any: """Load the entry point from its definition. If only a module is indicated by the value, return that module. Otherwise, return the named object. """ - match = self.pattern.match(self.value) + match = cast(Match, self.pattern.match(self.value)) module = import_module(match.group('module')) attrs = filter(None, (match.group('attr') or '').split('.')) return functools.reduce(getattr, attrs, module) @@ -268,7 +270,7 @@ def __repr__(self): """ return '%s(%r)' % (self.__class__.__name__, tuple(self)) - def select(self, **params): + def select(self, **params) -> EntryPoints: """ Select entry points from self that match the given parameters (typically group and/or name). @@ -304,19 +306,17 @@ def _from_text(text): class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" - hash: Optional["FileHash"] + hash: Optional[FileHash] size: int - dist: "Distribution" + dist: Distribution def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] - with self.locate().open(encoding=encoding) as stream: - return stream.read() + return self.locate().read_text(encoding=encoding) def read_binary(self) -> bytes: - with self.locate().open('rb') as stream: - return stream.read() + return self.locate().read_bytes() - def locate(self) -> pathlib.Path: + def locate(self) -> SimplePath: """Return a path-like object for this path""" return self.dist.locate_file(self) @@ -330,6 +330,7 @@ def __repr__(self) -> str: class DeprecatedNonAbstract: + # Required until Python 3.14 def __new__(cls, *args, **kwargs): all_names = { name for subclass in inspect.getmro(cls) for name in vars(subclass) @@ -349,25 +350,48 @@ def __new__(cls, *args, **kwargs): class Distribution(DeprecatedNonAbstract): - """A Python distribution package.""" + """ + An abstract Python distribution package. + + Custom providers may derive from this class and define + the abstract methods to provide a concrete implementation + for their environment. Some providers may opt to override + the default implementation of some properties to bypass + the file-reading mechanism. + """ @abc.abstractmethod def read_text(self, filename) -> Optional[str]: """Attempt to load metadata file given by the name. + Python distribution metadata is organized by blobs of text + typically represented as "files" in the metadata directory + (e.g. package-1.0.dist-info). These files include things + like: + + - METADATA: The distribution metadata including fields + like Name and Version and Description. + - entry_points.txt: A series of entry points as defined in + `the entry points spec `_. + - RECORD: A record of files according to + `this recording spec `_. + + A package may provide any set of files, including those + not listed here or none at all. + :param filename: The name of the file in the distribution info. :return: The text if found, otherwise None. """ @abc.abstractmethod - def locate_file(self, path: Union[str, os.PathLike[str]]) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: """ - Given a path to a file in this distribution, return a path + Given a path to a file in this distribution, return a SimplePath to it. """ @classmethod - def from_name(cls, name: str) -> "Distribution": + def from_name(cls, name: str) -> Distribution: """Return the Distribution for the given package name. :param name: The name of the distribution package to search for. @@ -385,16 +409,18 @@ def from_name(cls, name: str) -> "Distribution": raise PackageNotFoundError(name) @classmethod - def discover(cls, **kwargs) -> Iterable["Distribution"]: + def discover( + cls, *, context: Optional[DistributionFinder.Context] = None, **kwargs + ) -> Iterable[Distribution]: """Return an iterable of Distribution objects for all packages. Pass a ``context`` or pass keyword arguments for constructing a context. :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. + :return: Iterable of Distribution objects for packages matching + the context. """ - context = kwargs.pop('context', None) if context and kwargs: raise ValueError("cannot accept context and kwargs") context = context or DistributionFinder.Context(**kwargs) @@ -403,8 +429,8 @@ def discover(cls, **kwargs) -> Iterable["Distribution"]: ) @staticmethod - def at(path: Union[str, os.PathLike[str]]) -> "Distribution": - """Return a Distribution for the indicated metadata path + def at(path: str | os.PathLike[str]) -> Distribution: + """Return a Distribution for the indicated metadata path. :param path: a string or path-like object :return: a concrete Distribution instance for the path @@ -413,7 +439,7 @@ def at(path: Union[str, os.PathLike[str]]) -> "Distribution": @staticmethod def _discover_resolvers(): - """Search the meta_path for resolvers.""" + """Search the meta_path for resolvers (MetadataPathFinders).""" declared = ( getattr(finder, 'find_distributions', None) for finder in sys.meta_path ) @@ -424,8 +450,15 @@ def metadata(self) -> _meta.PackageMetadata: """Return the parsed metadata for this Distribution. The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. + metadata per the + `Core metadata specifications `_. + + Custom providers may provide the METADATA file or override this + property. """ + # deferred for performance (python/cpython#109829) + from . import _adapters + opt_text = ( self.read_text('METADATA') or self.read_text('PKG-INFO') @@ -454,6 +487,12 @@ def version(self) -> str: @property def entry_points(self) -> EntryPoints: + """ + Return EntryPoints for this distribution. + + Custom providers may provide the ``entry_points.txt`` file + or override this property. + """ return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property @@ -466,6 +505,10 @@ def files(self) -> Optional[List[PackagePath]]: (i.e. RECORD for dist-info, or installed-files.txt or SOURCES.txt for egg-info) is missing. Result may be empty if the metadata exists but is empty. + + Custom providers are recommended to provide a "RECORD" file (in + ``read_text``) or override this property to allow for callers to be + able to resolve filenames provided by the package. """ def make_file(name, hash=None, size_str=None): @@ -497,7 +540,7 @@ def skip_missing_files(package_paths): def _read_files_distinfo(self): """ - Read the lines of RECORD + Read the lines of RECORD. """ text = self.read_text('RECORD') return text and text.splitlines() @@ -524,7 +567,7 @@ def _read_files_egginfo_installed(self): paths = ( (subdir / name) .resolve() - .relative_to(self.locate_file('').resolve()) + .relative_to(self.locate_file('').resolve(), walk_up=True) .as_posix() for name in text.splitlines() ) @@ -611,6 +654,9 @@ def _load_json(self, filename): class DistributionFinder(MetaPathFinder): """ A MetaPathFinder capable of discovering installed distributions. + + Custom providers should implement this interface in order to + supply metadata. """ class Context: @@ -623,6 +669,17 @@ class Context: Each DistributionFinder may expect any parameters and should attempt to honor the canonical parameters defined below when appropriate. + + This mechanism gives a custom provider a means to + solicit additional details from the caller beyond + "name" and "path" when searching distributions. + For example, imagine a provider that exposes suites + of packages in either a "public" or "private" ``realm``. + A caller may wish to query only for distributions in + a particular realm and could call + ``distributions(realm="private")`` to signal to the + custom provider to only include distributions from that + realm. """ name = None @@ -658,11 +715,18 @@ def find_distributions(self, context=Context()) -> Iterable[Distribution]: class FastPath: """ - Micro-optimized class for searching a path for - children. + Micro-optimized class for searching a root for children. + + Root is a path on the file system that may contain metadata + directories either as natural directories or within a zip file. >>> FastPath('').children() ['...'] + + FastPath objects are cached and recycled for any given root. + + >>> FastPath('foobar') is FastPath('foobar') + True """ @functools.lru_cache() # type: ignore @@ -704,7 +768,19 @@ def lookup(self, mtime): class Lookup: + """ + A micro-optimized class for searching a (fast) path for metadata. + """ + def __init__(self, path: FastPath): + """ + Calculate all of the children representing metadata. + + From the children in the path, calculate early all of the + children that appear to represent metadata (infos) or legacy + metadata (eggs). + """ + base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") self.infos = FreezableDefaultDict(list) @@ -725,7 +801,10 @@ def __init__(self, path: FastPath): self.infos.freeze() self.eggs.freeze() - def search(self, prepared): + def search(self, prepared: Prepared): + """ + Yield all infos and eggs matching the Prepared query. + """ infos = ( self.infos[prepared.normalized] if prepared @@ -741,13 +820,28 @@ def search(self, prepared): class Prepared: """ - A prepared search for metadata on a possibly-named package. + A prepared search query for metadata on a possibly-named package. + + Pre-calculates the normalization to prevent repeated operations. + + >>> none = Prepared(None) + >>> none.normalized + >>> none.legacy_normalized + >>> bool(none) + False + >>> sample = Prepared('Sample__Pkg-name.foo') + >>> sample.normalized + 'sample_pkg_name_foo' + >>> sample.legacy_normalized + 'sample__pkg_name.foo' + >>> bool(sample) + True """ normalized = None legacy_normalized = None - def __init__(self, name): + def __init__(self, name: Optional[str]): self.name = name if name is None: return @@ -777,7 +871,7 @@ class MetadataPathFinder(DistributionFinder): @classmethod def find_distributions( cls, context=DistributionFinder.Context() - ) -> Iterable["PathDistribution"]: + ) -> Iterable[PathDistribution]: """ Find distributions. @@ -797,6 +891,7 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) + @classmethod def invalidate_caches(cls) -> None: FastPath.__new__.cache_clear() @@ -809,7 +904,7 @@ def __init__(self, path: SimplePath) -> None: """ self._path = path - def read_text(self, filename: Union[str, os.PathLike[str]]) -> Optional[str]: + def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: with suppress( FileNotFoundError, IsADirectoryError, @@ -823,7 +918,7 @@ def read_text(self, filename: Union[str, os.PathLike[str]]) -> Optional[str]: read_text.__doc__ = Distribution.read_text.__doc__ - def locate_file(self, path: Union[str, os.PathLike[str]]) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: return self._path.parent / path @property diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index f670016de7fef2..1927d0f624d82f 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import os from typing import Protocol from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload @@ -6,30 +9,27 @@ class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover + def __len__(self) -> int: ... # pragma: no cover - def __contains__(self, item: str) -> bool: - ... # pragma: no cover + def __contains__(self, item: str) -> bool: ... # pragma: no cover - def __getitem__(self, key: str) -> str: - ... # pragma: no cover + def __getitem__(self, key: str) -> str: ... # pragma: no cover - def __iter__(self) -> Iterator[str]: - ... # pragma: no cover + def __iter__(self) -> Iterator[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: None = None) -> Optional[str]: - ... # pragma: no cover + def get( + self, name: str, failobj: None = None + ) -> Optional[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: _T) -> Union[str, _T]: - ... # pragma: no cover + def get(self, name: str, failobj: _T) -> Union[str, _T]: ... # pragma: no cover # overload per python/importlib_metadata#435 @overload - def get_all(self, name: str, failobj: None = None) -> Optional[List[Any]]: - ... # pragma: no cover + def get_all( + self, name: str, failobj: None = None + ) -> Optional[List[Any]]: ... # pragma: no cover @overload def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]: @@ -44,20 +44,24 @@ def json(self) -> Dict[str, Union[str, List[str]]]: """ -class SimplePath(Protocol[_T]): +class SimplePath(Protocol): """ - A minimal subset of pathlib.Path required by PathDistribution. + A minimal subset of pathlib.Path required by Distribution. """ - def joinpath(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover + def joinpath( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover - def __truediv__(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover + def __truediv__( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover @property - def parent(self) -> _T: - ... # pragma: no cover + def parent(self) -> SimplePath: ... # pragma: no cover + + def read_text(self, encoding=None) -> str: ... # pragma: no cover + + def read_bytes(self) -> bytes: ... # pragma: no cover - def read_text(self) -> str: - ... # pragma: no cover + def exists(self) -> bool: ... # pragma: no cover diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index ae83cd07c4d4fb..ec4441c9116118 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -7,6 +7,16 @@ Anchor, ) +from ._functional import ( + contents, + is_resource, + open_binary, + open_text, + path, + read_binary, + read_text, +) + from .abc import ResourceReader @@ -16,4 +26,11 @@ 'ResourceReader', 'as_file', 'files', + 'contents', + 'is_resource', + 'open_binary', + 'open_text', + 'path', + 'read_binary', + 'read_text', ] diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index a3902535342612..c2c92254370f71 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -12,8 +12,6 @@ from typing import Union, Optional, cast from .abc import ResourceReader, Traversable -from ._adapters import wrap_spec - Package = Union[types.ModuleType, str] Anchor = Package @@ -27,6 +25,8 @@ def package_to_anchor(func): >>> files('a', 'b') Traceback (most recent call last): TypeError: files() takes from 0 to 1 positional arguments but 2 were given + + Remove this compatibility in Python 3.14. """ undefined = object() @@ -93,12 +93,13 @@ def _infer_caller(): """ def is_this_file(frame_info): - return frame_info.filename == __file__ + return frame_info.filename == stack[0].filename def is_wrapper(frame_info): return frame_info.function == 'wrapper' - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) + stack = inspect.stack() + not_this_file = itertools.filterfalse(is_this_file, stack) # also exclude 'wrapper' due to singledispatch in the call stack callers = itertools.filterfalse(is_wrapper, not_this_file) return next(callers).frame @@ -109,6 +110,9 @@ def from_package(package: types.ModuleType): Return a Traversable object for the given package. """ + # deferred for performance (python/cpython#109829) + from ._adapters import wrap_spec + spec = wrap_spec(package) reader = spec.loader.get_resource_reader(spec.name) return reader.files() @@ -179,7 +183,7 @@ def _(path): @contextlib.contextmanager def _temp_path(dir: tempfile.TemporaryDirectory): """ - Wrap tempfile.TemporyDirectory to return a pathlib object. + Wrap tempfile.TemporaryDirectory to return a pathlib object. """ with dir as result: yield pathlib.Path(result) diff --git a/Lib/importlib/resources/_functional.py b/Lib/importlib/resources/_functional.py new file mode 100644 index 00000000000000..f59416f2dd627d --- /dev/null +++ b/Lib/importlib/resources/_functional.py @@ -0,0 +1,81 @@ +"""Simplified function-based API for importlib.resources""" + +import warnings + +from ._common import files, as_file + + +_MISSING = object() + + +def open_binary(anchor, *path_names): + """Open for binary reading the *resource* within *package*.""" + return _get_resource(anchor, path_names).open('rb') + + +def open_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Open for text reading the *resource* within *package*.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.open('r', encoding=encoding, errors=errors) + + +def read_binary(anchor, *path_names): + """Read and return contents of *resource* within *package* as bytes.""" + return _get_resource(anchor, path_names).read_bytes() + + +def read_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Read and return contents of *resource* within *package* as str.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.read_text(encoding=encoding, errors=errors) + + +def path(anchor, *path_names): + """Return the path to the *resource* as an actual file system path.""" + return as_file(_get_resource(anchor, path_names)) + + +def is_resource(anchor, *path_names): + """Return ``True`` if there is a resource named *name* in the package, + + Otherwise returns ``False``. + """ + return _get_resource(anchor, path_names).is_file() + + +def contents(anchor, *path_names): + """Return an iterable over the named resources within the package. + + The iterable returns :class:`str` resources (e.g. files). + The iterable does not recurse into subdirectories. + """ + warnings.warn( + "importlib.resources.contents is deprecated. " + "Use files(anchor).iterdir() instead.", + DeprecationWarning, + stacklevel=1, + ) + return (resource.name for resource in _get_resource(anchor, path_names).iterdir()) + + +def _get_encoding_arg(path_names, encoding): + # For compatibility with versions where *encoding* was a positional + # argument, it needs to be given explicitly when there are multiple + # *path_names*. + # This limitation can be removed in Python 3.15. + if encoding is _MISSING: + if len(path_names) > 1: + raise TypeError( + "'encoding' argument required with multiple path names", + ) + else: + return 'utf-8' + return encoding + + +def _get_resource(anchor, path_names): + if anchor is None: + raise TypeError("anchor must be module or string, got None") + return files(anchor).joinpath(*path_names) diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index c3cdf769cbecb0..ccc5abbeb4e56e 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -1,7 +1,10 @@ import collections +import contextlib import itertools import pathlib import operator +import re +import warnings import zipfile from . import abc @@ -31,8 +34,10 @@ def files(self): class ZipReader(abc.TraversableResources): def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' + self.prefix = loader.prefix.replace('\\', '/') + if loader.is_package(module): + _, _, name = module.rpartition('.') + self.prefix += name + '/' self.archive = loader.archive def open_resource(self, resource): @@ -62,7 +67,7 @@ class MultiplexedPath(abc.Traversable): """ def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + self._paths = list(map(_ensure_traversable, remove_duplicates(paths))) if not self._paths: message = 'MultiplexedPath must contain at least one path' raise FileNotFoundError(message) @@ -130,7 +135,36 @@ class NamespaceReader(abc.TraversableResources): def __init__(self, namespace_path): if 'NamespacePath' not in str(namespace_path): raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) + self.path = MultiplexedPath(*map(self._resolve, namespace_path)) + + @classmethod + def _resolve(cls, path_str) -> abc.Traversable: + r""" + Given an item from a namespace path, resolve it to a Traversable. + + path_str might be a directory on the filesystem or a path to a + zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or + ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``. + """ + (dir,) = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) + return dir + + @classmethod + def _candidate_paths(cls, path_str): + yield pathlib.Path(path_str) + yield from cls._resolve_zip_path(path_str) + + @staticmethod + def _resolve_zip_path(path_str): + for match in reversed(list(re.finditer(r'[\\/]', path_str))): + with contextlib.suppress( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + PermissionError, + ): + inner = path_str[match.end() :].replace('\\', '/') + '/' + yield zipfile.Path(path_str[: match.start()], inner.lstrip('/')) def resource_path(self, resource): """ @@ -142,3 +176,21 @@ def resource_path(self, resource): def files(self): return self.path + + +def _ensure_traversable(path): + """ + Convert deprecated string arguments to traversables (pathlib.Path). + + Remove with Python 3.15. + """ + if not isinstance(path, str): + return path + + warnings.warn( + "String arguments are deprecated. Pass a Traversable instead.", + DeprecationWarning, + stacklevel=3, + ) + + return pathlib.Path(path) diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 7770c922c84fab..96f117fec62c10 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -88,7 +88,7 @@ def is_dir(self): def open(self, mode='r', *args, **kwargs): stream = self.parent.reader.open_binary(self.name) if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) + stream = io.TextIOWrapper(stream, *args, **kwargs) return stream def joinpath(self, name): diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 3ad71d31c2f438..2b564e9b52e0cb 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -5,7 +5,6 @@ from ._bootstrap import spec_from_loader from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER -from ._bootstrap_external import _RAW_MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache @@ -18,7 +17,7 @@ def source_hash(source_bytes): "Return the hash of *source_bytes* as used in hash-based pyc files." - return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + return _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) def resolve_name(name, package): @@ -145,7 +144,7 @@ class _incompatible_extension_module_restrictions: You can get the same effect as this function by implementing the basic interface of multi-phase init (PEP 489) and lying about - support for mulitple interpreters (or per-interpreter GIL). + support for multiple interpreters (or per-interpreter GIL). """ def __init__(self, *, disable_check): @@ -171,36 +170,57 @@ class _LazyModule(types.ModuleType): def __getattribute__(self, attr): """Trigger the load of the module and return the attribute.""" - # All module metadata must be garnered from __spec__ in order to avoid - # using mutated values. - # Stop triggering this method. - self.__class__ = types.ModuleType - # Get the original name to make sure no object substitution occurred - # in sys.modules. - original_name = self.__spec__.name - # Figure out exactly what attributes were mutated between the creation - # of the module and now. - attrs_then = self.__spec__.loader_state['__dict__'] - attrs_now = self.__dict__ - attrs_updated = {} - for key, value in attrs_now.items(): - # Code that set the attribute may have kept a reference to the - # assigned object, making identity more important than equality. - if key not in attrs_then: - attrs_updated[key] = value - elif id(attrs_now[key]) != id(attrs_then[key]): - attrs_updated[key] = value - self.__spec__.loader.exec_module(self) - # If exec_module() was used directly there is no guarantee the module - # object was put into sys.modules. - if original_name in sys.modules: - if id(self) != id(sys.modules[original_name]): - raise ValueError(f"module object for {original_name!r} " - "substituted in sys.modules during a lazy " - "load") - # Update after loading since that's what would happen in an eager - # loading situation. - self.__dict__.update(attrs_updated) + __spec__ = object.__getattribute__(self, '__spec__') + loader_state = __spec__.loader_state + with loader_state['lock']: + # Only the first thread to get the lock should trigger the load + # and reset the module's class. The rest can now getattr(). + if object.__getattribute__(self, '__class__') is _LazyModule: + __class__ = loader_state['__class__'] + + # Reentrant calls from the same thread must be allowed to proceed without + # triggering the load again. + # exec_module() and self-referential imports are the primary ways this can + # happen, but in any case we must return something to avoid deadlock. + if loader_state['is_loading']: + return __class__.__getattribute__(self, attr) + loader_state['is_loading'] = True + + __dict__ = __class__.__getattribute__(self, '__dict__') + + # All module metadata must be gathered from __spec__ in order to avoid + # using mutated values. + # Get the original name to make sure no object substitution occurred + # in sys.modules. + original_name = __spec__.name + # Figure out exactly what attributes were mutated between the creation + # of the module and now. + attrs_then = loader_state['__dict__'] + attrs_now = __dict__ + attrs_updated = {} + for key, value in attrs_now.items(): + # Code that set an attribute may have kept a reference to the + # assigned object, making identity more important than equality. + if key not in attrs_then: + attrs_updated[key] = value + elif id(attrs_now[key]) != id(attrs_then[key]): + attrs_updated[key] = value + __spec__.loader.exec_module(self) + # If exec_module() was used directly there is no guarantee the module + # object was put into sys.modules. + if original_name in sys.modules: + if id(self) != id(sys.modules[original_name]): + raise ValueError(f"module object for {original_name!r} " + "substituted in sys.modules during a lazy " + "load") + # Update after loading since that's what would happen in an eager + # loading situation. + __dict__.update(attrs_updated) + # Finally, stop triggering this method, if the module did not + # already update its own __class__. + if isinstance(self, _LazyModule): + object.__setattr__(self, '__class__', __class__) + return getattr(self, attr) def __delattr__(self, attr): @@ -235,6 +255,9 @@ def create_module(self, spec): def exec_module(self, module): """Make the module load lazily.""" + # Threading is only needed for lazy loading, and importlib.util can + # be pulled in at interpreter startup, so defer until needed. + import threading module.__spec__.loader = self.loader module.__loader__ = self.loader # Don't need to worry about deep-copying as trying to set an attribute @@ -244,5 +267,13 @@ def exec_module(self, module): loader_state = {} loader_state['__dict__'] = module.__dict__.copy() loader_state['__class__'] = module.__class__ + loader_state['lock'] = threading.RLock() + loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', + 'cache_from_source', 'decode_source', 'find_spec', + 'module_from_spec', 'resolve_name', 'source_from_cache', + 'source_hash', 'spec_from_file_location', 'spec_from_loader'] diff --git a/Lib/inspect.py b/Lib/inspect.py index f0b72662a9a0b2..08718d82e91582 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -6,9 +6,9 @@ Here are some of the useful functions provided by this module: - ismodule(), isclass(), ismethod(), isfunction(), isgeneratorfunction(), - isgenerator(), istraceback(), isframe(), iscode(), isbuiltin(), - isroutine() - check object types + ismodule(), isclass(), ismethod(), ispackage(), isfunction(), + isgeneratorfunction(), isgenerator(), istraceback(), isframe(), + iscode(), isbuiltin(), isroutine() - check object types getmembers() - get members of an object that satisfy a given condition getfile(), getsourcefile(), getsource() - find an object's source code @@ -24,8 +24,6 @@ stack(), trace() - get info about frames on the stack or in a traceback signature() - get a Signature object for the callable - - get_annotations() - safely compute an object's annotations """ # This module is in the public domain. No warranties. @@ -58,6 +56,7 @@ "CO_OPTIMIZED", "CO_VARARGS", "CO_VARKEYWORDS", + "CO_HAS_DOCSTRING", "ClassFoundException", "ClosureVars", "EndOfBlock", @@ -130,6 +129,7 @@ "ismethoddescriptor", "ismethodwrapper", "ismodule", + "ispackage", "isroutine", "istraceback", "markcoroutinefunction", @@ -142,6 +142,8 @@ import abc +from annotationlib import Format +from annotationlib import get_annotations # re-exported import ast import dis import collections.abc @@ -160,6 +162,7 @@ from keyword import iskeyword from operator import attrgetter from collections import namedtuple, OrderedDict +from weakref import ref as make_weakref # Create constants for the compiler flags in Include/code.h # We try to get them from dis to avoid duplication @@ -172,121 +175,6 @@ TPFLAGS_IS_ABSTRACT = 1 << 20 -def get_annotations(obj, *, globals=None, locals=None, eval_str=False): - """Compute the annotations dict for an object. - - obj may be a callable, class, or module. - Passing in an object of any other type raises TypeError. - - Returns a dict. get_annotations() returns a new dict every time - it's called; calling it twice on the same object will return two - different but equivalent dicts. - - This function handles several details for you: - - * If eval_str is true, values of type str will - be un-stringized using eval(). This is intended - for use with stringized annotations - ("from __future__ import annotations"). - * If obj doesn't have an annotations dict, returns an - empty dict. (Functions and methods always have an - annotations dict; classes, modules, and other types of - callables may not.) - * Ignores inherited annotations on classes. If a class - doesn't have its own annotations dict, returns an empty dict. - * All accesses to object members and dict values are done - using getattr() and dict.get() for safety. - * Always, always, always returns a freshly-created dict. - - eval_str controls whether or not values of type str are replaced - with the result of calling eval() on those values: - - * If eval_str is true, eval() is called on values of type str. - * If eval_str is false (the default), values of type str are unchanged. - - globals and locals are passed in to eval(); see the documentation - for eval() for more information. If either globals or locals is - None, this function may replace that value with a context-specific - default, contingent on type(obj): - - * If obj is a module, globals defaults to obj.__dict__. - * If obj is a class, globals defaults to - sys.modules[obj.__module__].__dict__ and locals - defaults to the obj class namespace. - * If obj is a callable, globals defaults to obj.__globals__, - although if obj is a wrapped function (using - functools.update_wrapper()) it is first unwrapped. - """ - if isinstance(obj, type): - # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, types.GetSetDescriptorType): - ann = None - else: - ann = None - - obj_globals = None - module_name = getattr(obj, '__module__', None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - obj_globals = getattr(module, '__dict__', None) - obj_locals = dict(vars(obj)) - unwrap = obj - elif isinstance(obj, types.ModuleType): - # module - ann = getattr(obj, '__annotations__', None) - obj_globals = getattr(obj, '__dict__') - obj_locals = None - unwrap = None - elif callable(obj): - # this includes types.Function, types.BuiltinFunctionType, - # types.BuiltinMethodType, functools.partial, functools.singledispatch, - # "class funclike" from Lib/test/test_inspect... on and on it goes. - ann = getattr(obj, '__annotations__', None) - obj_globals = getattr(obj, '__globals__', None) - obj_locals = None - unwrap = obj - else: - raise TypeError(f"{obj!r} is not a module, class, or callable.") - - if ann is None: - return {} - - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") - - if not ann: - return {} - - if not eval_str: - return dict(ann) - - if unwrap is not None: - while True: - if hasattr(unwrap, '__wrapped__'): - unwrap = unwrap.__wrapped__ - continue - if isinstance(unwrap, functools.partial): - unwrap = unwrap.func - continue - break - if hasattr(unwrap, "__globals__"): - obj_globals = unwrap.__globals__ - - if globals is None: - globals = obj_globals - if locals is None: - locals = obj_locals - - return_value = {key: - value if not isinstance(value, str) else eval(value, globals, locals) - for key, value in ann.items() } - return return_value - - # ----------------------------------------------------------- type-checking def ismodule(object): """Return true if the object is a module.""" @@ -300,15 +188,20 @@ def ismethod(object): """Return true if the object is an instance method.""" return isinstance(object, types.MethodType) +def ispackage(object): + """Return true if the object is a package.""" + return ismodule(object) and hasattr(object, "__path__") + def ismethoddescriptor(object): """Return true if the object is a method descriptor. But not if ismethod() or isclass() or isfunction() are true. This is new in Python 2.2, and, for example, is true of int.__add__. - An object passing this test has a __get__ attribute but not a __set__ - attribute, but beyond that the set of attributes varies. __name__ is - usually sensible, and __doc__ often is. + An object passing this test has a __get__ attribute, but not a + __set__ attribute or a __delete__ attribute. Beyond that, the set + of attributes varies; __name__ is usually sensible, and __doc__ + often is. Methods implemented via descriptors that also pass one of the other tests return false from the ismethoddescriptor() test, simply because @@ -318,7 +211,9 @@ def ismethoddescriptor(object): # mutual exclusion return False tp = type(object) - return hasattr(tp, "__get__") and not hasattr(tp, "__set__") + return (hasattr(tp, "__get__") + and not hasattr(tp, "__set__") + and not hasattr(tp, "__delete__")) def isdatadescriptor(object): """Return true if the object is a data descriptor. @@ -374,17 +269,24 @@ def isfunction(object): Function objects provide these attributes: __doc__ documentation string __name__ name with which this function was defined + __qualname__ qualified name of this function + __module__ name of the module the function was defined in or None __code__ code object containing compiled function bytecode __defaults__ tuple of any default values for arguments __globals__ global namespace in which this function was defined __annotations__ dict of parameter annotations - __kwdefaults__ dict of keyword only parameters with defaults""" + __kwdefaults__ dict of keyword only parameters with defaults + __dict__ namespace which is supporting arbitrary function attributes + __closure__ a tuple of cells or None + __type_params__ tuple of type parameters""" return isinstance(object, types.FunctionType) def _has_code_flag(f, flag): """Return true if ``f`` is a function (or a method or functools.partial - wrapper wrapping a function) whose code object has the given ``flag`` + wrapper wrapping a function or a functools.partialmethod wrapping a + function) whose code object has the given ``flag`` set in its flags.""" + f = functools._unwrap_partialmethod(f) while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) @@ -400,13 +302,13 @@ def isgeneratorfunction(obj): return _has_code_flag(obj, CO_GENERATOR) # A marker for markcoroutinefunction and iscoroutinefunction. -_is_coroutine_marker = object() +_is_coroutine_mark = object() def _has_coroutine_mark(f): while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) - return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_marker + return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_mark def markcoroutinefunction(func): """ @@ -414,7 +316,7 @@ def markcoroutinefunction(func): """ if hasattr(func, '__func__'): func = func.__func__ - func._is_coroutine_marker = _is_coroutine_marker + func._is_coroutine_marker = _is_coroutine_mark return func def iscoroutinefunction(obj): @@ -441,17 +343,18 @@ def isgenerator(object): """Return true if the object is a generator. Generator objects provide these attributes: - __iter__ defined to support iteration over container - close raises a new GeneratorExit exception inside the - generator to terminate the iteration gi_code code object gi_frame frame object or possibly None once the generator has been exhausted gi_running set to 1 when generator is executing, 0 otherwise - next return the next item from the container - send resumes the generator and "sends" a value that becomes + gi_yieldfrom object being iterated by yield from or None + + __iter__() defined to support iteration over container + close() raises a new GeneratorExit exception inside the + generator to terminate the iteration + send() resumes the generator and "sends" a value that becomes the result of the current yield-expression - throw used to raise an exception inside the generator""" + throw() used to raise an exception inside the generator""" return isinstance(object, types.GeneratorType) def iscoroutine(object): @@ -486,7 +389,11 @@ def isframe(object): f_lasti index of last attempted instruction in bytecode f_lineno current line number in Python source code f_locals local namespace seen by this frame - f_trace tracing function for this frame, or None""" + f_trace tracing function for this frame, or None + f_trace_lines is a tracing event triggered for each source line? + f_trace_opcodes are per-opcode events being requested? + + clear() used to clear all references to local variables""" return isinstance(object, types.FrameType) def iscode(object): @@ -503,6 +410,7 @@ def iscode(object): co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg | 16=nested | 32=generator | 64=nofree | 128=coroutine | 256=iterable_coroutine | 512=async_generator + | 0x4000000=has_docstring co_freevars tuple of names of free variables co_posonlyargcount number of positional only arguments co_kwonlyargcount number of keyword only arguments (not including ** arg) @@ -511,7 +419,12 @@ def iscode(object): co_names tuple of names other than arguments and function locals co_nlocals number of local variables co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_varnames tuple of names of arguments and local variables + co_qualname fully qualified function name + + co_lines() returns an iterator that yields successive bytecode ranges + co_positions() returns an iterator of source code positions for each bytecode instruction + replace() returns a copy of the code object with a new values""" return isinstance(object, types.CodeType) def isbuiltin(object): @@ -760,18 +673,14 @@ def unwrap(func, *, stop=None): :exc:`ValueError` is raised if a cycle is encountered. """ - if stop is None: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') - else: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting # Memoise by id to tolerate non-hashable objects, but store objects to # ensure they aren't destroyed, which would allow their IDs to be reused. memo = {id(f): f} recursion_limit = sys.getrecursionlimit() - while _is_wrapper(func): + while not isinstance(func, type) and hasattr(func, '__wrapped__'): + if stop is not None and stop(func): + break func = func.__wrapped__ id_func = id(func) if (id_func in memo) or (len(memo) >= recursion_limit): @@ -832,9 +741,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): @@ -957,6 +865,10 @@ def getsourcefile(object): elif any(filename.endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES): return None + elif filename.endswith(".fwork"): + # Apple mobile framework markers are another type of non-source file + return None + # return a filename found in the linecache even if it doesn't exist on disk if filename in linecache.cache: return filename @@ -987,6 +899,7 @@ def getmodule(object, _filename=None): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) + # Try the filename to modulename cache if _filename is not None and _filename in modulesbyfile: return sys.modules.get(modulesbyfile[_filename]) @@ -1032,79 +945,6 @@ class ClassFoundException(Exception): pass -class _ClassFinder(ast.NodeVisitor): - - def __init__(self, cls, tree, lines, qualname): - self.stack = [] - self.cls = cls - self.tree = tree - self.lines = lines - self.qualname = qualname - self.lineno_found = [] - - def visit_FunctionDef(self, node): - self.stack.append(node.name) - self.stack.append('') - self.generic_visit(node) - self.stack.pop() - self.stack.pop() - - visit_AsyncFunctionDef = visit_FunctionDef - - def visit_ClassDef(self, node): - self.stack.append(node.name) - if self.qualname == '.'.join(self.stack): - # Return the decorator for the class if present - if node.decorator_list: - line_number = node.decorator_list[0].lineno - else: - line_number = node.lineno - - # decrement by one since lines starts with indexing by zero - self.lineno_found.append((line_number - 1, node.end_lineno)) - self.generic_visit(node) - self.stack.pop() - - def get_lineno(self): - self.visit(self.tree) - lineno_found_number = len(self.lineno_found) - if lineno_found_number == 0: - raise OSError('could not find class definition') - elif lineno_found_number == 1: - return self.lineno_found[0][0] - else: - # We have multiple candidates for the class definition. - # Now we have to guess. - - # First, let's see if there are any method definitions - for member in self.cls.__dict__.values(): - if (isinstance(member, types.FunctionType) and - member.__module__ == self.cls.__module__): - for lineno, end_lineno in self.lineno_found: - if lineno <= member.__code__.co_firstlineno <= end_lineno: - return lineno - - class_strings = [(''.join(self.lines[lineno: end_lineno]), lineno) - for lineno, end_lineno in self.lineno_found] - - # Maybe the class has a docstring and it's unique? - if self.cls.__doc__: - ret = None - for candidate, lineno in class_strings: - if self.cls.__doc__.strip() in candidate: - if ret is None: - ret = lineno - else: - break - else: - if ret is not None: - return ret - - # We are out of ideas, just return the last one found, which is - # slightly better than previous ones - return self.lineno_found[-1][0] - - def findsource(object): """Return the entire source file and starting line number for an object. @@ -1122,7 +962,7 @@ def findsource(object): # Allow filenames in form of "" to pass through. # `doctest` monkeypatches `linecache` module to enable # inspection, so let `linecache.getlines` to be called. - if not (file.startswith('<') and file.endswith('>')): + if (not (file.startswith('<') and file.endswith('>'))) or file.endswith('.fwork'): raise OSError('source code not available') module = getmodule(object, file) @@ -1137,11 +977,13 @@ def findsource(object): return lines, 0 if isclass(object): - qualname = object.__qualname__ - source = ''.join(lines) - tree = ast.parse(source) - class_finder = _ClassFinder(object, tree, lines, qualname) - return lines, class_finder.get_lineno() + try: + lnum = vars(object)['__firstlineno__'] - 1 + except (TypeError, KeyError): + raise OSError('source code not available') + if lnum >= len(lines): + raise OSError('lineno is out of bounds') + return lines, lnum if ismethod(object): object = object.__func__ @@ -1155,15 +997,8 @@ def findsource(object): if not hasattr(object, 'co_firstlineno'): raise OSError('could not find function definition') lnum = object.co_firstlineno - 1 - pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(? 0: - try: - line = lines[lnum] - except IndexError: - raise OSError('lineno is out of bounds') - if pat.match(line): - break - lnum = lnum - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') return lines, lnum raise OSError('could not find code object') @@ -1492,7 +1327,9 @@ def getargvalues(frame): args, varargs, varkw = getargs(frame.f_code) return ArgInfo(args, varargs, varkw, frame.f_locals) -def formatannotation(annotation, base_module=None): +def formatannotation(annotation, base_module=None, *, quote_annotation_strings=True): + if not quote_annotation_strings and isinstance(annotation, str): + return annotation if getattr(annotation, '__module__', None) == 'typing': def repl(match): text = match.group() @@ -1837,9 +1674,16 @@ def _check_class(klass, attr): return entry.__dict__[attr] return _sentinel + @functools.lru_cache() -def _shadowed_dict_from_mro_tuple(mro): - for entry in mro: +def _shadowed_dict_from_weakref_mro_tuple(*weakref_mro): + for weakref_entry in weakref_mro: + # Normally we'd have to check whether the result of weakref_entry() + # is None here, in case the object the weakref is pointing to has died. + # In this specific case, however, we know that the only caller of this + # function is `_shadowed_dict()`, and that therefore this weakref is + # guaranteed to point to an object that is still alive. + entry = weakref_entry() dunder_dict = _get_dunder_dict_of_class(entry) if '__dict__' in dunder_dict: class_dict = dunder_dict['__dict__'] @@ -1849,8 +1693,19 @@ def _shadowed_dict_from_mro_tuple(mro): return class_dict return _sentinel + def _shadowed_dict(klass): - return _shadowed_dict_from_mro_tuple(_static_getmro(klass)) + # gh-118013: the inner function here is decorated with lru_cache for + # performance reasons, *but* make sure not to pass strong references + # to the items in the mro. Doing so can lead to unexpected memory + # consumption in cases where classes are dynamically created and + # destroyed, and the dynamically created classes happen to be the only + # objects that hold strong references to other objects that take up a + # significant amount of memory. + return _shadowed_dict_from_weakref_mro_tuple( + *[make_weakref(entry) for entry in _static_getmro(klass)] + ) + def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the @@ -2042,15 +1897,17 @@ def _signature_get_user_defined_method(cls, method_name): named ``method_name`` and returns it only if it is a pure python function. """ - try: - meth = getattr(cls, method_name) - except AttributeError: - return + if method_name == '__new__': + meth = getattr(cls, method_name, None) else: - if not isinstance(meth, _NonUserDefinedCallables): - # Once '__signature__' will be added to 'C'-level - # callables, this check won't be necessary - return meth + meth = getattr_static(cls, method_name, None) + if meth is None or isinstance(meth, _NonUserDefinedCallables): + # Once '__signature__' will be added to 'C'-level + # callables, this check won't be necessary + return None + if method_name != '__new__': + meth = _descriptor_get(meth, cls) + return meth def _signature_get_partial(wrapped_sig, partial, extra_args=()): @@ -2085,7 +1942,12 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): if param.kind is _POSITIONAL_ONLY: # If positional-only parameter is bound by partial, # it effectively disappears from the signature - new_params.pop(param_name) + # However, if it is a Placeholder it is not removed + # And also looses default value + if arg_value is functools.Placeholder: + new_params[param_name] = param.replace(default=_empty) + else: + new_params.pop(param_name) continue if param.kind is _POSITIONAL_OR_KEYWORD: @@ -2107,7 +1969,17 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): new_params[param_name] = param.replace(default=arg_value) else: # was passed as a positional argument - new_params.pop(param.name) + # Do not pop if it is a Placeholder + # also change kind to positional only + # and remove default + if arg_value is functools.Placeholder: + new_param = param.replace( + kind=_POSITIONAL_ONLY, + default=_empty + ) + new_params[param_name] = new_param + else: + new_params.pop(param_name) continue if param.kind is _KEYWORD_ONLY: @@ -2163,8 +2035,10 @@ def _signature_is_builtin(obj): ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) + # also be True for regular python classes. + # Can't use the `in` operator here, as it would + # invoke the custom __eq__ method. + obj is type or obj is object) def _signature_is_functionlike(obj): @@ -2267,7 +2141,12 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): module = None module_dict = {} + module_name = getattr(obj, '__module__', None) + if not module_name: + objclass = getattr(obj, '__objclass__', None) + module_name = getattr(objclass, '__module__', None) + if module_name: module = sys.modules.get(module_name, None) if module: @@ -2401,7 +2280,8 @@ def _signature_from_builtin(cls, func, skip_bound_arg=True): def _signature_from_function(cls, func, skip_bound_arg=True, - globals=None, locals=None, eval_str=False): + globals=None, locals=None, eval_str=False, + *, annotation_format=Format.VALUE): """Private helper: constructs Signature for the given python function.""" is_duck_function = False @@ -2427,7 +2307,8 @@ def _signature_from_function(cls, func, skip_bound_arg=True, positional = arg_names[:pos_count] keyword_only_count = func_code.co_kwonlyargcount keyword_only = arg_names[pos_count:pos_count + keyword_only_count] - annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str) + annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str, + format=annotation_format) defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ @@ -2495,13 +2376,23 @@ def _signature_from_function(cls, func, skip_bound_arg=True, __validate_parameters__=is_duck_function) +def _descriptor_get(descriptor, obj): + if isclass(descriptor): + return descriptor + get = getattr(type(descriptor), '__get__', _sentinel) + if get is _sentinel: + return descriptor + return get(descriptor, obj, type(obj)) + + def _signature_from_callable(obj, *, follow_wrapper_chains=True, skip_bound_arg=True, globals=None, locals=None, eval_str=False, - sigcls): + sigcls, + annotation_format=Format.VALUE): """Private helper function to get signature for arbitrary callable objects. @@ -2513,7 +2404,8 @@ def _signature_from_callable(obj, *, globals=globals, locals=locals, sigcls=sigcls, - eval_str=eval_str) + eval_str=eval_str, + annotation_format=annotation_format) if not callable(obj): raise TypeError('{!r} is not a callable object'.format(obj)) @@ -2546,22 +2438,14 @@ def _signature_from_callable(obj, *, pass else: if sig is not None: - # since __text_signature__ is not writable on classes, __signature__ - # may contain text (or be a callable that returns text); - # if so, convert it - o_sig = sig - if not isinstance(sig, (Signature, str)) and callable(sig): - sig = sig() - if isinstance(sig, str): - sig = _signature_fromstr(sigcls, obj, sig) if not isinstance(sig, Signature): raise TypeError( 'unexpected object {!r} in __signature__ ' - 'attribute'.format(o_sig)) + 'attribute'.format(sig)) return sig try: - partialmethod = obj._partialmethod + partialmethod = obj.__partialmethod__ except AttributeError: pass else: @@ -2585,25 +2469,30 @@ def _signature_from_callable(obj, *, sig_params = tuple(sig.parameters.values()) assert (not sig_params or first_wrapped_param is not sig_params[0]) + # If there were placeholders set, + # first param is transformed to positional only + if partialmethod.args.count(functools.Placeholder): + first_wrapped_param = first_wrapped_param.replace( + kind=Parameter.POSITIONAL_ONLY) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) + if isinstance(obj, functools.partial): + wrapped_sig = _get_signature_of(obj.func) + return _signature_get_partial(wrapped_sig, obj) + if isfunction(obj) or _signature_is_functionlike(obj): # If it's a pure Python function, or an object that is duck type # of a Python function (Cython functions, for instance), then: return _signature_from_function(sigcls, obj, skip_bound_arg=skip_bound_arg, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) if _signature_is_builtin(obj): return _signature_from_builtin(sigcls, obj, skip_bound_arg=skip_bound_arg) - if isinstance(obj, functools.partial): - wrapped_sig = _get_signature_of(obj.func) - return _signature_get_partial(wrapped_sig, obj) - - sig = None if isinstance(obj, type): # obj is a class or a metaclass @@ -2611,88 +2500,72 @@ def _signature_from_callable(obj, *, # in its metaclass call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: - sig = _get_signature_of(call) - else: - factory_method = None - new = _signature_get_user_defined_method(obj, '__new__') - init = _signature_get_user_defined_method(obj, '__init__') - - # Go through the MRO and see if any class has user-defined - # pure Python __new__ or __init__ method - for base in obj.__mro__: - # Now we check if the 'obj' class has an own '__new__' method - if new is not None and '__new__' in base.__dict__: - factory_method = new - break - # or an own '__init__' method - elif init is not None and '__init__' in base.__dict__: - factory_method = init - break + return _get_signature_of(call) - if factory_method is not None: - sig = _get_signature_of(factory_method) - - if sig is None: - # At this point we know, that `obj` is a class, with no user- - # defined '__init__', '__new__', or class-level '__call__' - - for base in obj.__mro__[:-1]: - # Since '__text_signature__' is implemented as a - # descriptor that extracts text signature from the - # class docstring, if 'obj' is derived from a builtin - # class, its own '__text_signature__' may be 'None'. - # Therefore, we go through the MRO (except the last - # class in there, which is 'object') to find the first - # class with non-empty text signature. - try: - text_sig = base.__text_signature__ - except AttributeError: - pass - else: - if text_sig: - # If 'base' class has a __text_signature__ attribute: - # return a signature based on it - return _signature_fromstr(sigcls, base, text_sig) - - # No '__text_signature__' was found for the 'obj' class. - # Last option is to check if its '__init__' is - # object.__init__ or type.__init__. - if type not in obj.__mro__: - # We have a class (not metaclass), but no user-defined - # __init__ or __new__ for it - if (obj.__init__ is object.__init__ and - obj.__new__ is object.__new__): - # Return a signature of 'object' builtin. - return sigcls.from_callable(object) - else: - raise ValueError( - 'no signature found for builtin type {!r}'.format(obj)) + new = _signature_get_user_defined_method(obj, '__new__') + init = _signature_get_user_defined_method(obj, '__init__') - elif not isinstance(obj, _NonUserDefinedCallables): + # Go through the MRO and see if any class has user-defined + # pure Python __new__ or __init__ method + for base in obj.__mro__: + # Now we check if the 'obj' class has an own '__new__' method + if new is not None and '__new__' in base.__dict__: + sig = _get_signature_of(new) + if skip_bound_arg: + sig = _signature_bound_method(sig) + return sig + # or an own '__init__' method + elif init is not None and '__init__' in base.__dict__: + return _get_signature_of(init) + + # At this point we know, that `obj` is a class, with no user- + # defined '__init__', '__new__', or class-level '__call__' + + for base in obj.__mro__[:-1]: + # Since '__text_signature__' is implemented as a + # descriptor that extracts text signature from the + # class docstring, if 'obj' is derived from a builtin + # class, its own '__text_signature__' may be 'None'. + # Therefore, we go through the MRO (except the last + # class in there, which is 'object') to find the first + # class with non-empty text signature. + try: + text_sig = base.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + # If 'base' class has a __text_signature__ attribute: + # return a signature based on it + return _signature_fromstr(sigcls, base, text_sig) + + # No '__text_signature__' was found for the 'obj' class. + # Last option is to check if its '__init__' is + # object.__init__ or type.__init__. + if type not in obj.__mro__: + # We have a class (not metaclass), but no user-defined + # __init__ or __new__ for it + if (obj.__init__ is object.__init__ and + obj.__new__ is object.__new__): + # Return a signature of 'object' builtin. + return sigcls.from_callable(object) + else: + raise ValueError( + 'no signature found for builtin type {!r}'.format(obj)) + + else: # An object with __call__ - # We also check that the 'obj' is not an instance of - # types.WrapperDescriptorType or types.MethodWrapperType to avoid - # infinite recursion (and even potential segfault) - call = _signature_get_user_defined_method(type(obj), '__call__') + call = getattr_static(type(obj), '__call__', None) if call is not None: try: - sig = _get_signature_of(call) - except ValueError as ex: - msg = 'no signature found for {!r}'.format(obj) - raise ValueError(msg) from ex - - if sig is not None: - # For classes and objects we skip the first parameter of their - # __call__, __new__, or __init__ methods - if skip_bound_arg: - return _signature_bound_method(sig) - else: - return sig - - if isinstance(obj, types.BuiltinFunctionType): - # Raise a nicer error message for builtins - msg = 'no signature found for builtin function {!r}'.format(obj) - raise ValueError(msg) + text_sig = obj.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + return _signature_fromstr(sigcls, obj, text_sig) + call = _descriptor_get(call, obj) + return _get_signature_of(call) raise ValueError('callable {!r} is not supported by signature'.format(obj)) @@ -2849,13 +2722,17 @@ def replace(self, *, name=_void, kind=_void, return type(self)(name, kind, default=default, annotation=annotation) def __str__(self): + return self._format() + + def _format(self, *, quote_annotation_strings=True): kind = self.kind formatted = self._name # Add annotation and default value if self._annotation is not _empty: - formatted = '{}: {}'.format(formatted, - formatannotation(self._annotation)) + annotation = formatannotation(self._annotation, + quote_annotation_strings=quote_annotation_strings) + formatted = '{}: {}'.format(formatted, annotation) if self._default is not _empty: if self._annotation is not _empty: @@ -3103,11 +2980,13 @@ def __init__(self, parameters=None, *, return_annotation=_empty, @classmethod def from_callable(cls, obj, *, - follow_wrapped=True, globals=None, locals=None, eval_str=False): + follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Constructs Signature for the given callable object.""" return _signature_from_callable(obj, sigcls=cls, follow_wrapper_chains=follow_wrapped, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) @property def parameters(self): @@ -3164,6 +3043,8 @@ def _bind(self, args, kwargs, *, partial=False): parameters_ex = () arg_vals = iter(args) + pos_only_param_in_kwargs = [] + while True: # Let's iterate through the positional arguments and corresponding # parameters @@ -3184,10 +3065,10 @@ def _bind(self, args, kwargs, *, partial=False): break elif param.name in kwargs: if param.kind == _POSITIONAL_ONLY: - msg = '{arg!r} parameter is positional only, ' \ - 'but was passed as a keyword' - msg = msg.format(arg=param.name) - raise TypeError(msg) from None + # Raise a TypeError once we are sure there is no + # **kwargs param later. + pos_only_param_in_kwargs.append(param) + continue parameters_ex = (param,) break elif (param.kind == _VAR_KEYWORD or @@ -3269,20 +3150,22 @@ def _bind(self, args, kwargs, *, partial=False): format(arg=param_name)) from None else: - if param.kind == _POSITIONAL_ONLY: - # This should never happen in case of a properly built - # Signature object (but let's have this check here - # to ensure correct behaviour just in case) - raise TypeError('{arg!r} parameter is positional only, ' - 'but was passed as a keyword'. \ - format(arg=param.name)) - arguments[param_name] = arg_val if kwargs: if kwargs_param is not None: # Process our '**kwargs'-like parameter arguments[kwargs_param.name] = kwargs + elif pos_only_param_in_kwargs: + raise TypeError( + 'got some positional-only arguments passed as ' + 'keyword arguments: {arg!r}'.format( + arg=', '.join( + param.name + for param in pos_only_param_in_kwargs + ), + ), + ) else: raise TypeError( 'got an unexpected keyword argument {arg!r}'.format( @@ -3318,19 +3201,24 @@ def __repr__(self): def __str__(self): return self.format() - def format(self, *, max_width=None): + def format(self, *, max_width=None, quote_annotation_strings=True): """Create a string representation of the Signature object. If *max_width* integer is passed, signature will try to fit into the *max_width*. If signature is longer than *max_width*, all parameters will be on separate lines. + + If *quote_annotation_strings* is False, annotations + in the signature are displayed without opening and closing quotation + marks. This is useful when the signature was created with the + STRING format or when ``from __future__ import annotations`` was used. """ result = [] render_pos_only_separator = False render_kw_only_separator = True for param in self.parameters.values(): - formatted = str(param) + formatted = param._format(quote_annotation_strings=quote_annotation_strings) kind = param.kind @@ -3367,16 +3255,19 @@ def format(self, *, max_width=None): rendered = '(\n {}\n)'.format(',\n '.join(result)) if self.return_annotation is not _empty: - anno = formatannotation(self.return_annotation) + anno = formatannotation(self.return_annotation, + quote_annotation_strings=quote_annotation_strings) rendered += ' -> {}'.format(anno) return rendered -def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False): +def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Get a signature object for the passed callable.""" return Signature.from_callable(obj, follow_wrapped=follow_wrapped, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) class BufferFlags(enum.IntFlag): diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index e398cc138308d9..703fa289dda1fb 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -239,7 +239,7 @@ def summarize_address_range(first, last): else: raise ValueError('unknown IP version') - ip_bits = first._max_prefixlen + ip_bits = first.max_prefixlen first_int = first._ip last_int = last._ip while first_int <= last_int: @@ -310,7 +310,7 @@ def collapse_addresses(addresses): [IPv4Network('192.0.2.0/24')] Args: - addresses: An iterator of IPv4Network or IPv6Network objects. + addresses: An iterable of IPv4Network or IPv6Network objects. Returns: An iterator of the collapsed IPv(4|6)Network objects. @@ -326,12 +326,12 @@ def collapse_addresses(addresses): # split IP addresses and networks for ip in addresses: if isinstance(ip, _BaseAddress): - if ips and ips[-1]._version != ip._version: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) ips.append(ip) - elif ip._prefixlen == ip._max_prefixlen: - if ips and ips[-1]._version != ip._version: + elif ip._prefixlen == ip.max_prefixlen: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) try: @@ -339,7 +339,7 @@ def collapse_addresses(addresses): except AttributeError: ips.append(ip.network_address) else: - if nets and nets[-1]._version != ip._version: + if nets and nets[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, nets[-1])) nets.append(ip) @@ -407,26 +407,21 @@ def reverse_pointer(self): """ return self._reverse_pointer() - @property - def version(self): - msg = '%200s has no version specified' % (type(self),) - raise NotImplementedError(msg) - def _check_int_address(self, address): if address < 0: msg = "%d (< 0) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._version)) + raise AddressValueError(msg % (address, self.version)) if address > self._ALL_ONES: msg = "%d (>= 2**%d) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._max_prefixlen, - self._version)) + raise AddressValueError(msg % (address, self.max_prefixlen, + self.version)) def _check_packed_address(self, address, expected_len): address_len = len(address) if address_len != expected_len: msg = "%r (len %d != %d) is not permitted as an IPv%d address" raise AddressValueError(msg % (address, address_len, - expected_len, self._version)) + expected_len, self.version)) @classmethod def _ip_int_from_prefix(cls, prefixlen): @@ -455,12 +450,12 @@ def _prefix_from_ip_int(cls, ip_int): ValueError: If the input intermingles zeroes & ones """ trailing_zeroes = _count_righthand_zero_bits(ip_int, - cls._max_prefixlen) - prefixlen = cls._max_prefixlen - trailing_zeroes + cls.max_prefixlen) + prefixlen = cls.max_prefixlen - trailing_zeroes leading_ones = ip_int >> trailing_zeroes all_ones = (1 << prefixlen) - 1 if leading_ones != all_ones: - byteslen = cls._max_prefixlen // 8 + byteslen = cls.max_prefixlen // 8 details = ip_int.to_bytes(byteslen, 'big') msg = 'Netmask pattern %r mixes zeroes & ones' raise ValueError(msg % details) @@ -492,7 +487,7 @@ def _prefix_from_prefix_string(cls, prefixlen_str): prefixlen = int(prefixlen_str) except ValueError: cls._report_invalid_netmask(prefixlen_str) - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen_str) return prefixlen @@ -542,7 +537,7 @@ def _split_addr_prefix(cls, address): """ # a packed address or integer if isinstance(address, (bytes, int)): - return address, cls._max_prefixlen + return address, cls.max_prefixlen if not isinstance(address, tuple): # Assume input argument to be string or any object representation @@ -552,7 +547,7 @@ def _split_addr_prefix(cls, address): # Constructing from a tuple (addr, [mask]) if len(address) > 1: return address - return address[0], cls._max_prefixlen + return address[0], cls.max_prefixlen def __reduce__(self): return self.__class__, (str(self),) @@ -577,14 +572,14 @@ def __int__(self): def __eq__(self, other): try: return (self._ip == other._ip - and self._version == other._version) + and self.version == other.version) except AttributeError: return NotImplemented def __lt__(self, other): if not isinstance(other, _BaseAddress): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self._ip != other._ip: @@ -613,7 +608,7 @@ def __hash__(self): return hash(hex(int(self._ip))) def _get_address_key(self): - return (self._version, self) + return (self.version, self) def __reduce__(self): return self.__class__, (self._ip,) @@ -649,15 +644,15 @@ def __format__(self, fmt): # Set some defaults if fmt_base == 'n': - if self._version == 4: + if self.version == 4: fmt_base = 'b' # Binary is default for ipv4 else: fmt_base = 'x' # Hex is default for ipv6 if fmt_base == 'b': - padlen = self._max_prefixlen + padlen = self.max_prefixlen else: - padlen = self._max_prefixlen // 4 + padlen = self.max_prefixlen // 4 if grouping: padlen += padlen // 4 - 1 @@ -716,7 +711,7 @@ def __getitem__(self, n): def __lt__(self, other): if not isinstance(other, _BaseNetwork): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self.network_address != other.network_address: @@ -727,7 +722,7 @@ def __lt__(self, other): def __eq__(self, other): try: - return (self._version == other._version and + return (self.version == other.version and self.network_address == other.network_address and int(self.netmask) == int(other.netmask)) except AttributeError: @@ -738,7 +733,7 @@ def __hash__(self): def __contains__(self, other): # always false if one is v4 and the other is v6. - if self._version != other._version: + if self.version != other.version: return False # dealing with another network. if isinstance(other, _BaseNetwork): @@ -829,7 +824,7 @@ def address_exclude(self, other): ValueError: If other is not completely contained by self. """ - if not self._version == other._version: + if not self.version == other.version: raise TypeError("%s and %s are not of the same version" % ( self, other)) @@ -901,10 +896,10 @@ def compare_networks(self, other): """ # does this need to raise a ValueError? - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same type' % ( self, other)) - # self._version == other._version below here: + # self.version == other.version below here: if self.network_address < other.network_address: return -1 if self.network_address > other.network_address: @@ -924,7 +919,7 @@ def _get_networks_key(self): and list.sort(). """ - return (self._version, self.network_address, self.netmask) + return (self.version, self.network_address, self.netmask) def subnets(self, prefixlen_diff=1, new_prefix=None): """The subnets which join to make the current subnet. @@ -952,7 +947,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): number means a larger network) """ - if self._prefixlen == self._max_prefixlen: + if self._prefixlen == self.max_prefixlen: yield self return @@ -967,7 +962,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): raise ValueError('prefix length diff must be > 0') new_prefixlen = self._prefixlen + prefixlen_diff - if new_prefixlen > self._max_prefixlen: + if new_prefixlen > self.max_prefixlen: raise ValueError( 'prefix length diff %d is invalid for netblock %s' % ( new_prefixlen, self)) @@ -1036,7 +1031,7 @@ def is_multicast(self): def _is_subnet_of(a, b): try: # Always false if one is v4 and the other is v6. - if a._version != b._version: + if a.version != b.version: raise TypeError(f"{a} and {b} are not of the same version") return (b.network_address <= a.network_address and b.broadcast_address >= a.broadcast_address) @@ -1086,7 +1081,11 @@ def is_private(self): """ return any(self.network_address in priv_network and self.broadcast_address in priv_network - for priv_network in self._constants._private_networks) + for priv_network in self._constants._private_networks) and all( + self.network_address not in network and + self.broadcast_address not in network + for network in self._constants._private_networks_exceptions + ) @property def is_global(self): @@ -1142,11 +1141,11 @@ class _BaseV4: """ __slots__ = () - _version = 4 + version = 4 # Equivalent to 255.255.255.255 or 32 bits of 1's. _ALL_ONES = (2**IPV4LENGTH) - 1 - _max_prefixlen = IPV4LENGTH + max_prefixlen = IPV4LENGTH # There are only a handful of valid v4 netmasks, so we cache them all # when constructed (see _make_netmask()). _netmask_cache = {} @@ -1166,7 +1165,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: try: @@ -1264,15 +1263,6 @@ def _reverse_pointer(self): reverse_octets = str(self).split('.')[::-1] return '.'.join(reverse_octets) + '.in-addr.arpa' - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv4Address(_BaseV4, _BaseAddress): """Represent and manipulate single IPv4 Addresses.""" @@ -1333,18 +1323,41 @@ def is_reserved(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ - return any(self in net for net in self._constants._private_networks) + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property @functools.lru_cache() def is_global(self): + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. + """ return self not in self._constants._public_network and not self.is_private @property @@ -1529,9 +1542,9 @@ def __init__(self, address, strict=True): self.network_address = IPv4Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == (self._max_prefixlen): + elif self._prefixlen == (self.max_prefixlen): self.hosts = lambda: [IPv4Address(addr)] @property @@ -1558,13 +1571,15 @@ class _IPv4Constants: _public_network = IPv4Network('100.64.0.0/10') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml _private_networks = [ IPv4Network('0.0.0.0/8'), IPv4Network('10.0.0.0/8'), IPv4Network('127.0.0.0/8'), IPv4Network('169.254.0.0/16'), IPv4Network('172.16.0.0/12'), - IPv4Network('192.0.0.0/29'), + IPv4Network('192.0.0.0/24'), IPv4Network('192.0.0.170/31'), IPv4Network('192.0.2.0/24'), IPv4Network('192.168.0.0/16'), @@ -1575,6 +1590,11 @@ class _IPv4Constants: IPv4Network('255.255.255.255/32'), ] + _private_networks_exceptions = [ + IPv4Network('192.0.0.9/32'), + IPv4Network('192.0.0.10/32'), + ] + _reserved_network = IPv4Network('240.0.0.0/4') _unspecified_address = IPv4Address('0.0.0.0') @@ -1594,11 +1614,11 @@ class _BaseV6: """ __slots__ = () - _version = 6 + version = 6 _ALL_ONES = (2**IPV6LENGTH) - 1 _HEXTET_COUNT = 8 _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') - _max_prefixlen = IPV6LENGTH + max_prefixlen = IPV6LENGTH # There are only a bunch of valid v6 netmasks, so we cache them all # when constructed (see _make_netmask()). @@ -1616,7 +1636,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: prefixlen = cls._prefix_from_prefix_string(arg) @@ -1878,15 +1898,6 @@ def _split_scope_id(ip_str): raise AddressValueError('Invalid IPv6 address: "%r"' % ip_str) return addr, scope_id - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv6Address(_BaseV6, _BaseAddress): """Represent and manipulate single IPv6 Addresses.""" @@ -1936,12 +1947,21 @@ def __init__(self, address): def _explode_shorthand_ip_string(self): ipv4_mapped = self.ipv4_mapped if ipv4_mapped is None: - long_form = super()._explode_shorthand_ip_string() - else: - prefix_len = 30 - raw_exploded_str = super()._explode_shorthand_ip_string() - long_form = "%s%s" % (raw_exploded_str[:prefix_len], str(ipv4_mapped)) - return long_form + return super()._explode_shorthand_ip_string() + prefix_len = 30 + raw_exploded_str = super()._explode_shorthand_ip_string() + return f"{raw_exploded_str[:prefix_len]}{ipv4_mapped!s}" + + def _reverse_pointer(self): + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + return super()._reverse_pointer() + prefix_len = 30 + raw_exploded_str = super()._explode_shorthand_ip_string()[:prefix_len] + # ipv4 encoded using hexadecimal nibbles instead of decimals + ipv4_int = ipv4_mapped._ip + reverse_chars = f"{raw_exploded_str}{ipv4_int:008x}"[::-1].replace(':', '') + return '.'.join(reverse_chars) + '.ip6.arpa' def _ipv4_mapped_ipv6_to_str(self): """Return convenient text representation of IPv4-mapped IPv6 address @@ -2009,6 +2029,9 @@ def is_multicast(self): See RFC 2373 2.7 for details. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_multicast return self in self._constants._multicast_network @property @@ -2020,6 +2043,9 @@ def is_reserved(self): reserved IPv6 Network ranges. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_reserved return any(self in x for x in self._constants._reserved_networks) @property @@ -2030,6 +2056,9 @@ def is_link_local(self): A boolean, True if the address is reserved per RFC 4291. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_link_local return self in self._constants._linklocal_network @property @@ -2049,28 +2078,46 @@ def is_site_local(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv6-special-registry, or is ipv4_mapped and is - reserved in the iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ ipv4_mapped = self.ipv4_mapped if ipv4_mapped is not None: return ipv4_mapped.is_private - return any(self in net for net in self._constants._private_networks) + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property def is_global(self): - """Test if this address is allocated for public networks. + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: - Returns: - A boolean, true if the address is not reserved per - iana-ipv6-special-registry. + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_global return not self.is_private @property @@ -2082,6 +2129,9 @@ def is_unspecified(self): RFC 2373 2.5.2. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_unspecified return self._ip == 0 @property @@ -2093,6 +2143,9 @@ def is_loopback(self): RFC 2373 2.5.3. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_loopback return self._ip == 1 @property @@ -2209,7 +2262,7 @@ def is_unspecified(self): @property def is_loopback(self): - return self._ip == 1 and self.network.is_loopback + return super().is_loopback and self.network.is_loopback class IPv6Network(_BaseV6, _BaseNetwork): @@ -2271,9 +2324,9 @@ def __init__(self, address, strict=True): self.network_address = IPv6Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == self._max_prefixlen: + elif self._prefixlen == self.max_prefixlen: self.hosts = lambda: [IPv6Address(addr)] def hosts(self): @@ -2310,19 +2363,33 @@ class _IPv6Constants: _multicast_network = IPv6Network('ff00::/8') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml _private_networks = [ IPv6Network('::1/128'), IPv6Network('::/128'), IPv6Network('::ffff:0:0/96'), + IPv6Network('64:ff9b:1::/48'), IPv6Network('100::/64'), IPv6Network('2001::/23'), - IPv6Network('2001:2::/48'), IPv6Network('2001:db8::/32'), - IPv6Network('2001:10::/28'), + # IANA says N/A, let's consider it not globally reachable to be safe + IPv6Network('2002::/16'), + # RFC 9637: https://www.rfc-editor.org/rfc/rfc9637.html#section-6-2.2 + IPv6Network('3fff::/20'), IPv6Network('fc00::/7'), IPv6Network('fe80::/10'), ] + _private_networks_exceptions = [ + IPv6Network('2001:1::1/128'), + IPv6Network('2001:1::2/128'), + IPv6Network('2001:3::/32'), + IPv6Network('2001:4:112::/48'), + IPv6Network('2001:20::/28'), + IPv6Network('2001:30::/28'), + ] + _reserved_networks = [ IPv6Network('::/8'), IPv6Network('100::/8'), IPv6Network('200::/7'), IPv6Network('400::/6'), diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index ed2c74771ea87d..1d972d22ded072 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -86,13 +86,13 @@ '[2.0, 1.0]' -Using json.tool from the shell to validate and pretty-print:: +Using json from the shell to validate and pretty-print:: - $ echo '{"json":"obj"}' | python -m json.tool + $ echo '{"json":"obj"}' | python -m json { "json": "obj" } - $ echo '{ 1.2:3.4}' | python -m json.tool + $ echo '{ 1.2:3.4}' | python -m json Expecting property name enclosed in double quotes: line 1 column 3 (char 2) """ __version__ = '2.0.9' diff --git a/Lib/json/__main__.py b/Lib/json/__main__.py new file mode 100644 index 00000000000000..1808eaddb62776 --- /dev/null +++ b/Lib/json/__main__.py @@ -0,0 +1,20 @@ +"""Command-line tool to validate and pretty-print JSON + +Usage:: + + $ echo '{"json":"obj"}' | python -m json + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m json + Expecting property name enclosed in double quotes: line 1 column 3 (char 2) + +""" +import json.tool + + +if __name__ == '__main__': + try: + json.tool.main() + except BrokenPipeError as exc: + raise SystemExit(exc.errno) diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index d69a45d6793069..ff4bfcdcc407b9 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -50,17 +50,18 @@ def __reduce__(self): } +HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS) STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) BACKSLASH = { '"': '"', '\\': '\\', '/': '/', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', } -def _decode_uXXXX(s, pos): - esc = s[pos + 1:pos + 5] - if len(esc) == 4 and esc[1] not in 'xX': +def _decode_uXXXX(s, pos, _m=HEXDIGITS.match): + esc = _m(s, pos + 1) + if esc is not None: try: - return int(esc, 16) + return int(esc.group(), 16) except ValueError: pass msg = "Invalid \\uXXXX escape" diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 45f547741885a8..b804224098e14f 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -174,7 +174,7 @@ def default(self, o): else: return list(iterable) # Let the base class default method raise the TypeError - return JSONEncoder.default(self, o) + return super().default(o) """ raise TypeError(f'Object of type {o.__class__.__name__} ' @@ -244,15 +244,18 @@ def floatstr(o, allow_nan=self.allow_nan, return text - if (_one_shot and c_make_encoder is not None - and self.indent is None): + if self.indent is None or isinstance(self.indent, str): + indent = self.indent + else: + indent = ' ' * self.indent + if _one_shot and c_make_encoder is not None: _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, + markers, self.default, _encoder, indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, + markers, self.default, _encoder, indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot) return _iterencode(o, 0) @@ -272,9 +275,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _intstr=int.__repr__, ): - if _indent is not None and not isinstance(_indent, str): - _indent = ' ' * _indent - def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' @@ -293,37 +293,40 @@ def _iterencode_list(lst, _current_indent_level): else: newline_indent = None separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: + for i, value in enumerate(lst): + if i: buf = separator - if isinstance(value, str): - yield buf + _encoder(value) - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, int): - # Subclasses of int/float may override __repr__, but we still - # want to encode them as integers/floats in JSON. One example - # within the standard library is IntEnum. - yield buf + _intstr(value) - elif isinstance(value, float): - # see comment above for int - yield buf + _floatstr(value) - else: - yield buf - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield buf + _encoder(value) + elif value is None: + yield buf + 'null' + elif value is True: + yield buf + 'true' + elif value is False: + yield buf + 'false' + elif isinstance(value, int): + # Subclasses of int/float may override __repr__, but we still + # want to encode them as integers/floats in JSON. One example + # within the standard library is IntEnum. + yield buf + _intstr(value) + elif isinstance(value, float): + # see comment above for int + yield buf + _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks + yield buf + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(lst).__name__} item {i}') + raise if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -382,28 +385,34 @@ def _iterencode_dict(dct, _current_indent_level): yield item_separator yield _encoder(key) yield _key_separator - if isinstance(value, str): - yield _encoder(value) - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, int): - # see comment for int/float in _make_iterencode - yield _intstr(value) - elif isinstance(value, float): - # see comment for int/float in _make_iterencode - yield _floatstr(value) - else: - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield _encoder(value) + elif value is None: + yield 'null' + elif value is True: + yield 'true' + elif value is False: + yield 'false' + elif isinstance(value, int): + # see comment for int/float in _make_iterencode + yield _intstr(value) + elif isinstance(value, float): + # see comment for int/float in _make_iterencode + yield _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(dct).__name__} item {key!r}') + raise if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -436,8 +445,14 @@ def _iterencode(o, _current_indent_level): if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = o - o = _default(o) - yield from _iterencode(o, _current_indent_level) + newobj = _default(o) + try: + yield from _iterencode(newobj, _current_indent_level) + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(o).__name__} object') + raise if markers is not None: del markers[markerid] return _iterencode diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py index 7a61cfc2d24dce..090897515fe2f3 100644 --- a/Lib/json/scanner.py +++ b/Lib/json/scanner.py @@ -9,7 +9,7 @@ __all__ = ['make_scanner'] NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', + r'(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?', (re.VERBOSE | re.MULTILINE | re.DOTALL)) def py_make_scanner(context): diff --git a/Lib/json/tool.py b/Lib/json/tool.py index fdfc3372bcca02..1ba91384c81f27 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -1,14 +1,7 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m json.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 3 (char 2) +"""Command-line tool to validate and pretty-print JSON +See `json.__main__` for a usage example (invocation as +`python -m json.tool` is supported for backwards compatibility). """ import argparse import json @@ -16,10 +9,9 @@ def main(): - prog = 'python -m json.tool' description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') - parser = argparse.ArgumentParser(prog=prog, description=description) + parser = argparse.ArgumentParser(description=description) parser.add_argument('infile', nargs='?', help='a JSON file to be validated or pretty-printed', default='-') @@ -86,4 +78,4 @@ def main(): try: main() except BrokenPipeError as exc: - sys.exit(exc.errno) + raise SystemExit(exc.errno) diff --git a/Lib/linecache.py b/Lib/linecache.py index 329a14053458b7..4b38a0464d8747 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,9 +5,6 @@ that name. """ -import sys -import os - __all__ = ["getline", "clearcache", "checkcache", "lazycache"] @@ -66,9 +63,14 @@ def checkcache(filename=None): size, mtime, lines, fullname = entry if mtime is None: continue # no-op for files loaded via a __loader__ + try: + # This import can fail if the interpreter is shutting down + import os + except ImportError: + return try: stat = os.stat(fullname) - except OSError: + except (OSError, ValueError): cache.pop(filename, None) continue if size != stat.st_size or mtime != stat.st_mtime: @@ -80,6 +82,11 @@ def updatecache(filename, module_globals=None): If something's wrong, print a message, discard the cache entry, and return an empty list.""" + # These imports are not at top level because linecache is in the critical + # path of the interpreter startup and importing os and sys take a lot of time + # and slows down the startup sequence. + import os + import sys import tokenize if filename in cache: @@ -128,16 +135,20 @@ def updatecache(filename, module_globals=None): try: stat = os.stat(fullname) break - except OSError: + except (OSError, ValueError): pass else: return [] + except ValueError: # may be raised by os.stat() + return [] try: with tokenize.open(fullname) as fp: lines = fp.readlines() except (OSError, UnicodeDecodeError, SyntaxError): return [] - if lines and not lines[-1].endswith('\n'): + if not lines: + lines = ['\n'] + elif not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname @@ -166,13 +177,11 @@ def lazycache(filename, module_globals): return False # Try for a __loader__, if available if module_globals and '__name__' in module_globals: - name = module_globals['__name__'] - if (loader := module_globals.get('__loader__')) is None: - if spec := module_globals.get('__spec__'): - try: - loader = spec.loader - except AttributeError: - pass + spec = module_globals.get('__spec__') + name = getattr(spec, 'name', None) or module_globals['__name__'] + loader = getattr(spec, 'loader', None) + if loader is None: + loader = module_globals.get('__loader__') get_source = getattr(loader, 'get_source', None) if name and get_source: diff --git a/Lib/locale.py b/Lib/locale.py index e0cb4c5449d556..d8c09f1123d318 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -1460,7 +1460,8 @@ def getpreferredencoding(do_setlocale=True): # to include every locale up to Windows Vista. # # NOTE: this mapping is incomplete. If your language is missing, please -# submit a bug report to the Python bug tracker at http://bugs.python.org/ +# submit a bug report as detailed in the Python devguide at: +# https://devguide.python.org/triage/issue-tracker/ # Make sure you include the missing language identifier and the suggested # locale code. # diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 684b58d5548f91..aa9b79d8cab4bb 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -37,7 +37,7 @@ 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', - 'warning', 'getLogRecordFactory', 'setLogRecordFactory', + 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort', 'raiseExceptions', 'getLevelNamesMapping', 'getHandlerByName', 'getHandlerNames'] @@ -56,7 +56,7 @@ # #_startTime is used as the base when calculating the relative time of events # -_startTime = time.time() +_startTime = time.time_ns() # #raiseExceptions is used to see if exceptions during handling should be @@ -300,7 +300,7 @@ def __init__(self, name, level, pathname, lineno, """ Initialize a logging record with interesting information. """ - ct = time.time() + ct = time.time_ns() self.name = name self.msg = msg # @@ -339,9 +339,17 @@ def __init__(self, name, level, pathname, lineno, self.stack_info = sinfo self.lineno = lineno self.funcName = func - self.created = ct - self.msecs = int((ct - int(ct)) * 1000) + 0.0 # see gh-89047 - self.relativeCreated = (self.created - _startTime) * 1000 + self.created = ct / 1e9 # ns to float seconds + # Get the number of whole milliseconds (0-999) in the fractional part of seconds. + # Eg: 1_677_903_920_999_998_503 ns --> 999_998_503 ns--> 999 ms + # Convert to float by adding 0.0 for historical reasons. See gh-89047 + self.msecs = (ct % 1_000_000_000) // 1_000_000 + 0.0 + if self.msecs == 999.0 and int(self.created) != ct // 1_000_000_000: + # ns -> sec conversion can round up, e.g: + # 1_677_903_920_999_999_900 ns --> 1_677_903_921.0 sec + self.msecs = 0.0 + + self.relativeCreated = (ct - _startTime) / 1e6 if logThreads: self.thread = threading.get_ident() self.threadName = threading.current_thread().name @@ -572,7 +580,7 @@ class Formatter(object): %(lineno)d Source line number where the logging call was issued (if available) %(funcName)s Function name - %(created)f Time when the LogRecord was created (time.time() + %(created)f Time when the LogRecord was created (time.time_ns() / 1e9 return value) %(asctime)s Textual time when the LogRecord was created %(msecs)d Millisecond portion of the creation time @@ -1522,6 +1530,11 @@ def warning(self, msg, *args, **kwargs): if self.isEnabledFor(WARNING): self._log(WARNING, msg, args, **kwargs) + def warn(self, msg, *args, **kwargs): + warnings.warn("The 'warn' method is deprecated, " + "use 'warning' instead", DeprecationWarning, 2) + self.warning(msg, *args, **kwargs) + def error(self, msg, *args, **kwargs): """ Log 'msg % args' with severity 'ERROR'. @@ -1898,6 +1911,11 @@ def warning(self, msg, *args, **kwargs): """ self.log(WARNING, msg, *args, **kwargs) + def warn(self, msg, *args, **kwargs): + warnings.warn("The 'warn' method is deprecated, " + "use 'warning' instead", DeprecationWarning, 2) + self.warning(msg, *args, **kwargs) + def error(self, msg, *args, **kwargs): """ Delegate an error call to the underlying logger. @@ -1949,18 +1967,11 @@ def hasHandlers(self): """ return self.logger.hasHandlers() - def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False): + def _log(self, level, msg, args, **kwargs): """ Low-level log implementation, proxied to allow nested logger adapters. """ - return self.logger._log( - level, - msg, - args, - exc_info=exc_info, - extra=extra, - stack_info=stack_info, - ) + return self.logger._log(level, msg, args, **kwargs) @property def manager(self): @@ -2020,7 +2031,7 @@ def basicConfig(**kwargs): that this argument is incompatible with 'filename' - if both are present, 'stream' is ignored. handlers If specified, this should be an iterable of already created - handlers, which will be added to the root handler. Any handler + handlers, which will be added to the root logger. Any handler in the list which does not have a formatter assigned will be assigned the formatter created in this function. force If this keyword is specified as true, any existing handlers @@ -2168,6 +2179,11 @@ def warning(msg, *args, **kwargs): basicConfig() root.warning(msg, *args, **kwargs) +def warn(msg, *args, **kwargs): + warnings.warn("The 'warn' function is deprecated, " + "use 'warning' instead", DeprecationWarning, 2) + warning(msg, *args, **kwargs) + def info(msg, *args, **kwargs): """ Log a message with severity 'INFO' on the root logger. If the logger has diff --git a/Lib/logging/config.py b/Lib/logging/config.py index de06090942d965..6a6a7f726f7e0c 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -497,6 +497,33 @@ def as_tuple(self, value): value = tuple(value) return value +def _is_queue_like_object(obj): + """Check that *obj* implements the Queue API.""" + if isinstance(obj, (queue.Queue, queue.SimpleQueue)): + return True + # defer importing multiprocessing as much as possible + from multiprocessing.queues import Queue as MPQueue + if isinstance(obj, MPQueue): + return True + # Depending on the multiprocessing start context, we cannot create + # a multiprocessing.managers.BaseManager instance 'mm' to get the + # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-119819). + # + # Since we only need an object implementing the Queue API, we only + # do a protocol check, but we do not use typing.runtime_checkable() + # and typing.Protocol to reduce import time (see gh-121723). + # + # Ideally, we would have wanted to simply use strict type checking + # instead of a protocol-based type checking since the latter does + # not check the method signatures. + # + # Note that only 'put_nowait' and 'get' are required by the logging + # queue handler and queue listener (see gh-124653) and that other + # methods are either optional or unused. + minimal_queue_interface = ['put_nowait', 'get'] + return all(callable(getattr(obj, method, None)) + for method in minimal_queue_interface) + class DictConfigurator(BaseConfigurator): """ Configure logging using a dictionary-like object to describe the @@ -667,10 +694,9 @@ def configure_formatter(self, config): except TypeError as te: if "'format'" not in str(te): raise - #Name of parameter changed from fmt to format. - #Retry with old name. - #This is so that code can be used with older Python versions - #(e.g. by Django) + # logging.Formatter and its subclasses expect the `fmt` + # parameter instead of `format`. Retry passing configuration + # with `fmt`. config['fmt'] = config.pop('format') config['()'] = factory result = self.configure_custom(config) @@ -726,16 +752,16 @@ def add_filters(self, filterer, filters): def _configure_queue_handler(self, klass, **kwargs): if 'queue' in kwargs: - q = kwargs['queue'] + q = kwargs.pop('queue') else: q = queue.Queue() # unbounded - rhl = kwargs.get('respect_handler_level', False) - if 'listener' in kwargs: - lklass = kwargs['listener'] - else: - lklass = logging.handlers.QueueListener - listener = lklass(q, *kwargs.get('handlers', []), respect_handler_level=rhl) - handler = klass(q) + + rhl = kwargs.pop('respect_handler_level', False) + lklass = kwargs.pop('listener', logging.handlers.QueueListener) + handlers = kwargs.pop('handlers', []) + + listener = lklass(q, *handlers, respect_handler_level=rhl) + handler = klass(q, **kwargs) handler.listener = listener return handler @@ -762,38 +788,39 @@ def configure_handler(self, config): klass = cname else: klass = self.resolve(cname) - if issubclass(klass, logging.handlers.MemoryHandler) and\ - 'target' in config: - # Special case for handler which refers to another handler - try: - tn = config['target'] - th = self.config['handlers'][tn] - if not isinstance(th, logging.Handler): - config.update(config_copy) # restore for deferred cfg - raise TypeError('target not configured yet') - config['target'] = th - except Exception as e: - raise ValueError('Unable to set target handler %r' % tn) from e + if issubclass(klass, logging.handlers.MemoryHandler): + if 'flushLevel' in config: + config['flushLevel'] = logging._checkLevel(config['flushLevel']) + if 'target' in config: + # Special case for handler which refers to another handler + try: + tn = config['target'] + th = self.config['handlers'][tn] + if not isinstance(th, logging.Handler): + config.update(config_copy) # restore for deferred cfg + raise TypeError('target not configured yet') + config['target'] = th + except Exception as e: + raise ValueError('Unable to set target handler %r' % tn) from e elif issubclass(klass, logging.handlers.QueueHandler): # Another special case for handler which refers to other handlers # if 'handlers' not in config: # raise ValueError('No handlers specified for a QueueHandler') if 'queue' in config: - from multiprocessing.queues import Queue as MPQueue qspec = config['queue'] - if not isinstance(qspec, (queue.Queue, MPQueue)): - if isinstance(qspec, str): - q = self.resolve(qspec) - if not callable(q): - raise TypeError('Invalid queue specifier %r' % qspec) - q = q() - elif isinstance(qspec, dict): - if '()' not in qspec: - raise TypeError('Invalid queue specifier %r' % qspec) - q = self.configure_custom(dict(qspec)) - else: + + if isinstance(qspec, str): + q = self.resolve(qspec) + if not callable(q): raise TypeError('Invalid queue specifier %r' % qspec) - config['queue'] = q + config['queue'] = q() + elif isinstance(qspec, dict): + if '()' not in qspec: + raise TypeError('Invalid queue specifier %r' % qspec) + config['queue'] = self.configure_custom(dict(qspec)) + elif not _is_queue_like_object(qspec): + raise TypeError('Invalid queue specifier %r' % qspec) + if 'listener' in config: lspec = config['listener'] if isinstance(lspec, type): @@ -979,7 +1006,8 @@ class ConfigSocketReceiver(ThreadingTCPServer): A simple TCP socket-based logging config receiver. """ - allow_reuse_address = 1 + allow_reuse_address = True + allow_reuse_port = True def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index e7f1322e4ba3d9..1cba64fd554100 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -23,11 +23,17 @@ To use, simply 'import logging.handlers' and log away! """ -import io, logging, socket, os, pickle, struct, time, re -from stat import ST_DEV, ST_INO, ST_MTIME +import copy +import io +import logging +import os +import pickle import queue +import re +import socket +import struct import threading -import copy +import time # # Some constants... @@ -187,15 +193,18 @@ def shouldRollover(self, record): Basically, see if the supplied record would cause the file to exceed the size limit we have. """ - # See bpo-45401: Never rollover anything other than regular files - if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): - return False if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? + pos = self.stream.tell() + if not pos: + # gh-116263: Never rollover an empty file + return False msg = "%s\n" % self.format(record) - self.stream.seek(0, 2) #due to non-posix-compliant Windows feature - if self.stream.tell() + len(msg) >= self.maxBytes: + if pos + len(msg) >= self.maxBytes: + # See bpo-45401: Never rollover anything other than regular files + if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): + return False return True return False @@ -232,19 +241,19 @@ def __init__(self, filename, when='h', interval=1, backupCount=0, if self.when == 'S': self.interval = 1 # one second self.suffix = "%Y-%m-%d_%H-%M-%S" - self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$" + extMatch = r"(? (plen + 1) and not fileName[plen+1].isdigit()): - continue - - if fileName[:plen] == prefix: - suffix = fileName[plen:] - # See bpo-45628: The date/time suffix could be anywhere in the - # filename - parts = suffix.split('.') - for part in parts: - if self.extMatch.match(part): + if self.namer is None: + prefix = baseName + '.' + plen = len(prefix) + for fileName in fileNames: + if fileName[:plen] == prefix: + suffix = fileName[plen:] + if self.extMatch.fullmatch(suffix): + result.append(os.path.join(dirName, fileName)) + else: + for fileName in fileNames: + # Our files could be just about anything after custom naming, + # but they should contain the datetime suffix. + # Try to find the datetime suffix in the file name and verify + # that the file name can be generated by this handler. + m = self.extMatch.search(fileName) + while m: + dfn = self.namer(self.baseFilename + "." + m[0]) + if os.path.basename(dfn) == fileName: result.append(os.path.join(dirName, fileName)) break + m = self.extMatch.search(fileName, m.start() + 1) + if len(result) < self.backupCount: result = [] else: @@ -410,17 +425,14 @@ def doRollover(self): then we have to get a list of matching filenames, sort them and remove the one with the oldest suffix. """ - if self.stream: - self.stream.close() - self.stream = None # get the time that this sequence started at and make it a TimeTuple currentTime = int(time.time()) - dstNow = time.localtime(currentTime)[-1] t = self.rolloverAt - self.interval if self.utc: timeTuple = time.gmtime(t) else: timeTuple = time.localtime(t) + dstNow = time.localtime(currentTime)[-1] dstThen = timeTuple[-1] if dstNow != dstThen: if dstNow: @@ -431,26 +443,19 @@ def doRollover(self): dfn = self.rotation_filename(self.baseFilename + "." + time.strftime(self.suffix, timeTuple)) if os.path.exists(dfn): - os.remove(dfn) + # Already rolled over. + return + + if self.stream: + self.stream.close() + self.stream = None self.rotate(self.baseFilename, dfn) if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) if not self.delay: self.stream = self._open() - newRolloverAt = self.computeRollover(currentTime) - while newRolloverAt <= currentTime: - newRolloverAt = newRolloverAt + self.interval - #If DST changes and midnight or weekly rollover, adjust for this. - if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc: - dstAtRollover = time.localtime(newRolloverAt)[-1] - if dstNow != dstAtRollover: - if not dstNow: # DST kicks in before next rollover, so we need to deduct an hour - addend = -3600 - else: # DST bows out before next rollover, so we need to add an hour - addend = 3600 - newRolloverAt += addend - self.rolloverAt = newRolloverAt + self.rolloverAt = self.computeRollover(currentTime) class WatchedFileHandler(logging.FileHandler): """ @@ -466,8 +471,7 @@ class WatchedFileHandler(logging.FileHandler): This handler is not appropriate for use under Windows, because under Windows open files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need - for such a handler. Furthermore, ST_INO is not supported under - Windows; stat always returns zero for this value. + for such a handler. This handler is based on a suggestion and patch by Chad J. Schroeder. @@ -483,9 +487,11 @@ def __init__(self, filename, mode='a', encoding=None, delay=False, self._statstream() def _statstream(self): - if self.stream: - sres = os.fstat(self.stream.fileno()) - self.dev, self.ino = sres[ST_DEV], sres[ST_INO] + if self.stream is None: + return + sres = os.fstat(self.stream.fileno()) + self.dev = sres.st_dev + self.ino = sres.st_ino def reopenIfNeeded(self): """ @@ -495,6 +501,9 @@ def reopenIfNeeded(self): has, close the old stream and reopen the file to get the current stream. """ + if self.stream is None: + return + # Reduce the chance of race conditions by stat'ing by path only # once and then fstat'ing our new fd if we opened a new log stream. # See issue #14632: Thanks to John Mulligan for the problem report @@ -502,18 +511,23 @@ def reopenIfNeeded(self): try: # stat the file by path, checking for existence sres = os.stat(self.baseFilename) + + # compare file system stat with that of our stream file handle + reopen = (sres.st_dev != self.dev or sres.st_ino != self.ino) except FileNotFoundError: - sres = None - # compare file system stat with that of our stream file handle - if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino: - if self.stream is not None: - # we have an open file handle, clean it up - self.stream.flush() - self.stream.close() - self.stream = None # See Issue #21742: _open () might fail. - # open a new file handle and get new stat info from that fd - self.stream = self._open() - self._statstream() + reopen = True + + if not reopen: + return + + # we have an open file handle, clean it up + self.stream.flush() + self.stream.close() + self.stream = None # See Issue #21742: _open () might fail. + + # open a new file handle and get new stat info from that fd + self.stream = self._open() + self._statstream() def emit(self, record): """ diff --git a/Lib/lzma.py b/Lib/lzma.py index 800f52198fbb79..946066aa0fba56 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -25,11 +25,11 @@ import io import os from _lzma import * -from _lzma import _encode_filter_properties, _decode_filter_properties +from _lzma import _encode_filter_properties, _decode_filter_properties # noqa: F401 import _compression -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 @@ -92,7 +92,7 @@ def __init__(self, filename=None, mode="r", *, """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if mode in ("r", "rb"): if check != -1: @@ -137,7 +137,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -153,12 +153,20 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 746811bd559412..b00d9e8634c785 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -750,9 +750,13 @@ def flush(self): _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() - # Make sure the new file's mode is the same as the old file's - mode = os.stat(self._path).st_mode - os.chmod(new_file.name, mode) + # Make sure the new file's mode and owner are the same as the old file's + info = os.stat(self._path) + os.chmod(new_file.name, info.st_mode) + try: + os.chown(new_file.name, info.st_uid, info.st_gid) + except (AttributeError, OSError): + pass try: os.rename(new_file.name, self._path) except FileExistsError: diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 51b99701c9d727..d7c4e8444f8dec 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -40,7 +40,7 @@ __all__ = [ "knownfiles", "inited", "MimeTypes", - "guess_type", "guess_all_extensions", "guess_extension", + "guess_type", "guess_file_type", "guess_all_extensions", "guess_extension", "add_type", "init", "read_mime_types", "suffix_map", "encodings_map", "types_map", "common_types" ] @@ -116,11 +116,17 @@ def guess_type(self, url, strict=True): mapped to '.tar.gz'. (This is table-driven too, using the dictionary suffix_map.) - Optional `strict' argument when False adds a bunch of commonly found, + Optional 'strict' argument when False adds a bunch of commonly found, but non-standard types. """ + # TODO: Deprecate accepting file paths (in particular path-like objects). url = os.fspath(url) - scheme, url = urllib.parse._splittype(url) + p = urllib.parse.urlparse(url) + if p.scheme and len(p.scheme) > 1: + scheme = p.scheme + url = p.path + else: + return self.guess_file_type(url, strict=strict) if scheme == 'data': # syntax of data URLs: # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data @@ -140,13 +146,25 @@ def guess_type(self, url, strict=True): if '=' in type or '/' not in type: type = 'text/plain' return type, None # never compressed, so encoding is None - base, ext = posixpath.splitext(url) + return self._guess_file_type(url, strict, posixpath.splitext) + + def guess_file_type(self, path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path instead of URL. + """ + path = os.fsdecode(path) + path = os.path.splitdrive(path)[1] + return self._guess_file_type(path, strict, os.path.splitext) + + def _guess_file_type(self, path, strict, splitext): + base, ext = splitext(path) while (ext_lower := ext.lower()) in self.suffix_map: - base, ext = posixpath.splitext(base + self.suffix_map[ext_lower]) + base, ext = splitext(base + self.suffix_map[ext_lower]) # encodings_map is case sensitive if ext in self.encodings_map: encoding = self.encodings_map[ext] - base, ext = posixpath.splitext(base) + base, ext = splitext(base) else: encoding = None ext = ext.lower() @@ -167,9 +185,9 @@ def guess_all_extensions(self, type, strict=True): Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, - but would be mapped to the MIME type `type' by guess_type(). + but would be mapped to the MIME type 'type' by guess_type(). - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ type = type.lower() @@ -186,11 +204,11 @@ def guess_extension(self, type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ extensions = self.guess_all_extensions(type, strict) @@ -296,7 +314,7 @@ def guess_type(url, strict=True): to ".tar.gz". (This is table-driven too, using the dictionary suffix_map). - Optional `strict' argument when false adds a bunch of commonly found, but + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -304,17 +322,27 @@ def guess_type(url, strict=True): return _db.guess_type(url, strict) +def guess_file_type(path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path instead of URL. + """ + if _db is None: + init() + return _db.guess_file_type(path, strict=strict) + + def guess_all_extensions(type, strict=True): """Guess the extensions for a file based on its MIME type. Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -327,10 +355,10 @@ def guess_extension(type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the - MIME type `type' by guess_type(). If no extension can be guessed for - `type', None is returned. + MIME type 'type' by guess_type(). If no extension can be guessed for + 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -546,6 +574,8 @@ def _default_mime_types(): '.csv' : 'text/csv', '.html' : 'text/html', '.htm' : 'text/html', + '.md' : 'text/markdown', + '.markdown': 'text/markdown', '.n3' : 'text/n3', '.txt' : 'text/plain', '.bat' : 'text/plain', @@ -555,9 +585,11 @@ def _default_mime_types(): '.pl' : 'text/plain', '.srt' : 'text/plain', '.rtx' : 'text/richtext', + '.rtf' : 'text/rtf', '.tsv' : 'text/tab-separated-values', '.vtt' : 'text/vtt', '.py' : 'text/x-python', + '.rst' : 'text/x-rst', '.etx' : 'text/x-setext', '.sgm' : 'text/x-sgml', '.sgml' : 'text/x-sgml', diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index a0a020f9eeb9b4..ac478ee7f51722 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -72,7 +72,12 @@ def _find_module(name, path=None): if isinstance(spec.loader, importlib.machinery.SourceFileLoader): kind = _PY_SOURCE - elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader): + elif isinstance( + spec.loader, ( + importlib.machinery.ExtensionFileLoader, + importlib.machinery.AppleFrameworkLoader, + ) + ): kind = _C_EXTENSION elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index dbbf106f680964..7e901cf2fb9852 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -11,15 +11,14 @@ import errno import io +import itertools import os import sys import socket import struct -import time import tempfile -import itertools +import time -import _multiprocessing from . import util @@ -28,6 +27,7 @@ _ForkingPickler = reduction.ForkingPickler try: + import _multiprocessing import _winapi from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE except ImportError: @@ -39,7 +39,9 @@ # # -BUFSIZE = 8192 +# 64 KiB is the default PIPE buffer size of most POSIX platforms. +BUFSIZE = 64 * 1024 + # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. @@ -392,7 +394,8 @@ def _recv(self, size, read=_read): handle = self._handle remaining = size while remaining > 0: - chunk = read(handle, remaining) + to_read = min(BUFSIZE, remaining) + chunk = read(handle, to_read) n = len(chunk) if n == 0: if remaining == size: @@ -476,8 +479,9 @@ def accept(self): ''' if self._listener is None: raise OSError('listener is closed') + c = self._listener.accept() - if self._authkey: + if self._authkey is not None: deliver_challenge(c, self._authkey) answer_challenge(c, self._authkey) return c @@ -1011,8 +1015,20 @@ def _exhaustive_wait(handles, timeout): # returning the first signalled might create starvation issues.) L = list(handles) ready = [] + # Windows limits WaitForMultipleObjects at 64 handles, and we use a + # few for synchronisation, so we switch to batched waits at 60. + if len(L) > 60: + try: + res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout) + except TimeoutError: + return [] + ready.extend(L[i] for i in res) + if res: + L = [h for i, h in enumerate(L) if i > res[0] & i not in res] + timeout = 0 while L: - res = _winapi.WaitForMultipleObjects(L, False, timeout) + short_L = L[:60] if len(L) > 60 else L + res = _winapi.WaitForMultipleObjects(short_L, False, timeout) if res == WAIT_TIMEOUT: break elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index de8a264829dff3..d0a3ad00e53ad8 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -167,7 +167,7 @@ def allow_connection_pickling(self): ''' # This is undocumented. In previous versions of multiprocessing # its only effect was to make socket objects inheritable on Windows. - from . import connection + from . import connection # noqa: F401 def set_executable(self, executable): '''Sets the path to a python.exe or pythonw.exe binary used to run @@ -259,13 +259,12 @@ def get_start_method(self, allow_none=False): def get_all_start_methods(self): """Returns a list of the supported start methods, default first.""" - if sys.platform == 'win32': - return ['spawn'] - else: - methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn'] - if reduction.HAVE_SEND_HANDLE: - methods.append('forkserver') - return methods + default = self._default_context.get_start_method() + start_method_names = [default] + start_method_names.extend( + name for name in _concrete_contexts if name != default + ) + return start_method_names # @@ -320,14 +319,15 @@ def _check_available(self): 'spawn': SpawnContext(), 'forkserver': ForkServerContext(), } - if sys.platform == 'darwin': - # bpo-33725: running arbitrary code after fork() is no longer reliable - # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. - _default_context = DefaultContext(_concrete_contexts['spawn']) + # bpo-33725: running arbitrary code after fork() is no longer reliable + # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. + # gh-84559: We changed everyones default to a thread safeish one in 3.14. + if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': + _default_context = DefaultContext(_concrete_contexts['forkserver']) else: - _default_context = DefaultContext(_concrete_contexts['fork']) + _default_context = DefaultContext(_concrete_contexts['spawn']) -else: +else: # Windows class SpawnProcess(process.BaseProcess): _start_method = 'spawn' diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 4642707dae2f4e..53b8c492675878 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,3 +1,4 @@ +import atexit import errno import os import selectors @@ -271,6 +272,8 @@ def sigchld_handler(*_unused): selector.close() unused_fds = [alive_r, child_w, sig_r, sig_w] unused_fds.extend(pid_to_fd.values()) + atexit._clear() + atexit.register(util._exit_function) code = _serve_one(child_r, fds, unused_fds, old_handlers) @@ -278,6 +281,7 @@ def sigchld_handler(*_unused): sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: + atexit._run_exitfuncs() os._exit(code) else: # Send pid to client process diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 76b915de74d94e..0f5f9f64c2de9e 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1152,10 +1152,10 @@ def set(self, value): BaseListProxy = MakeProxyType('BaseListProxy', ( - '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', - '__mul__', '__reversed__', '__rmul__', '__setitem__', - 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', - 'reverse', 'sort', '__imul__' + '__add__', '__contains__', '__delitem__', '__getitem__', '__imul__', + '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__', + 'append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop', + 'remove', 'reverse', 'sort', )) class ListProxy(BaseListProxy): def __iadd__(self, value): @@ -1169,16 +1169,20 @@ def __imul__(self, value): _BaseDictProxy = MakeProxyType('DictProxy', ( - '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', - '__setitem__', 'clear', 'copy', 'get', 'items', + '__contains__', '__delitem__', '__getitem__', '__ior__', '__iter__', + '__len__', '__or__', '__reversed__', '__ror__', + '__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' )) _BaseDictProxy._method_to_typeid_ = { '__iter__': 'Iterator', } class DictProxy(_BaseDictProxy): - __class_getitem__ = classmethod(types.GenericAlias) + def __ior__(self, value): + self._callmethod('__ior__', (value,)) + return self + __class_getitem__ = classmethod(types.GenericAlias) ArrayProxy = MakeProxyType('ArrayProxy', ( '__len__', '__getitem__', '__setitem__' diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 625981cf47627c..a57ef6bdad5ccc 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -1,3 +1,4 @@ +import atexit import os import signal @@ -66,10 +67,13 @@ def _launch(self, process_obj): self.pid = os.fork() if self.pid == 0: try: + atexit._clear() + atexit.register(util._exit_function) os.close(parent_r) os.close(parent_w) code = process_obj._bootstrap(parent_sentinel=child_r) finally: + atexit._run_exitfuncs() os._exit(code) else: os.close(child_w) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 49d4c7eea22411..62fb0ddbf91a5d 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -3,6 +3,7 @@ import signal import sys import _winapi +from subprocess import STARTUPINFO, STARTF_FORCEOFFFEEDBACK from .context import reduction, get_spawning_popen, set_spawning_popen from . import spawn @@ -74,7 +75,8 @@ def __init__(self, process_obj): try: hp, ht, pid, tid = _winapi.CreateProcess( python_exe, cmd, - None, None, False, 0, env, None, None) + None, None, False, 0, env, None, + STARTUPINFO(dwFlags=STARTF_FORCEOFFFEEDBACK)) _winapi.CloseHandle(ht) except: _winapi.CloseHandle(rhandle) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 271ba3fd325138..b45f7df476f7d8 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -310,11 +310,8 @@ def _bootstrap(self, parent_sentinel=None): # _run_after_forkers() is executed del old_process util.info('child process calling self.run()') - try: - self.run() - exitcode = 0 - finally: - util._exit_function() + self.run() + exitcode = 0 except SystemExit as e: if e.code is None: exitcode = 0 diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 852ae87b276861..925f043900004e 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -20,8 +20,6 @@ from queue import Empty, Full -import _multiprocessing - from . import connection from . import context _ForkingPickler = context.reduction.ForkingPickler diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 8e41f461cc934e..20ddd9c50e3d88 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -29,8 +29,12 @@ _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask') _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM) +def cleanup_noop(name): + raise RuntimeError('noop should never be registered or cleaned up') + _CLEANUP_FUNCS = { - 'noop': lambda: None, + 'noop': cleanup_noop, + 'dummy': lambda name: None, # Dummy resource used in tests } if os.name == 'posix': @@ -61,6 +65,7 @@ def __init__(self): self._lock = threading.RLock() self._fd = None self._pid = None + self._exitcode = None def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker @@ -84,9 +89,16 @@ def _stop(self): os.close(self._fd) self._fd = None - os.waitpid(self._pid, 0) + _, status = os.waitpid(self._pid, 0) + self._pid = None + try: + self._exitcode = os.waitstatus_to_exitcode(status) + except ValueError: + # os.waitstatus_to_exitcode may raise an exception for invalid values + self._exitcode = None + def getfd(self): self.ensure_running() return self._fd @@ -119,6 +131,7 @@ def ensure_running(self): pass self._fd = None self._pid = None + self._exitcode = None warnings.warn('resource_tracker: process died unexpectedly, ' 'relaunching. Some resources might leak.') @@ -221,6 +234,8 @@ def main(fd): pass cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()} + exit_code = 0 + try: # keep track of registered/unregistered resources with open(fd, 'rb') as f: @@ -242,6 +257,7 @@ def main(fd): else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: + exit_code = 3 try: sys.excepthook(*sys.exc_info()) except: @@ -251,10 +267,17 @@ def main(fd): for rtype, rtype_cache in cache.items(): if rtype_cache: try: - warnings.warn( - f'resource_tracker: There appear to be {len(rtype_cache)} ' - f'leaked {rtype} objects to clean up at shutdown: {rtype_cache}' - ) + exit_code = 1 + if rtype == 'dummy': + # The test 'dummy' resource is expected to leak. + # We skip the warning (and *only* the warning) for it. + pass + else: + warnings.warn( + f'resource_tracker: There appear to be ' + f'{len(rtype_cache)} leaked {rtype} objects to ' + f'clean up at shutdown: {rtype_cache}' + ) except Exception: pass for name in rtype_cache: @@ -265,6 +288,9 @@ def main(fd): try: _CLEANUP_FUNCS[rtype](name) except Exception as e: + exit_code = 2 warnings.warn('resource_tracker: %r: %s' % (name, e)) finally: pass + + sys.exit(exit_code) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 67e70fdc27cf31..99a8ce3320ad4e 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -539,6 +539,6 @@ def index(self, value): if value == entry: return position else: - raise ValueError(f"{value!r} not in this container") + raise ValueError("ShareableList.index(x): x not in list") __class_getitem__ = classmethod(types.GenericAlias) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 32871850ddec8b..b7192042b9cf47 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -14,7 +14,7 @@ import atexit import threading # we want threading to install it's # cleanup function before multiprocessing does -from subprocess import _args_from_interpreter_flags +from subprocess import _args_from_interpreter_flags # noqa: F401 from . import process @@ -102,11 +102,7 @@ def log_to_stderr(level=None): # Abstract socket support def _platform_supports_abstract_sockets(): - if sys.platform == "linux": - return True - if hasattr(sys, 'getandroidapilevel'): - return True - return False + return sys.platform in ("linux", "android") def is_abstract_socket_namespace(address): @@ -442,15 +438,13 @@ def _flush_std_streams(): def spawnv_passfds(path, args, passfds): import _posixsubprocess - import subprocess passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( args, [path], True, passfds, None, None, -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, - False, False, -1, None, None, None, -1, None, - subprocess._USE_VFORK) + False, False, -1, None, None, None, -1, None) finally: os.close(errpipe_read) os.close(errpipe_write) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index e7cbfe17ecb3c8..1b1873f08b608b 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -19,7 +19,6 @@ import os import sys -import stat import genericpath from genericpath import * @@ -29,7 +28,8 @@ "ismount","isreserved","expanduser","expandvars","normpath", "abspath","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", - "samefile", "sameopenfile", "samestat", "commonpath", "isjunction"] + "samefile", "sameopenfile", "samestat", "commonpath", "isjunction", + "isdevdrive"] def _get_bothseps(path): if isinstance(path, bytes): @@ -101,16 +101,14 @@ def join(path, *paths): if isinstance(path, bytes): sep = b'\\' seps = b'\\/' - colon = b':' + colon_seps = b':\\/' else: sep = '\\' seps = '\\/' - colon = ':' + colon_seps = ':\\/' try: - if not paths: - path[:0] + sep #23780: Ensure compatible data type even if p is null. result_drive, result_root, result_path = splitroot(path) - for p in map(os.fspath, paths): + for p in paths: p_drive, p_root, p_path = splitroot(p) if p_root: # Second path is absolute @@ -134,7 +132,7 @@ def join(path, *paths): result_path = result_path + p_path ## add separator between UNC and non-absolute path if (result_path and not result_root and - result_drive and result_drive[-1:] not in colon + seps): + result_drive and result_drive[-1] not in colon_seps): return result_drive + sep + result_path return result_drive + result_root + result_path except (TypeError, AttributeError, BytesWarning): @@ -168,56 +166,52 @@ def splitdrive(p): return drive, root + tail -def splitroot(p): - """Split a pathname into drive, root and tail. The drive is defined - exactly as in splitdrive(). On Windows, the root may be a single path - separator or an empty string. The tail contains anything after the root. - For example: +try: + from nt import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. - splitroot('//server/share/') == ('//server/share', '/', '') - splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') - splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') - splitroot('Windows/notepad') == ('', '', 'Windows/notepad') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'\\' - altsep = b'/' - colon = b':' - unc_prefix = b'\\\\?\\UNC\\' - empty = b'' - else: - sep = '\\' - altsep = '/' - colon = ':' - unc_prefix = '\\\\?\\UNC\\' - empty = '' - normp = p.replace(altsep, sep) - if normp[:1] == sep: - if normp[1:2] == sep: - # UNC drives, e.g. \\server\share or \\?\UNC\server\share - # Device drives, e.g. \\.\device or \\?\device - start = 8 if normp[:8].upper() == unc_prefix else 2 - index = normp.find(sep, start) - if index == -1: - return p, empty, empty - index2 = normp.find(sep, index + 1) - if index2 == -1: - return p, empty, empty - return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + The tail contains anything after the root.""" + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'\\' + altsep = b'/' + colon = b':' + unc_prefix = b'\\\\?\\UNC\\' + empty = b'' else: - # Relative path with root, e.g. \Windows - return empty, p[:1], p[1:] - elif normp[1:2] == colon: - if normp[2:3] == sep: - # Absolute drive-letter path, e.g. X:\Windows - return p[:2], p[2:3], p[3:] + sep = '\\' + altsep = '/' + colon = ':' + unc_prefix = '\\\\?\\UNC\\' + empty = '' + normp = p.replace(altsep, sep) + if normp[:1] == sep: + if normp[1:2] == sep: + # UNC drives, e.g. \\server\share or \\?\UNC\server\share + # Device drives, e.g. \\.\device or \\?\device + start = 8 if normp[:8].upper() == unc_prefix else 2 + index = normp.find(sep, start) + if index == -1: + return p, empty, empty + index2 = normp.find(sep, index + 1) + if index2 == -1: + return p, empty, empty + return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + else: + # Relative path with root, e.g. \Windows + return empty, p[:1], p[1:] + elif normp[1:2] == colon: + if normp[2:3] == sep: + # Absolute drive-letter path, e.g. X:\Windows + return p[:2], p[2:3], p[3:] + else: + # Relative path with drive, e.g. X:Windows + return p[:2], empty, p[2:] else: - # Relative path with drive, e.g. X:Windows - return p[:2], empty, p[2:] - else: - # Relative path, e.g. Windows - return empty, empty, p + # Relative path, e.g. Windows + return empty, empty, p # Split a path in head (everything up to the last '/') and tail (the @@ -269,33 +263,6 @@ def dirname(p): return split(p)[0] -# Is a path a junction? - -if hasattr(os.stat_result, 'st_reparse_tag'): - def isjunction(path): - """Test whether a path is a junction""" - try: - st = os.lstat(path) - except (OSError, ValueError, AttributeError): - return False - return bool(st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT) -else: - def isjunction(path): - """Test whether a path is a junction""" - os.fspath(path) - return False - - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - st = os.lstat(path) - except (OSError, ValueError): - return False - return True - # Is a path a mount point? # Any drive letter root (eg c:\) # Any share UNC (eg \\server\share) @@ -351,8 +318,8 @@ def isreserved(path): def _isreservedname(name): """Return true if the filename is reserved by the system.""" # Trailing dots and spaces are reserved. - if name.endswith(('.', ' ')) and name not in ('.', '..'): - return True + if name[-1:] in ('.', ' '): + return name not in ('.', '..') # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved. # ASCII control characters (0-31) are reserved. # Colon is reserved for file streams (e.g. "name:stream[:type]"). @@ -361,9 +328,7 @@ def _isreservedname(name): # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules # are complex and vary across Windows versions. On the side of # caution, return True for names that may not be reserved. - if name.partition('.')[0].rstrip(' ').upper() in _reserved_names: - return True - return False + return name.partition('.')[0].rstrip(' ').upper() in _reserved_names # Expand paths beginning with '~' or '~user'. @@ -381,24 +346,23 @@ def expanduser(path): If user or $HOME is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): + seps = b'\\/' tilde = b'~' else: + seps = '\\/' tilde = '~' if not path.startswith(tilde): return path i, n = 1, len(path) - while i < n and path[i] not in _get_bothseps(path): + while i < n and path[i] not in seps: i += 1 if 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] - elif not 'HOMEPATH' in os.environ: + elif 'HOMEPATH' not in os.environ: return path else: - try: - drive = os.environ['HOMEDRIVE'] - except KeyError: - drive = '' + drive = os.environ.get('HOMEDRIVE', '') userhome = join(drive, os.environ['HOMEPATH']) if i != 1: #~user @@ -549,7 +513,7 @@ def expandvars(path): # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! try: - from nt import _path_normpath + from nt import _path_normpath as normpath except ImportError: def normpath(path): @@ -588,14 +552,6 @@ def normpath(path): comps.append(curdir) return prefix + sep.join(comps) -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def _abspath_fallback(path): """Return the absolute version of a path as a fallback function in case @@ -738,7 +694,8 @@ def realpath(path, *, strict=False): new_unc_prefix = b'\\\\' cwd = os.getcwdb() # bpo-38081: Special case for realpath(b'nul') - if normcase(path) == normcase(os.fsencode(devnull)): + devnull = b'nul' + if normcase(path) == devnull: return b'\\\\.\\NUL' else: prefix = '\\\\?\\' @@ -746,7 +703,8 @@ def realpath(path, *, strict=False): new_unc_prefix = '\\\\' cwd = os.getcwd() # bpo-38081: Special case for realpath('nul') - if normcase(path) == normcase(devnull): + devnull = 'nul' + if normcase(path) == devnull: return '\\\\.\\NUL' had_prefix = path.startswith(prefix) if not had_prefix and not isabs(path): @@ -799,6 +757,9 @@ def realpath(path, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + if not path: + raise ValueError("no path specified") + if isinstance(path, bytes): sep = b'\\' curdir = b'.' @@ -810,22 +771,20 @@ def relpath(path, start=None): if start is None: start = curdir + else: + start = os.fspath(start) - if not path: - raise ValueError("no path specified") - - start = os.fspath(start) try: - start_abs = abspath(normpath(start)) - path_abs = abspath(normpath(path)) + start_abs = abspath(start) + path_abs = abspath(path) start_drive, _, start_rest = splitroot(start_abs) path_drive, _, path_rest = splitroot(path_abs) if normcase(start_drive) != normcase(path_drive): raise ValueError("path is on mount %r, start on mount %r" % ( path_drive, start_drive)) - start_list = [x for x in start_rest.split(sep) if x] - path_list = [x for x in path_rest.split(sep) if x] + start_list = start_rest.split(sep) if start_rest else [] + path_list = path_rest.split(sep) if path_rest else [] # Work out how much of the filepath is shared by start and path. i = 0 for e1, e2 in zip(start_list, path_list): @@ -836,29 +795,28 @@ def relpath(path, start=None): rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise -# Return the longest common sub-path of the sequence of paths given as input. +# Return the longest common sub-path of the iterable of paths given as input. # The function is case-insensitive and 'separator-insensitive', i.e. if the # only difference between two paths is the use of '\' versus '/' as separator, # they are deemed to be equal. # # However, the returned path will have the standard '\' separator (even if the # given paths had the alternative '/' separator) and will have the case of the -# first path given in the sequence. Additionally, any trailing separator is +# first path given in the iterable. Additionally, any trailing separator is # stripped from the returned path. def commonpath(paths): - """Given a sequence of path names, returns the longest common sub-path.""" - + """Given an iterable of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) if not paths: - raise ValueError('commonpath() arg is an empty sequence') + raise ValueError('commonpath() arg is an empty iterable') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'\\' altsep = b'/' @@ -872,9 +830,6 @@ def commonpath(paths): drivesplits = [splitroot(p.replace(altsep, sep).lower()) for p in paths] split_paths = [p.split(sep) for d, r, p in drivesplits] - if len({r for d, r, p in drivesplits}) != 1: - raise ValueError("Can't mix absolute and relative paths") - # Check that all drive letters or UNC paths match. The check is made only # now otherwise type errors for mixing strings and bytes would not be # caught. @@ -882,6 +837,12 @@ def commonpath(paths): raise ValueError("Paths don't have the same drive") drive, root, path = splitroot(paths[0].replace(altsep, sep)) + if len({r for d, r, p in drivesplits}) != 1: + if drive: + raise ValueError("Can't mix absolute and relative paths") + else: + raise ValueError("Can't mix rooted and not-rooted paths") + common = path.split(sep) common = [c for c in common if c and c != curdir] @@ -902,13 +863,15 @@ def commonpath(paths): try: - # The isdir(), isfile(), islink() and exists() implementations in - # genericpath use os.stat(). This is overkill on Windows. Use simpler + # The isdir(), isfile(), islink(), exists() and lexists() implementations + # in genericpath use os.stat(). This is overkill on Windows. Use simpler # builtin functions if they are available. from nt import _path_isdir as isdir from nt import _path_isfile as isfile from nt import _path_islink as islink + from nt import _path_isjunction as isjunction from nt import _path_exists as exists + from nt import _path_lexists as lexists except ImportError: # Use genericpath.* as imported above pass @@ -916,15 +879,12 @@ def commonpath(paths): try: from nt import _path_isdevdrive -except ImportError: - def isdevdrive(path): - """Determines whether the specified path is on a Windows Dev Drive.""" - # Never a Dev Drive - return False -else: def isdevdrive(path): """Determines whether the specified path is on a Windows Dev Drive.""" try: return _path_isdevdrive(abspath(path)) except OSError: return False +except ImportError: + # Use genericpath.isdevdrive as imported above + pass diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py index 61852aff58912d..6453f202c26d14 100644 --- a/Lib/nturl2path.py +++ b/Lib/nturl2path.py @@ -55,16 +55,11 @@ def pathname2url(p): if p[:4] == '\\\\?\\': p = p[4:] if p[:4].upper() == 'UNC\\': - p = '\\' + p[4:] + p = '\\\\' + p[4:] elif p[1:2] != ':': raise OSError('Bad path: ' + p) if not ':' in p: # No drive specifier, just convert slashes and quote the name - if p[:2] == '\\\\': - # path is something like \\host\path\on\remote\host - # convert this to ////host/path/on/remote/host - # (notice doubling of slashes at the start of the path) - p = '\\\\' + p components = p.split('\\') return urllib.parse.quote('/'.join(components)) comp = p.split(':', maxsplit=2) diff --git a/Lib/opcode.py b/Lib/opcode.py index 88f4df7c0e8c38..974f4d35e2a524 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -12,8 +12,8 @@ import _opcode from _opcode import stack_effect -from _opcode_metadata import (_specializations, _specialized_opmap, opmap, - HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, # noqa: F401 + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) # noqa: F401 EXTENDED_ARG = opmap['EXTENDED_ARG'] opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] @@ -36,6 +36,8 @@ _intrinsic_1_descs = _opcode.get_intrinsic1_descs() _intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_special_method_names = _opcode.get_special_method_names() +_common_constants = [AssertionError, NotImplementedError] _nb_ops = _opcode.get_nb_ops() hascompare = [opmap["COMPARE_OP"]] @@ -56,6 +58,9 @@ "COMPARE_OP": { "counter": 1, }, + "CONTAINS_OP": { + "counter": 1, + }, "BINARY_SUBSCR": { "counter": 1, }, @@ -80,6 +85,10 @@ "counter": 1, "func_version": 2, }, + "CALL_KW": { + "counter": 1, + "func_version": 2, + }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Lib/operator.py b/Lib/operator.py index 30116c1189a499..1b765522f85949 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -14,8 +14,8 @@ 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', - 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', - 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', + 'is_', 'is_none', 'is_not', 'is_not_none', 'isub', 'itemgetter', 'itruediv', + 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', 'setitem', 'sub', 'truediv', 'truth', 'xor'] @@ -66,6 +66,14 @@ def is_not(a, b): "Same as a is not b." return a is not b +def is_none(a): + "Same as a is None." + return a is None + +def is_not_none(a): + "Same as a is not None." + return a is not None + # Mathematical/Bitwise Operations *********************************************# def abs(a): @@ -239,7 +247,7 @@ class attrgetter: """ __slots__ = ('_attrs', '_call') - def __init__(self, attr, *attrs): + def __init__(self, attr, /, *attrs): if not attrs: if not isinstance(attr, str): raise TypeError('attribute name must be a string') @@ -257,7 +265,7 @@ def func(obj): return tuple(getter(obj) for getter in getters) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -276,7 +284,7 @@ class itemgetter: """ __slots__ = ('_items', '_call') - def __init__(self, item, *items): + def __init__(self, item, /, *items): if not items: self._items = (item,) def func(obj): @@ -288,7 +296,7 @@ def func(obj): return tuple(obj[i] for i in items) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -315,7 +323,7 @@ def __init__(self, name, /, *args, **kwargs): self._args = args self._kwargs = kwargs - def __call__(self, obj): + def __call__(self, obj, /): return getattr(obj, self._name)(*self._args, **self._kwargs) def __repr__(self): @@ -415,7 +423,7 @@ def ixor(a, b): except ImportError: pass else: - from _operator import __doc__ + from _operator import __doc__ # noqa: F401 # All of these "__func__ = func" assignments have to happen after importing # from _operator to make sure they're set to the right function diff --git a/Lib/os.py b/Lib/os.py index 7f38e14e7bdd96..aaa758d955fe4c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -64,6 +64,10 @@ def _get_exports_list(module): from posix import _have_functions except ImportError: pass + try: + from posix import _create_environ + except ImportError: + pass import posix __all__.extend(_get_exports_list(posix)) @@ -88,6 +92,10 @@ def _get_exports_list(module): from nt import _have_functions except ImportError: pass + try: + from nt import _create_environ + except ImportError: + pass else: raise ImportError('no os specific module found') @@ -281,6 +289,10 @@ def renames(old, new): __all__.extend(["makedirs", "removedirs", "renames"]) +# Private sentinel that makes walk() classify all symlinks and junctions as +# regular files. +_walk_symlinks_as_files = object() + def walk(top, topdown=True, onerror=None, followlinks=False): """Directory tree generator. @@ -361,58 +373,45 @@ def walk(top, topdown=True, onerror=None, followlinks=False): # minor reason when (say) a thousand readable directories are still # left to visit. try: - scandir_it = scandir(top) + with scandir(top) as entries: + for entry in entries: + try: + if followlinks is _walk_symlinks_as_files: + is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() + else: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) except OSError as error: if onerror is not None: onerror(error) continue - cont = False - with scandir_it: - while True: - try: - try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - cont = True - break - - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider the entry not to - # be a directory, same behaviour as os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) - - if not topdown and is_dir: - # Bottom-up: traverse into sub-directory, but exclude - # symlinks to directories if followlinks is False - if followlinks: - walk_into = True - else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider the - # entry not to be a symbolic link, same behaviour - # as os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - if cont: - continue - if topdown: # Yield before sub-directory traversal if going top down yield top, dirs, nondirs @@ -471,24 +470,59 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - if not follow_symlinks: - orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY, dir_fd=dir_fd) + stack = [(_fwalk_walk, (True, dir_fd, top, top, None))] + isbytes = isinstance(top, bytes) try: - if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and - path.samestat(orig_st, stat(topfd)))): - yield from _fwalk(topfd, top, isinstance(top, bytes), - topdown, onerror, follow_symlinks) + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) finally: - close(topfd) - - def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): + # Close any file descriptors still on the stack. + while stack: + action, value = stack.pop() + if action == _fwalk_close: + close(value) + + # Each item in the _fwalk() stack is a pair (action, args). + _fwalk_walk = 0 # args: (isroot, dirfd, toppath, topname, entry) + _fwalk_yield = 1 # args: (toppath, dirnames, filenames, topfd) + _fwalk_close = 2 # args: dirfd + + def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. + action, value = stack.pop() + if action == _fwalk_close: + close(value) + return + elif action == _fwalk_yield: + yield value + return + assert action == _fwalk_walk + isroot, dirfd, toppath, topname, entry = value + try: + if not follow_symlinks: + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + if entry is None: + orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd) + else: + orig_st = entry.stat(follow_symlinks=False) + topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) + except OSError as err: + if isroot: + raise + if onerror is not None: + onerror(err) + return + stack.append((_fwalk_close, topfd)) + if not follow_symlinks: + if isroot and not st.S_ISDIR(orig_st.st_mode): + return + if not path.samestat(orig_st, stat(topfd)): + return + scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -514,31 +548,18 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd + else: + stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) - for name in dirs if entries is None else zip(dirs, entries): - try: - if not follow_symlinks: - if topdown: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) - else: - assert entries is not None - name, entry = name - orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY, dir_fd=topfd) - except OSError as err: - if onerror is not None: - onerror(err) - continue - try: - if follow_symlinks or path.samestat(orig_st, stat(dirfd)): - dirpath = path.join(toppath, name) - yield from _fwalk(dirfd, dirpath, isbytes, - topdown, onerror, follow_symlinks) - finally: - close(dirfd) - - if not topdown: - yield toppath, dirs, nondirs, topfd + toppath = path.join(toppath, toppath[:0]) # Add trailing slash. + if entries is None: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, None)) + for name in dirs[::-1]) + else: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, entry)) + for name, entry in zip(dirs[::-1], entries[::-1])) __all__.append("fwalk") @@ -744,7 +765,18 @@ def __ror__(self, other): new.update(self) return new -def _createenviron(): + if _exists("_create_environ"): + def refresh(self): + data = _create_environ() + if name == 'nt': + data = {self.encodekey(key): value + for key, value in data.items()} + + # modify in-place to keep os.environb in sync + self._data.clear() + self._data.update(data) + +def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE def check_str(value): @@ -774,8 +806,8 @@ def decode(value): encode, decode) # unicode environ -environ = _createenviron() -del _createenviron +environ = _create_environ_mapping() +del _create_environ_mapping def getenv(key, default=None): diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 65ce836765c42b..5da3acd31997e5 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -5,871 +5,8 @@ operating systems. """ -import io -import ntpath -import os -import posixpath -import sys -import warnings -from itertools import chain -from _collections_abc import Sequence +from pathlib._abc import * +from pathlib._local import * -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - -from . import _abc - - -__all__ = [ - "UnsupportedOperation", - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - - -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_path', '_drv', '_root', '_tail') - - def __init__(self, path): - self._path = path - self._drv = path.drive - self._root = path.root - self._tail = path._tail - - def __len__(self): - return len(self._tail) - - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) - - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._path._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) - - def __repr__(self): - return "<{}.parents>".format(type(self._path).__name__) - - -UnsupportedOperation = _abc.UnsupportedOperation - - -class PurePath(_abc.PurePathBase): - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - - __slots__ = ( - # The `_raw_paths` slot stores unnormalized string paths. This is set - # in the `__init__()` method. - '_raw_paths', - - # The `_drv`, `_root` and `_tail_cached` slots store parsed and - # normalized parts of the path. They are set when any of the `drive`, - # `root` or `_tail` properties are accessed for the first time. The - # three-part division corresponds to the result of - # `os.path.splitroot()`, except that the tail is further split on path - # separators (i.e. it is a list of strings), and that the root and - # tail are normalized. - '_drv', '_root', '_tail_cached', - - # The `_str` slot stores the string representation of the path, - # computed from the drive, root and tail when `__str__()` is called - # for the first time. It's used to implement `_str_normcase` - '_str', - - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - ) - pathmod = os.path - - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def __init__(self, *args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - if arg.pathmod is ntpath and self.pathmod is posixpath: - # GH-103631: Convert separators for backwards compatibility. - paths.extend(path.replace('\\', '/') for path in arg._raw_paths) - else: - paths.extend(arg._raw_paths) - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - paths.append(path) - # Avoid calling super().__init__, as an optimisation - self._raw_paths = paths - - def joinpath(self, *pathsegments): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self.with_segments(self, *pathsegments) - - def __truediv__(self, key): - try: - return self.with_segments(self, key) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self.with_segments(key, self) - except TypeError: - return NotImplemented - - def __reduce__(self): - # Using the parts tuple helps share interned path parts - # when pickling related paths. - return (self.__class__, self.parts) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - - def __fspath__(self): - return str(self) - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - if _abc._is_case_sensitive(self.pathmod): - self._str_normcase_cached = str(self) - else: - self._str_normcase_cached = str(self).lower() - return self._str_normcase_cached - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) - return self._parts_normcase_cached - - def __lt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self.drive, self.root, - self._tail) or '.' - return self._str - - @classmethod - def _format_parsed_parts(cls, drv, root, tail): - if drv or root: - return drv + root + cls.pathmod.sep.join(tail) - elif tail and cls.pathmod.splitdrive(tail[0])[0]: - tail = ['.'] + tail - return cls.pathmod.sep.join(tail) - - def _from_parsed_parts(self, drv, root, tail): - path_str = self._format_parsed_parts(drv, root, tail) - path = self.with_segments(path_str) - path._str = path_str or '.' - path._drv = drv - path._root = root - path._tail_cached = tail - return path - - @classmethod - def _parse_path(cls, path): - if not path: - return '', '', [] - sep = cls.pathmod.sep - altsep = cls.pathmod.altsep - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = cls.pathmod.splitroot(path) - if not root and drv.startswith(sep) and not drv.endswith(sep): - drv_parts = drv.split(sep) - if len(drv_parts) == 4 and drv_parts[2] not in '?.': - # e.g. //server/share - root = sep - elif len(drv_parts) == 6: - # e.g. //?/unc/server/share - root = sep - parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] - return drv, root, parsed - - @property - def _raw_path(self): - """The joined but unnormalized path.""" - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self.pathmod.join(*paths) - return path - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - try: - return self._drv - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._drv - - @property - def root(self): - """The root of the path, if any.""" - try: - return self._root - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._root - - @property - def _tail(self): - try: - return self._tail_cached - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._tail_cached - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - return self.drive + self.root - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - if self.drive or self.root: - return (self.drive + self.root,) + tuple(self._tail) - else: - return tuple(self._tail) - - @property - def parent(self): - """The logical parent of the path.""" - drv = self.drive - root = self.root - tail = self._tail - if not tail: - return self - return self._from_parsed_parts(drv, root, tail[:-1]) - - @property - def parents(self): - """A sequence of this path's logical parents.""" - # The value of this property should not be cached on the path object, - # as doing so would introduce a reference cycle. - return _PathParents(self) - - @property - def name(self): - """The final path component, if any.""" - tail = self._tail - if not tail: - return '' - return tail[-1] - - def with_name(self, name): - """Return a new path with the file name changed.""" - m = self.pathmod - if not name or m.sep in name or (m.altsep and m.altsep in name) or name == '.': - raise ValueError(f"Invalid name {name!r}") - tail = self._tail.copy() - if not tail: - raise ValueError(f"{self!r} has an empty name") - tail[-1] = name - return self._from_parsed_parts(self.drive, self.root, tail) - - def relative_to(self, other, /, *_deprecated, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - for step, path in enumerate(chain([other], other.parents)): - if path == self or path in self.parents: - break - elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif path.name == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - parts = ['..'] * step + self._tail[len(path._tail):] - return self._from_parsed_parts('', '', parts) - - def is_relative_to(self, other, /, *_deprecated): - """Return True if the path is relative to another path or False. - """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - return other == self or other in self.parents - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - if self.pathmod is posixpath: - # Optimization: work with raw paths on POSIX. - for path in self._raw_paths: - if path.startswith('/'): - return True - return False - return self.pathmod.isabs(self) - - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " - "for removal in Python 3.15. Use os.path.isreserved() to " - "detect reserved paths on Windows.") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - if self.pathmod is ntpath: - return self.pathmod.isreserved(self) - return False - - def as_uri(self): - """Return the path as a URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - from urllib.parse import quote_from_bytes - return prefix + quote_from_bytes(os.fsencode(path)) - - @property - def _pattern_stack(self): - """Stack of path components, to be used with patterns in glob().""" - parts = self._tail.copy() - pattern = self._raw_path - if self.anchor: - raise NotImplementedError("Non-relative patterns are unsupported") - elif not parts: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - elif pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): - # GH-65238: pathlib doesn't preserve trailing slash. Add it back. - parts.append('') - parts.reverse() - return parts - - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" - # The string representation of an empty path is a single dot ('.'). Empty - # paths shouldn't match wildcards, so we change it to the empty string. - path_str = str(self) - return '' if path_str == '.' else path_str - -# Subclassing os.PathLike makes isinstance() checks slower, -# which in turn makes Path construction slower. Register instead! -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = ntpath - __slots__ = () - - -class Path(_abc.PathBase, PurePath): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - as_uri = PurePath.as_uri - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported on this system" - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def is_mount(self): - """ - Check if this path is a mount point - """ - return os.path.ismount(self) - - def is_junction(self): - """ - Whether this path is a junction. - """ - return os.path.isjunction(self) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def read_text(self, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, read it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return _abc.PathBase.read_text(self, encoding, errors, newline) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return _abc.PathBase.write_text(self, data, encoding, errors, newline) - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - return (self._make_child_relpath(name) for name in os.listdir(self)) - - def _scandir(self): - return os.scandir(self) - - def _make_child_entry(self, entry): - # Transform an entry yielded from _scandir() into a path object. - path_str = entry.name if str(self) == '.' else entry.path - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = self._tail + [entry.name] - return path - - def _make_child_relpath(self, name): - if not name: - return self - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{self.pathmod.sep}{name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - else: - path_str = name - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - return path - - def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - sys.audit("pathlib.Path.glob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - return _abc.PathBase.glob( - self, pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) - - def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - pattern = '**' / pattern - return _abc.PathBase.glob( - self, pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) - - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - return _abc.PathBase.walk( - self, top_down=top_down, on_error=on_error, follow_symlinks=follow_symlinks) - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - if self.root: - drive = os.path.splitroot(os.getcwd())[0] - return self._from_parsed_parts(drive, self.root, self._tail) - if self.drive: - # There is a CWD on each drive-letter drive. - cwd = os.path.abspath(self.drive) - else: - cwd = os.getcwd() - if not self._tail: - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - result = self.with_segments(cwd) - result._str = cwd - return result - drive, root, rel = os.path.splitroot(cwd) - if not rel: - return self._from_parsed_parts(drive, root, self._tail) - tail = rel.split(self.pathmod.sep) - tail.extend(self._tail) - return self._from_parsed_parts(drive, root, tail) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - return self.with_segments(os.path.realpath(self, strict=strict)) - - if pwd: - def owner(self, *, follow_symlinks=True): - """ - Return the login name of the file owner. - """ - uid = self.stat(follow_symlinks=follow_symlinks).st_uid - return pwd.getpwuid(uid).pw_name - - if grp: - def group(self, *, follow_symlinks=True): - """ - Return the group name of the file gid. - """ - gid = self.stat(follow_symlinks=follow_symlinks).st_gid - return grp.getgrgid(gid).gr_name - - if hasattr(os, "readlink"): - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - if hasattr(os, "symlink"): - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - os.symlink(target, self, target_is_directory) - - if hasattr(os, "link"): - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = os.path.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, tail + self._tail[1:]) - - return self - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - if not uri.startswith('file:'): - raise ValueError(f"URI does not start with 'file:': {uri!r}") - path = uri[5:] - if path[:3] == '///': - # Remove empty authority - path = path[2:] - elif path[:12] == '//localhost/': - # Remove 'localhost' authority - path = path[11:] - if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): - # Remove slash before DOS device/UNC path - path = path[1:] - if path[1:2] == '|': - # Replace bar with colon in DOS drive - path = path[:1] + ':' + path[2:] - from urllib.parse import unquote_to_bytes - path = cls(os.fsdecode(unquote_to_bytes(path))) - if not path.is_absolute(): - raise ValueError(f"URI is not absolute: {uri!r}") - return path - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") +__all__ = (_abc.__all__ + + _local.__all__) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index e4b1201a3703c3..f5eed6f025c250 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -12,138 +12,35 @@ """ import functools -from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL +import operator +import posixpath +from errno import EINVAL, EXDEV +from glob import _GlobberBase, _no_recurse_symlinks from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from pathlib._os import copyfileobj -# -# Internals -# -_WINERROR_NOT_READY = 21 # drive exists but is not accessible -_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 -_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself +__all__ = ["UnsupportedOperation"] -# EBADF - guard against macOS `stat` throwing EBADF -_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP) -_IGNORED_WINERRORS = ( - _WINERROR_NOT_READY, - _WINERROR_INVALID_NAME, - _WINERROR_CANT_RESOLVE_FILENAME) - -def _ignore_error(exception): - return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or - getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is attempted. + """ + pass @functools.cache -def _is_case_sensitive(pathmod): - return pathmod.normcase('Aa') == 'Aa' - -# -# Globbing helpers -# - -re = glob = None - - -@functools.lru_cache(maxsize=512) -def _compile_pattern(pat, sep, case_sensitive, recursive=True): - """Compile given glob pattern to a re.Pattern object (observing case - sensitivity).""" - global re, glob - if re is None: - import re, glob - - flags = re.NOFLAG if case_sensitive else re.IGNORECASE - regex = glob.translate(pat, recursive=recursive, include_hidden=True, seps=sep) - return re.compile(regex, flags=flags).match - - -def _select_special(paths, part): - """Yield special literal children of the given paths.""" - for path in paths: - yield path._make_child_relpath(part) - - -def _select_children(parent_paths, dir_only, follow_symlinks, match): - """Yield direct children of given paths, filtering by name and type.""" - if follow_symlinks is None: - follow_symlinks = True - for parent_path in parent_paths: - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with parent_path._scandir() as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - if dir_only: - try: - if not entry.is_dir(follow_symlinks=follow_symlinks): - continue - except OSError: - continue - if match(entry.name): - yield parent_path._make_child_entry(entry) - - -def _select_recursive(parent_paths, dir_only, follow_symlinks): - """Yield given paths and all their subdirectories, recursively.""" - if follow_symlinks is None: - follow_symlinks = False - for parent_path in parent_paths: - paths = [parent_path._make_child_relpath('')] - while paths: - path = paths.pop() - yield path - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with path._scandir() as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - try: - if entry.is_dir(follow_symlinks=follow_symlinks): - paths.append(path._make_child_entry(entry)) - continue - except OSError: - pass - if not dir_only: - yield path._make_child_entry(entry) - - -def _select_unique(paths): - """Yields the given paths, filtering out duplicates.""" - yielded = set() - try: - for path in paths: - path_str = str(path) - if path_str not in yielded: - yield path - yielded.add(path_str) - finally: - yielded.clear() - +def _is_case_sensitive(parser): + return parser.normcase('Aa') == 'Aa' -class UnsupportedOperation(NotImplementedError): - """An exception that is raised when an unsupported operation is called on - a path object. - """ - pass -class PathModuleBase: - """Base class for path modules, which do low-level path manipulation. +class ParserBase: + """Base class for path parsers, which do low-level path manipulation. - Path modules provide a subset of the os.path API, specifically those + Path parsers provide a subset of the os.path API, specifically those functions needed to provide PurePathBase functionality. Each PurePathBase - subclass references its path module via a 'pathmod' class attribute. + subclass references its path parser via a 'parser' class attribute. Every method in this base class raises an UnsupportedOperation exception. """ @@ -174,6 +71,12 @@ def splitdrive(self, path): drive. Either part may be empty.""" raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) + def splitext(self, path): + """Split the path into a pair (root, ext), where *ext* is empty or + begins with a period and contains at most one period, + and *root* is everything before the extension.""" + raise UnsupportedOperation(self._unsupported_msg('splitext()')) + def normcase(self, path): """Normalize the case of the path.""" raise UnsupportedOperation(self._unsupported_msg('normcase()')) @@ -184,6 +87,21 @@ def isabs(self, path): raise UnsupportedOperation(self._unsupported_msg('isabs()')) +class PathGlobber(_GlobberBase): + """ + Class providing shell-style globbing for path objects. + """ + + lexists = operator.methodcaller('exists', follow_symlinks=False) + add_slash = operator.methodcaller('joinpath', '') + scandir = operator.methodcaller('scandir') + + @staticmethod + def concat_path(path, text): + """Appends text to the given path.""" + return path.with_segments(path._raw_path + text) + + class PurePathBase: """Base class for pure path objects. @@ -203,10 +121,11 @@ class PurePathBase: # work from occurring when `resolve()` calls `stat()` or `readlink()`. '_resolving', ) - pathmod = PathModuleBase() + parser = ParserBase() + _globber = PathGlobber def __init__(self, path, *paths): - self._raw_path = self.pathmod.join(path, *paths) if paths else path + self._raw_path = self.parser.join(path, *paths) if paths else path if not isinstance(self._raw_path, str): raise TypeError( f"path should be a str, not {type(self._raw_path).__name__!r}") @@ -227,17 +146,17 @@ def __str__(self): def as_posix(self): """Return the string representation of the path with forward (/) slashes.""" - return str(self).replace(self.pathmod.sep, '/') + return str(self).replace(self.parser.sep, '/') @property def drive(self): """The drive prefix (letter or UNC path), if any.""" - return self.pathmod.splitdrive(self.anchor)[0] + return self.parser.splitdrive(self.anchor)[0] @property def root(self): """The root of the path, if any.""" - return self.pathmod.splitdrive(self.anchor)[1] + return self.parser.splitdrive(self.anchor)[1] @property def anchor(self): @@ -247,7 +166,7 @@ def anchor(self): @property def name(self): """The final path component, if any.""" - return self.pathmod.split(self._raw_path)[1] + return self.parser.split(self._raw_path)[1] @property def suffix(self): @@ -256,12 +175,7 @@ def suffix(self): This includes the leading period. For example: '.txt' """ - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[i:] - else: - return '' + return self.parser.splitext(self.name)[1] @property def suffixes(self): @@ -270,32 +184,36 @@ def suffixes(self): These include the leading periods. For example: ['.tar', '.gz'] """ - name = self.name - if name.endswith('.'): - return [] - name = name.lstrip('.') - return ['.' + suffix for suffix in name.split('.')[1:]] + split = self.parser.splitext + stem, suffix = split(self.name) + suffixes = [] + while suffix: + suffixes.append(suffix) + stem, suffix = split(stem) + return suffixes[::-1] @property def stem(self): """The final path component, minus its last suffix.""" - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[:i] - else: - return name + return self.parser.splitext(self.name)[0] def with_name(self, name): """Return a new path with the file name changed.""" - split = self.pathmod.split + split = self.parser.split if split(name)[0]: raise ValueError(f"Invalid name {name!r}") return self.with_segments(split(self._raw_path)[0], name) def with_stem(self, stem): """Return a new path with the stem changed.""" - return self.with_name(stem + self.suffix) + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) def with_suffix(self, suffix): """Return a new path with the file suffix changed. If the path @@ -303,14 +221,13 @@ def with_suffix(self, suffix): string, remove the suffix from the path. """ stem = self.stem - if not suffix: - return self.with_name(stem) - elif not stem: + if not stem: + # If the stem is empty, we can't make the suffix non-empty. raise ValueError(f"{self!r} has an empty name") - elif suffix.startswith('.') and len(suffix) > 1: - return self.with_name(stem + suffix) - else: + elif suffix and not suffix.startswith('.'): raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) def relative_to(self, other, *, walk_up=False): """Return the relative path to another path identified by the passed @@ -393,7 +310,7 @@ def _stack(self): uppermost parent of the path (equivalent to path.parents[-1]), and *parts* is a reversed list of parts following the anchor. """ - split = self.pathmod.split + split = self.parser.split path = self._raw_path parent, name = split(path) names = [] @@ -407,7 +324,7 @@ def _stack(self): def parent(self): """The logical parent of the path.""" path = self._raw_path - parent = self.pathmod.split(path)[0] + parent = self.parser.split(path)[0] if path != parent: parent = self.with_segments(parent) parent._resolving = self._resolving @@ -417,7 +334,7 @@ def parent(self): @property def parents(self): """A sequence of this path's logical parents.""" - split = self.pathmod.split + split = self.parser.split path = self._raw_path parent = split(path)[0] parents = [] @@ -430,15 +347,7 @@ def parents(self): def is_absolute(self): """True if the path is absolute (has both a root and, if applicable, a drive).""" - return self.pathmod.isabs(self._raw_path) - - @property - def _pattern_stack(self): - """Stack of path components, to be used with patterns in glob().""" - anchor, parts = self._stack - if anchor: - raise NotImplementedError("Non-relative patterns are unsupported") - return parts + return self.parser.isabs(self._raw_path) @property def _pattern_str(self): @@ -455,8 +364,8 @@ def match(self, path_pattern, *, case_sensitive=None): if not isinstance(path_pattern, PurePathBase): path_pattern = self.with_segments(path_pattern) if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.pathmod) - sep = path_pattern.pathmod.sep + case_sensitive = _is_case_sensitive(self.parser) + sep = path_pattern.parser.sep path_parts = self.parts[::-1] pattern_parts = path_pattern.parts[::-1] if not pattern_parts: @@ -465,8 +374,9 @@ def match(self, path_pattern, *, case_sensitive=None): return False if len(path_parts) > len(pattern_parts) and path_pattern.anchor: return False + globber = self._globber(sep, case_sensitive) for path_part, pattern_part in zip(path_parts, pattern_parts): - match = _compile_pattern(pattern_part, sep, case_sensitive, recursive=False) + match = globber.compile(pattern_part) if match(path_part) is None: return False return True @@ -479,8 +389,9 @@ def full_match(self, pattern, *, case_sensitive=None): if not isinstance(pattern, PurePathBase): pattern = self.with_segments(pattern) if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.pathmod) - match = _compile_pattern(pattern._pattern_str, pattern.pathmod.sep, case_sensitive) + case_sensitive = _is_case_sensitive(self.parser) + globber = self._globber(pattern.parser.sep, case_sensitive, recursive=True) + match = globber.compile(pattern._pattern_str) return match(self._pattern_str) is not None @@ -533,12 +444,7 @@ def exists(self, *, follow_symlinks=True): """ try: self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False return True @@ -548,14 +454,7 @@ def is_dir(self, *, follow_symlinks=True): """ try: return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_file(self, *, follow_symlinks=True): @@ -565,14 +464,7 @@ def is_file(self, *, follow_symlinks=True): """ try: return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_mount(self): @@ -601,13 +493,7 @@ def is_symlink(self): """ try: return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_junction(self): @@ -625,14 +511,7 @@ def is_block_device(self): """ try: return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_char_device(self): @@ -641,14 +520,7 @@ def is_char_device(self): """ try: return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_fifo(self): @@ -657,14 +529,7 @@ def is_fifo(self): """ try: return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_socket(self): @@ -673,14 +538,7 @@ def is_socket(self): """ try: return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def samefile(self, other_path): @@ -695,10 +553,43 @@ def samefile(self, other_path): return (st.st_ino == other_st.st_ino and st.st_dev == other_st.st_dev) + def _ensure_different_file(self, other_path): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + try: + if not self.samefile(other_path): + return + except (OSError, ValueError): + return + err = OSError(EINVAL, "Source and target are the same file") + err.filename = str(self) + err.filename2 = str(other_path) + raise err + + def _ensure_distinct_path(self, other_path): + """ + Raise OSError(EINVAL) if the other path is within this path. + """ + # Note: there is no straightforward, foolproof algorithm to determine + # if one directory is within another (a particularly perverse example + # would be a single network share mounted in one location via NFS, and + # in another location via CIFS), so we simply checks whether the + # other path is lexically equal to, or within, this path. + if self == other_path: + err = OSError(EINVAL, "Source and target are the same path") + elif self in other_path.parents: + err = OSError(EINVAL, "Source path is a parent of target path") + else: + return + err.filename = str(self) + err.filename2 = str(other_path) + raise err + def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ - Open the file pointed by this path and return a file object, as + Open the file pointed to by this path and return a file object, as the built-in open() function does. """ raise UnsupportedOperation(self._unsupported_msg('open()')) @@ -707,7 +598,7 @@ def read_bytes(self): """ Open the file in bytes mode, read it, and close the file. """ - with self.open(mode='rb') as f: + with self.open(mode='rb', buffering=0) as f: return f.read() def read_text(self, encoding=None, errors=None, newline=None): @@ -736,79 +627,50 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) - def iterdir(self): - """Yield path objects of the directory contents. + def scandir(self): + """Yield os.DirEntry objects of the directory contents. The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - raise UnsupportedOperation(self._unsupported_msg('iterdir()')) - - def _scandir(self): - # Emulate os.scandir(), which returns an object that can be used as a - # context manager. This method is called by walk() and glob(). - from contextlib import nullcontext - return nullcontext(self.iterdir()) + raise UnsupportedOperation(self._unsupported_msg('scandir()')) - def _make_child_entry(self, entry): - # Transform an entry yielded from _scandir() into a path object. - return entry + def iterdir(self): + """Yield path objects of the directory contents. - def _make_child_relpath(self, name): - return self.joinpath(name) + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + with self.scandir() as entries: + names = [entry.name for entry in entries] + return map(self.joinpath, names) - def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + def _glob_selector(self, parts, case_sensitive, recurse_symlinks): + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.parser) + case_pedantic = False + else: + # The user has expressed a case sensitivity choice, but we don't + # know the case sensitivity of the underlying filesystem, so we + # must use scandir() for everything, including non-wildcard parts. + case_pedantic = True + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) + return globber.selector(parts) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ if not isinstance(pattern, PurePathBase): pattern = self.with_segments(pattern) - if case_sensitive is None: - # TODO: evaluate case-sensitivity of each directory in _select_children(). - case_sensitive = _is_case_sensitive(self.pathmod) - - stack = pattern._pattern_stack - specials = ('', '.', '..') - filter_paths = False - deduplicate_paths = False - sep = self.pathmod.sep - paths = iter([self] if self.is_dir() else []) - while stack: - part = stack.pop() - if part in specials: - paths = _select_special(paths, part) - elif part == '**': - # Consume adjacent '**' components. - while stack and stack[-1] == '**': - stack.pop() - - # Consume adjacent non-special components and enable post-walk - # regex filtering, provided we're treating symlinks consistently. - if follow_symlinks is not None: - while stack and stack[-1] not in specials: - filter_paths = True - stack.pop() - - dir_only = bool(stack) - paths = _select_recursive(paths, dir_only, follow_symlinks) - if deduplicate_paths: - # De-duplicate if we've already seen a '**' component. - paths = _select_unique(paths) - deduplicate_paths = True - elif '**' in part: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - else: - dir_only = bool(stack) - match = _compile_pattern(part, sep, case_sensitive) - paths = _select_children(paths, dir_only, follow_symlinks, match) - if filter_paths: - # Filter out paths that don't match pattern. - prefix_len = len(str(self._make_child_relpath('_'))) - 1 - match = _compile_pattern(pattern._pattern_str, sep, case_sensitive) - paths = (path for path in paths if match(path._pattern_str, prefix_len)) - return paths - - def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + anchor, parts = pattern._stack + if anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + select = self._glob_selector(parts, case_sensitive, recurse_symlinks) + return select(self) + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): """Recursively yield all existing files (of any kind, including directories) matching the given relative pattern, anywhere in this subtree. @@ -816,52 +678,43 @@ def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): if not isinstance(pattern, PurePathBase): pattern = self.with_segments(pattern) pattern = '**' / pattern - return self.glob(pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) def walk(self, top_down=True, on_error=None, follow_symlinks=False): """Walk the directory tree from this directory, similar to os.walk().""" paths = [self] - while paths: path = paths.pop() if isinstance(path, tuple): yield path continue - - # We may not have read permission for self, in which case we can't - # get a list of the files the directory contains. os.walk() - # always suppressed the exception in that instance, rather than - # blow up for a minor reason when (say) a thousand readable - # directories are still left to visit. That logic is copied here. + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) try: - scandir_obj = path._scandir() + with path.scandir() as entries: + for entry in entries: + name = entry.name + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(path.joinpath(name)) + dirnames.append(name) + else: + filenames.append(name) + except OSError: + filenames.append(name) except OSError as error: if on_error is not None: on_error(error) - continue - - with scandir_obj as scandir_it: - dirnames = [] - filenames = [] if not top_down: - paths.append((path, dirnames, filenames)) - for entry in scandir_it: - try: - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - except OSError: - # Carried over from os.path.isdir(). - is_dir = False - - if is_dir: - if not top_down: - paths.append(path._make_child_entry(entry)) - dirnames.append(entry.name) - else: - filenames.append(entry.name) - + while not isinstance(paths.pop(), tuple): + pass + continue if top_down: yield path, dirnames, filenames - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] + paths += [path.joinpath(d) for d in reversed(dirnames)] def absolute(self): """Return an absolute version of this path @@ -906,65 +759,34 @@ def resolve(self, strict=False): """ if self._resolving: return self - path_root, parts = self._stack - path = self.with_segments(path_root) - try: - path = path.absolute() - except UnsupportedOperation: - path_tail = [] - else: - path_root, path_tail = path._stack - path_tail.reverse() - - # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported - # and (in non-strict mode) we can improve performance by not calling `stat()`. - querying = strict or getattr(self.readlink, '_supported', True) - link_count = 0 - while parts: - part = parts.pop() - if not part or part == '.': - continue - if part == '..': - if not path_tail: - if path_root: - # Delete '..' segment immediately following root - continue - elif path_tail[-1] != '..': - # Delete '..' segment and its predecessor - path_tail.pop() - continue - path_tail.append(part) - if querying and part != '..': - path = self.with_segments(path_root + self.pathmod.sep.join(path_tail)) + + def getcwd(): + return str(self.with_segments().absolute()) + + if strict or getattr(self.readlink, '_supported', True): + def lstat(path_str): + path = self.with_segments(path_str) path._resolving = True - try: - st = path.stat(follow_symlinks=False) - if S_ISLNK(st.st_mode): - # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are - # encountered during resolution. - link_count += 1 - if link_count >= self._max_symlinks: - raise OSError(ELOOP, "Too many symbolic links in path", self._raw_path) - target_root, target_parts = path.readlink()._stack - # If the symlink target is absolute (like '/etc/hosts'), set the current - # path to its uppermost parent (like '/'). - if target_root: - path_root = target_root - path_tail.clear() - else: - path_tail.pop() - # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to - # the stack of unresolved path parts. - parts.extend(target_parts) - continue - elif parts and not S_ISDIR(st.st_mode): - raise NotADirectoryError(ENOTDIR, "Not a directory", self._raw_path) - except OSError: - if strict: - raise - else: - querying = False - return self.with_segments(path_root + self.pathmod.sep.join(path_tail)) + return path.lstat() + + def readlink(path_str): + path = self.with_segments(path_str) + path._resolving = True + return str(path.readlink()) + else: + # If the user has *not* overridden the `readlink()` method, then + # symlinks are unsupported and (in non-strict mode) we can improve + # performance by not calling `path.lstat()`. + def skip(path_str): + # This exception will be internally consumed by `_realpath()`. + raise OSError("Operation skipped.") + + lstat = readlink = skip + + return self.with_segments(posixpath._realpath( + str(self), strict, self.parser.sep, + getcwd=getcwd, lstat=lstat, readlink=readlink, + maxlinks=self._max_symlinks)) def symlink_to(self, target, target_is_directory=False): """ @@ -973,6 +795,13 @@ def symlink_to(self, target, target_is_directory=False): """ raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) + def _symlink_to_target_of(self, link): + """ + Make this path a symlink with the same target as the given link. This + is used by copy(). + """ + self.symlink_to(link.readlink()) + def hardlink_to(self, target): """ Make this path a hard link pointing to the same file as *target*. @@ -993,6 +822,92 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ raise UnsupportedOperation(self._unsupported_msg('mkdir()')) + # Metadata keys supported by this path type. + _readable_metadata = _writable_metadata = frozenset() + + def _read_metadata(self, keys=None, *, follow_symlinks=True): + """ + Returns path metadata as a dict with string keys. + """ + raise UnsupportedOperation(self._unsupported_msg('_read_metadata()')) + + def _write_metadata(self, metadata, *, follow_symlinks=True): + """ + Sets path metadata from the given dict with string keys. + """ + raise UnsupportedOperation(self._unsupported_msg('_write_metadata()')) + + def _copy_metadata(self, target, *, follow_symlinks=True): + """ + Copies metadata (permissions, timestamps, etc) from this path to target. + """ + # Metadata types supported by both source and target. + keys = self._readable_metadata & target._writable_metadata + if keys: + metadata = self._read_metadata(keys, follow_symlinks=follow_symlinks) + target._write_metadata(metadata, follow_symlinks=follow_symlinks) + + def _copy_file(self, target): + """ + Copy the contents of this file to the given target. + """ + self._ensure_different_file(target) + with self.open('rb') as source_f: + try: + with target.open('wb') as target_f: + copyfileobj(source_f, target_f) + except IsADirectoryError as e: + if not target.exists(): + # Raise a less confusing exception. + raise FileNotFoundError( + f'Directory does not exist: {target}') from e + else: + raise + + def copy(self, target, *, follow_symlinks=True, dirs_exist_ok=False, + preserve_metadata=False): + """ + Recursively copy this file or directory tree to the given destination. + """ + if not isinstance(target, PathBase): + target = self.with_segments(target) + self._ensure_distinct_path(target) + stack = [(self, target)] + while stack: + src, dst = stack.pop() + if not follow_symlinks and src.is_symlink(): + dst._symlink_to_target_of(src) + if preserve_metadata: + src._copy_metadata(dst, follow_symlinks=False) + elif src.is_dir(): + children = src.iterdir() + dst.mkdir(exist_ok=dirs_exist_ok) + stack.extend((child, dst.joinpath(child.name)) + for child in children) + if preserve_metadata: + src._copy_metadata(dst) + else: + src._copy_file(dst) + if preserve_metadata: + src._copy_metadata(dst) + return target + + def copy_into(self, target_dir, *, follow_symlinks=True, + dirs_exist_ok=False, preserve_metadata=False): + """ + Copy this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif isinstance(target_dir, PathBase): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.copy(target, follow_symlinks=follow_symlinks, + dirs_exist_ok=dirs_exist_ok, + preserve_metadata=preserve_metadata) + def rename(self, target): """ Rename this path to the target path. @@ -1017,6 +932,38 @@ def replace(self, target): """ raise UnsupportedOperation(self._unsupported_msg('replace()')) + def move(self, target): + """ + Recursively move this file or directory tree to the given destination. + """ + self._ensure_different_file(target) + try: + return self.replace(target) + except UnsupportedOperation: + pass + except TypeError: + if not isinstance(target, PathBase): + raise + except OSError as err: + if err.errno != EXDEV: + raise + target = self.copy(target, follow_symlinks=False, preserve_metadata=True) + self._delete() + return target + + def move_into(self, target_dir): + """ + Move this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif isinstance(target_dir, PathBase): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.move(target) + def chmod(self, mode, *, follow_symlinks=True): """ Change the permissions of the path, like os.chmod(). @@ -1043,6 +990,30 @@ def rmdir(self): """ raise UnsupportedOperation(self._unsupported_msg('rmdir()')) + def _delete(self): + """ + Delete this file or directory (including all sub-directories). + """ + if self.is_symlink() or self.is_junction(): + self.unlink() + elif self.is_dir(): + self._rmtree() + else: + self.unlink() + + def _rmtree(self): + def on_error(err): + raise err + results = self.walk( + on_error=on_error, + top_down=False, # So we rmdir() empty directories. + follow_symlinks=False) + for dirpath, _, filenames in results: + for filename in filenames: + filepath = dirpath / filename + filepath.unlink() + dirpath.rmdir() + def owner(self, *, follow_symlinks=True): """ Return the login name of the file owner. diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py new file mode 100644 index 00000000000000..ef072b83d96904 --- /dev/null +++ b/Lib/pathlib/_local.py @@ -0,0 +1,950 @@ +import io +import ntpath +import operator +import os +import posixpath +import sys +from glob import _StringGlobber +from itertools import chain +from _collections_abc import Sequence + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from pathlib._os import (copyfile, file_metadata_keys, read_file_metadata, + write_file_metadata) +from pathlib._abc import UnsupportedOperation, PurePathBase, PathBase + + +__all__ = [ + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class PurePath(PurePathBase): + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unnormalized string paths. This is set + # in the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + parser = os.path + _globber = _StringGlobber + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.parser is not self.parser: + # GH-103631: Convert separators for backwards compatibility. + paths.append(arg.as_posix()) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + # Avoid calling super().__init__, as an optimisation + self._raw_paths = paths + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def __reduce__(self): + return self.__class__, tuple(self._raw_paths) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if self.parser is posixpath: + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.parser is other.parser + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.parser.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + def _from_parsed_string(self, path_str): + path = self.with_segments(path_str) + path._str = path_str or '.' + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.parser.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + return drv, root, [x for x in rel.split(sep) if x and x != '.'] + + @property + def _raw_path(self): + """The joined but unnormalized path.""" + paths = self._raw_paths + if len(paths) == 0: + path = '' + elif len(paths) == 1: + path = paths[0] + else: + path = self.parser.join(*paths) + return path + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if i != -1: + stem = name[:i] + # Stem must contain at least one non-dot character. + if stem.lstrip('.'): + return stem + return name + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name.lstrip('.') + i = name.rfind('.') + if i != -1: + return name[i:] + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + return ['.' + ext for ext in self.name.lstrip('.').split('.')[1:]] + + def relative_to(self, other, *, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if not isinstance(other, PurePath): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other): + """Return True if the path is relative to another path or False. + """ + if not isinstance(other, PurePath): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.parser is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.parser.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + import warnings + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + if self.parser is ntpath: + return self.parser.isreserved(self) + return False + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path_str = str(self) + return '' if path_str == '.' else path_str + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = ntpath + __slots__ = () + + +class Path(PathBase, PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + as_uri = PurePath.as_uri + + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported on this system" + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + if follow_symlinks: + return os.path.exists(self) + return os.path.lexists(self) + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + if follow_symlinks: + return os.path.isdir(self) + return PathBase.is_dir(self, follow_symlinks=follow_symlinks) + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + if follow_symlinks: + return os.path.isfile(self) + return PathBase.is_file(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + return os.path.islink(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.read_text(self, encoding, errors, newline) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.write_text(self, data, encoding, errors, newline) + + _remove_leading_dot = operator.itemgetter(slice(2, None)) + _remove_trailing_slash = operator.itemgetter(slice(-1)) + + def _filter_trailing_slash(self, paths): + sep = self.parser.sep + anchor_len = len(self.anchor) + for path_str in paths: + if len(path_str) > anchor_len and path_str[-1] == sep: + path_str = path_str[:-1] + yield path_str + + def scandir(self): + """Yield os.DirEntry objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return os.scandir(self) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + root_dir = str(self) + with os.scandir(root_dir) as scandir_it: + paths = [entry.path for entry in scandir_it] + if root_dir == '.': + paths = map(self._remove_leading_dot, paths) + return map(self._from_parsed_string, paths) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + if pattern.anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + parts = pattern._tail.copy() + if not parts: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + raw = pattern._raw_path + if raw[-1] in (self.parser.sep, self.parser.altsep): + # GH-65238: pathlib doesn't preserve trailing slash. Add it back. + parts.append('') + select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) + root = str(self) + paths = select(root) + + # Normalize results + if root == '.': + paths = map(self._remove_leading_dot, paths) + if parts[-1] == '': + paths = map(self._remove_trailing_slash, paths) + elif parts[-1] == '**': + paths = self._filter_trailing_slash(paths) + paths = map(self._from_parsed_string, paths) + return paths + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + pattern = '**' / pattern + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + root_dir = str(self) + if not follow_symlinks: + follow_symlinks = os._walk_symlinks_as_files + results = os.walk(root_dir, top_down, on_error, follow_symlinks) + for path_str, dirnames, filenames in results: + if root_dir == '.': + path_str = path_str[2:] + yield self._from_parsed_string(path_str), dirnames, filenames + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + return self._from_parsed_string(cwd) + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.parser.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + _readable_metadata = _writable_metadata = file_metadata_keys + _read_metadata = read_file_metadata + _write_metadata = write_file_metadata + + if copyfile: + def _copy_file(self, target): + """ + Copy the contents of this file to the given target. + """ + try: + target = os.fspath(target) + except TypeError: + if not isinstance(target, PathBase): + raise + PathBase._copy_file(self, target) + else: + copyfile(os.fspath(self), target) + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def _rmtree(self): + # Lazy import to improve module import time + import shutil + shutil.rmtree(self) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + return self.with_segments(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if os.name == 'nt': + def _symlink_to_target_of(self, link): + """ + Make this path a symlink with the same target as the given link. + This is used by copy(). + """ + self.symlink_to(link.readlink(), link.is_dir()) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py new file mode 100644 index 00000000000000..642b3a57c59a1d --- /dev/null +++ b/Lib/pathlib/_os.py @@ -0,0 +1,262 @@ +""" +Low-level OS functionality wrappers used by pathlib. +""" + +from errno import * +import os +import stat +import sys +try: + import fcntl +except ImportError: + fcntl = None +try: + import posix +except ImportError: + posix = None +try: + import _winapi +except ImportError: + _winapi = None + + +def _get_copy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + + +if fcntl and hasattr(fcntl, 'FICLONE'): + def _ficlone(source_fd, target_fd): + """ + Perform a lightweight copy of two files, where the data blocks are + copied only when modified. This is known as Copy on Write (CoW), + instantaneous copy or reflink. + """ + fcntl.ioctl(target_fd, fcntl.FICLONE, source_fd) +else: + _ficlone = None + + +if posix and hasattr(posix, '_fcopyfile'): + def _fcopyfile(source_fd, target_fd): + """ + Copy a regular file content using high-performance fcopyfile(3) + syscall (macOS). + """ + posix._fcopyfile(source_fd, target_fd, posix._COPYFILE_DATA) +else: + _fcopyfile = None + + +if hasattr(os, 'copy_file_range'): + def _copy_file_range(source_fd, target_fd): + """ + Copy data from one regular mmap-like fd to another by using a + high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side + copy. + This should work on Linux >= 4.5 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.copy_file_range(source_fd, target_fd, blocksize, + offset_dst=offset) + if sent == 0: + break # EOF + offset += sent +else: + _copy_file_range = None + + +if hasattr(os, 'sendfile'): + def _sendfile(source_fd, target_fd): + """Copy data from one regular mmap-like fd to another by using + high-performance sendfile(2) syscall. + This should work on Linux >= 2.6.33 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.sendfile(target_fd, source_fd, offset, blocksize) + if sent == 0: + break # EOF + offset += sent +else: + _sendfile = None + + +if _winapi and hasattr(_winapi, 'CopyFile2'): + def copyfile(source, target): + """ + Copy from one file to another using CopyFile2 (Windows only). + """ + _winapi.CopyFile2(source, target, 0) +else: + copyfile = None + + +def copyfileobj(source_f, target_f): + """ + Copy data from file-like object source_f to file-like object target_f. + """ + try: + source_fd = source_f.fileno() + target_fd = target_f.fileno() + except Exception: + pass # Fall through to generic code. + else: + try: + # Use OS copy-on-write where available. + if _ficlone: + try: + _ficlone(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EBADF, EOPNOTSUPP, ETXTBSY, EXDEV): + raise err + + # Use OS copy where available. + if _fcopyfile: + try: + _fcopyfile(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EINVAL, ENOTSUP): + raise err + if _copy_file_range: + try: + _copy_file_range(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (ETXTBSY, EXDEV): + raise err + if _sendfile: + try: + _sendfile(source_fd, target_fd) + return + except OSError as err: + if err.errno != ENOTSOCK: + raise err + except OSError as err: + # Produce more useful error messages. + err.filename = source_f.name + err.filename2 = target_f.name + raise err + + # Last resort: copy with fileobj read() and write(). + read_source = source_f.read + write_target = target_f.write + while buf := read_source(1024 * 1024): + write_target(buf) + + +# Kinds of metadata supported by the operating system. +file_metadata_keys = {'mode', 'times_ns'} +if hasattr(os.stat_result, 'st_flags'): + file_metadata_keys.add('flags') +if hasattr(os, 'listxattr'): + file_metadata_keys.add('xattrs') +file_metadata_keys = frozenset(file_metadata_keys) + + +def read_file_metadata(path, keys=None, *, follow_symlinks=True): + """ + Returns local path metadata as a dict with string keys. + """ + if keys is None: + keys = file_metadata_keys + assert keys.issubset(file_metadata_keys) + result = {} + for key in keys: + if key == 'xattrs': + try: + result['xattrs'] = [ + (attr, os.getxattr(path, attr, follow_symlinks=follow_symlinks)) + for attr in os.listxattr(path, follow_symlinks=follow_symlinks)] + except OSError as err: + if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + continue + st = os.stat(path, follow_symlinks=follow_symlinks) + if key == 'mode': + result['mode'] = stat.S_IMODE(st.st_mode) + elif key == 'times_ns': + result['times_ns'] = st.st_atime_ns, st.st_mtime_ns + elif key == 'flags': + result['flags'] = st.st_flags + return result + + +def write_file_metadata(path, metadata, *, follow_symlinks=True): + """ + Sets local path metadata from the given dict with string keys. + """ + assert frozenset(metadata.keys()).issubset(file_metadata_keys) + + def _nop(*args, ns=None, follow_symlinks=None): + pass + + if follow_symlinks: + # use the real function if it exists + def lookup(name): + return getattr(os, name, _nop) + else: + # use the real function only if it exists + # *and* it supports follow_symlinks + def lookup(name): + fn = getattr(os, name, _nop) + if fn in os.supports_follow_symlinks: + return fn + return _nop + + times_ns = metadata.get('times_ns') + if times_ns is not None: + lookup("utime")(path, ns=times_ns, follow_symlinks=follow_symlinks) + # We must copy extended attributes before the file is (potentially) + # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. + xattrs = metadata.get('xattrs') + if xattrs is not None: + for attr, value in xattrs: + try: + os.setxattr(path, attr, value, follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + mode = metadata.get('mode') + if mode is not None: + try: + lookup("chmod")(path, mode, follow_symlinks=follow_symlinks) + except NotImplementedError: + # if we got a NotImplementedError, it's because + # * follow_symlinks=False, + # * lchown() is unavailable, and + # * either + # * fchownat() is unavailable or + # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. + # (it returned ENOSUP.) + # therefore we're out of options--we simply cannot chown the + # symlink. give up, suppress the error. + # (which is what shutil always did in this circumstance.) + pass + flags = metadata.get('flags') + if flags is not None: + try: + lookup("chflags")(path, flags, follow_symlinks=follow_symlinks) + except OSError as why: + if why.errno not in (EOPNOTSUPP, ENOTSUP): + raise diff --git a/Lib/pdb.py b/Lib/pdb.py old mode 100755 new mode 100644 index 0754e8b628cf57..b7f6fd4323407e --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """ The Python Debugger Pdb ======================= @@ -77,17 +75,21 @@ import code import glob import token +import types import codeop import pprint import signal import inspect +import textwrap import tokenize -import functools +import itertools import traceback import linecache +import _colorize from contextlib import contextmanager -from typing import Union +from rlcompleter import Completer +from types import CodeType class Restart(Exception): @@ -116,13 +118,16 @@ def find_first_executable_line(code): return code.co_firstlineno def find_function(funcname, filename): - cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname)) + cre = re.compile(r'def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname)) try: fp = tokenize.open(filename) except OSError: - return None + lines = linecache.getlines(filename) + if not lines: + return None + fp = io.StringIO(''.join(lines)) funcdef = "" - funcstart = None + funcstart = 0 # consumer of this info expects the first line to be 1 with fp: for lineno, line in enumerate(fp, start=1): @@ -133,9 +138,12 @@ def find_function(funcname, filename): if funcdef: try: - funccode = compile(funcdef, filename, 'exec').co_consts[0] + code = compile(funcdef, filename, 'exec') except SyntaxError: continue + # We should always be able to find the code object here + funccode = next(c for c in code.co_consts if + isinstance(c, CodeType) and c.co_name == funcname) lineno_offset = find_first_executable_line(funccode) return funcname, filename, funcstart + lineno_offset - 1 return None @@ -155,51 +163,58 @@ def __repr__(self): return self -class _ScriptTarget(str): - def __new__(cls, val): - # Mutate self to be the "real path". - res = super().__new__(cls, os.path.realpath(val)) +class _ExecutableTarget: + filename: str + code: CodeType | str + namespace: dict - # Store the original path for error reporting. - res.orig = val - return res +class _ScriptTarget(_ExecutableTarget): + def __init__(self, target): + self._target = os.path.realpath(target) - def check(self): - if not os.path.exists(self): - print('Error:', self.orig, 'does not exist') + if not os.path.exists(self._target): + print(f'Error: {target} does not exist') sys.exit(1) - if os.path.isdir(self): - print('Error:', self.orig, 'is a directory') + if os.path.isdir(self._target): + print(f'Error: {target} is a directory') sys.exit(1) # If safe_path(-P) is not set, sys.path[0] is the directory # of pdb, and we should replace it with the directory of the script if not sys.flags.safe_path: - sys.path[0] = os.path.dirname(self) + sys.path[0] = os.path.dirname(self._target) + + def __repr__(self): + return self._target @property def filename(self): - return self + return self._target + + @property + def code(self): + # Open the file each time because the file may be modified + with io.open_code(self._target) as fp: + return f"exec(compile({fp.read()!r}, {self._target!r}, 'exec'))" @property def namespace(self): return dict( __name__='__main__', - __file__=self, + __file__=self._target, __builtins__=__builtins__, + __spec__=None, ) - @property - def code(self): - with io.open_code(self) as fp: - return f"exec(compile({fp.read()!r}, {self!r}, 'exec'))" +class _ModuleTarget(_ExecutableTarget): + def __init__(self, target): + self._target = target -class _ModuleTarget(str): - def check(self): + import runpy try: - self._details + _, self._spec, self._code = runpy._get_module_details(self._target) except ImportError as e: print(f"ImportError: {e}") sys.exit(1) @@ -207,24 +222,54 @@ def check(self): traceback.print_exc() sys.exit(1) - @functools.cached_property - def _details(self): - import runpy - return runpy._get_module_details(self) + def __repr__(self): + return self._target @property def filename(self): - return self.code.co_filename + return self._code.co_filename @property def code(self): - name, spec, code = self._details - return code + return self._code @property - def _spec(self): - name, spec, code = self._details - return spec + def namespace(self): + return dict( + __name__='__main__', + __file__=os.path.normcase(os.path.abspath(self.filename)), + __package__=self._spec.parent, + __loader__=self._spec.loader, + __spec__=self._spec, + __builtins__=__builtins__, + ) + + +class _ZipTarget(_ExecutableTarget): + def __init__(self, target): + import runpy + + self._target = os.path.realpath(target) + sys.path.insert(0, self._target) + try: + _, self._spec, self._code = runpy._get_main_module_details() + except ImportError as e: + print(f"ImportError: {e}") + sys.exit(1) + except Exception: + traceback.print_exc() + sys.exit(1) + + def __repr__(self): + return self._target + + @property + def filename(self): + return self._code.co_filename + + @property + def code(self): + return self._code @property def namespace(self): @@ -265,8 +310,10 @@ class Pdb(bdb.Bdb, cmd.Cmd): _file_mtime_table = {} + _last_pdb_instance = None + def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, - nosigint=False, readrc=True): + nosigint=False, readrc=True, mode=None): bdb.Bdb.__init__(self, skip=skip) cmd.Cmd.__init__(self, completekey, stdin, stdout) sys.audit("pdb.Pdb") @@ -278,6 +325,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.mainpyfile = '' self._wait_for_mainpyfile = False self.tb_lineno = {} + self.mode = mode # Try to load readline if it exists try: import readline @@ -306,10 +354,6 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, pass self.commands = {} # associates a command list to breakpoint numbers - self.commands_doprompt = {} # for each bp num, tells if the prompt - # must be disp. after execing the cmd list - self.commands_silent = {} # for each bp num, tells if the stack trace - # must be disp. after execing the cmd list self.commands_defining = False # True while in the process of defining # a command list self.commands_bnum = None # The breakpoint number for which we are @@ -318,6 +362,16 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self._chained_exceptions = tuple() self._chained_exception_index = 0 + def set_trace(self, frame=None, *, commands=None): + Pdb._last_pdb_instance = self + if frame is None: + frame = sys._getframe().f_back + + if commands is not None: + self.rcLines.extend(commands) + + super().set_trace(frame) + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt @@ -349,12 +403,10 @@ def setup(self, f, tb): self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] - # The f_locals dictionary is updated from the actual frame - # locals whenever the .f_locals accessor is called, so we - # cache it here to ensure that modifications are not overwritten. - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) + self._save_initial_file_mtime(self.curframe) + if self._chained_exceptions: self.set_convenience_variable( self.curframe, @@ -362,26 +414,12 @@ def setup(self, f, tb): self._chained_exceptions[self._chained_exception_index], ) - return self.execRcLines() - - # Can be executed earlier than 'setup' if desired - def execRcLines(self): - if not self.rcLines: - return - # local copy because of recursion - rcLines = self.rcLines - rcLines.reverse() - # execute every line only once - self.rcLines = [] - while rcLines: - line = rcLines.pop().strip() - if line and line[0] != '#': - if self.onecmd(line): - # if onecmd returns True, the command wants to exit - # from the interaction, save leftover rc lines - # to execute before next interaction - self.rcLines += reversed(rcLines) - return True + if self.rcLines: + self.cmdqueue = [ + line for line in self.rcLines + if line.strip() and not line.strip().startswith("#") + ] + self.rcLines = [] # Override Bdb methods @@ -397,12 +435,13 @@ def user_call(self, frame, argument_list): def user_line(self, frame): """This function is called when we stop or break at this line.""" if self._wait_for_mainpyfile: - if (self.mainpyfile != self.canonic(frame.f_code.co_filename) - or frame.f_lineno <= 0): + if (self.mainpyfile != self.canonic(frame.f_code.co_filename)): return self._wait_for_mainpyfile = False - if self.bp_commands(frame): - self.interaction(frame, None) + self.bp_commands(frame) + self.interaction(frame, None) + + user_opcode = user_line def bp_commands(self, frame): """Call every command that was set for the current active breakpoint @@ -415,18 +454,9 @@ def bp_commands(self, frame): self.currentbp in self.commands: currentbp = self.currentbp self.currentbp = 0 - lastcmd_back = self.lastcmd - self.setup(frame, None) for line in self.commands[currentbp]: - self.onecmd(line) - self.lastcmd = lastcmd_back - if not self.commands_silent[currentbp]: - self.print_stack_entry(self.stack[self.curindex]) - if self.commands_doprompt[currentbp]: - self._cmdloop() - self.forget() - return - return 1 + self.cmdqueue.append(line) + self.cmdqueue.append(f'_pdbcmd_restore_lastcmd {self.lastcmd}') def user_return(self, frame, return_value): """This function is called when a return trap is set here.""" @@ -469,9 +499,21 @@ def _cmdloop(self): except KeyboardInterrupt: self.message('--KeyboardInterrupt--') + def _save_initial_file_mtime(self, frame): + """save the mtime of the all the files in the frame stack in the file mtime table + if they haven't been saved yet.""" + while frame: + filename = frame.f_code.co_filename + if filename not in self._file_mtime_table: + try: + self._file_mtime_table[filename] = os.path.getmtime(filename) + except Exception: + pass + frame = frame.f_back + def _validate_file_mtime(self): - """Check if the source file of the current frame has been modified since - the last time we saw it. If so, give a warning.""" + """Check if the source file of the current frame has been modified. + If so, give a warning and reset the modify time to current.""" try: filename = self.curframe.f_code.co_filename mtime = os.path.getmtime(filename) @@ -481,11 +523,11 @@ def _validate_file_mtime(self): mtime != self._file_mtime_table[filename]): self.message(f"*** WARNING: file '{filename}' was edited, " "running stale code until the program is rerun") - self._file_mtime_table[filename] = mtime + self._file_mtime_table[filename] = mtime # Called before loop, handles display expressions # Set up convenience variable containers - def preloop(self): + def _show_display(self): displaying = self.displaying.get(self.curframe) if displaying: for expr, oldvalue in displaying.items(): @@ -570,13 +612,17 @@ def interaction(self, frame, tb_or_exc): if isinstance(tb_or_exc, BaseException): assert tb is not None, "main exception must have a traceback" with self._hold_exceptions(_chained_exceptions): - if self.setup(frame, tb): - # no interaction desired at this time (happens if .pdbrc contains - # a command like "continue") - self.forget() - return - self.print_stack_entry(self.stack[self.curindex]) + self.setup(frame, tb) + # We should print the stack entry if and only if the user input + # is expected, and we should print it right before the user input. + # We achieve this by appending _pdbcmd_print_frame_status to the + # command queue. If cmdqueue is not exhausted, the user input is + # not expected and we will not print the stack entry. + self.cmdqueue.append('_pdbcmd_print_frame_status') self._cmdloop() + # If _pdbcmd_print_frame_status is not used, pop it out + if self.cmdqueue and self.cmdqueue[-1] == '_pdbcmd_print_frame_status': + self.cmdqueue.pop() self.forget() def displayhook(self, obj): @@ -588,29 +634,108 @@ def displayhook(self, obj): self.message(repr(obj)) @contextmanager - def _disable_tab_completion(self): - if self.use_rawinput and self.completekey == 'tab': - try: - import readline - except ImportError: - yield - return - try: - readline.parse_and_bind('tab: self-insert') - yield - finally: - readline.parse_and_bind('tab: complete') - else: + def _disable_command_completion(self): + completenames = self.completenames + try: + self.completenames = self.completedefault yield + finally: + self.completenames = completenames + return + + def _exec_in_closure(self, source, globals, locals): + """ Run source code in closure so code object created within source + can find variables in locals correctly + + returns True if the source is executed, False otherwise + """ + + # Determine if the source should be executed in closure. Only when the + # source compiled to multiple code objects, we should use this feature. + # Otherwise, we can just raise an exception and normal exec will be used. + + code = compile(source, "", "exec") + if not any(isinstance(const, CodeType) for const in code.co_consts): + return False + + # locals could be a proxy which does not support pop + # copy it first to avoid modifying the original locals + locals_copy = dict(locals) + + locals_copy["__pdb_eval__"] = { + "result": None, + "write_back": {} + } + + # If the source is an expression, we need to print its value + try: + compile(source, "", "eval") + except SyntaxError: + pass + else: + source = "__pdb_eval__['result'] = " + source + + # Add write-back to update the locals + source = ("try:\n" + + textwrap.indent(source, " ") + "\n" + + "finally:\n" + + " __pdb_eval__['write_back'] = locals()") + + # Build a closure source code with freevars from locals like: + # def __pdb_outer(): + # var = None + # def __pdb_scope(): # This is the code object we want to execute + # nonlocal var + # + # return __pdb_scope.__code__ + source_with_closure = ("def __pdb_outer():\n" + + "\n".join(f" {var} = None" for var in locals_copy) + "\n" + + " def __pdb_scope():\n" + + "\n".join(f" nonlocal {var}" for var in locals_copy) + "\n" + + textwrap.indent(source, " ") + "\n" + + " return __pdb_scope.__code__" + ) + + # Get the code object of __pdb_scope() + # The exec fills locals_copy with the __pdb_outer() function and we can call + # that to get the code object of __pdb_scope() + ns = {} + try: + exec(source_with_closure, {}, ns) + except Exception: + return False + code = ns["__pdb_outer"]() + + cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars) + + try: + exec(code, globals, locals_copy, closure=cells) + except Exception: + return False + + # get the data we need from the statement + pdb_eval = locals_copy["__pdb_eval__"] + + # __pdb_eval__ should not be updated back to locals + pdb_eval["write_back"].pop("__pdb_eval__") + + # Write all local variables back to locals + locals.update(pdb_eval["write_back"]) + eval_result = pdb_eval["result"] + if eval_result is not None: + print(repr(eval_result)) + + return True def default(self, line): if line[:1] == '!': line = line[1:].strip() - locals = self.curframe_locals + locals = self.curframe.f_locals globals = self.curframe.f_globals try: + buffer = line if (code := codeop.compile_command(line + '\n', '', 'single')) is None: # Multi-line mode - with self._disable_tab_completion(): + with self._disable_command_completion(): buffer = line continue_prompt = "... " while (code := codeop.compile_command(buffer, '', 'single')) is None: @@ -633,6 +758,7 @@ def default(self, line): else: line = line.rstrip('\r\n') buffer += '\n' + line + self.lastcmd = buffer save_stdout = sys.stdout save_stdin = sys.stdin save_displayhook = sys.displayhook @@ -640,7 +766,8 @@ def default(self, line): sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook - exec(code, globals, locals) + if not self._exec_in_closure(buffer, globals, locals): + exec(code, globals, locals) finally: sys.stdout = save_stdout sys.stdin = save_stdin @@ -656,7 +783,7 @@ def _replace_convenience_variables(self, line): if "$" not in line: return line - dollar_start = dollar_end = -1 + dollar_start = dollar_end = (-1, -1) replace_variables = [] try: for t in tokenize.generate_tokens(io.StringIO(line).readline): @@ -711,7 +838,7 @@ def precmd(self, line): if marker >= 0: # queue up everything after marker next = line[marker+2:].lstrip() - self.cmdqueue.append(next) + self.cmdqueue.insert(0, next) line = line[:marker].rstrip() # Replace all the convenience variables @@ -727,7 +854,10 @@ def onecmd(self, line): a breakpoint command list definition. """ if not self.commands_defining: - self._validate_file_mtime() + if line.startswith('_pdbcmd'): + command, arg, line = self.parseline(line) + if hasattr(self, command): + return getattr(self, command)(arg) return cmd.Cmd.onecmd(self, line) else: return self.handle_command_def(line) @@ -736,14 +866,16 @@ def handle_command_def(self, line): """Handles one command line during command list definition.""" cmd, arg, line = self.parseline(line) if not cmd: - return - if cmd == 'silent': - self.commands_silent[self.commands_bnum] = True - return # continue to handle other cmd def in the cmd list - elif cmd == 'end': - self.cmdqueue = [] - return 1 # end of cmd list + return False + if cmd == 'end': + return True # end of cmd list + elif cmd == 'EOF': + print('') + return True # end of cmd list cmdlist = self.commands[self.commands_bnum] + if cmd == 'silent': + cmdlist.append('_pdbcmd_silence_frame_status') + return False # continue to handle other cmd def in the cmd list if arg: cmdlist.append(cmd+' '+arg) else: @@ -755,10 +887,8 @@ def handle_command_def(self, line): func = self.default # one of the resuming commands if func.__name__ in self.commands_resuming: - self.commands_doprompt[self.commands_bnum] = False - self.cmdqueue = [] - return 1 - return + return True + return False # interface abstraction functions @@ -788,7 +918,10 @@ def completenames(self, text, line, begidx, endidx): if commands: return commands else: - return self._complete_expression(text, line, begidx, endidx) + expressions = self._complete_expression(text, line, begidx, endidx) + if expressions: + return expressions + return self.completedefault(text, line, begidx, endidx) def _complete_location(self, text, line, begidx, endidx): # Complete a file/module/function location for break/tbreak/clear. @@ -823,7 +956,7 @@ def _complete_expression(self, text, line, begidx, endidx): # Collect globals and locals. It is usually not really sensible to also # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} if text.startswith("$"): # Complete convenience variables conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) @@ -845,6 +978,35 @@ def _complete_expression(self, text, line, begidx, endidx): # Complete a simple name. return [n for n in ns.keys() if n.startswith(text)] + def completedefault(self, text, line, begidx, endidx): + if text.startswith("$"): + # Complete convenience variables + conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) + return [f"${name}" for name in conv_vars if name.startswith(text[1:])] + + # Use rlcompleter to do the completion + state = 0 + matches = [] + completer = Completer(self.curframe.f_globals | self.curframe.f_locals) + while (match := completer.complete(text, state)) is not None: + matches.append(match) + state += 1 + return matches + + # Pdb meta commands, only intended to be used internally by pdb + + def _pdbcmd_print_frame_status(self, arg): + self.print_stack_trace(0) + self._validate_file_mtime() + self._show_display() + + def _pdbcmd_silence_frame_status(self, arg): + if self.cmdqueue and self.cmdqueue[-1] == '_pdbcmd_print_frame_status': + self.cmdqueue.pop() + + def _pdbcmd_restore_lastcmd(self, arg): + self.lastcmd = arg + # Command definitions, called by cmdloop() # The argument is the remaining string on the command line # Return true to exit from the command loop @@ -903,14 +1065,10 @@ def do_commands(self, arg): self.commands_bnum = bnum # Save old definitions for the case of a keyboard interrupt. if bnum in self.commands: - old_command_defs = (self.commands[bnum], - self.commands_doprompt[bnum], - self.commands_silent[bnum]) + old_commands = self.commands[bnum] else: - old_command_defs = None + old_commands = None self.commands[bnum] = [] - self.commands_doprompt[bnum] = True - self.commands_silent[bnum] = False prompt_back = self.prompt self.prompt = '(com) ' @@ -919,14 +1077,10 @@ def do_commands(self, arg): self.cmdloop() except KeyboardInterrupt: # Restore old definitions. - if old_command_defs: - self.commands[bnum] = old_command_defs[0] - self.commands_doprompt[bnum] = old_command_defs[1] - self.commands_silent[bnum] = old_command_defs[2] + if old_commands: + self.commands[bnum] = old_commands else: del self.commands[bnum] - del self.commands_doprompt[bnum] - del self.commands_silent[bnum] self.error('command definition aborted, old commands restored') finally: self.commands_defining = False @@ -934,7 +1088,7 @@ def do_commands(self, arg): complete_commands = _complete_bpnumber - def do_break(self, arg, temporary = 0): + def do_break(self, arg, temporary=False): """b(reak) [ ([filename:]lineno | function) [, condition] ] Without argument, list all breaks. @@ -995,7 +1149,7 @@ def do_break(self, arg, temporary = 0): try: func = eval(arg, self.curframe.f_globals, - self.curframe_locals) + self.curframe.f_locals) except: func = arg try: @@ -1049,7 +1203,7 @@ def do_tbreak(self, arg): Same arguments as break, but sets a temporary breakpoint: it is automatically deleted when first hit. """ - self.do_break(arg, 1) + self.do_break(arg, True) complete_tbreak = _complete_location @@ -1083,7 +1237,7 @@ def lineinfo(self, identifier): if f: fname = f item = parts[1] - answer = find_function(item, fname) + answer = find_function(item, self.canonic(fname)) return answer or failed def checkline(self, filename, lineno): @@ -1275,16 +1429,24 @@ def do_clear(self, arg): complete_cl = _complete_location def do_where(self, arg): - """w(here) + """w(here) [count] - Print a stack trace, with the most recent frame at the bottom. + Print a stack trace. If count is not specified, print the full stack. + If count is 0, print the current frame entry. If count is positive, + print count entries from the most recent frame. If count is negative, + print -count entries from the least recent frame. An arrow indicates the "current frame", which determines the context of most commands. 'bt' is an alias for this command. """ - if arg: - self._print_invalid_arg(arg) - return - self.print_stack_trace() + if not arg: + count = None + else: + try: + count = int(arg) + except ValueError: + self.error('Invalid count (%s)' % arg) + return + self.print_stack_trace(count) do_w = do_where do_bt = do_where @@ -1292,7 +1454,6 @@ def _select_frame(self, number): assert 0 <= number < len(self.stack) self.curindex = number self.curframe = self.stack[self.curindex][0] - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) self.print_stack_entry(self.stack[self.curindex]) self.lineno = None @@ -1446,6 +1607,11 @@ def do_run(self, arg): sys.argv. History, breakpoints, actions and debugger options are preserved. "restart" is an alias for "run". """ + if self.mode == 'inline': + self.error('run/restart command is disabled when pdb is running in inline mode.\n' + 'Use the command line interface to enable restarting your program\n' + 'e.g. "python -m pdb myscript.py"') + return if arg: import shlex argv0 = sys.argv[0:1] @@ -1533,7 +1699,7 @@ def do_debug(self, arg): """ sys.settrace(None) globals = self.curframe.f_globals - locals = self.curframe_locals + locals = self.curframe.f_locals p = Pdb(self.completekey, self.stdin, self.stdout) p.prompt = "(%s) " % self.prompt.strip() self.message("ENTERING RECURSIVE DEBUGGER") @@ -1578,7 +1744,7 @@ def do_args(self, arg): self._print_invalid_arg(arg) return co = self.curframe.f_code - dict = self.curframe_locals + dict = self.curframe.f_locals n = co.co_argcount + co.co_kwonlyargcount if co.co_flags & inspect.CO_VARARGS: n = n+1 if co.co_flags & inspect.CO_VARKEYWORDS: n = n+1 @@ -1598,15 +1764,15 @@ def do_retval(self, arg): if arg: self._print_invalid_arg(arg) return - if '__return__' in self.curframe_locals: - self.message(self._safe_repr(self.curframe_locals['__return__'], "retval")) + if '__return__' in self.curframe.f_locals: + self.message(self._safe_repr(self.curframe.f_locals['__return__'], "retval")) else: self.error('Not yet returned!') do_rv = do_retval def _getval(self, arg): try: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) except: self._error_exc() raise @@ -1614,7 +1780,7 @@ def _getval(self, arg): def _getval_except(self, arg, frame=None): try: if frame is None: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) else: return eval(arg, frame.f_globals, frame.f_locals) except BaseException as exc: @@ -1713,6 +1879,7 @@ def do_list(self, arg): self.message('[EOF]') except KeyboardInterrupt: pass + self._validate_file_mtime() do_l = do_list def do_longlist(self, arg): @@ -1731,6 +1898,7 @@ def do_longlist(self, arg): self.error(err) return self._print_lines(lines, lineno, breaklist, self.curframe) + self._validate_file_mtime() do_ll = do_longlist def do_source(self, arg): @@ -1858,7 +2026,7 @@ def do_interact(self, arg): Start an interactive interpreter whose global namespace contains all the (global and local) names found in the current scope. """ - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} console = _PdbInteractiveConsole(ns, message=self.message) console.interact(banner="*pdb interact start*", exitmsg="*exit from pdb interact command*") @@ -1930,7 +2098,7 @@ def complete_unalias(self, text, line, begidx, endidx): # List of all the commands making the program resume execution. commands_resuming = ['do_continue', 'do_step', 'do_next', 'do_return', - 'do_quit', 'do_jump'] + 'do_until', 'do_quit', 'do_jump'] # Print a traceback starting at the top stack frame. # The most recently entered frame is printed last; @@ -1939,10 +2107,22 @@ def complete_unalias(self, text, line, begidx, endidx): # It is also consistent with the up/down commands (which are # compatible with dbx and gdb: up moves towards 'main()' # and down moves towards the most recent stack frame). - - def print_stack_trace(self): + # * if count is None, prints the full stack + # * if count = 0, prints the current frame entry + # * if count < 0, prints -count least recent frame entries + # * if count > 0, prints count most recent frame entries + + def print_stack_trace(self, count=None): + if count is None: + stack_to_print = self.stack + elif count == 0: + stack_to_print = [self.stack[self.curindex]] + elif count < 0: + stack_to_print = self.stack[:-count] + else: + stack_to_print = self.stack[-count:] try: - for frame_lineno in self.stack: + for frame_lineno in stack_to_print: self.print_stack_entry(frame_lineno) except KeyboardInterrupt: pass @@ -2014,17 +2194,23 @@ def lookupmodule(self, filename): lookupmodule() translates (possibly incomplete) file or module name into an absolute file name. + + filename could be in format of: + * an absolute path like '/path/to/file.py' + * a relative path like 'file.py' or 'dir/file.py' + * a module name like 'module' or 'package.module' + + files and modules will be searched in sys.path. """ - if os.path.isabs(filename) and os.path.exists(filename): - return filename - f = os.path.join(sys.path[0], filename) - if os.path.exists(f) and self.canonic(f) == self.mainpyfile: - return f - root, ext = os.path.splitext(filename) - if ext == '': - filename = filename + '.py' + if not filename.endswith('.py'): + # A module is passed in so convert it to equivalent file + filename = filename.replace('.', os.sep) + '.py' + if os.path.isabs(filename): - return filename + if os.path.exists(filename): + return filename + return None + for dirname in sys.path: while os.path.islink(dirname): dirname = os.readlink(dirname) @@ -2033,7 +2219,7 @@ def lookupmodule(self, filename): return fullname return None - def _run(self, target: Union[_ModuleTarget, _ScriptTarget]): + def _run(self, target: _ExecutableTarget): # When bdb sets tracing, a number of call and line events happen # BEFORE debugger even reaches user's code (and the exact sequence of # events depends on python version). Take special measures to @@ -2171,18 +2357,22 @@ def runcall(*args, **kwds): """ return Pdb().runcall(*args, **kwds) -def set_trace(*, header=None): +def set_trace(*, header=None, commands=None): """Enter the debugger at the calling stack frame. This is useful to hard-code a breakpoint at a given point in a program, even if the code is not otherwise being debugged (e.g. when an assertion fails). If given, *header* is printed to the console - just before debugging begins. + just before debugging begins. *commands* is an optional list of + pdb commands to run when the debugger starts. """ - pdb = Pdb() + if Pdb._last_pdb_instance is not None: + pdb = Pdb._last_pdb_instance + else: + pdb = Pdb(mode='inline') if header is not None: pdb.message(header) - pdb.set_trace(sys._getframe().f_back) + pdb.set_trace(sys._getframe().f_back, commands=commands) # Post-Mortem interface @@ -2253,16 +2443,17 @@ def help(): def main(): import argparse - parser = argparse.ArgumentParser(prog="pdb", + parser = argparse.ArgumentParser(usage="%(prog)s [-h] [-c command] (-m module | pyfile) [args ...]", description=_usage, formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False) - parser.add_argument('-c', '--command', action='append', default=[], metavar='command') - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-m', metavar='module') - group.add_argument('pyfile', nargs='?') - parser.add_argument('args', nargs="*") + # We need to maunally get the script from args, because the first positional + # arguments could be either the script we need to debug, or the argument + # to the -m module + parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', + help='pdb commands to execute as if given in a .pdbrc file') + parser.add_argument('-m', metavar='module', dest='module') if len(sys.argv) == 1: # If no arguments were given (python -m pdb), print the whole help message. @@ -2270,25 +2461,47 @@ def main(): parser.print_help() sys.exit(2) - opts = parser.parse_args() - - if opts.m: - file = opts.m + opts, args = parser.parse_known_args() + + if opts.module: + # If a module is being debugged, we consider the arguments after "-m module" to + # be potential arguments to the module itself. We need to parse the arguments + # before "-m" to check if there is any invalid argument. + # e.g. "python -m pdb -m foo --spam" means passing "--spam" to "foo" + # "python -m pdb --spam -m foo" means passing "--spam" to "pdb" and is invalid + idx = sys.argv.index('-m') + args_to_pdb = sys.argv[1:idx] + # This will raise an error if there are invalid arguments + parser.parse_args(args_to_pdb) + else: + # If a script is being debugged, then pdb expects the script name as the first argument. + # Anything before the script is considered an argument to pdb itself, which would + # be invalid because it's not parsed by argparse. + invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) + if invalid_args: + parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") + sys.exit(2) + + if opts.module: + file = opts.module target = _ModuleTarget(file) else: - file = opts.pyfile - target = _ScriptTarget(file) - - target.check() + if not args: + parser.error("no module or script to run") + file = args.pop(0) + if file.endswith('.pyz'): + target = _ZipTarget(file) + else: + target = _ScriptTarget(file) - sys.argv[:] = [file] + opts.args # Hide "pdb.py" and pdb options from argument list + sys.argv[:] = [file] + args # Hide "pdb.py" and pdb options from argument list # Note on saving/restoring sys.argv: it's a good idea when sys.argv was # modified by the script being debugged. It's a bad idea when it was # changed by the user from the command line. There is a "restart" command # which allows explicit specification of command line arguments. - pdb = Pdb() - pdb.rcLines.extend(opts.command) + pdb = Pdb(mode='cli') + pdb.rcLines.extend(opts.commands) while True: try: pdb._run(target) @@ -2300,12 +2513,15 @@ def main(): print("The program exited via sys.exit(). Exit status:", end=' ') print(e) except BaseException as e: - traceback.print_exc() + traceback.print_exception(e, colorize=_colorize.can_colorize()) print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") - pdb.interaction(None, e) - print("Post mortem debugger finished. The " + target + - " will be restarted") + try: + pdb.interaction(None, e) + except Restart: + print("Restarting", target, "with arguments:") + print("\t" + " ".join(sys.argv[1:])) + continue if pdb._user_requested_quit: break print("The program finished and will be restarted") diff --git a/Lib/pickle.py b/Lib/pickle.py index 33c97c8c5efb28..ed8138beb908ee 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -51,7 +51,7 @@ bytes_types = (bytes, bytearray) # These are purely informational; no code uses these. -format_version = "4.0" # File format version we write +format_version = "5.0" # File format version we write compatible_formats = ["1.0", # Original protocol 0 "1.1", # Protocol 0 with INST added "1.2", # Original protocol 1 @@ -68,7 +68,7 @@ # The protocol we write by default. May be less than HIGHEST_PROTOCOL. # Only bump this if the oldest still supported version of Python already # includes it. -DEFAULT_PROTOCOL = 4 +DEFAULT_PROTOCOL = 5 class PickleError(Exception): """A common base class for the other pickling exceptions.""" @@ -313,37 +313,46 @@ def load_frame(self, frame_size): # Tools used for pickling. -def _getattribute(obj, name): - for subpath in name.split('.'): - if subpath == '': - raise AttributeError("Can't get local attribute {!r} on {!r}" - .format(name, obj)) - try: - parent = obj - obj = getattr(obj, subpath) - except AttributeError: - raise AttributeError("Can't get attribute {!r} on {!r}" - .format(name, obj)) from None - return obj, parent +def _getattribute(obj, dotted_path): + for subpath in dotted_path: + obj = getattr(obj, subpath) + return obj def whichmodule(obj, name): """Find the module an object belong to.""" + dotted_path = name.split('.') module_name = getattr(obj, '__module__', None) - if module_name is not None: - return module_name - # Protect the iteration by using a list copy of sys.modules against dynamic - # modules that trigger imports of other modules upon calls to getattr. - for module_name, module in sys.modules.copy().items(): - if (module_name == '__main__' - or module_name == '__mp_main__' # bpo-42406 - or module is None): - continue - try: - if _getattribute(module, name)[0] is obj: - return module_name - except AttributeError: - pass - return '__main__' + if '' in dotted_path: + raise PicklingError(f"Can't pickle local object {obj!r}") + if module_name is None: + # Protect the iteration by using a list copy of sys.modules against dynamic + # modules that trigger imports of other modules upon calls to getattr. + for module_name, module in sys.modules.copy().items(): + if (module_name == '__main__' + or module_name == '__mp_main__' # bpo-42406 + or module is None): + continue + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + pass + module_name = '__main__' + + try: + __import__(module_name, level=0) + module = sys.modules[module_name] + except (ImportError, ValueError, KeyError) as exc: + raise PicklingError(f"Can't pickle {obj!r}: {exc!s}") + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + raise PicklingError(f"Can't pickle {obj!r}: " + f"it's not found as {module_name}.{name}") + + raise PicklingError( + f"Can't pickle {obj!r}: it's not the same object as {module_name}.{name}") def encode_long(x): r"""Encode a long to a two's complement little-endian binary string. @@ -395,6 +404,13 @@ def decode_long(data): """ return int.from_bytes(data, byteorder='little', signed=True) +def _T(obj): + cls = type(obj) + module = cls.__module__ + if module in (None, 'builtins', '__main__'): + return cls.__qualname__ + return f'{module}.{cls.__qualname__}' + _NoValue = object() @@ -408,7 +424,7 @@ def __init__(self, file, protocol=None, *, fix_imports=True, The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. - The default protocol is 4. It was introduced in Python 3.4, and + The default protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest @@ -577,26 +593,29 @@ def save(self, obj, save_persistent_id=True): if reduce is not _NoValue: rv = reduce() else: - raise PicklingError("Can't pickle %r object: %r" % - (t.__name__, obj)) + raise PicklingError(f"Can't pickle {_T(t)} object") # Check for string returned by reduce(), meaning "save as global" if isinstance(rv, str): self.save_global(obj, rv) return - # Assert that reduce() returned a tuple - if not isinstance(rv, tuple): - raise PicklingError("%s must return string or tuple" % reduce) - - # Assert that it returned an appropriately sized tuple - l = len(rv) - if not (2 <= l <= 6): - raise PicklingError("Tuple returned by %s must have " - "two to six elements" % reduce) - - # Save the reduce() output and finally memoize the object - self.save_reduce(obj=obj, *rv) + try: + # Assert that reduce() returned a tuple + if not isinstance(rv, tuple): + raise PicklingError(f'__reduce__ must return a string or tuple, not {_T(rv)}') + + # Assert that it returned an appropriately sized tuple + l = len(rv) + if not (2 <= l <= 6): + raise PicklingError("tuple returned by __reduce__ " + "must contain 2 through 6 elements") + + # Save the reduce() output and finally memoize the object + self.save_reduce(obj=obj, *rv) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} object') + raise def persistent_id(self, obj): # This exists so a subclass can override it @@ -618,10 +637,12 @@ def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, *, obj=None): # This API is called by some subclasses - if not isinstance(args, tuple): - raise PicklingError("args from save_reduce() must be a tuple") if not callable(func): - raise PicklingError("func from save_reduce() must be callable") + raise PicklingError(f"first item of the tuple returned by __reduce__ " + f"must be callable, not {_T(func)}") + if not isinstance(args, tuple): + raise PicklingError(f"second item of the tuple returned by __reduce__ " + f"must be a tuple, not {_T(args)}") save = self.save write = self.write @@ -630,19 +651,30 @@ def save_reduce(self, func, args, state=None, listitems=None, if self.proto >= 2 and func_name == "__newobj_ex__": cls, args, kwargs = args if not hasattr(cls, "__new__"): - raise PicklingError("args[0] from {} args has no __new__" - .format(func_name)) + raise PicklingError("first argument to __newobj_ex__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError("args[0] from {} args has the wrong class" - .format(func_name)) + raise PicklingError(f"first argument to __newobj_ex__() " + f"must be {obj.__class__!r}, not {cls!r}") if self.proto >= 4: - save(cls) - save(args) - save(kwargs) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + save(kwargs) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ_EX) else: func = partial(cls.__new__, cls, *args, **kwargs) - save(func) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise save(()) write(REDUCE) elif self.proto >= 2 and func_name == "__newobj__": @@ -674,18 +706,33 @@ def save_reduce(self, func, args, state=None, listitems=None, # Python 2.2). cls = args[0] if not hasattr(cls, "__new__"): - raise PicklingError( - "args[0] from __newobj__ args has no __new__") + raise PicklingError("first argument to __newobj__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError( - "args[0] from __newobj__ args has the wrong class") + raise PicklingError(f"first argument to __newobj__() " + f"must be {obj.__class__!r}, not {cls!r}") args = args[1:] - save(cls) - save(args) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ) else: - save(func) - save(args) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor arguments') + raise write(REDUCE) if obj is not None: @@ -703,23 +750,35 @@ def save_reduce(self, func, args, state=None, listitems=None, # items and dict items (as (key, value) tuples), or None. if listitems is not None: - self._batch_appends(listitems) + self._batch_appends(listitems, obj) if dictitems is not None: - self._batch_setitems(dictitems) + self._batch_setitems(dictitems, obj) if state is not None: if state_setter is None: - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(BUILD) else: # If a state_setter is specified, call it instead of load_build # to update obj's with its previous state. # First, push state_setter and its tuple of expected arguments # (obj, state) onto the stack. - save(state_setter) + try: + save(state_setter) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state setter') + raise save(obj) # simple BINGET opcode as obj is already memoized. - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(TUPLE2) # Trigger a state_setter(obj, state) function call. write(REDUCE) @@ -782,14 +841,10 @@ def save_float(self, obj): self.write(FLOAT + repr(obj).encode("ascii") + b'\n') dispatch[float] = save_float - def save_bytes(self, obj): - if self.proto < 3: - if not obj: # bytes object is empty - self.save_reduce(bytes, (), obj=obj) - else: - self.save_reduce(codecs.encode, - (str(obj, 'latin1'), 'latin1'), obj=obj) - return + def _save_bytes_no_memo(self, obj): + # helper for writing bytes objects for protocol >= 3 + # without memoizing them + assert self.proto >= 3 n = len(obj) if n <= 0xff: self.write(SHORT_BINBYTES + pack("= 5 + # without memoizing them + assert self.proto >= 5 + n = len(obj) + if n >= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BYTEARRAY8 + pack("= self.framer._FRAME_SIZE_TARGET: - self._write_large_bytes(BYTEARRAY8 + pack("= 5") with obj.raw() as m: if not m.contiguous: @@ -832,10 +903,18 @@ def save_picklebuffer(self, obj): if in_band: # Write data in-band # XXX The C implementation avoids a copy here + buf = m.tobytes() + in_memo = id(buf) in self.memo if m.readonly: - self.save_bytes(m.tobytes()) + if in_memo: + self._save_bytes_no_memo(buf) + else: + self.save_bytes(buf) else: - self.save_bytearray(m.tobytes()) + if in_memo: + self._save_bytearray_no_memo(buf) + else: + self.save_bytearray(buf) else: # Write data out-of-band self.write(NEXT_BUFFER) @@ -879,8 +958,12 @@ def save_tuple(self, obj): save = self.save memo = self.memo if n <= 3 and self.proto >= 2: - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise # Subtle. Same as in the big comment below. if id(obj) in memo: get = self.get(memo[id(obj)][0]) @@ -894,8 +977,12 @@ def save_tuple(self, obj): # has more than 3 elements. write = self.write write(MARK) - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise if id(obj) in memo: # Subtle. d was not in memo when we entered save_tuple(), so @@ -925,38 +1012,52 @@ def save_list(self, obj): self.write(MARK + LIST) self.memoize(obj) - self._batch_appends(obj) + self._batch_appends(obj, obj) dispatch[list] = save_list _BATCHSIZE = 1000 - def _batch_appends(self, items): + def _batch_appends(self, items, obj): # Helper to batch up APPENDS sequences save = self.save write = self.write if not self.bin: - for x in items: - save(x) + for i, x in enumerate(items): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPEND) return it = iter(items) + start = 0 while True: tmp = list(islice(it, self._BATCHSIZE)) n = len(tmp) if n > 1: write(MARK) - for x in tmp: - save(x) + for i, x in enumerate(tmp, start): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPENDS) elif n: - save(tmp[0]) + try: + save(tmp[0]) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {start}') + raise write(APPEND) # else tmp is empty, and we're done if n < self._BATCHSIZE: return + start += n def save_dict(self, obj): if self.bin: @@ -965,11 +1066,11 @@ def save_dict(self, obj): self.write(MARK + DICT) self.memoize(obj) - self._batch_setitems(obj.items()) + self._batch_setitems(obj.items(), obj) dispatch[dict] = save_dict - def _batch_setitems(self, items): + def _batch_setitems(self, items, obj): # Helper to batch up SETITEMS sequences; proto >= 1 only save = self.save write = self.write @@ -977,7 +1078,11 @@ def _batch_setitems(self, items): if not self.bin: for k, v in items: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) return @@ -989,12 +1094,20 @@ def _batch_setitems(self, items): write(MARK) for k, v in tmp: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEMS) elif n: k, v = tmp[0] save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) # else tmp is empty, and we're done if n < self._BATCHSIZE: @@ -1017,8 +1130,12 @@ def save_set(self, obj): n = len(batch) if n > 0: write(MARK) - for item in batch: - save(item) + try: + for item in batch: + save(item) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise write(ADDITEMS) if n < self._BATCHSIZE: return @@ -1033,8 +1150,12 @@ def save_frozenset(self, obj): return write(MARK) - for item in obj: - save(item) + try: + for item in obj: + save(item) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise if id(obj) in self.memo: # If the object is already in the memo, this means it is @@ -1053,48 +1174,59 @@ def save_global(self, obj, name=None): if name is None: name = getattr(obj, '__qualname__', None) - if name is None: - name = obj.__name__ + if name is None: + name = obj.__name__ module_name = whichmodule(obj, name) - try: - __import__(module_name, level=0) - module = sys.modules[module_name] - obj2, parent = _getattribute(module, name) - except (ImportError, KeyError, AttributeError): - raise PicklingError( - "Can't pickle %r: it's not found as %s.%s" % - (obj, module_name, name)) from None - else: - if obj2 is not obj: - raise PicklingError( - "Can't pickle %r: it's not the same object as %s.%s" % - (obj, module_name, name)) - if self.proto >= 2: - code = _extension_registry.get((module_name, name)) - if code: - assert code > 0 + code = _extension_registry.get((module_name, name), _NoValue) + if code is not _NoValue: if code <= 0xff: - write(EXT1 + pack("= 3. + if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) - elif parent is not module: - self.save_reduce(getattr, (parent, lastname)) - elif self.proto >= 3: - write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + - bytes(name, "utf-8") + b'\n') + elif '.' in name: + # In protocol < 4, objects with multi-part __qualname__ + # are represented as + # getattr(getattr(..., attrname1), attrname2). + dotted_path = name.split('.') + name = dotted_path.pop(0) + save = self.save + for attrname in dotted_path: + save(getattr) + if self.proto < 2: + write(MARK) + self._save_toplevel_by_name(module_name, name) + for attrname in dotted_path: + save(attrname) + if self.proto < 2: + write(TUPLE) + else: + write(TUPLE2) + write(REDUCE) + else: + self._save_toplevel_by_name(module_name, name) + + self.memoize(obj) + + def _save_toplevel_by_name(self, module_name, name): + if self.proto >= 3: + # Non-ASCII identifiers are supported only with protocols >= 3. + encoding = "utf-8" else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING @@ -1103,15 +1235,19 @@ def save_global(self, obj, name=None): module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] - try: - write(GLOBAL + bytes(module_name, "ascii") + b'\n' + - bytes(name, "ascii") + b'\n') - except UnicodeEncodeError: - raise PicklingError( - "can't pickle global identifier '%s.%s' using " - "pickle protocol %i" % (module, name, self.proto)) from None - - self.memoize(obj) + encoding = "ascii" + try: + self.write(GLOBAL + bytes(module_name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle module identifier {module_name!r} using " + f"pickle protocol {self.proto}") + try: + self.write(bytes(name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle global identifier {name!r} using " + f"pickle protocol {self.proto}") def save_type(self, obj): if obj is type(None): @@ -1548,9 +1684,8 @@ def load_ext4(self): dispatch[EXT4[0]] = load_ext4 def get_extension(self, code): - nil = [] - obj = _extension_cache.get(code, nil) - if obj is not nil: + obj = _extension_cache.get(code, _NoValue) + if obj is not _NoValue: self.append(obj) return key = _inverted_registry.get(code) @@ -1572,8 +1707,13 @@ def find_class(self, module, name): elif module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) - if self.proto >= 4: - return _getattribute(sys.modules[module], name)[0] + if self.proto >= 4 and '.' in name: + dotted_path = name.split('.') + try: + return _getattribute(sys.modules[module], dotted_path) + except AttributeError: + raise AttributeError( + f"Can't resolve path {name!r} on module {module!r}") else: return getattr(sys.modules[module], name) diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 51ee4a7a2632ac..c462d26da97ce1 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -2429,8 +2429,6 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): + A memo entry isn't referenced before it's defined. + The markobject isn't stored in the memo. - - + A memo entry isn't redefined. """ # Most of the hair here is for sanity checks, but most of it is needed @@ -2484,7 +2482,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): assert opcode.name == "POP" numtopop = 0 else: - errormsg = markmsg = "no MARK exists on stack" + errormsg = "no MARK exists on stack" # Check for correct memo usage. if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): @@ -2494,9 +2492,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): else: assert arg is not None memo_idx = arg - if memo_idx in memo: - errormsg = "memo key %r already defined" % arg - elif not stack: + if not stack: errormsg = "stack is empty -- can't store into memo" elif stack[-1] is markobject: errormsg = "can't store markobject in the memo" diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index dccbec52aa731e..b84d72f2395d45 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -12,7 +12,7 @@ import warnings __all__ = [ - 'get_importer', 'iter_importers', 'get_loader', 'find_loader', + 'get_importer', 'iter_importers', 'walk_packages', 'iter_modules', 'get_data', 'read_code', 'extend_path', 'ModuleInfo', @@ -263,59 +263,6 @@ def iter_importers(fullname=""): yield get_importer(item) -def get_loader(module_or_name): - """Get a "loader" object for module_or_name - - Returns None if the module cannot be found or imported. - If the named module is not already imported, its containing package - (if any) is imported, in order to establish the package __path__. - """ - warnings._deprecated("pkgutil.get_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if module_or_name in sys.modules: - module_or_name = sys.modules[module_or_name] - if module_or_name is None: - return None - if isinstance(module_or_name, ModuleType): - module = module_or_name - loader = getattr(module, '__loader__', None) - if loader is not None: - return loader - if getattr(module, '__spec__', None) is None: - return None - fullname = module.__name__ - else: - fullname = module_or_name - return find_loader(fullname) - - -def find_loader(fullname): - """Find a "loader" object for fullname - - This is a backwards compatibility wrapper around - importlib.util.find_spec that converts most failures to ImportError - and only returns the loader rather than the full spec - """ - warnings._deprecated("pkgutil.find_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if fullname.startswith('.'): - msg = "Relative module name {!r} not supported".format(fullname) - raise ImportError(msg) - try: - spec = importlib.util.find_spec(fullname) - except (ImportError, AttributeError, TypeError, ValueError) as ex: - # This hack fixes an impedance mismatch between pkgutil and - # importlib, where the latter raises other errors for cases where - # pkgutil previously raised ImportError - msg = "Error while finding loader for {!r} ({}: {})" - raise ImportError(msg.format(fullname, type(ex), ex)) from ex - return spec.loader if spec is not None else None - - def extend_path(path, name): """Extend a package's path. diff --git a/Lib/platform.py b/Lib/platform.py old mode 100755 new mode 100644 index b56472235ee9e4..d6322c9d99d2f3 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ This module tries to retrieve as much platform-identifying data as possible. It makes this information available via function APIs. @@ -10,7 +8,8 @@ """ # This module is maintained by Marc-Andre Lemburg . # If you find problems, please submit bug reports/patches via the -# Python bug tracker (http://bugs.python.org) and assign them to "lemburg". +# Python issue tracker (https://github.com/python/cpython/issues) and +# mention "@malemburg". # # Still needed: # * support for MS-DOS (PythonDX ?) @@ -370,10 +369,7 @@ def win32_is_iot(): def win32_edition(): try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -432,10 +428,7 @@ def _win32_ver(version, csd, ptype): csd = 'SP' + csd[13:] try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -502,8 +495,32 @@ def mac_ver(release='', versioninfo=('', '', ''), machine=''): # If that also doesn't work return the default values return release, versioninfo, machine -def _java_getprop(name, default): +# A namedtuple for iOS version information. +IOSVersionInfo = collections.namedtuple( + "IOSVersionInfo", + ["system", "release", "model", "is_simulator"] +) + + +def ios_ver(system="", release="", model="", is_simulator=False): + """Get iOS version information, and return it as a namedtuple: + (system, release, model, is_simulator). + + If values can't be determined, they are set to values provided as + parameters. + """ + if sys.platform == "ios": + import _ios_support + result = _ios_support.get_platform_ios() + if result is not None: + return IOSVersionInfo(*result) + + return IOSVersionInfo(system, release, model, is_simulator) + + +def _java_getprop(name, default): + """This private helper is deprecated in 3.13 and will be removed in 3.15""" from java.lang import System try: value = System.getProperty(name) @@ -525,9 +542,11 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): given as parameters (which all default to ''). """ + import warnings + warnings._deprecated('java_ver', remove=(3, 15)) # Import the needed APIs try: - import java.lang + import java.lang # noqa: F401 except ImportError: return release, vendor, vminfo, osinfo @@ -546,6 +565,47 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): return release, vendor, vminfo, osinfo + +AndroidVer = collections.namedtuple( + "AndroidVer", "release api_level manufacturer model device is_emulator") + +def android_ver(release="", api_level=0, manufacturer="", model="", device="", + is_emulator=False): + if sys.platform == "android": + try: + from ctypes import CDLL, c_char_p, create_string_buffer + except ImportError: + pass + else: + # An NDK developer confirmed that this is an officially-supported + # API (https://stackoverflow.com/a/28416743). Use `getattr` to avoid + # private name mangling. + system_property_get = getattr(CDLL("libc.so"), "__system_property_get") + system_property_get.argtypes = (c_char_p, c_char_p) + + def getprop(name, default): + # https://android.googlesource.com/platform/bionic/+/refs/tags/android-5.0.0_r1/libc/include/sys/system_properties.h#39 + PROP_VALUE_MAX = 92 + buffer = create_string_buffer(PROP_VALUE_MAX) + length = system_property_get(name.encode("UTF-8"), buffer) + if length == 0: + # This API doesn’t distinguish between an empty property and + # a missing one. + return default + else: + return buffer.value.decode("UTF-8", "backslashreplace") + + release = getprop("ro.build.version.release", release) + api_level = int(getprop("ro.build.version.sdk", api_level)) + manufacturer = getprop("ro.product.manufacturer", manufacturer) + model = getprop("ro.product.model", model) + device = getprop("ro.product.device", device) + is_emulator = getprop("ro.kernel.qemu", "0") == "1" + + return AndroidVer( + release, api_level, manufacturer, model, device, is_emulator) + + ### System name aliasing def system_alias(system, release, version): @@ -617,7 +677,7 @@ def _platform(*args): if cleaned == platform: break platform = cleaned - while platform[-1] == '-': + while platform and platform[-1] == '-': platform = platform[:-1] return platform @@ -658,7 +718,7 @@ def _syscmd_file(target, default=''): default in case the command should fail. """ - if sys.platform in ('dos', 'win32', 'win16'): + if sys.platform in {'dos', 'win32', 'win16', 'ios', 'tvos', 'watchos'}: # XXX Others too ? return default @@ -822,6 +882,14 @@ def get_OpenVMS(): csid, cpu_number = vms_lib.getsyi('SYI$_CPU', 0) return 'Alpha' if cpu_number >= 128 else 'VAX' + # On the iOS simulator, os.uname returns the architecture as uname.machine. + # On device it returns the model name for some reason; but there's only one + # CPU architecture for iOS devices, so we know the right answer. + def get_ios(): + if sys.implementation._multiarch.endswith("simulator"): + return os.uname().machine + return 'arm64' + def from_subprocess(): """ Fall back to `uname -p` @@ -976,6 +1044,15 @@ def uname(): system = 'Windows' release = 'Vista' + # On Android, return the name and version of the OS rather than the kernel. + if sys.platform == 'android': + system = 'Android' + release = android_ver().release + + # Normalize responses on iOS + if sys.platform == 'ios': + system, release, _, _ = ios_ver() + vals = system, node, release, version, machine # Replace 'unknown' values with the more portable '' _uname_cache = uname_result(*map(_unknown_as_blank, vals)) @@ -1074,17 +1151,16 @@ def _sys_version(sys_version=None): if result is not None: return result - sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - if sys.platform.startswith('java'): # Jython + jython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" name = 'Jython' - match = sys_version_parser.match(sys_version) + match = jython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1111,7 +1187,14 @@ def _sys_version(sys_version=None): else: # CPython - match = sys_version_parser.match(sys_version) + cpython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'(?:experimental free-threading build\s+)?' # "free-threading-build" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + match = cpython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % @@ -1255,11 +1338,14 @@ def platform(aliased=False, terse=False): system, release, version = system_alias(system, release, version) if system == 'Darwin': - # macOS (darwin kernel) - macos_release = mac_ver()[0] - if macos_release: - system = 'macOS' - release = macos_release + # macOS and iOS both report as a "Darwin" kernel + if sys.platform == "ios": + system, release, _, _ = ios_ver() + else: + macos_release = mac_ver()[0] + if macos_release: + system = 'macOS' + release = macos_release if system == 'Windows': # MS platforms diff --git a/Lib/posixpath.py b/Lib/posixpath.py index e4f155e41a3221..fccca4e066b76f 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -22,6 +22,7 @@ altsep = None devnull = '/dev/null' +import errno import os import sys import stat @@ -35,7 +36,7 @@ "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames","relpath", - "commonpath", "isjunction"] + "commonpath", "isjunction","isdevdrive"] def _get_sep(path): @@ -77,12 +78,11 @@ def join(a, *p): sep = _get_sep(a) path = a try: - if not p: - path[:0] + sep #23780: Ensure compatible data type even if p is null. - for b in map(os.fspath, p): - if b.startswith(sep): + for b in p: + b = os.fspath(b) + if b.startswith(sep) or not path: path = b - elif not path or path.endswith(sep): + elif path.endswith(sep): path += b else: path += sep + b @@ -135,33 +135,30 @@ def splitdrive(p): return p[:0], p -def splitroot(p): - """Split a pathname into drive, root and tail. On Posix, drive is always - empty; the root may be empty, a single slash, or two slashes. The tail - contains anything after the root. For example: +try: + from posix import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. - splitroot('foo/bar') == ('', '', 'foo/bar') - splitroot('/foo/bar') == ('', '/', 'foo/bar') - splitroot('//foo/bar') == ('', '//', 'foo/bar') - splitroot('///foo/bar') == ('', '/', '//foo/bar') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'/' - empty = b'' - else: - sep = '/' - empty = '' - if p[:1] != sep: - # Relative path, e.g.: 'foo' - return empty, empty, p - elif p[1:2] != sep or p[2:3] == sep: - # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. - return empty, sep, p[1:] - else: - # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see - # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 - return empty, p[:2], p[2:] + The tail contains anything after the root.""" + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'/' + empty = b'' + else: + sep = '/' + empty = '' + if p[:1] != sep: + # Relative path, e.g.: 'foo' + return empty, empty, p + elif p[1:2] != sep or p[2:3] == sep: + # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. + return empty, sep, p[1:] + else: + # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see + # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + return empty, p[:2], p[2:] # Return the tail (basename) part of a path, same as split(path)[1]. @@ -187,26 +184,6 @@ def dirname(p): return head -# Is a path a junction? - -def isjunction(path): - """Test whether a path is a junction - Junctions are not a part of posix semantics""" - os.fspath(path) - return False - - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - os.lstat(path) - except (OSError, ValueError): - return False - return True - - # Is a path a mount point? # (Does this work for all UNIXes? Is it even guaranteed to work by Posix?) @@ -227,21 +204,17 @@ def ismount(path): parent = join(path, b'..') else: parent = join(path, '..') - parent = realpath(parent) try: s2 = os.lstat(parent) - except (OSError, ValueError): - return False + except OSError: + parent = realpath(parent) + try: + s2 = os.lstat(parent) + except OSError: + return False - dev1 = s1.st_dev - dev2 = s2.st_dev - if dev1 != dev2: - return True # path/.. on a different device as path - ino1 = s1.st_ino - ino2 = s2.st_ino - if ino1 == ino2: - return True # path/.. is the same i-node as path - return False + # path/.. on a different device as path or the same i-node as path + return s1.st_dev != s2.st_dev or s1.st_ino == s2.st_ino # Expand paths beginning with '~' or '~user'. @@ -290,7 +263,7 @@ def expanduser(path): return path name = path[1:i] if isinstance(name, bytes): - name = str(name, 'ASCII') + name = os.fsdecode(name) try: pwent = pwd.getpwnam(name) except KeyError: @@ -303,11 +276,8 @@ def expanduser(path): return path if isinstance(path, bytes): userhome = os.fsencode(userhome) - root = b'/' - else: - root = '/' - userhome = userhome.rstrip(root) - return (userhome + path[i:]) or root + userhome = userhome.rstrip(sep) + return (userhome + path[i:]) or sep # Expand paths containing shell variable substitutions. @@ -371,7 +341,7 @@ def expandvars(path): # if it contains symbolic links! try: - from posix import _path_normpath + from posix import _path_normpath as normpath except ImportError: def normpath(path): @@ -379,21 +349,19 @@ def normpath(path): path = os.fspath(path) if isinstance(path, bytes): sep = b'/' - empty = b'' dot = b'.' dotdot = b'..' else: sep = '/' - empty = '' dot = '.' dotdot = '..' - if path == empty: + if not path: return dot _, initial_slashes, path = splitroot(path) comps = path.split(sep) new_comps = [] for comp in comps: - if comp in (empty, dot): + if not comp or comp == dot: continue if (comp != dotdot or (not initial_slashes and not new_comps) or (new_comps and new_comps[-1] == dotdot)): @@ -404,24 +372,16 @@ def normpath(path): path = initial_slashes + sep.join(comps) return path or dot -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def abspath(path): """Return an absolute path.""" path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) + if isinstance(path, bytes): + if not path.startswith(b'/'): + path = join(os.getcwdb(), path) + else: + if not path.startswith('/'): + path = join(os.getcwd(), path) return normpath(path) @@ -432,72 +392,104 @@ def realpath(filename, *, strict=False): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) - path, ok = _joinrealpath(filename[:0], filename, strict, {}) - return abspath(path) - -# Join two paths, normalizing and eliminating any symbolic links -# encountered in the second path. -def _joinrealpath(path, rest, strict, seen): - if isinstance(path, bytes): + if isinstance(filename, bytes): sep = b'/' curdir = b'.' pardir = b'..' + getcwd = os.getcwdb else: sep = '/' curdir = '.' pardir = '..' - - if isabs(rest): - rest = rest[1:] - path = sep + getcwd = os.getcwd + return _realpath(filename, strict, sep, curdir, pardir, getcwd) + +def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, + getcwd=os.getcwd, lstat=os.lstat, readlink=os.readlink, maxlinks=None): + # The stack of unresolved path parts. When popped, a special value of None + # indicates that a symlink target has been resolved, and that the original + # symlink path can be retrieved by popping again. The [::-1] slice is a + # very fast way of spelling list(reversed(...)). + rest = filename.split(sep)[::-1] + + # The resolved path, which is absolute throughout this function. + # Note: getcwd() returns a normalized and symlink-free path. + path = sep if filename.startswith(sep) else getcwd() + + # Mapping from symlink paths to *fully resolved* symlink targets. If a + # symlink is encountered but not yet resolved, the value is None. This is + # used both to detect symlink loops and to speed up repeated traversals of + # the same links. + seen = {} + + # Number of symlinks traversed. When the number of traversals is limited + # by *maxlinks*, this is used instead of *seen* to detect symlink loops. + link_count = 0 while rest: - name, _, rest = rest.partition(sep) + name = rest.pop() + if name is None: + # resolved symlink target + seen[rest.pop()] = path + continue if not name or name == curdir: # current dir continue if name == pardir: # parent dir - if path: - path, name = split(path) - if name == pardir: - path = join(path, pardir, pardir) - else: - path = pardir + path = path[:path.rindex(sep)] or sep continue - newpath = join(path, name) + if path == sep: + newpath = path + name + else: + newpath = path + sep + name try: - st = os.lstat(newpath) + st = lstat(newpath) + if not stat.S_ISLNK(st.st_mode): + path = newpath + continue + elif maxlinks is not None: + link_count += 1 + if link_count > maxlinks: + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + elif newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + target = readlink(newpath) except OSError: if strict: raise - is_link = False - else: - is_link = stat.S_ISLNK(st.st_mode) - if not is_link: path = newpath continue # Resolve the symbolic link - if newpath in seen: - # Already seen this path - path = seen[newpath] - if path is not None: - # use cached value - continue - # The symlink is not resolved, so we must have a symlink loop. - if strict: - # Raise OSError(errno.ELOOP) - os.stat(newpath) - else: - # Return already resolved part + rest of the path unchanged. - return join(newpath, rest), False - seen[newpath] = None # not resolved symlink - path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen) - if not ok: - return join(path, rest), False - seen[newpath] = path # resolved symlink + if target.startswith(sep): + # Symlink target is absolute; reset resolved path. + path = sep + if maxlinks is None: + # Mark this symlink as seen but not fully resolved. + seen[newpath] = None + # Push the symlink path onto the stack, and signal its specialness + # by also pushing None. When these entries are popped, we'll + # record the fully-resolved symlink target in the 'seen' mapping. + rest.append(newpath) + rest.append(None) + # Push the unresolved symlink target parts onto the stack. + rest.extend(target.split(sep)[::-1]) - return path, True + return path supports_unicode_filenames = (sys.platform == 'darwin') @@ -505,10 +497,10 @@ def _joinrealpath(path, rest, strict, seen): def relpath(path, start=None): """Return a relative version of a path""" + path = os.fspath(path) if not path: raise ValueError("no path specified") - path = os.fspath(path) if isinstance(path, bytes): curdir = b'.' sep = b'/' @@ -524,15 +516,17 @@ def relpath(path, start=None): start = os.fspath(start) try: - start_list = [x for x in abspath(start).split(sep) if x] - path_list = [x for x in abspath(path).split(sep) if x] + start_tail = abspath(start).lstrip(sep) + path_tail = abspath(path).lstrip(sep) + start_list = start_tail.split(sep) if start_tail else [] + path_list = path_tail.split(sep) if path_tail else [] # Work out how much of the filepath is shared by start and path. i = len(commonprefix([start_list, path_list])) rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise @@ -546,10 +540,11 @@ def relpath(path, start=None): def commonpath(paths): """Given a sequence of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) + if not paths: raise ValueError('commonpath() arg is an empty sequence') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'/' curdir = b'.' @@ -561,7 +556,7 @@ def commonpath(paths): split_paths = [path.split(sep) for path in paths] try: - isabs, = set(p[:1] == sep for p in paths) + isabs, = {p.startswith(sep) for p in paths} except ValueError: raise ValueError("Can't mix absolute and relative paths") from None diff --git a/Lib/pprint.py b/Lib/pprint.py index 9314701db340c7..dc0953cec67a58 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -35,8 +35,6 @@ """ import collections as _collections -import dataclasses as _dataclasses -import re import sys as _sys import types as _types from io import StringIO as _StringIO @@ -54,6 +52,7 @@ def pprint(object, stream=None, indent=1, width=80, depth=None, *, underscore_numbers=underscore_numbers) printer.pprint(object) + def pformat(object, indent=1, width=80, depth=None, *, compact=False, sort_dicts=True, underscore_numbers=False): """Format a Python object into a pretty-printed representation.""" @@ -61,22 +60,27 @@ def pformat(object, indent=1, width=80, depth=None, *, compact=compact, sort_dicts=sort_dicts, underscore_numbers=underscore_numbers).pformat(object) + def pp(object, *args, sort_dicts=False, **kwargs): """Pretty-print a Python object""" pprint(object, *args, sort_dicts=sort_dicts, **kwargs) + def saferepr(object): """Version of repr() which can handle recursive data structures.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[0] + def isreadable(object): """Determine if saferepr(object) is readable by eval().""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[1] + def isrecursive(object): """Determine if object requires a recursive representation.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[2] + class _safe_key: """Helper function for key functions when sorting unorderable objects. @@ -99,10 +103,12 @@ def __lt__(self, other): return ((str(type(self.obj)), id(self.obj)) < \ (str(type(other.obj)), id(other.obj))) + def _safe_tuple(t): "Helper function for comparing 2-tuples" return _safe_key(t[0]), _safe_key(t[1]) + class PrettyPrinter: def __init__(self, indent=1, width=80, depth=None, stream=None, *, compact=False, sort_dicts=True, underscore_numbers=False): @@ -179,12 +185,15 @@ def _format(self, object, stream, indent, allowance, context, level): max_width = self._width - indent - allowance if len(rep) > max_width: p = self._dispatch.get(type(object).__repr__, None) + # Lazy import to improve module import time + from dataclasses import is_dataclass + if p is not None: context[objid] = 1 p(self, object, stream, indent, allowance, context, level + 1) del context[objid] return - elif (_dataclasses.is_dataclass(object) and + elif (is_dataclass(object) and not isinstance(object, type) and object.__dataclass_params__.repr and # Check dataclass has generated repr method. @@ -197,9 +206,12 @@ def _format(self, object, stream, indent, allowance, context, level): stream.write(rep) def _pprint_dataclass(self, object, stream, indent, allowance, context, level): + # Lazy import to improve module import time + from dataclasses import fields as dataclass_fields + cls_name = object.__class__.__name__ indent += len(cls_name) + 1 - items = [(f.name, getattr(object, f.name)) for f in _dataclasses.fields(object) if f.repr] + items = [(f.name, getattr(object, f.name)) for f in dataclass_fields(object) if f.repr] stream.write(cls_name + '(') self._format_namespace_items(items, stream, indent, allowance, context, level) stream.write(')') @@ -291,6 +303,9 @@ def _pprint_str(self, object, stream, indent, allowance, context, level): if len(rep) <= max_width1: chunks.append(rep) else: + # Lazy import to improve module import time + import re + # A list of alternating (non-space, space) strings parts = re.findall(r'\S*\s*', line) assert parts @@ -632,9 +647,11 @@ def _safe_repr(self, object, context, maxlevels, level): rep = repr(object) return rep, (rep and not rep.startswith('<')), False + _builtin_scalars = frozenset({str, bytes, bytearray, float, complex, bool, type(None)}) + def _recursion(object): return ("" % (type(object).__name__, id(object))) diff --git a/Lib/profile.py b/Lib/profile.py old mode 100755 new mode 100644 index 4b82523b03d64b..a5afb12c9d121a --- a/Lib/profile.py +++ b/Lib/profile.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python3 # # Class for profiling python code. rev 1.0 6/2/94 # @@ -387,8 +386,9 @@ def simulate_cmd_complete(self): def print_stats(self, sort=-1): import pstats - pstats.Stats(self).strip_dirs().sort_stats(sort). \ - print_stats() + if not isinstance(sort, tuple): + sort = (sort,) + pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() def dump_stats(self, file): with open(file, 'wb') as f: diff --git a/Lib/pstats.py b/Lib/pstats.py index 2f054bb4011e7f..46e18fb7592a77 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -83,7 +83,7 @@ class Stats: method now take arbitrarily many file names as arguments. All the print methods now take an argument that indicates how many lines - to print. If the arg is a floating point number between 0 and 1.0, then + to print. If the arg is a floating-point number between 0 and 1.0, then it is taken as a decimal percentage of the available lines to be printed (e.g., .1 means print 10% of all available lines). If it is an integer, it is taken to mean the number of lines of data that you wish to have @@ -611,7 +611,7 @@ def f8(x): if __name__ == '__main__': import cmd try: - import readline + import readline # noqa: F401 except ImportError: pass diff --git a/Lib/pty.py b/Lib/pty.py index 1d97994abef3c8..4b25ac32c8da14 100644 --- a/Lib/pty.py +++ b/Lib/pty.py @@ -32,27 +32,18 @@ def openpty(): except (AttributeError, OSError): pass master_fd, slave_name = _open_terminal() - slave_fd = slave_open(slave_name) - return master_fd, slave_fd - -def master_open(): - """master_open() -> (master_fd, slave_name) - Open a pty master and return the fd, and the filename of the slave end. - Deprecated, use openpty() instead.""" - - import warnings - warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 + slave_fd = os.open(slave_name, os.O_RDWR) try: - master_fd, slave_fd = os.openpty() - except (AttributeError, OSError): + from fcntl import ioctl, I_PUSH + except ImportError: + return master_fd, slave_fd + try: + ioctl(slave_fd, I_PUSH, "ptem") + ioctl(slave_fd, I_PUSH, "ldterm") + except OSError: pass - else: - slave_name = os.ttyname(slave_fd) - os.close(slave_fd) - return master_fd, slave_name - - return _open_terminal() + return master_fd, slave_fd def _open_terminal(): """Open pty master and return (master_fd, tty_name).""" @@ -66,26 +57,6 @@ def _open_terminal(): return (fd, '/dev/tty' + x + y) raise OSError('out of pty devices') -def slave_open(tty_name): - """slave_open(tty_name) -> slave_fd - Open the pty slave and acquire the controlling terminal, returning - opened filedescriptor. - Deprecated, use openpty() instead.""" - - import warnings - warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 - - result = os.open(tty_name, os.O_RDWR) - try: - from fcntl import ioctl, I_PUSH - except ImportError: - return result - try: - ioctl(result, I_PUSH, "ptem") - ioctl(result, I_PUSH, "ldterm") - except OSError: - pass - return result def fork(): """fork() -> (pid, master_fd) diff --git a/Lib/pydoc.py b/Lib/pydoc.py old mode 100755 new mode 100644 index 96aa1dfc1aacf6..c863794ea14ef9 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Generate Python documentation in HTML or text for interactive use. At the Python interactive prompt, calling help(thing) on a Python object @@ -72,10 +71,26 @@ class or function within a module or module in a package. If the import tokenize import urllib.parse import warnings +from annotationlib import Format from collections import deque from reprlib import Repr from traceback import format_exception_only +from _pyrepl.pager import (get_pager, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + +# Expose plain() as pydoc.plain() +from _pyrepl.pager import plain # noqa: F401 + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + # --------------------------------------------------------- common routines @@ -127,9 +142,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): @@ -199,12 +213,12 @@ def splitdoc(doc): def _getargspec(object): try: - signature = inspect.signature(object) + signature = inspect.signature(object, annotation_format=Format.STRING) if signature: name = getattr(object, '__name__', '') # function are always single-line and should not be formatted max_width = (80 - len(name)) if name != '' else None - return signature.format(max_width=max_width) + return signature.format(max_width=max_width, quote_annotation_strings=False) except (ValueError, TypeError): argspec = getattr(object, '__text_signature__', None) if argspec: @@ -225,6 +239,19 @@ def classname(object, modname): name = object.__module__ + '.' + name return name +def parentname(object, modname): + """Get a name of the enclosing class (qualified it with a module name + if necessary) or module.""" + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + return object.__module__ + '.' + name + else: + return name + else: + if object.__module__ != modname: + return object.__module__ + def isdata(object): """Check if an object is of a type that probably means it's data.""" return not (inspect.ismodule(object) or inspect.isclass(object) or @@ -301,7 +328,8 @@ def visiblename(name, all=None, obj=None): if name in {'__author__', '__builtins__', '__cached__', '__credits__', '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', - '__path__', '__qualname__', '__slots__', '__version__'}: + '__path__', '__qualname__', '__slots__', '__version__', + '__static_attributes__', '__firstlineno__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 @@ -319,13 +347,15 @@ def visiblename(name, all=None, obj=None): return not name.startswith('_') def classify_class_attrs(object): - """Wrap inspect.classify_class_attrs, with fixup for data descriptors.""" + """Wrap inspect.classify_class_attrs, with fixup for data descriptors and bound methods.""" results = [] for (name, kind, cls, value) in inspect.classify_class_attrs(object): if inspect.isdatadescriptor(value): kind = 'data descriptor' if isinstance(value, property) and value.fset is None: kind = 'readonly property' + elif kind == 'method' and _is_bound_method(value): + kind = 'static method' results.append((name, kind, cls, value)) return results @@ -537,7 +567,7 @@ def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): '_thread', 'zipimport') or (file.startswith(basedir) and not file.startswith(os.path.join(basedir, 'site-packages')))) and - object.__name__ not in ('xml.etree', 'test.pydoc_mod')): + object.__name__ not in ('xml.etree', 'test.test_pydoc.pydoc_mod')): if docloc.startswith(("http://", "https://")): docloc = "{}/{}.html".format(docloc.rstrip("/"), object.__name__.lower()) else: @@ -681,6 +711,25 @@ def classlink(self, object, modname): module.__name__, name, classname(object, modname)) return classname(object, modname) + def parentlink(self, object, modname): + """Make a link for the enclosing class or module.""" + link = None + name, module = object.__name__, sys.modules.get(object.__module__) + if hasattr(module, name) and getattr(module, name) is object: + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + link = '%s.html#%s' % (module.__name__, name) + else: + link = '#%s' % name + else: + if object.__module__ != modname: + link = '%s.html' % module.__name__ + if link: + return '%s' % (link, parentname(object, modname)) + else: + return parentname(object, modname) + def modulelink(self, object): """Make a link for a module.""" return '%s' % (object.__name__, object.__name__) @@ -822,9 +871,9 @@ def docmodule(self, object, name=None, mod=None, *ignored): cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key @@ -925,7 +974,7 @@ def spill(msg, attrs, predicate): push(self.docdata(value, name, mod)) else: push(self.document(value, name, mod, - funcs, classes, mdict, object)) + funcs, classes, mdict, object, homecls)) push('\n') return attrs @@ -1043,24 +1092,44 @@ def formatvalue(self, object): return self.grey('=' + self.repr(object)) def docroutine(self, object, name=None, mod=None, - funcs={}, classes={}, methods={}, cl=None): + funcs={}, classes={}, methods={}, cl=None, homecls=None): """Produce HTML documentation for a function or method object.""" realname = object.__name__ name = name or realname - anchor = (cl and cl.__name__ or '') + '-' + name + if homecls is None: + homecls = cl + anchor = ('' if cl is None else cl.__name__) + '-' + name note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + self.classlink(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % self.classlink(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % self.classlink( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % self.classlink(imclass,mod) + note = ' method of %s instance' % self.classlink( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % self.classlink(objclass, mod) + elif objclass is not homecls: + note = ' from ' + self.classlink(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = self.parentlink(imfunc, mod) + if pname: + note = ' from %s' % pname if (inspect.iscoroutinefunction(object) or inspect.isasyncgenfunction(object)): @@ -1071,10 +1140,13 @@ def docroutine(self, object, name=None, mod=None, if name == realname: title = '%s' % (anchor, realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): reallink = '%s' % ( cl.__name__ + '-' + realname, realname) - skipdocs = 1 + skipdocs = True + if note.startswith(' from '): + note = '' else: reallink = realname title = '%s = %s' % ( @@ -1087,7 +1159,8 @@ def docroutine(self, object, name=None, mod=None, # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + if not object.__annotations__: + argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' @@ -1102,7 +1175,7 @@ def docroutine(self, object, name=None, mod=None, doc = doc and '
    %s
    ' % doc return '
    %s
    %s
    \n' % (decl, doc) - def docdata(self, object, name=None, mod=None, cl=None): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): """Produce html documentation for a data descriptor.""" results = [] push = results.append @@ -1213,7 +1286,7 @@ def formattree(self, tree, modname, parent=None, prefix=''): entry, modname, c, prefix + ' ') return result - def docmodule(self, object, name=None, mod=None): + def docmodule(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given module object.""" name = object.__name__ # ignore the passed-in name synop, desc = splitdoc(getdoc(object)) @@ -1242,9 +1315,9 @@ def docmodule(self, object, name=None, mod=None): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] @@ -1392,7 +1465,7 @@ def spill(msg, attrs, predicate): push(self.docdata(value, name, mod)) else: push(self.document(value, - name, mod, object)) + name, mod, object, homecls)) return attrs def spilldescriptors(msg, attrs, predicate): @@ -1467,23 +1540,43 @@ def formatvalue(self, object): """Format an argument default value as text.""" return '=' + self.repr(object) - def docroutine(self, object, name=None, mod=None, cl=None): + def docroutine(self, object, name=None, mod=None, cl=None, homecls=None): """Produce text documentation for a function or method object.""" realname = object.__name__ name = name or realname + if homecls is None: + homecls = cl note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + classname(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % classname(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % classname( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % classname(imclass,mod) + note = ' method of %s instance' % classname( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % classname(objclass, mod) + elif objclass is not homecls: + note = ' from ' + classname(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = parentname(imfunc, mod) + if pname: + note = ' from %s' % pname if (inspect.iscoroutinefunction(object) or inspect.isasyncgenfunction(object)): @@ -1494,8 +1587,11 @@ def docroutine(self, object, name=None, mod=None, cl=None): if name == realname: title = self.bold(realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: - skipdocs = 1 + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): + skipdocs = True + if note.startswith(' from '): + note = '' title = self.bold(name) + ' = ' + realname argspec = None @@ -1506,7 +1602,8 @@ def docroutine(self, object, name=None, mod=None, cl=None): # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + if not object.__annotations__: + argspec = argspec[1:-1] if not argspec: argspec = '(...)' decl = asyncqualifier + title + argspec + note @@ -1517,7 +1614,7 @@ def docroutine(self, object, name=None, mod=None, cl=None): doc = getdoc(object) or '' return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') - def docdata(self, object, name=None, mod=None, cl=None): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): """Produce text documentation for a data descriptor.""" results = [] push = results.append @@ -1533,7 +1630,8 @@ def docdata(self, object, name=None, mod=None, cl=None): docproperty = docdata - def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=None): + def docother(self, object, name=None, mod=None, parent=None, *ignored, + maxlen=None, doc=None): """Produce text documentation for a data object.""" repr = self.repr(object) if maxlen: @@ -1554,140 +1652,11 @@ def bold(self, text): # --------------------------------------------------------- user interfaces -def pager(text): +def pager(text, title=''): """The first time this is called, determine what kind of pager to use.""" global pager - pager = getpager() - pager(text) - -def getpager(): - """Decide what method to use for paging through text.""" - if not hasattr(sys.stdin, "isatty"): - return plainpager - if not hasattr(sys.stdout, "isatty"): - return plainpager - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return plainpager - if sys.platform == "emscripten": - return plainpager - use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') - if use_pager: - if sys.platform == 'win32': # pipes completely broken in Windows - return lambda text: tempfilepager(plain(text), use_pager) - elif os.environ.get('TERM') in ('dumb', 'emacs'): - return lambda text: pipepager(plain(text), use_pager) - else: - return lambda text: pipepager(text, use_pager) - if os.environ.get('TERM') in ('dumb', 'emacs'): - return plainpager - if sys.platform == 'win32': - return lambda text: tempfilepager(plain(text), 'more <') - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return lambda text: pipepager(text, 'less') - - import tempfile - (fd, filename) = tempfile.mkstemp() - os.close(fd) - try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return lambda text: pipepager(text, 'more') - else: - return ttypager - finally: - os.unlink(filename) - -def plain(text): - """Remove boldface formatting from text.""" - return re.sub('.\b', '', text) - -def pipepager(text, cmd): - """Page through text by feeding it to another program.""" - import subprocess - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - errors='backslashreplace') - try: - with proc.stdin as pipe: - try: - pipe.write(text) - except KeyboardInterrupt: - # We've hereby abandoned whatever text hasn't been written, - # but the pager is still in control of the terminal. - pass - except OSError: - pass # Ignore broken pipes caused by quitting the pager program. - while True: - try: - proc.wait() - break - except KeyboardInterrupt: - # Ignore ctl-c like the pager itself does. Otherwise the pager is - # left running and the terminal is in raw mode and unusable. - pass - -def tempfilepager(text, cmd): - """Page through text by invoking a program on a temporary file.""" - import tempfile - with tempfile.TemporaryDirectory() as tempdir: - filename = os.path.join(tempdir, 'pydoc.out') - with open(filename, 'w', errors='backslashreplace', - encoding=os.device_encoding(0) if - sys.platform == 'win32' else None - ) as file: - file.write(text) - os.system(cmd + ' "' + filename + '"') - -def _escape_stdout(text): - # Escape non-encodable characters to avoid encoding errors later - encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' - return text.encode(encoding, 'backslashreplace').decode(encoding) - -def ttypager(text): - """Page through text on a text terminal.""" - lines = plain(_escape_stdout(text)).split('\n') - try: - import tty - fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) - tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) - except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None - getchar = lambda: sys.stdin.readline()[:-1][:1] - - try: - try: - h = int(os.environ.get('LINES', 0)) - except ValueError: - h = 0 - if h <= 1: - h = 25 - r = inc = h - 1 - sys.stdout.write('\n'.join(lines[:inc]) + '\n') - while lines[r:]: - sys.stdout.write('-- more --') - sys.stdout.flush() - c = getchar() - - if c in ('q', 'Q'): - sys.stdout.write('\r \r') - break - elif c in ('\r', '\n'): - sys.stdout.write('\r \r' + lines[r] + '\n') - r = r + 1 - continue - if c in ('b', 'B', '\x1b'): - r = r - inc - inc - if r < 0: r = 0 - sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') - r = r + inc - - finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) - -def plainpager(text): - """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(_escape_stdout(text))) + pager = get_pager() + pager(text, title) def describe(thing): """Produce a short description of the given thing.""" @@ -1714,6 +1683,13 @@ def describe(thing): return 'function ' + thing.__name__ if inspect.ismethod(thing): return 'method ' + thing.__name__ + if inspect.ismethodwrapper(thing): + return 'method wrapper ' + thing.__name__ + if inspect.ismethoddescriptor(thing): + try: + return 'method descriptor ' + thing.__name__ + except AttributeError: + pass return type(thing).__name__ def locate(path, forceload=0): @@ -1787,7 +1763,15 @@ def doc(thing, title='Python Library Documentation: %s', forceload=0, """Display text documentation, given an object or a path to an object.""" if output is None: try: - pager(render_doc(thing, title, forceload)) + if isinstance(thing, str): + what = thing + else: + what = getattr(thing, '__qualname__', None) + if not isinstance(what, str): + what = getattr(thing, '__name__', None) + if not isinstance(what, str): + what = type(thing).__name__ + ' object' + pager(render_doc(thing, title, forceload), f'Help on {what!s}') except ImportError as exc: if is_cli: raise @@ -1887,6 +1871,7 @@ class Helper: ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', @@ -1980,6 +1965,7 @@ class Helper: 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', @@ -2041,7 +2027,7 @@ def interact(self): if (len(request) > 2 and request[0] == request[-1] in ("'", '"') and request[0] not in request[1:-1]): request = request[1:-1] - if request.lower() in ('q', 'quit'): break + if request.lower() in ('q', 'quit', 'exit'): break if request == 'help': self.intro() else: @@ -2068,7 +2054,7 @@ def help(self, request, is_cli=False): elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too - doc(eval(request), 'Help on %s:', is_cli=is_cli) + doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli) elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) @@ -2093,7 +2079,7 @@ def intro(self): enter "modules spam". To quit this help utility and return to the interpreter, -enter "q" or "quit". +enter "q", "quit" or "exit". '''.format('%d.%d' % sys.version_info[:2])) def list(self, items, columns=4, width=80): @@ -2161,7 +2147,11 @@ def showtopic(self, topic, more_xrefs=''): text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) - pager(doc) + + if self._output is None: + pager(doc, f'Help on {topic!s}') + else: + self.output.write(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). @@ -2413,6 +2403,7 @@ def __init__(self, urlhandler, host, port): threading.Thread.__init__(self) self.serving = False self.error = None + self.docserver = None def run(self): """Start the server.""" @@ -2445,9 +2436,9 @@ def stop(self): thread = ServerThread(urlhandler, hostname, port) thread.start() - # Wait until thread.serving is True to make sure we are - # really up before returning. - while not thread.error and not thread.serving: + # Wait until thread.serving is True and thread.docserver is set + # to make sure we are really up before returning. + while not thread.error and not (thread.serving and thread.docserver): time.sleep(.01) return thread diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index d453c1f038ef07..165d3853f95e29 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Wed Jan 17 13:09:41 2024 +# Autogenerated by Sphinx on Tue Oct 15 22:34:02 2024 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -308,10 +308,10 @@ 'target.\n' 'The target is only evaluated once.\n' '\n' - 'An augmented assignment expression like "x += 1" can be ' - 'rewritten as\n' - '"x = x + 1" to achieve a similar, but not exactly equal ' - 'effect. In the\n' + 'An augmented assignment statement like "x += 1" can be ' + 'rewritten as "x\n' + '= x + 1" to achieve a similar, but not exactly equal effect. ' + 'In the\n' 'augmented version, "x" is only evaluated once. Also, when ' 'possible,\n' 'the actual operation is performed *in-place*, meaning that ' @@ -362,21 +362,21 @@ 'a single\n' 'target is allowed.\n' '\n' - 'For simple names as assignment targets, if in class or module ' - 'scope,\n' - 'the annotations are evaluated and stored in a special class or ' - 'module\n' - 'attribute "__annotations__" that is a dictionary mapping from ' - 'variable\n' - 'names (mangled if private) to evaluated annotations. This ' - 'attribute is\n' - 'writable and is automatically created at the start of class or ' - 'module\n' - 'body execution, if annotations are found statically.\n' - '\n' - 'For expressions as assignment targets, the annotations are ' + 'The assignment target is considered “simple” if it consists of ' + 'a\n' + 'single name that is not enclosed in parentheses. For simple ' + 'assignment\n' + 'targets, if in class or module scope, the annotations are ' + 'gathered in\n' + 'a lazily evaluated annotation scope. The annotations can be ' 'evaluated\n' - 'if in class or module scope, but not stored.\n' + 'using the "__annotations__" attribute of a class or module, or ' + 'using\n' + 'the facilities in the "annotationlib" module.\n' + '\n' + 'If the assignment target is not simple (an attribute, ' + 'subscript node,\n' + 'or parenthesized name), the annotation is never evaluated.\n' '\n' 'If a name is annotated in a function scope, then this name is ' 'local\n' @@ -386,12 +386,13 @@ '\n' 'If the right hand side is present, an annotated assignment ' 'performs\n' - 'the actual assignment before evaluating annotations (where\n' - 'applicable). If the right hand side is not present for an ' - 'expression\n' - 'target, then the interpreter evaluates the target except for ' - 'the last\n' - '"__setitem__()" or "__setattr__()" call.\n' + 'the actual assignment as if there was no annotation present. ' + 'If the\n' + 'right hand side is not present for an expression target, then ' + 'the\n' + 'interpreter evaluates the target except for the last ' + '"__setitem__()"\n' + 'or "__setattr__()" call.\n' '\n' 'See also:\n' '\n' @@ -415,11 +416,55 @@ 'Previously,\n' 'some expressions (like un-parenthesized tuple expressions) ' 'caused a\n' - 'syntax error.\n', + 'syntax error.\n' + '\n' + 'Changed in version 3.14: Annotations are now lazily evaluated ' + 'in a\n' + 'separate annotation scope. If the assignment target is not ' + 'simple,\n' + 'annotations are never evaluated.\n', + 'assignment-expressions': 'Assignment expressions\n' + '**********************\n' + '\n' + ' assignment_expression ::= [identifier ":="] ' + 'expression\n' + '\n' + 'An assignment expression (sometimes also called a ' + '“named expression”\n' + 'or “walrus”) assigns an "expression" to an ' + '"identifier", while also\n' + 'returning the value of the "expression".\n' + '\n' + 'One common use case is when handling matched ' + 'regular expressions:\n' + '\n' + ' if matching := pattern.search(data):\n' + ' do_something(matching)\n' + '\n' + 'Or, when processing a file stream in chunks:\n' + '\n' + ' while chunk := file.read(9000):\n' + ' process(chunk)\n' + '\n' + 'Assignment expressions must be surrounded by ' + 'parentheses when used as\n' + 'expression statements and when used as ' + 'sub-expressions in slicing,\n' + 'conditional, lambda, keyword-argument, and ' + 'comprehension-if\n' + 'expressions and in "assert", "with", and ' + '"assignment" statements. In\n' + 'all other places where they can be used, ' + 'parentheses are not required,\n' + 'including in "if" and "while" statements.\n' + '\n' + 'Added in version 3.8: See **PEP 572** for more ' + 'details about\n' + 'assignment expressions.\n', 'async': 'Coroutines\n' '**********\n' '\n' - 'New in version 3.5.\n' + 'Added in version 3.5.\n' '\n' '\n' 'Coroutine function definition\n' @@ -555,31 +600,67 @@ 'evaluate it\n' 'raises a "NameError" exception.\n' '\n' - '**Private name mangling:** When an identifier that ' - 'textually occurs in\n' - 'a class definition begins with two or more underscore ' - 'characters and\n' - 'does not end in two or more underscores, it is ' - 'considered a *private\n' - 'name* of that class. Private names are transformed to a ' - 'longer form\n' - 'before code is generated for them. The transformation ' - 'inserts the\n' - 'class name, with leading underscores removed and a ' - 'single underscore\n' - 'inserted, in front of the name. For example, the ' - 'identifier "__spam"\n' - 'occurring in a class named "Ham" will be transformed to ' - '"_Ham__spam".\n' - 'This transformation is independent of the syntactical ' + '\n' + 'Private name mangling\n' + '=====================\n' + '\n' + 'When an identifier that textually occurs in a class ' + 'definition begins\n' + 'with two or more underscore characters and does not end ' + 'in two or more\n' + 'underscores, it is considered a *private name* of that ' + 'class.\n' + '\n' + 'See also: The class specifications.\n' + '\n' + 'More precisely, private names are transformed to a ' + 'longer form before\n' + 'code is generated for them. If the transformed name is ' + 'longer than\n' + '255 characters, implementation-defined truncation may ' + 'happen.\n' + '\n' + 'The transformation is independent of the syntactical ' 'context in which\n' - 'the identifier is used. If the transformed name is ' - 'extremely long\n' - '(longer than 255 characters), implementation defined ' - 'truncation may\n' - 'happen. If the class name consists only of underscores, ' - 'no\n' - 'transformation is done.\n', + 'the identifier is used but only the following private ' + 'identifiers are\n' + 'mangled:\n' + '\n' + '* Any name used as the name of a variable that is ' + 'assigned or read or\n' + ' any name of an attribute being accessed.\n' + '\n' + ' The "__name__" attribute of nested functions, classes, ' + 'and type\n' + ' aliases is however not mangled.\n' + '\n' + '* The name of imported modules, e.g., "__spam" in ' + '"import __spam". If\n' + ' the module is part of a package (i.e., its name ' + 'contains a dot), the\n' + ' name is *not* mangled, e.g., the "__foo" in "import ' + '__foo.bar" is\n' + ' not mangled.\n' + '\n' + '* The name of an imported member, e.g., "__f" in "from ' + 'spam import\n' + ' __f".\n' + '\n' + 'The transformation rule is defined as follows:\n' + '\n' + '* The class name, with leading underscores removed and a ' + 'single\n' + ' leading underscore inserted, is inserted in front of ' + 'the identifier,\n' + ' e.g., the identifier "__spam" occurring in a class ' + 'named "Foo",\n' + ' "_Foo" or "__Foo" is transformed to "_Foo__spam".\n' + '\n' + '* If the class name consists only of underscores, the ' + 'transformation\n' + ' is the identity, e.g., the identifier "__spam" ' + 'occurring in a class\n' + ' named "_" or "__" is left as is.\n', 'atom-literals': 'Literals\n' '********\n' '\n' @@ -592,10 +673,10 @@ '\n' 'Evaluation of a literal yields an object of the given type ' '(string,\n' - 'bytes, integer, floating point number, complex number) with ' + 'bytes, integer, floating-point number, complex number) with ' 'the given\n' 'value. The value may be approximated in the case of ' - 'floating point\n' + 'floating-point\n' 'and imaginary (complex) literals. See section Literals for ' 'details.\n' '\n' @@ -722,9 +803,9 @@ '\n' 'object.__dir__(self)\n' '\n' - ' Called when "dir()" is called on the object. A ' - 'sequence must be\n' - ' returned. "dir()" converts the returned sequence to a ' + ' Called when "dir()" is called on the object. An ' + 'iterable must be\n' + ' returned. "dir()" converts the returned iterable to a ' 'list and\n' ' sorts it.\n' '\n' @@ -751,8 +832,8 @@ 'returned.\n' '\n' 'The "__dir__" function should accept no arguments, and ' - 'return a\n' - 'sequence of strings that represents the names accessible ' + 'return an\n' + 'iterable of strings that represents the names accessible ' 'on module. If\n' 'present, this function overrides the standard "dir()" ' 'search on a\n' @@ -792,7 +873,7 @@ 'Changed in version 3.5: "__class__" module attribute is ' 'now writable.\n' '\n' - 'New in version 3.7: "__getattr__" and "__dir__" module ' + 'Added in version 3.7: "__getattr__" and "__dir__" module ' 'attributes.\n' '\n' 'See also:\n' @@ -1053,11 +1134,13 @@ 'to the class\n' ' where it is defined. *__slots__* declared in parents ' 'are available\n' - ' in child classes. However, child subclasses will get a ' - '"__dict__"\n' - ' and *__weakref__* unless they also define *__slots__* ' - '(which should\n' - ' only contain names of any *additional* slots).\n' + ' in child classes. However, instances of a child ' + 'subclass will get a\n' + ' "__dict__" and *__weakref__* unless the subclass also ' + 'defines\n' + ' *__slots__* (which should only contain names of any ' + '*additional*\n' + ' slots).\n' '\n' '* If a class defines a slot also defined in a base ' 'class, the instance\n' @@ -1163,10 +1246,10 @@ 'target.\n' 'The target is only evaluated once.\n' '\n' - 'An augmented assignment expression like "x += 1" can be ' - 'rewritten as\n' - '"x = x + 1" to achieve a similar, but not exactly equal effect. ' - 'In the\n' + 'An augmented assignment statement like "x += 1" can be ' + 'rewritten as "x\n' + '= x + 1" to achieve a similar, but not exactly equal effect. In ' + 'the\n' 'augmented version, "x" is only evaluated once. Also, when ' 'possible,\n' 'the actual operation is performed *in-place*, meaning that ' @@ -1206,7 +1289,7 @@ '\n' ' await_expr ::= "await" primary\n' '\n' - 'New in version 3.5.\n', + 'Added in version 3.5.\n', 'binary': 'Binary arithmetic operations\n' '****************************\n' '\n' @@ -1239,7 +1322,11 @@ 'The "@" (at) operator is intended to be used for matrix\n' 'multiplication. No builtin Python types implement this operator.\n' '\n' - 'New in version 3.5.\n' + 'This operation can be customized using the special "__matmul__()" ' + 'and\n' + '"__rmatmul__()" methods.\n' + '\n' + 'Added in version 3.5.\n' '\n' 'The "/" (division) and "//" (floor division) operators yield the\n' 'quotient of their arguments. The numeric arguments are first\n' @@ -1251,17 +1338,19 @@ 'result. Division by zero raises the "ZeroDivisionError" ' 'exception.\n' '\n' - 'This operation can be customized using the special "__truediv__()" ' + 'The division operation can be customized using the special\n' + '"__truediv__()" and "__rtruediv__()" methods. The floor division\n' + 'operation can be customized using the special "__floordiv__()" ' 'and\n' - '"__floordiv__()" methods.\n' + '"__rfloordiv__()" methods.\n' '\n' 'The "%" (modulo) operator yields the remainder from the division ' 'of\n' 'the first argument by the second. The numeric arguments are ' 'first\n' 'converted to a common type. A zero right argument raises the\n' - '"ZeroDivisionError" exception. The arguments may be floating ' - 'point\n' + '"ZeroDivisionError" exception. The arguments may be ' + 'floating-point\n' 'numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals ' '"4*0.7 +\n' '0.34".) The modulo operator always yields a result with the same ' @@ -1288,13 +1377,13 @@ '\n' 'The *modulo* operation can be customized using the special ' '"__mod__()"\n' - 'method.\n' + 'and "__rmod__()" methods.\n' '\n' 'The floor division operator, the modulo operator, and the ' '"divmod()"\n' 'function are not defined for complex numbers. Instead, convert to ' 'a\n' - 'floating point number using the "abs()" function if appropriate.\n' + 'floating-point number using the "abs()" function if appropriate.\n' '\n' 'The "+" (addition) operator yields the sum of its arguments. The\n' 'arguments must either both be numbers or both be sequences of the ' @@ -1313,7 +1402,8 @@ 'The numeric arguments are first converted to a common type.\n' '\n' 'This operation can be customized using the special "__sub__()" ' - 'method.\n', + 'and\n' + '"__rsub__()" methods.\n', 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' @@ -2388,18 +2478,16 @@ 'An\n' 'expression-less "except" clause, if present, must be last; it ' 'matches\n' - 'any exception. For an "except" clause with an expression, that\n' - 'expression is evaluated, and the clause matches the exception if ' - 'the\n' - 'resulting object is “compatible” with the exception. An object ' - 'is\n' - 'compatible with an exception if the object is the class or a ' - '*non-\n' - 'virtual base class* of the exception object, or a tuple ' - 'containing an\n' - 'item that is the class or a non-virtual base class of the ' - 'exception\n' - 'object.\n' + 'any exception.\n' + '\n' + 'For an "except" clause with an expression, the expression must\n' + 'evaluate to an exception type or a tuple of exception types. ' + 'The\n' + 'raised exception matches an "except" clause whose expression ' + 'evaluates\n' + 'to the class or a *non-virtual base class* of the exception ' + 'object, or\n' + 'to a tuple that contains such a class.\n' '\n' 'If no "except" clause matches the exception, the search for an\n' 'exception handler continues in the surrounding code and on the\n' @@ -2548,13 +2636,16 @@ ' ...\n' " ExceptionGroup('', (BlockingIOError()))\n" '\n' - 'An "except*" clause must have a matching type, and this type ' - 'cannot be\n' - 'a subclass of "BaseExceptionGroup". It is not possible to mix ' - '"except"\n' - 'and "except*" in the same "try". "break", "continue" and ' - '"return"\n' - 'cannot appear in an "except*" clause.\n' + 'An "except*" clause must have a matching expression; it cannot ' + 'be\n' + '"except*:". Furthermore, this expression cannot contain ' + 'exception\n' + 'group types, because that would have ambiguous semantics.\n' + '\n' + 'It is not possible to mix "except" and "except*" in the same ' + '"try".\n' + '"break", "continue" and "return" cannot appear in an "except*" ' + 'clause.\n' '\n' '\n' '"else" clause\n' @@ -2765,7 +2856,7 @@ 'The "match" statement\n' '=====================\n' '\n' - 'New in version 3.10.\n' + 'Added in version 3.10.\n' '\n' 'The match statement is used for pattern matching. Syntax:\n' '\n' @@ -3052,7 +3143,7 @@ ' | "None"\n' ' | "True"\n' ' | "False"\n' - ' | signed_number: NUMBER | "-" NUMBER\n' + ' signed_number ::= ["-"] NUMBER\n' '\n' 'The rule "strings" and the token "NUMBER" are defined in the ' 'standard\n' @@ -3522,10 +3613,12 @@ ' parameter_list_no_posonly ::= defparameter ("," ' 'defparameter)* ["," [parameter_list_starargs]]\n' ' | parameter_list_starargs\n' - ' parameter_list_starargs ::= "*" [parameter] ("," ' + ' parameter_list_starargs ::= "*" [star_parameter] ("," ' 'defparameter)* ["," ["**" parameter [","]]]\n' ' | "**" parameter [","]\n' ' parameter ::= identifier [":" expression]\n' + ' star_parameter ::= identifier [":" ["*"] ' + 'expression]\n' ' defparameter ::= parameter ["=" expression]\n' ' funcname ::= identifier\n' '\n' @@ -3653,27 +3746,22 @@ 'expression"”\n' 'following the parameter name. Any parameter may have an ' 'annotation,\n' - 'even those of the form "*identifier" or "**identifier". ' - 'Functions may\n' - 'have “return” annotation of the form “"-> expression"” after ' - 'the\n' - 'parameter list. These annotations can be any valid Python ' - 'expression.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - 'dictionary keyed by the parameters’ names in the ' - '"__annotations__"\n' - 'attribute of the function object. If the "annotations" import ' - 'from\n' - '"__future__" is used, annotations are preserved as strings at ' - 'runtime\n' - 'which enables postponed evaluation. Otherwise, they are ' - 'evaluated\n' - 'when the function definition is executed. In this case ' - 'annotations\n' - 'may be evaluated in a different order than they appear in the ' - 'source\n' - 'code.\n' + 'even those of the form "*identifier" or "**identifier". (As a ' + 'special\n' + 'case, parameters of the form "*identifier" may have an ' + 'annotation “":\n' + '*expression"”.) Functions may have “return” annotation of the ' + 'form\n' + '“"-> expression"” after the parameter list. These annotations ' + 'can be\n' + 'any valid Python expression. The presence of annotations does ' + 'not\n' + 'change the semantics of a function. See Annotations for more\n' + 'information on annotations.\n' + '\n' + 'Changed in version 3.11: Parameters of the form “"*identifier"” ' + 'may\n' + 'have an annotation “": *expression"”. See **PEP 646**.\n' '\n' 'It is also possible to create anonymous functions (functions not ' 'bound\n' @@ -3849,7 +3937,7 @@ 'Coroutines\n' '==========\n' '\n' - 'New in version 3.5.\n' + 'Added in version 3.5.\n' '\n' '\n' 'Coroutine function definition\n' @@ -3976,13 +4064,18 @@ 'Type parameter lists\n' '====================\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' + '\n' + 'Changed in version 3.13: Support for default values was added ' + '(see\n' + '**PEP 696**).\n' '\n' ' type_params ::= "[" type_param ("," type_param)* "]"\n' ' type_param ::= typevar | typevartuple | paramspec\n' - ' typevar ::= identifier (":" expression)?\n' - ' typevartuple ::= "*" identifier\n' - ' paramspec ::= "**" identifier\n' + ' typevar ::= identifier (":" expression)? ("=" ' + 'expression)?\n' + ' typevartuple ::= "*" identifier ("=" expression)?\n' + ' paramspec ::= "**" identifier ("=" expression)?\n' '\n' 'Functions (including coroutines), classes and type aliases may ' 'contain\n' @@ -4081,21 +4174,41 @@ 'bounds or\n' 'constraints.\n' '\n' + 'All three flavors of type parameters can also have a *default ' + 'value*,\n' + 'which is used when the type parameter is not explicitly ' + 'provided. This\n' + 'is added by appending a single equals sign ("=") followed by an\n' + 'expression. Like the bounds and constraints of type variables, ' + 'the\n' + 'default value is not evaluated when the object is created, but ' + 'only\n' + 'when the type parameter’s "__default__" attribute is accessed. ' + 'To this\n' + 'end, the default value is evaluated in a separate annotation ' + 'scope. If\n' + 'no default value is specified for a type parameter, the ' + '"__default__"\n' + 'attribute is set to the special sentinel object ' + '"typing.NoDefault".\n' + '\n' 'The following example indicates the full set of allowed type ' 'parameter\n' 'declarations:\n' '\n' ' def overly_generic[\n' ' SimpleTypeVar,\n' + ' TypeVarWithDefault = int,\n' ' TypeVarWithBound: int,\n' ' TypeVarWithConstraints: (str, bytes),\n' - ' *SimpleTypeVarTuple,\n' - ' **SimpleParamSpec,\n' + ' *SimpleTypeVarTuple = (int, float),\n' + ' **SimpleParamSpec = (str, bytearray),\n' ' ](\n' ' a: SimpleTypeVar,\n' - ' b: TypeVarWithBound,\n' - ' c: Callable[SimpleParamSpec, TypeVarWithConstraints],\n' - ' *d: SimpleTypeVarTuple,\n' + ' b: TypeVarWithDefault,\n' + ' c: TypeVarWithBound,\n' + ' d: Callable[SimpleParamSpec, TypeVarWithConstraints],\n' + ' *e: SimpleTypeVarTuple,\n' ' ): ...\n' '\n' '\n' @@ -4238,6 +4351,56 @@ 'not\n' 'actually bound at runtime.\n' '\n' + '\n' + 'Annotations\n' + '===========\n' + '\n' + 'Changed in version 3.14: Annotations are now lazily evaluated ' + 'by\n' + 'default.\n' + '\n' + 'Variables and function parameters may carry *annotations*, ' + 'created by\n' + 'adding a colon after the name, followed by an expression:\n' + '\n' + ' x: annotation = 1\n' + ' def f(param: annotation): ...\n' + '\n' + 'Functions may also carry a return annotation following an ' + 'arrow:\n' + '\n' + ' def f() -> annotation: ...\n' + '\n' + 'Annotations are conventionally used for *type hints*, but this ' + 'is not\n' + 'enforced by the language, and in general annotations may ' + 'contain\n' + 'arbitrary expressions. The presence of annotations does not ' + 'change the\n' + 'runtime semantics of the code, except if some mechanism is used ' + 'that\n' + 'introspects and uses the annotations (such as "dataclasses" or\n' + '"functools.singledispatch()").\n' + '\n' + 'By default, annotations are lazily evaluated in a annotation ' + 'scope.\n' + 'This means that they are not evaluated when the code containing ' + 'the\n' + 'annotation is evaluated. Instead, the interpreter saves ' + 'information\n' + 'that can be used to evaluate the annotation later if requested. ' + 'The\n' + '"annotationlib" module provides tools for evaluating ' + 'annotations.\n' + '\n' + 'If the future statement "from __future__ import annotations" is\n' + 'present, all annotations are instead stored as strings:\n' + '\n' + ' >>> from __future__ import annotations\n' + ' >>> def f(param: annotation): ...\n' + ' >>> f.__annotations__\n' + " {'param': 'annotation'}\n" + '\n' '-[ Footnotes ]-\n' '\n' '[1] The exception is propagated to the invocation stack unless ' @@ -4400,7 +4563,7 @@ 'converted to\n' ' complex;\n' '\n' - '* otherwise, if either argument is a floating point number, ' + '* otherwise, if either argument is a floating-point number, ' 'the other\n' ' is converted to floating point;\n' '\n' @@ -4511,6 +4674,10 @@ ' It is not guaranteed that "__del__()" methods are called ' 'for\n' ' objects that still exist when the interpreter exits.\n' + ' "weakref.finalize" provides a straightforward way to ' + 'register a\n' + ' cleanup function to be called when an object is garbage ' + 'collected.\n' '\n' ' Note:\n' '\n' @@ -4724,7 +4891,7 @@ 'reflection,\n' ' and "__eq__()" and "__ne__()" are their own reflection. ' 'If the\n' - ' operands are of different types, and right operand’s ' + ' operands are of different types, and the right operand’s ' 'type is a\n' ' direct or indirect subclass of the left operand’s type, ' 'the\n' @@ -4734,6 +4901,11 @@ 'is not\n' ' considered.\n' '\n' + ' When no appropriate method returns any value other than\n' + ' "NotImplemented", the "==" and "!=" operators will fall ' + 'back to\n' + ' "is" and "is not", respectively.\n' + '\n' 'object.__hash__(self)\n' '\n' ' Called by built-in function "hash()" and for operations ' @@ -4921,7 +5093,7 @@ 'and continue running without the debugger using the "continue"\n' 'command.\n' '\n' - 'New in version 3.7: The built-in "breakpoint()", when called ' + 'Changed in version 3.7: The built-in "breakpoint()", when called ' 'with\n' 'defaults, can be used instead of "import pdb; pdb.set_trace()".\n' '\n' @@ -4935,8 +5107,8 @@ 'you are\n' 'in debug mode:\n' '\n' - ' > ...(3)double()\n' - ' -> return x * 2\n' + ' > ...(2)double()\n' + ' -> breakpoint()\n' ' (Pdb) p x\n' ' 3\n' ' (Pdb) continue\n' @@ -4965,11 +5137,11 @@ 'the\n' 'debugger upon program’s exit.\n' '\n' - 'New in version 3.2: "-c" option is introduced to execute ' + 'Changed in version 3.2: Added the "-c" option to execute ' 'commands as\n' - 'if given in a ".pdbrc" file, see Debugger Commands.\n' + 'if given in a ".pdbrc" file; see Debugger Commands.\n' '\n' - 'New in version 3.7: "-m" option is introduced to execute ' + 'Changed in version 3.7: Added the "-m" option to execute ' 'modules\n' 'similar to the way "python -m" does. As with a script, the ' 'debugger\n' @@ -5004,6 +5176,12 @@ ' 0\n' ' (Pdb)\n' '\n' + 'Changed in version 3.13: The implementation of **PEP 667** means ' + 'that\n' + 'name assignments made via "pdb" will immediately affect the ' + 'active\n' + 'scope, even when running inside an *optimized scope*.\n' + '\n' 'The module defines the following functions; each enters the ' 'debugger\n' 'in a slightly different way:\n' @@ -5045,7 +5223,7 @@ 'appears\n' ' as soon as the function is entered.\n' '\n' - 'pdb.set_trace(*, header=None)\n' + 'pdb.set_trace(*, header=None, commands=None)\n' '\n' ' Enter the debugger at the calling stack frame. This is ' 'useful to\n' @@ -5054,10 +5232,19 @@ ' code is not otherwise being debugged (e.g. when an assertion\n' ' fails). If given, *header* is printed to the console just ' 'before\n' - ' debugging begins.\n' + ' debugging begins. The *commands* argument, if given, is a ' + 'list of\n' + ' commands to execute when the debugger starts.\n' '\n' ' Changed in version 3.7: The keyword-only argument *header*.\n' '\n' + ' Changed in version 3.13: "set_trace()" will enter the ' + 'debugger\n' + ' immediately, rather than on the next line of code to be ' + 'executed.\n' + '\n' + ' Added in version 3.14: The *commands* argument.\n' + '\n' 'pdb.post_mortem(traceback=None)\n' '\n' ' Enter post-mortem debugging of the given *traceback* object. ' @@ -5080,7 +5267,7 @@ 'to access further features, you have to do this yourself:\n' '\n' "class pdb.Pdb(completekey='tab', stdin=None, stdout=None, " - 'skip=None, nosigint=False, readrc=True)\n' + 'skip=None, nosigint=False, readrc=True, mode=None)\n' '\n' ' "Pdb" is the debugger class.\n' '\n' @@ -5109,20 +5296,31 @@ 'Pdb\n' ' will load .pdbrc files from the filesystem.\n' '\n' + ' The *mode* argument specifies how the debugger was invoked. ' + 'It\n' + ' impacts the workings of some debugger commands. Valid values ' + 'are\n' + ' "\'inline\'" (used by the breakpoint() builtin), "\'cli\'" ' + '(used by the\n' + ' command line invocation) or "None" (for backwards compatible\n' + ' behaviour, as before the *mode* argument was added).\n' + '\n' ' Example call to enable tracing with *skip*:\n' '\n' " import pdb; pdb.Pdb(skip=['django.*']).set_trace()\n" '\n' ' Raises an auditing event "pdb.Pdb" with no arguments.\n' '\n' - ' New in version 3.1: The *skip* argument.\n' + ' Changed in version 3.1: Added the *skip* parameter.\n' '\n' - ' New in version 3.2: The *nosigint* argument. Previously, a ' - 'SIGINT\n' - ' handler was never set by Pdb.\n' + ' Changed in version 3.2: Added the *nosigint* parameter. ' + 'Previously,\n' + ' a SIGINT handler was never set by Pdb.\n' '\n' ' Changed in version 3.6: The *readrc* argument.\n' '\n' + ' Added in version 3.14: Added the *mode* argument.\n' + '\n' ' run(statement, globals=None, locals=None)\n' ' runeval(expression, globals=None, locals=None)\n' ' runcall(function, *args, **kwds)\n' @@ -5210,28 +5408,30 @@ '* "$_exception": the exception if the frame is raising an ' 'exception\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12: Added the *convenience variable* ' + 'feature.\n' '\n' 'If a file ".pdbrc" exists in the user’s home directory or in ' 'the\n' 'current directory, it is read with "\'utf-8\'" encoding and ' 'executed as\n' - 'if it had been typed at the debugger prompt. This is ' - 'particularly\n' - 'useful for aliases. If both files exist, the one in the home\n' - 'directory is read first and aliases defined there can be ' - 'overridden by\n' - 'the local file.\n' - '\n' - 'Changed in version 3.11: ".pdbrc" is now read with "\'utf-8\'" ' - 'encoding.\n' - 'Previously, it was read with the system locale encoding.\n' + 'if it had been typed at the debugger prompt, with the exception ' + 'that\n' + 'empty lines and lines starting with "#" are ignored. This is\n' + 'particularly useful for aliases. If both files exist, the one ' + 'in the\n' + 'home directory is read first and aliases defined there can be\n' + 'overridden by the local file.\n' '\n' 'Changed in version 3.2: ".pdbrc" can now contain commands that\n' 'continue debugging, such as "continue" or "next". Previously, ' 'these\n' 'commands had no effect.\n' '\n' + 'Changed in version 3.11: ".pdbrc" is now read with "\'utf-8\'" ' + 'encoding.\n' + 'Previously, it was read with the system locale encoding.\n' + '\n' 'h(elp) [command]\n' '\n' ' Without argument, print the list of available commands. With ' @@ -5243,13 +5443,20 @@ '"help\n' ' exec" must be entered to get help on the "!" command.\n' '\n' - 'w(here)\n' + 'w(here) [count]\n' '\n' ' Print a stack trace, with the most recent frame at the ' - 'bottom. An\n' - ' arrow (">") indicates the current frame, which determines ' - 'the\n' - ' context of most commands.\n' + 'bottom. if\n' + ' *count* is 0, print the current frame entry. If *count* is\n' + ' negative, print the least recent - *count* frames. If *count* ' + 'is\n' + ' positive, print the most recent *count* frames. An arrow ' + '(">")\n' + ' indicates the current frame, which determines the context of ' + 'most\n' + ' commands.\n' + '\n' + ' Changed in version 3.14: *count* argument is added.\n' '\n' 'd(own) [count]\n' '\n' @@ -5265,19 +5472,22 @@ '\n' 'b(reak) [([filename:]lineno | function) [, condition]]\n' '\n' - ' With a *lineno* argument, set a break there in the current ' - 'file.\n' + ' With a *lineno* argument, set a break at line *lineno* in ' + 'the\n' + ' current file. The line number may be prefixed with a ' + '*filename* and\n' + ' a colon, to specify a breakpoint in another file (possibly ' + 'one that\n' + ' hasn’t been loaded yet). The file is searched on ' + '"sys.path".\n' + ' Acceptable forms of *filename* are "/abspath/to/file.py",\n' + ' "relpath/file.py", "module" and "package.module".\n' + '\n' ' With a *function* argument, set a break at the first ' 'executable\n' - ' statement within that function. The line number may be ' - 'prefixed\n' - ' with a filename and a colon, to specify a breakpoint in ' - 'another\n' - ' file (probably one that hasn’t been loaded yet). The file ' - 'is\n' - ' searched on "sys.path". Note that each breakpoint is ' - 'assigned a\n' - ' number to which all the other breakpoint commands refer.\n' + ' statement within that function. *function* can be any ' + 'expression\n' + ' that evaluates to a function in the current namespace.\n' '\n' ' If a second argument is present, it is an expression which ' 'must\n' @@ -5289,6 +5499,9 @@ 'current\n' ' ignore count, and the associated condition if any.\n' '\n' + ' Each breakpoint is assigned a number to which all the other\n' + ' breakpoint commands refer.\n' + '\n' 'tbreak [([filename:]lineno | function) [, condition]]\n' '\n' ' Temporary breakpoint, which is removed automatically when it ' @@ -5369,25 +5582,27 @@ '\n' ' Specifying any command resuming execution (currently ' '"continue",\n' - ' "step", "next", "return", "jump", "quit" and their ' - 'abbreviations)\n' - ' terminates the command list (as if that command was ' - 'immediately\n' - ' followed by end). This is because any time you resume ' - 'execution\n' - ' (even with a simple next or step), you may encounter another\n' - ' breakpoint—which could have its own command list, leading to\n' - ' ambiguities about which list to execute.\n' - '\n' - ' If you use the "silent" command in the command list, the ' - 'usual\n' - ' message about stopping at a breakpoint is not printed. This ' - 'may be\n' - ' desirable for breakpoints that are to print a specific ' - 'message and\n' - ' then continue. If none of the other commands print anything, ' - 'you\n' - ' see no sign that the breakpoint was reached.\n' + ' "step", "next", "return", "until", "jump", "quit" and their\n' + ' abbreviations) terminates the command list (as if that ' + 'command was\n' + ' immediately followed by end). This is because any time you ' + 'resume\n' + ' execution (even with a simple next or step), you may ' + 'encounter\n' + ' another breakpoint—which could have its own command list, ' + 'leading\n' + ' to ambiguities about which list to execute.\n' + '\n' + ' If the list of commands contains the "silent" command, or a ' + 'command\n' + ' that resumes execution, then the breakpoint message ' + 'containing\n' + ' information about the frame is not displayed.\n' + '\n' + ' Changed in version 3.14: Frame information will not be ' + 'displayed if\n' + ' a command that resumes execution is present in the command ' + 'list.\n' '\n' 's(tep)\n' '\n' @@ -5466,14 +5681,14 @@ 'differs\n' ' from the current line.\n' '\n' - ' New in version 3.2: The ">>" marker.\n' + ' Changed in version 3.2: Added the ">>" marker.\n' '\n' 'll | longlist\n' '\n' ' List all source code for the current function or frame.\n' ' Interesting lines are marked as for "list".\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'a(rgs)\n' '\n' @@ -5506,7 +5721,7 @@ '\n' ' Try to get source code of *expression* and display it.\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'display [expression]\n' '\n' @@ -5565,7 +5780,7 @@ ' display lst[:]: [1] [old: []]\n' ' (Pdb)\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'undisplay [expression]\n' '\n' @@ -5574,34 +5789,32 @@ ' *expression*, clear all display expressions for the current ' 'frame.\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'interact\n' '\n' - ' Start an interactive interpreter (using the "code" module) ' - 'whose\n' - ' global namespace contains all the (global and local) names ' - 'found in\n' - ' the current scope. Use "exit()" or "quit()" to exit the ' - 'interpreter\n' - ' and return to the debugger.\n' + ' Start an interactive interpreter (using the "code" module) in ' + 'a new\n' + ' global namespace initialised from the local and global ' + 'namespaces\n' + ' for the current scope. Use "exit()" or "quit()" to exit the\n' + ' interpreter and return to the debugger.\n' '\n' ' Note:\n' '\n' - ' Because interact creates a new global namespace with the ' - 'current\n' - ' global and local namespace for execution, assignment to ' - 'variables\n' - ' will not affect the original namespaces. However, ' - 'modification to\n' - ' the mutable objects will be reflected in the original ' - 'namespaces.\n' + ' As "interact" creates a new dedicated namespace for code\n' + ' execution, assignments to variables will not affect the ' + 'original\n' + ' namespaces. However, modifications to any referenced ' + 'mutable\n' + ' objects will be reflected in the original namespaces as ' + 'usual.\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' - ' New in version 3.13: "exit()" and "quit()" can be used to ' + ' Changed in version 3.13: "exit()" and "quit()" can be used to ' 'exit\n' - ' "interact" command.\n' + ' the "interact" command.\n' '\n' ' Changed in version 3.13: "interact" directs its output to ' 'the\n' @@ -5675,6 +5888,10 @@ 'preserved.\n' ' "restart" is an alias for "run".\n' '\n' + ' Changed in version 3.14: "run" and "restart" commands are ' + 'disabled\n' + ' when the debugger is invoked in "\'inline\'" mode.\n' + '\n' 'q(uit)\n' '\n' ' Quit from the debugger. The program being executed is ' @@ -5742,7 +5959,7 @@ ' > example.py(10)middle()\n' ' -> return inner(0)\n' '\n' - ' New in version 3.13.\n' + ' Added in version 3.13.\n' '\n' '-[ Footnotes ]-\n' '\n' @@ -5807,7 +6024,8 @@ 'dict\n' 'items and earlier dictionary unpackings.\n' '\n' - 'New in version 3.5: Unpacking into dictionary displays, originally\n' + 'Added in version 3.5: Unpacking into dictionary displays, ' + 'originally\n' 'proposed by **PEP 448**.\n' '\n' 'A dict comprehension, in contrast to list and set comprehensions,\n' @@ -6115,9 +6333,12 @@ 'of the module "builtins". The global namespace is searched ' 'first. If\n' 'the names are not found there, the builtins namespace is ' - 'searched.\n' - 'The "global" statement must precede all uses of the listed ' - 'names.\n' + 'searched\n' + 'next. If the names are also not found in the builtins ' + 'namespace, new\n' + 'variables are created in the global namespace. The global ' + 'statement\n' + 'must precede all uses of the listed names.\n' '\n' 'The "global" statement has the same scope as a name binding ' 'operation\n' @@ -6180,17 +6401,18 @@ 'Annotation scopes\n' '-----------------\n' '\n' - 'Type parameter lists and "type" statements introduce ' - '*annotation\n' - 'scopes*, which behave mostly like function scopes, but with ' - 'some\n' - 'exceptions discussed below. *Annotations* currently do not use\n' - 'annotation scopes, but they are expected to use annotation ' - 'scopes in\n' - 'Python 3.13 when **PEP 649** is implemented.\n' + '*Annotations*, type parameter lists and "type" statements ' + 'introduce\n' + '*annotation scopes*, which behave mostly like function scopes, ' + 'but\n' + 'with some exceptions discussed below.\n' '\n' 'Annotation scopes are used in the following contexts:\n' '\n' + '* *Function annotations*.\n' + '\n' + '* *Variable annotations*.\n' + '\n' '* Type parameter lists for generic type aliases.\n' '\n' '* Type parameter lists for generic functions. A generic ' @@ -6205,8 +6427,9 @@ 'annotation\n' ' scope, but its decorators are not.\n' '\n' - '* The bounds and constraints for type variables (lazily ' - 'evaluated).\n' + '* The bounds, constraints, and default values for type ' + 'parameters\n' + ' (lazily evaluated).\n' '\n' '* The value of type aliases (lazily evaluated).\n' '\n' @@ -6243,31 +6466,40 @@ '\n' '* While annotation scopes have an internal name, that name is ' 'not\n' - ' reflected in the *__qualname__* of objects defined within the ' - 'scope.\n' - ' Instead, the "__qualname__" of such objects is as if the ' - 'object were\n' - ' defined in the enclosing scope.\n' + ' reflected in the *qualified name* of objects defined within ' + 'the\n' + ' scope. Instead, the "__qualname__" of such objects is as if ' + 'the\n' + ' object were defined in the enclosing scope.\n' + '\n' + 'Added in version 3.12: Annotation scopes were introduced in ' + 'Python\n' + '3.12 as part of **PEP 695**.\n' '\n' - 'New in version 3.12: Annotation scopes were introduced in ' - 'Python 3.12\n' - 'as part of **PEP 695**.\n' + 'Changed in version 3.13: Annotation scopes are also used for ' + 'type\n' + 'parameter defaults, as introduced by **PEP 696**.\n' + '\n' + 'Changed in version 3.14: Annotation scopes are now also used ' + 'for\n' + 'annotations, as specified in **PEP 649** and **PEP 749**.\n' '\n' '\n' 'Lazy evaluation\n' '---------------\n' '\n' - 'The values of type aliases created through the "type" statement ' - 'are\n' - '*lazily evaluated*. The same applies to the bounds and ' - 'constraints of\n' - 'type variables created through the type parameter syntax. This ' - 'means\n' - 'that they are not evaluated when the type alias or type ' - 'variable is\n' - 'created. Instead, they are only evaluated when doing so is ' - 'necessary\n' - 'to resolve an attribute access.\n' + 'Most annotation scopes are *lazily evaluated*. This includes\n' + 'annotations, the values of type aliases created through the ' + '"type"\n' + 'statement, and the bounds, constraints, and default values of ' + 'type\n' + 'variables created through the type parameter syntax. This means ' + 'that\n' + 'they are not evaluated when the type alias or type variable is\n' + 'created, or when the object carrying annotations is created. ' + 'Instead,\n' + 'they are only evaluated when necessary, for example when the\n' + '"__value__" attribute on a type alias is accessed.\n' '\n' 'Example:\n' '\n' @@ -6311,7 +6543,7 @@ 'looked up\n' 'as if they were used in the immediately enclosing scope.\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' '\n' 'Builtins and restricted execution\n' @@ -6445,12 +6677,17 @@ 'exprlists': 'Expression lists\n' '****************\n' '\n' - ' expression_list ::= expression ("," expression)* [","]\n' - ' starred_list ::= starred_item ("," starred_item)* ' + ' starred_expression ::= ["*"] or_expr\n' + ' flexible_expression ::= assignment_expression | ' + 'starred_expression\n' + ' flexible_expression_list ::= flexible_expression ("," ' + 'flexible_expression)* [","]\n' + ' starred_expression_list ::= starred_expression ("," ' + 'starred_expression)* [","]\n' + ' expression_list ::= expression ("," expression)* ' '[","]\n' - ' starred_expression ::= expression | (starred_item ",")* ' - '[starred_item]\n' - ' starred_item ::= assignment_expression | "*" or_expr\n' + ' yield_list ::= expression_list | ' + 'starred_expression "," [starred_expression_list]\n' '\n' 'Except when part of a list or set display, an expression list\n' 'containing at least one comma yields a tuple. The length of ' @@ -6466,23 +6703,26 @@ 'the\n' 'unpacking.\n' '\n' - 'New in version 3.5: Iterable unpacking in expression lists, ' - 'originally\n' - 'proposed by **PEP 448**.\n' + 'Added in version 3.5: Iterable unpacking in expression lists,\n' + 'originally proposed by **PEP 448**.\n' '\n' - 'The trailing comma is required only to create a single tuple ' - '(a.k.a. a\n' - '*singleton*); it is optional in all other cases. A single ' - 'expression\n' - 'without a trailing comma doesn’t create a tuple, but rather ' - 'yields the\n' - 'value of that expression. (To create an empty tuple, use an ' - 'empty pair\n' - 'of parentheses: "()".)\n', - 'floating': 'Floating point literals\n' + 'Added in version 3.11: Any item in an expression list may be ' + 'starred.\n' + 'See **PEP 646**.\n' + '\n' + 'A trailing comma is required only to create a one-item tuple, ' + 'such as\n' + '"1,"; it is optional in all other cases. A single expression ' + 'without a\n' + 'trailing comma doesn’t create a tuple, but rather yields the ' + 'value of\n' + 'that expression. (To create an empty tuple, use an empty pair ' + 'of\n' + 'parentheses: "()".)\n', + 'floating': 'Floating-point literals\n' '***********************\n' '\n' - 'Floating point literals are described by the following lexical\n' + 'Floating-point literals are described by the following lexical\n' 'definitions:\n' '\n' ' floatnumber ::= pointfloat | exponentfloat\n' @@ -6496,12 +6736,12 @@ 'using\n' 'radix 10. For example, "077e010" is legal, and denotes the same ' 'number\n' - 'as "77e10". The allowed range of floating point literals is\n' + 'as "77e10". The allowed range of floating-point literals is\n' 'implementation-dependent. As in integer literals, underscores ' 'are\n' 'supported for digit grouping.\n' '\n' - 'Some examples of floating point literals:\n' + 'Some examples of floating-point literals:\n' '\n' ' 3.14 10. .001 1e100 3.14e-10 0e0 ' '3.14_15_93\n' @@ -6884,7 +7124,7 @@ '\n' 'The "\'_\'" option signals the use of an underscore for a ' 'thousands\n' - 'separator for floating point presentation types and for ' + 'separator for floating-point presentation types and for ' 'integer\n' 'presentation type "\'d\'". For integer presentation types ' '"\'b\'", "\'o\'",\n' @@ -7011,11 +7251,11 @@ '\n' 'In addition to the above presentation types, integers can ' 'be formatted\n' - 'with the floating point presentation types listed below ' + 'with the floating-point presentation types listed below ' '(except "\'n\'"\n' 'and "None"). When doing so, "float()" is used to convert ' 'the integer\n' - 'to a floating point number before formatting.\n' + 'to a floating-point number before formatting.\n' '\n' 'The available presentation types for "float" and "Decimal" ' 'values are:\n' @@ -7041,12 +7281,10 @@ 'of "6" digits |\n' ' | | after the decimal point for "float", and ' 'shows all |\n' - ' | | coefficient digits for "Decimal". If no ' - 'digits follow the |\n' - ' | | decimal point, the decimal point is also ' - 'removed unless |\n' - ' | | the "#" option is ' - 'used. |\n' + ' | | coefficient digits for "Decimal". If ' + '"p=0", the decimal |\n' + ' | | point is omitted unless the "#" option is ' + 'used. |\n' ' ' '+-----------+------------------------------------------------------------+\n' ' | "\'E\'" | Scientific notation. Same as "\'e\'" ' @@ -7065,12 +7303,10 @@ 'decimal point for |\n' ' | | "float", and uses a precision large enough ' 'to show all |\n' - ' | | coefficient digits for "Decimal". If no ' - 'digits follow the |\n' - ' | | decimal point, the decimal point is also ' - 'removed unless |\n' - ' | | the "#" option is ' - 'used. |\n' + ' | | coefficient digits for "Decimal". If ' + '"p=0", the decimal |\n' + ' | | point is omitted unless the "#" option is ' + 'used. |\n' ' ' '+-----------+------------------------------------------------------------+\n' ' | "\'F\'" | Fixed-point notation. Same as "\'f\'", ' @@ -7155,18 +7391,22 @@ 'percent sign. |\n' ' ' '+-----------+------------------------------------------------------------+\n' - ' | None | For "float" this is the same as "\'g\'", ' - 'except that when |\n' - ' | | fixed-point notation is used to format the ' - 'result, it |\n' + ' | None | For "float" this is like the "\'g\'" type, ' + 'except that when |\n' + ' | | fixed- point notation is used to format ' + 'the result, it |\n' ' | | always includes at least one digit past ' - 'the decimal point. |\n' - ' | | The precision used is as large as needed ' - 'to represent the |\n' - ' | | given value faithfully. For "Decimal", ' - 'this is the same |\n' - ' | | as either "\'g\'" or "\'G\'" depending on ' - 'the value of |\n' + 'the decimal point, |\n' + ' | | and switches to the scientific notation ' + 'when "exp >= p - |\n' + ' | | 1". When the precision is not specified, ' + 'the latter will |\n' + ' | | be as large as needed to represent the ' + 'given value |\n' + ' | | faithfully. For "Decimal", this is the ' + 'same as either |\n' + ' | | "\'g\'" or "\'G\'" depending on the value ' + 'of |\n' ' | | "context.capitals" for the current decimal ' 'context. The |\n' ' | | overall effect is to match the output of ' @@ -7356,10 +7596,12 @@ ' parameter_list_no_posonly ::= defparameter ("," ' 'defparameter)* ["," [parameter_list_starargs]]\n' ' | parameter_list_starargs\n' - ' parameter_list_starargs ::= "*" [parameter] ("," ' + ' parameter_list_starargs ::= "*" [star_parameter] ("," ' 'defparameter)* ["," ["**" parameter [","]]]\n' ' | "**" parameter [","]\n' ' parameter ::= identifier [":" expression]\n' + ' star_parameter ::= identifier [":" ["*"] ' + 'expression]\n' ' defparameter ::= parameter ["=" expression]\n' ' funcname ::= identifier\n' '\n' @@ -7487,27 +7729,22 @@ 'expression"”\n' 'following the parameter name. Any parameter may have an ' 'annotation,\n' - 'even those of the form "*identifier" or "**identifier". ' - 'Functions may\n' - 'have “return” annotation of the form “"-> expression"” after ' - 'the\n' - 'parameter list. These annotations can be any valid Python ' - 'expression.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - 'dictionary keyed by the parameters’ names in the ' - '"__annotations__"\n' - 'attribute of the function object. If the "annotations" import ' - 'from\n' - '"__future__" is used, annotations are preserved as strings at ' - 'runtime\n' - 'which enables postponed evaluation. Otherwise, they are ' - 'evaluated\n' - 'when the function definition is executed. In this case ' - 'annotations\n' - 'may be evaluated in a different order than they appear in the ' - 'source\n' - 'code.\n' + 'even those of the form "*identifier" or "**identifier". (As a ' + 'special\n' + 'case, parameters of the form "*identifier" may have an ' + 'annotation “":\n' + '*expression"”.) Functions may have “return” annotation of the ' + 'form\n' + '“"-> expression"” after the parameter list. These annotations ' + 'can be\n' + 'any valid Python expression. The presence of annotations does ' + 'not\n' + 'change the semantics of a function. See Annotations for more\n' + 'information on annotations.\n' + '\n' + 'Changed in version 3.11: Parameters of the form “"*identifier"” ' + 'may\n' + 'have an annotation “": *expression"”. See **PEP 646**.\n' '\n' 'It is also possible to create anonymous functions (functions not ' 'bound\n' @@ -7581,7 +7818,7 @@ 'parameters, or as targets in "with" statements or "except" ' 'clauses, or\n' 'in a "for" target list, "class" definition, function definition,\n' - '"import" statement, or variable annotation.\n' + '"import" statement, or *variable annotations*.\n' '\n' '**CPython implementation detail:** The current implementation does ' 'not\n' @@ -7676,19 +7913,16 @@ '\n' 'Within the ASCII range (U+0001..U+007F), the valid characters ' 'for\n' - 'identifiers are the same as in Python 2.x: the uppercase and ' - 'lowercase\n' - 'letters "A" through "Z", the underscore "_" and, except for ' - 'the first\n' - 'character, the digits "0" through "9".\n' - '\n' - 'Python 3.0 introduces additional characters from outside the ' - 'ASCII\n' - 'range (see **PEP 3131**). For these characters, the ' - 'classification\n' - 'uses the version of the Unicode Character Database as ' - 'included in the\n' - '"unicodedata" module.\n' + 'identifiers include the uppercase and lowercase letters "A" ' + 'through\n' + '"Z", the underscore "_" and, except for the first character, ' + 'the\n' + 'digits "0" through "9". Python 3.0 introduced additional ' + 'characters\n' + 'from outside the ASCII range (see **PEP 3131**). For these\n' + 'characters, the classification uses the version of the ' + 'Unicode\n' + 'Character Database as included in the "unicodedata" module.\n' '\n' 'Identifiers are unlimited in length. Case is significant.\n' '\n' @@ -7738,8 +7972,8 @@ '\n' 'A non-normative HTML file listing all valid identifier ' 'characters for\n' - 'Unicode 15.1.0 can be found at\n' - 'https://www.unicode.org/Public/15.1.0/ucd/DerivedCoreProperties.txt\n' + 'Unicode 16.0.0 can be found at\n' + 'https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt\n' '\n' '\n' 'Keywords\n' @@ -7763,7 +7997,7 @@ 'Soft Keywords\n' '=============\n' '\n' - 'New in version 3.10.\n' + 'Added in version 3.10.\n' '\n' 'Some identifiers are only reserved under specific contexts. ' 'These are\n' @@ -7873,11 +8107,11 @@ '\n' 'An imaginary literal yields a complex number with a real part ' 'of 0.0.\n' - 'Complex numbers are represented as a pair of floating point ' + 'Complex numbers are represented as a pair of floating-point ' 'numbers\n' 'and have the same restrictions on their range. To create a ' 'complex\n' - 'number with a nonzero real part, add a floating point number to ' + 'number with a nonzero real part, add a floating-point number to ' 'it,\n' 'e.g., "(3+4j)". Some examples of imaginary literals:\n' '\n' @@ -8263,7 +8497,8 @@ 'in\n' 'square brackets:\n' '\n' - ' list_display ::= "[" [starred_list | comprehension] "]"\n' + ' list_display ::= "[" [flexible_expression_list | comprehension] ' + '"]"\n' '\n' 'A list display yields a new list object, the contents being ' 'specified\n' @@ -8391,9 +8626,12 @@ 'namespace\n' 'of the module "builtins". The global namespace is searched ' 'first. If\n' - 'the names are not found there, the builtins namespace is ' - 'searched.\n' - 'The "global" statement must precede all uses of the listed names.\n' + 'the names are not found there, the builtins namespace is searched\n' + 'next. If the names are also not found in the builtins namespace, ' + 'new\n' + 'variables are created in the global namespace. The global ' + 'statement\n' + 'must precede all uses of the listed names.\n' '\n' 'The "global" statement has the same scope as a name binding ' 'operation\n' @@ -8454,15 +8692,18 @@ 'Annotation scopes\n' '=================\n' '\n' - 'Type parameter lists and "type" statements introduce *annotation\n' - 'scopes*, which behave mostly like function scopes, but with some\n' - 'exceptions discussed below. *Annotations* currently do not use\n' - 'annotation scopes, but they are expected to use annotation scopes ' - 'in\n' - 'Python 3.13 when **PEP 649** is implemented.\n' + '*Annotations*, type parameter lists and "type" statements ' + 'introduce\n' + '*annotation scopes*, which behave mostly like function scopes, ' + 'but\n' + 'with some exceptions discussed below.\n' '\n' 'Annotation scopes are used in the following contexts:\n' '\n' + '* *Function annotations*.\n' + '\n' + '* *Variable annotations*.\n' + '\n' '* Type parameter lists for generic type aliases.\n' '\n' '* Type parameter lists for generic functions. A generic ' @@ -8476,8 +8717,8 @@ 'annotation\n' ' scope, but its decorators are not.\n' '\n' - '* The bounds and constraints for type variables (lazily ' - 'evaluated).\n' + '* The bounds, constraints, and default values for type parameters\n' + ' (lazily evaluated).\n' '\n' '* The value of type aliases (lazily evaluated).\n' '\n' @@ -8511,31 +8752,36 @@ ' can introduce new names.\n' '\n' '* While annotation scopes have an internal name, that name is not\n' - ' reflected in the *__qualname__* of objects defined within the ' - 'scope.\n' - ' Instead, the "__qualname__" of such objects is as if the object ' - 'were\n' - ' defined in the enclosing scope.\n' + ' reflected in the *qualified name* of objects defined within the\n' + ' scope. Instead, the "__qualname__" of such objects is as if the\n' + ' object were defined in the enclosing scope.\n' + '\n' + 'Added in version 3.12: Annotation scopes were introduced in ' + 'Python\n' + '3.12 as part of **PEP 695**.\n' '\n' - 'New in version 3.12: Annotation scopes were introduced in Python ' - '3.12\n' - 'as part of **PEP 695**.\n' + 'Changed in version 3.13: Annotation scopes are also used for type\n' + 'parameter defaults, as introduced by **PEP 696**.\n' + '\n' + 'Changed in version 3.14: Annotation scopes are now also used for\n' + 'annotations, as specified in **PEP 649** and **PEP 749**.\n' '\n' '\n' 'Lazy evaluation\n' '===============\n' '\n' - 'The values of type aliases created through the "type" statement ' - 'are\n' - '*lazily evaluated*. The same applies to the bounds and constraints ' - 'of\n' - 'type variables created through the type parameter syntax. This ' - 'means\n' - 'that they are not evaluated when the type alias or type variable ' - 'is\n' - 'created. Instead, they are only evaluated when doing so is ' - 'necessary\n' - 'to resolve an attribute access.\n' + 'Most annotation scopes are *lazily evaluated*. This includes\n' + 'annotations, the values of type aliases created through the ' + '"type"\n' + 'statement, and the bounds, constraints, and default values of ' + 'type\n' + 'variables created through the type parameter syntax. This means ' + 'that\n' + 'they are not evaluated when the type alias or type variable is\n' + 'created, or when the object carrying annotations is created. ' + 'Instead,\n' + 'they are only evaluated when necessary, for example when the\n' + '"__value__" attribute on a type alias is accessed.\n' '\n' 'Example:\n' '\n' @@ -8578,7 +8824,7 @@ 'looked up\n' 'as if they were used in the immediately enclosing scope.\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' '\n' 'Builtins and restricted execution\n' @@ -8635,37 +8881,41 @@ '\n' ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' '\n' - 'The "nonlocal" statement causes the listed identifiers to refer ' - 'to\n' - 'previously bound variables in the nearest enclosing scope ' - 'excluding\n' - 'globals. This is important because the default behavior for ' - 'binding is\n' - 'to search the local namespace first. The statement allows\n' - 'encapsulated code to rebind variables outside of the local ' - 'scope\n' - 'besides the global (module) scope.\n' - '\n' - 'Names listed in a "nonlocal" statement, unlike those listed in ' - 'a\n' - '"global" statement, must refer to pre-existing bindings in an\n' - 'enclosing scope (the scope in which a new binding should be ' - 'created\n' - 'cannot be determined unambiguously).\n' - '\n' - 'Names listed in a "nonlocal" statement must not collide with ' - 'pre-\n' - 'existing bindings in the local scope.\n' + 'When the definition of a function or class is nested (enclosed) ' + 'within\n' + 'the definitions of other functions, its nonlocal scopes are the ' + 'local\n' + 'scopes of the enclosing functions. The "nonlocal" statement ' + 'causes the\n' + 'listed identifiers to refer to names previously bound in ' + 'nonlocal\n' + 'scopes. It allows encapsulated code to rebind such nonlocal\n' + 'identifiers. If a name is bound in more than one nonlocal ' + 'scope, the\n' + 'nearest binding is used. If a name is not bound in any nonlocal ' + 'scope,\n' + 'or if there is no nonlocal scope, a "SyntaxError" is raised.\n' + '\n' + 'The nonlocal statement applies to the entire scope of a function ' + 'or\n' + 'class body. A "SyntaxError" is raised if a variable is used or\n' + 'assigned to prior to its nonlocal declaration in the scope.\n' '\n' 'See also:\n' '\n' ' **PEP 3104** - Access to Names in Outer Scopes\n' - ' The specification for the "nonlocal" statement.\n', + ' The specification for the "nonlocal" statement.\n' + '\n' + '**Programmer’s note:** "nonlocal" is a directive to the parser ' + 'and\n' + 'applies only to code parsed along with it. See the note for ' + 'the\n' + '"global" statement.\n', 'numbers': 'Numeric literals\n' '****************\n' '\n' - 'There are three types of numeric literals: integers, floating ' - 'point\n' + 'There are three types of numeric literals: integers, ' + 'floating-point\n' 'numbers, and imaginary numbers. There are no complex literals\n' '(complex numbers can be formed by adding a real number and an\n' 'imaginary number).\n' @@ -8756,7 +9006,7 @@ '"__rsub__()"\n' ' method, "type(y).__rsub__(y, x)" is called if ' '"type(x).__sub__(x,\n' - ' y)" returns *NotImplemented*.\n' + ' y)" returns "NotImplemented".\n' '\n' ' Note that ternary "pow()" will not try calling ' '"__rpow__()" (the\n' @@ -8799,14 +9049,18 @@ 'the result\n' ' (which could be, but does not have to be, *self*). If a ' 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of ' - 'a class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to ' - '"x =\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' + ' method is not defined, or if that method returns ' + '"NotImplemented",\n' + ' the augmented assignment falls back to the normal ' + 'methods. For\n' + ' instance, if *x* is an instance of a class with an ' + '"__iadd__()"\n' + ' method, "x += y" is equivalent to "x = x.__iadd__(y)" . ' + 'If\n' + ' "__iadd__()" does not exist, or if "x.__iadd__(y)" ' + 'returns\n' + ' "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" ' + 'are\n' ' considered, as with the evaluation of "x + y". In ' 'certain\n' ' situations, augmented assignment can result in ' @@ -8867,13 +9121,9 @@ ' of the object truncated to an "Integral" (typically an ' '"int").\n' '\n' - ' The built-in function "int()" falls back to ' - '"__trunc__()" if\n' - ' neither "__int__()" nor "__index__()" is defined.\n' - '\n' - ' Changed in version 3.11: The delegation of "int()" to ' - '"__trunc__()"\n' - ' is deprecated.\n', + ' Changed in version 3.14: "int()" no longer delegates to ' + 'the\n' + ' "__trunc__()" method.\n', 'objects': 'Objects, values and types\n' '*************************\n' '\n' @@ -8887,7 +9137,7 @@ 'Every object has an identity, a type and a value. An object’s\n' '*identity* never changes once it has been created; you may think ' 'of it\n' - 'as the object’s address in memory. The ‘"is"’ operator compares ' + 'as the object’s address in memory. The "is" operator compares ' 'the\n' 'identity of two objects; the "id()" function returns an integer\n' 'representing its identity.\n' @@ -8952,7 +9202,7 @@ 'Note that the use of the implementation’s tracing or debugging\n' 'facilities may keep objects alive that would normally be ' 'collectable.\n' - 'Also note that catching an exception with a ‘"try"…"except"’ ' + 'Also note that catching an exception with a "try"…"except" ' 'statement\n' 'may keep objects alive.\n' '\n' @@ -8967,8 +9217,9 @@ 'release the external resource, usually a "close()" method. ' 'Programs\n' 'are strongly recommended to explicitly close such objects. The\n' - '‘"try"…"finally"’ statement and the ‘"with"’ statement provide\n' - 'convenient ways to do this.\n' + '"try"…"finally" statement and the "with" statement provide ' + 'convenient\n' + 'ways to do this.\n' '\n' 'Some objects contain references to other objects; these are ' 'called\n' @@ -8992,16 +9243,22 @@ 'types, operations that compute new values may actually return a\n' 'reference to any existing object with the same type and value, ' 'while\n' - 'for mutable objects this is not allowed. E.g., after "a = 1; b = ' - '1",\n' - '"a" and "b" may or may not refer to the same object with the ' - 'value\n' - 'one, depending on the implementation, but after "c = []; d = []", ' - '"c"\n' - 'and "d" are guaranteed to refer to two different, unique, newly\n' - 'created empty lists. (Note that "c = d = []" assigns the same ' - 'object\n' - 'to both "c" and "d".)\n', + 'for mutable objects this is not allowed. For example, after "a = ' + '1; b\n' + '= 1", *a* and *b* may or may not refer to the same object with ' + 'the\n' + 'value one, depending on the implementation. This is because "int" ' + 'is\n' + 'an immutable type, so the reference to "1" can be reused. This\n' + 'behaviour depends on the implementation used, so should not be ' + 'relied\n' + 'upon, but is something to be aware of when making use of object\n' + 'identity tests. However, after "c = []; d = []", *c* and *d* are\n' + 'guaranteed to refer to two different, unique, newly created ' + 'empty\n' + 'lists. (Note that "e = f = []" assigns the *same* object to both ' + '*e*\n' + 'and *f*.)\n', 'operator-summary': 'Operator precedence\n' '*******************\n' '\n' @@ -9230,8 +9487,8 @@ '"complex"\n' 'number. (In earlier versions it raised a "ValueError".)\n' '\n' - 'This operation can be customized using the special "__pow__()" ' - 'method.\n', + 'This operation can be customized using the special "__pow__()" and\n' + '"__rpow__()" methods.\n', 'raise': 'The "raise" statement\n' '*********************\n' '\n' @@ -9345,10 +9602,7 @@ 'The try statement.\n' '\n' 'Changed in version 3.3: "None" is now permitted as "Y" in "raise X\n' - 'from Y".\n' - '\n' - 'New in version 3.3: The "__suppress_context__" attribute to ' - 'suppress\n' + 'from Y".Added the "__suppress_context__" attribute to suppress\n' 'automatic display of the exception context.\n' '\n' 'Changed in version 3.11: If the traceback of the active exception ' @@ -9491,7 +9745,7 @@ 'for\n' ' correctness.\n' '\n' - ' New in version 3.4.\n' + ' Added in version 3.4.\n' '\n' 'Note:\n' '\n' @@ -9648,9 +9902,12 @@ 'the\n' 'second argument.\n' '\n' - 'This operation can be customized using the special ' - '"__lshift__()" and\n' - '"__rshift__()" methods.\n' + 'The left shift operation can be customized using the special\n' + '"__lshift__()" and "__rlshift__()" methods. The right shift ' + 'operation\n' + 'can be customized using the special "__rshift__()" and ' + '"__rrshift__()"\n' + 'methods.\n' '\n' 'A right shift by *n* bits is defined as floor division by ' '"pow(2,n)".\n' @@ -9716,20 +9973,6 @@ 'not reported\n' 'by the "dir()" built-in function.\n' '\n' - 'object.__dict__\n' - '\n' - ' A dictionary or other mapping object used to store an ' - 'object’s\n' - ' (writable) attributes.\n' - '\n' - 'instance.__class__\n' - '\n' - ' The class to which a class instance belongs.\n' - '\n' - 'class.__bases__\n' - '\n' - ' The tuple of base classes of a class object.\n' - '\n' 'definition.__name__\n' '\n' ' The name of the class, function, method, descriptor, or ' @@ -9742,41 +9985,28 @@ 'descriptor, or\n' ' generator instance.\n' '\n' - ' New in version 3.3.\n' + ' Added in version 3.3.\n' '\n' - 'definition.__type_params__\n' + 'definition.__module__\n' '\n' - ' The type parameters of generic classes, functions, and ' - 'type\n' - ' aliases.\n' + ' The name of the module in which a class or function was ' + 'defined.\n' '\n' - ' New in version 3.12.\n' + 'definition.__doc__\n' '\n' - 'class.__mro__\n' + ' The documentation string of a class or function, or ' + '"None" if\n' + ' undefined.\n' '\n' - ' This attribute is a tuple of classes that are considered ' - 'when\n' - ' looking for base classes during method resolution.\n' - '\n' - 'class.mro()\n' - '\n' - ' This method can be overridden by a metaclass to customize ' - 'the\n' - ' method resolution order for its instances. It is called ' - 'at class\n' - ' instantiation, and its result is stored in "__mro__".\n' - '\n' - 'class.__subclasses__()\n' + 'definition.__type_params__\n' '\n' - ' Each class keeps a list of weak references to its ' - 'immediate\n' - ' subclasses. This method returns a list of all those ' - 'references\n' - ' still alive. The list is in definition order. Example:\n' + ' The type parameters of generic classes, functions, and ' + 'type\n' + ' aliases. For classes and functions that are not generic, ' + 'this will\n' + ' be an empty tuple.\n' '\n' - ' >>> int.__subclasses__()\n' - " [, , , " - "]\n", + ' Added in version 3.12.\n', 'specialnames': 'Special method names\n' '********************\n' '\n' @@ -9920,6 +10150,10 @@ ' It is not guaranteed that "__del__()" methods are called ' 'for\n' ' objects that still exist when the interpreter exits.\n' + ' "weakref.finalize" provides a straightforward way to ' + 'register a\n' + ' cleanup function to be called when an object is garbage ' + 'collected.\n' '\n' ' Note:\n' '\n' @@ -10133,8 +10367,8 @@ 'reflection,\n' ' and "__eq__()" and "__ne__()" are their own reflection. ' 'If the\n' - ' operands are of different types, and right operand’s type ' - 'is a\n' + ' operands are of different types, and the right operand’s ' + 'type is a\n' ' direct or indirect subclass of the left operand’s type, ' 'the\n' ' reflected method of the right operand has priority, ' @@ -10143,6 +10377,11 @@ 'is not\n' ' considered.\n' '\n' + ' When no appropriate method returns any value other than\n' + ' "NotImplemented", the "==" and "!=" operators will fall ' + 'back to\n' + ' "is" and "is not", respectively.\n' + '\n' 'object.__hash__(self)\n' '\n' ' Called by built-in function "hash()" and for operations ' @@ -10384,9 +10623,9 @@ '\n' 'object.__dir__(self)\n' '\n' - ' Called when "dir()" is called on the object. A sequence ' + ' Called when "dir()" is called on the object. An iterable ' 'must be\n' - ' returned. "dir()" converts the returned sequence to a ' + ' returned. "dir()" converts the returned iterable to a ' 'list and\n' ' sorts it.\n' '\n' @@ -10413,8 +10652,8 @@ 'returned.\n' '\n' 'The "__dir__" function should accept no arguments, and ' - 'return a\n' - 'sequence of strings that represents the names accessible on ' + 'return an\n' + 'iterable of strings that represents the names accessible on ' 'module. If\n' 'present, this function overrides the standard "dir()" search ' 'on a\n' @@ -10454,7 +10693,7 @@ 'Changed in version 3.5: "__class__" module attribute is now ' 'writable.\n' '\n' - 'New in version 3.7: "__getattr__" and "__dir__" module ' + 'Added in version 3.7: "__getattr__" and "__dir__" module ' 'attributes.\n' '\n' 'See also:\n' @@ -10713,11 +10952,13 @@ 'the class\n' ' where it is defined. *__slots__* declared in parents are ' 'available\n' - ' in child classes. However, child subclasses will get a ' - '"__dict__"\n' - ' and *__weakref__* unless they also define *__slots__* ' - '(which should\n' - ' only contain names of any *additional* slots).\n' + ' in child classes. However, instances of a child subclass ' + 'will get a\n' + ' "__dict__" and *__weakref__* unless the subclass also ' + 'defines\n' + ' *__slots__* (which should only contain names of any ' + '*additional*\n' + ' slots).\n' '\n' '* If a class defines a slot also defined in a base class, ' 'the instance\n' @@ -10823,7 +11064,7 @@ 'explicit\n' ' hint) can be accessed as "type(cls)".\n' '\n' - ' New in version 3.6.\n' + ' Added in version 3.6.\n' '\n' 'When a class is created, "type.__new__()" scans the class ' 'variables\n' @@ -10855,7 +11096,7 @@ '\n' ' See Creating the class object for more details.\n' '\n' - ' New in version 3.6.\n' + ' Added in version 3.6.\n' '\n' '\n' 'Metaclasses\n' @@ -11134,7 +11375,7 @@ 'built-in\n' 'types), including other ABCs.\n' '\n' - 'class.__instancecheck__(self, instance)\n' + 'type.__instancecheck__(self, instance)\n' '\n' ' Return true if *instance* should be considered a (direct ' 'or\n' @@ -11142,7 +11383,7 @@ 'implement\n' ' "isinstance(instance, class)".\n' '\n' - 'class.__subclasscheck__(self, subclass)\n' + 'type.__subclasscheck__(self, subclass)\n' '\n' ' Return true if *subclass* should be considered a (direct ' 'or\n' @@ -11457,7 +11698,7 @@ 'for\n' ' correctness.\n' '\n' - ' New in version 3.4.\n' + ' Added in version 3.4.\n' '\n' 'Note:\n' '\n' @@ -11682,7 +11923,7 @@ '"__rsub__()"\n' ' method, "type(y).__rsub__(y, x)" is called if ' '"type(x).__sub__(x,\n' - ' y)" returns *NotImplemented*.\n' + ' y)" returns "NotImplemented".\n' '\n' ' Note that ternary "pow()" will not try calling ' '"__rpow__()" (the\n' @@ -11725,14 +11966,17 @@ 'the result\n' ' (which could be, but does not have to be, *self*). If a ' 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of a ' - 'class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to "x ' - '=\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' + ' method is not defined, or if that method returns ' + '"NotImplemented",\n' + ' the augmented assignment falls back to the normal ' + 'methods. For\n' + ' instance, if *x* is an instance of a class with an ' + '"__iadd__()"\n' + ' method, "x += y" is equivalent to "x = x.__iadd__(y)" . ' + 'If\n' + ' "__iadd__()" does not exist, or if "x.__iadd__(y)" ' + 'returns\n' + ' "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" are\n' ' considered, as with the evaluation of "x + y". In ' 'certain\n' ' situations, augmented assignment can result in unexpected ' @@ -11793,13 +12037,9 @@ ' of the object truncated to an "Integral" (typically an ' '"int").\n' '\n' - ' The built-in function "int()" falls back to "__trunc__()" ' - 'if\n' - ' neither "__int__()" nor "__index__()" is defined.\n' - '\n' - ' Changed in version 3.11: The delegation of "int()" to ' - '"__trunc__()"\n' - ' is deprecated.\n' + ' Changed in version 3.14: "int()" no longer delegates to ' + 'the\n' + ' "__trunc__()" method.\n' '\n' '\n' 'With Statement Context Managers\n' @@ -11903,7 +12143,7 @@ 'will\n' 'raise a "TypeError".\n' '\n' - 'New in version 3.10.\n' + 'Added in version 3.10.\n' '\n' 'See also:\n' '\n' @@ -11952,7 +12192,7 @@ 'to\n' ' implement this method.\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' 'See also:\n' '\n' @@ -11966,6 +12206,71 @@ ' ABC for buffer types.\n' '\n' '\n' + 'Annotations\n' + '===========\n' + '\n' + 'Functions, classes, and modules may contain *annotations*, ' + 'which are a\n' + 'way to associate information (usually *type hints*) with a ' + 'symbol.\n' + '\n' + 'object.__annotations__\n' + '\n' + ' This attribute contains the annotations for an object. It ' + 'is lazily\n' + ' evaluated, so accessing the attribute may execute ' + 'arbitrary code\n' + ' and raise exceptions. If evaluation is successful, the ' + 'attribute is\n' + ' set to a dictionary mapping from variable names to ' + 'annotations.\n' + '\n' + ' Changed in version 3.14: Annotations are now lazily ' + 'evaluated.\n' + '\n' + 'object.__annotate__(format)\n' + '\n' + ' An *annotate function*. Returns a new dictionary object ' + 'mapping\n' + ' attribute/parameter names to their annotation values.\n' + '\n' + ' Takes a format parameter specifying the format in which ' + 'annotations\n' + ' values should be provided. It must be a member of the\n' + ' "annotationlib.Format" enum, or an integer with a value\n' + ' corresponding to a member of the enum.\n' + '\n' + ' If an annotate function doesn’t support the requested ' + 'format, it\n' + ' must raise "NotImplementedError". Annotate functions must ' + 'always\n' + ' support "VALUE" format; they must not raise ' + '"NotImplementedError()"\n' + ' when called with this format.\n' + '\n' + ' When called with "VALUE" format, an annotate function ' + 'may raise\n' + ' "NameError"; it must not raise "NameError" when called ' + 'requesting\n' + ' any other format.\n' + '\n' + ' If an object does not have any annotations, ' + '"__annotate__" should\n' + ' preferably be set to "None" (it can’t be deleted), rather ' + 'than set\n' + ' to a function that returns an empty dict.\n' + '\n' + ' Added in version 3.14.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 649** — Deferred evaluation of annotation using ' + 'descriptors\n' + ' Introduces lazy evaluation of annotations and the ' + '"__annotate__"\n' + ' function.\n' + '\n' + '\n' 'Special method lookup\n' '=====================\n' '\n' @@ -12116,7 +12421,7 @@ '‘Default\n' ' Case Folding’ of the Unicode Standard.\n' '\n' - ' New in version 3.3.\n' + ' Added in version 3.3.\n' '\n' 'str.center(width[, fillchar])\n' '\n' @@ -12284,7 +12589,7 @@ 'the\n' ' "LC_NUMERIC" locale in some cases.\n' '\n' - 'str.format_map(mapping)\n' + 'str.format_map(mapping, /)\n' '\n' ' Similar to "str.format(**mapping)", except that ' '"mapping" is used\n' @@ -12300,7 +12605,7 @@ "{country}'.format_map(Default(name='Guido'))\n" " 'Guido was born in country'\n" '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'str.index(sub[, start[, end]])\n' '\n' @@ -12344,7 +12649,7 @@ 'have code\n' ' points in the range U+0000-U+007F.\n' '\n' - ' New in version 3.7.\n' + ' Added in version 3.7.\n' '\n' 'str.isdecimal()\n' '\n' @@ -12584,7 +12889,7 @@ " >>> 'BaseTestCase'.removeprefix('Test')\n" " 'BaseTestCase'\n" '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' 'str.removesuffix(suffix, /)\n' '\n' @@ -12599,7 +12904,7 @@ " >>> 'TmpDirMixin'.removesuffix('Tests')\n" " 'TmpDirMixin'\n" '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' 'str.replace(old, new, count=-1)\n' '\n' @@ -12712,11 +13017,11 @@ ' and are deemed to delimit empty strings (for example,\n' ' "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', ' '\'2\']"). The *sep* argument\n' - ' may consist of multiple characters (for example,\n' - ' "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', ' - '\'3\']"). Splitting an\n' - ' empty string with a specified separator returns ' - '"[\'\']".\n' + ' may consist of multiple characters as a single ' + 'delimiter (to split\n' + ' with multiple delimiters, use "re.split()"). Splitting ' + 'an empty\n' + ' string with a specified separator returns "[\'\']".\n' '\n' ' For example:\n' '\n' @@ -12726,6 +13031,8 @@ " ['1', '2,3']\n" " >>> '1,2,,3,'.split(',')\n" " ['1', '2', '', '3', '']\n" + " >>> '1<>2<>3<4'.split('<>')\n" + " ['1', '2', '3<4']\n" '\n' ' If *sep* is not specified or is "None", a different ' 'splitting\n' @@ -13068,21 +13375,18 @@ 'greater must be expressed with escapes.\n' '\n' 'Both string and bytes literals may optionally be prefixed with a\n' - 'letter "\'r\'" or "\'R\'"; such strings are called *raw strings* ' - 'and treat\n' - 'backslashes as literal characters. As a result, in string ' - 'literals,\n' - '"\'\\U\'" and "\'\\u\'" escapes in raw strings are not treated ' - 'specially.\n' - 'Given that Python 2.x’s raw unicode literals behave differently ' - 'than\n' - 'Python 3.x’s the "\'ur\'" syntax is not supported.\n' - '\n' - 'New in version 3.3: The "\'rb\'" prefix of raw bytes literals has ' - 'been\n' - 'added as a synonym of "\'br\'".\n' + 'letter "\'r\'" or "\'R\'"; such constructs are called *raw ' + 'string\n' + 'literals* and *raw bytes literals* respectively and treat ' + 'backslashes\n' + 'as literal characters. As a result, in raw string literals, ' + '"\'\\U\'"\n' + 'and "\'\\u\'" escapes are not treated specially.\n' '\n' - 'New in version 3.3: Support for the unicode legacy literal\n' + 'Added in version 3.3: The "\'rb\'" prefix of raw bytes literals ' + 'has been\n' + 'added as a synonym of "\'br\'".Support for the unicode legacy ' + 'literal\n' '("u\'value\'") was reintroduced to simplify the maintenance of ' 'dual\n' 'Python 2.x and 3.x codebases. See **PEP 414** for more ' @@ -13268,7 +13572,8 @@ '*generic\n' 'class* will generally return a GenericAlias object.\n' '\n' - ' subscription ::= primary "[" expression_list "]"\n' + ' subscription ::= primary "[" flexible_expression_list ' + '"]"\n' '\n' 'When an object is subscripted, the interpreter will ' 'evaluate the\n' @@ -13287,13 +13592,18 @@ 'see\n' '__class_getitem__ versus __getitem__.\n' '\n' - 'If the expression list contains at least one comma, it will ' - 'evaluate\n' - 'to a "tuple" containing the items of the expression list. ' - 'Otherwise,\n' - 'the expression list will evaluate to the value of the ' - 'list’s sole\n' - 'member.\n' + 'If the expression list contains at least one comma, or if ' + 'any of the\n' + 'expressions are starred, the expression list will evaluate ' + 'to a\n' + '"tuple" containing the items of the expression list. ' + 'Otherwise, the\n' + 'expression list will evaluate to the value of the list’s ' + 'sole member.\n' + '\n' + 'Changed in version 3.11: Expressions in an expression list ' + 'may be\n' + 'starred. See **PEP 646**.\n' '\n' 'For built-in objects, there are two types of objects that ' 'support\n' @@ -13407,14 +13717,15 @@ 'clauses in turn until one is found that matches the exception. An\n' 'expression-less "except" clause, if present, must be last; it ' 'matches\n' - 'any exception. For an "except" clause with an expression, that\n' - 'expression is evaluated, and the clause matches the exception if the\n' - 'resulting object is “compatible” with the exception. An object is\n' - 'compatible with an exception if the object is the class or a *non-\n' - 'virtual base class* of the exception object, or a tuple containing ' - 'an\n' - 'item that is the class or a non-virtual base class of the exception\n' - 'object.\n' + 'any exception.\n' + '\n' + 'For an "except" clause with an expression, the expression must\n' + 'evaluate to an exception type or a tuple of exception types. The\n' + 'raised exception matches an "except" clause whose expression ' + 'evaluates\n' + 'to the class or a *non-virtual base class* of the exception object, ' + 'or\n' + 'to a tuple that contains such a class.\n' '\n' 'If no "except" clause matches the exception, the search for an\n' 'exception handler continues in the surrounding code and on the\n' @@ -13543,12 +13854,13 @@ ' ...\n' " ExceptionGroup('', (BlockingIOError()))\n" '\n' - 'An "except*" clause must have a matching type, and this type cannot ' - 'be\n' - 'a subclass of "BaseExceptionGroup". It is not possible to mix ' - '"except"\n' - 'and "except*" in the same "try". "break", "continue" and "return"\n' - 'cannot appear in an "except*" clause.\n' + 'An "except*" clause must have a matching expression; it cannot be\n' + '"except*:". Furthermore, this expression cannot contain exception\n' + 'group types, because that would have ambiguous semantics.\n' + '\n' + 'It is not possible to mix "except" and "except*" in the same "try".\n' + '"break", "continue" and "return" cannot appear in an "except*" ' + 'clause.\n' '\n' '\n' '"else" clause\n' @@ -13663,11 +13975,12 @@ 'See Implementing the arithmetic operations for more details.\n' '\n' 'Changed in version 3.9: Evaluating "NotImplemented" in a boolean\n' - 'context is deprecated. While it currently evaluates as true, it ' - 'will\n' - 'emit a "DeprecationWarning". It will raise a "TypeError" in a ' - 'future\n' - 'version of Python.\n' + 'context was deprecated.\n' + '\n' + 'Changed in version 3.14: Evaluating "NotImplemented" in a boolean\n' + 'context now raises a "TypeError". It previously evaluated to ' + '"True"\n' + 'and emitted a "DeprecationWarning" since Python 3.9.\n' '\n' '\n' 'Ellipsis\n' @@ -13709,7 +14022,7 @@ '\n' '* A sign is shown only when the number is negative.\n' '\n' - 'Python distinguishes between integers, floating point numbers, and\n' + 'Python distinguishes between integers, floating-point numbers, and\n' 'complex numbers:\n' '\n' '\n' @@ -13754,28 +14067,28 @@ '"numbers.Real" ("float")\n' '------------------------\n' '\n' - 'These represent machine-level double precision floating point ' + 'These represent machine-level double precision floating-point ' 'numbers.\n' 'You are at the mercy of the underlying machine architecture (and C ' 'or\n' 'Java implementation) for the accepted range and handling of ' 'overflow.\n' - 'Python does not support single-precision floating point numbers; ' + 'Python does not support single-precision floating-point numbers; ' 'the\n' 'savings in processor and memory usage that are usually the reason ' 'for\n' 'using these are dwarfed by the overhead of using objects in Python, ' 'so\n' 'there is no reason to complicate the language with two kinds of\n' - 'floating point numbers.\n' + 'floating-point numbers.\n' '\n' '\n' '"numbers.Complex" ("complex")\n' '-----------------------------\n' '\n' 'These represent complex numbers as a pair of machine-level double\n' - 'precision floating point numbers. The same caveats apply as for\n' - 'floating point numbers. The real and imaginary parts of a complex\n' + 'precision floating-point numbers. The same caveats apply as for\n' + 'floating-point numbers. The real and imaginary parts of a complex\n' 'number "z" can be retrieved through the read-only attributes ' '"z.real"\n' 'and "z.imag".\n' @@ -13791,14 +14104,18 @@ 'contains\n' 'the numbers 0, 1, …, *n*-1. Item *i* of sequence *a* is selected ' 'by\n' - '"a[i]".\n' + '"a[i]". Some sequences, including built-in sequences, interpret\n' + 'negative subscripts by adding the sequence length. For example,\n' + '"a[-2]" equals "a[n-2]", the second to last item of sequence a ' + 'with\n' + 'length "n".\n' '\n' 'Sequences also support slicing: "a[i:j]" selects all items with ' 'index\n' '*k* such that *i* "<=" *k* "<" *j*. When used as an expression, a\n' - 'slice is a sequence of the same type. This implies that the index ' - 'set\n' - 'is renumbered so that it starts at 0.\n' + 'slice is a sequence of the same type. The comment above about ' + 'negative\n' + 'indexes also applies to negative slice positions.\n' '\n' 'Some sequences also support “extended slicing” with a third “step”\n' 'parameter: "a[i:j:k]" selects all items of *a* with index *x* where ' @@ -13965,8 +14282,7 @@ 'however removing a key and re-inserting it will add it to the end\n' 'instead of keeping its old place.\n' '\n' - 'Dictionaries are mutable; they can be created by the "{...}" ' - 'notation\n' + 'Dictionaries are mutable; they can be created by the "{}" notation\n' '(see section Dictionary displays).\n' '\n' 'The extension modules "dbm.ndbm" and "dbm.gnu" provide additional\n' @@ -14016,13 +14332,15 @@ '| function.__closure__ | "None" or a ' '"tuple" of cells that contain bindings |\n' '| | for the ' - 'function’s free variables. A cell object |\n' - '| | has the ' - 'attribute "cell_contents". This can be |\n' - '| | used to get ' - 'the value of the cell, as well as set |\n' - '| | the ' - 'value. |\n' + 'names specified in the "co_freevars" |\n' + '| | attribute of ' + 'the function’s "code object". A cell |\n' + '| | object has ' + 'the attribute "cell_contents". This can |\n' + '| | be used to ' + 'get the value of the cell, as well as |\n' + '| | set the ' + 'value. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '\n' '\n' @@ -14037,8 +14355,8 @@ '|====================================================|====================================================|\n' '| function.__doc__ | The ' 'function’s documentation string, or "None" if |\n' - '| | unavailable. ' - 'Not inherited by subclasses. |\n' + '| | ' + 'unavailable. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| function.__name__ | The ' 'function’s name. See also: "__name__ |\n' @@ -14048,7 +14366,7 @@ '| function.__qualname__ | The ' 'function’s *qualified name*. See also: |\n' '| | ' - '"__qualname__ attributes". New in version 3.3. |\n' + '"__qualname__ attributes". Added in version 3.3. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| function.__module__ | The name of ' 'the module the function was defined |\n' @@ -14079,9 +14397,20 @@ '| | parameter ' 'names, and "\'return\'" for the return |\n' '| | annotation, ' - 'if provided. See also: Annotations |\n' - '| | Best ' - 'Practices. |\n' + 'if provided. See also: |\n' + '| | ' + '"object.__annotations__". Changed in version |\n' + '| | 3.14: ' + 'Annotations are now lazily evaluated. See |\n' + '| | **PEP ' + '649**. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__annotate__ | The ' + '*annotate function* for this function, or |\n' + '| | "None" if ' + 'the function has no annotations. See |\n' + '| | ' + '"object.__annotate__". Added in version 3.14. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| function.__kwdefaults__ | A ' '"dictionary" containing defaults for keyword- |\n' @@ -14091,7 +14420,7 @@ '| function.__type_params__ | A "tuple" ' 'containing the type parameters of a |\n' '| | generic ' - 'function. New in version 3.12. |\n' + 'function. Added in version 3.12. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '\n' 'Function objects also support getting and setting arbitrary\n' @@ -14186,21 +14515,10 @@ 'to\n' 'calling "f(C,1)" where "f" is the underlying function.\n' '\n' - 'Note that the transformation from function object to instance ' - 'method\n' - 'object happens each time the attribute is retrieved from the ' - 'instance.\n' - 'In some cases, a fruitful optimization is to assign the attribute ' - 'to a\n' - 'local variable and call that local variable. Also notice that this\n' - 'transformation only happens for user-defined functions; other ' - 'callable\n' - 'objects (and all non-callable objects) are retrieved without\n' - 'transformation. It is also important to note that user-defined\n' - 'functions which are attributes of a class instance are not ' - 'converted\n' - 'to bound methods; this *only* happens when the function is an\n' - 'attribute of the class.\n' + 'It is important to note that user-defined functions which are\n' + 'attributes of a class instance are not converted to bound methods;\n' + 'this *only* happens when the function is an attribute of the ' + 'class.\n' '\n' '\n' 'Generator functions\n' @@ -14337,43 +14655,279 @@ 'e.g.,\n' '"m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n' '\n' - 'Predefined (writable) attributes:\n' '\n' - ' "__name__"\n' - ' The module’s name.\n' + 'Import-related attributes on module objects\n' + '-------------------------------------------\n' + '\n' + 'Module objects have the following attributes that relate to the ' + 'import\n' + 'system. When a module is created using the machinery associated ' + 'with\n' + 'the import system, these attributes are filled in based on the\n' + 'module’s *spec*, before the *loader* executes and loads the ' + 'module.\n' + '\n' + 'To create a module dynamically rather than using the import ' + 'system,\n' + 'it’s recommended to use "importlib.util.module_from_spec()", which\n' + 'will set the various import-controlled attributes to appropriate\n' + 'values. It’s also possible to use the "types.ModuleType" ' + 'constructor\n' + 'to create modules directly, but this technique is more error-prone, ' + 'as\n' + 'most attributes must be manually set on the module object after it ' + 'has\n' + 'been created when using this approach.\n' '\n' - ' "__doc__"\n' - ' The module’s documentation string, or "None" if unavailable.\n' + 'Caution:\n' '\n' - ' "__file__"\n' - ' The pathname of the file from which the module was loaded, if ' - 'it\n' - ' was loaded from a file. The "__file__" attribute may be ' - 'missing\n' - ' for certain types of modules, such as C modules that are\n' - ' statically linked into the interpreter. For extension ' + ' With the exception of "__name__", it is **strongly** recommended\n' + ' that you rely on "__spec__" and its attributes instead of any of ' + 'the\n' + ' other individual attributes listed in this subsection. Note that\n' + ' updating an attribute on "__spec__" will not update the\n' + ' corresponding attribute on the module itself:\n' + '\n' + ' >>> import typing\n' + ' >>> typing.__name__, typing.__spec__.name\n' + " ('typing', 'typing')\n" + " >>> typing.__spec__.name = 'spelling'\n" + ' >>> typing.__name__, typing.__spec__.name\n' + " ('typing', 'spelling')\n" + " >>> typing.__name__ = 'keyboard_smashing'\n" + ' >>> typing.__name__, typing.__spec__.name\n' + " ('keyboard_smashing', 'spelling')\n" + '\n' + 'module.__name__\n' + '\n' + ' The name used to uniquely identify the module in the import ' + 'system.\n' + ' For a directly executed module, this will be set to ' + '""__main__"".\n' + '\n' + ' This attribute must be set to the fully qualified name of the\n' + ' module. It is expected to match the value of\n' + ' "module.__spec__.name".\n' + '\n' + 'module.__spec__\n' + '\n' + ' A record of the module’s import-system-related state.\n' + '\n' + ' Set to the "module spec" that was used when importing the ' + 'module.\n' + ' See Module specs for more details.\n' + '\n' + ' Added in version 3.4.\n' + '\n' + 'module.__package__\n' + '\n' + ' The *package* a module belongs to.\n' + '\n' + ' If the module is top-level (that is, not a part of any specific\n' + ' package) then the attribute should be set to "\'\'" (the empty\n' + ' string). Otherwise, it should be set to the name of the ' + 'module’s\n' + ' package (which can be equal to "module.__name__" if the module\n' + ' itself is a package). See **PEP 366** for further details.\n' + '\n' + ' This attribute is used instead of "__name__" to calculate ' + 'explicit\n' + ' relative imports for main modules. It defaults to "None" for\n' + ' modules created dynamically using the "types.ModuleType"\n' + ' constructor; use "importlib.util.module_from_spec()" instead to\n' + ' ensure the attribute is set to a "str".\n' + '\n' + ' It is **strongly** recommended that you use\n' + ' "module.__spec__.parent" instead of "module.__package__".\n' + ' "__package__" is now only used as a fallback if ' + '"__spec__.parent"\n' + ' is not set, and this fallback path is deprecated.\n' + '\n' + ' Changed in version 3.4: This attribute now defaults to "None" ' + 'for\n' + ' modules created dynamically using the "types.ModuleType"\n' + ' constructor. Previously the attribute was optional.\n' + '\n' + ' Changed in version 3.6: The value of "__package__" is expected ' + 'to\n' + ' be the same as "__spec__.parent". "__package__" is now only used ' + 'as\n' + ' a fallback during import resolution if "__spec__.parent" is not\n' + ' defined.\n' + '\n' + ' Changed in version 3.10: "ImportWarning" is raised if an import\n' + ' resolution falls back to "__package__" instead of\n' + ' "__spec__.parent".\n' + '\n' + ' Changed in version 3.12: Raise "DeprecationWarning" instead of\n' + ' "ImportWarning" when falling back to "__package__" during ' + 'import\n' + ' resolution.\n' + '\n' + ' Deprecated since version 3.13, will be removed in version 3.15:\n' + ' "__package__" will cease to be set or taken into consideration ' + 'by\n' + ' the import system or standard library.\n' + '\n' + 'module.__loader__\n' + '\n' + ' The *loader* object that the import machinery used to load the\n' + ' module.\n' + '\n' + ' This attribute is mostly useful for introspection, but can be ' + 'used\n' + ' for additional loader-specific functionality, for example ' + 'getting\n' + ' data associated with a loader.\n' + '\n' + ' "__loader__" defaults to "None" for modules created dynamically\n' + ' using the "types.ModuleType" constructor; use\n' + ' "importlib.util.module_from_spec()" instead to ensure the ' + 'attribute\n' + ' is set to a *loader* object.\n' + '\n' + ' It is **strongly** recommended that you use\n' + ' "module.__spec__.loader" instead of "module.__loader__".\n' + '\n' + ' Changed in version 3.4: This attribute now defaults to "None" ' + 'for\n' + ' modules created dynamically using the "types.ModuleType"\n' + ' constructor. Previously the attribute was optional.\n' + '\n' + ' Deprecated since version 3.12, removed in version 3.14: Setting\n' + ' "__loader__" on a module while failing to set "__spec__.loader" ' + 'is\n' + ' deprecated. In Python 3.14, "__loader__" will cease to be set ' + 'or\n' + ' taken into consideration by the import system or the standard\n' + ' library.\n' + '\n' + 'module.__path__\n' + '\n' + ' A (possibly empty) *sequence* of strings enumerating the ' + 'locations\n' + ' where the package’s submodules will be found. Non-package ' 'modules\n' - ' loaded dynamically from a shared library, it’s the pathname ' - 'of\n' - ' the shared library file.\n' + ' should not have a "__path__" attribute. See __path__ attributes ' + 'on\n' + ' modules for more details.\n' + '\n' + ' It is **strongly** recommended that you use\n' + ' "module.__spec__.submodule_search_locations" instead of\n' + ' "module.__path__".\n' '\n' - ' "__annotations__"\n' - ' A dictionary containing *variable annotations* collected ' - 'during\n' - ' module body execution. For best practices on working with\n' - ' "__annotations__", please see Annotations Best Practices.\n' + 'module.__file__\n' '\n' - 'Special read-only attribute: "__dict__" is the module’s namespace ' - 'as a\n' - 'dictionary object.\n' + 'module.__cached__\n' + '\n' + ' "__file__" and "__cached__" are both optional attributes that ' + 'may\n' + ' or may not be set. Both attributes should be a "str" when they ' + 'are\n' + ' available.\n' '\n' - '**CPython implementation detail:** Because of the way CPython ' - 'clears\n' - 'module dictionaries, the module dictionary will be cleared when ' + ' "__file__" indicates the pathname of the file from which the ' + 'module\n' + ' was loaded (if loaded from a file), or the pathname of the ' + 'shared\n' + ' library file for extension modules loaded dynamically from a ' + 'shared\n' + ' library. It might be missing for certain types of modules, such ' + 'as\n' + ' C modules that are statically linked into the interpreter, and ' 'the\n' - 'module falls out of scope even if the dictionary still has live\n' - 'references. To avoid this, copy the dictionary or keep the module\n' - 'around while using its dictionary directly.\n' + ' import system may opt to leave it unset if it has no semantic\n' + ' meaning (for example, a module loaded from a database).\n' + '\n' + ' If "__file__" is set then the "__cached__" attribute might also ' + 'be\n' + ' set, which is the path to any compiled version of the code ' + '(for\n' + ' example, a byte-compiled file). The file does not need to exist ' + 'to\n' + ' set this attribute; the path can simply point to where the ' + 'compiled\n' + ' file *would* exist (see **PEP 3147**).\n' + '\n' + ' Note that "__cached__" may be set even if "__file__" is not ' + 'set.\n' + ' However, that scenario is quite atypical. Ultimately, the ' + '*loader*\n' + ' is what makes use of the module spec provided by the *finder* ' + '(from\n' + ' which "__file__" and "__cached__" are derived). So if a loader ' + 'can\n' + ' load from a cached module but otherwise does not load from a ' + 'file,\n' + ' that atypical scenario may be appropriate.\n' + '\n' + ' It is **strongly** recommended that you use\n' + ' "module.__spec__.cached" instead of "module.__cached__".\n' + '\n' + ' Deprecated since version 3.13, will be removed in version 3.15:\n' + ' Setting "__cached__" on a module while failing to set\n' + ' "__spec__.cached" is deprecated. In Python 3.15, "__cached__" ' + 'will\n' + ' cease to be set or taken into consideration by the import system ' + 'or\n' + ' standard library.\n' + '\n' + '\n' + 'Other writable attributes on module objects\n' + '-------------------------------------------\n' + '\n' + 'As well as the import-related attributes listed above, module ' + 'objects\n' + 'also have the following writable attributes:\n' + '\n' + 'module.__doc__\n' + '\n' + ' The module’s documentation string, or "None" if unavailable. ' + 'See\n' + ' also: "__doc__ attributes".\n' + '\n' + 'module.__annotations__\n' + '\n' + ' A dictionary containing *variable annotations* collected during\n' + ' module body execution. For best practices on working with\n' + ' "__annotations__", see "annotationlib".\n' + '\n' + ' Changed in version 3.14: Annotations are now lazily evaluated. ' + 'See\n' + ' **PEP 649**.\n' + '\n' + 'module.__annotate__\n' + '\n' + ' The *annotate function* for this module, or "None" if the ' + 'module\n' + ' has no annotations. See also: "__annotate__" attributes.\n' + '\n' + ' Added in version 3.14.\n' + '\n' + '\n' + 'Module dictionaries\n' + '-------------------\n' + '\n' + 'Module objects also have the following special read-only ' + 'attribute:\n' + '\n' + 'module.__dict__\n' + '\n' + ' The module’s namespace as a dictionary object. Uniquely among ' + 'the\n' + ' attributes listed here, "__dict__" cannot be accessed as a ' + 'global\n' + ' variable from within a module; it can only be accessed as an\n' + ' attribute on module objects.\n' + '\n' + ' **CPython implementation detail:** Because of the way CPython\n' + ' clears module dictionaries, the module dictionary will be ' + 'cleared\n' + ' when the module falls out of scope even if the dictionary still ' + 'has\n' + ' live references. To avoid this, copy the dictionary or keep ' + 'the\n' + ' module around while using its dictionary directly.\n' '\n' '\n' 'Custom classes\n' @@ -14398,8 +14952,7 @@ 'to\n' 'a common ancestor. Additional details on the C3 MRO used by Python ' 'can\n' - 'be found in the documentation accompanying the 2.3 release at\n' - 'https://www.python.org/download/releases/2.3/mro/.\n' + 'be found at The Python 2.3 Method Resolution Order.\n' '\n' 'When a class attribute reference (for class "C", say) would yield ' 'a\n' @@ -14420,32 +14973,147 @@ 'A class object can be called (see above) to yield a class instance\n' '(see below).\n' '\n' - 'Special attributes:\n' '\n' - ' "__name__"\n' - ' The class name.\n' + 'Special attributes\n' + '------------------\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| Attribute | ' + 'Meaning |\n' + '|====================================================|====================================================|\n' + '| type.__name__ | The class’s ' + 'name. See also: "__name__ attributes". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__qualname__ | The class’s ' + '*qualified name*. See also: |\n' + '| | ' + '"__qualname__ attributes". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__module__ | The name of ' + 'the module in which the class was |\n' + '| | ' + 'defined. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__dict__ | A "mapping ' + 'proxy" providing a read-only view of |\n' + '| | the class’s ' + 'namespace. See also: "__dict__ |\n' + '| | ' + 'attributes". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__bases__ | A "tuple" ' + 'containing the class’s bases. In most |\n' + '| | cases, for a ' + 'class defined as "class X(A, B, C)", |\n' + '| | ' + '"X.__bases__" will be exactly equal to "(A, B, |\n' + '| | ' + 'C)". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__doc__ | The class’s ' + 'documentation string, or "None" if |\n' + '| | undefined. ' + 'Not inherited by subclasses. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__annotations__ | A dictionary ' + 'containing *variable annotations* |\n' + '| | collected ' + 'during class body execution. See also: |\n' + '| | ' + '"__annotations__ attributes". For best practices |\n' + '| | on working ' + 'with "__annotations__", please see |\n' + '| | ' + '"annotationlib". Caution: Accessing the |\n' + '| | ' + '"__annotations__" attribute of a class object |\n' + '| | directly may ' + 'yield incorrect results in the |\n' + '| | presence of ' + 'metaclasses. In addition, the |\n' + '| | attribute ' + 'may not exist for some classes. Use |\n' + '| | ' + '"annotationlib.get_annotations()" to retrieve |\n' + '| | class ' + 'annotations safely. Changed in version |\n' + '| | 3.14: ' + 'Annotations are now lazily evaluated. See |\n' + '| | **PEP ' + '649**. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__annotate__() | The ' + '*annotate function* for this class, or "None" |\n' + '| | if the class ' + 'has no annotations. See also: |\n' + '| | ' + '"__annotate__ attributes". Caution: Accessing |\n' + '| | the ' + '"__annotate__" attribute of a class object |\n' + '| | directly may ' + 'yield incorrect results in the |\n' + '| | presence of ' + 'metaclasses. Use |\n' + '| | ' + '"annotationlib.get_annotate_function()" to |\n' + '| | retrieve the ' + 'annotate function safely. Added in |\n' + '| | version ' + '3.14. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__type_params__ | A "tuple" ' + 'containing the type parameters of a |\n' + '| | generic ' + 'class. Added in version 3.12. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__static_attributes__ | A "tuple" ' + 'containing names of attributes of this |\n' + '| | class which ' + 'are assigned through "self.X" from any |\n' + '| | function in ' + 'its body. Added in version 3.13. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__firstlineno__ | The line ' + 'number of the first line of the class |\n' + '| | definition, ' + 'including decorators. Setting the |\n' + '| | "__module__" ' + 'attribute removes the |\n' + '| | ' + '"__firstlineno__" item from the type’s dictionary. |\n' + '| | Added in ' + 'version 3.13. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| type.__mro__ | The "tuple" ' + 'of classes that are considered when |\n' + '| | looking for ' + 'base classes during method resolution. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '\n' + '\n' + 'Special methods\n' + '---------------\n' '\n' - ' "__module__"\n' - ' The name of the module in which the class was defined.\n' + 'In addition to the special attributes described above, all Python\n' + 'classes also have the following two methods available:\n' '\n' - ' "__dict__"\n' - ' The dictionary containing the class’s namespace.\n' + 'type.mro()\n' '\n' - ' "__bases__"\n' - ' A tuple containing the base classes, in the order of their\n' - ' occurrence in the base class list.\n' + ' This method can be overridden by a metaclass to customize the\n' + ' method resolution order for its instances. It is called at ' + 'class\n' + ' instantiation, and its result is stored in "__mro__".\n' '\n' - ' "__doc__"\n' - ' The class’s documentation string, or "None" if undefined.\n' + 'type.__subclasses__()\n' '\n' - ' "__annotations__"\n' - ' A dictionary containing *variable annotations* collected ' - 'during\n' - ' class body execution. For best practices on working with\n' - ' "__annotations__", please see Annotations Best Practices.\n' + ' Each class keeps a list of weak references to its immediate\n' + ' subclasses. This method returns a list of all those references\n' + ' still alive. The list is in definition order. Example:\n' '\n' - ' "__type_params__"\n' - ' A tuple containing the type parameters of a generic class.\n' + ' >>> class A: pass\n' + ' >>> class B(A): pass\n' + ' >>> A.__subclasses__()\n' + " []\n" '\n' '\n' 'Class instances\n' @@ -14485,8 +15153,19 @@ 'they have methods with certain special names. See section Special\n' 'method names.\n' '\n' - 'Special attributes: "__dict__" is the attribute dictionary;\n' - '"__class__" is the instance’s class.\n' + '\n' + 'Special attributes\n' + '------------------\n' + '\n' + 'object.__class__\n' + '\n' + ' The class to which a class instance belongs.\n' + '\n' + 'object.__dict__\n' + '\n' + ' A dictionary or other mapping object used to store an object’s\n' + ' (writable) attributes. Not all instances have a "__dict__"\n' + ' attribute; see the section on __slots__ for more details.\n' '\n' '\n' 'I/O objects (also known as file objects)\n' @@ -14543,7 +15222,9 @@ 'name |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| codeobject.co_qualname | The fully ' - 'qualified function name |\n' + 'qualified function name Added in |\n' + '| | version ' + '3.11. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| codeobject.co_argcount | The total ' 'number of positional *parameters* |\n' @@ -14583,14 +15264,20 @@ '| codeobject.co_cellvars | A "tuple" ' 'containing the names of local variables |\n' '| | that are ' - 'referenced by nested functions inside the |\n' - '| | ' - 'function |\n' + 'referenced from at least one *nested |\n' + '| | scope* ' + 'inside the function |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| codeobject.co_freevars | A "tuple" ' - 'containing the names of free variables |\n' - '| | in the ' - 'function |\n' + 'containing the names of *free (closure) |\n' + '| | variables* ' + 'that a *nested scope* references in an |\n' + '| | outer scope. ' + 'See also "function.__closure__". |\n' + '| | Note: ' + 'references to global and builtin names are |\n' + '| | *not* ' + 'included. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| codeobject.co_code | A string ' 'representing the sequence of *bytecode* |\n' @@ -14675,10 +15362,10 @@ ' The iterator returns "tuple"s containing the "(start_line,\n' ' end_line, start_column, end_column)". The *i-th* tuple ' 'corresponds\n' - ' to the position of the source code that compiled to the *i-th*\n' - ' instruction. Column information is 0-indexed utf-8 byte offsets ' - 'on\n' - ' the given source line.\n' + ' to the position of the source code that compiled to the *i-th* ' + 'code\n' + ' unit. Column information is 0-indexed utf-8 byte offsets on the\n' + ' given source line.\n' '\n' ' This positional information can be missing. A non-exhaustive ' 'lists\n' @@ -14697,7 +15384,7 @@ ' When this occurs, some or all of the tuple elements can be ' '"None".\n' '\n' - ' New in version 3.11.\n' + ' Added in version 3.11.\n' '\n' ' Note:\n' '\n' @@ -14755,7 +15442,7 @@ 'but\n' ' have been eliminated by the *bytecode* compiler.\n' '\n' - ' New in version 3.10.\n' + ' Added in version 3.10.\n' '\n' ' See also:\n' '\n' @@ -14763,6 +15450,17 @@ 'tools.\n' ' The PEP that introduced the "co_lines()" method.\n' '\n' + 'codeobject.replace(**kwargs)\n' + '\n' + ' Return a copy of the code object with new values for the ' + 'specified\n' + ' fields.\n' + '\n' + ' Code objects are also supported by the generic function\n' + ' "copy.replace()".\n' + '\n' + ' Added in version 3.8.\n' + '\n' '\n' 'Frame objects\n' '-------------\n' @@ -14792,10 +15490,16 @@ '| | ' '""f_code"". |\n' '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_locals | The ' - 'dictionary used by the frame to look up local |\n' - '| | ' - 'variables |\n' + '| frame.f_locals | The mapping ' + 'used by the frame to look up local |\n' + '| | variables. ' + 'If the frame refers to an *optimized |\n' + '| | scope*, this ' + 'may return a write-through proxy |\n' + '| | object. ' + 'Changed in version 3.13: Return a proxy |\n' + '| | for ' + 'optimized scopes. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| frame.f_globals | The ' 'dictionary used by the frame to look up global |\n' @@ -14877,7 +15581,7 @@ ' "RuntimeError" is raised if the frame is currently executing or\n' ' suspended.\n' '\n' - ' New in version 3.4.\n' + ' Added in version 3.4.\n' '\n' ' Changed in version 3.13: Attempting to clear a suspended frame\n' ' raises "RuntimeError" (as has always been the case for ' @@ -15008,8 +15712,8 @@ 'around another object that alters the way in which that object is\n' 'retrieved from classes and class instances. The behaviour of class\n' 'method objects upon such retrieval is described above, under ' - '“User-\n' - 'defined methods”. Class method objects are created by the built-in\n' + '“instance\n' + 'methods”. Class method objects are created by the built-in\n' '"classmethod()" constructor.\n', 'typesfunctions': 'Functions\n' '*********\n' @@ -15077,21 +15781,23 @@ '\n' ' If no positional argument is given, an empty dictionary ' 'is created.\n' - ' If a positional argument is given and it is a mapping ' - 'object, a\n' - ' dictionary is created with the same key-value pairs as ' - 'the mapping\n' - ' object. Otherwise, the positional argument must be an ' - '*iterable*\n' - ' object. Each item in the iterable must itself be an ' - 'iterable with\n' - ' exactly two objects. The first object of each item ' - 'becomes a key\n' - ' in the new dictionary, and the second object the ' - 'corresponding\n' - ' value. If a key occurs more than once, the last value ' - 'for that key\n' - ' becomes the corresponding value in the new dictionary.\n' + ' If a positional argument is given and it defines a ' + '"keys()" method,\n' + ' a dictionary is created by calling "__getitem__()" on the ' + 'argument\n' + ' with each returned key from the method. Otherwise, the ' + 'positional\n' + ' argument must be an *iterable* object. Each item in the ' + 'iterable\n' + ' must itself be an iterable with exactly two elements. ' + 'The first\n' + ' element of each item becomes a key in the new dictionary, ' + 'and the\n' + ' second element the corresponding value. If a key occurs ' + 'more than\n' + ' once, the last value for that key becomes the ' + 'corresponding value\n' + ' in the new dictionary.\n' '\n' ' If keyword arguments are given, the keyword arguments and ' 'their\n' @@ -15204,7 +15910,7 @@ '\n' ' Return a shallow copy of the dictionary.\n' '\n' - ' classmethod fromkeys(iterable[, value])\n' + ' classmethod fromkeys(iterable, value=None, /)\n' '\n' ' Create a new dictionary with keys from *iterable* and ' 'values set\n' @@ -15220,7 +15926,7 @@ 'distinct\n' ' values, use a dict comprehension instead.\n' '\n' - ' get(key[, default])\n' + ' get(key, default=None)\n' '\n' ' Return the value for *key* if *key* is in the ' 'dictionary, else\n' @@ -15270,9 +15976,9 @@ 'dictionary. This\n' ' is a shortcut for "reversed(d.keys())".\n' '\n' - ' New in version 3.8.\n' + ' Added in version 3.8.\n' '\n' - ' setdefault(key[, default])\n' + ' setdefault(key, default=None)\n' '\n' ' If *key* is in the dictionary, return its value. If ' 'not, insert\n' @@ -15286,15 +15992,17 @@ '*other*,\n' ' overwriting existing keys. Return "None".\n' '\n' - ' "update()" accepts either another dictionary object or ' - 'an\n' - ' iterable of key/value pairs (as tuples or other ' - 'iterables of\n' - ' length two). If keyword arguments are specified, the ' - 'dictionary\n' - ' is then updated with those key/value pairs: ' - '"d.update(red=1,\n' - ' blue=2)".\n' + ' "update()" accepts either another object with a ' + '"keys()" method\n' + ' (in which case "__getitem__()" is called with every ' + 'key returned\n' + ' from the method). or an iterable of key/value pairs ' + '(as tuples\n' + ' or other iterables of length two). If keyword ' + 'arguments are\n' + ' specified, the dictionary is then updated with those ' + 'key/value\n' + ' pairs: "d.update(red=1, blue=2)".\n' '\n' ' values()\n' '\n' @@ -15321,7 +16029,7 @@ ' *other* take priority when *d* and *other* share ' 'keys.\n' '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' ' d |= other\n' '\n' @@ -15333,7 +16041,7 @@ 'and *other*\n' ' share keys.\n' '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' ' Dictionaries compare equal if and only if they have the ' 'same "(key,\n' @@ -15455,7 +16163,7 @@ 'original\n' ' dictionary to which the view refers.\n' '\n' - ' New in version 3.10.\n' + ' Added in version 3.10.\n' '\n' 'Keys views are set-like since their entries are unique and ' '*hashable*.\n' @@ -15959,8 +16667,8 @@ '| | also removes it from ' '*s* | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.remove(x)" | remove the first item from ' - '*s* | (3) |\n' + '| "s.remove(x)" | removes the first item from ' + '*s* | (3) |\n' '| | where "s[i]" is equal to ' '*x* | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' @@ -15972,7 +16680,9 @@ '\n' 'Notes:\n' '\n' - '1. *t* must have the same length as the slice it is replacing.\n' + '1. If *k* is not equal to "1", *t* must have the same length as ' + 'the\n' + ' slice it is replacing.\n' '\n' '2. The optional argument *i* defaults to "-1", so that by ' 'default the\n' @@ -15995,7 +16705,7 @@ 'mutable\n' ' sequence classes provide it.\n' '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' + ' Added in version 3.3: "clear()" and "copy()" methods.\n' '\n' '6. The value *n* is an integer, or an object implementing\n' ' "__index__()". Zero and negative values of *n* clear the ' @@ -16107,7 +16817,7 @@ '\n' ' For sorting examples and a brief sorting tutorial, see ' 'Sorting\n' - ' HOW TO.\n' + ' Techniques.\n' '\n' ' **CPython implementation detail:** While a list is being ' 'sorted,\n' @@ -16322,15 +17032,14 @@ 'objects\n' 'based on the sequence of values they define (instead of ' 'comparing\n' - 'based on object identity).\n' - '\n' - 'New in version 3.3: The "start", "stop" and "step" attributes.\n' + 'based on object identity).Added the "start", "stop" and "step"\n' + 'attributes.\n' '\n' 'See also:\n' '\n' ' * The linspace recipe shows how to implement a lazy version of ' 'range\n' - ' suitable for floating point applications.\n', + ' suitable for floating-point applications.\n', 'typesseq-mutable': 'Mutable Sequence Types\n' '**********************\n' '\n' @@ -16423,8 +17132,8 @@ '| | also removes it from ' '*s* | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.remove(x)" | remove the first item ' - 'from *s* | (3) |\n' + '| "s.remove(x)" | removes the first ' + 'item from *s* | (3) |\n' '| | where "s[i]" is equal ' 'to *x* | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' @@ -16437,8 +17146,9 @@ '\n' 'Notes:\n' '\n' - '1. *t* must have the same length as the slice it is ' - 'replacing.\n' + '1. If *k* is not equal to "1", *t* must have the same ' + 'length as the\n' + ' slice it is replacing.\n' '\n' '2. The optional argument *i* defaults to "-1", so that ' 'by default the\n' @@ -16464,7 +17174,8 @@ 'concrete mutable\n' ' sequence classes provide it.\n' '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' + ' Added in version 3.3: "clear()" and "copy()" ' + 'methods.\n' '\n' '6. The value *n* is an integer, or an object ' 'implementing\n' diff --git a/Lib/queue.py b/Lib/queue.py index 467ff4fcecb134..25beb46e30d6bd 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -10,7 +10,15 @@ except ImportError: SimpleQueue = None -__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] +__all__ = [ + 'Empty', + 'Full', + 'ShutDown', + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'SimpleQueue', +] try: @@ -72,10 +80,11 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises a ValueError if called more times than there were items placed in the queue. - - Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: unfinished = self.unfinished_tasks - 1 @@ -93,8 +102,6 @@ def join(self): to indicate the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. - - Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: while self.unfinished_tasks: @@ -227,25 +234,26 @@ def get_nowait(self): return self.get(block=False) def shutdown(self, immediate=False): - '''Shut-down the queue, making queue gets and puts raise. + '''Shut-down the queue, making queue gets and puts raise ShutDown. By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() will be unblocked, and also get() - and join() if 'immediate'. The ShutDown exception is raised. + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). ''' with self.mutex: self.is_shutdown = True if immediate: - n_items = self._qsize() while self._qsize(): self._get() if self.unfinished_tasks > 0: self.unfinished_tasks -= 1 - self.not_empty.notify_all() # release all blocked threads in `join()` self.all_tasks_done.notify_all() + # All getters need to re-check queue-empty to raise ShutDown + self.not_empty.notify_all() self.not_full.notify_all() # Override these methods to implement other queue organizations diff --git a/Lib/quopri.py b/Lib/quopri.py old mode 100755 new mode 100644 index f36cf7b3951cda..129fd2f5c7c28a --- a/Lib/quopri.py +++ b/Lib/quopri.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Conversions to/from quoted-printable transport encoding as per RFC 1521.""" # (Dec 1991 version). diff --git a/Lib/random.py b/Lib/random.py index 875beb2f8cf41c..8b9a270c429e4a 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -996,5 +996,75 @@ def _test(N=10_000): _os.register_at_fork(after_in_child=_inst.seed) +# ------------------------------------------------------ +# -------------- command-line interface ---------------- + + +def _parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-c", "--choice", nargs="+", + help="print a random choice") + group.add_argument( + "-i", "--integer", type=int, metavar="N", + help="print a random integer between 1 and N inclusive") + group.add_argument( + "-f", "--float", type=float, metavar="N", + help="print a random floating-point number between 0 and N inclusive") + group.add_argument( + "--test", type=int, const=10_000, nargs="?", + help=argparse.SUPPRESS) + parser.add_argument("input", nargs="*", + help="""\ +if no options given, output depends on the input + string or multiple: same as --choice + integer: same as --integer + float: same as --float""") + args = parser.parse_args(arg_list) + return args, parser.format_help() + + +def main(arg_list: list[str] | None = None) -> int | str: + args, help_text = _parse_args(arg_list) + + # Explicit arguments + if args.choice: + return choice(args.choice) + + if args.integer is not None: + return randint(1, args.integer) + + if args.float is not None: + return uniform(0, args.float) + + if args.test: + _test(args.test) + return "" + + # No explicit argument, select based on input + if len(args.input) == 1: + val = args.input[0] + try: + # Is it an integer? + val = int(val) + return randint(1, val) + except ValueError: + try: + # Is it a float? + val = float(val) + return uniform(0, val) + except ValueError: + # Split in case of space-separated string: "a b c" + return choice(val.split()) + + if len(args.input) >= 2: + return choice(args.input) + + return help_text + + if __name__ == '__main__': - _test() + print(main()) diff --git a/Lib/re/_casefix.py b/Lib/re/_casefix.py index 06507d08bee02b..fed2d84fc01473 100644 --- a/Lib/re/_casefix.py +++ b/Lib/re/_casefix.py @@ -1,4 +1,4 @@ -# Auto-generated by Tools/scripts/generate_re_casefix.py. +# Auto-generated by Tools/build/generate_re_casefix.py. # Maps the code of lowercased character to codes of different lowercased # characters which have the same uppercase. diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 7b888f877eb3dc..29109f8812ee7b 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -28,6 +28,8 @@ POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE), } +_CHARSET_ALL = [(NEGATE, None)] + def _combine_flags(flags, add_flags, del_flags, TYPE_FLAGS=_parser.TYPE_FLAGS): if add_flags & TYPE_FLAGS: @@ -84,17 +86,22 @@ def _compile(code, pattern, flags): code[skip] = _len(code) - skip elif op is IN: charset, hascased = _optimize_charset(av, iscased, tolower, fixes) - if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: - emit(IN_LOC_IGNORE) - elif not hascased: - emit(IN) - elif not fixes: # ascii - emit(IN_IGNORE) + if not charset: + emit(FAILURE) + elif charset == _CHARSET_ALL: + emit(ANY_ALL) else: - emit(IN_UNI_IGNORE) - skip = _len(code); emit(0) - _compile_charset(charset, flags, code) - code[skip] = _len(code) - skip + if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: + emit(IN_LOC_IGNORE) + elif not hascased: + emit(IN) + elif not fixes: # ascii + emit(IN_IGNORE) + else: + emit(IN_UNI_IGNORE) + skip = _len(code); emit(0) + _compile_charset(charset, flags, code) + code[skip] = _len(code) - skip elif op is ANY: if flags & SRE_FLAG_DOTALL: emit(ANY_ALL) @@ -277,6 +284,10 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): charmap[i] = 1 elif op is NEGATE: out.append((op, av)) + elif op is CATEGORY and tail and (CATEGORY, CH_NEGATE[av]) in tail: + # Optimize [\s\S] etc. + out = [] if out else _CHARSET_ALL + return out, False else: tail.append((op, av)) except IndexError: @@ -519,13 +530,18 @@ def _compile_info(code, pattern, flags): # look for a literal prefix prefix = [] prefix_skip = 0 - charset = [] # not used + charset = None # not used if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE): # look for literal prefix prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags) # if no prefix, look for charset prefix if not prefix: charset = _get_charset_prefix(pattern, flags) + if charset: + charset, hascased = _optimize_charset(charset) + assert not hascased + if charset == _CHARSET_ALL: + charset = None ## if prefix: ## print("*** PREFIX", prefix, prefix_skip) ## if charset: @@ -560,8 +576,6 @@ def _compile_info(code, pattern, flags): # generate overlap table code.extend(_generate_overlap_table(prefix)) elif charset: - charset, hascased = _optimize_charset(charset) - assert not hascased _compile_charset(charset, flags, code) code[skip] = len(code) - skip diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index 9c3c294ba448b4..d6f32302d37b2d 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -15,7 +15,7 @@ MAGIC = 20230612 -from _sre import MAXREPEAT, MAXGROUPS +from _sre import MAXREPEAT, MAXGROUPS # noqa: F401 # SRE standard exception (access as sre.error) # should this really be here? @@ -206,6 +206,8 @@ def _makecodes(*names): CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK } +CH_NEGATE = dict(zip(CHCODES[::2] + CHCODES[1::2], CHCODES[1::2] + CHCODES[::2])) + # flags SRE_FLAG_IGNORECASE = 2 # case insensitive SRE_FLAG_LOCALE = 4 # honour system locale diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index f3c779340fe230..0990255b22c219 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -807,14 +807,6 @@ def _parse(source, state, verbose, nested, first=False): state.grouprefpos[condgroup] = ( source.tell() - len(condname) - 1 ) - if not (condname.isdecimal() and condname.isascii()): - import warnings - warnings.warn( - "bad character in group name %s at position %d" % - (repr(condname) if source.istext else ascii(condname), - source.tell() - len(condname) - 1), - DeprecationWarning, stacklevel=nested + 6 - ) state.checklookbehindgroup(condgroup, source) item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): @@ -1038,14 +1030,6 @@ def addgroup(index, pos): if index >= MAXGROUPS: raise s.error("invalid group reference %d" % index, len(name) + 1) - if not (name.isdecimal() and name.isascii()): - import warnings - warnings.warn( - "bad character in group name %s at position %d" % - (repr(name) if s.istext else ascii(name), - s.tell() - len(name) - 1), - DeprecationWarning, stacklevel=5 - ) addgroup(index, len(name) + 1) elif c == "0": if s.next in OCTDIGITS: diff --git a/Lib/reprlib.py b/Lib/reprlib.py index 05bb1a0eb01795..19dbe3a07eb618 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -36,6 +36,17 @@ def wrapper(self): return decorating_function class Repr: + _lookup = { + 'tuple': 'builtins', + 'list': 'builtins', + 'array': 'array', + 'set': 'builtins', + 'frozenset': 'builtins', + 'deque': 'collections', + 'dict': 'builtins', + 'str': 'builtins', + 'int': 'builtins' + } def __init__( self, *, maxlevel=6, maxtuple=6, maxlist=6, maxarray=5, maxdict=4, @@ -60,14 +71,24 @@ def repr(self, x): return self.repr1(x, self.maxlevel) def repr1(self, x, level): - typename = type(x).__name__ + cls = type(x) + typename = cls.__name__ + if ' ' in typename: parts = typename.split() typename = '_'.join(parts) - if hasattr(self, 'repr_' + typename): - return getattr(self, 'repr_' + typename)(x, level) - else: - return self.repr_instance(x, level) + + method = getattr(self, 'repr_' + typename, None) + if method: + # not defined in this class + if typename not in self._lookup: + return method(x, level) + module = getattr(cls, '__module__', None) + # defined in this class and is the module intended + if module == self._lookup[typename]: + return method(x, level) + + return self.repr_instance(x, level) def _join(self, pieces, level): if self.indent is None: diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 206d6fb511cdf6..23eb0020f42e8a 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -35,6 +35,7 @@ import keyword import re import __main__ +import warnings __all__ = ["Completer"] @@ -88,10 +89,11 @@ def complete(self, text, state): return None if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) + with warnings.catch_warnings(action="ignore"): + if "." in text: + self.matches = self.attr_matches(text) + else: + self.matches = self.global_matches(text) try: return self.matches[state] except IndexError: diff --git a/Lib/sched.py b/Lib/sched.py index 14613cf29874da..fb20639d459967 100644 --- a/Lib/sched.py +++ b/Lib/sched.py @@ -11,7 +11,7 @@ implement simulated time by writing your own functions. This can also be used to integrate scheduling with STDWIN events; the delay function is allowed to modify the queue. Time can be expressed as -integers or floating point numbers, as long as it is consistent. +integers or floating-point numbers, as long as it is consistent. Events are specified by tuples (time, priority, action, argument, kwargs). As in UNIX, lower priority numbers mean higher priority; in this diff --git a/Lib/shutil.py b/Lib/shutil.py index c19ea0607208af..dd3e0e0c5da54b 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -44,10 +44,11 @@ else: _winapi = None -COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 +COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 256 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 -_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") +_USE_CP_SENDFILE = (hasattr(os, "sendfile") + and sys.platform.startswith(("linux", "android", "sunos"))) _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 @@ -55,7 +56,7 @@ __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "copytree", "move", "rmtree", "Error", "SpecialFileError", - "ExecError", "make_archive", "get_archive_formats", + "make_archive", "get_archive_formats", "register_archive_format", "unregister_archive_format", "get_unpack_formats", "register_unpack_format", "unregister_unpack_format", "unpack_archive", @@ -73,8 +74,6 @@ class SpecialFileError(OSError): """Raised when trying to do a kind of operation (e.g. copying) which is not supported on a special file (e.g. a named pipe)""" -class ExecError(OSError): - """Raised when a command could not be executed""" class ReadError(OSError): """Raised when an archive cannot be read""" @@ -111,7 +110,7 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags): def _fastcopy_sendfile(fsrc, fdst): """Copy data from one regular mmap-like fd to another by using high-performance sendfile(2) syscall. - This should work on Linux >= 2.6.33 only. + This should work on Linux >= 2.6.33, Android and Solaris. """ # Note: copyfileobj() is left alone in order to not introduce any # unexpected breakage. Possible risks by using zero-copy calls @@ -148,7 +147,7 @@ def _fastcopy_sendfile(fsrc, fdst): try: sent = os.sendfile(outfd, infd, offset, blocksize) except OSError as err: - # ...in oder to have a more informative exception. + # ...in order to have a more informative exception. err.filename = fsrc.name err.filename2 = fdst.name @@ -266,7 +265,7 @@ def copyfile(src, dst, *, follow_symlinks=True): return dst except _GiveupOnFastCopy: pass - # Linux + # Linux / Android / Solaris elif _USE_CP_SENDFILE: try: _fastcopy_sendfile(fsrc, fdst) @@ -555,7 +554,7 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't + links are copied. If the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. @@ -604,38 +603,37 @@ def _rmtree_islink(st): return stat.S_ISLNK(st.st_mode) # version vulnerable to race conditions -def _rmtree_unsafe(path, onexc): +def _rmtree_unsafe(path, dir_fd, onexc): + if dir_fd is not None: + raise NotImplementedError("dir_fd unavailable on this platform") try: - with os.scandir(path) as scandir_it: - entries = list(scandir_it) - except FileNotFoundError: + st = os.lstat(path) + except OSError as err: + onexc(os.lstat, path, err) return + try: + if _rmtree_islink(st): + # symlinks to directories are forbidden, see bug #1669 + raise OSError("Cannot call rmtree on a symbolic link") except OSError as err: - onexc(os.scandir, path, err) - entries = [] - for entry in entries: - fullname = entry.path - try: - is_dir = entry.is_dir(follow_symlinks=False) - except FileNotFoundError: - continue - except OSError: - is_dir = False - - if is_dir and not entry.is_junction(): + onexc(os.path.islink, path, err) + # can't continue even if onexc hook returns + return + def onerror(err): + if not isinstance(err, FileNotFoundError): + onexc(os.scandir, err.filename, err) + results = os.walk(path, topdown=False, onerror=onerror, followlinks=os._walk_symlinks_as_files) + for dirpath, dirnames, filenames in results: + for name in dirnames: + fullname = os.path.join(dirpath, name) try: - if entry.is_symlink(): - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or entry.is_dir above. - raise OSError("Cannot call rmtree on a symbolic link") + os.rmdir(fullname) except FileNotFoundError: continue except OSError as err: - onexc(os.path.islink, fullname, err) - continue - _rmtree_unsafe(fullname, onexc) - else: + onexc(os.rmdir, fullname, err) + for name in filenames: + fullname = os.path.join(dirpath, name) try: os.unlink(fullname) except FileNotFoundError: @@ -650,86 +648,101 @@ def _rmtree_unsafe(path, onexc): onexc(os.rmdir, path, err) # Version using fd-based APIs to protect against races -def _rmtree_safe_fd(topfd, path, onexc): +def _rmtree_safe_fd(path, dir_fd, onexc): + # While the unsafe rmtree works fine on bytes, the fd based does not. + if isinstance(path, bytes): + path = os.fsdecode(path) + stack = [(os.lstat, dir_fd, path, None)] + try: + while stack: + _rmtree_safe_fd_step(stack, onexc) + finally: + # Close any file descriptors still on the stack. + while stack: + func, fd, path, entry = stack.pop() + if func is not os.close: + continue + try: + os.close(fd) + except OSError as err: + onexc(os.close, path, err) + +def _rmtree_safe_fd_step(stack, onexc): + # Each stack item has four elements: + # * func: The first operation to perform: os.lstat, os.close or os.rmdir. + # Walking a directory starts with an os.lstat() to detect symlinks; in + # this case, func is updated before subsequent operations and passed to + # onexc() if an error occurs. + # * dirfd: Open file descriptor, or None if we're processing the top-level + # directory given to rmtree() and the user didn't supply dir_fd. + # * path: Path of file to operate upon. This is passed to onexc() if an + # error occurs. + # * orig_entry: os.DirEntry, or None if we're processing the top-level + # directory given to rmtree(). We used the cached stat() of the entry to + # save a call to os.lstat() when walking subdirectories. + func, dirfd, path, orig_entry = stack.pop() + name = path if orig_entry is None else orig_entry.name try: + if func is os.close: + os.close(dirfd) + return + if func is os.rmdir: + os.rmdir(name, dir_fd=dirfd) + return + + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + assert func is os.lstat + if orig_entry is None: + orig_st = os.lstat(name, dir_fd=dirfd) + else: + orig_st = orig_entry.stat(follow_symlinks=False) + + func = os.open # For error reporting. + topfd = os.open(name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dirfd) + + func = os.path.islink # For error reporting. + try: + if not os.path.samestat(orig_st, os.fstat(topfd)): + # Symlinks to directories are forbidden, see GH-46010. + raise OSError("Cannot call rmtree on a symbolic link") + stack.append((os.rmdir, dirfd, path, orig_entry)) + finally: + stack.append((os.close, topfd, path, orig_entry)) + + func = os.scandir # For error reporting. with os.scandir(topfd) as scandir_it: entries = list(scandir_it) - except FileNotFoundError: - return - except OSError as err: - err.filename = path - onexc(os.scandir, path, err) - return - for entry in entries: - fullname = os.path.join(path, entry.name) - try: - is_dir = entry.is_dir(follow_symlinks=False) - except FileNotFoundError: - continue - except OSError: - is_dir = False - else: - if is_dir: - try: - orig_st = entry.stat(follow_symlinks=False) - is_dir = stat.S_ISDIR(orig_st.st_mode) - except FileNotFoundError: - continue - except OSError as err: - onexc(os.lstat, fullname, err) - continue - if is_dir: + for entry in entries: + fullname = os.path.join(path, entry.name) try: - dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) - dirfd_closed = False + if entry.is_dir(follow_symlinks=False): + # Traverse into sub-directory. + stack.append((os.lstat, topfd, fullname, entry)) + continue except FileNotFoundError: continue - except OSError as err: - onexc(os.open, fullname, err) - else: - try: - if os.path.samestat(orig_st, os.fstat(dirfd)): - _rmtree_safe_fd(dirfd, fullname, onexc) - try: - os.close(dirfd) - except OSError as err: - # close() should not be retried after an error. - dirfd_closed = True - onexc(os.close, fullname, err) - dirfd_closed = True - try: - os.rmdir(entry.name, dir_fd=topfd) - except FileNotFoundError: - continue - except OSError as err: - onexc(os.rmdir, fullname, err) - else: - try: - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or stat.S_ISDIR above. - raise OSError("Cannot call rmtree on a symbolic " - "link") - except OSError as err: - onexc(os.path.islink, fullname, err) - finally: - if not dirfd_closed: - try: - os.close(dirfd) - except OSError as err: - onexc(os.close, fullname, err) - else: + except OSError: + pass try: os.unlink(entry.name, dir_fd=topfd) except FileNotFoundError: continue except OSError as err: onexc(os.unlink, fullname, err) + except FileNotFoundError as err: + if orig_entry is None or func is os.close: + err.filename = path + onexc(func, path, err) + except OSError as err: + err.filename = path + onexc(func, path, err) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and os.scandir in os.supports_fd and os.stat in os.supports_follow_symlinks) +_rmtree_impl = _rmtree_safe_fd if _use_fd_functions else _rmtree_unsafe def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): """Recursively delete a directory tree. @@ -773,66 +786,7 @@ def onexc(*args): exc_info = type(exc), exc, exc.__traceback__ return onerror(func, path, exc_info) - if _use_fd_functions: - # While the unsafe rmtree works fine on bytes, the fd based does not. - if isinstance(path, bytes): - path = os.fsdecode(path) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - try: - orig_st = os.lstat(path, dir_fd=dir_fd) - except OSError as err: - onexc(os.lstat, path, err) - return - try: - fd = os.open(path, os.O_RDONLY, dir_fd=dir_fd) - fd_closed = False - except OSError as err: - onexc(os.open, path, err) - return - try: - if os.path.samestat(orig_st, os.fstat(fd)): - _rmtree_safe_fd(fd, path, onexc) - try: - os.close(fd) - except OSError as err: - # close() should not be retried after an error. - fd_closed = True - onexc(os.close, path, err) - fd_closed = True - try: - os.rmdir(path, dir_fd=dir_fd) - except OSError as err: - onexc(os.rmdir, path, err) - else: - try: - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) - finally: - if not fd_closed: - try: - os.close(fd) - except OSError as err: - onexc(os.close, path, err) - else: - if dir_fd is not None: - raise NotImplementedError("dir_fd unavailable on this platform") - try: - st = os.lstat(path) - except OSError as err: - onexc(os.lstat, path, err) - return - try: - if _rmtree_islink(st): - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) - # can't continue even if onexc hook returns - return - return _rmtree_unsafe(path, onexc) + _rmtree_impl(path, dir_fd, onexc) # Allow introspection of whether or not the hardening against symlink # attacks is supported on the current platform @@ -1441,11 +1395,18 @@ def disk_usage(path): return _ntuple_diskusage(total, used, free) -def chown(path, user=None, group=None): +def chown(path, user=None, group=None, *, dir_fd=None, follow_symlinks=True): """Change owner user and group of the given path. user and group can be the uid/gid or the user/group names, and in that case, they are converted to their respective uid/gid. + + If dir_fd is set, it should be an open file descriptor to the directory to + be used as the root of *path* if it is relative. + + If follow_symlinks is set to False and the last element of the path is a + symbolic link, chown will modify the link itself and not the file being + referenced by the link. """ sys.audit('shutil.chown', path, user, group) @@ -1471,7 +1432,8 @@ def chown(path, user=None, group=None): if _group is None: raise LookupError("no such group: {!r}".format(group)) - os.chown(path, _user, _group) + os.chown(path, _user, _group, dir_fd=dir_fd, + follow_symlinks=follow_symlinks) def get_terminal_size(fallback=(80, 24)): """Get the size of the terminal window. @@ -1618,3 +1580,15 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if _access_check(name, mode): return name return None + +def __getattr__(name): + if name == "ExecError": + import warnings + warnings._deprecated( + "shutil.ExecError", + f"{warnings._DEPRECATED_MSG}; it " + "isn't raised by any shutil function.", + remove=(3, 16) + ) + return RuntimeError + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/site.py b/Lib/site.py index 0631f3f6115ec0..07a6361fad44e5 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -179,35 +179,46 @@ def addpackage(sitedir, name, known_paths): return _trace(f"Processing .pth file: {fullname!r}") try: - # locale encoding is not ideal especially on Windows. But we have used - # it for a long time. setuptools uses the locale encoding too. - f = io.TextIOWrapper(io.open_code(fullname), encoding="locale") + with io.open_code(fullname) as f: + pth_content = f.read() except OSError: return - with f: - for n, line in enumerate(f): - if line.startswith("#"): - continue - if line.strip() == "": + + try: + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") + except UnicodeDecodeError: + # Fallback to locale encoding for backward compatibility. + # We will deprecate this fallback in the future. + import locale + pth_content = pth_content.decode(locale.getencoding()) + _trace(f"Cannot read {fullname!r} as UTF-8. " + f"Using fallback encoding {locale.getencoding()!r}") + + for n, line in enumerate(pth_content.splitlines(), 1): + if line.startswith("#"): + continue + if line.strip() == "": + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) continue - try: - if line.startswith(("import ", "import\t")): - exec(line) - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - except Exception as exc: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(exc): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if dircase not in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print(f"Error processing line {n:d} of {fullname}:\n", + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break if reset: known_paths = None return known_paths @@ -280,8 +291,8 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -301,6 +312,10 @@ def joinuser(*args): # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' implementation = _get_implementation() implementation_lower = implementation.lower() @@ -311,7 +326,7 @@ def _get_path(userbase): if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' + return f'{userbase}/lib/python{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -379,6 +394,10 @@ def getsitepackages(prefixes=None): implementation = _get_implementation().lower() ver = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -386,7 +405,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - f"{implementation}{ver[0]}.{ver[1]}", + f"{implementation}{ver[0]}.{ver[1]}{abi_thread}", "site-packages") sitepackages.append(path) else: @@ -460,60 +479,88 @@ def gethistoryfile(): def enablerlcompleter(): """Enable default readline configuration on interactive prompts, by registering a sys.__interactivehook__. + """ + sys.__interactivehook__ = register_readline + + +def register_readline(): + """Configure readline completion on interactive prompts. If the readline module can be imported, the hook will set the Tab key as completion key and register ~/.python_history as history file. This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ - def register_readline(): - import atexit - try: - import readline - import rlcompleter - except ImportError: - return - - # Reading the initialization (config) file may not be enough to set a - # completion key, so we set one first and then read the file. - if readline.backend == 'editline': - readline.parse_and_bind('bind ^I rl_complete') + if not sys.flags.ignore_environment: + PYTHON_BASIC_REPL = os.getenv("PYTHON_BASIC_REPL") + else: + PYTHON_BASIC_REPL = False + + import atexit + try: + import readline + import rlcompleter # noqa: F401 + if PYTHON_BASIC_REPL: + CAN_USE_PYREPL = False else: - readline.parse_and_bind('tab: complete') + original_path = sys.path + sys.path = [p for p in original_path if p != ''] + try: + import _pyrepl.readline + import _pyrepl.unix_console + from _pyrepl.main import CAN_USE_PYREPL + finally: + sys.path = original_path + except ImportError: + return + + # Reading the initialization (config) file may not be enough to set a + # completion key, so we set one first and then read the file. + if readline.backend == 'editline': + readline.parse_and_bind('bind ^I rl_complete') + else: + readline.parse_and_bind('tab: complete') + + try: + readline.read_init_file() + except OSError: + # An OSError here could have many causes, but the most likely one + # is that there's no .inputrc file (or .editrc file in the case of + # Mac OS X + libedit) in the expected location. In that case, we + # want to ignore the exception. + pass + + if readline.get_current_history_length() == 0: + # If no history was loaded, default to .python_history, + # or PYTHON_HISTORY. + # The guard is necessary to avoid doubling history size at + # each interpreter exit when readline was already configured + # through a PYTHONSTARTUP hook, see: + # http://bugs.python.org/issue5845#msg198636 + history = gethistoryfile() + + if CAN_USE_PYREPL: + readline_module = _pyrepl.readline + exceptions = (OSError, *_pyrepl.unix_console._error) + else: + readline_module = readline + exceptions = OSError try: - readline.read_init_file() - except OSError: - # An OSError here could have many causes, but the most likely one - # is that there's no .inputrc file (or .editrc file in the case of - # Mac OS X + libedit) in the expected location. In that case, we - # want to ignore the exception. + readline_module.read_history_file(history) + except exceptions: pass - if readline.get_current_history_length() == 0: - # If no history was loaded, default to .python_history, - # or PYTHON_HISTORY. - # The guard is necessary to avoid doubling history size at - # each interpreter exit when readline was already configured - # through a PYTHONSTARTUP hook, see: - # http://bugs.python.org/issue5845#msg198636 - history = gethistoryfile() + def write_history(): try: - readline.read_history_file(history) - except OSError: + readline_module.write_history_file(history) + except (FileNotFoundError, PermissionError): + # home directory does not exist or is not writable + # https://bugs.python.org/issue19891 pass - def write_history(): - try: - readline.write_history_file(history) - except OSError: - # bpo-19891, bpo-41193: Home directory does not exist - # or is not writable, or the filesystem is read-only. - pass - - atexit.register(write_history) + atexit.register(write_history) - sys.__interactivehook__ = register_readline def venv(known_paths): global PREFIXES, ENABLE_USER_SITE @@ -574,7 +621,7 @@ def execsitecustomize(): """Run custom site specific code, if available.""" try: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError as exc: if exc.name == 'sitecustomize': pass @@ -594,7 +641,7 @@ def execusercustomize(): """Run custom user specific code, if available.""" try: try: - import usercustomize + import usercustomize # noqa: F401 except ImportError as exc: if exc.name == 'usercustomize': pass diff --git a/Lib/smtplib.py b/Lib/smtplib.py old mode 100755 new mode 100644 index b3cc68a789a7d8..84d6d858e7dec1 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - '''SMTP/ESMTP client class. This should follow RFC 821 (SMTP), RFC 1869 (ESMTP), RFC 2554 (SMTP @@ -105,7 +103,7 @@ class SMTPSenderRefused(SMTPResponseException): """Sender address refused. In addition to the attributes set by on all SMTPResponseException - exceptions, this sets `sender' to the string that the SMTP refused. + exceptions, this sets 'sender' to the string that the SMTP refused. """ def __init__(self, code, msg, sender): @@ -315,7 +313,7 @@ def _get_socket(self, host, port, timeout): def connect(self, host='localhost', port=0, source_address=None): """Connect to a host on a given port. - If the hostname ends with a colon (`:') followed by a number, and + If the hostname ends with a colon (':') followed by a number, and there is no port specified, that suffix will be stripped off and the number interpreted as the port number to use. @@ -346,7 +344,7 @@ def connect(self, host='localhost', port=0, source_address=None): return (code, msg) def send(self, s): - """Send `s' to the server.""" + """Send 's' to the server.""" if self.debuglevel > 0: self._print_debug('send:', repr(s)) if self.sock: diff --git a/Lib/socket.py b/Lib/socket.py index 77986fc2e48099..be37c24d6174a2 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -52,7 +52,9 @@ import _socket from _socket import * -import os, sys, io, selectors +import io +import os +import sys from enum import IntEnum, IntFlag try: @@ -110,102 +112,103 @@ def _intenum_converter(value, enum_klass): # WSA error codes if sys.platform.lower().startswith("win"): - errorTab = {} - errorTab[6] = "Specified event object handle is invalid." - errorTab[8] = "Insufficient memory available." - errorTab[87] = "One or more parameters are invalid." - errorTab[995] = "Overlapped operation aborted." - errorTab[996] = "Overlapped I/O event object not in signaled state." - errorTab[997] = "Overlapped operation will complete later." - errorTab[10004] = "The operation was interrupted." - errorTab[10009] = "A bad file handle was passed." - errorTab[10013] = "Permission denied." - errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT - errorTab[10022] = "An invalid operation was attempted." - errorTab[10024] = "Too many open files." - errorTab[10035] = "The socket operation would block." - errorTab[10036] = "A blocking operation is already in progress." - errorTab[10037] = "Operation already in progress." - errorTab[10038] = "Socket operation on nonsocket." - errorTab[10039] = "Destination address required." - errorTab[10040] = "Message too long." - errorTab[10041] = "Protocol wrong type for socket." - errorTab[10042] = "Bad protocol option." - errorTab[10043] = "Protocol not supported." - errorTab[10044] = "Socket type not supported." - errorTab[10045] = "Operation not supported." - errorTab[10046] = "Protocol family not supported." - errorTab[10047] = "Address family not supported by protocol family." - errorTab[10048] = "The network address is in use." - errorTab[10049] = "Cannot assign requested address." - errorTab[10050] = "Network is down." - errorTab[10051] = "Network is unreachable." - errorTab[10052] = "Network dropped connection on reset." - errorTab[10053] = "Software caused connection abort." - errorTab[10054] = "The connection has been reset." - errorTab[10055] = "No buffer space available." - errorTab[10056] = "Socket is already connected." - errorTab[10057] = "Socket is not connected." - errorTab[10058] = "The network has been shut down." - errorTab[10059] = "Too many references." - errorTab[10060] = "The operation timed out." - errorTab[10061] = "Connection refused." - errorTab[10062] = "Cannot translate name." - errorTab[10063] = "The name is too long." - errorTab[10064] = "The host is down." - errorTab[10065] = "The host is unreachable." - errorTab[10066] = "Directory not empty." - errorTab[10067] = "Too many processes." - errorTab[10068] = "User quota exceeded." - errorTab[10069] = "Disk quota exceeded." - errorTab[10070] = "Stale file handle reference." - errorTab[10071] = "Item is remote." - errorTab[10091] = "Network subsystem is unavailable." - errorTab[10092] = "Winsock.dll version out of range." - errorTab[10093] = "Successful WSAStartup not yet performed." - errorTab[10101] = "Graceful shutdown in progress." - errorTab[10102] = "No more results from WSALookupServiceNext." - errorTab[10103] = "Call has been canceled." - errorTab[10104] = "Procedure call table is invalid." - errorTab[10105] = "Service provider is invalid." - errorTab[10106] = "Service provider failed to initialize." - errorTab[10107] = "System call failure." - errorTab[10108] = "Service not found." - errorTab[10109] = "Class type not found." - errorTab[10110] = "No more results from WSALookupServiceNext." - errorTab[10111] = "Call was canceled." - errorTab[10112] = "Database query was refused." - errorTab[11001] = "Host not found." - errorTab[11002] = "Nonauthoritative host not found." - errorTab[11003] = "This is a nonrecoverable error." - errorTab[11004] = "Valid name, no data record requested type." - errorTab[11005] = "QoS receivers." - errorTab[11006] = "QoS senders." - errorTab[11007] = "No QoS senders." - errorTab[11008] = "QoS no receivers." - errorTab[11009] = "QoS request confirmed." - errorTab[11010] = "QoS admission error." - errorTab[11011] = "QoS policy failure." - errorTab[11012] = "QoS bad style." - errorTab[11013] = "QoS bad object." - errorTab[11014] = "QoS traffic control error." - errorTab[11015] = "QoS generic error." - errorTab[11016] = "QoS service type error." - errorTab[11017] = "QoS flowspec error." - errorTab[11018] = "Invalid QoS provider buffer." - errorTab[11019] = "Invalid QoS filter style." - errorTab[11020] = "Invalid QoS filter style." - errorTab[11021] = "Incorrect QoS filter count." - errorTab[11022] = "Invalid QoS object length." - errorTab[11023] = "Incorrect QoS flow count." - errorTab[11024] = "Unrecognized QoS object." - errorTab[11025] = "Invalid QoS policy object." - errorTab[11026] = "Invalid QoS flow descriptor." - errorTab[11027] = "Invalid QoS provider-specific flowspec." - errorTab[11028] = "Invalid QoS provider-specific filterspec." - errorTab[11029] = "Invalid QoS shape discard mode object." - errorTab[11030] = "Invalid QoS shaping rate object." - errorTab[11031] = "Reserved policy QoS element type." + errorTab = { + 6: "Specified event object handle is invalid.", + 8: "Insufficient memory available.", + 87: "One or more parameters are invalid.", + 995: "Overlapped operation aborted.", + 996: "Overlapped I/O event object not in signaled state.", + 997: "Overlapped operation will complete later.", + 10004: "The operation was interrupted.", + 10009: "A bad file handle was passed.", + 10013: "Permission denied.", + 10014: "A fault occurred on the network??", + 10022: "An invalid operation was attempted.", + 10024: "Too many open files.", + 10035: "The socket operation would block.", + 10036: "A blocking operation is already in progress.", + 10037: "Operation already in progress.", + 10038: "Socket operation on nonsocket.", + 10039: "Destination address required.", + 10040: "Message too long.", + 10041: "Protocol wrong type for socket.", + 10042: "Bad protocol option.", + 10043: "Protocol not supported.", + 10044: "Socket type not supported.", + 10045: "Operation not supported.", + 10046: "Protocol family not supported.", + 10047: "Address family not supported by protocol family.", + 10048: "The network address is in use.", + 10049: "Cannot assign requested address.", + 10050: "Network is down.", + 10051: "Network is unreachable.", + 10052: "Network dropped connection on reset.", + 10053: "Software caused connection abort.", + 10054: "The connection has been reset.", + 10055: "No buffer space available.", + 10056: "Socket is already connected.", + 10057: "Socket is not connected.", + 10058: "The network has been shut down.", + 10059: "Too many references.", + 10060: "The operation timed out.", + 10061: "Connection refused.", + 10062: "Cannot translate name.", + 10063: "The name is too long.", + 10064: "The host is down.", + 10065: "The host is unreachable.", + 10066: "Directory not empty.", + 10067: "Too many processes.", + 10068: "User quota exceeded.", + 10069: "Disk quota exceeded.", + 10070: "Stale file handle reference.", + 10071: "Item is remote.", + 10091: "Network subsystem is unavailable.", + 10092: "Winsock.dll version out of range.", + 10093: "Successful WSAStartup not yet performed.", + 10101: "Graceful shutdown in progress.", + 10102: "No more results from WSALookupServiceNext.", + 10103: "Call has been canceled.", + 10104: "Procedure call table is invalid.", + 10105: "Service provider is invalid.", + 10106: "Service provider failed to initialize.", + 10107: "System call failure.", + 10108: "Service not found.", + 10109: "Class type not found.", + 10110: "No more results from WSALookupServiceNext.", + 10111: "Call was canceled.", + 10112: "Database query was refused.", + 11001: "Host not found.", + 11002: "Nonauthoritative host not found.", + 11003: "This is a nonrecoverable error.", + 11004: "Valid name, no data record requested type.", + 11005: "QoS receivers.", + 11006: "QoS senders.", + 11007: "No QoS senders.", + 11008: "QoS no receivers.", + 11009: "QoS request confirmed.", + 11010: "QoS admission error.", + 11011: "QoS policy failure.", + 11012: "QoS bad style.", + 11013: "QoS bad object.", + 11014: "QoS traffic control error.", + 11015: "QoS generic error.", + 11016: "QoS service type error.", + 11017: "QoS flowspec error.", + 11018: "Invalid QoS provider buffer.", + 11019: "Invalid QoS filter style.", + 11020: "Invalid QoS filter style.", + 11021: "Incorrect QoS filter count.", + 11022: "Invalid QoS object length.", + 11023: "Incorrect QoS flow count.", + 11024: "Unrecognized QoS object.", + 11025: "Invalid QoS policy object.", + 11026: "Invalid QoS flow descriptor.", + 11027: "Invalid QoS provider-specific flowspec.", + 11028: "Invalid QoS provider-specific filterspec.", + 11029: "Invalid QoS shape discard mode object.", + 11030: "Invalid QoS shaping rate object.", + 11031: "Reserved policy QoS element type." + } __all__.append("errorTab") @@ -306,7 +309,8 @@ def makefile(self, mode="r", buffering=None, *, """makefile(...) -> an I/O stream connected to the socket The arguments are as for io.open() after the filename, except the only - supported mode values are 'r' (default), 'w' and 'b'. + supported mode values are 'r' (default), 'w', 'b', or a combination of + those. """ # XXX refactor to share code? if not set(mode) <= {"r", "w", "b"}: @@ -347,6 +351,9 @@ def makefile(self, mode="r", buffering=None, *, if hasattr(os, 'sendfile'): def _sendfile_use_sendfile(self, file, offset=0, count=None): + # Lazy import to improve module import time + import selectors + self._check_sendfile_params(file, offset, count) sockno = self.fileno() try: @@ -548,20 +555,18 @@ def fromfd(fd, family, type, proto=0): return socket(family, type, proto, nfd) if hasattr(_socket.socket, "sendmsg"): - import array - def send_fds(sock, buffers, fds, flags=0, address=None): """ send_fds(sock, buffers, fds[, flags[, address]]) -> integer Send the list of file descriptors fds over an AF_UNIX socket. """ + import array + return sock.sendmsg(buffers, [(_socket.SOL_SOCKET, _socket.SCM_RIGHTS, array.array("i", fds))]) __all__.append("send_fds") if hasattr(_socket.socket, "recvmsg"): - import array - def recv_fds(sock, bufsize, maxfds, flags=0): """ recv_fds(sock, bufsize, maxfds[, flags]) -> (data, list of file descriptors, msg_flags, address) @@ -569,6 +574,8 @@ def recv_fds(sock, bufsize, maxfds, flags=0): Receive up to maxfds file descriptors returning the message data and a list containing the descriptors. """ + import array + # Array of ints fds = array.array("i") msg, ancdata, flags, addr = sock.recvmsg(bufsize, @@ -591,16 +598,65 @@ def fromshare(info): return socket(0, 0, 0, info) __all__.append("fromshare") -if hasattr(_socket, "socketpair"): +# Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. +# This is used if _socket doesn't natively provide socketpair. It's +# always defined so that it can be patched in for testing purposes. +def _fallback_socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): + if family == AF_INET: + host = _LOCALHOST + elif family == AF_INET6: + host = _LOCALHOST_V6 + else: + raise ValueError("Only AF_INET and AF_INET6 socket address families " + "are supported") + if type != SOCK_STREAM: + raise ValueError("Only SOCK_STREAM socket type is supported") + if proto != 0: + raise ValueError("Only protocol zero is supported") + + # We create a connected TCP socket. Note the trick with + # setblocking(False) that prevents us from having to create a thread. + lsock = socket(family, type, proto) + try: + lsock.bind((host, 0)) + lsock.listen() + # On IPv6, ignore flow_info and scope_id + addr, port = lsock.getsockname()[:2] + csock = socket(family, type, proto) + try: + csock.setblocking(False) + try: + csock.connect((addr, port)) + except (BlockingIOError, InterruptedError): + pass + csock.setblocking(True) + ssock, _ = lsock.accept() + except: + csock.close() + raise + finally: + lsock.close() - def socketpair(family=None, type=SOCK_STREAM, proto=0): - """socketpair([family[, type[, proto]]]) -> (socket object, socket object) + # Authenticating avoids using a connection from something else + # able to connect to {host}:{port} instead of us. + # We expect only AF_INET and AF_INET6 families. + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise - Create a pair of socket objects from the sockets returned by the platform - socketpair() function. - The arguments are the same as for socket() except the default family is - AF_UNIX if defined on the platform; otherwise, the default is AF_INET. - """ + return (ssock, csock) + +if hasattr(_socket, "socketpair"): + def socketpair(family=None, type=SOCK_STREAM, proto=0): if family is None: try: family = AF_UNIX @@ -612,44 +668,7 @@ def socketpair(family=None, type=SOCK_STREAM, proto=0): return a, b else: - - # Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. - def socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): - if family == AF_INET: - host = _LOCALHOST - elif family == AF_INET6: - host = _LOCALHOST_V6 - else: - raise ValueError("Only AF_INET and AF_INET6 socket address families " - "are supported") - if type != SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with - # setblocking(False) that prevents us from having to create a thread. - lsock = socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen() - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except (BlockingIOError, InterruptedError): - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except: - csock.close() - raise - finally: - lsock.close() - return (ssock, csock) + socketpair = _fallback_socketpair __all__.append("socketpair") socketpair.__doc__ = """socketpair([family[, type[, proto]]]) -> (socket object, socket object) diff --git a/Lib/sqlite3/__init__.py b/Lib/sqlite3/__init__.py index 927267cf0b92ff..34a9c047dd607c 100644 --- a/Lib/sqlite3/__init__.py +++ b/Lib/sqlite3/__init__.py @@ -55,16 +55,3 @@ """ from sqlite3.dbapi2 import * -from sqlite3.dbapi2 import (_deprecated_names, - _deprecated_version_info, - _deprecated_version) - - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index b93b84384a0925..cfdee61403d1fa 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -65,7 +65,6 @@ def runsource(self, source, filename="", symbol="single"): def main(*args): parser = ArgumentParser( description="Python sqlite3 CLI", - prog="python -m sqlite3", ) parser.add_argument( "filename", type=str, default=":memory:", nargs="?", @@ -117,7 +116,7 @@ def main(*args): # No SQL provided; start the REPL. console = SqliteInteractiveConsole(con) try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg="") diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 56fc0461e6c922..0315760516edf8 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -25,9 +25,6 @@ import collections.abc from _sqlite3 import * -from _sqlite3 import _deprecated_version - -_deprecated_names = frozenset({"version", "version_info"}) paramstyle = "qmark" @@ -48,7 +45,7 @@ def TimeFromTicks(ticks): def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) -_deprecated_version_info = tuple(map(int, _deprecated_version.split("."))) + sqlite_version_info = tuple([int(x) for x in sqlite_version.split(".")]) Binary = memoryview @@ -97,12 +94,3 @@ def convert_timestamp(val): # Clean up namespace del(register_adapters_and_converters) - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 9dcce7dc76ced4..57e6a3b4f1e6eb 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -26,6 +26,7 @@ def _iterdump(connection, *, filter=None): writeable_schema = False cu = connection.cursor() + cu.row_factory = None # Make sure we get predictable results. # Disable foreign key constraints, if there is any foreign key violation. violations = cu.execute("PRAGMA foreign_key_check").fetchall() if violations: diff --git a/Lib/ssl.py b/Lib/ssl.py index 03d0121891ff4c..c8703b046cfd4b 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -513,18 +513,17 @@ def set_alpn_protocols(self, alpn_protocols): self._set_alpn_protocols(protos) def _load_windows_store_certs(self, storename, purpose): - certs = bytearray() try: for cert, encoding, trust in enum_certificates(storename): # CA certs are never PKCS#7 encoded if encoding == "x509_asn": if trust is True or purpose.oid in trust: - certs.extend(cert) + try: + self.load_verify_locations(cadata=cert) + except SSLError as exc: + warnings.warn(f"Bad certificate in Windows certificate store: {exc!s}") except PermissionError: warnings.warn("unable to enumerate Windows certificate store") - if certs: - self.load_verify_locations(cadata=certs) - return certs def load_default_certs(self, purpose=Purpose.SERVER_AUTH): if not isinstance(purpose, _ASN1Object): @@ -704,6 +703,16 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, else: raise ValueError(purpose) + # `VERIFY_X509_PARTIAL_CHAIN` makes OpenSSL's chain building behave more + # like RFC 3280 and 5280, which specify that chain building stops with the + # first trust anchor, even if that anchor is not self-signed. + # + # `VERIFY_X509_STRICT` makes OpenSSL more conservative about the + # certificates it accepts, including "disabling workarounds for + # some broken certificates." + context.verify_flags |= (_ssl.VERIFY_X509_PARTIAL_CHAIN | + _ssl.VERIFY_X509_STRICT) + if cafile or capath or cadata: context.load_verify_locations(cafile, capath, cadata) elif context.verify_mode != CERT_NONE: @@ -1155,11 +1164,21 @@ def getpeercert(self, binary_form=False): @_sslcopydoc def get_verified_chain(self): - return self._sslobj.get_verified_chain() + chain = self._sslobj.get_verified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] @_sslcopydoc def get_unverified_chain(self): - return self._sslobj.get_unverified_chain() + chain = self._sslobj.get_unverified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] @_sslcopydoc def selected_npn_protocol(self): diff --git a/Lib/stat.py b/Lib/stat.py index 9167ab185944fb..1b4ed1ebc940ef 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -2,7 +2,6 @@ Suggested usage: from stat import * """ -import sys # Indices for stat struct members in the tuple returned by os.stat() diff --git a/Lib/statistics.py b/Lib/statistics.py index 83aaedb04515e0..e2b59267f04f68 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -11,7 +11,7 @@ Function Description ================== ================================================== mean Arithmetic mean (average) of data. -fmean Fast, floating point arithmetic mean. +fmean Fast, floating-point arithmetic mean. geometric_mean Geometric mean of data. harmonic_mean Harmonic mean of data. median Median (middle value) of data. @@ -112,6 +112,8 @@ 'fmean', 'geometric_mean', 'harmonic_mean', + 'kde', + 'kde_random', 'linear_regression', 'mean', 'median', @@ -137,454 +139,157 @@ from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod -from math import isfinite, isinf +from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos from functools import reduce from operator import itemgetter from collections import Counter, namedtuple, defaultdict _SQRT2 = sqrt(2.0) +_random = random -# === Exceptions === +## Exceptions ############################################################## class StatisticsError(ValueError): pass -# === Private utilities === +## Measures of central tendency (averages) ################################# -def _sum(data): - """_sum(data) -> (type, sum, count) - - Return a high-precision sum of the given numeric data as a fraction, - together with the type to be converted to and the count of items. - - Examples - -------- - - >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) - (, Fraction(19, 2), 5) - - Some sources of round-off error will be avoided: - - # Built-in sum returns zero. - >>> _sum([1e50, 1, -1e50] * 1000) - (, Fraction(1000, 1), 3000) +def mean(data): + """Return the sample arithmetic mean of data. - Fractions and Decimals are also supported: + >>> mean([1, 2, 3, 4, 4]) + 2.8 >>> from fractions import Fraction as F - >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - (, Fraction(63, 20), 4) + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) >>> from decimal import Decimal as D - >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] - >>> _sum(data) - (, Fraction(6963, 10000), 4) + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + If ``data`` is empty, StatisticsError will be raised. - Mixed types are currently treated as an error, except that int is - allowed. """ - count = 0 - types = set() - types_add = types.add - partials = {} - partials_get = partials.get - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - partials[d] = partials_get(d, 0) + n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - # Sum all the partial sums using builtin sum. - total = sum(Fraction(n, d) for d, n in partials.items()) - T = reduce(_coerce, types, int) # or raise TypeError - return (T, total, count) + T, total, n = _sum(data) + if n < 1: + raise StatisticsError('mean requires at least one data point') + return _convert(total / n, T) -def _ss(data, c=None): - """Return the exact mean and sum of square deviations of sequence data. +def fmean(data, weights=None): + """Convert data to floats and compute the arithmetic mean. - Calculations are done in a single pass, allowing the input to be an iterator. + This runs faster than the mean() function and it always returns a float. + If the input dataset is empty, it raises a StatisticsError. - If given *c* is used the mean; otherwise, it is calculated from the data. - Use the *c* argument with care, as it can lead to garbage results. + >>> fmean([3.5, 4.0, 5.25]) + 4.25 """ - if c is not None: - T, ssd, count = _sum((d := x - c) * d for x in data) - return (T, ssd, c, count) - count = 0 - types = set() - types_add = types.add - sx_partials = defaultdict(int) - sxx_partials = defaultdict(int) - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - sx_partials[d] += n - sxx_partials[d] += n * n - if not count: - ssd = c = Fraction(0) - elif None in sx_partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - ssd = c = sx_partials[None] - assert not _isfinite(ssd) - else: - sx = sum(Fraction(n, d) for d, n in sx_partials.items()) - sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) - # This formula has poor numeric properties for floats, - # but with fractions it is exact. - ssd = (count * sxx - sx * sx) / count - c = sx / count - T = reduce(_coerce, types, int) # or raise TypeError - return (T, ssd, c, count) + if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + counter = count() + total = fsum(map(itemgetter(0), zip(data, counter))) + n = next(counter) + else: + total = fsum(data) -def _isfinite(x): - try: - return x.is_finite() # Likely a Decimal. - except AttributeError: - return math.isfinite(x) # Coerces to float first. + if not n: + raise StatisticsError('fmean requires at least one data point') + return total / n -def _coerce(T, S): - """Coerce types T and S to a common type, or raise TypeError. + if not isinstance(weights, (list, tuple)): + weights = list(weights) - Coercion rules are currently an implementation detail. See the CoerceTest - test class in test_statistics for details. - """ - # See http://bugs.python.org/issue24068. - assert T is not bool, "initial type T is bool" - # If the types are the same, no need to coerce anything. Put this - # first, so that the usual case (no coercion needed) happens as soon - # as possible. - if T is S: return T - # Mixed int & other coerce to the other type. - if S is int or S is bool: return T - if T is int: return S - # If one is a (strict) subclass of the other, coerce to the subclass. - if issubclass(S, T): return S - if issubclass(T, S): return T - # Ints coerce to the other type. - if issubclass(T, int): return S - if issubclass(S, int): return T - # Mixed fraction & float coerces to float (or float subclass). - if issubclass(T, Fraction) and issubclass(S, float): - return S - if issubclass(T, float) and issubclass(S, Fraction): - return T - # Any other combination is disallowed. - msg = "don't know how to coerce %s and %s" - raise TypeError(msg % (T.__name__, S.__name__)) + try: + num = sumprod(data, weights) + except ValueError: + raise StatisticsError('data and weights must be the same length') + den = fsum(weights) -def _exact_ratio(x): - """Return Real number x to exact (numerator, denominator) pair. + if not den: + raise StatisticsError('sum of weights must be non-zero') - >>> _exact_ratio(0.25) - (1, 4) + return num / den - x is expected to be an int, Fraction, Decimal or float. - """ - # XXX We should revisit whether using fractions to accumulate exact - # ratios is the right way to go. +def geometric_mean(data): + """Convert data to floats and compute the geometric mean. - # The integer ratios for binary floats can have numerators or - # denominators with over 300 decimal digits. The problem is more - # acute with decimal floats where the default decimal context - # supports a huge range of exponents from Emin=-999999 to - # Emax=999999. When expanded with as_integer_ratio(), numbers like - # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large - # numerators or denominators that will slow computation. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. - # When the integer ratios are accumulated as fractions, the size - # grows to cover the full range from the smallest magnitude to the - # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300), - # has a 616 digit numerator. Likewise, - # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000')) - # has 10,003 digit numerator. + Returns zero if the product of inputs is zero. - # This doesn't seem to have been problem in practice, but it is a - # potential pitfall. + No special efforts are made to achieve exact results. + (However, this may change in the future.) - try: - return x.as_integer_ratio() - except AttributeError: - pass - except (OverflowError, ValueError): - # float NAN or INF. - assert not _isfinite(x) - return (x, None) - try: - # x may be an Integral ABC. - return (x.numerator, x.denominator) - except AttributeError: - msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" - raise TypeError(msg) + >>> round(geometric_mean([54, 24, 36]), 9) + 36.0 + """ + n = 0 + found_zero = False -def _convert(value, T): - """Convert value to given numeric type T.""" - if type(value) is T: - # This covers the cases where T is Fraction, or where value is - # a NAN or INF (Decimal or float). - return value - if issubclass(T, int) and value.denominator != 1: - T = float - try: - # FIXME: what do we do if this overflows? - return T(value) - except TypeError: - if issubclass(T, Decimal): - return T(value.numerator) / T(value.denominator) - else: - raise + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) -def _fail_neg(values, errmsg='negative value'): - """Iterate over values, failing if any are less than zero.""" - for x in values: - if x < 0: - raise StatisticsError(errmsg) - yield x + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 + return exp(total / n) -def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: - """Rank order a dataset. The lowest value has rank 1. - Ties are averaged so that equal values receive the same rank: +def harmonic_mean(data, weights=None): + """Return the harmonic mean of data. - >>> data = [31, 56, 31, 25, 75, 18] - >>> _rank(data) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + The harmonic mean is the reciprocal of the arithmetic mean of the + reciprocals of the data. It can be used for averaging ratios or + rates, for example speeds. - The operation is idempotent: + Suppose a car travels 40 km/hr for 5 km and then speeds-up to + 60 km/hr for another 5 km. What is the average speed? - >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + >>> harmonic_mean([40, 60]) + 48.0 - It is possible to rank the data in reverse order so that the - highest value has rank 1. Also, a key-function can extract - the field to be ranked: + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? - >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] - >>> _rank(goals, key=itemgetter(1), reverse=True) - [2.0, 1.0, 3.0] + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 - Ranks are conventionally numbered starting from one; however, - setting *start* to zero allows the ranks to be used as array indices: + If ``data`` is empty, or any element is less than zero, + ``harmonic_mean`` will raise ``StatisticsError``. - >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] - >>> scores = [8.1, 7.3, 9.4, 8.3] - >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] - ['Bronze', 'Certificate', 'Gold', 'Silver'] - - """ - # If this function becomes public at some point, more thought - # needs to be given to the signature. A list of ints is - # plausible when ties is "min" or "max". When ties is "average", - # either list[float] or list[Fraction] is plausible. - - # Default handling of ties matches scipy.stats.mstats.spearmanr. - if ties != 'average': - raise ValueError(f'Unknown tie resolution method: {ties!r}') - if key is not None: - data = map(key, data) - val_pos = sorted(zip(data, count()), reverse=reverse) - i = start - 1 - result = [0] * len(val_pos) - for _, g in groupby(val_pos, key=itemgetter(0)): - group = list(g) - size = len(group) - rank = i + (size + 1) / 2 - for value, orig_pos in group: - result[orig_pos] = rank - i += size - return result - - -def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: - """Square root of n/m, rounded to the nearest integer using round-to-odd.""" - # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf - a = math.isqrt(n // m) - return a | (a*a*m != n) - - -# For 53 bit precision floats, the bit width used in -# _float_sqrt_of_frac() is 109. -_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 - - -def _float_sqrt_of_frac(n: int, m: int) -> float: - """Square root of n/m as a float, correctly rounded.""" - # See principle and proof sketch at: https://bugs.python.org/msg407078 - q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 - if q >= 0: - numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q - denominator = 1 - else: - numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) - denominator = 1 << -q - return numerator / denominator # Convert to float - - -def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: - """Square root of n/m as a Decimal, correctly rounded.""" - # Premise: For decimal, computing (n/m).sqrt() can be off - # by 1 ulp from the correctly rounded result. - # Method: Check the result, moving up or down a step if needed. - if n <= 0: - if not n: - return Decimal('0.0') - n, m = -n, -m - - root = (Decimal(n) / Decimal(m)).sqrt() - nr, dr = root.as_integer_ratio() - - plus = root.next_plus() - np, dp = plus.as_integer_ratio() - # test: n / m > ((root + plus) / 2) ** 2 - if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: - return plus - - minus = root.next_minus() - nm, dm = minus.as_integer_ratio() - # test: n / m < ((root + minus) / 2) ** 2 - if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: - return minus - - return root - - -# === Measures of central tendency (averages) === - -def mean(data): - """Return the sample arithmetic mean of data. - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - If ``data`` is empty, StatisticsError will be raised. - """ - T, total, n = _sum(data) - if n < 1: - raise StatisticsError('mean requires at least one data point') - return _convert(total / n, T) - - -def fmean(data, weights=None): - """Convert data to floats and compute the arithmetic mean. - - This runs faster than the mean() function and it always returns a float. - If the input dataset is empty, it raises a StatisticsError. - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - """ - if weights is None: - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) - total = fsum(data) - if not n: - raise StatisticsError('fmean requires at least one data point') - return total / n - if not isinstance(weights, (list, tuple)): - weights = list(weights) - try: - num = sumprod(data, weights) - except ValueError: - raise StatisticsError('data and weights must be the same length') - den = fsum(weights) - if not den: - raise StatisticsError('sum of weights must be non-zero') - return num / den - - -def geometric_mean(data): - """Convert data to floats and compute the geometric mean. - - Raises a StatisticsError if the input dataset is empty - or if it contains a negative value. - - Returns zero if the product of inputs is zero. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - >>> round(geometric_mean([54, 24, 36]), 9) - 36.0 - """ - n = 0 - found_zero = False - def count_positive(iterable): - nonlocal n, found_zero - for n, x in enumerate(iterable, start=1): - if x > 0.0 or math.isnan(x): - yield x - elif x == 0.0: - found_zero = True - else: - raise StatisticsError('No negative inputs allowed', x) - total = fsum(map(log, count_positive(data))) - if not n: - raise StatisticsError('Must have a non-empty dataset') - if math.isnan(total): - return math.nan - if found_zero: - return math.nan if total == math.inf else 0.0 - return exp(total / n) - - -def harmonic_mean(data, weights=None): - """Return the harmonic mean of data. - - The harmonic mean is the reciprocal of the arithmetic mean of the - reciprocals of the data. It can be used for averaging ratios or - rates, for example speeds. - - Suppose a car travels 40 km/hr for 5 km and then speeds-up to - 60 km/hr for another 5 km. What is the average speed? - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 - - If ``data`` is empty, or any element is less than zero, - ``harmonic_mean`` will raise ``StatisticsError``. """ if iter(data) is data: data = list(data) + errmsg = 'harmonic mean does not support negative values' + n = len(data) if n < 1: raise StatisticsError('harmonic_mean requires at least one data point') @@ -596,6 +301,7 @@ def harmonic_mean(data, weights=None): return x else: raise TypeError('unsupported type') + if weights is None: weights = repeat(1, n) sum_weights = n @@ -605,16 +311,19 @@ def harmonic_mean(data, weights=None): if len(weights) != n: raise StatisticsError('Number of weights does not match data size') _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg)) + try: data = _fail_neg(data, errmsg) T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data)) except ZeroDivisionError: return 0 + if total <= 0: raise StatisticsError('Weighted sum must be positive') + return _convert(sum_weights / total, T) -# FIXME: investigate ways to calculate medians without sorting? Quickselect? + def median(data): """Return the median (middle value) of numeric data. @@ -651,6 +360,9 @@ def median_low(data): 3 """ + # Potentially the sorting step could be replaced with a quickselect. + # However, it would require an excellent implementation to beat our + # highly optimized builtin sort. data = sorted(data) n = len(data) if n == 0: @@ -794,6 +506,7 @@ def multimode(data): ['b', 'd', 'f'] >>> multimode('') [] + """ counts = Counter(iter(data)) if not counts: @@ -802,120 +515,34 @@ def multimode(data): return [value for value, count in counts.items() if count == maxcount] -# Notes on methods for computing quantiles -# ---------------------------------------- -# -# There is no one perfect way to compute quantiles. Here we offer -# two methods that serve common needs. Most other packages -# surveyed offered at least one or both of these two, making them -# "standard" in the sense of "widely-adopted and reproducible". -# They are also easy to explain, easy to compute manually, and have -# straight-forward interpretations that aren't surprising. +## Measures of spread ###################################################### -# The default method is known as "R6", "PERCENTILE.EXC", or "expected -# value of rank order statistics". The alternative method is known as -# "R7", "PERCENTILE.INC", or "mode of rank order statistics". +def variance(data, xbar=None): + """Return the sample variance of data. -# For sample data where there is a positive probability for values -# beyond the range of the data, the R6 exclusive method is a -# reasonable choice. Consider a random sample of nine values from a -# population with a uniform distribution from 0.0 to 1.0. The -# distribution of the third ranked sample point is described by -# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and -# mean=0.300. Only the latter (which corresponds with R6) gives the -# desired cut point with 30% of the population falling below that -# value, making it comparable to a result from an inv_cdf() function. -# The R6 exclusive method is also idempotent. + data should be an iterable of Real-valued numbers, with at least two + values. The optional argument xbar, if given, should be the mean of + the data. If it is missing or None, the mean is automatically calculated. -# For describing population data where the end points are known to -# be included in the data, the R7 inclusive method is a reasonable -# choice. Instead of the mean, it uses the mode of the beta -# distribution for the interior points. Per Hyndman & Fan, "One nice -# property is that the vertices of Q7(p) divide the range into n - 1 -# intervals, and exactly 100p% of the intervals lie to the left of -# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + Use this function when your data is a sample from a population. To + calculate the variance from the entire population, see ``pvariance``. -# If needed, other methods could be added. However, for now, the -# position is that fewer options make for easier choices and that -# external packages can be used for anything more advanced. + Examples: -def quantiles(data, *, n=4, method='exclusive'): - """Divide *data* into *n* continuous intervals with equal probability. + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 - Returns a list of (n - 1) cut points separating the intervals. + If you have already calculated the mean of your data, you can pass it as + the optional second argument ``xbar`` to avoid recalculating it: - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate *data* in to 100 equal sized groups. + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 - The *data* can be any iterable containing sample. - The cut points are linearly interpolated between data points. - - If *method* is set to *inclusive*, *data* is treated as population - data. The minimum value is treated as the 0th percentile and the - maximum value is treated as the 100th percentile. - """ - if n < 1: - raise StatisticsError('n must be at least 1') - data = sorted(data) - ld = len(data) - if ld < 2: - if ld == 1: - return data * (n - 1) - raise StatisticsError('must have at least one data point') - if method == 'inclusive': - m = ld - 1 - result = [] - for i in range(1, n): - j, delta = divmod(i * m, n) - interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n - result.append(interpolated) - return result - if method == 'exclusive': - m = ld + 1 - result = [] - for i in range(1, n): - j = i * m // n # rescale i to m/n - j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 - delta = i*m - j*n # exact integer math - interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n - result.append(interpolated) - return result - raise ValueError(f'Unknown method: {method!r}') - - -# === Measures of spread === - -# See http://mathworld.wolfram.com/Variance.html -# http://mathworld.wolfram.com/SampleVariance.html - - -def variance(data, xbar=None): - """Return the sample variance of data. - - data should be an iterable of Real-valued numbers, with at least two - values. The optional argument xbar, if given, should be the mean of - the data. If it is missing or None, the mean is automatically calculated. - - Use this function when your data is a sample from a population. To - calculate the variance from the entire population, see ``pvariance``. - - Examples: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as - the optional second argument ``xbar`` to avoid recalculating it: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not check that ``xbar`` is actually the mean of - ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or - impossible results. + This function does not check that ``xbar`` is actually the mean of + ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or + impossible results. Decimals and Fractions are supported: @@ -928,6 +555,8 @@ def variance(data, xbar=None): Fraction(67, 108) """ + # http://mathworld.wolfram.com/SampleVariance.html + T, ss, c, n = _ss(data, xbar) if n < 2: raise StatisticsError('variance requires at least two data points') @@ -969,6 +598,8 @@ def pvariance(data, mu=None): Fraction(13, 72) """ + # http://mathworld.wolfram.com/Variance.html + T, ss, c, n = _ss(data, mu) if n < 1: raise StatisticsError('pvariance requires at least one data point') @@ -1011,46 +642,7 @@ def pstdev(data, mu=None): return _float_sqrt_of_frac(mss.numerator, mss.denominator) -def _mean_stdev(data): - """In one pass, compute the mean and sample standard deviation as floats.""" - T, ss, xbar, n = _ss(data) - if n < 2: - raise StatisticsError('stdev requires at least two data points') - mss = ss / (n - 1) - try: - return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) - except AttributeError: - # Handle Nans and Infs gracefully - return float(xbar), float(xbar) / float(ss) - -def _sqrtprod(x: float, y: float) -> float: - "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." - h = sqrt(x * y) - if not isfinite(h): - if isinf(h) and not isinf(x) and not isinf(y): - # Finite inputs overflowed, so scale down, and recompute. - scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) - return _sqrtprod(scale * x, scale * y) / scale - return h - if not h: - if x and y: - # Non-zero inputs underflowed, so scale up, and recompute. - # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) - scale = 2.0 ** 537 - return _sqrtprod(scale * x, scale * y) / scale - return h - # Improve accuracy with a differential correction. - # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 - d = sumprod((x, h), (y, -h)) - return h + d / (2.0 * h) - - -# === Statistics for relations between two inputs === - -# See https://en.wikipedia.org/wiki/Covariance -# https://en.wikipedia.org/wiki/Pearson_correlation_coefficient -# https://en.wikipedia.org/wiki/Simple_linear_regression - +## Statistics for relations between two inputs ############################# def covariance(x, y, /): """Covariance @@ -1069,6 +661,7 @@ def covariance(x, y, /): -7.5 """ + # https://en.wikipedia.org/wiki/Covariance n = len(x) if len(y) != n: raise StatisticsError('covariance requires that both inputs have same number of data points') @@ -1102,7 +695,10 @@ def correlation(x, y, /, *, method='linear'): Spearman's rank correlation coefficient is appropriate for ordinal data or for continuous data that doesn't meet the linear proportion requirement for Pearson's correlation coefficient. + """ + # https://en.wikipedia.org/wiki/Pearson_correlation_coefficient + # https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') @@ -1110,18 +706,22 @@ def correlation(x, y, /, *, method='linear'): raise StatisticsError('correlation requires at least two data points') if method not in {'linear', 'ranked'}: raise ValueError(f'Unknown method: {method!r}') + if method == 'ranked': start = (n - 1) / -2 # Center rankings around zero x = _rank(x, start=start) y = _rank(y, start=start) + else: xbar = fsum(x) / n ybar = fsum(y) / n x = [xi - xbar for xi in x] y = [yi - ybar for yi in y] + sxy = sumprod(x, y) sxx = sumprod(x, x) syy = sumprod(y, y) + try: return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: @@ -1169,27 +769,1019 @@ def linear_regression(x, y, /, *, proportional=False): LinearRegression(slope=2.90475..., intercept=0.0) """ + # https://en.wikipedia.org/wiki/Simple_linear_regression n = len(x) if len(y) != n: raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') + if not proportional: xbar = fsum(x) / n ybar = fsum(y) / n x = [xi - xbar for xi in x] # List because used three times below y = (yi - ybar for yi in y) # Generator because only used once below + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float sxx = sumprod(x, x) + try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: raise StatisticsError('x is constant') + intercept = 0.0 if proportional else ybar - slope * xbar return LinearRegression(slope=slope, intercept=intercept) -## Normal Distribution ##################################################### +## Kernel Density Estimation ############################################### + +_kernel_specs = {} + +def register(*kernels): + "Load the kernel's pdf, cdf, invcdf, and support into _kernel_specs." + def deco(builder): + spec = dict(zip(('pdf', 'cdf', 'invcdf', 'support'), builder())) + for kernel in kernels: + _kernel_specs[kernel] = spec + return builder + return deco + +@register('normal', 'gauss') +def normal_kernel(): + sqrt2pi = sqrt(2 * pi) + sqrt2 = sqrt(2) + pdf = lambda t: exp(-1/2 * t * t) / sqrt2pi + cdf = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) + invcdf = lambda t: _normal_dist_inv_cdf(t, 0.0, 1.0) + support = None + return pdf, cdf, invcdf, support + +@register('logistic') +def logistic_kernel(): + # 1.0 / (exp(t) + 2.0 + exp(-t)) + pdf = lambda t: 1/2 / (1.0 + cosh(t)) + cdf = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) + invcdf = lambda p: log(p / (1.0 - p)) + support = None + return pdf, cdf, invcdf, support + +@register('sigmoid') +def sigmoid_kernel(): + # (2/pi) / (exp(t) + exp(-t)) + c1 = 1 / pi + c2 = 2 / pi + c3 = pi / 2 + pdf = lambda t: c1 / cosh(t) + cdf = lambda t: c2 * atan(exp(t)) + invcdf = lambda p: log(tan(p * c3)) + support = None + return pdf, cdf, invcdf, support + +@register('rectangular', 'uniform') +def rectangular_kernel(): + pdf = lambda t: 1/2 + cdf = lambda t: 1/2 * t + 1/2 + invcdf = lambda p: 2.0 * p - 1.0 + support = 1.0 + return pdf, cdf, invcdf, support + +@register('triangular') +def triangular_kernel(): + pdf = lambda t: 1.0 - abs(t) + cdf = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 + invcdf = lambda p: sqrt(2.0*p) - 1.0 if p < 1/2 else 1.0 - sqrt(2.0 - 2.0*p) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('parabolic', 'epanechnikov') +def parabolic_kernel(): + pdf = lambda t: 3/4 * (1.0 - t * t) + cdf = lambda t: sumprod((-1/4, 3/4, 1/2), (t**3, t, 1.0)) + invcdf = lambda p: 2.0 * cos((acos(2.0*p - 1.0) + pi) / 3.0) + support = 1.0 + return pdf, cdf, invcdf, support + +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv + +def _quartic_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + if p < 0.0106: + return ((2.0 * p) ** 0.3838 - 1.0) * sign + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign + +@register('quartic', 'biweight') +def quartic_kernel(): + pdf = lambda t: 15/16 * (1.0 - t * t) ** 2 + cdf = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), + (t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_quartic_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +def _triweight_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + if 0.00001 < p < 0.499: + x -= 0.033 * sin(1.07 * tau * (p - 0.035)) + return x * sign + +@register('triweight') +def triweight_kernel(): + pdf = lambda t: 35/32 * (1.0 - t * t) ** 3 + cdf = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), + (t**7, t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_triweight_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('cosine') +def cosine_kernel(): + c1 = pi / 4 + c2 = pi / 2 + pdf = lambda t: c1 * cos(c2 * t) + cdf = lambda t: 1/2 * sin(c2 * t) + 1/2 + invcdf = lambda p: 2.0 * asin(2.0 * p - 1.0) / pi + support = 1.0 + return pdf, cdf, invcdf, support + +del register, normal_kernel, logistic_kernel, sigmoid_kernel +del rectangular_kernel, triangular_kernel, parabolic_kernel +del quartic_kernel, triweight_kernel, cosine_kernel + + +def kde(data, h, kernel='normal', *, cumulative=False): + """Kernel Density Estimation: Create a continuous probability density + function or cumulative distribution function from discrete samples. + + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point: + + normal (gauss) + logistic + sigmoid + + Kernels that only give weight to sample points within + the bandwidth: + + rectangular (uniform) + triangular + parabolic (epanechnikov) + quartic (biweight) + triweight + cosine + + If *cumulative* is true, will return a cumulative distribution function. + + A StatisticsError will be raised if the data sequence is empty. + + Example + ------- + + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + + Compute the area under the curve: + + >>> area = sum(f_hat(x) for x in range(-20, 20)) + >>> round(area, 4) + 1.0 + + Plot the estimated probability density function at + evenly spaced points from -6 to 10: + + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x + + Estimate P(4.5 < X <= 7.5), the probability that a new sample value + will be between 4.5 and 7.5: + + >>> cdf = kde(sample, h=1.5, cumulative=True) + >>> round(cdf(7.5) - cdf(4.5), 2) + 0.22 + + References + ---------- + + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf + + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use + + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ + + Kernel estimation of cumulative distribution function of a random variable with bounded support + https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf + + """ + + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + K = kernel_spec['pdf'] + W = kernel_spec['cdf'] + support = kernel_spec['support'] + + if support is None: + + def pdf(x): + return sum(K((x - x_i) / h) for x_i in data) / (len(data) * h) + + def cdf(x): + return sum(W((x - x_i) / h) for x_i in data) / len(data) + + else: + + sample = sorted(data) + bandwidth = h * support + + def pdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) + + def cdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum((W((x - x_i) / h) for x_i in supported), i) / n + + if cumulative: + cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' + return cdf + + else: + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' + return pdf + + +def kde_random(data, h, kernel='normal', *, seed=None): + """Return a function that makes a random selection from the estimated + probability density function created by kde(data, h, kernel). + + Providing a *seed* allows reproducible selections within a single + thread. The seed may be an integer, float, str, or bytes. + + A StatisticsError will be raised if the *data* sequence is empty. + + Example: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + """ + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + invcdf = kernel_spec['invcdf'] + + prng = _random.Random(seed) + random = prng.random + choice = prng.choice + + def rand(): + return choice(data) + h * invcdf(random()) + + rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' + + return rand + + +## Quantiles ############################################################### + +# There is no one perfect way to compute quantiles. Here we offer +# two methods that serve common needs. Most other packages +# surveyed offered at least one or both of these two, making them +# "standard" in the sense of "widely-adopted and reproducible". +# They are also easy to explain, easy to compute manually, and have +# straight-forward interpretations that aren't surprising. + +# The default method is known as "R6", "PERCENTILE.EXC", or "expected +# value of rank order statistics". The alternative method is known as +# "R7", "PERCENTILE.INC", or "mode of rank order statistics". + +# For sample data where there is a positive probability for values +# beyond the range of the data, the R6 exclusive method is a +# reasonable choice. Consider a random sample of nine values from a +# population with a uniform distribution from 0.0 to 1.0. The +# distribution of the third ranked sample point is described by +# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and +# mean=0.300. Only the latter (which corresponds with R6) gives the +# desired cut point with 30% of the population falling below that +# value, making it comparable to a result from an inv_cdf() function. +# The R6 exclusive method is also idempotent. + +# For describing population data where the end points are known to +# be included in the data, the R7 inclusive method is a reasonable +# choice. Instead of the mean, it uses the mode of the beta +# distribution for the interior points. Per Hyndman & Fan, "One nice +# property is that the vertices of Q7(p) divide the range into n - 1 +# intervals, and exactly 100p% of the intervals lie to the left of +# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + +# If needed, other methods could be added. However, for now, the +# position is that fewer options make for easier choices and that +# external packages can be used for anything more advanced. + +def quantiles(data, *, n=4, method='exclusive'): + """Divide *data* into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate *data* in to 100 equal sized groups. + + The *data* can be any iterable containing sample. + The cut points are linearly interpolated between data points. + + If *method* is set to *inclusive*, *data* is treated as population + data. The minimum value is treated as the 0th percentile and the + maximum value is treated as the 100th percentile. + + """ + if n < 1: + raise StatisticsError('n must be at least 1') + + data = sorted(data) + + ld = len(data) + if ld < 2: + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') + + if method == 'inclusive': + m = ld - 1 + result = [] + for i in range(1, n): + j, delta = divmod(i * m, n) + interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n + result.append(interpolated) + return result + + if method == 'exclusive': + m = ld + 1 + result = [] + for i in range(1, n): + j = i * m // n # rescale i to m/n + j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 + delta = i*m - j*n # exact integer math + interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n + result.append(interpolated) + return result + + raise ValueError(f'Unknown method: {method!r}') + + +## Normal Distribution ##################################################### + +class NormalDist: + "Normal distribution of a random variable" + # https://en.wikipedia.org/wiki/Normal_distribution + # https://en.wikipedia.org/wiki/Variance#Properties + + __slots__ = { + '_mu': 'Arithmetic mean of a normal distribution', + '_sigma': 'Standard deviation of a normal distribution', + } + + def __init__(self, mu=0.0, sigma=1.0): + "NormalDist where mu is the mean and sigma is the standard deviation." + if sigma < 0.0: + raise StatisticsError('sigma must be non-negative') + self._mu = float(mu) + self._sigma = float(sigma) + + @classmethod + def from_samples(cls, data): + "Make a normal distribution instance from sample data." + return cls(*_mean_stdev(data)) + + def samples(self, n, *, seed=None): + "Generate *n* samples for a given mean and standard deviation." + rnd = random.random if seed is None else random.Random(seed).random + inv_cdf = _normal_dist_inv_cdf + mu = self._mu + sigma = self._sigma + return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] + + def pdf(self, x): + "Probability density function. P(x <= X < x+dx) / dx" + variance = self._sigma * self._sigma + if not variance: + raise StatisticsError('pdf() not defined when sigma is zero') + diff = x - self._mu + return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) + + def cdf(self, x): + "Cumulative distribution function. P(X <= x)" + if not self._sigma: + raise StatisticsError('cdf() not defined when sigma is zero') + return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2))) + + def inv_cdf(self, p): + """Inverse cumulative distribution function. x : P(X <= x) = p + + Finds the value of the random variable such that the probability of + the variable being less than or equal to that value equals the given + probability. + + This function is also called the percent point function or quantile + function. + """ + if p <= 0.0 or p >= 1.0: + raise StatisticsError('p must be in the range 0.0 < p < 1.0') + return _normal_dist_inv_cdf(p, self._mu, self._sigma) + + def quantiles(self, n=4): + """Divide into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution in to 100 equal sized groups. + """ + return [self.inv_cdf(i / n) for i in range(1, n)] + + def overlap(self, other): + """Compute the overlapping coefficient (OVL) between two normal distributions. + + Measures the agreement between two normal probability distributions. + Returns a value between 0.0 and 1.0 giving the overlapping area in + the two underlying probability density functions. + + >>> N1 = NormalDist(2.4, 1.6) + >>> N2 = NormalDist(3.2, 2.0) + >>> N1.overlap(N2) + 0.8035050657330205 + """ + # See: "The overlapping coefficient as a measure of agreement between + # probability distributions and point estimation of the overlap of two + # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr + # http://dx.doi.org/10.1080/03610928908830127 + if not isinstance(other, NormalDist): + raise TypeError('Expected another NormalDist instance') + X, Y = self, other + if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity + X, Y = Y, X + X_var, Y_var = X.variance, Y.variance + if not X_var or not Y_var: + raise StatisticsError('overlap() not defined when sigma is zero') + dv = Y_var - X_var + dm = fabs(Y._mu - X._mu) + if not dv: + return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2)) + a = X._mu * Y_var - Y._mu * X_var + b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) + x1 = (a + b) / dv + x2 = (a - b) / dv + return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) + + def zscore(self, x): + """Compute the Standard Score. (x - mean) / stdev + + Describes *x* in terms of the number of standard deviations + above or below the mean of the normal distribution. + """ + # https://www.statisticshowto.com/probability-and-statistics/z-score/ + if not self._sigma: + raise StatisticsError('zscore() not defined when sigma is zero') + return (x - self._mu) / self._sigma + + @property + def mean(self): + "Arithmetic mean of the normal distribution." + return self._mu + + @property + def median(self): + "Return the median of the normal distribution" + return self._mu + + @property + def mode(self): + """Return the mode of the normal distribution + + The mode is the value x where which the probability density + function (pdf) takes its maximum value. + """ + return self._mu + + @property + def stdev(self): + "Standard deviation of the normal distribution." + return self._sigma + + @property + def variance(self): + "Square of the standard deviation." + return self._sigma * self._sigma + + def __add__(x1, x2): + """Add a constant or another NormalDist instance. + + If *other* is a constant, translate mu by the constant, + leaving sigma unchanged. + + If *other* is a NormalDist, add both the means and the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu + x2, x1._sigma) + + def __sub__(x1, x2): + """Subtract a constant or another NormalDist instance. + + If *other* is a constant, translate by the constant mu, + leaving sigma unchanged. + + If *other* is a NormalDist, subtract the means and add the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu - x2, x1._sigma) + + def __mul__(x1, x2): + """Multiply both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) + + def __truediv__(x1, x2): + """Divide both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) + + def __pos__(x1): + "Return a copy of the instance." + return NormalDist(x1._mu, x1._sigma) + + def __neg__(x1): + "Negates mu while keeping sigma the same." + return NormalDist(-x1._mu, x1._sigma) + + __radd__ = __add__ + + def __rsub__(x1, x2): + "Subtract a NormalDist from a constant or another NormalDist." + return -(x1 - x2) + + __rmul__ = __mul__ + + def __eq__(x1, x2): + "Two NormalDist objects are equal if their mu and sigma are both equal." + if not isinstance(x2, NormalDist): + return NotImplemented + return x1._mu == x2._mu and x1._sigma == x2._sigma + + def __hash__(self): + "NormalDist objects hash equal if their mu and sigma are both equal." + return hash((self._mu, self._sigma)) + + def __repr__(self): + return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + + def __getstate__(self): + return self._mu, self._sigma + + def __setstate__(self, state): + self._mu, self._sigma = state + + +## Private utilities ####################################################### + +def _sum(data): + """_sum(data) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. + + Examples + -------- + + >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) + (, Fraction(19, 2), 5) + + Some sources of round-off error will be avoided: + + # Built-in sum returns zero. + >>> _sum([1e50, 1, -1e50] * 1000) + (, Fraction(1000, 1), 3000) + + Fractions and Decimals are also supported: + + >>> from fractions import Fraction as F + >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) + (, Fraction(63, 20), 4) + + >>> from decimal import Decimal as D + >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] + >>> _sum(data) + (, Fraction(6963, 10000), 4) + + Mixed types are currently treated as an error, except that int is + allowed. + + """ + count = 0 + types = set() + types_add = types.add + partials = {} + partials_get = partials.get + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n + + if None in partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + total = sum(Fraction(n, d) for d, n in partials.items()) + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, total, count) + + +def _ss(data, c=None): + """Return the exact mean and sum of square deviations of sequence data. + + Calculations are done in a single pass, allowing the input to be an iterator. + + If given *c* is used the mean; otherwise, it is calculated from the data. + Use the *c* argument with care, as it can lead to garbage results. + + """ + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + + count = 0 + types = set() + types_add = types.add + sx_partials = defaultdict(int) + sxx_partials = defaultdict(int) + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + sx_partials[d] += n + sxx_partials[d] += n * n + + if not count: + ssd = c = Fraction(0) + + elif None in sx_partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + ssd = c = sx_partials[None] + assert not _isfinite(ssd) + + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) + # This formula has poor numeric properties for floats, + # but with fractions it is exact. + ssd = (count * sxx - sx * sx) / count + c = sx / count + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, ssd, c, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) + + +def _exact_ratio(x): + """Return Real number x to exact (numerator, denominator) pair. + + >>> _exact_ratio(0.25) + (1, 4) + + x is expected to be an int, Fraction, Decimal or float. + + """ + try: + return x.as_integer_ratio() + except AttributeError: + pass + except (OverflowError, ValueError): + # float NAN or INF. + assert not _isfinite(x) + return (x, None) + + try: + # x may be an Integral ABC. + return (x.numerator, x.denominator) + except AttributeError: + msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" + raise TypeError(msg) + + +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + + if issubclass(T, int) and value.denominator != 1: + T = float + + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator) / T(value.denominator) + else: + raise + + +def _fail_neg(values, errmsg='negative value'): + """Iterate over values, failing if any are less than zero.""" + for x in values: + if x < 0: + raise StatisticsError(errmsg) + yield x + + +def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: + """Rank order a dataset. The lowest value has rank 1. + + Ties are averaged so that equal values receive the same rank: + + >>> data = [31, 56, 31, 25, 75, 18] + >>> _rank(data) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + The operation is idempotent: + + >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + It is possible to rank the data in reverse order so that the + highest value has rank 1. Also, a key-function can extract + the field to be ranked: + + >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] + >>> _rank(goals, key=itemgetter(1), reverse=True) + [2.0, 1.0, 3.0] + + Ranks are conventionally numbered starting from one; however, + setting *start* to zero allows the ranks to be used as array indices: + + >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] + >>> scores = [8.1, 7.3, 9.4, 8.3] + >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] + ['Bronze', 'Certificate', 'Gold', 'Silver'] + + """ + # If this function becomes public at some point, more thought + # needs to be given to the signature. A list of ints is + # plausible when ties is "min" or "max". When ties is "average", + # either list[float] or list[Fraction] is plausible. + + # Default handling of ties matches scipy.stats.mstats.spearmanr. + if ties != 'average': + raise ValueError(f'Unknown tie resolution method: {ties!r}') + if key is not None: + data = map(key, data) + val_pos = sorted(zip(data, count()), reverse=reverse) + i = start - 1 + result = [0] * len(val_pos) + for _, g in groupby(val_pos, key=itemgetter(0)): + group = list(g) + size = len(group) + rank = i + (size + 1) / 2 + for value, orig_pos in group: + result[orig_pos] = rank + i += size + return result + + +def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: + """Square root of n/m, rounded to the nearest integer using round-to-odd.""" + # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf + a = math.isqrt(n // m) + return a | (a*a*m != n) + + +# For 53 bit precision floats, the bit width used in +# _float_sqrt_of_frac() is 109. +_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 + + +def _float_sqrt_of_frac(n: int, m: int) -> float: + """Square root of n/m as a float, correctly rounded.""" + # See principle and proof sketch at: https://bugs.python.org/msg407078 + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator # Convert to float + + +def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: + """Square root of n/m as a Decimal, correctly rounded.""" + # Premise: For decimal, computing (n/m).sqrt() can be off + # by 1 ulp from the correctly rounded result. + # Method: Check the result, moving up or down a step if needed. + if n <= 0: + if not n: + return Decimal('0.0') + n, m = -n, -m + + root = (Decimal(n) / Decimal(m)).sqrt() + nr, dr = root.as_integer_ratio() + + plus = root.next_plus() + np, dp = plus.as_integer_ratio() + # test: n / m > ((root + plus) / 2) ** 2 + if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: + return plus + + minus = root.next_minus() + nm, dm = minus.as_integer_ratio() + # test: n / m < ((root + minus) / 2) ** 2 + if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: + return minus + + return root + + +def _mean_stdev(data): + """In one pass, compute the mean and sample standard deviation as floats.""" + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) + try: + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) + except AttributeError: + # Handle Nans and Infs gracefully + return float(xbar), float(xbar) / float(ss) + + +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." + + h = sqrt(x * y) + + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h + + if not h: + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) def _normal_dist_inv_cdf(p, mu, sigma): @@ -1199,6 +1791,7 @@ def _normal_dist_inv_cdf(p, mu, sigma): # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. q = p - 0.5 + if fabs(q) <= 0.425: r = 0.180625 - q * q # Hash sum: 55.88319_28806_14901_4439 @@ -1220,6 +1813,7 @@ def _normal_dist_inv_cdf(p, mu, sigma): 1.0) x = num / den return mu + (x * sigma) + r = p if q <= 0.0 else 1.0 - p r = sqrt(-log(r)) if r <= 5.0: @@ -1260,9 +1854,11 @@ def _normal_dist_inv_cdf(p, mu, sigma): 1.36929_88092_27358_05310e-1) * r + 5.99832_20655_58879_37690e-1) * r + 1.0) + x = num / den if q < 0.0: x = -x + return mu + (x * sigma) @@ -1271,226 +1867,3 @@ def _normal_dist_inv_cdf(p, mu, sigma): from _statistics import _normal_dist_inv_cdf except ImportError: pass - - -class NormalDist: - "Normal distribution of a random variable" - # https://en.wikipedia.org/wiki/Normal_distribution - # https://en.wikipedia.org/wiki/Variance#Properties - - __slots__ = { - '_mu': 'Arithmetic mean of a normal distribution', - '_sigma': 'Standard deviation of a normal distribution', - } - - def __init__(self, mu=0.0, sigma=1.0): - "NormalDist where mu is the mean and sigma is the standard deviation." - if sigma < 0.0: - raise StatisticsError('sigma must be non-negative') - self._mu = float(mu) - self._sigma = float(sigma) - - @classmethod - def from_samples(cls, data): - "Make a normal distribution instance from sample data." - return cls(*_mean_stdev(data)) - - def samples(self, n, *, seed=None): - "Generate *n* samples for a given mean and standard deviation." - rnd = random.random if seed is None else random.Random(seed).random - inv_cdf = _normal_dist_inv_cdf - mu = self._mu - sigma = self._sigma - return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] - - def pdf(self, x): - "Probability density function. P(x <= X < x+dx) / dx" - variance = self._sigma * self._sigma - if not variance: - raise StatisticsError('pdf() not defined when sigma is zero') - diff = x - self._mu - return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) - - def cdf(self, x): - "Cumulative distribution function. P(X <= x)" - if not self._sigma: - raise StatisticsError('cdf() not defined when sigma is zero') - return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2))) - - def inv_cdf(self, p): - """Inverse cumulative distribution function. x : P(X <= x) = p - - Finds the value of the random variable such that the probability of - the variable being less than or equal to that value equals the given - probability. - - This function is also called the percent point function or quantile - function. - """ - if p <= 0.0 or p >= 1.0: - raise StatisticsError('p must be in the range 0.0 < p < 1.0') - return _normal_dist_inv_cdf(p, self._mu, self._sigma) - - def quantiles(self, n=4): - """Divide into *n* continuous intervals with equal probability. - - Returns a list of (n - 1) cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate the normal distribution in to 100 equal sized groups. - """ - return [self.inv_cdf(i / n) for i in range(1, n)] - - def overlap(self, other): - """Compute the overlapping coefficient (OVL) between two normal distributions. - - Measures the agreement between two normal probability distributions. - Returns a value between 0.0 and 1.0 giving the overlapping area in - the two underlying probability density functions. - - >>> N1 = NormalDist(2.4, 1.6) - >>> N2 = NormalDist(3.2, 2.0) - >>> N1.overlap(N2) - 0.8035050657330205 - """ - # See: "The overlapping coefficient as a measure of agreement between - # probability distributions and point estimation of the overlap of two - # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr - # http://dx.doi.org/10.1080/03610928908830127 - if not isinstance(other, NormalDist): - raise TypeError('Expected another NormalDist instance') - X, Y = self, other - if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity - X, Y = Y, X - X_var, Y_var = X.variance, Y.variance - if not X_var or not Y_var: - raise StatisticsError('overlap() not defined when sigma is zero') - dv = Y_var - X_var - dm = fabs(Y._mu - X._mu) - if not dv: - return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2)) - a = X._mu * Y_var - Y._mu * X_var - b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) - x1 = (a + b) / dv - x2 = (a - b) / dv - return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) - - def zscore(self, x): - """Compute the Standard Score. (x - mean) / stdev - - Describes *x* in terms of the number of standard deviations - above or below the mean of the normal distribution. - """ - # https://www.statisticshowto.com/probability-and-statistics/z-score/ - if not self._sigma: - raise StatisticsError('zscore() not defined when sigma is zero') - return (x - self._mu) / self._sigma - - @property - def mean(self): - "Arithmetic mean of the normal distribution." - return self._mu - - @property - def median(self): - "Return the median of the normal distribution" - return self._mu - - @property - def mode(self): - """Return the mode of the normal distribution - - The mode is the value x where which the probability density - function (pdf) takes its maximum value. - """ - return self._mu - - @property - def stdev(self): - "Standard deviation of the normal distribution." - return self._sigma - - @property - def variance(self): - "Square of the standard deviation." - return self._sigma * self._sigma - - def __add__(x1, x2): - """Add a constant or another NormalDist instance. - - If *other* is a constant, translate mu by the constant, - leaving sigma unchanged. - - If *other* is a NormalDist, add both the means and the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu + x2, x1._sigma) - - def __sub__(x1, x2): - """Subtract a constant or another NormalDist instance. - - If *other* is a constant, translate by the constant mu, - leaving sigma unchanged. - - If *other* is a NormalDist, subtract the means and add the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu - x2, x1._sigma) - - def __mul__(x1, x2): - """Multiply both mu and sigma by a constant. - - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) - - def __truediv__(x1, x2): - """Divide both mu and sigma by a constant. - - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) - - def __pos__(x1): - "Return a copy of the instance." - return NormalDist(x1._mu, x1._sigma) - - def __neg__(x1): - "Negates mu while keeping sigma the same." - return NormalDist(-x1._mu, x1._sigma) - - __radd__ = __add__ - - def __rsub__(x1, x2): - "Subtract a NormalDist from a constant or another NormalDist." - return -(x1 - x2) - - __rmul__ = __mul__ - - def __eq__(x1, x2): - "Two NormalDist objects are equal if their mu and sigma are both equal." - if not isinstance(x2, NormalDist): - return NotImplemented - return x1._mu == x2._mu and x1._sigma == x2._sigma - - def __hash__(self): - "NormalDist objects hash equal if their mu and sigma are both equal." - return hash((self._mu, self._sigma)) - - def __repr__(self): - return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' - - def __getstate__(self): - return self._mu, self._sigma - - def __setstate__(self, state): - self._mu, self._sigma = state diff --git a/Lib/struct.py b/Lib/struct.py index d6bba588636498..ff98e8c4cb3f1d 100644 --- a/Lib/struct.py +++ b/Lib/struct.py @@ -11,5 +11,5 @@ ] from _struct import * -from _struct import _clearcache -from _struct import __doc__ +from _struct import _clearcache # noqa: F401 +from _struct import __doc__ # noqa: F401 diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 20db7747d5db13..88f0230b05fbc7 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -79,10 +79,11 @@ if _mswindows: import _winapi - from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, # noqa: F401 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, + STARTF_FORCEONFEEDBACK, STARTF_FORCEOFFFEEDBACK, ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, @@ -93,6 +94,7 @@ "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", + "STARTF_FORCEONFEEDBACK", "STARTF_FORCEOFFFEEDBACK", "STARTUPINFO", "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", @@ -350,7 +352,7 @@ def _args_from_interpreter_flags(): if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'frozen_modules', 'showrefcount', 'utf8'): + 'frozen_modules', 'showrefcount', 'utf8', 'gil'): if opt in xoptions: value = xoptions[opt] if value is True: @@ -747,7 +749,6 @@ def _use_posix_spawn(): # These are primarily fail-safe knobs for negatives. A True value does not # guarantee the given libc/syscall API will be used. _USE_POSIX_SPAWN = _use_posix_spawn() -_USE_VFORK = True _HAVE_POSIX_SPAWN_CLOSEFROM = hasattr(os, 'POSIX_SPAWN_CLOSEFROM') @@ -839,6 +840,9 @@ def __init__(self, args, bufsize=-1, executable=None, if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") + if stdout is STDOUT: + raise ValueError("STDOUT can only be used for stderr") + if pipesize is None: pipesize = -1 # Restore default if not isinstance(pipesize, int): @@ -1586,6 +1590,8 @@ def _wait(self, timeout): """Internal implementation of wait() on Windows.""" if timeout is None: timeout_millis = _winapi.INFINITE + elif timeout <= 0: + timeout_millis = 0 else: timeout_millis = int(timeout * 1000) if self.returncode is None: @@ -1895,7 +1901,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, errpipe_read, errpipe_write, restore_signals, start_new_session, process_group, gid, gids, uid, umask, - preexec_fn, _USE_VFORK) + preexec_fn) self._child_created = True finally: # be sure the FD is closed no matter what diff --git a/Lib/symtable.py b/Lib/symtable.py index 17f820abd56660..7a30e1ac4ca378 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -1,13 +1,21 @@ """Interface to the compiler's internal symbol tables""" import _symtable -from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM, - DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE, - LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL) +from _symtable import ( + USE, + DEF_GLOBAL, # noqa: F401 + DEF_NONLOCAL, DEF_LOCAL, + DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS, + DEF_IMPORT, DEF_BOUND, DEF_ANNOT, + DEF_COMP_ITER, DEF_COMP_CELL, + SCOPE_OFF, SCOPE_MASK, + FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL +) import weakref +from enum import StrEnum -__all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"] +__all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] def symtable(code, filename, compile_type): """ Return the toplevel *SymbolTable* for the source code. @@ -39,6 +47,16 @@ def __call__(self, table, filename): _newSymbolTable = SymbolTableFactory() +class SymbolTableType(StrEnum): + MODULE = "module" + FUNCTION = "function" + CLASS = "class" + ANNOTATION = "annotation" + TYPE_ALIAS = "type alias" + TYPE_PARAMETERS = "type parameters" + TYPE_VARIABLE = "type variable" + + class SymbolTable: def __init__(self, raw_table, filename): @@ -62,23 +80,23 @@ def __repr__(self): def get_type(self): """Return the type of the symbol table. - The values returned are 'class', 'module', 'function', - 'annotation', 'TypeVar bound', 'type alias', and 'type parameter'. + The value returned is one of the values in + the ``SymbolTableType`` enumeration. """ if self._table.type == _symtable.TYPE_MODULE: - return "module" + return SymbolTableType.MODULE if self._table.type == _symtable.TYPE_FUNCTION: - return "function" + return SymbolTableType.FUNCTION if self._table.type == _symtable.TYPE_CLASS: - return "class" + return SymbolTableType.CLASS if self._table.type == _symtable.TYPE_ANNOTATION: - return "annotation" - if self._table.type == _symtable.TYPE_TYPE_VAR_BOUND: - return "TypeVar bound" + return SymbolTableType.ANNOTATION if self._table.type == _symtable.TYPE_TYPE_ALIAS: - return "type alias" - if self._table.type == _symtable.TYPE_TYPE_PARAM: - return "type parameter" + return SymbolTableType.TYPE_ALIAS + if self._table.type == _symtable.TYPE_TYPE_PARAMETERS: + return SymbolTableType.TYPE_PARAMETERS + if self._table.type == _symtable.TYPE_TYPE_VARIABLE: + return SymbolTableType.TYPE_VARIABLE assert False, f"unexpected type: {self._table.type}" def get_id(self): @@ -154,6 +172,10 @@ def get_children(self): for st in self._table.children] +def _get_scope(flags): # like _PyST_GetScope() + return (flags >> SCOPE_OFF) & SCOPE_MASK + + class Function(SymbolTable): # Default values for instance variables @@ -179,7 +201,7 @@ def get_locals(self): """ if self.__locals is None: locs = (LOCAL, CELL) - test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs + test = lambda x: _get_scope(x) in locs self.__locals = self.__idents_matching(test) return self.__locals @@ -188,7 +210,7 @@ def get_globals(self): """ if self.__globals is None: glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) - test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob + test = lambda x: _get_scope(x) in glob self.__globals = self.__idents_matching(test) return self.__globals @@ -203,7 +225,7 @@ def get_frees(self): """Return a tuple of free variables in the function. """ if self.__frees is None: - is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE + is_free = lambda x: _get_scope(x) == FREE self.__frees = self.__idents_matching(is_free) return self.__frees @@ -215,10 +237,45 @@ class Class(SymbolTable): def get_methods(self): """Return a tuple of methods declared in the class. """ + import warnings + typename = f'{self.__class__.__module__}.{self.__class__.__name__}' + warnings.warn(f'{typename}.get_methods() is deprecated ' + f'and will be removed in Python 3.16.', + DeprecationWarning, stacklevel=2) + if self.__methods is None: d = {} + + def is_local_symbol(ident): + flags = self._table.symbols.get(ident, 0) + return ((flags >> SCOPE_OFF) & SCOPE_MASK) == LOCAL + for st in self._table.children: - d[st.name] = 1 + # pick the function-like symbols that are local identifiers + if is_local_symbol(st.name): + match st.type: + case _symtable.TYPE_FUNCTION: + # generators are of type TYPE_FUNCTION with a ".0" + # parameter as a first parameter (which makes them + # distinguishable from a function named 'genexpr') + if st.name == 'genexpr' and '.0' in st.varnames: + continue + d[st.name] = 1 + case _symtable.TYPE_TYPE_PARAMETERS: + # Get the function-def block in the annotation + # scope 'st' with the same identifier, if any. + scope_name = st.name + for c in st.children: + if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: + # A generic generator of type TYPE_FUNCTION + # cannot be a direct child of 'st' (but it + # can be a descendant), e.g.: + # + # class A: + # type genexpr[genexpr] = (x for x in []) + assert scope_name != 'genexpr' or '.0' not in c.varnames + d[scope_name] = 1 + break self.__methods = tuple(d) return self.__methods @@ -228,7 +285,7 @@ class Symbol: def __init__(self, name, flags, namespaces=None, *, module_scope=False): self.__name = name self.__flags = flags - self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope() + self.__scope = _get_scope(flags) self.__namespaces = namespaces or () self.__module_scope = module_scope @@ -253,13 +310,18 @@ def is_referenced(self): """Return *True* if the symbol is used in its block. """ - return bool(self.__flags & _symtable.USE) + return bool(self.__flags & USE) def is_parameter(self): """Return *True* if the symbol is a parameter. """ return bool(self.__flags & DEF_PARAM) + def is_type_parameter(self): + """Return *True* if the symbol is a type parameter. + """ + return bool(self.__flags & DEF_TYPE_PARAM) + def is_global(self): """Return *True* if the symbol is global. """ @@ -292,6 +354,11 @@ def is_free(self): """ return bool(self.__scope == FREE) + def is_free_class(self): + """Return *True* if a class-scoped symbol is free from + the perspective of a method.""" + return bool(self.__flags & DEF_FREE_CLASS) + def is_imported(self): """Return *True* if the symbol is created from an import statement. @@ -302,6 +369,16 @@ def is_assigned(self): """Return *True* if a symbol is assigned to.""" return bool(self.__flags & DEF_LOCAL) + def is_comp_iter(self): + """Return *True* if the symbol is a comprehension iteration variable. + """ + return bool(self.__flags & DEF_COMP_ITER) + + def is_comp_cell(self): + """Return *True* if the symbol is a cell in an inlined comprehension. + """ + return bool(self.__flags & DEF_COMP_CELL) + def is_namespace(self): """Returns *True* if name binding introduces new namespace. diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 07ab27c7fb0c35..43f9276799b848 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -21,15 +21,16 @@ # Keys for get_config_var() that are never converted to Python integers. _ALWAYS_STR = { + 'IPHONEOS_DEPLOYMENT_TARGET', 'MACOSX_DEPLOYMENT_TARGET', } _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': @@ -57,6 +58,7 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + # Downstream distributors can overwrite the default install scheme. # This is done to support downstream modifications where distributors change # the installation layout (eg. different site-packages directory). @@ -75,10 +77,10 @@ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': @@ -114,8 +116,8 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -146,11 +148,11 @@ def joinuser(*args): 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', - 'include': '{userbase}/include/{implementation_lower}{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -290,6 +292,7 @@ def _get_preferred_schemes(): 'home': 'posix_home', 'user': 'osx_framework_user', } + return { 'prefix': 'posix_prefix', 'home': 'posix_home', @@ -337,7 +340,20 @@ def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" # _sysconfigdata is generated at build time, see _generate_posix_vars() name = _get_sysconfigdata_name() - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) + + # For cross builds, the path to the target's sysconfigdata must be specified + # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in + # sys.path can cause crashes when loaded by the host interpreter. + # Rely on truthiness as a valueless env variable is still an empty string. + # See OS X note in _generate_posix_vars re _sysconfigdata. + if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): + from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES + from importlib.util import module_from_spec + spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) + _temp = module_from_spec(spec) + spec.loader.exec_module(_temp) + else: + _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) build_time_vars = _temp.build_time_vars vars.update(build_time_vars) @@ -484,6 +500,9 @@ def _init_config_vars(): # the init-function. _CONFIG_VARS['userbase'] = _getuserbase() + # e.g., 't' for free-threaded or '' for default build + _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else '' + # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) if os.name == 'posix': @@ -598,10 +617,22 @@ def get_platform(): machine = machine.replace('/', '-') if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + if sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + else: + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" @@ -623,10 +654,15 @@ def get_platform(): if m: release = m.group() elif osname[:6] == "darwin": - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) return f"{osname}-{release}-{machine}" @@ -635,6 +671,10 @@ def get_python_version(): return _PY_VERSION_SHORT +def _get_python_version_abi(): + return _PY_VERSION_SHORT + get_config_var("abi_thread") + + def expand_makefile_vars(s, vars): """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in 'string' according to 'vars' (a dictionary mapping variable names to diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py old mode 100755 new mode 100644 index e2ac6837f157d5..c0097351b269f2 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """The Tab Nanny despises ambiguous indentation. She knows no mercy. tabnanny -- Detection of ambiguous indentation @@ -107,14 +105,14 @@ def check(file): errprint("%r: Token Error: %s" % (file, msg)) return - except SyntaxError as msg: - errprint("%r: Token Error: %s" % (file, msg)) - return - except IndentationError as msg: errprint("%r: Indentation Error: %s" % (file, msg)) return + except SyntaxError as msg: + errprint("%r: Syntax Error: %s" % (file, msg)) + return + except NannyNag as nag: badline = nag.get_lineno() line = nag.get_line() diff --git a/Lib/tarfile.py b/Lib/tarfile.py old mode 100755 new mode 100644 index 9775040cbe372c..a0fab46b24e249 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 #------------------------------------------------------------------- # tarfile.py #------------------------------------------------------------------- @@ -636,6 +635,10 @@ def __init__(self, fileobj, offset, size, name, blockinfo=None): def flush(self): pass + @property + def mode(self): + return 'rb' + def readable(self): return True @@ -842,6 +845,9 @@ def data_filter(member, dest_path): # Sentinel for replace() defaults, meaning "don't change the attribute" _KEEP = object() +# Header length is digits followed by a space. +_header_length_prefix_re = re.compile(br"([0-9]{1,20}) ") + class TarInfo(object): """Informational class which holds the details about an archive member given by a tar header block. @@ -872,7 +878,7 @@ class TarInfo(object): pax_headers = ('A dictionary containing key-value pairs of an ' 'associated pax extended header.'), sparse = 'Sparse member information.', - tarfile = None, + _tarfile = None, _sparse_structs = None, _link_target = None, ) @@ -901,6 +907,24 @@ def __init__(self, name=""): self.sparse = None # sparse member information self.pax_headers = {} # pax header information + @property + def tarfile(self): + import warnings + warnings.warn( + 'The undocumented "tarfile" attribute of TarInfo objects ' + + 'is deprecated and will be removed in Python 3.16', + DeprecationWarning, stacklevel=2) + return self._tarfile + + @tarfile.setter + def tarfile(self, tarfile): + import warnings + warnings.warn( + 'The undocumented "tarfile" attribute of TarInfo objects ' + + 'is deprecated and will be removed in Python 3.16', + DeprecationWarning, stacklevel=2) + self._tarfile = tarfile + @property def path(self): 'In pax headers, "name" is called "path".' @@ -1195,7 +1219,7 @@ def _create_pax_generic_header(cls, pax_headers, type, encoding): for keyword, value in pax_headers.items(): keyword = keyword.encode("utf-8") if binary: - # Try to restore the original byte representation of `value'. + # Try to restore the original byte representation of 'value'. # Needless to say, that the encoding must match the string. value = value.encode(encoding, "surrogateescape") else: @@ -1411,37 +1435,59 @@ def _proc_pax(self, tarfile): else: pax_headers = tarfile.pax_headers.copy() - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - # Parse pax header information. A record looks like that: # "%d %s=%s\n" % (length, keyword, value). length is the size # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") + # the newline. pos = 0 - while match := regex.match(buf, pos): - length, keyword = match.groups() - length = int(length) - if length == 0: + encoding = None + raw_headers = [] + while len(buf) > pos and buf[pos] != 0x00: + if not (match := _header_length_prefix_re.match(buf, pos)): + raise InvalidHeaderError("invalid header") + try: + length = int(match.group(1)) + except ValueError: + raise InvalidHeaderError("invalid header") + # Headers must be at least 5 bytes, shortest being '5 x=\n'. + # Value is allowed to be empty. + if length < 5: + raise InvalidHeaderError("invalid header") + if pos + length > len(buf): + raise InvalidHeaderError("invalid header") + + header_value_end_offset = match.start(1) + length - 1 # Last byte of the header + keyword_and_value = buf[match.end(1) + 1:header_value_end_offset] + raw_keyword, equals, raw_value = keyword_and_value.partition(b"=") + + # Check the framing of the header. The last character must be '\n' (0x0A) + if not raw_keyword or equals != b"=" or buf[header_value_end_offset] != 0x0A: raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] + raw_headers.append((length, raw_keyword, raw_value)) + + # Check if the pax header contains a hdrcharset field. This tells us + # the encoding of the path, linkpath, uname and gname fields. Normally, + # these fields are UTF-8 encoded but since POSIX.1-2008 tar + # implementations are allowed to store them as raw binary strings if + # the translation to UTF-8 fails. For the time being, we don't care about + # anything other than "BINARY". The only other value that is currently + # allowed by the standard is "ISO-IR 10646 2000 UTF-8" in other words UTF-8. + # Note that we only follow the initial 'hdrcharset' setting to preserve + # the initial behavior of the 'tarfile' module. + if raw_keyword == b"hdrcharset" and encoding is None: + if raw_value == b"BINARY": + encoding = tarfile.encoding + else: # This branch ensures only the first 'hdrcharset' header is used. + encoding = "utf-8" + + pos += length + + # If no explicit hdrcharset is set, we use UTF-8 as a default. + if encoding is None: + encoding = "utf-8" + # After parsing the raw headers we can decode them to text. + for length, raw_keyword, raw_value in raw_headers: # Normally, we could just use "utf-8" as the encoding and "strict" # as the error handler, but we better not take the risk. For # example, GNU tar <= 1.23 is known to store filenames it cannot @@ -1449,17 +1495,16 @@ def _proc_pax(self, tarfile): # hdrcharset=BINARY header). # We first try the strict standard encoding, and if that fails we # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", + keyword = self._decode_pax_field(raw_keyword, "utf-8", "utf-8", tarfile.errors) if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, + value = self._decode_pax_field(raw_value, encoding, tarfile.encoding, tarfile.errors) else: - value = self._decode_pax_field(value, "utf-8", "utf-8", + value = self._decode_pax_field(raw_value, "utf-8", "utf-8", tarfile.errors) pax_headers[keyword] = value - pos += length # Fetch the next header. try: @@ -1474,7 +1519,7 @@ def _proc_pax(self, tarfile): elif "GNU.sparse.size" in pax_headers: # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) + self._proc_gnusparse_00(next, raw_headers) elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": # GNU extended sparse format version 1.0. @@ -1496,15 +1541,24 @@ def _proc_pax(self, tarfile): return next - def _proc_gnusparse_00(self, next, pax_headers, buf): + def _proc_gnusparse_00(self, next, raw_headers): """Process a GNU tar extended sparse header, version 0.0. """ offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) + for _, keyword, value in raw_headers: + if keyword == b"GNU.sparse.offset": + try: + offsets.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + + elif keyword == b"GNU.sparse.numbytes": + try: + numbytes.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + next.sparse = list(zip(offsets, numbytes)) def _proc_gnusparse_01(self, next, pax_headers): @@ -1641,13 +1695,13 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, errorlevel=None, copybufsize=None, stream=False): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to + """Open an (uncompressed) tar archive 'name'. 'mode' is either 'r' to read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' + file or 'w' to create a new file overwriting an existing one. 'mode' defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. + If 'fileobj' is given, it is used for reading or writing data. If it + can be determined, 'mode' is overridden by 'fileobj's mode. + 'fileobj' is not closed, when TarFile is closed. """ modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} if mode not in modes: @@ -1706,6 +1760,8 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, # current position in the archive file self.inodes = {} # dictionary caching the inodes of # archive members already added + self._unames = {} # Cached mappings of uid -> uname + self._gnames = {} # Cached mappings of gid -> gname try: if self.mode == "r": @@ -1976,7 +2032,7 @@ def close(self): self.fileobj.close() def getmember(self, name): - """Return a TarInfo object for member `name'. If `name' can not be + """Return a TarInfo object for member 'name'. If 'name' can not be found in the archive, KeyError is raised. If a member occurs more than once in the archive, its last occurrence is assumed to be the most up-to-date version. @@ -2004,9 +2060,9 @@ def getnames(self): def gettarinfo(self, name=None, arcname=None, fileobj=None): """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by `name', or - specified as a file object `fileobj' with a file descriptor. If - given, `arcname' specifies an alternative name for the file in the + on an existing file. The file is either named by 'name', or + specified as a file object 'fileobj' with a file descriptor. If + given, 'arcname' specifies an alternative name for the file in the archive, otherwise, the name is taken from the 'name' attribute of 'fileobj', or the 'name' argument. The name should be a text string. @@ -2030,7 +2086,7 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): # Now, fill the TarInfo object with # information specific for the file. tarinfo = self.tarinfo() - tarinfo.tarfile = self # Not needed + tarinfo._tarfile = self # To be removed in 3.16. # Use os.stat or os.lstat, depending on if symlinks shall be resolved. if fileobj is None: @@ -2084,16 +2140,23 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): tarinfo.mtime = statres.st_mtime tarinfo.type = type tarinfo.linkname = linkname + + # Calls to pwd.getpwuid() and grp.getgrgid() tend to be expensive. To + # speed things up, cache the resolved usernames and group names. if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass + if tarinfo.uid not in self._unames: + try: + self._unames[tarinfo.uid] = pwd.getpwuid(tarinfo.uid)[0] + except KeyError: + self._unames[tarinfo.uid] = '' + tarinfo.uname = self._unames[tarinfo.uid] if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass + if tarinfo.gid not in self._gnames: + try: + self._gnames[tarinfo.gid] = grp.getgrgid(tarinfo.gid)[0] + except KeyError: + self._gnames[tarinfo.gid] = '' + tarinfo.gname = self._gnames[tarinfo.gid] if type in (CHRTYPE, BLKTYPE): if hasattr(os, "major") and hasattr(os, "minor"): @@ -2102,9 +2165,9 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If `verbose' is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. `members' is optional and must be a subset of the + """Print a table of contents to sys.stdout. If 'verbose' is False, only + the names of the members are printed. If it is True, an 'ls -l'-like + output is produced. 'members' is optional and must be a subset of the list returned by getmembers(). """ # Convert tarinfo type to stat type. @@ -2145,11 +2208,11 @@ def list(self, verbose=True, *, members=None): print() def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file `name' to the archive. `name' may be any type of file - (directory, fifo, symbolic link, etc.). If given, `arcname' + """Add the file 'name' to the archive. 'name' may be any type of file + (directory, fifo, symbolic link, etc.). If given, 'arcname' specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by - setting `recursive' to False. `filter' is a function + setting 'recursive' to False. 'filter' is a function that expects a TarInfo object argument and returns the changed TarInfo object, if it returns None the TarInfo object will be excluded from the archive. @@ -2196,13 +2259,16 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). + """Add the TarInfo object 'tarinfo' to the archive. If 'tarinfo' represents + a non zero-size regular file, the 'fileobj' argument should be a binary file, + and tarinfo.size bytes are read from it and added to the archive. + You can create TarInfo objects directly, or by using gettarinfo(). """ self._check("awx") + if fileobj is None and tarinfo.isreg() and tarinfo.size != 0: + raise ValueError("fileobj not provided for non zero-size regular file") + tarinfo = copy.copy(tarinfo) buf = tarinfo.tobuf(self.format, self.encoding, self.errors) @@ -2224,13 +2290,7 @@ def _get_filter_function(self, filter): if filter is None: filter = self.extraction_filter if filter is None: - import warnings - warnings.warn( - 'Python 3.14 will, by default, filter extracted tar ' - + 'archives and reject files or modify their metadata. ' - + 'Use the filter argument to control this behavior.', - DeprecationWarning) - return fully_trusted_filter + return data_filter if isinstance(filter, str): raise TypeError( 'String names are not supported for ' @@ -2248,12 +2308,12 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, filter=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only + directories afterwards. 'path' specifies a different directory + to extract to. 'members' is optional and must be a subset of the + list returned by getmembers(). If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called on each member just + The 'filter' function will be called on each member just before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. @@ -2293,13 +2353,13 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, filter=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` + as possible. 'member' may be a filename or a TarInfo object. You can + specify a different directory using 'path'. File attributes (owner, + mtime, mode) are set unless 'set_attrs' is False. If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called before extraction. + The 'filter' function will be called before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. """ @@ -2364,10 +2424,10 @@ def _handle_fatal_error(self, e): self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) def extractfile(self, member): - """Extract a member from the archive as a file object. `member' may be - a filename or a TarInfo object. If `member' is a regular file or + """Extract a member from the archive as a file object. 'member' may be + a filename or a TarInfo object. If 'member' is a regular file or a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If `member' does not appear + existing members, None is returned. If 'member' does not appear in the archive, KeyError is raised. """ self._check("r") @@ -2411,7 +2471,7 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, if upperdirs and not os.path.exists(upperdirs): # Create directories that are not part of the archive with # default permissions. - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if tarinfo.islnk() or tarinfo.issym(): self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) @@ -2565,7 +2625,7 @@ def chown(self, tarinfo, targetpath, numeric_owner): else: os.chown(targetpath, u, g) except (OSError, OverflowError) as e: - # OverflowError can be raised if an ID doesn't fit in `id_t` + # OverflowError can be raised if an ID doesn't fit in 'id_t' raise ExtractError("could not change owner") from e def chmod(self, tarinfo, targetpath): diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index 15586f15dfab30..493932d6c6d441 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -18,6 +18,7 @@ import socket import subprocess import sys +import textwrap import time import unittest @@ -492,29 +493,31 @@ def test_devpoll(self): self.check_elapsed_time(dt) -class FNTLEINTRTest(EINTRBaseTest): +class FCNTLEINTRTest(EINTRBaseTest): def _lock(self, lock_func, lock_name): self.addCleanup(os_helper.unlink, os_helper.TESTFN) - code = '\n'.join(( - "import fcntl, time", - "with open('%s', 'wb') as f:" % os_helper.TESTFN, - " fcntl.%s(f, fcntl.LOCK_EX)" % lock_name, - " time.sleep(%s)" % self.sleep_time)) - start_time = time.monotonic() - proc = self.subprocess(code) + rd1, wr1 = os.pipe() + rd2, wr2 = os.pipe() + for fd in (rd1, wr1, rd2, wr2): + self.addCleanup(os.close, fd) + code = textwrap.dedent(f""" + import fcntl, os, time + with open('{os_helper.TESTFN}', 'wb') as f: + fcntl.{lock_name}(f, fcntl.LOCK_EX) + os.write({wr1}, b"ok") + _ = os.read({rd2}, 2) # wait for parent process + time.sleep({self.sleep_time}) + """) + proc = self.subprocess(code, pass_fds=[wr1, rd2]) with kill_on_error(proc): with open(os_helper.TESTFN, 'wb') as f: # synchronize the subprocess + ok = os.read(rd1, 2) + self.assertEqual(ok, b"ok") + + # notify the child that the parent is ready start_time = time.monotonic() - for _ in support.sleeping_retry(support.LONG_TIMEOUT, error=False): - try: - lock_func(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - lock_func(f, fcntl.LOCK_UN) - except BlockingIOError: - break - else: - dt = time.monotonic() - start_time - raise Exception("failed to sync child in %.1f sec" % dt) + os.write(wr2, b"go") # the child locked the file just a moment ago for 'sleep_time' seconds # that means that the lock below will block for 'sleep_time' minus some diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index a2ddd133cf47c8..7edb35da463aa0 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -6,14 +6,51 @@ # (before the site module is run). import _testinternalcapi -import os import sys import unittest +from test import support from test.support import MS_WINDOWS MAX_HASH_SEED = 4294967295 + +BOOL_OPTIONS = [ + 'isolated', + 'use_environment', + 'dev_mode', + 'install_signal_handlers', + 'use_hash_seed', + 'faulthandler', + 'import_time', + 'code_debug_ranges', + 'show_ref_count', + 'dump_refs', + 'malloc_stats', + 'parse_argv', + 'site_import', + 'warn_default_encoding', + 'inspect', + 'interactive', + 'parser_debug', + 'write_bytecode', + 'quiet', + 'user_site_directory', + 'configure_c_stdio', + 'buffered_stdio', + 'use_frozen_modules', + 'safe_path', + 'pathconfig_warnings', + 'module_search_paths_set', + 'skip_source_first_line', + '_install_importlib', + '_init_main', + '_is_python_build', +] +if MS_WINDOWS: + BOOL_OPTIONS.append('legacy_windows_stdio') + + class SetConfigTests(unittest.TestCase): def setUp(self): self.old_config = _testinternalcapi.get_config() @@ -52,42 +89,15 @@ def test_set_invalid(self): ] # int (unsigned) - options = [ + int_options = [ '_config_init', - 'isolated', - 'use_environment', - 'dev_mode', - 'install_signal_handlers', - 'use_hash_seed', - 'faulthandler', - 'tracemalloc', - 'import_time', - 'code_debug_ranges', - 'show_ref_count', - 'dump_refs', - 'malloc_stats', - 'parse_argv', - 'site_import', 'bytes_warning', - 'inspect', - 'interactive', 'optimization_level', - 'parser_debug', - 'write_bytecode', + 'tracemalloc', 'verbose', - 'quiet', - 'user_site_directory', - 'configure_c_stdio', - 'buffered_stdio', - 'pathconfig_warnings', - 'module_search_paths_set', - 'skip_source_first_line', - '_install_importlib', - '_init_main', ] - if MS_WINDOWS: - options.append('legacy_windows_stdio') - for key in options: + int_options.extend(BOOL_OPTIONS) + for key in int_options: value_tests.append((key, invalid_uint)) type_tests.append((key, "abc")) type_tests.append((key, 2.0)) @@ -127,7 +137,8 @@ def test_set_invalid(self): 'warnoptions', 'module_search_paths', ): - value_tests.append((key, invalid_wstrlist)) + if key != 'xoptions': + value_tests.append((key, invalid_wstrlist)) type_tests.append((key, 123)) type_tests.append((key, "abc")) type_tests.append((key, [123])) @@ -148,19 +159,23 @@ def test_set_invalid(self): _testinternalcapi.set_config(config) def test_flags(self): + bool_options = set(BOOL_OPTIONS) for sys_attr, key, value in ( - ("debug", "parser_debug", 1), - ("inspect", "inspect", 2), - ("interactive", "interactive", 3), - ("optimize", "optimization_level", 4), - ("verbose", "verbose", 1), - ("bytes_warning", "bytes_warning", 10), - ("quiet", "quiet", 11), - ("isolated", "isolated", 12), + ("debug", "parser_debug", 2), + ("inspect", "inspect", 3), + ("interactive", "interactive", 4), + ("optimize", "optimization_level", 5), + ("verbose", "verbose", 6), + ("bytes_warning", "bytes_warning", 7), + ("quiet", "quiet", 8), + ("isolated", "isolated", 9), ): with self.subTest(sys=sys_attr, key=key, value=value): self.set_config(**{key: value, 'parse_argv': 0}) - self.assertEqual(getattr(sys.flags, sys_attr), value) + if key in bool_options: + self.assertEqual(getattr(sys.flags, sys_attr), int(bool(value))) + else: + self.assertEqual(getattr(sys.flags, sys_attr), value) self.set_config(write_bytecode=0) self.assertEqual(sys.flags.dont_write_bytecode, True) @@ -197,13 +212,26 @@ def test_flags(self): self.set_config(use_hash_seed=1, hash_seed=123) self.assertEqual(sys.flags.hash_randomization, 1) + if support.Py_GIL_DISABLED: + self.set_config(enable_gil=-1) + self.assertEqual(sys.flags.gil, None) + self.set_config(enable_gil=0) + self.assertEqual(sys.flags.gil, 0) + self.set_config(enable_gil=1) + self.assertEqual(sys.flags.gil, 1) + else: + # Builds without Py_GIL_DISABLED don't have + # PyConfig.enable_gil. sys.flags.gil is always defined to 1, for + # consistency. + self.assertEqual(sys.flags.gil, 1) + def test_options(self): self.check(warnoptions=[]) self.check(warnoptions=["default", "ignore"]) - self.set_config(xoptions=[]) + self.set_config(xoptions={}) self.assertEqual(sys._xoptions, {}) - self.set_config(xoptions=["dev", "tracemalloc=5"]) + self.set_config(xoptions={"dev": True, "tracemalloc": "5"}) self.assertEqual(sys._xoptions, {"dev": True, "tracemalloc": "5"}) def test_pathconfig(self): diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 834daa4df55fec..154662efce9412 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -1,31 +1,27 @@ import sys import types +import unittest -# Note: This test file can't import `unittest` since the runtime can't -# currently guarantee that it will not leak memory. Doing so will mark -# the test as passing but with reference leaks. This can safely import -# the `unittest` library once there's a strict guarantee of no leaks -# during runtime shutdown. # bpo-46417: Test that structseq types used by the sys module are still # valid when Py_Finalize()/Py_Initialize() are called multiple times. -class TestStructSeq: +class TestStructSeq(unittest.TestCase): # test PyTypeObject members - def _check_structseq(self, obj_type): + def check_structseq(self, obj_type): # ob_refcnt - assert sys.getrefcount(obj_type) > 1 + self.assertGreaterEqual(sys.getrefcount(obj_type), 1) # tp_base - assert issubclass(obj_type, tuple) + self.assertTrue(issubclass(obj_type, tuple)) # tp_bases - assert obj_type.__bases__ == (tuple,) + self.assertEqual(obj_type.__bases__, (tuple,)) # tp_dict - assert isinstance(obj_type.__dict__, types.MappingProxyType) + self.assertIsInstance(obj_type.__dict__, types.MappingProxyType) # tp_mro - assert obj_type.__mro__ == (obj_type, tuple, object) + self.assertEqual(obj_type.__mro__, (obj_type, tuple, object)) # tp_name - assert isinstance(type.__name__, str) + self.assertIsInstance(type.__name__, str) # tp_subclasses - assert obj_type.__subclasses__() == [] + self.assertEqual(obj_type.__subclasses__(), []) def test_sys_attrs(self): for attr_name in ( @@ -36,23 +32,30 @@ def test_sys_attrs(self): 'thread_info', # ThreadInfoType 'version_info', # VersionInfoType ): - attr = getattr(sys, attr_name) - self._check_structseq(type(attr)) + with self.subTest(attr=attr_name): + attr = getattr(sys, attr_name) + self.check_structseq(type(attr)) def test_sys_funcs(self): func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType if hasattr(sys, 'getwindowsversion'): func_names.append('getwindowsversion') # WindowsVersionType for func_name in func_names: - func = getattr(sys, func_name) - obj = func() - self._check_structseq(type(obj)) + with self.subTest(func=func_name): + func = getattr(sys, func_name) + obj = func() + self.check_structseq(type(obj)) try: - tests = TestStructSeq() - tests.test_sys_attrs() - tests.test_sys_funcs() + unittest.main( + module=( + '__main__' + if __name__ == '__main__' + # Avoiding a circular import: + else sys.modules['test._test_embed_structseq'] + ) + ) except SystemExit as exc: if exc.args[0] != 0: raise diff --git a/Lib/test/_test_monitoring_shutdown.py b/Lib/test/_test_monitoring_shutdown.py new file mode 100644 index 00000000000000..3d0fbec029dc09 --- /dev/null +++ b/Lib/test/_test_monitoring_shutdown.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +# gh-115832: An object destructor running during the final GC of interpreter +# shutdown triggered an infinite loop in the instrumentation code. + +import sys + +class CallableCycle: + def __init__(self): + self._cycle = self + + def __del__(self): + pass + + def __call__(self, code, instruction_offset): + pass + +def tracefunc(frame, event, arg): + pass + +def main(): + tool_id = sys.monitoring.PROFILER_ID + event_id = sys.monitoring.events.PY_START + + sys.monitoring.use_tool_id(tool_id, "test profiler") + sys.monitoring.set_events(tool_id, event_id) + sys.monitoring.register_callback(tool_id, event_id, CallableCycle()) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index c0d3ca50f17d69..065fc27b770438 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -22,7 +22,6 @@ import subprocess import struct import operator -import pathlib import pickle import weakref import warnings @@ -324,8 +323,9 @@ def test_set_executable(self): self.skipTest(f'test not appropriate for {self.TYPE}') paths = [ sys.executable, # str - sys.executable.encode(), # bytes - pathlib.Path(sys.executable) # os.PathLike + os.fsencode(sys.executable), # bytes + os_helper.FakePath(sys.executable), # os.PathLike + os_helper.FakePath(os.fsencode(sys.executable)), # os.PathLike bytes ] for path in paths: self.set_executable(path) @@ -1332,6 +1332,23 @@ def _on_queue_feeder_error(e, obj): self.assertTrue(not_serializable_obj.reduce_was_called) self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called) + def test_closed_queue_empty_exceptions(self): + # Assert that checking the emptiness of an unused closed queue + # does not raise an OSError. The rationale is that q.close() is + # a no-op upon construction and becomes effective once the queue + # has been used (e.g., by calling q.put()). + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.close() # this is a no-op since the feeder thread is None + q.join_thread() # this is also a no-op + self.assertTrue(q.empty()) + + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.put('foo') # make sure that the queue is 'used' + q.close() # close the feeder thread + q.join_thread() # make sure to join the feeder thread + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_closed_queue_put_get_exceptions(self): for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): q.close() @@ -2812,8 +2829,8 @@ def test_release_task_refs(self): self.pool.map(identity, objs) del objs - gc.collect() # For PyPy or other GCs. time.sleep(DELTA) # let threaded cleanup code run + support.gc_collect() # For PyPy or other GCs. self.assertEqual(set(wr() for wr in refs), {None}) # With a process pool, copies of the objects are returned, check # they were released too. @@ -3504,6 +3521,30 @@ def test_context(self): if self.TYPE == 'processes': self.assertRaises(OSError, l.accept) + def test_empty_authkey(self): + # bpo-43952: allow empty bytes as authkey + def handler(*args): + raise RuntimeError('Connection took too long...') + + def run(addr, authkey): + client = self.connection.Client(addr, authkey=authkey) + client.send(1729) + + key = b'' + + with self.connection.Listener(authkey=key) as listener: + thread = threading.Thread(target=run, args=(listener.address, key)) + thread.start() + try: + with listener.accept() as d: + self.assertEqual(d.recv(), 1729) + finally: + thread.join() + + if self.TYPE == 'processes': + with self.assertRaises(OSError): + listener.accept() + @unittest.skipUnless(util.abstract_sockets_supported, "test needs abstract socket support") def test_abstract_socket(self): @@ -3971,6 +4012,21 @@ def _new_shm_name(self, prefix): # test_multiprocessing_spawn, etc) in parallel. return prefix + str(os.getpid()) + def test_shared_memory_name_with_embedded_null(self): + name_tsmb = self._new_shm_name('test01_null') + sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) + self.addCleanup(sms.unlink) + with self.assertRaises(ValueError): + shared_memory.SharedMemory(name_tsmb + '\0a', create=False, size=512) + if shared_memory._USE_POSIX: + orig_name = sms._name + try: + sms._name = orig_name + '\0a' + with self.assertRaises(ValueError): + sms.unlink() + finally: + sms._name = orig_name + def test_shared_memory_basics(self): name_tsmb = self._new_shm_name('test01_tsmb') sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) @@ -4105,7 +4161,7 @@ def test_shared_memory_recreate(self): self.addCleanup(shm2.unlink) self.assertEqual(shm2._name, names[1]) - def test_invalid_shared_memory_cration(self): + def test_invalid_shared_memory_creation(self): # Test creating a shared memory segment with negative size with self.assertRaises(ValueError): sms_invalid = shared_memory.SharedMemory(create=True, size=-1) @@ -4623,7 +4679,7 @@ def make_finalizers(): old_interval = sys.getswitchinterval() old_threshold = gc.get_threshold() try: - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) gc.set_threshold(5, 5, 5) threads = [threading.Thread(target=run_finalizers), threading.Thread(target=make_finalizers)] @@ -5497,15 +5553,29 @@ def test_set_get(self): multiprocessing.set_start_method(old_method, force=True) self.assertGreaterEqual(count, 1) - def test_get_all(self): + def test_get_all_start_methods(self): methods = multiprocessing.get_all_start_methods() + self.assertIn('spawn', methods) if sys.platform == 'win32': self.assertEqual(methods, ['spawn']) + elif sys.platform == 'darwin': + self.assertEqual(methods[0], 'spawn') # The default is first. + # Whether these work or not, they remain available on macOS. + self.assertIn('fork', methods) + self.assertIn('forkserver', methods) else: - self.assertTrue(methods == ['fork', 'spawn'] or - methods == ['spawn', 'fork'] or - methods == ['fork', 'spawn', 'forkserver'] or - methods == ['spawn', 'fork', 'forkserver']) + # POSIX + self.assertIn('fork', methods) + if other_methods := set(methods) - {'fork', 'spawn'}: + # If there are more than those two, forkserver must be one. + self.assertEqual({'forkserver'}, other_methods) + # The default is the first method in the list. + self.assertIn(methods[0], {'forkserver', 'spawn'}, + msg='3.14+ default must not be fork') + if methods[0] == 'spawn': + # Confirm that the current default selection logic prefers + # forkserver vs spawn when available. + self.assertNotIn('forkserver', methods) def test_preload_resources(self): if multiprocessing.get_start_method() != 'forkserver': @@ -5609,8 +5679,9 @@ def create_and_register_resource(rtype): ''' for rtype in resource_tracker._CLEANUP_FUNCS: with self.subTest(rtype=rtype): - if rtype == "noop": + if rtype in ("noop", "dummy"): # Artefact resource type used by the resource_tracker + # or tests continue r, w = os.pipe() p = subprocess.Popen([sys.executable, @@ -5690,6 +5761,8 @@ def test_resource_tracker_sigterm(self): # Catchable signal (ignored by semaphore tracker) self.check_resource_tracker_death(signal.SIGTERM, False) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-125620: Skip on NetBSD due to long wait for SIGKILL process termination.") def test_resource_tracker_sigkill(self): # Uncatchable signal. self.check_resource_tracker_death(signal.SIGKILL, True) @@ -5730,6 +5803,38 @@ def test_too_long_name_resource(self): with self.assertRaises(ValueError): resource_tracker.register(too_long_name_resource, rtype) + def _test_resource_tracker_leak_resources(self, cleanup): + # We use a separate instance for testing, since the main global + # _resource_tracker may be used to watch test infrastructure. + from multiprocessing.resource_tracker import ResourceTracker + tracker = ResourceTracker() + tracker.ensure_running() + self.assertTrue(tracker._check_alive()) + + self.assertIsNone(tracker._exitcode) + tracker.register('somename', 'dummy') + if cleanup: + tracker.unregister('somename', 'dummy') + expected_exit_code = 0 + else: + expected_exit_code = 1 + + self.assertTrue(tracker._check_alive()) + self.assertIsNone(tracker._exitcode) + tracker._stop() + self.assertEqual(tracker._exitcode, expected_exit_code) + + def test_resource_tracker_exit_code(self): + """ + Test the exit code of the resource tracker. + + If no leaked resources were found, exit code should be 0, otherwise 1 + """ + for cleanup in [True, False]: + with self.subTest(cleanup=cleanup): + self._test_resource_tracker_leak_resources( + cleanup=cleanup, + ) class TestSimpleQueue(unittest.TestCase): @@ -5743,6 +5848,15 @@ def _test_empty(cls, queue, child_can_start, parent_can_continue): finally: parent_can_continue.set() + def test_empty_exceptions(self): + # Assert that checking emptiness of a closed queue raises + # an OSError, independently of whether the queue was used + # or not. This differs from Queue and JoinableQueue. + q = multiprocessing.SimpleQueue() + q.close() # close the pipe + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_empty(self): queue = multiprocessing.SimpleQueue() child_can_start = multiprocessing.Event() @@ -5992,12 +6106,30 @@ def _test_list(cls, obj): case.assertEqual(obj[0], 5) case.assertEqual(obj.count(5), 1) case.assertEqual(obj.index(5), 0) + obj += [7] + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7]) + obj *= 2 + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7, 5, 7]) + double_obj = obj * 2 + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + double_obj = 2 * obj + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + copied_obj = obj.copy() + case.assertIsInstance(copied_obj, list) + case.assertListEqual(list(copied_obj), [5, 7, 5, 7]) + obj.extend(double_obj + copied_obj) obj.sort() obj.reverse() for x in obj: pass - case.assertEqual(len(obj), 1) - case.assertEqual(obj.pop(0), 5) + case.assertEqual(len(obj), 16) + case.assertEqual(obj.pop(0), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_list(self): o = self.manager.list() @@ -6016,7 +6148,29 @@ def _test_dict(cls, obj): case.assertListEqual(list(obj.keys()), ['foo']) case.assertListEqual(list(obj.values()), [5]) case.assertDictEqual(obj.copy(), {'foo': 5}) - case.assertTupleEqual(obj.popitem(), ('foo', 5)) + obj |= {'bar': 6} + case.assertIsInstance(obj, multiprocessing.managers.DictProxy) + case.assertDictEqual(dict(obj), {'foo': 5, 'bar': 6}) + x = reversed(obj) + case.assertIsInstance(x, type(iter([]))) + case.assertListEqual(list(x), ['bar', 'foo']) + x = {'bar': 7, 'baz': 7} | obj + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 6, 'baz': 7}) + x = obj | {'bar': 7, 'baz': 7} + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 7, 'baz': 7}) + x = obj.fromkeys(['bar'], 6) + case.assertIsInstance(x, dict) + case.assertDictEqual(x, {'bar': 6}) + x = obj.popitem() + case.assertIsInstance(x, tuple) + case.assertTupleEqual(x, ('bar', 6)) + obj.setdefault('bar', 0) + obj.update({'bar': 7}) + case.assertEqual(obj.pop('bar'), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_dict(self): o = self.manager.dict() @@ -6089,6 +6243,29 @@ def submain(): pass self.assertFalse(err, msg=err.decode('utf-8')) +class _TestAtExit(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @classmethod + def _write_file_at_exit(self, output_path): + import atexit + def exit_handler(): + with open(output_path, 'w') as f: + f.write("deadbeef") + atexit.register(exit_handler) + + def test_atexit(self): + # gh-83856 + with os_helper.temp_dir() as temp_dir: + output_path = os.path.join(temp_dir, 'output.txt') + p = self.Process(target=self._write_file_at_exit, args=(output_path,)) + p.start() + p.join() + with open(output_path) as f: + self.assertEqual(f.read(), 'deadbeef') + + class MiscTestCase(unittest.TestCase): def test__all__(self): # Just make sure names in not_exported are excluded @@ -6113,6 +6290,24 @@ def test_spawn_sys_executable_none_allows_import(self): self.assertEqual(rc, 0) self.assertFalse(err, msg=err.decode('utf-8')) + def test_large_pool(self): + # + # gh-89240: Check that large pools are always okay + # + testfn = os_helper.TESTFN + self.addCleanup(os_helper.unlink, testfn) + with open(testfn, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + import multiprocessing + def f(x): return x*x + if __name__ == '__main__': + with multiprocessing.Pool(200) as p: + print(sum(p.map(f, range(1000)))) + ''')) + rc, out, err = script_helper.assert_python_ok(testfn) + self.assertEqual("332833500", out.decode('utf-8').strip()) + self.assertFalse(err, msg=err.decode('utf-8')) + # # Mixins diff --git a/Lib/test/archiver_tests.py b/Lib/test/archiver_tests.py index 1a4bbb9e5706c5..24745941b08923 100644 --- a/Lib/test/archiver_tests.py +++ b/Lib/test/archiver_tests.py @@ -3,6 +3,7 @@ import os import sys +from test.support import swap_attr from test.support import os_helper class OverwriteTests: @@ -153,3 +154,24 @@ def test_overwrite_broken_dir_symlink_as_implicit_dir(self): self.extractall(ar) self.assertTrue(os.path.islink(target)) self.assertFalse(os.path.exists(target2)) + + def test_concurrent_extract_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_concurrent_extract_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) diff --git a/Lib/test/archivetestdata/zipdir_backslash.zip b/Lib/test/archivetestdata/zipdir_backslash.zip new file mode 100644 index 00000000000000..979126ef5e37eb Binary files /dev/null and b/Lib/test/archivetestdata/zipdir_backslash.zip differ diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index de7d0da560a1c7..6df09d891433ea 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -556,6 +556,28 @@ def hook(event, args): sys.monitoring.register_callback(1, 1, None) +def test_winapi_createnamedpipe(pipe_name): + import _winapi + + def hook(event, args): + if event == "_winapi.CreateNamedPipe": + print(event, args) + + sys.addaudithook(hook) + _winapi.CreateNamedPipe(pipe_name, _winapi.PIPE_ACCESS_DUPLEX, 8, 2, 0, 0, 0, 0) + + +def test_assert_unicode(): + import sys + sys.addaudithook(lambda *args: None) + try: + sys.audit(9) + except TypeError: + pass + else: + raise RuntimeError("Expected sys.audit(9) to fail.") + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/bisect_cmd.py b/Lib/test/bisect_cmd.py index 5cb804bd469dc3..aee2e8ac120852 100755 --- a/Lib/test/bisect_cmd.py +++ b/Lib/test/bisect_cmd.py @@ -51,6 +51,7 @@ def python_cmd(): cmd = [sys.executable] cmd.extend(subprocess._args_from_interpreter_flags()) cmd.extend(subprocess._optim_args_from_interpreter_flags()) + cmd.extend(('-X', 'faulthandler')) return cmd @@ -77,9 +78,13 @@ def run_tests(args, tests, huntrleaks=None): write_tests(tmp, tests) cmd = python_cmd() - cmd.extend(['-m', 'test', '--matchfile', tmp]) + cmd.extend(['-u', '-m', 'test', '--matchfile', tmp]) cmd.extend(args.test_args) print("+ %s" % format_shell_args(cmd)) + + sys.stdout.flush() + sys.stderr.flush() + proc = subprocess.run(cmd) return proc.returncode finally: @@ -137,8 +142,8 @@ def main(): ntest = max(ntest // 2, 1) subtests = random.sample(tests, ntest) - print("[+] Iteration %s: run %s tests/%s" - % (iteration, len(subtests), len(tests))) + print(f"[+] Iteration {iteration}/{args.max_iter}: " + f"run {len(subtests)} tests/{len(tests)}") print() exitcode = run_tests(args, subtests) @@ -170,10 +175,10 @@ def main(): if len(tests) <= args.max_tests: print("Bisection completed in %s iterations and %s" % (iteration, datetime.timedelta(seconds=dt))) - sys.exit(1) else: print("Bisection failed after %s iterations and %s" % (iteration, datetime.timedelta(seconds=dt))) + sys.exit(1) if __name__ == "__main__": diff --git a/Lib/test/certdata/allsans.pem b/Lib/test/certdata/allsans.pem index e400e178a1f599..f6a63b9219d246 100644 --- a/Lib/test/certdata/allsans.pem +++ b/Lib/test/certdata/allsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDBGvj+Uy/VUyTR -mmIA1UEENThh0+pWODcvvUlkeIo+XTJ3FhF4/RVjImDHjozl28Xf2TzKnvQJa1KC -pqa7fr8cL9QMwk4pH+S4ulxOu02Bl3Yafx2oJVUML37vciJg+zkzPx1k3tXFjXkr -LGjZwOoufBC3AmPuq2xHFBzHrvp5/DIRH2slQFM9fpVZzN77gYyzxba0wCfCPpCf -eJFRyYKW8c7MXrwnM82YtE7Rlnf227EkCdMNaSeZLUIxeVpcnScqZl0SIbR3YEiV -0LPFkx0wJFm8qUEFU/h+0jamgy/ON+11nqmMlp3BjNi/JTVsa7N7A3dvdHC7VVlr -WnUgU6MoSniyL6ijpucyHtZzK2mJy0sHR8PadHKow0O423/5N8GKTSOvaGMXTjAe -OGs+9/P1ZYo3IjjQPz/NV3QlhK8zRqxF3cW0ekHHkT+/jZjCvSKm6mdbMQunKE1W -+dokAc815pb48Mzf1eWKd/7UyUf7CXussyAaJ3clpaK1sbbn9m0CAwEAAQKCAYAe -BaCCgdJk+xk1USg9cuo5ykBqzTSYlQLXdDlN2oO7sGehJhgvVEGX+QdM3ze+oM2B -wNd3tQDB2iKo11oCunDh4/m2xhq6wA+iPK8POoWRSUf+VJb6xlsTmurENV1s8IHz -GrPqM87OePFGqg/fEuQVuAotObzppVMfNdxHm0er4W6zRMw2rWqDnAOCQ5zDQ1/p -ryp5rYpA49M+R9NoAMlByHRbR7s+6Qnk3NuIMDmUcpF2xeQ/KIMUiHnLEU/gKDpi -bsk+VtyjlibR4zhh9/cJrLTApAIA+4eC176EJvKXCh5UIjd92JC7741HTNQXJpvG -9PXbzhyUCmncr04U+46snGHdwD+lG4LS7oBGACTLMtpcMrlgAm6XCg4T8gRVE/9n -FvCkqPHBR+vnhOxm+0x0yUY/DstJby6IPYPsfGK/s2n//j/vJrAZE1Pxlm9EPU13 -MRLcHstwjAc/NXRPnUN1DfcQvPLx6Tt6rqw3Wm1KO75kM+HZ56BX9/Bi1TgkiI0C -gcEA5JTlXssJ3W8Cz6w1ZtGsThHQBDbvHF2D5AdqO7y6/eqzCQgBQl9BTfXOzsvP -I1gf2CLEFBtGK09UjAuJQg90/NlKur7i7xt7HpAzEfGsDAL4P5BW5JnMNrzpJjjL -0uUDsPJlA75Wi29N2SFiaIslY0sZ6nckInat5GRe4O1AMSHoJ5suY9yTZTU3XB4O -A+XyddutI1GsFZgl8/8LyyNMcyNjxG3T5sr7IKf5/nIv6oMDjC2zLVZa8QS/MEnL -Kaa7AoHBANhEsxfcjw2MaPkrsqAsOP0dDf7g2rdz6wKT5BzZu9e+/E76NmvVDpns -e+kCjql9Os3/wonOMINvn1bTCQGTgk8+dw1fMyqg+zQCvH4ImcE6LSqhzblVHsIB -zZ7rW86trri1U9+olNHG4nwkus0i4LV8eeORns+j8DgXr6/eOvjX3ZW5TyU7/Qgm -SiSdBapzJbom3xJrbo9KQsrN5PVCOwuwrgY0o+2BeKyKhnt4uGv0bR+ii06EOJUA -WvjD7gLI9wKBwGVRXk3jH29IOm3EvjLh80bzfEmx89CV3tUfOEZcRGIyOsNhCfXa -dP7SWqWtDxZyhELwPgtPf43I7wfYQTHH2ioNQqN94ubrPmpwrkJg5cq5MkIyf2F6 -jlsg5xMrD6VeH4G6H25GWuQZJN9+fbkrHBpj+ovD3X9tLWzT1H5Miyx8BAQyM6DN -74Nn0C8Dn2C49vyor5i9JdK4ivIY9ahH8CYE5L73k3p0NFXoPtY61ORUyCjFROtu -oIa+fOQxgVzn6wKBwQC3DD7BnY7/Gq7m51ODOqrpoaPs7Qhyagyp298hhDD3hNEt -T56sWmLHaV/fcqipUDNrlGRmGzz4ooutA2YGDYIn7Gj7ym4WULcN6Jr92e25nLIJ -+XWUvjUQZFJThkXogxz1fZSGI7wCamHcTYJGipTDR54rPV+7w7hY4cN0CZbEdIE6 -buRMUZ/zO+VZZAYdpORz0N7SSlgDtAkgenCmHe64EEzbN8bgCcvHzl/RNfZyeSm7 -supSBJuXkfttvvg/JzUCgcEAlx0Pep9qCLvpk0WqzijBVHc3zK4wYxjhN2MBkF42 -SLWfogKpiPfIqxX6YF94roIA0VlW6Pj50v+sbPwq8nwsgFNhml80A4ODKr3O3Y3M -fXDBJW5W5ZRb/vhIKRjXyCSckSRfj7N8HUYjCLkxQansNWimrldmSet0H2mYJN0Y -JpBXdqpa76zoHzWpKFwD0fSVzvnMelPHSDCNOdIEHmR8e1x2F1/ufR/9/dBzPULY -HMj0OhQHoi8kJyMIj3+bQkbC +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCYEY+lIwS/w+gA +U5PfxSeL4qMF5TtFkrifL0oQaCjWFb8jeX2Rb1tKaObyrYmTURVWbMkI1LJOQWYt +GlANo2mvf4VD1wZZ/x/Tmylxe3A7itSTZgZuGXLHtFP9wk7yS46BlVcNq3ZoOi7k +k8NxTiUqgsmLg3Nj+Ifk8OI7c1BcP2pa1wZO/l3PTWqtFSLsFEvOK5N6ZPqILAUi +5vEkOiFefxVHHvD4N3IWSFgSwiDi9KhntdRwa+axsZxjq/o70FjQO5q5Quym9aDq +qo47PMykEb1gdTBQK1mPliMKsLi5zycI/ypm1PEcH53+9/F+U2B1bmvA/9PBx8T7 +hUyclBbQHCmdcXIYCgFr5AbGS3jFeVPiF7hCPj1k/jcpayXZAtWmqJTlkXKGbM68 +AgM/DHO0fGCYGRUw2vhV3ZIRCm0BqwforeFh+2oMyZjDjvo/uZjZNazdpcwvOy0c +lFkORwVI0zTIixXYSV4xN1BX821JCuhYubro89m+MSjxO14qgs0CAwEAAQKCAYAZ +txqF7+qNLSlN6NRASnw6BQzRYebIiJus25fYN2z0awAEFTbdQan75sprLlpt7Y/A +qivC0QkH/7oyFVmFPOWR1mwoQTPjQyfUJlB3Tsr3Xq488MtUkfwddkqfxlyIT6ud +ES6D8sNWs8QbRjuOLQtO6YgAji2UscH1PqDbMdBckSLAks0Pzab6d9p6w3DA4FvD +VQ4e6/WL0nnZ4ZjUqfnbm3zzJnHUX7fsubYfEfHyvzG9O/vdOPntgZ3zIvFxbPVc +/Dp8MRNgfQoRgAsc7VBQQifFNvva9SjvRsXwE7zQLQh9JszFpHWqIMevGwjdH8dO +1hHESqkRHL0sYwsqoGqbaHFrJVEA2Ljuxs/UsH6G2U6NuDVaYkqM8UwH0+EqGFtF +9qP8oh6Yvp+szI5yET4brW0Rlk8r2UqvQ5iSSl8SSWsNKR2wRs0PM0aNm13TDJmt +NOuxd/0ranG+NvBMdtLCwsoEplYDaaczYv9gbKHT64OlFAdGBkOytbS7eqOojmcC +gcEA0uQb9bPKbiMYSEJMAalOlruzvMiHBRW420IEuGyHJtocn/RK804Jj6gzu+WD +GT81rB9NXAt/3s6WDMSdrGrrBh/CQZJs50ok9hej0f6gsq1JNsWJ/c10rU/mqrTn +s++pdaRqbd+bgM/N6h79tJi3j4SgB9rJqjC+7qWlDJFWZOoDR59elc10xDA0AE0a +K6UvsJJGk4/v715K1Uh2qU5HYupuijNc0m3z2OMZ6zxTNNUdbSL4XF5X9e2+BwMQ +YrRzAoHBALiYeYWsmHGRdg++/p/9SE86YwyEYr+zrUiWenIPC1W5WQlkdFQqALmg +kjcXAXFFouC/DnKG65YOfGq772ekFuw06Lm4DEotQP02pESfAP9ulVufK2yJtbxi +m0PeN3JRrs/H72VW9hUSxCafNRyBeZ6nsfzEfEkPOdc+lRGeMHT9C7r7Y1asNk8A +Zl7wKZgv+Yvn8xhHWQj5z/T6K2z5CXFn6FERc0qV8+DggE9m3CjaaNxft6IodfHD +h1iakNtbvwKBwQCpWr/NRy131r0IQh0xdFoFGAUVxF8RSUli4hhSVe0O2TcFiLOr +wW5SK/wnlv75hlY+vABuu1lbfsDmzfnk3RORnm1sJP9JmbQm4AMRfw5jjl7uGiJf +a9+X0kNlsNMlH4ARVhCV3WzOO5KbwXlxzvYRzaqJxDwQbQbXNLRfbFNZxMcPfD8D +w7NSXXdVCpXKmOO8QytkEsHWkv07W+7WtWMEX0iXuPmAjwW0lWNaEd6r3by8yMlz +u9udRedFUEOXUFsCgcAX0g00c75EQXoTtBjVenC/UJCBh//aLwx4Znqsh0Z2LHHR +5XWher4XNiJIG57jCBJpoB30J3b1KS9i8peFL0aJ+pXhiV+EnuxZAJkYBdCyJYn+ +hb6rxeV+xta0XlOXW/UL+QfqcttUgtRvC3JmGEsibw9nx88l+mIDZZ8E4/3qytCd +s1zxTU3AyhNrwuALNH2mUSssgeB6aQot2a6K5GQUj00KUQ8om8sZxL6qAGL+npiT +f4KJ2WDG7u1jQKbat68CgcBONTGp3M3pdwc3OV/PMFq1+h1/BuyW/5qUtbsvXkrD +DZyXGY2SxJFMipakzOppusaQOYmO+VXKVDRJm8UG6fa2cdXjwm2kDxHB5E+v59VI ++uM8ErX6L52I0+rdoOU6AwpXyiVW5AXzMInOGHF4B7zJ1SA25RwVG44LbDw1cehW +MALXUdVCaegmvCCFq5TQlrlaEktxWk6Th92mXWbNbwUHlIGpcjEkASFvt90aBXWZ +w0YCQFVxx/K95nzKyRTjHTQ= -----END PRIVATE KEY----- Certificate: Data: @@ -47,42 +47,42 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=allsans Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:c1:1a:f8:fe:53:2f:d5:53:24:d1:9a:62:00:d5: - 41:04:35:38:61:d3:ea:56:38:37:2f:bd:49:64:78: - 8a:3e:5d:32:77:16:11:78:fd:15:63:22:60:c7:8e: - 8c:e5:db:c5:df:d9:3c:ca:9e:f4:09:6b:52:82:a6: - a6:bb:7e:bf:1c:2f:d4:0c:c2:4e:29:1f:e4:b8:ba: - 5c:4e:bb:4d:81:97:76:1a:7f:1d:a8:25:55:0c:2f: - 7e:ef:72:22:60:fb:39:33:3f:1d:64:de:d5:c5:8d: - 79:2b:2c:68:d9:c0:ea:2e:7c:10:b7:02:63:ee:ab: - 6c:47:14:1c:c7:ae:fa:79:fc:32:11:1f:6b:25:40: - 53:3d:7e:95:59:cc:de:fb:81:8c:b3:c5:b6:b4:c0: - 27:c2:3e:90:9f:78:91:51:c9:82:96:f1:ce:cc:5e: - bc:27:33:cd:98:b4:4e:d1:96:77:f6:db:b1:24:09: - d3:0d:69:27:99:2d:42:31:79:5a:5c:9d:27:2a:66: - 5d:12:21:b4:77:60:48:95:d0:b3:c5:93:1d:30:24: - 59:bc:a9:41:05:53:f8:7e:d2:36:a6:83:2f:ce:37: - ed:75:9e:a9:8c:96:9d:c1:8c:d8:bf:25:35:6c:6b: - b3:7b:03:77:6f:74:70:bb:55:59:6b:5a:75:20:53: - a3:28:4a:78:b2:2f:a8:a3:a6:e7:32:1e:d6:73:2b: - 69:89:cb:4b:07:47:c3:da:74:72:a8:c3:43:b8:db: - 7f:f9:37:c1:8a:4d:23:af:68:63:17:4e:30:1e:38: - 6b:3e:f7:f3:f5:65:8a:37:22:38:d0:3f:3f:cd:57: - 74:25:84:af:33:46:ac:45:dd:c5:b4:7a:41:c7:91: - 3f:bf:8d:98:c2:bd:22:a6:ea:67:5b:31:0b:a7:28: - 4d:56:f9:da:24:01:cf:35:e6:96:f8:f0:cc:df:d5: - e5:8a:77:fe:d4:c9:47:fb:09:7b:ac:b3:20:1a:27: - 77:25:a5:a2:b5:b1:b6:e7:f6:6d + 00:98:11:8f:a5:23:04:bf:c3:e8:00:53:93:df:c5: + 27:8b:e2:a3:05:e5:3b:45:92:b8:9f:2f:4a:10:68: + 28:d6:15:bf:23:79:7d:91:6f:5b:4a:68:e6:f2:ad: + 89:93:51:15:56:6c:c9:08:d4:b2:4e:41:66:2d:1a: + 50:0d:a3:69:af:7f:85:43:d7:06:59:ff:1f:d3:9b: + 29:71:7b:70:3b:8a:d4:93:66:06:6e:19:72:c7:b4: + 53:fd:c2:4e:f2:4b:8e:81:95:57:0d:ab:76:68:3a: + 2e:e4:93:c3:71:4e:25:2a:82:c9:8b:83:73:63:f8: + 87:e4:f0:e2:3b:73:50:5c:3f:6a:5a:d7:06:4e:fe: + 5d:cf:4d:6a:ad:15:22:ec:14:4b:ce:2b:93:7a:64: + fa:88:2c:05:22:e6:f1:24:3a:21:5e:7f:15:47:1e: + f0:f8:37:72:16:48:58:12:c2:20:e2:f4:a8:67:b5: + d4:70:6b:e6:b1:b1:9c:63:ab:fa:3b:d0:58:d0:3b: + 9a:b9:42:ec:a6:f5:a0:ea:aa:8e:3b:3c:cc:a4:11: + bd:60:75:30:50:2b:59:8f:96:23:0a:b0:b8:b9:cf: + 27:08:ff:2a:66:d4:f1:1c:1f:9d:fe:f7:f1:7e:53: + 60:75:6e:6b:c0:ff:d3:c1:c7:c4:fb:85:4c:9c:94: + 16:d0:1c:29:9d:71:72:18:0a:01:6b:e4:06:c6:4b: + 78:c5:79:53:e2:17:b8:42:3e:3d:64:fe:37:29:6b: + 25:d9:02:d5:a6:a8:94:e5:91:72:86:6c:ce:bc:02: + 03:3f:0c:73:b4:7c:60:98:19:15:30:da:f8:55:dd: + 92:11:0a:6d:01:ab:07:e8:ad:e1:61:fb:6a:0c:c9: + 98:c3:8e:fa:3f:b9:98:d9:35:ac:dd:a5:cc:2f:3b: + 2d:1c:94:59:0e:47:05:48:d3:34:c8:8b:15:d8:49: + 5e:31:37:50:57:f3:6d:49:0a:e8:58:b9:ba:e8:f3: + d9:be:31:28:f1:3b:5e:2a:82:cd Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: - DNS:allsans, othername:, othername:, email:user@example.org, DNS:www.example.org, DirName:/C=XY/L=Castle Anthrax/O=Python Software Foundation/CN=dirname example, URI:https://www.python.org/, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, Registered ID:1.2.3.4.5 + DNS:allsans, othername: 1.2.3.4::some other identifier, othername: 1.3.6.1.5.2.2::, email:user@example.org, DNS:www.example.org, DirName:/C=XY/L=Castle Anthrax/O=Python Software Foundation/CN=dirname example, URI:https://www.python.org/, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, Registered ID:1.2.3.4.5 X509v3 Key Usage: critical Digital Signature, Key Encipherment X509v3 Extended Key Usage: @@ -90,81 +90,78 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - D4:F1:D8:23:E0:A7:E9:CA:12:45:A0:0D:03:C2:25:A6:E8:65:BC:EE + 73:0D:4E:1F:4A:EC:F2:53:0F:53:FC:85:6F:CF:82:CD:A3:E8:11:8D X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 70:77:d8:82:b0:f4:ab:de:84:ce:88:32:63:5e:23:0f:b6:58: - a2:b1:65:ff:12:22:0b:88:a6:fa:06:40:9a:e7:63:a7:5d:ae: - 94:c5:68:3c:4b:e9:95:34:01:75:24:df:9d:6e:9b:e4:ff:3f: - 61:97:29:7b:ab:34:2c:14:d3:01:d2:eb:fb:84:40:db:12:54: - 7e:7a:44:bc:08:eb:9f:e2:15:0b:11:4f:25:d2:56:51:95:ad: - 6d:ad:07:aa:6a:61:f9:39:d5:82:8c:45:31:9f:2a:ff:18:98: - 49:0c:bb:17:ad:d5:24:d3:d1:c7:c4:10:3e:c4:79:26:58:f4: - c5:de:82:16:c4:c3:c4:a7:a3:62:22:41:90:36:0f:bc:4c:fd: - 6a:18:22:f2:87:e9:07:db:b4:3d:65:00:e4:70:f9:d6:e5:a8: - a1:b9:c9:9d:e7:5d:78:aa:98:d5:f8:f4:fd:5c:d9:4c:d0:6d: - bf:87:71:d3:5b:ec:f4:bf:46:f9:c8:f8:10:c5:72:af:c3:15: - b9:c4:06:67:0b:3f:f6:f4:64:c5:27:74:c1:6b:00:37:da:ea: - 18:36:77:36:a7:3e:80:2e:5d:54:0f:01:df:ce:9e:97:dd:c9: - f2:8b:59:82:c5:65:31:c8:73:20:fd:24:23:25:d8:00:df:90: - 93:26:76:08:0a:06:a9:0e:d3:d3:4c:6f:ef:a7:fb:de:eb:2a: - 40:b9:e4:b1:44:0c:37:ca:c6:9e:44:4a:b4:7c:2c:40:52:35: - bb:b3:71:28:3d:35:fd:be:c9:4f:54:b3:99:c5:5f:84:38:fb: - 2b:fb:ea:dd:88:e8:9d:c1:9b:67:87:3d:79:7b:3d:7e:61:1f: - 70:3c:b7:c8:4c:17:a5:0c:a3:28:c7:ab:48:11:14:f7:98:7a: - da:4e:fb:91:76:89:0a:a6:c6:72:e0:96:d9:f1:80:ea:68:90: - 37:5c:c6:69:c7:d7:bc:c7:d1:ae:5b:a9:12:59:c6:e4:6c:61: - a9:8b:ba:51:b3:13 + Signature Value: + a5:3d:12:87:b9:3b:3e:9c:1c:59:fb:d5:38:22:49:61:f3:c3: + 11:53:4b:4e:63:af:f2:3d:ef:24:67:45:bc:74:5c:4a:65:c5: + b4:bb:fe:84:b8:b6:ca:7d:fc:aa:ff:80:ae:67:1f:cb:c3:cd: + 8f:f9:75:8c:f9:d3:3f:db:f6:81:d8:06:42:c3:5d:a9:1e:a3: + 81:7d:57:ac:97:d9:bd:c8:ce:1e:ec:74:d7:94:d5:df:b1:ad: + ce:84:14:5c:8c:45:a4:a8:eb:67:ab:16:57:61:15:86:ae:11: + 1e:b5:10:42:de:84:76:88:9b:37:12:aa:a6:77:42:75:b4:c0: + 04:b3:75:45:e0:d7:aa:34:e3:07:c5:ed:f8:4e:f0:39:99:1f: + 5b:d8:4e:0c:ad:64:6d:09:07:3f:e3:e1:9f:1b:65:07:96:59: + 9a:b5:f1:4d:c3:ec:f7:32:a1:05:94:d1:0b:18:54:3c:67:cf: + 38:f5:2b:ec:cb:bd:79:be:f7:1b:b7:71:3f:c6:44:80:7f:00: + dc:3d:a0:07:c0:b5:1c:fb:52:f6:a0:f8:92:c6:c6:73:07:c5: + ca:0b:04:7c:55:51:e3:ba:93:32:17:bd:61:ae:cf:13:e4:5e: + 03:b2:51:11:c7:68:f5:08:b6:0e:57:49:11:3c:e3:f4:0e:e1: + 96:20:44:28:31:94:11:44:50:cf:17:70:8d:9c:14:c5:ed:94: + 4d:ba:94:9b:db:8b:9f:55:a1:5e:0a:90:bb:a0:0e:0d:3b:a0: + dd:4d:47:d1:cf:d0:47:6b:ff:6f:af:e4:83:40:73:e6:3a:59: + 40:bd:4f:3a:21:22:63:27:5d:02:26:67:89:1d:2f:19:c5:77: + e6:b5:67:4a:dd:b5:0e:de:46:34:57:c7:03:84:5d:cd:34:08: + 8f:47:c9:4d:7f:04:c0:4f:ff:68:52:bb:ae:84:0e:54:ce:5c: + 27:71:0f:a2:3f:9f:7a:49:a8:fa:0a:45:cf:96:42:a7:65:23: + b9:3e:40:eb:46:7b -----BEGIN CERTIFICATE----- -MIIHDTCCBXWgAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIHDzCCBXegAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2Fs -bHNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDBGvj+Uy/VUyTR -mmIA1UEENThh0+pWODcvvUlkeIo+XTJ3FhF4/RVjImDHjozl28Xf2TzKnvQJa1KC -pqa7fr8cL9QMwk4pH+S4ulxOu02Bl3Yafx2oJVUML37vciJg+zkzPx1k3tXFjXkr -LGjZwOoufBC3AmPuq2xHFBzHrvp5/DIRH2slQFM9fpVZzN77gYyzxba0wCfCPpCf -eJFRyYKW8c7MXrwnM82YtE7Rlnf227EkCdMNaSeZLUIxeVpcnScqZl0SIbR3YEiV -0LPFkx0wJFm8qUEFU/h+0jamgy/ON+11nqmMlp3BjNi/JTVsa7N7A3dvdHC7VVlr -WnUgU6MoSniyL6ijpucyHtZzK2mJy0sHR8PadHKow0O423/5N8GKTSOvaGMXTjAe -OGs+9/P1ZYo3IjjQPz/NV3QlhK8zRqxF3cW0ekHHkT+/jZjCvSKm6mdbMQunKE1W -+dokAc815pb48Mzf1eWKd/7UyUf7CXussyAaJ3clpaK1sbbn9m0CAwEAAaOCAt4w -ggLaMIIBMAYDVR0RBIIBJzCCASOCB2FsbHNhbnOgHgYDKgMEoBcMFXNvbWUgb3Ro -ZXIgaWRlbnRpZmllcqA1BgYrBgEFAgKgKzApoBAbDktFUkJFUk9TLlJFQUxNoRUw -E6ADAgEBoQwwChsIdXNlcm5hbWWBEHVzZXJAZXhhbXBsZS5vcmeCD3d3dy5leGFt -cGxlLm9yZ6RnMGUxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJh -eDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xGDAWBgNVBAMM -D2Rpcm5hbWUgZXhhbXBsZYYXaHR0cHM6Ly93d3cucHl0aG9uLm9yZy+HBH8AAAGH -EAAAAAAAAAAAAAAAAAAAAAGIBCoDBAUwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW -MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTU -8dgj4KfpyhJFoA0DwiWm6GW87jB9BgNVHSMEdjB0gBSziqCiunHxqCR51KRbJTYV -HknIzaFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy -ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp -UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w -eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo -dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E -PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y -ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAcHfYgrD0q96EzogyY14j -D7ZYorFl/xIiC4im+gZAmudjp12ulMVoPEvplTQBdSTfnW6b5P8/YZcpe6s0LBTT -AdLr+4RA2xJUfnpEvAjrn+IVCxFPJdJWUZWtba0Hqmph+TnVgoxFMZ8q/xiYSQy7 -F63VJNPRx8QQPsR5Jlj0xd6CFsTDxKejYiJBkDYPvEz9ahgi8ofpB9u0PWUA5HD5 -1uWoobnJneddeKqY1fj0/VzZTNBtv4dx01vs9L9G+cj4EMVyr8MVucQGZws/9vRk -xSd0wWsAN9rqGDZ3Nqc+gC5dVA8B386el93J8otZgsVlMchzIP0kIyXYAN+QkyZ2 -CAoGqQ7T00xv76f73usqQLnksUQMN8rGnkRKtHwsQFI1u7NxKD01/b7JT1SzmcVf -hDj7K/vq3YjoncGbZ4c9eXs9fmEfcDy3yEwXpQyjKMerSBEU95h62k77kXaJCqbG -cuCW2fGA6miQN1zGacfXvMfRrlupElnG5GxhqYu6UbMT +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +YWxsc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAJgRj6UjBL/D +6ABTk9/FJ4viowXlO0WSuJ8vShBoKNYVvyN5fZFvW0po5vKtiZNRFVZsyQjUsk5B +Zi0aUA2jaa9/hUPXBln/H9ObKXF7cDuK1JNmBm4Zcse0U/3CTvJLjoGVVw2rdmg6 +LuSTw3FOJSqCyYuDc2P4h+Tw4jtzUFw/alrXBk7+Xc9Naq0VIuwUS84rk3pk+ogs +BSLm8SQ6IV5/FUce8Pg3chZIWBLCIOL0qGe11HBr5rGxnGOr+jvQWNA7mrlC7Kb1 +oOqqjjs8zKQRvWB1MFArWY+WIwqwuLnPJwj/KmbU8Rwfnf738X5TYHVua8D/08HH +xPuFTJyUFtAcKZ1xchgKAWvkBsZLeMV5U+IXuEI+PWT+NylrJdkC1aaolOWRcoZs +zrwCAz8Mc7R8YJgZFTDa+FXdkhEKbQGrB+it4WH7agzJmMOO+j+5mNk1rN2lzC87 +LRyUWQ5HBUjTNMiLFdhJXjE3UFfzbUkK6Fi5uujz2b4xKPE7XiqCzQIDAQABo4IC +3jCCAtowggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBv +dGhlciBpZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2h +FTAToAMCAQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4 +YW1wbGUub3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRo +cmF4MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UE +AwwPZGlybmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAA +AYcQAAAAAAAAAAAAAAAAAAAAAYgEKgMEBTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0l +BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE +FHMNTh9K7PJTD1P8hW/Pgs2j6BGNMH0GA1UdIwR2MHSAFMAKKyhD3l/JfUflR5s2 +8mWMZzvioVGkTzBNMQswCQYDVQQGEwJYWTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3 +YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1jYS1zZXJ2ZXKCCQDLLYCZ +WmlSWzCBgwYIKwYBBQUHAQEEdzB1MDwGCCsGAQUFBzAChjBodHRwOi8vdGVzdGNh +LnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9weWNhY2VydC5jZXIwNQYIKwYBBQUHMAGG +KWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL29jc3AvMEMGA1Ud +HwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNh +L3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3DQEBCwUAA4IBgQClPRKHuTs+nBxZ+9U4 +Iklh88MRU0tOY6/yPe8kZ0W8dFxKZcW0u/6EuLbKffyq/4CuZx/Lw82P+XWM+dM/ +2/aB2AZCw12pHqOBfVesl9m9yM4e7HTXlNXfsa3OhBRcjEWkqOtnqxZXYRWGrhEe +tRBC3oR2iJs3Eqqmd0J1tMAEs3VF4NeqNOMHxe34TvA5mR9b2E4MrWRtCQc/4+Gf +G2UHllmatfFNw+z3MqEFlNELGFQ8Z8849Svsy715vvcbt3E/xkSAfwDcPaAHwLUc ++1L2oPiSxsZzB8XKCwR8VVHjupMyF71hrs8T5F4DslERx2j1CLYOV0kRPOP0DuGW +IEQoMZQRRFDPF3CNnBTF7ZRNupSb24ufVaFeCpC7oA4NO6DdTUfRz9BHa/9vr+SD +QHPmOllAvU86ISJjJ10CJmeJHS8ZxXfmtWdK3bUO3kY0V8cDhF3NNAiPR8lNfwTA +T/9oUruuhA5UzlwncQ+iP596Saj6CkXPlkKnZSO5PkDrRns= -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/b1930218.0 b/Lib/test/certdata/capath/b1930218.0 index 941d7919f8033e..6d773ee5b32432 100644 --- a/Lib/test/certdata/capath/b1930218.0 +++ b/Lib/test/certdata/capath/b1930218.0 @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBALGE009cBICRT4JJujAL9+jL+RTvPZ8LPwpi -/BsgpSDRYF+HWh8W0e2XcKbaGwMsfqBbPE4vFn4OiSmJ4RANONpqd183E7Moj3tc -dq2e6NP1nvWDqhAHjeZRmPB8DVLyDCEe2LmZJqklAye7XKsuMyei1iOog4dEKZ+X -tSRv17kK/Sjuu/tBWOodmd1EhquYvhzcy6mJHTZcqehHtfRSSKq1pGfvPtfi0zPe -mCnYerBZXOexDsz9n+v21ToOC8/+Cz2iv0UYzpTnqVVgiNTYhFB5BS5BA3SuZyb2 -WxIImM4Kl+0BD4lPF1z6Ph01JEeSMr/3pBgrPNBImeGizaPMUFMgtcbjZoV7VxDs -M0/Bd+cbfoHGxPNFIMCR3RN2ewOv9naOooNjV91jvLtaHBdSitYGSMwPx9NP6Noi -bIb5TlymKQc72FZMWbMgSQd7lITPK8McGk6HZJK6QuHmrX0d9lSQbyvps8xLKzMm -I/1lwDzwea3JwYHvNwTgJz6w7hW+UQIDAQABo1AwTjAdBgNVHQ4EFgQUs4qgorpx -8agkedSkWyU2FR5JyM0wHwYDVR0jBBgwFoAUs4qgorpx8agkedSkWyU2FR5JyM0w -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAYEAazIv5wUY6lzJlfTgwgxB -XxoKlcnHfQXuilYpNVBAt/6fe1scw2kvoMvSuJEvUBli9ycYbZV7UxYVolrcFOP7 -sTKpadumM0c8ux/S3HD5ai4M2Ixt5V0dQzxOkd6gyNqgSw6dXrYPSknwe7ZTnv01 -FFvjTbQYpjZh6I8zm9QF+VRm3+DLGKNO3BeooLPBqPTWncp/aFMa15Xa6NOeSABx -lZkRB8+WwH3OfTDoT+GDFjOh/1mbPkznOjgBnw9nTP0ti0rUAUY3M+gTaxWpHWh2 -RaKCM2kmMGAFyI+9tHWrvnqLSGhwQLQbUcXmeq1rT9sXwGBnLmNhmyxImbh2RaCe -zO8zHlBOq3LDZciyebM1gyF404tsOhjoZTI5uMCdcS81NorAF2LYiz7hIhgrTGOm -Dp0K+qtbNfuIkXdMjYydqc/8q8LmWgV7fgRuOc+Tzmc7esuvtjbh+3FkRdSm8M7v -dQSZaZrliAoQAnSJ7HWERIBI38H36TfOzpKSXIkiCHMf +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/ceff1710.0 b/Lib/test/certdata/capath/ceff1710.0 index 941d7919f8033e..6d773ee5b32432 100644 --- a/Lib/test/certdata/capath/ceff1710.0 +++ b/Lib/test/certdata/capath/ceff1710.0 @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBALGE009cBICRT4JJujAL9+jL+RTvPZ8LPwpi -/BsgpSDRYF+HWh8W0e2XcKbaGwMsfqBbPE4vFn4OiSmJ4RANONpqd183E7Moj3tc -dq2e6NP1nvWDqhAHjeZRmPB8DVLyDCEe2LmZJqklAye7XKsuMyei1iOog4dEKZ+X -tSRv17kK/Sjuu/tBWOodmd1EhquYvhzcy6mJHTZcqehHtfRSSKq1pGfvPtfi0zPe -mCnYerBZXOexDsz9n+v21ToOC8/+Cz2iv0UYzpTnqVVgiNTYhFB5BS5BA3SuZyb2 -WxIImM4Kl+0BD4lPF1z6Ph01JEeSMr/3pBgrPNBImeGizaPMUFMgtcbjZoV7VxDs -M0/Bd+cbfoHGxPNFIMCR3RN2ewOv9naOooNjV91jvLtaHBdSitYGSMwPx9NP6Noi -bIb5TlymKQc72FZMWbMgSQd7lITPK8McGk6HZJK6QuHmrX0d9lSQbyvps8xLKzMm -I/1lwDzwea3JwYHvNwTgJz6w7hW+UQIDAQABo1AwTjAdBgNVHQ4EFgQUs4qgorpx -8agkedSkWyU2FR5JyM0wHwYDVR0jBBgwFoAUs4qgorpx8agkedSkWyU2FR5JyM0w -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAYEAazIv5wUY6lzJlfTgwgxB -XxoKlcnHfQXuilYpNVBAt/6fe1scw2kvoMvSuJEvUBli9ycYbZV7UxYVolrcFOP7 -sTKpadumM0c8ux/S3HD5ai4M2Ixt5V0dQzxOkd6gyNqgSw6dXrYPSknwe7ZTnv01 -FFvjTbQYpjZh6I8zm9QF+VRm3+DLGKNO3BeooLPBqPTWncp/aFMa15Xa6NOeSABx -lZkRB8+WwH3OfTDoT+GDFjOh/1mbPkznOjgBnw9nTP0ti0rUAUY3M+gTaxWpHWh2 -RaKCM2kmMGAFyI+9tHWrvnqLSGhwQLQbUcXmeq1rT9sXwGBnLmNhmyxImbh2RaCe -zO8zHlBOq3LDZciyebM1gyF404tsOhjoZTI5uMCdcS81NorAF2LYiz7hIhgrTGOm -Dp0K+qtbNfuIkXdMjYydqc/8q8LmWgV7fgRuOc+Tzmc7esuvtjbh+3FkRdSm8M7v -dQSZaZrliAoQAnSJ7HWERIBI38H36TfOzpKSXIkiCHMf +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/cert3.pem b/Lib/test/certdata/cert3.pem new file mode 100644 index 00000000000000..a11dc614657d2f --- /dev/null +++ b/Lib/test/certdata/cert3.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/idnsans.pem b/Lib/test/certdata/idnsans.pem index cbcac7818ddc67..ebd7250cb4eae9 100644 --- a/Lib/test/certdata/idnsans.pem +++ b/Lib/test/certdata/idnsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQC8sqplTuHuLjbW -TL5SL2D1fw9U6WQzLVAF5gsyhd5lr2FpfYwjrob5Mav91aOLbJRTvoNyXsJ26FPS -0RycRGXbomcIEJxXGy9aI+0MLYBt1G5mgqCH+HcVCwPzCNlhVnTwvpgA7y8zs3+6 -ezZAPWkF0yWOMYLtTcq9A5GWeavt5VMgm1KZF3gO4k58oPyk3Ae9D0LAaYsX6DFi -BYx41eUR5UbSb5IYXaDd8d6jqW/jnYhgc6Cxkv1gTJFn87V5lrG0vYMSRUtWDQ9Y -Jh/EKAxjGw7AeY429p6TE4UoJhDmoFYR2NLvawhNIplxol/v0fs0veFQjI/UsTD8 -2tRfnYL4IX8szhLsE5/5Iq8aiLHjVbIMwmDYAa0P63Ap2kf1biSn9mpDL8lQazSo -yr8xzIq2QS5HMvGbeMAmS0ih10Zx84uVmkWlavgvtSflw8K/ZXT9c70rZp/TdBGY -95cOFsbg5U/20M/Llpis9tcBCaoVaYSFupatrP+p8y19qP2nebsCAwEAAQKCAYEA -uaYWWwHW6pzxOrnabcVLYX0WunW9LVShbIw97AElI2n/LuhkXh6xkK48BsqP0vaK -oDHJ5VYxgQdmoP03Zs8sX4BSWe7twg1u8wJxkA+cUXI1BAn0opHjpwJlalEEfe2v -s8PwjMrF59nsCq56W42PrDlms5UmuQ5WLsw6Co++hZmfxW7LPu+GIS6qBZfluNT5 -kBpZlDDCtkyteUD4SVI3wvmOSi+Wzv4e7P2wC9kByjENIcfhC5QQURRD4sA1hWCp -2SThYWqJOCEc2SvGgoqgTRaJuQ2aVG9qrntXt0N4V+WdJWXBK0jedkB2flLve1fR -KmDYuc9k/c1svmS3Y+iZohBha9H8jpuJmXYBxxg1iNg9m7qkfg8F8wxCYLQKB+U6 -tjRS7by+jSE08On7mpDDhJORnlh+rfEuWPPwAKQpLpdp76KDTvR++GvfOMUiOrFM -e9s5aXp+vcgkSSqYvigE+sFpCjQWwkGBkMdT16Pf9CzhQaM08YuLnzfLEYgLFw6R -AoHBAN5NQINBmlq/cptGSru66kfecqHfI7xHnnGWKAkto/B1x7Crrgs4Tk5b4vaA -JmAqatt5P1e7zco7uAXXebY5VURuH/30TlkuaB+oGFp0OMw6165n8RVPT2ZaDViK -ssJ9LT8fJ+23TWCCT2Z1zUlM/NnHAMjKOVsJK3/KEkVvlc7ROC7uVooc78AsQehg -zpL3GBYEeBukT8aNUMqUlesCsIs/dQHW7DzQL2xGkQagm5/PDsxaCsT7ynA8eL3X -TW+IXwKBwQDZTV3TaG6wqtL8y2DR0lN5jY/eYayX4e18iZ+XEZVTntPdVVyJIE4d -0A5ZfcILb9WE8R21iptROYSjcH/05j+3fQMJ1WAK0sNfGTUNNT3jYU8YzLvos+wW -G8E+mNMpFPWNvLV5Qrl4VvoifGh8AMvplUEz8uAzGJbXbRxUPcmjth2ph8zULEDn -/+o4OcT3gh1bp+HCqch0OuiJRn9qNUpsJG5GMm5FtjBjZM97ucZ1/0DaWl3JUxUN -/pueo3J9vCUCgcBg2Fjdlcvv8u2z1aijJmgATVm1SWfhE3ZkV50zem2sSTNotTJK -cwoyOveimeueA3ywBp9g0lFx5Bhkex3sFAggmrVXRoKHeZ8lA28woOdJmezybxfp -R7b4iQy9YRdFgZEfqawUdMHB5KNAqNt5LpANNBQUZX0dOt53eooBM/6Yri8CyxRq -cPbFysIfwWTdQ8Z7eRD2Qdv7TP9AcgDp9C8DSu7nkUEzsSKn0gpGT9vcgDEbN7Lv -ZB4qTT3wvoZeq5MCgcBIG18eDtJkN1sp3Yb0OTnP5QSvg3PVNngq0jQt2fzWMacW -FARP0HN7exW35n4kc2jD44q7OhJOAqsb3PHo3xqXlZkTg0WKceO4w9GR32/46spn -bVCRaFrX/z/BuM6hHD5bWRpS8aw/3YTFOsklFNKVYRyw01BIREmRlLhIz/QAKidv -oQt8AG9NTON44tqUUw3Q40WL5fEJeJ6/JrCTGrnmZrRdANEMuucVpFchNEVB1IC9 -tCzY6IPdD/atzojoZi0CgcB2x9oWLjJ0XJIp2pMAb8nCMVjkKrznKFjZbDm8EQBs -ou7pM2zkO3VRcWT1BXQocinJsjQqjQiTawP6IN2FQgT0d89V+pwd+jdvpdildQhP -1/6SErVRZV//oopKTsC6TIBL/EmW1TkP3ulQIZs8YklFgybeHdDyNFi+VgPXkVGe -IHp0nEzrui9q0YJsjHfFHBeGyzDSfbiBYiF7Auk66gYZbXufebP/LZNG/FIamPP3 -rwYIeeV1IVwk9tPBw6fGwrs= +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDeGGysqNqsz64+ +xsN+CUZDfCgPPhzUK0lDUlkEkp0ZeFeQm3+po2Y82gvJIValO5ChiXjPxSgJwoto +SHe4QZ8fWAlkX1d19A8xjjetx1K3dqZstqe3mPP7HQvfJlRkmAtVhFaUj1+SUx9K +s4HB4Lk2hqKUoq3TlRMpeSfJTXaHThVP7KZPWBzfM7pdpGYnznBpxgfymNrpTcXN +CjeR6lUlj/TExHiYLj4E0aODR/SUz0tBNdCQRcRDw+OyREe6Rk32Krl34Pr1dNKe +ER0tulhM2wRFsqw0a8vNs13I30BH4aXAr5bGgLCF2tXbP3KoeoR/GkQSWV14KrhT +xhKD3QEJ8gHjoHYKr4IdcFYcTakbI9E3THsQjQTUMJ/Xygza7OOr6pNPKY10tYfp +7WhyyY7cB/C/rHnUY0f58IZlRdMYgcLCUl7wzUxSVEMyr9Ary8XZZlxuO629k2ud +AIGwIi2TVq+UK0kNPZyGicGHMeEhoNc3YDMB/zYeqMr4KryI+mkCAwEAAQKCAYA4 +EENSnHdC+1P5bdRIew/dFjjIjD3bwyeF0oI9GMOGe+3ix5YE3QYAY2xpM7y7Dhu2 +40x3akXunMjzJKPwA8SmtWL9juG1mUvCjyt39yJmxJFDTSJuQrKIF694/6R7FjR6 +PGNcsgqGlewGv+SH6/HlFTxyN9SYXf/NztMfyimbAzd3Cv56dfwnzdeELu1IrCCN +WtuDvlk4XpUJasRXVadzyXCYwR3OEJJARik4CRBxBhjxl6OT38Co+IiAZiMTHw6z +jXeRuvyxXyYB76BQs9uCt05c5JV1I2OT1aKdpC0EbHlJEzJhmxrpdfBeoVdJUVVg +4b0QoocZE89KXOVmjmIKExxmjOjxH6o6qfxvDQWLspADD4D+69Zqjj6hFAfn3EUW +hcq3RHwuUheCE6i2xRtWPnA1ygqWN9mdNojX1+EXIv0Dh29kFWHpMUC29WWm9JA6 +Arx2UrtvoWGYotKxTDxZ0SV5ws/LzGjvMaT/dbXZHjywnB7sPfpcE9b91p8Q6VcC +gcEA+Wr44ZzyMirF9FYYA+KEBL91khmffhXzCDkRXqd1j2BCQWETWqvrUuAx+kwh +FZ5AABdip96w8Eaj8XBWb1c81mJUbOimbcqiV6B4GokCI/7u3FkOMPFRNBq41R/M +43JH6QLHNxhXI4OlZsxv5WHmR5N88kXEHhofBoSn14EHtdkK/Hu8ly1Wux4/da83 +2wanwlysmQkk1CnpJVT0st7HrKHP8YWDbcBvH+xysHOIOs22q3gvAF2rsdORidpR +fZnHAoHBAOP03brYpLU1xa5LHQz/B7uWfgJm3l7wRQ627YOfUBe4IjnH2y/kNKjv +ROVjaooB2gtya6Z6xVuY1m4LzUfYuRN1FdX1Np4NjvfgPNFViX+0HoxyAytXn++V +Iy1tCpL6X43Iqrj/VxBcX09vIpbsprW2d3dcIo/45abkAO6Uoa6HUmZoqrqrBA7b +LnuhEfBAyyN42OiXbHDqif7XAWC30yY6k0FfDaOSLiBkAOKZ9lkf7ei3eNr4SdOa +Tf5a/RVKTwKBwB7YfOkiCM3tfkfGcffhBqSzrO2hn5jvS/wjWqOTIDXYGLmPMN6Q +zmyUb3nd+mV7Cb05JylNoCJHCjVsyDPC3TJCPOCvMQ349nTR0qitcwdSmuXDWb7x +yTIhb+Rjp2olkwEdJ9gHeZdZy5XYCKqcnecSNWyc9jEm19lthHhha7uwmOw6vUsQ +/13q0rxSLB05SHwADBRtDhHzEPNd+1k3tggChv3+ng9vsg6HpnNuBlYHZOT12xI3 +g2ldme0rg9J9twKBwA4R1gGrT3czy3C3iCJ+Ny732e0yBjWb5NdEqSI/mgTsw4gH +ctrg3fMzWXBDE5dTB+8+77AF0dqWc121csUldj7iMifTi7xzn8hi2b4d5m+wYVZP +zyxEq0VxUguCuG1b8Lvij879S5Vh7iwL8vmXv65lhbgjQqraNOp5FimjmNsZ1Rcn +DKqa1ZRJKPROe7n1ddRJqDGq7vGFOGE3Sgl7Lxgj82TMhh37bsdnBLr3v8G+e8Oq +V1ZEjuH1myzA1vASdwKBwQDcMHbUKeoJUxyIlABFsWxhm0MKwL5ZRgo7rhH01rF2 +TF3mbAEHHsAEfkHgBVRooifbxglxUDy1olIyRk+kqs1gmXZA0UmE9lUqoX5+n/j8 +pgbCc3sV53x7JJ4BCnAn49XTOnTi2ILeQ3MvTqcj+sDKo+0T4Zq0r5d8ZrMikKbO +HJ4MMBGth645QKhgpb0XgltvSn8aceS4uTxIKrbNpZVnWj/VzuOYoXQmKnZko9p1 +dbyjt6PMeVXWj0tz8FB2DGE= -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=idnsans Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:bc:b2:aa:65:4e:e1:ee:2e:36:d6:4c:be:52:2f: - 60:f5:7f:0f:54:e9:64:33:2d:50:05:e6:0b:32:85: - de:65:af:61:69:7d:8c:23:ae:86:f9:31:ab:fd:d5: - a3:8b:6c:94:53:be:83:72:5e:c2:76:e8:53:d2:d1: - 1c:9c:44:65:db:a2:67:08:10:9c:57:1b:2f:5a:23: - ed:0c:2d:80:6d:d4:6e:66:82:a0:87:f8:77:15:0b: - 03:f3:08:d9:61:56:74:f0:be:98:00:ef:2f:33:b3: - 7f:ba:7b:36:40:3d:69:05:d3:25:8e:31:82:ed:4d: - ca:bd:03:91:96:79:ab:ed:e5:53:20:9b:52:99:17: - 78:0e:e2:4e:7c:a0:fc:a4:dc:07:bd:0f:42:c0:69: - 8b:17:e8:31:62:05:8c:78:d5:e5:11:e5:46:d2:6f: - 92:18:5d:a0:dd:f1:de:a3:a9:6f:e3:9d:88:60:73: - a0:b1:92:fd:60:4c:91:67:f3:b5:79:96:b1:b4:bd: - 83:12:45:4b:56:0d:0f:58:26:1f:c4:28:0c:63:1b: - 0e:c0:79:8e:36:f6:9e:93:13:85:28:26:10:e6:a0: - 56:11:d8:d2:ef:6b:08:4d:22:99:71:a2:5f:ef:d1: - fb:34:bd:e1:50:8c:8f:d4:b1:30:fc:da:d4:5f:9d: - 82:f8:21:7f:2c:ce:12:ec:13:9f:f9:22:af:1a:88: - b1:e3:55:b2:0c:c2:60:d8:01:ad:0f:eb:70:29:da: - 47:f5:6e:24:a7:f6:6a:43:2f:c9:50:6b:34:a8:ca: - bf:31:cc:8a:b6:41:2e:47:32:f1:9b:78:c0:26:4b: - 48:a1:d7:46:71:f3:8b:95:9a:45:a5:6a:f8:2f:b5: - 27:e5:c3:c2:bf:65:74:fd:73:bd:2b:66:9f:d3:74: - 11:98:f7:97:0e:16:c6:e0:e5:4f:f6:d0:cf:cb:96: - 98:ac:f6:d7:01:09:aa:15:69:84:85:ba:96:ad:ac: - ff:a9:f3:2d:7d:a8:fd:a7:79:bb + 00:de:18:6c:ac:a8:da:ac:cf:ae:3e:c6:c3:7e:09: + 46:43:7c:28:0f:3e:1c:d4:2b:49:43:52:59:04:92: + 9d:19:78:57:90:9b:7f:a9:a3:66:3c:da:0b:c9:21: + 56:a5:3b:90:a1:89:78:cf:c5:28:09:c2:8b:68:48: + 77:b8:41:9f:1f:58:09:64:5f:57:75:f4:0f:31:8e: + 37:ad:c7:52:b7:76:a6:6c:b6:a7:b7:98:f3:fb:1d: + 0b:df:26:54:64:98:0b:55:84:56:94:8f:5f:92:53: + 1f:4a:b3:81:c1:e0:b9:36:86:a2:94:a2:ad:d3:95: + 13:29:79:27:c9:4d:76:87:4e:15:4f:ec:a6:4f:58: + 1c:df:33:ba:5d:a4:66:27:ce:70:69:c6:07:f2:98: + da:e9:4d:c5:cd:0a:37:91:ea:55:25:8f:f4:c4:c4: + 78:98:2e:3e:04:d1:a3:83:47:f4:94:cf:4b:41:35: + d0:90:45:c4:43:c3:e3:b2:44:47:ba:46:4d:f6:2a: + b9:77:e0:fa:f5:74:d2:9e:11:1d:2d:ba:58:4c:db: + 04:45:b2:ac:34:6b:cb:cd:b3:5d:c8:df:40:47:e1: + a5:c0:af:96:c6:80:b0:85:da:d5:db:3f:72:a8:7a: + 84:7f:1a:44:12:59:5d:78:2a:b8:53:c6:12:83:dd: + 01:09:f2:01:e3:a0:76:0a:af:82:1d:70:56:1c:4d: + a9:1b:23:d1:37:4c:7b:10:8d:04:d4:30:9f:d7:ca: + 0c:da:ec:e3:ab:ea:93:4f:29:8d:74:b5:87:e9:ed: + 68:72:c9:8e:dc:07:f0:bf:ac:79:d4:63:47:f9:f0: + 86:65:45:d3:18:81:c2:c2:52:5e:f0:cd:4c:52:54: + 43:32:af:d0:2b:cb:c5:d9:66:5c:6e:3b:ad:bd:93: + 6b:9d:00:81:b0:22:2d:93:56:af:94:2b:49:0d:3d: + 9c:86:89:c1:87:31:e1:21:a0:d7:37:60:33:01:ff: + 36:1e:a8:ca:f8:2a:bc:88:fa:69 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,80 +90,77 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 5C:BE:18:7F:7B:3F:CE:99:66:80:79:53:4B:DD:33:1B:42:A5:7E:00 + C8:06:99:B7:E8:8F:EC:4F:3D:5C:89:6A:06:F5:77:2E:B0:E0:6A:9E X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 5d:7a:f8:81:e0:a7:c1:3f:39:eb:d3:52:2c:e1:cb:4d:29:b3: - 77:18:17:18:9e:12:fc:11:cc:3c:49:cb:6b:f4:4d:6c:b8:d2: - f4:e9:37:f8:6b:ed:f5:d7:f1:eb:5a:41:04:c7:f3:8c:da:e1: - 05:8e:ae:58:71:d9:01:8a:32:46:b2:dd:95:46:e1:ce:82:04: - fa:0b:1c:29:75:07:85:ce:cd:59:d4:cc:f3:56:b3:72:4d:cb: - 90:0f:ce:02:21:ce:5d:17:84:96:7f:6a:00:57:42:b7:24:5b: - 07:25:1e:77:a8:9d:da:41:09:8e:29:79:b4:b0:a1:45:c8:70: - ae:2c:86:24:ae:3d:9a:74:a7:04:78:d6:1f:1b:17:c5:c1:6d: - b1:1a:fd:f4:50:2e:61:16:84:89:d0:42:3f:b6:bf:bd:52:bd: - c8:3e:8e:87:b4:f0:bd:ad:c7:51:65:2f:77:e8:69:79:0e:03: - 63:89:e7:70:ad:c8:d1:2f:1a:a5:06:d2:90:db:7c:07:35:9a: - 0b:0e:85:87:d1:70:17:a7:88:0f:c6:b5:9c:88:00:fa:f9:b2: - 0a:19:5a:4b:8d:91:12:51:5e:0e:c1:d8:9e:02:78:d0:2d:24: - 09:fe:d4:97:3c:cb:a0:1f:9a:ab:f7:0f:e2:fa:64:23:4e:53: - 0a:15:3e:f5:04:01:86:29:8b:8e:24:40:2f:b1:90:87:5c:3b: - 7b:a7:4c:06:af:c3:90:7f:e9:c6:56:42:61:15:2c:83:f1:7c: - 4f:89:17:f3:a0:11:34:3f:8d:af:75:34:60:1e:e0:f2:f3:02: - e7:aa:b3:f7:9f:1c:f8:69:f4:fe:da:57:6e:1b:95:53:70:cd: - ed:b6:bb:2a:84:eb:ab:c3:a9:b4:d5:15:a0:b2:cc:81:2d:f1: - 56:c1:54:9b:5f:14:4c:5f:ad:5f:f5:06:ee:22:60:45:e4:50: - 35:64:ac:ac:ca:4a:bf:86:78:f8:53:2d:17:d8:e8:84:c8:07: - a4:c2:29:76:c7:1f + Signature Value: + 40:d1:6d:8e:a2:0b:91:4b:a8:c4:08:d0:f3:f9:8b:d0:a3:0b: + dc:00:22:8c:f1:2e:2b:e5:e6:b4:6e:ce:9d:cf:59:32:66:6c: + bb:0e:3b:1d:9c:05:d2:eb:a6:29:f8:74:4f:dc:83:3b:32:5a: + 2c:67:86:61:25:bc:bd:19:eb:20:c6:30:69:0e:4c:b2:e3:18: + ca:9e:fe:40:bc:1c:ad:8b:03:f5:04:be:90:ce:27:27:2f:83: + 14:57:8d:4f:a0:db:46:ce:e0:7d:e2:cf:7d:ea:0c:fd:8d:00: + 27:0a:db:0d:5f:e7:1e:52:25:1f:64:b9:30:5f:07:1a:10:a3: + 69:35:0e:dc:f8:23:f7:34:07:ce:c8:92:94:39:4d:d5:c3:ab: + 33:aa:f9:67:be:66:18:ac:67:14:5f:93:5f:68:48:04:ed:1e: + c9:74:28:b2:47:34:49:11:e4:7b:38:32:e5:dc:40:13:b4:69: + 75:39:43:db:7c:4a:f0:2b:94:cd:01:ba:4d:9b:9e:68:b3:ee: + 03:9e:7f:9c:0c:cf:9c:5c:cb:d4:33:d5:f0:e3:21:54:9a:13: + 6f:eb:1a:0f:f3:8b:e8:ef:eb:34:ba:09:77:39:2a:8a:4b:e1: + 7e:9f:b5:05:be:95:b6:92:5d:4c:35:47:38:64:38:5e:27:b8: + f9:34:94:2f:57:16:b0:f5:6a:21:3f:09:34:b9:dd:f8:d1:47: + 2c:c7:5e:7f:63:49:f4:5b:f4:d9:ea:66:fc:aa:64:27:f0:72: + d7:94:6f:86:0f:e7:3b:b3:d4:d9:30:67:b8:a2:c3:f7:4d:07: + 44:b3:70:67:dd:b1:21:ac:7c:2a:04:7b:2c:1d:df:0b:82:a9: + fb:df:88:72:47:1c:f5:5d:a3:f7:52:22:2d:ea:f4:2a:45:4f: + 9b:9d:63:95:59:f3:79:05:2b:f1:5b:3b:62:71:69:90:30:d7: + 7a:b2:c8:ec:68:e5:94:bb:97:00:d0:95:a7:fd:04:6c:f7:8b: + 28:c1:96:9b:6a:94 -----BEGIN CERTIFICATE----- -MIIGvTCCBSWgAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIGvzCCBSegAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2lk -bnNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC8sqplTuHuLjbW -TL5SL2D1fw9U6WQzLVAF5gsyhd5lr2FpfYwjrob5Mav91aOLbJRTvoNyXsJ26FPS -0RycRGXbomcIEJxXGy9aI+0MLYBt1G5mgqCH+HcVCwPzCNlhVnTwvpgA7y8zs3+6 -ezZAPWkF0yWOMYLtTcq9A5GWeavt5VMgm1KZF3gO4k58oPyk3Ae9D0LAaYsX6DFi -BYx41eUR5UbSb5IYXaDd8d6jqW/jnYhgc6Cxkv1gTJFn87V5lrG0vYMSRUtWDQ9Y -Jh/EKAxjGw7AeY429p6TE4UoJhDmoFYR2NLvawhNIplxol/v0fs0veFQjI/UsTD8 -2tRfnYL4IX8szhLsE5/5Iq8aiLHjVbIMwmDYAa0P63Ap2kf1biSn9mpDL8lQazSo -yr8xzIq2QS5HMvGbeMAmS0ih10Zx84uVmkWlavgvtSflw8K/ZXT9c70rZp/TdBGY -95cOFsbg5U/20M/Llpis9tcBCaoVaYSFupatrP+p8y19qP2nebsCAwEAAaOCAo4w -ggKKMIHhBgNVHREEgdkwgdaCB2lkbnNhbnOCH3huLS1rbmlnLTVxYS5pZG4ucHl0 -aG9udGVzdC5uZXSCLnhuLS1rbmlnc2dzc2NoZW4tbGNiMHcuaWRuYTIwMDMucHl0 -aG9udGVzdC5uZXSCLnhuLS1rbmlnc2djaGVuLWI0YTNkdW4uaWRuYTIwMDgucHl0 -aG9udGVzdC5uZXSCJHhuLS1ueGFzbXE2Yi5pZG5hMjAwMy5weXRob250ZXN0Lm5l -dIIkeG4tLW54YXNtbTFjLmlkbmEyMDA4LnB5dGhvbnRlc3QubmV0MA4GA1UdDwEB -/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/ -BAIwADAdBgNVHQ4EFgQUXL4Yf3s/zplmgHlTS90zG0KlfgAwfQYDVR0jBHYwdIAU -s4qgorpx8agkedSkWyU2FR5JyM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQK -DB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNh -LXNlcnZlcoIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKG -MGh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNl -cjA1BggrBgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2Evb2NzcC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250 -ZXN0Lm5ldC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGB -AF16+IHgp8E/OevTUizhy00ps3cYFxieEvwRzDxJy2v0TWy40vTpN/hr7fXX8eta -QQTH84za4QWOrlhx2QGKMkay3ZVG4c6CBPoLHCl1B4XOzVnUzPNWs3JNy5APzgIh -zl0XhJZ/agBXQrckWwclHneondpBCY4pebSwoUXIcK4shiSuPZp0pwR41h8bF8XB -bbEa/fRQLmEWhInQQj+2v71Svcg+joe08L2tx1FlL3foaXkOA2OJ53CtyNEvGqUG -0pDbfAc1mgsOhYfRcBeniA/GtZyIAPr5sgoZWkuNkRJRXg7B2J4CeNAtJAn+1Jc8 -y6Afmqv3D+L6ZCNOUwoVPvUEAYYpi44kQC+xkIdcO3unTAavw5B/6cZWQmEVLIPx -fE+JF/OgETQ/ja91NGAe4PLzAueqs/efHPhp9P7aV24blVNwze22uyqE66vDqbTV -FaCyzIEt8VbBVJtfFExfrV/1Bu4iYEXkUDVkrKzKSr+GePhTLRfY6ITIB6TCKXbH -Hw== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +aWRuc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAN4YbKyo2qzP +rj7Gw34JRkN8KA8+HNQrSUNSWQSSnRl4V5Cbf6mjZjzaC8khVqU7kKGJeM/FKAnC +i2hId7hBnx9YCWRfV3X0DzGON63HUrd2pmy2p7eY8/sdC98mVGSYC1WEVpSPX5JT +H0qzgcHguTaGopSirdOVEyl5J8lNdodOFU/spk9YHN8zul2kZifOcGnGB/KY2ulN +xc0KN5HqVSWP9MTEeJguPgTRo4NH9JTPS0E10JBFxEPD47JER7pGTfYquXfg+vV0 +0p4RHS26WEzbBEWyrDRry82zXcjfQEfhpcCvlsaAsIXa1ds/cqh6hH8aRBJZXXgq +uFPGEoPdAQnyAeOgdgqvgh1wVhxNqRsj0TdMexCNBNQwn9fKDNrs46vqk08pjXS1 +h+ntaHLJjtwH8L+sedRjR/nwhmVF0xiBwsJSXvDNTFJUQzKv0CvLxdlmXG47rb2T +a50AgbAiLZNWr5QrSQ09nIaJwYcx4SGg1zdgMwH/Nh6oyvgqvIj6aQIDAQABo4IC +jjCCAoowgeEGA1UdEQSB2TCB1oIHaWRuc2Fuc4IfeG4tLWtuaWctNXFhLmlkbi5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ3NzY2hlbi1sY2Iwdy5pZG5hMjAwMy5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ2NoZW4tYjRhM2R1bi5pZG5hMjAwOC5w +eXRob250ZXN0Lm5ldIIkeG4tLW54YXNtcTZiLmlkbmEyMDAzLnB5dGhvbnRlc3Qu +bmV0giR4bi0tbnhhc21tMWMuaWRuYTIwMDgucHl0aG9udGVzdC5uZXQwDgYDVR0P +AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB +Af8EAjAAMB0GA1UdDgQWBBTIBpm36I/sTz1ciWoG9XcusOBqnjB9BgNVHSMEdjB0 +gBTACisoQ95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNV +BAoMHVB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXIt +Y2Etc2VydmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcw +AoYwaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQu +Y2VyMDUGCCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9vY3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhv +bnRlc3QubmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOC +AYEAQNFtjqILkUuoxAjQ8/mL0KML3AAijPEuK+XmtG7Onc9ZMmZsuw47HZwF0uum +Kfh0T9yDOzJaLGeGYSW8vRnrIMYwaQ5MsuMYyp7+QLwcrYsD9QS+kM4nJy+DFFeN +T6DbRs7gfeLPfeoM/Y0AJwrbDV/nHlIlH2S5MF8HGhCjaTUO3Pgj9zQHzsiSlDlN +1cOrM6r5Z75mGKxnFF+TX2hIBO0eyXQoskc0SRHkezgy5dxAE7RpdTlD23xK8CuU +zQG6TZueaLPuA55/nAzPnFzL1DPV8OMhVJoTb+saD/OL6O/rNLoJdzkqikvhfp+1 +Bb6VtpJdTDVHOGQ4Xie4+TSUL1cWsPVqIT8JNLnd+NFHLMdef2NJ9Fv02epm/Kpk +J/By15Rvhg/nO7PU2TBnuKLD900HRLNwZ92xIax8KgR7LB3fC4Kp+9+Ickcc9V2j +91IiLer0KkVPm51jlVnzeQUr8Vs7YnFpkDDXerLI7GjllLuXANCVp/0EbPeLKMGW +m2qU -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.passwd.pem b/Lib/test/certdata/keycert.passwd.pem index c330c36d8f9fde..1739a3525fee2b 100644 --- a/Lib/test/certdata/keycert.passwd.pem +++ b/Lib/test/certdata/keycert.passwd.pem @@ -1,69 +1,69 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIhD+rJdxqb6ECAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDTdyjCP3riOSUfxix4aXEvBIIH -ECGkbsFabrcFMZcplw5jHMaOlG7rYjUzwDJ80JM8uzbv2Jb8SvNlns2+xmnEvH/M -mNvRmnXmplbVjH3XBMK8o2Psnr2V/a0j7/pgqpRxHykG+koOY4gzdt3MAg8JPbS2 -hymSl+Y5EpciO3xLfz4aFL1ZNqspQbO/TD13Ij7DUIy7xIRBMp4taoZCrP0cEBAZ -+wgu9m23I4dh3E8RUBzWyFFNic2MVVHrui6JbHc4dIHfyKLtXJDhUcS0vIC9PvcV -jhorh3UZC4lM+/jjXV5AhzQ0VrJ2tXAUX2dA144XHzkSH2QmwfnajPsci7BL2CGC -rjyTy4NfB/lDwU+55dqJZQSKXMxAapJMrtgw7LD5CKQcN6zmfhXGssJ7HQUXKkaX -I1YOFzuUD7oo56BVCnVswv0jX9RxrE5QYNreMlOP9cS+kIYH65N+PAhlURuQC14K -PgDkHn5knSa2UQA5tc5f7zdHOZhGRUfcjLP+KAWA3nh+/2OKw/X3zuPx75YT/FKe -tACPw5hjEpl62m9Xa0eWepZXwqkIOkzHMmCyNCsbC0mmRoEjmvfnslfsmnh4Dg/c -4YsTYMOLLIeCa+WIc38aA5W2lNO9lW0LwLhX1rP+GRVPv+TVHXlfoyaI+jp0iXrJ -t3xxT0gaiIR/VznyS7Py68QV/zB7VdqbsNzS7LdquHK1k8+7OYiWjY3gqyU40Iu2 -d1eSnIoDvQJwyYp7XYXbOlXNLY+s1Qb7yxcW3vXm0Bg3gKT8r1XHWJ9rj+CxAn5r -ysfkPs1JsesxzzQjwTiDNvHnBnZnwxuxfBr26ektEHmuAXSl8V6dzLN/aaPjpTj4 -CkE7KyqX3U9bLkp+ztl4xWKEmW44nskzm0+iqrtrxMyTfvvID4QrABjZL4zmWIqc -e3ZfA3AYk9VDIegk/YKGC5VZ8YS7ZXQ0ASK652XqJ7QlMKTxxV7zda6Fp4uW6/qN -ezt5wgbGGhZQXj2wDQmWNQYyG/juIgYTpCUA54U5XBIjuR6pg+Ytm0UrvNjsUoAC -wGelyqaLDq8U8jdIFYVTJy9aJjQOYXjsUJ0dZN2aGHSlju0ZGIZc49cTIVQ9BTC5 -Yc0Vlwzpl+LuA25DzKZNSb/ci0lO/cQGJ2uXQQgaNgdsHlu8nukENGJhnIzx4fzK -wEh3yHxhTRCzPPwDfXmx0IHXrPqJhSpAgaXBVIm8OjvmMxO+W75W4uLfNY/B7e2H -3cjklGuvkofOf7sEOrGUYf4cb6Obg8FpvHgpKo5Twwmoh/qvEKckBFqNhZXDDl88 -GbGlSEgyaAV1Ig8s1NJKBolWFa0juyPAwJ8vT1T4iwW7kQ7KXKt2UNn96K/HxkLu -pikvukz8oRHMlfVHa0R48UB1fFHwZLzPmwkpu6ancIxk3uO3yfhf6iDk3bmnyMlz -g3k/b6MrLYaOVByRxay85jH3Vvgqfgn6wa6BJ7xQ81eZ8B45gFuTH0J5JtLL7SH8 -darRPLCYfA+Ums9/H6pU5EXfd3yfjMIbvhCXHkJrrljkZ+th3p8dyto6wmYqIY6I -qR9sU+o6DhRaiP8tCICuhHxQpXylUM6WeJkJwduTJ8KWIvzsj4mReIKOl/oC2jSd -gIdKhb9Q3zj9ce4N5m6v66tyvjxGZ+xf3BvUPDD+LwZeXgf7OBsNVbXzQbzto594 -nbCzPocFi3gERE50ru4K70eQCy08TPG5NpOz+DDdO5vpAuMLYEuI7O3L+3GjW40Q -G5bu7H5/i7o/RWR67qhG/7p9kPw3nkUtYgnvnWaPMIuTfb4c2d069kjlfgWjIbbI -tpSKmm5DHlqTE4/ECAbIEDtSaw9dXHCdL3nh5+n428xDdGbjN4lT86tfu17EYKzl -ydH1RJ1LX3o3TEj9UkmDPt7LnftvwybMFEcP7hM2xD4lC++wKQs7Alg6dTkBnJV4 -5xU78WRntJkJTU7kFkpPKA0QfyCuSF1fAMoukDBkqUdOj6jE0BlJQlHk5iwgnJlt -uEdkTjHZEjIUxWC6llPcAzaPNlmnD45AgfEW+Jn21IvutmJiQAz5lm9Z9PXaR0C8 -hXB6owRY67C0YKQwXhoNf6xQun2xGBGYy5rPEEezX1S1tUH5GR/KW1Lh+FzFqHXI -ZEb5avfDqHKehGAjPON+Br7akuQ125M9LLjKuSyPaQzeeCAy356Xd7XzVwbPddbm -9S9WSPqzaPgh10chIHoNoC8HMd33dB5j9/Q6jrbU/oPlptu/GlorWblvJdcTuBGI -IVn45RFnkG8hCz0GJSNzW7+70YdESQbfJW79vssWMaiSjFE0pMyFXrFR5lBywBTx -PiGEUWtvrKG94X1TMlGUzDzDJOQNZ9dT94bonNe9pVmP5BP4/DzwwiWh6qrzWk6p -j8OE4cfCSh2WvHnhJbH7/N0v+JKjtxeIeJ16jx/K2oK5 +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQEKGM7Z40p3TpyCvl +LTPLsQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEJAwQy7HESbbXMoF +KXadaz8EggcQBlxqFXmWQ3YlUuWTTc+dUcmikIUmI8gUSB2doBxQWLkTzaNuQ/hj +uV/eU7VkNFswxrU1j+EEkoCAtClWrQ2aKTyQSM4YfhgN1vQJdOD8DypHtd0TaRS3 +VCSkJODqcWaCEc/Ypgb1KnB6UWhit1waPq4NVCZdiSF/ueRK9/iMlIRaejg61Qn+ +LVLcegWw47zTtCZKuZqFAO3PcvfC0cresGc6hrQrb9tjUyI9qCR6MeZSrHUJQ/Pb +T9e7OREQxCLxWEUb0/tMJH5k9HoG7waU5lELOyjGKXsflEIs+uBaWIssWp69KGLi +yas7BHZc8kxJNtigFQlmECZjoKvRxJWN4fOknxI8T+ttBovOtsIlVvocBfmH+Z2m +7a5VMBxD8Phk6zU4Pk0L2S6EIUCIYGS+kgHibgxZ58/QcoDZBrKOaJ/Md+Lhk2Rn +2JCuPe8CBTkB3V+xAJdaz7JAcgxz4TqwXFXA249lfYb8qY9XXcr67O83Cxwb95lu +skmxnhAKOwKIrhS1nKiefuwN5qOiA+nuPbJFadCOO5cdOSriG3s80ugDDvlPN8k6 +4b7XLulB38R2PdH/OHuF05QGQ3kOUbJWatMig+/09LNo9FiMaCczdIXQ6LGxcHc1 +G1Y3BqK17z3yULqHvxU4tIbj5Elt+X1mfKIfifmXBUujbz5rR1pBTfpnk+DFGcHZ +B/cXWFP0tDJAE7mvna3HimNDQMP58NbhZwFtFdU3H9R5n4R5MsYQ6+a+amQ0pZ1Y +XAaQ/iEg0WY8CEFO8zEvB9R05zMm6vycNtN8CbfBq9CJ0OaR1ymMW8pRVagsLDuh +D0T5ZtWZctE2+ImlwlpGDs91CX3zDxvW/3Bwf40PF4x7LJMt/tFzqQEovokvzhUw +0jX/kf42QhvydnDoxqczdVZOfbjHELA21U+JAeAAj9jhcCEYd50c2BVt4jhST6dP +pzNAqxc1RGwTU3K3nZkWKt4hMXPDGjb0pVqiOuPz/718Nsd6ck+Ko90gFX13mPLC +6FLPGjNUmE6f3TKjbRrQz3+IWMncCyo/JSWA85rldxD0XAem64h+2XppFWJsxJD5 +nH2prLckQUWBLljWNIQQyVuAWo0TVq3NCXDFGaP6GHNrZo9zZxDhxcp6v2Me3Mir +few9aQNb59Q1/A0qwiCf26Oe8JYTD14UCez41imxD+w721SO8jzLR80+ReemdwzE +B9fXh3T/lrCIDNB+F+GWA9wQUz8mzArKbhqQhl5gk39dnLahc4aN7GrkwL4hjhKW +O3X5bBdhhRlcub9SbXFC+tx/w6G/0roGMSvD7h8G/mw3QcUUwwDvOuI7qAqQ8FC+ +xcGWeHUiTGxp2t2gz8DSaeL2TbBMbHe+tRBmKlmmIrAq3AEllWzyze9ta5ZYt46P +njZzS+vbbA0mfcFAId7tQOjQ2/ygZePs5hg91revLzNMDRJ1IeBHnrVUuDA5D1HR +iFW+hhj1Lx1s0XAJBJcDos8xy/LZlRDLxJSHrMktZOELTYh7UZw4S2PNkOPms4kP +V4G6D/0wZh0t+T9zpH57ivLdyBQ2fsFwx3X3JfQkYmoFlwCz+uUb368TunQ/yZZY +fVh9Q9ndk6y6aor0wkORsBefVv3eN+2BDt6ZUpYu3Liim+W4C/SiKVfQ+5yDb/qe +KmX7kcmr/f13BWvEOUN4fzCMiw/N0aAgEejZQTwMgivvGSMcNaBiLfdvM6hmy7NE +uvgpLtK5v+YRcOLwiaVH7UL6Q61CJ+pcDNRvbJT/iVwidYUO16zTVR6NrtPhM62a +Ziq3Q77HD4D2KCRPEo3Xivc5RwITHeDgKHwpLRB00jiZ3iR0Wm5lI5VbyWLnmQ4x +CDPAZetJZGBkbhTmUVbL0m3HNLHfF9g+LWA23L9mRNCKRemL0c0UJRRJPaNDIx8p +8h27al4xI8nFav0TZLxmSw7mqnzyE7YAoe4EsmedOSXktHWN06dnczHu8AWrgNjY +Izv9XUmy9V5vsRG/lg09FyG/eZSgfs9rdmL/qeFXbbl2J33YTqrt6O3ZaEsgg42r +piZztung8Xro3VlFSAXZGrl7Z0AWwzaOvwreCxbAG1WtBmGgLrs98z627XgfBFYA +BPvJwn9f2GZzixiN6M8c2M5XueDE/Vpn4A/GKLJ8RXXxKtbGRXFfdF/M6mEgcpa1 +9pxAhzNTPaOp+SFbm/cFa43HZYZgDg1D9zth2ZII0ZITd5OpEaSNOsrOrhqXeVQU +isBpybgPqVQ50xUuRUyoHYAZClVe6+PBYVbvlXBlTWhSPY3leUm4AYt+fjZ4VFlb +p4I2KRmd68zXXwl2spkEsC96e9k28kHta55dO32gSwaXUUHRKnFmKB24jsleF6n3 +BOKxzsO9jr6vi0jUqZXvaCnmYWqS84wZH/1S1pgotaWdMQ/KT27xwHk= -----END ENCRYPTED PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODA4 -MjkxNDIzMTVaFw0yODA4MjYxNDIzMTVaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGP -ADCCAYoCggGBALKUqUtopT6E68kN+uJNEt34i2EbmG/bwjcD8IaMsgJPSsMO2Bpd -3S6qWgkCeOyCfmAwBxK2kNbxGb63ouysEv7l8GCTJTWv3hG/HQcejJpnAEGi6K1U -fDbyE/db6yZ12SoHVTGkadN4vYGCPd1Wj9ZO1F877SHQ8rDWX3xgTWkxN2ojBw44 -T8RHSDiG8D/CvG4uEy+VUszL+Uvny5y2poNSqvI3J56sptWSrh8nIIbkPZPBdUne -LYMOHTFK3ZjXSmhlXgziTxK71nnzM3Y9K9gxPnRqoXbvu/wFo55hQCkETiRkYgmm -jXcBMZ0TClQVnQWuLjMthRnWFZs4Lfmwqjs7FZD/61581R2BYehvpWbLvvuOJhwv -DFzexL2sXcAl7SsxbzeQKRHqGbIDfbnQTXfs3/VC6Ye5P82P2ucj+XC32N9piRmO -gCBP8L3ub+YzzdxikZN2gZXXE2jsb3QyE/R2LkWdWyshpKe+RsZP1SBRbHShUyOh -yJ90baoiEwj2mwIDAQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZI -hvcNAQELBQADggGBAHRUO/UIHl3jXQENewYayHxkIx8t7nu40iO2DXbicSijz5bo -5//xAB6RxhBAlsDBehgQP1uoZg+WJW+nHu3CIVOU3qZNZRaozxiCl2UFKcNqLOmx -R3NKpo1jYf4REQIeG8Yw9+hSWLRbshNteP6bKUUf+vanhg9+axyOEOH/iOQvgk/m -b8wA8wNa4ujWljPbTQnj7ry8RqhTM0GcAN5LSdSvcKcpzLcs3aYwh+Z8e30sQWna -F40sa5u7izgBTOrwpcDm/w5kC46vpRQ5fnbshVw6pne2by0mdMECASid/p25N103 -jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu -0x8evpvwuhdIyproejNFlBpKmW8OX7yKTCPPMC/VkX8Q1rVkxU0DQ6hmvwZlhoKa -9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW -HcVKQHyOeyvnINuBAQ== +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- - diff --git a/Lib/test/certdata/keycert.pem b/Lib/test/certdata/keycert.pem index 0d398633739a51..43ad52c2af507b 100644 --- a/Lib/test/certdata/keycert.pem +++ b/Lib/test/certdata/keycert.pem @@ -1,66 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/wIBADANBgkqhkiG9w0BAQEFAASCBukwggblAgEAAoIBgQCylKlLaKU+hOvJ -DfriTRLd+IthG5hv28I3A/CGjLICT0rDDtgaXd0uqloJAnjsgn5gMAcStpDW8Rm+ -t6LsrBL+5fBgkyU1r94Rvx0HHoyaZwBBouitVHw28hP3W+smddkqB1UxpGnTeL2B -gj3dVo/WTtRfO+0h0PKw1l98YE1pMTdqIwcOOE/ER0g4hvA/wrxuLhMvlVLMy/lL -58uctqaDUqryNyeerKbVkq4fJyCG5D2TwXVJ3i2DDh0xSt2Y10poZV4M4k8Su9Z5 -8zN2PSvYMT50aqF277v8BaOeYUApBE4kZGIJpo13ATGdEwpUFZ0Fri4zLYUZ1hWb -OC35sKo7OxWQ/+tefNUdgWHob6Vmy777jiYcLwxc3sS9rF3AJe0rMW83kCkR6hmy -A3250E137N/1QumHuT/Nj9rnI/lwt9jfaYkZjoAgT/C97m/mM83cYpGTdoGV1xNo -7G90MhP0di5FnVsrIaSnvkbGT9UgUWx0oVMjocifdG2qIhMI9psCAwEAAQKCAYBT -sHmaPmNaZj59jZCqp0YVQlpHWwBYQ5vD3pPE6oCttm0p9nXt/VkfenQRTthOtmT1 -POzDp00/feP7zeGLmqSYUjgRekPw4gdnN7Ip2PY5kdW77NWwDSzdLxuOS8Rq1MW9 -/Yu+ZPe3RBlDbT8C0IM+Atlh/BqIQ3zIxN4g0pzUlF0M33d6AYfYSzOcUhibOO7H -j84r+YXBNkIRgYKZYbutRXuZYaGuqejRpBj3voVu0d3Ntdb6lCWuClpB9HzfGN0c -RTv8g6UYO4sK3qyFn90ibIR/1GB9watvtoWVZqggiWeBzSWVWRsGEf9O+Cx4oJw1 -IphglhmhbgNksbj7bD24on/icldSOiVkoUemUOFmHWhCm4PnB1GmbD8YMfEdSbks -qDr1Ps1zg4mGOinVD/4cY7vuPFO/HCH07wfeaUGzRt4g0/yLr+XjVofOA3oowyxv -JAzr+niHA3lg5ecj4r7M68efwzN1OCyjMrVJw2RAzwvGxE+rm5NiT08SWlKQZnkC -gcEA4wvyLpIur/UB84nV3XVJ89UMNBLm++aTFzld047BLJtMaOhvNqx6Cl5c8VuW -l261KHjiVzpfNM3/A2LBQJcYkhX7avkqEXlj57cl+dCWAVwUzKmLJTPjfaTTZnYJ -xeN3dMYjJz2z2WtgvfvDoJLukVwIMmhTY8wtqqYyQBJ/l06pBsfw5TNvmVIOQHds -8ASOiFt+WRLk2bl9xrGGayqt3VV93KVRzF27cpjOgEcG74F3c0ZW9snERN7vIYwB -JfrlAoHBAMlahPwMP2TYylG8OzHe7EiehTekSO26LGh0Cq3wTGXYsK/q8hQCzL14 -kWW638vpwXL6L9ntvrd7hjzWRO3vX/VxnYEA6f0bpqHq1tZi6lzix5CTUN5McpDg -QnjenSJNrNjS1zEF8WeY9iLEuDI/M/iUW4y9R6s3WpgQhPDXpSvd2g3gMGRUYhxQ -Xna8auiJeYFq0oNaOxvJj+VeOfJ3ZMJttd+Y7gTOYZcbg3SdRb/kdxYki0RMD2hF -4ZvjJ6CTfwKBwQDiMqiZFTJGQwYqp4vWEmAW+I4r4xkUpWatoI2Fk5eI5T9+1PLX -uYXsho56NxEU1UrOg4Cb/p+TcBc8PErkGqR0BkpxDMOInTOXSrQe6lxIBoECVXc3 -HTbrmiay0a5y5GfCgxPKqIJhfcToAceoVjovv0y7S4yoxGZKuUEe7E8JY2iqRNAO -yOvKCCICv/hcN235E44RF+2/rDlOltagNej5tY6rIFkaDdgOF4bD7f9O5eEni1Bg -litfoesDtQP/3rECgcEAkQfvQ7D6tIPmbqsbJBfCr6fmoqZllT4FIJN84b50+OL0 -mTGsfjdqC4tdhx3sdu7/VPbaIqm5NmX10bowWgWSY7MbVME4yQPyqSwC5NbIonEC -d6N0mzoLR0kQ+Ai4u+2g82gicgAq2oj1uSNi3WZi48jQjHYFulCbo246o1NgeFFK -77WshYe2R1ioQfQDOU1URKCR0uTaMHClgfu112yiGd12JAD+aF3TM0kxDXz+sXI5 -SKy311DFxECZeXRLpcC3AoHBAJkNMJWTyPYbeVu+CTQkec8Uun233EkXa2kUNZc/ -5DuXDaK+A3DMgYRufTKSPpDHGaCZ1SYPInX1Uoe2dgVjWssRL2uitR4ENabDoAOA -ICVYXYYNagqQu5wwirF0QeaMXo1fjhuuHQh8GsMdXZvYEaAITZ9/NG5x/oY08+8H -kr78SMBOPy3XQn964uKG+e3JwpOG14GKABdAlrHKFXNWchu/6dgcYXB87mrC/GhO -zNwzC+QhFTZoOomFoqMgFWujng== +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODA4 -MjkxNDIzMTVaFw0yODA4MjYxNDIzMTVaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGP -ADCCAYoCggGBALKUqUtopT6E68kN+uJNEt34i2EbmG/bwjcD8IaMsgJPSsMO2Bpd -3S6qWgkCeOyCfmAwBxK2kNbxGb63ouysEv7l8GCTJTWv3hG/HQcejJpnAEGi6K1U -fDbyE/db6yZ12SoHVTGkadN4vYGCPd1Wj9ZO1F877SHQ8rDWX3xgTWkxN2ojBw44 -T8RHSDiG8D/CvG4uEy+VUszL+Uvny5y2poNSqvI3J56sptWSrh8nIIbkPZPBdUne -LYMOHTFK3ZjXSmhlXgziTxK71nnzM3Y9K9gxPnRqoXbvu/wFo55hQCkETiRkYgmm -jXcBMZ0TClQVnQWuLjMthRnWFZs4Lfmwqjs7FZD/61581R2BYehvpWbLvvuOJhwv -DFzexL2sXcAl7SsxbzeQKRHqGbIDfbnQTXfs3/VC6Ye5P82P2ucj+XC32N9piRmO -gCBP8L3ub+YzzdxikZN2gZXXE2jsb3QyE/R2LkWdWyshpKe+RsZP1SBRbHShUyOh -yJ90baoiEwj2mwIDAQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZI -hvcNAQELBQADggGBAHRUO/UIHl3jXQENewYayHxkIx8t7nu40iO2DXbicSijz5bo -5//xAB6RxhBAlsDBehgQP1uoZg+WJW+nHu3CIVOU3qZNZRaozxiCl2UFKcNqLOmx -R3NKpo1jYf4REQIeG8Yw9+hSWLRbshNteP6bKUUf+vanhg9+axyOEOH/iOQvgk/m -b8wA8wNa4ujWljPbTQnj7ry8RqhTM0GcAN5LSdSvcKcpzLcs3aYwh+Z8e30sQWna -F40sa5u7izgBTOrwpcDm/w5kC46vpRQ5fnbshVw6pne2by0mdMECASid/p25N103 -jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu -0x8evpvwuhdIyproejNFlBpKmW8OX7yKTCPPMC/VkX8Q1rVkxU0DQ6hmvwZlhoKa -9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW -HcVKQHyOeyvnINuBAQ== +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.pem.reference b/Lib/test/certdata/keycert.pem.reference new file mode 100644 index 00000000000000..5528b98fff9377 --- /dev/null +++ b/Lib/test/certdata/keycert.pem.reference @@ -0,0 +1,13 @@ +{'issuer': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'notAfter': 'Jan 29 11:51:12 2408 GMT', + 'notBefore': 'Oct 8 11:51:12 2024 GMT', + 'serialNumber': '4F1870457EB3050737FA5DE298392511BA9C0E92', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert2.pem b/Lib/test/certdata/keycert2.pem index e59d45439d4b6d..63e2181ed83e78 100644 --- a/Lib/test/certdata/keycert2.pem +++ b/Lib/test/certdata/keycert2.pem @@ -1,66 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCf8FWxi4oVlDVx -e8NDFgb+IYAGr/hZWuY1Zq7d7g57yPoxJrgt+bN89+U7qTduqyB2Hy8G0TqeACOr -IdpPZ8P7V5E5YiASwfJ72nbVo7qR9DAKA5FE8PU0bJFmFLjDDihc970zc4ilRDfR -WylUpj68nefOY4CzFzeiqVOLX2wezs7Z0hflkSXGBmC0j1FbQU2I3YJg3CKCabhT -tU6OyKItzjJ2vVaOoQ+B0Kv8leaRQ6ANZBAFQF2LepSy5F2+oSD+QHjPr+012V5D -mrsdIc9We8YyonS1u/3HI7lLohf3W+qFroQWjn0DJI56ScV1uEr/B0+hn2jBRTM5 -d1F9BeVWm1u8BOJu50CvOeuxiVLsxJpa4T41DJznJk5V+hE4hKvDKmlrwulsRp8o -jUEyUi8dzWOBRfAijIWv3qAPjGA/J33n6+PllCczC2BsVZhVmLqSMCwp1g2JTCM/ -KC7T4vOl/EGkm76fcmLeA1Ef8oUdRg+3T77VP+HqZ2JP06J8O8MCAwEAAQKCAYAw -YvJZ82BEJQGCIrIxMpHNAm+MFmKpDdIFp9oRdDrXgjcG9bLU3e1KSmkEgq4tggIh -GlAM3PHB6ULhPC2ixj7JZHWgCaqwYhKtG6vF+HGyRFDgRrIFTGyyfoICgxReloLp -lV2dGj/l19yXLuAzJtRmFdOSYhIGnGiNgnKvAKBiNajoxyHJpv7piPZqyc0QMZJ2 -bKVMDm02TSuhz4FDuzktaGtl9uQf5GQfnvTZRrRpkC70vigGnrFuSBiCgopF6NLq -6AXl8YS3Jcu2oGWrZDfS/GlG1QmvGGsmr9wndJSGG43jcpcRZt0g1nJNu4Fioq3e -7y6Gap9TEsciuQOv/6RD457XkNARmTQxFpEwmSgOPQn2pFcDspo71Ej7azzL/Z+3 -jvnVo3wxgxBcrpyh+vhBtJARp4pT4anW4PcD6IcPSOWbnI8Ldoj1XN5QkJcBcykK -6LmsAUqsmEQDNsmnGZWyYSCns4P2vUJi0hwQz8UiQwgAta3xnq4v5On7l3cq35kC -gcEA0+joOFbZBeGlCb27tDW4VCW0cQuczzuNEoBUKnsNSqy0nx1O7hgHm/f/NQDD -cpxiD15bRQ0KM9QbQC4dGaVoLsM07hUGk97dCxQPs2zot4CodCKGohs7E154tEDP -zVg3YS5mubUmqdqtn8ZCKeeZye/Tv2ageyF300sEgj2Cd7EZ8S4sB0PxZ2tqT3jy -cBL5cDruLEWuHIQjN7WwSjxnXocpb1OU7dJ+v4zFPCkSCOoa0DTTw4jFhPEOBdqV -T619AoHBAME3QyW4QVtU2Ct9u0B1XThhqSEyOpUrcH9nOoefggwP4WF3phVx16BG -aDKUIGQ62klRa5fi2eooxcjQRLv1sWO0UzssnO6ABMnGkUiRdrowo6xukNak0RTp -0gvNoJ0SZxGF0yWSCw1Rq3qP2Koj7XDumFChAzLMyUsnoOl29SA7GfXcZp1pZTiq -kOfFMWt0CIHu/EK03YWcd4vfQEq6lus39RCSXuL++Jva3yiEl5s069RFZvP1bNrD -emkfetDSPwKBwQClk+8fVnzs44sZOW9ZOEB3P57mVbSJGHb6Zdtd9hhEqP3Y9gWe -dJg9fmGjAJ23CAp3B7s5ER9PsAQ6+c0zJNNq9ox9G2CwWgtNhLdf81FDUPxPAktA -jxZx4/dcoOe+A5gCD0elA67aOUxA86DvLVA1QXeqrn3muBfwuUUknvs6mt8yXGl6 -o9QUgxHmVxLYD3tn/iPr4+ZP0c/Sz9yXpOsAKYxuuFg+G6N9+HiEsXKuFH4vAZgV -yODNJ61VVZ4lS+ECgcAqFqOl39E81+qO7sCPdgFsermg5ZQlUmUbG52AVZq6jesG -lE21disGWs/v1JyJuNg8CGRrnZriiycqa1PNreOKWImY5kr5GSHx4jNbn3RBcr70 -nNEoMJbq+1QqBgzqqkuRYZlxIbMOn6++7v6/cTwT0aWUSr6rnjhrCqLeuG8FKlqp -V+1ydLb79QvDsQzm30vLIggJb+ShakgQS/1xSdv+OR5FEd1hjTESokbiSJ/Ny2Vj -xAp9MgUYUmSj6ZuTSXkCgcAggshdRQLom/EK2pYwffIpKfBiyLbi+KIjKxkiPEsb -jrrQbvh9ZN6iAG3StVAYB5c6vewfeIlcDT0YJDyy1hGRLRG7vf9ubPf+n7Xp1y0W -oo9L9qfCHu0jmWwtinkFYjpTDkXlxXCG2v3TllNsNX/5afYo8sb9oxXHLTpBlwZB -fw6IgNZblWQevdgmUMTP9W2W7AZUxEz4gOM6lQkOwC3U59Dx2yO6rD3An6G1tlZF -2MClyf8o5d5ePObH8rkxrpY= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCtFmmyQ+HjjeBX +UE61fTenp2C9cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4v +ZADh/eIFOONvPebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5b +yhhpcOH1AmPCsiLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKTh +J+X8+PCY+FRObAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpO +G35/bW9jsQRpbeGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJ +izTuFV91pGmsXSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyU +Vr+odU0t7+YExBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp ++5cGWLOkcs8Xt0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAQKCAYAU +x+HCxFb6LLjS4tJrBHCf0VRSX9rf/7Ri2oYRXRUN1B3cTrZFeZyLdJyKggNiSy3E +GcF4lrxl2Xgaz8/cp3rxE5EYHv93hCQPZUbu4r2kZCLJsRPcGn3D/dyEKhCGyAi/ +fgsTkxyxnyK/9HXqhYyrZNu4rrh0SH63GKJ4GZsIFhZME95b0f6/e/3YJpSJFxKK +LF+xd/rjoWNOS62s1JqA/czpaXDqnJ0frNX5aQixLHnWUy/5qgRGNnY3/LOB0MuU +EjFizRSJp6gQQ64dIFgiMyUMi+Z6fw6xkTpN5EbmgFCwmaja1/hp9UZ+atlVpJsO +SB+OhF2wxB+bX2phGHJ9DD4RY/di1YKPvmH0+LITq4JiJWwJLgoCm7krzfVxZFZN +lgZPvdNw/Yt/Y3bN31FMw8DwRCThnL+MH83E58iCWQq2zKS1RssBBwFacI6Wf1ed +442NPIH47oi5qRKUxYHze6kFssSs4TkbR++VYHXLbTLjqfePLP92ZFxdOstx5jEC +gcEA48mx39dwr1oSk4uR+q8aDZIjPT+9D0IxcdAmL968QgG98hF2Tx7s5alR4y+9 +2FueR5fSqRweJfaDn77j5DVNXLD5jBz73pdwIkfqJHLfcsN7NnzCz3mxU9WP2F7R +W8KD3C2MtzGwQwOI3GT20+wJaA7p5n5zKR7AC0RxhC3U4zwH5nSjtY70A3dWXkn3 +F0j+B6ak0P0oN9VQUuvk5zw0IZ32uxef7poiHvFZpl1DQSgX/edT1CIT1s0nmVFl +w1ojAoHBAMKGX+TobGZfZFj/I7oIVQ7023pi516fZx/R0Y0FTmZzIKsdt3wrPmar +3K5dZGasNd0wzUsxX+vSjMtI1oERuPd4Rs6YqQHpX892uJ7L41/oBcFCc+Fk7jBY +TiRg/8/gOTaQe1nDR52K/R0sqBatRqInNMBDcBsVguBC4dTqP8CQ0CnEjAOYsk1s +1nYPKcfM36BUXBmSlnvUPWMm8KCT9Kko9ylHVq+d+ziDebfYpy6fTvti2NM31wTM +1prasEUaqQKBwQDQX14+5MapMd1SaVelmW5cwbVIvzjEb4npkj6MhdVzEELg4IZ4 +hFKzGDvXdoHVHKJi3YiQuC8ADUyE4kt4JCZbx2zQdmcVTTT/tweCRi8PvbDFvEU9 +JBZKGU+X38zmgr66uFRD9MlH1EDrU9TTMdW9Af+HoV7ZW87Tv82T25UmNXEIqORl +HpsrXIx+fmzxOQ1glFmq8BpNUO5EnJPtz43kvqrIpSjhTNAvvBqFbEUsom+oDWgK +4w2A7nTt9J8BoD8CgcBkYKa3HmBhazQC4JV097u8nglrXAH8R9EVEFZLqMNOBnaD +FjCKeF4Y6PJVX4fhm1eoLfihpnbS37EbbRiTPavutzgCf7AmdmCkU6Ts/FT2NmpR +0ZKuakCm3cpk51DZ2eBsEZ41MZmQ6Bm4pkSOfxeFsSl9VM9SioUgaCLUlZQUMCXa +h7ugV3kajuETxrtOiJ+UwjNMVuIkP971fTCKDA8iAyuXN2K5+JGcFewHPFr4qeg9 +vEIarCPeLD1JZzOyVRECgcEAhqeXfmFezNKGvr7qCflJe92LF9MStgJ7yE0OfADy +B+RZKeOwoqOO3VFR1piAn/DzrC2K9q+Of61gw+KWQOgMnsigqZ2mFLGChRjWhb7S +3G0DGOb4+DD9RR6wlFPFXSwVxSWGKrqNhOJik/IzVWvYxOOFjVt1adXZ3GftnYsv +nZCsS94H4kMiXr6UkbkjjxnZ1WkE9DaQJU27Mw1dtwb1ECzO0rty9B9L6b80MYW/ +nMhJV6sanCUex4nAChHD+VR7 -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIUF15VKdwjiTzzKgs6PnNpEekV9QQwDQYJKoZIhvcNAQEL +MIIEjjCCAvagAwIBAgIUCP/QP57z62jCsjf6l0ft/yzDxOkwDQYJKoZIhvcNAQEL BQAwYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhv -c3RuYW1lMB4XDTIxMDMxNzA4NDgyMFoXDTQwMDUxNjA4NDgyMFowYjELMAkGA1UE -BhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24g -U29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhvc3RuYW1lMIIBojAN -BgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAn/BVsYuKFZQ1cXvDQxYG/iGABq/4 -WVrmNWau3e4Oe8j6MSa4LfmzfPflO6k3bqsgdh8vBtE6ngAjqyHaT2fD+1eROWIg -EsHye9p21aO6kfQwCgORRPD1NGyRZhS4ww4oXPe9M3OIpUQ30VspVKY+vJ3nzmOA -sxc3oqlTi19sHs7O2dIX5ZElxgZgtI9RW0FNiN2CYNwigmm4U7VOjsiiLc4ydr1W -jqEPgdCr/JXmkUOgDWQQBUBdi3qUsuRdvqEg/kB4z6/tNdleQ5q7HSHPVnvGMqJ0 -tbv9xyO5S6IX91vqha6EFo59AySOeknFdbhK/wdPoZ9owUUzOXdRfQXlVptbvATi -budArznrsYlS7MSaWuE+NQyc5yZOVfoROISrwyppa8LpbEafKI1BMlIvHc1jgUXw -IoyFr96gD4xgPyd95+vj5ZQnMwtgbFWYVZi6kjAsKdYNiUwjPygu0+LzpfxBpJu+ -n3Ji3gNRH/KFHUYPt0++1T/h6mdiT9OifDvDAgMBAAGjGzAZMBcGA1UdEQQQMA6C -DGZha2Vob3N0bmFtZTANBgkqhkiG9w0BAQsFAAOCAYEARzdkuqa0Hexi/saMkdi3 -bubpQkc7X0RYKWnjy/PgcmbvQXLiWRMZOH9rMWvd5v+ZfkgAtsbOQuP8ycioNIFY -Il5SEmxHEN81z5UNSPLOib6ky13gzrnXRAxnnO7cICG7AaMu1dHv57fqjevcx/n/ -nxPNKwKL+TDpMw7ATVZw7Py7JciKyFAfwtkvt17j/ldvaQvuwmWHzyFVrQniQcQq -QEa4jy/Y/pXHAgCKq1qbe0ush17j1ChyH7l4SkF2xJKcYYQF5ipw8zg6WeOL2NFE -G1KDJN0SsMmM3PMN1e0lLQP3G+UaatervrKXu51QleKL32Xlby+pp1w9KKs39/Tb -RT8EMe9A6cecod6TL0ZUQHow6ykNYBkfSKDLTKWnL9ifZ0C/DvgmS7DpJg3oAa1e -GhIglMrgqJflTHAI/PvEsCKM1O0Un2dVGWsUCzPfhj1cKmagyb0Zd+2Tk9xGSRs9 -2ceXMxRCjOJwEHUCFuTYeqowabdlpi0nyPbSn7JIwCpT +c3RuYW1lMCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBiMQswCQYD +VQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhv +biBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDDAxmYWtlaG9zdG5hbWUwggGi +MA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCtFmmyQ+HjjeBXUE61fTenp2C9 +cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4vZADh/eIFOONv +PebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5byhhpcOH1AmPC +siLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKThJ+X8+PCY+FRO +bAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpOG35/bW9jsQRp +beGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJizTuFV91pGms +XSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyUVr+odU0t7+YE +xBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp+5cGWLOkcs8X +t0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAaM6MDgwFwYDVR0RBBAw +DoIMZmFrZWhvc3RuYW1lMB0GA1UdDgQWBBSAf/Z+TUHMOcr020NGG4geSnlNhTAN +BgkqhkiG9w0BAQsFAAOCAYEAIia0ULQmC510p78HZbiOb2BV8q+0AY0l6Cn/FRIL +Voqy7uB5oEYdDisla8epmBTU35+JmfrRHbP6IuSqKdcGj8VqsjZljhntSrB7rw4O +IqjbnxnfEuREjY2w+WGLvvdtVGXCBgfRmIItORFKpoOvLzLIi1lXeDq8QL97K4nM +tPsZpzILCBCoXmhh0MweCCNe1HSD8q6EbSDjVoxraBl0YK/l1ID08Fi5fwdddRX7 +txfx5NtoWWbsLZY0GdBtQmRCifs7P9lyhjUd87bJGd8WCBE4IzLwJiHEyNPI4WHe +8jmvKD/zO7mKxT9/jF7IBqwTHgQQ1uaRGytiXjRlfPu4Ez8ASR6rP7Hlua7h0Ba7 +OcDPcgM8rzN4zcfsvZ+8Cd84HXFjHhV6ZdRPbBWFWd8TlDufrWNuY4SyOtkyfE5E +lWcoRnkBGBaooZNcd72ZwZVO0gAS31bgs8ju+k02GIDLkuZSRNPeFsRQbcMu1Mhc +cFteFsqniLhzpkWJVQQgeGlV -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem b/Lib/test/certdata/keycert3.pem index f6887ba7a84e1a..097dc9f8e88091 100644 --- a/Lib/test/certdata/keycert3.pem +++ b/Lib/test/certdata/keycert3.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDFtLOteQlQojN7 -ztkux7m0hmGKkP1hh0hbKqTcD87jkLAqAwZWenjZMjCbbZ3vP+AObCIkYIKzPXY7 -Yi+H5M3O2mXIDxoHGjL/GWtoEyDNXvm9UC+MRuSOq2MaLHHQG0Rx2TxcYrMVUM7b -93rpN1LGRrCv1gISXM4EvEJooAR7Aadj0pG/o0fqDAdFjH6QZbhn1iZle+eGbjcf -dgH/H0F8dn1PPGoViHXicbsQ4kB6002Pf+aXP4b2QKAbflyNHEKHPHEOOTXrFjMd -c+bqKW24epEsMZI59qx9hU/4Rvp3/v+vEwTL7Nm7ilptzZn2cvGCW39LC0nNYLOz -kO3H8xwA75h6uykdB+WO/v2CKIK9M/ZO+9QNrmaokfKDamCk39b8hlCwNL6LsVpv -d3XTS5Wn4YWn92EqiltUJJoPo7pc7VTdWCg4zVFn4Q8Zh4NFNn/qTB8lEMgrsNTV -5cyZ7zhoBiUMSO45bmo2NsnE7ce/JUhlqe5uh0PT1MIBgTV+oDMCAwEAAQKCAYEA -udsy4gwblqK0tVnxz0lQqYV+os3EdO/BNHr1Oi7eNg2pngTz603812mYSjUVOHma -vtQmkH3twGQyBoc52Y1dcGzdK+IOfMjDUg7qao840ffL3I1J9ZwbdodlhZBsec94 -W3J1jP/4DDzICf8vm5g3h0+i/9m2Xt7BibAU2dg7/grC+lNUUoxDqaEfIOF/hW0q -muq1c8e0EisAROIh5FzUqhWVnWxU6eM7tuFlkuyu4whLLHB3LI466Lo+CTqT9M+v -jJYlvS5+AZW3qMBp6WOI8C+VIiBL178mo+Igkyyy5AYXcWeNkjp6ygRWvtWXIhCv -CI29mf+BP/54jAY0rQRXJ2UcSHXmM6PTDkE/L2OKeiY1Ou8gLOwun3yBVdbkXJMb -PWmUW4N8qSIJQ+vE2TDqmkqAT6m+ilzOXl1O+LLTvGyMnOiiSLXK9mC4ND3tqaQu -hvKivnI1doErcWUaIf1DHiJmLrGxrTCUKjCEoefqVq2/dDdtCfx7CqUvjl3DYKMB -AoHBAP+Vdi6D07gZFepEGCaJ+YH6cxEyO73CNnea/F1whVAzOv91kHS32jC9PAI3 -/wYlX+DLcN9mVF/q62V4SLZYfOxTPW4vWO0A45URe9s9Z795fdAcQ5jt3QFOVSnk -3XSaCkIOwckuwabGJi4+foiUEOnLLzQi1/g7x12dwejxVNhqhz5KFkOQPv8fQRed -sb5LVLYDeprsB2Vsx0fHwg4z9FvTIxLBeI7+sJD30lNpYZrCl/T9x4e1SV2Rwn2W -bghxgQKBwQDGBx07biZK9RB5g4qPl+G6vz0M+/KBfpwQbMYxSyct7u6gfGD9mWBO -qocIIr39Unac3kUL237Cn3HbgiGCRe7Mwd7XqnSSGWM5oWSlVQxEKTXYUlTbd9O9 -DKuyQGOl/AMEwD4ZbEOfQNmnd1U4nh1AV052FQY8Ry/atGFT9fApA/5X/bbenOwQ -YGDsokLzPf2BIDncpE+VNevUMoMI7EnySgjjfpL+cRld0qpLqBMo2h5VddeJ/5YM -1YcNfMQiw7MCgcEAwXqXuKa7A8aZvHpH/gS9CRRbP01TxFbdfLWrDeE8SnY9111c -Ob9kQTk/0D4rpK9uYXIgxD1m6iWghXQFN2TNTOnGuz7EhsYBgrt1k4Zsn5qND5oV -4hNPFsoB1nEW5EooMdGSCYaHuoSOKrvMdgAAvbu+xC0MaTJ3vfrK7Fik7h/WueTD -7emohuFWGVabU38bZZ5EljrPboxmX4Rs9uuFtG2lQ3GKnlVXvKaeZd6EsO9WsXPc -NHOcUmUhYokaSvIBAoHAGCxGJTsM8Zl4qVylTWH87A7sJOmccLJD2r1sdBf4cGL6 -PhzwugQ+/VtToGqdRo8Ka5u2Ufw5PQi5nVIFRSHERLpluW3VTQBMXHyXDJeVJ7zg -Fcf3E9NMxYcGbnvtrhVVSP8ulWvh1U7VQtwOSxsB9xixOzjVygXmkYvzVYxwBJG4 -OoV+DS6aomUhb8Fe6tJmX5zPc1+bV1t9ril8VVqCrFDdROfuiaDEt+8/Wnzp2dLG -YShBZ1cLugVWtw7D4nqBAoHAF29k64iAxY5Y4OOibVkqjUCPyqG2oxiXqgO7CxZp -FGUat5UtV2mIBlSENs1o5AZ1nPlgWtPtg0xVCaG2t/Rq7ugvUfAnAhUK6zX8FS+T -gCXE+7iKuuIJiCo13/iAwF/CLfuXvj4CZ71ta0wX9w99f1FcPEk0x+ytiyuWJK8K -tyubL34JwNrnkh/8e3LcV3L88Sk9ZmxeTz31f3cA3Fy2ZJOAUMD9dKXeKtY7azzt -MkhXedRsdLSKqMh0VGeGHoLS +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDuNnCqYWoguzKI +B3PwOsoRzI8wgNdxx3RiIJSItFISlAgB4XkYzPBpi5nFR8GCqq7IUQQ+iYwdZa3y +H6X11IdlrCx4gDdQFSFOegnG41oqv7GwBEFvq6xlowpoaGw+5/8/sJX/VU44rL9/ +fiMVTbehMhMS/z5nTSYSACV7sRXnwW4T6l2Od77FzPBjnaFFDNr5GIoiD4mNSJ35 +t05tliGcz0IyE5TGR6kIEFNIH5theHoJJ4JC4uLyWJweJpxDx10njWRwKN/tXm8F +nHQPMRAUl2wM07QZEI1yfzUr00/oG2dpPnPyNh+6p2ld8JS0g2/Pd2pQL8OtmM2c +DcN5E+qCEW1UVIJ5//ad8pUcoQwyRIBROJbakqv6FcVTdgTogoq5Vu+T6TVFLSIS +vVvpsV0MtT/H9/9TtCnqOwliE8s8kb3NjQT8e72VHNd43bVPE4DeQ+XlbMoKzV+G +OmSHSfzSOZB9kScSmGd5YD8783TbL8/7Nn1ERQdWqaixGI36+BECAwEAAQKCAYBV +Ubhevg9V89ZwdELpSxUu9NZgZ/VCck7UCplIsVUoBE8t5UULRfPhybdkuoOrulhp +tOLRR1ChAtcffohhmSJ5nwY6jqnBDCBmzD0OOEYGQ6xvv8Z0KcfQi2nh5WzHxy5b +8HJ5BmPC1tSr5FDKg5B+ssG0Lyl5tF8rWVQTjmSrIlO73Fhv+6GflPyQoVeutKEF +UO3Ar1H0AYtbcnUruPcHBBDQgMTrk6UOF1LM5U0wxwbmmnkEXeEtLeGUxv13JUMe +QTrSw0P+hn5uiDMwY5lI212ayorbwyNuzU0hGW7j98qa/S9MVSeRwv4bsbtBqwtT +ZVGL2TogjeJ+8+qDlv5tf3crD1pV0yu7uVUeIGrJZJdiiMM6N7/x3cIJtrLDaF1w +9kUqdQGWDWwRV3w0FaeXDVlUTJ3VFTZIinLo1Vmd3AXLn97eVy+68DWD6ELlMkfF +ro3bQbQu59DGswDF9+mMauhgvljnxY/wUnkyp8LGZt8w/RU/op8LcvPKFjQfX6UC +gcEA/RClguU/JTWQSF4Vjy5+DmrzSCXabrO7986lkOwPXEaXLsF2pGOU+sbxffXZ +PCQ4zU6aNYBMtNWLrVqBrPnI78XZQbmSG7zUDxWwmr1ttuSHGthdaZUVTOsFsnpE +R+1lj20G2T2ankzv0ICFz7BWyaYwyzDrABwuP189b3juuX6xGLlCO25/AnK+iTXQ +WF4ZURZiwDg4WpxvPsE2FFglN09TtDDCeQw/zv7Jh6jWtHZfivfv/wR8ILu72gde +LdinAoHBAPD5slojp2St6iWTaxcYYBc+4x+Rv7oyxeAyM/00iIAnf19oaAw3jki1 +yz6zeNnhaZuwa6ruvhJAESPc+pXkmoPpQ1yIRjKZc3jPVoaxYuSPgUEbHV6BQbjD +clIoGkC6ISsjanMq8Z3pK7e2bxY4mxGRQIFSFnTeK8m8nzq5dYisVtxxJFnV2fDG +NWDMdDYBh52vpkPjnm5OoOuilkEH2ygtZlT6bKAbnhi/FVjYCHk46inaYx8PakMu +LnvZI0OIhwKBwBGzYGBPeKM5o+Xr7sYdEmQfxvR88VJc8ADdS2dfm5NwvJJgpdPJ +w1nnIG0XDSLPxclWfiLP3o2ngiWV9wwKTKu4wwF94WJfStXjRn8MUOhCA9E04RPJ +gbvnlHZvZudBC6GElr4LOQ1phDypQLLOOsPQBAmyWj2fuvxjxQBPDSOcYPbBvog5 +qliZfgpK4U/NBShO0IlxZT+xQXa6PPYfVDsSKWCpKHEfEjeASshaXuowfW5S+U51 +GdmQSAtwCH5ccQKBwQCOLgW5gYfms2aPvSdWfR9VF9nSaqCBMCvoWDasky5mzucs +V+HsM2tUI09EM4h+pa02GyWruSmUgxCZ5GxFvJgedKc2FYG1oSysf0lCN69tw+4z +h9gQRpuMdGUjbF3xCuE/HqpUQWZGEamlv5JTvhpghx9ULibp1Zxob05Ty9E5TtYB +QxB7oN3yXkBoWLnIk6Z8t4KWU9rKosH3xfp5bDU2w3K5ePhWj3T8jOH/hZeaTqZ7 +A0uwq9u6v6jVkgxocEkCgcEAhyMPyiMqqq55qD6JlPY4zf4mWQJS4MnmWexH/5N1 +a6uogerwZnIjuH0/wNc2qzQSOwVJRqjlxRZbEkWST0JK/meXZ+e0nqaROWICutO6 +ciIf2wVjTzn1f85DHwPhUf3P2zIfpVFh+XuvSb32J+pzNwK40gMhirYMDSbsm/RQ +JrJHQwEX9BvI6IP5kaMOMlSwiG3Soo3MxeJWmozkz09+cs7BwMFsUKOdqruJHSaw +/Q06DQ9u9UyjmLyKv3IkN/cK -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:c5:b4:b3:ad:79:09:50:a2:33:7b:ce:d9:2e:c7: - b9:b4:86:61:8a:90:fd:61:87:48:5b:2a:a4:dc:0f: - ce:e3:90:b0:2a:03:06:56:7a:78:d9:32:30:9b:6d: - 9d:ef:3f:e0:0e:6c:22:24:60:82:b3:3d:76:3b:62: - 2f:87:e4:cd:ce:da:65:c8:0f:1a:07:1a:32:ff:19: - 6b:68:13:20:cd:5e:f9:bd:50:2f:8c:46:e4:8e:ab: - 63:1a:2c:71:d0:1b:44:71:d9:3c:5c:62:b3:15:50: - ce:db:f7:7a:e9:37:52:c6:46:b0:af:d6:02:12:5c: - ce:04:bc:42:68:a0:04:7b:01:a7:63:d2:91:bf:a3: - 47:ea:0c:07:45:8c:7e:90:65:b8:67:d6:26:65:7b: - e7:86:6e:37:1f:76:01:ff:1f:41:7c:76:7d:4f:3c: - 6a:15:88:75:e2:71:bb:10:e2:40:7a:d3:4d:8f:7f: - e6:97:3f:86:f6:40:a0:1b:7e:5c:8d:1c:42:87:3c: - 71:0e:39:35:eb:16:33:1d:73:e6:ea:29:6d:b8:7a: - 91:2c:31:92:39:f6:ac:7d:85:4f:f8:46:fa:77:fe: - ff:af:13:04:cb:ec:d9:bb:8a:5a:6d:cd:99:f6:72: - f1:82:5b:7f:4b:0b:49:cd:60:b3:b3:90:ed:c7:f3: - 1c:00:ef:98:7a:bb:29:1d:07:e5:8e:fe:fd:82:28: - 82:bd:33:f6:4e:fb:d4:0d:ae:66:a8:91:f2:83:6a: - 60:a4:df:d6:fc:86:50:b0:34:be:8b:b1:5a:6f:77: - 75:d3:4b:95:a7:e1:85:a7:f7:61:2a:8a:5b:54:24: - 9a:0f:a3:ba:5c:ed:54:dd:58:28:38:cd:51:67:e1: - 0f:19:87:83:45:36:7f:ea:4c:1f:25:10:c8:2b:b0: - d4:d5:e5:cc:99:ef:38:68:06:25:0c:48:ee:39:6e: - 6a:36:36:c9:c4:ed:c7:bf:25:48:65:a9:ee:6e:87: - 43:d3:d4:c2:01:81:35:7e:a0:33 + 00:ee:36:70:aa:61:6a:20:bb:32:88:07:73:f0:3a: + ca:11:cc:8f:30:80:d7:71:c7:74:62:20:94:88:b4: + 52:12:94:08:01:e1:79:18:cc:f0:69:8b:99:c5:47: + c1:82:aa:ae:c8:51:04:3e:89:8c:1d:65:ad:f2:1f: + a5:f5:d4:87:65:ac:2c:78:80:37:50:15:21:4e:7a: + 09:c6:e3:5a:2a:bf:b1:b0:04:41:6f:ab:ac:65:a3: + 0a:68:68:6c:3e:e7:ff:3f:b0:95:ff:55:4e:38:ac: + bf:7f:7e:23:15:4d:b7:a1:32:13:12:ff:3e:67:4d: + 26:12:00:25:7b:b1:15:e7:c1:6e:13:ea:5d:8e:77: + be:c5:cc:f0:63:9d:a1:45:0c:da:f9:18:8a:22:0f: + 89:8d:48:9d:f9:b7:4e:6d:96:21:9c:cf:42:32:13: + 94:c6:47:a9:08:10:53:48:1f:9b:61:78:7a:09:27: + 82:42:e2:e2:f2:58:9c:1e:26:9c:43:c7:5d:27:8d: + 64:70:28:df:ed:5e:6f:05:9c:74:0f:31:10:14:97: + 6c:0c:d3:b4:19:10:8d:72:7f:35:2b:d3:4f:e8:1b: + 67:69:3e:73:f2:36:1f:ba:a7:69:5d:f0:94:b4:83: + 6f:cf:77:6a:50:2f:c3:ad:98:cd:9c:0d:c3:79:13: + ea:82:11:6d:54:54:82:79:ff:f6:9d:f2:95:1c:a1: + 0c:32:44:80:51:38:96:da:92:ab:fa:15:c5:53:76: + 04:e8:82:8a:b9:56:ef:93:e9:35:45:2d:22:12:bd: + 5b:e9:b1:5d:0c:b5:3f:c7:f7:ff:53:b4:29:ea:3b: + 09:62:13:cb:3c:91:bd:cd:8d:04:fc:7b:bd:95:1c: + d7:78:dd:b5:4f:13:80:de:43:e5:e5:6c:ca:0a:cd: + 5f:86:3a:64:87:49:fc:d2:39:90:7d:91:27:12:98: + 67:79:60:3f:3b:f3:74:db:2f:cf:fb:36:7d:44:45: + 07:56:a9:a8:b1:18:8d:fa:f8:11 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,75 +90,72 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 85:75:10:25:D0:2C:80:50:24:1A:5B:57:70:DE:B5:CB:71:A9:3B:7B + 16:22:C9:AA:20:D4:F4:8E:ED:9C:A6:BB:23:F5:F2:29:44:ED:F2:1E X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 95:f3:56:bb:d5:8c:70:bd:d1:de:da:63:b0:29:d7:db:60:27: - d6:59:fd:61:1b:30:c6:d0:5d:73:7d:34:e1:68:e3:28:a6:89: - e6:60:bd:89:d3:0e:f4:72:ad:72:76:f8:86:21:fd:75:3c:f8: - 6d:be:9c:04:e1:82:03:69:6c:ae:d0:55:ba:5e:f2:ca:f5:0f: - 8e:d6:d9:8d:c8:56:46:f4:f8:ac:74:2a:19:7b:8e:47:70:1f: - fb:fb:bd:69:02:a1:a5:4a:6e:21:1c:04:14:15:55:bf:bf:24: - 43:c8:17:03:be:3e:2c:ea:db:c8:af:1d:fd:52:df:d6:15:49: - 9e:c2:44:69:ef:f1:45:43:83:b2:1e:cf:14:1c:13:3f:fe:9c: - 71:cb:e7:1b:18:56:36:a7:af:44:f1:0b:a1:79:44:46:f9:43: - 46:29:d8:b0:ca:49:4d:65:60:d3:f6:8e:74:bc:62:9e:1e:8d: - 4b:29:9a:b4:0d:f0:a2:77:5b:34:e4:11:2f:a7:25:c5:e5:07: - 76:12:ae:be:75:73:15:e4:0a:7d:53:38:56:3f:79:6d:6e:ca: - ed:80:ab:56:ed:7e:8b:1c:e7:e3:d4:62:30:22:70:e7:29:b2: - 03:3c:fe:fa:3d:f0:36:c0:4d:11:a2:99:d3:29:31:27:b8:c5: - b8:15:a3:3c:4f:9b:73:5e:2b:b2:fb:cb:fd:75:47:b8:17:bd: - 21:d8:e6:c1:b9:ff:73:81:d8:25:08:6d:08:5e:1c:a5:83:50: - de:67:e6:da:d0:8e:5a:d3:f2:2a:b1:3f:b8:80:21:07:6a:71: - 15:6d:05:eb:51:b3:59:8d:d4:15:46:7e:02:a8:13:01:16:99: - bd:03:cc:70:71:2a:23:16:78:af:d1:d5:01:9d:04:b4:63:93: - 9a:04:3a:92:2e:e6:7e:73:93:a5:fe:50:9b:bd:0e:ea:54:86: - 6f:7c:e5:14:77:fe:c2:28:5a:4a:0e:d7:2d:8c:e9:ed:61:29: - b2:53:ff:6c:04:bc + Signature Value: + 43:82:1f:18:b4:cb:7a:b7:4e:ac:c3:65:2e:b7:57:3a:f0:78: + b3:b2:20:3e:bd:7b:24:f7:e7:17:cc:62:c0:b8:63:f0:2e:30: + 0c:54:be:c9:94:7c:54:d4:40:06:6e:92:9d:72:2c:44:f5:e3: + c1:4a:3f:26:e4:21:ca:ba:ef:4f:a3:50:56:ee:89:5c:dc:5b: + 98:bc:0c:e9:db:16:98:f0:4c:3f:65:b9:d9:e0:2e:bd:83:9e: + c7:1b:f7:4f:87:5f:75:ff:67:5d:0a:ff:ce:fa:0b:f1:d5:8b: + 58:23:2b:92:27:b8:15:ff:66:c2:f9:68:e2:ed:64:a0:4a:e0: + 42:8b:ac:35:6a:67:64:f5:cb:a9:00:37:20:56:dd:3a:02:6e: + d8:0f:b4:5d:ff:92:a3:0d:65:fc:2e:ab:4e:37:bc:7e:d6:5b: + 1c:55:78:a0:6e:a9:2e:55:31:cf:61:f0:1b:4c:fe:fe:60:75: + 76:e8:d9:a1:f5:14:f0:43:33:4d:52:05:be:33:ff:da:31:94: + bd:0d:e5:4d:a8:ee:d1:f3:92:34:c2:36:91:37:3a:6d:ec:20: + df:b3:9e:a2:5d:d0:f0:a1:e2:ff:b7:85:ce:09:e3:c8:4a:6a: + 93:a4:ae:ec:c1:67:42:f7:8e:2f:64:ca:d7:76:72:bf:3f:ef: + 64:ff:10:dd:a2:3e:2d:f5:aa:02:cf:31:f0:6d:93:6b:56:4c: + b0:88:95:54:dc:d4:ac:dc:c1:88:27:8b:d9:9c:4e:66:93:f8: + dd:5e:74:41:26:9a:ef:e8:c8:46:5c:d5:f4:fc:cd:cc:67:c7: + 33:62:2e:70:9b:d1:39:de:34:fd:ad:9d:b9:48:61:e7:89:53: + 62:62:f0:40:6c:68:2f:e1:da:40:61:2a:50:af:c9:41:6d:98: + 16:af:cb:63:03:ec:04:99:e0:39:98:60:ae:aa:6a:57:9b:25: + 73:03:16:bf:78:c8:d7:2f:eb:fc:64:7f:c7:78:8c:34:15:5b: + 66:40:98:86:e6:0a -----BEGIN CERTIFICATE----- -MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAMW0s615CVCi -M3vO2S7HubSGYYqQ/WGHSFsqpNwPzuOQsCoDBlZ6eNkyMJttne8/4A5sIiRggrM9 -djtiL4fkzc7aZcgPGgcaMv8Za2gTIM1e+b1QL4xG5I6rYxoscdAbRHHZPFxisxVQ -ztv3euk3UsZGsK/WAhJczgS8QmigBHsBp2PSkb+jR+oMB0WMfpBluGfWJmV754Zu -Nx92Af8fQXx2fU88ahWIdeJxuxDiQHrTTY9/5pc/hvZAoBt+XI0cQoc8cQ45NesW -Mx1z5uopbbh6kSwxkjn2rH2FT/hG+nf+/68TBMvs2buKWm3NmfZy8YJbf0sLSc1g -s7OQ7cfzHADvmHq7KR0H5Y7+/YIogr0z9k771A2uZqiR8oNqYKTf1vyGULA0voux -Wm93ddNLlafhhaf3YSqKW1Qkmg+julztVN1YKDjNUWfhDxmHg0U2f+pMHyUQyCuw -1NXlzJnvOGgGJQxI7jluajY2ycTtx78lSGWp7m6HQ9PUwgGBNX6gMwIDAQABo4IB -wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV -HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E -FgQUhXUQJdAsgFAkGltXcN61y3GpO3swfQYDVR0jBHYwdIAUs4qgorpx8agkedSk -WyU2FR5JyM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst -gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 -Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw -AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD -VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAJXzVrvVjHC90d7a -Y7Ap19tgJ9ZZ/WEbMMbQXXN9NOFo4yimieZgvYnTDvRyrXJ2+IYh/XU8+G2+nATh -ggNpbK7QVbpe8sr1D47W2Y3IVkb0+Kx0Khl7jkdwH/v7vWkCoaVKbiEcBBQVVb+/ -JEPIFwO+Pizq28ivHf1S39YVSZ7CRGnv8UVDg7IezxQcEz/+nHHL5xsYVjanr0Tx -C6F5REb5Q0Yp2LDKSU1lYNP2jnS8Yp4ejUspmrQN8KJ3WzTkES+nJcXlB3YSrr51 -cxXkCn1TOFY/eW1uyu2Aq1btfosc5+PUYjAicOcpsgM8/vo98DbATRGimdMpMSe4 -xbgVozxPm3NeK7L7y/11R7gXvSHY5sG5/3OB2CUIbQheHKWDUN5n5trQjlrT8iqx -P7iAIQdqcRVtBetRs1mN1BVGfgKoEwEWmb0DzHBxKiMWeK/R1QGdBLRjk5oEOpIu -5n5zk6X+UJu9DupUhm985RR3/sIoWkoO1y2M6e1hKbJT/2wEvA== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem.reference b/Lib/test/certdata/keycert3.pem.reference new file mode 100644 index 00000000000000..84d2ca29953e46 --- /dev/null +++ b/Lib/test/certdata/keycert3.pem.reference @@ -0,0 +1,15 @@ +{'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), + 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), + 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Oct 28 14:23:16 2525 GMT', + 'notBefore': 'Aug 29 14:23:16 2018 GMT', + 'serialNumber': 'CB2D80995A69525C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert4.pem b/Lib/test/certdata/keycert4.pem index 1003d67fd075ec..fcaa3a1656d1a5 100644 --- a/Lib/test/certdata/keycert4.pem +++ b/Lib/test/certdata/keycert4.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC34y3S6iXdmdvd -M/2aFBe6CvRvZwhh1huGl7IQRtdoakPqMLlEdNHJtNeF5M27xLei+p4wt7N1Jyi0 -2keHQb1m9TqH5AruOkE2ti+15zEoKoU9aWydTiH+epKTT0yjg2NcKQjRUaWcbhzB -H4EMKuCIlzIIz8/EIKkOqhCDwq6+Fv3Ays+z7Bz+yR80ixivKu/l7SjxQ7z7R/kC -I7OViRcIO5QBQPj7VLvCTz4VA6u/LdXngK2HNuau6WXm5yNNQbqrB11AEJcYZf/c -VrneV4F+ZjLloAKgSn9GB8eWOyilTQ18TcKd+H2icipRaP/+QR/KPx5GK/SXU3my -qm62QOGI7t/5ktVdjGhs6tHZxw1SRiipiLYWbtVRrSxa4wYlgpgoUwvrvvtC5kAN -nTw1VGWsxcs+6a7+PocYnJiq7k4b5OAUb3Ryvl9DLAMy8NqpRWo4cHD/XQ3FCYwF -HlOSgx/dL5Se0i3dW1KzbP6OvaNg6nl/1EXPUsJ1ATS8nzvzhccCAwEAAQKCAYEA -nD3GvaJ9MeB802JNZBEWZ9jO/6jHknldQeq6POI0PF+t/NoRUH0BkyS4yucxdw0a -CrxulG5BaJUxHRkqFV5iE4zhgnzcXLXamyYJO8GIHtyiASAGTVIJyDNVPxztvTDx -x2iGOXPqBxP4Eo82EqSLywLMXHhVzAsEGZWeGpXb61+Vk62+9Nz1dfZlMTvOaWdO -Fkp/sx8e/1KT3KGBANlOXIxioP4Xj1Tbg6nY0fogf3vud5j52B1pu8xL7PkPIaFq -DEGz3XvWhBF/+Cs5iDeYz8eQpfQig7HdHVn2D8dZmzQgpLw1yGbPAnqrgopWfm7R -MqiyFe82p2t+vfSoG5jz28XxPtzBJV3ljxKxlbnclqu/CAYSjzaYohDzyhjdZOZI -r9DOfWOqu01Ha3EEsApn95fusHHGTH2FOy0u61FSTrfLfqsLw9WRJPWleirKikhf -SZzi223QrmzZMtuCF7VgTx3ghDhBmFD8uzVVQ1SwPZ8CgftRkFcn1llXIAfJ3iHB -AoHBAOg3DOIdtUVgpjMKhpAyuH54fYvGl7afIMNbKRle0kCiP45wtGJ43RPMqiR8 -1rxZB3+iapICI/lnhk3O7vVRkR64yiqQBcl/hXZ1BhyD6iDXWYmm5mcnymcoqfwc -p9TfzEPyGPb3SM2YlI0cSPRqM/jDvGvnDeKIpzEKvUlwJ59WoN2HOHTIXf+XbN5n -unpuTt6YKJvc48DrXsPnUzkCmUfbOmgHfeb9/qBs/8kY4YJMsZEjqf88o7mCJCIy -BtDxTwKBwQDKuOwE8e0GIA01ZHd6RfR+ZCvmp2oauxal4EJsBx+ZZnhEWGaSm1fE -Bf/ih074ghcSKoSrdYpD1xGZ6fGVWMx3jcL11yLDOUiiPDJsm8hUBZ0IW1qXyfCP -l7xy1bUkWwPXdmFuGp1exrcjooKrFNuTdYiK4nQZSKuCfXQRADrmEJmM+gYwhqI7 -4XsYo848B9A4hbY6RLEox4uvo/RmafY0iR0PMhVEc+ydNLKB/4LpahZqBQ4kTpMv -o4+rEvYt1gkCgcB08gx177ozx1nMCLf99N0/LBUmCIytNvR8DfPjyAIg9NUHOjFO -CkpkR0VEfO50Cm4hVD1RbOyLFRzpIJbtSvfHvg5qYv/XG3auUn8Sa0jE408/aKNO -PhbL3wnEYvYO2ep4KXtzHNQ4XmgprJ39IWMtG/5PZRx0ApgYtazgSDBcKXd4OTow -bhwQtUTpuNmMAPONXJnO7O5yYNbn2B7sbiedrYV7kJJSe4X5awtiTjp7sX4XdxuM -5BAcQ7NI2WLfZTcCgcBp/X9hIoATmMRvKwUQx+yJ/KO7Z8KhETpJJdR0mNDbqmit -Cy8t7cxYb+6WqLoQUivv0o0k/EJ7L8JDH76woAnfZB4P3RiOy69/K0wN3vFBhOHS -kbju7aU53lKoE7YuuOtsRrewEng/KlRsbDY3bqNTGLt4KegbpBQQGLmLffxNd1Zh -EAQWcP33ou9yNYrJdihWtQpOssWRlash/O32ceZJF3s7C6t068tFclz2fPocQdxQ -OC5pqy9nU/P0tOhDlMkCgcEAosaBJLIeAYlOU0+2uSx5g5mIqOOTyrDEmqqad6T/ -wkB7vW2QaoDvLL22Yrzdn9vQ0V0rqzhVtan7sq5pn/BQJAueZYN8rFxS3uuW+UQk -Nsc4GLJzU8Az/2DvqEIrnE7zRc5E1FOI9gKLrBlpJB2o0hVcBznDe05Gax6Kjqbm -jHqzyU73SpxpEy3OesClCeCQIMr47HaL9aSqaEX4U9bMpgHi0HgTTHqvJ5pch0hY -dYl+WAE9LAyF1DF29BirEXVw +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC/uCI/4eVSCIrx +//GcCCxeQP16Mtf0wAuINBR9uTySF8sEVoS6rNBwLeudZXaHmcp3jM7N2e9GjI8d +dQccPApKBwjAxhdT4XeuUTep4VraKWjLTWDvABHfCVFB1tX3HusEWaS8n/pH+N4L +UfnxS3HnFqGC5ib2DSgFTCnLQSj82iUgACdf87iZTons+kRvqYsdwpsgBUnO+w+o +c5o7Nj1bJJlTREsY6kQGY94as4pCkmqhoBTC7/le9wFGmXipRYi3pfSTOqAQJfqW +1VlQ4Bbc3lN5L+ONTmFXOgTJ5qhaS8cglnYElJI1QQTagyWQMHuBDIMpr3Iv/5qU +UzdIMzoMrpX6ien3Gan/jyjjzvP95jJvGujQudw/myXK8VOUThVtstRDFA3cIoV4 +UqMODz8w6qYU0GPYFBq//1UdpDrk/fBPBe3MLJT2+sB/kTzwxxYsKqcdjPG2Bt3Y +2nk7hCzE0KuueqCQPk/3fomY1TDyTlWOriAS40EYue5Rhx9qr8kCAwEAAQKCAYAL +X7EoeQ5Rv4/+q2B6EUIQlWp0RW/qZTpJ6k0M6GBfimnQ6BEXwgjbnt3LiKlvggAw +93mNXNCFLvGOndK+KhGyMpiiVx4rK8Ud4lObEHODXdGJvh1yEF7/DF51uXkYIA1x +RKAxUIxYmLsTkNlzJzaqrv0F9wF4t28YYVxZYpQ76/Un4Np1JtBcx/wGwxIsTbKj +IVhynd2zGdHj/He5643YSmOOPQ73e25tsp8KlnwtVuJclFKm/fWdXKoGtjQiVq2b +bViG7WTqb+a+R3IrO8CE2vAQ1ZqklP4Pg33AU1XFogRweqSkXvBGy1GHSSXCO4uX +gORj/bo/gAZ/eeufwqxBxJMS4Kbmxes/oYEgJidoaeA6DZjcKr+J3rFUt9j6rpxt +abeKBiK6wTHkKAdQwoccDdKFROvL4ZEu4zpjkmiqDrtIZEqtB8YCvoMpteIqrBhr +stLCDrftsysBTNGNn0THb4H5wx8nkZPCVyvaBD4N4Skp5rapP2bbrPhigcOoR8EC +gcEA4ToVg2b09tLWpmCg6XXssT1SvvFcSaJ3jjRKMfpmP5epWFiZ21qAOCbYhMNu +2G21AyNd1W7oVi6epXsB2YBOlSI0Fr6RuN0fDwg5jZLxEkEqDsL2QKsgoKHtGfBn +gX9jLQ5Lsy5/0CRXYCfPjdccbr2J5Gm2fU/jhhCi9ji/6b2RbgrRkIkr2I/J6KbS +sO+p1d93CFNagLMbdDspugDlIgj52OBMYO60nDlNU8O2pkAcLQTpWVqEjmoo4Q4K +oyOXAoHBANnqB93sLCndEiZNth7cOOEhNKCWRDkpGwDBbn2Dj3kBaXg8eCTG7tiF +FPyNVydYg+u0vcPIjj9URsOfpHc9MLSYtKAgCL3rSk+OQnROeg9hCjmB5Ucg7NTR +gSh/flfWp/q1EDeOVAu8B9DuBX1209br4xi78LnBP5/UzRjhuFEmvCvOqVCMCI7Y +I1k3l4NhOk87fKq5dbtxvNTU2eFTtt1+YjYwZFY+AUmx3WBz+aFwygks3S4SvN1J +LtLVYhOznwKBwQDC047WixIuDLX3WDD5orOroeNZHsn5PFv1HBBuaS9XpSatMH9u ++ztc12WGetQAze2+GDLMNNMv8cX0WZKBBfd0FBFA93pwkn6Sb0fxyoFUjCAIguen +iyB/M3M5c/blUz+EMxCSoA+aCkW2/NkS1lhXBwgoGLXuclPbnbqKCQ8h74TEzwD2 +6WGPRNqgsOYifj7IrjR2dDwehlCiW6c9qhaLOX5+94+6beK4HO1iHzN5Xo3A97Wv +QJjX5McV3yKeemMCgcA4N3reEo52IlULUqL4JSH7WkCkaP+iq2sO79fcQ3Ng6S9X +WGo6OqPlcbevS5s/SEOILDGEb5na1pgG4YlhRYTqIjb+1CTNMgUSrwWP0asFiqhD +m7IVfnX6lS23z+Q9LuBY+hr76hjeihyOFsmNy3jtCh+lAt8gXK1YQ2LB14FgVhjX +SFI/uFCA4VuFKaVJvGx5gkQwGvY3bCkl0t9+lMUpMPCPQD6yTP6yD1OoDWNJ9bn5 +UfyhZS4Z/EY7F9dcc8sCgcEAi8yW4gNCZSsGQRLh+fyDCSdak4ahATAeiI24231u +jK5+ncK9ZoZvzO/h+V5bw7ha59rD8aGmrLqzBVKEMi0PAT38psNH9ZtRt9y29YQ8 +WpyHtODMbphfbH4SFH3nj3PldRtBcOt0EDdp/4s64A9qdOWRx/vlJhdg0vtWTy5s +koyyOHhTcplpyPPq0BetYKl9UlXPVjus/RT1XV/lz2YrE8E9UAq+/r7ECoMExaZt +8j8J8O3lTWPxNTaQuiPiHA7E -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:b7:e3:2d:d2:ea:25:dd:99:db:dd:33:fd:9a:14: - 17:ba:0a:f4:6f:67:08:61:d6:1b:86:97:b2:10:46: - d7:68:6a:43:ea:30:b9:44:74:d1:c9:b4:d7:85:e4: - cd:bb:c4:b7:a2:fa:9e:30:b7:b3:75:27:28:b4:da: - 47:87:41:bd:66:f5:3a:87:e4:0a:ee:3a:41:36:b6: - 2f:b5:e7:31:28:2a:85:3d:69:6c:9d:4e:21:fe:7a: - 92:93:4f:4c:a3:83:63:5c:29:08:d1:51:a5:9c:6e: - 1c:c1:1f:81:0c:2a:e0:88:97:32:08:cf:cf:c4:20: - a9:0e:aa:10:83:c2:ae:be:16:fd:c0:ca:cf:b3:ec: - 1c:fe:c9:1f:34:8b:18:af:2a:ef:e5:ed:28:f1:43: - bc:fb:47:f9:02:23:b3:95:89:17:08:3b:94:01:40: - f8:fb:54:bb:c2:4f:3e:15:03:ab:bf:2d:d5:e7:80: - ad:87:36:e6:ae:e9:65:e6:e7:23:4d:41:ba:ab:07: - 5d:40:10:97:18:65:ff:dc:56:b9:de:57:81:7e:66: - 32:e5:a0:02:a0:4a:7f:46:07:c7:96:3b:28:a5:4d: - 0d:7c:4d:c2:9d:f8:7d:a2:72:2a:51:68:ff:fe:41: - 1f:ca:3f:1e:46:2b:f4:97:53:79:b2:aa:6e:b6:40: - e1:88:ee:df:f9:92:d5:5d:8c:68:6c:ea:d1:d9:c7: - 0d:52:46:28:a9:88:b6:16:6e:d5:51:ad:2c:5a:e3: - 06:25:82:98:28:53:0b:eb:be:fb:42:e6:40:0d:9d: - 3c:35:54:65:ac:c5:cb:3e:e9:ae:fe:3e:87:18:9c: - 98:aa:ee:4e:1b:e4:e0:14:6f:74:72:be:5f:43:2c: - 03:32:f0:da:a9:45:6a:38:70:70:ff:5d:0d:c5:09: - 8c:05:1e:53:92:83:1f:dd:2f:94:9e:d2:2d:dd:5b: - 52:b3:6c:fe:8e:bd:a3:60:ea:79:7f:d4:45:cf:52: - c2:75:01:34:bc:9f:3b:f3:85:c7 + 00:bf:b8:22:3f:e1:e5:52:08:8a:f1:ff:f1:9c:08: + 2c:5e:40:fd:7a:32:d7:f4:c0:0b:88:34:14:7d:b9: + 3c:92:17:cb:04:56:84:ba:ac:d0:70:2d:eb:9d:65: + 76:87:99:ca:77:8c:ce:cd:d9:ef:46:8c:8f:1d:75: + 07:1c:3c:0a:4a:07:08:c0:c6:17:53:e1:77:ae:51: + 37:a9:e1:5a:da:29:68:cb:4d:60:ef:00:11:df:09: + 51:41:d6:d5:f7:1e:eb:04:59:a4:bc:9f:fa:47:f8: + de:0b:51:f9:f1:4b:71:e7:16:a1:82:e6:26:f6:0d: + 28:05:4c:29:cb:41:28:fc:da:25:20:00:27:5f:f3: + b8:99:4e:89:ec:fa:44:6f:a9:8b:1d:c2:9b:20:05: + 49:ce:fb:0f:a8:73:9a:3b:36:3d:5b:24:99:53:44: + 4b:18:ea:44:06:63:de:1a:b3:8a:42:92:6a:a1:a0: + 14:c2:ef:f9:5e:f7:01:46:99:78:a9:45:88:b7:a5: + f4:93:3a:a0:10:25:fa:96:d5:59:50:e0:16:dc:de: + 53:79:2f:e3:8d:4e:61:57:3a:04:c9:e6:a8:5a:4b: + c7:20:96:76:04:94:92:35:41:04:da:83:25:90:30: + 7b:81:0c:83:29:af:72:2f:ff:9a:94:53:37:48:33: + 3a:0c:ae:95:fa:89:e9:f7:19:a9:ff:8f:28:e3:ce: + f3:fd:e6:32:6f:1a:e8:d0:b9:dc:3f:9b:25:ca:f1: + 53:94:4e:15:6d:b2:d4:43:14:0d:dc:22:85:78:52: + a3:0e:0f:3f:30:ea:a6:14:d0:63:d8:14:1a:bf:ff: + 55:1d:a4:3a:e4:fd:f0:4f:05:ed:cc:2c:94:f6:fa: + c0:7f:91:3c:f0:c7:16:2c:2a:a7:1d:8c:f1:b6:06: + dd:d8:da:79:3b:84:2c:c4:d0:ab:ae:7a:a0:90:3e: + 4f:f7:7e:89:98:d5:30:f2:4e:55:8e:ae:20:12:e3: + 41:18:b9:ee:51:87:1f:6a:af:c9 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,75 +90,72 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - C8:BD:A8:B4:C0:F2:32:10:73:47:9C:48:81:32:F8:BA:BB:26:84:97 + E4:3D:4C:DB:96:85:CC:90:ED:0F:58:1E:8D:4D:7D:34:6C:4E:11:10 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 76:87:76:4d:e4:0f:88:bf:2c:f3:58:67:c0:97:6c:cd:59:18: - 82:83:4c:04:19:a5:6d:aa:fa:64:3d:49:32:3e:e1:56:95:b2: - 13:f7:cf:d3:11:b0:72:b7:5b:e7:d7:85:69:51:3c:b6:54:80: - 45:2f:28:10:21:20:b9:ba:e9:27:5a:b7:3f:82:b7:69:f5:46: - f5:bf:a2:8b:17:7f:f2:14:d1:46:97:b5:8b:47:fb:9f:e8:5c: - 05:0e:9d:11:bd:7c:9a:03:84:0b:ca:29:66:4a:ca:0d:6f:09: - 1e:7a:27:c1:7f:03:96:70:8d:18:a5:2f:a4:98:a5:19:aa:8c: - 5d:1e:8c:3e:bb:6d:3b:c0:33:c0:15:e1:bd:09:3d:9f:e8:dc: - 12:d4:cb:44:1d:06:f5:e8:d6:4e:a1:2d:5c:9f:5d:1f:5b:2a: - c3:4d:40:8d:da:d1:78:80:d0:c6:31:72:10:48:8a:e9:10:7a: - 13:30:11:b2:9e:67:0e:ed:a1:aa:ec:73:2d:f0:b8:8a:22:75: - 0f:30:69:5c:50:7e:91:ce:da:91:c7:70:8c:65:ff:f6:58:fb: - 00:bd:45:cc:e2:e4:e3:e5:16:36:7d:f3:a2:4a:9c:45:ff:d9: - a5:16:e0:2f:b5:5b:6c:e6:8a:13:15:48:73:bd:7c:80:33:c3: - d4:3b:3a:1d:85:0e:a4:f7:f7:fb:48:0c:e9:a0:4b:5e:8a:5c: - 67:f8:25:02:6f:cd:72:c1:aa:5a:93:64:7c:14:20:43:e0:13: - 7f:0d:e1:0d:61:5e:2e:2c:cd:7a:2e:2a:ae:b6:75:6a:5f:a0: - 1a:9b:b6:67:2d:b0:a5:1c:54:bc:8c:70:7e:15:2b:c0:50:e3: - 03:bb:a4:a5:fc:45:01:c9:3f:a7:b8:18:dc:3e:08:07:a1:9b: - f5:bd:95:bd:49:e8:10:7c:91:7d:2d:c4:c2:98:b6:b7:51:69: - d7:0a:68:40:b5:0f:85:a0:a9:67:77:c6:68:cb:0e:58:34:b3: - 58:e7:c8:7c:09:67 + Signature Value: + 63:e9:cd:0f:ec:dd:27:a0:fc:76:16:a1:f2:5f:d3:07:9a:3f: + 97:f4:11:e8:20:e9:3c:dd:16:4f:20:62:71:73:a9:46:3e:85: + 6e:44:be:9d:08:4b:eb:80:08:d5:1b:93:5f:4a:0f:df:b7:1b: + 38:e2:2b:5c:68:48:ea:0a:58:5c:36:6c:79:a4:1e:b5:7e:ef: + cb:8d:5d:bd:7d:4a:e6:4e:dd:0b:87:10:ff:01:0e:9b:8b:bd: + de:1c:9a:25:fb:a2:e1:52:7b:8a:aa:08:37:b2:87:f7:45:9d: + 0b:ab:11:6b:0c:7f:db:ed:de:cc:1e:86:f0:be:30:25:6b:4b: + ff:f5:f2:99:ed:b0:b1:68:c1:44:0b:79:7b:81:95:b5:36:37: + 12:c4:99:9e:85:e8:2f:2d:cc:cd:d8:c6:f2:20:ec:6a:06:fd: + b8:fc:ff:02:11:95:4d:38:0d:42:8b:b2:43:eb:c2:b9:a4:e0: + 33:f1:da:25:2f:11:cf:57:b9:25:e5:ab:92:f2:b2:b5:11:2f: + bd:31:f7:55:eb:96:94:78:5a:ad:9c:8f:56:ec:34:a0:73:9c: + 01:c2:76:24:e0:f8:b3:c9:23:b9:ea:ab:c3:0a:d3:f8:53:44: + c7:12:37:76:44:d7:ee:25:93:f4:1c:1d:7c:fe:06:2c:fa:c5: + bb:c6:90:a4:57:fd:09:2e:da:af:f3:ac:e6:6d:7f:03:a1:26: + 36:9a:51:7d:8d:28:a7:5d:5d:37:57:cc:6a:11:2a:98:1d:57: + 71:32:6e:98:7c:12:28:ef:5c:2f:26:29:d4:56:0a:b1:23:d9: + 9d:35:20:1a:ec:cc:51:53:6e:ef:1c:e7:bc:3c:21:e7:64:31: + b1:4c:f3:55:b5:c4:93:17:d8:72:5a:05:2e:4a:e5:33:07:0e: + 7d:cf:22:73:0b:67:9f:b4:82:60:cd:71:f8:76:0c:c4:dc:98: + ee:49:f9:03:7f:0d:d8:c6:76:79:3c:28:ed:77:77:d3:7e:d5: + aa:1e:6e:2e:df:a5 -----BEGIN CERTIFICATE----- -MIIF9zCCBF+gAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF+TCCBGGgAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh -a2Vob3N0bmFtZTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBALfjLdLq -Jd2Z290z/ZoUF7oK9G9nCGHWG4aXshBG12hqQ+owuUR00cm014XkzbvEt6L6njC3 -s3UnKLTaR4dBvWb1OofkCu46QTa2L7XnMSgqhT1pbJ1OIf56kpNPTKODY1wpCNFR -pZxuHMEfgQwq4IiXMgjPz8QgqQ6qEIPCrr4W/cDKz7PsHP7JHzSLGK8q7+XtKPFD -vPtH+QIjs5WJFwg7lAFA+PtUu8JPPhUDq78t1eeArYc25q7pZebnI01BuqsHXUAQ -lxhl/9xWud5XgX5mMuWgAqBKf0YHx5Y7KKVNDXxNwp34faJyKlFo//5BH8o/HkYr -9JdTebKqbrZA4Yju3/mS1V2MaGzq0dnHDVJGKKmIthZu1VGtLFrjBiWCmChTC+u+ -+0LmQA2dPDVUZazFyz7prv4+hxicmKruThvk4BRvdHK+X0MsAzLw2qlFajhwcP9d -DcUJjAUeU5KDH90vlJ7SLd1bUrNs/o69o2DqeX/URc9SwnUBNLyfO/OFxwIDAQAB -o4IBwzCCAb8wFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA4GA1UdDwEB/wQEAwIF -oDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAd -BgNVHQ4EFgQUyL2otMDyMhBzR5xIgTL4ursmhJcwfQYDVR0jBHYwdIAUs4qgorpx -8agkedSkWyU2FR5JyM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZl -coIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6 -Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1Bggr -BgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2Nz -cC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5l -dC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAHaHdk3k -D4i/LPNYZ8CXbM1ZGIKDTAQZpW2q+mQ9STI+4VaVshP3z9MRsHK3W+fXhWlRPLZU -gEUvKBAhILm66Sdatz+Ct2n1RvW/oosXf/IU0UaXtYtH+5/oXAUOnRG9fJoDhAvK -KWZKyg1vCR56J8F/A5ZwjRilL6SYpRmqjF0ejD67bTvAM8AV4b0JPZ/o3BLUy0Qd -BvXo1k6hLVyfXR9bKsNNQI3a0XiA0MYxchBIiukQehMwEbKeZw7toarscy3wuIoi -dQ8waVxQfpHO2pHHcIxl//ZY+wC9Rczi5OPlFjZ986JKnEX/2aUW4C+1W2zmihMV -SHO9fIAzw9Q7Oh2FDqT39/tIDOmgS16KXGf4JQJvzXLBqlqTZHwUIEPgE38N4Q1h -Xi4szXouKq62dWpfoBqbtmctsKUcVLyMcH4VK8BQ4wO7pKX8RQHJP6e4GNw+CAeh -m/W9lb1J6BB8kX0txMKYtrdRadcKaEC1D4WgqWd3xmjLDlg0s1jnyHwJZw== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwM +ZmFrZWhvc3RuYW1lMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAv7gi +P+HlUgiK8f/xnAgsXkD9ejLX9MALiDQUfbk8khfLBFaEuqzQcC3rnWV2h5nKd4zO +zdnvRoyPHXUHHDwKSgcIwMYXU+F3rlE3qeFa2iloy01g7wAR3wlRQdbV9x7rBFmk +vJ/6R/jeC1H58Utx5xahguYm9g0oBUwpy0Eo/NolIAAnX/O4mU6J7PpEb6mLHcKb +IAVJzvsPqHOaOzY9WySZU0RLGOpEBmPeGrOKQpJqoaAUwu/5XvcBRpl4qUWIt6X0 +kzqgECX6ltVZUOAW3N5TeS/jjU5hVzoEyeaoWkvHIJZ2BJSSNUEE2oMlkDB7gQyD +Ka9yL/+alFM3SDM6DK6V+onp9xmp/48o487z/eYybxro0LncP5slyvFTlE4VbbLU +QxQN3CKFeFKjDg8/MOqmFNBj2BQav/9VHaQ65P3wTwXtzCyU9vrAf5E88McWLCqn +HYzxtgbd2Np5O4QsxNCrrnqgkD5P936JmNUw8k5Vjq4gEuNBGLnuUYcfaq/JAgMB +AAGjggHDMIIBvzAXBgNVHREEEDAOggxmYWtlaG9zdG5hbWUwDgYDVR0PAQH/BAQD +AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAA +MB0GA1UdDgQWBBTkPUzbloXMkO0PWB6NTX00bE4REDB9BgNVHSMEdjB0gBTACiso +Q95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5 +dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2Vy +dmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0 +cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUG +CCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9v +Y3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3Qu +bmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAY+nN +D+zdJ6D8dhah8l/TB5o/l/QR6CDpPN0WTyBicXOpRj6FbkS+nQhL64AI1RuTX0oP +37cbOOIrXGhI6gpYXDZseaQetX7vy41dvX1K5k7dC4cQ/wEOm4u93hyaJfui4VJ7 +iqoIN7KH90WdC6sRawx/2+3ezB6G8L4wJWtL//Xyme2wsWjBRAt5e4GVtTY3EsSZ +noXoLy3MzdjG8iDsagb9uPz/AhGVTTgNQouyQ+vCuaTgM/HaJS8Rz1e5JeWrkvKy +tREvvTH3VeuWlHharZyPVuw0oHOcAcJ2JOD4s8kjueqrwwrT+FNExxI3dkTX7iWT +9BwdfP4GLPrFu8aQpFf9CS7ar/Os5m1/A6EmNppRfY0op11dN1fMahEqmB1XcTJu +mHwSKO9cLyYp1FYKsSPZnTUgGuzMUVNu7xznvDwh52QxsUzzVbXEkxfYcloFLkrl +MwcOfc8icwtnn7SCYM1x+HYMxNyY7kn5A38N2MZ2eTwo7Xd3037Vqh5uLt+l -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycertecc.pem b/Lib/test/certdata/keycertecc.pem index 81daa4ccb94217..d503d49dc85f15 100644 --- a/Lib/test/certdata/keycertecc.pem +++ b/Lib/test/certdata/keycertecc.pem @@ -1,8 +1,8 @@ -----BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBcNwE+cm17mmr7Yg6d -0DNCnheGFOjkYH4tYzTyCkcZGShkmF/tKhIqb3imKz0Kx9+hZANiAATyp8ws6CuN -OI2/3MC4jZVSkmoDzm/X/ZrkEm4TVHKPSZ6kzZRpmmUlLS9l7SQZSLYyDAFBFzoG -JJYHhZNQXEO7HFszn6KnvLjhwS6ddzlaHPziEknrSr0OKhJmdJHrQAQ= +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBuhPMBtySJ8hFhn5hU +1yvdDujmB8ajjXNlkQ9GhGdghQIAfKmP9vWtSY+1+BRjpAahZANiAAQKLvQoetPb +S5tbgExB5vSDUtgc7hK1j6AN++En6AkW8KkK+O/31jkItqBoSQz8ts+VCU5cloTf +H6i5AS9zUkHr4DEyn+XxfiutzcNQPdQIBmygWAsUpa1XlZf8ExMFf/A= -----END PRIVATE KEY----- Certificate: Data: @@ -13,19 +13,19 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc Subject Public Key Info: Public Key Algorithm: id-ecPublicKey Public-Key: (384 bit) pub: - 04:f2:a7:cc:2c:e8:2b:8d:38:8d:bf:dc:c0:b8:8d: - 95:52:92:6a:03:ce:6f:d7:fd:9a:e4:12:6e:13:54: - 72:8f:49:9e:a4:cd:94:69:9a:65:25:2d:2f:65:ed: - 24:19:48:b6:32:0c:01:41:17:3a:06:24:96:07:85: - 93:50:5c:43:bb:1c:5b:33:9f:a2:a7:bc:b8:e1:c1: - 2e:9d:77:39:5a:1c:fc:e2:12:49:eb:4a:bd:0e:2a: - 12:66:74:91:eb:40:04 + 04:0a:2e:f4:28:7a:d3:db:4b:9b:5b:80:4c:41:e6: + f4:83:52:d8:1c:ee:12:b5:8f:a0:0d:fb:e1:27:e8: + 09:16:f0:a9:0a:f8:ef:f7:d6:39:08:b6:a0:68:49: + 0c:fc:b6:cf:95:09:4e:5c:96:84:df:1f:a8:b9:01: + 2f:73:52:41:eb:e0:31:32:9f:e5:f1:7e:2b:ad:cd: + c3:50:3d:d4:08:06:6c:a0:58:0b:14:a5:ad:57:95: + 97:fc:13:13:05:7f:f0 ASN1 OID: secp384r1 NIST CURVE: P-384 X509v3 extensions: @@ -38,69 +38,66 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 79:11:98:86:15:4F:48:F4:31:0B:D2:CC:C8:26:3A:09:07:5D:96:40 + F3:39:E5:90:3F:58:6F:55:9D:91:D4:86:B8:1B:14:35:09:AC:06:97 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 6e:42:e8:a2:2d:28:14:e3:25:5c:c1:7e:54:e9:3a:ff:30:db: - 94:ba:b2:f6:5f:ae:9a:c1:90:b3:4f:ce:65:1d:84:64:c0:71: - 2c:44:8e:7e:00:79:f5:8c:4a:1d:34:13:44:de:99:2e:db:53: - ee:ec:74:97:4d:59:1a:09:82:4f:98:75:91:a7:a0:b9:da:5e: - 68:f5:32:85:be:36:3d:83:d4:ee:f9:87:67:31:85:41:53:9a: - e7:05:96:13:1c:88:2e:7f:33:b1:ee:bd:f9:50:52:24:ed:3d: - 92:95:6e:30:c3:af:74:a9:ee:15:bb:da:7c:14:50:8e:e3:99: - ea:ba:b4:37:8a:50:61:26:de:01:93:b8:a2:6b:d9:c7:38:5e: - b2:f8:96:3d:a8:9f:7d:0c:71:d4:7e:cc:a0:57:af:7e:ce:3f: - a7:a7:27:68:c1:28:d7:4f:44:c1:b4:93:c3:c7:35:2b:50:c3: - 8e:2c:d0:46:c1:3f:e1:67:d3:f0:81:ae:f3:5c:3e:4f:d5:a8: - 07:8f:e0:eb:ef:d8:dc:47:e0:3d:58:eb:de:0e:7f:b2:58:cb: - 5c:f1:2f:65:7e:0f:0d:cc:ca:ba:83:53:63:bc:dd:18:0c:ee: - ed:ec:96:88:d0:38:c5:d7:ab:e7:55:79:7b:6d:ba:c0:a0:e9: - 5c:ca:7c:fb:f8:70:c7:fb:f5:b2:b5:74:cb:f7:c0:0d:20:9f: - 1d:b7:4c:bf:8a:8d:cd:e3:bc:4e:30:78:02:12:a0:9b:d5:8f: - 49:3c:95:91:76:6e:7c:54:dc:61:7a:2e:20:ed:35:25:e0:c5: - 17:50:02:83:00:74:8f:f0:1c:97:96:08:fc:2e:63:a4:f7:97: - 87:43:2a:32:04:2d:4c:f9:1a:07:bf:68:91:fc:50:21:a1:3c: - 8d:8f:fb:83:57:83:1f:b6:55:5c:55:2f:58:64:ad:f3:27:ba: - d0:e3:cd:58:01:a3:c9:ba:1d:95:dc:30:d5:af:b9:20:ad:d9: - 48:ba:8d:9a:66:ee + Signature Value: + b8:d9:bf:b8:88:0a:01:2b:aa:64:32:e8:97:e1:0f:ee:34:40: + ef:71:fc:e5:f4:a2:3b:26:00:e2:19:3b:3e:cb:8e:2b:51:4c: + 30:ff:ab:44:9d:28:8d:d2:9c:32:e3:6b:96:73:1c:9d:55:76: + b2:bf:af:b8:51:ed:fd:04:e7:0d:fa:8c:2a:68:01:cb:92:90: + 7d:d1:31:d8:6c:f4:b8:4b:ea:62:59:1e:31:4e:f7:17:ae:3f: + f0:b6:ff:f8:6e:64:e3:6e:e9:19:4f:d0:1e:84:1e:df:56:49: + ae:90:a9:e0:7b:70:e6:97:7f:08:f2:03:49:82:7d:58:8e:a5: + a0:88:59:f3:a1:ab:b7:dd:6b:9a:2b:79:64:9c:d3:07:7c:ec: + a3:8d:56:77:28:10:c9:42:a5:fa:81:8e:35:ad:f7:28:da:58: + cc:a0:61:fa:0d:75:fd:26:ab:07:88:c5:64:21:f1:98:1c:e3: + 13:51:38:a7:59:a5:59:88:b9:10:4d:c1:79:70:3f:36:9e:08: + 75:53:7f:77:a3:3d:5d:16:b7:3b:a2:f6:eb:41:9b:56:16:80: + 34:96:57:f0:3a:3d:91:43:17:51:2b:64:2c:d7:e2:25:45:85: + f3:5f:73:f8:7d:aa:a3:6d:61:e8:ba:c5:90:5e:16:50:bc:79: + b9:4e:df:66:db:ae:95:80:15:da:4e:bf:8f:4e:9d:e3:7d:b0: + ab:8b:72:93:3c:56:1b:92:d3:67:07:d5:5a:ad:98:16:69:ea: + 46:fe:e0:d0:f4:ae:95:d3:b5:80:7d:f2:64:4d:14:c0:97:d2: + f3:91:bb:e6:43:d0:7e:39:14:0b:95:ab:c1:56:fd:26:79:f8: + 8c:69:eb:bc:74:0e:ea:cd:94:62:7e:30:58:01:7f:84:ee:9d: + 58:9c:fc:8f:75:1e:d9:86:cb:f4:fd:4c:af:b2:f7:69:ea:58: + cb:7c:93:ce:3d:86:f0:46:01:df:86:02:a1:6c:0f:fb:13:84: + d6:75:23:f9:17:21 -----BEGIN CERTIFICATE----- -MIIEyzCCAzOgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEzTCCAzWgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv -Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATyp8ws6CuNOI2/3MC4 -jZVSkmoDzm/X/ZrkEm4TVHKPSZ6kzZRpmmUlLS9l7SQZSLYyDAFBFzoGJJYHhZNQ -XEO7HFszn6KnvLjhwS6ddzlaHPziEknrSr0OKhJmdJHrQASjggHEMIIBwDAYBgNV -HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU -BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUeRGY -hhVPSPQxC9LMyCY6CQddlkAwfQYDVR0jBHYwdIAUs4qgorpx8agkedSkWyU2FR5J -yM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMstgJlaaVJb -MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 -aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 -cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww -OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 -b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAG5C6KItKBTjJVzBflTpOv8w -25S6svZfrprBkLNPzmUdhGTAcSxEjn4AefWMSh00E0TemS7bU+7sdJdNWRoJgk+Y -dZGnoLnaXmj1MoW+Nj2D1O75h2cxhUFTmucFlhMciC5/M7HuvflQUiTtPZKVbjDD -r3Sp7hW72nwUUI7jmeq6tDeKUGEm3gGTuKJr2cc4XrL4lj2on30McdR+zKBXr37O -P6enJ2jBKNdPRMG0k8PHNStQw44s0EbBP+Fn0/CBrvNcPk/VqAeP4Ovv2NxH4D1Y -694Of7JYy1zxL2V+Dw3MyrqDU2O83RgM7u3slojQOMXXq+dVeXttusCg6VzKfPv4 -cMf79bK1dMv3wA0gnx23TL+Kjc3jvE4weAISoJvVj0k8lZF2bnxU3GF6LiDtNSXg -xRdQAoMAdI/wHJeWCPwuY6T3l4dDKjIELUz5Gge/aJH8UCGhPI2P+4NXgx+2VVxV -L1hkrfMnutDjzVgBo8m6HZXcMNWvuSCt2Ui6jZpm7g== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEWMBQGA1UEAwwN +bG9jYWxob3N0LWVjYzB2MBAGByqGSM49AgEGBSuBBAAiA2IABAou9Ch609tLm1uA +TEHm9INS2BzuErWPoA374SfoCRbwqQr47/fWOQi2oGhJDPy2z5UJTlyWhN8fqLkB +L3NSQevgMTKf5fF+K63Nw1A91AgGbKBYCxSlrVeVl/wTEwV/8KOCAcQwggHAMBgG +A1UdEQQRMA+CDWxvY2FsaG9zdC1lY2MwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTz +OeWQP1hvVZ2R1Ia4GxQ1CawGlzB9BgNVHSMEdjB0gBTACisoQ95fyX1H5UebNvJl +jGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp +UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w +eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo +dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E +PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y +ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAuNm/uIgKASuqZDLol+EP +7jRA73H85fSiOyYA4hk7PsuOK1FMMP+rRJ0ojdKcMuNrlnMcnVV2sr+vuFHt/QTn +DfqMKmgBy5KQfdEx2Gz0uEvqYlkeMU73F64/8Lb/+G5k427pGU/QHoQe31ZJrpCp +4Htw5pd/CPIDSYJ9WI6loIhZ86Grt91rmit5ZJzTB3zso41WdygQyUKl+oGONa33 +KNpYzKBh+g11/SarB4jFZCHxmBzjE1E4p1mlWYi5EE3BeXA/Np4IdVN/d6M9XRa3 +O6L260GbVhaANJZX8Do9kUMXUStkLNfiJUWF819z+H2qo21h6LrFkF4WULx5uU7f +ZtuulYAV2k6/j06d432wq4tykzxWG5LTZwfVWq2YFmnqRv7g0PSuldO1gH3yZE0U +wJfS85G75kPQfjkUC5WrwVb9Jnn4jGnrvHQO6s2UYn4wWAF/hO6dWJz8j3Ue2YbL +9P1Mr7L3aepYy3yTzj2G8EYB34YCoWwP+xOE1nUj+Rch -----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.ca.pem b/Lib/test/certdata/leaf-missing-aki.ca.pem new file mode 100644 index 00000000000000..36b202ae02ee54 --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.ca.pem @@ -0,0 +1,13 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN CERTIFICATE----- +MIIBkDCCATWgAwIBAgIUGjIb/aYm9u9fBh2o4GAYRJwk5XIwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAaMRgwFgYDVQQDDA94NTA5LWxpbWJvLXJvb3QwWTATBgcq +hkjOPQIBBggqhkjOPQMBBwNCAARUzBhjMOkO911U65Fvs4YmL1YPNj63P9Fa+g9U +KrUqiIy8WjaDXdIe8g8Zj0TalpbU1gYCs3atteMxgIp6qxwHo1cwVTAPBgNVHRMB +Af8EBTADAQH/MAsGA1UdDwQEAwICBDAWBgNVHREEDzANggtleGFtcGxlLmNvbTAd +BgNVHQ4EFgQUcv1fyqgezMGzmo+lhmUkdUuAbIowCgYIKoZIzj0EAwIDSQAwRgIh +AIOErPSRlWpnyMub9UgtPF/lSzdvnD4Q8KjLQppHx6oPAiEA373p4L/HvUbs0xg8 +6/pLyn0RT02toKKJcMV3ChohLtM= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.keycert.pem b/Lib/test/certdata/leaf-missing-aki.keycert.pem new file mode 100644 index 00000000000000..0fd2ab39bf2c70 --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.keycert.pem @@ -0,0 +1,18 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIF5Re+/FP3rg+7c1odKEQPXhb9V65kXnlZIWHDG9gKrLoAoGCCqGSM49 +AwEHoUQDQgAE1WAQMdC7ims7T9lpK9uzaCuKqHb/oNMbGjh1f10pOHv3Z+oAvsqF +Sv3hGzreu69YLy01afA6sUCf1AA/95dKkg== +-----END EC PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIBjjCCATWgAwIBAgIUVlBgclml+OXlrWzZfcgYCiNm96UwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAWMRQwEgYDVQQDDAtleGFtcGxlLmNvbTBZMBMGByqGSM49 +AgEGCCqGSM49AwEHA0IABNVgEDHQu4prO0/ZaSvbs2griqh2/6DTGxo4dX9dKTh7 +92fqAL7KhUr94Rs63ruvWC8tNWnwOrFAn9QAP/eXSpKjWzBZMB0GA1UdDgQWBBS3 +yYRQQwo3syjGVQ8Yw7/XRZHbpzALBgNVHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwFgYDVR0RBA8wDYILZXhhbXBsZS5jb20wCgYIKoZIzj0EAwIDRwAwRAIg +BVq7lw4Y5MPEyisPhowMWd4KnERupdM5qeImDO+dD7ICIE/ksd6Wz1b8rMAfllNV +yiYst9lfwTd2SkFgdDNUDFud +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/make_ssl_certs.py b/Lib/test/certdata/make_ssl_certs.py index 94a35a64ab1abe..18e614496385fd 100644 --- a/Lib/test/certdata/make_ssl_certs.py +++ b/Lib/test/certdata/make_ssl_certs.py @@ -1,6 +1,7 @@ """Make the custom certificate and private key files used by test_ssl and friends.""" +import argparse import os import pprint import shutil @@ -8,7 +9,8 @@ from subprocess import * startdate = "20180829142316Z" -enddate = "20371028142316Z" +enddate_default = "25251028142316Z" +days_default = "140000" req_template = """ [ default ] @@ -79,8 +81,8 @@ default_startdate = {startdate} enddate = {enddate} default_enddate = {enddate} - default_days = 7000 - default_crl_days = 7000 + default_days = {days} + default_crl_days = {days} certificate = pycacert.pem private_key = pycakey.pem serial = $dir/serial @@ -109,14 +111,15 @@ subjectKeyIdentifier=hash authorityKeyIdentifier=keyid:always,issuer - basicConstraints = CA:true + basicConstraints = critical, CA:true + keyUsage = critical, digitalSignature, keyCertSign, cRLSign """ here = os.path.abspath(os.path.dirname(__file__)) -def make_cert_key(hostname, sign=False, extra_san='', +def make_cert_key(cmdlineargs, hostname, sign=False, extra_san='', ext='req_x509_extensions_full', key='rsa:3072'): print("creating cert for " + hostname) tempnames = [] @@ -129,13 +132,13 @@ def make_cert_key(hostname, sign=False, extra_san='', hostname=hostname, extra_san=extra_san, startdate=startdate, - enddate=enddate + enddate=cmdlineargs.enddate, + days=cmdlineargs.days ) with open(req_file, 'w') as f: f.write(req) - args = ['req', '-new', '-nodes', '-days', '7000', + args = ['req', '-new', '-nodes', '-days', cmdlineargs.days, '-newkey', key, '-keyout', key_file, - '-extensions', ext, '-config', req_file] if sign: with tempfile.NamedTemporaryFile(delete=False) as f: @@ -144,7 +147,7 @@ def make_cert_key(hostname, sign=False, extra_san='', args += ['-out', reqfile ] else: - args += ['-x509', '-out', cert_file ] + args += ['-extensions', ext, '-x509', '-out', cert_file ] check_call(['openssl'] + args) if sign: @@ -174,7 +177,7 @@ def make_cert_key(hostname, sign=False, extra_san='', def unmake_ca(): shutil.rmtree(TMP_CADIR) -def make_ca(): +def make_ca(cmdlineargs): os.mkdir(TMP_CADIR) with open(os.path.join('cadir','index.txt'),'a+') as f: pass # empty file @@ -191,7 +194,8 @@ def make_ca(): hostname='our-ca-server', extra_san='', startdate=startdate, - enddate=enddate + enddate=cmdlineargs.enddate, + days=cmdlineargs.days ) t.write(req) t.flush() @@ -218,14 +222,22 @@ def make_ca(): shutil.copy('capath/ceff1710.0', 'capath/b1930218.0') -def print_cert(path): +def write_cert_reference(path): import _ssl - pprint.pprint(_ssl._test_decode_cert(path)) + refdata = pprint.pformat(_ssl._test_decode_cert(path)) + print(refdata) + with open(path + '.reference', 'w') as f: + print(refdata, file=f) if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Make the custom certificate and private key files used by test_ssl and friends.') + parser.add_argument('--days', default=days_default) + parser.add_argument('--enddate', default=enddate_default) + cmdlineargs = parser.parse_args() + os.chdir(here) - cert, key = make_cert_key('localhost', ext='req_x509_extensions_simple') + cert, key = make_cert_key(cmdlineargs, 'localhost', ext='req_x509_extensions_simple') with open('ssl_cert.pem', 'w') as f: f.write(cert) with open('ssl_key.pem', 'w') as f: @@ -242,24 +254,26 @@ def print_cert(path): f.write(cert) # For certificate matching tests - make_ca() - cert, key = make_cert_key('fakehostname', ext='req_x509_extensions_simple') + make_ca(cmdlineargs) + cert, key = make_cert_key(cmdlineargs, 'fakehostname', ext='req_x509_extensions_simple') with open('keycert2.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('localhost', sign=True) + cert, key = make_cert_key(cmdlineargs, 'localhost', sign=True) with open('keycert3.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('fakehostname', sign=True) + check_call(['openssl', 'x509', '-outform', 'pem', '-in', 'keycert3.pem', '-out', 'cert3.pem']) + + cert, key = make_cert_key(cmdlineargs, 'fakehostname', sign=True) with open('keycert4.pem', 'w') as f: f.write(key) f.write(cert) cert, key = make_cert_key( - 'localhost-ecc', sign=True, key='param:secp384r1.pem' + cmdlineargs, 'localhost-ecc', sign=True, key='param:secp384r1.pem' ) with open('keycertecc.pem', 'w') as f: f.write(key) @@ -279,7 +293,7 @@ def print_cert(path): 'RID.1 = 1.2.3.4.5', ] - cert, key = make_cert_key('allsans', sign=True, extra_san='\n'.join(extra_san)) + cert, key = make_cert_key(cmdlineargs, 'allsans', sign=True, extra_san='\n'.join(extra_san)) with open('allsans.pem', 'w') as f: f.write(key) f.write(cert) @@ -296,17 +310,17 @@ def print_cert(path): ] # IDN SANS, signed - cert, key = make_cert_key('idnsans', sign=True, extra_san='\n'.join(extra_san)) + cert, key = make_cert_key(cmdlineargs, 'idnsans', sign=True, extra_san='\n'.join(extra_san)) with open('idnsans.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('nosan', sign=True, ext='req_x509_extensions_nosan') + cert, key = make_cert_key(cmdlineargs, 'nosan', sign=True, ext='req_x509_extensions_nosan') with open('nosan.pem', 'w') as f: f.write(key) f.write(cert) unmake_ca() - print("update Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") - print_cert('keycert.pem') - print_cert('keycert3.pem') + print("Writing out reference data for Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") + write_cert_reference('keycert.pem') + write_cert_reference('keycert3.pem') diff --git a/Lib/test/certdata/nosan.pem b/Lib/test/certdata/nosan.pem index ec10cdcabb9e35..c397342fa86f6e 100644 --- a/Lib/test/certdata/nosan.pem +++ b/Lib/test/certdata/nosan.pem @@ -1,130 +1,137 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCv3sUoOE4F7Pye -AT2Q6XpXrGUOu1fYgdnItLLLhvn7ACuHMj7TA5UKXxsepJn5m2Ji9LvAbksr1IWd -LZAvNgjwsUR+E4HbY108BhVt9sk3HFkvE0OOFbAa14ICtYPe18P/4Hv6Zfu/GJDU -rwXHNCUu0p6i/mospZ5O3sx5MgVaShknGAEC3Kp7zOgusMmE8XSbkNQa3ARMkW4o -kTqWKAeAHDjVFVyyhzZQmo+gaLzhWfJVSZhlJsuiLoZGGrVTq85EiXsE4l8rPaI+ -mKkVzWP13IZW+Fx1tiIktumdHWb1OQWrvm8AiT9b8PcFCUUrvhQFcLDSCZjKlQ0t -RWrSSKrrVsSldOreqRLtpjGzFJpGnTcvslL7rP5pg5DjBsYmVcDjrmRuJuhGq52X -/6HEC97GouVK8tT1LVMv1wufVPn+i9TzwxOuRWeUvVqLAJgWQ9N3yKdymH+VrpZk -/oB9ScyDakGezZBW5CeOQbNJ8WoX58jNxefGjtqKxmyztu43r3ECAwEAAQKCAYBQ -fVoqYCqFV8L95X9x1QljGsldhqxbsIIl811o/KtoDtndFEfgd2E8z+4vhhHaRR0w -QOW02kWZF7jXCMVWdhp9XgQE15S0/bLsB7TDERFiIZ1HiD+AxbhFcKBV8REbahCQ -CQN0xDwFZ47RaBDy7JCf71EfM+UP7fSYECvww83jVspQNBIyZx+3bT5OMCbqqz88 -+3m3mT52dJDADEeN9WAJZ+Ey1IYKRwu6tCJLvePEF1BrbDVNBgZogXZ+mzalxpjr -4RpGPMMa+VWc8HmDVd+LtpwKJcQD00GvUP4fNywn+5jvNWl54FdQiTLPrieTWxas -XUQ2crxP7Aqr2/vsU5Ruru5uF7H+ssMHp9YQDhpJ2+SVhQ9P+/loXCuKGt+BrB2Z -MlitO3f+vfRtzATmJ8G0qFrOqZK1A/qsiyIze240C1hAl3oy2xpZqTDGp4gRWwoi -OIN0HmH9UbP7bbNQY1x/zstTbza4/7rGb1+DZKeZIMu7QjBCU0rtsJpGtUvcQGEC -gcEA42GMYSL/HljZMF1LsDhTX/cmP8FDNgONhWYxT+w0Csnj1usLNBaT63dYnEiW -QKydRR4casAR1Kdy4Yfcy2lCy1kCfwqkQYk8fxSsOSHRjUfwC1SnfdYlwKFMxw4a -oZF0R4oVCBYrfP+8kqrj+5gs/gXblsw72XkYtbCdIriKKdmUzTx7MegzSqh2PVRi -rJzuwCZQ/O0NfhwdOHxLQDo0dgD+vv9e+KOSoJ9FDv8HH1tnolpRMdkSA8AJR/Nk -DXt1AoHBAMYBfTKQZ2jqLKybe4tP+YKjvjVp8vJx0iNUXFN/P6hBaSBOgq85uxXL -X3s7N/pkOCjyE95B8QusIkbnbfdyEP89O4bTbUHPXyAkHyRkR7Vny49HYuaR/aXQ -mXC0J2z5bXVpCQ514l/R/Io3wBph+hbG3To7pp9pMOV4qzvibUZaTZFwH+q+xDwf -SKSFy3fcomgH4/K5/QuKVj0jOUQsYjQQWb8GukS2KZK3zYJIAG1bBcsCVpSuBdW0 -eCZgqjnwjQKBwCUyUwWc9QEg5b68tGIKhNEhHDe3xOf0ItWcxxpc+JJ/Pm9tGfMW -cnJFntBKK5I+6qdg6qMn8oLINcnhMORxvsSHNhpUQlSaP7RGTHo4JxCmoQUpfxDd -1GUzvdyeWQrvQYdmdlRRVCHpsA6KOCtzVIDlsmtz06Ka5cjrMHl6mNeJyYbdiwW6 -B5ICBv23bUDxlzkFy5/ko51qufkAlErYeraHKSVTn1SrZZQzGdf/LkoZ6NUtUzUF -XqYQZzRHA6oU9QKBwDslzLljC5D6ivfQxln6POV6dmJMUOd9erFVDPNgSqq/R2EA -MueXDjzXcKFGMlWYxHHuxmKZPiEnfWHC1kWZjFxCdVq0I6oKATd/stHTJtyYseUO -BQwtRiDXLE7PcguKgtkU1EC+lC3dc1vyhW8cH3HYW9N+aCqsaI/TuQr9e3kNlqhA -XzhnXgU7rx5+XSZkARukZ8JlLqLY4yQGNqAXxgoZbEW1A8VsyQRr5XbqfT4td5CK -FUT6qwGIlG+aZp9CLQKBwQCQkwdW9A/Q4Ffq8+XTL1hJ24m/q11OLAPODUypOhWw -OCbX2fkv59pSBe6niZDBls1NpHB9mzalBrJCfU+yKC667gKcKULOnWULIoOQvmcg -Ka3hkkW28gTnCjfDIYm3IdsLjc67zJplOixaKgxhO8NtJZGtg0oLIrofG8EYRInv -OmtGw+XE+s4TVs6WgXnEg9zWQ5ZYtqQVn6PT5jsz+Nrvipi61HWHVBd7g+78ojps -3suWxl0FvgzTW5HD16WRXeI= +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAQKCAYAE +A1FQwmMxHOhUbuAIRoUInwhMeLs7ZKJx8OsGPDSCTGRGGnAFD/rSP63cNy71T+AF +gggpiGQegz9KkeGfhQwM1GssLGk8WnCp2fnqbAwyhkZzpOs+WEJSCsKMFDc3S0GU +hChIYAJFtgRLJnAFM+b2V0uIfHCz26hGqKOdDb/Wumzd8zPhH+/SNI5hdM7xStez +2d/dlqVZ21c0ghiMLZ/M2qrz8EEvWccc4BEuYtqJYNpdgAauLIGF0SxByPt/oZMf +4fa0zOVfAL6F5cPFkycRzuz97Lx+ZdtLuUBx/3y8Sk24TR1QutwaFhbwpRSQSUrg +xT7ZmS24rQhaP8i7+ROLGy0CPlLMc/D3WvvtdLDWOUF30azURvAFcqKA87Bp05bR +HI/ewXaSB26J34MUmBHy5dBjld+HX6hQInPhCTZu7gNDOrPaEc8Ug50c1r7HkYRl +VBv/4BQo3YtNdoKhL08MVuNbcxrzge3kI0mbQu7qDSyrve02WnQ1zijJ916NbgEC +gcEA6oFNNtWzVw/7/MBxUZ1tE5VW5U2SjRvQZlcPTJ9Ub16byhOCdNvE17uyYLt5 +diqoVAvSGWiJ+OKsmw3XTLdhxzWeZ1a9ZiCozsmIfHMQmL9t3MxzTnMEZCfcvxz0 +DPvAN0REUtwRkMbt7vC+g01Eom9UpAH7hKXkc3pTgtkoyFeebC/YneGtdOTVATKU +WMfsY6Biye7P1D8V7btjLI8LTnv90ygoP0uSrX53hp7k4EFAeXyT9VN5wMXLw81a +5K69AoHBANXZsvSSIwe7fjspNUU9yELnEokDIdBn4ARX6x5PonL2KB0iVUzWgjHb +vs23GAzAgKKY+S8VLYptBLUBlsZ/pSLfZLWd1ywxeHmk/nkzuqlAw1jwuUlMq6Vr +BJmfhpOoXeSrQwvkL/uOejdrDSmLAFiIV3l9LtkmIJzfQSSy2QB8mfWwni0vxOJT +mIqZE01rfRTNLIqxO3+0d7pDqArVvWxYqF4C8kihUhNjkB5NaURUTzDQ3Y33aAvT +zAm697nGSQKBwQDEQ8ed9ykL2sLpfR7aUclytHBvpYbcNsUqgf66ADeopiP48m8i +4rRSYjMepok3jugmv2XuAgJHnV8cvm7NNEXPdl7G2l/V08u0lhN3JM5lKQIH4801 +gSnRsVMdWFwhaaosFySfvLOu2e9VJYQtXEPvNwI96bLaCAW1aFHwl1N8qWhb34eK +S9DinopvYCesTlbX4uoLW6XxW4M83rJYHrg1zaxYR6m3n8Z5EflzYBTqY3JUuyES +F/U0k9bAX2SNNHkCgcALk8mYbADxfjkLQuPbZ8jbtl7OhBjki3sZQRk9ftowlxr8 +2Mr9ae+Ke3cM9AidSB6urtFutxrMD7LdicR74pUyGh39pxnrDpKTI1eTgDVuzE7H +FeEyErCIOA77siM7AzZyFsN+dVATslbzgRwpT5kpMdhqf1h18RZ656tDLVuKJzS+ +lF073QYvqo7rkfX1jwgqhCERMR8jfsWsk9UZIREsOHCFBmvPesxSuGUo/s/gHyBa +aDRWZzp+yWyWakTXDeECgcBaPIxLK0ky89Mv8jTSyIJIMhQj8+z8wFLJFQG5vF/Z +1taLIDY0stU8HzKflTJ5v72r2jpIvu5YgDEgl+kKjyqLMWSu4nBZ2OoJet49S6d3 +X0YL2VzsIiSG+8cmVvBBf7iZCHDKiCM20dzzhZq8BCLiaPq+Zu2No2xEJyE0m8rc +I66z6fV1BWQXLe6vPC8EVvBWz3Ybje0czUdZnZ44qV6qyZzREXuqbC0qS5RanE4Z ++Ps9b3TezE7gDII6SbUwhZo= -----END PRIVATE KEY----- Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: cb:2d:80:99:5a:69:52:61 Signature Algorithm: sha256WithRSAEncryption Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=nosan Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:af:de:c5:28:38:4e:05:ec:fc:9e:01:3d:90:e9: - 7a:57:ac:65:0e:bb:57:d8:81:d9:c8:b4:b2:cb:86: - f9:fb:00:2b:87:32:3e:d3:03:95:0a:5f:1b:1e:a4: - 99:f9:9b:62:62:f4:bb:c0:6e:4b:2b:d4:85:9d:2d: - 90:2f:36:08:f0:b1:44:7e:13:81:db:63:5d:3c:06: - 15:6d:f6:c9:37:1c:59:2f:13:43:8e:15:b0:1a:d7: - 82:02:b5:83:de:d7:c3:ff:e0:7b:fa:65:fb:bf:18: - 90:d4:af:05:c7:34:25:2e:d2:9e:a2:fe:6a:2c:a5: - 9e:4e:de:cc:79:32:05:5a:4a:19:27:18:01:02:dc: - aa:7b:cc:e8:2e:b0:c9:84:f1:74:9b:90:d4:1a:dc: - 04:4c:91:6e:28:91:3a:96:28:07:80:1c:38:d5:15: - 5c:b2:87:36:50:9a:8f:a0:68:bc:e1:59:f2:55:49: - 98:65:26:cb:a2:2e:86:46:1a:b5:53:ab:ce:44:89: - 7b:04:e2:5f:2b:3d:a2:3e:98:a9:15:cd:63:f5:dc: - 86:56:f8:5c:75:b6:22:24:b6:e9:9d:1d:66:f5:39: - 05:ab:be:6f:00:89:3f:5b:f0:f7:05:09:45:2b:be: - 14:05:70:b0:d2:09:98:ca:95:0d:2d:45:6a:d2:48: - aa:eb:56:c4:a5:74:ea:de:a9:12:ed:a6:31:b3:14: - 9a:46:9d:37:2f:b2:52:fb:ac:fe:69:83:90:e3:06: - c6:26:55:c0:e3:ae:64:6e:26:e8:46:ab:9d:97:ff: - a1:c4:0b:de:c6:a2:e5:4a:f2:d4:f5:2d:53:2f:d7: - 0b:9f:54:f9:fe:8b:d4:f3:c3:13:ae:45:67:94:bd: - 5a:8b:00:98:16:43:d3:77:c8:a7:72:98:7f:95:ae: - 96:64:fe:80:7d:49:cc:83:6a:41:9e:cd:90:56:e4: - 27:8e:41:b3:49:f1:6a:17:e7:c8:cd:c5:e7:c6:8e: - da:8a:c6:6c:b3:b6:ee:37:af:71 + 00:c3:e5:00:c7:04:14:ea:1b:98:3f:39:6e:a4:55: + be:a8:4c:72:cc:e2:ae:b0:5f:8d:98:a1:f3:c6:06: + 35:85:94:87:8c:5f:bf:00:d4:fc:d1:be:02:8b:a3: + d4:57:6c:13:9b:fd:0f:3d:ad:bb:6c:2f:d7:5d:f8: + 57:14:68:d3:fb:18:d2:e7:5f:e3:29:78:e9:cc:38: + be:8c:ec:bc:e2:84:58:b4:3b:24:66:bc:2c:ad:58: + 85:0c:62:41:fc:96:a1:ee:c9:dc:1d:51:13:ac:6e: + a1:a4:da:be:d2:6d:ef:17:cc:36:df:f5:d3:67:09: + 86:67:a0:e2:4c:88:6b:e9:d3:c0:f9:14:51:89:2f: + e2:bb:37:dc:42:10:70:a9:1c:9e:66:17:2c:b0:ce: + e9:e8:47:7b:ac:12:05:7e:d1:bf:27:b7:6f:3b:45: + c2:e8:04:d6:8e:44:22:ed:e8:bd:45:4a:c4:9b:ea: + 0d:cd:4b:f0:14:34:d0:db:d5:52:e8:6b:de:87:14: + b2:f7:ac:10:aa:4a:dd:f2:48:21:ad:cf:79:67:70: + 41:b9:1d:70:51:34:45:a8:91:ae:12:2a:a1:c3:38: + ff:33:0d:96:71:b6:63:70:5a:9e:90:b4:76:85:8f: + 8a:d8:fd:79:cc:1a:27:a4:26:02:06:72:bc:de:07: + c8:c0:83:01:82:f6:ed:b7:62:8e:7f:21:7e:4e:86: + 04:88:6e:a8:f4:9a:d6:15:47:06:70:25:bb:33:5d: + 54:27:ec:13:5b:43:33:45:36:09:e9:24:fa:49:c4: + 0e:31:0d:1a:f8:d7:92:2c:01:01:e6:ee:e8:2d:dc: + 6d:1a:5f:63:db:6d:76:31:3a:91:7f:51:df:b7:25: + 0b:b9:bd:81:9b:70:c5:31:a8:e3:da:1e:2d:c2:f1: + 05:13:ad:cb:c8:44:19:b8:5b:62:f6:7d:dd:f1:00: + da:42:e9:16:9f:45:55:e3:7d:e1:85:7e:4d:29:c8: + 0a:e5:bc:8c:98:08:6b:f1:01:e5 Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 27:04:5B:37:51:0D:92:FD:2C:60:61:FE:4F:9C:66:20:41:2C:08:45 + X509v3 Authority Key Identifier: + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 Signature Algorithm: sha256WithRSAEncryption - 91:42:c2:15:57:42:47:77:e7:0f:c5:55:26:b1:5b:c3:5e:ba: - 81:db:e1:a4:9f:b8:42:5a:21:c9:8c:18:ae:0f:90:ab:9a:24: - e7:d2:78:fc:bd:97:29:b1:5c:46:1f:5b:b8:d2:a7:87:f1:50: - 53:5b:d3:be:57:74:bd:e5:75:db:50:81:f7:37:95:0b:69:ef: - 39:8c:5c:82:d5:64:62:d5:8b:e9:e0:31:e1:73:d2:5a:2c:de: - 43:5a:06:e5:d3:4d:d0:35:e0:9f:c2:73:31:bc:35:69:d4:fb: - 7d:f0:1a:33:f7:f6:25:72:9c:a6:84:05:08:f6:b5:e8:04:10: - f1:1f:f2:95:ad:a1:f8:d8:80:a5:eb:75:43:99:33:90:0c:79: - fc:c0:87:08:95:20:aa:c2:81:0b:22:6f:56:f4:8f:2a:23:f8: - 40:47:1c:03:a5:b1:04:0a:04:4a:df:d0:88:a8:bc:31:f2:42: - 9b:d8:11:14:9e:e3:68:ea:07:2c:15:de:d2:36:5a:15:38:ed: - d2:af:0e:b4:b6:1d:a0:57:94:ea:c3:c7:4c:14:57:81:00:57: - 94:d3:b0:27:69:d7:48:02:6c:e5:97:f7:be:22:7c:38:24:af: - b2:b0:7b:08:75:1e:ca:2e:c7:41:ef:8b:74:cf:c9:c3:6f:39: - b9:52:41:18:c6:70:24:54:51:04:fe:5f:88:70:35:e5:1c:8e: - d6:67:69:44:44:33:9b:8c:fe:a5:b9:95:48:66:84:f3:1a:04: - ab:a3:57:c1:b6:b4:2f:28:12:45:2b:cb:42:d3:f4:a5:ce:7b: - 6c:1f:e4:c8:a9:e7:d4:6d:c8:27:2d:69:26:c5:e8:73:10:54: - 1f:c3:bf:fd:aa:f5:95:6f:f6:ca:d5:06:8f:1b:79:93:e3:86: - ba:8d:fe:a8:10:8f:95:3e:14:09:bf:ca:88:59:e2:93:b6:ec: - 03:a9:7e:dd:1f:5f:13:d3:29:b3:a6:f3:6a:df:30:53:44:c8: - cd:e5:82:57:bc:9c + Signature Value: + a0:b7:48:56:c6:b3:92:26:3f:d8:92:3c:ed:72:b5:89:ea:fd: + c9:66:da:ba:6a:8e:0d:04:ea:b2:fd:1f:e4:29:da:1e:c7:8f: + 5a:f0:88:74:dd:b3:f0:5e:a7:c4:77:13:cf:a8:19:fb:f2:2d: + ee:47:b4:0c:7c:5b:d3:dc:2f:2a:5c:bf:43:22:1c:91:d8:03: + 7d:44:90:c0:2d:fe:9e:7c:8b:ef:39:4e:b3:87:99:c8:eb:c2: + b7:cf:86:65:05:52:8c:15:b9:6a:8d:cd:e3:2a:29:d1:f5:87: + 42:11:c3:2e:42:ec:ed:26:55:8d:f3:ad:66:f4:79:72:f7:9e: + ed:bc:0c:5a:a7:74:ab:dc:57:e8:5c:99:b6:32:8f:7e:58:6e: + 70:48:ea:5d:a7:fa:b1:fc:c0:e9:50:3a:a4:53:21:e7:8b:77: + 54:2d:bb:64:1e:fc:88:86:90:c1:03:90:17:bb:3c:cf:ce:e8: + 48:32:c2:07:e5:8e:4f:93:a9:1a:e2:f5:93:a8:01:9f:30:26: + 1e:ed:b5:62:e9:25:c5:b0:32:e1:fc:bd:d6:48:b4:70:a9:e2: + cd:f6:a9:42:cb:bb:24:39:b9:34:fc:b9:cb:09:01:f0:5e:7e: + ef:b5:59:d6:88:31:a9:4c:be:7d:5b:de:4d:ec:84:1b:a1:1b: + d8:7d:83:cb:f1:04:c9:f1:f3:a4:08:05:3c:b5:96:13:1d:37: + 8a:23:83:22:86:72:17:13:5e:e8:89:06:58:cd:89:42:71:12: + e5:47:fc:f7:6e:96:28:8f:19:b9:d7:86:5b:c5:62:14:e1:5b: + 06:e7:e0:66:7e:fc:b7:9e:a9:99:14:e5:0a:d6:df:8f:b5:a2: + 1a:74:54:30:f6:f4:bf:1b:43:1d:be:4f:38:92:55:10:7b:d8: + 4f:e0:33:0f:40:2e:58:ec:9c:78:1b:43:17:b3:cb:0b:f5:34: + e2:7e:11:a1:90:b6:3c:79:6a:0b:91:ce:0b:8d:d5:60:e4:6d: + c8:2a:3d:40:6d:17 -----BEGIN CERTIFICATE----- -MIIEJDCCAowCCQDLLYCZWmlSYTANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJY -WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV -BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTgwODI5MTQyMzE2WhcNMzcxMDI4MTQyMzE2 -WjBbMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNV -BAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ4wDAYDVQQDDAVub3NhbjCC -AaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAK/exSg4TgXs/J4BPZDpeles -ZQ67V9iB2ci0ssuG+fsAK4cyPtMDlQpfGx6kmfmbYmL0u8BuSyvUhZ0tkC82CPCx -RH4TgdtjXTwGFW32yTccWS8TQ44VsBrXggK1g97Xw//ge/pl+78YkNSvBcc0JS7S -nqL+aiylnk7ezHkyBVpKGScYAQLcqnvM6C6wyYTxdJuQ1BrcBEyRbiiROpYoB4Ac -ONUVXLKHNlCaj6BovOFZ8lVJmGUmy6IuhkYatVOrzkSJewTiXys9oj6YqRXNY/Xc -hlb4XHW2IiS26Z0dZvU5Bau+bwCJP1vw9wUJRSu+FAVwsNIJmMqVDS1FatJIqutW -xKV06t6pEu2mMbMUmkadNy+yUvus/mmDkOMGxiZVwOOuZG4m6EarnZf/ocQL3sai -5Ury1PUtUy/XC59U+f6L1PPDE65FZ5S9WosAmBZD03fIp3KYf5WulmT+gH1JzINq -QZ7NkFbkJ45Bs0nxahfnyM3F58aO2orGbLO27jevcQIDAQABMA0GCSqGSIb3DQEB -CwUAA4IBgQCRQsIVV0JHd+cPxVUmsVvDXrqB2+Gkn7hCWiHJjBiuD5CrmiTn0nj8 -vZcpsVxGH1u40qeH8VBTW9O+V3S95XXbUIH3N5ULae85jFyC1WRi1Yvp4DHhc9Ja -LN5DWgbl003QNeCfwnMxvDVp1Pt98Boz9/YlcpymhAUI9rXoBBDxH/KVraH42ICl -63VDmTOQDHn8wIcIlSCqwoELIm9W9I8qI/hARxwDpbEECgRK39CIqLwx8kKb2BEU -nuNo6gcsFd7SNloVOO3Srw60th2gV5Tqw8dMFFeBAFeU07AnaddIAmzll/e+Inw4 -JK+ysHsIdR7KLsdB74t0z8nDbzm5UkEYxnAkVFEE/l+IcDXlHI7WZ2lERDObjP6l -uZVIZoTzGgSro1fBtrQvKBJFK8tC0/SlzntsH+TIqefUbcgnLWkmxehzEFQfw7/9 -qvWVb/bK1QaPG3mT44a6jf6oEI+VPhQJv8qIWeKTtuwDqX7dH18T0ymzpvNq3zBT -RMjN5YJXvJw= +MIIEbzCCAtegAwIBAgIJAMstgJlaaVJhMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowWzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEOMAwGA1UEAwwF +bm9zYW4wggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAaNCMEAw +HQYDVR0OBBYEFCcEWzdRDZL9LGBh/k+cZiBBLAhFMB8GA1UdIwQYMBaAFMAKKyhD +3l/JfUflR5s28mWMZzviMA0GCSqGSIb3DQEBCwUAA4IBgQCgt0hWxrOSJj/Ykjzt +crWJ6v3JZtq6ao4NBOqy/R/kKdoex49a8Ih03bPwXqfEdxPPqBn78i3uR7QMfFvT +3C8qXL9DIhyR2AN9RJDALf6efIvvOU6zh5nI68K3z4ZlBVKMFblqjc3jKinR9YdC +EcMuQuztJlWN861m9Hly957tvAxap3Sr3FfoXJm2Mo9+WG5wSOpdp/qx/MDpUDqk +UyHni3dULbtkHvyIhpDBA5AXuzzPzuhIMsIH5Y5Pk6ka4vWTqAGfMCYe7bVi6SXF +sDLh/L3WSLRwqeLN9qlCy7skObk0/LnLCQHwXn7vtVnWiDGpTL59W95N7IQboRvY +fYPL8QTJ8fOkCAU8tZYTHTeKI4MihnIXE17oiQZYzYlCcRLlR/z3bpYojxm514Zb +xWIU4VsG5+Bmfvy3nqmZFOUK1t+PtaIadFQw9vS/G0Mdvk84klUQe9hP4DMPQC5Y +7Jx4G0MXs8sL9TTifhGhkLY8eWoLkc4LjdVg5G3IKj1AbRc= -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycacert.pem b/Lib/test/certdata/pycacert.pem index 360cd57426a5a8..c2c8b1ecdcfc97 100644 --- a/Lib/test/certdata/pycacert.pem +++ b/Lib/test/certdata/pycacert.pem @@ -7,93 +7,96 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:b1:84:d3:4f:5c:04:80:91:4f:82:49:ba:30:0b: - f7:e8:cb:f9:14:ef:3d:9f:0b:3f:0a:62:fc:1b:20: - a5:20:d1:60:5f:87:5a:1f:16:d1:ed:97:70:a6:da: - 1b:03:2c:7e:a0:5b:3c:4e:2f:16:7e:0e:89:29:89: - e1:10:0d:38:da:6a:77:5f:37:13:b3:28:8f:7b:5c: - 76:ad:9e:e8:d3:f5:9e:f5:83:aa:10:07:8d:e6:51: - 98:f0:7c:0d:52:f2:0c:21:1e:d8:b9:99:26:a9:25: - 03:27:bb:5c:ab:2e:33:27:a2:d6:23:a8:83:87:44: - 29:9f:97:b5:24:6f:d7:b9:0a:fd:28:ee:bb:fb:41: - 58:ea:1d:99:dd:44:86:ab:98:be:1c:dc:cb:a9:89: - 1d:36:5c:a9:e8:47:b5:f4:52:48:aa:b5:a4:67:ef: - 3e:d7:e2:d3:33:de:98:29:d8:7a:b0:59:5c:e7:b1: - 0e:cc:fd:9f:eb:f6:d5:3a:0e:0b:cf:fe:0b:3d:a2: - bf:45:18:ce:94:e7:a9:55:60:88:d4:d8:84:50:79: - 05:2e:41:03:74:ae:67:26:f6:5b:12:08:98:ce:0a: - 97:ed:01:0f:89:4f:17:5c:fa:3e:1d:35:24:47:92: - 32:bf:f7:a4:18:2b:3c:d0:48:99:e1:a2:cd:a3:cc: - 50:53:20:b5:c6:e3:66:85:7b:57:10:ec:33:4f:c1: - 77:e7:1b:7e:81:c6:c4:f3:45:20:c0:91:dd:13:76: - 7b:03:af:f6:76:8e:a2:83:63:57:dd:63:bc:bb:5a: - 1c:17:52:8a:d6:06:48:cc:0f:c7:d3:4f:e8:da:22: - 6c:86:f9:4e:5c:a6:29:07:3b:d8:56:4c:59:b3:20: - 49:07:7b:94:84:cf:2b:c3:1c:1a:4e:87:64:92:ba: - 42:e1:e6:ad:7d:1d:f6:54:90:6f:2b:e9:b3:cc:4b: - 2b:33:26:23:fd:65:c0:3c:f0:79:ad:c9:c1:81:ef: - 37:04:e0:27:3e:b0:ee:15:be:51 + 00:e6:ba:e2:e4:c1:c2:0c:1c:3e:62:d8:b9:5c:57: + 2e:52:b8:83:c5:88:3a:e6:9a:7a:f5:64:16:33:eb: + 37:6e:2f:7b:f3:68:03:45:65:47:5d:71:10:59:ca: + 2b:1b:00:6c:81:14:61:f4:86:59:3f:ea:fd:78:37: + 16:9d:43:f1:c4:f6:69:8c:c5:29:06:88:9e:26:22: + 04:ac:04:d8:87:34:48:39:eb:6b:f2:0b:92:aa:c3: + 6e:63:51:51:6b:c2:ad:ff:5c:c8:2f:b2:1b:9c:20: + 8a:40:3e:a2:2f:6a:ea:c8:d9:37:43:5c:dc:ed:92: + e2:d9:40:d2:61:9f:71:8a:f5:ed:39:ba:a8:5e:3e: + b5:21:63:10:d8:6f:b4:e2:11:01:0b:10:e8:bb:fb: + 62:ef:48:55:bc:f5:d2:9c:ab:68:ae:95:25:19:f2: + 97:7d:1a:dc:66:ea:88:5e:86:e4:cb:cb:69:4d:5e: + b0:a3:fb:6c:31:e4:28:60:5e:90:f1:d4:2e:10:50: + e1:85:f0:0d:5c:bd:dd:45:24:08:19:3e:1c:93:66: + 8f:2b:da:53:7d:04:1c:0e:42:c4:68:5e:a6:cd:a9: + 18:ed:a7:cd:6a:d0:d1:86:ba:90:ff:b7:4c:de:c7: + 43:24:6d:c7:1c:6b:9c:81:e7:e1:1b:57:25:90:a9: + 0e:c9:56:f3:f6:6b:5e:2d:b4:2e:40:50:9b:42:63: + d2:d6:99:1c:38:dc:cf:2b:2c:a7:72:f1:c7:5e:63: + 34:76:48:f4:3e:88:13:9e:86:16:53:2f:74:fb:87: + 01:8d:22:a4:68:33:ee:13:6c:7a:06:14:54:56:17: + 57:57:98:34:d0:0b:66:09:e3:88:09:f8:a5:15:1c: + 10:73:d0:88:50:99:5e:18:65:3b:ff:31:27:1b:5e: + c6:aa:41:fd:2d:2f:18:a7:c0:f2:ab:c7:22:b5:0b: + 69:d2:73:d1:bb:d0:1c:3d:fa:a4:35:62:cd:33:86: + c7:a0:23:0f:b9:6a:d5:d2:6d:8d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD - - X509v3 Basic Constraints: + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + X509v3 Basic Constraints: critical CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign Signature Algorithm: sha256WithRSAEncryption - 6b:32:2f:e7:05:18:ea:5c:c9:95:f4:e0:c2:0c:41:5f:1a:0a: - 95:c9:c7:7d:05:ee:8a:56:29:35:50:40:b7:fe:9f:7b:5b:1c: - c3:69:2f:a0:cb:d2:b8:91:2f:50:19:62:f7:27:18:6d:95:7b: - 53:16:15:a2:5a:dc:14:e3:fb:b1:32:a9:69:db:a6:33:47:3c: - bb:1f:d2:dc:70:f9:6a:2e:0c:d8:8c:6d:e5:5d:1d:43:3c:4e: - 91:de:a0:c8:da:a0:4b:0e:9d:5e:b6:0f:4a:49:f0:7b:b6:53: - 9e:fd:35:14:5b:e3:4d:b4:18:a6:36:61:e8:8f:33:9b:d4:05: - f9:54:66:df:e0:cb:18:a3:4e:dc:17:a8:a0:b3:c1:a8:f4:d6: - 9d:ca:7f:68:53:1a:d7:95:da:e8:d3:9e:48:00:71:95:99:11: - 07:cf:96:c0:7d:ce:7d:30:e8:4f:e1:83:16:33:a1:ff:59:9b: - 3e:4c:e7:3a:38:01:9f:0f:67:4c:fd:2d:8b:4a:d4:01:46:37: - 33:e8:13:6b:15:a9:1d:68:76:45:a2:82:33:69:26:30:60:05: - c8:8f:bd:b4:75:ab:be:7a:8b:48:68:70:40:b4:1b:51:c5:e6: - 7a:ad:6b:4f:db:17:c0:60:67:2e:63:61:9b:2c:48:99:b8:76: - 45:a0:9e:cc:ef:33:1e:50:4e:ab:72:c3:65:c8:b2:79:b3:35: - 83:21:78:d3:8b:6c:3a:18:e8:65:32:39:b8:c0:9d:71:2f:35: - 36:8a:c0:17:62:d8:8b:3e:e1:22:18:2b:4c:63:a6:0e:9d:0a: - fa:ab:5b:35:fb:88:91:77:4c:8d:8c:9d:a9:cf:fc:ab:c2:e6: - 5a:05:7b:7e:04:6e:39:cf:93:ce:67:3b:7a:cb:af:b6:36:e1: - fb:71:64:45:d4:a6:f0:ce:ef:75:04:99:69:9a:e5:88:0a:10: - 02:74:89:ec:75:84:44:80:48:df:c1:f7:e9:37:ce:ce:92:92: - 5c:89:22:08:73:1f + Signature Value: + bc:e3:56:22:03:e4:5c:b9:67:ff:94:bc:75:9c:00:85:b0:d5: + 9c:c4:c3:29:66:5e:8b:b2:a9:a6:30:86:71:1a:6b:f2:00:c5: + 82:ab:5f:50:04:2a:fb:ed:b8:4c:b9:00:1b:49:57:92:11:cd: + a2:bc:cb:0f:a8:b4:61:f8:14:ca:a0:ec:40:17:ba:55:a1:c4: + bc:a6:b2:5a:ef:f4:20:10:77:47:d0:a0:c5:58:b9:6c:b5:10: + 7b:85:4a:43:a3:fb:2c:01:b9:77:17:b0:be:a0:ee:ae:ae:4d: + 67:86:48:89:57:86:78:ea:3c:ed:f0:41:35:8d:71:68:55:f9: + f2:e9:ac:32:d4:c6:a2:ef:ec:54:e6:c4:8e:2c:fd:bd:aa:60: + 56:65:33:95:ea:10:c6:74:04:eb:2a:6e:9b:11:f6:61:00:aa: + fd:ec:f2:0b:b1:4b:11:cd:93:eb:df:98:ae:4c:b4:07:04:4a: + e5:ef:ff:52:58:75:f5:3e:a4:71:e1:4a:72:5c:a9:8f:d4:aa: + 88:f0:6a:71:b4:c3:00:5f:99:6e:d7:91:af:6c:98:0d:64:c2: + 24:c7:9e:05:11:68:5e:24:62:e3:2e:45:ec:a3:34:f2:a3:9d: + 4d:e5:32:18:2f:74:fc:11:f1:36:50:4f:a0:40:29:68:5c:43: + 4c:23:6c:5d:72:c4:ec:52:76:eb:dc:b2:bc:1f:a6:c4:06:66: + 9b:5c:c7:cc:ca:f2:d1:25:4f:de:a5:1f:8d:e4:0c:49:b6:cf: + 85:40:a1:b9:1f:c6:c7:19:15:07:63:34:93:d0:57:a0:5a:70: + ec:af:4a:1c:72:17:1d:74:a3:6c:31:45:0b:33:7a:a1:b8:46: + db:c7:0e:64:4c:6f:b7:99:04:82:43:1f:e0:59:d6:99:21:27: + 28:09:40:ae:fc:c4:23:aa:a0:0c:08:05:2a:92:1c:db:23:9e: + d1:d5:63:ae:39:13:a3:12:88:5a:43:3c:4a:6e:32:f0:84:9f: + f9:09:0c:91:e7:b8 -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBALGE009cBICRT4JJujAL9+jL+RTvPZ8LPwpi -/BsgpSDRYF+HWh8W0e2XcKbaGwMsfqBbPE4vFn4OiSmJ4RANONpqd183E7Moj3tc -dq2e6NP1nvWDqhAHjeZRmPB8DVLyDCEe2LmZJqklAye7XKsuMyei1iOog4dEKZ+X -tSRv17kK/Sjuu/tBWOodmd1EhquYvhzcy6mJHTZcqehHtfRSSKq1pGfvPtfi0zPe -mCnYerBZXOexDsz9n+v21ToOC8/+Cz2iv0UYzpTnqVVgiNTYhFB5BS5BA3SuZyb2 -WxIImM4Kl+0BD4lPF1z6Ph01JEeSMr/3pBgrPNBImeGizaPMUFMgtcbjZoV7VxDs -M0/Bd+cbfoHGxPNFIMCR3RN2ewOv9naOooNjV91jvLtaHBdSitYGSMwPx9NP6Noi -bIb5TlymKQc72FZMWbMgSQd7lITPK8McGk6HZJK6QuHmrX0d9lSQbyvps8xLKzMm -I/1lwDzwea3JwYHvNwTgJz6w7hW+UQIDAQABo1AwTjAdBgNVHQ4EFgQUs4qgorpx -8agkedSkWyU2FR5JyM0wHwYDVR0jBBgwFoAUs4qgorpx8agkedSkWyU2FR5JyM0w -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAYEAazIv5wUY6lzJlfTgwgxB -XxoKlcnHfQXuilYpNVBAt/6fe1scw2kvoMvSuJEvUBli9ycYbZV7UxYVolrcFOP7 -sTKpadumM0c8ux/S3HD5ai4M2Ixt5V0dQzxOkd6gyNqgSw6dXrYPSknwe7ZTnv01 -FFvjTbQYpjZh6I8zm9QF+VRm3+DLGKNO3BeooLPBqPTWncp/aFMa15Xa6NOeSABx -lZkRB8+WwH3OfTDoT+GDFjOh/1mbPkznOjgBnw9nTP0ti0rUAUY3M+gTaxWpHWh2 -RaKCM2kmMGAFyI+9tHWrvnqLSGhwQLQbUcXmeq1rT9sXwGBnLmNhmyxImbh2RaCe -zO8zHlBOq3LDZciyebM1gyF404tsOhjoZTI5uMCdcS81NorAF2LYiz7hIhgrTGOm -Dp0K+qtbNfuIkXdMjYydqc/8q8LmWgV7fgRuOc+Tzmc7esuvtjbh+3FkRdSm8M7v -dQSZaZrliAoQAnSJ7HWERIBI38H36TfOzpKSXIkiCHMf +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycakey.pem b/Lib/test/certdata/pycakey.pem index 819bdef1ff9bfc..0248f985545e20 100644 --- a/Lib/test/certdata/pycakey.pem +++ b/Lib/test/certdata/pycakey.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCxhNNPXASAkU+C -SbowC/foy/kU7z2fCz8KYvwbIKUg0WBfh1ofFtHtl3Cm2hsDLH6gWzxOLxZ+Dokp -ieEQDTjaandfNxOzKI97XHatnujT9Z71g6oQB43mUZjwfA1S8gwhHti5mSapJQMn -u1yrLjMnotYjqIOHRCmfl7Ukb9e5Cv0o7rv7QVjqHZndRIarmL4c3MupiR02XKno -R7X0UkiqtaRn7z7X4tMz3pgp2HqwWVznsQ7M/Z/r9tU6DgvP/gs9or9FGM6U56lV -YIjU2IRQeQUuQQN0rmcm9lsSCJjOCpftAQ+JTxdc+j4dNSRHkjK/96QYKzzQSJnh -os2jzFBTILXG42aFe1cQ7DNPwXfnG36BxsTzRSDAkd0TdnsDr/Z2jqKDY1fdY7y7 -WhwXUorWBkjMD8fTT+jaImyG+U5cpikHO9hWTFmzIEkHe5SEzyvDHBpOh2SSukLh -5q19HfZUkG8r6bPMSyszJiP9ZcA88HmtycGB7zcE4Cc+sO4VvlECAwEAAQKCAYB7 -gUnzALYxLOgAYYMkQm9si9zz768TpCNr+ooj5YZ9Wq6OSAEveBT+FErQCxaYErDW -qCNA0gn4Eezj9YWcQVa4vzHmEM+n6iRJU39ONC0Qqua5Ma10EY1sHIEnb2dlufku -YeOu3RrEu3eCgRxsDGySuvv5OxinV4kN++KPQzD3EOopPE+U81YFLCsMgsyfPlmm -gwc/IKIuXDHp5Vp2bXkZK98CYLV8RddjUw7SrkZNwx6cI9eET0CgTs7y4SrevoOy -jCdnA0j1HvL8AbLQuYoXo9fdGYDeq55hyYlxSMYLaEToZG3DJ0UAldrT+r7x52D8 -2QMnJUo2XHzVYPlXPJIAkFJisZZ36TkBvywCgXZMMLibPo9U6V0nfkybTtXKoory -nmgBv+XSGSNrVWMiygpDPqpX1G6bBgqUX3CiTlxtSkYYz1M4Vgj2cux5XEPTnVCq -CLVzvNIXZt1RyzXPxGWpPidCjOaiWBRT4u1Dol9fs3PmVvDaRxcKo9nspiUHCfEC -gcEA4GgxZ+IJwpAMHkdYId0oxjKgTqIg+Ua+EwfUoQT10ERl/k/V4cDwJRHT8lML -rKhTNQJMEE040jq+6mPJDl1KqMb/v05Q7fF22ToGw1HkZwK52O6CeEiJW4/J6bR1 -pZGN0irsa6GvzV65Y6gZVFEUl0JPRf8wPvQHXsWAw8/2LuXkXjV0ieIMq4pbWJf4 -kaid7dYLHnobiP9RVk7BGr7ifmCshoPjWp4TRMwYf6iIZrqMxUSX0QY8Xsqx6bch -LLx/AoHBAMqCvvwUKTrF4gKh5jyl6T6DTZ/Dujaz7BuAJdsSSHvuTa/Y1EfsQHZN -jABn89ZqHYDiyyCuVFO3dqhLtsPjhyFMSXj+98JYcL3FGKnqQqRTwtzzx2P2lV5X -U0WhrNRb3iLu79Tr8pE/2EPnvTr+J5b0DHEeRyM72LWs43zrDYHorH0/Aa5Qd37F -gDLCTBEl8jO5irRuAIq/KV9ZFnn8JDjNGVpXgHPW3354ON1YaMLnPASk7FQizSOQ -QZAsyxtdLwKBwGUosvTYYXvygXP4x1LkpmfKFJe94E1exXpAsmovmTvcSXn9tTXC -Sr77LWb0ZrPbYT7pHS7QEMg8MSnp941hIrG4mzs666KHkgLUdI4B0YtaIDsZMXlV -gY3j4KpYbhxH4/2U2eSfC2fxxnKVKW3n6vdQrfmo0q/eQ6BGOgiLK7fybCLHyBQL -8Zg2k3z5bNUEhMTdE0AW3WjBZ4IXmFcdK26616r/szJ7RcZilrydVXexqpmWlTVl -sTst9kucAPlwswKBwQCwf7my/GNezR8Jik+fZj7edBQQfcdra+8JnOvhfpLcKLte -2s1RjjA0q6usou1bYAsszP2bEzV97XWmgq7dFg4tUE7s/NO1d91zGDhBx2Gj1TkN -2A5dKonOuq9iDeITB6qYqcUvvyEfxRRZQr2jj+WzZCr/4BLCO6PJ29A9jKOuKLtF -QcfWRF2RiNMN6lffzkHFIR4p2YHxa2DEsGGtmbt8Ig3Jtl/HFmydzmxJRoev71dY -+ODdB6PhLhZmcRPoWpMCgcEAhGArwL68GwwRMqAX79gMv8tVT0CJnDyGk5mD/ZIB -Nzt0yQFO7rTEa1l1vAtOiVJ9IpAak2lgbEwodOfGnQst7lujNYDFzTRPTFt/lID1 -u6JBxmqawOSlqa00bt4l2YsTZV+BfSznBP6XO1PK4iR3o5G3NkoKJjZWm3e3asHk -6eTeMLcsIJ+Fp7gG0ve2EdQwhVSVMFEu4Q4C2FcJeU++L4kYpY7sTnAjUtiLvtHn -yp3jllEn3CBD8Uhs4B+sL/6p +MIIG/AIBADANBgkqhkiG9w0BAQEFAASCBuYwggbiAgEAAoIBgQDmuuLkwcIMHD5i +2LlcVy5SuIPFiDrmmnr1ZBYz6zduL3vzaANFZUddcRBZyisbAGyBFGH0hlk/6v14 +NxadQ/HE9mmMxSkGiJ4mIgSsBNiHNEg562vyC5Kqw25jUVFrwq3/XMgvshucIIpA +PqIvaurI2TdDXNztkuLZQNJhn3GK9e05uqhePrUhYxDYb7TiEQELEOi7+2LvSFW8 +9dKcq2iulSUZ8pd9Gtxm6ohehuTLy2lNXrCj+2wx5ChgXpDx1C4QUOGF8A1cvd1F +JAgZPhyTZo8r2lN9BBwOQsRoXqbNqRjtp81q0NGGupD/t0zex0Mkbccca5yB5+Eb +VyWQqQ7JVvP2a14ttC5AUJtCY9LWmRw43M8rLKdy8cdeYzR2SPQ+iBOehhZTL3T7 +hwGNIqRoM+4TbHoGFFRWF1dXmDTQC2YJ44gJ+KUVHBBz0IhQmV4YZTv/MScbXsaq +Qf0tLxinwPKrxyK1C2nSc9G70Bw9+qQ1Ys0zhsegIw+5atXSbY0CAwEAAQKCAYBf +4VWcPjBHHA2Iwgr1Jn1nfqmzklL3tUZXZwoa9SoJrc3Sbmy9j8LCP9PNnEehZuGw +GipClPnNp/dA15OcMrnrYYKnLt9Hico+inBqk3Dvbnh9KSmoYcrHD4N13jr5juMD +dSjzOQ5kKNmKrPx0u/dpE2r1oUdlql5+bYN/ceSbHGtCTCDfWSun/iTn7DO8pdhL +IvGz/Fk2mlaWuYiV9lz//5Z1W+w73sesNNYKgf/d+F9/+VNqMXbanLdypJmTBNp9 +eoS3eLk3ycoiGHCnVNm28IjeElFkUOJKVXY39BoMbS/x1MUHjuZDxdOcEaCV9iOr +adg7d91srwJlHGGX3IkC+08J5OJZtfTEMcB3i9g2omOxLAFfapeRxZdD6zMdxWUG +PUNIiy7UKaHe7JJ+fftnk0QBu5DzyrdP0fryFNz9nySot+gzVlU9idTMnIyE73UJ +foPowKpmxDOYNw0ZGXBIP/lMRNXkQM765P1NJpCN0Ud+kBl7GdFMkI++GFjiH3kC +gcEA+VIsOliXn5xfU2zXZb2q/bPPlCBDw/EzIelRTFVqUC2kL3oh4NlDNWAFgcfX +rQUCCoY9rEKYdTQI0w47K4AkJYajnDG2zHj6fkaPcLweyEDa9Hw2ezDDiwz2uwH2 +4HW+JBZ0I7rRuSRXTSJrzFYYVU6tsr54Kl85l2bmL8V2XVYj4ko9uddsyH0F1Ejt +BYPBghr01uEHsO3nIcNMl10fS9eXdhMTr/haXdDxpFXjR5AO/ouQEyGrXk6LPJpn +rUE7AoHBAOzpN8lQDQlf0JcYFBvFd7JdrhpbaIdATmT0NM/Mh+4B8qy5qnIJBdeQ +9T46sHmHzvhtGHmnP+dckKJi1Y/8Lt6noWNYDRnmrDol046ce2Nj8gxDGBT1kG4n +GBoKkA1Gj5N5yWMB8cGTVzQ+tViRMkn9+jmxRaNsW1Y0j7lMtVz0hJOUscbUALV1 +fYGH7zPiqfHoKoz7UuB2OlCKOH0+V96lAO4EgKMQNcEKMs9+Tg13fHTAyodxy/CY +tjUcKpef1wKBwBE24EDjDw0BMf/Dmxe2QdEkkieLFsK3q60iu+9GUoHYtOZmS2KH +/cD4sUilsLmMh/iMDkQPkRE+l4FjESjOvzAsHK3TLOjvTXRckNja1FFFURjiXqyg +0E+QhJSi7RXQa2F4f2pcItDitnhn8QN5ylJRjWKzDf729jYC78/KlYKaSP393Ecx +nZw2LanboynnT/wYumD/xpUrx/Kn1mj5EAkfiKCpbomO30Zs/9I17+xoAPEIV9lK +UNfBGpIDozbuMwKBwFlZmAWf4FrRvSzPEv5qWjt2I2yjXufrs+VVSPm6LOXx7CGC +oKsDhiWH8UZ4AgjD1KZTFvECyBItEgt8dQkp1k95L1/1XHORURFZJNHbaJnSnv5K +67Ez8DXrHqbrpuqq2wmG3BIwMIqOVExK/kAZ+rp3REEv/5CkFEqN5kq/iIM3YSz7 +3pSbbm0Bk8UfjHKoIOowYqPrQZWQYWvwxV9O/PrmhlQ+dHmLaoqUmxcwjqV7k//A +mmG85GqoXcfoCJRI3wKBwBdnxBzg17TMFufuvX2Bc/M9MqL3+vlwH6SDdr+2yYKA +hiD8Ur2OwtDGHnV4m3NeA/Guyz32H4CzvFAnpzlvMow/dvfp9JUcpdeidhIBZy8V +D7VODIiCyyTAb3g1LK0+HTEHAVRFihbNXhub/P6NckFXw0MJJQOvNNsYQnJTUngY +oxqdt1HeAujEwBrRSfrOGE2K8FVJ/MYf4PmxTIocICBk1/BmNHsUeJ5yFUDBweh5 +UJN6yp5PiGwvW8WFl4waXw== -----END PRIVATE KEY----- diff --git a/Lib/test/certdata/revocation.crl b/Lib/test/certdata/revocation.crl index 621675eb5c183d..a10586d1f79345 100644 --- a/Lib/test/certdata/revocation.crl +++ b/Lib/test/certdata/revocation.crl @@ -1,14 +1,14 @@ -----BEGIN X509 CRL----- -MIICJjCBjwIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +MIICKDCBkQIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j -YS1zZXJ2ZXIXDTIxMDMxNzA4NDgyMFoXDTQwMDUxNjA4NDgyMFqgDjAMMAoGA1Ud -FAQDAgEAMA0GCSqGSIb3DQEBCwUAA4IBgQCd2GrHb4zr2R8eK7YMHwlkgICxbWP1 -4nuEi55yzUcmMcCZJ6ZQV3yYqTlAULGQ9qWAUdhsyH+yu3hRKFKHQv0DAdKKxgow -66YasAQQ99DskXOPxmRoIA7qtIWZbLtBwHQJWh+uUFlTdUXitGIX5Xie74xu5YIr -moa3QeuZyG5+gigSTUyst5T/J/cHfBzlAJLc2k3Ty4EPYXKHCVnrZWJbRmxq199l -A7S+eBb9qWXSYXCn6v+EZ76pUS3u/66kZ86PO3h9294BzdhxbCJ27dQXNHw6owe2 -Iyiv0aWx+TNSGSf4yCqaYTH6RtEoviI3h/inVFHNGgjlMzdaGw/0I3bkB0rt2WSR -Vck37HnXvQvVEkgO/39C0WKZus6m4gmOgZcbJbXaR8uIR5Hmw3SEyGEPEIBu6tXV -BLJOSOSu2vVUH5GUIrpvK9FTySKYa+MGryoPasuqZNfwpaXK+ON2G6QsmcXPWZY0 -Dry6t0w2geW6UYVGmb831i8ZP3JVVVwcwi0= +YS1zZXJ2ZXIXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWqAOMAwwCgYD +VR0UBAMCAQAwDQYJKoZIhvcNAQELBQADggGBAKN4S2g4wCeU1fO5TSckAwxgdzdh +pY28f4musnQt7l37MzB2gmJVDSCZfQyrUnfSEST15WEY7CZVyTlbsu6gYKK53Yej +j3ORBfGUgzaz62Hs8di7SrHDzWUlNCFa47YFWDmtj96KTX1AItnpkCCE58Wfpivp +Hu+YINFpi/2vI2nvP/xcfvgT3dXek9kyz+2jHmadxcn2VerSBZZ9fiZk/k4NzgoI +JdiSswtN1c5GelHQfftwRXbWqsp6TvgHC5MagDuHh5Bj7/DftI7nCy0IT5GnP8lS +ZqmXUMpa8zbtSNSTIk0XepmypNW8HHMQbfJp0y7yOQ4pPyXICrjYTg7wKpODRcm3 +BRN89vvNfCszMU41glVfQG+2Po5uAMTl1hX8WYSj0+Xxrdg+wgJead4S5Sq3CgMT +bKsH2Dqh43L8BTxuxzLQyduK0gKSl8vlN7a9Bzm3IXYlyk+kKSyo4jP8XK79pj1k +1JglMFM9jpoMF2VmNjiROtVEl2tbDGwlvpjWYQ== -----END X509 CRL----- diff --git a/Lib/test/certdata/ssl_cert.pem b/Lib/test/certdata/ssl_cert.pem index de596717bd855f..6db52404942210 100644 --- a/Lib/test/certdata/ssl_cert.pem +++ b/Lib/test/certdata/ssl_cert.pem @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODA4 -MjkxNDIzMTVaFw0yODA4MjYxNDIzMTVaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGP -ADCCAYoCggGBALKUqUtopT6E68kN+uJNEt34i2EbmG/bwjcD8IaMsgJPSsMO2Bpd -3S6qWgkCeOyCfmAwBxK2kNbxGb63ouysEv7l8GCTJTWv3hG/HQcejJpnAEGi6K1U -fDbyE/db6yZ12SoHVTGkadN4vYGCPd1Wj9ZO1F877SHQ8rDWX3xgTWkxN2ojBw44 -T8RHSDiG8D/CvG4uEy+VUszL+Uvny5y2poNSqvI3J56sptWSrh8nIIbkPZPBdUne -LYMOHTFK3ZjXSmhlXgziTxK71nnzM3Y9K9gxPnRqoXbvu/wFo55hQCkETiRkYgmm -jXcBMZ0TClQVnQWuLjMthRnWFZs4Lfmwqjs7FZD/61581R2BYehvpWbLvvuOJhwv -DFzexL2sXcAl7SsxbzeQKRHqGbIDfbnQTXfs3/VC6Ye5P82P2ucj+XC32N9piRmO -gCBP8L3ub+YzzdxikZN2gZXXE2jsb3QyE/R2LkWdWyshpKe+RsZP1SBRbHShUyOh -yJ90baoiEwj2mwIDAQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZI -hvcNAQELBQADggGBAHRUO/UIHl3jXQENewYayHxkIx8t7nu40iO2DXbicSijz5bo -5//xAB6RxhBAlsDBehgQP1uoZg+WJW+nHu3CIVOU3qZNZRaozxiCl2UFKcNqLOmx -R3NKpo1jYf4REQIeG8Yw9+hSWLRbshNteP6bKUUf+vanhg9+axyOEOH/iOQvgk/m -b8wA8wNa4ujWljPbTQnj7ry8RqhTM0GcAN5LSdSvcKcpzLcs3aYwh+Z8e30sQWna -F40sa5u7izgBTOrwpcDm/w5kC46vpRQ5fnbshVw6pne2by0mdMECASid/p25N103 -jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu -0x8evpvwuhdIyproejNFlBpKmW8OX7yKTCPPMC/VkX8Q1rVkxU0DQ6hmvwZlhoKa -9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW -HcVKQHyOeyvnINuBAQ== +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/ssl_key.passwd.pem b/Lib/test/certdata/ssl_key.passwd.pem index 46de61ab85b136..e60a66c2798784 100644 --- a/Lib/test/certdata/ssl_key.passwd.pem +++ b/Lib/test/certdata/ssl_key.passwd.pem @@ -1,42 +1,42 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQI072N7W+PDDMCAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBA/AuaRNi4vE4KGqI4In+70BIIH -ENGS5Vex5NID873frmd1UZEHZ+O/Bd0wDb+NUpIqesHkRYf7kKi6Gnr+nKQ/oVVn -Lm3JjE7c8ECP0OkOOXmiXuWL1SkzBBWqCI4stSGUPvBiHsGwNnvJAaGjUffgMlcC -aJOA2+dnejLkzblq4CB2LQdm06N3Xoe9tyqtQaUHxfzJAf5Ydd8uj7vpKN2MMhY7 -icIPJwSyh0N7S6XWVtHEokr9Kp4y2hS5a+BgCWV1/1z0aF7agnSVndmT1VR+nWmc -lM14k+lethmHMB+fsNSjnqeJ7XOPlOTHqhiZ9bBSTgF/xr5Bck/NiKRzHjdovBox -TKg+xchaBhpRh7wBPBIlNJeHmIjv+8obOKjKU98Ig/7R9+IryZaNcKAH0PuOT+Sw -QHXiCGQbOiYHB9UyhDTWiB7YVjd8KHefOFxfHzOQb/iBhbv1x3bTl3DgepvRN6VO -dIsPLoIZe42sdf9GeMsk8mGJyZUQ6AzsfhWk3grb/XscizPSvrNsJ2VL1R7YTyT3 -3WA4ZXR1EqvXnWL7N/raemQjy62iOG6t7fcF5IdP9CMbWP+Plpsz4cQW7FtesCTq -a5ZXraochQz361ODFNIeBEGU+0qqXUtZDlmos/EySkZykSeU/L0bImS62VGE3afo -YXBmznTTT9kkFkqv7H0MerfJsrE/wF8puP3GM01DW2JRgXRpSWlvbPV/2LnMtRuD -II7iH4rWDtTjCN6BWKAgDOnPkc9sZ4XulqT32lcUeV6LTdMBfq8kMEc8eDij1vUT -maVCRpuwaq8EIT3lVgNLufHiG96ojlyYtj3orzw22IjkgC/9ee8UDik9CqbMVmFf -fVHhsw8LNSg8Q4bmwm5Eg2w2it2gtI68+mwr75oCxuJ/8OMjW21Prj8XDh5reie2 -c0lDKQOFZ9UnLU1bXR/6qUM+JFKR4DMq+fOCuoQSVoyVUEOsJpvBOYnYZN9cxsZm -vh9dKafMEcKZ8flsbr+gOmOw7+Py2ifSlf25E/Frb1W4gtbTb0LQVHb6+drutrZj -8HEu4CnHYFCD4ZnOJb26XlZCb8GFBddW86yJYyUqMMV6Q1aJfAOAglsTo1LjIMOZ -byo0BTAmwUevU/iuOXQ4qRBXXcoidDcTCrxfUSPG9wdt9l+m5SdQpWqfQ+fx5O7m -SLlrHyZCiPSFMtC9DxqjIklHjf5W3wslGLgaD30YXa4VDYkRihf3CNsxGQ+tVvef -l0ZjoAitF7Gaua06IESmKnpHe23dkr1cjYq+u2IV+xGH8LeExdwsQ9kpuTeXPnQs -JOA99SsFx1ct32RrwjxnDDsiNkaViTKo9GDkV3jQTfoFgAVqfSgg9wGXpqUqhNG7 -TiSIHCowllLny2zn4XrXCy2niD3VDt0skb3l/PaegHE2z7S5YY85nQtYwpLiwB9M -SQ08DYKxPBZYKtS2iZ/fsA1gjSRQDPg/SIxMhUC3M3qH8iWny1Lzl25F2Uq7VVEX -LdTUtaby49jRTT3CQGr5n6z7bMbUegiY7h8WmOekuThGDH+4xZp6+rDP4GFk4FeK -JcF70vMQYIjQZhadic6olv+9VtUP42ltGG/yP9a3eWRkzfAf2eCh6B1rYdgEWwE8 -rlcZzwM+y6eUmeNF2FVWB8iWtTMQHy+dYNPM+Jtus1KQKxiiq/yCRs7nWvzWRFWA -HRyqV0J6/lqgm4FvfktFt1T0W+mDoLJOR2/zIwMy2lgL5zeHuR3SaMJnCikJbqKS -HB3UvrhAWUcZqdH29+FhVWeM7ybyF1Wccmf+IIC/ePLa6gjtqPV8lG/5kbpcpnB6 -UQY8WWaKMxyr3jJ9bAX5QKshchp04cDecOLZrpFGNNQngR8RxSEkiIgAqNxWunIu -KrdBDrupv/XAgEOclmgToY3iywLJSV5gHAyHWDUhRH4cFCLiGPl4XIcnXOuTze3H -3j+EYSiS3v3DhHjp33YU2pXlJDjiYsKzAXejEh66++Y8qaQdCAad3ruWRCzW3kgk -Md0A1VGzntTnQsewvExQEMZH2LtYIsPv3KCYGeSAuLabX4tbGk79PswjnjLLEOr0 -Ghf6RF6qf5/iFyJoG4vrbKT8kx6ywh0InILCdjUunuDskIBxX6tEcr9XwajoIvb2 -kcmGdjam5kKLS7QOWQTl8/r/cuFes0dj34cX5Qpq+Gd7tRq/D+b0207926Cxvftv -qQ1cVn8HiLxKkZzd3tpf2xnoV1zkTL0oHrNg+qzxoxXUTUcwtIf1d/HRbYEAhi/d -bBBoFeftEHWNq+sJgS9bH+XNzo/yK4u04B5miOq8v4CSkJdzu+ZdF22d4cjiGmtQ -8BTmcn0Unzm+u5H0+QSZe54QBHJGNXXOIKMTkgnOdW27g4DbI1y7fCqJiSMbRW6L -oHmMfbdB3GWqGbsUkhY8i6h9op0MU6WOX7ea2Rxyt4t6 +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQMMUN+qWFiwTbIjjb +cLEtYQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEGGzvsc6e5Zn2M9S +sdoMbaIEggcQ0mQYuMcujHtFfRKLmfclrf/dHOYgnVYc2c9MSR3jfoOwnTdBR1Op +MUuO4ukqrxGkrJBy+MLb1oPEGrnvn39pdBMZx4CBmdwX0jDq36UL9EU5/m071mT+ +Xoe/z/RHtrtmbpA9XKu7n/5gLi9jg57qorss8r5MaPHghm2ZOqaQZIOOMCF+4gNU +Cmp/NHL9TUM/sfQpaw9YM4VuTRyHNO3OiDKaCCz2romFrgM1TYz/I5/n5MMR543J +dsyIFzef3jHYfaXTl97v9ibGVpIUCKfHgYlQf41H+Me890AP4HaS0e2y+73/UwjB +4YSzPVm+lCiBWhFDbMuRmPY3YDQOA+TV06oCO8/aMamPRsryl8g7iOQAIz4hQdoP +ZTXtSZ+F68YmfPjKAj3NNQosvLzfuSfxZ9HqyvZ/0w1eead7pdSs7uvCEvN2Zcvk +a9EWy1bM895zD3DwrxqGnSYbitXouIOf9zHsl2lRUK44XmvgTHoNaJFIVyzVNgQr +e1YJM7LbEvErspa6cZTz7DCR44g9cYBi393XEt5yl4NCtq2PFq762pNqpsQqt5/Y +pLX5w12npf+OZ1WqJla89FabqLaJ1blmDEgfko0XriofW5gWLByvhYINtt+u5x/5 +QdTya2gWBxI2K+sAeGIt913RBAus8hLEDnVGDX/FEthz1OOKLXp0G6HQmqUom5PU +/O36GszhWIz1Q/SpMfZkdHfEfbJFCx0mpaJZTlC01hg+OpnwWxUvO2dxrK46xAzd +/obsVQRLOuyTxjIrQd+vVZRIcOroTHVUbh7qV2NV1sdQbsKnc4mYK8jjHsb/v/O3 +lcOFJX2V55z7FdEOi7qKKn3l5j6SNmdMGlGde16692VCdYo4Wjs2ufkYryJoJQ7L +O3AeoXVrpc2JwKj6GFKN3Bw6A3XEXTShWdHS8DAuHdbLBhHESeb/qA76qNInw/od +PXtsRNBWaS87rh7GGWrO2ULHBk0k5hLa9puXgAZVh5NcrVuujWM2spGPHGJWhvDt +ffOJAe0Kra8b1CHOC9aRUuHnapuu/mAD+XZTAt/UUNq0hdTDO1VTxL4lHpZjmWJ3 +OvmTp1HpA8FpSmzSBw6H0beVT9LXDbkSax86E2YQJPcC9IjQARoaL+7ba1XdXeSQ +k796zrq4YD1SB09qRVVar8nBSHUYAzG2gbAWgAQgOjwtC5nT1Que3kI7weG2nFZ4 +B1MfBmpEqPwMs8O5hurdYmfpBRoudivKrK5si+LB7og1IwVD2IS8mm55/Kmnugf4 +YCEiGu5s9F+njqEZBV7UPAabaFph6Iw3Jfmcg0YUZV60hY6O3EieonfwVBBKWBQp +K0SUs24Ld80B3oYWuqYI+MQlSprskRKFBfx7hp2PsQU9jAVPc8NohmfM5IJAFBwo +rCqAOsRsrRZnVoaRP89X50VuBg6ROiu/cmI5sqou3u30Pndzmlbcg8ekWj+1qrur +prva6uPc0oiCgFTMksFnhKvlHErIm+ceGvNSib+rWNomaL5cjpgnqAG6lZMP7N+p +QvH+ABchP+fJyyl/XuaOLmcXDSkIcadfjJSHACwmGRas3unfOYEyBg29oPu6PNlW +jXFJb7OzWaQfFNcCnHzvYGTvmrbg7VqBVqmVRYX3r6WNDJeaSwDJpxYNNQlTugbm +FOlD9JTeHyZ17rrqiCitbQwBVBpWOhBEIkFD+3JL1ZdGjyc3rcvbERLr/UN649vS +FcMOLiEvFjFdirq6Pe9fx/VD1GrIzXaEhvyCePSeoV6eILm7SSFTVCm+JMtiBlBi +ZjDxlUhQGuMg3IwZRqjfUR90wo3QWe5XOgM9mJa5qY/Yaa9YpFEJcd4mYnXcPLIz +eY8lqGAhROwJhGRhQoTQXucsSEhAkUBpRiE7UCN8OjezvAeFn0+6XyHeST+QZenX +ixAKJ27lFn/njvL4sDocd7ZvXpb3P5ZxCRakMnjunQQyjtUlJmPrNjs+ZlPenLuh +UA3Oj5d96dDqzgZNLxbDHKr6B+CMApBrwUDcum09PYgJ5xZ/Hrct4iSa57Gi528L +l0dcVgPHd80oIn/vyhjVYkkXNMrRhTJIgLuh0KsddZ7/8Xvxma9W8hNQ1sYwE0Yp +RqLgMRdFpTSN8hKUTfRvjzbrW4nQ2XVqAdyM6PEkdrPCBZczdGisp9oMF6woI/pA +ZKNxdUr0DG3tVBJ7z6qfdp2j+yHvhqQt+ohmk3YldPKpXfYz20ZJ52URE8vbCj2K +NZ/MVEACbg6FDgNQ1bIKD61pKL72+SjyKW1wQfIkeqMfOBI8BGclp7BL6dGRrVHN +PLg2j9gsgZ3XsiJOFtJ6Q3UABkeUrbRvAFQXPM8+keWU4VP/99dCJTvnEQTM6O8U +gz2NA4j1ZEOmB5L+hmf2gW+xgW6eeMjylj5JdeAliNaUqIhOju30vo4= -----END ENCRYPTED PRIVATE KEY----- diff --git a/Lib/test/certdata/ssl_key.pem b/Lib/test/certdata/ssl_key.pem index 1ea4578d81ecc4..1ad9e5e2b832cf 100644 --- a/Lib/test/certdata/ssl_key.pem +++ b/Lib/test/certdata/ssl_key.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/wIBADANBgkqhkiG9w0BAQEFAASCBukwggblAgEAAoIBgQCylKlLaKU+hOvJ -DfriTRLd+IthG5hv28I3A/CGjLICT0rDDtgaXd0uqloJAnjsgn5gMAcStpDW8Rm+ -t6LsrBL+5fBgkyU1r94Rvx0HHoyaZwBBouitVHw28hP3W+smddkqB1UxpGnTeL2B -gj3dVo/WTtRfO+0h0PKw1l98YE1pMTdqIwcOOE/ER0g4hvA/wrxuLhMvlVLMy/lL -58uctqaDUqryNyeerKbVkq4fJyCG5D2TwXVJ3i2DDh0xSt2Y10poZV4M4k8Su9Z5 -8zN2PSvYMT50aqF277v8BaOeYUApBE4kZGIJpo13ATGdEwpUFZ0Fri4zLYUZ1hWb -OC35sKo7OxWQ/+tefNUdgWHob6Vmy777jiYcLwxc3sS9rF3AJe0rMW83kCkR6hmy -A3250E137N/1QumHuT/Nj9rnI/lwt9jfaYkZjoAgT/C97m/mM83cYpGTdoGV1xNo -7G90MhP0di5FnVsrIaSnvkbGT9UgUWx0oVMjocifdG2qIhMI9psCAwEAAQKCAYBT -sHmaPmNaZj59jZCqp0YVQlpHWwBYQ5vD3pPE6oCttm0p9nXt/VkfenQRTthOtmT1 -POzDp00/feP7zeGLmqSYUjgRekPw4gdnN7Ip2PY5kdW77NWwDSzdLxuOS8Rq1MW9 -/Yu+ZPe3RBlDbT8C0IM+Atlh/BqIQ3zIxN4g0pzUlF0M33d6AYfYSzOcUhibOO7H -j84r+YXBNkIRgYKZYbutRXuZYaGuqejRpBj3voVu0d3Ntdb6lCWuClpB9HzfGN0c -RTv8g6UYO4sK3qyFn90ibIR/1GB9watvtoWVZqggiWeBzSWVWRsGEf9O+Cx4oJw1 -IphglhmhbgNksbj7bD24on/icldSOiVkoUemUOFmHWhCm4PnB1GmbD8YMfEdSbks -qDr1Ps1zg4mGOinVD/4cY7vuPFO/HCH07wfeaUGzRt4g0/yLr+XjVofOA3oowyxv -JAzr+niHA3lg5ecj4r7M68efwzN1OCyjMrVJw2RAzwvGxE+rm5NiT08SWlKQZnkC -gcEA4wvyLpIur/UB84nV3XVJ89UMNBLm++aTFzld047BLJtMaOhvNqx6Cl5c8VuW -l261KHjiVzpfNM3/A2LBQJcYkhX7avkqEXlj57cl+dCWAVwUzKmLJTPjfaTTZnYJ -xeN3dMYjJz2z2WtgvfvDoJLukVwIMmhTY8wtqqYyQBJ/l06pBsfw5TNvmVIOQHds -8ASOiFt+WRLk2bl9xrGGayqt3VV93KVRzF27cpjOgEcG74F3c0ZW9snERN7vIYwB -JfrlAoHBAMlahPwMP2TYylG8OzHe7EiehTekSO26LGh0Cq3wTGXYsK/q8hQCzL14 -kWW638vpwXL6L9ntvrd7hjzWRO3vX/VxnYEA6f0bpqHq1tZi6lzix5CTUN5McpDg -QnjenSJNrNjS1zEF8WeY9iLEuDI/M/iUW4y9R6s3WpgQhPDXpSvd2g3gMGRUYhxQ -Xna8auiJeYFq0oNaOxvJj+VeOfJ3ZMJttd+Y7gTOYZcbg3SdRb/kdxYki0RMD2hF -4ZvjJ6CTfwKBwQDiMqiZFTJGQwYqp4vWEmAW+I4r4xkUpWatoI2Fk5eI5T9+1PLX -uYXsho56NxEU1UrOg4Cb/p+TcBc8PErkGqR0BkpxDMOInTOXSrQe6lxIBoECVXc3 -HTbrmiay0a5y5GfCgxPKqIJhfcToAceoVjovv0y7S4yoxGZKuUEe7E8JY2iqRNAO -yOvKCCICv/hcN235E44RF+2/rDlOltagNej5tY6rIFkaDdgOF4bD7f9O5eEni1Bg -litfoesDtQP/3rECgcEAkQfvQ7D6tIPmbqsbJBfCr6fmoqZllT4FIJN84b50+OL0 -mTGsfjdqC4tdhx3sdu7/VPbaIqm5NmX10bowWgWSY7MbVME4yQPyqSwC5NbIonEC -d6N0mzoLR0kQ+Ai4u+2g82gicgAq2oj1uSNi3WZi48jQjHYFulCbo246o1NgeFFK -77WshYe2R1ioQfQDOU1URKCR0uTaMHClgfu112yiGd12JAD+aF3TM0kxDXz+sXI5 -SKy311DFxECZeXRLpcC3AoHBAJkNMJWTyPYbeVu+CTQkec8Uun233EkXa2kUNZc/ -5DuXDaK+A3DMgYRufTKSPpDHGaCZ1SYPInX1Uoe2dgVjWssRL2uitR4ENabDoAOA -ICVYXYYNagqQu5wwirF0QeaMXo1fjhuuHQh8GsMdXZvYEaAITZ9/NG5x/oY08+8H -kr78SMBOPy3XQn964uKG+e3JwpOG14GKABdAlrHKFXNWchu/6dgcYXB87mrC/GhO -zNwzC+QhFTZoOomFoqMgFWujng== +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn -----END PRIVATE KEY----- diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 168f6f73f6186f..e22324efc490be 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -550,10 +550,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + if (PyBytes_Check(args[0])) { + if (PyBytes_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 1 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[0])); + goto exit; + } a = PyBytes_AS_STRING(args[0])[0]; } - else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + else if (PyByteArray_Check(args[0])) { + if (PyByteArray_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 1 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[0])); + goto exit; + } a = PyByteArray_AS_STRING(args[0])[0]; } else { @@ -563,10 +577,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } b = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } b = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -576,10 +604,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + if (PyBytes_Check(args[2])) { + if (PyBytes_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 3 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[2])); + goto exit; + } c = PyBytes_AS_STRING(args[2])[0]; } - else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + else if (PyByteArray_Check(args[2])) { + if (PyByteArray_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 3 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[2])); + goto exit; + } c = PyByteArray_AS_STRING(args[2])[0]; } else { @@ -589,10 +631,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + if (PyBytes_Check(args[3])) { + if (PyBytes_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 4 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[3])); + goto exit; + } d = PyBytes_AS_STRING(args[3])[0]; } - else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + else if (PyByteArray_Check(args[3])) { + if (PyByteArray_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 4 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[3])); + goto exit; + } d = PyByteArray_AS_STRING(args[3])[0]; } else { @@ -602,10 +658,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + if (PyBytes_Check(args[4])) { + if (PyBytes_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 5 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[4])); + goto exit; + } e = PyBytes_AS_STRING(args[4])[0]; } - else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + else if (PyByteArray_Check(args[4])) { + if (PyByteArray_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 5 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[4])); + goto exit; + } e = PyByteArray_AS_STRING(args[4])[0]; } else { @@ -615,10 +685,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 6) { goto skip_optional; } - if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + if (PyBytes_Check(args[5])) { + if (PyBytes_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 6 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[5])); + goto exit; + } f = PyBytes_AS_STRING(args[5])[0]; } - else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + else if (PyByteArray_Check(args[5])) { + if (PyByteArray_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 6 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[5])); + goto exit; + } f = PyByteArray_AS_STRING(args[5])[0]; } else { @@ -628,10 +712,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 7) { goto skip_optional; } - if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + if (PyBytes_Check(args[6])) { + if (PyBytes_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 7 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[6])); + goto exit; + } g = PyBytes_AS_STRING(args[6])[0]; } - else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + else if (PyByteArray_Check(args[6])) { + if (PyByteArray_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 7 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[6])); + goto exit; + } g = PyByteArray_AS_STRING(args[6])[0]; } else { @@ -641,10 +739,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 8) { goto skip_optional; } - if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + if (PyBytes_Check(args[7])) { + if (PyBytes_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 8 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[7])); + goto exit; + } h = PyBytes_AS_STRING(args[7])[0]; } - else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + else if (PyByteArray_Check(args[7])) { + if (PyByteArray_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 8 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[7])); + goto exit; + } h = PyByteArray_AS_STRING(args[7])[0]; } else { @@ -654,10 +766,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 9) { goto skip_optional; } - if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + if (PyBytes_Check(args[8])) { + if (PyBytes_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 9 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[8])); + goto exit; + } i = PyBytes_AS_STRING(args[8])[0]; } - else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + else if (PyByteArray_Check(args[8])) { + if (PyByteArray_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 9 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[8])); + goto exit; + } i = PyByteArray_AS_STRING(args[8])[0]; } else { @@ -667,10 +793,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 10) { goto skip_optional; } - if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + if (PyBytes_Check(args[9])) { + if (PyBytes_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 10 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[9])); + goto exit; + } j = PyBytes_AS_STRING(args[9])[0]; } - else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + else if (PyByteArray_Check(args[9])) { + if (PyByteArray_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 10 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[9])); + goto exit; + } j = PyByteArray_AS_STRING(args[9])[0]; } else { @@ -680,10 +820,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 11) { goto skip_optional; } - if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + if (PyBytes_Check(args[10])) { + if (PyBytes_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 11 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[10])); + goto exit; + } k = PyBytes_AS_STRING(args[10])[0]; } - else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + else if (PyByteArray_Check(args[10])) { + if (PyByteArray_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 11 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[10])); + goto exit; + } k = PyByteArray_AS_STRING(args[10])[0]; } else { @@ -693,10 +847,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 12) { goto skip_optional; } - if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + if (PyBytes_Check(args[11])) { + if (PyBytes_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 12 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[11])); + goto exit; + } l = PyBytes_AS_STRING(args[11])[0]; } - else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + else if (PyByteArray_Check(args[11])) { + if (PyByteArray_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 12 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[11])); + goto exit; + } l = PyByteArray_AS_STRING(args[11])[0]; } else { @@ -706,10 +874,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 13) { goto skip_optional; } - if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + if (PyBytes_Check(args[12])) { + if (PyBytes_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 13 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[12])); + goto exit; + } m = PyBytes_AS_STRING(args[12])[0]; } - else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + else if (PyByteArray_Check(args[12])) { + if (PyByteArray_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 13 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[12])); + goto exit; + } m = PyByteArray_AS_STRING(args[12])[0]; } else { @@ -719,10 +901,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 14) { goto skip_optional; } - if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + if (PyBytes_Check(args[13])) { + if (PyBytes_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 14 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[13])); + goto exit; + } n = PyBytes_AS_STRING(args[13])[0]; } - else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + else if (PyByteArray_Check(args[13])) { + if (PyByteArray_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 14 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[13])); + goto exit; + } n = PyByteArray_AS_STRING(args[13])[0]; } else { @@ -740,7 +936,7 @@ static PyObject * test_char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, char f, char g, char h, char i, char j, char k, char l, char m, char n) -/*[clinic end generated code: output=98589f02422fe6b1 input=e42330417a44feac]*/ +/*[clinic end generated code: output=ff11e203248582df input=e42330417a44feac]*/ /*[clinic input] @@ -1028,7 +1224,10 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[2]) != 1) { - _PyArg_BadArgument("test_int_converter", "argument 3", "a unicode character", args[2]); + PyErr_Format(PyExc_TypeError, + "test_int_converter(): argument 3 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[2])); goto exit; } c = PyUnicode_READ_CHAR(args[2], 0); @@ -1048,7 +1247,7 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) static PyObject * test_int_converter_impl(PyObject *module, int a, int b, int c, myenum d) -/*[clinic end generated code: output=5aed87a7589eefb2 input=d20541fc1ca0553e]*/ +/*[clinic end generated code: output=fbcfb7554688663d input=d20541fc1ca0553e]*/ /*[clinic input] @@ -1980,7 +2179,7 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2014,7 +2213,7 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec static PyObject * test_keywords_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=73d46a9ae3320f96 input=0d3484844749c05b]*/ +/*[clinic end generated code: output=13ba007e1c842a37 input=0d3484844749c05b]*/ /*[clinic input] @@ -2050,7 +2249,7 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2084,7 +2283,7 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=c9f02a41f425897d input=384adc78bfa0bff7]*/ +/*[clinic end generated code: output=789799a6d2d6eb4d input=384adc78bfa0bff7]*/ /*[clinic input] @@ -2121,7 +2320,7 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2168,7 +2367,7 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO static PyObject * test_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=b35d4e66f7283e46 input=eda7964f784f4607]*/ +/*[clinic end generated code: output=42430dd8ea5afde6 input=eda7964f784f4607]*/ /*[clinic input] @@ -2207,7 +2406,7 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2265,7 +2464,7 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=ede7e6e65106bf2b input=209387a4815e5082]*/ +/*[clinic end generated code: output=f312c35c380d2bf9 input=209387a4815e5082]*/ /*[clinic input] @@ -2303,7 +2502,7 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2350,7 +2549,7 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=36d4df939a4c3eef input=18393cc64fa000f4]*/ +/*[clinic end generated code: output=3937da2a8233ebe0 input=18393cc64fa000f4]*/ /*[clinic input] @@ -2386,7 +2585,7 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2420,7 +2619,7 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=4835f4b6cf386c28 input=1767b0ebdf06060e]*/ +/*[clinic end generated code: output=6b4f6dd5f4db3877 input=1767b0ebdf06060e]*/ /*[clinic input] @@ -2457,7 +2656,7 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2491,7 +2690,7 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P static PyObject * test_posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *c) -/*[clinic end generated code: output=2570ea156a8d3cb5 input=9042f2818f664839]*/ +/*[clinic end generated code: output=8bef2a8198e70b26 input=9042f2818f664839]*/ /*[clinic input] @@ -2530,7 +2729,7 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2567,7 +2766,7 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=aaa0e6b5ce02900d input=29546ebdca492fea]*/ +/*[clinic end generated code: output=a44b8ae8300955e1 input=29546ebdca492fea]*/ /*[clinic input] @@ -2606,7 +2805,7 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2655,7 +2854,7 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na static PyObject * test_posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=1d9f2d8420d0a85f input=cdf5a9625e554e9b]*/ +/*[clinic end generated code: output=cae6647c9e8e0238 input=cdf5a9625e554e9b]*/ /*[clinic input] @@ -2693,7 +2892,7 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2740,7 +2939,7 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n static PyObject * test_posonly_keywords_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=a83caa0505b296cf input=1581299d21d16f14]*/ +/*[clinic end generated code: output=6526fd08aafa2149 input=1581299d21d16f14]*/ /*[clinic input] @@ -2779,7 +2978,7 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_ PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2833,7 +3032,7 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_ static PyObject * test_posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=0b24fba3dc04d26b input=408798ec3d42949f]*/ +/*[clinic end generated code: output=b8d01e98443738c2 input=408798ec3d42949f]*/ /*[clinic input] @@ -2873,7 +3072,7 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2922,7 +3121,7 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg static PyObject * test_posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=592b217bca2f7bcc input=8d8e5643bbbc2309]*/ +/*[clinic end generated code: output=81d71c288f13d4dc input=8d8e5643bbbc2309]*/ /*[clinic input] @@ -2961,7 +3160,7 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3008,7 +3207,7 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_posonly_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=b8b00420826bc11f input=f7e5eed94f75fff0]*/ +/*[clinic end generated code: output=a717d2a1a3310289 input=f7e5eed94f75fff0]*/ /*[clinic input] @@ -3048,7 +3247,7 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3102,7 +3301,7 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=3b9ee879ebee285a input=1e557dc979d120fd]*/ +/*[clinic end generated code: output=0f50b4b8d45cf2de input=1e557dc979d120fd]*/ /*[clinic input] @@ -3144,7 +3343,7 @@ test_posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssi PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3196,7 +3395,7 @@ static PyObject * test_posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=d380f84f81cc0e45 input=c3884a4f956fdc89]*/ +/*[clinic end generated code: output=8dac8d2a4e6105fa input=c3884a4f956fdc89]*/ /*[clinic input] @@ -3236,7 +3435,7 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3285,7 +3484,7 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss static PyObject * test_posonly_keywords_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=ee629e962cb06992 input=68d01d7c0f6dafb0]*/ +/*[clinic end generated code: output=5a96d521e6414f5d input=68d01d7c0f6dafb0]*/ /*[clinic input] @@ -3328,7 +3527,7 @@ test_posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3389,7 +3588,7 @@ static PyObject * test_posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=a2721babb42ecfd1 input=d0883d45876f186c]*/ +/*[clinic end generated code: output=d5a474dcd5dc3e9f input=d0883d45876f186c]*/ /*[clinic input] @@ -3432,7 +3631,7 @@ test_posonly_keywords_opt2_kwonly_opt(PyObject *module, PyObject *const *args, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3498,7 +3697,7 @@ static PyObject * test_posonly_keywords_opt2_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=0626203eedb6e7e8 input=c95e2e1ec93035ad]*/ +/*[clinic end generated code: output=ac239c5ee8a74408 input=c95e2e1ec93035ad]*/ /*[clinic input] @@ -3543,7 +3742,7 @@ test_posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3617,7 +3816,7 @@ test_posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f) -/*[clinic end generated code: output=07d8acc04558a5a0 input=9914857713c5bbf8]*/ +/*[clinic end generated code: output=638bbd0005639342 input=9914857713c5bbf8]*/ /*[clinic input] test_keyword_only_parameter @@ -3935,8 +4134,8 @@ test_vararg_and_posonly a: object - *args: object / + *args: object [clinic start generated code]*/ @@ -3949,36 +4148,32 @@ PyDoc_STRVAR(test_vararg_and_posonly__doc__, {"test_vararg_and_posonly", _PyCFunction_CAST(test_vararg_and_posonly), METH_FASTCALL, test_vararg_and_posonly__doc__}, static PyObject * -test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args); +test_vararg_and_posonly_impl(PyObject *module, PyObject *a, Py_ssize_t nargs, + PyObject *const *args); static PyObject * test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 1; PyObject *a; - PyObject *__clinic_args = NULL; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("test_vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { goto exit; } a = args[0]; - __clinic_args = PyTuple_New(nargs - 1); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 1; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); - } - return_value = test_vararg_and_posonly_impl(module, a, __clinic_args); + __clinic_args = args + 1; + return_value = test_vararg_and_posonly_impl(module, a, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } static PyObject * -test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=79b75dc07decc8d6 input=08dc2bf7afbf1613]*/ +test_vararg_and_posonly_impl(PyObject *module, PyObject *a, Py_ssize_t nargs, + PyObject *const *args) +/*[clinic end generated code: output=dc2dd9483cc0459e input=9cfa748bbff09877]*/ /*[clinic input] test_vararg @@ -4013,7 +4208,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4048,7 +4243,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject static PyObject * test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=880365c61ae205d7 input=81d33815ad1bae6e]*/ +/*[clinic end generated code: output=1411e464f358a7ba input=81d33815ad1bae6e]*/ /*[clinic input] test_vararg_with_default @@ -4085,7 +4280,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4131,7 +4326,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=291e9a5a09831128 input=6e110b54acd9b22d]*/ +/*[clinic end generated code: output=f09d4b917063ca41 input=6e110b54acd9b22d]*/ /*[clinic input] test_vararg_with_only_defaults @@ -4168,7 +4363,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4219,7 +4414,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize static PyObject * test_vararg_with_only_defaults_impl(PyObject *module, PyObject *args, int b, PyObject *c) -/*[clinic end generated code: output=dd21b28f0db26a4b input=fa56a709a035666e]*/ +/*[clinic end generated code: output=cc6590b8805d5433 input=fa56a709a035666e]*/ /*[clinic input] test_paramname_module @@ -4486,7 +4681,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4521,7 +4716,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ static PyObject * Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a) -/*[clinic end generated code: output=d89b99e83d442be0 input=af158077bd237ef9]*/ +/*[clinic end generated code: output=a3a968137b0f320a input=af158077bd237ef9]*/ /*[clinic input] @@ -4721,7 +4916,7 @@ Test_an_metho_arg_named_arg_impl(TestObj *self, int arg) /*[clinic input] Test.__init__ *args: object - / + Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ @@ -4732,14 +4927,14 @@ PyDoc_STRVAR(Test___init____doc__, "Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE."); static int -Test___init___impl(TestObj *self, PyObject *args); +Test___init___impl(TestObj *self, Py_ssize_t nargs, PyObject *const *args); static int Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) { int return_value = -1; PyTypeObject *base_tp = TestType; - PyObject *__clinic_args = NULL; + PyObject *const *__clinic_args = NULL; if ((Py_IS_TYPE(self, base_tp) || Py_TYPE(self)->tp_new == base_tp->tp_new) && @@ -4749,24 +4944,23 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_GetSlice(0, -1); - return_value = Test___init___impl((TestObj *)self, __clinic_args); + __clinic_args = _PyTuple_CAST(args)->ob_item; + return_value = Test___init___impl((TestObj *)self, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } static int -Test___init___impl(TestObj *self, PyObject *args) -/*[clinic end generated code: output=0ed1009fe0dcf98d input=96c3ddc0cd38fc0c]*/ +Test___init___impl(TestObj *self, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=6a64b417c9080a73 input=2a8bd0033c9ac772]*/ /*[clinic input] @classmethod Test.__new__ *args: object - / + Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ @@ -4777,14 +4971,14 @@ PyDoc_STRVAR(Test__doc__, "Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE."); static PyObject * -Test_impl(PyTypeObject *type, PyObject *args); +Test_impl(PyTypeObject *type, Py_ssize_t nargs, PyObject *const *args); static PyObject * Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; PyTypeObject *base_tp = TestType; - PyObject *__clinic_args = NULL; + PyObject *const *__clinic_args = NULL; if ((type == base_tp || type->tp_init == base_tp->tp_init) && !_PyArg_NoKeywords("Test", kwargs)) { @@ -4793,17 +4987,16 @@ Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_GetSlice(0, -1); - return_value = Test_impl(type, __clinic_args); + __clinic_args = _PyTuple_CAST(args)->ob_item; + return_value = Test_impl(type, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } static PyObject * -Test_impl(PyTypeObject *type, PyObject *args) -/*[clinic end generated code: output=8b219f6633e2a2e9 input=26a672e2e9750120]*/ +Test_impl(PyTypeObject *type, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=bf22f942407383a5 input=70ad829df3dd9b84]*/ /*[clinic input] @@ -4834,7 +5027,7 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4868,7 +5061,7 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) static int Test___init___impl(TestObj *self, PyObject *a) -/*[clinic end generated code: output=0b9ca79638ab3ecb input=a8f9222a6ab35c59]*/ +/*[clinic end generated code: output=0e1239b9bc247bc1 input=a8f9222a6ab35c59]*/ /*[clinic input] @@ -5004,12 +5197,16 @@ Test_property_set_impl(TestObj *self, PyObject *value); static int Test_property_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) { - return Test_property_set_impl(self, value); + int return_value; + + return_value = Test_property_set_impl(self, value); + + return return_value; } static int Test_property_set_impl(TestObj *self, PyObject *value) -/*[clinic end generated code: output=9797cd03c5204ddb input=3bc3f46a23c83a88]*/ +/*[clinic end generated code: output=d51023f17c4ac3a1 input=3bc3f46a23c83a88]*/ /*[clinic input] output push @@ -5098,7 +5295,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(i), }, + .ob_item = { _Py_LATIN1_CHR('i'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -5133,7 +5330,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * mangled_c_keyword_identifier_impl(PyObject *module, int int_value) -/*[clinic end generated code: output=f24b37e0368e0eb8 input=060876448ab567a2]*/ +/*[clinic end generated code: output=01a8088b57632916 input=060876448ab567a2]*/ /*[clinic input] @@ -5327,52 +5524,6 @@ Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, /*[clinic end generated code: output=4fda8a7f2547137c input=fc72ef4b4cfafabc]*/ -/*[clinic input] -Test.__init__ -> long -Test overriding the __init__ return converter -[clinic start generated code]*/ - -PyDoc_STRVAR(Test___init____doc__, -"Test()\n" -"--\n" -"\n" -"Test overriding the __init__ return converter"); - -static long -Test___init___impl(TestObj *self); - -static int -Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int return_value = -1; - PyTypeObject *base_tp = TestType; - long _return_value; - - if ((Py_IS_TYPE(self, base_tp) || - Py_TYPE(self)->tp_new == base_tp->tp_new) && - !_PyArg_NoPositional("Test", args)) { - goto exit; - } - if ((Py_IS_TYPE(self, base_tp) || - Py_TYPE(self)->tp_new == base_tp->tp_new) && - !_PyArg_NoKeywords("Test", kwargs)) { - goto exit; - } - _return_value = Test___init___impl((TestObj *)self); - if ((_return_value == -1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromLong(_return_value); - -exit: - return return_value; -} - -static long -Test___init___impl(TestObj *self) -/*[clinic end generated code: output=daf6ee12c4e443fb input=311af0dc7f17e8e9]*/ - - /*[clinic input] fn_with_default_binop_expr arg: object(c_default='CONST_A + CONST_B') = a+b @@ -5491,7 +5642,7 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -5525,7 +5676,7 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py static PyObject * docstr_fallback_to_converter_default_impl(PyObject *module, str a) -/*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ +/*[clinic end generated code: output=3fff7b702f0065cb input=0cbe6a4d24bc2274]*/ /*[clinic input] diff --git a/Lib/test/crashers/README b/Lib/test/crashers/README index 0259a0688cbc67..7111946b93b280 100644 --- a/Lib/test/crashers/README +++ b/Lib/test/crashers/README @@ -8,13 +8,10 @@ Each test should fail when run from the command line: ./python Lib/test/crashers/weakref_in_del.py Put as much info into a docstring or comments to help determine the cause of the -failure, as well as a bugs.python.org issue number if it exists. Particularly -note if the cause is system or environment dependent and what the variables are. +failure, as well as an issue number or link if it exists. +Particularly note if the cause is system or environment dependent and +what the variables are. Once the crash is fixed, the test case should be moved into an appropriate test (even if it was originally from the test suite). This ensures the regression doesn't happen again. And if it does, it should be easier to track down. - -Also see Lib/test_crashers.py which exercises the crashers in this directory. -In particular, make sure to add any new infinite loop crashers to the black -list so it doesn't try to run them. diff --git a/Lib/test/crashers/bogus_code_obj.py b/Lib/test/crashers/bogus_code_obj.py index e71b3582cf2d76..b3ff07995c95ed 100644 --- a/Lib/test/crashers/bogus_code_obj.py +++ b/Lib/test/crashers/bogus_code_obj.py @@ -12,8 +12,8 @@ """ -import types +def f(): + pass -co = types.CodeType(0, 0, 0, 0, 0, 0, b'\x04\x00\x71\x00', - (), (), (), '', '', 1, b'') -exec(co) +f.__code__ = f.__code__.replace(co_code=b"") +f() diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 53ad5e57ada017..dbe25ef57dea83 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1,7 +1,4 @@ -"""Test date/time type. - -See https://www.zope.dev/Members/fdrake/DateTimeWiki/TestCases -""" +"""Test the datetime module.""" import bisect import copy import decimal @@ -13,6 +10,7 @@ import re import struct import sys +import textwrap import unittest import warnings @@ -22,6 +20,7 @@ from test import support from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST +from test.support import script_helper, warnings_helper import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -37,6 +36,10 @@ import _testcapi except ImportError: _testcapi = None +try: + import _interpreters +except ModuleNotFoundError: + _interpreters = None # Needed by test_datetime import _strptime @@ -1103,6 +1106,85 @@ def test_delta_non_days_ignored(self): dt2 = dt - delta self.assertEqual(dt2, dt - days) + def test_strptime(self): + inputs = [ + # Basic valid cases + (date(1998, 2, 3), '1998-02-03', '%Y-%m-%d'), + (date(2004, 12, 2), '2004-12-02', '%Y-%m-%d'), + + # Edge cases: Leap year + (date(2020, 2, 29), '2020-02-29', '%Y-%m-%d'), # Valid leap year date + + # bpo-34482: Handle surrogate pairs + (date(2004, 12, 2), '2004-12\ud80002', '%Y-%m\ud800%d'), + (date(2004, 12, 2), '2004\ud80012-02', '%Y\ud800%m-%d'), + + # Month/day variations + (date(2004, 2, 1), '2004-02', '%Y-%m'), # No day provided + (date(2004, 2, 1), '02-2004', '%m-%Y'), # Month and year swapped + + # Different day-month-year formats + (date(2004, 12, 2), '02/12/2004', '%d/%m/%Y'), # Day/Month/Year + (date(2004, 12, 2), '12/02/2004', '%m/%d/%Y'), # Month/Day/Year + + # Different separators + (date(2023, 9, 24), '24.09.2023', '%d.%m.%Y'), # Dots as separators + (date(2023, 9, 24), '24-09-2023', '%d-%m-%Y'), # Dashes + (date(2023, 9, 24), '2023/09/24', '%Y/%m/%d'), # Slashes + + # Handling years with fewer digits + (date(127, 2, 3), '0127-02-03', '%Y-%m-%d'), + (date(99, 2, 3), '0099-02-03', '%Y-%m-%d'), + (date(5, 2, 3), '0005-02-03', '%Y-%m-%d'), + + # Variations on ISO 8601 format + (date(2023, 9, 25), '2023-W39-1', '%G-W%V-%u'), # ISO week date (Week 39, Monday) + (date(2023, 9, 25), '2023-268', '%Y-%j'), # Year and day of the year (Julian) + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = date.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), date) + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit dates are allowed. + strptime = date.strptime + with self.assertRaises(ValueError): + # %y does require two digits. + newdate = strptime('01/02/3', '%d/%m/%y') + + d1 = date(2003, 2, 1) + d2 = date(2003, 1, 2) + d3 = date(2003, 1, 25) + inputs = [ + ('%d', '1/02/03', '%d/%m/%y', d1), + ('%m', '01/2/03', '%d/%m/%y', d1), + ('%j', '2/03', '%j/%y', d2), + ('%w', '6/04/03', '%w/%U/%y', d1), + # %u requires a single digit. + ('%W', '6/4/2003', '%u/%W/%Y', d1), + ('%V', '6/4/2003', '%u/%V/%G', d3), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + date.strptime('02-29', '%m-%d') + with self._assertNotWarns(DeprecationWarning): + date.strptime('20-03-14', '%y-%m-%d') + date.strptime('02-29,2024', '%m-%d,%Y') + class SubclassDate(date): sub_var = 1 @@ -1335,6 +1417,11 @@ def test_insane_fromtimestamp(self): self.assertRaises(OverflowError, self.theclass.fromtimestamp, insane) + def test_fromtimestamp_with_none_arg(self): + # See gh-120268 for more details + with self.assertRaises(TypeError): + self.theclass.fromtimestamp(None) + def test_today(self): import time @@ -1686,18 +1773,35 @@ def test_bool(self): self.assertTrue(self.theclass.max) def test_strftime_y2k(self): - for y in (1, 49, 70, 99, 100, 999, 1000, 1970): - d = self.theclass(y, 1, 1) - # Issue 13305: For years < 1000, the value is not always - # padded to 4 digits across platforms. The C standard - # assumes year >= 1900, so it does not specify the number - # of digits. - if d.strftime("%Y") != '%04d' % y: - # Year 42 returns '42', not padded - self.assertEqual(d.strftime("%Y"), '%d' % y) - # '0042' is obtained anyway - if support.has_strftime_extensions: - self.assertEqual(d.strftime("%4Y"), '%04d' % y) + # Test that years less than 1000 are 0-padded; note that the beginning + # of an ISO 8601 year may fall in an ISO week of the year before, and + # therefore needs an offset of -1 when formatting with '%G'. + dataset = ( + (1, 0), + (49, -1), + (70, 0), + (99, 0), + (100, -1), + (999, 0), + (1000, 0), + (1970, 0), + ) + specifiers = 'YG' + if _time.strftime('%F', (1900, 1, 1, 0, 0, 0, 0, 1, 0)) == '1900-01-01': + specifiers += 'FC' + for year, g_offset in dataset: + for specifier in specifiers: + with self.subTest(year=year, specifier=specifier): + d = self.theclass(year, 1, 1) + if specifier == 'G': + year += g_offset + if specifier == 'C': + expected = f"{year // 100:02d}" + else: + expected = f"{year:04d}" + if specifier == 'F': + expected += f"-01-01" + self.assertEqual(d.strftime(f"%{specifier}"), expected) def test_replace(self): cls = self.theclass @@ -1723,11 +1827,24 @@ def test_replace(self): def test_subclass_replace(self): class DateSubclass(self.theclass): - pass + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result dt = DateSubclass(2012, 1, 1) - self.assertIs(type(dt.replace(year=2013)), DateSubclass) - self.assertIs(type(copy.replace(dt, year=2013)), DateSubclass) + + test_cases = [ + ('self.replace', dt.replace(year=2013)), + ('copy.replace', copy.replace(dt, year=2013)), + ] + + for name, res in test_cases: + with self.subTest(name): + self.assertIs(type(res), DateSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.month, 1) + self.assertEqual(res.extra, 7) def test_subclass_date(self): @@ -1914,6 +2031,10 @@ def test_fromisoformat_fails(self): '2009-02-29', # Invalid leap day '2019-W53-1', # No week 53 in 2019 '2020-W54-1', # No week 54 + '0000-W25-1', # Invalid year + '10000-W25-1', # Invalid year + '2020-W25-0', # Invalid day-of-week + '2020-W25-8', # Invalid day-of-week '2009\ud80002\ud80028', # Separators are surrogate codepoints ] @@ -2690,7 +2811,8 @@ def test_utcnow(self): def test_strptime(self): string = '2004-12-01 13:02:47.197' format = '%Y-%m-%d %H:%M:%S.%f' - expected = _strptime._strptime_datetime(self.theclass, string, format) + expected = _strptime._strptime_datetime_datetime(self.theclass, string, + format) got = self.theclass.strptime(string, format) self.assertEqual(expected, got) self.assertIs(type(expected), self.theclass) @@ -2704,8 +2826,8 @@ def test_strptime(self): ] for string, format in inputs: with self.subTest(string=string, format=format): - expected = _strptime._strptime_datetime(self.theclass, string, - format) + expected = _strptime._strptime_datetime_datetime(self.theclass, + string, format) got = self.theclass.strptime(string, format) self.assertEqual(expected, got) @@ -2780,6 +2902,20 @@ def test_strptime_single_digit(self): newdate = strptime(string, format) self.assertEqual(newdate, target, msg=reason) + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + self.theclass.strptime('02-29', '%m-%d') + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + self.theclass.strptime('03-14.159265', '%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('20-03-14.159265', '%y-%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('02-29,2024', '%m-%d,%Y') + def test_more_timetuple(self): # This tests fields beyond those tested by the TestDate.test_timetuple. t = self.theclass(2004, 12, 31, 6, 22, 33) @@ -2813,11 +2949,32 @@ def test_more_strftime(self): self.assertEqual(t.strftime("%z"), "-0200" + z) self.assertEqual(t.strftime("%:z"), "-02:00:" + z) - # bpo-34482: Check that surrogates don't cause a crash. - try: - t.strftime('%y\ud800%m %H\ud800%M') - except UnicodeEncodeError: - pass + def test_strftime_special(self): + t = self.theclass(2004, 12, 31, 6, 22, 33, 47) + s1 = t.strftime('%c') + s2 = t.strftime('%B') + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d') + self.assertEqual(t.strftime('\U0001f4bb%c\U0001f40d%B'), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(t.strftime('%c\U0001f4bb%B\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(t.strftime('\ud83d'), '\ud83d') + self.assertEqual(t.strftime('\udc0d'), '\udc0d') + self.assertEqual(t.strftime('\ud83d%c\udc0d%B'), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(t.strftime('%c\ud83d%B\udc0d'), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(t.strftime('%c\udc0d%B\ud83d'), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d') + self.assertEqual(t.strftime('%c\ud83d\udc0d%B'), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(t.strftime('%c\udcf0\udc9f\udc90\udc8d%B'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(t.strftime('\0'), '\0') + self.assertEqual(t.strftime('\0'*1000), '\0'*1000) + self.assertEqual(t.strftime('\0%c\0%B'), f'\0{s1}\0{s2}') + self.assertEqual(t.strftime('%c\0%B\0'), f'{s1}\0{s2}\0') def test_extract(self): dt = self.theclass(2002, 3, 4, 18, 45, 3, 1234) @@ -3025,6 +3182,26 @@ def __new__(cls, *args, **kwargs): self.assertIsInstance(dt, DateTimeSubclass) self.assertEqual(dt.extra, 7) + def test_subclass_replace_fold(self): + class DateTimeSubclass(self.theclass): + pass + + dt = DateTimeSubclass(2012, 1, 1) + dt2 = DateTimeSubclass(2012, 1, 1, fold=1) + + test_cases = [ + ('self.replace', dt.replace(year=2013), 0), + ('self.replace', dt2.replace(year=2013), 1), + ('copy.replace', copy.replace(dt, year=2013), 0), + ('copy.replace', copy.replace(dt2, year=2013), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), DateTimeSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.fold, fold) + def test_fromisoformat_datetime(self): # Test that isoformat() is reversible base_dates = [ @@ -3266,6 +3443,9 @@ def test_fromisoformat_datetime_examples(self): ('2025-01-02T03:04:05,678+00:00:10', self.theclass(2025, 1, 2, 3, 4, 5, 678000, tzinfo=timezone(timedelta(seconds=10)))), + ('2025-01-02T24:00:00', self.theclass(2025, 1, 3, 0, 0, 0)), + ('2025-01-31T24:00:00', self.theclass(2025, 2, 1, 0, 0, 0)), + ('2025-12-31T24:00:00', self.theclass(2026, 1, 1, 0, 0, 0)) ] for input_str, expected in examples: @@ -3297,11 +3477,17 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:', # Ends with time separator '2009-04-19T12:30:', # Ends with time separator '2009-04-19T12:30:45.', # Ends with time separator - '2009-04-19T12:30:45.123456+', # Ends with timzone separator - '2009-04-19T12:30:45.123456-', # Ends with timzone separator + '2009-04-19T12:30:45.123456+', # Ends with timezone separator + '2009-04-19T12:30:45.123456-', # Ends with timezone separator '2009-04-19T12:30:45.123456-05:00a', # Extra text '2009-04-19T12:30:45.123-05:00a', # Extra text '2009-04-19T12:30:45-05:00a', # Extra text + '2009-04-19T24:00:00.000001', # Has non-zero microseconds on 24:00 + '2009-04-19T24:00:01.000000', # Has non-zero seconds on 24:00 + '2009-04-19T24:01:00.000000', # Has non-zero minutes on 24:00 + '2009-04-32T24:00:00.000000', # Day is invalid before wrapping due to 24:00 + '2009-13-01T24:00:00.000000', # Month is invalid before wrapping due to 24:00 + '9999-12-31T24:00:00.000000', # Year is invalid after wrapping due to 24:00 ] for bad_str in bad_strs: @@ -3571,6 +3757,33 @@ def test_strftime(self): # gh-85432: The parameter was named "fmt" in the pure-Python impl. t.strftime(format="%f") + def test_strftime_special(self): + t = self.theclass(1, 2, 3, 4) + s1 = t.strftime('%I%p%Z') + s2 = t.strftime('%X') + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d') + self.assertEqual(t.strftime('\U0001f4bb%I%p%Z\U0001f40d%X'), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(t.strftime('%I%p%Z\U0001f4bb%X\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(t.strftime('\ud83d'), '\ud83d') + self.assertEqual(t.strftime('\udc0d'), '\udc0d') + self.assertEqual(t.strftime('\ud83d%I%p%Z\udc0d%X'), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(t.strftime('%I%p%Z\ud83d%X\udc0d'), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(t.strftime('%I%p%Z\udc0d%X\ud83d'), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d') + self.assertEqual(t.strftime('%I%p%Z\ud83d\udc0d%X'), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(t.strftime('%I%p%Z\udcf0\udc9f\udc90\udc8d%X'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(t.strftime('\0'), '\0') + self.assertEqual(t.strftime('\0'*1000), '\0'*1000) + self.assertEqual(t.strftime('\0%I%p%Z\0%X'), f'\0{s1}\0{s2}') + self.assertEqual(t.strftime('%I%p%Z\0%X\0'), f'{s1}\0{s2}\0') + def test_format(self): t = self.theclass(1, 2, 3, 4) self.assertEqual(t.__format__(''), str(t)) @@ -3664,6 +3877,78 @@ def test_compat_unpickle(self): derived = loads(data, encoding='latin1') self.assertEqual(derived, expected) + def test_strptime(self): + # bpo-34482: Check that surrogates are handled properly. + inputs = [ + (self.theclass(13, 2, 47, 197000), '13:02:47.197', '%H:%M:%S.%f'), + (self.theclass(13, 2, 47, 197000), '13:02\ud80047.197', '%H:%M\ud800%S.%f'), + (self.theclass(13, 2, 47, 197000), '13\ud80002:47.197', '%H\ud800%M:%S.%f'), + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = self.theclass.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), self.theclass) + + def test_strptime_tz(self): + strptime = self.theclass.strptime + self.assertEqual(strptime("+0002", "%z").utcoffset(), 2 * MINUTE) + self.assertEqual(strptime("-0002", "%z").utcoffset(), -2 * MINUTE) + self.assertEqual( + strptime("-00:02:01.000003", "%z").utcoffset(), + -timedelta(minutes=2, seconds=1, microseconds=3) + ) + # Only local timezone and UTC are supported + for tzseconds, tzname in ((0, 'UTC'), (0, 'GMT'), + (-_time.timezone, _time.tzname[0])): + if tzseconds < 0: + sign = '-' + seconds = -tzseconds + else: + sign ='+' + seconds = tzseconds + hours, minutes = divmod(seconds//60, 60) + tstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) + with self.subTest(tstr=tstr): + t = strptime(tstr, "%z %Z") + self.assertEqual(t.utcoffset(), timedelta(seconds=tzseconds)) + self.assertEqual(t.tzname(), tzname) + self.assertIs(type(t), self.theclass) + + # Can produce inconsistent time + tstr, fmt = "+1234 UTC", "%z %Z" + t = strptime(tstr, fmt) + self.assertEqual(t.utcoffset(), 12 * HOUR + 34 * MINUTE) + self.assertEqual(t.tzname(), 'UTC') + # yet will roundtrip + self.assertEqual(t.strftime(fmt), tstr) + + # Produce naive time if no %z is provided + self.assertEqual(strptime("UTC", "%Z").tzinfo, None) + + def test_strptime_errors(self): + for tzstr in ("-2400", "-000", "z"): + with self.assertRaises(ValueError): + self.theclass.strptime(tzstr, "%z") + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit times are allowed. + t = self.theclass(4, 5, 6) + inputs = [ + ('%H', '4:05:06', '%H:%M:%S', t), + ('%M', '04:5:06', '%H:%M:%S', t), + ('%S', '04:05:6', '%H:%M:%S', t), + ('%I', '4am:05:06', '%I%p:%M:%S', t), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = self.theclass.strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + def test_bool(self): # time is always True. cls = self.theclass @@ -3705,11 +3990,28 @@ def test_replace(self): def test_subclass_replace(self): class TimeSubclass(self.theclass): - pass + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result ctime = TimeSubclass(12, 30) - self.assertIs(type(ctime.replace(hour=10)), TimeSubclass) - self.assertIs(type(copy.replace(ctime, hour=10)), TimeSubclass) + ctime2 = TimeSubclass(12, 30, fold=1) + + test_cases = [ + ('self.replace', ctime.replace(hour=10), 0), + ('self.replace', ctime2.replace(hour=10), 1), + ('copy.replace', copy.replace(ctime, hour=10), 0), + ('copy.replace', copy.replace(ctime2, hour=10), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), TimeSubclass) + self.assertEqual(res.hour, 10) + self.assertEqual(res.minute, 30) + self.assertEqual(res.extra, 7) + self.assertEqual(res.fold, fold) def test_subclass_time(self): @@ -4005,9 +4307,8 @@ def tzname(self, dt): return self.tz self.assertRaises(TypeError, t.strftime, "%Z") # Issue #6697: - if '_Fast' in self.__class__.__name__: - Badtzname.tz = '\ud800' - self.assertRaises(ValueError, t.strftime, "%Z") + Badtzname.tz = '\ud800' + self.assertEqual(t.strftime("%Z"), '\ud800') def test_hash_edge_cases(self): # Offsets that overflow a basic time. @@ -4219,7 +4520,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=tstr): t_rt = self.theclass.fromisoformat(tstr) - assert t == t_rt, t_rt + assert t == t_rt def test_fromisoformat_timespecs(self): time_bases = [ @@ -4343,6 +4644,8 @@ def test_fromisoformat_fails(self): '12:30:45.123456-', # Extra at end of microsecond time '12:30:45.123456+', # Extra at end of microsecond time '12:30:45.123456+12:00:30a', # Extra at end of full time + '12.5', # Decimal mark at end of hour + '12:30,5', # Decimal mark at end of minute ] for bad_str in bad_strs: @@ -5435,42 +5738,50 @@ def fromutc(self, dt): class Oddballs(unittest.TestCase): - def test_bug_1028306(self): + def test_date_datetime_comparison(self): + # bpo-1028306, bpo-5516 (gh-49766) # Trying to compare a date to a datetime should act like a mixed- # type comparison, despite that datetime is a subclass of date. as_date = date.today() as_datetime = datetime.combine(as_date, time()) - self.assertTrue(as_date != as_datetime) - self.assertTrue(as_datetime != as_date) - self.assertFalse(as_date == as_datetime) - self.assertFalse(as_datetime == as_date) - self.assertRaises(TypeError, lambda: as_date < as_datetime) - self.assertRaises(TypeError, lambda: as_datetime < as_date) - self.assertRaises(TypeError, lambda: as_date <= as_datetime) - self.assertRaises(TypeError, lambda: as_datetime <= as_date) - self.assertRaises(TypeError, lambda: as_date > as_datetime) - self.assertRaises(TypeError, lambda: as_datetime > as_date) - self.assertRaises(TypeError, lambda: as_date >= as_datetime) - self.assertRaises(TypeError, lambda: as_datetime >= as_date) - - # Nevertheless, comparison should work with the base-class (date) - # projection if use of a date method is forced. - self.assertEqual(as_date.__eq__(as_datetime), True) - different_day = (as_date.day + 1) % 20 + 1 - as_different = as_datetime.replace(day= different_day) - self.assertEqual(as_date.__eq__(as_different), False) + date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) + datetime_sc = SubclassDatetime(as_date.year, as_date.month, + as_date.day, 0, 0, 0) + for d in (as_date, date_sc): + for dt in (as_datetime, datetime_sc): + for x, y in (d, dt), (dt, d): + self.assertTrue(x != y) + self.assertFalse(x == y) + self.assertRaises(TypeError, lambda: x < y) + self.assertRaises(TypeError, lambda: x <= y) + self.assertRaises(TypeError, lambda: x > y) + self.assertRaises(TypeError, lambda: x >= y) # And date should compare with other subclasses of date. If a # subclass wants to stop this, it's up to the subclass to do so. - date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) - self.assertEqual(as_date, date_sc) - self.assertEqual(date_sc, as_date) - # Ditto for datetimes. - datetime_sc = SubclassDatetime(as_datetime.year, as_datetime.month, - as_date.day, 0, 0, 0) - self.assertEqual(as_datetime, datetime_sc) - self.assertEqual(datetime_sc, as_datetime) + for x, y in ((as_date, date_sc), + (date_sc, as_date), + (as_datetime, datetime_sc), + (datetime_sc, as_datetime)): + self.assertTrue(x == y) + self.assertFalse(x != y) + self.assertFalse(x < y) + self.assertFalse(x > y) + self.assertTrue(x <= y) + self.assertTrue(x >= y) + + # Nevertheless, comparison should work if other object is an instance + # of date or datetime class with overridden comparison operators. + # So special methods should return NotImplemented, as if + # date and datetime were independent classes. + for x, y in (as_date, as_datetime), (as_datetime, as_date): + self.assertEqual(x.__eq__(y), NotImplemented) + self.assertEqual(x.__ne__(y), NotImplemented) + self.assertEqual(x.__lt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__ge__(y), NotImplemented) def test_extra_attributes(self): with self.assertWarns(DeprecationWarning): @@ -6696,6 +7007,123 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) + def test_type_check_in_subinterp(self): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + + script = textwrap.dedent(f""" + if {_interpreters is None}: + import _testcapi as module + module.test_datetime_capi() + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + def run(type_checker, obj): + if not type_checker(obj, True): + raise TypeError(f'{{type(obj)}} is not C API type') + + import _datetime + run(module.datetime_check_date, _datetime.date.today()) + run(module.datetime_check_datetime, _datetime.datetime.now()) + run(module.datetime_check_time, _datetime.time(12, 30)) + run(module.datetime_check_delta, _datetime.timedelta(1)) + run(module.datetime_check_tzinfo, _datetime.tzinfo()) + """) + if _interpreters is None: + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + else: + for name in ('isolated', 'legacy'): + with self.subTest(name): + config = _interpreters.new_config(name).__dict__ + ret = support.run_in_subinterp_with_config(script, **config) + self.assertEqual(ret, 0) + + +class ExtensionModuleTests(unittest.TestCase): + + def setUp(self): + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + @support.cpython_only + def test_gh_120161(self): + with self.subTest('simple'): + script = textwrap.dedent(""" + import datetime + from _ast import Tuple + f = lambda: None + Tuple.dims = property(f, f) + + class tzutc(datetime.tzinfo): + pass + """) + script_helper.assert_python_ok('-c', script) + + with self.subTest('complex'): + script = textwrap.dedent(""" + import asyncio + import datetime + from typing import Type + + class tzutc(datetime.tzinfo): + pass + _EPOCHTZ = datetime.datetime(1970, 1, 1, tzinfo=tzutc()) + + class FakeDateMeta(type): + def __instancecheck__(self, obj): + return True + class FakeDate(datetime.date, metaclass=FakeDateMeta): + pass + def pickle_fake_date(datetime_) -> Type[FakeDate]: + # A pickle function for FakeDate + return FakeDate + """) + script_helper.assert_python_ok('-c', script) + + def test_update_type_cache(self): + # gh-120782 + script = textwrap.dedent(""" + import sys + for i in range(5): + import _datetime + assert _datetime.date.max > _datetime.date.min + assert _datetime.time.max > _datetime.time.min + assert _datetime.datetime.max > _datetime.datetime.min + assert _datetime.timedelta.max > _datetime.timedelta.min + assert _datetime.date.__dict__["min"] is _datetime.date.min + assert _datetime.date.__dict__["max"] is _datetime.date.max + assert _datetime.date.__dict__["resolution"] is _datetime.date.resolution + assert _datetime.time.__dict__["min"] is _datetime.time.min + assert _datetime.time.__dict__["max"] is _datetime.time.max + assert _datetime.time.__dict__["resolution"] is _datetime.time.resolution + assert _datetime.datetime.__dict__["min"] is _datetime.datetime.min + assert _datetime.datetime.__dict__["max"] is _datetime.datetime.max + assert _datetime.datetime.__dict__["resolution"] is _datetime.datetime.resolution + assert _datetime.timedelta.__dict__["min"] is _datetime.timedelta.min + assert _datetime.timedelta.__dict__["max"] is _datetime.timedelta.max + assert _datetime.timedelta.__dict__["resolution"] is _datetime.timedelta.resolution + assert _datetime.timezone.__dict__["min"] is _datetime.timezone.min + assert _datetime.timezone.__dict__["max"] is _datetime.timezone.max + assert _datetime.timezone.__dict__["utc"] is _datetime.timezone.utc + assert isinstance(_datetime.timezone.min, _datetime.tzinfo) + assert isinstance(_datetime.timezone.max, _datetime.tzinfo) + assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) + del sys.modules['_datetime'] + """) + script_helper.assert_python_ok('-c', script) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/decimaltestdata/ddFMA.decTest b/Lib/test/decimaltestdata/ddFMA.decTest index 9094fc015bde18..7f2e52303747f8 100644 --- a/Lib/test/decimaltestdata/ddFMA.decTest +++ b/Lib/test/decimaltestdata/ddFMA.decTest @@ -1663,7 +1663,7 @@ ddfma375087 fma 1 12345678 1E-33 -> 12345678.00000001 Inexac ddfma375088 fma 1 12345678 1E-34 -> 12345678.00000001 Inexact Rounded ddfma375089 fma 1 12345678 1E-35 -> 12345678.00000001 Inexact Rounded --- desctructive subtraction (from remainder tests) +-- destructive subtraction (from remainder tests) -- +++ some of these will be off-by-one remainder vs remainderNear diff --git a/Lib/test/decimaltestdata/ddQuantize.decTest b/Lib/test/decimaltestdata/ddQuantize.decTest index 91776201694dd6..e1c5674d9ac042 100644 --- a/Lib/test/decimaltestdata/ddQuantize.decTest +++ b/Lib/test/decimaltestdata/ddQuantize.decTest @@ -462,7 +462,7 @@ ddqua520 quantize 1.234 1e359 -> 0E+359 Inexact Rounded ddqua521 quantize 123.456 1e359 -> 0E+359 Inexact Rounded ddqua522 quantize 1.234 1e359 -> 0E+359 Inexact Rounded ddqua523 quantize 123.456 1e359 -> 0E+359 Inexact Rounded --- next four are "won't fit" overfl +-- next four are "won't fit" overflow ddqua526 quantize 1.234 1e-299 -> NaN Invalid_operation ddqua527 quantize 123.456 1e-299 -> NaN Invalid_operation ddqua528 quantize 1.234 1e-299 -> NaN Invalid_operation diff --git a/Lib/test/decimaltestdata/ddRemainder.decTest b/Lib/test/decimaltestdata/ddRemainder.decTest index 5bd1e32d01ef03..b1866d39a2868c 100644 --- a/Lib/test/decimaltestdata/ddRemainder.decTest +++ b/Lib/test/decimaltestdata/ddRemainder.decTest @@ -422,7 +422,7 @@ ddrem757 remainder 1 sNaN -> NaN Invalid_operation ddrem758 remainder 1000 sNaN -> NaN Invalid_operation ddrem759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs ddrem760 remainder NaN1 NaN7 -> NaN1 ddrem761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation ddrem762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/ddRemainderNear.decTest b/Lib/test/decimaltestdata/ddRemainderNear.decTest index 6ba64ebafe9a18..bbe82ea3747d16 100644 --- a/Lib/test/decimaltestdata/ddRemainderNear.decTest +++ b/Lib/test/decimaltestdata/ddRemainderNear.decTest @@ -450,7 +450,7 @@ ddrmn757 remaindernear 1 sNaN -> NaN Invalid_operation ddrmn758 remaindernear 1000 sNaN -> NaN Invalid_operation ddrmn759 remaindernear Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs ddrmn760 remaindernear NaN1 NaN7 -> NaN1 ddrmn761 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation ddrmn762 remaindernear NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/dqRemainder.decTest b/Lib/test/decimaltestdata/dqRemainder.decTest index bae8eae5269d43..e0aaca3747e229 100644 --- a/Lib/test/decimaltestdata/dqRemainder.decTest +++ b/Lib/test/decimaltestdata/dqRemainder.decTest @@ -418,7 +418,7 @@ dqrem757 remainder 1 sNaN -> NaN Invalid_operation dqrem758 remainder 1000 sNaN -> NaN Invalid_operation dqrem759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs dqrem760 remainder NaN1 NaN7 -> NaN1 dqrem761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation dqrem762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/dqRemainderNear.decTest b/Lib/test/decimaltestdata/dqRemainderNear.decTest index b850626fe4e04d..2c5c3f5074ea23 100644 --- a/Lib/test/decimaltestdata/dqRemainderNear.decTest +++ b/Lib/test/decimaltestdata/dqRemainderNear.decTest @@ -450,7 +450,7 @@ dqrmn757 remaindernear 1 sNaN -> NaN Invalid_operation dqrmn758 remaindernear 1000 sNaN -> NaN Invalid_operation dqrmn759 remaindernear Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs dqrmn760 remaindernear NaN1 NaN7 -> NaN1 dqrmn761 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation dqrmn762 remaindernear NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/exp.decTest b/Lib/test/decimaltestdata/exp.decTest index 6a7af23b625530..e01d7a8f92db1f 100644 --- a/Lib/test/decimaltestdata/exp.decTest +++ b/Lib/test/decimaltestdata/exp.decTest @@ -28,7 +28,7 @@ rounding: half_even maxExponent: 384 minexponent: -383 --- basics (examples in specificiation, etc.) +-- basics (examples in specification, etc.) expx001 exp -Infinity -> 0 expx002 exp -10 -> 0.0000453999298 Inexact Rounded expx003 exp -1 -> 0.367879441 Inexact Rounded diff --git a/Lib/test/decimaltestdata/remainder.decTest b/Lib/test/decimaltestdata/remainder.decTest index 7a1061b1e6f0f8..4f59b332877afe 100644 --- a/Lib/test/decimaltestdata/remainder.decTest +++ b/Lib/test/decimaltestdata/remainder.decTest @@ -435,7 +435,7 @@ remx757 remainder 1 sNaN -> NaN Invalid_operation remx758 remainder 1000 sNaN -> NaN Invalid_operation remx759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs remx760 remainder NaN1 NaN7 -> NaN1 remx761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation remx762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/remainderNear.decTest b/Lib/test/decimaltestdata/remainderNear.decTest index b768b9e0cf6370..000b1424d8ab48 100644 --- a/Lib/test/decimaltestdata/remainderNear.decTest +++ b/Lib/test/decimaltestdata/remainderNear.decTest @@ -498,7 +498,7 @@ rmnx758 remaindernear 1000 sNaN -> NaN Invalid_operation rmnx759 remaindernear Inf sNaN -> NaN Invalid_operation rmnx760 remaindernear NaN sNaN -> NaN Invalid_operation --- propaging NaNs +-- propagating NaNs rmnx761 remaindernear NaN1 NaN7 -> NaN1 rmnx762 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation rmnx763 remaindernear NaN3 -sNaN9 -> -NaN9 Invalid_operation diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 217ee15d4c8af5..f2649aa2d41fef 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -40,11 +40,11 @@ BaseException ├── ReferenceError ├── RuntimeError │ ├── NotImplementedError + │ ├── PythonFinalizationError │ └── RecursionError ├── StopAsyncIteration ├── StopIteration ├── SyntaxError - │ └── IncompleteInputError │ └── IndentationError │ └── TabError ├── SystemError diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 0053bce4292f64..8bef04cba81138 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -164,6 +164,7 @@ def __init__(self, **kwargs) -> None: self.match_tests: TestFilter = [] self.pgo = False self.pgo_extended = False + self.tsan = False self.worker_json = None self.start = None self.timeout = None @@ -172,6 +173,8 @@ def __init__(self, **kwargs) -> None: self.fail_rerun = False self.tempdir = None self._add_python_opts = True + self.xmlpath = None + self.single_process = False super().__init__(**kwargs) @@ -305,6 +308,12 @@ def _create_parser(): group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, help='run PROCESSES processes at once') + group.add_argument('--single-process', action='store_true', + dest='single_process', + help='always run all tests sequentially in ' + 'a single process, ignore -jN option, ' + 'and failed tests are also rerun sequentially ' + 'in the same process') group.add_argument('-T', '--coverage', action='store_true', dest='trace', help='turn on code coverage tracing using the trace ' @@ -333,6 +342,8 @@ def _create_parser(): help='enable Profile Guided Optimization (PGO) training') group.add_argument('--pgo-extended', action='store_true', help='enable extended PGO training (slower training)') + group.add_argument('--tsan', dest='tsan', action='store_true', + help='run a subset of test cases that are proper for the TSAN test') group.add_argument('--fail-env-changed', action='store_true', help='if a test file alters the environment, mark ' 'the test as failed') @@ -347,6 +358,8 @@ def _create_parser(): help='override the working directory for the test run') group.add_argument('--cleanup', action='store_true', help='remove old test_python_* directories') + group.add_argument('--bisect', action='store_true', + help='if some tests fail, run test.bisect_cmd on them') group.add_argument('--dont-add-python-opts', dest='_add_python_opts', action='store_false', help="internal option, don't use it") @@ -415,9 +428,7 @@ def _parse_args(args, **kwargs): # Continuous Integration (CI): common options for fast/slow CI modes if ns.slow_ci or ns.fast_ci: # Similar to options: - # - # -j0 --randomize --fail-env-changed --fail-rerun --rerun - # --slowest --verbose3 + # -j0 --randomize --fail-env-changed --rerun --slowest --verbose3 if ns.use_mp is None: ns.use_mp = 0 ns.randomize = True @@ -429,6 +440,10 @@ def _parse_args(args, **kwargs): else: ns._add_python_opts = False + # --singleprocess overrides -jN option + if ns.single_process: + ns.use_mp = None + # When both --slow-ci and --fast-ci options are present, # --slow-ci has the priority if ns.slow_ci: @@ -501,17 +516,19 @@ def _parse_args(args, **kwargs): ns.randomize = True if ns.verbose: ns.header = True + # When -jN option is used, a worker process does not use --verbose3 # and so -R 3:3 -jN --verbose3 just works as expected: there is no false # alarm about memory leak. if ns.huntrleaks and ns.verbose3 and ns.use_mp is None: - ns.verbose3 = False # run_single_test() replaces sys.stdout with io.StringIO if verbose3 # is true. In this case, huntrleaks sees an write into StringIO as # a memory leak, whereas it is not (gh-71290). + ns.verbose3 = False print("WARNING: Disable --verbose3 because it's incompatible with " "--huntrleaks without -jN option", file=sys.stderr) + if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py index 817624d79e9263..41372e427ffd03 100644 --- a/Lib/test/libregrtest/filter.py +++ b/Lib/test/libregrtest/filter.py @@ -27,6 +27,11 @@ def _is_full_match_test(pattern): return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) +def get_match_tests(): + global _test_patterns + return _test_patterns + + def set_match_tests(patterns): global _test_matchers, _test_patterns diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index ee890b5b1db4cd..4ac95e23a56b8f 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -23,6 +23,7 @@ "test_future_stmt", "test_gdb", "test_inspect", + "test_pydoc", "test_multiprocessing_fork", "test_multiprocessing_forkserver", "test_multiprocessing_spawn", diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index a125706927393c..fa1d4d575c8fd4 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -43,7 +43,10 @@ def log(self, line: str = '') -> None: def get_load_avg(self) -> float | None: if hasattr(os, 'getloadavg'): - return os.getloadavg()[0] + try: + return os.getloadavg()[0] + except OSError: + pass if self.win_load_tracker is not None: return self.win_load_tracker.getloadavg() return None diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 7ca1b1cb65ae40..11ebb09fbb4745 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,8 +7,7 @@ import time import trace -from test import support -from test.support import os_helper, MS_WINDOWS +from test.support import os_helper, MS_WINDOWS, flush_std_streams from .cmdline import _parse_args, Namespace from .findtests import findtests, split_test_packages, list_cases @@ -19,6 +18,7 @@ from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME +from .tsan import setup_tsan_tests from .utils import ( StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, strip_py_suffix, count, format_duration, @@ -57,6 +57,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.quiet: bool = ns.quiet self.pgo: bool = ns.pgo self.pgo_extended: bool = ns.pgo_extended + self.tsan: bool = ns.tsan # Test results self.results: TestResults = TestResults() @@ -73,6 +74,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.want_cleanup: bool = ns.cleanup self.want_rerun: bool = ns.rerun self.want_run_leaks: bool = ns.runleaks + self.want_bisect: bool = ns.bisect self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) self.want_add_python_opts: bool = (_add_python_opts @@ -86,12 +88,13 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.cmdline_args: TestList = ns.args # Workers - if ns.use_mp is None: - num_workers = 0 # run sequentially + self.single_process: bool = ns.single_process + if self.single_process or ns.use_mp is None: + num_workers = 0 # run sequentially in a single process elif ns.use_mp <= 0: - num_workers = -1 # use the number of CPUs + num_workers = -1 # run in parallel, use the number of CPUs else: - num_workers = ns.use_mp + num_workers = ns.use_mp # run in parallel self.num_workers: int = num_workers self.worker_json: StrJSON | None = ns.worker_json @@ -183,6 +186,9 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList # add default PGO tests if no tests are specified setup_pgo_tests(self.cmdline_args, self.pgo_extended) + if self.tsan: + setup_tsan_tests(self.cmdline_args) + exclude_tests = set() if self.exclude: for arg in self.cmdline_args: @@ -230,7 +236,7 @@ def list_tests(tests: TestTuple): def _rerun_failed_tests(self, runtests: RunTests): # Configure the runner to re-run tests - if self.num_workers == 0: + if self.num_workers == 0 and not self.single_process: # Always run tests in fresh processes to have more deterministic # initial state. Don't re-run tests in parallel but limit to a # single worker process to have side effects (on the system load @@ -240,7 +246,6 @@ def _rerun_failed_tests(self, runtests: RunTests): tests, match_tests_dict = self.results.prepare_rerun() # Re-run failed tests - self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") runtests = runtests.copy( tests=tests, rerun=True, @@ -250,7 +255,15 @@ def _rerun_failed_tests(self, runtests: RunTests): match_tests_dict=match_tests_dict, output_on_failure=False) self.logger.set_tests(runtests) - self._run_tests_mp(runtests, self.num_workers) + + msg = f"Re-running {len(tests)} failed tests in verbose mode" + if not self.single_process: + msg = f"{msg} in subprocesses" + self.log(msg) + self._run_tests_mp(runtests, self.num_workers) + else: + self.log(msg) + self.run_tests_sequentially(runtests) return runtests def rerun_failed_tests(self, runtests: RunTests): @@ -273,6 +286,55 @@ def rerun_failed_tests(self, runtests: RunTests): self.display_result(rerun_runtests) + def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: + print() + title = f"Bisect {test}" + if progress: + title = f"{title} ({progress})" + print(title) + print("#" * len(title)) + print() + + cmd = runtests.create_python_cmd() + cmd.extend([ + "-u", "-m", "test.bisect_cmd", + # Limit to 25 iterations (instead of 100) to not abuse CI resources + "--max-iter", "25", + "-v", + # runtests.match_tests is not used (yet) for bisect_cmd -i arg + ]) + cmd.extend(runtests.bisect_cmd_args()) + cmd.append(test) + print("+", shlex.join(cmd), flush=True) + + flush_std_streams() + + import subprocess + proc = subprocess.run(cmd, timeout=runtests.timeout) + exitcode = proc.returncode + + title = f"{title}: exit code {exitcode}" + print(title) + print("#" * len(title)) + print(flush=True) + + if exitcode: + print(f"Bisect failed with exit code {exitcode}") + return False + + return True + + def run_bisect(self, runtests: RunTests) -> None: + tests, _ = self.results.prepare_rerun(clear=False) + + for index, name in enumerate(tests, 1): + if len(tests) > 1: + progress = f"{index}/{len(tests)}" + else: + progress = "" + if not self._run_bisect(runtests, name, progress): + return + def display_result(self, runtests): # If running the test suite for PGO then no one cares about results. if runtests.pgo: @@ -295,9 +357,7 @@ def run_test( namespace = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=namespace) result = namespace['result'] - # Mypy doesn't know about this attribute yet, - # but it will do soon: https://github.com/python/typeshed/pull/11091 - result.covered_lines = list(tracer.counts) # type: ignore[attr-defined] + result.covered_lines = list(tracer.counts) else: result = run_single_test(test_name, runtests) @@ -318,7 +378,7 @@ def run_tests_sequentially(self, runtests) -> None: tests = count(jobs, 'test') else: tests = 'tests' - msg = f"Run {tests} sequentially" + msg = f"Run {tests} sequentially in a single process" if runtests.timeout: msg += " (timeout: %s)" % format_duration(runtests.timeout) self.log(msg) @@ -461,8 +521,8 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: setup_process() - if self.hunt_refleak and not self.num_workers: - # gh-109739: WindowsLoadTracker thread interfers with refleak check + if (runtests.hunt_refleak is not None) and (not self.num_workers): + # gh-109739: WindowsLoadTracker thread interferes with refleak check use_load_tracker = False else: # WindowsLoadTracker is only needed on Windows @@ -481,6 +541,9 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if self.want_rerun and self.results.need_rerun(): self.rerun_failed_tests(runtests) + + if self.want_bisect and self.results.need_rerun(): + self.run_bisect(runtests) finally: if use_load_tracker: self.logger.stop_load_tracker() @@ -527,6 +590,7 @@ def _add_cross_compile_opts(self, regrtest_opts): '_PYTHON_PROJECT_BASE', '_PYTHON_HOST_PLATFORM', '_PYTHON_SYSCONFIGDATA_NAME', + "_PYTHON_SYSCONFIGDATA_PATH", 'PYTHONPATH' } old_environ = os.environ @@ -540,7 +604,7 @@ def _add_cross_compile_opts(self, regrtest_opts): keep_environ = True if cross_compile and hostrunner: - if self.num_workers == 0: + if self.num_workers == 0 and not self.single_process: # For now use only two cores for cross-compiled builds; # hostrunner can be expensive. regrtest_opts.extend(['-j', '2']) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 7da16cf721f097..fa447a4336a399 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,3 +1,4 @@ +import os import sys import warnings from inspect import isabstract @@ -23,6 +24,30 @@ def _get_dump(cls): cls._abc_negative_cache, cls._abc_negative_cache_version) +def save_support_xml(filename): + if support.junit_xml_list is None: + return + + import pickle + with open(filename, 'xb') as fp: + pickle.dump(support.junit_xml_list, fp) + support.junit_xml_list = None + + +def restore_support_xml(filename): + try: + fp = open(filename, 'rb') + except FileNotFoundError: + return + + import pickle + with fp: + xml_list = pickle.load(fp) + os.unlink(filename) + + support.junit_xml_list = xml_list + + def runtest_refleak(test_name, test_func, hunt_refleak: HuntRefleak, quiet: bool): @@ -85,14 +110,18 @@ def get_pooled_int(value): getunicodeinternedsize = sys.getunicodeinternedsize fd_count = os_helper.fd_count # initialize variables to make pyflakes quiet - rc_before = alloc_before = fd_before = interned_before = 0 + rc_before = alloc_before = fd_before = interned_immortal_before = 0 if not quiet: - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, - flush=True) - - results = None + print("beginning", repcount, "repetitions. Showing number of leaks " + "(. for 0 or less, X for 10 or more)", + file=sys.stderr) + numbers = ("1234567890"*(repcount//10 + 1))[:repcount] + numbers = numbers[:warmups] + ':' + numbers[warmups:] + print(numbers, file=sys.stderr, flush=True) + + xml_filename = 'refleak-xml.tmp' + result = None dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() @@ -100,10 +129,11 @@ def get_pooled_int(value): current = refleak_helper._hunting_for_refleaks refleak_helper._hunting_for_refleaks = True try: - results = test_func() + result = test_func() finally: refleak_helper._hunting_for_refleaks = current + save_support_xml(xml_filename) dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() @@ -111,22 +141,40 @@ def get_pooled_int(value): # Also, readjust the reference counts and alloc blocks by ignoring # any strings that might have been interned during test_func. These # strings will be deallocated at runtime shutdown - interned_after = getunicodeinternedsize() - alloc_after = getallocatedblocks() - interned_after - rc_after = gettotalrefcount() - interned_after * 2 + interned_immortal_after = getunicodeinternedsize( + # Use an internal-only keyword argument that mypy doesn't know yet + _only_immortal=True) # type: ignore[call-arg] + alloc_after = getallocatedblocks() - interned_immortal_after + rc_after = gettotalrefcount() fd_after = fd_count() - if not quiet: - print('.', end='', file=sys.stderr, flush=True) - rc_deltas[i] = get_pooled_int(rc_after - rc_before) alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) fd_deltas[i] = get_pooled_int(fd_after - fd_before) + if not quiet: + # use max, not sum, so total_leaks is one of the pooled ints + total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i]) + if total_leaks <= 0: + symbol = '.' + elif total_leaks < 10: + symbol = ( + '.', '1', '2', '3', '4', '5', '6', '7', '8', '9', + )[total_leaks] + else: + symbol = 'X' + if i == warmups: + print(' ', end='', file=sys.stderr, flush=True) + print(symbol, end='', file=sys.stderr, flush=True) + del total_leaks + del symbol + alloc_before = alloc_after rc_before = rc_after fd_before = fd_after - interned_before = interned_after + interned_immortal_before = interned_immortal_after + + restore_support_xml(xml_filename) if not quiet: print(file=sys.stderr) @@ -158,15 +206,21 @@ def check_fd_deltas(deltas): ]: # ignore warmup runs deltas = deltas[warmups:] - if checker(deltas): + failing = checker(deltas) + suspicious = any(deltas) + if failing or suspicious: msg = '%s leaked %s %s, sum=%s' % ( test_name, deltas, item_name, sum(deltas)) - print(msg, file=sys.stderr, flush=True) - with open(filename, "a", encoding="utf-8") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True - return (failed, results) + print(msg, end='', file=sys.stderr) + if failing: + print(file=sys.stderr, flush=True) + with open(filename, "a", encoding="utf-8") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + else: + print(' (this is fine)', file=sys.stderr, flush=True) + return (failed, result) def dash_R_cleanup(fs, ps, pic, zdc, abcs): @@ -188,21 +242,24 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): zipimport._zip_directory_cache.update(zdc) # Clear ABC registries, restoring previously saved ABC registries. - # ignore deprecation warning for collections.abc.ByteString abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: - for ref in abcs.get(obj, set()): - if ref() is not None: - obj.register(ref()) + refs = abcs.get(obj, None) + if refs is not None: + obj._abc_registry_clear() + for ref in refs: + subclass = ref() + if subclass is not None: + obj.register(subclass) obj._abc_caches_clear() # Clear caches clear_caches() - # Clear type cache at the end: previous function calls can modify types - sys._clear_type_cache() + # Clear other caches last (previous function calls can re-populate them): + sys._clear_internal_caches() def warm_caches(): diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index a41ea8aba028c3..0e28435bc7d629 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -18,7 +18,7 @@ class TestResults: - def __init__(self): + def __init__(self) -> None: self.bad: TestList = [] self.good: TestList = [] self.rerun_bad: TestList = [] @@ -38,22 +38,22 @@ def __init__(self): # used by -T with -j self.covered_lines: set[Location] = set() - def is_all_good(self): + def is_all_good(self) -> bool: return (not self.bad and not self.skipped and not self.interrupted and not self.worker_bug) - def get_executed(self): + def get_executed(self) -> set[TestName]: return (set(self.good) | set(self.bad) | set(self.skipped) | set(self.resource_denied) | set(self.env_changed) | set(self.run_no_tests)) - def no_tests_run(self): + def no_tests_run(self) -> bool: return not any((self.good, self.bad, self.skipped, self.interrupted, self.env_changed)) - def get_state(self, fail_env_changed): + def get_state(self, fail_env_changed: bool) -> str: state = [] if self.bad: state.append("FAILURE") @@ -138,7 +138,7 @@ def get_coverage_results(self) -> trace.CoverageResults: def need_rerun(self): return bool(self.rerun_results) - def prepare_rerun(self) -> tuple[TestTuple, FilterDict]: + def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: tests: TestList = [] match_tests_dict = {} for result in self.rerun_results: @@ -149,11 +149,12 @@ def prepare_rerun(self) -> tuple[TestTuple, FilterDict]: if match_tests: match_tests_dict[result.test_name] = match_tests - # Clear previously failed tests - self.rerun_bad.extend(self.bad) - self.bad.clear() - self.env_changed.clear() - self.rerun_results.clear() + if clear: + # Clear previously failed tests + self.rerun_bad.extend(self.bad) + self.bad.clear() + self.env_changed.clear() + self.rerun_results.clear() return (tuple(tests), match_tests_dict) @@ -203,7 +204,7 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): omitted = set(tests) - self.get_executed() # less important - all_tests.append((omitted, "test", "{} omitted:")) + all_tests.append((sorted(omitted), "test", "{} omitted:")) if not quiet: all_tests.append((self.skipped, "test", "{} skipped:")) all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):")) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 18a0342f0611cf..387ddf9614cf79 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -22,7 +22,7 @@ from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, - format_duration, print_warning, count, plural, get_signal_name) + format_duration, print_warning, count, plural) from .worker import create_worker_process, USE_PROCESS_GROUP if MS_WINDOWS: @@ -79,8 +79,12 @@ class MultiprocessResult: err_msg: str | None = None +class WorkerThreadExited: + """Indicates that a worker thread has exited""" + ExcStr = str QueueOutput = tuple[Literal[False], MultiprocessResult] | tuple[Literal[True], ExcStr] +QueueContent = QueueOutput | WorkerThreadExited class ExitThread(Exception): @@ -138,14 +142,20 @@ def _kill(self) -> None: return self._killed = True - if USE_PROCESS_GROUP: + use_killpg = USE_PROCESS_GROUP + if use_killpg: + parent_sid = os.getsid(0) + sid = os.getsid(popen.pid) + use_killpg = (sid != parent_sid) + + if use_killpg: what = f"{self} process group" else: what = f"{self} process" print(f"Kill {what}", file=sys.stderr, flush=True) try: - if USE_PROCESS_GROUP: + if use_killpg: os.killpg(popen.pid, signal.SIGKILL) else: popen.kill() @@ -209,7 +219,7 @@ def _run_process(self, runtests: WorkerRunTests, output_fd: int, self._popen = None def create_stdout(self, stack: contextlib.ExitStack) -> TextIO: - """Create stdout temporay file (file descriptor).""" + """Create stdout temporary file (file descriptor).""" if MS_WINDOWS: # gh-95027: When stdout is not a TTY, Python uses the ANSI code @@ -356,7 +366,7 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: err_msg=None, state=State.TIMEOUT) if retcode != 0: - name = get_signal_name(retcode) + name = support.get_signal_name(retcode) if name: retcode = f"{retcode} ({name})" raise WorkerError(self.test_name, f"Exit code {retcode}", stdout, @@ -376,8 +386,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: def run(self) -> None: fail_fast = self.runtests.fail_fast fail_env_changed = self.runtests.fail_env_changed - while not self._stopped: - try: + try: + while not self._stopped: try: test_name = next(self.pending) except StopIteration: @@ -396,11 +406,12 @@ def run(self) -> None: if mp_result.result.must_stop(fail_fast, fail_env_changed): break - except ExitThread: - break - except BaseException: - self.output.put((True, traceback.format_exc())) - break + except ExitThread: + pass + except BaseException: + self.output.put((True, traceback.format_exc())) + finally: + self.output.put(WorkerThreadExited()) def _wait_completed(self) -> None: popen = self._popen @@ -458,8 +469,9 @@ def __init__(self, num_workers: int, runtests: RunTests, self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results + self.live_worker_count = 0 - self.output: queue.Queue[QueueOutput] = queue.Queue() + self.output: queue.Queue[QueueContent] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) self.timeout = runtests.timeout @@ -497,6 +509,7 @@ def start_workers(self) -> None: self.log(msg) for worker in self.workers: worker.start() + self.live_worker_count += 1 def stop_workers(self) -> None: start_time = time.monotonic() @@ -511,14 +524,18 @@ def _get_result(self) -> QueueOutput | None: # bpo-46205: check the status of workers every iteration to avoid # waiting forever on an empty queue. - while any(worker.is_alive() for worker in self.workers): + while self.live_worker_count > 0: if use_faulthandler: faulthandler.dump_traceback_later(MAIN_PROCESS_TIMEOUT, exit=True) # wait for a thread try: - return self.output.get(timeout=PROGRESS_UPDATE) + result = self.output.get(timeout=PROGRESS_UPDATE) + if isinstance(result, WorkerThreadExited): + self.live_worker_count -= 1 + continue + return result except queue.Empty: pass @@ -528,12 +545,6 @@ def _get_result(self) -> QueueOutput | None: if running: self.log(running) - # all worker threads are done: consume pending results - try: - return self.output.get(timeout=0) - except queue.Empty: - return None - def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result pgo = self.runtests.pgo diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index edd72276320e41..8e9779524c56a2 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -2,7 +2,9 @@ import dataclasses import json import os +import shlex import subprocess +import sys from typing import Any from test import support @@ -67,6 +69,11 @@ class HuntRefleak: runs: int filename: StrPath + def bisect_cmd_args(self) -> list[str]: + # Ignore filename since it can contain colon (":"), + # and usually it's not used. Use the default filename. + return ["-R", f"{self.warmups}:{self.runs}:"] + @dataclasses.dataclass(slots=True, frozen=True) class RunTests: @@ -137,6 +144,49 @@ def json_file_use_stdout(self) -> bool: or support.is_wasi ) + def create_python_cmd(self) -> list[str]: + python_opts = support.args_from_interpreter_flags() + if self.python_cmd is not None: + executable = self.python_cmd + # Remove -E option, since --python=COMMAND can set PYTHON + # environment variables, such as PYTHONPATH, in the worker + # process. + python_opts = [opt for opt in python_opts if opt != "-E"] + else: + executable = (sys.executable,) + cmd = [*executable, *python_opts] + if '-u' not in python_opts: + cmd.append('-u') # Unbuffered stdout and stderr + if self.coverage: + cmd.append("-Xpresite=test.cov") + return cmd + + def bisect_cmd_args(self) -> list[str]: + args = [] + if self.fail_fast: + args.append("--failfast") + if self.fail_env_changed: + args.append("--fail-env-changed") + if self.timeout: + args.append(f"--timeout={self.timeout}") + if self.hunt_refleak is not None: + args.extend(self.hunt_refleak.bisect_cmd_args()) + if self.test_dir: + args.extend(("--testdir", self.test_dir)) + if self.memory_limit: + args.extend(("--memlimit", self.memory_limit)) + if self.gc_threshold: + args.append(f"--threshold={self.gc_threshold}") + if self.use_resources: + args.extend(("-u", ','.join(self.use_resources))) + if self.python_cmd: + cmd = shlex.join(self.python_cmd) + args.extend(("--python", cmd)) + if self.randomize: + args.append(f"--randomize") + args.append(f"--randseed={self.random_seed}") + return args + @dataclasses.dataclass(slots=True, frozen=True) class WorkerRunTests(RunTests): diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 235029d8620ff5..17323e7f9cf730 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -57,7 +57,10 @@ def _run_suite(suite): result = runner.run(suite) if support.junit_xml_list is not None: - support.junit_xml_list.append(result.get_xml_element()) + import xml.etree.ElementTree as ET + xml_elem = result.get_xml_element() + xml_str = ET.tostring(xml_elem).decode('ascii') + support.junit_xml_list.append(xml_str) if not result.testsRun and not result.skipped and not result.errors: raise support.TestDidNotRun @@ -280,9 +283,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: xml_list = support.junit_xml_list if xml_list: - import xml.etree.ElementTree as ET - result.xml_data = [ET.tostring(x).decode('us-ascii') - for x in xml_list] + result.xml_data = xml_list finally: if use_timeout: faulthandler.cancel_dump_traceback_later() diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index de23fdd59ded95..1820f354572521 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,6 +9,7 @@ import traceback import unittest from test import support +from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False @@ -65,23 +66,24 @@ def _add_result(self, test, capture=False, **args): if capture: if self._stdout_buffer is not None: stdout = self._stdout_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-out').text = stdout + ET.SubElement(e, 'system-out').text = sanitize_xml(stdout) if self._stderr_buffer is not None: stderr = self._stderr_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-err').text = stderr + ET.SubElement(e, 'system-err').text = sanitize_xml(stderr) for k, v in args.items(): if not k or not v: continue + e2 = ET.SubElement(e, k) if hasattr(v, 'items'): for k2, v2 in v.items(): if k2: - e2.set(k2, str(v2)) + e2.set(k2, sanitize_xml(str(v2))) else: - e2.text = str(v2) + e2.text = sanitize_xml(str(v2)) else: - e2.text = str(v) + e2.text = sanitize_xml(str(v)) @classmethod def __makeErrorDict(cls, err_type, err_value, err_tb): diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py new file mode 100644 index 00000000000000..dd18ae2584f5d8 --- /dev/null +++ b/Lib/test/libregrtest/tsan.py @@ -0,0 +1,33 @@ +# Set of tests run by default if --tsan is specified. The tests below were +# chosen because they use threads and run in a reasonable amount of time. + +TSAN_TESTS = [ + # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909). + 'test_capi.test_mem', + 'test_capi.test_pyatomic', + 'test_code', + 'test_enum', + 'test_functools', + 'test_httpservers', + 'test_imaplib', + 'test_importlib', + 'test_io', + 'test_logging', + 'test_queue', + 'test_signal', + 'test_socket', + 'test_sqlite3', + 'test_ssl', + 'test_syslog', + 'test_thread', + 'test_threadedtempfile', + 'test_threading', + 'test_threading_local', + 'test_threadsignals', + 'test_weakref', +] + + +def setup_tsan_tests(cmdline_args): + if not cmdline_args: + cmdline_args[:] = TSAN_TESTS[:] diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index b30025d962413c..d6be4ad049d14a 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -5,6 +5,7 @@ import os.path import platform import random +import re import shlex import signal import subprocess @@ -263,6 +264,12 @@ def clear_caches(): for f in typing._cleanups: f() + import inspect + abs_classes = filter(inspect.isabstract, typing.__dict__.values()) + for abc in abs_classes: + for obj in abc.__subclasses__() + [abc]: + obj._abc_caches_clear() + try: fractions = sys.modules['fractions'] except KeyError: @@ -275,7 +282,16 @@ def clear_caches(): except KeyError: pass else: - inspect._shadowed_dict_from_mro_tuple.cache_clear() + inspect._shadowed_dict_from_weakref_mro_tuple.cache_clear() + inspect._filesbymodname.clear() + inspect.modulesbyfile.clear() + + try: + importlib_metadata = sys.modules['importlib.metadata'] + except KeyError: + pass + else: + importlib_metadata.FastPath.__new__.cache_clear() def get_build_info(): @@ -284,29 +300,78 @@ def get_build_info(): config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' cflags = sysconfig.get_config_var('PY_CFLAGS') or '' - cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or '' + cflags += ' ' + (sysconfig.get_config_var('PY_CFLAGS_NODIST') or '') ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' build = [] # --disable-gil if sysconfig.get_config_var('Py_GIL_DISABLED'): - build.append("free_threading") + if not sys.flags.ignore_environment: + PYTHON_GIL = os.environ.get('PYTHON_GIL', None) + if PYTHON_GIL: + PYTHON_GIL = (PYTHON_GIL == '1') + else: + PYTHON_GIL = None + + free_threading = "free_threading" + if PYTHON_GIL is not None: + free_threading = f"{free_threading} GIL={int(PYTHON_GIL)}" + build.append(free_threading) if hasattr(sys, 'gettotalrefcount'): # --with-pydebug build.append('debug') - if '-DNDEBUG' in (cflags + cflags_nodist): + if '-DNDEBUG' in cflags: build.append('without_assert') else: build.append('release') if '--with-assertions' in config_args: build.append('with_assert') - elif '-DNDEBUG' not in (cflags + cflags_nodist): + elif '-DNDEBUG' not in cflags: build.append('with_assert') + # --enable-experimental-jit + tier2 = re.search('-D_Py_TIER2=([0-9]+)', cflags) + if tier2: + tier2 = int(tier2.group(1)) + + if not sys.flags.ignore_environment: + PYTHON_JIT = os.environ.get('PYTHON_JIT', None) + if PYTHON_JIT: + PYTHON_JIT = (PYTHON_JIT != '0') + else: + PYTHON_JIT = None + + if tier2 == 1: # =yes + if PYTHON_JIT == False: + jit = 'JIT=off' + else: + jit = 'JIT' + elif tier2 == 3: # =yes-off + if PYTHON_JIT: + jit = 'JIT' + else: + jit = 'JIT=off' + elif tier2 == 4: # =interpreter + if PYTHON_JIT == False: + jit = 'JIT-interpreter=off' + else: + jit = 'JIT-interpreter' + elif tier2 == 6: # =interpreter-off (Secret option!) + if PYTHON_JIT: + jit = 'JIT-interpreter' + else: + jit = 'JIT-interpreter=off' + elif '-D_Py_JIT' in cflags: + jit = 'JIT' + else: + jit = None + if jit: + build.append(jit) + # --enable-framework=name framework = sysconfig.get_config_var('PYTHONFRAMEWORK') if framework: @@ -327,6 +392,11 @@ def get_build_info(): if support.check_cflags_pgo(): # PGO (--enable-optimizations) optimizations.append('PGO') + + if support.check_bolt_optimized(): + # BOLT (--enable-bolt) + optimizations.append('BOLT') + if optimizations: build.append('+'.join(optimizations)) @@ -422,7 +492,7 @@ def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: # the tests. The name of the dir includes the pid to allow parallel # testing (see the -j option). # Emscripten and WASI have stubbed getpid(), Emscripten has only - # milisecond clock resolution. Use randint() instead. + # millisecond clock resolution. Use randint() instead. if support.is_emscripten or support.is_wasi: nounce = random.randint(0, 1_000_000) else: @@ -675,23 +745,23 @@ def cleanup_temp_dir(tmp_dir: StrPath): print("Remove file: %s" % name) os_helper.unlink(name) -WINDOWS_STATUS = { - 0xC0000005: "STATUS_ACCESS_VIOLATION", - 0xC00000FD: "STATUS_STACK_OVERFLOW", - 0xC000013A: "STATUS_CONTROL_C_EXIT", -} - -def get_signal_name(exitcode): - if exitcode < 0: - signum = -exitcode - try: - return signal.Signals(signum).name - except ValueError: - pass - - try: - return WINDOWS_STATUS[exitcode] - except KeyError: - pass - return None +ILLEGAL_XML_CHARS_RE = re.compile( + '[' + # Control characters; newline (\x0A and \x0D) and TAB (\x09) are legal + '\x00-\x08\x0B\x0C\x0E-\x1F' + # Surrogate characters + '\uD800-\uDFFF' + # Special Unicode characters + '\uFFFE' + '\uFFFF' + # Match multiple sequential invalid characters for better efficiency + ']+') + +def _sanitize_xml_replace(regs): + text = regs[0] + return ''.join(f'\\x{ord(ch):02x}' if ch <= '\xff' else ascii(ch)[1:-1] + for ch in text) + +def sanitize_xml(text): + return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text) diff --git a/Lib/test/libregrtest/win_utils.py b/Lib/test/libregrtest/win_utils.py index 5736cdfd3c7292..b51fde0af57d1f 100644 --- a/Lib/test/libregrtest/win_utils.py +++ b/Lib/test/libregrtest/win_utils.py @@ -24,6 +24,10 @@ class WindowsLoadTracker(): """ def __init__(self): + # make __del__ not fail if pre-flight test fails + self._running = None + self._stopped = None + # Pre-flight test for access to the performance data; # `PermissionError` will be raised if not allowed winreg.QueryInfoKey(winreg.HKEY_PERFORMANCE_DATA) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 7a6d33d4499943..86cc30835fdbda 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -3,7 +3,6 @@ import os from typing import Any, NoReturn -from test import support from test.support import os_helper, Py_DEBUG from .setup import setup_process, setup_test_dir @@ -15,27 +14,17 @@ USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) +NEED_TTY = { + 'test_ioctl', +} def create_worker_process(runtests: WorkerRunTests, output_fd: int, tmp_dir: StrPath | None = None) -> subprocess.Popen: - python_cmd = runtests.python_cmd worker_json = runtests.as_json() - python_opts = support.args_from_interpreter_flags() - if python_cmd is not None: - executable = python_cmd - # Remove -E option, since --python=COMMAND can set PYTHON environment - # variables, such as PYTHONPATH, in the worker process. - python_opts = [opt for opt in python_opts if opt != "-E"] - else: - executable = (sys.executable,) - if runtests.coverage: - python_opts.append("-Xpresite=test.cov") - cmd = [*executable, *python_opts, - '-u', # Unbuffered stdout and stderr - '-m', 'test.libregrtest.worker', - worker_json] + cmd = runtests.create_python_cmd() + cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) if tmp_dir is not None: @@ -61,7 +50,10 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, close_fds=True, cwd=work_dir, ) - if USE_PROCESS_GROUP: + + # Don't use setsid() in tests using TTY + test_name = runtests.tests[0] + if USE_PROCESS_GROUP and test_name not in NEED_TTY: kwargs['start_new_session'] = True # Pass json_file to the worker process diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index d9ab21d4941cdb..dbc5ef4f9f2cd5 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, Py_C_RECURSION_LIMIT +from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) @@ -191,6 +191,14 @@ def test_setslice(self): self.assertRaises(TypeError, a.__setitem__) + def test_slice_assign_iterator(self): + x = self.type2test(range(5)) + x[0:3] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0, 3, 4])) + + x[:] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0])) + def test_delslice(self): a = self.type2test([0, 1]) del a[1:2] @@ -562,3 +570,8 @@ def test_exhausted_iterator(self): self.assertEqual(list(exhit), []) self.assertEqual(list(empit), [9]) self.assertEqual(a, self.type2test([1, 2, 3, 9])) + + # gh-115733: Crash when iterating over exhausted iterator + exhit = iter(self.type2test([1, 2, 3])) + for _ in exhit: + next(exhit, 1) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 024c6debcd4a54..8c8f8901f00178 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -1013,6 +1013,10 @@ def multipass(self, results, n): self.assertEqual(self.barrier.n_waiting, 0) self.assertFalse(self.barrier.broken) + def test_constructor(self): + self.assertRaises(ValueError, self.barriertype, parties=0) + self.assertRaises(ValueError, self.barriertype, parties=-1) + def test_barrier(self, passes=1): """ Test that a barrier is passed in lockstep diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index b4cfce19a7174e..ed89a81a6ea685 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -1,7 +1,7 @@ # tests common to dict and UserDict import unittest import collections -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class BasicTestMappingProtocol(unittest.TestCase): @@ -624,7 +624,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/mathdata/cmath_testcases.txt b/Lib/test/mathdata/cmath_testcases.txt index 0165e17634f41c..7b98b5a2998413 100644 --- a/Lib/test/mathdata/cmath_testcases.txt +++ b/Lib/test/mathdata/cmath_testcases.txt @@ -371,9 +371,9 @@ acosh1002 acosh 0.0 inf -> inf 1.5707963267948966 acosh1003 acosh 2.3 inf -> inf 1.5707963267948966 acosh1004 acosh -0.0 inf -> inf 1.5707963267948966 acosh1005 acosh -2.3 inf -> inf 1.5707963267948966 -acosh1006 acosh 0.0 nan -> nan nan +acosh1006 acosh 0.0 nan -> nan 1.5707963267948966 ignore-imag-sign acosh1007 acosh 2.3 nan -> nan nan -acosh1008 acosh -0.0 nan -> nan nan +acosh1008 acosh -0.0 nan -> nan 1.5707963267948966 ignore-imag-sign acosh1009 acosh -2.3 nan -> nan nan acosh1010 acosh -inf 0.0 -> inf 3.1415926535897931 acosh1011 acosh -inf 2.3 -> inf 3.1415926535897931 @@ -1992,9 +1992,9 @@ tanh0065 tanh 1.797e+308 0.0 -> 1.0 0.0 --special values tanh1000 tanh 0.0 0.0 -> 0.0 0.0 -tanh1001 tanh 0.0 inf -> nan nan invalid +tanh1001 tanh 0.0 inf -> 0.0 nan invalid tanh1002 tanh 2.3 inf -> nan nan invalid -tanh1003 tanh 0.0 nan -> nan nan +tanh1003 tanh 0.0 nan -> 0.0 nan tanh1004 tanh 2.3 nan -> nan nan tanh1005 tanh inf 0.0 -> 1.0 0.0 tanh1006 tanh inf 0.7 -> 1.0 0.0 @@ -2009,7 +2009,7 @@ tanh1014 tanh nan 2.3 -> nan nan tanh1015 tanh nan inf -> nan nan tanh1016 tanh nan nan -> nan nan tanh1017 tanh 0.0 -0.0 -> 0.0 -0.0 -tanh1018 tanh 0.0 -inf -> nan nan invalid +tanh1018 tanh 0.0 -inf -> 0.0 nan invalid tanh1019 tanh 2.3 -inf -> nan nan invalid tanh1020 tanh inf -0.0 -> 1.0 -0.0 tanh1021 tanh inf -0.7 -> 1.0 -0.0 @@ -2022,9 +2022,9 @@ tanh1027 tanh nan -0.0 -> nan -0.0 tanh1028 tanh nan -2.3 -> nan nan tanh1029 tanh nan -inf -> nan nan tanh1030 tanh -0.0 -0.0 -> -0.0 -0.0 -tanh1031 tanh -0.0 -inf -> nan nan invalid +tanh1031 tanh -0.0 -inf -> -0.0 nan invalid tanh1032 tanh -2.3 -inf -> nan nan invalid -tanh1033 tanh -0.0 nan -> nan nan +tanh1033 tanh -0.0 nan -> -0.0 nan tanh1034 tanh -2.3 nan -> nan nan tanh1035 tanh -inf -0.0 -> -1.0 -0.0 tanh1036 tanh -inf -0.7 -> -1.0 -0.0 @@ -2035,7 +2035,7 @@ tanh1040 tanh -inf -3.5 -> -1.0 -0.0 tanh1041 tanh -inf -inf -> -1.0 0.0 ignore-imag-sign tanh1042 tanh -inf nan -> -1.0 0.0 ignore-imag-sign tanh1043 tanh -0.0 0.0 -> -0.0 0.0 -tanh1044 tanh -0.0 inf -> nan nan invalid +tanh1044 tanh -0.0 inf -> -0.0 nan invalid tanh1045 tanh -2.3 inf -> nan nan invalid tanh1046 tanh -inf 0.0 -> -1.0 0.0 tanh1047 tanh -inf 0.7 -> -1.0 0.0 @@ -2307,9 +2307,9 @@ tan0066 tan -8.79645943005142 0.0 -> 0.7265425280053614098 0.0 -- special values tan1000 tan -0.0 0.0 -> -0.0 0.0 -tan1001 tan -inf 0.0 -> nan nan invalid +tan1001 tan -inf 0.0 -> nan 0.0 invalid tan1002 tan -inf 2.2999999999999998 -> nan nan invalid -tan1003 tan nan 0.0 -> nan nan +tan1003 tan nan 0.0 -> nan 0.0 tan1004 tan nan 2.2999999999999998 -> nan nan tan1005 tan -0.0 inf -> -0.0 1.0 tan1006 tan -0.69999999999999996 inf -> -0.0 1.0 @@ -2324,7 +2324,7 @@ tan1014 tan -2.2999999999999998 nan -> nan nan tan1015 tan -inf nan -> nan nan tan1016 tan nan nan -> nan nan tan1017 tan 0.0 0.0 -> 0.0 0.0 -tan1018 tan inf 0.0 -> nan nan invalid +tan1018 tan inf 0.0 -> nan 0.0 invalid tan1019 tan inf 2.2999999999999998 -> nan nan invalid tan1020 tan 0.0 inf -> 0.0 1.0 tan1021 tan 0.69999999999999996 inf -> 0.0 1.0 @@ -2337,9 +2337,9 @@ tan1027 tan 0.0 nan -> 0.0 nan tan1028 tan 2.2999999999999998 nan -> nan nan tan1029 tan inf nan -> nan nan tan1030 tan 0.0 -0.0 -> 0.0 -0.0 -tan1031 tan inf -0.0 -> nan nan invalid +tan1031 tan inf -0.0 -> nan -0.0 invalid tan1032 tan inf -2.2999999999999998 -> nan nan invalid -tan1033 tan nan -0.0 -> nan nan +tan1033 tan nan -0.0 -> nan -0.0 tan1034 tan nan -2.2999999999999998 -> nan nan tan1035 tan 0.0 -inf -> 0.0 -1.0 tan1036 tan 0.69999999999999996 -inf -> 0.0 -1.0 @@ -2350,7 +2350,7 @@ tan1040 tan 3.5 -inf -> 0.0 -1.0 tan1041 tan inf -inf -> -0.0 -1.0 ignore-real-sign tan1042 tan nan -inf -> -0.0 -1.0 ignore-real-sign tan1043 tan -0.0 -0.0 -> -0.0 -0.0 -tan1044 tan -inf -0.0 -> nan nan invalid +tan1044 tan -inf -0.0 -> nan -0.0 invalid tan1045 tan -inf -2.2999999999999998 -> nan nan invalid tan1046 tan -0.0 -inf -> -0.0 -1.0 tan1047 tan -0.69999999999999996 -inf -> -0.0 -1.0 diff --git a/Lib/test/mathdata/ieee754.txt b/Lib/test/mathdata/ieee754.txt index a8b8a0a2148f00..0dac69040ad011 100644 --- a/Lib/test/mathdata/ieee754.txt +++ b/Lib/test/mathdata/ieee754.txt @@ -51,7 +51,7 @@ nan >>> INF / INF nan -However unambigous operations with inf return inf: +However unambiguous operations with inf return inf: >>> INF * INF inf >>> 1.5 * INF @@ -116,7 +116,7 @@ inf >>> 0 ** -1 Traceback (most recent call last): ... -ZeroDivisionError: 0.0 cannot be raised to a negative power +ZeroDivisionError: zero to a negative power >>> pow(0, NAN) nan diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 93e7dbbd103934..8ef8fae44f1d25 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -26,7 +26,7 @@ from test import support from test.support import os_helper from test.support import ( - TestFailed, run_with_locale, no_tracing, + TestFailed, run_with_locales, no_tracing, _2G, _4G, bigmemtest ) from test.support.import_helper import forget @@ -144,6 +144,14 @@ class E(C): def __getinitargs__(self): return () +import __main__ +__main__.C = C +C.__module__ = "__main__" +__main__.D = D +D.__module__ = "__main__" +__main__.E = E +E.__module__ = "__main__" + # Simple mutable object. class Object: pass @@ -157,14 +165,6 @@ def __reduce__(self): # Shouldn't support the recursion itself return K, (self.value,) -import __main__ -__main__.C = C -C.__module__ = "__main__" -__main__.D = D -D.__module__ = "__main__" -__main__.E = E -E.__module__ = "__main__" - class myint(int): def __init__(self, x): self.str = str(x) @@ -1179,6 +1179,166 @@ def test_compat_unpickle(self): self.assertIs(type(unpickled), collections.UserDict) self.assertEqual(unpickled, collections.UserDict({1: 2})) + def test_load_global(self): + self.assertIs(self.loads(b'cbuiltins\nstr\n.'), str) + self.assertIs(self.loads(b'cmath\nlog\n.'), math.log) + self.assertIs(self.loads(b'cos.path\njoin\n.'), os.path.join) + self.assertIs(self.loads(b'\x80\x04cbuiltins\nstr.upper\n.'), str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04cm\xc3\xb6dule\ngl\xc3\xb6bal\n.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'c\xff\nlog\n.') + self.assertRaises(UnicodeDecodeError, self.loads, b'cmath\n\xff\n.') + self.assertRaises(self.truncated_errors, self.loads, b'c\nlog\n.') + self.assertRaises(self.truncated_errors, self.loads, b'cmath\n\n.') + self.assertRaises(self.truncated_errors, self.loads, b'\x80\x04cmath\n\n.') + + def test_load_stack_global(self): + self.assertIs(self.loads(b'\x8c\x08builtins\x8c\x03str\x93.'), str) + self.assertIs(self.loads(b'\x8c\x04math\x8c\x03log\x93.'), math.log) + self.assertIs(self.loads(b'\x8c\x07os.path\x8c\x04join\x93.'), + os.path.join) + self.assertIs(self.loads(b'\x80\x04\x8c\x08builtins\x8c\x09str.upper\x93.'), + str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04\x8c\x07m\xc3\xb6dule\x8c\x07gl\xc3\xb6bal\x93.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x01\xff\x8c\x03log\x93.') + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x04math\x8c\x01\xff\x93.') + self.assertRaises(ValueError, self.loads, b'\x8c\x00\x8c\x03log\x93.') + self.assertRaises(AttributeError, self.loads, b'\x8c\x04math\x8c\x00\x93.') + self.assertRaises(AttributeError, self.loads, b'\x80\x04\x8c\x04math\x8c\x00\x93.') + + self.assertRaises(pickle.UnpicklingError, self.loads, b'N\x8c\x03log\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x8c\x04mathN\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x80\x04\x8c\x04mathN\x93.') + + def test_find_class(self): + unpickler = self.unpickler(io.BytesIO()) + unpickler_nofix = self.unpickler(io.BytesIO(), fix_imports=False) + unpickler4 = self.unpickler(io.BytesIO(b'\x80\x04N.')) + unpickler4.load() + + self.assertIs(unpickler.find_class('__builtin__', 'str'), str) + self.assertRaises(ModuleNotFoundError, + unpickler_nofix.find_class, '__builtin__', 'str') + self.assertIs(unpickler.find_class('builtins', 'str'), str) + self.assertIs(unpickler_nofix.find_class('builtins', 'str'), str) + self.assertIs(unpickler.find_class('math', 'log'), math.log) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + + self.assertIs(unpickler4.find_class('builtins', 'str.upper'), str.upper) + with self.assertRaisesRegex(AttributeError, + r"module 'builtins' has no attribute 'str\.upper'"): + unpickler.find_class('builtins', 'str.upper') + + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): + unpickler.find_class('math', 'spam') + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): + unpickler4.find_class('math', 'spam') + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.spam'"): + unpickler.find_class('math', 'log.spam') + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.spam' on module 'math'") as cm: + unpickler4.find_class('math', 'log.spam') + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute 'spam'") + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.\.spam'"): + unpickler.find_class('math', 'log..spam') + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.\.spam' on module 'math'") as cm: + unpickler4.find_class('math', 'log..spam') + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute ''") + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): + unpickler.find_class('math', '') + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): + unpickler4.find_class('math', '') + self.assertRaises(ModuleNotFoundError, unpickler.find_class, 'spam', 'log') + self.assertRaises(ValueError, unpickler.find_class, '', 'log') + + self.assertRaises(TypeError, unpickler.find_class, None, 'log') + self.assertRaises(TypeError, unpickler.find_class, 'math', None) + self.assertRaises((TypeError, AttributeError), unpickler4.find_class, 'math', None) + + def test_custom_find_class(self): + def loads(data): + class Unpickler(self.unpickler): + def find_class(self, module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + @staticmethod + def find_class(module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + @classmethod + def find_class(cls, module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + pass + def find_class(module_name, global_name): + return (module_name, global_name) + unpickler = Unpickler(io.BytesIO(data)) + unpickler.find_class = find_class + return unpickler.load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def test_bad_ext_code(self): + # unregistered extension code + self.check_unpickling_error(ValueError, b'\x82\x01.') + self.check_unpickling_error(ValueError, b'\x82\xff.') + self.check_unpickling_error(ValueError, b'\x83\x01\x00.') + self.check_unpickling_error(ValueError, b'\x83\xff\xff.') + self.check_unpickling_error(ValueError, b'\x84\x01\x00\x00\x00.') + self.check_unpickling_error(ValueError, b'\x84\xff\xff\xff\x7f.') + # EXT specifies code <= 0 + self.check_unpickling_error(pickle.UnpicklingError, b'\x82\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x83\x00\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x80.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\xff\xff\xff\xff.') + + @support.cpython_only + def test_bad_ext_inverted_registry(self): + code = 1 + def check(key, exc): + with support.swap_item(copyreg._inverted_registry, code, key): + with self.assertRaises(exc): + self.loads(b'\x82\x01.') + check(None, ValueError) + check((), ValueError) + check((__name__,), (TypeError, ValueError)) + check((__name__, "MyList", "x"), (TypeError, ValueError)) + check((__name__, None), (TypeError, ValueError)) + check((None, "MyList"), (TypeError, ValueError)) + def test_bad_reduce(self): self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0) self.check_unpickling_error(TypeError, b'N)R.') @@ -1443,6 +1603,789 @@ def t(): [ToBeUnpickled] * 2) +class AbstractPicklingErrorTests: + # Subclass must define self.dumps, self.pickler. + + def test_bad_reduce_result(self): + obj = REX([print, ()]) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + '__reduce__ must return a string or tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((print,)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((print, (), None, None, None, None, None)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_bad_reconstructor(self): + obj = REX((42, ())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'first item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_reconstructor(self): + obj = REX((UnpickleableCallable(), ())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + + def test_bad_reconstructor_args(self): + obj = REX((print, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_reconstructor_args(self): + obj = REX((print, (1, 2, UNPICKLEABLE))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_args(self): + obj = REX((copyreg.__newobj__, ())) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((IndexError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'tuple index out of range', + '__newobj__ expected at least 1 argument, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj__, [REX])) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_class(self): + obj = REX((copyreg.__newobj__, (NoNew(),))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj__() has no __new__', + f'first argument to __newobj__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_wrong_newobj_class(self): + obj = REX((copyreg.__newobj__, (str,))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj__() must be {REX!r}, not {str!r}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_newobj_class(self): + class LocalREX(REX): pass + obj = LocalREX((copyreg.__newobj__, (LocalREX,))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + + def test_unpickleable_newobj_args(self): + obj = REX((copyreg.__newobj__, (REX, 1, 2, UNPICKLEABLE))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_ex_args(self): + obj = REX((copyreg.__newobj_ex__, ())) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((ValueError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 3, got 0)', + '__newobj_ex__ expected 3 arguments, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj_ex__, 42)) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj_ex__, (REX, 42, {}))) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'Value after * must be an iterable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second argument to __newobj_ex__() must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj_ex__, (REX, (), []))) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'functools.partial() argument after ** must be a mapping, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'third argument to __newobj_ex__() must be a dict, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_ex__class(self): + obj = REX((copyreg.__newobj_ex__, (NoNew(), (), {}))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj_ex__() has no __new__', + f'first argument to __newobj_ex__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_wrong_newobj_ex_class(self): + if self.pickler is not pickle._Pickler: + self.skipTest('only verified in the Python implementation') + obj = REX((copyreg.__newobj_ex__, (str, (), {}))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj_ex__() must be {REX}, not {str}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_newobj_ex_class(self): + class LocalREX(REX): pass + obj = LocalREX((copyreg.__newobj_ex__, (LocalREX, (), {}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + + def test_unpickleable_newobj_ex_args(self): + obj = REX((copyreg.__newobj_ex__, (REX, (1, 2, UNPICKLEABLE), {}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_newobj_ex_kwargs(self): + obj = REX((copyreg.__newobj_ex__, (REX, (), {'a': UNPICKLEABLE}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_state(self): + obj = REX_state(UNPICKLEABLE) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_state state', + 'when serializing test.pickletester.REX_state object']) + + def test_bad_state_setter(self): + if self.pickler is pickle._Pickler: + self.skipTest('only verified in the C implementation') + obj = REX((print, (), 'state', None, None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'sixth item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_state_setter(self): + obj = REX((print, (), 'state', None, None, UnpickleableCallable())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state setter', + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_state_with_state_setter(self): + obj = REX((print, (), UNPICKLEABLE, None, None, print)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state', + 'when serializing test.pickletester.REX object']) + + def test_bad_object_list_items(self): + # Issue4176: crash when 4th and 5th items of __reduce__() + # are not iterators + obj = REX((list, (), None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + if self.pickler is not pickle._Pickler: + # Python implementation is less strict and also accepts iterables. + obj = REX((list, (), None, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_object_list_items(self): + obj = REX_six([1, 2, UNPICKLEABLE]) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_six item 2', + 'when serializing test.pickletester.REX_six object']) + + def test_bad_object_dict_items(self): + # Issue4176: crash when 4th and 5th items of __reduce__() + # are not iterators + obj = REX((dict, (), None, None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fifth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + for proto in protocols: + obj = REX((dict, (), None, None, iter([('a',)]))) + with self.subTest(proto=proto): + with self.assertRaises((ValueError, TypeError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 2, got 1)', + 'dict items iterator must return 2-tuples'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + if self.pickler is not pickle._Pickler: + # Python implementation is less strict and also accepts iterables. + obj = REX((dict, (), None, None, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'dict items iterator must return 2-tuples') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_object_dict_items(self): + obj = REX_seven({'a': UNPICKLEABLE}) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing test.pickletester.REX_seven item 'a'", + 'when serializing test.pickletester.REX_seven object']) + + def test_unpickleable_list_items(self): + obj = [1, [2, 3, UNPICKLEABLE]] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 2', + 'when serializing list item 1']) + for n in [0, 1, 1000, 1005]: + obj = [*range(n), UNPICKLEABLE] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + f'when serializing list item {n}']) + + def test_unpickleable_tuple_items(self): + obj = (1, (2, 3, UNPICKLEABLE)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1']) + obj = (*range(10), UNPICKLEABLE) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 10']) + + def test_unpickleable_dict_items(self): + obj = {'a': {'b': UNPICKLEABLE}} + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'b'", + "when serializing dict item 'a'"]) + for n in [0, 1, 1000, 1005]: + obj = dict.fromkeys(range(n)) + obj['a'] = UNPICKLEABLE + for proto in protocols: + with self.subTest(proto=proto, n=n): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'"]) + + def test_unpickleable_set_items(self): + obj = {UNPICKLEABLE} + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing set element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing set reconstructor arguments']) + + def test_unpickleable_frozenset_items(self): + obj = frozenset({frozenset({UNPICKLEABLE})}) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing frozenset element', + 'when serializing frozenset element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments', + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments']) + + def test_global_lookup_error(self): + # Global name does not exist + obj = REX('spam') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as {__name__}.spam") + self.assertEqual(str(cm.exception.__context__), + f"module '{__name__}' has no attribute 'spam'") + + obj.__module__ = 'nonexisting' + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: No module named 'nonexisting'") + self.assertEqual(str(cm.exception.__context__), + "No module named 'nonexisting'") + + obj.__module__ = '' + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: Empty module name") + self.assertEqual(str(cm.exception.__context__), + "Empty module name") + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as __main__.spam") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'spam'") + + def test_nonencodable_global_name_error(self): + for proto in protocols[:4]: + with self.subTest(proto=proto): + name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff' + obj = REX(name) + obj.__module__ = __name__ + with support.swap_item(globals(), name, obj): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle global identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) + + def test_nonencodable_module_name_error(self): + for proto in protocols[:4]: + with self.subTest(proto=proto): + name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff' + obj = REX('test') + obj.__module__ = name + mod = types.SimpleNamespace(test=obj) + with support.swap_item(sys.modules, name, mod): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle module identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) + + def test_nested_lookup_error(self): + # Nested name does not exist + obj = REX('AbstractPickleTests.spam') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as {__name__}.AbstractPickleTests.spam") + self.assertEqual(str(cm.exception.__context__), + "type object 'AbstractPickleTests' has no attribute 'spam'") + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as __main__.AbstractPickleTests.spam") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'AbstractPickleTests'") + + def test_wrong_object_lookup_error(self): + # Name is bound to different object + obj = REX('AbstractPickleTests') + obj.__module__ = __name__ + AbstractPickleTests.ham = [] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not the same object as {__name__}.AbstractPickleTests") + self.assertIsNone(cm.exception.__context__) + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as __main__.AbstractPickleTests") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'AbstractPickleTests'") + + def test_local_lookup_error(self): + # Test that whichmodule() errors out cleanly when looking up + # an assumed globally-reachable object fails. + def f(): + pass + # Since the function is local, lookup will fail + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") + # Same without a __module__ attribute (exercises a different path + # in _pickle.c). + del f.__module__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") + # Yet a different path. + f.__name__ = f.__qualname__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") + + def test_reduce_ex_None(self): + c = REX_None() + with self.assertRaises(TypeError): + self.dumps(c) + + def test_reduce_None(self): + c = R_None() + with self.assertRaises(TypeError): + self.dumps(c) + + @no_tracing + def test_bad_getattr(self): + # Issue #3514: crash when there is an infinite loop in __getattr__ + x = BadGetattr() + for proto in range(2): + with support.infinite_recursion(25): + self.assertRaises(RuntimeError, self.dumps, x, proto) + for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): + s = self.dumps(x, proto) + + def test_picklebuffer_error(self): + # PickleBuffer forbidden with protocol < 5 + pb = pickle.PickleBuffer(b"foobar") + for proto in range(0, 5): + with self.subTest(proto=proto): + with self.assertRaises(pickle.PickleError) as cm: + self.dumps(pb, proto) + self.assertEqual(str(cm.exception), + 'PickleBuffer can only be pickled with protocol >= 5') + + def test_non_continuous_buffer(self): + for proto in protocols[5:]: + with self.subTest(proto=proto): + pb = pickle.PickleBuffer(memoryview(b"foobar")[::2]) + with self.assertRaises((pickle.PicklingError, BufferError)): + self.dumps(pb, proto) + + def test_buffer_callback_error(self): + def buffer_callback(buffers): + raise CustomError + pb = pickle.PickleBuffer(b"foobar") + with self.assertRaises(CustomError): + self.dumps(pb, 5, buffer_callback=buffer_callback) + + def test_evil_pickler_mutating_collection(self): + # https://github.com/python/cpython/issues/92930 + global Clearer + class Clearer: + pass + + def check(collection): + class EvilPickler(self.pickler): + def persistent_id(self, obj): + if isinstance(obj, Clearer): + collection.clear() + return None + pickler = EvilPickler(io.BytesIO(), proto) + try: + pickler.dump(collection) + except RuntimeError as e: + expected = "changed size during iteration" + self.assertIn(expected, str(e)) + + for proto in protocols: + check([Clearer()]) + check([Clearer(), Clearer()]) + check({Clearer()}) + check({Clearer(), Clearer()}) + check({Clearer(): 1}) + check({Clearer(): 1, Clearer(): 2}) + check({1: Clearer(), 2: Clearer()}) + + @support.cpython_only + def test_bad_ext_code(self): + # This should never happen in normal circumstances, because the type + # and the value of the extension code is checked in copyreg.add_extension(). + key = (__name__, 'MyList') + def check(code, exc): + assert key not in copyreg._extension_registry + assert code not in copyreg._inverted_registry + with (support.swap_item(copyreg._extension_registry, key, code), + support.swap_item(copyreg._inverted_registry, code, key)): + for proto in protocols[2:]: + with self.assertRaises(exc): + self.dumps(MyList, proto) + + check(object(), TypeError) + check(None, TypeError) + check(-1, (RuntimeError, struct.error)) + check(0, RuntimeError) + check(2**31, (RuntimeError, OverflowError, struct.error)) + check(2**1000, (OverflowError, struct.error)) + check(-2**1000, (OverflowError, struct.error)) + class AbstractPickleTests: # Subclass must define self.dumps, self.loads. @@ -1845,6 +2788,25 @@ def test_bytes(self): p = self.dumps(s, proto) self.assert_is_copy(s, self.loads(p)) + def test_bytes_memoization(self): + for proto in protocols: + for array_type in [bytes, ZeroCopyBytes]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + x, y = self.loads(p) + self.assertIs(x, y) + self.assert_is_copy((b, b), (x, y)) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1, b2 = array_type(s), array_type(s) + p = self.dumps((b1, b2), proto) + # Note that (b1, b2) = self.loads(p) might have identical + # components, i.e., b1 is b2, but this is not always the + # case if the content is large (equality still holds). + self.assert_is_copy((b1, b2), self.loads(p)) + def test_bytearray(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: @@ -1864,13 +2826,31 @@ def test_bytearray(self): self.assertNotIn(b'bytearray', p) self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) - def test_bytearray_memoization_bug(self): + def test_bytearray_memoization(self): for proto in protocols: - for s in b'', b'xyz', b'xyz'*100: - b = bytearray(s) - p = self.dumps((b, b), proto) - b1, b2 = self.loads(p) - self.assertIs(b1, b2) + for array_type in [bytearray, ZeroCopyBytearray]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + b1, b2 = self.loads(p) + self.assertIs(b1, b2) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1a, b2a = array_type(s), array_type(s) + # Unlike bytes, equal but independent bytearray objects are + # never identical. + self.assertIsNot(b1a, b2a) + + p = self.dumps((b1a, b2a), proto) + b1b, b2b = self.loads(p) + self.assertIsNot(b1b, b2b) + + self.assertIsNot(b1a, b1b) + self.assert_is_copy(b1a, b1b) + + self.assertIsNot(b2a, b2b) + self.assert_is_copy(b2a, b2b) def test_ints(self): for proto in protocols: @@ -1915,7 +2895,7 @@ def test_float(self): got = self.loads(pickle) self.assert_is_copy(value, got) - @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @run_with_locales('LC_ALL', 'de_DE', 'fr_FR', '') def test_float_format(self): # make sure that floats are formatted locale independent with proto 0 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') @@ -2416,55 +3396,12 @@ def test_reduce_calls_base(self): y = self.loads(s) self.assertEqual(y._reduce_called, 1) - def test_reduce_ex_None(self): - c = REX_None() - with self.assertRaises(TypeError): - self.dumps(c) - - def test_reduce_None(self): - c = R_None() - with self.assertRaises(TypeError): - self.dumps(c) - def test_pickle_setstate_None(self): c = C_None_setstate() p = self.dumps(c) with self.assertRaises(TypeError): self.loads(p) - @no_tracing - def test_bad_getattr(self): - # Issue #3514: crash when there is an infinite loop in __getattr__ - x = BadGetattr() - for proto in range(2): - with support.infinite_recursion(25): - self.assertRaises(RuntimeError, self.dumps, x, proto) - for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): - s = self.dumps(x, proto) - - def test_reduce_bad_iterator(self): - # Issue4176: crash when 4th and 5th items of __reduce__() - # are not iterators - class C(object): - def __reduce__(self): - # 4th item is not an iterator - return list, (), None, [], None - class D(object): - def __reduce__(self): - # 5th item is not an iterator - return dict, (), None, None, [] - - # Python implementation is less strict and also accepts iterables. - for proto in protocols: - try: - self.dumps(C(), proto) - except pickle.PicklingError: - pass - try: - self.dumps(D(), proto) - except pickle.PicklingError: - pass - def test_many_puts_and_gets(self): # Test that internal data structures correctly deal with lots of # puts/gets. @@ -2781,6 +3718,18 @@ class Recursive: self.assertIs(unpickled, Recursive) del Recursive.mod # break reference loop + def test_recursive_nested_names2(self): + global Recursive + class Recursive: + pass + Recursive.ref = Recursive + Recursive.__qualname__ = 'Recursive.ref' + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(Recursive, proto)) + self.assertIs(unpickled, Recursive) + del Recursive.ref # break reference loop + def test_py_methods(self): global PyMethodsTest class PyMethodsTest: @@ -2901,27 +3850,6 @@ def test_compat_pickle(self): self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) self.assertIs(type(self.loads(pickled)), type(val)) - def test_local_lookup_error(self): - # Test that whichmodule() errors out cleanly when looking up - # an assumed globally-reachable object fails. - def f(): - pass - # Since the function is local, lookup will fail - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Same without a __module__ attribute (exercises a different path - # in _pickle.c). - del f.__module__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Yet a different path. - f.__name__ = f.__qualname__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # # PEP 574 tests below # @@ -3032,20 +3960,6 @@ def test_oob_buffers_writable_to_readonly(self): self.assertIs(type(new), type(obj)) self.assertEqual(new, obj) - def test_picklebuffer_error(self): - # PickleBuffer forbidden with protocol < 5 - pb = pickle.PickleBuffer(b"foobar") - for proto in range(0, 5): - with self.assertRaises(pickle.PickleError): - self.dumps(pb, proto) - - def test_buffer_callback_error(self): - def buffer_callback(buffers): - 1/0 - pb = pickle.PickleBuffer(b"foobar") - with self.assertRaises(ZeroDivisionError): - self.dumps(pb, 5, buffer_callback=buffer_callback) - def test_buffers_error(self): pb = pickle.PickleBuffer(b"foobar") for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): @@ -3137,37 +4051,6 @@ def __reduce__(self): expected = "changed size during iteration" self.assertIn(expected, str(e)) - def test_evil_pickler_mutating_collection(self): - # https://github.com/python/cpython/issues/92930 - if not hasattr(self, "pickler"): - raise self.skipTest(f"{type(self)} has no associated pickler type") - - global Clearer - class Clearer: - pass - - def check(collection): - class EvilPickler(self.pickler): - def persistent_id(self, obj): - if isinstance(obj, Clearer): - collection.clear() - return None - pickler = EvilPickler(io.BytesIO(), proto) - try: - pickler.dump(collection) - except RuntimeError as e: - expected = "changed size during iteration" - self.assertIn(expected, str(e)) - - for proto in protocols: - check([Clearer()]) - check([Clearer(), Clearer()]) - check({Clearer()}) - check({Clearer(), Clearer()}) - check({Clearer(): 1}) - check({Clearer(): 1, Clearer(): 2}) - check({1: Clearer(), 2: Clearer()}) - class BigmemPickleTests: @@ -3298,6 +4181,18 @@ def test_huge_str_64b(self, size): # Test classes for reduce_ex +class R: + def __init__(self, reduce=None): + self.reduce = reduce + def __reduce__(self, proto): + return self.reduce + +class REX: + def __init__(self, reduce_ex=None): + self.reduce_ex = reduce_ex + def __reduce_ex__(self, proto): + return self.reduce_ex + class REX_one(object): """No __reduce_ex__ here, but inheriting it from object""" _reduce_called = 0 @@ -3388,6 +4283,19 @@ def __getstate__(self): __setstate__ = None +class CustomError(Exception): + pass + +class Unpickleable: + def __reduce__(self): + raise CustomError + +UNPICKLEABLE = Unpickleable() + +class UnpickleableCallable(Unpickleable): + def __call__(self, *args, **kwargs): + pass + # Test classes for newobj @@ -3437,7 +4345,9 @@ class MyIntWithNew2(MyIntWithNew): class SlotList(MyList): __slots__ = ["foo"] -class SimpleNewObj(int): +# Ruff "redefined while unused" false positive here due to `global` variables +# being assigned (and then restored) from within test methods earlier in the file +class SimpleNewObj(int): # noqa: F811 def __init__(self, *args, **kwargs): # raise an error, to make sure this isn't called raise TypeError("SimpleNewObj.__init__() didn't expect to get called") @@ -3456,6 +4366,12 @@ class BadGetattr: def __getattr__(self, key): self.foo +class NoNew: + def __getattribute__(self, name): + if name == '__new__': + raise AttributeError + return super().__getattribute__(name) + class AbstractPickleModuleTests: @@ -3528,7 +4444,7 @@ def raises_oserror(self, *args, **kwargs): raise OSError @property def bad_property(self): - 1/0 + raise CustomError # File without read and readline class F: @@ -3549,23 +4465,23 @@ class F: class F: read = bad_property readline = raises_oserror - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad readline class F: readline = bad_property read = raises_oserror - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad readline, no read class F: readline = bad_property - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad read, no readline class F: read = bad_property - self.assertRaises((AttributeError, ZeroDivisionError), self.Unpickler, F()) + self.assertRaises((AttributeError, CustomError), self.Unpickler, F()) # File with bad peek class F: @@ -3574,7 +4490,7 @@ class F: readline = raises_oserror try: self.Unpickler(F()) - except ZeroDivisionError: + except CustomError: pass # File with bad readinto @@ -3584,7 +4500,7 @@ class F: readline = raises_oserror try: self.Unpickler(F()) - except ZeroDivisionError: + except CustomError: pass def test_pickler_bad_file(self): @@ -3597,8 +4513,8 @@ class F: class F: @property def write(self): - 1/0 - self.assertRaises(ZeroDivisionError, self.Pickler, F()) + raise CustomError + self.assertRaises(CustomError, self.Pickler, F()) def check_dumps_loads_oob_buffers(self, dumps, loads): # No need to do the full gamut of tests here, just enough to @@ -3706,9 +4622,15 @@ def test_return_correct_type(self): def test_protocol0_is_ascii_only(self): non_ascii_str = "\N{EMPTY SET}" - self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0) + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(non_ascii_str, 0) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.' - self.assertRaises(pickle.UnpicklingError, self.loads, pickled) + with self.assertRaises(pickle.UnpicklingError) as cm: + self.loads(pickled) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') class AbstractPicklerUnpicklerObjectTests: @@ -3978,8 +4900,11 @@ class MyClass: # NotImplemented self.assertIs(math_log, math.log) - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: p.dump(g) + self.assertRegex(str(cm.exception), + r'(__reduce__|)' + r' must return (a )?string or tuple') with self.assertRaisesRegex( ValueError, 'The reducer just failed'): diff --git a/Lib/test/pyclbr_input.py b/Lib/test/pyclbr_input.py index 19ccd62dead8ee..5535edbfa7795b 100644 --- a/Lib/test/pyclbr_input.py +++ b/Lib/test/pyclbr_input.py @@ -12,17 +12,19 @@ class B (object): def bm(self): pass class C (B): - foo = Other().foo - om = Other.om - d = 10 - # XXX: This causes test_pyclbr.py to fail, but only because the - # introspection-based is_method() code in the test can't - # distinguish between this and a genuine method function like m(). - # The pyclbr.py module gets this right as it parses the text. + # This one is correctly considered by both test_pyclbr.py and pyclbr.py + # as a non-method of C. + foo = Other().foo + + # This causes test_pyclbr.py to fail, but only because the + # introspection-based is_method() code in the test can't + # distinguish between this and a genuine method function like m(). # - #f = f + # The pyclbr.py module gets this right as it parses the text. + om = Other.om + f = f def m(self): pass @@ -31,3 +33,53 @@ def sm(self): pass @classmethod def cm(self): pass + +# Check that mangling is correctly handled + +class a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class ___: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 814358746d6d8a..0b2e4b1c1988c4 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -105,9 +105,13 @@ def collect_sys(info_add): ) copy_attributes(info_add, sys, 'sys.%s', attributes) - call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel') - call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion') - call_func(info_add, 'sys.getrecursionlimit', sys, 'getrecursionlimit') + for func in ( + '_is_gil_enabled', + 'getandroidapilevel', + 'getrecursionlimit', + 'getwindowsversion', + ): + call_func(info_add, f'sys.{func}', sys, func) encoding = sys.getfilesystemencoding() if hasattr(sys, 'getfilesystemencodeerrors'): @@ -179,6 +183,9 @@ def collect_platform(info_add): info_add(f'platform.freedesktop_os_release[{key}]', os_release[key]) + if sys.platform == 'android': + call_func(info_add, 'platform.android_ver', platform, 'android_ver') + def collect_locale(info_add): import locale @@ -287,6 +294,7 @@ def format_groups(groups): "HOMEDRIVE", "HOMEPATH", "IDLESTARTUP", + "IPHONEOS_DEPLOYMENT_TARGET", "LANG", "LDFLAGS", "LDSHARED", @@ -326,6 +334,7 @@ def format_groups(groups): "_PYTHON_HOST_PLATFORM", "_PYTHON_PROJECT_BASE", "_PYTHON_SYSCONFIGDATA_NAME", + "_PYTHON_SYSCONFIGDATA_PATH", "__PYVENV_LAUNCHER__", # Sanitizer options @@ -509,6 +518,7 @@ def collect_sysconfig(info_add): 'MACHDEP', 'MULTIARCH', 'OPT', + 'PGO_PROF_USE_FLAG', 'PY_CFLAGS', 'PY_CFLAGS_NODIST', 'PY_CORE_LDFLAGS', @@ -520,6 +530,7 @@ def collect_sysconfig(info_add): 'Py_GIL_DISABLED', 'SHELL', 'SOABI', + 'TEST_MODULES', 'abs_builddir', 'abs_srcdir', 'prefix', @@ -543,7 +554,6 @@ def collect_sysconfig(info_add): for name in ( 'WITH_DOC_STRINGS', 'WITH_DTRACE', - 'WITH_FREELISTS', 'WITH_MIMALLOC', 'WITH_PYMALLOC', 'WITH_VALGRIND', diff --git a/Lib/test/reperf.py b/Lib/test/reperf.py deleted file mode 100644 index e93bacdb5843c6..00000000000000 --- a/Lib/test/reperf.py +++ /dev/null @@ -1,23 +0,0 @@ -import re -import time - -def main(): - s = "\13hello\14 \13world\14 " * 1000 - p = re.compile(r"([\13\14])") - timefunc(10, p.sub, "", s) - timefunc(10, p.split, s) - timefunc(10, p.findall, s) - -def timefunc(n, func, *args, **kw): - t0 = time.perf_counter() - try: - for i in range(n): - result = func(*args, **kw) - return result - finally: - t1 = time.perf_counter() - if n > 1: - print(n, "times", end=' ') - print(func.__name__, "%.3f" % (t1-t0), "CPU seconds") - -main() diff --git a/Lib/test/sortperf.py b/Lib/test/sortperf.py deleted file mode 100644 index 14a9d827ed57c5..00000000000000 --- a/Lib/test/sortperf.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Sort performance test. - -See main() for command line syntax. -See tabulate() for output format. - -""" - -import sys -import time -import random -import marshal -import tempfile -import os - -td = tempfile.gettempdir() - -def randfloats(n): - """Return a list of n random floats in [0, 1).""" - # Generating floats is expensive, so this writes them out to a file in - # a temp directory. If the file already exists, it just reads them - # back in and shuffles them a bit. - fn = os.path.join(td, "rr%06d" % n) - try: - fp = open(fn, "rb") - except OSError: - r = random.random - result = [r() for i in range(n)] - try: - try: - fp = open(fn, "wb") - marshal.dump(result, fp) - fp.close() - fp = None - finally: - if fp: - try: - os.unlink(fn) - except OSError: - pass - except OSError as msg: - print("can't write", fn, ":", msg) - else: - result = marshal.load(fp) - fp.close() - # Shuffle it a bit... - for i in range(10): - i = random.randrange(n) - temp = result[:i] - del result[:i] - temp.reverse() - result.extend(temp) - del temp - assert len(result) == n - return result - -def flush(): - sys.stdout.flush() - -def doit(L): - t0 = time.perf_counter() - L.sort() - t1 = time.perf_counter() - print("%6.2f" % (t1-t0), end=' ') - flush() - -def tabulate(r): - r"""Tabulate sort speed for lists of various sizes. - - The sizes are 2**i for i in r (the argument, a list). - - The output displays i, 2**i, and the time to sort arrays of 2**i - floating point numbers with the following properties: - - *sort: random data - \sort: descending data - /sort: ascending data - 3sort: ascending, then 3 random exchanges - +sort: ascending, then 10 random at the end - %sort: ascending, then randomly replace 1% of the elements w/ random values - ~sort: many duplicates - =sort: all equal - !sort: worst case scenario - - """ - cases = tuple([ch + "sort" for ch in r"*\/3+%~=!"]) - fmt = ("%2s %7s" + " %6s"*len(cases)) - print(fmt % (("i", "2**i") + cases)) - for i in r: - n = 1 << i - L = randfloats(n) - print("%2d %7d" % (i, n), end=' ') - flush() - doit(L) # *sort - L.reverse() - doit(L) # \sort - doit(L) # /sort - - # Do 3 random exchanges. - for dummy in range(3): - i1 = random.randrange(n) - i2 = random.randrange(n) - L[i1], L[i2] = L[i2], L[i1] - doit(L) # 3sort - - # Replace the last 10 with random floats. - if n >= 10: - L[-10:] = [random.random() for dummy in range(10)] - doit(L) # +sort - - # Replace 1% of the elements at random. - for dummy in range(n // 100): - L[random.randrange(n)] = random.random() - doit(L) # %sort - - # Arrange for lots of duplicates. - if n > 4: - del L[4:] - L = L * (n // 4) - # Force the elements to be distinct objects, else timings can be - # artificially low. - L = list(map(lambda x: --x, L)) - doit(L) # ~sort - del L - - # All equal. Again, force the elements to be distinct objects. - L = list(map(abs, [-0.5] * n)) - doit(L) # =sort - del L - - # This one looks like [3, 2, 1, 0, 0, 1, 2, 3]. It was a bad case - # for an older implementation of quicksort, which used the median - # of the first, last and middle elements as the pivot. - half = n // 2 - L = list(range(half - 1, -1, -1)) - L.extend(range(half)) - # Force to float, so that the timings are comparable. This is - # significantly faster if we leave them as ints. - L = list(map(float, L)) - doit(L) # !sort - print() - -def main(): - """Main program when invoked as a script. - - One argument: tabulate a single row. - Two arguments: tabulate a range (inclusive). - Extra arguments are used to seed the random generator. - - """ - # default range (inclusive) - k1 = 15 - k2 = 20 - if sys.argv[1:]: - # one argument: single point - k1 = k2 = int(sys.argv[1]) - if sys.argv[2:]: - # two arguments: specify range - k2 = int(sys.argv[2]) - if sys.argv[3:]: - # derive random seed from remaining arguments - x = 1 - for a in sys.argv[3:]: - x = 69069 * x + hash(a) - random.seed(x) - r = range(k1, k2+1) # include the end point - tabulate(r) - -if __name__ == '__main__': - main() diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index cecf309dca9194..4b82d51b4508ac 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -1132,8 +1132,8 @@ def test_capitalize_nonascii(self): self.checkequal('\u2160\u2171\u2172', '\u2170\u2171\u2172', 'capitalize') # check with Ll chars with no upper - nothing changes here - self.checkequal('\u019b\u1d00\u1d86\u0221\u1fb7', - '\u019b\u1d00\u1d86\u0221\u1fb7', 'capitalize') + self.checkequal('\u1d00\u1d86\u0221\u1fb7', + '\u1d00\u1d86\u0221\u1fb7', 'capitalize') def test_startswith(self): self.checkequal(True, 'hello', 'startswith', 'he') @@ -1503,19 +1503,19 @@ def test_find_etc_raise_correct_error_messages(self): # issue 11828 s = 'hello' x = 'x' - self.assertRaisesRegex(TypeError, r'^find\(', s.find, + self.assertRaisesRegex(TypeError, r'^find\b', s.find, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rfind\(', s.rfind, + self.assertRaisesRegex(TypeError, r'^rfind\b', s.rfind, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^index\(', s.index, + self.assertRaisesRegex(TypeError, r'^index\b', s.index, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rindex\(', s.rindex, + self.assertRaisesRegex(TypeError, r'^rindex\b', s.rindex, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^count\(', s.count, + self.assertRaisesRegex(TypeError, r'^count\b', s.count, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^startswith\(', s.startswith, + self.assertRaisesRegex(TypeError, r'^startswith\b', s.startswith, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^endswith\(', s.endswith, + self.assertRaisesRegex(TypeError, r'^endswith\b', s.endswith, x, None, None, None) # issue #15534 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 5b091fb2fd32dc..7c1ef42a4970d7 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -4,8 +4,8 @@ raise ImportError('support must be imported from the test package') import contextlib -import dataclasses import functools +import inspect import _opcode import os import re @@ -26,10 +26,10 @@ "Error", "TestFailed", "TestDidNotRun", "ResourceDenied", # io "record_original_stdout", "get_original_stdout", "captured_stdout", - "captured_stdin", "captured_stderr", + "captured_stdin", "captured_stderr", "captured_output", # unittest "is_resource_enabled", "requires", "requires_freebsd_version", - "requires_linux_version", "requires_mac_ver", + "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", "requires_gzip", "requires_bz2", "requires_lzma", "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", @@ -56,9 +56,12 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "Py_C_RECURSION_LIMIT", + "Py_DEBUG", "exceeds_recursion_limit", "get_c_recursion_limit", "skip_on_s390x", "without_optimizer", + "force_not_colorized", + "BrokenIter", + "in_systemd_nspawn_sync_suppressed", ] @@ -250,22 +253,16 @@ class USEROBJECTFLAGS(ctypes.Structure): # process not running under the same user id as the current console # user. To avoid that, raise an exception if the window manager # connection is not available. - from ctypes import cdll, c_int, pointer, Structure - from ctypes.util import find_library - - app_services = cdll.LoadLibrary(find_library("ApplicationServices")) - - if app_services.CGMainDisplayID() == 0: - reason = "gui tests cannot run without OS X window manager" + import subprocess + try: + rc = subprocess.run(["launchctl", "managername"], + capture_output=True, check=True) + managername = rc.stdout.decode("utf-8").strip() + except subprocess.CalledProcessError: + reason = "unable to detect macOS launchd job manager" else: - class ProcessSerialNumber(Structure): - _fields_ = [("highLongOfPSN", c_int), - ("lowLongOfPSN", c_int)] - psn = ProcessSerialNumber() - psn_p = pointer(psn) - if ( (app_services.GetCurrentProcess(psn_p) < 0) or - (app_services.SetFrontProcess(psn_p) < 0) ): - reason = "cannot run without OS X gui process" + if managername != "Aqua": + reason = f"{managername=} -- can only run in a macOS GUI session" # check on every platform whether tkinter can actually do anything if not reason: @@ -387,7 +384,7 @@ def skip_if_buildbot(reason=None): reason = 'not suitable for buildbots' try: isbuildbot = getpass.getuser().lower() == 'buildbot' - except (KeyError, EnvironmentError) as err: + except (KeyError, OSError) as err: warnings.warn(f'getpass.getuser() failed {err}.', RuntimeWarning) isbuildbot = False return unittest.skipIf(isbuildbot, reason) @@ -515,12 +512,18 @@ def has_no_debug_ranges(): def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(has_no_debug_ranges(), reason) + MS_WINDOWS = (sys.platform == 'win32') # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') -is_android = hasattr(sys, 'getandroidapilevel') +is_android = sys.platform == "android" + +def skip_android_selinux(name): + return unittest.skipIf( + sys.platform == "android", f"Android blocks {name} with SELinux" + ) if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' @@ -532,24 +535,33 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" -# Apple mobile platforms (iOS/tvOS/watchOS) are POSIX-like but do not -# have subprocess or fork support. is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} is_apple = is_apple_mobile or sys.platform == "darwin" has_fork_support = hasattr(os, "fork") and not ( + # WASM and Apple mobile platforms do not support subprocesses. is_emscripten or is_wasi or is_apple_mobile + + # Although Android supports fork, it's unsafe to call it from Python because + # all Android apps are multi-threaded. + or is_android ) def requires_fork(): return unittest.skipUnless(has_fork_support, "requires working os.fork()") has_subprocess_support = not ( + # WASM and Apple mobile platforms do not support subprocesses. is_emscripten or is_wasi or is_apple_mobile + + # Although Android supports subproceses, they're almost never useful in + # practice (see PEP 738). And most of the tests that use them are calling + # sys.executable, which won't work when Python is embedded in an Android app. + or is_android ) def requires_subprocess(): @@ -788,6 +800,16 @@ def disable_gc(): if have_gc: gc.enable() +@contextlib.contextmanager +def gc_threshold(*args): + import gc + old_threshold = gc.get_threshold() + gc.set_threshold(*args) + try: + yield + finally: + gc.set_threshold(*old_threshold) + def python_is_optimized(): """Find if Python was built with optimizations.""" @@ -817,7 +839,27 @@ def check_cflags_pgo(): return any(option in cflags_nodist for option in pgo_options) +def check_bolt_optimized(): + # Always return false, if the platform is WASI, + # because BOLT optimization does not support WASM binary. + if is_wasi: + return False + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + return '--enable-bolt' in config_args + + Py_GIL_DISABLED = bool(sysconfig.get_config_var('Py_GIL_DISABLED')) + +def requires_gil_enabled(msg="needs the GIL enabled"): + """Decorator for skipping tests on the free-threaded build.""" + return unittest.skipIf(Py_GIL_DISABLED, msg) + +def expected_failure_if_gil_disabled(): + """Expect test failure if the GIL is disabled.""" + if Py_GIL_DISABLED: + return unittest.expectedFailure + return lambda test_case: test_case + if Py_GIL_DISABLED: _header = 'PHBBInP' else: @@ -834,8 +876,16 @@ def calcvobjsize(fmt): return struct.calcsize(_vheader + fmt + _align) -_TPFLAGS_HAVE_GC = 1<<14 +_TPFLAGS_STATIC_BUILTIN = 1<<1 +_TPFLAGS_DISALLOW_INSTANTIATION = 1<<7 +_TPFLAGS_IMMUTABLETYPE = 1<<8 _TPFLAGS_HEAPTYPE = 1<<9 +_TPFLAGS_BASETYPE = 1<<10 +_TPFLAGS_READY = 1<<12 +_TPFLAGS_READYING = 1<<13 +_TPFLAGS_HAVE_GC = 1<<14 +_TPFLAGS_BASE_EXC_SUBCLASS = 1<<30 +_TPFLAGS_TYPE_SUBCLASS = 1<<31 def check_sizeof(test, o, size): try: @@ -852,8 +902,8 @@ def check_sizeof(test, o, size): test.assertEqual(result, size, msg) #======================================================================= -# Decorator for running a function in a different locale, correctly resetting -# it afterwards. +# Decorator/context manager for running a code in a different locale, +# correctly resetting it afterwards. @contextlib.contextmanager def run_with_locale(catstr, *locales): @@ -864,16 +914,21 @@ def run_with_locale(catstr, *locales): except AttributeError: # if the test author gives us an invalid category string raise - except: + except Exception: # cannot retrieve original locale, so do nothing locale = orig_locale = None + if '' not in locales: + raise unittest.SkipTest('no locales') else: for loc in locales: try: locale.setlocale(category, loc) break - except: + except locale.Error: pass + else: + if '' not in locales: + raise unittest.SkipTest(f'no locales {locales}') try: yield @@ -881,6 +936,46 @@ def run_with_locale(catstr, *locales): if locale and orig_locale: locale.setlocale(category, orig_locale) +#======================================================================= +# Decorator for running a function in multiple locales (if they are +# availasble) and resetting the original locale afterwards. + +def run_with_locales(catstr, *locales): + def deco(func): + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + dry_run = '' in locales + try: + import locale + category = getattr(locale, catstr) + orig_locale = locale.setlocale(category) + except AttributeError: + # if the test author gives us an invalid category string + raise + except Exception: + # cannot retrieve original locale, so do nothing + pass + else: + try: + for loc in locales: + with self.subTest(locale=loc): + try: + locale.setlocale(category, loc) + except locale.Error: + self.skipTest(f'no locale {loc!r}') + else: + dry_run = False + func(self, *args, **kwargs) + finally: + locale.setlocale(category, orig_locale) + if dry_run: + # no locales available, so just run the test + # with the current locale + with self.subTest(locale=None): + func(self, *args, **kwargs) + return wrapper + return deco + #======================================================================= # Decorator for running a function in a specific timezone, correctly # resetting it afterwards. @@ -1130,6 +1225,26 @@ def coverage_wrapper(*args, **kwargs): return coverage_wrapper +def no_rerun(reason): + """Skip rerunning for a particular test. + + WARNING: Use this decorator with care; skipping rerunning makes it + impossible to find reference leaks. Provide a clear reason for skipping the + test using the 'reason' parameter. + """ + def deco(func): + assert not isinstance(func, type), func + _has_run = False + def wrapper(self): + nonlocal _has_run + if _has_run: + self.skipTest(reason) + func(self) + _has_run = True + return wrapper + return deco + + def refcount_test(test): """Decorator for tests which involve reference counting. @@ -1143,11 +1258,17 @@ def refcount_test(test): def requires_limited_api(test): try: - import _testcapi + import _testcapi # noqa: F401 + import _testlimitedcapi # noqa: F401 except ImportError: - return unittest.skip('needs _testcapi module')(test) + return unittest.skip('needs _testcapi and _testlimitedcapi modules')(test) return test + +# Windows build doesn't support --disable-test-modules feature, so there's no +# 'TEST_MODULES' var in config +TEST_MODULES_ENABLED = (sysconfig.get_config_var('TEST_MODULES') or 'yes') == 'yes' + def requires_specialization(test): return unittest.skipUnless( _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) @@ -1695,7 +1816,10 @@ def run_in_subinterp(code): module is enabled. """ _check_tracemalloc() - import _testcapi + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") return _testcapi.run_in_subinterp(code) @@ -1705,11 +1829,25 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): module is enabled. """ _check_tracemalloc() - import _testinternalcapi + try: + import _testinternalcapi + except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") if own_gil is not None: assert 'gil' not in config, (own_gil, config) - config['gil'] = 2 if own_gil else 1 - return _testinternalcapi.run_in_subinterp_with_config(code, **config) + config['gil'] = 'own' if own_gil else 'shared' + else: + gil = config['gil'] + if gil == 0: + config['gil'] = 'default' + elif gil == 1: + config['gil'] = 'shared' + elif gil == 2: + config['gil'] = 'own' + elif not isinstance(gil, str): + raise NotImplementedError(gil) + config = types.SimpleNamespace(**config) + return _testinternalcapi.run_in_subinterp_with_config(code, config) def _check_tracemalloc(): @@ -1727,19 +1865,22 @@ def _check_tracemalloc(): def check_free_after_iterating(test, iter, cls, args=()): - class A(cls): - def __del__(self): - nonlocal done - done = True - try: - next(it) - except StopIteration: - pass - done = False - it = iter(A(*args)) - # Issue 26494: Shouldn't crash - test.assertRaises(StopIteration, next, it) + def wrapper(): + class A(cls): + def __del__(self): + nonlocal done + done = True + try: + next(it) + except StopIteration: + pass + + it = iter(A(*args)) + # Issue 26494: Shouldn't crash + test.assertRaises(StopIteration, next, it) + + wrapper() # The sequence should be deallocated just after the end of iterating gc_collect() test.assertTrue(done) @@ -1778,18 +1919,18 @@ def missing_compiler_executable(cmd_names=[]): return cmd[0] -_is_android_emulator = None +_old_android_emulator = None def setswitchinterval(interval): # Setting a very low gil interval on the Android emulator causes python # to hang (issue #26939). - minimum_interval = 1e-5 + minimum_interval = 1e-4 # 100 us if is_android and interval < minimum_interval: - global _is_android_emulator - if _is_android_emulator is None: - import subprocess - _is_android_emulator = (subprocess.check_output( - ['getprop', 'ro.kernel.qemu']).strip() == b'1') - if _is_android_emulator: + global _old_android_emulator + if _old_android_emulator is None: + import platform + av = platform.android_ver() + _old_android_emulator = av.is_emulator and av.api_level < 24 + if _old_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) @@ -1864,12 +2005,18 @@ def restore(self): def with_pymalloc(): - import _testcapi + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED def with_mimalloc(): - import _testcapi + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") return _testcapi.WITH_MIMALLOC @@ -2079,7 +2226,15 @@ def skip_if_broken_multiprocessing_synchronize(): # bpo-38377: On Linux, creating a semaphore fails with OSError # if the current user does not have the permission to create # a file in /dev/shm/ directory. - synchronize.Lock(ctx=None) + import multiprocessing + synchronize.Lock(ctx=multiprocessing.get_context('fork')) + # The explicit fork mp context is required in order for + # TestResourceTracker.test_resource_tracker_reused to work. + # synchronize creates a new multiprocessing.resource_tracker + # process at module import time via the above call in that + # scenario. Awkward. This enables gh-84559. No code involved + # should have threads at that point so fork() should be safe. + except OSError as exc: raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") @@ -2189,7 +2344,7 @@ def clear_ignored_deprecations(*tokens: object) -> None: def requires_venv_with_pip(): # ensurepip requires zlib to open ZIP archives (.whl binary wheel packages) try: - import zlib + import zlib # noqa: F401 except ImportError: return unittest.skipIf(True, "venv: ensurepip requires zlib") @@ -2228,16 +2383,25 @@ def _findwheel(pkgname): # and returns the path to the venv directory and the path to the python executable @contextlib.contextmanager def setup_venv_with_pip_setuptools_wheel(venv_dir): + import shlex import subprocess from .os_helper import temp_cwd + def run_command(cmd): + if verbose: + print() + print('Run:', ' '.join(map(shlex.quote, cmd))) + subprocess.run(cmd, check=True) + else: + subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True) + with temp_cwd() as temp_dir: # Create virtual environment to get setuptools cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) venv = os.path.join(temp_dir, venv_dir) @@ -2252,10 +2416,7 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): '-m', 'pip', 'install', _findwheel('setuptools'), _findwheel('wheel')] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) yield python @@ -2378,6 +2539,46 @@ def sleeping_retry(timeout, err_msg=None, /, delay = min(delay * 2, max_delay) +class CPUStopwatch: + """Context manager to roughly time a CPU-bound operation. + + Disables GC. Uses CPU time if it can (i.e. excludes sleeps & time of + other processes). + + N.B.: + - This *includes* time spent in other threads. + - Some systems only have a coarse resolution; check + stopwatch.clock_info.rseolution if. + + Usage: + + with ProcessStopwatch() as stopwatch: + ... + elapsed = stopwatch.seconds + resolution = stopwatch.clock_info.resolution + """ + def __enter__(self): + get_time = time.process_time + clock_info = time.get_clock_info('process_time') + if get_time() <= 0: # some platforms like WASM lack process_time() + get_time = time.monotonic + clock_info = time.get_clock_info('monotonic') + self.context = disable_gc() + self.context.__enter__() + self.get_time = get_time + self.clock_info = clock_info + self.start_time = get_time() + return self + + def __exit__(self, *exc): + try: + end_time = self.get_time() + finally: + result = self.context.__exit__(*exc) + self.seconds = end_time - self.start_time + return result + + @contextlib.contextmanager def adjust_int_max_str_digits(max_digits): """Temporarily change the integer string conversion length limit.""" @@ -2389,43 +2590,39 @@ def adjust_int_max_str_digits(max_digits): sys.set_int_max_str_digits(current) -def _get_c_recursion_limit(): +def get_c_recursion_limit(): try: import _testcapi return _testcapi.Py_C_RECURSION_LIMIT - except (ImportError, AttributeError): - # Originally taken from Include/cpython/pystate.h . - if sys.platform == 'win32': - return 4000 - else: - return 10000 + except ImportError: + raise unittest.SkipTest('requires _testcapi') + -# The default C recursion limit. -Py_C_RECURSION_LIMIT = _get_c_recursion_limit() +def exceeds_recursion_limit(): + """For recursion tests, easily exceeds default recursion limit.""" + return get_c_recursion_limit() * 3 -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = Py_C_RECURSION_LIMIT * 3 -#Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') +# Windows doesn't have os.uname() but it doesn't support s390x. +is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' +skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') Py_TRACE_REFS = hasattr(sys, 'getobjects') # Decorator to disable optimizer while a function run def without_optimizer(func): try: - import _testinternalcapi + from _testinternalcapi import get_optimizer, set_optimizer except ImportError: return func @functools.wraps(func) def wrapper(*args, **kwargs): - save_opt = _testinternalcapi.get_optimizer() + save_opt = get_optimizer() try: - _testinternalcapi.set_optimizer(None) + set_optimizer(None) return func(*args, **kwargs) finally: - _testinternalcapi.set_optimizer(save_opt) + set_optimizer(save_opt) return wrapper @@ -2454,3 +2651,284 @@ def copy_python_src_ignore(path, names): 'build', } return ignored + + +# XXX Move this to the inspect module? +def walk_class_hierarchy(top, *, topdown=True): + # This is based on the logic in os.walk(). + assert isinstance(top, type), repr(top) + stack = [top] + while stack: + top = stack.pop() + if isinstance(top, tuple): + yield top + continue + + subs = type(top).__subclasses__(top) + if topdown: + # Yield before subclass traversal if going top down. + yield top, subs + # Traverse into subclasses. + for sub in reversed(subs): + stack.append(sub) + else: + # Yield after subclass traversal if going bottom up. + stack.append((top, subs)) + # Traverse into subclasses. + for sub in reversed(subs): + stack.append(sub) + + +def iter_builtin_types(): + # First try the explicit route. + try: + import _testinternalcapi + except ImportError: + _testinternalcapi = None + if _testinternalcapi is not None: + yield from _testinternalcapi.get_static_builtin_types() + return + + # Fall back to making a best-effort guess. + if hasattr(object, '__flags__'): + # Look for any type object with the Py_TPFLAGS_STATIC_BUILTIN flag set. + import datetime + seen = set() + for cls, subs in walk_class_hierarchy(object): + if cls in seen: + continue + seen.add(cls) + if not (cls.__flags__ & _TPFLAGS_STATIC_BUILTIN): + # Do not walk its subclasses. + subs[:] = [] + continue + yield cls + else: + # Fall back to a naive approach. + seen = set() + for obj in __builtins__.values(): + if not isinstance(obj, type): + continue + cls = obj + # XXX? + if cls.__module__ != 'builtins': + continue + if cls == ExceptionGroup: + # It's a heap type. + continue + if cls in seen: + continue + seen.add(cls) + yield cls + + +# XXX Move this to the inspect module? +def iter_name_in_mro(cls, name): + """Yield matching items found in base.__dict__ across the MRO. + + The descriptor protocol is not invoked. + + list(iter_name_in_mro(cls, name))[0] is roughly equivalent to + find_name_in_mro() in Objects/typeobject.c (AKA PyType_Lookup()). + + inspect.getattr_static() is similar. + """ + # This can fail if "cls" is weird. + for base in inspect._static_getmro(cls): + # This can fail if "base" is weird. + ns = inspect._get_dunder_dict_of_class(base) + try: + obj = ns[name] + except KeyError: + continue + yield obj, base + + +# XXX Move this to the inspect module? +def find_name_in_mro(cls, name, default=inspect._sentinel): + for res in iter_name_in_mro(cls, name): + # Return the first one. + return res + if default is not inspect._sentinel: + return default, None + raise AttributeError(name) + + +# XXX The return value should always be exactly the same... +def identify_type_slot_wrappers(): + try: + import _testinternalcapi + except ImportError: + _testinternalcapi = None + if _testinternalcapi is not None: + names = {n: None for n in _testinternalcapi.identify_type_slot_wrappers()} + return list(names) + else: + raise NotImplementedError + + +def iter_slot_wrappers(cls): + def is_slot_wrapper(name, value): + if not isinstance(value, types.WrapperDescriptorType): + assert not repr(value).startswith(' bool: + """ + Test whether the test suite is runing in systemd-nspawn + with ``--suppress-sync=true``. + + This can be used to skip tests that rely on ``fsync()`` calls + and similar not being intercepted. + """ + + if not hasattr(os, "O_SYNC"): + return False + + try: + with open("/run/systemd/container", "rb") as fp: + if fp.read().rstrip() != b"systemd-nspawn": + return False + except FileNotFoundError: + return False + + # If systemd-nspawn is used, O_SYNC flag will immediately + # trigger EINVAL. Otherwise, ENOENT will be given instead. + import errno + try: + fd = os.open(__file__, os.O_RDONLY | os.O_SYNC) + except OSError as err: + if err.errno == errno.EINVAL: + return True + else: + os.close(fd) + + return False diff --git a/Lib/test/support/asynchat.py b/Lib/test/support/asynchat.py index 38c47a1fda683e..a8c6b28a9e1a91 100644 --- a/Lib/test/support/asynchat.py +++ b/Lib/test/support/asynchat.py @@ -1,5 +1,5 @@ # TODO: This module was deprecated and removed from CPython 3.12 -# Now it is a test-only helper. Any attempts to rewrite exising tests that +# Now it is a test-only helper. Any attempts to rewrite existing tests that # are using this module and remove it completely are appreciated! # See: https://github.com/python/cpython/issues/72719 diff --git a/Lib/test/support/asyncore.py b/Lib/test/support/asyncore.py index b397aca5568079..870e42837640de 100644 --- a/Lib/test/support/asyncore.py +++ b/Lib/test/support/asyncore.py @@ -1,5 +1,5 @@ # TODO: This module was deprecated and removed from CPython 3.12 -# Now it is a test-only helper. Any attempts to rewrite exising tests that +# Now it is a test-only helper. Any attempts to rewrite existing tests that # are using this module and remove it completely are appreciated! # See: https://github.com/python/cpython/issues/72719 diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index a4845065a5322e..f6426c3e285b2d 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,7 +3,11 @@ import unittest import dis import io -from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object +import opcode +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None _UNSPECIFIED = object() @@ -65,16 +69,14 @@ class CompilationStepTestCase(unittest.TestCase): class Label: pass - def assertInstructionsMatch(self, actual_, expected_): - # get two lists where each entry is a label or - # an instruction tuple. Normalize the labels to the - # instruction count of the target, and compare the lists. + def assertInstructionsMatch(self, actual_seq, expected): + # get an InstructionSequence and an expected list, where each + # entry is a label or an instruction tuple. Construct an expected + # instruction sequence and compare with the one given. - self.assertIsInstance(actual_, list) - self.assertIsInstance(expected_, list) - - actual = self.normalize_insts(actual_) - expected = self.normalize_insts(expected_) + self.assertIsInstance(expected, list) + actual = actual_seq.get_instructions() + expected = self.seq_from_insts(expected).get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -84,10 +86,8 @@ def assertInstructionsMatch(self, actual_, expected_): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) - # crop comparison to the provided expected values - if len(act) > len(exp): - act = act[:len(exp)] - self.assertEqual(exp, act) + idx = max([p[0] for p in enumerate(exp) if p[1] != -1]) + self.assertEqual(exp[:idx], act[:idx]) def resolveAndRemoveLabels(self, insts): idx = 0 @@ -102,54 +102,57 @@ def resolveAndRemoveLabels(self, insts): return res - def normalize_insts(self, insts): - """ Map labels to instruction index. - Map opcodes to opnames. - """ - insts = self.resolveAndRemoveLabels(insts) - res = [] - for item in insts: - assert isinstance(item, tuple) - opcode, oparg, *loc = item - opcode = dis.opmap.get(opcode, opcode) - if isinstance(oparg, self.Label): - arg = oparg.value - else: - arg = oparg if opcode in self.HAS_ARG else None - opcode = dis.opname[opcode] - res.append((opcode, arg, *loc)) - return res + def seq_from_insts(self, insts): + labels = {item for item in insts if isinstance(item, self.Label)} + for i, lbl in enumerate(labels): + lbl.value = i - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] + seq = _testinternalcapi.new_instruction_sequence() for item in insts: - assert isinstance(item, tuple) - inst = list(item) - opcode = dis.opmap[inst[0]] - oparg = inst[1] - loc = inst[2:] + [-1] * (6 - len(inst)) - res.append((opcode, oparg, *loc)) - return res + if isinstance(item, self.Label): + seq.use_label(item.value) + else: + op = item[0] + if isinstance(op, str): + op = opcode.opmap[op] + arg, *loc = item[1:] + if isinstance(arg, self.Label): + arg = arg.value + loc = loc + [-1] * (4 - len(loc)) + seq.addop(op, arg or 0, *loc) + return seq + + def check_instructions(self, insts): + for inst in insts: + if isinstance(inst, self.Label): + continue + op, arg, *loc = inst + if isinstance(op, str): + op = opcode.opmap[op] + self.assertEqual(op in opcode.hasarg, + arg is not None, + f"{opcode.opname[op]=} {arg=}") + self.assertTrue(all(isinstance(l, int) for l in loc)) +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CodegenTestCase(CompilationStepTestCase): def generate_code(self, ast): - insts, _ = compiler_codegen(ast, "my_file.py", 0) + insts, _ = _testinternalcapi.compiler_codegen(ast, "my_file.py", 0) return insts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CfgOptimizationTestCase(CompilationStepTestCase): - def get_optimized(self, insts, consts, nlocals=0): - insts = self.normalize_insts(insts) - insts = self.complete_insts_info(insts) - insts = optimize_cfg(insts, consts, nlocals) + def get_optimized(self, seq, consts, nlocals=0): + insts = _testinternalcapi.optimize_cfg(seq, consts, nlocals) return insts, consts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class AssemblerTestCase(CompilationStepTestCase): def get_code_object(self, filename, insts, metadata): - co = assemble_code_object(filename, insts, metadata) + co = _testinternalcapi.assemble_code_object(filename, insts, metadata) return co diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py index db93eea5e912e0..40f58a2f59c6c3 100644 --- a/Lib/test/support/hypothesis_helper.py +++ b/Lib/test/support/hypothesis_helper.py @@ -5,6 +5,13 @@ except ImportError: from . import _hypothesis_stubs as hypothesis else: + # Regrtest changes to use a tempdir as the working directory, so we have + # to tell Hypothesis to use the original in order to persist the database. + from .os_helper import SAVEDCWD + from hypothesis.configuration import set_hypothesis_home_dir + + set_hypothesis_home_dir(os.path.join(SAVEDCWD, ".hypothesis")) + # When using the real Hypothesis, we'll configure it to ignore occasional # slow tests (avoiding flakiness from random VM slowness in CI). hypothesis.settings.register_profile( diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 3d804f2b590108..2b91bdcf9cd859 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -58,8 +58,8 @@ def make_legacy_pyc(source): :return: The file system path to the legacy pyc file. """ pyc_file = importlib.util.cache_from_source(source) - up_one = os.path.dirname(os.path.abspath(source)) - legacy_pyc = os.path.join(up_one, source + 'c') + assert source.endswith('.py') + legacy_pyc = source + 'c' shutil.move(pyc_file, legacy_pyc) return legacy_pyc @@ -114,7 +114,7 @@ def multi_interp_extensions_check(enabled=True): This only applies to modules that haven't been imported yet. It overrides the PyInterpreterConfig.check_multi_interp_extensions setting (see support.run_in_subinterp_with_config() and - _xxsubinterpreters.create()). + _interpreters.create()). Also see importlib.utils.allowing_all_extensions(). """ @@ -268,6 +268,18 @@ def modules_cleanup(oldmodules): sys.modules.update(oldmodules) +@contextlib.contextmanager +def isolated_modules(): + """ + Save modules on entry and cleanup on exit. + """ + (saved,) = modules_setup() + try: + yield + finally: + modules_cleanup(saved) + + def mock_register_at_fork(func): # bpo-30599: Mock os.register_at_fork() when importing the random module, # since this function doesn't allow to unregister callbacks and would leak diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 15a908e9663593..e067f259364d2a 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -2,11 +2,11 @@ import threading import weakref -import _xxsubinterpreters as _interpreters +import _interpreters # aliases: -from _xxsubinterpreters import ( - InterpreterError, InterpreterNotFoundError, +from _interpreters import ( + InterpreterError, InterpreterNotFoundError, NotShareableError, is_shareable, ) @@ -14,7 +14,8 @@ __all__ = [ 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', 'Interpreter', - 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed', + 'NotShareableError', 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', ] @@ -42,7 +43,11 @@ def __getattr__(name): {formatted} """.strip() -class ExecFailure(RuntimeError): +class ExecutionFailed(InterpreterError): + """An unhandled exception happened during execution. + + This is raised from Interpreter.exec() and Interpreter.call(). + """ def __init__(self, excinfo): msg = excinfo.formatted @@ -68,51 +73,78 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=True) - return Interpreter(id) + id = _interpreters.create(reqrefs=True) + return Interpreter(id, _ownsref=True) def list_all(): """Return all existing interpreters.""" - return [Interpreter(id) for id in _interpreters.list_all()] + return [Interpreter(id, _whence=whence) + for id, whence in _interpreters.list_all(require_ready=True)] def get_current(): """Return the currently running interpreter.""" - id = _interpreters.get_current() - return Interpreter(id) + id, whence = _interpreters.get_current() + return Interpreter(id, _whence=whence) def get_main(): """Return the main interpreter.""" - id = _interpreters.get_main() - return Interpreter(id) + id, whence = _interpreters.get_main() + assert whence == _interpreters.WHENCE_RUNTIME, repr(whence) + return Interpreter(id, _whence=whence) _known = weakref.WeakValueDictionary() class Interpreter: - """A single Python interpreter.""" + """A single Python interpreter. + + Attributes: + + "id" - the unique process-global ID number for the interpreter + "whence" - indicates where the interpreter was created + + If the interpreter wasn't created by this module + then any method that modifies the interpreter will fail, + i.e. .close(), .prepare_main(), .exec(), and .call() + """ + + _WHENCE_TO_STR = { + _interpreters.WHENCE_UNKNOWN: 'unknown', + _interpreters.WHENCE_RUNTIME: 'runtime init', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + _interpreters.WHENCE_STDLIB: '_interpreters module', + } - def __new__(cls, id, /): + def __new__(cls, id, /, _whence=None, _ownsref=None): # There is only one instance for any given ID. if not isinstance(id, int): raise TypeError(f'id must be an int, got {id!r}') id = int(id) + if _whence is None: + if _ownsref: + _whence = _interpreters.WHENCE_STDLIB + else: + _whence = _interpreters.whence(id) + assert _whence in cls._WHENCE_TO_STR, repr(_whence) + if _ownsref is None: + _ownsref = (_whence == _interpreters.WHENCE_STDLIB) try: self = _known[id] assert hasattr(self, '_ownsref') except KeyError: - # This may raise InterpreterNotFoundError: - _interpreters._incref(id) - try: - self = super().__new__(cls) - self._id = id - self._ownsref = True - except BaseException: - _interpreters._deccref(id) - raise + self = super().__new__(cls) _known[id] = self + self._id = id + self._whence = _whence + self._ownsref = _ownsref + if _ownsref: + # This may raise InterpreterNotFoundError: + _interpreters.incref(id) return self def __repr__(self): @@ -124,12 +156,20 @@ def __hash__(self): def __del__(self): self._decref() + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + def _decref(self): if not self._ownsref: return self._ownsref = False try: - _interpreters._decref(self.id) + _interpreters.decref(self._id) except InterpreterNotFoundError: pass @@ -137,17 +177,24 @@ def _decref(self): def id(self): return self._id + @property + def whence(self): + return self._WHENCE_TO_STR[self._whence] + def is_running(self): """Return whether or not the identified interpreter is running.""" return _interpreters.is_running(self._id) + # Everything past here is available only to interpreters created by + # interpreters.create(). + def close(self): """Finalize and destroy the interpreter. Attempting to destroy the current interpreter results - in a RuntimeError. + in an InterpreterError. """ - return _interpreters.destroy(self._id) + return _interpreters.destroy(self._id, restrict=True) def prepare_main(self, ns=None, /, **kwargs): """Bind the given values into the interpreter's __main__. @@ -155,9 +202,9 @@ def prepare_main(self, ns=None, /, **kwargs): The values must be shareable. """ ns = dict(ns, **kwargs) if ns is not None else kwargs - _interpreters.set___main___attrs(self._id, ns) + _interpreters.set___main___attrs(self._id, ns, restrict=True) - def exec_sync(self, code, /): + def exec(self, code, /): """Run the given source code in the interpreter. This is essentially the same as calling the builtin "exec" @@ -166,22 +213,46 @@ def exec_sync(self, code, /): There is no return value. - If the code raises an unhandled exception then an ExecFailure - is raised, which summarizes the unhandled exception. The actual - exception is discarded because objects cannot be shared between - interpreters. + If the code raises an unhandled exception then an ExecutionFailed + exception is raised, which summarizes the unhandled exception. + The actual exception is discarded because objects cannot be + shared between interpreters. This blocks the current Python thread until done. During that time, the previous interpreter is allowed to run in other threads. """ - excinfo = _interpreters.exec(self._id, code) + excinfo = _interpreters.exec(self._id, code, restrict=True) if excinfo is not None: - raise ExecFailure(excinfo) + raise ExecutionFailed(excinfo) + + def call(self, callable, /): + """Call the object in the interpreter with given args/kwargs. + + Only functions that take no arguments and have no closure + are supported. - def run(self, code, /): + The return value is discarded. + + If the callable raises an exception then the error display + (including full traceback) is send back between the interpreters + and an ExecutionFailed exception is raised, much like what + happens with Interpreter.exec(). + """ + # XXX Support args and kwargs. + # XXX Support arbitrary callables. + # XXX Support returning the return value (e.g. via pickle). + excinfo = _interpreters.call(self._id, callable, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def call_in_thread(self, callable, /): + """Return a new thread that calls the object in the interpreter. + + The return value and any raised exception are discarded. + """ def task(): - self.exec_sync(code) + self.call(callable) t = threading.Thread(target=task) t.start() return t diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/test/support/interpreters/_crossinterp.py new file mode 100644 index 00000000000000..544e197ba4c028 --- /dev/null +++ b/Lib/test/support/interpreters/_crossinterp.py @@ -0,0 +1,102 @@ +"""Common code between queues and channels.""" + + +class ItemInterpreterDestroyed(Exception): + """Raised when trying to get an item whose interpreter was destroyed.""" + + +class classonly: + """A non-data descriptor that makes a value only visible on the class. + + This is like the "classmethod" builtin, but does not show up on + instances of the class. It may be used as a decorator. + """ + + def __init__(self, value): + self.value = value + self.getter = classmethod(value).__get__ + self.name = None + + def __set_name__(self, cls, name): + if self.name is not None: + raise TypeError('already used') + self.name = name + + def __get__(self, obj, cls): + if obj is not None: + raise AttributeError(self.name) + # called on the class + return self.getter(None, cls) + + +class UnboundItem: + """Represents a cross-interpreter item no longer bound to an interpreter. + + An item is unbound when the interpreter that added it to the + cross-interpreter container is destroyed. + """ + + __slots__ = () + + @classonly + def singleton(cls, kind, module, name='UNBOUND'): + doc = cls.__doc__.replace('cross-interpreter container', kind) + doc = doc.replace('cross-interpreter', kind) + subclass = type( + f'Unbound{kind.capitalize()}Item', + (cls,), + dict( + _MODULE=module, + _NAME=name, + __doc__=doc, + ), + ) + return object.__new__(subclass) + + _MODULE = __name__ + _NAME = 'UNBOUND' + + def __new__(cls): + raise Exception(f'use {cls._MODULE}.{cls._NAME}') + + def __repr__(self): + return f'{self._MODULE}.{self._NAME}' +# return f'interpreters.queues.UNBOUND' + + +UNBOUND = object.__new__(UnboundItem) +UNBOUND_ERROR = object() +UNBOUND_REMOVE = object() + +_UNBOUND_CONSTANT_TO_FLAG = { + UNBOUND_REMOVE: 1, + UNBOUND_ERROR: 2, + UNBOUND: 3, +} +_UNBOUND_FLAG_TO_CONSTANT = {v: k + for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()} + + +def serialize_unbound(unbound): + op = unbound + try: + flag = _UNBOUND_CONSTANT_TO_FLAG[op] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {op!r}') + return flag, + + +def resolve_unbound(flag, exctype_destroyed): + try: + op = _UNBOUND_FLAG_TO_CONSTANT[flag] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {flag!r}') + if op is UNBOUND_REMOVE: + # "remove" not possible here + raise NotImplementedError + elif op is UNBOUND_ERROR: + raise exctype_destroyed("item's original interpreter destroyed") + elif op is UNBOUND: + return UNBOUND + else: + raise NotImplementedError(repr(op)) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index 75a5a60f54f926..d2bd93d77f7169 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -1,36 +1,69 @@ """Cross-interpreter Channels High Level Module.""" import time -import _xxinterpchannels as _channels +import _interpchannels as _channels +from . import _crossinterp # aliases: -from _xxinterpchannels import ( +from _interpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, ChannelEmptyError, ChannelNotEmptyError, ) +from ._crossinterp import ( + UNBOUND_ERROR, UNBOUND_REMOVE, +) __all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', 'create', 'list_all', 'SendChannel', 'RecvChannel', 'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError', + 'ItemInterpreterDestroyed', ] -def create(): +class ItemInterpreterDestroyed(ChannelError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(*, unbounditems=UNBOUND): """Return (recv, send) for a new cross-interpreter channel. The channel may be used to pass data safely between interpreters. + + "unbounditems" sets the default for the send end of the channel. + See SendChannel.send() for supported values. The default value + is UNBOUND, which replaces the unbound item when received. """ - cid = _channels.create() - recv, send = RecvChannel(cid), SendChannel(cid) + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + cid = _channels.create(unboundop) + recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound) return recv, send def list_all(): """Return a list of (recv, send) for all open channels.""" - return [(RecvChannel(cid), SendChannel(cid)) - for cid in _channels.list_all()] + return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound)) + for cid, unbound in _channels.list_all()] class _ChannelEnd: @@ -38,7 +71,8 @@ class _ChannelEnd: _end = None - def __init__(self, cid): + def __new__(cls, cid): + self = super().__new__(cls) if self._end == 'send': cid = _channels._channel_id(cid, send=True, force=True) elif self._end == 'recv': @@ -46,6 +80,7 @@ def __init__(self, cid): else: raise NotImplementedError(self._end) self._id = cid + return self def __repr__(self): return f'{type(self).__name__}(id={int(self._id)})' @@ -61,6 +96,14 @@ def __eq__(self, other): return NotImplemented return other._id == self._id + # for pickling: + def __getnewargs__(self): + return (int(self._id),) + + # for pickling: + def __getstate__(self): + return None + @property def id(self): return self._id @@ -96,12 +139,15 @@ def recv(self, timeout=None, *, if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout - obj = _channels.recv(self._id, _sentinel) + obj, unboundop = _channels.recv(self._id, _sentinel) while obj is _sentinel: time.sleep(_delay) if timeout is not None and time.time() >= end: raise TimeoutError - obj = _channels.recv(self._id, _sentinel) + obj, unboundop = _channels.recv(self._id, _sentinel) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) return obj def recv_nowait(self, default=_NOT_SET): @@ -112,9 +158,13 @@ def recv_nowait(self, default=_NOT_SET): is the same as recv(). """ if default is _NOT_SET: - return _channels.recv(self._id) + obj, unboundop = _channels.recv(self._id) else: - return _channels.recv(self._id, default) + obj, unboundop = _channels.recv(self._id, default) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + return obj def close(self): _channels.close(self._id, recv=True) @@ -125,43 +175,79 @@ class SendChannel(_ChannelEnd): _end = 'send' + def __new__(cls, cid, *, _unbound=None): + if _unbound is None: + try: + op = _channels.get_channel_defaults(cid) + _unbound = (op,) + except ChannelNotFoundError: + _unbound = _serialize_unbound(UNBOUND) + self = super().__new__(cls, cid) + self._unbound = _unbound + return self + @property def is_closed(self): info = self._info return info.closed or info.closing - def send(self, obj, timeout=None): + def send(self, obj, timeout=None, *, + unbound=None, + ): """Send the object (i.e. its data) to the channel's receiving end. This blocks until the object is received. """ - _channels.send(self._id, obj, timeout=timeout, blocking=True) + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True) - def send_nowait(self, obj): + def send_nowait(self, obj, *, + unbound=None, + ): """Send the object to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) # XXX Note that at the moment channel_send() only ever returns # None. This should be fixed when channel_send_wait() is added. # See bpo-32604 and gh-19829. - return _channels.send(self._id, obj, blocking=False) + return _channels.send(self._id, obj, unboundop, blocking=False) - def send_buffer(self, obj, timeout=None): + def send_buffer(self, obj, timeout=None, *, + unbound=None, + ): """Send the object's buffer to the channel's receiving end. This blocks until the object is received. """ - _channels.send_buffer(self._id, obj, timeout=timeout, blocking=True) + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + _channels.send_buffer(self._id, obj, unboundop, + timeout=timeout, blocking=True) - def send_buffer_nowait(self, obj): + def send_buffer_nowait(self, obj, *, + unbound=None, + ): """Send the object's buffer to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - return _channels.send_buffer(self._id, obj, blocking=False) + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + return _channels.send_buffer(self._id, obj, unboundop, blocking=False) def close(self): _channels.close(self._id, send=True) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index aead0c40ca9667..deb8e8613af731 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -1,50 +1,91 @@ """Cross-interpreter Queues High Level Module.""" +import pickle import queue import time import weakref -import _xxinterpqueues as _queues +import _interpqueues as _queues +from . import _crossinterp # aliases: -from _xxinterpqueues import ( +from _interpqueues import ( QueueError, QueueNotFoundError, ) +from ._crossinterp import ( + UNBOUND_ERROR, UNBOUND_REMOVE, +) __all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', 'create', 'list_all', 'Queue', 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', + 'ItemInterpreterDestroyed', ] -class QueueEmpty(_queues.QueueEmpty, queue.Empty): +class QueueEmpty(QueueError, queue.Empty): """Raised from get_nowait() when the queue is empty. It is also raised from get() if it times out. """ -class QueueFull(_queues.QueueFull, queue.Full): +class QueueFull(QueueError, queue.Full): """Raised from put_nowait() when the queue is full. It is also raised from put() if it times out. """ -def create(maxsize=0): +class ItemInterpreterDestroyed(QueueError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + +_SHARED_ONLY = 0 +_PICKLED = 1 + + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. + + "syncobj" sets the default for Queue.put() + and Queue.put_nowait(). + + "unbounditems" likewise sets the default. See Queue.put() for + supported values. The default value is UNBOUND, which replaces + the unbound item. """ - qid = _queues.create(maxsize) - return Queue(qid) + fmt = _SHARED_ONLY if syncobj else _PICKLED + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + qid = _queues.create(maxsize, fmt, unboundop) + return Queue(qid, _fmt=fmt, _unbound=unbound) def list_all(): """Return a list of all open queues.""" - return [Queue(qid) - for qid in _queues.list_all()] - + return [Queue(qid, _fmt=fmt, _unbound=(unboundop,)) + for qid, fmt, unboundop in _queues.list_all()] _known_queues = weakref.WeakValueDictionary() @@ -52,17 +93,28 @@ def list_all(): class Queue: """A cross-interpreter queue.""" - def __new__(cls, id, /): + def __new__(cls, id, /, *, _fmt=None, _unbound=None): # There is only one instance for any given ID. if isinstance(id, int): id = int(id) else: raise TypeError(f'id must be an int, got {id!r}') + if _fmt is None: + if _unbound is None: + _fmt, op = _queues.get_queue_defaults(id) + _unbound = (op,) + else: + _fmt, _ = _queues.get_queue_defaults(id) + elif _unbound is None: + _, op = _queues.get_queue_defaults(id) + _unbound = (op,) try: self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id + self._fmt = _fmt + self._unbound = _unbound _known_queues[id] = self _queues.bind(id) return self @@ -83,6 +135,14 @@ def __repr__(self): def __hash__(self): return hash(self._id) + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + @property def id(self): return self._id @@ -105,34 +165,96 @@ def qsize(self): return _queues.get_count(self._id) def put(self, obj, timeout=None, *, + syncobj=None, + unbound=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. This blocks while the queue is full. + + If "syncobj" is None (the default) then it uses the + queue's default, set with create_queue(). + + If "syncobj" is false then all objects are supported, + at the expense of worse performance. + + If "syncobj" is true then the object must be "shareable". + Examples of "shareable" objects include the builtin singletons, + str, and memoryview. One benefit is that such objects are + passed through the queue efficiently. + + The key difference, though, is conceptual: the corresponding + object returned from Queue.get() will be strictly equivalent + to the given obj. In other words, the two objects will be + effectively indistinguishable from each other, even if the + object is mutable. The received object may actually be the + same object, or a copy (immutable values only), or a proxy. + Regardless, the received object should be treated as though + the original has been shared directly, whether or not it + actually is. That's a slightly different and stronger promise + than just (initial) equality, which is all "syncobj=False" + can promise. + + "unbound" controls the behavior of Queue.get() for the given + object if the current interpreter (calling put()) is later + destroyed. + + If "unbound" is None (the default) then it uses the + queue's default, set with create_queue(), + which is usually UNBOUND. + + If "unbound" is UNBOUND_ERROR then get() will raise an + ItemInterpreterDestroyed exception if the original interpreter + has been destroyed. This does not otherwise affect the queue; + the next call to put() will work like normal, returning the next + item in the queue. + + If "unbound" is UNBOUND_REMOVE then the item will be removed + from the queue as soon as the original interpreter is destroyed. + Be aware that this will introduce an imbalance between put() + and get() calls. + + If "unbound" is UNBOUND then it is returned by get() in place + of the unbound item. """ + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) if timeout is not None: timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout + if fmt is _PICKLED: + obj = pickle.dumps(obj) while True: try: - _queues.put(self._id, obj) - except _queues.QueueFull as exc: + _queues.put(self._id, obj, fmt, unboundop) + except QueueFull as exc: if timeout is not None and time.time() >= end: - exc.__class__ = QueueFull raise # re-raise time.sleep(_delay) else: break - def put_nowait(self, obj): - try: - return _queues.put(self._id, obj) - except _queues.QueueFull as exc: - exc.__class__ = QueueFull - raise # re-raise + def put_nowait(self, obj, *, syncobj=None, unbound=None): + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + if fmt is _PICKLED: + obj = pickle.dumps(obj) + _queues.put(self._id, obj, fmt, unboundop) def get(self, timeout=None, *, _delay=10 / 1000, # 10 milliseconds @@ -140,6 +262,10 @@ def get(self, timeout=None, *, """Return the next object from the queue. This blocks while the queue is empty. + + If the next item's original interpreter has been destroyed + then the "next object" is determined by the value of the + "unbound" argument to put(). """ if timeout is not None: timeout = int(timeout) @@ -148,12 +274,20 @@ def get(self, timeout=None, *, end = time.time() + timeout while True: try: - return _queues.get(self._id) - except _queues.QueueEmpty as exc: + obj, fmt, unboundop = _queues.get(self._id) + except QueueEmpty as exc: if timeout is not None and time.time() >= end: - exc.__class__ = QueueEmpty raise # re-raise time.sleep(_delay) + else: + break + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY return obj def get_nowait(self): @@ -163,10 +297,17 @@ def get_nowait(self): is the same as get(). """ try: - return _queues.get(self._id) - except _queues.QueueEmpty as exc: - exc.__class__ = QueueEmpty + obj, fmt, unboundop = _queues.get(self._id) + except QueueEmpty as exc: raise # re-raise + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY + return obj -_queues._register_queue_type(Queue) +_queues._register_heap_types(Queue, QueueEmpty, QueueFull) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 22787e32b5f3ab..891405943b78c5 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -198,6 +198,23 @@ def skip_unless_symlink(test): return test if ok else unittest.skip(msg)(test) +_can_hardlink = None + +def can_hardlink(): + global _can_hardlink + if _can_hardlink is None: + # Android blocks hard links using SELinux + # (https://stackoverflow.com/q/32365690). + _can_hardlink = hasattr(os, "link") and not support.is_android + return _can_hardlink + + +def skip_unless_hardlink(test): + ok = can_hardlink() + msg = "requires hardlink support" + return test if ok else unittest.skip(msg)(test) + + _can_xattr = None @@ -595,7 +612,7 @@ def __fspath__(self): def fd_count(): """Count the number of open file descriptors. """ - if sys.platform.startswith(('linux', 'freebsd', 'emscripten')): + if sys.platform.startswith(('linux', 'android', 'freebsd', 'emscripten')): fd_path = "/proc/self/fd" elif sys.platform == "darwin": fd_path = "/dev/fd" @@ -615,7 +632,8 @@ def fd_count(): if hasattr(os, 'sysconf'): try: MAXFD = os.sysconf("SC_OPEN_MAX") - except OSError: + except (OSError, ValueError): + # gh-118201: ValueError is raised intermittently on iOS pass old_modes = None diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index 759020c33aa700..46ce950433ddf6 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -63,30 +63,32 @@ class _PythonRunResult(collections.namedtuple("_PythonRunResult", """Helper for reporting Python subprocess run results""" def fail(self, cmd_line): """Provide helpful details about failed subcommand runs""" - # Limit to 80 lines to ASCII characters - maxlen = 80 * 100 + # Limit to 300 lines of ASCII characters + maxlen = 300 * 100 out, err = self.out, self.err if len(out) > maxlen: out = b'(... truncated stdout ...)' + out[-maxlen:] if len(err) > maxlen: err = b'(... truncated stderr ...)' + err[-maxlen:] - out = out.decode('ascii', 'replace').rstrip() - err = err.decode('ascii', 'replace').rstrip() - raise AssertionError("Process return code is %d\n" - "command line: %r\n" - "\n" - "stdout:\n" - "---\n" - "%s\n" - "---\n" - "\n" - "stderr:\n" - "---\n" - "%s\n" - "---" - % (self.rc, cmd_line, - out, - err)) + out = out.decode('utf8', 'replace').rstrip() + err = err.decode('utf8', 'replace').rstrip() + + exitcode = self.rc + signame = support.get_signal_name(exitcode) + if signame: + exitcode = f"{exitcode} ({signame})" + raise AssertionError(f"Process return code is {exitcode}\n" + f"command line: {cmd_line!r}\n" + f"\n" + f"stdout:\n" + f"---\n" + f"{out}\n" + f"---\n" + f"\n" + f"stderr:\n" + f"---\n" + f"{err}\n" + f"---") # Executing the interpreter in a subprocess @@ -232,9 +234,13 @@ def make_script(script_dir, script_basename, source, omit_suffix=False): if not omit_suffix: script_filename += os.extsep + 'py' script_name = os.path.join(script_dir, script_filename) - # The script should be encoded to UTF-8, the default string encoding - with open(script_name, 'w', encoding='utf-8') as script_file: - script_file.write(source) + if isinstance(source, str): + # The script should be encoded to UTF-8, the default string encoding + with open(script_name, 'w', encoding='utf-8') as script_file: + script_file.write(source) + else: + with open(script_name, 'wb') as script_file: + script_file.write(source) importlib.invalidate_caches() return script_name diff --git a/Lib/test/support/smtpd.py b/Lib/test/support/smtpd.py index 6052232ec2b585..6537679db9ad24 100755 --- a/Lib/test/support/smtpd.py +++ b/Lib/test/support/smtpd.py @@ -7,7 +7,7 @@ --nosetuid -n - This program generally tries to setuid `nobody', unless this flag is + This program generally tries to setuid 'nobody', unless this flag is set. The setuid call will fail if this program is not run as root (in which case, use this flag). @@ -17,7 +17,7 @@ --class classname -c classname - Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by + Use 'classname' as the concrete SMTP proxy class. Uses 'PureProxy' by default. --size limit @@ -39,8 +39,8 @@ Version: %(__version__)s -If localhost is not given then `localhost' is used, and if localport is not -given then 8025 is used. If remotehost is not given then `localhost' is used, +If localhost is not given then 'localhost' is used, and if localport is not +given then 8025 is used. If remotehost is not given then 'localhost' is used, and if remoteport is not given, then 25 is used. """ @@ -633,7 +633,8 @@ def __init__(self, localaddr, remoteaddr, " be set to True at the same time") asyncore.dispatcher.__init__(self, map=map) try: - gai_results = socket.getaddrinfo(*localaddr, + family = 0 if socket.has_ipv6 else socket.AF_INET + gai_results = socket.getaddrinfo(*localaddr, family=family, type=socket.SOCK_STREAM) self.create_socket(gai_results[0][0], gai_results[0][1]) # try to re-use a server port if possible @@ -672,9 +673,9 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): message to. data is a string containing the entire full text of the message, - headers (if supplied) and all. It has been `de-transparencied' + headers (if supplied) and all. It has been 'de-transparencied' according to RFC 821, Section 4.5.2. In other words, a line - containing a `.' followed by other text has had the leading dot + containing a '.' followed by other text has had the leading dot removed. kwargs is a dictionary containing additional information. It is @@ -685,7 +686,7 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): ['BODY=8BITMIME', 'SMTPUTF8']. 'rcpt_options': same, for the rcpt command. - This function should return None for a normal `250 Ok' response; + This function should return None for a normal '250 Ok' response; otherwise, it should return the desired response string in RFC 821 format. diff --git a/Lib/test/support/testcase.py b/Lib/test/support/testcase.py index 1e4363b15783eb..fad1e4cb3499c0 100644 --- a/Lib/test/support/testcase.py +++ b/Lib/test/support/testcase.py @@ -1,3 +1,6 @@ +from math import copysign, isnan + + class ExceptionIsLikeMixin: def assertExceptionIsLike(self, exc, template): """ @@ -23,3 +26,40 @@ def assertExceptionIsLike(self, exc, template): self.assertEqual(len(exc.exceptions), len(template.exceptions)) for e, t in zip(exc.exceptions, template.exceptions): self.assertExceptionIsLike(e, t) + + +class FloatsAreIdenticalMixin: + def assertFloatsAreIdentical(self, x, y): + """Fail unless floats x and y are identical, in the sense that: + (1) both x and y are nans, or + (2) both x and y are infinities, with the same sign, or + (3) both x and y are zeros, with the same sign, or + (4) x and y are both finite and nonzero, and x == y + + """ + msg = 'floats {!r} and {!r} are not identical' + + if isnan(x) or isnan(y): + if isnan(x) and isnan(y): + return + elif x == y: + if x != 0.0: + return + # both zero; check that signs match + elif copysign(1.0, x) == copysign(1.0, y): + return + else: + msg += ': zeros have different signs' + self.fail(msg.format(x, y)) + + +class ComplexesAreIdenticalMixin(FloatsAreIdenticalMixin): + def assertComplexesAreIdentical(self, x, y): + """Fail unless complex numbers x and y have equal values and signs. + + In particular, if x and y both have real (or imaginary) part + zero, but the zeros have different signs, this test will fail. + + """ + self.assertFloatsAreIdentical(x.real, y.real) + self.assertFloatsAreIdentical(x.imag, y.imag) diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index c87cde4b3d1fab..e405056c8ffcb5 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -3,12 +3,6 @@ from test.support import warnings_helper import os import sys -import types - -try: - import _multiprocessing -except ModuleNotFoundError: - _multiprocessing = None if support.check_sanitizer(address=True, memory=True): @@ -36,17 +30,6 @@ class FailedImport(RuntimeError): class AllTest(unittest.TestCase): - def setUp(self): - # concurrent.futures uses a __getattr__ hook. Its __all__ triggers - # import of a submodule, which fails when _multiprocessing is not - # available. - if _multiprocessing is None: - sys.modules["_multiprocessing"] = types.ModuleType("_multiprocessing") - - def tearDown(self): - if _multiprocessing is None: - sys.modules.pop("_multiprocessing") - def check_all(self, modname): names = {} with warnings_helper.check_warnings( @@ -120,7 +103,7 @@ def test_all(self): # In case _socket fails to build, make this test fail more gracefully # than an AttributeError somewhere deep in concurrent.futures, email # or unittest. - import _socket + import _socket # noqa: F401 ignored = [] failed_imports = [] diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py new file mode 100644 index 00000000000000..d55b97ade68cef --- /dev/null +++ b/Lib/test/test__colorize.py @@ -0,0 +1,59 @@ +import contextlib +import sys +import unittest +import unittest.mock +import _colorize +from test.support import force_not_colorized + +ORIGINAL_CAN_COLORIZE = _colorize.can_colorize + + +def setUpModule(): + _colorize.can_colorize = lambda: False + + +def tearDownModule(): + _colorize.can_colorize = ORIGINAL_CAN_COLORIZE + + +class TestColorizeFunction(unittest.TestCase): + @force_not_colorized + def test_colorized_detection_checks_for_environment_variables(self): + if sys.platform == "win32": + virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", + return_value=True) + else: + virtual_patching = contextlib.nullcontext() + with virtual_patching: + + flags = unittest.mock.MagicMock(ignore_environment=False) + with (unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.flags", flags), + unittest.mock.patch("_colorize.can_colorize", ORIGINAL_CAN_COLORIZE)): + isatty_mock.return_value = True + with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): + self.assertEqual(_colorize.can_colorize(), False) + isatty_mock.return_value = False + with unittest.mock.patch("os.environ", {}): + self.assertEqual(_colorize.can_colorize(), False) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__interpchannels.py similarity index 62% rename from Lib/test/test__xxinterpchannels.py rename to Lib/test/test__interpchannels.py index cc2ed7849b0c0f..4a7f04b9df9843 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -8,14 +8,21 @@ from test.support import import_helper -from test.test__xxsubinterpreters import ( - interpreters, +_channels = import_helper.import_module('_interpchannels') +from test.support.interpreters import _crossinterp +from test.test__interpreters import ( + _interpreters, _run_output, clean_up_interpreters, ) -channels = import_helper.import_module('_xxinterpchannels') +REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND] + + +# Additional tests are found in Lib/test/test_interpreters/test_channels.py. +# New tests should be added there. +# XXX The tests here should be moved there. See the note under LowLevelTests. ################################## @@ -24,9 +31,19 @@ def recv_wait(cid): while True: try: - return channels.recv(cid) - except channels.ChannelEmptyError: + obj, unboundop = _channels.recv(cid) + except _channels.ChannelEmptyError: time.sleep(0.1) + else: + assert unboundop is None, repr(unboundop) + return obj + + +def recv_nowait(cid, *args, unbound=False): + obj, unboundop = _channels.recv(cid, *args) + assert (unboundop is None) != unbound, repr(unboundop) + return obj + #@contextmanager #def run_threaded(id, source, **shared): @@ -44,14 +61,15 @@ def run_interp(id, source, **shared): def _run_interp(id, source, shared, _mainns={}): source = dedent(source) - main = interpreters.get_main() + main, *_ = _interpreters.get_main() if main == id: - if interpreters.get_current() != main: + cur, *_ = _interpreters.get_current() + if cur != main: raise RuntimeError # XXX Run a func? exec(source, _mainns) else: - interpreters.run_string(id, source, shared) + _interpreters.run_string(id, source, shared) class Interpreter(namedtuple('Interpreter', 'name id')): @@ -66,7 +84,7 @@ def from_raw(cls, raw): raise NotImplementedError def __new__(cls, name=None, id=None): - main = interpreters.get_main() + main, *_ = _interpreters.get_main() if id == main: if not name: name = 'main' @@ -84,7 +102,7 @@ def __new__(cls, name=None, id=None): name = 'main' id = main else: - id = interpreters.create() + id = _interpreters.create() self = super().__new__(cls, name, id) return self @@ -95,7 +113,7 @@ def __new__(cls, name=None, id=None): def expect_channel_closed(): try: yield - except channels.ChannelClosedError: + except _channels.ChannelClosedError: pass else: assert False, 'channel not closed' @@ -182,7 +200,7 @@ def run_action(cid, action, end, state, *, hideclosed=True): try: result = _run_action(cid, action, end, state) - except channels.ChannelClosedError: + except _channels.ChannelClosedError: if not hideclosed and not expectfail: raise result = state.close() @@ -195,18 +213,18 @@ def run_action(cid, action, end, state, *, hideclosed=True): def _run_action(cid, action, end, state): if action == 'use': if end == 'send': - channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) return state.incr() elif end == 'recv': if not state.pending: try: - channels.recv(cid) - except channels.ChannelEmptyError: + _channels.recv(cid) + except _channels.ChannelEmptyError: return state else: raise Exception('expected ChannelEmptyError') else: - channels.recv(cid) + recv_nowait(cid) return state.decr() else: raise ValueError(end) @@ -214,7 +232,7 @@ def _run_action(cid, action, end, state): kwargs = {} if end in ('recv', 'send'): kwargs[end] = True - channels.close(cid, **kwargs) + _channels.close(cid, **kwargs) return state.close() elif action == 'force-close': kwargs = { @@ -222,17 +240,17 @@ def _run_action(cid, action, end, state): } if end in ('recv', 'send'): kwargs[end] = True - channels.close(cid, **kwargs) + _channels.close(cid, **kwargs) return state.close(force=True) else: raise ValueError(action) def clean_up_channels(): - for cid in channels.list_all(): + for cid, _ in _channels.list_all(): try: - channels.destroy(cid) - except channels.ChannelNotFoundError: + _channels.destroy(cid) + except _channels.ChannelNotFoundError: pass # already destroyed @@ -249,25 +267,25 @@ def tearDown(self): class ChannelIDTests(TestBase): def test_default_kwargs(self): - cid = channels._channel_id(10, force=True) + cid = _channels._channel_id(10, force=True) self.assertEqual(int(cid), 10) self.assertEqual(cid.end, 'both') def test_with_kwargs(self): - cid = channels._channel_id(10, send=True, force=True) + cid = _channels._channel_id(10, send=True, force=True) self.assertEqual(cid.end, 'send') - cid = channels._channel_id(10, send=True, recv=False, force=True) + cid = _channels._channel_id(10, send=True, recv=False, force=True) self.assertEqual(cid.end, 'send') - cid = channels._channel_id(10, recv=True, force=True) + cid = _channels._channel_id(10, recv=True, force=True) self.assertEqual(cid.end, 'recv') - cid = channels._channel_id(10, recv=True, send=False, force=True) + cid = _channels._channel_id(10, recv=True, send=False, force=True) self.assertEqual(cid.end, 'recv') - cid = channels._channel_id(10, send=True, recv=True, force=True) + cid = _channels._channel_id(10, send=True, recv=True, force=True) self.assertEqual(cid.end, 'both') def test_coerce_id(self): @@ -275,47 +293,47 @@ class Int(str): def __index__(self): return 10 - cid = channels._channel_id(Int(), force=True) + cid = _channels._channel_id(Int(), force=True) self.assertEqual(int(cid), 10) def test_bad_id(self): - self.assertRaises(TypeError, channels._channel_id, object()) - self.assertRaises(TypeError, channels._channel_id, 10.0) - self.assertRaises(TypeError, channels._channel_id, '10') - self.assertRaises(TypeError, channels._channel_id, b'10') - self.assertRaises(ValueError, channels._channel_id, -1) - self.assertRaises(OverflowError, channels._channel_id, 2**64) + self.assertRaises(TypeError, _channels._channel_id, object()) + self.assertRaises(TypeError, _channels._channel_id, 10.0) + self.assertRaises(TypeError, _channels._channel_id, '10') + self.assertRaises(TypeError, _channels._channel_id, b'10') + self.assertRaises(ValueError, _channels._channel_id, -1) + self.assertRaises(OverflowError, _channels._channel_id, 2**64) def test_bad_kwargs(self): with self.assertRaises(ValueError): - channels._channel_id(10, send=False, recv=False) + _channels._channel_id(10, send=False, recv=False) def test_does_not_exist(self): - cid = channels.create() - with self.assertRaises(channels.ChannelNotFoundError): - channels._channel_id(int(cid) + 1) # unforced + cid = _channels.create(REPLACE) + with self.assertRaises(_channels.ChannelNotFoundError): + _channels._channel_id(int(cid) + 1) # unforced def test_str(self): - cid = channels._channel_id(10, force=True) + cid = _channels._channel_id(10, force=True) self.assertEqual(str(cid), '10') def test_repr(self): - cid = channels._channel_id(10, force=True) + cid = _channels._channel_id(10, force=True) self.assertEqual(repr(cid), 'ChannelID(10)') - cid = channels._channel_id(10, send=True, force=True) + cid = _channels._channel_id(10, send=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10, send=True)') - cid = channels._channel_id(10, recv=True, force=True) + cid = _channels._channel_id(10, recv=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') - cid = channels._channel_id(10, send=True, recv=True, force=True) + cid = _channels._channel_id(10, send=True, recv=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10)') def test_equality(self): - cid1 = channels.create() - cid2 = channels._channel_id(int(cid1)) - cid3 = channels.create() + cid1 = _channels.create(REPLACE) + cid2 = _channels._channel_id(int(cid1)) + cid3 = _channels.create(REPLACE) self.assertTrue(cid1 == cid1) self.assertTrue(cid1 == cid2) @@ -335,11 +353,11 @@ def test_equality(self): self.assertTrue(cid1 != cid3) def test_shareable(self): - chan = channels.create() + chan = _channels.create(REPLACE) - obj = channels.create() - channels.send(chan, obj, blocking=False) - got = channels.recv(chan) + obj = _channels.create(REPLACE) + _channels.send(chan, obj, blocking=False) + got = recv_nowait(chan) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -350,33 +368,33 @@ def test_shareable(self): class ChannelTests(TestBase): def test_create_cid(self): - cid = channels.create() - self.assertIsInstance(cid, channels.ChannelID) + cid = _channels.create(REPLACE) + self.assertIsInstance(cid, _channels.ChannelID) def test_sequential_ids(self): - before = channels.list_all() - id1 = channels.create() - id2 = channels.create() - id3 = channels.create() - after = channels.list_all() + before = [cid for cid, _ in _channels.list_all()] + id1 = _channels.create(REPLACE) + id2 = _channels.create(REPLACE) + id3 = _channels.create(REPLACE) + after = [cid for cid, _ in _channels.list_all()] self.assertEqual(id2, int(id1) + 1) self.assertEqual(id3, int(id2) + 1) self.assertEqual(set(after) - set(before), {id1, id2, id3}) def test_ids_global(self): - id1 = interpreters.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(""" - import _xxinterpchannels as _channels - cid = _channels.create() + import _interpchannels as _channels + cid = _channels.create(3) print(cid) """)) cid1 = int(out.strip()) - id2 = interpreters.create() + id2 = _interpreters.create() out = _run_output(id2, dedent(""" - import _xxinterpchannels as _channels - cid = _channels.create() + import _interpchannels as _channels + cid = _channels.create(3) print(cid) """)) cid2 = int(out.strip()) @@ -385,81 +403,81 @@ def test_ids_global(self): def test_channel_list_interpreters_none(self): """Test listing interpreters for a channel with no associations.""" - # Test for channel with no associated interpreters. - cid = channels.create() - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + # Test for channel with no associated _interpreters. + cid = _channels.create(REPLACE) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, []) self.assertEqual(recv_interps, []) def test_channel_list_interpreters_basic(self): - """Test basic listing channel interpreters.""" - interp0 = interpreters.get_main() - cid = channels.create() - channels.send(cid, "send", blocking=False) + """Test basic listing channel _interpreters.""" + interp0, *_ = _interpreters.get_main() + cid = _channels.create(REPLACE) + _channels.send(cid, "send", blocking=False) # Test for a channel that has one end associated to an interpreter. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, []) - interp1 = interpreters.create() + interp1 = _interpreters.create() _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) # Test for channel that has both ends associated to an interpreter. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, [interp1]) def test_channel_list_interpreters_multiple(self): """Test listing interpreters for a channel with many associations.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - cid = channels.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + interp2 = _interpreters.create() + interp3 = _interpreters.create() + cid = _channels.create(REPLACE) - channels.send(cid, "send", blocking=False) + _channels.send(cid, "send", blocking=False) _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.send({cid}, "send", blocking=False) """)) _run_output(interp2, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) _run_output(interp3, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(set(send_interps), {interp0, interp1}) self.assertEqual(set(recv_interps), {interp2, interp3}) def test_channel_list_interpreters_destroyed(self): """Test listing channel interpreters with a destroyed interpreter.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - cid = channels.create() - channels.send(cid, "send", blocking=False) + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + cid = _channels.create(REPLACE) + _channels.send(cid, "send", blocking=False) _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) # Should be one interpreter associated with each end. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, [interp1]) - interpreters.destroy(interp1) + _interpreters.destroy(interp1) # Destroyed interpreter should not be listed. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, []) @@ -467,104 +485,104 @@ def test_channel_list_interpreters_released(self): """Test listing channel interpreters with a released channel.""" # Set up one channel with main interpreter on the send end and two # subinterpreters on the receive end. - interp0 = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - cid = channels.create() - channels.send(cid, "data", blocking=False) + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + interp2 = _interpreters.create() + cid = _channels.create(REPLACE) + _channels.send(cid, "data", blocking=False) _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) - channels.send(cid, "data", blocking=False) + _channels.send(cid, "data", blocking=False) _run_output(interp2, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) # Check the setup. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 1) self.assertEqual(len(recv_interps), 2) # Release the main interpreter from the send end. - channels.release(cid, send=True) - # Send end should have no associated interpreters. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + _channels.release(cid, send=True) + # Send end should have no associated _interpreters. + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 0) self.assertEqual(len(recv_interps), 2) # Release one of the subinterpreters from the receive end. _run_output(interp2, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.release({cid}) """)) # Receive end should have the released interpreter removed. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 0) self.assertEqual(recv_interps, [interp1]) def test_channel_list_interpreters_closed(self): """Test listing channel interpreters with a closed channel.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - cid = channels.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + cid = _channels.create(REPLACE) # Put something in the channel so that it's not empty. - channels.send(cid, "send", blocking=False) + _channels.send(cid, "send", blocking=False) # Check initial state. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 1) self.assertEqual(len(recv_interps), 0) # Force close the channel. - channels.close(cid, force=True) + _channels.close(cid, force=True) # Both ends should raise an error. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=False) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=False) def test_channel_list_interpreters_closed_send_end(self): """Test listing channel interpreters with a channel's send end closed.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - cid = channels.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + cid = _channels.create(REPLACE) # Put something in the channel so that it's not empty. - channels.send(cid, "send", blocking=False) + _channels.send(cid, "send", blocking=False) # Check initial state. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 1) self.assertEqual(len(recv_interps), 0) # Close the send end of the channel. - channels.close(cid, send=True) + _channels.close(cid, send=True) # Send end should raise an error. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) # Receive end should not be closed (since channel is not empty). - recv_interps = channels.list_interpreters(cid, send=False) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(recv_interps), 0) # Close the receive end of the channel from a subinterpreter. _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.close({cid}, force=True) """)) return # Both ends should raise an error. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=False) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=False) def test_allowed_types(self): - cid = channels.create() + cid = _channels.create(REPLACE) objects = [ None, 'spam', @@ -573,8 +591,8 @@ def test_allowed_types(self): ] for obj in objects: with self.subTest(obj): - channels.send(cid, obj, blocking=False) - got = channels.recv(cid) + _channels.send(cid, obj, blocking=False) + got = recv_nowait(cid) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -583,16 +601,16 @@ def test_allowed_types(self): # XXX What about between interpreters? def test_run_string_arg_unresolved(self): - cid = channels.create() - interp = interpreters.create() + cid = _channels.create(REPLACE) + interp = _interpreters.create() - interpreters.set___main___attrs(interp, dict(cid=cid.send)) + _interpreters.set___main___attrs(interp, dict(cid=cid.send)) out = _run_output(interp, dedent(""" - import _xxinterpchannels as _channels + import _interpchannels as _channels print(cid.end) _channels.send(cid, b'spam', blocking=False) """)) - obj = channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') self.assertEqual(out.strip(), 'send') @@ -602,17 +620,17 @@ def test_run_string_arg_unresolved(self): # Note: this test caused crashes on some buildbots (bpo-33615). @unittest.skip('disabled until high-level channels exist') def test_run_string_arg_resolved(self): - cid = channels.create() - cid = channels._channel_id(cid, _resolve=True) - interp = interpreters.create() + cid = _channels.create(REPLACE) + cid = _channels._channel_id(cid, _resolve=True) + interp = _interpreters.create() out = _run_output(interp, dedent(""" - import _xxinterpchannels as _channels + import _interpchannels as _channels print(chan.id.end) _channels.send(chan.id, b'spam', blocking=False) """), dict(chan=cid.send)) - obj = channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') self.assertEqual(out.strip(), 'send') @@ -621,65 +639,65 @@ def test_run_string_arg_resolved(self): # send/recv def test_send_recv_main(self): - cid = channels.create() + cid = _channels.create(REPLACE) orig = b'spam' - channels.send(cid, orig, blocking=False) - obj = channels.recv(cid) + _channels.send(cid, orig, blocking=False) + obj = recv_nowait(cid) self.assertEqual(obj, orig) self.assertIsNot(obj, orig) def test_send_recv_same_interpreter(self): - id1 = interpreters.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(""" - import _xxinterpchannels as _channels - cid = _channels.create() + import _interpchannels as _channels + cid = _channels.create(REPLACE) orig = b'spam' _channels.send(cid, orig, blocking=False) - obj = _channels.recv(cid) + obj, _ = _channels.recv(cid) assert obj is not orig assert obj == orig """)) def test_send_recv_different_interpreters(self): - cid = channels.create() - id1 = interpreters.create() + cid = _channels.create(REPLACE) + id1 = _interpreters.create() out = _run_output(id1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) - obj = channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') def test_send_recv_different_threads(self): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): obj = recv_wait(cid) - channels.send(cid, obj) + _channels.send(cid, obj) t = threading.Thread(target=f) t.start() - channels.send(cid, b'spam') + _channels.send(cid, b'spam') obj = recv_wait(cid) t.join() self.assertEqual(obj, b'spam') def test_send_recv_different_interpreters_and_threads(self): - cid = channels.create() - id1 = interpreters.create() + cid = _channels.create(REPLACE) + id1 = _interpreters.create() out = None def f(): nonlocal out out = _run_output(id1, dedent(f""" import time - import _xxinterpchannels as _channels + import _interpchannels as _channels while True: try: - obj = _channels.recv({cid}) + obj, _ = _channels.recv({cid}) break except _channels.ChannelEmptyError: time.sleep(0.1) @@ -689,38 +707,38 @@ def f(): t = threading.Thread(target=f) t.start() - channels.send(cid, b'spam') + _channels.send(cid, b'spam') obj = recv_wait(cid) t.join() self.assertEqual(obj, b'eggs') def test_send_not_found(self): - with self.assertRaises(channels.ChannelNotFoundError): - channels.send(10, b'spam') + with self.assertRaises(_channels.ChannelNotFoundError): + _channels.send(10, b'spam') def test_recv_not_found(self): - with self.assertRaises(channels.ChannelNotFoundError): - channels.recv(10) + with self.assertRaises(_channels.ChannelNotFoundError): + _channels.recv(10) def test_recv_empty(self): - cid = channels.create() - with self.assertRaises(channels.ChannelEmptyError): - channels.recv(cid) + cid = _channels.create(REPLACE) + with self.assertRaises(_channels.ChannelEmptyError): + _channels.recv(cid) def test_recv_default(self): default = object() - cid = channels.create() - obj1 = channels.recv(cid, default) - channels.send(cid, None, blocking=False) - channels.send(cid, 1, blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'eggs', blocking=False) - obj2 = channels.recv(cid, default) - obj3 = channels.recv(cid, default) - obj4 = channels.recv(cid) - obj5 = channels.recv(cid, default) - obj6 = channels.recv(cid, default) + cid = _channels.create(REPLACE) + obj1 = recv_nowait(cid, default) + _channels.send(cid, None, blocking=False) + _channels.send(cid, 1, blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'eggs', blocking=False) + obj2 = recv_nowait(cid, default) + obj3 = recv_nowait(cid, default) + obj4 = recv_nowait(cid) + obj5 = recv_nowait(cid, default) + obj6 = recv_nowait(cid, default) self.assertIs(obj1, default) self.assertIs(obj2, None) @@ -731,32 +749,33 @@ def test_recv_default(self): def test_recv_sending_interp_destroyed(self): with self.subTest('closed'): - cid1 = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid1 = _channels.create(REPLACE) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.send({cid1}, b'spam', blocking=False) """)) - interpreters.destroy(interp) + _interpreters.destroy(interp) with self.assertRaisesRegex(RuntimeError, f'channel {cid1} is closed'): - channels.recv(cid1) + _channels.recv(cid1) del cid1 with self.subTest('still open'): - cid2 = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid2 = _channels.create(REPLACE) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.send({cid2}, b'spam', blocking=False) """)) - channels.send(cid2, b'eggs', blocking=False) - interpreters.destroy(interp) + _channels.send(cid2, b'eggs', blocking=False) + _interpreters.destroy(interp) - channels.recv(cid2) + recv_nowait(cid2, unbound=True) + recv_nowait(cid2, unbound=False) with self.assertRaisesRegex(RuntimeError, f'channel {cid2} is empty'): - channels.recv(cid2) + _channels.recv(cid2) del cid2 #------------------- @@ -764,9 +783,9 @@ def test_recv_sending_interp_destroyed(self): def test_send_buffer(self): buf = bytearray(b'spamspamspam') - cid = channels.create() - channels.send_buffer(cid, buf, blocking=False) - obj = channels.recv(cid) + cid = _channels.create(REPLACE) + _channels.send_buffer(cid, buf, blocking=False) + obj = recv_nowait(cid) self.assertIsNot(obj, buf) self.assertIsInstance(obj, memoryview) @@ -784,18 +803,18 @@ def build_send_waiter(self, obj, *, buffer=False): # We want a long enough sleep that send() actually has to wait. if buffer: - send = channels.send_buffer + send = _channels.send_buffer else: - send = channels.send + send = _channels.send - cid = channels.create() + cid = _channels.create(REPLACE) try: started = time.monotonic() send(cid, obj, blocking=False) stopped = time.monotonic() - channels.recv(cid) + recv_nowait(cid) finally: - channels.destroy(cid) + _channels.destroy(cid) delay = stopped - started # seconds delay *= 3 @@ -807,14 +826,14 @@ def test_send_blocking_waiting(self): received = None obj = b'spam' wait = self.build_send_waiter(obj) - cid = channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received wait() received = recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send(cid, obj, blocking=True) + _channels.send(cid, obj, blocking=True) t.join() self.assertEqual(received, obj) @@ -823,14 +842,14 @@ def test_send_buffer_blocking_waiting(self): received = None obj = bytearray(b'spam') wait = self.build_send_waiter(obj, buffer=True) - cid = channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received wait() received = recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send_buffer(cid, obj, blocking=True) + _channels.send_buffer(cid, obj, blocking=True) t.join() self.assertEqual(received, obj) @@ -838,13 +857,13 @@ def f(): def test_send_blocking_no_wait(self): received = None obj = b'spam' - cid = channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received received = recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send(cid, obj, blocking=True) + _channels.send(cid, obj, blocking=True) t.join() self.assertEqual(received, obj) @@ -852,13 +871,13 @@ def f(): def test_send_buffer_blocking_no_wait(self): received = None obj = bytearray(b'spam') - cid = channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received received = recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send_buffer(cid, obj, blocking=True) + _channels.send_buffer(cid, obj, blocking=True) t.join() self.assertEqual(received, obj) @@ -867,25 +886,25 @@ def test_send_timeout(self): obj = b'spam' with self.subTest('non-blocking with timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(ValueError): - channels.send(cid, obj, blocking=False, timeout=0.1) + _channels.send(cid, obj, blocking=False, timeout=0.1) with self.subTest('timeout hit'): - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(TimeoutError): - channels.send(cid, obj, blocking=True, timeout=0.1) - with self.assertRaises(channels.ChannelEmptyError): - received = channels.recv(cid) + _channels.send(cid, obj, blocking=True, timeout=0.1) + with self.assertRaises(_channels.ChannelEmptyError): + received = recv_nowait(cid) print(repr(received)) with self.subTest('timeout not hit'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send(cid, obj, blocking=True, timeout=10) + _channels.send(cid, obj, blocking=True, timeout=10) t.join() def test_send_buffer_timeout(self): @@ -904,25 +923,25 @@ def test_send_buffer_timeout(self): obj = bytearray(b'spam') with self.subTest('non-blocking with timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(ValueError): - channels.send_buffer(cid, obj, blocking=False, timeout=0.1) + _channels.send_buffer(cid, obj, blocking=False, timeout=0.1) with self.subTest('timeout hit'): - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(TimeoutError): - channels.send_buffer(cid, obj, blocking=True, timeout=0.1) - with self.assertRaises(channels.ChannelEmptyError): - received = channels.recv(cid) + _channels.send_buffer(cid, obj, blocking=True, timeout=0.1) + with self.assertRaises(_channels.ChannelEmptyError): + received = recv_nowait(cid) print(repr(received)) with self.subTest('timeout not hit'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send_buffer(cid, obj, blocking=True, timeout=10) + _channels.send_buffer(cid, obj, blocking=True, timeout=10) t.join() def test_send_closed_while_waiting(self): @@ -930,25 +949,25 @@ def test_send_closed_while_waiting(self): wait = self.build_send_waiter(obj) with self.subTest('without timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): wait() - channels.close(cid, force=True) + _channels.close(cid, force=True) t = threading.Thread(target=f) t.start() - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, obj, blocking=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, obj, blocking=True) t.join() with self.subTest('with timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): wait() - channels.close(cid, force=True) + _channels.close(cid, force=True) t = threading.Thread(target=f) t.start() - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, obj, blocking=True, timeout=30) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, obj, blocking=True, timeout=30) t.join() def test_send_buffer_closed_while_waiting(self): @@ -968,73 +987,73 @@ def test_send_buffer_closed_while_waiting(self): wait = self.build_send_waiter(obj, buffer=True) with self.subTest('without timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): wait() - channels.close(cid, force=True) + _channels.close(cid, force=True) t = threading.Thread(target=f) t.start() - with self.assertRaises(channels.ChannelClosedError): - channels.send_buffer(cid, obj, blocking=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send_buffer(cid, obj, blocking=True) t.join() with self.subTest('with timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): wait() - channels.close(cid, force=True) + _channels.close(cid, force=True) t = threading.Thread(target=f) t.start() - with self.assertRaises(channels.ChannelClosedError): - channels.send_buffer(cid, obj, blocking=True, timeout=30) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send_buffer(cid, obj, blocking=True, timeout=30) t.join() #------------------- # close def test_close_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.close(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_multiple_users(self): - cid = channels.create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + id1 = _interpreters.create() + id2 = _interpreters.create() + _interpreters.run_string(id1, dedent(f""" + import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) - interpreters.run_string(id2, dedent(f""" - import _xxinterpchannels as _channels + _interpreters.run_string(id2, dedent(f""" + import _interpchannels as _channels _channels.recv({cid}) """)) - channels.close(cid) + _channels.close(cid) - excsnap = interpreters.run_string(id1, dedent(f""" + excsnap = _interpreters.run_string(id1, dedent(f""" _channels.send({cid}, b'spam') """)) self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') - excsnap = interpreters.run_string(id2, dedent(f""" + excsnap = _interpreters.run_string(id2, dedent(f""" _channels.send({cid}, b'spam') """)) self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') def test_close_multiple_times(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.close(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.close(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(cid) def test_close_empty(self): tests = [ @@ -1045,149 +1064,149 @@ def test_close_empty(self): ] for send, recv in tests: with self.subTest((send, recv)): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid, send=send, recv=recv) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.close(cid, send=send, recv=recv) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_defaults_with_unused_items(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) - with self.assertRaises(channels.ChannelNotEmptyError): - channels.close(cid) - channels.recv(cid) - channels.send(cid, b'eggs', blocking=False) + with self.assertRaises(_channels.ChannelNotEmptyError): + _channels.close(cid) + recv_nowait(cid) + _channels.send(cid, b'eggs', blocking=False) def test_close_recv_with_unused_items_unforced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - - with self.assertRaises(channels.ChannelNotEmptyError): - channels.close(cid, recv=True) - channels.recv(cid) - channels.send(cid, b'eggs', blocking=False) - channels.recv(cid) - channels.recv(cid) - channels.close(cid, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + + with self.assertRaises(_channels.ChannelNotEmptyError): + _channels.close(cid, recv=True) + recv_nowait(cid) + _channels.send(cid, b'eggs', blocking=False) + recv_nowait(cid) + recv_nowait(cid) + _channels.close(cid, recv=True) def test_close_send_with_unused_items_unforced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, send=True) - - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - channels.recv(cid) - channels.recv(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, send=True) + + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + recv_nowait(cid) + recv_nowait(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_both_with_unused_items_unforced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - - with self.assertRaises(channels.ChannelNotEmptyError): - channels.close(cid, recv=True, send=True) - channels.recv(cid) - channels.send(cid, b'eggs', blocking=False) - channels.recv(cid) - channels.recv(cid) - channels.close(cid, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + + with self.assertRaises(_channels.ChannelNotEmptyError): + _channels.close(cid, recv=True, send=True) + recv_nowait(cid) + _channels.send(cid, b'eggs', blocking=False) + recv_nowait(cid) + recv_nowait(cid) + _channels.close(cid, recv=True) def test_close_recv_with_unused_items_forced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, recv=True, force=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, recv=True, force=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_send_with_unused_items_forced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, send=True, force=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, send=True, force=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_both_with_unused_items_forced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, send=True, recv=True, force=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, send=True, recv=True, force=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_never_used(self): - cid = channels.create() - channels.close(cid) + cid = _channels.create(REPLACE) + _channels.close(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'spam') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'spam') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_by_unassociated_interp(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.close({cid}, force=True) """)) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.close(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(cid) def test_close_used_multiple_times_by_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid, force=True) - - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.close(cid, force=True) + + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_channel_list_interpreters_invalid_channel(self): - cid = channels.create() + cid = _channels.create(REPLACE) # Test for invalid channel ID. - with self.assertRaises(channels.ChannelNotFoundError): - channels.list_interpreters(1000, send=True) + with self.assertRaises(_channels.ChannelNotFoundError): + _channels.list_interpreters(1000, send=True) - channels.close(cid) + _channels.close(cid) # Test for a channel that has been closed. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) def test_channel_list_interpreters_invalid_args(self): # Tests for invalid arguments passed to the API. - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(TypeError): - channels.list_interpreters(cid) + _channels.list_interpreters(cid) class ChannelReleaseTests(TestBase): @@ -1234,125 +1253,125 @@ class ChannelReleaseTests(TestBase): """ def test_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid, send=True, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_multiple_users(self): - cid = channels.create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + id1 = _interpreters.create() + id2 = _interpreters.create() + _interpreters.run_string(id1, dedent(f""" + import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) out = _run_output(id2, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + obj, _ = _channels.recv({cid}) _channels.release({cid}) print(repr(obj)) """)) - interpreters.run_string(id1, dedent(f""" + _interpreters.run_string(id1, dedent(f""" _channels.release({cid}) """)) self.assertEqual(out.strip(), "b'spam'") def test_no_kwargs(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.release(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_multiple_times(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid, send=True, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.release(cid, send=True, recv=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.release(cid, send=True, recv=True) def test_with_unused_items(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.release(cid, send=True, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_never_used(self): - cid = channels.create() - channels.release(cid) + cid = _channels.create(REPLACE) + _channels.release(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'spam') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'spam') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_by_unassociated_interp(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.release({cid}) """)) - obj = channels.recv(cid) - channels.release(cid) + obj = recv_nowait(cid) + _channels.release(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') self.assertEqual(obj, b'spam') def test_close_if_unassociated(self): # XXX Something's not right with this test... - cid = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels obj = _channels.send({cid}, b'spam', blocking=False) _channels.release({cid}) """)) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_partially(self): # XXX Is partial close too weird/confusing? - cid = channels.create() - channels.send(cid, None, blocking=False) - channels.recv(cid) - channels.send(cid, b'spam', blocking=False) - channels.release(cid, send=True) - obj = channels.recv(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, None, blocking=False) + recv_nowait(cid) + _channels.send(cid, b'spam', blocking=False) + _channels.release(cid, send=True) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') def test_used_multiple_times_by_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid, send=True, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) class ChannelCloseFixture(namedtuple('ChannelCloseFixture', @@ -1422,18 +1441,18 @@ def clean_up(self): def _new_channel(self, creator): if creator.name == 'main': - return channels.create() + return _channels.create(REPLACE) else: - ch = channels.create() + ch = _channels.create(REPLACE) run_interp(creator.id, f""" - import _xxsubinterpreters + import _interpreters cid = _xxsubchannels.create() # We purposefully send back an int to avoid tying the # channel to the other interpreter. _xxsubchannels.send({ch}, int(cid), blocking=False) - del _xxsubinterpreters + del _interpreters """) - self._cid = channels.recv(ch) + self._cid = recv_nowait(ch) return self._cid def _get_interpreter(self, interp): @@ -1458,13 +1477,13 @@ def _prep_interpreter(self, interp): if interp.name == 'main': return run_interp(interp.id, f""" - import _xxinterpchannels as channels - import test.test__xxinterpchannels as helpers + import _interpchannels as channels + import test.test__interpchannels as helpers ChannelState = helpers.ChannelState try: cid except NameError: - cid = channels._channel_id({self.cid}) + cid = _channels._channel_id({self.cid}) """) @@ -1651,7 +1670,7 @@ def run_action(self, fix, action, *, hideclosed=True): ) fix.record_action(action, result) else: - _cid = channels.create() + _cid = _channels.create(REPLACE) run_interp(interp.id, f""" result = helpers.run_action( {fix.cid}, @@ -1660,12 +1679,12 @@ def run_action(self, fix, action, *, hideclosed=True): {repr(fix.state)}, hideclosed={hideclosed}, ) - channels.send({_cid}, result.pending.to_bytes(1, 'little'), blocking=False) - channels.send({_cid}, b'X' if result.closed else b'', blocking=False) + _channels.send({_cid}, result.pending.to_bytes(1, 'little'), blocking=False) + _channels.send({_cid}, b'X' if result.closed else b'', blocking=False) """) result = ChannelState( - pending=int.from_bytes(channels.recv(_cid), 'little'), - closed=bool(channels.recv(_cid)), + pending=int.from_bytes(recv_nowait(_cid), 'little'), + closed=bool(recv_nowait(_cid)), ) fix.record_action(action, result) @@ -1688,42 +1707,42 @@ def _close(self, fix, *, force): if not fix.expect_closed_error(): self.run_action(fix, close, hideclosed=False) else: - with self.assertRaises(channels.ChannelClosedError): + with self.assertRaises(_channels.ChannelClosedError): self.run_action(fix, close, hideclosed=False) def _assert_closed_in_interp(self, fix, interp=None): if interp is None or interp.name == 'main': - with self.assertRaises(channels.ChannelClosedError): - channels.recv(fix.cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(fix.cid, b'spam') - with self.assertRaises(channels.ChannelClosedError): - channels.close(fix.cid) - with self.assertRaises(channels.ChannelClosedError): - channels.close(fix.cid, force=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(fix.cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(fix.cid, b'spam') + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(fix.cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(fix.cid, force=True) else: run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.recv(cid) + _channels.recv(cid) """) run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) """) run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.close(cid) + _channels.close(cid) """) run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.close(cid, force=True) + _channels.close(cid, force=True) """) def _assert_closed(self, fix): self.assertTrue(fix.state.closed) for _ in range(fix.state.pending): - channels.recv(fix.cid) + recv_nowait(fix.cid) self._assert_closed_in_interp(fix) for interp in ('same', 'other'): diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__interpreters.py similarity index 73% rename from Lib/test/test__xxsubinterpreters.py rename to Lib/test/test__interpreters.py index a76e4d0ade5b8a..14cd50bd30502c 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__interpreters.py @@ -7,15 +7,15 @@ import threading import unittest -import _testinternalcapi from test import support from test.support import import_helper from test.support import os_helper from test.support import script_helper -interpreters = import_helper.import_module('_xxsubinterpreters') -from _xxsubinterpreters import InterpreterNotFoundError +_interpreters = import_helper.import_module('_interpreters') +_testinternalcapi = import_helper.import_module('_testinternalcapi') +from _interpreters import InterpreterNotFoundError ################################## @@ -36,18 +36,18 @@ def _captured_script(script): def _run_output(interp, request): script, rpipe = _captured_script(request) with rpipe: - interpreters.run_string(interp, script) + _interpreters.run_string(interp, script) return rpipe.read() def _wait_for_interp_to_run(interp, timeout=None): # bpo-37224: Running this test file in multiprocesses will fail randomly. # The failure reason is that the thread can't acquire the cpu to - # run subinterpreter eariler than the main thread in multiprocess. + # run subinterpreter earlier than the main thread in multiprocess. if timeout is None: timeout = support.SHORT_TIMEOUT for _ in support.sleeping_retry(timeout, error=False): - if interpreters.is_running(interp): + if _interpreters.is_running(interp): break else: raise RuntimeError('interp is not running') @@ -57,7 +57,7 @@ def _wait_for_interp_to_run(interp, timeout=None): def _running(interp): r, w = os.pipe() def run(): - interpreters.run_string(interp, dedent(f""" + _interpreters.run_string(interp, dedent(f""" # wait for "signal" with open({r}, encoding="utf-8") as rpipe: rpipe.read() @@ -75,12 +75,12 @@ def run(): def clean_up_interpreters(): - for id in interpreters.list_all(): + for id, *_ in _interpreters.list_all(): if id == 0: # main continue try: - interpreters.destroy(id) - except RuntimeError: + _interpreters.destroy(id) + except _interpreters.InterpreterError: pass # already destroyed @@ -112,7 +112,7 @@ def test_default_shareables(self): for obj in shareables: with self.subTest(obj): self.assertTrue( - interpreters.is_shareable(obj)) + _interpreters.is_shareable(obj)) def test_not_shareable(self): class Cheese: @@ -141,7 +141,7 @@ class SubBytes(bytes): for obj in not_shareables: with self.subTest(repr(obj)): self.assertFalse( - interpreters.is_shareable(obj)) + _interpreters.is_shareable(obj)) class ShareableTypeTests(unittest.TestCase): @@ -230,8 +230,8 @@ class ModuleTests(TestBase): def test_import_in_interpreter(self): _run_output( - interpreters.create(), - 'import _xxsubinterpreters as _interpreters', + _interpreters.create(), + 'import _interpreters', ) @@ -241,45 +241,45 @@ def test_import_in_interpreter(self): class ListAllTests(TestBase): def test_initial(self): - main = interpreters.get_main() - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main]) def test_after_creating(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + first = _interpreters.create() + second = _interpreters.create() + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main, first, second]) def test_after_destroying(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - interpreters.destroy(first) - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + first = _interpreters.create() + second = _interpreters.create() + _interpreters.destroy(first) + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main, second]) class GetCurrentTests(TestBase): def test_main(self): - main = interpreters.get_main() - cur = interpreters.get_current() + main, *_ = _interpreters.get_main() + cur, *_ = _interpreters.get_current() self.assertEqual(cur, main) self.assertIsInstance(cur, int) def test_subinterpreter(self): - main = interpreters.get_main() - interp = interpreters.create() + main, *_ = _interpreters.get_main() + interp = _interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - cur = _interpreters.get_current() + import _interpreters + cur, *_ = _interpreters.get_current() print(cur) assert isinstance(cur, int) """)) cur = int(out.strip()) - _, expected = interpreters.list_all() + _, expected = [id for id, *_ in _interpreters.list_all()] self.assertEqual(cur, expected) self.assertNotEqual(cur, main) @@ -287,17 +287,17 @@ def test_subinterpreter(self): class GetMainTests(TestBase): def test_from_main(self): - [expected] = interpreters.list_all() - main = interpreters.get_main() + [expected] = [id for id, *_ in _interpreters.list_all()] + main, *_ = _interpreters.get_main() self.assertEqual(main, expected) self.assertIsInstance(main, int) def test_from_subinterpreter(self): - [expected] = interpreters.list_all() - interp = interpreters.create() + [expected] = [id for id, *_ in _interpreters.list_all()] + interp = _interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - main = _interpreters.get_main() + import _interpreters + main, *_ = _interpreters.get_main() print(main) assert isinstance(main, int) """)) @@ -308,22 +308,22 @@ def test_from_subinterpreter(self): class IsRunningTests(TestBase): def test_main(self): - main = interpreters.get_main() - self.assertTrue(interpreters.is_running(main)) + main, *_ = _interpreters.get_main() + self.assertTrue(_interpreters.is_running(main)) @unittest.skip('Fails on FreeBSD') def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interpreters.is_running(interp)) + interp = _interpreters.create() + self.assertFalse(_interpreters.is_running(interp)) with _running(interp): - self.assertTrue(interpreters.is_running(interp)) - self.assertFalse(interpreters.is_running(interp)) + self.assertTrue(_interpreters.is_running(interp)) + self.assertFalse(_interpreters.is_running(interp)) def test_from_subinterpreter(self): - interp = interpreters.create() + interp = _interpreters.create() out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters if _interpreters.is_running({interp}): print(True) else: @@ -332,34 +332,35 @@ def test_from_subinterpreter(self): self.assertEqual(out.strip(), 'True') def test_already_destroyed(self): - interp = interpreters.create() - interpreters.destroy(interp) + interp = _interpreters.create() + _interpreters.destroy(interp) with self.assertRaises(InterpreterNotFoundError): - interpreters.is_running(interp) + _interpreters.is_running(interp) def test_does_not_exist(self): with self.assertRaises(InterpreterNotFoundError): - interpreters.is_running(1_000_000) + _interpreters.is_running(1_000_000) def test_bad_id(self): with self.assertRaises(ValueError): - interpreters.is_running(-1) + _interpreters.is_running(-1) class CreateTests(TestBase): def test_in_main(self): - id = interpreters.create() + id = _interpreters.create() self.assertIsInstance(id, int) - self.assertIn(id, interpreters.list_all()) + after = [id for id, *_ in _interpreters.list_all()] + self.assertIn(id, after) @unittest.skip('enable this test when working on pystate.c') def test_unique_id(self): seen = set() for _ in range(100): - id = interpreters.create() - interpreters.destroy(id) + id = _interpreters.create() + _interpreters.destroy(id) seen.add(id) self.assertEqual(len(seen), 100) @@ -369,7 +370,7 @@ def test_in_thread(self): id = None def f(): nonlocal id - id = interpreters.create() + id = _interpreters.create() lock.acquire() lock.release() @@ -377,29 +378,31 @@ def f(): with lock: t.start() t.join() - self.assertIn(id, interpreters.list_all()) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertIn(id, after) def test_in_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) assert isinstance(id, int) """)) id2 = int(out.strip()) - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1, id2}) def test_in_threaded_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() id2 = None def f(): nonlocal id2 out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) """)) @@ -409,144 +412,155 @@ def f(): t.start() t.join() - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1, id2}) def test_after_destroy_all(self): - before = set(interpreters.list_all()) + before = set(id for id, *_ in _interpreters.list_all()) # Create 3 subinterpreters. ids = [] for _ in range(3): - id = interpreters.create() + id = _interpreters.create() ids.append(id) # Now destroy them. for id in ids: - interpreters.destroy(id) + _interpreters.destroy(id) # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id}) + id = _interpreters.create() + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, before | {id}) def test_after_destroy_some(self): - before = set(interpreters.list_all()) + before = set(id for id, *_ in _interpreters.list_all()) # Create 3 subinterpreters. - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() + id1 = _interpreters.create() + id2 = _interpreters.create() + id3 = _interpreters.create() # Now destroy 2 of them. - interpreters.destroy(id1) - interpreters.destroy(id3) + _interpreters.destroy(id1) + _interpreters.destroy(id3) # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id, id2}) + id = _interpreters.create() + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, before | {id, id2}) class DestroyTests(TestBase): def test_one(self): - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() - self.assertIn(id2, interpreters.list_all()) - interpreters.destroy(id2) - self.assertNotIn(id2, interpreters.list_all()) - self.assertIn(id1, interpreters.list_all()) - self.assertIn(id3, interpreters.list_all()) + id1 = _interpreters.create() + id2 = _interpreters.create() + id3 = _interpreters.create() + before = set(id for id, *_ in _interpreters.list_all()) + self.assertIn(id2, before) + + _interpreters.destroy(id2) + + after = set(id for id, *_ in _interpreters.list_all()) + self.assertNotIn(id2, after) + self.assertIn(id1, after) + self.assertIn(id3, after) def test_all(self): - before = set(interpreters.list_all()) + initial = set(id for id, *_ in _interpreters.list_all()) ids = set() for _ in range(3): - id = interpreters.create() + id = _interpreters.create() ids.add(id) - self.assertEqual(set(interpreters.list_all()), before | ids) + before = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(before, initial | ids) for id in ids: - interpreters.destroy(id) - self.assertEqual(set(interpreters.list_all()), before) + _interpreters.destroy(id) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, initial) def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(RuntimeError): - interpreters.destroy(main) + main, = [id for id, *_ in _interpreters.list_all()] + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.destroy(main) def f(): - with self.assertRaises(RuntimeError): - interpreters.destroy(main) + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.destroy(main) t = threading.Thread(target=f) t.start() t.join() def test_already_destroyed(self): - id = interpreters.create() - interpreters.destroy(id) + id = _interpreters.create() + _interpreters.destroy(id) with self.assertRaises(InterpreterNotFoundError): - interpreters.destroy(id) + _interpreters.destroy(id) def test_does_not_exist(self): with self.assertRaises(InterpreterNotFoundError): - interpreters.destroy(1_000_000) + _interpreters.destroy(1_000_000) def test_bad_id(self): with self.assertRaises(ValueError): - interpreters.destroy(-1) + _interpreters.destroy(-1) def test_from_current(self): - main, = interpreters.list_all() - id = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id = _interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters try: _interpreters.destroy({id}) - except RuntimeError: + except _interpreters.InterpreterError: pass """) - interpreters.run_string(id, script) - self.assertEqual(set(interpreters.list_all()), {main, id}) + _interpreters.run_string(id, script) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id}) def test_from_sibling(self): - main, = interpreters.list_all() - id1 = interpreters.create() - id2 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() + id2 = _interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.destroy({id2}) """) - interpreters.run_string(id1, script) + _interpreters.run_string(id1, script) - self.assertEqual(set(interpreters.list_all()), {main, id1}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1}) def test_from_other_thread(self): - id = interpreters.create() + id = _interpreters.create() def f(): - interpreters.destroy(id) + _interpreters.destroy(id) t = threading.Thread(target=f) t.start() t.join() def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + interp = _interpreters.create() with _running(interp): - self.assertTrue(interpreters.is_running(interp), + self.assertTrue(_interpreters.is_running(interp), msg=f"Interp {interp} should be running before destruction.") - with self.assertRaises(RuntimeError, + with self.assertRaises(_interpreters.InterpreterError, msg=f"Should not be able to destroy interp {interp} while it's still running."): - interpreters.destroy(interp) - self.assertTrue(interpreters.is_running(interp)) + _interpreters.destroy(interp) + self.assertTrue(_interpreters.is_running(interp)) class RunStringTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def test_success(self): script, file = _captured_script('print("it worked!", end="")') with file: - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) out = file.read() self.assertEqual(out, 'it worked!') @@ -555,7 +569,7 @@ def test_in_thread(self): script, file = _captured_script('print("it worked!", end="")') with file: def f(): - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) t = threading.Thread(target=f) t.start() @@ -565,7 +579,7 @@ def f(): self.assertEqual(out, 'it worked!') def test_create_thread(self): - subinterp = interpreters.create() + subinterp = _interpreters.create() script, file = _captured_script(""" import threading def f(): @@ -576,7 +590,7 @@ def f(): t.join() """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, 'it worked!') @@ -584,7 +598,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create(isolated=True) + subinterp = _interpreters.create('isolated') script, file = _captured_script(f""" import threading def f(): @@ -598,13 +612,13 @@ def f(): print('{expected}', end='') """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create(isolated=False) + subinterp = _interpreters.create('legacy') script, file = _captured_script(""" import threading def f(): @@ -615,13 +629,13 @@ def f(): t.join() """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, 'it worked!') def test_shareable_types(self): - interp = interpreters.create() + interp = _interpreters.create() objects = [ None, 'spam', @@ -630,15 +644,15 @@ def test_shareable_types(self): ] for obj in objects: with self.subTest(obj): - interpreters.set___main___attrs(interp, dict(obj=obj)) - interpreters.run_string( + _interpreters.set___main___attrs(interp, dict(obj=obj)) + _interpreters.run_string( interp, f'assert(obj == {obj!r})', ) def test_os_exec(self): expected = 'spam spam spam spam spam' - subinterp = interpreters.create() + subinterp = _interpreters.create() script, file = _captured_script(f""" import os, sys try: @@ -647,7 +661,7 @@ def test_os_exec(self): print('{expected}', end='') """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, expected) @@ -668,7 +682,7 @@ def test_fork(self): with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) file.seek(0) content = file.read() @@ -676,31 +690,31 @@ def test_fork(self): def test_already_running(self): with _running(self.id): - with self.assertRaises(RuntimeError): - interpreters.run_string(self.id, 'print("spam")') + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.run_string(self.id, 'print("spam")') def test_does_not_exist(self): id = 0 - while id in interpreters.list_all(): + while id in set(id for id, *_ in _interpreters.list_all()): id += 1 with self.assertRaises(InterpreterNotFoundError): - interpreters.run_string(id, 'print("spam")') + _interpreters.run_string(id, 'print("spam")') def test_error_id(self): with self.assertRaises(ValueError): - interpreters.run_string(-1, 'print("spam")') + _interpreters.run_string(-1, 'print("spam")') def test_bad_id(self): with self.assertRaises(TypeError): - interpreters.run_string('spam', 'print("spam")') + _interpreters.run_string('spam', 'print("spam")') def test_bad_script(self): with self.assertRaises(TypeError): - interpreters.run_string(self.id, 10) + _interpreters.run_string(self.id, 10) def test_bytes_for_script(self): with self.assertRaises(TypeError): - interpreters.run_string(self.id, b'print("spam")') + _interpreters.run_string(self.id, b'print("spam")') def test_with_shared(self): r, w = os.pipe() @@ -721,8 +735,8 @@ def test_with_shared(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.set___main___attrs(self.id, shared) - interpreters.run_string(self.id, script) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -732,7 +746,7 @@ def test_with_shared(self): self.assertIsNone(ns['cheddar']) def test_shared_overwrites(self): - interpreters.run_string(self.id, dedent(""" + _interpreters.run_string(self.id, dedent(""" spam = 'eggs' ns1 = dict(vars()) del ns1['__builtins__'] @@ -743,8 +757,8 @@ def test_shared_overwrites(self): ns2 = dict(vars()) del ns2['__builtins__'] """) - interpreters.set___main___attrs(self.id, shared) - interpreters.run_string(self.id, script) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) r, w = os.pipe() script = dedent(f""" @@ -754,7 +768,7 @@ def test_shared_overwrites(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -775,8 +789,8 @@ def test_shared_overwrites_default_vars(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.set___main___attrs(self.id, shared) - interpreters.run_string(self.id, script) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -784,7 +798,7 @@ def test_shared_overwrites_default_vars(self): def test_main_reused(self): r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" spam = True ns = dict(vars()) @@ -798,7 +812,7 @@ def test_main_reused(self): ns1 = pickle.load(chan) r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" eggs = False ns = dict(vars()) @@ -827,7 +841,7 @@ def test_execution_namespace_is_main(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -835,7 +849,6 @@ def test_execution_namespace_is_main(self): ns.pop('__loader__') self.assertEqual(ns, { '__name__': '__main__', - '__annotations__': {}, '__doc__': None, '__package__': None, '__spec__': None, @@ -848,7 +861,7 @@ def test_still_running_at_exit(self): script = dedent(""" from textwrap import dedent import threading - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() def f(): _interpreters.run_string(id, dedent(''' @@ -872,13 +885,13 @@ class RunFailedTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def add_module(self, modname, text): import tempfile tempdir = tempfile.mkdtemp() self.addCleanup(lambda: os_helper.rmtree(tempdir)) - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" import sys sys.path.insert(0, {tempdir!r}) """)) @@ -900,11 +913,11 @@ class NeverError(Exception): pass raise NeverError # never raised """).format(dedent(text)) if fails: - err = interpreters.run_string(self.id, script) + err = _interpreters.run_string(self.id, script) self.assertIsNot(err, None) return err else: - err = interpreters.run_string(self.id, script) + err = _interpreters.run_string(self.id, script) self.assertIs(err, None) return None except: @@ -1029,7 +1042,7 @@ class RunFuncTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def test_success(self): r, w = os.pipe() @@ -1039,8 +1052,8 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') - interpreters.set___main___attrs(self.id, dict(w=w)) - interpreters.run_func(self.id, script) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1056,8 +1069,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') def f(): - interpreters.set___main___attrs(self.id, dict(w=w)) - interpreters.run_func(self.id, script) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) t = threading.Thread(target=f) t.start() t.join() @@ -1077,8 +1090,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') code = script.__code__ - interpreters.set___main___attrs(self.id, dict(w=w)) - interpreters.run_func(self.id, code) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, code) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1091,7 +1104,7 @@ def script(): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) # XXX This hasn't been fixed yet. @unittest.expectedFailure @@ -1099,38 +1112,38 @@ def test_return_value(self): def script(): return 'spam' with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) def test_args(self): with self.subTest('args'): def script(a, b=0): assert a == b with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('*args'): def script(*args): assert not args with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('**kwargs'): def script(**kwargs): assert not kwargs with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('kwonly'): def script(*, spam=True): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('posonly'): def script(spam, /): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) if __name__ == '__main__': diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 0947464bb8c04e..7e6e296c069abb 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -1,4 +1,4 @@ -from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, localeconv, Error) +from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error) try: from _locale import (RADIXCHAR, THOUSEP, nl_langinfo) except ImportError: @@ -26,7 +26,10 @@ 'bs_BA', 'fr_LU', 'kl_GL', 'fa_IR', 'de_BE', 'sv_SE', 'it_CH', 'uk_UA', 'eu_ES', 'vi_VN', 'af_ZA', 'nb_NO', 'en_DK', 'tg_TJ', 'ps_AF', 'en_US', 'fr_FR.ISO8859-1', 'fr_FR.UTF-8', 'fr_FR.ISO8859-15@euro', - 'ru_RU.KOI8-R', 'ko_KR.eucKR'] + 'ru_RU.KOI8-R', 'ko_KR.eucKR', + 'ja_JP.UTF-8', 'lzh_TW.UTF-8', 'my_MM.UTF-8', 'or_IN.UTF-8', 'shn_MM.UTF-8', + 'ar_AE.UTF-8', 'bn_IN.UTF-8', 'mr_IN.UTF-8', 'th_TH.TIS620', +] def setUpModule(): global candidate_locales @@ -74,6 +77,19 @@ def accept(loc): 'ps_AF': ('\u066b', '\u066c'), } +known_alt_digits = { + 'C': (0, {}), + 'en_US': (0, {}), + 'fa_IR': (100, {0: '\u06f0\u06f0', 10: '\u06f1\u06f0', 99: '\u06f9\u06f9'}), + 'ja_JP': (100, {1: '\u4e00', 10: '\u5341', 99: '\u4e5d\u5341\u4e5d'}), + 'lzh_TW': (32, {0: '\u3007', 10: '\u5341', 31: '\u5345\u4e00'}), + 'my_MM': (100, {0: '\u1040\u1040', 10: '\u1041\u1040', 99: '\u1049\u1049'}), + 'or_IN': (100, {0: '\u0b66', 10: '\u0b67\u0b66', 99: '\u0b6f\u0b6f'}), + 'shn_MM': (100, {0: '\u1090\u1090', 10: '\u1091\u1090', 99: '\u1099\u1099'}), + 'ar_AE': (100, {0: '\u0660', 10: '\u0661\u0660', 99: '\u0669\u0669'}), + 'bn_IN': (100, {0: '\u09e6', 10: '\u09e7\u09e6', 99: '\u09ef\u09ef'}), +} + if sys.platform == 'win32': # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) del known_numerics['ps_AF'] @@ -115,16 +131,17 @@ def numeric_tester(self, calc_type, calc_value, data_type, used_locale): def test_lc_numeric_nl_langinfo(self): # Test nl_langinfo against known values tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue for li, lc in ((RADIXCHAR, "decimal_point"), (THOUSEP, "thousands_sep")): if self.numeric_tester('nl_langinfo', nl_langinfo(li), lc, loc): tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) if not tested: self.skipTest('no suitable locales') @@ -135,10 +152,10 @@ def test_lc_numeric_nl_langinfo(self): def test_lc_numeric_localeconv(self): # Test localeconv against known values tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue formatting = localeconv() @@ -146,6 +163,7 @@ def test_lc_numeric_localeconv(self): "thousands_sep"): if self.numeric_tester('localeconv', formatting[lc], lc, loc): tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) if not tested: self.skipTest('no suitable locales') @@ -153,10 +171,10 @@ def test_lc_numeric_localeconv(self): def test_lc_numeric_basic(self): # Test nl_langinfo against localeconv tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue for li, lc in ((RADIXCHAR, "decimal_point"), @@ -173,6 +191,42 @@ def test_lc_numeric_basic(self): nl_radixchar, li_radixchar, loc, set_locale)) tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ALT_DIGITS'), "requires locale.ALT_DIGITS") + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten, bpo-46390" + ) + def test_alt_digits_nl_langinfo(self): + # Test nl_langinfo(ALT_DIGITS) + tested = False + for loc in candidate_locales: + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + + with self.subTest(locale=loc): + alt_digits = nl_langinfo(locale.ALT_DIGITS) + self.assertIsInstance(alt_digits, str) + alt_digits = alt_digits.split(';') if alt_digits else [] + if alt_digits: + self.assertGreaterEqual(len(alt_digits), 10, alt_digits) + loc1 = loc.split('.', 1)[0] + if loc1 in known_alt_digits: + count, samples = known_alt_digits[loc1] + if count and not alt_digits: + self.skipTest(f'ALT_DIGITS is not set for locale {loc!r} on this platform') + self.assertEqual(len(alt_digits), count, alt_digits) + for i in samples: + self.assertEqual(alt_digits[i], samples[i]) + tested = True if not tested: self.skipTest('no suitable locales') @@ -180,10 +234,10 @@ def test_float_parsing(self): # Bug #1391872: Test whether float parsing is okay on European # locales. tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue @@ -199,6 +253,7 @@ def test_float_parsing(self): self.assertRaises(ValueError, float, localeconv()['decimal_point'].join(['1', '23'])) tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) if not tested: self.skipTest('no suitable locales') diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 10f04b64dda40e..d5cf014d40daf8 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -17,7 +17,7 @@ def check_bool_function_result(self, func, ops, expected): self.assertEqual(func(op), expected) def test_invalid_opcodes(self): - invalid = [-100, -1, 255, 512, 513, 1000] + invalid = [-100, -1, 512, 513, 1000] self.check_bool_function_result(_opcode.is_valid, invalid, False) self.check_bool_function_result(_opcode.has_arg, invalid, False) self.check_bool_function_result(_opcode.has_const, invalid, False) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py new file mode 100644 index 00000000000000..a9abd61b9a8d92 --- /dev/null +++ b/Lib/test/test_android.py @@ -0,0 +1,444 @@ +import io +import platform +import queue +import re +import subprocess +import sys +import unittest +from _android_support import TextLogStream +from array import array +from contextlib import ExitStack, contextmanager +from threading import Thread +from test.support import LOOPBACK_TIMEOUT +from time import time +from unittest.mock import patch + + +if sys.platform != "android": + raise unittest.SkipTest("Android-specific") + +api_level = platform.android_ver().api_level + +# (name, level, fileno) +STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)] + + +# Test redirection of stdout and stderr to the Android log. +class TestAndroidOutput(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.logcat_process = subprocess.Popen( + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" + ) + self.logcat_queue = queue.Queue() + + def logcat_thread(): + for line in self.logcat_process.stdout: + self.logcat_queue.put(line.rstrip("\n")) + self.logcat_process.stdout.close() + self.logcat_thread = Thread(target=logcat_thread) + self.logcat_thread.start() + + from ctypes import CDLL, c_char_p, c_int + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) + + def assert_logs(self, level, tag, expected, **kwargs): + for line in expected: + self.assert_log(level, tag, line, **kwargs) + + def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): + deadline = time() + timeout + while True: + try: + line = self.logcat_queue.get(timeout=(deadline - time())) + except queue.Empty: + self.fail(f"line not found: {expected!r}") + if match := re.fullmatch(fr"(.)/{tag}: (.*)", line): + try: + self.assertEqual(level, match[1]) + self.assertEqual(expected, match[2]) + break + except AssertionError: + if not skip: + raise + + def tearDown(self): + self.logcat_process.terminate() + self.logcat_process.wait(LOOPBACK_TIMEOUT) + self.logcat_thread.join(LOOPBACK_TIMEOUT) + + @contextmanager + def unbuffered(self, stream): + stream.reconfigure(write_through=True) + try: + yield + finally: + stream.reconfigure(write_through=False) + + # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't + # test them directly. Detect this mode and use some temporary streams with + # the same properties. + def stream_context(self, stream_name, level): + # https://developer.android.com/ndk/reference/group/logging + prio = {"I": 4, "W": 5}[level] + + stack = ExitStack() + stack.enter_context(self.subTest(stream_name)) + stream = getattr(sys, stream_name) + native_stream = getattr(sys, f"__{stream_name}__") + if isinstance(stream, io.StringIO): + stack.enter_context( + patch( + f"sys.{stream_name}", + TextLogStream( + prio, f"python.{stream_name}", native_stream.fileno(), + errors="backslashreplace" + ), + ) + ) + return stack + + def test_str(self): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): + stream = getattr(sys, stream_name) + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) + self.assertEqual("UTF-8", stream.encoding) + self.assertEqual("backslashreplace", stream.errors) + self.assertIs(stream.line_buffering, True) + self.assertIs(stream.write_through, False) + + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) + if lines is None: + lines = [s] + self.assert_logs(level, tag, lines) + + # Single-line messages, + with self.unbuffered(stream): + write("", []) + + write("a") + write("Hello") + write("Hello world") + write(" ") + write(" ") + + # Non-ASCII text + write("ol\u00e9") # Spanish + write("\u4e2d\u6587") # Chinese + + # Non-BMP emoji + write("\U0001f600") + + # Non-encodable surrogates + write("\ud800\udc00", [r"\ud800\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", [r"\udc80"]) + + # Null characters are logged using "modified UTF-8". + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) + + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. + write("\nx", [""]) + write("\na\n", ["x", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d"]) + write("xx", []) + write("f\n\ng", ["exxf", ""]) + write("\n", ["g"]) + + # Since this is a line-based logging system, line buffering + # cannot be turned off, i.e. a newline always causes a flush. + stream.reconfigure(line_buffering=False) + self.assertIs(stream.line_buffering, True) + + # However, buffering can be turned off completely if you want a + # flush after every write. + with self.unbuffered(stream): + write("\nx", ["", "x"]) + write("\na\n", ["", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d", "e"]) + write("xx", ["xx"]) + write("f\n\ng", ["f", "", "g"]) + write("\n", [""]) + + # "\r\n" should be translated into "\n". + write("hello\r\n", ["hello"]) + write("hello\r\nworld\r\n", ["hello", "world"]) + write("\r\n", [""]) + + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + write(CustomStr("custom\n"), ["custom"], write_len=7) + + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + # Manual flushing is supported. + write("hello", []) + stream.flush() + self.assert_log(level, tag, "hello") + write("hello", []) + write("world", []) + stream.flush() + self.assert_log(level, tag, "helloworld") + + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE in _android_support.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). + # + # ASCII (1 byte per character) + write(("foobar" * 700) + "\n", # 4200 bytes in + [("foobar" * 666) + "foob", # 4000 bytes out + "ar" + ("foobar" * 33)]) # 200 bytes out + + # "Full-width" digits 0-9 (3 bytes per character) + s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" + write((s * 150) + "\n", # 4500 bytes in + [s * 100, # 3000 bytes out + s * 50]) # 1500 bytes out + + s = "0123456789" + write(s * 200, []) # 2000 bytes in + write(s * 150, []) # 1500 bytes in + write(s * 51, [s * 350]) # 510 bytes in, 3500 bytes out + write("\n", [s * 51]) # 0 bytes in, 510 bytes out + + def test_bytes(self): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): + stream = getattr(sys, stream_name).buffer + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) + + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) + if lines is None: + lines = [b.decode()] + self.assert_logs(level, tag, lines) + + # Single-line messages, + write(b"", []) + + write(b"a") + write(b"Hello") + write(b"Hello world") + write(b" ") + write(b" ") + + # Non-ASCII text + write(b"ol\xc3\xa9") # Spanish + write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese + + # Non-BMP emoji + write(b"\xf0\x9f\x98\x80") + + # Null bytes are logged using "modified UTF-8". + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) + + # Invalid UTF-8 + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) + + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. + # + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) + write(b"\n", [""]) + write(b"b\n", ["b"]) + write(b"c\n\n", ["c", ""]) + write(b"d\ne", ["d", "e"]) + write(b"xx", ["xx"]) + write(b"f\n\ng", ["f", "", "g"]) + write(b"\n", [""]) + + # "\r\n" should be translated into "\n". + write(b"hello\r\n", ["hello"]) + write(b"hello\r\nworld\r\n", ["hello", "world"]) + write(b"\r\n", [""]) + + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. + for obj in ["", "hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + +class TestAndroidRateLimit(unittest.TestCase): + def test_rate_limit(self): + # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 + PER_MESSAGE_OVERHEAD = 28 + + # https://developer.android.com/ndk/reference/group/logging + ANDROID_LOG_DEBUG = 3 + + # To avoid flooding the test script output, use a different tag rather + # than stdout or stderr. + tag = "python.rate_limit" + stream = TextLogStream(ANDROID_LOG_DEBUG, tag) + + # Make a test message which consumes 1 KB of the logcat buffer. + message = "Line {:03d} " + message += "." * ( + 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0)) + ) + "\n" + + # To avoid depending on the performance of the test device, we mock the + # passage of time. + mock_now = time() + + def mock_time(): + # Avoid division by zero by simulating a small delay. + mock_sleep(0.0001) + return mock_now + + def mock_sleep(duration): + nonlocal mock_now + mock_now += duration + + # See _android_support.py. The default values of these parameters work + # well across a wide range of devices, but we'll use smaller values to + # ensure a quick and reliable test that doesn't flood the log too much. + MAX_KB_PER_SECOND = 100 + BUCKET_KB = 10 + with ( + patch("_android_support.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), + patch("_android_support.BUCKET_SIZE", BUCKET_KB * 1024), + patch("_android_support.sleep", mock_sleep), + patch("_android_support.time", mock_time), + ): + # Make sure the token bucket is full. + stream.write("Initial message to reset _prev_write_time") + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + line_num = 0 + + # Write BUCKET_KB messages, and return the rate at which they were + # accepted in KB per second. + def write_bucketful(): + nonlocal line_num + start = mock_time() + max_line_num = line_num + BUCKET_KB + while line_num < max_line_num: + stream.write(message.format(line_num)) + line_num += 1 + return BUCKET_KB / (mock_time() - start) + + # The first bucketful should be written with minimal delay. The + # factor of 2 here is not arbitrary: it verifies that the system can + # write fast enough to empty the bucket within two bucketfuls, which + # the next part of the test depends on. + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) + + # Write another bucketful to empty the token bucket completely. + write_bucketful() + + # The next bucketful should be written at the rate limit. + self.assertAlmostEqual( + write_bucketful(), MAX_KB_PER_SECOND, + delta=MAX_KB_PER_SECOND * 0.1 + ) + + # Once the token bucket refills, we should go back to full speed. + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py new file mode 100644 index 00000000000000..2ca7058c14398c --- /dev/null +++ b/Lib/test/test_annotationlib.py @@ -0,0 +1,1139 @@ +"""Tests for the annotations module.""" + +import annotationlib +import builtins +import collections +import functools +import itertools +import pickle +import unittest +from annotationlib import ( + Format, + ForwardRef, + get_annotations, + get_annotate_function, + annotations_to_string, + value_to_string, +) +from typing import Unpack + +from test import support +from test.test_inspect import inspect_stock_annotations +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 + + +def times_three(fn): + @functools.wraps(fn) + def wrapper(a, b): + return fn(a * 3, b * 3) + + return wrapper + + +class MyClass: + def __repr__(self): + return "my repr" + + +class TestFormat(unittest.TestCase): + def test_enum(self): + self.assertEqual(Format.VALUE.value, 1) + self.assertEqual(Format.VALUE, 1) + + self.assertEqual(Format.FORWARDREF.value, 2) + self.assertEqual(Format.FORWARDREF, 2) + + self.assertEqual(Format.STRING.value, 3) + self.assertEqual(Format.STRING, 3) + + +class TestForwardRefFormat(unittest.TestCase): + def test_closure(self): + def inner(arg: x): + pass + + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) + fwdref = anno["arg"] + self.assertIsInstance(fwdref, annotationlib.ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = 1 + self.assertEqual(fwdref.evaluate(), x) + + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) + self.assertEqual(anno["arg"], x) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) + self.assertIs(anno["x"], int) + fwdref = anno["y"] + self.assertIsInstance(fwdref, annotationlib.ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "doesntexist") + with self.assertRaises(NameError): + fwdref.evaluate() + self.assertEqual(fwdref.evaluate(globals={"doesntexist": 1}), 1) + + def test_nonexistent_attribute(self): + def f( + x: some.module, + y: some[module], + z: some(module), + alpha: some | obj, + beta: +some, + gamma: some < obj, + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno, ForwardRef("some.module")) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno, ForwardRef("some[module]")) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno, ForwardRef("some(module)")) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno, ForwardRef("some | obj")) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno, ForwardRef("+some")) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno, ForwardRef("some < obj")) + + +class TestSourceFormat(unittest.TestCase): + def test_closure(self): + x = 0 + + def inner(arg: x): + pass + + anno = annotationlib.get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_closure_undefined(self): + if False: + x = 0 + + def inner(arg: x): + pass + + anno = annotationlib.get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) + + def test_expressions(self): + def f( + add: a + b, + sub: a - b, + mul: a * b, + matmul: a @ b, + truediv: a / b, + mod: a % b, + lshift: a << b, + rshift: a >> b, + or_: a | b, + xor: a ^ b, + and_: a & b, + floordiv: a // b, + pow_: a**b, + lt: a < b, + le: a <= b, + eq: a == b, + ne: a != b, + gt: a > b, + ge: a >= b, + invert: ~a, + neg: -a, + pos: +a, + getitem: a[b], + getattr: a.b, + call: a(b, *c, d=e), # **kwargs are not supported + *args: *a, + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "add": "a + b", + "sub": "a - b", + "mul": "a * b", + "matmul": "a @ b", + "truediv": "a / b", + "mod": "a % b", + "lshift": "a << b", + "rshift": "a >> b", + "or_": "a | b", + "xor": "a ^ b", + "and_": "a & b", + "floordiv": "a // b", + "pow_": "a ** b", + "lt": "a < b", + "le": "a <= b", + "eq": "a == b", + "ne": "a != b", + "gt": "a > b", + "ge": "a >= b", + "invert": "~a", + "neg": "-a", + "pos": "+a", + "getitem": "a[b]", + "getattr": "a.b", + "call": "a(b, *c, d=e)", + "args": "*a", + }, + ) + + def test_reverse_ops(self): + def f( + radd: 1 + a, + rsub: 1 - a, + rmul: 1 * a, + rmatmul: 1 @ a, + rtruediv: 1 / a, + rmod: 1 % a, + rlshift: 1 << a, + rrshift: 1 >> a, + ror: 1 | a, + rxor: 1 ^ a, + rand: 1 & a, + rfloordiv: 1 // a, + rpow: 1**a, + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "radd": "1 + a", + "rsub": "1 - a", + "rmul": "1 * a", + "rmatmul": "1 @ a", + "rtruediv": "1 / a", + "rmod": "1 % a", + "rlshift": "1 << a", + "rrshift": "1 >> a", + "ror": "1 | a", + "rxor": "1 ^ a", + "rand": "1 & a", + "rfloordiv": "1 // a", + "rpow": "1 ** a", + }, + ) + + def test_nested_expressions(self): + def f( + nested: list[Annotated[set[int], "set of ints", 4j]], + set: {a + b}, # single element because order is not guaranteed + dict: {a + b: c + d, "key": e + g}, + list: [a, b, c], + tuple: (a, b, c), + slice: (a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d]), + extended_slice: a[:, :, c:d], + unpack1: [*a], + unpack2: [*a, b, c], + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "nested": "list[Annotated[set[int], 'set of ints', 4j]]", + "set": "{a + b}", + "dict": "{a + b: c + d, 'key': e + g}", + "list": "[a, b, c]", + "tuple": "(a, b, c)", + "slice": "(a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d])", + "extended_slice": "a[:, :, c:d]", + "unpack1": "[*a]", + "unpack2": "[*a, b, c]", + }, + ) + + def test_unsupported_operations(self): + format_msg = "Cannot stringify annotation containing string formatting" + + def f(fstring: f"{a}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + annotationlib.get_annotations(f, format=Format.STRING) + + def f(fstring_format: f"{a:02d}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + annotationlib.get_annotations(f, format=Format.STRING) + + +class TestForwardRefClass(unittest.TestCase): + def test_special_attrs(self): + # Forward refs provide a different introspection API. __name__ and + # __qualname__ make little sense for forward refs as they can store + # complex typing expressions. + fr = annotationlib.ForwardRef("set[Any]") + self.assertFalse(hasattr(fr, "__name__")) + self.assertFalse(hasattr(fr, "__qualname__")) + self.assertEqual(fr.__module__, "annotationlib") + # Forward refs are currently unpicklable once they contain a code object. + fr.__forward_code__ # fill the cache + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(fr, proto) + + def test_evaluate_with_type_params(self): + class Gen[T]: + alias = int + + with self.assertRaises(NameError): + ForwardRef("T").evaluate() + with self.assertRaises(NameError): + ForwardRef("T").evaluate(type_params=()) + with self.assertRaises(NameError): + ForwardRef("T").evaluate(owner=int) + + (T,) = Gen.__type_params__ + self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) + self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) + + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(type_params=Gen.__type_params__) + self.assertIs(ForwardRef("alias").evaluate(owner=Gen), int) + # If you pass custom locals, we don't look at the owner's locals + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(owner=Gen, locals={}) + # But if the name exists in the locals, it works + self.assertIs( + ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str + ) + + def test_fwdref_with_module(self): + self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) + self.assertIs( + ForwardRef("Counter", module="collections").evaluate(), collections.Counter + ) + self.assertEqual( + ForwardRef("Counter[int]", module="collections").evaluate(), + collections.Counter[int], + ) + + with self.assertRaises(NameError): + # If globals are passed explicitly, we don't look at the module dict + ForwardRef("Format", module="annotationlib").evaluate(globals={}) + + def test_fwdref_to_builtin(self): + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int", module="collections").evaluate(), int) + self.assertIs(ForwardRef("int", owner=str).evaluate(), int) + + # builtins are still searched with explicit globals + self.assertIs(ForwardRef("int").evaluate(globals={}), int) + + # explicit values in globals have precedence + obj = object() + self.assertIs(ForwardRef("int").evaluate(globals={"int": obj}), obj) + + def test_fwdref_value_is_cached(self): + fr = ForwardRef("hello") + with self.assertRaises(NameError): + fr.evaluate() + self.assertIs(fr.evaluate(globals={"hello": str}), str) + self.assertIs(fr.evaluate(), str) + + def test_fwdref_with_owner(self): + self.assertEqual( + ForwardRef("Counter[int]", owner=collections).evaluate(), + collections.Counter[int], + ) + + def test_name_lookup_without_eval(self): + # test the codepath where we look up simple names directly in the + # namespaces without going through eval() + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int").evaluate(locals={"int": str}), str) + self.assertIs( + ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), + float, + ) + self.assertIs(ForwardRef("int").evaluate(globals={"int": str}), str) + with support.swap_attr(builtins, "int", dict): + self.assertIs(ForwardRef("int").evaluate(), dict) + + with self.assertRaises(NameError): + ForwardRef("doesntexist").evaluate() + + def test_fwdref_invalid_syntax(self): + fr = ForwardRef("if") + with self.assertRaises(SyntaxError): + fr.evaluate() + fr = ForwardRef("1+") + with self.assertRaises(SyntaxError): + fr.evaluate() + + +class TestGetAnnotations(unittest.TestCase): + def test_builtin_type(self): + self.assertEqual(annotationlib.get_annotations(int), {}) + self.assertEqual(annotationlib.get_annotations(object), {}) + + def test_custom_metaclass(self): + class Meta(type): + pass + + class C(metaclass=Meta): + x: int + + self.assertEqual(annotationlib.get_annotations(C), {"x": int}) + + def test_missing_dunder_dict(self): + class NoDict(type): + @property + def __dict__(cls): + raise AttributeError + + b: str + + class C1(metaclass=NoDict): + a: int + + self.assertEqual(annotationlib.get_annotations(C1), {"a": int}) + self.assertEqual( + annotationlib.get_annotations(C1, format=Format.FORWARDREF), + {"a": int}, + ) + self.assertEqual( + annotationlib.get_annotations(C1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(annotationlib.get_annotations(NoDict), {"b": str}) + self.assertEqual( + annotationlib.get_annotations(NoDict, format=Format.FORWARDREF), + {"b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(NoDict, format=Format.STRING), + {"b": "str"}, + ) + + def test_format(self): + def f1(a: int): + pass + + def f2(a: undefined): + pass + + self.assertEqual( + annotationlib.get_annotations(f1, format=Format.VALUE), + {"a": int}, + ) + self.assertEqual(annotationlib.get_annotations(f1, format=1), {"a": int}) + + fwd = annotationlib.ForwardRef("undefined") + self.assertEqual( + annotationlib.get_annotations(f2, format=Format.FORWARDREF), + {"a": fwd}, + ) + self.assertEqual(annotationlib.get_annotations(f2, format=2), {"a": fwd}) + + self.assertEqual( + annotationlib.get_annotations(f1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(annotationlib.get_annotations(f1, format=3), {"a": "int"}) + + with self.assertRaises(ValueError): + annotationlib.get_annotations(f1, format=0) + + with self.assertRaises(ValueError): + annotationlib.get_annotations(f1, format=4) + + def test_custom_object_with_annotations(self): + class C: + def __init__(self): + self.__annotations__ = {"x": int, "y": str} + + self.assertEqual(annotationlib.get_annotations(C()), {"x": int, "y": str}) + + def test_custom_format_eval_str(self): + def foo(): + pass + + with self.assertRaises(ValueError): + annotationlib.get_annotations(foo, format=Format.FORWARDREF, eval_str=True) + annotationlib.get_annotations(foo, format=Format.STRING, eval_str=True) + + def test_stock_annotations(self): + def foo(a: int, b: str): + pass + + for format in (Format.VALUE, Format.FORWARDREF): + with self.subTest(format=format): + self.assertEqual( + annotationlib.get_annotations(foo, format=format), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(foo, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + + foo.__annotations__ = {"a": "foo", "b": "str"} + for format in Format: + with self.subTest(format=format): + self.assertEqual( + annotationlib.get_annotations(foo, format=format), + {"a": "foo", "b": "str"}, + ) + + self.assertEqual( + annotationlib.get_annotations(foo, eval_str=True, locals=locals()), + {"a": foo, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(foo, eval_str=True, globals=locals()), + {"a": foo, "b": str}, + ) + + def test_stock_annotations_in_module(self): + isa = inspect_stock_annotations + + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), {"a": int, "b": str} + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(annotationlib, **kwargs), {} + ) # annotations module has no annotations + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), {"a": int, "b": str} + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": int, "b": str, "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": int, "b": str, "c": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(annotationlib, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + self.assertEqual( + annotationlib.get_annotations(isa, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(annotationlib, format=Format.STRING), + {}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, format=Format.STRING), + {}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.unannotated_function, format=Format.STRING + ), + {}, + ) + + def test_stock_annotations_on_wrapper(self): + isa = inspect_stock_annotations + + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + annotationlib.get_annotations(wrapped), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, format=Format.FORWARDREF), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=False), + {"a": int, "b": str, "return": isa.MyClass}, + ) + + def test_stringized_annotations_in_module(self): + isa = inspect_stringized_annotations + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.STRING}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + {"format": Format.STRING, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": "int", "b": "'str'", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": "'int'", "b": "'str'", "c": "'MyClass'"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), {"a": int, "b": str} + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + def test_stringized_annotations_in_empty_module(self): + isa2 = inspect_stringized_annotations_2 + self.assertEqual(annotationlib.get_annotations(isa2), {}) + self.assertEqual(annotationlib.get_annotations(isa2, eval_str=True), {}) + self.assertEqual(annotationlib.get_annotations(isa2, eval_str=False), {}) + + def test_stringized_annotations_on_wrapper(self): + isa = inspect_stringized_annotations + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + annotationlib.get_annotations(wrapped), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=False), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + + def test_stringized_annotations_on_class(self): + isa = inspect_stringized_annotations + # test that local namespace lookups work + self.assertEqual( + annotationlib.get_annotations(isa.MyClassWithLocalAnnotations), + {"x": "mytype"}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.MyClassWithLocalAnnotations, eval_str=True + ), + {"x": int}, + ) + + def test_modify_annotations(self): + def f(x: int): + pass + + self.assertEqual(annotationlib.get_annotations(f), {"x": int}) + self.assertEqual( + annotationlib.get_annotations(f, format=Format.FORWARDREF), + {"x": int}, + ) + + f.__annotations__["x"] = str + # The modification is reflected in VALUE (the default) + self.assertEqual(annotationlib.get_annotations(f), {"x": str}) + # ... and also in FORWARDREF, which tries __annotations__ if available + self.assertEqual( + annotationlib.get_annotations(f, format=Format.FORWARDREF), + {"x": str}, + ) + # ... but not in STRING which always uses __annotate__ + self.assertEqual( + annotationlib.get_annotations(f, format=Format.STRING), + {"x": "int"}, + ) + + def test_non_dict_annotations(self): + class WeirdAnnotations: + @property + def __annotations__(self): + return "not a dict" + + wa = WeirdAnnotations() + for format in Format: + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotations__ is neither a dict nor None" + ), + ): + annotationlib.get_annotations(wa, format=format) + + def test_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": int} + + ha = HasAnnotations() + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.VALUE), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.FORWARDREF), {"x": int} + ) + + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.STRING), {"x": "int"} + ) + + def test_raising_annotations_on_custom_object(self): + class HasRaisingAnnotations: + @property + def __annotations__(self): + return {"x": undefined} + + hra = HasRaisingAnnotations() + + with self.assertRaises(NameError): + annotationlib.get_annotations(hra, format=Format.VALUE) + + with self.assertRaises(NameError): + annotationlib.get_annotations(hra, format=Format.FORWARDREF) + + undefined = float + self.assertEqual( + annotationlib.get_annotations(hra, format=Format.VALUE), {"x": float} + ) + + def test_forwardref_prefers_annotations(self): + class HasBoth: + @property + def __annotations__(self): + return {"x": int} + + @property + def __annotate__(self): + return lambda format: {"x": str} + + hb = HasBoth() + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.VALUE), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.FORWARDREF), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.STRING), {"x": str} + ) + + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = annotationlib.get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = annotationlib.get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars( + self, + ): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = annotationlib.get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = annotationlib.get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + annotationlib.get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = annotationlib.get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None}, + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + annotationlib.get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars( + self, + ): + self.assertEqual( + annotationlib.get_annotations( + inspect_stringized_annotations_pep695.E, eval_str=True + ), + {"x": str}, + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.F_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__), + ) + self.assertNotEqual( + set(results.F_meth_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()).intersection( + results.F.__type_params__ + ), + set(), + ) + + self.assertEqual(results.G_annotations, {"x": str}) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__), + ) + + +class TestCallEvaluateFunction(unittest.TestCase): + def test_evaluation(self): + def evaluate(format, exc=NotImplementedError): + if format != 1: + raise exc + return undefined + + with self.assertRaises(NameError): + annotationlib.call_evaluate_function(evaluate, Format.VALUE) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.FORWARDREF), + annotationlib.ForwardRef("undefined"), + ) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.STRING), + "undefined", + ) + + +class MetaclassTests(unittest.TestCase): + def test_annotated_meta(self): + class Meta(type): + a: int + + class X(metaclass=Meta): + pass + + class Y(metaclass=Meta): + b: float + + self.assertEqual(get_annotations(Meta), {"a": int}) + self.assertEqual(get_annotate_function(Meta)(Format.VALUE), {"a": int}) + + self.assertEqual(get_annotations(X), {}) + self.assertIs(get_annotate_function(X), None) + + self.assertEqual(get_annotations(Y), {"b": float}) + self.assertEqual(get_annotate_function(Y)(Format.VALUE), {"b": float}) + + def test_unannotated_meta(self): + class Meta(type): + pass + + class X(metaclass=Meta): + a: str + + class Y(X): + pass + + self.assertEqual(get_annotations(Meta), {}) + self.assertIs(get_annotate_function(Meta), None) + + self.assertEqual(get_annotations(Y), {}) + self.assertIs(get_annotate_function(Y), None) + + self.assertEqual(get_annotations(X), {"a": str}) + self.assertEqual(get_annotate_function(X)(Format.VALUE), {"a": str}) + + def test_ordering(self): + # Based on a sample by David Ellis + # https://discuss.python.org/t/pep-749-implementing-pep-649/54974/38 + + def make_classes(): + class Meta(type): + a: int + expected_annotations = {"a": int} + + class A(type, metaclass=Meta): + b: float + expected_annotations = {"b": float} + + class B(metaclass=A): + c: str + expected_annotations = {"c": str} + + class C(B): + expected_annotations = {} + + class D(metaclass=Meta): + expected_annotations = {} + + return Meta, A, B, C, D + + classes = make_classes() + class_count = len(classes) + for order in itertools.permutations(range(class_count), class_count): + names = ", ".join(classes[i].__name__ for i in order) + with self.subTest(names=names): + classes = make_classes() # Regenerate classes + for i in order: + get_annotations(classes[i]) + for c in classes: + with self.subTest(c=c): + self.assertEqual(get_annotations(c), c.expected_annotations) + annotate_func = get_annotate_function(c) + if c.expected_annotations: + self.assertEqual( + annotate_func(Format.VALUE), c.expected_annotations + ) + else: + self.assertIs(annotate_func, None) + + +class TestGetAnnotateFunction(unittest.TestCase): + def test_static_class(self): + self.assertIsNone(get_annotate_function(object)) + self.assertIsNone(get_annotate_function(int)) + + def test_unannotated_class(self): + class C: + pass + + self.assertIsNone(get_annotate_function(C)) + + D = type("D", (), {}) + self.assertIsNone(get_annotate_function(D)) + + def test_annotated_class(self): + class C: + a: int + + self.assertEqual(get_annotate_function(C)(Format.VALUE), {"a": int}) + + +class TestToSource(unittest.TestCase): + def test_value_to_string(self): + self.assertEqual(value_to_string(int), "int") + self.assertEqual(value_to_string(MyClass), "test.test_annotationlib.MyClass") + self.assertEqual(value_to_string(len), "len") + self.assertEqual(value_to_string(value_to_string), "value_to_string") + self.assertEqual(value_to_string(times_three), "times_three") + self.assertEqual(value_to_string(...), "...") + self.assertEqual(value_to_string(None), "None") + self.assertEqual(value_to_string(1), "1") + self.assertEqual(value_to_string("1"), "'1'") + self.assertEqual(value_to_string(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(value_to_string(MyClass()), "my repr") + + def test_annotations_to_string(self): + self.assertEqual(annotations_to_string({}), {}) + self.assertEqual(annotations_to_string({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_string({"x": "int"}), {"x": "int"}) + self.assertEqual( + annotations_to_string({"x": int, "y": str}), {"x": "int", "y": "str"} + ) + + +class TestAnnotationLib(unittest.TestCase): + def test__all__(self): + support.check__all__(self, annotationlib) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 86d6e81a71642b..ba9876570385d3 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -6,8 +6,11 @@ import io import operator import os +import py_compile +import re import shutil import stat +import subprocess import sys import textwrap import tempfile @@ -15,10 +18,22 @@ import argparse import warnings -from test.support import os_helper, captured_stderr +from enum import StrEnum +from pathlib import Path +from test.support import REPO_ROOT +from test.support import TEST_HOME_DIR +from test.support import captured_stderr +from test.support import import_helper +from test.support import os_helper +from test.support import requires_subprocess +from test.support import script_helper +from test.test_tools import skip_if_missing from unittest import mock +py = os.path.basename(sys.executable) + + class StdIOBuffer(io.TextIOWrapper): '''Replacement for writable io.StringIO that behaves more like real file @@ -280,16 +295,18 @@ def test_failures(self, tester): parser = self._get_parser(tester) for args_str in tester.failures: args = args_str.split() - with tester.assertRaises(ArgumentParserError, msg=args): - parser.parse_args(args) + with tester.subTest(args=args): + with tester.assertRaises(ArgumentParserError, msg=args): + parser.parse_args(args) def test_successes(self, tester): parser = self._get_parser(tester) for args, expected_ns in tester.successes: if isinstance(args, str): args = args.split() - result_ns = self._parse_args(parser, args) - tester.assertEqual(expected_ns, result_ns) + with tester.subTest(args=args): + result_ns = self._parse_args(parser, args) + tester.assertEqual(expected_ns, result_ns) # add tests for each combination of an optionals adding method # and an arg parsing method @@ -378,15 +395,22 @@ class TestOptionalsSingleDashAmbiguous(ParserTestCase): """Test Optionals that partially match but are not subsets""" argument_signatures = [Sig('-foobar'), Sig('-foorab')] - failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b'] + failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b', + '-f=a', '-foo=b'] successes = [ ('', NS(foobar=None, foorab=None)), ('-foob a', NS(foobar='a', foorab=None)), + ('-foob=a', NS(foobar='a', foorab=None)), ('-foor a', NS(foobar=None, foorab='a')), + ('-foor=a', NS(foobar=None, foorab='a')), ('-fooba a', NS(foobar='a', foorab=None)), + ('-fooba=a', NS(foobar='a', foorab=None)), ('-foora a', NS(foobar=None, foorab='a')), + ('-foora=a', NS(foobar=None, foorab='a')), ('-foobar a', NS(foobar='a', foorab=None)), + ('-foobar=a', NS(foobar='a', foorab=None)), ('-foorab a', NS(foobar=None, foorab='a')), + ('-foorab=a', NS(foobar=None, foorab='a')), ] @@ -619,9 +643,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -677,7 +701,7 @@ class TestOptionalsChoices(ParserTestCase): argument_signatures = [ Sig('-f', choices='abc'), Sig('-g', type=int, choices=range(5))] - failures = ['a', '-f d', '-fad', '-ga', '-g 6'] + failures = ['a', '-f d', '-f ab', '-fad', '-ga', '-g 6'] successes = [ ('', NS(f=None, g=None)), ('-f a', NS(f='a', g=None)), @@ -765,49 +789,6 @@ def test_const(self): self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) - def test_deprecated_init_kw(self): - # See gh-92248 - parser = argparse.ArgumentParser() - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-a', - action=argparse.BooleanOptionalAction, - type=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-b', - action=argparse.BooleanOptionalAction, - type=bool, - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-c', - action=argparse.BooleanOptionalAction, - metavar=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-d', - action=argparse.BooleanOptionalAction, - metavar='d', - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-e', - action=argparse.BooleanOptionalAction, - choices=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-f', - action=argparse.BooleanOptionalAction, - choices=(), - ) - class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" @@ -916,7 +897,9 @@ class TestOptionalsAllowLongAbbreviation(ParserTestCase): successes = [ ('', NS(foo=None, foobaz=None, fooble=False)), ('--foo 7', NS(foo='7', foobaz=None, fooble=False)), + ('--foo=7', NS(foo='7', foobaz=None, fooble=False)), ('--fooba a', NS(foo=None, foobaz='a', fooble=False)), + ('--fooba=a', NS(foo=None, foobaz='a', fooble=False)), ('--foobl --foo g', NS(foo='g', foobaz=None, fooble=True)), ] @@ -955,6 +938,23 @@ class TestOptionalsDisallowLongAbbreviationPrefixChars(ParserTestCase): ] +class TestOptionalsDisallowSingleDashLongAbbreviation(ParserTestCase): + """Do not allow abbreviations of long options at all""" + + parser_signature = Sig(allow_abbrev=False) + argument_signatures = [ + Sig('-foo'), + Sig('-foodle', action='store_true'), + Sig('-foonly'), + ] + failures = ['-foon 3', '-food', '-food -foo 2'] + successes = [ + ('', NS(foo=None, foodle=False, foonly=None)), + ('-foo 3', NS(foo='3', foodle=False, foonly=None)), + ('-foonly 7 -foodle -foo 2', NS(foo='2', foodle=True, foonly='7')), + ] + + class TestDisallowLongAbbreviationAllowsShortGrouping(ParserTestCase): """Do not allow abbreviations of long options at all""" @@ -993,6 +993,34 @@ class TestDisallowLongAbbreviationAllowsShortGroupingPrefix(ParserTestCase): ] +class TestStrEnumChoices(TestCase): + class Color(StrEnum): + RED = "red" + GREEN = "green" + BLUE = "blue" + + def test_parse_enum_value(self): + parser = argparse.ArgumentParser() + parser.add_argument('--color', choices=self.Color) + args = parser.parse_args(['--color', 'red']) + self.assertEqual(args.color, self.Color.RED) + + def test_help_message_contains_enum_choices(self): + parser = argparse.ArgumentParser() + parser.add_argument('--color', choices=self.Color, help='Choose a color') + self.assertIn('[--color {red,green,blue}]', parser.format_usage()) + self.assertIn(' --color {red,green,blue}', parser.format_help()) + + def test_invalid_enum_value_raises_error(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--color', choices=self.Color) + self.assertRaisesRegex( + argparse.ArgumentError, + r"invalid choice: 'yellow' \(choose from red, green, blue\)", + parser.parse_args, + ['--color', 'yellow'], + ) + # ================ # Positional tests # ================ @@ -1042,8 +1070,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1076,8 +1104,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1090,9 +1118,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), @@ -1132,57 +1160,87 @@ class TestPositionalsNargs2None(ParserTestCase): class TestPositionalsNargsNoneZeroOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with unlimited""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='*')] - failures = ['', '--foo'] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='*')] + failures = ['', '--foo', 'a b -x X c'] successes = [ - ('a', NS(foo='a', bar=[])), - ('a b', NS(foo='a', bar=['b'])), - ('a b c', NS(foo='a', bar=['b', 'c'])), + ('a', NS(x=None, foo='a', bar=[])), + ('a b', NS(x=None, foo='a', bar=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b', 'c'])), + ('-x X a', NS(x='X', foo='a', bar=[])), + ('a -x X', NS(x='X', foo='a', bar=[])), + ('-x X a b', NS(x='X', foo='a', bar=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOneOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with one or more""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='+')] - failures = ['', '--foo', 'a'] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='+')] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=['b'])), - ('a b c', NS(foo='a', bar=['b', 'c'])), + ('a b', NS(x=None, foo='a', bar=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b', 'c'])), + ('-x X a b', NS(x='X', foo='a', bar=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOptional(ParserTestCase): """Test a Positional with no nargs followed by one with an Optional""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='?')] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='?')] failures = ['', '--foo', 'a b c'] successes = [ - ('a', NS(foo='a', bar=None)), - ('a b', NS(foo='a', bar='b')), + ('a', NS(x=None, foo='a', bar=None)), + ('a b', NS(x=None, foo='a', bar='b')), + ('-x X a', NS(x='X', foo='a', bar=None)), + ('a -x X', NS(x='X', foo='a', bar=None)), + ('-x X a b', NS(x='X', foo='a', bar='b')), + ('a -x X b', NS(x='X', foo='a', bar='b')), + ('a b -x X', NS(x='X', foo='a', bar='b')), ] class TestPositionalsNargsZeroOrMoreNone(ParserTestCase): """Test a Positional with unlimited nargs followed by one with none""" - argument_signatures = [Sig('foo', nargs='*'), Sig('bar')] - failures = ['', '--foo'] + argument_signatures = [Sig('-x'), Sig('foo', nargs='*'), Sig('bar')] + failures = ['', '--foo', 'a -x X b', 'a -x X b c', 'a b -x X c'] successes = [ - ('a', NS(foo=[], bar='a')), - ('a b', NS(foo=['a'], bar='b')), - ('a b c', NS(foo=['a', 'b'], bar='c')), + ('a', NS(x=None, foo=[], bar='a')), + ('a b', NS(x=None, foo=['a'], bar='b')), + ('a b c', NS(x=None, foo=['a', 'b'], bar='c')), + ('-x X a', NS(x='X', foo=[], bar='a')), + ('a -x X', NS(x='X', foo=[], bar='a')), + ('-x X a b', NS(x='X', foo=['a'], bar='b')), + ('a b -x X', NS(x='X', foo=['a'], bar='b')), + ('-x X a b c', NS(x='X', foo=['a', 'b'], bar='c')), + ('a b c -x X', NS(x='X', foo=['a', 'b'], bar='c')), ] class TestPositionalsNargsOneOrMoreNone(ParserTestCase): """Test a Positional with one or more nargs followed by one with none""" - argument_signatures = [Sig('foo', nargs='+'), Sig('bar')] - failures = ['', '--foo', 'a'] + argument_signatures = [Sig('-x'), Sig('foo', nargs='+'), Sig('bar')] + failures = ['', '--foo', 'a', 'a -x X b c', 'a b -x X c'] successes = [ - ('a b', NS(foo=['a'], bar='b')), - ('a b c', NS(foo=['a', 'b'], bar='c')), + ('a b', NS(x=None, foo=['a'], bar='b')), + ('a b c', NS(x=None, foo=['a', 'b'], bar='c')), + ('-x X a b', NS(x='X', foo=['a'], bar='b')), + ('a -x X b', NS(x='X', foo=['a'], bar='b')), + ('a b -x X', NS(x='X', foo=['a'], bar='b')), + ('-x X a b c', NS(x='X', foo=['a', 'b'], bar='c')), + ('a b c -x X', NS(x='X', foo=['a', 'b'], bar='c')), ] @@ -1267,14 +1325,21 @@ class TestPositionalsNargsNoneZeroOrMore1(ParserTestCase): """Test three Positionals: no nargs, unlimited nargs and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='*'), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a'] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=[], baz=['b'])), - ('a b c', NS(foo='a', bar=['b'], baz=['c'])), + ('a b', NS(x=None, foo='a', bar=[], baz=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b'], baz=['c'])), + ('-x X a b', NS(x='X', foo='a', bar=[], baz=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=[], baz=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=[], baz=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b'], baz=['c'])), ] @@ -1282,14 +1347,22 @@ class TestPositionalsNargsNoneOneOrMore1(ParserTestCase): """Test three Positionals: no nargs, one or more nargs and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='+'), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a', 'b'] + failures = ['', '--foo', 'a', 'b', 'a b -x X c d', 'a b c -x X d'] successes = [ - ('a b c', NS(foo='a', bar=['b'], baz=['c'])), - ('a b c d', NS(foo='a', bar=['b', 'c'], baz=['d'])), + ('a b c', NS(x=None, foo='a', bar=['b'], baz=['c'])), + ('a b c d', NS(x=None, foo='a', bar=['b', 'c'], baz=['d'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b -x X c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('-x X a b c d', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), + ('a -x X b c d', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), + ('a b c d -x X', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), ] @@ -1297,14 +1370,21 @@ class TestPositionalsNargsNoneOptional1(ParserTestCase): """Test three Positionals: no nargs, optional narg and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='?', default=0.625), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a'] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=0.625, baz=['b'])), - ('a b c', NS(foo='a', bar='b', baz=['c'])), + ('a b', NS(x=None, foo='a', bar=0.625, baz=['b'])), + ('a b c', NS(x=None, foo='a', bar='b', baz=['c'])), + ('-x X a b', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar='b', baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar='b', baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar='b', baz=['c'])), ] @@ -1520,6 +1600,9 @@ class TestNargsRemainder(ParserTestCase): successes = [ ('X', NS(x='X', y=[], z=None)), ('-z Z X', NS(x='X', y=[], z='Z')), + ('-z Z X A B', NS(x='X', y=['A', 'B'], z='Z')), + ('X -z Z A B', NS(x='X', y=['-z', 'Z', 'A', 'B'], z=None)), + ('X A -z Z B', NS(x='X', y=['A', '-z', 'Z', 'B'], z=None)), ('X A B -z Z', NS(x='X', y=['A', 'B', '-z', 'Z'], z=None)), ('X Y --foo', NS(x='X', y=['Y', '--foo'], z=None)), ] @@ -1556,18 +1639,24 @@ class TestDefaultSuppress(ParserTestCase): """Test actions with suppressed defaults""" argument_signatures = [ - Sig('foo', nargs='?', default=argparse.SUPPRESS), - Sig('bar', nargs='*', default=argparse.SUPPRESS), + Sig('foo', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('bar', nargs='*', type=int, default=argparse.SUPPRESS), Sig('--baz', action='store_true', default=argparse.SUPPRESS), + Sig('--qux', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('--quux', nargs='*', type=int, default=argparse.SUPPRESS), ] - failures = ['-x'] + failures = ['-x', 'a', '1 a'] successes = [ ('', NS()), - ('a', NS(foo='a')), - ('a b', NS(foo='a', bar=['b'])), + ('1', NS(foo=1)), + ('1 2', NS(foo=1, bar=[2])), ('--baz', NS(baz=True)), - ('a --baz', NS(foo='a', baz=True)), - ('--baz a b', NS(foo='a', bar=['b'], baz=True)), + ('1 --baz', NS(foo=1, baz=True)), + ('--baz 1 2', NS(foo=1, bar=[2], baz=True)), + ('--qux', NS(qux=None)), + ('--qux 1', NS(qux=1)), + ('--quux', NS(quux=[])), + ('--quux 1 2', NS(quux=[1, 2])), ] @@ -1691,27 +1780,43 @@ def convert_arg_line_to_args(self, arg_line): # Type conversion tests # ===================== +def FileType(*args, **kwargs): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'FileType is deprecated', + PendingDeprecationWarning, __name__) + return argparse.FileType(*args, **kwargs) + + +class TestFileTypeDeprecation(TestCase): + + def test(self): + with self.assertWarns(PendingDeprecationWarning) as cm: + argparse.FileType() + self.assertIn('FileType is deprecated', str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + class TestFileTypeRepr(TestCase): def test_r(self): - type = argparse.FileType('r') + type = FileType('r') self.assertEqual("FileType('r')", repr(type)) def test_wb_1(self): - type = argparse.FileType('wb', 1) + type = FileType('wb', 1) self.assertEqual("FileType('wb', 1)", repr(type)) def test_r_latin(self): - type = argparse.FileType('r', encoding='latin_1') + type = FileType('r', encoding='latin_1') self.assertEqual("FileType('r', encoding='latin_1')", repr(type)) def test_w_big5_ignore(self): - type = argparse.FileType('w', encoding='big5', errors='ignore') + type = FileType('w', encoding='big5', errors='ignore') self.assertEqual("FileType('w', encoding='big5', errors='ignore')", repr(type)) def test_r_1_replace(self): - type = argparse.FileType('r', 1, errors='replace') + type = FileType('r', 1, errors='replace') self.assertEqual("FileType('r', 1, errors='replace')", repr(type)) @@ -1765,7 +1870,6 @@ def __eq__(self, other): text = text.decode('ascii') return self.name == other.name == text - class TestFileTypeR(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for reading files""" @@ -1778,8 +1882,8 @@ def setUp(self): self.create_readonly_file('readonly') argument_signatures = [ - Sig('-x', type=argparse.FileType()), - Sig('spam', type=argparse.FileType('r')), + Sig('-x', type=FileType()), + Sig('spam', type=FileType('r')), ] failures = ['-x', '', 'non-existent-file.txt'] successes = [ @@ -1799,7 +1903,7 @@ def setUp(self): file.close() argument_signatures = [ - Sig('-c', type=argparse.FileType('r'), default='no-file.txt'), + Sig('-c', type=FileType('r'), default='no-file.txt'), ] # should provoke no such file error failures = [''] @@ -1818,8 +1922,8 @@ def setUp(self): file.write(file_name) argument_signatures = [ - Sig('-x', type=argparse.FileType('rb')), - Sig('spam', type=argparse.FileType('rb')), + Sig('-x', type=FileType('rb')), + Sig('spam', type=FileType('rb')), ] failures = ['-x', ''] successes = [ @@ -1857,8 +1961,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('w')), - Sig('spam', type=argparse.FileType('w')), + Sig('-x', type=FileType('w')), + Sig('spam', type=FileType('w')), ] failures = ['-x', '', 'readonly'] successes = [ @@ -1880,8 +1984,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('x')), - Sig('spam', type=argparse.FileType('x')), + Sig('-x', type=FileType('x')), + Sig('spam', type=FileType('x')), ] failures = ['-x', '', 'readonly', 'writable'] successes = [ @@ -1895,8 +1999,8 @@ class TestFileTypeWB(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for writing binary files""" argument_signatures = [ - Sig('-x', type=argparse.FileType('wb')), - Sig('spam', type=argparse.FileType('wb')), + Sig('-x', type=FileType('wb')), + Sig('spam', type=FileType('wb')), ] failures = ['-x', ''] successes = [ @@ -1912,8 +2016,8 @@ class TestFileTypeXB(TestFileTypeX): "Test the FileType option/argument type for writing new binary files only" argument_signatures = [ - Sig('-x', type=argparse.FileType('xb')), - Sig('spam', type=argparse.FileType('xb')), + Sig('-x', type=FileType('xb')), + Sig('spam', type=FileType('xb')), ] successes = [ ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), @@ -1925,7 +2029,7 @@ class TestFileTypeOpenArgs(TestCase): """Test that open (the builtin) is correctly called""" def test_open_args(self): - FT = argparse.FileType + FT = FileType cases = [ (FT('rb'), ('rb', -1, None, None)), (FT('w', 1), ('w', 1, None, None)), @@ -1940,7 +2044,7 @@ def test_open_args(self): def test_invalid_file_type(self): with self.assertRaises(ValueError): - argparse.FileType('b')('-test') + FileType('b')('-test') class TestFileTypeMissingInitialization(TestCase): @@ -1951,7 +2055,7 @@ class TestFileTypeMissingInitialization(TestCase): def test(self): parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('-x', type=argparse.FileType) self.assertEqual( @@ -2136,6 +2240,130 @@ class TestActionExtend(ParserTestCase): ] +class TestNegativeNumber(ParserTestCase): + """Test parsing negative numbers""" + + argument_signatures = [ + Sig('--int', type=int), + Sig('--float', type=float), + Sig('--complex', type=complex), + ] + failures = [ + '--float -_.45', + '--float -1__000.0', + '--float -1.0.0', + '--int -1__000', + '--int -1.0', + '--complex -1__000.0j', + '--complex -1.0jj', + '--complex -_.45j', + ] + successes = [ + ('--int -1000 --float -1000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000 --float -1_000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000_000 --float -1_000_000.0', NS(int=-1000000, float=-1000000.0, complex=None)), + ('--float -1_000.0', NS(int=None, float=-1000.0, complex=None)), + ('--float -1_000_000.0_0', NS(int=None, float=-1000000.0, complex=None)), + ('--float -.5', NS(int=None, float=-0.5, complex=None)), + ('--float -.5_000', NS(int=None, float=-0.5, complex=None)), + ('--float -1e3', NS(int=None, float=-1000, complex=None)), + ('--float -1e-3', NS(int=None, float=-0.001, complex=None)), + ('--complex -1j', NS(int=None, float=None, complex=-1j)), + ('--complex -1_000j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1_000.0j', NS(int=None, float=None, complex=-1000.0j)), + ('--complex -1e3j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), + ] + +class TestArgumentAndSubparserSuggestions(TestCase): + """Test error handling and suggestion when a user makes a typo""" + + def test_wrong_argument_error_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from bar, baz)", + excinfo.exception.stderr + ) + + def test_wrong_argument_error_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz', maybe you meant" + " 'bar'? (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz' (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_no_suggestion_implicit(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_empty(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from )", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_int(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, 2]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_mixed_types(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, '2']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + class TestInvalidAction(TestCase): """Test invalid user defined Action""" @@ -2149,11 +2377,24 @@ def test_invalid_type(self): self.assertRaises(NotImplementedError, parser.parse_args, ['--foo', 'bar']) def test_modified_invalid_action(self): - parser = ErrorRaisingArgumentParser() + parser = argparse.ArgumentParser(exit_on_error=False) action = parser.add_argument('--foo') # Someone got crazy and did this action.type = 1 - self.assertRaises(ArgumentParserError, parser.parse_args, ['--foo', 'bar']) + self.assertRaisesRegex(TypeError, '1 is not callable', + parser.parse_args, ['--foo', 'bar']) + action.type = () + self.assertRaisesRegex(TypeError, r'\(\) is not callable', + parser.parse_args, ['--foo', 'bar']) + # It is impossible to distinguish a TypeError raised due to a mismatch + # of the required function arguments from a TypeError raised for an incorrect + # argument value, and using the heavy inspection machinery is not worthwhile + # as it does not reliably work in all cases. + # Therefore, a generic ArgumentError is raised to handle this logical error. + action.type = pow + self.assertRaisesRegex(argparse.ArgumentError, + "argument --foo: invalid pow value: 'bar'", + parser.parse_args, ['--foo', 'bar']) # ================ @@ -2190,7 +2431,9 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, else: subparsers_kwargs['help'] = 'command help' subparsers = parser.add_subparsers(**subparsers_kwargs) - self.assertArgumentParserError(parser.add_subparsers) + self.assertRaisesRegex(ValueError, + 'cannot have multiple subparser arguments', + parser.add_subparsers) # add first sub-parser parser1_kwargs = dict(description='1 description') @@ -2200,14 +2443,14 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser1_kwargs['aliases'] = ['1alias1', '1alias2'] parser1 = subparsers.add_parser('1', **parser1_kwargs) parser1.add_argument('-w', type=int, help='w help') - parser1.add_argument('x', choices='abc', help='x help') + parser1.add_argument('x', choices=['a', 'b', 'c'], help='x help') # add second sub-parser parser2_kwargs = dict(description='2 description') if subparser_help: parser2_kwargs['help'] = '2 help' parser2 = subparsers.add_parser('2', **parser2_kwargs) - parser2.add_argument('-y', choices='123', help='y help') + parser2.add_argument('-y', choices=['1', '2', '3'], help='y help') parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser @@ -2274,6 +2517,68 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_to_class_namespace(self): + class C: + pass + self.assertEqual( + self.parser.parse_known_args('0.5 1 b -w 7 -p'.split(), namespace=C), + (C, ['-p']), + ) + self.assertIs(C.foo, False) + self.assertEqual(C.bar, 0.5) + self.assertEqual(C.w, 7) + self.assertEqual(C.x, 'b') + + def test_abbreviation(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('--foodle') + parser.add_argument('--foonly') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('bar') + parser1.add_argument('--fo') + parser1.add_argument('--foonew') + + self.assertEqual(parser.parse_args(['--food', 'baz', 'bar']), + NS(foodle='baz', foonly=None, fo=None, foonew=None)) + self.assertEqual(parser.parse_args(['--foon', 'baz', 'bar']), + NS(foodle=None, foonly='baz', fo=None, foonew=None)) + self.assertArgumentParserError(parser.parse_args, ['--fo', 'baz', 'bar']) + self.assertEqual(parser.parse_args(['bar', '--fo', 'baz']), + NS(foodle=None, foonly=None, fo='baz', foonew=None)) + self.assertEqual(parser.parse_args(['bar', '--foo', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertEqual(parser.parse_args(['bar', '--foon', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertArgumentParserError(parser.parse_args, ['bar', '--food', 'baz']) + + def test_parse_known_args_with_single_dash_option(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('-k', '--known', action='count', default=0) + parser.add_argument('-n', '--new', action='count', default=0) + self.assertEqual(parser.parse_known_args(['-k', '-u']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-u', '-k']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-ku']), + (NS(known=1, new=0), ['-u'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=u']) + self.assertEqual(parser.parse_known_args(['-uk']), + (NS(known=0, new=0), ['-uk'])) + self.assertEqual(parser.parse_known_args(['-u=k']), + (NS(known=0, new=0), ['-u=k'])) + self.assertEqual(parser.parse_known_args(['-kunknown']), + (NS(known=1, new=0), ['-unknown'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=unknown']) + self.assertEqual(parser.parse_known_args(['-ku=nknown']), + (NS(known=1, new=0), ['-u=nknown'])) + self.assertEqual(parser.parse_known_args(['-knew']), + (NS(known=1, new=1), ['-ew'])) + self.assertArgumentParserError(parser.parse_known_args, ['-kn=ew']) + self.assertArgumentParserError(parser.parse_known_args, ['-k-new']) + self.assertArgumentParserError(parser.parse_known_args, ['-kn-ew']) + self.assertEqual(parser.parse_known_args(['-kne-w']), + (NS(known=1, new=1), ['-e-w'])) + def test_dest(self): parser = ErrorRaisingArgumentParser() parser.add_argument('--foo', action='store_true') @@ -2324,18 +2629,6 @@ def test_required_subparsers_no_destination_error(self): 'error: the following arguments are required: {foo,bar}\n$' ) - def test_wrong_argument_subparsers_no_destination_error(self): - parser = ErrorRaisingArgumentParser() - subparsers = parser.add_subparsers(required=True) - subparsers.add_parser('foo') - subparsers.add_parser('bar') - with self.assertRaises(ArgumentParserError) as excinfo: - parser.parse_args(('baz',)) - self.assertRegex( - excinfo.exception.stderr, - r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from 'foo', 'bar'\)\n$" - ) - def test_optional_subparsers(self): parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers(dest='command', required=False) @@ -2471,6 +2764,29 @@ def test_parser_command_help(self): --foo foo help ''')) + def assert_bad_help(self, context_type, func, *args, **kwargs): + with self.assertRaisesRegex(ValueError, 'badly formed help string') as cm: + func(*args, **kwargs) + self.assertIsInstance(cm.exception.__context__, context_type) + + def test_invalid_subparsers_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(ValueError, parser.add_subparsers, help='%Y-%m-%d') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(KeyError, parser.add_subparsers, help='%(spam)s') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(TypeError, parser.add_subparsers, help='%(prog)d') + + def test_invalid_subparser_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + subparsers = parser.add_subparsers() + self.assert_bad_help(ValueError, subparsers.add_parser, '1', + help='%Y-%m-%d') + self.assert_bad_help(KeyError, subparsers.add_parser, '1', + help='%(spam)s') + self.assert_bad_help(TypeError, subparsers.add_parser, '1', + help='%(prog)d') + def test_subparser_title_help(self): parser = ErrorRaisingArgumentParser(prog='PROG', description='main description') @@ -2612,6 +2928,31 @@ def test_interleaved_groups(self): result = parser.parse_args('1 2 3 4'.split()) self.assertEqual(expected, result) +class TestGroupConstructor(TestCase): + def test_group_prefix_chars(self): + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-+') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_group_prefix_chars_default(self): + # "default" isn't quite the right word here, but it's the same as + # the parser's default prefix so it's a good test + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + # =================== # Parent parser tests # =================== @@ -2648,8 +2989,6 @@ def setUp(self): group.add_argument('-a', action='store_true') group.add_argument('-b', action='store_true') - self.main_program = os.path.basename(sys.argv[0]) - def test_single_parent(self): parser = ErrorRaisingArgumentParser(parents=[self.wxyz_parent]) self.assertEqual(parser.parse_args('-y 1 2 --w 3'.split()), @@ -2660,7 +2999,7 @@ def test_single_parent_mutex(self): parser = ErrorRaisingArgumentParser(parents=[self.ab_mutex_parent]) self._test_mutex_ab(parser.parse_args) - def test_single_granparent_mutex(self): + def test_single_grandparent_mutex(self): parents = [self.ab_mutex_parent] parser = ErrorRaisingArgumentParser(add_help=False, parents=parents) parser = ErrorRaisingArgumentParser(parents=[parser]) @@ -2739,11 +3078,10 @@ def test_subparser_parents_mutex(self): def test_parent_help(self): parents = [self.abcd_parent, self.wxyz_parent] - parser = ErrorRaisingArgumentParser(parents=parents) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=parents) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-b B] [--d D] [--w W] [-y Y] a z + usage: PROG [-h] [-b B] [--d D] [--w W] [-y Y] a z positional arguments: a @@ -2759,7 +3097,7 @@ def test_parent_help(self): x: -y Y - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_groups_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) @@ -2769,15 +3107,14 @@ def test_groups_parents(self): m = parent.add_mutually_exclusive_group() m.add_argument('-y') m.add_argument('-z') - parser = ErrorRaisingArgumentParser(parents=[parent]) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) self.assertRaises(ArgumentParserError, parser.parse_args, ['-y', 'Y', '-z', 'Z']) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-w W] [-x X] [-y Y | -z Z] + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] options: -h, --help show this help message and exit @@ -2789,11 +3126,40 @@ def test_groups_parents(self): -w W -x X - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_wrong_type_parents(self): self.assertRaises(TypeError, ErrorRaisingArgumentParser, parents=[1]) + def test_mutex_groups_parents(self): + parent = ErrorRaisingArgumentParser(add_help=False) + g = parent.add_argument_group(title='g', description='gd') + g.add_argument('-w') + g.add_argument('-x') + m = g.add_mutually_exclusive_group() + m.add_argument('-y') + m.add_argument('-z') + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) + + self.assertRaises(ArgumentParserError, parser.parse_args, + ['-y', 'Y', '-z', 'Z']) + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] + + options: + -h, --help show this help message and exit + + g: + gd + + -w W + -x X + -y Y + -z Z + ''')) + # ============================== # Mutually exclusive group tests # ============================== @@ -2836,6 +3202,50 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_optional_order(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--foo') + group.add_argument('bar', nargs='?') + expected = '''\ + usage: PROG [-h] (--foo FOO | bar) + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('bar', nargs='?') + group.add_argument('--foo') + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): + self.maxDiff = None + parser = ErrorRaisingArgumentParser(prog='PROG') + commands = parser.add_subparsers(title="commands", dest="command") + cmd_foo = commands.add_parser("foo") + group = cmd_foo.add_mutually_exclusive_group() + group.add_argument('--verbose', action='store_true', help=argparse.SUPPRESS) + group.add_argument('--quiet', action='store_true', help=argparse.SUPPRESS) + longopt = '--' + 'long'*32 + longmeta = 'LONG'*32 + cmd_foo.add_argument(longopt) + expected = f'''\ + usage: PROG foo [-h] + [{longopt} {longmeta}] + + options: + -h, --help show this help message and exit + {longopt} {longmeta} + ''' + self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) + def test_empty_group(self): # See issue 26952 parser = argparse.ArgumentParser() @@ -2849,26 +3259,30 @@ def test_failures_when_not_required(self): parse_args = self.get_parser(required=False).parse_args error = ArgumentParserError for args_string in self.failures: - self.assertRaises(error, parse_args, args_string.split()) + with self.subTest(args=args_string): + self.assertRaises(error, parse_args, args_string.split()) def test_failures_when_required(self): parse_args = self.get_parser(required=True).parse_args error = ArgumentParserError for args_string in self.failures + ['']: - self.assertRaises(error, parse_args, args_string.split()) + with self.subTest(args=args_string): + self.assertRaises(error, parse_args, args_string.split()) def test_successes_when_not_required(self): parse_args = self.get_parser(required=False).parse_args successes = self.successes + self.successes_when_not_required for args_string, expected_ns in successes: - actual_ns = parse_args(args_string.split()) - self.assertEqual(actual_ns, expected_ns) + with self.subTest(args=args_string): + actual_ns = parse_args(args_string.split()) + self.assertEqual(actual_ns, expected_ns) def test_successes_when_required(self): parse_args = self.get_parser(required=True).parse_args for args_string, expected_ns in self.successes: - actual_ns = parse_args(args_string.split()) - self.assertEqual(actual_ns, expected_ns) + with self.subTest(args=args_string): + actual_ns = parse_args(args_string.split()) + self.assertEqual(actual_ns, expected_ns) def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage @@ -2951,12 +3365,12 @@ def get_parser(self, required=None): ] usage_when_not_required = '''\ - usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] - [--klmno KLMNO | --pqrst PQRST] + usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] [--klmno KLMNO | + --pqrst PQRST] ''' usage_when_required = '''\ - usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] - (--klmno KLMNO | --pqrst PQRST) + usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] (--klmno KLMNO | + --pqrst PQRST) ''' help = '''\ @@ -3045,7 +3459,7 @@ def get_parser(self, required): group = parser.add_mutually_exclusive_group(required=required) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') - group.add_argument('badger', nargs='*', default='X', help='BADGER') + group.add_argument('badger', nargs='*', help='BADGER') return parser failures = [ @@ -3056,13 +3470,13 @@ def get_parser(self, required): '--foo X Y', ] successes = [ - ('--foo', NS(foo=True, spam=None, badger='X')), - ('--spam S', NS(foo=False, spam='S', badger='X')), + ('--foo', NS(foo=True, spam=None, badger=[])), + ('--spam S', NS(foo=False, spam='S', badger=[])), ('X', NS(foo=False, spam=None, badger=['X'])), ('X Y Z', NS(foo=False, spam=None, badger=['X', 'Y', 'Z'])), ] successes_when_not_required = [ - ('', NS(foo=False, spam=None, badger='X')), + ('', NS(foo=False, spam=None, badger=[])), ] usage_when_not_required = '''\ @@ -3255,6 +3669,111 @@ def get_parser(self, required): test_successes_when_not_required = None test_successes_when_required = None + +class TestMutuallyExclusiveOptionalOptional(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('--bar', nargs='?') + return parser + + failures = [ + '--foo X --bar Y', + '--foo X --bar', + ] + successes = [ + ('--foo X', NS(foo='X', bar=None)), + ('--bar X', NS(foo=None, bar='X')), + ('--bar', NS(foo=None, bar=None)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=None)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | --bar [BAR]) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | --bar [BAR]] + ''' + help = '''\ + + options: + -h, --help show this help message and exit + --foo FOO + --bar [BAR] + ''' + + +class TestMutuallyExclusiveOptionalWithDefault(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('--bar', type=bool, default=True) + return parser + + failures = [ + '--foo X --bar Y', + '--foo X --bar=', + ] + successes = [ + ('--foo X', NS(foo='X', bar=True)), + ('--bar X', NS(foo=None, bar=True)), + ('--bar=', NS(foo=None, bar=False)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=True)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | --bar BAR) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | --bar BAR] + ''' + help = '''\ + + options: + -h, --help show this help message and exit + --foo FOO + --bar BAR + ''' + + +class TestMutuallyExclusivePositionalWithDefault(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('bar', nargs='?', type=bool, default=True) + return parser + + failures = [ + '--foo X Y', + ] + successes = [ + ('--foo X', NS(foo='X', bar=True)), + ('X', NS(foo=None, bar=True)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=True)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | bar) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | bar] + ''' + help = '''\ + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' + # ================================================= # Mutually exclusive group in parent parser tests # ================================================= @@ -4206,6 +4725,158 @@ class TestHelpUsagePositionalsOnlyWrap(HelpTestCase): version = '' +class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): + # https://github.com/python/cpython/issues/62549 + # https://github.com/python/cpython/issues/89743 + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('-n1', metavar='()', help='n1'), + Sig('-o1', metavar='(1, 2)', help='o1'), + Sig('-u1', metavar=' (uu) ', help='u1'), + Sig('-v1', metavar='( vv )', help='v1'), + Sig('-w1', metavar='(w)w', help='w1'), + Sig('-x1', metavar='x(x)', help='x1'), + Sig('-y1', metavar='yy)', help='y1'), + Sig('-z1', metavar='(zz', help='z1'), + Sig('-n2', metavar='[]', help='n2'), + Sig('-o2', metavar='[1, 2]', help='o2'), + Sig('-u2', metavar=' [uu] ', help='u2'), + Sig('-v2', metavar='[ vv ]', help='v2'), + Sig('-w2', metavar='[w]w', help='w2'), + Sig('-x2', metavar='x[x]', help='x2'), + Sig('-y2', metavar='yy]', help='y2'), + Sig('-z2', metavar='[zz', help='z2'), + ] + + usage = '''\ + usage: PROG [-h] [-n1 ()] [-o1 (1, 2)] [-u1 (uu) ] [-v1 ( vv )] [-w1 (w)w] + [-x1 x(x)] [-y1 yy)] [-z1 (zz] [-n2 []] [-o2 [1, 2]] [-u2 [uu] ] + [-v2 [ vv ]] [-w2 [w]w] [-x2 x[x]] [-y2 yy]] [-z2 [zz] + ''' + help = usage + '''\ + + options: + -h, --help show this help message and exit + -n1 () n1 + -o1 (1, 2) o1 + -u1 (uu) u1 + -v1 ( vv ) v1 + -w1 (w)w w1 + -x1 x(x) x1 + -y1 yy) y1 + -z1 (zz z1 + -n2 [] n2 + -o2 [1, 2] o2 + -u2 [uu] u2 + -v2 [ vv ] v2 + -w2 [w]w w2 + -x2 x[x] x2 + -y2 yy] y2 + -z2 [zz z2 + ''' + version = '' + + +class TestHelpUsageNoWhitespaceCrash(TestCase): + + def test_all_suppressed_mutex_followed_by_long_arg(self): + # https://github.com/python/cpython/issues/62090 + # https://github.com/python/cpython/issues/96310 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument('--spam', help=argparse.SUPPRESS) + parser.add_argument('--eggs-eggs-eggs-eggs-eggs-eggs') + usage = textwrap.dedent('''\ + usage: PROG [-h] + [--eggs-eggs-eggs-eggs-eggs-eggs EGGS_EGGS_EGGS_EGGS_EGGS_EGGS] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_newline_in_metavar(self): + # https://github.com/python/cpython/issues/77048 + mapping = ['123456', '12345', '12345', '123'] + parser = argparse.ArgumentParser('11111111111111') + parser.add_argument('-v', '--verbose', + help='verbose mode', action='store_true') + parser.add_argument('targets', + help='installation targets', + nargs='+', + metavar='\n'.join(mapping)) + usage = textwrap.dedent('''\ + usage: 11111111111111 [-h] [-v] + 123456 + 12345 + 12345 + 123 [123456 + 12345 + 12345 + 123 ...] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_empty_metavar_required_arg(self): + # https://github.com/python/cpython/issues/82091 + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('--nil', metavar='', required=True) + parser.add_argument('--a', metavar='A' * 70) + usage = ( + 'usage: PROG [-h] --nil \n' + ' [--a AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + 'AAAAAAAAAAAAAAAAAAAAAAA]\n' + ) + self.assertEqual(parser.format_usage(), usage) + + def test_all_suppressed_mutex_with_optional_nargs(self): + # https://github.com/python/cpython/issues/98666 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument( + '--param1', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + mutex.add_argument( + '--param2', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + usage = 'usage: PROG [-h]\n' + self.assertEqual(parser.format_usage(), usage) + + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + gg = g.add_mutually_exclusive_group() + gg.add_argument("--hax") + gg.add_argument("--hox", help=argparse.SUPPRESS) + gg.add_argument("--hex") + g.add_argument("--eggs") + parser.add_argument("--num") + + usage = textwrap.dedent('''\ + usage: PROG [-h] [--spam SPAM | [--hax HAX | --hex HEX] | --eggs EGGS] + [--num NUM] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_long_mutex_groups_wrap(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('--op1', metavar='MET', nargs='?') + g.add_argument('--op2', metavar=('MET1', 'MET2'), nargs='*') + g.add_argument('--op3', nargs='*') + g.add_argument('--op4', metavar=('MET1', 'MET2'), nargs='+') + g.add_argument('--op5', nargs='+') + g.add_argument('--op6', nargs=3) + g.add_argument('--op7', metavar=('MET1', 'MET2', 'MET3'), nargs=3) + + usage = textwrap.dedent('''\ + usage: PROG [-h] [--op1 [MET] | --op2 [MET1 [MET2 ...]] | --op3 [OP3 ...] | + --op4 MET1 [MET2 ...] | --op5 OP5 [OP5 ...] | --op6 OP6 OP6 OP6 | + --op7 MET1 MET2 MET3] + ''') + self.assertEqual(parser.format_usage(), usage) + + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" @@ -4215,7 +4886,7 @@ class TestHelpVariableExpansion(HelpTestCase): help='x %(prog)s %(default)s %(type)s %%'), Sig('-y', action='store_const', default=42, const='XXX', help='y %(prog)s %(default)s %(const)s'), - Sig('--foo', choices='abc', + Sig('--foo', choices=['a', 'b', 'c'], help='foo %(prog)s %(default)s %(choices)s'), Sig('--bar', default='baz', choices=[1, 2], metavar='BBB', help='bar %(prog)s %(default)s %(dest)s'), @@ -4458,7 +5129,7 @@ class TestHelpNone(HelpTestCase): version = '' -class TestHelpTupleMetavar(HelpTestCase): +class TestHelpTupleMetavarOptional(HelpTestCase): """Test specifying metavar as a tuple""" parser_signature = Sig(prog='PROG') @@ -4485,6 +5156,34 @@ class TestHelpTupleMetavar(HelpTestCase): version = '' +class TestHelpTupleMetavarPositional(HelpTestCase): + """Test specifying metavar on a Positional as a tuple""" + + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('w', help='w help', nargs='+', metavar=('W1', 'W2')), + Sig('x', help='x help', nargs='*', metavar=('X1', 'X2')), + Sig('y', help='y help', nargs=3, metavar=('Y1', 'Y2', 'Y3')), + Sig('z', help='z help', nargs='?', metavar=('Z1',)), + ] + argument_group_signatures = [] + usage = '''\ + usage: PROG [-h] W1 [W2 ...] [X1 [X2 ...]] Y1 Y2 Y3 [Z1] + ''' + help = usage + '''\ + + positional arguments: + W1 W2 w help + X1 X2 x help + Y1 Y2 Y3 y help + Z1 z help + + options: + -h, --help show this help message and exit + ''' + version = '' + + class TestHelpRawText(HelpTestCase): """Test the RawTextHelpFormatter""" @@ -4778,6 +5477,46 @@ def custom_type(string): version = '' +class TestHelpUsageLongSubparserCommand(TestCase): + """Test that subparser commands are formatted correctly in help""" + maxDiff = None + + def test_parent_help(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, max_help_position=50) + + parent_parser = argparse.ArgumentParser( + prog='PROG', + formatter_class=custom_formatter + ) + + cmd_subparsers = parent_parser.add_subparsers(title="commands", + metavar='CMD', + help='command to use') + cmd_subparsers.add_parser("add", + help="add something") + + cmd_subparsers.add_parser("remove", + help="remove something") + + cmd_subparsers.add_parser("a-very-long-command", + help="command that does something") + + parser_help = parent_parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] CMD ... + + options: + -h, --help show this help message and exit + + commands: + CMD command to use + add add something + remove remove something + a-very-long-command command that does something + ''')) + + # ===================================== # Optional/Positional constructor tests # ===================================== @@ -4785,15 +5524,15 @@ def custom_type(string): class TestInvalidArgumentConstructors(TestCase): """Test a bunch of invalid Argument constructors""" - def assertTypeError(self, *args, **kwargs): + def assertTypeError(self, *args, errmsg=None, **kwargs): parser = argparse.ArgumentParser() - self.assertRaises(TypeError, parser.add_argument, - *args, **kwargs) + self.assertRaisesRegex(TypeError, errmsg, parser.add_argument, + *args, **kwargs) - def assertValueError(self, *args, **kwargs): + def assertValueError(self, *args, errmsg=None, **kwargs): parser = argparse.ArgumentParser() - self.assertRaises(ValueError, parser.add_argument, - *args, **kwargs) + self.assertRaisesRegex(ValueError, errmsg, parser.add_argument, + *args, **kwargs) def test_invalid_keyword_arguments(self): self.assertTypeError('-x', bar=None) @@ -4803,73 +5542,99 @@ def test_invalid_keyword_arguments(self): def test_missing_destination(self): self.assertTypeError() - for action in ['append', 'store']: - self.assertTypeError(action=action) + for action in ['store', 'append', 'extend']: + with self.subTest(action=action): + self.assertTypeError(action=action) def test_invalid_option_strings(self): - self.assertValueError('--') - self.assertValueError('---') + self.assertTypeError('-', errmsg='dest= is required') + self.assertTypeError('--', errmsg='dest= is required') + self.assertTypeError('---', errmsg='dest= is required') def test_invalid_prefix(self): - self.assertValueError('--foo', '+foo') + self.assertValueError('--foo', '+foo', + errmsg='must start with a character') def test_invalid_type(self): - self.assertValueError('--foo', type='int') - self.assertValueError('--foo', type=(int, float)) + self.assertTypeError('--foo', type='int', + errmsg="'int' is not callable") + self.assertTypeError('--foo', type=(int, float), + errmsg='is not callable') def test_invalid_action(self): - self.assertValueError('-x', action='foo') - self.assertValueError('foo', action='baz') - self.assertValueError('--foo', action=('store', 'append')) - parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: - parser.add_argument("--foo", action="store-true") - self.assertIn('unknown action', str(cm.exception)) + self.assertValueError('-x', action='foo', + errmsg='unknown action') + self.assertValueError('foo', action='baz', + errmsg='unknown action') + self.assertValueError('--foo', action=('store', 'append'), + errmsg='unknown action') + self.assertValueError('--foo', action="store-true", + errmsg='unknown action') + + def test_invalid_help(self): + self.assertValueError('--foo', help='%Y-%m-%d', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(spam)s', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(prog)d', + errmsg='badly formed help string') def test_multiple_dest(self): parser = argparse.ArgumentParser() parser.add_argument(dest='foo') - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('bar', dest='baz') - self.assertIn('dest supplied twice for positional argument', + self.assertIn('dest supplied twice for positional argument,' + ' did you mean metavar?', str(cm.exception)) def test_no_argument_actions(self): for action in ['store_const', 'store_true', 'store_false', 'append_const', 'count']: - for attrs in [dict(type=int), dict(nargs='+'), - dict(choices='ab')]: - self.assertTypeError('-x', action=action, **attrs) + with self.subTest(action=action): + for attrs in [dict(type=int), dict(nargs='+'), + dict(choices=['a', 'b'])]: + with self.subTest(attrs=attrs): + self.assertTypeError('-x', action=action, **attrs) + self.assertTypeError('x', action=action, **attrs) + self.assertValueError('x', action=action, + errmsg=f"action '{action}' is not valid for positional arguments") + self.assertTypeError('-x', action=action, nargs=0) + self.assertValueError('x', action=action, nargs=0, + errmsg='nargs for positionals must be != 0') def test_no_argument_no_const_actions(self): # options with zero arguments for action in ['store_true', 'store_false', 'count']: + with self.subTest(action=action): + # const is always disallowed + self.assertTypeError('-x', const='foo', action=action) - # const is always disallowed - self.assertTypeError('-x', const='foo', action=action) - - # nargs is always disallowed - self.assertTypeError('-x', nargs='*', action=action) + # nargs is always disallowed + self.assertTypeError('-x', nargs='*', action=action) def test_more_than_one_argument_actions(self): - for action in ['store', 'append']: - - # nargs=0 is disallowed - self.assertValueError('-x', nargs=0, action=action) - self.assertValueError('spam', nargs=0, action=action) - - # const is disallowed with non-optional arguments - for nargs in [1, '*', '+']: - self.assertValueError('-x', const='foo', - nargs=nargs, action=action) - self.assertValueError('spam', const='foo', - nargs=nargs, action=action) + for action in ['store', 'append', 'extend']: + with self.subTest(action=action): + # nargs=0 is disallowed + action_name = 'append' if action == 'extend' else action + self.assertValueError('-x', nargs=0, action=action, + errmsg=f'nargs for {action_name} actions must be != 0') + self.assertValueError('spam', nargs=0, action=action, + errmsg='nargs for positionals must be != 0') + + # const is disallowed with non-optional arguments + for nargs in [1, '*', '+']: + self.assertValueError('-x', const='foo', + nargs=nargs, action=action) + self.assertValueError('spam', const='foo', + nargs=nargs, action=action) def test_required_const_actions(self): for action in ['store_const', 'append_const']: - - # nargs is always disallowed - self.assertTypeError('-x', nargs='+', action=action) + with self.subTest(action=action): + # nargs is always disallowed + self.assertTypeError('-x', nargs='+', action=action) def test_parsers_action_missing_params(self): self.assertTypeError('command', action='parsers') @@ -4988,14 +5753,18 @@ def test_subparser_conflict(self): parser = argparse.ArgumentParser() sp = parser.add_subparsers() sp.add_parser('fullname', aliases=['alias']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'fullname') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'alias') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['fullname']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['alias']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser: fullname', + sp.add_parser, 'fullname') + self.assertRaisesRegex(ValueError, + 'conflicting subparser: alias', + sp.add_parser, 'alias') + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: fullname', + sp.add_parser, 'other', aliases=['fullname']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: alias', + sp.add_parser, 'other', aliases=['alias']) # ============================= @@ -5540,8 +6309,34 @@ def test_zero_or_more_optional(self): args = parser.parse_args([]) self.assertEqual(NS(x=[]), args) - def test_double_dash(self): - parser = argparse.ArgumentParser() + +class TestDoubleDash(TestCase): + def test_single_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('-f', '--foo') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo=--']) + self.assertEqual(NS(foo='--', bar=[]), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument -f/--foo: expected one argument', + parser.parse_args, ['--foo', '--']) + args = parser.parse_args(['-f--']) + self.assertEqual(NS(foo='--', bar=[]), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument -f/--foo: expected one argument', + parser.parse_args, ['-f', '--']) + args = parser.parse_args(['--foo', 'a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--foo', 'c']) + self.assertEqual(NS(foo='c', bar=['a', 'b']), args) + args = parser.parse_args(['a', '--', 'b', '--foo', 'c']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--foo', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c', '--foo', 'd']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--', 'c', '--foo', 'd']), args) + + def test_multiple_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('-f', '--foo', nargs='*') parser.add_argument('bar', nargs='*') @@ -5555,6 +6350,91 @@ def test_double_dash(self): self.assertEqual(NS(foo=[], bar=[]), args) args = parser.parse_args(['--foo', 'a', 'b', '--', 'c', 'd']) self.assertEqual(NS(foo=['a', 'b'], bar=['c', 'd']), args) + args = parser.parse_args(['a', 'b', '--foo', 'c', 'd']) + self.assertEqual(NS(foo=['c', 'd'], bar=['a', 'b']), args) + args = parser.parse_args(['a', '--', 'b', '--foo', 'c', 'd']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--foo', 'c', 'd']), args) + args, argv = parser.parse_known_args(['a', 'b', '--foo', 'c', '--', 'd']) + self.assertEqual(NS(foo=['c'], bar=['a', 'b']), args) + self.assertEqual(argv, ['--', 'd']) + + def test_multiple_double_dashes(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--', 'a', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + args = parser.parse_args(['--', '--', 'a', '--', 'b', 'c']) + self.assertEqual(NS(foo='--', bar=['a', '--', 'b', 'c']), args) + + def test_remainder(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + parser.add_argument('bar', nargs='...') + + args = parser.parse_args(['--', 'a', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo') + parser.add_argument('bar', nargs='...') + args = parser.parse_args(['--foo', 'a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['--', 'b', '--', 'c']), args) + + def test_subparser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('run') + parser1.add_argument('-f') + parser1.add_argument('bar', nargs='*') + + args = parser.parse_args(['x', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['x', 'run', 'a', 'b', '--', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', 'run', 'a', '--', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', 'run', '--', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', '--', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['--', 'x', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['x', 'run', '--', 'a', '--', 'b']) + self.assertEqual(NS(foo='x', f=None, bar=['a', '--', 'b']), args) + args = parser.parse_args(['x', '--', 'run', '--', 'a', '--', 'b']) + self.assertEqual(NS(foo='x', f=None, bar=['a', '--', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + "invalid choice: '--'", + parser.parse_args, ['--', 'x', '--', 'run', 'a', 'b']) + + def test_subparser_after_multiple_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', nargs='*') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('run') + parser1.add_argument('-f') + parser1.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo', 'x', 'y', '--', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo=['x', 'y'], f='c', bar=['a', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + "invalid choice: '--'", + parser.parse_args, ['--foo', 'x', '--', '--', 'run', 'a', 'b']) # =========================== @@ -5576,14 +6456,25 @@ def test_basic(self): args, extras = parser.parse_known_args(argv) # cannot parse the '1,2,3' - self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[]), args) - self.assertEqual(["1", "2", "3"], extras) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args) + self.assertEqual(["2", "3"], extras) + args, extras = parser.parse_known_intermixed_args(argv) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual([], extras) + # unknown optionals go into extras + argv = 'cmd --foo x --error 1 2 --bar y 3'.split() + args, extras = parser.parse_known_intermixed_args(argv) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual(['--error'], extras) argv = 'cmd --foo x 1 --error 2 --bar y 3'.split() args, extras = parser.parse_known_intermixed_args(argv) - # unknown optionals go into extras - self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args) - self.assertEqual(['--error', '2', '3'], extras) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual(['--error'], extras) + argv = 'cmd --foo x 1 2 --error --bar y 3'.split() + args, extras = parser.parse_known_intermixed_args(argv) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual(['--error'], extras) # restores attributes that were temporarily changed self.assertIsNone(parser.usage) @@ -5602,38 +6493,48 @@ def test_remainder(self): parser.parse_intermixed_args(argv) self.assertRegex(str(cm.exception), r'\.\.\.') - def test_exclusive(self): - # mutually exclusive group; intermixed works fine - parser = ErrorRaisingArgumentParser(prog='PROG') + def test_required_exclusive(self): + # required mutually exclusive group; intermixed works fine + parser = argparse.ArgumentParser(prog='PROG', exit_on_error=False) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') parser.add_argument('badger', nargs='*', default='X', help='BADGER') + args = parser.parse_intermixed_args('--foo 1 2'.split()) + self.assertEqual(NS(badger=['1', '2'], foo=True, spam=None), args) args = parser.parse_intermixed_args('1 --foo 2'.split()) self.assertEqual(NS(badger=['1', '2'], foo=True, spam=None), args) - self.assertRaises(ArgumentParserError, parser.parse_intermixed_args, '1 2'.split()) + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --foo --spam is required', + parser.parse_intermixed_args, '1 2'.split()) self.assertEqual(group.required, True) - def test_exclusive_incompatible(self): - # mutually exclusive group including positional - fail - parser = ErrorRaisingArgumentParser(prog='PROG') + def test_required_exclusive_with_positional(self): + # required mutually exclusive group with positional argument + parser = argparse.ArgumentParser(prog='PROG', exit_on_error=False) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') group.add_argument('badger', nargs='*', default='X', help='BADGER') - self.assertRaises(TypeError, parser.parse_intermixed_args, []) + args = parser.parse_intermixed_args(['--foo']) + self.assertEqual(NS(foo=True, spam=None, badger='X'), args) + args = parser.parse_intermixed_args(['a', 'b']) + self.assertEqual(NS(foo=False, spam=None, badger=['a', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --foo --spam badger is required', + parser.parse_intermixed_args, []) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument badger: not allowed with argument --foo', + parser.parse_intermixed_args, ['--foo', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument badger: not allowed with argument --foo', + parser.parse_intermixed_args, ['a', '--foo', 'b']) self.assertEqual(group.required, True) def test_invalid_args(self): parser = ErrorRaisingArgumentParser(prog='PROG') self.assertRaises(ArgumentParserError, parser.parse_intermixed_args, ['a']) - parser = ErrorRaisingArgumentParser(prog='PROG') - parser.add_argument('--foo', nargs="*") - parser.add_argument('foo') - with captured_stderr() as stderr: - parser.parse_intermixed_args(['hello', '--foo']) - self.assertIn("UserWarning", stderr.getvalue()) class TestIntermixedMessageContentError(TestCase): # case where Intermixed gives different error message @@ -5652,7 +6553,7 @@ def test_missing_argument_name_in_message(self): with self.assertRaises(ArgumentParserError) as cm: parser.parse_intermixed_args([]) msg = str(cm.exception) - self.assertNotRegex(msg, 'req_pos') + self.assertRegex(msg, 'req_pos') self.assertRegex(msg, 'req_opt') # ========================== @@ -5902,7 +6803,8 @@ def test_help_with_metavar(self): class TestExitOnError(TestCase): def setUp(self): - self.parser = argparse.ArgumentParser(exit_on_error=False) + self.parser = argparse.ArgumentParser(exit_on_error=False, + fromfile_prefix_chars='@') self.parser.add_argument('--integers', metavar='N', type=int) def test_exit_on_error_with_good_args(self): @@ -5913,6 +6815,285 @@ def test_exit_on_error_with_bad_args(self): with self.assertRaises(argparse.ArgumentError): self.parser.parse_args('--integers a'.split()) + def test_unrecognized_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_args, '--foo bar'.split()) + + def test_unrecognized_intermixed_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_intermixed_args, '--foo bar'.split()) + + def test_required_args(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', metavar='BaZ') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, BaZ$', + self.parser.parse_args, []) + + def test_required_args_n(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs=3) + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_n_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs=3, metavar=('B', 'A', 'Z')) + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, B, A, Z$', + self.parser.parse_args, []) + + def test_required_args_optional(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='?') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_args_zero_or_more(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='*') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_args_one_or_more(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='+') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_one_or_more_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='+', metavar=('BaZ1', 'BaZ2')) + self.assertRaisesRegex(argparse.ArgumentError, + r'the following arguments are required: bar, BaZ1\[, BaZ2]$', + self.parser.parse_args, []) + + def test_required_args_remainder(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='...') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_mutually_exclusive_args(self): + group = self.parser.add_mutually_exclusive_group(required=True) + group.add_argument('--bar') + group.add_argument('--baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --bar --baz is required', + self.parser.parse_args, []) + + def test_conflicting_mutually_exclusive_args_optional_with_metavar(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='?', metavar='BaZ') + self.assertRaisesRegex(argparse.ArgumentError, + 'argument BaZ: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument --bar: not allowed with argument BaZ$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar1(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='*', metavar=('BAZ1',)) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument BAZ1: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument --bar: not allowed with argument BAZ1$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar2(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='*', metavar=('BAZ1', 'BAZ2')) + self.assertRaisesRegex(argparse.ArgumentError, + r'argument BAZ1\[, BAZ2]: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + r'argument --bar: not allowed with argument BAZ1\[, BAZ2]$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_ambiguous_option(self): + self.parser.add_argument('--foobaz') + self.parser.add_argument('--fooble', action='store_true') + self.parser.add_argument('--foogle') + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble", + self.parser.parse_args, ['--foob']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob=1 could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob=1']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob', '1', '--foogle', '2']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob=1 could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob=1', '--foogle', '2']) + + def test_os_error(self): + self.parser.add_argument('file') + self.assertRaisesRegex(argparse.ArgumentError, + "No such file or directory: 'no-such-file'", + self.parser.parse_args, ['@no-such-file']) + + +class TestProgName(TestCase): + source = textwrap.dedent('''\ + import argparse + parser = argparse.ArgumentParser() + parser.parse_args() + ''') + + def setUp(self): + self.dirname = 'package' + os_helper.FS_NONASCII + self.addCleanup(os_helper.rmtree, self.dirname) + os.mkdir(self.dirname) + + def make_script(self, dirname, basename, *, compiled=False): + script_name = script_helper.make_script(dirname, basename, self.source) + if not compiled: + return script_name + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + return pyc_file + + def make_zip_script(self, script_name, name_in_zip=None): + zip_name, _ = script_helper.make_zip_script(self.dirname, 'test_zip', + script_name, name_in_zip) + return zip_name + + def check_usage(self, expected, *args, **kwargs): + res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) + self.assertEqual(res.out.splitlines()[0].decode(), + f'usage: {expected} [-h]') + + def test_script(self, compiled=False): + basename = os_helper.TESTFN + script_name = self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(os.path.basename(script_name), script_name, '-h') + + def test_script_compiled(self): + self.test_script(compiled=True) + + def test_directory(self, compiled=False): + dirname = os.path.join(self.dirname, os_helper.TESTFN) + os.mkdir(dirname) + self.make_script(dirname, '__main__', compiled=compiled) + self.check_usage(f'{py} {dirname}', dirname) + dirname2 = os.path.join(os.curdir, dirname) + self.check_usage(f'{py} {dirname2}', dirname2) + + def test_directory_compiled(self): + self.test_directory(compiled=True) + + def test_module(self, compiled=False): + basename = 'module' + os_helper.FS_NONASCII + modulename = f'{self.dirname}.{basename}' + self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(f'{py} -m {modulename}', + '-m', modulename, PYTHONPATH=os.curdir) + + def test_module_compiled(self): + self.test_module(compiled=True) + + def test_package(self, compiled=False): + basename = 'subpackage' + os_helper.FS_NONASCII + packagename = f'{self.dirname}.{basename}' + subdirname = os.path.join(self.dirname, basename) + os.mkdir(subdirname) + self.make_script(subdirname, '__main__', compiled=compiled) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename, PYTHONPATH=os.curdir) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename + '.__main__', PYTHONPATH=os.curdir) + + def test_package_compiled(self): + self.test_package(compiled=True) + + def test_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + zip_name = self.make_zip_script(script_name) + self.check_usage(f'{py} {zip_name}', zip_name) + + def test_zipfile_compiled(self): + self.test_zipfile(compiled=True) + + def test_directory_in_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + name_in_zip = 'package/subpackage/__main__' + ('.py', '.pyc')[compiled] + zip_name = self.make_zip_script(script_name, name_in_zip) + dirname = os.path.join(zip_name, 'package', 'subpackage') + self.check_usage(f'{py} {dirname}', dirname) + + def test_directory_in_zipfile_compiled(self): + self.test_directory_in_zipfile(compiled=True) + +# ================= +# Translation tests +# ================= + +pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py' +snapshot_path = Path(TEST_HOME_DIR) / 'translationdata' / 'argparse' / 'msgids.txt' + +msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)', re.DOTALL) +msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"') + + +@requires_subprocess() +class TestTranslations(unittest.TestCase): + + def test_translations(self): + # Test messages extracted from the argparse module against a snapshot + skip_if_missing('i18n') + res = generate_po_file(stdout_only=False) + self.assertEqual(res.returncode, 0) + self.assertEqual(res.stderr, '') + msgids = extract_msgids(res.stdout) + snapshot = snapshot_path.read_text().splitlines() + self.assertListEqual(msgids, snapshot) + + +def generate_po_file(*, stdout_only=True): + res = subprocess.run([sys.executable, pygettext, + '--no-location', '-o', '-', argparse.__file__], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if stdout_only: + return res.stdout + return res + + +def extract_msgids(po): + msgids = [] + for msgid in msgid_pattern.findall(po): + msgid_string = ''.join(msgid_string_pattern.findall(msgid)) + msgid_string = msgid_string.replace(r'\"', '"') + if msgid_string: + msgids.append(msgid_string) + return sorted(msgids) + + +def update_translation_snapshots(): + contents = generate_po_file() + msgids = extract_msgids(contents) + snapshot_path.write_text('\n'.join(msgids)) + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. @@ -5921,4 +7102,8 @@ def tearDownModule(): if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots() + sys.exit(0) unittest.main() diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index a219fa365e7f20..f621f343eb062a 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1014,6 +1014,29 @@ def test_pop(self): array.array(self.typecode, self.example[3:]+self.example[:-1]) ) + def test_clear(self): + a = array.array(self.typecode, self.example) + with self.assertRaises(TypeError): + a.clear(42) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode, self.example) + a.clear() + a.append(self.example[2]) + a.append(self.example[3]) + self.assertEqual(a, array.array(self.typecode, self.example[2:4])) + + with memoryview(a): + with self.assertRaises(BufferError): + a.clear() + def test_reverse(self): a = array.array(self.typecode, self.example) self.assertRaises(TypeError, a.reverse, 42) @@ -1469,8 +1492,8 @@ def test_byteswap(self): if a.itemsize==1: self.assertEqual(a, b) else: - # On alphas treating the byte swapped bit patters as - # floats/doubles results in floating point exceptions + # On alphas treating the byte swapped bit patterns as + # floats/doubles results in floating-point exceptions # => compare the 8bit string values instead self.assertNotEqual(a.tobytes(), b.tobytes()) b.byteswap() diff --git a/Lib/test/test_ast/__init__.py b/Lib/test/test_ast/__init__.py new file mode 100644 index 00000000000000..9a89d27ba9f979 --- /dev/null +++ b/Lib/test/test_ast/__init__.py @@ -0,0 +1,7 @@ +import os + +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_ast/data/ast_repr.txt b/Lib/test/test_ast/data/ast_repr.txt new file mode 100644 index 00000000000000..3778b9e70a4605 --- /dev/null +++ b/Lib/test/test_ast/data/ast_repr.txt @@ -0,0 +1,209 @@ +Module(body=[Expr(value=Constant(value='module docstring', kind=None))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...), ..., arg(...)], vararg=arg(...), kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), ..., Dict(...)]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Expr(value=Constant(...))], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='object', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='A', ctx=Load(...)), Name(id='B', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Delete(targets=[Name(id='v', ctx=Del(...))])], type_ignores=[]) +Module(body=[Assign(targets=[Name(id='v', ctx=Store(...))], value=Constant(value=1, kind=None), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[List(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Subscript(value=Name(...), slice=Name(...), ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Add(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Sub(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=MatMult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Div(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mod(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Pow(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=LShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=RShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitOr(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitXor(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitAnd(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=FloorDiv(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[Pass(...)])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[If(...)])])], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...)), withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[Raise(exc=None, cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Name(id='Exception', ctx=Load(...)), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[], orelse=[], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[]) +Module(body=[Global(names=['v'])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[Pass()], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Break()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Continue()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=List(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...), comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...)), Expr(value=Await(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncFor(target=Name(...), iter=Name(...), body=[Expr(...)], orelse=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncWith(items=[withitem(...)], body=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[None, Constant(...)], values=[Dict(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Starred(...), Constant(...)]))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Yield(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=YieldFrom(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=ListComp(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Call(func=Name(...), args=[GeneratorExp(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Attribute(value=Attribute(...), attr='c', ctx=Load(...))], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=NamedExpr(target=Name(...), value=Constant(...)))], type_ignores=[]) +Module(body=[If(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), ..., arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...), ..., Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None)], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)]), match_case(pattern=MatchAs(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=True, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=False, kind=None))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=And(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=Or(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Add(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Sub(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Div(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=MatMult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=FloorDiv(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Pow(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mod(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=RShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=LShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitXor(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitOr(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitAnd(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Not(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=UAdd(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=USub(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Invert(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=Lambda(args=arguments(...), body=Constant(...)))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[], values=[]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Constant(...), ops=[Lt(...), Lt(...)], comparators=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Eq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[LtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[GtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotEq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Is(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[IsNot(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[In(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotIn(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Constant(...), ..., Starred(...)], keywords=[keyword(...), keyword(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Starred(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[GeneratorExp(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=10, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1j, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value='string', kind=None))], type_ignores=[]) +Module(body=[Expr(value=Attribute(value=Name(...), attr='b', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=Name(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Name(id='v', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Attribute(...), args=[Subscript(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=IfExp(test=Name(...), body=Call(...), orelse=Call(...)))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[Constant(...), ..., Constant(...)]))], type_ignores=[]) \ No newline at end of file diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py new file mode 100644 index 00000000000000..28d32b2941f30d --- /dev/null +++ b/Lib/test/test_ast/snippets.py @@ -0,0 +1,601 @@ +import ast +import sys + +from test.test_ast.utils import to_tuple + + +# These tests are compiled through "exec" +# There should be at least one test per statement +exec_tests = [ + # Module docstring + "'module docstring'", + # FunctionDef + "def f(): pass", + # FunctionDef with docstring + "def f(): 'function docstring'", + # FunctionDef with arg + "def f(a): pass", + # FunctionDef with arg and default value + "def f(a=0): pass", + # FunctionDef with varargs + "def f(*args): pass", + # FunctionDef with varargs as TypeVarTuple + "def f(*args: *Ts): pass", + # FunctionDef with varargs as unpacked Tuple + "def f(*args: *tuple[int, ...]): pass", + # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple + "def f(*args: *tuple[int, *Ts]): pass", + # FunctionDef with kwargs + "def f(**kwargs): pass", + # FunctionDef with all kind of args and docstring + "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", + # FunctionDef with type annotation on return involving unpacking + "def f() -> tuple[*Ts]: pass", + "def f() -> tuple[int, *Ts]: pass", + "def f() -> tuple[int, *tuple[int, ...]]: pass", + # ClassDef + "class C:pass", + # ClassDef with docstring + "class C: 'docstring for class C'", + # ClassDef, new style class + "class C(object): pass", + # Classdef with multiple bases + "class C(A, B): pass", + # Return + "def f():return 1", + "def f():return", + # Delete + "del v", + # Assign + "v = 1", + "a,b = c", + "(a,b) = c", + "[a,b] = c", + "a[b] = c", + # AnnAssign with unpacked types + "x: tuple[*Ts]", + "x: tuple[int, *Ts]", + "x: tuple[int, *tuple[str, ...]]", + # AugAssign + "v += 1", + "v -= 1", + "v *= 1", + "v @= 1", + "v /= 1", + "v %= 1", + "v **= 1", + "v <<= 1", + "v >>= 1", + "v |= 1", + "v ^= 1", + "v &= 1", + "v //= 1", + # For + "for v in v:pass", + # For-Else + "for v in v:\n pass\nelse:\n pass", + # While + "while v:pass", + # While-Else + "while v:\n pass\nelse:\n pass", + # If-Elif-Else + "if v:pass", + "if a:\n pass\nelif b:\n pass", + "if a:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelif b:\n pass\nelif b:\n pass\nelse:\n pass", + # With + "with x: pass", + "with x, y: pass", + "with x as y: pass", + "with x as y, z as q: pass", + "with (x as y): pass", + "with (x, y): pass", + # Raise + "raise", + "raise Exception('string')", + "raise Exception", + "raise Exception('string') from None", + # TryExcept + "try:\n pass\nexcept Exception:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass", + # TryFinally + "try:\n pass\nfinally:\n pass", + # TryStarExcept + "try:\n pass\nexcept* Exception:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass", + # TryExceptFinallyElse + "try:\n pass\nexcept Exception:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + # Assert + "assert v", + # Assert with message + "assert v, 'message'", + # Import + "import sys", + "import foo as bar", + # ImportFrom + "from sys import x as y", + "from sys import v", + # Global + "global v", + # Expr + "1", + # Pass, + "pass", + # Break + "for v in v:break", + # Continue + "for v in v:continue", + # for statements with naked tuples (see http://bugs.python.org/issue6704) + "for a,b in c: pass", + "for (a,b) in c: pass", + "for [a,b] in c: pass", + # Multiline generator expression (test for .lineno & .col_offset) + """( + ( + Aa + , + Bb + ) + for + Aa + , + Bb in Cc + )""", + # dictcomp + "{a : b for w in x for m in p if g}", + # dictcomp with naked tuple + "{a : b for v,w in x}", + # setcomp + "{r for l in x if g}", + # setcomp with naked tuple + "{r for l,m in x}", + # AsyncFunctionDef + "async def f():\n 'async function'\n await something()", + # AsyncFor + "async def f():\n async for e in i: 1\n else: 2", + # AsyncWith + "async def f():\n async with a as b: 1", + # PEP 448: Additional Unpacking Generalizations + "{**{1:2}, 2:3}", + "{*{1, 2}, 3}", + # Function with yield (from) + "def f(): yield 1", + "def f(): yield from []", + # Asynchronous comprehensions + "async def f():\n [i async for b in c]", + # Decorated FunctionDef + "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", + # Decorated AsyncFunctionDef + "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", + # Decorated ClassDef + "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", + # Decorator with generator argument + "@deco(a for a in b)\ndef f(): pass", + # Decorator with attribute + "@a.b.c\ndef f(): pass", + # Simple assignment expression + "(a := 1)", + # Assignment expression in if statement + "if a := foo(): pass", + # Assignment expression in while + "while a := foo(): pass", + # Positional-only arguments + "def f(a, /,): pass", + "def f(a, /, c, d, e): pass", + "def f(a, /, c, *, d, e): pass", + "def f(a, /, c, *, d, e, **kwargs): pass", + # Positional-only arguments with defaults + "def f(a=1, /,): pass", + "def f(a=1, /, b=2, c=4): pass", + "def f(a=1, /, b=2, *, c=4): pass", + "def f(a=1, /, b=2, *, c): pass", + "def f(a=1, /, b=2, *, c=4, **kwargs): pass", + "def f(a=1, /, b=2, *, c, **kwargs): pass", + # Type aliases + "type X = int", + "type X[T] = int", + "type X[T, *Ts, **P] = (T, Ts, P)", + "type X[T: int, *Ts, **P] = (T, Ts, P)", + "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", + "type X[T: int = 1, *Ts = 2, **P =3] = (T, Ts, P)", + # Generic classes + "class X[T]: pass", + "class X[T, *Ts, **P]: pass", + "class X[T: int, *Ts, **P]: pass", + "class X[T: (int, str), *Ts, **P]: pass", + "class X[T: int = 1, *Ts = 2, **P = 3]: pass", + # Generic functions + "def f[T](): pass", + "def f[T, *Ts, **P](): pass", + "def f[T: int, *Ts, **P](): pass", + "def f[T: (int, str), *Ts, **P](): pass", + "def f[T: int = 1, *Ts = 2, **P = 3](): pass", + # Match + "match x:\n\tcase 1:\n\t\tpass", + # Match with _ + "match x:\n\tcase 1:\n\t\tpass\n\tcase _:\n\t\tpass", +] + +# These are compiled through "single" +# because of overlap with "eval", it just tests what +# can't be tested with "eval" +single_tests = [ + "1+2" +] + +# These are compiled through "eval" +# It should test all expressions +eval_tests = [ + # Constant(value=None) + "None", + # True + "True", + # False + "False", + # BoolOp + "a and b", + "a or b", + # BinOp + "a + b", + "a - b", + "a * b", + "a / b", + "a @ b", + "a // b", + "a ** b", + "a % b", + "a >> b", + "a << b", + "a ^ b", + "a | b", + "a & b", + # UnaryOp + "not v", + "+v", + "-v", + "~v", + # Lambda + "lambda:None", + # Dict + "{ 1:2 }", + # Empty dict + "{}", + # Set + "{None,}", + # Multiline dict (test for .lineno & .col_offset) + """{ + 1 + : + 2 + }""", + # Multiline list + """[ + 1 + , + 1 + ]""", + # Multiline tuple + """( + 1 + , + )""", + # Multiline set + """{ + 1 + , + 1 + }""", + # ListComp + "[a for b in c if d]", + # GeneratorExp + "(a for b in c if d)", + # SetComp + "{a for b in c if d}", + # DictComp + "{k: v for k, v in c if d}", + # Comprehensions with multiple for targets + "[(a,b) for a,b in c]", + "[(a,b) for (a,b) in c]", + "[(a,b) for [a,b] in c]", + "{(a,b) for a,b in c}", + "{(a,b) for (a,b) in c}", + "{(a,b) for [a,b] in c}", + "((a,b) for a,b in c)", + "((a,b) for (a,b) in c)", + "((a,b) for [a,b] in c)", + # Async comprehensions - async comprehensions can't work outside an asynchronous function + # + # Yield - yield expressions can't work outside a function + # + # Compare + "1 < 2 < 3", + "a == b", + "a <= b", + "a >= b", + "a != b", + "a is b", + "a is not b", + "a in b", + "a not in b", + # Call without argument + "f()", + # Call + "f(1,2,c=3,*d,**e)", + # Call with multi-character starred + "f(*[0, 1])", + # Call with a generator argument + "f(a for a in b)", + # Constant(value=int()) + "10", + # Complex num + "1j", + # Constant(value=str()) + "'string'", + # Attribute + "a.b", + # Subscript + "a[b:c]", + # Name + "v", + # List + "[1,2,3]", + # Empty list + "[]", + # Tuple + "1,2,3", + # Tuple + "(1,2,3)", + # Empty tuple + "()", + # Combination + "a.b.c.d(a.b[1:2])", + # Slice + "[5][1:]", + "[5][:1]", + "[5][::1]", + "[5][1:1:1]", + # IfExp + "foo() if x else bar()", + # JoinedStr and FormattedValue + "f'{a}'", + "f'{a:.2f}'", + "f'{a!r}'", + "f'foo({a})'", +] + + +def main(): + if __name__ != '__main__': + return + if sys.argv[1:] == ['-g']: + for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), + (eval_tests, "eval")): + print(kind+"_results = [") + for statement in statements: + tree = ast.parse(statement, "?", kind) + print("%r," % (to_tuple(tree),)) + print("]") + print("main()") + raise SystemExit + +#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast/snippets.py -g ##### +exec_results = [ +('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), +('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 19), 'C', [('Name', (1, 8, 1, 9), 'A', ('Load',)), ('Name', (1, 11, 1, 12), 'B', ('Load',))], [], [('Pass', (1, 15, 1, 19))], [], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 14), None)], [], None, None, [])], []), +('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), +('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), +('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 8), [('Subscript', (1, 0, 1, 4), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Store',))], ('Name', (1, 7, 1, 8), 'c', ('Load',)), None)], []), +('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Sub',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('MatMult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Div',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mod',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Pow',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('LShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('RShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitOr',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitXor',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitAnd',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('FloorDiv',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), +('Module', [('For', (1, 0, 4, 6), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))], None)], []), +('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), +('Module', [('While', (1, 0, 4, 6), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), +('Module', [('If', (1, 0, 10, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 10, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('If', (5, 0, 10, 6), ('Name', (5, 5, 5, 6), 'b', ('Load',)), [('Pass', (6, 2, 6, 6))], [('If', (7, 0, 10, 6), ('Name', (7, 5, 7, 6), 'b', ('Load',)), [('Pass', (8, 2, 8, 6))], [('Pass', (10, 2, 10, 6))])])])])], []), +('Module', [('With', (1, 0, 1, 12), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None)], [('Pass', (1, 8, 1, 12))], None)], []), +('Module', [('With', (1, 0, 1, 15), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None), ('withitem', ('Name', (1, 8, 1, 9), 'y', ('Load',)), None)], [('Pass', (1, 11, 1, 15))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), +('Module', [('With', (1, 0, 1, 19), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), ('Name', (1, 11, 1, 12), 'y', ('Store',)))], [('Pass', (1, 15, 1, 19))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), None), ('withitem', ('Name', (1, 9, 1, 10), 'y', ('Load',)), None)], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('Raise', (1, 0, 1, 5), None, None)], []), +('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), +('Module', [('Raise', (1, 0, 1, 15), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), None)], []), +('Module', [('Raise', (1, 0, 1, 35), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), ('Constant', (1, 31, 1, 35), None, None))], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('TryStar', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), +('Module', [('Assert', (1, 0, 1, 19), ('Name', (1, 7, 1, 8), 'v', ('Load',)), ('Constant', (1, 10, 1, 19), 'message', None))], []), +('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), +('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')])], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0)], []), +('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), +('Module', [('Global', (1, 0, 1, 8), ['v'])], []), +('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), +('Module', [('Pass', (1, 0, 1, 4))], []), +('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), +('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), +('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), +('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 16), ('Yield', (1, 9, 1, 16), ('Constant', (1, 15, 1, 16), 1, None)))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 22), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 22), ('YieldFrom', (1, 9, 1, 22), ('List', (1, 20, 1, 22), [], ('Load',))))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), +('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), +('Module', [('If', (1, 0, 1, 19), ('NamedExpr', (1, 3, 1, 13), ('Name', (1, 3, 1, 4), 'a', ('Store',)), ('Call', (1, 8, 1, 13), ('Name', (1, 8, 1, 11), 'foo', ('Load',)), [], [])), [('Pass', (1, 15, 1, 19))], [])], []), +('Module', [('While', (1, 0, 1, 22), ('NamedExpr', (1, 6, 1, 16), ('Name', (1, 6, 1, 7), 'a', ('Store',)), ('Call', (1, 11, 1, 16), ('Name', (1, 11, 1, 14), 'foo', ('Load',)), [], [])), [('Pass', (1, 18, 1, 22))], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), +('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []), +('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []), +('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 21, 1, 24), 'Ts', None), ('ParamSpec', (1, 26, 1, 29), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 43), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 39, 1, 43))], [], None, None, [('TypeVar', (1, 6, 1, 16), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Constant', (1, 15, 1, 16), 1, None)), ('TypeVarTuple', (1, 18, 1, 25), 'Ts', ('Constant', (1, 24, 1, 25), 2, None)), ('ParamSpec', (1, 27, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))])], []), +('Module', [('Match', (1, 0, 3, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))])])], []), +('Module', [('Match', (1, 0, 5, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))]), ('match_case', ('MatchAs', (4, 6, 4, 7), None, None), None, [('Pass', (5, 2, 5, 6))])])], []), +] +single_results = [ +('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), +] +eval_results = [ +('Expression', ('Constant', (1, 0, 1, 4), None, None)), +('Expression', ('Constant', (1, 0, 1, 4), True, None)), +('Expression', ('Constant', (1, 0, 1, 5), False, None)), +('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), +('Expression', ('BoolOp', (1, 0, 1, 6), ('Or',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Sub',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Div',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('MatMult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('FloorDiv',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Pow',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mod',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('RShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('LShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitXor',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitOr',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitAnd',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('UAdd',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('USub',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('Invert',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), +('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), +('Expression', ('Dict', (1, 0, 1, 2), [], [])), +('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), +('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), +('Expression', ('List', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 4, 6), [('Constant', (2, 6, 2, 7), 1, None)], ('Load',))), +('Expression', ('Set', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)])), +('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('DictComp', (1, 0, 1, 25), ('Name', (1, 1, 1, 2), 'k', ('Load',)), ('Name', (1, 4, 1, 5), 'v', ('Load',)), [('comprehension', ('Tuple', (1, 10, 1, 14), [('Name', (1, 10, 1, 11), 'k', ('Store',)), ('Name', (1, 13, 1, 14), 'v', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [('Name', (1, 23, 1, 24), 'd', ('Load',))], 0)])), +('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Eq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('LtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('GtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotEq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Is',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('IsNot',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('In',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotIn',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Call', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [], [])), +('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), +('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), +('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), +('Expression', ('Constant', (1, 0, 1, 2), 10, None)), +('Expression', ('Constant', (1, 0, 1, 2), 1j, None)), +('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), +('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), +('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), +('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), ('Constant', (1, 4, 1, 5), 1, None), None, None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), None, ('Constant', (1, 5, 1, 6), 1, None), None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 8), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 7), None, None, ('Constant', (1, 6, 1, 7), 1, None)), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 10), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 9), ('Constant', (1, 4, 1, 5), 1, None), ('Constant', (1, 6, 1, 7), 1, None), ('Constant', (1, 8, 1, 9), 1, None)), ('Load',))), +('Expression', ('IfExp', (1, 0, 1, 21), ('Name', (1, 9, 1, 10), 'x', ('Load',)), ('Call', (1, 0, 1, 5), ('Name', (1, 0, 1, 3), 'foo', ('Load',)), [], []), ('Call', (1, 16, 1, 21), ('Name', (1, 16, 1, 19), 'bar', ('Load',)), [], []))), +('Expression', ('JoinedStr', (1, 0, 1, 6), [('FormattedValue', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), +] +main() diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast/test_ast.py similarity index 59% rename from Lib/test/test_ast.py rename to Lib/test/test_ast/test_ast.py index 3789ac22e3899c..67ab8cf6baf657 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1,5 +1,6 @@ import ast import builtins +import copy import dis import enum import os @@ -8,9 +9,8 @@ import textwrap import types import unittest -import warnings import weakref -from functools import partial +from pathlib import Path from textwrap import dedent try: import _testinternalcapi @@ -18,277 +18,28 @@ _testinternalcapi = None from test import support -from test.support.import_helper import import_fresh_module from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin +from test.test_ast.utils import to_tuple +from test.test_ast.snippets import ( + eval_tests, eval_results, exec_tests, exec_results, single_tests, single_results +) + + +STDLIB = os.path.dirname(ast.__file__) +STDLIB_FILES = [fn for fn in os.listdir(STDLIB) if fn.endswith(".py")] +STDLIB_FILES.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) + +AST_REPR_DATA_FILE = Path(__file__).parent / "data" / "ast_repr.txt" + +def ast_repr_get_test_cases() -> list[str]: + return exec_tests + eval_tests + + +def ast_repr_update_snapshots() -> None: + data = [repr(ast.parse(test)) for test in ast_repr_get_test_cases()] + AST_REPR_DATA_FILE.write_text("\n".join(data)) -def to_tuple(t): - if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis: - return t - elif isinstance(t, list): - return [to_tuple(e) for e in t] - result = [t.__class__.__name__] - if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): - result.append((t.lineno, t.col_offset)) - if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): - result[-1] += (t.end_lineno, t.end_col_offset) - if t._fields is None: - return tuple(result) - for f in t._fields: - result.append(to_tuple(getattr(t, f))) - return tuple(result) - - -# These tests are compiled through "exec" -# There should be at least one test per statement -exec_tests = [ - # None - "None", - # Module docstring - "'module docstring'", - # FunctionDef - "def f(): pass", - # FunctionDef with docstring - "def f(): 'function docstring'", - # FunctionDef with arg - "def f(a): pass", - # FunctionDef with arg and default value - "def f(a=0): pass", - # FunctionDef with varargs - "def f(*args): pass", - # FunctionDef with varargs as TypeVarTuple - "def f(*args: *Ts): pass", - # FunctionDef with varargs as unpacked Tuple - "def f(*args: *tuple[int, ...]): pass", - # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple - "def f(*args: *tuple[int, *Ts]): pass", - # FunctionDef with kwargs - "def f(**kwargs): pass", - # FunctionDef with all kind of args and docstring - "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", - # FunctionDef with type annotation on return involving unpacking - "def f() -> tuple[*Ts]: pass", - "def f() -> tuple[int, *Ts]: pass", - "def f() -> tuple[int, *tuple[int, ...]]: pass", - # ClassDef - "class C:pass", - # ClassDef with docstring - "class C: 'docstring for class C'", - # ClassDef, new style class - "class C(object): pass", - # Return - "def f():return 1", - # Delete - "del v", - # Assign - "v = 1", - "a,b = c", - "(a,b) = c", - "[a,b] = c", - # AnnAssign with unpacked types - "x: tuple[*Ts]", - "x: tuple[int, *Ts]", - "x: tuple[int, *tuple[str, ...]]", - # AugAssign - "v += 1", - # For - "for v in v:pass", - # While - "while v:pass", - # If - "if v:pass", - # If-Elif - "if a:\n pass\nelif b:\n pass", - # If-Elif-Else - "if a:\n pass\nelif b:\n pass\nelse:\n pass", - # With - "with x as y: pass", - "with x as y, z as q: pass", - "with (x as y): pass", - "with (x, y): pass", - # Raise - "raise Exception('string')", - # TryExcept - "try:\n pass\nexcept Exception:\n pass", - # TryFinally - "try:\n pass\nfinally:\n pass", - # TryStarExcept - "try:\n pass\nexcept* Exception:\n pass", - # Assert - "assert v", - # Import - "import sys", - # ImportFrom - "from sys import v", - # Global - "global v", - # Expr - "1", - # Pass, - "pass", - # Break - "for v in v:break", - # Continue - "for v in v:continue", - # for statements with naked tuples (see http://bugs.python.org/issue6704) - "for a,b in c: pass", - "for (a,b) in c: pass", - "for [a,b] in c: pass", - # Multiline generator expression (test for .lineno & .col_offset) - """( - ( - Aa - , - Bb - ) - for - Aa - , - Bb in Cc - )""", - # dictcomp - "{a : b for w in x for m in p if g}", - # dictcomp with naked tuple - "{a : b for v,w in x}", - # setcomp - "{r for l in x if g}", - # setcomp with naked tuple - "{r for l,m in x}", - # AsyncFunctionDef - "async def f():\n 'async function'\n await something()", - # AsyncFor - "async def f():\n async for e in i: 1\n else: 2", - # AsyncWith - "async def f():\n async with a as b: 1", - # PEP 448: Additional Unpacking Generalizations - "{**{1:2}, 2:3}", - "{*{1, 2}, 3}", - # Asynchronous comprehensions - "async def f():\n [i async for b in c]", - # Decorated FunctionDef - "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", - # Decorated AsyncFunctionDef - "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", - # Decorated ClassDef - "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", - # Decorator with generator argument - "@deco(a for a in b)\ndef f(): pass", - # Decorator with attribute - "@a.b.c\ndef f(): pass", - # Simple assignment expression - "(a := 1)", - # Positional-only arguments - "def f(a, /,): pass", - "def f(a, /, c, d, e): pass", - "def f(a, /, c, *, d, e): pass", - "def f(a, /, c, *, d, e, **kwargs): pass", - # Positional-only arguments with defaults - "def f(a=1, /,): pass", - "def f(a=1, /, b=2, c=4): pass", - "def f(a=1, /, b=2, *, c=4): pass", - "def f(a=1, /, b=2, *, c): pass", - "def f(a=1, /, b=2, *, c=4, **kwargs): pass", - "def f(a=1, /, b=2, *, c, **kwargs): pass", - # Type aliases - "type X = int", - "type X[T] = int", - "type X[T, *Ts, **P] = (T, Ts, P)", - "type X[T: int, *Ts, **P] = (T, Ts, P)", - "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", - # Generic classes - "class X[T]: pass", - "class X[T, *Ts, **P]: pass", - "class X[T: int, *Ts, **P]: pass", - "class X[T: (int, str), *Ts, **P]: pass", - # Generic functions - "def f[T](): pass", - "def f[T, *Ts, **P](): pass", - "def f[T: int, *Ts, **P](): pass", - "def f[T: (int, str), *Ts, **P](): pass", -] - -# These are compiled through "single" -# because of overlap with "eval", it just tests what -# can't be tested with "eval" -single_tests = [ - "1+2" -] - -# These are compiled through "eval" -# It should test all expressions -eval_tests = [ - # None - "None", - # BoolOp - "a and b", - # BinOp - "a + b", - # UnaryOp - "not v", - # Lambda - "lambda:None", - # Dict - "{ 1:2 }", - # Empty dict - "{}", - # Set - "{None,}", - # Multiline dict (test for .lineno & .col_offset) - """{ - 1 - : - 2 - }""", - # ListComp - "[a for b in c if d]", - # GeneratorExp - "(a for b in c if d)", - # Comprehensions with multiple for targets - "[(a,b) for a,b in c]", - "[(a,b) for (a,b) in c]", - "[(a,b) for [a,b] in c]", - "{(a,b) for a,b in c}", - "{(a,b) for (a,b) in c}", - "{(a,b) for [a,b] in c}", - "((a,b) for a,b in c)", - "((a,b) for (a,b) in c)", - "((a,b) for [a,b] in c)", - # Yield - yield expressions can't work outside a function - # - # Compare - "1 < 2 < 3", - # Call - "f(1,2,c=3,*d,**e)", - # Call with multi-character starred - "f(*[0, 1])", - # Call with a generator argument - "f(a for a in b)", - # Num - "10", - # Str - "'string'", - # Attribute - "a.b", - # Subscript - "a[b:c]", - # Name - "v", - # List - "[1,2,3]", - # Empty list - "[]", - # Tuple - "1,2,3", - # Tuple - "(1,2,3)", - # Empty tuple - "()", - # Combination - "a.b.c.d(a.b[1:2])", -] - -# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension -# excepthandler, arguments, keywords, alias class AST_Tests(unittest.TestCase): maxDiff = None @@ -333,6 +84,23 @@ def test_AST_objects(self): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) + def test_AST_fields_NULL_check(self): + # See: https://github.com/python/cpython/issues/126105 + old_value = ast.AST._fields + + def cleanup(): + ast.AST._fields = old_value + self.addCleanup(cleanup) + + del ast.AST._fields + + msg = "type object 'ast.AST' has no attribute '_fields'" + # Both examples used to crash: + with self.assertRaisesRegex(AttributeError, msg): + ast.AST(arg1=123) + with self.assertRaisesRegex(AttributeError, msg): + ast.AST() + def test_AST_garbage_collection(self): class X: pass @@ -492,151 +260,59 @@ def test_base_classes(self): self.assertTrue(issubclass(ast.comprehension, ast.AST)) self.assertTrue(issubclass(ast.Gt, ast.AST)) - def test_import_deprecated(self): - ast = import_fresh_module('ast') - depr_regex = ( - r'ast\.{} is deprecated and will be removed in Python 3.14; ' - r'use ast\.Constant instead' - ) - for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': - with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): - getattr(ast, name) - - def test_field_attr_existence_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): - item = getattr(ast, name) - if self._is_ast_node(name, item): - with self.subTest(item): - with self.assertWarns(DeprecationWarning): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - def test_field_attr_existence(self): for name, item in ast.__dict__.items(): - # These emit DeprecationWarnings - if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: - continue # constructor has a different signature if name == 'Index': continue if self._is_ast_node(name, item): - x = item() + x = self._construct_ast_class(item) if isinstance(x, ast.AST): self.assertIs(type(x._fields), tuple) + def _construct_ast_class(self, cls): + kwargs = {} + for name, typ in cls.__annotations__.items(): + if typ is str: + kwargs[name] = 'capybara' + elif typ is int: + kwargs[name] = 42 + elif typ is object: + kwargs[name] = b'capybara' + elif isinstance(typ, type) and issubclass(typ, ast.AST): + kwargs[name] = self._construct_ast_class(typ) + return cls(**kwargs) + def test_arguments(self): x = ast.arguments() self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', 'kw_defaults', 'kwarg', 'defaults')) - - with self.assertRaises(AttributeError): - x.args + self.assertEqual(x.__annotations__, { + 'posonlyargs': list[ast.arg], + 'args': list[ast.arg], + 'vararg': ast.arg | None, + 'kwonlyargs': list[ast.arg], + 'kw_defaults': list[ast.expr], + 'kwarg': ast.arg | None, + 'defaults': list[ast.expr], + }) + + self.assertEqual(x.args, []) self.assertIsNone(x.vararg) x = ast.arguments(*range(1, 8)) self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) - def test_field_attr_writable_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - def test_field_attr_writable(self): - x = ast.Constant() + x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) - def test_classattrs_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - x = ast.Num() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_classattrs(self): - x = ast.Constant() + with self.assertWarns(DeprecationWarning): + x = ast.Constant() self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): @@ -651,7 +327,7 @@ def test_classattrs(self): with self.assertRaises(AttributeError): x.foobar - x = ast.Constant(lineno=2) + x = ast.Constant(lineno=2, value=3) self.assertEqual(x.lineno, 2) x = ast.Constant(42, lineno=0) @@ -662,8 +338,9 @@ def test_classattrs(self): self.assertRaises(TypeError, ast.Constant, 1, None, 2) self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') + # Arbitrary keyword arguments are supported (but deprecated) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) @@ -678,190 +355,6 @@ def test_classattrs(self): self.assertIs(ast.Constant(None).value, None) self.assertIs(ast.Constant(...).value, ...) - def test_realtype(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.assertIs(type(ast.Num(42)), ast.Constant) - self.assertIs(type(ast.Num(4.25)), ast.Constant) - self.assertIs(type(ast.Num(4.25j)), ast.Constant) - self.assertIs(type(ast.Str('42')), ast.Constant) - self.assertIs(type(ast.Bytes(b'42')), ast.Constant) - self.assertIs(type(ast.NameConstant(True)), ast.Constant) - self.assertIs(type(ast.NameConstant(False)), ast.Constant) - self.assertIs(type(ast.NameConstant(None)), ast.Constant) - self.assertIs(type(ast.Ellipsis()), ast.Constant) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - def test_isinstance(self): - from ast import Constant - - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - cls_depr_msg = ( - 'ast.{} is deprecated and will be removed in Python 3.14; ' - 'use ast.Constant instead' - ) - - assertNumDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") - ) - assertStrDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") - ) - assertBytesDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") - ) - assertNameConstantDeprecated = partial( - self.assertWarnsRegex, - DeprecationWarning, - cls_depr_msg.format("NameConstant") - ) - assertEllipsisDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") - ) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - n = Num(arg) - with assertNumDeprecated(): - self.assertIsInstance(n, Num) - - with assertStrDeprecated(): - s = Str('42') - with assertStrDeprecated(): - self.assertIsInstance(s, Str) - - with assertBytesDeprecated(): - b = Bytes(b'42') - with assertBytesDeprecated(): - self.assertIsInstance(b, Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - n = NameConstant(arg) - with assertNameConstantDeprecated(): - self.assertIsInstance(n, NameConstant) - - with assertEllipsisDeprecated(): - e = Ellipsis() - with assertEllipsisDeprecated(): - self.assertIsInstance(e, Ellipsis) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertIsInstance(Constant(arg), Num) - - with assertStrDeprecated(): - self.assertIsInstance(Constant('42'), Str) - - with assertBytesDeprecated(): - self.assertIsInstance(Constant(b'42'), Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - self.assertIsInstance(Constant(arg), NameConstant) - - with assertEllipsisDeprecated(): - self.assertIsInstance(Constant(...), Ellipsis) - - with assertStrDeprecated(): - s = Str('42') - assertNumDeprecated(self.assertNotIsInstance, s, Num) - assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) - - with assertNumDeprecated(): - n = Num(42) - assertStrDeprecated(self.assertNotIsInstance, n, Str) - assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) - - with assertNameConstantDeprecated(): - n = NameConstant(True) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - with assertNameConstantDeprecated(): - n = NameConstant(False) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - for arg in '42', True, False: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(arg), Num) - - assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) - - class S(str): pass - with assertStrDeprecated(): - self.assertIsInstance(Constant(S('42')), Str) - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(S('42')), Num) - - def test_constant_subclasses_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, 'spam') - self.assertIs(type(n), N) - self.assertIsInstance(n, N) - self.assertIsInstance(n, ast.Num) - self.assertNotIsInstance(n, N2) - self.assertNotIsInstance(ast.Num(42), N) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertIs(type(n), N) - - self.assertEqual([str(w.message) for w in wlog], [ - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - ]) - def test_constant_subclasses(self): class N(ast.Constant): def __init__(self, *args, **kwargs): @@ -888,8 +381,9 @@ def test_module(self): self.assertEqual(x.body, body) def test_nodeclasses(self): - # Zero arguments constructor explicitly allowed - x = ast.BinOp() + # Zero arguments constructor explicitly allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp() self.assertEqual(x._fields, ('left', 'op', 'right')) # Random attribute allowed too @@ -927,8 +421,9 @@ def test_nodeclasses(self): self.assertEqual(x.right, 3) self.assertEqual(x.lineno, 0) - # Random kwargs also allowed - x = ast.BinOp(1, 2, 3, foobarbaz=42) + # Random kwargs also allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp(1, 2, 3, foobarbaz=42) self.assertEqual(x.foobarbaz, 42) def test_no_fields(self): @@ -936,20 +431,12 @@ def test_no_fields(self): x = ast.Sub() self.assertEqual(x._fields, ()) - def test_pickling(self): - import pickle - - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): - ast2 = pickle.loads(pickle.dumps(ast, protocol)) - self.assertEqual(to_tuple(ast2), to_tuple(ast)) - def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "", "exec") - self.assertIn("but got None: for node, attr, source in tests: self.assert_none_check(node, attr, source) + def test_repr(self) -> None: + snapshots = AST_REPR_DATA_FILE.read_text().split("\n") + for test, snapshot in zip(ast_repr_get_test_cases(), snapshots, strict=True): + with self.subTest(test_input=test): + self.assertEqual(repr(ast.parse(test)), snapshot) + + def test_repr_large_input_crash(self): + # gh-125010: Fix use-after-free in ast repr() + source = "0x0" + "e" * 10_000 + with self.assertRaisesRegex(ValueError, + r"Exceeds the limit \(\d+ digits\)"): + repr(ast.Constant(value=eval(source))) + + +class CopyTests(unittest.TestCase): + """Test copying and pickling AST nodes.""" + + @staticmethod + def iter_ast_classes(): + """Iterate over the (native) subclasses of ast.AST recursively. + + This excludes the special class ast.Index since its constructor + returns an integer. + """ + def do(cls): + if cls.__module__ != 'ast': + return + if cls is ast.Index: + return + + yield cls + for sub in cls.__subclasses__(): + yield from do(sub) + + yield from do(ast.AST) + + def test_pickling(self): + import pickle + + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + for code in exec_tests: + with self.subTest(code=code, protocol=protocol): + tree = compile(code, "?", "exec", 0x400) + ast2 = pickle.loads(pickle.dumps(tree, protocol)) + self.assertEqual(to_tuple(ast2), to_tuple(tree)) + + def test_copy_with_parents(self): + # gh-120108 + code = """ + ('',) + while i < n: + if ch == '': + ch = format[i] + if ch == '': + if freplace is None: + '' % getattr(object) + elif ch == '': + if zreplace is None: + if hasattr: + offset = object.utcoffset() + if offset is not None: + if offset.days < 0: + offset = -offset + h = divmod(timedelta(hours=0)) + if u: + zreplace = '' % (sign,) + elif s: + zreplace = '' % (sign,) + else: + zreplace = '' % (sign,) + elif ch == '': + if Zreplace is None: + Zreplace = '' + if hasattr(object): + s = object.tzname() + if s is not None: + Zreplace = s.replace('') + newformat.append(Zreplace) + else: + push('') + else: + push(ch) + + """ + tree = ast.parse(textwrap.dedent(code)) + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + child.parent = node + try: + with support.infinite_recursion(200): + tree2 = copy.deepcopy(tree) + finally: + # Singletons like ast.Load() are shared; make sure we don't + # leave them mutated after this test. + for node in ast.walk(tree): + if hasattr(node, "parent"): + del node.parent + + for node in ast.walk(tree2): + for child in ast.iter_child_nodes(node): + if hasattr(child, "parent") and not isinstance(child, ( + ast.expr_context, ast.boolop, ast.unaryop, ast.cmpop, ast.operator, + )): + self.assertEqual(to_tuple(child.parent), to_tuple(node)) + + def test_replace_interface(self): + for klass in self.iter_ast_classes(): + with self.subTest(klass=klass): + self.assertTrue(hasattr(klass, '__replace__')) + + fields = set(klass._fields) + with self.subTest(klass=klass, fields=fields): + node = klass(**dict.fromkeys(fields)) + # forbid positional arguments in replace() + self.assertRaises(TypeError, copy.replace, node, 1) + self.assertRaises(TypeError, node.__replace__, 1) + + def test_replace_native(self): + for klass in self.iter_ast_classes(): + fields = set(klass._fields) + attributes = set(klass._attributes) + + with self.subTest(klass=klass, fields=fields, attributes=attributes): + # use of object() to ensure that '==' and 'is' + # behave similarly in ast.compare(node, repl) + old_fields = {field: object() for field in fields} + old_attrs = {attr: object() for attr in attributes} + + # check shallow copy + node = klass(**old_fields) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + # check when passing using attributes (they may be optional!) + node = klass(**old_fields, **old_attrs) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + + for field in fields: + # check when we sometimes have attributes and sometimes not + for init_attrs in [{}, old_attrs]: + node = klass(**old_fields, **init_attrs) + # only change a single field (do not change attributes) + new_value = object() + repl = copy.replace(node, **{field: new_value}) + for f in fields: + old_value = old_fields[f] + # assert that there is no side-effect + self.assertIs(getattr(node, f), old_value) + # check the changes + if f != field: + self.assertIs(getattr(repl, f), old_value) + else: + self.assertIs(getattr(repl, f), new_value) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + for attribute in attributes: + node = klass(**old_fields, **old_attrs) + # only change a single attribute (do not change fields) + new_attr = object() + repl = copy.replace(node, **{attribute: new_attr}) + for a in attributes: + old_attr = old_attrs[a] + # assert that there is no side-effect + self.assertIs(getattr(node, a), old_attr) + # check the changes + if a != attribute: + self.assertIs(getattr(repl, a), old_attr) + else: + self.assertIs(getattr(repl, a), new_attr) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + def test_replace_accept_known_class_fields(self): + nid, ctx = object(), object() + + node = ast.Name(id=nid, ctx=ctx) + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + + new_nid = object() + repl = copy.replace(node, id=new_nid) + # assert that there is no side-effect + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + # check the changes + self.assertIs(repl.id, new_nid) + self.assertIs(repl.ctx, node.ctx) # no changes + + def test_replace_accept_known_class_attributes(self): + node = ast.parse('x').body[0].value + self.assertEqual(node.id, 'x') + self.assertEqual(node.lineno, 1) + + # constructor allows any type so replace() should do the same + lineno = object() + repl = copy.replace(node, lineno=lineno) + # assert that there is no side-effect + self.assertEqual(node.lineno, 1) + # check the changes + self.assertEqual(repl.id, node.id) + self.assertEqual(repl.ctx, node.ctx) + self.assertEqual(repl.lineno, lineno) + + _, _, state = node.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], 1) + + _, _, state = repl.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], lineno) + + def test_replace_accept_known_custom_class_fields(self): + class MyNode(ast.AST): + _fields = ('name', 'data') + __annotations__ = {'name': str, 'data': object} + __match_args__ = ('name', 'data') + + name, data = 'name', object() + + node = MyNode(name, data) + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check shallow copy + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the shallow copy + self.assertIs(repl.name, name) + self.assertIs(repl.data, data) + + node = MyNode(name, data) + repl_data = object() + # replace custom but known field + repl = copy.replace(node, data=repl_data) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the changes + self.assertIs(repl.name, node.name) + self.assertIs(repl.data, repl_data) + + def test_replace_accept_known_custom_class_attributes(self): + class MyNode(ast.AST): + x = 0 + y = 1 + _attributes = ('x', 'y') + + node = MyNode() + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + + y = object() + repl = copy.replace(node, y=y) + # assert that there is no side-effect + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + # check the changes + self.assertEqual(repl.x, 0) + self.assertEqual(repl.y, y) + + def test_replace_ignore_known_custom_instance_fields(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # assert initial values + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # shallow copy, but drops extra fields + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'x') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + # change known native field + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + def test_replace_reject_missing_field(self): + # case: warn if deleted field is not replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + msg = "Name.__replace__ missing 1 keyword argument: 'id'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node) + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + + # case: do not raise if deleted field is replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + + def test_replace_reject_known_custom_instance_fields_commits(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # explicit rejection of known instance fields + self.assertTrue(hasattr(node, 'extra')) + msg = "Name.__replace__ got an unexpected keyword argument 'extra'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, extra=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + + def test_replace_reject_unknown_instance_fields(self): + node = ast.parse('x').body[0].value + context = node.ctx + + # explicit rejection of unknown extra fields + self.assertRaises(AttributeError, getattr, node, 'unknown') + msg = "Name.__replace__ got an unexpected keyword argument 'unknown'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, unknown=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertRaises(AttributeError, getattr, node, 'unknown') + class ASTHelpers_Test(unittest.TestCase): maxDiff = None @@ -1198,21 +1171,20 @@ def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], " - "keywords=[]))], type_ignores=[])" + "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])" ) self.assertEqual(ast.dump(node, annotate_fields=False), "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')], []))], [])" + "Constant('and cheese')]))])" ) self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " - "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " + "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])" + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" ) def test_dump_indent(self): @@ -1225,9 +1197,7 @@ def test_dump_indent(self): func=Name(id='spam', ctx=Load()), args=[ Name(id='eggs', ctx=Load()), - Constant(value='and cheese')], - keywords=[]))], - type_ignores=[])""") + Constant(value='and cheese')]))])""") self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ Module( \t[ @@ -1236,9 +1206,7 @@ def test_dump_indent(self): \t\t\t\tName('spam', Load()), \t\t\t\t[ \t\t\t\t\tName('eggs', Load()), -\t\t\t\t\tConstant('and cheese')], -\t\t\t\t[]))], -\t[])""") +\t\t\t\t\tConstant('and cheese')]))])""") self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ Module( body=[ @@ -1265,7 +1233,6 @@ def test_dump_indent(self): col_offset=11, end_lineno=1, end_col_offset=23)], - keywords=[], lineno=1, col_offset=0, end_lineno=1, @@ -1273,8 +1240,7 @@ def test_dump_indent(self): lineno=1, col_offset=0, end_lineno=1, - end_col_offset=24)], - type_ignores=[])""") + end_col_offset=24)])""") def test_dump_incomplete(self): node = ast.Raise(lineno=3, col_offset=4) @@ -1304,6 +1270,119 @@ def test_dump_incomplete(self): self.assertEqual(ast.dump(node, annotate_fields=False), "Raise(cause=Name('e', Load()))" ) + # Arguments: + node = ast.arguments(args=[ast.arg("x")]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments(args=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments() + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments()", + ) + # Classes: + node = ast.ClassDef( + 'T', + [], + [ast.keyword('a', ast.Constant(None))], + [], + [ast.Name('dataclass', ctx=ast.Load())], + ) + self.assertEqual(ast.dump(node), + "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", + ) + + def test_dump_show_empty(self): + def check_node(node, empty, full, **kwargs): + with self.subTest(show_empty=False): + self.assertEqual( + ast.dump(node, show_empty=False, **kwargs), + empty, + ) + with self.subTest(show_empty=True): + self.assertEqual( + ast.dump(node, show_empty=True, **kwargs), + full, + ) + + def check_text(code, empty, full, **kwargs): + check_node(ast.parse(code), empty, full, **kwargs) + + check_node( + ast.arguments(), + empty="arguments()", + full="arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[])", + ) + + check_node( + # Corner case: there are no real `Name` instances with `id=''`: + ast.Name(id='', ctx=ast.Load()), + empty="Name(id='', ctx=Load())", + full="Name(id='', ctx=Load())", + ) + + check_node( + ast.MatchSingleton(value=None), + empty="MatchSingleton(value=None)", + full="MatchSingleton(value=None)", + ) + + check_node( + ast.Constant(value=None), + empty="Constant(value=None)", + full="Constant(value=None)", + ) + + check_node( + ast.Constant(value=''), + empty="Constant(value='')", + full="Constant(value='')", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))], decorator_list=[], type_params=[])], type_ignores=[])", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], decorator_list=[], type_params=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)], type_ignores=[])", + include_attributes=True, + ) + + check_text( + 'spam(eggs, "and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], keywords=[]))], type_ignores=[])", + ) + + check_text( + 'spam(eggs, text="and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))], type_ignores=[])", + ) + + check_text( + "import _ast as ast; from module import sub", + empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)])", + full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", + ) def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') @@ -1314,8 +1393,9 @@ def test_copy_location(self): 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' 'col_offset=0, end_lineno=1, end_col_offset=5))' ) - src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1) - new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None)) + func = ast.Name('spam', ast.Load()) + src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func) + new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func)) self.assertIsNone(new.end_lineno) self.assertIsNone(new.end_col_offset) self.assertEqual(new.lineno, 1) @@ -1331,14 +1411,13 @@ def test_fix_missing_locations(self): "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " - "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=12)], lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], " - "type_ignores=[])" + "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" ) def test_increment_lineno(self): @@ -1410,6 +1489,12 @@ def test_get_docstring(self): node = ast.parse('async def foo():\n """spam\n ham"""') self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') + node = ast.parse('async def foo():\n """spam\n ham"""') + self.assertEqual(ast.get_docstring(node.body[0], clean=False), 'spam\n ham') + + node = ast.parse('x') + self.assertRaises(TypeError, ast.get_docstring, node.body[0]) + def test_get_docstring_none(self): self.assertIsNone(ast.get_docstring(ast.parse(''))) node = ast.parse('x = "not docstring"') @@ -1434,6 +1519,9 @@ def test_get_docstring_none(self): node = ast.parse('async def foo():\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('async def foo():\n 42') + self.assertIsNone(ast.get_docstring(node.body[0])) + def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): node = ast.parse( '"""line one\nline two"""\n\n' @@ -1574,15 +1662,15 @@ def test_level_as_none(self): self.assertIn('sleep', ns) def test_recursion_direct(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e with self.assertRaises(RecursionError): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") def test_recursion_indirect(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = f f.operand = e with self.assertRaises(RecursionError): @@ -1961,32 +2049,6 @@ def test_call(self): call = ast.Call(func, args, bad_keywords) self.expr(call, "must have Load context") - def test_num(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") @@ -2026,31 +2088,17 @@ def test_list(self): def test_tuple(self): self._sequence(ast.Tuple) - def test_nameconstant(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import NameConstant - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.expr(ast.NameConstant(4)) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - @support.requires_resource('cpu') def test_stdlib_validates(self): - stdlib = os.path.dirname(ast.__file__) - tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] - tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) - for module in tests: + for module in STDLIB_FILES: with self.subTest(module): - fn = os.path.join(stdlib, module) + fn = os.path.join(STDLIB, module) with open(fn, "r", encoding="utf-8") as fp: source = fp.read() mod = ast.parse(source, fn) compile(mod, fn, "exec") + mod2 = ast.parse(source, fn) + self.assertTrue(ast.compare(mod, mod2)) constant_1 = ast.Constant(1) pattern_1 = ast.MatchValue(constant_1) @@ -2228,7 +2276,7 @@ def test_validation(self): "got an invalid type in Constant: list") def test_singletons(self): - for const in (None, False, True, Ellipsis, b'', frozenset()): + for const in (None, False, True, Ellipsis, b''): with self.subTest(const=const): value = self.compile_constant(const) self.assertIs(value, const) @@ -2272,7 +2320,7 @@ def get_load_const(self, tree): co = compile(tree, '', 'exec') consts = [] for instr in dis.get_instructions(co): - if instr.opname == 'LOAD_CONST' or instr.opname == 'RETURN_CONST': + if instr.opcode in dis.hasconst: consts.append(instr.argval) return consts @@ -2280,7 +2328,7 @@ def get_load_const(self, tree): def test_load_const(self): consts = [None, True, False, - 124, + 1000, 2.0, 3j, "unicode", @@ -2692,76 +2740,15 @@ def test_source_segment_missing_info(self): self.assertIsNone(ast.get_source_segment(s, x)) self.assertIsNone(ast.get_source_segment(s, y)) -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, 'Num', node.n)) - def visit_Str(self, node): - log.append((node.lineno, 'Str', node.s)) - def visit_Bytes(self, node): - log.append((node.lineno, 'Bytes', node.s)) - def visit_NameConstant(self, node): - log.append((node.lineno, 'NameConstant', node.value)) - def visit_Ellipsis(self, node): - log.append((node.lineno, 'Ellipsis', ...)) - mod = ast.parse(dedent('''\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - ''')) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - visitor.visit(mod) - self.assertEqual(log, [ - (1, 'Num', 42), - (2, 'Num', 4.25), - (3, 'Num', 4.25j), - (4, 'Str', 'string'), - (5, 'Bytes', b'bytes'), - (6, 'NameConstant', True), - (7, 'NameConstant', None), - (8, 'Ellipsis', ...), - ]) - self.assertEqual([str(w.message) for w in wlog], [ - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Str is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Bytes is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_Ellipsis is deprecated; add visit_Constant', - ]) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer - - def assertASTTransformation(self, tranformer_class, + +class NodeTransformerTests(ASTTestMixin, unittest.TestCase): + def assertASTTransformation(self, transformer_class, initial_code, expected_code): initial_ast = ast.parse(dedent(initial_code)) expected_ast = ast.parse(dedent(expected_code)) - tranformer = tranformer_class() - result_ast = ast.fix_missing_locations(tranformer.visit(initial_ast)) + transformer = transformer_class() + result_ast = ast.fix_missing_locations(transformer.visit(initial_ast)) self.assertASTEqual(result_ast, expected_ast) @@ -2870,6 +2857,126 @@ def visit_Call(self, node: ast.Call): self.assertASTTransformation(PrintToLog, code, expected) +class ASTConstructorTests(unittest.TestCase): + """Test the autogenerated constructors for AST nodes.""" + + def test_FunctionDef(self): + args = ast.arguments() + self.assertEqual(args.args, []) + self.assertEqual(args.posonlyargs, []) + with self.assertWarnsRegex(DeprecationWarning, + r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): + node = ast.FunctionDef(args=args) + self.assertFalse(hasattr(node, "name")) + self.assertEqual(node.decorator_list, []) + node = ast.FunctionDef(name='foo', args=args) + self.assertEqual(node.name, 'foo') + self.assertEqual(node.decorator_list, []) + + def test_expr_context(self): + name = ast.Name("x") + self.assertEqual(name.id, "x") + self.assertIsInstance(name.ctx, ast.Load) + + name2 = ast.Name("x", ast.Store()) + self.assertEqual(name2.id, "x") + self.assertIsInstance(name2.ctx, ast.Store) + + name3 = ast.Name("x", ctx=ast.Del()) + self.assertEqual(name3.id, "x") + self.assertIsInstance(name3.ctx, ast.Del) + + with self.assertWarnsRegex(DeprecationWarning, + r"Name\.__init__ missing 1 required positional argument: 'id'"): + name3 = ast.Name() + + def test_custom_subclass_with_no_fields(self): + class NoInit(ast.AST): + pass + + obj = NoInit() + self.assertIsInstance(obj, NoInit) + self.assertEqual(obj.__dict__, {}) + + def test_fields_but_no_field_types(self): + class Fields(ast.AST): + _fields = ('a',) + + obj = Fields() + with self.assertRaises(AttributeError): + obj.a + obj = Fields(a=1) + self.assertEqual(obj.a, 1) + + def test_fields_and_types(self): + class FieldsAndTypes(ast.AST): + _fields = ('a',) + _field_types = {'a': int | None} + a: int | None = None + + obj = FieldsAndTypes() + self.assertIs(obj.a, None) + obj = FieldsAndTypes(a=1) + self.assertEqual(obj.a, 1) + + def test_custom_attributes(self): + class MyAttrs(ast.AST): + _attributes = ("a", "b") + + obj = MyAttrs(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + with self.assertWarnsRegex(DeprecationWarning, + r"MyAttrs.__init__ got an unexpected keyword argument 'c'."): + obj = MyAttrs(c=3) + + def test_fields_and_types_no_default(self): + class FieldsAndTypesNoDefault(ast.AST): + _fields = ('a',) + _field_types = {'a': int} + + with self.assertWarnsRegex(DeprecationWarning, + r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\."): + obj = FieldsAndTypesNoDefault() + with self.assertRaises(AttributeError): + obj.a + obj = FieldsAndTypesNoDefault(a=1) + self.assertEqual(obj.a, 1) + + def test_incomplete_field_types(self): + class MoreFieldsThanTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None} + a: int | None = None + b: int | None = None + + with self.assertWarnsRegex( + DeprecationWarning, + r"Field 'b' is missing from MoreFieldsThanTypes\._field_types" + ): + obj = MoreFieldsThanTypes() + self.assertIs(obj.a, None) + self.assertIs(obj.b, None) + + obj = MoreFieldsThanTypes(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + def test_complete_field_types(self): + class _AllFieldTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None, 'b': list[str]} + # This must be set explicitly + a: int | None = None + # This will add an implicit empty list default + b: list[str] + + obj = _AllFieldTypes() + self.assertIs(obj.a, None) + self.assertEqual(obj.b, []) + + @support.cpython_only class ModuleStateTests(unittest.TestCase): # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. @@ -2971,154 +3078,303 @@ def test_cli_file_input(self): self.assertEqual(res.rc, 0) -def main(): - if __name__ != '__main__': - return - if sys.argv[1:] == ['-g']: - for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), - (eval_tests, "eval")): - print(kind+"_results = [") - for statement in statements: - tree = ast.parse(statement, "?", kind) - print("%r," % (to_tuple(tree),)) - print("]") - print("main()") - raise SystemExit - unittest.main() +class ASTOptimiziationTests(unittest.TestCase): + binop = { + "+": ast.Add(), + "-": ast.Sub(), + "*": ast.Mult(), + "/": ast.Div(), + "%": ast.Mod(), + "<<": ast.LShift(), + ">>": ast.RShift(), + "|": ast.BitOr(), + "^": ast.BitXor(), + "&": ast.BitAnd(), + "//": ast.FloorDiv(), + "**": ast.Pow(), + } + + unaryop = { + "~": ast.Invert(), + "+": ast.UAdd(), + "-": ast.USub(), + } + + def wrap_expr(self, expr): + return ast.Module(body=[ast.Expr(value=expr)]) + + def wrap_statement(self, statement): + return ast.Module(body=[statement]) + + def assert_ast(self, code, non_optimized_target, optimized_target): + non_optimized_tree = ast.parse(code, optimize=-1) + optimized_tree = ast.parse(code, optimize=1) + + # Is a non-optimized tree equal to a non-optimized target? + self.assertTrue( + ast.compare(non_optimized_tree, non_optimized_target), + f"{ast.dump(non_optimized_target)} must equal " + f"{ast.dump(non_optimized_tree)}", + ) + + # Is a optimized tree equal to a non-optimized target? + self.assertFalse( + ast.compare(optimized_tree, non_optimized_target), + f"{ast.dump(non_optimized_target)} must not equal " + f"{ast.dump(non_optimized_tree)}" + ) + + # Is a optimized tree is equal to an optimized target? + self.assertTrue( + ast.compare(optimized_tree, optimized_target), + f"{ast.dump(optimized_target)} must equal " + f"{ast.dump(optimized_tree)}", + ) + + def create_binop(self, operand, left=ast.Constant(1), right=ast.Constant(1)): + return ast.BinOp(left=left, op=self.binop[operand], right=right) + + def test_folding_binop(self): + code = "1 %s 1" + operators = self.binop.keys() + + for op in operators: + result_code = code % op + non_optimized_target = self.wrap_expr(self.create_binop(op)) + optimized_target = self.wrap_expr(ast.Constant(value=eval(result_code))) + + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) -#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### -exec_results = [ -('Module', [('Expr', (1, 0, 1, 4), ('Constant', (1, 0, 1, 4), None, None))], []), -('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), -('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), -('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), -('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), -('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), -('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), -('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), -('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), -('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), -('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), -('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), -('Module', [('With', (1, 0, 1, 19), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), ('Name', (1, 11, 1, 12), 'y', ('Store',)))], [('Pass', (1, 15, 1, 19))], None)], []), -('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), None), ('withitem', ('Name', (1, 9, 1, 10), 'y', ('Load',)), None)], [('Pass', (1, 13, 1, 17))], None)], []), -('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), -('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), -('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), -('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), -('Module', [('Global', (1, 0, 1, 8), ['v'])], []), -('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), -('Module', [('Pass', (1, 0, 1, 4))], []), -('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), -('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), -('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), -('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), -('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), -('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts'), ('ParamSpec', (1, 15, 1, 18), 'P')], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',))), ('TypeVarTuple', (1, 15, 1, 18), 'Ts'), ('ParamSpec', (1, 20, 1, 23), 'P')], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 22, 1, 25), 'Ts'), ('ParamSpec', (1, 27, 1, 30), 'P')], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), -('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts'), ('ParamSpec', (1, 16, 1, 19), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',))), ('TypeVarTuple', (1, 16, 1, 19), 'Ts'), ('ParamSpec', (1, 21, 1, 24), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 23, 1, 26), 'Ts'), ('ParamSpec', (1, 28, 1, 31), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts'), ('ParamSpec', (1, 14, 1, 17), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',))), ('TypeVarTuple', (1, 14, 1, 17), 'Ts'), ('ParamSpec', (1, 19, 1, 22), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 21, 1, 24), 'Ts'), ('ParamSpec', (1, 26, 1, 29), 'P')])], []), -] -single_results = [ -('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), -] -eval_results = [ -('Expression', ('Constant', (1, 0, 1, 4), None, None)), -('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), -('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), -('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), -('Expression', ('Dict', (1, 0, 1, 2), [], [])), -('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), -('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), -('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), -('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), -('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), -('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), -('Expression', ('Constant', (1, 0, 1, 2), 10, None)), -('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), -('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), -('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), -('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), -] -main() + # Multiplication of constant tuples must be folded + code = "(1,) * 3" + non_optimized_target = self.wrap_expr(self.create_binop("*", ast.Tuple(elts=[ast.Constant(value=1)]), ast.Constant(value=3))) + optimized_target = self.wrap_expr(ast.Constant(eval(code))) + + self.assert_ast(code, non_optimized_target, optimized_target) + + def test_folding_unaryop(self): + code = "%s1" + operators = self.unaryop.keys() + + def create_unaryop(operand): + return ast.UnaryOp(op=self.unaryop[operand], operand=ast.Constant(1)) + + for op in operators: + result_code = code % op + non_optimized_target = self.wrap_expr(create_unaryop(op)) + optimized_target = self.wrap_expr(ast.Constant(eval(result_code))) + + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) + + def test_folding_not(self): + code = "not (1 %s (1,))" + operators = { + "in": ast.In(), + "is": ast.Is(), + } + opt_operators = { + "is": ast.IsNot(), + "in": ast.NotIn(), + } + + def create_notop(operand): + return ast.UnaryOp(op=ast.Not(), operand=ast.Compare( + left=ast.Constant(value=1), + ops=[operators[operand]], + comparators=[ast.Tuple(elts=[ast.Constant(value=1)])] + )) + + for op in operators.keys(): + result_code = code % op + non_optimized_target = self.wrap_expr(create_notop(op)) + optimized_target = self.wrap_expr( + ast.Compare(left=ast.Constant(1), ops=[opt_operators[op]], comparators=[ast.Constant(value=(1,))]) + ) + + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) + + def test_folding_format(self): + code = "'%s' % (a,)" + + non_optimized_target = self.wrap_expr( + ast.BinOp( + left=ast.Constant(value="%s"), + op=ast.Mod(), + right=ast.Tuple(elts=[ast.Name(id='a')])) + ) + optimized_target = self.wrap_expr( + ast.JoinedStr( + values=[ + ast.FormattedValue(value=ast.Name(id='a'), conversion=115) + ] + ) + ) + + self.assert_ast(code, non_optimized_target, optimized_target) + + + def test_folding_tuple(self): + code = "(1,)" + + non_optimized_target = self.wrap_expr(ast.Tuple(elts=[ast.Constant(1)])) + optimized_target = self.wrap_expr(ast.Constant(value=(1,))) + + self.assert_ast(code, non_optimized_target, optimized_target) + + def test_folding_comparator(self): + code = "1 %s %s1%s" + operators = [("in", ast.In()), ("not in", ast.NotIn())] + braces = [ + ("[", "]", ast.List, (1,)), + ("{", "}", ast.Set, frozenset({1})), + ] + for left, right, non_optimized_comparator, optimized_comparator in braces: + for op, node in operators: + non_optimized_target = self.wrap_expr(ast.Compare( + left=ast.Constant(1), ops=[node], + comparators=[non_optimized_comparator(elts=[ast.Constant(1)])] + )) + optimized_target = self.wrap_expr(ast.Compare( + left=ast.Constant(1), ops=[node], + comparators=[ast.Constant(value=optimized_comparator)] + )) + self.assert_ast(code % (op, left, right), non_optimized_target, optimized_target) + + def test_folding_iter(self): + code = "for _ in %s1%s: pass" + braces = [ + ("[", "]", ast.List, (1,)), + ("{", "}", ast.Set, frozenset({1})), + ] + + for left, right, ast_cls, optimized_iter in braces: + non_optimized_target = self.wrap_statement(ast.For( + target=ast.Name(id="_", ctx=ast.Store()), + iter=ast_cls(elts=[ast.Constant(1)]), + body=[ast.Pass()] + )) + optimized_target = self.wrap_statement(ast.For( + target=ast.Name(id="_", ctx=ast.Store()), + iter=ast.Constant(value=optimized_iter), + body=[ast.Pass()] + )) + + self.assert_ast(code % (left, right), non_optimized_target, optimized_target) + + def test_folding_subscript(self): + code = "(1,)[0]" + + non_optimized_target = self.wrap_expr( + ast.Subscript(value=ast.Tuple(elts=[ast.Constant(value=1)]), slice=ast.Constant(value=0)) + ) + optimized_target = self.wrap_expr(ast.Constant(value=1)) + + self.assert_ast(code, non_optimized_target, optimized_target) + + def test_folding_type_param_in_function_def(self): + code = "def foo[%s = 1 + 1](): pass" + + unoptimized_binop = self.create_binop("+") + unoptimized_type_params = [ + ("T", "T", ast.TypeVar), + ("**P", "P", ast.ParamSpec), + ("*Ts", "Ts", ast.TypeVarTuple), + ] + + for type, name, type_param in unoptimized_type_params: + result_code = code % type + optimized_target = self.wrap_statement( + ast.FunctionDef( + name='foo', + args=ast.arguments(), + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=ast.Constant(2))] + ) + ) + non_optimized_target = self.wrap_statement( + ast.FunctionDef( + name='foo', + args=ast.arguments(), + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=unoptimized_binop)] + ) + ) + self.assert_ast(result_code, non_optimized_target, optimized_target) + + def test_folding_type_param_in_class_def(self): + code = "class foo[%s = 1 + 1]: pass" + + unoptimized_binop = self.create_binop("+") + unoptimized_type_params = [ + ("T", "T", ast.TypeVar), + ("**P", "P", ast.ParamSpec), + ("*Ts", "Ts", ast.TypeVarTuple), + ] + + for type, name, type_param in unoptimized_type_params: + result_code = code % type + optimized_target = self.wrap_statement( + ast.ClassDef( + name='foo', + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=ast.Constant(2))] + ) + ) + non_optimized_target = self.wrap_statement( + ast.ClassDef( + name='foo', + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=unoptimized_binop)] + ) + ) + self.assert_ast(result_code, non_optimized_target, optimized_target) + + def test_folding_type_param_in_type_alias(self): + code = "type foo[%s = 1 + 1] = 1" + + unoptimized_binop = self.create_binop("+") + unoptimized_type_params = [ + ("T", "T", ast.TypeVar), + ("**P", "P", ast.ParamSpec), + ("*Ts", "Ts", ast.TypeVarTuple), + ] + + for type, name, type_param in unoptimized_type_params: + result_code = code % type + optimized_target = self.wrap_statement( + ast.TypeAlias( + name=ast.Name(id='foo', ctx=ast.Store()), + type_params=[type_param(name=name, default_value=ast.Constant(2))], + value=ast.Constant(value=1), + ) + ) + non_optimized_target = self.wrap_statement( + ast.TypeAlias( + name=ast.Name(id='foo', ctx=ast.Store()), + type_params=[type_param(name=name, default_value=unoptimized_binop)], + value=ast.Constant(value=1), + ) + ) + self.assert_ast(result_code, non_optimized_target, optimized_target) + + +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + ast_repr_update_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_ast/utils.py b/Lib/test/test_ast/utils.py new file mode 100644 index 00000000000000..145e89ee94e935 --- /dev/null +++ b/Lib/test/test_ast/utils.py @@ -0,0 +1,15 @@ +def to_tuple(t): + if t is None or isinstance(t, (str, int, complex, float, bytes)) or t is Ellipsis: + return t + elif isinstance(t, list): + return [to_tuple(e) for e in t] + result = [t.__class__.__name__] + if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): + result.append((t.lineno, t.col_offset)) + if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): + result[-1] += (t.end_lineno, t.end_col_offset) + if t._fields is None: + return tuple(result) + for f in t._fields: + result.append(to_tuple(getattr(t, f))) + return tuple(result) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 39605dca3886c8..4f2278bb263681 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -393,15 +393,159 @@ async def gen(): r'anext\(\): asynchronous generator is already running'): an.__next__() + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + an.send(None) + + def test_async_gen_asend_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_asend_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.asend(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.athrow(None) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + def test_async_gen_3_arg_deprecation_warning(self): async def gen(): yield 123 with self.assertWarns(DeprecationWarning): x = gen().athrow(GeneratorExit, GeneratorExit(), None) - with self.assertWarnsRegex(RuntimeWarning, - f"coroutine method 'athrow' of '{gen.__qualname__}' " - f"was never awaited"): + with self.assertRaises(GeneratorExit): + x.send(None) del x gc_collect() @@ -427,6 +571,54 @@ async def gen(): self.assertTrue(inspect.isawaitable(aclose)) aclose.close() + def test_async_gen_asend_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + + def test_async_gen_athrow_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + gen = agen.athrow(MyExc) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + class AsyncGenAsyncioTest(unittest.TestCase): @@ -1572,11 +1764,8 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) - with self.assertWarnsRegex(RuntimeWarning, - f"coroutine method 'aclose' of '{async_iterate.__qualname__}' " - f"was never awaited"): - del message, messages - gc_collect() + del message, messages + gc_collect() def test_async_gen_expression_01(self): async def arange(n): @@ -1630,10 +1819,7 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) - with self.assertWarnsRegex(RuntimeWarning, - f"coroutine method 'aclose' of '{async_iterate.__qualname__}' " - f"was never awaited"): - gc_collect() + gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): @@ -1671,6 +1857,62 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_throw_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + nxt = it.aclose() + with self.assertRaises(StopIteration): + nxt.throw(GeneratorExit) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(GeneratorExit) + + def test_async_gen_throw_custom_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.aclose() + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + + def test_async_gen_throw_custom_same_athrow_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.athrow(MyException) + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + def test_async_gen_aclose_twice_with_different_coros(self): # Regression test for https://bugs.python.org/issue39606 async def async_iterate(): @@ -1752,7 +1994,67 @@ async def gen(): g.aclose() gc_collect() + def test_aclose_throw(self): + async def gen(): + return + yield + + class MyException(Exception): + pass + + g = gen() + with self.assertRaises(MyException): + g.aclose().throw(MyException) + + del g + gc_collect() # does not warn unawaited + + def test_asend_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + + + def test_athrow_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(Exception) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.send(None) + del gen2 + gc_collect() # does not warn unawaited if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 82071edb252570..c14a0bb180d79b 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -3,6 +3,7 @@ import concurrent.futures import errno import math +import platform import socket import sys import threading @@ -231,6 +232,22 @@ def test_set_default_executor_error(self): self.assertIsNone(self.loop._default_executor) + def test_shutdown_default_executor_timeout(self): + class DummyExecutor(concurrent.futures.ThreadPoolExecutor): + def shutdown(self, wait=True, *, cancel_futures=False): + if wait: + time.sleep(0.1) + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + executor = DummyExecutor() + self.loop.set_default_executor(executor) + + with self.assertWarnsRegex(RuntimeWarning, + "The executor did not finishing joining"): + self.loop.run_until_complete( + self.loop.shutdown_default_executor(timeout=0.01)) + def test_call_soon(self): def cb(): pass @@ -1414,6 +1431,10 @@ def test_create_connection_no_inet_pton(self, m_socket): self._test_create_connection_ip_addr(m_socket, False) @patch_socket + @unittest.skipIf( + support.is_android and platform.android_ver().api_level < 23, + "Issue gh-71123: this fails on Android before API level 23" + ) def test_create_connection_service_name(self, m_socket): m_socket.getaddrinfo = socket.getaddrinfo sock = m_socket.socket.return_value diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 0f8212dbec47be..31d2a00dbb8c9c 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -213,6 +213,52 @@ async def run(): self.run_coro(run()) + def test_staggered_race_with_eager_tasks(self): + # See https://github.com/python/cpython/issues/124309 + + async def fail(): + await asyncio.sleep(0) + raise ValueError("no good") + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: asyncio.sleep(2, result="sleep2"), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: fail() + ], + delay=0.25 + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIsInstance(excs[2], ValueError) + + self.run_coro(run()) + + def test_staggered_race_with_eager_tasks_no_delay(self): + # See https://github.com/python/cpython/issues/124309 + async def fail(): + raise ValueError("no good") + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: fail(), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: asyncio.sleep(0, result="sleep0"), + ], + delay=None + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], ValueError) + self.assertEqual(len(excs), 2) + + self.run_coro(run()) + class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): Task = tasks._PyTask @@ -241,6 +287,18 @@ class DummyLoop: _, out, err = assert_python_ok("-c", code) self.assertFalse(err) + def test_issue122332(self): + async def coro(): + pass + + async def run(): + task = self.loop.create_task(coro()) + await task + self.assertIsNone(task.get_coro()) + + self.run_coro(run()) + + class AsyncTaskCounter: def __init__(self, loop, *, task_class, eager): self.suspense_count = 0 diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index c92c88bd5b2429..4dcf9f0e4037b6 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1125,12 +1125,16 @@ def test_create_server_ssl_match_failed(self): # incorrect server_hostname f_c = self.loop.create_connection(MyProto, host, port, ssl=sslcontext_client) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with mock.patch.object(self.loop, 'call_exception_handler'): with test_utils.disable_logger(): - with self.assertRaisesRegex( - ssl.CertificateError, - "IP address mismatch, certificate is not valid for " - "'127.0.0.1'"): + with self.assertRaisesRegex(ssl.CertificateError, regex): self.loop.run_until_complete(f_c) # close connection @@ -1374,6 +1378,80 @@ def test_create_datagram_endpoint_sock(self): tr.close() self.loop.run_until_complete(pr.done) + def test_datagram_send_to_non_listening_address(self): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages after + # sending a message to an address that wasn't listening. + loop = self.loop + + class Protocol(asyncio.DatagramProtocol): + + _received_datagram = None + + def datagram_received(self, data, addr): + self._received_datagram.set_result(data) + + async def wait_for_datagram_received(self): + self._received_datagram = loop.create_future() + result = await asyncio.wait_for(self._received_datagram, 10) + self._received_datagram = None + return result + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + transport_1, protocol_1 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_1) + ) + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + transport_2, protocol_2 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_2) + ) + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address that + # is not listening + socket_3 = create_socket() + transport_3, protocol_3 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_3) + ) + addr_3 = socket_3.getsockname() + transport_3.abort() + + transport_1.sendto(b'a', addr=addr_2) + self.assertEqual(loop.run_until_complete( + protocol_2.wait_for_datagram_received() + ), b'a') + + transport_2.sendto(b'b', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'b') + + # this should send to an address that isn't listening + transport_1.sendto(b'c', addr=addr_3) + loop.run_until_complete(asyncio.sleep(0)) + + # transport 1 should still be able to receive messages after sending to + # an address that wasn't listening + transport_2.sendto(b'd', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'd') + + transport_1.close() + transport_2.close() + def test_internal_fds(self): loop = self.create_event_loop() if not isinstance(loop, selector_events.BaseSelectorEventLoop): @@ -2131,24 +2209,8 @@ def test_remove_fds_after_closing(self): else: import selectors - class UnixEventLoopTestsMixin(EventLoopTestsMixin): - def setUp(self): - super().setUp() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) - - def tearDown(self): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - super().tearDown() - - if hasattr(selectors, 'KqueueSelector'): - class KqueueEventLoopTests(UnixEventLoopTestsMixin, + class KqueueEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2173,7 +2235,7 @@ def test_write_pty(self): super().test_write_pty() if hasattr(selectors, 'EpollSelector'): - class EPollEventLoopTests(UnixEventLoopTestsMixin, + class EPollEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2181,7 +2243,7 @@ def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.EpollSelector()) if hasattr(selectors, 'PollSelector'): - class PollEventLoopTests(UnixEventLoopTestsMixin, + class PollEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2189,7 +2251,7 @@ def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.PollSelector()) # Should always exist. - class SelectEventLoopTests(UnixEventLoopTestsMixin, + class SelectEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2250,7 +2312,7 @@ def test_handle_repr(self): h = asyncio.Handle(noop, (1, 2), self.loop) filename, lineno = test_utils.get_function_source(noop) self.assertEqual(repr(h), - '' + '' % (filename, lineno)) # cancelled handle @@ -2268,14 +2330,14 @@ def test_handle_repr(self): # partial function cb = functools.partial(noop, 1, 2) h = asyncio.Handle(cb, (3,), self.loop) - regex = (r'^$' + regex = (r'^$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) # partial function with keyword args cb = functools.partial(noop, x=1) h = asyncio.Handle(cb, (2, 3), self.loop) - regex = (r'^$' + regex = (r'^$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) @@ -2286,7 +2348,7 @@ def test_handle_repr(self): h = asyncio.Handle(cb, (), self.loop) cb_regex = r'' - cb_regex = fr'functools.partialmethod\({cb_regex}, , \)\(\)' + cb_regex = fr'functools.partialmethod\({cb_regex}\)\(\)' regex = fr'^$' self.assertRegex(repr(h), regex) @@ -2316,6 +2378,24 @@ def test_handle_repr_debug(self): '' % (filename, lineno, create_filename, create_lineno)) + # partial function + cb = functools.partial(noop, 1, 2) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + def test_handle_source_traceback(self): loop = asyncio.get_event_loop_policy().new_event_loop() loop.set_debug(True) @@ -2620,9 +2700,6 @@ def test_event_loop_policy(self): self.assertRaises(NotImplementedError, policy.get_event_loop) self.assertRaises(NotImplementedError, policy.set_event_loop, object()) self.assertRaises(NotImplementedError, policy.new_event_loop) - self.assertRaises(NotImplementedError, policy.get_child_watcher) - self.assertRaises(NotImplementedError, policy.set_child_watcher, - object()) def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() @@ -2737,20 +2814,8 @@ def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) - if sys.platform != 'win32': - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) - def tearDown(self): try: - if sys.platform != 'win32': - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - super().tearDown() finally: self.loop.close() diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index d3e8efec1c04c2..3a4291e3a68ca6 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -5,6 +5,7 @@ import re import sys import threading +import traceback import unittest from unittest import mock from types import GenericAlias @@ -31,6 +32,25 @@ def last_cb(): pass +class ReachableCode(Exception): + """Exception to raise to indicate that some code was reached. + + Use this exception if using mocks is not a good alternative. + """ + + +class SimpleEvilEventLoop(asyncio.base_events.BaseEventLoop): + """Base class for UAF and other evil stuff requiring an evil event loop.""" + + def get_debug(self): # to suppress tracebacks + return False + + def __del__(self): + # Automatically close the evil event loop to avoid warnings. + if not self.is_closed() and not self.is_running(): + self.close() + + class DuckFuture: # Class that does not inherit from Future but aims to be duck-type # compatible with it. @@ -416,6 +436,24 @@ def test_copy_state(self): _copy_future_state(f_cancelled, newf_cancelled) self.assertTrue(newf_cancelled.cancelled()) + try: + raise concurrent.futures.InvalidStateError + except BaseException as e: + f_exc = e + + f_conexc = self._new_future(loop=self.loop) + f_conexc.set_exception(f_exc) + + newf_conexc = self._new_future(loop=self.loop) + _copy_future_state(f_conexc, newf_conexc) + self.assertTrue(newf_conexc.done()) + try: + newf_conexc.result() + except BaseException as e: + newf_exc = e # assertRaises context manager drops the traceback + newf_tb = ''.join(traceback.format_tb(newf_exc.__traceback__)) + self.assertEqual(newf_tb.count('raise concurrent.futures.InvalidStateError'), 1) + def test_iter(self): fut = self._new_future(loop=self.loop) @@ -640,6 +678,28 @@ def __del__(self): fut = self._new_future(loop=self.loop) fut.set_result(Evil()) + def test_future_cancelled_result_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.result() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + + def test_future_cancelled_exception_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.exception() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') @@ -656,6 +716,24 @@ def test_future_del_segfault(self): with self.assertRaises(AttributeError): del fut._log_traceback + def test_callbacks_copy(self): + # See https://github.com/python/cpython/issues/125789 + # In C implementation, the `_callbacks` attribute + # always returns a new list to avoid mutations of internal state + + fut = self._new_future(loop=self.loop) + f1 = lambda _: 1 + f2 = lambda _: 2 + fut.add_done_callback(f1) + fut.add_done_callback(f2) + callbacks = fut._callbacks + self.assertIsNot(callbacks, fut._callbacks) + fut.remove_done_callback(f1) + callbacks = fut._callbacks + self.assertIsNot(callbacks, fut._callbacks) + fut.remove_done_callback(f2) + self.assertIsNone(fut._callbacks) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') @@ -888,6 +966,107 @@ def __eq__(self, other): fut.remove_done_callback(evil()) + def test_evil_call_soon_list_mutation(self): + # see: https://github.com/python/cpython/issues/125969 + called_on_fut_callback0 = False + + pad = lambda: ... + + def evil_call_soon(*args, **kwargs): + nonlocal called_on_fut_callback0 + if called_on_fut_callback0: + # Called when handling fut->fut_callbacks[0] + # and mutates the length fut->fut_callbacks. + fut.remove_done_callback(int) + fut.remove_done_callback(pad) + else: + called_on_fut_callback0 = True + + fake_event_loop = SimpleEvilEventLoop() + fake_event_loop.call_soon = evil_call_soon + + with mock.patch.object(self, 'loop', fake_event_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), fake_event_loop) + + fut.add_done_callback(str) # sets fut->fut_callback0 + fut.add_done_callback(int) # sets fut->fut_callbacks[0] + fut.add_done_callback(pad) # sets fut->fut_callbacks[1] + fut.add_done_callback(pad) # sets fut->fut_callbacks[2] + fut.set_result("boom") + + # When there are no more callbacks, the Python implementation + # returns an empty list but the C implementation returns None. + self.assertIn(fut._callbacks, (None, [])) + + def test_use_after_free_on_fut_callback_0_with_evil__eq__(self): + # Special thanks to Nico-Posada for the original PoC. + # See https://github.com/python/cpython/issues/125966. + + fut = self._new_future() + + class cb_pad: + def __eq__(self, other): + return True + + class evil(cb_pad): + def __eq__(self, other): + fut.remove_done_callback(None) + return NotImplemented + + fut.add_done_callback(cb_pad()) + fut.remove_done_callback(evil()) + + def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): + # see: https://github.com/python/cpython/issues/125984 + + class EvilEventLoop(SimpleEvilEventLoop): + def call_soon(self, *args, **kwargs): + super().call_soon(*args, **kwargs) + raise ReachableCode + + def __getattribute__(self, name): + nonlocal fut_callback_0 + if name == 'call_soon': + fut.remove_done_callback(fut_callback_0) + del fut_callback_0 + return object.__getattribute__(self, name) + + evil_loop = EvilEventLoop() + with mock.patch.object(self, 'loop', evil_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), evil_loop) + + fut_callback_0 = lambda: ... + fut.add_done_callback(fut_callback_0) + self.assertRaises(ReachableCode, fut.set_result, "boom") + + def test_use_after_free_on_fut_context_0_with_evil__getattribute__(self): + # see: https://github.com/python/cpython/issues/125984 + + class EvilEventLoop(SimpleEvilEventLoop): + def call_soon(self, *args, **kwargs): + super().call_soon(*args, **kwargs) + raise ReachableCode + + def __getattribute__(self, name): + if name == 'call_soon': + # resets the future's event loop + fut.__init__(loop=SimpleEvilEventLoop()) + return object.__getattribute__(self, name) + + evil_loop = EvilEventLoop() + with mock.patch.object(self, 'loop', evil_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), evil_loop) + + fut_callback_0 = mock.Mock() + fut_context_0 = mock.Mock() + fut.add_done_callback(fut_callback_0, context=fut_context_0) + del fut_context_0 + del fut_callback_0 + self.assertRaises(ReachableCode, fut.set_result, "boom") + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index a0884bffe6b0de..c3bff760f7307e 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -39,7 +39,7 @@ async def test_lock(self): with self.assertRaisesRegex( TypeError, - "object Lock can't be used in 'await' expression" + "'Lock' object can't be awaited" ): await lock @@ -77,7 +77,7 @@ async def test_lock_by_with_statement(self): self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, - r"object \w+ can't be used in 'await' expression" + r"'\w+' object can't be awaited" ): with await lock: pass @@ -941,7 +941,7 @@ async def test_semaphore(self): with self.assertRaisesRegex( TypeError, - "object Semaphore can't be used in 'await' expression", + "'Semaphore' object can't be awaited", ): await sem @@ -1194,14 +1194,14 @@ async def c3(result): self.assertEqual([2, 3], result) async def test_acquire_fifo_order_4(self): - # Test that a successfule `acquire()` will wake up multiple Tasks + # Test that a successful `acquire()` will wake up multiple Tasks # that were waiting in the Semaphore queue due to FIFO rules. sem = asyncio.Semaphore(0) result = [] count = 0 async def c1(result): - # First task immediatlly waits for semaphore. It will be awoken by c2. + # First task immediately waits for semaphore. It will be awoken by c2. self.assertEqual(sem._value, 0) await sem.acquire() # We should have woken up all waiting tasks now. @@ -1270,7 +1270,7 @@ async def test_barrier(self): self.assertIn("filling", repr(barrier)) with self.assertRaisesRegex( TypeError, - "object Barrier can't be used in 'await' expression", + "'Barrier' object can't be awaited", ): await barrier @@ -1475,13 +1475,13 @@ async def coro(): # first time waiting await barrier.wait() - # after wainting once for all tasks + # after waiting once for all tasks if rewait_n > 0: rewait_n -= 1 # wait again only for rewait tasks await barrier.wait() else: - # wait for end of draining state` + # wait for end of draining state await barrier_nowaiting.wait() # wait for other waiting tasks await barrier.wait() @@ -1780,7 +1780,7 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) async def test_abort_barrier_when_exception_then_resetting(self): - # test from threading.Barrier: see `lock_tests.test_abort_and_reset`` + # test from threading.Barrier: see `lock_tests.test_abort_and_reset` barrier1 = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) results1 = [] diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index dc25a46985e349..84c5f99129585b 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -77,7 +77,7 @@ async def test(lock): self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, - "can't be used in 'await' expression" + "can't be awaited" ): with await lock: pass @@ -124,10 +124,10 @@ def foo(): yield self.assertFalse(asyncio.iscoroutine(foo())) - def test_iscoroutinefunction(self): async def foo(): pass - self.assertTrue(asyncio.iscoroutinefunction(foo)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(asyncio.iscoroutinefunction(foo)) def test_async_def_coroutines(self): async def bar(): diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index c42856e578b8cc..4b3d551dd7b3a2 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -585,11 +585,10 @@ def test_sendto_memoryview(self): def test_sendto_no_data(self): transport = self.datagram_transport() - transport._buffer.append((b'data', ('0.0.0.0', 12345))) - transport.sendto(b'', ()) - self.assertFalse(self.sock.sendto.called) - self.assertEqual( - [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'', addr=('0.0.0.0', 1234)) def test_sendto_buffer(self): transport = self.datagram_transport() @@ -628,6 +627,19 @@ def test_sendto_buffer_memoryview(self): list(transport._buffer)) self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + @mock.patch('asyncio.proactor_events.logger') def test_sendto_exception(self, m_log): data = b'data' @@ -1006,9 +1018,9 @@ def setUp(self): self.addCleanup(self.file.close) super().setUp() - def make_socket(self, cleanup=True): + def make_socket(self, cleanup=True, blocking=False): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setblocking(False) + sock.setblocking(blocking) sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024) if cleanup: @@ -1070,6 +1082,11 @@ def test_sock_sendfile_not_regular_file(self): 0, None)) self.assertEqual(self.file.tell(), 0) + def test_blocking_socket(self): + self.loop.set_debug(True) + sock = self.make_socket(blocking=True) + with self.assertRaisesRegex(ValueError, "must be non-blocking"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 2d058ccf6a8c72..5019e9a293525d 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -522,5 +522,204 @@ class PriorityQueueJoinTests(_QueueJoinTestMixin, unittest.IsolatedAsyncioTestCa q_class = asyncio.PriorityQueue +class _QueueShutdownTestMixin: + q_class = None + + def assertRaisesShutdown(self, msg="Didn't appear to shut-down queue"): + return self.assertRaises(asyncio.QueueShutDown, msg=msg) + + async def test_format(self): + q = self.q_class() + q.shutdown() + self.assertEqual(q._format(), 'maxsize=0 shutdown') + + async def test_shutdown_empty(self): + # Test shutting down an empty queue + + # Setup empty queue, and join() and get() tasks + q = self.q_class() + loop = asyncio.get_running_loop() + get_task = loop.create_task(q.get()) + await asyncio.sleep(0) # want get task pending before shutdown + + # Perform shut-down + q.shutdown(immediate=False) # unfinished tasks: 0 -> 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task successfully finishes + await q.join() + + # Ensure get() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(get_task.done()) + with self.assertRaisesShutdown(): + await get_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + async def test_shutdown_nonempty(self): + # Test shutting down a non-empty queue + + # Setup full queue with 1 item, and join() and put() tasks + q = self.q_class(maxsize=1) + loop = asyncio.get_running_loop() + + q.put_nowait("data") + join_task = loop.create_task(q.join()) + put_task = loop.create_task(q.put("data2")) + + # Ensure put() task is not finished + await asyncio.sleep(0) + self.assertFalse(put_task.done()) + + # Perform shut-down + q.shutdown(immediate=False) # unfinished tasks: 1 -> 1 + + self.assertEqual(q.qsize(), 1) + + # Ensure put() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(put_task.done()) + with self.assertRaisesShutdown(): + await put_task + + # Ensure get() succeeds on enqueued item + self.assertEqual(await q.get(), "data") + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task, and join() task succeeds + q.task_done() + + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate(self): + # Test immediately shutting down a queue + + # Setup queue with 1 item, and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + join_task = loop.create_task(q.join()) + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 1 -> 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there are no unfinished tasks + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate_with_unfinished(self): + # Test immediately shutting down a queue with unfinished tasks + + # Setup queue with 2 items (1 retrieved), and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + q.put_nowait("data") + join_task = loop.create_task(q.join()) + self.assertEqual(await q.get(), "data") + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 2 -> 1 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task + q.task_done() + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + +class QueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.Queue + + +class LifoQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.LifoQueue + + +class PriorityQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.PriorityQueue + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 266f057f0776c3..45f70d09a2083a 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -93,8 +93,8 @@ async def main(): def test_asyncio_run_only_coro(self): for o in {1, lambda: None}: with self.subTest(obj=o), \ - self.assertRaisesRegex(ValueError, - 'a coroutine was expected'): + self.assertRaisesRegex(TypeError, + 'an awaitable is required'): asyncio.run(o) def test_asyncio_run_debug(self): @@ -319,19 +319,28 @@ async def f(): def test_run_non_coro(self): with asyncio.Runner() as runner: with self.assertRaisesRegex( - ValueError, - "a coroutine was expected" + TypeError, + "an awaitable is required" ): runner.run(123) def test_run_future(self): with asyncio.Runner() as runner: - with self.assertRaisesRegex( - ValueError, - "a coroutine was expected" - ): - fut = runner.get_loop().create_future() - runner.run(fut) + fut = runner.get_loop().create_future() + fut.set_result('done') + self.assertEqual('done', runner.run(fut)) + + def test_run_awaitable(self): + class MyAwaitable: + def __await__(self): + return self.run().__await__() + + @staticmethod + async def run(): + return 'done' + + with asyncio.Runner() as runner: + self.assertEqual('done', runner.run(MyAwaitable())) def test_explicit_close(self): runner = asyncio.Runner() diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index c22b780b5edcb8..aaeda33dd0c677 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1280,11 +1280,10 @@ def test_sendto_memoryview(self): def test_sendto_no_data(self): transport = self.datagram_transport() - transport._buffer.append((b'data', ('0.0.0.0', 12345))) - transport.sendto(b'', ()) - self.assertFalse(self.sock.sendto.called) + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) self.assertEqual( - [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + self.sock.sendto.call_args[0], (b'', ('0.0.0.0', 1234))) def test_sendto_buffer(self): transport = self.datagram_transport() @@ -1320,6 +1319,18 @@ def test_sendto_buffer_memoryview(self): list(transport._buffer)) self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_tryagain(self): data = b'data' diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index d33ff197bbfa1d..2509d4382cdebd 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -93,13 +93,10 @@ async def wait_closed(self): class SendfileBase: - # 256 KiB plus small unaligned to buffer chunk - # Newer versions of Windows seems to have increased its internal - # buffer and tries to send as much of the data as it can as it - # has some form of buffering for this which is less than 256KiB - # on newer server versions and Windows 11. - # So DATA should be larger than 256 KiB to make this test reliable. - DATA = b"x" * (1024 * 256 + 1) + # Linux >= 6.10 seems buffering up to 17 pages of data. + # So DATA should be large enough to make this test reliable even with a + # 64 KiB page configuration. + DATA = b"x" * (1024 * 17 * 64 + 1) # Reduce socket buffer size to test on relative small data sets. BUF_SIZE = 4 * 1024 # 4 KiB diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 918faac909b9bf..60a40cc8349fed 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -125,8 +125,12 @@ async def main(srv): class TestServer2(unittest.IsolatedAsyncioTestCase): async def test_wait_closed_basic(self): - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -137,7 +141,8 @@ async def serve(*args): self.assertFalse(task1.done()) # active count != 0, not closed: should block - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) task2 = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task1.done()) @@ -152,7 +157,8 @@ async def serve(*args): self.assertFalse(task2.done()) self.assertFalse(task3.done()) - srv._detach() + wr.close() + await wr.wait_closed() # active count == 0, closed: should unblock await task1 await task2 @@ -161,8 +167,12 @@ async def serve(*args): async def test_wait_closed_race(self): # Test a regression in 3.12.0, should be fixed in 3.12.1 - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -170,13 +180,91 @@ async def serve(*args): task = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task.done()) - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) loop = asyncio.get_running_loop() loop.call_soon(srv.close) - loop.call_soon(srv._detach) + loop.call_soon(wr.close) await srv.wait_closed() + async def test_close_clients(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + self.addCleanup(wr.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.close_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + async def test_abort_clients(self): + async def serve(rd, wr): + fut.set_result((rd, wr)) + await wr.wait_closed() + + fut = asyncio.Future() + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) + self.addCleanup(c_wr.close) + + (s_rd, s_wr) = await fut + + # Limit the socket buffers so we can more reliably overfill them + s_sock = s_wr.get_extra_info('socket') + s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + c_sock = c_wr.get_extra_info('socket') + c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + + # Get the reader in to a paused state by sending more than twice + # the configured limit + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + while c_wr.transport.is_reading(): + await asyncio.sleep(0) + + # Get the writer in a waiting state by sending data until the + # kernel stops accepting more data in the send buffer. + # gh-122136: getsockopt() does not reliably report the buffer size + # available for message content. + # We loop until we start filling up the asyncio buffer. + # To avoid an infinite loop we cap at 10 times the expected value + c_bufsize = c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) + s_bufsize = s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) + for i in range(10): + s_wr.write(b'a' * c_bufsize) + s_wr.write(b'a' * s_bufsize) + if s_wr.transport.get_write_buffer_size() > 0: + break + self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + srv.close() + srv.abort_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) # Test the various corner cases of Unix server socket removal diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index 075113cbe8e4a6..acef24a703ba38 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -555,12 +555,93 @@ class SelectEventLoopTests(BaseSockTestsMixin, def create_event_loop(self): return asyncio.SelectorEventLoop() + class ProactorEventLoopTests(BaseSockTestsMixin, test_utils.TestCase): def create_event_loop(self): return asyncio.ProactorEventLoop() + + async def _basetest_datagram_send_to_non_listening_address(self, + recvfrom): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages + # after sending a message to an address that wasn't listening. + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address + # that is not listening + socket_3 = create_socket() + addr_3 = socket_3.getsockname() + socket_3.shutdown(socket.SHUT_RDWR) + socket_3.close() + + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + socket_2_recv_task = self.loop.create_task(recvfrom(socket_2)) + await asyncio.sleep(0) + + await self.loop.sock_sendto(socket_1, b'a', addr_2) + self.assertEqual(await socket_2_recv_task, b'a') + + await self.loop.sock_sendto(socket_2, b'b', addr_1) + self.assertEqual(await socket_1_recv_task, b'b') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # this should send to an address that isn't listening + await self.loop.sock_sendto(socket_1, b'c', addr_3) + self.assertEqual(await socket_1_recv_task, b'') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # socket 1 should still be able to receive messages after sending + # to an address that wasn't listening + socket_2.sendto(b'd', addr_1) + self.assertEqual(await socket_1_recv_task, b'd') + + socket_1.shutdown(socket.SHUT_RDWR) + socket_1.close() + socket_2.shutdown(socket.SHUT_RDWR) + socket_2.close() + + + def test_datagram_send_to_non_listening_address_recvfrom(self): + async def recvfrom(socket): + data, _ = await self.loop.sock_recvfrom(socket, 4096) + return data + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom)) + + + def test_datagram_send_to_non_listening_address_recvfrom_into(self): + async def recvfrom_into(socket): + buf = bytearray(4096) + length, _ = await self.loop.sock_recvfrom_into(socket, buf, + 4096) + return buf[:length] + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom_into)) + else: import selectors diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index f5f0afeab51c9e..761904c5146b6a 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -109,6 +109,54 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_connection_lost_when_busy(self): + # gh-118950: SSLProtocol.connection_lost not being called when OSError + # is thrown on asyncio.write. + sock = mock.Mock() + sock.fileno = mock.Mock(return_value=12345) + sock.send = mock.Mock(side_effect=BrokenPipeError) + + # construct StreamWriter chain that contains loop dependant logic this emulates + # what _make_ssl_transport() does in BaseSelectorEventLoop + reader = asyncio.StreamReader(limit=2 ** 16, loop=self.loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) + ssl_proto = self.ssl_protocol(proto=protocol) + + # emulate reading decompressed data + sslobj = mock.Mock() + sslobj.read.side_effect = ssl.SSLWantReadError + sslobj.write.side_effect = ssl.SSLWantReadError + ssl_proto._sslobj = sslobj + + # emulate outgoing data + data = b'An interesting message' + + outgoing = mock.Mock() + outgoing.read = mock.Mock(return_value=data) + outgoing.pending = len(data) + ssl_proto._outgoing = outgoing + + # use correct socket transport to initialize the SSLProtocol + self.loop._make_socket_transport(sock, ssl_proto) + + transport = ssl_proto._app_transport + writer = asyncio.StreamWriter(transport, protocol, reader, self.loop) + + async def main(): + # writes data to transport + async def write(): + writer.write(data) + await writer.drain() + + # try to write for the first time + await write() + # try to write for the second time, this raises as the connection_lost + # callback should be done with error + with self.assertRaises(ConnectionResetError): + await write() + + self.loop.run_until_complete(main()) + def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = self.loop.create_future() diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py new file mode 100644 index 00000000000000..74941f704c4890 --- /dev/null +++ b/Lib/test/test_asyncio/test_staggered.py @@ -0,0 +1,124 @@ +import asyncio +import unittest +from asyncio.staggered import staggered_race + +from test import support + +support.requires_working_socket(module=True) + + +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + +class StaggeredTests(unittest.IsolatedAsyncioTestCase): + async def test_empty(self): + winner, index, excs = await staggered_race( + [], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(excs, []) + + async def test_one_successful(self): + async def coro(index): + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 0') + self.assertEqual(index, 0) + self.assertEqual(excs, [None]) + + async def test_first_error_second_successful(self): + async def coro(index): + if index == 0: + raise ValueError(index) + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIs(excs[1], None) + + async def test_first_timeout_second_successful(self): + async def coro(index): + if index == 0: + await asyncio.sleep(10) # much bigger than delay + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=0.1, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIs(excs[1], None) + + async def test_none_successful(self): + async def coro(index): + raise ValueError(index) + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIsInstance(excs[1], ValueError) + + + async def test_multiple_winners(self): + event = asyncio.Event() + + async def coro(index): + await event.wait() + return index + + async def do_set(): + event.set() + await asyncio.Event().wait() + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + do_set, + ], + delay=0.1, + ) + self.assertIs(winner, 0) + self.assertIs(index, 0) + self.assertEqual(len(excs), 3) + self.assertIsNone(excs[0], None) + self.assertIsInstance(excs[1], asyncio.CancelledError) + self.assertIsInstance(excs[2], asyncio.CancelledError) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 210990593adfa9..dbe5646c2b7c08 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -383,6 +383,10 @@ def test_readuntil_separator(self): stream = asyncio.StreamReader(loop=self.loop) with self.assertRaisesRegex(ValueError, 'Separator should be'): self.loop.run_until_complete(stream.readuntil(separator=b'')) + with self.assertRaisesRegex(ValueError, 'Separator should be'): + self.loop.run_until_complete(stream.readuntil(separator=(b'',))) + with self.assertRaisesRegex(ValueError, 'Separator should contain'): + self.loop.run_until_complete(stream.readuntil(separator=())) def test_readuntil_multi_chunks(self): stream = asyncio.StreamReader(loop=self.loop) @@ -466,6 +470,55 @@ def test_readuntil_limit_found_sep(self): self.assertEqual(b'some dataAAA', stream._buffer) + def test_readuntil_multi_separator(self): + stream = asyncio.StreamReader(loop=self.loop) + + # Simple case + stream.feed_data(b'line 1\nline 2\r') + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 1\n', data) + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 2\r', data) + self.assertEqual(b'', stream._buffer) + + # First end position matches, even if that's a longer match + stream.feed_data(b'ABCDEFG') + data = self.loop.run_until_complete(stream.readuntil((b'DEF', b'BCDE'))) + self.assertEqual(b'ABCDE', data) + self.assertEqual(b'FG', stream._buffer) + + def test_readuntil_multi_separator_limit(self): + stream = asyncio.StreamReader(loop=self.loop, limit=3) + stream.feed_data(b'some dataA') + + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'is found') as cm: + self.loop.run_until_complete(stream.readuntil((b'A', b'ome dataA'))) + + self.assertEqual(b'some dataA', stream._buffer) + + def test_readuntil_multi_separator_negative_offset(self): + # If the buffer is big enough for the smallest separator (but does + # not contain it) but too small for the largest, `offset` must not + # become negative. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'data') + + readuntil_task = self.loop.create_task(stream.readuntil((b'A', b'long sep'))) + self.loop.call_soon(stream.feed_data, b'Z') + self.loop.call_soon(stream.feed_data, b'Aaaa') + + data = self.loop.run_until_complete(readuntil_task) + self.assertEqual(b'dataZA', data) + self.assertEqual(b'aaa', stream._buffer) + + def test_readuntil_bytearray(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'some data\r\n') + data = self.loop.run_until_complete(stream.readuntil(bytearray(b'\r\n'))) + self.assertEqual(b'some data\r\n', data) + self.assertEqual(b'', stream._buffer) + def test_readexactly_zero_or_less(self): # Read exact number of bytes (zero or less). stream = asyncio.StreamReader(loop=self.loop) @@ -769,52 +822,6 @@ async def client(addr): self.assertEqual(msg1, b"hello world 1!\n") self.assertEqual(msg2, b"hello world 2!\n") - @unittest.skipIf(sys.platform == 'win32', "Don't have pipes") - @requires_subprocess() - def test_read_all_from_pipe_reader(self): - # See asyncio issue 168. This test is derived from the example - # subprocess_attach_read_pipe.py, but we configure the - # StreamReader's limit so that twice it is less than the size - # of the data writer. Also we must explicitly attach a child - # watcher to the event loop. - - code = """\ -import os, sys -fd = int(sys.argv[1]) -os.write(fd, b'data') -os.close(fd) -""" - rfd, wfd = os.pipe() - args = [sys.executable, '-c', code, str(wfd)] - - pipe = open(rfd, 'rb', 0) - reader = asyncio.StreamReader(loop=self.loop, limit=1) - protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) - transport, _ = self.loop.run_until_complete( - self.loop.connect_read_pipe(lambda: protocol, pipe)) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - try: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(watcher) - create = asyncio.create_subprocess_exec( - *args, - pass_fds={wfd}, - ) - proc = self.loop.run_until_complete(create) - self.loop.run_until_complete(proc.wait()) - finally: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - - os.close(wfd) - data = self.loop.run_until_complete(reader.read(-1)) - self.assertEqual(data, b'data') - def test_streamreader_constructor_without_loop(self): with self.assertRaisesRegex(RuntimeError, 'no current event loop'): asyncio.StreamReader() @@ -1130,6 +1137,31 @@ async def inner(httpd): self.assertEqual(messages, []) + def test_unclosed_server_resource_warnings(self): + async def inner(rd, wr): + fut.set_result(True) + with self.assertWarns(ResourceWarning) as cm: + del wr + gc.collect() + self.assertEqual(len(cm.warnings), 1) + self.assertTrue(str(cm.warnings[0].message).startswith("unclosed None: def test_subprocess_protocol_events(self): # gh-108973: Test that all subprocess protocol methods are called. - # The protocol methods are not called in a determistic order. + # The protocol methods are not called in a deterministic order. # The order depends on the event loop and the operating system. events = [] fds = [1, 2] @@ -872,60 +864,47 @@ async def main(): self.loop.run_until_complete(main()) + @unittest.skipIf(sys.platform != 'linux', "Linux only") + def test_subprocess_send_signal_race(self): + # See https://github.com/python/cpython/issues/87744 + async def main(): + for _ in range(10): + proc = await asyncio.create_subprocess_exec('sleep', '0.1') + await asyncio.sleep(0.1) + try: + proc.send_signal(signal.SIGUSR1) + except ProcessLookupError: + pass + self.assertNotEqual(await proc.wait(), 255) + + self.loop.run_until_complete(main()) + if sys.platform != 'win32': # Unix class SubprocessWatcherMixin(SubprocessMixin): - Watcher = None - def setUp(self): super().setUp() policy = asyncio.get_event_loop_policy() self.loop = policy.new_event_loop() self.set_event_loop(self.loop) - watcher = self._get_watcher() - watcher.attach_loop(self.loop) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - policy.set_child_watcher(watcher) + def test_watcher_implementation(self): + loop = self.loop + watcher = loop._watcher + if unix_events.can_use_pidfd(): + self.assertIsInstance(watcher, unix_events._PidfdChildWatcher) + else: + self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher) - def tearDown(self): - super().tearDown() - policy = asyncio.get_event_loop_policy() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = policy.get_child_watcher() - policy.set_child_watcher(None) - watcher.attach_loop(None) - watcher.close() class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - - def _get_watcher(self): - return unix_events.ThreadedChildWatcher() - - class SubprocessSafeWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - def _get_watcher(self): - with self.assertWarns(DeprecationWarning): - return unix_events.SafeChildWatcher() - - class MultiLoopChildWatcherTests(test_utils.TestCase): - - def test_warns(self): - with self.assertWarns(DeprecationWarning): - unix_events.MultiLoopChildWatcher() - - class SubprocessFastWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - def _get_watcher(self): - with self.assertWarns(DeprecationWarning): - return unix_events.FastChildWatcher() + def setUp(self): + # Force the use of the threaded child watcher + unix_events.can_use_pidfd = mock.Mock(return_value=False) + super().setUp() @unittest.skipUnless( unix_events.can_use_pidfd(), @@ -934,70 +913,8 @@ def _get_watcher(self): class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - def _get_watcher(self): - return unix_events.PidfdChildWatcher() - - - class GenericWatcherTests(test_utils.TestCase): - - def test_create_subprocess_fails_with_inactive_watcher(self): - watcher = mock.create_autospec(asyncio.AbstractChildWatcher) - watcher.is_active.return_value = False - - async def execute(): - asyncio.set_child_watcher(watcher) - - with self.assertRaises(RuntimeError): - await subprocess.create_subprocess_exec( - os_helper.FakePath(sys.executable), '-c', 'pass') + pass - watcher.add_child_handler.assert_not_called() - - with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertIsNone(runner.run(execute())) - self.assertListEqual(watcher.mock_calls, [ - mock.call.__enter__(), - mock.call.is_active(), - mock.call.__exit__(RuntimeError, mock.ANY, mock.ANY), - ], watcher.mock_calls) - - - @unittest.skipUnless( - unix_events.can_use_pidfd(), - "operating system does not support pidfds", - ) - def test_create_subprocess_with_pidfd(self): - async def in_thread(): - proc = await asyncio.create_subprocess_exec( - *PROGRAM_CAT, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - stdout, stderr = await proc.communicate(b"some data") - return proc.returncode, stdout - - async def main(): - # asyncio.Runner did not call asyncio.set_event_loop() - with warnings.catch_warnings(): - warnings.simplefilter('error', DeprecationWarning) - # get_event_loop() raises DeprecationWarning if - # set_event_loop() was never called and RuntimeError if - # it was called at least once. - with self.assertRaises((RuntimeError, DeprecationWarning)): - asyncio.get_event_loop_policy().get_event_loop() - return await asyncio.to_thread(asyncio.run, in_thread()) - with self.assertWarns(DeprecationWarning): - asyncio.set_child_watcher(asyncio.PidfdChildWatcher()) - try: - with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: - returncode, stdout = runner.run(main()) - self.assertEqual(returncode, 0) - self.assertEqual(stdout, b'some data') - finally: - with self.assertWarns(DeprecationWarning): - asyncio.set_child_watcher(None) else: # Windows class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 7a18362b54e469..1b4de96a572fb9 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1,16 +1,17 @@ # Adapted with permission from the EdgeDB project; # license: PSFL. - +import sys +import gc import asyncio import contextvars import contextlib from asyncio import taskgroups import unittest +import warnings from test.test_asyncio.utils import await_without_task - # To prevent a warning "test altered the execution environment" def tearDownModule(): asyncio.set_event_loop_policy(None) @@ -28,6 +29,15 @@ def get_error_types(eg): return {type(exc) for exc in eg.exceptions} +def no_other_refs(): + # due to gh-124392 coroutines now refer to their locals + coro = asyncio.current_task().get_coro() + frame = sys._getframe(1) + while coro.cr_frame != frame: + coro = coro.cr_await + return [coro] + + class TestTaskGroup(unittest.IsolatedAsyncioTestCase): async def test_taskgroup_01(self): @@ -738,10 +748,7 @@ async def coro2(g): await asyncio.sleep(1) except asyncio.CancelledError: with self.assertRaises(RuntimeError): - g.create_task(c1 := coro1()) - # We still have to await c1 to avoid a warning - with self.assertRaises(ZeroDivisionError): - await c1 + g.create_task(coro1()) with self.assertRaises(ExceptionGroup) as cm: async with taskgroups.TaskGroup() as g: @@ -797,22 +804,25 @@ async def test_taskgroup_double_enter(self): pass async def test_taskgroup_finished(self): - tg = taskgroups.TaskGroup() - async with tg: - pass - coro = asyncio.sleep(0) - with self.assertRaisesRegex(RuntimeError, "is finished"): - tg.create_task(coro) - # We still have to await coro to avoid a warning - await coro + async def create_task_after_tg_finish(): + tg = taskgroups.TaskGroup() + async with tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + + # Make sure the coroutine was closed when submitted to the inactive tg + # (if not closed, a RuntimeWarning should have been raised) + with warnings.catch_warnings(record=True) as w: + await create_task_after_tg_finish() + self.assertEqual(len(w), 0) async def test_taskgroup_not_entered(self): tg = taskgroups.TaskGroup() coro = asyncio.sleep(0) with self.assertRaisesRegex(RuntimeError, "has not been entered"): tg.create_task(coro) - # We still have to await coro to avoid a warning - await coro async def test_taskgroup_without_parent_task(self): tg = taskgroups.TaskGroup() @@ -821,8 +831,171 @@ async def test_taskgroup_without_parent_task(self): coro = asyncio.sleep(0) with self.assertRaisesRegex(RuntimeError, "has not been entered"): tg.create_task(coro) - # We still have to await coro to avoid a warning - await coro + + def test_coro_closed_when_tg_closed(self): + async def run_coro_after_tg_closes(): + async with taskgroups.TaskGroup() as tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + loop = asyncio.get_event_loop() + loop.run_until_complete(run_coro_after_tg_closes()) + + async def test_cancelling_level_preserved(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(raise_after(0.0, RuntimeError)) + except* RuntimeError: + pass + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_nested_groups_both_cancelled(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as outer_tg: + try: + async with asyncio.TaskGroup() as inner_tg: + inner_tg.create_task(raise_after(0, RuntimeError)) + outer_tg.create_task(raise_after(0, ValueError)) + except* RuntimeError: + pass + else: + self.fail("RuntimeError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 1) + except* ValueError: + pass + else: + self.fail("ValueError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_error_and_cancel(self): + event = asyncio.Event() + + async def raise_error(): + event.set() + await asyncio.sleep(0) + raise RuntimeError() + + async def inner(): + try: + async with taskgroups.TaskGroup() as tg: + tg.create_task(raise_error()) + await asyncio.sleep(1) + self.fail("Sleep in group should have been cancelled") + except* RuntimeError: + self.assertEqual(asyncio.current_task().cancelling(), 1) + self.assertEqual(asyncio.current_task().cancelling(), 1) + await asyncio.sleep(1) + self.fail("Sleep after group should have been cancelled") + + async def outer(): + t = asyncio.create_task(inner()) + await event.wait() + self.assertEqual(t.cancelling(), 0) + t.cancel() + self.assertEqual(t.cancelling(), 1) + with self.assertRaises(asyncio.CancelledError): + await t + self.assertTrue(t.cancelled()) + + await outer() + + async def test_exception_refcycles_direct(self): + """Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except ExceptionGroup as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_errors(self): + """Test that TaskGroup deletes self._errors, and __aexit__ args""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except* _Done as excs: + exc = excs.exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_parent_task(self): + """Test that TaskGroup deletes self._parent_task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + try: + async with asyncio.TaskGroup() as tg2: + tg2.create_task(coro_fn()) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_propagate_cancellation_error(self): + """Test that TaskGroup deletes propagate_cancellation_error""" + tg = asyncio.TaskGroup() + exc = None + + try: + async with asyncio.timeout(-1): + async with tg: + await asyncio.sleep(0) + except TimeoutError as e: + exc = e.__cause__ + + self.assertIsInstance(exc, asyncio.CancelledError) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_base_error(self): + """Test that TaskGroup deletes self._base_error""" + class MyKeyboardInterrupt(KeyboardInterrupt): + pass + + tg = asyncio.TaskGroup() + exc = None + + try: + async with tg: + raise MyKeyboardInterrupt + except MyKeyboardInterrupt as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) if __name__ == "__main__": diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 4dfaff847edb90..9d2d356631b42c 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1,6 +1,7 @@ """Tests for tasks.py.""" import collections +import contextlib import contextvars import gc import io @@ -19,6 +20,7 @@ from test.test_asyncio import utils as test_utils from test import support from test.support.script_helper import assert_python_ok +from test.support.warnings_helper import ignore_warnings def tearDownModule(): @@ -85,6 +87,7 @@ class BaseTaskTests: Task = None Future = None + all_tasks = None def new_task(self, loop, coro, name='TestTask', context=None): return self.__class__.Task(coro, loop=loop, name=name, context=context) @@ -683,6 +686,30 @@ def on_timeout(): finally: loop.close() + def test_uncancel_resets_must_cancel(self): + + async def coro(): + await fut + return 42 + + loop = asyncio.new_event_loop() + fut = asyncio.Future(loop=loop) + task = self.new_task(loop, coro()) + loop.run_until_complete(asyncio.sleep(0)) # Get task waiting for fut + fut.set_result(None) # Make task runnable + try: + task.cancel() # Enter cancelled state + self.assertEqual(task.cancelling(), 1) + self.assertTrue(task._must_cancel) + + task.uncancel() # Undo cancellation + self.assertEqual(task.cancelling(), 0) + self.assertFalse(task._must_cancel) + finally: + res = loop.run_until_complete(task) + self.assertEqual(res, 42) + loop.close() + def test_cancel(self): def gen(): @@ -1409,12 +1436,6 @@ def gen(): yield 0.01 yield 0 - loop = self.new_test_loop(gen) - # disable "slow callback" warning - loop.slow_callback_duration = 1.0 - completed = set() - time_shifted = False - async def sleeper(dt, x): nonlocal time_shifted await asyncio.sleep(dt) @@ -1424,21 +1445,78 @@ async def sleeper(dt, x): loop.advance_time(0.14) return x - a = sleeper(0.01, 'a') - b = sleeper(0.01, 'b') - c = sleeper(0.15, 'c') + async def try_iterator(awaitables): + values = [] + for f in asyncio.as_completed(awaitables): + values.append(await f) + return values - async def foo(): + async def try_async_iterator(awaitables): values = [] - for f in asyncio.as_completed([b, c, a]): + async for f in asyncio.as_completed(awaitables): values.append(await f) return values - res = loop.run_until_complete(self.new_task(loop, foo())) - self.assertAlmostEqual(0.15, loop.time()) - self.assertTrue('a' in res[:2]) - self.assertTrue('b' in res[:2]) - self.assertEqual(res[2], 'c') + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + # disable "slow callback" warning + loop.slow_callback_duration = 1.0 + + completed = set() + time_shifted = False + + a = sleeper(0.01, 'a') + b = sleeper(0.01, 'b') + c = sleeper(0.15, 'c') + + res = loop.run_until_complete(self.new_task(loop, foo([b, c, a]))) + self.assertAlmostEqual(0.15, loop.time()) + self.assertTrue('a' in res[:2]) + self.assertTrue('b' in res[:2]) + self.assertEqual(res[2], 'c') + + def test_as_completed_same_tasks_in_as_out(self): + # Ensures that asynchronously iterating as_completed's iterator + # yields awaitables are the same awaitables that were passed in when + # those awaitables are futures. + async def try_async_iterator(awaitables): + awaitables_out = set() + async for out_aw in asyncio.as_completed(awaitables): + awaitables_out.add(out_aw) + return awaitables_out + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + # Coroutines shouldn't be yielded back as finished coroutines + # can't be re-used. + awaitables_in = frozenset( + (coro(0), coro(1), coro(2), coro(3)) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in - awaitables_out != awaitables_in: + raise self.failureException('Got original coroutines ' + 'out of as_completed iterator.') + + # Tasks should be yielded back. + coro_obj_a = coro('a') + task_b = loop.create_task(coro('b')) + coro_obj_c = coro('c') + task_d = loop.create_task(coro('d')) + awaitables_in = frozenset( + (coro_obj_a, task_b, coro_obj_c, task_d) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in & awaitables_out != {task_b, task_d}: + raise self.failureException('Only tasks should be yielded ' + 'from as_completed iterator ' + 'as-is.') def test_as_completed_with_timeout(self): @@ -1448,12 +1526,7 @@ def gen(): yield 0 yield 0.1 - loop = self.new_test_loop(gen) - - a = loop.create_task(asyncio.sleep(0.1, 'a')) - b = loop.create_task(asyncio.sleep(0.15, 'b')) - - async def foo(): + async def try_iterator(): values = [] for f in asyncio.as_completed([a, b], timeout=0.12): if values: @@ -1465,16 +1538,33 @@ async def foo(): values.append((2, exc)) return values - res = loop.run_until_complete(self.new_task(loop, foo())) - self.assertEqual(len(res), 2, res) - self.assertEqual(res[0], (1, 'a')) - self.assertEqual(res[1][0], 2) - self.assertIsInstance(res[1][1], asyncio.TimeoutError) - self.assertAlmostEqual(0.12, loop.time()) + async def try_async_iterator(): + values = [] + try: + async for f in asyncio.as_completed([a, b], timeout=0.12): + v = await f + values.append((1, v)) + loop.advance_time(0.02) + except asyncio.TimeoutError as exc: + values.append((2, exc)) + return values - # move forward to close generator - loop.advance_time(10) - loop.run_until_complete(asyncio.wait([a, b])) + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + a = loop.create_task(asyncio.sleep(0.1, 'a')) + b = loop.create_task(asyncio.sleep(0.15, 'b')) + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(len(res), 2, res) + self.assertEqual(res[0], (1, 'a')) + self.assertEqual(res[1][0], 2) + self.assertIsInstance(res[1][1], asyncio.TimeoutError) + self.assertAlmostEqual(0.12, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) def test_as_completed_with_unused_timeout(self): @@ -1483,19 +1573,75 @@ def gen(): yield 0 yield 0.01 - loop = self.new_test_loop(gen) - - a = asyncio.sleep(0.01, 'a') - - async def foo(): + async def try_iterator(): for f in asyncio.as_completed([a], timeout=1): v = await f self.assertEqual(v, 'a') - loop.run_until_complete(self.new_task(loop, foo())) + async def try_async_iterator(): + async for f in asyncio.as_completed([a], timeout=1): + v = await f + self.assertEqual(v, 'a') - def test_as_completed_reverse_wait(self): + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + a = asyncio.sleep(0.01, 'a') + loop = self.new_test_loop(gen) + loop.run_until_complete(self.new_task(loop, foo())) + loop.close() + + def test_as_completed_resume_iterator(self): + # Test that as_completed returns an iterator that can be resumed + # the next time iteration is performed (i.e. if __iter__ is called + # again) + async def try_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + for f in iterator: + collected.append(await f) + return collected + + async def try_async_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + async for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + async for f in iterator: + collected.append(await f) + return collected + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + results = loop.run_until_complete( + foo((coro(0), coro(1), coro(2), coro(3))) + ) + self.assertCountEqual(results, (0, 1, 2, 3)) + def test_as_completed_reverse_wait(self): + # Tests the plain iterator style of as_completed iteration to + # ensure that the first future awaited resolves to the first + # completed awaitable from the set we passed in, even if it wasn't + # the first future generated by as_completed. def gen(): yield 0 yield 0.05 @@ -1522,7 +1668,8 @@ async def test(): loop.run_until_complete(test()) def test_as_completed_concurrent(self): - + # Ensure that more than one future or coroutine yielded from + # as_completed can be awaited concurrently. def gen(): when = yield self.assertAlmostEqual(0.05, when) @@ -1530,38 +1677,55 @@ def gen(): self.assertAlmostEqual(0.05, when) yield 0.05 - a = asyncio.sleep(0.05, 'a') - b = asyncio.sleep(0.05, 'b') - fs = {a, b} + async def try_iterator(fs): + return list(asyncio.as_completed(fs)) - async def test(): - futs = list(asyncio.as_completed(fs)) - self.assertEqual(len(futs), 2) - done, pending = await asyncio.wait( - [asyncio.ensure_future(fut) for fut in futs] - ) - self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + async def try_async_iterator(fs): + return [f async for f in asyncio.as_completed(fs)] - loop = self.new_test_loop(gen) - loop.run_until_complete(test()) + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + a = asyncio.sleep(0.05, 'a') + b = asyncio.sleep(0.05, 'b') + fs = {a, b} + + async def test(): + futs = await runner(fs) + self.assertEqual(len(futs), 2) + done, pending = await asyncio.wait( + [asyncio.ensure_future(fut) for fut in futs] + ) + self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + + loop = self.new_test_loop(gen) + loop.run_until_complete(test()) def test_as_completed_duplicate_coroutines(self): async def coro(s): return s - async def runner(): + async def try_iterator(): result = [] c = coro('ham') for f in asyncio.as_completed([c, c, coro('spam')]): result.append(await f) return result - fut = self.new_task(self.loop, runner()) - self.loop.run_until_complete(fut) - result = fut.result() - self.assertEqual(set(result), {'ham', 'spam'}) - self.assertEqual(len(result), 2) + async def try_async_iterator(): + result = [] + c = coro('ham') + async for f in asyncio.as_completed([c, c, coro('spam')]): + result.append(await f) + return result + + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + fut = self.new_task(self.loop, runner()) + self.loop.run_until_complete(fut) + result = fut.result() + self.assertEqual(set(result), {'ham', 'spam'}) + self.assertEqual(len(result), 2) def test_as_completed_coroutine_without_loop(self): async def coro(): @@ -1570,8 +1734,8 @@ async def coro(): a = coro() self.addCleanup(a.close) - futs = asyncio.as_completed([a]) with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + futs = asyncio.as_completed([a]) list(futs) def test_as_completed_coroutine_use_running_loop(self): @@ -1776,6 +1940,7 @@ async def notmutch(): self.assertFalse(task.cancelled()) self.assertIs(task.exception(), base_exc) + @ignore_warnings(category=DeprecationWarning) def test_iscoroutinefunction(self): def fn(): pass @@ -1793,6 +1958,7 @@ async def fn2(): self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) self.assertTrue(asyncio.iscoroutinefunction(mock.AsyncMock())) + @ignore_warnings(category=DeprecationWarning) def test_coroutine_non_gen_function(self): async def func(): return 'test' @@ -2044,14 +2210,32 @@ async def coro(): self.assertEqual(res, 42) def test_as_completed_invalid_args(self): + # as_completed() expects a list of futures, not a future instance + # TypeError should be raised either on iterator construction or first + # iteration + + # Plain iterator fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(fut) + next(iterator) + coro = coroutine_function() + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(coro) + next(iterator) + coro.close() - # as_completed() expects a list of futures, not a future instance - self.assertRaises(TypeError, self.loop.run_until_complete, - asyncio.as_completed(fut)) + # Async iterator + async def try_async_iterator(aw): + async for f in asyncio.as_completed(aw): + break + + fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(fut)) coro = coroutine_function() - self.assertRaises(TypeError, self.loop.run_until_complete, - asyncio.as_completed(coro)) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(coro)) coro.close() def test_wait_invalid_args(self): @@ -2087,7 +2271,7 @@ async def kill_me(loop): coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) - self.assertEqual(asyncio.all_tasks(loop=self.loop), {task}) + self.assertEqual(self.all_tasks(loop=self.loop), {task}) asyncio.set_event_loop(None) @@ -2102,7 +2286,7 @@ async def kill_me(loop): # no more reference to kill_me() task: the task is destroyed by the GC support.gc_collect() - self.assertEqual(asyncio.all_tasks(loop=self.loop), set()) + self.assertEqual(self.all_tasks(loop=self.loop), set()) mock_handler.assert_called_with(self.loop, { 'message': 'Task was destroyed but it is pending!', @@ -2251,7 +2435,7 @@ async def coro(): message = m_log.error.call_args[0][0] self.assertIn('Task was destroyed but it is pending', message) - self.assertEqual(asyncio.all_tasks(self.loop), set()) + self.assertEqual(self.all_tasks(self.loop), set()) def test_create_task_with_noncoroutine(self): with self.assertRaisesRegex(TypeError, @@ -2504,6 +2688,28 @@ def test_get_context(self): finally: loop.close() + def test_proper_refcounts(self): + # see: https://github.com/python/cpython/issues/126083 + class Break: + def __str__(self): + raise RuntimeError("break") + + obj = object() + initial_refcount = sys.getrefcount(obj) + + coro = coroutine_function() + loop = asyncio.new_event_loop() + task = asyncio.Task.__new__(asyncio.Task) + + for _ in range(5): + with self.assertRaisesRegex(RuntimeError, 'break'): + task.__init__(coro, loop=loop, context=obj, name=Break()) + + coro.close() + del task + + self.assertEqual(sys.getrefcount(obj), initial_refcount) + def add_subclass_tests(cls): BaseTask = cls.Task @@ -2551,6 +2757,7 @@ async def func(): # Add patched Task & Future back to the test case cls.Task = Task cls.Future = Future + cls.all_tasks = tasks.all_tasks # Add an extra unit-test cls.test_subclasses_ctask_cfuture = test_subclasses_ctask_cfuture @@ -2624,6 +2831,7 @@ class CTask_CFuture_Tests(BaseTaskTests, SetMethodsTest, Task = getattr(tasks, '_CTask', None) Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) @support.refcount_test def test_refleaks_in_task___init__(self): @@ -2655,6 +2863,7 @@ class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) @unittest.skipUnless(hasattr(tasks, '_CTask'), @@ -2664,6 +2873,7 @@ class CTaskSubclass_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) @unittest.skipUnless(hasattr(futures, '_CFuture'), @@ -2673,6 +2883,7 @@ class PyTask_CFutureSubclass_Tests(BaseTaskTests, test_utils.TestCase): Future = getattr(futures, '_CFuture', None) Task = tasks._PyTask + all_tasks = tasks._py_all_tasks @unittest.skipUnless(hasattr(tasks, '_CTask'), @@ -2681,6 +2892,7 @@ class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) @unittest.skipUnless(hasattr(futures, '_CFuture'), @@ -2689,6 +2901,7 @@ class PyTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = tasks._PyTask Future = getattr(futures, '_CFuture', None) + all_tasks = staticmethod(tasks._py_all_tasks) class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, @@ -2696,6 +2909,7 @@ class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, Task = tasks._PyTask Future = futures._PyFuture + all_tasks = staticmethod(tasks._py_all_tasks) @add_subclass_tests @@ -2735,6 +2949,7 @@ class BaseTaskIntrospectionTests: _unregister_task = None _enter_task = None _leave_task = None + all_tasks = None def test__register_task_1(self): class TaskLike: @@ -2748,9 +2963,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), {task}) + self.assertEqual(self.all_tasks(loop), {task}) self._unregister_task(task) def test__register_task_2(self): @@ -2764,9 +2979,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), {task}) + self.assertEqual(self.all_tasks(loop), {task}) self._unregister_task(task) def test__register_task_3(self): @@ -2780,9 +2995,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._unregister_task(task) def test__enter_task(self): @@ -2833,13 +3048,13 @@ def test__unregister_task(self): task.get_loop = lambda: loop self._register_task(task) self._unregister_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) def test__unregister_task_not_registered(self): task = mock.Mock() loop = mock.Mock() self._unregister_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): @@ -2847,6 +3062,7 @@ class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _unregister_task = staticmethod(tasks._py_unregister_task) _enter_task = staticmethod(tasks._py_enter_task) _leave_task = staticmethod(tasks._py_leave_task) + all_tasks = staticmethod(tasks._py_all_tasks) @unittest.skipUnless(hasattr(tasks, '_c_register_task'), @@ -2857,6 +3073,7 @@ class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _unregister_task = staticmethod(tasks._c_unregister_task) _enter_task = staticmethod(tasks._c_enter_task) _leave_task = staticmethod(tasks._c_leave_task) + all_tasks = staticmethod(tasks._c_all_tasks) else: _register_task = _unregister_task = _enter_task = _leave_task = None @@ -2923,14 +3140,14 @@ def test_asyncio_module_compiled(self): # fail on systems where C modules were successfully compiled # (hence the test for _functools etc), but _asyncio somehow didn't. try: - import _functools - import _json - import _pickle + import _functools # noqa: F401 + import _json # noqa: F401 + import _pickle # noqa: F401 except ImportError: self.skipTest('C modules are not available') else: try: - import _asyncio + import _asyncio # noqa: F401 except ImportError: self.fail('_asyncio module is missing') diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py index 1138a93e0f78ec..774380270a7d70 100644 --- a/Lib/test/test_asyncio/test_threads.py +++ b/Lib/test/test_asyncio/test_threads.py @@ -30,7 +30,9 @@ async def test_to_thread_once(self): func.assert_called_once() async def test_to_thread_concurrent(self): - func = mock.Mock() + calls = [] + def func(): + calls.append(1) futs = [] for _ in range(10): @@ -38,7 +40,7 @@ async def test_to_thread_concurrent(self): futs.append(fut) await asyncio.gather(*futs) - self.assertEqual(func.call_count, 10) + self.assertEqual(sum(calls), 10) async def test_to_thread_args_kwargs(self): # Unlike run_in_executor(), to_thread() should directly accept kwargs. diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index 1f7f9ee696a525..f5543e191d07ff 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -220,7 +220,7 @@ async def test_nested_timeouts_loop_busy(self): # Pretend the loop is busy for a while. time.sleep(0.1) await asyncio.sleep(1) - # TimeoutError was cought by (2) + # TimeoutError was caught by (2) await asyncio.sleep(10) # This sleep should be interrupted by (1) t1 = loop.time() self.assertTrue(t0 <= t1 <= t0 + 1) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 59ef9f5f58cabc..9ae54b6887010b 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -6,7 +6,6 @@ import multiprocessing from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests import os -import pathlib import signal import socket import stat @@ -304,20 +303,20 @@ def test_create_unix_server_existing_path_sock(self): self.loop.run_until_complete(srv.wait_closed()) @socket_helper.skip_unless_bind_unix_socket - def test_create_unix_server_pathlib(self): + def test_create_unix_server_pathlike(self): with test_utils.unix_socket_path() as path: - path = pathlib.Path(path) + path = os_helper.FakePath(path) srv_coro = self.loop.create_unix_server(lambda: None, path) srv = self.loop.run_until_complete(srv_coro) srv.close() self.loop.run_until_complete(srv.wait_closed()) - def test_create_unix_connection_pathlib(self): + def test_create_unix_connection_pathlike(self): with test_utils.unix_socket_path() as path: - path = pathlib.Path(path) + path = os_helper.FakePath(path) coro = self.loop.create_unix_connection(lambda: None, path) with self.assertRaises(FileNotFoundError): - # If pathlib.Path wasn't supported, the exception would be + # If path-like object weren't supported, the exception would be # different. self.loop.run_until_complete(coro) @@ -1113,697 +1112,6 @@ def test_write_eof_pending(self): self.assertFalse(self.protocol.connection_lost.called) -class AbstractChildWatcherTests(unittest.TestCase): - - def test_warns_on_subclassing(self): - with self.assertWarns(DeprecationWarning): - class MyWatcher(asyncio.AbstractChildWatcher): - pass - - def test_not_implemented(self): - f = mock.Mock() - watcher = asyncio.AbstractChildWatcher() - self.assertRaises( - NotImplementedError, watcher.add_child_handler, f, f) - self.assertRaises( - NotImplementedError, watcher.remove_child_handler, f) - self.assertRaises( - NotImplementedError, watcher.attach_loop, f) - self.assertRaises( - NotImplementedError, watcher.close) - self.assertRaises( - NotImplementedError, watcher.is_active) - self.assertRaises( - NotImplementedError, watcher.__enter__) - self.assertRaises( - NotImplementedError, watcher.__exit__, f, f, f) - - -class BaseChildWatcherTests(unittest.TestCase): - - def test_not_implemented(self): - f = mock.Mock() - watcher = unix_events.BaseChildWatcher() - self.assertRaises( - NotImplementedError, watcher._do_waitpid, f) - - -class ChildWatcherTestsMixin: - - ignore_warnings = mock.patch.object(log.logger, "warning") - - def setUp(self): - super().setUp() - self.loop = self.new_test_loop() - self.running = False - self.zombies = {} - - with mock.patch.object( - self.loop, "add_signal_handler") as self.m_add_signal_handler: - self.watcher = self.create_watcher() - self.watcher.attach_loop(self.loop) - - def waitpid(self, pid, flags): - if isinstance(self.watcher, asyncio.SafeChildWatcher) or pid != -1: - self.assertGreater(pid, 0) - try: - if pid < 0: - return self.zombies.popitem() - else: - return pid, self.zombies.pop(pid) - except KeyError: - pass - if self.running: - return 0, 0 - else: - raise ChildProcessError() - - def add_zombie(self, pid, status): - self.zombies[pid] = status - - def waitstatus_to_exitcode(self, status): - if status > 32768: - return status - 32768 - elif 32700 < status < 32768: - return status - 32768 - else: - return status - - def test_create_watcher(self): - self.m_add_signal_handler.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - - def waitpid_mocks(func): - def wrapped_func(self): - def patch(target, wrapper): - return mock.patch(target, wraps=wrapper, - new_callable=mock.Mock) - - with patch('asyncio.unix_events.waitstatus_to_exitcode', self.waitstatus_to_exitcode), \ - patch('os.waitpid', self.waitpid) as m_waitpid: - func(self, m_waitpid) - return wrapped_func - - @waitpid_mocks - def test_sigchld(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(42, callback, 9, 10, 14) - - self.assertFalse(callback.called) - - # child is running - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - # child terminates (returncode 12) - self.running = False - self.add_zombie(42, EXITCODE(12)) - self.watcher._sig_chld() - - callback.assert_called_once_with(42, 12, 9, 10, 14) - - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(42, EXITCODE(13)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - # sigchld called again - self.zombies.clear() - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_two_children(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register child 1 - with self.watcher: - self.running = True - self.watcher.add_child_handler(43, callback1, 7, 8) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register child 2 - with self.watcher: - self.watcher.add_child_handler(44, callback2, 147, 18) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # children are running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 1 terminates (signal 3) - self.add_zombie(43, SIGNAL(3)) - self.watcher._sig_chld() - - callback1.assert_called_once_with(43, -3, 7, 8) - self.assertFalse(callback2.called) - - callback1.reset_mock() - - # child 2 still running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 2 terminates (code 108) - self.add_zombie(44, EXITCODE(108)) - self.running = False - self.watcher._sig_chld() - - callback2.assert_called_once_with(44, 108, 147, 18) - self.assertFalse(callback1.called) - - callback2.reset_mock() - - # ensure that the children are effectively reaped - self.add_zombie(43, EXITCODE(14)) - self.add_zombie(44, EXITCODE(15)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # sigchld called again - self.zombies.clear() - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_two_children_terminating_together(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register child 1 - with self.watcher: - self.running = True - self.watcher.add_child_handler(45, callback1, 17, 8) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register child 2 - with self.watcher: - self.watcher.add_child_handler(46, callback2, 1147, 18) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # children are running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 1 terminates (code 78) - # child 2 terminates (signal 5) - self.add_zombie(45, EXITCODE(78)) - self.add_zombie(46, SIGNAL(5)) - self.running = False - self.watcher._sig_chld() - - callback1.assert_called_once_with(45, 78, 17, 8) - callback2.assert_called_once_with(46, -5, 1147, 18) - - callback1.reset_mock() - callback2.reset_mock() - - # ensure that the children are effectively reaped - self.add_zombie(45, EXITCODE(14)) - self.add_zombie(46, EXITCODE(15)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_race_condition(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - # child terminates before being registered - self.add_zombie(50, EXITCODE(4)) - self.watcher._sig_chld() - - self.watcher.add_child_handler(50, callback, 1, 12) - - callback.assert_called_once_with(50, 4, 1, 12) - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(50, SIGNAL(1)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_replace_handler(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(51, callback1, 19) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register the same child again - with self.watcher: - self.watcher.add_child_handler(51, callback2, 21) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child terminates (signal 8) - self.running = False - self.add_zombie(51, SIGNAL(8)) - self.watcher._sig_chld() - - callback2.assert_called_once_with(51, -8, 21) - self.assertFalse(callback1.called) - - callback2.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(51, EXITCODE(13)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_remove_handler(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(52, callback, 1984) - - self.assertFalse(callback.called) - - # unregister the child - self.watcher.remove_child_handler(52) - - self.assertFalse(callback.called) - - # child terminates (code 99) - self.running = False - self.add_zombie(52, EXITCODE(99)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_unknown_status(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(53, callback, -19) - - self.assertFalse(callback.called) - - # terminate with unknown status - self.zombies[53] = 1178 - self.running = False - self.watcher._sig_chld() - - callback.assert_called_once_with(53, 1178, -19) - - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(53, EXITCODE(101)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_remove_child_handler(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - callback3 = mock.Mock() - - # register children - with self.watcher: - self.running = True - self.watcher.add_child_handler(54, callback1, 1) - self.watcher.add_child_handler(55, callback2, 2) - self.watcher.add_child_handler(56, callback3, 3) - - # remove child handler 1 - self.assertTrue(self.watcher.remove_child_handler(54)) - - # remove child handler 2 multiple times - self.assertTrue(self.watcher.remove_child_handler(55)) - self.assertFalse(self.watcher.remove_child_handler(55)) - self.assertFalse(self.watcher.remove_child_handler(55)) - - # all children terminate - self.add_zombie(54, EXITCODE(0)) - self.add_zombie(55, EXITCODE(1)) - self.add_zombie(56, EXITCODE(2)) - self.running = False - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - callback3.assert_called_once_with(56, 2, 3) - - @waitpid_mocks - def test_sigchld_unhandled_exception(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(57, callback) - - # raise an exception - m_waitpid.side_effect = ValueError - - with mock.patch.object(log.logger, - 'error') as m_error: - - self.assertEqual(self.watcher._sig_chld(), None) - self.assertTrue(m_error.called) - - @waitpid_mocks - def test_sigchld_child_reaped_elsewhere(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(58, callback) - - self.assertFalse(callback.called) - - # child terminates - self.running = False - self.add_zombie(58, EXITCODE(4)) - - # waitpid is called elsewhere - os.waitpid(58, os.WNOHANG) - - m_waitpid.reset_mock() - - # sigchld - with self.ignore_warnings: - self.watcher._sig_chld() - - if isinstance(self.watcher, asyncio.FastChildWatcher): - # here the FastChildWatcher enters a deadlock - # (there is no way to prevent it) - self.assertFalse(callback.called) - else: - callback.assert_called_once_with(58, 255) - - @waitpid_mocks - def test_sigchld_unknown_pid_during_registration(self, m_waitpid): - # register two children - callback1 = mock.Mock() - callback2 = mock.Mock() - - with self.ignore_warnings, self.watcher: - self.running = True - # child 1 terminates - self.add_zombie(591, EXITCODE(7)) - # an unknown child terminates - self.add_zombie(593, EXITCODE(17)) - - self.watcher._sig_chld() - - self.watcher.add_child_handler(591, callback1) - self.watcher.add_child_handler(592, callback2) - - callback1.assert_called_once_with(591, 7) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_set_loop(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(60, callback) - - # attach a new loop - old_loop = self.loop - self.loop = self.new_test_loop() - patch = mock.patch.object - - with patch(old_loop, "remove_signal_handler") as m_old_remove, \ - patch(self.loop, "add_signal_handler") as m_new_add: - - self.watcher.attach_loop(self.loop) - - m_old_remove.assert_called_once_with( - signal.SIGCHLD) - m_new_add.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - - # child terminates - self.running = False - self.add_zombie(60, EXITCODE(9)) - self.watcher._sig_chld() - - callback.assert_called_once_with(60, 9) - - @waitpid_mocks - def test_set_loop_race_condition(self, m_waitpid): - # register 3 children - callback1 = mock.Mock() - callback2 = mock.Mock() - callback3 = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(61, callback1) - self.watcher.add_child_handler(62, callback2) - self.watcher.add_child_handler(622, callback3) - - # detach the loop - old_loop = self.loop - self.loop = None - - with mock.patch.object( - old_loop, "remove_signal_handler") as m_remove_signal_handler: - - with self.assertWarnsRegex( - RuntimeWarning, 'A loop is being detached'): - self.watcher.attach_loop(None) - - m_remove_signal_handler.assert_called_once_with( - signal.SIGCHLD) - - # child 1 & 2 terminate - self.add_zombie(61, EXITCODE(11)) - self.add_zombie(62, SIGNAL(5)) - - # SIGCHLD was not caught - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - self.assertFalse(callback3.called) - - # attach a new loop - self.loop = self.new_test_loop() - - with mock.patch.object( - self.loop, "add_signal_handler") as m_add_signal_handler: - - self.watcher.attach_loop(self.loop) - - m_add_signal_handler.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - callback1.assert_called_once_with(61, 11) # race condition! - callback2.assert_called_once_with(62, -5) # race condition! - self.assertFalse(callback3.called) - - callback1.reset_mock() - callback2.reset_mock() - - # child 3 terminates - self.running = False - self.add_zombie(622, EXITCODE(19)) - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - callback3.assert_called_once_with(622, 19) - - @waitpid_mocks - def test_close(self, m_waitpid): - # register two children - callback1 = mock.Mock() - - with self.watcher: - self.running = True - # child 1 terminates - self.add_zombie(63, EXITCODE(9)) - # other child terminates - self.add_zombie(65, EXITCODE(18)) - self.watcher._sig_chld() - - self.watcher.add_child_handler(63, callback1) - self.watcher.add_child_handler(64, callback1) - - self.assertEqual(len(self.watcher._callbacks), 1) - if isinstance(self.watcher, asyncio.FastChildWatcher): - self.assertEqual(len(self.watcher._zombies), 1) - - with mock.patch.object( - self.loop, - "remove_signal_handler") as m_remove_signal_handler: - - self.watcher.close() - - m_remove_signal_handler.assert_called_once_with( - signal.SIGCHLD) - self.assertFalse(self.watcher._callbacks) - if isinstance(self.watcher, asyncio.FastChildWatcher): - self.assertFalse(self.watcher._zombies) - - -class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): - def create_watcher(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return asyncio.SafeChildWatcher() - - -class FastChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): - def create_watcher(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return asyncio.FastChildWatcher() - - -class PolicyTests(unittest.TestCase): - - def create_policy(self): - return asyncio.DefaultEventLoopPolicy() - - @mock.patch('asyncio.unix_events.can_use_pidfd') - def test_get_default_child_watcher(self, m_can_use_pidfd): - m_can_use_pidfd.return_value = False - policy = self.create_policy() - self.assertIsNone(policy._watcher) - with self.assertWarns(DeprecationWarning): - watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.ThreadedChildWatcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - m_can_use_pidfd.return_value = True - policy = self.create_policy() - self.assertIsNone(policy._watcher) - with self.assertWarns(DeprecationWarning): - watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.PidfdChildWatcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - def test_get_child_watcher_after_set(self): - policy = self.create_policy() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = asyncio.FastChildWatcher() - policy.set_child_watcher(watcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - def test_get_child_watcher_thread(self): - - def f(): - policy.set_event_loop(policy.new_event_loop()) - - self.assertIsInstance(policy.get_event_loop(), - asyncio.AbstractEventLoop) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = policy.get_child_watcher() - - self.assertIsInstance(watcher, asyncio.SafeChildWatcher) - self.assertIsNone(watcher._loop) - - policy.get_event_loop().close() - - policy = self.create_policy() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - policy.set_child_watcher(asyncio.SafeChildWatcher()) - - th = threading.Thread(target=f) - th.start() - th.join() - - def test_child_watcher_replace_mainloop_existing(self): - policy = self.create_policy() - loop = policy.new_event_loop() - policy.set_event_loop(loop) - - # Explicitly setup SafeChildWatcher, - # default ThreadedChildWatcher has no _loop property - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - policy.set_child_watcher(watcher) - watcher.attach_loop(loop) - - self.assertIs(watcher._loop, loop) - - new_loop = policy.new_event_loop() - policy.set_event_loop(new_loop) - - self.assertIs(watcher._loop, new_loop) - - policy.set_event_loop(None) - - self.assertIs(watcher._loop, None) - - loop.close() - new_loop.close() - - class TestFunctional(unittest.TestCase): def setUp(self): @@ -1904,6 +1212,7 @@ async def test_fork_not_share_event_loop(self): wait_process(pid, exitcode=0) @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_signal_handling(self): self.addCleanup(multiprocessing_cleanup_tests) @@ -1950,6 +1259,7 @@ async def func(): self.assertTrue(child_handled.is_set()) @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_run(self): self.addCleanup(multiprocessing_cleanup_tests) @@ -1969,6 +1279,7 @@ async def child_main(): self.assertEqual(result.value, 42) @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_subprocess(self): self.addCleanup(multiprocessing_cleanup_tests) diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index d52f32534a0cfe..11a8eeeab37634 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -249,8 +249,8 @@ async def test_cancel_wait_for(self): await self._test_cancel_wait_for(60.0) async def test_wait_for_cancel_suppressed(self): - # GH-86296: Supressing CancelledError is discouraged - # but if a task subpresses CancelledError and returns a value, + # GH-86296: Suppressing CancelledError is discouraged + # but if a task suppresses CancelledError and returns a value, # `wait_for` should return the value instead of raising CancelledError. # This is the same behavior as `asyncio.timeout`. diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 6e6c90a247b291..0c128c599ba011 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -36,7 +36,23 @@ def data_received(self, data): self.trans.close() -class ProactorLoopCtrlC(test_utils.TestCase): +class WindowsEventsTestCase(test_utils.TestCase): + def _unraisablehook(self, unraisable): + # Storing unraisable.object can resurrect an object which is being + # finalized. Storing unraisable.exc_value creates a reference cycle. + self._unraisable = unraisable + print(unraisable) + + def setUp(self): + self._prev_unraisablehook = sys.unraisablehook + self._unraisable = None + sys.unraisablehook = self._unraisablehook + + def tearDown(self): + sys.unraisablehook = self._prev_unraisablehook + self.assertIsNone(self._unraisable) + +class ProactorLoopCtrlC(WindowsEventsTestCase): def test_ctrl_c(self): @@ -58,7 +74,7 @@ def SIGINT_after_delay(): thread.join() -class ProactorMultithreading(test_utils.TestCase): +class ProactorMultithreading(WindowsEventsTestCase): def test_run_from_nonmain_thread(self): finished = False @@ -79,7 +95,7 @@ def func(): self.assertTrue(finished) -class ProactorTests(test_utils.TestCase): +class ProactorTests(WindowsEventsTestCase): def setUp(self): super().setUp() @@ -283,8 +299,32 @@ async def probe(): return "done" - -class WinPolicyTests(test_utils.TestCase): + def test_loop_restart(self): + # We're fishing for the "RuntimeError: <_overlapped.Overlapped object at XXX> + # still has pending operation at deallocation, the process may crash" error + stop = threading.Event() + def threadMain(): + while not stop.is_set(): + self.loop.call_soon_threadsafe(lambda: None) + time.sleep(0.01) + thr = threading.Thread(target=threadMain) + + # In 10 60-second runs of this test prior to the fix: + # time in seconds until failure: (none), 15.0, 6.4, (none), 7.6, 8.3, 1.7, 22.2, 23.5, 8.3 + # 10 seconds had a 50% failure rate but longer would be more costly + end_time = time.time() + 10 # Run for 10 seconds + self.loop.call_soon(thr.start) + while not self._unraisable: # Stop if we got an unraisable exc + self.loop.stop() + self.loop.run_forever() + if time.time() >= end_time: + break + + stop.set() + thr.join() + + +class WinPolicyTests(WindowsEventsTestCase): def test_selector_win_policy(self): async def main(): diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 44943e1fa7bc4e..b8dbe7feaac3f4 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -15,6 +15,7 @@ import unittest import weakref import warnings +from ast import literal_eval from unittest import mock from http.server import HTTPServer @@ -56,24 +57,8 @@ def data_file(*filename): ONLYKEY = data_file('certdata', 'ssl_key.pem') SIGNED_CERTFILE = data_file('certdata', 'keycert3.pem') SIGNING_CA = data_file('certdata', 'pycacert.pem') -PEERCERT = { - 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), - 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), - 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), - 'issuer': ((('countryName', 'XY'),), - (('organizationName', 'Python Software Foundation CA'),), - (('commonName', 'our-ca-server'),)), - 'notAfter': 'Oct 28 14:23:16 2037 GMT', - 'notBefore': 'Aug 29 14:23:16 2018 GMT', - 'serialNumber': 'CB2D80995A69525C', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} - +with open(data_file('certdata', 'keycert3.pem.reference')) as file: + PEERCERT = literal_eval(file.read()) def simple_server_sslcontext(): server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) @@ -301,12 +286,17 @@ def run_udp_echo_server(*, host='127.0.0.1', port=0): family, type, proto, _, sockaddr = addr_info[0] sock = socket.socket(family, type, proto) sock.bind((host, port)) + sockname = sock.getsockname() thread = threading.Thread(target=lambda: echo_datagrams(sock)) thread.start() try: - yield sock.getsockname() + yield sockname finally: - sock.sendto(b'STOP', sock.getsockname()) + # gh-122187: use a separate socket to send the stop message to avoid + # TSan reported race on the same socket. + sock2 = socket.socket(family, type, proto) + sock2.sendto(b'STOP', sockname) + sock2.close() thread.join() @@ -547,25 +537,6 @@ def close_loop(loop): loop._default_executor.shutdown(wait=True) loop.close() - policy = support.maybe_get_event_loop_policy() - if policy is not None: - try: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = policy.get_child_watcher() - except NotImplementedError: - # watcher is not implemented by EventLoopPolicy, e.g. Windows - pass - else: - if isinstance(watcher, asyncio.ThreadedChildWatcher): - # Wait for subprocess to finish, but not forever - for thread in list(watcher._threads.values()): - thread.join(timeout=support.SHORT_TIMEOUT) - if thread.is_alive(): - raise RuntimeError(f"thread {thread} still alive: " - "subprocess still running") - - def set_event_loop(self, loop, *, cleanup=True): if loop is None: raise AssertionError('loop is None') diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index cd0a4e2264865d..ddd9f951143df7 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -89,6 +89,7 @@ def test_excepthook(self): ) def test_unraisablehook(self): + import_helper.import_module("_testcapi") returncode, events, stderr = self.run_python("test_unraisablehook") if returncode: self.fail(stderr) @@ -139,6 +140,7 @@ def test_gc(self): ) + @support.requires_resource('network') def test_http(self): import_helper.import_module("http.client") returncode, events, stderr = self.run_python("test_http_client") @@ -209,7 +211,7 @@ def test_threading(self): expected = [ ("_thread.start_new_thread", "(, (), None)"), ("test.test_func", "()"), - ("_thread.start_joinable_thread", "(,)"), + ("_thread.start_joinable_thread", "(, 1, None)"), ("test.test_func", "()"), ] @@ -290,6 +292,27 @@ def test_sys_monitoring_register_callback(self): self.assertEqual(actual, expected) + def test_winapi_createnamedpipe(self): + winapi = import_helper.import_module("_winapi") + + pipe_name = r"\\.\pipe\LOCAL\test_winapi_createnamed_pipe" + returncode, events, stderr = self.run_python("test_winapi_createnamedpipe", pipe_name) + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("_winapi.CreateNamedPipe", f"({pipe_name!r}, 3, 8)")] + + self.assertEqual(actual, expected) + + def test_assert_unicode(self): + # See gh-126018 + returncode, _, stderr = self.run_python("test_assert_unicode") + if returncode: + self.fail(stderr) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index f6171d3ed4efd7..409c8c109e885f 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -545,6 +545,40 @@ def test_b85encode(self): self.check_other_types(base64.b85encode, b"www.python.org", b'cXxL#aCvlSZ*DGca%T') + def test_z85encode(self): + eq = self.assertEqual + + tests = { + b'': b'', + b'www.python.org': b'CxXl-AcVLsz/dgCA+t', + bytes(range(255)): b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""", + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""": + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""", + b'no padding..': b'zF{UpvpS[.zF7NO', + b'zero compression\x00\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg00000', + b'zero compression\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg0000', + b"""Boundary:\x00\x00\x00\x00""": b"""lt}0:wmoI7iSGcW00""", + b'Space compr: ': b'q/DePwGUG3ze:IRarR^H', + b'\xff': b'@@', + b'\xff'*2: b'%nJ', + b'\xff'*3: b'%nS9', + b'\xff'*4: b'%nSc0', + } + + for data, res in tests.items(): + eq(base64.z85encode(data), res) + + self.check_other_types(base64.z85encode, b"www.python.org", + b'CxXl-AcVLsz/dgCA+t') + def test_a85decode(self): eq = self.assertEqual @@ -626,6 +660,41 @@ def test_b85decode(self): self.check_other_types(base64.b85decode, b'cXxL#aCvlSZ*DGca%T', b"www.python.org") + def test_z85decode(self): + eq = self.assertEqual + + tests = { + b'': b'', + b'CxXl-AcVLsz/dgCA+t': b'www.python.org', + b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""": bytes(range(255)), + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""": + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""", + b'zF{UpvpS[.zF7NO': b'no padding..', + b'Ds.bnay/tbAb]JhB7]Mg00000': b'zero compression\x00\x00\x00\x00', + b'Ds.bnay/tbAb]JhB7]Mg0000': b'zero compression\x00\x00\x00', + b"""lt}0:wmoI7iSGcW00""": b"""Boundary:\x00\x00\x00\x00""", + b'q/DePwGUG3ze:IRarR^H': b'Space compr: ', + b'@@': b'\xff', + b'%nJ': b'\xff'*2, + b'%nS9': b'\xff'*3, + b'%nSc0': b'\xff'*4, + } + + for data, res in tests.items(): + eq(base64.z85decode(data), res) + eq(base64.z85decode(data.decode("ascii")), res) + + self.check_other_types(base64.z85decode, b'CxXl-AcVLsz/dgCA+t', + b'www.python.org') + def test_a85_padding(self): eq = self.assertEqual @@ -707,6 +776,21 @@ def test_b85decode_errors(self): self.assertRaises(ValueError, base64.b85decode, b'|NsC') self.assertRaises(ValueError, base64.b85decode, b'|NsC1') + def test_z85decode_errors(self): + illegal = list(range(33)) + \ + list(b'"\',;_`|\\~') + \ + list(range(128, 256)) + for c in illegal: + with self.assertRaises(ValueError, msg=bytes([c])): + base64.z85decode(b'0000' + bytes([c])) + + # b'\xff\xff\xff\xff' encodes to b'%nSc0', the following will overflow: + self.assertRaises(ValueError, base64.z85decode, b'%') + self.assertRaises(ValueError, base64.z85decode, b'%n') + self.assertRaises(ValueError, base64.z85decode, b'%nS') + self.assertRaises(ValueError, base64.z85decode, b'%nSc') + self.assertRaises(ValueError, base64.z85decode, b'%nSc1') + def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode, base64.standard_b64decode, @@ -714,7 +798,8 @@ def test_decode_nonascii_str(self): base64.b32decode, base64.b16decode, base64.b85decode, - base64.a85decode) + base64.a85decode, + base64.z85decode) for f in decode_funcs: self.assertRaises(ValueError, f, 'with non-ascii \xcb') diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index 4c3cf0b964ae56..e599b02c17d9c0 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -78,6 +78,9 @@ def test_inheritance(self): last_depth = depth finally: inheritance_tree.close() + + # Underscore-prefixed (private) exceptions don't need to be documented + exc_set = set(e for e in exc_set if not e.startswith('_')) self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set) interface_tests = ("length", "args", "str", "repr") @@ -129,7 +132,7 @@ class Value(str): d[HashThisKeyWillClearTheDict()] = Value() # refcount of Value() is 1 now - # Exception.__setstate__ should aquire a strong reference of key and + # Exception.__setstate__ should acquire a strong reference of key and # value in the dict. Otherwise, Value()'s refcount would go below # zero in the tp_hash call in PyObject_SetAttr(), and it would cause # crash in GC. diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 568c88e326c087..f15dae13eb384e 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -228,6 +228,10 @@ def user_exception(self, frame, exc_info): self.process_event('exception', frame) self.next_set_method() + def user_opcode(self, frame): + self.process_event('opcode', frame) + self.next_set_method() + def do_clear(self, arg): # The temporary breakpoints are deleted in user_line(). bp_list = [self.currentbp] @@ -366,7 +370,7 @@ def next_set_method(self): set_method = getattr(self, 'set_' + set_type) # The following set methods give back control to the tracer. - if set_type in ('step', 'continue', 'quit'): + if set_type in ('step', 'stepinstr', 'continue', 'quit'): set_method() return elif set_type in ('next', 'return'): @@ -610,6 +614,15 @@ def test_step_next_on_last_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) + def test_stepinstr(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('stepinstr', ), + ('opcode', 2, 'tfunc_main'), ('next', ), + ('line', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + def test_next(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -1033,8 +1046,9 @@ def main(): ('return', 1, ''), ('quit', ), ] import test_module_for_bdb + ns = {'test_module_for_bdb': test_module_for_bdb} with TracerRun(self) as tracer: - tracer.runeval('test_module_for_bdb.main()', globals(), locals()) + tracer.runeval('test_module_for_bdb.main()', ns, ns) class IssuesTestCase(BaseTestCase): """Test fixed bdb issues.""" @@ -1203,6 +1217,19 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) + def test_next_to_botframe(self): + # gh-125422 + # Check that next command won't go to the bottom frame. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, ''), ('step', ), + ('return', 2, ''), ('next', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '', 'exec')) + class TestRegressions(unittest.TestCase): def test_format_stack_entry_no_lineno(self): diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 82dea8a6d731ea..1f3b6746ce4a62 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -139,13 +139,21 @@ def assertLeadingPadding(data, non_strict_mode_expected_result: bytes): def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result) + def assertExcessPadding(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result) + # Test excess data exceptions assertExcessData(b'ab==a', b'i') assertExcessData(b'ab===', b'i') + assertExcessData(b'ab====', b'i') assertExcessData(b'ab==:', b'i') assertExcessData(b'abc=a', b'i\xb7') assertExcessData(b'abc=:', b'i\xb7') assertExcessData(b'ab==\n', b'i') + assertExcessData(b'abc==', b'i\xb7') + assertExcessData(b'abc===', b'i\xb7') + assertExcessData(b'abc====', b'i\xb7') + assertExcessData(b'abc=====', b'i\xb7') # Test non-base64 data exceptions assertNonBase64Data(b'\nab==', b'i') @@ -157,8 +165,15 @@ def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): assertLeadingPadding(b'=', b'') assertLeadingPadding(b'==', b'') assertLeadingPadding(b'===', b'') + assertLeadingPadding(b'====', b'') + assertLeadingPadding(b'=====', b'') assertDiscontinuousPadding(b'ab=c=', b'i\xb7') assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b') + assertExcessPadding(b'abcd=', b'i\xb7\x1d') + assertExcessPadding(b'abcd==', b'i\xb7\x1d') + assertExcessPadding(b'abcd===', b'i\xb7\x1d') + assertExcessPadding(b'abcd====', b'i\xb7\x1d') + assertExcessPadding(b'abcd=====', b'i\xb7\x1d') def test_base64errors(self): diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 5b1b95b9c82064..ae938d12c9401b 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -130,10 +130,10 @@ def native_type_range(fmt): for fmt in fmtdict['@']: fmtdict['@'][fmt] = native_type_range(fmt) -# Format codes suppported by the memoryview object +# Format codes supported by the memoryview object MEMORYVIEW = NATIVE.copy() -# Format codes suppported by array.array +# Format codes supported by array.array ARRAY = NATIVE.copy() for k in NATIVE: if not k in "bBhHiIlLfd": @@ -168,7 +168,7 @@ def randrange_fmt(mode, char, obj): if char == 'c': x = bytes([x]) if obj == 'numpy' and x == b'\x00': - # http://projects.scipy.org/numpy/ticket/1925 + # https://github.com/numpy/numpy/issues/2518 x = b'\x01' if char == '?': x = bool(x) @@ -1918,7 +1918,7 @@ def test_ndarray_random(self): if numpy_array: shape = t[3] if 0 in shape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 z = numpy_array_from_structure(items, fmt, t) self.verify(x, obj=None, itemsize=z.itemsize, fmt=fmt, readonly=False, @@ -1950,7 +1950,7 @@ def test_ndarray_random_invalid(self): except Exception as e: numpy_err = e.__class__ - if 0: # http://projects.scipy.org/numpy/ticket/1910 + if 0: # https://github.com/numpy/numpy/issues/2503 self.assertTrue(numpy_err) def test_ndarray_random_slice_assign(self): @@ -1996,7 +1996,7 @@ def test_ndarray_random_slice_assign(self): if numpy_array: if 0 in lshape or 0 in rshape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 zl = numpy_array_from_structure(litems, fmt, tl) zr = numpy_array_from_structure(ritems, fmt, tr) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 9a0bf524e3943f..eb5906f8944c8e 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -4,6 +4,7 @@ import asyncio import builtins import collections +import contextlib import decimal import fractions import gc @@ -16,6 +17,7 @@ import random import re import sys +import textwrap import traceback import types import typing @@ -30,6 +32,7 @@ from operator import neg from test import support from test.support import (cpython_only, swap_attr, maybe_get_event_loop_policy) +from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok from test.support.warnings_helper import check_warnings @@ -46,6 +49,8 @@ x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) +# used as proof of globals being used +A_GLOBAL_VALUE = 123 class Squares: @@ -410,7 +415,7 @@ def test_compile_top_level_await_no_coro(self): "socket.accept is broken" ) def test_compile_top_level_await(self): - """Test whether code some top level await can be compiled. + """Test whether code with top level await can be compiled. Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set, and make sure the generated code object has the CO_COROUTINE flag @@ -424,6 +429,7 @@ async def arange(n): yield i modes = ('single', 'exec') + optimizations = (-1, 0, 1, 2) code_samples = [ '''a = await asyncio.sleep(0, result=1)''', '''async for i in arange(1): @@ -436,34 +442,52 @@ async def arange(n): '''a = [x async for x in arange(2) async for x in arange(2)][1]''', '''a = [x async for x in (x async for x in arange(5))][1]''', '''a, = [1 for x in {x async for x in arange(1)}]''', - '''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''' + '''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''', + # gh-121637: Make sure we correctly handle the case where the + # async code is optimized away + '''assert not await asyncio.sleep(0); a = 1''', + '''assert [x async for x in arange(1)]; a = 1''', + '''assert {x async for x in arange(1)}; a = 1''', + '''assert {x: x async for x in arange(1)}; a = 1''', + ''' + if (a := 1) and __debug__: + async with asyncio.Lock() as l: + pass + ''', + ''' + if (a := 1) and __debug__: + async for x in arange(2): + pass + ''', ] policy = maybe_get_event_loop_policy() try: - for mode, code_sample in product(modes, code_samples): - source = dedent(code_sample) - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) - - co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - - self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, - msg=f"source={source} mode={mode}") + for mode, code_sample, optimize in product(modes, code_samples, optimizations): + with self.subTest(mode=mode, code_sample=code_sample, optimize=optimize): + source = dedent(code_sample) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + compile(source, '?', mode, optimize=optimize) - # test we can create and advance a function type - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - async_f = FunctionType(co, globals_) - asyncio.run(async_f()) - self.assertEqual(globals_['a'], 1) - - # test we can await-eval, - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - asyncio.run(eval(co, globals_)) - self.assertEqual(globals_['a'], 1) + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + optimize=optimize) + + self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, + msg=f"source={source} mode={mode}") + + # test we can create and advance a function type + globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} + async_f = FunctionType(co, globals_) + asyncio.run(async_f()) + self.assertEqual(globals_['a'], 1) + + # test we can await-eval, + globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} + asyncio.run(eval(co, globals_)) + self.assertEqual(globals_['a'], 1) finally: asyncio.set_event_loop_policy(policy) @@ -660,6 +684,16 @@ def test_divmod(self): self.assertAlmostEqual(result[1], exp_result[1]) self.assertRaises(TypeError, divmod) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 1, 0, + ) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 0.0, 0, + ) def test_eval(self): self.assertEqual(eval('1+1'), 2) @@ -684,6 +718,11 @@ def __getitem__(self, key): raise ValueError self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_eval_kwargs(self): + data = {"A_GLOBAL_VALUE": 456} + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", globals=data), 456) + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", locals=data), 123) + def test_general_eval(self): # Tests that general mappings can be used for the locals argument @@ -777,6 +816,19 @@ def test_exec(self): del l['__builtins__'] self.assertEqual((g, l), ({'a': 1}, {'b': 2})) + def test_exec_kwargs(self): + g = {} + exec('global z\nz = 1', globals=g) + if '__builtins__' in g: + del g['__builtins__'] + self.assertEqual(g, {'z': 1}) + + # if we only set locals, the global assignment will not + # reach this locals dictionary + g = {} + exec('global z\nz = 1', locals=g) + self.assertEqual(g, {}) + def test_exec_globals(self): code = compile("print('Hello World!')", "", "exec") # no builtin function @@ -1728,6 +1780,8 @@ def test_sum(self): self.assertRaises(TypeError, sum, [], '') self.assertRaises(TypeError, sum, [], b'') self.assertRaises(TypeError, sum, [], bytearray()) + self.assertRaises(OverflowError, sum, [1.0, 10**1000]) + self.assertRaises(OverflowError, sum, [1j, 10**1000]) class BadSeq: def __getitem__(self, index): @@ -1738,6 +1792,11 @@ def __getitem__(self, index): sum(([x] for x in range(10)), empty) self.assertEqual(empty, []) + xs = [complex(random.random() - .5, random.random() - .5) + for _ in range(10000)] + self.assertEqual(sum(xs), complex(sum(z.real for z in xs), + sum(z.imag for z in xs))) + @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "sum accuracy not guaranteed on machines with double rounding") @@ -1745,6 +1804,13 @@ def __getitem__(self, index): def test_sum_accuracy(self): self.assertEqual(sum([0.1] * 10), 1.0) self.assertEqual(sum([1.0, 10E100, 1.0, -10E100]), 2.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100, 2j]), 2+2j) + self.assertEqual(sum([2+1j, 10E100j, 1j, -10E100j]), 2+2j) + self.assertEqual(sum([1j, 1, 10E100j, 1j, 1.0, -10E100j]), 2+2j) + self.assertEqual(sum([2j, 1., 10E100, 1., -10E100]), 2+2j) + self.assertEqual(sum([1.0, 10**100, 1.0, -10**100]), 2.0) + self.assertEqual(sum([2j, 1.0, 10**100, 1.0, -10**100]), 2+2j) + self.assertEqual(sum([0.1j]*10 + [fractions.Fraction(1, 10)]), 0.1+1j) def test_type(self): self.assertEqual(type(''), type('123')) @@ -2105,19 +2171,36 @@ def test_construct_singletons(self): self.assertRaises(TypeError, tp, 1, 2) self.assertRaises(TypeError, tp, a=1, b=2) - def test_warning_notimplemented(self): - # Issue #35712: NotImplemented is a sentinel value that should never + def test_bool_notimplemented(self): + # GH-79893: NotImplemented is a sentinel value that should never # be evaluated in a boolean context (virtually all such use cases # are a result of accidental misuse implementing rich comparison # operations in terms of one another). - # For the time being, it will continue to evaluate as a true value, but - # issue a deprecation warning (with the eventual intent to make it - # a TypeError). - self.assertWarns(DeprecationWarning, bool, NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertTrue(NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertFalse(not NotImplemented) + msg = "NotImplemented should not be used in a boolean context" + self.assertRaisesRegex(TypeError, msg, bool, NotImplemented) + with self.assertRaisesRegex(TypeError, msg): + if NotImplemented: + pass + with self.assertRaisesRegex(TypeError, msg): + not NotImplemented + + def test_singleton_attribute_access(self): + for singleton in (NotImplemented, Ellipsis): + with self.subTest(singleton): + self.assertIs(type(singleton), singleton.__class__) + self.assertIs(type(singleton).__class__, type) + + # Missing instance attributes: + with self.assertRaises(AttributeError): + singleton.prop = 1 + with self.assertRaises(AttributeError): + singleton.prop + + # Missing class attributes: + with self.assertRaises(TypeError): + type(singleton).prop = 1 + with self.assertRaises(AttributeError): + type(singleton).prop class TestBreakpoint(unittest.TestCase): @@ -2331,7 +2414,8 @@ def child(wpipe): print(ascii(input(prompt)), file=wpipe) except BaseException as e: print(ascii(f'{e.__class__.__name__}: {e!s}'), file=wpipe) - lines = self.run_child(child, terminal_input + b"\r\n") + with self.detach_readline(): + lines = self.run_child(child, terminal_input + b"\r\n") # Check we did exercise the GNU readline path self.assertIn(lines[0], {'tty = True', 'tty = False'}) if lines[0] != 'tty = True': @@ -2344,28 +2428,36 @@ def child(wpipe): expected = terminal_input.decode(sys.stdin.encoding) # what else? self.assertEqual(input_result, expected) - def test_input_tty(self): - # Test input() functionality when wired to a tty (the code path - # is different and invokes GNU readline if available). - self.check_input_tty("prompt", b"quux") - - def skip_if_readline(self): + @contextlib.contextmanager + def detach_readline(self): # bpo-13886: When the readline module is loaded, PyOS_Readline() uses # the readline implementation. In some cases, the Python readline # callback rlhandler() is called by readline with a string without - # non-ASCII characters. Skip tests on non-ASCII characters if the - # readline module is loaded, since test_builtin is not intended to test + # non-ASCII characters. + # Unlink readline temporarily from PyOS_Readline() for those tests, + # since test_builtin is not intended to test # the readline module, but the builtins module. - if 'readline' in sys.modules: - self.skipTest("the readline module is loaded") + if "readline" in sys.modules: + c = import_module("ctypes") + fp_api = "PyOS_ReadlineFunctionPointer" + prev_value = c.c_void_p.in_dll(c.pythonapi, fp_api).value + c.c_void_p.in_dll(c.pythonapi, fp_api).value = None + try: + yield + finally: + c.c_void_p.in_dll(c.pythonapi, fp_api).value = prev_value + else: + yield + + def test_input_tty(self): + # Test input() functionality when wired to a tty + self.check_input_tty("prompt", b"quux") def test_input_tty_non_ascii(self): - self.skip_if_readline() # Check stdin/stdout encoding is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xc3\xa9", "utf-8") def test_input_tty_non_ascii_unicode_errors(self): - self.skip_if_readline() # Check stdin/stdout error handler is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") @@ -2375,14 +2467,12 @@ def test_input_tty_null_in_prompt(self): 'null characters') def test_input_tty_nonencodable_prompt(self): - self.skip_if_readline() self.check_input_tty("prompté", b"quux", "ascii", stdout_errors='strict', expected="UnicodeEncodeError: 'ascii' codec can't encode " "character '\\xe9' in position 6: ordinal not in " "range(128)") def test_input_tty_nondecodable_input(self): - self.skip_if_readline() self.check_input_tty("prompt", b"quux\xe9", "ascii", stdin_errors='strict', expected="UnicodeDecodeError: 'ascii' codec can't decode " "byte 0xe9 in position 4: ordinal not in " @@ -2484,9 +2574,9 @@ def __del__(self): class ImmortalTests(unittest.TestCase): if sys.maxsize < (1 << 32): - IMMORTAL_REFCOUNT = (1 << 30) - 1 + IMMORTAL_REFCOUNT = 3 << 29 else: - IMMORTAL_REFCOUNT = (1 << 32) - 1 + IMMORTAL_REFCOUNT = 3 << 30 IMMORTALS = (None, True, False, Ellipsis, NotImplemented, *range(-5, 257)) @@ -2517,6 +2607,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2595,6 +2686,17 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + def test_type_typeparams(self): class A[T]: pass diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a3804a945f2e3a..9e1985bb3a7639 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -991,13 +991,13 @@ def test_translate(self): self.assertEqual(c, b'hllo') def test_sq_item(self): - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') obj = self.type2test((42,)) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, -2) + _testlimitedcapi.sequence_getitem(obj, -2) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, 1) - self.assertEqual(_testcapi.sequence_getitem(obj, 0), 42) + _testlimitedcapi.sequence_getitem(obj, 1) + self.assertEqual(_testlimitedcapi.sequence_getitem(obj, 0), 42) class BytesTest(BaseBytesTest, unittest.TestCase): @@ -1256,7 +1256,7 @@ class SubBytes(bytes): class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') def test_getitem_error(self): b = bytearray(b'python') @@ -1354,7 +1354,7 @@ def setitem_as_mapping(b, i, val): b[i] = val def setitem_as_sequence(b, i, val): - self._testcapi.sequence_setitem(b, i, val) + self._testlimitedcapi.sequence_setitem(b, i, val) def do_tests(setitem): b = bytearray([1, 2, 3]) @@ -1401,7 +1401,7 @@ def del_as_mapping(b, i): del b[i] def del_as_sequence(b, i): - self._testcapi.sequence_delitem(b, i) + self._testlimitedcapi.sequence_delitem(b, i) def do_tests(delete): b = bytearray(range(10)) @@ -1599,6 +1599,13 @@ def test_extend(self): a = bytearray(b'') a.extend([Indexable(ord('a'))]) self.assertEqual(a, b'a') + a = bytearray(b'abc') + self.assertRaisesRegex(TypeError, # Override for string. + "expected iterable of integers; got: 'str'", + a.extend, 'def') + self.assertRaisesRegex(TypeError, # But not for others. + "can't extend bytearray with float", + a.extend, 1.0) def test_remove(self): b = bytearray(b'hello') @@ -1803,7 +1810,7 @@ def __index__(self): with self.subTest("tp_as_sequence"): b = bytearray(b'Now you see me...') with self.assertRaises(IndexError): - self._testcapi.sequence_setitem(b, 0, Boom()) + self._testlimitedcapi.sequence_setitem(b, 0, Boom()) class AssortedBytesTest(unittest.TestCase): diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 1f0b9adc3698b4..7d786be1d25b1c 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -3,19 +3,19 @@ import array import unittest +import io from io import BytesIO, DEFAULT_BUFFER_SIZE import os import pickle import glob import tempfile -import pathlib import random import shutil import subprocess import threading from test.support import import_helper from test.support import threading_helper -from test.support.os_helper import unlink +from test.support.os_helper import unlink, FakePath import _compression import sys @@ -476,7 +476,6 @@ def testReadlinesNoNewline(self): self.assertEqual(xlines, [b'Test']) def testContextProtocol(self): - f = None with BZ2File(self.filename, "wb") as f: f.write(b"xxx") f = BZ2File(self.filename, "rb") @@ -537,26 +536,208 @@ def testMultiStreamOrdering(self): with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(), data1 + data2) + def testOpenFilename(self): + with BZ2File(self.filename, "wb") as f: + f.write(b'content') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, "ab") as f: + f.write(b'appendix') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithName(self): + with open(self.filename, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithoutName(self): + bio = BytesIO() + with BZ2File(bio, 'wb') as f: + f.write(b'content') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + with BZ2File(bio, 'ab') as f: + f.write(b'appendix') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + bio.seek(0) + with BZ2File(bio, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'rb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + def testOpenFileWithIntName(self): + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with open(fd, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with open(fd, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + def testOpenBytesFilename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with BZ2File(bytes_filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, bytes_filename) with BZ2File(bytes_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with BZ2File(str_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, str_filename) def testOpenPathLikeFilename(self): - filename = pathlib.Path(self.filename) + filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, self.filename) with BZ2File(filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, self.filename) def testDecompressLimited(self): """Decompressed data buffering should be limited""" @@ -577,6 +758,9 @@ def testReadBytesIO(self): with BZ2File(bio) as bz2f: self.assertRaises(TypeError, bz2f.read, float()) self.assertEqual(bz2f.read(), self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'rb') self.assertFalse(bio.closed) def testPeekBytesIO(self): @@ -592,6 +776,9 @@ def testWriteBytesIO(self): with BZ2File(bio, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'wb') self.assertEqual(ext_decompress(bio.getvalue()), self.TEXT) self.assertFalse(bio.closed) diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 7334a325ba22f0..e4b0b8c451fd45 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -26,17 +26,16 @@ TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"] # Apply some platform dependent overrides -if sys.platform.startswith("linux"): - if support.is_android: - # Android defaults to using UTF-8 for all system interfaces - EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" - EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" - else: - # Linux distros typically alias the POSIX locale directly to the C - # locale. - # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be - # able to check this case unconditionally - EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") +if sys.platform == "android": + # Android defaults to using UTF-8 for all system interfaces + EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" + EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" +elif sys.platform.startswith("linux"): + # Linux distros typically alias the POSIX locale directly to the C + # locale. + # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be + # able to check this case unconditionally + EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") elif sys.platform.startswith("aix"): # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1" diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 1f9ffc5e9a5c33..f119d89c0ec39a 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -985,7 +985,7 @@ def assertFailure(self, *args): def test_help(self): stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) - self.assertIn(b'calendar.py', stdout) + self.assertIn(b' -m calendar ', stdout) self.assertIn(b'--help', stdout) # special case: stdout but sys.exit() diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 2a6a5d287b04ee..68e3b2a0d4d932 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -5,12 +5,15 @@ import _testcapi except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None import struct import collections import itertools import gc import contextlib -import sys import types @@ -42,11 +45,16 @@ def test_frames_are_popped_after_failed_calls(self): # recovering from failed calls: def f(): pass - for _ in range(1000): - try: - f(None) - except TypeError: + class C: + def m(self): pass + callables = [f, C.m, [].__len__] + for c in callables: + for _ in range(1000): + try: + c(None) + except TypeError: + pass # BOOM! @@ -240,6 +248,7 @@ def test_module_not_callable_suggestion(self): self.assertRaisesRegex(TypeError, msg, mod) +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestCallingConventions(unittest.TestCase): """Test calling using various C calling conventions (METH_*) from Python @@ -437,6 +446,7 @@ def static_method(): NULL_OR_EMPTY = object() + class FastCallTests(unittest.TestCase): """Test calling using various callables from C """ @@ -480,42 +490,43 @@ class FastCallTests(unittest.TestCase): ] # Add all the calling conventions and variants of C callables - _instance = _testcapi.MethInstance() - for obj, expected_self in ( - (_testcapi, _testcapi), # module-level function - (_instance, _instance), # bound method - (_testcapi.MethClass, _testcapi.MethClass), # class method on class - (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. - (_testcapi.MethStatic, None), # static method - ): - CALLS_POSARGS.extend([ - (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), - (obj.meth_varargs_keywords, - (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), - (obj.meth_fastcall, (), (expected_self, ())), - (obj.meth_fastcall_keywords, - (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall_keywords, - (), (expected_self, (), NULL_OR_EMPTY)), - (obj.meth_noargs, (), expected_self), - (obj.meth_o, (123, ), (expected_self, 123)), - ]) - - CALLS_KWARGS.extend([ - (obj.meth_varargs_keywords, - (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), - (obj.meth_varargs_keywords, - (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), - (obj.meth_varargs_keywords, - (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall_keywords, - (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), - (obj.meth_fastcall_keywords, - (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), - (obj.meth_fastcall_keywords, - (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), - ]) + if _testcapi: + _instance = _testcapi.MethInstance() + for obj, expected_self in ( + (_testcapi, _testcapi), # module-level function + (_instance, _instance), # bound method + (_testcapi.MethClass, _testcapi.MethClass), # class method on class + (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. + (_testcapi.MethStatic, None), # static method + ): + CALLS_POSARGS.extend([ + (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), + (obj.meth_varargs_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), + (obj.meth_fastcall, (), (expected_self, ())), + (obj.meth_fastcall_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (), (expected_self, (), NULL_OR_EMPTY)), + (obj.meth_noargs, (), expected_self), + (obj.meth_o, (123, ), (expected_self, 123)), + ]) + + CALLS_KWARGS.extend([ + (obj.meth_varargs_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_varargs_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_varargs_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + ]) def check_result(self, result, expected): if isinstance(expected, tuple) and expected[-1] is NULL_OR_EMPTY: @@ -523,6 +534,7 @@ def check_result(self, result, expected): expected = (*expected[:-1], result[-1]) self.assertEqual(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall_dict(self): # Test PyObject_VectorcallDict() @@ -542,6 +554,7 @@ def test_vectorcall_dict(self): result = _testcapi.pyobject_fastcalldict(func, args, kwargs) self.check_result(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall(self): # Test PyObject_Vectorcall() @@ -606,6 +619,7 @@ def testfunction_kw(self, *, kw): ADAPTIVE_WARMUP_DELAY = 2 +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestPEP590(unittest.TestCase): def test_method_descriptor_flag(self): @@ -837,12 +851,17 @@ def get_a(x): @requires_limited_api def test_vectorcall_limited_incoming(self): from _testcapi import pyobject_vectorcall - obj = _testcapi.LimitedVectorCallClass() - self.assertEqual(pyobject_vectorcall(obj, (), ()), "vectorcall called") + for cls in (_testlimitedcapi.LimitedVectorCallClass, + _testlimitedcapi.LimitedRelativeVectorCallClass): + with self.subTest(cls=cls): + obj = cls() + self.assertEqual( + pyobject_vectorcall(obj, (), ()), + "vectorcall called") @requires_limited_api def test_vectorcall_limited_outgoing(self): - from _testcapi import call_vectorcall + from _testlimitedcapi import call_vectorcall args_captured = [] kwargs_captured = [] @@ -858,7 +877,7 @@ def f(*args, **kwargs): @requires_limited_api def test_vectorcall_limited_outgoing_method(self): - from _testcapi import call_vectorcall_method + from _testlimitedcapi import call_vectorcall_method args_captured = [] kwargs_captured = [] @@ -1018,6 +1037,7 @@ class TestRecursion(unittest.TestCase): @skip_on_s390x @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_capi/__init__.py b/Lib/test/test_capi/__init__.py index 4b16ecc31156a5..5a8ba6845399e0 100644 --- a/Lib/test/test_capi/__init__.py +++ b/Lib/test/test_capi/__init__.py @@ -1,5 +1,12 @@ import os +import unittest from test.support import load_package_tests +from test.support import TEST_MODULES_ENABLED + + +if not TEST_MODULES_ENABLED: + raise unittest.SkipTest("requires test modules") + def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_capi/check_config.py b/Lib/test/test_capi/check_config.py index eb99ae16f2b69e..bc4f3a0beb9c1e 100644 --- a/Lib/test/test_capi/check_config.py +++ b/Lib/test/test_capi/check_config.py @@ -10,7 +10,7 @@ def import_singlephase(): assert '_testsinglephase' not in sys.modules try: - import _testsinglephase + import _testsinglephase # noqa: F401 except ImportError: sys.modules.pop('_testsinglephase', None) return False diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index 97ed939928c360..6a626813f23379 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -4,6 +4,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None @@ -74,7 +75,7 @@ def assertTypedEqual(self, actual, expected): def test_object_str(self): # Test PyObject_Str() - object_str = _testcapi.object_str + object_str = _testlimitedcapi.object_str self.assertTypedEqual(object_str(''), '') self.assertTypedEqual(object_str('abc'), 'abc') self.assertTypedEqual(object_str('\U0001f40d'), '\U0001f40d') @@ -87,7 +88,7 @@ def test_object_str(self): def test_object_repr(self): # Test PyObject_Repr() - object_repr = _testcapi.object_repr + object_repr = _testlimitedcapi.object_repr self.assertTypedEqual(object_repr(''), "''") self.assertTypedEqual(object_repr('abc'), "'abc'") self.assertTypedEqual(object_repr('\U0001f40d'), "'\U0001f40d'") @@ -100,7 +101,7 @@ def test_object_repr(self): def test_object_ascii(self): # Test PyObject_ASCII() - object_ascii = _testcapi.object_ascii + object_ascii = _testlimitedcapi.object_ascii self.assertTypedEqual(object_ascii(''), "''") self.assertTypedEqual(object_ascii('abc'), "'abc'") self.assertTypedEqual(object_ascii('\U0001f40d'), r"'\U0001f40d'") @@ -113,7 +114,7 @@ def test_object_ascii(self): def test_object_bytes(self): # Test PyObject_Bytes() - object_bytes = _testcapi.object_bytes + object_bytes = _testlimitedcapi.object_bytes self.assertTypedEqual(object_bytes(b''), b'') self.assertTypedEqual(object_bytes(b'abc'), b'abc') self.assertTypedEqual(object_bytes(BytesSubclass(b'abc')), b'abc') @@ -132,7 +133,7 @@ def test_object_bytes(self): self.assertTypedEqual(object_bytes(NULL), b'') def test_object_getattr(self): - xgetattr = _testcapi.object_getattr + xgetattr = _testlimitedcapi.object_getattr obj = TestObject() obj.a = 11 setattr(obj, '\U0001f40d', 22) @@ -146,7 +147,7 @@ def test_object_getattr(self): # CRASHES xgetattr(NULL, 'a') def test_object_getattrstring(self): - getattrstring = _testcapi.object_getattrstring + getattrstring = _testlimitedcapi.object_getattrstring obj = TestObject() obj.a = 11 setattr(obj, '\U0001f40d', 22) @@ -188,7 +189,7 @@ def test_object_getoptionalattrstring(self): # CRASHES getoptionalattrstring(NULL, b'a') def test_object_hasattr(self): - xhasattr = _testcapi.object_hasattr + xhasattr = _testlimitedcapi.object_hasattr obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -212,7 +213,7 @@ def test_object_hasattr(self): # CRASHES xhasattr(NULL, 'a') def test_object_hasattrstring(self): - hasattrstring = _testcapi.object_hasattrstring + hasattrstring = _testlimitedcapi.object_hasattrstring obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -264,7 +265,7 @@ def test_object_hasattrstringwitherror(self): # CRASHES hasattrstring(NULL, b'a') def test_object_setattr(self): - xsetattr = _testcapi.object_setattr + xsetattr = _testlimitedcapi.object_setattr obj = TestObject() xsetattr(obj, 'a', 5) self.assertEqual(obj.a, 5) @@ -284,7 +285,7 @@ def test_object_setattr(self): # CRASHES xsetattr(NULL, 'a', 5) def test_object_setattrstring(self): - setattrstring = _testcapi.object_setattrstring + setattrstring = _testlimitedcapi.object_setattrstring obj = TestObject() setattrstring(obj, b'a', 5) self.assertEqual(obj.a, 5) @@ -305,7 +306,7 @@ def test_object_setattrstring(self): # CRASHES setattrstring(NULL, b'a', 5) def test_object_delattr(self): - xdelattr = _testcapi.object_delattr + xdelattr = _testlimitedcapi.object_delattr obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -322,7 +323,7 @@ def test_object_delattr(self): # CRASHES xdelattr(NULL, 'a') def test_object_delattrstring(self): - delattrstring = _testcapi.object_delattrstring + delattrstring = _testlimitedcapi.object_delattrstring obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -340,7 +341,7 @@ def test_object_delattrstring(self): def test_mapping_check(self): - check = _testcapi.mapping_check + check = _testlimitedcapi.mapping_check self.assertTrue(check({1: 2})) self.assertTrue(check([1, 2])) self.assertTrue(check((1, 2))) @@ -351,7 +352,7 @@ def test_mapping_check(self): self.assertFalse(check(NULL)) def test_mapping_size(self): - for size in _testcapi.mapping_size, _testcapi.mapping_length: + for size in _testlimitedcapi.mapping_size, _testlimitedcapi.mapping_length: self.assertEqual(size({1: 2}), 1) self.assertEqual(size([1, 2]), 2) self.assertEqual(size((1, 2)), 2) @@ -363,7 +364,7 @@ def test_mapping_size(self): self.assertRaises(SystemError, size, NULL) def test_object_getitem(self): - getitem = _testcapi.object_getitem + getitem = _testlimitedcapi.object_getitem dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertRaises(KeyError, getitem, dct, 'b') @@ -383,7 +384,7 @@ def test_object_getitem(self): self.assertRaises(SystemError, getitem, NULL, 'a') def test_mapping_getitemstring(self): - getitemstring = _testcapi.mapping_getitemstring + getitemstring = _testlimitedcapi.mapping_getitemstring dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitemstring(dct, b'a'), 1) self.assertRaises(KeyError, getitemstring, dct, b'b') @@ -437,7 +438,7 @@ def test_mapping_getoptionalitemstring(self): # CRASHES getitemstring(NULL, b'a') def test_mapping_haskey(self): - haskey = _testcapi.mapping_haskey + haskey = _testlimitedcapi.mapping_haskey dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskey(dct, 'a')) self.assertFalse(haskey(dct, 'b')) @@ -486,7 +487,7 @@ def test_mapping_haskey(self): 'null argument to internal routine') def test_mapping_haskeystring(self): - haskeystring = _testcapi.mapping_haskeystring + haskeystring = _testlimitedcapi.mapping_haskeystring dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskeystring(dct, b'a')) self.assertFalse(haskeystring(dct, b'b')) @@ -527,7 +528,7 @@ def test_mapping_haskeystring(self): "null argument to internal routine") def test_mapping_haskeywitherror(self): - haskey = _testcapi.mapping_haskeywitherror + haskey = _testlimitedcapi.mapping_haskeywitherror dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskey(dct, 'a')) self.assertFalse(haskey(dct, 'b')) @@ -548,7 +549,7 @@ def test_mapping_haskeywitherror(self): # CRASHES haskey(NULL, 'a')) def test_mapping_haskeystringwitherror(self): - haskeystring = _testcapi.mapping_haskeystringwitherror + haskeystring = _testlimitedcapi.mapping_haskeystringwitherror dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskeystring(dct, b'a')) self.assertFalse(haskeystring(dct, b'b')) @@ -565,7 +566,7 @@ def test_mapping_haskeystringwitherror(self): # CRASHES haskeystring(NULL, b'a') def test_object_setitem(self): - setitem = _testcapi.object_setitem + setitem = _testlimitedcapi.object_setitem dct = {} setitem(dct, 'a', 5) self.assertEqual(dct, {'a': 5}) @@ -591,7 +592,7 @@ def test_object_setitem(self): self.assertRaises(SystemError, setitem, NULL, 'a', 5) def test_mapping_setitemstring(self): - setitemstring = _testcapi.mapping_setitemstring + setitemstring = _testlimitedcapi.mapping_setitemstring dct = {} setitemstring(dct, b'a', 5) self.assertEqual(dct, {'a': 5}) @@ -611,7 +612,7 @@ def test_mapping_setitemstring(self): self.assertRaises(SystemError, setitemstring, NULL, b'a', 5) def test_object_delitem(self): - for delitem in _testcapi.object_delitem, _testcapi.mapping_delitem: + for delitem in _testlimitedcapi.object_delitem, _testlimitedcapi.mapping_delitem: dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitem(dct, 'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -637,7 +638,7 @@ def test_object_delitem(self): self.assertRaises(SystemError, delitem, NULL, 'a') def test_mapping_delitemstring(self): - delitemstring = _testcapi.mapping_delitemstring + delitemstring = _testlimitedcapi.mapping_delitemstring dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitemstring(dct, b'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -677,23 +678,23 @@ def items(self): for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), dict_obj, OrderedDict(dict_obj), Mapping1(dict_obj), Mapping2(dict_obj)]: - self.assertListEqual(_testcapi.mapping_keys(mapping), + self.assertListEqual(_testlimitedcapi.mapping_keys(mapping), list(mapping.keys())) - self.assertListEqual(_testcapi.mapping_values(mapping), + self.assertListEqual(_testlimitedcapi.mapping_values(mapping), list(mapping.values())) - self.assertListEqual(_testcapi.mapping_items(mapping), + self.assertListEqual(_testlimitedcapi.mapping_items(mapping), list(mapping.items())) def test_mapping_keys_valuesitems_bad_arg(self): - self.assertRaises(AttributeError, _testcapi.mapping_keys, object()) - self.assertRaises(AttributeError, _testcapi.mapping_values, object()) - self.assertRaises(AttributeError, _testcapi.mapping_items, object()) - self.assertRaises(AttributeError, _testcapi.mapping_keys, []) - self.assertRaises(AttributeError, _testcapi.mapping_values, []) - self.assertRaises(AttributeError, _testcapi.mapping_items, []) - self.assertRaises(SystemError, _testcapi.mapping_keys, NULL) - self.assertRaises(SystemError, _testcapi.mapping_values, NULL) - self.assertRaises(SystemError, _testcapi.mapping_items, NULL) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, []) + self.assertRaises(SystemError, _testlimitedcapi.mapping_keys, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_values, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_items, NULL) class BadMapping: def keys(self): @@ -703,12 +704,12 @@ def values(self): def items(self): return None bad_mapping = BadMapping() - self.assertRaises(TypeError, _testcapi.mapping_keys, bad_mapping) - self.assertRaises(TypeError, _testcapi.mapping_values, bad_mapping) - self.assertRaises(TypeError, _testcapi.mapping_items, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_keys, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_values, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_items, bad_mapping) def test_sequence_check(self): - check = _testcapi.sequence_check + check = _testlimitedcapi.sequence_check self.assertFalse(check({1: 2})) self.assertTrue(check([1, 2])) self.assertTrue(check((1, 2))) @@ -719,7 +720,7 @@ def test_sequence_check(self): # CRASHES check(NULL) def test_sequence_size(self): - for size in _testcapi.sequence_size, _testcapi.sequence_length: + for size in _testlimitedcapi.sequence_size, _testlimitedcapi.sequence_length: self.assertEqual(size([1, 2]), 2) self.assertEqual(size((1, 2)), 2) self.assertEqual(size('abc'), 3) @@ -731,7 +732,7 @@ def test_sequence_size(self): self.assertRaises(SystemError, size, NULL) def test_sequence_getitem(self): - getitem = _testcapi.sequence_getitem + getitem = _testlimitedcapi.sequence_getitem lst = ['a', 'b', 'c'] self.assertEqual(getitem(lst, 1), 'b') self.assertEqual(getitem(lst, -1), 'c') @@ -744,7 +745,7 @@ def test_sequence_getitem(self): self.assertRaises(SystemError, getitem, NULL, 1) def test_sequence_concat(self): - concat = _testcapi.sequence_concat + concat = _testlimitedcapi.sequence_concat self.assertEqual(concat(['a', 'b'], [1, 2]), ['a', 'b', 1, 2]) self.assertEqual(concat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) @@ -757,7 +758,7 @@ def test_sequence_concat(self): self.assertRaises(SystemError, concat, NULL, []) def test_sequence_repeat(self): - repeat = _testcapi.sequence_repeat + repeat = _testlimitedcapi.sequence_repeat self.assertEqual(repeat(['a', 'b'], 2), ['a', 'b', 'a', 'b']) self.assertEqual(repeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) self.assertEqual(repeat(['a', 'b'], 0), []) @@ -771,7 +772,7 @@ def test_sequence_repeat(self): self.assertRaises(SystemError, repeat, NULL, 2) def test_sequence_inplaceconcat(self): - inplaceconcat = _testcapi.sequence_inplaceconcat + inplaceconcat = _testlimitedcapi.sequence_inplaceconcat lst = ['a', 'b'] res = inplaceconcat(lst, [1, 2]) self.assertEqual(res, ['a', 'b', 1, 2]) @@ -790,7 +791,7 @@ def test_sequence_inplaceconcat(self): self.assertRaises(SystemError, inplaceconcat, NULL, []) def test_sequence_inplacerepeat(self): - inplacerepeat = _testcapi.sequence_inplacerepeat + inplacerepeat = _testlimitedcapi.sequence_inplacerepeat lst = ['a', 'b'] res = inplacerepeat(lst, 2) self.assertEqual(res, ['a', 'b', 'a', 'b']) @@ -807,7 +808,7 @@ def test_sequence_inplacerepeat(self): self.assertRaises(SystemError, inplacerepeat, NULL, 2) def test_sequence_setitem(self): - setitem = _testcapi.sequence_setitem + setitem = _testlimitedcapi.sequence_setitem lst = ['a', 'b', 'c'] setitem(lst, 1, 'x') self.assertEqual(lst, ['a', 'x', 'c']) @@ -825,7 +826,7 @@ def test_sequence_setitem(self): self.assertRaises(SystemError, setitem, NULL, 1, 'x') def test_sequence_delitem(self): - delitem = _testcapi.sequence_delitem + delitem = _testlimitedcapi.sequence_delitem lst = ['a', 'b', 'c'] delitem(lst, 1) self.assertEqual(lst, ['a', 'c']) @@ -840,7 +841,7 @@ def test_sequence_delitem(self): self.assertRaises(SystemError, delitem, NULL, 1) def test_sequence_setslice(self): - setslice = _testcapi.sequence_setslice + setslice = _testlimitedcapi.sequence_setslice # Correct case: for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: @@ -882,7 +883,7 @@ def __setitem__(self, index, value): self.assertRaises(SystemError, setslice, NULL, 1, 3, 'xy') def test_sequence_delslice(self): - delslice = _testcapi.sequence_delslice + delslice = _testlimitedcapi.sequence_delslice # Correct case: for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: @@ -920,7 +921,7 @@ def __delitem__(self, index): self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) def test_sequence_count(self): - count = _testcapi.sequence_count + count = _testlimitedcapi.sequence_count lst = ['a', 'b', 'a'] self.assertEqual(count(lst, 'a'), 2) @@ -935,7 +936,7 @@ def test_sequence_count(self): self.assertRaises(SystemError, count, NULL, 'a') def test_sequence_contains(self): - contains = _testcapi.sequence_contains + contains = _testlimitedcapi.sequence_contains lst = ['a', 'b', 'a'] self.assertEqual(contains(lst, 'a'), 1) @@ -954,7 +955,7 @@ def test_sequence_contains(self): # CRASHES contains(NULL, 'a') def test_sequence_index(self): - index = _testcapi.sequence_index + index = _testlimitedcapi.sequence_index lst = ['a', 'b', 'a'] self.assertEqual(index(lst, 'a'), 0) @@ -974,7 +975,7 @@ def test_sequence_index(self): self.assertRaises(SystemError, index, NULL, 'a') def test_sequence_list(self): - xlist = _testcapi.sequence_list + xlist = _testlimitedcapi.sequence_list self.assertEqual(xlist(['a', 'b', 'c']), ['a', 'b', 'c']) self.assertEqual(xlist(('a', 'b', 'c')), ['a', 'b', 'c']) self.assertEqual(xlist(iter(['a', 'b', 'c'])), ['a', 'b', 'c']) @@ -984,7 +985,7 @@ def test_sequence_list(self): self.assertRaises(SystemError, xlist, NULL) def test_sequence_tuple(self): - xtuple = _testcapi.sequence_tuple + xtuple = _testlimitedcapi.sequence_tuple self.assertEqual(xtuple(['a', 'b', 'c']), ('a', 'b', 'c')) self.assertEqual(xtuple(('a', 'b', 'c')), ('a', 'b', 'c')) self.assertEqual(xtuple(iter(['a', 'b', 'c'])), ('a', 'b', 'c')) @@ -993,12 +994,51 @@ def test_sequence_tuple(self): self.assertRaises(TypeError, xtuple, 42) self.assertRaises(SystemError, xtuple, NULL) - def test_number_check(self): - number_check = _testcapi.number_check - self.assertTrue(number_check(1 + 1j)) - self.assertTrue(number_check(1)) - self.assertTrue(number_check(0.5)) - self.assertFalse(number_check("1 + 1j")) + def test_object_generichash(self): + # Test PyObject_GenericHash() + generichash = _testcapi.object_generichash + for obj in object(), 1, 'string', []: + self.assertEqual(generichash(obj), object.__hash__(obj)) + + def run_iter_api_test(self, next_func): + for data in (), [], (1, 2, 3), [1 , 2, 3], "123": + with self.subTest(data=data): + items = [] + it = iter(data) + while (item := next_func(it)) is not None: + items.append(item) + self.assertEqual(items, list(data)) + + class Broken: + def __init__(self): + self.count = 0 + + def __next__(self): + if self.count < 3: + self.count += 1 + return self.count + else: + raise TypeError('bad type') + + it = Broken() + self.assertEqual(next_func(it), 1) + self.assertEqual(next_func(it), 2) + self.assertEqual(next_func(it), 3) + with self.assertRaisesRegex(TypeError, 'bad type'): + next_func(it) + + def test_iter_next(self): + from _testcapi import PyIter_Next + self.run_iter_api_test(PyIter_Next) + # CRASHES PyIter_Next(10) + + def test_iter_nextitem(self): + from _testcapi import PyIter_NextItem + self.run_iter_api_test(PyIter_NextItem) + + regex = "expected.*iterator.*got.*'int'" + with self.assertRaisesRegex(TypeError, regex): + PyIter_NextItem(10) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index 833122c4e319d8..39099f6b82240f 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -1,7 +1,7 @@ import unittest from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None @@ -19,7 +19,7 @@ def __bytes__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyByteArray_Check() - check = _testcapi.bytearray_check + check = _testlimitedcapi.bytearray_check self.assertTrue(check(bytearray(b'abc'))) self.assertFalse(check(b'abc')) self.assertTrue(check(ByteArraySubclass(b'abc'))) @@ -32,7 +32,7 @@ def test_check(self): def test_checkexact(self): # Test PyByteArray_CheckExact() - check = _testcapi.bytearray_checkexact + check = _testlimitedcapi.bytearray_checkexact self.assertTrue(check(bytearray(b'abc'))) self.assertFalse(check(b'abc')) self.assertFalse(check(ByteArraySubclass(b'abc'))) @@ -45,7 +45,7 @@ def test_checkexact(self): def test_fromstringandsize(self): # Test PyByteArray_FromStringAndSize() - fromstringandsize = _testcapi.bytearray_fromstringandsize + fromstringandsize = _testlimitedcapi.bytearray_fromstringandsize self.assertEqual(fromstringandsize(b'abc'), bytearray(b'abc')) self.assertEqual(fromstringandsize(b'abc', 2), bytearray(b'ab')) @@ -62,7 +62,7 @@ def test_fromstringandsize(self): def test_fromobject(self): # Test PyByteArray_FromObject() - fromobject = _testcapi.bytearray_fromobject + fromobject = _testlimitedcapi.bytearray_fromobject self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) @@ -77,7 +77,7 @@ def test_fromobject(self): def test_size(self): # Test PyByteArray_Size() - size = _testcapi.bytearray_size + size = _testlimitedcapi.bytearray_size self.assertEqual(size(bytearray(b'abc')), 3) self.assertEqual(size(ByteArraySubclass(b'abc')), 3) @@ -88,7 +88,7 @@ def test_size(self): def test_asstring(self): """Test PyByteArray_AsString()""" - asstring = _testcapi.bytearray_asstring + asstring = _testlimitedcapi.bytearray_asstring self.assertEqual(asstring(bytearray(b'abc'), 4), b'abc\0') self.assertEqual(asstring(ByteArraySubclass(b'abc'), 4), b'abc\0') @@ -100,7 +100,7 @@ def test_asstring(self): def test_concat(self): """Test PyByteArray_Concat()""" - concat = _testcapi.bytearray_concat + concat = _testlimitedcapi.bytearray_concat ba = bytearray(b'abc') self.assertEqual(concat(ba, b'def'), bytearray(b'abcdef')) @@ -133,7 +133,7 @@ def test_concat(self): def test_resize(self): """Test PyByteArray_Resize()""" - resize = _testcapi.bytearray_resize + resize = _testlimitedcapi.bytearray_resize ba = bytearray(b'abcdef') self.assertEqual(resize(ba, 3), 0) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index bb5d724ff187d4..5908d79e14029b 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -1,6 +1,7 @@ import unittest from test.support import import_helper +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testcapi = import_helper.import_module('_testcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX @@ -19,7 +20,7 @@ def __bytes__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyBytes_Check() - check = _testcapi.bytes_check + check = _testlimitedcapi.bytes_check self.assertTrue(check(b'abc')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) @@ -33,7 +34,7 @@ def test_check(self): def test_checkexact(self): # Test PyBytes_CheckExact() - check = _testcapi.bytes_checkexact + check = _testlimitedcapi.bytes_checkexact self.assertTrue(check(b'abc')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) @@ -47,11 +48,13 @@ def test_checkexact(self): def test_fromstringandsize(self): # Test PyBytes_FromStringAndSize() - fromstringandsize = _testcapi.bytes_fromstringandsize + fromstringandsize = _testlimitedcapi.bytes_fromstringandsize self.assertEqual(fromstringandsize(b'abc'), b'abc') self.assertEqual(fromstringandsize(b'abc', 2), b'ab') self.assertEqual(fromstringandsize(b'abc\0def'), b'abc\0def') + self.assertEqual(fromstringandsize(b'a'), b'a') + self.assertEqual(fromstringandsize(b'a', 1), b'a') self.assertEqual(fromstringandsize(b'', 0), b'') self.assertEqual(fromstringandsize(NULL, 0), b'') self.assertEqual(len(fromstringandsize(NULL, 3)), 3) @@ -65,7 +68,7 @@ def test_fromstringandsize(self): def test_fromstring(self): # Test PyBytes_FromString() - fromstring = _testcapi.bytes_fromstring + fromstring = _testlimitedcapi.bytes_fromstring self.assertEqual(fromstring(b'abc\0def'), b'abc') self.assertEqual(fromstring(b''), b'') @@ -74,7 +77,7 @@ def test_fromstring(self): def test_fromobject(self): # Test PyBytes_FromObject() - fromobject = _testcapi.bytes_fromobject + fromobject = _testlimitedcapi.bytes_fromobject self.assertEqual(fromobject(b'abc'), b'abc') self.assertEqual(fromobject(bytearray(b'abc')), b'abc') @@ -88,7 +91,7 @@ def test_fromobject(self): def test_size(self): # Test PyBytes_Size() - size = _testcapi.bytes_size + size = _testlimitedcapi.bytes_size self.assertEqual(size(b'abc'), 3) self.assertEqual(size(BytesSubclass(b'abc')), 3) @@ -100,7 +103,7 @@ def test_size(self): def test_asstring(self): """Test PyBytes_AsString()""" - asstring = _testcapi.bytes_asstring + asstring = _testlimitedcapi.bytes_asstring self.assertEqual(asstring(b'abc', 4), b'abc\0') self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') @@ -111,8 +114,8 @@ def test_asstring(self): def test_asstringandsize(self): """Test PyBytes_AsStringAndSize()""" - asstringandsize = _testcapi.bytes_asstringandsize - asstringandsize_null = _testcapi.bytes_asstringandsize_null + asstringandsize = _testlimitedcapi.bytes_asstringandsize + asstringandsize_null = _testlimitedcapi.bytes_asstringandsize_null self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) @@ -128,7 +131,7 @@ def test_asstringandsize(self): def test_repr(self): # Test PyBytes_Repr() - bytes_repr = _testcapi.bytes_repr + bytes_repr = _testlimitedcapi.bytes_repr self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") @@ -149,7 +152,7 @@ def test_repr(self): def test_concat(self, concat=None): """Test PyBytes_Concat()""" if concat is None: - concat = _testcapi.bytes_concat + concat = _testlimitedcapi.bytes_concat self.assertEqual(concat(b'abc', b'def'), b'abcdef') self.assertEqual(concat(b'a\0b', b'c\0d'), b'a\0bc\0d') @@ -182,11 +185,11 @@ def test_concat(self, concat=None): def test_concatanddel(self): """Test PyBytes_ConcatAndDel()""" - self.test_concat(_testcapi.bytes_concatanddel) + self.test_concat(_testlimitedcapi.bytes_concatanddel) def test_decodeescape(self): """Test PyBytes_DecodeEscape()""" - decodeescape = _testcapi.bytes_decodeescape + decodeescape = _testlimitedcapi.bytes_decodeescape self.assertEqual(decodeescape(b'abc'), b'abc') self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), @@ -217,6 +220,75 @@ def test_decodeescape(self): # CRASHES decodeescape(b'abc', NULL, -1) # CRASHES decodeescape(NULL, NULL, 1) + def test_resize(self): + """Test _PyBytes_Resize()""" + resize = _testcapi.bytes_resize + + for new in True, False: + self.assertEqual(resize(b'abc', 0, new), b'') + self.assertEqual(resize(b'abc', 1, new), b'a') + self.assertEqual(resize(b'abc', 2, new), b'ab') + self.assertEqual(resize(b'abc', 3, new), b'abc') + b = resize(b'abc', 4, new) + self.assertEqual(len(b), 4) + self.assertEqual(b[:3], b'abc') + + self.assertEqual(resize(b'a', 0, new), b'') + self.assertEqual(resize(b'a', 1, new), b'a') + b = resize(b'a', 2, new) + self.assertEqual(len(b), 2) + self.assertEqual(b[:1], b'a') + + self.assertEqual(resize(b'', 0, new), b'') + self.assertEqual(len(resize(b'', 1, new)), 1) + self.assertEqual(len(resize(b'', 2, new)), 2) + + self.assertRaises(SystemError, resize, b'abc', -1, False) + self.assertRaises(SystemError, resize, bytearray(b'abc'), 3, False) + + # CRASHES resize(NULL, 0, False) + # CRASHES resize(NULL, 3, False) + + def test_join(self): + """Test PyBytes_Join()""" + bytes_join = _testcapi.bytes_join + + self.assertEqual(bytes_join(b'', []), b'') + self.assertEqual(bytes_join(b'sep', []), b'') + + self.assertEqual(bytes_join(b'', [b'a', b'b', b'c']), b'abc') + self.assertEqual(bytes_join(b'-', [b'a', b'b', b'c']), b'a-b-c') + self.assertEqual(bytes_join(b' - ', [b'a', b'b', b'c']), b'a - b - c') + self.assertEqual(bytes_join(b'-', [bytearray(b'abc'), + memoryview(b'def')]), + b'abc-def') + + self.assertEqual(bytes_join(b'-', iter([b'a', b'b', b'c'])), b'a-b-c') + + # invalid 'sep' argument + with self.assertRaises(TypeError): + bytes_join(bytearray(b'sep'), []) + with self.assertRaises(TypeError): + bytes_join(memoryview(b'sep'), []) + with self.assertRaises(TypeError): + bytes_join('', []) # empty Unicode string + with self.assertRaises(TypeError): + bytes_join('unicode', []) + with self.assertRaises(TypeError): + bytes_join(123, []) + with self.assertRaises(SystemError): + self.assertEqual(bytes_join(NULL, [b'a', b'b', b'c']), b'abc') + + # invalid 'iterable' argument + with self.assertRaises(TypeError): + bytes_join(b'', [b'bytes', 'unicode']) + with self.assertRaises(TypeError): + bytes_join(b'', [b'bytes', 123]) + with self.assertRaises(TypeError): + bytes_join(b'', 123) + with self.assertRaises(SystemError): + bytes_join(b'', NULL) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 682c56979c6dfa..a557e35e68915d 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -1,13 +1,20 @@ -import unittest +import codecs +import contextlib +import io +import re import sys +import unittest +import unittest.mock as mock +import _testcapi from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None +BAD_ARGUMENT = re.escape('bad argument type for built-in operation') -class CAPITest(unittest.TestCase): +class CAPIUnicodeTest(unittest.TestCase): # TODO: Test the following functions: # # PyUnicode_BuildEncodingMap @@ -27,7 +34,7 @@ class CAPITest(unittest.TestCase): def test_fromencodedobject(self): """Test PyUnicode_FromEncodedObject()""" - fromencodedobject = _testcapi.unicode_fromencodedobject + fromencodedobject = _testlimitedcapi.unicode_fromencodedobject self.assertEqual(fromencodedobject(b'abc', NULL), 'abc') self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc') @@ -52,7 +59,7 @@ def test_fromencodedobject(self): def test_decode(self): """Test PyUnicode_Decode()""" - decode = _testcapi.unicode_decode + decode = _testlimitedcapi.unicode_decode self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]') self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]') @@ -70,7 +77,7 @@ def test_decode(self): def test_asencodedstring(self): """Test PyUnicode_AsEncodedString()""" - asencodedstring = _testcapi.unicode_asencodedstring + asencodedstring = _testlimitedcapi.unicode_asencodedstring self.assertEqual(asencodedstring('abc', NULL), b'abc') self.assertEqual(asencodedstring('abc', 'ascii'), b'abc') @@ -93,7 +100,7 @@ def test_asencodedstring(self): def test_decodeutf8(self): """Test PyUnicode_DecodeUTF8()""" - decodeutf8 = _testcapi.unicode_decodeutf8 + decodeutf8 = _testlimitedcapi.unicode_decodeutf8 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') @@ -113,7 +120,7 @@ def test_decodeutf8(self): def test_decodeutf8stateful(self): """Test PyUnicode_DecodeUTF8Stateful()""" - decodeutf8stateful = _testcapi.unicode_decodeutf8stateful + decodeutf8stateful = _testlimitedcapi.unicode_decodeutf8stateful for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') @@ -136,7 +143,7 @@ def test_decodeutf8stateful(self): def test_asutf8string(self): """Test PyUnicode_AsUTF8String()""" - asutf8string = _testcapi.unicode_asutf8string + asutf8string = _testlimitedcapi.unicode_asutf8string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf8string(s), s.encode('utf-8')) @@ -148,7 +155,7 @@ def test_asutf8string(self): def test_decodeutf16(self): """Test PyUnicode_DecodeUTF16()""" - decodeutf16 = _testcapi.unicode_decodeutf16 + decodeutf16 = _testlimitedcapi.unicode_decodeutf16 naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -192,7 +199,7 @@ def test_decodeutf16(self): def test_decodeutf16stateful(self): """Test PyUnicode_DecodeUTF16Stateful()""" - decodeutf16stateful = _testcapi.unicode_decodeutf16stateful + decodeutf16stateful = _testlimitedcapi.unicode_decodeutf16stateful naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -238,7 +245,7 @@ def test_decodeutf16stateful(self): def test_asutf16string(self): """Test PyUnicode_AsUTF16String()""" - asutf16string = _testcapi.unicode_asutf16string + asutf16string = _testlimitedcapi.unicode_asutf16string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf16string(s), s.encode('utf-16')) @@ -250,7 +257,7 @@ def test_asutf16string(self): def test_decodeutf32(self): """Test PyUnicode_DecodeUTF8()""" - decodeutf32 = _testcapi.unicode_decodeutf32 + decodeutf32 = _testlimitedcapi.unicode_decodeutf32 naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -290,7 +297,7 @@ def test_decodeutf32(self): def test_decodeutf32stateful(self): """Test PyUnicode_DecodeUTF32Stateful()""" - decodeutf32stateful = _testcapi.unicode_decodeutf32stateful + decodeutf32stateful = _testlimitedcapi.unicode_decodeutf32stateful naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -342,7 +349,7 @@ def test_decodeutf32stateful(self): def test_asutf32string(self): """Test PyUnicode_AsUTF32String()""" - asutf32string = _testcapi.unicode_asutf32string + asutf32string = _testlimitedcapi.unicode_asutf32string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf32string(s), s.encode('utf-32')) @@ -354,7 +361,7 @@ def test_asutf32string(self): def test_decodelatin1(self): """Test PyUnicode_DecodeLatin1()""" - decodelatin1 = _testcapi.unicode_decodelatin1 + decodelatin1 = _testlimitedcapi.unicode_decodelatin1 self.assertEqual(decodelatin1(b'abc'), 'abc') self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc') @@ -365,7 +372,7 @@ def test_decodelatin1(self): def test_aslatin1string(self): """Test PyUnicode_AsLatin1String()""" - aslatin1string = _testcapi.unicode_aslatin1string + aslatin1string = _testlimitedcapi.unicode_aslatin1string self.assertEqual(aslatin1string('abc'), b'abc') self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2') @@ -377,7 +384,7 @@ def test_aslatin1string(self): def test_decodeascii(self): """Test PyUnicode_DecodeASCII()""" - decodeascii = _testcapi.unicode_decodeascii + decodeascii = _testlimitedcapi.unicode_decodeascii self.assertEqual(decodeascii(b'abc'), 'abc') self.assertEqual(decodeascii(b'abc', 'strict'), 'abc') @@ -392,7 +399,7 @@ def test_decodeascii(self): def test_asasciistring(self): """Test PyUnicode_AsASCIIString()""" - asasciistring = _testcapi.unicode_asasciistring + asasciistring = _testlimitedcapi.unicode_asasciistring self.assertEqual(asasciistring('abc'), b'abc') @@ -403,7 +410,7 @@ def test_asasciistring(self): def test_decodecharmap(self): """Test PyUnicode_DecodeCharmap()""" - decodecharmap = _testcapi.unicode_decodecharmap + decodecharmap = _testlimitedcapi.unicode_decodecharmap self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac') self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac') @@ -426,7 +433,7 @@ def test_decodecharmap(self): def test_ascharmapstring(self): """Test PyUnicode_AsCharmapString()""" - ascharmapstring = _testcapi.unicode_ascharmapstring + ascharmapstring = _testlimitedcapi.unicode_ascharmapstring self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7') self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7') @@ -443,7 +450,7 @@ def test_ascharmapstring(self): def test_decodeunicodeescape(self): """Test PyUnicode_DecodeUnicodeEscape()""" - decodeunicodeescape = _testcapi.unicode_decodeunicodeescape + decodeunicodeescape = _testlimitedcapi.unicode_decodeunicodeescape self.assertEqual(decodeunicodeescape(b'abc'), 'abc') self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\') @@ -467,7 +474,7 @@ def test_decodeunicodeescape(self): def test_asunicodeescapestring(self): """Test PyUnicode_AsUnicodeEscapeString()""" - asunicodeescapestring = _testcapi.unicode_asunicodeescapestring + asunicodeescapestring = _testlimitedcapi.unicode_asunicodeescapestring self.assertEqual(asunicodeescapestring('abc'), b'abc') self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\') @@ -481,7 +488,7 @@ def test_asunicodeescapestring(self): def test_decoderawunicodeescape(self): """Test PyUnicode_DecodeRawUnicodeEscape()""" - decoderawunicodeescape = _testcapi.unicode_decoderawunicodeescape + decoderawunicodeescape = _testlimitedcapi.unicode_decoderawunicodeescape self.assertEqual(decoderawunicodeescape(b'abc'), 'abc') self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\') @@ -503,7 +510,7 @@ def test_decoderawunicodeescape(self): def test_asrawunicodeescapestring(self): """Test PyUnicode_AsRawUnicodeEscapeString()""" - asrawunicodeescapestring = _testcapi.unicode_asrawunicodeescapestring + asrawunicodeescapestring = _testlimitedcapi.unicode_asrawunicodeescapestring self.assertEqual(asrawunicodeescapestring('abc'), b'abc') self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\') @@ -516,5 +523,383 @@ def test_asrawunicodeescapestring(self): # CRASHES asrawunicodeescapestring(NULL) +class CAPICodecs(unittest.TestCase): + + def setUp(self): + # Encoding names are normalized internally by converting them + # to lowercase and their hyphens are replaced by underscores. + self.encoding_name = 'test.test_capi.test_codecs.codec_reversed' + # Make sure that our custom codec is not already registered (that + # way we know whether we correctly unregistered the custom codec + # after a test or not). + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + # create the search function without registering yet + self._create_custom_codec() + + def _create_custom_codec(self): + def codec_encoder(m, errors='strict'): + return (type(m)().join(reversed(m)), len(m)) + + def codec_decoder(c, errors='strict'): + return (type(c)().join(reversed(c)), len(c)) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return codec_encoder(input) + + class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return codec_decoder(input) + + class StreamReader(codecs.StreamReader): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + info = codecs.CodecInfo( + encode=codec_encoder, + decode=codec_decoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + name=self.encoding_name + ) + + def search_function(encoding): + if encoding == self.encoding_name: + return info + return None + + self.codec_info = info + self.search_function = search_function + + @contextlib.contextmanager + def use_custom_encoder(self): + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + codecs.register(self.search_function) + yield + codecs.unregister(self.search_function) + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + + def test_codec_register(self): + search_function, encoding = self.search_function, self.encoding_name + # register the search function using the C API + self.assertIsNone(_testcapi.codec_register(search_function)) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIs(codecs.lookup(encoding), search_function(encoding)) + self.assertEqual(codecs.encode('123', encoding=encoding), '321') + # unregister the search function using the regular API + codecs.unregister(search_function) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_unregister(self): + search_function, encoding = self.search_function, self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding) + # register the search function using the regular API + codecs.register(search_function) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIsNotNone(codecs.lookup(encoding)) + # unregister the search function using the C API + self.assertIsNone(_testcapi.codec_unregister(search_function)) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_known_encoding(self): + self.assertRaises(LookupError, codecs.lookup, 'unknown-codec') + self.assertFalse(_testcapi.codec_known_encoding('unknown-codec')) + self.assertFalse(_testcapi.codec_known_encoding('unknown_codec')) + self.assertFalse(_testcapi.codec_known_encoding('UNKNOWN-codec')) + + encoding_name = self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding_name) + + codecs.register(self.search_function) + self.addCleanup(codecs.unregister, self.search_function) + + for name in [ + encoding_name, + encoding_name.upper(), + encoding_name.replace('_', '-'), + ]: + with self.subTest(name): + self.assertTrue(_testcapi.codec_known_encoding(name)) + + def test_codec_encode(self): + encode = _testcapi.codec_encode + self.assertEqual(encode('a', 'utf-8', NULL), b'a') + self.assertEqual(encode('a', 'utf-8', 'strict'), b'a') + self.assertEqual(encode('[é]', 'ascii', 'ignore'), b'[]') + + self.assertRaises(TypeError, encode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + encode('a', NULL, 'strict') + + def test_codec_decode(self): + decode = _testcapi.codec_decode + + s = 'a\xa1\u4f60\U0001f600' + b = s.encode() + + self.assertEqual(decode(b, 'utf-8', 'strict'), s) + self.assertEqual(decode(b, 'utf-8', NULL), s) + self.assertEqual(decode(b, 'latin1', 'strict'), b.decode('latin1')) + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', 'strict') + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', NULL) + self.assertEqual(decode(b, 'ascii', 'replace'), 'a' + '\ufffd'*9) + + # _codecs.decode() only reports an unknown error handling name when + # the corresponding error handling function is used; this difers + # from PyUnicode_Decode() which checks that both the encoding and + # the error handling name are recognized before even attempting to + # call the decoder. + self.assertEqual(decode(b'', 'utf-8', 'unknown-error-handler'), '') + self.assertEqual(decode(b'a', 'utf-8', 'unknown-error-handler'), 'a') + + self.assertRaises(TypeError, decode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + decode(b, NULL, 'strict') + + def test_codec_encoder(self): + codec_encoder = _testcapi.codec_encoder + + with self.use_custom_encoder(): + encoder = codec_encoder(self.encoding_name) + self.assertIs(encoder, self.codec_info.encode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_encoder(NULL) + + def test_codec_decoder(self): + codec_decoder = _testcapi.codec_decoder + + with self.use_custom_encoder(): + decoder = codec_decoder(self.encoding_name) + self.assertIs(decoder, self.codec_info.decode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_decoder(NULL) + + def test_codec_incremental_encoder(self): + codec_incremental_encoder = _testcapi.codec_incremental_encoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + encoder = codec_incremental_encoder(encoding, errors) + self.assertIsInstance(encoder, self.codec_info.incrementalencoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_encoder(NULL, 'strict') + + def test_codec_incremental_decoder(self): + codec_incremental_decoder = _testcapi.codec_incremental_decoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + decoder = codec_incremental_decoder(encoding, errors) + self.assertIsInstance(decoder, self.codec_info.incrementaldecoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_decoder(NULL, 'strict') + + def test_codec_stream_reader(self): + codec_stream_reader = _testcapi.codec_stream_reader + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_reader(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamreader) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_reader(NULL, stream, 'strict') + + def test_codec_stream_writer(self): + codec_stream_writer = _testcapi.codec_stream_writer + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_writer(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamwriter) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_writer(NULL, stream, 'strict') + + +class CAPICodecErrors(unittest.TestCase): + + @classmethod + def _generate_exception_args(cls): + for objlen in range(5): + maxind = 2 * max(2, objlen) + for start in range(-maxind, maxind + 1): + for end in range(-maxind, maxind + 1): + yield objlen, start, end + + @classmethod + def generate_encode_errors(cls): + return tuple( + UnicodeEncodeError('utf-8', '0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_decode_errors(cls): + return tuple( + UnicodeDecodeError('utf-8', b'0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_translate_errors(cls): + return tuple( + UnicodeTranslateError('0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def setUpClass(cls): + cls.unicode_encode_errors = cls.generate_encode_errors() + cls.unicode_decode_errors = cls.generate_decode_errors() + cls.unicode_translate_errors = cls.generate_translate_errors() + cls.all_unicode_errors = ( + cls.unicode_encode_errors + + cls.unicode_decode_errors + + cls.unicode_translate_errors + ) + cls.bad_unicode_errors = ( + ValueError(), + ) + + def test_codec_register_error(self): + # for cleaning up between tests + from _codecs import _unregister_error as _codecs_unregister_error + + self.assertRaises(LookupError, _testcapi.codec_lookup_error, 'custom') + + def custom_error_handler(exc): + raise exc + + error_handler = mock.Mock(wraps=custom_error_handler) + _testcapi.codec_register_error('custom', error_handler) + self.addCleanup(_codecs_unregister_error, 'custom') + + self.assertRaises(UnicodeEncodeError, codecs.encode, + '\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + error_handler.reset_mock() + + self.assertRaises(UnicodeDecodeError, codecs.decode, + b'\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + + # _codecs._unregister_error directly delegates to the internal C + # function so a Python-level function test is sufficient (it is + # tested in test_codeccallbacks). + + def test_codec_lookup_error(self): + codec_lookup_error = _testcapi.codec_lookup_error + self.assertIs(codec_lookup_error(NULL), codecs.strict_errors) + self.assertIs(codec_lookup_error('strict'), codecs.strict_errors) + self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors) + self.assertIs(codec_lookup_error('replace'), codecs.replace_errors) + self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors) + self.assertIs(codec_lookup_error('backslashreplace'), codecs.backslashreplace_errors) + self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors) + self.assertRaises(LookupError, codec_lookup_error, 'unknown') + + def test_codec_strict_errors_handler(self): + handler = _testcapi.codec_strict_errors + for exc in self.all_unicode_errors + self.bad_unicode_errors: + with self.subTest(handler=handler, exc=exc): + self.assertRaises(type(exc), handler, exc) + + def test_codec_ignore_errors_handler(self): + handler = _testcapi.codec_ignore_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_replace_errors_handler(self): + handler = _testcapi.codec_replace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_xmlcharrefreplace_errors_handler(self): + handler = _testcapi.codec_xmlcharrefreplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def test_codec_backslashreplace_errors_handler(self): + handler = _testcapi.codec_backslashreplace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_namereplace_errors_handler(self): + handler = _testlimitedcapi.codec_namereplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def do_test_codec_errors_handler(self, handler, exceptions): + at_least_one = False + for exc in exceptions: + # See https://github.com/python/cpython/issues/123378 and related + # discussion and issues for details. + if self._exception_may_crash(exc): + continue + + at_least_one = True + with self.subTest(handler=handler, exc=exc): + # test that the handler does not crash + self.assertIsInstance(handler(exc), tuple) + + if exceptions: + self.assertTrue(at_least_one, "all exceptions are crashing") + + for bad_exc in ( + self.bad_unicode_errors + + tuple(e for e in self.all_unicode_errors if e not in exceptions) + ): + with self.subTest('bad type', handler=handler, exc=bad_exc): + self.assertRaises(TypeError, handler, bad_exc) + + @classmethod + def _exception_may_crash(cls, exc): + """Indicate whether a Unicode exception might currently crash + the interpreter when used by a built-in codecs error handler. + + Until gh-123378 is fixed, we skip the tests for these exceptions. + + This should only be used by "do_test_codec_errors_handler". + """ + message, start, end = exc.object, exc.start, exc.end + match exc: + case UnicodeEncodeError(): + return end < start or (end - start) >= len(message) + case UnicodeDecodeError(): + # The case "end - start >= len(message)" does not crash. + return end < start + case UnicodeTranslateError(): + # Test "end <= start" because PyCodec_ReplaceErrors checks + # the Unicode kind of a 0-length string which by convention + # is PyUnicode_1BYTE_KIND and not PyUnicode_2BYTE_KIND as + # the handler currently expects. + return end <= start or (end - start) >= len(message) + return False + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py index a5b59558e7f851..368edfbf2ce97e 100644 --- a/Lib/test/test_capi/test_complex.py +++ b/Lib/test/test_capi/test_complex.py @@ -10,6 +10,7 @@ _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None INF = float("inf") @@ -25,7 +26,7 @@ def __complex__(self): class CAPIComplexTest(unittest.TestCase): def test_check(self): # Test PyComplex_Check() - check = _testcapi.complex_check + check = _testlimitedcapi.complex_check self.assertTrue(check(1+2j)) self.assertTrue(check(ComplexSubclass(1+2j))) @@ -38,7 +39,7 @@ def test_check(self): def test_checkexact(self): # PyComplex_CheckExact() - checkexact = _testcapi.complex_checkexact + checkexact = _testlimitedcapi.complex_checkexact self.assertTrue(checkexact(1+2j)) self.assertFalse(checkexact(ComplexSubclass(1+2j))) @@ -57,13 +58,13 @@ def test_fromccomplex(self): def test_fromdoubles(self): # Test PyComplex_FromDoubles() - fromdoubles = _testcapi.complex_fromdoubles + fromdoubles = _testlimitedcapi.complex_fromdoubles self.assertEqual(fromdoubles(1.0, 2.0), 1.0+2.0j) def test_realasdouble(self): # Test PyComplex_RealAsDouble() - realasdouble = _testcapi.complex_realasdouble + realasdouble = _testlimitedcapi.complex_realasdouble self.assertEqual(realasdouble(1+2j), 1.0) self.assertEqual(realasdouble(-1+0j), -1.0) @@ -98,7 +99,7 @@ def test_realasdouble(self): def test_imagasdouble(self): # Test PyComplex_ImagAsDouble() - imagasdouble = _testcapi.complex_imagasdouble + imagasdouble = _testlimitedcapi.complex_imagasdouble self.assertEqual(imagasdouble(1+2j), 2.0) self.assertEqual(imagasdouble(1-1j), -1.0) @@ -225,7 +226,11 @@ def test_py_c_pow(self): self.assertEqual(_py_c_pow(0j, -1)[1], errno.EDOM) self.assertEqual(_py_c_pow(0j, 1j)[1], errno.EDOM) - self.assertEqual(_py_c_pow(*[DBL_MAX+1j]*2)[0], complex(*[INF]*2)) + max_num = DBL_MAX+1j + self.assertEqual(_py_c_pow(max_num, max_num), + (complex(INF, INF), errno.ERANGE)) + self.assertEqual(_py_c_pow(max_num, 2), + (complex(INF, INF), errno.ERANGE)) def test_py_c_abs(self): diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py new file mode 100644 index 00000000000000..71fb9ae45c7c30 --- /dev/null +++ b/Lib/test/test_capi/test_config.py @@ -0,0 +1,379 @@ +""" +Tests PyConfig_Get() and PyConfig_Set() C API (PEP 741). +""" +import os +import sys +import sysconfig +import types +import unittest +from test import support +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +# Is the Py_STATS macro defined? +Py_STATS = hasattr(sys, '_stats_on') + + +class CAPITests(unittest.TestCase): + def test_config_get(self): + # Test PyConfig_Get() + config_get = _testcapi.config_get + config_names = _testcapi.config_names + + TEST_VALUE = { + str: "TEST_MARKER_STR", + str | None: "TEST_MARKER_OPT_STR", + list[str]: ("TEST_MARKER_STR_TUPLE",), + dict[str, str | bool]: {"x": "value", "y": True}, + } + + # read config options and check their type + options = [ + ("allocator", int, None), + ("argv", list[str], "argv"), + ("base_exec_prefix", str | None, "base_exec_prefix"), + ("base_executable", str | None, "_base_executable"), + ("base_prefix", str | None, "base_prefix"), + ("buffered_stdio", bool, None), + ("bytes_warning", int, None), + ("check_hash_pycs_mode", str, None), + ("code_debug_ranges", bool, None), + ("configure_c_stdio", bool, None), + ("coerce_c_locale", bool, None), + ("coerce_c_locale_warn", bool, None), + ("configure_locale", bool, None), + ("cpu_count", int, None), + ("dev_mode", bool, None), + ("dump_refs", bool, None), + ("dump_refs_file", str | None, None), + ("exec_prefix", str | None, "exec_prefix"), + ("executable", str | None, "executable"), + ("faulthandler", bool, None), + ("filesystem_encoding", str, None), + ("filesystem_errors", str, None), + ("hash_seed", int, None), + ("home", str | None, None), + ("import_time", bool, None), + ("inspect", bool, None), + ("install_signal_handlers", bool, None), + ("int_max_str_digits", int, None), + ("interactive", bool, None), + ("isolated", bool, None), + ("malloc_stats", bool, None), + ("module_search_paths", list[str], "path"), + ("optimization_level", int, None), + ("orig_argv", list[str], "orig_argv"), + ("parser_debug", bool, None), + ("parse_argv", bool, None), + ("pathconfig_warnings", bool, None), + ("perf_profiling", int, None), + ("platlibdir", str, "platlibdir"), + ("prefix", str | None, "prefix"), + ("program_name", str, None), + ("pycache_prefix", str | None, "pycache_prefix"), + ("quiet", bool, None), + ("run_command", str | None, None), + ("run_filename", str | None, None), + ("run_module", str | None, None), + ("safe_path", bool, None), + ("show_ref_count", bool, None), + ("site_import", bool, None), + ("skip_source_first_line", bool, None), + ("stdio_encoding", str, None), + ("stdio_errors", str, None), + ("stdlib_dir", str | None, "_stdlib_dir"), + ("tracemalloc", int, None), + ("use_environment", bool, None), + ("use_frozen_modules", bool, None), + ("use_hash_seed", bool, None), + ("user_site_directory", bool, None), + ("utf8_mode", bool, None), + ("verbose", int, None), + ("warn_default_encoding", bool, None), + ("warnoptions", list[str], "warnoptions"), + ("write_bytecode", bool, None), + ("xoptions", dict[str, str | bool], "_xoptions"), + ] + if support.Py_DEBUG: + options.append(("run_presite", str | None, None)) + if sysconfig.get_config_var('Py_GIL_DISABLED'): + options.append(("enable_gil", int, None)) + if support.MS_WINDOWS: + options.extend(( + ("legacy_windows_stdio", bool, None), + ("legacy_windows_fs_encoding", bool, None), + )) + if Py_STATS: + options.extend(( + ("_pystats", bool, None), + )) + + for name, option_type, sys_attr in options: + with self.subTest(name=name, option_type=option_type, + sys_attr=sys_attr): + value = config_get(name) + if isinstance(option_type, types.GenericAlias): + self.assertIsInstance(value, option_type.__origin__) + if option_type.__origin__ == dict: + key_type = option_type.__args__[0] + value_type = option_type.__args__[1] + for item in value.items(): + self.assertIsInstance(item[0], key_type) + self.assertIsInstance(item[1], value_type) + else: + item_type = option_type.__args__[0] + for item in value: + self.assertIsInstance(item, item_type) + else: + self.assertIsInstance(value, option_type) + + if sys_attr is not None: + expected = getattr(sys, sys_attr) + self.assertEqual(expected, value) + + override = TEST_VALUE[option_type] + with support.swap_attr(sys, sys_attr, override): + self.assertEqual(config_get(name), override) + + # check that the test checks all options + self.assertEqual(sorted(name for name, option_type, sys_attr in options), + sorted(config_names())) + + def test_config_get_sys_flags(self): + # Test PyConfig_Get() + config_get = _testcapi.config_get + + # compare config options with sys.flags + for flag, name, negate in ( + ("debug", "parser_debug", False), + ("inspect", "inspect", False), + ("interactive", "interactive", False), + ("optimize", "optimization_level", False), + ("dont_write_bytecode", "write_bytecode", True), + ("no_user_site", "user_site_directory", True), + ("no_site", "site_import", True), + ("ignore_environment", "use_environment", True), + ("verbose", "verbose", False), + ("bytes_warning", "bytes_warning", False), + ("quiet", "quiet", False), + # "hash_randomization" is tested below + ("isolated", "isolated", False), + ("dev_mode", "dev_mode", False), + ("utf8_mode", "utf8_mode", False), + ("warn_default_encoding", "warn_default_encoding", False), + ("safe_path", "safe_path", False), + ("int_max_str_digits", "int_max_str_digits", False), + # "gil" is tested below + ): + with self.subTest(flag=flag, name=name, negate=negate): + value = config_get(name) + if negate: + value = not value + self.assertEqual(getattr(sys.flags, flag), value) + + self.assertEqual(sys.flags.hash_randomization, + config_get('use_hash_seed') == 0 + or config_get('hash_seed') != 0) + + if sysconfig.get_config_var('Py_GIL_DISABLED'): + value = config_get('enable_gil') + expected = (value if value != -1 else None) + self.assertEqual(sys.flags.gil, expected) + + def test_config_get_non_existent(self): + # Test PyConfig_Get() on non-existent option name + config_get = _testcapi.config_get + nonexistent_key = 'NONEXISTENT_KEY' + err_msg = f'unknown config option name: {nonexistent_key}' + with self.assertRaisesRegex(ValueError, err_msg): + config_get(nonexistent_key) + + def test_config_get_write_bytecode(self): + # PyConfig_Get("write_bytecode") gets sys.dont_write_bytecode + # as an integer + config_get = _testcapi.config_get + with support.swap_attr(sys, "dont_write_bytecode", 0): + self.assertEqual(config_get('write_bytecode'), 1) + with support.swap_attr(sys, "dont_write_bytecode", "yes"): + self.assertEqual(config_get('write_bytecode'), 0) + with support.swap_attr(sys, "dont_write_bytecode", []): + self.assertEqual(config_get('write_bytecode'), 1) + + def test_config_getint(self): + # Test PyConfig_GetInt() + config_getint = _testcapi.config_getint + + # PyConfig_MEMBER_INT type + self.assertEqual(config_getint('verbose'), sys.flags.verbose) + + # PyConfig_MEMBER_UINT type + self.assertEqual(config_getint('isolated'), sys.flags.isolated) + + # PyConfig_MEMBER_ULONG type + self.assertIsInstance(config_getint('hash_seed'), int) + + # PyPreConfig member + self.assertIsInstance(config_getint('allocator'), int) + + # platlibdir type is str + with self.assertRaises(TypeError): + config_getint('platlibdir') + + def test_get_config_names(self): + names = _testcapi.config_names() + self.assertIsInstance(names, frozenset) + for name in names: + self.assertIsInstance(name, str) + + def test_config_set_sys_attr(self): + # Test PyConfig_Set() with sys attributes + config_get = _testcapi.config_get + config_set = _testcapi.config_set + + # mutable configuration option mapped to sys attributes + for name, sys_attr, option_type in ( + ('argv', 'argv', list[str]), + ('base_exec_prefix', 'base_exec_prefix', str | None), + ('base_executable', '_base_executable', str | None), + ('base_prefix', 'base_prefix', str | None), + ('exec_prefix', 'exec_prefix', str | None), + ('executable', 'executable', str | None), + ('module_search_paths', 'path', list[str]), + ('platlibdir', 'platlibdir', str), + ('prefix', 'prefix', str | None), + ('pycache_prefix', 'pycache_prefix', str | None), + ('stdlib_dir', '_stdlib_dir', str | None), + ('warnoptions', 'warnoptions', list[str]), + ('xoptions', '_xoptions', dict[str, str | bool]), + ): + with self.subTest(name=name): + if option_type == str: + test_values = ('TEST_REPLACE',) + invalid_types = (1, None) + elif option_type == str | None: + test_values = ('TEST_REPLACE', None) + invalid_types = (123,) + elif option_type == list[str]: + test_values = (['TEST_REPLACE'], []) + invalid_types = ('text', 123, [123]) + else: # option_type == dict[str, str | bool]: + test_values = ({"x": "value", "y": True},) + invalid_types = ('text', 123, ['option'], + {123: 'value'}, + {'key': b'bytes'}) + + old_opt_value = config_get(name) + old_sys_value = getattr(sys, sys_attr) + try: + for value in test_values: + config_set(name, value) + self.assertEqual(config_get(name), value) + self.assertEqual(getattr(sys, sys_attr), value) + + for value in invalid_types: + with self.assertRaises(TypeError): + config_set(name, value) + finally: + setattr(sys, sys_attr, old_sys_value) + config_set(name, old_opt_value) + + def test_config_set_sys_flag(self): + # Test PyConfig_Set() with sys.flags + config_get = _testcapi.config_get + config_set = _testcapi.config_set + + # mutable configuration option mapped to sys.flags + class unsigned_int(int): + pass + + def expect_int(value): + value = int(value) + return (value, value) + + def expect_bool(value): + value = int(bool(value)) + return (value, value) + + def expect_bool_not(value): + value = bool(value) + return (int(value), int(not value)) + + for name, sys_flag, option_type, expect_func in ( + # (some flags cannot be set, see comments below.) + ('parser_debug', 'debug', bool, expect_bool), + ('inspect', 'inspect', bool, expect_bool), + ('interactive', 'interactive', bool, expect_bool), + ('optimization_level', 'optimize', unsigned_int, expect_int), + ('write_bytecode', 'dont_write_bytecode', bool, expect_bool_not), + # user_site_directory + # site_import + ('use_environment', 'ignore_environment', bool, expect_bool_not), + ('verbose', 'verbose', unsigned_int, expect_int), + ('bytes_warning', 'bytes_warning', unsigned_int, expect_int), + ('quiet', 'quiet', bool, expect_bool), + # hash_randomization + # isolated + # dev_mode + # utf8_mode + # warn_default_encoding + # safe_path + ('int_max_str_digits', 'int_max_str_digits', unsigned_int, expect_int), + # gil + ): + if name == "int_max_str_digits": + new_values = (0, 5_000, 999_999) + invalid_values = (-1, 40) # value must 0 or >= 4300 + invalid_types = (1.0, "abc") + elif option_type == int: + new_values = (False, True, 0, 1, 5, -5) + invalid_values = () + invalid_types = (1.0, "abc") + else: + new_values = (False, True, 0, 1, 5) + invalid_values = (-5,) + invalid_types = (1.0, "abc") + + with self.subTest(name=name): + old_value = config_get(name) + try: + for value in new_values: + expected, expect_flag = expect_func(value) + + config_set(name, value) + self.assertEqual(config_get(name), expected) + self.assertEqual(getattr(sys.flags, sys_flag), expect_flag) + if name == "write_bytecode": + self.assertEqual(getattr(sys, "dont_write_bytecode"), + expect_flag) + if name == "int_max_str_digits": + self.assertEqual(sys.get_int_max_str_digits(), + expect_flag) + + for value in invalid_values: + with self.assertRaises(ValueError): + config_set(name, value) + + for value in invalid_types: + with self.assertRaises(TypeError): + config_set(name, value) + finally: + config_set(name, old_value) + + def test_config_set_read_only(self): + # Test PyConfig_Set() on read-only options + config_set = _testcapi.config_set + for name, value in ( + ("allocator", 0), # PyPreConfig member + ("cpu_count", 8), + ("dev_mode", True), + ("filesystem_encoding", "utf-8"), + ): + with self.subTest(name=name, value=value): + with self.assertRaisesRegex(ValueError, r"read-only"): + config_set(name, value) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index cca6145bc90c04..e726e3d813d888 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -2,7 +2,11 @@ from collections import OrderedDict, UserDict from types import MappingProxyType from test import support -import _testcapi +from test.support import import_helper + + +_testcapi = import_helper.import_module("_testcapi") +_testlimitedcapi = import_helper.import_module("_testlimitedcapi") NULL = None @@ -25,7 +29,7 @@ def gen(): class CAPITest(unittest.TestCase): def test_dict_check(self): - check = _testcapi.dict_check + check = _testlimitedcapi.dict_check self.assertTrue(check({1: 2})) self.assertTrue(check(OrderedDict({1: 2}))) self.assertFalse(check(UserDict({1: 2}))) @@ -34,7 +38,7 @@ def test_dict_check(self): # CRASHES check(NULL) def test_dict_checkexact(self): - check = _testcapi.dict_checkexact + check = _testlimitedcapi.dict_checkexact self.assertTrue(check({1: 2})) self.assertFalse(check(OrderedDict({1: 2}))) self.assertFalse(check(UserDict({1: 2}))) @@ -43,7 +47,7 @@ def test_dict_checkexact(self): # CRASHES check(NULL) def test_dict_new(self): - dict_new = _testcapi.dict_new + dict_new = _testlimitedcapi.dict_new dct = dict_new() self.assertEqual(dct, {}) self.assertIs(type(dct), dict) @@ -51,7 +55,7 @@ def test_dict_new(self): self.assertIsNot(dct2, dct) def test_dictproxy_new(self): - dictproxy_new = _testcapi.dictproxy_new + dictproxy_new = _testlimitedcapi.dictproxy_new for dct in {1: 2}, OrderedDict({1: 2}), UserDict({1: 2}): proxy = dictproxy_new(dct) self.assertIs(type(proxy), MappingProxyType) @@ -67,7 +71,7 @@ def test_dictproxy_new(self): # CRASHES dictproxy_new(NULL) def test_dict_copy(self): - copy = _testcapi.dict_copy + copy = _testlimitedcapi.dict_copy for dct in {1: 2}, OrderedDict({1: 2}): dct_copy = copy(dct) self.assertIs(type(dct_copy), dict) @@ -79,7 +83,7 @@ def test_dict_copy(self): self.assertRaises(SystemError, copy, NULL) def test_dict_clear(self): - clear = _testcapi.dict_clear + clear = _testlimitedcapi.dict_clear dct = {1: 2} clear(dct) self.assertEqual(dct, {}) @@ -98,7 +102,7 @@ def test_dict_clear(self): # CRASHES? clear(NULL) def test_dict_size(self): - size = _testcapi.dict_size + size = _testlimitedcapi.dict_size self.assertEqual(size({1: 2}), 1) self.assertEqual(size(OrderedDict({1: 2})), 1) @@ -109,7 +113,7 @@ def test_dict_size(self): self.assertRaises(SystemError, size, NULL) def test_dict_getitem(self): - getitem = _testcapi.dict_getitem + getitem = _testlimitedcapi.dict_getitem dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertIs(getitem(dct, 'b'), KeyError) @@ -131,7 +135,7 @@ def test_dict_getitem(self): # CRASHES getitem(NULL, 'a') def test_dict_getitemstring(self): - getitemstring = _testcapi.dict_getitemstring + getitemstring = _testlimitedcapi.dict_getitemstring dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitemstring(dct, b'a'), 1) self.assertIs(getitemstring(dct, b'b'), KeyError) @@ -188,7 +192,7 @@ def test_dict_getitemstringref(self): # CRASHES getitemstring(NULL, b'a') def test_dict_getitemwitherror(self): - getitem = _testcapi.dict_getitemwitherror + getitem = _testlimitedcapi.dict_getitemwitherror dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertIs(getitem(dct, 'b'), KeyError) @@ -206,7 +210,7 @@ def test_dict_getitemwitherror(self): # CRASHES getitem(NULL, 'a') def test_dict_contains(self): - contains = _testcapi.dict_contains + contains = _testlimitedcapi.dict_contains dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(contains(dct, 'a')) self.assertFalse(contains(dct, 'b')) @@ -238,7 +242,7 @@ def test_dict_contains_string(self): # CRASHES contains(NULL, b'a') def test_dict_setitem(self): - setitem = _testcapi.dict_setitem + setitem = _testlimitedcapi.dict_setitem dct = {} setitem(dct, 'a', 5) self.assertEqual(dct, {'a': 5}) @@ -258,7 +262,7 @@ def test_dict_setitem(self): # CRASHES setitem(NULL, 'a', 5) def test_dict_setitemstring(self): - setitemstring = _testcapi.dict_setitemstring + setitemstring = _testlimitedcapi.dict_setitemstring dct = {} setitemstring(dct, b'a', 5) self.assertEqual(dct, {'a': 5}) @@ -277,7 +281,7 @@ def test_dict_setitemstring(self): # CRASHES setitemstring(NULL, b'a', 5) def test_dict_delitem(self): - delitem = _testcapi.dict_delitem + delitem = _testlimitedcapi.dict_delitem dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitem(dct, 'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -298,7 +302,7 @@ def test_dict_delitem(self): # CRASHES delitem(NULL, 'a') def test_dict_delitemstring(self): - delitemstring = _testcapi.dict_delitemstring + delitemstring = _testlimitedcapi.dict_delitemstring dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitemstring(dct, b'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -371,21 +375,21 @@ def items(self): return None dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} for mapping in [dict_obj, DictSubclass(dict_obj), BadMapping(dict_obj)]: - self.assertListEqual(_testcapi.dict_keys(mapping), + self.assertListEqual(_testlimitedcapi.dict_keys(mapping), list(dict_obj.keys())) - self.assertListEqual(_testcapi.dict_values(mapping), + self.assertListEqual(_testlimitedcapi.dict_values(mapping), list(dict_obj.values())) - self.assertListEqual(_testcapi.dict_items(mapping), + self.assertListEqual(_testlimitedcapi.dict_items(mapping), list(dict_obj.items())) def test_dict_keys_valuesitems_bad_arg(self): for mapping in UserDict(), [], object(): - self.assertRaises(SystemError, _testcapi.dict_keys, mapping) - self.assertRaises(SystemError, _testcapi.dict_values, mapping) - self.assertRaises(SystemError, _testcapi.dict_items, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_keys, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_values, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_items, mapping) def test_dict_next(self): - dict_next = _testcapi.dict_next + dict_next = _testlimitedcapi.dict_next self.assertIsNone(dict_next({}, 0)) dct = {'a': 1, 'b': 2, 'c': 3} pos = 0 @@ -402,7 +406,7 @@ def test_dict_next(self): # CRASHES dict_next(NULL, 0) def test_dict_update(self): - update = _testcapi.dict_update + update = _testlimitedcapi.dict_update for cls1 in dict, DictSubclass: for cls2 in dict, DictSubclass, UserDict: dct = cls1({'a': 1, 'b': 2}) @@ -417,7 +421,7 @@ def test_dict_update(self): self.assertRaises(SystemError, update, NULL, {}) def test_dict_merge(self): - merge = _testcapi.dict_merge + merge = _testlimitedcapi.dict_merge for cls1 in dict, DictSubclass: for cls2 in dict, DictSubclass, UserDict: dct = cls1({'a': 1, 'b': 2}) @@ -435,7 +439,7 @@ def test_dict_merge(self): self.assertRaises(SystemError, merge, NULL, {}, 0) def test_dict_mergefromseq2(self): - mergefromseq2 = _testcapi.dict_mergefromseq2 + mergefromseq2 = _testlimitedcapi.dict_mergefromseq2 for cls1 in dict, DictSubclass: for cls2 in list, iter: dct = cls1({'a': 1, 'b': 2}) diff --git a/Lib/test/test_capi/test_eval.py b/Lib/test/test_capi/test_eval.py new file mode 100644 index 00000000000000..20ef2695ef3e27 --- /dev/null +++ b/Lib/test/test_capi/test_eval.py @@ -0,0 +1,103 @@ +import sys +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + +class Tests(unittest.TestCase): + def test_eval_get_func_name(self): + eval_get_func_name = _testlimitedcapi.eval_get_func_name + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_name(function_example), + "function_example") + self.assertEqual(eval_get_func_name(A.method_example), + "method_example") + self.assertEqual(eval_get_func_name(A().method_example), + "method_example") + self.assertEqual(eval_get_func_name(sum), "sum") # c function + self.assertEqual(eval_get_func_name(A), "type") + + def test_eval_get_func_desc(self): + eval_get_func_desc = _testlimitedcapi.eval_get_func_desc + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_desc(function_example), + "()") + self.assertEqual(eval_get_func_desc(A.method_example), + "()") + self.assertEqual(eval_get_func_desc(A().method_example), + "()") + self.assertEqual(eval_get_func_desc(sum), "()") # c function + self.assertEqual(eval_get_func_desc(A), " object") + + def test_eval_getlocals(self): + # Test PyEval_GetLocals() + x = 1 + self.assertEqual(_testlimitedcapi.eval_getlocals(), + {'self': self, + 'x': 1}) + + y = 2 + self.assertEqual(_testlimitedcapi.eval_getlocals(), + {'self': self, + 'x': 1, + 'y': 2}) + + def test_eval_getglobals(self): + # Test PyEval_GetGlobals() + self.assertEqual(_testlimitedcapi.eval_getglobals(), + globals()) + + def test_eval_getbuiltins(self): + # Test PyEval_GetBuiltins() + self.assertEqual(_testlimitedcapi.eval_getbuiltins(), + globals()['__builtins__']) + + def test_eval_getframe(self): + # Test PyEval_GetFrame() + self.assertEqual(_testlimitedcapi.eval_getframe(), + sys._getframe()) + + def test_eval_getframe_builtins(self): + # Test PyEval_GetFrameBuiltins() + self.assertEqual(_testlimitedcapi.eval_getframe_builtins(), + sys._getframe().f_builtins) + + def test_eval_getframe_globals(self): + # Test PyEval_GetFrameGlobals() + self.assertEqual(_testlimitedcapi.eval_getframe_globals(), + sys._getframe().f_globals) + + def test_eval_getframe_locals(self): + # Test PyEval_GetFrameLocals() + self.assertEqual(_testlimitedcapi.eval_getframe_locals(), + sys._getframe().f_locals) + + def test_eval_get_recursion_limit(self): + # Test Py_GetRecursionLimit() + self.assertEqual(_testlimitedcapi.eval_get_recursion_limit(), + sys.getrecursionlimit()) + + def test_eval_set_recursion_limit(self): + # Test Py_SetRecursionLimit() + old_limit = sys.getrecursionlimit() + try: + limit = old_limit + 123 + _testlimitedcapi.eval_set_recursion_limit(limit) + self.assertEqual(sys.getrecursionlimit(), limit) + finally: + sys.setrecursionlimit(old_limit) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_eval_code_ex.py b/Lib/test/test_capi/test_eval_code_ex.py index 2d28e5289eff94..b298e5007e5e7d 100644 --- a/Lib/test/test_capi/test_eval_code_ex.py +++ b/Lib/test/test_capi/test_eval_code_ex.py @@ -1,11 +1,16 @@ import unittest +import builtins +from collections import UserDict from test.support import import_helper +from test.support import swap_attr # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +NULL = None + class PyEval_EvalCodeExTests(unittest.TestCase): @@ -13,43 +18,108 @@ def test_simple(self): def f(): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, dict(a=1)), 1) - - # Need to force the compiler to use LOAD_NAME - # def test_custom_locals(self): - # def f(): - # return + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, dict(a=1)), 1) + + self.assertRaises(NameError, eval_code_ex, code, {}) + self.assertRaises(SystemError, eval_code_ex, code, UserDict(a=1)) + self.assertRaises(SystemError, eval_code_ex, code, []) + self.assertRaises(SystemError, eval_code_ex, code, 1) + # CRASHES eval_code_ex(code, NULL) + # CRASHES eval_code_ex(1, {}) + # CRASHES eval_code_ex(NULL, {}) + + def test_custom_locals(self): + # Monkey-patch __build_class__ to get a class code object. + code = None + def build_class(func, name, /, *bases, **kwds): + nonlocal code + code = func.__code__ + + with swap_attr(builtins, '__build_class__', build_class): + class A: + # Uses LOAD_NAME for a + r[:] = [a] + + eval_code_ex = _testcapi.eval_code_ex + results = [] + g = dict(a=1, r=results) + self.assertIsNone(eval_code_ex(code, g)) + self.assertEqual(results, [1]) + self.assertIsNone(eval_code_ex(code, g, dict(a=2))) + self.assertEqual(results, [2]) + self.assertIsNone(eval_code_ex(code, g, UserDict(a=3))) + self.assertEqual(results, [3]) + self.assertIsNone(eval_code_ex(code, g, {})) + self.assertEqual(results, [1]) + self.assertIsNone(eval_code_ex(code, g, NULL)) + self.assertEqual(results, [1]) + + self.assertRaises(TypeError, eval_code_ex, code, g, []) + self.assertRaises(TypeError, eval_code_ex, code, g, 1) + self.assertRaises(NameError, eval_code_ex, code, dict(r=results), {}) + self.assertRaises(NameError, eval_code_ex, code, dict(r=results), NULL) + self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), []) + self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), 1) def test_with_args(self): def f(a, b, c): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (1, 2, 3)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (1, 2, 3)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2, 3, 4)) def test_with_kwargs(self): def f(a, b, c): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), dict(a=1, b=2, c=3)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), dict(a=1, b=2, c=3)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2, c=3, d=4)) def test_with_default(self): def f(a): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (1,)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (1,)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, ()) def test_with_kwarg_default(self): def f(*, a): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (), dict(a=1)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), dict(a=1)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), {}) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), NULL) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), UserDict(a=1)) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), []) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), 1) def test_with_closure(self): a = 1 + b = 2 def f(): + b return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (), {}, f.__closure__), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__), 1) + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__[::-1]), 2) + + # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, ()), 1) + # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, NULL), 1) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 1d158e3586e98d..b22ddd8ad858d4 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -3,11 +3,12 @@ import re import sys import unittest +import textwrap from test import support from test.support import import_helper from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok from test.support.testcase import ExceptionIsLikeMixin from .test_misc import decode_stderr @@ -68,6 +69,47 @@ def test_exc_info(self): else: self.assertTrue(False) + def test_warn_with_stacklevel(self): + code = textwrap.dedent('''\ + import _testcapi + + def foo(): + _testcapi.function_set_warning() + + foo() # line 6 + + + foo() # line 9 + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + self.assertEqual(warnings, [ + b':6: RuntimeWarning: Testing PyErr_WarnEx', + b' foo() # line 6', + b':9: RuntimeWarning: Testing PyErr_WarnEx', + b' foo() # line 9', + ]) + + def test_warn_during_finalization(self): + code = textwrap.dedent('''\ + import _testcapi + + class Foo: + def foo(self): + _testcapi.function_set_warning() + def __del__(self): + self.foo() + + ref = Foo() + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + # Due to the finalization of the interpreter, the source will be omitted + # because the ``warnings`` module cannot be imported at this time + self.assertEqual(warnings, [ + b':7: RuntimeWarning: Testing PyErr_WarnEx', + ]) + class Test_FatalError(unittest.TestCase): diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index cb94d562645916..92c987794142c9 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -9,6 +9,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None @@ -29,7 +30,7 @@ class CAPIFloatTest(unittest.TestCase): def test_check(self): # Test PyFloat_Check() - check = _testcapi.float_check + check = _testlimitedcapi.float_check self.assertTrue(check(4.25)) self.assertTrue(check(FloatSubclass(4.25))) @@ -41,7 +42,7 @@ def test_check(self): def test_checkexact(self): # Test PyFloat_CheckExact() - checkexact = _testcapi.float_checkexact + checkexact = _testlimitedcapi.float_checkexact self.assertTrue(checkexact(4.25)) self.assertFalse(checkexact(FloatSubclass(4.25))) @@ -53,7 +54,7 @@ def test_checkexact(self): def test_fromstring(self): # Test PyFloat_FromString() - fromstring = _testcapi.float_fromstring + fromstring = _testlimitedcapi.float_fromstring self.assertEqual(fromstring("4.25"), 4.25) self.assertEqual(fromstring(b"4.25"), 4.25) @@ -72,13 +73,13 @@ def test_fromstring(self): def test_fromdouble(self): # Test PyFloat_FromDouble() - fromdouble = _testcapi.float_fromdouble + fromdouble = _testlimitedcapi.float_fromdouble self.assertEqual(fromdouble(4.25), 4.25) def test_asdouble(self): # Test PyFloat_AsDouble() - asdouble = _testcapi.float_asdouble + asdouble = _testlimitedcapi.float_asdouble class BadFloat3: def __float__(self): @@ -109,19 +110,19 @@ def __float__(self): def test_getinfo(self): # Test PyFloat_GetInfo() - getinfo = _testcapi.float_getinfo + getinfo = _testlimitedcapi.float_getinfo self.assertEqual(getinfo(), sys.float_info) def test_getmax(self): # Test PyFloat_GetMax() - getmax = _testcapi.float_getmax + getmax = _testlimitedcapi.float_getmax self.assertEqual(getmax(), sys.float_info.max) def test_getmin(self): # Test PyFloat_GetMax() - getmin = _testcapi.float_getmin + getmin = _testlimitedcapi.float_getmin self.assertEqual(getmin(), sys.float_info.min) diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 12039803ba543e..703d228f92e713 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -4,11 +4,18 @@ import sys from test import support from test.support import import_helper +from test.support import script_helper from test.support import warnings_helper +from test.support.testcase import FloatsAreIdenticalMixin # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') from _testcapi import getargs_keywords, getargs_keyword_only +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = NULL + # > How about the following counterproposal. This also changes some of # > the other format codes to be a little more regular. # > @@ -430,11 +437,7 @@ def test_K(self): self.assertEqual(VERY_LARGE & ULLONG_MAX, getargs_K(VERY_LARGE)) -class Float_TestCase(unittest.TestCase): - def assertEqualWithSign(self, actual, expected): - self.assertEqual(actual, expected) - self.assertEqual(math.copysign(1, actual), math.copysign(1, expected)) - +class Float_TestCase(unittest.TestCase, FloatsAreIdenticalMixin): def test_f(self): from _testcapi import getargs_f self.assertEqual(getargs_f(4.25), 4.25) @@ -456,10 +459,10 @@ def test_f(self): self.assertEqual(getargs_f(DBL_MAX), INF) self.assertEqual(getargs_f(-DBL_MAX), -INF) if FLT_MIN > DBL_MIN: - self.assertEqualWithSign(getargs_f(DBL_MIN), 0.0) - self.assertEqualWithSign(getargs_f(-DBL_MIN), -0.0) - self.assertEqualWithSign(getargs_f(0.0), 0.0) - self.assertEqualWithSign(getargs_f(-0.0), -0.0) + self.assertFloatsAreIdentical(getargs_f(DBL_MIN), 0.0) + self.assertFloatsAreIdentical(getargs_f(-DBL_MIN), -0.0) + self.assertFloatsAreIdentical(getargs_f(0.0), 0.0) + self.assertFloatsAreIdentical(getargs_f(-0.0), -0.0) r = getargs_f(NAN) self.assertNotEqual(r, r) @@ -488,8 +491,8 @@ def test_d(self): self.assertEqual(getargs_d(x), x) self.assertRaises(OverflowError, getargs_d, 1< int -> object) - for value in (INT_MIN, INT_MAX, -1, 0, 1, 123): + def check_long_asint(self, func, min_val, max_val, *, + use_index=True, + mask=False, + negative_value_error=OverflowError): + # round trip (object -> C integer -> object) + values = (0, 1, 1234, max_val) + if min_val < 0: + values += (-1, min_val) + for value in values: + with self.subTest(value=value): + self.assertEqual(func(value), value) + self.assertEqual(func(IntSubclass(value)), value) + if use_index: + self.assertEqual(func(Index(value)), value) + + if use_index: + self.assertEqual(func(MyIndexAndInt()), 10) + else: + self.assertRaises(TypeError, func, Index(42)) + self.assertRaises(TypeError, func, MyIndexAndInt()) + + if mask: + self.assertEqual(func(min_val - 1), max_val) + self.assertEqual(func(max_val + 1), min_val) + self.assertEqual(func(-1 << 1000), 0) + self.assertEqual(func(1 << 1000), 0) + else: + self.assertRaises(negative_value_error, func, min_val - 1) + self.assertRaises(negative_value_error, func, -1 << 1000) + self.assertRaises(OverflowError, func, max_val + 1) + self.assertRaises(OverflowError, func, 1 << 1000) + self.assertRaises(TypeError, func, 1.0) + self.assertRaises(TypeError, func, b'2') + self.assertRaises(TypeError, func, '3') + self.assertRaises(SystemError, func, NULL) + + def check_long_asintandoverflow(self, func, min_val, max_val): + # round trip (object -> C integer -> object) + for value in (min_val, max_val, -1, 0, 1, 1234): with self.subTest(value=value): - self.assertEqual(PyLong_AsInt(value), value) - self.assertEqual(PyLong_AsInt(IntSubclass(42)), 42) - self.assertEqual(PyLong_AsInt(Index(42)), 42) - self.assertEqual(PyLong_AsInt(MyIndexAndInt()), 10) + self.assertEqual(func(value), (value, 0)) + self.assertEqual(func(IntSubclass(value)), (value, 0)) + self.assertEqual(func(Index(value)), (value, 0)) - # bound checking - self.assertRaises(OverflowError, PyLong_AsInt, INT_MIN - 1) - self.assertRaises(OverflowError, PyLong_AsInt, INT_MAX + 1) + self.assertEqual(func(MyIndexAndInt()), (10, 0)) - # invalid type - self.assertRaises(TypeError, PyLong_AsInt, 1.0) - self.assertRaises(TypeError, PyLong_AsInt, b'2') - self.assertRaises(TypeError, PyLong_AsInt, '3') - self.assertRaises(SystemError, PyLong_AsInt, NULL) + self.assertEqual(func(min_val - 1), (-1, -1)) + self.assertEqual(func(max_val + 1), (-1, +1)) + + # CRASHES func(1.0) + # CRASHES func(NULL) + + def test_long_asint(self): + # Test PyLong_AsInt() + PyLong_AsInt = _testlimitedcapi.PyLong_AsInt + from _testcapi import INT_MIN, INT_MAX + self.check_long_asint(PyLong_AsInt, INT_MIN, INT_MAX) def test_long_aslong(self): # Test PyLong_AsLong() and PyLong_FromLong() - aslong = _testcapi.pylong_aslong + aslong = _testlimitedcapi.pylong_aslong from _testcapi import LONG_MIN, LONG_MAX - # round trip (object -> long -> object) - for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslong(value), value) - - self.assertEqual(aslong(IntSubclass(42)), 42) - self.assertEqual(aslong(Index(42)), 42) - self.assertEqual(aslong(MyIndexAndInt()), 10) - - self.assertRaises(OverflowError, aslong, LONG_MIN - 1) - self.assertRaises(OverflowError, aslong, LONG_MAX + 1) - self.assertRaises(TypeError, aslong, 1.0) - self.assertRaises(TypeError, aslong, b'2') - self.assertRaises(TypeError, aslong, '3') - self.assertRaises(SystemError, aslong, NULL) + self.check_long_asint(aslong, LONG_MIN, LONG_MAX) def test_long_aslongandoverflow(self): # Test PyLong_AsLongAndOverflow() - aslongandoverflow = _testcapi.pylong_aslongandoverflow + aslongandoverflow = _testlimitedcapi.pylong_aslongandoverflow from _testcapi import LONG_MIN, LONG_MAX - # round trip (object -> long -> object) - for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslongandoverflow(value), (value, 0)) - - self.assertEqual(aslongandoverflow(IntSubclass(42)), (42, 0)) - self.assertEqual(aslongandoverflow(Index(42)), (42, 0)) - self.assertEqual(aslongandoverflow(MyIndexAndInt()), (10, 0)) - - self.assertEqual(aslongandoverflow(LONG_MIN - 1), (-1, -1)) - self.assertEqual(aslongandoverflow(LONG_MAX + 1), (-1, 1)) - # CRASHES aslongandoverflow(1.0) - # CRASHES aslongandoverflow(NULL) + self.check_long_asintandoverflow(aslongandoverflow, LONG_MIN, LONG_MAX) def test_long_asunsignedlong(self): # Test PyLong_AsUnsignedLong() and PyLong_FromUnsignedLong() - asunsignedlong = _testcapi.pylong_asunsignedlong + asunsignedlong = _testlimitedcapi.pylong_asunsignedlong from _testcapi import ULONG_MAX - # round trip (object -> unsigned long -> object) - for value in (ULONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlong(value), value) - - self.assertEqual(asunsignedlong(IntSubclass(42)), 42) - self.assertRaises(TypeError, asunsignedlong, Index(42)) - self.assertRaises(TypeError, asunsignedlong, MyIndexAndInt()) - - self.assertRaises(OverflowError, asunsignedlong, -1) - self.assertRaises(OverflowError, asunsignedlong, ULONG_MAX + 1) - self.assertRaises(TypeError, asunsignedlong, 1.0) - self.assertRaises(TypeError, asunsignedlong, b'2') - self.assertRaises(TypeError, asunsignedlong, '3') - self.assertRaises(SystemError, asunsignedlong, NULL) + self.check_long_asint(asunsignedlong, 0, ULONG_MAX, + use_index=False) def test_long_asunsignedlongmask(self): # Test PyLong_AsUnsignedLongMask() - asunsignedlongmask = _testcapi.pylong_asunsignedlongmask + asunsignedlongmask = _testlimitedcapi.pylong_asunsignedlongmask from _testcapi import ULONG_MAX - # round trip (object -> unsigned long -> object) - for value in (ULONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlongmask(value), value) - - self.assertEqual(asunsignedlongmask(IntSubclass(42)), 42) - self.assertEqual(asunsignedlongmask(Index(42)), 42) - self.assertEqual(asunsignedlongmask(MyIndexAndInt()), 10) - - self.assertEqual(asunsignedlongmask(-1), ULONG_MAX) - self.assertEqual(asunsignedlongmask(ULONG_MAX + 1), 0) - self.assertRaises(TypeError, asunsignedlongmask, 1.0) - self.assertRaises(TypeError, asunsignedlongmask, b'2') - self.assertRaises(TypeError, asunsignedlongmask, '3') - self.assertRaises(SystemError, asunsignedlongmask, NULL) + self.check_long_asint(asunsignedlongmask, 0, ULONG_MAX, mask=True) def test_long_aslonglong(self): # Test PyLong_AsLongLong() and PyLong_FromLongLong() - aslonglong = _testcapi.pylong_aslonglong + aslonglong = _testlimitedcapi.pylong_aslonglong from _testcapi import LLONG_MIN, LLONG_MAX - # round trip (object -> long long -> object) - for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslonglong(value), value) - - self.assertEqual(aslonglong(IntSubclass(42)), 42) - self.assertEqual(aslonglong(Index(42)), 42) - self.assertEqual(aslonglong(MyIndexAndInt()), 10) - - self.assertRaises(OverflowError, aslonglong, LLONG_MIN - 1) - self.assertRaises(OverflowError, aslonglong, LLONG_MAX + 1) - self.assertRaises(TypeError, aslonglong, 1.0) - self.assertRaises(TypeError, aslonglong, b'2') - self.assertRaises(TypeError, aslonglong, '3') - self.assertRaises(SystemError, aslonglong, NULL) + self.check_long_asint(aslonglong, LLONG_MIN, LLONG_MAX) def test_long_aslonglongandoverflow(self): # Test PyLong_AsLongLongAndOverflow() - aslonglongandoverflow = _testcapi.pylong_aslonglongandoverflow + aslonglongandoverflow = _testlimitedcapi.pylong_aslonglongandoverflow from _testcapi import LLONG_MIN, LLONG_MAX - # round trip (object -> long long -> object) - for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslonglongandoverflow(value), (value, 0)) - - self.assertEqual(aslonglongandoverflow(IntSubclass(42)), (42, 0)) - self.assertEqual(aslonglongandoverflow(Index(42)), (42, 0)) - self.assertEqual(aslonglongandoverflow(MyIndexAndInt()), (10, 0)) - - self.assertEqual(aslonglongandoverflow(LLONG_MIN - 1), (-1, -1)) - self.assertEqual(aslonglongandoverflow(LLONG_MAX + 1), (-1, 1)) - # CRASHES aslonglongandoverflow(1.0) - # CRASHES aslonglongandoverflow(NULL) + self.check_long_asintandoverflow(aslonglongandoverflow, LLONG_MIN, LLONG_MAX) def test_long_asunsignedlonglong(self): # Test PyLong_AsUnsignedLongLong() and PyLong_FromUnsignedLongLong() - asunsignedlonglong = _testcapi.pylong_asunsignedlonglong + asunsignedlonglong = _testlimitedcapi.pylong_asunsignedlonglong from _testcapi import ULLONG_MAX - # round trip (object -> unsigned long long -> object) - for value in (ULLONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlonglong(value), value) - - self.assertEqual(asunsignedlonglong(IntSubclass(42)), 42) - self.assertRaises(TypeError, asunsignedlonglong, Index(42)) - self.assertRaises(TypeError, asunsignedlonglong, MyIndexAndInt()) - - self.assertRaises(OverflowError, asunsignedlonglong, -1) - self.assertRaises(OverflowError, asunsignedlonglong, ULLONG_MAX + 1) - self.assertRaises(TypeError, asunsignedlonglong, 1.0) - self.assertRaises(TypeError, asunsignedlonglong, b'2') - self.assertRaises(TypeError, asunsignedlonglong, '3') - self.assertRaises(SystemError, asunsignedlonglong, NULL) + self.check_long_asint(asunsignedlonglong, 0, ULLONG_MAX, use_index=False) def test_long_asunsignedlonglongmask(self): # Test PyLong_AsUnsignedLongLongMask() - asunsignedlonglongmask = _testcapi.pylong_asunsignedlonglongmask + asunsignedlonglongmask = _testlimitedcapi.pylong_asunsignedlonglongmask from _testcapi import ULLONG_MAX - # round trip (object -> unsigned long long -> object) - for value in (ULLONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlonglongmask(value), value) - - self.assertEqual(asunsignedlonglongmask(IntSubclass(42)), 42) - self.assertEqual(asunsignedlonglongmask(Index(42)), 42) - self.assertEqual(asunsignedlonglongmask(MyIndexAndInt()), 10) - - self.assertEqual(asunsignedlonglongmask(-1), ULLONG_MAX) - self.assertEqual(asunsignedlonglongmask(ULLONG_MAX + 1), 0) - self.assertRaises(TypeError, asunsignedlonglongmask, 1.0) - self.assertRaises(TypeError, asunsignedlonglongmask, b'2') - self.assertRaises(TypeError, asunsignedlonglongmask, '3') - self.assertRaises(SystemError, asunsignedlonglongmask, NULL) + self.check_long_asint(asunsignedlonglongmask, 0, ULLONG_MAX, mask=True) def test_long_as_ssize_t(self): # Test PyLong_AsSsize_t() and PyLong_FromSsize_t() - as_ssize_t = _testcapi.pylong_as_ssize_t + as_ssize_t = _testlimitedcapi.pylong_as_ssize_t from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX - # round trip (object -> Py_ssize_t -> object) - for value in (PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(as_ssize_t(value), value) - - self.assertEqual(as_ssize_t(IntSubclass(42)), 42) - self.assertRaises(TypeError, as_ssize_t, Index(42)) - self.assertRaises(TypeError, as_ssize_t, MyIndexAndInt()) - - self.assertRaises(OverflowError, as_ssize_t, PY_SSIZE_T_MIN - 1) - self.assertRaises(OverflowError, as_ssize_t, PY_SSIZE_T_MAX + 1) - self.assertRaises(TypeError, as_ssize_t, 1.0) - self.assertRaises(TypeError, as_ssize_t, b'2') - self.assertRaises(TypeError, as_ssize_t, '3') - self.assertRaises(SystemError, as_ssize_t, NULL) + self.check_long_asint(as_ssize_t, PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, + use_index=False) def test_long_as_size_t(self): # Test PyLong_AsSize_t() and PyLong_FromSize_t() - as_size_t = _testcapi.pylong_as_size_t + as_size_t = _testlimitedcapi.pylong_as_size_t from _testcapi import SIZE_MAX - # round trip (object -> size_t -> object) - for value in (SIZE_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(as_size_t(value), value) - - self.assertEqual(as_size_t(IntSubclass(42)), 42) - self.assertRaises(TypeError, as_size_t, Index(42)) - self.assertRaises(TypeError, as_size_t, MyIndexAndInt()) - - self.assertRaises(OverflowError, as_size_t, -1) - self.assertRaises(OverflowError, as_size_t, SIZE_MAX + 1) - self.assertRaises(TypeError, as_size_t, 1.0) - self.assertRaises(TypeError, as_size_t, b'2') - self.assertRaises(TypeError, as_size_t, '3') - self.assertRaises(SystemError, as_size_t, NULL) + self.check_long_asint(as_size_t, 0, SIZE_MAX, use_index=False) def test_long_asdouble(self): # Test PyLong_AsDouble() - asdouble = _testcapi.pylong_asdouble + asdouble = _testlimitedcapi.pylong_asdouble MAX = int(sys.float_info.max) for value in (-MAX, MAX, -1, 0, 1, 1234): with self.subTest(value=value): @@ -401,8 +304,8 @@ def test_long_asdouble(self): def test_long_asvoidptr(self): # Test PyLong_AsVoidPtr() - fromvoidptr = _testcapi.pylong_fromvoidptr - asvoidptr = _testcapi.pylong_asvoidptr + fromvoidptr = _testlimitedcapi.pylong_fromvoidptr + asvoidptr = _testlimitedcapi.pylong_asvoidptr obj = object() x = fromvoidptr(obj) y = fromvoidptr(NULL) @@ -423,6 +326,348 @@ def test_long_asvoidptr(self): self.assertRaises(OverflowError, asvoidptr, -2**1000) # CRASHES asvoidptr(NULL) + def _test_long_aspid(self, aspid): + # Test PyLong_AsPid() + from _testcapi import SIZEOF_PID_T + bits = 8 * SIZEOF_PID_T + PID_T_MIN = -2**(bits-1) + PID_T_MAX = 2**(bits-1) - 1 + self.check_long_asint(aspid, PID_T_MIN, PID_T_MAX) + + def test_long_aspid(self): + self._test_long_aspid(_testcapi.pylong_aspid) + + def test_long_aspid_limited(self): + self._test_long_aspid(_testlimitedcapi.pylong_aspid) + + @support.bigmemtest(2**32, memuse=0.35) + def test_long_asnativebytes_huge(self, size): + asnativebytes = _testcapi.pylong_asnativebytes + v = 1 << size + buffer = bytearray(size * 2 // 15 + 10) + r = asnativebytes(v, buffer, 0, -1) + self.assertEqual(r, size // 8 + 1) + self.assertEqual(buffer.count(0), len(buffer)) + r = asnativebytes(v, buffer, len(buffer), -1) + self.assertEqual(r, size // 8 + 1) + self.assertEqual(buffer.count(0), len(buffer) - 1) + + def test_long_asnativebytes(self): + import math + from _testcapi import ( + pylong_asnativebytes as asnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 + MAX_USIZE = 2 ** (SZ * 8) - 1 + if support.verbose: + print(f"SIZEOF_SIZE={SZ}\n{MAX_SSIZE=:016X}\n{MAX_USIZE=:016X}") + + # These tests check that the requested buffer size is correct. + # This matches our current implementation: We only specify that the + # return value is a size *sufficient* to hold the result when queried + # using n_bytes=0. If our implementation changes, feel free to update + # the expectations here -- or loosen them to be range checks. + # (i.e. 0 *could* be stored in 1 byte and 512 in 2) + for v, expect in [ + (0, SZ), + (512, SZ), + (-512, SZ), + (MAX_SSIZE, SZ), + (MAX_USIZE, SZ + 1), + (-MAX_SSIZE, SZ), + (-MAX_USIZE, SZ + 1), + (2**255-1, 32), + (-(2**255-1), 32), + (2**255, 33), + (-(2**255), 33), # if you ask, we'll say 33, but 32 would do + (2**256-1, 33), + (-(2**256-1), 33), + (2**256, 33), + (-(2**256), 33), + ]: + with self.subTest(f"sizeof-{v:X}"): + buffer = bytearray(b"\x5a") + self.assertEqual(expect, asnativebytes(v, buffer, 0, -1), + "PyLong_AsNativeBytes(v, , 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") + # Also check via the __index__ path. + # We pass Py_ASNATIVEBYTES_NATIVE_ENDIAN | ALLOW_INDEX + self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, 3 | 16), + "PyLong_AsNativeBytes(Index(v), , 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") + + # Test that we populate n=2 bytes but do not overwrite more. + buffer = bytearray(b"\x99"*3) + self.assertEqual(2, asnativebytes(4, buffer, 2, 0), # BE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 0) // BE") + self.assertEqual(buffer, b"\x00\x04\x99") + self.assertEqual(2, asnativebytes(4, buffer, 2, 1), # LE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 1) // LE") + self.assertEqual(buffer, b"\x04\x00\x99") + + # We request as many bytes as `expect_be` contains, and always check + # the result (both big and little endian). We check the return value + # independently, since the buffer should always be filled correctly even + # if we need more bytes + for v, expect_be, expect_n in [ + (0, b'\x00', 1), + (0, b'\x00' * 2, 2), + (0, b'\x00' * 8, min(8, SZ)), + (1, b'\x01', 1), + (1, b'\x00' * 10 + b'\x01', min(11, SZ)), + (42, b'\x2a', 1), + (42, b'\x00' * 10 + b'\x2a', min(11, SZ)), + (-1, b'\xff', 1), + (-1, b'\xff' * 10, min(11, SZ)), + (-42, b'\xd6', 1), + (-42, b'\xff' * 10 + b'\xd6', min(11, SZ)), + # Extracts 255 into a single byte, but requests 2 + # (this is currently a special case, and "should" request SZ) + (255, b'\xff', 2), + (255, b'\x00\xff', 2), + (256, b'\x01\x00', 2), + (0x80, b'\x00' * 7 + b'\x80', min(8, SZ)), + # Extracts successfully (unsigned), but requests 9 bytes + (2**63, b'\x80' + b'\x00' * 7, 9), + (2**63, b'\x00\x80' + b'\x00' * 7, 9), + # Extracts into 8 bytes, but if you provide 9 we'll say 9 + (-2**63, b'\x80' + b'\x00' * 7, 8), + (-2**63, b'\xff\x80' + b'\x00' * 7, 9), + + (2**255-1, b'\x7f' + b'\xff' * 31, 32), + (-(2**255-1), b'\x80' + b'\x00' * 30 + b'\x01', 32), + # Request extra bytes, but result says we only needed 32 + (-(2**255-1), b'\xff\x80' + b'\x00' * 30 + b'\x01', 32), + (-(2**255-1), b'\xff\xff\x80' + b'\x00' * 30 + b'\x01', 32), + + # Extracting 256 bits of integer will request 33 bytes, but still + # copy as many bits as possible into the buffer. So we *can* copy + # into a 32-byte buffer, though negative number may be unrecoverable + (2**256-1, b'\xff' * 32, 33), + (2**256-1, b'\x00' + b'\xff' * 32, 33), + (-(2**256-1), b'\x00' * 31 + b'\x01', 33), + (-(2**256-1), b'\xff' + b'\x00' * 31 + b'\x01', 33), + (-(2**256-1), b'\xff\xff' + b'\x00' * 31 + b'\x01', 33), + # However, -2**255 precisely will extract into 32 bytes and return + # success. For bigger buffers, it will still succeed, but will + # return 33 + (-(2**255), b'\x80' + b'\x00' * 31, 32), + (-(2**255), b'\xff\x80' + b'\x00' * 31, 33), + + # The classic "Windows HRESULT as negative number" case + # HRESULT hr; + # PyLong_AsNativeBytes(<-2147467259>, &hr, sizeof(HRESULT), -1) + # assert(hr == E_FAIL) + (-2147467259, b'\x80\x00\x40\x05', 4), + ]: + with self.subTest(f"{v:X}-{len(expect_be)}bytes"): + n = len(expect_be) + # Fill the buffer with dummy data to ensure all bytes + # are overwritten. + buffer = bytearray(b"\xa5"*n) + expect_le = expect_be[::-1] + + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 0), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_be, buffer[:n], "") + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 1), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_le, buffer[:n], "") + + # Test cases that do not request size for a sign bit when we pass the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag + for v, expect_be, expect_n in [ + (255, b'\xff', 1), + # We pass a 2 byte buffer so it just uses the whole thing + (255, b'\x00\xff', 2), + + (2**63, b'\x80' + b'\x00' * 7, 8), + # We pass a 9 byte buffer so it uses the whole thing + (2**63, b'\x00\x80' + b'\x00' * 7, 9), + + (2**256-1, b'\xff' * 32, 32), + # We pass a 33 byte buffer so it uses the whole thing + (2**256-1, b'\x00' + b'\xff' * 32, 33), + ]: + with self.subTest(f"{v:X}-{len(expect_be)}bytes-unsigned"): + n = len(expect_be) + buffer = bytearray(b"\xa5"*n) + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 4), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 5), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + + # Ensure Py_ASNATIVEBYTES_REJECT_NEGATIVE raises on negative value + with self.assertRaises(ValueError): + asnativebytes(-1, buffer, 0, 8) + + # Ensure omitting Py_ASNATIVEBYTES_ALLOW_INDEX raises on __index__ value + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, -1) + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, 3) + + # Check a few error conditions. These are validated in code, but are + # unspecified in docs, so if we make changes to the implementation, it's + # fine to just update these tests rather than preserve the behaviour. + with self.assertRaises(TypeError): + asnativebytes('not a number', buffer, 0, -1) + + def test_long_asnativebytes_fuzz(self): + import math + from random import Random + from _testcapi import ( + pylong_asnativebytes as asnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + + rng = Random() + # Allocate bigger buffer than actual values are going to be + buffer = bytearray(260) + + for _ in range(1000): + n = rng.randrange(1, 256) + bytes_be = bytes([ + # Ensure the most significant byte is nonzero + rng.randrange(1, 256), + *[rng.randrange(256) for _ in range(n - 1)] + ]) + bytes_le = bytes_be[::-1] + v = int.from_bytes(bytes_le, 'little') + + expect_1 = expect_2 = (SZ, n) + if bytes_be[0] & 0x80: + # All values are positive, so if MSB is set, expect extra bit + # when we request the size or have a large enough buffer + expect_1 = (SZ, n + 1) + # When passing Py_ASNATIVEBYTES_UNSIGNED_BUFFER, we expect the + # return to be exactly the right size. + expect_2 = (n,) + + try: + actual = asnativebytes(v, buffer, 0, -1) + self.assertIn(actual, expect_1) + + actual = asnativebytes(v, buffer, len(buffer), 0) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_be, buffer[-n:]) + + actual = asnativebytes(v, buffer, len(buffer), 1) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_le, buffer[:n]) + + actual = asnativebytes(v, buffer, n, 4) + self.assertIn(actual, expect_2, bytes_be.hex()) + actual = asnativebytes(v, buffer, n, 5) + self.assertIn(actual, expect_2, bytes_be.hex()) + except AssertionError as ex: + value_hex = ''.join(reversed([ + f'{b:02X}{"" if i % 8 else "_"}' + for i, b in enumerate(bytes_le, start=1) + ])).strip('_') + if support.verbose: + print() + print(n, 'bytes') + print('hex =', value_hex) + print('int =', v) + raise + raise AssertionError(f"Value: 0x{value_hex}") from ex + + def test_long_fromnativebytes(self): + import math + from _testcapi import ( + pylong_fromnativebytes as fromnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 + MAX_USIZE = 2 ** (SZ * 8) - 1 + + for v_be, expect_s, expect_u in [ + (b'\x00', 0, 0), + (b'\x01', 1, 1), + (b'\xff', -1, 255), + (b'\x00\xff', 255, 255), + (b'\xff\xff', -1, 65535), + ]: + with self.subTest(f"{expect_s}-{expect_u:X}-{len(v_be)}bytes"): + n = len(v_be) + v_le = v_be[::-1] + + self.assertEqual(expect_s, fromnativebytes(v_be, n, 0, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + self.assertEqual(expect_s, fromnativebytes(v_le, n, 1, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + self.assertEqual(expect_u, fromnativebytes(v_be, n, 0, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + self.assertEqual(expect_u, fromnativebytes(v_le, n, 1, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + + # Check native endian when the result would be the same either + # way and we can test it. + if v_be == v_le: + self.assertEqual(expect_s, fromnativebytes(v_be, n, -1, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + self.assertEqual(expect_u, fromnativebytes(v_be, n, -1, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + + # Swap the unsigned request for tests and use the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag instead + self.assertEqual(expect_u, fromnativebytes(v_be, n, 4, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + + def test_long_getsign(self): + # Test PyLong_GetSign() + getsign = _testcapi.pylong_getsign + self.assertEqual(getsign(1), 1) + self.assertEqual(getsign(123456), 1) + self.assertEqual(getsign(-2), -1) + self.assertEqual(getsign(0), 0) + self.assertEqual(getsign(True), 1) + self.assertEqual(getsign(IntSubclass(-11)), -1) + self.assertEqual(getsign(False), 0) + + self.assertRaises(TypeError, getsign, 1.0) + self.assertRaises(TypeError, getsign, Index(123)) + + # CRASHES getsign(NULL) + + def test_long_asint32(self): + # Test PyLong_AsInt32() and PyLong_FromInt32() + to_int32 = _testlimitedcapi.pylong_asint32 + from _testcapi import INT32_MIN, INT32_MAX + self.check_long_asint(to_int32, INT32_MIN, INT32_MAX) + + def test_long_asint64(self): + # Test PyLong_AsInt64() and PyLong_FromInt64() + as_int64 = _testlimitedcapi.pylong_asint64 + from _testcapi import INT64_MIN, INT64_MAX + self.check_long_asint(as_int64, INT64_MIN, INT64_MAX) + + def test_long_asuint32(self): + # Test PyLong_AsUInt32() and PyLong_FromUInt32() + as_uint32 = _testlimitedcapi.pylong_asuint32 + from _testcapi import UINT32_MAX + self.check_long_asint(as_uint32, 0, UINT32_MAX, + negative_value_error=ValueError) + + def test_long_asuint64(self): + # Test PyLong_AsUInt64() and PyLong_FromUInt64() + as_uint64 = _testlimitedcapi.pylong_asuint64 + from _testcapi import UINT64_MAX + self.check_long_asint(as_uint64, 0, UINT64_MAX, + negative_value_error=ValueError) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 04f17a9ec9e72a..6ab7b685c2e18b 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -148,12 +148,12 @@ class C(): pass self.assertIn(b'MemoryError', out) *_, count = line.split(b' ') count = int(count) - self.assertLessEqual(count, i*5) - self.assertGreaterEqual(count, i*5-2) + self.assertLessEqual(count, i*10) + self.assertGreaterEqual(count, i*10-4) -# Py_GIL_DISABLED requires mimalloc (not malloc) -@unittest.skipIf(support.Py_GIL_DISABLED, 'need malloc') +# free-threading requires mimalloc (not malloc) +@support.requires_gil_enabled() class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 67fbef4f269814..80e705a37c4c5e 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -17,7 +17,6 @@ import time import types import unittest -import warnings import weakref import operator from test import support @@ -26,6 +25,8 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import requires_limited_api +from test.support import expected_failure_if_gil_disabled +from test.support import Py_GIL_DISABLED from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end try: import _posixsubprocess @@ -40,13 +41,14 @@ except ImportError: _testsinglephase = None try: - import _xxsubinterpreters as _interpreters + import _interpreters except ModuleNotFoundError: _interpreters = None # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +import _testlimitedcapi import _testinternalcapi @@ -117,7 +119,7 @@ def __len__(self): return 1 with self.assertRaisesRegex(TypeError, 'indexing'): _posixsubprocess.fork_exec( - 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -125,7 +127,7 @@ def __len__(self): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -135,7 +137,7 @@ def __len__(self): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) + Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") @@ -536,14 +538,19 @@ def __del__(self): self.assertEqual(new_type_refcnt, sys.getrefcount(A)) def test_heaptype_with_dict(self): - inst = _testcapi.HeapCTypeWithDict() - inst.foo = 42 - self.assertEqual(inst.foo, 42) - self.assertEqual(inst.dictobj, inst.__dict__) - self.assertEqual(inst.dictobj, {"foo": 42}) + for cls in ( + _testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict, + ): + with self.subTest(cls=cls): + inst = cls() + inst.foo = 42 + self.assertEqual(inst.foo, 42) + self.assertEqual(inst.dictobj, inst.__dict__) + self.assertEqual(inst.dictobj, {"foo": 42}) - inst = _testcapi.HeapCTypeWithDict() - self.assertEqual({}, inst.__dict__) + inst = cls() + self.assertEqual({}, inst.__dict__) def test_heaptype_with_managed_dict(self): inst = _testcapi.HeapCTypeWithManagedDict() @@ -580,10 +587,15 @@ def test_heaptype_with_negative_dict(self): self.assertEqual({}, inst.__dict__) def test_heaptype_with_weakref(self): - inst = _testcapi.HeapCTypeWithWeakref() - ref = weakref.ref(inst) - self.assertEqual(ref(), inst) - self.assertEqual(inst.weakreflist, ref) + for cls in ( + _testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref, + ): + with self.subTest(cls=cls): + inst = cls() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + self.assertEqual(inst.weakreflist, ref) def test_heaptype_with_managed_weakref(self): inst = _testcapi.HeapCTypeWithManagedWeakref() @@ -707,63 +719,76 @@ def test_heaptype_with_custom_metaclass_custom_new(self): with self.assertRaisesRegex(TypeError, msg): t = _testcapi.pytype_fromspec_meta(metaclass) - def test_heaptype_with_custom_metaclass_deprecation(self): + def test_heaptype_base_with_custom_metaclass(self): metaclass = _testcapi.HeapCTypeMetaclassCustomNew - # gh-103968: a metaclass with custom tp_new is deprecated, but still - # allowed for functions that existed in 3.11 - # (PyType_FromSpecWithBases is used here). class Base(metaclass=metaclass): pass # Class creation from C - with warnings_helper.check_warnings( - ('.* _testcapi.Subclass .* custom tp_new.*in Python 3.14.*', DeprecationWarning), - ): + msg = "Metaclasses with custom tp_new are not supported." + with self.assertRaisesRegex(TypeError, msg): sub = _testcapi.make_type_with_base(Base) - self.assertTrue(issubclass(sub, Base)) - self.assertIsInstance(sub, metaclass) - def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): + def test_heaptype_with_tp_vectorcall(self): + tp = _testcapi.HeapCTypeVectorcall + v0 = tp.__new__(tp) + v0.__init__() + v1 = tp() + self.assertEqual(v0.value, 2) + self.assertEqual(v1.value, 1) - with self.assertRaises(TypeError): - class Both1(_testcapi.HeapCTypeWithWeakref, _testcapi.HeapCTypeWithDict): - pass - with self.assertRaises(TypeError): - class Both2(_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref): - pass + def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): + for weakref_cls in (_testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref): + for dict_cls in (_testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict): + with self.subTest(weakref_cls=weakref_cls, dict_cls=dict_cls): + + with self.assertRaises(TypeError): + class Both1(weakref_cls, dict_cls): + pass + with self.assertRaises(TypeError): + class Both2(dict_cls, weakref_cls): + pass def test_multiple_inheritance_ctypes_with_weakref_or_dict_and_other_builtin(self): + for dict_cls in (_testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict): + for weakref_cls in (_testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref): + with self.subTest(dict_cls=dict_cls, weakref_cls=weakref_cls): + + with self.assertRaises(TypeError): + class C1(dict_cls, list): + pass - with self.assertRaises(TypeError): - class C1(_testcapi.HeapCTypeWithDict, list): - pass - - with self.assertRaises(TypeError): - class C2(_testcapi.HeapCTypeWithWeakref, list): - pass + with self.assertRaises(TypeError): + class C2(weakref_cls, list): + pass - class C3(_testcapi.HeapCTypeWithManagedDict, list): - pass - class C4(_testcapi.HeapCTypeWithManagedWeakref, list): - pass + class C3(_testcapi.HeapCTypeWithManagedDict, list): + pass + class C4(_testcapi.HeapCTypeWithManagedWeakref, list): + pass - inst = C3() - inst.append(0) - str(inst.__dict__) + inst = C3() + inst.append(0) + str(inst.__dict__) - inst = C4() - inst.append(0) - str(inst.__weakref__) + inst = C4() + inst.append(0) + str(inst.__weakref__) - for cls in (_testcapi.HeapCTypeWithManagedDict, _testcapi.HeapCTypeWithManagedWeakref): - for cls2 in (_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref): - class S(cls, cls2): - pass - class B1(C3, cls): - pass - class B2(C4, cls): - pass + for cls in (_testcapi.HeapCTypeWithManagedDict, + _testcapi.HeapCTypeWithManagedWeakref): + for cls2 in (dict_cls, weakref_cls): + class S(cls, cls2): + pass + class B1(C3, cls): + pass + class B2(C4, cls): + pass def test_pytype_fromspec_with_repeated_slots(self): for variant in range(2): @@ -771,33 +796,11 @@ def test_pytype_fromspec_with_repeated_slots(self): with self.assertRaises(SystemError): _testcapi.create_type_from_repeated_slots(variant) - @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_immutable_type_with_mutable_base(self): - # Add deprecation warning here so it's removed in 3.14 - warnings._deprecated( - 'creating immutable classes with mutable bases', remove=(3, 14)) - - class MutableBase: - def meth(self): - return 'original' - - with self.assertWarns(DeprecationWarning): - ImmutableSubclass = _testcapi.make_immutable_type_with_base( - MutableBase) - instance = ImmutableSubclass() + class MutableBase: ... - self.assertEqual(instance.meth(), 'original') - - # Cannot override the static type's method - with self.assertRaisesRegex( - TypeError, - "cannot set 'meth' attribute of immutable type"): - ImmutableSubclass.meth = lambda self: 'overridden' - self.assertEqual(instance.meth(), 'original') - - # Can change the method on the mutable base - MutableBase.meth = lambda self: 'changed' - self.assertEqual(instance.meth(), 'changed') + with self.assertRaisesRegex(TypeError, 'Creating immutable type'): + _testcapi.make_immutable_type_with_base(MutableBase) def test_pynumber_tobase(self): from _testcapi import pynumber_tobase @@ -886,36 +889,6 @@ def __init__(self): _testcapi.clear_managed_dict(c) self.assertEqual(c.__dict__, {}) - def test_eval_get_func_name(self): - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(_testcapi.eval_get_func_name(function_example), - "function_example") - self.assertEqual(_testcapi.eval_get_func_name(A.method_example), - "method_example") - self.assertEqual(_testcapi.eval_get_func_name(A().method_example), - "method_example") - self.assertEqual(_testcapi.eval_get_func_name(sum), "sum") # c function - self.assertEqual(_testcapi.eval_get_func_name(A), "type") - - def test_eval_get_func_desc(self): - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(_testcapi.eval_get_func_desc(function_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(A.method_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(A().method_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(sum), "()") # c function - self.assertEqual(_testcapi.eval_get_func_desc(A), " object") - def test_function_get_code(self): import types @@ -1099,21 +1072,152 @@ class Data(_testcapi.ObjExtraData): del d.extra self.assertIsNone(d.extra) - def test_get_type_module_name(self): + def test_get_type_name(self): + class MyType: + pass + + from _testcapi import ( + get_type_name, get_type_qualname, + get_type_fullyqualname, get_type_module_name) + from collections import OrderedDict ht = _testcapi.get_heaptype_for_name() - for cls, expected in { - int: 'builtins', - OrderedDict: 'collections', - ht: '_testcapi', - }.items(): - with self.subTest(repr(cls)): - modname = _testinternalcapi.get_type_module_name(cls) - self.assertEqual(modname, expected) + for cls, fullname, modname, qualname, name in ( + (int, + 'int', + 'builtins', + 'int', + 'int'), + (OrderedDict, + 'collections.OrderedDict', + 'collections', + 'OrderedDict', + 'OrderedDict'), + (ht, + '_testcapi.HeapTypeNameType', + '_testcapi', + 'HeapTypeNameType', + 'HeapTypeNameType'), + (MyType, + f'{__name__}.CAPITest.test_get_type_name..MyType', + __name__, + 'CAPITest.test_get_type_name..MyType', + 'MyType'), + ): + with self.subTest(cls=repr(cls)): + self.assertEqual(get_type_fullyqualname(cls), fullname) + self.assertEqual(get_type_module_name(cls), modname) + self.assertEqual(get_type_qualname(cls), qualname) + self.assertEqual(get_type_name(cls), name) + # override __module__ ht.__module__ = 'test_module' - modname = _testinternalcapi.get_type_module_name(ht) - self.assertEqual(modname, 'test_module') + self.assertEqual(get_type_fullyqualname(ht), 'test_module.HeapTypeNameType') + self.assertEqual(get_type_module_name(ht), 'test_module') + self.assertEqual(get_type_qualname(ht), 'HeapTypeNameType') + self.assertEqual(get_type_name(ht), 'HeapTypeNameType') + + # override __name__ and __qualname__ + MyType.__name__ = 'my_name' + MyType.__qualname__ = 'my_qualname' + self.assertEqual(get_type_fullyqualname(MyType), f'{__name__}.my_qualname') + self.assertEqual(get_type_module_name(MyType), __name__) + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # override also __module__ + MyType.__module__ = 'my_module' + self.assertEqual(get_type_fullyqualname(MyType), 'my_module.my_qualname') + self.assertEqual(get_type_module_name(MyType), 'my_module') + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # PyType_GetFullyQualifiedName() ignores the module if it's "builtins" + # or "__main__" of it is not a string + MyType.__module__ = 'builtins' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = '__main__' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = 123 + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + + def test_get_base_by_token(self): + def get_base_by_token(src, key, comparable=True): + def run(use_mro): + find_first = _testcapi.pytype_getbasebytoken + ret1, result = find_first(src, key, use_mro, True) + ret2, no_result = find_first(src, key, use_mro, False) + self.assertIn(ret1, (0, 1)) + self.assertEqual(ret1, result is not None) + self.assertEqual(ret1, ret2) + self.assertIsNone(no_result) + return result + + found_in_mro = run(True) + found_in_bases = run(False) + if comparable: + self.assertIs(found_in_mro, found_in_bases) + return found_in_mro + return found_in_mro, found_in_bases + + create_type = _testcapi.create_type_with_token + get_token = _testcapi.get_tp_token + + Py_TP_USE_SPEC = _testcapi.Py_TP_USE_SPEC + self.assertEqual(Py_TP_USE_SPEC, 0) + + A1 = create_type('_testcapi.A1', Py_TP_USE_SPEC) + self.assertTrue(get_token(A1) != Py_TP_USE_SPEC) + + B1 = create_type('_testcapi.B1', id(self)) + self.assertTrue(get_token(B1) == id(self)) + + tokenA1 = get_token(A1) + # find A1 from A1 + found = get_base_by_token(A1, tokenA1) + self.assertIs(found, A1) + + # no token in static types + STATIC = type(1) + self.assertEqual(get_token(STATIC), 0) + found = get_base_by_token(STATIC, tokenA1) + self.assertIs(found, None) + + # no token in pure subtypes + class A2(A1): pass + self.assertEqual(get_token(A2), 0) + # find A1 + class Z(STATIC, B1, A2): pass + found = get_base_by_token(Z, tokenA1) + self.assertIs(found, A1) + + # searching for NULL token is an error + with self.assertRaises(SystemError): + get_base_by_token(Z, 0) + with self.assertRaises(SystemError): + get_base_by_token(STATIC, 0) + + # share the token with A1 + C1 = create_type('_testcapi.C1', tokenA1) + self.assertTrue(get_token(C1) == tokenA1) + + # find C1 first by shared token + class Z(C1, A2): pass + found = get_base_by_token(Z, tokenA1) + self.assertIs(found, C1) + # B1 not found + found = get_base_by_token(Z, get_token(B1)) + self.assertIs(found, None) + + with self.assertRaises(TypeError): + _testcapi.pytype_getbasebytoken( + 'not a type', id(self), True, False) + + def test_gen_get_code(self): + def genf(): yield + gen = genf() + self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) + @requires_limited_api class TestHeapTypeRelative(unittest.TestCase): @@ -1124,7 +1228,7 @@ def test_heaptype_relative_sizes(self): # Test subclassing using "relative" basicsize, see PEP 697 def check(extra_base_size, extra_size): Base, Sub, instance, data_ptr, data_offset, data_size = ( - _testcapi.make_sized_heaptypes( + _testlimitedcapi.make_sized_heaptypes( extra_base_size, -extra_size)) # no alignment shenanigans when inheriting directly @@ -1152,11 +1256,11 @@ def check(extra_base_size, extra_size): # we don't reserve (requested + alignment) or more data self.assertLess(data_size - extra_size, - _testcapi.ALIGNOF_MAX_ALIGN_T) + _testlimitedcapi.ALIGNOF_MAX_ALIGN_T) # The offsets/sizes we calculated should be aligned. - self.assertEqual(data_offset % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) - self.assertEqual(data_size % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) + self.assertEqual(data_offset % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) + self.assertEqual(data_size % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, object.__basicsize__, @@ -1182,7 +1286,7 @@ def test_heaptype_inherit_itemsize(self): object.__basicsize__+1}) for extra_size in sizes: with self.subTest(extra_size=extra_size): - Sub = _testcapi.subclass_var_heaptype( + Sub = _testlimitedcapi.subclass_var_heaptype( _testcapi.HeapCCollection, -extra_size, 0, 0) collection = Sub(1, 2, 3) collection.set_data_to_3s() @@ -1196,7 +1300,7 @@ def test_heaptype_invalid_inheritance(self): with self.assertRaises(SystemError, msg="Cannot extend variable-size class without " + "Py_TPFLAGS_ITEMS_AT_END"): - _testcapi.subclass_heaptype(int, -8, 0) + _testlimitedcapi.subclass_heaptype(int, -8, 0) def test_heaptype_relative_members(self): """Test HeapCCollection subclasses work properly""" @@ -1209,7 +1313,7 @@ def test_heaptype_relative_members(self): for offset in sizes: with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size, offset=offset): if offset < extra_size: - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, -extra_size, offset, True) Base = Sub.mro()[1] instance = Sub() @@ -1228,29 +1332,29 @@ def test_heaptype_relative_members(self): instance.set_memb_relative(0) else: with self.assertRaises(SystemError): - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, -extra_size, offset, True) with self.assertRaises(SystemError): - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, extra_size, offset, True) with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size): with self.assertRaises(SystemError): - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, -extra_size, -1, True) def test_heaptype_relative_members_errors(self): with self.assertRaisesRegex( SystemError, r"With Py_RELATIVE_OFFSET, basicsize must be negative"): - _testcapi.make_heaptype_with_member(0, 1234, 0, True) + _testlimitedcapi.make_heaptype_with_member(0, 1234, 0, True) with self.assertRaisesRegex( SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): - _testcapi.make_heaptype_with_member(0, -8, 1234, True) + _testlimitedcapi.make_heaptype_with_member(0, -8, 1234, True) with self.assertRaisesRegex( SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): - _testcapi.make_heaptype_with_member(0, -8, -1, True) + _testlimitedcapi.make_heaptype_with_member(0, -8, -1, True) - Sub = _testcapi.make_heaptype_with_member(0, -8, 0, True) + Sub = _testlimitedcapi.make_heaptype_with_member(0, -8, 0, True) instance = Sub() with self.assertRaisesRegex( SystemError, r"PyMember_GetOne used with Py_RELATIVE_OFFSET"): @@ -1259,6 +1363,53 @@ def test_heaptype_relative_members_errors(self): SystemError, r"PyMember_SetOne used with Py_RELATIVE_OFFSET"): instance.set_memb_relative(0) + def test_heaptype_relative_special_members_errors(self): + for member_name in "__vectorcalloffset__", "__dictoffset__", "__weaklistoffset__": + with self.subTest(member_name=member_name): + with self.assertRaisesRegex( + SystemError, + r"With Py_RELATIVE_OFFSET, basicsize must be negative."): + _testlimitedcapi.make_heaptype_with_member( + basicsize=sys.getsizeof(object()) + 100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"Member offset out of range \(0\.\.-basicsize\)"): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-8, + add_relative_flag=True, + member_name=member_name, + member_offset=-1, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"type of %s must be Py_T_PYSSIZET" % member_name): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"flags for %s must be " % member_name): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=0, + ) + def test_pyobject_getitemdata_error(self): """Test PyObject_GetItemData fails on unsupported types""" with self.assertRaises(TypeError): @@ -1270,13 +1421,132 @@ def test_pyobject_getitemdata_error(self): _testcapi.pyobject_getitemdata(0) + def test_function_get_closure(self): + from types import CellType + + def regular_function(): ... + def unused_one_level(arg1): + def inner(arg2, arg3): ... + return inner + def unused_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): ... + return inner + return decorator + def with_one_level(arg1): + def inner(arg2, arg3): + return arg1 + arg2 + arg3 + return inner + def with_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): + return arg1 + arg2 + arg3 + arg4 + arg5 + arg6 + return inner + return decorator + + # Functions without closures: + self.assertIsNone(_testcapi.function_get_closure(regular_function)) + self.assertIsNone(regular_function.__closure__) + + func = unused_one_level(1) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + func = unused_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + # Functions with closures: + func = with_one_level(5) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 1) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + self.assertTrue(all(isinstance(cell, CellType) for cell in closure)) + self.assertTrue(closure[0].cell_contents, 5) + + func = with_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 4) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + self.assertTrue(all(isinstance(cell, CellType) for cell in closure)) + self.assertEqual([cell.cell_contents for cell in closure], + [1, 2, 3, 4]) + + def test_function_get_closure_error(self): + with self.assertRaises(SystemError): + _testcapi.function_get_closure(1) + with self.assertRaises(SystemError): + _testcapi.function_get_closure(None) + + def test_function_set_closure(self): + from types import CellType + + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + func = function_without_closure + _testcapi.function_set_closure(func, (CellType(1), CellType(1))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 1]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 1]) + + func = function_with_closure(1) + _testcapi.function_set_closure(func, + (CellType(1), CellType(2), CellType(3))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 2, 3]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 2, 3]) + + def test_function_set_closure_none(self): + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + _testcapi.function_set_closure(function_without_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_without_closure)) + self.assertIsNone(function_without_closure.__closure__) + + _testcapi.function_set_closure(function_with_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_with_closure)) + self.assertIsNone(function_with_closure.__closure__) + + def test_function_set_closure_errors(self): + def function_without_closure(): ... + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(None, ()) # not a function + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(function_without_closure, 1) + self.assertIsNone(function_without_closure.__closure__) # no change + + # NOTE: this works, but goes against the docs: + _testcapi.function_set_closure(function_without_closure, (1, 2)) + self.assertEqual( + _testcapi.function_get_closure(function_without_closure), (1, 2)) + self.assertEqual(function_without_closure.__closure__, (1, 2)) + + class TestPendingCalls(unittest.TestCase): # See the comment in ceval.c (at the "handle_eval_breaker" label) # about when pending calls get run. This is especially relevant # here for creating deterministic tests. - def pendingcalls_submit(self, l, n): + def main_pendingcalls_submit(self, l, n): def callback(): #this function can be interrupted by thread switching so let's #use an atomic operation @@ -1291,12 +1561,27 @@ def callback(): if _testcapi._pending_threadfunc(callback): break - def pendingcalls_wait(self, l, n, context = None): + def pendingcalls_submit(self, l, n, *, main=True, ensure=False): + def callback(): + #this function can be interrupted by thread switching so let's + #use an atomic operation + l.append(None) + + if main: + return _testcapi._pending_threadfunc(callback, n, + blocking=False, + ensure_added=ensure) + else: + return _testinternalcapi.pending_threadfunc(callback, n, + blocking=False, + ensure_added=ensure) + + def pendingcalls_wait(self, l, numadded, context = None): #now, stick around until l[0] has grown to 10 count = 0 - while len(l) != n: + while len(l) != numadded: #this busy loop is where we expect to be interrupted to - #run our callbacks. Note that callbacks are only run on the + #run our callbacks. Note that some callbacks are only run on the #main thread if False and support.verbose: print("(%i)"%(len(l),),) @@ -1306,12 +1591,12 @@ def pendingcalls_wait(self, l, n, context = None): continue count += 1 self.assertTrue(count < 10000, - "timeout waiting for %i callbacks, got %i"%(n, len(l))) + "timeout waiting for %i callbacks, got %i"%(numadded, len(l))) if False and support.verbose: print("(%i)"%(len(l),)) @threading_helper.requires_working_threading() - def test_pendingcalls_threaded(self): + def test_main_pendingcalls_threaded(self): #do every callback on a separate thread n = 32 #total callbacks @@ -1325,15 +1610,15 @@ class foo(object):pass context.lock = threading.Lock() context.event = threading.Event() - threads = [threading.Thread(target=self.pendingcalls_thread, + threads = [threading.Thread(target=self.main_pendingcalls_thread, args=(context,)) for i in range(context.nThreads)] with threading_helper.start_threads(threads): self.pendingcalls_wait(context.l, n, context) - def pendingcalls_thread(self, context): + def main_pendingcalls_thread(self, context): try: - self.pendingcalls_submit(context.l, context.n) + self.main_pendingcalls_submit(context.l, context.n) finally: with context.lock: context.nFinished += 1 @@ -1343,20 +1628,54 @@ def pendingcalls_thread(self, context): if nFinished == context.nThreads: context.event.set() - def test_pendingcalls_non_threaded(self): + def test_main_pendingcalls_non_threaded(self): #again, just using the main thread, likely they will all be dispatched at #once. It is ok to ask for too many, because we loop until we find a slot. #the loop can be interrupted to dispatch. #there are only 32 dispatch slots, so we go for twice that! l = [] n = 64 - self.pendingcalls_submit(l, n) + self.main_pendingcalls_submit(l, n) self.pendingcalls_wait(l, n) - def test_gen_get_code(self): - def genf(): yield - gen = genf() - self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) + def test_max_pending(self): + with self.subTest('main-only'): + maxpending = 32 + + l = [] + added = self.pendingcalls_submit(l, 1, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, 1) + + l = [] + added = self.pendingcalls_submit(l, maxpending, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + l = [] + added = self.pendingcalls_submit(l, maxpending+1, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + with self.subTest('not main-only'): + # Per-interpreter pending calls has a much higher limit + # on how many may be pending at a time. + maxpending = 300 + + l = [] + added = self.pendingcalls_submit(l, 1, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, 1) + + l = [] + added = self.pendingcalls_submit(l, maxpending, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + l = [] + added = self.pendingcalls_submit(l, maxpending+1, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) class PendingTask(types.SimpleNamespace): @@ -1796,6 +2115,7 @@ def test_py_config_isoloated_per_interpreter(self): # double checked at the time this test was written. config = _testinternalcapi.get_config() config['int_max_str_digits'] = 55555 + config['parse_argv'] = 0 _testinternalcapi.set_config(config) sub_value = _testinternalcapi.get_config()['int_max_str_digits'] assert sub_value == 55555, sub_value @@ -1848,15 +2168,30 @@ def test_configured_settings(self): kwlist[-2] = 'check_multi_interp_extensions' kwlist[-1] = 'own_gil' - # expected to work - for config, expected in { + expected_to_work = { (True, True, True, True, True, True, True): (ALL_FLAGS, True), (True, False, False, False, False, False, False): (OBMALLOC, False), (False, False, False, True, False, True, False): (THREADS | EXTENSIONS, False), - }.items(): + } + + expected_to_fail = { + (False, False, False, False, False, False, False), + } + + # gh-117649: The free-threaded build does not currently allow + # setting check_multi_interp_extensions to False. + if Py_GIL_DISABLED: + for config in list(expected_to_work.keys()): + kwargs = dict(zip(kwlist, config)) + if not kwargs['check_multi_interp_extensions']: + del expected_to_work[config] + expected_to_fail.add(config) + + # expected to work + for config, expected in expected_to_work.items(): kwargs = dict(zip(kwlist, config)) exp_flags, exp_gil = expected expected = { @@ -1880,9 +2215,7 @@ def test_configured_settings(self): self.assertEqual(settings, expected) # expected to fail - for config in [ - (False, False, False, False, False, False, False), - ]: + for config in expected_to_fail: kwargs = dict(zip(kwlist, config)) with self.subTest(config): script = textwrap.dedent(f''' @@ -1890,11 +2223,14 @@ def test_configured_settings(self): _testinternalcapi.get_interp_settings() raise NotImplementedError('unreachable') ''') - with self.assertRaises(RuntimeError): + with self.assertRaises(_interpreters.InterpreterError): support.run_in_subinterp_with_config(script, **kwargs) @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + # gh-117649: The free-threaded build does not currently allow overriding + # the check_multi_interp_extensions setting. + @expected_failure_if_gil_disabled() def test_overridden_setting_extensions_subinterp_check(self): """ PyInterpreterConfig.check_multi_interp_extensions can be overridden @@ -1946,6 +2282,9 @@ def check(enabled, override): } r, w = os.pipe() + if Py_GIL_DISABLED: + # gh-117649: The test fails before `w` is closed + self.addCleanup(os.close, w) script = textwrap.dedent(f''' from test.test_capi.check_config import run_singlephase_check run_singlephase_check({override}, {w}) @@ -1990,6 +2329,9 @@ def test_mutate_exception(self): self.assertFalse(hasattr(binascii.Error, "foobar")) @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + # gh-117649: The free-threaded build does not currently support sharing + # extension module state between interpreters. + @expected_failure_if_gil_disabled() def test_module_state_shared_in_global(self): """ bpo-44050: Extension module state should be shared between interpreters @@ -1999,6 +2341,13 @@ def test_module_state_shared_in_global(self): self.addCleanup(os.close, r) self.addCleanup(os.close, w) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + script = textwrap.dedent(f""" import importlib.machinery import importlib.util @@ -2006,7 +2355,7 @@ def test_module_state_shared_in_global(self): fullname = '_test_module_state_shared' origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + loader = importlib.machinery.{loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) attr_id = str(id(module.Error)).encode() @@ -2023,134 +2372,513 @@ def test_module_state_shared_in_global(self): @requires_subinterpreters -class InterpreterIDTests(unittest.TestCase): +class InterpreterConfigTests(unittest.TestCase): + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + def iter_all_configs(self): + for use_main_obmalloc in (True, False): + for allow_fork in (True, False): + for allow_exec in (True, False): + for allow_threads in (True, False): + for allow_daemon in (True, False): + for checkext in (True, False): + for gil in ('shared', 'own', 'default'): + yield types.SimpleNamespace( + use_main_obmalloc=use_main_obmalloc, + allow_fork=allow_fork, + allow_exec=allow_exec, + allow_threads=allow_threads, + allow_daemon_threads=allow_daemon, + check_multi_interp_extensions=checkext, + gil=gil, + ) + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def test_predefined_config(self): + def check(name, expected): + expected = self.supported[expected] + args = (name,) if name else () + + config1 = _interpreters.new_config(*args) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(*args) + self.assert_ns_equal(config2, expected) + self.assertIsNot(config2, expected) + self.assertIsNot(config2, config1) + + with self.subTest('default'): + check(None, 'isolated') + + for name in self.supported: + with self.subTest(name): + check(name, name) + + def test_update_from_dict(self): + for name, vanilla in self.supported.items(): + with self.subTest(f'noop ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'change all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in self.gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in self.gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('unsupported field'): + for name in self.supported: + with self.assertRaises(ValueError): + _interpreters.new_config(name, spam=True) + + # Bad values for bool fields. + for field, value in vars(self.supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'unsupported value ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'unsupported value(gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'unsupported value (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_interp_init(self): + questionable = [ + # strange + dict( + allow_fork=True, + allow_exec=False, + ), + dict( + gil='shared', + use_main_obmalloc=False, + ), + # risky + dict( + allow_fork=True, + allow_threads=True, + ), + # ought to be invalid? + dict( + allow_threads=False, + allow_daemon_threads=True, + ), + dict( + gil='own', + use_main_obmalloc=True, + ), + ] + invalid = [ + dict( + use_main_obmalloc=False, + check_multi_interp_extensions=False + ), + ] + if Py_GIL_DISABLED: + invalid.append(dict(check_multi_interp_extensions=False)) + def match(config, override_cases): + ns = vars(config) + for overrides in override_cases: + if dict(ns, **overrides) == ns: + return True + return False + + def check(config): + script = 'pass' + rc = _testinternalcapi.run_in_subinterp_with_config(script, config) + self.assertEqual(rc, 0) + + for config in self.iter_all_configs(): + if config.gil == 'default': + continue + if match(config, invalid): + with self.subTest(f'invalid: {config}'): + with self.assertRaises(_interpreters.InterpreterError): + check(config) + elif match(config, questionable): + with self.subTest(f'questionable: {config}'): + check(config) + else: + with self.subTest(f'valid: {config}'): + check(config) - InterpreterID = _testcapi.get_interpreterid_type() + def test_get_config(self): + @contextlib.contextmanager + def new_interp(config): + interpid = _interpreters.create(config, reqrefs=False) + try: + yield interpid + finally: + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass - def new_interpreter(self): - def ensure_destroyed(interpid): + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + if Py_GIL_DISABLED: + expected.check_multi_interp_extensions = False + interpid, *_ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + with new_interp('isolated') as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + with new_interp('legacy') as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config( + 'empty', + use_main_obmalloc=True, + gil='shared', + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + ) + with new_interp(orig) as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + +@requires_subinterpreters +class InterpreterIDTests(unittest.TestCase): + + def add_interp_cleanup(self, interpid): + def ensure_destroyed(): try: _interpreters.destroy(interpid) except _interpreters.InterpreterNotFoundError: pass + self.addCleanup(ensure_destroyed) + + def new_interpreter(self): id = _interpreters.create() - self.addCleanup(lambda: ensure_destroyed(id)) + self.add_interp_cleanup(id) return id - def test_with_int(self): - id = self.InterpreterID(10, force=True) + def test_conversion_int(self): + convert = _testinternalcapi.normalize_interp_id + interpid = convert(10) + self.assertEqual(interpid, 10) - self.assertEqual(int(id), 10) - - def test_coerce_id(self): - class Int(str): + def test_conversion_coerced(self): + convert = _testinternalcapi.normalize_interp_id + class MyInt(str): def __index__(self): return 10 + interpid = convert(MyInt()) + self.assertEqual(interpid, 10) + + def test_conversion_from_interpreter(self): + convert = _testinternalcapi.normalize_interp_id + interpid = self.new_interpreter() + converted = convert(interpid) + self.assertEqual(converted, interpid) - id = self.InterpreterID(Int(), force=True) - self.assertEqual(int(id), 10) + def test_conversion_bad(self): + convert = _testinternalcapi.normalize_interp_id - def test_bad_id(self): for badid in [ object(), 10.0, '10', b'10', ]: - with self.subTest(badid): + with self.subTest(f'bad: {badid!r}'): with self.assertRaises(TypeError): - self.InterpreterID(badid) + convert(badid) badid = -1 - with self.subTest(badid): + with self.subTest(f'bad: {badid!r}'): with self.assertRaises(ValueError): - self.InterpreterID(badid) + convert(badid) badid = 2**64 - with self.subTest(badid): + with self.subTest(f'bad: {badid!r}'): with self.assertRaises(OverflowError): - self.InterpreterID(badid) + convert(badid) - def test_exists(self): - id = self.new_interpreter() - with self.assertRaises(_interpreters.InterpreterNotFoundError): - self.InterpreterID(int(id) + 1) # unforced + def test_lookup_exists(self): + interpid = self.new_interpreter() + self.assertTrue( + _testinternalcapi.interpreter_exists(interpid)) - def test_does_not_exist(self): - id = self.new_interpreter() - with self.assertRaises(_interpreters.InterpreterNotFoundError): - self.InterpreterID(int(id) + 1) # unforced + def test_lookup_does_not_exist(self): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) - def test_destroyed(self): - id = _interpreters.create() - _interpreters.destroy(id) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - self.InterpreterID(id) # unforced - - def test_str(self): - id = self.InterpreterID(10, force=True) - self.assertEqual(str(id), '10') - - def test_repr(self): - id = self.InterpreterID(10, force=True) - self.assertEqual(repr(id), 'InterpreterID(10)') - - def test_equality(self): - id1 = self.new_interpreter() - id2 = self.InterpreterID(id1) - id3 = self.InterpreterID( - self.new_interpreter()) - - self.assertTrue(id2 == id2) # identity - self.assertTrue(id2 == id1) # int-equivalent - self.assertTrue(id1 == id2) # reversed - self.assertTrue(id2 == int(id2)) - self.assertTrue(id2 == float(int(id2))) - self.assertTrue(float(int(id2)) == id2) - self.assertFalse(id2 == float(int(id2)) + 0.1) - self.assertFalse(id2 == str(int(id2))) - self.assertFalse(id2 == 2**1000) - self.assertFalse(id2 == float('inf')) - self.assertFalse(id2 == 'spam') - self.assertFalse(id2 == id3) - - self.assertFalse(id2 != id2) - self.assertFalse(id2 != id1) - self.assertFalse(id1 != id2) - self.assertTrue(id2 != id3) - - def test_linked_lifecycle(self): - id1 = _interpreters.create() - _testcapi.unlink_interpreter_refcount(id1) + def test_lookup_destroyed(self): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) + + def get_refcount_helpers(self): + return ( + _testinternalcapi.get_interpreter_refcount, + (lambda id: _interpreters.incref(id, implieslink=False)), + _interpreters.decref, + ) + + def test_linked_lifecycle_does_not_exist(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + with self.subTest('never existed'): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + with self.subTest('destroyed'): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + def test_linked_lifecycle_initial(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount, _, _ = self.get_refcount_helpers() + + # A new interpreter will start out not linked, with a refcount of 0. + interpid = self.new_interpreter() + linked = is_linked(interpid) + refcount = get_refcount(interpid) + + self.assertFalse(linked) + self.assertEqual(refcount, 0) + + def test_linked_lifecycle_never_linked(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + # Incref will not automatically link it. + incref(interpid) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + # It isn't linked so it isn't destroyed. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 0) + 0, get_refcount(interpid)) + + def test_linked_lifecycle_link_unlink(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + + interpid = self.new_interpreter() + + # Linking at refcount 0 does not destroy the interpreter. + link(interpid) + self.assertTrue( + exists(interpid)) + self.assertTrue( + is_linked(interpid)) + + # Unlinking at refcount 0 does not destroy the interpreter. + unlink(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) + + def test_linked_lifecycle_link_incref_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + # Linking it will not change the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) - id2 = self.InterpreterID(id1) + # Decref with a refcount of 0 is not allowed. + incref(interpid) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 1) + 1, get_refcount(interpid)) + + # When linked, decref back to 0 destroys the interpreter. + decref(interpid) + self.assertFalse( + exists(interpid)) + + def test_linked_lifecycle_incref_link(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount, incref, _ = self.get_refcount_helpers() - # The interpreter isn't linked to ID objects, so it isn't destroyed. - del id2 + interpid = self.new_interpreter() + + incref(interpid) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 0) + 1, get_refcount(interpid)) - _testcapi.link_interpreter_refcount(id1) + # Linking it will not reset the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 0) + 1, get_refcount(interpid)) + + def test_linked_lifecycle_link_incref_unlink_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() - id3 = self.InterpreterID(id1) + interpid = self.new_interpreter() + + link(interpid) + self.assertTrue( + is_linked(interpid)) + + incref(interpid) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 1) + 1, get_refcount(interpid)) - # The interpreter is linked now so is destroyed. - del id3 - with self.assertRaises(_interpreters.InterpreterNotFoundError): - _testinternalcapi.get_interpreter_refcount(id1) + # Unlinking it will not change the refcount. + unlink(interpid) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + # Unlinked: decref back to 0 does not destroys the interpreter. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) class BuiltinStaticTypesTests(unittest.TestCase): @@ -2245,6 +2973,22 @@ def callback(): t.start() t.join() + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_thread_gilstate_in_clear(self): + # See https://github.com/python/cpython/issues/119585 + class C: + def __del__(self): + _testcapi.gilstate_ensure_release() + + # Thread-local variables are destroyed in `PyThreadState_Clear()`. + local_var = threading.local() + + def callback(): + local_var.x = C() + + _testcapi._test_thread_state(callback) + @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_gilstate_ensure_no_deadlock(self): @@ -2264,25 +3008,36 @@ def test_gilstate_matches_current(self): _testcapi.test_current_tstate_matches() +def get_test_funcs(mod, exclude_prefix=None): + funcs = {} + for name in dir(mod): + if not name.startswith('test_'): + continue + if exclude_prefix is not None and name.startswith(exclude_prefix): + continue + funcs[name] = getattr(mod, name) + return funcs + + class Test_testcapi(unittest.TestCase): - locals().update((name, getattr(_testcapi, name)) - for name in dir(_testcapi) - if name.startswith('test_')) + locals().update(get_test_funcs(_testcapi)) # Suppress warning from PyUnicode_FromUnicode(). @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_widechar(self): - _testcapi.test_widechar() + _testlimitedcapi.test_widechar() def test_version_api_data(self): self.assertEqual(_testcapi.Py_Version, sys.hexversion) +class Test_testlimitedcapi(unittest.TestCase): + locals().update(get_test_funcs(_testlimitedcapi)) + + class Test_testinternalcapi(unittest.TestCase): - locals().update((name, getattr(_testinternalcapi, name)) - for name in dir(_testinternalcapi) - if name.startswith('test_') - and not name.startswith('test_lock_')) + locals().update(get_test_funcs(_testinternalcapi, + exclude_prefix='test_lock_')) @threading_helper.requires_working_threading() @@ -2303,7 +3058,12 @@ class Test_ModuleStateAccess(unittest.TestCase): def setUp(self): fullname = '_testmultiphase_meth_state_access' # XXX origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = importlib.machinery.AppleFrameworkLoader(fullname, origin) + else: + loader = importlib.machinery.ExtensionFileLoader(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) diff --git a/Lib/test/test_capi/test_number.py b/Lib/test/test_capi/test_number.py new file mode 100644 index 00000000000000..04f85d2395083a --- /dev/null +++ b/Lib/test/test_capi/test_number.py @@ -0,0 +1,320 @@ +import itertools +import operator +import sys +import unittest +import warnings + +from test.support import cpython_only, import_helper + +_testcapi = import_helper.import_module('_testcapi') +from _testcapi import PY_SSIZE_T_MAX, PY_SSIZE_T_MIN + +try: + from _testbuffer import ndarray +except ImportError: + ndarray = None + +NULL = None + +class BadDescr: + def __get__(self, obj, objtype=None): + raise RuntimeError + +class WithDunder: + def _meth(self, *args): + if self.val: + return self.val + if self.exc: + raise self.exc + @classmethod + def with_val(cls, val): + obj = super().__new__(cls) + obj.val = val + obj.exc = None + setattr(cls, cls.methname, cls._meth) + return obj + + @classmethod + def with_exc(cls, exc): + obj = super().__new__(cls) + obj.val = None + obj.exc = exc + setattr(cls, cls.methname, cls._meth) + return obj + +class HasBadAttr: + def __new__(cls): + obj = super().__new__(cls) + setattr(cls, cls.methname, BadDescr()) + return obj + + +class IndexLike(WithDunder): + methname = '__index__' + +class IntLike(WithDunder): + methname = '__int__' + +class FloatLike(WithDunder): + methname = '__float__' + + +def subclassof(base): + return type(base.__name__ + 'Subclass', (base,), {}) + + +class SomeError(Exception): + pass + +class OtherError(Exception): + pass + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyNumber_Check() + check = _testcapi.number_check + + self.assertTrue(check(1)) + self.assertTrue(check(IndexLike.with_val(1))) + self.assertTrue(check(IntLike.with_val(99))) + self.assertTrue(check(0.5)) + self.assertTrue(check(FloatLike.with_val(4.25))) + self.assertTrue(check(1+2j)) + + self.assertFalse(check([])) + self.assertFalse(check("abc")) + self.assertFalse(check(object())) + self.assertFalse(check(NULL)) + + def test_unary_ops(self): + methmap = {'__neg__': _testcapi.number_negative, # PyNumber_Negative() + '__pos__': _testcapi.number_positive, # PyNumber_Positive() + '__abs__': _testcapi.number_absolute, # PyNumber_Absolute() + '__invert__': _testcapi.number_invert} # PyNumber_Invert() + + for name, func in methmap.items(): + # Generic object, has no tp_as_number structure + self.assertRaises(TypeError, func, object()) + + # C-API function accepts NULL + self.assertRaises(SystemError, func, NULL) + + # Behave as corresponding unary operation + op = getattr(operator, name) + for x in [0, 42, -1, 3.14, 1+2j]: + try: + op(x) + except TypeError: + self.assertRaises(TypeError, func, x) + else: + self.assertEqual(func(x), op(x)) + + def test_binary_ops(self): + methmap = {'__add__': _testcapi.number_add, # PyNumber_Add() + '__sub__': _testcapi.number_subtract, # PyNumber_Subtract() + '__mul__': _testcapi.number_multiply, # PyNumber_Multiply() + '__matmul__': _testcapi.number_matrixmultiply, # PyNumber_MatrixMultiply() + '__floordiv__': _testcapi.number_floordivide, # PyNumber_FloorDivide() + '__truediv__': _testcapi.number_truedivide, # PyNumber_TrueDivide() + '__mod__': _testcapi.number_remainder, # PyNumber_Remainder() + '__divmod__': _testcapi.number_divmod, # PyNumber_Divmod() + '__lshift__': _testcapi.number_lshift, # PyNumber_Lshift() + '__rshift__': _testcapi.number_rshift, # PyNumber_Rshift() + '__and__': _testcapi.number_and, # PyNumber_And() + '__xor__': _testcapi.number_xor, # PyNumber_Xor() + '__or__': _testcapi.number_or, # PyNumber_Or() + '__pow__': _testcapi.number_power, # PyNumber_Power() + '__iadd__': _testcapi.number_inplaceadd, # PyNumber_InPlaceAdd() + '__isub__': _testcapi.number_inplacesubtract, # PyNumber_InPlaceSubtract() + '__imul__': _testcapi.number_inplacemultiply, # PyNumber_InPlaceMultiply() + '__imatmul__': _testcapi.number_inplacematrixmultiply, # PyNumber_InPlaceMatrixMultiply() + '__ifloordiv__': _testcapi.number_inplacefloordivide, # PyNumber_InPlaceFloorDivide() + '__itruediv__': _testcapi.number_inplacetruedivide, # PyNumber_InPlaceTrueDivide() + '__imod__': _testcapi.number_inplaceremainder, # PyNumber_InPlaceRemainder() + '__ilshift__': _testcapi.number_inplacelshift, # PyNumber_InPlaceLshift() + '__irshift__': _testcapi.number_inplacershift, # PyNumber_InPlaceRshift() + '__iand__': _testcapi.number_inplaceand, # PyNumber_InPlaceAnd() + '__ixor__': _testcapi.number_inplacexor, # PyNumber_InPlaceXor() + '__ior__': _testcapi.number_inplaceor, # PyNumber_InPlaceOr() + '__ipow__': _testcapi.number_inplacepower, # PyNumber_InPlacePower() + } + + for name, func in methmap.items(): + cases = [0, 42, 3.14, -1, 123, 1+2j] + + # Generic object, has no tp_as_number structure + for x in cases: + self.assertRaises(TypeError, func, object(), x) + self.assertRaises(TypeError, func, x, object()) + + # Behave as corresponding binary operation + op = getattr(operator, name, divmod) + for x, y in itertools.combinations(cases, 2): + try: + op(x, y) + except (TypeError, ValueError, ZeroDivisionError) as exc: + self.assertRaises(exc.__class__, func, x, y) + else: + self.assertEqual(func(x, y), op(x, y)) + + # CRASHES func(NULL, object()) + # CRASHES func(object(), NULL) + + @unittest.skipIf(ndarray is None, "needs _testbuffer") + def test_misc_add(self): + # PyNumber_Add(), PyNumber_InPlaceAdd() + add = _testcapi.number_add + inplaceadd = _testcapi.number_inplaceadd + + # test sq_concat/sq_inplace_concat slots + a, b, r = [1, 2], [3, 4], [1, 2, 3, 4] + self.assertEqual(add(a, b), r) + self.assertEqual(a, [1, 2]) + self.assertRaises(TypeError, add, ndarray([1], (1,)), 2) + a, b, r = [1, 2], [3, 4], [1, 2, 3, 4] + self.assertEqual(inplaceadd(a, b), r) + self.assertEqual(a, r) + self.assertRaises(TypeError, inplaceadd, ndarray([1], (1,)), 2) + + @unittest.skipIf(ndarray is None, "needs _testbuffer") + def test_misc_multiply(self): + # PyNumber_Multiply(), PyNumber_InPlaceMultiply() + multiply = _testcapi.number_multiply + inplacemultiply = _testcapi.number_inplacemultiply + + # test sq_repeat/sq_inplace_repeat slots + a, b, r = [1], 2, [1, 1] + self.assertEqual(multiply(a, b), r) + self.assertEqual((a, b), ([1], 2)) + self.assertEqual(multiply(b, a), r) + self.assertEqual((a, b), ([1], 2)) + self.assertEqual(multiply([1], -1), []) + self.assertRaises(TypeError, multiply, ndarray([1], (1,)), 2) + self.assertRaises(TypeError, multiply, [1], 0.5) + self.assertRaises(OverflowError, multiply, [1], PY_SSIZE_T_MAX + 1) + self.assertRaises(MemoryError, multiply, [1, 2], PY_SSIZE_T_MAX//2 + 1) + a, b, r = [1], 2, [1, 1] + self.assertEqual(inplacemultiply(a, b), r) + self.assertEqual((a, b), (r, 2)) + a = [1] + self.assertEqual(inplacemultiply(b, a), r) + self.assertEqual((a, b), ([1], 2)) + self.assertRaises(TypeError, inplacemultiply, ndarray([1], (1,)), 2) + self.assertRaises(OverflowError, inplacemultiply, [1], PY_SSIZE_T_MAX + 1) + self.assertRaises(MemoryError, inplacemultiply, [1, 2], PY_SSIZE_T_MAX//2 + 1) + + def test_misc_power(self): + # PyNumber_Power() + power = _testcapi.number_power + + class HasPow(WithDunder): + methname = '__pow__' + + # ternary op + self.assertEqual(power(4, 11, 5), pow(4, 11, 5)) + self.assertRaises(TypeError, power, 4, 11, 1.25) + self.assertRaises(TypeError, power, 4, 11, HasPow.with_val(NotImplemented)) + self.assertRaises(TypeError, power, 4, 11, object()) + + def test_long(self): + # Test PyNumber_Long() + long = _testcapi.number_long + + self.assertEqual(long(42), 42) + self.assertEqual(long(1.25), 1) + self.assertEqual(long("42"), 42) + self.assertEqual(long(b"42"), 42) + self.assertEqual(long(bytearray(b"42")), 42) + self.assertEqual(long(memoryview(b"42")), 42) + self.assertEqual(long(IndexLike.with_val(99)), 99) + self.assertEqual(long(IntLike.with_val(99)), 99) + + self.assertRaises(TypeError, long, IntLike.with_val(1.0)) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, long, IntLike.with_val(True)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(long(IntLike.with_val(True)), 1) + self.assertRaises(RuntimeError, long, IntLike.with_exc(RuntimeError)) + + self.assertRaises(TypeError, long, 1j) + self.assertRaises(TypeError, long, object()) + self.assertRaises(SystemError, long, NULL) + + def test_float(self): + # Test PyNumber_Float() + float_ = _testcapi.number_float + + self.assertEqual(float_(1.25), 1.25) + self.assertEqual(float_(123), 123.) + self.assertEqual(float_("1.25"), 1.25) + + self.assertEqual(float_(FloatLike.with_val(4.25)), 4.25) + self.assertEqual(float_(IndexLike.with_val(99)), 99.0) + self.assertEqual(float_(IndexLike.with_val(-1)), -1.0) + + self.assertRaises(TypeError, float_, FloatLike.with_val(687)) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, float_, FloatLike.with_val(subclassof(float)(4.25))) + with self.assertWarns(DeprecationWarning): + self.assertEqual(float_(FloatLike.with_val(subclassof(float)(4.25))), 4.25) + self.assertRaises(RuntimeError, float_, FloatLike.with_exc(RuntimeError)) + + self.assertRaises(TypeError, float_, IndexLike.with_val(1.25)) + self.assertRaises(OverflowError, float_, IndexLike.with_val(2**2000)) + + self.assertRaises(TypeError, float_, 1j) + self.assertRaises(TypeError, float_, object()) + self.assertRaises(SystemError, float_, NULL) + + def test_index(self): + # Test PyNumber_Index() + index = _testcapi.number_index + + self.assertEqual(index(11), 11) + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, index, IndexLike.with_val(True)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(index(IndexLike.with_val(True)), 1) + self.assertRaises(TypeError, index, IndexLike.with_val(1.0)) + self.assertRaises(RuntimeError, index, IndexLike.with_exc(RuntimeError)) + + self.assertRaises(TypeError, index, 1.25) + self.assertRaises(TypeError, index, "42") + self.assertRaises(TypeError, index, object()) + self.assertRaises(SystemError, index, NULL) + + def test_tobase(self): + # Test PyNumber_ToBase() + tobase = _testcapi.number_tobase + + self.assertEqual(tobase(10, 2), bin(10)) + self.assertEqual(tobase(11, 8), oct(11)) + self.assertEqual(tobase(16, 10), str(16)) + self.assertEqual(tobase(13, 16), hex(13)) + + self.assertRaises(SystemError, tobase, NULL, 2) + self.assertRaises(SystemError, tobase, 2, 3) + self.assertRaises(TypeError, tobase, 1.25, 2) + self.assertRaises(TypeError, tobase, "42", 2) + + def test_asssizet(self): + # Test PyNumber_AsSsize_t() + asssizet = _testcapi.number_asssizet + + for n in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + self.assertEqual(asssizet(n, OverflowError), n) + self.assertEqual(asssizet(PY_SSIZE_T_MAX+10, NULL), PY_SSIZE_T_MAX) + self.assertEqual(asssizet(PY_SSIZE_T_MIN-10, NULL), PY_SSIZE_T_MIN) + + self.assertRaises(OverflowError, asssizet, PY_SSIZE_T_MAX + 10, OverflowError) + self.assertRaises(RuntimeError, asssizet, PY_SSIZE_T_MAX + 10, RuntimeError) + self.assertRaises(SystemError, asssizet, NULL, TypeError) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py new file mode 100644 index 00000000000000..cc9c9b688f00e2 --- /dev/null +++ b/Lib/test/test_capi/test_object.py @@ -0,0 +1,135 @@ +import enum +import unittest +from test.support import import_helper +from test.support import os_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testcapi = import_helper.import_module('_testcapi') + + +class Constant(enum.IntEnum): + Py_CONSTANT_NONE = 0 + Py_CONSTANT_FALSE = 1 + Py_CONSTANT_TRUE = 2 + Py_CONSTANT_ELLIPSIS = 3 + Py_CONSTANT_NOT_IMPLEMENTED = 4 + Py_CONSTANT_ZERO = 5 + Py_CONSTANT_ONE = 6 + Py_CONSTANT_EMPTY_STR = 7 + Py_CONSTANT_EMPTY_BYTES = 8 + Py_CONSTANT_EMPTY_TUPLE = 9 + + INVALID_CONSTANT = Py_CONSTANT_EMPTY_TUPLE + 1 + + +class GetConstantTest(unittest.TestCase): + def check_get_constant(self, get_constant): + self.assertIs(get_constant(Constant.Py_CONSTANT_NONE), None) + self.assertIs(get_constant(Constant.Py_CONSTANT_FALSE), False) + self.assertIs(get_constant(Constant.Py_CONSTANT_TRUE), True) + self.assertIs(get_constant(Constant.Py_CONSTANT_ELLIPSIS), Ellipsis) + self.assertIs(get_constant(Constant.Py_CONSTANT_NOT_IMPLEMENTED), NotImplemented) + + for constant_id, constant_type, value in ( + (Constant.Py_CONSTANT_ZERO, int, 0), + (Constant.Py_CONSTANT_ONE, int, 1), + (Constant.Py_CONSTANT_EMPTY_STR, str, ""), + (Constant.Py_CONSTANT_EMPTY_BYTES, bytes, b""), + (Constant.Py_CONSTANT_EMPTY_TUPLE, tuple, ()), + ): + with self.subTest(constant_id=constant_id): + obj = get_constant(constant_id) + self.assertEqual(type(obj), constant_type, obj) + self.assertEqual(obj, value) + + with self.assertRaises(SystemError): + get_constant(Constant.INVALID_CONSTANT) + + def test_get_constant(self): + self.check_get_constant(_testlimitedcapi.get_constant) + + def test_get_constant_borrowed(self): + self.check_get_constant(_testlimitedcapi.get_constant_borrowed) + + +class PrintTest(unittest.TestCase): + def testPyObjectPrintObject(self): + + class PrintableObject: + + def __repr__(self): + return "spam spam spam" + + def __str__(self): + return "egg egg egg" + + obj = PrintableObject() + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.call_pyobject_print(obj, output_filename, False) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), repr(obj)) + + # Test str printing + _testcapi.call_pyobject_print(obj, output_filename, True) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), str(obj)) + + def testPyObjectPrintNULL(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.pyobject_print_null(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), '') + + def testPyObjectPrintNoRefObject(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + correct_output = _testcapi.pyobject_print_noref_object(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), correct_output) + + def testPyObjectPrintOSError(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + open(output_filename, "w+").close() + with self.assertRaises(OSError): + _testcapi.pyobject_print_os_error(output_filename) + + +class ClearWeakRefsNoCallbacksTest(unittest.TestCase): + """Test PyUnstable_Object_ClearWeakRefsNoCallbacks""" + def test_ClearWeakRefsNoCallbacks(self): + """Ensure PyUnstable_Object_ClearWeakRefsNoCallbacks works""" + import weakref + import gc + class C: + pass + obj = C() + messages = [] + ref = weakref.ref(obj, lambda: messages.append("don't add this")) + self.assertIs(ref(), obj) + self.assertFalse(messages) + _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) + self.assertIsNone(ref()) + gc.collect() + self.assertFalse(messages) + + def test_ClearWeakRefsNoCallbacks_no_weakref_support(self): + """Don't fail on objects that don't support weakrefs""" + import weakref + obj = object() + with self.assertRaises(TypeError): + ref = weakref.ref(obj) + _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 5c8c0596610303..f1ab72180d714d 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1,10 +1,17 @@ import contextlib -import opcode +import sys import textwrap import unittest +import gc +import os -import _testinternalcapi +import _opcode +from test.support import script_helper, requires_specialization, import_helper + +_testinternalcapi = import_helper.import_module("_testinternalcapi") + +from _testinternalcapi import TIER2_THRESHOLD @contextlib.contextmanager def temporary_optimizer(opt): @@ -26,18 +33,21 @@ def clear_executors(func): func.__code__ = func.__code__.replace() +@requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") class TestOptimizerAPI(unittest.TestCase): - def test_get_counter_optimizer_dealloc(self): + def test_new_counter_optimizer_dealloc(self): # See gh-108727 def f(): - _testinternalcapi.get_counter_optimizer() + _testinternalcapi.new_counter_optimizer() f() def test_get_set_optimizer(self): old = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() try: _testinternalcapi.set_optimizer(opt) self.assertEqual(_testinternalcapi.get_optimizer(), opt) @@ -58,12 +68,13 @@ def loop(): loop = ns['loop'] for repeat in range(5): - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() with temporary_optimizer(opt): self.assertEqual(opt.get_count(), 0) with clear_executors(loop): loop() - self.assertEqual(opt.get_count(), 1000) + # Subtract because optimizer doesn't kick in sooner + self.assertEqual(opt.get_count(), 1000 - TIER2_THRESHOLD) def test_long_loop(self): "Check that we aren't confused by EXTENDED_ARG" @@ -75,7 +86,7 @@ def nop(): pass def long_loop(): - for _ in range(10): + for _ in range(20): nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); @@ -86,11 +97,11 @@ def long_loop(): """), ns, ns) long_loop = ns['long_loop'] - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() with temporary_optimizer(opt): self.assertEqual(opt.get_count(), 0) long_loop() - self.assertEqual(opt.get_count(), 10) + self.assertEqual(opt.get_count(), 20 - TIER2_THRESHOLD) # Need iterations to warm up def test_code_restore_for_ENTER_EXECUTOR(self): def testfunc(x): @@ -98,7 +109,7 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() with temporary_optimizer(opt): testfunc(1000) code, replace_code = testfunc.__code__, testfunc.__code__.replace() @@ -109,21 +120,31 @@ def testfunc(x): def get_first_executor(func): code = func.__code__ co_code = code.co_code - JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] for i in range(0, len(co_code), 2): - if co_code[i] == JUMP_BACKWARD: - try: - return _testinternalcapi.get_executor(code, i) - except ValueError: - pass + try: + return _opcode.get_executor(code, i) + except ValueError: + pass return None +def iter_opnames(ex): + for item in ex: + yield item[0] + + +def get_opnames(ex): + return list(iter_opnames(ex)) + + +@requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") class TestExecutorInvalidation(unittest.TestCase): def setUp(self): self.old = _testinternalcapi.get_optimizer() - self.opt = _testinternalcapi.get_counter_optimizer() + self.opt = _testinternalcapi.new_counter_optimizer() _testinternalcapi.set_optimizer(self.opt) def tearDown(self): @@ -155,7 +176,7 @@ def f{n}(): self.assertTrue(exe.is_valid()) # Assert that the correct executors are invalidated # and check that nothing crashes when we invalidate - # an executor mutliple times. + # an executor multiple times. for i in (4,3,2,1,0): _testinternalcapi.invalidate_executors(objects[i]) for exe in executors[i:]: @@ -172,7 +193,7 @@ def f(): pass """), ns, ns) f = ns['f'] - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): f() exe = get_first_executor(f) @@ -181,6 +202,26 @@ def f(): _testinternalcapi.invalidate_executors(f.__code__) self.assertFalse(exe.is_valid()) + def test_sys__clear_internal_caches(self): + def f(): + for _ in range(1000): + pass + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + sys._clear_internal_caches() + self.assertFalse(exe.is_valid()) + exe = get_first_executor(f) + self.assertIsNone(exe) + + +@requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUops(unittest.TestCase): def test_basic_loop(self): @@ -189,15 +230,15 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(1000) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_SET_IP", uops) - self.assertIn("_LOAD_FAST", uops) + uops = get_opnames(ex) + self.assertIn("_JUMP_TO_TOP", uops) + self.assertIn("_LOAD_FAST_0", uops) def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" @@ -233,10 +274,11 @@ def many_vars(): z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 while z9 > 0: z9 = z9 - 1 + +z9 """), ns, ns) many_vars = ns["many_vars"] - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): ex = get_first_executor(many_vars) self.assertIsNone(ex) @@ -244,7 +286,8 @@ def many_vars(): ex = get_first_executor(many_vars) self.assertIsNotNone(ex) - self.assertIn(("_LOAD_FAST", 259, 0), list(ex)) + self.assertTrue(any((opcode, oparg, operand) == ("_LOAD_FAST", 259, 0) + for opcode, oparg, _, operand in list(ex))) def test_unspecialized_unpack(self): # An example of an unspecialized opcode @@ -258,14 +301,14 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_UNPACK_SEQUENCE", uops) def test_pop_jump_if_false(self): @@ -274,13 +317,13 @@ def testfunc(n): while i < n: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_TRUE_POP", uops) def test_pop_jump_if_none(self): @@ -289,14 +332,15 @@ def testfunc(a): if x is None: x = 0 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(range(20)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) def test_pop_jump_if_not_none(self): def testfunc(a): @@ -305,14 +349,15 @@ def testfunc(a): if x is not None: x = 0 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(range(20)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_NONE_POP", uops) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) def test_pop_jump_if_true(self): def testfunc(n): @@ -320,13 +365,13 @@ def testfunc(n): while not i >= n: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_FALSE_POP", uops) def test_jump_backward(self): @@ -335,13 +380,13 @@ def testfunc(n): while i < n: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_JUMP_TO_TOP", uops) def test_jump_forward(self): @@ -355,13 +400,13 @@ def testfunc(n): a += 1 return a - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) # Since there is no JUMP_FORWARD instruction, # look for indirect evidence: the += operator self.assertIn("_BINARY_OP_ADD_INT", uops) @@ -373,7 +418,7 @@ def testfunc(n): total += i return total - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): total = testfunc(20) self.assertEqual(total, 190) @@ -382,7 +427,7 @@ def testfunc(n): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -394,7 +439,7 @@ def testfunc(a): total += i return total - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): a = list(range(20)) total = testfunc(a) @@ -404,7 +449,7 @@ def testfunc(a): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -416,7 +461,7 @@ def testfunc(a): total += i return total - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): a = tuple(range(20)) total = testfunc(a) @@ -426,7 +471,7 @@ def testfunc(a): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -436,7 +481,7 @@ def testfunc(it): for x in it: pass - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): a = [1, 2, 3] it = iter(a) @@ -452,13 +497,13 @@ def dummy(x): for i in range(n): dummy(i) - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_PUSH_FRAME", uops) self.assertIn("_BINARY_OP_ADD_INT", uops) @@ -470,13 +515,13 @@ def testfunc(n): else: i = 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_FALSE_POP", uops) def test_for_iter_tier_two(self): @@ -498,7 +543,7 @@ def testfunc(n, m): x += 1000*i + j return x - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): x = testfunc(10, 10) @@ -506,7 +551,7 @@ def testfunc(n, m): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) def test_confidence_score(self): @@ -527,18 +572,914 @@ def testfunc(n): bits += 1 return bits - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): x = testfunc(20) self.assertEqual(x, 40) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - ops = [opname for opname, _, _ in ex] - count = ops.count("_GUARD_IS_TRUE_POP") - # Because Each 'if' halves the score, the second branch is - # too much already. - self.assertEqual(count, 1) + ops = list(iter_opnames(ex)) + #Since branch is 50/50 the trace could go either way. + count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") + self.assertLessEqual(count, 2) + + +@requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") +class TestUopsOptimization(unittest.TestCase): + + def _run_with_optimizer(self, testfunc, arg): + res = None + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + res = testfunc(arg) + + ex = get_first_executor(testfunc) + return res, ex + + + def test_int_type_propagation(self): + def testfunc(loops): + num = 0 + for i in range(loops): + x = num + num + a = x + 1 + num += 1 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertIsNotNone(ex) + self.assertEqual(res, 63) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_both_int_count), 1) + + def test_int_type_propagation_through_frame(self): + def double(x): + return x + x + def testfunc(loops): + num = 0 + for i in range(loops): + x = num + num + a = double(x) + num += 1 + return a + + opt = _testinternalcapi.new_uop_optimizer() + res = None + with temporary_optimizer(opt): + res = testfunc(32) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + self.assertEqual(res, 124) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_both_int_count), 1) + + def test_int_type_propagation_from_frame(self): + def double(x): + return x + x + def testfunc(loops): + num = 0 + for i in range(loops): + a = double(num) + x = a + a + num += 1 + return x + + opt = _testinternalcapi.new_uop_optimizer() + res = None + with temporary_optimizer(opt): + res = testfunc(32) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + self.assertEqual(res, 124) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_both_int_count), 1) + + def test_int_impure_region(self): + def testfunc(loops): + num = 0 + while num < loops: + x = num + num + y = 1 + x // 2 + a = x + y + num += 1 + return a + + res, ex = self._run_with_optimizer(testfunc, 64) + self.assertIsNotNone(ex) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + self.assertGreaterEqual(len(binop_count), 3) + + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + self.assertNotIn("_CHECK_PEP_523", uops) + + def test_int_type_propagate_through_range(self): + def testfunc(n): + + for i in range(n): + x = i + i + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 62) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_BOTH_INT", uops) + + def test_int_value_numbering(self): + def testfunc(n): + + y = 1 + for i in range(n): + x = y + z = x + a = z + b = a + res = x + z + a + b + return res + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 4) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_BOTH_INT", uops) + guard_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertEqual(len(guard_count), 1) + + def test_comprehension(self): + def testfunc(n): + for _ in range(n): + return [i for i in range(n)] + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, list(range(32))) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_BINARY_OP_ADD_INT", uops) + + def test_call_py_exact_args_disappearing(self): + def dummy(x): + return x+1 + + def testfunc(n): + for i in range(n): + dummy(i) + + opt = _testinternalcapi.new_uop_optimizer() + # Trigger specialization + testfunc(8) + with temporary_optimizer(opt): + del dummy + gc.collect() + + def dummy(x): + return x + 2 + testfunc(32) + + ex = get_first_executor(testfunc) + # Honestly as long as it doesn't crash it's fine. + # Whether we get an executor or not is non-deterministic, + # because it's decided by when the function is freed. + # This test is a little implementation specific. + + def test_promote_globals_to_constants(self): + + result = script_helper.run_python_until_end('-c', textwrap.dedent(""" + import _testinternalcapi + import opcode + import _opcode + + def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + for i in range(0, len(co_code), 2): + try: + return _opcode.get_executor(code, i) + except ValueError: + pass + return None + + def get_opnames(ex): + return {item[0] for item in ex} + + def testfunc(n): + for i in range(n): + x = range(i) + return x + + opt = _testinternalcapi.new_uop_optimizer() + _testinternalcapi.set_optimizer(opt) + testfunc(64) + + ex = get_first_executor(testfunc) + assert ex is not None + uops = get_opnames(ex) + assert "_LOAD_GLOBAL_BUILTINS" not in uops + assert "_LOAD_CONST_INLINE_BORROW_WITH_NULL" in uops + """)) + self.assertEqual(result[0].rc, 0, result) + + def test_float_add_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a + 0.25 + a = a + 0.25 + a = a + 0.25 + a = a + 0.25 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, 33.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + + def test_float_subtract_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a - 0.25 + a = a - 0.25 + a = a - 0.25 + a = a - 0.25 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, -31.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + + def test_float_multiply_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a * 1.0 + a = a * 1.0 + a = a * 1.0 + a = a * 1.0 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, 1.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + + def test_add_unicode_propagation(self): + def testfunc(n): + a = "" + for _ in range(n): + a + a + a + a + a + a + a + a + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, "") + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_UNICODE"] + self.assertLessEqual(len(guard_both_unicode_count), 1) + self.assertIn("_BINARY_OP_ADD_UNICODE", uops) + + def test_compare_op_type_propagation_float(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_FLOAT", uops) + + def test_compare_op_type_propagation_int(self): + def testfunc(n): + a = 1 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_both_int_count), 1) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_int_partial(self): + def testfunc(n): + a = 1 + for _ in range(n): + if a > 2: + x = 0 + if a < 2: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_left_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_left_int_count), 1) + self.assertEqual(len(guard_both_int_count), 0) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_float_partial(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + if a > 2.0: + x = 0 + if a < 2.0: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_left_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_left_float_count), 1) + self.assertEqual(len(guard_both_float_count), 0) + self.assertIn("_COMPARE_OP_FLOAT", uops) + + def test_compare_op_type_propagation_unicode(self): + def testfunc(n): + a = "" + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_UNICODE"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_STR", uops) + + def test_type_inconsistency(self): + ns = {} + src = textwrap.dedent(""" + def testfunc(n): + for i in range(n): + x = _test_global + _test_global + """) + exec(src, ns, ns) + testfunc = ns['testfunc'] + ns['_test_global'] = 0 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNone(ex) + ns['_test_global'] = 1 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_BOTH_INT", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + # Try again, but between the runs, set the global to a float. + # This should result in no executor the second time. + ns = {} + exec(src, ns, ns) + testfunc = ns['testfunc'] + ns['_test_global'] = 0 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNone(ex) + ns['_test_global'] = 3.14 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNone(ex) + + def test_combine_stack_space_checks_sequential(self): + def dummy12(x): + return x - 1 + def dummy13(y): + z = y + 2 + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(7) + c, d = dummy13(9) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 832) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # sequential calls: max(12, 13) == 13 + largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_nested(self): + def dummy12(x): + return x + 3 + def dummy15(y): + z = dummy12(y) + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b, c = dummy15(2) + a += b + c + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 224) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # nested calls: 15 + 12 == 27 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy12.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls(self): + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(5) + c, d, e = dummy18(2) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 800) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(12, 18 + max(12, 13)) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls_different_order(self): + # same as `several_calls` but with top-level calls reversed + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + c, d, e = dummy18(2) + b = dummy12(5) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 800) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(18 + max(12, 13), 12) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_complex(self): + def dummy0(x): + return x + def dummy1(x): + return dummy0(x) + def dummy2(x): + return dummy1(x) + def dummy3(x): + return dummy0(x) + def dummy4(x): + y = dummy0(x) + return dummy3(y) + def dummy5(x): + return dummy2(x) + def dummy6(x): + y = dummy5(x) + z = dummy0(y) + return dummy4(z) + def testfunc(n): + a = 0; + for _ in range(32): + b = dummy5(1) + c = dummy0(1) + d = dummy6(1) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 96) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 15) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 15) + + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy6.__code__) + + _testinternalcapi.get_co_framesize(dummy5.__code__) + + _testinternalcapi.get_co_framesize(dummy2.__code__) + + _testinternalcapi.get_co_framesize(dummy1.__code__) + + _testinternalcapi.get_co_framesize(dummy0.__code__) + ) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_large_framesize(self): + # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is + # actually doing its job. Note that the resulting trace hits + # UOP_MAX_TRACE_LENGTH, but since all _CHECK_STACK_SPACEs happen early, this + # test is still meaningful. + repetitions = 10000 + ns = {} + header = """ + def dummy_large(a0): + """ + body = "".join([f""" + a{n+1} = a{n} + 1 + """ for n in range(repetitions)]) + return_ = f""" + return a{repetitions-1} + """ + exec(textwrap.dedent(header + body + return_), ns, ns) + dummy_large = ns['dummy_large'] + + # this is something like: + # + # def dummy_large(a0): + # a1 = a0 + 1 + # a2 = a1 + 1 + # .... + # a9999 = a9998 + 1 + # return a9999 + + def dummy15(z): + y = dummy_large(z) + return y + 3 + + def testfunc(n): + b = 0 + for _ in range(n): + b += dummy15(7) + return b + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 32 * (repetitions + 9)) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + + # this hits a different case during trace projection in refcount test runs only, + # so we need to account for both possibilities + self.assertIn(uop_names.count("_CHECK_STACK_SPACE"), [0, 1]) + if uop_names.count("_CHECK_STACK_SPACE") == 0: + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy_large.__code__) + ) + else: + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_recursion(self): + def dummy15(x): + while x > 0: + return dummy15(x - 1) + return 42 + def testfunc(n): + a = 0 + for _ in range(n): + a += dummy15(n) + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 42 * 32) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 1) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_many_nested(self): + # overflow the trace_stack + def dummy_a(x): + return x + def dummy_b(x): + return dummy_a(x) + def dummy_c(x): + return dummy_b(x) + def dummy_d(x): + return dummy_c(x) + def dummy_e(x): + return dummy_d(x) + def dummy_f(x): + return dummy_e(x) + def dummy_g(x): + return dummy_f(x) + def dummy_h(x): + return dummy_g(x) + def testfunc(n): + a = 0 + for _ in range(n): + a += dummy_h(n) + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 32 * 32) + self.assertIsNone(ex) + + def test_return_generator(self): + def gen(): + yield None + def testfunc(n): + for i in range(n): + gen() + return i + res, ex = self._run_with_optimizer(testfunc, 20) + self.assertEqual(res, 19) + self.assertIsNotNone(ex) + self.assertIn("_RETURN_GENERATOR", get_opnames(ex)) + + def test_for_iter_gen(self): + def gen(n): + for i in range(n): + yield i + def testfunc(n): + g = gen(n) + s = 0 + for i in g: + s += i + return s + res, ex = self._run_with_optimizer(testfunc, 20) + self.assertEqual(res, 190) + self.assertIsNotNone(ex) + self.assertIn("_FOR_ITER_GEN_FRAME", get_opnames(ex)) + + def test_modified_local_is_seen_by_optimized_code(self): + l = sys._getframe().f_locals + a = 1 + s = 0 + for j in range(1 << 10): + a + a + l["xa"[j >> 9]] = 1.0 + s += a + self.assertIs(type(a), float) + self.assertIs(type(s), float) + self.assertEqual(s, 1024.0) + + def test_guard_type_version_removed(self): + def thing(a): + x = 0 + for _ in range(100): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_removed_inlined(self): + """ + Verify that the guard type version if we have an inlined function + """ + + def fn(): + pass + + def thing(a): + x = 0 + for _ in range(100): + x += a.attr + fn() + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_not_removed(self): + """ + Verify that the guard type version is not removed if we modify the class + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + # for the first 90 iterations we set the attribute on this dummy function which shouldn't + # trigger the type watcher + # then after 90 it should trigger it and stop optimizing + # Note that the code needs to be in this weird form so it's optimized inline without any control flow + setattr((Foo, Bar)[i < 90], "attr", 2) + x += a.attr + return x + + class Foo: + attr = 1 + + class Bar: + pass + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + + self.assertIsNotNone(ex) + self.assertEqual(res, 219) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 2) + + + @unittest.expectedFailure + def test_guard_type_version_not_removed_escaping(self): + """ + Verify that the guard type version is not removed if have an escaping function + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + # eval should be escaping and so should cause optimization to stop and preserve both type versions + eval("None") + x += a.attr + return x + + class Foo: + attr = 1 + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + # Note: This will actually be 1 for noe + # https://github.com/python/cpython/pull/119365#discussion_r1626220129 + self.assertEqual(guard_type_version_count, 2) + + + def test_guard_type_version_executor_invalidated(self): + """ + Verify that the executor is invalided on a type change. + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + self.assertEqual(res, 200) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_GUARD_TYPE_VERSION"), 1) + self.assertTrue(ex.is_valid()) + Foo.attr = 0 + self.assertFalse(ex.is_valid()) + + def test_type_version_doesnt_segfault(self): + """ + Tests that setting a type version doesn't cause a segfault when later looking at the stack. + """ + + # Minimized from mdp.py benchmark + + class A: + def __init__(self): + self.attr = {} + + def method(self, arg): + self.attr[arg] = None + + def fn(a): + for _ in range(100): + (_ for _ in []) + (_ for _ in [a.method(None)]) + + fn(A()) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py new file mode 100644 index 00000000000000..894f66b437a39c --- /dev/null +++ b/Lib/test/test_capi/test_run.py @@ -0,0 +1,118 @@ +import os +import unittest +from collections import UserDict +from test.support import import_helper +from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE + +NULL = None +_testcapi = import_helper.import_module('_testcapi') +Py_single_input = _testcapi.Py_single_input +Py_file_input = _testcapi.Py_file_input +Py_eval_input = _testcapi.Py_eval_input + + +class DictSubclass(dict): + pass + + +class CAPITest(unittest.TestCase): + # TODO: Test the following functions: + # + # PyRun_SimpleStringFlags + # PyRun_AnyFileExFlags + # PyRun_SimpleFileExFlags + # PyRun_InteractiveOneFlags + # PyRun_InteractiveOneObject + # PyRun_InteractiveLoopFlags + # PyRun_String (may be a macro) + # PyRun_AnyFile (may be a macro) + # PyRun_AnyFileEx (may be a macro) + # PyRun_AnyFileFlags (may be a macro) + # PyRun_SimpleString (may be a macro) + # PyRun_SimpleFile (may be a macro) + # PyRun_SimpleFileEx (may be a macro) + # PyRun_InteractiveOne (may be a macro) + # PyRun_InteractiveLoop (may be a macro) + # PyRun_File (may be a macro) + # PyRun_FileEx (may be a macro) + # PyRun_FileFlags (may be a macro) + + def test_run_stringflags(self): + # Test PyRun_StringFlags(). + # XXX: fopen() uses different path encoding than Python on Windows. + def run(s, *args): + return _testcapi.run_stringflags(s, Py_file_input, *args) + source = b'a\n' + + self.assertIsNone(run(b'a\n', dict(a=1))) + self.assertIsNone(run(b'a\n', dict(a=1), {})) + self.assertIsNone(run(b'a\n', {}, dict(a=1))) + self.assertIsNone(run(b'a\n', {}, UserDict(a=1))) + + self.assertRaises(NameError, run, b'a\n', {}) + self.assertRaises(NameError, run, b'a\n', {}, {}) + self.assertRaises(TypeError, run, b'a\n', dict(a=1), []) + self.assertRaises(TypeError, run, b'a\n', dict(a=1), 1) + + self.assertIsNone(run(b'a\n', DictSubclass(a=1))) + self.assertIsNone(run(b'a\n', DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, b'a\n', DictSubclass()) + + self.assertIsNone(run(b'\xc3\xa4\n', {'\xe4': 1})) + self.assertRaises(SyntaxError, run, b'\xe4\n', {}) + + self.assertRaises(SystemError, run, b'a\n', NULL) + self.assertRaises(SystemError, run, b'a\n', NULL, {}) + self.assertRaises(SystemError, run, b'a\n', NULL, dict(a=1)) + self.assertRaises(SystemError, run, b'a\n', UserDict()) + self.assertRaises(SystemError, run, b'a\n', UserDict(), {}) + self.assertRaises(SystemError, run, b'a\n', UserDict(), dict(a=1)) + + # CRASHES run(NULL, {}) + + def test_run_fileexflags(self): + # Test PyRun_FileExFlags(). + filename = os.fsencode(TESTFN if os.name != 'nt' else TESTFN_ASCII) + with open(filename, 'wb') as fp: + fp.write(b'a\n') + self.addCleanup(unlink, filename) + def run(*args): + return _testcapi.run_fileexflags(filename, Py_file_input, *args) + + self.assertIsNone(run(dict(a=1))) + self.assertIsNone(run(dict(a=1), {})) + self.assertIsNone(run({}, dict(a=1))) + self.assertIsNone(run({}, UserDict(a=1))) + self.assertIsNone(run(dict(a=1), {}, 1)) # closeit = True + + self.assertRaises(NameError, run, {}) + self.assertRaises(NameError, run, {}, {}) + self.assertRaises(TypeError, run, dict(a=1), []) + self.assertRaises(TypeError, run, dict(a=1), 1) + + self.assertIsNone(run(DictSubclass(a=1))) + self.assertIsNone(run(DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, DictSubclass()) + + self.assertRaises(SystemError, run, NULL) + self.assertRaises(SystemError, run, NULL, {}) + self.assertRaises(SystemError, run, NULL, dict(a=1)) + self.assertRaises(SystemError, run, UserDict()) + self.assertRaises(SystemError, run, UserDict(), {}) + self.assertRaises(SystemError, run, UserDict(), dict(a=1)) + + @unittest.skipUnless(TESTFN_UNDECODABLE, 'only works if there are undecodable paths') + @unittest.skipIf(os.name == 'nt', 'does not work on Windows') + def test_run_fileexflags_with_undecodable_filename(self): + run = _testcapi.run_fileexflags + try: + with open(TESTFN_UNDECODABLE, 'wb') as fp: + fp.write(b'a\n') + self.addCleanup(unlink, TESTFN_UNDECODABLE) + except OSError: + self.skipTest('undecodable paths are not supported') + self.assertIsNone(run(TESTFN_UNDECODABLE, Py_file_input, dict(a=1))) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_capi/test_set.py b/Lib/test/test_capi/test_set.py index 5235f81543e0b6..62d90a3f94326d 100644 --- a/Lib/test/test_capi/test_set.py +++ b/Lib/test/test_capi/test_set.py @@ -2,8 +2,10 @@ from test.support import import_helper -# Skip this test if the _testcapi or _testinternalcapi modules aren't available. +# Skip this test if the _testcapi, _testlimitedcapi or _testinternalcapi +# modules aren't available. _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') class set_subclass(set): @@ -23,7 +25,7 @@ def assertImmutable(self, action, *args): class TestSetCAPI(BaseSetTests, unittest.TestCase): def test_set_check(self): - check = _testcapi.set_check + check = _testlimitedcapi.set_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -33,7 +35,7 @@ def test_set_check(self): # CRASHES: check(NULL) def test_set_check_exact(self): - check = _testcapi.set_checkexact + check = _testlimitedcapi.set_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -43,7 +45,7 @@ def test_set_check_exact(self): # CRASHES: check(NULL) def test_frozenset_check(self): - check = _testcapi.frozenset_check + check = _testlimitedcapi.frozenset_check self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -53,7 +55,7 @@ def test_frozenset_check(self): # CRASHES: check(NULL) def test_frozenset_check_exact(self): - check = _testcapi.frozenset_checkexact + check = _testlimitedcapi.frozenset_checkexact self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -63,7 +65,7 @@ def test_frozenset_check_exact(self): # CRASHES: check(NULL) def test_anyset_check(self): - check = _testcapi.anyset_check + check = _testlimitedcapi.anyset_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -74,7 +76,7 @@ def test_anyset_check(self): # CRASHES: check(NULL) def test_anyset_check_exact(self): - check = _testcapi.anyset_checkexact + check = _testlimitedcapi.anyset_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -85,7 +87,7 @@ def test_anyset_check_exact(self): # CRASHES: check(NULL) def test_set_new(self): - set_new = _testcapi.set_new + set_new = _testlimitedcapi.set_new self.assertEqual(set_new().__class__, set) self.assertEqual(set_new(), set()) self.assertEqual(set_new((1, 1, 2)), {1, 2}) @@ -98,7 +100,7 @@ def test_set_new(self): set_new((1, {})) def test_frozenset_new(self): - frozenset_new = _testcapi.frozenset_new + frozenset_new = _testlimitedcapi.frozenset_new self.assertEqual(frozenset_new().__class__, frozenset) self.assertEqual(frozenset_new(), frozenset()) self.assertEqual(frozenset_new((1, 1, 2)), frozenset({1, 2})) @@ -111,7 +113,7 @@ def test_frozenset_new(self): frozenset_new((1, {})) def test_set_size(self): - get_size = _testcapi.set_size + get_size = _testlimitedcapi.set_size self.assertEqual(get_size(set()), 0) self.assertEqual(get_size(frozenset()), 0) self.assertEqual(get_size({1, 1, 2}), 2) @@ -134,7 +136,7 @@ def test_set_get_size(self): # CRASHES: get_size(object()) def test_set_contains(self): - contains = _testcapi.set_contains + contains = _testlimitedcapi.set_contains for cls in (set, frozenset, set_subclass, frozenset_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -147,7 +149,7 @@ def test_set_contains(self): # CRASHES: contains(NULL, NULL) def test_add(self): - add = _testcapi.set_add + add = _testlimitedcapi.set_add for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -165,7 +167,7 @@ def test_add(self): # CRASHES: add(NULL, NULL) def test_discard(self): - discard = _testcapi.set_discard + discard = _testlimitedcapi.set_discard for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -187,7 +189,7 @@ def test_discard(self): # CRASHES: discard(NULL, NULL) def test_pop(self): - pop = _testcapi.set_pop + pop = _testlimitedcapi.set_pop orig = (1, 2) for cls in (set, set_subclass): with self.subTest(cls=cls): @@ -204,7 +206,7 @@ def test_pop(self): # CRASHES: pop(NULL) def test_clear(self): - clear = _testcapi.set_clear + clear = _testlimitedcapi.set_clear for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -263,3 +265,7 @@ def test_set_next_entry(self): with self.assertRaises(SystemError): set_next(object(), 0) # CRASHES: set_next(NULL, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index a294c3b13a5c30..6b27dc512a7d15 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -1,6 +1,5 @@ import unittest from test.support import import_helper -from test.support import warnings_helper # Skip this test if the _testcapi module isn't available. import_helper.import_module('_testcapi') @@ -81,36 +80,22 @@ def _test_int_range(self, name, minval, maxval, *, hardlimit=None, self._test_warn(name, maxval+1, minval) self._test_warn(name, hardmaxval) - if indexlimit is None: - indexlimit = hardlimit - if not indexlimit: + if indexlimit is False: self.assertRaises(TypeError, setattr, ts, name, Index(minval)) self.assertRaises(TypeError, setattr, ts, name, Index(maxval)) else: - hardminindexval, hardmaxindexval = indexlimit self._test_write(name, Index(minval), minval) - if minval < hardminindexval: - self._test_write(name, Index(hardminindexval), hardminindexval) - if maxval < hardmaxindexval: - self._test_write(name, Index(maxval), maxval) - else: - self._test_write(name, Index(hardmaxindexval), hardmaxindexval) - self._test_overflow(name, Index(hardminindexval-1)) - if name in ('T_UINT', 'T_ULONG'): - self.assertRaises(TypeError, setattr, self.ts, name, - Index(hardmaxindexval+1)) - self.assertRaises(TypeError, setattr, self.ts, name, - Index(2**1000)) - else: - self._test_overflow(name, Index(hardmaxindexval+1)) - self._test_overflow(name, Index(2**1000)) + self._test_write(name, Index(maxval), maxval) + self._test_overflow(name, Index(hardminval-1)) + self._test_overflow(name, Index(hardmaxval+1)) + self._test_overflow(name, Index(2**1000)) self._test_overflow(name, Index(-2**1000)) - if hardminindexval < minval and name != 'T_ULONGLONG': - self._test_warn(name, Index(hardminindexval)) - self._test_warn(name, Index(minval-1)) - if maxval < hardmaxindexval: - self._test_warn(name, Index(maxval+1)) - self._test_warn(name, Index(hardmaxindexval)) + if hardminval < minval: + self._test_warn(name, Index(hardminval)) + self._test_warn(name, Index(minval-1), maxval) + if maxval < hardmaxval: + self._test_warn(name, Index(maxval+1), minval) + self._test_warn(name, Index(hardmaxval)) def test_bool(self): ts = self.ts @@ -138,14 +123,12 @@ def test_int(self): self._test_int_range('T_INT', INT_MIN, INT_MAX, hardlimit=(LONG_MIN, LONG_MAX)) self._test_int_range('T_UINT', 0, UINT_MAX, - hardlimit=(LONG_MIN, ULONG_MAX), - indexlimit=(LONG_MIN, LONG_MAX)) + hardlimit=(LONG_MIN, ULONG_MAX)) def test_long(self): self._test_int_range('T_LONG', LONG_MIN, LONG_MAX) self._test_int_range('T_ULONG', 0, ULONG_MAX, - hardlimit=(LONG_MIN, ULONG_MAX), - indexlimit=(LONG_MIN, LONG_MAX)) + hardlimit=(LONG_MIN, ULONG_MAX)) def test_py_ssize_t(self): self._test_int_range('T_PYSSIZET', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, indexlimit=False) @@ -153,7 +136,7 @@ def test_py_ssize_t(self): def test_longlong(self): self._test_int_range('T_LONGLONG', LLONG_MIN, LLONG_MAX) self._test_int_range('T_ULONGLONG', 0, ULLONG_MAX, - indexlimit=(LONG_MIN, LONG_MAX)) + hardlimit=(LONG_MIN, ULLONG_MAX)) def test_bad_assignments(self): ts = self.ts diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py index 08bf0370fc59b5..54a8e026d883d4 100644 --- a/Lib/test/test_capi/test_sys.py +++ b/Lib/test/test_capi/test_sys.py @@ -5,9 +5,9 @@ from test.support import import_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None NULL = None @@ -20,10 +20,10 @@ class CAPITest(unittest.TestCase): maxDiff = None @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getobject(self): # Test PySys_GetObject() - getobject = _testcapi.sys_getobject + getobject = _testlimitedcapi.sys_getobject self.assertIs(getobject(b'stdout'), sys.stdout) with support.swap_attr(sys, '\U0001f40d', 42): @@ -38,10 +38,10 @@ def test_sys_getobject(self): # CRASHES getobject(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_setobject(self): # Test PySys_SetObject() - setobject = _testcapi.sys_setobject + setobject = _testlimitedcapi.sys_setobject value = ['value'] value2 = ['value2'] @@ -70,10 +70,10 @@ def test_sys_setobject(self): # CRASHES setobject(NULL, value) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getxoptions(self): # Test PySys_GetXOptions() - getxoptions = _testcapi.sys_getxoptions + getxoptions = _testlimitedcapi.sys_getxoptions self.assertIs(getxoptions(), sys._xoptions) diff --git a/Lib/test/test_capi/test_time.py b/Lib/test/test_capi/test_time.py new file mode 100644 index 00000000000000..989c158818ccf0 --- /dev/null +++ b/Lib/test/test_capi/test_time.py @@ -0,0 +1,78 @@ +import time +import unittest +from test.support import import_helper +_testcapi = import_helper.import_module('_testcapi') + + +PyTime_MIN = _testcapi.PyTime_MIN +PyTime_MAX = _testcapi.PyTime_MAX +SEC_TO_NS = 10 ** 9 +DAY_TO_SEC = (24 * 60 * 60) +# Worst clock resolution: maximum delta between two clock reads. +CLOCK_RES = 0.050 + + +class CAPITest(unittest.TestCase): + def test_min_max(self): + # PyTime_t is just int64_t + self.assertEqual(PyTime_MIN, -2**63) + self.assertEqual(PyTime_MAX, 2**63 - 1) + + def test_assecondsdouble(self): + # Test PyTime_AsSecondsDouble() + def ns_to_sec(ns): + if abs(ns) % SEC_TO_NS == 0: + return float(ns // SEC_TO_NS) + else: + return float(ns) / SEC_TO_NS + + seconds = ( + 0, + 1, + DAY_TO_SEC, + 365 * DAY_TO_SEC, + ) + values = { + PyTime_MIN, + PyTime_MIN + 1, + PyTime_MAX - 1, + PyTime_MAX, + } + for second in seconds: + ns = second * SEC_TO_NS + values.add(ns) + # test nanosecond before/after to test rounding + values.add(ns - 1) + values.add(ns + 1) + for ns in list(values): + if (-ns) > PyTime_MAX: + continue + values.add(-ns) + for ns in sorted(values): + with self.subTest(ns=ns): + self.assertEqual(_testcapi.PyTime_AsSecondsDouble(ns), + ns_to_sec(ns)) + + def check_clock(self, c_func, py_func): + t1 = c_func() + t2 = py_func() + self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) + + def test_monotonic(self): + # Test PyTime_Monotonic() and PyTime_MonotonicRaw() + self.check_clock(_testcapi.PyTime_Monotonic, time.monotonic) + self.check_clock(_testcapi.PyTime_MonotonicRaw, time.monotonic) + + def test_perf_counter(self): + # Test PyTime_PerfCounter() and PyTime_PerfCounterRaw() + self.check_clock(_testcapi.PyTime_PerfCounter, time.perf_counter) + self.check_clock(_testcapi.PyTime_PerfCounterRaw, time.perf_counter) + + def test_time(self): + # Test PyTime_Time() and PyTime_TimeRaw() + self.check_clock(_testcapi.PyTime_Time, time.time) + self.check_clock(_testcapi.PyTime_TimeRaw, time.time) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py new file mode 100644 index 00000000000000..e6b49caeb51f32 --- /dev/null +++ b/Lib/test/test_capi/test_tuple.py @@ -0,0 +1,261 @@ +import unittest +import sys +from collections import namedtuple +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None +PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN +PY_SSIZE_T_MAX = _testcapi.PY_SSIZE_T_MAX + +class TupleSubclass(tuple): + pass + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyTuple_Check() + check = _testlimitedcapi.tuple_check + + self.assertTrue(check((1, 2))) + self.assertTrue(check(())) + self.assertTrue(check(TupleSubclass((1, 2)))) + self.assertFalse(check({1: 2})) + self.assertFalse(check([1, 2])) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_tuple_checkexact(self): + # Test PyTuple_CheckExact() + check = _testlimitedcapi.tuple_checkexact + + self.assertTrue(check((1, 2))) + self.assertTrue(check(())) + self.assertFalse(check(TupleSubclass((1, 2)))) + self.assertFalse(check({1: 2})) + self.assertFalse(check([1, 2])) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_tuple_new(self): + # Test PyTuple_New() + tuple_new = _testlimitedcapi.tuple_new + size = _testlimitedcapi.tuple_size + checknull = _testcapi._check_tuple_item_is_NULL + + tup1 = tuple_new(0) + self.assertEqual(tup1, ()) + self.assertEqual(size(tup1), 0) + self.assertIs(type(tup1), tuple) + tup2 = tuple_new(1) + self.assertIs(type(tup2), tuple) + self.assertEqual(size(tup2), 1) + self.assertIsNot(tup2, tup1) + self.assertTrue(checknull(tup2, 0)) + + self.assertRaises(SystemError, tuple_new, -1) + self.assertRaises(SystemError, tuple_new, PY_SSIZE_T_MIN) + self.assertRaises(MemoryError, tuple_new, PY_SSIZE_T_MAX) + + def test_tuple_pack(self): + # Test PyTuple_Pack() + pack = _testlimitedcapi.tuple_pack + + self.assertEqual(pack(0), ()) + self.assertEqual(pack(1, [1]), ([1],)) + self.assertEqual(pack(2, [1], [2]), ([1], [2])) + + self.assertRaises(SystemError, pack, PY_SSIZE_T_MIN) + self.assertRaises(SystemError, pack, -1) + self.assertRaises(MemoryError, pack, PY_SSIZE_T_MAX) + + # CRASHES pack(1, NULL) + # CRASHES pack(2, [1]) + + def test_tuple_size(self): + # Test PyTuple_Size() + size = _testlimitedcapi.tuple_size + + self.assertEqual(size(()), 0) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size(TupleSubclass((1, 2))), 2) + + self.assertRaises(SystemError, size, []) + self.assertRaises(SystemError, size, 42) + self.assertRaises(SystemError, size, object()) + + # CRASHES size(NULL) + + def test_tuple_get_size(self): + # Test PyTuple_GET_SIZE() + size = _testcapi.tuple_get_size + + self.assertEqual(size(()), 0) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size(TupleSubclass((1, 2))), 2) + + def test_tuple_getitem(self): + # Test PyTuple_GetItem() + getitem = _testlimitedcapi.tuple_getitem + + tup = ([1], [2], [3]) + self.assertEqual(getitem(tup, 0), [1]) + self.assertEqual(getitem(tup, 2), [3]) + + tup2 = TupleSubclass(([1], [2], [3])) + self.assertEqual(getitem(tup2, 0), [1]) + self.assertEqual(getitem(tup2, 2), [3]) + + self.assertRaises(IndexError, getitem, tup, PY_SSIZE_T_MIN) + self.assertRaises(IndexError, getitem, tup, -1) + self.assertRaises(IndexError, getitem, tup, len(tup)) + self.assertRaises(IndexError, getitem, tup, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, getitem, [1, 2, 3], 1) + self.assertRaises(SystemError, getitem, 42, 1) + + # CRASHES getitem(NULL, 0) + + def test_tuple_get_item(self): + # Test PyTuple_GET_ITEM() + get_item = _testcapi.tuple_get_item + + tup = ([1], [2], [3]) + self.assertEqual(get_item(tup, 0), [1]) + self.assertEqual(get_item(tup, 2), [3]) + + tup2 = TupleSubclass(([1], [2], [3])) + self.assertEqual(get_item(tup2, 0), [1]) + self.assertEqual(get_item(tup2, 2), [3]) + + # CRASHES get_item(NULL, 0) + + def test_tuple_getslice(self): + # Test PyTuple_GetSlice() + getslice = _testlimitedcapi.tuple_getslice + + # empty + tup = ([1], [2], [3]) + self.assertEqual(getslice(tup, PY_SSIZE_T_MIN, 0), ()) + self.assertEqual(getslice(tup, -1, 0), ()) + self.assertEqual(getslice(tup, 3, PY_SSIZE_T_MAX), ()) + self.assertEqual(getslice(tup, 1, 1), ()) + self.assertEqual(getslice(tup, 2, 1), ()) + tup = TupleSubclass(([1], [2], [3])) + self.assertEqual(getslice(tup, PY_SSIZE_T_MIN, 0), ()) + self.assertEqual(getslice(tup, -1, 0), ()) + self.assertEqual(getslice(tup, 3, PY_SSIZE_T_MAX), ()) + self.assertEqual(getslice(tup, 1, 1), ()) + self.assertEqual(getslice(tup, 2, 1), ()) + + # slice + tup = ([1], [2], [3], [4]) + self.assertEqual(getslice(tup, 1, 3), ([2], [3])) + tup = TupleSubclass(([1], [2], [3], [4])) + self.assertEqual(getslice(tup, 1, 3), ([2], [3])) + + # whole + tup = ([1], [2], [3]) + self.assertEqual(getslice(tup, 0, 3), tup) + self.assertEqual(getslice(tup, 0, 100), tup) + self.assertEqual(getslice(tup, -100, 100), tup) + tup = TupleSubclass(([1], [2], [3])) + self.assertEqual(getslice(tup, 0, 3), tup) + self.assertEqual(getslice(tup, 0, 100), tup) + self.assertEqual(getslice(tup, -100, 100), tup) + + self.assertRaises(SystemError, getslice, [[1], [2], [3]], 0, 0) + self.assertRaises(SystemError, getslice, 42, 0, 0) + + # CRASHES getslice(NULL, 0, 0) + + def test_tuple_setitem(self): + # Test PyTuple_SetItem() + setitem = _testlimitedcapi.tuple_setitem + checknull = _testcapi._check_tuple_item_is_NULL + + tup = ([1], [2]) + self.assertEqual(setitem(tup, 0, []), ([], [2])) + self.assertEqual(setitem(tup, 1, []), ([1], [])) + + tup2 = setitem(tup, 1, NULL) + self.assertTrue(checknull(tup2, 1)) + + tup2 = TupleSubclass(([1], [2])) + self.assertRaises(SystemError, setitem, tup2, 0, []) + + self.assertRaises(IndexError, setitem, tup, PY_SSIZE_T_MIN, []) + self.assertRaises(IndexError, setitem, tup, -1, []) + self.assertRaises(IndexError, setitem, tup, len(tup), []) + self.assertRaises(IndexError, setitem, tup, PY_SSIZE_T_MAX, []) + self.assertRaises(SystemError, setitem, [1], 0, []) + self.assertRaises(SystemError, setitem, 42, 0, []) + + # CRASHES setitem(NULL, 0, []) + + def test_tuple_set_item(self): + # Test PyTuple_SET_ITEM() + set_item = _testcapi.tuple_set_item + checknull = _testcapi._check_tuple_item_is_NULL + + tup = ([1], [2]) + self.assertEqual(set_item(tup, 0, []), ([], [2])) + self.assertEqual(set_item(tup, 1, []), ([1], [])) + + tup2 = set_item(tup, 1, NULL) + self.assertTrue(checknull(tup2, 1)) + + tup2 = TupleSubclass(([1], [2])) + self.assertIs(set_item(tup2, 0, []), tup2) + self.assertEqual(tup2, ([], [2])) + + # CRASHES set_item(tup, -1, []) + # CRASHES set_item(tup, len(tup), []) + # CRASHES set_item([1], 0, []) + # CRASHES set_item(NULL, 0, []) + + def test__tuple_resize(self): + # Test _PyTuple_Resize() + resize = _testcapi._tuple_resize + checknull = _testcapi._check_tuple_item_is_NULL + + a = () + b = resize(a, 0, False) + self.assertEqual(len(a), 0) + self.assertEqual(len(b), 0) + b = resize(a, 2, False) + self.assertEqual(len(a), 0) + self.assertEqual(len(b), 2) + self.assertTrue(checknull(b, 0)) + self.assertTrue(checknull(b, 1)) + + a = ([1], [2], [3]) + b = resize(a, 3) + self.assertEqual(b, a) + b = resize(a, 2) + self.assertEqual(b, a[:2]) + b = resize(a, 5) + self.assertEqual(len(b), 5) + self.assertEqual(b[:3], a) + self.assertTrue(checknull(b, 3)) + self.assertTrue(checknull(b, 4)) + + a = () + self.assertRaises(MemoryError, resize, a, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, resize, a, -1) + self.assertRaises(SystemError, resize, a, PY_SSIZE_T_MIN) + # refcount > 1 + a = (1, 2, 3) + self.assertRaises(SystemError, resize, a, 3, False) + self.assertRaises(SystemError, resize, a, 0, False) + # non-tuple + self.assertRaises(SystemError, resize, [1, 2, 3], 0, False) + self.assertRaises(SystemError, resize, NULL, 0, False) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py new file mode 100644 index 00000000000000..54c83e09f892a0 --- /dev/null +++ b/Lib/test/test_capi/test_type.py @@ -0,0 +1,66 @@ +from test.support import import_helper +import unittest + +_testcapi = import_helper.import_module('_testcapi') + + +class TypeTests(unittest.TestCase): + def test_freeze(self): + # test PyType_Freeze() + type_freeze = _testcapi.type_freeze + + # simple case, no inherante + class MyType: + pass + MyType.attr = "mutable" + + type_freeze(MyType) + err_msg = "cannot set 'attr' attribute of immutable type 'MyType'" + with self.assertRaisesRegex(TypeError, err_msg): + # the class is now immutable + MyType.attr = "immutable" + + # test MRO: PyType_Freeze() requires base classes to be immutable + class A: pass + class B: pass + class C(B): pass + class D(A, C): pass + + self.assertEqual(D.mro(), [D, A, C, B, object]) + with self.assertRaises(TypeError): + type_freeze(D) + + type_freeze(A) + type_freeze(B) + type_freeze(C) + # all parent classes are now immutable, so D can be made immutable + # as well + type_freeze(D) + + def test_freeze_meta(self): + """test PyType_Freeze() with overridden MRO""" + type_freeze = _testcapi.type_freeze + + class Base: + value = 1 + + class Meta(type): + def mro(cls): + return (cls, Base, object) + + class FreezeThis(metaclass=Meta): + """This has `Base` in the MRO, but not tp_bases""" + + self.assertEqual(FreezeThis.value, 1) + + with self.assertRaises(TypeError): + type_freeze(FreezeThis) + + Base.value = 2 + self.assertEqual(FreezeThis.value, 2) + + type_freeze(Base) + with self.assertRaises(TypeError): + Base.value = 3 + type_freeze(FreezeThis) + self.assertEqual(FreezeThis.value, 2) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index bb6161abf4da81..65d8242ad3fc60 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -8,10 +8,18 @@ from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None try: import _testinternalcapi except ImportError: _testinternalcapi = None +try: + import ctypes +except ImportError: + ctypes = None NULL = None @@ -84,10 +92,10 @@ def test_fill(self): # TODO: Test PyUnicode_Fill() with non-modifiable unicode. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_writechar(self): """Test PyUnicode_WriteChar()""" - from _testcapi import unicode_writechar as writechar + from _testlimitedcapi import unicode_writechar as writechar strings = [ # one string for every kind @@ -115,10 +123,10 @@ def test_writechar(self): # unicode. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_resize(self): """Test PyUnicode_Resize()""" - from _testcapi import unicode_resize as resize + from _testlimitedcapi import unicode_resize as resize strings = [ # all strings have exactly 3 characters @@ -141,10 +149,10 @@ def test_resize(self): # and with NULL as the address. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_append(self): """Test PyUnicode_Append()""" - from _testcapi import unicode_append as append + from _testlimitedcapi import unicode_append as append strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -169,10 +177,10 @@ def test_append(self): # TODO: Check reference counts. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_appendanddel(self): """Test PyUnicode_AppendAndDel()""" - from _testcapi import unicode_appendanddel as appendanddel + from _testlimitedcapi import unicode_appendanddel as appendanddel strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -196,10 +204,10 @@ def test_appendanddel(self): # TODO: Check reference counts. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromstringandsize(self): """Test PyUnicode_FromStringAndSize()""" - from _testcapi import unicode_fromstringandsize as fromstringandsize + from _testlimitedcapi import unicode_fromstringandsize as fromstringandsize self.assertEqual(fromstringandsize(b'abc'), 'abc') self.assertEqual(fromstringandsize(b'abc', 2), 'ab') @@ -221,10 +229,10 @@ def test_fromstringandsize(self): self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MAX) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromstring(self): """Test PyUnicode_FromString()""" - from _testcapi import unicode_fromstring as fromstring + from _testlimitedcapi import unicode_fromstring as fromstring self.assertEqual(fromstring(b'abc'), 'abc') self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') @@ -273,10 +281,10 @@ def test_fromkindanddata(self): # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_substring(self): """Test PyUnicode_Substring()""" - from _testcapi import unicode_substring as substring + from _testlimitedcapi import unicode_substring as substring strings = [ 'ab', 'ab\xa1\xa2', @@ -297,10 +305,10 @@ def test_substring(self): # CRASHES substring(NULL, 0, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_getlength(self): """Test PyUnicode_GetLength()""" - from _testcapi import unicode_getlength as getlength + from _testlimitedcapi import unicode_getlength as getlength for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -311,10 +319,10 @@ def test_getlength(self): # CRASHES getlength(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_readchar(self): """Test PyUnicode_ReadChar()""" - from _testcapi import unicode_readchar as readchar + from _testlimitedcapi import unicode_readchar as readchar for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -330,10 +338,10 @@ def test_readchar(self): # CRASHES readchar(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromobject(self): """Test PyUnicode_FromObject()""" - from _testcapi import unicode_fromobject as fromobject + from _testlimitedcapi import unicode_fromobject as fromobject for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -348,13 +356,13 @@ def test_fromobject(self): self.assertRaises(TypeError, fromobject, []) # CRASHES fromobject(NULL) + @unittest.skipIf(ctypes is None, 'need ctypes') def test_from_format(self): """Test PyUnicode_FromFormat()""" # Length modifiers "j" and "t" are not tested here because ctypes does # not expose types for intmax_t and ptrdiff_t. - # _testcapi.test_string_from_format() has a wider coverage of all + # _testlimitedcapi.test_string_from_format() has a wider coverage of all # formats. - import_helper.import_module('ctypes') from ctypes import ( c_char_p, pythonapi, py_object, sizeof, @@ -411,8 +419,29 @@ def check_format(expected, format, *args): # truncated string check_format('abc', b'%.3s', b'abcdef') + check_format('abc[', + b'%.6s', 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\u20ac', + b'%.7s', 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\ufffd', + b'%.5s', b'abc[\xff]') + check_format('abc[', + b'%.6s', b'abc[\xe2\x82]') + check_format('abc[\ufffd]', + b'%.7s', b'abc[\xe2\x82]') check_format('abc[\ufffd', - b'%.5s', 'abc[\u20ac]'.encode('utf8')) + b'%.7s', b'abc[\xe2\x82\0') + check_format(' abc[', + b'%10.6s', 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\u20ac', + b'%10.7s', 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\ufffd', + b'%10.5s', b'abc[\xff]') + check_format(' abc[', + b'%10.6s', b'abc[\xe2\x82]') + check_format(' abc[\ufffd]', + b'%10.7s', b'abc[\xe2\x82]') + check_format("'\\u20acABC'", b'%A', '\u20acABC') check_format("'\\u20", @@ -425,10 +454,31 @@ def check_format(expected, format, *args): b'%.3S', '\u20acABCDEF') check_format('\u20acAB', b'%.3U', '\u20acABCDEF') + check_format('\u20acAB', b'%.3V', '\u20acABCDEF', None) + check_format('abc[', + b'%.6V', None, 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\u20ac', + b'%.7V', None, 'abc[\u20ac]'.encode('utf8')) check_format('abc[\ufffd', - b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) + b'%.5V', None, b'abc[\xff]') + check_format('abc[', + b'%.6V', None, b'abc[\xe2\x82]') + check_format('abc[\ufffd]', + b'%.7V', None, b'abc[\xe2\x82]') + check_format(' abc[', + b'%10.6V', None, 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\u20ac', + b'%10.7V', None, 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\ufffd', + b'%10.5V', None, b'abc[\xff]') + check_format(' abc[', + b'%10.6V', None, b'abc[\xe2\x82]') + check_format(' abc[\ufffd]', + b'%10.7V', None, b'abc[\xe2\x82]') + check_format(' abc[\ufffd', + b'%10.7V', None, b'abc[\xe2\x82\0') # following tests comes from #7330 # test width modifier and precision modifier with %S @@ -646,6 +696,40 @@ def check_format(expected, format, *args): check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + # test %T + check_format('type: str', + b'type: %T', py_object("abc")) + check_format(f'type: st', + b'type: %.2T', py_object("abc")) + check_format(f'type: str', + b'type: %10T', py_object("abc")) + + class LocalType: + pass + obj = LocalType() + fullname = f'{__name__}.{LocalType.__qualname__}' + check_format(f'type: {fullname}', + b'type: %T', py_object(obj)) + fullname_alt = f'{__name__}:{LocalType.__qualname__}' + check_format(f'type: {fullname_alt}', + b'type: %#T', py_object(obj)) + + # test %N + check_format('type: str', + b'type: %N', py_object(str)) + check_format(f'type: st', + b'type: %.2N', py_object(str)) + check_format(f'type: str', + b'type: %10N', py_object(str)) + + check_format(f'type: {fullname}', + b'type: %N', py_object(type(obj))) + check_format(f'type: {fullname_alt}', + b'type: %#N', py_object(type(obj))) + with self.assertRaisesRegex(TypeError, "%N argument must be a type"): + check_format('type: str', + b'type: %N', py_object("abc")) + # test variable width and precision check_format(' abc', b'%*s', c_int(5), b'abc') check_format('ab', b'%.*s', c_int(2), b'abc') @@ -707,10 +791,10 @@ def check_format(expected, format, *args): PyUnicode_FromFormat, b'%+i', c_int(10)) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_interninplace(self): """Test PyUnicode_InternInPlace()""" - from _testcapi import unicode_interninplace as interninplace + from _testlimitedcapi import unicode_interninplace as interninplace s = b'abc'.decode() r = interninplace(s) @@ -720,10 +804,10 @@ def test_interninplace(self): # CRASHES interninplace(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_internfromstring(self): """Test PyUnicode_InternFromString()""" - from _testcapi import unicode_internfromstring as internfromstring + from _testlimitedcapi import unicode_internfromstring as internfromstring self.assertEqual(internfromstring(b'abc'), 'abc') self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') @@ -734,10 +818,10 @@ def test_internfromstring(self): # CRASHES internfromstring(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromwidechar(self): """Test PyUnicode_FromWideChar()""" - from _testcapi import unicode_fromwidechar as fromwidechar + from _testlimitedcapi import unicode_fromwidechar as fromwidechar from _testcapi import SIZEOF_WCHAR_T if SIZEOF_WCHAR_T == 2: @@ -769,11 +853,11 @@ def test_fromwidechar(self): self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidechar(self): """Test PyUnicode_AsWideChar()""" - from _testcapi import unicode_aswidechar - from _testcapi import unicode_aswidechar_null + from _testlimitedcapi import unicode_aswidechar + from _testlimitedcapi import unicode_aswidechar_null from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidechar('abcdef', 2) @@ -817,11 +901,11 @@ def test_aswidechar(self): self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidecharstring(self): """Test PyUnicode_AsWideCharString()""" - from _testcapi import unicode_aswidecharstring - from _testcapi import unicode_aswidecharstring_null + from _testlimitedcapi import unicode_aswidecharstring + from _testlimitedcapi import unicode_aswidecharstring_null from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidecharstring('abc') @@ -893,10 +977,10 @@ def test_asucs4copy(self): # CRASHES asucs4copy(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromordinal(self): """Test PyUnicode_FromOrdinal()""" - from _testcapi import unicode_fromordinal as fromordinal + from _testlimitedcapi import unicode_fromordinal as fromordinal self.assertEqual(fromordinal(0x61), 'a') self.assertEqual(fromordinal(0x20ac), '\u20ac') @@ -922,11 +1006,11 @@ def test_asutf8(self): # CRASHES unicode_asutf8(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_asutf8andsize(self): """Test PyUnicode_AsUTF8AndSize()""" - from _testcapi import unicode_asutf8andsize - from _testcapi import unicode_asutf8andsize_null + from _testlimitedcapi import unicode_asutf8andsize + from _testlimitedcapi import unicode_asutf8andsize_null self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3)) self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6)) @@ -945,10 +1029,10 @@ def test_asutf8andsize(self): # CRASHES unicode_asutf8andsize_null(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_getdefaultencoding(self): """Test PyUnicode_GetDefaultEncoding()""" - from _testcapi import unicode_getdefaultencoding as getdefaultencoding + from _testlimitedcapi import unicode_getdefaultencoding as getdefaultencoding self.assertEqual(getdefaultencoding(), b'utf-8') @@ -973,10 +1057,10 @@ def test_transform_decimal_and_space(self): # CRASHES transform_decimal(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_concat(self): """Test PyUnicode_Concat()""" - from _testcapi import unicode_concat as concat + from _testlimitedcapi import unicode_concat as concat self.assertEqual(concat('abc', 'def'), 'abcdef') self.assertEqual(concat('abc', 'где'), 'abcгде') @@ -994,10 +1078,10 @@ def test_concat(self): # CRASHES concat('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_split(self): """Test PyUnicode_Split()""" - from _testcapi import unicode_split as split + from _testlimitedcapi import unicode_split as split self.assertEqual(split('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) self.assertEqual(split('a|b|c|d', '|', 2), ['a', 'b', 'c|d']) @@ -1022,10 +1106,10 @@ def test_split(self): # CRASHES split(NULL, '|') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_rsplit(self): """Test PyUnicode_RSplit()""" - from _testcapi import unicode_rsplit as rsplit + from _testlimitedcapi import unicode_rsplit as rsplit self.assertEqual(rsplit('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) self.assertEqual(rsplit('a|b|c|d', '|', 2), ['a|b', 'c', 'd']) @@ -1051,10 +1135,10 @@ def test_rsplit(self): # CRASHES rsplit(NULL, '|') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_partition(self): """Test PyUnicode_Partition()""" - from _testcapi import unicode_partition as partition + from _testlimitedcapi import unicode_partition as partition self.assertEqual(partition('a|b|c', '|'), ('a', '|', 'b|c')) self.assertEqual(partition('a||b||c', '||'), ('a', '||', 'b||c')) @@ -1071,10 +1155,10 @@ def test_partition(self): # CRASHES partition('a|b|c', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_rpartition(self): """Test PyUnicode_RPartition()""" - from _testcapi import unicode_rpartition as rpartition + from _testlimitedcapi import unicode_rpartition as rpartition self.assertEqual(rpartition('a|b|c', '|'), ('a|b', '|', 'c')) self.assertEqual(rpartition('a||b||c', '||'), ('a||b', '||', 'c')) @@ -1091,10 +1175,10 @@ def test_rpartition(self): # CRASHES rpartition('a|b|c', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_splitlines(self): """Test PyUnicode_SplitLines()""" - from _testcapi import unicode_splitlines as splitlines + from _testlimitedcapi import unicode_splitlines as splitlines self.assertEqual(splitlines('a\nb\rc\r\nd'), ['a', 'b', 'c', 'd']) self.assertEqual(splitlines('a\nb\rc\r\nd', True), @@ -1109,10 +1193,10 @@ def test_splitlines(self): # CRASHES splitlines(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_translate(self): """Test PyUnicode_Translate()""" - from _testcapi import unicode_translate as translate + from _testlimitedcapi import unicode_translate as translate self.assertEqual(translate('abcd', {ord('a'): 'A', ord('b'): ord('B'), ord('c'): '<>'}), 'AB<>d') self.assertEqual(translate('абвг', {ord('а'): 'А', ord('б'): ord('Б'), ord('в'): '<>'}), 'АБ<>г') @@ -1134,10 +1218,10 @@ def test_translate(self): # CRASHES translate(NULL, []) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_join(self): """Test PyUnicode_Join()""" - from _testcapi import unicode_join as join + from _testlimitedcapi import unicode_join as join self.assertEqual(join('|', ['a', 'b', 'c']), 'a|b|c') self.assertEqual(join('|', ['a', '', 'c']), 'a||c') self.assertEqual(join('', ['a', 'b', 'c']), 'abc') @@ -1152,10 +1236,10 @@ def test_join(self): self.assertRaises(SystemError, join, '|', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_count(self): """Test PyUnicode_Count()""" - from _testcapi import unicode_count + from _testlimitedcapi import unicode_count for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1183,10 +1267,10 @@ def test_count(self): # CRASHES unicode_count(str, NULL, 0, len(str)) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_tailmatch(self): """Test PyUnicode_Tailmatch()""" - from _testcapi import unicode_tailmatch as tailmatch + from _testlimitedcapi import unicode_tailmatch as tailmatch str = 'ababahalamaha' self.assertEqual(tailmatch(str, 'aba', 0, len(str), -1), 1) @@ -1218,10 +1302,10 @@ def test_tailmatch(self): # CRASHES tailmatch(str, NULL, 0, len(str), -1) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_find(self): """Test PyUnicode_Find()""" - from _testcapi import unicode_find as find + from _testlimitedcapi import unicode_find as find for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1259,10 +1343,10 @@ def test_find(self): # CRASHES find(str, NULL, 0, len(str), 1) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_findchar(self): """Test PyUnicode_FindChar()""" - from _testcapi import unicode_findchar + from _testlimitedcapi import unicode_findchar for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1295,10 +1379,10 @@ def test_findchar(self): # CRASHES unicode_findchar(NULL, ord('!'), 0, len(str), 1), 1) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_replace(self): """Test PyUnicode_Replace()""" - from _testcapi import unicode_replace as replace + from _testlimitedcapi import unicode_replace as replace str = 'abracadabra' self.assertEqual(replace(str, 'a', '='), '=br=c=d=br=') @@ -1326,10 +1410,10 @@ def test_replace(self): # CRASHES replace(NULL, 'a', '=') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_compare(self): """Test PyUnicode_Compare()""" - from _testcapi import unicode_compare as compare + from _testlimitedcapi import unicode_compare as compare self.assertEqual(compare('abc', 'abc'), 0) self.assertEqual(compare('abc', 'def'), -1) @@ -1348,10 +1432,10 @@ def test_compare(self): # CRASHES compare('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_comparewithasciistring(self): """Test PyUnicode_CompareWithASCIIString()""" - from _testcapi import unicode_comparewithasciistring as comparewithasciistring + from _testlimitedcapi import unicode_comparewithasciistring as comparewithasciistring self.assertEqual(comparewithasciistring('abc', b'abc'), 0) self.assertEqual(comparewithasciistring('abc', b'def'), -1) @@ -1365,11 +1449,11 @@ def test_comparewithasciistring(self): # CRASHES comparewithasciistring(NULL, b'abc') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_equaltoutf8(self): # Test PyUnicode_EqualToUTF8() - from _testcapi import unicode_equaltoutf8 as equaltoutf8 - from _testcapi import unicode_asutf8andsize as asutf8andsize + from _testlimitedcapi import unicode_equaltoutf8 as equaltoutf8 + from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -1411,11 +1495,11 @@ def test_equaltoutf8(self): '\ud801'.encode("utf8", "surrogatepass")), 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_equaltoutf8andsize(self): # Test PyUnicode_EqualToUTF8AndSize() - from _testcapi import unicode_equaltoutf8andsize as equaltoutf8andsize - from _testcapi import unicode_asutf8andsize as asutf8andsize + from _testlimitedcapi import unicode_equaltoutf8andsize as equaltoutf8andsize + from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -1480,10 +1564,10 @@ def check_not_equal_encoding(text, encoding): # CRASHES equaltoutf8andsize('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_richcompare(self): """Test PyUnicode_RichCompare()""" - from _testcapi import unicode_richcompare as richcompare + from _testlimitedcapi import unicode_richcompare as richcompare LT, LE, EQ, NE, GT, GE = range(6) strings = ('abc', 'абв', '\U0001f600', 'abc\0') @@ -1508,10 +1592,10 @@ def test_richcompare(self): # CRASHES richcompare('abc', NULL, op) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_format(self): """Test PyUnicode_Format()""" - from _testcapi import unicode_format as format + from _testlimitedcapi import unicode_format as format self.assertEqual(format('x=%d!', 42), 'x=42!') self.assertEqual(format('x=%d!', (42,)), 'x=42!') @@ -1521,10 +1605,10 @@ def test_format(self): self.assertRaises(SystemError, format, NULL, 42) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_contains(self): """Test PyUnicode_Contains()""" - from _testcapi import unicode_contains as contains + from _testlimitedcapi import unicode_contains as contains self.assertEqual(contains('abcd', ''), 1) self.assertEqual(contains('abcd', 'b'), 1) @@ -1543,10 +1627,10 @@ def test_contains(self): # CRASHES contains('abcd', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_isidentifier(self): """Test PyUnicode_IsIdentifier()""" - from _testcapi import unicode_isidentifier as isidentifier + from _testlimitedcapi import unicode_isidentifier as isidentifier self.assertEqual(isidentifier("a"), 1) self.assertEqual(isidentifier("b0"), 1) @@ -1638,5 +1722,220 @@ def test_pep393_utf8_caching_bug(self): self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) +class PyUnicodeWriterTest(unittest.TestCase): + def create_writer(self, size): + return _testcapi.PyUnicodeWriter(size) + + def test_basic(self): + writer = self.create_writer(100) + + # test PyUnicodeWriter_WriteUTF8() + writer.write_utf8(b'var', -1) + + # test PyUnicodeWriter_WriteChar() + writer.write_char('=') + + # test PyUnicodeWriter_WriteSubstring() + writer.write_substring("[long]", 1, 5) + + # test PyUnicodeWriter_WriteStr() + writer.write_str(" value ") + + # test PyUnicodeWriter_WriteRepr() + writer.write_repr("repr") + + self.assertEqual(writer.finish(), + "var=long value 'repr'") + + def test_utf8(self): + writer = self.create_writer(0) + writer.write_utf8(b"ascii", -1) + writer.write_char('-') + writer.write_utf8(b"latin1=\xC3\xA9", -1) + writer.write_char('-') + writer.write_utf8(b"euro=\xE2\x82\xAC", -1) + writer.write_char('.') + self.assertEqual(writer.finish(), + "ascii-latin1=\xE9-euro=\u20AC.") + + def test_invalid_utf8(self): + writer = self.create_writer(0) + with self.assertRaises(UnicodeDecodeError): + writer.write_utf8(b"invalid=\xFF", -1) + + def test_recover_utf8_error(self): + # test recovering from PyUnicodeWriter_WriteUTF8() error + writer = self.create_writer(0) + writer.write_utf8(b"value=", -1) + + # write fails with an invalid string + with self.assertRaises(UnicodeDecodeError): + writer.write_utf8(b"invalid\xFF", -1) + + # retry write with a valid string + writer.write_utf8(b"valid", -1) + + self.assertEqual(writer.finish(), + "value=valid") + + def test_decode_utf8(self): + # test PyUnicodeWriter_DecodeUTF8Stateful() + writer = self.create_writer(0) + writer.decodeutf8stateful(b"ign\xFFore", -1, b"ignore") + writer.write_char('-') + writer.decodeutf8stateful(b"replace\xFF", -1, b"replace") + writer.write_char('-') + + # incomplete trailing UTF-8 sequence + writer.decodeutf8stateful(b"incomplete\xC3", -1, b"replace") + + self.assertEqual(writer.finish(), + "ignore-replace\uFFFD-incomplete\uFFFD") + + def test_decode_utf8_consumed(self): + # test PyUnicodeWriter_DecodeUTF8Stateful() with consumed + writer = self.create_writer(0) + + # valid string + consumed = writer.decodeutf8stateful(b"text", -1, b"strict", True) + self.assertEqual(consumed, 4) + writer.write_char('-') + + # non-ASCII + consumed = writer.decodeutf8stateful(b"\xC3\xA9-\xE2\x82\xAC", 6, b"strict", True) + self.assertEqual(consumed, 6) + writer.write_char('-') + + # invalid UTF-8 (consumed is 0 on error) + with self.assertRaises(UnicodeDecodeError): + writer.decodeutf8stateful(b"invalid\xFF", -1, b"strict", True) + + # ignore error handler + consumed = writer.decodeutf8stateful(b"more\xFF", -1, b"ignore", True) + self.assertEqual(consumed, 5) + writer.write_char('-') + + # incomplete trailing UTF-8 sequence + consumed = writer.decodeutf8stateful(b"incomplete\xC3", -1, b"ignore", True) + self.assertEqual(consumed, 10) + + self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete") + + def test_widechar(self): + writer = self.create_writer(0) + writer.write_widechar("latin1=\xE9") + writer.write_widechar("-") + writer.write_widechar("euro=\u20AC") + writer.write_char("-") + writer.write_widechar("max=\U0010ffff") + writer.write_char('.') + self.assertEqual(writer.finish(), + "latin1=\xE9-euro=\u20AC-max=\U0010ffff.") + + def test_ucs4(self): + writer = self.create_writer(0) + writer.write_ucs4("ascii IGNORED", 5) + writer.write_char("-") + writer.write_ucs4("latin1=\xe9", 8) + writer.write_char("-") + writer.write_ucs4("euro=\u20ac", 6) + writer.write_char("-") + writer.write_ucs4("max=\U0010ffff", 5) + writer.write_char(".") + self.assertEqual(writer.finish(), + "ascii-latin1=\xE9-euro=\u20AC-max=\U0010ffff.") + + # Test some special characters + writer = self.create_writer(0) + # Lone surrogate character + writer.write_ucs4("lone\uDC80", 5) + writer.write_char("-") + # Surrogate pair + writer.write_ucs4("pair\uDBFF\uDFFF", 5) + writer.write_char("-") + writer.write_ucs4("null[\0]", 7) + self.assertEqual(writer.finish(), + "lone\udc80-pair\udbff-null[\0]") + + # invalid size + writer = self.create_writer(0) + with self.assertRaises(ValueError): + writer.write_ucs4("text", -1) + + def test_substring_empty(self): + writer = self.create_writer(0) + writer.write_substring("abc", 1, 1) + self.assertEqual(writer.finish(), '') + + +@unittest.skipIf(ctypes is None, 'need ctypes') +class PyUnicodeWriterFormatTest(unittest.TestCase): + def create_writer(self, size): + return _testcapi.PyUnicodeWriter(size) + + def writer_format(self, writer, *args): + from ctypes import c_char_p, pythonapi, c_int, c_void_p + _PyUnicodeWriter_Format = getattr(pythonapi, "PyUnicodeWriter_Format") + _PyUnicodeWriter_Format.argtypes = (c_void_p, c_char_p,) + _PyUnicodeWriter_Format.restype = c_int + + if _PyUnicodeWriter_Format(writer.get_pointer(), *args) < 0: + raise ValueError("PyUnicodeWriter_Format failed") + + def test_format(self): + from ctypes import c_int + writer = self.create_writer(0) + self.writer_format(writer, b'%s %i', b'abc', c_int(123)) + writer.write_char('.') + self.assertEqual(writer.finish(), 'abc 123.') + + def test_recover_error(self): + # test recovering from PyUnicodeWriter_Format() error + writer = self.create_writer(0) + self.writer_format(writer, b"%s ", b"Hello") + + # PyUnicodeWriter_Format() fails with an invalid format string + with self.assertRaises(ValueError): + self.writer_format(writer, b"%s\xff", b"World") + + # Retry PyUnicodeWriter_Format() with a valid format string + self.writer_format(writer, b"%s.", b"World") + + self.assertEqual(writer.finish(), 'Hello World.') + + def test_unicode_equal(self): + unicode_equal = _testlimitedcapi.unicode_equal + + def copy(text): + return text.encode().decode() + + self.assertTrue(unicode_equal("", "")) + self.assertTrue(unicode_equal("abc", "abc")) + self.assertTrue(unicode_equal("abc", copy("abc"))) + self.assertTrue(unicode_equal("\u20ac", copy("\u20ac"))) + self.assertTrue(unicode_equal("\U0010ffff", copy("\U0010ffff"))) + + self.assertFalse(unicode_equal("abc", "abcd")) + self.assertFalse(unicode_equal("\u20ac", "\u20ad")) + self.assertFalse(unicode_equal("\U0010ffff", "\U0010fffe")) + + # str subclass + self.assertTrue(unicode_equal("abc", Str("abc"))) + self.assertTrue(unicode_equal(Str("abc"), "abc")) + self.assertFalse(unicode_equal("abc", Str("abcd"))) + self.assertFalse(unicode_equal(Str("abc"), "abcd")) + + # invalid type + for invalid_type in (b'bytes', 123, ("tuple",)): + with self.subTest(invalid_type=invalid_type): + with self.assertRaises(TypeError): + unicode_equal("abc", invalid_type) + with self.assertRaises(TypeError): + unicode_equal(invalid_type, "abc") + + # CRASHES unicode_equal("abc", NULL) + # CRASHES unicode_equal(NULL, "abc") + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index ae062b1bda26b7..8644479d83d5ed 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -1,7 +1,10 @@ import unittest +import contextvars from contextlib import contextmanager, ExitStack -from test.support import catch_unraisable_exception, import_helper +from test.support import ( + catch_unraisable_exception, import_helper, + gc_collect) # Skip this test if the _testcapi module isn't available. @@ -94,6 +97,23 @@ def test_dealloc(self): del d self.assert_events(["dealloc"]) + def test_object_dict(self): + class MyObj: pass + o = MyObj() + + with self.watcher() as wid: + self.watch(wid, o.__dict__) + o.foo = "bar" + o.foo = "baz" + del o.foo + self.assert_events(["new:foo:bar", "mod:foo:baz", "del:foo"]) + + with self.watcher() as wid: + self.watch(wid, o.__dict__) + for _ in range(100): + o.foo = "bar" + self.assert_events(["new:foo:bar"] + ["mod:foo:bar"] * 99) + def test_unwatch(self): d = {} with self.watcher() as wid: @@ -280,8 +300,10 @@ class C: pass self.watch(wid, C) with catch_unraisable_exception() as cm: C.foo = "bar" - self.assertEqual(cm.unraisable.err_msg, - f"Exception ignored in type watcher callback #0 for {C!r}") + self.assertEqual( + cm.unraisable.err_msg, + f"Exception ignored in type watcher callback #1 for {C!r}", + ) self.assertIs(cm.unraisable.object, None) self.assertEqual(str(cm.unraisable.exc_value), "boom!") self.assert_events([]) @@ -372,6 +394,7 @@ def code_watcher(self, which_watcher): def assert_event_counts(self, exp_created_0, exp_destroyed_0, exp_created_1, exp_destroyed_1): + gc_collect() # code objects are collected by GC in free-threaded build self.assertEqual( exp_created_0, _testcapi.get_code_watcher_num_created_events(0)) self.assertEqual( @@ -432,6 +455,7 @@ def test_dealloc_error(self): with self.code_watcher(2): with catch_unraisable_exception() as cm: del co + gc_collect() self.assertEqual(str(cm.unraisable.exc_value), "boom!") @@ -563,5 +587,92 @@ def test_allocate_too_many_watchers(self): _testcapi.allocate_too_many_func_watchers() +class TestContextObjectWatchers(unittest.TestCase): + @contextmanager + def context_watcher(self, which_watcher): + wid = _testcapi.add_context_watcher(which_watcher) + try: + switches = _testcapi.get_context_switches(which_watcher) + except ValueError: + switches = None + try: + yield switches + finally: + _testcapi.clear_context_watcher(wid) + + def assert_event_counts(self, want_0, want_1): + self.assertEqual(len(_testcapi.get_context_switches(0)), want_0) + self.assertEqual(len(_testcapi.get_context_switches(1)), want_1) + + def test_context_object_events_dispatched(self): + # verify that all counts are zero before any watchers are registered + self.assert_event_counts(0, 0) + + # verify that all counts remain zero when a context object is + # entered and exited with no watchers registered + ctx = contextvars.copy_context() + ctx.run(self.assert_event_counts, 0, 0) + self.assert_event_counts(0, 0) + + # verify counts are as expected when first watcher is registered + with self.context_watcher(0): + self.assert_event_counts(0, 0) + ctx.run(self.assert_event_counts, 1, 0) + self.assert_event_counts(2, 0) + + # again with second watcher registered + with self.context_watcher(1): + self.assert_event_counts(2, 0) + ctx.run(self.assert_event_counts, 3, 1) + self.assert_event_counts(4, 2) + + # verify counts are reset and don't change after both watchers are cleared + ctx.run(self.assert_event_counts, 0, 0) + self.assert_event_counts(0, 0) + + def test_callback_error(self): + ctx_outer = contextvars.copy_context() + ctx_inner = contextvars.copy_context() + unraisables = [] + + def _in_outer(): + with self.context_watcher(2): + with catch_unraisable_exception() as cm: + ctx_inner.run(lambda: unraisables.append(cm.unraisable)) + unraisables.append(cm.unraisable) + + try: + ctx_outer.run(_in_outer) + self.assertEqual([x.err_msg for x in unraisables], + ["Exception ignored in Py_CONTEXT_SWITCHED " + f"watcher callback for {ctx!r}" + for ctx in [ctx_inner, ctx_outer]]) + self.assertEqual([str(x.exc_value) for x in unraisables], + ["boom!", "boom!"]) + finally: + # Break reference cycle + unraisables = None + + def test_clear_out_of_range_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"Invalid context watcher ID -1"): + _testcapi.clear_context_watcher(-1) + with self.assertRaisesRegex(ValueError, r"Invalid context watcher ID 8"): + _testcapi.clear_context_watcher(8) # CONTEXT_MAX_WATCHERS = 8 + + def test_clear_unassigned_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"No context watcher set for ID 1"): + _testcapi.clear_context_watcher(1) + + def test_allocate_too_many_watchers(self): + with self.assertRaisesRegex(RuntimeError, r"no more context watcher IDs available"): + _testcapi.allocate_too_many_context_watchers() + + def test_exit_base_context(self): + ctx = contextvars.Context() + _testcapi.clear_context_stack() + with self.context_watcher(0) as switches: + ctx.run(lambda: None) + self.assertEqual(switches, [ctx, None]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py new file mode 100644 index 00000000000000..54859f9ff7622d --- /dev/null +++ b/Lib/test/test_cext/__init__.py @@ -0,0 +1,111 @@ +# gh-116869: Build a basic C test extension to check that the Python C API +# does not emit C compiler warnings. +# +# The Python C API must be compatible with building +# with the -Werror=declaration-after-statement compiler flag. + +import os.path +import shlex +import shutil +import subprocess +import unittest +from test import support + + +SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c') +SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') + + +# With MSVC on a debug build, the linker fails with: cannot open file +# 'python311.lib', it should look 'python311_d.lib'. +@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG, + 'test fails on Windows debug build') +# Building and running an extension in clang sanitizing mode is not +# straightforward +@support.skip_if_sanitizer('test does not work with analyzing builds', + address=True, memory=True, ub=True, thread=True) +# the test uses venv+pip: skip if it's not available +@support.requires_venv_with_pip() +@support.requires_subprocess() +@support.requires_resource('cpu') +class TestExt(unittest.TestCase): + # Default build with no options + def test_build(self): + self.check_build('_test_cext') + + def test_build_c11(self): + self.check_build('_test_c11_cext', std='c11') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") + def test_build_c99(self): + self.check_build('_test_c99_cext', std='c99') + + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_test_limited_cext', limited=True) + + @support.requires_gil_enabled('broken for now with Free Threading') + def test_build_limited_c11(self): + self.check_build('_test_limited_c11_cext', limited=True, std='c11') + + def check_build(self, extension_name, std=None, limited=False): + venv_dir = 'env' + with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: + self._check_build(extension_name, python_exe, + std=std, limited=limited) + + def _check_build(self, extension_name, python_exe, std, limited): + pkg_dir = 'pkg' + os.mkdir(pkg_dir) + shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) + shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) + + def run_cmd(operation, cmd): + env = os.environ.copy() + if std: + env['CPYTHON_TEST_STD'] = std + if limited: + env['CPYTHON_TEST_LIMITED'] = '1' + env['CPYTHON_TEST_EXT_NAME'] = extension_name + if support.verbose: + print('Run:', ' '.join(map(shlex.quote, cmd))) + subprocess.run(cmd, check=True, env=env) + else: + proc = subprocess.run(cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + if proc.returncode: + print('Run:', ' '.join(map(shlex.quote, cmd))) + print(proc.stdout, end='') + self.fail( + f"{operation} failed with exit code {proc.returncode}") + + # Build and install the C extension + cmd = [python_exe, '-X', 'dev', + '-m', 'pip', 'install', '--no-build-isolation', + os.path.abspath(pkg_dir)] + if support.verbose: + cmd.append('-v') + run_cmd('Install', cmd) + + # Do a reference run. Until we test that running python + # doesn't leak references (gh-94755), run it so one can manually check + # -X showrefcount results against this baseline. + cmd = [python_exe, + '-X', 'dev', + '-X', 'showrefcount', + '-c', 'pass'] + run_cmd('Reference run', cmd) + + # Import the C extension + cmd = [python_exe, + '-X', 'dev', + '-X', 'showrefcount', + '-c', f"import {extension_name}"] + run_cmd('Import', cmd) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c new file mode 100644 index 00000000000000..b76abe1d74c628 --- /dev/null +++ b/Lib/test/test_cext/extension.c @@ -0,0 +1,89 @@ +// gh-116869: Basic C test extension to check that the Python C API +// does not emit C compiler warnings. + +// Always enable assertions +#undef NDEBUG + +#include "Python.h" + +#ifndef MODULE_NAME +# error "MODULE_NAME macro must be defined" +#endif + +#define _STR(NAME) #NAME +#define STR(NAME) _STR(NAME) + +PyDoc_STRVAR(_testcext_add_doc, +"add(x, y)\n" +"\n" +"Return the sum of two integers: x + y."); + +static PyObject * +_testcext_add(PyObject *Py_UNUSED(module), PyObject *args) +{ + long i, j, res; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) { + return NULL; + } + res = i + j; + return PyLong_FromLong(res); +} + + +static PyMethodDef _testcext_methods[] = { + {"add", _testcext_add, METH_VARARGS, _testcext_add_doc}, + {NULL, NULL, 0, NULL} // sentinel +}; + + +static int +_testcext_exec( +#ifdef __STDC_VERSION__ + PyObject *module +#else + PyObject *Py_UNUSED(module) +#endif + ) +{ +#ifdef __STDC_VERSION__ + if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) { + return -1; + } +#endif + + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + + return 0; +} + +static PyModuleDef_Slot _testcext_slots[] = { + {Py_mod_exec, (void*)_testcext_exec}, + {0, NULL} +}; + + +PyDoc_STRVAR(_testcext_doc, "C test extension."); + +static struct PyModuleDef _testcext_module = { + PyModuleDef_HEAD_INIT, // m_base + STR(MODULE_NAME), // m_name + _testcext_doc, // m_doc + 0, // m_size + _testcext_methods, // m_methods + _testcext_slots, // m_slots + NULL, // m_traverse + NULL, // m_clear + NULL, // m_free +}; + + +#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define FUNC_NAME(NAME) _FUNC_NAME(NAME) + +PyMODINIT_FUNC +FUNC_NAME(MODULE_NAME)(void) +{ + return PyModuleDef_Init(&_testcext_module); +} diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py new file mode 100644 index 00000000000000..e97749b45ea6f3 --- /dev/null +++ b/Lib/test/test_cext/setup.py @@ -0,0 +1,109 @@ +# gh-91321: Build a basic C test extension to check that the Python C API is +# compatible with C and does not emit C compiler warnings. +import os +import platform +import shlex +import sys +import sysconfig +from test import support + +from setuptools import setup, Extension + + +SOURCE = 'extension.c' + +if not support.MS_WINDOWS: + # C compiler flags for GCC and clang + CFLAGS = [ + # The purpose of test_cext extension is to check that building a C + # extension using the Python C API does not emit C compiler warnings. + '-Werror', + + # gh-120593: Check the 'const' qualifier + '-Wcast-qual', + ] + if not support.Py_GIL_DISABLED: + CFLAGS.append( + # gh-116869: The Python C API must be compatible with building + # with the -Werror=declaration-after-statement compiler flag. + '-Werror=declaration-after-statement', + ) +else: + # MSVC compiler flags + CFLAGS = [ + # Display warnings level 1 to 4 + '/W4', + # Treat all compiler warnings as compiler errors + '/WX', + ] + + +def main(): + std = os.environ.get("CPYTHON_TEST_STD", "") + module_name = os.environ["CPYTHON_TEST_EXT_NAME"] + limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + + cflags = list(CFLAGS) + cflags.append(f'-DMODULE_NAME={module_name}') + + # Add -std=STD or /std:STD (MSVC) compiler flag + if std: + if support.MS_WINDOWS: + cflags.append(f'/std:{std}') + else: + cflags.append(f'-std={std}') + + # Remove existing -std or /std options from CC command line. + # Python adds -std=c11 option. + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + if support.MS_WINDOWS: + std_prefix = '/std' + else: + std_prefix = '-std' + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + + # Define Py_LIMITED_API macro + if limited: + version = sys.hexversion + cflags.append(f'-DPy_LIMITED_API={version:#x}') + + # On Windows, add PCbuild\amd64\ to include and library directories + include_dirs = [] + library_dirs = [] + if support.MS_WINDOWS: + srcdir = sysconfig.get_config_var('srcdir') + machine = platform.uname().machine + pcbuild = os.path.join(srcdir, 'PCbuild', machine) + if os.path.exists(pcbuild): + # pyconfig.h is generated in PCbuild\amd64\ + include_dirs.append(pcbuild) + # python313.lib is generated in PCbuild\amd64\ + library_dirs.append(pcbuild) + print(f"Add PCbuild directory: {pcbuild}") + + # Display information to help debugging + for env_name in ('CC', 'CFLAGS'): + if env_name in os.environ: + print(f"{env_name} env var: {os.environ[env_name]!r}") + else: + print(f"{env_name} env var: ") + print(f"extra_compile_args: {cflags!r}") + + ext = Extension( + module_name, + sources=[SOURCE], + extra_compile_args=cflags, + include_dirs=include_dirs, + library_dirs=library_dirs) + setup(name=f'internal_{module_name}', + version='0.0', + ext_modules=[ext]) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index d59271435e9eb0..7720cf157fa9ae 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,7 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest - +from test.support import cpython_only, import_helper, script_helper testmeths = [ @@ -448,15 +448,15 @@ def __delattr__(self, *args): def testHasAttrString(self): import sys from test.support import import_helper - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') class A: def __init__(self): self.attr = 1 a = A() - self.assertEqual(_testcapi.object_hasattrstring(a, b"attr"), 1) - self.assertEqual(_testcapi.object_hasattrstring(a, b"noattr"), 0) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"attr"), 1) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"noattr"), 0) self.assertIsNone(sys.exception()) def testDel(self): @@ -503,6 +503,56 @@ def __eq__(self, other): return 1 self.assertRaises(TypeError, hash, C2()) + def testPredefinedAttrs(self): + o = object() + + class Custom: + pass + + c = Custom() + + methods = ( + '__class__', '__delattr__', '__dir__', '__eq__', '__format__', + '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', + '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', + '__new__', '__reduce__', '__reduce_ex__', '__repr__', + '__setattr__', '__sizeof__', '__str__', '__subclasshook__' + ) + for name in methods: + with self.subTest(name): + self.assertTrue(callable(getattr(object, name, None))) + self.assertTrue(callable(getattr(o, name, None))) + self.assertTrue(callable(getattr(Custom, name, None))) + self.assertTrue(callable(getattr(c, name, None))) + + not_defined = [ + '__abs__', '__aenter__', '__aexit__', '__aiter__', '__anext__', + '__await__', '__bool__', '__bytes__', '__ceil__', + '__complex__', '__contains__', '__del__', '__delete__', + '__delitem__', '__divmod__', '__enter__', '__exit__', + '__float__', '__floor__', '__get__', '__getattr__', '__getitem__', + '__index__', '__int__', '__invert__', '__iter__', '__len__', + '__length_hint__', '__missing__', '__neg__', '__next__', + '__objclass__', '__pos__', '__rdivmod__', '__reversed__', + '__round__', '__set__', '__setitem__', '__trunc__' + ] + augment = ( + 'add', 'and', 'floordiv', 'lshift', 'matmul', 'mod', 'mul', 'pow', + 'rshift', 'sub', 'truediv', 'xor' + ) + not_defined.extend(map("__{}__".format, augment)) + not_defined.extend(map("__r{}__".format, augment)) + not_defined.extend(map("__i{}__".format, augment)) + for name in not_defined: + with self.subTest(name): + self.assertFalse(hasattr(object, name)) + self.assertFalse(hasattr(o, name)) + self.assertFalse(hasattr(Custom, name)) + self.assertFalse(hasattr(c, name)) + + # __call__() is defined on the metaclass but not the class + self.assertFalse(hasattr(o, "__call__")) + self.assertFalse(hasattr(c, "__call__")) def testSFBug532646(self): # Test for SF bug 532646 @@ -788,5 +838,181 @@ def __init__(self, obj): Type(i) self.assertEqual(calls, 100) + def test_specialization_class_call_doesnt_crash(self): + # gh-123185 + + class Foo: + def __init__(self, arg): + pass + + for _ in range(8): + try: + Foo() + except: + pass + + +from _testinternalcapi import has_inline_values + +Py_TPFLAGS_MANAGED_DICT = (1 << 2) + +class Plain: + pass + + +class WithAttrs: + + def __init__(self): + self.a = 1 + self.b = 2 + self.c = 3 + self.d = 4 + + +class TestInlineValues(unittest.TestCase): + + def test_flags(self): + self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + + def test_has_inline_values(self): + c = Plain() + self.assertTrue(has_inline_values(c)) + del c.__dict__ + self.assertFalse(has_inline_values(c)) + + def test_instances(self): + self.assertTrue(has_inline_values(Plain())) + self.assertTrue(has_inline_values(WithAttrs())) + + def test_inspect_dict(self): + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__ + self.assertTrue(has_inline_values(c)) + + def test_update_dict(self): + d = { "e": 5, "f": 6 } + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__.update(d) + self.assertTrue(has_inline_values(c)) + + @staticmethod + def set_100(obj): + for i in range(100): + setattr(obj, f"a{i}", i) + + def check_100(self, obj): + for i in range(100): + self.assertEqual(getattr(obj, f"a{i}"), i) + + def test_many_attributes(self): + class C: pass + c = C() + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + c = C() + self.assertTrue(has_inline_values(c)) + + def test_many_attributes_with_dict(self): + class C: pass + c = C() + d = c.__dict__ + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + + def test_bug_117750(self): + "Aborted on 3.13a6" + class C: + def __init__(self): + self.__dict__.clear() + + obj = C() + self.assertEqual(obj.__dict__, {}) + obj.foo = None # Aborted here + self.assertEqual(obj.__dict__, {"foo":None}) + + def test_store_attr_deleted_dict(self): + class Foo: + pass + + f = Foo() + del f.__dict__ + f.a = 3 + self.assertEqual(f.a, 3) + + def test_rematerialize_object_dict(self): + # gh-121860: rematerializing an object's managed dictionary after it + # had been deleted caused a crash. + class Foo: pass + f = Foo() + f.__dict__["attr"] = 1 + del f.__dict__ + + # Using a str subclass is a way to trigger the re-materialization + class StrSubclass(str): pass + self.assertFalse(hasattr(f, StrSubclass("attr"))) + + # Changing the __class__ also triggers the re-materialization + class Bar: pass + f.__class__ = Bar + self.assertIsInstance(f, Bar) + self.assertEqual(f.__dict__, {}) + + def test_store_attr_type_cache(self): + """Verifies that the type cache doesn't provide a value which is + inconsistent from the dict.""" + class X: + def __del__(inner_self): + v = C.a + self.assertEqual(v, C.__dict__['a']) + + class C: + a = X() + + # prime the cache + C.a + C.a + + # destructor shouldn't be able to see inconsistent state + C.a = X() + C.a = X() + + @cpython_only + def test_detach_materialized_dict_no_memory(self): + # Skip test if _testcapi is not available: + import_helper.import_module('_testcapi') + + code = """if 1: + import test.support + import _testcapi + + class A: + def __init__(self): + self.a = 1 + self.b = 2 + a = A() + d = a.__dict__ + with test.support.catch_unraisable_exception() as ex: + _testcapi.set_nomemory(0, 1) + del a + assert ex.unraisable.exc_type is MemoryError + try: + d["a"] + except KeyError: + pass + else: + assert False, "KeyError not raised" + """ + rc, out, err = script_helper.assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertFalse(out, msg=out.decode('utf-8')) + self.assertFalse(err, msg=err.decode('utf-8')) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index e987ce54605497..d492ea1d76aa3a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -5,7 +5,7 @@ from functools import partial from test import support, test_tools from test.support import os_helper -from test.support.os_helper import TESTFN, unlink +from test.support.os_helper import TESTFN, unlink, rmtree from textwrap import dedent from unittest import TestCase import inspect @@ -17,14 +17,26 @@ test_tools.skip_if_missing('clinic') with test_tools.imports_under_tool('clinic'): import libclinic + from libclinic import ClinicError, unspecified, NULL, fail + from libclinic.converters import int_converter, str_converter, self_converter + from libclinic.function import ( + Module, Class, Function, FunctionKind, Parameter, + permute_optional_groups, permute_right_option_groups, + permute_left_option_groups) import clinic - from clinic import DSLParser - - -def _make_clinic(*, filename='clinic_tests'): - clang = clinic.CLanguage(filename) - c = clinic.Clinic(clang, filename=filename, limited_capi=False) - c.block_parser = clinic.BlockParser('', clang) + from libclinic.clanguage import CLanguage + from libclinic.converter import converters, legacy_converters + from libclinic.return_converters import return_converters, int_return_converter + from libclinic.block_parser import Block, BlockParser + from libclinic.codegen import BlockPrinter, Destination + from libclinic.dsl_parser import DSLParser + from libclinic.cli import parse_file, Clinic + + +def _make_clinic(*, filename='clinic_tests', limited_capi=False): + clang = CLanguage(filename) + c = Clinic(clang, filename=filename, limited_capi=limited_capi) + c.block_parser = BlockParser('', clang) return c @@ -43,7 +55,7 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, if strip: code = code.strip() errmsg = re.escape(errmsg) - with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + with tc.assertRaisesRegex(ClinicError, errmsg) as cm: parser(code) if filename is not None: tc.assertEqual(cm.exception.filename, filename) @@ -52,6 +64,20 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, return cm.exception +def restore_dict(converters, old_converters): + converters.clear() + converters.update(old_converters) + + +def save_restore_converters(testcase): + testcase.addCleanup(restore_dict, converters, + converters.copy()) + testcase.addCleanup(restore_dict, legacy_converters, + legacy_converters.copy()) + testcase.addCleanup(restore_dict, return_converters, + return_converters.copy()) + + class ClinicWholeFileTest(TestCase): maxDiff = None @@ -60,6 +86,7 @@ def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): filename=filename, lineno=lineno) def setUp(self): + save_restore_converters(self) self.clinic = _make_clinic(filename="test.c") def test_eol(self): @@ -121,11 +148,11 @@ def test_whitespace_before_stop_line(self): self.expect_failure(raw, err, filename="test.c", lineno=2) def test_parse_with_body_prefix(self): - clang = clinic.CLanguage(None) + clang = CLanguage(None) clang.body_prefix = "//" clang.start_line = "//[{dsl_name} start]" clang.stop_line = "//[{dsl_name} stop]" - cl = clinic.Clinic(clang, filename="test.c", limited_capi=False) + cl = Clinic(clang, filename="test.c", limited_capi=False) raw = dedent(""" //[clinic start] //module test @@ -295,7 +322,7 @@ def __init__(self): """ self.expect_failure(block, err, lineno=8) - def test_multiple_star_in_args(self): + def test_star_after_vararg(self): err = "'my_test_func' uses '*' more than once." block = """ /*[clinic input] @@ -309,6 +336,20 @@ def test_multiple_star_in_args(self): """ self.expect_failure(block, err, lineno=6) + def test_vararg_after_star(self): + err = "'my_test_func' uses '*' more than once." + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + * + *args: object + kw_arg: object + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + def test_module_already_got_one(self): err = "Already defined module 'm'!" block = """ @@ -635,14 +676,69 @@ class C "void *" "" err = "Illegal C basename: '.illegal.'" self.expect_failure(block, err, lineno=7) + def test_cloned_forced_text_signature(self): + block = dedent(""" + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + param a + b: object = NULL + / + + docstring + [clinic start generated code]*/ + + /*[clinic input] + dst = src + [clinic start generated code]*/ + """) + self.clinic.parse(block) + self.addCleanup(rmtree, "clinic") + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + + src_docstring_lines = funcs[0].docstring.split("\n") + dst_docstring_lines = funcs[1].docstring.split("\n") + + # Signatures are copied. + self.assertEqual(src_docstring_lines[0], "src($module, a[, b])") + self.assertEqual(dst_docstring_lines[0], "dst($module, a[, b])") + + # Param docstrings are copied. + self.assertIn(" param a", src_docstring_lines) + self.assertIn(" param a", dst_docstring_lines) + + # Docstrings are not copied. + self.assertIn("docstring", src_docstring_lines) + self.assertNotIn("docstring", dst_docstring_lines) + + def test_cloned_forced_text_signature_illegal(self): + block = """ + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + b: object = NULL + / + [clinic start generated code]*/ + + /*[clinic input] + @text_signature "($module, a_override[, b])" + dst = src + [clinic start generated code]*/ + """ + err = "Cannot use @text_signature when cloning a function" + self.expect_failure(block, err, lineno=11) + class ParseFileUnitTest(TestCase): def expect_parsing_failure( self, *, filename, expected_error, verify=True, output=None ): errmsg = re.escape(dedent(expected_error).strip()) - with self.assertRaisesRegex(clinic.ClinicError, errmsg): - clinic.parse_file(filename, limited_capi=False) + with self.assertRaisesRegex(ClinicError, errmsg): + parse_file(filename, limited_capi=False) def test_parse_file_no_extension(self) -> None: self.expect_parsing_failure( @@ -663,7 +759,7 @@ def test_parse_file_strange_extension(self) -> None: class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): - computed = clinic.permute_optional_groups(l, m, r) + computed = permute_optional_groups(l, m, r) self.assertEqual(output, computed) def test_range(self): @@ -705,13 +801,13 @@ def test_right_only(self): def test_have_left_options_but_required_is_empty(self): def fn(): - clinic.permute_optional_groups(['a'], [], []) + permute_optional_groups(['a'], [], []) self.assertRaises(ValueError, fn) class ClinicLinearFormatTest(TestCase): def _test(self, input, output, **kwargs): - computed = clinic.linear_format(input, **kwargs) + computed = libclinic.linear_format(input, **kwargs) self.assertEqual(output, computed) def test_empty_strings(self): @@ -761,6 +857,19 @@ def test_multiline_substitution(self): def """, name='bingle\nbungle\n') + def test_text_before_block_marker(self): + regex = re.escape("found before '{marker}'") + with self.assertRaisesRegex(ClinicError, regex): + libclinic.linear_format("no text before marker for you! {marker}", + marker="not allowed!") + + def test_text_after_block_marker(self): + regex = re.escape("found after '{marker}'") + with self.assertRaisesRegex(ClinicError, regex): + libclinic.linear_format("{marker} no text after marker for you!", + marker="not allowed!") + + class InertParser: def __init__(self, clinic): pass @@ -778,13 +887,12 @@ def parse(self, block): class ClinicBlockParserTest(TestCase): def _test(self, input, output): - language = clinic.CLanguage(None) + language = CLanguage(None) - blocks = list(clinic.BlockParser(input, language)) - writer = clinic.BlockPrinter(language) - c = _make_clinic() + blocks = list(BlockParser(input, language)) + writer = BlockPrinter(language) for block in blocks: - writer.print_block(block, limited_capi=c.limited_capi, header_includes=c.includes) + writer.print_block(block) output = writer.f.getvalue() assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input) @@ -809,8 +917,8 @@ def test_round_trip_2(self): """) def _test_clinic(self, input, output): - language = clinic.CLanguage(None) - c = clinic.Clinic(language, filename="file", limited_capi=False) + language = CLanguage(None) + c = Clinic(language, filename="file", limited_capi=False) c.parsers['inert'] = InertParser(c) c.parsers['copy'] = CopyParser(c) computed = c.parse(input) @@ -843,7 +951,7 @@ class ClinicParserTest(TestCase): def parse(self, text): c = _make_clinic() parser = DSLParser(c) - block = clinic.Block(text) + block = Block(text) parser.parse(block) return block @@ -851,8 +959,8 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): block = self.parse(text) s = block.signatures self.assertEqual(len(s), signatures_in_block) - assert isinstance(s[0], clinic.Module) - assert isinstance(s[function_index], clinic.Function) + assert isinstance(s[0], Module) + assert isinstance(s[function_index], Function) return s[function_index] def expect_failure(self, block, err, *, @@ -867,7 +975,7 @@ def checkDocstring(self, fn, expected): def test_trivial(self): parser = DSLParser(_make_clinic()) - block = clinic.Block(""" + block = Block(""" module os os.access """) @@ -896,7 +1004,7 @@ def test_param(self): self.assertEqual(2, len(function.parameters)) p = function.parameters['path'] self.assertEqual('path', p.name) - self.assertIsInstance(p.converter, clinic.int_converter) + self.assertIsInstance(p.converter, int_converter) def test_param_default(self): function = self.parse_function(""" @@ -995,7 +1103,7 @@ def test_param_no_docstring(self): """) self.assertEqual(3, len(function.parameters)) conv = function.parameters['something_else'].converter - self.assertIsInstance(conv, clinic.str_converter) + self.assertIsInstance(conv, str_converter) def test_param_default_parameters_out_of_order(self): err = ( @@ -1156,7 +1264,7 @@ def test_base_invalid_syntax(self): Function 'stat' has an invalid parameter declaration: \s+'invalid syntax: int = 42' """).strip() - with self.assertRaisesRegex(clinic.ClinicError, err): + with self.assertRaisesRegex(ClinicError, err): self.parse_function(block) def test_param_default_invalid_syntax(self): @@ -1188,7 +1296,7 @@ def test_return_converter(self): module os os.stat -> int """) - self.assertIsInstance(function.return_converter, clinic.int_return_converter) + self.assertIsInstance(function.return_converter, int_return_converter) def test_return_converter_invalid_syntax(self): block = """ @@ -1693,13 +1801,43 @@ def test_parameters_required_after_depr_star2(self): ) self.expect_failure(block, err, lineno=4) + def test_parameters_required_after_depr_star3(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + *args: object + b: int + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + def test_depr_star_must_come_before_star(self): block = """ module foo foo.bar - this: int + a: int * * [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '* [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_before_vararg(self): + block = """ + module foo + foo.bar + a: int + *args: object + * [from 3.14] + b: int Docstring. """ err = "Function 'bar': '* [from ...]' must precede '*'" @@ -1814,7 +1952,7 @@ def test_double_slash(self): err = "Function 'bar' uses '/' more than once." self.expect_failure(block, err) - def test_mix_star_and_slash(self): + def test_slash_after_star(self): block = """ module foo foo.bar @@ -1827,6 +1965,19 @@ def test_mix_star_and_slash(self): err = "Function 'bar': '/' must precede '*'" self.expect_failure(block, err) + def test_slash_after_vararg(self): + block = """ + module foo + foo.bar + x: int + y: int + *args: object + z: int + / + """ + err = "Function 'bar': '/' must precede '*'" + self.expect_failure(block, err) + def test_depr_star_must_come_after_slash(self): block = """ module foo @@ -1866,6 +2017,19 @@ def test_star_must_come_after_depr_slash(self): err = "Function 'bar': '/ [from ...]' must precede '*'" self.expect_failure(block, err, lineno=4) + def test_vararg_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + *args: object + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + def test_depr_slash_must_come_after_slash(self): block = """ module foo @@ -1893,14 +2057,14 @@ def test_parameters_not_permitted_after_slash_for_now(self): self.expect_failure(block, err) def test_parameters_no_more_than_one_vararg(self): - err = "Too many var args" + err = "Function 'bar' uses '*' more than once." block = """ module foo foo.bar *vararg1: object *vararg2: object """ - self.expect_failure(block, err, lineno=0) + self.expect_failure(block, err, lineno=3) def test_function_not_at_column_0(self): function = self.parse_function(""" @@ -2004,7 +2168,7 @@ def test_directive(self): parser = DSLParser(_make_clinic()) parser.flag = False parser.directives['setflag'] = lambda : setattr(parser, 'flag', True) - block = clinic.Block("setflag") + block = Block("setflag") parser.parse(block) self.assertTrue(parser.flag) @@ -2012,7 +2176,7 @@ def test_legacy_converters(self): block = self.parse('module os\nos.access\n path: "s"') module, function = block.signatures conv = (function.parameters['path']).converter - self.assertIsInstance(conv, clinic.str_converter) + self.assertIsInstance(conv, str_converter) def test_legacy_converters_non_string_constant_annotation(self): err = "Annotations must be either a name, a function call, or a string" @@ -2133,6 +2297,14 @@ class Foo "" "" expected_error = err_template.format(invalid_kind) self.expect_failure(block, expected_error, lineno=3) + def test_init_cannot_define_a_return_type(self): + block = """ + class Foo "" "" + Foo.__init__ -> long + """ + expected_error = "__init__ methods cannot define a return type" + self.expect_failure(block, expected_error, lineno=1) + def test_invalid_getset(self): annotations = ["@getter", "@setter"] for annotation in annotations: @@ -2154,7 +2326,7 @@ class Foo "" "" obj: int / """ - expected_error = f"{annotation} method cannot define parameters" + expected_error = f"{annotation} methods cannot define parameters" self.expect_failure(block, expected_error) def test_setter_docstring(self): @@ -2261,14 +2433,14 @@ def test_unused_param(self): def test_scaffolding(self): # test repr on special values - self.assertEqual(repr(clinic.unspecified), '') - self.assertEqual(repr(clinic.NULL), '') + self.assertEqual(repr(unspecified), '') + self.assertEqual(repr(NULL), '') # test that fail fails with support.captured_stdout() as stdout: errmsg = 'The igloos are melting' - with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: - clinic.fail(errmsg, filename='clown.txt', line_number=69) + with self.assertRaisesRegex(ClinicError, errmsg) as cm: + fail(errmsg, filename='clown.txt', line_number=69) exc = cm.exception self.assertEqual(exc.filename, 'clown.txt') self.assertEqual(exc.lineno, 69) @@ -2286,10 +2458,10 @@ def test_non_ascii_character_in_docstring(self): self.parse(block) # The line numbers are off; this is a known limitation. expected = dedent("""\ - Warning in file 'clinic_tests' on line 0: + Warning: Non-ascii characters are not allowed in docstrings: 'á' - Warning in file 'clinic_tests' on line 0: + Warning: Non-ascii characters are not allowed in docstrings: 'ü', 'á', 'ß' """) @@ -2390,7 +2562,7 @@ def test_state_func_docstring_no_summary(self): docstring1 docstring2 """ - self.expect_failure(block, err, lineno=0) + self.expect_failure(block, err, lineno=3) def test_state_func_docstring_only_one_param_template(self): err = "You may not specify {parameters} more than once in a docstring!" @@ -2404,12 +2576,34 @@ def test_state_func_docstring_only_one_param_template(self): these are the params again: {parameters} """ - self.expect_failure(block, err, lineno=0) + self.expect_failure(block, err, lineno=7) + + def test_kind_defining_class(self): + function = self.parse_function(""" + module m + class m.C "PyObject *" "" + m.C.meth + cls: defining_class + """, signatures_in_block=3, function_index=2) + p = function.parameters['cls'] + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + def test_disallow_defining_class_at_module_level(self): + err = "A 'defining_class' parameter cannot be defined at module level." + block = """ + module m + m.func + cls: defining_class + """ + self.expect_failure(block, err, lineno=2) class ClinicExternalTest(TestCase): maxDiff = None + def setUp(self): + save_restore_converters(self) + def run_clinic(self, *args): with ( support.captured_stdout() as out, @@ -2500,7 +2694,7 @@ def test_cli_force(self): # Verify by checking the checksum. checksum = ( "/*[clinic end generated code: " - "output=c16447c01510dfb3 input=9543a8d2da235301]*/\n" + "output=0acbef4794cb933e input=9543a8d2da235301]*/\n" ) with open(fn, encoding='utf-8') as f: generated = f.read() @@ -2634,9 +2828,9 @@ def test_cli_converters(self): bool() double() float() - init() int() long() + object() Py_ssize_t() size_t() unsigned_int() @@ -3184,44 +3378,85 @@ def test_keyword_only_parameter(self): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) - def test_posonly_vararg(self): - with self.assertRaises(TypeError): - ac_tester.posonly_vararg() - self.assertEqual(ac_tester.posonly_vararg(1, 2), (1, 2, ())) - self.assertEqual(ac_tester.posonly_vararg(1, b=2), (1, 2, ())) - self.assertEqual(ac_tester.posonly_vararg(1, 2, 3, 4), (1, 2, (3, 4))) - with self.assertRaises(TypeError): - ac_tester.posonly_vararg(b=4) - with self.assertRaises(TypeError): - ac_tester.posonly_vararg(1, 2, 3, b=4) - - def test_vararg_and_posonly(self): - with self.assertRaises(TypeError): - ac_tester.vararg_and_posonly() - with self.assertRaises(TypeError): - ac_tester.vararg_and_posonly(1, b=2) - self.assertEqual(ac_tester.vararg_and_posonly(1, 2, 3, 4), (1, (2, 3, 4))) - - def test_vararg(self): - with self.assertRaises(TypeError): - ac_tester.vararg() - with self.assertRaises(TypeError): - ac_tester.vararg(1, b=2) - self.assertEqual(ac_tester.vararg(1, 2, 3, 4), (1, (2, 3, 4))) - - def test_vararg_with_default(self): - with self.assertRaises(TypeError): - ac_tester.vararg_with_default() - self.assertEqual(ac_tester.vararg_with_default(1, b=False), (1, (), False)) - self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4), (1, (2, 3, 4), False)) - self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4, b=True), (1, (2, 3, 4), True)) + def test_varpos(self): + # fn(*args) + fn = ac_tester.varpos + self.assertEqual(fn(), ((),)) + self.assertEqual(fn(1, 2), ((1, 2),)) - def test_vararg_with_only_defaults(self): - self.assertEqual(ac_tester.vararg_with_only_defaults(), ((), None)) - self.assertEqual(ac_tester.vararg_with_only_defaults(b=2), ((), 2)) - self.assertEqual(ac_tester.vararg_with_only_defaults(1, b=2), ((1, ), 2)) - self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4), ((1, 2, 3, 4), None)) - self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + def test_posonly_varpos(self): + # fn(a, b, /, *args) + fn = ac_tester.posonly_varpos + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1) + self.assertRaises(TypeError, fn, 1, b=2) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + + def test_posonly_poskw_varpos(self): + # fn(a, /, b, *args) + fn = ac_tester.posonly_poskw_varpos + self.assertRaises(TypeError, fn) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, b=2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + self.assertRaises(TypeError, fn, b=4) + self.assertRaises(TypeError, fn, 1, 2, 3, b=4) + + def test_poskw_varpos(self): + # fn(a, *args) + fn = ac_tester.poskw_varpos + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1, b=2) + self.assertEqual(fn(a=1), (1, ())) + self.assertRaises(TypeError, fn, 1, a=2) + self.assertEqual(fn(1), (1, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4))) + + def test_poskw_varpos_kwonly_opt(self): + # fn(a, *args, b=False) + fn = ac_tester.poskw_varpos_kwonly_opt + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), True)) + self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4), False)) + self.assertEqual(fn(1, 2, 3, 4, b=5), (1, (2, 3, 4), True)) + self.assertEqual(fn(a=1), (1, (), False)) + self.assertEqual(fn(a=1, b=2), (1, (), True)) + + def test_poskw_varpos_kwonly_opt2(self): + # fn(a, *args, b=False, c=False) + fn = ac_tester.poskw_varpos_kwonly_opt2 + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), 2, False)) + self.assertEqual(fn(1, b=2, c=3), (1, (), 2, 3)) + self.assertEqual(fn(1, 2, 3), (1, (2, 3), False, False)) + self.assertEqual(fn(1, 2, 3, b=4), (1, (2, 3), 4, False)) + self.assertEqual(fn(1, 2, 3, b=4, c=5), (1, (2, 3), 4, 5)) + self.assertEqual(fn(a=1), (1, (), False, False)) + self.assertEqual(fn(a=1, b=2), (1, (), 2, False)) + self.assertEqual(fn(a=1, b=2, c=3), (1, (), 2, 3)) + + def test_varpos_kwonly_opt(self): + # fn(*args, b=False) + fn = ac_tester.varpos_kwonly_opt + self.assertEqual(fn(), ((), False)) + self.assertEqual(fn(b=2), ((), 2)) + self.assertEqual(fn(1, b=2), ((1, ), 2)) + self.assertEqual(fn(1, 2, 3, 4), ((1, 2, 3, 4), False)) + self.assertEqual(fn(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + + def test_varpos_kwonly_req_opt(self): + fn = ac_tester.varpos_kwonly_req_opt + self.assertRaises(TypeError, fn) + self.assertEqual(fn(a=1), ((), 1, False, False)) + self.assertEqual(fn(a=1, b=2), ((), 1, 2, False)) + self.assertEqual(fn(a=1, b=2, c=3), ((), 1, 2, 3)) + self.assertRaises(TypeError, fn, 1) + self.assertEqual(fn(1, a=2), ((1,), 2, False, False)) + self.assertEqual(fn(1, a=2, b=3), ((1,), 2, 3, False)) + self.assertEqual(fn(1, a=2, b=3, c=4), ((1,), 2, 3, 4)) def test_gh_32092_oob(self): ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) @@ -3245,35 +3480,19 @@ def test_gh_99240_double_free(self): ac_tester.gh_99240_double_free('a', '\0b') def test_null_or_tuple_for_varargs(self): + # fn(name, *constraints, covariant=False) + fn = ac_tester.null_or_tuple_for_varargs # All of these should not crash: - valid_args_for_test = [ - (('a',), {}, - ('a', (), False)), - (('a', 1, 2, 3), {'covariant': True}, - ('a', (1, 2, 3), True)), - ((), {'name': 'a'}, - ('a', (), False)), - ((), {'name': 'a', 'covariant': True}, - ('a', (), True)), - ((), {'covariant': True, 'name': 'a'}, - ('a', (), True)), - ] - for args, kwargs, expected in valid_args_for_test: - with self.subTest(args=args, kwargs=kwargs): - self.assertEqual( - ac_tester.null_or_tuple_for_varargs(*args, **kwargs), - expected, - ) + self.assertEqual(fn('a'), ('a', (), False)) + self.assertEqual(fn('a', 1, 2, 3, covariant=True), ('a', (1, 2, 3), True)) + self.assertEqual(fn(name='a'), ('a', (), False)) + self.assertEqual(fn(name='a', covariant=True), ('a', (), True)) + self.assertEqual(fn(covariant=True, name='a'), ('a', (), True)) - def test_null_or_tuple_for_varargs_error(self): - with self.assertRaises(TypeError): - ac_tester.null_or_tuple_for_varargs(covariant=True) - with self.assertRaises(TypeError): - ac_tester.null_or_tuple_for_varargs(1, name='a') - with self.assertRaises(TypeError): - ac_tester.null_or_tuple_for_varargs(1, 2, 3, name='a', covariant=True) - with self.assertRaises(TypeError): - ac_tester.null_or_tuple_for_varargs(1, 2, 3, covariant=True, name='a') + self.assertRaises(TypeError, fn, covariant=True) + self.assertRaises(TypeError, fn, 1, name='a') + self.assertRaises(TypeError, fn, 1, 2, 3, name='a', covariant=True) + self.assertRaises(TypeError, fn, 1, 2, 3, covariant=True, name='a') def test_cloned_func_exception_message(self): incorrect_arg = -1 # f1() and f2() accept a single str @@ -3288,26 +3507,78 @@ def test_cloned_func_with_converter_exception_message(self): func = getattr(ac_tester, name) self.assertEqual(func(), name) - def test_meth_method_no_params(self): + def test_get_defining_class(self): obj = ac_tester.TestClass() - meth = obj.meth_method_no_params + meth = obj.get_defining_class + self.assertIs(obj.get_defining_class(), ac_tester.TestClass) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass) + check = partial(self.assertRaisesRegex, TypeError, "no arguments") check(meth, 1) check(meth, a=1) - def test_meth_method_no_params_capi(self): + def test_get_defining_class_capi(self): from _testcapi import pyobject_vectorcall obj = ac_tester.TestClass() - meth = obj.meth_method_no_params + meth = obj.get_defining_class pyobject_vectorcall(meth, None, None) pyobject_vectorcall(meth, (), None) pyobject_vectorcall(meth, (), ()) pyobject_vectorcall(meth, None, ()) + self.assertIs(pyobject_vectorcall(meth, (), ()), ac_tester.TestClass) check = partial(self.assertRaisesRegex, TypeError, "no arguments") check(pyobject_vectorcall, meth, (1,), None) check(pyobject_vectorcall, meth, (1,), ("a",)) + def test_get_defining_class_arg(self): + obj = ac_tester.TestClass() + self.assertEqual(obj.get_defining_class_arg("arg"), + (ac_tester.TestClass, "arg")) + self.assertEqual(obj.get_defining_class_arg(arg=123), + (ac_tester.TestClass, 123)) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass, arg="arg") + + # wrong number of arguments + with self.assertRaises(TypeError): + obj.get_defining_class_arg() + with self.assertRaises(TypeError): + obj.get_defining_class_arg("arg1", "arg2") + + def test_defclass_varpos(self): + # fn(*args) + cls = ac_tester.TestClass + obj = cls() + fn = obj.defclass_varpos + self.assertEqual(fn(), (cls, ())) + self.assertEqual(fn(1, 2), (cls, (1, 2))) + fn = cls.defclass_varpos + self.assertRaises(TypeError, fn) + self.assertEqual(fn(obj), (cls, ())) + self.assertEqual(fn(obj, 1, 2), (cls, (1, 2))) + + def test_defclass_posonly_varpos(self): + # fn(a, b, /, *args) + cls = ac_tester.TestClass + obj = cls() + fn = obj.defclass_posonly_varpos + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1) + self.assertEqual(fn(1, 2), (cls, 1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (cls, 1, 2, (3, 4))) + fn = cls.defclass_posonly_varpos + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, obj) + self.assertRaises(TypeError, fn, obj, 1) + self.assertEqual(fn(obj, 1, 2), (cls, 1, 2, ())) + self.assertEqual(fn(obj, 1, 2, 3, 4), (cls, 1, 2, (3, 4))) + def test_depr_star_new(self): cls = ac_tester.DeprStarNew cls() @@ -3594,6 +3865,46 @@ def test_depr_multi(self): self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") +class LimitedCAPIOutputTests(unittest.TestCase): + + def setUp(self): + self.clinic = _make_clinic(limited_capi=True) + + @staticmethod + def wrap_clinic_input(block): + return dedent(f""" + /*[clinic input] + output everything buffer + {block} + [clinic start generated code]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + + def test_limited_capi_float(self): + block = self.wrap_clinic_input(""" + func + f: float + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("float f;", generated) + self.assertIn("f = (float) PyFloat_AsDouble", generated) + + def test_limited_capi_double(self): + block = self.wrap_clinic_input(""" + func + f: double + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("double f;", generated) + self.assertIn("f = PyFloat_AsDouble", generated) + + try: import _testclinic_limited except ImportError: @@ -3624,6 +3935,53 @@ def test_my_int_sum(self): with self.assertRaises(TypeError): _testclinic_limited.my_int_sum(1, "str") + def test_my_double_sum(self): + for func in ( + _testclinic_limited.my_float_sum, + _testclinic_limited.my_double_sum, + ): + with self.subTest(func=func.__name__): + self.assertEqual(func(1.0, 2.5), 3.5) + with self.assertRaises(TypeError): + func() + with self.assertRaises(TypeError): + func(1) + with self.assertRaises(TypeError): + func(1., "2") + + def test_get_file_descriptor(self): + # test 'file descriptor' converter: call PyObject_AsFileDescriptor() + get_fd = _testclinic_limited.get_file_descriptor + + class MyInt(int): + pass + + class MyFile: + def __init__(self, fd): + self._fd = fd + def fileno(self): + return self._fd + + for fd in (0, 1, 2, 5, 123_456): + self.assertEqual(get_fd(fd), fd) + + myint = MyInt(fd) + self.assertEqual(get_fd(myint), fd) + + myfile = MyFile(fd) + self.assertEqual(get_fd(myfile), fd) + + with self.assertRaises(OverflowError): + get_fd(2**256) + with self.assertWarnsRegex(RuntimeWarning, + "bool is used as a file descriptor"): + get_fd(True) + with self.assertRaises(TypeError): + get_fd(1.0) + with self.assertRaises(TypeError): + get_fd("abc") + with self.assertRaises(TypeError): + get_fd(None) class PermutationTests(unittest.TestCase): @@ -3637,7 +3995,7 @@ def test_permute_left_option_groups(self): (1, 2, 3), ) data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. - actual = tuple(clinic.permute_left_option_groups(data)) + actual = tuple(permute_left_option_groups(data)) self.assertEqual(actual, expected) def test_permute_right_option_groups(self): @@ -3648,7 +4006,7 @@ def test_permute_right_option_groups(self): (1, 2, 3), ) data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. - actual = tuple(clinic.permute_right_option_groups(data)) + actual = tuple(permute_right_option_groups(data)) self.assertEqual(actual, expected) def test_permute_optional_groups(self): @@ -3727,7 +4085,7 @@ def test_permute_optional_groups(self): for params in dataset: with self.subTest(**params): left, required, right, expected = params.values() - permutations = clinic.permute_optional_groups(left, required, right) + permutations = permute_optional_groups(left, required, right) actual = tuple(permutations) self.assertEqual(actual, expected) @@ -3868,15 +4226,15 @@ def test_suffix_all_lines(self): class ClinicReprTests(unittest.TestCase): def test_Block_repr(self): - block = clinic.Block("foo") + block = Block("foo") expected_repr = "" self.assertEqual(repr(block), expected_repr) - block2 = clinic.Block("bar", "baz", [], "eggs", "spam") + block2 = Block("bar", "baz", [], "eggs", "spam") expected_repr_2 = "" self.assertEqual(repr(block2), expected_repr_2) - block3 = clinic.Block( + block3 = Block( input="longboi_" * 100, dsl_name="wow_so_long", signatures=[], @@ -3891,47 +4249,44 @@ def test_Block_repr(self): def test_Destination_repr(self): c = _make_clinic() - destination = clinic.Destination( + destination = Destination( "foo", type="file", clinic=c, args=("eggs",) ) self.assertEqual( repr(destination), "" ) - destination2 = clinic.Destination("bar", type="buffer", clinic=c) + destination2 = Destination("bar", type="buffer", clinic=c) self.assertEqual(repr(destination2), "") def test_Module_repr(self): - module = clinic.Module("foo", _make_clinic()) + module = Module("foo", _make_clinic()) self.assertRegex(repr(module), r"") def test_Class_repr(self): - cls = clinic.Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') + cls = Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') self.assertRegex(repr(cls), r"") def test_FunctionKind_repr(self): self.assertEqual( - repr(clinic.FunctionKind.INVALID), "" - ) - self.assertEqual( - repr(clinic.FunctionKind.CLASS_METHOD), "" + repr(FunctionKind.CLASS_METHOD), "" ) def test_Function_and_Parameter_reprs(self): - function = clinic.Function( + function = Function( name='foo', module=_make_clinic(), cls=None, c_basename=None, full_name='foofoo', - return_converter=clinic.init_return_converter(), - kind=clinic.FunctionKind.METHOD_INIT, + return_converter=int_return_converter(), + kind=FunctionKind.METHOD_INIT, coexist=False ) self.assertEqual(repr(function), "") - converter = clinic.self_converter('bar', 'bar', function) - parameter = clinic.Parameter( + converter = self_converter('bar', 'bar', function) + parameter = Parameter( "bar", kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, function=function, diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 57f80d5d8cd016..a96a5780b31b6f 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -1,4 +1,5 @@ from test.support import requires_IEEE_754, cpython_only, import_helper +from test.support.testcase import ComplexesAreIdenticalMixin from test.test_math import parse_testfile, test_file import test.test_math as test_math import unittest @@ -49,7 +50,7 @@ (INF, NAN) ]] -class CMathTests(unittest.TestCase): +class CMathTests(ComplexesAreIdenticalMixin, unittest.TestCase): # list of all functions in cmath test_functions = [getattr(cmath, fname) for fname in [ 'acos', 'acosh', 'asin', 'asinh', 'atan', 'atanh', @@ -65,39 +66,6 @@ def setUp(self): def tearDown(self): self.test_values.close() - def assertFloatIdentical(self, x, y): - """Fail unless floats x and y are identical, in the sense that: - (1) both x and y are nans, or - (2) both x and y are infinities, with the same sign, or - (3) both x and y are zeros, with the same sign, or - (4) x and y are both finite and nonzero, and x == y - - """ - msg = 'floats {!r} and {!r} are not identical' - - if math.isnan(x) or math.isnan(y): - if math.isnan(x) and math.isnan(y): - return - elif x == y: - if x != 0.0: - return - # both zero; check that signs match - elif math.copysign(1.0, x) == math.copysign(1.0, y): - return - else: - msg += ': zeros have different signs' - self.fail(msg.format(x, y)) - - def assertComplexIdentical(self, x, y): - """Fail unless complex numbers x and y have equal values and signs. - - In particular, if x and y both have real (or imaginary) part - zero, but the zeros have different signs, this test will fail. - - """ - self.assertFloatIdentical(x.real, y.real) - self.assertFloatIdentical(x.imag, y.imag) - def rAssertAlmostEqual(self, a, b, rel_err = 2e-15, abs_err = 5e-323, msg=None): """Fail if the two floating-point numbers are not almost equal. @@ -555,7 +523,7 @@ def test_isinf(self): @requires_IEEE_754 def testTanhSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.tanh(z), z) + self.assertComplexesAreIdentical(cmath.tanh(z), z) # The algorithm used for atan and atanh makes use of the system # log1p function; If that system function doesn't respect the sign @@ -564,12 +532,12 @@ def testTanhSign(self): @requires_IEEE_754 def testAtanSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.atan(z), z) + self.assertComplexesAreIdentical(cmath.atan(z), z) @requires_IEEE_754 def testAtanhSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.atanh(z), z) + self.assertComplexesAreIdentical(cmath.atanh(z), z) class IsCloseTests(test_math.IsCloseTests): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 1fe3b2fe53c0b6..35725718152c56 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -5,11 +5,13 @@ import os import subprocess import sys +import sysconfig import tempfile import textwrap import unittest from test import support from test.support import os_helper +from test.support import force_not_colorized from test.support.script_helper import ( spawn_python, kill_python, assert_python_ok, assert_python_failure, interpreter_requires_environment @@ -38,6 +40,7 @@ def verify_valid_flag(self, cmd_line): self.assertNotIn(b'Traceback', err) return out + @support.cpython_only def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -48,14 +51,17 @@ def test_help(self): self.assertNotIn(b'-X dev', out) self.assertLess(len(lines), 50) + @support.cpython_only def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) + @support.cpython_only def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) + @support.cpython_only def test_help_all(self): out = self.verify_valid_flag('--help-all') lines = out.splitlines() @@ -74,6 +80,7 @@ def test_optimize(self): def test_site_flag(self): self.verify_valid_flag('-S') + @support.cpython_only def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': @@ -139,6 +146,7 @@ def run_python(*args): else: self.assertEqual(err, b'') + @support.cpython_only def test_xoption_frozen_modules(self): tests = { ('=on', 'FrozenImporter'), @@ -153,6 +161,7 @@ def test_xoption_frozen_modules(self): res = assert_python_ok(*cmd) self.assertRegex(res.out.decode('utf-8'), expected) + @support.cpython_only def test_env_var_frozen_modules(self): tests = { ('on', 'FrozenImporter'), @@ -579,6 +588,7 @@ def test_del___main__(self): print("del sys.modules['__main__']", file=script) assert_python_ok(filename) + @support.cpython_only def test_unknown_options(self): rc, out, err = assert_python_failure('-E', '-z') self.assertIn(b'Unknown option: -z', err) @@ -634,15 +644,13 @@ def test_sys_flags_set(self): PYTHONDONTWRITEBYTECODE=value, PYTHONVERBOSE=value, ) - dont_write_bytecode = int(bool(value)) + expected_bool = int(bool(value)) code = ( "import sys; " "sys.stderr.write(str(sys.flags)); " f"""sys.exit(not ( - sys.flags.debug == sys.flags.optimize == - sys.flags.verbose == - {expected} - and sys.flags.dont_write_bytecode == {dont_write_bytecode} + sys.flags.optimize == sys.flags.verbose == {expected} + and sys.flags.debug == sys.flags.dont_write_bytecode == {expected_bool} ))""" ) with self.subTest(envar_value=value): @@ -693,6 +701,7 @@ def run_xdev(self, *args, check_exitcode=True, xdev=True): self.assertEqual(proc.returncode, 0, proc) return proc.stdout.rstrip() + @support.cpython_only def test_xdev(self): # sys.flags.dev_mode code = "import sys; print(sys.flags.dev_mode)" @@ -729,7 +738,7 @@ def test_xdev(self): # Memory allocator debug hooks try: - import _testinternalcapi + import _testinternalcapi # noqa: F401 except ImportError: pass else: @@ -746,7 +755,7 @@ def test_xdev(self): # Faulthandler try: - import faulthandler + import faulthandler # noqa: F401 except ImportError: pass else: @@ -871,6 +880,118 @@ def test_pythondevmode_env(self): self.assertEqual(proc.stdout.rstrip(), 'True') self.assertEqual(proc.returncode, 0, proc) + def test_python_gil(self): + cases = [ + # (env, opt, expected, msg) + ('1', None, '1', "PYTHON_GIL=1"), + (None, '1', '1', "-X gil=1"), + ] + + if support.Py_GIL_DISABLED: + cases.extend( + [ + (None, None, 'None', "no options set"), + ('0', None, '0', "PYTHON_GIL=0"), + ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"), + (None, '0', '0', "-X gil=0"), + ] + ) + else: + cases.extend( + [ + (None, None, '1', '-X gil=0 (unsupported by this build)'), + ('1', None, '1', 'PYTHON_GIL=0 (unsupported by this build)'), + ] + ) + code = "import sys; print(sys.flags.gil)" + environ = dict(os.environ) + + for env, opt, expected, msg in cases: + with self.subTest(msg, env=env, opt=opt): + environ.pop('PYTHON_GIL', None) + if env is not None: + environ['PYTHON_GIL'] = env + extra_args = [] + if opt is not None: + extra_args = ['-X', f'gil={opt}'] + + proc = subprocess.run([sys.executable, *extra_args, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, env=environ) + self.assertEqual(proc.returncode, 0, proc) + self.assertEqual(proc.stdout.rstrip(), expected) + self.assertEqual(proc.stderr, '') + + def test_python_asyncio_debug(self): + code = "import asyncio; print(asyncio.get_event_loop().get_debug())" + rc, out, err = assert_python_ok('-c', code, PYTHONASYNCIODEBUG='1') + self.assertIn(b'True', out) + + @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") + def test_python_dump_refs(self): + code = 'import sys; sys._clear_type_cache()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') + self.assertEqual(rc, 0) + + @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") + def test_python_dump_refs_file(self): + with tempfile.NamedTemporaryFile() as dump_file: + code = 'import sys; sys._clear_type_cache()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) + self.assertEqual(rc, 0) + with open(dump_file.name, 'r') as file: + contents = file.read() + self.assertIn('Remaining objects', contents) + + @unittest.skipUnless(sys.platform == 'darwin', 'PYTHONEXECUTABLE only works on macOS') + def test_python_executable(self): + code = 'import sys; print(sys.executable)' + expected = "/busr/bbin/bpython" + rc, out, err = assert_python_ok('-c', code, PYTHONEXECUTABLE=expected) + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_fs_encoding(self): + code = "import sys; print(sys.getfilesystemencoding())" + expected = 'mbcs' + rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSFSENCODING='1') + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_stdio(self): + code = "import sys; print(sys.stdin.encoding, sys.stdout.encoding)" + expected = 'cp' + rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSSTDIO='1') + self.assertIn(expected.encode(), out) + + @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), + "PYTHONMALLOCSTATS doesn't work with ASAN") + def test_python_malloc_stats(self): + code = "pass" + rc, out, err = assert_python_ok('-c', code, PYTHONMALLOCSTATS='1') + self.assertIn(b'Small block threshold', err) + + def test_python_user_base(self): + code = "import site; print(site.USER_BASE)" + expected = "/custom/userbase" + rc, out, err = assert_python_ok('-c', code, PYTHONUSERBASE=expected) + self.assertIn(expected.encode(), out) + + def test_python_basic_repl(self): + # Currently this only tests that the env var is set. See test_pyrepl.test_python_basic_repl. + code = "import os; print('PYTHON_BASIC_REPL' in os.environ)" + expected = "True" + rc, out, err = assert_python_ok('-c', code, PYTHON_BASIC_REPL='1') + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(sysconfig.get_config_var('HAVE_PERF_TRAMPOLINE'), "Requires HAVE_PERF_TRAMPOLINE support") + def test_python_perf_jit_support(self): + code = "import sys; print(sys.is_stack_trampoline_active())" + expected = "True" + rc, out, err = assert_python_ok('-c', code, PYTHON_PERF_JIT_SUPPORT='1') + self.assertIn(expected.encode(), out) + @unittest.skipUnless(sys.platform == 'win32', 'bpo-32457 only applies on Windows') def test_argv0_normalization(self): @@ -883,6 +1004,7 @@ def test_argv0_normalization(self): self.assertEqual(proc.returncode, 0, proc) self.assertEqual(proc.stdout.strip(), b'0') + @support.cpython_only def test_parsing_error(self): args = [sys.executable, '-I', '--unknown-option'] proc = subprocess.run(args, @@ -939,7 +1061,7 @@ def test_cpu_count_default(self): self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHON_CPU_COUNT='1234') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - es = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') + res = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) def res2int(self, res): @@ -986,6 +1108,7 @@ def test_sys_flags_not_set(self): class SyntaxErrorTests(unittest.TestCase): + @force_not_colorized def check_string(self, code): proc = subprocess.run([sys.executable, "-"], input=code, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 3a5a8abf81e43d..f30107225ff612 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -543,7 +543,7 @@ def test_pep_409_verbiage(self): script = textwrap.dedent("""\ try: raise ValueError - except: + except ValueError: raise NameError from None """) with os_helper.temp_dir() as script_dir: diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 46bebfc7af675b..93c65a82508dcb 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -17,7 +17,7 @@ freevars: () nlocals: 2 flags: 3 -consts: ('None', '') +consts: ('',) >>> dump(f(4).__code__) name: g @@ -86,7 +86,7 @@ cellvars: () freevars: () nlocals: 0 -flags: 3 +flags: 67108867 consts: ("'doc string'", 'None') >>> def keywordonly_args(a,b,*,k1): @@ -123,6 +123,61 @@ flags: 3 consts: ('None',) +>>> def has_docstring(x: str): +... 'This is a one-line doc string' +... x += x +... x += "hello world" +... # co_flags should be 0x4000003 = 67108867 +... return x + +>>> dump(has_docstring.__code__) +name: has_docstring +argcount: 1 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: ('x',) +cellvars: () +freevars: () +nlocals: 1 +flags: 67108867 +consts: ("'This is a one-line doc string'", "'hello world'") + +>>> async def async_func_docstring(x: str, y: str): +... "This is a docstring from async function" +... import asyncio +... await asyncio.sleep(1) +... # co_flags should be 0x4000083 = 67108995 +... return x + y + +>>> dump(async_func_docstring.__code__) +name: async_func_docstring +argcount: 2 +posonlyargcount: 0 +kwonlyargcount: 0 +names: ('asyncio', 'sleep') +varnames: ('x', 'y', 'asyncio') +cellvars: () +freevars: () +nlocals: 3 +flags: 67108995 +consts: ("'This is a docstring from async function'", 'None') + +>>> def no_docstring(x, y, z): +... return x + "hello" + y + z + "world" + +>>> dump(no_docstring.__code__) +name: no_docstring +argcount: 3 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: ('x', 'y', 'z') +cellvars: () +freevars: () +nlocals: 3 +flags: 3 +consts: ("'hello'", "'world'") """ import copy @@ -141,11 +196,10 @@ ctypes = None from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, - gc_collect) + gc_collect, Py_GIL_DISABLED) from test.support.script_helper import assert_python_ok -from test.support import threading_helper -from test.support.bytecode_helper import (BytecodeTestCase, - instructions_with_positions) +from test.support import threading_helper, import_helper +from test.support.bytecode_helper import instructions_with_positions from opcode import opmap, opname COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -176,7 +230,7 @@ class CodeTest(unittest.TestCase): @cpython_only def test_newempty(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") co = _testcapi.code_newempty("filename", "funcname", 15) self.assertEqual(co.co_filename, "filename") self.assertEqual(co.co_name, "funcname") @@ -255,10 +309,11 @@ def func(): return x code = func.__code__ - # different co_name, co_varnames, co_consts + # Different co_name, co_varnames, co_consts. + # Must have the same number of constants and + # variables or we get crashes. def func2(): y = 2 - z = 3 return y code2 = func2.__code__ @@ -570,11 +625,30 @@ def f(a='str_value'): self.assertIsInterned(f()) @cpython_only + @unittest.skipIf(Py_GIL_DISABLED, "free-threaded build interns all string constants") def test_interned_string_with_null(self): co = compile(r'res = "str\0value!"', '?', 'exec') v = self.find_const(co.co_consts, 'str\0value!') self.assertIsNotInterned(v) + @cpython_only + @unittest.skipUnless(Py_GIL_DISABLED, "does not intern all constants") + def test_interned_constants(self): + # compile separately to avoid compile time de-duping + + globals = {} + exec(textwrap.dedent(""" + def func1(): + return (0.0, (1, 2, "hello")) + """), globals) + + exec(textwrap.dedent(""" + def func2(): + return (0.0, (1, 2, "hello")) + """), globals) + + self.assertTrue(globals["func1"]() is globals["func2"]()) + class CodeWeakRefTest(unittest.TestCase): @@ -761,7 +835,7 @@ def f(): co_code=bytes( [ dis.opmap["RESUME"], 0, - dis.opmap["LOAD_ASSERTION_ERROR"], 0, + dis.opmap["LOAD_COMMON_CONSTANT"], 0, dis.opmap["RAISE_VARARGS"], 1, ] ), @@ -835,6 +909,7 @@ def test_free_called(self): SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) del f + gc_collect() # For free-threaded build self.assertEqual(LAST_FREED, 100) def test_get_set(self): @@ -866,13 +941,18 @@ def __init__(self, f, test): def run(self): del self.f gc_collect() - self.test.assertEqual(LAST_FREED, 500) + # gh-117683: In the free-threaded build, the code object's + # destructor may still be running concurrently in the main + # thread. + if not Py_GIL_DISABLED: + self.test.assertEqual(LAST_FREED, 500) SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) tt = ThreadTest(f, self) del f tt.start() tt.join() + gc_collect() # For free-threaded build self.assertEqual(LAST_FREED, 500) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 259778a5cade98..37c7bc772ed8c7 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -1,5 +1,6 @@ "Test InteractiveConsole and InteractiveInterpreter from code module" import sys +import traceback import unittest from textwrap import dedent from contextlib import ExitStack @@ -30,6 +31,7 @@ def mock_sys(self): class TestInteractiveConsole(unittest.TestCase, MockSys): + maxDiff = None def setUp(self): self.console = code.InteractiveConsole() @@ -61,21 +63,151 @@ def test_console_stderr(self): raise AssertionError("no console stdout") def test_syntax_error(self): - self.infunc.side_effect = ["undefined", EOFError('Finished')] + self.infunc.side_effect = ["def f():", + " x = ?", + "", + EOFError('Finished')] self.console.interact() - for call in self.stderr.method_calls: - if 'NameError' in ''.join(call[1]): - break - else: - raise AssertionError("No syntax error from console") + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 2', + ' x = ?', + ' ^', + 'SyntaxError: invalid syntax']) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + def test_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 1', + ' 1', + 'IndentationError: unexpected indent']) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + def test_unicode_error(self): + self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[output.index('\n') + 1:] + self.assertTrue(output.startswith('UnicodeEncodeError: '), output) + self.assertIs(self.sysmod.last_type, UnicodeEncodeError) + self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_sysexcepthook(self): - self.infunc.side_effect = ["raise ValueError('')", + self.infunc.side_effect = ["def f():", + " raise ValueError('BOOM!')", + "", + "f()", + EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + 'Traceback (most recent call last):\n', + ' File "", line 1, in \n', + ' File "", line 2, in f\n', + 'ValueError: BOOM!\n']) + + def test_sysexcepthook_syntax_error(self): + self.infunc.side_effect = ["def f():", + " x = ?", + "", EOFError('Finished')] hook = mock.Mock() self.sysmod.excepthook = hook self.console.interact() - self.assertTrue(hook.called) + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 2\n', + ' x = ?\n', + ' ^\n', + 'SyntaxError: invalid syntax\n']) + + def test_sysexcepthook_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 1\n', + ' 1\n', + 'IndentationError: unexpected indent\n']) + + def test_sysexcepthook_crashing_doesnt_close_repl(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + self.sysmod.excepthook = 1 + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("'int' object is not callable"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_BaseException(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + s = "not so fast" + def raise_base(*args, **kwargs): + raise BaseException(s) + self.sysmod.excepthook = raise_base + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("not so fast"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_SystemExit_gets_through(self): + self.infunc.side_effect = ["1/0"] + def raise_base(*args, **kwargs): + raise SystemExit + self.sysmod.excepthook = raise_base + with self.assertRaises(SystemExit): + self.console.interact() def test_banner(self): # with banner @@ -134,6 +266,11 @@ def test_cause_tb(self): ValueError """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_context_tb(self): self.infunc.side_effect = ["try: ham\nexcept: eggs\n", @@ -152,6 +289,11 @@ def test_context_tb(self): NameError: name 'eggs' is not defined """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, NameError) + self.assertIs(type(self.sysmod.last_value), NameError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) class TestInteractiveConsoleLocalExit(unittest.TestCase, MockSys): diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 4991330489d139..86e5e5c1474674 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1,3 +1,4 @@ +from _codecs import _unregister_error as _codecs_unregister_error import codecs import html.entities import itertools @@ -1210,7 +1211,6 @@ def replace_with_long(exc): '\ufffd\x00\x00' ) - def test_fake_error_class(self): handlers = [ codecs.strict_errors, @@ -1235,6 +1235,31 @@ class FakeUnicodeError(Exception): with self.assertRaises((TypeError, FakeUnicodeError)): handler(FakeUnicodeError()) + def test_reject_unregister_builtin_error_handler(self): + for name in [ + 'strict', 'ignore', 'replace', 'backslashreplace', 'namereplace', + 'xmlcharrefreplace', 'surrogateescape', 'surrogatepass', + ]: + with self.subTest(name): + self.assertRaises(ValueError, _codecs_unregister_error, name) + + def test_unregister_custom_error_handler(self): + def custom_handler(exc): + raise exc + + custom_name = 'test.test_unregister_custom_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + codecs.register_error(custom_name, custom_handler) + self.assertIs(codecs.lookup_error(custom_name), custom_handler) + self.assertTrue(_codecs_unregister_error(custom_name)) + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + + def test_unregister_custom_unknown_error_handler(self): + unknown_name = 'test.test_unregister_custom_unknown_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + self.assertFalse(_codecs_unregister_error(unknown_name)) + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index ff511a625a0194..290656f070503a 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2,7 +2,6 @@ import contextlib import copy import io -import locale import pickle import sys import unittest @@ -13,9 +12,9 @@ from test.support import os_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None try: import _testinternalcapi except ImportError: @@ -482,11 +481,11 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(4*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(8*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) def test_partial(self): self.check_partial( @@ -666,11 +665,11 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(b"\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(b"\xff\xff\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) def test_partial(self): self.check_partial( @@ -1356,13 +1355,29 @@ def test_decode(self): def test_decode_invalid(self): testcases = [ - (b"xn--w&", "strict", UnicodeError()), + (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")), + (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")), + (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")), + (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")), + (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")), + (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")), + (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")), + (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")), (b"xn--w&", "ignore", "xn-"), ] for puny, errors, expected in testcases: with self.subTest(puny=puny, errors=errors): if isinstance(expected, Exception): - self.assertRaises(UnicodeError, puny.decode, "punycode", errors) + with self.assertRaises(UnicodeDecodeError) as cm: + puny.decode("punycode", errors) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, puny) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) else: self.assertEqual(puny.decode("punycode", errors), expected) @@ -1532,7 +1547,7 @@ def test_nameprep(self): orig = str(orig, "utf-8", "surrogatepass") if prepped is None: # Input contains prohibited characters - self.assertRaises(UnicodeError, nameprep, orig) + self.assertRaises(UnicodeEncodeError, nameprep, orig) else: prepped = str(prepped, "utf-8", "surrogatepass") try: @@ -1542,11 +1557,46 @@ def test_nameprep(self): class IDNACodecTest(unittest.TestCase): + + invalid_decode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFFpython.org", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFFhon.org", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF.org", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"python.\xFForg", 7, 8, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"python.o\xFFrg", 8, 9, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"python.org\xFF", 10, 11, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.org", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.org", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.org", 13, 14, "")), + ] + invalid_encode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"foo.{'\xff'*60}", 4, 64, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "あさ.\u034f", 3, 4, "")), + ] + def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") self.assertEqual(str(b"python.org.", "idna"), "python.org.") self.assertEqual(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org") self.assertEqual(str(b"xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"XN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"xN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"Xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"bugs.xn--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + self.assertEqual(str(b"bugs.XN--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + + def test_builtin_decode_invalid(self): + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + case.decode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start, msg=f'reason: {exc.reason}') + self.assertEqual(exc.end, expected.end) def test_builtin_encode(self): self.assertEqual("python.org".encode("idna"), b"python.org") @@ -1554,10 +1604,21 @@ def test_builtin_encode(self): self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org") self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") + def test_builtin_encode_invalid(self): + for case, expected in self.invalid_encode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + case.encode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_builtin_decode_length_limit(self): - with self.assertRaisesRegex(UnicodeError, "way too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "way too long"): (b"xn--016c"+b"a"*1100).decode("idna") - with self.assertRaisesRegex(UnicodeError, "too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "too long"): (b"xn--016c"+b"a"*70).decode("idna") def test_stream(self): @@ -1595,6 +1656,39 @@ def test_incremental_decode(self): self.assertEqual(decoder.decode(b"rg."), "org.") self.assertEqual(decoder.decode(b"", True), "") + def test_incremental_decode_invalid(self): + iterdecode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFF", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"o\xFF", 1, 2, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"org\xFF", 3, 4, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.", 13, 14, "")), + ] + for case, expected in iterdecode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + list(codecs.iterdecode((bytes([c]) for c in case), "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + decoder = codecs.getincrementaldecoder("idna")() + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + decoder.decode(case) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_incremental_encode(self): self.assertEqual( b"".join(codecs.iterencode("python.org", "idna")), @@ -1623,6 +1717,23 @@ def test_incremental_encode(self): self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.") self.assertEqual(encoder.encode("", True), b"") + def test_incremental_encode_invalid(self): + iterencode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"{'\xff'*60}", 0, 60, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "\u034f", 0, 1, "")), + ] + for case, expected in iterencode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + list(codecs.iterencode(case, "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + # codecs.getincrementalencoder.encode() does not throw an error + def test_errors(self): """Only supports "strict" error handler""" "python.org".encode("idna", "strict") @@ -1700,16 +1811,10 @@ def test_getwriter(self): self.assertRaises(TypeError, codecs.getwriter) self.assertRaises(LookupError, codecs.getwriter, "__spam__") + @support.run_with_locale('LC_CTYPE', 'tr_TR') def test_lookup_issue1813(self): # Issue #1813: under Turkish locales, lookup of some codecs failed # because 'I' is lowercased as "ı" (dotless i) - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') c = codecs.lookup('ASCII') self.assertEqual(c.name, 'ascii') @@ -2112,14 +2217,14 @@ def test_basics(self): "encoding=%r" % encoding) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_basics_capi(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: if encoding not in broken_unicode_with_stateful: # check incremental decoder/encoder (fetched via the C API) try: - cencoder = _testcapi.codec_incrementalencoder(encoding) + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding) except LookupError: # no IncrementalEncoder pass else: @@ -2128,7 +2233,7 @@ def test_basics_capi(self): for c in s: encodedresult += cencoder.encode(c) encodedresult += cencoder.encode("", True) - cdecoder = _testcapi.codec_incrementaldecoder(encoding) + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding) decodedresult = "" for c in encodedresult: decodedresult += cdecoder.decode(bytes([c])) @@ -2139,12 +2244,12 @@ def test_basics_capi(self): if encoding not in ("idna", "mbcs"): # check incremental decoder/encoder with errors argument try: - cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore") + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore") except LookupError: # no IncrementalEncoder pass else: encodedresult = b"".join(cencoder.encode(c) for c in s) - cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore") + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore") decodedresult = "".join(cdecoder.decode(bytes([c])) for c in encodedresult) self.assertEqual(decodedresult, s, @@ -2846,7 +2951,7 @@ def test_seek0(self): bytes_transform_encodings.append("zlib_codec") transform_aliases["zlib_codec"] = ["zip", "zlib"] try: - import bz2 + import bz2 # noqa: F401 except ImportError: pass else: diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 7e6f811e17cfa2..a24d3e3ea142b7 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1,5 +1,6 @@ """Unit tests for collections.py.""" +import array import collections import copy import doctest @@ -25,7 +26,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString, Buffer +from collections.abc import Buffer class TestUserObjects(unittest.TestCase): @@ -541,7 +542,7 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) - n = support.EXCEEDS_RECURSION_LIMIT + n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) n = len(names) @@ -1934,28 +1935,6 @@ def assert_index_same(seq1, seq2, index_args): assert_index_same( nativeseq, seqseq, (letter, start, stop)) - def test_ByteString(self): - for sample in [bytes, bytearray]: - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(sample(), ByteString) - self.assertTrue(issubclass(sample, ByteString)) - for sample in [str, list, tuple]: - with self.assertWarns(DeprecationWarning): - self.assertNotIsInstance(sample(), ByteString) - self.assertFalse(issubclass(sample, ByteString)) - with self.assertWarns(DeprecationWarning): - self.assertNotIsInstance(memoryview(b""), ByteString) - self.assertFalse(issubclass(memoryview, ByteString)) - with self.assertWarns(DeprecationWarning): - self.validate_abstract_methods(ByteString, '__getitem__', '__len__') - - with self.assertWarns(DeprecationWarning): - class X(ByteString): pass - - with self.assertWarns(DeprecationWarning): - # No metaclass conflict - class Z(ByteString, Awaitable): pass - def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) @@ -1972,6 +1951,7 @@ def test_MutableSequence(self): for sample in [list, bytearray, deque]: self.assertIsInstance(sample(), MutableSequence) self.assertTrue(issubclass(sample, MutableSequence)) + self.assertTrue(issubclass(array.array, MutableSequence)) self.assertFalse(issubclass(str, MutableSequence)) self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index ebb479f2de7c63..85ae71c1f77b28 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,5 +1,9 @@ +import contextlib import dis +import io +import itertools import math +import opcode import os import unittest import sys @@ -9,9 +13,14 @@ import types import textwrap import warnings +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + from test import support -from test.support import (script_helper, requires_debug_ranges, - requires_specialization, Py_C_RECURSION_LIMIT) +from test.support import (script_helper, requires_debug_ranges, run_code, + requires_specialization, get_c_recursion_limit) from test.support.bytecode_helper import instructions_with_positions from test.support.os_helper import FakePath @@ -112,7 +121,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = int(Py_C_RECURSION_LIMIT * 0.9) + repeat = int(get_c_recursion_limit() * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -436,8 +445,8 @@ class A: def f(): __mangled = 1 __not_mangled__ = 2 - import __mangled_mod - import __package__.module + import __mangled_mod # noqa: F401 + import __package__.module # noqa: F401 self.assertIn("_A__mangled", A.f.__code__.co_varnames) self.assertIn("__not_mangled__", A.f.__code__.co_varnames) @@ -470,6 +479,19 @@ def test_dead_code_with_except_handler_compiles(self): x = 2 """), '', 'exec') + def test_try_except_in_while_with_chained_condition_compiles(self): + # see gh-124871 + compile(textwrap.dedent(""" + name_1, name_2, name_3 = 1, 2, 3 + while name_3 <= name_2 > name_1: + try: + raise + except: + pass + finally: + pass + """), '', 'exec') + def test_compile_invalid_namedexpr(self): # gh-109351 m = ast.Module( @@ -497,6 +519,58 @@ def test_compile_invalid_namedexpr(self): with self.assertRaisesRegex(TypeError, "NamedExpr target must be a Name"): compile(ast.fix_missing_locations(m), "", "exec") + def test_compile_redundant_jumps_and_nops_after_moving_cold_blocks(self): + # See gh-120367 + code=textwrap.dedent(""" + try: + pass + except: + pass + else: + match name_2: + case b'': + pass + finally: + something + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + + def test_compile_redundant_jump_after_convert_pseudo_ops(self): + # See gh-120367 + code=textwrap.dedent(""" + if name_2: + pass + else: + try: + pass + except: + pass + ~name_5 + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + def test_compile_ast(self): fname = __file__ if fname.lower().endswith('pyc'): @@ -525,11 +599,11 @@ def test_compile_ast(self): self.assertRaises(TypeError, compile, co1, '', 'eval') # raise exception when node type is no start node - self.assertRaises(TypeError, compile, _ast.If(), '', 'exec') + self.assertRaises(TypeError, compile, _ast.If(test=_ast.Name(id='x', ctx=_ast.Load())), '', 'exec') # raise exception when node has invalid children ast = _ast.Module() - ast.body = [_ast.BoolOp()] + ast.body = [_ast.BoolOp(op=_ast.Or())] self.assertRaises(TypeError, compile, ast, '', 'exec') def test_compile_invalid_typealias(self): @@ -632,9 +706,10 @@ def test_yet_more_evil_still_undecodable(self): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): # Expected limit is Py_C_RECURSION_LIMIT - fail_depth = Py_C_RECURSION_LIMIT + 1 - crash_depth = Py_C_RECURSION_LIMIT * 100 - success_depth = int(Py_C_RECURSION_LIMIT * 0.8) + limit = get_c_recursion_limit() + fail_depth = limit + 1 + crash_depth = limit * 100 + success_depth = int(limit * 0.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth @@ -700,7 +775,6 @@ def check_same_constant(const): self.assertEqual(repr(f1()), repr(const)) check_same_constant(None) - check_same_constant(0) check_same_constant(0.0) check_same_constant(b'abc') check_same_constant('abc') @@ -747,7 +821,7 @@ def f(): return "unused" self.assertEqual(f.__code__.co_consts, - ("docstring", "used")) + (f.__doc__, "used")) @support.cpython_only def test_remove_unused_consts_no_docstring(self): @@ -760,7 +834,7 @@ def f(): return "unused" self.assertEqual(f.__code__.co_consts, - (None, "used")) + (True, "used")) @support.cpython_only def test_remove_unused_consts_extended_args(self): @@ -778,9 +852,9 @@ def test_remove_unused_consts_extended_args(self): eval(compile(code, "file.py", "exec"), g) exec(code, g) f = g['f'] - expected = tuple([None, '', 1] + [f't{i}' for i in range(N)]) + expected = tuple([''] + [f't{i}' for i in range(N)]) self.assertEqual(f.__code__.co_consts, expected) - expected = "".join(expected[3:]) + expected = "".join(expected[1:]) self.assertEqual(expected, f()) # Stripping unused constants is not a strict requirement for the @@ -792,7 +866,7 @@ def test_strip_unused_None(self): def f1(): "docstring" return 42 - self.assertEqual(f1.__code__.co_consts, ("docstring", 42)) + self.assertEqual(f1.__code__.co_consts, (f1.__doc__,)) # This is a regression test for a CPython specific peephole optimizer # implementation bug present in a few releases. It's assertion verifies @@ -809,9 +883,88 @@ def unused_code_at_end(): # RETURN_VALUE opcode. This does not always crash an interpreter. # When you build with the clang memory sanitizer it reliably aborts. self.assertEqual( - 'RETURN_CONST', + 'RETURN_VALUE', list(dis.get_instructions(unused_code_at_end))[-1].opname) + @support.cpython_only + def test_docstring(self): + src = textwrap.dedent(""" + def with_docstring(): + "docstring" + + def two_strings(): + "docstring" + "not docstring" + + def with_fstring(): + f"not docstring" + + def with_const_expression(): + "also" + " not docstring" + """) + + for opt in [0, 1, 2]: + with self.subTest(opt=opt): + code = compile(src, "", "exec", optimize=opt) + ns = {} + exec(code, ns) + + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + self.assertEqual(ns['two_strings'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + self.assertIsNone(ns['two_strings'].__doc__) + self.assertIsNone(ns['with_fstring'].__doc__) + self.assertIsNone(ns['with_const_expression'].__doc__) + + @support.cpython_only + def test_docstring_interactive_mode(self): + srcs = [ + """def with_docstring(): + "docstring" + """, + """class with_docstring: + "docstring" + """, + ] + + for opt in [0, 1, 2]: + for src in srcs: + with self.subTest(opt=opt, src=src): + code = compile(textwrap.dedent(src), "", "single", optimize=opt) + ns = {} + exec(code, ns) + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + + @support.cpython_only + def test_docstring_omitted(self): + # See gh-115347 + src = textwrap.dedent(""" + def f(): + "docstring1" + def h(): + "docstring2" + return 42 + + class C: + "docstring3" + pass + + return h + """) + for opt in [-1, 0, 1, 2]: + for mode in ["exec", "single"]: + with self.subTest(opt=opt, mode=mode): + code = compile(src, "", mode, optimize=opt) + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.dis(code) + self.assertNotIn('NOP', output.getvalue()) + def test_dont_merge_constants(self): # Issue #25843: compile() must not merge constants which are equal # but have a different type. @@ -828,7 +981,6 @@ def check_different_constants(const1, const2): self.assertEqual(repr(f1()), repr(const1)) self.assertEqual(repr(f2()), repr(const2)) - check_different_constants(0, 0.0) check_different_constants(+0.0, -0.0) check_different_constants((0,), (0.0,)) check_different_constants('a', b'a') @@ -891,8 +1043,8 @@ def unused_block_while_else(): for func in funcs: opcodes = list(dis.get_instructions(func)) - self.assertLessEqual(len(opcodes), 3) - self.assertEqual('RETURN_CONST', opcodes[-1].opname) + self.assertLessEqual(len(opcodes), 4) + self.assertEqual('RETURN_VALUE', opcodes[-1].opname) self.assertEqual(None, opcodes[-1].argval) def test_false_while_loop(self): @@ -909,8 +1061,8 @@ def continue_in_while(): # Check that we did not raise but we also don't generate bytecode for func in funcs: opcodes = list(dis.get_instructions(func)) - self.assertEqual(2, len(opcodes)) - self.assertEqual('RETURN_CONST', opcodes[1].opname) + self.assertEqual(3, len(opcodes)) + self.assertEqual('RETURN_VALUE', opcodes[-1].opname) self.assertEqual(None, opcodes[1].argval) def test_consts_in_conditionals(self): @@ -1021,6 +1173,8 @@ def no_code2(): for func in (no_code1, no_code2): with self.subTest(func=func): + if func is no_code1 and no_code1.__doc__ is None: + continue code = func.__code__ [(start, end, line)] = code.co_lines() self.assertEqual(start, 0) @@ -1088,9 +1242,9 @@ def return_genexp(): x in y) - genexp_lines = [0, 2, 0] + genexp_lines = [0, 4, 2, 0, 4] - genexp_code = return_genexp.__code__.co_consts[1] + genexp_code = return_genexp.__code__.co_consts[0] code_lines = self.get_code_lines(genexp_code) self.assertEqual(genexp_lines, code_lines) @@ -1230,6 +1384,14 @@ def check_op_count(func, op, expected): actual += 1 self.assertEqual(actual, expected) + def check_consts(func, typ, expected): + slice_consts = 0 + consts = func.__code__.co_consts + for instr in dis.Bytecode(func): + if instr.opname == "LOAD_CONST" and isinstance(consts[instr.oparg], typ): + slice_consts += 1 + self.assertEqual(slice_consts, expected) + def load(): return x[a:b] + x [a:] + x[:b] + x[:] @@ -1245,15 +1407,30 @@ def long_slice(): def aug(): x[a:b] += y - check_op_count(load, "BINARY_SLICE", 4) + def aug_const(): + x[1:2] += y + + def compound_const_slice(): + x[1:2:3, 4:5:6] = y + + check_op_count(load, "BINARY_SLICE", 3) check_op_count(load, "BUILD_SLICE", 0) - check_op_count(store, "STORE_SLICE", 4) + check_consts(load, slice, 1) + check_op_count(store, "STORE_SLICE", 3) check_op_count(store, "BUILD_SLICE", 0) + check_consts(store, slice, 1) check_op_count(long_slice, "BUILD_SLICE", 1) check_op_count(long_slice, "BINARY_SLICE", 0) check_op_count(aug, "BINARY_SLICE", 1) check_op_count(aug, "STORE_SLICE", 1) check_op_count(aug, "BUILD_SLICE", 0) + check_op_count(aug_const, "BINARY_SLICE", 0) + check_op_count(aug_const, "STORE_SLICE", 0) + check_consts(aug_const, slice, 1) + check_op_count(compound_const_slice, "BINARY_SLICE", 0) + check_op_count(compound_const_slice, "BUILD_SLICE", 0) + check_consts(compound_const_slice, slice, 0) + check_consts(compound_const_slice, tuple, 1) def test_compare_positions(self): for opname_prefix, op in [ @@ -1377,6 +1554,55 @@ def f(): for kw in ("except", "except*"): exec(code % kw, g, l); + def test_regression_gh_120225(self): + async def name_4(): + match b'': + case True: + pass + case name_5 if f'e': + {name_3: name_4 async for name_2 in name_5} + case []: + pass + [[]] + +class TestBooleanExpression(unittest.TestCase): + class Value: + def __init__(self): + self.called = 0 + + def __bool__(self): + self.called += 1 + return self.value + + class Yes(Value): + value = True + + class No(Value): + value = False + + def test_short_circuit_and(self): + v = [self.Yes(), self.No(), self.Yes()] + res = v[0] and v[1] and v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_short_circuit_or(self): + v = [self.No(), self.Yes(), self.No()] + res = v[0] or v[1] or v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_compound(self): + # See gh-124285 + v = [self.No(), self.Yes(), self.Yes(), self.Yes()] + res = v[0] and v[1] or v[2] or v[3] + self.assertIs(res, v[2]) + self.assertEqual([e.called for e in v], [1, 0, 1, 0]) + + v = [self.No(), self.No(), self.Yes(), self.Yes(), self.No()] + res = v[0] or v[1] and v[2] or v[3] or v[4] + self.assertIs(res, v[3]) + self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): @@ -1498,6 +1724,7 @@ def test_multiline_boolean_expression(self): self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', line=4, end_line=4, column=8, end_column=13, occurrence=2) + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") def test_multiline_assert(self): snippet = textwrap.dedent("""\ assert (a > 0 and @@ -1505,11 +1732,11 @@ def test_multiline_assert(self): ccc == 1000000), "error msg" """) compiled_code, _ = self.check_positions_against_ast(snippet) - self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_ASSERTION_ERROR', + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_COMMON_CONSTANT', line=1, end_line=3, column=0, end_column=36, occurrence=1) # The "error msg": self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', - line=3, end_line=3, column=25, end_column=36, occurrence=4) + line=3, end_line=3, column=25, end_column=36, occurrence=2) self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', line=1, end_line=3, column=0, end_column=36, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'RAISE_VARARGS', @@ -1531,8 +1758,8 @@ def test_multiline_generator_expression(self): line=1, end_line=2, column=1, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=8, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', - line=1, end_line=6, column=0, end_column=32, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=4, end_line=4, column=7, end_column=14, occurrence=1) def test_multiline_async_generator_expression(self): snippet = textwrap.dedent("""\ @@ -1548,7 +1775,7 @@ def test_multiline_async_generator_expression(self): self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', line=1, end_line=2, column=1, end_column=8, occurrence=2) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_list_comprehension(self): @@ -1586,7 +1813,7 @@ async def f(): line=2, end_line=3, column=5, end_column=12, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=2, end_line=3, column=5, end_column=12, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_set_comprehension(self): @@ -1624,7 +1851,7 @@ async def f(): line=2, end_line=3, column=5, end_column=12, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=2, end_line=3, column=5, end_column=12, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_dict_comprehension(self): @@ -1662,7 +1889,7 @@ async def f(): line=2, end_line=3, column=5, end_column=11, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=2, end_line=3, column=5, end_column=11, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_matchcase_sequence(self): @@ -1925,11 +2152,135 @@ def test_column_offset_deduplication(self): def test_load_super_attr(self): source = "class C:\n def __init__(self):\n super().__init__()" - code = compile(source, "", "exec").co_consts[0].co_consts[1] + for const in compile(source, "", "exec").co_consts[0].co_consts: + if isinstance(const, types.CodeType): + code = const + break self.assertOpcodeSourcePositionIs( code, "LOAD_GLOBAL", line=3, end_line=3, column=4, end_column=9 ) + def test_lambda_return_position(self): + snippets = [ + "f = lambda: x", + "f = lambda: 42", + "f = lambda: 1 + 2", + "f = lambda: a + b", + ] + for snippet in snippets: + with self.subTest(snippet=snippet): + lamb = run_code(snippet)["f"] + positions = lamb.__code__.co_positions() + # assert that all positions are within the lambda + for i, pos in enumerate(positions): + with self.subTest(i=i, pos=pos): + start_line, end_line, start_col, end_col = pos + if i == 0 and start_col == end_col == 0: + # ignore the RESUME in the beginning + continue + self.assertEqual(start_line, 1) + self.assertEqual(end_line, 1) + code_start = snippet.find(":") + 2 + code_end = len(snippet) + self.assertGreaterEqual(start_col, code_start) + self.assertLessEqual(end_col, code_end) + self.assertGreaterEqual(end_col, start_col) + self.assertLessEqual(end_col, code_end) + + def test_return_in_with_positions(self): + # See gh-98442 + def f(): + with xyz: + 1 + 2 + 3 + 4 + return R + + # All instructions should have locations on a single line + for instr in dis.get_instructions(f): + start_line, end_line, _, _ = instr.positions + self.assertEqual(start_line, end_line) + + # Expect four `LOAD_CONST None` instructions: + # three for the no-exception __exit__ call, and one for the return. + # They should all have the locations of the context manager ('xyz'). + + load_none = [instr for instr in dis.get_instructions(f) if + instr.opname == 'LOAD_CONST' and instr.argval is None] + return_value = [instr for instr in dis.get_instructions(f) if + instr.opname == 'RETURN_VALUE'] + + self.assertEqual(len(load_none), 4) + self.assertEqual(len(return_value), 2) + for instr in load_none + return_value: + start_line, end_line, start_col, end_col = instr.positions + self.assertEqual(start_line, f.__code__.co_firstlineno + 1) + self.assertEqual(end_line, f.__code__.co_firstlineno + 1) + self.assertEqual(start_col, 17) + self.assertEqual(end_col, 20) + + +class TestStaticAttributes(unittest.TestCase): + + def test_basic(self): + class C: + def f(self): + self.a = self.b = 42 + # read fields are not included + self.f() + self.arr[3] + + self.assertIsInstance(C.__static_attributes__, tuple) + self.assertEqual(sorted(C.__static_attributes__), ['a', 'b']) + + def test_nested_function(self): + class C: + def f(self): + self.x = 1 + self.y = 2 + self.x = 3 # check deduplication + + def g(self, obj): + self.y = 4 + self.z = 5 + + def h(self, a): + self.u = 6 + self.v = 7 + + obj.self = 8 + + self.assertEqual(sorted(C.__static_attributes__), ['u', 'v', 'x', 'y', 'z']) + + def test_nested_class(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D: + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(C.D.__static_attributes__), ['y', 'z']) + + def test_subclass(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D(C): + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(D.__static_attributes__), ['y', 'z']) + class TestExpressionStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object @@ -2331,6 +2682,67 @@ def test_return_inside_async_with_block(self): """ self.check_stack_size(snippet, async_=True) +@support.cpython_only +@unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') +class TestInstructionSequence(unittest.TestCase): + def compare_instructions(self, seq, expected): + self.assertEqual([(opcode.opname[i[0]],) + i[1:] for i in seq.get_instructions()], + expected) + + def test_basics(self): + seq = _testinternalcapi.new_instruction_sequence() + + def add_op(seq, opname, oparg, bl, bc=0, el=0, ec=0): + seq.addop(opcode.opmap[opname], oparg, bl, bc, el, el) + + add_op(seq, 'LOAD_CONST', 1, 1) + add_op(seq, 'JUMP', lbl1 := seq.new_label(), 2) + add_op(seq, 'LOAD_CONST', 1, 3) + add_op(seq, 'JUMP', lbl2 := seq.new_label(), 4) + seq.use_label(lbl1) + add_op(seq, 'LOAD_CONST', 2, 4) + seq.use_label(lbl2) + add_op(seq, 'RETURN_VALUE', 0, 3) + + expected = [('LOAD_CONST', 1, 1), + ('JUMP', 4, 2), + ('LOAD_CONST', 1, 3), + ('JUMP', 5, 4), + ('LOAD_CONST', 2, 4), + ('RETURN_VALUE', None, 3), + ] + + self.compare_instructions(seq, [ex + (0,0,0) for ex in expected]) + + def test_nested(self): + seq = _testinternalcapi.new_instruction_sequence() + seq.addop(opcode.opmap['LOAD_CONST'], 1, 1, 0, 0, 0) + nested = _testinternalcapi.new_instruction_sequence() + nested.addop(opcode.opmap['LOAD_CONST'], 2, 2, 0, 0, 0) + + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(nested, [('LOAD_CONST', 2, 2, 0, 0, 0)]) + + seq.add_nested(nested) + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) + + def test_static_attributes_are_sorted(self): + code = ( + 'class T:\n' + ' def __init__(self):\n' + ' self.{V1} = 10\n' + ' self.{V2} = 10\n' + ' def foo(self):\n' + ' self.{V3} = 10\n' + ) + attributes = ("a", "b", "c") + for perm in itertools.permutations(attributes): + var_names = {f'V{i + 1}': name for i, name in enumerate(perm)} + ns = run_code(code.format(**var_names)) + t = ns['T'] + self.assertEqual(t.__static_attributes__, attributes) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 0ec6013dc11e3e..3a34c6822bc079 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -4,7 +4,6 @@ import importlib.util import io import os -import pathlib import py_compile import shutil import struct @@ -19,7 +18,7 @@ # compileall relies on ProcessPoolExecutor if ProcessPoolExecutor exists # and it can function. from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests - from concurrent.futures import ProcessPoolExecutor + from concurrent.futures import ProcessPoolExecutor # noqa: F401 from concurrent.futures.process import _check_system_limits _check_system_limits() _have_multiprocessing = True @@ -31,6 +30,7 @@ from test.support import script_helper from test.test_py_compile import without_source_date_epoch from test.test_py_compile import SourceDateEpochTestMeta +from test.support.os_helper import FakePath def get_pyc(script, opt): @@ -156,28 +156,28 @@ def test_compile_file_pathlike(self): self.assertFalse(os.path.isfile(self.bc_path)) # we should also test the output with support.captured_stdout() as stdout: - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path))) + self.assertTrue(compileall.compile_file(FakePath(self.source_path))) self.assertRegex(stdout.getvalue(), r'Compiling ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_ddir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - ddir=pathlib.Path('ddir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + ddir=FakePath('ddir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_stripdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - stripdir=pathlib.Path('stripdir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + stripdir=FakePath('stripdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_prependdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - prependdir=pathlib.Path('prependdir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + prependdir=FakePath('prependdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) @@ -228,22 +228,22 @@ def test_optimize(self): def test_compile_dir_pathlike(self): self.assertFalse(os.path.isfile(self.bc_path)) with support.captured_stdout() as stdout: - compileall.compile_dir(pathlib.Path(self.directory)) + compileall.compile_dir(FakePath(self.directory)) line = stdout.getvalue().splitlines()[0] self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_dir_pathlike_stripdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), - stripdir=pathlib.Path('stripdir_path'), + self.assertTrue(compileall.compile_dir(FakePath(self.directory), + stripdir=FakePath('stripdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_dir_pathlike_prependdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), - prependdir=pathlib.Path('prependdir_path'), + self.assertTrue(compileall.compile_dir(FakePath(self.directory), + prependdir=FakePath('prependdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) @@ -502,19 +502,25 @@ def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') with open(self.source_path, 'w', encoding='utf-8') as file: - file.write('# -*- coding: utf-8 -*-\n') - file.write('print u"\u20ac"\n') + # Intentional syntax error: bytes can only contain + # ASCII literal characters. + file.write('b"\u20ac"') def tearDown(self): shutil.rmtree(self.directory) def test_error(self): - try: - orig_stdout = sys.stdout - sys.stdout = io.TextIOWrapper(io.BytesIO(),encoding='ascii') - compileall.compile_dir(self.directory) - finally: - sys.stdout = orig_stdout + buffer = io.TextIOWrapper(io.BytesIO(), encoding='ascii') + with contextlib.redirect_stdout(buffer): + compiled = compileall.compile_dir(self.directory) + self.assertFalse(compiled) # should not be successful + buffer.seek(0) + res = buffer.read() + self.assertIn( + 'SyntaxError: bytes can only contain ASCII literal characters', + res, + ) + self.assertNotIn('UnicodeEncodeError', res) class CommandLineTestsBase: @@ -977,7 +983,7 @@ class CommandLineTestsNoSourceEpoch(CommandLineTestsBase, -@unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') +@os_helper.skip_unless_hardlink class HardlinkDedupTestsBase: # Test hardlink_dupes parameter of compileall.compile_dir() diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 5696433e529d0a..c4962e3599986e 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -1,3 +1,6 @@ +import dis +import io +import textwrap import types from test.support.bytecode_helper import AssemblerTestCase @@ -22,11 +25,13 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault('filename', filename) return metadata - def assemble_test(self, insts, metadata, expected): + def insts_to_code_object(self, insts, metadata): metadata = self.complete_metadata(metadata) - insts = self.complete_insts_info(insts) + seq = self.seq_from_insts(insts) + return self.get_code_object(metadata['filename'], seq, metadata) - co = self.get_code_object(metadata['filename'], insts, metadata) + def assemble_test(self, insts, metadata, expected): + co = self.insts_to_code_object(insts, metadata) self.assertIsInstance(co, types.CodeType) expected_metadata = {} @@ -66,7 +71,7 @@ def test_simple_expr(self): ('BINARY_OP', 0, 1), # '+' ('LOAD_CONST', 0, 1), # 2 ('BINARY_OP', 11, 1), # '/' - ('RETURN_VALUE', 1), + ('RETURN_VALUE', None, 1), ] expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} self.assemble_test(insts, metadata, expected) @@ -79,7 +84,7 @@ def inner(): return x return inner() % 2 - inner_code = mod_two.__code__.co_consts[1] + inner_code = mod_two.__code__.co_consts[0] assert isinstance(inner_code, types.CodeType) metadata = { @@ -97,14 +102,48 @@ def inner(): ('LOAD_CLOSURE', 0, 1), ('BUILD_TUPLE', 1, 1), ('LOAD_CONST', 1, 1), - ('MAKE_FUNCTION', 0, 2), + ('MAKE_FUNCTION', None, 2), ('SET_FUNCTION_ATTRIBUTE', 8, 2), - ('PUSH_NULL', 0, 1), + ('PUSH_NULL', None, 1), ('CALL', 0, 2), # (lambda: x)() ('LOAD_CONST', 2, 2), # 2 ('BINARY_OP', 6, 2), # % - ('RETURN_VALUE', 0, 2) + ('RETURN_VALUE', None, 2) ] expected = {(0,): 0, (1,): 1, (2,): 0, (120,): 0, (121,): 1} self.assemble_test(instructions, metadata, expected) + + + def test_exception_table(self): + metadata = { + 'filename' : 'exc.py', + 'name' : 'exc', + 'consts' : {2 : 0}, + } + + # code for "try: pass\n except: pass" + insts = [ + ('RESUME', 0), + ('SETUP_FINALLY', 4), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ('SETUP_CLEANUP', 10), + ('PUSH_EXC_INFO', None), + ('POP_TOP', None), + ('POP_EXCEPT', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ('COPY', 3), + ('POP_EXCEPT', None), + ('RERAISE', 1), + ] + co = self.insts_to_code_object(insts, metadata) + output = io.StringIO() + dis.dis(co, file=output) + exc_table = textwrap.dedent(""" + ExceptionTable: + L1 to L2 -> L2 [0] + L2 to L3 -> L3 [1] lasti + """) + self.assertTrue(output.getvalue().endswith(exc_table)) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index dbeadd9ca47c63..2dd7cf65ee3c2a 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -1,4 +1,5 @@ +import textwrap from test.support.bytecode_helper import CodegenTestCase # Tests for the code-generation stage of the compiler. @@ -6,11 +7,19 @@ class IsolatedCodeGenTests(CodegenTestCase): + def assertInstructionsMatch_recursive(self, insts, expected_insts): + expected_nested = [i for i in expected_insts if isinstance(i, list)] + expected_insts = [i for i in expected_insts if not isinstance(i, list)] + self.assertInstructionsMatch(insts, expected_insts) + self.assertEqual(len(insts.get_nested()), len(expected_nested)) + for n_insts, n_expected in zip(insts.get_nested(), expected_nested): + self.assertInstructionsMatch_recursive(n_insts, n_expected) + def codegen_test(self, snippet, expected_insts): import ast - a = ast.parse(snippet, "my_file.py", "exec"); + a = ast.parse(snippet, "my_file.py", "exec") insts = self.generate_code(a) - self.assertInstructionsMatch(insts, expected_insts) + self.assertInstructionsMatch_recursive(insts, expected_insts) def test_if_expression(self): snippet = "42 if True else 24" @@ -20,13 +29,13 @@ def test_if_expression(self): ('LOAD_CONST', 0, 1), ('TO_BOOL', 0, 1), ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), - ('LOAD_CONST', 1, 1), + ('LOAD_SMALL_INT', 42, 1), ('JUMP_NO_INTERRUPT', exit_lbl := self.Label()), false_lbl, - ('LOAD_CONST', 2, 1), + ('LOAD_SMALL_INT', 24, 1), exit_lbl, ('POP_TOP', None), - ('LOAD_CONST', 3), + ('LOAD_CONST', 1), ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) @@ -40,6 +49,7 @@ def test_for_loop(self): ('GET_ITER', None, 1), loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), + ('NOP', None, 1, 1), ('STORE_NAME', 1, 1), ('LOAD_NAME', 2, 2), ('PUSH_NULL', None, 2), @@ -54,3 +64,96 @@ def test_for_loop(self): ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) + + def test_function(self): + snippet = textwrap.dedent(""" + def f(x): + return x + 42 + """) + expected = [ + # Function definition + ('RESUME', 0), + ('LOAD_CONST', 0), + ('MAKE_FUNCTION', None), + ('STORE_NAME', 0), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + [ + # Function body + ('RESUME', 0), + ('LOAD_FAST', 0), + ('LOAD_SMALL_INT', 42), + ('BINARY_OP', 0), + ('RETURN_VALUE', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ] + ] + self.codegen_test(snippet, expected) + + def test_nested_functions(self): + snippet = textwrap.dedent(""" + def f(): + def h(): + return 12 + def g(): + x = 1 + y = 2 + z = 3 + u = 4 + return 42 + """) + expected = [ + # Function definition + ('RESUME', 0), + ('LOAD_CONST', 0), + ('MAKE_FUNCTION', None), + ('STORE_NAME', 0), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + [ + # Function body + ('RESUME', 0), + ('LOAD_CONST', 1), + ('MAKE_FUNCTION', None), + ('STORE_FAST', 0), + ('LOAD_CONST', 2), + ('MAKE_FUNCTION', None), + ('STORE_FAST', 1), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + [ + ('RESUME', 0), + ('NOP', None), + ('LOAD_SMALL_INT', 12), + ('RETURN_VALUE', None), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + ], + [ + ('RESUME', 0), + ('LOAD_SMALL_INT', 1), + ('STORE_FAST', 0), + ('LOAD_SMALL_INT', 2), + ('STORE_FAST', 1), + ('LOAD_SMALL_INT', 3), + ('STORE_FAST', 2), + ('LOAD_SMALL_INT', 4), + ('STORE_FAST', 3), + ('NOP', None), + ('LOAD_SMALL_INT', 42), + ('RETURN_VALUE', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ], + ], + ] + self.codegen_test(snippet, expected) + + def test_syntax_error__return_not_in_function(self): + snippet = "return 42" + with self.assertRaisesRegex(SyntaxError, "'return' outside function") as cm: + self.codegen_test(snippet, None) + self.assertIsNone(cm.exception.text) + self.assertEqual(cm.exception.offset, 1) + self.assertEqual(cm.exception.end_offset, 10) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index b057121f285dc7..ecc97315e50d31 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -1,15 +1,17 @@ import unittest import sys from test import support +from test.support.testcase import ComplexesAreIdenticalMixin from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) from random import random -from math import atan2, isnan, copysign +from math import isnan, copysign import operator INF = float("inf") NAN = float("nan") +DBL_MAX = sys.float_info.max # These tests ensure that complex math does the right thing ZERO_DIVISION = ( @@ -20,7 +22,38 @@ (1, 0+0j), ) -class ComplexTest(unittest.TestCase): +class WithIndex: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + +class WithFloat: + def __init__(self, value): + self.value = value + def __float__(self): + return self.value + +class ComplexSubclass(complex): + pass + +class OtherComplexSubclass(complex): + pass + +class MyInt: + def __init__(self, value): + self.value = value + + def __int__(self): + return self.value + +class WithComplex: + def __init__(self, value): + self.value = value + def __complex__(self): + return self.value + +class ComplexTest(ComplexesAreIdenticalMixin, unittest.TestCase): def assertAlmostEqual(self, a, b): if isinstance(a, complex): @@ -49,29 +82,6 @@ def assertCloseAbs(self, x, y, eps=1e-9): # check that relative difference < eps self.assertTrue(abs((x-y)/y) < eps) - def assertFloatsAreIdentical(self, x, y): - """assert that floats x and y are identical, in the sense that: - (1) both x and y are nans, or - (2) both x and y are infinities, with the same sign, or - (3) both x and y are zeros, with the same sign, or - (4) x and y are both finite and nonzero, and x == y - - """ - msg = 'floats {!r} and {!r} are not identical' - - if isnan(x) or isnan(y): - if isnan(x) and isnan(y): - return - elif x == y: - if x != 0.0: - return - # both zero; check that signs match - elif copysign(1.0, x) == copysign(1.0, y): - return - else: - msg += ': zeros have different signs' - self.fail(msg.format(x, y)) - def assertClose(self, x, y, eps=1e-9): """Return true iff complexes x and y "are close".""" self.assertCloseAbs(x.real, y.real, eps) @@ -117,6 +127,33 @@ def test_truediv(self): self.assertTrue(isnan(z.real)) self.assertTrue(isnan(z.imag)) + self.assertComplexesAreIdentical(complex(INF, 1)/(0.0+1j), + complex(NAN, -INF)) + + # test recover of infs if numerator has infs and denominator is finite + self.assertComplexesAreIdentical(complex(INF, -INF)/(1+0j), + complex(INF, -INF)) + self.assertComplexesAreIdentical(complex(INF, INF)/(0.0+1j), + complex(INF, -INF)) + self.assertComplexesAreIdentical(complex(NAN, INF)/complex(2**1000, 2**-1000), + complex(INF, INF)) + self.assertComplexesAreIdentical(complex(INF, NAN)/complex(2**1000, 2**-1000), + complex(INF, -INF)) + + # test recover of zeros if denominator is infinite + self.assertComplexesAreIdentical((1+1j)/complex(INF, INF), (0.0+0j)) + self.assertComplexesAreIdentical((1+1j)/complex(INF, -INF), (0.0+0j)) + self.assertComplexesAreIdentical((1+1j)/complex(-INF, INF), + complex(0.0, -0.0)) + self.assertComplexesAreIdentical((1+1j)/complex(-INF, -INF), + complex(-0.0, 0)) + self.assertComplexesAreIdentical((INF+1j)/complex(INF, INF), + complex(NAN, NAN)) + self.assertComplexesAreIdentical(complex(1, INF)/complex(INF, INF), + complex(NAN, NAN)) + self.assertComplexesAreIdentical(complex(INF, 1)/complex(1, INF), + complex(NAN, NAN)) + def test_truediv_zero_division(self): for a, b in ZERO_DIVISION: with self.assertRaises(ZeroDivisionError): @@ -339,137 +376,118 @@ def test_conjugate(self): self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) def test_constructor(self): - class NS: - def __init__(self, value): self.value = value - def __complex__(self): return self.value - self.assertEqual(complex(NS(1+10j)), 1+10j) - self.assertRaises(TypeError, complex, NS(None)) - self.assertRaises(TypeError, complex, {}) - self.assertRaises(TypeError, complex, NS(1.5)) - self.assertRaises(TypeError, complex, NS(1)) - self.assertRaises(TypeError, complex, object()) - self.assertRaises(TypeError, complex, NS(4.25+0.5j), object()) - - self.assertAlmostEqual(complex("1+10j"), 1+10j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10.0), 10+0j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10+0j), 10+0j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10.0), 1+10j) - self.assertAlmostEqual(complex(3.14+0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14), 3.14+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(3.14+0j, 0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14, 0.0), 3.14+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(0j, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0.0, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0j, 3.14), 3.14j) - self.assertAlmostEqual(complex(0.0, 3.14), 3.14j) - self.assertAlmostEqual(complex("1"), 1+0j) - self.assertAlmostEqual(complex("1j"), 1j) - self.assertAlmostEqual(complex(), 0) - self.assertAlmostEqual(complex("-1"), -1) - self.assertAlmostEqual(complex("+1"), +1) - self.assertAlmostEqual(complex("(1+2j)"), 1+2j) - self.assertAlmostEqual(complex("(1.3+2.2j)"), 1.3+2.2j) - self.assertAlmostEqual(complex("3.14+1J"), 3.14+1j) - self.assertAlmostEqual(complex(" ( +3.14-6J )"), 3.14-6j) - self.assertAlmostEqual(complex(" ( +3.14-J )"), 3.14-1j) - self.assertAlmostEqual(complex(" ( +3.14+j )"), 3.14+1j) - self.assertAlmostEqual(complex("J"), 1j) - self.assertAlmostEqual(complex("( j )"), 1j) - self.assertAlmostEqual(complex("+J"), 1j) - self.assertAlmostEqual(complex("( -j)"), -1j) - self.assertAlmostEqual(complex('1e-500'), 0.0 + 0.0j) - self.assertAlmostEqual(complex('-1e-500j'), 0.0 - 0.0j) - self.assertAlmostEqual(complex('-1e-500+1e-500j'), -0.0 + 0.0j) - self.assertEqual(complex('1-1j'), 1.0 - 1j) - self.assertEqual(complex('1J'), 1j) - - class complex2(complex): pass - self.assertAlmostEqual(complex(complex2(1+1j)), 1+1j) - self.assertAlmostEqual(complex(real=17, imag=23), 17+23j) - self.assertAlmostEqual(complex(real=17+23j), 17+23j) - self.assertAlmostEqual(complex(real=17+23j, imag=23), 17+46j) - self.assertAlmostEqual(complex(real=1+2j, imag=3+4j), -3+5j) + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex(), 0.0, 0.0) + check(complex(10), 10.0, 0.0) + check(complex(4.25), 4.25, 0.0) + check(complex(4.25+0j), 4.25, 0.0) + check(complex(4.25+0.5j), 4.25, 0.5) + check(complex(ComplexSubclass(4.25+0.5j)), 4.25, 0.5) + check(complex(WithComplex(4.25+0.5j)), 4.25, 0.5) + + check(complex(1, 10), 1.0, 10.0) + check(complex(1, 10.0), 1.0, 10.0) + check(complex(1, 4.25), 1.0, 4.25) + check(complex(1.0, 10), 1.0, 10.0) + check(complex(4.25, 10), 4.25, 10.0) + check(complex(1.0, 10.0), 1.0, 10.0) + check(complex(4.25, 0.5), 4.25, 0.5) + + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25+0j, 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not .*ComplexSubclass"): + check(complex(ComplexSubclass(4.25+0j), 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not .*WithComplex"): + check(complex(WithComplex(4.25+0j), 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25j, 0), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not complex"): + check(complex(0, 4.25+0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not .*ComplexSubclass"): + check(complex(0, ComplexSubclass(4.25+0j)), 0.0, 4.25) + with self.assertRaisesRegex(TypeError, + "argument 'imag' must be a real number, not .*WithComplex"): + complex(0, WithComplex(4.25+0j)) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not complex"): + check(complex(0.0, 4.25j), -4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25+0j, 0j), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25j, 0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25+0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25j), -4.25, 0.0) + + check(complex(real=4.25), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(real=4.25+0j), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(real=4.25+1.5j), 4.25, 1.5) + check(complex(imag=1.5), 0.0, 1.5) + check(complex(real=4.25, imag=1.5), 4.25, 1.5) + check(complex(4.25, imag=1.5), 4.25, 1.5) # check that the sign of a zero in the real or imaginary part - # is preserved when constructing from two floats. (These checks - # are harmless on systems without support for signed zeros.) - def split_zeros(x): - """Function that produces different results for 0. and -0.""" - return atan2(x, -1.) - - self.assertEqual(split_zeros(complex(1., 0.).imag), split_zeros(0.)) - self.assertEqual(split_zeros(complex(1., -0.).imag), split_zeros(-0.)) - self.assertEqual(split_zeros(complex(0., 1.).real), split_zeros(0.)) - self.assertEqual(split_zeros(complex(-0., 1.).real), split_zeros(-0.)) - - c = 3.14 + 1j - self.assertTrue(complex(c) is c) - del c - - self.assertRaises(TypeError, complex, "1", "1") - self.assertRaises(TypeError, complex, 1, "1") - - # SF bug 543840: complex(string) accepts strings with \0 - # Fixed in 2.3. - self.assertRaises(ValueError, complex, '1+1j\0j') - - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, float, 5+3j) - self.assertRaises(ValueError, complex, "") - self.assertRaises(TypeError, complex, None) - self.assertRaisesRegex(TypeError, "not 'NoneType'", complex, None) - self.assertRaises(ValueError, complex, "\0") - self.assertRaises(ValueError, complex, "3\09") - self.assertRaises(TypeError, complex, "1", "2") - self.assertRaises(TypeError, complex, "1", 42) - self.assertRaises(TypeError, complex, 1, "2") - self.assertRaises(ValueError, complex, "1+") - self.assertRaises(ValueError, complex, "1+1j+1j") - self.assertRaises(ValueError, complex, "--") - self.assertRaises(ValueError, complex, "(1+2j") - self.assertRaises(ValueError, complex, "1+2j)") - self.assertRaises(ValueError, complex, "1+(2j)") - self.assertRaises(ValueError, complex, "(1+2j)123") - self.assertRaises(ValueError, complex, "x") - self.assertRaises(ValueError, complex, "1j+2") - self.assertRaises(ValueError, complex, "1e1ej") - self.assertRaises(ValueError, complex, "1e++1ej") - self.assertRaises(ValueError, complex, ")1+2j(") - self.assertRaisesRegex( - TypeError, - "first argument must be a string or a number, not 'dict'", - complex, {1:2}, 1) - self.assertRaisesRegex( - TypeError, - "second argument must be a number, not 'dict'", - complex, 1, {1:2}) - # the following three are accepted by Python 2.6 - self.assertRaises(ValueError, complex, "1..1j") - self.assertRaises(ValueError, complex, "1.11.1j") - self.assertRaises(ValueError, complex, "1e1.1j") - - # check that complex accepts long unicode strings - self.assertEqual(type(complex("1"*500)), complex) - # check whitespace processing - self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) - # Invalid unicode string - # See bpo-34087 - self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + # is preserved when constructing from two floats. + for x in 1.0, -1.0: + for y in 0.0, -0.0: + check(complex(x, y), x, y) + check(complex(y, x), y, x) + + c = complex(4.25, 1.5) + self.assertIs(complex(c), c) + c2 = ComplexSubclass(c) + self.assertEqual(c2, c) + self.assertIs(type(c2), ComplexSubclass) + del c, c2 + + self.assertRaisesRegex(TypeError, + "argument must be a string or a number, not dict", + complex, {}) + self.assertRaisesRegex(TypeError, + "argument must be a string or a number, not NoneType", + complex, None) + self.assertRaisesRegex(TypeError, + "argument 'real' must be a real number, not dict", + complex, {1:2}, 0) + self.assertRaisesRegex(TypeError, + "argument 'real' must be a real number, not str", + complex, '1', 0) + self.assertRaisesRegex(TypeError, + "argument 'imag' must be a real number, not dict", + complex, 0, {1:2}) + self.assertRaisesRegex(TypeError, + "argument 'imag' must be a real number, not str", + complex, 0, '1') + + self.assertRaises(TypeError, complex, WithComplex(1.5)) + self.assertRaises(TypeError, complex, WithComplex(1)) + self.assertRaises(TypeError, complex, WithComplex(None)) + self.assertRaises(TypeError, complex, WithComplex(4.25+0j), object()) + self.assertRaises(TypeError, complex, WithComplex(1.5), object()) + self.assertRaises(TypeError, complex, WithComplex(1), object()) + self.assertRaises(TypeError, complex, WithComplex(None), object()) class EvilExc(Exception): pass @@ -480,33 +498,33 @@ def __complex__(self): self.assertRaises(EvilExc, complex, evilcomplex()) - class float2: - def __init__(self, value): - self.value = value - def __float__(self): - return self.value - - self.assertAlmostEqual(complex(float2(42.)), 42) - self.assertAlmostEqual(complex(real=float2(17.), imag=float2(23.)), 17+23j) - self.assertRaises(TypeError, complex, float2(None)) - - class MyIndex: - def __init__(self, value): - self.value = value - def __index__(self): - return self.value - - self.assertAlmostEqual(complex(MyIndex(42)), 42.0+0.0j) - self.assertAlmostEqual(complex(123, MyIndex(42)), 123.0+42.0j) - self.assertRaises(OverflowError, complex, MyIndex(2**2000)) - self.assertRaises(OverflowError, complex, 123, MyIndex(2**2000)) + check(complex(WithFloat(4.25)), 4.25, 0.0) + check(complex(WithFloat(4.25), 1.5), 4.25, 1.5) + check(complex(1.5, WithFloat(4.25)), 1.5, 4.25) + self.assertRaises(TypeError, complex, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(42), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(None)) + self.assertRaises(TypeError, complex, WithFloat(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(None)) + + check(complex(WithIndex(42)), 42.0, 0.0) + check(complex(WithIndex(42), 1.5), 42.0, 1.5) + check(complex(1.5, WithIndex(42)), 1.5, 42.0) + self.assertRaises(OverflowError, complex, WithIndex(2**2000)) + self.assertRaises(OverflowError, complex, WithIndex(2**2000), 1.5) + self.assertRaises(OverflowError, complex, 1.5, WithIndex(2**2000)) + self.assertRaises(TypeError, complex, WithIndex(None)) + self.assertRaises(TypeError, complex, WithIndex(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithIndex(None)) class MyInt: def __int__(self): return 42 self.assertRaises(TypeError, complex, MyInt()) - self.assertRaises(TypeError, complex, 123, MyInt()) + self.assertRaises(TypeError, complex, MyInt(), 1.5) + self.assertRaises(TypeError, complex, 1.5, MyInt()) class complex0(complex): """Test usage of __complex__() when inheriting from 'complex'""" @@ -526,9 +544,9 @@ class complex2(complex): def __complex__(self): return None - self.assertEqual(complex(complex0(1j)), 42j) + check(complex(complex0(1j)), 0.0, 42.0) with self.assertWarns(DeprecationWarning): - self.assertEqual(complex(complex1(1j)), 2j) + check(complex(complex1(1j)), 0.0, 2.0) self.assertRaises(TypeError, complex, complex2(1j)) def test___complex__(self): @@ -536,36 +554,93 @@ def test___complex__(self): self.assertEqual(z.__complex__(), z) self.assertEqual(type(z.__complex__()), complex) - class complex_subclass(complex): - pass - - z = complex_subclass(3 + 4j) + z = ComplexSubclass(3 + 4j) self.assertEqual(z.__complex__(), 3 + 4j) self.assertEqual(type(z.__complex__()), complex) @support.requires_IEEE_754 def test_constructor_special_numbers(self): - class complex2(complex): - pass for x in 0.0, -0.0, INF, -INF, NAN: for y in 0.0, -0.0, INF, -INF, NAN: with self.subTest(x=x, y=y): z = complex(x, y) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex2(x, y) - self.assertIs(type(z), complex2) + z = ComplexSubclass(x, y) + self.assertIs(type(z), ComplexSubclass) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex(complex2(x, y)) + z = complex(ComplexSubclass(x, y)) self.assertIs(type(z), complex) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex2(complex(x, y)) - self.assertIs(type(z), complex2) + z = ComplexSubclass(complex(x, y)) + self.assertIs(type(z), ComplexSubclass) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) + def test_constructor_from_string(self): + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex("1"), 1.0, 0.0) + check(complex("1j"), 0.0, 1.0) + check(complex("-1"), -1.0, 0.0) + check(complex("+1"), 1.0, 0.0) + check(complex("1+2j"), 1.0, 2.0) + check(complex("(1+2j)"), 1.0, 2.0) + check(complex("(1.5+4.25j)"), 1.5, 4.25) + check(complex("4.25+1J"), 4.25, 1.0) + check(complex(" ( +4.25-6J )"), 4.25, -6.0) + check(complex(" ( +4.25-J )"), 4.25, -1.0) + check(complex(" ( +4.25+j )"), 4.25, 1.0) + check(complex("J"), 0.0, 1.0) + check(complex("( j )"), 0.0, 1.0) + check(complex("+J"), 0.0, 1.0) + check(complex("( -j)"), 0.0, -1.0) + check(complex('1-1j'), 1.0, -1.0) + check(complex('1J'), 0.0, 1.0) + + check(complex('1e-500'), 0.0, 0.0) + check(complex('-1e-500j'), 0.0, -0.0) + check(complex('1e-500+1e-500j'), 0.0, 0.0) + check(complex('-1e-500+1e-500j'), -0.0, 0.0) + check(complex('1e-500-1e-500j'), 0.0, -0.0) + check(complex('-1e-500-1e-500j'), -0.0, -0.0) + + # SF bug 543840: complex(string) accepts strings with \0 + # Fixed in 2.3. + self.assertRaises(ValueError, complex, '1+1j\0j') + self.assertRaises(ValueError, complex, "") + self.assertRaises(ValueError, complex, "\0") + self.assertRaises(ValueError, complex, "3\09") + self.assertRaises(ValueError, complex, "1+") + self.assertRaises(ValueError, complex, "1+1j+1j") + self.assertRaises(ValueError, complex, "--") + self.assertRaises(ValueError, complex, "(1+2j") + self.assertRaises(ValueError, complex, "1+2j)") + self.assertRaises(ValueError, complex, "1+(2j)") + self.assertRaises(ValueError, complex, "(1+2j)123") + self.assertRaises(ValueError, complex, "x") + self.assertRaises(ValueError, complex, "1j+2") + self.assertRaises(ValueError, complex, "1e1ej") + self.assertRaises(ValueError, complex, "1e++1ej") + self.assertRaises(ValueError, complex, ")1+2j(") + # the following three are accepted by Python 2.6 + self.assertRaises(ValueError, complex, "1..1j") + self.assertRaises(ValueError, complex, "1.11.1j") + self.assertRaises(ValueError, complex, "1e1.1j") + + # check that complex accepts long unicode strings + self.assertIs(type(complex("1"*500)), complex) + # check whitespace processing + self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) + # Invalid unicode string + # See bpo-34087 + self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + def test_constructor_negative_nans_from_string(self): self.assertEqual(copysign(1., complex("-nan").real), -1.) self.assertEqual(copysign(1., complex("-nanj").imag), -1.) @@ -584,10 +659,39 @@ def test_underscores(self): if not any(ch in lit for ch in 'xXoObB'): self.assertRaises(ValueError, complex, lit) + def test_from_number(self, cls=complex): + def eq(actual, expected): + self.assertEqual(actual, expected) + self.assertIs(type(actual), cls) + + eq(cls.from_number(3.14), 3.14+0j) + eq(cls.from_number(3.14j), 3.14j) + eq(cls.from_number(314), 314.0+0j) + eq(cls.from_number(OtherComplexSubclass(3.14, 2.72)), 3.14+2.72j) + eq(cls.from_number(WithComplex(3.14+2.72j)), 3.14+2.72j) + eq(cls.from_number(WithFloat(3.14)), 3.14+0j) + eq(cls.from_number(WithIndex(314)), 314.0+0j) + + cNAN = complex(NAN, NAN) + x = cls.from_number(cNAN) + self.assertTrue(x != x) + self.assertIs(type(x), cls) + if cls is complex: + self.assertIs(cls.from_number(cNAN), cNAN) + + self.assertRaises(TypeError, cls.from_number, '3.14') + self.assertRaises(TypeError, cls.from_number, b'3.14') + self.assertRaises(TypeError, cls.from_number, MyInt(314)) + self.assertRaises(TypeError, cls.from_number, {}) + self.assertRaises(TypeError, cls.from_number) + + def test_from_number_subclass(self): + self.test_from_number(ComplexSubclass) + def test_hash(self): for x in range(-30, 30): self.assertEqual(hash(x), hash(complex(x, 0))) - x /= 3.0 # now check against floating point + x /= 3.0 # now check against floating-point self.assertEqual(hash(x), hash(complex(x, 0.))) self.assertNotEqual(hash(2000005 - 1j), -1) @@ -597,6 +701,8 @@ def test_abs(self): for num in nums: self.assertAlmostEqual((num.real**2 + num.imag**2) ** 0.5, abs(num)) + self.assertRaises(OverflowError, abs, complex(DBL_MAX, DBL_MAX)) + def test_repr_str(self): def test(v, expected, test_fn=self.assertEqual): test_fn(repr(v), expected) @@ -642,9 +748,6 @@ def test(v, expected, test_fn=self.assertEqual): test(complex(-0., -0.), "(-0-0j)") def test_pos(self): - class ComplexSubclass(complex): - pass - self.assertEqual(+(1+6j), 1+6j) self.assertEqual(+ComplexSubclass(1, 6), 1+6j) self.assertIs(type(+ComplexSubclass(1, 6)), complex) @@ -664,8 +767,8 @@ def test_getnewargs(self): def test_plus_minus_0j(self): # test that -0j and 0j literals are not identified z1, z2 = 0j, -0j - self.assertEqual(atan2(z1.imag, -1.), atan2(0., -1.)) - self.assertEqual(atan2(z2.imag, -1.), atan2(-0., -1.)) + self.assertFloatsAreIdentical(z1.imag, 0.0) + self.assertFloatsAreIdentical(z2.imag, -0.0) @support.requires_IEEE_754 def test_negated_imaginary_literal(self): @@ -700,8 +803,7 @@ def test_repr_roundtrip(self): for y in vals: z = complex(x, y) roundtrip = complex(repr(z)) - self.assertFloatsAreIdentical(z.real, roundtrip.real) - self.assertFloatsAreIdentical(z.imag, roundtrip.imag) + self.assertComplexesAreIdentical(z, roundtrip) # if we predefine some constants, then eval(repr(z)) should # also work, except that it might change the sign of zeros diff --git a/Lib/test/test_concurrent_futures/__init__.py b/Lib/test/test_concurrent_futures/__init__.py index 430fa93aa456a2..b38bd38d338f0e 100644 --- a/Lib/test/test_concurrent_futures/__init__.py +++ b/Lib/test/test_concurrent_futures/__init__.py @@ -1,10 +1,12 @@ import os.path import unittest from test import support -from test.support import import_helper +from test.support import threading_helper + + +# Adjust if we ever have a platform with processes but not threads. +threading_helper.requires_working_threading(module=True) -# Skip tests if _multiprocessing wasn't built. -import_helper.import_module('_multiprocessing') if support.check_sanitizer(address=True, memory=True): # gh-90791: Skip the test because it is too slow when Python is built diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 6a79fe69ec37cf..b97d9ffd94b1f8 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -1,6 +1,5 @@ import threading import time -import unittest import weakref from concurrent import futures from test import support @@ -24,6 +23,7 @@ def make_dummy_object(_): class ExecutorTest: + # Executor.shutdown() and context manager usage is tested by # ExecutorShutdownTest. def test_submit(self): @@ -53,7 +53,8 @@ def test_map_exception(self): i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5]) self.assertEqual(i.__next__(), (0, 1)) self.assertEqual(i.__next__(), (0, 1)) - self.assertRaises(ZeroDivisionError, i.__next__) + with self.assertRaises(ZeroDivisionError): + i.__next__() @support.requires_resource('walltime') def test_map_timeout(self): @@ -83,24 +84,34 @@ def test_no_stale_references(self): # references. my_object = MyObject() my_object_collected = threading.Event() - my_object_callback = weakref.ref( - my_object, lambda obj: my_object_collected.set()) - fut = self.executor.submit(my_object.my_method) + def set_event(): + if Py_GIL_DISABLED: + # gh-117688 Avoid deadlock by setting the event in a + # background thread. The current thread may be in the middle + # of the my_object_collected.wait() call, which holds locks + # needed by my_object_collected.set(). + threading.Thread(target=my_object_collected.set).start() + else: + my_object_collected.set() + my_object_callback = weakref.ref(my_object, lambda obj: set_event()) + # Deliberately discarding the future. + self.executor.submit(my_object.my_method) del my_object if Py_GIL_DISABLED: # Due to biased reference counting, my_object might only be # deallocated while the thread that created it runs -- if the # thread is paused waiting on an event, it may not merge the - # refcount of the queued object. For that reason, we wait for the - # task to finish (so that it's no longer referenced) and force a - # GC to ensure that it is collected. - fut.result() # Wait for the task to finish. - support.gc_collect() + # refcount of the queued object. For that reason, we alternate + # between running the GC and waiting for the event. + wait_time = 0 + collected = False + while not collected and wait_time <= support.SHORT_TIMEOUT: + support.gc_collect() + collected = my_object_collected.wait(timeout=1.0) + wait_time += 1.0 else: - del fut # Deliberately discard the future. - - collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) + collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) self.assertTrue(collected, "Stale reference not collected within timeout.") diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py index 3c30c4558c0b3e..ee589753a3e456 100644 --- a/Lib/test/test_concurrent_futures/test_deadlock.py +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -145,7 +145,7 @@ def test_exit_at_task_unpickle(self): self._check_crash(BrokenProcessPool, id, ExitAtUnpickle()) def test_error_at_task_unpickle(self): - # gh-109832: Restore stderr overriden by _raise_error_ignore_stderr() + # gh-109832: Restore stderr overridden by _raise_error_ignore_stderr() self.addCleanup(setattr, sys, 'stderr', sys.stderr) # Check problem occurring while unpickling a task on workers @@ -183,7 +183,7 @@ def test_error_during_result_pickle_on_worker(self): self._check_crash(PicklingError, _return_instance, ErrorAtPickle) def test_error_during_result_unpickle_in_result_handler(self): - # gh-109832: Restore stderr overriden by _raise_error_ignore_stderr() + # gh-109832: Restore stderr overridden by _raise_error_ignore_stderr() self.addCleanup(setattr, sys, 'stderr', sys.stderr) # Check problem occurring while unpickling a task in @@ -236,7 +236,7 @@ def test_shutdown_deadlock_pickle(self): executor_manager.join() def test_crash_big_data(self): - # Test that there is a clean exception instad of a deadlock when a + # Test that there is a clean exception instead of a deadlock when a # child process crashes while some data is being written into the # queue. # https://github.com/python/cpython/issues/94777 diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index ce01e0ff0f287a..df640929309318 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -3,7 +3,11 @@ import queue import time import unittest +import sys +import io from concurrent.futures._base import BrokenExecutor +from concurrent.futures.process import _check_system_limits + from logging.handlers import QueueHandler from test import support @@ -109,6 +113,37 @@ def _assert_logged(self, msg): create_executor_tests(globals(), FailingInitializerMixin) +@unittest.skipIf(sys.platform == "win32", "Resource Tracker doesn't run on Windows") +class FailingInitializerResourcesTest(unittest.TestCase): + """ + Source: https://github.com/python/cpython/issues/104090 + """ + + def _test(self, test_class): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + + runner = unittest.TextTestRunner(stream=io.StringIO()) + runner.run(test_class('test_initializer')) + + # GH-104090: + # Stop resource tracker manually now, so we can verify there are not leaked resources by checking + # the process exit code + from multiprocessing.resource_tracker import _resource_tracker + _resource_tracker._stop() + + self.assertEqual(_resource_tracker._exitcode, 0) + + def test_spawn(self): + self._test(ProcessPoolSpawnFailingInitializerTest) + + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + def test_forkserver(self): + self._test(ProcessPoolForkserverFailingInitializerTest) + + def setUpModule(): setup_module() diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py new file mode 100644 index 00000000000000..ea1512fc830d0c --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -0,0 +1,377 @@ +import asyncio +import contextlib +import io +import os +import pickle +import sys +import time +import unittest +from concurrent.futures.interpreter import ( + ExecutionFailed, BrokenInterpreterPool, +) +import _interpreters +from test import support +import test.test_asyncio.utils as testasyncio_utils +from test.support.interpreters import queues + +from .executor import ExecutorTest, mul +from .util import BaseTestCase, InterpreterPoolMixin, setup_module + + +def noop(): + pass + + +def write_msg(fd, msg): + os.write(fd, msg + b'\0') + + +def read_msg(fd): + msg = b'' + while ch := os.read(fd, 1): + if ch == b'\0': + return msg + msg += ch + + +def get_current_name(): + return __name__ + + +def fail(exctype, msg=None): + raise exctype(msg) + + +def get_current_interpid(*extra): + interpid, _ = _interpreters.get_current() + return (interpid, *extra) + + +class InterpretersMixin(InterpreterPoolMixin): + + def pipe(self): + r, w = os.pipe() + self.addCleanup(lambda: os.close(r)) + self.addCleanup(lambda: os.close(w)) + return r, w + + +class PickleShenanigans: + """Succeeds with pickle.dumps(), but fails with pickle.loads()""" + def __init__(self, value): + if value == 1: + raise RuntimeError("gotcha") + + def __reduce__(self): + return (self.__class__, (1,)) + + +class InterpreterPoolExecutorTest( + InterpretersMixin, ExecutorTest, BaseTestCase): + + @unittest.expectedFailure + def test_init_script(self): + msg1 = b'step: init' + msg2 = b'step: run' + r, w = self.pipe() + initscript = f""" + import os + msg = {msg2!r} + os.write({w}, {msg1!r} + b'\\0') + """ + script = f""" + os.write({w}, msg + b'\\0') + """ + os.write(w, b'\0') + + executor = self.executor_type(initializer=initscript) + before_init = os.read(r, 100) + fut = executor.submit(script) + after_init = read_msg(r) + fut.result() + after_run = read_msg(r) + + self.assertEqual(before_init, b'\0') + self.assertEqual(after_init, msg1) + self.assertEqual(after_run, msg2) + + @unittest.expectedFailure + def test_init_script_args(self): + with self.assertRaises(ValueError): + self.executor_type(initializer='pass', initargs=('spam',)) + + def test_init_func(self): + msg = b'step: init' + r, w = self.pipe() + os.write(w, b'\0') + + executor = self.executor_type( + initializer=write_msg, initargs=(w, msg)) + before = os.read(r, 100) + executor.submit(mul, 10, 10) + after = read_msg(r) + + self.assertEqual(before, b'\0') + self.assertEqual(after, msg) + + def test_init_closure(self): + count = 0 + def init1(): + assert count == 0, count + def init2(): + nonlocal count + count += 1 + + with self.assertRaises(pickle.PicklingError): + self.executor_type(initializer=init1) + with self.assertRaises(pickle.PicklingError): + self.executor_type(initializer=init2) + + def test_init_instance_method(self): + class Spam: + def initializer(self): + raise NotImplementedError + spam = Spam() + + with self.assertRaises(pickle.PicklingError): + self.executor_type(initializer=spam.initializer) + + def test_init_shared(self): + msg = b'eggs' + r, w = self.pipe() + script = f"""if True: + import os + if __name__ != '__main__': + import __main__ + spam = __main__.spam + os.write({w}, spam + b'\\0') + """ + + executor = self.executor_type(shared={'spam': msg}) + fut = executor.submit(exec, script) + fut.result() + after = read_msg(r) + + self.assertEqual(after, msg) + + @unittest.expectedFailure + def test_init_exception_in_script(self): + executor = self.executor_type(initializer='raise Exception("spam")') + with executor: + with contextlib.redirect_stderr(io.StringIO()) as stderr: + fut = executor.submit('pass') + with self.assertRaises(BrokenInterpreterPool): + fut.result() + stderr = stderr.getvalue() + self.assertIn('ExecutionFailed: Exception: spam', stderr) + self.assertIn('Uncaught in the interpreter:', stderr) + self.assertIn('The above exception was the direct cause of the following exception:', + stderr) + + def test_init_exception_in_func(self): + executor = self.executor_type(initializer=fail, + initargs=(Exception, 'spam')) + with executor: + with contextlib.redirect_stderr(io.StringIO()) as stderr: + fut = executor.submit(noop) + with self.assertRaises(BrokenInterpreterPool): + fut.result() + stderr = stderr.getvalue() + self.assertIn('ExecutionFailed: Exception: spam', stderr) + self.assertIn('Uncaught in the interpreter:', stderr) + self.assertIn('The above exception was the direct cause of the following exception:', + stderr) + + @unittest.expectedFailure + def test_submit_script(self): + msg = b'spam' + r, w = self.pipe() + script = f""" + import os + os.write({w}, __name__.encode('utf-8') + b'\\0') + """ + executor = self.executor_type() + + fut = executor.submit(script) + res = fut.result() + after = read_msg(r) + + self.assertEqual(after, b'__main__') + self.assertIs(res, None) + + def test_submit_closure(self): + spam = True + def task1(): + return spam + def task2(): + nonlocal spam + spam += 1 + return spam + + executor = self.executor_type() + with self.assertRaises(pickle.PicklingError): + executor.submit(task1) + with self.assertRaises(pickle.PicklingError): + executor.submit(task2) + + def test_submit_local_instance(self): + class Spam: + def __init__(self): + self.value = True + + executor = self.executor_type() + with self.assertRaises(pickle.PicklingError): + executor.submit(Spam) + + def test_submit_instance_method(self): + class Spam: + def run(self): + return True + spam = Spam() + + executor = self.executor_type() + with self.assertRaises(pickle.PicklingError): + executor.submit(spam.run) + + def test_submit_func_globals(self): + executor = self.executor_type() + fut = executor.submit(get_current_name) + name = fut.result() + + self.assertEqual(name, __name__) + self.assertNotEqual(name, '__main__') + + @unittest.expectedFailure + def test_submit_exception_in_script(self): + fut = self.executor.submit('raise Exception("spam")') + with self.assertRaises(Exception) as captured: + fut.result() + self.assertIs(type(captured.exception), Exception) + self.assertEqual(str(captured.exception), 'spam') + cause = captured.exception.__cause__ + self.assertIs(type(cause), ExecutionFailed) + for attr in ('__name__', '__qualname__', '__module__'): + self.assertEqual(getattr(cause.excinfo.type, attr), + getattr(Exception, attr)) + self.assertEqual(cause.excinfo.msg, 'spam') + + def test_submit_exception_in_func(self): + fut = self.executor.submit(fail, Exception, 'spam') + with self.assertRaises(Exception) as captured: + fut.result() + self.assertIs(type(captured.exception), Exception) + self.assertEqual(str(captured.exception), 'spam') + cause = captured.exception.__cause__ + self.assertIs(type(cause), ExecutionFailed) + for attr in ('__name__', '__qualname__', '__module__'): + self.assertEqual(getattr(cause.excinfo.type, attr), + getattr(Exception, attr)) + self.assertEqual(cause.excinfo.msg, 'spam') + + def test_saturation(self): + blocker = queues.create() + executor = self.executor_type(4, shared=dict(blocker=blocker)) + + for i in range(15 * executor._max_workers): + executor.submit(exec, 'import __main__; __main__.blocker.get()') + #executor.submit('blocker.get()') + self.assertEqual(len(executor._threads), executor._max_workers) + for i in range(15 * executor._max_workers): + blocker.put_nowait(None) + executor.shutdown(wait=True) + + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") + def test_idle_thread_reuse(self): + executor = self.executor_type() + executor.submit(mul, 21, 2).result() + executor.submit(mul, 6, 7).result() + executor.submit(mul, 3, 14).result() + self.assertEqual(len(executor._threads), 1) + executor.shutdown(wait=True) + + def test_pickle_errors_propagate(self): + # GH-125864: Pickle errors happen before the script tries to execute, so the + # queue used to wait infinitely. + + fut = self.executor.submit(PickleShenanigans(0)) + with self.assertRaisesRegex(RuntimeError, "gotcha"): + fut.result() + + +class AsyncioTest(InterpretersMixin, testasyncio_utils.TestCase): + + @classmethod + def setUpClass(cls): + # Most uses of asyncio will implicitly call set_event_loop_policy() + # with the default policy if a policy hasn't been set already. + # If that happens in a test, like here, we'll end up with a failure + # when --fail-env-changed is used. That's why the other tests that + # use asyncio are careful to set the policy back to None and why + # we're careful to do so here. We also validate that no other + # tests left a policy in place, just in case. + policy = support.maybe_get_event_loop_policy() + assert policy is None, policy + cls.addClassCleanup(lambda: asyncio.set_event_loop_policy(None)) + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + self.executor = self.executor_type() + self.addCleanup(lambda: self.executor.shutdown()) + + def tearDown(self): + if not self.loop.is_closed(): + testasyncio_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + def test_run_in_executor(self): + unexpected, _ = _interpreters.get_current() + + func = get_current_interpid + fut = self.loop.run_in_executor(self.executor, func, 'yo') + interpid, res = self.loop.run_until_complete(fut) + + self.assertEqual(res, 'yo') + self.assertNotEqual(interpid, unexpected) + + def test_run_in_executor_cancel(self): + executor = self.executor_type() + + called = False + + def patched_call_soon(*args): + nonlocal called + called = True + + func = time.sleep + fut = self.loop.run_in_executor(self.executor, func, 0.05) + fut.cancel() + self.loop.run_until_complete( + self.loop.shutdown_default_executor()) + self.loop.close() + self.loop.call_soon = patched_call_soon + self.loop.call_soon_threadsafe = patched_call_soon + time.sleep(0.4) + self.assertFalse(called) + + def test_default_executor(self): + unexpected, _ = _interpreters.get_current() + + self.loop.set_default_executor(self.executor) + fut = self.loop.run_in_executor(None, get_current_interpid) + interpid, = self.loop.run_until_complete(fut) + + self.assertNotEqual(interpid, unexpected) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 7fc59a05f3deac..8b1bdaa33d8f5c 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -116,6 +116,7 @@ def test_saturation(self): for _ in range(job_count): sem.release() + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_process_reuse_one(self): executor = self.executor assert executor._max_workers >= 4 @@ -201,13 +202,13 @@ def test_python_finalization_error(self): # QueueFeederThread. orig_start_new_thread = threading._start_joinable_thread nthread = 0 - def mock_start_new_thread(func, *args): + def mock_start_new_thread(func, *args, **kwargs): nonlocal nthread if nthread >= 1: raise RuntimeError("can't create new thread at " "interpreter shutdown") nthread += 1 - return orig_start_new_thread(func, *args) + return orig_start_new_thread(func, *args, **kwargs) with support.swap_attr(threading, '_start_joinable_thread', mock_start_new_thread): diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 45dab7a75fdd50..7a4065afd46fc8 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -247,7 +247,9 @@ def test_cancel_futures_wait_false(self): # Errors in atexit hooks don't change the process exit code, check # stderr manually. self.assertFalse(err) - self.assertEqual(out.strip(), b"apple") + # gh-116682: stdout may be empty if shutdown happens before task + # starts executing. + self.assertIn(out.strip(), [b"apple", b""]) class ProcessPoolShutdownTest(ExecutorShutdownTest): diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 5926a632aa4bec..2b5bea9f4055a2 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -41,6 +41,7 @@ def acquire_lock(lock): sem.release() executor.shutdown(wait=True) + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_thread_reuse(self): executor = self.executor_type() executor.submit(mul, 21, 2).result() @@ -49,6 +50,7 @@ def test_idle_thread_reuse(self): self.assertEqual(len(executor._threads), 1) executor.shutdown(wait=True) + @support.requires_fork() @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') @support.requires_resource('cpu') def test_hang_global_shutdown_lock(self): diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index ff486202092c81..108cf54bf79e6f 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -142,7 +142,7 @@ def test_pending_calls_race(self): def future_func(): event.wait() oldswitchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) try: fs = {self.executor.submit(future_func) for i in range(100)} event.set() diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index dc48bec796b87f..52baab51340fc9 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -74,6 +74,10 @@ class ThreadPoolMixin(ExecutorMixin): executor_type = futures.ThreadPoolExecutor +class InterpreterPoolMixin(ExecutorMixin): + executor_type = futures.InterpreterPoolExecutor + + class ProcessPoolForkMixin(ExecutorMixin): executor_type = futures.ProcessPoolExecutor ctx = "fork" @@ -85,6 +89,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() @@ -111,11 +117,14 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() def create_executor_tests(remote_globals, mixin, bases=(BaseTestCase,), executor_mixins=(ThreadPoolMixin, + InterpreterPoolMixin, ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin)): @@ -136,6 +145,12 @@ def strip_mixin(name): def setup_module(): - unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + try: + _check_system_limits() + except NotImplementedError: + pass + else: + unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 2d7dfbde7082ee..e3c5d08dd1e7d1 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,7 +2,6 @@ import configparser import io import os -import pathlib import textwrap import unittest @@ -646,6 +645,21 @@ def test_weird_errors(self): "'opt' in section 'Bar' already exists") self.assertEqual(e.args, ("Bar", "opt", "", None)) + def test_get_after_duplicate_option_error(self): + cf = self.newconfig() + ini = textwrap.dedent("""\ + [Foo] + x{equals}1 + y{equals}2 + y{equals}3 + """.format(equals=self.delimiters[0])) + if self.strict: + with self.assertRaises(configparser.DuplicateOptionError): + cf.read_string(ini) + else: + cf.read_string(ini) + self.assertEqual(cf.get('Foo', 'x'), '1') + def test_write(self): config_string = ( "[Long Line]\n" @@ -730,12 +744,12 @@ def test_read_returns_file_list(self): self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a Path object: cf = self.newconfig() - parsed_files = cf.read(pathlib.Path(file1), encoding="utf-8") + parsed_files = cf.read(os_helper.FakePath(file1), encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we passed both a filename and a Path object: cf = self.newconfig() - parsed_files = cf.read([pathlib.Path(file1), file1], encoding="utf-8") + parsed_files = cf.read([os_helper.FakePath(file1), file1], encoding="utf-8") self.assertEqual(parsed_files, [file1, file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: @@ -1555,6 +1569,30 @@ def test_source_as_bytes(self): "'[badbad'" ) + def test_keys_without_value_with_extra_whitespace(self): + lines = [ + '[SECT]\n', + 'KEY1\n', + ' KEY2 = VAL2\n', # note the Space before the key! + ] + parser = configparser.ConfigParser( + comment_prefixes="", + allow_no_value=True, + strict=False, + delimiters=('=',), + interpolation=None, + ) + with self.assertRaises(configparser.MultilineContinuationError) as dse: + parser.read_file(lines) + self.assertEqual( + str(dse.exception), + "Key without value continued with an indented line.\n" + "file: '', line: 3\n" + "' KEY2 = VAL2\\n'" + ) + + + class CoverageOneHundredTestCase(unittest.TestCase): """Covers edge cases in the codebase.""" @@ -2076,6 +2114,67 @@ def test_instance_assignment(self): self.assertEqual(cfg['two'].getlen('one'), 5) +class SectionlessTestCase(unittest.TestCase): + + def fromstring(self, string): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.read_string(string) + return cfg + + def test_no_first_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + [sect1] + c = 3 + """) + + self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg1.sections())) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg1['sect1']['c']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + #self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg2.sections())) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg2['sect1']['c']) + + def test_no_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + """) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg1.sections()) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg2.sections()) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + + def test_add_section(self): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.add_section(configparser.UNNAMED_SECTION) + cfg.set(configparser.UNNAMED_SECTION, 'a', '1') + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + + def test_disabled_error(self): + with self.assertRaises(configparser.MissingSectionHeaderError): + configparser.ConfigParser().read_string("a = 1") + + with self.assertRaises(configparser.UnnamedSectionDisabledError): + configparser.ConfigParser().add_section(configparser.UNNAMED_SECTION) + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, configparser, not_exported={"Error"}) diff --git a/Lib/test/test_contains.py b/Lib/test/test_contains.py index 471d04a76ca4e4..259c954aa1d616 100644 --- a/Lib/test/test_contains.py +++ b/Lib/test/test_contains.py @@ -24,8 +24,11 @@ def test_common_tests(self): self.assertNotIn(0, b) self.assertIn(1, c) self.assertNotIn(0, c) - self.assertRaises(TypeError, lambda: 1 in a) - self.assertRaises(TypeError, lambda: 1 not in a) + msg = "argument of type 'base_set' is not a container or iterable" + with self.assertRaisesRegex(TypeError, msg): + 1 in a + with self.assertRaisesRegex(TypeError, msg): + 1 not in a # test char in string self.assertIn('c', 'abc') diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index 255be306156c0b..b06b9df9f5b0b8 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -60,6 +60,14 @@ def test_context_var_repr_1(self): c.reset(t) self.assertIn(' used ', repr(t)) + @isolated_context + def test_token_repr_1(self): + c = contextvars.ContextVar('a') + tok = c.set(1) + self.assertRegex(repr(tok), + r"^ at 0x[0-9a-fA-F]+>$") + def test_context_subclassing_1(self): with self.assertRaisesRegex(TypeError, 'not an acceptable base type'): class MyContextVar(contextvars.ContextVar): diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 36c3abca80f894..cf6519598037e9 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -444,12 +444,10 @@ class FileContextTestCase(unittest.TestCase): def testWithOpen(self): tfn = tempfile.mktemp() try: - f = None with open(tfn, "w", encoding="utf-8") as f: self.assertFalse(f.closed) f.write("Booh\n") self.assertTrue(f.closed) - f = None with self.assertRaises(ZeroDivisionError): with open(tfn, "r", encoding="utf-8") as f: self.assertFalse(f.closed) diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 89102373759ca0..3dec64cc9a2414 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -972,6 +972,10 @@ class C: copy.replace(c, x=1, error=2) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, copy, not_exported={"dispatch_table", "error"}) + def global_foo(x, y): return x+y diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index d848bfbd46c83b..e6d65e7d90abb1 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -11,6 +11,10 @@ from test.support import import_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok +try: + import _testcapi +except ImportError: + _testcapi = None class AsyncYieldFrom: @@ -970,13 +974,13 @@ def test_await_1(self): async def foo(): await 1 - with self.assertRaisesRegex(TypeError, "object int can.t.*await"): + with self.assertRaisesRegex(TypeError, "'int' object can.t be awaited"): run_async(foo()) def test_await_2(self): async def foo(): await [] - with self.assertRaisesRegex(TypeError, "object list can.t.*await"): + with self.assertRaisesRegex(TypeError, "'list' object can.t be awaited"): run_async(foo()) def test_await_3(self): @@ -1036,7 +1040,7 @@ class Awaitable: async def foo(): return await Awaitable() with self.assertRaisesRegex( - TypeError, "object Awaitable can't be used in 'await' expression"): + TypeError, "'Awaitable' object can't be awaited"): run_async(foo()) @@ -1181,7 +1185,7 @@ async def f(): async def g(): try: raise KeyError - except: + except KeyError: return await f() _, result = run_async(g()) @@ -2445,6 +2449,7 @@ def test_unawaited_warning_during_shutdown(self): @support.cpython_only +@unittest.skipIf(_testcapi is None, "requires _testcapi") class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index c6039bd17b0662..efd79448c66104 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -1,10 +1,10 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os.path +import shlex import shutil -import unittest import subprocess -import sysconfig +import unittest from test import support @@ -12,36 +12,41 @@ SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') -# gh-110119: pip does not currently support 't' in the ABI flag use by -# --disable-gil builds. Once it does, we can remove this skip. -@unittest.skipIf(support.Py_GIL_DISABLED, - 'test does not work with --disable-gil') +# With MSVC on a debug build, the linker fails with: cannot open file +# 'python311.lib', it should look 'python311_d.lib'. +@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG, + 'test fails on Windows debug build') +# Building and running an extension in clang sanitizing mode is not +# straightforward +@support.skip_if_sanitizer('test does not work with analyzing builds', + address=True, memory=True, ub=True, thread=True) +# the test uses venv+pip: skip if it's not available +@support.requires_venv_with_pip() @support.requires_subprocess() +@support.requires_resource('cpu') class TestCPPExt(unittest.TestCase): - @support.requires_resource('cpu') - def test_build_cpp11(self): - self.check_build(False, '_testcpp11ext') + def test_build(self): + self.check_build('_testcppext') - @support.requires_resource('cpu') def test_build_cpp03(self): - self.check_build(True, '_testcpp03ext') + self.check_build('_testcpp03ext', std='c++03') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11") + def test_build_cpp11(self): + self.check_build('_testcpp11ext', std='c++11') + + # Only test C++14 on MSVC. + # On s390x RHEL7, GCC 4.8.5 doesn't support C++14. + @unittest.skipIf(not support.MS_WINDOWS, "need Windows") + def test_build_cpp14(self): + self.check_build('_testcpp14ext', std='c++14') - # With MSVC, the linker fails with: cannot open file 'python311.lib' - # https://github.com/python/cpython/pull/32175#issuecomment-1111175897 - @unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') - # Building and running an extension in clang sanitizing mode is not - # straightforward - @unittest.skipIf( - '-fsanitize' in (sysconfig.get_config_var('PY_CFLAGS') or ''), - 'test does not work with analyzing builds') - # the test uses venv+pip: skip if it's not available - @support.requires_venv_with_pip() - def check_build(self, std_cpp03, extension_name): + def check_build(self, extension_name, std=None): venv_dir = 'env' with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: - self._check_build(std_cpp03, extension_name, python_exe) + self._check_build(extension_name, python_exe, std=std) - def _check_build(self, std_cpp03, extension_name, python_exe): + def _check_build(self, extension_name, python_exe, std): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) @@ -49,10 +54,11 @@ def _check_build(self, std_cpp03, extension_name, python_exe): def run_cmd(operation, cmd): env = os.environ.copy() - env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11' + if std: + env['CPYTHON_TEST_CPP_STD'] = std env['CPYTHON_TEST_EXT_NAME'] = extension_name if support.verbose: - print('Run:', ' '.join(cmd)) + print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) else: proc = subprocess.run(cmd, @@ -61,6 +67,7 @@ def run_cmd(operation, cmd): stderr=subprocess.STDOUT, text=True) if proc.returncode: + print('Run:', ' '.join(map(shlex.quote, cmd))) print(proc.stdout, end='') self.fail( f"{operation} failed with exit code {proc.returncode}") @@ -69,6 +76,8 @@ def run_cmd(operation, cmd): cmd = [python_exe, '-X', 'dev', '-m', 'pip', 'install', '--no-build-isolation', os.path.abspath(pkg_dir)] + if support.verbose: + cmd.append('-v') run_cmd('Install', cmd) # Do a reference run. Until we test that running python diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index 90669b10cb2c6d..ab485b629b7788 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -8,10 +8,8 @@ #include "Python.h" -#if __cplusplus >= 201103 -# define NAME _testcpp11ext -#else -# define NAME _testcpp03ext +#ifndef MODULE_NAME +# error "MODULE_NAME macro must be defined" #endif #define _STR(NAME) #NAME @@ -160,7 +158,7 @@ PyType_Slot VirtualPyObject_Slots[] = { }; PyType_Spec VirtualPyObject_Spec = { - /* .name */ STR(NAME) ".VirtualPyObject", + /* .name */ STR(MODULE_NAME) ".VirtualPyObject", /* .basicsize */ sizeof(VirtualPyObject), /* .itemsize */ 0, /* .flags */ Py_TPFLAGS_DEFAULT, @@ -227,6 +225,10 @@ _testcppext_exec(PyObject *module) if (!result) return -1; Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + return 0; } @@ -240,7 +242,7 @@ PyDoc_STRVAR(_testcppext_doc, "C++ test extension."); static struct PyModuleDef _testcppext_module = { PyModuleDef_HEAD_INIT, // m_base - STR(NAME), // m_name + STR(MODULE_NAME), // m_name _testcppext_doc, // m_doc 0, // m_size _testcppext_methods, // m_methods @@ -254,7 +256,7 @@ static struct PyModuleDef _testcppext_module = { #define FUNC_NAME(NAME) _FUNC_NAME(NAME) PyMODINIT_FUNC -FUNC_NAME(NAME)(void) +FUNC_NAME(MODULE_NAME)(void) { return PyModuleDef_Init(&_testcppext_module); } diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index c7ba1efb4dd05a..d97b238b8d1477 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -1,8 +1,8 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os +import platform import shlex -import sys import sysconfig from test import support @@ -10,6 +10,7 @@ SOURCE = 'extension.cpp' + if not support.MS_WINDOWS: # C++ compiler flags for GCC and clang CPPFLAGS = [ @@ -19,34 +20,77 @@ '-Werror', ] else: - # Don't pass any compiler flag to MSVC - CPPFLAGS = [] + # MSVC compiler flags + CPPFLAGS = [ + # Display warnings level 1 to 4 + '/W4', + # Treat all compiler warnings as compiler errors + '/WX', + ] def main(): cppflags = list(CPPFLAGS) - std = os.environ["CPYTHON_TEST_CPP_STD"] - name = os.environ["CPYTHON_TEST_EXT_NAME"] + std = os.environ.get("CPYTHON_TEST_CPP_STD", "") + module_name = os.environ["CPYTHON_TEST_EXT_NAME"] - cppflags = [*CPPFLAGS, f'-std={std}'] + cppflags = list(CPPFLAGS) + cppflags.append(f'-DMODULE_NAME={module_name}') + + # Add -std=STD or /std:STD (MSVC) compiler flag + if std: + if support.MS_WINDOWS: + cppflags.append(f'/std:{std}') + else: + cppflags.append(f'-std={std}') # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 # option emits a C++ compiler warning. Remove "-std11" option from the # CC command. cmd = (sysconfig.get_config_var('CC') or '') if cmd is not None: + if support.MS_WINDOWS: + std_prefix = '/std' + else: + std_prefix = '-std' cmd = shlex.split(cmd) - cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] cmd = shlex.join(cmd) # CC env var overrides sysconfig CC variable in setuptools os.environ['CC'] = cmd - cpp_ext = Extension( - name, + # On Windows, add PCbuild\amd64\ to include and library directories + include_dirs = [] + library_dirs = [] + if support.MS_WINDOWS: + srcdir = sysconfig.get_config_var('srcdir') + machine = platform.uname().machine + pcbuild = os.path.join(srcdir, 'PCbuild', machine) + if os.path.exists(pcbuild): + # pyconfig.h is generated in PCbuild\amd64\ + include_dirs.append(pcbuild) + # python313.lib is generated in PCbuild\amd64\ + library_dirs.append(pcbuild) + print(f"Add PCbuild directory: {pcbuild}") + + # Display information to help debugging + for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): + if env_name in os.environ: + print(f"{env_name} env var: {os.environ[env_name]!r}") + else: + print(f"{env_name} env var: ") + print(f"extra_compile_args: {cppflags!r}") + + ext = Extension( + module_name, sources=[SOURCE], language='c++', - extra_compile_args=cppflags) - setup(name='internal' + name, version='0.0', ext_modules=[cpp_ext]) + extra_compile_args=cppflags, + include_dirs=include_dirs, + library_dirs=library_dirs) + setup(name=f'internal_{module_name}', + version='0.0', + ext_modules=[ext]) if __name__ == "__main__": diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 27e8a767903777..65720871d5c5f0 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -30,6 +30,59 @@ def test_bad_counter_during_dealloc(self): self.assertEqual(cm.unraisable.exc_type, TypeError) + def test_crash_with_not_enough_args(self): + # gh-126220 + import _lsprof + + for profile in [_lsprof.Profiler(), cProfile.Profile()]: + for method in [ + "_pystart_callback", + "_pyreturn_callback", + "_ccall_callback", + "_creturn_callback", + ]: + with self.subTest(profile=profile, method=method): + method_obj = getattr(profile, method) + with self.assertRaises(TypeError): + method_obj() # should not crash + + def test_evil_external_timer(self): + # gh-120289 + # Disabling profiler in external timer should not crash + import _lsprof + class EvilTimer(): + def __init__(self, disable_count): + self.count = 0 + self.disable_count = disable_count + + def __call__(self): + self.count += 1 + if self.count == self.disable_count: + profiler_with_evil_timer.disable() + return self.count + + # this will trigger external timer to disable profiler at + # call event - in initContext in _lsprof.c + with support.catch_unraisable_exception() as cm: + profiler_with_evil_timer = _lsprof.Profiler(EvilTimer(1)) + profiler_with_evil_timer.enable() + # Make a call to trigger timer + (lambda: None)() + profiler_with_evil_timer.disable() + profiler_with_evil_timer.clear() + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + + # this will trigger external timer to disable profiler at + # return event - in Stop in _lsprof.c + with support.catch_unraisable_exception() as cm: + profiler_with_evil_timer = _lsprof.Profiler(EvilTimer(2)) + profiler_with_evil_timer.enable() + # Make a call to trigger timer + (lambda: None)() + profiler_with_evil_timer.disable() + profiler_with_evil_timer.clear() + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + def test_profile_enable_disable(self): prof = self.profilerclass() # Make sure we clean ourselves up if the test fails for some reason. diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 21a4cb586ff665..ce5c03659f1979 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -64,8 +64,7 @@ def _test_arg_valid(self, ctor, arg): ctor(arg, delimiter='\t', skipinitialspace=True) ctor(arg, escapechar='\t', skipinitialspace=True) ctor(arg, quotechar='\t', skipinitialspace=True) - self.assertRaises(ValueError, ctor, arg, - delimiter=' ', skipinitialspace=True) + ctor(arg, delimiter=' ', skipinitialspace=True) self.assertRaises(ValueError, ctor, arg, escapechar=' ', skipinitialspace=True) self.assertRaises(ValueError, ctor, arg, @@ -192,9 +191,6 @@ def _write_error_test(self, exc, fields, **kwargs): def test_write_arg_valid(self): self._write_error_test(csv.Error, None) - self._write_test((), '') - self._write_test([None], '""') - self._write_error_test(csv.Error, [None], quoting = csv.QUOTE_NONE) # Check that exceptions are passed up the chain self._write_error_test(OSError, BadIterable()) class BadList: @@ -208,7 +204,6 @@ class BadItem: def __str__(self): raise OSError self._write_error_test(OSError, [BadItem()]) - def test_write_bigfield(self): # This exercises the buffer realloc functionality bigstring = 'X' * 50000 @@ -270,9 +265,11 @@ def test_write_lineterminator(self): writer = csv.writer(sio, lineterminator=lineterminator) writer.writerow(['a', 'b']) writer.writerow([1, 2]) + writer.writerow(['\r', '\n']) self.assertEqual(sio.getvalue(), f'a,b{lineterminator}' - f'1,2{lineterminator}') + f'1,2{lineterminator}' + f'"\r","\n"{lineterminator}') def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') @@ -315,6 +312,49 @@ def test_writerows_with_none(self): fileobj.seek(0) self.assertEqual(fileobj.read(), 'a\r\n""\r\n') + + def test_write_empty_fields(self): + self._write_test((), '') + self._write_test([''], '""') + self._write_error_test(csv.Error, [''], quoting=csv.QUOTE_NONE) + self._write_test([''], '""', quoting=csv.QUOTE_STRINGS) + self._write_test([''], '""', quoting=csv.QUOTE_NOTNULL) + self._write_test([None], '""') + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_STRINGS) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NOTNULL) + self._write_test(['', ''], ',') + self._write_test([None, None], ',') + + def test_write_empty_fields_space_delimiter(self): + self._write_test([''], '""', delimiter=' ', skipinitialspace=False) + self._write_test([''], '""', delimiter=' ', skipinitialspace=True) + self._write_test([None], '""', delimiter=' ', skipinitialspace=False) + self._write_test([None], '""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True) + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False) + self._write_test([None, None], '"" ""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False, + quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, ['', ''], + delimiter=' ', skipinitialspace=True, + quoting=csv.QUOTE_NONE) + for quoting in csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True, + quoting=quoting) + + for quoting in csv.QUOTE_NONE, csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_error_test(csv.Error, [None, None], + delimiter=' ', skipinitialspace=True, + quoting=quoting) + def test_writerows_errors(self): with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) @@ -414,6 +454,10 @@ def test_read_quoting(self): quoting=csv.QUOTE_STRINGS) self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') + self._read_test(['1\\.5,\\.5,.5'], [[1.5, 0.5, 0.5]], + quoting=csv.QUOTE_NONNUMERIC, escapechar='\\') + self._read_test(['1\\.5,\\.5,"\\.5"'], [[1.5, 0.5, ".5"]], + quoting=csv.QUOTE_STRINGS, escapechar='\\') def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], @@ -429,6 +473,14 @@ def test_read_skipinitialspace(self): [[None, None, None]], skipinitialspace=True, quoting=csv.QUOTE_STRINGS) + def test_read_space_delimiter(self): + self._read_test(['a b', ' a ', ' ', ''], + [['a', '', '', 'b'], ['', '', 'a', '', ''], ['', '', ''], []], + delimiter=' ', skipinitialspace=False) + self._read_test(['a b', ' a ', ' ', ''], + [['a', 'b'], ['a', ''], [''], []], + delimiter=' ', skipinitialspace=True) + def test_read_bigfield(self): # This exercises the buffer realloc functionality and field size # limits. @@ -461,22 +513,44 @@ def test_read_linenum(self): self.assertEqual(r.line_num, 3) def test_roundtrip_quoteed_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj) - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj)): - self.assertEqual(row, rows[i]) + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator) + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj)): + self.assertEqual(row, rows[i]) def test_roundtrip_escaped_unquoted_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\") - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\")): - self.assertEqual(row,rows[i]) + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator, + quoting=csv.QUOTE_NONE, escapechar="\\") + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj, + quoting=csv.QUOTE_NONE, + escapechar="\\")): + self.assertEqual(row, rows[i]) + class TestDialectRegistry(unittest.TestCase): def test_registry_badargs(self): @@ -555,10 +629,10 @@ class space(csv.excel): escapechar = "\\" with TemporaryFile("w+", encoding="utf-8") as fileobj: - fileobj.write("abc def\nc1ccccc1 benzene\n") + fileobj.write("abc def\nc1ccccc1 benzene\n") fileobj.seek(0) reader = csv.reader(fileobj, dialect=space()) - self.assertEqual(next(reader), ["abc", "def"]) + self.assertEqual(next(reader), ["abc", "", "", "def"]) self.assertEqual(next(reader), ["c1ccccc1", "benzene"]) def compare_dialect_123(self, expected, *writeargs, **kwwriteargs): @@ -1029,6 +1103,12 @@ class mydialect(csv.Dialect): mydialect.quoting = None self.assertRaises(csv.Error, mydialect) + mydialect.quoting = 42 + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + 'bad "quoting" value') + mydialect.doublequote = True mydialect.quoting = csv.QUOTE_ALL mydialect.quotechar = '"' @@ -1164,8 +1244,9 @@ class mydialect(csv.Dialect): self.assertRaises(csv.Error, create_invalid, field_name, 5) self.assertRaises(ValueError, create_invalid, field_name, "\n") self.assertRaises(ValueError, create_invalid, field_name, "\r") - self.assertRaises(ValueError, create_invalid, field_name, " ", - skipinitialspace=True) + if field_name != "delimiter": + self.assertRaises(ValueError, create_invalid, field_name, " ", + skipinitialspace=True) class TestSniffer(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_aligned_structures.py b/Lib/test/test_ctypes/test_aligned_structures.py new file mode 100644 index 00000000000000..a208fb9a00966a --- /dev/null +++ b/Lib/test/test_ctypes/test_aligned_structures.py @@ -0,0 +1,286 @@ +from ctypes import ( + c_char, c_uint32, c_uint16, c_ubyte, c_byte, alignment, sizeof, + BigEndianStructure, LittleEndianStructure, + BigEndianUnion, LittleEndianUnion, +) +import struct +import unittest + + +class TestAlignedStructures(unittest.TestCase): + def test_aligned_string(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}i12x16s", 7, b"hello world!")) + class Aligned(base): + _align_ = 16 + _fields_ = [ + ('value', c_char * 12) + ] + + class Main(base): + _fields_ = [ + ('first', c_uint32), + ('string', Aligned), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.first, 7) + self.assertEqual(main.string.value, b'hello world!') + self.assertEqual(bytes(main.string), b'hello world!\0\0\0\0') + self.assertEqual(Main.string.offset, 16) + self.assertEqual(Main.string.size, 16) + self.assertEqual(alignment(main.string), 16) + self.assertEqual(alignment(main), 16) + + def test_aligned_structures(self): + for base, data in ( + (LittleEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + (BigEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + ): + class SomeBools(base): + _align_ = 4 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ] + class Main(base): + _fields_ = [ + ("x", c_ubyte), + ("y", SomeBools), + ("z", c_ubyte), + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(SomeBools), 4) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.y), 4) + self.assertEqual(Main.x.size, 1) + self.assertEqual(Main.y.offset, 4) + self.assertEqual(Main.y.size, 4) + self.assertEqual(main.y.bool1, True) + self.assertEqual(main.y.bool2, False) + self.assertEqual(Main.z.offset, 8) + self.assertEqual(main.z, 7) + + def test_oversized_structure(self): + data = bytearray(b"\0" * 8) + for base in (LittleEndianStructure, BigEndianStructure): + class SomeBoolsTooBig(base): + _align_ = 8 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ("bool3", c_ubyte), + ] + class Main(base): + _fields_ = [ + ("y", SomeBoolsTooBig), + ("z", c_uint32), + ] + with self.assertRaises(ValueError) as ctx: + Main.from_buffer(data) + self.assertEqual( + ctx.exception.args[0], + 'Buffer size too small (4 instead of at least 8 bytes)' + ) + + def test_aligned_subclasses(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class UnalignedSub(base): + x: c_uint32 + _fields_ = [ + ("x", c_uint32), + ] + + class AlignedStruct(UnalignedSub): + _align_ = 8 + _fields_ = [ + ("y", c_uint32), + ] + + class Main(base): + _fields_ = [ + ("a", c_uint32), + ("b", AlignedStruct) + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(main.b), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(sizeof(main), 16) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 3) + self.assertEqual(main.b.y, 4) + self.assertEqual(Main.b.offset, 8) + self.assertEqual(Main.b.size, 8) + + def test_aligned_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class AlignedUnion(ubase): + _align_ = 8 + _fields_ = [ + ("a", c_uint32), + ("b", c_ubyte * 7), + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", AlignedUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(bytes(main.union.b), data[8:-1]) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + + def test_aligned_struct_in_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class Sub(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint32), + ("y", c_uint32), + ] + + class MainUnion(ubase): + _fields_ = [ + ("a", c_uint32), + ("b", Sub), + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", MainUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(Main.first.size, 4) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(main.union.b.x, 3) + self.assertEqual(main.union.b.y, 4) + + def test_smaller_aligned_subclassed_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}H2xI", 1, 0xD60102D7)) + class SubUnion(ubase): + _align_ = 2 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + + class MainUnion(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint16), + ("union", MainUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.union.num, 0xD60102D7) + self.assertEqual(main.union.unsigned, data[4]) + self.assertEqual(main.union.signed, data[4] - 256) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.union), 4) + self.assertEqual(Main.union.offset, 4) + self.assertEqual(Main.union.size, 4) + self.assertEqual(Main.first.size, 2) + + def test_larger_aligned_subclassed_union(self): + for ubase, e in ( + (LittleEndianUnion, "<"), + (BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}I4x", 0xD60102D6)) + class SubUnion(ubase): + _align_ = 8 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + + class Main(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main), 8) + self.assertEqual(main.num, 0xD60102D6) + self.assertEqual(main.unsigned, 0xD6) + self.assertEqual(main.signed, -42) + + def test_aligned_packed_structures(self): + for sbase, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}B2H4xB", 1, 2, 3, 4)) + + class Inner(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint16), + ("y", c_uint16), + ] + + class Main(sbase): + _pack_ = 1 + _fields_ = [ + ("a", c_ubyte), + ("b", Inner), + ("c", c_ubyte), + ] + + main = Main.from_buffer(data) + self.assertEqual(sizeof(main), 10) + self.assertEqual(Main.b.offset, 1) + # Alignment == 8 because _pack_ wins out. + self.assertEqual(alignment(main.b), 8) + # Size is still 8 though since inside this Structure, it will have + # effect. + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(Main.c.offset, 9) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 2) + self.assertEqual(main.b.y, 3) + self.assertEqual(main.c, 4) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_ctypes/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py index 774316e227ff73..c80fdff5de685d 100644 --- a/Lib/test/test_ctypes/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -1,8 +1,7 @@ import ctypes import sys import unittest -import warnings -from ctypes import (Structure, Array, sizeof, addressof, +from ctypes import (Structure, Array, ARRAY, sizeof, addressof, create_string_buffer, create_unicode_buffer, c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulonglong, c_float, c_double, c_longdouble) @@ -17,13 +16,6 @@ c_long, c_ulonglong, c_float, c_double, c_longdouble -def ARRAY(*args): - # ignore DeprecationWarning in tests - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - return ctypes.ARRAY(*args) - - class ArrayTestCase(unittest.TestCase): def test_inheritance_hierarchy(self): self.assertEqual(Array.mro(), [Array, _CData, object]) @@ -37,6 +29,22 @@ def test_type_flags(self): self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewArray = PyCArrayType.__new__(PyCArrayType, 'NewArray', (Array,), {}) + for cls in Array, NewArray: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Array): + _type_ = c_int + _length_ = 13 + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCArrayType.__init__(T, 'ptr', (), {}) + def test_simple(self): # create classes holding simple numeric types, and check # various properties. @@ -237,7 +245,7 @@ def test_empty_element_struct(self): class EmptyStruct(Structure): _fields_ = [] - obj = (EmptyStruct * 2)() # bpo37188: Floating point exception + obj = (EmptyStruct * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_empty_element_array(self): @@ -245,7 +253,7 @@ class EmptyArray(Array): _type_ = c_int _length_ = 0 - obj = (EmptyArray * 2)() # bpo37188: Floating point exception + obj = (EmptyArray * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_bpo36504_signed_int_overflow(self): @@ -259,10 +267,6 @@ def test_bpo36504_signed_int_overflow(self): def test_large_array(self, size): c_char * size - def test_deprecation(self): - with self.assertWarns(DeprecationWarning): - CharArray = ctypes.ARRAY(c_char, 3) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index ca75e748256083..cc62b1a22a3b06 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import unittest from ctypes import (Structure, CDLL, CFUNCTYPE, @@ -6,6 +5,8 @@ c_short, c_int, c_long, c_longlong, c_byte, c_wchar, c_float, c_double, ArgumentError) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index d43c56ad371fbd..19ba2f4484e7da 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -1,12 +1,16 @@ -import _ctypes_test import os +import sys import unittest from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, LittleEndianStructure, BigEndianStructure, c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, - c_uint32, c_uint64, - c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) + c_uint8, c_uint16, c_uint32, c_uint64, + c_short, c_ushort, c_int, c_uint, c_long, c_ulong, + c_longlong, c_ulonglong, + Union) from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class BITS(Structure): @@ -32,27 +36,88 @@ class BITS(Structure): func.argtypes = POINTER(BITS), c_char +class BITS_msvc(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + + +try: + func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc +except AttributeError as err: + # The MSVC struct must be available on Windows; it's optional elsewhere + if support.MS_WINDOWS: + raise err + func_msvc = None +else: + func_msvc.argtypes = POINTER(BITS_msvc), c_char + + class C_Test(unittest.TestCase): def test_ints(self): for i in range(512): for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) - # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior - @support.skip_if_sanitizer(ub=True) def test_shorts(self): b = BITS() name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. if func(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. self.skipTest("Compiler does not support signed short bitfields") for i in range(256): for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) + + @unittest.skipUnless(func_msvc, "need MSVC or __attribute__((ms_struct))") + def test_shorts_msvc_mode(self): + b = BITS_msvc() + name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. + if func_msvc(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. + self.skipTest("Compiler does not support signed short bitfields") + for i in range(256): + for name in "MNOPQRS": + with self.subTest(i=i, name=name): + b = BITS_msvc() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func_msvc(byref(b), name.encode('ascii'))) signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) @@ -86,39 +151,47 @@ class X(Structure): def test_signed(self): for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)*2) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("dummy", c_typ), + ("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)*2) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) def test_unsigned(self): for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) + for layout in "ms", "gcc-sysv": + with self.subTest(layout=layout): + return self.get_except(type(Structure), "X", (), + {"_fields_": fields, "layout": layout}) def test_nonint_types(self): # bit fields are not allowed on non-integer types. @@ -135,9 +208,15 @@ def test_nonint_types(self): result = self.fail_fields(("a", c_char, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char')) - class Dummy(Structure): + class Empty(Structure): _fields_ = [] + result = self.fail_fields(("a", Empty, 1)) + self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) + + class Dummy(Structure): + _fields_ = [("x", c_int)] + result = self.fail_fields(("a", Dummy, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) @@ -148,22 +227,28 @@ def test_c_wchar(self): def test_single_bitfield_size(self): for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + result = self.fail_fields(("a", c_typ, -1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - result = self.fail_fields(("a", c_typ, 0)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + result = self.fail_fields(("a", c_typ, 0)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - class X(Structure): - _fields_ = [("a", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, sizeof(c_typ)*8)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) def test_multi_bitfields_size(self): class X(Structure): @@ -235,6 +320,161 @@ class X(Structure): else: self.assertEqual(sizeof(X), sizeof(c_int) * 2) + def test_mixed_5(self): + class X(Structure): + _fields_ = [ + ('A', c_uint, 1), + ('B', c_ushort, 16)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + def test_mixed_6(self): + class X(Structure): + _fields_ = [ + ('A', c_ulonglong, 1), + ('B', c_uint, 32)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_7(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + self.assertEqual(16, sizeof(X)) + + def test_mixed_8(self): + class Foo(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1), + ] + + class Bar(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1), + ] + self.assertEqual(sizeof(Foo), sizeof(Bar)) + + def test_mixed_9(self): + class X(Structure): + _fields_ = [ + ("A", c_uint8), + ("B", c_uint32, 1), + ] + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_10(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32, 1), + ("B", c_uint64, 1), + ] + if sys.platform == 'win32': + self.assertEqual(8, alignment(X)) + self.assertEqual(16, sizeof(X)) + else: + self.assertEqual(8, alignment(X)) + self.assertEqual(8, sizeof(X)) + + def test_gh_95496(self): + for field_width in range(1, 33): + class TestStruct(Structure): + _fields_ = [ + ("Field1", c_uint32, field_width), + ("Field2", c_uint8, 8) + ] + + cmd = TestStruct() + cmd.Field2 = 1 + self.assertEqual(1, cmd.Field2) + + def test_gh_84039(self): + class Bad(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + + class GoodA(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ] + + + class Good(Structure): + _pack_ = 1 + _fields_ = [ + ("a", GoodA), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + self.assertEqual(3, sizeof(Bad)) + self.assertEqual(3, sizeof(Good)) + + def test_gh_73939(self): + class MyStructure(Structure): + _pack_ = 1 + _fields_ = [ + ("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2) + ] + self.assertEqual(8, sizeof(MyStructure)) + + def test_gh_86098(self): + class X(Structure): + _fields_ = [ + ("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16) + ] + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message class X(Structure): @@ -288,6 +528,21 @@ class Big(BigEndianStructure): x.c = 2 self.assertEqual(b, b'\xab\xcd\xef\x12') + def test_union_bitfield(self): + class BitfieldUnion(Union): + _fields_ = [("a", c_uint32, 1), + ("b", c_uint32, 2), + ("c", c_uint32, 3)] + self.assertEqual(sizeof(BitfieldUnion), 4) + b = bytearray(4) + x = BitfieldUnion.from_buffer(b) + x.a = 1 + self.assertEqual(int.from_bytes(b).bit_count(), 1) + x.b = 3 + self.assertEqual(int.from_bytes(b).bit_count(), 2) + x.c = 7 + self.assertEqual(int.from_bytes(b).bit_count(), 3) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py new file mode 100644 index 00000000000000..eb77d6d7782478 --- /dev/null +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -0,0 +1,152 @@ +import unittest +import ctypes +from ctypes import POINTER, c_void_p + +from ._support import PyCSimpleType + + +class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): + def tearDown(self): + # to not leak references, we must clean _pointer_type_cache + ctypes._reset_cache() + + def test_creating_pointer_in_dunder_new_1(self): + # Test metaclass whose instances are C types; when the type is + # created it automatically creates a pointer type for itself. + # The pointer type is also an instance of the metaclass. + # Such an implementation is used in `IUnknown` of the `comtypes` + # project. See gh-124520. + + class ct_meta(type): + def __new__(cls, name, bases, namespace): + self = super().__new__(cls, name, bases, namespace) + + # Avoid recursion: don't set up a pointer to + # a pointer (to a pointer...) + if bases == (c_void_p,): + # When creating PtrBase itself, the name + # is not yet available + return self + if issubclass(self, PtrBase): + return self + + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + ctypes._pointer_type_cache[self] = p + return self + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + class CtBase(object, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + class Sub2(Sub): + pass + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertTrue(issubclass(POINTER(Sub2), Sub2)) + self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub))) + self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase))) + + def test_creating_pointer_in_dunder_new_2(self): + # A simpler variant of the above, used in `CoClass` of the `comtypes` + # project. + + class ct_meta(type): + def __new__(cls, name, bases, namespace): + self = super().__new__(cls, name, bases, namespace) + if isinstance(self, p_meta): + return self + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + ctypes._pointer_type_cache[self] = p + return self + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertTrue(issubclass(POINTER(Sub), Sub)) + + def test_creating_pointer_in_dunder_init_1(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_1) + if bases == (c_void_p,): + return + if issubclass(self, PtrBase): + return + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + class CtBase(object, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + class Sub2(Sub): + pass + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertTrue(issubclass(POINTER(Sub2), Sub2)) + self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub))) + self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase))) + + def test_creating_pointer_in_dunder_init_2(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_2) + if isinstance(self, p_meta): + return + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertTrue(issubclass(POINTER(Sub), Sub)) diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index 19f4158c0ac846..8f483dfe1db801 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import functools import gc @@ -14,6 +13,8 @@ c_float, c_double, c_longdouble, py_object) from ctypes.util import find_library from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class Callbacks(unittest.TestCase): @@ -106,7 +107,7 @@ def test_pyobject(self): def test_unsupported_restype_1(self): # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. + # functions, the type must have a non-NULL stginfo->setfunc. # POINTER(c_double), for example, is not supported. prototype = self.functype.__func__(POINTER(c_double)) @@ -148,9 +149,10 @@ def callback(a, b): print(f"a={a}, b={b}, c={c}") return c dll = cdll[_ctypes_test.__file__] - # With no fix for i38748, the next line will raise OSError and cause the test to fail. - self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) - + with support.captured_stdout() as out: + # With no fix for i38748, the next line will raise OSError and cause the test to fail. + self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) + self.assertEqual(out.getvalue(), "a=5, b=10, c=15\n") if hasattr(ctypes, 'WINFUNCTYPE'): class StdcallCallbacks(Callbacks): diff --git a/Lib/test/test_ctypes/test_cfuncs.py b/Lib/test/test_ctypes/test_cfuncs.py index 6ff0878a35da2f..48330c4b0a763b 100644 --- a/Lib/test/test_ctypes/test_cfuncs.py +++ b/Lib/test/test_ctypes/test_cfuncs.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import unittest from ctypes import (CDLL, @@ -6,6 +5,8 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class CFunctions(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_checkretval.py b/Lib/test/test_ctypes/test_checkretval.py index 5dc9e25aa38226..9d6bfdb845e6c7 100644 --- a/Lib/test/test_ctypes/test_checkretval.py +++ b/Lib/test/test_ctypes/test_checkretval.py @@ -1,7 +1,8 @@ -import _ctypes_test import ctypes import unittest from ctypes import CDLL, c_int +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class CHECKED(c_int): diff --git a/Lib/test/test_ctypes/test_find.py b/Lib/test/test_ctypes/test_find.py index 7732ff37308848..85b28617d2d754 100644 --- a/Lib/test/test_ctypes/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -129,5 +129,28 @@ def test_gh114257(self): self.assertIsNone(find_library("libc")) +@unittest.skipUnless(sys.platform == 'android', 'Test only valid for Android') +class FindLibraryAndroid(unittest.TestCase): + def test_find(self): + for name in [ + "c", "m", # POSIX + "z", # Non-POSIX, but present on Linux + "log", # Not present on Linux + ]: + with self.subTest(name=name): + path = find_library(name) + self.assertIsInstance(path, str) + self.assertEqual( + os.path.dirname(path), + "/system/lib64" if "64" in os.uname().machine + else "/system/lib") + self.assertEqual(os.path.basename(path), f"lib{name}.so") + self.assertTrue(os.path.isfile(path), path) + + for name in ["libc", "nonexistent"]: + with self.subTest(name=name): + self.assertIsNone(find_library(name)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index 0eed39484fb39e..8362fb16d94dcd 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -1,8 +1,9 @@ -import _ctypes_test import ctypes import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, sizeof, _CFuncPtr, c_void_p, c_char_p, c_char, c_int, c_uint, c_long) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -29,6 +30,12 @@ def test_type_flags(self): self.assertTrue(_CFuncPtr.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(_CFuncPtr.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + CdeclCallback = CFUNCTYPE(c_int, c_int, c_int) + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCFuncPtrType.__init__(CdeclCallback, 'ptr', (), {}) + def test_basic(self): X = WINFUNCTYPE(c_int, c_int, c_int) diff --git a/Lib/test/test_ctypes/test_functions.py b/Lib/test/test_ctypes/test_functions.py index 04e8582ff1e427..3454b83d43e1e7 100644 --- a/Lib/test/test_ctypes/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -1,12 +1,13 @@ -import _ctypes_test import ctypes import sys import unittest from ctypes import (CDLL, Structure, Array, CFUNCTYPE, - byref, POINTER, pointer, ArgumentError, + byref, POINTER, pointer, ArgumentError, sizeof, c_char, c_wchar, c_byte, c_char_p, c_wchar_p, c_short, c_int, c_long, c_longlong, c_void_p, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from _ctypes import _Pointer, _SimpleCData @@ -71,7 +72,8 @@ def callback(*args): self.assertEqual(str(cm.exception), "argument 1: TypeError: one character bytes, " - "bytearray or integer expected") + "bytearray, or an integer in range(256) expected, " + "not bytes of length 3") def test_wchar_parm(self): f = dll._testfunc_i_bhilfd @@ -83,14 +85,27 @@ def test_wchar_parm(self): with self.assertRaises(ArgumentError) as cm: f(1, 2, 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: unicode string expected " - "instead of int instance") + "argument 2: TypeError: a unicode character expected, " + "not instance of int") with self.assertRaises(ArgumentError) as cm: f(1, "abc", 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: one character unicode string " - "expected") + "argument 2: TypeError: a unicode character expected, " + "not a string of length 3") + + with self.assertRaises(ArgumentError) as cm: + f(1, "", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: a unicode character expected, " + "not a string of length 0") + + if sizeof(c_wchar) < 4: + with self.assertRaises(ArgumentError) as cm: + f(1, "\U0001f40d", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: the string '\\U0001f40d' " + "cannot be converted to a single wchar_t character") def test_c_char_p_parm(self): """Test the error message when converting an incompatible type to c_char_p.""" diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py new file mode 100644 index 00000000000000..d61754d6d49e70 --- /dev/null +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -0,0 +1,731 @@ +"""Test CTypes structs, unions, bitfields against C equivalents. + +The types here are auto-converted to C source at +`Modules/_ctypes/_ctypes_test_generated.c.h`, which is compiled into +_ctypes_test. + +Run this module to regenerate the files: + +./python Lib/test/test_ctypes/test_generated_structs.py > Modules/_ctypes/_ctypes_test_generated.c.h +""" + +import unittest +from test.support import import_helper +import re +from dataclasses import dataclass +from functools import cached_property + +import ctypes +from ctypes import Structure, Union +from ctypes import sizeof, alignment, pointer, string_at +_ctypes_test = import_helper.import_module("_ctypes_test") + + +# ctypes erases the difference between `c_int` and e.g.`c_int16`. +# To keep it, we'll use custom subclasses with the C name stashed in `_c_name`: +class c_bool(ctypes.c_bool): + _c_name = '_Bool' + +# To do it for all the other types, use some metaprogramming: +for c_name, ctypes_name in { + 'signed char': 'c_byte', + 'short': 'c_short', + 'int': 'c_int', + 'long': 'c_long', + 'long long': 'c_longlong', + 'unsigned char': 'c_ubyte', + 'unsigned short': 'c_ushort', + 'unsigned int': 'c_uint', + 'unsigned long': 'c_ulong', + 'unsigned long long': 'c_ulonglong', + **{f'{u}int{n}_t': f'c_{u}int{n}' + for u in ('', 'u') + for n in (8, 16, 32, 64)} +}.items(): + ctype = getattr(ctypes, ctypes_name) + newtype = type(ctypes_name, (ctype,), {'_c_name': c_name}) + globals()[ctypes_name] = newtype + + +# Register structs and unions to test + +TESTCASES = {} +def register(name=None, set_name=False): + def decorator(cls, name=name): + if name is None: + name = cls.__name__ + assert name.isascii() # will be used in _PyUnicode_EqualToASCIIString + assert name.isidentifier() # will be used as a C identifier + assert name not in TESTCASES + TESTCASES[name] = cls + if set_name: + cls.__name__ = name + return cls + return decorator + +@register() +class SingleInt(Structure): + _fields_ = [('a', c_int)] + +@register() +class SingleInt_Union(Union): + _fields_ = [('a', c_int)] + + +@register() +class SingleU32(Structure): + _fields_ = [('a', c_uint32)] + + +@register() +class SimpleStruct(Structure): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class SimpleUnion(Union): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class ManyTypes(Structure): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class ManyTypesU(Union): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class Nested(Structure): + _fields_ = [ + ('a', SimpleStruct), ('b', SimpleUnion), ('anon', SimpleStruct), + ] + _anonymous_ = ['anon'] + + +@register() +class Packed1(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 1 + + +@register() +class Packed2(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 2 + + +@register() +class Packed3(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 4 + + +@register() +class Packed4(Structure): + def _maybe_skip(): + # `_pack_` enables MSVC-style packing, but keeps platform-specific + # alignments. + # The C code we generate for GCC/clang currently uses + # `__attribute__((ms_struct))`, which activates MSVC layout *and* + # alignments, that is, sizeof(basic type) == alignment(basic type). + # On a Pentium, int64 is 32-bit aligned, so the two won't match. + # The expected behavior is instead tested in + # StructureTestCase.test_packed, over in test_structures.py. + if sizeof(c_int64) != alignment(c_int64): + raise unittest.SkipTest('cannot test on this platform') + + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 8 + +@register() +class X86_32EdgeCase(Structure): + # On a Pentium, long long (int64) is 32-bit aligned, + # so these are packed tightly. + _fields_ = [('a', c_int32), ('b', c_int64), ('c', c_int32)] + +@register() +class MSBitFieldExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('a', c_uint, 4), + ('b', c_uint, 5), + ('c', c_uint, 7)] + +@register() +class MSStraddlingExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('first', c_uint, 9), + ('second', c_uint, 7), + ('may_straddle', c_uint, 30), + ('last', c_uint, 18)] + +@register() +class IntBits(Structure): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits(Structure): + _fields_ = [*IntBits._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class IntBits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [*IntBits_MSVC._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +# Skipped for now -- we don't always match the alignment +#@register() +class IntBits_Union(Union): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +# Skipped for now -- we don't always match the alignment +#@register() +class BitsUnion(Union): + _fields_ = [*IntBits_Union._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class I64Bits(Structure): + _fields_ = [("a", c_int64, 1), + ("b", c_int64, 62), + ("c", c_int64, 1)] + +@register() +class U64Bits(Structure): + _fields_ = [("a", c_uint64, 1), + ("b", c_uint64, 62), + ("c", c_uint64, 1)] + +for n in 8, 16, 32, 64: + for signedness in '', 'u': + ctype = globals()[f'c_{signedness}int{n}'] + + @register(f'Struct331_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, 3), + ("c", ctype, 1)] + + @register(f'Struct1x1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct1nx1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("full", ctype), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct3xx_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, n-2), + ("c", ctype, n-2)] + +@register() +class Mixed1(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 4)] + +@register() +class Mixed2(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int32, 32)] + +@register() +class Mixed3(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + +@register() +class Mixed4(Structure): + _fields_ = [("a", c_short, 4), + ("b", c_short, 4), + ("c", c_int, 24), + ("d", c_short, 4), + ("e", c_short, 4), + ("f", c_int, 24)] + +@register() +class Mixed5(Structure): + _fields_ = [('A', c_uint, 1), + ('B', c_ushort, 16)] + +@register() +class Mixed6(Structure): + _fields_ = [('A', c_ulonglong, 1), + ('B', c_uint, 32)] + +@register() +class Mixed7(Structure): + _fields_ = [("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + +@register() +class Mixed8_a(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed8_b(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed9(Structure): + _fields_ = [("A", c_uint8), + ("B", c_uint32, 1)] + +@register() +class Mixed10(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_95496(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_84039_bad(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_84039_good_a(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1)] + +@register() +class Example_gh_84039_good(Structure): + _pack_ = 1 + _fields_ = [("a", Example_gh_84039_good_a), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_73939(Structure): + _pack_ = 1 + _fields_ = [("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2)] + +@register() +class Example_gh_86098(Structure): + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class Example_gh_86098_pack(Structure): + _pack_ = 1 + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class AnonBitfields(Structure): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + _anonymous_ = ["_"] + _fields_ = [("_", X), ('y', c_byte)] + + +class GeneratedTest(unittest.TestCase): + def test_generated_data(self): + """Check that a ctypes struct/union matches its C equivalent. + + This compares with data from get_generated_test_data(), a list of: + - name (str) + - size (int) + - alignment (int) + - for each field, three snapshots of memory, as bytes: + - memory after the field is set to -1 + - memory after the field is set to 1 + - memory after the field is set to 0 + + or: + - None + - reason to skip the test (str) + + This does depend on the C compiler keeping padding bits zero. + Common compilers seem to do so. + """ + for name, cls in TESTCASES.items(): + with self.subTest(name=name): + if _maybe_skip := getattr(cls, '_maybe_skip', None): + _maybe_skip() + expected = iter(_ctypes_test.get_generated_test_data(name)) + expected_name = next(expected) + if expected_name is None: + self.skipTest(next(expected)) + self.assertEqual(name, expected_name) + self.assertEqual(sizeof(cls), next(expected)) + with self.subTest('alignment'): + self.assertEqual(alignment(cls), next(expected)) + obj = cls() + ptr = pointer(obj) + for field in iterfields(cls): + for value in -1, 1, 0: + with self.subTest(field=field.full_name, value=value): + field.set_to(obj, value) + py_mem = string_at(ptr, sizeof(obj)) + c_mem = next(expected) + if py_mem != c_mem: + # Generate a helpful failure message + lines, requires = dump_ctype(cls) + m = "\n".join([str(field), 'in:', *lines]) + self.assertEqual(py_mem.hex(), c_mem.hex(), m) + + +# The rest of this file is generating C code from a ctypes type. +# This is only meant for (and tested with) the known inputs in this file! + +def c_str_repr(string): + """Return a string as a C literal""" + return '"' + re.sub('([\"\'\\\\\n])', r'\\\1', string) + '"' + +def dump_simple_ctype(tp, variable_name='', semi=''): + """Get C type name or declaration of a scalar type + + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + length = getattr(tp, '_length_', None) + if length is not None: + return f'{dump_simple_ctype(tp._type_, variable_name)}[{length}]{semi}' + assert not issubclass(tp, (Structure, Union)) + return f'{tp._c_name}{maybe_space(variable_name)}{semi}' + + +def dump_ctype(tp, struct_or_union_tag='', variable_name='', semi=''): + """Get C type name or declaration of a ctype + + struct_or_union_tag: name of the struct or union + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + requires = set() + if issubclass(tp, (Structure, Union)): + attributes = [] + pushes = [] + pops = [] + pack = getattr(tp, '_pack_', None) + if pack is not None: + pushes.append(f'#pragma pack(push, {pack})') + pops.append(f'#pragma pack(pop)') + layout = getattr(tp, '_layout_', None) + if layout == 'ms' or pack: + # The 'ms_struct' attribute only works on x86 and PowerPC + requires.add( + 'defined(MS_WIN32) || (' + '(defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (' + 'defined(__GNUC__) || defined(__clang__)))' + ) + attributes.append('ms_struct') + if attributes: + a = f' GCC_ATTR({", ".join(attributes)})' + else: + a = '' + lines = [f'{struct_or_union(tp)}{a}{maybe_space(struct_or_union_tag)} ' +'{'] + for fielddesc in tp._fields_: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + if f_name in getattr(tp, '_anonymous_', ()): + f_name = '' + if f_bits is None: + subsemi = ';' + else: + if f_tp not in (c_int, c_uint): + # XLC can reportedly only handle int & unsigned int + # bitfields (the only types required by C spec) + requires.add('!defined(__xlc__)') + subsemi = f' :{f_bits};' + sub_lines, sub_requires = dump_ctype( + f_tp, variable_name=f_name, semi=subsemi) + requires.update(sub_requires) + for line in sub_lines: + lines.append(' ' + line) + lines.append(f'}}{maybe_space(variable_name)}{semi}') + return [*pushes, *lines, *reversed(pops)], requires + else: + return [dump_simple_ctype(tp, variable_name, semi)], requires + +def struct_or_union(cls): + if issubclass(cls, Structure): + return 'struct' + if issubclass(cls, Union): + return 'union' + raise TypeError(cls) + +def maybe_space(string): + if string: + return ' ' + string + return string + +def unpack_field_desc(f_name, f_tp, f_bits=None): + """Unpack a _fields_ entry into a (name, type, bits) triple""" + return f_name, f_tp, f_bits + +@dataclass +class FieldInfo: + """Information about a (possibly nested) struct/union field""" + name: str + tp: type + bits: int | None # number if this is a bit field + parent_type: type + parent: 'FieldInfo' #| None + + @cached_property + def attr_path(self): + """Attribute names to get at the value of this field""" + if self.name in getattr(self.parent_type, '_anonymous_', ()): + selfpath = () + else: + selfpath = (self.name,) + if self.parent: + return (*self.parent.attr_path, *selfpath) + else: + return selfpath + + @cached_property + def full_name(self): + """Attribute names to get at the value of this field""" + return '.'.join(self.attr_path) + + def set_to(self, obj, new): + """Set the field on a given Structure/Union instance""" + for attr_name in self.attr_path[:-1]: + obj = getattr(obj, attr_name) + setattr(obj, self.attr_path[-1], new) + + @cached_property + def root(self): + if self.parent is None: + return self + else: + return self.parent + + @cached_property + def descriptor(self): + return getattr(self.parent_type, self.name) + + def __repr__(self): + qname = f'{self.root.parent_type.__name__}.{self.full_name}' + try: + desc = self.descriptor + except AttributeError: + desc = '???' + return f'<{type(self).__name__} for {qname}: {desc}>' + +def iterfields(tp, parent=None): + """Get *leaf* fields of a structure or union, as FieldInfo""" + try: + fields = tp._fields_ + except AttributeError: + yield parent + else: + for fielddesc in fields: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + sub = FieldInfo(f_name, f_tp, f_bits, tp, parent) + yield from iterfields(f_tp, sub) + + +if __name__ == '__main__': + # Dump C source to stdout + def output(string): + print(re.compile(r'^ +$', re.MULTILINE).sub('', string).lstrip('\n')) + output(""" + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + + // Append VALUE to the result. + #define APPEND(ITEM) { \\ + PyObject *item = ITEM; \\ + if (!item) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + int rv = PyList_Append(result, item); \\ + Py_DECREF(item); \\ + if (rv < 0) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \\ + TYPE v = VAL; \\ + TARGET = v; \\ + APPEND(PyBytes_FromStringAndSize( \\ + (char*)&value, sizeof(value))); \\ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \\ + SET_AND_APPEND(TYPE, TARGET, -1) \\ + SET_AND_APPEND(TYPE, TARGET, 1) \\ + SET_AND_APPEND(TYPE, TARGET, 0) \\ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + """) + for name, cls in TESTCASES.items(): + output(""" + if (PyUnicode_CompareWithASCIIString(name, %s) == 0) { + """ % c_str_repr(name)) + lines, requires = dump_ctype(cls, struct_or_union_tag=name, semi=';') + if requires: + output(f""" + #if {" && ".join(f'({r})' for r in sorted(requires))} + """) + for line in lines: + output(' ' + line) + typename = f'{struct_or_union(cls)} {name}' + output(f""" + {typename} value = {{0}}; + APPEND(PyUnicode_FromString({c_str_repr(name)})); + APPEND(PyLong_FromLong(sizeof({typename}))); + APPEND(PyLong_FromLong(_Alignof({typename}))); + """.rstrip()) + for field in iterfields(cls): + f_tp = dump_simple_ctype(field.tp) + output(f"""\ + TEST_FIELD({f_tp}, value.{field.full_name}); + """.rstrip()) + if requires: + output(f""" + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + """) + output(""" + return result; + } + """) + + output(""" + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + """) diff --git a/Lib/test/test_ctypes/test_internals.py b/Lib/test/test_ctypes/test_internals.py index 94c9a86c2d06df..778da6573da975 100644 --- a/Lib/test/test_ctypes/test_internals.py +++ b/Lib/test/test_ctypes/test_internals.py @@ -28,7 +28,7 @@ def test_ints(self): self.assertEqual(ci._objects, None) def test_c_char_p(self): - s = b"Hello, World" + s = "Hello, World".encode("ascii") refcnt = sys.getrefcount(s) cs = c_char_p(s) self.assertEqual(refcnt + 1, sys.getrefcount(s)) diff --git a/Lib/test/test_ctypes/test_libc.py b/Lib/test/test_ctypes/test_libc.py index 09c76db0bd0b17..df7dbc0ae264a0 100644 --- a/Lib/test/test_ctypes/test_libc.py +++ b/Lib/test/test_ctypes/test_libc.py @@ -1,8 +1,10 @@ -import _ctypes_test +import ctypes import math import unittest from ctypes import (CDLL, CFUNCTYPE, POINTER, create_string_buffer, sizeof, c_void_p, c_char, c_int, c_double, c_size_t) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") lib = CDLL(_ctypes_test.__file__) @@ -20,6 +22,31 @@ def test_sqrt(self): self.assertEqual(lib.my_sqrt(4.0), 2.0) self.assertEqual(lib.my_sqrt(2.0), math.sqrt(2.0)) + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type and libffi >= 3.3.0") + def test_csqrt(self): + lib.my_csqrt.argtypes = ctypes.c_double_complex, + lib.my_csqrt.restype = ctypes.c_double_complex + self.assertEqual(lib.my_csqrt(4), 2+0j) + self.assertAlmostEqual(lib.my_csqrt(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrt(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtf.argtypes = ctypes.c_float_complex, + lib.my_csqrtf.restype = ctypes.c_float_complex + self.assertAlmostEqual(lib.my_csqrtf(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtf(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtl.argtypes = ctypes.c_longdouble_complex, + lib.my_csqrtl.restype = ctypes.c_longdouble_complex + self.assertAlmostEqual(lib.my_csqrtl(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtl(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + def test_qsort(self): comparefunc = CFUNCTYPE(c_int, POINTER(c_char), POINTER(c_char)) lib.my_qsort.argtypes = c_void_p, c_size_t, c_size_t, comparefunc diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 59d7f51935f3cd..fc1eecb77e17e3 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -1,5 +1,4 @@ import _ctypes -import _ctypes_test import ctypes import os import shutil @@ -10,6 +9,7 @@ from ctypes import CDLL, cdll, addressof, c_void_p, c_char_p from ctypes.util import find_library from test.support import import_helper, os_helper +_ctypes_test = import_helper.import_module("_ctypes_test") libc_name = None @@ -42,10 +42,7 @@ def test_load(self): self.skipTest('could not find library to load') CDLL(test_lib) CDLL(os.path.basename(test_lib)) - class CTypesTestPathLikeCls: - def __fspath__(self): - return test_lib - CDLL(CTypesTestPathLikeCls()) + CDLL(os_helper.FakePath(test_lib)) self.assertRaises(OSError, CDLL, self.unknowndll) def test_load_version(self): @@ -59,11 +56,15 @@ def test_load_version(self): self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) def test_find(self): + found = False for name in ("c", "m"): lib = find_library(name) if lib: + found = True cdll.LoadLibrary(lib) CDLL(lib) + if not found: + self.skipTest("Could not find c and m libraries") @unittest.skipUnless(os.name == "nt", 'test specific to Windows') diff --git a/Lib/test/test_ctypes/test_numbers.py b/Lib/test/test_ctypes/test_numbers.py index 29108a28ec16e1..c57c58eb002328 100644 --- a/Lib/test/test_ctypes/test_numbers.py +++ b/Lib/test/test_ctypes/test_numbers.py @@ -1,12 +1,15 @@ import array +import ctypes import struct import sys import unittest +from itertools import combinations from operator import truth from ctypes import (byref, sizeof, alignment, c_char, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble, c_bool) +from test.support.testcase import ComplexesAreIdenticalMixin def valid_ranges(*types): @@ -38,8 +41,28 @@ def valid_ranges(*types): signed_ranges = valid_ranges(*signed_types) bool_values = [True, False, 0, 1, -1, 5000, 'test', [], [1]] +class IntLike: + def __int__(self): + return 2 -class NumberTestCase(unittest.TestCase): +class IndexLike: + def __index__(self): + return 2 + +class FloatLike: + def __float__(self): + return 2.0 + +class ComplexLike: + def __complex__(self): + return 1+1j + + +INF = float("inf") +NAN = float("nan") + + +class NumberTestCase(unittest.TestCase, ComplexesAreIdenticalMixin): def test_default_init(self): # default values are set to zero @@ -86,9 +109,6 @@ def test_byref(self): def test_floats(self): # c_float and c_double can be created from # Python int and float - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() for t in float_types: self.assertEqual(t(2.0).value, 2.0) @@ -96,18 +116,34 @@ def __float__(self): self.assertEqual(t(2).value, 2.0) self.assertEqual(t(f).value, 2.0) + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex(self): + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + self.assertEqual(t(1).value, 1+0j) + self.assertEqual(t(1.0).value, 1+0j) + self.assertEqual(t(1+0.125j).value, 1+0.125j) + self.assertEqual(t(IndexLike()).value, 2+0j) + self.assertEqual(t(FloatLike()).value, 2+0j) + self.assertEqual(t(ComplexLike()).value, 1+1j) + + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex_round_trip(self): + # Ensure complexes transformed exactly. The CMPLX macro should + # preserve special components (like inf/nan or signed zero). + values = [complex(*_) for _ in combinations([1, -1, 0.0, -0.0, 2, + -3, INF, -INF, NAN], 2)] + for z in values: + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + with self.subTest(z=z, type=t): + self.assertComplexesAreIdentical(z, t(z).value) + def test_integers(self): - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() - class IntLike: - def __int__(self): - return 2 d = IntLike() - class IndexLike: - def __index__(self): - return 2 i = IndexLike() # integers cannot be constructed from floats, # but from integer-like objects diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index d1eeee6b0306fe..f89521cf8b3a67 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,10 +1,9 @@ -import _ctypes_test import unittest import test.support from ctypes import (CDLL, PyDLL, ArgumentError, Structure, Array, Union, _Pointer, _SimpleCData, _CFuncPtr, - POINTER, pointer, byref, + POINTER, pointer, byref, sizeof, c_void_p, c_char_p, c_wchar_p, py_object, c_bool, c_char, c_wchar, @@ -14,6 +13,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class SimpleTypesTestCase(unittest.TestCase): @@ -86,19 +87,33 @@ def test_c_char(self): with self.assertRaises(TypeError) as cm: c_char.from_param(b"abc") self.assertEqual(str(cm.exception), - "one character bytes, bytearray or integer expected") + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length 3") def test_c_wchar(self): with self.assertRaises(TypeError) as cm: c_wchar.from_param("abc") self.assertEqual(str(cm.exception), - "one character unicode string expected") + "a unicode character expected, not a string of length 3") + with self.assertRaises(TypeError) as cm: + c_wchar.from_param("") + self.assertEqual(str(cm.exception), + "a unicode character expected, not a string of length 0") with self.assertRaises(TypeError) as cm: c_wchar.from_param(123) self.assertEqual(str(cm.exception), - "unicode string expected instead of int instance") + "a unicode character expected, not instance of int") + + if sizeof(c_wchar) < 4: + with self.assertRaises(TypeError) as cm: + c_wchar.from_param('\U0001f40d') + self.assertEqual(str(cm.exception), + "the string '\\U0001f40d' cannot be converted to " + "a single wchar_t character") + + def test_int_pointers(self): LPINT = POINTER(c_int) diff --git a/Lib/test/test_ctypes/test_pickling.py b/Lib/test/test_ctypes/test_pickling.py index 0ca42a68f0675f..9d433fc69de391 100644 --- a/Lib/test/test_ctypes/test_pickling.py +++ b/Lib/test/test_ctypes/test_pickling.py @@ -1,9 +1,10 @@ -import _ctypes_test import pickle import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, pointer, c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, c_int, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 8cf2114c282cab..fc558e10ba40c5 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -1,4 +1,3 @@ -import _ctypes_test import array import ctypes import sys @@ -10,6 +9,8 @@ c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCPointerType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -33,6 +34,11 @@ def test_type_flags(self): self.assertTrue(_Pointer.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(_Pointer.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCPointerType.__init__(POINTER(c_byte), 'ptr', (), {}) + def test_pointer_crash(self): class A(POINTER(c_ulong)): diff --git a/Lib/test/test_ctypes/test_prototypes.py b/Lib/test/test_ctypes/test_prototypes.py index 81eb4562c740fd..63ae799ea86ab2 100644 --- a/Lib/test/test_ctypes/test_prototypes.py +++ b/Lib/test/test_ctypes/test_prototypes.py @@ -18,12 +18,13 @@ # # In this case, there would have to be an additional reference to the argument... -import _ctypes_test import unittest from ctypes import (CDLL, CFUNCTYPE, POINTER, ArgumentError, pointer, byref, sizeof, addressof, create_string_buffer, c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, c_short, c_int, c_long, c_longlong, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") testdll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py index 77da35855928a4..1072a109833261 100644 --- a/Lib/test/test_ctypes/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -47,7 +47,7 @@ def test_PyLong_Long(self): @support.refcount_test def test_PyObj_FromPtr(self): - s = "abc def ghi jkl" + s = object() ref = sys.getrefcount(s) # id(python-object) is the address pyobj = _ctypes.PyObj_FromPtr(id(s)) diff --git a/Lib/test/test_ctypes/test_refcounts.py b/Lib/test/test_ctypes/test_refcounts.py index a90588ca9bb1b6..012722d8486218 100644 --- a/Lib/test/test_ctypes/test_refcounts.py +++ b/Lib/test/test_ctypes/test_refcounts.py @@ -1,9 +1,11 @@ -import _ctypes_test import ctypes import gc import sys import unittest from test import support +from test.support import import_helper +from test.support import script_helper +_ctypes_test = import_helper.import_module("_ctypes_test") MyCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int) @@ -109,5 +111,18 @@ def func(): func() +class ModuleIsolationTest(unittest.TestCase): + def test_finalize(self): + # check if gc_decref() succeeds + script = ( + "import ctypes;" + "import sys;" + "del sys.modules['_ctypes'];" + "import _ctypes;" + "exit()" + ) + script_helper.assert_python_ok("-c", script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_returnfuncptrs.py b/Lib/test/test_ctypes/test_returnfuncptrs.py index 4010e511e75ade..337801b226ab06 100644 --- a/Lib/test/test_ctypes/test_returnfuncptrs.py +++ b/Lib/test/test_ctypes/test_returnfuncptrs.py @@ -1,6 +1,7 @@ -import _ctypes_test import unittest from ctypes import CDLL, CFUNCTYPE, ArgumentError, c_char_p, c_void_p, c_char +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class ReturnFuncPtrTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_simplesubclasses.py b/Lib/test/test_ctypes/test_simplesubclasses.py index c96798e67f23f7..4e4bef3690f66a 100644 --- a/Lib/test/test_ctypes/test_simplesubclasses.py +++ b/Lib/test/test_ctypes/test_simplesubclasses.py @@ -26,6 +26,29 @@ def test_type_flags(self): self.assertTrue(_SimpleCData.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(_SimpleCData.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewT = PyCSimpleType.__new__(PyCSimpleType, 'NewT', (_SimpleCData,), {}) + for cls in _SimpleCData, NewT: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(_SimpleCData): + _type_ = "i" + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCSimpleType.__init__(T, 'ptr', (), {}) + + def test_swapped_type_creation(self): + cls = PyCSimpleType.__new__(PyCSimpleType, '', (), {'_type_': 'i'}) + with self.assertRaises(TypeError): + PyCSimpleType.__init__(cls) + PyCSimpleType.__init__(cls, '', (), {'_type_': 'i'}) + self.assertEqual(cls.__ctype_le__.__dict__.get('_type_'), 'i') + self.assertEqual(cls.__ctype_be__.__dict__.get('_type_'), 'i') + def test_compare(self): self.assertEqual(MyInt(3), MyInt(3)) self.assertNotEqual(MyInt(42), MyInt(43)) diff --git a/Lib/test/test_ctypes/test_slicing.py b/Lib/test/test_ctypes/test_slicing.py index a592d911cbe6ca..66f9e530104cac 100644 --- a/Lib/test/test_ctypes/test_slicing.py +++ b/Lib/test/test_ctypes/test_slicing.py @@ -1,7 +1,8 @@ -import _ctypes_test import unittest from ctypes import (CDLL, POINTER, sizeof, c_byte, c_short, c_int, c_long, c_char, c_wchar, c_char_p) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class SlicesTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_stringptr.py b/Lib/test/test_ctypes/test_stringptr.py index 67c61c6c3e17e6..bb6045b250ffce 100644 --- a/Lib/test/test_ctypes/test_stringptr.py +++ b/Lib/test/test_ctypes/test_stringptr.py @@ -1,9 +1,10 @@ -import _ctypes_test import sys import unittest from test import support from ctypes import (CDLL, Structure, POINTER, create_string_buffer, c_char, c_char_p) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") lib = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index f474a02fa8db06..b5e165f3bae929 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -4,7 +4,9 @@ Py_TPFLAGS_IMMUTABLETYPE) -class StructFieldsTestCase(unittest.TestCase): +NOTHING = object() + +class FieldsTestBase: # Structure/Union classes must get 'finalized' sooner or # later, when one of these things happen: # @@ -14,42 +16,47 @@ class StructFieldsTestCase(unittest.TestCase): # 4. The type is subclassed # # When they are finalized, assigning _fields_ is no longer allowed. + + def assert_final_fields(self, cls, expected=NOTHING): + self.assertRaises(AttributeError, setattr, cls, "_fields_", []) + self.assertEqual(getattr(cls, "_fields_", NOTHING), expected) + def test_1_A(self): - class X(Structure): + class X(self.cls): pass self.assertEqual(sizeof(X), 0) # not finalized X._fields_ = [] # finalized - self.assertRaises(AttributeError, setattr, X, "_fields_", []) + self.assert_final_fields(X, expected=[]) def test_1_B(self): - class X(Structure): + class X(self.cls): _fields_ = [] # finalized - self.assertRaises(AttributeError, setattr, X, "_fields_", []) + self.assert_final_fields(X, expected=[]) def test_2(self): - class X(Structure): + class X(self.cls): pass X() - self.assertRaises(AttributeError, setattr, X, "_fields_", []) + self.assert_final_fields(X) def test_3(self): - class X(Structure): + class X(self.cls): pass - class Y(Structure): + class Y(self.cls): _fields_ = [("x", X)] # finalizes X - self.assertRaises(AttributeError, setattr, X, "_fields_", []) + self.assert_final_fields(X) def test_4(self): - class X(Structure): + class X(self.cls): pass class Y(X): pass - self.assertRaises(AttributeError, setattr, X, "_fields_", []) + self.assert_final_fields(X) Y._fields_ = [] - self.assertRaises(AttributeError, setattr, X, "_fields_", []) + self.assert_final_fields(X) def test_5(self): - class X(Structure): + class X(self.cls): _fields_ = (("char", c_char * 5),) x = X(b'#' * 5) @@ -59,17 +66,10 @@ class X(Structure): def test_6(self): self.assertRaises(TypeError, CField) - def test_cfield_type_flags(self): - self.assertTrue(CField.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) - self.assertTrue(CField.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) - - def test_cfield_inheritance_hierarchy(self): - self.assertEqual(CField.mro(), [CField, object]) - def test_gh99275(self): - class BrokenStructure(Structure): + class BrokenStructure(self.cls): def __init_subclass__(cls, **kwargs): - cls._fields_ = [] # This line will fail, `stgdict` is not ready + cls._fields_ = [] # This line will fail, `stginfo` is not ready with self.assertRaisesRegex(TypeError, 'ctypes state is not initialized'): @@ -78,26 +78,28 @@ class Subclass(BrokenStructure): ... # __set__ and __get__ should raise a TypeError in case their self # argument is not a ctype instance. def test___set__(self): - class MyCStruct(Structure): + class MyCStruct(self.cls): _fields_ = (("field", c_int),) self.assertRaises(TypeError, MyCStruct.field.__set__, 'wrong type self', 42) - class MyCUnion(Union): - _fields_ = (("field", c_int),) - self.assertRaises(TypeError, - MyCUnion.field.__set__, 'wrong type self', 42) - def test___get__(self): - class MyCStruct(Structure): + class MyCStruct(self.cls): _fields_ = (("field", c_int),) self.assertRaises(TypeError, MyCStruct.field.__get__, 'wrong type self', 42) - class MyCUnion(Union): - _fields_ = (("field", c_int),) - self.assertRaises(TypeError, - MyCUnion.field.__get__, 'wrong type self', 42) +class StructFieldsTestCase(unittest.TestCase, FieldsTestBase): + cls = Structure + + def test_cfield_type_flags(self): + self.assertTrue(CField.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + + def test_cfield_inheritance_hierarchy(self): + self.assertEqual(CField.mro(), [CField, object]) + +class UnionFieldsTestCase(unittest.TestCase, FieldsTestBase): + cls = Union if __name__ == "__main__": diff --git a/Lib/test/test_ctypes/test_structunion.py b/Lib/test/test_ctypes/test_structunion.py new file mode 100644 index 00000000000000..973ac3b2f1919d --- /dev/null +++ b/Lib/test/test_ctypes/test_structunion.py @@ -0,0 +1,353 @@ +"""Common tests for ctypes.Structure and ctypes.Union""" + +import unittest +from ctypes import (Structure, Union, POINTER, sizeof, alignment, + c_char, c_byte, c_ubyte, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from ._support import (_CData, PyCStructType, UnionType, + Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) +from struct import calcsize + + +class StructUnionTestBase: + formats = {"c": c_char, + "b": c_byte, + "B": c_ubyte, + "h": c_short, + "H": c_ushort, + "i": c_int, + "I": c_uint, + "l": c_long, + "L": c_ulong, + "q": c_longlong, + "Q": c_ulonglong, + "f": c_float, + "d": c_double, + } + + def test_subclass(self): + class X(self.cls): + _fields_ = [("a", c_int)] + + class Y(X): + _fields_ = [("b", c_int)] + + class Z(X): + pass + + self.assertEqual(sizeof(X), sizeof(c_int)) + self.check_sizeof(Y, + struct_size=sizeof(c_int)*2, + union_size=sizeof(c_int)) + self.assertEqual(sizeof(Z), sizeof(c_int)) + self.assertEqual(X._fields_, [("a", c_int)]) + self.assertEqual(Y._fields_, [("b", c_int)]) + self.assertEqual(Z._fields_, [("a", c_int)]) + + def test_subclass_delayed(self): + class X(self.cls): + pass + self.assertEqual(sizeof(X), 0) + X._fields_ = [("a", c_int)] + + class Y(X): + pass + self.assertEqual(sizeof(Y), sizeof(X)) + Y._fields_ = [("b", c_int)] + + class Z(X): + pass + + self.assertEqual(sizeof(X), sizeof(c_int)) + self.check_sizeof(Y, + struct_size=sizeof(c_int)*2, + union_size=sizeof(c_int)) + self.assertEqual(sizeof(Z), sizeof(c_int)) + self.assertEqual(X._fields_, [("a", c_int)]) + self.assertEqual(Y._fields_, [("b", c_int)]) + self.assertEqual(Z._fields_, [("a", c_int)]) + + def test_inheritance_hierarchy(self): + self.assertEqual(self.cls.mro(), [self.cls, _CData, object]) + self.assertEqual(type(self.metacls), type) + + def test_type_flags(self): + for cls in self.cls, self.metacls: + with self.subTest(cls=cls): + self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewClass = self.metacls.__new__(self.metacls, 'NewClass', + (self.cls,), {}) + for cls in self.cls, NewClass: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(self.cls): + _fields_ = [("x", c_char), + ("y", c_char)] + with self.assertRaisesRegex(SystemError, "already initialized"): + self.metacls.__init__(T, 'ptr', (), {}) + + def test_alignment(self): + class X(self.cls): + _fields_ = [("x", c_char * 3)] + self.assertEqual(alignment(X), calcsize("s")) + self.assertEqual(sizeof(X), calcsize("3s")) + + class Y(self.cls): + _fields_ = [("x", c_char * 3), + ("y", c_int)] + self.assertEqual(alignment(Y), alignment(c_int)) + self.check_sizeof(Y, + struct_size=calcsize("3s i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class SI(self.cls): + _fields_ = [("a", X), + ("b", Y)] + self.assertEqual(alignment(SI), max(alignment(Y), alignment(X))) + self.check_sizeof(SI, + struct_size=calcsize("3s0i 3si 0i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class IS(self.cls): + _fields_ = [("b", Y), + ("a", X)] + + self.assertEqual(alignment(SI), max(alignment(X), alignment(Y))) + self.check_sizeof(IS, + struct_size=calcsize("3si 3s 0i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class XX(self.cls): + _fields_ = [("a", X), + ("b", X)] + self.assertEqual(alignment(XX), alignment(X)) + self.check_sizeof(XX, + struct_size=calcsize("3s 3s 0s"), + union_size=calcsize("3s")) + + def test_empty(self): + # I had problems with these + # + # Although these are pathological cases: Empty Structures! + class X(self.cls): + _fields_ = [] + + # Is this really the correct alignment, or should it be 0? + self.assertTrue(alignment(X) == 1) + self.assertTrue(sizeof(X) == 0) + + class XX(self.cls): + _fields_ = [("a", X), + ("b", X)] + + self.assertEqual(alignment(XX), 1) + self.assertEqual(sizeof(XX), 0) + + def test_fields(self): + # test the offset and size attributes of Structure/Union fields. + class X(self.cls): + _fields_ = [("x", c_int), + ("y", c_char)] + + self.assertEqual(X.x.offset, 0) + self.assertEqual(X.x.size, sizeof(c_int)) + + if self.cls == Structure: + self.assertEqual(X.y.offset, sizeof(c_int)) + else: + self.assertEqual(X.y.offset, 0) + self.assertEqual(X.y.size, sizeof(c_char)) + + # readonly + self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) + self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) + + # XXX Should we check nested data types also? + # offset is always relative to the class... + + def test_invalid_field_types(self): + class POINT(self.cls): + pass + self.assertRaises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)]) + + def test_invalid_name(self): + # field name must be string + def declare_with_name(name): + class S(self.cls): + _fields_ = [(name, c_int)] + + self.assertRaises(TypeError, declare_with_name, b"x") + + def test_intarray_fields(self): + class SomeInts(self.cls): + _fields_ = [("a", c_int * 4)] + + # can use tuple to initialize array (but not list!) + self.assertEqual(SomeInts((1, 2)).a[:], [1, 2, 0, 0]) + self.assertEqual(SomeInts((1, 2)).a[::], [1, 2, 0, 0]) + self.assertEqual(SomeInts((1, 2)).a[::-1], [0, 0, 2, 1]) + self.assertEqual(SomeInts((1, 2)).a[::2], [1, 0]) + self.assertEqual(SomeInts((1, 2)).a[1:5:6], [2]) + self.assertEqual(SomeInts((1, 2)).a[6:4:-1], []) + self.assertEqual(SomeInts((1, 2, 3, 4)).a[:], [1, 2, 3, 4]) + self.assertEqual(SomeInts((1, 2, 3, 4)).a[::], [1, 2, 3, 4]) + # too long + # XXX Should raise ValueError?, not RuntimeError + self.assertRaises(RuntimeError, SomeInts, (1, 2, 3, 4, 5)) + + def test_huge_field_name(self): + # issue12881: segfault with large structure field names + def create_class(length): + class S(self.cls): + _fields_ = [('x' * length, c_int)] + + for length in [10 ** i for i in range(0, 8)]: + try: + create_class(length) + except MemoryError: + # MemoryErrors are OK, we just don't want to segfault + pass + + def test_abstract_class(self): + class X(self.cls): + _abstract_ = "something" + with self.assertRaisesRegex(TypeError, r"^abstract class$"): + X() + + def test_methods(self): + self.assertIn("in_dll", dir(type(self.cls))) + self.assertIn("from_address", dir(type(self.cls))) + self.assertIn("in_dll", dir(type(self.cls))) + + +class StructureTestCase(unittest.TestCase, StructUnionTestBase): + cls = Structure + metacls = PyCStructType + + def test_metaclass_name(self): + self.assertEqual(self.metacls.__name__, "PyCStructType") + + def check_sizeof(self, cls, *, struct_size, union_size): + self.assertEqual(sizeof(cls), struct_size) + + def test_simple_structs(self): + for code, tp in self.formats.items(): + class X(Structure): + _fields_ = [("x", c_char), + ("y", tp)] + self.assertEqual((sizeof(X), code), + (calcsize("c%c0%c" % (code, code)), code)) + + +class UnionTestCase(unittest.TestCase, StructUnionTestBase): + cls = Union + metacls = UnionType + + def test_metaclass_name(self): + self.assertEqual(self.metacls.__name__, "UnionType") + + def check_sizeof(self, cls, *, struct_size, union_size): + self.assertEqual(sizeof(cls), union_size) + + def test_simple_unions(self): + for code, tp in self.formats.items(): + class X(Union): + _fields_ = [("x", c_char), + ("y", tp)] + self.assertEqual((sizeof(X), code), + (calcsize("%c" % (code)), code)) + + +class PointerMemberTestBase: + def test(self): + # a Structure/Union with a POINTER field + class S(self.cls): + _fields_ = [("array", POINTER(c_int))] + + s = S() + # We can assign arrays of the correct type + s.array = (c_int * 3)(1, 2, 3) + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [1, 2, 3]) + + s.array[0] = 42 + + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [42, 2, 3]) + + s.array[0] = 1 + + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [1, 2, 3]) + +class PointerMemberTestCase_Struct(unittest.TestCase, PointerMemberTestBase): + cls = Structure + + def test_none_to_pointer_fields(self): + class S(self.cls): + _fields_ = [("x", c_int), + ("p", POINTER(c_int))] + + s = S() + s.x = 12345678 + s.p = None + self.assertEqual(s.x, 12345678) + +class PointerMemberTestCase_Union(unittest.TestCase, PointerMemberTestBase): + cls = Union + + def test_none_to_pointer_fields(self): + class S(self.cls): + _fields_ = [("x", c_int), + ("p", POINTER(c_int))] + + s = S() + s.x = 12345678 + s.p = None + self.assertFalse(s.p) # NULL pointers are falsy + + +class TestRecursiveBase: + def test_contains_itself(self): + class Recursive(self.cls): + pass + + try: + Recursive._fields_ = [("next", Recursive)] + except AttributeError as details: + self.assertIn("Structure or union cannot contain itself", + str(details)) + else: + self.fail("Structure or union cannot contain itself") + + + def test_vice_versa(self): + class First(self.cls): + pass + class Second(self.cls): + pass + + First._fields_ = [("second", Second)] + + try: + Second._fields_ = [("first", First)] + except AttributeError as details: + self.assertIn("_fields_ is final", str(details)) + else: + self.fail("AttributeError not raised") + +class TestRecursiveStructure(unittest.TestCase, TestRecursiveBase): + cls = Structure + +class TestRecursiveUnion(unittest.TestCase, TestRecursiveBase): + cls = Union diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 98bc4bdcac9306..0ec238e04b74cd 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -1,191 +1,24 @@ -import _ctypes_test +"""Tests for ctypes.Structure + +Features common with Union should go in test_structunion.py instead. +""" + from platform import architecture as _architecture import struct import sys import unittest -from ctypes import (CDLL, Array, Structure, Union, POINTER, sizeof, byref, alignment, +from ctypes import (CDLL, Structure, Union, POINTER, sizeof, byref, c_void_p, c_char, c_wchar, c_byte, c_ubyte, - c_uint8, c_uint16, c_uint32, - c_short, c_ushort, c_int, c_uint, - c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) + c_uint8, c_uint16, c_uint32, c_int, c_uint, + c_long, c_ulong, c_longlong, c_float, c_double) from ctypes.util import find_library -from struct import calcsize from collections import namedtuple from test import support -from ._support import (_CData, PyCStructType, Py_TPFLAGS_DISALLOW_INSTANTIATION, - Py_TPFLAGS_IMMUTABLETYPE) - - -class SubclassesTest(unittest.TestCase): - def test_subclass(self): - class X(Structure): - _fields_ = [("a", c_int)] - - class Y(X): - _fields_ = [("b", c_int)] - - class Z(X): - pass - - self.assertEqual(sizeof(X), sizeof(c_int)) - self.assertEqual(sizeof(Y), sizeof(c_int)*2) - self.assertEqual(sizeof(Z), sizeof(c_int)) - self.assertEqual(X._fields_, [("a", c_int)]) - self.assertEqual(Y._fields_, [("b", c_int)]) - self.assertEqual(Z._fields_, [("a", c_int)]) - - def test_subclass_delayed(self): - class X(Structure): - pass - self.assertEqual(sizeof(X), 0) - X._fields_ = [("a", c_int)] - - class Y(X): - pass - self.assertEqual(sizeof(Y), sizeof(X)) - Y._fields_ = [("b", c_int)] - - class Z(X): - pass - - self.assertEqual(sizeof(X), sizeof(c_int)) - self.assertEqual(sizeof(Y), sizeof(c_int)*2) - self.assertEqual(sizeof(Z), sizeof(c_int)) - self.assertEqual(X._fields_, [("a", c_int)]) - self.assertEqual(Y._fields_, [("b", c_int)]) - self.assertEqual(Z._fields_, [("a", c_int)]) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class StructureTestCase(unittest.TestCase): - formats = {"c": c_char, - "b": c_byte, - "B": c_ubyte, - "h": c_short, - "H": c_ushort, - "i": c_int, - "I": c_uint, - "l": c_long, - "L": c_ulong, - "q": c_longlong, - "Q": c_ulonglong, - "f": c_float, - "d": c_double, - } - - def test_inheritance_hierarchy(self): - self.assertEqual(Structure.mro(), [Structure, _CData, object]) - - self.assertEqual(PyCStructType.__name__, "PyCStructType") - self.assertEqual(type(PyCStructType), type) - - - def test_type_flags(self): - for cls in Structure, PyCStructType: - with self.subTest(cls=cls): - self.assertTrue(Structure.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) - self.assertFalse(Structure.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) - - def test_simple_structs(self): - for code, tp in self.formats.items(): - class X(Structure): - _fields_ = [("x", c_char), - ("y", tp)] - self.assertEqual((sizeof(X), code), - (calcsize("c%c0%c" % (code, code)), code)) - - def test_unions(self): - for code, tp in self.formats.items(): - class X(Union): - _fields_ = [("x", c_char), - ("y", tp)] - self.assertEqual((sizeof(X), code), - (calcsize("%c" % (code)), code)) - - def test_struct_alignment(self): - class X(Structure): - _fields_ = [("x", c_char * 3)] - self.assertEqual(alignment(X), calcsize("s")) - self.assertEqual(sizeof(X), calcsize("3s")) - - class Y(Structure): - _fields_ = [("x", c_char * 3), - ("y", c_int)] - self.assertEqual(alignment(Y), alignment(c_int)) - self.assertEqual(sizeof(Y), calcsize("3si")) - - class SI(Structure): - _fields_ = [("a", X), - ("b", Y)] - self.assertEqual(alignment(SI), max(alignment(Y), alignment(X))) - self.assertEqual(sizeof(SI), calcsize("3s0i 3si 0i")) - - class IS(Structure): - _fields_ = [("b", Y), - ("a", X)] - - self.assertEqual(alignment(SI), max(alignment(X), alignment(Y))) - self.assertEqual(sizeof(IS), calcsize("3si 3s 0i")) - - class XX(Structure): - _fields_ = [("a", X), - ("b", X)] - self.assertEqual(alignment(XX), alignment(X)) - self.assertEqual(sizeof(XX), calcsize("3s 3s 0s")) - - def test_empty(self): - # I had problems with these - # - # Although these are pathological cases: Empty Structures! - class X(Structure): - _fields_ = [] - - class Y(Union): - _fields_ = [] - - # Is this really the correct alignment, or should it be 0? - self.assertTrue(alignment(X) == alignment(Y) == 1) - self.assertTrue(sizeof(X) == sizeof(Y) == 0) - - class XX(Structure): - _fields_ = [("a", X), - ("b", X)] - - self.assertEqual(alignment(XX), 1) - self.assertEqual(sizeof(XX), 0) - - def test_fields(self): - # test the offset and size attributes of Structure/Union fields. - class X(Structure): - _fields_ = [("x", c_int), - ("y", c_char)] - - self.assertEqual(X.x.offset, 0) - self.assertEqual(X.x.size, sizeof(c_int)) - - self.assertEqual(X.y.offset, sizeof(c_int)) - self.assertEqual(X.y.size, sizeof(c_char)) - - # readonly - self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) - self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) - - class X(Union): - _fields_ = [("x", c_int), - ("y", c_char)] - - self.assertEqual(X.x.offset, 0) - self.assertEqual(X.x.size, sizeof(c_int)) - - self.assertEqual(X.y.offset, 0) - self.assertEqual(X.y.size, sizeof(c_char)) - - # readonly - self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) - self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) - - # XXX Should we check nested data types also? - # offset is always relative to the class... - def test_packed(self): class X(Structure): _fields_ = [("a", c_byte), @@ -272,36 +105,6 @@ class POINT(Structure): pt = POINT(y=2, x=1) self.assertEqual((pt.x, pt.y), (1, 2)) - def test_invalid_field_types(self): - class POINT(Structure): - pass - self.assertRaises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)]) - - def test_invalid_name(self): - # field name must be string - def declare_with_name(name): - class S(Structure): - _fields_ = [(name, c_int)] - - self.assertRaises(TypeError, declare_with_name, b"x") - - def test_intarray_fields(self): - class SomeInts(Structure): - _fields_ = [("a", c_int * 4)] - - # can use tuple to initialize array (but not list!) - self.assertEqual(SomeInts((1, 2)).a[:], [1, 2, 0, 0]) - self.assertEqual(SomeInts((1, 2)).a[::], [1, 2, 0, 0]) - self.assertEqual(SomeInts((1, 2)).a[::-1], [0, 0, 2, 1]) - self.assertEqual(SomeInts((1, 2)).a[::2], [1, 0]) - self.assertEqual(SomeInts((1, 2)).a[1:5:6], [2]) - self.assertEqual(SomeInts((1, 2)).a[6:4:-1], []) - self.assertEqual(SomeInts((1, 2, 3, 4)).a[:], [1, 2, 3, 4]) - self.assertEqual(SomeInts((1, 2, 3, 4)).a[::], [1, 2, 3, 4]) - # too long - # XXX Should raise ValueError?, not RuntimeError - self.assertRaises(RuntimeError, SomeInts, (1, 2, 3, 4, 5)) - def test_nested_initializers(self): # test initializing nested structures class Phone(Structure): @@ -356,37 +159,12 @@ class Person(Structure): self.assertEqual(msg, "(Phone) TypeError: too many initializers") - def test_huge_field_name(self): - # issue12881: segfault with large structure field names - def create_class(length): - class S(Structure): - _fields_ = [('x' * length, c_int)] - - for length in [10 ** i for i in range(0, 8)]: - try: - create_class(length) - except MemoryError: - # MemoryErrors are OK, we just don't want to segfault - pass - def get_except(self, func, *args): try: func(*args) except Exception as detail: return detail.__class__, str(detail) - def test_abstract_class(self): - class X(Structure): - _abstract_ = "something" - # try 'X()' - cls, msg = self.get_except(eval, "X()", locals()) - self.assertEqual((cls, msg), (TypeError, "abstract class")) - - def test_methods(self): - self.assertIn("in_dll", dir(type(Structure))) - self.assertIn("from_address", dir(type(Structure))) - self.assertIn("in_dll", dir(type(Structure))) - def test_positional_args(self): # see also http://bugs.python.org/issue5042 class W(Structure): @@ -489,6 +267,8 @@ class X(Structure): self.assertEqual(s.second, got.second) def _test_issue18060(self, Vector): + # Regression tests for gh-62260 + # The call to atan2() should succeed if the # class fields were correctly cloned in the # subclasses. Otherwise, it will segfault. @@ -507,8 +287,8 @@ def _test_issue18060(self, Vector): @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") def test_issue18060_a(self): # This test case calls - # PyCStructUnionType_update_stgdict() for each - # _fields_ assignment, and PyCStgDict_clone() + # PyCStructUnionType_update_stginfo() for each + # _fields_ assignment, and PyCStgInfo_clone() # for the Mid and Vector class definitions. class Base(Structure): _fields_ = [('y', c_double), @@ -523,7 +303,7 @@ class Vector(Mid): pass @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") def test_issue18060_b(self): # This test case calls - # PyCStructUnionType_update_stgdict() for each + # PyCStructUnionType_update_stginfo() for each # _fields_ assignment. class Base(Structure): _fields_ = [('y', c_double), @@ -538,7 +318,7 @@ class Vector(Mid): @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") def test_issue18060_c(self): # This test case calls - # PyCStructUnionType_update_stgdict() for each + # PyCStructUnionType_update_stginfo() for each # _fields_ assignment. class Base(Structure): _fields_ = [('y', c_double)] @@ -680,6 +460,7 @@ class Test3E(Structure): self.assertEqual(result.data[i], float(i+1)) def test_38368(self): + # Regression test for gh-82549 class U(Union): _fields_ = [ ('f1', c_uint8 * 16), @@ -701,9 +482,9 @@ class U(Union): self.assertEqual(f2, [0x4567, 0x0123, 0xcdef, 0x89ab, 0x3210, 0x7654, 0xba98, 0xfedc]) - @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + @unittest.skipIf(True, 'Test disabled for now - see gh-60779/gh-60780') def test_union_by_value(self): - # See bpo-16575 + # See gh-60779 # These should mirror the structures in Modules/_ctypes/_ctypes_test.c @@ -782,9 +563,9 @@ class Test5(Structure): self.assertEqual(test5.nested.an_int, 0) self.assertEqual(test5.another_int, 0) - @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + @unittest.skipIf(True, 'Test disabled for now - see gh-60779/gh-60780') def test_bitfield_by_value(self): - # See bpo-16576 + # See gh-60780 # These should mirror the structures in Modules/_ctypes/_ctypes_test.c @@ -864,75 +645,5 @@ class Test8(Union): 'a union by value, which is unsupported.') -class PointerMemberTestCase(unittest.TestCase): - - def test(self): - # a Structure with a POINTER field - class S(Structure): - _fields_ = [("array", POINTER(c_int))] - - s = S() - # We can assign arrays of the correct type - s.array = (c_int * 3)(1, 2, 3) - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [1, 2, 3]) - - # The following are bugs, but are included here because the unittests - # also describe the current behaviour. - # - # This fails with SystemError: bad arg to internal function - # or with IndexError (with a patch I have) - - s.array[0] = 42 - - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [42, 2, 3]) - - s.array[0] = 1 - - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [1, 2, 3]) - - def test_none_to_pointer_fields(self): - class S(Structure): - _fields_ = [("x", c_int), - ("p", POINTER(c_int))] - - s = S() - s.x = 12345678 - s.p = None - self.assertEqual(s.x, 12345678) - - -class TestRecursiveStructure(unittest.TestCase): - def test_contains_itself(self): - class Recursive(Structure): - pass - - try: - Recursive._fields_ = [("next", Recursive)] - except AttributeError as details: - self.assertIn("Structure or union cannot contain itself", - str(details)) - else: - self.fail("Structure or union cannot contain itself") - - - def test_vice_versa(self): - class First(Structure): - pass - class Second(Structure): - pass - - First._fields_ = [("second", Second)] - - try: - Second._fields_ = [("first", First)] - except AttributeError as details: - self.assertIn("_fields_ is final", str(details)) - else: - self.fail("AttributeError not raised") - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_unicode.py b/Lib/test/test_ctypes/test_unicode.py index 2ddc7c56544e35..d9e17371d13572 100644 --- a/Lib/test/test_ctypes/test_unicode.py +++ b/Lib/test/test_ctypes/test_unicode.py @@ -1,6 +1,7 @@ -import _ctypes_test import ctypes import unittest +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class UnicodeTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_unions.py b/Lib/test/test_ctypes/test_unions.py deleted file mode 100644 index cf5344bdf19165..00000000000000 --- a/Lib/test/test_ctypes/test_unions.py +++ /dev/null @@ -1,18 +0,0 @@ -import unittest -from ctypes import Union -from ._support import (_CData, UnionType, Py_TPFLAGS_DISALLOW_INSTANTIATION, - Py_TPFLAGS_IMMUTABLETYPE) - - -class ArrayTestCase(unittest.TestCase): - def test_inheritance_hierarchy(self): - self.assertEqual(Union.mro(), [Union, _CData, object]) - - self.assertEqual(UnionType.__name__, "UnionType") - self.assertEqual(type(UnionType), type) - - def test_type_flags(self): - for cls in Union, UnionType: - with self.subTest(cls=Union): - self.assertTrue(Union.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) - self.assertFalse(Union.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) diff --git a/Lib/test/test_ctypes/test_values.py b/Lib/test/test_ctypes/test_values.py index d0b4803dff8529..1b757e020d5ce2 100644 --- a/Lib/test/test_ctypes/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -2,7 +2,6 @@ A testcase which accesses *values* in a dll. """ -import _ctypes_test import _imp import importlib.util import sys @@ -15,10 +14,14 @@ class ValuesTestCase(unittest.TestCase): + def setUp(self): + _ctypes_test = import_helper.import_module("_ctypes_test") + self.ctdll = CDLL(_ctypes_test.__file__) + def test_an_integer(self): # This test checks and changes an integer stored inside the # _ctypes_test dll/shared lib. - ctdll = CDLL(_ctypes_test.__file__) + ctdll = self.ctdll an_integer = c_int.in_dll(ctdll, "an_integer") x = an_integer.value self.assertEqual(x, ctdll.get_an_integer()) @@ -30,8 +33,7 @@ def test_an_integer(self): self.assertEqual(x, ctdll.get_an_integer()) def test_undefined(self): - ctdll = CDLL(_ctypes_test.__file__) - self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") + self.assertRaises(ValueError, c_int.in_dll, self.ctdll, "Undefined_Symbol") class PythonValuesTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_win32.py b/Lib/test/test_ctypes/test_win32.py index 4aaecd8d38f98f..31919118670613 100644 --- a/Lib/test/test_ctypes/test_win32.py +++ b/Lib/test/test_ctypes/test_win32.py @@ -1,6 +1,5 @@ # Windows specific tests -import _ctypes_test import ctypes import errno import sys @@ -9,6 +8,7 @@ _pointer_type_cache, c_void_p, c_char, c_int, c_long) from test import support +from test.support import import_helper from ._support import Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE @@ -36,6 +36,7 @@ def test_noargs(self): @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class ReturnStructSizesTestCase(unittest.TestCase): def test_sizes(self): + _ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) for i in range(1, 11): fields = [ (f"f{f}", c_char) for f in range(1, i + 1)] @@ -116,6 +117,7 @@ class RECT(Structure): ("right", c_long), ("bottom", c_long)] + _ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) pt = POINT(15, 25) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 83d10dd8579074..cc3aa561cd4c42 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -1081,6 +1081,14 @@ def test_resize_term(self): self.assertEqual(curses.LINES, lines) self.assertEqual(curses.COLS, cols) + with self.assertRaises(OverflowError): + curses.resize_term(35000, 1) + with self.assertRaises(OverflowError): + curses.resize_term(1, 35000) + # GH-120378: Overflow failure in resize_term() causes refresh to fail + tmp = curses.initscr() + tmp.erase() + @requires_curses_func('resizeterm') def test_resizeterm(self): curses.update_lines_cols() @@ -1095,6 +1103,14 @@ def test_resizeterm(self): self.assertEqual(curses.LINES, lines) self.assertEqual(curses.COLS, cols) + with self.assertRaises(OverflowError): + curses.resizeterm(35000, 1) + with self.assertRaises(OverflowError): + curses.resizeterm(1, 35000) + # GH-120378: Overflow failure in resizeterm() causes refresh to fail + tmp = curses.initscr() + tmp.erase() + def test_ungetch(self): curses.ungetch(b'A') self.assertEqual(self.stdscr.getkey(), 'A') diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 272d427875ae40..2e6c49e29ce828 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -17,11 +17,14 @@ from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol, DefaultDict from typing import get_type_hints from collections import deque, OrderedDict, namedtuple, defaultdict -from functools import total_ordering +from copy import deepcopy +from functools import total_ordering, wraps import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation. +from test import support + # Just any custom exception we can catch. class CustomError(Exception): pass @@ -58,7 +61,7 @@ class C: x: int = field(default=1, default_factory=int) def test_field_repr(self): - int_field = field(default=1, init=True, repr=False) + int_field = field(default=1, init=True, repr=False, doc='Docstring') int_field.name = "id" repr_output = repr(int_field) expected_output = "Field(name='id',type=None," \ @@ -66,6 +69,7 @@ def test_field_repr(self): "init=True,repr=False,hash=None," \ "compare=True,metadata=mappingproxy({})," \ f"kw_only={MISSING!r}," \ + "doc='Docstring'," \ "_field_type=None)" self.assertEqual(repr_output, expected_output) @@ -1315,6 +1319,29 @@ def __post_init__(self, init_base, init_derived): c = C(10, 11, 50, 51) self.assertEqual(vars(c), {'x': 21, 'y': 101}) + def test_init_var_name_shadowing(self): + # Because dataclasses rely exclusively on `__annotations__` for + # handling InitVar and `__annotations__` preserves shadowed definitions, + # you can actually shadow an InitVar with a method or property. + # + # This only works when there is no default value; `dataclasses` uses the + # actual name (which will be bound to the shadowing method) for default + # values. + @dataclass + class C: + shadowed: InitVar[int] + _shadowed: int = field(init=False) + + def __post_init__(self, shadowed): + self._shadowed = shadowed * 2 + + @property + def shadowed(self): + return self._shadowed * 3 + + c = C(5) + self.assertEqual(c.shadowed, 30) + def test_default_factory(self): # Test a factory that returns a new list. @dataclass @@ -1522,6 +1549,24 @@ class A(types.GenericAlias): self.assertTrue(is_dataclass(type(a))) self.assertTrue(is_dataclass(a)) + def test_is_dataclass_inheritance(self): + @dataclass + class X: + y: int + + class Z(X): + pass + + self.assertTrue(is_dataclass(X), "X should be a dataclass") + self.assertTrue( + is_dataclass(Z), + "Z should be a dataclass because it inherits from X", + ) + z_instance = Z(y=5) + self.assertTrue( + is_dataclass(z_instance), + "z_instance should be a dataclass because it is an instance of Z", + ) def test_helper_fields_with_class_instance(self): # Check that we can call fields() on either a class or instance, @@ -2216,6 +2261,7 @@ def assertDocStrEqual(self, a, b): # whitespace stripped. self.assertEqual(a.replace(' ', ''), b.replace(' ', '')) + @support.requires_docstrings def test_existing_docstring_not_overridden(self): @dataclass class C: @@ -2468,6 +2514,15 @@ def __repr__(self): class TestEq(unittest.TestCase): + def test_recursive_eq(self): + # Test a class with recursive child + @dataclass + class C: + recursive: object = ... + c = C() + c.recursive = c + self.assertEqual(c, c) + def test_no_eq(self): # Test a class with no __eq__ and eq=False. @dataclass(eq=False) @@ -3122,6 +3177,48 @@ class C: with self.assertRaisesRegex(TypeError, 'unhashable type'): hash(C({})) + def test_frozen_deepcopy_without_slots(self): + # see: https://github.com/python/cpython/issues/89683 + @dataclass(frozen=True, slots=False) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + def test_frozen_deepcopy_with_slots(self): + # see: https://github.com/python/cpython/issues/89683 + with self.subTest('generated __slots__'): + @dataclass(frozen=True, slots=True) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + with self.subTest('user-defined __slots__ and no __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + s: str + + # with user-defined slots, __getstate__ and __setstate__ are not + # automatically added, hence the error + err = r"^cannot\ assign\ to\ field\ 's'$" + self.assertRaisesRegex(FrozenInstanceError, err, deepcopy, C('')) + + with self.subTest('user-defined __slots__ and __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + __getstate__ = dataclasses._dataclass_getstate + __setstate__ = dataclasses._dataclass_setstate + + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + class TestSlots(unittest.TestCase): def test_simple(self): @@ -3208,7 +3305,7 @@ class Base(Root4): j: str h: str - self.assertEqual(Base.__slots__, ('y', )) + self.assertEqual(Base.__slots__, ('y',)) @dataclass(slots=True) class Derived(Base): @@ -3218,7 +3315,7 @@ class Derived(Base): k: str h: str - self.assertEqual(Derived.__slots__, ('z', )) + self.assertEqual(Derived.__slots__, ('z',)) @dataclass class AnotherDerived(Base): @@ -3226,6 +3323,24 @@ class AnotherDerived(Base): self.assertNotIn('__slots__', AnotherDerived.__dict__) + def test_slots_with_docs(self): + class Root: + __slots__ = {'x': 'x'} + + @dataclass(slots=True) + class Base(Root): + y1: int = field(doc='y1') + y2: int + + self.assertEqual(Base.__slots__, {'y1': 'y1', 'y2': None}) + + @dataclass(slots=True) + class Child(Base): + z1: int = field(doc='z1') + z2: int + + self.assertEqual(Child.__slots__, {'z1': 'z1', 'z2': None}) + def test_cant_inherit_from_iterator_slots(self): class Root: @@ -3495,6 +3610,174 @@ class A(Base): self.assertIs(a.__weakref__, a_ref) + def test_dataclass_derived_weakref_slot(self): + class A: + pass + + @dataclass(slots=True, weakref_slot=True) + class B(A): + pass + + self.assertEqual(B.__slots__, ()) + B() + + def test_dataclass_derived_generic(self): + T = typing.TypeVar('T') + + @dataclass(slots=True, weakref_slot=True) + class A(typing.Generic[T]): + pass + self.assertEqual(A.__slots__, ('__weakref__',)) + self.assertTrue(A.__weakref__) + A() + + @dataclass(slots=True, weakref_slot=True) + class B[T2]: + pass + self.assertEqual(B.__slots__, ('__weakref__',)) + self.assertTrue(B.__weakref__) + B() + + def test_dataclass_derived_generic_from_base(self): + T = typing.TypeVar('T') + + class RawBase: ... + + @dataclass(slots=True, weakref_slot=True) + class C1(typing.Generic[T], RawBase): + pass + self.assertEqual(C1.__slots__, ()) + self.assertTrue(C1.__weakref__) + C1() + @dataclass(slots=True, weakref_slot=True) + class C2(RawBase, typing.Generic[T]): + pass + self.assertEqual(C2.__slots__, ()) + self.assertTrue(C2.__weakref__) + C2() + + @dataclass(slots=True, weakref_slot=True) + class D[T2](RawBase): + pass + self.assertEqual(D.__slots__, ()) + self.assertTrue(D.__weakref__) + D() + + def test_dataclass_derived_generic_from_slotted_base(self): + T = typing.TypeVar('T') + + class WithSlots: + __slots__ = ('a', 'b') + + @dataclass(slots=True, weakref_slot=True) + class E1(WithSlots, Generic[T]): + pass + self.assertEqual(E1.__slots__, ('__weakref__',)) + self.assertTrue(E1.__weakref__) + E1() + @dataclass(slots=True, weakref_slot=True) + class E2(Generic[T], WithSlots): + pass + self.assertEqual(E2.__slots__, ('__weakref__',)) + self.assertTrue(E2.__weakref__) + E2() + + @dataclass(slots=True, weakref_slot=True) + class F[T2](WithSlots): + pass + self.assertEqual(F.__slots__, ('__weakref__',)) + self.assertTrue(F.__weakref__) + F() + + def test_dataclass_derived_generic_from_slotted_base(self): + T = typing.TypeVar('T') + + class WithWeakrefSlot: + __slots__ = ('__weakref__',) + + @dataclass(slots=True, weakref_slot=True) + class G1(WithWeakrefSlot, Generic[T]): + pass + self.assertEqual(G1.__slots__, ()) + self.assertTrue(G1.__weakref__) + G1() + @dataclass(slots=True, weakref_slot=True) + class G2(Generic[T], WithWeakrefSlot): + pass + self.assertEqual(G2.__slots__, ()) + self.assertTrue(G2.__weakref__) + G2() + + @dataclass(slots=True, weakref_slot=True) + class H[T2](WithWeakrefSlot): + pass + self.assertEqual(H.__slots__, ()) + self.assertTrue(H.__weakref__) + H() + + def test_dataclass_slot_dict(self): + class WithDictSlot: + __slots__ = ('__dict__',) + + @dataclass(slots=True) + class A(WithDictSlot): ... + + self.assertEqual(A.__slots__, ()) + self.assertEqual(A().__dict__, {}) + A() + + @support.cpython_only + def test_dataclass_slot_dict_ctype(self): + # https://github.com/python/cpython/issues/123935 + from test.support import import_helper + # Skips test if `_testcapi` is not present: + _testcapi = import_helper.import_module('_testcapi') + + @dataclass(slots=True) + class HasDictOffset(_testcapi.HeapCTypeWithDict): + __dict__: dict = {} + self.assertNotEqual(_testcapi.HeapCTypeWithDict.__dictoffset__, 0) + self.assertEqual(HasDictOffset.__slots__, ()) + + @dataclass(slots=True) + class DoesNotHaveDictOffset(_testcapi.HeapCTypeWithWeakref): + __dict__: dict = {} + self.assertEqual(_testcapi.HeapCTypeWithWeakref.__dictoffset__, 0) + self.assertEqual(DoesNotHaveDictOffset.__slots__, ('__dict__',)) + + @support.cpython_only + def test_slots_with_wrong_init_subclass(self): + # TODO: This test is for a kinda-buggy behavior. + # Ideally, it should be fixed and `__init_subclass__` + # should be fully supported in the future versions. + # See https://github.com/python/cpython/issues/91126 + class WrongSuper: + def __init_subclass__(cls, arg): + pass + + with self.assertRaisesRegex( + TypeError, + "missing 1 required positional argument: 'arg'", + ): + @dataclass(slots=True) + class WithWrongSuper(WrongSuper, arg=1): + pass + + class CorrectSuper: + args = [] + def __init_subclass__(cls, arg="default"): + cls.args.append(arg) + + @dataclass(slots=True) + class WithCorrectSuper(CorrectSuper): + pass + + # __init_subclass__ is called twice: once for `WithCorrectSuper` + # and once for `WithCorrectSuper__slots__` new class + # that we create internally. + self.assertEqual(CorrectSuper.args, ["default", "default"]) + + class TestDescriptors(unittest.TestCase): def test_set_name(self): # See bpo-33141. @@ -4034,6 +4317,23 @@ def test_funny_class_names_names(self): C = make_dataclass(classname, ['a', 'b']) self.assertEqual(C.__name__, classname) + def test_dataclass_decorator_default(self): + C = make_dataclass('C', [('x', int)], decorator=dataclass) + c = C(10) + self.assertEqual(c.x, 10) + + def test_dataclass_custom_decorator(self): + def custom_dataclass(cls, *args, **kwargs): + dc = dataclass(cls, *args, **kwargs) + dc.__custom__ = True + return dc + + C = make_dataclass('C', [('x', int)], decorator=custom_dataclass) + c = C(10) + self.assertEqual(c.x, 10) + self.assertEqual(c.__custom__, True) + + class TestReplace(unittest.TestCase): def test(self): @dataclass(frozen=True) @@ -4637,6 +4937,140 @@ def test_make_dataclass(self): self.assertTrue(fields(B)[0].kw_only) self.assertFalse(fields(B)[1].kw_only) + def test_deferred_annotations(self): + @dataclass + class A: + x: undefined + y: ClassVar[undefined] + + fs = fields(A) + self.assertEqual(len(fs), 1) + self.assertEqual(fs[0].name, 'x') + + +class TestZeroArgumentSuperWithSlots(unittest.TestCase): + def test_zero_argument_super(self): + @dataclass(slots=True) + class A: + def foo(self): + super() + + A().foo() + + def test_dunder_class_with_old_property(self): + @dataclass(slots=True) + class A: + def _get_foo(slf): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + return __class__ + + def _set_foo(slf, value): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + + def _del_foo(slf): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + + foo = property(_get_foo, _set_foo, _del_foo) + + a = A() + self.assertIs(a.foo, A) + a.foo = 4 + del a.foo + + def test_dunder_class_with_new_property(self): + @dataclass(slots=True) + class A: + @property + def foo(slf): + return slf.__class__ + + @foo.setter + def foo(slf, value): + self.assertIs(__class__, type(slf)) + + @foo.deleter + def foo(slf): + self.assertIs(__class__, type(slf)) + + a = A() + self.assertIs(a.foo, A) + a.foo = 4 + del a.foo + + # Test the parts of a property individually. + def test_slots_dunder_class_property_getter(self): + @dataclass(slots=True) + class A: + @property + def foo(slf): + return __class__ + + a = A() + self.assertIs(a.foo, A) + + def test_slots_dunder_class_property_setter(self): + @dataclass(slots=True) + class A: + foo = property() + @foo.setter + def foo(slf, val): + self.assertIs(__class__, type(slf)) + + a = A() + a.foo = 4 + + def test_slots_dunder_class_property_deleter(self): + @dataclass(slots=True) + class A: + foo = property() + @foo.deleter + def foo(slf): + self.assertIs(__class__, type(slf)) + + a = A() + del a.foo + + def test_wrapped(self): + def mydecorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + return wrapper + + @dataclass(slots=True) + class A: + @mydecorator + def foo(self): + super() + + A().foo() + + def test_remembered_class(self): + # Apply the dataclass decorator manually (not when the class + # is created), so that we can keep a reference to the + # undecorated class. + class A: + def cls(self): + return __class__ + + self.assertIs(A().cls(), A) + + B = dataclass(slots=True)(A) + self.assertIs(B().cls(), B) + + # This is undesirable behavior, but is a function of how + # modifying __class__ in the closure works. I'm not sure this + # should be tested or not: I don't really want to guarantee + # this behavior, but I don't want to lose the point that this + # is how it works. + + # The underlying class is "broken" by changing its __class__ + # in A.foo() to B. This normally isn't a problem, because no + # one will be keeping a reference to the underlying class A. + self.assertIs(A().cls(), B) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 3859733a4fe65b..005187f13e665f 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -1,5 +1,6 @@ import unittest import sys +import functools from test.support.import_helper import import_fresh_module @@ -39,21 +40,26 @@ def load_tests(loader, tests, pattern): for cls in test_classes: cls.__name__ += suffix cls.__qualname__ += suffix - @classmethod - def setUpClass(cls_, module=module): - cls_._save_sys_modules = sys.modules.copy() - sys.modules[TESTS] = module - sys.modules['datetime'] = module.datetime_module - if hasattr(module, '_pydatetime'): - sys.modules['_pydatetime'] = module._pydatetime - sys.modules['_strptime'] = module._strptime - @classmethod - def tearDownClass(cls_): - sys.modules.clear() - sys.modules.update(cls_._save_sys_modules) - cls.setUpClass = setUpClass - cls.tearDownClass = tearDownClass - tests.addTests(loader.loadTestsFromTestCase(cls)) + + @functools.wraps(cls, updated=()) + class Wrapper(cls): + @classmethod + def setUpClass(cls_, module=module): + cls_._save_sys_modules = sys.modules.copy() + sys.modules[TESTS] = module + sys.modules['datetime'] = module.datetime_module + if hasattr(module, '_pydatetime'): + sys.modules['_pydatetime'] = module._pydatetime + sys.modules['_strptime'] = module._strptime + super().setUpClass() + + @classmethod + def tearDownClass(cls_): + super().tearDownClass() + sys.modules.clear() + sys.modules.update(cls_._save_sys_modules) + + tests.addTests(loader.loadTestsFromTestCase(Wrapper)) return tests diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index e3924d8ec8b5c1..4be7c5649da68a 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -6,6 +6,13 @@ from test.support import import_helper from test.support import os_helper + +try: + from dbm import sqlite3 as dbm_sqlite3 +except ImportError: + dbm_sqlite3 = None + + try: from dbm import ndbm except ImportError: @@ -213,6 +220,27 @@ def test_whichdb_ndbm(self): for path in fnames: self.assertIsNone(self.dbm.whichdb(path)) + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3(self): + # Databases created by dbm.sqlite3 are detected correctly. + with dbm_sqlite3.open(_fname, "c") as db: + db["key"] = "value" + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") + + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3_existing_db(self): + # Existing sqlite3 databases are detected correctly. + sqlite3 = import_helper.import_module("sqlite3") + try: + # Create an empty database. + with sqlite3.connect(_fname) as cx: + cx.execute("CREATE TABLE dummy(database)") + cx.commit() + finally: + cx.close() + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") + + def setUp(self): self.addCleanup(cleaunup_test_dir) setup_test_dir() diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py new file mode 100644 index 00000000000000..2e1f2d32924bad --- /dev/null +++ b/Lib/test/test_dbm_sqlite3.py @@ -0,0 +1,310 @@ +import sys +import unittest +from contextlib import closing +from functools import partial +from pathlib import Path +from test.support import import_helper, os_helper + +dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") +# N.B. The test will fail on some platforms without sqlite3 +# if the sqlite3 import is above the import of dbm.sqlite3. +# This is deliberate: if the import helper managed to import dbm.sqlite3, +# we must inevitably be able to import sqlite3. Else, we have a problem. +import sqlite3 +from dbm.sqlite3 import _normalize_uri + + +class _SQLiteDbmTests(unittest.TestCase): + + def setUp(self): + self.filename = os_helper.TESTFN + db = dbm_sqlite3.open(self.filename, "c") + db.close() + + def tearDown(self): + for suffix in "", "-wal", "-shm": + os_helper.unlink(self.filename + suffix) + + +class URI(unittest.TestCase): + + def test_uri_substitutions(self): + dataset = ( + ("/absolute/////b/c", "/absolute/b/c"), + ("PRE#MID##END", "PRE%23MID%23%23END"), + ("%#?%%#", "%25%23%3F%25%25%23"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + self.assertTrue(_normalize_uri(path).endswith(normalized)) + + @unittest.skipUnless(sys.platform == "win32", "requires Windows") + def test_uri_windows(self): + dataset = ( + # Relative subdir. + (r"2018\January.xlsx", + "2018/January.xlsx"), + # Absolute with drive letter. + (r"C:\Projects\apilibrary\apilibrary.sln", + "/C:/Projects/apilibrary/apilibrary.sln"), + # Relative with drive letter. + (r"C:Projects\apilibrary\apilibrary.sln", + "/C:Projects/apilibrary/apilibrary.sln"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + if not Path(path).is_absolute(): + self.skipTest(f"skipping relative path: {path!r}") + self.assertTrue(_normalize_uri(path).endswith(normalized)) + + +class ReadOnly(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + with dbm_sqlite3.open(self.filename, "w") as db: + db[b"key1"] = "value1" + db[b"key2"] = "value2" + self.db = dbm_sqlite3.open(self.filename, "r") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_readonly_read(self): + self.assertEqual(self.db[b"key1"], b"value1") + self.assertEqual(self.db[b"key2"], b"value2") + + def test_readonly_write(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[b"new"] = "value" + + def test_readonly_delete(self): + with self.assertRaises(dbm_sqlite3.error): + del self.db[b"key1"] + + def test_readonly_keys(self): + self.assertEqual(self.db.keys(), [b"key1", b"key2"]) + + def test_readonly_iter(self): + self.assertEqual([k for k in self.db], [b"key1", b"key2"]) + + +class ReadWrite(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def db_content(self): + with closing(sqlite3.connect(self.filename)) as cx: + keys = [r[0] for r in cx.execute("SELECT key FROM Dict")] + vals = [r[0] for r in cx.execute("SELECT value FROM Dict")] + return keys, vals + + def test_readwrite_unique_key(self): + self.db["key"] = "value" + self.db["key"] = "other" + keys, vals = self.db_content() + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"other"]) + + def test_readwrite_delete(self): + self.db["key"] = "value" + self.db["new"] = "other" + + del self.db[b"new"] + keys, vals = self.db_content() + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"value"]) + + del self.db[b"key"] + keys, vals = self.db_content() + self.assertEqual(keys, []) + self.assertEqual(vals, []) + + def test_readwrite_null_key(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[None] = "value" + + def test_readwrite_null_value(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[b"key"] = None + + +class Misuse(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_misuse_double_create(self): + self.db["key"] = "value" + with dbm_sqlite3.open(self.filename, "c") as db: + self.assertEqual(db[b"key"], b"value") + + def test_misuse_double_close(self): + self.db.close() + + def test_misuse_invalid_flag(self): + regex = "must be.*'r'.*'w'.*'c'.*'n', not 'invalid'" + with self.assertRaisesRegex(ValueError, regex): + dbm_sqlite3.open(self.filename, flag="invalid") + + def test_misuse_double_delete(self): + self.db["key"] = "value" + del self.db[b"key"] + with self.assertRaises(KeyError): + del self.db[b"key"] + + def test_misuse_invalid_key(self): + with self.assertRaises(KeyError): + self.db[b"key"] + + def test_misuse_iter_close1(self): + self.db["1"] = 1 + it = iter(self.db) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_iter_close2(self): + self.db["1"] = 1 + self.db["2"] = 2 + it = iter(self.db) + next(it) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_use_after_close(self): + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + self.db[b"read"] + with self.assertRaises(dbm_sqlite3.error): + self.db[b"write"] = "value" + with self.assertRaises(dbm_sqlite3.error): + del self.db[b"del"] + with self.assertRaises(dbm_sqlite3.error): + len(self.db) + with self.assertRaises(dbm_sqlite3.error): + self.db.keys() + + def test_misuse_reinit(self): + with self.assertRaises(dbm_sqlite3.error): + self.db.__init__("new.db", flag="n", mode=0o666) + + def test_misuse_empty_filename(self): + for flag in "r", "w", "c", "n": + with self.assertRaises(dbm_sqlite3.error): + db = dbm_sqlite3.open("", flag="c") + + +class DataTypes(_SQLiteDbmTests): + + dataset = ( + # (raw, coerced) + (42, b"42"), + (3.14, b"3.14"), + ("string", b"string"), + (b"bytes", b"bytes"), + ) + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_datatypes_values(self): + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db["key"] = raw + self.assertEqual(self.db[b"key"], coerced) + + def test_datatypes_keys(self): + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db[raw] = "value" + self.assertEqual(self.db[coerced], b"value") + # Raw keys are silently coerced to bytes. + self.assertEqual(self.db[raw], b"value") + del self.db[raw] + + def test_datatypes_replace_coerced(self): + self.db["10"] = "value" + self.db[b"10"] = "value" + self.db[10] = "value" + self.assertEqual(self.db.keys(), [b"10"]) + + +class CorruptDatabase(_SQLiteDbmTests): + """Verify that database exceptions are raised as dbm.sqlite3.error.""" + + def setUp(self): + super().setUp() + with closing(sqlite3.connect(self.filename)) as cx: + with cx: + cx.execute("DROP TABLE IF EXISTS Dict") + cx.execute("CREATE TABLE Dict (invalid_schema)") + + def check(self, flag, fn, should_succeed=False): + with closing(dbm_sqlite3.open(self.filename, flag)) as db: + with self.assertRaises(dbm_sqlite3.error): + fn(db) + + @staticmethod + def read(db): + return db["key"] + + @staticmethod + def write(db): + db["key"] = "value" + + @staticmethod + def iter(db): + next(iter(db)) + + @staticmethod + def keys(db): + db.keys() + + @staticmethod + def del_(db): + del db["key"] + + @staticmethod + def len_(db): + len(db) + + def test_corrupt_readwrite(self): + for flag in "r", "w", "c": + with self.subTest(flag=flag): + check = partial(self.check, flag=flag) + check(fn=self.read) + check(fn=self.write) + check(fn=self.iter) + check(fn=self.keys) + check(fn=self.del_) + check(fn=self.len_) + + def test_corrupt_force_new(self): + with closing(dbm_sqlite3.open(self.filename, "n")) as db: + db["foo"] = "write" + _ = db[b"foo"] + next(iter(db)) + del db[b"foo"] + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 1423bc61c7f690..bc6c6427740949 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -34,11 +34,10 @@ import locale from test.support import (is_resource_enabled, requires_IEEE_754, requires_docstrings, - check_sanitizer, check_disallow_instantiation) from test.support import (TestFailed, run_with_locale, cpython_only, - darwin_malloc_err_warning, is_emscripten) + darwin_malloc_err_warning) from test.support.import_helper import import_fresh_module from test.support import threading_helper from test.support import warnings_helper @@ -813,6 +812,29 @@ def test_explicit_context_create_from_float(self): x = random.expovariate(0.01) * (random.random() * 2.0 - 1.0) self.assertEqual(x, float(nc.create_decimal(x))) # roundtrip + def test_from_number(self, cls=None): + Decimal = self.decimal.Decimal + if cls is None: + cls = Decimal + + def check(arg, expected): + d = cls.from_number(arg) + self.assertIs(type(d), cls) + self.assertEqual(d, expected) + + check(314, Decimal(314)) + check(3.14, Decimal.from_float(3.14)) + check(Decimal('3.14'), Decimal('3.14')) + self.assertRaises(TypeError, cls.from_number, 3+4j) + self.assertRaises(TypeError, cls.from_number, '314') + self.assertRaises(TypeError, cls.from_number, (0, (3, 1, 4), 0)) + self.assertRaises(TypeError, cls.from_number, object()) + + def test_from_number_subclass(self, cls=None): + class DecimalSubclass(self.decimal.Decimal): + pass + self.test_from_number(DecimalSubclass) + def test_unicode_digits(self): Decimal = self.decimal.Decimal @@ -1110,6 +1132,13 @@ def test_formatting(self): ('z>z6.1f', '-0.', 'zzz0.0'), ('x>z6.1f', '-0.', 'xxx0.0'), ('🖤>z6.1f', '-0.', '🖤🖤🖤0.0'), # multi-byte fill char + ('\x00>z6.1f', '-0.', '\x00\x00\x000.0'), # null fill char + + # issue 114563 ('z' format on F type in cdecimal) + ('z3,.10F', '-6.24E-323', '0.0000000000'), + + # issue 91060 ('#' format in cdecimal) + ('#', '0', '0.'), # issue 6850 ('a=-7.0', '0.12345', 'aaaa0.1'), @@ -1247,7 +1276,7 @@ def test_deprecated_N_format(self): self.assertRaises(ValueError, format, h, '10Nf') self.assertRaises(ValueError, format, h, 'Nx') - @run_with_locale('LC_ALL', 'ps_AF') + @run_with_locale('LC_ALL', 'ps_AF', '') def test_wide_char_separator_decimal_point(self): # locale with wide char separator and decimal point Decimal = self.decimal.Decimal @@ -2065,7 +2094,9 @@ def test_tonum_methods(self): #to quantize, which is already extensively tested test_triples = [ ('123.456', -4, '0E+4'), + ('-123.456', -4, '-0E+4'), ('123.456', -3, '0E+3'), + ('-123.456', -3, '-0E+3'), ('123.456', -2, '1E+2'), ('123.456', -1, '1.2E+2'), ('123.456', 0, '123'), @@ -4373,7 +4404,8 @@ def test_module_attributes(self): self.assertEqual(C.__version__, P.__version__) - self.assertEqual(dir(C), dir(P)) + self.assertLessEqual(set(dir(C)), set(dir(P))) + self.assertEqual([n for n in dir(C) if n[:2] != '__'], sorted(P.__all__)) def test_context_attributes(self): @@ -4710,9 +4742,33 @@ def test_py_exact_power(self): c.prec = 1 x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('3e-6')) + c.prec = 2 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.6e-6')) + c.prec = 3 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 28 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 201 x = Decimal(2**578) ** Decimal("-0.5") + # See https://github.com/python/cpython/issues/118027 + # Testing for an exact power could appear to hang, in the Python + # version, as it attempted to compute 10**(MAX_EMAX + 1). + # Fixed via https://github.com/python/cpython/pull/118503. + c.prec = P.MAX_PREC + c.Emax = P.MAX_EMAX + c.Emin = P.MIN_EMIN + c.traps[P.Inexact] = 1 + D2 = Decimal(2) + # If the bug is still present, the next statement won't complete. + res = D2 ** 117 + self.assertEqual(res, 1 << 117) + def test_py_immutability_operations(self): # Do operations and check that it didn't change internal objects. Decimal = P.Decimal @@ -5636,47 +5692,6 @@ def __abs__(self): self.assertEqual(Decimal.from_float(cls(101.1)), Decimal.from_float(101.1)) - # Issue 41540: - @unittest.skipIf(sys.platform.startswith("aix"), - "AIX: default ulimit: test is flaky because of extreme over-allocation") - @unittest.skipIf(is_emscripten, "Test is unstable on Emscripten") - @unittest.skipIf(check_sanitizer(address=True, memory=True), - "ASAN/MSAN sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_maxcontext_exact_arith(self): - - # Make sure that exact operations do not raise MemoryError due - # to huge intermediate values when the context precision is very - # large. - - # The following functions fill the available precision and are - # therefore not suitable for large precisions (by design of the - # specification). - MaxContextSkip = ['logical_invert', 'next_minus', 'next_plus', - 'logical_and', 'logical_or', 'logical_xor', - 'next_toward', 'rotate', 'shift'] - - Decimal = C.Decimal - Context = C.Context - localcontext = C.localcontext - - # Here only some functions that are likely candidates for triggering a - # MemoryError are tested. deccheck.py has an exhaustive test. - maxcontext = Context(prec=C.MAX_PREC, Emin=C.MIN_EMIN, Emax=C.MAX_EMAX) - with localcontext(maxcontext): - self.assertEqual(Decimal(0).exp(), 1) - self.assertEqual(Decimal(1).ln(), 0) - self.assertEqual(Decimal(1).log10(), 0) - self.assertEqual(Decimal(10**2).log10(), 2) - self.assertEqual(Decimal(10**223).log10(), 223) - self.assertEqual(Decimal(10**19).logb(), 19) - self.assertEqual(Decimal(4).sqrt(), 2) - self.assertEqual(Decimal("40E9").sqrt(), Decimal('2.0E+5')) - self.assertEqual(divmod(Decimal(10), 3), (3, 1)) - self.assertEqual(Decimal(10) // 3, 3) - self.assertEqual(Decimal(4) / 2, 2) - self.assertEqual(Decimal(400) ** -1, Decimal('0.0025')) - def test_c_immutable_types(self): SignalDict = type(C.Context().flags) SignalDictMixin = SignalDict.__bases__[0] @@ -5726,6 +5741,20 @@ def test_c_signaldict_segfault(self): with self.assertRaisesRegex(ValueError, err_msg): sd.copy() + def test_format_fallback_capitals(self): + # Fallback to _pydecimal formatting (triggered by `#` format which + # is unsupported by mpdecimal) should honor the current context. + x = C.Decimal('6.09e+23') + self.assertEqual(format(x, '#'), '6.09E+23') + with C.localcontext(capitals=0): + self.assertEqual(format(x, '#'), '6.09e+23') + + def test_format_fallback_rounding(self): + y = C.Decimal('6.09') + self.assertEqual(format(y, '#.1f'), '6.1') + with C.localcontext(rounding=C.ROUND_DOWN): + self.assertEqual(format(y, '#.1f'), '6.0') + @requires_docstrings @requires_cdecimal class SignatureTest(unittest.TestCase): @@ -5889,13 +5918,17 @@ def load_tests(loader, tests, pattern): if TODO_TESTS is None: from doctest import DocTestSuite, IGNORE_EXCEPTION_DETAIL + orig_context = orig_sys_decimal.getcontext().copy() for mod in C, P: if not mod: continue def setUp(slf, mod=mod): sys.modules['decimal'] = mod - def tearDown(slf): + init(mod) + def tearDown(slf, mod=mod): sys.modules['decimal'] = orig_sys_decimal + mod.setcontext(ORIGINAL_CONTEXT[mod].copy()) + orig_sys_decimal.setcontext(orig_context.copy()) optionflags = IGNORE_EXCEPTION_DETAIL if mod is C else 0 sys.modules['decimal'] = mod tests.addTest(DocTestSuite(mod, setUp=setUp, tearDown=tearDown, @@ -5910,8 +5943,8 @@ def setUpModule(): TEST_ALL = ARITH if ARITH is not None else is_resource_enabled('decimal') def tearDownModule(): - if C: C.setcontext(ORIGINAL_CONTEXT[C]) - P.setcontext(ORIGINAL_CONTEXT[P]) + if C: C.setcontext(ORIGINAL_CONTEXT[C].copy()) + P.setcontext(ORIGINAL_CONTEXT[P].copy()) if not C: warnings.warn('C tests skipped: no module named _decimal.', UserWarning) diff --git a/Lib/test/test_decorators.py b/Lib/test/test_decorators.py index 3a4fc959f6f8a7..78361be9fa1e61 100644 --- a/Lib/test/test_decorators.py +++ b/Lib/test/test_decorators.py @@ -1,5 +1,5 @@ import unittest -from types import MethodType + def funcattrs(**kwds): def decorate(func): diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index ae1dfacd7262e4..4679f297fd7f4a 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -166,7 +166,7 @@ def test_contains(self): with self.assertRaises(RuntimeError): n in d - def test_contains_count_stop_crashes(self): + def test_contains_count_index_stop_crashes(self): class A: def __eq__(self, other): d.clear() @@ -178,6 +178,10 @@ def __eq__(self, other): with self.assertRaises(RuntimeError): _ = d.count(3) + d = deque([A()]) + with self.assertRaises(RuntimeError): + d.index(0) + def test_extend(self): d = deque('a') self.assertRaises(TypeError, d.extend, 1) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index beeab6cb7f254c..aa801b9c4f7ad9 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1314,7 +1314,7 @@ class X(object): # Inherit from object on purpose to check some backwards compatibility paths class X(object): __slots__ = "a" - with self.assertRaisesRegex(AttributeError, "'X' object has no attribute 'a'"): + with self.assertRaisesRegex(AttributeError, "'test.test_descr.ClassPropertiesAndMethods.test_slots..X' object has no attribute 'a'"): X().a # Test string subclass in `__slots__`, see gh-98783 @@ -1593,8 +1593,11 @@ def f(cls, arg): self.fail("classmethod shouldn't accept keyword args") cm = classmethod(f) - cm_dict = {'__annotations__': {}, - '__doc__': "f docstring", + cm_dict = {'__doc__': ( + "f docstring" + if support.HAVE_DOCSTRINGS + else None + ), '__module__': __name__, '__name__': 'f', '__qualname__': f.__qualname__} @@ -1606,6 +1609,55 @@ def f(cls, arg): del cm.x self.assertNotHasAttr(cm, "x") + def test_classmethod_staticmethod_annotations(self): + for deco in (classmethod, staticmethod): + @deco + def unannotated(cls): pass + @deco + def annotated(cls) -> int: pass + + for method in (annotated, unannotated): + with self.subTest(deco=deco, method=method): + with self.assertRaises(AttributeError): + del unannotated.__annotations__ + + original_annotations = dict(method.__wrapped__.__annotations__) + self.assertNotIn('__annotations__', method.__dict__) + self.assertEqual(method.__annotations__, original_annotations) + self.assertIn('__annotations__', method.__dict__) + + new_annotations = {"a": "b"} + method.__annotations__ = new_annotations + self.assertEqual(method.__annotations__, new_annotations) + self.assertEqual(method.__wrapped__.__annotations__, original_annotations) + + del method.__annotations__ + self.assertEqual(method.__annotations__, original_annotations) + + original_annotate = method.__wrapped__.__annotate__ + self.assertNotIn('__annotate__', method.__dict__) + self.assertIs(method.__annotate__, original_annotate) + self.assertIn('__annotate__', method.__dict__) + + new_annotate = lambda: {"annotations": 1} + method.__annotate__ = new_annotate + self.assertIs(method.__annotate__, new_annotate) + self.assertIs(method.__wrapped__.__annotate__, original_annotate) + + del method.__annotate__ + self.assertIs(method.__annotate__, original_annotate) + + def test_staticmethod_annotations_without_dict_access(self): + # gh-125017: this used to crash + class Spam: + def __new__(cls, x, y): + pass + + self.assertEqual(Spam.__new__.__annotations__, {}) + obj = Spam.__dict__['__new__'] + self.assertIsInstance(obj, staticmethod) + self.assertEqual(obj.__annotations__, {}) + @support.refcount_test def test_refleaks_in_classmethod___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') @@ -1683,10 +1735,10 @@ class D(C): self.assertEqual(d.foo(1), (d, 1)) self.assertEqual(D.foo(d, 1), (d, 1)) sm = staticmethod(None) - self.assertEqual(sm.__dict__, {'__doc__': None}) + self.assertEqual(sm.__dict__, {'__doc__': None.__doc__}) sm.x = 42 self.assertEqual(sm.x, 42) - self.assertEqual(sm.__dict__, {"x" : 42, '__doc__': None}) + self.assertEqual(sm.__dict__, {"x" : 42, '__doc__': None.__doc__}) del sm.x self.assertNotHasAttr(sm, "x") @@ -3187,8 +3239,6 @@ class classic: class C(base): def __init__(self, value): self.value = int(value) - def __cmp__(self_, other): - self.fail("shouldn't call __cmp__") def __eq__(self, other): if isinstance(other, C): return self.value == other.value @@ -3983,6 +4033,20 @@ def test_ipow_exception_text(self): y = x ** 2 self.assertIn('unsupported operand type(s) for **', str(cm.exception)) + def test_pow_wrapper_error_messages(self): + self.assertRaisesRegex(TypeError, + 'expected 1 or 2 arguments, got 0', + int().__pow__) + self.assertRaisesRegex(TypeError, + 'expected 1 or 2 arguments, got 3', + int().__pow__, 1, 2, 3) + self.assertRaisesRegex(TypeError, + 'expected 1 or 2 arguments, got 0', + int().__rpow__) + self.assertRaisesRegex(TypeError, + 'expected 1 or 2 arguments, got 3', + int().__rpow__, 1, 2, 3) + def test_mutable_bases(self): # Testing mutable bases... @@ -4590,18 +4654,16 @@ def test_special_unbound_method_types(self): def test_not_implemented(self): # Testing NotImplemented... # all binary methods should be able to return a NotImplemented - import operator def specialmethod(self, other): return NotImplemented def check(expr, x, y): - try: - exec(expr, {'x': x, 'y': y, 'operator': operator}) - except TypeError: - pass - else: - self.fail("no TypeError from %r" % (expr,)) + with ( + self.subTest(expr=expr, x=x, y=y), + self.assertRaises(TypeError), + ): + exec(expr, {'x': x, 'y': y}) N1 = sys.maxsize + 1 # might trigger OverflowErrors instead of # TypeErrors @@ -4622,12 +4684,23 @@ def check(expr, x, y): ('__and__', 'x & y', 'x &= y'), ('__or__', 'x | y', 'x |= y'), ('__xor__', 'x ^ y', 'x ^= y')]: - rname = '__r' + name[2:] + # Defines 'left' magic method: A = type('A', (), {name: specialmethod}) a = A() check(expr, a, a) check(expr, a, N1) check(expr, a, N2) + # Defines 'right' magic method: + rname = '__r' + name[2:] + B = type('B', (), {rname: specialmethod}) + b = B() + check(expr, b, b) + check(expr, a, b) + check(expr, b, a) + check(expr, b, N1) + check(expr, b, N2) + check(expr, N1, b) + check(expr, N2, b) if iexpr: check(iexpr, a, a) check(iexpr, a, N1) @@ -5075,8 +5148,10 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', + '__static_attributes__', '__weakref__', + 'meth']) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5085,7 +5160,7 @@ def test_iter_values(self): it = self.C.__dict__.values() self.assertNotIsInstance(it, list) values = list(it) - self.assertEqual(len(values), 5) + self.assertEqual(len(values), 7) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5095,8 +5170,10 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', + '__static_attributes__', '__weakref__', + 'meth']) def test_dict_type_with_metaclass(self): # Testing type of __dict__ when metaclass set... diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 13e3ea41bdb76c..828440a993a975 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -8,7 +8,7 @@ # of much interest anymore), and a few were fiddled to make the output # deterministic. -from test.support import sortdict +from test.support import sortdict # noqa: F401 import doctest import unittest @@ -39,16 +39,16 @@ def merge(self, other): Here's the new type at work: >>> print(defaultdict) # show our type - + >>> print(type(defaultdict)) # its metatype >>> a = defaultdict(default=0.0) # create an instance >>> print(a) # show the instance {} >>> print(type(a)) # show its type - + >>> print(a.__class__) # show its class - + >>> print(type(a) is a.__class__) # its type is its class True >>> a[1] = 3.25 # modify the instance @@ -99,7 +99,7 @@ def merge(self, other): >>> print(sortdict(a.__dict__)) {'default': -1000, 'x1': 100, 'x2': 200} >>> -""" +""" % {'modname': __name__} class defaultdict2(dict): __slots__ = ['default'] @@ -264,19 +264,19 @@ def merge(self, other): ... print("classmethod", cls, y) >>> C.foo(1) - classmethod 1 + classmethod 1 >>> c = C() >>> c.foo(1) - classmethod 1 + classmethod 1 >>> class D(C): ... pass >>> D.foo(1) - classmethod 1 + classmethod 1 >>> d = D() >>> d.foo(1) - classmethod 1 + classmethod 1 This prints "classmethod __main__.D 1" both times; in other words, the class passed as the first argument of foo() is the class involved in the @@ -292,18 +292,18 @@ class passed as the first argument of foo() is the class involved in the >>> E.foo(1) E.foo() called - classmethod 1 + classmethod 1 >>> e = E() >>> e.foo(1) E.foo() called - classmethod 1 + classmethod 1 In this example, the call to C.foo() from E.foo() will see class C as its first argument, not class E. This is to be expected, since the call specifies the class C. But it stresses the difference between these class methods and methods defined in metaclasses (where an upcall to a metamethod would pass the target class as an explicit first argument). -""" +""" % {'modname': __name__} test_5 = """ diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 620d0ca4f4c2da..4030716efb51f9 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper, Py_C_RECURSION_LIMIT +from test.support import import_helper, get_c_recursion_limit class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) @@ -1476,6 +1476,24 @@ def test_dict_items_result_gc_reversed(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + def test_store_evilattr(self): + class EvilAttr: + def __init__(self, d): + self.d = d + + def __del__(self): + if 'attr' in self.d: + del self.d['attr'] + gc.collect() + + class Obj: + pass + + obj = Obj() + obj.__dict__ = {} + for _ in range(10): + obj.attr = EvilAttr(obj.__dict__) + def test_str_nonstr(self): # cpython uses a different lookup function if the dict only contains # `str` keys. Make sure the unoptimized path is used when a non-`str` diff --git a/Lib/test/test_dict_version.py b/Lib/test/test_dict_version.py deleted file mode 100644 index 243084c75c42bc..00000000000000 --- a/Lib/test/test_dict_version.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -Test implementation of the PEP 509: dictionary versioning. -""" -import unittest -from test.support import import_helper - -# PEP 509 is implemented in CPython but other Python implementations -# don't require to implement it -_testcapi = import_helper.import_module('_testcapi') - - -class DictVersionTests(unittest.TestCase): - type2test = dict - - def setUp(self): - self.seen_versions = set() - self.dict = None - - def check_version_unique(self, mydict): - version = _testcapi.dict_get_version(mydict) - self.assertNotIn(version, self.seen_versions) - self.seen_versions.add(version) - - def check_version_changed(self, mydict, method, *args, **kw): - result = method(*args, **kw) - self.check_version_unique(mydict) - return result - - def check_version_dont_change(self, mydict, method, *args, **kw): - version1 = _testcapi.dict_get_version(mydict) - self.seen_versions.add(version1) - - result = method(*args, **kw) - - version2 = _testcapi.dict_get_version(mydict) - self.assertEqual(version2, version1, "version changed") - - return result - - def new_dict(self, *args, **kw): - d = self.type2test(*args, **kw) - self.check_version_unique(d) - return d - - def test_constructor(self): - # new empty dictionaries must all have an unique version - empty1 = self.new_dict() - empty2 = self.new_dict() - empty3 = self.new_dict() - - # non-empty dictionaries must also have an unique version - nonempty1 = self.new_dict(x='x') - nonempty2 = self.new_dict(x='x', y='y') - - def test_copy(self): - d = self.new_dict(a=1, b=2) - - d2 = self.check_version_dont_change(d, d.copy) - - # dict.copy() must create a dictionary with a new unique version - self.check_version_unique(d2) - - def test_setitem(self): - d = self.new_dict() - - # creating new keys must change the version - self.check_version_changed(d, d.__setitem__, 'x', 'x') - self.check_version_changed(d, d.__setitem__, 'y', 'y') - - # changing values must change the version - self.check_version_changed(d, d.__setitem__, 'x', 1) - self.check_version_changed(d, d.__setitem__, 'y', 2) - - def test_setitem_same_value(self): - value = object() - d = self.new_dict() - - # setting a key must change the version - self.check_version_changed(d, d.__setitem__, 'key', value) - - # setting a key to the same value with dict.__setitem__ - # must change the version - self.check_version_dont_change(d, d.__setitem__, 'key', value) - - # setting a key to the same value with dict.update - # must change the version - self.check_version_dont_change(d, d.update, key=value) - - d2 = self.new_dict(key=value) - self.check_version_dont_change(d, d.update, d2) - - def test_setitem_equal(self): - class AlwaysEqual: - def __eq__(self, other): - return True - - value1 = AlwaysEqual() - value2 = AlwaysEqual() - self.assertTrue(value1 == value2) - self.assertFalse(value1 != value2) - self.assertIsNot(value1, value2) - - d = self.new_dict() - self.check_version_changed(d, d.__setitem__, 'key', value1) - self.assertIs(d['key'], value1) - - # setting a key to a value equal to the current value - # with dict.__setitem__() must change the version - self.check_version_changed(d, d.__setitem__, 'key', value2) - self.assertIs(d['key'], value2) - - # setting a key to a value equal to the current value - # with dict.update() must change the version - self.check_version_changed(d, d.update, key=value1) - self.assertIs(d['key'], value1) - - d2 = self.new_dict(key=value2) - self.check_version_changed(d, d.update, d2) - self.assertIs(d['key'], value2) - - def test_setdefault(self): - d = self.new_dict() - - # setting a key with dict.setdefault() must change the version - self.check_version_changed(d, d.setdefault, 'key', 'value1') - - # don't change the version if the key already exists - self.check_version_dont_change(d, d.setdefault, 'key', 'value2') - - def test_delitem(self): - d = self.new_dict(key='value') - - # deleting a key with dict.__delitem__() must change the version - self.check_version_changed(d, d.__delitem__, 'key') - - # don't change the version if the key doesn't exist - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.__delitem__, 'key') - - def test_pop(self): - d = self.new_dict(key='value') - - # pop() must change the version if the key exists - self.check_version_changed(d, d.pop, 'key') - - # pop() must not change the version if the key does not exist - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.pop, 'key') - - def test_popitem(self): - d = self.new_dict(key='value') - - # popitem() must change the version if the dict is not empty - self.check_version_changed(d, d.popitem) - - # popitem() must not change the version if the dict is empty - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.popitem) - - def test_update(self): - d = self.new_dict(key='value') - - # update() calling with no argument must not change the version - self.check_version_dont_change(d, d.update) - - # update() must change the version - self.check_version_changed(d, d.update, key='new value') - - d2 = self.new_dict(key='value 3') - self.check_version_changed(d, d.update, d2) - - def test_clear(self): - d = self.new_dict(key='value') - - # clear() must change the version if the dict is not empty - self.check_version_changed(d, d.clear) - - # clear() must not change the version if the dict is empty - self.check_version_dont_change(d, d.clear) - - -class Dict(dict): - pass - - -class DictSubtypeVersionTests(DictVersionTests): - type2test = Dict - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index 472e3dfa0d8a0a..26b56dac5032fa 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -1,5 +1,8 @@ +import traceback import unittest +from test.support import BrokenIter + # For scope testing. g = "Global variable" @@ -127,6 +130,41 @@ def test_star_expression(self): self.assertEqual({i: i*i for i in [*range(4)]}, expected) self.assertEqual({i: i*i for i in (*range(4),)}, expected) + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + def init_raises(): + try: + {x:x for x in BrokenIter(init_raises=True)} + except Exception as e: + return e + + def next_raises(): + try: + {x:x for x in BrokenIter(next_raises=True)} + except Exception as e: + return e + + def iter_raises(): + try: + {x:x for x in BrokenIter(iter_raises=True)} + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index cad568b6ac4c2d..d9881611c19c43 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -2,7 +2,7 @@ import copy import pickle import unittest -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class DictSetTest(unittest.TestCase): @@ -279,7 +279,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(Py_C_RECURSION_LIMIT//2 + 100): + for i in range(get_c_recursion_limit()//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 6afd90af8442ad..9e217249be7332 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -272,10 +272,30 @@ def test_make_file_usascii_charset_with_nonascii_input(self): self.assertIn('content="text/html; charset=us-ascii"', output) self.assertIn('ımplıcıt', output) +class TestDiffer(unittest.TestCase): + def test_close_matches_aligned(self): + # Of the 4 closely matching pairs, we want 1 to match with 3, + # and 2 with 4, to align with a "top to bottom" mental model. + a = ["cat\n", "dog\n", "close match 1\n", "close match 2\n"] + b = ["close match 3\n", "close match 4\n", "kitten\n", "puppy\n"] + m = difflib.Differ().compare(a, b) + self.assertEqual(list(m), + ['- cat\n', + '- dog\n', + '- close match 1\n', + '? ^\n', + '+ close match 3\n', + '? ^\n', + '- close match 2\n', + '? ^\n', + '+ close match 4\n', + '? ^\n', + '+ kitten\n', + '+ puppy\n']) class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): - args = ['one', 'two', 'Original', 'Current', + args = [['one'], ['two'], 'Original', 'Current', '2005-01-26 23:30:50', '2010-04-02 10:20:52'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], [ @@ -287,7 +307,7 @@ def test_tab_delimiter(self): "--- Current\t2010-04-02 10:20:52"]) def test_no_trailing_tab_on_empty_filedate(self): - args = ['one', 'two', 'Original', 'Current'] + args = [['one'], ['two'], 'Original', 'Current'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], ["--- Original", "+++ Current"]) @@ -427,6 +447,28 @@ def assertDiff(expect, actual): lineterm=b'') assertDiff(expect, actual) + +class TestInputTypes(unittest.TestCase): + def _assert_type_error(self, msg, generator, *args): + with self.assertRaises(TypeError) as ctx: + list(generator(*args)) + self.assertEqual(msg, str(ctx.exception)) + + def test_input_type_checks(self): + unified = difflib.unified_diff + context = difflib.context_diff + + expect = "input must be a sequence of strings, not str" + self._assert_type_error(expect, unified, 'a', ['b']) + self._assert_type_error(expect, context, 'a', ['b']) + + self._assert_type_error(expect, unified, ['a'], 'b') + self._assert_type_error(expect, context, ['a'], 'b') + + expect = "lines to compare must be str, not NoneType (None)" + self._assert_type_error(expect, unified, ['a'], [None]) + self._assert_type_error(expect, context, ['a'], [None]) + def test_mixed_types_content(self): # type of input content must be consistent: all str or all bytes a = [b'hello'] @@ -475,10 +517,6 @@ def test_mixed_types_dates(self): b = ['bar\n'] list(difflib.unified_diff(a, b, 'a', 'b', datea, dateb)) - def _assert_type_error(self, msg, generator, *args): - with self.assertRaises(TypeError) as ctx: - list(generator(*args)) - self.assertEqual(msg, str(ctx.exception)) class TestJunkAPIs(unittest.TestCase): def test_is_line_junk_true(self): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index a5917da346dded..3c6570afa50d45 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1,5 +1,6 @@ # Minimal tests for dis module +import ast import contextlib import dis import functools @@ -47,42 +48,46 @@ def cm(cls, x): %3d RESUME 0 %3d LOAD_FAST 1 (x) - LOAD_CONST 1 (1) + LOAD_SMALL_INT 1 COMPARE_OP 72 (==) LOAD_FAST 0 (self) STORE_ATTR 0 (x) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ % (_C.__init__.__code__.co_firstlineno, _C.__init__.__code__.co_firstlineno + 1,) dis_c_instance_method_bytes = """\ RESUME 0 LOAD_FAST 1 - LOAD_CONST 1 + LOAD_SMALL_INT 1 COMPARE_OP 72 (==) LOAD_FAST 0 STORE_ATTR 0 - RETURN_CONST 0 + LOAD_CONST 0 + RETURN_VALUE """ dis_c_class_method = """\ %3d RESUME 0 %3d LOAD_FAST 1 (x) - LOAD_CONST 1 (1) + LOAD_SMALL_INT 1 COMPARE_OP 72 (==) LOAD_FAST 0 (cls) STORE_ATTR 0 (x) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ % (_C.cm.__code__.co_firstlineno, _C.cm.__code__.co_firstlineno + 2,) dis_c_static_method = """\ %3d RESUME 0 %3d LOAD_FAST 0 (x) - LOAD_CONST 1 (1) + LOAD_SMALL_INT 1 COMPARE_OP 72 (==) STORE_FAST 0 (x) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ % (_C.sm.__code__.co_firstlineno, _C.sm.__code__.co_firstlineno + 2,) # Class disassembling info has an extra newline at end. @@ -109,7 +114,8 @@ def _f(a): CALL 1 POP_TOP -%3d RETURN_CONST 1 (1) +%3d LOAD_SMALL_INT 1 + RETURN_VALUE """ % (_f.__code__.co_firstlineno, _f.__code__.co_firstlineno + 1, _f.__code__.co_firstlineno + 2) @@ -122,11 +128,23 @@ def _f(a): 14 CALL 1 22 POP_TOP -%3d 24 RETURN_CONST 1 (1) +%3d 24 LOAD_SMALL_INT 1 + 26 RETURN_VALUE """ % (_f.__code__.co_firstlineno, _f.__code__.co_firstlineno + 1, _f.__code__.co_firstlineno + 2) +dis_f_with_positions_format = f"""\ +%-14s RESUME 0 + +%-14s LOAD_GLOBAL 1 (print + NULL) +%-14s LOAD_FAST 0 (a) +%-14s CALL 1 +%-14s POP_TOP + +%-14s LOAD_SMALL_INT 1 +%-14s RETURN_VALUE +""" dis_f_co_code = """\ RESUME 0 @@ -134,7 +152,8 @@ def _f(a): LOAD_FAST 0 CALL 1 POP_TOP - RETURN_CONST 1 + LOAD_SMALL_INT 1 + RETURN_VALUE """ def bug708901(): @@ -146,9 +165,9 @@ def bug708901(): %3d RESUME 0 %3d LOAD_GLOBAL 1 (range + NULL) - LOAD_CONST 1 (1) + LOAD_SMALL_INT 1 -%3d LOAD_CONST 2 (10) +%3d LOAD_SMALL_INT 10 %3d CALL 2 GET_ITER @@ -159,7 +178,8 @@ def bug708901(): %3d L2: END_FOR POP_TOP - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ % (bug708901.__code__.co_firstlineno, bug708901.__code__.co_firstlineno + 1, bug708901.__code__.co_firstlineno + 2, @@ -176,14 +196,14 @@ def bug1333982(x=[]): dis_bug1333982 = """\ %3d RESUME 0 -%3d LOAD_ASSERTION_ERROR - LOAD_CONST 1 ( at 0x..., file "%s", line %d>) +%3d LOAD_COMMON_CONSTANT 0 (AssertionError) + LOAD_CONST 0 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST 0 (x) GET_ITER CALL 0 -%3d LOAD_CONST 2 (1) +%3d LOAD_SMALL_INT 1 %3d BINARY_OP 0 (+) CALL 0 @@ -206,16 +226,17 @@ def bug42562(): dis_bug42562 = """\ RESUME 0 - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ # Extended arg followed by NOP code_bug_45757 = bytes([ - opcode.opmap['EXTENDED_ARG'], 0x01, # EXTENDED_ARG 0x01 - opcode.opmap['NOP'], 0xFF, # NOP 0xFF - opcode.opmap['EXTENDED_ARG'], 0x01, # EXTENDED_ARG 0x01 - opcode.opmap['LOAD_CONST'], 0x29, # LOAD_CONST 0x29 - opcode.opmap['RETURN_VALUE'], 0x00, # RETURN_VALUE 0x00 + opcode.opmap['EXTENDED_ARG'], 0x01, + opcode.opmap['NOP'], 0xFF, + opcode.opmap['EXTENDED_ARG'], 0x01, + opcode.opmap['LOAD_CONST'], 0x29, + opcode.opmap['RETURN_VALUE'], 0x00, ]) dis_bug_45757 = """\ @@ -252,25 +273,27 @@ def wrap_func_w_kwargs(): %3d RESUME 0 %3d LOAD_GLOBAL 1 (func_w_kwargs + NULL) - LOAD_CONST 1 (1) - LOAD_CONST 2 (2) - LOAD_CONST 3 (5) - LOAD_CONST 4 (('c',)) + LOAD_SMALL_INT 1 + LOAD_SMALL_INT 2 + LOAD_SMALL_INT 5 + LOAD_CONST 0 (('c',)) CALL_KW 3 POP_TOP - RETURN_CONST 0 (None) + LOAD_CONST 1 (None) + RETURN_VALUE """ % (wrap_func_w_kwargs.__code__.co_firstlineno, wrap_func_w_kwargs.__code__.co_firstlineno + 1) dis_intrinsic_1_2 = """\ 0 RESUME 0 - 1 LOAD_CONST 0 (0) - LOAD_CONST 1 (('*',)) + 1 LOAD_SMALL_INT 0 + LOAD_CONST 0 (('*',)) IMPORT_NAME 0 (math) CALL_INTRINSIC_1 2 (INTRINSIC_IMPORT_STAR) POP_TOP - RETURN_CONST 2 (None) + LOAD_CONST 1 (None) + RETURN_VALUE """ dis_intrinsic_1_5 = """\ @@ -296,7 +319,8 @@ def wrap_func_w_kwargs(): %3d LOAD_GLOBAL 0 (spam) POP_TOP - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ _BIG_LINENO_FORMAT2 = """\ @@ -304,17 +328,20 @@ def wrap_func_w_kwargs(): %4d LOAD_GLOBAL 0 (spam) POP_TOP - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ dis_module_expected_results = """\ Disassembly of f: 4 RESUME 0 - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE Disassembly of g: 5 RESUME 0 - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ @@ -324,7 +351,7 @@ def wrap_func_w_kwargs(): 0 RESUME 0 1 LOAD_NAME 0 (x) - LOAD_CONST 0 (1) + LOAD_SMALL_INT 1 BINARY_OP 0 (+) RETURN_VALUE """ @@ -335,10 +362,11 @@ def wrap_func_w_kwargs(): 0 RESUME 0 1 LOAD_NAME 0 (x) - LOAD_CONST 0 (1) + LOAD_SMALL_INT 1 BINARY_OP 0 (+) STORE_NAME 0 (x) - RETURN_CONST 1 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ annot_stmt_str = """\ @@ -352,32 +380,40 @@ def wrap_func_w_kwargs(): dis_annot_stmt_str = """\ 0 RESUME 0 - 2 SETUP_ANNOTATIONS - LOAD_CONST 0 (1) + 2 LOAD_SMALL_INT 1 STORE_NAME 0 (x) - LOAD_NAME 1 (int) - LOAD_NAME 2 (__annotations__) - LOAD_CONST 1 ('x') - STORE_SUBSCR - 3 LOAD_NAME 3 (fun) + 4 LOAD_SMALL_INT 1 + LOAD_NAME 1 (lst) + LOAD_NAME 2 (fun) PUSH_NULL - LOAD_CONST 0 (1) + LOAD_SMALL_INT 0 CALL 1 - LOAD_NAME 2 (__annotations__) - LOAD_CONST 2 ('y') STORE_SUBSCR - 4 LOAD_CONST 0 (1) - LOAD_NAME 4 (lst) - LOAD_NAME 3 (fun) - PUSH_NULL - LOAD_CONST 3 (0) - CALL 1 - STORE_SUBSCR - LOAD_NAME 1 (int) - POP_TOP - RETURN_CONST 4 (None) + 2 LOAD_CONST 0 (", line 2>) + MAKE_FUNCTION + STORE_NAME 3 (__annotate__) + LOAD_CONST 1 (None) + RETURN_VALUE +""" + +fn_with_annotate_str = """ +def foo(a: int, b: str) -> str: + return a * b +""" + +dis_fn_with_annotate_str = """\ + 0 RESUME 0 + + 2 LOAD_CONST 0 (", line 2>) + MAKE_FUNCTION + LOAD_CONST 1 (", line 2>) + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 16 (annotate) + STORE_NAME 0 (foo) + LOAD_CONST 2 (None) + RETURN_VALUE """ compound_stmt_str = """\ @@ -389,17 +425,16 @@ def wrap_func_w_kwargs(): dis_compound_stmt_str = """\ 0 RESUME 0 - 1 LOAD_CONST 0 (0) + 1 LOAD_SMALL_INT 0 STORE_NAME 0 (x) - 2 NOP + 2 L1: NOP - 3 L1: LOAD_NAME 0 (x) - LOAD_CONST 1 (1) + 3 LOAD_NAME 0 (x) + LOAD_SMALL_INT 1 BINARY_OP 13 (+=) STORE_NAME 0 (x) - - 2 JUMP_BACKWARD 7 (to L1) + JUMP_BACKWARD 8 (to L1) """ dis_traceback = """\ @@ -407,8 +442,8 @@ def wrap_func_w_kwargs(): %4d NOP -%4d L1: LOAD_CONST 1 (1) - LOAD_CONST 2 (0) +%4d L1: LOAD_SMALL_INT 1 + LOAD_SMALL_INT 0 --> BINARY_OP 11 (/) POP_TOP @@ -465,18 +500,18 @@ def _fstring(a, b, c, d): %3d LOAD_FAST 0 (a) FORMAT_SIMPLE - LOAD_CONST 1 (' ') + LOAD_CONST 0 (' ') LOAD_FAST 1 (b) - LOAD_CONST 2 ('4') + LOAD_CONST 1 ('4') FORMAT_WITH_SPEC - LOAD_CONST 1 (' ') + LOAD_CONST 0 (' ') LOAD_FAST 2 (c) CONVERT_VALUE 2 (repr) FORMAT_SIMPLE - LOAD_CONST 1 (' ') + LOAD_CONST 0 (' ') LOAD_FAST 3 (d) CONVERT_VALUE 2 (repr) - LOAD_CONST 2 ('4') + LOAD_CONST 1 ('4') FORMAT_WITH_SPEC BUILD_STRING 7 RETURN_VALUE @@ -491,21 +526,27 @@ def _with(c): %4d RESUME 0 %4d LOAD_FAST 0 (c) - BEFORE_WITH + COPY 1 + LOAD_SPECIAL 1 (__exit__) + SWAP 2 + SWAP 3 + LOAD_SPECIAL 0 (__enter__) + CALL 0 L1: POP_TOP -%4d LOAD_CONST 1 (1) +%4d LOAD_SMALL_INT 1 STORE_FAST 1 (x) %4d L2: LOAD_CONST 0 (None) LOAD_CONST 0 (None) LOAD_CONST 0 (None) - CALL 2 + CALL 3 POP_TOP -%4d LOAD_CONST 2 (2) +%4d LOAD_SMALL_INT 2 STORE_FAST 2 (y) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE %4d L3: PUSH_EXC_INFO WITH_EXCEPT_START @@ -516,17 +557,19 @@ def _with(c): L5: POP_EXCEPT POP_TOP POP_TOP + POP_TOP -%4d LOAD_CONST 2 (2) +%4d LOAD_SMALL_INT 2 STORE_FAST 2 (y) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE -- L6: COPY 3 POP_EXCEPT RERAISE 1 ExceptionTable: - L1 to L2 -> L3 [1] lasti - L3 to L5 -> L6 [3] lasti + L1 to L2 -> L3 [2] lasti + L3 to L5 -> L6 [4] lasti """ % (_with.__code__.co_firstlineno, _with.__code__.co_firstlineno + 1, _with.__code__.co_firstlineno + 2, @@ -547,7 +590,12 @@ async def _asyncwith(c): L1: RESUME 0 %4d LOAD_FAST 0 (c) - BEFORE_ASYNC_WITH + COPY 1 + LOAD_SPECIAL 3 (__aexit__) + SWAP 2 + SWAP 3 + LOAD_SPECIAL 2 (__aenter__) + CALL 0 GET_AWAITABLE 1 LOAD_CONST 0 (None) L2: SEND 3 (to L5) @@ -557,13 +605,13 @@ async def _asyncwith(c): L5: END_SEND L6: POP_TOP -%4d LOAD_CONST 1 (1) +%4d LOAD_SMALL_INT 1 STORE_FAST 1 (x) %4d L7: LOAD_CONST 0 (None) LOAD_CONST 0 (None) LOAD_CONST 0 (None) - CALL 2 + CALL 3 GET_AWAITABLE 2 LOAD_CONST 0 (None) L8: SEND 3 (to L11) @@ -573,14 +621,15 @@ async def _asyncwith(c): L11: END_SEND POP_TOP -%4d LOAD_CONST 2 (2) +%4d LOAD_SMALL_INT 2 STORE_FAST 2 (y) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE %4d L12: CLEANUP_THROW - L13: JUMP_BACKWARD_NO_INTERRUPT 25 (to L5) + L13: JUMP_BACKWARD_NO_INTERRUPT 26 (to L5) L14: CLEANUP_THROW - L15: JUMP_BACKWARD_NO_INTERRUPT 9 (to L11) + L15: JUMP_BACKWARD_NO_INTERRUPT 10 (to L11) L16: PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 @@ -598,10 +647,12 @@ async def _asyncwith(c): L23: POP_EXCEPT POP_TOP POP_TOP + POP_TOP -%4d LOAD_CONST 2 (2) +%4d LOAD_SMALL_INT 2 STORE_FAST 2 (y) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE -- L24: COPY 3 POP_EXCEPT @@ -610,16 +661,16 @@ async def _asyncwith(c): RERAISE 1 ExceptionTable: L1 to L3 -> L25 [0] lasti - L3 to L4 -> L12 [3] + L3 to L4 -> L12 [4] L4 to L6 -> L25 [0] lasti - L6 to L7 -> L16 [1] lasti + L6 to L7 -> L16 [2] lasti L7 to L9 -> L25 [0] lasti L9 to L10 -> L14 [2] L10 to L13 -> L25 [0] lasti L14 to L15 -> L25 [0] lasti - L16 to L18 -> L24 [3] lasti - L18 to L19 -> L20 [6] - L19 to L23 -> L24 [3] lasti + L16 to L18 -> L24 [4] lasti + L18 to L19 -> L20 [7] + L19 to L23 -> L24 [4] lasti L23 to L25 -> L25 [0] lasti """ % (_asyncwith.__code__.co_firstlineno, _asyncwith.__code__.co_firstlineno + 1, @@ -688,7 +739,8 @@ def _tryfinallyconst(b): PUSH_NULL CALL 0 POP_TOP - RETURN_CONST 1 (1) + LOAD_SMALL_INT 1 + RETURN_VALUE -- L1: PUSH_EXC_INFO @@ -733,7 +785,7 @@ def foo(x): %4d LOAD_FAST 0 (y) BUILD_TUPLE 1 - LOAD_CONST 1 () + LOAD_CONST 0 () MAKE_FUNCTION SET_FUNCTION_ATTRIBUTE 8 (closure) STORE_FAST 1 (foo) @@ -782,6 +834,7 @@ def foo(x): POP_TOP L1: RESUME 0 LOAD_FAST 0 (.0) + GET_ITER L2: FOR_ITER 10 (to L3) STORE_FAST 1 (z) LOAD_DEREF 2 (x) @@ -793,7 +846,8 @@ def foo(x): JUMP_BACKWARD 12 (to L2) L3: END_FOR POP_TOP - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE -- L4: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) RERAISE 1 @@ -830,9 +884,9 @@ def loop_test(): %3d RESUME_CHECK 0 %3d BUILD_LIST 0 - LOAD_CONST 1 ((1, 2, 3)) + LOAD_CONST 0 ((1, 2, 3)) LIST_EXTEND 1 - LOAD_CONST 2 (3) + LOAD_SMALL_INT 3 BINARY_OP 5 (*) GET_ITER L1: FOR_ITER_LIST 14 (to L2) @@ -840,13 +894,14 @@ def loop_test(): %3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST 0 (i) - CALL_PY_WITH_DEFAULTS 1 + CALL_PY_GENERAL 1 POP_TOP JUMP_BACKWARD 16 (to L1) %3d L2: END_FOR POP_TOP - RETURN_CONST 0 (None) + LOAD_CONST_IMMORTAL 1 (None) + RETURN_VALUE """ % (loop_test.__code__.co_firstlineno, loop_test.__code__.co_firstlineno + 1, loop_test.__code__.co_firstlineno + 2, @@ -858,12 +913,13 @@ def extended_arg_quick(): dis_extended_arg_quick_code = """\ %3d RESUME 0 -%3d LOAD_CONST 1 (Ellipsis) +%3d LOAD_CONST 0 (Ellipsis) EXTENDED_ARG 1 UNPACK_EX 256 POP_TOP STORE_FAST 0 (_) - RETURN_CONST 0 (None) + LOAD_CONST 1 (None) + RETURN_VALUE """% (extended_arg_quick.__code__.co_firstlineno, extended_arg_quick.__code__.co_firstlineno + 1,) @@ -950,6 +1006,82 @@ def test_dis(self): def test_dis_with_offsets(self): self.do_disassembly_test(_f, dis_f_with_offsets, show_offsets=True) + @requires_debug_ranges() + def test_dis_with_all_positions(self): + def format_instr_positions(instr): + values = tuple('?' if p is None else p for p in instr.positions) + return '%s:%s-%s:%s' % (values[0], values[2], values[1], values[3]) + + instrs = list(dis.get_instructions(_f)) + for instr in instrs: + with self.subTest(instr=instr): + self.assertTrue(all(p is not None for p in instr.positions)) + positions = tuple(map(format_instr_positions, instrs)) + expected = dis_f_with_positions_format % positions + self.do_disassembly_test(_f, expected, show_positions=True) + + @requires_debug_ranges() + def test_dis_with_some_positions(self): + code = ("def f():\n" + " try: pass\n" + " finally:pass") + f = compile(ast.parse(code), "?", "exec").co_consts[0] + + expect = '\n'.join([ + '1:0-1:0 RESUME 0', + '', + '2:3-3:15 NOP', + '', + '3:11-3:15 LOAD_CONST 0 (None)', + '3:11-3:15 RETURN_VALUE', + '', + ' -- L1: PUSH_EXC_INFO', + '', + '3:11-3:15 RERAISE 0', + '', + ' -- L2: COPY 3', + ' -- POP_EXCEPT', + ' -- RERAISE 1', + 'ExceptionTable:', + ' L1 to L2 -> L2 [1] lasti', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + + @requires_debug_ranges() + def test_dis_with_linenos_but_no_columns(self): + code = "def f():\n\tx = 1" + tree = ast.parse(code) + func = tree.body[0] + ass_x = func.body[0].targets[0] + # remove columns information but keep line information + ass_x.col_offset = ass_x.end_col_offset = -1 + f = compile(tree, "?", "exec").co_consts[0] + + expect = '\n'.join([ + '1:0-1:0 RESUME 0', + '', + '2:5-2:6 LOAD_SMALL_INT 1', + '2:?-2:? STORE_FAST 0 (x)', + '2:?-2:? LOAD_CONST 0 (None)', + '2:?-2:? RETURN_VALUE', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + + def test_dis_with_no_positions(self): + def f(): + pass + + f.__code__ = f.__code__.replace(co_linetable=b'') + expect = '\n'.join([ + ' RESUME 0', + ' LOAD_CONST 0 (None)', + ' RETURN_VALUE', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + def test_bug_708901(self): self.do_disassembly_test(bug708901, dis_bug708901) @@ -1014,6 +1146,7 @@ def test_disassemble_str(self): self.do_disassembly_test(expr_str, dis_expr_str) self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str) self.do_disassembly_test(annot_stmt_str, dis_annot_stmt_str) + self.do_disassembly_test(fn_with_annotate_str, dis_fn_with_annotate_str) self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str) def test_disassemble_bytes(self): @@ -1152,7 +1285,7 @@ def test_binary_specialize(self): 0 RESUME_CHECK 0 1 LOAD_NAME 0 (a) - LOAD_CONST 0 (0) + LOAD_SMALL_INT 0 %s RETURN_VALUE """ @@ -1172,7 +1305,7 @@ def test_load_attr_specialize(self): load_attr_quicken = """\ 0 RESUME_CHECK 0 - 1 LOAD_CONST 0 ('a') + 1 LOAD_CONST_IMMORTAL 0 ('a') LOAD_ATTR_SLOT 0 (__class__) RETURN_VALUE """ @@ -1189,7 +1322,7 @@ def test_call_specialize(self): 1 LOAD_NAME 0 (str) PUSH_NULL - LOAD_CONST 0 (1) + LOAD_SMALL_INT 1 CALL_STR_1 1 RETURN_VALUE """ @@ -1201,21 +1334,42 @@ def test_call_specialize(self): @cpython_only @requires_specialization def test_loop_quicken(self): - import _testinternalcapi # Loop can trigger a quicken where the loop is located - self.code_quicken(loop_test, 1) + self.code_quicken(loop_test, 4) got = self.get_disassembly(loop_test, adaptive=True) expected = dis_loop_test_quickened_code - if _testinternalcapi.get_optimizer(): - # We *may* see ENTER_EXECUTOR in the disassembly. This is a - # temporary hack to keep the test working until dis is able to - # handle the instruction correctly (GH-112383): - got = got.replace( - "ENTER_EXECUTOR 16", - "JUMP_BACKWARD 16 (to L1)", - ) self.do_disassembly_compare(got, expected) + @cpython_only + @requires_specialization + def test_loop_with_conditional_at_end_is_quickened(self): + def for_loop_true(x): + for i in range(10): + if x: + pass + + for_loop_true(True) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_true, adaptive=True)) + + def for_loop_false(x): + for i in range(10): + if x: + pass + + for_loop_false(False) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_false, adaptive=True)) + + def while_loop(): + i = 0 + while i < 10: + i += 1 + + while_loop() + self.assertIn('COMPARE_OP_INT', + self.get_disassembly(while_loop, adaptive=True)) + @cpython_only def test_extended_arg_quick(self): got = self.get_disassembly(extended_arg_quick) @@ -1311,7 +1465,7 @@ def get_disassembly(self, func, lasti=-1, wrapper=True, **kwargs): Kw-only arguments: 0 Number of locals: 1 Stack size: \\d+ -Flags: OPTIMIZED, NEWLOCALS +Flags: OPTIMIZED, NEWLOCALS, HAS_DOCSTRING Constants: {code_info_consts} Names: @@ -1337,8 +1491,8 @@ def f(c=c): Stack size: \\d+ Flags: OPTIMIZED, NEWLOCALS, VARARGS, VARKEYWORDS, GENERATOR Constants: - 0: None - 1: + 0: + 1: None Variable names: 0: a 1: b @@ -1356,10 +1510,12 @@ def f(c=c): 2: [abedfxyz] 3: [abedfxyz] 4: [abedfxyz] - 5: [abedfxyz]""" + 5: [abedfxyz] + 6: [abedfxyz] + 7: [abedfxyz]""" # NOTE: the order of the cell variables above depends on dictionary order! -co_tricky_nested_f = tricky.__func__.__code__.co_consts[1] +co_tricky_nested_f = tricky.__func__.__code__.co_consts[0] code_info_tricky_nested_f = """\ Filename: (.*) @@ -1392,8 +1548,6 @@ def f(c=c): Number of locals: 0 Stack size: \\d+ Flags: 0x0 -Constants: - 0: 1 Names: 0: x""" @@ -1407,8 +1561,7 @@ def f(c=c): Stack size: \\d+ Flags: 0x0 Constants: - 0: 1 - 1: None + 0: None Names: 0: x""" @@ -1422,8 +1575,7 @@ def f(c=c): Stack size: \\d+ Flags: 0x0 Constants: - 0: 0 - 1: 1 + 0: None Names: 0: x""" @@ -1444,7 +1596,6 @@ async def async_def(): Flags: OPTIMIZED, NEWLOCALS, COROUTINE Constants: 0: None - 1: 1 Names: 0: b 1: c @@ -1528,9 +1679,9 @@ def jumpy(): # End fodder for opinfo generation tests expected_outer_line = 1 _line_offset = outer.__code__.co_firstlineno - 1 -code_object_f = outer.__code__.co_consts[1] +code_object_f = outer.__code__.co_consts[0] expected_f_line = code_object_f.co_firstlineno - _line_offset -code_object_inner = code_object_f.co_consts[1] +code_object_inner = code_object_f.co_consts[0] expected_inner_line = code_object_inner.co_firstlineno - _line_offset expected_jumpy_line = 1 @@ -1583,198 +1734,204 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=52, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=26, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='BUILD_LIST', opcode=47, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='BUILD_MAP', opcode=48, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None), - Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None), + Instruction(opname='MAKE_CELL', opcode=92, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=92, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=48, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=107, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=80, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=80, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=1, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=1, argval=1, argrepr='', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_LIST', opcode=43, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_MAP', opcode=44, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=62, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=52, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=26, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None), - Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=58, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=92, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=92, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=48, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=107, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=80, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=80, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=80, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=80, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=62, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=88, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=58, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=80, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=80, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=80, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=80, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=84, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=36, start_offset=36, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='GET_ITER', opcode=19, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='FOR_ITER', opcode=72, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='END_FOR', opcode=11, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=87, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=208, argrepr='to L9', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=6, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=45, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=176, argrepr='to L7', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=7, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=1, argval=190, argrepr='to L8', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=20, argval=230, argrepr='to L10', offset=188, start_offset=188, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=190, start_offset=190, starts_line=True, line_number=11, label=8, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=192, start_offset=192, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=208, argrepr='to L9', offset=200, start_offset=200, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=40, argval=128, argrepr='to L6', offset=204, start_offset=204, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=9, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='NOP', opcode=30, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=10, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=7, argval=0, argrepr='0', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=45, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=110, arg=1, argval='dodgy', argrepr='dodgy', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=248, start_offset=248, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=258, start_offset=258, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=270, start_offset=270, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=2, argval=2, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=286, start_offset=286, starts_line=True, line_number=28, label=11, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=296, start_offset=296, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=306, start_offset=306, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=310, start_offset=310, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='WITH_EXCEPT_START', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=1, argval=328, argrepr='to L12', offset=322, start_offset=322, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=12, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=26, argval=286, argrepr='to L11', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=338, start_offset=338, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=346, start_offset=346, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='CHECK_EXC_MATCH', opcode=7, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=14, argval=390, argrepr='to L13', offset=358, start_offset=358, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=364, start_offset=364, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=374, start_offset=374, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=52, argval=286, argrepr='to L11', offset=388, start_offset=388, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=390, start_offset=390, starts_line=True, line_number=22, label=13, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=392, start_offset=392, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=410, start_offset=410, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=424, start_offset=424, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='GET_ITER', opcode=16, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='FOR_ITER', opcode=67, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=107, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=4, argval=4, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=54, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=72, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=6, argval=6, argrepr='', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=54, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=72, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=74, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=83, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=33, argval=194, argrepr='to L8', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=1, argval=1, argrepr='', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=42, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=107, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=6, argval=6, argrepr='', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=54, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=2, argval=176, argrepr='to L6', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=72, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=4, argval=4, argrepr='', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=54, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=2, argval=192, argrepr='to L7', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=72, arg=39, argval=114, argrepr='to L5', offset=188, start_offset=188, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_FORWARD', opcode=74, arg=11, argval=216, argrepr='to L9', offset=192, start_offset=192, starts_line=True, line_number=17, label=7, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=79, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=1, argval=1, argrepr='', offset=218, start_offset=218, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=0, argval=0, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=42, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=57, arg=1, argval=1, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=90, arg=1, argval=1, argrepr='__exit__', offset=232, start_offset=232, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=112, arg=2, argval=2, argrepr='', offset=234, start_offset=234, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=112, arg=3, argval=3, argrepr='', offset=236, start_offset=236, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=90, arg=0, argval=0, argrepr='__enter__', offset=238, start_offset=238, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=0, argval=0, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=107, arg=1, argval='dodgy', argrepr='dodgy', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=250, start_offset=250, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=79, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=260, start_offset=260, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=276, start_offset=276, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=3, argval=3, argrepr='', offset=278, start_offset=278, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=286, start_offset=286, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=288, start_offset=288, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=79, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=298, start_offset=298, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=310, start_offset=310, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=41, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=1, argval=332, argrepr='to L11', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='RERAISE', opcode=100, arg=2, argval=2, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=73, arg=28, argval=288, argrepr='to L10', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=344, start_offset=344, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=352, start_offset=352, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=14, argval=396, argrepr='to L12', offset=364, start_offset=364, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=370, start_offset=370, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=79, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=380, start_offset=380, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=390, start_offset=390, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=392, start_offset=392, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=73, arg=54, argval=288, argrepr='to L10', offset=394, start_offset=394, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=0, argval=0, argrepr='', offset=396, start_offset=396, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=398, start_offset=398, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=402, start_offset=402, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=404, start_offset=404, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=406, start_offset=406, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=79, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=416, start_offset=416, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=0, argval=0, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=430, start_offset=430, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, label=None, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=4, start_offset=4, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), ] @@ -1841,6 +1998,7 @@ def test_co_positions(self): (2, 2, 8, 9), (1, 3, 0, 1), (1, 3, 0, 1), + (1, 3, 0, 1), (1, 3, 0, 1) ] self.assertEqual(positions, expected) @@ -1920,6 +2078,24 @@ def f(x, y, z): dis.dis(f.__code__, file=output, show_caches=True) self.assertIn("L1:", output.getvalue()) + def test_is_op_format(self): + output = io.StringIO() + dis.dis("a is b", file=output, show_caches=True) + self.assertIn("IS_OP 0 (is)", output.getvalue()) + + output = io.StringIO() + dis.dis("a is not b", file=output, show_caches=True) + self.assertIn("IS_OP 1 (is not)", output.getvalue()) + + def test_contains_op_format(self): + output = io.StringIO() + dis.dis("a in b", file=output, show_caches=True) + self.assertIn("CONTAINS_OP 0 (in)", output.getvalue()) + + output = io.StringIO() + dis.dis("a not in b", file=output, show_caches=True) + self.assertIn("CONTAINS_OP 1 (not in)", output.getvalue()) + def test_baseopname_and_baseopcode(self): # Standard instructions for name, code in dis.opmap.items(): @@ -1986,6 +2162,22 @@ def f(opcode, oparg, offset, *init_args): self.assertEqual(f(opcode.opmap["BINARY_OP"], 3, *args), (3, '<<')) self.assertEqual(f(opcode.opmap["CALL_INTRINSIC_1"], 2, *args), (2, 'INTRINSIC_IMPORT_STAR')) + def test_custom_arg_resolver(self): + class MyArgResolver(dis.ArgResolver): + def offset_from_jump_arg(self, op, arg, offset): + return arg + 1 + + def get_label_for_offset(self, offset): + return 2 * offset + + def f(opcode, oparg, offset, *init_args): + arg_resolver = MyArgResolver(*init_args) + return arg_resolver.get_argval_argrepr(opcode, oparg, offset) + offset = 42 + self.assertEqual(f(opcode.opmap["JUMP_BACKWARD"], 1, offset), (2, 'to L4')) + self.assertEqual(f(opcode.opmap["SETUP_FINALLY"], 2, offset), (3, 'to L6')) + + def get_instructions(self, code): return dis._get_instructions_bytes(code) @@ -2136,20 +2328,20 @@ def test_bytecode_co_positions(self): class TestBytecodeTestCase(BytecodeTestCase): def test_assert_not_in_with_op_not_in_bytecode(self): code = compile("a = 1", "", "exec") - self.assertInBytecode(code, "LOAD_CONST", 1) + self.assertInBytecode(code, "LOAD_SMALL_INT", 1) self.assertNotInBytecode(code, "LOAD_NAME") self.assertNotInBytecode(code, "LOAD_NAME", "a") def test_assert_not_in_with_arg_not_in_bytecode(self): code = compile("a = 1", "", "exec") - self.assertInBytecode(code, "LOAD_CONST") - self.assertInBytecode(code, "LOAD_CONST", 1) + self.assertInBytecode(code, "LOAD_SMALL_INT") + self.assertInBytecode(code, "LOAD_SMALL_INT", 1) self.assertNotInBytecode(code, "LOAD_CONST", 2) def test_assert_not_in_with_arg_in_bytecode(self): code = compile("a = 1", "", "exec") with self.assertRaises(AssertionError): - self.assertNotInBytecode(code, "LOAD_CONST", 1) + self.assertNotInBytecode(code, "LOAD_SMALL_INT", 1) class TestFinderMethods(unittest.TestCase): def test__find_imports(self): diff --git a/Lib/test/test_doctest/decorator_mod.py b/Lib/test/test_doctest/decorator_mod.py new file mode 100644 index 00000000000000..9f106888411202 --- /dev/null +++ b/Lib/test/test_doctest/decorator_mod.py @@ -0,0 +1,10 @@ +# This module is used in `doctest_lineno.py`. +import functools + + +def decorator(f): + @functools.wraps(f) + def inner(): + return f() + + return inner diff --git a/Lib/test/test_doctest/doctest_lineno.py b/Lib/test/test_doctest/doctest_lineno.py index 677c569cf710eb..0dbcd9a11eaba2 100644 --- a/Lib/test/test_doctest/doctest_lineno.py +++ b/Lib/test/test_doctest/doctest_lineno.py @@ -67,3 +67,12 @@ def property_with_doctest(self): # https://github.com/python/cpython/issues/99433 str_wrapper = object().__str__ + + +# https://github.com/python/cpython/issues/115392 +from test.test_doctest.decorator_mod import decorator + +@decorator +@decorator +def func_with_docstring_wrapped(): + """Some unrelated info.""" diff --git a/Lib/test/test_doctest/sample_doctest_skip.py b/Lib/test/test_doctest/sample_doctest_skip.py new file mode 100644 index 00000000000000..1b83dec1f8c4dc --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_skip.py @@ -0,0 +1,49 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving skips. +""" + +def no_skip_pass(): + """ + >>> 2 + 2 + 4 + """ + +def no_skip_fail(): + """ + >>> 2 + 2 + 5 + """ + +def single_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + """ + +def double_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 # doctest: +SKIP + 6 + """ + +def partial_skip_pass(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 + 6 + """ + +def partial_skip_fail(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 2 + 2 + 5 + """ + +def no_examples(): + """A docstring with no examples should not be counted as run or skipped.""" diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 7015255db1f7f0..b1e165fe16b54f 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -4,7 +4,6 @@ from test import support from test.support import import_helper -from test.support.pty_helper import FakeInput # used in doctests import doctest import functools import os @@ -18,8 +17,12 @@ import contextlib -if not support.has_subprocess_support: - raise unittest.SkipTest("test_CLI requires subprocess support.") +def doctest_skip_if(condition): + def decorator(func): + if condition and support.HAVE_DOCSTRINGS: + func.__doc__ = ">>> pass # doctest: +SKIP" + return func + return decorator # NOTE: There are some additional tests relating to interaction with @@ -466,7 +469,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -685,6 +688,7 @@ def basics(): r""" None test.test_doctest.doctest_lineno.MethodWrapper.method_without_docstring 61 test.test_doctest.doctest_lineno.MethodWrapper.property_with_doctest 4 test.test_doctest.doctest_lineno.func_with_docstring + 77 test.test_doctest.doctest_lineno.func_with_docstring_wrapped 12 test.test_doctest.doctest_lineno.func_with_doctest None test.test_doctest.doctest_lineno.func_without_docstring @@ -734,7 +738,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 830 < len(tests) < 860 # approximate number of objects with docstrings + >>> 750 < len(tests) < 800 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests @@ -887,6 +891,10 @@ def basics(): r""" DocTestRunner is used to run DocTest test cases, and to accumulate statistics. Here's a simple DocTest case we can use: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... ''' ... >>> x = 12 @@ -941,6 +949,8 @@ def basics(): r""" 6 ok TestResults(failed=1, attempted=3) + + >>> _colorize.COLORIZE = save_colorize """ def verbose_flag(): r""" The `verbose` flag makes the test runner generate more detailed @@ -1016,12 +1026,16 @@ def exceptions(): r""" lines between the first line and the type/value may be omitted or replaced with any other string: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... ''' ... >>> x = 12 ... >>> print(x//0) ... Traceback (most recent call last): - ... ZeroDivisionError: integer division or modulo by zero + ... ZeroDivisionError: division by zero ... ''' >>> test = doctest.DocTestFinder().find(f)[0] >>> doctest.DocTestRunner(verbose=False).run(test) @@ -1038,7 +1052,7 @@ def exceptions(): r""" ... >>> print('pre-exception output', x//0) ... pre-exception output ... Traceback (most recent call last): - ... ZeroDivisionError: integer division or modulo by zero + ... ZeroDivisionError: division by zero ... ''' >>> test = doctest.DocTestFinder().find(f)[0] >>> doctest.DocTestRunner(verbose=False).run(test) @@ -1049,7 +1063,7 @@ def exceptions(): r""" print('pre-exception output', x//0) Exception raised: ... - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero TestResults(failed=1, attempted=2) Exception messages may contain newlines: @@ -1244,8 +1258,10 @@ def exceptions(): r""" Exception raised: Traceback (most recent call last): ... - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero TestResults(failed=1, attempted=1) + + >>> _colorize.COLORIZE = save_colorize """ def displayhook(): r""" Test that changing sys.displayhook doesn't matter for doctest. @@ -1287,6 +1303,10 @@ def optionflags(): r""" The DONT_ACCEPT_TRUE_FOR_1 flag disables matches between True/False and 1/0: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... '>>> True\n1\n' @@ -1706,6 +1726,7 @@ def optionflags(): r""" Clean up. >>> del doctest.OPTIONFLAGS_BY_NAME[unlikely] + >>> _colorize.COLORIZE = save_colorize """ @@ -1716,6 +1737,10 @@ def option_directives(): r""" single example. To turn an option on for an example, follow that example with a comment of the form ``# doctest: +OPTION``: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): r''' ... >>> print(list(range(10))) # should fail: no ellipsis ... [0, 1, ..., 9] @@ -1923,6 +1948,8 @@ def option_directives(): r""" >>> test = doctest.DocTestParser().get_doctest(s, {}, 's', 's.py', 0) Traceback (most recent call last): ValueError: line 0 of the doctest for s has an option directive on a line with no example: '# doctest: +ELLIPSIS' + + >>> _colorize.COLORIZE = save_colorize """ def test_testsource(): r""" @@ -1977,6 +2004,7 @@ def test_debug(): r""" Create some fake stdin input, to feed to the debugger: + >>> from test.support.pty_helper import FakeInput >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput(['next', 'print(x)', 'continue']) @@ -2006,6 +2034,10 @@ def test_pdb_set_trace(): with a version that restores stdout. This is necessary for you to see debugger output. + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> doc = ''' ... >>> x = 42 ... >>> raise Exception('clé') @@ -2020,6 +2052,7 @@ def test_pdb_set_trace(): To demonstrate this, we'll create a fake standard input that captures our debugger input: + >>> from test.support.pty_helper import FakeInput >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ ... 'print(x)', # print data defined by the example @@ -2028,8 +2061,7 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin - --Return-- - > (1)()->None + > (1)() -> import pdb; pdb.set_trace() (Pdb) print(x) 42 @@ -2059,8 +2091,7 @@ def test_pdb_set_trace(): ... runner.run(test) ... finally: ... sys.stdin = real_stdin - --Return-- - > (3)calls_set_trace()->None + > (3)calls_set_trace() -> import pdb; pdb.set_trace() (Pdb) print(y) 2 @@ -2086,6 +2117,7 @@ def test_pdb_set_trace(): >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ + ... 'step', # return event of g ... 'list', # list source from example 2 ... 'next', # return from g() ... 'list', # list source from example 1 @@ -2096,6 +2128,9 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin ... # doctest: +NORMALIZE_WHITESPACE + > (3)g() + -> import pdb; pdb.set_trace() + (Pdb) step --Return-- > (3)g()->None -> import pdb; pdb.set_trace() @@ -2128,6 +2163,8 @@ def test_pdb_set_trace(): Got: 9 TestResults(failed=1, attempted=3) + + >>> _colorize.COLORIZE = save_colorize """ def test_pdb_set_trace_nested(): @@ -2156,8 +2193,10 @@ def test_pdb_set_trace_nested(): >>> parser = doctest.DocTestParser() >>> runner = doctest.DocTestRunner(verbose=False) >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> from test.support.pty_helper import FakeInput >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ + ... 'step', ... 'print(y)', # print data defined in the function ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', ... 'up', 'print(x)', @@ -2171,6 +2210,9 @@ def test_pdb_set_trace_nested(): ... finally: ... sys.stdin = real_stdin ... # doctest: +REPORT_NDIFF + > (4)calls_set_trace() + -> import pdb; pdb.set_trace() + (Pdb) step > (5)calls_set_trace() -> self.f1() (Pdb) print(y) @@ -2246,6 +2288,16 @@ def test_DocTestSuite(): >>> suite.run(unittest.TestResult()) + If all examples in a docstring are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_skip') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 2 + We can use the current module: >>> suite = test.test_doctest.sample_doctest.test_suite() @@ -2358,25 +2410,6 @@ def test_DocFileSuite(): >>> suite.run(unittest.TestResult()) - Support for using a package's __loader__.get_data() is also - provided. - - >>> import unittest, pkgutil, test - >>> added_loader = False - >>> if not hasattr(test, '__loader__'): - ... test.__loader__ = pkgutil.get_loader(test) - ... added_loader = True - >>> try: - ... suite = doctest.DocFileSuite('test_doctest.txt', - ... 'test_doctest2.txt', - ... 'test_doctest4.txt', - ... package='test.test_doctest') - ... suite.run(unittest.TestResult()) - ... finally: - ... if added_loader: - ... del test.__loader__ - - '/' should be used as a path separator. It will be converted to a native separator at run time: @@ -2417,6 +2450,18 @@ def test_DocFileSuite(): Traceback (most recent call last): ValueError: Package may only be specified for module-relative paths. + If all examples in a file are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest4.txt', + ... 'test_doctest_skip.txt') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 1 + You can specify initial global variables: >>> suite = doctest.DocFileSuite('test_doctest.txt', @@ -2518,7 +2563,7 @@ def __call__(self, *args, **kwargs): self.func(*args, **kwargs) @Wrapper -def test_look_in_unwrapped(): +def wrapped(): """ Docstrings in wrapped functions must be detected as well. @@ -2526,6 +2571,35 @@ def test_look_in_unwrapped(): 'one other test' """ +def test_look_in_unwrapped(): + """ + Ensure that wrapped doctests work correctly. + + >>> import doctest + >>> doctest.run_docstring_examples( + ... wrapped, {}, name=wrapped.__name__, verbose=True) + Finding tests in wrapped + Trying: + 'one other test' + Expecting: + 'one other test' + ok + """ + +@doctest_skip_if(support.check_impl_detail(cpython=False)) +def test_wrapped_c_func(): + """ + # https://github.com/python/cpython/issues/117692 + >>> import binascii + >>> from test.test_doctest.decorator_mod import decorator + + >>> c_func_wrapped = decorator(binascii.b2a_hex) + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(c_func_wrapped) + >>> for test in tests: + ... print(test.lineno, test.name) + None b2a_hex + """ + def test_unittest_reportflags(): """Default unittest reporting flags can be set to control reporting @@ -2611,7 +2685,11 @@ def test_testfile(): r""" called with the name of a file, which is taken to be relative to the calling module. The return value is (#failures, #tests). -We don't want `-v` in sys.argv for these tests. +We don't want color or `-v` in sys.argv for these tests. + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> save_argv = sys.argv >>> if '-v' in sys.argv: @@ -2627,9 +2705,9 @@ def test_testfile(): r""" ... NameError: name 'favorite_color' is not defined ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in test_doctest.txt - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. TestResults(failed=1, attempted=2) >>> doctest.master = None # Reset master. @@ -2656,9 +2734,9 @@ def test_testfile(): r""" Got: 'red' ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in test_doctest.txt - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. TestResults(failed=1, attempted=2) >>> doctest.master = None # Reset master. @@ -2688,10 +2766,10 @@ def test_testfile(): r""" b ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in test_doctest.txt - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. @@ -2748,7 +2826,7 @@ def test_testfile(): r""" ********************************************************************** ... ********************************************************************** - 1 items had failures: + 1 item had failures: 2 of 2 in test_doctest4.txt ***Test Failed*** 2 failures. TestResults(failed=2, attempted=2) @@ -2771,14 +2849,15 @@ def test_testfile(): r""" Expecting: 'b\u0105r' ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in test_doctest4.txt - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. >>> sys.argv = save_argv + >>> _colorize.COLORIZE = save_colorize """ class TestImporter(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): @@ -2916,6 +2995,10 @@ def test_testmod(): r""" def test_unicode(): """ Check doctest with a non-ascii filename: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> doc = ''' ... >>> raise Exception('clé') ... ''' @@ -2941,8 +3024,12 @@ def test_unicode(): """ raise Exception('clé') Exception: clé TestResults(failed=1, attempted=1) + + >>> _colorize.COLORIZE = save_colorize """ + +@doctest_skip_if(not support.has_subprocess_support) def test_CLI(): r""" The doctest module can be used to run doctests against an arbitrary file. These tests test this CLI functionality. @@ -2996,10 +3083,10 @@ def test_CLI(): r""" Expecting: 'a' ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in myfile.doc - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. Now we'll write a couple files, one with three tests, the other a python module @@ -3073,7 +3160,7 @@ def test_CLI(): r""" Got: 'ajkml' ********************************************************************** - 1 items had failures: + 1 item had failures: 2 of 3 in myfile.doc ***Test Failed*** 2 failures. @@ -3100,9 +3187,9 @@ def test_CLI(): r""" Got: 'abcdef' ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in myfile.doc - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. The fifth test uses verbose with the two options, so we should get verbose success output for the tests in both files: @@ -3125,10 +3212,10 @@ def test_CLI(): r""" Expecting: 'a...l' ok - 1 items passed all tests: + 1 item passed all tests: 3 tests in myfile.doc - 3 tests in 1 items. - 3 passed and 0 failed. + 3 tests in 1 item. + 3 passed. Test passed. Trying: 1 + 1 @@ -3140,12 +3227,12 @@ def test_CLI(): r""" Expecting: 'abc def' ok - 1 items had no tests: + 1 item had no tests: myfile2 - 1 items passed all tests: + 1 item passed all tests: 2 tests in myfile2.test_func 2 tests in 2 items. - 2 passed and 0 failed. + 2 passed. Test passed. We should also check some typical error cases. @@ -3233,6 +3320,10 @@ def test_run_doctestsuite_multiple_times(): def test_exception_with_note(note): """ + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> test_exception_with_note('Note') Traceback (most recent call last): ... @@ -3282,6 +3373,8 @@ def test_exception_with_note(note): ValueError: message note TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize """ exc = ValueError('Text') exc.add_note(note) @@ -3362,6 +3455,10 @@ def test_syntax_error_subclass_from_stdlib(): def test_syntax_error_with_incorrect_expected_note(): """ + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... r''' ... >>> exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) @@ -3390,6 +3487,8 @@ def test_syntax_error_with_incorrect_expected_note(): note1 note2 TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize """ diff --git a/Lib/test/test_doctest/test_doctest_skip.txt b/Lib/test/test_doctest/test_doctest_skip.txt new file mode 100644 index 00000000000000..f340e2b8141253 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip.txt @@ -0,0 +1,4 @@ +This is a sample doctest in a text file, in which all examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 3928bbab4423c2..21bece26b893c6 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -4,7 +4,7 @@ import sys import unittest -from test.support import is_wasi, Py_DEBUG, swap_item, swap_attr +from test.support import swap_item, swap_attr, is_wasi, Py_DEBUG class RebindBuiltinsTests(unittest.TestCase): @@ -134,7 +134,7 @@ def test_eval_gives_lambda_custom_globals(self): self.assertEqual(foo(), 7) - @unittest.skipIf(is_wasi and Py_DEBUG, "stack depth too shallow in pydebug WASI") + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_load_global_specialization_failure_keeps_oparg(self): # https://github.com/python/cpython/issues/91625 class MyGlobals(dict): diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index bdb0e55f21069f..95224e19f67ce5 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -801,6 +801,10 @@ def test_get_quoted_string_header_ends_in_qcontent(self): self.assertEqual(qs.content, 'bob') self.assertEqual(qs.quoted_value, ' "bob"') + def test_get_quoted_string_cfws_only_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_quoted_string(' (foo) ') + def test_get_quoted_string_no_quoted_string(self): with self.assertRaises(errors.HeaderParseError): parser.get_quoted_string(' (ab) xyz') @@ -1135,6 +1139,10 @@ def test_get_local_part_complex_obsolete_invalid(self): '@python.org') self.assertEqual(local_part.local_part, 'Fred.A.Johnson and dogs') + def test_get_local_part_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_local_part('') + def test_get_local_part_no_part_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_local_part(' (foo) ') @@ -1387,6 +1395,10 @@ def test_get_domain_obsolete(self): '') self.assertEqual(domain.domain, 'example.com') + def test_get_domain_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_domain("") + def test_get_domain_no_non_cfws_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_domain(" (foo)\t") @@ -1512,6 +1524,10 @@ def test_get_obs_route_no_route_before_end_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_obs_route('(foo) @example.com,') + def test_get_obs_route_no_route_before_end_raises2(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('(foo) @example.com, (foo) ') + def test_get_obs_route_no_route_before_special_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_obs_route('(foo) [abc],') @@ -1520,6 +1536,14 @@ def test_get_obs_route_no_route_before_special_raises2(self): with self.assertRaises(errors.HeaderParseError): parser.get_obs_route('(foo) @example.com [abc],') + def test_get_obs_route_no_domain_after_at_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('@') + + def test_get_obs_route_no_domain_after_at_raises2(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('@example.com, @') + # get_angle_addr def test_get_angle_addr_simple(self): @@ -1646,6 +1670,14 @@ def test_get_angle_addr_ends_at_special(self): self.assertIsNone(angle_addr.route) self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + def test_get_angle_addr_empty_raise(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('') + + def test_get_angle_addr_left_angle_only_raise(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('<') + def test_get_angle_addr_no_angle_raise(self): with self.assertRaises(errors.HeaderParseError): parser.get_angle_addr('(foo) ') @@ -1805,6 +1837,32 @@ def test_get_name_addr_qs_name(self): self.assertIsNone(name_addr.route) self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + def test_get_name_addr_ending_with_dot_without_space(self): + name_addr = self._test_get_x(parser.get_name_addr, + 'John X.', + 'John X.', + '"John X."', + [errors.ObsoleteHeaderDefect], + '') + self.assertEqual(name_addr.display_name, 'John X.') + self.assertEqual(name_addr.local_part, 'jxd') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'jxd@example.com') + + def test_get_name_addr_starting_with_dot(self): + name_addr = self._test_get_x(parser.get_name_addr, + '. Doe ', + '. Doe ', + '". Doe" ', + [errors.InvalidHeaderDefect, errors.ObsoleteHeaderDefect], + '') + self.assertEqual(name_addr.display_name, '. Doe') + self.assertEqual(name_addr.local_part, 'jxd') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'jxd@example.com') + def test_get_name_addr_with_route(self): name_addr = self._test_get_x(parser.get_name_addr, '"Roy.A.Bear" <@two.example.com: dinsdale@example.com>', @@ -1831,6 +1889,10 @@ def test_get_name_addr_ends_at_special(self): self.assertIsNone(name_addr.route) self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + def test_get_name_addr_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_name_addr('') + def test_get_name_addr_no_content_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_name_addr(' (foo) ') @@ -2698,6 +2760,35 @@ def test_get_msg_id_no_angle_end(self): ) self.assertEqual(msg_id.token_type, 'msg-id') + def test_get_msg_id_empty_id_left(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("<@domain>") + + def test_get_msg_id_empty_id_right(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("") + + def test_get_msg_id_no_id_right(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("") + + def test_get_msg_id_ws_only_local(self): + msg_id = self._test_get_x( + parser.get_msg_id, + "< @domain>", + "< @domain>", + "< @domain>", + [errors.ObsoleteHeaderDefect], + "" + ) + self.assertEqual(msg_id.token_type, 'msg-id') + @parameterize @@ -2985,6 +3076,19 @@ def test_address_list_with_unicode_names_in_quotes(self): '=?utf-8?q?H=C3=BCbsch?= Kaktus ,\n' ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= \n') + def test_address_list_with_list_separator_after_fold(self): + a = 'x' * 66 + '@example.com' + to = f'{a}, "Hübsch Kaktus" ' + self._test(parser.get_address_list(to)[0], + f'{a},\n =?utf-8?q?H=C3=BCbsch?= Kaktus \n') + + a = '.' * 79 + to = f'"{a}" , "Hübsch Kaktus" ' + self._test(parser.get_address_list(to)[0], + f'{a}\n' + ' , =?utf-8?q?H=C3=BCbsch?= Kaktus ' + '\n') + # XXX Need tests with comments on various sides of a unicode token, # and with unicode tokens in the comments. Spaces inside the quotes # currently don't do the right thing. diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 39d4ace8d4a1d8..65ddbabcaa1997 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -337,6 +337,21 @@ def test_nonascii_as_string_without_cte(self): msg = email.message_from_bytes(source) self.assertEqual(msg.as_string(), expected) + def test_nonascii_as_string_with_ascii_charset(self): + m = textwrap.dedent("""\ + MIME-Version: 1.0 + Content-type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 8bit + + Test if non-ascii messages with no Content-Transfer-Encoding set + can be as_string'd: + Föö bär + """) + source = m.encode('iso-8859-1') + expected = source.decode('ascii', 'replace') + msg = email.message_from_bytes(source) + self.assertEqual(msg.as_string(), expected) + def test_nonascii_as_string_without_content_type_and_cte(self): m = textwrap.dedent("""\ MIME-Version: 1.0 @@ -4166,6 +4181,21 @@ def test_8bit_in_uuencode_body(self): self.assertEqual(msg.get_payload(decode=True), '<,.V', + [errors.ObsoleteHeaderDefect], + '"John X." ', + 'John X.', + 'jxd@example.com', + 'jxd', + 'example.com', + None), + + 'name_starting_with_dot': + ('. Doe ', + [errors.InvalidHeaderDefect, errors.ObsoleteHeaderDefect], + '". Doe" ', + '. Doe', + 'jxd@example.com', + 'jxd', + 'example.com', + None), + } # XXX: Need many more examples, and in particular some with names in @@ -1628,7 +1648,7 @@ def test_address_display_names(self): 'Lôrem ipsum dôlôr sit amet, cônsectetuer adipiscing. ' 'Suspendisse pôtenti. Aliquam nibh. Suspendisse pôtenti.', '=?utf-8?q?L=C3=B4rem_ipsum_d=C3=B4l=C3=B4r_sit_amet=2C_c' - '=C3=B4nsectetuer?=\n =?utf-8?q?adipiscing=2E_Suspendisse' + '=C3=B4nsectetuer?=\n =?utf-8?q?_adipiscing=2E_Suspendisse' '_p=C3=B4tenti=2E_Aliquam_nibh=2E?=\n Suspendisse =?utf-8' '?q?p=C3=B4tenti=2E?=', ), diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index c6b9c80efe1b54..baa35fd68e49c5 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -26,6 +26,7 @@ class PolicyAPITests(unittest.TestCase): 'raise_on_defect': False, 'mangle_from_': True, 'message_factory': None, + 'verify_generated_headers': True, } # These default values are the ones set on email.policy.default. # If any of these defaults change, the docs must be updated. @@ -294,6 +295,31 @@ def test_short_maxlen_error(self): with self.assertRaises(email.errors.HeaderParseError): policy.fold("Subject", subject) + def test_verify_generated_headers(self): + """Turning protection off allows header injection""" + policy = email.policy.default.clone(verify_generated_headers=False) + for text in ( + 'Header: Value\r\nBad: Injection\r\n', + 'Header: NoNewLine' + ): + with self.subTest(text=text): + message = email.message_from_string( + "Header: Value\r\n\r\nBody", + policy=policy, + ) + class LiteralHeader(str): + name = 'Header' + def fold(self, **kwargs): + return self + + del message['Header'] + message['Header'] = LiteralHeader(text) + + self.assertEqual( + message.as_string(), + f"{text}\nBody", + ) + # XXX: Need subclassing tests. # For adding subclassed objects, make sure the usual rules apply (subclass # wins), but that the order still works (right overrides left). diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index d04b3909efa643..4e6201e13c87f9 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -3,9 +3,7 @@ import test.support import time import unittest -import sys -import os.path -import zoneinfo + class DateTimeTests(unittest.TestCase): @@ -154,10 +152,6 @@ def test_variable_tzname(self): t1 = utils.localtime(t0) self.assertEqual(t1.tzname(), 'EET') - def test_isdst_deprecation(self): - with self.assertWarns(DeprecationWarning): - t0 = datetime.datetime(1990, 1, 1) - t1 = utils.localtime(t0, isdst=True) # Issue #24836: The timezone files are out of date (pre 2011k) # on Mac OS X Snow Leopard. diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 6c60854bbd76cc..5e886b6c8c38ec 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1,10 +1,12 @@ # Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) from test import support -from test.support import import_helper, os_helper, MS_WINDOWS +from test.libregrtest.utils import get_build_info +from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS import unittest from collections import namedtuple import contextlib +import io import json import os import os.path @@ -19,6 +21,13 @@ if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") + +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + + MACOS = (sys.platform == 'darwin') PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 @@ -41,6 +50,8 @@ INIT_LOOPS = 4 MAX_HASH_SEED = 4294967295 +ABI_THREAD = 't' if sysconfig.get_config_var('Py_GIL_DISABLED') else '' + # If we are running from a build dir, but the stdlib has been installed, # some tests need to expect different results. @@ -158,7 +169,8 @@ def run_repeated_init_and_subinterpreters(self): # Parse the line from the loop. The first line is the main # interpreter and the 3 afterward are subinterpreters. interp = Interp(*match.groups()) - if support.verbose > 1: + if support.verbose > 2: + # 5 lines per pass is super-spammy, so limit that to -vvv print(interp) self.assertTrue(interp.interp) self.assertTrue(interp.tstate) @@ -269,6 +281,10 @@ def test_pre_initialization_api(self): """ env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) out, err = self.run_embedded_interpreter("test_pre_initialization_api", env=env) + if support.verbose > 1: + print() + print(out) + print(err) if MS_WINDOWS: expected_path = self.test_exe else: @@ -286,6 +302,10 @@ def test_pre_initialization_sys_options(self): env['PYTHONPATH'] = os.pathsep.join(sys.path) out, err = self.run_embedded_interpreter( "test_pre_initialization_sys_options", env=env) + if support.verbose > 1: + print() + print(out) + print(err) expected_output = ( "sys.warnoptions: ['once', 'module', 'default']\n" "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n" @@ -352,6 +372,7 @@ def test_simple_initialization_api(self): self.assertEqual(out, 'Finalized\n' * INIT_LOOPS) @support.requires_specialization + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): # https://github.com/python/cpython/issues/92031 @@ -368,6 +389,9 @@ def is_specialized(f): opname in opcode._specialized_opmap # Exclude superinstructions: and "__" not in opname + # LOAD_CONST_IMMORTAL is "specialized", but is + # inserted during quickening. + and opname != "LOAD_CONST_IMMORTAL" ): return True return False @@ -396,6 +420,101 @@ def test_ucnhash_capi_reset(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '9\n' * INIT_LOOPS) + def test_datetime_reset_strptime(self): + code = ( + "import datetime;" + "d = datetime.datetime.strptime('2000-01-01', '%Y-%m-%d');" + "print(d.strftime('%Y%m%d'))" + ) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '20000101\n' * INIT_LOOPS) + + def test_static_types_inherited_slots(self): + script = textwrap.dedent(""" + import test.support + results = [] + for cls in test.support.iter_builtin_types(): + for attr, _ in test.support.iter_slot_wrappers(cls): + wrapper = getattr(cls, attr) + res = (cls, attr, wrapper) + results.append(res) + results = ((repr(c), a, repr(w)) for c, a, w in results) + """) + def collate_results(raw): + results = {} + for cls, attr, wrapper in raw: + key = cls, attr + assert key not in results, (results, key, wrapper) + results[key] = wrapper + return results + + ns = {} + exec(script, ns, ns) + main_results = collate_results(ns['results']) + del ns + + script += textwrap.dedent(""" + import json + import sys + text = json.dumps(list(results)) + print(text, file=sys.stderr) + """) + out, err = self.run_embedded_interpreter( + "test_repeated_init_exec", script, script) + _results = err.split('--- Loop #')[1:] + (_embedded, _reinit, + ) = [json.loads(res.rpartition(' ---\n')[-1]) for res in _results] + embedded_results = collate_results(_embedded) + reinit_results = collate_results(_reinit) + + for key, expected in main_results.items(): + cls, attr = key + for src, results in [ + ('embedded', embedded_results), + ('reinit', reinit_results), + ]: + with self.subTest(src, cls=cls, slotattr=attr): + actual = results.pop(key) + self.assertEqual(actual, expected) + self.maxDiff = None + self.assertEqual(embedded_results, {}) + self.assertEqual(reinit_results, {}) + + self.assertEqual(out, '') + + def test_getargs_reset_static_parser(self): + # Test _PyArg_Parser initializations via _PyArg_UnpackKeywords() + # https://github.com/python/cpython/issues/122334 + code = textwrap.dedent(""" + try: + import _ssl + except ModuleNotFoundError: + _ssl = None + if _ssl is not None: + _ssl.txt2obj(txt='1.3') + print('1') + + import _queue + _queue.SimpleQueue().put_nowait(item=None) + print('2') + + import _zoneinfo + _zoneinfo.ZoneInfo.clear_cache(only_keys=['Foo/Bar']) + print('3') + """) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '1\n2\n3\n' * INIT_LOOPS) + + +def config_dev_mode(preconfig, config): + preconfig['allocator'] = PYMEM_ALLOCATOR_DEBUG + preconfig['dev_mode'] = 1 + config['dev_mode'] = 1 + config['warnoptions'] = ['default'] + config['faulthandler'] = 1 + + +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): maxDiff = 4096 UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape') @@ -409,30 +528,30 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): PRE_CONFIG_COMPAT = { '_config_init': API_COMPAT, 'allocator': PYMEM_ALLOCATOR_NOT_SET, - 'parse_argv': 0, - 'configure_locale': 1, - 'coerce_c_locale': 0, - 'coerce_c_locale_warn': 0, - 'utf8_mode': 0, + 'parse_argv': False, + 'configure_locale': True, + 'coerce_c_locale': False, + 'coerce_c_locale_warn': False, + 'utf8_mode': False, } if MS_WINDOWS: PRE_CONFIG_COMPAT.update({ - 'legacy_windows_fs_encoding': 0, + 'legacy_windows_fs_encoding': False, }) PRE_CONFIG_PYTHON = dict(PRE_CONFIG_COMPAT, _config_init=API_PYTHON, - parse_argv=1, + parse_argv=True, coerce_c_locale=GET_DEFAULT_CONFIG, utf8_mode=GET_DEFAULT_CONFIG, ) PRE_CONFIG_ISOLATED = dict(PRE_CONFIG_COMPAT, _config_init=API_ISOLATED, - configure_locale=0, - isolated=1, - use_environment=0, - utf8_mode=0, - dev_mode=0, - coerce_c_locale=0, + configure_locale=False, + isolated=True, + use_environment=False, + utf8_mode=False, + dev_mode=False, + coerce_c_locale=False, ) COPY_PRE_CONFIG = [ @@ -443,35 +562,35 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT = { '_config_init': API_COMPAT, - 'isolated': 0, - 'use_environment': 1, - 'dev_mode': 0, + 'isolated': False, + 'use_environment': True, + 'dev_mode': False, - 'install_signal_handlers': 1, - 'use_hash_seed': 0, + 'install_signal_handlers': True, + 'use_hash_seed': False, 'hash_seed': 0, 'int_max_str_digits': sys.int_info.default_max_str_digits, 'cpu_count': -1, - 'faulthandler': 0, + 'faulthandler': False, 'tracemalloc': 0, 'perf_profiling': 0, - 'import_time': 0, - 'code_debug_ranges': 1, - 'show_ref_count': 0, - 'dump_refs': 0, + 'import_time': False, + 'code_debug_ranges': True, + 'show_ref_count': False, + 'dump_refs': False, 'dump_refs_file': None, - 'malloc_stats': 0, + 'malloc_stats': False, 'filesystem_encoding': GET_DEFAULT_CONFIG, 'filesystem_errors': GET_DEFAULT_CONFIG, 'pycache_prefix': None, 'program_name': GET_DEFAULT_CONFIG, - 'parse_argv': 0, + 'parse_argv': False, 'argv': [""], 'orig_argv': [], - 'xoptions': [], + 'xoptions': {}, 'warnoptions': [], 'pythonpath_env': None, @@ -484,71 +603,73 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix': GET_DEFAULT_CONFIG, 'base_exec_prefix': GET_DEFAULT_CONFIG, 'module_search_paths': GET_DEFAULT_CONFIG, - 'module_search_paths_set': 1, + 'module_search_paths_set': True, 'platlibdir': sys.platlibdir, 'stdlib_dir': GET_DEFAULT_CONFIG, - 'site_import': 1, + 'site_import': True, 'bytes_warning': 0, - 'warn_default_encoding': 0, - 'inspect': 0, - 'interactive': 0, + 'warn_default_encoding': False, + 'inspect': False, + 'interactive': False, 'optimization_level': 0, - 'parser_debug': 0, - 'write_bytecode': 1, + 'parser_debug': False, + 'write_bytecode': True, 'verbose': 0, - 'quiet': 0, - 'user_site_directory': 1, - 'configure_c_stdio': 0, - 'buffered_stdio': 1, + 'quiet': False, + 'user_site_directory': True, + 'configure_c_stdio': False, + 'buffered_stdio': True, 'stdio_encoding': GET_DEFAULT_CONFIG, 'stdio_errors': GET_DEFAULT_CONFIG, - 'skip_source_first_line': 0, + 'skip_source_first_line': False, 'run_command': None, 'run_module': None, 'run_filename': None, 'sys_path_0': None, - '_install_importlib': 1, + '_install_importlib': True, 'check_hash_pycs_mode': 'default', - 'pathconfig_warnings': 1, - '_init_main': 1, + 'pathconfig_warnings': True, + '_init_main': True, 'use_frozen_modules': not support.Py_DEBUG, - 'safe_path': 0, + 'safe_path': False, '_is_python_build': IGNORE_CONFIG, } if Py_STATS: - CONFIG_COMPAT['_pystats'] = 0 + CONFIG_COMPAT['_pystats'] = False if support.Py_DEBUG: CONFIG_COMPAT['run_presite'] = None + if support.Py_GIL_DISABLED: + CONFIG_COMPAT['enable_gil'] = -1 if MS_WINDOWS: CONFIG_COMPAT.update({ - 'legacy_windows_stdio': 0, + 'legacy_windows_stdio': False, }) CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, - configure_c_stdio=1, - parse_argv=2, + configure_c_stdio=True, + parse_argv=True, ) CONFIG_ISOLATED = dict(CONFIG_COMPAT, _config_init=API_ISOLATED, - isolated=1, - use_environment=0, - user_site_directory=0, - safe_path=1, - dev_mode=0, - install_signal_handlers=0, - use_hash_seed=0, - faulthandler=0, - tracemalloc=0, + isolated=True, + use_environment=False, + user_site_directory=False, + safe_path=True, + dev_mode=False, + install_signal_handlers=False, + use_hash_seed=False, + faulthandler=False, + tracemalloc=False, perf_profiling=0, - pathconfig_warnings=0, + pathconfig_warnings=False, ) if MS_WINDOWS: - CONFIG_ISOLATED['legacy_windows_stdio'] = 0 + CONFIG_ISOLATED['legacy_windows_stdio'] = False # global config DEFAULT_GLOBAL_CONFIG = { @@ -735,6 +856,9 @@ def check_config(self, configs, expected): if value is self.IGNORE_CONFIG: config.pop(key, None) del expected[key] + # Resolve bool/int mismatches to reduce noise in diffs + if isinstance(value, (bool, int)) and isinstance(config.get(key), (bool, int)): + expected[key] = type(config[key])(expected[key]) self.assertEqual(config, expected) def check_global_config(self, configs): @@ -824,23 +948,23 @@ def test_init_compat_config(self): def test_init_global_config(self): preconfig = { - 'utf8_mode': 1, + 'utf8_mode': True, } config = { 'program_name': './globalvar', - 'site_import': 0, - 'bytes_warning': 1, + 'site_import': False, + 'bytes_warning': True, 'warnoptions': ['default::BytesWarning'], - 'inspect': 1, - 'interactive': 1, + 'inspect': True, + 'interactive': True, 'optimization_level': 2, - 'write_bytecode': 0, - 'verbose': 1, - 'quiet': 1, - 'buffered_stdio': 0, + 'write_bytecode': False, + 'verbose': True, + 'quiet': True, + 'buffered_stdio': False, - 'user_site_directory': 0, - 'pathconfig_warnings': 0, + 'user_site_directory': False, + 'pathconfig_warnings': False, } self.check_all_configs("test_init_global_config", config, preconfig, api=API_COMPAT) @@ -848,18 +972,18 @@ def test_init_global_config(self): def test_init_from_config(self): preconfig = { 'allocator': ALLOCATOR_FOR_CONFIG, - 'utf8_mode': 1, + 'utf8_mode': True, } config = { - 'install_signal_handlers': 0, - 'use_hash_seed': 1, + 'install_signal_handlers': False, + 'use_hash_seed': True, 'hash_seed': 123, 'tracemalloc': 2, 'perf_profiling': 0, - 'import_time': 1, - 'code_debug_ranges': 0, - 'show_ref_count': 1, - 'malloc_stats': 1, + 'import_time': True, + 'code_debug_ranges': False, + 'show_ref_count': True, + 'malloc_stats': True, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', @@ -872,13 +996,13 @@ def test_init_from_config(self): '-X', 'cmdline_xoption', '-c', 'pass', 'arg2'], - 'parse_argv': 2, - 'xoptions': [ - 'config_xoption1=3', - 'config_xoption2=', - 'config_xoption3', - 'cmdline_xoption', - ], + 'parse_argv': True, + 'xoptions': { + 'config_xoption1': '3', + 'config_xoption2': '', + 'config_xoption3': True, + 'cmdline_xoption': True, + }, 'warnoptions': [ 'cmdline_warnoption', 'default::BytesWarning', @@ -886,26 +1010,26 @@ def test_init_from_config(self): ], 'run_command': 'pass\n', - 'site_import': 0, + 'site_import': False, 'bytes_warning': 1, - 'inspect': 1, - 'interactive': 1, + 'inspect': True, + 'interactive': True, 'optimization_level': 2, - 'write_bytecode': 0, + 'write_bytecode': False, 'verbose': 1, - 'quiet': 1, - 'configure_c_stdio': 1, - 'buffered_stdio': 0, - 'user_site_directory': 0, - 'faulthandler': 1, + 'quiet': True, + 'configure_c_stdio': True, + 'buffered_stdio': False, + 'user_site_directory': False, + 'faulthandler': True, 'platlibdir': 'my_platlibdir', 'module_search_paths': self.IGNORE_CONFIG, - 'safe_path': 1, + 'safe_path': True, 'int_max_str_digits': 31337, 'cpu_count': 4321, 'check_hash_pycs_mode': 'always', - 'pathconfig_warnings': 0, + 'pathconfig_warnings': False, } if Py_STATS: config['_pystats'] = 1 @@ -917,29 +1041,30 @@ def test_init_compat_env(self): 'allocator': ALLOCATOR_FOR_CONFIG, } config = { - 'use_hash_seed': 1, + 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, 'perf_profiling': 0, - 'import_time': 1, - 'code_debug_ranges': 0, - 'malloc_stats': 1, - 'inspect': 1, + 'import_time': True, + 'code_debug_ranges': False, + 'malloc_stats': True, + 'inspect': True, 'optimization_level': 2, 'pythonpath_env': '/my/path', 'pycache_prefix': 'env_pycache_prefix', - 'write_bytecode': 0, + 'write_bytecode': False, 'verbose': 1, - 'buffered_stdio': 0, + 'buffered_stdio': False, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', - 'user_site_directory': 0, - 'faulthandler': 1, + 'user_site_directory': False, + 'faulthandler': True, 'warnoptions': ['EnvVar'], 'platlibdir': 'env_platlibdir', 'module_search_paths': self.IGNORE_CONFIG, - 'safe_path': 1, + 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = 1 @@ -952,32 +1077,33 @@ def test_init_python_env(self): 'utf8_mode': 1, } config = { - 'use_hash_seed': 1, + 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, 'perf_profiling': 0, - 'import_time': 1, - 'code_debug_ranges': 0, - 'malloc_stats': 1, - 'inspect': 1, + 'import_time': True, + 'code_debug_ranges': False, + 'malloc_stats': True, + 'inspect': True, 'optimization_level': 2, 'pythonpath_env': '/my/path', 'pycache_prefix': 'env_pycache_prefix', - 'write_bytecode': 0, + 'write_bytecode': False, 'verbose': 1, - 'buffered_stdio': 0, + 'buffered_stdio': False, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', - 'user_site_directory': 0, - 'faulthandler': 1, + 'user_site_directory': False, + 'faulthandler': True, 'warnoptions': ['EnvVar'], 'platlibdir': 'env_platlibdir', 'module_search_paths': self.IGNORE_CONFIG, - 'safe_path': 1, + 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: - config['_pystats'] = 1 + config['_pystats'] = True self.check_all_configs("test_init_python_env", config, preconfig, api=API_PYTHON) @@ -998,40 +1124,38 @@ def test_init_env_dev_mode_alloc(self): api=API_COMPAT) def test_init_dev_mode(self): - preconfig = { - 'allocator': PYMEM_ALLOCATOR_DEBUG, - } + preconfig = {} config = { - 'faulthandler': 1, - 'dev_mode': 1, + 'faulthandler': True, + 'dev_mode': True, 'warnoptions': ['default'], } + config_dev_mode(preconfig, config) self.check_all_configs("test_init_dev_mode", config, preconfig, api=API_PYTHON) def test_preinit_parse_argv(self): # Pre-initialize implicitly using argv: make sure that -X dev # is used to configure the allocation in preinitialization - preconfig = { - 'allocator': PYMEM_ALLOCATOR_DEBUG, - } + preconfig = {} config = { 'argv': ['script.py'], 'orig_argv': ['python3', '-X', 'dev', '-P', 'script.py'], 'run_filename': os.path.abspath('script.py'), - 'dev_mode': 1, - 'faulthandler': 1, + 'dev_mode': True, + 'faulthandler': True, 'warnoptions': ['default'], - 'xoptions': ['dev'], - 'safe_path': 1, + 'xoptions': {'dev': True}, + 'safe_path': True, } + config_dev_mode(preconfig, config) self.check_all_configs("test_preinit_parse_argv", config, preconfig, api=API_PYTHON) def test_preinit_dont_parse_argv(self): # -X dev must be ignored by isolated preconfiguration preconfig = { - 'isolated': 0, + 'isolated': False, } argv = ["python3", "-E", "-I", "-P", @@ -1041,37 +1165,37 @@ def test_preinit_dont_parse_argv(self): config = { 'argv': argv, 'orig_argv': argv, - 'isolated': 0, + 'isolated': False, } self.check_all_configs("test_preinit_dont_parse_argv", config, preconfig, api=API_ISOLATED) def test_init_isolated_flag(self): config = { - 'isolated': 1, - 'safe_path': 1, - 'use_environment': 0, - 'user_site_directory': 0, + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, } self.check_all_configs("test_init_isolated_flag", config, api=API_PYTHON) def test_preinit_isolated1(self): # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set config = { - 'isolated': 1, - 'safe_path': 1, - 'use_environment': 0, - 'user_site_directory': 0, + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, } self.check_all_configs("test_preinit_isolated1", config, api=API_COMPAT) def test_preinit_isolated2(self): # _PyPreConfig.isolated=0, _PyCoreConfig.isolated=1 config = { - 'isolated': 1, - 'safe_path': 1, - 'use_environment': 0, - 'user_site_directory': 0, + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, } self.check_all_configs("test_preinit_isolated2", config, api=API_COMPAT) @@ -1114,12 +1238,12 @@ def modify_path(path): def test_init_sys_add(self): config = { 'faulthandler': 1, - 'xoptions': [ - 'config_xoption', - 'cmdline_xoption', - 'sysadd_xoption', - 'faulthandler', - ], + 'xoptions': { + 'config_xoption': True, + 'cmdline_xoption': True, + 'sysadd_xoption': True, + 'faulthandler': True, + }, 'warnoptions': [ 'ignore:::cmdline_warnoption', 'ignore:::sysadd_warnoption', @@ -1139,7 +1263,7 @@ def test_init_run_main(self): 'orig_argv': ['python3', '-c', code, 'arg2'], 'program_name': './python3', 'run_command': code + '\n', - 'parse_argv': 2, + 'parse_argv': True, 'sys_path_0': '', } self.check_all_configs("test_init_run_main", config, api=API_PYTHON) @@ -1154,8 +1278,8 @@ def test_init_main(self): 'arg2'], 'program_name': './python3', 'run_command': code + '\n', - 'parse_argv': 2, - '_init_main': 0, + 'parse_argv': True, + '_init_main': False, 'sys_path_0': '', } self.check_all_configs("test_init_main", config, @@ -1164,12 +1288,12 @@ def test_init_main(self): def test_init_parse_argv(self): config = { - 'parse_argv': 2, + 'parse_argv': True, 'argv': ['-c', 'arg1', '-v', 'arg3'], 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', 'run_command': 'pass\n', - 'use_environment': 0, + 'use_environment': False, } self.check_all_configs("test_init_parse_argv", config, api=API_PYTHON) @@ -1178,7 +1302,7 @@ def test_init_dont_parse_argv(self): 'parse_argv': 0, } config = { - 'parse_argv': 0, + 'parse_argv': False, 'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', @@ -1261,11 +1385,11 @@ def module_search_paths(self, prefix=None, exec_prefix=None): ver = sys.version_info return [ os.path.join(prefix, sys.platlibdir, - f'python{ver.major}{ver.minor}.zip'), + f'python{ver.major}{ver.minor}{ABI_THREAD}.zip'), os.path.join(prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}'), os.path.join(exec_prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}', 'lib-dynload'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload'), ] @contextlib.contextmanager @@ -1319,7 +1443,7 @@ def test_init_setpythonhome(self): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1360,7 +1484,7 @@ def test_init_is_python_build_with_home(self): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1491,7 +1615,7 @@ def test_init_pyvenv_cfg(self): if not MS_WINDOWS: lib_dynload = os.path.join(pyvenv_home, sys.platlibdir, - f'python{ver.major}.{ver.minor}', + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload') os.makedirs(lib_dynload) else: @@ -1533,12 +1657,12 @@ def test_init_pyvenv_cfg(self): config['base_prefix'] = pyvenv_home config['prefix'] = pyvenv_home config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') - config['use_frozen_modules'] = int(not support.Py_DEBUG) + config['use_frozen_modules'] = bool(not support.Py_DEBUG) else: # cannot reliably assume stdlib_dir here because it # depends too much on our build. But it ought to be found config['stdlib_dir'] = self.IGNORE_CONFIG - config['use_frozen_modules'] = int(not support.Py_DEBUG) + config['use_frozen_modules'] = bool(not support.Py_DEBUG) env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1586,7 +1710,6 @@ def test_global_pathconfig(self): # The global path configuration (_Py_path_config) must be a copy # of the path configuration of PyInterpreter.config (PyConfig). ctypes = import_helper.import_module('ctypes') - _testinternalcapi = import_helper.import_module('_testinternalcapi') def get_func(name): func = getattr(ctypes.pythonapi, name) @@ -1603,7 +1726,7 @@ def get_func(name): config = _testinternalcapi.get_configs()['config'] - self.assertEqual(Py_GetPath().split(os.path.pathsep), + self.assertEqual(tuple(Py_GetPath().split(os.path.pathsep)), config['module_search_paths']) self.assertEqual(Py_GetPrefix(), config['prefix']) self.assertEqual(Py_GetExecPrefix(), config['exec_prefix']) @@ -1625,16 +1748,15 @@ def test_init_warnoptions(self): 'ignore:::PySys_AddWarnOption2', # PySys_AddWarnOption() 'ignore:::PyConfig_BeforeRead', # PyConfig.warnoptions 'ignore:::PyConfig_AfterRead'] # PyWideStringList_Append() - preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG) + preconfig = {} config = { - 'dev_mode': 1, - 'faulthandler': 1, 'bytes_warning': 1, - 'warnoptions': warnoptions, 'orig_argv': ['python3', '-Wignore:::cmdline1', '-Wignore:::cmdline2'], } + config_dev_mode(preconfig, config) + config['warnoptions'] = warnoptions self.check_all_configs("test_init_warnoptions", config, preconfig, api=API_PYTHON) @@ -1647,27 +1769,57 @@ def test_init_set_config(self): self.check_all_configs("test_init_set_config", config, api=API_ISOLATED) + def test_initconfig_api(self): + preconfig = { + 'configure_locale': True, + } + config = { + 'pycache_prefix': 'conf_pycache_prefix', + 'xoptions': {'faulthandler': True}, + 'hash_seed': 10, + 'use_hash_seed': True, + 'perf_profiling': 2, + } + config_dev_mode(preconfig, config) + # Temporarily enable ignore_stderr=True to ignore warnings on JIT builds + # See gh-126255 for more information + self.check_all_configs("test_initconfig_api", config, preconfig, + api=API_ISOLATED, ignore_stderr=True) + + def test_initconfig_get_api(self): + self.run_embedded_interpreter("test_initconfig_get_api") + + def test_initconfig_exit(self): + self.run_embedded_interpreter("test_initconfig_exit") + + def test_initconfig_module(self): + self.run_embedded_interpreter("test_initconfig_module") + def test_get_argc_argv(self): self.run_embedded_interpreter("test_get_argc_argv") # ignore output def test_init_use_frozen_modules(self): tests = { - ('=on', 1), - ('=off', 0), - ('=', 1), - ('', 1), + ('=on', True), + ('=off', False), + ('=', True), + ('', True), } for raw, expected in tests: optval = f'frozen_modules{raw}' + if raw.startswith('='): + xoption_value = raw[1:] + else: + xoption_value = True config = { - 'parse_argv': 2, + 'parse_argv': True, 'argv': ['-c'], 'orig_argv': ['./argv0', '-X', optval, '-c', 'pass'], 'program_name': './argv0', 'run_command': 'pass\n', - 'use_environment': 1, - 'xoptions': [optval], + 'use_environment': True, + 'xoptions': {'frozen_modules': xoption_value}, 'use_frozen_modules': expected, } env = {'TESTFROZEN': raw[1:]} if raw else None @@ -1699,6 +1851,13 @@ def test_init_main_interpreter_settings(self): self.assertEqual(out, expected) + @threading_helper.requires_working_threading() + def test_init_in_background_thread(self): + # gh-123022: Check that running Py_Initialize() in a background + # thread doesn't crash. + out, err = self.run_embedded_interpreter("test_init_in_background_thread") + self.assertEqual(err, "") + class SetConfigTests(unittest.TestCase): def test_set_config(self): @@ -1782,6 +1941,7 @@ def test_unicode_id_init(self): # See bpo-44133 @unittest.skipIf(os.name == 'nt', 'Py_FrozenMain is not exported on Windows') + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_frozenmain(self): env = dict(os.environ) env['PYTHONUNBUFFERED'] = '1' @@ -1792,9 +1952,9 @@ def test_frozenmain(self): sys.argv ['./argv0', '-E', 'arg1', 'arg2'] config program_name: ./argv0 config executable: {executable} - config use_environment: 1 - config configure_c_stdio: 1 - config buffered_stdio: 0 + config use_environment: True + config configure_c_stdio: True + config buffered_stdio: False """).lstrip() self.assertEqual(out, expected) @@ -1829,7 +1989,11 @@ def test_no_memleak(self): @unittest.skipUnless(support.Py_DEBUG, '-X presite requires a Python debug build') def test_presite(self): - cmd = [sys.executable, "-I", "-X", "presite=test.reperf", "-c", "print('cmd')"] + cmd = [ + sys.executable, + "-I", "-X", "presite=test._test_embed_structseq", + "-c", "print('unique-python-message')", + ] proc = subprocess.run( cmd, stdout=subprocess.PIPE, @@ -1838,9 +2002,8 @@ def test_presite(self): ) self.assertEqual(proc.returncode, 0) out = proc.stdout.strip() - self.assertIn("10 times sub", out) - self.assertIn("CPU seconds", out) - self.assertIn("cmd", out) + self.assertIn("Tests passed", out) + self.assertIn("unique-python-message", out) class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase): diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index a4b36a90d8815e..6d3c91b0b6d9f9 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,7 +6,6 @@ import test.support import unittest import unittest.mock -from importlib.resources.abc import Traversable from pathlib import Path import ensurepip diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 39c1ae0ad5a078..b9e13fb8c3585e 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -447,7 +447,7 @@ def spam(cls): def test_bad_new_super(self): with self.assertRaisesRegex( TypeError, - 'has no members defined', + 'do not use .super...__new__;', ): class BadSuper(self.enum_type): def __new__(cls, value): @@ -1048,6 +1048,22 @@ class TestPlainEnumFunction(_EnumTests, _PlainOutputTests, unittest.TestCase): class TestPlainFlagClass(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag + def test_none_member(self): + class FlagWithNoneMember(Flag): + A = 1 + E = None + + self.assertEqual(FlagWithNoneMember.A.value, 1) + self.assertIs(FlagWithNoneMember.E.value, None) + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with |"): + FlagWithNoneMember.A | FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with &"): + FlagWithNoneMember.E & FlagWithNoneMember.A + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with \^"): + FlagWithNoneMember.A ^ FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be inverted"): + ~FlagWithNoneMember.E + class TestPlainFlagFunction(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag @@ -1479,6 +1495,27 @@ class SpamEnum(Enum): spam = nonmember(SpamEnumIsInner) self.assertTrue(SpamEnum.spam is SpamEnumIsInner) + def test_using_members_as_nonmember(self): + class Example(Flag): + A = 1 + B = 2 + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + + class Example(Flag): + A = auto() + B = auto() + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + def test_nested_classes_in_enum_with_member(self): """Support locally-defined nested classes.""" class Outer(Enum): @@ -1851,6 +1888,25 @@ def test_wrong_inheritance_order(self): class Wrong(Enum, str): NotHere = 'error before this point' + def test_raise_custom_error_on_creation(self): + class InvalidRgbColorError(ValueError): + def __init__(self, r, g, b): + self.r = r + self.g = g + self.b = b + super().__init__(f'({r}, {g}, {b}) is not a valid RGB color') + + with self.assertRaises(InvalidRgbColorError): + class RgbColor(Enum): + RED = (255, 0, 0) + GREEN = (0, 255, 0) + BLUE = (0, 0, 255) + INVALID = (256, 0, 0) + + def __init__(self, r, g, b): + if not all(0 <= val <= 255 for val in (r, g, b)): + raise InvalidRgbColorError(r, g, b) + def test_intenum_transitivity(self): class number(IntEnum): one = 1 @@ -3393,6 +3449,43 @@ def __new__(cls, int_value, *value_aliases): self.assertIs(Types(2), Types.NetList) self.assertIs(Types('nl'), Types.NetList) + def test_second_tuple_item_is_falsey(self): + class Cardinal(Enum): + RIGHT = (1, 0) + UP = (0, 1) + LEFT = (-1, 0) + DOWN = (0, -1) + self.assertIs(Cardinal(1, 0), Cardinal.RIGHT) + self.assertIs(Cardinal(-1, 0), Cardinal.LEFT) + + def test_no_members(self): + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Enum(7) + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Flag(7) + + def test_empty_names(self): + for nothing in '', [], {}: + for e_type in None, int: + empty_enum = Enum('empty_enum', nothing, type=e_type) + self.assertEqual(len(empty_enum), 0) + self.assertRaisesRegex(TypeError, 'has no members', empty_enum, 0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', names=0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', 0, type=int) + + def test_nonhashable_matches_hashable(self): # issue 125710 + class Directions(Enum): + DOWN_ONLY = frozenset({"sc"}) + UP_ONLY = frozenset({"cs"}) + UNRESTRICTED = frozenset({"sc", "cs"}) + self.assertIs(Directions({"sc"}), Directions.DOWN_ONLY) + class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" @@ -4017,6 +4110,8 @@ def test_global_repr_conform1(self): @reraise_if_not_enum(NoName) def test_global_enum_str(self): + self.assertEqual(repr(NoName.ONE), 'test_enum.ONE') + self.assertEqual(repr(NoName(0)), 'test_enum.NoName(0)') self.assertEqual(str(NoName.ONE & NoName.TWO), 'NoName(0)') self.assertEqual(str(NoName(0)), 'NoName(0)') @@ -4851,22 +4946,22 @@ class Color(enum.Enum) | The value of the Enum member. | | ---------------------------------------------------------------------- - | Methods inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: | - | __contains__(value) from enum.EnumType + | __contains__(value) | Return True if `value` is in `cls`. | | `value` is in `cls` if: | 1) `value` is a member of `cls`, or | 2) `value` is the value of one of the `cls`'s members. | - | __getitem__(name) from enum.EnumType + | __getitem__(name) | Return the member matching `name`. | - | __iter__() from enum.EnumType + | __iter__() | Return members in definition order. | - | __len__() from enum.EnumType + | __len__() | Return the number of members (no aliases) | | ---------------------------------------------------------------------- @@ -4891,11 +4986,11 @@ class Color(enum.Enum) | | Data and other attributes defined here: | - | YELLOW = + | CYAN = | | MAGENTA = | - | CYAN = + | YELLOW = | | ---------------------------------------------------------------------- | Data descriptors inherited from enum.Enum: @@ -4905,7 +5000,18 @@ class Color(enum.Enum) | value | | ---------------------------------------------------------------------- - | Data descriptors inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: + | + | __contains__(value) + | + | __getitem__(name) + | + | __iter__() + | + | __len__() + | + | ---------------------------------------------------------------------- + | Readonly properties inherited from enum.EnumType: | | __members__""" @@ -5111,7 +5217,57 @@ class Unhashable: self.assertIn('python', Unhashable) self.assertEqual(Unhashable.name.value, 'python') self.assertEqual(Unhashable.name.name, 'name') - _test_simple_enum(Unhashable, Unhashable) + _test_simple_enum(CheckedUnhashable, Unhashable) + ## + class CheckedComplexStatus(IntEnum): + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + # + @_simple_enum(IntEnum) + class ComplexStatus: + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + _test_simple_enum(CheckedComplexStatus, ComplexStatus) + # + # + class CheckedComplexFlag(IntFlag): + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'outer upper half' + PANTS = 2, 'lower half' + self.assertIs(CheckedComplexFlag.SHIRT, CheckedComplexFlag.VEST) + # + @_simple_enum(IntFlag) + class ComplexFlag: + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'uppert half' + PANTS = 2, 'lower half' + _test_simple_enum(CheckedComplexFlag, ComplexFlag) class MiscTestCase(unittest.TestCase): @@ -5198,7 +5354,7 @@ def test_convert_value_lookup_priority(self): filter=lambda x: x.startswith('CONVERT_TEST_')) # We don't want the reverse lookup value to vary when there are # multiple possible names for a given value. It should always - # report the first lexigraphical name in that case. + # report the first lexicographical name in that case. self.assertEqual(test_type(5).name, 'CONVERT_TEST_NAME_A') def test_convert_int(self): diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index be4fd73bfdc36b..e377383450e19d 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -1,6 +1,7 @@ """test script for a few new invalid token catches""" import sys +from codecs import BOM_UTF8 from test import support from test.support import os_helper from test.support import script_helper @@ -11,67 +12,158 @@ class EOFTestCase(unittest.TestCase): def test_EOF_single_quote(self): expect = "unterminated string literal (detected at line 1) (, line 1)" for quote in ("'", "\""): - try: + with self.assertRaises(SyntaxError) as cm: eval(f"""{quote}this is a test\ """) - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - self.assertEqual(msg.offset, 1) - else: - raise support.TestFailed + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.offset, 1) def test_EOFS(self): - expect = ("unterminated triple-quoted string literal (detected at line 1) (, line 1)") - try: - eval("""'''this is a test""") - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - self.assertEqual(msg.offset, 1) - else: - raise support.TestFailed + expect = ("unterminated triple-quoted string literal (detected at line 3) (, line 1)") + with self.assertRaises(SyntaxError) as cm: + eval("""ä = '''thîs is \na \ntest""") + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + + with self.assertRaises(SyntaxError) as cm: + eval("""ä = '''thîs is \na \ntest""".encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + + with self.assertRaises(SyntaxError) as cm: + eval(BOM_UTF8 + """ä = '''thîs is \na \ntest""".encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + + with self.assertRaises(SyntaxError) as cm: + eval("""# coding: latin1\nä = '''thîs is \na \ntest""".encode('latin1')) + self.assertEqual(str(cm.exception), "unterminated triple-quoted string literal (detected at line 4) (, line 2)") + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) def test_EOFS_with_file(self): expect = ("(, line 1)") with os_helper.temp_dir() as temp_dir: - file_name = script_helper.make_script(temp_dir, 'foo', """'''this is \na \ntest""") - rc, out, err = script_helper.assert_python_failure(file_name) - self.assertIn(b'unterminated triple-quoted string literal (detected at line 3)', err) + file_name = script_helper.make_script(temp_dir, 'foo', + """ä = '''thîs is \na \ntest""") + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 3)']) + + file_name = script_helper.make_script(temp_dir, 'foo', + """ä = '''thîs is \na \ntest""".encode()) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 3)']) + + file_name = script_helper.make_script(temp_dir, 'foo', + BOM_UTF8 + """ä = '''thîs is \na \ntest""".encode()) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 3)']) + + file_name = script_helper.make_script(temp_dir, 'foo', + """# coding: latin1\nä = '''thîs is \na \ntest""".encode('latin1')) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 4)']) @warnings_helper.ignore_warnings(category=SyntaxWarning) def test_eof_with_line_continuation(self): expect = "unexpected EOF while parsing (, line 1)" - try: + with self.assertRaises(SyntaxError) as cm: compile('"\\Xhh" \\', '', 'exec') - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - else: - raise support.TestFailed + self.assertEqual(str(cm.exception), expect) def test_line_continuation_EOF(self): """A continuation at the end of input must be an error; bpo2180.""" expect = 'unexpected EOF while parsing (, line 1)' - with self.assertRaises(SyntaxError) as excinfo: - exec('x = 5\\') - self.assertEqual(str(excinfo.exception), expect) - with self.assertRaises(SyntaxError) as excinfo: + with self.assertRaises(SyntaxError) as cm: + exec('ä = 5\\') + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: + exec('ä = 5\\'.encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: + exec('# coding:latin1\nä = 5\\'.encode('latin1')) + self.assertEqual(str(cm.exception), + 'unexpected EOF while parsing (, line 2)') + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: + exec(BOM_UTF8 + 'ä = 5\\'.encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: exec('\\') - self.assertEqual(str(excinfo.exception), expect) + self.assertEqual(str(cm.exception), expect) @unittest.skipIf(not sys.executable, "sys.executable required") def test_line_continuation_EOF_from_file_bpo2180(self): """Ensure tok_nextc() does not add too many ending newlines.""" with os_helper.temp_dir() as temp_dir: file_name = script_helper.make_script(temp_dir, 'foo', '\\') - rc, out, err = script_helper.assert_python_failure(file_name) - self.assertIn(b'unexpected EOF while parsing', err) - self.assertIn(b'line 1', err) - self.assertIn(b'\\', err) - - file_name = script_helper.make_script(temp_dir, 'foo', 'y = 6\\') - rc, out, err = script_helper.assert_python_failure(file_name) - self.assertIn(b'unexpected EOF while parsing', err) - self.assertIn(b'line 1', err) - self.assertIn(b'y = 6\\', err) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-2:], [ + ' \\', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-3][-8:], ', line 1', err) + + file_name = script_helper.make_script(temp_dir, 'foo', 'ä = 6\\') + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + ' ä = 6\\', + ' ^', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-4][-8:], ', line 1', err) + + file_name = script_helper.make_script(temp_dir, 'foo', + '# coding:latin1\n' + 'ä = 7\\'.encode('latin1')) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + ' ä = 7\\', + ' ^', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-4][-8:], ', line 2', err) + + file_name = script_helper.make_script(temp_dir, 'foo', + BOM_UTF8 + 'ä = 8\\'.encode()) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + ' ä = 8\\', + ' ^', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-4][-8:], ', line 1', err) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 20122679223843..b4fc290b1f32b6 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): e = ExceptionGroup('eg', [e]) return e diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c7e76414ff0715..5beeac3adfc065 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -7,24 +7,28 @@ import pickle import weakref import errno +from codecs import BOM_UTF8 +from itertools import product from textwrap import dedent from test.support import (captured_stderr, check_impl_detail, cpython_only, gc_collect, no_tracing, script_helper, - SuppressCrashReport) + SuppressCrashReport, + force_not_colorized) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink from test.support.warnings_helper import check_warnings from test import support try: + import _testcapi from _testcapi import INT_MAX except ImportError: + _testcapi = None INT_MAX = 2**31 - 1 - class NaiveException(Exception): def __init__(self, x): self.x = x @@ -40,6 +44,7 @@ def __str__(self): # XXX This is not really enough, each *operation* should be tested! + class ExceptionTests(unittest.TestCase): def raise_catch(self, exc, excname): @@ -301,6 +306,7 @@ def baz(): { 6 0="""''', 5, 13) + check('b"fooжжж"'.encode(), 1, 1, 1, 10) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 7) @@ -313,8 +319,8 @@ def baz(): check('def f():\n global x\n nonlocal x', 2, 3) # Errors thrown by future.c - check('from __future__ import doesnt_exist', 1, 1) - check('from __future__ import braces', 1, 1) + check('from __future__ import doesnt_exist', 1, 24) + check('from __future__ import braces', 1, 24) check('x=1\nfrom __future__ import division', 2, 1) check('foo(1=2)', 1, 5) check('def f():\n x, y: int', 2, 3) @@ -344,8 +350,8 @@ def __init__(self_): class InvalidException: pass + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi1(): - import _testcapi try: _testcapi.raise_exception(BadException, 1) except TypeError as err: @@ -355,8 +361,8 @@ def test_capi1(): else: self.fail("Expected exception") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi2(): - import _testcapi try: _testcapi.raise_exception(BadException, 0) except RuntimeError as err: @@ -369,8 +375,8 @@ def test_capi2(): else: self.fail("Expected exception") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi3(): - import _testcapi self.assertRaises(SystemError, _testcapi.raise_exception, InvalidException, 1) @@ -1331,6 +1337,29 @@ def test_unicode_errors_no_object(self): for klass in klasses: self.assertEqual(str(klass.__new__(klass)), "") + def test_unicode_error_str_does_not_crash(self): + # Test that str(UnicodeError(...)) does not crash. + # See https://github.com/python/cpython/issues/123378. + + for start, end, objlen in product( + range(-5, 5), + range(-5, 5), + range(7), + ): + obj = 'a' * objlen + with self.subTest('encode', objlen=objlen, start=start, end=end): + exc = UnicodeEncodeError('utf-8', obj, start, end, '') + self.assertIsInstance(str(exc), str) + + with self.subTest('translate', objlen=objlen, start=start, end=end): + exc = UnicodeTranslateError(obj, start, end, '') + self.assertIsInstance(str(exc), str) + + encoded = obj.encode() + with self.subTest('decode', objlen=objlen, start=start, end=end): + exc = UnicodeDecodeError('utf-8', encoded, start, end, '') + self.assertIsInstance(str(exc), str) + @no_tracing def test_badisinstance(self): # Bug #2542: if issubclass(e, MyException) raises an exception, @@ -1380,6 +1409,7 @@ def foo(): @cpython_only def test_recursion_normalizing_exception(self): + import_module("_testinternalcapi") # Issue #22898. # Test that a RecursionError is raised when tstate->recursion_depth is # equal to recursion_limit in PyErr_NormalizeException() and check @@ -1421,7 +1451,7 @@ def gen(): next(generator) recursionlimit = sys.getrecursionlimit() try: - recurse(support.EXCEEDS_RECURSION_LIMIT) + recurse(support.exceeds_recursion_limit()) finally: sys.setrecursionlimit(recursionlimit) print('Done.') @@ -1434,6 +1464,7 @@ def gen(): self.assertIn(b'Done.', out) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_recursion_normalizing_infinite_exception(self): # Issue #30697. Test that a RecursionError is raised when # maximum recursion depth has been exceeded when creating @@ -1447,7 +1478,8 @@ def test_recursion_normalizing_infinite_exception(self): """ rc, out, err = script_helper.assert_python_failure("-c", code) self.assertEqual(rc, 1) - self.assertIn(b'RecursionError: maximum recursion depth exceeded', err) + expected = b'RecursionError: maximum recursion depth exceeded' + self.assertTrue(expected in err, msg=f"{expected!r} not found in {err[:3_000]!r}... (truncated)") self.assertIn(b'Done.', out) @@ -1502,6 +1534,7 @@ def recurse_in_body_and_except(): # Python built with Py_TRACE_REFS fail with a fatal error in # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no # memory left and the size of the Python frames stack is greater than @@ -1524,6 +1557,7 @@ def recurse(cnt): self.assertIn(b'MemoryError', err) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_MemoryError(self): # PyErr_NoMemory always raises the same exception instance. # Check that the traceback is not doubled. @@ -1543,8 +1577,8 @@ def raiseMemError(): self.assertEqual(tb1, tb2) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_exception_with_doc(self): - import _testcapi doc2 = "This is a test docstring." doc4 = "This is another test docstring." @@ -1583,6 +1617,7 @@ class C(object): self.assertEqual(error5.__doc__, "") @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_cleanup(self): # Issue #5437: preallocated MemoryError instances should not keep # traceback objects alive. @@ -1673,6 +1708,7 @@ def test_unhandled(self): # Python built with Py_TRACE_REFS fail with a fatal error in # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_in_PyErr_PrintEx(self): code = """if 1: import _testcapi @@ -1791,6 +1827,7 @@ class TestException(MemoryError): gc_collect() + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_in_subinterp(self): # gh-109894: subinterpreters shouldn't count on last resort memory error # when MemoryError is raised through PyErr_NoMemory() call, @@ -1805,6 +1842,29 @@ def test_memory_error_in_subinterp(self): rc, _, err = script_helper.assert_python_ok("-c", code) self.assertIn(b'MemoryError', err) + def test_keyerror_context(self): + # Make sure that _PyErr_SetKeyError() chains exceptions + try: + err1 = None + err2 = None + try: + d = {} + try: + raise ValueError("bug") + except Exception as exc: + err1 = exc + d[1] + except Exception as exc: + err2 = exc + + self.assertIsInstance(err1, ValueError) + self.assertIsInstance(err2, KeyError) + self.assertEqual(err2.__context__, err1) + finally: + # Break any potential reference cycle + exc1 = None + exc2 = None + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): @@ -1824,6 +1884,8 @@ def f(): except self.failureException: with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) + else: + self.fail("assertRaisesRegex should have failed.") self.assertIn("aab", err.getvalue()) @@ -1974,16 +2036,21 @@ def test_copy_pickle(self): self.assertEqual(exc.path, orig.path) +def run_script(source): + if isinstance(source, str): + with open(TESTFN, 'w', encoding='utf-8') as testfile: + testfile.write(dedent(source)) + else: + with open(TESTFN, 'wb') as testfile: + testfile.write(source) + _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) + return err.decode('utf-8').splitlines() + class AssertionErrorTests(unittest.TestCase): def tearDown(self): unlink(TESTFN) - def write_source(self, source): - with open(TESTFN, 'w') as testfile: - testfile.write(dedent(source)) - _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - return err.decode('utf-8').splitlines() - + @force_not_colorized def test_assertion_error_location(self): cases = [ ('assert None', @@ -2014,11 +2081,32 @@ def test_assertion_error_location(self): 'AssertionError', ], ), - ('assert 1 > 2, "message"', + ('assert 1 > 2, "messäge"', [ - ' assert 1 > 2, "message"', + ' assert 1 > 2, "messäge"', ' ^^^^^', - 'AssertionError: message', + 'AssertionError: messäge', + ], + ), + ('assert 1 > 2, "messäge"'.encode(), + [ + ' assert 1 > 2, "messäge"', + ' ^^^^^', + 'AssertionError: messäge', + ], + ), + ('# coding: latin1\nassert 1 > 2, "messäge"'.encode('latin1'), + [ + ' assert 1 > 2, "messäge"', + ' ^^^^^', + 'AssertionError: messäge', + ], + ), + (BOM_UTF8 + 'assert 1 > 2, "messäge"'.encode(), + [ + ' assert 1 > 2, "messäge"', + ' ^^^^^', + 'AssertionError: messäge', ], ), @@ -2056,10 +2144,11 @@ def test_assertion_error_location(self): ), ] for source, expected in cases: - with self.subTest(source): - result = self.write_source(source) + with self.subTest(source=source): + result = run_script(source) self.assertEqual(result[-3:], expected) + @force_not_colorized def test_multiline_not_highlighted(self): cases = [ (""" @@ -2086,12 +2175,15 @@ def test_multiline_not_highlighted(self): ), ] for source, expected in cases: - with self.subTest(source): - result = self.write_source(source) + with self.subTest(source=source): + result = run_script(source) self.assertEqual(result[-len(expected):], expected) class SyntaxErrorTests(unittest.TestCase): + maxDiff = None + + @force_not_colorized def test_range_of_offsets(self): cases = [ # Basic range from 2->7 @@ -2183,45 +2275,106 @@ def test_range_of_offsets(self): the_exception = exc def test_encodings(self): + self.addCleanup(unlink, TESTFN) source = ( '# -*- coding: cp437 -*-\n' '"┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))\n' ) - try: - with open(TESTFN, 'w', encoding='cp437') as testfile: - testfile.write(source) - rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - err = err.decode('utf-8').splitlines() - - self.assertEqual(err[-3], ' "┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))') - self.assertEqual(err[-2], ' ^^^^^^^^^^^^^^^^^^^') - finally: - unlink(TESTFN) + err = run_script(source.encode('cp437')) + self.assertEqual(err[-3], ' "┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))') + self.assertEqual(err[-2], ' ^^^^^^^^^^^^^^^^^^^') # Check backwards tokenizer errors source = '# -*- coding: ascii -*-\n\n(\n' - try: - with open(TESTFN, 'w', encoding='ascii') as testfile: - testfile.write(source) - rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - err = err.decode('utf-8').splitlines() - - self.assertEqual(err[-3], ' (') - self.assertEqual(err[-2], ' ^') - finally: - unlink(TESTFN) + err = run_script(source) + self.assertEqual(err[-3], ' (') + self.assertEqual(err[-2], ' ^') def test_non_utf8(self): # Check non utf-8 characters - try: - with open(TESTFN, 'bw') as testfile: - testfile.write(b"\x89") - rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - err = err.decode('utf-8').splitlines() + self.addCleanup(unlink, TESTFN) + err = run_script(b"\x89") + self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1]) - self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1]) - finally: - unlink(TESTFN) + def test_string_source(self): + def try_compile(source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + return cm.exception + + exc = try_compile('return "ä"') + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('return "ä"'.encode()) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile(BOM_UTF8 + 'return "ä"'.encode()) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('# coding: latin1\nreturn "ä"'.encode('latin1')) + self.assertEqual(str(exc), "'return' outside function (, line 2)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('return "ä" #' + 'ä'*1000) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('return "ä" # ' + 'ä'*1000) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + def test_file_source(self): + self.addCleanup(unlink, TESTFN) + err = run_script('return "ä"') + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script('return "ä"'.encode()) + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script(BOM_UTF8 + 'return "ä"'.encode()) + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script('# coding: latin1\nreturn "ä"'.encode('latin1')) + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script('return "ä" #' + 'ä'*1000) + self.assertEqual(err[-2:], [ + ' ^^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + self.assertEqual(err[-3][:100], ' return "ä" #' + 'ä'*84) + + err = run_script('return "ä" # ' + 'ä'*1000) + self.assertEqual(err[-2:], [ + ' ^^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + self.assertEqual(err[-3][:100], ' return "ä" # ' + 'ä'*83) def test_attributes_new_constructor(self): args = ("bad.py", 1, 2, "abcdefg", 1, 100) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py new file mode 100644 index 00000000000000..d896fec73d1971 --- /dev/null +++ b/Lib/test/test_external_inspection.py @@ -0,0 +1,84 @@ +import unittest +import os +import textwrap +import importlib +import sys +from test.support import os_helper, SHORT_TIMEOUT +from test.support.script_helper import make_script + +import subprocess + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _testexternalinspection import PROCESS_VM_READV_SUPPORTED + from _testexternalinspection import get_stack_trace +except ImportError: + raise unittest.SkipTest("Test only runs when _testexternalinspection is available") + +def _make_test_script(script_dir, script_basename, source): + to_return = make_script(script_dir, script_basename, source) + importlib.invalidate_caches() + return to_return + +class TestGetStackTrace(unittest.TestCase): + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import time, sys, os + def bar(): + for x in range(100): + if x == 50: + baz() + def baz(): + foo() + + def foo(): + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(1000) + + bar() + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + + expected_stack_trace = [ + 'foo', + 'baz', + 'bar', + '' + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_self_trace(self): + stack_trace = get_stack_trace(os.getpid()) + self.assertEqual(stack_trace[0], "test_self_trace") + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index d0473500a17735..60815be96e14eb 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -236,7 +236,7 @@ def test_sigfpe(self): faulthandler._sigfpe() """, 3, - 'Floating point exception') + 'Floating-point exception') @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGBUS'), 'need signal.SIGBUS') @@ -266,6 +266,7 @@ def test_sigill(self): 5, 'Illegal instruction') + @unittest.skipIf(_testcapi is None, 'need _testcapi') def check_fatal_error_func(self, release_gil): # Test that Py_FatalError() dumps a traceback with support.SuppressCrashReport(): @@ -574,10 +575,12 @@ def run(self): lineno = 8 else: lineno = 10 + # When the traceback is dumped, the waiter thread may be in the + # `self.running.set()` call or in `self.stop.wait()`. regex = r""" ^Thread 0x[0-9a-f]+ \(most recent call first\): (?: File ".*threading.py", line [0-9]+ in [_a-z]+ - ){{1,3}} File "", line 23 in run + ){{1,3}} File "", line (?:22|23) in run File ".*threading.py", line [0-9]+ in _bootstrap_inner File ".*threading.py", line [0-9]+ in _bootstrap diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 6d734d052454d3..bb784cbe2ce036 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -117,7 +117,9 @@ def test_fcntl_bad_file(self): @cpython_only def test_fcntl_bad_file_overflow(self): - from _testcapi import INT_MAX, INT_MIN + _testcapi = import_module("_testcapi") + INT_MAX = _testcapi.INT_MAX + INT_MIN = _testcapi.INT_MIN # Issue 15989 with self.assertRaises(OverflowError): fcntl.fcntl(INT_MAX + 1, fcntl.F_SETFL, os.O_NONBLOCK) @@ -129,8 +131,9 @@ def test_fcntl_bad_file_overflow(self): fcntl.fcntl(BadFile(INT_MIN - 1), fcntl.F_SETFL, os.O_NONBLOCK) @unittest.skipIf( - platform.machine().startswith('arm') and platform.system() == 'Linux', - "ARM Linux returns EINVAL for F_NOTIFY DN_MULTISHOT") + (platform.machine().startswith("arm") and platform.system() == "Linux") + or platform.system() == "Android", + "this platform returns EINVAL for F_NOTIFY DN_MULTISHOT") def test_fcntl_64_bit(self): # Issue #1309352: fcntl shouldn't fail when the third arg fits in a # C 'long' but not in a C 'int'. @@ -188,7 +191,7 @@ def test_lockf_share(self): @cpython_only def test_flock_overflow(self): - import _testcapi + _testcapi = import_module("_testcapi") self.assertRaises(OverflowError, fcntl.flock, _testcapi.INT_MAX+1, fcntl.LOCK_SH) diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index 9df55278693531..1206032a93566e 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -126,7 +126,7 @@ def testMethods(self): # it must also return None if an exception was given try: 1/0 - except: + except ZeroDivisionError: self.assertEqual(self.f.__exit__(*sys.exc_info()), None) def testReadWhenWriting(self): diff --git a/Lib/test/test_file_eintr.py b/Lib/test/test_file_eintr.py index f9236f45ca4be8..466f94e389f986 100644 --- a/Lib/test/test_file_eintr.py +++ b/Lib/test/test_file_eintr.py @@ -21,8 +21,8 @@ raise unittest.SkipTest("test module requires subprocess") # Test import all of the things we're about to try testing up front. -import _io -import _pyio +import _io # noqa: F401 +import _pyio # noqa: F401 @unittest.skipUnless(os.name == 'posix', 'tests requires a posix system.') class TestFileIOSignalInterrupt: diff --git a/Lib/test/test_filecmp.py b/Lib/test/test_filecmp.py index 9b5ac12bccc58f..2c83667b22feb4 100644 --- a/Lib/test/test_filecmp.py +++ b/Lib/test/test_filecmp.py @@ -1,5 +1,6 @@ import filecmp import os +import re import shutil import tempfile import unittest @@ -8,11 +9,24 @@ from test.support import os_helper +def _create_file_shallow_equal(template_path, new_path): + """create a file with the same size and mtime but different content.""" + shutil.copy2(template_path, new_path) + with open(new_path, 'r+b') as f: + next_char = bytearray(f.read(1)) + next_char[0] = (next_char[0] + 1) % 256 + f.seek(0) + f.write(next_char) + shutil.copystat(template_path, new_path) + assert os.stat(new_path).st_size == os.stat(template_path).st_size + assert os.stat(new_path).st_mtime == os.stat(template_path).st_mtime + class FileCompareTestCase(unittest.TestCase): def setUp(self): self.name = os_helper.TESTFN self.name_same = os_helper.TESTFN + '-same' self.name_diff = os_helper.TESTFN + '-diff' + self.name_same_shallow = os_helper.TESTFN + '-same-shallow' data = 'Contents of file go here.\n' for name in [self.name, self.name_same, self.name_diff]: with open(name, 'w', encoding="utf-8") as output: @@ -20,12 +34,19 @@ def setUp(self): with open(self.name_diff, 'a+', encoding="utf-8") as output: output.write('An extra line.\n') + + for name in [self.name_same, self.name_diff]: + shutil.copystat(self.name, name) + + _create_file_shallow_equal(self.name, self.name_same_shallow) + self.dir = tempfile.gettempdir() def tearDown(self): os.unlink(self.name) os.unlink(self.name_same) os.unlink(self.name_diff) + os.unlink(self.name_same_shallow) def test_matching(self): self.assertTrue(filecmp.cmp(self.name, self.name), @@ -36,12 +57,17 @@ def test_matching(self): "Comparing file to identical file fails") self.assertTrue(filecmp.cmp(self.name, self.name_same, shallow=False), "Comparing file to identical file fails") + self.assertTrue(filecmp.cmp(self.name, self.name_same_shallow), + "Shallow identical files should be considered equal") def test_different(self): self.assertFalse(filecmp.cmp(self.name, self.name_diff), "Mismatched files compare as equal") self.assertFalse(filecmp.cmp(self.name, self.dir), "File and directory compare as equal") + self.assertFalse(filecmp.cmp(self.name, self.name_same_shallow, + shallow=False), + "Mismatched file to shallow identical file compares as equal") def test_cache_clear(self): first_compare = filecmp.cmp(self.name, self.name_same, shallow=False) @@ -56,6 +82,8 @@ def setUp(self): self.dir = os.path.join(tmpdir, 'dir') self.dir_same = os.path.join(tmpdir, 'dir-same') self.dir_diff = os.path.join(tmpdir, 'dir-diff') + self.dir_diff_file = os.path.join(tmpdir, 'dir-diff-file') + self.dir_same_shallow = os.path.join(tmpdir, 'dir-same-shallow') # Another dir is created under dir_same, but it has a name from the # ignored list so it should not affect testing results. @@ -63,7 +91,17 @@ def setUp(self): self.caseinsensitive = os.path.normcase('A') == os.path.normcase('a') data = 'Contents of file go here.\n' - for dir in (self.dir, self.dir_same, self.dir_diff, self.dir_ignored): + + shutil.rmtree(self.dir, True) + os.mkdir(self.dir) + subdir_path = os.path.join(self.dir, 'subdir') + os.mkdir(subdir_path) + dir_file_path = os.path.join(self.dir, "file") + with open(dir_file_path, 'w', encoding="utf-8") as output: + output.write(data) + + for dir in (self.dir_same, self.dir_same_shallow, + self.dir_diff, self.dir_diff_file): shutil.rmtree(dir, True) os.mkdir(dir) subdir_path = os.path.join(dir, 'subdir') @@ -72,14 +110,25 @@ def setUp(self): fn = 'FiLe' # Verify case-insensitive comparison else: fn = 'file' - with open(os.path.join(dir, fn), 'w', encoding="utf-8") as output: - output.write(data) + + file_path = os.path.join(dir, fn) + + if dir is self.dir_same_shallow: + _create_file_shallow_equal(dir_file_path, file_path) + else: + shutil.copy2(dir_file_path, file_path) with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: output.write('An extra file.\n') + # Add different file2 with respect to dir_diff + with open(os.path.join(self.dir_diff_file, 'file2'), 'w', encoding="utf-8") as output: + output.write('Different contents.\n') + + def tearDown(self): - for dir in (self.dir, self.dir_same, self.dir_diff): + for dir in (self.dir, self.dir_same, self.dir_diff, + self.dir_same_shallow, self.dir_diff_file): shutil.rmtree(dir) def test_default_ignores(self): @@ -102,25 +151,65 @@ def test_cmpfiles(self): shallow=False), "Comparing directory to same fails") - # Add different file2 - with open(os.path.join(self.dir, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - - self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_same, + self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_diff_file, ['file', 'file2']) == (['file'], ['file2'], []), "Comparing mismatched directories fails") + def test_cmpfiles_invalid_names(self): + # See https://github.com/python/cpython/issues/122400. + for file, desc in [ + ('\x00', 'NUL bytes filename'), + (__file__ + '\x00', 'filename with embedded NUL bytes'), + ("\uD834\uDD1E.py", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + ('a' * 1_000_000, 'very long filename'), + ]: + for other_dir in [self.dir, self.dir_same, self.dir_diff]: + with self.subTest(f'cmpfiles: {desc}', other_dir=other_dir): + res = filecmp.cmpfiles(self.dir, other_dir, [file]) + self.assertTupleEqual(res, ([], [], [file])) + + def test_dircmp_invalid_names(self): + for bad_dir, desc in [ + ('\x00', 'NUL bytes dirname'), + (f'Top{os.sep}Mid\x00', 'dirname with embedded NUL bytes'), + ("\uD834\uDD1E", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + ('a' * 1_000_000, 'very long dirname'), + ]: + d1 = filecmp.dircmp(self.dir, bad_dir) + d2 = filecmp.dircmp(bad_dir, self.dir) + for target in [ + # attributes where os.listdir() raises OSError or ValueError + 'left_list', 'right_list', + 'left_only', 'right_only', 'common', + ]: + with self.subTest(f'dircmp(ok, bad): {desc}', target=target): + with self.assertRaises((OSError, ValueError)): + getattr(d1, target) + with self.subTest(f'dircmp(bad, ok): {desc}', target=target): + with self.assertRaises((OSError, ValueError)): + getattr(d2, target) def _assert_lists(self, actual, expected): """Assert that two lists are equal, up to ordering.""" self.assertEqual(sorted(actual), sorted(expected)) + def test_dircmp_identical_directories(self): + self._assert_dircmp_identical_directories() + self._assert_dircmp_identical_directories(shallow=False) + + def test_dircmp_different_file(self): + self._assert_dircmp_different_file() + self._assert_dircmp_different_file(shallow=False) - def test_dircmp(self): + def test_dircmp_different_directories(self): + self._assert_dircmp_different_directories() + self._assert_dircmp_different_directories(shallow=False) + + def _assert_dircmp_identical_directories(self, **options): # Check attributes for comparison of two identical directories left_dir, right_dir = self.dir, self.dir_same - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) if self.caseinsensitive: @@ -142,9 +231,10 @@ def test_dircmp(self): ] self._assert_report(d.report, expected_report) + def _assert_dircmp_different_directories(self, **options): # Check attributes for comparison of two different directories (right) left_dir, right_dir = self.dir, self.dir_diff - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'subdir']) @@ -164,12 +254,8 @@ def test_dircmp(self): self._assert_report(d.report, expected_report) # Check attributes for comparison of two different directories (left) - left_dir, right_dir = self.dir, self.dir_diff - shutil.move( - os.path.join(self.dir_diff, 'file2'), - os.path.join(self.dir, 'file2') - ) - d = filecmp.dircmp(left_dir, right_dir) + left_dir, right_dir = self.dir_diff, self.dir + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'file2', 'subdir']) @@ -180,27 +266,62 @@ def test_dircmp(self): self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, []) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), - "Only in {} : ['file2']".format(self.dir), + "diff {} {}".format(self.dir_diff, self.dir), + "Only in {} : ['file2']".format(self.dir_diff), "Identical files : ['file']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) - # Add different file2 - with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - d = filecmp.dircmp(self.dir, self.dir_diff) + + def _assert_dircmp_different_file(self, **options): + # A different file2 + d = filecmp.dircmp(self.dir_diff, self.dir_diff_file, **options) self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, ['file2']) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), + "diff {} {}".format(self.dir_diff, self.dir_diff_file), "Identical files : ['file']", "Differing files : ['file2']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) + def test_dircmp_no_shallow_different_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow, shallow=False) + self.assertEqual(d.same_files, []) + self.assertEqual(d.diff_files, ['file']) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Differing files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + + def test_dircmp_shallow_same_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow) + self.assertEqual(d.same_files, ['file']) + self.assertEqual(d.diff_files, []) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Identical files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + + def test_dircmp_shallow_is_keyword_only(self): + with self.assertRaisesRegex( + TypeError, + re.escape("dircmp.__init__() takes from 3 to 5 positional arguments but 6 were given"), + ): + filecmp.dircmp(self.dir, self.dir_same, None, None, True) + self.assertIsInstance( + filecmp.dircmp(self.dir, self.dir_same, None, None, shallow=True), + filecmp.dircmp, + ) + def test_dircmp_subdirs_type(self): """Check that dircmp.subdirs respects subclassing.""" class MyDirCmp(filecmp.dircmp): diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index b3ad41d2588c4c..b340ef7ed1621c 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -23,10 +23,9 @@ from io import BytesIO, StringIO from fileinput import FileInput, hook_encoded -from pathlib import Path from test.support import verbose -from test.support.os_helper import TESTFN +from test.support.os_helper import TESTFN, FakePath from test.support.os_helper import unlink as safe_unlink from test.support import os_helper from test import support @@ -478,23 +477,23 @@ def test_iteration_buffering(self): self.assertRaises(StopIteration, next, fi) self.assertEqual(src.linesread, []) - def test_pathlib_file(self): - t1 = Path(self.writeTmp("Pathlib file.")) + def test_pathlike_file(self): + t1 = FakePath(self.writeTmp("Path-like file.")) with FileInput(t1, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') self.assertEqual(fi.lineno(), 1) self.assertEqual(fi.filelineno(), 1) self.assertEqual(fi.filename(), os.fspath(t1)) - def test_pathlib_file_inplace(self): - t1 = Path(self.writeTmp('Pathlib file.')) + def test_pathlike_file_inplace(self): + t1 = FakePath(self.writeTmp('Path-like file.')) with FileInput(t1, inplace=True, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') print('Modified %s' % line) with open(t1, encoding="utf-8") as f: - self.assertEqual(f.read(), 'Modified Pathlib file.\n') + self.assertEqual(f.read(), 'Modified Path-like file.\n') class MockFileInput: diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 06d5a8abf32083..0611d1749f41c1 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -17,6 +17,7 @@ TESTFN, TESTFN_ASCII, TESTFN_UNICODE, make_bad_fd, ) from test.support.warnings_helper import check_warnings +from test.support.import_helper import import_module from collections import UserList import _io # C implementation of io @@ -597,7 +598,7 @@ class COtherFileTests(OtherFileTests, unittest.TestCase): @cpython_only def testInvalidFd_overflow(self): # Issue 15989 - import _testcapi + _testcapi = import_module("_testcapi") self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MAX + 1) self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MIN - 1) diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py index 1d134430909d84..42871f8a09b16b 100644 --- a/Lib/test/test_finalization.py +++ b/Lib/test/test_finalization.py @@ -13,7 +13,7 @@ def with_tp_del(cls): class C(object): def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.with_tp_del') + raise unittest.SkipTest('requires _testcapi.with_tp_del') return C try: @@ -22,7 +22,7 @@ def __new__(cls, *args, **kwargs): def without_gc(cls): class C: def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.without_gc') + raise unittest.SkipTest('requires _testcapi.without_gc') return C from test import support diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 5bd640617d6874..f588e16b70123a 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -8,6 +8,7 @@ import unittest from test import support +from test.support.testcase import FloatsAreIdenticalMixin from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) from math import isinf, isnan, copysign, ldexp @@ -32,6 +33,28 @@ class FloatSubclass(float): class OtherFloatSubclass(float): pass +class MyIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + +class MyInt: + def __init__(self, value): + self.value = value + + def __int__(self): + return self.value + +class FloatLike: + def __init__(self, value): + self.value = value + + def __float__(self): + return self.value + + class GeneralFloatCases(unittest.TestCase): def test_float(self): @@ -152,7 +175,7 @@ def check(s): # non-UTF-8 byte string check(b'123\xa0') - @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE') + @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE', '') def test_float_with_comma(self): # set locale to something that doesn't use '.' for the decimal point # float must not accept the locale specific decimal point but @@ -181,10 +204,6 @@ def test_float_with_comma(self): def test_floatconversion(self): # Make sure that calls to __float__() work properly - class Foo1(object): - def __float__(self): - return 42. - class Foo2(float): def __float__(self): return 42. @@ -206,45 +225,29 @@ class FooStr(str): def __float__(self): return float(str(self)) + 1 - self.assertEqual(float(Foo1()), 42.) + self.assertEqual(float(FloatLike(42.)), 42.) self.assertEqual(float(Foo2()), 42.) with self.assertWarns(DeprecationWarning): self.assertEqual(float(Foo3(21)), 42.) self.assertRaises(TypeError, float, Foo4(42)) self.assertEqual(float(FooStr('8')), 9.) - class Foo5: - def __float__(self): - return "" - self.assertRaises(TypeError, time.sleep, Foo5()) + self.assertRaises(TypeError, time.sleep, FloatLike("")) # Issue #24731 - class F: - def __float__(self): - return OtherFloatSubclass(42.) + f = FloatLike(OtherFloatSubclass(42.)) with self.assertWarns(DeprecationWarning): - self.assertEqual(float(F()), 42.) + self.assertEqual(float(f), 42.) with self.assertWarns(DeprecationWarning): - self.assertIs(type(float(F())), float) + self.assertIs(type(float(f)), float) with self.assertWarns(DeprecationWarning): - self.assertEqual(FloatSubclass(F()), 42.) + self.assertEqual(FloatSubclass(f), 42.) with self.assertWarns(DeprecationWarning): - self.assertIs(type(FloatSubclass(F())), FloatSubclass) - - class MyIndex: - def __init__(self, value): - self.value = value - def __index__(self): - return self.value + self.assertIs(type(FloatSubclass(f)), FloatSubclass) self.assertEqual(float(MyIndex(42)), 42.0) self.assertRaises(OverflowError, float, MyIndex(2**2000)) - - class MyInt: - def __int__(self): - return 42 - - self.assertRaises(TypeError, float, MyInt()) + self.assertRaises(TypeError, float, MyInt(42)) def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): @@ -277,6 +280,37 @@ def __new__(cls, arg, newarg=None): self.assertEqual(float(u), 2.5) self.assertEqual(u.newarg, 3) + def assertEqualAndType(self, actual, expected_value, expected_type): + self.assertEqual(actual, expected_value) + self.assertIs(type(actual), expected_type) + + def test_from_number(self, cls=float): + def eq(actual, expected): + self.assertEqual(actual, expected) + self.assertIs(type(actual), cls) + + eq(cls.from_number(3.14), 3.14) + eq(cls.from_number(314), 314.0) + eq(cls.from_number(OtherFloatSubclass(3.14)), 3.14) + eq(cls.from_number(FloatLike(3.14)), 3.14) + eq(cls.from_number(MyIndex(314)), 314.0) + + x = cls.from_number(NAN) + self.assertTrue(x != x) + self.assertIs(type(x), cls) + if cls is float: + self.assertIs(cls.from_number(NAN), NAN) + + self.assertRaises(TypeError, cls.from_number, '3.14') + self.assertRaises(TypeError, cls.from_number, b'3.14') + self.assertRaises(TypeError, cls.from_number, 3.14j) + self.assertRaises(TypeError, cls.from_number, MyInt(314)) + self.assertRaises(TypeError, cls.from_number, {}) + self.assertRaises(TypeError, cls.from_number) + + def test_from_number_subclass(self): + self.test_from_number(FloatSubclass) + def test_is_integer(self): self.assertFalse((1.1).is_integer()) self.assertTrue((1.).is_integer()) @@ -829,7 +863,7 @@ def test_short_repr(self): self.assertEqual(repr(float(negs)), str(float(negs))) @support.requires_IEEE_754 -class RoundTestCase(unittest.TestCase): +class RoundTestCase(unittest.TestCase, FloatsAreIdenticalMixin): def test_inf_nan(self): self.assertRaises(OverflowError, round, INF) @@ -859,10 +893,10 @@ def test_large_n(self): def test_small_n(self): for n in [-308, -309, -400, 1-2**31, -2**31, -2**31-1, -2**100]: - self.assertEqual(round(123.456, n), 0.0) - self.assertEqual(round(-123.456, n), -0.0) - self.assertEqual(round(1e300, n), 0.0) - self.assertEqual(round(1e-320, n), 0.0) + self.assertFloatsAreIdentical(round(123.456, n), 0.0) + self.assertFloatsAreIdentical(round(-123.456, n), -0.0) + self.assertFloatsAreIdentical(round(1e300, n), 0.0) + self.assertFloatsAreIdentical(round(1e-320, n), 0.0) def test_overflow(self): self.assertRaises(OverflowError, round, 1.6e308, -308) @@ -949,6 +983,13 @@ def test_None_ndigits(self): self.assertEqual(x, 2) self.assertIsInstance(x, int) + @support.cpython_only + def test_round_with_none_arg_direct_call(self): + for val in [(1.0).__round__(None), + round(1.0), + round(1.0, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) # Beginning with Python 2.6 float has cross platform compatible # ways to create and represent inf and nan @@ -1053,21 +1094,14 @@ def test_nan_signs(self): fromHex = float.fromhex toHex = float.hex -class HexFloatTestCase(unittest.TestCase): +class HexFloatTestCase(FloatsAreIdenticalMixin, unittest.TestCase): MAX = fromHex('0x.fffffffffffff8p+1024') # max normal MIN = fromHex('0x1p-1022') # min normal TINY = fromHex('0x0.0000000000001p-1022') # min subnormal EPS = fromHex('0x0.0000000000001p0') # diff between 1.0 and next float up def identical(self, x, y): - # check that floats x and y are identical, or that both - # are NaNs - if isnan(x) or isnan(y): - if isnan(x) == isnan(y): - return - elif x == y and (x != 0.0 or copysign(1.0, x) == copysign(1.0, y)): - return - self.fail('%r not identical to %r' % (x, y)) + self.assertFloatsAreIdentical(x, y) def test_ends(self): self.identical(self.MIN, ldexp(1.0, -1022)) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 6fa49dbc0b730c..9dde63e40d06db 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -4,6 +4,7 @@ import re import test.support as support import unittest +from test.support.import_helper import import_module maxsize = support.MAX_Py_ssize_t @@ -35,7 +36,7 @@ def testformat(formatstr, args, output=None, limit=None, overflowok=False): # when 'limit' is specified, it determines how many characters # must match exactly; lengths must always match. # ex: limit=5, '12345678' matches '12345___' - # (mainly for floating point format tests for which an exact match + # (mainly for floating-point format tests for which an exact match # can't be guaranteed due to rounding and representation errors) elif output and limit is not None and ( len(result)!=len(output) or result[:limit]!=output[:limit]): @@ -303,9 +304,9 @@ def test_str_format(self): test_exc('%c', sys.maxunicode+1, OverflowError, "%c arg not in range(0x110000)") #test_exc('%c', 2**128, OverflowError, "%c arg not in range(0x110000)") - test_exc('%c', 3.14, TypeError, "%c requires int or char") - test_exc('%c', 'ab', TypeError, "%c requires int or char") - test_exc('%c', b'x', TypeError, "%c requires int or char") + test_exc('%c', 3.14, TypeError, "%c requires an int or a unicode character, not float") + test_exc('%c', 'ab', TypeError, "%c requires an int or a unicode character, not a string of length 2") + test_exc('%c', b'x', TypeError, "%c requires an int or a unicode character, not bytes") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: @@ -369,11 +370,11 @@ def __bytes__(self): test_exc(b"%c", 2**128, OverflowError, "%c arg not in range(256)") test_exc(b"%c", b"Za", TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not a bytes object of length 2") test_exc(b"%c", "Y", TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not str") test_exc(b"%c", 3.14, TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not float") test_exc(b"%b", "Xc", TypeError, "%b requires a bytes-like object, " "or an object that implements __bytes__, not 'str'") @@ -478,7 +479,8 @@ def test_precision(self): @support.cpython_only def test_precision_c_limits(self): - from _testcapi import INT_MAX + _testcapi = import_module("_testcapi") + INT_MAX = _testcapi.INT_MAX f = 1.2 with self.assertRaises(ValueError) as cm: diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index af3cb214ab0ac1..98dccbec9566ac 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -91,6 +91,204 @@ class DummyFraction(fractions.Fraction): def _components(r): return (r.numerator, r.denominator) +def typed_approx_eq(a, b): + return type(a) == type(b) and (a == b or math.isclose(a, b)) + +class Symbolic: + """Simple non-numeric class for testing mixed arithmetic. + It is not Integral, Rational, Real or Complex, and cannot be converted + to int, float or complex. but it supports some arithmetic operations. + """ + def __init__(self, value): + self.value = value + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} * {other}') + def __rmul__(self, other): + return self.__class__(f'{other} * {self}') + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} / {other}') + def __rtruediv__(self, other): + return self.__class__(f'{other} / {self}') + def __mod__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} % {other}') + def __rmod__(self, other): + return self.__class__(f'{other} % {self}') + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} ** {other}') + def __rpow__(self, other): + return self.__class__(f'{other} ** {self}') + def __eq__(self, other): + if other.__class__ != self.__class__: + return NotImplemented + return self.value == other.value + def __str__(self): + return f'{self.value}' + def __repr__(self): + return f'{self.__class__.__name__}({self.value!r})' + +class SymbolicReal(Symbolic): + pass +numbers.Real.register(SymbolicReal) + +class SymbolicComplex(Symbolic): + pass +numbers.Complex.register(SymbolicComplex) + +class Rat: + """Simple Rational class for testing mixed arithmetic.""" + def __init__(self, n, d): + self.numerator = n + self.denominator = d + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator * other.numerator, + self.denominator * other.denominator) + def __rmul__(self, other): + return self.__class__(other.numerator * self.numerator, + other.denominator * self.denominator) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator * other.denominator, + self.denominator * other.numerator) + def __rtruediv__(self, other): + return self.__class__(other.numerator * self.denominator, + other.denominator * self.numerator) + def __mod__(self, other): + if isinstance(other, F): + return NotImplemented + d = self.denominator * other.numerator + return self.__class__(self.numerator * other.denominator % d, d) + def __rmod__(self, other): + d = other.denominator * self.numerator + return self.__class__(other.numerator * self.denominator % d, d) + + return self.__class__(other.numerator / self.numerator, + other.denominator / self.denominator) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator ** other, + self.denominator ** other) + def __float__(self): + return self.numerator / self.denominator + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return (typed_approx_eq(self.numerator, other.numerator) and + typed_approx_eq(self.denominator, other.denominator)) + def __repr__(self): + return f'{self.__class__.__name__}({self.numerator!r}, {self.denominator!r})' +numbers.Rational.register(Rat) + +class Root: + """Simple Real class for testing mixed arithmetic.""" + def __init__(self, v, n=F(2)): + self.base = v + self.degree = n + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base * other**self.degree, self.degree) + def __rmul__(self, other): + return self.__class__(other**self.degree * self.base, self.degree) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base / other**self.degree, self.degree) + def __rtruediv__(self, other): + return self.__class__(other**self.degree / self.base, self.degree) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base, self.degree / other) + def __float__(self): + return float(self.base) ** (1 / float(self.degree)) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.base, other.base) and typed_approx_eq(self.degree, other.degree) + def __repr__(self): + return f'{self.__class__.__name__}({self.base!r}, {self.degree!r})' +numbers.Real.register(Root) + +class Polar: + """Simple Complex class for testing mixed arithmetic.""" + def __init__(self, r, phi): + self.r = r + self.phi = phi + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r * other, self.phi) + def __rmul__(self, other): + return self.__class__(other * self.r, self.phi) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r / other, self.phi) + def __rtruediv__(self, other): + return self.__class__(other / self.r, -self.phi) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r ** other, self.phi * other) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.r, other.r) and typed_approx_eq(self.phi, other.phi) + def __repr__(self): + return f'{self.__class__.__name__}({self.r!r}, {self.phi!r})' +numbers.Complex.register(Polar) + +class Rect: + """Other simple Complex class for testing mixed arithmetic.""" + def __init__(self, x, y): + self.x = x + self.y = y + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.x * other, self.y * other) + def __rmul__(self, other): + return self.__class__(other * self.x, other * self.y) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.x / other, self.y / other) + def __rtruediv__(self, other): + r = self.x * self.x + self.y * self.y + return self.__class__(other * (self.x / r), other * (self.y / r)) + def __rpow__(self, other): + return Polar(other ** self.x, math.log(other) * self.y) + def __complex__(self): + return complex(self.x, self.y) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.x, other.x) and typed_approx_eq(self.y, other.y) + def __repr__(self): + return f'{self.__class__.__name__}({self.x!r}, {self.y!r})' +numbers.Complex.register(Rect) + +class RectComplex(Rect, complex): + pass + +class Ratio: + def __init__(self, ratio): + self._ratio = ratio + def as_integer_ratio(self): + return self._ratio + class FractionTest(unittest.TestCase): @@ -163,6 +361,38 @@ def testInitFromDecimal(self): self.assertRaises(OverflowError, F, Decimal('inf')) self.assertRaises(OverflowError, F, Decimal('-inf')) + def testInitFromIntegerRatio(self): + self.assertEqual((7, 3), _components(F(Ratio((7, 3))))) + errmsg = (r"argument should be a string or a Rational instance or " + r"have the as_integer_ratio\(\) method") + # the type also has an "as_integer_ratio" attribute. + self.assertRaisesRegex(TypeError, errmsg, F, Ratio) + # bad ratio + self.assertRaises(TypeError, F, Ratio(7)) + self.assertRaises(ValueError, F, Ratio((7,))) + self.assertRaises(ValueError, F, Ratio((7, 3, 1))) + # only single-argument form + self.assertRaises(TypeError, F, Ratio((3, 7)), 11) + self.assertRaises(TypeError, F, 2, Ratio((-10, 9))) + + # as_integer_ratio not defined in a class + class A: + pass + a = A() + a.as_integer_ratio = lambda: (9, 5) + self.assertEqual((9, 5), _components(F(a))) + + # as_integer_ratio defined in a metaclass + class M(type): + def as_integer_ratio(self): + return (11, 9) + class B(metaclass=M): + pass + self.assertRaisesRegex(TypeError, errmsg, F, B) + self.assertRaisesRegex(TypeError, errmsg, F, B()) + self.assertRaises(TypeError, F.from_number, B) + self.assertRaises(TypeError, F.from_number, B()) + def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) self.assertEqual((3, 2), _components(F("3/2"))) @@ -368,6 +598,37 @@ def testFromDecimal(self): ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("snan")) + def testFromNumber(self, cls=F): + def check(arg, numerator, denominator): + f = cls.from_number(arg) + self.assertIs(type(f), cls) + self.assertEqual(f.numerator, numerator) + self.assertEqual(f.denominator, denominator) + + check(10, 10, 1) + check(2.5, 5, 2) + check(Decimal('2.5'), 5, 2) + check(F(22, 7), 22, 7) + check(DummyFraction(22, 7), 22, 7) + check(Rat(22, 7), 22, 7) + check(Ratio((22, 7)), 22, 7) + self.assertRaises(TypeError, cls.from_number, 3+4j) + self.assertRaises(TypeError, cls.from_number, '5/2') + self.assertRaises(TypeError, cls.from_number, []) + self.assertRaises(OverflowError, cls.from_number, float('inf')) + self.assertRaises(OverflowError, cls.from_number, Decimal('inf')) + + # as_integer_ratio not defined in a class + class A: + pass + a = A() + a.as_integer_ratio = lambda: (9, 5) + check(a, 9, 5) + + def testFromNumber_subclass(self): + self.testFromNumber(DummyFraction) + + def test_is_integer(self): self.assertTrue(F(1, 1).is_integer()) self.assertTrue(F(-1, 1).is_integer()) @@ -593,6 +854,7 @@ def testMixedArithmetic(self): self.assertTypedEquals(0.9, 1.0 - F(1, 10)) self.assertTypedEquals(0.9 + 0j, (1.0 + 0j) - F(1, 10)) + def testMixedMultiplication(self): self.assertTypedEquals(F(1, 10), F(1, 10) * 1) self.assertTypedEquals(0.1, F(1, 10) * 1.0) self.assertTypedEquals(0.1 + 0j, F(1, 10) * (1.0 + 0j)) @@ -600,6 +862,29 @@ def testMixedArithmetic(self): self.assertTypedEquals(0.1, 1.0 * F(1, 10)) self.assertTypedEquals(0.1 + 0j, (1.0 + 0j) * F(1, 10)) + self.assertTypedEquals(F(3, 2) * DummyFraction(5, 3), F(5, 2)) + self.assertTypedEquals(DummyFraction(5, 3) * F(3, 2), F(5, 2)) + self.assertTypedEquals(F(3, 2) * Rat(5, 3), Rat(15, 6)) + self.assertTypedEquals(Rat(5, 3) * F(3, 2), F(5, 2)) + + self.assertTypedEquals(F(3, 2) * Root(4), Root(F(9, 1))) + self.assertTypedEquals(Root(4) * F(3, 2), 3.0) + self.assertEqual(F(3, 2) * SymbolicReal('X'), SymbolicReal('3/2 * X')) + self.assertRaises(TypeError, operator.mul, SymbolicReal('X'), F(3, 2)) + + self.assertTypedEquals(F(3, 2) * Polar(4, 2), Polar(F(6, 1), 2)) + self.assertTypedEquals(F(3, 2) * Polar(4.0, 2), Polar(6.0, 2)) + self.assertTypedEquals(F(3, 2) * Rect(4, 3), Rect(F(6, 1), F(9, 2))) + self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), RectComplex(6.0, 4.5)) + self.assertRaises(TypeError, operator.mul, Polar(4, 2), F(3, 2)) + self.assertTypedEquals(Rect(4, 3) * F(3, 2), 6.0 + 4.5j) + self.assertEqual(F(3, 2) * SymbolicComplex('X'), SymbolicComplex('3/2 * X')) + self.assertRaises(TypeError, operator.mul, SymbolicComplex('X'), F(3, 2)) + + self.assertEqual(F(3, 2) * Symbolic('X'), Symbolic('3/2 * X')) + self.assertRaises(TypeError, operator.mul, Symbolic('X'), F(3, 2)) + + def testMixedDivision(self): self.assertTypedEquals(F(1, 10), F(1, 10) / 1) self.assertTypedEquals(0.1, F(1, 10) / 1.0) self.assertTypedEquals(0.1 + 0j, F(1, 10) / (1.0 + 0j)) @@ -607,6 +892,28 @@ def testMixedArithmetic(self): self.assertTypedEquals(10.0, 1.0 / F(1, 10)) self.assertTypedEquals(10.0 + 0j, (1.0 + 0j) / F(1, 10)) + self.assertTypedEquals(F(3, 2) / DummyFraction(3, 5), F(5, 2)) + self.assertTypedEquals(DummyFraction(5, 3) / F(2, 3), F(5, 2)) + self.assertTypedEquals(F(3, 2) / Rat(3, 5), Rat(15, 6)) + self.assertTypedEquals(Rat(5, 3) / F(2, 3), F(5, 2)) + + self.assertTypedEquals(F(2, 3) / Root(4), Root(F(1, 9))) + self.assertTypedEquals(Root(4) / F(2, 3), 3.0) + self.assertEqual(F(3, 2) / SymbolicReal('X'), SymbolicReal('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, SymbolicReal('X'), F(3, 2)) + + self.assertTypedEquals(F(3, 2) / Polar(4, 2), Polar(F(3, 8), -2)) + self.assertTypedEquals(F(3, 2) / Polar(4.0, 2), Polar(0.375, -2)) + self.assertTypedEquals(F(3, 2) / Rect(4, 3), Rect(0.24, 0.18)) + self.assertRaises(TypeError, operator.truediv, Polar(4, 2), F(2, 3)) + self.assertTypedEquals(Rect(4, 3) / F(2, 3), 6.0 + 4.5j) + self.assertEqual(F(3, 2) / SymbolicComplex('X'), SymbolicComplex('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, SymbolicComplex('X'), F(3, 2)) + + self.assertEqual(F(3, 2) / Symbolic('X'), Symbolic('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, Symbolic('X'), F(2, 3)) + + def testMixedIntegerDivision(self): self.assertTypedEquals(0, F(1, 10) // 1) self.assertTypedEquals(0.0, F(1, 10) // 1.0) self.assertTypedEquals(10, 1 // F(1, 10)) @@ -631,6 +938,26 @@ def testMixedArithmetic(self): self.assertTypedTupleEquals(divmod(-0.1, float('inf')), divmod(F(-1, 10), float('inf'))) self.assertTypedTupleEquals(divmod(-0.1, float('-inf')), divmod(F(-1, 10), float('-inf'))) + self.assertTypedEquals(F(3, 2) % DummyFraction(3, 5), F(3, 10)) + self.assertTypedEquals(DummyFraction(5, 3) % F(2, 3), F(1, 3)) + self.assertTypedEquals(F(3, 2) % Rat(3, 5), Rat(3, 6)) + self.assertTypedEquals(Rat(5, 3) % F(2, 3), F(1, 3)) + + self.assertRaises(TypeError, operator.mod, F(2, 3), Root(4)) + self.assertTypedEquals(Root(4) % F(3, 2), 0.5) + self.assertEqual(F(3, 2) % SymbolicReal('X'), SymbolicReal('3/2 % X')) + self.assertRaises(TypeError, operator.mod, SymbolicReal('X'), F(3, 2)) + + self.assertRaises(TypeError, operator.mod, F(3, 2), Polar(4, 2)) + self.assertRaises(TypeError, operator.mod, F(3, 2), RectComplex(4, 3)) + self.assertRaises(TypeError, operator.mod, Rect(4, 3), F(2, 3)) + self.assertEqual(F(3, 2) % SymbolicComplex('X'), SymbolicComplex('3/2 % X')) + self.assertRaises(TypeError, operator.mod, SymbolicComplex('X'), F(3, 2)) + + self.assertEqual(F(3, 2) % Symbolic('X'), Symbolic('3/2 % X')) + self.assertRaises(TypeError, operator.mod, Symbolic('X'), F(2, 3)) + + def testMixedPower(self): # ** has more interesting conversion rules. self.assertTypedEquals(F(100, 1), F(1, 10) ** -2) self.assertTypedEquals(F(100, 1), F(10, 1) ** 2) @@ -647,6 +974,40 @@ def testMixedArithmetic(self): self.assertRaises(ZeroDivisionError, operator.pow, F(0, 1), -2) + self.assertTypedEquals(F(3, 2) ** Rat(3, 1), F(27, 8)) + self.assertTypedEquals(F(3, 2) ** Rat(-3, 1), F(8, 27)) + self.assertTypedEquals(F(-3, 2) ** Rat(-3, 1), F(-8, 27)) + self.assertTypedEquals(F(9, 4) ** Rat(3, 2), 3.375) + self.assertIsInstance(F(4, 9) ** Rat(-3, 2), float) + self.assertAlmostEqual(F(4, 9) ** Rat(-3, 2), 3.375) + self.assertAlmostEqual(F(-4, 9) ** Rat(-3, 2), 3.375j) + self.assertTypedEquals(Rat(9, 4) ** F(3, 2), 3.375) + self.assertTypedEquals(Rat(3, 2) ** F(3, 1), Rat(27, 8)) + self.assertTypedEquals(Rat(3, 2) ** F(-3, 1), F(8, 27)) + self.assertIsInstance(Rat(4, 9) ** F(-3, 2), float) + self.assertAlmostEqual(Rat(4, 9) ** F(-3, 2), 3.375) + + self.assertTypedEquals(Root(4) ** F(2, 3), Root(4, 3.0)) + self.assertTypedEquals(Root(4) ** F(2, 1), Root(4, F(1))) + self.assertTypedEquals(Root(4) ** F(-2, 1), Root(4, -F(1))) + self.assertTypedEquals(Root(4) ** F(-2, 3), Root(4, -3.0)) + self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('3/2 ** X')) + self.assertEqual(SymbolicReal('X') ** F(3, 2), SymbolicReal('X ** 1.5')) + + self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(F(9,4), 0.0)) + self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(F(1), 0.0)) + self.assertTypedEquals(F(3, 2) ** RectComplex(2, 0), Polar(2.25, 0.0)) + self.assertTypedEquals(F(1, 1) ** RectComplex(2, 3), Polar(1.0, 0.0)) + self.assertTypedEquals(Polar(4, 2) ** F(3, 2), Polar(8.0, 3.0)) + self.assertTypedEquals(Polar(4, 2) ** F(3, 1), Polar(64, 6)) + self.assertTypedEquals(Polar(4, 2) ** F(-3, 1), Polar(0.015625, -6)) + self.assertTypedEquals(Polar(4, 2) ** F(-3, 2), Polar(0.125, -3.0)) + self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('3/2 ** X')) + self.assertEqual(SymbolicComplex('X') ** F(3, 2), SymbolicComplex('X ** 1.5')) + + self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('3/2 ** X')) + self.assertEqual(Symbolic('X') ** F(3, 2), Symbolic('X ** 1.5')) + def testMixingWithDecimal(self): # Decimal refuses mixed arithmetic (but not mixed comparisons) self.assertRaises(TypeError, operator.add, @@ -1314,6 +1675,39 @@ def test_float_format_testfile(self): self.assertEqual(float(format(f, fmt2)), float(rhs)) self.assertEqual(float(format(-f, fmt2)), float('-' + rhs)) + def test_complex_handling(self): + # See issue gh-102840 for more details. + + a = F(1, 2) + b = 1j + message = "unsupported operand type(s) for %s: '%s' and '%s'" + # test forward + self.assertRaisesMessage(TypeError, + message % ("%", "Fraction", "complex"), + operator.mod, a, b) + self.assertRaisesMessage(TypeError, + message % ("//", "Fraction", "complex"), + operator.floordiv, a, b) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "Fraction", "complex"), + divmod, a, b) + # test reverse + self.assertRaisesMessage(TypeError, + message % ("%", "complex", "Fraction"), + operator.mod, b, a) + self.assertRaisesMessage(TypeError, + message % ("//", "complex", "Fraction"), + operator.floordiv, b, a) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "complex", "Fraction"), + divmod, b, a) + + def test_three_argument_pow(self): + message = "unsupported operand type(s) for ** or pow(): '%s', '%s', '%s'" + self.assertRaisesMessage(TypeError, + message % ("Fraction", "int", "int"), + pow, F(3), 4, 5) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index baed03d92b9e56..11f191700ccef0 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -1,10 +1,9 @@ -import gc +import copy import operator import re import sys import textwrap import threading -import types import unittest import weakref try: @@ -12,9 +11,11 @@ except ImportError: _testcapi = None +from collections.abc import Mapping from test import support -from test.support import threading_helper +from test.support import import_helper, threading_helper from test.support.script_helper import assert_python_ok +from test import mapping_tests class ClearTest(unittest.TestCase): @@ -198,14 +199,6 @@ def inner(): tb = tb.tb_next return frames - def test_locals(self): - f, outer, inner = self.make_frames() - outer_locals = outer.f_locals - self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) - self.assertEqual(outer_locals, {'x': 5, 'y': 6}) - inner_locals = inner.f_locals - self.assertEqual(inner_locals, {'x': 5, 'z': 7}) - def test_clear_locals(self): # Test f_locals after clear() (issue #21897) f, outer, inner = self.make_frames() @@ -217,8 +210,8 @@ def test_clear_locals(self): def test_locals_clear_locals(self): # Test f_locals before and after clear() (to exercise caching) f, outer, inner = self.make_frames() - outer.f_locals - inner.f_locals + self.assertNotEqual(outer.f_locals, {}) + self.assertNotEqual(inner.f_locals, {}) outer.clear() inner.clear() self.assertEqual(outer.f_locals, {}) @@ -269,6 +262,382 @@ def inner(): r"^$" % (file_repr, offset + 5)) +class TestFrameLocals(unittest.TestCase): + def test_scope(self): + class A: + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + + self.assertEqual(A.x, 2) + self.assertEqual(A.y, 2) + + def f(): + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + self.assertEqual(x, 2) + self.assertEqual(locals()['y'], 2) + f() + + def test_closure(self): + x = 1 + y = 2 + + def f(): + z = x + y + d = sys._getframe().f_locals + self.assertEqual(d['x'], 1) + self.assertEqual(d['y'], 2) + d['x'] = 2 + d['y'] = 3 + + f() + self.assertEqual(x, 2) + self.assertEqual(y, 3) + + def test_as_dict(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + # self, x, y, d + self.assertEqual(len(d), 4) + self.assertIs(d['d'], d) + self.assertEqual(set(d.keys()), set(['x', 'y', 'd', 'self'])) + self.assertEqual(len(d.values()), 4) + self.assertIn(1, d.values()) + self.assertEqual(len(d.items()), 4) + self.assertIn(('x', 1), d.items()) + self.assertEqual(d.__getitem__('x'), 1) + d.__setitem__('x', 2) + self.assertEqual(d['x'], 2) + self.assertEqual(d.get('x'), 2) + self.assertIs(d.get('non_exist', None), None) + self.assertEqual(d.__len__(), 4) + self.assertEqual(set([key for key in d]), set(['x', 'y', 'd', 'self'])) + self.assertIn('x', d) + self.assertTrue(d.__contains__('x')) + + self.assertEqual(reversed(d), list(reversed(d.keys()))) + + d.update({'x': 3, 'z': 4}) + self.assertEqual(d['x'], 3) + self.assertEqual(d['z'], 4) + + with self.assertRaises(TypeError): + d.update([1, 2]) + + self.assertEqual(d.setdefault('x', 5), 3) + self.assertEqual(d.setdefault('new', 5), 5) + self.assertEqual(d['new'], 5) + + with self.assertRaises(KeyError): + d['non_exist'] + + def test_as_number(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + self.assertIn('z', d | {'z': 3}) + d |= {'z': 3} + self.assertEqual(d['z'], 3) + d |= {'y': 3} + self.assertEqual(d['y'], 3) + with self.assertRaises(TypeError): + d |= 3 + with self.assertRaises(TypeError): + _ = d | [3] + + def test_non_string_key(self): + d = sys._getframe().f_locals + d[1] = 2 + self.assertEqual(d[1], 2) + + def test_write_with_hidden(self): + def f(): + f_locals = [sys._getframe().f_locals for b in [0]][0] + f_locals['b'] = 2 + f_locals['c'] = 3 + self.assertEqual(b, 2) + self.assertEqual(c, 3) + b = 0 + c = 0 + f() + + def test_local_objects(self): + o = object() + k = '.'.join(['a', 'b', 'c']) + f_locals = sys._getframe().f_locals + f_locals['o'] = f_locals['k'] + self.assertEqual(o, 'a.b.c') + + def test_copy(self): + x = 0 + d = sys._getframe().f_locals + d_copy = d.copy() + self.assertIsInstance(d_copy, dict) + self.assertEqual(d_copy['x'], 0) + d_copy['x'] = 1 + self.assertEqual(x, 0) + + def test_update_with_self(self): + def f(): + f_locals = sys._getframe().f_locals + f_locals.update(f_locals) + f_locals.update(f_locals) + f_locals.update(f_locals) + f() + + def test_repr(self): + x = 1 + # Introduce a reference cycle + frame = sys._getframe() + self.assertEqual(repr(frame.f_locals), repr(dict(frame.f_locals))) + + def test_delete(self): + x = 1 + d = sys._getframe().f_locals + + # This needs to be tested before f_extra_locals is created + with self.assertRaisesRegex(KeyError, 'non_exist'): + del d['non_exist'] + + with self.assertRaises(KeyError): + d.pop('non_exist') + + with self.assertRaisesRegex(ValueError, 'local variables'): + del d['x'] + + with self.assertRaises(AttributeError): + d.clear() + + with self.assertRaises(ValueError): + d.pop('x') + + with self.assertRaises(ValueError): + d.pop('x', None) + + # 'm', 'n' is stored in f_extra_locals + d['m'] = 1 + d['n'] = 1 + + with self.assertRaises(KeyError): + d.pop('non_exist') + + del d['m'] + self.assertEqual(d.pop('n'), 1) + + self.assertNotIn('m', d) + self.assertNotIn('n', d) + + self.assertEqual(d.pop('n', 2), 2) + + @support.cpython_only + def test_sizeof(self): + proxy = sys._getframe().f_locals + support.check_sizeof(self, proxy, support.calcobjsize("P")) + + def test_unsupport(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(TypeError): + copy.copy(d) + + with self.assertRaises(TypeError): + copy.deepcopy(d) + + def test_is_mapping(self): + x = 1 + d = sys._getframe().f_locals + self.assertIsInstance(d, Mapping) + match d: + case {"x": value}: + self.assertEqual(value, 1) + kind = "mapping" + case _: + kind = "other" + self.assertEqual(kind, "mapping") + + def _x_stringlikes(self): + class StringSubclass(str): + pass + + class ImpostorX: + def __hash__(self): + return hash('x') + + def __eq__(self, other): + return other == 'x' + + return StringSubclass('x'), ImpostorX(), 'x' + + def test_proxy_key_stringlikes_overwrite(self): + def f(obj): + x = 1 + proxy = sys._getframe().f_locals + proxy[obj] = 2 + return ( + list(proxy.keys()), + dict(proxy), + proxy + ) + + for obj in self._x_stringlikes(): + with self.subTest(cls=type(obj).__name__): + + keys_snapshot, proxy_snapshot, proxy = f(obj) + expected_keys = ['obj', 'x', 'proxy'] + expected_dict = {'obj': 'x', 'x': 2, 'proxy': proxy} + self.assertEqual(proxy.keys(), expected_keys) + self.assertEqual(proxy, expected_dict) + self.assertEqual(keys_snapshot, expected_keys) + self.assertEqual(proxy_snapshot, expected_dict) + + def test_proxy_key_stringlikes_ftrst_write(self): + def f(obj): + proxy = sys._getframe().f_locals + proxy[obj] = 2 + self.assertEqual(x, 2) + x = 1 + + for obj in self._x_stringlikes(): + with self.subTest(cls=type(obj).__name__): + f(obj) + + def test_proxy_key_unhashables(self): + class StringSubclass(str): + __hash__ = None + + class ObjectSubclass: + __hash__ = None + + proxy = sys._getframe().f_locals + + for obj in StringSubclass('x'), ObjectSubclass(): + with self.subTest(cls=type(obj).__name__): + with self.assertRaises(TypeError): + proxy[obj] + with self.assertRaises(TypeError): + proxy[obj] = 0 + + def test_constructor(self): + FrameLocalsProxy = type([sys._getframe().f_locals + for x in range(1)][0]) + self.assertEqual(FrameLocalsProxy.__name__, 'FrameLocalsProxy') + + def make_frame(): + x = 1 + y = 2 + return sys._getframe() + + proxy = FrameLocalsProxy(make_frame()) + self.assertEqual(proxy, {'x': 1, 'y': 2}) + + # constructor expects 1 frame argument + with self.assertRaises(TypeError): + FrameLocalsProxy() # no arguments + with self.assertRaises(TypeError): + FrameLocalsProxy(123) # wrong type + with self.assertRaises(TypeError): + FrameLocalsProxy(frame=sys._getframe()) # no keyword arguments + + +class FrameLocalsProxyMappingTests(mapping_tests.TestHashMappingProtocol): + """Test that FrameLocalsProxy behaves like a Mapping (with exceptions)""" + + def _f(*args, **kwargs): + def _f(): + return sys._getframe().f_locals + return _f() + type2test = _f + + @unittest.skipIf(True, 'Locals proxies for different frames never compare as equal') + def test_constructor(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: del proxy[key] fails') + def test_write(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.popitem') + def test_popitem(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.pop') + def test_pop(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.clear') + def test_clear(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.fromkeys') + def test_fromkeys(self): + pass + + # no del + def test_getitem(self): + mapping_tests.BasicTestMappingProtocol.test_getitem(self) + d = self._full_mapping({'a': 1, 'b': 2}) + self.assertEqual(d['a'], 1) + self.assertEqual(d['b'], 2) + d['c'] = 3 + d['a'] = 4 + self.assertEqual(d['c'], 3) + self.assertEqual(d['a'], 4) + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.update') + def test_update(self): + pass + + # proxy.copy returns a regular dict + def test_copy(self): + d = self._full_mapping({1:1, 2:2, 3:3}) + self.assertEqual(d.copy(), {1:1, 2:2, 3:3}) + d = self._empty_mapping() + self.assertEqual(d.copy(), d) + self.assertRaises(TypeError, d.copy, None) + + self.assertIsInstance(d.copy(), dict) + + @unittest.skipIf(True, 'Locals proxies for different frames never compare as equal') + def test_eq(self): + pass + + +class TestFrameCApi(unittest.TestCase): + def test_basic(self): + x = 1 + ctypes = import_helper.import_module('ctypes') + PyEval_GetFrameLocals = ctypes.pythonapi.PyEval_GetFrameLocals + PyEval_GetFrameLocals.restype = ctypes.py_object + frame_locals = PyEval_GetFrameLocals() + self.assertTrue(type(frame_locals), dict) + self.assertEqual(frame_locals['x'], 1) + frame_locals['x'] = 2 + self.assertEqual(x, 1) + + PyEval_GetFrameGlobals = ctypes.pythonapi.PyEval_GetFrameGlobals + PyEval_GetFrameGlobals.restype = ctypes.py_object + frame_globals = PyEval_GetFrameGlobals() + self.assertTrue(type(frame_globals), dict) + self.assertIs(frame_globals, globals()) + + PyEval_GetFrameBuiltins = ctypes.pythonapi.PyEval_GetFrameBuiltins + PyEval_GetFrameBuiltins.restype = ctypes.py_object + frame_builtins = PyEval_GetFrameBuiltins() + self.assertEqual(frame_builtins, __builtins__) + + PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals + PyFrame_GetLocals.argtypes = [ctypes.py_object] + PyFrame_GetLocals.restype = ctypes.py_object + frame = sys._getframe() + f_locals = PyFrame_GetLocals(frame) + self.assertTrue(f_locals['x'], 1) + f_locals['x'] = 2 + self.assertEqual(x, 2) + + class TestIncompleteFrameAreInvisible(unittest.TestCase): def test_issue95818(self): @@ -293,69 +662,6 @@ def gen(): """) assert_python_ok("-c", code) - @support.cpython_only - def test_sneaky_frame_object(self): - - def trace(frame, event, arg): - """ - Don't actually do anything, just force a frame object to be created. - """ - - def callback(phase, info): - """ - Yo dawg, I heard you like frames, so I'm allocating a frame while - you're allocating a frame, so you can have a frame while you have a - frame! - """ - nonlocal sneaky_frame_object - sneaky_frame_object = sys._getframe().f_back.f_back - # We're done here: - gc.callbacks.remove(callback) - - def f(): - while True: - yield - - old_threshold = gc.get_threshold() - old_callbacks = gc.callbacks[:] - old_enabled = gc.isenabled() - old_trace = sys.gettrace() - try: - # Stop the GC for a second while we set things up: - gc.disable() - # Create a paused generator: - g = f() - next(g) - # Move all objects to the oldest generation, and tell the GC to run - # on the *very next* allocation: - gc.collect() - gc.set_threshold(1, 0, 0) - # Okay, so here's the nightmare scenario: - # - We're tracing the resumption of a generator, which creates a new - # frame object. - # - The allocation of this frame object triggers a collection - # *before* the frame object is actually created. - # - During the collection, we request the exact same frame object. - # This test does it with a GC callback, but in real code it would - # likely be a trace function, weakref callback, or finalizer. - # - The collection finishes, and the original frame object is - # created. We now have two frame objects fighting over ownership - # of the same interpreter frame! - sys.settrace(trace) - gc.callbacks.append(callback) - sneaky_frame_object = None - gc.enable() - next(g) - # g.gi_frame should be the frame object from the callback (the - # one that was *requested* second, but *created* first): - self.assertIs(g.gi_frame, sneaky_frame_object) - finally: - gc.set_threshold(*old_threshold) - gc.callbacks[:] = old_callbacks - sys.settrace(old_trace) - if old_enabled: - gc.enable() - @support.cpython_only @threading_helper.requires_working_threading() def test_sneaky_frame_object_teardown(self): diff --git a/Lib/test/test_free_threading/__init__.py b/Lib/test/test_free_threading/__init__.py new file mode 100644 index 00000000000000..34e1ad30e11097 --- /dev/null +++ b/Lib/test/test_free_threading/__init__.py @@ -0,0 +1,11 @@ +import os +import unittest + +from test import support + + +if not support.Py_GIL_DISABLED: + raise unittest.SkipTest("GIL enabled") + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py new file mode 100644 index 00000000000000..80daf0d9cae9e0 --- /dev/null +++ b/Lib/test/test_free_threading/test_dict.py @@ -0,0 +1,147 @@ +import gc +import time +import unittest +import weakref + +from ast import Or +from functools import partial +from threading import Thread +from unittest import TestCase + +try: + import _testcapi +except ImportError: + _testcapi = None + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestDict(TestCase): + def test_racing_creation_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with shared keys""" + class C(int): + pass + + self.racing_creation(C) + + def test_racing_creation_no_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with an ordinary dict""" + self.racing_creation(Or) + + def test_racing_creation_inline_values_invalid(self): + """Verify that re-creating a dict after we have invalid inline values + is thread safe""" + class C: + pass + + def make_obj(): + a = C() + # Make object, make inline values invalid, and then delete dict + a.__dict__ = {} + del a.__dict__ + return a + + self.racing_creation(make_obj) + + def test_racing_creation_nonmanaged_dict(self): + """Verify that explicit creation of an unmanaged dict is thread safe + outside of the normal attribute setting code path""" + def make_obj(): + def f(): pass + return f + + def set(func, name, val): + # Force creation of the dict via PyObject_GenericGetDict + func.__dict__[name] = val + + self.racing_creation(make_obj, set) + + def racing_creation(self, cls, set=setattr): + objects = [] + processed = [] + + OBJECT_COUNT = 100 + THREAD_COUNT = 10 + CUR = 0 + + for i in range(OBJECT_COUNT): + objects.append(cls()) + + def writer_func(name): + last = -1 + while True: + if CUR == last: + continue + elif CUR == OBJECT_COUNT: + break + + obj = objects[CUR] + set(obj, name, name) + last = CUR + processed.append(name) + + writers = [] + for x in range(THREAD_COUNT): + writer = Thread(target=partial(writer_func, f"a{x:02}")) + writers.append(writer) + writer.start() + + for i in range(OBJECT_COUNT): + CUR = i + while len(processed) != THREAD_COUNT: + time.sleep(0.001) + processed.clear() + + CUR = OBJECT_COUNT + + for writer in writers: + writer.join() + + for obj_idx, obj in enumerate(objects): + assert ( + len(obj.__dict__) == THREAD_COUNT + ), f"{len(obj.__dict__)} {obj.__dict__!r} {obj_idx}" + for i in range(THREAD_COUNT): + assert f"a{i:02}" in obj.__dict__, f"a{i:02} missing at {obj_idx}" + + def test_racing_set_dict(self): + """Races assigning to __dict__ should be thread safe""" + + def f(): pass + l = [] + THREAD_COUNT = 10 + class MyDict(dict): pass + + def writer_func(l): + for i in range(1000): + d = MyDict() + l.append(weakref.ref(d)) + f.__dict__ = d + + lists = [] + writers = [] + for x in range(THREAD_COUNT): + thread_list = [] + lists.append(thread_list) + writer = Thread(target=partial(writer_func, thread_list)) + writers.append(writer) + + for writer in writers: + writer.start() + + for writer in writers: + writer.join() + + f.__dict__ = {} + gc.collect() + + for thread_list in lists: + for ref in thread_list: + self.assertIsNone(ref()) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_gc.py b/Lib/test/test_free_threading/test_gc.py new file mode 100644 index 00000000000000..401067fe9c612c --- /dev/null +++ b/Lib/test/test_free_threading/test_gc.py @@ -0,0 +1,61 @@ +import unittest + +import threading +from threading import Thread +from unittest import TestCase +import gc + +from test.support import threading_helper + + +class MyObj: + pass + + +@threading_helper.requires_working_threading() +class TestGC(TestCase): + def test_get_objects(self): + event = threading.Event() + + def gc_thread(): + for i in range(100): + o = gc.get_objects() + event.set() + + def mutator_thread(): + while not event.is_set(): + o1 = MyObj() + o2 = MyObj() + o3 = MyObj() + o4 = MyObj() + + gcs = [Thread(target=gc_thread)] + mutators = [Thread(target=mutator_thread) for _ in range(4)] + with threading_helper.start_threads(gcs + mutators): + pass + + def test_get_referrers(self): + event = threading.Event() + + obj = MyObj() + + def gc_thread(): + for i in range(100): + o = gc.get_referrers(obj) + event.set() + + def mutator_thread(): + while not event.is_set(): + d1 = { "key": obj } + d2 = { "key": obj } + d3 = { "key": obj } + d4 = { "key": obj } + + gcs = [Thread(target=gc_thread) for _ in range(2)] + mutators = [Thread(target=mutator_thread) for _ in range(4)] + with threading_helper.start_threads(gcs + mutators): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py new file mode 100644 index 00000000000000..a705161369e8dd --- /dev/null +++ b/Lib/test/test_free_threading/test_list.py @@ -0,0 +1,76 @@ +import unittest + +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + + +NTHREAD = 10 +OBJECT_COUNT = 5_000 + + +class C: + def __init__(self, v): + self.v = v + + +@threading_helper.requires_working_threading() +class TestList(TestCase): + def test_racing_iter_append(self): + l = [] + + def writer_func(): + for i in range(OBJECT_COUNT): + l.append(C(i + OBJECT_COUNT)) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(NTHREAD): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + def test_racing_iter_extend(self): + l = [] + + def writer_func(): + for i in range(OBJECT_COUNT): + l.extend([C(i + OBJECT_COUNT)]) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(NTHREAD): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py new file mode 100644 index 00000000000000..8fec01715531cb --- /dev/null +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -0,0 +1,248 @@ +"""Tests monitoring, sys.settrace, and sys.setprofile in a multi-threaded +environment to verify things are thread-safe in a free-threaded build""" + +import sys +import time +import unittest +import weakref + +from sys import monitoring +from test.support import threading_helper +from threading import Thread, _PyRLock +from unittest import TestCase + + +class InstrumentationMultiThreadedMixin: + thread_count = 10 + func_count = 50 + fib = 12 + + def after_threads(self): + """Runs once after all the threads have started""" + pass + + def during_threads(self): + """Runs repeatedly while the threads are still running""" + pass + + def work(self, n, funcs): + """Fibonacci function which also calls a bunch of random functions""" + for func in funcs: + func() + if n < 2: + return n + return self.work(n - 1, funcs) + self.work(n - 2, funcs) + + def start_work(self, n, funcs): + # With the GIL builds we need to make sure that the hooks have + # a chance to run as it's possible to run w/o releasing the GIL. + time.sleep(0.1) + self.work(n, funcs) + + def after_test(self): + """Runs once after the test is done""" + pass + + def test_instrumentation(self): + # Setup a bunch of functions which will need instrumentation... + funcs = [] + for i in range(self.func_count): + x = {} + exec("def f(): pass", x) + funcs.append(x["f"]) + + threads = [] + for i in range(self.thread_count): + # Each thread gets a copy of the func list to avoid contention + t = Thread(target=self.start_work, args=(self.fib, list(funcs))) + t.start() + threads.append(t) + + self.after_threads() + + while True: + any_alive = False + for t in threads: + if t.is_alive(): + any_alive = True + break + + if not any_alive: + break + + self.during_threads() + + self.after_test() + + +class MonitoringTestMixin: + def setUp(self): + for i in range(6): + if monitoring.get_tool(i) is None: + self.tool_id = i + monitoring.use_tool_id(i, self.__class__.__name__) + break + + def tearDown(self): + monitoring.free_tool_id(self.tool_id) + + +@threading_helper.requires_working_threading() +class SetPreTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Sets tracing one time after the threads have started""" + + def setUp(self): + super().setUp() + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def after_threads(self): + sys.settrace(self.trace_func) + + +@threading_helper.requires_working_threading() +class MonitoringMultiThreaded( + MonitoringTestMixin, InstrumentationMultiThreadedMixin, TestCase +): + """Uses sys.monitoring and repeatedly toggles instrumentation on and off""" + + def setUp(self): + super().setUp() + self.set = False + self.called = False + monitoring.register_callback( + self.tool_id, monitoring.events.LINE, self.callback + ) + + def tearDown(self): + monitoring.set_events(self.tool_id, 0) + super().tearDown() + + def callback(self, *args): + self.called = True + + def after_test(self): + self.assertTrue(self.called) + + def during_threads(self): + if self.set: + monitoring.set_events( + self.tool_id, monitoring.events.CALL | monitoring.events.LINE + ) + else: + monitoring.set_events(self.tool_id, 0) + self.set = not self.set + + +@threading_helper.requires_working_threading() +class SetTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses sys.settrace and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def tearDown(self): + sys.settrace(None) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def during_threads(self): + if self.set: + sys.settrace(self.trace_func) + else: + sys.settrace(None) + self.set = not self.set + + +@threading_helper.requires_working_threading() +class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses sys.setprofile and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def tearDown(self): + sys.setprofile(None) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def during_threads(self): + if self.set: + sys.setprofile(self.trace_func) + else: + sys.setprofile(None) + self.set = not self.set + + +@threading_helper.requires_working_threading() +class MonitoringMisc(MonitoringTestMixin, TestCase): + def register_callback(self): + def callback(*args): + pass + + for i in range(200): + monitoring.register_callback(self.tool_id, monitoring.events.LINE, callback) + + self.refs.append(weakref.ref(callback)) + + def test_register_callback(self): + self.refs = [] + threads = [] + for i in range(50): + t = Thread(target=self.register_callback) + t.start() + threads.append(t) + + for thread in threads: + thread.join() + + monitoring.register_callback(self.tool_id, monitoring.events.LINE, None) + for ref in self.refs: + self.assertEqual(ref(), None) + + def test_set_local_trace_opcodes(self): + def trace(frame, event, arg): + frame.f_trace_opcodes = True + return trace + + loops = 1_000 + + sys.settrace(trace) + try: + l = _PyRLock() + + def f(): + for i in range(loops): + with l: + pass + + t = Thread(target=f) + t.start() + for i in range(loops): + with l: + pass + t.join() + finally: + sys.settrace(None) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py new file mode 100644 index 00000000000000..758f74f54d0b56 --- /dev/null +++ b/Lib/test/test_free_threading/test_slots.py @@ -0,0 +1,43 @@ +import threading +from test.support import threading_helper +from unittest import TestCase + + +def run_in_threads(targets): + """Run `targets` in separate threads""" + threads = [ + threading.Thread(target=target) + for target in targets + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +@threading_helper.requires_working_threading() +class TestSlots(TestCase): + + def test_object(self): + class Spam: + __slots__ = [ + "eggs", + ] + + def __init__(self, initial_value): + self.eggs = initial_value + + spam = Spam(0) + iters = 20_000 + + def writer(): + for _ in range(iters): + spam.eggs += 1 + + def reader(): + for _ in range(iters): + eggs = spam.eggs + assert type(eggs) is int + assert 0 <= eggs <= iters + + run_in_threads([writer, reader, reader, reader]) diff --git a/Lib/test/test_free_threading/test_str.py b/Lib/test/test_free_threading/test_str.py new file mode 100644 index 00000000000000..72044e979b0f48 --- /dev/null +++ b/Lib/test/test_free_threading/test_str.py @@ -0,0 +1,74 @@ +import unittest + +from itertools import cycle +from threading import Event, Thread +from unittest import TestCase + +from test.support import threading_helper + +@threading_helper.requires_working_threading() +class TestStr(TestCase): + def test_racing_join_extend(self): + '''Test joining a string being extended by another thread''' + l = [] + ITERS = 100 + READERS = 10 + done_event = Event() + def writer_func(): + for i in range(ITERS): + l.extend(map(str, range(i))) + l.clear() + done_event.set() + def reader_func(): + while not done_event.is_set(): + ''.join(l) + writer = Thread(target=writer_func) + readers = [] + for x in range(READERS): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + def test_racing_join_replace(self): + ''' + Test joining a string of characters being replaced with ephemeral + strings by another thread. + ''' + l = [*'abcdefg'] + MAX_ORDINAL = 1_000 + READERS = 10 + done_event = Event() + + def writer_func(): + for i, c in zip(cycle(range(len(l))), + map(chr, range(128, MAX_ORDINAL))): + l[i] = c + done_event.set() + + def reader_func(): + while not done_event.is_set(): + ''.join(l) + ''.join(l) + ''.join(l) + ''.join(l) + + writer = Thread(target=writer_func) + readers = [] + for x in range(READERS): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_tokenize.py b/Lib/test/test_free_threading/test_tokenize.py new file mode 100644 index 00000000000000..860cfec4d710f4 --- /dev/null +++ b/Lib/test/test_free_threading/test_tokenize.py @@ -0,0 +1,57 @@ +import io +import time +import unittest +import tokenize +from functools import partial +from threading import Thread + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestTokenize(unittest.TestCase): + def test_tokenizer_iter(self): + source = io.StringIO("for _ in a:\n pass") + it = tokenize._tokenize.TokenizerIter(source.readline, extra_tokens=False) + + tokens = [] + def next_token(it): + while True: + try: + r = next(it) + tokens.append(tokenize.TokenInfo._make(r)) + time.sleep(0.03) + except StopIteration: + return + + threads = [] + for _ in range(5): + threads.append(Thread(target=partial(next_token, it))) + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + expected_tokens = [ + tokenize.TokenInfo(type=1, string='for', start=(1, 0), end=(1, 3), line='for _ in a:\n'), + tokenize.TokenInfo(type=1, string='_', start=(1, 4), end=(1, 5), line='for _ in a:\n'), + tokenize.TokenInfo(type=1, string='in', start=(1, 6), end=(1, 8), line='for _ in a:\n'), + tokenize.TokenInfo(type=1, string='a', start=(1, 9), end=(1, 10), line='for _ in a:\n'), + tokenize.TokenInfo(type=11, string=':', start=(1, 10), end=(1, 11), line='for _ in a:\n'), + tokenize.TokenInfo(type=4, string='', start=(1, 11), end=(1, 11), line='for _ in a:\n'), + tokenize.TokenInfo(type=5, string='', start=(2, -1), end=(2, -1), line=' pass'), + tokenize.TokenInfo(type=1, string='pass', start=(2, 2), end=(2, 6), line=' pass'), + tokenize.TokenInfo(type=4, string='', start=(2, 6), end=(2, 6), line=' pass'), + tokenize.TokenInfo(type=6, string='', start=(2, -1), end=(2, -1), line=' pass'), + tokenize.TokenInfo(type=0, string='', start=(2, -1), end=(2, -1), line=' pass'), + ] + + tokens.sort() + expected_tokens.sort() + self.assertListEqual(tokens, expected_tokens) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py new file mode 100644 index 00000000000000..51463b6bb8c1b4 --- /dev/null +++ b/Lib/test/test_free_threading/test_type.py @@ -0,0 +1,141 @@ +import threading +import unittest + +from concurrent.futures import ThreadPoolExecutor +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + + + +NTHREADS = 6 +BOTTOM = 0 +TOP = 1000 +ITERS = 100 + +class A: + attr = 1 + +@threading_helper.requires_working_threading() +class TestType(TestCase): + def test_attr_cache(self): + def read(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + A.attr + + def write(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + # Make _PyType_Lookup cache hot first + A.attr + A.attr + x = A.attr + x += 1 + A.attr = x + + + with ThreadPoolExecutor(NTHREADS) as pool: + pool.submit(read, (1,)) + pool.submit(write, (1,)) + pool.shutdown(wait=True) + + def test_attr_cache_consistency(self): + class C: + x = 0 + + DONE = False + def writer_func(): + for i in range(3000): + C.x + C.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = C.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def test_attr_cache_consistency_subclass(self): + class C: + x = 0 + + class D(C): + pass + + DONE = False + def writer_func(): + for i in range(3000): + D.x + D.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = D.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def test___class___modification(self): + loops = 200 + + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(loops): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for i in range(NTHREADS): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + def run_one(self, writer_func, reader_func): + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_zip.py b/Lib/test/test_free_threading/test_zip.py new file mode 100644 index 00000000000000..b4c5837b1d8936 --- /dev/null +++ b/Lib/test/test_free_threading/test_zip.py @@ -0,0 +1,41 @@ +import unittest +from threading import Thread + +from test.support import threading_helper + + +class ZipThreading(unittest.TestCase): + @staticmethod + def work(enum): + while True: + try: + next(enum) + except StopIteration: + break + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_threading(self): + number_of_threads = 8 + number_of_iterations = 8 + n = 40_000 + enum = zip(range(n), range(n)) + for _ in range(number_of_iterations): + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread( + target=self.work, + args=[ + enum, + ], + ) + ) + for t in worker_threads: + t.start() + for t in worker_threads: + t.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 27c7f70cef32e3..c359f2ecce01f1 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -8,13 +8,13 @@ # Unicode identifiers in tests is allowed by PEP 3131. import ast +import datetime import dis import os import re import types import decimal import unittest -import warnings from test import support from test.support.os_helper import temp_cwd from test.support.script_helper import assert_python_failure, assert_python_ok @@ -744,13 +744,13 @@ def test_comments(self): }''', 'A complex trick: 2') self.assertEqual(f''' { -40 # fourty +40 # forty + # plus 2 # two }''', '\n42') self.assertEqual(f''' { -40 # fourty +40 # forty + # plus 2 # two }''', '\n42') @@ -896,6 +896,7 @@ def test_missing_expression(self): "f'{:2}'", "f'''{\t\f\r\n:a}'''", "f'{:'", + "F'{[F'{:'}[F'{:'}]]]", ]) self.assertAllRaise(SyntaxError, @@ -1602,6 +1603,12 @@ def f(a): self.assertEqual(f'{f(a=4)}', '3=') self.assertEqual(x, 4) + # Check debug expressions in format spec + y = 20 + self.assertEqual(f"{2:{y=}}", "yyyyyyyyyyyyyyyyyyy2") + self.assertEqual(f"{datetime.datetime.now():h1{y=}h2{y=}h3{y=}}", + 'h1y=20h2y=20h3y=20') + # Make sure __format__ is being called. class C: def __format__(self, s): @@ -1615,9 +1622,11 @@ def __repr__(self): self.assertEqual(f'{C()=: }', 'C()=FORMAT- ') self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x') self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********') + self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20') self.assertRaises(SyntaxError, eval, "f'{C=]'") + # Make sure leading and following text works. x = 'foo' self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 81115e9db888cf..bed0e6d973b8da 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -543,8 +543,8 @@ def test_set_pasv(self): self.assertFalse(self.client.passiveserver) def test_voidcmd(self): - self.client.voidcmd('echo 200') - self.client.voidcmd('echo 299') + self.assertEqual(self.client.voidcmd('echo 200'), '200') + self.assertEqual(self.client.voidcmd('echo 299'), '299') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index 8a4d4ae6663653..fd43e8a7f3c994 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -98,7 +98,12 @@ def test___globals__(self): (AttributeError, TypeError)) def test___builtins__(self): - self.assertIs(self.b.__builtins__, __builtins__) + if __name__ == "__main__": + builtins_dict = __builtins__.__dict__ + else: + builtins_dict = __builtins__ + + self.assertIs(self.b.__builtins__, builtins_dict) self.cannot_set_attr(self.b, '__builtins__', 2, (AttributeError, TypeError)) @@ -108,7 +113,7 @@ def func(s): return len(s) ns = {} func2 = type(func)(func.__code__, ns) self.assertIs(func2.__globals__, ns) - self.assertIs(func2.__builtins__, __builtins__) + self.assertIs(func2.__builtins__, builtins_dict) # Make sure that the function actually works. self.assertEqual(func2("abc"), 3) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 7c66b906d308ba..d590af090abc6e 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -6,6 +6,7 @@ from itertools import permutations import pickle from random import choice +import re import sys from test import support import threading @@ -31,10 +32,6 @@ decimal = import_helper.import_fresh_module('decimal', fresh=['_decimal']) -_partial_types = [py_functools.partial] -if c_functools: - _partial_types.append(c_functools.partial) - @contextlib.contextmanager def replaced_module(name, replacement): @@ -189,6 +186,19 @@ def test_nested_optimization(self): flat = partial(signature, 'asdf', bar=True) self.assertEqual(signature(nested), signature(flat)) + def test_nested_optimization_bug(self): + partial = self.partial + class Builder: + def __call__(self, tag, *children, **attrib): + return (tag, children, attrib) + + def __getattr__(self, tag): + return partial(self, tag) + + B = Builder() + m = B.m + assert m(1, 2, a=2) == ('m', (1, 2), dict(a=2)) + def test_nested_partial_with_attribute(self): # see issue 25137 partial = self.partial @@ -201,16 +211,58 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') + def test_placeholders_trailing_raise(self): + PH = self.module.Placeholder + for args in [(PH,), (0, PH), (0, PH, 1, PH, PH, PH)]: + with self.assertRaises(TypeError): + self.partial(capture, *args) + + def test_placeholders(self): + PH = self.module.Placeholder + # 1 Placeholder + args = (PH, 0) + p = self.partial(capture, *args) + actual_args, actual_kwds = p('x') + self.assertEqual(actual_args, ('x', 0)) + self.assertEqual(actual_kwds, {}) + # 2 Placeholders + args = (PH, 0, PH, 1) + p = self.partial(capture, *args) + with self.assertRaises(TypeError): + p('x') + actual_args, actual_kwds = p('x', 'y') + self.assertEqual(actual_args, ('x', 0, 'y', 1)) + self.assertEqual(actual_kwds, {}) + + def test_placeholders_optimization(self): + PH = self.module.Placeholder + p = self.partial(capture, PH, 0) + p2 = self.partial(p, PH, 1, 2, 3) + self.assertEqual(p2.args, (PH, 0, 1, 2, 3)) + p3 = self.partial(p2, -1, 4) + actual_args, actual_kwds = p3(5) + self.assertEqual(actual_args, (-1, 0, 1, 2, 3, 4, 5)) + self.assertEqual(actual_kwds, {}) + # inner partial has placeholders and outer partial has no args case + p = self.partial(capture, PH, 0) + p2 = self.partial(p) + self.assertEqual(p2.args, (PH, 0)) + self.assertEqual(p2(1), ((1, 0), {})) + + def test_construct_placeholder_singleton(self): + PH = self.module.Placeholder + tp = type(PH) + self.assertIs(tp(), PH) + self.assertRaises(TypeError, tp, 1, 2) + self.assertRaises(TypeError, tp, a=1, b=2) + def test_repr(self): args = (object(), object()) args_repr = ', '.join(repr(a) for a in args) kwargs = {'a': object(), 'b': object()} kwargs_reprs = ['a={a!r}, b={b!r}'.format_map(kwargs), 'b={b!r}, a={a!r}'.format_map(kwargs)] - if self.partial in _partial_types: - name = 'functools.partial' - else: - name = self.partial.__name__ + name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) self.assertEqual(f'{name}({capture!r})', repr(f)) @@ -229,10 +281,7 @@ def test_repr(self): for kwargs_repr in kwargs_reprs]) def test_recursive_repr(self): - if self.partial in _partial_types: - name = 'functools.partial' - else: - name = self.partial.__name__ + name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) f.__setstate__((f, (), {}, {})) @@ -308,6 +357,23 @@ def test_setstate(self): self.assertEqual(f(2), ((2,), {})) self.assertEqual(f(), ((), {})) + # Set State with placeholders + PH = self.module.Placeholder + f = self.partial(signature) + f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[]))) + self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[]))) + msg_regex = re.escape("missing positional arguments in 'partial' call; " + "expected at least 1, got 0") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + f() + self.assertEqual(f(2), ((2, 1), dict(a=10))) + + # Trailing Placeholder error + f = self.partial(signature) + msg_regex = re.escape("trailing Placeholders are not allowed") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) + def test_setstate_errors(self): f = self.partial(signature) self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) @@ -344,8 +410,10 @@ def test_recursive_pickle(self): f.__setstate__((f, (), {}, {})) try: for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(RecursionError): - pickle.dumps(f, proto) + # gh-117008: Small limit since pickle uses C stack memory + with support.infinite_recursion(100): + with self.assertRaises(RecursionError): + pickle.dumps(f, proto) finally: f.__setstate__((capture, (), {}, {})) @@ -390,6 +458,21 @@ def __getitem__(self, key): f = self.partial(object) self.assertRaises(TypeError, f.__setstate__, BadSequence()) + def test_partial_as_method(self): + class A: + meth = self.partial(capture, 1, a=2) + cmeth = classmethod(self.partial(capture, 1, a=2)) + smeth = staticmethod(self.partial(capture, 1, a=2)) + + a = A() + self.assertEqual(A.meth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + self.assertEqual(A.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) + self.assertEqual(A.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + self.assertEqual(a.meth(3, b=4), ((1, a, 3), {'a': 2, 'b': 4})) + self.assertEqual(a.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) + self.assertEqual(a.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): if c_functools: @@ -436,6 +519,19 @@ def __str__(self): self.assertIn('astr', r) self.assertIn("['sth']", r) + def test_placeholders_refcount_smoke(self): + PH = self.module.Placeholder + # sum supports vector call + lst1, start = [], [] + sum_lists = self.partial(sum, PH, start) + for i in range(10): + sum_lists([lst1, lst1]) + # collections.ChainMap initializer does not support vectorcall + map1, map2 = {}, {} + partial_cm = self.partial(collections.ChainMap, PH, map1) + for i in range(10): + partial_cm(map2, map2) + class TestPartialPy(TestPartial, unittest.TestCase): module = py_functools @@ -460,6 +556,19 @@ class TestPartialCSubclass(TestPartialC): class TestPartialPySubclass(TestPartialPy): partial = PyPartialSubclass + def test_subclass_optimization(self): + # `partial` input to `partial` subclass + p = py_functools.partial(min, 2) + p2 = self.partial(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) + # `partial` subclass input to `partial` subclass + p = self.partial(min, 2) + p2 = self.partial(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) + + class TestPartialMethod(unittest.TestCase): class A(object): @@ -564,6 +673,14 @@ class B: method = functools.partialmethod(func=capture, a=1) def test_repr(self): + self.assertEqual(repr(vars(self.A)['nothing']), + 'functools.partialmethod({})'.format(capture)) + self.assertEqual(repr(vars(self.A)['positional']), + 'functools.partialmethod({}, 1)'.format(capture)) + self.assertEqual(repr(vars(self.A)['keywords']), + 'functools.partialmethod({}, a=2)'.format(capture)) + self.assertEqual(repr(vars(self.A)['spec_keywords']), + 'functools.partialmethod({}, self=1, func=2)'.format(capture)) self.assertEqual(repr(vars(self.A)['both']), 'functools.partialmethod({}, 3, b=4)'.format(capture)) @@ -589,6 +706,20 @@ def f(a, b, /): p = functools.partial(f, 1) self.assertEqual(p(2), f(1, 2)) + def test_subclass_optimization(self): + class PartialMethodSubclass(functools.partialmethod): + pass + # `partialmethod` input to `partialmethod` subclass + p = functools.partialmethod(min, 2) + p2 = PartialMethodSubclass(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) + # `partialmethod` subclass input to `partialmethod` subclass + p = PartialMethodSubclass(min, 2) + p2 = PartialMethodSubclass(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) + class TestUpdateWrapper(unittest.TestCase): @@ -705,6 +836,34 @@ def wrapper(): self.assertTrue(wrapper.__doc__.startswith('max(')) self.assertEqual(wrapper.__annotations__, {}) + def test_update_type_wrapper(self): + def wrapper(*args): pass + + functools.update_wrapper(wrapper, type) + self.assertEqual(wrapper.__name__, 'type') + self.assertEqual(wrapper.__annotations__, {}) + self.assertEqual(wrapper.__type_params__, ()) + + def test_update_wrapper_annotations(self): + def inner(x: int): pass + def wrapper(*args): pass + + functools.update_wrapper(wrapper, inner) + self.assertEqual(wrapper.__annotations__, {'x': int}) + self.assertIs(wrapper.__annotate__, inner.__annotate__) + + def with_forward_ref(x: undefined): pass + def wrapper(*args): pass + + functools.update_wrapper(wrapper, with_forward_ref) + + self.assertIs(wrapper.__annotate__, with_forward_ref.__annotate__) + with self.assertRaises(NameError): + wrapper.__annotations__ + + undefined = str + self.assertEqual(wrapper.__annotations__, {'x': undefined}) + class TestWraps(TestUpdateWrapper): @@ -942,8 +1101,13 @@ def mycmp(x, y): @unittest.skipIf(support.MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_cmp_to_signature(self): - self.assertEqual(str(Signature.from_callable(self.cmp_to_key)), - '(mycmp)') + sig = Signature.from_callable(self.cmp_to_key) + self.assertEqual(str(sig), '(mycmp)') + def mycmp(x, y): + return y - x + sig = Signature.from_callable(self.cmp_to_key(mycmp)) + self.assertEqual(str(sig), '(obj)') + @unittest.skipUnless(c_functools, 'requires the C _functools module') @@ -1858,9 +2022,10 @@ def test_staticmethod(x): self.assertIsNone(ref()) def test_common_signatures(self): - def orig(): ... + def orig(a, /, b, c=True): ... lru = self.module.lru_cache(1)(orig) + self.assertEqual(str(Signature.from_callable(lru)), '(a, /, b, c=True)') self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') @@ -1875,7 +2040,7 @@ def fib(n): return fib(n-1) + fib(n-2) if not support.Py_DEBUG: - depth = support.Py_C_RECURSION_LIMIT*2//7 + depth = support.get_c_recursion_limit()*2//7 with support.infinite_recursion(): fib(depth) if self.module == c_functools: @@ -2706,7 +2871,10 @@ def static_func(arg: int) -> str: A().static_func ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual(A.func.__name__, 'func') @@ -2795,7 +2963,10 @@ def decorated_classmethod(cls, arg: int) -> str: WithSingleDispatch().decorated_classmethod ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual( @@ -2867,11 +3038,26 @@ def _(arg: typing.Union[int, typing.Iterable[str]]): def test_invalid_positional_argument(self): @functools.singledispatch - def f(*args): + def f(*args, **kwargs): pass msg = 'f requires at least 1 positional argument' with self.assertRaisesRegex(TypeError, msg): f() + msg = 'f requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + f(a=1) + + def test_invalid_positional_argument_singledispatchmethod(self): + class A: + @functools.singledispatchmethod + def t(self, *args, **kwargs): + pass + msg = 't requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + A().t() + msg = 't requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + A().t(a=1) def test_union(self): @functools.singledispatch @@ -2995,6 +3181,27 @@ def _(arg: typing.List[float] | bytes): self.assertEqual(f(""), "default") self.assertEqual(f(b""), "default") + def test_forward_reference(self): + @functools.singledispatch + def f(arg, arg2=None): + return "default" + + @f.register + def _(arg: str, arg2: undefined = None): + return "forward reference" + + self.assertEqual(f(1), "default") + self.assertEqual(f(""), "forward reference") + + def test_unresolved_forward_reference(self): + @functools.singledispatch + def f(arg): + return "default" + + with self.assertRaisesRegex(TypeError, "is an unresolved forward reference"): + @f.register + def _(arg: undefined): + return "forward reference" class CachedCostItem: _cost = 1 @@ -3123,7 +3330,10 @@ def test_access_from_class(self): self.assertIsInstance(CachedCostItem.cost, py_functools.cached_property) def test_doc(self): - self.assertEqual(CachedCostItem.cost.__doc__, "The cost of the item.") + self.assertEqual(CachedCostItem.cost.__doc__, + ("The cost of the item." + if support.HAVE_DOCSTRINGS + else None)) def test_module(self): self.assertEqual(CachedCostItem.cost.__module__, CachedCostItem.__module__) diff --git a/Lib/test/test_future_stmt/nested_scope.py b/Lib/test/test_future_stmt/nested_scope.py index 3d7fc860a37655..a8433a42cbb6b0 100644 --- a/Lib/test/test_future_stmt/nested_scope.py +++ b/Lib/test/test_future_stmt/nested_scope.py @@ -1,6 +1,6 @@ """This is a test""" -from __future__ import nested_scopes; import site +from __future__ import nested_scopes; import site # noqa: F401 def f(x): def g(y): diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 2c8ceb664cb362..42c6cb3fefac33 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -67,19 +67,19 @@ def test_future_single_import(self): with import_helper.CleanImport( 'test.test_future_stmt.test_future_single_import', ): - from test.test_future_stmt import test_future_single_import + from test.test_future_stmt import test_future_single_import # noqa: F401 def test_future_multiple_imports(self): with import_helper.CleanImport( 'test.test_future_stmt.test_future_multiple_imports', ): - from test.test_future_stmt import test_future_multiple_imports + from test.test_future_stmt import test_future_multiple_imports # noqa: F401 def test_future_multiple_features(self): with import_helper.CleanImport( "test.test_future_stmt.test_future_multiple_features", ): - from test.test_future_stmt import test_future_multiple_features + from test.test_future_stmt import test_future_multiple_features # noqa: F401 def test_unknown_future_flag(self): code = """ @@ -88,7 +88,7 @@ def test_unknown_future_flag(self): """ self.assertSyntaxError( code, lineno=2, - message='future feature rested_snopes is not defined', + message='future feature rested_snopes is not defined', offset=24, ) def test_future_import_not_on_top(self): @@ -137,23 +137,23 @@ def test_future_import_star(self): code = """ from __future__ import * """ - self.assertSyntaxError(code, message='future feature * is not defined') + self.assertSyntaxError(code, message='future feature * is not defined', offset=24) def test_future_import_braces(self): code = """ from __future__ import braces """ # Congrats, you found an easter egg! - self.assertSyntaxError(code, message='not a chance') + self.assertSyntaxError(code, message='not a chance', offset=24) code = """ from __future__ import nested_scopes, braces """ - self.assertSyntaxError(code, message='not a chance') + self.assertSyntaxError(code, message='not a chance', offset=39) def test_module_with_future_import_not_on_top(self): with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future + from test.test_future_stmt import badsyntax_future # noqa: F401 self.check_syntax_error(cm.exception, "badsyntax_future", lineno=3) def test_ensure_flags_dont_clash(self): @@ -171,26 +171,6 @@ def test_ensure_flags_dont_clash(self): } self.assertCountEqual(set(flags.values()), flags.values()) - def test_parserhack(self): - # test that the parser.c::future_hack function works as expected - # Note: although this test must pass, it's not testing the original - # bug as of 2.6 since the with statement is not optional and - # the parser hack disabled. If a new keyword is introduced in - # 2.6, change this to refer to the new future import. - try: - exec("from __future__ import print_function; print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - - try: - exec("from __future__ import (print_function); print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - def test_unicode_literals_exec(self): scope = {} exec("from __future__ import unicode_literals; x = ''", {}, scope) @@ -203,6 +183,25 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) + def test_future_dotted_import(self): + with self.assertRaises(ImportError): + exec("from .__future__ import spam") + + code = dedent( + """ + from __future__ import print_function + from ...__future__ import ham + """ + ) + with self.assertRaises(ImportError): + exec(code) + + code = """ + from .__future__ import nested_scopes + from __future__ import barry_as_FLUFL + """ + self.assertSyntaxError(code, lineno=2) + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index b01f344cb14a1a..2b3c0d3baddeaf 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,11 +1,14 @@ import unittest import unittest.mock +from test import support from test.support import (verbose, refcount_test, - cpython_only, requires_subprocess, Py_GIL_DISABLED) + cpython_only, requires_subprocess, + requires_gil_enabled, + Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script -from test.support import threading_helper +from test.support import threading_helper, gc_threshold import gc import sys @@ -16,17 +19,16 @@ import weakref try: + import _testcapi from _testcapi import with_tp_del + from _testcapi import ContainerNoGC except ImportError: + _testcapi = None def with_tp_del(cls): class C(object): def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.with_tp_del') + raise unittest.SkipTest('requires _testcapi.with_tp_del') return C - -try: - from _testcapi import ContainerNoGC -except ImportError: ContainerNoGC = None ### Support code @@ -225,7 +227,9 @@ def test_function(self): exec("def f(): pass\n", d) gc.collect() del d - self.assertEqual(gc.collect(), 2) + # In the free-threaded build, the count returned by `gc.collect()` + # is 3 because it includes f's code object. + self.assertIn(gc.collect(), (2, 3)) def test_function_tp_clear_leaves_consistent_state(self): # https://github.com/python/cpython/issues/91636 @@ -363,6 +367,7 @@ def __del__(self): # To minimize variations, though, we first store the get_count() results # and check them at the end. @refcount_test + @requires_gil_enabled('needs precise allocation counts') def test_get_count(self): gc.collect() a, b, c = gc.get_count() @@ -383,19 +388,11 @@ def test_collect_generations(self): # each call to collect(N) x = [] gc.collect(0) - # x is now in gen 1 + # x is now in the old gen a, b, c = gc.get_count() - gc.collect(1) - # x is now in gen 2 - d, e, f = gc.get_count() - gc.collect(2) - # x is now in gen 3 - g, h, i = gc.get_count() - # We don't check a, d, g since their exact values depends on + # We don't check a since its exact values depends on # internal implementation details of the interpreter. self.assertEqual((b, c), (1, 0)) - self.assertEqual((e, f), (0, 1)) - self.assertEqual((h, i), (0, 0)) def test_trashcan(self): class Ouch: @@ -477,7 +474,7 @@ def run_thread(): make_nested() old_switchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-5) + support.setswitchinterval(1e-5) try: exit = [] threads = [] @@ -688,6 +685,7 @@ def do_work(): @cpython_only @requires_subprocess() + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_garbage_at_shutdown(self): import subprocess code = """if 1: @@ -822,7 +820,7 @@ def test_get_objects(self): any(l is element for element in gc.get_objects()) ) - @unittest.skipIf(Py_GIL_DISABLED, 'need generational GC') + @requires_gil_enabled('need generational GC') def test_get_objects_generations(self): gc.collect() l = [] @@ -830,42 +828,10 @@ def test_get_objects_generations(self): self.assertTrue( any(l is element for element in gc.get_objects(generation=0)) ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=0) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=1) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=2) + gc.collect() self.assertFalse( any(l is element for element in gc.get_objects(generation=0)) ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=2)) - ) del l gc.collect() @@ -1075,6 +1041,112 @@ class Z: callback.assert_not_called() gc.enable() + @cpython_only + def test_get_referents_on_capsule(self): + # gh-124538: Calling gc.get_referents() on an untracked capsule must not crash. + import _datetime + import _socket + untracked_capsule = _datetime.datetime_CAPI + tracked_capsule = _socket.CAPI + + # For whoever sees this in the future: if this is failing + # after making datetime's capsule tracked, that's fine -- this isn't something + # users are relying on. Just find a different capsule that is untracked. + self.assertFalse(gc.is_tracked(untracked_capsule)) + self.assertTrue(gc.is_tracked(tracked_capsule)) + + self.assertEqual(len(gc.get_referents(untracked_capsule)), 0) + gc.get_referents(tracked_capsule) + + @cpython_only + def test_get_objects_during_gc(self): + # gh-125859: Calling gc.get_objects() or gc.get_referrers() during a + # collection should not crash. + test = self + collected = False + + class GetObjectsOnDel: + def __del__(self): + nonlocal collected + collected = True + objs = gc.get_objects() + # NB: can't use "in" here because some objects override __eq__ + for obj in objs: + test.assertTrue(obj is not self) + test.assertEqual(gc.get_referrers(self), []) + + obj = GetObjectsOnDel() + obj.cycle = obj + del obj + + gc.collect() + self.assertTrue(collected) + + +class IncrementalGCTests(unittest.TestCase): + + def setUp(self): + # Reenable GC as it is disabled module-wide + gc.enable() + + def tearDown(self): + gc.disable() + + @requires_gil_enabled("Free threading does not support incremental GC") + # Use small increments to emulate longer running process in a shorter time + @gc_threshold(200, 10) + def test_incremental_gc_handles_fast_cycle_creation(self): + + class LinkedList: + + #Use slots to reduce number of implicit objects + __slots__ = "next", "prev", "surprise" + + def __init__(self, next=None, prev=None): + self.next = next + if next is not None: + next.prev = self + self.prev = prev + if prev is not None: + prev.next = self + + def make_ll(depth): + head = LinkedList() + for i in range(depth): + head = LinkedList(head, head.prev) + return head + + head = make_ll(1000) + count = 1000 + + # There will be some objects we aren't counting, + # e.g. the gc stats dicts. This test checks + # that the counts don't grow, so we try to + # correct for the uncounted objects + # This is just an estimate. + CORRECTION = 20 + + enabled = gc.isenabled() + gc.enable() + olds = [] + for i in range(20_000): + newhead = make_ll(20) + count += 20 + newhead.surprise = head + olds.append(newhead) + if len(olds) == 20: + stats = gc.get_stats() + young = stats[0] + incremental = stats[1] + old = stats[2] + collected = young['collected'] + incremental['collected'] + old['collected'] + count += CORRECTION + live = count - collected + self.assertLess(live, 25000) + del olds[:] + if not enabled: + gc.disable() + class GCCallbackTests(unittest.TestCase): def setUp(self): @@ -1197,6 +1269,7 @@ def test_collect_garbage(self): self.assertEqual(len(gc.garbage), 0) + @requires_subprocess() @unittest.skipIf(BUILD_WITH_NDEBUG, 'built with -NDEBUG') def test_refcount_errors(self): @@ -1316,6 +1389,10 @@ def callback(ignored): junk = [] i = 0 detector = GC_Detector() + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: @@ -1329,6 +1406,7 @@ def callback(ignored): # with an empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref @@ -1383,6 +1461,10 @@ def __del__(self): detector = GC_Detector() junk = [] i = 0 + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: @@ -1396,6 +1478,32 @@ def __del__(self): # empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) + def test_indirect_calls_with_gc_disabled(self): + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") + junk.append([]) # this will eventually trigger gc + + try: + gc.disable() + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + break + junk.append([]) # this may eventually trigger gc (if it is enabled) + + self.assertEqual(i, 10001) + finally: + gc.enable() + class PythonFinalizationTests(unittest.TestCase): def test_ast_fini(self): diff --git a/Lib/test/test_gdb/__init__.py b/Lib/test/test_gdb/__init__.py index 99557739af6748..0dd721780233e6 100644 --- a/Lib/test/test_gdb/__init__.py +++ b/Lib/test/test_gdb/__init__.py @@ -24,6 +24,9 @@ if support.check_cflags_pgo(): raise unittest.SkipTest("test_gdb is not reliable on PGO builds") +if support.check_bolt_optimized(): + raise unittest.SkipTest("test_gdb is not reliable on BOLT optimized builds") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_gdb/test_backtrace.py b/Lib/test/test_gdb/test_backtrace.py index c41e7cb7c210de..714853c7b4732d 100644 --- a/Lib/test/test_gdb/test_backtrace.py +++ b/Lib/test/test_gdb/test_backtrace.py @@ -49,6 +49,7 @@ def test_bt_full(self): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") + @support.requires_gil_enabled() @support.requires_resource('cpu') def test_threads(self): 'Verify that "py-bt" indicates threads that are waiting for the GIL' diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index ca1228ee7008a9..ff9a52b7adac8a 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -31,8 +31,9 @@ def skip_if_different_mount_drives(): with test_tools.imports_under_tool("cases_generator"): from analyzer import StackItem import parser - from stack import Stack + from stack import Local, Stack import tier1_generator + import optimizer_generator def handle_stderr(): @@ -59,9 +60,9 @@ def test_effect_sizes(self): stack.pop(y) stack.pop(x) for out in outputs: - stack.push(out) - self.assertEqual(stack.base_offset.to_c(), "-1 - oparg*2 - oparg") - self.assertEqual(stack.top_offset.to_c(), "1 - oparg*2 - oparg + oparg*4") + stack.push(Local.undefined(out)) + self.assertEqual(stack.base_offset.to_c(), "-1 - oparg - oparg*2") + self.assertEqual(stack.top_offset.to_c(), "1 - oparg - oparg*2 + oparg*4") class TestGeneratedCases(unittest.TestCase): @@ -121,7 +122,7 @@ def run_cases_test(self, input: str, expected: str): def test_inst_no_args(self): input = """ inst(OP, (--)) { - spam(); + SPAM(); } """ output = """ @@ -129,7 +130,7 @@ def test_inst_no_args(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - spam(); + SPAM(); DISPATCH(); } """ @@ -138,7 +139,8 @@ def test_inst_no_args(self): def test_inst_one_pop(self): input = """ inst(OP, (value --)) { - spam(); + SPAM(value); + DEAD(value); } """ output = """ @@ -146,10 +148,11 @@ def test_inst_one_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - spam(); + SPAM(value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -158,7 +161,7 @@ def test_inst_one_pop(self): def test_inst_one_push(self): input = """ inst(OP, (-- res)) { - spam(); + res = SPAM(); } """ output = """ @@ -166,10 +169,11 @@ def test_inst_one_push(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *res; - spam(); + _PyStackRef res; + res = SPAM(); stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -178,7 +182,8 @@ def test_inst_one_push(self): def test_inst_one_push_one_pop(self): input = """ inst(OP, (value -- res)) { - spam(); + res = SPAM(value); + DEAD(value); } """ output = """ @@ -186,10 +191,10 @@ def test_inst_one_push_one_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - spam(); + res = SPAM(value); stack_pointer[-1] = res; DISPATCH(); } @@ -199,7 +204,9 @@ def test_inst_one_push_one_pop(self): def test_binary_op(self): input = """ inst(OP, (left, right -- res)) { - spam(); + res = SPAM(left, right); + INPUTS_DEAD(); + } """ output = """ @@ -207,14 +214,15 @@ def test_binary_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - spam(); + res = SPAM(left, right); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -223,7 +231,8 @@ def test_binary_op(self): def test_overlap(self): input = """ inst(OP, (left, right -- left, result)) { - spam(); + result = SPAM(left, right); + INPUTS_DEAD(); } """ output = """ @@ -231,25 +240,26 @@ def test_overlap(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *right; - PyObject *left; - PyObject *result; + _PyStackRef left; + _PyStackRef right; + _PyStackRef result; right = stack_pointer[-1]; left = stack_pointer[-2]; - spam(); + result = SPAM(left, right); stack_pointer[-1] = result; DISPATCH(); } """ self.run_cases_test(input, output) - def test_predictions_and_eval_breaker(self): + def test_predictions(self): input = """ - inst(OP1, (arg -- rest)) { + inst(OP1, (arg -- res)) { + res = Py_None; } inst(OP3, (arg -- res)) { DEOPT_IF(xxx); - CHECK_EVAL_BREAKER(); + res = Py_None; } family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; """ @@ -259,10 +269,9 @@ def test_predictions_and_eval_breaker(self): next_instr += 1; INSTRUCTION_STATS(OP1); PREDICTED(OP1); - PyObject *arg; - PyObject *rest; - arg = stack_pointer[-1]; - stack_pointer[-1] = rest; + _PyStackRef res; + res = Py_None; + stack_pointer[-1] = res; DISPATCH(); } @@ -271,17 +280,76 @@ def test_predictions_and_eval_breaker(self): next_instr += 1; INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); - PyObject *arg; - PyObject *res; - arg = stack_pointer[-1]; + _PyStackRef res; DEOPT_IF(xxx, OP1); + res = Py_None; + stack_pointer[-1] = res; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_sync_sp(self): + input = """ + inst(A, (arg -- res)) { + SYNC_SP(); + escaping_call(); + res = Py_None; + } + inst(B, (arg -- res)) { + res = Py_None; + SYNC_SP(); + escaping_call(); + } + """ + output = """ + TARGET(A) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(A); + _PyStackRef res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = Py_None; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(B) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(B); + _PyStackRef res; + res = Py_None; stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } """ self.run_cases_test(input, output) + + def test_pep7_condition(self): + input = """ + inst(OP, (arg1 -- out)) { + if (arg1) + out = 0; + else { + out = 1; + } + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + def test_error_if_plain(self): input = """ inst(OP, (--)) { @@ -320,7 +388,10 @@ def test_error_if_plain_with_comment(self): def test_error_if_pop(self): input = """ inst(OP, (left, right -- res)) { + SPAM(left, right); + INPUTS_DEAD(); ERROR_IF(cond, label); + res = 0; } """ output = """ @@ -328,14 +399,45 @@ def test_error_if_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + SPAM(left, right); if (cond) goto pop_2_label; + res = 0; stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_pop_with_result(self): + input = """ + inst(OP, (left, right -- res)) { + res = SPAM(left, right); + INPUTS_DEAD(); + ERROR_IF(cond, label); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = SPAM(left, right); + if (cond) goto pop_2_label; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -348,14 +450,16 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(OP); - PyObject *value; - value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; uint32_t extra = read_u32(&this_instr[2].cache); + (void)extra; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -384,10 +488,12 @@ def test_macro_instruction(self): } op(OP2, (extra/2, arg2, left, right -- res)) { res = op2(arg2, left, right); + INPUTS_DEAD(); } macro(OP) = OP1 + cache/2 + OP2; inst(OP3, (unused/5, arg2, left, right -- res)) { res = op3(arg2, left, right); + INPUTS_DEAD(); } family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; """ @@ -397,40 +503,52 @@ def test_macro_instruction(self): next_instr += 6; INSTRUCTION_STATS(OP); PREDICTED(OP); - _Py_CODEUNIT *this_instr = next_instr - 6; - PyObject *right; - PyObject *left; - PyObject *arg2; - PyObject *res; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef arg2; + _PyStackRef res; // _OP1 - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* Skip 2 cache entries */ // OP2 - arg2 = stack_pointer[-3]; { + arg2 = stack_pointer[-3]; uint32_t extra = read_u32(&this_instr[4].cache); + (void)extra; + _PyFrame_SetStackPointer(frame, stack_pointer); res = op2(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer[-3] = res; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(OP1) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(OP1); - PyObject *right; - PyObject *left; + _PyStackRef left; + _PyStackRef right; right = stack_pointer[-1]; left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -439,17 +557,20 @@ def test_macro_instruction(self): next_instr += 6; INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *arg2; - PyObject *res; + _PyStackRef arg2; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; /* Skip 5 cache entries */ right = stack_pointer[-1]; left = stack_pointer[-2]; arg2 = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); res = op3(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-3] = res; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -458,7 +579,7 @@ def test_macro_instruction(self): def test_unused_caches(self): input = """ inst(OP, (unused/1, unused/2 --)) { - body(); + body; } """ output = """ @@ -468,7 +589,7 @@ def test_unused_caches(self): INSTRUCTION_STATS(OP); /* Skip 1 cache entry */ /* Skip 2 cache entries */ - body(); + body; DISPATCH(); } """ @@ -476,7 +597,7 @@ def test_unused_caches(self): def test_pseudo_instruction_no_flags(self): input = """ - pseudo(OP) = { + pseudo(OP, (in -- out1, out2)) = { OP1, }; @@ -495,7 +616,7 @@ def test_pseudo_instruction_no_flags(self): def test_pseudo_instruction_with_flags(self): input = """ - pseudo(OP, (HAS_ARG, HAS_JUMP)) = { + pseudo(OP, (in1, in2 --), (HAS_ARG, HAS_JUMP)) = { OP1, }; @@ -512,10 +633,40 @@ def test_pseudo_instruction_with_flags(self): """ self.run_cases_test(input, output) + def test_pseudo_instruction_as_sequence(self): + input = """ + pseudo(OP, (in -- out1, out2)) = [ + OP1, OP2 + ]; + + inst(OP1, (--)) { + } + + inst(OP2, (--)) { + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + + TARGET(OP2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_input(self): input = """ inst(OP, (below, values[oparg*2], above --)) { - spam(); + SPAM(values, oparg); } """ output = """ @@ -523,14 +674,11 @@ def test_array_input(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *above; - PyObject **values; - PyObject *below; - above = stack_pointer[-1]; + _PyStackRef *values; values = &stack_pointer[-1 - oparg*2]; - below = stack_pointer[-2 - oparg*2]; - spam(); + SPAM(values, oparg); stack_pointer += -2 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -539,7 +687,9 @@ def test_array_input(self): def test_array_output(self): input = """ inst(OP, (unused, unused -- below, values[oparg*3], above)) { - spam(values, oparg); + SPAM(values, oparg); + below = 0; + above = 0; } """ output = """ @@ -547,14 +697,17 @@ def test_array_output(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *below; - PyObject **values; - PyObject *above; + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; values = &stack_pointer[-1]; - spam(values, oparg); + SPAM(values, oparg); + below = 0; + above = 0; stack_pointer[-2] = below; stack_pointer[-1 + oparg*3] = above; stack_pointer += oparg*3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -563,7 +716,8 @@ def test_array_output(self): def test_array_input_output(self): input = """ inst(OP, (values[oparg] -- values[oparg], above)) { - spam(values, oparg); + SPAM(values, oparg); + above = 0; } """ output = """ @@ -571,12 +725,14 @@ def test_array_input_output(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject **values; - PyObject *above; + _PyStackRef *values; + _PyStackRef above; values = &stack_pointer[-oparg]; - spam(values, oparg); + SPAM(values, oparg); + above = 0; stack_pointer[0] = above; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -593,12 +749,13 @@ def test_array_error_if(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject **values; - PyObject *extra; - values = &stack_pointer[-oparg]; - extra = stack_pointer[-1 - oparg]; - if (oparg == 0) { stack_pointer += -1 - oparg; goto somewhere; } + if (oparg == 0) { + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto somewhere; + } stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -607,7 +764,10 @@ def test_array_error_if(self): def test_cond_effect(self): input = """ inst(OP, (aa, input if ((oparg & 1) == 1), cc -- xx, output if (oparg & 2), zz)) { - output = spam(oparg, input); + output = SPAM(oparg, aa, cc, input); + INPUTS_DEAD(); + xx = 0; + zz = 0; } """ output = """ @@ -615,20 +775,23 @@ def test_cond_effect(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *cc; - PyObject *input = NULL; - PyObject *aa; - PyObject *xx; - PyObject *output = NULL; - PyObject *zz; + _PyStackRef aa; + _PyStackRef input = PyStackRef_NULL; + _PyStackRef cc; + _PyStackRef xx; + _PyStackRef output = PyStackRef_NULL; + _PyStackRef zz; cc = stack_pointer[-1]; if ((oparg & 1) == 1) { input = stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)]; } aa = stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)]; - output = spam(oparg, input); + output = SPAM(oparg, aa, cc, input); + xx = 0; + zz = 0; stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)] = xx; if (oparg & 2) stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)] = output; stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0)] = zz; stack_pointer += -(((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -637,10 +800,14 @@ def test_cond_effect(self): def test_macro_cond_effect(self): input = """ op(A, (left, middle, right --)) { - # Body of A + USE(left, middle, right); + INPUTS_DEAD(); } op(B, (-- deep, extra if (oparg), res)) { - # Body of B + deep = -1; + res = 0; + extra = 1; + INPUTS_DEAD(); } macro(M) = A + B; """ @@ -649,27 +816,30 @@ def test_macro_cond_effect(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - PyObject *right; - PyObject *middle; - PyObject *left; - PyObject *deep; - PyObject *extra = NULL; - PyObject *res; + _PyStackRef left; + _PyStackRef middle; + _PyStackRef right; + _PyStackRef deep; + _PyStackRef extra = PyStackRef_NULL; + _PyStackRef res; // A - right = stack_pointer[-1]; - middle = stack_pointer[-2]; - left = stack_pointer[-3]; { - # Body of A + right = stack_pointer[-1]; + middle = stack_pointer[-2]; + left = stack_pointer[-3]; + USE(left, middle, right); } // B { - # Body of B + deep = -1; + res = 0; + extra = 1; } stack_pointer[-3] = deep; if (oparg) stack_pointer[-2] = extra; stack_pointer[-2 + ((oparg) ? 1 : 0)] = res; stack_pointer += -1 + ((oparg) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -678,10 +848,10 @@ def test_macro_cond_effect(self): def test_macro_push_push(self): input = """ op(A, (-- val1)) { - val1 = spam(); + val1 = SPAM(); } op(B, (-- val2)) { - val2 = spam(); + val2 = SPAM(); } macro(M) = A + B; """ @@ -690,19 +860,20 @@ def test_macro_push_push(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - PyObject *val1; - PyObject *val2; + _PyStackRef val1; + _PyStackRef val2; // A { - val1 = spam(); + val1 = SPAM(); } // B { - val2 = spam(); + val2 = SPAM(); } stack_pointer[0] = val1; stack_pointer[1] = val2; stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -711,10 +882,10 @@ def test_macro_push_push(self): def test_override_inst(self): input = """ inst(OP, (--)) { - spam(); + spam; } override inst(OP, (--)) { - ham(); + ham; } """ output = """ @@ -722,7 +893,7 @@ def test_override_inst(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - ham(); + ham; DISPATCH(); } """ @@ -731,11 +902,11 @@ def test_override_inst(self): def test_override_op(self): input = """ op(OP, (--)) { - spam(); + spam; } macro(M) = OP; override op(OP, (--)) { - ham(); + ham; } """ output = """ @@ -743,7 +914,7 @@ def test_override_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - ham(); + ham; DISPATCH(); } """ @@ -752,7 +923,7 @@ def test_override_op(self): def test_annotated_inst(self): input = """ pure inst(OP, (--)) { - ham(); + ham; } """ output = """ @@ -760,7 +931,7 @@ def test_annotated_inst(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - ham(); + ham; DISPATCH(); } """ @@ -769,7 +940,7 @@ def test_annotated_inst(self): def test_annotated_op(self): input = """ pure op(OP, (--)) { - spam(); + SPAM(); } macro(M) = OP; """ @@ -778,7 +949,7 @@ def test_annotated_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - spam(); + SPAM(); DISPATCH(); } """ @@ -786,12 +957,661 @@ def test_annotated_op(self): input = """ pure register specializing op(OP, (--)) { - spam(); + SPAM(); } macro(M) = OP; """ self.run_cases_test(input, output) + def test_deopt_and_exit(self): + input = """ + pure op(OP, (arg1 -- out)) { + DEOPT_IF(1); + EXIT_IF(1); + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_of_one(self): + input = """ + inst(OP, (arg[1] -- out[1])) { + out[0] = arg[0]; + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef *out; + arg = &stack_pointer[-1]; + out = &stack_pointer[-1]; + out[0] = arg[0]; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pointer_to_stackref(self): + input = """ + inst(OP, (arg: _PyStackRef * -- out)) { + out = *arg; + DEAD(arg); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef out; + arg = (_PyStackRef *)stack_pointer[-1].bits; + out = *arg; + stack_pointer[-1] = out; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_cached_value(self): + input = """ + op(FIRST, (arg1 -- out)) { + out = arg1; + } + + op(SECOND, (unused -- unused)) { + } + + macro(BOTH) = FIRST + SECOND; + """ + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_unused_named_values(self): + input = """ + op(OP, (named -- named)) { + } + + macro(INST) = OP; + """ + output = """ + TARGET(INST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INST); + DISPATCH(); + } + + """ + self.run_cases_test(input, output) + + def test_used_unused_used(self): + input = """ + op(FIRST, (w -- w)) { + USE(w); + } + + op(SECOND, (x -- x)) { + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef w; + _PyStackRef y; + // FIRST + { + w = stack_pointer[-1]; + USE(w); + } + // SECOND + { + } + // THIRD + { + y = w; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_used_used(self): + input = """ + op(FIRST, (w -- w)) { + } + + op(SECOND, (x -- x)) { + USE(x); + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + // FIRST + { + } + // SECOND + { + x = stack_pointer[-1]; + USE(x); + } + // THIRD + { + y = x; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_flush(self): + input = """ + op(FIRST, ( -- a, b)) { + a = 0; + b = 1; + } + + op(SECOND, (a, b -- )) { + USE(a, b); + INPUTS_DEAD(); + } + + macro(TEST) = FIRST + flush + SECOND; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 0; + b = 1; + } + // flush + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + // SECOND + { + USE(a, b); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_on_error_peeks(self): + + input = """ + op(FIRST, (x, y -- a, b)) { + a = x; + DEAD(x); + b = y; + DEAD(y); + } + + op(SECOND, (a, b -- a, b)) { + } + + op(THIRD, (j, k --)) { + INPUTS_DEAD(); // Mark j and k as used + ERROR_IF(cond, error); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + _PyStackRef a; + _PyStackRef b; + // FIRST + { + y = stack_pointer[-1]; + x = stack_pointer[-2]; + a = x; + b = y; + } + // SECOND + { + } + // THIRD + { + // Mark j and k as used + if (cond) goto pop_2_error; + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_push_then_error(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a -- a, b)) { + b = 1; + ERROR_IF(cond, error); + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 1; + } + // SECOND + { + b = 1; + if (cond) { + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_true(self): + + input = """ + inst(OP1, ( --)) { + ERROR_IF(true, here); + } + inst(OP2, ( --)) { + ERROR_IF(1, there); + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + goto here; + } + + TARGET(OP2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + goto there; + } + """ + self.run_cases_test(input, output) + + def test_scalar_array_inconsistency(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_size_inconsistency(self): + + input = """ + op(FIRST, ( -- a[2])) { + a[0] = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_reload(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_stack_reload_only(self): + + input = """ + inst(BALANCED, ( -- )) { + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_only(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_instruction_size_macro(self): + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + """ + + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + frame->return_offset = 1 ; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + # Two instructions of different sizes referencing the same + # uop containing the `INSTRUCTION_SIZE` macro is not allowed. + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + macro(OP2) = unused/1 + OP; + """ + + output = "" # No output needed as this should raise an error. + with self.assertRaisesRegex(SyntaxError, "All instructions containing a uop"): + self.run_cases_test(input, output) + + def test_escaping_call_next_to_cmacro(self): + input = """ + inst(OP, (--)) { + #ifdef Py_GIL_DISABLED + escaping_call(); + #else + another_escaping_call(); + #endif + yet_another_escaping_call(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #else + _PyFrame_SetStackPointer(frame, stack_pointer); + another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + yet_another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + +class TestGeneratedAbstractCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_input2_filename = os.path.join(self.temp_dir, "input2.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_input2_filename, + self.temp_output_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, input2: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with open(self.temp_input2_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input2) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + optimizer_generator.generate_tier2_abstract_from_files( + [self.temp_input_filename, self.temp_input2_filename], + self.temp_output_filename + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual = "".join(lines) + self.assertEqual(actual.strip(), expected.strip()) + + def test_overridden_abstract(self): + input = """ + pure op(OP, (--)) { + SPAM(); + } + """ + input2 = """ + pure op(OP, (--)) { + eggs(); + } + """ + output = """ + case OP: { + eggs(); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_overridden_abstract_args(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + op(OP2, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + input2 = """ + op(OP, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + output = """ + case OP: { + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; + arg1 = stack_pointer[-1]; + out = EGGS(arg1); + stack_pointer[-1] = out; + break; + } + + case OP2: { + _Py_UopsSymbol *out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_no_overridden_case(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + + pure op(OP2, (arg1 -- out)) { + } + + """ + input2 = """ + pure op(OP2, (arg1 -- out)) { + out = NULL; + } + """ + output = """ + case OP: { + _Py_UopsSymbol *out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + + case OP2: { + _Py_UopsSymbol *out; + out = NULL; + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_missing_override_failure(self): + input = """ + pure op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + pure op(OTHER, (arg1 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(AssertionError, "All abstract uops"): + self.run_cases_test(input, input2, output) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index d48f0d47ba1962..bf2cb1160723b0 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -6,6 +6,8 @@ import unittest import weakref import inspect +import textwrap +import types from test import support @@ -89,9 +91,12 @@ def gen(): self.assertEqual(gc.garbage, old_garbage) def test_lambda_generator(self): - # Issue #23192: Test that a lambda returning a generator behaves + # bpo-23192, gh-119897: Test that a lambda returning a generator behaves # like the equivalent function f = lambda: (yield 1) + self.assertIsInstance(f(), types.GeneratorType) + self.assertEqual(next(f()), 1) + def g(): return (yield 1) # test 'yield from' @@ -108,6 +113,27 @@ def g3(): return (yield from f()) gen.send(2) self.assertEqual(cm.exception.value, 2) + def test_generator_resurrect(self): + # Test that a resurrected generator still has a valid gi_code + resurrected = [] + + # Resurrect a generator in a finalizer + exec(textwrap.dedent(""" + def gen(): + try: + yield + except: + resurrected.append(g) + + g = gen() + next(g) + """), {"resurrected": resurrected}) + + support.gc_collect() + + self.assertEqual(len(resurrected), 1) + self.assertIsInstance(resurrected[0].gi_code, types.CodeType) + class GeneratorTest(unittest.TestCase): @@ -242,6 +268,79 @@ def loop(): #This should not raise loop() + +class ModifyUnderlyingIterableTest(unittest.TestCase): + iterables = [ + range(0), + range(20), + [1, 2, 3], + (2,), + {13, 48, 211}, + frozenset((15, 8, 6)), + {1: 2, 3: 4}, + ] + + non_iterables = [ + None, + 42, + 3.0, + 2j, + ] + + def genexpr(self): + return (x for x in range(10)) + + def genfunc(self): + def gen(it): + for x in it: + yield x + return gen(range(10)) + + def process_tests(self, get_generator): + for obj in self.iterables: + g_obj = get_generator(obj) + with self.subTest(g_obj=g_obj, obj=obj): + self.assertListEqual(list(g_obj), list(obj)) + + g_iter = get_generator(iter(obj)) + with self.subTest(g_iter=g_iter, obj=obj): + self.assertListEqual(list(g_iter), list(obj)) + + err_regex = "'.*' object is not iterable" + for obj in self.non_iterables: + g_obj = get_generator(obj) + with self.subTest(g_obj=g_obj): + self.assertRaisesRegex(TypeError, err_regex, list, g_obj) + + def test_modify_f_locals(self): + def modify_f_locals(g, local, obj): + g.gi_frame.f_locals[local] = obj + return g + + def get_generator_genexpr(obj): + return modify_f_locals(self.genexpr(), '.0', obj) + + def get_generator_genfunc(obj): + return modify_f_locals(self.genfunc(), 'it', obj) + + self.process_tests(get_generator_genexpr) + self.process_tests(get_generator_genfunc) + + def test_new_gen_from_gi_code(self): + def new_gen_from_gi_code(g, obj): + generator_func = types.FunctionType(g.gi_code, {}) + return generator_func(obj) + + def get_generator_genexpr(obj): + return new_gen_from_gi_code(self.genexpr(), obj) + + def get_generator_genfunc(obj): + return new_gen_from_gi_code(self.genfunc(), obj) + + self.process_tests(get_generator_genexpr) + self.process_tests(get_generator_genfunc) + + class ExceptionTest(unittest.TestCase): # Tests for the issue #23353: check that the currently handled exception # is correctly saved/restored in PyEval_EvalFrameEx(). @@ -532,6 +631,26 @@ def f(): with self.assertRaises(RuntimeError): gen.close() + def test_close_releases_frame_locals(self): + # See gh-118272 + + class Foo: + pass + + f = Foo() + f_wr = weakref.ref(f) + + def genfn(): + a = f + yield + + g = genfn() + next(g) + del f + g.close() + support.gc_collect() + self.assertIsNone(f_wr()) + class GeneratorThrowTest(unittest.TestCase): @@ -887,7 +1006,7 @@ def b(): File "", line 1, in ? File "", line 2, in g File "", line 2, in f - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero >>> next(k) # and the generator cannot be resumed Traceback (most recent call last): File "", line 1, in ? @@ -2223,6 +2342,16 @@ def printsolution(self, x): ... SyntaxError: 'yield' outside function +>>> f=lambda: (yield from (1,2)), (yield from (3,4)) +Traceback (most recent call last): + ... +SyntaxError: 'yield from' outside function + +>>> yield from [1,2] +Traceback (most recent call last): + ... +SyntaxError: 'yield from' outside function + >>> def f(): x = yield = y Traceback (most recent call last): ... diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 04cb810d9babbf..3048d038c782d4 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -49,7 +49,7 @@ ShareableList = None from os import DirEntry from re import Pattern, Match -from types import GenericAlias, MappingProxyType, AsyncGeneratorType +from types import GenericAlias, MappingProxyType, AsyncGeneratorType, CoroutineType, GeneratorType from tempfile import TemporaryDirectory, SpooledTemporaryFile from urllib.parse import SplitResult, ParseResult from unittest.case import _AssertRaisesContext @@ -57,6 +57,10 @@ from weakref import WeakSet, ReferenceType, ref import typing from typing import Unpack +try: + from tkinter import Event +except ImportError: + Event = None from typing import TypeVar T = TypeVar('T') @@ -96,7 +100,7 @@ class BaseTest(unittest.TestCase): """Test basics.""" - generic_types = [type, tuple, list, dict, set, frozenset, enumerate, + generic_types = [type, tuple, list, dict, set, frozenset, enumerate, memoryview, defaultdict, deque, SequenceMatcher, dircmp, @@ -120,6 +124,7 @@ class BaseTest(unittest.TestCase): KeysView, ItemsView, ValuesView, Sequence, MutableSequence, MappingProxyType, AsyncGeneratorType, + GeneratorType, CoroutineType, DirEntry, chain, LoggerAdapter, StreamHandler, @@ -138,6 +143,8 @@ class BaseTest(unittest.TestCase): if ValueProxy is not None: generic_types.extend((ValueProxy, DictProxy, ListProxy, ApplyResult, MPSimpleQueue, MPQueue, MPJoinableQueue)) + if Event is not None: + generic_types.append(Event) def test_subscriptable(self): for t in self.generic_types: diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index aece757fc1933e..e530b463966819 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -1,5 +1,6 @@ import unittest from test import support +from test.support.import_helper import import_module class TestMROEntry(unittest.TestCase): @@ -277,7 +278,9 @@ def __class_getitem__(cls, item): class CAPITest(unittest.TestCase): def test_c_class(self): - from _testcapi import Generic, GenericAlias + _testcapi = import_module("_testcapi") + Generic = _testcapi.Generic + GenericAlias = _testcapi.GenericAlias self.assertIsInstance(Generic.__class_getitem__(int), GenericAlias) IntGeneric = Generic[int] diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index f407ee3caf154c..6d2593cb4cf228 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -135,6 +135,9 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename), False) self.assertIs(self.pathmodule.exists(bfilename), False) + self.assertIs(self.pathmodule.lexists(filename), False) + self.assertIs(self.pathmodule.lexists(bfilename), False) + create_file(filename) self.assertIs(self.pathmodule.exists(filename), True) @@ -145,14 +148,17 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename + '\x00'), False) self.assertIs(self.pathmodule.exists(bfilename + b'\x00'), False) - if self.pathmodule is not genericpath: - self.assertIs(self.pathmodule.lexists(filename), True) - self.assertIs(self.pathmodule.lexists(bfilename), True) + self.assertIs(self.pathmodule.lexists(filename), True) + self.assertIs(self.pathmodule.lexists(bfilename), True) + + self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) + self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) - self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) - self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) + # Keyword arguments are accepted + self.assertIs(self.pathmodule.exists(path=filename), True) + self.assertIs(self.pathmodule.lexists(path=filename), True) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") @unittest.skipIf(is_emscripten, "Emscripten pipe fds have no stat") diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index 4f2d3cdcc7943e..7fb58a67368576 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -223,7 +223,7 @@ next(g) File "", line 1, in g = (10 // i for i in (5, 0, 2)) - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero >>> next(g) Traceback (most recent call last): File "", line 1, in -toplevel- diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 2f7aa69efc184a..d5dcdad9614ecc 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -557,7 +557,7 @@ def test_framework_macos(self): ns.add_known_dir("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload") ns.add_known_file("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/os.py") - # This is definitely not the stdlib (see discusion in bpo-46890) + # This is definitely not the stdlib (see discussion in bpo-46890) #ns.add_known_file("/Library/Frameworks/lib/python98.zip") expected = dict( @@ -605,7 +605,7 @@ def test_alt_framework_macos(self): ns.add_known_dir("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload") ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/os.py") - # This is definitely not the stdlib (see discusion in bpo-46890) + # This is definitely not the stdlib (see discussion in bpo-46890) #ns.add_known_xfile("/Library/lib/python98.zip") expected = dict( executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", @@ -844,6 +844,7 @@ def test_explicitly_set_stdlib_dir(self): PYDEBUGEXT="", VERSION_MAJOR=9, # fixed version number for ease VERSION_MINOR=8, # of testing + ABI_THREAD="", PYWINVER=None, EXE_SUFFIX=None, diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index aa5fac8eca1354..b72640bd871ba6 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -4,7 +4,9 @@ import shutil import sys import unittest +import warnings +from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -41,6 +43,11 @@ def setUp(self): os.symlink(self.norm('broken'), self.norm('sym1')) os.symlink('broken', self.norm('sym2')) os.symlink(os.path.join('a', 'bcd'), self.norm('sym3')) + self.open_dirfd() + + def open_dirfd(self): + if self.dir_fd is not None: + os.close(self.dir_fd) if {os.open, os.stat} <= os.supports_dir_fd and os.scandir in os.supports_fd: self.dir_fd = os.open(self.tempdir, os.O_RDONLY | os.O_DIRECTORY) else: @@ -333,6 +340,35 @@ def test_recursive_glob(self): eq(glob.glob('**', recursive=True, include_hidden=True), [join(*i) for i in full+rec]) + def test_glob_non_directory(self): + eq = self.assertSequencesEqual_noorder + eq(self.rglob('EF'), self.joins(('EF',))) + eq(self.rglob('EF', ''), []) + eq(self.rglob('EF', '*'), []) + eq(self.rglob('EF', '**'), []) + eq(self.rglob('nonexistent'), []) + eq(self.rglob('nonexistent', ''), []) + eq(self.rglob('nonexistent', '*'), []) + eq(self.rglob('nonexistent', '**'), []) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_glob_named_pipe(self): + path = os.path.join(self.tempdir, 'mypipe') + os.mkfifo(path) + + # gh-117127: Reopen self.dir_fd to pick up directory changes + self.open_dirfd() + + self.assertEqual(self.rglob('mypipe'), [path]) + self.assertEqual(self.rglob('mypipe*'), [path]) + self.assertEqual(self.rglob('mypipe', ''), []) + self.assertEqual(self.rglob('mypipe', 'sub'), []) + self.assertEqual(self.rglob('mypipe', '*'), []) + + + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_glob_many_open_files(self): depth = 30 base = os.path.join(self.tempdir, 'deep') @@ -350,6 +386,36 @@ def test_glob_many_open_files(self): for it in iters: self.assertEqual(next(it), p) + def test_glob0(self): + with self.assertWarns(DeprecationWarning): + glob.glob0(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob0(self.tempdir, 'a'), ['a']) + eq(glob.glob0(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob0(self.tempdir, '.b*'), []) + eq(glob.glob0(self.tempdir, 'b'), []) + eq(glob.glob0(self.tempdir, '?'), []) + eq(glob.glob0(self.tempdir, '*a'), []) + eq(glob.glob0(self.tempdir, 'a*'), []) + + def test_glob1(self): + with self.assertWarns(DeprecationWarning): + glob.glob1(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob1(self.tempdir, 'a'), ['a']) + eq(glob.glob1(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob1(self.tempdir, '.b*'), ['.bb']) + eq(glob.glob1(self.tempdir, 'b'), []) + eq(glob.glob1(self.tempdir, '?'), ['a']) + eq(glob.glob1(self.tempdir, '*a'), ['a', 'aaa']) + eq(glob.glob1(self.tempdir, 'a*'), ['a', 'aaa', 'aab']) + def test_translate_matching(self): match = re.compile(glob.translate('*')).match self.assertIsNotNone(match('foo')) @@ -429,9 +495,9 @@ def fn(pat): self.assertEqual(fn('?'), r'(?s:[^/])\Z') self.assertEqual(fn('**'), r'(?s:.*)\Z') self.assertEqual(fn('**/**'), r'(?s:.*)\Z') - self.assertRaises(ValueError, fn, '***') - self.assertRaises(ValueError, fn, 'a**') - self.assertRaises(ValueError, fn, '**b') + self.assertEqual(fn('***'), r'(?s:[^/]*)\Z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('**b'), r'(?s:[^/]*b)\Z') self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/[^/]*\.[^/]*/.*)\Z') def test_translate_seps(self): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 8501006b799262..6a841587f49166 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -3,6 +3,7 @@ from test.support import check_syntax_error from test.support import import_helper +import annotationlib import inspect import unittest import sys @@ -164,7 +165,7 @@ def test_floats(self): x = 3.14 x = 314. x = 0.314 - # XXX x = 000.314 + x = 000.314 x = .314 x = 3e14 x = 3E14 @@ -306,16 +307,6 @@ def test_eof_error(self): var_annot_global: int # a global annotated is necessary for test_var_annot -# custom namespace for testing __annotations__ - -class CNS: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - self._dct[item.lower()] = value - def __getitem__(self, item): - return self._dct[item] - class GrammarTests(unittest.TestCase): @@ -446,22 +437,12 @@ class F(C, A): self.assertEqual(E.__annotations__, {}) self.assertEqual(F.__annotations__, {}) - - def test_var_annot_metaclass_semantics(self): - class CMeta(type): - @classmethod - def __prepare__(metacls, name, bases, **kwds): - return {'__annotations__': CNS()} - class CC(metaclass=CMeta): - XX: 'ANNOT' - self.assertEqual(CC.__annotations__['xx'], 'ANNOT') - def test_var_annot_module_semantics(self): self.assertEqual(test.__annotations__, {}) self.assertEqual(ann_module.__annotations__, - {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) + {'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) self.assertEqual(ann_module.M.__annotations__, - {'123': 123, 'o': type}) + {'o': type}) self.assertEqual(ann_module2.__annotations__, {}) def test_var_annot_in_module(self): @@ -476,51 +457,12 @@ def test_var_annot_in_module(self): ann_module3.D_bad_ann(5) def test_var_annot_simple_exec(self): - gns = {}; lns= {} + gns = {}; lns = {} exec("'docstring'\n" - "__annotations__[1] = 2\n" "x: int = 5\n", gns, lns) - self.assertEqual(lns["__annotations__"], {1: 2, 'x': int}) - with self.assertRaises(KeyError): - gns['__annotations__'] - - def test_var_annot_custom_maps(self): - # tests with custom locals() and __annotations__ - ns = {'__annotations__': CNS()} - exec('X: int; Z: str = "Z"; (w): complex = 1j', ns) - self.assertEqual(ns['__annotations__']['x'], int) - self.assertEqual(ns['__annotations__']['z'], str) + self.assertEqual(lns["__annotate__"](annotationlib.Format.VALUE), {'x': int}) with self.assertRaises(KeyError): - ns['__annotations__']['w'] - nonloc_ns = {} - class CNS2: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('x: int = 1', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], int) - - def test_var_annot_refleak(self): - # complex case: custom locals plus custom __annotations__ - # this was causing refleak - cns = CNS() - nonloc_ns = {'__annotations__': cns} - class CNS2: - def __init__(self): - self._dct = {'__annotations__': cns} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('X: str', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], str) + gns['__annotate__'] def test_var_annot_rhs(self): ns = {} diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 128f933787a3f6..bf6e1703db8451 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -5,7 +5,6 @@ import functools import io import os -import pathlib import struct import sys import unittest @@ -79,16 +78,18 @@ def test_write(self): f.close() def test_write_read_with_pathlike_file(self): - filename = pathlib.Path(self.filename) + filename = os_helper.FakePath(self.filename) with gzip.GzipFile(filename, 'w') as f: f.write(data1 * 50) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) with gzip.GzipFile(filename, 'a') as f: f.write(data1) with gzip.GzipFile(filename) as f: d = f.read() self.assertEqual(d, data1 * 51) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) # The following test_write_xy methods test that write accepts # the corresponding bytes-like object type as input @@ -472,15 +473,122 @@ def test_textio_readlines(self): with io.TextIOWrapper(f, encoding="ascii") as t: self.assertEqual(t.readlines(), lines) + def test_fileobj_with_name(self): + with open(self.filename, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + def test_fileobj_from_fdopen(self): # Issue #13781: Opening a GzipFile for writing fails when using a # fileobj created with os.fdopen(). - fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT) - with os.fdopen(fd, "wb") as f: - with gzip.GzipFile(fileobj=f, mode="w") as g: - pass + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + with os.fdopen(fd, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with os.fdopen(fd, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with os.fdopen(fd, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with os.fdopen(fd, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) def test_fileobj_mode(self): + self.assertEqual(gzip.READ, 'rb') + self.assertEqual(gzip.WRITE, 'wb') gzip.GzipFile(self.filename, "wb").close() with open(self.filename, "r+b") as f: with gzip.GzipFile(fileobj=f, mode='r') as g: @@ -508,17 +616,69 @@ def test_fileobj_mode(self): def test_bytes_filename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with gzip.GzipFile(bytes_filename, "wb") as f: f.write(data1 * 50) + self.assertEqual(f.name, bytes_filename) with gzip.GzipFile(bytes_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with gzip.GzipFile(str_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, str_filename) + + def test_fileobj_without_name(self): + bio = io.BytesIO() + with gzip.GzipFile(fileobj=bio, mode='wb') as f: + f.write(data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + bio.seek(0) + with gzip.GzipFile(fileobj=bio, mode='rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + + def test_fileobj_and_filename(self): + filename2 = self.filename + 'new' + with (open(self.filename, 'wb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='wb') as f): + f.write(data1 * 50) + self.assertEqual(f.name, filename2) + with (open(self.filename, 'rb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='rb') as f): + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, filename2) + # Sanity check that we are actually operating on the right file. + with gzip.GzipFile(self.filename, 'rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, self.filename) def test_decompress_limited(self): """Decompressed data buffering should be limited""" @@ -553,8 +713,18 @@ def test_compress_mtime(self): f.read(1) # to set mtime attribute self.assertEqual(f.mtime, mtime) + def test_compress_mtime_default(self): + # test for gh-125260 + datac = gzip.compress(data1, mtime=0) + datac2 = gzip.compress(data1) + self.assertEqual(datac, datac2) + datac3 = gzip.compress(data1, mtime=None) + self.assertNotEqual(datac, datac3) + with gzip.GzipFile(fileobj=io.BytesIO(datac3), mode="rb") as f: + f.read(1) # to set mtime attribute + self.assertGreater(f.mtime, 1) + def test_compress_correct_level(self): - # gzip.compress calls with mtime == 0 take a different code path. for mtime in (0, 42): with self.subTest(mtime=mtime): nocompress = gzip.compress(data1, compresslevel=0, mtime=mtime) @@ -562,6 +732,17 @@ def test_compress_correct_level(self): self.assertIn(data1, nocompress) self.assertNotIn(data1, yescompress) + def test_issue112346(self): + # The OS byte should be 255, this should not change between Python versions. + for mtime in (0, 42): + with self.subTest(mtime=mtime): + compress = gzip.compress(data1, compresslevel=1, mtime=mtime) + self.assertEqual( + struct.unpack(" TreeMaker: diff --git a/Lib/test/test_importlib/resources/data01/__init__.py b/Lib/test/test_importlib/metadata/data/__init__.py similarity index 100% rename from Lib/test/test_importlib/resources/data01/__init__.py rename to Lib/test/test_importlib/metadata/data/__init__.py diff --git a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl similarity index 100% rename from Lib/test/test_importlib/data/example-21.12-py3-none-any.whl rename to Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl diff --git a/Lib/test/test_importlib/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg similarity index 100% rename from Lib/test/test_importlib/data/example-21.12-py3.6.egg rename to Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg diff --git a/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl similarity index 100% rename from Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl rename to Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl diff --git a/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py new file mode 100644 index 00000000000000..ba73b743394169 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py @@ -0,0 +1,2 @@ +def main(): + return 'example' diff --git a/Lib/test/test_importlib/metadata/data/sources/example/setup.py b/Lib/test/test_importlib/metadata/data/sources/example/setup.py new file mode 100644 index 00000000000000..479488a0348186 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name='example', + version='21.12', + license='Apache Software License', + packages=['example'], + entry_points={ + 'console_scripts': ['example = example:main', 'Example=example:main'], + }, +) diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py new file mode 100644 index 00000000000000..de645c2e8bc75b --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py @@ -0,0 +1,2 @@ +def main(): + return "example" diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml new file mode 100644 index 00000000000000..011f4751fb9e32 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'trampolim' +requires = ['trampolim'] + +[project] +name = 'example2' +version = '1.0.0' + +[project.scripts] +example = 'example2:main' diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py similarity index 81% rename from Lib/test/test_importlib/fixtures.py rename to Lib/test/test_importlib/metadata/fixtures.py index 8c973356b5660d..826b1b3259b4cd 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -1,15 +1,14 @@ -import os import sys import copy import json import shutil import pathlib -import tempfile import textwrap import functools import contextlib -from test.support.os_helper import FS_NONASCII +from test.support import import_helper +from test.support import os_helper from test.support import requires_zlib from . import _path @@ -26,29 +25,12 @@ @contextlib.contextmanager -def tempdir(): - tmpdir = tempfile.mkdtemp() - try: - yield pathlib.Path(tmpdir) - finally: - shutil.rmtree(tmpdir) - - -@contextlib.contextmanager -def save_cwd(): - orig = os.getcwd() - try: - yield - finally: - os.chdir(orig) - - -@contextlib.contextmanager -def tempdir_as_cwd(): - with tempdir() as tmp: - with save_cwd(): - os.chdir(str(tmp)) - yield tmp +def tmp_path(): + """ + Like os_helper.temp_dir, but yields a pathlib.Path. + """ + with os_helper.temp_dir() as path: + yield pathlib.Path(path) @contextlib.contextmanager @@ -69,7 +51,7 @@ def setUp(self): class SiteDir(Fixtures): def setUp(self): super().setUp() - self.site_dir = self.fixtures.enter_context(tempdir()) + self.site_dir = self.fixtures.enter_context(tmp_path()) class OnSysPath(Fixtures): @@ -85,6 +67,7 @@ def add_sys_path(dir): def setUp(self): super().setUp() self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + self.fixtures.enter_context(import_helper.isolated_modules()) class SiteBuilder(SiteDir): @@ -141,15 +124,13 @@ class DistInfoPkgEditable(DistInfoPkg): some_hash = '524127ce937f7cb65665130c695abd18ca386f60bb29687efb976faa1596fdcc' files: FilesSpec = { 'distinfo_pkg-1.0.0.dist-info': { - 'direct_url.json': json.dumps( - { - "archive_info": { - "hash": f"sha256={some_hash}", - "hashes": {"sha256": f"{some_hash}"}, - }, - "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", - } - ) + 'direct_url.json': json.dumps({ + "archive_info": { + "hash": f"sha256={some_hash}", + "hashes": {"sha256": f"{some_hash}"}, + }, + "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", + }) }, } @@ -253,6 +234,40 @@ def main(): } +class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module.json + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../../../etc/jupyter/jupyter_notebook_config.d/relative.json + /etc/jupyter/jupyter_notebook_config.d/absolute.json + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } + + class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): files: FilesSpec = { "egg_with_no_modules_pkg.egg-info": { @@ -338,7 +353,9 @@ def record_names(file_defs): class FileBuilder: def unicode_filename(self): - return FS_NONASCII or self.skip("File system does not support non-ascii.") + return os_helper.FS_NONASCII or self.skip( + "File system does not support non-ascii." + ) def DALS(str): @@ -348,7 +365,7 @@ def DALS(str): @requires_zlib() class ZipFixtures: - root = 'test.test_importlib.data' + root = 'test.test_importlib.metadata.data' def _fixture_on_path(self, filename): pkg_file = resources.files(self.root).joinpath(filename) diff --git a/Lib/test/test_importlib/stubs.py b/Lib/test/test_importlib/metadata/stubs.py similarity index 100% rename from Lib/test/test_importlib/stubs.py rename to Lib/test/test_importlib/metadata/stubs.py diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/metadata/test_api.py similarity index 98% rename from Lib/test/test_importlib/test_metadata_api.py rename to Lib/test/test_importlib/metadata/test_api.py index 33c6e85ee94753..2256e0c502e46f 100644 --- a/Lib/test/test_importlib/test_metadata_api.py +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -29,6 +29,7 @@ class APITests( fixtures.EggInfoPkg, fixtures.EggInfoPkgPipInstalledNoToplevel, fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgPipInstalledExternalDataFiles, fixtures.EggInfoPkgSourcesFallback, fixtures.DistInfoPkg, fixtures.DistInfoPkgWithDot, @@ -109,7 +110,7 @@ def test_entry_points_unique_packages_normalized(self): Entry points should only be exposed for the first package on sys.path with a given name (even when normalized). """ - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) alt_pkg = { "DistInfo_pkg-1.1.0.dist-info": { diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/metadata/test_main.py similarity index 95% rename from Lib/test/test_importlib/test_main.py rename to Lib/test/test_importlib/metadata/test_main.py index 0a769b89841234..e4218076f8cb0e 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -2,6 +2,7 @@ import pickle import unittest import warnings +import importlib import importlib.metadata import contextlib from test.support import os_helper @@ -137,7 +138,7 @@ def test_unique_distributions(self): fixtures.build_files(self.make_pkg('abc'), self.site_dir) before = list(_unique(distributions())) - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) fixtures.build_files(self.make_pkg('ABC'), alt_site_dir) after = list(_unique(distributions())) @@ -308,12 +309,10 @@ def test_sortable(self): """ EntryPoint objects are sortable, but result is undefined. """ - sorted( - [ - EntryPoint(name='b', value='val', group='group'), - EntryPoint(name='a', value='val', group='group'), - ] - ) + sorted([ + EntryPoint(name='b', value='val', group='group'), + EntryPoint(name='a', value='val', group='group'), + ]) class FileSystem( @@ -380,18 +379,16 @@ def test_packages_distributions_all_module_types(self): 'all_distributions-1.0.0.dist-info': metadata, } for i, suffix in enumerate(suffixes): - files.update( - { - f'importable-name {i}{suffix}': '', - f'in_namespace_{i}': { - f'mod{suffix}': '', - }, - f'in_package_{i}': { - '__init__.py': '', - f'mod{suffix}': '', - }, - } - ) + files.update({ + f'importable-name {i}{suffix}': '', + f'in_namespace_{i}': { + f'mod{suffix}': '', + }, + f'in_package_{i}': { + '__init__.py': '', + f'mod{suffix}': '', + }, + }) metadata.update(RECORD=fixtures.build_record(files)) fixtures.build_files(files, prefix=self.site_dir) diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py similarity index 100% rename from Lib/test/test_importlib/test_zip.py rename to Lib/test/test_importlib/metadata/test_zip.py diff --git a/Lib/test/test_importlib/resources/data01/binary.file b/Lib/test/test_importlib/resources/data01/binary.file deleted file mode 100644 index eaf36c1daccfdf..00000000000000 Binary files a/Lib/test/test_importlib/resources/data01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file b/Lib/test/test_importlib/resources/data01/subdirectory/binary.file deleted file mode 100644 index eaf36c1daccfdf..00000000000000 Binary files a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/utf-16.file b/Lib/test/test_importlib/resources/data01/utf-16.file deleted file mode 100644 index 2cb772295ef4b4..00000000000000 Binary files a/Lib/test/test_importlib/resources/data01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/utf-8.file b/Lib/test/test_importlib/resources/data01/utf-8.file deleted file mode 100644 index 1c0132ad90a192..00000000000000 --- a/Lib/test/test_importlib/resources/data01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/data02/one/__init__.py b/Lib/test/test_importlib/resources/data02/one/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data02/one/resource1.txt b/Lib/test/test_importlib/resources/data02/one/resource1.txt deleted file mode 100644 index 61a813e40174a6..00000000000000 --- a/Lib/test/test_importlib/resources/data02/one/resource1.txt +++ /dev/null @@ -1 +0,0 @@ -one resource diff --git a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt b/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt deleted file mode 100644 index 48f587a2d0ac53..00000000000000 --- a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt +++ /dev/null @@ -1 +0,0 @@ -a resource \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/data02/two/__init__.py b/Lib/test/test_importlib/resources/data02/two/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data02/two/resource2.txt b/Lib/test/test_importlib/resources/data02/two/resource2.txt deleted file mode 100644 index a80ce46ea362e2..00000000000000 --- a/Lib/test/test_importlib/resources/data02/two/resource2.txt +++ /dev/null @@ -1 +0,0 @@ -two resource diff --git a/Lib/test/test_importlib/resources/data03/__init__.py b/Lib/test/test_importlib/resources/data03/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/resource1.txt b/Lib/test/test_importlib/resources/data03/namespace/resource1.txt deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/namespacedata01/binary.file b/Lib/test/test_importlib/resources/namespacedata01/binary.file deleted file mode 100644 index eaf36c1daccfdf..00000000000000 Binary files a/Lib/test/test_importlib/resources/namespacedata01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-16.file b/Lib/test/test_importlib/resources/namespacedata01/utf-16.file deleted file mode 100644 index 2cb772295ef4b4..00000000000000 Binary files a/Lib/test/test_importlib/resources/namespacedata01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file b/Lib/test/test_importlib/resources/namespacedata01/utf-8.file deleted file mode 100644 index 1c0132ad90a192..00000000000000 --- a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index 1a13f043a86f03..4e4e0e9c337f23 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -19,25 +18,21 @@ def test_contents(self): assert self.expected <= contents -class ContentsDiskTests(ContentsTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ContentsDiskTests(ContentsTests, util.DiskSetup, unittest.TestCase): + pass class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): pass -class ContentsNamespaceTests(ContentsTests, unittest.TestCase): +class ContentsNamespaceTests(ContentsTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + expected = { # no __init__ because of namespace design - # no subdirectory as incidental difference in fixture 'binary.file', + 'subdirectory', 'utf-16.file', 'utf-8.file', } - - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py index 73127209a2761b..640f90fc0dd91a 100644 --- a/Lib/test/test_importlib/resources/test_custom.py +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -5,6 +5,7 @@ from test.support import os_helper from importlib import resources +from importlib.resources import abc from importlib.resources.abc import TraversableResources, ResourceReader from . import util @@ -39,8 +40,9 @@ def setUp(self): self.addCleanup(self.fixtures.close) def test_custom_loader(self): - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + temp_dir = pathlib.Path(self.fixtures.enter_context(os_helper.temp_dir())) loader = SimpleLoader(MagicResources(temp_dir)) pkg = util.create_package_from_loader(loader) files = resources.files(pkg) - assert files is temp_dir + assert isinstance(files, abc.Traversable) + assert list(files.iterdir()) == [] diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 1450cfb310926a..933894dce2c045 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,4 +1,7 @@ -import typing +import os +import pathlib +import py_compile +import shutil import textwrap import unittest import warnings @@ -7,11 +10,8 @@ from importlib import resources from importlib.resources.abc import Traversable -from . import data01 from . import util -from . import _path -from test.support import os_helper -from test.support import import_helper +from test.support import os_helper, import_helper @contextlib.contextmanager @@ -32,13 +32,14 @@ def test_read_text(self): actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') assert actual == 'Hello, UTF-8 world!\n' - @unittest.skipUnless( - hasattr(typing, 'runtime_checkable'), - "Only suitable when typing supports runtime_checkable", - ) def test_traversable(self): assert isinstance(resources.files(self.data), Traversable) + def test_joinpath_with_multiple_args(self): + files = resources.files(self.data) + binfile = files.joinpath('subdirectory', 'binary.file') + self.assertTrue(binfile.is_file()) + def test_old_parameter(self): """ Files used to take a 'package' parameter. Make sure anyone @@ -48,66 +49,125 @@ def test_old_parameter(self): resources.files(package=self.data) -class OpenDiskTests(FilesTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): + pass class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass -class OpenNamespaceTests(FilesTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 +class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + - self.data = namespacedata01 +class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' -class SiteDir: - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - self.site_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir)) - self.fixtures.enter_context(import_helper.CleanImport()) +class DirectSpec: + """ + Override behavior of ModuleSetup to write a full spec directly. + """ + MODULE = 'unused' + + def load_fixture(self, name): + self.tree_on_path(self.spec) + + +class ModulesFiles: + spec = { + 'mod.py': '', + 'res.txt': 'resources are the best', + } -class ModulesFilesTests(SiteDir, unittest.TestCase): def test_module_resources(self): """ A module can have resources found adjacent to the module. """ - spec = { - 'mod.py': '', - 'res.txt': 'resources are the best', - } - _path.build(spec, self.site_dir) import mod actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') - assert actual == spec['res.txt'] + assert actual == self.spec['res.txt'] + +class ModuleFilesDiskTests(DirectSpec, util.DiskSetup, ModulesFiles, unittest.TestCase): + pass + + +class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.TestCase): + pass + + +class ImplicitContextFiles: + set_val = textwrap.dedent( + f""" + import {resources.__name__} as res + val = res.files().joinpath('res.txt').read_text(encoding='utf-8') + """ + ) + spec = { + 'somepkg': { + '__init__.py': set_val, + 'submod.py': set_val, + 'res.txt': 'resources are the best', + }, + 'frozenpkg': { + '__init__.py': set_val.replace(resources.__name__, 'c_resources'), + 'res.txt': 'resources are the best', + }, + } -class ImplicitContextFilesTests(SiteDir, unittest.TestCase): - def test_implicit_files(self): + def test_implicit_files_package(self): """ Without any parameter, files() will infer the location as the caller. """ - spec = { - 'somepkg': { - '__init__.py': textwrap.dedent( - """ - import importlib.resources as res - val = res.files().joinpath('res.txt').read_text(encoding='utf-8') - """ - ), - 'res.txt': 'resources are the best', - }, - } - _path.build(spec, self.site_dir) assert importlib.import_module('somepkg').val == 'resources are the best' + def test_implicit_files_submodule(self): + """ + Without any parameter, files() will infer the location as the caller. + """ + assert importlib.import_module('somepkg.submod').val == 'resources are the best' + + def _compile_importlib(self): + """ + Make a compiled-only copy of the importlib resources package. + + Currently only code is copied, as importlib resources doesn't itself + have any resources. + """ + bin_site = self.fixtures.enter_context(os_helper.temp_dir()) + c_resources = pathlib.Path(bin_site, 'c_resources') + sources = pathlib.Path(resources.__file__).parent + + for source_path in sources.glob('**/*.py'): + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix('.pyc') + py_compile.compile(source_path, c_path) + self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) + + def test_implicit_files_with_compiled_importlib(self): + """ + Caller detection works for compiled-only resources module. + + python/cpython#123085 + """ + self._compile_importlib() + assert importlib.import_module('frozenpkg').val == 'resources are the best' + + +class ImplicitContextFilesDiskTests( + DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase +): + pass + + +class ImplicitContextFilesZipTests( + DirectSpec, util.ZipSetup, ImplicitContextFiles, unittest.TestCase +): + pass + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py new file mode 100644 index 00000000000000..4317abf3162c52 --- /dev/null +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -0,0 +1,255 @@ +import unittest +import os +import importlib + +from test.support import warnings_helper + +from importlib import resources + +from . import util + +# Since the functional API forwards to Traversable, we only test +# filesystem resources here -- not zip files, namespace packages etc. +# We do test for two kinds of Anchor, though. + + +class StringAnchorMixin: + anchor01 = 'data01' + anchor02 = 'data02' + + +class ModuleAnchorMixin: + @property + def anchor01(self): + return importlib.import_module('data01') + + @property + def anchor02(self): + return importlib.import_module('data02') + + +class FunctionalAPIBase(util.DiskSetup): + def setUp(self): + super().setUp() + self.load_fixture('data02') + + def _gen_resourcetxt_path_parts(self): + """Yield various names of a text file in anchor02, each in a subTest""" + for path_parts in ( + ('subdirectory', 'subsubdir', 'resource.txt'), + ('subdirectory/subsubdir/resource.txt',), + ('subdirectory/subsubdir', 'resource.txt'), + ): + with self.subTest(path_parts=path_parts): + yield path_parts + + def assertEndsWith(self, string, suffix): + """Assert that `string` ends with `suffix`. + + Used to ignore an architecture-specific UTF-16 byte-order mark.""" + self.assertEqual(string[-len(suffix) :], suffix) + + def test_read_text(self): + self.assertEqual( + resources.read_text(self.anchor01, 'utf-8.file'), + 'Hello, UTF-8 world!\n', + ) + self.assertEqual( + resources.read_text( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + encoding='utf-8', + ), + 'a resource', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ), + 'a resource', + ) + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.read_text(self.anchor01) + with self.assertRaises(OSError): + resources.read_text(self.anchor01, 'no-such-file') + with self.assertRaises(UnicodeDecodeError): + resources.read_text(self.anchor01, 'utf-16.file') + self.assertEqual( + resources.read_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ), + '\x00\x01\x02\x03', + ) + self.assertEndsWith( # ignore the BOM + resources.read_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_read_binary(self): + self.assertEqual( + resources.read_binary(self.anchor01, 'utf-8.file'), + b'Hello, UTF-8 world!\n', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_binary(self.anchor02, *path_parts), + b'a resource', + ) + + def test_open_text(self): + with resources.open_text(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ) as f: + self.assertEqual(f.read(), 'a resource') + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.open_text(self.anchor01) + with self.assertRaises(OSError): + resources.open_text(self.anchor01, 'no-such-file') + with resources.open_text(self.anchor01, 'utf-16.file') as f: + with self.assertRaises(UnicodeDecodeError): + f.read() + with resources.open_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ) as f: + self.assertEqual(f.read(), '\x00\x01\x02\x03') + with resources.open_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ) as f: + self.assertEndsWith( # ignore the BOM + f.read(), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_open_binary(self): + with resources.open_binary(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_binary( + self.anchor02, + *path_parts, + ) as f: + self.assertEqual(f.read(), b'a resource') + + def test_path(self): + with resources.path(self.anchor01, 'utf-8.file') as path: + with open(str(path), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + with resources.path(self.anchor01) as path: + with open(os.path.join(path, 'utf-8.file'), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + + def test_is_resource(self): + is_resource = resources.is_resource + self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) + self.assertFalse(is_resource(self.anchor01, 'no_such_file')) + self.assertFalse(is_resource(self.anchor01)) + self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertTrue(is_resource(self.anchor02, *path_parts)) + + def test_contents(self): + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01) + self.assertGreaterEqual( + set(c), + {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, + ) + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, 'utf-8.file')) + + for path_parts in self._gen_resourcetxt_path_parts(): + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, *path_parts)) + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01, 'subdirectory') + self.assertGreaterEqual( + set(c), + {'binary.file'}, + ) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_common_errors(self): + for func in ( + resources.read_text, + resources.read_binary, + resources.open_text, + resources.open_binary, + resources.path, + resources.is_resource, + resources.contents, + ): + with self.subTest(func=func): + # Rejecting None anchor + with self.assertRaises(TypeError): + func(None) + # Rejecting invalid anchor type + with self.assertRaises((TypeError, AttributeError)): + func(1234) + # Unknown module + with self.assertRaises(ModuleNotFoundError): + func('$missing module$') + + def test_text_errors(self): + for func in ( + resources.read_text, + resources.open_text, + ): + with self.subTest(func=func): + # Multiple path arguments need explicit encoding argument. + with self.assertRaises(TypeError): + func( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + ) + + +class FunctionalAPITest_StringAnchor( + StringAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass + + +class FunctionalAPITest_ModuleAnchor( + ModuleAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 86becb4bfaad37..8c00378ad3cc9c 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -24,7 +23,7 @@ def test_open_binary(self): target = resources.files(self.data) / 'binary.file' with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, b'\x00\x01\x02\x03') + self.assertEqual(result, bytes(range(4))) def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' @@ -65,21 +64,21 @@ def test_open_text_FileNotFoundError(self): target.open(encoding='utf-8') -class OpenDiskTests(OpenTests, unittest.TestCase): - def setUp(self): - self.data = data01 - +class OpenDiskTests(OpenTests, util.DiskSetup, unittest.TestCase): + pass -class OpenDiskNamespaceTests(OpenTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class OpenDiskNamespaceTests(OpenTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass +class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 34a6bdd2d58b91..378dc7a2baeb23 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -1,8 +1,8 @@ import io +import pathlib import unittest from importlib import resources -from . import data01 from . import util @@ -15,23 +15,16 @@ def execute(self, package, path): class PathTests: def test_reading(self): """ - Path should be readable. - - Test also implicitly verifies the returned object is a pathlib.Path - instance. + Path should be readable and a pathlib.Path instance. """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: + self.assertIsInstance(path, pathlib.Path) self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) - # pathlib.Path.read_text() was introduced in Python 3.5. - with path.open('r', encoding='utf-8') as file: - text = file.read() - self.assertEqual('Hello, UTF-8 world!\n', text) - + self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) -class PathDiskTests(PathTests, unittest.TestCase): - data = data01 +class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase): def test_natural_path(self): # Guarantee the internal implementation detail that # file-system-backed resources do not get the tempdir diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 088982681e8b0c..59c237d964121e 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -1,7 +1,7 @@ import unittest from importlib import import_module, resources -from . import data01 + from . import util @@ -18,7 +18,7 @@ def execute(self, package, path): class ReadTests: def test_read_bytes(self): result = resources.files(self.data).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4))) def test_read_text_default_encoding(self): result = ( @@ -51,30 +51,42 @@ def test_read_text_with_errors(self): ) -class ReadDiskTests(ReadTests, unittest.TestCase): - data = data01 +class ReadDiskTests(ReadTests, util.DiskSetup, unittest.TestCase): + pass class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) def test_read_submodule_resource_by_name(self): result = ( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .read_bytes() + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) + +class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' -class ReadNamespaceTests(ReadTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def test_read_submodule_resource(self): + submodule = import_module('namespacedata01.subdirectory') + result = resources.files(submodule).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(12, 16))) + + def test_read_submodule_resource_by_name(self): + result = ( + resources.files('namespacedata01.subdirectory') + .joinpath('binary.file') + .read_bytes() + ) + self.assertEqual(result, bytes(range(12, 16))) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 8670f72a334585..ed5693ab416798 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -1,17 +1,21 @@ import os.path -import sys import pathlib import unittest from importlib import import_module from importlib.readers import MultiplexedPath, NamespaceReader +from . import util -class MultiplexedPathTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - path = pathlib.Path(__file__).parent / 'namespacedata01' - cls.folder = str(path) + +class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def setUp(self): + super().setUp() + self.folder = pathlib.Path(self.data.__path__[0]) + self.data01 = pathlib.Path(self.load_fixture('data01').__file__).parent + self.data02 = pathlib.Path(self.load_fixture('data02').__file__).parent def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -19,7 +23,7 @@ def test_init_no_paths(self): def test_init_file(self): with self.assertRaises(NotADirectoryError): - MultiplexedPath(os.path.join(self.folder, 'binary.file')) + MultiplexedPath(self.folder / 'binary.file') def test_iterdir(self): contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} @@ -27,12 +31,13 @@ def test_iterdir(self): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} + ) def test_iterdir_duplicate(self): - data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) contents = { - path.name for path in MultiplexedPath(self.folder, data01).iterdir() + path.name for path in MultiplexedPath(self.folder, self.data01).iterdir() } for remove in ('__pycache__', '__init__.pyc'): try: @@ -60,17 +65,16 @@ def test_open_file(self): path.open() def test_join_path(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - path = MultiplexedPath(self.folder, data01) + prefix = str(self.folder.parent) + path = MultiplexedPath(self.folder, self.data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'binary.file'), ) - self.assertEqual( - str(path.joinpath('subdirectory'))[len(prefix) + 1 :], - os.path.join('data01', 'subdirectory'), - ) + sub = path.joinpath('subdirectory') + assert isinstance(sub, MultiplexedPath) + assert 'namespacedata01' in str(sub) + assert 'data01' in str(sub) self.assertEqual( str(path.joinpath('imaginary'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'imaginary'), @@ -82,10 +86,8 @@ def test_join_path_compound(self): assert not path.joinpath('imaginary/foo.py').exists() def test_join_path_common_subdir(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - data02 = os.path.join(prefix, 'data02') - path = MultiplexedPath(data01, data02) + prefix = str(self.data02.parent) + path = MultiplexedPath(self.data01, self.data02) self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) self.assertEqual( str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], @@ -105,16 +107,8 @@ def test_name(self): ) -class NamespaceReaderTest(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) +class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' def test_init_error(self): with self.assertRaises(ValueError): @@ -124,7 +118,7 @@ def test_resource_path(self): namespacedata01 = import_module('namespacedata01') reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + root = self.data.__path__[0] self.assertEqual( reader.resource_path('binary.file'), os.path.join(root, 'binary.file') ) @@ -133,9 +127,8 @@ def test_resource_path(self): ) def test_files(self): - namespacedata01 = import_module('namespacedata01') - reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + reader = NamespaceReader(self.data.__spec__.submodule_search_locations) + root = self.data.__path__[0] self.assertIsInstance(reader.files(), MultiplexedPath) self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index 6f75cf57f03d02..fcede14b891a84 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,15 +1,7 @@ -import contextlib -import sys import unittest -import uuid -import pathlib -from . import data01 -from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper, os_helper -from test.support.os_helper import unlink class ResourceTests: @@ -29,9 +21,8 @@ def test_is_dir(self): self.assertTrue(target.is_dir()) -class ResourceDiskTests(ResourceTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ResourceDiskTests(ResourceTests, util.DiskSetup, unittest.TestCase): + pass class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): @@ -42,33 +33,39 @@ def names(traversable): return {item.name for item in traversable.iterdir()} -class ResourceLoaderTests(unittest.TestCase): +class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): def test_resource_contents(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) self.assertEqual(names(resources.files(package)), {'A', 'B', 'C'}) def test_is_file(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('B').is_file()) def test_is_dir(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('D').is_dir()) def test_resource_missing(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertFalse(resources.files(package).joinpath('Z').is_file()) -class ResourceCornerCaseTests(unittest.TestCase): +class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): def test_package_has_no_reader_fallback(self): """ Test odd ball packages which: @@ -77,7 +74,7 @@ def test_package_has_no_reader_fallback(self): # 3. Are not in a zip file """ module = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) # Give the module a dummy loader. module.__loader__ = object() @@ -88,43 +85,39 @@ def test_package_has_no_reader_fallback(self): self.assertFalse(resources.files(module).joinpath('A').is_file()) -class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore - +class ResourceFromZipsTest01(util.ZipSetup, unittest.TestCase): def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .is_file() + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - names(resources.files('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) def test_as_file_directory(self): - with resources.as_file(resources.files('ziptestdata')) as data: - assert data.name == 'ziptestdata' + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' assert data.is_dir() assert data.joinpath('subdirectory').is_dir() assert len(list(data.iterdir())) assert not data.parent.exists() -class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore +class ResourceFromZipsTest02(util.ZipSetup, unittest.TestCase): + MODULE = 'data02' def test_unrelated_contents(self): """ @@ -132,93 +125,48 @@ def test_unrelated_contents(self): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - names(resources.files('ziptestdata.one')), + names(resources.files('data02.one')), {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - names(resources.files('ziptestdata.two')), + names(resources.files('data02.two')), {'__init__.py', 'resource2.txt'}, ) -@contextlib.contextmanager -def zip_on_path(dir): - data_path = pathlib.Path(zipdata01.__file__) - source_zip_path = data_path.parent.joinpath('ziptestdata.zip') - zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' - zip_path.write_bytes(source_zip_path.read_bytes()) - sys.path.append(str(zip_path)) - import_module('ziptestdata') - - try: - yield - finally: - with contextlib.suppress(ValueError): - sys.path.remove(str(zip_path)) - - with contextlib.suppress(KeyError): - del sys.path_importer_cache[str(zip_path)] - del sys.modules['ziptestdata'] - - with contextlib.suppress(OSError): - unlink(zip_path) - - -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetup, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(zip_on_path(temp_dir)) - def test_iterdir_does_not_keep_open(self): - [item.name for item in resources.files('ziptestdata').iterdir()] + [item.name for item in resources.files('data01').iterdir()] def test_is_file_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').is_file() + resources.files('data01').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('not-present').is_file() + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - resources.as_file(resources.files('ziptestdata') / 'binary.file') + resources.as_file(resources.files('data01') / 'binary.file') def test_entered_path_does_not_keep_open(self): """ Mimic what certifi does on import to make its bundle available for the process duration. """ - resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').read_bytes() + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('utf-8.file').read_text( - encoding='utf-8' - ) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) - +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( resources.files(import_module('namespacedata01')) @@ -237,7 +185,9 @@ def test_submodule_contents(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) def test_submodule_contents_by_name(self): contents = names(resources.files('namespacedata01')) @@ -245,7 +195,41 @@ def test_submodule_contents_by_name(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) + + def test_submodule_sub_contents(self): + contents = names(resources.files(import_module('namespacedata01.subdirectory'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + def test_submodule_sub_contents_by_name(self): + contents = names(resources.files('namespacedata01.subdirectory')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + +class ResourceFromNamespaceDiskTests( + util.DiskSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' + + +class ResourceFromNamespaceZipTests( + util.ZipSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/update-zips.py b/Lib/test/test_importlib/resources/update-zips.py deleted file mode 100755 index 231334aa7e38b4..00000000000000 --- a/Lib/test/test_importlib/resources/update-zips.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Generate the zip test data files. - -Run to build the tests/zipdataNN/ziptestdata.zip files from -files in tests/dataNN. - -Replaces the file with the working copy, but does commit anything -to the source repo. -""" - -import contextlib -import os -import pathlib -import zipfile - - -def main(): - """ - >>> from unittest import mock - >>> monkeypatch = getfixture('monkeypatch') - >>> monkeypatch.setattr(zipfile, 'ZipFile', mock.MagicMock()) - >>> print(); main() # print workaround for bpo-32509 - - ...data01... -> ziptestdata/... - ... - ...data02... -> ziptestdata/... - ... - """ - suffixes = '01', '02' - tuple(map(generate, suffixes)) - - -def generate(suffix): - root = pathlib.Path(__file__).parent.relative_to(os.getcwd()) - zfpath = root / f'zipdata{suffix}/ziptestdata.zip' - with zipfile.ZipFile(zfpath, 'w') as zf: - for src, rel in walk(root / f'data{suffix}'): - dst = 'ziptestdata' / pathlib.PurePosixPath(rel.as_posix()) - print(src, '->', dst) - zf.write(src, dst) - - -def walk(datapath): - for dirpath, dirnames, filenames in os.walk(datapath): - with contextlib.suppress(ValueError): - dirnames.remove('__pycache__') - for filename in filenames: - res = pathlib.Path(dirpath) / filename - rel = res.relative_to(datapath) - yield res, rel - - -__name__ == '__main__' and main() diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index dbe6ee81476699..e2d995f596317d 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -4,11 +4,12 @@ import sys import types import pathlib +import contextlib -from . import data01 -from . import zipdata01 from importlib.resources.abc import ResourceReader -from test.support import import_helper +from test.support import import_helper, os_helper +from . import zip as zip_ +from . import _path from importlib.machinery import ModuleSpec @@ -67,7 +68,7 @@ def create_package(file=None, path=None, is_package=True, contents=()): ) -class CommonTests(metaclass=abc.ABCMeta): +class CommonTestsBase(metaclass=abc.ABCMeta): """ Tests shared by test_open, test_path, and test_read. """ @@ -83,34 +84,34 @@ def test_package_name(self): """ Passing in the package name should succeed. """ - self.execute(data01.__name__, 'utf-8.file') + self.execute(self.data.__name__, 'utf-8.file') def test_package_object(self): """ Passing in the package itself should succeed. """ - self.execute(data01, 'utf-8.file') + self.execute(self.data, 'utf-8.file') def test_string_path(self): """ Passing in a string for the path should succeed. """ path = 'utf-8.file' - self.execute(data01, path) + self.execute(self.data, path) def test_pathlib_path(self): """ Passing in a pathlib.PurePath object for the path should succeed. """ path = pathlib.PurePath('utf-8.file') - self.execute(data01, path) + self.execute(self.data, path) def test_importing_module_as_side_effect(self): """ The anchor package can already be imported. """ - del sys.modules[data01.__name__] - self.execute(data01.__name__, 'utf-8.file') + del sys.modules[self.data.__name__] + self.execute(self.data.__name__, 'utf-8.file') def test_missing_path(self): """ @@ -140,40 +141,66 @@ def test_useless_loader(self): self.execute(package, 'utf-8.file') -class ZipSetupBase: - ZIP_MODULE = None - - @classmethod - def setUpClass(cls): - data_path = pathlib.Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass - +fixtures = dict( + data01={ + '__init__.py': '', + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + '__init__.py': '', + 'binary.file': bytes(range(4, 8)), + }, + }, + data02={ + '__init__.py': '', + 'one': {'__init__.py': '', 'resource1.txt': 'one resource'}, + 'two': {'__init__.py': '', 'resource2.txt': 'two resource'}, + 'subdirectory': {'subsubdir': {'resource.txt': 'a resource'}}, + }, + namespacedata01={ + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + 'binary.file': bytes(range(12, 16)), + }, + }, +) + + +class ModuleSetup: def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + self.fixtures.enter_context(import_helper.isolated_modules()) + self.data = self.load_fixture(self.MODULE) + + def load_fixture(self, module): + self.tree_on_path({module: fixtures[module]}) + return importlib.import_module(module) + + +class ZipSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(spec, modules))) + ) + + +class DiskSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + _path.build(spec, pathlib.Path(temp_dir)) + self.fixtures.enter_context(import_helper.DirsOnSysPath(temp_dir)) -class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore +class CommonTests(DiskSetup, CommonTestsBase): + pass diff --git a/Lib/test/test_importlib/resources/zip.py b/Lib/test/test_importlib/resources/zip.py new file mode 100755 index 00000000000000..fc453f02060a31 --- /dev/null +++ b/Lib/test/test_importlib/resources/zip.py @@ -0,0 +1,24 @@ +""" +Generate zip test data files. +""" + +import zipfile + + +def make_zip_file(tree, dst): + """ + Zip the files in tree into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for name, contents in walk(tree): + zf.writestr(name, contents) + zipfile._path.CompleteDirs.inject(zf) + return dst + + +def walk(tree, prefix=''): + for name, contents in tree.items(): + if isinstance(contents, dict): + yield from walk(contents, prefix=f'{prefix}{name}/') + else: + yield f'{prefix}{name}', contents diff --git a/Lib/test/test_importlib/resources/zipdata01/__init__.py b/Lib/test/test_importlib/resources/zipdata01/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb0739f87e9..00000000000000 Binary files a/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/resources/zipdata02/__init__.py b/Lib/test/test_importlib/resources/zipdata02/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512d2807e..00000000000000 Binary files a/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 2a35f3dcb7210c..51ea5270b1a928 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -6,8 +6,10 @@ import os.path import sys +from test import support from test.support import import_helper from test.support import os_helper +import traceback import types import unittest @@ -353,6 +355,20 @@ def test_module_missing_spec(self): with self.assertRaises(ModuleNotFoundError): self.init.reload(module) + def test_reload_traceback_with_non_str(self): + # gh-125519 + with support.captured_stdout() as stdout: + try: + self.init.reload("typing") + except TypeError as exc: + traceback.print_exception(exc, file=stdout) + else: + self.fail("Expected TypeError to be raised") + printed_traceback = stdout.getvalue() + self.assertIn("TypeError", printed_traceback) + self.assertNotIn("AttributeError", printed_traceback) + self.assertNotIn("module.__spec__.name", printed_traceback) + (Frozen_ReloadTests, Source_ReloadTests @@ -437,5 +453,44 @@ def test_everyone_has___spec__(self): ) = test_util.test_both(StartupTests, machinery=machinery) +class TestModuleAll(unittest.TestCase): + def test_machinery(self): + extra = ( + # from importlib._bootstrap and importlib._bootstrap_external + 'AppleFrameworkLoader', + 'BYTECODE_SUFFIXES', + 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', + 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', + 'FileFinder', + 'FrozenImporter', + 'ModuleSpec', + 'NamespaceLoader', + 'OPTIMIZED_BYTECODE_SUFFIXES', + 'PathFinder', + 'SOURCE_SUFFIXES', + 'SourceFileLoader', + 'SourcelessFileLoader', + 'WindowsRegistryFinder', + ) + support.check__all__(self, machinery['Source'], extra=extra) + + def test_util(self): + extra = ( + # from importlib.abc, importlib._bootstrap + # and importlib._bootstrap_external + 'Loader', + 'MAGIC_NUMBER', + 'cache_from_source', + 'decode_source', + 'module_from_spec', + 'source_from_cache', + 'spec_from_file_location', + 'spec_from_loader', + ) + support.check__all__(self, util['Source'], extra=extra) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index cc993f333e355a..5c6e0303528906 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -2,9 +2,12 @@ from importlib import abc from importlib import util import sys +import time +import threading import types import unittest +from test.support import threading_helper from test.test_importlib import util as test_util @@ -40,6 +43,7 @@ class TestingImporter(abc.MetaPathFinder, abc.Loader): module_name = 'lazy_loader_test' mutated_name = 'changed' loaded = None + load_count = 0 source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name) def find_spec(self, name, path, target=None): @@ -48,8 +52,10 @@ def find_spec(self, name, path, target=None): return util.spec_from_loader(name, util.LazyLoader(self)) def exec_module(self, module): + time.sleep(0.01) # Simulate a slow load. exec(self.source_code, module.__dict__) self.loaded = module + self.load_count += 1 class LazyLoaderTests(unittest.TestCase): @@ -59,8 +65,9 @@ def test_init(self): # Classes that don't define exec_module() trigger TypeError. util.LazyLoader(object) - def new_module(self, source_code=None): - loader = TestingImporter() + def new_module(self, source_code=None, loader=None): + if loader is None: + loader = TestingImporter() if source_code is not None: loader.source_code = source_code spec = util.spec_from_loader(TestingImporter.module_name, @@ -140,6 +147,83 @@ def test_module_already_in_sys(self): # Force the load; just care that no exception is raised. module.__name__ + @threading_helper.requires_working_threading() + def test_module_load_race(self): + with test_util.uncache(TestingImporter.module_name): + loader = TestingImporter() + module = self.new_module(loader=loader) + self.assertEqual(loader.load_count, 0) + + class RaisingThread(threading.Thread): + exc = None + def run(self): + try: + super().run() + except Exception as exc: + self.exc = exc + + def access_module(): + return module.attr + + threads = [] + for _ in range(2): + threads.append(thread := RaisingThread(target=access_module)) + thread.start() + + # Races could cause errors + for thread in threads: + thread.join() + self.assertIsNone(thread.exc) + + # Or multiple load attempts + self.assertEqual(loader.load_count, 1) + + def test_lazy_self_referential_modules(self): + # Directory modules with submodules that reference the parent can attempt to access + # the parent module during a load. Verify that this common pattern works with lazy loading. + # json is a good example in the stdlib. + json_modules = [name for name in sys.modules if name.startswith('json')] + with test_util.uncache(*json_modules): + # Standard lazy loading, unwrapped + spec = util.find_spec('json') + loader = util.LazyLoader(spec.loader) + spec.loader = loader + module = util.module_from_spec(spec) + sys.modules['json'] = module + loader.exec_module(module) + + # Trigger load with attribute lookup, ensure expected behavior + test_load = module.loads('{}') + self.assertEqual(test_load, {}) + + def test_lazy_module_type_override(self): + # Verify that lazy loading works with a module that modifies + # its __class__ to be a custom type. + + # Example module from PEP 726 + module = self.new_module(source_code="""\ +import sys +from types import ModuleType + +CONSTANT = 3.14 + +class ImmutableModule(ModuleType): + def __setattr__(self, name, value): + raise AttributeError('Read-only attribute!') + + def __delattr__(self, name): + raise AttributeError('Read-only attribute!') + +sys.modules[__name__].__class__ = ImmutableModule +""") + sys.modules[TestingImporter.module_name] = module + self.assertIsInstance(module, util._LazyModule) + self.assertEqual(module.CONSTANT, 3.14) + with self.assertRaises(AttributeError): + module.CONSTANT = 2.71 + with self.assertRaises(AttributeError): + del module.CONSTANT + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 072e198795d394..cbbdada3b010a7 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -286,25 +286,24 @@ def test_project3_succeeds(self): class ZipWithMissingDirectory(NamespacePackageTest): paths = ['missing_directory.zip'] + # missing_directory.zip contains: + # Length Date Time Name + # --------- ---------- ----- ---- + # 29 2012-05-03 18:13 foo/one.py + # 0 2012-05-03 20:57 bar/ + # 38 2012-05-03 20:57 bar/two.py + # --------- ------- + # 67 3 files - @unittest.expectedFailure def test_missing_directory(self): - # This will fail because missing_directory.zip contains: - # Length Date Time Name - # --------- ---------- ----- ---- - # 29 2012-05-03 18:13 foo/one.py - # 0 2012-05-03 20:57 bar/ - # 38 2012-05-03 20:57 bar/two.py - # --------- ------- - # 67 3 files - - # Because there is no 'foo/', the zipimporter currently doesn't - # know that foo is a namespace package - import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + + def test_missing_directory2(self): + import foo + self.assertFalse(hasattr(foo, 'one')) def test_present_directory(self): - # This succeeds because there is a "bar/" in the zip file import bar.two self.assertEqual(bar.two.attr, 'missing_directory foo two') diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py index 80aa3609c6f96e..02318926f35eee 100644 --- a/Lib/test/test_importlib/test_spec.py +++ b/Lib/test/test_importlib/test_spec.py @@ -502,7 +502,8 @@ def test_spec_from_loader_is_package_true_with_fileloader(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -601,7 +602,8 @@ def test_spec_from_file_location_smsl_empty(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -626,7 +628,8 @@ def test_spec_from_file_location_smsl_default(self): self.assertEqual(spec.loader, self.pkgloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 5072be86cfd112..9af1e4d505c66e 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -13,6 +13,7 @@ import shutil import threading import unittest +from test import support from test.support import verbose from test.support.import_helper import forget, mock_register_at_fork from test.support.os_helper import (TESTFN, unlink, rmtree) @@ -260,7 +261,7 @@ def setUpModule(): try: old_switchinterval = sys.getswitchinterval() unittest.addModuleCleanup(sys.setswitchinterval, old_switchinterval) - sys.setswitchinterval(1e-5) + support.setswitchinterval(1e-5) except AttributeError: pass diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index fe5e7b31d9c32b..668042782bdc5f 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -27,7 +27,7 @@ except ImportError: _testmultiphase = None try: - import _xxsubinterpreters as _interpreters + import _interpreters except ModuleNotFoundError: _interpreters = None @@ -577,7 +577,7 @@ def test_cache_from_source_respects_pycache_prefix_relative(self): with util.temporary_pycache_prefix(pycache_prefix): self.assertEqual( self.util.cache_from_source(path, optimization=''), - expect) + os.path.normpath(expect)) @unittest.skipIf(sys.implementation.cache_tag is None, 'requires sys.implementation.cache_tag to not be None') @@ -656,7 +656,7 @@ def test_magic_number(self): class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): - interpid = _interpreters.create(isolated=True) + interpid = _interpreters.create('isolated') def ensure_destroyed(): try: _interpreters.destroy(interpid) @@ -669,7 +669,7 @@ def ensure_destroyed(): raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) @@ -682,6 +682,9 @@ def ensure_destroyed(): raise ImportError(excsnap.msg) @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + # gh-117649: single-phase init modules are not currently supported in + # subinterpreters in the free-threaded build + @support.expected_failure_if_gil_disabled() def test_single_phase_init_module(self): script = textwrap.dedent(''' from importlib.util import _incompatible_extension_module_restrictions @@ -706,14 +709,22 @@ def test_single_phase_init_module(self): self.run_with_own_gil(script) @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + @support.requires_gil_enabled("gh-117649: not supported in free-threaded build") def test_incomplete_multi_phase_init_module(self): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + prescript = textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} name = '_test_shared_gil_only' filename = {_testmultiphase.__file__!r} - loader = ExtensionFileLoader(name, filename) + loader = {loader}(name, filename) spec = spec_from_loader(name, loader) ''') diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index c25be096e52874..edbe78545a2536 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -6,13 +6,17 @@ import marshal import os import os.path +from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper import unittest import sys import tempfile import types +_testsinglephase = import_helper.import_module("_testsinglephase") + BUILTINS = types.SimpleNamespace() BUILTINS.good_name = None @@ -22,25 +26,39 @@ if 'importlib' not in sys.builtin_module_names: BUILTINS.bad_name = 'importlib' -EXTENSIONS = types.SimpleNamespace() -EXTENSIONS.path = None -EXTENSIONS.ext = None -EXTENSIONS.filename = None -EXTENSIONS.file_path = None -EXTENSIONS.name = '_testsinglephase' - -def _extension_details(): - global EXTENSIONS - for path in sys.path: - for ext in machinery.EXTENSION_SUFFIXES: - filename = EXTENSIONS.name + ext - file_path = os.path.join(path, filename) - if os.path.exists(file_path): - EXTENSIONS.path = path - EXTENSIONS.ext = ext - EXTENSIONS.filename = filename - EXTENSIONS.file_path = file_path - return +if support.is_wasi: + # dlopen() is a shim for WASI as of WASI SDK which fails by default. + # We don't provide an implementation, so tests will fail. + # But we also don't want to turn off dynamic loading for those that provide + # a working implementation. + def _extension_details(): + global EXTENSIONS + EXTENSIONS = None +else: + EXTENSIONS = types.SimpleNamespace() + EXTENSIONS.path = None + EXTENSIONS.ext = None + EXTENSIONS.filename = None + EXTENSIONS.file_path = None + EXTENSIONS.name = '_testsinglephase' + + def _extension_details(): + global EXTENSIONS + for path in sys.path: + for ext in machinery.EXTENSION_SUFFIXES: + # Apple mobile platforms mechanically load .so files, + # but the findable files are labelled .fwork + if is_apple_mobile: + ext = ext.replace(".so", ".fwork") + + filename = EXTENSIONS.name + ext + file_path = os.path.join(path, filename) + if os.path.exists(file_path): + EXTENSIONS.path = path + EXTENSIONS.ext = ext + EXTENSIONS.filename = filename + EXTENSIONS.file_path = file_path + return _extension_details() diff --git a/Lib/test/test_inspect/inspect_deferred_annotations.py b/Lib/test/test_inspect/inspect_deferred_annotations.py new file mode 100644 index 00000000000000..bb59ef1035b3c1 --- /dev/null +++ b/Lib/test/test_inspect/inspect_deferred_annotations.py @@ -0,0 +1,2 @@ +def f(x: undefined): + pass diff --git a/Lib/test/test_inspect/inspect_fodder.py b/Lib/test/test_inspect/inspect_fodder.py index 60ba7aa78394e8..febd54c86fe1d1 100644 --- a/Lib/test/test_inspect/inspect_fodder.py +++ b/Lib/test/test_inspect/inspect_fodder.py @@ -68,9 +68,9 @@ class FesteringGob(MalodorousPervert, ParrotDroppings): def abuse(self, a, b, c): pass - @property - def contradiction(self): + def _getter(self): pass + contradiction = property(_getter) async def lobbest(grenade): pass diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 8639cf2e72cd7a..43fda6622537fc 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -310,3 +310,62 @@ def f(): class cls310: def g(): pass + +# line 314 +class ClassWithCodeObject: + import sys + code = sys._getframe(0).f_code + +import enum + +# line 321 +class enum322(enum.Enum): + A = 'a' + +# line 325 +class enum326(enum.IntEnum): + A = 1 + +# line 329 +class flag330(enum.Flag): + A = 1 + +# line 333 +class flag334(enum.IntFlag): + A = 1 + +# line 337 +simple_enum338 = enum.Enum('simple_enum338', 'A') +simple_enum339 = enum.IntEnum('simple_enum339', 'A') +simple_flag340 = enum.Flag('simple_flag340', 'A') +simple_flag341 = enum.IntFlag('simple_flag341', 'A') + +import typing + +# line 345 +class nt346(typing.NamedTuple): + x: int + y: int + +# line 350 +nt351 = typing.NamedTuple('nt351', (('x', int), ('y', int))) + +# line 353 +class td354(typing.TypedDict): + x: int + y: int + +# line 358 +td359 = typing.TypedDict('td359', (('x', int), ('y', int))) + +import dataclasses + +# line 363 +@dataclasses.dataclass +class dc364: + x: int + y: int + +# line 369 +dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) +dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) diff --git a/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py new file mode 100644 index 00000000000000..39bfe2edb03f30 --- /dev/null +++ b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py @@ -0,0 +1,87 @@ +from __future__ import annotations +from typing import Callable, Unpack + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: Unpack[Ts], z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +# Eggs is `int` in globals, a TypeVar in type_params, and `str` in locals: +class E[Eggs]: + Eggs = str + x: Eggs + + + +def nested(): + from types import SimpleNamespace + from inspect import get_annotations + + Eggs = bytes + Spam = memoryview + + + class F[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + # Eggs is `int` in globals, `bytes` in the function scope, + # a TypeVar in the type_params, and `str` in locals: + class G[Eggs]: + Eggs = str + x: Eggs + + + return SimpleNamespace( + F=F, + F_annotations=get_annotations(F, eval_str=True), + F_meth_annotations=get_annotations(F.generic_method, eval_str=True), + G_annotations=get_annotations(G, eval_str=True), + generic_func=generic_function, + generic_func_annotations=get_annotations(generic_function, eval_str=True) + ) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 4611f62b293ff9..a4430a868676e2 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1,9 +1,11 @@ +from annotationlib import Format, ForwardRef import asyncio import builtins import collections import copy import datetime import functools +import gc import importlib import inspect import io @@ -25,6 +27,7 @@ import unittest import unittest.mock import warnings +import weakref try: @@ -32,28 +35,26 @@ except ImportError: ThreadPoolExecutor = None -from test.support import cpython_only +from test.support import cpython_only, import_helper from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support.import_helper import DirsOnSysPath, ready_to_import -from test.support.os_helper import TESTFN +from test.support.os_helper import TESTFN, temp_cwd from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python -from test.support import has_subprocess_support, SuppressCrashReport +from test.support import has_subprocess_support from test import support -from . import inspect_fodder as mod -from . import inspect_fodder2 as mod2 -from . import inspect_stock_annotations -from . import inspect_stringized_annotations -from . import inspect_stringized_annotations_2 +from test.test_inspect import inspect_fodder as mod +from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_deferred_annotations # Functions tested in this suite: # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, -# isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, +# isbuiltin, isroutine, isgenerator, ispackage, isgeneratorfunction, getmembers, # getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, -# getclasstree, getargvalues, formatargvalues, -# currentframe, stack, trace, isdatadescriptor, -# ismethodwrapper +# getclasstree, getargvalues, formatargvalues, currentframe, +# stack, trace, ismethoddescriptor, isdatadescriptor, ismethodwrapper # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. @@ -104,7 +105,7 @@ def unsorted_keyword_only_parameters_fn(*, throw, out, the, baby, with_, class IsTestBase(unittest.TestCase): predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode, inspect.isframe, inspect.isfunction, inspect.ismethod, - inspect.ismodule, inspect.istraceback, + inspect.ismodule, inspect.istraceback, inspect.ispackage, inspect.isgenerator, inspect.isgeneratorfunction, inspect.iscoroutine, inspect.iscoroutinefunction, inspect.isasyncgen, inspect.isasyncgenfunction, @@ -120,10 +121,13 @@ def istest(self, predicate, exp): predicate == inspect.iscoroutinefunction) and \ other == inspect.isfunction: continue - self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp)) + if predicate == inspect.ispackage and other == inspect.ismodule: + self.assertTrue(predicate(obj), '%s(%s)' % (predicate.__name__, exp)) + else: + self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp)) def test__all__(self): - support.check__all__(self, inspect, not_exported=("modulesbyfile",)) + support.check__all__(self, inspect, not_exported=("modulesbyfile",), extra=("get_annotations",)) def generator_function_example(self): for i in range(2): @@ -175,6 +179,7 @@ def test_excluding_predicates(self): self.istest(inspect.ismethod, 'git.argue') self.istest(inspect.ismethod, 'mod.custom_method') self.istest(inspect.ismodule, 'mod') + self.istest(inspect.ismethoddescriptor, 'int.__add__') self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory') self.istest(inspect.isgenerator, '(x for x in range(2))') self.istest(inspect.isgeneratorfunction, 'generator_function_example') @@ -199,19 +204,51 @@ def test_excluding_predicates(self): self.assertFalse(inspect.ismethodwrapper(int)) self.assertFalse(inspect.ismethodwrapper(type("AnyClass", (), {}))) + def test_ispackage(self): + self.istest(inspect.ispackage, 'asyncio') + self.istest(inspect.ispackage, 'importlib') + self.assertFalse(inspect.ispackage(inspect)) + self.assertFalse(inspect.ispackage(mod)) + self.assertFalse(inspect.ispackage(':)')) + + class FakePackage: + __path__ = None + self.assertFalse(inspect.ispackage(FakePackage())) def test_iscoroutine(self): async_gen_coro = async_generator_function_example(1) gen_coro = gen_coroutine_function_example(1) coro = coroutine_function_example(1) + class PMClass: + async_generator_partialmethod_example = functools.partialmethod( + async_generator_function_example) + coroutine_partialmethod_example = functools.partialmethod( + coroutine_function_example) + gen_coroutine_partialmethod_example = functools.partialmethod( + gen_coroutine_function_example) + + # partialmethods on the class, bound to an instance + pm_instance = PMClass() + async_gen_coro_pmi = pm_instance.async_generator_partialmethod_example + gen_coro_pmi = pm_instance.gen_coroutine_partialmethod_example + coro_pmi = pm_instance.coroutine_partialmethod_example + + # partialmethods on the class, unbound but accessed via the class + async_gen_coro_pmc = PMClass.async_generator_partialmethod_example + gen_coro_pmc = PMClass.gen_coroutine_partialmethod_example + coro_pmc = PMClass.coroutine_partialmethod_example + self.assertFalse( inspect.iscoroutinefunction(gen_coroutine_function_example)) self.assertFalse( inspect.iscoroutinefunction( functools.partial(functools.partial( gen_coroutine_function_example)))) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmi)) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmc)) + self.assertFalse(inspect.iscoroutinefunction(inspect)) self.assertFalse(inspect.iscoroutine(gen_coro)) self.assertTrue( @@ -220,6 +257,8 @@ def test_iscoroutine(self): inspect.isgeneratorfunction( functools.partial(functools.partial( gen_coroutine_function_example)))) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmi)) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmc)) self.assertTrue(inspect.isgenerator(gen_coro)) async def _fn3(): @@ -285,6 +324,8 @@ def do_something_static(): inspect.iscoroutinefunction( functools.partial(functools.partial( coroutine_function_example)))) + self.assertTrue(inspect.iscoroutinefunction(coro_pmi)) + self.assertTrue(inspect.iscoroutinefunction(coro_pmc)) self.assertTrue(inspect.iscoroutine(coro)) self.assertFalse( @@ -297,6 +338,8 @@ def do_something_static(): inspect.isgeneratorfunction( functools.partial(functools.partial( coroutine_function_example)))) + self.assertFalse(inspect.isgeneratorfunction(coro_pmi)) + self.assertFalse(inspect.isgeneratorfunction(coro_pmc)) self.assertFalse(inspect.isgenerator(coro)) self.assertFalse( @@ -311,6 +354,8 @@ def do_something_static(): inspect.isasyncgenfunction( functools.partial(functools.partial( async_generator_function_example)))) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmi)) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmc)) self.assertTrue(inspect.isasyncgen(async_gen_coro)) coro.close(); gen_coro.close(); # silence warnings @@ -371,6 +416,8 @@ def test_isroutine(self): self.assertFalse(inspect.isroutine(type)) self.assertFalse(inspect.isroutine(int)) self.assertFalse(inspect.isroutine(type('some_class', (), {}))) + # partial + self.assertTrue(inspect.isroutine(functools.partial(mod.spam))) def test_isclass(self): self.istest(inspect.isclass, 'mod.StupidGit') @@ -639,7 +686,10 @@ def test_cleandoc(self): @cpython_only def test_c_cleandoc(self): - import _testinternalcapi + try: + import _testinternalcapi + except ImportError: + return unittest.skip("requires _testinternalcapi") func = _testinternalcapi.compiler_cleandoc for i, (input, expected) in enumerate(self.cleandoc_testdata): with self.subTest(i=i): @@ -701,6 +751,18 @@ def test_getsourcefile(self): finally: del linecache.cache[co.co_filename] + def test_getsource_empty_file(self): + with temp_cwd() as cwd: + with open('empty_file.py', 'w'): + pass + sys.path.insert(0, cwd) + try: + import empty_file + self.assertEqual(inspect.getsource(empty_file), '\n') + self.assertEqual(inspect.getsourcelines(empty_file), (['\n'], 0)) + finally: + sys.path.remove(cwd) + def test_getfile(self): self.assertEqual(inspect.getfile(mod.StupidGit), mod.__file__) @@ -771,6 +833,62 @@ def monkey(filename, module_globals=None): def test_getsource_on_code_object(self): self.assertSourceEqual(mod.eggs.__code__, 12, 18) + def test_getsource_on_generated_class(self): + A = type('A', (unittest.TestCase,), {}) + self.assertEqual(inspect.getsourcefile(A), __file__) + self.assertEqual(inspect.getfile(A), __file__) + self.assertIs(inspect.getmodule(A), sys.modules[__name__]) + self.assertRaises(OSError, inspect.getsource, A) + self.assertRaises(OSError, inspect.getsourcelines, A) + self.assertIsNone(inspect.getcomments(A)) + + def test_getsource_on_class_without_firstlineno(self): + __firstlineno__ = 1 + class C: + nonlocal __firstlineno__ + self.assertRaises(OSError, inspect.getsource, C) + +class TestGetsourceStdlib(unittest.TestCase): + # Test Python implementations of the stdlib modules + + def test_getsource_stdlib_collections_abc(self): + import collections.abc + lines, lineno = inspect.getsourcelines(collections.abc.Sequence) + self.assertEqual(lines[0], 'class Sequence(Reversible, Collection):\n') + src = inspect.getsource(collections.abc.Sequence) + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_tomllib(self): + import tomllib + self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) + self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) + + def test_getsource_stdlib_abc(self): + # Pure Python implementation + abc = import_helper.import_fresh_module('abc', blocked=['_abc']) + with support.swap_item(sys.modules, 'abc', abc): + self.assertRaises(OSError, inspect.getsource, abc.ABCMeta) + self.assertRaises(OSError, inspect.getsourcelines, abc.ABCMeta) + # With C acceleration + import abc + try: + src = inspect.getsource(abc.ABCMeta) + lines, lineno = inspect.getsourcelines(abc.ABCMeta) + except OSError: + pass + else: + self.assertEqual(lines[0], ' class ABCMeta(type):\n') + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_decimal(self): + # Pure Python implementation + decimal = import_helper.import_fresh_module('decimal', blocked=['_decimal']) + with support.swap_item(sys.modules, 'decimal', decimal): + src = inspect.getsource(decimal.Decimal) + lines, lineno = inspect.getsourcelines(decimal.Decimal) + self.assertEqual(lines[0], 'class Decimal(object):\n') + self.assertEqual(src.splitlines(True), lines) + class TestGetsourceInteractive(unittest.TestCase): def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; @@ -865,6 +983,29 @@ def test_anonymous(self): # as argument to another function. self.assertSourceEqual(mod2.anonymous, 55, 55) + def test_enum(self): + self.assertSourceEqual(mod2.enum322, 322, 323) + self.assertSourceEqual(mod2.enum326, 326, 327) + self.assertSourceEqual(mod2.flag330, 330, 331) + self.assertSourceEqual(mod2.flag334, 334, 335) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum338) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum339) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag340) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag341) + + def test_namedtuple(self): + self.assertSourceEqual(mod2.nt346, 346, 348) + self.assertRaises(OSError, inspect.getsource, mod2.nt351) + + def test_typeddict(self): + self.assertSourceEqual(mod2.td354, 354, 356) + self.assertRaises(OSError, inspect.getsource, mod2.td359) + + def test_dataclass(self): + self.assertSourceEqual(mod2.dc364, 364, 367) + self.assertRaises(OSError, inspect.getsource, mod2.dc370) + self.assertRaises(OSError, inspect.getsource, mod2.dc371) + class TestBlockComments(GetSourceBase): fodderModule = mod @@ -928,7 +1069,7 @@ def test_findsource_without_filename(self): self.assertRaises(IOError, inspect.findsource, co) self.assertRaises(IOError, inspect.getsource, co) - def test_findsource_with_out_of_bounds_lineno(self): + def test_findsource_on_func_with_out_of_bounds_lineno(self): mod_len = len(inspect.getsource(mod)) src = '\n' * 2* mod_len + "def f(): pass" co = compile(src, mod.__file__, "exec") @@ -936,12 +1077,26 @@ def test_findsource_with_out_of_bounds_lineno(self): eval(co, g, l) func = l['f'] self.assertEqual(func.__code__.co_firstlineno, 1+2*mod_len) - with self.assertRaisesRegex(IOError, "lineno is out of bounds"): + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): inspect.findsource(func) + def test_findsource_on_class_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "class A: pass" + co = compile(src, mod.__file__, "exec") + g, l = {'__name__': mod.__name__}, {} + eval(co, g, l) + cls = l['A'] + self.assertEqual(cls.__firstlineno__, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(cls) + def test_getsource_on_method(self): self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) + def test_getsource_on_class_code_object(self): + self.assertSourceEqual(mod2.ClassWithCodeObject.code, 315, 317) + def test_nested_func(self): self.assertSourceEqual(mod2.cls135.func136, 136, 139) @@ -962,7 +1117,11 @@ def test_class_decorator(self): self.assertSourceEqual(mod2.cls196, 194, 201) self.assertSourceEqual(mod2.cls196.cls200, 198, 201) + @support.requires_docstrings def test_class_inside_conditional(self): + # We skip this test when docstrings are not present, + # because docstrings are one of the main factors of + # finding the correct class in the source code. self.assertSourceEqual(mod2.cls238.cls239, 239, 240) def test_multiple_children_classes(self): @@ -1172,7 +1331,7 @@ def test_getfullargspec_builtin_methods(self): @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_func(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.docstring_with_signature_with_defaults spec = inspect.getfullargspec(builtin) self.assertEqual(spec.defaults[0], 'avocado') @@ -1181,7 +1340,7 @@ def test_getfullargspec_builtin_func(self): @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_func_no_signature(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.docstring_no_signature with self.assertRaises(TypeError): inspect.getfullargspec(builtin) @@ -1215,7 +1374,7 @@ def test_getfullargspec_builtin_func_no_signature(self): (dict.__class_getitem__, meth_type_o), ] try: - import _stat + import _stat # noqa: F401 except ImportError: # if the _stat extension is not available, stat.S_IMODE() is # implemented in Python, not in C @@ -1479,6 +1638,56 @@ def f(self): self.assertIn(('f', b.f), inspect.getmembers(b)) self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod)) + def test_getmembers_custom_dir(self): + class CorrectDir: + def __init__(self, attr): + self.attr = attr + def method(self): + return self.attr + 1 + def __dir__(self): + return ['attr', 'method'] + + cd = CorrectDir(5) + self.assertEqual(inspect.getmembers(cd), [ + ('attr', 5), + ('method', cd.method), + ]) + self.assertEqual(inspect.getmembers(cd, inspect.ismethod), [ + ('method', cd.method), + ]) + + def test_getmembers_custom_broken_dir(self): + # inspect.getmembers calls `dir()` on the passed object inside. + # if `__dir__` mentions some non-existent attribute, + # we still need to return others correctly. + class BrokenDir: + existing = 1 + def method(self): + return self.existing + 1 + def __dir__(self): + return ['method', 'missing', 'existing'] + + bd = BrokenDir() + self.assertEqual(inspect.getmembers(bd), [ + ('existing', 1), + ('method', bd.method), + ]) + self.assertEqual(inspect.getmembers(bd, inspect.ismethod), [ + ('method', bd.method), + ]) + + def test_getmembers_custom_duplicated_dir(self): + # Duplicates in `__dir__` must not fail and return just one result. + class DuplicatedDir: + attr = 1 + def __dir__(self): + return ['attr', 'attr'] + + dd = DuplicatedDir() + self.assertEqual(inspect.getmembers(dd), [ + ('attr', 1), + ]) + def test_getmembers_VirtualAttribute(self): class M(type): def __getattr__(cls, name): @@ -1523,105 +1732,6 @@ class C(metaclass=M): attrs = [a[0] for a in inspect.getmembers(C)] self.assertNotIn('missing', attrs) - def test_get_annotations_with_stock_annotations(self): - def foo(a:int, b:str): pass - self.assertEqual(inspect.get_annotations(foo), {'a': int, 'b': str}) - - foo.__annotations__ = {'a': 'foo', 'b':'str'} - self.assertEqual(inspect.get_annotations(foo), {'a': 'foo', 'b': 'str'}) - - self.assertEqual(inspect.get_annotations(foo, eval_str=True, locals=locals()), {'a': foo, 'b': str}) - self.assertEqual(inspect.get_annotations(foo, eval_str=True, globals=locals()), {'a': foo, 'b': str}) - - isa = inspect_stock_annotations - self.assertEqual(inspect.get_annotations(isa), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(inspect), {}) # inspect module has no annotations - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass}) - self.assertEqual(inspect.get_annotations(inspect, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(inspect, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) - - def times_three(fn): - @functools.wraps(fn) - def wrapper(a, b): - return fn(a*3, b*3) - return wrapper - - wrapped = times_three(isa.function) - self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) - self.assertIsNot(wrapped.__globals__, isa.function.__globals__) - self.assertEqual(inspect.get_annotations(wrapped), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) - - def test_get_annotations_with_stringized_annotations(self): - isa = inspect_stringized_annotations - self.assertEqual(inspect.get_annotations(isa), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.function), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function2), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function3), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) - - isa2 = inspect_stringized_annotations_2 - self.assertEqual(inspect.get_annotations(isa2), {}) - self.assertEqual(inspect.get_annotations(isa2, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa2, eval_str=False), {}) - - def times_three(fn): - @functools.wraps(fn) - def wrapper(a, b): - return fn(a*3, b*3) - return wrapper - - wrapped = times_three(isa.function) - self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) - self.assertIsNot(wrapped.__globals__, isa.function.__globals__) - self.assertEqual(inspect.get_annotations(wrapped), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - - # test that local namespace lookups work - self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations), {'x': 'mytype'}) - self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), {'x': int}) - class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): @@ -1630,6 +1740,122 @@ def test_typing_replacement(self): self.assertEqual(inspect.formatannotation(ann1), 'Union[List[testModule.typing.A], int]') +class TestIsMethodDescriptor(unittest.TestCase): + + def test_custom_descriptors(self): + class MethodDescriptor: + def __get__(self, *_): pass + class MethodDescriptorSub(MethodDescriptor): + pass + class DataDescriptorWithNoGet: + def __set__(self, *_): pass + class DataDescriptorWithGetSet: + def __get__(self, *_): pass + def __set__(self, *_): pass + class DataDescriptorWithGetDelete: + def __get__(self, *_): pass + def __delete__(self, *_): pass + class DataDescriptorSub(DataDescriptorWithNoGet, + DataDescriptorWithGetDelete): + pass + + # Custom method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptor()), + '__get__ and no __set__/__delete__ => method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptorSub()), + '__get__ (inherited) and no __set__/__delete__' + ' => method descriptor') + + # Custom data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithNoGet()), + '__set__ (and no __get__) => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetSet()), + '__get__ and __set__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetDelete()), + '__get__ and __delete__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorSub()), + '__get__, __set__ and __delete__ => not a method descriptor') + + # Classes of descriptors (are *not* descriptors themselves): + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptor)) + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptorSub)) + self.assertFalse(inspect.ismethoddescriptor(DataDescriptorSub)) + + def test_builtin_descriptors(self): + builtin_slot_wrapper = int.__add__ # This one is mentioned in docs. + class Owner: + def instance_method(self): pass + @classmethod + def class_method(cls): pass + @staticmethod + def static_method(): pass + @property + def a_property(self): pass + class Slotermeyer: + __slots__ = 'a_slot', + def function(): + pass + a_lambda = lambda: None + + # Example builtin method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(builtin_slot_wrapper), + 'a builtin slot wrapper is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['class_method']), + 'a classmethod object is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['static_method']), + 'a staticmethod object is a method descriptor') + + # Example builtin data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(Owner.__dict__['a_property']), + 'a property is not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(Slotermeyer.__dict__['a_slot']), + 'a slot is not a method descriptor') + + # `types.MethodType`/`types.FunctionType` instances (they *are* + # method descriptors, but `ismethoddescriptor()` explicitly + # excludes them): + self.assertFalse(inspect.ismethoddescriptor(Owner().instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().static_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.static_method)) + self.assertFalse(inspect.ismethoddescriptor(function)) + self.assertFalse(inspect.ismethoddescriptor(a_lambda)) + self.assertTrue(inspect.ismethoddescriptor(functools.partial(function))) + + def test_descriptor_being_a_class(self): + class MethodDescriptorMeta(type): + def __get__(self, *_): pass + class ClassBeingMethodDescriptor(metaclass=MethodDescriptorMeta): + pass + # `ClassBeingMethodDescriptor` itself *is* a method descriptor, + # but it is *also* a class, and `ismethoddescriptor()` explicitly + # excludes classes. + self.assertFalse( + inspect.ismethoddescriptor(ClassBeingMethodDescriptor), + 'classes (instances of type) are explicitly excluded') + + def test_non_descriptors(self): + class Test: + pass + self.assertFalse(inspect.ismethoddescriptor(Test())) + self.assertFalse(inspect.ismethoddescriptor(Test)) + self.assertFalse(inspect.ismethoddescriptor([42])) + self.assertFalse(inspect.ismethoddescriptor(42)) + + class TestIsDataDescriptor(unittest.TestCase): def test_custom_descriptors(self): @@ -2251,6 +2477,13 @@ def __dict__(self): self.assertEqual(inspect.getattr_static(foo, 'a'), 3) self.assertFalse(test.called) + class Bar(Foo): pass + + bar = Bar() + bar.a = 5 + self.assertEqual(inspect.getattr_static(bar, 'a'), 3) + self.assertFalse(test.called) + def test_mutated_mro(self): test = self test.called = False @@ -2355,6 +2588,21 @@ def __getattribute__(self, attr): self.assertFalse(test.called) + def test_cache_does_not_cause_classes_to_persist(self): + # regression test for gh-118013: + # check that the internal _shadowed_dict cache does not cause + # dynamically created classes to have extended lifetimes even + # when no other strong references to those classes remain. + # Since these classes can themselves hold strong references to + # other objects, this can cause unexpected memory consumption. + class Foo: pass + Foo.instance = Foo() + weakref_to_class = weakref.ref(Foo) + inspect.getattr_static(Foo.instance, 'whatever', 'irrelevant') + del Foo + gc.collect() + self.assertIsNone(weakref_to_class()) + class TestGetGeneratorState(unittest.TestCase): @@ -2842,7 +3090,7 @@ def test_staticmethod(*args): # NOQA @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_builtins(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") def test_unbound_method(o): """Use this to test unbound methods (things that should have a self)""" @@ -2899,9 +3147,12 @@ def p(name): return signature.parameters[name].default # This doesn't work now. # (We don't have a valid signature for "type" in 3.4) + class ThisWorksNow: + __call__ = type + # TODO: Support type. + self.assertEqual(ThisWorksNow()(1), int) + self.assertEqual(ThisWorksNow()('A', (), {}).__name__, 'A') with self.assertRaisesRegex(ValueError, "no signature found"): - class ThisWorksNow: - __call__ = type test_callable(ThisWorksNow()) # Regression test for issue #20786 @@ -2920,7 +3171,7 @@ class ThisWorksNow: @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_decorated_builtins(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") func = _testcapi.docstring_with_signature_with_defaults def decorator(func): @@ -2941,7 +3192,7 @@ def wrapper_like(*args, **kwargs) -> int: pass @cpython_only def test_signature_on_builtins_no_signature(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") with self.assertRaisesRegex(ValueError, 'no signature found for builtin'): inspect.signature(_testcapi.docstring_no_signature) @@ -2979,7 +3230,7 @@ def test_signature_on_builtins_no_signature(self): (dict.__class_getitem__, meth_o), ] try: - import _stat + import _stat # noqa: F401 except ImportError: # if the _stat extension is not available, stat.S_IMODE() is # implemented in Python, not in C @@ -2991,6 +3242,13 @@ def test_signature_on_builtins_no_signature(self): self.assertEqual(inspect.signature(builtin), inspect.signature(template)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_parsing_with_defaults(self): + _testcapi = import_helper.import_module("_testcapi") + meth = _testcapi.DocStringUnrepresentableSignatureTest.with_default + self.assertEqual(str(inspect.signature(meth)), '(self, /, x=1)') + def test_signature_on_non_function(self): with self.assertRaisesRegex(TypeError, 'is not a callable object'): inspect.signature(42) @@ -3108,6 +3366,10 @@ def m1d(*args, **kwargs): int)) def test_signature_on_classmethod(self): + self.assertEqual(self.signature(classmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @classmethod def foo(cls, arg1, *, arg2=1): @@ -3126,6 +3388,10 @@ def foo(cls, arg1, *, arg2=1): ...)) def test_signature_on_staticmethod(self): + self.assertEqual(self.signature(staticmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @staticmethod def foo(cls, *, arg): @@ -3144,7 +3410,7 @@ def foo(cls, *, arg): ...)) def test_signature_on_partial(self): - from functools import partial + from functools import partial, Placeholder def test(): pass @@ -3199,6 +3465,25 @@ def test(a, b, *, c, d): ('d', ..., ..., "keyword_only")), ...)) + # With Placeholder + self.assertEqual(self.signature(partial(test, Placeholder, 1)), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, Placeholder, 1, c=2)), + ((('a', ..., ..., "positional_only"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + # Ensure unittest.mock.ANY & similar do not get picked up as a Placeholder + self.assertEqual(self.signature(partial(test, unittest.mock.ANY, 1, c=2)), + ((('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + def test(a, *args, b, **kwargs): pass @@ -3246,6 +3531,15 @@ def test(a, *args, b, **kwargs): ('kwargs', ..., ..., "var_keyword")), ...)) + # With Placeholder + p = partial(test, Placeholder, Placeholder, 1, b=0, test=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + def test(a, b, c:int) -> 42: pass @@ -3350,6 +3644,34 @@ def foo(a, b, /, c, d, **kwargs): ('kwargs', ..., ..., 'var_keyword')), ...)) + # Positional only With Placeholder + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + # Optionals Positional With Placeholder + def foo(a=0, b=1, /, c=2, d=3): + pass + + # Positional + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only")), + ...)) + + # Positional or Keyword - transformed to positional + p = partial(foo, Placeholder, 1, Placeholder, 1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "positional_only")), + ...)) + def test_signature_on_partialmethod(self): from functools import partialmethod @@ -3362,18 +3684,32 @@ def test(): inspect.signature(Spam.ham) class Spam: - def test(it, a, *, c) -> 'spam': + def test(it, a, b, *, c) -> 'spam': pass ham = partialmethod(test, c=1) + bar = partialmethod(test, functools.Placeholder, 1, c=1) self.assertEqual(self.signature(Spam.ham, eval_str=False), ((('it', ..., ..., 'positional_or_keyword'), ('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), ('c', 1, ..., 'keyword_only')), 'spam')) self.assertEqual(self.signature(Spam().ham, eval_str=False), ((('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + # With Placeholder + self.assertEqual(self.signature(Spam.bar, eval_str=False), + ((('it', ..., ..., 'positional_only'), + ('a', ..., ..., 'positional_only'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + self.assertEqual(self.signature(Spam().bar, eval_str=False), + ((('a', ..., ..., 'positional_only'), ('c', 1, ..., 'keyword_only')), 'spam')) @@ -3389,7 +3725,7 @@ def test(self: 'anno', x): def test_signature_on_fake_partialmethod(self): def foo(a): pass - foo._partialmethod = 'spam' + foo.__partialmethod__ = 'spam' self.assertEqual(str(inspect.signature(foo)), '(a)') def test_signature_on_decorated(self): @@ -3484,6 +3820,98 @@ def __init__(self, b): ((('a', ..., ..., "positional_or_keyword"),), ...)) + with self.subTest('classmethod'): + class CM(type): + @classmethod + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class CM(type): + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) + class C(metaclass=CM): + def __init__(self, c): + pass + + self.assertEqual(C(1), (2, C, 1)) + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class CM(type): + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class CM(type): + __call__ = ':'.join + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(['a', 'bc']), 'a:bc') + # BUG: Returns '' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class CM(type): + __call__ = (2).__pow__ + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(3), 8) + self.assertEqual(C(3, 7), 1) + # BUG: Returns '' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature((0).__pow__)) + class CM(type): def __new__(mcls, name, bases, dct, *, foo=1): return super().__new__(mcls, name, bases, dct) @@ -3545,6 +3973,169 @@ def __init__(self, b): ('bar', 2, ..., "keyword_only")), ...)) + def test_signature_on_class_with_init(self): + class C: + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + pass + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(lambda x, a, b: None, 2) + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_class_with_new(self): + with self.subTest('FunctionType'): + class C: + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(lambda x, cls, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(lambda cls, x, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __new__ = str.__subclasscheck__ + + self.assertEqual(C(), False) + # TODO: Support BuiltinMethodType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + with self.subTest('MethodWrapperType'): + class C: + __new__ = type.__or__.__get__(int, type) + + self.assertEqual(C(), C | int) + # TODO: Support MethodWrapperType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + # TODO: Test ClassMethodDescriptorType + + with self.subTest('MethodDescriptorType'): + class C: + __new__ = type.__dict__['__subclasscheck__'] + + self.assertEqual(C(C), True) + self.assertEqual(self.signature(C), self.signature(C.__subclasscheck__)) + + with self.subTest('WrapperDescriptorType'): + class C: + __new__ = type.__or__ + + self.assertEqual(C(int), C | int) + # TODO: Support WrapperDescriptorType + # self.assertEqual(self.signature(C), self.signature(C.__or__)) + self.assertRaises(ValueError, self.signature, C) + def test_signature_on_subclass(self): class A: def __new__(cls, a=1, *args, **kwargs): @@ -3598,8 +4189,11 @@ class D(C): pass # Test meta-classes without user-defined __init__ or __new__ class C(type): pass class D(C): pass + self.assertEqual(C('A', (), {}).__name__, 'A') + # TODO: Support type. with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(C), None) + self.assertEqual(D('A', (), {}).__name__, 'A') with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(D), None) @@ -3649,16 +4243,140 @@ class Bar(Spam, Foo): ((('a', ..., ..., "positional_or_keyword"),), ...)) - class Wrapped: - pass - Wrapped.__wrapped__ = lambda a: None - self.assertEqual(self.signature(Wrapped), + with self.subTest('classmethod'): + class C: + @classmethod + def __call__(cls, a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __call__(a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class C: + __call__ = A().call + + self.assertEqual(C()(1), 1) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) + + c = C() + self.assertEqual(c(1), (2, c, 1)) + self.assertEqual(self.signature(C()), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + + self.assertEqual(C()(1), (2, 1)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __call__ = ':'.join + + self.assertEqual(C()(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class C: + __call__ = (2).__pow__ + + self.assertEqual(C()(3), 8) + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('ClassMethodDescriptorType'): + class C(dict): + __call__ = dict.__dict__['fromkeys'] + + res = C()([1, 2], 3) + self.assertEqual(res, {1: 3, 2: 3}) + self.assertEqual(type(res), C) + self.assertEqual(self.signature(C()), self.signature(dict.fromkeys)) + + with self.subTest('MethodDescriptorType'): + class C(str): + __call__ = str.join + + self.assertEqual(C(':')(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('WrapperDescriptorType'): + class C(int): + __call__ = int.__pow__ + + self.assertEqual(C(2)(3), 8) + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('MemberDescriptorType'): + class C: + __slots__ = '__call__' + c = C() + c.__call__ = lambda a: a + self.assertEqual(c(1), 1) + self.assertEqual(self.signature(c), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_callable_objects_with_text_signature_attr(self): + class C: + __text_signature__ = '(a, /, b, c=True)' + def __call__(self, *args, **kwargs): + pass + + self.assertEqual(self.signature(C), ((), ...)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_only"), + ('b', ..., ..., "positional_or_keyword"), + ('c', True, ..., "positional_or_keyword"), + ), + ...)) + + c = C() + c.__text_signature__ = '(x, y)' + self.assertEqual(self.signature(c), + ((('x', ..., ..., "positional_or_keyword"), + ('y', ..., ..., "positional_or_keyword"), + ), + ...)) + + def test_signature_on_wrapper(self): + class Wrapper: + def __call__(self, b): + pass + wrapper = Wrapper() + wrapper.__wrapped__ = lambda a: None + self.assertEqual(self.signature(wrapper), ((('a', ..., ..., "positional_or_keyword"),), ...)) # wrapper loop: - Wrapped.__wrapped__ = Wrapped + wrapper = Wrapper() + wrapper.__wrapped__ = wrapper with self.assertRaisesRegex(ValueError, 'wrapper loop'): - self.signature(Wrapped) + self.signature(wrapper) def test_signature_on_lambdas(self): self.assertEqual(self.signature((lambda a=10: a)), @@ -3916,6 +4634,18 @@ def func( expected_multiline, ) + def test_signature_format_unquote(self): + def func(x: 'int') -> 'str': ... + + self.assertEqual( + inspect.signature(func).format(), + "(x: 'int') -> 'str'" + ) + self.assertEqual( + inspect.signature(func).format(quote_annotation_strings=False), + "(x: int) -> str" + ) + def test_signature_replace_parameters(self): def test(a, b) -> 42: pass @@ -4148,6 +4878,26 @@ def test_signature_eval_str(self): par('b', PORK, annotation=tuple), ))) + def test_signature_annotation_format(self): + ida = inspect_deferred_annotations + sig = inspect.Signature + par = inspect.Parameter + PORK = inspect.Parameter.POSITIONAL_OR_KEYWORD + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func=signature_func): + self.assertEqual( + signature_func(ida.f, annotation_format=Format.STRING), + sig([par("x", PORK, annotation="undefined")]) + ) + self.assertEqual( + signature_func(ida.f, annotation_format=Format.FORWARDREF), + sig([par("x", PORK, annotation=ForwardRef("undefined"))]) + ) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f, annotation_format=Format.VALUE) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f) + def test_signature_none_annotation(self): class funclike: # Has to be callable, and have correct @@ -4173,38 +4923,6 @@ def foo(): pass self.assertEqual(signature_func(foo), inspect.Signature()) self.assertEqual(inspect.get_annotations(foo), {}) - def test_signature_as_str(self): - self.maxDiff = None - class S: - __signature__ = '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - - def test_signature_as_callable(self): - # __signature__ should be either a staticmethod or a bound classmethod - class S: - @classmethod - def __signature__(cls): - return '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - - class S: - @staticmethod - def __signature__(): - return '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - def test_signature_on_derived_classes(self): # gh-105080: Make sure that signatures are consistent on derived classes @@ -4223,6 +4941,16 @@ class D2(D1): self.assertEqual(inspect.signature(D2), inspect.signature(D1)) + def test_signature_on_non_comparable(self): + class NoncomparableCallable: + def __call__(self, a): + pass + def __eq__(self, other): + 1/0 + self.assertEqual(self.signature(NoncomparableCallable()), + ((('a', ..., ..., 'positional_or_keyword'),), + ...)) + class TestParameterObject(unittest.TestCase): def test_signature_parameter_kinds(self): @@ -4588,15 +5316,30 @@ def test(a_po, b_po, c_po=3, /, foo=42, *, bar=50, **kwargs): self.assertEqual(self.call(test, 1, 2, foo=4, bar=5), (1, 2, 3, 4, 5, {})) - with self.assertRaisesRegex(TypeError, "but was passed as a keyword"): - self.call(test, 1, 2, foo=4, bar=5, c_po=10) + self.assertEqual(self.call(test, 1, 2, foo=4, bar=5, c_po=10), + (1, 2, 3, 4, 5, {'c_po': 10})) + + self.assertEqual(self.call(test, 1, 2, 30, c_po=31, foo=4, bar=5), + (1, 2, 30, 4, 5, {'c_po': 31})) + + self.assertEqual(self.call(test, 1, 2, 30, foo=4, bar=5, c_po=31), + (1, 2, 30, 4, 5, {'c_po': 31})) - with self.assertRaisesRegex(TypeError, "parameter is positional only"): - self.call(test, 1, 2, c_po=4) + self.assertEqual(self.call(test, 1, 2, c_po=4), + (1, 2, 3, 42, 50, {'c_po': 4})) - with self.assertRaisesRegex(TypeError, "parameter is positional only"): + with self.assertRaisesRegex(TypeError, "missing 2 required positional arguments"): self.call(test, a_po=1, b_po=2) + def without_var_kwargs(c_po=3, d_po=4, /): + return c_po, d_po + + with self.assertRaisesRegex( + TypeError, + "positional-only arguments passed as keyword arguments: 'c_po, d_po'", + ): + self.call(without_var_kwargs, c_po=33, d_po=44) + def test_signature_bind_with_self_arg(self): # Issue #17071: one of the parameters is named "self def test(a, self, b): @@ -4630,7 +5373,7 @@ def test_signature_bind_implicit_arg(self): # Issue #19611: getcallargs should work with comprehensions def make_set(): return set(z * z for z in range(5)) - gencomp_code = make_set.__code__.co_consts[1] + gencomp_code = make_set.__code__.co_consts[0] gencomp_func = types.FunctionType(gencomp_code, {}) iterator = iter(range(5)) @@ -4800,10 +5543,17 @@ class TestSignatureDefinitions(unittest.TestCase): # This test case provides a home for checking that particular APIs # have signatures available for introspection + @staticmethod + def is_public(name): + return not name.startswith('_') or name.startswith('__') and name.endswith('__') + @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") - def test_builtins_have_signatures(self): + def _test_module_has_signatures(self, module, + no_signature=(), unsupported_signature=(), + methods_no_signature={}, methods_unsupported_signature={}, + good_exceptions=()): # This checks all builtin callables in CPython have signatures # A few have signatures Signature can't yet handle, so we skip those # since they will have to wait until PEP 457 adds the required @@ -4812,48 +5562,273 @@ def test_builtins_have_signatures(self): # reasons, so we also skip those for the time being, but design # the test to fail in order to indicate when it needs to be # updated. - no_signature = set() - # These need PEP 457 groups - needs_groups = {"range", "slice", "dir", "getattr", - "next", "iter", "vars"} - no_signature |= needs_groups - # These have unrepresentable parameter default values of NULL - needs_null = {"anext"} - no_signature |= needs_null - # These need *args support in Argument Clinic - needs_varargs = {"min", "max", "__build_class__"} - no_signature |= needs_varargs - # These builtin types are expected to provide introspection info - types_with_signatures = { - 'bool', 'classmethod', 'complex', 'enumerate', 'filter', 'float', - 'frozenset', 'list', 'map', 'memoryview', 'object', 'property', - 'reversed', 'set', 'staticmethod', 'tuple', 'zip' - } + no_signature = no_signature or set() # Check the signatures we expect to be there - ns = vars(builtins) + ns = vars(module) + try: + names = set(module.__all__) + except AttributeError: + names = set(name for name in ns if self.is_public(name)) for name, obj in sorted(ns.items()): + if name not in names: + continue if not callable(obj): continue - # The builtin types haven't been converted to AC yet - if isinstance(obj, type) and (name not in types_with_signatures): - # Note that this also skips all the exception types + if (isinstance(obj, type) and + issubclass(obj, BaseException) and + name not in good_exceptions): no_signature.add(name) - if (name in no_signature): - # Not yet converted - continue - if name in {'classmethod', 'staticmethod'}: - # Bug gh-112006: inspect.unwrap() does not work with types - # with the __wrapped__ data descriptor. - continue - with self.subTest(builtin=name): - self.assertIsNotNone(inspect.signature(obj)) + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', builtin=name): + self.assertIsNotNone(inspect.signature(obj)) + if isinstance(obj, type): + with self.subTest(type=name): + self._test_builtin_methods_have_signatures(obj, + methods_no_signature.get(name, ()), + methods_unsupported_signature.get(name, ())) # Check callables that haven't been converted don't claim a signature # This ensures this test will start failing as more signatures are # added, so the affected items can be moved into the scope of the # regression test above - for name in no_signature - needs_null: - with self.subTest(builtin=name): - self.assertIsNone(ns[name].__text_signature__) + for name in no_signature: + with self.subTest('none', builtin=name): + obj = ns[name] + self.assertIsNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + for name in unsupported_signature: + with self.subTest('unsupported', builtin=name): + obj = ns[name] + self.assertIsNotNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + + def _test_builtin_methods_have_signatures(self, cls, no_signature, unsupported_signature): + ns = vars(cls) + for name in ns: + obj = getattr(cls, name, None) + if not callable(obj) or isinstance(obj, type): + continue + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', method=name): + self.assertIsNotNone(inspect.signature(obj)) + for name in no_signature: + with self.subTest('none', method=name): + self.assertIsNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + for name in unsupported_signature: + with self.subTest('unsupported', method=name): + self.assertIsNotNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + + def test_builtins_have_signatures(self): + no_signature = {'type', 'super', 'bytearray', 'bytes', 'dict', 'int', 'str'} + # These need PEP 457 groups + needs_groups = {"range", "slice", "dir", "getattr", + "next", "iter", "vars"} + no_signature |= needs_groups + # These have unrepresentable parameter default values of NULL + unsupported_signature = {"anext"} + # These need *args support in Argument Clinic + needs_varargs = {"min", "max", "__build_class__"} + no_signature |= needs_varargs + + methods_no_signature = { + 'dict': {'update'}, + 'object': {'__class__'}, + } + methods_unsupported_signature = { + 'bytearray': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'bytes': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'dict': {'pop'}, + 'memoryview': {'cast', 'hex'}, + 'str': {'count', 'endswith', 'find', 'index', 'maketrans', 'rfind', 'rindex', 'startswith'}, + } + self._test_module_has_signatures(builtins, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_types_module_has_signatures(self): + unsupported_signature = {'CellType'} + methods_no_signature = { + 'AsyncGeneratorType': {'athrow'}, + 'CoroutineType': {'throw'}, + 'GeneratorType': {'throw'}, + } + self._test_module_has_signatures(types, + unsupported_signature=unsupported_signature, + methods_no_signature=methods_no_signature) + + def test_sys_module_has_signatures(self): + no_signature = {'getsizeof', 'set_asyncgen_hooks'} + no_signature |= {name for name in ['getobjects'] + if hasattr(sys, name)} + self._test_module_has_signatures(sys, no_signature) + + def test_abc_module_has_signatures(self): + import abc + self._test_module_has_signatures(abc) + + def test_atexit_module_has_signatures(self): + import atexit + self._test_module_has_signatures(atexit) + + def test_codecs_module_has_signatures(self): + import codecs + methods_no_signature = {'StreamReader': {'charbuffertype'}} + self._test_module_has_signatures(codecs, + methods_no_signature=methods_no_signature) + + def test_collections_module_has_signatures(self): + no_signature = {'OrderedDict', 'defaultdict'} + unsupported_signature = {'deque'} + methods_no_signature = { + 'OrderedDict': {'update'}, + } + methods_unsupported_signature = { + 'deque': {'index'}, + 'OrderedDict': {'pop'}, + 'UserString': {'maketrans'}, + } + self._test_module_has_signatures(collections, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_collections_abc_module_has_signatures(self): + import collections.abc + self._test_module_has_signatures(collections.abc) + + def test_errno_module_has_signatures(self): + import errno + self._test_module_has_signatures(errno) + + def test_faulthandler_module_has_signatures(self): + import faulthandler + unsupported_signature = {'dump_traceback', 'dump_traceback_later', 'enable'} + unsupported_signature |= {name for name in ['register'] + if hasattr(faulthandler, name)} + self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) + + def test_functools_module_has_signatures(self): + unsupported_signature = {"reduce"} + self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) + + def test_gc_module_has_signatures(self): + import gc + no_signature = {'set_threshold'} + self._test_module_has_signatures(gc, no_signature) + + def test_io_module_has_signatures(self): + methods_no_signature = { + 'BufferedRWPair': {'read', 'peek', 'read1', 'readinto', 'readinto1', 'write'}, + } + self._test_module_has_signatures(io, + methods_no_signature=methods_no_signature) + + def test_itertools_module_has_signatures(self): + import itertools + no_signature = {'islice', 'repeat'} + self._test_module_has_signatures(itertools, no_signature) + + def test_locale_module_has_signatures(self): + import locale + self._test_module_has_signatures(locale) + + def test_marshal_module_has_signatures(self): + import marshal + self._test_module_has_signatures(marshal) + + def test_operator_module_has_signatures(self): + import operator + self._test_module_has_signatures(operator) + + def test_os_module_has_signatures(self): + unsupported_signature = {'chmod', 'utime'} + unsupported_signature |= {name for name in + ['get_terminal_size', 'posix_spawn', 'posix_spawnp', + 'register_at_fork', 'startfile'] + if hasattr(os, name)} + self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) + + def test_pwd_module_has_signatures(self): + pwd = import_helper.import_module('pwd') + self._test_module_has_signatures(pwd) + + def test_re_module_has_signatures(self): + import re + methods_no_signature = {'Match': {'group'}} + self._test_module_has_signatures(re, + methods_no_signature=methods_no_signature, + good_exceptions={'error', 'PatternError'}) + + def test_signal_module_has_signatures(self): + import signal + self._test_module_has_signatures(signal) + + def test_stat_module_has_signatures(self): + import stat + self._test_module_has_signatures(stat) + + def test_string_module_has_signatures(self): + import string + self._test_module_has_signatures(string) + + def test_symtable_module_has_signatures(self): + import symtable + self._test_module_has_signatures(symtable) + + def test_sysconfig_module_has_signatures(self): + import sysconfig + self._test_module_has_signatures(sysconfig) + + def test_threading_module_has_signatures(self): + import threading + self._test_module_has_signatures(threading) + + def test_thread_module_has_signatures(self): + import _thread + no_signature = {'RLock'} + self._test_module_has_signatures(_thread, no_signature) + + def test_time_module_has_signatures(self): + no_signature = { + 'asctime', 'ctime', 'get_clock_info', 'gmtime', 'localtime', + 'strftime', 'strptime' + } + no_signature |= {name for name in + ['clock_getres', 'clock_settime', 'clock_settime_ns', + 'pthread_getcpuclockid'] + if hasattr(time, name)} + self._test_module_has_signatures(time, no_signature) + + def test_tokenize_module_has_signatures(self): + import tokenize + self._test_module_has_signatures(tokenize) + + def test_tracemalloc_module_has_signatures(self): + import tracemalloc + self._test_module_has_signatures(tracemalloc) + + def test_typing_module_has_signatures(self): + import typing + no_signature = {'ParamSpec', 'ParamSpecArgs', 'ParamSpecKwargs', + 'Text', 'TypeAliasType', 'TypeVar', 'TypeVarTuple'} + methods_no_signature = { + 'Generic': {'__class_getitem__', '__init_subclass__'}, + } + methods_unsupported_signature = { + 'Text': {'count', 'find', 'index', 'rfind', 'rindex', 'startswith', 'endswith', 'maketrans'}, + } + self._test_module_has_signatures(typing, no_signature, + methods_no_signature=methods_no_signature, + methods_unsupported_signature=methods_unsupported_signature) + + def test_warnings_module_has_signatures(self): + unsupported_signature = {'warn', 'warn_explicit'} + self._test_module_has_signatures(warnings, unsupported_signature=unsupported_signature) + + def test_weakref_module_has_signatures(self): + import weakref + no_signature = {'ReferenceType', 'ref'} + self._test_module_has_signatures(weakref, no_signature) def test_python_function_override_signature(self): def func(*args, **kwargs): @@ -4885,6 +5860,7 @@ def func(*args, **kwargs): with self.assertRaises(ValueError): inspect.signature(func) + @support.requires_docstrings def test_base_class_have_text_signature(self): # see issue 43118 from test.typinganndata.ann_module7 import BufferedReader @@ -4970,6 +5946,14 @@ def test_recursion_limit(self): with self.assertRaisesRegex(ValueError, 'wrapper loop'): inspect.unwrap(obj) + def test_wrapped_descriptor(self): + self.assertIs(inspect.unwrap(NTimesUnwrappable), NTimesUnwrappable) + self.assertIs(inspect.unwrap(staticmethod), staticmethod) + self.assertIs(inspect.unwrap(classmethod), classmethod) + self.assertIs(inspect.unwrap(staticmethod(classmethod)), classmethod) + self.assertIs(inspect.unwrap(classmethod(staticmethod)), staticmethod) + + class TestMain(unittest.TestCase): def test_only_source(self): module = importlib.import_module('unittest') diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 0bf55facad9fed..8870d7aa5d663d 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -1,5 +1,4 @@ import sys -import time import unittest from unittest import mock @@ -12,6 +11,11 @@ except ImportError: _pylong = None +try: + import _decimal +except ImportError: + _decimal = None + L = [ ('0', 0), ('1', 1), @@ -397,68 +401,8 @@ def __trunc__(self): class JustTrunc(base): def __trunc__(self): return 42 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(JustTrunc()), 42) - - class ExceptionalTrunc(base): - def __trunc__(self): - 1 / 0 - with self.assertRaises(ZeroDivisionError), \ - self.assertWarns(DeprecationWarning): - int(ExceptionalTrunc()) - - for trunc_result_base in (object, Classic): - class Index(trunc_result_base): - def __index__(self): - return 42 - - class TruncReturnsNonInt(base): - def __trunc__(self): - return Index() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class Intable(trunc_result_base): - def __int__(self): - return 42 - - class TruncReturnsNonIndex(base): - def __trunc__(self): - return Intable() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class NonIntegral(trunc_result_base): - def __trunc__(self): - # Check that we avoid infinite recursion. - return NonIntegral() - - class TruncReturnsNonIntegral(base): - def __trunc__(self): - return NonIntegral() - try: - with self.assertWarns(DeprecationWarning): - int(TruncReturnsNonIntegral()) - except TypeError as e: - self.assertEqual(str(e), - "__trunc__ returned non-Integral" - " (type NonIntegral)") - else: - self.fail("Failed to raise TypeError with %s" % - ((base, trunc_result_base),)) - - # Regression test for bugs.python.org/issue16060. - class BadInt(trunc_result_base): - def __int__(self): - return 42.0 - - class TruncReturnsBadInt(base): - def __trunc__(self): - return BadInt() - - with self.assertRaises(TypeError), \ - self.assertWarns(DeprecationWarning): - int(TruncReturnsBadInt()) + with self.assertRaises(TypeError): + int(JustTrunc()) def test_int_subclass_with_index(self): class MyIndex(int): @@ -509,18 +453,6 @@ class BadInt2(int): def __int__(self): return True - class TruncReturnsBadIndex: - def __trunc__(self): - return BadIndex() - - class TruncReturnsBadInt: - def __trunc__(self): - return BadInt() - - class TruncReturnsIntSubclass: - def __trunc__(self): - return True - bad_int = BadIndex() with self.assertWarns(DeprecationWarning): n = int(bad_int) @@ -544,26 +476,6 @@ def __trunc__(self): self.assertEqual(n, 1) self.assertIs(type(n), int) - bad_int = TruncReturnsBadIndex() - with self.assertWarns(DeprecationWarning): - n = int(bad_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - - bad_int = TruncReturnsBadInt() - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, int, bad_int) - - good_int = TruncReturnsIntSubclass() - with self.assertWarns(DeprecationWarning): - n = int(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - with self.assertWarns(DeprecationWarning): - n = IntSubclass(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), IntSubclass) - def test_error_message(self): def check(s, base=None): with self.assertRaises(ValueError, @@ -604,6 +516,13 @@ def test_issue31619(self): self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + @support.cpython_only + def test_round_with_none_arg_direct_call(self): + for val in [(1).__round__(None), + round(1), + round(1, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) class IntStrDigitLimitsTests(unittest.TestCase): @@ -664,84 +583,78 @@ def test_denial_of_service_prevented_int_to_str(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 50_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic huge_int = int(f'0x{"c"*65_000}', base=16) # 78268 decimal digits. digits = 78_268 - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): huge_decimal = str(huge_int) - seconds_to_convert = get_time() - start self.assertEqual(len(huge_decimal), digits) # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') # We test with the limit almost at the size needed to check performance. # The performant limit check is slightly fuzzy, give it a some room. with support.adjust_int_max_str_digits(int(.995 * digits)): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): str(huge_int) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge_int = int(f'0x{"c"*500_000}', base=16) # 602060 digits. - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds said Zen based cloud VM. str(extra_huge_int) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLess(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLess(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_denial_of_service_prevented_str_to_int(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 100_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic digits = 133700 huge = '8'*digits - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): int(huge) - seconds_to_convert = get_time() - start # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') with support.adjust_int_max_str_digits(digits - 1): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): int(huge) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge = '7'*1_200_000 - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds in the Zen based cloud VM. int(extra_huge) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_power_of_two_bases_unlimited(self): """The limit does not apply to power of 2 bases.""" @@ -835,17 +748,28 @@ def tearDown(self): sys.set_int_max_str_digits(self._previous_limit) super().tearDown() - def test_pylong_int_to_decimal(self): - n = (1 << 100_000) - 1 - suffix = '9883109375' + def _test_pylong_int_to_decimal(self, n, suffix): s = str(n) - assert s[-10:] == suffix - s = str(-n) - assert s[-10:] == suffix - s = '%d' % n - assert s[-10:] == suffix - s = b'%d' % n - assert s[-10:] == suffix.encode('ascii') + self.assertEqual(s[-10:], suffix) + s2 = str(-n) + self.assertEqual(s2, '-' + s) + s3 = '%d' % n + self.assertEqual(s3, s) + s4 = b'%d' % n + self.assertEqual(s4, s.encode('ascii')) + + def test_pylong_int_to_decimal(self): + self._test_pylong_int_to_decimal((1 << 100_000), '9883109376') + self._test_pylong_int_to_decimal((1 << 100_000) - 1, '9883109375') + self._test_pylong_int_to_decimal(10**30_000, '0000000000') + self._test_pylong_int_to_decimal(10**30_000 - 1, '9999999999') + self._test_pylong_int_to_decimal(3**60_000, '9313200001') + + @support.requires_resource('cpu') + def test_pylong_int_to_decimal_2(self): + self._test_pylong_int_to_decimal(2**1_000_000, '2747109376') + self._test_pylong_int_to_decimal(10**300_000, '0000000000') + self._test_pylong_int_to_decimal(3**600_000, '3132000001') def test_pylong_int_divmod(self): n = (1 << 100_000) @@ -901,6 +825,100 @@ def test_pylong_misbehavior_error_path_from_str( with self.assertRaises(RuntimeError): int(big_value) + def test_pylong_roundtrip(self): + from random import randrange, getrandbits + bits = 5000 + while bits <= 1_000_000: + bits += randrange(-100, 101) # break bitlength patterns + hibit = 1 << (bits - 1) + n = hibit | getrandbits(bits - 1) + assert n.bit_length() == bits + sn = str(n) + self.assertFalse(sn.startswith('0')) + self.assertEqual(n, int(sn)) + bits <<= 1 + + @support.requires_resource('cpu') + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_pylong_roundtrip_huge(self): + # k blocks of 1234567890 + k = 1_000_000 # so 10 million digits in all + tentoten = 10**10 + n = 1234567890 * ((tentoten**k - 1) // (tentoten - 1)) + sn = "1234567890" * k + self.assertEqual(n, int(sn)) + self.assertEqual(sn, str(n)) + + @support.requires_resource('cpu') + @unittest.skipUnless(_pylong, "_pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_failsafe(self): + # While I believe the number of GUARD digits in this function is + # always enough so that no more than one correction step is ever + # needed, the code has a "failsafe" path that takes over if I'm + # wrong about that. We have no input that reaches that block. + # Here we test a contrived input that _does_ reach that block, + # provided the number of guard digits is reduced to 1. + sn = "9" * 2000156 + n = 10**len(sn) - 1 + orig_spread = _pylong._spread.copy() + _pylong._spread.clear() + try: + self.assertEqual(n, _pylong._dec_str_to_int_inner(sn, GUARD=1)) + self.assertIn(999, _pylong._spread) + finally: + _pylong._spread.clear() + _pylong._spread.update(orig_spread) + + @unittest.skipUnless(_pylong, "pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_monster(self): + # I don't think anyone has enough RAM to build a string long enough + # for this function to complain. So lie about the string length. + + class LyingStr(str): + def __len__(self): + return int((1 << 47) / _pylong._LOG_10_BASE_256) + + liar = LyingStr("42") + # We have to pass the liar directly to the complaining function. If we + # just try `int(liar)`, earlier layers will replace it with plain old + # "43". + # Embedding `len(liar)` into the f-string failed on the WASI testbot + # (don't know what that is): + # OverflowError: cannot fit 'int' into an index-sized integer + # So a random stab at worming around that. + self.assertRaisesRegex(ValueError, + f"^cannot convert string of len {liar.__len__()} to int$", + _pylong._dec_str_to_int_inner, + liar) + + @unittest.skipUnless(_pylong, "_pylong module required") + def test_pylong_compute_powers(self): + # Basic sanity tests. See end of _pylong.py for manual heavy tests. + def consumer(w, base, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit or w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + d = _pylong.compute_powers(w, base, limit, need_hi=need_hi) + self.assertEqual(d.keys(), need) + for k, v in d.items(): + self.assertEqual(v, base ** k) + + for base in 2, 5: + for need_hi in False, True: + for limit in 1, 11: + for w in range(250, 550): + consumer(w, base, limit, need_hi) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_interpreters/__init__.py b/Lib/test/test_interpreters/__init__.py index 4b16ecc31156a5..e3d189c4efcd27 100644 --- a/Lib/test/test_interpreters/__init__.py +++ b/Lib/test/test_interpreters/__init__.py @@ -1,5 +1,9 @@ import os -from test.support import load_package_tests +from test.support import load_package_tests, Py_GIL_DISABLED +import unittest + +if Py_GIL_DISABLED: + raise unittest.SkipTest("GIL disabled") def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index aefd326977095f..a9befbba64daa0 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,15 +1,32 @@ import os -import threading +import pickle from textwrap import dedent +import threading +import types import unittest from test import support from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +_interpreters = import_helper.import_module('_interpreters') +from test.support import Py_GIL_DISABLED from test.support import interpreters -from test.support.interpreters import InterpreterNotFoundError -from .utils import _captured_script, _run_output, _running, TestBase +from test.support import force_not_colorized +from test.support.interpreters import ( + InterpreterError, InterpreterNotFoundError, ExecutionFailed, +) +from .utils import ( + _captured_script, _run_output, _running, TestBase, + requires_test_modules, _testinternalcapi, +) + + +WHENCE_STR_UNKNOWN = 'unknown' +WHENCE_STR_RUNTIME = 'runtime init' +WHENCE_STR_LEGACY_CAPI = 'legacy C-API' +WHENCE_STR_CAPI = 'C-API' +WHENCE_STR_XI = 'cross-interpreter C-API' +WHENCE_STR_STDLIB = '_interpreters module' class ModuleTests(TestBase): @@ -37,6 +54,9 @@ def test_in_main(self): self.assertIsInstance(interp, interpreters.Interpreter) self.assertIn(interp, interpreters.list_all()) + # GH-126221: Passing an invalid Unicode character used to cause a SystemError + self.assertRaises(UnicodeEncodeError, _interpreters.create, '\udc80') + def test_in_thread(self): lock = threading.Lock() interp = None @@ -155,6 +175,18 @@ def test_idempotent(self): id2 = id(interp) self.assertNotEqual(id1, id2) + @requires_test_modules + def test_created_with_capi(self): + expected = _testinternalcapi.next_interpreter_id() + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print((interp.id, interp.whence)) + """) + interpid, whence = eval(text) + self.assertEqual(interpid, expected) + self.assertEqual(whence, WHENCE_STR_CAPI) + class ListAllTests(TestBase): @@ -197,6 +229,33 @@ def test_idempotent(self): for interp1, interp2 in zip(actual, expected): self.assertIs(interp1, interp2) + def test_created_with_capi(self): + mainid, *_ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected = [ + (mainid, WHENCE_STR_RUNTIME), + (interpid1, WHENCE_STR_STDLIB), + (interpid2, WHENCE_STR_STDLIB), + (interpid3, WHENCE_STR_STDLIB), + (interpid4, WHENCE_STR_CAPI), + (interpid5, WHENCE_STR_STDLIB), + ] + expected2 = expected[:-2] + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.create() + print( + [(i.id, i.whence) for i in interpreters.list_all()]) + """) + res = eval(text) + res2 = [(i.id, i.whence) for i in interpreters.list_all()] + self.assertEqual(res, expected) + self.assertEqual(res2, expected2) + class InterpreterObjectTests(TestBase): @@ -249,6 +308,38 @@ def test_id_readonly(self): with self.assertRaises(AttributeError): interp.id = 1_000_000 + def test_whence(self): + main = interpreters.get_main() + interp = interpreters.create() + + with self.subTest('main'): + self.assertEqual(main.whence, WHENCE_STR_RUNTIME) + + with self.subTest('from _interpreters'): + self.assertEqual(interp.whence, WHENCE_STR_STDLIB) + + with self.subTest('from C-API'): + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(repr(interp.whence)) + """) + whence = eval(text) + self.assertEqual(whence, WHENCE_STR_CAPI) + + with self.subTest('readonly'): + for value in [ + None, + WHENCE_STR_UNKNOWN, + WHENCE_STR_RUNTIME, + WHENCE_STR_STDLIB, + WHENCE_STR_CAPI, + ]: + with self.assertRaises(AttributeError): + interp.whence = value + with self.assertRaises(AttributeError): + main.whence = value + def test_hashable(self): interp = interpreters.create() expected = hash(interp.id) @@ -261,6 +352,12 @@ def test_equality(self): self.assertEqual(interp1, interp1) self.assertNotEqual(interp1, interp2) + def test_pickle(self): + interp = interpreters.create() + data = pickle.dumps(interp) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, interp) + class TestInterpreterIsRunning(TestBase): @@ -268,6 +365,7 @@ def test_main(self): main = interpreters.get_main() self.assertTrue(main.is_running()) + # XXX Is this still true? @unittest.skip('Fails on FreeBSD') def test_subinterpreter(self): interp = interpreters.create() @@ -280,7 +378,7 @@ def test_subinterpreter(self): def test_finished(self): r, w = self.pipe() interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os os.write({w}, b'x') """) @@ -290,7 +388,7 @@ def test_finished(self): def test_from_subinterpreter(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters if _interpreters.is_running({interp.id}): print(True) else: @@ -312,7 +410,7 @@ def test_with_only_background_threads(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading @@ -326,9 +424,58 @@ def task(): self.assertFalse(interp.is_running()) os.write(w_thread, DONE) - interp.exec_sync('t.join()') + interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), FINISHED) + def test_created_with_capi(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(interp.is_running()) + """) + def parse_results(text): + self.assertNotEqual(text, "") + try: + return eval(text) + except Exception: + raise Exception(repr(text)) + + with self.subTest('running __main__ (from self)'): + with self.interpreter_from_capi() as interpid: + text = self.run_from_capi(interpid, script, main=True) + running = parse_results(text) + self.assertTrue(running) + + with self.subTest('running, but not __main__ (from self)'): + text = self.run_temp_from_capi(script) + running = parse_results(text) + self.assertFalse(running) + + with self.subTest('running __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + before = interp.is_running() + with self.running_from_capi(interpid, main=True): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertTrue(during) + self.assertFalse(after) + + with self.subTest('running, but not __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + before = interp.is_running() + with self.running_from_capi(interpid, main=False): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertFalse(during) + self.assertFalse(after) + + with self.subTest('not running (from other)'): + with self.interpreter_obj_from_capi() as (interp, _): + running = interp.is_running() + self.assertFalse(running) + class TestInterpreterClose(TestBase): @@ -356,11 +503,11 @@ def test_all(self): def test_main(self): main, = interpreters.list_all() - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterError): main.close() def f(): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterError): main.close() t = threading.Thread(target=f) @@ -381,7 +528,7 @@ def test_from_current(self): interp = interpreters.Interpreter({interp.id}) try: interp.close() - except RuntimeError: + except interpreters.InterpreterError: print('failed') """)) self.assertEqual(out.strip(), 'failed') @@ -393,7 +540,7 @@ def test_from_sibling(self): interp2 = interpreters.create() self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) - interp1.exec_sync(dedent(f""" + interp1.exec(dedent(f""" from test.support import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() @@ -411,12 +558,13 @@ def f(): t.start() t.join() + # XXX Is this still true? @unittest.skip('Fails on FreeBSD') def test_still_running(self): main, = interpreters.list_all() interp = interpreters.create() with _running(interp): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterError): interp.close() self.assertTrue(interp.is_running()) @@ -427,7 +575,7 @@ def test_subthreads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading import time @@ -451,6 +599,53 @@ def task(): self.assertEqual(os.read(r_interp, 1), FINISHED) + def test_created_with_capi(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + interp.close() + """) + + with self.subTest('running __main__ (from self)'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*unrecognized'): + self.run_from_capi(interpid, script, main=True) + + with self.subTest('running, but not __main__ (from self)'): + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*unrecognized'): + self.run_temp_from_capi(script) + + with self.subTest('running __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=True): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + # Make sure it wssn't closed. + self.assertTrue( + self.interp_exists(interpid)) + + # The rest would be skipped until we deal with running threads when + # interp.close() is called. However, the "whence" restrictions + # trigger first. + + with self.subTest('running, but not __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=False): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + # Make sure it wssn't closed. + self.assertTrue( + self.interp_exists(interpid)) + + with self.subTest('not running (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + self.assertTrue( + self.interp_exists(interpid)) + class TestInterpreterPrepareMain(TestBase): @@ -503,28 +698,47 @@ def test_not_shareable(self): interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) # Make sure neither was actually bound. - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('print(foo)') - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('print(spam)') + with self.assertRaises(ExecutionFailed): + interp.exec('print(foo)') + with self.assertRaises(ExecutionFailed): + interp.exec('print(spam)') + + def test_running(self): + interp = interpreters.create() + interp.prepare_main({'spam': True}) + with self.running(interp): + with self.assertRaisesRegex(InterpreterError, 'running'): + interp.prepare_main({'spam': False}) + interp.exec('assert spam is True') + @requires_test_modules + def test_created_with_capi(self): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.prepare_main({'spam': True}) + with self.assertRaisesRegex(ExecutionFailed, 'NameError'): + self.run_from_capi(interpid, 'assert spam is True') -class TestInterpreterExecSync(TestBase): + +class TestInterpreterExec(TestBase): def test_success(self): interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - interp.exec_sync(script) - out = file.read() + script, results = _captured_script('print("it worked!", end="")') + with results: + interp.exec(script) + results = results.final() + results.raise_if_failed() + out = results.stdout self.assertEqual(out, 'it worked!') def test_failure(self): interp = interpreters.create() - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('raise Exception') + with self.assertRaises(ExecutionFailed): + interp.exec('raise Exception') + @force_not_colorized def test_display_preserved_exception(self): tempdir = self.temp_dir() modfile = self.make_module('spam', tempdir, text=""" @@ -542,21 +756,21 @@ def script(): spam.eggs() interp = interpreters.create() - interp.exec_sync(script) + interp.exec(script) """) stdout, stderr = self.assert_python_failure(scriptfile) self.maxDiff = None - interpmod_line, = (l for l in stderr.splitlines() if ' exec_sync' in l) - # File "{interpreters.__file__}", line 179, in exec_sync + interpmod_line, = (l for l in stderr.splitlines() if ' exec' in l) + # File "{interpreters.__file__}", line 179, in exec self.assertEqual(stderr, dedent(f"""\ Traceback (most recent call last): File "{scriptfile}", line 9, in - interp.exec_sync(script) - ~~~~~~~~~~~~~~~~^^^^^^^^ + interp.exec(script) + ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} - raise ExecFailure(excinfo) - test.support.interpreters.ExecFailure: RuntimeError: uh-oh! + raise ExecutionFailed(excinfo) + test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -575,15 +789,17 @@ def script(): def test_in_thread(self): interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: + script, results = _captured_script('print("it worked!", end="")') + with results: def f(): - interp.exec_sync(script) + interp.exec(script) t = threading.Thread(target=f) t.start() t.join() - out = file.read() + results = results.final() + results.raise_if_failed() + out = results.stdout self.assertEqual(out, 'it worked!') @@ -604,28 +820,29 @@ def test_fork(self): with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) - interp.exec_sync(script) + interp.exec(script) file.seek(0) content = file.read() self.assertEqual(content, expected) + # XXX Is this still true? @unittest.skip('Fails on FreeBSD') def test_already_running(self): interp = interpreters.create() with _running(interp): with self.assertRaises(RuntimeError): - interp.exec_sync('print("spam")') + interp.exec('print("spam")') def test_bad_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.exec_sync(10) + interp.exec(10) def test_bytes_for_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.exec_sync(b'print("spam")') + interp.exec(b'print("spam")') def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -636,7 +853,7 @@ def test_with_background_threads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading @@ -648,46 +865,234 @@ def task(): t.start() os.write({w_interp}, {RAN!r}) """) - interp.exec_sync(f"""if True: + interp.exec(f"""if True: os.write({w_interp}, {RAN!r}) """) os.write(w_thread, DONE) - interp.exec_sync('t.join()') + interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), FINISHED) - # test_xxsubinterpreters covers the remaining - # Interpreter.exec_sync() behavior. + def test_created_with_capi(self): + with self.interpreter_obj_from_capi() as (interp, _): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.exec('raise Exception("it worked!")') + # test__interpreters covers the remaining + # Interpreter.exec() behavior. -class TestInterpreterRun(TestBase): - def test_success(self): +def call_func_noop(): + pass + + +def call_func_return_shareable(): + return (1, None) + + +def call_func_return_not_shareable(): + return [1, 2, 3] + + +def call_func_failure(): + raise Exception('spam!') + + +def call_func_ident(value): + return value + + +def get_call_func_closure(value): + def call_func_closure(): + return value + return call_func_closure + + +class Spam: + + @staticmethod + def noop(): + pass + + @classmethod + def from_values(cls, *values): + return cls(values) + + def __init__(self, value): + self.value = value + + def __call__(self, *args, **kwargs): + return (self.value, args, kwargs) + + def __eq__(self, other): + if not isinstance(other, Spam): + return NotImplemented + return self.value == other.value + + def run(self, *args, **kwargs): + return (self.value, args, kwargs) + + +def call_func_complex(op, /, value=None, *args, exc=None, **kwargs): + if exc is not None: + raise exc + if op == '': + raise ValueError('missing op') + elif op == 'ident': + if args or kwargs: + raise Exception((args, kwargs)) + return value + elif op == 'full-ident': + return (value, args, kwargs) + elif op == 'globals': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return __name__ + elif op == 'interpid': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return interpreters.get_current().id + elif op == 'closure': + if args or kwargs: + raise Exception((args, kwargs)) + return get_call_func_closure(value) + elif op == 'custom': + if args or kwargs: + raise Exception((args, kwargs)) + return Spam(value) + elif op == 'custom-inner': + if args or kwargs: + raise Exception((args, kwargs)) + class Eggs(Spam): + pass + return Eggs(value) + elif not isinstance(op, str): + raise TypeError(op) + else: + raise NotImplementedError(op) + + +class TestInterpreterCall(TestBase): + + # signature + # - blank + # - args + # - kwargs + # - args, kwargs + # return + # - nothing (None) + # - simple + # - closure + # - custom + # ops: + # - do nothing + # - fail + # - echo + # - do complex, relative to interpreter + # scope + # - global func + # - local closure + # - returned closure + # - callable type instance + # - type + # - classmethod + # - staticmethod + # - instance method + # exception + # - builtin + # - custom + # - preserves info (e.g. SyntaxError) + # - matching error display + + def test_call(self): interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - t = interp.run(script) - t.join() - out = file.read() - self.assertEqual(out, 'it worked!') + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + res = interp.call(callable) + self.assertIs(res, None) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + with self.assertRaises(Exception): + if args or kwargs: + raise Exception((args, kwargs)) + interp.call(callable) + + with self.assertRaises(ExecutionFailed): + interp.call(call_func_failure) + + def test_call_in_thread(self): + interp = interpreters.create() - def test_failure(self): - caught = False - def excepthook(args): - nonlocal caught - caught = True - threading.excepthook = excepthook - try: - interp = interpreters.create() - t = interp.run('raise Exception') + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNone(ctx.caught) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + if args or kwargs: + continue + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNotNone(ctx.caught) + + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(call_func_failure) t.join() - - self.assertTrue(caught) - except BaseException: - threading.excepthook = threading.__excepthook__ + self.assertIsNotNone(ctx.caught) class TestIsShareable(TestBase): @@ -742,6 +1147,521 @@ class SubBytes(bytes): interpreters.is_shareable(obj)) +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_new_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + default = _interpreters.new_config('isolated') + with self.subTest('no arg'): + config = _interpreters.new_config() + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + with self.subTest('default'): + config1 = _interpreters.new_config('default') + self.assert_ns_equal(config1, default) + self.assertIsNot(config1, default) + + config2 = _interpreters.new_config('default') + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + for arg in ['', 'default']: + with self.subTest(f'default ({arg!r})'): + config = _interpreters.new_config(arg) + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + for name, vanilla in supported.items(): + with self.subTest(f'supported ({name})'): + expected = vanilla + config1 = _interpreters.new_config(name) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(name) + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + with self.subTest(f'noop override ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'override all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('extra override'): + with self.assertRaises(ValueError): + _interpreters.new_config(spam=True) + + # Bad values for bool fields. + for field, value in vars(supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'bad override ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_get_main(self): + interpid, whence = _interpreters.get_main() + self.assertEqual(interpid, 0) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_RUNTIME) + + def test_get_current(self): + with self.subTest('main'): + main, *_ = _interpreters.get_main() + interpid, whence = _interpreters.get_current() + self.assertEqual(interpid, main) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + + script = f""" + import _interpreters + interpid, whence = _interpreters.get_current() + print((interpid, whence)) + """ + def parse_stdout(text): + interpid, whence = eval(text) + return interpid, whence + + with self.subTest('from _interpreters'): + orig = _interpreters.create() + text = self.run_and_capture(orig, script) + interpid, whence = parse_stdout(text) + self.assertEqual(interpid, orig) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) + + with self.subTest('from C-API'): + last = 0 + for id, *_ in _interpreters.list_all(): + last = max(last, id) + expected = last + 1 + text = self.run_temp_from_capi(script) + interpid, whence = parse_stdout(text) + self.assertEqual(interpid, expected) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) + + def test_list_all(self): + mainid, *_ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + expected = [ + (mainid, _interpreters.WHENCE_RUNTIME), + (interpid1, _interpreters.WHENCE_STDLIB), + (interpid2, _interpreters.WHENCE_STDLIB), + (interpid3, _interpreters.WHENCE_STDLIB), + ] + + with self.subTest('main'): + res = _interpreters.list_all() + self.assertEqual(res, expected) + + with self.subTest('via interp from _interpreters'): + text = self.run_and_capture(interpid2, f""" + import _interpreters + print( + _interpreters.list_all()) + """) + + res = eval(text) + self.assertEqual(res, expected) + + with self.subTest('via interp from C-API'): + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected2 = expected + [ + (interpid4, _interpreters.WHENCE_CAPI), + (interpid5, _interpreters.WHENCE_STDLIB), + ] + expected3 = expected + [ + (interpid5, _interpreters.WHENCE_STDLIB), + ] + text = self.run_temp_from_capi(f""" + import _interpreters + _interpreters.create() + print( + _interpreters.list_all()) + """) + res2 = eval(text) + res3 = _interpreters.list_all() + self.assertEqual(res2, expected2) + self.assertEqual(res3, expected3) + + def test_create(self): + isolated = _interpreters.new_config('isolated') + legacy = _interpreters.new_config('legacy') + default = isolated + + with self.subTest('no args'): + interpid = _interpreters.create() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('config: None'): + interpid = _interpreters.create(None) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('config: \'empty\''): + with self.assertRaises(InterpreterError): + # The "empty" config isn't viable on its own. + _interpreters.create('empty') + + for arg, expected in { + '': default, + 'default': default, + 'isolated': isolated, + 'legacy': legacy, + }.items(): + with self.subTest(f'str arg: {arg!r}'): + interpid = _interpreters.create(arg) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config('empty') + orig.use_main_obmalloc = True + orig.check_multi_interp_extensions = bool(Py_GIL_DISABLED) + orig.gil = 'shared' + interpid = _interpreters.create(orig) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + with self.subTest('missing fields'): + orig = _interpreters.new_config() + del orig.gil + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('extra fields'): + orig = _interpreters.new_config() + orig.spam = True + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('whence'): + interpid = _interpreters.create() + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_STDLIB) + + @requires_test_modules + def test_destroy(self): + with self.subTest('from _interpreters'): + interpid = _interpreters.create() + before = [id for id, *_ in _interpreters.list_all()] + _interpreters.destroy(interpid) + after = [id for id, *_ in _interpreters.list_all()] + + self.assertIn(interpid, before) + self.assertNotIn(interpid, after) + self.assertFalse( + self.interp_exists(interpid)) + + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + with self.assertRaises(InterpreterError): + # It is the current interpreter. + _interpreters.destroy(interpid) + + with self.subTest('from C-API'): + interpid = _testinternalcapi.create_interpreter() + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.destroy(interpid, restrict=True) + self.assertTrue( + self.interp_exists(interpid)) + _interpreters.destroy(interpid) + self.assertFalse( + self.interp_exists(interpid)) + + def test_get_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + if Py_GIL_DISABLED: + expected.check_multi_interp_extensions = False + interpid, *_ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + interpid = _interpreters.create('isolated') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + interpid = _interpreters.create('legacy') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('from C-API'): + orig = _interpreters.new_config('isolated') + with self.interpreter_from_capi(orig) as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.get_config(interpid, restrict=True) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + @requires_test_modules + def test_whence(self): + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + + with self.subTest('stdlib'): + interpid = _interpreters.create() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) + + for orig, name in { + _interpreters.WHENCE_UNKNOWN: 'not ready', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + }.items(): + with self.subTest(f'from C-API ({orig}: {name})'): + with self.interpreter_from_capi(whence=orig) as interpid: + whence = _interpreters.whence(interpid) + self.assertEqual(whence, orig) + + with self.subTest('from C-API, running'): + text = self.run_temp_from_capi(dedent(f""" + import _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=True) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) + + with self.subTest('from legacy C-API, running'): + ... + text = self.run_temp_from_capi(dedent(f""" + import _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=False) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI) + + def test_is_running(self): + def check(interpid, expected): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.is_running(interpid, restrict=True) + running = _interpreters.is_running(interpid) + self.assertIs(running, expected) + + with self.subTest('from _interpreters (running)'): + interpid = _interpreters.create() + with self.running(interpid): + running = _interpreters.is_running(interpid) + self.assertTrue(running) + + with self.subTest('from _interpreters (not running)'): + interpid = _interpreters.create() + running = _interpreters.is_running(interpid) + self.assertFalse(running) + + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + check(interpid, True) + + with self.subTest('from C-API (running __main__)'): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=True): + check(interpid, True) + + with self.subTest('from C-API (running, but not __main__)'): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=False): + check(interpid, False) + + with self.subTest('from C-API (not running)'): + with self.interpreter_from_capi() as interpid: + check(interpid, False) + + def test_exec(self): + with self.subTest('run script'): + interpid = _interpreters.create() + script, results = _captured_script('print("it worked!", end="")') + with results: + exc = _interpreters.exec(interpid, script) + results = results.final() + results.raise_if_failed() + out = results.stdout + self.assertEqual(out, 'it worked!') + + with self.subTest('uncaught exception'): + interpid = _interpreters.create() + script, results = _captured_script(""" + raise Exception('uh-oh!') + print("it worked!", end="") + """) + with results: + exc = _interpreters.exec(interpid, script) + out = results.stdout() + self.assertEqual(out, '') + self.assert_ns_equal(exc, types.SimpleNamespace( + type=types.SimpleNamespace( + __name__='Exception', + __qualname__='Exception', + __module__='builtins', + ), + msg='uh-oh!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + )) + + with self.subTest('from C-API'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.exec(interpid, 'raise Exception("it worked!")', + restrict=True) + exc = _interpreters.exec(interpid, 'raise Exception("it worked!")') + self.assertIsNot(exc, None) + self.assertEqual(exc.msg, 'it worked!') + + def test_call(self): + with self.subTest('no args'): + interpid = _interpreters.create() + exc = _interpreters.call(interpid, call_func_return_shareable) + self.assertIs(exc, None) + + with self.subTest('uncaught exception'): + interpid = _interpreters.create() + exc = _interpreters.call(interpid, call_func_failure) + self.assertEqual(exc, types.SimpleNamespace( + type=types.SimpleNamespace( + __name__='Exception', + __qualname__='Exception', + __module__='builtins', + ), + msg='spam!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + )) + + @requires_test_modules + def test_set___main___attrs(self): + with self.subTest('from _interpreters'): + interpid = _interpreters.create() + before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + before2 = _interpreters.exec(interpid, 'assert ham == 42') + self.assertEqual(before1.type.__name__, 'NameError') + self.assertEqual(before2.type.__name__, 'NameError') + + _interpreters.set___main___attrs(interpid, dict( + spam='eggs', + ham=42, + )) + after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + after2 = _interpreters.exec(interpid, 'assert ham == 42') + after3 = _interpreters.exec(interpid, 'assert spam == 42') + self.assertIs(after1, None) + self.assertIs(after2, None) + self.assertEqual(after3.type.__name__, 'AssertionError') + + with self.subTest('from C-API'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.set___main___attrs(interpid, {'spam': True}, + restrict=True) + _interpreters.set___main___attrs(interpid, {'spam': True}) + rc = _testinternalcapi.exec_interpreter( + interpid, + 'assert spam is True', + ) + self.assertEqual(rc, 0) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 3c3e18832d4168..eada18f99d04db 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -1,3 +1,5 @@ +import importlib +import pickle import threading from textwrap import dedent import unittest @@ -5,12 +7,30 @@ from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -_channels = import_helper.import_module('_xxinterpchannels') +_channels = import_helper.import_module('_interpchannels') from test.support import interpreters from test.support.interpreters import channels from .utils import _run_output, TestBase +class LowLevelTests(TestBase): + + # The behaviors in the low-level module is important in as much + # as it is exercised by the high-level module. Therefore the + # most # important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + # Additional tests are found in Lib/test/test__interpchannels.py. + # XXX Those should be either moved to LowLevelTests or eliminated + # in favor of high-level tests in this file. + + def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). + importlib.reload(channels) + + class TestChannels(TestBase): def test_create(self): @@ -28,6 +48,7 @@ def test_list_all(self): self.assertEqual(after, created) def test_shareable(self): + interp = interpreters.create() rch, sch = channels.create() self.assertTrue( @@ -40,8 +61,25 @@ def test_shareable(self): rch2 = rch.recv() sch2 = rch.recv() + interp.prepare_main(rch=rch, sch=sch) + sch.send_nowait(rch) + sch.send_nowait(sch) + interp.exec(dedent(""" + rch2 = rch.recv() + sch2 = rch.recv() + assert rch2 == rch + assert sch2 == sch + + sch.send_nowait(rch2) + sch.send_nowait(sch2) + """)) + rch3 = rch.recv() + sch3 = rch.recv() + self.assertEqual(rch2, rch) self.assertEqual(sch2, sch) + self.assertEqual(rch3, rch) + self.assertEqual(sch3, sch) def test_is_closed(self): rch, sch = channels.create() @@ -81,6 +119,12 @@ def test_equality(self): self.assertEqual(ch1, ch1) self.assertNotEqual(ch1, ch2) + def test_pickle(self): + ch, _ = channels.create() + data = pickle.dumps(ch) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + class TestSendChannelAttrs(TestBase): @@ -106,6 +150,12 @@ def test_equality(self): self.assertEqual(ch1, ch1) self.assertNotEqual(ch1, ch2) + def test_pickle(self): + _, ch = channels.create() + data = pickle.dumps(ch) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + class TestSendRecv(TestBase): @@ -120,7 +170,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import channels r, s = channels.create() orig = b'spam' @@ -193,7 +243,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import channels r, s = channels.create() orig = b'spam' @@ -322,6 +372,228 @@ def test_send_buffer_nowait(self): obj[4:8] = b'ham.' self.assertEqual(obj, buf) + def test_send_cleared_with_subinterpreter(self): + def common(rch, sch, unbound=None, presize=0): + if not unbound: + extraargs = '' + elif unbound is channels.UNBOUND: + extraargs = ', unbound=channels.UNBOUND' + elif unbound is channels.UNBOUND_ERROR: + extraargs = ', unbound=channels.UNBOUND_ERROR' + elif unbound is channels.UNBOUND_REMOVE: + extraargs = ', unbound=channels.UNBOUND_REMOVE' + else: + raise NotImplementedError(repr(unbound)) + interp = interpreters.create() + + _run_output(interp, dedent(f""" + from test.support.interpreters import channels + sch = channels.SendChannel({sch.id}) + obj1 = b'spam' + obj2 = b'eggs' + sch.send_nowait(obj1{extraargs}) + sch.send_nowait(obj2{extraargs}) + """)) + self.assertEqual( + _channels.get_count(rch.id), + presize + 2, + ) + + if presize == 0: + obj1 = rch.recv() + self.assertEqual(obj1, b'spam') + self.assertEqual( + _channels.get_count(rch.id), + presize + 1, + ) + + return interp + + with self.subTest('default'): # UNBOUND + rch, sch = channels.create() + interp = common(rch, sch) + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + obj1 = rch.recv() + self.assertEqual(_channels.get_count(rch.id), 0) + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND'): + rch, sch = channels.create() + interp = common(rch, sch, channels.UNBOUND) + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + obj1 = rch.recv() + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND_ERROR'): + rch, sch = channels.create() + interp = common(rch, sch, channels.UNBOUND_ERROR) + + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND_REMOVE'): + rch, sch = channels.create() + + interp = common(rch, sch, channels.UNBOUND_REMOVE) + del interp + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + sch.send_nowait(b'ham', unbound=channels.UNBOUND_REMOVE) + self.assertEqual(_channels.get_count(rch.id), 1) + interp = common(rch, sch, channels.UNBOUND_REMOVE, 1) + self.assertEqual(_channels.get_count(rch.id), 3) + sch.send_nowait(42, unbound=channels.UNBOUND_REMOVE) + self.assertEqual(_channels.get_count(rch.id), 4) + del interp + self.assertEqual(_channels.get_count(rch.id), 2) + obj1 = rch.recv() + obj2 = rch.recv() + self.assertEqual(obj1, b'ham') + self.assertEqual(obj2, 42) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + def test_send_cleared_with_subinterpreter_mixed(self): + rch, sch = channels.create() + interp = interpreters.create() + + # If we don't associate the main interpreter with the channel + # then the channel will be automatically closed when interp + # is destroyed. + sch.send_nowait(None) + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 0) + + _run_output(interp, dedent(f""" + from test.support.interpreters import channels + sch = channels.SendChannel({sch.id}) + sch.send_nowait(1, unbound=channels.UNBOUND) + sch.send_nowait(2, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(3) + sch.send_nowait(4, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(5, unbound=channels.UNBOUND) + """)) + self.assertEqual(_channels.get_count(rch.id), 5) + + del interp + self.assertEqual(_channels.get_count(rch.id), 4) + + obj1 = rch.recv() + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 3) + + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 2) + + obj2 = rch.recv() + self.assertIs(obj2, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 1) + + obj3 = rch.recv() + self.assertIs(obj3, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + + def test_send_cleared_with_subinterpreter_multiple(self): + rch, sch = channels.create() + interp1 = interpreters.create() + interp2 = interpreters.create() + + sch.send_nowait(1) + _run_output(interp1, dedent(f""" + from test.support.interpreters import channels + rch = channels.RecvChannel({rch.id}) + sch = channels.SendChannel({sch.id}) + obj1 = rch.recv() + sch.send_nowait(2, unbound=channels.UNBOUND) + sch.send_nowait(obj1, unbound=channels.UNBOUND_REMOVE) + """)) + _run_output(interp2, dedent(f""" + from test.support.interpreters import channels + rch = channels.RecvChannel({rch.id}) + sch = channels.SendChannel({sch.id}) + obj2 = rch.recv() + obj1 = rch.recv() + """)) + self.assertEqual(_channels.get_count(rch.id), 0) + sch.send_nowait(3) + _run_output(interp1, dedent(""" + sch.send_nowait(4, unbound=channels.UNBOUND) + # interp closed here + sch.send_nowait(5, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(6, unbound=channels.UNBOUND) + """)) + _run_output(interp2, dedent(""" + sch.send_nowait(7, unbound=channels.UNBOUND_ERROR) + # interp closed here + sch.send_nowait(obj1, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(obj2, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(8, unbound=channels.UNBOUND) + """)) + _run_output(interp1, dedent(""" + sch.send_nowait(9, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(10, unbound=channels.UNBOUND) + """)) + self.assertEqual(_channels.get_count(rch.id), 10) + + obj3 = rch.recv() + self.assertEqual(obj3, 3) + self.assertEqual(_channels.get_count(rch.id), 9) + + obj4 = rch.recv() + self.assertEqual(obj4, 4) + self.assertEqual(_channels.get_count(rch.id), 8) + + del interp1 + self.assertEqual(_channels.get_count(rch.id), 6) + + # obj5 was removed + + obj6 = rch.recv() + self.assertIs(obj6, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 5) + + obj7 = rch.recv() + self.assertEqual(obj7, 7) + self.assertEqual(_channels.get_count(rch.id), 4) + + del interp2 + self.assertEqual(_channels.get_count(rch.id), 3) + + # obj1 + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 2) + + # obj2 was removed + + obj8 = rch.recv() + self.assertIs(obj8, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 1) + + # obj9 was removed + + obj10 = rch.recv() + self.assertIs(obj10, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index c2917d839904f9..ac24f6568acd95 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -10,7 +10,7 @@ from test.support import import_helper from test.support import os_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +import_helper.import_module('_interpreters') from .utils import TestBase @@ -124,7 +124,7 @@ def test_sys_path_0(self): orig = sys.path[0] interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import json import sys print(json.dumps({{ @@ -164,6 +164,7 @@ def test_sys_path_0(self): class FinalizationTests(TestBase): + @support.requires_subprocess() def test_gh_109793(self): # Make sure finalization finishes and the correct error code # is reported, even when subinterpreters get cleaned up at the end. diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 2a8ca99c1f6e3f..18f83d097eb360 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -1,25 +1,99 @@ +import importlib +import pickle import threading from textwrap import dedent import unittest -import time -from test.support import import_helper +from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. -_queues = import_helper.import_module('_xxinterpqueues') +_queues = import_helper.import_module('_interpqueues') from test.support import interpreters -from test.support.interpreters import queues -from .utils import _run_output, TestBase +from test.support.interpreters import queues, _crossinterp +from .utils import _run_output, TestBase as _TestBase -class TestBase(TestBase): +REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND] + + +def get_num_queues(): + return len(_queues.list_all()) + + +class TestBase(_TestBase): def tearDown(self): - for qid in _queues.list_all(): + for qid, _, _ in _queues.list_all(): try: _queues.destroy(qid) except Exception: pass +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). + importlib.reload(queues) + + def test_create_destroy(self): + qid = _queues.create(2, 0, REPLACE) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + with self.assertRaises(queues.QueueNotFoundError): + _queues.get(qid) + with self.assertRaises(queues.QueueNotFoundError): + _queues.destroy(qid) + + def test_not_destroyed(self): + # It should have cleaned up any remaining queues. + stdout, stderr = self.assert_python_ok( + '-c', + dedent(f""" + import {_queues.__name__} as _queues + _queues.create(2, 0, {REPLACE}) + """), + ) + self.assertEqual(stdout, '') + if Py_DEBUG: + self.assertNotEqual(stderr, '') + else: + self.assertEqual(stderr, '') + + def test_bind_release(self): + with self.subTest('typical'): + qid = _queues.create(2, 0, REPLACE) + _queues.bind(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('bind too much'): + qid = _queues.create(2, 0, REPLACE) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('nested'): + qid = _queues.create(2, 0, REPLACE) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('release without binding'): + qid = _queues.create(2, 0, REPLACE) + with self.assertRaises(queues.QueueError): + _queues.release(qid) + + class QueueTests(TestBase): def test_create(self): @@ -51,20 +125,20 @@ def test_shareable(self): queue1 = queues.create() interp = interpreters.create() - interp.exec_sync(dedent(f""" + interp.exec(dedent(f""" from test.support.interpreters import queues queue1 = queues.Queue({queue1.id}) """)); with self.subTest('same interpreter'): queue2 = queues.create() - queue1.put(queue2) + queue1.put(queue2, syncobj=True) queue3 = queue1.get() self.assertIs(queue3, queue2) with self.subTest('from current interpreter'): queue4 = queues.create() - queue1.put(queue4) + queue1.put(queue4, syncobj=True) out = _run_output(interp, dedent(""" queue4 = queue1.get() print(queue4.id) @@ -75,7 +149,7 @@ def test_shareable(self): with self.subTest('from subinterpreter'): out = _run_output(interp, dedent(""" queue5 = queues.create() - queue1.put(queue5) + queue1.put(queue5, syncobj=True) print(queue5.id) """)) qid = int(out) @@ -112,13 +186,19 @@ def test_equality(self): self.assertEqual(queue1, queue1) self.assertNotEqual(queue1, queue2) + def test_pickle(self): + queue = queues.create() + data = pickle.dumps(queue) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, queue) + class TestQueueOps(TestBase): def test_empty(self): queue = queues.create() before = queue.empty() - queue.put(None) + queue.put(None, syncobj=True) during = queue.empty() queue.get() after = queue.empty() @@ -133,7 +213,7 @@ def test_full(self): queue = queues.create(3) for _ in range(3): actual.append(queue.full()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.full()) for _ in range(3): queue.get() @@ -147,16 +227,16 @@ def test_qsize(self): queue = queues.create() for _ in range(3): actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) for _ in range(3): queue.get() actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) @@ -165,30 +245,91 @@ def test_qsize(self): def test_put_get_main(self): expected = list(range(20)) - queue = queues.create() - for i in range(20): - queue.put(i) - actual = [queue.get() for _ in range(20)] + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create() + for i in range(20): + queue.put(i, **kwds) + actual = [queue.get() for _ in range(20)] - self.assertEqual(actual, expected) + self.assertEqual(actual, expected) def test_put_timeout(self): - queue = queues.create(2) - queue.put(None) - queue.put(None) - with self.assertRaises(queues.QueueFull): - queue.put(None, timeout=0.1) - queue.get() - queue.put(None) + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put(None, **kwds) + queue.put(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1, **kwds) + queue.get() + queue.put(None, **kwds) def test_put_nowait(self): - queue = queues.create(2) - queue.put_nowait(None) - queue.put_nowait(None) - with self.assertRaises(queues.QueueFull): - queue.put_nowait(None) - queue.get() - queue.put_nowait(None) + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put_nowait(None, **kwds) + queue.put_nowait(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None, **kwds) + queue.get() + queue.put_nowait(None, **kwds) + + def test_put_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + ]: + with self.subTest(repr(obj)): + queue = queues.create() + + queue.put(obj, syncobj=True) + obj2 = queue.get() + self.assertEqual(obj2, obj) + + queue.put(obj, syncobj=True) + obj2 = queue.get_nowait() + self.assertEqual(obj2, obj) + + for obj in [ + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj, syncobj=True) + + def test_put_not_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + # not shareable + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + + queue.put(obj, syncobj=False) + obj2 = queue.get() + self.assertEqual(obj2, obj) + + queue.put(obj, syncobj=False) + obj2 = queue.get_nowait() + self.assertEqual(obj2, obj) def test_get_timeout(self): queue = queues.create() @@ -200,17 +341,54 @@ def test_get_nowait(self): with self.assertRaises(queues.QueueEmpty): queue.get_nowait() + def test_put_get_default_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=True) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + get = getattr(queue, methname) + for i in range(20): + queue.put(i) + actual = [get() for _ in range(20)] + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj) + + def test_put_get_default_not_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=False) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + get = getattr(queue, methname) + + for i in range(20): + queue.put(i) + actual = [get() for _ in range(20)] + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + queue.put(obj) + obj2 = get() + self.assertEqual(obj, obj2) + self.assertIsNot(obj, obj2) + def test_put_get_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import queues queue = queues.create() - orig = b'spam' - queue.put(orig) - obj = queue.get() - assert obj == orig, 'expected: obj == orig' - assert obj is not orig, 'expected: obj is not orig' """)) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + interp.exec(dedent(f""" + orig = b'spam' + queue.put(orig, syncobj=True) + obj = queue.{methname}() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) def test_put_get_different_interpreters(self): interp = interpreters.create() @@ -218,56 +396,239 @@ def test_put_get_different_interpreters(self): queue2 = queues.create() self.assertEqual(len(queues.list_all()), 2) - obj1 = b'spam' - queue1.put(obj1) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + obj1 = b'spam' + queue1.put(obj1, syncobj=True) + + out = _run_output( + interp, + dedent(f""" + from test.support.interpreters import queues + queue1 = queues.Queue({queue1.id}) + queue2 = queues.Queue({queue2.id}) + assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' + obj = queue1.{methname}() + assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0' + assert obj == b'spam', 'expected: obj == obj1' + # When going to another interpreter we get a copy. + assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' + obj2 = b'eggs' + print(id(obj2)) + assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' + queue2.put(obj2, syncobj=True) + assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' + """)) + self.assertEqual(len(queues.list_all()), 2) + self.assertEqual(queue1.qsize(), 0) + self.assertEqual(queue2.qsize(), 1) + + get = getattr(queue2, methname) + obj2 = get() + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) - out = _run_output( - interp, - dedent(f""" + def test_put_cleared_with_subinterpreter(self): + def common(queue, unbound=None, presize=0): + if not unbound: + extraargs = '' + elif unbound is queues.UNBOUND: + extraargs = ', unbound=queues.UNBOUND' + elif unbound is queues.UNBOUND_ERROR: + extraargs = ', unbound=queues.UNBOUND_ERROR' + elif unbound is queues.UNBOUND_REMOVE: + extraargs = ', unbound=queues.UNBOUND_REMOVE' + else: + raise NotImplementedError(repr(unbound)) + interp = interpreters.create() + + _run_output(interp, dedent(f""" from test.support.interpreters import queues - queue1 = queues.Queue({queue1.id}) - queue2 = queues.Queue({queue2.id}) - assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' - obj = queue1.get() - assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0' - assert obj == b'spam', 'expected: obj == obj1' - # When going to another interpreter we get a copy. - assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' + queue = queues.Queue({queue.id}) + obj1 = b'spam' obj2 = b'eggs' - print(id(obj2)) - assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' - queue2.put(obj2) - assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' + queue.put(obj1, syncobj=True{extraargs}) + queue.put(obj2, syncobj=True{extraargs}) """)) - self.assertEqual(len(queues.list_all()), 2) - self.assertEqual(queue1.qsize(), 0) - self.assertEqual(queue2.qsize(), 1) + self.assertEqual(queue.qsize(), presize + 2) - obj2 = queue2.get() - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) + if presize == 0: + obj1 = queue.get() + self.assertEqual(obj1, b'spam') + self.assertEqual(queue.qsize(), presize + 1) - def test_put_cleared_with_subinterpreter(self): + return interp + + with self.subTest('default'): # UNBOUND + queue = queues.create() + interp = common(queue) + del interp + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND'): + queue = queues.create() + interp = common(queue, queues.UNBOUND) + del interp + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND_ERROR'): + queue = queues.create() + interp = common(queue, queues.UNBOUND_ERROR) + + del interp + self.assertEqual(queue.qsize(), 1) + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND_REMOVE'): + queue = queues.create() + + interp = common(queue, queues.UNBOUND_REMOVE) + del interp + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + queue.put(b'ham', unbound=queues.UNBOUND_REMOVE) + self.assertEqual(queue.qsize(), 1) + interp = common(queue, queues.UNBOUND_REMOVE, 1) + self.assertEqual(queue.qsize(), 3) + queue.put(42, unbound=queues.UNBOUND_REMOVE) + self.assertEqual(queue.qsize(), 4) + del interp + self.assertEqual(queue.qsize(), 2) + obj1 = queue.get() + obj2 = queue.get() + self.assertEqual(obj1, b'ham') + self.assertEqual(obj2, 42) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + def test_put_cleared_with_subinterpreter_mixed(self): + queue = queues.create() interp = interpreters.create() + _run_output(interp, dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + queue.put(1, syncobj=True, unbound=queues.UNBOUND) + queue.put(2, syncobj=True, unbound=queues.UNBOUND_ERROR) + queue.put(3, syncobj=True) + queue.put(4, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(5, syncobj=True, unbound=queues.UNBOUND) + """)) + self.assertEqual(queue.qsize(), 5) + + del interp + self.assertEqual(queue.qsize(), 4) + + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 3) + + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + self.assertEqual(queue.qsize(), 2) + + obj2 = queue.get() + self.assertIs(obj2, queues.UNBOUND) + self.assertEqual(queue.qsize(), 1) + + obj3 = queue.get() + self.assertIs(obj3, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + + def test_put_cleared_with_subinterpreter_multiple(self): queue = queues.create() + interp1 = interpreters.create() + interp2 = interpreters.create() - out = _run_output( - interp, - dedent(f""" - from test.support.interpreters import queues - queue = queues.Queue({queue.id}) - obj1 = b'spam' - obj2 = b'eggs' - queue.put(obj1) - queue.put(obj2) - """)) + queue.put(1, syncobj=True) + _run_output(interp1, dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + obj1 = queue.get() + queue.put(2, syncobj=True, unbound=queues.UNBOUND) + queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_REMOVE) + """)) + _run_output(interp2, dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + obj2 = queue.get() + obj1 = queue.get() + """)) + self.assertEqual(queue.qsize(), 0) + queue.put(3) + _run_output(interp1, dedent(""" + queue.put(4, syncobj=True, unbound=queues.UNBOUND) + # interp closed here + queue.put(5, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(6, syncobj=True, unbound=queues.UNBOUND) + """)) + _run_output(interp2, dedent(""" + queue.put(7, syncobj=True, unbound=queues.UNBOUND_ERROR) + # interp closed here + queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_ERROR) + queue.put(obj2, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(8, syncobj=True, unbound=queues.UNBOUND) + """)) + _run_output(interp1, dedent(""" + queue.put(9, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(10, syncobj=True, unbound=queues.UNBOUND) + """)) + self.assertEqual(queue.qsize(), 10) + + obj3 = queue.get() + self.assertEqual(obj3, 3) + self.assertEqual(queue.qsize(), 9) + + obj4 = queue.get() + self.assertEqual(obj4, 4) + self.assertEqual(queue.qsize(), 8) + + del interp1 + self.assertEqual(queue.qsize(), 6) + + # obj5 was removed + + obj6 = queue.get() + self.assertIs(obj6, queues.UNBOUND) + self.assertEqual(queue.qsize(), 5) + + obj7 = queue.get() + self.assertEqual(obj7, 7) + self.assertEqual(queue.qsize(), 4) + + del interp2 + self.assertEqual(queue.qsize(), 3) + + # obj1 + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() self.assertEqual(queue.qsize(), 2) - obj1 = queue.get() - self.assertEqual(obj1, b'spam') + # obj2 was removed + + obj8 = queue.get() + self.assertIs(obj8, queues.UNBOUND) self.assertEqual(queue.qsize(), 1) - del interp + # obj9 was removed + + obj10 = queue.get() + self.assertIs(obj10, queues.UNBOUND) self.assertEqual(queue.qsize(), 0) def test_put_get_different_threads(self): @@ -281,12 +642,12 @@ def f(): break except queues.QueueEmpty: continue - queue2.put(obj) + queue2.put(obj, syncobj=True) t = threading.Thread(target=f) t.start() orig = b'spam' - queue1.put(orig) + queue1.put(orig, syncobj=True) obj = queue2.get() t.join() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index 3cc570b3bf7128..e400535b2a0e4e 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -5,7 +5,7 @@ from test.support import import_helper from test.support import threading_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +import_helper.import_module('_interpreters') from test.support import interpreters from .utils import TestBase diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 3a37ed09dd8943..3cab76d0f279e0 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -1,29 +1,344 @@ +from collections import namedtuple import contextlib +import json import os import os.path +#import select import subprocess import sys import tempfile -import threading from textwrap import dedent +import threading +import types import unittest +import warnings from test import support -from test.support import os_helper +# We would use test.support.import_helper.import_module(), +# but the indirect import of test.support.os_helper causes refleaks. +try: + import _interpreters +except ImportError as exc: + raise unittest.SkipTest(str(exc)) from test.support import interpreters -def _captured_script(script): - r, w = os.pipe() - indented = script.replace('\n', '\n ') - wrapped = dedent(f""" - import contextlib - with open({w}, 'w', encoding='utf-8') as spipe: - with contextlib.redirect_stdout(spipe): +try: + import _testinternalcapi + import _testcapi +except ImportError: + _testinternalcapi = None + _testcapi = None + +def requires_test_modules(func): + return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) + + +def _dump_script(text): + lines = text.splitlines() + print() + print('-' * 20) + for i, line in enumerate(lines, 1): + print(f' {i:>{len(str(len(lines)))}} {line}') + print('-' * 20) + + +def _close_file(file): + try: + if hasattr(file, 'close'): + file.close() + else: + os.close(file) + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + + +def pack_exception(exc=None): + captured = _interpreters.capture_exception(exc) + data = dict(captured.__dict__) + data['type'] = dict(captured.type.__dict__) + return json.dumps(data) + + +def unpack_exception(packed): + try: + data = json.loads(packed) + except json.decoder.JSONDecodeError: + warnings.warn('incomplete exception data', RuntimeWarning) + print(packed if isinstance(packed, str) else packed.decode('utf-8')) + return None + exc = types.SimpleNamespace(**data) + exc.type = types.SimpleNamespace(**exc.type) + return exc; + + +class CapturingResults: + + STDIO = dedent("""\ + with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}: + _captured_std{stream} = io.StringIO() + with contextlib.redirect_std{stream}(_captured_std{stream}): + ######################### + # begin wrapped script + {indented} - """) - return wrapped, open(r, encoding='utf-8') + + # end wrapped script + ######################### + text = _captured_std{stream}.getvalue() + _spipe_{stream}.write(text.encode('utf-8')) + """)[:-1] + EXC = dedent("""\ + with open({w_pipe}, 'wb', buffering=0) as _spipe_exc: + try: + ######################### + # begin wrapped script + + {indented} + + # end wrapped script + ######################### + except Exception as exc: + text = _interp_utils.pack_exception(exc) + _spipe_exc.write(text.encode('utf-8')) + """)[:-1] + + @classmethod + def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False): + script = dedent(script).strip(os.linesep) + imports = [ + f'import {__name__} as _interp_utils', + ] + wrapped = script + + # Handle exc. + if exc: + exc = os.pipe() + r_exc, w_exc = exc + indented = wrapped.replace('\n', '\n ') + wrapped = cls.EXC.format( + w_pipe=w_exc, + indented=indented, + ) + else: + exc = None + + # Handle stdout. + if stdout: + imports.extend([ + 'import contextlib, io', + ]) + stdout = os.pipe() + r_out, w_out = stdout + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_out, + indented=indented, + stream='out', + ) + else: + stdout = None + + # Handle stderr. + if stderr == 'stdout': + stderr = None + elif stderr: + if not stdout: + imports.extend([ + 'import contextlib, io', + ]) + stderr = os.pipe() + r_err, w_err = stderr + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_err, + indented=indented, + stream='err', + ) + else: + stderr = None + + if wrapped == script: + raise NotImplementedError + else: + for line in imports: + wrapped = f'{line}{os.linesep}{wrapped}' + + results = cls(stdout, stderr, exc) + return wrapped, results + + def __init__(self, out, err, exc): + self._rf_out = None + self._rf_err = None + self._rf_exc = None + self._w_out = None + self._w_err = None + self._w_exc = None + + if out is not None: + r_out, w_out = out + self._rf_out = open(r_out, 'rb', buffering=0) + self._w_out = w_out + + if err is not None: + r_err, w_err = err + self._rf_err = open(r_err, 'rb', buffering=0) + self._w_err = w_err + + if exc is not None: + r_exc, w_exc = exc + self._rf_exc = open(r_exc, 'rb', buffering=0) + self._w_exc = w_exc + + self._buf_out = b'' + self._buf_err = b'' + self._buf_exc = b'' + self._exc = None + + self._closed = False + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + @property + def closed(self): + return self._closed + + def close(self): + if self._closed: + return + self._closed = True + + if self._w_out is not None: + _close_file(self._w_out) + self._w_out = None + if self._w_err is not None: + _close_file(self._w_err) + self._w_err = None + if self._w_exc is not None: + _close_file(self._w_exc) + self._w_exc = None + + self._capture() + + if self._rf_out is not None: + _close_file(self._rf_out) + self._rf_out = None + if self._rf_err is not None: + _close_file(self._rf_err) + self._rf_err = None + if self._rf_exc is not None: + _close_file(self._rf_exc) + self._rf_exc = None + + def _capture(self): + # Ideally this is called only after the script finishes + # (and thus has closed the write end of the pipe. + if self._rf_out is not None: + chunk = self._rf_out.read(100) + while chunk: + self._buf_out += chunk + chunk = self._rf_out.read(100) + if self._rf_err is not None: + chunk = self._rf_err.read(100) + while chunk: + self._buf_err += chunk + chunk = self._rf_err.read(100) + if self._rf_exc is not None: + chunk = self._rf_exc.read(100) + while chunk: + self._buf_exc += chunk + chunk = self._rf_exc.read(100) + + def _unpack_stdout(self): + return self._buf_out.decode('utf-8') + + def _unpack_stderr(self): + return self._buf_err.decode('utf-8') + + def _unpack_exc(self): + if self._exc is not None: + return self._exc + if not self._buf_exc: + return None + self._exc = unpack_exception(self._buf_exc) + return self._exc + + def stdout(self): + if self.closed: + return self.final().stdout + self._capture() + return self._unpack_stdout() + + def stderr(self): + if self.closed: + return self.final().stderr + self._capture() + return self._unpack_stderr() + + def exc(self): + if self.closed: + return self.final().exc + self._capture() + return self._unpack_exc() + + def final(self, *, force=False): + try: + return self._final + except AttributeError: + if not self._closed: + if not force: + raise Exception('no final results available yet') + else: + return CapturedResults.Proxy(self) + self._final = CapturedResults( + self._unpack_stdout(), + self._unpack_stderr(), + self._unpack_exc(), + ) + return self._final + + +class CapturedResults(namedtuple('CapturedResults', 'stdout stderr exc')): + + class Proxy: + def __init__(self, capturing): + self._capturing = capturing + def _finish(self): + if self._capturing is None: + return + self._final = self._capturing.final() + self._capturing = None + def __iter__(self): + self._finish() + yield from self._final + def __len__(self): + self._finish() + return len(self._final) + def __getattr__(self, name): + self._finish() + if name.startswith('_'): + raise AttributeError(name) + return getattr(self._final, name) + + def raise_if_failed(self): + if self.exc is not None: + raise interpreters.ExecutionFailed(self.exc) + + +def _captured_script(script, *, stdout=True, stderr=False, exc=False): + return CapturingResults.wrap_script( + script, + stdout=stdout, + stderr=stderr, + exc=exc, + ) def clean_up_interpreters(): @@ -32,24 +347,24 @@ def clean_up_interpreters(): continue try: interp.close() - except RuntimeError: + except _interpreters.InterpreterError: pass # already destroyed def _run_output(interp, request, init=None): - script, rpipe = _captured_script(request) - with rpipe: + script, results = _captured_script(request) + with results: if init: interp.prepare_main(init) - interp.exec_sync(script) - return rpipe.read() + interp.exec(script) + return results.stdout() @contextlib.contextmanager def _running(interp): r, w = os.pipe() def run(): - interp.exec_sync(dedent(f""" + interp.exec(dedent(f""" # wait for "signal" with open({r}) as rpipe: rpipe.read() @@ -67,6 +382,9 @@ def run(): class TestBase(unittest.TestCase): + def tearDown(self): + clean_up_interpreters() + def pipe(self): def ensure_closed(fd): try: @@ -81,9 +399,22 @@ def ensure_closed(fd): def temp_dir(self): tempdir = tempfile.mkdtemp() tempdir = os.path.realpath(tempdir) + from test.support import os_helper self.addCleanup(lambda: os_helper.rmtree(tempdir)) return tempdir + @contextlib.contextmanager + def captured_thread_exception(self): + ctx = types.SimpleNamespace(caught=None) + def excepthook(args): + ctx.caught = args + orig_excepthook = threading.excepthook + threading.excepthook = excepthook + try: + yield ctx + finally: + threading.excepthook = orig_excepthook + def make_script(self, filename, dirname=None, text=None): if text: text = dedent(text) @@ -143,5 +474,213 @@ def assert_python_failure(self, *argv): self.assertNotEqual(exitcode, 0) return stdout, stderr - def tearDown(self): - clean_up_interpreters() + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def _run_string(self, interp, script): + wrapped, results = _captured_script(script, exc=False) + #_dump_script(wrapped) + with results: + if isinstance(interp, interpreters.Interpreter): + interp.exec(script) + else: + err = _interpreters.run_string(interp, wrapped) + if err is not None: + return None, err + return results.stdout(), None + + def run_and_capture(self, interp, script): + text, err = self._run_string(interp, script) + if err is not None: + raise interpreters.ExecutionFailed(err) + else: + return text + + def interp_exists(self, interpid): + try: + _interpreters.whence(interpid) + except _interpreters.InterpreterNotFoundError: + return False + else: + return True + + @requires_test_modules + @contextlib.contextmanager + def interpreter_from_capi(self, config=None, whence=None): + if config is False: + if whence is None: + whence = _interpreters.WHENCE_LEGACY_CAPI + else: + assert whence in (_interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN), repr(whence) + config = None + elif config is True: + config = _interpreters.new_config('default') + elif config is None: + if whence not in ( + _interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN, + ): + config = _interpreters.new_config('legacy') + elif isinstance(config, str): + config = _interpreters.new_config(config) + + if whence is None: + whence = _interpreters.WHENCE_XI + + interpid = _testinternalcapi.create_interpreter(config, whence=whence) + try: + yield interpid + finally: + try: + _testinternalcapi.destroy_interpreter(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + @contextlib.contextmanager + def interpreter_obj_from_capi(self, config='legacy'): + with self.interpreter_from_capi(config) as interpid: + interp = interpreters.Interpreter( + interpid, + _whence=_interpreters.WHENCE_CAPI, + _ownsref=False, + ) + yield interp, interpid + + @contextlib.contextmanager + def capturing(self, script): + wrapped, capturing = _captured_script(script, stdout=True, exc=True) + #_dump_script(wrapped) + with capturing: + yield wrapped, capturing.final(force=True) + + @requires_test_modules + def run_from_capi(self, interpid, script, *, main=False): + with self.capturing(script) as (wrapped, results): + rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main) + assert rc == 0, rc + results.raise_if_failed() + return results.stdout + + @contextlib.contextmanager + def _running(self, run_interp, exec_interp): + token = b'\0' + r_in, w_in = self.pipe() + r_out, w_out = self.pipe() + + def close(): + _close_file(r_in) + _close_file(w_in) + _close_file(r_out) + _close_file(w_out) + + # Start running (and wait). + script = dedent(f""" + import os + try: + # handshake + token = os.read({r_in}, 1) + os.write({w_out}, token) + # Wait for the "done" message. + os.read({r_in}, 1) + except BrokenPipeError: + pass + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + """) + failed = None + def run(): + nonlocal failed + try: + run_interp(script) + except Exception as exc: + failed = exc + close() + t = threading.Thread(target=run) + t.start() + + # handshake + try: + os.write(w_in, token) + token2 = os.read(r_out, 1) + assert token2 == token, (token2, token) + except OSError: + t.join() + if failed is not None: + raise failed + + # CM __exit__() + try: + try: + yield + finally: + # Send "done". + os.write(w_in, b'\0') + finally: + close() + t.join() + if failed is not None: + raise failed + + @contextlib.contextmanager + def running(self, interp): + if isinstance(interp, int): + interpid = interp + def exec_interp(script): + exc = _interpreters.exec(interpid, script) + assert exc is None, exc + run_interp = exec_interp + else: + def run_interp(script): + text = self.run_and_capture(interp, script) + assert text == '', repr(text) + def exec_interp(script): + interp.exec(script) + with self._running(run_interp, exec_interp): + yield + + @requires_test_modules + @contextlib.contextmanager + def running_from_capi(self, interpid, *, main=False): + def run_interp(script): + text = self.run_from_capi(interpid, script, main=main) + assert text == '', repr(text) + def exec_interp(script): + rc = _testinternalcapi.exec_interpreter(interpid, script) + assert rc == 0, rc + with self._running(run_interp, exec_interp): + yield + + @requires_test_modules + def run_temp_from_capi(self, script, config='legacy'): + if config is False: + # Force using Py_NewInterpreter(). + run_in_interp = (lambda s, c: _testcapi.run_in_subinterp(s)) + config = None + else: + run_in_interp = _testinternalcapi.run_in_subinterp_with_config + if config is True: + config = 'default' + if isinstance(config, str): + config = _interpreters.new_config(config) + with self.capturing(script) as (wrapped, results): + rc = run_in_interp(wrapped, config) + assert rc == 0, rc + results.raise_if_failed() + return results.stdout diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index cc387afa391909..aa1b8268592ff7 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -40,7 +40,7 @@ from test.support.script_helper import ( assert_python_ok, assert_python_failure, run_python_until_end) from test.support import ( - import_helper, is_apple, os_helper, skip_if_sanitizer, threading_helper, warnings_helper + import_helper, is_apple, os_helper, threading_helper, warnings_helper, ) from test.support.os_helper import FakePath @@ -257,6 +257,27 @@ class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO): UnsupportedOperation = pyio.UnsupportedOperation +class MockCharPseudoDevFileIO(MockFileIO): + # GH-95782 + # ftruncate() does not work on these special files (and CPython then raises + # appropriate exceptions), so truncate() does not have to be accounted for + # here. + def __init__(self, data): + super().__init__(data) + + def seek(self, *args): + return 0 + + def tell(self, *args): + return 0 + +class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO): + pass + +class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO): + pass + + class MockNonBlockWriterIO: def __init__(self): @@ -618,11 +639,9 @@ def test_large_file_ops(self): def test_with_open(self): for bufsize in (0, 100): - f = None with self.open(os_helper.TESTFN, "wb", bufsize) as f: f.write(b"xxx") self.assertEqual(f.closed, True) - f = None try: with self.open(os_helper.TESTFN, "wb", bufsize) as f: 1/0 @@ -1139,7 +1158,7 @@ class APIMismatchTest(unittest.TestCase): def test_RawIOBase_io_in_pyio_match(self): """Test that pyio RawIOBase class has all c RawIOBase methods""" mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__',)) + ignore=('__weakref__', '__static_attributes__')) self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') def test_RawIOBase_pyio_in_io_match(self): @@ -1648,23 +1667,34 @@ def test_truncate_on_read_only(self): self.assertRaises(self.UnsupportedOperation, bufio.truncate) self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0) + def test_tell_character_device_file(self): + # GH-95782 + # For the (former) bug in BufferedIO to manifest, the wrapped IO obj + # must be able to produce at least 2 bytes. + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.read(1), b"1") + self.assertEqual(buf.tell(), 0) + + def test_seek_character_device_file(self): + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.seek(1, io.SEEK_SET), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.read(1), b"1") + + # In the C implementation, tell() sets the BufferedIO's abs_pos to 0, + # which means that the next seek() could return a negative offset if it + # does not sanity-check: + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedReaderTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO([b"abc"]) bufio = self.tp(rawio) @@ -2020,19 +2050,6 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedWriterTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO() bufio = self.tp(rawio) @@ -2542,19 +2559,6 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedRandomTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_garbage_collection(self): CBufferedReaderTest.test_garbage_collection(self) CBufferedWriterTest.test_garbage_collection(self) @@ -4010,6 +4014,28 @@ def write(self, data): t.write("x"*chunk_size) self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) + def test_issue119506(self): + chunk_size = 8192 + + class MockIO(self.MockRawIO): + written = False + def write(self, data): + if not self.written: + self.written = True + t.write("middle") + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf) + t.write("abc") + t.write("def") + # writing data which size >= chunk_size cause flushing buffer before write. + t.write("g" * chunk_size) + t.flush() + + self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], + buf._write_stack) + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio @@ -4880,7 +4906,7 @@ def load_tests(loader, tests, pattern): # classes in the __dict__ of each test. mocks = (MockRawIO, MisbehavedRawIO, MockFileIO, CloseFailureIO, MockNonBlockWriterIO, MockUnseekableIO, MockRawIOWithoutRead, - SlowFlushRawIO) + SlowFlushRawIO, MockCharPseudoDevFileIO) all_members = io.__all__ c_io_ns = {name : getattr(io, name) for name in all_members} py_io_ns = {name : getattr(pyio, name) for name in all_members} diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py index 7b7067eb7b61d4..04934dfa16a5f0 100644 --- a/Lib/test/test_ioctl.py +++ b/Lib/test/test_ioctl.py @@ -66,23 +66,15 @@ def test_ioctl_mutate_2048(self): # Test with a larger buffer, just for the record. self._check_ioctl_mutate_len(2048) - def test_ioctl_signed_unsigned_code_param(self): - if not pty: - raise unittest.SkipTest('pty module required') + @unittest.skipIf(pty is None, 'pty module required') + def test_ioctl_set_window_size(self): mfd, sfd = pty.openpty() try: - if termios.TIOCSWINSZ < 0: - set_winsz_opcode_maybe_neg = termios.TIOCSWINSZ - set_winsz_opcode_pos = termios.TIOCSWINSZ & 0xffffffff - else: - set_winsz_opcode_pos = termios.TIOCSWINSZ - set_winsz_opcode_maybe_neg, = struct.unpack("i", - struct.pack("I", termios.TIOCSWINSZ)) - - our_winsz = struct.pack("HHHH",80,25,0,0) - # test both with a positive and potentially negative ioctl code - new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_pos, our_winsz) - new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_maybe_neg, our_winsz) + # (rows, columns, xpixel, ypixel) + our_winsz = struct.pack("HHHH", 20, 40, 0, 0) + result = fcntl.ioctl(mfd, termios.TIOCSWINSZ, our_winsz) + new_winsz = struct.unpack("HHHH", result) + self.assertEqual(new_winsz[:2], (20, 40)) finally: os.close(mfd) os.close(sfd) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index b4952acc2b61b1..b1ac2b94f41b38 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2189,11 +2189,17 @@ def testIPv6AddressTooLarge(self): ipaddress.ip_address('FFFF::c000:201%scope')) def testIPVersion(self): + self.assertEqual(ipaddress.IPv4Address.version, 4) + self.assertEqual(ipaddress.IPv6Address.version, 6) + self.assertEqual(self.ipv4_address.version, 4) self.assertEqual(self.ipv6_address.version, 6) self.assertEqual(self.ipv6_scoped_address.version, 6) def testMaxPrefixLength(self): + self.assertEqual(ipaddress.IPv4Address.max_prefixlen, 32) + self.assertEqual(ipaddress.IPv6Address.max_prefixlen, 128) + self.assertEqual(self.ipv4_interface.max_prefixlen, 32) self.assertEqual(self.ipv6_interface.max_prefixlen, 128) self.assertEqual(self.ipv6_scoped_interface.max_prefixlen, 128) @@ -2288,6 +2294,10 @@ def testReservedIpv4(self): self.assertEqual(True, ipaddress.ip_address( '172.31.255.255').is_private) self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private) + self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global) + self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global) self.assertEqual(True, ipaddress.ip_address('169.254.100.200').is_link_local) @@ -2313,6 +2323,7 @@ def testPrivateNetworks(self): self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) + self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) @@ -2329,8 +2340,8 @@ def testPrivateNetworks(self): self.assertEqual(True, ipaddress.ip_network("::/128").is_private) self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) - self.assertEqual(True, ipaddress.ip_network("2001::/23").is_private) self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) + self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private) self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) @@ -2409,6 +2420,22 @@ def testReservedIpv6(self): self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified) self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified) + self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global) + self.assertFalse(ipaddress.ip_address('2001::').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::1').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::2').is_global) + self.assertFalse(ipaddress.ip_address('2001:2::').is_global) + self.assertTrue(ipaddress.ip_address('2001:3::').is_global) + self.assertFalse(ipaddress.ip_address('2001:4::').is_global) + self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global) + self.assertFalse(ipaddress.ip_address('2001:10::').is_global) + self.assertTrue(ipaddress.ip_address('2001:20::').is_global) + self.assertTrue(ipaddress.ip_address('2001:30::').is_global) + self.assertFalse(ipaddress.ip_address('2001:40::').is_global) + self.assertFalse(ipaddress.ip_address('2002::').is_global) + # gh-124217: conform with RFC 9637 + self.assertFalse(ipaddress.ip_address('3fff::').is_global) + # some generic IETF reserved addresses self.assertEqual(True, ipaddress.ip_address('100::').is_reserved) self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved) @@ -2421,12 +2448,52 @@ def testIpv4Mapped(self): self.assertEqual(ipaddress.ip_address('::ffff:c0a8:101').ipv4_mapped, ipaddress.ip_address('192.168.1.1')) + def testIpv4MappedProperties(self): + # Test that an IPv4 mapped IPv6 address has + # the same properties as an IPv4 address. + for addr4 in ( + "178.62.3.251", # global + "169.254.169.254", # link local + "127.0.0.1", # loopback + "224.0.0.1", # multicast + "192.168.0.1", # private + "0.0.0.0", # unspecified + "100.64.0.1", # public and not global + ): + with self.subTest(addr4): + ipv4 = ipaddress.IPv4Address(addr4) + ipv6 = ipaddress.IPv6Address(f"::ffff:{addr4}") + + self.assertEqual(ipv4.is_global, ipv6.is_global) + self.assertEqual(ipv4.is_private, ipv6.is_private) + self.assertEqual(ipv4.is_reserved, ipv6.is_reserved) + self.assertEqual(ipv4.is_multicast, ipv6.is_multicast) + self.assertEqual(ipv4.is_unspecified, ipv6.is_unspecified) + self.assertEqual(ipv4.is_link_local, ipv6.is_link_local) + self.assertEqual(ipv4.is_loopback, ipv6.is_loopback) + def testIpv4MappedPrivateCheck(self): self.assertEqual( True, ipaddress.ip_address('::ffff:192.168.1.1').is_private) self.assertEqual( False, ipaddress.ip_address('::ffff:172.32.0.0').is_private) + def testIpv4MappedLoopbackCheck(self): + # test networks + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.100.200.254/128').is_loopback) + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.42.0.0/112').is_loopback) + self.assertEqual(False, ipaddress.ip_network( + '::ffff:128.0.0.0').is_loopback) + # test addresses + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.100.200.254').is_loopback) + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.42.0.0').is_loopback) + self.assertEqual(False, ipaddress.ip_address( + '::ffff:128.0.0.0').is_loopback) + def testAddrExclude(self): addr1 = ipaddress.ip_network('10.1.1.0/24') addr2 = ipaddress.ip_network('10.1.1.0/26') @@ -2550,12 +2617,42 @@ def testExplodeShortHandIpStr(self): self.assertEqual('192.168.178.1', addr4.exploded) def testReversePointer(self): - addr1 = ipaddress.IPv4Address('127.0.0.1') - addr2 = ipaddress.IPv6Address('2001:db8::1') - self.assertEqual('1.0.0.127.in-addr.arpa', addr1.reverse_pointer) - self.assertEqual('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.' + - 'b.d.0.1.0.0.2.ip6.arpa', - addr2.reverse_pointer) + for addr_v4, expected in [ + ('127.0.0.1', '1.0.0.127.in-addr.arpa'), + # test vector: https://www.rfc-editor.org/rfc/rfc1035, §3.5 + ('10.2.0.52', '52.0.2.10.in-addr.arpa'), + ]: + with self.subTest('ipv4_reverse_pointer', addr=addr_v4): + addr = ipaddress.IPv4Address(addr_v4) + self.assertEqual(addr.reverse_pointer, expected) + + for addr_v6, expected in [ + ( + '2001:db8::1', ( + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.' + '0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.' + 'ip6.arpa' + ) + ), + ( + '::FFFF:192.168.1.35', ( + '3.2.1.0.8.a.0.c.f.f.f.f.0.0.0.0.' + '0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.' + 'ip6.arpa' + ) + ), + # test vector: https://www.rfc-editor.org/rfc/rfc3596, §2.5 + ( + '4321:0:1:2:3:4:567:89ab', ( + 'b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.' + '2.0.0.0.1.0.0.0.0.0.0.0.1.2.3.4.' + 'ip6.arpa' + ) + ) + ]: + with self.subTest('ipv6_reverse_pointer', addr=addr_v6): + addr = ipaddress.IPv6Address(addr_v6) + self.assertEqual(addr.reverse_pointer, expected) def testIntRepresentation(self): self.assertEqual(16909060, int(self.ipv4_address)) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 7f759fb3317146..95a119ba683e09 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - for cnt in range(support.EXCEEDS_RECURSION_LIMIT): + for cnt in range(support.exceeds_recursion_limit()): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 30aedb0db3bb3d..1b9f3cf76240ad 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -5,11 +5,13 @@ from test.support import cpython_only from test.support.os_helper import TESTFN, unlink from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ +from test.support import BrokenIter import pickle import collections.abc import functools import contextlib import builtins +import traceback # Test result of triple loop (too big to inline) TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2), @@ -302,7 +304,7 @@ def __eq__(self, other): # listiter_reduce_general self.assertEqual( run("reversed", orig["reversed"](list(range(8)))), - (iter, ([],)) + (reversed, ([],)) ) for case in types: @@ -1143,6 +1145,46 @@ def test_error_iter(self): self.assertRaises(TypeError, iter, typ()) self.assertRaises(ZeroDivisionError, iter, BadIterableClass()) + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + def init_raises(): + try: + for x in BrokenIter(init_raises=True): + pass + except Exception as e: + return e + + def next_raises(): + try: + for x in BrokenIter(next_raises=True): + pass + except Exception as e: + return e + + def iter_raises(): + try: + for x in BrokenIter(iter_raises=True): + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 9af0730ea98004..8469de998ba014 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1,7 +1,7 @@ import doctest import unittest from test import support -from test.support import threading_helper +from test.support import threading_helper, script_helper from itertools import * import weakref from decimal import Decimal @@ -15,26 +15,6 @@ import struct import threading import gc -import warnings - -def pickle_deprecated(testfunc): - """ Run the test three times. - First, verify that a Deprecation Warning is raised. - Second, run normally but with DeprecationWarnings temporarily disabled. - Third, run with warnings promoted to errors. - """ - def inner(self): - with self.assertWarns(DeprecationWarning): - testfunc(self) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - testfunc(self) - with warnings.catch_warnings(): - warnings.simplefilter("error", category=DeprecationWarning) - with self.assertRaises((DeprecationWarning, AssertionError, SystemError)): - testfunc(self) - - return inner maxsize = support.MAX_Py_ssize_t minsize = -maxsize-1 @@ -144,7 +124,6 @@ def expand(it, i=0): c = expand(compare[took:]) self.assertEqual(a, c); - @pickle_deprecated def test_accumulate(self): self.assertEqual(list(accumulate(range(10))), # one positional arg [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]) @@ -171,9 +150,6 @@ def test_accumulate(self): [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0]) with self.assertRaises(TypeError): list(accumulate(s, chr)) # unary-operation - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, accumulate(range(10))) # test pickling - self.pickletest(proto, accumulate(range(10), initial=7)) self.assertEqual(list(accumulate([10, 5, 1], initial=None)), [10, 15, 16]) self.assertEqual(list(accumulate([10, 5, 1], initial=100)), [100, 110, 115, 116]) self.assertEqual(list(accumulate([], initial=100)), [100]) @@ -245,58 +221,12 @@ def test_chain_from_iterable(self): self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) - @pickle_deprecated - def test_chain_reducible(self): - for oper in [copy.deepcopy] + picklecopiers: - it = chain('abc', 'def') - self.assertEqual(list(oper(it)), list('abcdef')) - self.assertEqual(next(it), 'a') - self.assertEqual(list(oper(it)), list('bcdef')) - - self.assertEqual(list(oper(chain(''))), []) - self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd')) - self.assertRaises(TypeError, list, oper(chain(2, 3))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef')) - - @pickle_deprecated - def test_chain_setstate(self): - self.assertRaises(TypeError, chain().__setstate__, ()) - self.assertRaises(TypeError, chain().__setstate__, []) - self.assertRaises(TypeError, chain().__setstate__, 0) - self.assertRaises(TypeError, chain().__setstate__, ([],)) - self.assertRaises(TypeError, chain().__setstate__, (iter([]), [])) - it = chain() - it.__setstate__((iter(['abc', 'def']),)) - self.assertEqual(list(it), ['a', 'b', 'c', 'd', 'e', 'f']) - it = chain() - it.__setstate__((iter(['abc', 'def']), iter(['ghi']))) - self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f']) - - @pickle_deprecated def test_combinations(self): self.assertRaises(TypeError, combinations, 'abc') # missing r argument self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments self.assertRaises(TypeError, combinations, None) # pool is not iterable self.assertRaises(ValueError, combinations, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(combinations('abc', 32))), []) # r > n - - self.assertEqual(list(op(combinations('ABCD', 2))), - [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - testIntermediate = combinations('ABCD', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - - self.assertEqual(list(op(combinations(range(4), 3))), - [(0,1,2), (0,1,3), (0,2,3), (1,2,3)]) - testIntermediate = combinations(range(4), 3) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [(0,1,3), (0,2,3), (1,2,3)]) - def combinations1(iterable, r): 'Pure python version shown in the docs' pool = tuple(iterable) @@ -350,9 +280,6 @@ def combinations3(iterable, r): self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, combinations(values, r)) # test pickling - @support.bigaddrspacetest def test_combinations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -364,7 +291,6 @@ def test_combinations_tuple_reuse(self): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) - @pickle_deprecated def test_combinations_with_replacement(self): cwr = combinations_with_replacement self.assertRaises(TypeError, cwr, 'abc') # missing r argument @@ -372,15 +298,6 @@ def test_combinations_with_replacement(self): self.assertRaises(TypeError, cwr, None) # pool is not iterable self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(cwr('ABC', 2))), - [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - testIntermediate = cwr('ABC', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - - def cwr1(iterable, r): 'Pure python version shown in the docs' # number items returned: (n+r-1)! / r! / (n-1)! when n>0 @@ -438,22 +355,18 @@ def numcombs(n, r): self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cwr(values,r)) # test pickling - @support.bigaddrspacetest def test_combinations_with_replacement_overflow(self): with self.assertRaises((OverflowError, MemoryError)): combinations_with_replacement("AA", 2**30) - # Test implementation detail: tuple re-use + # Test implementation detail: tuple re-use @support.impl_detail("tuple reuse is specific to CPython") def test_combinations_with_replacement_tuple_reuse(self): cwr = combinations_with_replacement self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) - @pickle_deprecated def test_permutations(self): self.assertRaises(TypeError, permutations) # too few arguments self.assertRaises(TypeError, permutations, 'abc', 2, 1) # too many arguments @@ -514,9 +427,6 @@ def permutations2(iterable, r=None): self.assertEqual(result, list(permutations(values, None))) # test r as None self.assertEqual(result, list(permutations(values))) # test default r - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, permutations(values, r)) # test pickling - @support.bigaddrspacetest def test_permutations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -560,7 +470,6 @@ def test_combinatorics(self): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm - @pickle_deprecated def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF')) @@ -577,24 +486,6 @@ def test_compress(self): self.assertRaises(TypeError, compress, range(6)) # too few args self.assertRaises(TypeError, compress, range(6), None) # too many args - # check copy, deepcopy, pickle - for op in [lambda a:copy.copy(a), lambda a:copy.deepcopy(a)] + picklecopiers: - for data, selectors, result1, result2 in [ - ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'), - ('ABCDEF', [0,0,0,0,0,0], '', ''), - ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'), - ('ABCDEF', [1,0,1], 'AC', 'C'), - ('ABC', [0,1,1,1,1,1], 'BC', 'C'), - ]: - - self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1)) - self.assertEqual(list(op(compress(data, selectors))), list(result1)) - testIntermediate = compress(data, selectors) - if result1: - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), list(result2)) - - @pickle_deprecated def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)]) @@ -632,19 +523,10 @@ def test_count(self): r2 = 'count(%r)'.__mod__(i) self.assertEqual(r1, r2) - # check copy, deepcopy, pickle - for value in -3, 3, maxsize-5, maxsize+5: - c = count(value) - self.assertEqual(next(copy.copy(c)), value) - self.assertEqual(next(copy.deepcopy(c)), value) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(value)) - #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() - @pickle_deprecated - def test_count_with_stride(self): + def test_count_with_step(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) self.assertEqual(lzip('abc',count(start=2,step=3)), [('a', 2), ('b', 5), ('c', 8)]) @@ -687,17 +569,28 @@ def test_count_with_stride(self): c = count(10, 1.0) self.assertEqual(type(next(c)), int) self.assertEqual(type(next(c)), float) - for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): - for j in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 1, 10, sys.maxsize-5, sys.maxsize+5): - # Test repr - r1 = repr(count(i, j)) - if j == 1: - r2 = ('count(%r)' % i) - else: - r2 = ('count(%r, %r)' % (i, j)) - self.assertEqual(r1, r2) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(i, j)) + + @threading_helper.requires_working_threading() + def test_count_threading(self, step=1): + # this test verifies multithreading consistency, which is + # mostly for testing builds without GIL, but nice to test anyway + count_to = 10_000 + num_threads = 10 + c = count(step=step) + def counting_thread(): + for i in range(count_to): + next(c) + threads = [] + for i in range(num_threads): + thread = threading.Thread(target=counting_thread) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + self.assertEqual(next(c), count_to * num_threads * step) + + def test_count_with_step_threading(self): + self.test_count_threading(step=5) def test_cycle(self): self.assertEqual(take(10, cycle('abc')), list('abcabcabca')) @@ -706,113 +599,6 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) - @pickle_deprecated - def test_cycle_copy_pickle(self): - # check copy, deepcopy, pickle - c = cycle('abc') - self.assertEqual(next(c), 'a') - #simple copy currently not supported, because __reduce__ returns - #an internal iterator - #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab')) - self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab')) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('bcabcabcab')) - next(c) - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('cabcabcabc')) - next(c) - next(c) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cycle('abc')) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - # test with partial consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(2)] # consume 2 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - # test with completely consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(7)] # consume 7 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - @pickle_deprecated - def test_cycle_unpickle_compat(self): - testcases = [ - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - ] - assert len(testcases) == 20 - for t in testcases: - it = pickle.loads(t) - self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) - - @pickle_deprecated - def test_cycle_setstate(self): - # Verify both modes for restoring state - - # Mode 0 is efficient. It uses an incompletely consumed input - # iterator to build a cycle object and then passes in state with - # a list of previously consumed values. There is no data - # overlap between the two. - c = cycle('defg') - c.__setstate__((list('abc'), 0)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # Mode 1 is inefficient. It starts with a cycle object built - # from an iterator over the remaining elements in a partial - # cycle and then passes in state with all of the previously - # seen values (this overlaps values included in the iterator). - c = cycle('defg') - c.__setstate__((list('abcdefg'), 1)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # The first argument to setstate needs to be a tuple - with self.assertRaises(TypeError): - cycle('defg').__setstate__([list('abcdefg'), 0]) - - # The first argument in the setstate tuple must be a list - with self.assertRaises(TypeError): - c = cycle('defg') - c.__setstate__((tuple('defg'), 0)) - take(20, c) - - # The second argument in the setstate tuple must be an int - with self.assertRaises(TypeError): - cycle('defg').__setstate__((list('abcdefg'), 'x')) - - self.assertRaises(TypeError, cycle('').__setstate__, ()) - self.assertRaises(TypeError, cycle('').__setstate__, ([],)) - - @pickle_deprecated def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) @@ -831,15 +617,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check normal pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for elem in g: - self.assertEqual(k, elem[0]) - dup.append(elem) - self.assertEqual(s, dup) - # Check nested case dup = [] for k, g in groupby(s, testR): @@ -850,18 +627,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check nested and pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2), proto)): - for elem in ig: - self.assertEqual(k, elem[0]) - self.assertEqual(ik, elem[2]) - dup.append(elem) - self.assertEqual(s, dup) - - # Check case where inner iterator is not used keys = [k for k, g in groupby(s, testR)] expectedkeys = set([r[0] for r in s]) @@ -881,13 +646,6 @@ def test_groupby(self): list(it) # exhaust the groupby iterator self.assertEqual(list(g3), []) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = groupby(s, testR) - _, g = next(it) - next(it) - next(it) - self.assertEqual(list(pickle.loads(pickle.dumps(g, proto))), []) - # Exercise pipes and filters style s = 'abracadabra' # sort s | uniq @@ -970,7 +728,6 @@ def test_filter(self): c = filter(isEven, range(6)) self.pickletest(proto, c) - @pickle_deprecated def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0]) @@ -981,8 +738,6 @@ def test_filterfalse(self): self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7) self.assertRaises(TypeError, filterfalse, isEven, 3) self.assertRaises(TypeError, next, filterfalse(range(6), range(6))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, filterfalse(isEven, range(6))) def test_zip(self): # XXX This is rather silly now that builtin zip() calls zip()... @@ -1001,33 +756,12 @@ def test_zip(self): lzip('abc', 'def')) @support.impl_detail("tuple reuse is specific to CPython") - @pickle_deprecated def test_zip_tuple_reuse(self): ids = list(map(id, zip('abc', 'def'))) self.assertEqual(min(ids), max(ids)) ids = list(map(id, list(zip('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - # check copy, deepcopy, pickle - ans = [(x,y) for x, y in copy.copy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count()), proto))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - testIntermediate = zip('abc',count()) - next(testIntermediate) - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate, proto))] - self.assertEqual(ans, [('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip('abc', count())) - def test_ziplongest(self): for args in [ ['abc', range(6)], @@ -1077,14 +811,6 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - @pickle_deprecated - def test_zip_longest_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip_longest("abc", "def")) - self.pickletest(proto, zip_longest("abc", "defgh")) - self.pickletest(proto, zip_longest("abc", "defgh", fillvalue=1)) - self.pickletest(proto, zip_longest("", "defgh")) - def test_zip_longest_bad_iterable(self): exception = TypeError() @@ -1266,12 +992,16 @@ def product1(*args, **kwds): else: return - def product2(*args, **kwds): + def product2(*iterables, repeat=1): 'Pure python version used in docs' - pools = list(map(tuple, args)) * kwds.get('repeat', 1) + if repeat < 0: + raise ValueError('repeat argument cannot be negative') + pools = [tuple(pool) for pool in iterables] * repeat + result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] + for prod in result: yield tuple(prod) @@ -1296,34 +1026,6 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) - @pickle_deprecated - def test_product_pickling(self): - # check copy, deepcopy, pickle - for args, result in [ - ([], [()]), # zero iterables - (['ab'], [('a',), ('b',)]), # one iterable - ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables - ([range(0), range(2), range(3)], []), # first iterable with zero length - ([range(2), range(0), range(3)], []), # middle iterable with zero length - ([range(2), range(3), range(0)], []), # last iterable with zero length - ]: - self.assertEqual(list(copy.copy(product(*args))), result) - self.assertEqual(list(copy.deepcopy(product(*args))), result) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, product(*args)) - - @pickle_deprecated - def test_product_issue_25021(self): - # test that indices are properly clamped to the length of the tuples - p = product((1, 2),(3,)) - p.__setstate__((0, 0x1000)) # will access tuple element 1 if not clamped - self.assertEqual(next(p), (2, 3)) - # test that empty tuple in the list will result in an immediate StopIteration - p = product((1, 2), (), (3,)) - p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped - self.assertRaises(StopIteration, next, p) - - @pickle_deprecated def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) self.assertEqual(lzip(range(3),repeat('a')), @@ -1342,21 +1044,12 @@ def test_repeat(self): list(r) self.assertEqual(repr(r), 'repeat((1+0j), 0)') - # check copy, deepcopy, pickle - c = repeat(object='a', times=10) - self.assertEqual(next(c), 'a') - self.assertEqual(take(2, copy.copy(c)), list('a' * 2)) - self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2)) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, repeat(object='a', times=10)) - def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', -1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', -2)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") - @pickle_deprecated def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), [0**1, 1**2, 2**3]) @@ -1374,20 +1067,6 @@ def test_map(self): self.assertRaises(ValueError, next, map(errfunc, [4], [5])) self.assertRaises(TypeError, next, map(onearg, [4], [5])) - # check copy, deepcopy, pickle - ans = [('a',0),('b',1),('c',2)] - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.copy(c)), ans) - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = map(tupleize, 'abc', count()) - self.pickletest(proto, c) - - @pickle_deprecated def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), [0**1, 1**2, 2**3]) @@ -1402,20 +1081,6 @@ def test_starmap(self): self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)])) self.assertRaises(TypeError, next, starmap(onearg, [(4,5)])) - # check copy, deepcopy, pickle - ans = [0**1, 1**2, 2**3] - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.copy(c)), ans) - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.pickletest(proto, c) - - @pickle_deprecated def test_islice(self): for args in [ # islice(args) should agree with range(args) (10, 20, 3), @@ -1472,21 +1137,6 @@ def test_islice(self): self.assertEqual(list(islice(c, 1, 3, 50)), [1]) self.assertEqual(next(c), 3) - # check copy, deepcopy, pickle - for args in [ # islice(args) should agree with range(args) - (10, 20, 3), - (10, 3, 20), - (10, 20), - (10, 3), - (20,) - ]: - self.assertEqual(list(copy.copy(islice(range(100), *args))), - list(range(*args))) - self.assertEqual(list(copy.deepcopy(islice(range(100), *args))), - list(range(*args))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, islice(range(100), *args)) - # Issue #21321: check source iterator is not referenced # from islice() after the latter has been exhausted it = (x for x in (1, 2)) @@ -1510,7 +1160,6 @@ def __index__(self): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) - @pickle_deprecated def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(takewhile(underten, data)), [1, 3, 5]) @@ -1524,14 +1173,6 @@ def test_takewhile(self): self.assertEqual(list(t), [1, 1, 1]) self.assertRaises(StopIteration, next, t) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5]) - self.assertEqual(list(copy.deepcopy(takewhile(underten, data))), - [1, 3, 5]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, takewhile(underten, data)) - - @pickle_deprecated def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) @@ -1542,14 +1183,6 @@ def test_dropwhile(self): self.assertRaises(TypeError, next, dropwhile(10, [(4,5)])) self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)])) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8]) - self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))), - [20, 2, 4, 6, 8]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, dropwhile(underten, data)) - - @pickle_deprecated def test_tee(self): n = 200 @@ -1616,10 +1249,11 @@ def test_tee(self): self.assertEqual(len(result), n) self.assertEqual([list(x) for x in result], [list('abc')]*n) - # tee pass-through to copyable iterator + # tee objects are independent (see bug gh-123884) a, b = tee('abc') c, d = tee(a) - self.assertTrue(a is c) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) # test tee_new t1, t2 = tee('abc') @@ -1629,7 +1263,7 @@ def test_tee(self): t3 = tnew(t1) self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) - # test that tee objects are weak referencable + # test that tee objects are weak referenceable a, b = tee(range(10)) p = weakref.proxy(a) self.assertEqual(getattr(p, '__class__'), type(b)) @@ -1664,40 +1298,13 @@ def test_tee(self): self.assertEqual(list(a), long_ans[100:]) self.assertEqual(list(b), long_ans[60:]) - # check deepcopy - a, b = tee('abc') - self.assertEqual(list(copy.deepcopy(a)), ans) - self.assertEqual(list(copy.deepcopy(b)), ans) - self.assertEqual(list(a), ans) - self.assertEqual(list(b), ans) - a, b = tee(range(10000)) - self.assertEqual(list(copy.deepcopy(a)), long_ans) - self.assertEqual(list(copy.deepcopy(b)), long_ans) - self.assertEqual(list(a), long_ans) - self.assertEqual(list(b), long_ans) - - # check partially consumed deepcopy - a, b = tee('abc') - take(2, a) - take(1, b) - self.assertEqual(list(copy.deepcopy(a)), ans[2:]) - self.assertEqual(list(copy.deepcopy(b)), ans[1:]) - self.assertEqual(list(a), ans[2:]) - self.assertEqual(list(b), ans[1:]) - a, b = tee(range(10000)) - take(100, a) - take(60, b) - self.assertEqual(list(copy.deepcopy(a)), long_ans[100:]) - self.assertEqual(list(copy.deepcopy(b)), long_ans[60:]) - self.assertEqual(list(a), long_ans[100:]) - self.assertEqual(list(b), long_ans[60:]) - - # check pickle - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, iter(tee('abc'))) - a, b = tee('abc') - self.pickletest(proto, a, compare=ans) - self.pickletest(proto, b, compare=ans) + def test_tee_dealloc_segfault(self): + # gh-115874: segfaults when accessing module state in tp_dealloc. + script = ( + "import typing, copyreg, itertools; " + "copyreg.buggy_tee = itertools.tee(())" + ) + script_helper.assert_python_ok("-c", script) # Issue 13454: Crash when deleting backward iterator from tee() def test_tee_del_backward(self): @@ -1813,6 +1420,13 @@ def test_zip_longest_result_gc(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only + def test_pairwise_result_gc(self): + # Ditto for pairwise. + it = pairwise([None, None]) + gc.collect() + self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only def test_immutable_types(self): from itertools import _grouper, _tee, _tee_dataobject @@ -1851,33 +1465,6 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) - @pickle_deprecated - def test_accumulate_reducible(self): - # check copy, deepcopy, pickle - data = [1, 2, 3, 4, 5] - accumulated = [1, 3, 6, 10, 15] - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = accumulate(data) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[:]) - self.assertEqual(next(it), 1) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[1:]) - it = accumulate(data) - self.assertEqual(next(it), 1) - self.assertEqual(list(copy.deepcopy(it)), accumulated[1:]) - self.assertEqual(list(copy.copy(it)), accumulated[1:]) - - @pickle_deprecated - def test_accumulate_reducible_none(self): - # Issue #25718: total is None - it = accumulate([None, None, None], operator.is_) - self.assertEqual(next(it), None) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it_copy = pickle.loads(pickle.dumps(it, proto)) - self.assertEqual(list(it_copy), [True, False]) - self.assertEqual(list(copy.deepcopy(it)), [True, False]) - self.assertEqual(list(copy.copy(it)), [True, False]) - def test_chain(self): self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF') @@ -1985,27 +1572,169 @@ def batched_recipe(iterable, n): self.assertEqual(r1, r2) self.assertEqual(e1, e2) + + def test_groupby_recipe(self): + + # Begin groupby() recipe ####################################### + + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: + return + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass + + # End groupby() recipe ######################################### + + # Check whether it accepts arguments correctly + self.assertEqual([], list(groupby([]))) + self.assertEqual([], list(groupby([], key=id))) + self.assertRaises(TypeError, list, groupby('abc', [])) + if False: + # Test not applicable to the recipe + self.assertRaises(TypeError, list, groupby('abc', None)) + self.assertRaises(TypeError, groupby, 'abc', lambda x:x, 10) + + # Check normal input + s = [(0, 10, 20), (0, 11,21), (0,12,21), (1,13,21), (1,14,22), + (2,15,22), (3,16,23), (3,17,23)] + dup = [] + for k, g in groupby(s, lambda r:r[0]): + for elem in g: + self.assertEqual(k, elem[0]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check nested case + dup = [] + for k, g in groupby(s, testR): + for ik, ig in groupby(g, testR2): + for elem in ig: + self.assertEqual(k, elem[0]) + self.assertEqual(ik, elem[2]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check case where inner iterator is not used + keys = [k for k, g in groupby(s, testR)] + expectedkeys = set([r[0] for r in s]) + self.assertEqual(set(keys), expectedkeys) + self.assertEqual(len(keys), len(expectedkeys)) + + # Check case where inner iterator is used after advancing the groupby + # iterator + s = list(zip('AABBBAAAA', range(9))) + it = groupby(s, testR) + _, g1 = next(it) + _, g2 = next(it) + _, g3 = next(it) + self.assertEqual(list(g1), []) + self.assertEqual(list(g2), []) + self.assertEqual(next(g3), ('A', 5)) + list(it) # exhaust the groupby iterator + self.assertEqual(list(g3), []) + + # Exercise pipes and filters style + s = 'abracadabra' + # sort s | uniq + r = [k for k, g in groupby(sorted(s))] + self.assertEqual(r, ['a', 'b', 'c', 'd', 'r']) + # sort s | uniq -d + r = [k for k, g in groupby(sorted(s)) if list(islice(g,1,2))] + self.assertEqual(r, ['a', 'b', 'r']) + # sort s | uniq -c + r = [(len(list(g)), k) for k, g in groupby(sorted(s))] + self.assertEqual(r, [(5, 'a'), (2, 'b'), (1, 'c'), (1, 'd'), (2, 'r')]) + # sort s | uniq -c | sort -rn | head -3 + r = sorted([(len(list(g)) , k) for k, g in groupby(sorted(s))], reverse=True)[:3] + self.assertEqual(r, [(5, 'a'), (2, 'r'), (2, 'b')]) + + # iter.__next__ failure + class ExpectedError(Exception): + pass + def delayed_raise(n=0): + for i in range(n): + yield 'yo' + raise ExpectedError + def gulp(iterable, keyp=None, func=list): + return [func(g) for k, g in groupby(iterable, keyp)] + + # iter.__next__ failure on outer object + self.assertRaises(ExpectedError, gulp, delayed_raise(0)) + # iter.__next__ failure on inner object + self.assertRaises(ExpectedError, gulp, delayed_raise(1)) + + # __eq__ failure + class DummyCmp: + def __eq__(self, dst): + raise ExpectedError + s = [DummyCmp(), DummyCmp(), None] + + # __eq__ failure on outer object + self.assertRaises(ExpectedError, gulp, s, func=id) + # __eq__ failure on inner object + self.assertRaises(ExpectedError, gulp, s) + + # keyfunc failure + def keyfunc(obj): + if keyfunc.skip > 0: + keyfunc.skip -= 1 + return obj + else: + raise ExpectedError + + # keyfunc failure on outer object + keyfunc.skip = 0 + self.assertRaises(ExpectedError, gulp, [None], keyfunc) + keyfunc.skip = 1 + self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + + @staticmethod def islice(iterable, *args): + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step def test_islice_recipe(self): self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) @@ -2025,6 +1754,172 @@ def test_islice_recipe(self): self.assertEqual(next(c), 3) + def test_tee_recipe(self): + + # Begin tee() recipe ########################################### + + def tee(iterable, n=2): + if n < 0: + raise ValueError + if n == 0: + return () + iterator = _tee(iterable) + result = [iterator] + for _ in range(n - 1): + result.append(_tee(iterator)) + return tuple(result) + + class _tee: + + def __init__(self, iterable): + it = iter(iterable) + if isinstance(it, _tee): + self.iterator = it.iterator + self.link = it.link + else: + self.iterator = it + self.link = [None, None] + + def __iter__(self): + return self + + def __next__(self): + link = self.link + if link[1] is None: + link[0] = next(self.iterator) + link[1] = [None, None] + value, self.link = link + return value + + # End tee() recipe ############################################# + + n = 200 + + a, b = tee([]) # test empty iterator + self.assertEqual(list(a), []) + self.assertEqual(list(b), []) + + a, b = tee(irange(n)) # test 100% interleaved + self.assertEqual(lzip(a,b), lzip(range(n), range(n))) + + a, b = tee(irange(n)) # test 0% interleaved + self.assertEqual(list(a), list(range(n))) + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of leading iterator + for i in range(100): + self.assertEqual(next(a), i) + del a + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of trailing iterator + for i in range(100): + self.assertEqual(next(a), i) + del b + self.assertEqual(list(a), list(range(100, n))) + + for j in range(5): # test randomly interleaved + order = [0]*n + [1]*n + random.shuffle(order) + lists = ([], []) + its = tee(irange(n)) + for i in order: + value = next(its[i]) + lists[i].append(value) + self.assertEqual(lists[0], list(range(n))) + self.assertEqual(lists[1], list(range(n))) + + # test argument format checking + self.assertRaises(TypeError, tee) + self.assertRaises(TypeError, tee, 3) + self.assertRaises(TypeError, tee, [1,2], 'x') + self.assertRaises(TypeError, tee, [1,2], 3, 'x') + + # tee object should be instantiable + a, b = tee('abc') + c = type(a)('def') + self.assertEqual(list(c), list('def')) + + # test long-lagged and multi-way split + a, b, c = tee(range(2000), 3) + for i in range(100): + self.assertEqual(next(a), i) + self.assertEqual(list(b), list(range(2000))) + self.assertEqual([next(c), next(c)], list(range(2))) + self.assertEqual(list(a), list(range(100,2000))) + self.assertEqual(list(c), list(range(2,2000))) + + # test invalid values of n + self.assertRaises(TypeError, tee, 'abc', 'invalid') + self.assertRaises(ValueError, tee, [], -1) + + for n in range(5): + result = tee('abc', n) + self.assertEqual(type(result), tuple) + self.assertEqual(len(result), n) + self.assertEqual([list(x) for x in result], [list('abc')]*n) + + # tee objects are independent (see bug gh-123884) + a, b = tee('abc') + c, d = tee(a) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) + + # test tee_new + t1, t2 = tee('abc') + tnew = type(t1) + self.assertRaises(TypeError, tnew) + self.assertRaises(TypeError, tnew, 10) + t3 = tnew(t1) + self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) + + # test that tee objects are weak referencable + a, b = tee(range(10)) + p = weakref.proxy(a) + self.assertEqual(getattr(p, '__class__'), type(b)) + del a + gc.collect() # For PyPy or other GCs. + self.assertRaises(ReferenceError, getattr, p, '__class__') + + ans = list('abc') + long_ans = list(range(10000)) + + # Tests not applicable to the tee() recipe + if False: + # check copy + a, b = tee('abc') + self.assertEqual(list(copy.copy(a)), ans) + self.assertEqual(list(copy.copy(b)), ans) + a, b = tee(list(range(10000))) + self.assertEqual(list(copy.copy(a)), long_ans) + self.assertEqual(list(copy.copy(b)), long_ans) + + # check partially consumed copy + a, b = tee('abc') + take(2, a) + take(1, b) + self.assertEqual(list(copy.copy(a)), ans[2:]) + self.assertEqual(list(copy.copy(b)), ans[1:]) + self.assertEqual(list(a), ans[2:]) + self.assertEqual(list(b), ans[1:]) + a, b = tee(range(10000)) + take(100, a) + take(60, b) + self.assertEqual(list(copy.copy(a)), long_ans[100:]) + self.assertEqual(list(copy.copy(b)), long_ans[60:]) + self.assertEqual(list(a), long_ans[100:]) + self.assertEqual(list(b), long_ans[60:]) + + # Issue 13454: Crash when deleting backward iterator from tee() + forward, backward = tee(repeat(None, 2000)) # 20000000 + try: + any(forward) # exhaust the iterator + del backward + except: + del forward, backward + raise + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 124045b13184b3..2250af964c022b 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -8,14 +8,40 @@ class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) + self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) def test_float(self): rval = self.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) + self.assertIsInstance(rval, float) self.assertEqual(rval, 1.0) + def test_nonascii_digits_rejected(self): + # JSON specifies only ascii digits, see gh-125687 + for num in ["1\uff10", "0.\uff10", "0e\uff10"]: + with self.assertRaises(self.JSONDecodeError): + self.loads(num) + + def test_bytes(self): + self.assertEqual(self.loads(b"1"), 1) + + def test_parse_constant(self): + for constant, expected in [ + ("Infinity", "INFINITY"), + ("-Infinity", "-INFINITY"), + ("NaN", "NAN"), + ]: + self.assertEqual( + self.loads(constant, parse_constant=str.upper), expected + ) + + def test_constant_invalid_case(self): + for constant in [ + "nan", "NAN", "naN", "infinity", "INFINITY", "inFiniTy" + ]: + with self.assertRaises(self.JSONDecodeError): + self.loads(constant) + def test_empty_objects(self): self.assertEqual(self.loads('{}'), {}) self.assertEqual(self.loads('[]'), []) @@ -88,7 +114,8 @@ def test_string_with_utf8_bom(self): self.json.load(StringIO(bom_json)) self.assertIn('BOM', str(cm.exception)) # make sure that the BOM is not detected in the middle of a string - bom_in_str = '"{}"'.format(''.encode('utf-8-sig').decode('utf-8')) + bom = ''.encode('utf-8-sig').decode('utf-8') + bom_in_str = f'"{bom}"' self.assertEqual(self.loads(bom_in_str), '\ufeff') self.assertEqual(self.json.load(StringIO(bom_in_str)), '\ufeff') diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 3ce16684a08272..811880a15c8020 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -8,6 +8,24 @@ def test_default(self): self.dumps(type, default=repr), self.dumps(repr(type))) + def test_bad_default(self): + def default(obj): + if obj is NotImplemented: + raise ValueError + if obj is ...: + return NotImplemented + if obj is type: + return collections + return [...] + + with self.assertRaises(ValueError) as cm: + self.dumps(type, default=default) + self.assertEqual(cm.exception.__notes__, + ['when serializing ellipsis object', + 'when serializing list item 0', + 'when serializing module object', + 'when serializing type object']) + def test_ordereddict(self): od = collections.OrderedDict(a=1, b=2, c=3, d=4) od.move_to_end('b') diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 4bbc6c71489a83..6a39b72a09df35 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -23,8 +23,7 @@ def test_encode_basestring_ascii(self): for input_string, expect in CASES: result = self.json.encoder.encode_basestring_ascii(input_string) self.assertEqual(result, expect, - '{0!r} != {1!r} for {2}({3!r})'.format( - result, expect, fname, input_string)) + f'{result!r} != {expect!r} for {fname}({input_string!r})') def test_ordered_dict(self): # See issue 6105 diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index d6bce605e21463..7c1696cc66d12b 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -89,7 +89,7 @@ def test_failures(self): except self.JSONDecodeError: pass else: - self.fail("Expected failure for fail{0}.json: {1!r}".format(idx, doc)) + self.fail(f"Expected failure for fail{idx}.json: {doc!r}") def test_non_string_keys_dict(self): data = {'a' : 1, (1, 2) : 2} @@ -100,8 +100,27 @@ def test_non_string_keys_dict(self): def test_not_serializable(self): import sys with self.assertRaisesRegex(TypeError, - 'Object of type module is not JSON serializable'): + 'Object of type module is not JSON serializable') as cm: self.dumps(sys) + self.assertFalse(hasattr(cm.exception, '__notes__')) + + with self.assertRaises(TypeError) as cm: + self.dumps([1, [2, 3, sys]]) + self.assertEqual(cm.exception.__notes__, + ['when serializing list item 2', + 'when serializing list item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps((1, (2, 3, sys))) + self.assertEqual(cm.exception.__notes__, + ['when serializing tuple item 2', + 'when serializing tuple item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps({'a': {'b': sys}}) + self.assertEqual(cm.exception.__notes__, + ["when serializing dict item 'b'", + "when serializing dict item 'a'"]) def test_truncated_input(self): test_cases = [ diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 164ff2013eb552..290207e9c15b88 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -12,8 +12,8 @@ def test_listrecursion(self): x.append(x) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]) else: self.fail("didn't raise ValueError on list recursion") x = [] @@ -21,8 +21,8 @@ def test_listrecursion(self): x.append(y) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]*2) else: self.fail("didn't raise ValueError on alternating list recursion") y = [] @@ -35,8 +35,8 @@ def test_dictrecursion(self): x["test"] = x try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing dict item 'test'"]) else: self.fail("didn't raise ValueError on dict recursion") x = {} @@ -60,8 +60,10 @@ def default(self, o): enc.recurse = True try: enc.encode(JSONTestObject) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, + ["when serializing list item 0", + "when serializing type object"]) else: self.fail("didn't raise ValueError on default recursion") diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index 2d3ee8a8bf0f92..cca556a3b95bab 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -116,6 +116,11 @@ def test_bad_escapes(self): '"\\u012z"', '"\\u0x12"', '"\\u0X12"', + '"\\u{0}"'.format("\uff10" * 4), + '"\\u 123"', + '"\\u-123"', + '"\\u+123"', + '"\\u1_23"', '"\\ud834\\"', '"\\ud834\\u"', '"\\ud834\\ud"', @@ -127,6 +132,11 @@ def test_bad_escapes(self): '"\\ud834\\udd2z"', '"\\ud834\\u0x20"', '"\\ud834\\u0X20"', + '"\\ud834\\u{0}"'.format("\uff10" * 4), + '"\\ud834\\u 123"', + '"\\ud834\\u-123"', + '"\\ud834\\u+123"', + '"\\ud834\\u1_23"', ] for s in bad_escapes: with self.assertRaises(self.JSONDecodeError, msg=s): diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 2b63810d53981e..5da7cdcad709fa 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -11,7 +11,7 @@ @support.requires_subprocess() -class TestTool(unittest.TestCase): +class TestMain(unittest.TestCase): data = """ [["blorpie"],[ "whoops" ] , [ @@ -19,6 +19,7 @@ class TestTool(unittest.TestCase): "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field" :"yes"} ] """ + module = 'json' expect_without_sort_keys = textwrap.dedent("""\ [ @@ -87,7 +88,7 @@ class TestTool(unittest.TestCase): """) def test_stdin_stdout(self): - args = sys.executable, '-m', 'json.tool' + args = sys.executable, '-m', self.module process = subprocess.run(args, input=self.data, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, self.expect) self.assertEqual(process.stderr, '') @@ -101,7 +102,7 @@ def _create_infile(self, data=None): def test_infile_stdout(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', infile) + rc, out, err = assert_python_ok('-m', self.module, infile) self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) self.assertEqual(err, b'') @@ -115,7 +116,7 @@ def test_non_ascii_infile(self): ''').encode() infile = self._create_infile(data) - rc, out, err = assert_python_ok('-m', 'json.tool', infile) + rc, out, err = assert_python_ok('-m', self.module, infile) self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), expect.splitlines()) @@ -124,7 +125,7 @@ def test_non_ascii_infile(self): def test_infile_outfile(self): infile = self._create_infile() outfile = os_helper.TESTFN + '.out' - rc, out, err = assert_python_ok('-m', 'json.tool', infile, outfile) + rc, out, err = assert_python_ok('-m', self.module, infile, outfile) self.addCleanup(os.remove, outfile) with open(outfile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) @@ -134,7 +135,7 @@ def test_infile_outfile(self): def test_writing_in_place(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', infile, infile) + rc, out, err = assert_python_ok('-m', self.module, infile, infile) with open(infile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) self.assertEqual(rc, 0) @@ -142,20 +143,20 @@ def test_writing_in_place(self): self.assertEqual(err, b'') def test_jsonlines(self): - args = sys.executable, '-m', 'json.tool', '--json-lines' + args = sys.executable, '-m', self.module, '--json-lines' process = subprocess.run(args, input=self.jsonlines_raw, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, self.jsonlines_expect) self.assertEqual(process.stderr, '') def test_help_flag(self): - rc, out, err = assert_python_ok('-m', 'json.tool', '-h') + rc, out, err = assert_python_ok('-m', self.module, '-h') self.assertEqual(rc, 0) self.assertTrue(out.startswith(b'usage: ')) self.assertEqual(err, b'') def test_sort_keys_flag(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', '--sort-keys', infile) + rc, out, err = assert_python_ok('-m', self.module, '--sort-keys', infile) self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), self.expect_without_sort_keys.encode().splitlines()) @@ -169,7 +170,7 @@ def test_indent(self): 2 ] ''') - args = sys.executable, '-m', 'json.tool', '--indent', '2' + args = sys.executable, '-m', self.module, '--indent', '2' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -177,7 +178,7 @@ def test_indent(self): def test_no_indent(self): input_ = '[1,\n2]' expect = '[1, 2]\n' - args = sys.executable, '-m', 'json.tool', '--no-indent' + args = sys.executable, '-m', self.module, '--no-indent' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -185,7 +186,7 @@ def test_no_indent(self): def test_tab(self): input_ = '[1, 2]' expect = '[\n\t1,\n\t2\n]\n' - args = sys.executable, '-m', 'json.tool', '--tab' + args = sys.executable, '-m', self.module, '--tab' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -193,7 +194,7 @@ def test_tab(self): def test_compact(self): input_ = '[ 1 ,\n 2]' expect = '[1,2]\n' - args = sys.executable, '-m', 'json.tool', '--compact' + args = sys.executable, '-m', self.module, '--compact' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -202,7 +203,7 @@ def test_no_ensure_ascii_flag(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' self.addCleanup(os.remove, outfile) - assert_python_ok('-m', 'json.tool', '--no-ensure-ascii', infile, outfile) + assert_python_ok('-m', self.module, '--no-ensure-ascii', infile, outfile) with open(outfile, "rb") as f: lines = f.read().splitlines() # asserting utf-8 encoded output file @@ -213,7 +214,7 @@ def test_ensure_ascii_default(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' self.addCleanup(os.remove, outfile) - assert_python_ok('-m', 'json.tool', infile, outfile) + assert_python_ok('-m', self.module, infile, outfile) with open(outfile, "rb") as f: lines = f.read().splitlines() # asserting an ascii encoded output file @@ -222,11 +223,16 @@ def test_ensure_ascii_default(self): @unittest.skipIf(sys.platform =="win32", "The test is failed with ValueError on Windows") def test_broken_pipe_error(self): - cmd = [sys.executable, '-m', 'json.tool'] + cmd = [sys.executable, '-m', self.module] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE) - # bpo-39828: Closing before json.tool attempts to write into stdout. + # bpo-39828: Closing before json attempts to write into stdout. proc.stdout.close() proc.communicate(b'"{}"') self.assertEqual(proc.returncode, errno.EPIPE) + + +@support.requires_subprocess() +class TestTool(TestMain): + module = 'json.tool' diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 2e8bba2775256a..68629cceeb9be9 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -20,12 +20,17 @@ def test_encoding4(self): def test_encoding5(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps(u, ensure_ascii=False) - self.assertEqual(j, '"{0}"'.format(u)) + self.assertEqual(j, f'"{u}"') def test_encoding6(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps([u], ensure_ascii=False) - self.assertEqual(j, '["{0}"]'.format(u)) + self.assertEqual(j, f'["{u}"]') + + def test_encoding7(self): + u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' + j = self.dumps(u + "\n", ensure_ascii=False) + self.assertEqual(j, f'"{u}\\n"') def test_big_unicode_encode(self): u = '\U0001d120' @@ -34,13 +39,13 @@ def test_big_unicode_encode(self): def test_big_unicode_decode(self): u = 'z\U0001d120x' - self.assertEqual(self.loads('"' + u + '"'), u) + self.assertEqual(self.loads(f'"{u}"'), u) self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u) def test_unicode_decode(self): for i in range(0, 0xd7ff): u = chr(i) - s = '"\\u{0:04x}"'.format(i) + s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) def test_unicode_preservation(self): diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 3b0930fe69e30e..41f7b70e5cfe81 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -2,7 +2,6 @@ """ import os -import stat import sys import unittest import socket @@ -29,7 +28,7 @@ def setUp(self): mode = 'w+b' with self.open(TESTFN, mode) as f: - current_size = os.fstat(f.fileno())[stat.ST_SIZE] + current_size = os.fstat(f.fileno()).st_size if current_size == size+1: return @@ -40,13 +39,13 @@ def setUp(self): f.seek(size) f.write(b'a') f.flush() - self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1) + self.assertEqual(os.fstat(f.fileno()).st_size, size+1) @classmethod def tearDownClass(cls): with cls.open(TESTFN, 'wb'): pass - if not os.stat(TESTFN)[stat.ST_SIZE] == 0: + if not os.stat(TESTFN).st_size == 0: raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') unlink(TESTFN2) @@ -67,7 +66,7 @@ def test_large_read(self, _size): self.assertEqual(f.tell(), size + 1) def test_osstat(self): - self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1) + self.assertEqual(os.stat(TESTFN).st_size, size+1) def test_seek_read(self): with self.open(TESTFN, 'rb') as f: @@ -142,6 +141,9 @@ def test_truncate(self): f.truncate(1) self.assertEqual(f.tell(), 0) # else pointer moved f.seek(0) + # Verify readall on a truncated file is well behaved. read() + # without a size can be unbounded, this should get just the byte + # that remains. self.assertEqual(len(f.read()), 1) # else wasn't truncated def test_seekable(self): @@ -152,6 +154,22 @@ def test_seekable(self): f.seek(pos) self.assertTrue(f.seekable()) + @bigmemtest(size=size, memuse=2, dry_run=False) + def test_seek_readall(self, _size): + # Seek which doesn't change position should readall successfully. + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(0, os.SEEK_CUR), 0) + self.assertEqual(len(f.read()), size + 1) + + # Seek which changes (or might change) position should readall + # successfully. + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(20, os.SEEK_SET), 20) + self.assertEqual(len(f.read()), size - 19) + + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(-3, os.SEEK_END), size - 2) + self.assertEqual(len(f.read()), 3) def skip_no_disk_space(path, required): def decorator(fun): diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 2528a51240fbf7..58baae25df3df7 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -764,3 +764,11 @@ def test_shebang_command_in_venv(self): with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: data = self.run_py([script], env=env) self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") + + def test_shebang_executable_extension(self): + with self.script('#! /usr/bin/env python3.99') as script: + data = self.run_py([script], expect_returncode=103) + expect = "# Search PATH for python3.99.exe" + actual = [line.strip() for line in data["stderr"].splitlines() + if line.startswith("# Search PATH")] + self.assertEqual([expect], actual) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 72dd40136cfdb2..6f5955791407ea 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -5,6 +5,7 @@ import os.path import tempfile import tokenize +from importlib.machinery import ModuleSpec from test import support from test.support import os_helper @@ -82,6 +83,10 @@ def test_getlines(self): class EmptyFile(GetLineTestsGoodData, unittest.TestCase): file_list = [] + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, ['\n']) + class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): file_list = ['\n'] @@ -97,6 +102,16 @@ class BadUnicode_WithDeclaration(GetLineTestsBadData, unittest.TestCase): file_byte_string = b'# coding=utf-8\n\x80abc' +class FakeLoader: + def get_source(self, fullname): + return f'source for {fullname}' + + +class NoSourceLoader: + def get_source(self, fullname): + return None + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -238,6 +253,64 @@ def raise_memoryerror(*args, **kwargs): self.assertEqual(lines3, []) self.assertEqual(linecache.getlines(FILENAME), lines) + def test_loader(self): + filename = 'scheme://path' + + for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': loader} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + for spec in (None, object(), ModuleSpec('', FakeLoader())): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + linecache.clearcache() + spec = ModuleSpec('x.y.z', FakeLoader()) + module_globals = {'__name__': 'a.b.c', '__loader__': spec.loader, + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for x.y.z\n']) + + def test_invalid_names(self): + for name, desc in [ + ('\x00', 'NUL bytes filename'), + (__file__ + '\x00', 'filename with embedded NUL bytes'), + # A filename with surrogate codes. A UnicodeEncodeError is raised + # by os.stat() upon querying, which is a subclass of ValueError. + ("\uD834\uDD1E.py", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + # For POSIX platforms, an OSError will be raised but for Windows + # platforms, a ValueError is raised due to the path_t converter. + # See: https://github.com/python/cpython/issues/122170 + ('a' * 1_000_000, 'very long filename'), + ]: + with self.subTest(f'updatecache: {desc}'): + linecache.clearcache() + lines = linecache.updatecache(name) + self.assertListEqual(lines, []) + self.assertNotIn(name, linecache.cache) + + # hack into the cache (it shouldn't be allowed + # but we never know what people do...) + for key, fullname in [(name, 'ok'), ('key', name), (name, name)]: + with self.subTest(f'checkcache: {desc}', + key=key, fullname=fullname): + linecache.clearcache() + linecache.cache[key] = (0, 1234, [], fullname) + linecache.checkcache(key) + self.assertNotIn(key, linecache.cache) + + # just to be sure that we did not mess with cache + linecache.clearcache() + class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 2969c6e2f98a23..ad7accf2099f43 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -96,6 +96,11 @@ def imul(a, b): a *= b self.assertRaises((MemoryError, OverflowError), mul, lst, n) self.assertRaises((MemoryError, OverflowError), imul, lst, n) + def test_empty_slice(self): + x = [] + x[:] = x + self.assertEqual(x, []) + def test_list_resize_overflow(self): # gh-97616: test new_allocated * sizeof(PyObject*) overflow # check in list_resize() @@ -229,6 +234,31 @@ def __eq__(self, other): list4 = [1] self.assertFalse(list3 == list4) + def test_lt_operator_modifying_operand(self): + # See gh-120298 + class evil: + def __lt__(self, other): + other.clear() + return NotImplemented + + a = [[evil()]] + with self.assertRaises(TypeError): + a[0] < a + + def test_list_index_modifing_operand(self): + # See gh-120384 + class evil: + def __init__(self, lst): + self.lst = lst + def __iter__(self): + yield from self.lst + self.lst.clear() + + lst = list(range(5)) + operand = evil(lst) + with self.assertRaises(ValueError): + lst[::-1] = operand + @cpython_only def test_preallocation(self): iterable = [0] * 10 @@ -269,6 +299,15 @@ def __eq__(self, other): lst = [X(), X()] X() in lst + def test_tier2_invalidates_iterator(self): + # GH-121012 + for _ in range(100): + a = [1, 2, 3] + it = iter(a) + for _ in it: + pass + a.append(4) + self.assertEqual(list(it), []) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 2868dd01545b95..cffdeeacc5d73b 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -1,8 +1,11 @@ import doctest import textwrap +import traceback import types import unittest +from test.support import BrokenIter + doctests = """ ########### Tests borrowed from or inspired by test_genexps.py ############ @@ -168,6 +171,31 @@ def test_references___class__(self): """ self._check_in_scopes(code, raises=NameError) + def test_references___class___defined(self): + code = """ + __class__ = 2 + res = [__class__ for x in [1]] + """ + self._check_in_scopes( + code, outputs={"res": [2]}, scopes=["module", "function"]) + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_references___class___enclosing(self): + code = """ + __class__ = 2 + class C: + res = [__class__ for x in [1]] + res = C.res + """ + self._check_in_scopes(code, raises=NameError) + + def test_super_and_class_cell_in_sibling_comps(self): + code = """ + [super for _ in [1]] + [__class__ for _ in [1]] + """ + self._check_in_scopes(code, raises=NameError) + def test_inner_cell_shadows_outer(self): code = """ items = [(lambda: i) for i in range(5)] @@ -581,7 +609,7 @@ def test_comp_in_try_except(self): result = snapshot = None try: result = [{func}(value) for value in value] - except: + except ValueError: snapshot = value raise """ @@ -615,16 +643,21 @@ def test_exception_in_post_comp_call(self): value = [1, None] try: [v for v in value].sort() - except: + except TypeError: pass """ self._check_in_scopes(code, {"value": [1, None]}) def test_frame_locals(self): code = """ - val = [sys._getframe().f_locals for a in [0]][0]["a"] + val = "a" in [sys._getframe().f_locals for a in [0]][0] """ import sys + self._check_in_scopes(code, {"val": False}, ns={"sys": sys}) + + code = """ + val = [sys._getframe().f_locals["a"] for a in [0]][0] + """ self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) def _recursive_replace(self, maybe_code): @@ -666,6 +699,56 @@ def test_code_replace_extended_arg(self): self._check_in_scopes(code, expected) self._check_in_scopes(code, expected, exec_func=self._replacing_exec) + def test_multiple_comprehension_name_reuse(self): + code = """ + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"y": [3]}, ns={"x": 3}) + + code = """ + x = 2 + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"x": 2, "y": [3]}, ns={"x": 3}, scopes=["class"]) + self._check_in_scopes(code, {"x": 2, "y": [2]}, ns={"x": 3}, scopes=["function", "module"]) + + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + def init_raises(): + try: + [x for x in BrokenIter(init_raises=True)] + except Exception as e: + return e + + def next_raises(): + try: + [x for x in BrokenIter(next_raises=True)] + except Exception as e: + return e + + def iter_raises(): + try: + [x for x in BrokenIter(iter_raises=True)] + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index da4bd79746a476..00e93d8e78443d 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -355,6 +355,8 @@ def setUp(self): is_emscripten or is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") def test_strcoll_with_diacritic(self): self.assertLess(locale.strcoll('à', 'b'), 0) @@ -364,6 +366,8 @@ def test_strcoll_with_diacritic(self): is_emscripten or is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") def test_strxfrm_with_diacritic(self): self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 888523227c2ac4..e72f222e1c7eeb 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -60,6 +60,7 @@ import weakref from http.server import HTTPServer, BaseHTTPRequestHandler +from unittest.mock import patch from urllib.parse import urlparse, parse_qs from socketserver import (ThreadingUDPServer, DatagramRequestHandler, ThreadingTCPServer, StreamRequestHandler) @@ -80,6 +81,9 @@ skip_if_asan_fork = unittest.skipIf( support.HAVE_ASAN_FORK_BUG, "libasan has a pthread_create() dead lock related to thread+fork") +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") class BaseTest(unittest.TestCase): @@ -603,7 +607,7 @@ def test_name(self): def test_builtin_handlers(self): # We can't actually *use* too many handlers in the tests, # but we can try instantiating them with various options - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): for existing in (True, False): fn = make_temp_file() if not existing: @@ -653,21 +657,21 @@ def test_builtin_handlers(self): self.assertFalse(h.shouldFlush(r)) h.close() - def test_path_objects(self): + def test_pathlike_objects(self): """ - Test that Path objects are accepted as filename arguments to handlers. + Test that path-like objects are accepted as filename arguments to handlers. See Issue #27493. """ fn = make_temp_file() os.unlink(fn) - pfn = pathlib.Path(fn) + pfn = os_helper.FakePath(fn) cases = ( (logging.FileHandler, (pfn, 'w')), (logging.handlers.RotatingFileHandler, (pfn, 'a')), (logging.handlers.TimedRotatingFileHandler, (pfn, 'h')), ) - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): cases += ((logging.handlers.WatchedFileHandler, (pfn, 'w')),) for cls, args in cases: h = cls(*args, encoding="utf-8") @@ -731,6 +735,7 @@ def remove_loop(fname, tries): @support.requires_fork() @threading_helper.requires_working_threading() @skip_if_asan_fork + @skip_if_tsan_fork def test_post_fork_child_no_deadlock(self): """Ensure child logging locks are not held; bpo-6721 & bpo-36533.""" class _OurHandler(logging.Handler): @@ -1033,6 +1038,7 @@ class TestTCPServer(ControlMixin, ThreadingTCPServer): """ allow_reuse_address = True + allow_reuse_port = True def __init__(self, addr, handler, poll_interval=0.5, bind_and_activate=True): @@ -2187,7 +2193,8 @@ def test_output(self): self.handled.clear() msg = "sp\xe4m" logger.error(msg) - self.handled.wait() + handled = self.handled.wait(support.SHORT_TIMEOUT) + self.assertTrue(handled, "HTTP request timed out") self.assertEqual(self.log_data.path, '/frob') self.assertEqual(self.command, method) if method == 'GET': @@ -2361,6 +2368,32 @@ class CustomListener(logging.handlers.QueueListener): class CustomQueue(queue.Queue): pass +class CustomQueueProtocol: + def __init__(self, maxsize=0): + self.queue = queue.Queue(maxsize) + + def __getattr__(self, attribute): + queue = object.__getattribute__(self, 'queue') + return getattr(queue, attribute) + +class CustomQueueFakeProtocol(CustomQueueProtocol): + # An object implementing the minimial Queue API for + # the logging module but with incorrect signatures. + # + # The object will be considered a valid queue class since we + # do not check the signatures (only callability of methods) + # but will NOT be usable in production since a TypeError will + # be raised due to the extra argument in 'put_nowait'. + def put_nowait(self): + pass + +class CustomQueueWrongProtocol(CustomQueueProtocol): + put_nowait = None + +class MinimalQueueProtocol: + def put_nowait(self, x): pass + def get(self): pass + def queueMaker(): return queue.Queue() @@ -3030,6 +3063,30 @@ def format(self, record): }, } + config18 = { + "version": 1, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + }, + "buffering": { + "class": "logging.handlers.MemoryHandler", + "capacity": 5, + "target": "console", + "level": "DEBUG", + "flushLevel": "ERROR" + } + }, + "loggers": { + "mymodule": { + "level": "DEBUG", + "handlers": ["buffering"], + "propagate": "true" + } + } + } + bad_format = { "version": 1, "formatters": { @@ -3516,6 +3573,11 @@ def test_config17_ok(self): h = logging._handlers['hand1'] self.assertEqual(h.formatter.custom_property, 'value') + def test_config18_ok(self): + self.apply_config(self.config18) + handler = logging.getLogger('mymodule').handlers[0] + self.assertEqual(handler.flushLevel, logging.ERROR) + def setup_via_listener(self, text, verify=None): text = text.encode("utf-8") # Ask for a randomly assigned port (by using port 0) @@ -3693,16 +3755,16 @@ def test_baseconfig(self): 'adict': { 'd': 'e', 'f': 3 , 'alpha numeric 1 with spaces' : 5, - 'aplha numeric 1 %( - © ©ß¯' : 9, + 'alpha numeric 1 %( - © ©ß¯' : 9, 'alpha numeric ] 1 with spaces' : 15, - 'aplha ]] numeric 1 %( - © ©ß¯]' : 19, - ' aplha [ numeric 1 %( - © ©ß¯] ' : 11, - ' aplha ' : 32, + 'alpha ]] numeric 1 %( - © ©ß¯]' : 19, + ' alpha [ numeric 1 %( - © ©ß¯] ' : 11, + ' alpha ' : 32, '' : 10, 'nest4' : { 'd': 'e', 'f': 3 , 'alpha numeric 1 with spaces' : 5, - 'aplha numeric 1 %( - © ©ß¯' : 9, + 'alpha numeric 1 %( - © ©ß¯' : 9, '' : 10, 'somelist' : ('g', ('h', 'i'), 'j'), 'somedict' : { @@ -3724,14 +3786,14 @@ def test_baseconfig(self): self.assertEqual(bc.convert('cfg://adict.d'), 'e') self.assertEqual(bc.convert('cfg://adict[f]'), 3) self.assertEqual(bc.convert('cfg://adict[alpha numeric 1 with spaces]'), 5) - self.assertEqual(bc.convert('cfg://adict[aplha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[alpha numeric 1 %( - © ©ß¯]'), 9) self.assertEqual(bc.convert('cfg://adict[]'), 10) self.assertEqual(bc.convert('cfg://adict.nest4.d'), 'e') self.assertEqual(bc.convert('cfg://adict.nest4[d]'), 'e') self.assertEqual(bc.convert('cfg://adict[nest4].d'), 'e') self.assertEqual(bc.convert('cfg://adict[nest4][f]'), 3) self.assertEqual(bc.convert('cfg://adict[nest4][alpha numeric 1 with spaces]'), 5) - self.assertEqual(bc.convert('cfg://adict[nest4][aplha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[nest4][alpha numeric 1 %( - © ©ß¯]'), 9) self.assertEqual(bc.convert('cfg://adict[nest4][]'), 10) self.assertEqual(bc.convert('cfg://adict[nest4][somelist][0]'), 'g') self.assertEqual(bc.convert('cfg://adict[nest4][somelist][1][0]'), 'h') @@ -3751,8 +3813,8 @@ def test_baseconfig(self): self.assertRaises(ValueError, bc.convert, 'cfg://!') self.assertRaises(KeyError, bc.convert, 'cfg://adict[2]') self.assertRaises(KeyError, bc.convert, 'cfg://adict[alpha numeric ] 1 with spaces]') - self.assertRaises(ValueError, bc.convert, 'cfg://adict[ aplha ]] numeric 1 %( - © ©ß¯] ]') - self.assertRaises(ValueError, bc.convert, 'cfg://adict[ aplha [ numeric 1 %( - © ©ß¯] ]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ alpha ]] numeric 1 %( - © ©ß¯] ]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ alpha [ numeric 1 %( - © ©ß¯] ]') def test_namedtuple(self): # see bpo-39142 @@ -3863,19 +3925,18 @@ def do_queuehandler_configuration(self, qspec, lspec): self.addCleanup(os.remove, fn) @threading_helper.requires_working_threading() + @support.requires_subprocess() def test_config_queue_handler(self): - q = CustomQueue() - dq = { - '()': __name__ + '.CustomQueue', - 'maxsize': 10 - } + qs = [CustomQueue(), CustomQueueProtocol()] + dqs = [{'()': f'{__name__}.{cls}', 'maxsize': 10} + for cls in ['CustomQueue', 'CustomQueueProtocol']] dl = { '()': __name__ + '.listenerMaker', 'arg1': None, 'arg2': None, 'respect_handler_level': True } - qvalues = (None, __name__ + '.queueMaker', __name__ + '.CustomQueue', dq, q) + qvalues = (None, __name__ + '.queueMaker', __name__ + '.CustomQueue', *dqs, *qs) lvalues = (None, __name__ + '.CustomListener', dl, CustomListener) for qspec, lspec in itertools.product(qvalues, lvalues): self.do_queuehandler_configuration(qspec, lspec) @@ -3891,6 +3952,140 @@ def test_config_queue_handler(self): msg = str(ctx.exception) self.assertEqual(msg, "Unable to configure handler 'ah'") + def _apply_simple_queue_listener_configuration(self, qspec): + self.apply_config({ + "version": 1, + "handlers": { + "queue_listener": { + "class": "logging.handlers.QueueHandler", + "queue": qspec, + }, + }, + }) + + @threading_helper.requires_working_threading() + @support.requires_subprocess() + @patch("multiprocessing.Manager") + def test_config_queue_handler_does_not_create_multiprocessing_manager(self, manager): + # gh-120868, gh-121723, gh-124653 + + for qspec in [ + {"()": "queue.Queue", "maxsize": -1}, + queue.Queue(), + # queue.SimpleQueue does not inherit from queue.Queue + queue.SimpleQueue(), + # CustomQueueFakeProtocol passes the checks but will not be usable + # since the signatures are incompatible. Checking the Queue API + # without testing the type of the actual queue is a trade-off + # between usability and the work we need to do in order to safely + # check that the queue object correctly implements the API. + CustomQueueFakeProtocol(), + MinimalQueueProtocol(), + ]: + with self.subTest(qspec=qspec): + self._apply_simple_queue_listener_configuration(qspec) + manager.assert_not_called() + + @patch("multiprocessing.Manager") + def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_manager(self, manager): + # gh-120868, gh-121723 + + for qspec in [object(), CustomQueueWrongProtocol()]: + with self.subTest(qspec=qspec), self.assertRaises(ValueError): + self._apply_simple_queue_listener_configuration(qspec) + manager.assert_not_called() + + @skip_if_tsan_fork + @support.requires_subprocess() + @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" + " assertions in multiprocessing") + def test_config_reject_simple_queue_handler_multiprocessing_context(self): + # multiprocessing.SimpleQueue does not implement 'put_nowait' + # and thus cannot be used as a queue-like object (gh-124653) + + import multiprocessing + + if support.MS_WINDOWS: + start_methods = ['spawn'] + else: + start_methods = ['spawn', 'fork', 'forkserver'] + + for start_method in start_methods: + with self.subTest(start_method=start_method): + ctx = multiprocessing.get_context(start_method) + qspec = ctx.SimpleQueue() + with self.assertRaises(ValueError): + self._apply_simple_queue_listener_configuration(qspec) + + @skip_if_tsan_fork + @support.requires_subprocess() + @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" + " assertions in multiprocessing") + def test_config_queue_handler_multiprocessing_context(self): + # regression test for gh-121723 + if support.MS_WINDOWS: + start_methods = ['spawn'] + else: + start_methods = ['spawn', 'fork', 'forkserver'] + for start_method in start_methods: + with self.subTest(start_method=start_method): + ctx = multiprocessing.get_context(start_method) + with ctx.Manager() as manager: + q = manager.Queue() + records = [] + # use 1 process and 1 task per child to put 1 record + with ctx.Pool(1, initializer=self._mpinit_issue121723, + initargs=(q, "text"), maxtasksperchild=1): + records.append(q.get(timeout=60)) + self.assertTrue(q.empty()) + self.assertEqual(len(records), 1) + + @staticmethod + def _mpinit_issue121723(qspec, message_to_log): + # static method for pickling support + logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': True, + 'handlers': { + 'log_to_parent': { + 'class': 'logging.handlers.QueueHandler', + 'queue': qspec + } + }, + 'root': {'handlers': ['log_to_parent'], 'level': 'DEBUG'} + }) + # log a message (this creates a record put in the queue) + logging.getLogger().info(message_to_log) + + @skip_if_tsan_fork + @support.requires_subprocess() + def test_multiprocessing_queues(self): + # See gh-119819 + + cd = copy.deepcopy(self.config_queue_handler) + from multiprocessing import Queue as MQ, Manager as MM + q1 = MQ() # this can't be pickled + q2 = MM().Queue() # a proxy queue for use when pickling is needed + q3 = MM().JoinableQueue() # a joinable proxy queue + for qspec in (q1, q2, q3): + fn = make_temp_file('.log', 'test_logging-cmpqh-') + cd['handlers']['h1']['filename'] = fn + cd['handlers']['ah']['queue'] = qspec + qh = None + try: + self.apply_config(cd) + qh = logging.getHandlerByName('ah') + self.assertEqual(sorted(logging.getHandlerNames()), ['ah', 'h1']) + self.assertIsNotNone(qh.listener) + self.assertIs(qh.queue, qspec) + self.assertIs(qh.listener.queue, qspec) + finally: + h = logging.getHandlerByName('h1') + if h: + self.addCleanup(closeFileHandler, h, fn) + else: + self.addCleanup(os.remove, fn) + def test_90195(self): # See gh-90195 config = { @@ -3941,6 +4136,35 @@ def test_111615(self): } logging.config.dictConfig(config) + # gh-118868: check if kwargs are passed to logging QueueHandler + def test_kwargs_passing(self): + class CustomQueueHandler(logging.handlers.QueueHandler): + def __init__(self, *args, **kwargs): + super().__init__(queue.Queue()) + self.custom_kwargs = kwargs + + custom_kwargs = {'foo': 'bar'} + + config = { + 'version': 1, + 'handlers': { + 'custom': { + 'class': CustomQueueHandler, + **custom_kwargs + }, + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['custom'] + } + } + + logging.config.dictConfig(config) + + handler = logging.getHandlerByName('custom') + self.assertEqual(handler.custom_kwargs, custom_kwargs) + + class ManagerTest(BaseTest): def test_manager_loggerclass(self): logged = [] @@ -4146,6 +4370,7 @@ def test_queue_listener_with_multiple_handlers(self): import multiprocessing from unittest.mock import patch + @skip_if_tsan_fork @threading_helper.requires_working_threading() class QueueListenerTest(BaseTest): """ @@ -4547,6 +4772,77 @@ def test_issue_89047(self): s = f.format(r) self.assertNotIn('.1000', s) + def test_msecs_has_no_floating_point_precision_loss(self): + # See issue gh-102402 + tests = ( + # time_ns is approx. 2023-03-04 04:25:20 UTC + # (time_ns, expected_msecs_value) + (1_677_902_297_100_000_000, 100.0), # exactly 100ms + (1_677_903_920_999_998_503, 999.0), # check truncating doesn't round + (1_677_903_920_000_998_503, 0.0), # check truncating doesn't round + (1_677_903_920_999_999_900, 0.0), # check rounding up + ) + for ns, want in tests: + with patch('time.time_ns') as patched_ns: + patched_ns.return_value = ns + record = logging.makeLogRecord({'msg': 'test'}) + with self.subTest(ns): + self.assertEqual(record.msecs, want) + self.assertEqual(record.created, ns / 1e9) + self.assertAlmostEqual(record.created - int(record.created), + record.msecs / 1e3, + delta=1e-3) + + def test_relativeCreated_has_higher_precision(self): + # See issue gh-102402. + # Run the code in the subprocess, because the time module should + # be patched before the first import of the logging package. + # Temporary unloading and re-importing the logging package has + # side effects (including registering the atexit callback and + # references leak). + start_ns = 1_677_903_920_000_998_503 # approx. 2023-03-04 04:25:20 UTC + offsets_ns = (200, 500, 12_354, 99_999, 1_677_903_456_999_123_456) + code = textwrap.dedent(f""" + start_ns = {start_ns!r} + offsets_ns = {offsets_ns!r} + start_monotonic_ns = start_ns - 1 + + import time + # Only time.time_ns needs to be patched for the current + # implementation, but patch also other functions to make + # the test less implementation depending. + old_time_ns = time.time_ns + old_time = time.time + old_monotonic_ns = time.monotonic_ns + old_monotonic = time.monotonic + time_ns_result = start_ns + time.time_ns = lambda: time_ns_result + time.time = lambda: time.time_ns()/1e9 + time.monotonic_ns = lambda: time_ns_result - start_monotonic_ns + time.monotonic = lambda: time.monotonic_ns()/1e9 + try: + import logging + + for offset_ns in offsets_ns: + # mock for log record creation + time_ns_result = start_ns + offset_ns + record = logging.makeLogRecord({{'msg': 'test'}}) + print(record.created, record.relativeCreated) + finally: + time.time_ns = old_time_ns + time.time = old_time + time.monotonic_ns = old_monotonic_ns + time.monotonic = old_monotonic + """) + rc, out, err = assert_python_ok("-c", code) + out = out.decode() + for offset_ns, line in zip(offsets_ns, out.splitlines(), strict=True): + with self.subTest(offset_ns=offset_ns): + created, relativeCreated = map(float, line.split()) + self.assertAlmostEqual(created, (start_ns + offset_ns) / 1e9, places=6) + # After PR gh-102412, precision (places) increases from 3 to 7 + self.assertAlmostEqual(relativeCreated, offset_ns / 1e6, places=7) + class TestBufferingFormatter(logging.BufferingFormatter): def formatHeader(self, records): @@ -4581,7 +4877,7 @@ def test_formatting(self): r.addHandler(h) try: raise RuntimeError('deliberate mistake') - except: + except RuntimeError: logging.exception('failed', stack_info=True) r.removeHandler(h) h.close() @@ -4975,6 +5271,7 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): else: return results + @skip_if_tsan_fork def test_multiprocessing(self): support.skip_if_broken_multiprocessing_synchronize() multiprocessing_imported = 'multiprocessing' in sys.modules @@ -5478,6 +5775,7 @@ def test_critical(self): self.assertEqual(record.levelno, logging.CRITICAL) self.assertEqual(record.msg, msg) self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.funcName, 'test_critical') def test_is_enabled_for(self): old_disable = self.adapter.logger.manager.disable @@ -5496,15 +5794,9 @@ def test_has_handlers(self): self.assertFalse(self.adapter.hasHandlers()) def test_nested(self): - class Adapter(logging.LoggerAdapter): - prefix = 'Adapter' - - def process(self, msg, kwargs): - return f"{self.prefix} {msg}", kwargs - msg = 'Adapters can be nested, yo.' - adapter = Adapter(logger=self.logger, extra=None) - adapter_adapter = Adapter(logger=adapter, extra=None) + adapter = PrefixAdapter(logger=self.logger, extra=None) + adapter_adapter = PrefixAdapter(logger=adapter, extra=None) adapter_adapter.prefix = 'AdapterAdapter' self.assertEqual(repr(adapter), repr(adapter_adapter)) adapter_adapter.log(logging.CRITICAL, msg, self.recording) @@ -5513,6 +5805,7 @@ def process(self, msg, kwargs): self.assertEqual(record.levelno, logging.CRITICAL) self.assertEqual(record.msg, f"Adapter AdapterAdapter {msg}") self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.funcName, 'test_nested') orig_manager = adapter_adapter.manager self.assertIs(adapter.manager, orig_manager) self.assertIs(self.logger.manager, orig_manager) @@ -5528,6 +5821,61 @@ def process(self, msg, kwargs): self.assertIs(adapter.manager, orig_manager) self.assertIs(self.logger.manager, orig_manager) + def test_styled_adapter(self): + # Test an example from the Cookbook. + records = self.recording.records + adapter = StyleAdapter(self.logger) + adapter.warning('Hello, {}!', 'world') + self.assertEqual(str(records[-1].msg), 'Hello, world!') + self.assertEqual(records[-1].funcName, 'test_styled_adapter') + adapter.log(logging.WARNING, 'Goodbye {}.', 'world') + self.assertEqual(str(records[-1].msg), 'Goodbye world.') + self.assertEqual(records[-1].funcName, 'test_styled_adapter') + + def test_nested_styled_adapter(self): + records = self.recording.records + adapter = PrefixAdapter(self.logger) + adapter.prefix = '{}' + adapter2 = StyleAdapter(adapter) + adapter2.warning('Hello, {}!', 'world') + self.assertEqual(str(records[-1].msg), '{} Hello, world!') + self.assertEqual(records[-1].funcName, 'test_nested_styled_adapter') + adapter2.log(logging.WARNING, 'Goodbye {}.', 'world') + self.assertEqual(str(records[-1].msg), '{} Goodbye world.') + self.assertEqual(records[-1].funcName, 'test_nested_styled_adapter') + + def test_find_caller_with_stacklevel(self): + the_level = 1 + trigger = self.adapter.warning + + def innermost(): + trigger('test', stacklevel=the_level) + + def inner(): + innermost() + + def outer(): + inner() + + records = self.recording.records + outer() + self.assertEqual(records[-1].funcName, 'innermost') + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'inner') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'outer') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'test_find_caller_with_stacklevel') + self.assertGreater(records[-1].lineno, lineno) + def test_extra_in_records(self): self.adapter = logging.LoggerAdapter(logger=self.logger, extra={'foo': '1'}) @@ -5569,6 +5917,30 @@ def test_extra_merged_log_call_has_precedence(self): self.assertEqual(record.foo, '2') +class PrefixAdapter(logging.LoggerAdapter): + prefix = 'Adapter' + + def process(self, msg, kwargs): + return f"{self.prefix} {msg}", kwargs + + +class Message: + def __init__(self, fmt, args): + self.fmt = fmt + self.args = args + + def __str__(self): + return self.fmt.format(*self.args) + + +class StyleAdapter(logging.LoggerAdapter): + def log(self, level, msg, /, *args, stacklevel=1, **kwargs): + if self.isEnabledFor(level): + msg, kwargs = self.process(msg, kwargs) + self.logger.log(level, Message(msg, args), **kwargs, + stacklevel=stacklevel+1) + + class LoggerTest(BaseTest, AssertErrorMessage): def setUp(self): @@ -5844,13 +6216,28 @@ def test_emit_after_closing_in_write_mode(self): self.assertEqual(fp.read().strip(), '1') class RotatingFileHandlerTest(BaseFileTest): - @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") def test_should_not_rollover(self): - # If maxbytes is zero rollover never occurs + # If file is empty rollover never occurs + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=1) + self.assertFalse(rh.shouldRollover(None)) + rh.close() + + # If maxBytes is zero rollover never occurs + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=0) + self.assertFalse(rh.shouldRollover(None)) + rh.close() + + with open(self.fn, 'wb') as f: + f.write(b'\n') rh = logging.handlers.RotatingFileHandler( self.fn, encoding="utf-8", maxBytes=0) self.assertFalse(rh.shouldRollover(None)) rh.close() + + @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") + def test_should_not_rollover_non_file(self): # bpo-45401 - test with special file # We set maxBytes to 1 so that rollover would normally happen, except # for the check for regular files @@ -5860,18 +6247,47 @@ def test_should_not_rollover(self): rh.close() def test_should_rollover(self): - rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8", maxBytes=1) + with open(self.fn, 'wb') as f: + f.write(b'\n') + rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8", maxBytes=2) self.assertTrue(rh.shouldRollover(self.next_rec())) rh.close() def test_file_created(self): # checks that the file is created and assumes it was created # by us + os.unlink(self.fn) rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8") rh.emit(self.next_rec()) self.assertLogFile(self.fn) rh.close() + def test_max_bytes(self, delay=False): + kwargs = {'delay': delay} if delay else {} + os.unlink(self.fn) + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", backupCount=2, maxBytes=100, **kwargs) + self.assertIs(os.path.exists(self.fn), not delay) + small = logging.makeLogRecord({'msg': 'a'}) + large = logging.makeLogRecord({'msg': 'b'*100}) + self.assertFalse(rh.shouldRollover(small)) + self.assertFalse(rh.shouldRollover(large)) + rh.emit(small) + self.assertLogFile(self.fn) + self.assertFalse(os.path.exists(self.fn + ".1")) + self.assertFalse(rh.shouldRollover(small)) + self.assertTrue(rh.shouldRollover(large)) + rh.emit(large) + self.assertTrue(os.path.exists(self.fn)) + self.assertLogFile(self.fn + ".1") + self.assertFalse(os.path.exists(self.fn + ".2")) + self.assertTrue(rh.shouldRollover(small)) + self.assertTrue(rh.shouldRollover(large)) + rh.close() + + def test_max_bytes_delay(self): + self.test_max_bytes(delay=True) + def test_rollover_filenames(self): def namer(name): return name + ".test" @@ -5880,11 +6296,15 @@ def namer(name): rh.namer = namer rh.emit(self.next_rec()) self.assertLogFile(self.fn) + self.assertFalse(os.path.exists(namer(self.fn + ".1"))) rh.emit(self.next_rec()) self.assertLogFile(namer(self.fn + ".1")) + self.assertFalse(os.path.exists(namer(self.fn + ".2"))) rh.emit(self.next_rec()) self.assertLogFile(namer(self.fn + ".2")) self.assertFalse(os.path.exists(namer(self.fn + ".3"))) + rh.emit(self.next_rec()) + self.assertFalse(os.path.exists(namer(self.fn + ".3"))) rh.close() def test_namer_rotator_inheritance(self): @@ -6005,6 +6425,52 @@ def test_rollover(self): print(tf.read()) self.assertTrue(found, msg=msg) + def test_rollover_at_midnight(self, weekly=False): + os_helper.unlink(self.fn) + now = datetime.datetime.now() + atTime = now.time() + if not 0.1 < atTime.microsecond/1e6 < 0.9: + # The test requires all records to be emitted within + # the range of the same whole second. + time.sleep((0.1 - atTime.microsecond/1e6) % 1.0) + now = datetime.datetime.now() + atTime = now.time() + atTime = atTime.replace(microsecond=0) + fmt = logging.Formatter('%(asctime)s %(message)s') + when = f'W{now.weekday()}' if weekly else 'MIDNIGHT' + for i in range(3): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) + fh.emit(r2) + fh.close() + self.assertLogFile(self.fn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + + os.utime(self.fn, (now.timestamp() - 1,)*2) + for i in range(2): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) + fh.emit(r2) + fh.close() + rolloverDate = now - datetime.timedelta(days=7 if weekly else 1) + otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' + self.assertLogFile(otherfn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing2 {i}', line) + with open(otherfn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + + def test_rollover_at_weekday(self): + self.test_rollover_at_midnight(weekly=True) + def test_invalid(self): assertRaises = self.assertRaises assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, @@ -6014,22 +6480,47 @@ def test_invalid(self): assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, self.fn, 'W7', encoding="utf-8", delay=True) + # TODO: Test for utc=False. def test_compute_rollover_daily_attime(self): currentTime = 0 + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', + utc=True, atTime=None) + try: + actual = rh.computeRollover(currentTime) + self.assertEqual(actual, currentTime + 24 * 60 * 60) + + actual = rh.computeRollover(currentTime + 24 * 60 * 60 - 1) + self.assertEqual(actual, currentTime + 24 * 60 * 60) + + actual = rh.computeRollover(currentTime + 24 * 60 * 60) + self.assertEqual(actual, currentTime + 48 * 60 * 60) + + actual = rh.computeRollover(currentTime + 25 * 60 * 60) + self.assertEqual(actual, currentTime + 48 * 60 * 60) + finally: + rh.close() + atTime = datetime.time(12, 0, 0) rh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when='MIDNIGHT', interval=1, backupCount=0, + self.fn, encoding="utf-8", when='MIDNIGHT', utc=True, atTime=atTime) try: actual = rh.computeRollover(currentTime) self.assertEqual(actual, currentTime + 12 * 60 * 60) + actual = rh.computeRollover(currentTime + 12 * 60 * 60 - 1) + self.assertEqual(actual, currentTime + 12 * 60 * 60) + + actual = rh.computeRollover(currentTime + 12 * 60 * 60) + self.assertEqual(actual, currentTime + 36 * 60 * 60) + actual = rh.computeRollover(currentTime + 13 * 60 * 60) self.assertEqual(actual, currentTime + 36 * 60 * 60) finally: rh.close() - #@unittest.skipIf(True, 'Temporarily skipped while failures investigated.') + # TODO: Test for utc=False. def test_compute_rollover_weekly_attime(self): currentTime = int(time.time()) today = currentTime - currentTime % 86400 @@ -6054,14 +6545,28 @@ def test_compute_rollover_weekly_attime(self): expected += 12 * 60 * 60 # Add in adjustment for today expected += today + actual = rh.computeRollover(today) if actual != expected: print('failed in timezone: %d' % time.timezone) print('local vars: %s' % locals()) self.assertEqual(actual, expected) + + actual = rh.computeRollover(today + 12 * 60 * 60 - 1) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + if day == wday: # goes into following week expected += 7 * 24 * 60 * 60 + actual = rh.computeRollover(today + 12 * 60 * 60) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + actual = rh.computeRollover(today + 13 * 60 * 60) if actual != expected: print('failed in timezone: %d' % time.timezone) @@ -6079,7 +6584,7 @@ def test_compute_files_to_delete(self): for i in range(10): times.append(dt.strftime('%Y-%m-%d_%H-%M-%S')) dt += datetime.timedelta(seconds=5) - prefixes = ('a.b', 'a.b.c', 'd.e', 'd.e.f') + prefixes = ('a.b', 'a.b.c', 'd.e', 'd.e.f', 'g') files = [] rotators = [] for prefix in prefixes: @@ -6092,10 +6597,22 @@ def test_compute_files_to_delete(self): if prefix.startswith('a.b'): for t in times: files.append('%s.log.%s' % (prefix, t)) - else: - rotator.namer = lambda name: name.replace('.log', '') + '.log' + elif prefix.startswith('d.e'): + def namer(filename): + dirname, basename = os.path.split(filename) + basename = basename.replace('.log', '') + '.log' + return os.path.join(dirname, basename) + rotator.namer = namer for t in times: files.append('%s.%s.log' % (prefix, t)) + elif prefix == 'g': + def namer(filename): + dirname, basename = os.path.split(filename) + basename = 'g' + basename[6:] + '.oldlog' + return os.path.join(dirname, basename) + rotator.namer = namer + for t in times: + files.append('g%s.oldlog' % t) # Create empty files for fn in files: p = os.path.join(wd, fn) @@ -6105,18 +6622,423 @@ def test_compute_files_to_delete(self): for i, prefix in enumerate(prefixes): rotator = rotators[i] candidates = rotator.getFilesToDelete() - self.assertEqual(len(candidates), 3) + self.assertEqual(len(candidates), 3, candidates) if prefix.startswith('a.b'): p = '%s.log.' % prefix for c in candidates: d, fn = os.path.split(c) self.assertTrue(fn.startswith(p)) - else: + elif prefix.startswith('d.e'): for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.endswith('.log')) + self.assertTrue(fn.endswith('.log'), fn) self.assertTrue(fn.startswith(prefix + '.') and fn[len(prefix) + 2].isdigit()) + elif prefix == 'g': + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.endswith('.oldlog')) + self.assertTrue(fn.startswith('g') and fn[1].isdigit()) + + def test_compute_files_to_delete_same_filename_different_extensions(self): + # See GH-93205 for background + wd = pathlib.Path(tempfile.mkdtemp(prefix='test_logging_')) + self.addCleanup(shutil.rmtree, wd) + times = [] + dt = datetime.datetime.now() + n_files = 10 + for _ in range(n_files): + times.append(dt.strftime('%Y-%m-%d_%H-%M-%S')) + dt += datetime.timedelta(seconds=5) + prefixes = ('a.log', 'a.log.b') + files = [] + rotators = [] + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = logging.handlers.TimedRotatingFileHandler(wd / prefix, when='s', + interval=5, + backupCount=backupCount, + delay=True) + rotators.append(rotator) + for t in times: + files.append('%s.%s' % (prefix, t)) + for t in times: + files.append('a.log.%s.c' % t) + # Create empty files + for f in files: + (wd / f).touch() + # Now the checks that only the correct files are offered up for deletion + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = rotators[i] + candidates = rotator.getFilesToDelete() + self.assertEqual(len(candidates), n_files - backupCount, candidates) + matcher = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\Z") + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.startswith(prefix+'.')) + suffix = fn[(len(prefix)+1):] + self.assertRegex(suffix, matcher) + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_MIDNIGHT_local(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False) + + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 12, 0, 0)) + + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 5, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 10, 11, 59, 59), DT(2012, 3, 10, 12, 0)) + test(DT(2012, 3, 10, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 10, 13, 0), DT(2012, 3, 11, 12, 0)) + + test(DT(2012, 11, 3, 11, 59, 59), DT(2012, 11, 3, 12, 0)) + test(DT(2012, 11, 3, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 3, 13, 0), DT(2012, 11, 4, 12, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(2, 0, 0)) + + test(DT(2012, 3, 10, 1, 59, 59), DT(2012, 3, 10, 2, 0)) + # 2:00:00 is the same as 3:00:00 at 2012-3-11. + test(DT(2012, 3, 10, 2, 0), DT(2012, 3, 11, 3, 0)) + test(DT(2012, 3, 10, 3, 0), DT(2012, 3, 11, 3, 0)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 0)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 2, 0)) + test(DT(2012, 3, 11, 4, 0), DT(2012, 3, 12, 2, 0)) + + test(DT(2012, 11, 3, 1, 59, 59), DT(2012, 11, 3, 2, 0)) + test(DT(2012, 11, 3, 2, 0), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 3, 3, 0), DT(2012, 11, 4, 2, 0)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 5, 2, 0)) + test(DT(2012, 11, 4, 3, 0), DT(2012, 11, 5, 2, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(2, 30, 0)) + + test(DT(2012, 3, 10, 2, 29, 59), DT(2012, 3, 10, 2, 30)) + # No time 2:30:00 at 2012-3-11. + test(DT(2012, 3, 10, 2, 30), DT(2012, 3, 11, 3, 30)) + test(DT(2012, 3, 10, 3, 0), DT(2012, 3, 11, 3, 30)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 2, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 12, 2, 30)) + + test(DT(2012, 11, 3, 2, 29, 59), DT(2012, 11, 3, 2, 30)) + test(DT(2012, 11, 3, 2, 30), DT(2012, 11, 4, 2, 30)) + test(DT(2012, 11, 3, 3, 0), DT(2012, 11, 4, 2, 30)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(1, 30, 0)) + + test(DT(2012, 3, 11, 1, 29, 59), DT(2012, 3, 11, 1, 30)) + test(DT(2012, 3, 11, 1, 30), DT(2012, 3, 12, 1, 30)) + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 12, 1, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 1, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 12, 1, 30)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 29, 59), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 30), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 5, 1, 30)) + # It is weird, but the rollover date jumps back from 2012-11-5 + # to 2012-11-4. + test(DT(2012, 11, 4, 1, 0, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 29, 59, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 30, fold=1), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 2, 30), DT(2012, 11, 5, 1, 30)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_W6_local(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False) + + test(DT(2012, 3, 4, 23, 59, 59), DT(2012, 3, 5, 0, 0)) + test(DT(2012, 3, 5, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 5, 1, 0), DT(2012, 3, 12, 0, 0)) + + test(DT(2012, 10, 28, 23, 59, 59), DT(2012, 10, 29, 0, 0)) + test(DT(2012, 10, 29, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 29, 1, 0), DT(2012, 11, 5, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(0, 0, 0)) + + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 18, 0, 0)) + + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 11, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 4, 11, 59, 59), DT(2012, 3, 4, 12, 0)) + test(DT(2012, 3, 4, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 4, 13, 0), DT(2012, 3, 11, 12, 0)) + + test(DT(2012, 10, 28, 11, 59, 59), DT(2012, 10, 28, 12, 0)) + test(DT(2012, 10, 28, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 28, 13, 0), DT(2012, 11, 4, 12, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(2, 0, 0)) + + test(DT(2012, 3, 4, 1, 59, 59), DT(2012, 3, 4, 2, 0)) + # 2:00:00 is the same as 3:00:00 at 2012-3-11. + test(DT(2012, 3, 4, 2, 0), DT(2012, 3, 11, 3, 0)) + test(DT(2012, 3, 4, 3, 0), DT(2012, 3, 11, 3, 0)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 0)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 2, 0)) + test(DT(2012, 3, 11, 4, 0), DT(2012, 3, 18, 2, 0)) + + test(DT(2012, 10, 28, 1, 59, 59), DT(2012, 10, 28, 2, 0)) + test(DT(2012, 10, 28, 2, 0), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 10, 28, 3, 0), DT(2012, 11, 4, 2, 0)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 11, 2, 0)) + test(DT(2012, 11, 4, 3, 0), DT(2012, 11, 11, 2, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(2, 30, 0)) + + test(DT(2012, 3, 4, 2, 29, 59), DT(2012, 3, 4, 2, 30)) + # No time 2:30:00 at 2012-3-11. + test(DT(2012, 3, 4, 2, 30), DT(2012, 3, 11, 3, 30)) + test(DT(2012, 3, 4, 3, 0), DT(2012, 3, 11, 3, 30)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 2, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 18, 2, 30)) + + test(DT(2012, 10, 28, 2, 29, 59), DT(2012, 10, 28, 2, 30)) + test(DT(2012, 10, 28, 2, 30), DT(2012, 11, 4, 2, 30)) + test(DT(2012, 10, 28, 3, 0), DT(2012, 11, 4, 2, 30)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(1, 30, 0)) + + test(DT(2012, 3, 11, 1, 29, 59), DT(2012, 3, 11, 1, 30)) + test(DT(2012, 3, 11, 1, 30), DT(2012, 3, 18, 1, 30)) + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 18, 1, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 1, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 18, 1, 30)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 29, 59), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 30), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 11, 1, 30)) + # It is weird, but the rollover date jumps back from 2012-11-11 + # to 2012-11-4. + test(DT(2012, 11, 4, 1, 0, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 29, 59, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 30, fold=1), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 2, 30), DT(2012, 11, 11, 1, 30)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_MIDNIGHT_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3) + + test(DT(2012, 3, 8, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 9, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 9, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 13, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 14, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 14, 0, 0)) + + test(DT(2012, 11, 1, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 2, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 2, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 6, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 7, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 7, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 8, 11, 59, 59), DT(2012, 3, 10, 12, 0)) + test(DT(2012, 3, 8, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 8, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 10, 11, 59, 59), DT(2012, 3, 12, 12, 0)) + test(DT(2012, 3, 10, 12, 0), DT(2012, 3, 13, 12, 0)) + test(DT(2012, 3, 10, 13, 0), DT(2012, 3, 13, 12, 0)) + + test(DT(2012, 11, 1, 11, 59, 59), DT(2012, 11, 3, 12, 0)) + test(DT(2012, 11, 1, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 1, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 3, 11, 59, 59), DT(2012, 11, 5, 12, 0)) + test(DT(2012, 11, 3, 12, 0), DT(2012, 11, 6, 12, 0)) + test(DT(2012, 11, 3, 13, 0), DT(2012, 11, 6, 12, 0)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_W6_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3) + + test(DT(2012, 2, 19, 23, 59, 59), DT(2012, 3, 5, 0, 0)) + test(DT(2012, 2, 20, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 2, 20, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 4, 23, 59, 59), DT(2012, 3, 19, 0, 0)) + test(DT(2012, 3, 5, 0, 0), DT(2012, 3, 26, 0, 0)) + test(DT(2012, 3, 5, 1, 0), DT(2012, 3, 26, 0, 0)) + + test(DT(2012, 10, 14, 23, 59, 59), DT(2012, 10, 29, 0, 0)) + test(DT(2012, 10, 15, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 15, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 28, 23, 59, 59), DT(2012, 11, 12, 0, 0)) + test(DT(2012, 10, 29, 0, 0), DT(2012, 11, 19, 0, 0)) + test(DT(2012, 10, 29, 1, 0), DT(2012, 11, 19, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(0, 0, 0)) + + test(DT(2012, 2, 25, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 2, 26, 0, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 2, 26, 1, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 25, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 4, 1, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 4, 1, 0, 0)) + + test(DT(2012, 10, 20, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 10, 21, 0, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 10, 21, 1, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 18, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 25, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 25, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 2, 18, 11, 59, 59), DT(2012, 3, 4, 12, 0)) + test(DT(2012, 2, 19, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 2, 19, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 4, 11, 59, 59), DT(2012, 3, 18, 12, 0)) + test(DT(2012, 3, 4, 12, 0), DT(2012, 3, 25, 12, 0)) + test(DT(2012, 3, 4, 13, 0), DT(2012, 3, 25, 12, 0)) + + test(DT(2012, 10, 14, 11, 59, 59), DT(2012, 10, 28, 12, 0)) + test(DT(2012, 10, 14, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 14, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 28, 11, 59, 59), DT(2012, 11, 11, 12, 0)) + test(DT(2012, 10, 28, 12, 0), DT(2012, 11, 18, 12, 0)) + test(DT(2012, 10, 28, 13, 0), DT(2012, 11, 18, 12, 0)) + + fh.close() def secs(**kw): @@ -6130,40 +7052,49 @@ def secs(**kw): # current time (epoch start) is a Thursday, W0 means Monday ('W0', secs(days=4, hours=24)), ): - def test_compute_rollover(self, when=when, exp=exp): - rh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=when, interval=1, backupCount=0, utc=True) - currentTime = 0.0 - actual = rh.computeRollover(currentTime) - if exp != actual: - # Failures occur on some systems for MIDNIGHT and W0. - # Print detailed calculation for MIDNIGHT so we can try to see - # what's going on - if when == 'MIDNIGHT': - try: - if rh.utc: - t = time.gmtime(currentTime) - else: - t = time.localtime(currentTime) - currentHour = t[3] - currentMinute = t[4] - currentSecond = t[5] - # r is the number of seconds left between now and midnight - r = logging.handlers._MIDNIGHT - ((currentHour * 60 + - currentMinute) * 60 + - currentSecond) - result = currentTime + r - print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) - print('currentHour: %s' % currentHour, file=sys.stderr) - print('currentMinute: %s' % currentMinute, file=sys.stderr) - print('currentSecond: %s' % currentSecond, file=sys.stderr) - print('r: %s' % r, file=sys.stderr) - print('result: %s' % result, file=sys.stderr) - except Exception as e: - print('exception in diagnostic code: %s' % e, file=sys.stderr) - self.assertEqual(exp, actual) - rh.close() - setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover) + for interval in 1, 3: + def test_compute_rollover(self, when=when, interval=interval, exp=exp): + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, interval=interval, backupCount=0, utc=True) + currentTime = 0.0 + actual = rh.computeRollover(currentTime) + if when.startswith('W'): + exp += secs(days=7*(interval-1)) + else: + exp *= interval + if exp != actual: + # Failures occur on some systems for MIDNIGHT and W0. + # Print detailed calculation for MIDNIGHT so we can try to see + # what's going on + if when == 'MIDNIGHT': + try: + if rh.utc: + t = time.gmtime(currentTime) + else: + t = time.localtime(currentTime) + currentHour = t[3] + currentMinute = t[4] + currentSecond = t[5] + # r is the number of seconds left between now and midnight + r = logging.handlers._MIDNIGHT - ((currentHour * 60 + + currentMinute) * 60 + + currentSecond) + result = currentTime + r + print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) + print('currentHour: %s' % currentHour, file=sys.stderr) + print('currentMinute: %s' % currentMinute, file=sys.stderr) + print('currentSecond: %s' % currentSecond, file=sys.stderr) + print('r: %s' % r, file=sys.stderr) + print('result: %s' % result, file=sys.stderr) + except Exception as e: + print('exception in diagnostic code: %s' % e, file=sys.stderr) + self.assertEqual(exp, actual) + rh.close() + name = "test_compute_rollover_%s" % when + if interval > 1: + name += "_interval" + test_compute_rollover.__name__ = name + setattr(TimedRotatingFileHandlerTest, name, test_compute_rollover) @unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.') diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index d299c34cec076d..19978118c80dba 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -386,15 +386,6 @@ def __long__(self): return 42 self.assertRaises(TypeError, int, JustLong()) - class LongTrunc: - # __long__ should be ignored in 3.x - def __long__(self): - return 42 - def __trunc__(self): - return 1729 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(LongTrunc()), 1729) - def check_float_conversion(self, n): # Check that int -> float conversion behaviour matches # that of the pure Python version above. @@ -482,6 +473,12 @@ def test_float_conversion(self): self.check_float_conversion(value) self.check_float_conversion(-value) + @support.requires_IEEE_754 + @support.bigmemtest(2**32, memuse=0.2) + def test_float_conversion_huge_integer(self, size): + v = 1 << size + self.assertRaises(OverflowError, float, v) + def test_float_overflow(self): for x in -2.0, -1.0, 0.0, 1.0, 2.0: self.assertEqual(float(int(x)), x) @@ -623,6 +620,56 @@ def __lt__(self, other): eq(x > y, Rcmp > 0) eq(x >= y, Rcmp >= 0) + @support.requires_IEEE_754 + @support.bigmemtest(2**32, memuse=0.2) + def test_mixed_compares_huge_integer(self, size): + v = 1 << size + f = sys.float_info.max + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, True) + self.assertIs(f <= v, True) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + f = float('inf') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, True) + self.assertIs(f >= v, True) + f = float('nan') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + + del v + v = (-1) << size + f = -sys.float_info.max + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, True) + self.assertIs(f >= v, True) + f = float('-inf') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, True) + self.assertIs(f <= v, True) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + f = float('nan') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + def test__format__(self): self.assertEqual(format(123456789, 'd'), '123456789') self.assertEqual(format(123456789, 'd'), '123456789') @@ -942,9 +989,12 @@ def test_huge_lshift_of_zero(self): self.assertEqual(0 << (sys.maxsize + 1), 0) @support.cpython_only - @support.bigmemtest(sys.maxsize + 1000, memuse=2/15 * 2, dry_run=False) + @support.bigmemtest(2**32, memuse=0.2) def test_huge_lshift(self, size): - self.assertEqual(1 << (sys.maxsize + 1000), 1 << 1000 << sys.maxsize) + v = 5 << size + self.assertEqual(v.bit_length(), size + 3) + self.assertEqual(v.bit_count(), 2) + self.assertEqual(v >> size, 5) def test_huge_rshift(self): huge_shift = 1 << 1000 @@ -956,11 +1006,13 @@ def test_huge_rshift(self): self.assertEqual(-2**128 >> huge_shift, -1) @support.cpython_only - @support.bigmemtest(sys.maxsize + 500, memuse=2/15, dry_run=False) + @support.bigmemtest(2**32, memuse=0.2) def test_huge_rshift_of_huge(self, size): - huge = ((1 << 500) + 11) << sys.maxsize - self.assertEqual(huge >> (sys.maxsize + 1), (1 << 499) + 5) - self.assertEqual(huge >> (sys.maxsize + 1000), 0) + huge = ((1 << 500) + 11) << size + self.assertEqual(huge.bit_length(), size + 501) + self.assertEqual(huge.bit_count(), 4) + self.assertEqual(huge >> (size + 1), (1 << 499) + 5) + self.assertEqual(huge >> (size + 1000), 0) def test_small_rshift(self): self.assertEqual(42 >> 1, 21) @@ -1639,6 +1691,8 @@ class MyInt(int): MyInt.__basicsize__ + MyInt.__itemsize__ * ndigits ) + # GH-117195 -- This shouldn't crash + object.__sizeof__(1) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 65e6488c5d7b10..22478c14fb4a65 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -2,7 +2,6 @@ import array from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE import os -import pathlib import pickle import random import sys @@ -12,7 +11,7 @@ from test.support import _4G, bigmemtest from test.support.import_helper import import_module from test.support.os_helper import ( - TESTFN, unlink + TESTFN, unlink, FakePath ) lzma = import_module("lzma") @@ -548,23 +547,29 @@ def test_init(self): pass def test_init_with_PathLike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename, COMPRESSED_XZ): with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) with LZMAFile(filename, "a") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT * 2) + self.assertEqual(f.name, TESTFN) def test_init_with_filename(self): with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(TESTFN) as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'rb') with LZMAFile(TESTFN, "w") as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') def test_init_mode(self): with TempFile(TESTFN): @@ -585,11 +590,11 @@ def test_init_with_x_mode(self): self.addCleanup(unlink, TESTFN) for mode in ("x", "xb"): unlink(TESTFN) - with LZMAFile(TESTFN, mode): + with LZMAFile(TESTFN, mode) as f: pass + self.assertEqual(f.mode, 'wb') with self.assertRaises(FileExistsError): - with LZMAFile(TESTFN, mode): - pass + LZMAFile(TESTFN, mode) def test_init_bad_mode(self): with self.assertRaises(ValueError): @@ -867,16 +872,74 @@ def test_read_from_file(self): with LZMAFile(TESTFN) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, TESTFN) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) def test_read_from_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(bytes_filename) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, bytes_filename) + + def test_read_from_fileobj(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with open(TESTFN, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_from_fileobj_with_int_name(self): + with TempFile(TESTFN, COMPRESSED_XZ): + fd = os.open(TESTFN, os.O_RDONLY) + with open(fd, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) def test_read_incomplete(self): with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f: @@ -1005,6 +1068,8 @@ def test_write(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: f.write(INPUT) + with self.assertRaises(AttributeError): + f.name expected = lzma.compress(INPUT) self.assertEqual(dst.getvalue(), expected) with BytesIO() as dst: @@ -1041,16 +1106,35 @@ def test_write_append(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: f.write(part1) + self.assertEqual(f.mode, 'wb') with LZMAFile(dst, "a") as f: f.write(part2) + self.assertEqual(f.mode, 'wb') with LZMAFile(dst, "a") as f: f.write(part3) + self.assertEqual(f.mode, 'wb') self.assertEqual(dst.getvalue(), expected) def test_write_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + expected = lzma.compress(INPUT) with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) @@ -1058,13 +1142,66 @@ def test_write_to_file(self): unlink(TESTFN) def test_write_to_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) try: with LZMAFile(bytes_filename, "w") as f: f.write(INPUT) + self.assertEqual(f.name, bytes_filename) + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj(self): + try: + with open(TESTFN, "wb") as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj_with_int_name(self): + try: + fd = os.open(TESTFN, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + expected = lzma.compress(INPUT) with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) @@ -1079,10 +1216,13 @@ def test_write_append_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(part1) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: f.write(part2) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: f.write(part3) + self.assertEqual(f.mode, 'wb') with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) finally: @@ -1276,15 +1416,17 @@ def test_filename(self): self.assertEqual(f.read(), INPUT * 2) def test_with_pathlike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename): with lzma.open(filename, "wb") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) with open(filename, "rb") as f: file_data = lzma.decompress(f.read()) self.assertEqual(file_data, INPUT) with lzma.open(filename, "rb") as f: self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) def test_bad_params(self): # Test invalid parameter combinations. diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index c52c014185bec7..3ecb5eab26d4b9 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -1,7 +1,6 @@ import os import sys import time -import stat import socket import email import email.message @@ -9,7 +8,9 @@ import io import tempfile from test import support +from test.support import import_helper from test.support import os_helper +from test.support import refleak_helper from test.support import socket_helper import unittest import textwrap @@ -701,8 +702,7 @@ def _check_basics(self, factory=None): self.assertEqual(self._box._factory, factory) for subdir in '', 'tmp', 'new', 'cur': path = os.path.join(self._path, subdir) - mode = os.stat(path)[stat.ST_MODE] - self.assertTrue(stat.S_ISDIR(mode), "Not a directory: '%s'" % path) + self.assertTrue(os.path.isdir(path), f"Not a directory: {path!r}") def test_list_folders(self): # List folders @@ -1081,6 +1081,47 @@ def test_permissions_after_flush(self): self.assertEqual(os.stat(self._path).st_mode, mode) + @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') + def test_ownership_after_flush(self): + # See issue gh-117467 + + pwd = import_helper.import_module('pwd') + grp = import_helper.import_module('grp') + st = os.stat(self._path) + + for e in pwd.getpwall(): + if e.pw_uid != st.st_uid: + other_uid = e.pw_uid + break + else: + self.skipTest("test needs more than one user") + + for e in grp.getgrall(): + if e.gr_gid != st.st_gid: + other_gid = e.gr_gid + break + else: + self.skipTest("test needs more than one group") + + try: + os.chown(self._path, other_uid, other_gid) + except OSError: + self.skipTest('test needs root privilege') + # Change permissions as in test_permissions_after_flush. + mode = st.st_mode | 0o666 + os.chmod(self._path, mode) + + self._box.add(self._template % 0) + i = self._box.add(self._template % 1) + # Need to remove one message to make flush() create a new file + self._box.remove(i) + self._box.flush() + + st = os.stat(self._path) + self.assertEqual(st.st_uid, other_uid) + self.assertEqual(st.st_gid, other_gid) + self.assertEqual(st.st_mode, mode) + class _TestMboxMMDF(_TestSingleFile): @@ -2443,6 +2484,9 @@ def test__all__(self): def tearDownModule(): support.reap_children() + # reap_children may have re-populated caches: + if refleak_helper.hunting_for_refleaks(): + sys._clear_internal_caches() if __name__ == '__main__': diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 615568e6af2102..64ee1ba867d592 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -118,7 +118,7 @@ def test_code(self): def test_many_codeobjects(self): # Issue2957: bad recursion count on code objects # more than MAX_MARSHAL_STACK_DEPTH - count = support.EXCEEDS_RECURSION_LIMIT + count = support.exceeds_recursion_limit() codes = (ExceptionTestCase.test_exceptions.__code__,) * count marshal.loads(marshal.dumps(codes)) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index ad382fc2b59891..fecafd53aa6e6f 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -187,6 +187,9 @@ def result_check(expected, got, ulp_tol=5, abs_tol=0.0): # Check exactly equal (applies also to strings representing exceptions) if got == expected: + if not got and not expected: + if math.copysign(1, got) != math.copysign(1, expected): + return f"expected {expected}, got {got} (zero has wrong sign)" return None failure = "not equal" @@ -657,7 +660,7 @@ def testFsum(self): def msum(iterable): """Full precision summation. Compute sum(iterable) without any intermediate accumulation of error. Based on the 'lsum' function - at http://code.activestate.com/recipes/393090/ + at https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/ """ tmant, texp = 0, 0 @@ -809,11 +812,13 @@ def testHypot(self): # Test allowable types (those with __float__) self.assertEqual(hypot(12.0, 5.0), 13.0) self.assertEqual(hypot(12, 5), 13) - self.assertEqual(hypot(1, -1), math.sqrt(2)) - self.assertEqual(hypot(1, FloatLike(-1.)), math.sqrt(2)) + self.assertEqual(hypot(0.75, -1), 1.25) + self.assertEqual(hypot(-1, 0.75), 1.25) + self.assertEqual(hypot(0.75, FloatLike(-1.)), 1.25) + self.assertEqual(hypot(FloatLike(-1.), 0.75), 1.25) self.assertEqual(hypot(Decimal(12), Decimal(5)), 13) self.assertEqual(hypot(Fraction(12, 32), Fraction(5, 32)), Fraction(13, 32)) - self.assertEqual(hypot(bool(1), bool(0), bool(1), bool(1)), math.sqrt(3)) + self.assertEqual(hypot(True, False, True, True, True), 2.0) # Test corner cases self.assertEqual(hypot(0.0, 0.0), 0.0) # Max input is zero @@ -969,9 +974,9 @@ def testDist(self): self.assertEqual(dist((D(14), D(1)), (D(2), D(-4))), D(13)) self.assertEqual(dist((F(14, 32), F(1, 32)), (F(2, 32), F(-4, 32))), F(13, 32)) - self.assertEqual(dist((True, True, False, True, False), - (True, False, True, True, False)), - sqrt(2.0)) + self.assertEqual(dist((True, True, False, False, True, True), + (True, False, True, False, False, False)), + 2.0) # Test corner cases self.assertEqual(dist((13.25, 12.5, -3.25), @@ -1120,6 +1125,15 @@ def __index__(self): with self.assertRaises(TypeError): math.isqrt(value) + @support.bigmemtest(2**32, memuse=0.85) + def test_isqrt_huge(self, size): + if size & 1: + size += 1 + v = 1 << size + w = math.isqrt(v) + self.assertEqual(w.bit_length(), size // 2 + 1) + self.assertEqual(w.bit_count(), 1) + def test_lcm(self): lcm = math.lcm self.assertEqual(lcm(0, 0), 0) @@ -1261,6 +1275,13 @@ def testLog10(self): self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + @support.bigmemtest(2**32, memuse=0.2) + def test_log_huge_integer(self, size): + v = 1 << size + self.assertAlmostEqual(math.log2(v), size) + self.assertAlmostEqual(math.log(v), size * 0.6931471805599453) + self.assertAlmostEqual(math.log10(v), size * 0.3010299956639812) + def testSumProd(self): sumprod = math.sumprod Decimal = decimal.Decimal @@ -1397,7 +1418,7 @@ def __repr__(self): return f'Flt({int(self)})' def baseline_sumprod(p, q): - """This defines the target behavior including expections and special values. + """This defines the target behavior including exceptions and special values. However, it is subject to rounding errors, so float inputs should be exactly representable with only a few bits. """ @@ -1881,7 +1902,7 @@ def testTan(self): try: self.assertTrue(math.isnan(math.tan(INF))) self.assertTrue(math.isnan(math.tan(NINF))) - except: + except ValueError: self.assertRaises(ValueError, math.tan, INF) self.assertRaises(ValueError, math.tan, NINF) self.assertTrue(math.isnan(math.tan(NAN))) @@ -2051,6 +2072,13 @@ def test_testfile(self): except OverflowError: result = 'OverflowError' + # C99+ says for math.h's sqrt: If the argument is +∞ or ±0, it is + # returned, unmodified. On another hand, for csqrt: If z is ±0+0i, + # the result is +0+0i. Lets correct zero sign of er to follow + # first convention. + if id in ['sqrt0002', 'sqrt0003', 'sqrt1001', 'sqrt1023']: + er = math.copysign(er, ar) + # Default tolerances ulp_tol, abs_tol = 5, 0.0 @@ -2613,6 +2641,244 @@ def test_fractions(self): self.assertAllNotClose(fraction_examples, rel_tol=1e-9) +class FMATests(unittest.TestCase): + """ Tests for math.fma. """ + + def test_fma_nan_results(self): + # Selected representative values. + values = [ + -math.inf, -1e300, -2.3, -1e-300, -0.0, + 0.0, 1e-300, 2.3, 1e300, math.inf, math.nan + ] + + # If any input is a NaN, the result should be a NaN, too. + for a, b in itertools.product(values, repeat=2): + self.assertIsNaN(math.fma(math.nan, a, b)) + self.assertIsNaN(math.fma(a, math.nan, b)) + self.assertIsNaN(math.fma(a, b, math.nan)) + + def test_fma_infinities(self): + # Cases involving infinite inputs or results. + positives = [1e-300, 2.3, 1e300, math.inf] + finites = [-1e300, -2.3, -1e-300, -0.0, 0.0, 1e-300, 2.3, 1e300] + non_nans = [-math.inf, -2.3, -0.0, 0.0, 2.3, math.inf] + + # ValueError due to inf * 0 computation. + for c in non_nans: + for infinity in [math.inf, -math.inf]: + for zero in [0.0, -0.0]: + with self.assertRaises(ValueError): + math.fma(infinity, zero, c) + with self.assertRaises(ValueError): + math.fma(zero, infinity, c) + + # ValueError when a*b and c both infinite of opposite signs. + for b in positives: + with self.assertRaises(ValueError): + math.fma(math.inf, b, -math.inf) + with self.assertRaises(ValueError): + math.fma(math.inf, -b, math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, -b, -math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, b, math.inf) + with self.assertRaises(ValueError): + math.fma(b, math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(-b, math.inf, math.inf) + with self.assertRaises(ValueError): + math.fma(-b, -math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(b, -math.inf, math.inf) + + # Infinite result when a*b and c both infinite of the same sign. + for b in positives: + self.assertEqual(math.fma(math.inf, b, math.inf), math.inf) + self.assertEqual(math.fma(math.inf, -b, -math.inf), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, math.inf), math.inf) + self.assertEqual(math.fma(-math.inf, b, -math.inf), -math.inf) + self.assertEqual(math.fma(b, math.inf, math.inf), math.inf) + self.assertEqual(math.fma(-b, math.inf, -math.inf), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, math.inf), math.inf) + self.assertEqual(math.fma(b, -math.inf, -math.inf), -math.inf) + + # Infinite result when a*b finite, c infinite. + for a, b in itertools.product(finites, finites): + self.assertEqual(math.fma(a, b, math.inf), math.inf) + self.assertEqual(math.fma(a, b, -math.inf), -math.inf) + + # Infinite result when a*b infinite, c finite. + for b, c in itertools.product(positives, finites): + self.assertEqual(math.fma(math.inf, b, c), math.inf) + self.assertEqual(math.fma(-math.inf, b, c), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, c), math.inf) + self.assertEqual(math.fma(math.inf, -b, c), -math.inf) + + self.assertEqual(math.fma(b, math.inf, c), math.inf) + self.assertEqual(math.fma(b, -math.inf, c), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, c), math.inf) + self.assertEqual(math.fma(-b, math.inf, c), -math.inf) + + # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 + # properly: it doesn't use the right sign when the result is zero. + @unittest.skipIf( + sys.platform.startswith(("freebsd", "wasi", "netbsd")) + or (sys.platform == "android" and platform.machine() == "x86_64"), + f"this platform doesn't implement IEE 754-2008 properly") + def test_fma_zero_result(self): + nonnegative_finites = [0.0, 1e-300, 2.3, 1e300] + + # Zero results from exact zero inputs. + for b in nonnegative_finites: + self.assertIsPositiveZero(math.fma(0.0, b, 0.0)) + self.assertIsPositiveZero(math.fma(0.0, b, -0.0)) + self.assertIsNegativeZero(math.fma(0.0, -b, -0.0)) + self.assertIsPositiveZero(math.fma(0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, -0.0)) + self.assertIsNegativeZero(math.fma(-0.0, b, -0.0)) + self.assertIsPositiveZero(math.fma(-0.0, b, 0.0)) + + self.assertIsPositiveZero(math.fma(b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(b, 0.0, -0.0)) + self.assertIsNegativeZero(math.fma(-b, 0.0, -0.0)) + self.assertIsPositiveZero(math.fma(-b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, -0.0)) + self.assertIsNegativeZero(math.fma(b, -0.0, -0.0)) + self.assertIsPositiveZero(math.fma(b, -0.0, 0.0)) + + # Exact zero result from nonzero inputs. + self.assertIsPositiveZero(math.fma(2.0, 2.0, -4.0)) + self.assertIsPositiveZero(math.fma(2.0, -2.0, 4.0)) + self.assertIsPositiveZero(math.fma(-2.0, -2.0, -4.0)) + self.assertIsPositiveZero(math.fma(-2.0, 2.0, 4.0)) + + # Underflow to zero. + tiny = 1e-300 + self.assertIsPositiveZero(math.fma(tiny, tiny, 0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, 0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, 0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, 0.0)) + self.assertIsPositiveZero(math.fma(tiny, tiny, -0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, -0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, -0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, -0.0)) + + # Corner case where rounding the multiplication would + # give the wrong result. + x = float.fromhex('0x1p-500') + y = float.fromhex('0x1p-550') + z = float.fromhex('0x1p-1000') + self.assertIsNegativeZero(math.fma(x-y, x+y, -z)) + self.assertIsPositiveZero(math.fma(y-x, x+y, z)) + self.assertIsNegativeZero(math.fma(y-x, -(x+y), -z)) + self.assertIsPositiveZero(math.fma(x-y, -(x+y), z)) + + def test_fma_overflow(self): + a = b = float.fromhex('0x1p512') + c = float.fromhex('0x1p1023') + # Overflow from multiplication. + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + self.assertEqual(math.fma(a, b/2.0, 0.0), c) + # Overflow from the addition. + with self.assertRaises(OverflowError): + math.fma(a, b/2.0, c) + # No overflow, even though a*b overflows a float. + self.assertEqual(math.fma(a, b, -c), c) + + # Extreme case: a * b is exactly at the overflow boundary, so the + # tiniest offset makes a difference between overflow and a finite + # result. + a = float.fromhex('0x1.ffffffc000000p+511') + b = float.fromhex('0x1.0000002000000p+512') + c = float.fromhex('0x0.0000000000001p-1022') + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + with self.assertRaises(OverflowError): + math.fma(a, b, c) + self.assertEqual(math.fma(a, b, -c), + float.fromhex('0x1.fffffffffffffp+1023')) + + # Another extreme case: here a*b is about as large as possible subject + # to math.fma(a, b, c) being finite. + a = float.fromhex('0x1.ae565943785f9p+512') + b = float.fromhex('0x1.3094665de9db8p+512') + c = float.fromhex('0x1.fffffffffffffp+1023') + self.assertEqual(math.fma(a, b, -c), c) + + def test_fma_single_round(self): + a = float.fromhex('0x1p-50') + self.assertEqual(math.fma(a - 1.0, a + 1.0, 1.0), a*a) + + def test_random(self): + # A collection of randomly generated inputs for which the naive FMA + # (with two rounds) gives a different result from a singly-rounded FMA. + + # tuples (a, b, c, expected) + test_values = [ + ('0x1.694adde428b44p-1', '0x1.371b0d64caed7p-1', + '0x1.f347e7b8deab8p-4', '0x1.19f10da56c8adp-1'), + ('0x1.605401ccc6ad6p-2', '0x1.ce3a40bf56640p-2', + '0x1.96e3bf7bf2e20p-2', '0x1.1af6d8aa83101p-1'), + ('0x1.e5abd653a67d4p-2', '0x1.a2e400209b3e6p-1', + '0x1.a90051422ce13p-1', '0x1.37d68cc8c0fbbp+0'), + ('0x1.f94e8efd54700p-2', '0x1.123065c812cebp-1', + '0x1.458f86fb6ccd0p-1', '0x1.ccdcee26a3ff3p-1'), + ('0x1.bd926f1eedc96p-1', '0x1.eee9ca68c5740p-1', + '0x1.960c703eb3298p-2', '0x1.3cdcfb4fdb007p+0'), + ('0x1.27348350fbccdp-1', '0x1.3b073914a53f1p-1', + '0x1.e300da5c2b4cbp-1', '0x1.4c51e9a3c4e29p+0'), + ('0x1.2774f00b3497bp-1', '0x1.7038ec336bff0p-2', + '0x1.2f6f2ccc3576bp-1', '0x1.99ad9f9c2688bp-1'), + ('0x1.51d5a99300e5cp-1', '0x1.5cd74abd445a1p-1', + '0x1.8880ab0bbe530p-1', '0x1.3756f96b91129p+0'), + ('0x1.73cb965b821b8p-2', '0x1.218fd3d8d5371p-1', + '0x1.d1ea966a1f758p-2', '0x1.5217b8fd90119p-1'), + ('0x1.4aa98e890b046p-1', '0x1.954d85dff1041p-1', + '0x1.122b59317ebdfp-1', '0x1.0bf644b340cc5p+0'), + ('0x1.e28f29e44750fp-1', '0x1.4bcc4fdcd18fep-1', + '0x1.fd47f81298259p-1', '0x1.9b000afbc9995p+0'), + ('0x1.d2e850717fe78p-3', '0x1.1dd7531c303afp-1', + '0x1.e0869746a2fc2p-2', '0x1.316df6eb26439p-1'), + ('0x1.cf89c75ee6fbap-2', '0x1.b23decdc66825p-1', + '0x1.3d1fe76ac6168p-1', '0x1.00d8ea4c12abbp+0'), + ('0x1.3265ae6f05572p-2', '0x1.16d7ec285f7a2p-1', + '0x1.0b8405b3827fbp-1', '0x1.5ef33c118a001p-1'), + ('0x1.c4d1bf55ec1a5p-1', '0x1.bc59618459e12p-2', + '0x1.ce5b73dc1773dp-1', '0x1.496cf6164f99bp+0'), + ('0x1.d350026ac3946p-1', '0x1.9a234e149a68cp-2', + '0x1.f5467b1911fd6p-2', '0x1.b5cee3225caa5p-1'), + ] + for a_hex, b_hex, c_hex, expected_hex in test_values: + a = float.fromhex(a_hex) + b = float.fromhex(b_hex) + c = float.fromhex(c_hex) + expected = float.fromhex(expected_hex) + self.assertEqual(math.fma(a, b, c), expected) + self.assertEqual(math.fma(b, a, c), expected) + + # Custom assertions. + def assertIsNaN(self, value): + self.assertTrue( + math.isnan(value), + msg="Expected a NaN, got {!r}".format(value) + ) + + def assertIsPositiveZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) > 0, + msg="Expected a positive zero, got {!r}".format(value) + ) + + def assertIsNegativeZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) < 0, + msg="Expected a negative zero, got {!r}".format(value) + ) + + def load_tests(loader, tests, pattern): from doctest import DocFileSuite tests.addTest(DocFileSuite(os.path.join("mathdata", "ieee754.txt"))) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 8192502a40791b..95629ed862d6eb 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -801,7 +801,7 @@ def test_sizeof(self): def _test_cow_mutation(self, mutation): # Common code for all BytesIO copy-on-write mutation tests. - imm = b' ' * 1024 + imm = (' ' * 1024).encode("ascii") old_rc = sys.getrefcount(imm) memio = self.ioclass(imm) self.assertEqual(sys.getrefcount(imm), old_rc + 1) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 0eb2a367603cfc..2d4bf5f1408df8 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -18,6 +18,10 @@ from test.support import import_helper +class MyObject: + pass + + class AbstractMemoryTests: source_bytes = b"abcdef" @@ -228,8 +232,6 @@ def __init__(self, base): self.m = memoryview(base) class MySource(tp): pass - class MyObject: - pass # Create a reference cycle through a memoryview object. # This exercises mbuf_clear(). @@ -656,5 +658,26 @@ def __bool__(self): m[0] = MyBool() self.assertEqual(ba[:8], b'\0'*8) + def test_buffer_reference_loop(self): + m = memoryview(b'abc').__buffer__(0) + o = MyObject() + o.m = m + o.o = o + wr = weakref.ref(o) + del m, o + gc.collect() + self.assertIsNone(wr()) + + def test_picklebuffer_reference_loop(self): + pb = pickle.PickleBuffer(memoryview(b'abc')) + o = MyObject() + o.pb = pb + o.o = o + wr = weakref.ref(o) + del pb, o + gc.collect() + self.assertIsNone(wr()) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index 36e8ab4cda3dad..b37b7defe84d1c 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -164,9 +164,11 @@ ... d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['foo'] = 4 d['foo'] = 42 d['bar'] = 123 + d['__static_attributes__'] = () >>> Use a metaclass that doesn't derive from type. @@ -182,12 +184,12 @@ ... b = 24 ... meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] kw: [] >>> type(C) is dict True >>> print(sorted(C.items())) - [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] >>> And again, with a __prepare__ attribute. @@ -205,11 +207,13 @@ prepare: C () [('other', 'booh')] d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['a'] = 1 d['a'] = 2 d['b'] = 3 + d['__static_attributes__'] = () meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 2), ('b', 3)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] kw: [('other', 'booh')] >>> diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 01bba0ac2eed5a..58f6a4dfae08ba 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -1,6 +1,6 @@ import io import mimetypes -import pathlib +import os import sys import unittest.mock @@ -35,20 +35,28 @@ def setUp(self): def test_case_sensitivity(self): eq = self.assertEqual - eq(self.db.guess_type("foobar.HTML"), self.db.guess_type("foobar.html")) - eq(self.db.guess_type("foobar.TGZ"), self.db.guess_type("foobar.tgz")) - eq(self.db.guess_type("foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) def test_default_data(self): eq = self.assertEqual - eq(self.db.guess_type("foo.html"), ("text/html", None)) - eq(self.db.guess_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) def test_data_urls(self): eq = self.assertEqual @@ -62,7 +70,7 @@ def test_file_parsing(self): eq = self.assertEqual sio = io.StringIO("x-application/x-unittest pyunit\n") self.db.readfp(sio) - eq(self.db.guess_type("foo.pyunit"), + eq(self.db.guess_file_type("foo.pyunit"), ("x-application/x-unittest", None)) eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") @@ -74,11 +82,19 @@ def test_read_mime_types(self): with os_helper.temp_dir() as directory: data = "x-application/x-unittest pyunit\n" - file = pathlib.Path(directory, "sample.mimetype") - file.write_text(data, encoding="utf-8") + file = os.path.join(directory, "sample.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) mime_dict = mimetypes.read_mime_types(file) eq(mime_dict[".pyunit"], "x-application/x-unittest") + data = "x-application/x-unittest2 pyunit2\n" + file = os.path.join(directory, "sample2.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) + mime_dict = mimetypes.read_mime_types(os_helper.FakePath(file)) + eq(mime_dict[".pyunit2"], "x-application/x-unittest2") + # bpo-41048: read_mime_types should read the rule file with 'utf-8' encoding. # Not with locale encoding. _bootlocale has been imported because io.open(...) # uses it. @@ -94,12 +110,12 @@ def test_read_mime_types(self): def test_non_standard_types(self): eq = self.assertEqual # First try strict - eq(self.db.guess_type('foo.xul', strict=True), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict - eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') @@ -109,15 +125,51 @@ def test_filename_with_url_delimiters(self): # compared to when interpreted as filename because of the semicolon. eq = self.assertEqual gzip_expected = ('application/x-tar', 'gzip') - eq(self.db.guess_type(";1.tar.gz"), gzip_expected) - eq(self.db.guess_type("?1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";&1=123;?.tar.gz"), gzip_expected) - eq(self.db.guess_type("?k1=v1&k2=v2.tar.gz"), gzip_expected) + for name in ( + ';1.tar.gz', + '?1.tar.gz', + '#1.tar.gz', + '#1#.tar.gz', + ';1#.tar.gz', + ';&1=123;?.tar.gz', + '?k1=v1&k2=v2.tar.gz', + ): + for prefix in ('', '/', '\\', + 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', + '//share/server/', '\\\\share\\server\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) + eq(self.db.guess_type(path), gzip_expected) + expected = (None, None) if os.name == 'nt' else gzip_expected + for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) + eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') + result = self.db.guess_type('http://host.html') + msg = 'URL only has a host name, not a file' + self.assertSequenceEqual(result, (None, None), msg) + result = self.db.guess_type('http://example.com/host.html') + msg = 'Should be text/html' + self.assertSequenceEqual(result, ('text/html', None), msg) + result = self.db.guess_type('http://example.com/host.html#x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + result = self.db.guess_type('http://example.com/host.html?q=x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + def test_guess_all_types(self): # First try strict. Use a set here for testing the results because if # test_urllib2 is run before test_mimetypes, global state is modified @@ -187,6 +239,8 @@ def check_extensions(): self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') self.assertEqual(mimetypes.guess_extension('text/html'), '.html') self.assertEqual(mimetypes.guess_extension('text/plain'), '.txt') + self.assertEqual(mimetypes.guess_extension('text/rtf'), '.rtf') + self.assertEqual(mimetypes.guess_extension('text/x-rst'), '.rst') self.assertEqual(mimetypes.guess_extension('video/mpeg'), '.mpeg') self.assertEqual(mimetypes.guess_extension('video/quicktime'), '.mov') @@ -214,22 +268,38 @@ def test_init_stability(self): def test_path_like_ob(self): filename = "LICENSE.txt" - filepath = pathlib.Path(filename) - filepath_with_abs_dir = pathlib.Path('/dir/'+filename) - filepath_relative = pathlib.Path('../dir/'+filename) - path_dir = pathlib.Path('./') + filepath = os_helper.FakePath(filename) + filepath_with_abs_dir = os_helper.FakePath('/dir/'+filename) + filepath_relative = os_helper.FakePath('../dir/'+filename) + path_dir = os_helper.FakePath('./') - expected = self.db.guess_type(filename) + expected = self.db.guess_file_type(filename) + self.assertEqual(self.db.guess_file_type(filepath), expected) self.assertEqual(self.db.guess_type(filepath), expected) + self.assertEqual(self.db.guess_file_type( + filepath_with_abs_dir), expected) self.assertEqual(self.db.guess_type( filepath_with_abs_dir), expected) + self.assertEqual(self.db.guess_file_type(filepath_relative), expected) self.assertEqual(self.db.guess_type(filepath_relative), expected) + + self.assertEqual(self.db.guess_file_type(path_dir), (None, None)) self.assertEqual(self.db.guess_type(path_dir), (None, None)) + def test_bytes_path(self): + self.assertEqual(self.db.guess_file_type(b'foo.html'), + self.db.guess_file_type('foo.html')) + self.assertEqual(self.db.guess_file_type(b'foo.tar.gz'), + self.db.guess_file_type('foo.tar.gz')) + self.assertEqual(self.db.guess_file_type(b'foo.tgz'), + self.db.guess_file_type('foo.tgz')) + def test_keywords_args_api(self): + self.assertEqual(self.db.guess_file_type( + path="foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_type( - url="foo.html", strict=True), ("text/html", None)) + url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( type='image/jpg', strict=True), []) self.assertEqual(self.db.guess_extension( diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index ac759757d24659..b2a299ed172967 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,8 +1,10 @@ from test.support import ( requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple, + in_systemd_nspawn_sync_suppressed, ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink +from test.support.script_helper import assert_python_ok import unittest import errno import os @@ -12,6 +14,7 @@ import socket import string import sys +import textwrap import weakref # Skip test if we can't import mmap. @@ -837,7 +840,8 @@ def test_flush_return_value(self): mm.write(b'python') result = mm.flush() self.assertIsNone(result) - if sys.platform.startswith('linux'): + if (sys.platform.startswith(('linux', 'android')) + and not in_systemd_nspawn_sync_suppressed()): # 'offset' must be a multiple of mmap.PAGESIZE on Linux. # See bpo-34754 for details. self.assertRaises(OSError, mm.flush, 1, len(b'python')) @@ -1058,6 +1062,81 @@ def __exit__(self, exc_type, exc_value, traceback): with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): m.write_byte(X()) + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + @unittest.skipUnless(hasattr(mmap.mmap, '_protect'), 'test needs debug build') + def test_access_violations(self): + from test.support.os_helper import TESTFN + + code = textwrap.dedent(""" + import faulthandler + import mmap + import os + import sys + from contextlib import suppress + + # Prevent logging access violations to stderr. + faulthandler.disable() + + PAGESIZE = mmap.PAGESIZE + PAGE_NOACCESS = 0x01 + + with open(sys.argv[1], 'bw+') as f: + f.write(b'A'* PAGESIZE) + f.flush() + + m = mmap.mmap(f.fileno(), PAGESIZE) + m._protect(PAGE_NOACCESS, 0, PAGESIZE) + with suppress(OSError): + m.read(PAGESIZE) + assert False, 'mmap.read() did not raise' + with suppress(OSError): + m.read_byte() + assert False, 'mmap.read_byte() did not raise' + with suppress(OSError): + m.readline() + assert False, 'mmap.readline() did not raise' + with suppress(OSError): + m.write(b'A'* PAGESIZE) + assert False, 'mmap.write() did not raise' + with suppress(OSError): + m.write_byte(0) + assert False, 'mmap.write_byte() did not raise' + with suppress(OSError): + m[0] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0:10] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0:10:2] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0] = 1 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m[0:10] = b'A'* 10 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m[0:10:2] = b'A'* 5 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m.move(0, 10, 1) + assert False, 'mmap.move() did not raise' + with suppress(OSError): + list(m) # test mmap_item + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m.find(b'A') + assert False, 'mmap.find() did not raise' + with suppress(OSError): + m.rfind(b'A') + assert False, 'mmap.rfind() did not raise' + """) + rt, stdout, stderr = assert_python_ok("-c", code, TESTFN) + self.assertEqual(stdout.strip(), b'') + self.assertEqual(stderr.strip(), b'') + + class LargeMmapTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index 98d1cbe824df12..22132b01c8a056 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -357,6 +357,8 @@ def test_annotations_are_created_correctly(self): ann_module4 = import_helper.import_fresh_module( 'test.typinganndata.ann_module4', ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 60b6326bfbad5e..b640aa08e4a812 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -3,12 +3,18 @@ import collections import dis import functools +import math import operator import sys import textwrap import types import unittest -import asyncio + +import test.support +from test.support import requires_specialization, script_helper +from test.support.import_helper import import_module + +_testcapi = test.support.import_helper.import_module("_testcapi") PAIR = (0,1) @@ -35,12 +41,19 @@ def g1(): TEST_TOOL2 = 3 TEST_TOOL3 = 4 +def nth_line(func, offset): + return func.__code__.co_firstlineno + offset + class MonitoringBasicTest(unittest.TestCase): + def tearDown(self): + sys.monitoring.free_tool_id(TEST_TOOL) + def test_has_objects(self): m = sys.monitoring m.events m.use_tool_id + m.clear_tool_id m.free_tool_id m.get_tool m.get_events @@ -68,6 +81,43 @@ def test_tool(self): with self.assertRaises(ValueError): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.CALL) + def test_clear(self): + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, lambda *args: events.append(args)) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + def f(): + a = 1 + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + + f() + sys.monitoring.clear_tool_id(TEST_TOOL) + f() + + # the first f() should trigger a PY_START and a LINE event + # the second f() after clear_tool_id should not trigger any event + # the callback function should be cleared as well + self.assertEqual(len(events), 2) + callback = sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertIs(callback, None) + + sys.monitoring.free_tool_id(TEST_TOOL) + + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + f() + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + f() + # the first f() should trigger a LINE event, and even if we use the + # tool id immediately after freeing it, the second f() should not + # trigger any event + self.assertEqual(len(events), 1) + sys.monitoring.free_tool_id(TEST_TOOL) + class MonitoringTestBase: @@ -529,8 +579,8 @@ def test_lines_single(self): f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMonitoringTest.test_lines_single.__code__.co_firstlineno - self.assertEqual(events, [start+7, 16, start+8]) + start = nth_line(LineMonitoringTest.test_lines_single, 0) + self.assertEqual(events, [start+7, nth_line(f1, 1), start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -547,8 +597,13 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno - self.assertEqual(events, [start+7, 23, 24, 23, 24, 23, start+8]) + start = nth_line(LineMonitoringTest.test_lines_loop, 0) + floop_1 = nth_line(floop, 1) + floop_2 = nth_line(floop, 2) + self.assertEqual( + events, + [start+7, floop_1, floop_2, floop_1, floop_2, floop_1, start+8] + ) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -569,8 +624,8 @@ def test_lines_two(self): sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) - start = LineMonitoringTest.test_lines_two.__code__.co_firstlineno - expected = [start+10, 16, start+11] + start = nth_line(LineMonitoringTest.test_lines_two, 0) + expected = [start+10, nth_line(f1, 1), start+11] self.assertEqual(events, expected) self.assertEqual(events2, expected) finally: @@ -641,6 +696,17 @@ def func2(): self.check_lines(func2, [1,2,3,4,5,6]) + def test_generator_with_line(self): + + def f(): + def a(): + yield + def b(): + yield from a() + next(b()) + + self.check_lines(f, [1,3,5,4,2,4]) + class TestDisable(MonitoringTestBase, unittest.TestCase): def gen(self, cond): @@ -719,8 +785,6 @@ def get_events(self, func, tool, recorders): def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): events = self.get_events(func, tool, recorders) - if events != expected: - print(events, file = sys.stderr) self.assertEqual(events, expected) def check_balanced(self, func, recorders): @@ -808,16 +872,42 @@ def func1(): self.check_events(func1, [("raise", KeyError)]) def test_implicit_stop_iteration(self): + """Generators are documented as raising a StopIteration + when they terminate. + However, we don't do that if we can avoid it, for speed. + sys.monitoring handles that by injecting a STOP_ITERATION + event when we would otherwise have skip the RAISE event. + This test checks that both paths record an equivalent event. + """ def gen(): yield 1 return 2 - def implicit_stop_iteration(): - for _ in gen(): + def implicit_stop_iteration(iterator=None): + if iterator is None: + iterator = gen() + for _ in iterator: pass - self.check_events(implicit_stop_iteration, [("raise", StopIteration)], recorders=(StopiterationRecorder,)) + recorders=(ExceptionRecorder, StopiterationRecorder,) + expected = [("raise", StopIteration)] + + # Make sure that the loop is unspecialized, and that it will not + # re-specialize immediately, so that we can we can test the + # unspecialized version of the loop first. + # Note: this assumes that we don't specialize loops over sets. + implicit_stop_iteration(set(range(100))) + + # This will record a RAISE event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) + + # Now specialize, so that we see a STOP_ITERATION event. + for _ in range(100): + implicit_stop_iteration() + + # This will record a STOP_ITERATION event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) initial = [ ("raise", ZeroDivisionError), @@ -955,6 +1045,7 @@ def func(): ) self.assertEqual(events[0], ("throw", IndexError)) + @requires_specialization def test_no_unwind_for_shim_frame(self): class B: @@ -1114,6 +1205,7 @@ def func1(): ('instruction', 'func1', 10), ('instruction', 'func1', 12), ('instruction', 'func1', 14), + ('instruction', 'func1', 16), ('line', 'get_events', 11)]) def test_c_call(self): @@ -1138,6 +1230,7 @@ def func2(): ('instruction', 'func2', 40), ('instruction', 'func2', 42), ('instruction', 'func2', 44), + ('instruction', 'func2', 46), ('line', 'get_events', 11)]) def test_try_except(self): @@ -1171,6 +1264,7 @@ def func3(): ('instruction', 'func3', 30), ('instruction', 'func3', 32), ('instruction', 'func3', 34), + ('instruction', 'func3', 36), ('line', 'get_events', 11)]) def test_with_restart(self): @@ -1191,6 +1285,7 @@ def func1(): ('instruction', 'func1', 10), ('instruction', 'func1', 12), ('instruction', 'func1', 14), + ('instruction', 'func1', 16), ('line', 'get_events', 11)]) sys.monitoring.restart_events() @@ -1207,6 +1302,7 @@ def func1(): ('instruction', 'func1', 10), ('instruction', 'func1', 12), ('instruction', 'func1', 14), + ('instruction', 'func1', 16), ('line', 'get_events', 11)]) def test_turn_off_only_instruction(self): @@ -1394,9 +1490,27 @@ class BranchRecorder(JumpRecorder): +class JumpOffsetRecorder: + + event_type = E.JUMP + name = "jump" + + def __init__(self, events, offsets=False): + self.events = events + + def __call__(self, code, from_, to): + self.events.append((self.name, code.co_name, from_, to)) + +class BranchOffsetRecorder(JumpOffsetRecorder): + + event_type = E.BRANCH + name = "branch" + + JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder +BRANCH_OFFSET_RECORDERS = BranchOffsetRecorder, class TestBranchAndJumpEvents(CheckEvents): maxDiff = None @@ -1486,6 +1600,24 @@ def func(): ('return', 'func', None), ('line', 'get_events', 11)]) + def test_while_offset_consistency(self): + + def foo(n=0): + while n<4: + pass + n += 1 + return None + + in_loop = ('branch', 'foo', 10, 14) + exit_loop = ('branch', 'foo', 10, 30) + self.check_events(foo, recorders = BRANCH_OFFSET_RECORDERS, expected = [ + in_loop, + in_loop, + in_loop, + in_loop, + exit_loop]) + + class TestLoadSuperAttr(CheckEvents): RECORDERS = CallRecorder, LineRecorder, CRaiseRecorder, CReturnRecorder @@ -1546,7 +1678,7 @@ def f(): ('line', 'method', 2), ('line', 'method', 3), ('line', 'method', 2), - ('call', 'method', 1), + ('call', 'method', d["b"]), ('line', 'method', 1), ('line', 'method', 1), ('line', 'get_events', 11), @@ -1793,20 +1925,55 @@ def test_gh108976(self): sys.monitoring.set_events(0, E.LINE | E.INSTRUCTION) sys.monitoring.set_events(0, 0) + def test_call_function_ex(self): + def f(a=1, b=2): + return a + b + args = (1, 2) + empty_args = [] + + call_data = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, lambda code, offset, callable, arg0: call_data.append((callable, arg0))) + sys.monitoring.set_events(0, E.CALL) + f(*args) + f(*empty_args) + sys.monitoring.set_events(0, 0) + self.assertEqual(call_data[0], (f, 1)) + self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) + + def test_instruction_explicit_callback(self): + # gh-122247 + # Calling the instruction event callback explicitly should not + # crash CPython + def callback(code, instruction_offset): + pass + + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.register_callback(0, sys.monitoring.events.INSTRUCTION, callback) + sys.monitoring.set_events(0, sys.monitoring.events.INSTRUCTION) + callback(None, 0) # call the *same* handler while it is registered + sys.monitoring.restart_events() + sys.monitoring.set_events(0, 0) + class TestOptimizer(MonitoringTestBase, unittest.TestCase): def setUp(self): - import _testinternalcapi - self.old_opt = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() - _testinternalcapi.set_optimizer(opt) + _testinternalcapi = import_module("_testinternalcapi") + if hasattr(_testinternalcapi, "get_optimizer"): + self.old_opt = _testinternalcapi.get_optimizer() + opt = _testinternalcapi.new_counter_optimizer() + _testinternalcapi.set_optimizer(opt) super(TestOptimizer, self).setUp() def tearDown(self): - import _testinternalcapi super(TestOptimizer, self).tearDown() - _testinternalcapi.set_optimizer(self.old_opt) + import _testinternalcapi + if hasattr(_testinternalcapi, "get_optimizer"): + _testinternalcapi.set_optimizer(self.old_opt) def test_for_loop(self): def test_func(x): @@ -1845,3 +2012,208 @@ def test_func(recorder): sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.set_events(TEST_TOOL, 0) self.assertGreater(len(events), 250) + +class TestMonitoringAtShutdown(unittest.TestCase): + + def test_monitoring_live_at_shutdown(self): + # gh-115832: An object destructor running during the final GC of + # interpreter shutdown triggered an infinite loop in the + # instrumentation code. + script = test.support.findfile("_test_monitoring_shutdown.py") + script_helper.run_test_script(script) + + +class TestCApiEventGeneration(MonitoringTestBase, unittest.TestCase): + + class Scope: + def __init__(self, *args): + self.args = args + + def __enter__(self): + _testcapi.monitoring_enter_scope(*self.args) + + def __exit__(self, *args): + _testcapi.monitoring_exit_scope() + + def setUp(self): + super(TestCApiEventGeneration, self).setUp() + + capi = _testcapi + + self.codelike = capi.CodeLike(2) + + self.cases = [ + # (Event, function, *args) + ( 1, E.PY_START, capi.fire_event_py_start), + ( 1, E.PY_RESUME, capi.fire_event_py_resume), + ( 1, E.PY_YIELD, capi.fire_event_py_yield, 10), + ( 1, E.PY_RETURN, capi.fire_event_py_return, 20), + ( 2, E.CALL, capi.fire_event_call, callable, 40), + ( 1, E.JUMP, capi.fire_event_jump, 60), + ( 1, E.BRANCH, capi.fire_event_branch, 70), + ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)), + ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), + ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), + ( 1, E.PY_UNWIND, capi.fire_event_py_unwind, ValueError(6)), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, 7), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, StopIteration(8)), + ] + + self.EXPECT_RAISED_EXCEPTION = [E.PY_THROW, E.RAISE, E.EXCEPTION_HANDLED, E.PY_UNWIND] + + + def check_event_count(self, event, func, args, expected, callback_raises=None): + class Counter: + def __init__(self, callback_raises): + self.callback_raises = callback_raises + self.count = 0 + + def __call__(self, *args): + self.count += 1 + if self.callback_raises: + exc = self.callback_raises + self.callback_raises = None + raise exc + + try: + counter = Counter(callback_raises) + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + try: + func(*args) + except ValueError as e: + self.assertIsInstance(expected, ValueError) + self.assertEqual(str(e), str(expected)) + return + else: + self.assertEqual(counter.count, expected) + + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_fire_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + self.check_event_count(event, function, args_, expected) + + def test_missing_exception(self): + for _, event, function, *args in self.cases: + if event not in self.EXPECT_RAISED_EXCEPTION: + continue + assert args and isinstance(args[-1], BaseException) + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args[:-1]) + (None,) + evt = int(math.log2(event)) + expected = ValueError(f"Firing event {evt} with no exception set") + self.check_event_count(event, function, args_, expected) + + def test_fire_event_failing_callback(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + exc = OSError(42) + with self.assertRaises(type(exc)): + self.check_event_count(event, function, args_, expected, + callback_raises=exc) + + + CANNOT_DISABLE = { E.PY_THROW, E.RAISE, E.RERAISE, + E.EXCEPTION_HANDLED, E.PY_UNWIND } + + def check_disable(self, event, func, args, expected): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.disable = True + if event in self.CANNOT_DISABLE: + # use try-except rather then assertRaises to avoid + # events from framework code + try: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + except ValueError: + pass + else: + self.Error("Expected a ValueError") + else: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected - 1) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_disable_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(2) + with self.subTest(function.__name__): + args_ = (self.codelike, 0) + tuple(args) + self.check_disable(event, function, args_, expected) + + def test_enter_scope_two_events(self): + try: + yield_counter = CounterWithDisable() + unwind_counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_YIELD, yield_counter) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, unwind_counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_YIELD | E.PY_UNWIND) + + yield_value = int(math.log2(E.PY_YIELD)) + unwind_value = int(math.log2(E.PY_UNWIND)) + cl = _testcapi.CodeLike(2) + common_args = (cl, 0) + with self.Scope(cl, yield_value, unwind_value): + yield_counter.count = 0 + unwind_counter.count = 0 + + _testcapi.fire_event_py_unwind(*common_args, ValueError(42)) + assert(yield_counter.count == 0) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 1) + assert(unwind_counter.count == 1) + + yield_counter.disable = True + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + finally: + sys.monitoring.set_events(TEST_TOOL, 0) diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index 600c4446fd5cd4..1c6905bd1ee586 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -64,7 +64,7 @@ class TestConsoleIO(unittest.TestCase): # CREATE_NEW_CONSOLE creates a "popup" window. @requires_resource('gui') def run_in_separated_process(self, code): - # Run test in a seprated process to avoid stdin conflicts. + # Run test in a separated process to avoid stdin conflicts. # See: gh-110147 cmd = [sys.executable, '-c', code] subprocess.run(cmd, check=True, capture_output=True, diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 6451df14696933..1b55f1e70b32f5 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -12,6 +12,7 @@ from test import support from test.support import os_helper from test.support.os_helper import TESTFN +from test.support.import_helper import import_module ALL_CJKENCODINGS = [ # _codecs_cn @@ -212,7 +213,7 @@ def test_issue5640(self): @support.cpython_only def test_subinterp(self): # bpo-42846: Test a CJK codec in a subinterpreter - import _testcapi + _testcapi = import_module("_testcapi") encoding = 'cp932' text = "Python の開発は、1990 年ごろから開始されています。" code = textwrap.dedent(""" @@ -303,7 +304,7 @@ def test_setstate_validates_input(self): self.assertRaises(TypeError, decoder.setstate, 123) self.assertRaises(TypeError, decoder.setstate, ("invalid", 0)) self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid")) - self.assertRaises(UnicodeError, decoder.setstate, (b"123456789", 0)) + self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0)) class Test_StreamReader(unittest.TestCase): def test_bug1728403(self): diff --git a/Lib/test/test_multiprocessing_fork/__init__.py b/Lib/test/test_multiprocessing_fork/__init__.py index aa1fff50b28f5f..b35e82879d7fe2 100644 --- a/Lib/test/test_multiprocessing_fork/__init__.py +++ b/Lib/test/test_multiprocessing_fork/__init__.py @@ -12,5 +12,8 @@ if sys.platform == 'darwin': raise unittest.SkipTest("test may crash on macOS (bpo-33725)") +if support.check_sanitizer(thread=True): + raise unittest.SkipTest("TSAN doesn't support threads after fork") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 7b2fa844827ae9..cf44080670dc2e 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -298,6 +298,82 @@ def test_named_expression_invalid_set_comprehension_iterable_expression(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope + def test_named_expression_invalid_rebinding_dict_comprehension_iteration_variable(self): + cases = [ + ("Key reuse", 'i', "{(i := 0): 1 for i in range(5)}"), + ("Value reuse", 'i', "{1: (i := 0) for i in range(5)}"), + ("Both reuse", 'i', "{(i := 0): (i := 0) for i in range(5)}"), + ("Nested reuse", 'j', "{{(j := 0): 1 for i in range(5)} for j in range(5)}"), + ("Reuse inner loop target", 'j', "{(j := 0): 1 for i in range(5) for j in range(5)}"), + ("Unpacking key reuse", 'i', "{(i := 0): 1 for i, j in {(0, 1)}}"), + ("Unpacking value reuse", 'i', "{1: (i := 0) for i, j in {(0, 1)}}"), + ("Reuse in loop condition", 'i', "{i+1: 1 for i in range(5) if (i := 0)}"), + ("Unreachable reuse", 'i', "{(False or (i:=0)): 1 for i in range(5)}"), + ("Unreachable nested reuse", 'i', + "{i: j for i in range(5) for j in range(5) if True or (i:=10)}"), + # Regression tests from https://github.com/python/cpython/issues/87447 + ("Complex expression: a", "a", + "{(a := 1): 1 for a, (*b, c[d+e::f(g)], h.i) in j}"), + ("Complex expression: b", "b", + "{(b := 1): 1 for a, (*b, c[d+e::f(g)], h.i) in j}"), + ] + for case, target, code in cases: + msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'" + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_rebinding_dict_comprehension_inner_loop(self): + cases = [ + ("Inner reuse", 'j', "{i: 1 for i in range(5) if (j := 0) for j in range(5)}"), + ("Inner unpacking reuse", 'j', "{i: 1 for i in range(5) if (j := 0) for j, k in {(0, 1)}}"), + ] + for case, target, code in cases: + msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'" + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_dict_comprehension_iterable_expression(self): + cases = [ + ("Top level", "{i: 1 for i in (i := range(5))}"), + ("Inside tuple", "{i: 1 for i in (2, 3, i := range(5))}"), + ("Inside list", "{i: 1 for i in [2, 3, i := range(5)]}"), + ("Different name", "{i: 1 for i in (j := range(5))}"), + ("Lambda expression", "{i: 1 for i in (lambda:(j := range(5)))()}"), + ("Inner loop", "{i: 1 for i in range(5) for j in (i := range(5))}"), + ("Nested comprehension", "{i: 1 for i in {j: 2 for j in (k := range(5))}}"), + ("Nested comprehension condition", "{i: 1 for i in {j: 2 for j in range(5) if (j := True)}}"), + ("Nested comprehension body", "{i: 1 for i in {(j := True) for j in range(5)}}"), + ] + msg = "assignment expression cannot be used in a comprehension iterable expression" + for case, code in cases: + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_mangled_class_variables(self): + code = """class Foo: + def bar(self): + [[(__x:=2) for _ in range(2)] for __x in range(2)] + """ + + with self.assertRaisesRegex(SyntaxError, + "assignment expression cannot rebind comprehension iteration variable '__x'"): + exec(code, {}, {}) + class NamedExpressionAssignmentTest(unittest.TestCase): @@ -351,7 +427,7 @@ def test_named_expression_assignment_09(self): def test_named_expression_assignment_10(self): if (match := 10) == 10: - pass + self.assertEqual(match, 10) else: self.fail("variable was not assigned using named expression") def test_named_expression_assignment_11(self): @@ -393,7 +469,7 @@ def test_named_expression_assignment_14(self): def test_named_expression_assignment_15(self): while a := False: - pass # This will not run + self.fail("While body executed") # This will not run self.assertEqual(a, False) @@ -674,6 +750,18 @@ def test_named_expression_scope_in_genexp(self): for idx, elem in enumerate(genexp): self.assertEqual(elem, b[idx] + a) + def test_named_expression_scope_mangled_names(self): + class Foo: + def f(self_): + global __x1 + __x1 = 0 + [_Foo__x1 := 1 for a in [2]] + self.assertEqual(__x1, 1) + [__x1 := 2 for a in [3]] + self.assertEqual(__x1, 2) + + Foo().f() + self.assertEqual(_Foo__x1, 2) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 9cb03e3cd5de8d..64cbfaaaaa0690 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -374,6 +374,7 @@ def test_normpath(self): tester("ntpath.normpath('\\\\foo\\')", '\\\\foo\\') tester("ntpath.normpath('\\\\foo')", '\\\\foo') tester("ntpath.normpath('\\\\')", '\\\\') + tester("ntpath.normpath('//?/UNC/server/share/..')", '\\\\?\\UNC\\server\\share\\') def test_realpath_curdir(self): expected = ntpath.normpath(os.getcwd()) @@ -866,43 +867,47 @@ def test_commonpath(self): def check(paths, expected): tester(('ntpath.commonpath(%r)' % paths).replace('\\\\', '\\'), expected) - def check_error(exc, paths): - self.assertRaises(exc, ntpath.commonpath, paths) - self.assertRaises(exc, ntpath.commonpath, - [os.fsencode(p) for p in paths]) - + def check_error(paths, expected): + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths[::-1]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths[::-1]]) + + self.assertRaises(TypeError, ntpath.commonpath, None) self.assertRaises(ValueError, ntpath.commonpath, []) - check_error(ValueError, ['C:\\Program Files', 'Program Files']) - check_error(ValueError, ['C:\\Program Files', 'C:Program Files']) - check_error(ValueError, ['\\Program Files', 'Program Files']) - check_error(ValueError, ['Program Files', 'C:\\Program Files']) - check(['C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files'], - 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files\\'], - 'C:\\Program Files') - check(['C:\\\\Program Files', 'C:\\Program Files\\\\'], - 'C:\\Program Files') - check(['C:\\.\\Program Files', 'C:\\Program Files\\.'], - 'C:\\Program Files') - check(['C:\\', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Projects'], 'C:\\') - check(['C:\\Program Files\\', 'C:\\Projects'], 'C:\\') - - check(['C:\\Program Files\\Foo', 'C:/Program Files/Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'c:/program files/bar'], - 'C:\\Program Files') - check(['c:/program files/bar', 'C:\\Program Files\\Foo'], - 'c:\\program files') - - check_error(ValueError, ['C:\\Program Files', 'D:\\Program Files']) + self.assertRaises(ValueError, ntpath.commonpath, iter([])) + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'C:Foo'], "Can't mix absolute and relative paths") + check_error(['C:\\Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'Foo'], "Paths don't have the same drive") + check_error(['C:Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'Foo'], "Paths don't have the same drive") + check_error(['\\Foo', 'Foo'], "Can't mix rooted and not-rooted paths") + + check(['C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo\\'], 'C:\\Foo') + check(['C:\\\\Foo', 'C:\\Foo\\\\'], 'C:\\Foo') + check(['C:\\.\\Foo', 'C:\\Foo\\.'], 'C:\\Foo') + check(['C:\\', 'C:\\baz'], 'C:\\') + check(['C:\\Bar', 'C:\\baz'], 'C:\\') + check(['C:\\Foo', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Bar', 'C:\\Baz'], 'C:\\') + check(['C:\\Bar\\', 'C:\\Baz'], 'C:\\') + + check(['C:\\Foo\\Bar', 'C:/Foo/Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'c:/foo/baz'], 'C:\\Foo') + check(['c:/foo/bar', 'C:\\Foo\\Baz'], 'c:\\foo') + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'D:\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'D:Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'D:Foo'], "Paths don't have the same drive") check(['spam'], 'spam') check(['spam', 'spam'], 'spam') @@ -916,20 +921,16 @@ def check_error(exc, paths): check([''], '') check(['', 'spam\\alot'], '') - check_error(ValueError, ['', '\\spam\\alot']) - - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['Program Files', b'C:\\Program Files\\Foo']) + + # gh-117381: Logical error messages + check_error(['', '\\spam\\alot'], "Can't mix rooted and not-rooted paths") + + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['Foo', b'C:\\Foo\\Baz']) @unittest.skipIf(is_emscripten, "Emscripten cannot fstat unnamed files.") def test_sameopenfile(self): @@ -1094,6 +1095,27 @@ def test_isfile_driveletter(self): raise unittest.SkipTest('SystemDrive is not defined or malformed') self.assertFalse(os.path.isfile('\\\\.\\' + drive)) + @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()") + def test_isfile_anonymous_pipe(self): + pr, pw = os.pipe() + try: + self.assertFalse(ntpath.isfile(pr)) + finally: + os.close(pr) + os.close(pw) + + @unittest.skipIf(sys.platform != 'win32', "windows only") + def test_isfile_named_pipe(self): + import _winapi + named_pipe = f'//./PIPE/python_isfile_test_{os.getpid()}' + h = _winapi.CreateNamedPipe(named_pipe, + _winapi.PIPE_ACCESS_INBOUND, + 0, 1, 0, 0, 0, 0) + try: + self.assertFalse(ntpath.isfile(named_pipe)) + finally: + _winapi.CloseHandle(h) + @unittest.skipIf(sys.platform != 'win32', "windows only") def test_con_device(self): self.assertFalse(os.path.isfile(r"\\.\CON")) @@ -1107,14 +1129,22 @@ def test_fast_paths_in_use(self): # There are fast paths of these functions implemented in posixmodule.c. # Confirm that they are being used, and not the Python fallbacks in # genericpath.py. + self.assertTrue(os.path.splitroot is nt._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is nt._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) self.assertTrue(os.path.isdir is nt._path_isdir) self.assertFalse(inspect.isfunction(os.path.isdir)) self.assertTrue(os.path.isfile is nt._path_isfile) self.assertFalse(inspect.isfunction(os.path.isfile)) self.assertTrue(os.path.islink is nt._path_islink) self.assertFalse(inspect.isfunction(os.path.islink)) + self.assertTrue(os.path.isjunction is nt._path_isjunction) + self.assertFalse(inspect.isfunction(os.path.isjunction)) self.assertTrue(os.path.exists is nt._path_exists) self.assertFalse(inspect.isfunction(os.path.exists)) + self.assertTrue(os.path.lexists is nt._path_lexists) + self.assertFalse(inspect.isfunction(os.path.lexists)) @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 2b2783d57be8f4..cdcddb0d717f23 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -4,17 +4,20 @@ import threading import types import unittest -from test.support import threading_helper, check_impl_detail +from test.support import threading_helper, check_impl_detail, requires_specialization +from test.support.import_helper import import_module # Skip this module on other interpreters, it is cpython specific: if check_impl_detail(cpython=False): raise unittest.SkipTest('implementation detail specific to cpython') -import _testinternalcapi +_testinternalcapi = import_module("_testinternalcapi") def disabling_optimizer(func): def wrapper(*args, **kwargs): + if not hasattr(_testinternalcapi, "get_optimizer"): + return func(*args, **kwargs) old_opt = _testinternalcapi.get_optimizer() _testinternalcapi.set_optimizer(None) try: @@ -25,6 +28,13 @@ def wrapper(*args, **kwargs): return wrapper +class TestBase(unittest.TestCase): + def assert_specialized(self, f, opname): + instructions = dis.get_instructions(f, adaptive=True) + opnames = {instruction.opname for instruction in instructions} + self.assertIn(opname, opnames) + + class TestLoadSuperAttrCache(unittest.TestCase): def test_descriptor_not_double_executed_on_spec_fail(self): calls = [] @@ -476,7 +486,7 @@ def f(): self.assertFalse(f()) -class TestCallCache(unittest.TestCase): +class TestCallCache(TestBase): def test_too_many_defaults_0(self): def f(): pass @@ -504,9 +514,33 @@ def f(x, y): f(None) f() + @disabling_optimizer + @requires_specialization + def test_assign_init_code(self): + class MyClass: + def __init__(self): + pass + + def instantiate(): + return MyClass() + + # Trigger specialization + for _ in range(1025): + instantiate() + self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") + + def count_args(self, *args): + self.num_args = len(args) + + # Set MyClass.__init__.__code__ to a code object that uses different + # args + MyClass.__init__.__code__ = count_args.__code__ + instantiate() + @threading_helper.requires_working_threading() -class TestRacesDoNotCrash(unittest.TestCase): +@requires_specialization +class TestRacesDoNotCrash(TestBase): # Careful with these. Bigger numbers have a higher chance of catching bugs, # but you can also burn through a *ton* of type/dict/function versions: ITEMS = 1000 @@ -514,11 +548,6 @@ class TestRacesDoNotCrash(unittest.TestCase): WARMUPS = 2 WRITERS = 2 - def assert_specialized(self, f, opname): - instructions = dis.get_instructions(f, adaptive=True) - opnames = {instruction.opname for instruction in instructions} - self.assertIn(opname, opnames) - @disabling_optimizer def assert_races_do_not_crash( self, opname, get_items, read, write, *, check_items=False @@ -1021,6 +1050,7 @@ def write(items): class C: pass +@requires_specialization class TestInstanceDict(unittest.TestCase): def setUp(self): @@ -1042,20 +1072,13 @@ def test_dict_materialization(self): c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) + self.assertEqual(c.__dict__, {"a":1, "b": 2}) def test_dict_dematerialization(self): c = C() c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) for _ in range(100): c.a self.assertEqual( @@ -1070,10 +1093,6 @@ def test_dict_dematerialization_multiple_refs(self): d = c.__dict__ for _ in range(100): c.a - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) self.assertIs(c.__dict__, d) def test_dict_dematerialization_copy(self): @@ -1136,6 +1155,50 @@ class D(dict): pass {'a':1, 'b':2} ) + def test_125868(self): + + def make_special_dict(): + """Create a dictionary an object with a this table: + index | key | value + ----- | --- | ----- + 0 | 'b' | 'value' + 1 | 'b' | NULL + """ + class A: + pass + a = A() + a.a = 1 + a.b = 2 + d = a.__dict__.copy() + del d['a'] + del d['b'] + d['b'] = "value" + return d + + class NoInlineAorB: + pass + for i in range(ord('c'), ord('z')): + setattr(NoInlineAorB(), chr(i), i) + + c = NoInlineAorB() + c.a = 0 + c.b = 1 + self.assertFalse(_testinternalcapi.has_inline_values(c)) + + def f(o, n): + for i in range(n): + o.b = i + # Prime f to store to dict slot 1 + f(c, 100) + + test_obj = NoInlineAorB() + test_obj.__dict__ = make_special_dict() + self.assertEqual(test_obj.b, "value") + + #This should set x.b = 0 + f(test_obj, 1) + self.assertEqual(test_obj.b, 0) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6b4ff..f7cc8331b8d844 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -39,16 +39,19 @@ class C: pass def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) - self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + self.assertEqual(ns['__annotations__'], {1: 2}) def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): - del globals()['__annotations__'] + globals().pop('__annotations__', None) class C: - del __annotations__ - with self.assertRaises(NameError): - x: int + try: + del __annotations__ + except NameError: + pass + x: int + self.assertEqual(C.__annotations__, {"x": int}) def test_raise_class_exceptions(self): diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 1db738d228b1b9..812d46482e238a 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -1,6 +1,9 @@ import unittest +import inspect import pickle import sys +from decimal import Decimal +from fractions import Fraction from test import support from test.support import import_helper @@ -344,6 +347,26 @@ def test_is_not(self): self.assertFalse(operator.is_not(a, b)) self.assertTrue(operator.is_not(a,c)) + def test_is_none(self): + operator = self.module + a = 'xyzpdq' + b = '' + c = None + self.assertRaises(TypeError, operator.is_none) + self.assertFalse(operator.is_none(a)) + self.assertFalse(operator.is_none(b)) + self.assertTrue(operator.is_none(c)) + + def test_is_not_none(self): + operator = self.module + a = 'xyzpdq' + b = '' + c = None + self.assertRaises(TypeError, operator.is_not_none) + self.assertTrue(operator.is_not_none(a)) + self.assertTrue(operator.is_not_none(b)) + self.assertFalse(operator.is_not_none(c)) + def test_attrgetter(self): operator = self.module class A: @@ -508,6 +531,44 @@ def __getitem__(self, other): return 5 # so that C is a sequence self.assertEqual(operator.ixor (c, 5), "ixor") self.assertEqual(operator.iconcat (c, c), "iadd") + def test_iconcat_without_getitem(self): + operator = self.module + + msg = "'int' object can't be concatenated" + with self.assertRaisesRegex(TypeError, msg): + operator.iconcat(1, 0.5) + + def test_index(self): + operator = self.module + class X: + def __index__(self): + return 1 + + self.assertEqual(operator.index(X()), 1) + self.assertEqual(operator.index(0), 0) + self.assertEqual(operator.index(1), 1) + self.assertEqual(operator.index(2), 2) + with self.assertRaises((AttributeError, TypeError)): + operator.index(1.5) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Fraction(3, 7)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Decimal(1)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(None) + + def test_not_(self): + operator = self.module + class C: + def __bool__(self): + raise SyntaxError + self.assertRaises(TypeError, operator.not_) + self.assertRaises(SyntaxError, operator.not_, C()) + self.assertFalse(operator.not_(5)) + self.assertFalse(operator.not_([0])) + self.assertTrue(operator.not_(0)) + self.assertTrue(operator.not_([])) + def test_length_hint(self): operator = self.module class X(object): @@ -533,6 +594,13 @@ def __length_hint__(self): with self.assertRaises(LookupError): operator.length_hint(X(LookupError)) + class Y: pass + + msg = "'str' object cannot be interpreted as an integer" + with self.assertRaisesRegex(TypeError, msg): + operator.length_hint(X(2), "abc") + self.assertEqual(operator.length_hint(Y(), 10), 10) + def test_call(self): operator = self.module @@ -555,6 +623,28 @@ def test_dunder_is_original(self): if dunder: self.assertIs(dunder, orig) + def test_attrgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.attrgetter) + self.assertEqual(str(sig), '(attr, /, *attrs)') + sig = inspect.signature(operator.attrgetter('x', 'z', 'y')) + self.assertEqual(str(sig), '(obj, /)') + + def test_itemgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.itemgetter) + self.assertEqual(str(sig), '(item, /, *items)') + sig = inspect.signature(operator.itemgetter(2, 3, 5)) + self.assertEqual(str(sig), '(obj, /)') + + def test_methodcaller_signature(self): + operator = self.module + sig = inspect.signature(operator.methodcaller) + self.assertEqual(str(sig), '(name, /, *args, **kwargs)') + sig = inspect.signature(operator.methodcaller('foo', 2, y=3)) + self.assertEqual(str(sig), '(obj, /)') + + class PyOperatorTestCase(OperatorTestCase, unittest.TestCase): module = py_operator diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py index c8554c40df4b2d..fac4d1a4ab44c4 100644 --- a/Lib/test/test_optimizer.py +++ b/Lib/test/test_optimizer.py @@ -7,6 +7,9 @@ class TestRareEventCounters(unittest.TestCase): + def setUp(self): + _testinternalcapi.reset_rare_event_counters() + def test_set_class(self): class A: pass @@ -74,5 +77,14 @@ def func(x=0): _testinternalcapi.get_rare_event_counters()["func_modification"] ) + +class TestOptimizerSymbols(unittest.TestCase): + + @unittest.skipUnless(hasattr(_testinternalcapi, "uop_symbols_test"), + "requires _testinternalcapi.uop_symbols_test") + def test_optimizer_symbols(self): + _testinternalcapi.uop_symbols_test() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 4571b23dfe7c1a..9f131a9110dccb 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -2,7 +2,9 @@ import contextlib import copy import gc +import operator import pickle +import re from random import randrange, shuffle import struct import sys @@ -739,11 +741,44 @@ def test_ordered_dict_items_result_gc(self): # when it's mutated and returned from __next__: self.assertTrue(gc.is_tracked(next(it))) + +class _TriggerSideEffectOnEqual: + count = 0 # number of calls to __eq__ + trigger = 1 # count value when to trigger side effect + + def __eq__(self, other): + if self.__class__.count == self.__class__.trigger: + self.side_effect() + self.__class__.count += 1 + return True + + def __hash__(self): + # all instances represent the same key + return -1 + + def side_effect(self): + raise NotImplementedError + class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase): module = py_coll OrderedDict = py_coll.OrderedDict + def test_issue119004_attribute_error(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + del dict1[TODEL] + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + # This causes an AttributeError due to the linked list being changed + msg = re.escape("'NoneType' object has no attribute 'key'") + self.assertRaisesRegex(AttributeError, msg, operator.eq, dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + class CPythonBuiltinDictTests(unittest.TestCase): """Builtin dict preserves insertion order. @@ -764,8 +799,85 @@ class CPythonBuiltinDictTests(unittest.TestCase): del method +class CPythonOrderedDictSideEffects: + + def check_runtime_error_issue119004(self, dict1, dict2): + msg = re.escape("OrderedDict mutated during iteration") + self.assertRaisesRegex(RuntimeError, msg, operator.eq, dict1, dict2) + + def test_issue119004_change_size_by_clear(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + dict1.clear() + + dict1 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, {}) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + def test_issue119004_change_size_by_delete_key(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + del dict1[TODEL] + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + def test_issue119004_change_linked_list_by_clear(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + dict1.clear() + dict1['a'] = dict1['b'] = 'c' + + dict1 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys(('a', 'b'), 'c')) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + def test_issue119004_change_linked_list_by_delete_key(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + del dict1[TODEL] + dict1['a'] = 'c' + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, {0: None, 'a': 'c', 4.2: None}) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + def test_issue119004_change_size_by_delete_key_in_dict_eq(self): + class Key(_TriggerSideEffectOnEqual): + trigger = 0 + def side_effect(self): + del dict1[TODEL] + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.assertEqual(Key.count, 0) + # the side effect is in dict.__eq__ and modifies the length + self.assertNotEqual(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + @unittest.skipUnless(c_coll, 'requires the C version of the collections module') -class CPythonOrderedDictTests(OrderedDictTests, unittest.TestCase): +class CPythonOrderedDictTests(OrderedDictTests, + CPythonOrderedDictSideEffects, + unittest.TestCase): module = c_coll OrderedDict = c_coll.OrderedDict diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 2c8823ae47c726..307f0f11ddc33f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -13,6 +13,7 @@ import locale import os import pickle +import platform import select import selectors import shutil @@ -34,7 +35,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper -from test.support import set_recursion_limit +from test.support import infinite_recursion from test.support import warnings_helper from platform import win32_is_iot @@ -56,8 +57,10 @@ except (ImportError, AttributeError): all_users = [] try: + import _testcapi from _testcapi import INT_MAX, PY_SSIZE_T_MAX except ImportError: + _testcapi = None INT_MAX = PY_SSIZE_T_MAX = sys.maxsize try: @@ -825,7 +828,7 @@ def ns_to_sec(ns): return (ns * 1e-9) + 0.5e-9 def test_utime_by_indexed(self): - # pass times as floating point seconds as the second indexed parameter + # pass times as floating-point seconds as the second indexed parameter def set_time(filename, ns): atime_ns, mtime_ns = ns atime = self.ns_to_sec(atime_ns) @@ -1295,9 +1298,56 @@ def test_ror_operator(self): self._test_underlying_process_env('_A_', '') self._test_underlying_process_env(overridden_key, original_value) + def test_refresh(self): + # Test os.environ.refresh() + has_environb = hasattr(os, 'environb') + + # Test with putenv() which doesn't update os.environ + os.environ['test_env'] = 'python_value' + os.putenv("test_env", "new_value") + self.assertEqual(os.environ['test_env'], 'python_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'python_value') + + os.environ.refresh() + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + # Test with unsetenv() which doesn't update os.environ + os.unsetenv('test_env') + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + os.environ.refresh() + self.assertNotIn('test_env', os.environ) + if has_environb: + self.assertNotIn(b'test_env', os.environb) + + if has_environb: + # test os.environb.refresh() with putenv() + os.environb[b'test_env'] = b'python_value2' + os.putenv("test_env", "new_value2") + self.assertEqual(os.environb[b'test_env'], b'python_value2') + self.assertEqual(os.environ['test_env'], 'python_value2') + + os.environb.refresh() + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + # test os.environb.refresh() with unsetenv() + os.unsetenv('test_env') + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + os.environb.refresh() + self.assertNotIn(b'test_env', os.environb) + self.assertNotIn('test_env', os.environ) class WalkTests(unittest.TestCase): """Tests for os.walk().""" + is_fwalk = False # Wrapper to hide minor differences between os.walk and os.fwalk # to tests both functions with the same code base @@ -1332,14 +1382,14 @@ def setUp(self): self.sub11_path = join(self.sub1_path, "SUB11") sub2_path = join(self.walk_path, "SUB2") sub21_path = join(sub2_path, "SUB21") - tmp1_path = join(self.walk_path, "tmp1") + self.tmp1_path = join(self.walk_path, "tmp1") tmp2_path = join(self.sub1_path, "tmp2") tmp3_path = join(sub2_path, "tmp3") tmp5_path = join(sub21_path, "tmp3") self.link_path = join(sub2_path, "link") t2_path = join(os_helper.TESTFN, "TEST2") tmp4_path = join(os_helper.TESTFN, "TEST2", "tmp4") - broken_link_path = join(sub2_path, "broken_link") + self.broken_link_path = join(sub2_path, "broken_link") broken_link2_path = join(sub2_path, "broken_link2") broken_link3_path = join(sub2_path, "broken_link3") @@ -1349,13 +1399,13 @@ def setUp(self): os.makedirs(sub21_path) os.makedirs(t2_path) - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: + for path in self.tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: with open(path, "x", encoding='utf-8') as f: f.write("I'm " + path + " and proud of it. Blame test_os.\n") if os_helper.can_symlink(): os.symlink(os.path.abspath(t2_path), self.link_path) - os.symlink('broken', broken_link_path, True) + os.symlink('broken', self.broken_link_path, True) os.symlink(join('tmp3', 'broken'), broken_link2_path, True) os.symlink(join('SUB21', 'tmp5'), broken_link3_path, True) self.sub2_tree = (sub2_path, ["SUB21", "link"], @@ -1451,6 +1501,11 @@ def test_walk_symlink(self): else: self.fail("Didn't follow symlink with followlinks=True") + walk_it = self.walk(self.broken_link_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1472,6 +1527,73 @@ def test_walk_bad_dir(self): finally: os.rename(path1new, path1) + def test_walk_bad_dir2(self): + walk_it = self.walk('nonexisting') + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk('nonexisting', follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe(self): + path = os_helper.TESTFN + '-pipe' + os.mkfifo(path) + self.addCleanup(os.unlink, path) + + walk_it = self.walk(path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe2(self): + path = os_helper.TESTFN + '-dir' + os.mkdir(path) + self.addCleanup(shutil.rmtree, path) + os.mkfifo(os.path.join(path, 'mypipe')) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + next(walk_it) + self.assertRaises(StopIteration, next, walk_it) + self.assertEqual(errors, []) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(root, path) + self.assertEqual(dirs, []) + self.assertEqual(files, ['mypipe']) + dirs.extend(files) + files.clear() + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + if self.is_fwalk: + self.assertEqual(errors, []) + else: + self.assertEqual(len(errors), 1, errors) + self.assertIsInstance(errors[0], NotADirectoryError) + def test_walk_many_open_files(self): depth = 30 base = os.path.join(os_helper.TESTFN, 'deep') @@ -1496,7 +1618,7 @@ def test_walk_many_open_files(self): def test_walk_above_recursion_limit(self): depth = 50 os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) - with set_recursion_limit(depth - 5): + with infinite_recursion(depth - 5): all = list(self.walk(self.walk_path)) sub2_path = self.sub2_tree[0] @@ -1537,6 +1659,7 @@ def test_walk_above_recursion_limit(self): @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): """Tests for os.fwalk().""" + is_fwalk = True def walk(self, top, **kwargs): for root, dirs, files, root_fd in self.fwalk(top, **kwargs): @@ -1592,6 +1715,9 @@ def test_yields_correct_dir_fd(self): @unittest.skipIf( support.is_emscripten, "Cannot dup stdout on Emscripten" ) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) def test_fd_leak(self): # Since we're opening a lot of FDs, we must be careful to avoid leaks: # we both check that calling fwalk() a large number of times doesn't @@ -1605,10 +1731,29 @@ def test_fd_leak(self): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) + @unittest.skipIf( + support.is_emscripten, "Cannot dup stdout on Emscripten" + ) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) + def test_fd_finalization(self): + # Check that close()ing the fwalk() generator closes FDs + def getfd(): + fd = os.dup(1) + os.close(fd) + return fd + for topdown in (False, True): + old_fd = getfd() + it = self.fwalk(os_helper.TESTFN, topdown=topdown) + self.assertEqual(getfd(), old_fd) + next(it) + self.assertGreater(getfd(), old_fd) + it.close() + self.assertEqual(getfd(), old_fd) + # fwalk() keeps file descriptors open test_walk_many_open_files = None - # fwalk() still uses recursion - test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): @@ -1731,6 +1876,19 @@ def test_exist_ok_existing_regular_file(self): self.assertRaises(OSError, os.makedirs, path, exist_ok=True) os.remove(path) + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + base = os_helper.TESTFN + path = os.path.abspath(os.path.join(os_helper.TESTFN, 'dir')) + os.mkdir(path, mode=0o700) + out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem") + os.rmdir(path) + out = out.strip().rsplit(" ", 1)[1] + self.assertEqual( + out, + '"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"', + ) + def tearDown(self): path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', 'dir6') @@ -2193,9 +2351,13 @@ def test_chmod(self): @unittest.skipIf(support.is_wasi, "Cannot create invalid FD on WASI.") class TestInvalidFD(unittest.TestCase): - singles = ["fchdir", "dup", "fdatasync", "fstat", - "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] - singles_fildes = {"fchdir", "fdatasync", "fsync"} + singles = ["fchdir", "dup", "fstat", "fstatvfs", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir"} + # systemd-nspawn --suppress-sync=true does not verify fd passed + # fdatasync() and fsync(), and always returns success + if not support.in_systemd_nspawn_sync_suppressed(): + singles += ["fdatasync", "fsync"] + singles_fildes |= {"fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): @@ -2285,6 +2447,7 @@ def test_fchown(self): support.is_emscripten or support.is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_fpathconf(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") @@ -2492,8 +2655,10 @@ def test_listdir(self): # test listdir without arguments current_directory = os.getcwd() try: - os.chdir(os.sep) - self.assertEqual(set(os.listdir()), set(os.listdir(os.sep))) + # The root directory is not readable on Android, so use a directory + # we created ourselves. + os.chdir(self.dir) + self.assertEqual(set(os.listdir()), expected) finally: os.chdir(current_directory) @@ -3012,7 +3177,8 @@ class Win32NtTests(unittest.TestCase): def test_getfinalpathname_handles(self): nt = import_helper.import_module('nt') ctypes = import_helper.import_module('ctypes') - import ctypes.wintypes + # Ruff false positive -- it thinks we're redefining `ctypes` here + import ctypes.wintypes # noqa: F811 kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE @@ -3124,9 +3290,8 @@ def test_stat_inaccessible_file(self): self.skipTest("Unable to create inaccessible file") def cleanup(): - # Give delete permission. We are the file owner, so we can do this - # even though we removed all permissions earlier. - subprocess.check_output([ICACLS, filename, "/grant", "Everyone:(D)"], + # Give delete permission to the owner (us) + subprocess.check_output([ICACLS, filename, "/grant", "*WD:(D)"], stderr=subprocess.STDOUT) os.unlink(filename) @@ -3506,31 +3671,30 @@ class ProgramPriorityTests(unittest.TestCase): """Tests for os.getpriority() and os.setpriority().""" def test_set_get_priority(self): - base = os.getpriority(os.PRIO_PROCESS, os.getpid()) - os.setpriority(os.PRIO_PROCESS, os.getpid(), base + 1) - try: - new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) - if base >= 19 and new_prio <= 19: - raise unittest.SkipTest("unable to reliably test setpriority " - "at current nice level of %s" % base) - else: - self.assertEqual(new_prio, base + 1) - finally: - try: - os.setpriority(os.PRIO_PROCESS, os.getpid(), base) - except OSError as err: - if err.errno != errno.EACCES: - raise + code = f"""if 1: + import os + os.setpriority(os.PRIO_PROCESS, os.getpid(), {base} + 1) + print(os.getpriority(os.PRIO_PROCESS, os.getpid())) + """ + + # Subprocess inherits the current process' priority. + _, out, _ = assert_python_ok("-c", code) + new_prio = int(out) + # nice value cap is 19 for linux and 20 for FreeBSD + if base >= 19 and new_prio <= base: + raise unittest.SkipTest("unable to reliably test setpriority " + "at current nice level of %s" % base) + else: + self.assertEqual(new_prio, base + 1) @unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()") class TestSendfile(unittest.IsolatedAsyncioTestCase): DATA = b"12345abcde" * 16 * 1024 # 160 KiB - SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ - not sys.platform.startswith("solaris") and \ - not sys.platform.startswith("sunos") + SUPPORT_HEADERS_TRAILERS = ( + not sys.platform.startswith(("linux", "android", "solaris", "sunos"))) requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') requires_32b = unittest.skipUnless(sys.maxsize < 2**32, @@ -3853,7 +4017,12 @@ def test_does_not_crash(self): try: size = os.get_terminal_size() except OSError as e: - if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY): + known_errnos = [errno.EINVAL, errno.ENOTTY] + if sys.platform == "android": + # The Android testbed redirects the native stdout to a pipe, + # which returns a different error code. + known_errnos.append(errno.EACCES) + if sys.platform == "win32" or e.errno in known_errnos: # Under win32 a generic OSError can be thrown if the # handle cannot be retrieved self.skipTest("failed to query terminal size") @@ -4007,9 +4176,15 @@ def test_eventfd_select(self): @unittest.skipUnless(hasattr(os, 'timerfd_create'), 'requires os.timerfd_create') @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # Tolerate a difference of 1 ms - CLOCK_RES_NS = 1_000_000 - CLOCK_RES = CLOCK_RES_NS * 1e-9 + # 1 ms accuracy is reliably achievable on every platform except Android + # emulators, where we allow 10 ms (gh-108277). + if sys.platform == "android" and platform.android_ver().is_emulator: + CLOCK_RES_PLACES = 2 + else: + CLOCK_RES_PLACES = 3 + + CLOCK_RES = 10 ** -CLOCK_RES_PLACES + CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) def timerfd_create(self, *args, **kwargs): fd = os.timerfd_create(*args, **kwargs) @@ -4031,18 +4206,18 @@ def test_timerfd_initval(self): # 1st call next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) - self.assertAlmostEqual(interval2, 0.0, places=3) - self.assertAlmostEqual(next_expiration, 0.0, places=3) + self.assertAlmostEqual(interval2, 0.0, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, 0.0, places=self.CLOCK_RES_PLACES) # 2nd call next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) # timerfd_gettime next_expiration, interval2 = os.timerfd_gettime(fd) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) def test_timerfd_non_blocking(self): fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) @@ -4096,8 +4271,8 @@ def test_timerfd_interval(self): # timerfd_gettime next_expiration, interval2 = os.timerfd_gettime(fd) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) count = 3 t = time.perf_counter() @@ -4128,8 +4303,8 @@ def test_timerfd_TFD_TIMER_ABSTIME(self): # timerfd_gettime # Note: timerfd_gettime returns relative values even if TFD_TIMER_ABSTIME is specified. next_expiration, interval2 = os.timerfd_gettime(fd) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, offset, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, offset, places=self.CLOCK_RES_PLACES) t = time.perf_counter() count_signaled = self.read_count_signaled(fd) @@ -4838,7 +5013,7 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): os.name == 'nt') def test_attributes(self): - link = hasattr(os, 'link') + link = os_helper.can_hardlink() symlink = os_helper.can_symlink() dirname = os.path.join(self.path, "dir") @@ -5251,8 +5426,9 @@ def test_fork(self): else: assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") - @unittest.skipUnless(sys.platform in ("linux", "darwin"), + @unittest.skipUnless(sys.platform in ("linux", "android", "darwin"), "Only Linux and macOS detect this today.") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_fork_warns_when_non_python_thread_exists(self): code = """if 1: import os, threading, warnings @@ -5279,20 +5455,21 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - def test_fork_at_exit(self): + def test_fork_at_finalization(self): code = """if 1: import atexit import os - def exit_handler(): - pid = os.fork() - if pid != 0: - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + pid = os.fork() + if pid != 0: + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(b"", out) + self.assertEqual(b"OK\n", out) self.assertIn(b"can't fork at interpreter shutdown", err) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 2b166451243775..c7104bfda90f6c 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1,3 +1,4 @@ +import contextlib import io import os import sys @@ -15,22 +16,45 @@ from test.support import import_helper from test.support import is_emscripten, is_wasi -from test.support import set_recursion_limit +from test.support import infinite_recursion +from test.support import swap_attr from test.support import os_helper from test.support.os_helper import TESTFN, FakePath from test.test_pathlib import test_pathlib_abc from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows, needs_symlinks +try: + import fcntl +except ImportError: + fcntl = None try: import grp, pwd except ImportError: grp = pwd = None +try: + import posix +except ImportError: + posix = None root_in_posix = False if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) + +def patch_replace(old_test): + def new_replace(self, target): + raise OSError(errno.EXDEV, "Cross-device link", self, target) + + def new_test(self): + old_replace = self.cls.replace + self.cls.replace = new_replace + try: + old_test(self) + finally: + self.cls.replace = old_replace + return new_test + # # Tests for the pure classes. # @@ -65,7 +89,7 @@ def test_concrete_class(self): p = self.cls('a') self.assertIs(type(p), expected) - def test_concrete_pathmod(self): + def test_concrete_parser(self): if self.cls is pathlib.PurePosixPath: expected = posixpath elif self.cls is pathlib.PureWindowsPath: @@ -73,19 +97,19 @@ def test_concrete_pathmod(self): else: expected = os.path p = self.cls('a') - self.assertIs(p.pathmod, expected) + self.assertIs(p.parser, expected) - def test_different_pathmods_unequal(self): + def test_different_parsers_unequal(self): p = self.cls('a') - if p.pathmod is posixpath: + if p.parser is posixpath: q = pathlib.PureWindowsPath('a') else: q = pathlib.PurePosixPath('a') self.assertNotEqual(p, q) - def test_different_pathmods_unordered(self): + def test_different_parsers_unordered(self): p = self.cls('a') - if p.pathmod is posixpath: + if p.parser is posixpath: q = pathlib.PureWindowsPath('a') else: q = pathlib.PurePosixPath('a') @@ -107,17 +131,26 @@ def test_constructor_nested(self): self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) self.assertEqual(P(P('./a:b')), P('./a:b')) + @needs_windows + def test_constructor_nested_foreign_flavour(self): + # See GH-125069. + p1 = pathlib.PurePosixPath('b/c:\\d') + p2 = pathlib.PurePosixPath('b/', 'c:\\d') + self.assertEqual(p1, p2) + self.assertEqual(self.cls(p1), self.cls('b/c:/d')) + self.assertEqual(self.cls(p2), self.cls('b/c:/d')) + def _check_parse_path(self, raw_path, *expected): - sep = self.pathmod.sep + sep = self.parser.sep actual = self.cls._parse_path(raw_path.replace('/', sep)) self.assertEqual(actual, expected) - if altsep := self.pathmod.altsep: + if altsep := self.parser.altsep: actual = self.cls._parse_path(raw_path.replace('/', altsep)) self.assertEqual(actual, expected) def test_parse_path_common(self): check = self._check_parse_path - sep = self.pathmod.sep + sep = self.parser.sep check('', '', '', []) check('a', '', '', ['a']) check('a/', '', '', ['a']) @@ -139,15 +172,6 @@ def test_empty_path(self): # Special case for the empty path. self._check_str('.', ('',)) - def test_parts_interning(self): - P = self.cls - p = P('/usr/bin/foo') - q = P('/usr/local/bin') - # 'usr' - self.assertIs(p.parts[1], q.parts[1]) - # 'bin' - self.assertIs(p.parts[2], q.parts[3]) - def test_join_nested(self): P = self.cls p = P('a/b').joinpath(P('c')) @@ -311,19 +335,6 @@ def test_with_stem_empty(self): self.assertRaises(ValueError, P('a/b').with_stem, '') self.assertRaises(ValueError, P('a/b').with_stem, '.') - def test_relative_to_several_args(self): - P = self.cls - p = P('a/b') - with self.assertWarns(DeprecationWarning): - p.relative_to('a', 'b') - p.relative_to('a', 'b', walk_up=True) - - def test_is_relative_to_several_args(self): - P = self.cls - p = P('a/b') - with self.assertWarns(DeprecationWarning): - p.is_relative_to('a', 'b') - def test_is_reserved_deprecated(self): P = self.cls p = P('a/b') @@ -523,10 +534,10 @@ class PathTest(test_pathlib_abc.DummyPathTest, PurePathTest): def setUp(self): super().setUp() - os.chmod(self.pathmod.join(self.base, 'dirE'), 0) + os.chmod(self.parser.join(self.base, 'dirE'), 0) def tearDown(self): - os.chmod(self.pathmod.join(self.base, 'dirE'), 0o777) + os.chmod(self.parser.join(self.base, 'dirE'), 0o777) os_helper.rmtree(self.base) def tempdir(self): @@ -541,8 +552,8 @@ def test_matches_pathbase_api(self): path_names = {name for name in dir(pathlib._abc.PathBase) if name[0] != '_'} self.assertEqual(our_names, path_names) for attr_name in our_names: - if attr_name == 'pathmod': - # On Windows, Path.pathmod is ntpath, but PathBase.pathmod is + if attr_name == 'parser': + # On Windows, Path.parser is ntpath, but PathBase.parser is # posixpath, and so their docstrings differ. continue our_attr = getattr(self.cls, attr_name) @@ -557,9 +568,9 @@ def test_concrete_class(self): p = self.cls('a') self.assertIs(type(p), expected) - def test_unsupported_pathmod(self): - if self.cls.pathmod is os.path: - self.skipTest("path flavour is supported") + def test_unsupported_parser(self): + if self.cls.parser is os.path: + self.skipTest("path parser is supported") else: self.assertRaises(pathlib.UnsupportedOperation, self.cls) @@ -666,6 +677,190 @@ def test_open_unbuffered(self): self.assertIsInstance(f, io.RawIOBase) self.assertEqual(f.read().strip(), b"this is file A") + def test_copy_file_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'fileA' + if hasattr(os, 'chmod'): + os.chmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source, stat.UF_NODUMP) + source_st = source.stat() + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + target_st = target.stat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @os_helper.skip_unless_xattr + def test_copy_file_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'fileA' + os.setxattr(source, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertEqual(os.getxattr(target, b'user.foo'), b'42') + + @needs_symlinks + def test_copy_link_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'linkA' + if hasattr(os, 'lchmod'): + os.lchmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(source, stat.UF_NODUMP) + source_st = source.lstat() + target = base / 'copyA' + source.copy(target, follow_symlinks=False, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + target_st = target.lstat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + def test_copy_error_handling(self): + def make_raiser(err): + def raiser(*args, **kwargs): + raise OSError(err, os.strerror(err)) + return raiser + + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + + # Raise non-fatal OSError from all available fast copy functions. + with contextlib.ExitStack() as ctx: + if fcntl and hasattr(fcntl, 'FICLONE'): + ctx.enter_context(mock.patch('fcntl.ioctl', make_raiser(errno.EXDEV))) + if posix and hasattr(posix, '_fcopyfile'): + ctx.enter_context(mock.patch('posix._fcopyfile', make_raiser(errno.ENOTSUP))) + if hasattr(os, 'copy_file_range'): + ctx.enter_context(mock.patch('os.copy_file_range', make_raiser(errno.EXDEV))) + if hasattr(os, 'sendfile'): + ctx.enter_context(mock.patch('os.sendfile', make_raiser(errno.ENOTSOCK))) + + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + # Raise fatal OSError from first available fast copy function. + if fcntl and hasattr(fcntl, 'FICLONE'): + patchpoint = 'fcntl.ioctl' + elif posix and hasattr(posix, '_fcopyfile'): + patchpoint = 'posix._fcopyfile' + elif hasattr(os, 'copy_file_range'): + patchpoint = 'os.copy_file_range' + elif hasattr(os, 'sendfile'): + patchpoint = 'os.sendfile' + else: + return + with mock.patch(patchpoint, make_raiser(errno.ENOENT)): + self.assertRaises(FileNotFoundError, source.copy, target) + + @unittest.skipIf(sys.platform == "win32" or sys.platform == "wasi", "directories are always readable on Windows and WASI") + @unittest.skipIf(root_in_posix, "test fails with root privilege") + def test_copy_dir_no_read_permission(self): + base = self.cls(self.base) + source = base / 'dirE' + target = base / 'copyE' + self.assertRaises(PermissionError, source.copy, target) + self.assertFalse(target.exists()) + + def test_copy_dir_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'dirC' + if hasattr(os, 'chmod'): + os.chmod(source / 'dirD', stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source / 'fileC', stat.UF_NODUMP) + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + + for subpath in ['.', 'fileC', 'dirD', 'dirD/fileD']: + source_st = source.joinpath(subpath).stat() + target_st = target.joinpath(subpath).stat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @os_helper.skip_unless_xattr + def test_copy_dir_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'dirC' + source_file = source.joinpath('dirD', 'fileD') + os.setxattr(source_file, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + target_file = target.joinpath('dirD', 'fileD') + self.assertEqual(os.getxattr(target_file, b'user.foo'), b'42') + + @patch_replace + def test_move_file_other_fs(self): + self.test_move_file() + + @patch_replace + def test_move_file_to_file_other_fs(self): + self.test_move_file_to_file() + + @patch_replace + def test_move_file_to_dir_other_fs(self): + self.test_move_file_to_dir() + + @patch_replace + def test_move_dir_other_fs(self): + self.test_move_dir() + + @patch_replace + def test_move_dir_to_dir_other_fs(self): + self.test_move_dir_to_dir() + + @patch_replace + def test_move_dir_into_itself_other_fs(self): + self.test_move_dir_into_itself() + + @patch_replace + @needs_symlinks + def test_move_file_symlink_other_fs(self): + self.test_move_file_symlink() + + @patch_replace + @needs_symlinks + def test_move_file_symlink_to_itself_other_fs(self): + self.test_move_file_symlink_to_itself() + + @patch_replace + @needs_symlinks + def test_move_dir_symlink_other_fs(self): + self.test_move_dir_symlink() + + @patch_replace + @needs_symlinks + def test_move_dir_symlink_to_itself_other_fs(self): + self.test_move_dir_symlink_to_itself() + + @patch_replace + @needs_symlinks + def test_move_dangling_symlink_other_fs(self): + self.test_move_dangling_symlink() + + @patch_replace + def test_move_into_other_os(self): + self.test_move_into() + + @patch_replace + def test_move_into_empty_name_other_os(self): + self.test_move_into_empty_name() + def test_resolve_nonexist_relative_issue38671(self): p = self.cls('non', 'exist') @@ -777,26 +972,109 @@ def test_group_no_follow_symlinks(self): self.assertEqual(expected_gid, gid_2) self.assertEqual(expected_name, link.group(follow_symlinks=False)) - def test_unlink(self): - p = self.cls(self.base) / 'fileA' - p.unlink() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) - - def test_unlink_missing_ok(self): - p = self.cls(self.base) / 'fileAAA' - self.assertFileNotFound(p.unlink) - p.unlink(missing_ok=True) - - def test_rmdir(self): - p = self.cls(self.base) / 'dirA' - for q in p.iterdir(): - q.unlink() - p.rmdir() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_delete_unwritable(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + child_file_path = tmp / 'a' + child_dir_path = tmp / 'b' + child_file_path.write_text("") + child_dir_path.mkdir() + old_dir_mode = tmp.stat().st_mode + old_child_file_mode = child_file_path.stat().st_mode + old_child_dir_mode = child_dir_path.stat().st_mode + # Make unwritable. + new_mode = stat.S_IREAD | stat.S_IEXEC + try: + child_file_path.chmod(new_mode) + child_dir_path.chmod(new_mode) + tmp.chmod(new_mode) + + self.assertRaises(PermissionError, tmp._delete) + finally: + tmp.chmod(old_dir_mode) + child_file_path.chmod(old_child_file_mode) + child_dir_path.chmod(old_child_dir_mode) + + @needs_windows + def test_delete_inner_junction(self): + import _winapi + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + _winapi.CreateJunction(str(dir2), str(link1)) + link2 = dir1 / 'link2' + _winapi.CreateJunction(str(dir3), str(link2)) + link3 = dir1 / 'link3' + _winapi.CreateJunction(str(file1), str(link3)) + # make sure junctions are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + + @needs_windows + def test_delete_outer_junction(self): + import _winapi + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + src = tmp / 'cheese' + dst = tmp / 'shop' + src.mkdir() + spam = src / 'spam' + spam.write_text('') + _winapi.CreateJunction(str(src), str(dst)) + dst._delete() + self.assertFalse(dst.exists()) + self.assertTrue(spam.exists()) + self.assertTrue(src.exists()) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_delete_on_named_pipe(self): + p = self.cls(self.base, 'pipe') + os.mkfifo(p) + p._delete() + self.assertFalse(p.exists()) - @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") + p = self.cls(self.base, 'dir') + p.mkdir() + os.mkfifo(p / 'mypipe') + p._delete() + self.assertFalse(p.exists()) + + def test_delete_does_not_choke_on_failing_lstat(self): + try: + orig_lstat = os.lstat + tmp = self.cls(self.base, 'delete') + + def raiser(fn, *args, **kwargs): + if fn != tmp: + raise OSError() + else: + return orig_lstat(fn) + + os.lstat = raiser + + tmp.mkdir() + foo = tmp / 'foo' + foo.write_text('') + tmp._delete() + finally: + os.lstat = orig_lstat + + @os_helper.skip_unless_hardlink def test_hardlink_to(self): P = self.cls(self.base) target = P / 'fileA' @@ -809,7 +1087,7 @@ def test_hardlink_to(self): self.assertTrue(target.exists()) # Linking to a str of a relative path. link2 = P / 'dirA' / 'fileAAA' - target2 = self.pathmod.join(TESTFN, 'fileA') + target2 = self.parser.join(TESTFN, 'fileA') link2.hardlink_to(target2) self.assertEqual(os.stat(target2).st_size, size) self.assertTrue(link2.exists()) @@ -834,7 +1112,7 @@ def test_rename(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Renaming to a str of a relative path. - r = self.pathmod.join(TESTFN, 'fileAAA') + r = self.parser.join(TESTFN, 'fileAAA') renamed_q = q.rename(r) self.assertEqual(renamed_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) @@ -851,7 +1129,7 @@ def test_replace(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Replacing another (existing) path. - r = self.pathmod.join(TESTFN, 'dirB', 'fileB') + r = self.parser.join(TESTFN, 'dirB', 'fileB') replaced_q = q.replace(r) self.assertEqual(replaced_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) @@ -1060,9 +1338,9 @@ def test_symlink_to_unsupported(self): def test_is_junction(self): P = self.cls(self.base) - with mock.patch.object(P.pathmod, 'isjunction'): - self.assertEqual(P.is_junction(), P.pathmod.isjunction.return_value) - P.pathmod.isjunction.assert_called_once_with(P) + with mock.patch.object(P.parser, 'isjunction'): + self.assertEqual(P.is_junction(), P.parser.isjunction.return_value) + P.parser.isjunction.assert_called_once_with(P) @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") @unittest.skipIf(sys.platform == "vxworks", @@ -1103,15 +1381,15 @@ def test_is_socket_true(self): self.assertIs(self.cls(self.base, 'mysock\x00').is_socket(), False) def test_is_char_device_true(self): - # Under Unix, /dev/null should generally be a char device. - P = self.cls('/dev/null') + # os.devnull should generally be a char device. + P = self.cls(os.devnull) if not P.exists(): - self.skipTest("/dev/null required") + self.skipTest("null device required") self.assertTrue(P.is_char_device()) self.assertFalse(P.is_block_device()) self.assertFalse(P.is_file()) - self.assertIs(self.cls('/dev/null\udfff').is_char_device(), False) - self.assertIs(self.cls('/dev/null\x00').is_char_device(), False) + self.assertIs(self.cls(f'{os.devnull}\udfff').is_char_device(), False) + self.assertIs(self.cls(f'{os.devnull}\x00').is_char_device(), False) def test_is_mount_root(self): if os.name == 'nt': @@ -1121,8 +1399,8 @@ def test_is_mount_root(self): self.assertTrue(R.is_mount()) self.assertFalse((R / '\udfff').is_mount()) - def test_passing_kwargs_deprecated(self): - with self.assertWarns(DeprecationWarning): + def test_passing_kwargs_errors(self): + with self.assertRaises(TypeError): self.cls(foo="bar") def setUpWalk(self): @@ -1199,7 +1477,7 @@ def test_walk_above_recursion_limit(self): path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True) - with set_recursion_limit(recursion_limit): + with infinite_recursion(recursion_limit): list(base.walk()) list(base.walk(top_down=False)) @@ -1239,7 +1517,7 @@ def test_glob_above_recursion_limit(self): path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True) - with set_recursion_limit(recursion_limit): + with infinite_recursion(recursion_limit): list(base.glob('**/')) def test_glob_pathlike(self): @@ -1250,6 +1528,26 @@ def test_glob_pathlike(self): self.assertEqual(expect, set(p.glob(P(pattern)))) self.assertEqual(expect, set(p.glob(FakePath(pattern)))) + @needs_symlinks + def test_glob_dot(self): + P = self.cls + with os_helper.change_cwd(P(self.base, "dirC")): + self.assertEqual( + set(P('.').glob('*')), {P("fileC"), P("novel.txt"), P("dirD")}) + self.assertEqual( + set(P('.').glob('**')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD"), P(".")}) + self.assertEqual( + set(P('.').glob('**/*')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD")}) + self.assertEqual( + set(P('.').glob('**/*/*')), {P("dirD/fileD")}) + + def test_glob_inaccessible(self): + P = self.cls + p = P(self.base, "mydir1", "mydir2") + p.mkdir(parents=True) + p.parent.chmod(0) + self.assertEqual(set(p.glob('*')), set()) + def test_rglob_pathlike(self): P = self.cls p = P(self.base, "dirC") @@ -1276,18 +1574,20 @@ def test_absolute_posix(self): ) @needs_posix def test_open_mode(self): - old_mask = os.umask(0) + # Unmask all permissions except world-write, which may + # not be supported on some filesystems (see GH-85633.) + old_mask = os.umask(0o002) self.addCleanup(os.umask, old_mask) p = self.cls(self.base) with (p / 'new_file').open('wb'): pass - st = os.stat(self.pathmod.join(self.base, 'new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o664) + os.umask(0o026) with (p / 'other_new_file').open('wb'): pass - st = os.stat(self.pathmod.join(self.base, 'other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o640) @needs_posix def test_resolve_root(self): @@ -1305,18 +1605,20 @@ def test_resolve_root(self): ) @needs_posix def test_touch_mode(self): - old_mask = os.umask(0) + # Unmask all permissions except world-write, which may + # not be supported on some filesystems (see GH-85633.) + old_mask = os.umask(0o002) self.addCleanup(os.umask, old_mask) p = self.cls(self.base) (p / 'new_file').touch() - st = os.stat(self.pathmod.join(self.base, 'new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o664) + os.umask(0o026) (p / 'other_new_file').touch() - st = os.stat(self.pathmod.join(self.base, 'other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o640) (p / 'masked_new_file').touch(mode=0o750) - st = os.stat(self.pathmod.join(self.base, 'masked_new_file')) + st = os.stat(self.parser.join(self.base, 'masked_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) @unittest.skipUnless(hasattr(pwd, 'getpwall'), diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 1d30deca8f7a1b..4ab804850e9c3e 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1,13 +1,15 @@ import collections +import contextlib import io import os import errno import stat import unittest -from pathlib._abc import UnsupportedOperation, PathModuleBase, PurePathBase, PathBase +from pathlib._abc import UnsupportedOperation, ParserBase, PurePathBase, PathBase import posixpath +from test.support import is_wasi from test.support.os_helper import TESTFN @@ -38,8 +40,8 @@ def test_is_notimplemented(self): self.assertTrue(isinstance(UnsupportedOperation(), NotImplementedError)) -class PathModuleBaseTest(unittest.TestCase): - cls = PathModuleBase +class ParserBaseTest(unittest.TestCase): + cls = ParserBase def test_unsupported_operation(self): m = self.cls() @@ -49,6 +51,7 @@ def test_unsupported_operation(self): self.assertRaises(e, m.join, 'foo') self.assertRaises(e, m.split, 'foo') self.assertRaises(e, m.splitdrive, 'foo') + self.assertRaises(e, m.splitext, 'foo') self.assertRaises(e, m.normcase, 'foo') self.assertRaises(e, m.isabs, 'foo') @@ -109,13 +112,13 @@ def test_magic_methods(self): self.assertIs(P.__gt__, object.__gt__) self.assertIs(P.__ge__, object.__ge__) - def test_pathmod(self): - self.assertIsInstance(self.cls.pathmod, PathModuleBase) + def test_parser(self): + self.assertIsInstance(self.cls.parser, ParserBase) class DummyPurePath(PurePathBase): __slots__ = () - pathmod = posixpath + parser = posixpath def __eq__(self, other): if not isinstance(other, DummyPurePath): @@ -137,14 +140,14 @@ class DummyPurePathTest(unittest.TestCase): def setUp(self): name = self.id().split('.')[-1] - if name in _tests_needing_posix and self.cls.pathmod is not posixpath: + if name in _tests_needing_posix and self.cls.parser is not posixpath: self.skipTest('requires POSIX-flavoured path class') - if name in _tests_needing_windows and self.cls.pathmod is posixpath: + if name in _tests_needing_windows and self.cls.parser is posixpath: self.skipTest('requires Windows-flavoured path class') p = self.cls('a') - self.pathmod = p.pathmod - self.sep = self.pathmod.sep - self.altsep = self.pathmod.altsep + self.parser = p.parser + self.sep = self.parser.sep + self.altsep = self.parser.altsep def test_constructor_common(self): P = self.cls @@ -512,8 +515,6 @@ def test_full_match_common(self): self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) self.assertFalse(P('a/b/c.py').full_match('/a/b/c.py/**')) self.assertFalse(P('a/b/c.py').full_match('/**/a/b/c.py')) - self.assertRaises(ValueError, P('a').full_match, '**a/b/c') - self.assertRaises(ValueError, P('a').full_match, 'a/b/c**') # Case-sensitive flag self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) self.assertTrue(P('A.py').full_match('a.PY', case_sensitive=False)) @@ -790,8 +791,12 @@ def test_suffix_common(self): self.assertEqual(P('/a/.hg.rc').suffix, '.rc') self.assertEqual(P('a/b.tar.gz').suffix, '.gz') self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('a/trailing.dot.').suffix, '.') + self.assertEqual(P('/a/trailing.dot.').suffix, '.') + self.assertEqual(P('a/..d.o.t..').suffix, '.') + self.assertEqual(P('a/inn.er..dots').suffix, '.dots') + self.assertEqual(P('photo').suffix, '') + self.assertEqual(P('photo.jpg').suffix, '.jpg') @needs_windows def test_suffix_windows(self): @@ -808,8 +813,8 @@ def test_suffix_windows(self): self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('c:a/trailing.dot.').suffix, '.') + self.assertEqual(P('c:/a/trailing.dot.').suffix, '.') self.assertEqual(P('//My.py/Share.php').suffix, '') self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') @@ -829,8 +834,12 @@ def test_suffixes_common(self): self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('/a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('a/..d.o.t..').suffixes, ['.o', '.t', '.', '.']) + self.assertEqual(P('a/inn.er..dots').suffixes, ['.er', '.', '.dots']) + self.assertEqual(P('photo').suffixes, []) + self.assertEqual(P('photo.jpg').suffixes, ['.jpg']) @needs_windows def test_suffixes_windows(self): @@ -849,8 +858,8 @@ def test_suffixes_windows(self): self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) self.assertEqual(P('//My.py/Share.php').suffixes, []) self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('c:a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('c:/a/trailing.dot.').suffixes, ['.dot', '.']) def test_stem_empty(self): P = self.cls @@ -866,8 +875,11 @@ def test_stem_common(self): self.assertEqual(P('a/.hgrc').stem, '.hgrc') self.assertEqual(P('a/.hg.rc').stem, '.hg') self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') + self.assertEqual(P('a/trailing.dot.').stem, 'trailing.dot') + self.assertEqual(P('a/..d.o.t..').stem, '..d.o.t.') + self.assertEqual(P('a/inn.er..dots').stem, 'inn.er.') + self.assertEqual(P('photo').stem, 'photo') + self.assertEqual(P('photo.jpg').stem, 'photo') @needs_windows def test_stem_windows(self): @@ -881,8 +893,8 @@ def test_stem_windows(self): self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') self.assertEqual(P('c:a/.hg.rc').stem, '.hg') self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') + self.assertEqual(P('c:a/trailing.dot.').stem, 'trailing.dot') + def test_with_name_common(self): P = self.cls self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) @@ -930,16 +942,16 @@ def test_with_stem_common(self): self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) - self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) + self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d.')) + self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d.')) @needs_windows def test_with_stem_windows(self): P = self.cls self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d.')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d.')) self.assertRaises(ValueError, P('c:').with_stem, 'd') self.assertRaises(ValueError, P('c:/').with_stem, 'd') self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') @@ -957,6 +969,8 @@ def test_with_stem_empty(self): self.assertEqual(P('/').with_stem('d'), P('/d')) self.assertEqual(P('a/b').with_stem(''), P('a/')) self.assertEqual(P('a/b').with_stem('.'), P('a/.')) + self.assertRaises(ValueError, P('foo.gz').with_stem, '') + self.assertRaises(ValueError, P('/a/b/foo.gz').with_stem, '') def test_with_stem_seps(self): P = self.cls @@ -973,6 +987,11 @@ def test_with_suffix_common(self): # Stripping suffix. self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + # Single dot + self.assertEqual(P('a/b').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b').with_suffix('.'), P('/a/b.')) + self.assertEqual(P('a/b.py').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b.py').with_suffix('.'), P('/a/b.')) @needs_windows def test_with_suffix_windows(self): @@ -998,6 +1017,7 @@ def test_with_suffix_windows(self): self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') + self.assertRaises(TypeError, P('c:a/b').with_suffix, None) def test_with_suffix_empty(self): P = self.cls @@ -1005,17 +1025,17 @@ def test_with_suffix_empty(self): self.assertRaises(ValueError, P('').with_suffix, '.gz') self.assertRaises(ValueError, P('/').with_suffix, '.gz') - def test_with_suffix_seps(self): + def test_with_suffix_invalid(self): P = self.cls # Invalid suffix. self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') self.assertRaises(ValueError, P('a/b').with_suffix, '/') - self.assertRaises(ValueError, P('a/b').with_suffix, '.') self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') self.assertRaises(ValueError, P('a/b').with_suffix, './.d') self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') + self.assertRaises(TypeError, P('a/b').with_suffix, None) def test_relative_to_common(self): P = self.cls @@ -1405,13 +1425,31 @@ def close(self): 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') +class DummyDirEntry: + """ + Minimal os.DirEntry-like object. Returned from DummyPath.scandir(). + """ + __slots__ = ('name', '_is_symlink', '_is_dir') + + def __init__(self, name, is_symlink, is_dir): + self.name = name + self._is_symlink = is_symlink + self._is_dir = is_dir + + def is_symlink(self): + return self._is_symlink + + def is_dir(self, *, follow_symlinks=True): + return self._is_dir and (follow_symlinks or not self._is_symlink) + + class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in memory. """ __slots__ = () - pathmod = posixpath + parser = posixpath _files = {} _directories = {} @@ -1429,10 +1467,10 @@ def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) def stat(self, *, follow_symlinks=True): - if follow_symlinks: - path = str(self.resolve()) + if follow_symlinks or self.name in ('', '.', '..'): + path = str(self.resolve(strict=True)) else: - path = str(self.parent.resolve() / self.name) + path = str(self.parent.resolve(strict=True) / self.name) if path in self._files: st_mode = stat.S_IFREG elif path in self._directories: @@ -1445,7 +1483,7 @@ def stat(self, *, follow_symlinks=True): def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): - if buffering != -1: + if buffering != -1 and not (buffering == 0 and 'b' in mode): raise NotImplementedError path_obj = self.resolve() path = str(path_obj) @@ -1472,32 +1510,75 @@ def open(self, mode='r', buffering=-1, encoding=None, stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) return stream - def iterdir(self): - path = str(self.resolve()) - if path in self._files: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) - elif path in self._directories: - return (self / name for name in self._directories[path]) + @contextlib.contextmanager + def scandir(self): + path = self.resolve() + path_str = str(path) + if path_str in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path_str) + elif path_str in self._directories: + yield iter([path.joinpath(name)._dir_entry for name in self._directories[path_str]]) else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) + raise FileNotFoundError(errno.ENOENT, "File not found", path_str) + + @property + def _dir_entry(self): + path_str = str(self) + is_symlink = path_str in self._symlinks + is_directory = (path_str in self._directories + if not is_symlink + else self._symlinks[path_str][1]) + return DummyDirEntry(self.name, is_symlink, is_directory) def mkdir(self, mode=0o777, parents=False, exist_ok=False): - path = str(self.resolve()) - if path in self._directories: + path = str(self.parent.resolve() / self.name) + parent = str(self.parent.resolve()) + if path in self._directories or path in self._symlinks: if exist_ok: return else: raise FileExistsError(errno.EEXIST, "File exists", path) try: if self.name: - self._directories[str(self.parent)].add(self.name) + self._directories[parent].add(self.name) self._directories[path] = set() except KeyError: if not parents: - raise FileNotFoundError(errno.ENOENT, "File not found", str(self.parent)) from None + raise FileNotFoundError(errno.ENOENT, "File not found", parent) from None self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) + def unlink(self, missing_ok=False): + path_obj = self.parent.resolve(strict=True) / self.name + path = str(path_obj) + name = path_obj.name + parent = str(path_obj.parent) + if path in self._directories: + raise IsADirectoryError(errno.EISDIR, "Is a directory", path) + elif path in self._files: + self._directories[parent].remove(name) + del self._files[path] + elif path in self._symlinks: + self._directories[parent].remove(name) + del self._symlinks[path] + elif not missing_ok: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def rmdir(self): + path_obj = self.parent.resolve(strict=True) / self.name + path = str(path_obj) + if path in self._files or path in self._symlinks: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) + elif path not in self._directories: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + elif self._directories[path]: + raise OSError(errno.ENOTEMPTY, "Directory not empty", path) + else: + name = path_obj.name + parent = str(path_obj.parent) + self._directories[parent].remove(name) + del self._directories[path] + class DummyPathTest(DummyPurePathTest): """Tests for PathBase methods that use stat(), open() and iterdir().""" @@ -1530,7 +1611,7 @@ def setUp(self): name = self.id().split('.')[-1] if name in _tests_needing_symlinks and not self.can_symlink: self.skipTest('requires symlinks') - pathmod = self.cls.pathmod + parser = self.cls.parser p = self.cls(self.base) p.mkdir(parents=True) p.joinpath('dirA').mkdir() @@ -1551,9 +1632,11 @@ def setUp(self): if self.can_symlink: p.joinpath('linkA').symlink_to('fileA') p.joinpath('brokenLink').symlink_to('non-existing') - p.joinpath('linkB').symlink_to('dirB') - p.joinpath('dirA', 'linkC').symlink_to(pathmod.join('..', 'dirB')) - p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB')) + p.joinpath('linkB').symlink_to('dirB', target_is_directory=True) + p.joinpath('dirA', 'linkC').symlink_to( + parser.join('..', 'dirB'), target_is_directory=True) + p.joinpath('dirB', 'linkD').symlink_to( + parser.join('..', 'dirB'), target_is_directory=True) p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') def tearDown(self): @@ -1573,13 +1656,13 @@ def assertFileNotFound(self, func, *args, **kwargs): self.assertEqual(cm.exception.errno, errno.ENOENT) def assertEqualNormCase(self, path_a, path_b): - normcase = self.pathmod.normcase + normcase = self.parser.normcase self.assertEqual(normcase(path_a), normcase(path_b)) def test_samefile(self): - pathmod = self.pathmod - fileA_path = pathmod.join(self.base, 'fileA') - fileB_path = pathmod.join(self.base, 'dirB', 'fileB') + parser = self.parser + fileA_path = parser.join(self.base, 'fileA') + fileB_path = parser.join(self.base, 'dirB', 'fileB') p = self.cls(fileA_path) pp = self.cls(fileA_path) q = self.cls(fileB_path) @@ -1588,7 +1671,7 @@ def test_samefile(self): self.assertFalse(p.samefile(fileB_path)) self.assertFalse(p.samefile(q)) # Test the non-existent file case - non_existent = pathmod.join(self.base, 'foo') + non_existent = parser.join(self.base, 'foo') r = self.cls(non_existent) self.assertRaises(FileNotFoundError, p.samefile, r) self.assertRaises(FileNotFoundError, p.samefile, non_existent) @@ -1677,6 +1760,436 @@ def test_write_text_with_newlines(self): self.assertEqual((p / 'fileA').read_bytes(), b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') + def test_copy_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + @needs_symlinks + def test_copy_symlink_follow_symlinks_true(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertFalse(target.is_symlink()) + self.assertEqual(source.read_text(), target.read_text()) + + @needs_symlinks + def test_copy_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.copy, source) + + @needs_symlinks + def test_copy_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_directory_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_into_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'linkB' / 'copyB' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + self.assertFalse(target.exists()) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC') + target.symlink_to(base / 'fileA') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC' / 'dirD') + target.symlink_to(base / 'dirC') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + def test_copy_file_to_existing_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' / 'fileB' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_file_to_existing_directory(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirA' + self.assertRaises(OSError, source.copy, target) + + @needs_symlinks + def test_copy_file_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @needs_symlinks + def test_copy_file_to_existing_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + def test_copy_file_empty(self): + base = self.cls(self.base) + source = base / 'empty' + target = base / 'copyA' + source.write_bytes(b'') + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertEqual(target.read_bytes(), b'') + + def test_copy_file_to_itself(self): + base = self.cls(self.base) + source = base / 'empty' + source.write_bytes(b'') + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + def test_copy_dir_simple(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + def test_copy_dir_complex(self, follow_symlinks=True): + def ordered_walk(path): + for dirpath, dirnames, filenames in path.walk(follow_symlinks=follow_symlinks): + dirnames.sort() + filenames.sort() + yield dirpath, dirnames, filenames + base = self.cls(self.base) + source = base / 'dirC' + + if self.can_symlink: + # Add some symlinks + source.joinpath('linkC').symlink_to('fileC') + source.joinpath('linkD').symlink_to('dirD', target_is_directory=True) + + # Perform the copy + target = base / 'copyC' + result = source.copy(target, follow_symlinks=follow_symlinks) + self.assertEqual(result, target) + + # Compare the source and target trees + source_walk = ordered_walk(source) + target_walk = ordered_walk(target) + for source_item, target_item in zip(source_walk, target_walk, strict=True): + self.assertEqual(source_item[0].relative_to(source), + target_item[0].relative_to(target)) # dirpath + self.assertEqual(source_item[1], target_item[1]) # dirnames + self.assertEqual(source_item[2], target_item[2]) # filenames + # Compare files and symlinks + for filename in source_item[2]: + source_file = source_item[0].joinpath(filename) + target_file = target_item[0].joinpath(filename) + if follow_symlinks or not source_file.is_symlink(): + # Regular file. + self.assertEqual(source_file.read_bytes(), target_file.read_bytes()) + elif source_file.is_dir(): + # Symlink to directory. + self.assertTrue(target_file.is_dir()) + self.assertEqual(source_file.readlink(), target_file.readlink()) + else: + # Symlink to file. + self.assertEqual(source_file.read_bytes(), target_file.read_bytes()) + self.assertEqual(source_file.readlink(), target_file.readlink()) + + def test_copy_dir_complex_follow_symlinks_false(self): + self.test_copy_dir_complex(follow_symlinks=False) + + def test_copy_dir_to_existing_directory(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + target.mkdir() + target.joinpath('dirD').mkdir() + self.assertRaises(FileExistsError, source.copy, target) + + def test_copy_dir_to_existing_directory_dirs_exist_ok(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + target.mkdir() + target.joinpath('dirD').mkdir() + result = source.copy(target, dirs_exist_ok=True) + self.assertEqual(result, target) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + def test_copy_dir_to_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + def test_copy_dir_into_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC' / 'dirD' / 'copyC' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + self.assertFalse(target.exists()) + + @needs_symlinks + def test_copy_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'source' + target = base / 'target' + + source.mkdir() + source.joinpath('link').symlink_to('nonexistent') + + self.assertRaises(FileNotFoundError, source.copy, target) + + target2 = base / 'target2' + result = source.copy(target2, follow_symlinks=False) + self.assertEqual(result, target2) + self.assertTrue(target2.joinpath('link').is_symlink()) + self.assertEqual(target2.joinpath('link').readlink(), self.cls('nonexistent')) + + def test_copy_into(self): + base = self.cls(self.base) + source = base / 'fileA' + target_dir = base / 'dirA' + result = source.copy_into(target_dir) + self.assertEqual(result, target_dir / 'fileA') + self.assertTrue(result.exists()) + self.assertEqual(source.read_text(), result.read_text()) + + def test_copy_into_empty_name(self): + source = self.cls('') + target_dir = self.base + self.assertRaises(ValueError, source.copy_into, target_dir) + + def test_move_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'fileA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + + def test_move_file_to_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'dirB' / 'fileB' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + + def test_move_file_to_dir(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + + def test_move_file_to_itself(self): + base = self.cls(self.base) + source = base / 'fileA' + self.assertRaises(OSError, source.move, source) + + def test_move_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + def test_move_dir_to_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertTrue(target.exists()) + + def test_move_dir_to_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + self.assertRaises(OSError, source.move, source) + self.assertTrue(source.exists()) + + def test_move_dir_into_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC' / 'bar' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertFalse(target.exists()) + + @needs_symlinks + def test_move_file_symlink(self): + base = self.cls(self.base) + source = base / 'linkA' + source_readlink = source.readlink() + target = base / 'linkA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_file_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dir_symlink(self): + base = self.cls(self.base) + source = base / 'linkB' + source_readlink = source.readlink() + target = base / 'linkB_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_dir_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'brokenLink' + source_readlink = source.readlink() + target = base / 'brokenLink_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + def test_move_into(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target_dir = base / 'dirA' + result = source.move_into(target_dir) + self.assertEqual(result, target_dir / 'fileA') + self.assertFalse(source.exists()) + self.assertTrue(result.exists()) + self.assertEqual(source_text, result.read_text()) + + def test_move_into_empty_name(self): + source = self.cls('') + target_dir = self.base + self.assertRaises(ValueError, source.move_into, target_dir) + def test_iterdir(self): P = self.cls p = P(self.base) @@ -1706,6 +2219,23 @@ def test_iterdir_nodir(self): self.assertIn(cm.exception.errno, (errno.ENOTDIR, errno.ENOENT, errno.EINVAL)) + def test_scandir(self): + p = self.cls(self.base) + with p.scandir() as entries: + self.assertTrue(list(entries)) + with p.scandir() as entries: + for entry in entries: + child = p / entry.name + self.assertIsNotNone(entry) + self.assertEqual(entry.name, child.name) + self.assertEqual(entry.is_symlink(), + child.is_symlink()) + self.assertEqual(entry.is_dir(follow_symlinks=False), + child.is_dir(follow_symlinks=False)) + if entry.name != 'brokenLinkLoop': + self.assertEqual(entry.is_dir(), child.is_dir()) + + def test_glob_common(self): def _check(glob, expected): self.assertEqual(set(glob), { P(self.base, q) for q in expected }) @@ -1741,8 +2271,9 @@ def _check(glob, expected): def test_glob_posix(self): P = self.cls p = P(self.base) + q = p / "FILEa" given = set(p.glob("FILEa")) - expect = set() + expect = {q} if q.exists() else set() self.assertEqual(given, expect) self.assertEqual(set(p.glob("FILEa*")), set()) @@ -1753,8 +2284,6 @@ def test_glob_windows(self): self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) - self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) def test_glob_empty_pattern(self): P = self.cls @@ -1776,9 +2305,9 @@ def _check(path, pattern, case_sensitive, expected): _check(path, "dirb/file*", False, ["dirB/fileB"]) @needs_symlinks - def test_glob_follow_symlinks_common(self): + def test_glob_recurse_symlinks_common(self): def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=True) + actual = {path for path in path.glob(glob, recurse_symlinks=True) if path.parts.count("linkD") <= 1} # exclude symlink loop. self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls @@ -1812,88 +2341,54 @@ def _check(path, glob, expected): _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "*/dirD/**/", ["dirC/dirD/"]) - @needs_symlinks - def test_glob_no_follow_symlinks_common(self): + def test_rglob_recurse_symlinks_false(self): def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=False)} + actual = set(path.rglob(glob, recurse_symlinks=False)) self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls p = P(self.base) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"]) - _check(p, "dir*/*/..", ["dirC/dirD/.."]) - _check(p, "dir*/**", [ - "dirA/", "dirA/linkC", - "dirB/", "dirB/fileB", "dirB/linkD", - "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", - "dirE/"]) - _check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "dir*/*/**/", ["dirC/dirD/"]) - _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**", ["dirC/dirD/../dirD/", "dirC/dirD/../dirD/fileD"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) - _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "*/dirD/**/", ["dirC/dirD/"]) - - def test_rglob_common(self): - def _check(glob, expected): - self.assertEqual(set(glob), {P(self.base, q) for q in expected}) - P = self.cls - p = P(self.base) it = p.rglob("fileA") self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.rglob("fileB"), ["dirB/fileB"]) - _check(p.rglob("**/fileB"), ["dirB/fileB"]) - _check(p.rglob("*/fileA"), []) - if not self.can_symlink: - _check(p.rglob("*/fileB"), ["dirB/fileB"]) - else: - _check(p.rglob("*/fileB"), ["dirB/fileB", "dirB/linkD/fileB", - "linkB/fileB", "dirA/linkC/fileB"]) - _check(p.rglob("file*"), ["fileA", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD"]) - if not self.can_symlink: - _check(p.rglob("*/"), [ - "dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/", - ]) - else: - _check(p.rglob("*/"), [ + _check(p, "fileA", ["fileA"]) + _check(p, "fileB", ["dirB/fileB"]) + _check(p, "**/fileB", ["dirB/fileB"]) + _check(p, "*/fileA", []) + + if self.can_symlink: + _check(p, "*/fileB", ["dirB/fileB", "dirB/linkD/fileB", + "linkB/fileB", "dirA/linkC/fileB"]) + _check(p, "*/", [ "dirA/", "dirA/linkC/", "dirB/", "dirB/linkD/", "dirC/", - "dirC/dirD/", "dirE/", "linkB/", - ]) - _check(p.rglob(""), ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + "dirC/dirD/", "dirE/", "linkB/"]) + else: + _check(p, "*/fileB", ["dirB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "", ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) p = P(self.base, "dirC") - _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) - _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**"), ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**/"), ["dirC/dirD/"]) - _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) - _check(p.rglob("*/"), ["dirC/dirD/"]) - _check(p.rglob(""), ["dirC/", "dirC/dirD/"]) - _check(p.rglob("**"), [ - "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) - _check(p.rglob("**/"), ["dirC/", "dirC/dirD/"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "**/file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "dir*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/**/", ["dirC/dirD/"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + _check(p, "**", ["dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) + _check(p, "**/", ["dirC/", "dirC/dirD/"]) # gh-91616, a re module regression - _check(p.rglob("*.txt"), ["dirC/novel.txt"]) - _check(p.rglob("*.*"), ["dirC/novel.txt"]) + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) @needs_posix def test_rglob_posix(self): P = self.cls p = P(self.base, "dirC") + q = p / "dirD" / "FILEd" given = set(p.rglob("FILEd")) - expect = set() + expect = {q} if q.exists() else set() self.assertEqual(given, expect) self.assertEqual(set(p.rglob("FILEd*")), set()) @@ -1903,12 +2398,11 @@ def test_rglob_windows(self): p = P(self.base, "dirC") self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) @needs_symlinks - def test_rglob_follow_symlinks_common(self): + def test_rglob_recurse_symlinks_common(self): def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=True) + actual = {path for path in path.rglob(glob, recurse_symlinks=True) if path.parts.count("linkD") <= 1} # exclude symlink loop. self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls @@ -1937,37 +2431,12 @@ def _check(path, glob, expected): _check(p, "*.txt", ["dirC/novel.txt"]) _check(p, "*.*", ["dirC/novel.txt"]) - @needs_symlinks - def test_rglob_no_follow_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", ["dirB/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", ]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "", ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) - - p = P(self.base, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD/"]) - _check(p, "", ["dirC/", "dirC/dirD/"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - @needs_symlinks def test_rglob_symlink_loop(self): # Don't get fooled by symlink loops (Issue #26012). P = self.cls p = P(self.base) - given = set(p.rglob('*')) + given = set(p.rglob('*', recurse_symlinks=False)) expect = {'brokenLink', 'dirA', 'dirA/linkC', 'dirB', 'dirB/fileB', 'dirB/linkD', @@ -1981,6 +2450,8 @@ def test_rglob_symlink_loop(self): } self.assertEqual(given, {p / x for x in expect}) + # See https://github.com/WebAssembly/wasi-filesystem/issues/26 + @unittest.skipIf(is_wasi, "WASI resolution of '..' parts doesn't match POSIX") def test_glob_dotdot(self): # ".." is not special in globs. P = self.cls @@ -1991,7 +2462,11 @@ def test_glob_dotdot(self): self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) self.assertEqual(set(p.glob("dirA/../file*/..")), set()) self.assertEqual(set(p.glob("../xyzzy")), set()) - self.assertEqual(set(p.glob("xyzzy/..")), set()) + if self.cls.parser is posixpath: + self.assertEqual(set(p.glob("xyzzy/..")), set()) + else: + # ".." segments are normalized first on Windows, so this path is stat()able. + self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) @needs_symlinks @@ -2006,7 +2481,7 @@ def test_glob_permissions(self): if i % 2: link.symlink_to(P(self.base, "dirE", "nonexistent")) else: - link.symlink_to(P(self.base, "dirC")) + link.symlink_to(P(self.base, "dirC"), target_is_directory=True) self.assertEqual(len(set(base.glob("*"))), 100) self.assertEqual(len(set(base.glob("*/"))), 50) @@ -2055,15 +2530,15 @@ def test_resolve_common(self): p.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ENOENT) # Non-strict - pathmod = self.pathmod + parser = self.parser self.assertEqualNormCase(str(p.resolve(strict=False)), - pathmod.join(self.base, 'foo')) + parser.join(self.base, 'foo')) p = P(self.base, 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - pathmod.join(self.base, 'foo', 'in', 'spam')) + parser.join(self.base, 'foo', 'in', 'spam')) p = P(self.base, '..', 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - pathmod.join(pathmod.dirname(self.base), 'foo', 'in', 'spam')) + parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) # These are all relative symlinks. p = P(self.base, 'dirB', 'fileB') self._check_resolve_relative(p, p) @@ -2078,7 +2553,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', 'spam'), False) p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if self.cls.pathmod is not posixpath: + if self.cls.parser is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', @@ -2090,7 +2565,7 @@ def test_resolve_common(self): # Now create absolute symlinks. d = self.tempdir() P(self.base, 'dirA', 'linkX').symlink_to(d) - P(self.base, str(d), 'linkY').symlink_to(self.pathmod.join(self.base, 'dirB')) + P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) # Non-strict @@ -2098,7 +2573,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), False) p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if self.cls.pathmod is not posixpath: + if self.cls.parser is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) @@ -2110,11 +2585,11 @@ def test_resolve_common(self): @needs_symlinks def test_resolve_dot(self): # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - pathmod = self.pathmod + parser = self.parser p = self.cls(self.base) p.joinpath('0').symlink_to('.', target_is_directory=True) - p.joinpath('1').symlink_to(pathmod.join('0', '0'), target_is_directory=True) - p.joinpath('2').symlink_to(pathmod.join('1', '1'), target_is_directory=True) + p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) q = p / '2' self.assertEqual(q.resolve(strict=True), p) r = q / '3' / '4' @@ -2142,11 +2617,11 @@ def test_resolve_loop(self): p = self.cls(self.base, 'linkZ', 'foo') self.assertEqual(p.resolve(strict=False), p) # Loops with absolute symlinks. - self.cls(self.base, 'linkU').symlink_to(self.pathmod.join(self.base, 'linkU/inside')) + self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) self._check_symlink_loop(self.base, 'linkU') - self.cls(self.base, 'linkV').symlink_to(self.pathmod.join(self.base, 'linkV')) + self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) self._check_symlink_loop(self.base, 'linkV') - self.cls(self.base, 'linkW').symlink_to(self.pathmod.join(self.base, 'linkW/../linkW')) + self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) self._check_symlink_loop(self.base, 'linkW') # Non-strict q = self.cls(self.base, 'linkW', 'foo') @@ -2318,11 +2793,11 @@ def test_is_char_device_false(self): def _check_complex_symlinks(self, link0_target): # Test solving a non-looping chain of symlinks (issue #19887). - pathmod = self.pathmod + parser = self.parser P = self.cls(self.base) - P.joinpath('link1').symlink_to(pathmod.join('link0', 'link0'), target_is_directory=True) - P.joinpath('link2').symlink_to(pathmod.join('link1', 'link1'), target_is_directory=True) - P.joinpath('link3').symlink_to(pathmod.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) # Resolve absolute paths. @@ -2372,7 +2847,90 @@ def test_complex_symlinks_relative(self): @needs_symlinks def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(self.pathmod.join('dirA', '..')) + self._check_complex_symlinks(self.parser.join('dirA', '..')) + + def test_unlink(self): + p = self.cls(self.base) / 'fileA' + p.unlink() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_unlink_missing_ok(self): + p = self.cls(self.base) / 'fileAAA' + self.assertFileNotFound(p.unlink) + p.unlink(missing_ok=True) + + def test_rmdir(self): + p = self.cls(self.base) / 'dirA' + for q in p.iterdir(): + q.unlink() + p.rmdir() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_delete_file(self): + p = self.cls(self.base) / 'fileA' + p._delete() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_delete_dir(self): + base = self.cls(self.base) + base.joinpath('dirA')._delete() + self.assertRaises(FileNotFoundError, base.joinpath('dirA').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirA', 'linkC').lstat) + base.joinpath('dirB')._delete() + self.assertRaises(FileNotFoundError, base.joinpath('dirB').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'fileB').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'linkD').lstat) + base.joinpath('dirC')._delete() + self.assertRaises(FileNotFoundError, base.joinpath('dirC').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD', 'fileD').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'fileC').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'novel.txt').stat) + + @needs_symlinks + def test_delete_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir_ = tmp / 'dir' + dir_.mkdir() + link = tmp / 'link' + link.symlink_to(dir_) + link._delete() + self.assertTrue(dir_.exists()) + self.assertFalse(link.exists(follow_symlinks=False)) + + @needs_symlinks + def test_delete_inner_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + link1.symlink_to(dir2) + link2 = dir1 / 'link2' + link2.symlink_to(dir3) + link3 = dir1 / 'link3' + link3.symlink_to(file1) + # make sure symlinks are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + + def test_delete_missing(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + # filename is guaranteed not to exist + filename = tmp / 'foo' + self.assertRaises(FileNotFoundError, filename._delete) def setUpWalk(self): # Build: @@ -2411,7 +2969,7 @@ def setUpWalk(self): f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") if self.can_symlink: - self.link_path.symlink_to(t2_path) + self.link_path.symlink_to(t2_path, target_is_directory=True) broken_link_path.symlink_to('broken') broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) @@ -2529,15 +3087,19 @@ class DummyPathWithSymlinks(DummyPath): def readlink(self): path = str(self.parent.resolve() / self.name) if path in self._symlinks: - return self.with_segments(self._symlinks[path]) + return self.with_segments(self._symlinks[path][0]) elif path in self._files or path in self._directories: raise OSError(errno.EINVAL, "Not a symlink", path) else: raise FileNotFoundError(errno.ENOENT, "File not found", path) def symlink_to(self, target, target_is_directory=False): - self._directories[str(self.parent)].add(self.name) - self._symlinks[str(self)] = str(target) + path = str(self.parent.resolve() / self.name) + parent = str(self.parent.resolve()) + if path in self._symlinks: + raise FileExistsError(errno.EEXIST, "File exists", path) + self._directories[parent].add(self.name) + self._symlinks[path] = str(target), target_is_directory class DummyPathWithSymlinksTest(DummyPathTest): diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 298e78ccee3875..dae6d898964cff 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -1,6 +1,7 @@ import array import collections import dataclasses +import dis import enum import inspect import sys @@ -2957,6 +2958,14 @@ def test_invalid_syntax_3(self): pass """) + def test_len1_tuple_sequence_pattern_comma(self): + # correct syntax would be `case(*x,):` + self.assert_syntax_error(""" + match ...: + case (*x): + pass + """) + def test_mapping_pattern_keys_may_only_match_literals_and_attribute_lookups(self): self.assert_syntax_error(""" match ...: @@ -3006,6 +3015,13 @@ def test_multiple_assignments_to_name_in_pattern_5(self): pass """) + def test_multiple_assignments_to_name_in_pattern_6(self): + self.assert_syntax_error(""" + match ...: + case a as a + 1: # NAME and expression with no () + pass + """) + def test_multiple_starred_names_in_sequence_pattern_0(self): self.assert_syntax_error(""" match ...: @@ -3369,6 +3385,24 @@ class Keys: self.assertIs(y, None) self.assertIs(z, None) +class TestSourceLocations(unittest.TestCase): + def test_jump_threading(self): + # See gh-123048 + def f(): + x = 0 + v = 1 + match v: + case 1: + if x < 0: + x = 1 + case 2: + if x < 0: + x = 1 + x += 1 + + for inst in dis.get_instructions(f): + if inst.opcode in dis.hasjump: + self.assertIsNotNone(inst.positions.lineno, "jump without location") class TestTracing(unittest.TestCase): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2b0795cdad707e..e5f9848319021a 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -10,11 +10,12 @@ import subprocess import textwrap import linecache +import zipapp from contextlib import ExitStack, redirect_stdout from io import StringIO from test import support -from test.support import os_helper +from test.support import force_not_colorized, os_helper from test.support.import_helper import import_module from test.support.pty_helper import run_pty, FakeInput from unittest.mock import patch @@ -45,7 +46,6 @@ def test_pdb_displayhook(): >>> def test_function(foo, bar): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'foo', @@ -54,8 +54,8 @@ def test_pdb_displayhook(): ... 'continue', ... ]): ... test_function(1, None) - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) foo 1 (Pdb) bar @@ -97,6 +97,7 @@ def test_pdb_basic_commands(): ... print(ret) >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to line ret = test_function_2('baz') ... 'step', # entering the function call ... 'args', # display function args ... 'list', # list function source @@ -121,6 +122,9 @@ def test_pdb_basic_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) step @@ -144,7 +148,7 @@ def test_pdb_basic_commands(): [EOF] (Pdb) bt ... - (25)() + (26)() -> test_function() (3)test_function() -> ret = test_function_2('baz') @@ -254,6 +258,8 @@ def test_pdb_breakpoint_commands(): ... 'clear 3', ... 'break', ... 'condition 1', + ... 'commands 1', + ... 'EOF', # Simulate Ctrl-D/Ctrl-Z from user, should end input ... 'enable 1', ... 'clear 1', ... 'commands 2', @@ -274,8 +280,8 @@ def test_pdb_breakpoint_commands(): ... 'continue', ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) break 4, + @@ -309,6 +315,9 @@ def test_pdb_breakpoint_commands(): 2 breakpoint keep yes at :4 (Pdb) condition 1 Breakpoint 1 is now unconditional. + (Pdb) commands 1 + (com) EOF + (Pdb) enable 1 Enabled breakpoint 1 at :3 (Pdb) clear 1 @@ -354,6 +363,130 @@ def test_pdb_breakpoint_commands(): 4 """ +def test_pdb_breakpoint_on_annotated_function_def(): + """Test breakpoints on function definitions with annotation. + + >>> def foo[T](): + ... return 0 + + >>> def bar() -> int: + ... return 0 + + >>> def foobar[T]() -> int: + ... return 0 + + >>> reset_Breakpoint() + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... pass + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'break foo', + ... 'break bar', + ... 'break foobar', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) break foo + Breakpoint 1 at :2 + (Pdb) break bar + Breakpoint 2 at :2 + (Pdb) break foobar + Breakpoint 3 at :2 + (Pdb) continue + """ + +def test_pdb_commands(): + """Test the commands command of pdb. + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... print(1) + ... print(2) + ... print(3) + + >>> reset_Breakpoint() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'b 3', + ... 'commands', + ... 'silent', # suppress the frame status output + ... 'p "hello"', + ... 'end', + ... 'b 4', + ... 'commands', + ... 'until 5', # no output, should stop at line 5 + ... 'continue', # hit breakpoint at line 3 + ... '', # repeat continue, hit breakpoint at line 4 then `until` to line 5 + ... '', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) b 3 + Breakpoint 1 at :3 + (Pdb) commands + (com) silent + (com) p "hello" + (com) end + (Pdb) b 4 + Breakpoint 2 at :4 + (Pdb) commands + (com) until 5 + (Pdb) continue + 'hello' + (Pdb) + 1 + 2 + > (5)test_function() + -> print(3) + (Pdb) + 3 + """ + +def test_pdb_breakpoint_with_filename(): + """Breakpoints with filename:lineno + + >>> def test_function(): + ... # inspect_fodder2 is a great module as the line number is stable + ... from test.test_inspect import inspect_fodder2 as mod2 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... mod2.func88() + ... mod2.func114() + ... # Be a good citizen and clean up the mess + ... reset_Breakpoint() + + First, need to clear bdb state that might be left over from previous tests. + Otherwise, the new breakpoints might get assigned different numbers. + + >>> reset_Breakpoint() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS + ... 'break test.test_inspect.inspect_fodder2:90', + ... 'continue', # will stop at func88 + ... 'break test/test_inspect/inspect_fodder2.py:115', + ... 'continue', # will stop at func114 + ... 'continue', + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) break test.test_inspect.inspect_fodder2:90 + Breakpoint 1 at ...inspect_fodder2.py:90 + (Pdb) continue + > ...inspect_fodder2.py(90)func88() + -> return 90 + (Pdb) break test/test_inspect/inspect_fodder2.py:115 + Breakpoint 2 at ...inspect_fodder2.py:115 + (Pdb) continue + > ...inspect_fodder2.py(115)func114() + -> return 115 + (Pdb) continue + """ + def test_pdb_breakpoints_preserved_across_interactive_sessions(): """Breakpoints are remembered between interactive sessions @@ -421,6 +554,43 @@ def test_pdb_breakpoints_preserved_across_interactive_sessions(): (Pdb) continue """ +def test_pdb_break_anywhere(): + """Test break_anywhere() method of Pdb. + + >>> def outer(): + ... def inner(): + ... import pdb + ... import sys + ... p = pdb.Pdb(nosigint=True, readrc=False) + ... p.set_trace() + ... frame = sys._getframe() + ... print(p.break_anywhere(frame)) # inner + ... print(p.break_anywhere(frame.f_back)) # outer + ... print(p.break_anywhere(frame.f_back.f_back)) # caller + ... inner() + + >>> def caller(): + ... outer() + + >>> def test_function(): + ... caller() + + >>> reset_Breakpoint() + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'b 3', + ... 'c', + ... ]): + ... test_function() + > (6)inner() + -> p.set_trace() + (Pdb) b 3 + Breakpoint 1 at :3 + (Pdb) c + True + False + False + """ + def test_pdb_pp_repr_exc(): """Test that do_p/do_pp do not swallow exceptions. @@ -438,8 +608,7 @@ def test_pdb_pp_repr_exc(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p obj *** Exception: repr_exc @@ -448,6 +617,37 @@ def test_pdb_pp_repr_exc(): (Pdb) continue """ +def test_pdb_empty_line(): + """Test that empty line repeats the last command. + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... y = 2 + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'p x', + ... '', # Should repeat p x + ... 'n ;; p 0 ;; p x', # Fill cmdqueue with multiple commands + ... '', # Should still repeat p x + ... 'continue', + ... ]): + ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p x + 1 + (Pdb) + 1 + (Pdb) n ;; p 0 ;; p x + 0 + 1 + > (4)test_function() + -> y = 2 + (Pdb) + 1 + (Pdb) continue + """ def do_nothing(): pass @@ -479,6 +679,7 @@ def test_list_commands(): ... ret = test_function_2('baz') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to the test function line ... 'list', # list first function ... 'step', # step into second function ... 'list', # list second function @@ -494,6 +695,9 @@ def test_list_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) list @@ -574,8 +778,7 @@ def test_pdb_whatis_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) whatis myvar @@ -604,6 +807,7 @@ def test_pdb_display_command(): ... a = 4 >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'display +', ... 'display', ... 'display a', @@ -619,6 +823,9 @@ def test_pdb_display_command(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> a = 1 (Pdb) display + @@ -667,6 +874,7 @@ def test_pdb_alias_command(): ... o.method() >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'alias pi', ... 'alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}")', ... 'alias ps pi self', @@ -685,6 +893,9 @@ def test_pdb_alias_command(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> o.method() (Pdb) alias pi @@ -727,7 +938,7 @@ def test_pdb_where_command(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() >>> def f(): - ... g(); + ... g() >>> def test_function(): ... f() @@ -735,50 +946,114 @@ def test_pdb_where_command(): >>> with PdbTestInput([ # doctest: +ELLIPSIS ... 'w', ... 'where', + ... 'w 1', + ... 'w invalid', ... 'u', ... 'w', + ... 'w 0', + ... 'w 100', + ... 'w -100', ... 'continue', ... ]): ... test_function() - --Return-- - > (2)g()->None + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) w ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); - > (2)g()->None + -> g() + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) where ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); - > (2)g()->None + -> g() + > (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 1 + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w invalid + *** Invalid count (invalid) (Pdb) u > (2)f() - -> g(); + -> g() (Pdb) w ... - (8)() + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 0 + > (2)f() + -> g() + (Pdb) w 100 + ... + (13)() -> test_function() (2)test_function() -> f() > (2)f() - -> g(); - (2)g()->None + -> g() + (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w -100 + ... + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) continue + """ + +def test_pdb_restart_command(): + """Test restart command + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False, mode='inline').set_trace() + ... x = 1 + + >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'restart', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False, mode='inline').set_trace() + (Pdb) restart + *** run/restart command is disabled when pdb is running in inline mode. + Use the command line interface to enable restarting your program + e.g. "python -m pdb myscript.py" (Pdb) continue """ +def test_pdb_commands_with_set_trace(): + """Test that commands can be passed to Pdb.set_trace() + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace(commands=['p x', 'c']) + + >>> test_function() + 1 + """ + # skip this test if sys.flags.no_site = True; # exit() isn't defined unless there's a site module. @@ -810,8 +1085,7 @@ def test_pdb_interact_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (4)test_function()->None + > (4)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) interact *pdb interact start* @@ -843,7 +1117,7 @@ def test_convenience_variables(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... try: ... raise Exception('test') - ... except: + ... except Exception: ... pass ... return 1 @@ -851,6 +1125,7 @@ def test_convenience_variables(): ... util_function() >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # Step to try statement ... '$_frame.f_lineno', # Check frame convenience variable ... '$ _frame', # This should be a syntax error ... '$a = 10', # Set a convenience variable @@ -873,6 +1148,9 @@ def test_convenience_variables(): ... 'continue', ... ]): ... test_function() + > (2)util_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)util_function() -> try: (Pdb) $_frame.f_lineno @@ -911,7 +1189,7 @@ def test_convenience_variables(): Exception('test') (Pdb) next > (5)util_function() - -> except: + -> except Exception: (Pdb) $_exception *** KeyError: '_exception' (Pdb) return @@ -1138,7 +1416,7 @@ def test_post_mortem_context_of_the_cause(): def test_post_mortem_from_none(): """Test post mortem traceback debugging of chained exception - In particular that cause from None (which sets __supress_context__ to True) + In particular that cause from None (which sets __suppress_context__ to True) does not show context. @@ -1346,6 +1624,7 @@ def test_post_mortem(): ... print('Not reached.') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # step to test_function_2() line ... 'next', # step over exception-raising call ... 'bt', # get a backtrace ... 'list', # list code of test_function() @@ -1357,6 +1636,9 @@ def test_post_mortem(): ... test_function() ... except ZeroDivisionError: ... print('Correctly reraised.') + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> test_function_2() (Pdb) next @@ -1366,7 +1648,7 @@ def test_post_mortem(): -> test_function_2() (Pdb) bt ... - (10)() + (11)() -> test_function() > (3)test_function() -> test_function_2() @@ -1393,6 +1675,58 @@ def test_post_mortem(): """ +def test_pdb_return_to_different_file(): + """When pdb returns to a different file, it should not skip if f_trace is + not already set + + >>> import pprint + + >>> class A: + ... def __repr__(self): + ... return 'A' + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... pprint.pprint(A()) + + >>> reset_Breakpoint() + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'b A.__repr__', + ... 'continue', + ... 'return', + ... 'next', + ... 'return', + ... 'return', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) b A.__repr__ + Breakpoint 1 at :3 + (Pdb) continue + > (3)__repr__() + -> return 'A' + (Pdb) return + --Return-- + > (3)__repr__()->'A' + -> return 'A' + (Pdb) next + > ...pprint.py..._safe_repr() + -> return rep,... + (Pdb) return + --Return-- + > ...pprint.py..._safe_repr()->('A'...) + -> return rep,... + (Pdb) return + --Return-- + > ...pprint.py...format()->('A'...) + -> return... + (Pdb) continue + A + """ + + def test_pdb_skip_modules(): """This illustrates the simple case of module skipping. @@ -1403,9 +1737,13 @@ def test_pdb_skip_modules(): >>> with PdbTestInput([ ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() + > (3)skip_module() + -> import pdb; pdb.Pdb(skip=['stri*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (4)skip_module() -> string.capwords('FOO') (Pdb) step @@ -1420,7 +1758,6 @@ def test_pdb_invalid_arg(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'a = 3', @@ -1429,8 +1766,8 @@ def test_pdb_invalid_arg(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) a = 3 *** Invalid argument: = 3 Usage: a(rgs) @@ -1464,10 +1801,14 @@ def test_pdb_skip_modules_with_callback(): ... 'step', ... 'step', ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() ... pass # provides something to "step" to + > (4)skip_module() + -> import pdb; pdb.Pdb(skip=['module_to_skip*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (5)skip_module() -> mod.foo_pony(callback) (Pdb) step @@ -1486,7 +1827,7 @@ def test_pdb_skip_modules_with_callback(): > (5)skip_module()->None -> mod.foo_pony(callback) (Pdb) step - > (10)() + > (11)() -> pass # provides something to "step" to (Pdb) continue """ @@ -1505,6 +1846,7 @@ def test_pdb_continue_in_bottomframe(): ... print(4) >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'step', ... 'next', ... 'break 7', ... 'continue', @@ -1513,6 +1855,9 @@ def test_pdb_continue_in_bottomframe(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> inst.set_trace() + (Pdb) step > (4)test_function() -> inst.botframe = sys._getframe() # hackery to get the right botframe (Pdb) next @@ -1604,8 +1949,8 @@ def test_next_until_return_at_return_event(): ... 'return', ... 'continue']): ... test_function() - > (3)test_function() - -> test_function_2() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_function_2 Breakpoint 1 at :2 (Pdb) continue @@ -1664,12 +2009,16 @@ def test_pdb_next_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1724,12 +2073,16 @@ def test_pdb_next_command_for_coroutine(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'next', ... 'step', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1784,12 +2137,16 @@ def test_pdb_next_command_for_asyncgen(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'step', ... 'next', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1842,11 +2199,15 @@ def test_pdb_return_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'return', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1897,9 +2258,13 @@ def test_pdb_return_command_for_coroutine(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1932,11 +2297,15 @@ def test_pdb_until_command_for_generator(): ... print("finished") >>> with PdbTestInput(['step', + ... 'step', ... 'until 4', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -1988,9 +2357,13 @@ def test_pdb_until_command_for_coroutine(): ... print("finished") >>> with PdbTestInput(['step', + ... 'step', ... 'until 8', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -2029,8 +2402,8 @@ def test_pdb_next_command_in_generator_for_loop(): ... 'next', ... 'continue']): ... test_function() - > (3)test_function() - -> for i in test_gen(): + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_gen Breakpoint 1 at :2 (Pdb) continue @@ -2069,11 +2442,15 @@ def test_pdb_next_command_subiterator(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'next', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -2102,34 +2479,105 @@ def test_pdb_multiline_statement(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'def f(x):', ... ' return x * 2', ... '', - ... 'f(2)', + ... 'val = 2', + ... 'if val > 0:', + ... ' val = f(val)', + ... '', + ... '', # empty line should repeat the multi-line statement + ... 'val', ... 'c' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) def f(x): ... return x * 2 ... - (Pdb) f(2) - 4 + (Pdb) val = 2 + (Pdb) if val > 0: + ... val = f(val) + ... + (Pdb) + (Pdb) val + 8 + (Pdb) c + """ + +def test_pdb_closure(): + """Test for all expressions/statements that involve closure + + >>> k = 0 + >>> g = 1 + >>> def test_function(): + ... x = 2 + ... g = 3 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'k', + ... 'g', + ... 'y = y', + ... 'global g; g', + ... 'global g; (lambda: g)()', + ... '(lambda: x)()', + ... '(lambda: g)()', + ... 'lst = [n for n in range(10) if (n % x) == 0]', + ... 'lst', + ... 'sum(n for n in lst if n > x)', + ... 'x = 1; raise Exception()', + ... 'x', + ... 'def f():', + ... ' return x', + ... '', + ... 'f()', + ... 'c' + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) k + 0 + (Pdb) g + 3 + (Pdb) y = y + *** NameError: name 'y' is not defined + (Pdb) global g; g + 1 + (Pdb) global g; (lambda: g)() + 1 + (Pdb) (lambda: x)() + 2 + (Pdb) (lambda: g)() + 3 + (Pdb) lst = [n for n in range(10) if (n % x) == 0] + (Pdb) lst + [0, 2, 4, 6, 8] + (Pdb) sum(n for n in lst if n > x) + 18 + (Pdb) x = 1; raise Exception() + *** Exception + (Pdb) x + 1 + (Pdb) def f(): + ... return x + ... + (Pdb) f() + 1 (Pdb) c """ def test_pdb_show_attribute_and_item(): - """Test for multiline statement + """Test for expressions with command prefix >>> def test_function(): ... n = lambda x: x ... c = {"a": 1} ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'c["a"]', @@ -2143,8 +2591,8 @@ def test_pdb_show_attribute_and_item(): ... 'c' ... ]): ... test_function() - > (5)test_function() - -> pass + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) c["a"] 1 (Pdb) c.get("a") @@ -2163,6 +2611,49 @@ def test_pdb_show_attribute_and_item(): (Pdb) c """ +# doctest will modify pdb.set_trace during the test, so we need to backup +# the original function to use it in the test +original_pdb_settrace = pdb.set_trace + +def test_pdb_with_inline_breakpoint(): + """Hard-coded breakpoint() calls should invoke the same debugger instance + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb().set_trace() + ... original_pdb_settrace() + ... x = 2 + + >>> with PdbTestInput(['display x', + ... 'n', + ... 'n', + ... 'n', + ... 'n', + ... 'undisplay', + ... 'c']): + ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb().set_trace() + (Pdb) display x + display x: 1 + (Pdb) n + > (4)test_function() + -> original_pdb_settrace() + (Pdb) n + > (4)test_function() + -> original_pdb_settrace() + (Pdb) n + > (5)test_function() + -> x = 2 + (Pdb) n + --Return-- + > (5)test_function()->None + -> x = 2 + display x: 2 [old: 1] + (Pdb) undisplay + (Pdb) c + """ + def test_pdb_issue_20766(): """Test for reference leaks when the SIGINT handler is set. @@ -2178,12 +2669,12 @@ def test_pdb_issue_20766(): >>> with PdbTestInput(['continue', ... 'continue']): ... test_function() - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 1: - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 2: """ @@ -2204,8 +2695,8 @@ def test_pdb_issue_43318(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) clear :3 @@ -2237,12 +2728,16 @@ def test_pdb_issue_gh_91742(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 5', ... 'continue' ... ]): ... test_function() + > (11)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (12)test_function() -> about() (Pdb) step @@ -2280,6 +2775,7 @@ def test_pdb_issue_gh_94215(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 3', @@ -2292,6 +2788,9 @@ def test_pdb_issue_gh_94215(): ... 'continue' ... ]): ... test_function() + > (8)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (9)test_function() -> func() (Pdb) step @@ -2331,7 +2830,7 @@ def test_pdb_issue_gh_94215(): def test_pdb_issue_gh_101673(): """See GH-101673 - Make sure ll won't revert local variable assignment + Make sure ll and switching frames won't revert local variable assignment >>> def test_function(): ... a = 1 @@ -2341,11 +2840,14 @@ def test_pdb_issue_gh_101673(): ... '!a = 2', ... 'll', ... 'p a', + ... 'u', + ... 'p a', + ... 'd', + ... 'p a', ... 'continue' ... ]): ... test_function() - --Return-- - > (3)test_function()->None + > (3)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) !a = 2 (Pdb) ll @@ -2354,6 +2856,16 @@ def test_pdb_issue_gh_101673(): 3 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p a 2 + (Pdb) u + > (11)() + -> test_function() + (Pdb) p a + *** NameError: name 'a' is not defined + (Pdb) d + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p a + 2 (Pdb) continue """ @@ -2369,16 +2881,16 @@ def test_pdb_issue_gh_103225(): ... a = 1 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... b = 2 - > (7)() - -> b = 2 + > (6)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) longlist 1 with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE 2 'longlist', 3 'continue' 4 ]): 5 a = 1 - 6 import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - 7 -> b = 2 + 6 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + 7 b = 2 (Pdb) continue """ @@ -2397,9 +2909,8 @@ def test_pdb_issue_gh_101517(): ... 'continue' ... ]): ... test_function() - --Return-- - > (None)test_function()->None - -> Warning: lineno is None + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2415,9 +2926,8 @@ def test_pdb_issue_gh_108976(): ... 'continue' ... ]): ... test_function() - bdb.Bdb.dispatch: unknown debugging event: 'opcode' - > (5)test_function() - -> a = 1 + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2436,9 +2946,8 @@ def test_pdb_issue_gh_80731(): ... raise ValueError('Correct') ... except ValueError: ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass - > (10)() - -> pass + > (9)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) import sys (Pdb) sys.exc_info() (, ValueError('Correct'), ) @@ -2452,6 +2961,7 @@ def test_pdb_ambiguous_statements(): Make sure that ambiguous statements prefixed by '!' are properly disambiguated >>> with PdbTestInput([ + ... 's', # step to the print line ... '! n = 42', # disambiguated statement: reassign the name n ... 'n', # advance the debugger into the print() ... 'continue' @@ -2460,6 +2970,9 @@ def test_pdb_ambiguous_statements(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... print(f"The value of n is {n}") > (8)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s + > (9)() -> print(f"The value of n is {n}") (Pdb) ! n = 42 (Pdb) n @@ -2488,8 +3001,8 @@ def test_pdb_f_trace_lines(): ... 'continue' ... ]): ... test_function() - > (6)test_function() - -> if frame.f_trace_lines != False: + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2512,7 +3025,6 @@ def test_pdb_function_break(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ... 'break foo', @@ -2522,8 +3034,8 @@ def test_pdb_function_break(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break foo Breakpoint ... at :1 (Pdb) break bar @@ -2553,6 +3065,7 @@ def test_pdb_issue_gh_65052(): ... A() >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ... 's', + ... 's', ... 'retval', ... 'continue', ... 'args', @@ -2561,6 +3074,9 @@ def test_pdb_issue_gh_65052(): ... 'continue', ... ]): ... test_function() + > (3)__new__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)__new__() -> return object.__new__(cls) (Pdb) s @@ -2570,8 +3086,8 @@ def test_pdb_issue_gh_65052(): (Pdb) retval *** repr(retval) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) continue - > (7)__init__() - -> self.a = 1 + > (6)__init__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) args self = *** repr(self) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) display self @@ -2603,12 +3119,12 @@ def _run_pdb(self, pdb_args, commands, cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**env, 'PYTHONIOENCODING': 'utf-8'} ) as proc: stdout, stderr = proc.communicate(str.encode(commands)) - stdout = stdout and bytes.decode(stdout) - stderr = stderr and bytes.decode(stderr) + stdout = bytes.decode(stdout) if isinstance(stdout, bytes) else stdout + stderr = bytes.decode(stderr) if isinstance(stderr, bytes) else stderr self.assertEqual( proc.returncode, expected_returncode, @@ -2618,13 +3134,32 @@ def _run_pdb(self, pdb_args, commands, def run_pdb_script(self, script, commands, expected_returncode=0, - extra_env=None): + extra_env=None, + script_args=None, + pdbrc=None, + remove_home=False): """Run 'script' lines with pdb and the pdb 'commands'.""" filename = 'main.py' with open(filename, 'w') as f: f.write(textwrap.dedent(script)) + + if pdbrc is not None: + with open('.pdbrc', 'w') as f: + f.write(textwrap.dedent(pdbrc)) + self.addCleanup(os_helper.unlink, '.pdbrc') self.addCleanup(os_helper.unlink, filename) - return self._run_pdb([filename], commands, expected_returncode, extra_env) + + homesave = None + if remove_home: + homesave = os.environ.pop('HOME', None) + try: + if script_args is None: + script_args = [] + stdout, stderr = self._run_pdb([filename] + script_args, commands, expected_returncode, extra_env) + finally: + if homesave is not None: + os.environ['HOME'] = homesave + return stdout, stderr def run_pdb_module(self, script, commands): """Runs the script code as part of a module""" @@ -2695,6 +3230,18 @@ def bœr(): ('bœr', 2), ) + def test_spec(self): + # Test that __main__.__spec__ is set to None when running a script + script = """ + import __main__ + print(__main__.__spec__) + """ + + commands = "continue" + + stdout, _ = self.run_pdb_script(script, commands) + self.assertIn('None', stdout) + def test_find_function_first_executable_line(self): code = textwrap.dedent("""\ def foo(): pass @@ -2728,7 +3275,7 @@ def test_issue7964(self): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, ) self.addCleanup(proc.stdout.close) stdout, stderr = proc.communicate(b'quit\n') @@ -2812,7 +3359,7 @@ def start_pdb(): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env={**os.environ, 'PYTHONIOENCODING': 'utf-8'} ) self.addCleanup(proc.stdout.close) @@ -2842,7 +3389,7 @@ def start_pdb(): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'} ) self.addCleanup(proc.stdout.close) @@ -2850,6 +3397,7 @@ def start_pdb(): self.assertNotIn(b'Error', stdout, "Got an error running test script under PDB") + @force_not_colorized def test_issue16180(self): # A syntax error in the debuggee. script = "def f: pass\n" @@ -2858,11 +3406,12 @@ def test_issue16180(self): stdout, stderr = self.run_pdb_script( script, commands ) - self.assertIn(expected, stdout, + self.assertIn(expected, stderr, '\n\nExpected:\n{}\nGot:\n{}\n' 'Fail to handle a syntax error in the debuggee.' - .format(expected, stdout)) + .format(expected, stderr)) + @force_not_colorized def test_issue84583(self): # A syntax error from ast.literal_eval should not make pdb exit. script = "import ast; ast.literal_eval('')\n" @@ -2872,11 +3421,12 @@ def test_issue84583(self): quit """ stdout, stderr = self.run_pdb_script(script, commands) - # The code should appear 3 times in the stdout: - # 1. when pdb starts - # 2. when the exception is raised, in trackback - # 3. in where command - self.assertEqual(stdout.count("ast.literal_eval('')"), 3) + # The code should appear 3 times in the stdout/stderr: + # 1. when pdb starts (stdout) + # 2. when the exception is raised, in trackback (stderr) + # 3. in where command (stdout) + self.assertEqual(stdout.count("ast.literal_eval('')"), 2) + self.assertEqual(stderr.count("ast.literal_eval('')"), 1) def test_issue26053(self): # run command of pdb prompt echoes the correct args @@ -2892,37 +3442,144 @@ def test_issue26053(self): self.assertRegex(res, "Restarting .* with arguments:\na b c") self.assertRegex(res, "Restarting .* with arguments:\nd e f") - def test_readrc_kwarg(self): + def test_issue58956(self): + # Set a breakpoint in a function that already exists on the call stack + # should enable the trace function for the frame. + script = """ + import bar + def foo(): + ret = bar.bar() + pass + foo() + """ + commands = """ + b bar.bar + c + b main.py:5 + c + p ret + quit + """ + bar = """ + def bar(): + return 42 + """ + with open('bar.py', 'w') as f: + f.write(textwrap.dedent(bar)) + self.addCleanup(os_helper.unlink, 'bar.py') + stdout, stderr = self.run_pdb_script(script, commands) + lines = stdout.splitlines() + self.assertIn('-> pass', lines) + self.assertIn('(Pdb) 42', lines) + + def test_step_into_botframe(self): + # gh-125422 + # pdb should not be able to step into the botframe (bdb.py) + script = "x = 1" + commands = """ + step + step + step + quit + """ + stdout, _ = self.run_pdb_script(script, commands) + self.assertIn("The program finished", stdout) + self.assertNotIn("bdb.py", stdout) + + def test_pdbrc_basic(self): script = textwrap.dedent(""" - import pdb; pdb.Pdb(readrc=False).set_trace() + a = 1 + b = 2 + """) - print('hello') + pdbrc = textwrap.dedent(""" + # Comments should be fine + n + p f"{a+8=}" """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w') as f: - f.write("invalid\n") - - with open('main.py', 'w') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - with proc: - stdout, stderr = proc.communicate(b'q\n') - self.assertNotIn(b"NameError: name 'invalid' is not defined", - stdout) + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertNotIn("SyntaxError", stdout) + self.assertIn("a+8=9", stdout) + self.assertIn("-> b = 2", stdout) - finally: - if save_home is not None: - os.environ['HOME'] = save_home + def test_pdbrc_empty_line(self): + """Test that empty lines in .pdbrc are ignored.""" + + script = textwrap.dedent(""" + a = 1 + b = 2 + c = 3 + """) + + pdbrc = textwrap.dedent(""" + n + + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("b = 2", stdout) + self.assertNotIn("c = 3", stdout) + + def test_pdbrc_alias(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}") + until 6 + pi a + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("a.attr = 1", stdout) + + def test_pdbrc_semicolon(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 5;;c;;n + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("-> b = 2", stdout) + + def test_pdbrc_commands(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 6 + commands 1 ;; p a;; end + c + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("<__main__.A object at", stdout) + + def test_readrc_kwarg(self): + script = textwrap.dedent(""" + print('hello') + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc='invalid', remove_home=True) + self.assertIn("NameError: name 'invalid' is not defined", stdout) def test_readrc_homedir(self): save_home = os.environ.pop("HOME", None) @@ -2937,46 +3594,17 @@ def test_readrc_homedir(self): if save_home is not None: os.environ["HOME"] = save_home - def test_read_pdbrc_with_ascii_encoding(self): - script = textwrap.dedent(""" - import pdb; pdb.Pdb().set_trace() - print('hello') - """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w', encoding='utf-8') as f: - f.write("Fran\u00E7ais") - - with open('main.py', 'w', encoding='utf-8') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - env = {'PYTHONIOENCODING': 'ascii'} - if sys.platform == 'win32': - env['PYTHONLEGACYWINDOWSSTDIO'] = 'non-empty-string' - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ, **env} - ) - with proc: - stdout, stderr = proc.communicate(b'c\n') - self.assertIn(b"UnicodeEncodeError: \'ascii\' codec can\'t encode character " - b"\'\\xe7\' in position 21: ordinal not in range(128)", stderr) - - finally: - if save_home is not None: - os.environ['HOME'] = save_home - def test_header(self): stdout = StringIO() header = 'Nobody expects... blah, blah, blah' with ExitStack() as resources: resources.enter_context(patch('sys.stdout', stdout)) + # patch pdb.Pdb.set_trace() to avoid entering the debugger resources.enter_context(patch.object(pdb.Pdb, 'set_trace')) + # We need to manually clear pdb.Pdb._last_pdb_instance so a + # new instance with stdout redirected could be created when + # pdb.set_trace() is called. + pdb.Pdb._last_pdb_instance = None pdb.set_trace(header=header) self.assertEqual(stdout.getvalue(), header + '\n') @@ -3001,6 +3629,31 @@ def test_module_is_run_as_main(self): stdout, stderr = self.run_pdb_module(script, commands) self.assertTrue(any("SUCCESS" in l for l in stdout.splitlines()), stdout) + def test_run_module_with_args(self): + commands = """ + continue + """ + self._run_pdb(["calendar", "-m"], commands, expected_returncode=2) + + stdout, _ = self._run_pdb(["-m", "calendar", "1"], commands) + self.assertIn("December", stdout) + + stdout, _ = self._run_pdb(["-m", "calendar", "--type", "text"], commands) + self.assertIn("December", stdout) + + def test_run_script_with_args(self): + script = """ + import sys + print(sys.argv[1:]) + """ + commands = """ + continue + quit + """ + + stdout, stderr = self.run_pdb_script(script, commands, script_args=["--bar", "foo"]) + self.assertIn("['--bar', 'foo']", stdout) + def test_breakpoint(self): script = """ if __name__ == '__main__': @@ -3068,9 +3721,9 @@ def test_dir_as_script(self): def test_invalid_cmd_line_options(self): stdout, stderr = self._run_pdb(["-c"], "", expected_returncode=2) - self.assertIn(f"pdb: error: argument -c/--command: expected one argument", stdout.split('\n')[1]) + self.assertIn(f"pdb: error: argument -c/--command: expected one argument", stderr.split('\n')[1]) stdout, stderr = self._run_pdb(["--spam", "-m", "pdb"], "", expected_returncode=2) - self.assertIn(f"pdb: error: unrecognized arguments: --spam", stdout.split('\n')[1]) + self.assertIn(f"pdb: error: unrecognized arguments: --spam", stderr.split('\n')[1]) def test_blocks_at_first_code_line(self): script = """ @@ -3090,10 +3743,12 @@ def test_file_modified_after_execution(self): print("hello") """ + # the time.sleep is needed for low-resolution filesystems like HFS+ commands = """ filename = $_frame.f_code.co_filename f = open(filename, "w") f.write("print('goodbye')") + import time; time.sleep(1) f.close() ll """ @@ -3102,11 +3757,32 @@ def test_file_modified_after_execution(self): self.assertIn("WARNING:", stdout) self.assertIn("was edited", stdout) + def test_file_modified_and_immediately_restarted(self): + script = """ + print("hello") + """ + + # the time.sleep is needed for low-resolution filesystems like HFS+ + commands = """ + filename = $_frame.f_code.co_filename + f = open(filename, "w") + f.write("print('goodbye')") + import time; time.sleep(1) + f.close() + restart + """ + + stdout, stderr = self.run_pdb_script(script, commands) + self.assertNotIn("WARNING:", stdout) + self.assertNotIn("was edited", stdout) + def test_file_modified_after_execution_with_multiple_instances(self): + # the time.sleep is needed for low-resolution filesystems like HFS+ script = """ import pdb; pdb.Pdb().set_trace() with open(__file__, "w") as f: f.write("print('goodbye')\\n" * 5) + import time; time.sleep(1) import pdb; pdb.Pdb().set_trace() """ @@ -3125,7 +3801,7 @@ def test_file_modified_after_execution_with_multiple_instances(self): cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}, ) as proc: stdout, _ = proc.communicate(str.encode(commands)) @@ -3166,6 +3842,23 @@ def change_file(content, filename): # the file as up to date self.assertNotIn("WARNING:", stdout) + def test_post_mortem_restart(self): + script = """ + def foo(): + raise ValueError("foo") + foo() + """ + + commands = """ + continue + restart + continue + quit + """ + + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("Restarting", stdout) + def test_relative_imports(self): self.module_name = 't_main' os_helper.rmtree(self.module_name) @@ -3420,12 +4113,46 @@ def _create_fake_frozen_module(): # verify that pdb found the source of the "frozen" function self.assertIn('x = "Sentinel string for gh-93696"', stdout, "Sentinel statement not found") + def test_empty_file(self): + script = '' + commands = 'q\n' + # We check that pdb stopped at line 0, but anything reasonable + # is acceptable here, as long as it does not halt + stdout, _ = self.run_pdb_script(script, commands) + self.assertIn('main.py(0)', stdout) + stdout, _ = self.run_pdb_module(script, commands) + self.assertIn('__main__.py(0)', stdout) + def test_non_utf8_encoding(self): script_dir = os.path.join(os.path.dirname(__file__), 'encoded_modules') for filename in os.listdir(script_dir): if filename.endswith(".py"): self._run_pdb([os.path.join(script_dir, filename)], 'q') + def test_zipapp(self): + with os_helper.temp_dir() as temp_dir: + os.mkdir(os.path.join(temp_dir, 'source')) + script = textwrap.dedent( + """ + def f(x): + return x + 1 + f(21 + 21) + """ + ) + with open(os.path.join(temp_dir, 'source', '__main__.py'), 'w') as f: + f.write(script) + zipapp.create_archive(os.path.join(temp_dir, 'source'), + os.path.join(temp_dir, 'zipapp.pyz')) + stdout, _ = self._run_pdb([os.path.join(temp_dir, 'zipapp.pyz')], '\n'.join([ + 'b f', + 'c', + 'p x', + 'q' + ])) + self.assertIn('42', stdout) + self.assertIn('return x + 1', stdout) + + class ChecklineTests(unittest.TestCase): def setUp(self): linecache.clearcache() # Pdb.checkline() uses linecache.getline() @@ -3511,6 +4238,57 @@ def test_expression_completion(self): self.assertIn(b'species', output) self.assertIn(b'$_frame', output) + def test_builtin_completion(self): + script = textwrap.dedent(""" + value = "speci" + import pdb; pdb.Pdb().set_trace() + """) + + # Complete: print(value + 'al') + input = b"pri\tval\t + 'al')\n" + + # Continue + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'special', output) + + def test_local_namespace(self): + script = textwrap.dedent(""" + def f(): + original = "I live Pythin" + import pdb; pdb.Pdb().set_trace() + f() + """) + + # Complete: original.replace('i', 'o') + input = b"orig\t.repl\t('i', 'o')\n" + + # Continue + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'I love Python', output) + + def test_multiline_completion(self): + script = textwrap.dedent(""" + import pdb; pdb.Pdb().set_trace() + """) + + input = b"def func():\n" + # Complete: \treturn 40 + 2 + input += b"\tret\t 40 + 2\n" + input += b"\n" + # Complete: func() + input += b"fun\t()\n" + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'42', output) + def load_tests(loader, tests, pattern): from test import test_pdb diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 2ea186c85c8823..c7da151dce3b37 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -114,7 +114,7 @@ def f(): return None self.assertNotInBytecode(f, 'LOAD_GLOBAL') - self.assertInBytecode(f, 'RETURN_CONST', None) + self.assertInBytecode(f, 'LOAD_CONST', None) self.check_lnotab(f) def test_while_one(self): @@ -131,7 +131,7 @@ def f(): def test_pack_unpack(self): for line, elem in ( - ('a, = a,', 'RETURN_CONST',), + ('a, = a,', 'LOAD_CONST',), ('a, b = a, b', 'SWAP',), ('a, b, c = a, b, c', 'SWAP',), ): @@ -162,7 +162,7 @@ def test_folding_of_tuples_of_constants(self): # One LOAD_CONST for the tuple, one for the None return value load_consts = [instr for instr in dis.get_instructions(code) if instr.opname == 'LOAD_CONST'] - self.assertEqual(len(load_consts), 1) + self.assertEqual(len(load_consts), 2) self.check_lnotab(code) # Bug 1053819: Tuple of constants misidentified when presented with: @@ -248,14 +248,17 @@ def test_folding_of_binops_on_constants(self): ): with self.subTest(line=line): code = compile(line, '', 'single') - self.assertInBytecode(code, 'LOAD_CONST', elem) + if isinstance(elem, int): + self.assertInBytecode(code, 'LOAD_SMALL_INT', elem) + else: + self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('BINARY_')) self.check_lnotab(code) # Verify that unfoldables are skipped code = compile('a=2+"b"', '', 'single') - self.assertInBytecode(code, 'LOAD_CONST', 2) + self.assertInBytecode(code, 'LOAD_SMALL_INT', 2) self.assertInBytecode(code, 'LOAD_CONST', 'b') self.check_lnotab(code) @@ -307,7 +310,10 @@ def test_folding_of_unaryops_on_constants(self): ): with self.subTest(line=line): code = compile(line, '', 'single') - self.assertInBytecode(code, 'LOAD_CONST', elem) + if isinstance(elem, int): + self.assertInBytecode(code, 'LOAD_SMALL_INT', elem) + else: + self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('UNARY_')) self.check_lnotab(code) @@ -766,7 +772,7 @@ def test_load_fast_unknown_after_error_2(self): def f(): try: 1 / 0 - except: + except ZeroDivisionError: print(a, b, c, d, e, f, g) a = b = c = d = e = f = g = 1 self.assertInBytecode(f, 'LOAD_FAST_CHECK') @@ -932,23 +938,6 @@ def f(): self.assertNotInBytecode(f, "LOAD_FAST_CHECK") return f - def test_deleting_local_warns_and_assigns_none(self): - f = self.make_function_with_no_checks() - co_code = f.__code__.co_code - def trace(frame, event, arg): - if event == 'line' and frame.f_lineno == 4: - del frame.f_locals["x"] - sys.settrace(None) - return None - return trace - e = r"assigning None to unbound local 'x'" - with self.assertWarnsRegex(RuntimeWarning, e): - sys.settrace(trace) - f() - self.assertInBytecode(f, "LOAD_FAST") - self.assertNotInBytecode(f, "LOAD_FAST_CHECK") - self.assertEqual(f.__code__.co_code, co_code) - def test_modifying_local_does_not_add_check(self): f = self.make_function_with_no_checks() def trace(frame, event, arg): @@ -981,10 +970,15 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): def cfg_optimization_test(self, insts, expected_insts, consts=None, expected_consts=None, nlocals=0): + + self.check_instructions(insts) + self.check_instructions(expected_insts) + if expected_consts is None: expected_consts = consts - opt_insts, opt_consts = self.get_optimized(insts, consts, nlocals) - expected_insts = self.normalize_insts(expected_insts) + seq = self.seq_from_insts(insts) + opt_insts, opt_consts = self.get_optimized(seq, consts, nlocals) + expected_insts = self.seq_from_insts(expected_insts).get_instructions() self.assertInstructionsMatch(opt_insts, expected_insts) self.assertEqual(opt_consts, expected_consts) @@ -993,17 +987,19 @@ def test_conditional_jump_forward_non_const_condition(self): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), - ('RETURN_CONST', 1, 13), + ('LOAD_CONST', 1, 13), + ('RETURN_VALUE', None, 13), lbl, - ('RETURN_CONST', 2, 14), + ('LOAD_CONST', 2, 14), + ('RETURN_VALUE', None, 14), ] self.cfg_optimization_test(insts, expected_insts, @@ -1020,12 +1016,13 @@ def test_conditional_jump_forward_const_condition(self): ('LOAD_CONST', 2, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ - ('NOP', 11), - ('NOP', 12), - ('RETURN_CONST', 1, 14), + ('NOP', None, 11), + ('NOP', None, 12), + ('LOAD_CONST', 1, 14), + ('RETURN_VALUE', None, 14), ] self.cfg_optimization_test(insts, expected_insts, @@ -1038,14 +1035,14 @@ def test_conditional_jump_backward_non_const_condition(self): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected = [ lbl := self.Label(), ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) @@ -1056,15 +1053,35 @@ def test_conditional_jump_backward_const_condition(self): ('LOAD_CONST', 3, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected_insts = [ lbl := self.Label(), - ('NOP', 11), + ('NOP', None, 11), ('JUMP', lbl, 12), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + def test_except_handler_label(self): + insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('POP_BLOCK', None, -1), + ('LOAD_CONST', 1, 11), + ('RETURN_VALUE', None, 11), + handler, + ('LOAD_CONST', 2, 12), + ('RETURN_VALUE', None, 12), + ] + expected_insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('LOAD_CONST', 1, 11), + ('RETURN_VALUE', None, 11), + handler, + ('LOAD_CONST', 2, 12), + ('RETURN_VALUE', None, 12), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + def test_no_unsafe_static_swap(self): # We can't change order of two stores to the same location insts = [ @@ -1074,17 +1091,18 @@ def test_no_unsafe_static_swap(self): ('SWAP', 3, 4), ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), + ('POP_TOP', None, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), + ('NOP', None, 3), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), - ('RETURN_CONST', 0) + ('POP_TOP', None, 4), + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) @@ -1097,15 +1115,16 @@ def test_dead_store_elimination_in_same_lineno(self): ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), - ('POP_TOP', 0, 4), + ('NOP', None, 3), + ('POP_TOP', None, 4), ('STORE_FAST', 1, 4), - ('RETURN_CONST', 0, 5) + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) @@ -1118,7 +1137,7 @@ def test_no_dead_store_elimination_in_different_lineno(self): ('STORE_FAST', 1, 5), ('STORE_FAST', 1, 6), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), @@ -1127,7 +1146,8 @@ def test_no_dead_store_elimination_in_different_lineno(self): ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 5), ('STORE_FAST', 1, 6), - ('RETURN_CONST', 0, 5) + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) @@ -1153,7 +1173,7 @@ def get_insts(lno1, lno2, op1, op2): op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' expected_insts = [ ('LOAD_NAME', 0, 10), - ('NOP', 0, 4), + ('NOP', None, 4), (op, 0, 5), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 9e273e99e387a4..1411e55dd0f293 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -13,9 +13,7 @@ from test.support import os_helper, import_helper from test.support.script_helper import assert_python_ok -_py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST") -_pgo_flag = sysconfig.get_config_var("PGO_PROF_USE_FLAG") -if _pgo_flag and _py_cflags_nodist and _pgo_flag in _py_cflags_nodist: +if support.check_cflags_pgo(): raise unittest.SkipTest("peg_generator test disabled under PGO build") test_tools.skip_if_missing("peg_generator") diff --git a/Lib/test/test_peg_generator/test_grammar_validator.py b/Lib/test/test_peg_generator/test_grammar_validator.py index 72c3d2054fa8f9..c7f20e1de802ce 100644 --- a/Lib/test/test_peg_generator/test_grammar_validator.py +++ b/Lib/test/test_peg_generator/test_grammar_validator.py @@ -4,7 +4,7 @@ test_tools.skip_if_missing("peg_generator") with test_tools.imports_under_tool("peg_generator"): from pegen.grammar_parser import GeneratedParser as GrammarParser - from pegen.validator import SubRuleValidator, ValidationError + from pegen.validator import SubRuleValidator, ValidationError, RaiseRuleValidator from pegen.testutil import parse_string from pegen.grammar import Grammar @@ -49,3 +49,13 @@ def test_rule_with_collision_after_some_other_rules(self) -> None: with self.assertRaises(ValidationError): for rule_name, rule in grammar.rules.items(): validator.validate_rule(rule_name, rule) + + def test_raising_valid_rule(self) -> None: + grammar_source = """ + start: NAME { RAISE_SYNTAX_ERROR("this is not allowed") } + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = RaiseRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 86db767b99a228..d8606521345390 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -484,7 +484,7 @@ def test_left_recursive(self) -> None: def test_python_expr(self) -> None: grammar = """ - start: expr NEWLINE? $ { ast.Expression(expr, lineno=1, col_offset=0) } + start: expr NEWLINE? $ { ast.Expression(expr) } expr: ( expr '+' term { ast.BinOp(expr, ast.Add(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } | expr '-' term { ast.BinOp(expr, ast.Sub(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } | term { term } @@ -506,6 +506,14 @@ def test_python_expr(self) -> None: val = eval(code) self.assertEqual(val, 3.0) + def test_f_string_in_action(self) -> None: + grammar = """ + start: n=NAME NEWLINE? $ { f"name -> {n.string}" } + """ + parser_class = make_parser(grammar) + node = parse_string("a", parser_class) + self.assertEqual(node.strip(), "name -> a") + def test_nullable(self) -> None: grammar_source = """ start: sign NUMBER @@ -893,7 +901,7 @@ def test_unreachable_implicit3(self) -> None: def test_locations_in_alt_action_and_group(self) -> None: grammar = """ - start: t=term NEWLINE? $ { ast.Expression(t, LOCATIONS) } + start: t=term NEWLINE? $ { ast.Expression(t) } term: | l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, LOCATIONS) } | l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, LOCATIONS) } diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 040be63da11447..1e74990878007a 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -62,11 +62,13 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -76,14 +78,27 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - perf_lines = perf_file_contents.splitlines(); - expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + perf_lines = perf_file_contents.splitlines() + expected_symbols = [ + f"py::foo:{script}", + f"py::bar:{script}", + f"py::baz:{script}", + ] for expected_symbol in expected_symbols: - perf_line = next((line for line in perf_lines if expected_symbol in line), None) - self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_line = next( + (line for line in perf_lines if expected_symbol in line), None + ) + self.assertIsNotNone( + perf_line, f"Could not find {expected_symbol} in perf file" + ) perf_addr = perf_line.split(" ")[0] - self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") - self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") + self.assertFalse( + perf_addr.startswith("0x"), "Address should not be prefixed with 0x" + ) + self.assertTrue( + set(perf_addr).issubset(string.hexdigits), + "Address should contain only hex characters", + ) def test_trampoline_works_with_forks(self): code = """if 1: @@ -117,11 +132,13 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -166,11 +183,13 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -191,14 +210,14 @@ def test_sys_api_with_existing_trampoline(self): sys.activate_stack_trampoline("perf") sys.activate_stack_trampoline("perf") """ - assert_python_ok("-c", code) + assert_python_ok("-c", code, PYTHON_JIT="0") def test_sys_api_with_invalid_trampoline(self): code = """if 1: import sys sys.activate_stack_trampoline("invalid") """ - rc, out, err = assert_python_failure("-c", code) + rc, out, err = assert_python_failure("-c", code, PYTHON_JIT="0") self.assertIn("invalid backend: invalid", err.decode()) def test_sys_api_get_status(self): @@ -209,10 +228,10 @@ def test_sys_api_get_status(self): sys.deactivate_stack_trampoline() assert sys.is_stack_trampoline_active() is False """ - assert_python_ok("-c", code) + assert_python_ok("-c", code, PYTHON_JIT="0") -def is_unwinding_reliable(): +def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") if not cflags: return False @@ -247,8 +266,9 @@ def perf_command_works(): "-c", 'print("hello")', ) + env = {**os.environ, "PYTHON_JIT": "0"} stdout = subprocess.check_output( - cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT + cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env ) except (subprocess.SubprocessError, OSError): return False @@ -259,14 +279,26 @@ def perf_command_works(): return True -def run_perf(cwd, *args, **env_vars): +def run_perf(cwd, *args, use_jit=False, **env_vars): + env = os.environ.copy() if env_vars: - env = os.environ.copy() env.update(env_vars) - else: - env = None + env["PYTHON_JIT"] = "0" output_file = cwd + "/perf_output.perf" - base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + if not use_jit: + base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + else: + base_cmd = ( + "perf", + "record", + "-g", + "--call-graph=dwarf,65528", + "-F99", + "-k1", + "-o", + output_file, + "--", + ) proc = subprocess.run( base_cmd + args, stdout=subprocess.PIPE, @@ -274,9 +306,21 @@ def run_perf(cwd, *args, **env_vars): env=env, ) if proc.returncode: - print(proc.stderr) + print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") + if use_jit: + jit_output_file = cwd + "/jit_output.dump" + command = ("perf", "inject", "-j", "-i", output_file, "-o", jit_output_file) + proc = subprocess.run( + command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env + ) + if proc.returncode: + print(proc.stderr) + raise ValueError(f"Perf failed with return code {proc.returncode}") + # Copy the jit_output_file to the output_file + os.rename(jit_output_file, output_file) + base_cmd = ("perf", "script") proc = subprocess.run( ("perf", "script", "-i", output_file), @@ -290,20 +334,9 @@ def run_perf(cwd, *args, **env_vars): ) -@unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(is_unwinding_reliable(), "Unwinding is unreliable") -class TestPerfProfiler(unittest.TestCase): - def setUp(self): - super().setUp() - self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) - - def tearDown(self) -> None: - super().tearDown() - files_to_delete = ( - set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files - ) - for file in files_to_delete: - file.unlink() +class TestPerfProfilerMixin: + def run_perf(self, script_dir, perf_mode, script): + raise NotImplementedError() def test_python_calls_appear_in_the_stack_if_perf_activated(self): with temp_dir() as script_dir: @@ -322,14 +355,14 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, "-Xperf", script) + stdout, stderr = self.run_perf(script_dir, script) self.assertEqual(stderr, "") self.assertIn(f"py::foo:{script}", stdout) self.assertIn(f"py::bar:{script}", stdout) self.assertIn(f"py::baz:{script}", stdout) - def test_python_calls_do_not_appear_in_the_stack_if_perf_activated(self): + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): with temp_dir() as script_dir: code = """if 1: def foo(n): @@ -346,13 +379,39 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, script) + stdout, stderr = self.run_perf( + script_dir, script, activate_trampoline=False + ) self.assertEqual(stderr, "") self.assertNotIn(f"py::foo:{script}", stdout) self.assertNotIn(f"py::bar:{script}", stdout) self.assertNotIn(f"py::baz:{script}", stdout) + +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless( + is_unwinding_reliable_with_frame_pointers(), + "Unwinding is unreliable with frame pointers", +) +class TestPerfProfiler(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf(script_dir, sys.executable, "-Xperf", script) + return run_perf(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + def test_pre_fork_compile(self): code = """if 1: import sys @@ -370,7 +429,7 @@ def bar_fork(): foo_fork() def foo(): - pass + import time; time.sleep(1) def bar(): foo() @@ -393,11 +452,13 @@ def compile_trampolines_for_all_functions(): with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], universal_newlines=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -423,12 +484,49 @@ def compile_trampolines_for_all_functions(): # identical in both the parent and child perf-map files. perf_file_lines = perf_file_contents.split("\n") for line in perf_file_lines: - if ( - f"py::foo_fork:{script}" in line - or f"py::bar_fork:{script}" in line - ): + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: self.assertIn(line, child_perf_file_contents) +def _is_perf_version_at_least(major, minor): + # The output of perf --version looks like "perf version 6.7-3" but + # it can also be perf version "perf version 5.15.143" + try: + output = subprocess.check_output(["perf", "--version"], text=True) + except (subprocess.CalledProcessError, FileNotFoundError): + return False + version = output.split()[2] + version = version.split("-")[0] + version = version.split(".") + version = tuple(map(int, version)) + return version >= (major, minor) + + +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless( + _is_perf_version_at_least(6, 6), "perf command may not work due to a perf bug" +) +class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf( + script_dir, sys.executable, "-Xperf_jit", script, use_jit=True + ) + return run_perf(script_dir, sys.executable, script, use_jit=True) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("jit*.dump")) + self.perf_files |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = set(pathlib.Path("/tmp/").glob("jit*.dump")) + files_to_delete |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + files_to_delete = files_to_delete - self.perf_files + for file in files_to_delete: + file.unlink() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index a17adb89f55360..d4c6fe0124af18 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -2,7 +2,11 @@ import sys import unittest -from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry +try: + from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry +except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") + if sys.platform != 'linux': raise unittest.SkipTest('Linux only') diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 5e187e5189d117..c84e507cdf645f 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -16,6 +16,7 @@ from test.pickletester import AbstractHookTests from test.pickletester import AbstractUnpickleTests +from test.pickletester import AbstractPicklingErrorTests from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests from test.pickletester import AbstractPersistentPicklerTests @@ -55,6 +56,18 @@ def loads(self, buf, **kwds): return u.load() +class PyPicklingErrorTests(AbstractPicklingErrorTests, unittest.TestCase): + + pickler = pickle._Pickler + + def dumps(self, arg, proto=None, **kwargs): + f = io.BytesIO() + p = self.pickler(f, proto, **kwargs) + p.dump(arg) + f.seek(0) + return bytes(f.read()) + + class PyPicklerTests(AbstractPickleTests, unittest.TestCase): pickler = pickle._Pickler @@ -88,6 +101,8 @@ def loads(self, buf, **kwds): return pickle.loads(buf, **kwds) test_framed_write_sizes_with_delayed_writer = None + test_find_class = None + test_custom_find_class = None class PersistentPicklerUnpicklerMixin(object): @@ -267,6 +282,9 @@ class CUnpicklerTests(PyUnpicklerTests): bad_stack_errors = (pickle.UnpicklingError,) truncated_errors = (pickle.UnpicklingError,) + class CPicklingErrorTests(PyPicklingErrorTests): + pickler = _pickle.Pickler + class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler @@ -564,11 +582,12 @@ def test_exceptions(self): if exc in (BlockingIOError, ResourceWarning, StopAsyncIteration, + PythonFinalizationError, RecursionError, EncodingWarning, BaseExceptionGroup, ExceptionGroup, - IncompleteInputError): + _IncompleteInputError): continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index d37af79e878a2e..d8ff7a25cbc4b7 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -1,3 +1,4 @@ +import io import pickle import pickletools from test import support @@ -62,6 +63,305 @@ def test_optimize_binput_and_memoize(self): self.assertNotIn(pickle.BINPUT, pickled2) +class SimpleReader: + def __init__(self, data): + self.data = data + self.pos = 0 + + def read(self, n): + data = self.data[self.pos: self.pos + n] + self.pos += n + return data + + def readline(self): + nl = self.data.find(b'\n', self.pos) + 1 + if not nl: + nl = len(self.data) + data = self.data[self.pos: nl] + self.pos = nl + return data + + +class GenopsTests(unittest.TestCase): + def test_genops(self): + it = pickletools.genops(b'(I123\nK\x12J\x12\x34\x56\x78t.') + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 0), + ('INT', 123, 1), + ('BININT1', 0x12, 6), + ('BININT', 0x78563412, 8), + ('TUPLE', None, 13), + ('STOP', None, 14), + ]) + + def test_from_file(self): + f = io.BytesIO(b'prefix(I123\nK\x12J\x12\x34\x56\x78t.suffix') + self.assertEqual(f.read(6), b'prefix') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 6), + ('INT', 123, 7), + ('BININT1', 0x12, 12), + ('BININT', 0x78563412, 14), + ('TUPLE', None, 19), + ('STOP', None, 20), + ]) + self.assertEqual(f.read(), b'suffix') + + def test_without_pos(self): + f = SimpleReader(b'(I123\nK\x12J\x12\x34\x56\x78t.') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, None), + ('INT', 123, None), + ('BININT1', 0x12, None), + ('BININT', 0x78563412, None), + ('TUPLE', None, None), + ('STOP', None, None), + ]) + + def test_no_stop(self): + it = pickletools.genops(b'N') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + 'pickle exhausted before seeing STOP'): + next(it) + + def test_truncated_data(self): + it = pickletools.genops(b'I123') + with self.assertRaisesRegex(ValueError, + 'no newline found when trying to read stringnl'): + next(it) + it = pickletools.genops(b'J\x12\x34') + with self.assertRaisesRegex(ValueError, + 'not enough data in stream to read int4'): + next(it) + + def test_unknown_opcode(self): + it = pickletools.genops(b'N\xff') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position 1, opcode b'\\xff' unknown"): + next(it) + + def test_unknown_opcode_without_pos(self): + f = SimpleReader(b'N\xff') + it = pickletools.genops(f) + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position , opcode b'\\xff' unknown"): + next(it) + + +class DisTests(unittest.TestCase): + maxDiff = None + + def check_dis(self, data, expected, **kwargs): + out = io.StringIO() + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def check_dis_error(self, data, expected, expected_error, **kwargs): + out = io.StringIO() + with self.assertRaisesRegex(ValueError, expected_error): + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def test_mark(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_indentlevel(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''', indentlevel=2) + + def test_mark_without_pos(self): + self.check_dis(SimpleReader(b'(N(tl.'), '''\ +( MARK +N NONE +( MARK +t TUPLE (MARK at unknown opcode offset) +l LIST (MARK at unknown opcode offset) +. STOP +highest protocol among opcodes = 0 +''') + + def test_no_mark(self): + self.check_dis_error(b'Nt.', '''\ + 0: N NONE + 1: t TUPLE +''', 'no MARK exists on stack') + + def test_put(self): + self.check_dis(b'Np0\nq\x01r\x02\x00\x00\x00\x94.', '''\ + 0: N NONE + 1: p PUT 0 + 4: q BINPUT 1 + 6: r LONG_BINPUT 2 + 11: \\x94 MEMOIZE (as 3) + 12: . STOP +highest protocol among opcodes = 4 +''') + + def test_put_redefined(self): + self.check_dis(b'Np1\np1\nq\x01r\x01\x00\x00\x00\x94.', '''\ + 0: N NONE + 1: p PUT 1 + 4: p PUT 1 + 7: q BINPUT 1 + 9: r LONG_BINPUT 1 + 14: \\x94 MEMOIZE (as 1) + 15: . STOP +highest protocol among opcodes = 4 +''') + + def test_put_empty_stack(self): + self.check_dis_error(b'p0\n', '''\ + 0: p PUT 0 +''', "stack is empty -- can't store into memo") + + def test_put_markobject(self): + self.check_dis_error(b'(p0\n', '''\ + 0: ( MARK + 1: p PUT 0 +''', "can't store markobject in the memo") + + def test_get(self): + self.check_dis(b'(Np1\ng1\nh\x01j\x01\x00\x00\x00t.', '''\ + 0: ( MARK + 1: N NONE + 2: p PUT 1 + 5: g GET 1 + 8: h BINGET 1 + 10: j LONG_BINGET 1 + 15: t TUPLE (MARK at 0) + 16: . STOP +highest protocol among opcodes = 1 +''') + + def test_get_without_put(self): + self.check_dis_error(b'g1\n.', '''\ + 0: g GET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'h\x01.', '''\ + 0: h BINGET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'j\x01\x00\x00\x00.', '''\ + 0: j LONG_BINGET 1 +''', 'memo key 1 has never been stored into') + + def test_memo(self): + memo = {} + self.check_dis(b'Np1\n.', '''\ + 0: N NONE + 1: p PUT 1 + 4: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + self.check_dis(b'g1\n.', '''\ + 0: g GET 1 + 3: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + + def test_mark_pop(self): + self.check_dis(b'(N00N.', '''\ + 0: ( MARK + 1: N NONE + 2: 0 POP + 3: 0 POP (MARK at 0) + 4: N NONE + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_too_small_stack(self): + self.check_dis_error(b'a', '''\ + 0: a APPEND +''', 'tries to pop 2 items from stack with only 0 items') + self.check_dis_error(b']a', '''\ + 0: ] EMPTY_LIST + 1: a APPEND +''', 'tries to pop 2 items from stack with only 1 items') + + def test_no_stop(self): + self.check_dis_error(b'N', '''\ + 0: N NONE +''', 'pickle exhausted before seeing STOP') + + def test_truncated_data(self): + self.check_dis_error(b'NI123', '''\ + 0: N NONE +''', 'no newline found when trying to read stringnl') + self.check_dis_error(b'NJ\x12\x34', '''\ + 0: N NONE +''', 'not enough data in stream to read int4') + + def test_unknown_opcode(self): + self.check_dis_error(b'N\xff', '''\ + 0: N NONE +''', r"at position 1, opcode b'\\xff' unknown") + + def test_stop_not_empty_stack(self): + self.check_dis_error(b']N.', '''\ + 0: ] EMPTY_LIST + 1: N NONE + 2: . STOP +highest protocol among opcodes = 1 +''', r'stack not empty after STOP: \[list\]') + + def test_annotate(self): + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=1) + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + self.check_dis(b'(((((((ttttttt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: ( MARK Push markobject onto the stack. + 2: ( MARK Push markobject onto the stack. + 3: ( MARK Push markobject onto the stack. + 4: ( MARK Push markobject onto the stack. + 5: ( MARK Push markobject onto the stack. + 6: ( MARK Push markobject onto the stack. + 7: t TUPLE (MARK at 6) Build a tuple out of the topmost stack slice, after markobject. + 8: t TUPLE (MARK at 5) Build a tuple out of the topmost stack slice, after markobject. + 9: t TUPLE (MARK at 4) Build a tuple out of the topmost stack slice, after markobject. + 10: t TUPLE (MARK at 3) Build a tuple out of the topmost stack slice, after markobject. + 11: t TUPLE (MARK at 2) Build a tuple out of the topmost stack slice, after markobject. + 12: t TUPLE (MARK at 1) Build a tuple out of the topmost stack slice, after markobject. + 13: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 14: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + + class MiscTestCase(unittest.TestCase): def test__all__(self): not_exported = { diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py index eed0fd1c6b73fa..a7a1c2affbe1fb 100644 --- a/Lib/test/test_pkg.py +++ b/Lib/test/test_pkg.py @@ -94,7 +94,7 @@ def mkhier(self, descr): def test_1(self): hier = [("t1", None), ("t1 __init__.py", "")] self.mkhier(hier) - import t1 + import t1 # noqa: F401 def test_2(self): hier = [ @@ -124,7 +124,7 @@ def test_2(self): from t2 import sub from t2.sub import subsub - from t2.sub.subsub import spam + from t2.sub.subsub import spam # noqa: F401 self.assertEqual(sub.__name__, "t2.sub") self.assertEqual(subsub.__name__, "t2.sub.subsub") self.assertEqual(sub.subsub.__name__, "t2.sub.subsub") diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 6fcd726345eeac..736b83711def03 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -12,6 +12,10 @@ import shutil import zipfile +from test.support.import_helper import DirsOnSysPath +from test.support.os_helper import FakePath +from test.test_importlib.util import uncache + # Note: pkgutil.walk_packages is currently tested in test_runpy. This is # a hack to get a major issue resolved for 3.3b2. Longer term, it should # be moved back here, perhaps by factoring out the helper code for @@ -118,7 +122,7 @@ def test_issue44061_iter_modules(self): # make sure iter_modules accepts Path objects names = [] - for moduleinfo in pkgutil.iter_modules([Path(zip_file)]): + for moduleinfo in pkgutil.iter_modules([FakePath(zip_file)]): self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo) names.append(moduleinfo.name) self.assertEqual(names, [pkg]) @@ -318,6 +322,38 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) + def test_name_resolution_import_rebinding(self): + # The same data is also used for testing import in test_import and + # mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + class PkgutilPEP302Tests(unittest.TestCase): @@ -486,7 +522,43 @@ def test_mixed_namespace(self): del sys.modules['foo.bar'] del sys.modules['foo.baz'] - # XXX: test .pkg files + + def test_extend_path_argument_types(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + # If the input path is not a list it is returned unchanged + self.assertEqual('notalist', pkgutil.extend_path('notalist', 'foo')) + self.assertEqual(('not', 'a', 'list'), pkgutil.extend_path(('not', 'a', 'list'), 'foo')) + self.assertEqual(123, pkgutil.extend_path(123, 'foo')) + self.assertEqual(None, pkgutil.extend_path(None, 'foo')) + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] + + + def test_extend_path_pkg_files(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + with open(os.path.join(dirname_0, 'bar.pkg'), 'w') as pkg_file: + pkg_file.write('\n'.join([ + 'baz', + '/foo/bar/baz', + '', + '#comment' + ])) + + extended_paths = pkgutil.extend_path(sys.path, 'bar') + + self.assertEqual(extended_paths[:-2], sys.path) + self.assertEqual(extended_paths[-2], 'baz') + self.assertEqual(extended_paths[-1], '/foo/bar/baz') + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] class NestedNamespacePackageTest(unittest.TestCase): @@ -535,67 +607,6 @@ class ImportlibMigrationTests(unittest.TestCase): # PEP 302 emulation in this module is in the process of being # deprecated in favour of importlib proper - @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_missing_loader_attribute(self): - global __loader__ - this_loader = __loader__ - del __loader__ - try: - self.assertIsNotNone(pkgutil.get_loader(__name__)) - finally: - __loader__ = this_loader - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_missing_spec_attribute(self): - name = 'spam' - mod = type(sys)(name) - del mod.__spec__ - with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - self.assertIsNone(loader) - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_spec_attribute_none(self): - name = 'spam' - mod = type(sys)(name) - mod.__spec__ = None - with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - self.assertIsNone(loader) - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_None_in_sys_modules(self): - name = 'totally bogus' - sys.modules[name] = None - try: - loader = pkgutil.get_loader(name) - finally: - del sys.modules[name] - self.assertIsNone(loader) - - def test_get_loader_is_deprecated(self): - with check_warnings( - (r".*\bpkgutil.get_loader\b.*", DeprecationWarning), - ): - res = pkgutil.get_loader("sys") - self.assertIsNotNone(res) - - def test_find_loader_is_deprecated(self): - with check_warnings( - (r".*\bpkgutil.find_loader\b.*", DeprecationWarning), - ): - res = pkgutil.find_loader("sys") - self.assertIsNotNone(res) - - @ignore_warnings(category=DeprecationWarning) - def test_find_loader_missing_module(self): - name = 'totally bogus' - loader = pkgutil.find_loader(name) - self.assertIsNone(loader) - def test_get_importer_avoids_emulation(self): # We use an illegal path so *none* of the path hooks should fire with check_warnings() as w: diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 648e18d0150ef0..40d5fb338ce563 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -10,6 +10,14 @@ from test import support from test.support import os_helper +try: + # Some of the iOS tests need ctypes to operate. + # Confirm that the ctypes module is available + # is available. + import _ctypes +except ImportError: + _ctypes = None + FEDORA_OS_RELEASE = """\ NAME=Fedora VERSION="32 (Thirty Two)" @@ -219,6 +227,30 @@ def test_uname(self): self.assertEqual(res[-1], res.processor) self.assertEqual(len(res), 6) + if os.name == "posix": + uname = os.uname() + self.assertEqual(res.node, uname.nodename) + self.assertEqual(res.version, uname.version) + self.assertEqual(res.machine, uname.machine) + + if sys.platform == "android": + self.assertEqual(res.system, "Android") + self.assertEqual(res.release, platform.android_ver().release) + elif sys.platform == "ios": + # Platform module needs ctypes for full operation. If ctypes + # isn't available, there's no ObjC module, and dummy values are + # returned. + if _ctypes: + self.assertIn(res.system, {"iOS", "iPadOS"}) + self.assertEqual(res.release, platform.ios_ver().release) + else: + self.assertEqual(res.system, "") + self.assertEqual(res.release, "") + else: + self.assertEqual(res.system, uname.sysname) + self.assertEqual(res.release, uname.release) + + @unittest.skipUnless(sys.platform.startswith('win'), "windows only test") def test_uname_win32_without_wmi(self): def raises_oserror(*a): @@ -318,12 +350,44 @@ def raises_oserror(*a): platform._uname_cache = None def test_java_ver(self): - res = platform.java_ver() - if sys.platform == 'java': # Is never actually checked in CI - self.assertTrue(all(res)) + import re + msg = re.escape( + "'java_ver' is deprecated and slated for removal in Python 3.15" + ) + with self.assertWarnsRegex(DeprecationWarning, msg): + res = platform.java_ver() + self.assertEqual(len(res), 4) + @unittest.skipUnless(support.MS_WINDOWS, 'This test only makes sense on Windows') def test_win32_ver(self): - res = platform.win32_ver() + release1, version1, csd1, ptype1 = 'a', 'b', 'c', 'd' + res = platform.win32_ver(release1, version1, csd1, ptype1) + self.assertEqual(len(res), 4) + release, version, csd, ptype = res + if release: + # Currently, release names always come from internal dicts, + # but this could change over time. For now, we just check that + # release is something different from what we have passed. + self.assertNotEqual(release, release1) + if version: + # It is rather hard to test explicit version without + # going deep into the details. + self.assertIn('.', version) + for v in version.split('.'): + int(v) # should not fail + if csd: + self.assertTrue(csd.startswith('SP'), msg=csd) + if ptype: + if os.cpu_count() > 1: + self.assertIn('Multiprocessor', ptype) + else: + self.assertIn('Uniprocessor', ptype) + + @unittest.skipIf(support.MS_WINDOWS, 'This test only makes sense on non Windows') + def test_win32_ver_on_non_windows(self): + release, version, csd, ptype = 'a', '1.0', 'c', 'd' + res = platform.win32_ver(release, version, csd, ptype) + self.assertSequenceEqual(res, (release, version, csd, ptype), seq_type=tuple) def test_mac_ver(self): res = platform.mac_ver() @@ -377,6 +441,56 @@ def test_mac_ver_with_fork(self): # parent support.wait_process(pid, exitcode=0) + def test_ios_ver(self): + result = platform.ios_ver() + + # ios_ver is only fully available on iOS where ctypes is available. + if sys.platform == "ios" and _ctypes: + system, release, model, is_simulator = result + # Result is a namedtuple + self.assertEqual(result.system, system) + self.assertEqual(result.release, release) + self.assertEqual(result.model, model) + self.assertEqual(result.is_simulator, is_simulator) + + # We can't assert specific values without reproducing the logic of + # ios_ver(), so we check that the values are broadly what we expect. + + # System is either iOS or iPadOS, depending on the test device + self.assertIn(system, {"iOS", "iPadOS"}) + + # Release is a numeric version specifier with at least 2 parts + parts = release.split(".") + self.assertGreaterEqual(len(parts), 2) + self.assertTrue(all(part.isdigit() for part in parts)) + + # If this is a simulator, we get a high level device descriptor + # with no identifying model number. If this is a physical device, + # we get a model descriptor like "iPhone13,1" + if is_simulator: + self.assertIn(model, {"iPhone", "iPad"}) + else: + self.assertTrue( + (model.startswith("iPhone") or model.startswith("iPad")) + and "," in model + ) + + self.assertEqual(type(is_simulator), bool) + else: + # On non-iOS platforms, calling ios_ver doesn't fail; you get + # default values + self.assertEqual(result.system, "") + self.assertEqual(result.release, "") + self.assertEqual(result.model, "") + self.assertFalse(result.is_simulator) + + # Check the fallback values can be overridden by arguments + override = platform.ios_ver("Foo", "Bar", "Whiz", True) + self.assertEqual(override.system, "Foo") + self.assertEqual(override.release, "Bar") + self.assertEqual(override.model, "Whiz") + self.assertTrue(override.is_simulator) + @unittest.skipIf(support.is_emscripten, "Does not apply to Emscripten") def test_libc_ver(self): # check that libc_ver(executable) doesn't raise an exception @@ -426,6 +540,43 @@ def test_libc_ver(self): self.assertEqual(platform.libc_ver(filename, chunksize=chunksize), ('glibc', '1.23.4')) + def test_android_ver(self): + res = platform.android_ver() + self.assertIsInstance(res, tuple) + self.assertEqual(res, (res.release, res.api_level, res.manufacturer, + res.model, res.device, res.is_emulator)) + + if sys.platform == "android": + for name in ["release", "manufacturer", "model", "device"]: + with self.subTest(name): + value = getattr(res, name) + self.assertIsInstance(value, str) + self.assertNotEqual(value, "") + + self.assertIsInstance(res.api_level, int) + self.assertGreaterEqual(res.api_level, sys.getandroidapilevel()) + + self.assertIsInstance(res.is_emulator, bool) + + # When not running on Android, it should return the default values. + else: + self.assertEqual(res.release, "") + self.assertEqual(res.api_level, 0) + self.assertEqual(res.manufacturer, "") + self.assertEqual(res.model, "") + self.assertEqual(res.device, "") + self.assertEqual(res.is_emulator, False) + + # Default values may also be overridden using parameters. + res = platform.android_ver( + "alpha", 1, "bravo", "charlie", "delta", True) + self.assertEqual(res.release, "alpha") + self.assertEqual(res.api_level, 1) + self.assertEqual(res.manufacturer, "bravo") + self.assertEqual(res.model, "charlie") + self.assertEqual(res.device, "delta") + self.assertEqual(res.is_emulator, True) + @support.cpython_only def test__comparable_version(self): from platform import _comparable_version as V diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index 001f86f2893f2f..b231b05f864ab9 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -13,7 +13,6 @@ import subprocess import binascii import collections -import time import zoneinfo from test import support from test.support import os_helper diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 1847ae95db9292..5675db8d1cab6e 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -172,7 +172,10 @@ def test_poll3(self): @cpython_only def test_poll_c_limits(self): - from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX + try: + from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX + except ImportError: + raise unittest.SkipTest("requires _testcapi") pollster = select.poll() pollster.register(1) diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index 1a193814d7535d..eea0625012da6d 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -2,6 +2,7 @@ import dis import pickle +import types import unittest from test.support import check_syntax_error @@ -440,7 +441,9 @@ def f(x: not (int is int), /): ... # without constant folding we end up with # COMPARE_OP(is), IS_OP (0) # with constant folding we should expect a IS_OP (1) - codes = [(i.opname, i.argval) for i in dis.get_instructions(g)] + code_obj = next(const for const in g.__code__.co_consts + if isinstance(const, types.CodeType) and const.co_name == "__annotate__") + codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)] self.assertNotIn(('UNARY_NOT', None), codes) self.assertIn(('IS_OP', 1), codes) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index a45f620e18dc1d..35016b83a477fc 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -411,8 +411,10 @@ def test_posix_fallocate(self): # issue33655: Also ignore EINVAL on *BSD since ZFS is also # often used there. if inst.errno == errno.EINVAL and sys.platform.startswith( - ('sunos', 'freebsd', 'netbsd', 'openbsd', 'gnukfreebsd')): + ('sunos', 'freebsd', 'openbsd', 'gnukfreebsd')): raise unittest.SkipTest("test may fail on ZFS filesystems") + elif inst.errno == errno.EOPNOTSUPP and sys.platform.startswith("netbsd"): + raise unittest.SkipTest("test may fail on FFS filesystems") else: raise finally: @@ -564,6 +566,7 @@ def test_dup(self): @unittest.skipUnless(hasattr(posix, 'confstr'), 'test needs posix.confstr()') + @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_confstr(self): self.assertRaises(ValueError, posix.confstr, "CS_garbage") self.assertEqual(len(posix.confstr("CS_PATH")) > 0, True) @@ -703,7 +706,8 @@ def test_makedev(self): self.assertEqual(posix.major(dev), major) self.assertRaises(TypeError, posix.major, float(dev)) self.assertRaises(TypeError, posix.major) - self.assertRaises((ValueError, OverflowError), posix.major, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.major, x) minor = posix.minor(dev) self.assertIsInstance(minor, int) @@ -711,13 +715,23 @@ def test_makedev(self): self.assertEqual(posix.minor(dev), minor) self.assertRaises(TypeError, posix.minor, float(dev)) self.assertRaises(TypeError, posix.minor) - self.assertRaises((ValueError, OverflowError), posix.minor, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.minor, x) self.assertEqual(posix.makedev(major, minor), dev) self.assertRaises(TypeError, posix.makedev, float(major), minor) self.assertRaises(TypeError, posix.makedev, major, float(minor)) self.assertRaises(TypeError, posix.makedev, major) self.assertRaises(TypeError, posix.makedev) + for x in -2, 2**32, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.makedev, x, minor) + self.assertRaises((ValueError, OverflowError), posix.makedev, major, x) + + if sys.platform == 'linux': + NODEV = -1 + self.assertEqual(posix.major(NODEV), NODEV) + self.assertEqual(posix.minor(NODEV), NODEV) + self.assertEqual(posix.makedev(NODEV, NODEV), NODEV) def _test_all_chown_common(self, chown_func, first_param, stat_func): """Common code for chown, fchown and lchown tests.""" @@ -1257,7 +1271,7 @@ def test_sched_priority(self): self.assertIsInstance(lo, int) self.assertIsInstance(hi, int) self.assertGreaterEqual(hi, lo) - # Apple plaforms return 15 without checking the argument. + # Apple platforms return 15 without checking the argument. if not is_apple: self.assertRaises(OSError, posix.sched_get_priority_min, -23) self.assertRaises(OSError, posix.sched_get_priority_max, -23) @@ -1270,9 +1284,10 @@ def test_get_and_set_scheduler_and_param(self): self.assertIn(mine, possible_schedulers) try: parent = posix.sched_getscheduler(os.getppid()) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + # POSIX specifies EPERM, but Android returns EACCES. Both errno + # values are mapped to PermissionError. + pass else: self.assertIn(parent, possible_schedulers) self.assertRaises(OSError, posix.sched_getscheduler, -1) @@ -1287,9 +1302,8 @@ def test_get_and_set_scheduler_and_param(self): try: posix.sched_setscheduler(0, mine, param) posix.sched_setparam(0, param) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + pass self.assertRaises(OSError, posix.sched_setparam, -1, param) self.assertRaises(OSError, posix.sched_setscheduler, -1, mine, param) @@ -1335,12 +1349,21 @@ def test_sched_getaffinity(self): def test_sched_setaffinity(self): mask = posix.sched_getaffinity(0) self.addCleanup(posix.sched_setaffinity, 0, list(mask)) + if len(mask) > 1: # Empty masks are forbidden mask.pop() posix.sched_setaffinity(0, mask) self.assertEqual(posix.sched_getaffinity(0), mask) - self.assertRaises(OSError, posix.sched_setaffinity, 0, []) + + try: + posix.sched_setaffinity(0, []) + # gh-117061: On RHEL9, sched_setaffinity(0, []) does not fail + except OSError: + # sched_setaffinity() manual page documents EINVAL error + # when the mask is empty. + pass + self.assertRaises(ValueError, posix.sched_setaffinity, 0, [-10]) self.assertRaises(ValueError, posix.sched_setaffinity, 0, map(int, "0X")) self.assertRaises(OverflowError, posix.sched_setaffinity, 0, [1<<128]) @@ -2119,6 +2142,13 @@ def test_stat(self): with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): os.stat("file", dir_fd=0) + def test_ptsname_r(self): + self._verify_available("HAVE_PTSNAME_R") + if self.mac_ver >= (10, 13, 4): + self.assertIn("HAVE_PTSNAME_R", posix._have_functions) + else: + self.assertNotIn("HAVE_PTSNAME_R", posix._have_functions) + def test_access(self): self._verify_available("HAVE_FACCESSAT") if self.mac_ver >= (10, 10): diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 86ce1b1d41ba61..ca5cf42f8fcd71 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,11 +1,12 @@ +import inspect import os import posixpath import sys import unittest from posixpath import realpath, abspath, dirname, basename from test import test_genericpath -from test.support import import_helper -from test.support import os_helper +from test.support import get_attribute, import_helper +from test.support import cpython_only, os_helper from test.support.os_helper import FakePath from unittest import mock @@ -56,6 +57,8 @@ def test_join(self): self.assertEqual(fn(b"/foo", b"bar", b"baz"), b"/foo/bar/baz") self.assertEqual(fn(b"/foo/", b"bar/", b"baz/"), b"/foo/bar/baz/") + self.assertEqual(fn("a", ""), "a/") + self.assertEqual(fn("a", "", ""), "a/") self.assertEqual(fn("a", "b"), "a/b") self.assertEqual(fn("a", "b/"), "a/b/") self.assertEqual(fn("a/", "b"), "a/b") @@ -281,6 +284,16 @@ def fake_lstat(path): def test_isjunction(self): self.assertFalse(posixpath.isjunction(ABSTFN)) + @unittest.skipIf(sys.platform == 'win32', "Fast paths are not for win32") + @cpython_only + def test_fast_paths_in_use(self): + # There are fast paths of these functions implemented in posixmodule.c. + # Confirm that they are being used, and not the Python fallbacks + self.assertTrue(os.path.splitroot is posix._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is posix._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) + def test_expanduser(self): self.assertEqual(posixpath.expanduser("foo"), "foo") self.assertEqual(posixpath.expanduser(b"foo"), b"foo") @@ -342,6 +355,24 @@ def test_expanduser_pwd(self): for path in ('~', '~/.local', '~vstinner/'): self.assertEqual(posixpath.expanduser(path), path) + @unittest.skipIf(sys.platform == "vxworks", + "no home directory on VxWorks") + def test_expanduser_pwd2(self): + pwd = import_helper.import_module('pwd') + for all_entry in get_attribute(pwd, 'getpwall')(): + name = all_entry.pw_name + + # gh-121200: pw_dir can be different between getpwall() and + # getpwnam(), so use getpwnam() pw_dir as expanduser() does. + entry = pwd.getpwnam(name) + home = entry.pw_dir + home = home.rstrip('/') or '/' + + with self.subTest(all_entry=all_entry, entry=entry): + self.assertEqual(posixpath.expanduser('~' + name), home) + self.assertEqual(posixpath.expanduser(os.fsencode('~' + name)), + os.fsencode(home)) + NORMPATH_CASES = [ ("", "."), ("/", "/"), @@ -456,6 +487,15 @@ def test_realpath_relative(self): finally: os_helper.unlink(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_missing_pardir(self): + try: + os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN) + self.assertEqual(realpath("nonexistent/../" + os_helper.TESTFN), ABSTFN + "1") + finally: + os_helper.unlink(os_helper.TESTFN) + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_symlink_loops(self): @@ -475,7 +515,7 @@ def test_realpath_symlink_loops(self): self.assertEqual(realpath(ABSTFN+"1/../x"), dirname(ABSTFN) + "/x") os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "y"), - ABSTFN + "y") + ABSTFN + "x") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "1"), ABSTFN + "1") @@ -637,10 +677,29 @@ def test_realpath_resolve_first(self): safe_rmdir(ABSTFN + "/k") safe_rmdir(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + def test_realpath_unreadable_symlink(self): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + self.assertEqual(realpath(ABSTFN), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/foo'), ABSTFN + '/foo') + self.assertEqual(realpath(ABSTFN + '/../foo'), dirname(ABSTFN) + '/foo') + self.assertEqual(realpath(ABSTFN + '/foo/..'), ABSTFN) + with self.assertRaises(PermissionError): + realpath(ABSTFN, strict=True) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os.unlink(ABSTFN) + def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") try: curdir = os.path.split(os.getcwd())[-1] + self.assertRaises(TypeError, posixpath.relpath, None) self.assertRaises(ValueError, posixpath.relpath, "") self.assertEqual(posixpath.relpath("a"), "a") self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") @@ -703,7 +762,9 @@ def check_error(exc, paths): self.assertRaises(exc, posixpath.commonpath, [os.fsencode(p) for p in paths]) + self.assertRaises(TypeError, posixpath.commonpath, None) self.assertRaises(ValueError, posixpath.commonpath, []) + self.assertRaises(ValueError, posixpath.commonpath, iter([])) check_error(ValueError, ['/usr', 'usr']) check_error(ValueError, ['usr', '/usr']) diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 4e6fed1ab969ac..dfbc2a06e7346f 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -8,7 +8,6 @@ import pprint import random import re -import test.support import types import unittest diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 5f1bfd9e30db98..f4805a1d6c6602 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -188,38 +188,6 @@ def test_string_in_loop_on_same_line(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - def test_stream_redirection_hint_for_py2_migration(self): - # Test correct hint produced for Py2 redirection syntax - with self.assertRaises(TypeError) as context: - print >> sys.stderr, "message" - self.assertIn('Did you mean "print(, ' - 'file=)"?', str(context.exception)) - - # Test correct hint is produced in the case where RHS implements - # __rrshift__ but returns NotImplemented - with self.assertRaises(TypeError) as context: - print >> 42 - self.assertIn('Did you mean "print(, ' - 'file=)"?', str(context.exception)) - - # Test stream redirection hint is specific to print - with self.assertRaises(TypeError) as context: - max >> sys.stderr - self.assertNotIn('Did you mean ', str(context.exception)) - - # Test stream redirection hint is specific to rshift - with self.assertRaises(TypeError) as context: - print << sys.stderr - self.assertNotIn('Did you mean', str(context.exception)) - - # Ensure right operand implementing rrshift still works - class OverrideRRShift: - def __rrshift__(self, lhs): - return 42 # Force result independent of LHS - - self.assertEqual(print >> OverrideRRShift(), 42) - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index a1dfc9abbb8ef7..0f16b92334999c 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -7,7 +7,7 @@ from difflib import unified_diff from io import StringIO from test.support.os_helper import TESTFN, unlink, temp_dir, change_cwd -from contextlib import contextmanager +from contextlib import contextmanager, redirect_stdout import profile from test.profilee import testfunc, timer @@ -92,6 +92,11 @@ def test_run(self): self.profilermodule.run("int('1')", filename=TESTFN) self.assertTrue(os.path.exists(TESTFN)) + def test_run_with_sort_by_values(self): + with redirect_stdout(StringIO()) as f: + self.profilermodule.run("int('1')", sort=('tottime', 'stdname')) + self.assertIn("Ordered by: internal time, standard name", f.getvalue()) + def test_runctx(self): with silent(): self.profilermodule.runctx("testfunc()", globals(), locals()) diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index c12c908d2ee32d..cea241b0f200d0 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -183,6 +183,77 @@ def test_refleaks_in___init__(self): fake_prop.__init__('fget', 'fset', 'fdel', 'doc') self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + @support.refcount_test + def test_gh_115618(self): + # Py_XDECREF() was improperly called for None argument + # in property methods. + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + prop = property() + refs_before = gettotalrefcount() + for i in range(100): + prop = prop.getter(None) + self.assertIsNone(prop.fget) + for i in range(100): + prop = prop.setter(None) + self.assertIsNone(prop.fset) + for i in range(100): + prop = prop.deleter(None) + self.assertIsNone(prop.fdel) + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_property_name(self): + def getter(self): + return 42 + + def setter(self, value): + pass + + class A: + @property + def foo(self): + return 1 + + @foo.setter + def oof(self, value): + pass + + bar = property(getter) + baz = property(None, setter) + + self.assertEqual(A.foo.__name__, 'foo') + self.assertEqual(A.oof.__name__, 'oof') + self.assertEqual(A.bar.__name__, 'bar') + self.assertEqual(A.baz.__name__, 'baz') + + A.quux = property(getter) + self.assertEqual(A.quux.__name__, 'getter') + A.quux.__name__ = 'myquux' + self.assertEqual(A.quux.__name__, 'myquux') + self.assertEqual(A.bar.__name__, 'bar') # not affected + A.quux.__name__ = None + self.assertIsNone(A.quux.__name__) + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(None, setter).__name__ + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(1).__name__ + + class Err: + def __getattr__(self, attr): + raise RuntimeError('fail') + + p = property(Err()) + with self.assertRaisesRegex(RuntimeError, 'fail'): + p.__name__ + + p.__name__ = 'not_fail' + self.assertEqual(p.__name__, 'not_fail') + def test_property_set_name_incorrect_args(self): p = property() @@ -224,6 +295,7 @@ class PropertySubSlots(property): class PropertySubclassTests(unittest.TestCase): + @support.requires_docstrings def test_slots_docstring_copy_exception(self): # A special case error that we preserve despite the GH-98963 behavior # that would otherwise silently ignore this error. @@ -366,7 +438,7 @@ def getter3(self): self.assertEqual(p2.__doc__, "doc-A") # Case-3: with no user-provided doc new getter doc - # takes precendence + # takes precedence p = property(getter2, None, None, None) p2 = p.getter(getter3) @@ -391,6 +463,40 @@ def getter3(self): self.assertEqual(p.__doc__, "user") self.assertEqual(p2.__doc__, "user") + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_prefer_explicit_doc(self): + # Issue 25757: subclasses of property lose docstring + self.assertEqual(property(doc="explicit doc").__doc__, "explicit doc") + self.assertEqual(PropertySub(doc="explicit doc").__doc__, "explicit doc") + + class Foo: + spam = PropertySub(doc="spam explicit doc") + + @spam.getter + def spam(self): + """ignored as doc already set""" + return 1 + + def _stuff_getter(self): + """ignored as doc set directly""" + stuff = PropertySub(doc="stuff doc argument", fget=_stuff_getter) + + #self.assertEqual(Foo.spam.__doc__, "spam explicit doc") + self.assertEqual(Foo.stuff.__doc__, "stuff doc argument") + + def test_property_no_doc_on_getter(self): + # If a property's getter has no __doc__ then the property's doc should + # be None; test that this is consistent with subclasses as well; see + # GH-2487 + class NoDoc: + @property + def __doc__(self): + raise AttributeError + + self.assertEqual(property(NoDoc()).__doc__, None) + self.assertEqual(PropertySub(NoDoc()).__doc__, None) + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_setter_copies_getter_docstring(self): diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 3f2bac0155fd9e..dee94533c74549 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,7 +1,6 @@ -import sys import unittest from test.support import ( - is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose + is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -9,9 +8,8 @@ # Skip these tests if termios is not available import_module('termios') -# Skip tests on WASM platforms, plus iOS/tvOS/watchOS -if is_apple_mobile or is_emscripten or is_wasi: - raise unittest.SkipTest(f"pty tests not required on {sys.platform}") +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("pty is not available on this platform") import errno import os diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index c4e6551f605782..64387296e84621 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -227,7 +227,8 @@ class PyCompileCLITestCase(unittest.TestCase): def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') - self.cache_path = importlib.util.cache_from_source(self.source_path) + self.cache_path = importlib.util.cache_from_source(self.source_path, + optimization='' if __debug__ else 1) with open(self.source_path, 'w') as file: file.write('x = 123\n') @@ -250,6 +251,7 @@ def pycompilecmd_failure(self, *args): return script_helper.assert_python_failure('-m', 'py_compile', *args) def test_stdin(self): + self.assertFalse(os.path.exists(self.cache_path)) result = self.pycompilecmd('-', input=self.source_path) self.assertEqual(result.returncode, 0) self.assertEqual(result.stdout, b'') diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index c7c5419ffe3e37..4bf0576586cca5 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -78,7 +78,8 @@ def ismethod(oclass, obj, name): objname = obj.__name__ if objname.startswith("__") and not objname.endswith("__"): - objname = "_%s%s" % (oclass.__name__, objname) + if stripped_typename := oclass.__name__.lstrip('_'): + objname = f"_{stripped_typename}{objname}" return objname == name # Make sure the toplevel functions and classes are the same. @@ -109,14 +110,20 @@ def ismethod(oclass, obj, name): actualMethods = [] for m in py_item.__dict__.keys(): + if m == "__annotate__": + continue if ismethod(py_item, getattr(py_item, m), m): actualMethods.append(m) - foundMethods = [] - for m in value.methods.keys(): - if m[:2] == '__' and m[-2:] != '__': - foundMethods.append('_'+name+m) - else: - foundMethods.append(m) + + if stripped_typename := name.lstrip('_'): + foundMethods = [] + for m in value.methods.keys(): + if m.startswith('__') and not m.endswith('__'): + foundMethods.append(f"_{stripped_typename}{m}") + else: + foundMethods.append(m) + else: + foundMethods = list(value.methods.keys()) try: self.assertListEq(foundMethods, actualMethods, ignore) @@ -150,8 +157,9 @@ def test_easy(self): "DocTestCase", '_DocTestSuite')) self.checkModule('difflib', ignore=("Match",)) - def test_decorators(self): - self.checkModule('test.pyclbr_input', ignore=['om']) + def test_cases(self): + # see test.pyclbr_input for the rationale behind the ignored symbols + self.checkModule('test.pyclbr_input', ignore=['om', 'f']) def test_nested(self): mb = pyclbr @@ -226,7 +234,7 @@ def test_others(self): cm( 'pdb', # pyclbr does not handle elegantly `typing` or properties - ignore=('Union', '_ModuleTarget', '_ScriptTarget'), + ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget'), ) cm('pydoc', ignore=('input', 'output',)) # properties diff --git a/Lib/test/test_pydoc/__init__.py b/Lib/test/test_pydoc/__init__.py new file mode 100644 index 00000000000000..f2a39a3fe29c7f --- /dev/null +++ b/Lib/test/test_pydoc/__init__.py @@ -0,0 +1,6 @@ +import os +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/pydoc_mod.py b/Lib/test/test_pydoc/pydoc_mod.py similarity index 100% rename from Lib/test/pydoc_mod.py rename to Lib/test/test_pydoc/pydoc_mod.py diff --git a/Lib/test/pydocfodder.py b/Lib/test/test_pydoc/pydocfodder.py similarity index 72% rename from Lib/test/pydocfodder.py rename to Lib/test/test_pydoc/pydocfodder.py index a3ef2231243954..3cc2d5bd57fe5b 100644 --- a/Lib/test/pydocfodder.py +++ b/Lib/test/test_pydoc/pydocfodder.py @@ -2,6 +2,12 @@ import types +def global_func(x, y): + """Module global function""" + +def global_func2(x, y): + """Module global function 2""" + class A: "A class." @@ -26,7 +32,7 @@ def A_classmethod(cls, x): "A class method defined in A." A_classmethod = classmethod(A_classmethod) - def A_staticmethod(): + def A_staticmethod(x, y): "A static method defined in A." A_staticmethod = staticmethod(A_staticmethod) @@ -61,6 +67,30 @@ def BD_method(self): def BCD_method(self): "Method defined in B, C and D." + @classmethod + def B_classmethod(cls, x): + "A class method defined in B." + + global_func = global_func # same name + global_func_alias = global_func + global_func2_alias = global_func2 + B_classmethod_alias = B_classmethod + A_classmethod_ref = A.A_classmethod + A_staticmethod = A.A_staticmethod # same name + A_staticmethod_alias = A.A_staticmethod + A_method_ref = A().A_method + A_method_alias = A.A_method + B_method_alias = B_method + count = list.count # same name + list_count = list.count + __repr__ = object.__repr__ # same name + object_repr = object.__repr__ + get = {}.get # same name + dict_get = {}.get + +B.B_classmethod_ref = B.B_classmethod + + class C(A): "A class, derived from A." @@ -136,3 +166,23 @@ def __call__(self, inst): submodule = types.ModuleType(__name__ + '.submodule', """A submodule, which should appear in its parent's summary""") + +global_func_alias = global_func +A_classmethod = A.A_classmethod # same name +A_classmethod2 = A.A_classmethod +A_classmethod3 = B.A_classmethod +A_staticmethod = A.A_staticmethod # same name +A_staticmethod_alias = A.A_staticmethod +A_staticmethod_ref = A().A_staticmethod +A_staticmethod_ref2 = B().A_staticmethod +A_method = A().A_method # same name +A_method2 = A().A_method +A_method3 = B().A_method +B_method = B.B_method # same name +B_method2 = B.B_method +count = list.count # same name +list_count = list.count +__repr__ = object.__repr__ # same name +object_repr = object.__repr__ +get = {}.get # same name +dict_get = {}.get diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py similarity index 74% rename from Lib/test/test_pydoc.py rename to Lib/test/test_pydoc/test_pydoc.py index 99b19d01783a10..2a4d3ab73db608 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -17,6 +17,7 @@ import types import typing import unittest +import unittest.mock import urllib.parse import xml.etree import xml.etree.ElementTree @@ -30,11 +31,12 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, spawn_python) from test.support import threading_helper -from test.support import (reap_children, captured_output, captured_stdout, +from test.support import (reap_children, captured_stdout, captured_stderr, is_emscripten, is_wasi, requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) -from test import pydoc_mod +from test.test_pydoc import pydoc_mod +from test.test_pydoc import pydocfodder class nonascii: @@ -51,7 +53,7 @@ class nonascii: expected_text_pattern = """ NAME - test.pydoc_mod - This is a test module for test_pydoc + test.test_pydoc.pydoc_mod - This is a test module for test_pydoc %s CLASSES builtins.object @@ -75,6 +77,11 @@ class A(builtins.object) | __weakref__%s class B(builtins.object) + | Methods defined here: + | + | __annotate__(...) + | + | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__%s @@ -85,8 +92,6 @@ class B(builtins.object) | Data and other attributes defined here: | | NO_MEANING = 'eggs' - | - | __annotations__ = {'NO_MEANING': } class C(builtins.object) | Methods defined here: @@ -102,7 +107,7 @@ class C(builtins.object) | ---------------------------------------------------------------------- | Class methods defined here: | - | __class_getitem__(item) from builtins.type + | __class_getitem__(item) | | ---------------------------------------------------------------------- | Data descriptors defined here: @@ -124,7 +129,7 @@ class C(builtins.object) DATA __xyz__ = 'X, Y and Z' - c_alias = test.pydoc_mod.C[int] + c_alias = test.test_pydoc.pydoc_mod.C[int] list_alias1 = typing.List[int] list_alias2 = list[int] type_union1 = typing.Union[int, str] @@ -147,7 +152,7 @@ class C(builtins.object) for s in expected_data_docstrings) html2text_of_expected = """ -test.pydoc_mod (version 1.2.3.4) +test.test_pydoc.pydoc_mod (version 1.2.3.4) This is a test module for test_pydoc Modules @@ -166,7 +171,7 @@ class A(builtins.object) Methods defined here: __init__() Wow, I have no function! - + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ dictionary for instance variables @@ -174,14 +179,17 @@ class A(builtins.object) list of weak references to the object class B(builtins.object) + Methods defined here: + __annotate__(...) + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ dictionary for instance variables __weakref__ list of weak references to the object + ---------------------------------------------------------------------- Data and other attributes defined here: NO_MEANING = 'eggs' - __annotations__ = {'NO_MEANING': } class C(builtins.object) @@ -191,8 +199,10 @@ class C(builtins.object) is_it_true(self) Return self.get_answer() say_no(self) + ---------------------------------------------------------------------- Class methods defined here: - __class_getitem__(item) from builtins.type + __class_getitem__(item) + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ dictionary for instance variables @@ -209,7 +219,7 @@ class C(builtins.object) Data __xyz__ = 'X, Y and Z' - c_alias = test.pydoc_mod.C[int] + c_alias = test.test_pydoc.pydoc_mod.C[int] list_alias1 = typing.List[int] list_alias2 = list[int] type_union1 = typing.Union[int, str] @@ -330,11 +340,15 @@ def get_pydoc_html(module): loc = "
    Module Docs" return output.strip(), loc +def clean_text(doc): + # clean up the extra text formatting that pydoc performs + return re.sub('\b.', '', doc) + def get_pydoc_link(module): "Returns a documentation web link of a module" abspath = os.path.abspath dirname = os.path.dirname - basedir = dirname(dirname(abspath(__file__))) + basedir = dirname(dirname(dirname(abspath(__file__)))) doc = pydoc.TextDoc() loc = doc.getdocloc(module, basedir=basedir) return loc @@ -347,10 +361,7 @@ def get_pydoc_text(module): loc = "\nMODULE DOCS\n " + loc + "\n" output = doc.docmodule(module) - - # clean up the extra text formatting that pydoc performs - patt = re.compile('\b.') - output = patt.sub('', output) + output = clean_text(output) return output.strip(), loc def get_html_title(text): @@ -367,12 +378,18 @@ def html2text(html): Tailored for pydoc tests only. """ html = html.replace("
    ", "\n") + html = html.replace("
    ", "-"*70) html = re.sub("<.*?>", "", html) html = pydoc.replace(html, " ", " ", ">", ">", "<", "<") return html class PydocBaseTest(unittest.TestCase): + def tearDown(self): + # Self-testing. Mocking only works if sys.modules['pydoc'] and pydoc + # are the same. But some pydoc functions reload the module and change + # sys.modules, so check that it was restored. + self.assertIs(sys.modules['pydoc'], pydoc) def _restricted_walk_packages(self, walk_packages, path=None): """ @@ -404,6 +421,8 @@ def call_url_handler(self, url, expected_title): class PydocDocTest(unittest.TestCase): maxDiff = None + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') @@ -444,6 +463,14 @@ class BinaryInteger(enum.IntEnum): doc = pydoc.render_doc(BinaryInteger) self.assertIn('BinaryInteger.zero', doc) + def test_slotted_dataclass_with_field_docs(self): + import dataclasses + @dataclasses.dataclass(slots=True) + class My: + x: int = dataclasses.field(doc='Docstring for x') + doc = pydoc.render_doc(My) + self.assertIn('Docstring for x', doc) + def test_mixed_case_module_names_are_lower_cased(self): # issue16484 doc_link = get_pydoc_link(xml.etree.ElementTree) @@ -483,7 +510,7 @@ def test_not_here(self): @requires_docstrings def test_not_ascii(self): - result = run_pydoc('test.test_pydoc.nonascii', PYTHONIOENCODING='ascii') + result = run_pydoc('test.test_pydoc.test_pydoc.nonascii', PYTHONIOENCODING='ascii') encoded = nonascii.__doc__.encode('ascii', 'backslashreplace') self.assertIn(encoded, result) @@ -652,40 +679,195 @@ def test_fail_help_output_redirect(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') + @unittest.mock.patch('pydoc.pager') @requires_docstrings - def test_help_output_redirect(self): + def test_help_output_redirect(self, pager_mock): # issue 940286, if output is set in Helper, then all output from # Helper.help should be redirected - getpager_old = pydoc.getpager - getpager_new = lambda: (lambda x: x) self.maxDiff = None - buf = StringIO() - helper = pydoc.Helper(output=buf) unused, doc_loc = get_pydoc_text(pydoc_mod) - module = "test.pydoc_mod" + module = "test.test_pydoc.pydoc_mod" help_header = """ - Help on module test.pydoc_mod in test: + Help on module test.test_pydoc.pydoc_mod in test.test_pydoc: """.lstrip() help_header = textwrap.dedent(help_header) expected_help_pattern = help_header + expected_text_pattern - pydoc.getpager = getpager_new - try: - with captured_output('stdout') as output, \ - captured_output('stderr') as err: - helper.help(module) + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(module) + result = buf.getvalue().strip() + expected_text = expected_help_pattern % ( + (doc_loc,) + + expected_text_data_docstrings + + (inspect.getabsfile(pydoc_mod),)) + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertEqual(expected_text, result) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_help_output_redirect_various_requests(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.help should be redirected + + def run_pydoc_for_request(request, expected_text_part): + """Helper function to run pydoc with its output redirected""" + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(request) result = buf.getvalue().strip() - expected_text = expected_help_pattern % ( - (doc_loc,) + - expected_text_data_docstrings + - (inspect.getabsfile(pydoc_mod),)) - self.assertEqual('', output.getvalue()) + self.assertEqual('', output.getvalue(), msg=f'failed on request "{request}"') + self.assertEqual('', err.getvalue(), msg=f'failed on request "{request}"') + self.assertIn(expected_text_part, result, msg=f'failed on request "{request}"') + pager_mock.assert_not_called() + + self.maxDiff = None + + # test for "keywords" + run_pydoc_for_request('keywords', 'Here is a list of the Python keywords.') + # test for "symbols" + run_pydoc_for_request('symbols', 'Here is a list of the punctuation symbols') + # test for "topics" + run_pydoc_for_request('topics', 'Here is a list of available topics.') + # test for "modules" skipped, see test_modules() + # test for symbol "%" + run_pydoc_for_request('%', 'The power operator') + # test for special True, False, None keywords + run_pydoc_for_request('True', 'class bool(int)') + run_pydoc_for_request('False', 'class bool(int)') + run_pydoc_for_request('None', 'class NoneType(object)') + # test for keyword "assert" + run_pydoc_for_request('assert', 'The "assert" statement') + # test for topic "TYPES" + run_pydoc_for_request('TYPES', 'The standard type hierarchy') + # test for "pydoc.Helper.help" + run_pydoc_for_request('pydoc.Helper.help', 'Help on function help in pydoc.Helper:') + # test for pydoc.Helper.help + run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:') + # test for pydoc.Helper() instance skipped because it is always meant to be interactive + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_help_output_pager(self): + def run_pydoc_pager(request, what, expected_first_line): + with (captured_stdout() as output, + captured_stderr() as err, + unittest.mock.patch('pydoc.pager') as pager_mock, + self.subTest(repr(request))): + helper = pydoc.Helper() + helper.help(request) self.assertEqual('', err.getvalue()) - self.assertEqual(expected_text, result) - finally: - pydoc.getpager = getpager_old + self.assertEqual('\n', output.getvalue()) + pager_mock.assert_called_once() + result = clean_text(pager_mock.call_args.args[0]) + self.assertEqual(result.splitlines()[0], expected_first_line) + self.assertEqual(pager_mock.call_args.args[1], f'Help on {what}') + + run_pydoc_pager('%', 'EXPRESSIONS', 'Operator precedence') + run_pydoc_pager('True', 'bool object', 'Help on bool object:') + run_pydoc_pager(True, 'bool object', 'Help on bool object:') + run_pydoc_pager('assert', 'assert', 'The "assert" statement') + run_pydoc_pager('TYPES', 'TYPES', 'The standard type hierarchy') + run_pydoc_pager('pydoc.Helper.help', 'pydoc.Helper.help', + 'Help on function help in pydoc.Helper:') + run_pydoc_pager(pydoc.Helper.help, 'Helper.help', + 'Help on function help in module pydoc:') + run_pydoc_pager('str', 'str', 'Help on class str in module builtins:') + run_pydoc_pager(str, 'str', 'Help on class str in module builtins:') + run_pydoc_pager('str.upper', 'str.upper', + 'Help on method descriptor upper in str:') + run_pydoc_pager(str.upper, 'str.upper', + 'Help on method descriptor upper:') + run_pydoc_pager(''.upper, 'str.upper', + 'Help on built-in function upper:') + run_pydoc_pager(str.__add__, + 'str.__add__', 'Help on method descriptor __add__:') + run_pydoc_pager(''.__add__, + 'str.__add__', 'Help on method wrapper __add__:') + run_pydoc_pager(int.numerator, 'int.numerator', + 'Help on getset descriptor builtins.int.numerator:') + run_pydoc_pager(list[int], 'list', + 'Help on GenericAlias in module builtins:') + run_pydoc_pager('sys', 'sys', 'Help on built-in module sys:') + run_pydoc_pager(sys, 'sys', 'Help on built-in module sys:') + + def test_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('with') + helptext = showtopic_io.getvalue() + self.assertIn('The "with" statement', helptext) + + def test_fail_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('abd') + expected = "no documentation found for 'abd'" + self.assertEqual(expected, showtopic_io.getvalue().strip()) + + @unittest.mock.patch('pydoc.pager') + def test_fail_showtopic_output_redirect(self, pager_mock): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.showtopic("abd") + expected = "no documentation found for 'abd'" + self.assertEqual(expected, buf.getvalue().strip()) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_showtopic_output_redirect(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.showtopic should be redirected + self.maxDiff = None + + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.showtopic('with') + result = buf.getvalue().strip() + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertIn('The "with" statement', result) + + pager_mock.assert_not_called() + + def test_lambda_with_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"return": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a, b, c) -> int", helptext) + + def test_lambda_without_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int)", helptext) + + def test_lambda_with_return_and_params_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int, "return": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int) -> int", helptext) def test_namedtuple_fields(self): Person = namedtuple('Person', ['nickname', 'firstname']) @@ -798,8 +980,7 @@ def itemconfigure(self, tagOrId, cnf=None, **kw): b_size = A.a_size doc = pydoc.render_doc(B) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''\ Python Library Documentation: class B in module %s @@ -887,13 +1068,12 @@ def __init__(self, ... doc = pydoc.render_doc(A) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''Python Library Documentation: class A in module %s class A(builtins.object) | A( - | arg1: collections.abc.Callable[[int, int, int], str], + | arg1: Callable[[int, int, int], str], | arg2: Literal['some value', 'other value'], | arg3: Annotated[int, 'some docs about this type'] | ) -> None @@ -902,7 +1082,7 @@ class A(builtins.object) | | __init__( | self, - | arg1: collections.abc.Callable[[int, int, int], str], + | arg1: Callable[[int, int, int], str], | arg2: Literal['some value', 'other value'], | arg3: Annotated[int, 'some docs about this type'] | ) -> None @@ -925,12 +1105,11 @@ def func( ... doc = pydoc.render_doc(func) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''Python Library Documentation: function func in module %s func( - arg1: collections.abc.Callable[[typing.Annotated[int, 'Some doc']], str], + arg1: Callable[[Annotated[int, 'Some doc']], str], arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8] ) -> Annotated[int, 'Some other'] ''' % __name__) @@ -942,8 +1121,7 @@ def function_with_really_long_name_so_annotations_can_be_rather_small( ... doc = pydoc.render_doc(function_with_really_long_name_so_annotations_can_be_rather_small) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''Python Library Documentation: function function_with_really_long_name_so_annotations_can_be_rather_small in module %s function_with_really_long_name_so_annotations_can_be_rather_small( @@ -957,8 +1135,7 @@ def function_with_really_long_name_so_annotations_can_be_rather_small( second_very_long_parameter_name: ... doc = pydoc.render_doc(does_not_have_name) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''Python Library Documentation: function in module %s lambda very_long_parameter_name_that_should_not_fit_into_a_single_line, second_very_long_parameter_name @@ -1077,7 +1254,8 @@ def test_url_search_package_error(self): sys.path.insert(0, TESTFN) try: with self.assertRaisesRegex(ValueError, "ouch"): - import test_error_package # Sanity check + # Sanity check + import test_error_package # noqa: F401 text = self.call_url_handler("search?key=test_error_package", "Pydoc: Search Results") @@ -1129,19 +1307,34 @@ def test_modules_search_builtin(self): self.assertTrue(result.startswith(expected)) def test_importfile(self): - loaded_pydoc = pydoc.importfile(pydoc.__file__) + try: + loaded_pydoc = pydoc.importfile(pydoc.__file__) - self.assertIsNot(loaded_pydoc, pydoc) - self.assertEqual(loaded_pydoc.__name__, 'pydoc') - self.assertEqual(loaded_pydoc.__file__, pydoc.__file__) - self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) + self.assertIsNot(loaded_pydoc, pydoc) + self.assertEqual(loaded_pydoc.__name__, 'pydoc') + self.assertEqual(loaded_pydoc.__file__, pydoc.__file__) + self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) + finally: + sys.modules['pydoc'] = pydoc + + +class Rect: + @property + def area(self): + '''Area of the rect''' + return self.w * self.h + + +class Square(Rect): + area = property(lambda self: self.side**2) class TestDescriptions(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) def test_module(self): # Check that pydocfodder module can be described - from test import pydocfodder doc = pydoc.render_doc(pydocfodder) self.assertIn("pydocfodder", doc) @@ -1201,8 +1394,8 @@ def foo(data: typing.List[typing.Any], T = typing.TypeVar('T') class C(typing.Generic[T], typing.Mapping[int, str]): ... self.assertEqual(pydoc.render_doc(foo).splitlines()[-1], - 'f\x08fo\x08oo\x08o(data: List[Any], x: int)' - ' -> Iterator[Tuple[int, Any]]') + 'f\x08fo\x08oo\x08o(data: typing.List[typing.Any], x: int)' + ' -> typing.Iterator[typing.Tuple[int, typing.Any]]') self.assertEqual(pydoc.render_doc(C).splitlines()[2], 'class C\x08C(collections.abc.Mapping, typing.Generic)') @@ -1244,7 +1437,7 @@ def test_unbound_python_method(self): @requires_docstrings def test_unbound_builtin_method(self): self.assertEqual(self._get_summary_line(_pickle.Pickler.dump), - "dump(self, obj, /)") + "dump(self, obj, /) unbound _pickle.Pickler method") # these no longer include "self" def test_bound_python_method(self): @@ -1296,7 +1489,7 @@ def test_module_level_callable_o(self): def test_unbound_builtin_method_noargs(self): self.assertEqual(self._get_summary_line(str.lower), - "lower(self, /)") + "lower(self, /) unbound builtins.str method") def test_bound_builtin_method_noargs(self): self.assertEqual(self._get_summary_line(''.lower), @@ -1304,7 +1497,7 @@ def test_bound_builtin_method_noargs(self): def test_unbound_builtin_method_o(self): self.assertEqual(self._get_summary_line(set.add), - "add(self, object, /)") + "add(self, object, /) unbound builtins.set method") def test_bound_builtin_method_o(self): self.assertEqual(self._get_summary_line(set().add), @@ -1312,7 +1505,7 @@ def test_bound_builtin_method_o(self): def test_unbound_builtin_method_coexist_o(self): self.assertEqual(self._get_summary_line(set.__contains__), - "__contains__(self, object, /)") + "__contains__(self, object, /) unbound builtins.set method") def test_bound_builtin_method_coexist_o(self): self.assertEqual(self._get_summary_line(set().__contains__), @@ -1320,24 +1513,24 @@ def test_bound_builtin_method_coexist_o(self): def test_unbound_builtin_classmethod_noargs(self): self.assertEqual(self._get_summary_line(datetime.datetime.__dict__['utcnow']), - "utcnow(type, /)") + "utcnow(type, /) unbound datetime.datetime method") def test_bound_builtin_classmethod_noargs(self): self.assertEqual(self._get_summary_line(datetime.datetime.utcnow), - "utcnow() method of builtins.type instance") + "utcnow() class method of datetime.datetime") def test_unbound_builtin_classmethod_o(self): self.assertEqual(self._get_summary_line(dict.__dict__['__class_getitem__']), - "__class_getitem__(type, object, /)") + "__class_getitem__(type, object, /) unbound builtins.dict method") def test_bound_builtin_classmethod_o(self): self.assertEqual(self._get_summary_line(dict.__class_getitem__), - "__class_getitem__(object, /) method of builtins.type instance") + "__class_getitem__(object, /) class method of builtins.dict") @support.cpython_only @requires_docstrings def test_module_level_callable_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.func_with_unrepresentable_signature self.assertEqual(self._get_summary_line(builtin), "func_with_unrepresentable_signature(a, b=)") @@ -1347,7 +1540,7 @@ def test_module_level_callable_unrepresentable_default(self): def test_builtin_staticmethod_unrepresentable_default(self): self.assertEqual(self._get_summary_line(str.maketrans), "maketrans(x, y=, z=, /)") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.staticmeth), "staticmeth(a, b=)") @@ -1356,11 +1549,13 @@ def test_builtin_staticmethod_unrepresentable_default(self): @requires_docstrings def test_unbound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line(dict.pop), - "pop(self, key, default=, /)") - import _testcapi + "pop(self, key, default=, /) " + "unbound builtins.dict method") + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.meth), - "meth(self, /, a, b=)") + "meth(self, /, a, b=) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") @support.cpython_only @requires_docstrings @@ -1368,7 +1563,7 @@ def test_bound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line({}.pop), "pop(key, default=, /) " "method of builtins.dict instance") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") obj = _testcapi.DocStringUnrepresentableSignatureTest() self.assertEqual(self._get_summary_line(obj.meth), "meth(a, b=) " @@ -1377,19 +1572,21 @@ def test_bound_builtin_method_unrepresentable_default(self): @support.cpython_only @requires_docstrings def test_unbound_builtin_classmethod_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest descr = cls.__dict__['classmeth'] self.assertEqual(self._get_summary_line(descr), - "classmeth(type, /, a, b=)") + "classmeth(type, /, a, b=) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") @support.cpython_only @requires_docstrings def test_bound_builtin_classmethod_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.classmeth), - "classmeth(a, b=) method of builtins.type instance") + "classmeth(a, b=) class method of " + "_testcapi.DocStringUnrepresentableSignatureTest") def test_overridden_text_signature(self): class C: @@ -1420,10 +1617,10 @@ def smeth(*args, **kwargs): self.assertEqual(self._get_summary_line(C.meth), "meth" + unbound) self.assertEqual(self._get_summary_line(C().meth), - "meth" + bound + " method of test.test_pydoc.C instance") + "meth" + bound + " method of test.test_pydoc.test_pydoc.C instance") C.cmeth.__func__.__text_signature__ = text_signature self.assertEqual(self._get_summary_line(C.cmeth), - "cmeth" + bound + " method of builtins.type instance") + "cmeth" + bound + " class method of test.test_pydoc.test_pydoc.C") C.smeth.__text_signature__ = text_signature self.assertEqual(self._get_summary_line(C.smeth), "smeth" + unbound) @@ -1460,13 +1657,13 @@ def cm(cls, x): 'cm(...)\n' ' A class method\n') self.assertEqual(self._get_summary_lines(X.cm), """\ -cm(x) method of builtins.type instance +cm(x) class method of test.test_pydoc.test_pydoc.X A class method """) self.assertIn(""" | Class methods defined here: | - | cm(x) from builtins.type + | cm(x) | A class method """, pydoc.plain(pydoc.render_doc(X))) @@ -1522,13 +1719,13 @@ def test_namedtuple_field_descriptor(self): @requires_docstrings def test_property(self): - class Rect: - @property - def area(self): - '''Area of the rect''' - return self.w * self.h - self.assertEqual(self._get_summary_lines(Rect.area), """\ +area + Area of the rect +""") + # inherits the docstring from Rect.area + self.assertEqual(self._get_summary_lines(Square.area), """\ +area Area of the rect """) self.assertIn(""" @@ -1623,12 +1820,150 @@ def a_fn_with_https_link(): ) +class PydocFodderTest(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + def getsection(self, text, beginline, endline): + lines = text.splitlines() + beginindex, endindex = 0, None + if beginline is not None: + beginindex = lines.index(beginline) + if endline is not None: + endindex = lines.index(endline, beginindex) + return lines[beginindex:endindex] + + def test_text_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.TextDoc() + result = doc.docclass(cls) + result = clean_text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f' | Methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_method_alias = A_method(self)', lines) + self.assertIn(' | B_method_alias = B_method(self)', lines) + self.assertIn(' | A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn(' | A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn(' | global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | global_func_alias = global_func(x, y)', lines) + self.assertIn(' | global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | count(self, value, /) from builtins.list', lines) + self.assertIn(' | list_count = count(self, value, /)', lines) + self.assertIn(' | __repr__(self, /) from builtins.object', lines) + self.assertIn(' | object_repr = __repr__(self, /)', lines) + + lines = self.getsection(result, f' | Static methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn(' | B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn(' | A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + self.assertIn(' | get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' | dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + + lines = self.getsection(result, f' | Class methods {where}:', ' | ' + '-'*70) + self.assertIn(' | B_classmethod(x)', lines) + self.assertIn(' | B_classmethod_alias = B_classmethod(x)', lines) + + def test_html_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.HTMLDoc() + result = doc.docclass(cls) + result = html2text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f'Methods {where}:', '-'*70) + self.assertIn('A_method_alias = A_method(self)', lines) + self.assertIn('B_method_alias = B_method(self)', lines) + self.assertIn('A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn('A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn('global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('global_func_alias = global_func(x, y)', lines) + self.assertIn('global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('count(self, value, /) from builtins.list', lines) + self.assertIn('list_count = count(self, value, /)', lines) + self.assertIn('__repr__(self, /) from builtins.object', lines) + self.assertIn('object_repr = __repr__(self, /)', lines) + + lines = self.getsection(result, f'Static methods {where}:', '-'*70) + self.assertIn('A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn('B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn('A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + + lines = self.getsection(result, f'Class methods {where}:', '-'*70) + self.assertIn('B_classmethod(x)', lines) + self.assertIn('B_classmethod_alias = B_classmethod(x)', lines) + + def test_text_doc_inherited_routines_in_class(self): + self.test_text_doc_routines_in_class(pydocfodder.D) + + def test_html_doc_inherited_routines_in_class(self): + self.test_html_doc_routines_in_class(pydocfodder.D) + + def test_text_doc_routines_in_module(self): + doc = pydoc.TextDoc() + result = doc.docmodule(pydocfodder) + result = clean_text(result) + lines = self.getsection(result, 'FUNCTIONS', 'FILE') + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn(' A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + + def test_html_doc_routines_in_module(self): + doc = pydoc.HTMLDoc() + result = doc.docmodule(pydocfodder) + result = html2text(result) + lines = self.getsection(result, ' Functions', None) + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn('A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + + @unittest.skipIf( is_emscripten or is_wasi, "Socket server not available on Emscripten/WASI." ) class PydocServerTest(unittest.TestCase): """Tests for pydoc._start_server""" + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) def test_server(self): # Minimal test that starts the server, checks that it works, then stops @@ -1691,9 +2026,14 @@ def test_url_requests(self): ("foobar", "Pydoc: Error - foobar"), ] - with self.restrict_walk_packages(): - for url, title in requests: - self.call_url_handler(url, title) + self.assertIs(sys.modules['pydoc'], pydoc) + try: + with self.restrict_walk_packages(): + for url, title in requests: + self.call_url_handler(url, title) + finally: + # Some requests reload the module and change sys.modules. + sys.modules['pydoc'] = pydoc class TestHelper(unittest.TestCase): @@ -1703,6 +2043,9 @@ def test_keywords(self): class PydocWithMetaClasses(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') @requires_docstrings diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index d941a1a8f9ebc6..1d56ccd71cf962 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -755,5 +755,59 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ReparseDeferralTest(unittest.TestCase): + def test_getter_setter_round_trip(self): + parser = expat.ParserCreate() + enabled = (expat.version_info >= (2, 6, 0)) + + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + parser.SetReparseDeferralEnabled(False) + self.assertIs(parser.GetReparseDeferralEnabled(), False) + parser.SetReparseDeferralEnabled(True) + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + + def test_reparse_deferral_enabled(self): + if expat.version_info < (2, 6, 0): + self.skipTest(f'Expat {expat.version_info} does not ' + 'support reparse deferral') + + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + self.assertTrue(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: no. + self.assertEqual(started, []) + + parser.Parse(b'', True) + + self.assertEqual(started, ['doc']) + + def test_reparse_deferral_disabled(self): + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + if expat.version_info >= (2, 6, 0): + parser.SetReparseDeferralEnabled(False) + self.assertFalse(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: yes. + self.assertEqual(started, ['doc']) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py new file mode 100644 index 00000000000000..8359d9844623c2 --- /dev/null +++ b/Lib/test/test_pyrepl/__init__.py @@ -0,0 +1,15 @@ +import os +import sys +from test.support import requires, load_package_tests +from test.support.import_helper import import_module + +if sys.platform != "win32": + # On non-Windows platforms, testing pyrepl currently requires that the + # 'curses' resource be given on the regrtest command line using the -u + # option. Additionally, we need to attempt to import curses and readline. + requires("curses") + curses = import_module("curses") + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pyrepl/__main__.py b/Lib/test/test_pyrepl/__main__.py new file mode 100644 index 00000000000000..cbe9e01d0df820 --- /dev/null +++ b/Lib/test/test_pyrepl/__main__.py @@ -0,0 +1,4 @@ +import unittest +from test.test_pyrepl import load_tests + +unittest.main() diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py new file mode 100644 index 00000000000000..672d4896c92283 --- /dev/null +++ b/Lib/test/test_pyrepl/support.py @@ -0,0 +1,168 @@ +import os +from code import InteractiveConsole +from functools import partial +from typing import Iterable +from unittest.mock import MagicMock + +from _pyrepl.console import Console, Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig +from _pyrepl.simple_interact import _strip_final_indent + + +def multiline_input(reader: ReadlineAlikeReader, namespace: dict | None = None): + saved = reader.more_lines + try: + reader.more_lines = partial(more_lines, namespace=namespace) + reader.ps1 = reader.ps2 = ">>>" + reader.ps3 = reader.ps4 = "..." + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + +def more_lines(text: str, namespace: dict | None = None): + if namespace is None: + namespace = {} + src = _strip_final_indent(text) + console = InteractiveConsole(namespace, filename="") + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + +def code_to_events(code: str): + for c in code: + yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) + + +def clean_screen(screen: Iterable[str]): + """Cleans color and console characters out of a screen output. + + This is useful for screen testing, it increases the test readability since + it strips out all the unreadable side of the screen. + """ + output = [] + for line in screen: + if line.startswith(">>>") or line.startswith("..."): + line = line[3:] + output.append(line) + return "\n".join(output).strip() + + +def prepare_reader(console: Console, **kwargs): + config = ReadlineConfig(readline_completer=kwargs.pop("readline_completer", None)) + reader = ReadlineAlikeReader(console=console, config=config) + reader.more_lines = partial(more_lines, namespace=None) + reader.paste_mode = True # Avoid extra indents + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) + + for key, val in kwargs.items(): + setattr(reader, key, val) + + return reader + + +def prepare_console(events: Iterable[Event], **kwargs) -> MagicMock | Console: + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def handle_all_events( + events, prepare_console=prepare_console, prepare_reader=prepare_reader +): + console = prepare_console(events) + reader = prepare_reader(console) + try: + while True: + reader.handle1() + except StopIteration: + pass + except KeyboardInterrupt: + pass + return reader, console + + +handle_events_narrow_console = partial( + handle_all_events, + prepare_console=partial(prepare_console, width=10), +) + + +def make_clean_env() -> dict[str, str]: + clean_env = os.environ.copy() + for k in clean_env.copy(): + if k.startswith("PYTHON"): + clean_env.pop(k) + clean_env.pop("FORCE_COLOR", None) + clean_env.pop("NO_COLOR", None) + return clean_env + + +class FakeConsole(Console): + def __init__(self, events, encoding="utf-8") -> None: + self.events = iter(events) + self.encoding = encoding + self.screen = [] + self.height = 100 + self.width = 80 + + def get_event(self, block: bool = True) -> Event | None: + return next(self.events) + + def getpending(self) -> Event: + return self.get_event(block=False) + + def getheightwidth(self) -> tuple[int, int]: + return self.height, self.width + + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: + pass + + def prepare(self) -> None: + pass + + def restore(self) -> None: + pass + + def move_cursor(self, x: int, y: int) -> None: + pass + + def set_cursor_vis(self, visible: bool) -> None: + pass + + def push_char(self, char: int | bytes) -> None: + pass + + def beep(self) -> None: + pass + + def clear(self) -> None: + pass + + def finish(self) -> None: + pass + + def flushoutput(self) -> None: + pass + + def forgetinput(self) -> None: + pass + + def wait(self, timeout: float | None = None) -> bool: + return True + + def repaint(self) -> None: + pass diff --git a/Lib/test/test_pyrepl/test_input.py b/Lib/test/test_pyrepl/test_input.py new file mode 100644 index 00000000000000..c78c876c2c4c2a --- /dev/null +++ b/Lib/test/test_pyrepl/test_input.py @@ -0,0 +1,102 @@ +import unittest + +from _pyrepl.console import Event +from _pyrepl.input import KeymapTranslator + + +class KeymapTranslatorTests(unittest.TestCase): + def test_push_single_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_multiple_keys(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_invalid_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "b") + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["b"])) + + def test_push_invalid_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_character_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_character_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + evt3 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key_interrupted(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + evt3 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_invalid_key_with_unicode_category(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "\u0003") # Control character + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["\u0003"])) + + def test_empty(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + self.assertTrue(translator.empty()) + evt = Event("key", "a") + translator.push(evt) + self.assertFalse(translator.empty()) + translator.get() + self.assertTrue(translator.empty()) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py new file mode 100644 index 00000000000000..e0ee310e2c4dbc --- /dev/null +++ b/Lib/test/test_pyrepl/test_interact.py @@ -0,0 +1,262 @@ +import contextlib +import io +import unittest +from unittest.mock import patch +from textwrap import dedent + +from test.support import force_not_colorized + +from _pyrepl.console import InteractiveColoredConsole +from _pyrepl.simple_interact import _more_lines + +class TestSimpleInteract(unittest.TestCase): + def test_multiple_statements(self): + namespace = {} + code = dedent("""\ + class A: + def foo(self): + + + pass + + class B: + def bar(self): + pass + + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with ( + patch.object(InteractiveColoredConsole, "showsyntaxerror") as showsyntaxerror, + patch.object(InteractiveColoredConsole, "runsource", wraps=console.runsource) as runsource, + contextlib.redirect_stdout(f), + ): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + showsyntaxerror.assert_not_called() + + + def test_multiple_statements_output(self): + namespace = {} + code = dedent("""\ + b = 1 + b + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "1\n") + + def test_empty(self): + namespace = {} + code = "" + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "") + + def test_runsource_compiles_and_runs_code(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + with patch.object(console, "runcode") as mock_runcode: + console.runsource(source) + mock_runcode.assert_called_once() + + def test_runsource_returns_false_for_successful_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + + @force_not_colorized + def test_runsource_returns_false_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertIn('SyntaxError', f.getvalue()) + + @force_not_colorized + def test_runsource_show_syntax_error_location(self): + console = InteractiveColoredConsole() + source = "def f(x, x): ..." + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + r = """ + def f(x, x): ... + ^ +SyntaxError: duplicate argument 'x' in function definition""" + self.assertIn(r, f.getvalue()) + + def test_runsource_shows_syntax_error_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() + source = dedent("""\ + match 1: + case {0: _, 0j: _}: + pass + """) + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() + + def test_runsource_survives_null_bytes(self): + console = InteractiveColoredConsole() + source = "\x00\n" + f = io.StringIO() + with contextlib.redirect_stdout(f), contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertIn("source code string cannot contain null bytes", f.getvalue()) + + def test_no_active_future(self): + console = InteractiveColoredConsole() + source = dedent("""\ + x: int = 1 + print(__annotate__(1)) + """) + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertEqual(f.getvalue(), "{'x': }\n") + + def test_future_annotations(self): + console = InteractiveColoredConsole() + source = dedent("""\ + from __future__ import annotations + def g(x: int): ... + print(g.__annotations__) + """) + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertEqual(f.getvalue(), "{'x': 'int'}\n") + + def test_future_barry_as_flufl(self): + console = InteractiveColoredConsole() + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource("from __future__ import barry_as_FLUFL\n") + result = console.runsource("""print("black" <> 'blue')\n""") + self.assertFalse(result) + self.assertEqual(f.getvalue(), "True\n") + + +class TestMoreLines(unittest.TestCase): + def test_invalid_syntax_single_line(self): + namespace = {} + code = "if foo" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_empty_line(self): + namespace = {} + code = "" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_valid_single_statement(self): + namespace = {} + code = "foo = 1" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiline_single_assignment(self): + namespace = {} + code = dedent("""\ + foo = [ + 1, + 2, + 3, + ]""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiline_single_block(self): + namespace = {} + code = dedent("""\ + def foo(): + '''docs''' + + return 1""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_statements_single_line(self): + namespace = {} + code = "foo = 1;bar = 2" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiple_statements(self): + namespace = {} + code = dedent("""\ + import time + + foo = 1""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_blocks(self): + namespace = {} + code = dedent("""\ + from dataclasses import dataclass + + @dataclass + class Point: + x: float + y: float""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_blocks_empty_newline(self): + namespace = {} + code = dedent("""\ + from dataclasses import dataclass + + @dataclass + class Point: + x: float + y: float + """) + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiple_blocks_indented_newline(self): + namespace = {} + code = ( + "from dataclasses import dataclass\n" + "\n" + "@dataclass\n" + "class Point:\n" + " x: float\n" + " y: float\n" + " " + ) + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_incomplete_statement(self): + namespace = {} + code = "if foo:" + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) diff --git a/Lib/test/test_pyrepl/test_keymap.py b/Lib/test/test_pyrepl/test_keymap.py new file mode 100644 index 00000000000000..2c97066b2c7043 --- /dev/null +++ b/Lib/test/test_pyrepl/test_keymap.py @@ -0,0 +1,120 @@ +import string +import unittest + +from _pyrepl.keymap import _keynames, _escapes, parse_keys, compile_keymap, KeySpecError + + +class TestParseKeys(unittest.TestCase): + def test_single_character(self): + """Ensure that single ascii characters or single digits are parsed as single characters.""" + test_cases = [(key, [key]) for key in string.ascii_letters + string.digits] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_keynames(self): + """Ensure that keynames are parsed to their corresponding mapping. + + A keyname is expected to be of the following form: \\ such as \\ + which would get parsed as "left". + """ + test_cases = [(f"\\<{keyname}>", [parsed_keyname]) for keyname, parsed_keyname in _keynames.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_escape_sequences(self): + """Ensure that escaping sequences are parsed to their corresponding mapping.""" + test_cases = [(f"\\{escape}", [parsed_escape]) for escape, parsed_escape in _escapes.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_control_sequences(self): + """Ensure that supported control sequences are parsed successfully.""" + keys = ["@", "[", "]", "\\", "^", "_", "\\", "\\"] + keys.extend(string.ascii_letters) + test_cases = [(f"\\C-{key}", chr(ord(key) & 0x1F)) for key in []] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_meta_sequences(self): + self.assertEqual(parse_keys("\\M-a"), ["\033", "a"]) + self.assertEqual(parse_keys("\\M-b"), ["\033", "b"]) + self.assertEqual(parse_keys("\\M-c"), ["\033", "c"]) + + def test_combinations(self): + self.assertEqual(parse_keys("\\C-a\\n\\"), ["\x01", "\n", "up"]) + self.assertEqual(parse_keys("\\M-a\\t\\"), ["\033", "a", "\t", "down"]) + + def test_keyspec_errors(self): + cases = [ + ("\\Ca", "\\C must be followed by `-'"), + ("\\ca", "\\C must be followed by `-'"), + ("\\C-\\C-", "doubled \\C-"), + ("\\Ma", "\\M must be followed by `-'"), + ("\\ma", "\\M must be followed by `-'"), + ("\\M-\\M-", "doubled \\M-"), + ("\\", "unrecognised keyname"), + ("\\大", "unknown backslash escape"), + ("\\C-\\", "\\C- followed by invalid key") + ] + for test_keys, expected_err in cases: + with self.subTest(f"{test_keys} should give error {expected_err}"): + with self.assertRaises(KeySpecError) as e: + parse_keys(test_keys) + self.assertIn(expected_err, str(e.exception)) + + def test_index_errors(self): + test_cases = ["\\", "\\C", "\\C-\\C"] + for test_keys in test_cases: + with self.assertRaises(IndexError): + parse_keys(test_keys) + + +class TestCompileKeymap(unittest.TestCase): + def test_empty_keymap(self): + keymap = {} + result = compile_keymap(keymap) + self.assertEqual(result, {}) + + def test_single_keymap(self): + keymap = {b"a": "action"} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": "action"}) + + def test_nested_keymap(self): + keymap = {b"a": {b"b": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action"}}) + + def test_empty_value(self): + keymap = {b"a": {b"": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action"}}) + + def test_multiple_empty_values(self): + keymap = {b"a": {b"": "action1", b"b": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action1", b"b": "action2"}}) + + def test_multiple_keymaps(self): + keymap = {b"a": {b"b": "action1", b"c": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action1", b"c": "action2"}}) + + def test_nested_multiple_keymaps(self): + keymap = {b"a": {b"b": {b"c": "action"}}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": {b"c": "action"}}}) + + def test_clashing_definitions(self): + km = {b'a': 'c', b'a' + b'b': 'd'} + with self.assertRaises(KeySpecError): + compile_keymap(km) + + def test_non_bytes_key(self): + with self.assertRaises(TypeError): + compile_keymap({123: 'a'}) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py new file mode 100644 index 00000000000000..f29a7ffbd7cafd --- /dev/null +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -0,0 +1,1346 @@ +import io +import itertools +import os +import pathlib +import re +import rlcompleter +import select +import subprocess +import sys +import tempfile +from unittest import TestCase, skipUnless, skipIf +from unittest.mock import patch +from test.support import force_not_colorized +from test.support import SHORT_TIMEOUT +from test.support.import_helper import import_module +from test.support.os_helper import unlink + +from .support import ( + FakeConsole, + handle_all_events, + handle_events_narrow_console, + more_lines, + multiline_input, + code_to_events, + clean_screen, + make_clean_env, +) +from _pyrepl.console import Event +from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig, + _ReadlineWrapper) +from _pyrepl.readline import multiline_input as readline_multiline_input + +try: + import pty +except ImportError: + pty = None + + +class ReplTestCase(TestCase): + def run_repl( + self, + repl_input: str | list[str], + env: dict | None = None, + *, + cmdline_args: list[str] | None = None, + cwd: str | None = None, + ) -> tuple[str, int]: + temp_dir = None + if cwd is None: + temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + cwd = temp_dir.name + try: + return self._run_repl( + repl_input, env=env, cmdline_args=cmdline_args, cwd=cwd + ) + finally: + if temp_dir is not None: + temp_dir.cleanup() + + def _run_repl( + self, + repl_input: str | list[str], + *, + env: dict | None, + cmdline_args: list[str] | None, + cwd: str, + ) -> tuple[str, int]: + assert pty + master_fd, slave_fd = pty.openpty() + cmd = [sys.executable, "-i", "-u"] + if env is None: + cmd.append("-I") + elif "PYTHON_HISTORY" not in env: + env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history") + if cmdline_args is not None: + cmd.extend(cmdline_args) + + try: + import termios + except ModuleNotFoundError: + pass + else: + term_attr = termios.tcgetattr(slave_fd) + term_attr[6][termios.VREPRINT] = 0 # pass through CTRL-R + term_attr[6][termios.VINTR] = 0 # pass through CTRL-C + termios.tcsetattr(slave_fd, termios.TCSANOW, term_attr) + + process = subprocess.Popen( + cmd, + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + cwd=cwd, + text=True, + close_fds=True, + env=env if env else os.environ, + ) + os.close(slave_fd) + if isinstance(repl_input, list): + repl_input = "\n".join(repl_input) + "\n" + os.write(master_fd, repl_input.encode("utf-8")) + + output = [] + while select.select([master_fd], [], [], SHORT_TIMEOUT)[0]: + try: + data = os.read(master_fd, 1024).decode("utf-8") + if not data: + break + except OSError: + break + output.append(data) + else: + os.close(master_fd) + process.kill() + self.fail(f"Timeout while waiting for output, got: {''.join(output)}") + + os.close(master_fd) + try: + exit_code = process.wait(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + exit_code = process.wait() + return "".join(output), exit_code + + +class TestCursorPosition(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_up_arrow_simple(self): + # fmt: off + code = ( + "def f():\n" + " ...\n" + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 1)) + console.move_cursor.assert_called_once_with(0, 1) + + def test_down_arrow_end_of_input(self): + # fmt: off + code = ( + "def f():\n" + " ...\n" + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 2)) + console.move_cursor.assert_called_once_with(0, 2) + + def test_left_arrow_simple(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (4, 0)) + console.move_cursor.assert_called_once_with(4, 0) + + def test_right_arrow_end_of_line(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (5, 0)) + console.move_cursor.assert_called_once_with(5, 0) + + def test_cursor_position_simple_character(self): + events = itertools.chain(code_to_events("k")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 1 for simple character + self.assertEqual(reader.cxy, (1, 0)) + + def test_cursor_position_double_width_character(self): + events = itertools.chain(code_to_events("樂")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_character_move_left(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 0) + self.assertEqual(reader.cxy, (0, 0)) + + def test_cursor_position_double_width_character_move_left_right(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_characters_move_up(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor at end of first line + self.assertEqual(reader.pos, len(for_loop)) + self.assertEqual(reader.cxy, (len(for_loop), 0)) + + def test_cursor_position_double_width_characters_move_up_down(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor here (showing 2nd line only): + # < ' 可口可乐; 可口可樂'> + # ^ + self.assertEqual(reader.pos, 19) + self.assertEqual(reader.cxy, (10, 1)) + + def test_cursor_position_multiple_double_width_characters_move_left(self): + events = itertools.chain( + code_to_events("' 可口可乐; 可口可樂'"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 10) + + # 1 for quote, 1 for space, 2 per wide character, + # 1 for semicolon, 1 for space, 2 per wide character + self.assertEqual(reader.cxy, (16, 0)) + + def test_cursor_position_move_up_to_eol(self): + first_line = "for _ in _:" + second_line = " hello" + + # fmt: off + code = ( + f"{first_line}\n" + f"{second_line}\n" + " h\n" + " hel" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 1, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual( + reader.pos, len(first_line) + len(second_line) + 1 + ) # +1 for newline + self.assertEqual(reader.cxy, (len(second_line), 1)) + + def test_cursor_position_move_down_to_eol(self): + last_line = " hel" + + # fmt: off + code = ( + "for _ in _:\n" + " hello\n" + " h\n" + f"{last_line}" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 3, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual(reader.pos, len(code)) + self.assertEqual(reader.cxy, (len(last_line), 3)) + + def test_cursor_position_multiple_mixed_lines_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " x = '可口可乐; 可口可樂'\n" + " y = 'abckdfjskldfjslkdjf'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + + reader, _ = handle_all_events(events) + + # By moving left, we're before the s: + # y = 'abckdfjskldfjslkdjf' + # ^ + # And we should move before the semi-colon despite the different offset + # x = '可口可乐; 可口可樂' + # ^ + self.assertEqual(reader.pos, 22) + self.assertEqual(reader.cxy, (15, 1)) + + def test_cursor_position_after_wrap_and_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + reader, _ = handle_events_narrow_console(events) + + # The code looks like this: + # def foo()\ + # : + # hello + # After moving up we should be after the colon in line 2 + self.assertEqual(reader.pos, 10) + self.assertEqual(reader.cxy, (1, 1)) + + +class TestPyReplAutoindent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_auto_indent_default(self): + # fmt: off + input_code = ( + "def f():\n" + "pass\n\n" + ) + + output_code = ( + "def f():\n" + " pass\n" + " " + ) + # fmt: on + + def test_auto_indent_continuation(self): + # auto indenting according to previous user indentation + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_prev_block(self): + # auto indenting according to indentation in different block + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + code_to_events( + "def g():\n" + "pass\n\n" + ), + ) + + output_code = ( + "def g():\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output1 = multiline_input(reader) + output2 = multiline_input(reader) + self.assertEqual(output2, output_code) + + def test_auto_indent_multiline(self): + # fmt: off + events = itertools.chain( + code_to_events( + "def f():\n" + "pass" + ), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new line should be autoindented + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + code_to_events( + "pass" + ), + [ + # go to end of last line + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # double newline to terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_with_comment(self): + # fmt: off + events = code_to_events( + "def f(): # foo\n" + "pass\n\n" + ) + + output_code = ( + "def f(): # foo\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_with_multicomment(self): + # fmt: off + events = code_to_events( + "def f(): ## foo\n" + "pass\n\n" + ) + + output_code = ( + "def f(): ## foo\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_ignore_comments(self): + # fmt: off + events = code_to_events( + "pass #:\n" + ) + + output_code = ( + "pass #:" + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + +class TestPyReplOutput(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + reader.can_colorize = False + return reader + + def test_stdin_is_tty(self): + # Used during test log analysis to figure out if a TTY was available. + try: + if os.isatty(sys.stdin.fileno()): + return + except OSError as ose: + self.skipTest(f"stdin tty check failed: {ose}") + else: + self.skipTest("stdin is not a tty") + + def test_stdout_is_tty(self): + # Used during test log analysis to figure out if a TTY was available. + try: + if os.isatty(sys.stdout.fileno()): + return + except OSError as ose: + self.skipTest(f"stdout tty check failed: {ose}") + else: + self.skipTest("stdout is not a tty") + + def test_basic(self): + reader = self.prepare_reader(code_to_events("1+1\n")) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_get_line_buffer_returns_str(self): + reader = self.prepare_reader(code_to_events("\n")) + wrapper = _ReadlineWrapper(f_in=None, f_out=None, reader=reader) + self.assertIs(type(wrapper.get_line_buffer()), str) + + def test_multiline_edit(self): + events = itertools.chain( + code_to_events("def f():\n...\n\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="delete", raw=bytearray(b"\x7F")), + Event(evt="key", data="right", raw=bytearray(b"g")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="p", raw=bytearray(b"p")), + Event(evt="key", data="a", raw=bytearray(b"a")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "def f():\n ...\n ") + self.assertEqual(clean_screen(reader.screen), "def f():\n ...") + output = multiline_input(reader) + self.assertEqual(output, "def g():\n pass\n ") + self.assertEqual(clean_screen(reader.screen), "def g():\n pass") + + def test_history_navigation_with_up_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_history_with_multiline_entries(self): + code = "def foo():\nx = 1\ny = 2\nz = 3\n\ndef bar():\nreturn 42\n\n" + events = list(itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + )) + + reader = self.prepare_reader(events) + output = multiline_input(reader) + output = multiline_input(reader) + output = multiline_input(reader) + self.assertEqual( + clean_screen(reader.screen), + 'def foo():\n x = 1\n y = 2\n z = 3' + ) + self.assertEqual(output, "def foo():\n x = 1\n y = 2\n z = 3\n ") + + + def test_history_navigation_with_down_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_history_search(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n3+3\n"), + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") + output = multiline_input(reader) + self.assertEqual(output, "3+3") + self.assertEqual(clean_screen(reader.screen), "3+3") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_control_character(self): + events = code_to_events("c\x1d\n") + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, "c\x1d") + self.assertEqual(clean_screen(reader.screen), "c") + + def test_history_search_backward(self): + # Test history search backward with "imp" input + events = itertools.chain( + code_to_events("import os\n"), + code_to_events("imp"), + [ + Event(evt='key', data='page up', raw=bytearray(b'\x1b[5~')), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + # fill the history + reader = self.prepare_reader(events) + multiline_input(reader) + + # search for "imp" in history + output = multiline_input(reader) + self.assertEqual(output, "import os") + self.assertEqual(clean_screen(reader.screen), "import os") + + def test_history_search_backward_empty(self): + # Test history search backward with an empty input + events = itertools.chain( + code_to_events("import os\n"), + [ + Event(evt='key', data='page up', raw=bytearray(b'\x1b[5~')), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + # fill the history + reader = self.prepare_reader(events) + multiline_input(reader) + + # search backward in history + output = multiline_input(reader) + self.assertEqual(output, "import os") + self.assertEqual(clean_screen(reader.screen), "import os") + + +class TestPyReplCompleter(TestCase): + def prepare_reader(self, events, namespace): + console = FakeConsole(events) + config = ReadlineConfig() + config.readline_completer = rlcompleter.Completer(namespace).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + @patch("rlcompleter._readline_available", False) + def test_simple_completion(self): + events = code_to_events("os.getpid\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getpid()") + + def test_completion_with_many_options(self): + # Test with something that initially displays many options + # and then complete from one of them. The first time tab is + # pressed, the options are displayed (which corresponds to + # when the repl shows [ not unique ]) and the second completes + # from one of them. + events = code_to_events("os.\t\tO_AP\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.O_APPEND") + + def test_empty_namespace_completion(self): + events = code_to_events("os.geten\t\n") + namespace = {} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.geten") + + def test_global_namespace_completion(self): + events = code_to_events("py\t\n") + namespace = {"python": None} + reader = self.prepare_reader(events, namespace) + output = multiline_input(reader, namespace) + self.assertEqual(output, "python") + + def test_updown_arrow_with_completion_menu(self): + """Up arrow in the middle of unfinished tab completion when the menu is displayed + should work and trigger going back in history. Down arrow should subsequently + get us back to the incomplete command.""" + code = "import os\nos.\t\t" + namespace = {"os": os} + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events, namespace=namespace) + output = multiline_input(reader, namespace) + # This is the first line, nothing to see here + self.assertEqual(output, "import os") + # This is the second line. We pressed up and down arrows + # so we should end up where we were when we initiated tab completion. + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.") + + @patch("_pyrepl.readline._ReadlineWrapper.get_reader") + @patch("sys.stderr", new_callable=io.StringIO) + def test_completion_with_warnings(self, mock_stderr, mock_get_reader): + class Dummy: + @property + def test_func(self): + import warnings + + warnings.warn("warnings\n") + return None + + dummy = Dummy() + events = code_to_events("dummy.test_func.\t\n\n") + namespace = {"dummy": dummy} + reader = self.prepare_reader(events, namespace) + mock_get_reader.return_value = reader + output = readline_multiline_input(more_lines, ">>>", "...") + self.assertEqual(output, "dummy.test_func.__") + self.assertEqual(mock_stderr.getvalue(), "") + + +class TestPasteEvent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_paste(self): + # fmt: off + code = ( + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:\n" + " pass\n" + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines(self): + # fmt: off + code = ( + "def f():\n" + " x = y\n" + " \n" + " y = z\n" + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines_not_in_paste_mode(self): + # fmt: off + code = ( + "def f():\n" + "x = y\n" + "\n" + "y = z\n\n" + ) + + expected = ( + "def f():\n" + " x = y\n" + " " + ) + # fmt: on + + events = code_to_events(code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, expected) + + def test_paste_not_in_paste_mode(self): + # fmt: off + input_code = ( + "def a():\n" + "for x in range(10):\n" + "if x%2:\n" + "print(x)\n" + "else:\n" + "pass\n\n" + ) + + output_code = ( + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:" + ) + # fmt: on + + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste(self): + """Test that bracketed paste using \x1b[200~ and \x1b[201~ works.""" + # fmt: off + input_code = ( + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" + ) + + output_code = ( + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" + ) + # fmt: on + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste_single_line(self): + input_code = "oneline" + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, input_code) + + +@skipUnless(pty, "requires pty") +class TestDumbTerminal(ReplTestCase): + def test_dumb_terminal_exits_cleanly(self): + env = os.environ.copy() + env.update({"TERM": "dumb"}) + output, exit_code = self.run_repl("exit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("warning: can't use pyrepl", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + +@skipUnless(pty, "requires pty") +@skipIf((os.environ.get("TERM") or "dumb") == "dumb", "can't use pyrepl in dumb terminal") +class TestMain(ReplTestCase): + def setUp(self): + # Cleanup from PYTHON* variables to isolate from local + # user settings, see #121359. Such variables should be + # added later in test methods to patched os.environ. + patcher = patch('os.environ', new=make_clean_env()) + self.addCleanup(patcher.stop) + patcher.start() + + @force_not_colorized + def test_exposed_globals_in_repl(self): + pre = "['__builtins__'" + post = "'__loader__', '__name__', '__package__', '__spec__']" + output, exit_code = self.run_repl(["sorted(dir())", "exit()"]) + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertEqual(exit_code, 0) + + # if `__main__` is not a file (impossible with pyrepl) + case1 = f"{pre}, '__doc__', {post}" in output + + # if `__main__` is an uncached .py file (no .pyc) + case2 = f"{pre}, '__doc__', '__file__', {post}" in output + + # if `__main__` is a cached .pyc file and the .py source exists + case3 = f"{pre}, '__cached__', '__doc__', '__file__', {post}" in output + + # if `__main__` is a cached .pyc file but there's no .py source file + case4 = f"{pre}, '__cached__', '__doc__', {post}" in output + + self.assertTrue(case1 or case2 or case3 or case4, output) + + def _assertMatchOK( + self, var: str, expected: str | re.Pattern, actual: str + ) -> None: + if isinstance(expected, re.Pattern): + self.assertTrue( + expected.match(actual), + f"{var}={actual} does not match {expected.pattern}", + ) + else: + self.assertEqual( + actual, + expected, + f"expected {var}={expected}, got {var}={actual}", + ) + + @force_not_colorized + def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False): + clean_env = make_clean_env() + clean_env["NO_COLOR"] = "1" # force_not_colorized doesn't touch subprocesses + + with tempfile.TemporaryDirectory() as td: + blue = pathlib.Path(td) / "blue" + blue.mkdir() + mod = blue / "calx.py" + mod.write_text("FOO = 42", encoding="utf-8") + commands = [ + "print(f'^{" + var + "=}')" for var in expectations + ] + ["exit()"] + if as_file and as_module: + self.fail("as_file and as_module are mutually exclusive") + elif as_file: + output, exit_code = self.run_repl( + commands, + cmdline_args=[str(mod)], + env=clean_env, + ) + elif as_module: + output, exit_code = self.run_repl( + commands, + cmdline_args=["-m", "blue.calx"], + env=clean_env, + cwd=td, + ) + else: + self.fail("Choose one of as_file or as_module") + + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + + self.assertEqual(exit_code, 0) + for var, expected in expectations.items(): + with self.subTest(var=var, expected=expected): + if m := re.search(rf"\^{var}=(.+?)[\r\n]", output): + self._assertMatchOK(var, expected, actual=m.group(1)) + else: + self.fail(f"{var}= not found in output: {output!r}\n\n{output}") + + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + def test_inspect_keeps_globals_from_inspected_file(self): + expectations = { + "FOO": "42", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, as_file=True) + + def test_inspect_keeps_globals_from_inspected_module(self): + expectations = { + "FOO": "42", + "__name__": "'__main__'", + "__package__": "'blue'", + "__file__": re.compile(r"^'.*calx.py'$"), + } + self._run_repl_globals_test(expectations, as_module=True) + + @force_not_colorized + def test_python_basic_repl(self): + env = os.environ.copy() + commands = ("from test.support import initialized_with_pyrepl\n" + "initialized_with_pyrepl()\n" + "exit()\n") + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env) + if "can\'t use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertEqual(exit_code, 0) + self.assertIn("True", output) + self.assertNotIn("False", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("False", output) + self.assertNotIn("True", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + # The site module must not load _pyrepl if PYTHON_BASIC_REPL is set + commands = ("import sys\n" + "print('_pyrepl' in sys.modules)\n" + "exit()\n") + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("False", output) + self.assertNotIn("True", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + @force_not_colorized + def test_bad_sys_excepthook_doesnt_crash_pyrepl(self): + env = os.environ.copy() + commands = ("import sys\n" + "sys.excepthook = 1\n" + "1/0\n" + "exit()\n") + + def check(output, exitcode): + self.assertIn("Error in sys.excepthook:", output) + self.assertEqual(output.count("'int' object is not callable"), 1) + self.assertIn("Original exception was:", output) + self.assertIn("division by zero", output) + self.assertEqual(exitcode, 0) + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env) + if "can\'t use pyrepl" in output: + self.skipTest("pyrepl not available") + check(output, exit_code) + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + check(output, exit_code) + + def test_not_wiping_history_file(self): + # skip, if readline module is not available + import_module('readline') + + hfile = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(unlink, hfile.name) + env = os.environ.copy() + env["PYTHON_HISTORY"] = hfile.name + commands = "123\nspam\nexit()\n" + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("123", output) + self.assertIn("spam", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + + hfile.file.truncate() + hfile.close() + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("123", output) + self.assertIn("spam", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + + @force_not_colorized + def test_correct_filename_in_syntaxerrors(self): + env = os.environ.copy() + commands = "a b c\nexit()\n" + output, exit_code = self.run_repl(commands, env=env) + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertIn("SyntaxError: invalid syntax", output) + self.assertIn("", output) + commands = " b\nexit()\n" + output, exit_code = self.run_repl(commands, env=env) + self.assertIn("IndentationError: unexpected indent", output) + self.assertIn("", output) + + @force_not_colorized + def test_proper_tracebacklimit(self): + env = os.environ.copy() + for set_tracebacklimit in [True, False]: + commands = ("import sys\n" + + ("sys.tracebacklimit = 1\n" if set_tracebacklimit else "") + + "def x1(): 1/0\n\n" + "def x2(): x1()\n\n" + "def x3(): x2()\n\n" + "x3()\n" + "exit()\n") + + for basic_repl in [True, False]: + if basic_repl: + env["PYTHON_BASIC_REPL"] = "1" + else: + env.pop("PYTHON_BASIC_REPL", None) + with self.subTest(set_tracebacklimit=set_tracebacklimit, + basic_repl=basic_repl): + output, exit_code = self.run_repl(commands, env=env) + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertIn("in x1", output) + if set_tracebacklimit: + self.assertNotIn("in x2", output) + self.assertNotIn("in x3", output) + self.assertNotIn("in ", output) + else: + self.assertIn("in x2", output) + self.assertIn("in x3", output) + self.assertIn("in ", output) + + def test_null_byte(self): + output, exit_code = self.run_repl("\x00\nexit()\n") + self.assertEqual(exit_code, 0) + self.assertNotIn("TypeError", output) + + def test_readline_history_file(self): + # skip, if readline module is not available + readline = import_module('readline') + if readline.backend != "editline": + self.skipTest("GNU readline is not affected by this issue") + + hfile = tempfile.NamedTemporaryFile() + self.addCleanup(unlink, hfile.name) + env = os.environ.copy() + env["PYTHON_HISTORY"] = hfile.name + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl("spam \nexit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("spam ", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + self.assertIn("spam\\040", pathlib.Path(hfile.name).read_text()) + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl("exit\n", env=env) + self.assertEqual(exit_code, 0) + self.assertNotIn("\\040", pathlib.Path(hfile.name).read_text()) + + def test_keyboard_interrupt_after_isearch(self): + output, exit_code = self.run_repl(["\x12", "\x03", "exit"]) + self.assertEqual(exit_code, 0) diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py new file mode 100644 index 00000000000000..6c72a1d39c55df --- /dev/null +++ b/Lib/test/test_pyrepl/test_reader.py @@ -0,0 +1,314 @@ +import itertools +import functools +import rlcompleter +from unittest import TestCase +from unittest.mock import MagicMock + +from .support import handle_all_events, handle_events_narrow_console, code_to_events, prepare_reader +from _pyrepl.console import Event +from _pyrepl.reader import Reader + + +class TestReader(TestCase): + def assert_screen_equals(self, reader, expected): + actual = reader.screen + expected = expected.split("\n") + self.assertListEqual(actual, expected) + + def test_calc_screen_wrap_simple(self): + events = code_to_events(10 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_calc_screen_wrap_wide_characters(self): + events = code_to_events(8 * "a" + "樂") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") + + def test_calc_screen_wrap_three_lines(self): + events = code_to_events(20 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") + + def test_calc_screen_prompt_handling(self): + def prepare_reader_keep_prompts(*args, **kwargs): + reader = prepare_reader(*args, **kwargs) + del reader.get_prompt + reader.ps1 = ">>> " + reader.ps2 = ">>> " + reader.ps3 = "... " + reader.ps4 = "" + reader.can_colorize = False + reader.paste_mode = False + return reader + + events = code_to_events("if some_condition:\nsome_function()") + reader, _ = handle_events_narrow_console( + events, + prepare_reader=prepare_reader_keep_prompts, + ) + # fmt: off + self.assert_screen_equals( + reader, + ( + ">>> if so\\\n" + "me_condit\\\n" + "ion:\n" + "... s\\\n" + "ome_funct\\\n" + "ion()" + ) + ) + # fmt: on + + def test_calc_screen_wrap_three_lines_mixed_character(self): + # fmt: off + code = ( + "def f():\n" + f" {8*"a"}\n" + f" {5*"樂"}" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + + # fmt: off + self.assert_screen_equals(reader, ( + "def f():\n" + f" {7*"a"}\\\n" + "a\n" + f" {3*"樂"}\\\n" + "樂樂" + )) + # fmt: on + + def test_calc_screen_backspace(self): + events = itertools.chain( + code_to_events("aaa"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "aa") + + def test_calc_screen_wrap_removes_after_backspace(self): + events = itertools.chain( + code_to_events(10 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, 9 * "a") + + def test_calc_screen_backspace_in_second_line_after_wrap(self): + events = itertools.chain( + code_to_events(11 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_setpos_for_xy_simple(self): + events = code_to_events("11+11") + reader, _ = handle_all_events(events) + reader.setpos_from_xy(0, 0) + self.assertEqual(reader.pos, 0) + + def test_control_characters(self): + code = 'flag = "🏳️‍🌈"' + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, 'flag = "🏳️\\u200d🌈"') + + def test_setpos_from_xy_multiple_lines(self): + # fmt: off + code = ( + "def foo():\n" + " return 1" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_all_events(events) + reader.setpos_from_xy(2, 1) + self.assertEqual(reader.pos, 13) + + def test_setpos_from_xy_after_wrap(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(2, 2) + self.assertEqual(reader.pos, 13) + + def test_setpos_fromxy_in_wrapped_line(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(0, 1) + self.assertEqual(reader.pos, 9) + + def test_up_arrow_after_ctrl_r(self): + events = iter( + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ] + ) + + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "") + + def test_newline_within_block_trailing_whitespace(self): + # fmt: off + code = ( + "def foo():\n" + "a = 1\n" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new lines in-block shouldn't terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + # end of line 2 + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # a double new line in-block should terminate the block + # even if its followed by whitespace + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + no_paste_reader = functools.partial(prepare_reader, paste_mode=False) + reader, _ = handle_all_events(events, prepare_reader=no_paste_reader) + + expected = ( + "def foo():\n" + " \n" + " \n" + " a = 1\n" + " \n" + " " # HistoricalReader will trim trailing whitespace + ) + self.assert_screen_equals(reader, expected) + self.assertTrue(reader.finished) + + def test_input_hook_is_called_if_set(self): + input_hook = MagicMock() + def _prepare_console(events): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + console.input_hook = input_hook + return console + + events = code_to_events("a") + reader, _ = handle_all_events(events, prepare_console=_prepare_console) + + self.assertEqual(len(input_hook.mock_calls), 4) + + def test_keyboard_interrupt_clears_screen(self): + namespace = {"itertools": itertools} + code = "import itertools\nitertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), # Two tabs for completion + Event(evt='key', data='\t', raw=bytearray(b'\t')), + Event(evt='key', data='\x03', raw=bytearray(b'\x03')), # Ctrl-C + ]) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + self.assertEqual(reader.calc_screen(), code.split("\n")) + + def test_prompt_length(self): + # Handles simple ASCII prompt + ps1 = ">>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 4) + + # Handles ANSI escape sequences + ps1 = "\033[0;32m>>> \033[0m" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles ANSI escape sequences bracketed in \001 .. \002 + ps1 = "\001\033[0;32m\002>>> \001\033[0m\002" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles wide characters in prompt + ps1 = "樂>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 5) + + # Handles wide characters AND ANSI sequences together + ps1 = "\001\033[0;32m\002樂>\001\033[0m\002> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m樂>\033[0m> ") + self.assertEqual(l, 5) + + def test_completions_updated_on_key_press(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), # Two tabs for completion + Event(evt='key', data='\t', raw=bytearray(b'\t')), + ], code_to_events("a")) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + actual = reader.screen + self.assertEqual(len(actual), 2) + self.assertEqual(actual[0].rstrip(), "itertools.accumulate(") + self.assertEqual(actual[1], f"{code}a") + + def test_key_press_on_tab_press_once(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), + ], code_to_events("a")) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + self.assert_screen_equals(reader, f"{code}a") diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py new file mode 100644 index 00000000000000..e3bbabcb0089fb --- /dev/null +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -0,0 +1,314 @@ +import itertools +import sys +import unittest +from functools import partial +from unittest import TestCase +from unittest.mock import MagicMock, call, patch, ANY + +from .support import handle_all_events, code_to_events + +try: + from _pyrepl.console import Event + from _pyrepl.unix_console import UnixConsole +except ImportError: + pass + + +def unix_console(events, **kwargs): + console = UnixConsole() + console.get_event = MagicMock(side_effect=events) + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +handle_events_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console), +) +handle_events_narrow_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, width=5), +) +handle_events_short_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, height=1), +) +handle_events_unix_console_height_3 = partial( + handle_all_events, prepare_console=partial(unix_console, height=3) +) + + +TERM_CAPABILITIES = { + "bel": b"\x07", + "civis": b"\x1b[?25l", + "clear": b"\x1b[H\x1b[2J", + "cnorm": b"\x1b[?12l\x1b[?25h", + "cub": b"\x1b[%p1%dD", + "cub1": b"\x08", + "cud": b"\x1b[%p1%dB", + "cud1": b"\n", + "cuf": b"\x1b[%p1%dC", + "cuf1": b"\x1b[C", + "cup": b"\x1b[%i%p1%d;%p2%dH", + "cuu": b"\x1b[%p1%dA", + "cuu1": b"\x1b[A", + "dch1": b"\x1b[P", + "dch": b"\x1b[%p1%dP", + "el": b"\x1b[K", + "hpa": b"\x1b[%i%p1%dG", + "ich": b"\x1b[%p1%d@", + "ich1": None, + "ind": b"\n", + "pad": None, + "ri": b"\x1bM", + "rmkx": b"\x1b[?1l\x1b>", + "smkx": b"\x1b[?1h\x1b=", +} + + +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") +@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) +@patch( + "_pyrepl.curses.tparm", + lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), +) +@patch("_pyrepl.curses.setupterm", lambda a, b: None) +@patch( + "termios.tcgetattr", + lambda _: [ + 27394, + 3, + 19200, + 536872399, + 38400, + 38400, + [ + b"\x04", + b"\xff", + b"\xff", + b"\x7f", + b"\x17", + b"\x15", + b"\x12", + b"\x00", + b"\x03", + b"\x1c", + b"\x1a", + b"\x19", + b"\x11", + b"\x13", + b"\x16", + b"\x0f", + b"\x01", + b"\x00", + b"\x14", + b"\x00", + ], + ], +) +@patch("termios.tcsetattr", lambda a, b, c: None) +@patch("os.write") +class TestConsole(TestCase): + def test_simple_addition(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"4") + con.restore() + + def test_wrap(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, con = handle_events_narrow_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"\\") + _os_write.assert_any_call(ANY, b"\n") + _os_write.assert_any_call(ANY, b"4") + con.restore() + + def test_cursor_left(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + con.restore() + + def test_cursor_left_right(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuf"] + b":1") + con.restore() + + def test_cursor_up(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + con.restore() + + def test_cursor_up_down(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cud"] + b":1") + con.restore() + + def test_cursor_back_write(self, _os_write): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, b"2") + con.restore() + + def test_multiline_function_move_up_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, con = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + con.restore() + + def test_multiline_function_move_up_down_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, con = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ind"] + b":") + con.restore() + + def test_resize_bigger_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_short_unix_console(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ri"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b"def f():"), + ] + ) + console.restore() + con.restore() + + def test_resize_smaller_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_unix_console_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ind"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b" foo"), + ] + ) + console.restore() + con.restore() diff --git a/Lib/test/test_pyrepl/test_unix_eventqueue.py b/Lib/test/test_pyrepl/test_unix_eventqueue.py new file mode 100644 index 00000000000000..301f79927a741f --- /dev/null +++ b/Lib/test/test_pyrepl/test_unix_eventqueue.py @@ -0,0 +1,117 @@ +import tempfile +import unittest +import sys +from unittest.mock import patch + +try: + from _pyrepl.console import Event + from _pyrepl.unix_eventqueue import EventQueue +except ImportError: + pass + +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") +@patch("_pyrepl.curses.tigetstr", lambda x: b"") +class TestUnixEventQueue(unittest.TestCase): + def setUp(self): + self.file = tempfile.TemporaryFile() + + def tearDown(self) -> None: + self.file.close() + + def test_get(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.get(), event) + + def test_empty(self): + eq = EventQueue(self.file.fileno(), "utf-8") + self.assertTrue(eq.empty()) + eq.insert(Event("key", "a", b"a")) + self.assertFalse(eq.empty()) + + def test_flush_buf(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.buf.extend(b"test") + self.assertEqual(eq.flush_buf(), b"test") + self.assertEqual(eq.buf, bytearray()) + + def test_insert(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.events[0], event) + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": "b"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "b") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_without_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"c": "d"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "a") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.push("b") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "c") + eq.push("d") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "d") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.flush_buf() + eq.push("\033") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\033") + eq.push("b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "b") + + def test_push_special_key(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("A") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + + def test_push_unrecognized_escape_sequence(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("Z") + self.assertEqual(len(eq.events), 3) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "[") + self.assertEqual(eq.events[2].evt, "key") + self.assertEqual(eq.events[2].data, "Z") diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py new file mode 100644 index 00000000000000..4a3b2baf64a944 --- /dev/null +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -0,0 +1,334 @@ +import sys +import unittest + +if sys.platform != "win32": + raise unittest.SkipTest("test only relevant on win32") + + +import itertools +from functools import partial +from typing import Iterable +from unittest import TestCase +from unittest.mock import MagicMock, call + +from .support import handle_all_events, code_to_events + +try: + from _pyrepl.console import Event, Console + from _pyrepl.windows_console import ( + WindowsConsole, + MOVE_LEFT, + MOVE_RIGHT, + MOVE_UP, + MOVE_DOWN, + ERASE_IN_LINE, + ) +except ImportError: + pass + + +class WindowsConsoleTests(TestCase): + def console(self, events, **kwargs) -> Console: + console = WindowsConsole() + console.get_event = MagicMock(side_effect=events) + console._scroll = MagicMock() + console._hide_cursor = MagicMock() + console._show_cursor = MagicMock() + console._getscrollbacksize = MagicMock(42) + console.out = MagicMock() + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + def handle_events(self, events: Iterable[Event], **kwargs): + return handle_all_events(events, partial(self.console, **kwargs)) + + def handle_events_narrow(self, events): + return self.handle_events(events, width=5) + + def handle_events_short(self, events): + return self.handle_events(events, height=1) + + def handle_events_height_3(self, events): + return self.handle_events(events, height=3) + + def test_simple_addition(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_wrap(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events_narrow(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"\\") + con.out.write.assert_any_call(b"\n") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_resize_wider(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events_narrow(events) + + console.height = 20 + console.width = 80 + console.getheightwidth = MagicMock(lambda _: (20, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(self.move_right(2)) + con.out.write.assert_any_call(self.move_up(2)) + con.out.write.assert_any_call(b"567890") + + con.restore() + + def test_resize_narrower(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events(events) + + console.height = 20 + console.width = 4 + console.getheightwidth = MagicMock(lambda _: (20, 4)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(b"456\\") + con.out.write.assert_any_call(b"789\\") + + con.restore() + + def test_cursor_left(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.restore() + + def test_cursor_left_right(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(self.move_right()) + con.restore() + + def test_cursor_up(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_cursor_up_down(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.out.write.assert_any_call(self.move_down()) + con.restore() + + def test_cursor_back_write(self): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(b"21") + con.restore() + + def test_multiline_function_move_up_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(5)) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_multiline_function_move_up_down_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(8)) + con.out.write.assert_any_call(self.erase_in_line()) + con.restore() + + def test_resize_bigger_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_short(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(b"def f():"), + call(self.move_left(3)), + call(self.move_down()), + ] + ) + console.restore() + con.restore() + + def test_resize_smaller_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(self.erase_in_line()), + call(b" foo"), + ] + ) + console.restore() + con.restore() + + def move_up(self, lines=1): + return MOVE_UP.format(lines).encode("utf8") + + def move_down(self, lines=1): + return MOVE_DOWN.format(lines).encode("utf8") + + def move_left(self, cols=1): + return MOVE_LEFT.format(cols).encode("utf8") + + def move_right(self, cols=1): + return MOVE_RIGHT.format(cols).encode("utf8") + + def erase_in_line(self): + return ERASE_IN_LINE.encode("utf8") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index e3d4d566cdda48..7f4fe357034b71 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -316,92 +316,104 @@ def test_shutdown_all_methods_in_one_thread(self): def test_shutdown_immediate_all_methods_in_one_thread(self): return self._shutdown_all_methods_in_one_thread(True) - def _write_msg_thread(self, q, n, results, delay, - i_when_exec_shutdown, - event_start, event_end): - event_start.wait() - for i in range(1, n+1): + def _write_msg_thread(self, q, n, results, + i_when_exec_shutdown, event_shutdown, + barrier_start): + # All `write_msg_threads` + # put several items into the queue. + for i in range(0, i_when_exec_shutdown//2): + q.put((i, 'LOYD')) + # Wait for the barrier to be complete. + barrier_start.wait() + + for i in range(i_when_exec_shutdown//2, n): try: q.put((i, "YDLO")) - results.append(True) except self.queue.ShutDown: results.append(False) - # triggers shutdown of queue - if i == i_when_exec_shutdown: - event_end.set() - time.sleep(delay) - # end of all puts - q.join() + break - def _read_msg_thread(self, q, nb, results, delay, event_start): - event_start.wait() - block = True - while nb: - time.sleep(delay) + # Trigger queue shutdown. + if i == i_when_exec_shutdown: + # Only one thread should call shutdown(). + if not event_shutdown.is_set(): + event_shutdown.set() + results.append(True) + + def _read_msg_thread(self, q, results, barrier_start): + # Get at least one item. + q.get(True) + q.task_done() + # Wait for the barrier to be complete. + barrier_start.wait() + while True: try: - # Get at least one message - q.get(block) - block = False + q.get(False) q.task_done() - results.append(True) - nb -= 1 except self.queue.ShutDown: - results.append(False) - nb -= 1 + results.append(True) + break except self.queue.Empty: pass - q.join() - def _shutdown_thread(self, q, event_end, immediate): + def _shutdown_thread(self, q, results, event_end, immediate): event_end.wait() q.shutdown(immediate) - q.join() + results.append(q.qsize() == 0) - def _join_thread(self, q, delay, event_start): - event_start.wait() - time.sleep(delay) + def _join_thread(self, q, barrier_start): + # Wait for the barrier to be complete. + barrier_start.wait() q.join() def _shutdown_all_methods_in_many_threads(self, immediate): + # Run a 'multi-producers/consumers queue' use case, + # with enough items into the queue. + # When shutdown, all running threads will be joined. q = self.type2test() ps = [] - ev_start = threading.Event() - ev_exec_shutdown = threading.Event() res_puts = [] res_gets = [] - delay = 1e-4 - read_process = 4 - nb_msgs = read_process * 16 - nb_msgs_r = nb_msgs // read_process - when_exec_shutdown = nb_msgs // 2 - lprocs = ( - (self._write_msg_thread, 1, (q, nb_msgs, res_puts, delay, - when_exec_shutdown, - ev_start, ev_exec_shutdown)), - (self._read_msg_thread, read_process, (q, nb_msgs_r, - res_gets, delay*2, - ev_start)), - (self._join_thread, 2, (q, delay*2, ev_start)), - (self._shutdown_thread, 1, (q, ev_exec_shutdown, immediate)), - ) - # start all threds + res_shutdown = [] + write_threads = 4 + read_threads = 6 + join_threads = 2 + nb_msgs = 1024*64 + nb_msgs_w = nb_msgs // write_threads + when_exec_shutdown = nb_msgs_w // 2 + # Use of a Barrier to ensure that + # - all write threads put all their items into the queue, + # - all read thread get at least one item from the queue, + # and keep on running until shutdown. + # The join thread is started only when shutdown is immediate. + nparties = write_threads + read_threads + if immediate: + nparties += join_threads + barrier_start = threading.Barrier(nparties) + ev_exec_shutdown = threading.Event() + lprocs = [ + (self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts, + when_exec_shutdown, ev_exec_shutdown, + barrier_start)), + (self._read_msg_thread, read_threads, (q, res_gets, barrier_start)), + (self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)), + ] + if immediate: + lprocs.append((self._join_thread, join_threads, (q, barrier_start))) + # start all threads. for func, n, args in lprocs: for i in range(n): ps.append(threading.Thread(target=func, args=args)) ps[-1].start() - # set event in order to run q.shutdown() - ev_start.set() - - if not immediate: - assert(len(res_gets) == len(res_puts)) - assert(res_gets.count(True) == res_puts.count(True)) - else: - assert(len(res_gets) <= len(res_puts)) - assert(res_gets.count(True) <= res_puts.count(True)) - - for thread in ps[1:]: + for thread in ps: thread.join() + self.assertTrue(True in res_puts) + self.assertEqual(res_gets.count(True), read_threads) + if immediate: + self.assertListEqual(res_shutdown, [True]) + self.assertTrue(q.empty()) + def test_shutdown_all_methods_in_many_threads(self): return self._shutdown_all_methods_in_many_threads(False) @@ -473,7 +485,7 @@ def _shutdown_get(self, immediate): else: thrds = ( # on shutdown(immediate=False) - # one of these threads shoud raise Shutdown + # one of these threads should raise Shutdown (self._get, (q, go, results)), (self._get, (q, go, results)), (self._get, (q, go, results)), @@ -564,7 +576,6 @@ def _shutdown_put_join(self, immediate): results = [] go = threading.Event() q.put("Y") - nb = q.qsize() # queue not fulled thrds = ( @@ -575,13 +586,19 @@ def _shutdown_put_join(self, immediate): for func, params in thrds: threads.append(threading.Thread(target=func, args=params)) threads[-1].start() - self.assertEqual(q.unfinished_tasks, nb) - for i in range(nb): - t = threading.Thread(target=q.task_done) - t.start() - threads.append(t) + self.assertEqual(q.unfinished_tasks, 1) + q.shutdown(immediate) go.set() + + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + else: + result = q.get() + self.assertEqual(result, "Y") + q.task_done() + for t in threads: t.join() @@ -618,6 +635,23 @@ def test_shutdown_get_task_done_join(self): self.assertEqual(results, [True]*len(thrds)) + def test_shutdown_pending_get(self): + def get(): + try: + results.append(q.get()) + except Exception as e: + results.append(e) + + q = self.type2test() + results = [] + get_thread = threading.Thread(target=get) + get_thread.start() + q.shutdown(immediate=False) + get_thread.join(timeout=10.0) + self.assertFalse(get_thread.is_alive()) + self.assertEqual(len(results), 1) + self.assertIsInstance(results[0], self.queue.ShutDown) + class QueueTest(BaseQueueTestMixin): diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 6d26a61bee4292..dcf0753bc828f3 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -48,7 +48,7 @@ def test_except_reraise(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: try: raise KeyError("caught") except KeyError: @@ -60,7 +60,7 @@ def test_finally_reraise(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: try: raise KeyError("caught") finally: @@ -73,7 +73,7 @@ def nested_reraise(): def reraise(): try: raise TypeError("foo") - except: + except TypeError: nested_reraise() self.assertRaises(TypeError, reraise) @@ -81,7 +81,7 @@ def test_raise_from_None(self): try: try: raise TypeError("foo") - except: + except TypeError: raise ValueError() from None except ValueError as e: self.assertIsInstance(e.__context__, TypeError) @@ -91,7 +91,7 @@ def test_with_reraise1(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: with Context(): pass raise @@ -101,7 +101,7 @@ def test_with_reraise2(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: with Context(): raise KeyError("caught") raise @@ -111,7 +111,7 @@ def test_yield_reraise(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: yield 1 raise g = reraise() @@ -314,7 +314,7 @@ def test_instance_context_instance_raise(self): try: try: raise context - except: + except IndexError: raise OSError() except OSError as e: self.assertIs(e.__context__, context) @@ -326,7 +326,7 @@ def test_class_context_instance_raise(self): try: try: raise context - except: + except IndexError: raise OSError() except OSError as e: self.assertIsNot(e.__context__, context) @@ -339,7 +339,7 @@ def test_class_context_class_raise(self): try: try: raise context - except: + except IndexError: raise OSError except OSError as e: self.assertIsNot(e.__context__, context) @@ -351,7 +351,7 @@ def test_c_exception_context(self): try: try: 1/0 - except: + except ZeroDivisionError: raise OSError except OSError as e: self.assertIsInstance(e.__context__, ZeroDivisionError) @@ -362,7 +362,7 @@ def test_c_exception_raise(self): try: try: 1/0 - except: + except ZeroDivisionError: xyzzy except NameError as e: self.assertIsInstance(e.__context__, ZeroDivisionError) @@ -459,7 +459,7 @@ def f(): try: try: raise ValueError - except: + except ValueError: del g raise KeyError except Exception as e: @@ -475,7 +475,7 @@ class C: def __del__(self): try: 1/0 - except: + except ZeroDivisionError: raise def f(): diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index b1e4ef4197d130..51f9193b269eee 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -4,6 +4,7 @@ import os import time import pickle +import shlex import warnings import test.support @@ -1397,5 +1398,47 @@ def test_after_fork(self): support.wait_process(pid, exitcode=0) +class CommandLineTest(unittest.TestCase): + def test_parse_args(self): + args, help_text = random._parse_args(shlex.split("--choice a b c")) + self.assertEqual(args.choice, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--integer 5")) + self.assertEqual(args.integer, 5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--float 2.5")) + self.assertEqual(args.float, 2.5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("a b c")) + self.assertEqual(args.input, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("5")) + self.assertEqual(args.input, ["5"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("2.5")) + self.assertEqual(args.input, ["2.5"]) + self.assertTrue(help_text.startswith("usage: ")) + + def test_main(self): + for command, expected in [ + ("--choice a b c", "b"), + ('"a b c"', "b"), + ("a b c", "b"), + ("--choice 'a a' 'b b' 'c c'", "b b"), + ("'a a' 'b b' 'c c'", "b b"), + ("--integer 5", 4), + ("5", 4), + ("--float 2.5", 2.1110546288126204), + ("2.5", 2.1110546288126204), + ]: + random.seed(0) + self.assertEqual(random.main(shlex.split(command)), expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 993a7d6e264a1f..ff95f54026e172 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,12 +1,11 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, check_disallow_instantiation, is_emscripten, is_wasi, - warnings_helper, SHORT_TIMEOUT) + warnings_helper, SHORT_TIMEOUT, CPUStopwatch, requires_resource) import locale import re import string import sys -import time import unittest import warnings from re import Scanner @@ -14,7 +13,7 @@ # some platforms lack working multiprocessing try: - import _multiprocessing + import _multiprocessing # noqa: F401 except ImportError: multiprocessing = None else: @@ -884,31 +883,137 @@ def test_named_unicode_escapes(self): self.checkPatternError(br'\N{LESS-THAN SIGN}', r'bad escape \N', 0) self.checkPatternError(br'[\N{LESS-THAN SIGN}]', r'bad escape \N', 1) - def test_string_boundaries(self): + def test_word_boundaries(self): # See http://bugs.python.org/issue10713 - self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), - "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc", re.ASCII).group(1), "abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc").group(1), b"abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc", re.LOCALE).group(1), b"abc") + self.assertEqual(re.search(r"\b(ьюя)\b", "ьюя").group(1), "ьюя") + self.assertIsNone(re.search(r"\b(ьюя)\b", "ьюя", re.ASCII)) + # There's a word boundary between a word and a non-word. + self.assertTrue(re.match(r".\b", "a=")) + self.assertTrue(re.match(r".\b", "a=", re.ASCII)) + self.assertTrue(re.match(br".\b", b"a=")) + self.assertTrue(re.match(br".\b", b"a=", re.LOCALE)) + self.assertTrue(re.match(r".\b", "я=")) + self.assertIsNone(re.match(r".\b", "я=", re.ASCII)) + # There's a word boundary between a non-word and a word. + self.assertTrue(re.match(r".\b", "=a")) + self.assertTrue(re.match(r".\b", "=a", re.ASCII)) + self.assertTrue(re.match(br".\b", b"=a")) + self.assertTrue(re.match(br".\b", b"=a", re.LOCALE)) + self.assertTrue(re.match(r".\b", "=я")) + self.assertIsNone(re.match(r".\b", "=я", re.ASCII)) + # There is no word boundary inside a word. + self.assertIsNone(re.match(r".\b", "ab")) + self.assertIsNone(re.match(r".\b", "ab", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"ab")) + self.assertIsNone(re.match(br".\b", b"ab", re.LOCALE)) + self.assertIsNone(re.match(r".\b", "юя")) + self.assertIsNone(re.match(r".\b", "юя", re.ASCII)) + # There is no word boundary between a non-word characters. + self.assertIsNone(re.match(r".\b", "=-")) + self.assertIsNone(re.match(r".\b", "=-", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"=-")) + self.assertIsNone(re.match(br".\b", b"=-", re.LOCALE)) + # There is no non-boundary match between a word and a non-word. + self.assertIsNone(re.match(r".\B", "a=")) + self.assertIsNone(re.match(r".\B", "a=", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"a=")) + self.assertIsNone(re.match(br".\B", b"a=", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "я=")) + self.assertTrue(re.match(r".\B", "я=", re.ASCII)) + # There is no non-boundary match between a non-word and a word. + self.assertIsNone(re.match(r".\B", "=a")) + self.assertIsNone(re.match(r".\B", "=a", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"=a")) + self.assertIsNone(re.match(br".\B", b"=a", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "=я")) + self.assertTrue(re.match(r".\B", "=я", re.ASCII)) + # There's a non-boundary match inside a word. + self.assertTrue(re.match(r".\B", "ab")) + self.assertTrue(re.match(r".\B", "ab", re.ASCII)) + self.assertTrue(re.match(br".\B", b"ab")) + self.assertTrue(re.match(br".\B", b"ab", re.LOCALE)) + self.assertTrue(re.match(r".\B", "юя")) + self.assertTrue(re.match(r".\B", "юя", re.ASCII)) + # There's a non-boundary match between a non-word characters. + self.assertTrue(re.match(r".\B", "=-")) + self.assertTrue(re.match(r".\B", "=-", re.ASCII)) + self.assertTrue(re.match(br".\B", b"=-")) + self.assertTrue(re.match(br".\B", b"=-", re.LOCALE)) # There's a word boundary at the start of a string. self.assertTrue(re.match(r"\b", "abc")) + self.assertTrue(re.match(r"\b", "abc", re.ASCII)) + self.assertTrue(re.match(br"\b", b"abc")) + self.assertTrue(re.match(br"\b", b"abc", re.LOCALE)) + self.assertTrue(re.match(r"\b", "ьюя")) + self.assertIsNone(re.match(r"\b", "ьюя", re.ASCII)) + # There's a word boundary at the end of a string. + self.assertTrue(re.fullmatch(r".+\b", "abc")) + self.assertTrue(re.fullmatch(r".+\b", "abc", re.ASCII)) + self.assertTrue(re.fullmatch(br".+\b", b"abc")) + self.assertTrue(re.fullmatch(br".+\b", b"abc", re.LOCALE)) + self.assertTrue(re.fullmatch(r".+\b", "ьюя")) + self.assertIsNone(re.search(r"\b", "ьюя", re.ASCII)) # A non-empty string includes a non-boundary zero-length match. - self.assertTrue(re.search(r"\B", "abc")) + self.assertEqual(re.search(r"\B", "abc").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "abc", re.ASCII).span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc").span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc", re.LOCALE).span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя", re.ASCII).span(), (0, 0)) # There is no non-boundary match at the start of a string. - self.assertFalse(re.match(r"\B", "abc")) + self.assertIsNone(re.match(r"\B", "abc")) + self.assertIsNone(re.match(r"\B", "abc", re.ASCII)) + self.assertIsNone(re.match(br"\B", b"abc")) + self.assertIsNone(re.match(br"\B", b"abc", re.LOCALE)) + self.assertIsNone(re.match(r"\B", "ьюя")) + self.assertTrue(re.match(r"\B", "ьюя", re.ASCII)) + # There is no non-boundary match at the end of a string. + self.assertIsNone(re.fullmatch(r".+\B", "abc")) + self.assertIsNone(re.fullmatch(r".+\B", "abc", re.ASCII)) + self.assertIsNone(re.fullmatch(br".+\B", b"abc")) + self.assertIsNone(re.fullmatch(br".+\B", b"abc", re.LOCALE)) + self.assertIsNone(re.fullmatch(r".+\B", "ьюя")) + self.assertTrue(re.fullmatch(r".+\B", "ьюя", re.ASCII)) # However, an empty string contains no word boundaries, and also no # non-boundaries. - self.assertIsNone(re.search(r"\B", "")) + self.assertIsNone(re.search(r"\b", "")) + self.assertIsNone(re.search(r"\b", "", re.ASCII)) + self.assertIsNone(re.search(br"\b", b"")) + self.assertIsNone(re.search(br"\b", b"", re.LOCALE)) # This one is questionable and different from the perlre behaviour, # but describes current behavior. - self.assertIsNone(re.search(r"\b", "")) + self.assertIsNone(re.search(r"\B", "")) + self.assertIsNone(re.search(r"\B", "", re.ASCII)) + self.assertIsNone(re.search(br"\B", b"")) + self.assertIsNone(re.search(br"\B", b"", re.LOCALE)) # A single word-character string has two boundaries, but no # non-boundary gaps. self.assertEqual(len(re.findall(r"\b", "a")), 2) + self.assertEqual(len(re.findall(r"\b", "a", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\b", b"a")), 2) + self.assertEqual(len(re.findall(br"\b", b"a", re.LOCALE)), 2) self.assertEqual(len(re.findall(r"\B", "a")), 0) + self.assertEqual(len(re.findall(r"\B", "a", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\B", b"a")), 0) + self.assertEqual(len(re.findall(br"\B", b"a", re.LOCALE)), 0) # If there are no words, there are no boundaries self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) # Can match around the whitespace. self.assertEqual(len(re.findall(r"\B", " ")), 2) + self.assertEqual(len(re.findall(r"\B", " ", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\B", b" ")), 2) + self.assertEqual(len(re.findall(br"\B", b" ", re.LOCALE)), 2) def test_bigcharset(self): self.assertEqual(re.match("([\u2222\u2223])", @@ -1116,47 +1221,76 @@ def test_not_literal(self): def test_possible_set_operations(self): s = bytes(range(128)).decode() - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[0-9--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('-./0123456789')) + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: + self.assertEqual(re.findall(r'[0-9--2]', s), list('-./0123456789')) + self.assertEqual(w.filename, __file__) + self.assertEqual(re.findall(r'[--1]', s), list('-./01')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[%--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-1")) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference ') as w: p = re.compile(r'[%--]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-")) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[0-9&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: + self.assertEqual(re.findall(r'[0-8&&1]', s), list('&012345678')) + self.assertEqual(w.filename, __file__) + + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[\d&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) + self.assertEqual(re.findall(r'[&&1]', s), list('&1')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[0-9||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[\d||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) + self.assertEqual(re.findall(r'[||1]', s), list('1|')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[0-9~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[\d~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) + self.assertEqual(re.findall(r'[~~1]', s), list('1~')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[0-9]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789[]')) + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: + self.assertEqual(re.findall(r'[[0-8]|]', s), list('012345678[]')) + self.assertEqual(w.filename, __file__) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[:digit:]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list(':[]dgit')) def test_search_coverage(self): @@ -1229,7 +1363,7 @@ def test_pickling(self): newpat = pickle.loads(pickled) self.assertEqual(newpat, oldpat) # current pickle expects the _compile() reconstructor in re module - from re import _compile + from re import _compile # noqa: F401 def test_copying(self): import copy @@ -2282,19 +2416,21 @@ def test_bug_40736(self): with self.assertRaisesRegex(TypeError, "got 'type'"): re.search("x*", type) + # gh-117594: The test is not slow by itself, but it relies on + # the absolute computation time and can fail on very slow computers. + @requires_resource('cpu') def test_search_anchor_at_beginning(self): s = 'x'*10**7 - start = time.perf_counter() - for p in r'\Ay', r'^y': - self.assertIsNone(re.search(p, s)) - self.assertEqual(re.split(p, s), [s]) - self.assertEqual(re.findall(p, s), []) - self.assertEqual(list(re.finditer(p, s)), []) - self.assertEqual(re.sub(p, '', s), s) - t = time.perf_counter() - start + with CPUStopwatch() as stopwatch: + for p in r'\Ay', r'^y': + self.assertIsNone(re.search(p, s)) + self.assertEqual(re.split(p, s), [s]) + self.assertEqual(re.findall(p, s), []) + self.assertEqual(list(re.finditer(p, s)), []) + self.assertEqual(re.sub(p, '', s), s) # Without optimization it takes 1 second on my computer. # With optimization -- 0.0003 seconds. - self.assertLess(t, 0.1) + self.assertLess(stopwatch.seconds, 0.1) def test_possessive_quantifiers(self): """Test Possessive Quantifiers @@ -2472,6 +2608,24 @@ def test_regression_gh94675(self): def test_fail(self): self.assertEqual(re.search(r'12(?!)|3', '123')[0], '3') + def test_character_set_any(self): + # The union of complementary character sets matches any character + # and is equivalent to "(?s:.)". + s = '1x\n' + for p in r'[\s\S]', r'[\d\D]', r'[\w\W]', r'[\S\s]', r'\s|\S': + with self.subTest(pattern=p): + self.assertEqual(re.findall(p, s), list(s)) + self.assertEqual(re.fullmatch('(?:' + p + ')+', s).group(), s) + + def test_character_set_none(self): + # Negation of the union of complementary character sets does not match + # any character. + s = '1x\n' + for p in r'[^\s\S]', r'[^\d\D]', r'[^\w\W]', r'[^\S\s]': + with self.subTest(pattern=p): + self.assertIsNone(re.search(p, s)) + self.assertIsNone(re.search('(?s:.)' + p, s)) + def get_debug_out(pat): with captured_stdout() as out: diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 5e0e6f8dfac651..50e77cbbb6be13 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -12,6 +12,7 @@ from test.support.os_helper import unlink, temp_dir, TESTFN from test.support.pty_helper import run_pty from test.support.script_helper import assert_python_ok +from test.support.threading_helper import requires_working_threading # Skip tests if there is no readline module readline = import_module('readline') @@ -132,6 +133,32 @@ def test_nonascii_history(self): self.assertEqual(readline.get_history_item(1), "entrée 1") self.assertEqual(readline.get_history_item(2), "entrée 22") + def test_write_read_limited_history(self): + previous_length = readline.get_history_length() + self.addCleanup(readline.set_history_length, previous_length) + + readline.clear_history() + readline.add_history("first line") + readline.add_history("second line") + readline.add_history("third line") + + readline.set_history_length(2) + self.assertEqual(readline.get_history_length(), 2) + readline.write_history_file(TESTFN) + self.addCleanup(os.remove, TESTFN) + + readline.clear_history() + self.assertEqual(readline.get_current_history_length(), 0) + self.assertEqual(readline.get_history_length(), 2) + + readline.read_history_file(TESTFN) + self.assertEqual(readline.get_history_item(1), "second line") + self.assertEqual(readline.get_history_item(2), "third line") + self.assertEqual(readline.get_history_item(3), None) + + # Readline seems to report an additional history element. + self.assertIn(readline.get_current_history_length(), (2, 3)) + class TestReadline(unittest.TestCase): @@ -323,6 +350,50 @@ def test_history_size(self): self.assertEqual(len(lines), history_size) self.assertEqual(lines[-1].strip(), b"last input") + @requires_working_threading() + def test_gh123321_threadsafe(self): + """gh-123321: readline should be thread-safe and not crash""" + script = textwrap.dedent(r""" + import threading + from test.support.threading_helper import join_thread + + def func(): + input() + + thread1 = threading.Thread(target=func) + thread2 = threading.Thread(target=func) + thread1.start() + thread2.start() + join_thread(thread1) + join_thread(thread2) + print("done") + """) + + output = run_pty(script, input=b"input1\rinput2\r") + + self.assertIn(b"done", output) + + + def test_write_read_limited_history(self): + previous_length = readline.get_history_length() + self.addCleanup(readline.set_history_length, previous_length) + + readline.add_history("first line") + readline.add_history("second line") + readline.add_history("third line") + + readline.set_history_length(2) + self.assertEqual(readline.get_history_length(), 2) + readline.write_history_file(TESTFN) + self.addCleanup(os.remove, TESTFN) + + readline.read_history_file(TESTFN) + # Without clear_history() there's no good way to test if + # the correct entries are present (we're combining history limiting and + # possible deduplication with arbitrary previous content). + # So, we've only tested that the read did not fail. + # See TestHistoryManipulation for the full test. + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 89562fa5eac62c..d4f4a69a7a38c1 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -21,13 +21,16 @@ import tempfile import textwrap import unittest +from xml.etree import ElementTree + from test import support -from test.support import os_helper, without_optimizer +from test.support import import_helper +from test.support import os_helper from test.libregrtest import cmdline from test.libregrtest import main from test.libregrtest import setup from test.libregrtest import utils -from test.libregrtest.filter import set_match_tests, match_test +from test.libregrtest.filter import get_match_tests, set_match_tests, match_test from test.libregrtest.result import TestStats from test.libregrtest.utils import normalize_test_name @@ -399,7 +402,7 @@ def test_unknown_option(self): self.checkError(['--unknown-option'], 'unrecognized arguments: --unknown-option') - def check_ci_mode(self, args, use_resources, rerun=True): + def create_regrtest(self, args): ns = cmdline._parse_args(args) # Check Regrtest attributes which are more reliable than Namespace @@ -411,6 +414,10 @@ def check_ci_mode(self, args, use_resources, rerun=True): regrtest = main.Regrtest(ns) + return regrtest + + def check_ci_mode(self, args, use_resources, rerun=True): + regrtest = self.create_regrtest(args) self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) @@ -455,6 +462,33 @@ def test_dont_add_python_opts(self): ns = cmdline._parse_args(args) self.assertFalse(ns._add_python_opts) + def test_bisect(self): + args = ['--bisect'] + regrtest = self.create_regrtest(args) + self.assertTrue(regrtest.want_bisect) + + def test_verbose3_huntrleaks(self): + args = ['-R', '3:10', '--verbose3'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertIsNotNone(regrtest.hunt_refleak) + self.assertEqual(regrtest.hunt_refleak.warmups, 3) + self.assertEqual(regrtest.hunt_refleak.runs, 10) + self.assertFalse(regrtest.output_on_failure) + + def test_single_process(self): + args = ['-j2', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + + args = ['--fast-ci', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + @dataclasses.dataclass(slots=True) class Rerun: @@ -1147,7 +1181,7 @@ def test_run(self): stats=TestStats(4, 1), forever=True) - @without_optimizer + @support.without_optimizer def check_leak(self, code, what, *, run_workers=False): test = self.create_test('huntrleaks', code=code) @@ -1162,8 +1196,8 @@ def check_leak(self, code, what, *, run_workers=False): stderr=subprocess.STDOUT) self.check_executed_tests(output, [test], failed=test, stats=1) - line = 'beginning 6 repetitions\n123456\n......\n' - self.check_line(output, re.escape(line)) + line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n' + self.check_line(output, line) line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) self.assertIn(line2, output) @@ -1192,6 +1226,47 @@ def test_huntrleaks(self): def test_huntrleaks_mp(self): self.check_huntrleaks(run_workers=True) + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def test_huntrleaks_bisect(self): + # test --huntrleaks --bisect + code = textwrap.dedent(""" + import unittest + + GLOBAL_LIST = [] + + class RefLeakTest(unittest.TestCase): + def test1(self): + pass + + def test2(self): + pass + + def test3(self): + GLOBAL_LIST.append(object()) + + def test4(self): + pass + """) + + test = self.create_test('huntrleaks', code=code) + + filename = 'reflog.txt' + self.addCleanup(os_helper.unlink, filename) + cmd = ['--huntrleaks', '3:3:', '--bisect', test] + output = self.run_tests(*cmd, + exitcode=EXITCODE_BAD_TEST, + stderr=subprocess.STDOUT) + + self.assertIn(f"Bisect {test}", output) + self.assertIn(f"Bisect {test}: exit code 0", output) + + # test3 is the one which leaks + self.assertIn("Bisection completed in", output) + self.assertIn( + "Tests (1):\n" + f"* {test}.RefLeakTest.test3\n", + output) + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak @@ -1674,6 +1749,9 @@ def test_other_bug(self): @support.cpython_only def test_uncollectable(self): + # Skip test if _testcapi is missing + import_helper.import_module('_testcapi') + code = textwrap.dedent(r""" import _testcapi import gc @@ -2056,6 +2134,10 @@ def test_unload_tests(self): def check_add_python_opts(self, option): # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python + + # Skip test if _testinternalcapi is missing + import_helper.import_module('_testinternalcapi') + code = textwrap.dedent(r""" import sys import unittest @@ -2118,10 +2200,8 @@ def test_add_python_opts(self): @unittest.skipIf(support.is_android, 'raising SIGSEGV on Android is unreliable') def test_worker_output_on_failure(self): - try: - from faulthandler import _sigsegv - except ImportError: - self.skipTest("need faulthandler._sigsegv") + # Skip test if faulthandler is missing + import_helper.import_module('faulthandler') code = textwrap.dedent(r""" import faulthandler @@ -2176,6 +2256,44 @@ def test_pass(self): self.check_executed_tests(output, testname, stats=1, parallel=True) self.assertNotIn('SPAM SPAM SPAM', output) + def test_xml(self): + code = textwrap.dedent(r""" + import unittest + from test import support + + class VerboseTests(unittest.TestCase): + def test_failed(self): + print("abc \x1b def") + self.fail() + """) + testname = self.create_test(code=code) + + # Run sequentially + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + output = self.run_tests(testname, "--junit-xml", filename, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=testname, + stats=TestStats(1, 1, 0)) + + # Test generated XML + with open(filename, encoding="utf8") as fp: + content = fp.read() + + testsuite = ElementTree.fromstring(content) + self.assertEqual(int(testsuite.get('tests')), 1) + self.assertEqual(int(testsuite.get('errors')), 0) + self.assertEqual(int(testsuite.get('failures')), 1) + + testcase = testsuite[0][0] + self.assertEqual(testcase.get('status'), 'run') + self.assertEqual(testcase.get('result'), 'completed') + self.assertGreater(float(testcase.get('time')), 0) + for out in testcase.iter('system-out'): + self.assertEqual(out.text, r"abc \x1b def") + class TestUtils(unittest.TestCase): def test_format_duration(self): @@ -2211,15 +2329,6 @@ def test_normalize_test_name(self): self.assertIsNone(normalize('setUpModule (test.test_x)', is_error=True)) self.assertIsNone(normalize('tearDownModule (test.test_module)', is_error=True)) - def test_get_signal_name(self): - for exitcode, expected in ( - (-int(signal.SIGINT), 'SIGINT'), - (-int(signal.SIGSEGV), 'SIGSEGV'), - (3221225477, "STATUS_ACCESS_VIOLATION"), - (0xC00000FD, "STATUS_STACK_OVERFLOW"), - ): - self.assertEqual(utils.get_signal_name(exitcode), expected, exitcode) - def test_format_resources(self): format_resources = utils.format_resources ALL_RESOURCES = utils.ALL_RESOURCES @@ -2248,6 +2357,10 @@ def __init__(self, test_id): def id(self): return self.test_id + # Restore patterns once the test completes + patterns = get_match_tests() + self.addCleanup(set_match_tests, patterns) + test_access = Test('test.test_os.FileTests.test_access') test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') test_copy = Test('test.test_shutil.TestCopy.test_copy') @@ -2354,6 +2467,25 @@ def id(self): self.assertTrue(match_test(test_chdir)) self.assertFalse(match_test(test_copy)) + def test_sanitize_xml(self): + sanitize_xml = utils.sanitize_xml + + # escape invalid XML characters + self.assertEqual(sanitize_xml('abc \x1b\x1f def'), + r'abc \x1b\x1f def') + self.assertEqual(sanitize_xml('nul:\x00, bell:\x07'), + r'nul:\x00, bell:\x07') + self.assertEqual(sanitize_xml('surrogate:\uDC80'), + r'surrogate:\udc80') + self.assertEqual(sanitize_xml('illegal \uFFFE and \uFFFF'), + r'illegal \ufffe and \uffff') + + # no escape for valid XML characters + self.assertEqual(sanitize_xml('a\n\tb'), + 'a\n\tb') + self.assertEqual(sanitize_xml('valid t\xe9xt \u20ac'), + 'valid t\xe9xt \u20ac') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index a28d1595f44533..e764e60560db23 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -1,13 +1,26 @@ """Test the interactive interpreter.""" -import sys import os -import unittest +import select import subprocess +import sys +import unittest from textwrap import dedent from test import support -from test.support import cpython_only, has_subprocess_support, SuppressCrashReport +from test.support import ( + cpython_only, + has_subprocess_support, + os_helper, + SuppressCrashReport, + SHORT_TIMEOUT, +) from test.support.script_helper import kill_python +from test.support.import_helper import import_module + +try: + import pty +except ImportError: + pty = None if not has_subprocess_support: @@ -28,7 +41,7 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): # path may be used by Py_GetPath() to build the default module search # path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "") - cmd_line = [stdin_fname, '-E', '-i'] + cmd_line = [stdin_fname, '-I', '-i'] cmd_line.extend(args) # Set TERM=vt100, for the rationale see the comments in spawn_python() of @@ -64,6 +77,7 @@ class TestInteractiveInterpreter(unittest.TestCase): # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') def test_no_memory(self): + import_module("_testcapi") # Issue #30696: Fix the interactive interpreter looping endlessly when # no memory. Check also that the fix does not break the interactive # loop when an exception is raised. @@ -173,6 +187,19 @@ def foo(x): ] self.assertEqual(traceback_lines, expected_lines) + def test_runsource_show_syntax_error_location(self): + user_input = dedent("""def f(x, x): ... + """) + p = spawn_repl() + p.stdin.write(user_input) + output = kill_python(p) + expected_lines = [ + ' def f(x, x): ...', + ' ^', + "SyntaxError: duplicate argument 'x' in function definition" + ] + self.assertEqual(output.splitlines()[4:-1], expected_lines) + def test_interactive_source_is_in_linecache(self): user_input = dedent(""" def foo(x): @@ -193,7 +220,58 @@ def bar(x): expected = "(30, None, [\'def foo(x):\\n\', \' return x + 1\\n\', \'\\n\'], \'\')" self.assertIn(expected, output, expected) + def test_asyncio_repl_reaches_python_startup_script(self): + with os_helper.temp_dir() as tmpdir: + script = os.path.join(tmpdir, "pythonstartup.py") + with open(script, "w") as f: + f.write("print('pythonstartup done!')" + os.linesep) + f.write("exit(0)" + os.linesep) + + env = os.environ.copy() + env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".asyncio_history") + env["PYTHONSTARTUP"] = script + subprocess.check_call( + [sys.executable, "-m", "asyncio"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + timeout=SHORT_TIMEOUT, + ) + @unittest.skipUnless(pty, "requires pty") + def test_asyncio_repl_is_ok(self): + m, s = pty.openpty() + cmd = [sys.executable, "-I", "-m", "asyncio"] + env = os.environ.copy() + proc = subprocess.Popen( + cmd, + stdin=s, + stdout=s, + stderr=s, + text=True, + close_fds=True, + env=env, + ) + os.close(s) + os.write(m, b"await asyncio.sleep(0)\n") + os.write(m, b"exit()\n") + output = [] + while select.select([m], [], [], SHORT_TIMEOUT)[0]: + try: + data = os.read(m, 1024).decode("utf-8") + if not data: + break + except OSError: + break + output.append(data) + os.close(m) + try: + exit_code = proc.wait(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + exit_code = proc.wait() + + self.assertEqual(exit_code, 0, "".join(output)) class TestInteractiveModeSyntaxErrors(unittest.TestCase): @@ -213,5 +291,42 @@ def f(): self.assertEqual(traceback_lines, expected_lines) +class TestAsyncioREPLContextVars(unittest.TestCase): + def test_toplevel_contextvars_sync(self): + user_input = dedent("""\ + from contextvars import ContextVar + var = ContextVar("var", default="failed") + var.set("ok") + """) + p = spawn_repl("-m", "asyncio") + p.stdin.write(user_input) + user_input2 = dedent(""" + print(f"toplevel contextvar test: {var.get()}") + """) + p.stdin.write(user_input2) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + expected = "toplevel contextvar test: ok" + self.assertIn(expected, output, expected) + + def test_toplevel_contextvars_async(self): + user_input = dedent("""\ + from contextvars import ContextVar + var = ContextVar('var', default='failed') + """) + p = spawn_repl("-m", "asyncio") + p.stdin.write(user_input) + user_input2 = "async def set_var(): var.set('ok')\n" + p.stdin.write(user_input2) + user_input3 = "await set_var()\n" + p.stdin.write(user_input3) + user_input4 = "print(f'toplevel contextvar test: {var.get()}')\n" + p.stdin.write(user_input4) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + expected = "toplevel contextvar test: ok" + self.assertIn(expected, output, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 3e93b561c143d8..ffeb1fba7b80c6 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -580,6 +580,50 @@ def test_invalid_indent(self): with self.assertRaisesRegex(expected_error, expected_msg): r.repr(test_object) + def test_shadowed_stdlib_array(self): + # Issue #113570: repr() should not be fooled by an array + class array: + def __repr__(self): + return "not array.array" + + self.assertEqual(r(array()), "not array.array") + + def test_shadowed_builtin(self): + # Issue #113570: repr() should not be fooled + # by a shadowed builtin function + class list: + def __repr__(self): + return "not builtins.list" + + self.assertEqual(r(list()), "not builtins.list") + + def test_custom_repr(self): + class MyRepr(Repr): + + def repr_TextIOWrapper(self, obj, level): + if obj.name in {'', '', ''}: + return obj.name + return repr(obj) + + aRepr = MyRepr() + self.assertEqual(aRepr.repr(sys.stdin), "") + + def test_custom_repr_class_with_spaces(self): + class TypeWithSpaces: + pass + + t = TypeWithSpaces() + type(t).__name__ = "type with spaces" + self.assertEqual(type(t).__name__, "type with spaces") + + class MyRepr(Repr): + def repr_type_with_spaces(self, obj, level): + return "Type With Spaces" + + + aRepr = MyRepr() + self.assertEqual(aRepr.repr(t), "Type With Spaces") + def write_file(path, text): with open(path, 'w', encoding='ASCII') as fp: fp.write(text) diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index 317e7ca8f8c853..d23d3623235f38 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -138,7 +138,7 @@ def test_pagesize(self): self.assertIsInstance(pagesize, int) self.assertGreaterEqual(pagesize, 0) - @unittest.skipUnless(sys.platform == 'linux', 'test requires Linux') + @unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux only') def test_linux_constants(self): for attr in ['MSGQUEUE', 'NICE', 'RTPRIO', 'RTTIME', 'SIGPENDING']: with contextlib.suppress(AttributeError): diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 5f449cdc05c6ba..b967c7623c57b0 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -28,9 +28,6 @@ def __gt__(self, other): def __ge__(self, other): return self.x >= other - def __cmp__(self, other): - raise support.TestFailed("Number.__cmp__() should not be called") - def __repr__(self): return "Number(%r)" % (self.x, ) @@ -53,9 +50,6 @@ def __setitem__(self, i, v): def __bool__(self): raise TypeError("Vectors cannot be used in Boolean contexts") - def __cmp__(self, other): - raise support.TestFailed("Vector.__cmp__() should not be called") - def __repr__(self): return "Vector(%r)" % (self.data, ) diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 273ce2cf5c7dd2..1cff6a218f8d75 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -55,7 +55,7 @@ def test_attr_matches(self): if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) expected = sorted({'None.%s%s' % (x, - '()' if x == '__init_subclass__' + '()' if x in ('__init_subclass__', '__class__') else '' if x == '__doc__' else '(') for x in dir(None)}) diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 9d76764c75be3e..b64383f6546f31 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -15,7 +15,7 @@ from test.support import (infinite_recursion, no_tracing, verbose, requires_subprocess, requires_resource) from test.support.import_helper import forget, make_legacy_pyc, unload -from test.support.os_helper import create_empty_file, temp_dir +from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script @@ -657,14 +657,13 @@ def test_basic_script(self): self._check_script(script_name, "", script_name, script_name, expect_spec=False) - def test_basic_script_with_path_object(self): + def test_basic_script_with_pathlike_object(self): with temp_dir() as script_dir: mod_name = 'script' - script_name = pathlib.Path(self._make_test_script(script_dir, - mod_name)) - self._check_script(script_name, "", - os.fsdecode(script_name), - os.fsdecode(script_name), + script_name = self._make_test_script(script_dir, mod_name) + self._check_script(FakePath(script_name), "", + script_name, + script_name, expect_spec=False) def test_basic_script_no_suffix(self): diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index eda4e6a46df437..0d0f86c145b499 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -16,15 +16,17 @@ from xml.sax.handler import (feature_namespaces, feature_external_ges, LexicalHandler) from xml.sax.xmlreader import InputSource, AttributesImpl, AttributesNSImpl +from xml import sax from io import BytesIO, StringIO import codecs import os.path +import pyexpat import shutil import sys from urllib.error import URLError import urllib.request from test.support import os_helper -from test.support import findfile +from test.support import findfile, check__all__ from test.support.os_helper import FakePath, TESTFN @@ -1214,6 +1216,56 @@ def test_expat_incremental_reset(self): self.assertEqual(result.getvalue(), start + b"text") + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + def test_flush_reparse_deferral_enabled(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + self.assertEqual(result.getvalue(), start) # i.e. no elements started + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + + def test_flush_reparse_deferral_disabled(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + parser._parser.SetReparseDeferralEnabled(False) + self.assertEqual(result.getvalue(), start) # i.e. no elements started + + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + # ===== Locator support def test_expat_locator_noinfo(self): @@ -1506,5 +1558,20 @@ def characters(self, content): self.assertEqual(self.char_index, 2) +class TestModuleAll(unittest.TestCase): + def test_all(self): + extra = ( + 'ContentHandler', + 'ErrorHandler', + 'InputSource', + 'SAXException', + 'SAXNotRecognizedException', + 'SAXNotSupportedException', + 'SAXParseException', + 'SAXReaderNotAvailable', + ) + check__all__(self, sax, extra=extra) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 6e46dfa96a664f..24a366efc6ca05 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -810,6 +810,30 @@ def dig(self): gc_collect() # For PyPy or other GCs. self.assertIsNone(ref()) + def test_multiple_nesting(self): + # Regression test for https://github.com/python/cpython/issues/121863 + class MultiplyNested: + def f1(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g(_MultiplyNested__arg=2) + + def f2(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g + + inst = MultiplyNested() + with self.assertRaises(TypeError): + inst.f1() + + closure = inst.f2() + with self.assertRaises(TypeError): + closure(_MultiplyNested__arg=2) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index d9102eb98a54a6..a8531d466e56e7 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -635,6 +635,16 @@ def __le__(self, some_set): myset >= myobj self.assertTrue(myobj.le_called) + def test_set_membership(self): + myfrozenset = frozenset(range(3)) + myset = {myfrozenset, "abc", 1} + self.assertIn(set(range(3)), myset) + self.assertNotIn(set(range(1)), myset) + myset.discard(set(range(3))) + self.assertEqual(myset, {"abc", 1}) + self.assertRaises(KeyError, myset.remove, set(range(1))) + self.assertRaises(KeyError, myset.remove, set(range(3))) + class SetSubclass(set): pass diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py index 976fa885bd8ef9..0bb02ef11f6b4b 100644 --- a/Lib/test/test_setcomps.py +++ b/Lib/test/test_setcomps.py @@ -1,6 +1,9 @@ import doctest +import traceback import unittest +from test.support import BrokenIter + doctests = """ ########### Tests mostly copied from test_listcomps.py ############ @@ -148,6 +151,42 @@ """ +class SetComprehensionTest(unittest.TestCase): + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + def init_raises(): + try: + {x for x in BrokenIter(init_raises=True)} + except Exception as e: + return e + + def next_raises(): + try: + {x for x in BrokenIter(next_raises=True)} + except Exception as e: + return e + + def iter_raises(): + try: + {x for x in BrokenIter(iter_raises=True)} + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index d96dad4eb9475d..37e54d23b22516 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -558,25 +558,23 @@ def test_rmtree_uses_safe_fd_version_if_available(self): os.listdir in os.supports_fd and os.stat in os.supports_follow_symlinks) if _use_fd_functions: - self.assertTrue(shutil._use_fd_functions) self.assertTrue(shutil.rmtree.avoids_symlink_attacks) tmp_dir = self.mkdtemp() d = os.path.join(tmp_dir, 'a') os.mkdir(d) try: - real_rmtree = shutil._rmtree_safe_fd + real_open = os.open class Called(Exception): pass def _raiser(*args, **kwargs): raise Called - shutil._rmtree_safe_fd = _raiser + os.open = _raiser self.assertRaises(Called, shutil.rmtree, d) finally: - shutil._rmtree_safe_fd = real_rmtree + os.open = real_open else: - self.assertFalse(shutil._use_fd_functions) self.assertFalse(shutil.rmtree.avoids_symlink_attacks) - @unittest.skipUnless(shutil._use_fd_functions, "requires safe rmtree") + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "requires safe rmtree") def test_rmtree_fails_on_close(self): # Test that the error handler is called for failed os.close() and that # os.close() is only called once for a file descriptor. @@ -611,7 +609,7 @@ def onexc(*args): self.assertEqual(errors[1][1], dir1) self.assertEqual(close_count, 2) - @unittest.skipUnless(shutil._use_fd_functions, "dir_fd is not supported") + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "dir_fd is not supported") def test_rmtree_with_dir_fd(self): tmp_dir = self.mkdtemp() victim = 'killme' @@ -625,7 +623,7 @@ def test_rmtree_with_dir_fd(self): shutil.rmtree(victim, dir_fd=dir_fd) self.assertFalse(os.path.exists(fullname)) - @unittest.skipIf(shutil._use_fd_functions, "dir_fd is supported") + @unittest.skipIf(shutil.rmtree.avoids_symlink_attacks, "dir_fd is supported") def test_rmtree_with_dir_fd_unsupported(self): tmp_dir = self.mkdtemp() with self.assertRaises(NotImplementedError): @@ -667,6 +665,23 @@ def test_rmtree_on_junction(self): finally: shutil.rmtree(TESTFN, ignore_errors=True) + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_rmtree_on_named_pipe(self): + os.mkfifo(TESTFN) + try: + with self.assertRaises(NotADirectoryError): + shutil.rmtree(TESTFN) + self.assertTrue(os.path.exists(TESTFN)) + finally: + os.unlink(TESTFN) + + os.mkdir(TESTFN) + os.mkfifo(os.path.join(TESTFN, 'mypipe')) + shutil.rmtree(TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") @os_helper.skip_if_dac_override @@ -724,6 +739,16 @@ def _onexc(fn, path, exc): shutil.rmtree(TESTFN) raise + def test_rmtree_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = os.path.join(TESTFN, *(['d'] * directory_depth)) + os.makedirs(base) + + with support.infinite_recursion(recursion_limit): + shutil.rmtree(TESTFN) + class TestCopyTree(BaseTest, unittest.TestCase): @@ -880,10 +905,10 @@ def test_copytree_arg_types_of_ignore(self): os.mkdir(os.path.join(src_dir, 'test_dir', 'subdir')) write_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') - invokations = [] + invocations = [] def _ignore(src, names): - invokations.append(src) + invocations.append(src) self.assertIsInstance(src, str) self.assertIsInstance(names, list) self.assertEqual(len(names), len(set(names))) @@ -897,7 +922,7 @@ def _ignore(src, names): 'test.txt'))) dst_dir = join(self.mkdtemp(), 'destination') - shutil.copytree(pathlib.Path(src_dir), dst_dir, ignore=_ignore) + shutil.copytree(FakePath(src_dir), dst_dir, ignore=_ignore) self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) @@ -908,7 +933,7 @@ def _ignore(src, names): self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) - self.assertEqual(len(invokations), 9) + self.assertEqual(len(invocations), 9) def test_copytree_retains_permissions(self): tmp_dir = self.mkdtemp() @@ -1598,42 +1623,6 @@ class TestArchives(BaseTest, unittest.TestCase): ### shutil.make_archive - @support.requires_zlib() - def test_make_tarball(self): - # creating something to tar - root_dir, base_dir = self._create_files('') - - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - tarball = make_archive(rel_base_name, 'gztar', root_dir, '.') - - # check if the compressed tarball was created - self.assertEqual(tarball, base_name + '.tar.gz') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r:gz') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - - # trying an uncompressed one - with os_helper.change_cwd(work_dir), no_chdir: - tarball = make_archive(rel_base_name, 'tar', root_dir, '.') - self.assertEqual(tarball, base_name + '.tar') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - def _tarinfo(self, path): with tarfile.open(path) as tar: names = tar.getnames() @@ -1654,6 +1643,92 @@ def _create_files(self, base_dir='dist'): write_file((root_dir, 'outer'), 'xxx') return root_dir, base_dir + @support.requires_zlib() + def test_make_tarfile(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir) + # check if the compressed tarball was created + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'tar', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + # check if the uncompressed tarball was created + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'tar', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist/sub', 'dist/sub/file3']) + + @support.requires_zlib() + def test_make_tarfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'gztar') + self.assertEqual(archive, base_name + '.tar.gz') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r:gz') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', base_dir=base_dir) + self.assertEqual(archive, base_name + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + def test_make_tarfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + @support.requires_zlib() @unittest.skipUnless(shutil.which('tar'), 'Need the tar command to run') @@ -1703,40 +1778,89 @@ def test_tarfile_vs_tar(self): @support.requires_zlib() def test_make_zipfile(self): - # creating something to zip root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'zip', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'zip', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/sub/', 'dist/sub/file3']) - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir) + @support.requires_zlib() + def test_make_zipfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'zip') + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + root_dir, base_dir = self._create_files() + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', base_dir=base_dir) + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3', - 'outer']) - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir, base_dir) - - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3']) + @support.requires_zlib() + def test_make_zipfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) @support.requires_zlib() @unittest.skipUnless(shutil.which('zip'), @@ -1785,7 +1909,10 @@ def test_unzip_zipfile(self): subprocess.check_output(zip_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: details = exc.output.decode(errors="replace") - if 'unrecognized option: t' in details: + if any(message in details for message in [ + 'unrecognized option: t', # BusyBox + 'invalid option -- t', # Android + ]): self.skipTest("unzip doesn't support -t") msg = "{}\n\n**Unzip Output**\n{}" self.fail(msg.format(exc, details)) @@ -1906,17 +2033,19 @@ def archiver(base_name, base_dir, **kw): unregister_archive_format('xxx') def test_make_tarfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'tar'), 'test.tar') self.assertTrue(os.path.isfile('test.tar')) @support.requires_zlib() def test_make_zipfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'zip'), 'test.zip') self.assertTrue(os.path.isfile('test.zip')) @@ -1937,10 +2066,11 @@ def test_register_archive_format(self): self.assertNotIn('xxx', formats) def test_make_tarfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1949,6 +2079,7 @@ def test_make_tarfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1959,10 +2090,11 @@ def test_make_tarfile_rootdir_nodir(self): @support.requires_zlib() def test_make_zipfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1971,6 +2103,7 @@ def test_make_zipfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1985,7 +2118,7 @@ def check_unpack_archive(self, format, **kwargs): self.check_unpack_archive_with_converter( format, lambda path: path, **kwargs) self.check_unpack_archive_with_converter( - format, pathlib.Path, **kwargs) + format, FakePath, **kwargs) self.check_unpack_archive_with_converter(format, FakePath, **kwargs) def check_unpack_archive_with_converter(self, format, converter, **kwargs): @@ -2015,9 +2148,6 @@ def check_unpack_archive_with_converter(self, format, converter, **kwargs): def check_unpack_tarball(self, format): self.check_unpack_archive(format, filter='fully_trusted') self.check_unpack_archive(format, filter='data') - with warnings_helper.check_warnings( - ('Python 3.14', DeprecationWarning)): - self.check_unpack_archive(format) def test_unpack_archive_tar(self): self.check_unpack_tarball('tar') @@ -2090,7 +2220,9 @@ def test_disk_usage(self): def test_chown(self): dirname = self.mkdtemp() filename = tempfile.mktemp(dir=dirname) + linkname = os.path.join(dirname, "chown_link") write_file(filename, 'testing chown function') + os.symlink(filename, linkname) with self.assertRaises(ValueError): shutil.chown(filename) @@ -2111,7 +2243,7 @@ def test_chown(self): gid = os.getgid() def check_chown(path, uid=None, gid=None): - s = os.stat(filename) + s = os.stat(path) if uid is not None: self.assertEqual(uid, s.st_uid) if gid is not None: @@ -2147,6 +2279,36 @@ def check_chown(path, uid=None, gid=None): shutil.chown(dirname, user, group) check_chown(dirname, uid, gid) + dirfd = os.open(dirname, os.O_RDONLY) + self.addCleanup(os.close, dirfd) + basename = os.path.basename(filename) + baselinkname = os.path.basename(linkname) + shutil.chown(basename, uid, gid, dir_fd=dirfd) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, dir_fd=dirfd) + check_chown(filename, uid) + shutil.chown(basename, group=gid, dir_fd=dirfd) + check_chown(filename, gid=gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, uid, gid) + shutil.chown(linkname, uid, follow_symlinks=True) + check_chown(filename, uid) + shutil.chown(baselinkname, group=gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, gid=gid) + shutil.chown(baselinkname, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + + with self.assertRaises(TypeError): + shutil.chown(filename, uid, dir_fd=dirname) + + with self.assertRaises(FileNotFoundError): + shutil.chown('missingfile', uid, gid, dir_fd=dirfd) + + with self.assertRaises(ValueError): + shutil.chown(filename, dir_fd=dirfd) + @support.requires_subprocess() class TestWhich(BaseTest, unittest.TestCase): @@ -2518,12 +2680,12 @@ def test_move_file_to_dir(self): def test_move_file_to_dir_pathlike_src(self): # Move a pathlike file to another location on the same filesystem. - src = pathlib.Path(self.src_file) + src = FakePath(self.src_file) self._check_move_file(src, self.dst_dir, self.dst_file) def test_move_file_to_dir_pathlike_dst(self): # Move a file to another pathlike location on the same filesystem. - dst = pathlib.Path(self.dst_dir) + dst = FakePath(self.dst_dir) self._check_move_file(self.src_file, dst, self.dst_file) @mock_rename @@ -3234,7 +3396,7 @@ def test_module_all_attribute(self): self.assertTrue(hasattr(shutil, '__all__')) target_api = ['copyfileobj', 'copyfile', 'copymode', 'copystat', 'copy', 'copy2', 'copytree', 'move', 'rmtree', 'Error', - 'SpecialFileError', 'ExecError', 'make_archive', + 'SpecialFileError', 'make_archive', 'get_archive_formats', 'register_archive_format', 'unregister_archive_format', 'get_unpack_formats', 'register_unpack_format', 'unregister_unpack_format', @@ -3243,6 +3405,8 @@ def test_module_all_attribute(self): if hasattr(os, 'statvfs') or os.name == 'nt': target_api.append('disk_usage') self.assertEqual(set(shutil.__all__), set(target_api)) + with self.assertWarns(DeprecationWarning): + from shutil import ExecError if __name__ == '__main__': diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 61fb047caf6dab..704a0090bdbc0f 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -123,6 +123,8 @@ def __repr__(self): self.assertEqual(signal.getsignal(signal.SIGHUP), hup) self.assertEqual(0, argument.repr_count) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124083: strsignal is not supported on NetBSD") def test_strsignal(self): self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) @@ -698,7 +700,7 @@ def handler(signum, frame): @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") class SiginterruptTest(unittest.TestCase): - def readpipe_interrupted(self, interrupt): + def readpipe_interrupted(self, interrupt, timeout=support.SHORT_TIMEOUT): """Perform a read during which a signal will arrive. Return True if the read is interrupted by the signal and raises an exception. Return False if it returns normally. @@ -746,7 +748,7 @@ def handler(signum, frame): # wait until the child process is loaded and has started first_line = process.stdout.readline() - stdout, stderr = process.communicate(timeout=support.SHORT_TIMEOUT) + stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired: process.kill() return False @@ -777,7 +779,7 @@ def test_siginterrupt_off(self): # If a signal handler is installed and siginterrupt is called with # a false value for the second argument, when that signal arrives, it # does not interrupt a syscall that's in progress. - interrupted = self.readpipe_interrupted(False) + interrupted = self.readpipe_interrupted(False, timeout=2) self.assertFalse(interrupted) @@ -1325,15 +1327,18 @@ def test_stress_delivery_simultaneous(self): def handler(signum, frame): sigs.append(signum) - self.setsig(signal.SIGUSR1, handler) + # On Android, SIGUSR1 is unreliable when used in close proximity to + # another signal – see Android/testbed/app/src/main/python/main.py. + # So we use a different signal. + self.setsig(signal.SIGUSR2, handler) self.setsig(signal.SIGALRM, handler) # for ITIMER_REAL expected_sigs = 0 while expected_sigs < N: # Hopefully the SIGALRM will be received somewhere during - # initial processing of SIGUSR1. + # initial processing of SIGUSR2. signal.setitimer(signal.ITIMER_REAL, 1e-6 + random.random() * 1e-5) - os.kill(os.getpid(), signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGUSR2) expected_sigs += 2 # Wait for handlers to run to avoid signal coalescing @@ -1345,6 +1350,7 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") + @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 0502181854f52b..035913cdd05f34 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -328,13 +328,13 @@ def test_getsitepackages(self): if sys.platlibdir != "lib": self.assertEqual(len(dirs), 2) wanted = os.path.join('xoxo', sys.platlibdir, - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[0], wanted) else: self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[-1], wanted) else: @@ -513,7 +513,7 @@ def test_sitecustomize_executed(self): # If sitecustomize is available, it should have been imported. if "sitecustomize" not in sys.modules: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError: pass else: diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index 2e0dc1aa276f35..d765746987bc4b 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -2,6 +2,7 @@ from test import support from test.support import import_helper from test.support import socket_helper +import os import smtplib import socket @@ -9,6 +10,8 @@ support.requires("network") +SMTP_TEST_SERVER = os.getenv('CPYTHON_TEST_SMTP_SERVER', 'smtp.gmail.com') + def check_ssl_verifiy(host, port): context = ssl.create_default_context() with socket.create_connection((host, port)) as sock: @@ -22,7 +25,7 @@ def check_ssl_verifiy(host, port): class SmtpTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 587 def test_connect_starttls(self): @@ -44,7 +47,7 @@ def test_connect_starttls(self): class SmtpSSLTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 465 def test_connect(self): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 17964234992062..f2bc52ba6e8701 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -3,7 +3,6 @@ from test.support import ( is_apple, os_helper, refleak_helper, socket_helper, threading_helper ) - import _thread as thread import array import contextlib @@ -37,6 +36,10 @@ import fcntl except ImportError: fcntl = None +try: + import _testcapi +except ImportError: + _testcapi = None support.requires_working_socket(module=True) @@ -46,6 +49,7 @@ VSOCKPORT = 1234 AIX = platform.system() == "AIX" +WSL = "microsoft-standard-WSL" in platform.release() try: import _socket @@ -156,8 +160,8 @@ def _have_socket_qipcrtr(): def _have_socket_vsock(): """Check whether AF_VSOCK sockets are supported on this host.""" - ret = get_cid() is not None - return ret + cid = get_cid() + return (cid is not None) def _have_socket_bluetooth(): @@ -208,7 +212,10 @@ def socket_setdefaulttimeout(timeout): HAVE_SOCKET_VSOCK = _have_socket_vsock() -HAVE_SOCKET_UDPLITE = hasattr(socket, "IPPROTO_UDPLITE") +# Older Android versions block UDPLITE with SELinux. +HAVE_SOCKET_UDPLITE = ( + hasattr(socket, "IPPROTO_UDPLITE") + and not (support.is_android and platform.android_ver().api_level < 29)) HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() @@ -510,10 +517,9 @@ def clientTearDown(self): ThreadableTest.clientTearDown(self) @unittest.skipIf(fcntl is None, "need fcntl") +@unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') -@unittest.skipUnless(get_cid() != 2, - "This test can only be run on a virtual guest.") class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): @@ -526,6 +532,7 @@ def setUp(self): self.serv.bind((socket.VMADDR_CID_ANY, VSOCKPORT)) self.serv.listen() self.serverExplicitReady() + self.serv.settimeout(support.LOOPBACK_TIMEOUT) self.conn, self.connaddr = self.serv.accept() self.addCleanup(self.conn.close) @@ -534,6 +541,9 @@ def clientSetUp(self): self.cli = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) self.addCleanup(self.cli.close) cid = get_cid() + if cid in (socket.VMADDR_CID_HOST, socket.VMADDR_CID_ANY): + # gh-119461: Use the local communication address (loopback) + cid = socket.VMADDR_CID_LOCAL self.cli.connect((cid, VSOCKPORT)) def testStream(self): @@ -582,19 +592,27 @@ class SocketPairTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): unittest.TestCase.__init__(self, methodName=methodName) ThreadableTest.__init__(self) + self.cli = None + self.serv = None + + def socketpair(self): + # To be overridden by some child classes. + return socket.socketpair() def setUp(self): - self.serv, self.cli = socket.socketpair() + self.serv, self.cli = self.socketpair() def tearDown(self): - self.serv.close() + if self.serv: + self.serv.close() self.serv = None def clientSetUp(self): pass def clientTearDown(self): - self.cli.close() + if self.cli: + self.cli.close() self.cli = None ThreadableTest.clientTearDown(self) @@ -1093,6 +1111,7 @@ def test_sethostname(self): @unittest.skipUnless(hasattr(socket, 'if_nameindex'), 'socket.if_nameindex() not available.') + @support.skip_android_selinux('if_nameindex') def testInterfaceNameIndex(self): interfaces = socket.if_nameindex() for index, name in interfaces: @@ -1109,6 +1128,7 @@ def testInterfaceNameIndex(self): @unittest.skipUnless(hasattr(socket, 'if_indextoname'), 'socket.if_indextoname() not available.') + @support.skip_android_selinux('if_indextoname') def testInvalidInterfaceIndexToName(self): self.assertRaises(OSError, socket.if_indextoname, 0) self.assertRaises(OverflowError, socket.if_indextoname, -1) @@ -1128,6 +1148,7 @@ def testInvalidInterfaceIndexToName(self): @unittest.skipUnless(hasattr(socket, 'if_nametoindex'), 'socket.if_nametoindex() not available.') + @support.skip_android_selinux('if_nametoindex') def testInvalidInterfaceNameToIndex(self): self.assertRaises(TypeError, socket.if_nametoindex, 0) self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') @@ -1167,6 +1188,7 @@ def testNtoH(self): self.assertRaises(OverflowError, func, 1<<34) @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testNtoHErrors(self): import _testcapi s_good_values = [0, 1, 2, 0xffff] @@ -1196,8 +1218,8 @@ def testGetServBy(self): # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. if ( - sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform == 'linux' + sys.platform.startswith( + ('linux', 'android', 'freebsd', 'netbsd', 'gnukfreebsd')) or is_apple ): # avoid the 'echo' service on this platform, as there is an @@ -1214,9 +1236,8 @@ def testGetServBy(self): else: raise OSError # Try same call with optional protocol omitted - # Issue #26936: Android getservbyname() was broken before API 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue gh-71123: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): port2 = socket.getservbyname(service) eq(port, port2) # Try udp, but don't barf if it doesn't exist @@ -1227,8 +1248,9 @@ def testGetServBy(self): else: eq(udpport, port) # Now make sure the lookup by port returns the same service name - # Issue #26936: Android getservbyport() is broken. - if not support.is_android: + # Issue #26936: when the protocol is omitted, this fails on Android + # before API level 28. + if not (support.is_android and platform.android_ver().api_level < 28): eq(socket.getservbyport(port2), service) eq(socket.getservbyport(port, 'tcp'), service) if udpport is not None: @@ -1573,9 +1595,8 @@ def testGetaddrinfo(self): socket.getaddrinfo('::1', 80) # port can be a string service name such as "http", a numeric # port number or None - # Issue #26936: Android getaddrinfo() was broken before API level 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue #26936: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): socket.getaddrinfo(HOST, "http") socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) @@ -1633,6 +1654,7 @@ def testGetaddrinfo(self): except socket.gaierror: pass + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_getaddrinfo_int_port_overflow(self): # gh-74895: Test that getaddrinfo does not raise OverflowError on port. # @@ -1650,7 +1672,7 @@ def test_getaddrinfo_int_port_overflow(self): try: socket.getaddrinfo(None, ULONG_MAX + 1, type=socket.SOCK_STREAM) except OverflowError: - # Platforms differ as to what values consitute a getaddrinfo() error + # Platforms differ as to what values constitute a getaddrinfo() error # return. Some fail for LONG_MAX+1, others ULONG_MAX+1, and Windows # silently accepts such huge "port" aka "service" numeric values. self.fail("Either no error or socket.gaierror expected.") @@ -1826,6 +1848,7 @@ def test_listen_backlog(self): srv.listen() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_listen_backlog_overflow(self): # Issue 15989 import _testcapi @@ -1858,6 +1881,7 @@ def test_getfqdn_filter_localhost(self): @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") + @support.skip_android_selinux('if_nameindex') def test_getaddrinfo_ipv6_scopeid_symbolic(self): # Just pick up any network interface (Linux, Mac OS X) (ifindex, test_interface) = socket.if_nameindex()[0] @@ -1891,6 +1915,7 @@ def test_getaddrinfo_ipv6_scopeid_numeric(self): @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") + @support.skip_android_selinux('if_nameindex') def test_getnameinfo_ipv6_scopeid_symbolic(self): # Just pick up any network interface. (ifindex, test_interface) = socket.if_nameindex()[0] @@ -2504,6 +2529,7 @@ def testVSOCKConstants(self): socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE socket.VMADDR_CID_ANY socket.VMADDR_PORT_ANY + socket.VMADDR_CID_LOCAL socket.VMADDR_CID_HOST socket.VM_SOCKETS_INVALID_VERSION socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID @@ -2707,22 +2733,29 @@ def testDup(self): def _testDup(self): self.serv_conn.send(MSG) - def testShutdown(self): - # Testing shutdown() + def check_shutdown(self): + # Test shutdown() helper msg = self.cli_conn.recv(1024) self.assertEqual(msg, MSG) - # wait for _testShutdown to finish: on OS X, when the server + # wait for _testShutdown[_overflow] to finish: on OS X, when the server # closes the connection the client also becomes disconnected, # and the client's shutdown call will fail. (Issue #4397.) self.done.wait() + def testShutdown(self): + self.check_shutdown() + def _testShutdown(self): self.serv_conn.send(MSG) self.serv_conn.shutdown(2) - testShutdown_overflow = support.cpython_only(testShutdown) + @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def testShutdown_overflow(self): + self.check_shutdown() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def _testShutdown_overflow(self): import _testcapi self.serv_conn.send(MSG) @@ -3193,7 +3226,7 @@ def _testSendmsgTimeout(self): # Linux supports MSG_DONTWAIT when sending, but in general, it # only works when receiving. Could add other platforms if they # support it too. - @skipWithClientIf(sys.platform not in {"linux"}, + @skipWithClientIf(sys.platform not in {"linux", "android"}, "MSG_DONTWAIT not known to work on this platform when " "sending") def testSendmsgDontWait(self): @@ -4778,15 +4811,13 @@ def testInterruptedSendmsgTimeout(self): class TCPCloserTest(ThreadedTCPSocketTest): - def testClose(self): - conn, addr = self.serv.accept() - conn.close() + conn, _ = self.serv.accept() - sd = self.cli - read, write, err = select.select([sd], [], [], 1.0) - self.assertEqual(read, [sd]) - self.assertEqual(sd.recv(1), b'') + read, _, _ = select.select([conn], [], [], support.SHORT_TIMEOUT) + self.assertEqual(read, [conn]) + self.assertEqual(conn.recv(1), b'x') + conn.close() # Calling close() many times should be safe. conn.close() @@ -4794,7 +4825,10 @@ def testClose(self): def _testClose(self): self.cli.connect((HOST, self.port)) - time.sleep(1.0) + self.cli.send(b'x') + read, _, _ = select.select([self.cli], [], [], support.SHORT_TIMEOUT) + self.assertEqual(read, [self.cli]) + self.assertEqual(self.cli.recv(1), b'') class BasicSocketPairTest(SocketPairTest): @@ -4832,6 +4866,112 @@ def _testSend(self): self.assertEqual(msg, MSG) +class PurePythonSocketPairTest(SocketPairTest): + # Explicitly use socketpair AF_INET or AF_INET6 to ensure that is the + # code path we're using regardless platform is the pure python one where + # `_socket.socketpair` does not exist. (AF_INET does not work with + # _socket.socketpair on many platforms). + def socketpair(self): + # called by super().setUp(). + try: + return socket.socketpair(socket.AF_INET6) + except OSError: + return socket.socketpair(socket.AF_INET) + + # Local imports in this class make for easy security fix backporting. + + def setUp(self): + if hasattr(_socket, "socketpair"): + self._orig_sp = socket.socketpair + # This forces the version using the non-OS provided socketpair + # emulation via an AF_INET socket in Lib/socket.py. + socket.socketpair = socket._fallback_socketpair + else: + # This platform already uses the non-OS provided version. + self._orig_sp = None + super().setUp() + + def tearDown(self): + super().tearDown() + if self._orig_sp is not None: + # Restore the default socket.socketpair definition. + socket.socketpair = self._orig_sp + + def test_recv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _test_recv(self): + self.cli.send(MSG) + + def test_send(self): + self.serv.send(MSG) + + def _test_send(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + + def test_ipv4(self): + cli, srv = socket.socketpair(socket.AF_INET) + cli.close() + srv.close() + + def _test_ipv4(self): + pass + + @unittest.skipIf(not hasattr(_socket, 'IPPROTO_IPV6') or + not hasattr(_socket, 'IPV6_V6ONLY'), + "IPV6_V6ONLY option not supported") + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test') + def test_ipv6(self): + cli, srv = socket.socketpair(socket.AF_INET6) + cli.close() + srv.close() + + def _test_ipv6(self): + pass + + def test_injected_authentication_failure(self): + orig_getsockname = socket.socket.getsockname + inject_sock = None + + def inject_getsocketname(self): + nonlocal inject_sock + sockname = orig_getsockname(self) + # Connect to the listening socket ahead of the + # client socket. + if inject_sock is None: + inject_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + inject_sock.setblocking(False) + try: + inject_sock.connect(sockname[:2]) + except (BlockingIOError, InterruptedError): + pass + inject_sock.setblocking(True) + return sockname + + sock1 = sock2 = None + try: + socket.socket.getsockname = inject_getsocketname + with self.assertRaises(OSError): + sock1, sock2 = socket.socketpair() + finally: + socket.socket.getsockname = orig_getsockname + if inject_sock: + inject_sock.close() + if sock1: # This cleanup isn't needed on a successful test. + sock1.close() + if sock2: + sock2.close() + + def _test_injected_authentication_failure(self): + # No-op. Exists for base class threading infrastructure to call. + # We could refactor this test into its own lesser class along with the + # setUp and tearDown code to construct an ideal; it is simpler to keep + # it here and live with extra overhead one this _one_ failure test. + pass + + class NonBlockingTCPTests(ThreadedTCPSocketTest): def __init__(self, methodName='runTest'): @@ -4879,6 +5019,7 @@ def _testSetBlocking(self): pass @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testSetBlocking_overflow(self): # Issue 15989 import _testcapi @@ -5631,7 +5772,7 @@ def test_setblocking_invalidfd(self): sock.setblocking(False) -@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +@unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux specific test') class TestLinuxAbstractNamespace(unittest.TestCase): UNIX_PATH_MAX = 108 @@ -5756,7 +5897,8 @@ def testUnencodableAddr(self): self.addCleanup(os_helper.unlink, path) self.assertEqual(self.sock.getsockname(), path) - @unittest.skipIf(sys.platform == 'linux', 'Linux specific test') + @unittest.skipIf(sys.platform in ('linux', 'android'), + 'Linux behavior is tested by TestLinuxAbstractNamespace') def testEmptyAddress(self): # Test that binding empty address fails. self.assertRaises(OSError, self.sock.bind, "") @@ -6690,6 +6832,28 @@ class TestMacOSTCPFlags(unittest.TestCase): def test_tcp_keepalive(self): self.assertTrue(socket.TCP_KEEPALIVE) +@unittest.skipUnless(hasattr(socket, 'TCP_QUICKACK'), 'need socket.TCP_QUICKACK') +class TestQuickackFlag(unittest.TestCase): + def check_set_quickack(self, sock): + # quickack already true by default on some OS distributions + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + if opt: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 0) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertFalse(opt) + + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 1) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertTrue(opt) + + def test_set_quickack(self): + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + self.check_set_quickack(sock) + @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") class TestMSWindowsTCPFlags(unittest.TestCase): @@ -6703,7 +6867,9 @@ class TestMSWindowsTCPFlags(unittest.TestCase): 'TCP_KEEPCNT', # available starting with Windows 10 1709 'TCP_KEEPIDLE', - 'TCP_KEEPINTVL' + 'TCP_KEEPINTVL', + # available starting with Windows 7 / Server 2008 R2 + 'TCP_QUICKACK', } def test_new_tcp_flags(self): diff --git a/Lib/test/test_sort.py b/Lib/test/test_sort.py index 3b6ad4d17b0416..2a7cfb7affaa21 100644 --- a/Lib/test/test_sort.py +++ b/Lib/test/test_sort.py @@ -128,6 +128,27 @@ def bad_key(x): x = [e for e, i in augmented] # a stable sort of s check("stability", x, s) + def test_small_stability(self): + from itertools import product + from operator import itemgetter + + # Exhaustively test stability across all lists of small lengths + # and only a few distinct elements. + # This can provoke edge cases that randomization is unlikely to find. + # But it can grow very expensive quickly, so don't overdo it. + NELTS = 3 + MAXSIZE = 9 + + pick0 = itemgetter(0) + for length in range(MAXSIZE + 1): + # There are NELTS ** length distinct lists. + for t in product(range(NELTS), repeat=length): + xs = list(zip(t, range(length))) + # Stability forced by index in each element. + forced = sorted(xs) + # Use key= to hide the index from compares. + native = sorted(xs, key=pick0) + self.assertEqual(forced, native) #============================================================================== class TestBugs(unittest.TestCase): diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 303f9e03b5383f..d014a9ce841607 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -34,7 +34,9 @@ def expect_failure(self, *args): def test_cli_help(self): out = self.expect_success("-h") - self.assertIn("usage: python -m sqlite3", out) + self.assertIn("usage: ", out) + self.assertIn(" [-h] [-v] [filename] [sql]", out) + self.assertIn("Python sqlite3 CLI", out) def test_cli_version(self): out = self.expect_success("-v") diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 588272448bbfda..488b401fb0054d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -28,14 +28,14 @@ import threading import unittest import urllib.parse +import warnings from test.support import ( SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, is_apple, is_emscripten, is_wasi ) from test.support import gc_collect -from test.support import threading_helper -from _testcapi import INT_MAX, ULLONG_MAX +from test.support import threading_helper, import_helper from os import SEEK_SET, SEEK_CUR, SEEK_END from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath @@ -48,17 +48,6 @@ def test_api_level(self): self.assertEqual(sqlite.apilevel, "2.0", "apilevel is %s, should be 2.0" % sqlite.apilevel) - def test_deprecated_version(self): - msg = "deprecated and will be removed in Python 3.14" - for attr in "version", "version_info": - with self.subTest(attr=attr): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite, attr) - self.assertEqual(cm.filename, __file__) - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite.dbapi2, attr) - self.assertEqual(cm.filename, __file__) - def test_thread_safety(self): self.assertIn(sqlite.threadsafety, {0, 1, 3}, "threadsafety is %d, should be 0, 1 or 3" % @@ -590,6 +579,11 @@ def test_connection_resource_warning(self): del cx gc_collect() + def test_connection_signature(self): + from inspect import signature + sig = signature(self.cx) + self.assertEqual(str(sig), "(sql, /)") + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): @@ -884,9 +878,21 @@ def test_execute_named_param_and_sequence(self): msg = "Binding.*is a named parameter" for query, params in dataset: with self.subTest(query=query, params=params): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: + with self.assertRaisesRegex(sqlite.ProgrammingError, msg) as cm: self.cu.execute(query, params) - self.assertEqual(cm.filename, __file__) + + def test_execute_indexed_nameless_params(self): + # See gh-117995: "'?1' is considered a named placeholder" + for query, params, expected in ( + ("select ?1, ?2", (1, 2), (1, 2)), + ("select ?2, ?1", (1, 2), (2, 1)), + ): + with self.subTest(query=query, params=params): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + cu = self.cu.execute(query, params) + actual, = cu.fetchall() + self.assertEqual(actual, expected) def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER @@ -1202,7 +1208,6 @@ def test_blob_seek_and_tell(self): def test_blob_seek_error(self): msg_oor = "offset out of blob range" msg_orig = "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END" - msg_of = "seek offset results in overflow" dataset = ( (ValueError, msg_oor, lambda: self.blob.seek(1000)), @@ -1214,12 +1219,15 @@ def test_blob_seek_error(self): with self.subTest(exc=exc, msg=msg, fn=fn): self.assertRaisesRegex(exc, msg, fn) + def test_blob_seek_overflow_error(self): # Force overflow errors + msg_of = "seek offset results in overflow" + _testcapi = import_helper.import_module("_testcapi") self.blob.seek(1, SEEK_SET) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_CUR) + self.blob.seek(_testcapi.INT_MAX, SEEK_CUR) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_END) + self.blob.seek(_testcapi.INT_MAX, SEEK_END) def test_blob_read(self): buf = self.blob.read() @@ -1379,14 +1387,17 @@ def test_blob_get_item_error(self): with self.subTest(idx=idx): with self.assertRaisesRegex(IndexError, "index out of range"): self.blob[idx] - with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): - self.blob[ULLONG_MAX] # Provoke read error self.cx.execute("update test set b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob[0] + def test_blob_get_item_error_bigint(self): + _testcapi = import_helper.import_module("_testcapi") + with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): + self.blob[_testcapi.ULLONG_MAX] + def test_blob_set_item_error(self): with self.assertRaisesRegex(TypeError, "cannot be interpreted"): self.blob[0] = b"multiple" @@ -1423,7 +1434,7 @@ def test_blob_sequence_not_supported(self): self.blob + self.blob with self.assertRaisesRegex(TypeError, "unsupported operand"): self.blob * 5 - with self.assertRaisesRegex(TypeError, "is not iterable"): + with self.assertRaisesRegex(TypeError, "is not.+iterable"): b"a" in self.blob def test_blob_context_manager(self): diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 7261b7f0dc93d0..550cea41976441 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -10,6 +10,7 @@ class DumpTests(MemoryDatabaseMixin, unittest.TestCase): def test_table_dump(self): expected_sqls = [ + "PRAGMA foreign_keys=OFF;", """CREATE TABLE "index"("index" blob);""" , """INSERT INTO "index" VALUES(X'01');""" @@ -48,7 +49,7 @@ def test_table_dump(self): expected_sqls = [ "PRAGMA foreign_keys=OFF;", "BEGIN TRANSACTION;", - *expected_sqls, + *expected_sqls[1:], "COMMIT;", ] [self.assertEqual(expected_sqls[i], actual_sqls[i]) @@ -190,6 +191,21 @@ def __getitem__(self, index): got = list(self.cx.iterdump()) self.assertEqual(expected, got) + def test_dump_custom_row_factory(self): + # gh-118221: iterdump should be able to cope with custom row factories. + def dict_factory(cu, row): + fields = [col[0] for col in cu.description] + return dict(zip(fields, row)) + + self.cx.row_factory = dict_factory + CREATE_TABLE = "CREATE TABLE test(t);" + expected = ["BEGIN TRANSACTION;", CREATE_TABLE, "COMMIT;"] + + self.cu.execute(CREATE_TABLE) + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + self.assertEqual(self.cx.row_factory, dict_factory) + def test_dump_virtual_tables(self): # gh-64662 expected = [ diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 1b18230d83577d..de5110a1cc4b6d 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -3,6 +3,8 @@ import sys import unittest import unittest.mock +from ast import literal_eval +from threading import Thread from test import support from test.support import import_helper from test.support import os_helper @@ -38,7 +40,7 @@ ssl = import_helper.import_module("ssl") import _ssl -from ssl import TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType +from ssl import Purpose, TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType Py_DEBUG_WIN32 = support.Py_DEBUG and sys.platform == 'win32' @@ -82,52 +84,32 @@ def data_file(*name): CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") CAFILE_CACERT = data_file("capath", "5ed36f99.0") -CERTFILE_INFO = { - 'issuer': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'notAfter': 'Aug 26 14:23:15 2028 GMT', - 'notBefore': 'Aug 29 14:23:15 2018 GMT', - 'serialNumber': '98A7CF88C74A32ED', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} +with open(data_file('keycert.pem.reference')) as file: + CERTFILE_INFO = literal_eval(file.read()) # empty CRL CRLFILE = data_file("revocation.crl") # Two keys and certs signed by the same CA (for SNI tests) SIGNED_CERTFILE = data_file("keycert3.pem") +SINGED_CERTFILE_ONLY = data_file("cert3.pem") SIGNED_CERTFILE_HOSTNAME = 'localhost' -SIGNED_CERTFILE_INFO = { - 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), - 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), - 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), - 'issuer': ((('countryName', 'XY'),), - (('organizationName', 'Python Software Foundation CA'),), - (('commonName', 'our-ca-server'),)), - 'notAfter': 'Oct 28 14:23:16 2037 GMT', - 'notBefore': 'Aug 29 14:23:16 2018 GMT', - 'serialNumber': 'CB2D80995A69525C', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} +with open(data_file('keycert3.pem.reference')) as file: + SIGNED_CERTFILE_INFO = literal_eval(file.read()) SIGNED_CERTFILE2 = data_file("keycert4.pem") SIGNED_CERTFILE2_HOSTNAME = 'fakehostname' SIGNED_CERTFILE_ECC = data_file("keycertecc.pem") SIGNED_CERTFILE_ECC_HOSTNAME = 'localhost-ecc' +# A custom testcase, extracted from `rfc5280::aki::leaf-missing-aki` in x509-limbo: +# The leaf (server) certificate has no AKI, which is forbidden under RFC 5280. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +LEAF_MISSING_AKI_CERTFILE = data_file("leaf-missing-aki.keycert.pem") +LEAF_MISSING_AKI_CERTFILE_HOSTNAME = "example.com" +LEAF_MISSING_AKI_CA = data_file("leaf-missing-aki.ca.pem") + # Same certificate as pycacert.pem, but without extra text in file SIGNING_CA = data_file("capath", "ceff1710.0") # cert with all kinds of subject alt names @@ -296,11 +278,19 @@ def test_wrap_socket(sock, *, return context.wrap_socket(sock, **kwargs) +USE_SAME_TEST_CONTEXT = False +_TEST_CONTEXT = None + def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True): """Create context client_context, server_context, hostname = testing_context() """ + global _TEST_CONTEXT + if USE_SAME_TEST_CONTEXT: + if _TEST_CONTEXT is not None: + return _TEST_CONTEXT + if server_cert == SIGNED_CERTFILE: hostname = SIGNED_CERTFILE_HOSTNAME elif server_cert == SIGNED_CERTFILE2: @@ -318,6 +308,10 @@ def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True): if server_chain: server_context.load_verify_locations(SIGNING_CA) + if USE_SAME_TEST_CONTEXT: + if _TEST_CONTEXT is not None: + _TEST_CONTEXT = client_context, server_context, hostname + return client_context, server_context, hostname @@ -402,6 +396,7 @@ def test_random(self): ssl.RAND_add(bytearray(b"this is a random bytearray object"), 75.0) def test_parse_cert(self): + self.maxDiff = None # note that this uses an 'unofficial' function in _ssl.c, # provided solely for this test, to exercise the certificate # parsing code @@ -544,7 +539,7 @@ def test_openssl_version(self): else: openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" self.assertTrue( - s.startswith((openssl_ver, libressl_ver)), + s.startswith((openssl_ver, libressl_ver, "AWS-LC")), (s, t, hex(n)) ) @@ -1162,24 +1157,30 @@ def test_load_cert_chain(self): with self.assertRaises(OSError) as cm: ctx.load_cert_chain(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(BADCERT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(EMPTYCERT) # Separate key and cert ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_cert_chain(ONLYCERT, ONLYKEY) ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(ONLYCERT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(ONLYKEY) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) # Mismatching key and cert ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + key values mismatch # OpenSSL + | + KEY_VALUES_MISMATCH # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.SSLError, regex): ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) # Password protected key and cert ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) @@ -1247,7 +1248,7 @@ def test_load_verify_locations(self): with self.assertRaises(OSError) as cm: ctx.load_verify_locations(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_verify_locations(BADCERT) ctx.load_verify_locations(CERTFILE, CAPATH) ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) @@ -1497,6 +1498,10 @@ def test_create_default_context(self): self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_CLIENT) self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_PARTIAL_CHAIN, + ssl.VERIFY_X509_PARTIAL_CHAIN) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_STRICT, + ssl.VERIFY_X509_STRICT) self.assertTrue(ctx.check_hostname) self._assert_context_options(ctx) @@ -1651,9 +1656,10 @@ def test_lib_reason(self): with self.assertRaises(ssl.SSLError) as cm: ctx.load_dh_params(CERTFILE) self.assertEqual(cm.exception.library, 'PEM') - self.assertEqual(cm.exception.reason, 'NO_START_LINE') + regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)" + self.assertRegex(cm.exception.reason, regex) s = str(cm.exception) - self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + self.assertTrue("NO_START_LINE" in s, s) def test_subclass(self): # Check that the appropriate SSLError subclass is raised @@ -1833,7 +1839,13 @@ def test_connect_fail(self): s = test_wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED) self.addCleanup(s.close) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, s.connect, self.server_addr) def test_connect_ex(self): @@ -1901,7 +1913,13 @@ def test_connect_with_context_fail(self): server_hostname=SIGNED_CERTFILE_HOSTNAME ) self.addCleanup(s.close) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, s.connect, self.server_addr) def test_connect_capath(self): @@ -2118,14 +2136,16 @@ def test_bio_handshake(self): self.assertIsNone(sslobj.version()) self.assertIsNone(sslobj.shared_ciphers()) self.assertRaises(ValueError, sslobj.getpeercert) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": self.assertIsNone(sslobj.get_channel_binding('tls-unique')) self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) self.assertTrue(sslobj.cipher()) self.assertIsNone(sslobj.shared_ciphers()) self.assertIsNotNone(sslobj.version()) self.assertTrue(sslobj.getpeercert()) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": self.assertTrue(sslobj.get_channel_binding('tls-unique')) try: self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) @@ -2397,16 +2417,18 @@ def run(self): self.write(msg.lower()) except OSError as e: # handles SSLError and socket errors + if isinstance(e, ConnectionError): + # OpenSSL 1.1.1 sometimes raises + # ConnectionResetError when connection is not + # shut down gracefully. + if self.server.chatty and support.verbose: + print(f" Connection reset by peer: {self.addr}") + + self.close() + self.running = False + return if self.server.chatty and support.verbose: - if isinstance(e, ConnectionError): - # OpenSSL 1.1.1 sometimes raises - # ConnectionResetError when connection is not - # shut down gracefully. - print( - f" Connection reset by peer: {self.addr}" - ) - else: - handle_error("Test server failure:\n") + handle_error("Test server failure:\n") try: self.write(b"ERROR\n") except OSError: @@ -2792,6 +2814,44 @@ def test_echo(self): 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', str(e.exception)) + @unittest.skipUnless(support.Py_GIL_DISABLED, "test is only useful if the GIL is disabled") + def test_ssl_in_multiple_threads(self): + # See GH-124984: OpenSSL is not thread safe. + threads = [] + + global USE_SAME_TEST_CONTEXT + USE_SAME_TEST_CONTEXT = True + try: + for func in ( + self.test_echo, + self.test_alpn_protocols, + self.test_getpeercert, + self.test_crl_check, + self.test_check_hostname_idn, + self.test_wrong_cert_tls12, + self.test_wrong_cert_tls13, + ): + # Be careful with the number of threads here. + # Too many can result in failing tests. + for num in range(5): + with self.subTest(func=func, num=num): + threads.append(Thread(target=func)) + + with threading_helper.catch_threading_exception() as cm: + for thread in threads: + with self.subTest(thread=thread): + thread.start() + + for thread in threads: + with self.subTest(thread=thread): + thread.join() + if cm.exc_value is not None: + # Some threads can skip their test + if not isinstance(cm.exc_value, unittest.SkipTest): + raise cm.exc_value + finally: + USE_SAME_TEST_CONTEXT = False + def test_getpeercert(self): if support.verbose: sys.stdout.write("\n") @@ -2850,11 +2910,16 @@ def test_crl_check(self): client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with server: with client_context.wrap_socket(socket.socket(), server_hostname=hostname) as s: - with self.assertRaisesRegex(ssl.SSLError, - "certificate verify failed"): + with self.assertRaisesRegex(ssl.SSLError, regex): s.connect((HOST, server.port)) # now load a CRL file. The CRL file is signed by the CA. @@ -2885,12 +2950,16 @@ def test_check_hostname(self): # incorrect hostname should raise an exception server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with server: with client_context.wrap_socket(socket.socket(), server_hostname="invalid") as s: - with self.assertRaisesRegex( - ssl.CertificateError, - "Hostname mismatch, certificate is not valid for 'invalid'."): + with self.assertRaisesRegex(ssl.CertificateError, regex): s.connect((HOST, server.port)) # missing server_hostname arg should cause an exception, too @@ -2946,6 +3015,38 @@ def test_ecc_cert(self): cipher = s.cipher()[0].split('-') self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + @unittest.skipUnless(IS_OPENSSL_3_0_0, + "test requires RFC 5280 check added in OpenSSL 3.0+") + def test_verify_strict(self): + # verification fails by default, since the server cert is non-conforming + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + hostname = LEAF_MISSING_AKI_CERTFILE_HOSTNAME + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + + # explicitly disabling VERIFY_X509_STRICT allows it to succeed + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + client_context.verify_flags &= ~ssl.VERIFY_X509_STRICT + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + def test_dual_rsa_ecc(self): client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) client_context.load_verify_locations(SIGNING_CA) @@ -3093,8 +3194,10 @@ def test_wrong_cert_tls13(self): suppress_ragged_eofs=False) as s: s.connect((HOST, server.port)) with self.assertRaisesRegex( - ssl.SSLError, - 'alert unknown ca|EOF occurred' + OSError, + 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA|' + 'closed by the remote host|Connection reset by peer|' + 'Broken pipe' ): # TLS 1.3 perform client cert exchange after handshake s.write(b'data') @@ -3158,13 +3261,21 @@ def test_ssl_cert_verify_error(self): server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: try: s.connect((HOST, server.port)) + self.fail("Expected connection failure") except ssl.SSLError as e: msg = 'unable to get local issuer certificate' self.assertIsInstance(e, ssl.SSLCertVerificationError) self.assertEqual(e.verify_code, 20) self.assertEqual(e.verify_message, msg) - self.assertIn(msg, repr(e)) - self.assertIn('certificate verify failed', repr(e)) + # Allow for flexible libssl error messages. + regex = f"({msg}|CERTIFICATE_VERIFY_FAILED)" + self.assertRegex(repr(e), regex) + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRegex(repr(e), regex) def test_PROTOCOL_TLS(self): """Connecting to an SSLv23 server with various client options""" @@ -3696,7 +3807,7 @@ def test_no_shared_ciphers(self): server_hostname=hostname) as s: with self.assertRaises(OSError): s.connect((HOST, server.port)) - self.assertIn("no shared cipher", server.conn_errors[0]) + self.assertIn("NO_SHARED_CIPHER", server.conn_errors[0]) def test_version_basic(self): """ @@ -3784,7 +3895,7 @@ def test_min_max_version_mismatch(self): server_hostname=hostname) as s: with self.assertRaises(ssl.SSLError) as e: s.connect((HOST, server.port)) - self.assertIn("alert", str(e.exception)) + self.assertRegex(str(e.exception), "(alert|ALERT)") @requires_tls_version('SSLv3') def test_min_max_version_sslv3(self): @@ -3826,6 +3937,10 @@ def test_tls_unique_channel_binding(self): client_context, server_context, hostname = testing_context() + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server = ThreadedEchoServer(context=server_context, chatty=True, connectionchatty=False) @@ -3926,7 +4041,7 @@ def test_dh_params(self): cipher = stats["cipher"][0] parts = cipher.split("-") if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: - self.fail("Non-DH cipher: " + cipher[0]) + self.fail("Non-DH key exchange: " + cipher[0]) def test_ecdh_curve(self): # server secp384r1, client auto @@ -4093,8 +4208,9 @@ def cb_raising(ssl_sock, server_name, initial_context): chatty=False, sni_name='supermessage') - self.assertEqual(cm.exception.reason, - 'SSLV3_ALERT_HANDSHAKE_FAILURE') + # Allow for flexible libssl error messages. + regex = "(SSLV3_ALERT_HANDSHAKE_FAILURE|NO_PRIVATE_VALUE)" + self.assertRegex(cm.exception.reason, regex) self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) def test_sni_callback_wrong_return_type(self): @@ -4446,8 +4562,12 @@ def msg_cb(conn, direction, version, content_type, msg_type, data): # test sometimes fails with EOF error. Test passes as long as # server aborts connection with an error. with self.assertRaisesRegex( - ssl.SSLError, - '(certificate required|EOF occurred)' + OSError, + ('certificate required' + '|EOF occurred' + '|closed by the remote host' + '|Connection reset by peer' + '|Broken pipe') ): # receive CertificateRequest data = s.recv(1024) @@ -4626,6 +4746,40 @@ def test_internal_chain_client(self): ssl.PEM_cert_to_DER_cert(pem), der ) + def test_certificate_chain(self): + client_context, server_context, hostname = testing_context( + server_chain=False + ) + server = ThreadedEchoServer(context=server_context, chatty=False) + + with open(SIGNING_CA) as f: + expected_ca_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with open(SINGED_CERTFILE_ONLY) as f: + expected_ee_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + s.connect((HOST, server.port)) + vc = s.get_verified_chain() + self.assertEqual(len(vc), 2) + + ee, ca = vc + self.assertIsInstance(ee, bytes) + self.assertIsInstance(ca, bytes) + self.assertEqual(expected_ca_cert, ca) + self.assertEqual(expected_ee_cert, ee) + + uvc = s.get_unverified_chain() + self.assertEqual(len(uvc), 1) + self.assertIsInstance(uvc[0], bytes) + + self.assertEqual(ee, uvc[0]) + self.assertNotEqual(ee, ca) + def test_internal_chain_server(self): client_context, server_context, hostname = testing_context() client_context.load_cert_chain(SIGNED_CERTFILE) @@ -4871,7 +5025,7 @@ def run(self): pass # closed, protocol error, etc. def non_linux_skip_if_other_okay_error(self, err): - if sys.platform == "linux": + if sys.platform in ("linux", "android"): return # Expect the full test setup to always work on Linux. if (isinstance(err, ConnectionResetError) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 8bd373976426ef..fa08dc6a25b0ea 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -6,7 +6,10 @@ import sys import unittest from test.support.import_helper import import_module -from _testcapi import get_feature_macros +try: + from _testcapi import get_feature_macros +except ImportError: + raise unittest.SkipTest("requires _testcapi") feature_macros = get_feature_macros() @@ -224,6 +227,9 @@ def test_windows_feature_macros(self): "PyEval_EvalFrameEx", "PyEval_GetBuiltins", "PyEval_GetFrame", + "PyEval_GetFrameBuiltins", + "PyEval_GetFrameGlobals", + "PyEval_GetFrameLocals", "PyEval_GetFuncDesc", "PyEval_GetFuncName", "PyEval_GetGlobals", @@ -261,7 +267,6 @@ def test_windows_feature_macros(self): "PyExc_IOError", "PyExc_ImportError", "PyExc_ImportWarning", - "PyExc_IncompleteInputError", "PyExc_IndentationError", "PyExc_IndexError", "PyExc_InterruptedError", @@ -366,6 +371,7 @@ def test_windows_feature_macros(self): "PyInterpreterState_New", "PyIter_Check", "PyIter_Next", + "PyIter_NextItem", "PyIter_Send", "PyListIter_Type", "PyListRevIter_Type", @@ -385,23 +391,31 @@ def test_windows_feature_macros(self): "PyLongRangeIter_Type", "PyLong_AsDouble", "PyLong_AsInt", + "PyLong_AsInt32", + "PyLong_AsInt64", "PyLong_AsLong", "PyLong_AsLongAndOverflow", "PyLong_AsLongLong", "PyLong_AsLongLongAndOverflow", "PyLong_AsSize_t", "PyLong_AsSsize_t", + "PyLong_AsUInt32", + "PyLong_AsUInt64", "PyLong_AsUnsignedLong", "PyLong_AsUnsignedLongLong", "PyLong_AsUnsignedLongLongMask", "PyLong_AsUnsignedLongMask", "PyLong_AsVoidPtr", "PyLong_FromDouble", + "PyLong_FromInt32", + "PyLong_FromInt64", "PyLong_FromLong", "PyLong_FromLongLong", "PyLong_FromSize_t", "PyLong_FromSsize_t", "PyLong_FromString", + "PyLong_FromUInt32", + "PyLong_FromUInt64", "PyLong_FromUnsignedLong", "PyLong_FromUnsignedLongLong", "PyLong_FromVoidPtr", @@ -699,14 +713,19 @@ def test_windows_feature_macros(self): "PyTuple_Size", "PyTuple_Type", "PyType_ClearCache", + "PyType_Freeze", "PyType_FromMetaclass", "PyType_FromModuleAndSpec", "PyType_FromSpec", "PyType_FromSpecWithBases", "PyType_GenericAlloc", "PyType_GenericNew", + "PyType_GetBaseByToken", "PyType_GetFlags", + "PyType_GetFullyQualifiedName", "PyType_GetModule", + "PyType_GetModuleByDef", + "PyType_GetModuleName", "PyType_GetModuleState", "PyType_GetName", "PyType_GetQualName", @@ -787,6 +806,7 @@ def test_windows_feature_macros(self): "PyUnicode_DecodeUnicodeEscape", "PyUnicode_EncodeFSDefault", "PyUnicode_EncodeLocale", + "PyUnicode_Equal", "PyUnicode_EqualToUTF8", "PyUnicode_EqualToUTF8AndSize", "PyUnicode_FSConverter", @@ -854,6 +874,8 @@ def test_windows_feature_macros(self): "Py_GetArgcArgv", "Py_GetBuildInfo", "Py_GetCompiler", + "Py_GetConstant", + "Py_GetConstantBorrowed", "Py_GetCopyright", "Py_GetExecPrefix", "Py_GetPath", @@ -879,12 +901,14 @@ def test_windows_feature_macros(self): "Py_MakePendingCalls", "Py_NewInterpreter", "Py_NewRef", + "Py_REFCNT", "Py_ReprEnter", "Py_ReprLeave", "Py_SetPath", "Py_SetProgramName", "Py_SetPythonHome", "Py_SetRecursionLimit", + "Py_TYPE", "Py_UTF8Mode", "Py_VaBuildValue", "Py_Version", diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index bf2c254c9ee7d9..c69baa4bf4d1b1 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1072,7 +1072,7 @@ def test_no_inplace_modifications(self): def test_order_doesnt_matter(self): # Test that the order of data points doesn't change the result. - # CAUTION: due to floating point rounding errors, the result actually + # CAUTION: due to floating-point rounding errors, the result actually # may depend on the order. Consider this test representing an ideal. # To avoid this test failing, only test with exact values such as ints # or Fractions. @@ -2353,6 +2353,183 @@ def test_mixed_int_and_float(self): self.assertAlmostEqual(actual_mean, expected_mean, places=5) +class TestKDE(unittest.TestCase): + + def test_kde(self): + kde = statistics.kde + StatisticsError = statistics.StatisticsError + + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # The approximate integral of a PDF should be close to 1.0 + + def integrate(func, low, high, steps=10_000): + "Numeric approximation of a definite function integral." + dx = (high - low) / steps + midpoints = (low + (i + 1/2) * dx for i in range(steps)) + return sum(map(func, midpoints)) * dx + + for kernel in kernels: + with self.subTest(kernel=kernel): + f_hat = kde(sample, h=1.5, kernel=kernel) + area = integrate(f_hat, -20, 20) + self.assertAlmostEqual(area, 1.0, places=4) + + # Check CDF against an integral of the PDF + + data = [3, 5, 10, 12] + h = 2.3 + x = 10.5 + for kernel in kernels: + with self.subTest(kernel=kernel): + cdf = kde(data, h, kernel, cumulative=True) + f_hat = kde(data, h, kernel) + area = integrate(f_hat, -20, x, 100_000) + self.assertAlmostEqual(cdf(x), area, places=4) + + # Check error cases + + with self.assertRaises(StatisticsError): + kde([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde(sample, h=0.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde(sample, h=-1.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde(sample, h=1.0, kernel='bogus') # Invalid kernel + with self.assertRaises(TypeError): + kde(sample, 1.0, 'gauss', True) # Positional cumulative argument + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + f_hat = kde(sample, h, kernel) + self.assertEqual(f_hat.__name__, 'pdf') + self.assertIn(kernel, f_hat.__doc__) + self.assertIn(repr(h), f_hat.__doc__) + + # Test closed interval for the support boundaries. + # In particular, 'uniform' should non-zero at the boundaries. + + f_hat = kde([0], 1.0, 'uniform') + self.assertEqual(f_hat(-1.0), 1/2) + self.assertEqual(f_hat(1.0), 1/2) + + # Test online updates to data + + data = [1, 2] + f_hat = kde(data, 5.0, 'triangular') + self.assertEqual(f_hat(100), 0.0) + data.append(100) + self.assertGreater(f_hat(100), 0.0) + + def test_kde_kernel_specs(self): + # White-box test for the kernel formulas in isolation from + # their downstream use in kde() and kde_random() + kernel_specs = statistics._kernel_specs + + # Verify that cdf / invcdf will round trip + xarr = [i/100 for i in range(-100, 101)] + parr = [i/1000 + 5/10000 for i in range(1000)] + for kernel, spec in kernel_specs.items(): + cdf = spec['cdf'] + invcdf = spec['invcdf'] + with self.subTest(kernel=kernel): + for x in xarr: + self.assertAlmostEqual(invcdf(cdf(x)), x, places=6) + for p in parr: + self.assertAlmostEqual(cdf(invcdf(p)), p, places=11) + + @support.requires_resource('cpu') + def test_kde_random(self): + kde_random = statistics.kde_random + StatisticsError = statistics.StatisticsError + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # Smoke test + + for kernel in kernels: + with self.subTest(kernel=kernel): + rand = kde_random(sample, h=1.5, kernel=kernel) + selections = [rand() for i in range(10)] + + # Check error cases + + with self.assertRaises(StatisticsError): + kde_random([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde_random(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde_random(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde_random(sample, h=-1.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde_random(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde_random(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde_random(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + rand = kde_random(sample, h, kernel) + self.assertEqual(rand.__name__, 'rand') + self.assertIn(kernel, rand.__doc__) + self.assertIn(repr(h), rand.__doc__) + + # Approximate distribution test: Compare a random sample to the expected distribution + + data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] + xarr = [x / 10 for x in range(-100, 250)] + n = 1_000_000 + h = 1.75 + dx = 0.1 + + def p_observed(x): + # P(x <= X < x+dx) + i = bisect.bisect_left(big_sample, x) + j = bisect.bisect_left(big_sample, x + dx) + return (j - i) / len(big_sample) + + def p_expected(x): + # P(x <= X < x+dx) + return F_hat(x + dx) - F_hat(x) + + for kernel in kernels: + with self.subTest(kernel=kernel): + + rand = kde_random(data, h, kernel, seed=8675309**2) + big_sample = sorted([rand() for i in range(n)]) + F_hat = statistics.kde(data, h, kernel, cumulative=True) + + for x in xarr: + self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.0005)) + + # Test online updates to data + + data = [1, 2] + rand = kde_random(data, 5, 'triangular') + self.assertLess(max([rand() for i in range(5000)]), 10) + data.append(100) + self.assertGreater(max(rand() for i in range(5000)), 10) + + class TestQuantiles(unittest.TestCase): def test_specific_cases(self): @@ -2622,7 +2799,7 @@ def test_sqrtprod_helper_function_fundamentals(self): @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "accuracy not guaranteed on machines with double rounding") - @support.cpython_only # Allow for a weaker sumprod() implmentation + @support.cpython_only # Allow for a weaker sumprod() implementation def test_sqrtprod_helper_function_improved_accuracy(self): # Test a known example where accuracy is improved x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index b4927113db44e3..4de6c1cba152bd 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -1578,7 +1578,7 @@ def __int__(self): self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) - self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + self.assertRaisesRegex(TypeError, r'%c requires an int or a unicode character, not .*\.PseudoFloat', operator.mod, '%c', pi) class RaisingNumber: def __int__(self): @@ -1665,7 +1665,7 @@ def test_startswith_endswith_errors(self): self.assertIn('str', exc) self.assertIn('tuple', exc) - @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '') def test_format_float(self): # should not format with a comma, but always with C locale self.assertEqual('1.0', '%.1f' % 1.0) @@ -1736,8 +1736,6 @@ def __str__(self): 'character buffers are decoded to unicode' ) - self.assertRaises(TypeError, str, 42, 42, 42) - def test_constructor_keyword_args(self): """Pass various keyword argument combinations to the constructor.""" # The object argument can be passed as a keyword. @@ -2432,8 +2430,10 @@ def __repr__(self): self.assertEqual(repr(s1()), '\\n') def test_printable_repr(self): - self.assertEqual(repr('\U00010000'), "'%c'" % (0x10000,)) # printable - self.assertEqual(repr('\U00014000'), "'\\U00014000'") # nonprintable + # printable + self.assertEqual(repr('\U00010000'), "'%c'" % (0x10000,)) + # nonprintable (private use area) + self.assertEqual(repr('\U00100001'), "'\\U00100001'") # This test only affects 32-bit platforms because expandtabs can only take # an int as the max value, not a 64-bit C long. If expandtabs is changed @@ -2651,6 +2651,47 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) + def test_str_invalid_call(self): + # too many args + with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): + str("too", "many", "argu", "ments") + with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): + str(1, "", "", 1) + + # no such kw arg + with self.assertRaisesRegex(TypeError, r"str\(\) got an unexpected keyword argument 'test'"): + str(test=1) + + # 'encoding' must be str + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, 1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, encoding=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not bytes"): + str(b"x", b"ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not bytes"): + str(b"x", encoding=b"ascii") + + # 'errors' must be str + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, 1, 1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not int"): + str(1, errors=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not int"): + str(1, "", errors=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not bytes"): + str(b"x", "ascii", b"strict") + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not bytes"): + str(b"x", "ascii", errors=b"strict") + + # both positional and kwarg + with self.assertRaisesRegex(TypeError, r"argument for str\(\) given by name \('encoding'\) and position \(2\)"): + str(b"x", "utf-8", encoding="ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) takes at most 3 arguments \(4 given\)"): + str(b"x", "utf-8", "ignore", encoding="ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) takes at most 3 arguments \(4 given\)"): + str(b"x", "utf-8", "strict", errors="ignore") + class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index cebfc8927862a7..752e31359cf206 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -183,8 +183,7 @@ class Y1900Tests(unittest.TestCase): def test_y_before_1900(self): # Issue #13674, #19634 t = (1899, 1, 1, 0, 0, 0, 0, 0, 0) - if (sys.platform == "win32" - or sys.platform.startswith(("aix", "sunos", "solaris"))): + if sys.platform.startswith(("aix", "sunos", "solaris")): with self.assertRaises(ValueError): time.strftime("%y", t) else: diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py index 371e8193b3544d..c7c6f684cd33f0 100644 --- a/Lib/test/test_string_literals.py +++ b/Lib/test/test_string_literals.py @@ -131,7 +131,7 @@ def test_eval_str_invalid_escape(self): self.assertEqual(exc.lineno, 1) self.assertEqual(exc.offset, 1) - # Check that the warning is raised ony once if there are syntax errors + # Check that the warning is raised only once if there are syntax errors with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always', category=SyntaxWarning) diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 05c8afc907ad3c..9f5cfca9c7f124 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -5,13 +5,22 @@ import locale import re import os +import platform import sys from test import support -from test.support import skip_if_buggy_ucrt_strfptime +from test.support import warnings_helper +from test.support import skip_if_buggy_ucrt_strfptime, run_with_locales from datetime import date as datetime_date import _strptime +libc_ver = platform.libc_ver() +if libc_ver[0] == 'glibc': + glibc_ver = tuple(map(int, libc_ver[1].split('.'))) +else: + glibc_ver = None + + class getlang_Tests(unittest.TestCase): """Test _getlang""" def test_basic(self): @@ -120,7 +129,7 @@ def setUp(self): def test_pattern(self): # Test TimeRE.pattern - pattern_string = self.time_re.pattern(r"%a %A %d") + pattern_string = self.time_re.pattern(r"%a %A %d %Y") self.assertTrue(pattern_string.find(self.locale_time.a_weekday[2]) != -1, "did not find abbreviated weekday in pattern string '%s'" % pattern_string) @@ -160,10 +169,11 @@ def test_compile(self): found.group('b'))) for directive in ('a','A','b','B','c','d','G','H','I','j','m','M','p', 'S','u','U','V','w','W','x','X','y','Y','Z','%'): - compiled = self.time_re.compile("%" + directive) - found = compiled.match(time.strftime("%" + directive)) + fmt = "%d %Y" if directive == 'd' else "%" + directive + compiled = self.time_re.compile(fmt) + found = compiled.match(time.strftime(fmt)) self.assertTrue(found, "Matching failed on '%s' using '%s' regex" % - (time.strftime("%" + directive), + (time.strftime(fmt), compiled.pattern)) def test_blankpattern(self): @@ -206,8 +216,8 @@ class StrptimeTests(unittest.TestCase): """Tests for _strptime.strptime.""" def setUp(self): - """Create testing time tuple.""" - self.time_tuple = time.gmtime() + """Create testing time tuples.""" + self.time_tuple = time.localtime() def test_ValueError(self): # Make sure ValueError is raised when match fails or format is bad @@ -282,59 +292,73 @@ def test_strptime_exception_context(self): # additional check for IndexError branch (issue #19545) with self.assertRaises(ValueError) as e: _strptime._strptime_time('19', '%Y %') - self.assertIs(e.exception.__suppress_context__, True) + self.assertIsNone(e.exception.__context__) def test_unconverteddata(self): # Check ValueError is raised when there is unconverted data self.assertRaises(ValueError, _strptime._strptime_time, "10 12", "%m") - def helper(self, directive, position): + def roundtrip(self, fmt, position, time_tuple=None): """Helper fxn in testing.""" - strf_output = time.strftime("%" + directive, self.time_tuple) - strp_output = _strptime._strptime_time(strf_output, "%" + directive) - self.assertTrue(strp_output[position] == self.time_tuple[position], - "testing of '%s' directive failed; '%s' -> %s != %s" % - (directive, strf_output, strp_output[position], - self.time_tuple[position])) + if time_tuple is None: + time_tuple = self.time_tuple + strf_output = time.strftime(fmt, time_tuple) + strp_output = _strptime._strptime_time(strf_output, fmt) + self.assertEqual(strp_output[position], time_tuple[position], + "testing of %r format failed; %r -> %r != %r" % + (fmt, strf_output, strp_output[position], + time_tuple[position])) + if support.verbose >= 3: + print("testing of %r format: %r -> %r" % + (fmt, strf_output, strp_output[position])) def test_year(self): # Test that the year is handled properly - for directive in ('y', 'Y'): - self.helper(directive, 0) + self.roundtrip('%Y', 0) + self.roundtrip('%y', 0) + self.roundtrip('%Y', 0, (1900, 1, 1, 0, 0, 0, 0, 1, 0)) + # Must also make sure %y values are correct for bounds set by Open Group - for century, bounds in ((1900, ('69', '99')), (2000, ('00', '68'))): - for bound in bounds: - strp_output = _strptime._strptime_time(bound, '%y') - expected_result = century + int(bound) - self.assertTrue(strp_output[0] == expected_result, - "'y' test failed; passed in '%s' " - "and returned '%s'" % (bound, strp_output[0])) + strptime = _strptime._strptime_time + self.assertEqual(strptime('00', '%y')[0], 2000) + self.assertEqual(strptime('68', '%y')[0], 2068) + self.assertEqual(strptime('69', '%y')[0], 1969) + self.assertEqual(strptime('99', '%y')[0], 1999) def test_month(self): # Test for month directives - for directive in ('B', 'b', 'm'): - self.helper(directive, 1) + self.roundtrip('%m', 1) + + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', '') + def test_month_locale(self): + # Test for month directives + self.roundtrip('%B', 1) + self.roundtrip('%b', 1) + for m in range(1, 13): + self.roundtrip('%B', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) + self.roundtrip('%b', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) def test_day(self): # Test for day directives - self.helper('d', 2) + self.roundtrip('%d %Y', 2) def test_hour(self): # Test hour directives - self.helper('H', 3) - strf_output = time.strftime("%I %p", self.time_tuple) - strp_output = _strptime._strptime_time(strf_output, "%I %p") - self.assertTrue(strp_output[3] == self.time_tuple[3], - "testing of '%%I %%p' directive failed; '%s' -> %s != %s" % - (strf_output, strp_output[3], self.time_tuple[3])) + self.roundtrip('%H', 3) + + # NB: Only works on locales with AM/PM + @run_with_locales('LC_TIME', 'C', 'en_US', 'ja_JP') + def test_hour_locale(self): + # Test hour directives + self.roundtrip('%I %p', 3) def test_minute(self): # Test minute directives - self.helper('M', 4) + self.roundtrip('%M', 4) def test_second(self): # Test second directives - self.helper('S', 5) + self.roundtrip('%S', 5) def test_fraction(self): # Test microseconds @@ -345,12 +369,18 @@ def test_fraction(self): def test_weekday(self): # Test weekday directives - for directive in ('A', 'a', 'w', 'u'): - self.helper(directive,6) + self.roundtrip('%w', 6) + self.roundtrip('%u', 6) + + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', '') + def test_weekday_locale(self): + # Test weekday directives + self.roundtrip('%A', 6) + self.roundtrip('%a', 6) def test_julian(self): # Test julian directives - self.helper('j', 7) + self.roundtrip('%j', 7) def test_offset(self): one_hour = 60 * 60 @@ -447,20 +477,121 @@ def test_bad_timezone(self): "time.daylight set to %s and passing in %s" % (time.tzname, tz_value, time.daylight, tz_name)) - def test_date_time(self): + # NB: Does not roundtrip in some locales due to the ambiguity of + # the date and time representation (bugs in locales?): + # * Seconds are not included: bem_ZM, bokmal, ff_SN, nb_NO, nn_NO, + # no_NO, norwegian, nynorsk. + # * Hours are in 12-hour notation without AM/PM indication: hy_AM, + # id_ID, ms_MY. + # * Year is not included: ha_NG. + # * Use non-Gregorian calendar: lo_LA, thai, th_TH. + # On Windows: ar_IN, ar_SA, fa_IR, ps_AF. + # + # BUG: Generates regexp that does not match the current date and time + # for lzh_TW. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'he_IL', 'eu_ES', 'ar_AE', 'mfe_MU', 'yo_NG', + 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', + 'my_MM', 'or_IN', 'shn_MM', 'az_IR') + def test_date_time_locale(self): # Test %c directive - for position in range(6): - self.helper('c', position) - - def test_date(self): + loc = locale.getlocale(locale.LC_TIME)[0] + if glibc_ver and glibc_ver < (2, 31) and loc == 'br_FR': + self.skipTest('%c in locale br_FR does not include time') + now = time.time() + self.roundtrip('%c', slice(0, 6), time.localtime(now)) + # 1 hour 20 minutes 30 seconds ago + self.roundtrip('%c', slice(0, 6), time.localtime(now - 4830)) + # 12 hours ago + self.roundtrip('%c', slice(0, 6), time.localtime(now - 12*3600)) + # different days of the week + for i in range(1, 7): + self.roundtrip('%c', slice(0, 6), time.localtime(now - i*24*3600)) + # different months + for i in range(1, 12): + self.roundtrip('%c', slice(0, 6), time.localtime(now - i*30*24*3600)) + # different year + self.roundtrip('%c', slice(0, 6), time.localtime(now - 366*24*3600)) + + # NB: Dates before 1969 do not roundtrip on some locales: + # az_IR, bo_CN, bo_IN, dz_BT, eu_ES, eu_FR, fa_IR, or_IN. + @support.run_with_tz('STD-1DST,M4.1.0,M10.1.0') + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'he_IL', 'ar_AE', 'mfe_MU', 'yo_NG', + 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', + 'my_MM', 'shn_MM') + def test_date_time_locale2(self): + # Test %c directive + loc = locale.getlocale(locale.LC_TIME)[0] + if sys.platform.startswith('sunos'): + if loc in ('ar_AE',): + self.skipTest(f'locale {loc!r} may not work on this platform') + self.roundtrip('%c', slice(0, 6), (1900, 1, 1, 0, 0, 0, 0, 1, 0)) + self.roundtrip('%c', slice(0, 6), (1800, 1, 1, 0, 0, 0, 0, 1, 0)) + + # NB: Does not roundtrip because use non-Gregorian calendar: + # lo_LA, thai, th_TH. On Windows: ar_IN, ar_SA, fa_IR, ps_AF. + # BUG: Generates regexp that does not match the current date + # for lzh_TW. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'he_IL', 'eu_ES', 'ar_AE', + 'az_IR', 'my_MM', 'or_IN', 'shn_MM') + def test_date_locale(self): # Test %x directive - for position in range(0,3): - self.helper('x', position) - - def test_time(self): + now = time.time() + self.roundtrip('%x', slice(0, 3), time.localtime(now)) + # different days of the week + for i in range(1, 7): + self.roundtrip('%x', slice(0, 3), time.localtime(now - i*24*3600)) + # different months + for i in range(1, 12): + self.roundtrip('%x', slice(0, 3), time.localtime(now - i*30*24*3600)) + # different year + self.roundtrip('%x', slice(0, 3), time.localtime(now - 366*24*3600)) + + # NB: Dates before 1969 do not roundtrip on many locales, including C. + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten, bpo-46390" + ) + @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM') + def test_date_locale2(self): + # Test %x directive + loc = locale.getlocale(locale.LC_TIME)[0] + if sys.platform.startswith('sunos'): + if loc in ('en_US', 'de_DE', 'ar_AE'): + self.skipTest(f'locale {loc!r} may not work on this platform') + self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0)) + self.roundtrip('%x', slice(0, 3), (1800, 1, 1, 0, 0, 0, 0, 1, 0)) + + # NB: Does not roundtrip in some locales due to the ambiguity of + # the time representation (bugs in locales?): + # * Seconds are not included: bokmal, ff_SN, nb_NO, nn_NO, no_NO, + # norwegian, nynorsk. + # * Hours are in 12-hour notation without AM/PM indication: hy_AM, + # ms_MY, sm_WS. + # BUG: Generates regexp that does not match the current time for lzh_TW. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'aa_ET', 'am_ET', 'az_IR', 'byn_ER', 'fa_IR', 'gez_ET', + 'my_MM', 'om_ET', 'or_IN', 'shn_MM', 'sid_ET', 'so_SO', + 'ti_ET', 'tig_ER', 'wal_ET') + def test_time_locale(self): # Test %X directive - for position in range(3,6): - self.helper('X', position) + loc = locale.getlocale(locale.LC_TIME)[0] + pos = slice(3, 6) + if glibc_ver and glibc_ver < (2, 29) and loc in { + 'aa_ET', 'am_ET', 'byn_ER', 'gez_ET', 'om_ET', + 'sid_ET', 'so_SO', 'ti_ET', 'tig_ER', 'wal_ET'}: + # Hours are in 12-hour notation without AM/PM indication. + # Ignore hours. + pos = slice(4, 6) + now = time.time() + self.roundtrip('%X', pos, time.localtime(now)) + # 1 hour 20 minutes 30 seconds ago + self.roundtrip('%X', pos, time.localtime(now - 4830)) + # 12 hours ago + self.roundtrip('%X', pos, time.localtime(now - 12*3600)) def test_percent(self): # Make sure % signs are handled properly @@ -497,9 +628,11 @@ def test_escaping(self): need_escaping = r".^$*+?{}\[]|)(" self.assertTrue(_strptime._strptime_time(need_escaping, need_escaping)) + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-70647 def test_feb29_on_leap_year_without_year(self): time.strptime("Feb 29", "%b %d") + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-70647 def test_mar1_comes_after_feb29_even_when_omitting_the_year(self): self.assertLess( time.strptime("Feb 29", "%b %d"), @@ -679,25 +812,25 @@ class CacheTests(unittest.TestCase): def test_time_re_recreation(self): # Make sure cache is recreated when current locale does not match what # cached object was created with. - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") _strptime._strptime_time("2005", "%Y") _strptime._TimeRE_cache.locale_time.lang = "Ni" original_time_re = _strptime._TimeRE_cache - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertIsNot(original_time_re, _strptime._TimeRE_cache) self.assertEqual(len(_strptime._regex_cache), 1) def test_regex_cleanup(self): # Make sure cached regexes are discarded when cache becomes "full". try: - del _strptime._regex_cache['%d'] + del _strptime._regex_cache['%d %Y'] except KeyError: pass bogus_key = 0 while len(_strptime._regex_cache) <= _strptime._CACHE_MAX_SIZE: _strptime._regex_cache[bogus_key] = None bogus_key += 1 - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertEqual(len(_strptime._regex_cache), 1) def test_new_localetime(self): @@ -705,24 +838,19 @@ def test_new_localetime(self): # is created. locale_time_id = _strptime._TimeRE_cache.locale_time _strptime._TimeRE_cache.locale_time.lang = "Ni" - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertIsNot(locale_time_id, _strptime._TimeRE_cache.locale_time) def test_TimeRE_recreation_locale(self): # The TimeRE instance should be recreated upon changing the locale. - locale_info = locale.getlocale(locale.LC_TIME) - try: - locale.setlocale(locale.LC_TIME, ('en_US', 'UTF8')) - except locale.Error: - self.skipTest('test needs en_US.UTF8 locale') - try: - _strptime._strptime_time('10', '%d') + with support.run_with_locale('LC_TIME', 'en_US.UTF8'): + _strptime._strptime_time('10 2004', '%d %Y') # Get id of current cache object. first_time_re = _strptime._TimeRE_cache try: # Change the locale and force a recreation of the cache. locale.setlocale(locale.LC_TIME, ('de_DE', 'UTF8')) - _strptime._strptime_time('10', '%d') + _strptime._strptime_time('10 2004', '%d %Y') # Get the new cache object's id. second_time_re = _strptime._TimeRE_cache # They should not be equal. @@ -732,10 +860,6 @@ def test_TimeRE_recreation_locale(self): # to the resetting to the original locale. except locale.Error: self.skipTest('test needs de_DE.UTF8 locale') - # Make sure we don't trample on the locale setting once we leave the - # test. - finally: - locale.setlocale(locale.LC_TIME, locale_info) @support.run_with_tz('STD-1DST,M4.1.0,M10.1.0') def test_TimeRE_recreation_timezone(self): diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 15f6ee06ffe19b..5fee9fbb92acf4 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -1,4 +1,5 @@ from collections import abc +from itertools import combinations import array import gc import math @@ -11,12 +12,22 @@ from test import support from test.support import import_helper from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin ISBIGENDIAN = sys.byteorder == "big" integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N' byteorders = '', '@', '=', '<', '>', '!' +INF = float('inf') +NAN = float('nan') + +try: + struct.pack('C', 1j) + have_c_complex = True +except struct.error: + have_c_complex = False + def iter_integer_formats(byteorders=byteorders): for code in integer_codes: for byteorder in byteorders: @@ -33,7 +44,7 @@ def bigendian_to_native(value): else: return string_reverse(value) -class StructTest(unittest.TestCase): +class StructTest(ComplexesAreIdenticalMixin, unittest.TestCase): def test_isbigendian(self): self.assertEqual((struct.pack('=i', 1)[0] == 0), ISBIGENDIAN) @@ -96,6 +107,13 @@ def test_new_features(self): ('10s', b'helloworld', b'helloworld', b'helloworld', 0), ('11s', b'helloworld', b'helloworld\0', b'helloworld\0', 1), ('20s', b'helloworld', b'helloworld'+10*b'\0', b'helloworld'+10*b'\0', 1), + ('0p', b'helloworld', b'', b'', 1), + ('1p', b'helloworld', b'\x00', b'\x00', 1), + ('2p', b'helloworld', b'\x01h', b'\x01h', 1), + ('10p', b'helloworld', b'\x09helloworl', b'\x09helloworl', 1), + ('11p', b'helloworld', b'\x0Ahelloworld', b'\x0Ahelloworld', 0), + ('12p', b'helloworld', b'\x0Ahelloworld\0', b'\x0Ahelloworld\0', 1), + ('20p', b'helloworld', b'\x0Ahelloworld'+9*b'\0', b'\x0Ahelloworld'+9*b'\0', 1), ('b', 7, b'\7', b'\7', 0), ('b', -7, b'\371', b'\371', 0), ('B', 7, b'\7', b'\7', 0), @@ -339,6 +357,7 @@ def assertStructError(func, *args, **kwargs): def test_p_code(self): # Test p ("Pascal string") code. for code, input, expected, expectedback in [ + ('0p', b'abc', b'', b''), ('p', b'abc', b'\x00', b''), ('1p', b'abc', b'\x00', b''), ('2p', b'abc', b'\x01a', b'a'), @@ -521,6 +540,9 @@ def __bool__(self): for c in [b'\x01', b'\x7f', b'\xff', b'\x0f', b'\xf0']: self.assertTrue(struct.unpack('>?', c)[0]) + self.assertTrue(struct.unpack('E', '>C', '"), ('a' * 100, True, 0, @@ -1532,7 +1528,8 @@ def test_repr(self): (["ls"], False, None, ""), (["ls", '--my-opts', 'a' * 100], False, None, ""), - (path_cmd, False, 7, f"") + (os_helper.FakePath("my-tool.py"), False, 7, + ">") ] with unittest.mock.patch.object(subprocess.Popen, '_execute_child'): for cmd, shell, code, sx in cases: @@ -1607,6 +1604,22 @@ def test_class_getitems(self): self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) + @unittest.skipUnless(hasattr(subprocess, '_winapi'), + 'need subprocess._winapi') + def test_wait_negative_timeout(self): + proc = subprocess.Popen(ZERO_RETURN_CMD) + with proc: + patch = mock.patch.object( + subprocess._winapi, + 'WaitForSingleObject', + return_value=subprocess._winapi.WAIT_OBJECT_0) + with patch as mock_wait: + proc.wait(-1) # negative timeout + mock_wait.assert_called_once_with(proc._handle, 0) + proc.returncode = None + + self.assertEqual(proc.wait(), 0) + class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): @@ -1747,6 +1760,13 @@ def test_capture_output(self): self.assertIn(b'BDFL', cp.stdout) self.assertIn(b'FLUFL', cp.stderr) + def test_stdout_stdout(self): + # run() refuses to accept stdout=STDOUT + with self.assertRaises(ValueError, + msg=("STDOUT can only be used for stderr")): + self.run_python("print('will not be run')", + stdout=subprocess.STDOUT) + def test_stdout_with_capture_output_arg(self): # run() refuses to accept 'stdout' with 'capture_output' tf = tempfile.TemporaryFile() @@ -3258,7 +3278,7 @@ def __int__(self): 1, 2, 3, 4, True, True, 0, None, None, None, -1, - None, True) + None) self.assertIn('fds_to_keep', str(c.exception)) finally: if not gc_enabled: @@ -3382,35 +3402,17 @@ def test_preexec_at_exit(self): def dummy(): pass - def exit_handler(): - subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) - @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), - "vfork() not enabled by configure.") - @mock.patch("subprocess._fork_exec") - @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) - def test__use_vfork(self, mock_fork_exec): - self.assertTrue(subprocess._USE_VFORK) # The default value regardless. - mock_fork_exec.side_effect = RuntimeError("just testing args") - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - mock_fork_exec.assert_called_once() - # NOTE: These assertions are *ugly* as they require the last arg - # to remain the have_vfork boolean. We really need to refactor away - # from the giant "wall of args" internal C extension API. - self.assertTrue(mock_fork_exec.call_args.args[-1]) - with mock.patch.object(subprocess, '_USE_VFORK', False): - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) - @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), "vfork() not enabled by configure.") @unittest.skipIf(sys.platform != "linux", "Linux only, requires strace.") @@ -3457,7 +3459,6 @@ def test_vfork_used_when_expected(self): # Test that each individual thing that would disable the use of vfork # actually disables it. for sub_name, preamble, sp_kwarg, expect_permission_error in ( - ("!use_vfork", "subprocess._USE_VFORK = False", "", False), ("preexec", "", "preexec_fn=lambda: None", False), ("setgid", "", f"group={os.getgid()}", True), ("setuid", "", f"user={os.getuid()}", True), diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index f4a8d434ed1b8c..d6d08ee53f821c 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -18,10 +18,11 @@ def test_untested_modules_can_be_imported(self): self.fail('{} has tests even though test_sundry claims ' 'otherwise'.format(name)) - import html.entities + import html.entities # noqa: F401 try: - import tty # Not available on Windows + # Not available on Windows + import tty # noqa: F401 except ImportError: if support.verbose: print("skipping tty") diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 256b416caaa584..149016635522c3 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -1,9 +1,12 @@ """Unit tests for zero-argument super() & related machinery.""" +import copy +import pickle import textwrap +import threading import unittest from unittest.mock import patch -from test.support import import_helper +from test.support import import_helper, threading_helper ADAPTIVE_WARMUP_DELAY = 2 @@ -505,6 +508,107 @@ def some(cls): for _ in range(ADAPTIVE_WARMUP_DELAY): C.some(C) + @threading_helper.requires_working_threading() + def test___class___modification_multithreaded(self): + """ Note: this test isn't actually testing anything on its own. + It requires a sys audithook to be set to crash on older Python. + This should be the case anyways as our test suite sets + an audit hook. + """ + + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(200): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for _ in range(6): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + def test_special_methods(self): + for e in E(), E: + s = super(C, e) + self.assertEqual(s.__reduce__, e.__reduce__) + self.assertEqual(s.__reduce_ex__, e.__reduce_ex__) + self.assertEqual(s.__getstate__, e.__getstate__) + self.assertFalse(hasattr(s, '__getnewargs__')) + self.assertFalse(hasattr(s, '__getnewargs_ex__')) + self.assertFalse(hasattr(s, '__setstate__')) + self.assertFalse(hasattr(s, '__copy__')) + self.assertFalse(hasattr(s, '__deepcopy__')) + + def test_pickling(self): + e = E() + e.x = 1 + s = super(C, e) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + u = pickle.loads(pickle.dumps(s, proto)) + self.assertEqual(u.f(), s.f()) + self.assertIs(type(u), type(s)) + self.assertIs(type(u.__self__), E) + self.assertEqual(u.__self__.x, 1) + self.assertIs(u.__thisclass__, C) + self.assertIs(u.__self_class__, E) + + s = super(C, E) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + u = pickle.loads(pickle.dumps(s, proto)) + self.assertEqual(u.cm(), s.cm()) + self.assertEqual(u.f, s.f) + self.assertIs(type(u), type(s)) + self.assertIs(u.__self__, E) + self.assertIs(u.__thisclass__, C) + self.assertIs(u.__self_class__, E) + + def test_shallow_copying(self): + s = super(C, E()) + self.assertIs(copy.copy(s), s) + s = super(C, E) + self.assertIs(copy.copy(s), s) + + def test_deep_copying(self): + e = E() + e.x = [1] + s = super(C, e) + u = copy.deepcopy(s) + self.assertEqual(u.f(), s.f()) + self.assertIs(type(u), type(s)) + self.assertIsNot(u, s) + self.assertIs(type(u.__self__), E) + self.assertIsNot(u.__self__, e) + self.assertIsNot(u.__self__.x, e.x) + self.assertEqual(u.__self__.x, [1]) + self.assertIs(u.__thisclass__, C) + self.assertIs(u.__self_class__, E) + + s = super(C, E) + u = copy.deepcopy(s) + self.assertEqual(u.cm(), s.cm()) + self.assertEqual(u.f, s.f) + self.assertIsNot(u, s) + self.assertIs(type(u), type(s)) + self.assertIs(u.__self__, E) + self.assertIs(u.__thisclass__, C) + self.assertIs(u.__self_class__, E) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index d160cbf0645b47..9a3cf140d81241 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -3,6 +3,7 @@ import io import os import shutil +import signal import socket import stat import subprocess @@ -70,7 +71,7 @@ def test_get_original_stdout(self): self.assertEqual(support.get_original_stdout(), sys.stdout) def test_unload(self): - import sched + import sched # noqa: F401 self.assertIn("sched", sys.modules) import_helper.unload("sched") self.assertNotIn("sched", sys.modules) @@ -547,13 +548,14 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') + @unittest.skipIf(support.is_apple_mobile, "Unstable on Apple Mobile") @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): - # We cannot test the absolute value of fd_count(): on old Linux - # kernel or glibc versions, os.urandom() keeps a FD open on - # /dev/urandom device and Python has 4 FD opens instead of 3. - # Test is unstable on Emscripten. The platform starts and stops + # We cannot test the absolute value of fd_count(): on old Linux kernel + # or glibc versions, os.urandom() keeps a FD open on /dev/urandom + # device and Python has 4 FD opens instead of 3. Test is unstable on + # Emscripten and Apple Mobile platforms; these platforms start and stop # background threads that use pipes and epoll fds. start = os_helper.fd_count() fd = os.open(__file__, os.O_RDONLY) @@ -732,6 +734,17 @@ def test_copy_python_src_ignore(self): self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored) + def test_get_signal_name(self): + for exitcode, expected in ( + (-int(signal.SIGINT), 'SIGINT'), + (-int(signal.SIGSEGV), 'SIGSEGV'), + (128 + int(signal.SIGABRT), 'SIGABRT'), + (3221225477, "STATUS_ACCESS_VIOLATION"), + (0xC00000FD, "STATUS_STACK_OVERFLOW"), + ): + self.assertEqual(support.get_signal_name(exitcode), expected, + exitcode) + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 92b78a8086a83d..24d89b09d946ad 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -1,6 +1,9 @@ """ Test the API of the symtable module. """ + +import re +import textwrap import symtable import unittest @@ -13,7 +16,7 @@ glob = 42 some_var = 12 -some_non_assigned_global_var = 11 +some_non_assigned_global_var: int some_assigned_global_var = 11 class Mine: @@ -49,10 +52,124 @@ def namespace_test(): pass def generic_spam[T](a): pass -class GenericMine[T: int]: +class GenericMine[T: int, U: (int, str) = int]: pass """ +TEST_COMPLEX_CLASS_CODE = """ +# The following symbols are defined in ComplexClass +# without being introduced by a 'global' statement. +glob_unassigned_meth: Any +glob_unassigned_meth_pep_695: Any + +glob_unassigned_async_meth: Any +glob_unassigned_async_meth_pep_695: Any + +def glob_assigned_meth(): pass +def glob_assigned_meth_pep_695[T](): pass + +async def glob_assigned_async_meth(): pass +async def glob_assigned_async_meth_pep_695[T](): pass + +# The following symbols are defined in ComplexClass after +# being introduced by a 'global' statement (and therefore +# are not considered as local symbols of ComplexClass). +glob_unassigned_meth_ignore: Any +glob_unassigned_meth_pep_695_ignore: Any + +glob_unassigned_async_meth_ignore: Any +glob_unassigned_async_meth_pep_695_ignore: Any + +def glob_assigned_meth_ignore(): pass +def glob_assigned_meth_pep_695_ignore[T](): pass + +async def glob_assigned_async_meth_ignore(): pass +async def glob_assigned_async_meth_pep_695_ignore[T](): pass + +class ComplexClass: + a_var = 1234 + a_genexpr = (x for x in []) + a_lambda = lambda x: x + + type a_type_alias = int + type a_type_alias_pep_695[T] = list[T] + + class a_class: pass + class a_class_pep_695[T]: pass + + def a_method(self): pass + def a_method_pep_695[T](self): pass + + async def an_async_method(self): pass + async def an_async_method_pep_695[T](self): pass + + @classmethod + def a_classmethod(cls): pass + @classmethod + def a_classmethod_pep_695[T](self): pass + + @classmethod + async def an_async_classmethod(cls): pass + @classmethod + async def an_async_classmethod_pep_695[T](self): pass + + @staticmethod + def a_staticmethod(): pass + @staticmethod + def a_staticmethod_pep_695[T](self): pass + + @staticmethod + async def an_async_staticmethod(): pass + @staticmethod + async def an_async_staticmethod_pep_695[T](self): pass + + # These ones will be considered as methods because of the 'def' although + # they are *not* valid methods at runtime since they are not decorated + # with @staticmethod. + def a_fakemethod(): pass + def a_fakemethod_pep_695[T](): pass + + async def an_async_fakemethod(): pass + async def an_async_fakemethod_pep_695[T](): pass + + # Check that those are still considered as methods + # since they are not using the 'global' keyword. + def glob_unassigned_meth(): pass + def glob_unassigned_meth_pep_695[T](): pass + + async def glob_unassigned_async_meth(): pass + async def glob_unassigned_async_meth_pep_695[T](): pass + + def glob_assigned_meth(): pass + def glob_assigned_meth_pep_695[T](): pass + + async def glob_assigned_async_meth(): pass + async def glob_assigned_async_meth_pep_695[T](): pass + + # The following are not picked as local symbols because they are not + # visible by the class at runtime (this is equivalent to having the + # definitions outside of the class). + global glob_unassigned_meth_ignore + def glob_unassigned_meth_ignore(): pass + global glob_unassigned_meth_pep_695_ignore + def glob_unassigned_meth_pep_695_ignore[T](): pass + + global glob_unassigned_async_meth_ignore + async def glob_unassigned_async_meth_ignore(): pass + global glob_unassigned_async_meth_pep_695_ignore + async def glob_unassigned_async_meth_pep_695_ignore[T](): pass + + global glob_assigned_meth_ignore + def glob_assigned_meth_ignore(): pass + global glob_assigned_meth_pep_695_ignore + def glob_assigned_meth_pep_695_ignore[T](): pass + + global glob_assigned_async_meth_ignore + async def glob_assigned_async_meth_ignore(): pass + global glob_assigned_async_meth_pep_695_ignore + async def glob_assigned_async_meth_pep_695_ignore[T](): pass +""" + def find_block(block, name): for ch in block.get_children(): @@ -65,6 +182,7 @@ class SymtableTest(unittest.TestCase): top = symtable.symtable(TEST_CODE, "?", "exec") # These correspond to scopes in TEST_CODE Mine = find_block(top, "Mine") + a_method = find_block(Mine, "a_method") spam = find_block(top, "spam") internal = find_block(spam, "internal") @@ -78,6 +196,7 @@ class SymtableTest(unittest.TestCase): GenericMine = find_block(top, "GenericMine") GenericMine_inner = find_block(GenericMine, "GenericMine") T = find_block(GenericMine, "T") + U = find_block(GenericMine, "U") def test_type(self): self.assertEqual(self.top.get_type(), "module") @@ -87,13 +206,14 @@ def test_type(self): self.assertEqual(self.internal.get_type(), "function") self.assertEqual(self.foo.get_type(), "function") self.assertEqual(self.Alias.get_type(), "type alias") - self.assertEqual(self.GenericAlias.get_type(), "type parameter") + self.assertEqual(self.GenericAlias.get_type(), "type parameters") self.assertEqual(self.GenericAlias_inner.get_type(), "type alias") - self.assertEqual(self.generic_spam.get_type(), "type parameter") + self.assertEqual(self.generic_spam.get_type(), "type parameters") self.assertEqual(self.generic_spam_inner.get_type(), "function") - self.assertEqual(self.GenericMine.get_type(), "type parameter") + self.assertEqual(self.GenericMine.get_type(), "type parameters") self.assertEqual(self.GenericMine_inner.get_type(), "class") - self.assertEqual(self.T.get_type(), "TypeVar bound") + self.assertEqual(self.T.get_type(), "type variable") + self.assertEqual(self.U.get_type(), "type variable") def test_id(self): self.assertGreater(self.top.get_id(), 0) @@ -205,12 +325,14 @@ def test_assigned(self): def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') - st2 = st1.get_children()[0] + st2 = st1.get_children()[1] + self.assertEqual(st2.get_type(), "function") self.assertTrue(st2.lookup('x').is_local()) self.assertTrue(st2.lookup('x').is_annotated()) self.assertFalse(st2.lookup('x').is_global()) st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') - st4 = st3.get_children()[0] + st4 = st3.get_children()[1] + self.assertEqual(st4.get_type(), "function") self.assertTrue(st4.lookup('x').is_local()) self.assertFalse(st4.lookup('x').is_annotated()) @@ -237,8 +359,86 @@ def test_name(self): self.assertEqual(self.spam.lookup("x").get_name(), "x") self.assertEqual(self.Mine.get_name(), "Mine") - def test_class_info(self): - self.assertEqual(self.Mine.get_methods(), ('a_method',)) + def test_class_get_methods(self): + deprecation_mess = ( + re.escape('symtable.Class.get_methods() is deprecated ' + 'and will be removed in Python 3.16.') + ) + + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(self.Mine.get_methods(), ('a_method',)) + + top = symtable.symtable(TEST_COMPLEX_CLASS_CODE, "?", "exec") + this = find_block(top, "ComplexClass") + + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), ( + 'a_method', 'a_method_pep_695', + 'an_async_method', 'an_async_method_pep_695', + 'a_classmethod', 'a_classmethod_pep_695', + 'an_async_classmethod', 'an_async_classmethod_pep_695', + 'a_staticmethod', 'a_staticmethod_pep_695', + 'an_async_staticmethod', 'an_async_staticmethod_pep_695', + 'a_fakemethod', 'a_fakemethod_pep_695', + 'an_async_fakemethod', 'an_async_fakemethod_pep_695', + 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', + 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', + 'glob_assigned_meth', 'glob_assigned_meth_pep_695', + 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', + )) + + # Test generator expressions that are of type TYPE_FUNCTION + # but will not be reported by get_methods() since they are + # not functions per se. + # + # Other kind of comprehensions such as list, set or dict + # expressions do not have the TYPE_FUNCTION type. + + def check_body(body, expected_methods): + indented = textwrap.indent(body, ' ' * 4) + top = symtable.symtable(f"class A:\n{indented}", "?", "exec") + this = find_block(top, "A") + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), expected_methods) + + # statements with 'genexpr' inside it + GENEXPRS = ( + 'x = (x for x in [])', + 'x = (x async for x in [])', + 'type x[genexpr = (x for x in [])] = (x for x in [])', + 'type x[genexpr = (x async for x in [])] = (x async for x in [])', + 'genexpr = (x for x in [])', + 'genexpr = (x async for x in [])', + 'type genexpr[genexpr = (x for x in [])] = (x for x in [])', + 'type genexpr[genexpr = (x async for x in [])] = (x async for x in [])', + ) + + for gen in GENEXPRS: + # test generator expression + with self.subTest(gen=gen): + check_body(gen, ()) + + # test generator expression + variable named 'genexpr' + with self.subTest(gen=gen, isvar=True): + check_body('\n'.join((gen, 'genexpr = 1')), ()) + check_body('\n'.join(('genexpr = 1', gen)), ()) + + for paramlist in ('()', '(x)', '(x, y)', '(z: T)'): + for func in ( + f'def genexpr{paramlist}:pass', + f'async def genexpr{paramlist}:pass', + f'def genexpr[T]{paramlist}:pass', + f'async def genexpr[T]{paramlist}:pass', + ): + with self.subTest(func=func): + # test function named 'genexpr' + check_body(func, ('genexpr',)) + + for gen in GENEXPRS: + with self.subTest(gen=gen, func=func): + # test generator expression + function named 'genexpr' + check_body('\n'.join((gen, func)), ('genexpr',)) + check_body('\n'.join((func, gen)), ('genexpr',)) def test_filename_correct(self): ### Bug tickler: SyntaxError file name correct whether error raised @@ -299,12 +499,56 @@ def test_symbol_repr(self): "") self.assertEqual(repr(self.other_internal.lookup("some_var")), "") + self.assertEqual(repr(self.GenericMine.lookup("T")), + "") + + st1 = symtable.symtable("[x for x in [1]]", "?", "exec") + self.assertEqual(repr(st1.lookup("x")), + "") + + st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") + self.assertEqual(repr(st2.lookup("x")), + "") + + st3 = symtable.symtable("def f():\n" + " x = 1\n" + " class A:\n" + " x = 2\n" + " def method():\n" + " return x\n", + "?", "exec") + # child 0 is for __annotate__ + func_f = st3.get_children()[1] + class_A = func_f.get_children()[0] + self.assertEqual(repr(class_A.lookup('x')), + "") def test_symtable_entry_repr(self): expected = f"" self.assertEqual(repr(self.top._table), expected) +class ComprehensionTests(unittest.TestCase): + def get_identifiers_recursive(self, st, res): + res.extend(st.get_identifiers()) + for ch in st.get_children(): + self.get_identifiers_recursive(ch, res) + + def test_loopvar_in_only_one_scope(self): + # ensure that the loop variable appears only once in the symtable + comps = [ + "[x for x in [1]]", + "{x for x in [1]}", + "{x:x*x for x in [1]}", + ] + for comp in comps: + with self.subTest(comp=comp): + st = symtable.symtable(comp, "?", "exec") + ids = [] + self.get_identifiers_recursive(st, ids) + self.assertEqual(len([x for x in ids if x == 'x']), 1) + + class CommandLineTest(unittest.TestCase): maxDiff = None diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index ee9b873d9023f0..132e2b839627bc 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -59,6 +59,18 @@ Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> def __debug__(): pass +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> async def __debug__(): pass +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> class __debug__: pass +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + >>> del __debug__ Traceback (most recent call last): SyntaxError: cannot delete __debug__ @@ -786,6 +798,9 @@ >>> __debug__: int Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> x.__debug__: int +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ >>> f(a=) Traceback (most recent call last): SyntaxError: expected argument value expression @@ -1182,6 +1197,24 @@ Traceback (most recent call last): SyntaxError: expected ':' + >>> match x: + ... case a, __debug__, b: + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> match x: + ... case a, b, *__debug__: + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> match x: + ... case Foo(a, __debug__=1, b=2): + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + >>> if x = 3: ... pass Traceback (most recent call last): @@ -1213,6 +1246,22 @@ Traceback (most recent call last): SyntaxError: expected '(' + >>> def f -> int: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f -> int: # type: int + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f[T]: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> def f[T] -> str: + Traceback (most recent call last): + SyntaxError: expected '(' + Parenthesized arguments in function definitions >>> def f(x, (y, z), w): @@ -1259,6 +1308,15 @@ Traceback (most recent call last): SyntaxError: expected 'except' or 'finally' block +Custom error message for __debug__ as exception variable + + >>> try: + ... pass + ... except TypeError as __debug__: + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + Custom error message for try block mixing except and except* >>> try: @@ -1301,6 +1359,36 @@ Traceback (most recent call last): SyntaxError: cannot have both 'except' and 'except*' on the same 'try' +Better error message for using `except as` with not a name: + + >>> try: + ... pass + ... except TypeError as obj.attr: + ... pass + Traceback (most recent call last): + SyntaxError: cannot use except statement with attribute + + >>> try: + ... pass + ... except TypeError as obj[1]: + ... pass + Traceback (most recent call last): + SyntaxError: cannot use except statement with subscript + + >>> try: + ... pass + ... except* TypeError as (obj, name): + ... pass + Traceback (most recent call last): + SyntaxError: cannot use except* statement with tuple + + >>> try: + ... pass + ... except* TypeError as 1: + ... pass + Traceback (most recent call last): + SyntaxError: cannot use except* statement with literal + Ensure that early = are not matched by the parser as invalid comparisons >>> f(2, 4, x=34); 1 $ 2 Traceback (most recent call last): @@ -1506,6 +1594,19 @@ Traceback (most recent call last): IndentationError: expected an indented block after class definition on line 1 + >>> class C(__debug__=42): ... + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> class Meta(type): + ... def __new__(*args, **kwargs): + ... pass + + >>> class C(metaclass=Meta, __debug__=42): + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + >>> match something: ... pass Traceback (most recent call last): @@ -1692,6 +1793,26 @@ Traceback (most recent call last): SyntaxError: Did you mean to use 'from ... import ...' instead? +>>> import __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> import a as __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> import a.b.c as __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> from a import __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> from a import b as __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. @@ -1699,6 +1820,18 @@ Traceback (most recent call last): SyntaxError: invalid syntax +>>> from i import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> from .. import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + >>> (): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated @@ -1712,6 +1845,49 @@ Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated +# 'not' after operators: + +>>> 3 + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 * not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> ~ not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + not -1 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +# Check that we don't introduce misleading errors +>>> not 1 */ 2 +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not + 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + Corner-cases that used to fail to raise the correct error: >>> def f(*, x=lambda __debug__:0): pass @@ -1756,53 +1932,55 @@ ... case 42 as 1+2+4: ... ... Traceback (most recent call last): - SyntaxError: invalid pattern target + SyntaxError: cannot use expression as pattern target >>> match ...: - ... case Foo(z=1, y=2, x): + ... case 42 as a.b: ... ... Traceback (most recent call last): - SyntaxError: positional patterns follow keyword patterns + SyntaxError: cannot use attribute as pattern target >>> match ...: - ... case Foo(a, z=1, y=2, x): + ... case 42 as (a, b): ... ... Traceback (most recent call last): - SyntaxError: positional patterns follow keyword patterns + SyntaxError: cannot use tuple as pattern target >>> match ...: - ... case Foo(z=1, x, y=2): + ... case 42 as (a + 1): ... ... Traceback (most recent call last): - SyntaxError: positional patterns follow keyword patterns + SyntaxError: cannot use expression as pattern target >>> match ...: - ... case C(a=b, c, d=e, f, g=h, i, j=k, ...): + ... case (32 as x) | (42 as a()): ... ... Traceback (most recent call last): - SyntaxError: positional patterns follow keyword patterns + SyntaxError: cannot use function call as pattern target -Non-matching 'elif'/'else' statements: - - >>> if a == b: + >>> match ...: + ... case Foo(z=1, y=2, x): ... ... - ... elif a == c: Traceback (most recent call last): - SyntaxError: 'elif' must match an if-statement here + SyntaxError: positional patterns follow keyword patterns - >>> if x == y: + >>> match ...: + ... case Foo(a, z=1, y=2, x): ... ... - ... else: Traceback (most recent call last): - SyntaxError: 'else' must match a valid statement here + SyntaxError: positional patterns follow keyword patterns - >>> elif m == n: + >>> match ...: + ... case Foo(z=1, x, y=2): + ... ... Traceback (most recent call last): - SyntaxError: 'elif' must match an if-statement here + SyntaxError: positional patterns follow keyword patterns - >>> else: + >>> match ...: + ... case C(a=b, c, d=e, f, g=h, i, j=k, ...): + ... ... Traceback (most recent call last): - SyntaxError: 'else' must match a valid statement here + SyntaxError: positional patterns follow keyword patterns Uses of the star operator which should fail: @@ -1868,22 +2046,22 @@ >>> A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[*(1:2)] = 1 Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> del A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[*:] and A[:*] >>> A[*:] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[:*] Traceback (most recent call last): ... @@ -1894,7 +2072,7 @@ >>> A[*] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[**] @@ -1972,16 +2150,116 @@ def f(x: *b) Invalid expressions in type scopes: + >>> type A[] = int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> class A[]: ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[](): ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[]() + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> async def some[]: # type: int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> async def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> async def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> async def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + >>> type A[T: (x:=3)] = int Traceback (most recent call last): ... SyntaxError: named expression cannot be used within a TypeVar bound + >>> type A[T: ((x:= 3), int)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> type A[T = ((x:=3))] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> def f[T: (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> def f[T: (int, (yield))](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> def f[T = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> def f[*Ts = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> def f[**P = [(yield), int]](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + >>> type A[T: (yield 3)] = int Traceback (most recent call last): ... SyntaxError: yield expression cannot be used within a TypeVar bound + >>> type A[T: (int, (yield 3))] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> type A[T = (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + >>> type A[T: (await 3)] = int Traceback (most recent call last): ... @@ -1992,6 +2270,31 @@ def f(x: *b) ... SyntaxError: yield expression cannot be used within a TypeVar bound + >>> class A[T: (yield 3)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (int, (yield 3))]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> class A[T = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> class A[*Ts = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> class A[**P = [(yield), int]]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + >>> type A = (x := 3) Traceback (most recent call last): ... @@ -2012,6 +2315,14 @@ def f(x: *b) ... SyntaxError: yield expression cannot be used within a type alias + >>> type __debug__ = int + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> class A[__debug__]: pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + >>> class A[T]((x := 3)): ... Traceback (most recent call last): ... @@ -2038,11 +2349,23 @@ def f(x: *b) >>> f(**x, *) Traceback (most recent call last): - SyntaxError: iterable argument unpacking follows keyword argument unpacking + SyntaxError: Invalid star expression >>> f(x, *:) Traceback (most recent call last): - SyntaxError: invalid syntax + SyntaxError: Invalid star expression + + >>> f(x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression """ import re @@ -2059,8 +2382,8 @@ def _check_error(self, code, errtext, lineno=None, offset=None, end_lineno=None, end_offset=None): """Check that compiling code raises SyntaxError with errtext. - errtext is a regular expression that must be present in the - test of the exception raised. If subclass is specified, it + errtest is a regular expression that must be present in the + test of the exception raised. If subclass is specified it is the expected subclass of SyntaxError (e.g. IndentationError). """ try: @@ -2084,22 +2407,6 @@ def _check_error(self, code, errtext, else: self.fail("compile() did not raise SyntaxError") - def _check_noerror(self, code, - errtext="compile() raised unexpected SyntaxError", - filename="", mode="exec", subclass=None): - """Check that compiling code does not raise a SyntaxError. - - errtext is the message passed to self.fail if there is - a SyntaxError. If the subclass parameter is specified, - it is the subclass of SyntaxError (e.g. IndentationError) - that the raised error is checked against. - """ - try: - compile(code, filename, mode) - except SyntaxError as err: - if (not subclass) or isinstance(err, subclass): - self.fail(errtext) - def test_expression_with_assignment(self): self._check_error( "print(end1 + end2 = ' ')", @@ -2325,13 +2632,40 @@ def bug(): code += "): yield a" return code - CO_MAXBLOCKS = 20 # static nesting limit of the compiler + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @support.cpython_only + def test_async_with_statement_many_context_managers(self): + # See gh-116767 + + def get_code(n): + code = [ textwrap.dedent(""" + async def bug(): + async with ( + a + """) ] + for i in range(n): + code.append(f" as a{i}, a\n") + code.append("): yield a") + return "".join(code) + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block - for n in range(CO_MAXBLOCKS): + for n in range(MAX_MANAGERS): with self.subTest(f"within range: {n=}"): compile(get_code(n), "", "exec") - for n in range(CO_MAXBLOCKS, CO_MAXBLOCKS + 5): + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): with self.subTest(f"out of range: {n=}"): self._check_error(get_code(n), "too many statically nested blocks") @@ -2469,29 +2803,11 @@ def test_syntax_error_on_deeply_nested_blocks(self): while 20: while 21: while 22: - break + while 23: + break """ self._check_error(source, "too many statically nested blocks") - def test_syntax_error_non_matching_elif_else_statements(self): - # Check bpo-45759: 'elif' statements that doesn't match an - # if-statement or 'else' statements that doesn't match any - # valid else-able statement (e.g. 'while') - self._check_error( - "elif m == n:\n ...", - "'elif' must match an if-statement here") - self._check_error( - "else:\n ...", - "'else' must match a valid statement here") - self._check_noerror("if a == b:\n ...\nelif a == c:\n ...") - self._check_noerror("if x == y:\n ...\nelse:\n ...") - self._check_error( - "else = 123", - "invalid syntax") - self._check_error( - "elif 55 = 123", - "cannot assign to literal here") - @support.cpython_only def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" @@ -2508,6 +2824,39 @@ def test_deep_invalid_rule(self): with self.assertRaises(SyntaxError): compile(source, "", "exec") + def test_except_stmt_invalid_as_expr(self): + self._check_error( + textwrap.dedent( + """ + try: + pass + except ValueError as obj.attr: + pass + """ + ), + errtext="cannot use except statement with attribute", + lineno=4, + end_lineno=4, + offset=22, + end_offset=22 + len("obj.attr"), + ) + + def test_match_stmt_invalid_as_expr(self): + self._check_error( + textwrap.dedent( + """ + match 1: + case x as obj.attr: + ... + """ + ), + errtext="cannot use attribute as pattern target", + lineno=3, + end_lineno=3, + offset=15, + end_offset=15 + len("obj.attr"), + ) + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 71671a5a984256..c0862d7d15f39e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -16,6 +16,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure from test.support import threading_helper from test.support import import_helper +from test.support import force_not_colorized try: from test.support import interpreters except ImportError: @@ -145,6 +146,7 @@ def f(): class ExceptHookTest(unittest.TestCase): + @force_not_colorized def test_original_excepthook(self): try: raise ValueError(42) @@ -156,6 +158,7 @@ def test_original_excepthook(self): self.assertRaises(TypeError, sys.__excepthook__) + @force_not_colorized def test_excepthook_bytes_filename(self): # bpo-37467: sys.excepthook() must not crash if a filename # is a bytes string @@ -203,6 +206,20 @@ def test_exit(self): self.assertEqual(out, b'') self.assertEqual(err, b'') + # gh-125842: Windows uses 32-bit unsigned integers for exit codes + # so a -1 exit code is sometimes interpreted as 0xffff_ffff. + rc, out, err = assert_python_failure('-c', 'import sys; sys.exit(0xffff_ffff)') + self.assertIn(rc, (-1, 0xff, 0xffff_ffff)) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + # Overflow results in a -1 exit code, which may be converted to 0xff + # or 0xffff_ffff. + rc, out, err = assert_python_failure('-c', 'import sys; sys.exit(2**128)') + self.assertIn(rc, (-1, 0xff, 0xffff_ffff)) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + # call with integer argument with self.assertRaises(SystemExit) as cm: sys.exit(42) @@ -391,10 +408,15 @@ def test_dlopenflags(self): @test.support.refcount_test def test_refcount(self): - # n here must be a global in order for this test to pass while - # tracing with a python function. Tracing calls PyFrame_FastToLocals - # which will add a copy of any locals to the frame object, causing - # the reference count to increase by 2 instead of 1. + # n here originally had to be a global in order for this test to pass + # while tracing with a python function. Tracing used to call + # PyFrame_FastToLocals, which would add a copy of any locals to the + # frame object, causing the ref count to increase by 2 instead of 1. + # While that no longer happens (due to PEP 667), this test case retains + # its original global-based implementation + # PEP 683's immortal objects also made this point moot, since the + # refcount for None doesn't change anyway. Maybe this test should be + # using a different constant value? (e.g. an integer) global n self.assertRaises(TypeError, sys.getrefcount) c = sys.getrefcount(None) @@ -562,7 +584,8 @@ def g456(): # And the next record must be for g456(). filename, lineno, funcname, sourceline = stack[i+1] self.assertEqual(funcname, "g456") - self.assertTrue(sourceline.startswith("if leave_g.wait(")) + self.assertTrue((sourceline.startswith("if leave_g.wait(") or + sourceline.startswith("g_raised.set()"))) finally: # Reap the spawned thread. leave_g.set() @@ -668,7 +691,7 @@ def test_thread_info(self): self.assertEqual(len(info), 3) self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) - if sys.platform.startswith(("linux", "freebsd")): + if sys.platform.startswith(("linux", "android", "freebsd")): self.assertEqual(info.name, "pthread") elif sys.platform == "win32": self.assertEqual(info.name, "nt") @@ -722,33 +745,70 @@ def __hash__(self): if has_is_interned: self.assertIs(sys._is_interned(S("abc")), False) + @support.cpython_only @requires_subinterpreters def test_subinterp_intern_dynamically_allocated(self): + # Implementation detail: Dynamically allocated strings + # are distinct between interpreters s = "never interned before" + str(random.randrange(0, 10**9)) t = sys.intern(s) self.assertIs(t, s) interp = interpreters.create() - interp.exec_sync(textwrap.dedent(f''' + interp.exec(textwrap.dedent(f''' import sys - t = sys.intern({s!r}) + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + assert id(t) != {id(s)}, (id(t), {id(s)}) assert id(t) != {id(t)}, (id(t), {id(t)}) ''')) + @support.cpython_only @requires_subinterpreters def test_subinterp_intern_statically_allocated(self): + # Implementation detail: Statically allocated strings are shared + # between interpreters. # See Tools/build/generate_global_objects.py for the list # of strings that are always statically allocated. - s = '__init__' - t = sys.intern(s) + for s in ('__init__', 'CANCELLED', '', 'utf-8', + '{{', '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff', + ): + with self.subTest(s=s): + t = sys.intern(s) - interp = interpreters.create() - interp.exec_sync(textwrap.dedent(f''' - import sys - t = sys.intern({s!r}) - assert id(t) == {id(t)}, (id(t), {id(t)}) - ''')) + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + assert id(t) == {id(t)}, (id(t), {id(t)}) + ''')) + + @support.cpython_only + @requires_subinterpreters + def test_subinterp_intern_singleton(self): + # Implementation detail: singletons are used for 0- and 1-character + # latin1 strings. + for s in '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff': + with self.subTest(s=s): + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + assert id(s) == {id(s)} + t = sys.intern(s) + ''')) + self.assertTrue(sys._is_interned(s)) def test_sys_flags(self): self.assertTrue(sys.flags) @@ -792,6 +852,7 @@ def test_sys_getwindowsversion_no_instantiation(self): def test_clear_type_cache(self): sys._clear_type_cache() + @force_not_colorized @support.requires_subprocess() def test_ioencoding(self): env = dict(os.environ) @@ -995,14 +1056,10 @@ def test_debugmallocstats(self): # Output of sys._debugmallocstats() depends on configure flags. # The sysconfig vars are not available on Windows. if sys.platform != "win32": - with_freelists = sysconfig.get_config_var("WITH_FREELISTS") with_pymalloc = sysconfig.get_config_var("WITH_PYMALLOC") - if with_freelists: - self.assertIn(b"free PyDictObjects", err) + self.assertIn(b"free PyDictObjects", err) if with_pymalloc: self.assertIn(b'Small block threshold', err) - if not with_freelists and not with_pymalloc: - self.assertFalse(err) # The function has no parameter self.assertRaises(TypeError, sys._debugmallocstats, True) @@ -1048,6 +1105,12 @@ def test_getallocatedblocks(self): c = sys.getallocatedblocks() self.assertIn(c, range(b - 50, b + 50)) + def test_is_gil_enabled(self): + if support.Py_GIL_DISABLED: + self.assertIs(type(sys._is_gil_enabled()), bool) + else: + self.assertTrue(sys._is_gil_enabled()) + def test_is_finalizing(self): self.assertIs(sys.is_finalizing(), False) # Don't use the atexit module because _Py_Finalizing is only set @@ -1101,13 +1164,13 @@ def __del__(self): self.assertEqual(stdout.rstrip(), b"") self.assertEqual(stderr.rstrip(), b"") - @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), - 'need sys.getandroidapilevel()') + @unittest.skipUnless(sys.platform == "android", "Android only") def test_getandroidapilevel(self): level = sys.getandroidapilevel() self.assertIsInstance(level, int) self.assertGreater(level, 0) + @force_not_colorized @support.requires_subprocess() def test_sys_tracebacklimit(self): code = """if 1: @@ -1390,7 +1453,7 @@ class SizeofTest(unittest.TestCase): def setUp(self): self.P = struct.calcsize('P') self.longdigit = sys.int_info.sizeof_digit - import _testinternalcapi + _testinternalcapi = import_helper.import_module("_testinternalcapi") self.gc_headsize = _testinternalcapi.SIZEOF_PYGC_HEAD self.managed_pre_header_size = _testinternalcapi.SIZEOF_MANAGED_PRE_HEADER @@ -1550,10 +1613,11 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + INTERPRETER_FRAME = '9PhcP' + check(x, size('3PiccPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass - check(func, size('15Pi')) + check(func, size('16Pi')) class c(): @staticmethod def foo(): @@ -1567,7 +1631,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('PP4P4c7P2ic??2P')) + check(get_gen(), size('6P4c' + INTERPRETER_FRAME + 'P')) # iterator check(iter('abc'), size('lP')) # callable-iterator @@ -1595,7 +1659,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module - check(unittest, size('PnPPP')) + if support.Py_GIL_DISABLED: + check(unittest, size('PPPPPP')) + else: + check(unittest, size('PPPPP')) # None check(None, size('')) # NotImplementedType @@ -1653,6 +1720,7 @@ def delx(self): del self.__x fmt = 'P2nPI13Pl4Pn9Pn12PIPc' s = vsize(fmt) check(int, s) + typeid = 'n' if support.Py_GIL_DISABLED else '' # class s = vsize(fmt + # PyTypeObject '4P' # PyAsyncMethods @@ -1660,8 +1728,9 @@ def delx(self): del self.__x '3P' # PyMappingMethods '10P' # PySequenceMethods '2P' # PyBufferProcs - '6P' - '1PIP' # Specializer cache + '7P' + '1PIP' # Specializer cache + + typeid # heap type id (free-threaded only) ) class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries @@ -1708,11 +1777,15 @@ class newstyleclass(object): pass # TODO: add check that forces layout of unicodefields # weakref import weakref - check(weakref.ref(int), size('2Pn3P')) + if support.Py_GIL_DISABLED: + expected = size('2Pn4P') + else: + expected = size('2Pn3P') + check(weakref.ref(int), expected) # weakproxy # XXX # weakcallableproxy - check(weakref.proxy(int), size('2Pn3P')) + check(weakref.proxy(int), expected) def check_slots(self, obj, base, extra): expected = sys.getsizeof(base) + struct.calcsize(extra) @@ -1762,7 +1835,8 @@ def test_pythontypes(self): # symtable entry # XXX # sys.flags - check(sys.flags, vsize('') + self.P * len(sys.flags)) + # FIXME: The +1 will not be necessary once gh-122575 is fixed + check(sys.flags, vsize('') + self.P * (1 + len(sys.flags))) def test_asyncgen_hooks(self): old = sys.get_asyncgen_hooks() @@ -1770,6 +1844,21 @@ def test_asyncgen_hooks(self): self.assertIsNone(old.finalizer) firstiter = lambda *a: None + finalizer = lambda *a: None + + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter=firstiter, finalizer="invalid") + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + + # gh-118473 + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter="invalid", finalizer=finalizer) + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + sys.set_asyncgen_hooks(firstiter=firstiter) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) @@ -1777,7 +1866,6 @@ def test_asyncgen_hooks(self): self.assertIs(hooks.finalizer, None) self.assertIs(hooks[1], None) - finalizer = lambda *a: None sys.set_asyncgen_hooks(finalizer=finalizer) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index 32e03d7cd25dbe..311a4d2cafe88d 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -124,7 +124,7 @@ def f(p): def test_caught_exception(self): def f(p): try: 1/0 - except: pass + except ZeroDivisionError: pass f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -133,7 +133,7 @@ def f(p): def test_caught_nested_exception(self): def f(p): try: 1/0 - except: pass + except ZeroDivisionError: pass f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -156,9 +156,9 @@ def f(p): def g(p): try: f(p) - except: + except ZeroDivisionError: try: f(p) - except: pass + except ZeroDivisionError: pass f_ident = ident(f) g_ident = ident(g) self.check_events(g, [(1, 'call', g_ident), @@ -187,7 +187,7 @@ def g(p): def test_raise_twice(self): def f(p): try: 1/0 - except: 1/0 + except ZeroDivisionError: 1/0 f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -196,7 +196,7 @@ def f(p): def test_raise_reraise(self): def f(p): try: 1/0 - except: raise + except ZeroDivisionError: raise f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -320,7 +320,7 @@ def f(p): def test_caught_exception(self): def f(p): try: 1/0 - except: pass + except ZeroDivisionError: pass f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -479,6 +479,20 @@ def f(): sys.setprofile(lambda *args: None) f() + def test_method_with_c_function(self): + # gh-122029 + # When we have a PyMethodObject whose im_func is a C function, we + # should record both the call and the return. f = classmethod(repr) + # is just a way to create a PyMethodObject with a C function. + class A: + f = classmethod(repr) + events = [] + sys.setprofile(lambda frame, event, args: events.append(event)) + A().f() + sys.setprofile(None) + # The last c_call is the call to sys.setprofile + self.assertEqual(events, ['c_call', 'c_return', 'c_call']) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 125f40227118f6..95cf0d1ec2d9ab 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1650,15 +1650,15 @@ def func(): EXPECTED_EVENTS = [ (0, 'call'), (2, 'line'), - (1, 'line'), (-3, 'call'), (-2, 'line'), (-2, 'return'), - (4, 'line'), (1, 'line'), + (4, 'line'), + (2, 'line'), (-2, 'call'), (-2, 'return'), - (1, 'return'), + (2, 'return'), ] # C level events should be the same as expected and the same as Python level. @@ -2857,7 +2857,7 @@ def test_no_jump_from_exception_event(output): output.append(1) 1 / 0 - @jump_test(3, 2, [2, 5], event='return') + @jump_test(3, 2, [2, 2, 5], event='return') def test_jump_from_yield(output): def gen(): output.append(2) @@ -3039,7 +3039,7 @@ def test_trace_unpack_long_sequence(self): def test_trace_lots_of_globals(self): - count = min(1000, int(support.Py_C_RECURSION_LIMIT * 0.8)) + count = min(1000, int(support.get_c_recursion_limit() * 0.8)) code = """if 1: def f(): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index bb87bf00dc2d1a..1ade49281b4e26 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,3 +1,5 @@ +import platform +import re import unittest import sys import os @@ -6,7 +8,11 @@ from copy import copy from test.support import ( - captured_stdout, PythonSymlink, requires_subprocess, is_wasi + captured_stdout, + is_apple_mobile, + is_wasi, + PythonSymlink, + requires_subprocess, ) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, @@ -151,7 +157,7 @@ def test_posix_venv_scheme(self): binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') # Resolve the paths in an imaginary venv/ directory @@ -226,6 +232,11 @@ def test_get_config_vars(self): self.assertTrue(cvars) def test_get_platform(self): + # Check the actual platform returns something reasonable. + actual_platform = get_platform() + self.assertIsInstance(actual_platform, str) + self.assertTrue(actual_platform) + # windows XP, 32bits os.name = 'nt' sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) ' @@ -341,9 +352,26 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'linux-i686') + # Android + os.name = 'posix' + sys.platform = 'android' + get_config_vars()['ANDROID_API_LEVEL'] = 9 + for machine, abi in { + 'x86_64': 'x86_64', + 'i686': 'x86', + 'aarch64': 'arm64_v8a', + 'armv7l': 'armeabi_v7a', + }.items(): + with self.subTest(machine): + self._set_uname(('Linux', 'localhost', '3.18.91+', + '#1 Tue Jan 9 20:35:43 UTC 2018', machine)) + self.assertEqual(get_platform(), f'android-9-{abi}') + # XXX more platforms to tests here @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't distribute header files in the runtime environment") def test_get_config_h_filename(self): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) @@ -389,8 +417,8 @@ def test_user_similar(self): if name == 'platlib': # Replace "/lib64/python3.11/site-packages" suffix # with "/lib/python3.11/site-packages". - py_version_short = sysconfig.get_python_version() - suffix = f'python{py_version_short}/site-packages' + py_version_abi = sysconfig._get_python_version_abi() + suffix = f'python{py_version_abi}/site-packages' expected = expected.replace(f'/{sys.platlibdir}/{suffix}', f'/lib/{suffix}') self.assertEqual(user_path, expected) @@ -421,6 +449,9 @@ def test_library(self): self.assertTrue(library.startswith(f'python{major}{minor}')) self.assertTrue(library.endswith('.dll')) self.assertEqual(library, ldlibrary) + elif is_apple_mobile: + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + self.assertEqual(ldlibrary, f"{framework}.framework/{framework}") else: self.assertTrue(library.startswith(f'libpython{major}.{minor}')) self.assertTrue(library.endswith('.a')) @@ -474,6 +505,8 @@ def test_platform_in_subprocess(self): self.assertEqual(my_platform, test_platform) @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') @@ -516,12 +549,9 @@ def test_EXT_SUFFIX_in_vars(self): vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - @unittest.skipUnless(sys.platform == 'linux' and - hasattr(sys.implementation, '_multiarch'), - 'multiarch-specific test') - def test_triplet_in_ext_suffix(self): + @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + def test_linux_ext_suffix(self): ctypes = import_module('ctypes') - import platform, re machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') if re.match('(aarch64|arm|mips|ppc|powerpc|s390|sparc)', machine): @@ -534,6 +564,19 @@ def test_triplet_in_ext_suffix(self): self.assertTrue(suffix.endswith(expected_suffixes), f'unexpected suffix {suffix!r}') + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') + def test_android_ext_suffix(self): + machine = platform.machine() + suffix = sysconfig.get_config_var('EXT_SUFFIX') + expected_triplet = { + "x86_64": "x86_64-linux-android", + "i686": "i686-linux-android", + "aarch64": "aarch64-linux-android", + "armv7l": "arm-linux-androideabi", + }[machine] + self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), + f"{machine=}, {suffix=}") + @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') @@ -544,6 +587,8 @@ class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_get_makefile_filename(self): makefile = sysconfig.get_makefile_filename() self.assertTrue(os.path.isfile(makefile), makefile) diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index cc122cafc7985c..30dcb3e3c4f4f9 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -315,7 +315,7 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failur def test_with_errored_file(self): """Should displays error when errored python file is given.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: - stderr = f"{file_path!r}: Token Error: " + stderr = f"{file_path!r}: Indentation Error: " stderr += ('unindent does not match any outer indentation level' ' (, line 3)') self.validate_cmd(file_path, stderr=stderr, expect_failure=True) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 51f070e96047a6..54d329a15d4d25 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -386,7 +386,7 @@ def test_is_tarfile_erroneous(self): self.assertFalse(tarfile.is_tarfile(tmpname)) # is_tarfile works on path-like objects - self.assertFalse(tarfile.is_tarfile(pathlib.Path(tmpname))) + self.assertFalse(tarfile.is_tarfile(os_helper.FakePath(tmpname))) # is_tarfile works on file objects with open(tmpname, "rb") as fobj: @@ -400,7 +400,7 @@ def test_is_tarfile_valid(self): self.assertTrue(tarfile.is_tarfile(self.tarname)) # is_tarfile works on path-like objects - self.assertTrue(tarfile.is_tarfile(pathlib.Path(self.tarname))) + self.assertTrue(tarfile.is_tarfile(os_helper.FakePath(self.tarname))) # is_tarfile works on file objects with open(self.tarname, "rb") as fobj: @@ -507,19 +507,36 @@ def test_length_zero_header(self): with tarfile.open(support.findfile('recursion.tar', subdir='archivetestdata')): pass - def test_extractfile_name(self): + def test_extractfile_attrs(self): # gh-74468: TarFile.name must name a file, not a parent archive. file = self.tar.getmember('ustar/regtype') with self.tar.extractfile(file) as fobj: self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) + self.assertIs(fobj.closed, False) + self.assertIs(fobj.closed, True) + self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) class MiscReadTestBase(CommonReadTest): - def requires_name_attribute(self): - pass + is_stream = False def test_no_name_argument(self): - self.requires_name_attribute() with open(self.tarname, "rb") as fobj: self.assertIsInstance(fobj.name, str) with tarfile.open(fileobj=fobj, mode=self.mode) as tar: @@ -552,7 +569,6 @@ def test_int_name_attribute(self): self.assertIsNone(tar.name) def test_bytes_name_attribute(self): - self.requires_name_attribute() tarname = os.fsencode(self.tarname) with open(tarname, 'rb') as fobj: self.assertIsInstance(fobj.name, bytes) @@ -560,21 +576,23 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) - def test_pathlike_name(self): - tarname = pathlib.Path(self.tarname) + def test_pathlike_name(self, tarname=None): + if tarname is None: + tarname = self.tarname + expected = os.path.abspath(tarname) + tarname = os_helper.FakePath(tarname) with tarfile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with self.taropen(tarname) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with tarfile.TarFile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) if self.suffix == '': with tarfile.TarFile(tarname, mode='r') as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) + + def test_pathlike_bytes_name(self): + self.test_pathlike_name(os.fsencode(self.tarname)) def test_illegal_mode_arg(self): with open(tmpname, 'wb'): @@ -704,6 +722,24 @@ def format_mtime(mtime): tar.close() os_helper.rmtree(DIR) + @staticmethod + def test_extractall_default_filter(): + # Test that the default filter is now "data", and the other filter types are not used. + DIR = pathlib.Path(TEMPDIR) / "extractall_default_filter" + with ( + os_helper.temp_dir(DIR), + tarfile.open(tarname, encoding="iso8859-1") as tar, + unittest.mock.patch("tarfile.data_filter", wraps=tarfile.data_filter) as mock_data_filter, + unittest.mock.patch("tarfile.tar_filter", wraps=tarfile.tar_filter) as mock_tar_filter, + unittest.mock.patch("tarfile.fully_trusted_filter", wraps=tarfile.fully_trusted_filter) as mock_ft_filter + ): + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + + mock_data_filter.assert_called() + mock_ft_filter.assert_not_called() + mock_tar_filter.assert_not_called() + @os_helper.skip_unless_working_chmod def test_extract_directory(self): dirtype = "ustar/dirtype" @@ -720,24 +756,24 @@ def test_extract_directory(self): finally: os_helper.rmtree(DIR) - def test_extractall_pathlike_name(self): - DIR = pathlib.Path(TEMPDIR) / "extractall" + def test_extractall_pathlike_dir(self): + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: directories = [t for t in tar if t.isdir()] - tar.extractall(DIR, directories, filter='fully_trusted') + tar.extractall(os_helper.FakePath(DIR), directories, filter='fully_trusted') for tarinfo in directories: - path = DIR / tarinfo.name + path = os.path.join(DIR, tarinfo.name) self.assertEqual(os.path.getmtime(path), tarinfo.mtime) - def test_extract_pathlike_name(self): + def test_extract_pathlike_dir(self): dirtype = "ustar/dirtype" - DIR = pathlib.Path(TEMPDIR) / "extractall" + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: tarinfo = tar.getmember(dirtype) - tar.extract(tarinfo, path=DIR, filter='fully_trusted') - extracted = DIR / dirtype + tar.extract(tarinfo, path=os_helper.FakePath(DIR), filter='fully_trusted') + extracted = os.path.join(DIR, dirtype) self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) def test_init_close_fobj(self): @@ -796,17 +832,16 @@ class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("BZ2File have no name attribute") + pass class LzmaMiscReadTest(LzmaTest, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("LZMAFile have no name attribute") + pass class StreamReadTest(CommonReadTest, unittest.TestCase): prefix="r|" + is_stream = True def test_read_through(self): # Issue #11224: A poorly designed _FileInFile.read() method @@ -1167,7 +1202,7 @@ def _fs_supports_holes(): # # The function returns False if page size is larger than 4 KiB. # For example, ppc64 uses pages of 64 KiB. - if sys.platform.startswith("linux"): + if sys.platform.startswith(("linux", "android")): # Linux evidentially has 512 byte st_blocks units. name = os.path.join(TEMPDIR, "sparse-test") with open(name, "wb") as fobj: @@ -1226,6 +1261,48 @@ def test_pax_number_fields(self): finally: tar.close() + def test_pax_header_bad_formats(self): + # The fields from the pax header have priority over the + # TarInfo. + pax_header_replacements = ( + b" foo=bar\n", + b"0 \n", + b"1 \n", + b"2 \n", + b"3 =\n", + b"4 =a\n", + b"1000000 foo=bar\n", + b"0 foo=bar\n", + b"-12 foo=bar\n", + b"000000000000000000000000036 foo=bar\n", + ) + pax_headers = {"foo": "bar"} + + for replacement in pax_header_replacements: + with self.subTest(header=replacement): + tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT, + encoding="iso8859-1") + try: + t = tarfile.TarInfo() + t.name = "pax" # non-ASCII + t.uid = 1 + t.pax_headers = pax_headers + tar.addfile(t) + finally: + tar.close() + + with open(tmpname, "rb") as f: + data = f.read() + self.assertIn(b"11 foo=bar\n", data) + data = data.replace(b"11 foo=bar\n", replacement) + + with open(tmpname, "wb") as f: + f.truncate() + f.write(data) + + with self.assertRaisesRegex(tarfile.ReadError, r"method tar: ReadError\('invalid header'\)"): + tarfile.open(tmpname, encoding="iso8859-1") + class WriteTestBase(TarTest): # Put all write tests in here that are supposed to be tested @@ -1350,11 +1427,11 @@ def test_ordered_recursion(self): def test_gettarinfo_pathlike_name(self): with tarfile.open(tmpname, self.mode) as tar: - path = pathlib.Path(TEMPDIR) / "file" + path = os.path.join(TEMPDIR, "file") with open(path, "wb") as fobj: fobj.write(b"aaa") - tarinfo = tar.gettarinfo(path) - tarinfo2 = tar.gettarinfo(os.fspath(path)) + tarinfo = tar.gettarinfo(os_helper.FakePath(path)) + tarinfo2 = tar.gettarinfo(path) self.assertIsInstance(tarinfo.name, str) self.assertEqual(tarinfo.name, tarinfo2.name) self.assertEqual(tarinfo.size, 3) @@ -1568,6 +1645,12 @@ def write(self, data): pax_headers={'non': 'empty'}) self.assertFalse(f.closed) + def test_missing_fileobj(self): + with tarfile.open(tmpname, self.mode) as tar: + tarinfo = tar.gettarinfo(tarname) + with self.assertRaises(ValueError): + tar.addfile(tarinfo) + class GzipWriteTest(GzipTest, WriteTest): pass @@ -1901,10 +1984,10 @@ def test_create_existing_taropen(self): self.assertIn("spameggs42", names[0]) def test_create_pathlike_name(self): - with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + with tarfile.open(os_helper.FakePath(tmpname), self.mode) as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) @@ -1915,10 +1998,10 @@ def test_create_pathlike_name(self): self.assertIn('spameggs42', names[0]) def test_create_taropen_pathlike_name(self): - with self.taropen(pathlib.Path(tmpname), "x") as tobj: + with self.taropen(os_helper.FakePath(tmpname), "x") as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) @@ -3239,7 +3322,8 @@ def test_add(self): tar = tarfile.open(fileobj=bio, mode='w', format=tarformat) tarinfo = tar.gettarinfo(tarname) try: - tar.addfile(tarinfo) + with open(tarname, 'rb') as f: + tar.addfile(tarinfo, f) except Exception: if tarformat == tarfile.USTAR_FORMAT: # In the old, limited format, adding might fail for @@ -3254,7 +3338,8 @@ def test_add(self): replaced = tarinfo.replace(**{attr_name: None}) with self.assertRaisesRegex(ValueError, f"{attr_name}"): - tar.addfile(replaced) + with open(tarname, 'rb') as f: + tar.addfile(replaced, f) def test_list(self): # Change some metadata to None, then compare list() output @@ -3961,15 +4046,6 @@ def test_data_filter(self): self.assertIs(filtered.name, tarinfo.name) self.assertIs(filtered.type, tarinfo.type) - def test_default_filter_warns(self): - """Ensure the default filter warns""" - with ArchiveMaker() as arc: - arc.add('foo') - with warnings_helper.check_warnings( - ('Python 3.14', DeprecationWarning)): - with self.check_context(arc.open(), None): - self.expect_file('foo') - def test_change_default_filter_on_instance(self): tar = tarfile.TarFile(tarname, 'r') def strict_filter(tarinfo, path): diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index ebdb58f91d3d8a..d479f7d7515d9b 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -51,7 +51,7 @@ def test_eval_null_in_result(self): def test_eval_surrogates_in_result(self): tcl = self.interp - self.assertIn(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') + self.assertEqual(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') def testEvalException(self): tcl = self.interp @@ -61,11 +61,30 @@ def testEvalException2(self): tcl = self.interp self.assertRaises(TclError,tcl.eval,'this is wrong') + def test_eval_returns_tcl_obj(self): + tcl = self.interp.tk + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + a = tcl.eval('set a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + self.assertEqual(a, expected) + def testCall(self): tcl = self.interp tcl.call('set','a','1') self.assertEqual(tcl.call('set','a'),'1') + def test_call_passing_null(self): + tcl = self.interp + tcl.call('set', 'a', 'a\0b') # ASCII-only + self.assertEqual(tcl.getvar('a'), 'a\x00b') + self.assertEqual(tcl.call('set', 'a'), 'a\x00b') + self.assertEqual(tcl.eval('set a'), 'a\x00b') + + tcl.call('set', 'a', '\u20ac\0') # non-ASCII + self.assertEqual(tcl.getvar('a'), '\u20ac\x00') + self.assertEqual(tcl.call('set', 'a'), '\u20ac\x00') + self.assertEqual(tcl.eval('set a'), '\u20ac\x00') + def testCallException(self): tcl = self.interp self.assertRaises(TclError,tcl.call,'set','a') @@ -74,11 +93,35 @@ def testCallException2(self): tcl = self.interp self.assertRaises(TclError,tcl.call,'this','is','wrong') + def test_call_returns_tcl_obj(self): + tcl = self.interp.tk + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + a = tcl.call('set', 'a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + if self.wantobjects: + self.assertEqual(str(a), expected) + self.assertEqual(a.string, expected) + self.assertEqual(a.typename, 'regexp') + else: + self.assertEqual(a, expected) + def testSetVar(self): tcl = self.interp tcl.setvar('a','1') self.assertEqual(tcl.eval('set a'),'1') + def test_setvar_passing_null(self): + tcl = self.interp + tcl.setvar('a', 'a\0b') # ASCII-only + self.assertEqual(tcl.getvar('a'), 'a\x00b') + self.assertEqual(tcl.call('set', 'a'), 'a\x00b') + self.assertEqual(tcl.eval('set a'), 'a\x00b') + + tcl.setvar('a', '\u20ac\0') # non-ASCII + self.assertEqual(tcl.getvar('a'), '\u20ac\x00') + self.assertEqual(tcl.call('set', 'a'), '\u20ac\x00') + self.assertEqual(tcl.eval('set a'), '\u20ac\x00') + def testSetVarArray(self): tcl = self.interp tcl.setvar('a(1)','1') @@ -102,6 +145,18 @@ def testGetVarArrayException(self): tcl = self.interp self.assertRaises(TclError,tcl.getvar,'a(1)') + def test_getvar_returns_tcl_obj(self): + tcl = self.interp.tk + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + a = tcl.getvar('a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + if self.wantobjects: + self.assertEqual(str(a), expected) + self.assertEqual(a.string, expected) + self.assertEqual(a.typename, 'regexp') + else: + self.assertEqual(a, expected) + def testUnsetVar(self): tcl = self.interp tcl.setvar('a',1) @@ -219,10 +274,18 @@ def test_evalfile_surrogates_in_result(self): with open(filename, 'wb') as f: f.write(b""" set a "<\xed\xa0\xbd\xed\xb2\xbb>" + """) + if tcl_version >= (9, 0): + self.assertRaises(TclError, tcl.evalfile, filename) + else: + tcl.evalfile(filename) + self.assertEqual(tcl.eval('set a'), '<\U0001f4bb>') + + with open(filename, 'wb') as f: + f.write(b""" set b "<\\ud83d\\udcbb>" """) tcl.evalfile(filename) - self.assertEqual(tcl.eval('set a'), '<\U0001f4bb>') self.assertEqual(tcl.eval('set b'), '<\U0001f4bb>') def testEvalFileException(self): @@ -482,29 +545,36 @@ def testfunc(arg): return arg self.interp.createcommand('testfunc', testfunc) self.addCleanup(self.interp.tk.deletecommand, 'testfunc') - def check(value, expected=None, *, eq=self.assertEqual): - if expected is None: - expected = value + def check(value, expected1=None, expected2=None, *, eq=self.assertEqual): + expected = value + if self.wantobjects >= 2: + if expected2 is not None: + expected = expected2 + expected_type = type(expected) + else: + if expected1 is not None: + expected = expected1 + expected_type = str nonlocal result result = None r = self.interp.call('testfunc', value) - self.assertIsInstance(result, str) + self.assertIsInstance(result, expected_type) eq(result, expected) - self.assertIsInstance(r, str) + self.assertIsInstance(r, expected_type) eq(r, expected) def float_eq(actual, expected): self.assertAlmostEqual(float(actual), expected, delta=abs(expected) * 1e-10) - check(True, '1') - check(False, '0') + check(True, '1', 1) + check(False, '0', 0) check('string') check('string\xbd') check('string\u20ac') check('string\U0001f4bb') if sys.platform != 'win32': - check('<\udce2\udc82\udcac>', '<\u20ac>') - check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>') + check('<\udce2\udc82\udcac>', '<\u20ac>', '<\u20ac>') + check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>', '<\U0001f4bb>') check('') check(b'string', 'string') check(b'string\xe2\x82\xac', 'string\xe2\x82\xac') @@ -526,9 +596,31 @@ def float_eq(actual, expected): check(float('inf'), eq=float_eq) check(-float('inf'), eq=float_eq) # XXX NaN representation can be not parsable by float() - check((), '') - check((1, (2,), (3, 4), '5 6', ()), '1 2 {3 4} {5 6} {}') - check([1, [2,], [3, 4], '5 6', []], '1 2 {3 4} {5 6} {}') + check((), '', '') + check((1, (2,), (3, 4), '5 6', ()), + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) + check([1, [2,], [3, 4], '5 6', []], + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) + + def test_passing_tcl_obj(self): + tcl = self.interp.tk + a = None + def testfunc(arg): + nonlocal a + a = arg + self.interp.createcommand('testfunc', testfunc) + self.addCleanup(self.interp.tk.deletecommand, 'testfunc') + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + tcl.eval(r'testfunc $a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + if self.wantobjects >= 2: + self.assertEqual(str(a), expected) + self.assertEqual(a.string, expected) + self.assertEqual(a.typename, 'regexp') + else: + self.assertEqual(a, expected) def test_splitlist(self): splitlist = self.interp.tk.splitlist @@ -654,6 +746,7 @@ def test_new_tcl_obj(self): support.check_disallow_instantiation(self, _tkinter.TkttType) support.check_disallow_instantiation(self, _tkinter.TkappType) + class BigmemTclTest(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index b64b6a4f2baeb5..a5e182cef23dc5 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -13,6 +13,7 @@ import weakref import gc import shutil +import subprocess from unittest import mock import unittest @@ -62,16 +63,10 @@ def test_infer_return_type_multiples_and_none(self): tempfile._infer_return_type(b'', None, '') def test_infer_return_type_pathlib(self): - self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/'))) + self.assertIs(str, tempfile._infer_return_type(os_helper.FakePath('/'))) def test_infer_return_type_pathlike(self): - class Path: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - + Path = os_helper.FakePath self.assertIs(str, tempfile._infer_return_type(Path('/'))) self.assertIs(bytes, tempfile._infer_return_type(Path(b'/'))) self.assertIs(str, tempfile._infer_return_type('', Path(''))) @@ -442,7 +437,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir).write(b"blat") - self.do_create(dir=pathlib.Path(dir)).write(b"blat") + self.do_create(dir=os_helper.FakePath(dir)).write(b"blat") finally: support.gc_collect() # For PyPy or other GCs. os.rmdir(dir) @@ -680,7 +675,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir) - self.do_create(dir=pathlib.Path(dir)) + self.do_create(dir=os_helper.FakePath(dir)) finally: os.rmdir(dir) @@ -781,7 +776,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: os.rmdir(self.do_create(dir=dir)) - os.rmdir(self.do_create(dir=pathlib.Path(dir))) + os.rmdir(self.do_create(dir=os_helper.FakePath(dir))) finally: os.rmdir(dir) @@ -803,6 +798,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # Use icacls.exe to extract the users with some level of access + # Main thing we are testing is that the BUILTIN\Users group has + # no access. The exact ACL is going to vary based on which user + # is running the test. + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir], encoding="oem").casefold() + finally: + os.rmdir(dir) + + dir = dir.casefold() + users = set() + found_user = False + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line.removeprefix(dir).strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".casefold(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py index 58698ffac2d981..d6eb00175b4c85 100644 --- a/Lib/test/test_termios.py +++ b/Lib/test/test_termios.py @@ -3,6 +3,7 @@ import sys import tempfile import unittest +from test import support from test.support.import_helper import import_module termios = import_module('termios') @@ -18,10 +19,16 @@ def setUp(self): tmp = self.enterContext(tempfile.TemporaryFile(mode='wb', buffering=0)) self.bad_fd = tmp.fileno() - def assertRaisesTermiosError(self, errno, callable, *args): + def assertRaisesTermiosError(self, err, callable, *args): + # Some versions of Android return EACCES when calling termios functions + # on a regular file. + errs = [err] + if sys.platform == 'android' and err == errno.ENOTTY: + errs.append(errno.EACCES) + with self.assertRaises(termios.error) as cm: callable(*args) - self.assertEqual(cm.exception.args[0], errno) + self.assertIn(cm.exception.args[0], errs) def test_tcgetattr(self): attrs = termios.tcgetattr(self.fd) @@ -90,16 +97,18 @@ def test_tcsetattr_errors(self): self.assertRaises(TypeError, termios.tcsetattr, object(), termios.TCSANOW, attrs) self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW) + @support.skip_android_selinux('tcsendbreak') def test_tcsendbreak(self): try: termios.tcsendbreak(self.fd, 1) except termios.error as exc: - if exc.args[0] == errno.ENOTTY and sys.platform.startswith('freebsd'): + if exc.args[0] == errno.ENOTTY and sys.platform.startswith(('freebsd', "netbsd")): self.skipTest('termios.tcsendbreak() is not supported ' 'with pseudo-terminals (?) on this platform') raise termios.tcsendbreak(self.stream, 1) + @support.skip_android_selinux('tcsendbreak') def test_tcsendbreak_errors(self): self.assertRaises(OverflowError, termios.tcsendbreak, self.fd, 2**1000) self.assertRaises(TypeError, termios.tcsendbreak, self.fd, 0.0) @@ -110,10 +119,12 @@ def test_tcsendbreak_errors(self): self.assertRaises(TypeError, termios.tcsendbreak, object(), 0) self.assertRaises(TypeError, termios.tcsendbreak, self.fd) + @support.skip_android_selinux('tcdrain') def test_tcdrain(self): termios.tcdrain(self.fd) termios.tcdrain(self.stream) + @support.skip_android_selinux('tcdrain') def test_tcdrain_errors(self): self.assertRaisesTermiosError(errno.ENOTTY, termios.tcdrain, self.bad_fd) self.assertRaises(ValueError, termios.tcdrain, -1) @@ -136,12 +147,14 @@ def test_tcflush_errors(self): self.assertRaises(TypeError, termios.tcflush, object(), termios.TCIFLUSH) self.assertRaises(TypeError, termios.tcflush, self.fd) + @support.skip_android_selinux('tcflow') def test_tcflow(self): termios.tcflow(self.fd, termios.TCOOFF) termios.tcflow(self.fd, termios.TCOON) termios.tcflow(self.fd, termios.TCIOFF) termios.tcflow(self.fd, termios.TCION) + @support.skip_android_selinux('tcflow') def test_tcflow_errors(self): self.assertRaisesTermiosError(errno.EINVAL, termios.tcflow, self.fd, -1) self.assertRaises(OverflowError, termios.tcflow, self.fd, 2**1000) @@ -211,6 +224,15 @@ def test_constants(self): self.assertLess(termios.VTIME, termios.NCCS) self.assertLess(termios.VMIN, termios.NCCS) + def test_ioctl_constants(self): + # gh-119770: ioctl() constants must be positive + for name in dir(termios): + if not name.startswith('TIO'): + continue + value = getattr(termios, name) + with self.subTest(name=name): + self.assertGreaterEqual(value, 0) + def test_exception(self): self.assertTrue(issubclass(termios.error, Exception)) self.assertFalse(issubclass(termios.error, OSError)) diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 931cb4b797e0b2..d94e04250c9307 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -189,8 +189,8 @@ def task(): with threading_helper.wait_threads_exit(): handle = thread.start_joinable_thread(task) handle.join() - with self.assertRaisesRegex(ValueError, "not joinable"): - handle.join() + # Subsequent join() calls should succeed + handle.join() def test_joinable_not_joined(self): handle_destroyed = thread.allocate_lock() @@ -233,58 +233,109 @@ def task(): with self.assertRaisesRegex(RuntimeError, "Cannot join current thread"): raise errors[0] - def test_detach_from_self(self): - errors = [] - handles = [] - start_joinable_thread_returned = thread.allocate_lock() - start_joinable_thread_returned.acquire() - thread_detached = thread.allocate_lock() - thread_detached.acquire() + def test_join_then_self_join(self): + # make sure we can't deadlock in the following scenario with + # threads t0 and t1 (see comment in `ThreadHandle_join()` for more + # details): + # + # - t0 joins t1 + # - t1 self joins + def make_lock(): + lock = thread.allocate_lock() + lock.acquire() + return lock + + error = None + self_joiner_handle = None + self_joiner_started = make_lock() + self_joiner_barrier = make_lock() + def self_joiner(): + nonlocal error + + self_joiner_started.release() + self_joiner_barrier.acquire() - def task(): - start_joinable_thread_returned.acquire() try: - handles[0].detach() + self_joiner_handle.join() except Exception as e: - errors.append(e) - finally: - thread_detached.release() + error = e + + joiner_started = make_lock() + def joiner(): + joiner_started.release() + self_joiner_handle.join() with threading_helper.wait_threads_exit(): - handle = thread.start_joinable_thread(task) - handles.append(handle) - start_joinable_thread_returned.release() - thread_detached.acquire() - with self.assertRaisesRegex(ValueError, "not joinable"): - handle.join() + self_joiner_handle = thread.start_joinable_thread(self_joiner) + # Wait for the self-joining thread to start + self_joiner_started.acquire() + + # Start the thread that joins the self-joiner + joiner_handle = thread.start_joinable_thread(joiner) + + # Wait for the joiner to start + joiner_started.acquire() + + # Not great, but I don't think there's a deterministic way to make + # sure that the self-joining thread has been joined. + time.sleep(0.1) - assert len(errors) == 0 + # Unblock the self-joiner + self_joiner_barrier.release() - def test_detach_then_join(self): + self_joiner_handle.join() + joiner_handle.join() + + with self.assertRaisesRegex(RuntimeError, "Cannot join current thread"): + raise error + + def test_join_with_timeout(self): lock = thread.allocate_lock() lock.acquire() - def task(): + def thr(): lock.acquire() with threading_helper.wait_threads_exit(): - handle = thread.start_joinable_thread(task) - # detach() returns even though the thread is blocked on lock - handle.detach() - # join() then cannot be called anymore - with self.assertRaisesRegex(ValueError, "not joinable"): - handle.join() + handle = thread.start_joinable_thread(thr) + handle.join(0.1) + self.assertFalse(handle.is_done()) lock.release() + handle.join() + self.assertTrue(handle.is_done()) - def test_join_then_detach(self): - def task(): + def test_join_unstarted(self): + handle = thread._ThreadHandle() + with self.assertRaisesRegex(RuntimeError, "thread not started"): + handle.join() + + def test_set_done_unstarted(self): + handle = thread._ThreadHandle() + with self.assertRaisesRegex(RuntimeError, "thread not started"): + handle._set_done() + + def test_start_duplicate_handle(self): + lock = thread.allocate_lock() + lock.acquire() + + def func(): + lock.acquire() + + handle = thread._ThreadHandle() + with threading_helper.wait_threads_exit(): + thread.start_joinable_thread(func, handle=handle) + with self.assertRaisesRegex(RuntimeError, "thread already started"): + thread.start_joinable_thread(func, handle=handle) + lock.release() + handle.join() + + def test_start_with_none_handle(self): + def func(): pass with threading_helper.wait_threads_exit(): - handle = thread.start_joinable_thread(task) + handle = thread.start_joinable_thread(func, handle=None) handle.join() - with self.assertRaisesRegex(ValueError, "not joinable"): - handle.detach() class Barrier: diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 1ab223b81e939e..3ca5f5ce1b7068 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,10 +3,11 @@ """ import test.support -from test.support import threading_helper, requires_subprocess +from test.support import threading_helper, requires_subprocess, requires_gil_enabled from test.support import verbose, cpython_only, os_helper from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support import force_not_colorized import random import sys @@ -47,6 +48,8 @@ def skip_unless_reliable_fork(test): return unittest.skip("due to known OS bug related to thread+fork")(test) if support.HAVE_ASAN_FORK_BUG: return unittest.skip("libasan has a pthread_create() dead lock related to thread+fork")(test) + if support.check_sanitizer(thread=True): + return unittest.skip("TSAN doesn't support threads after fork")(test) return test @@ -406,7 +409,7 @@ def run(self): def test_limbo_cleanup(self): # Issue 7481: Failure to start thread should cleanup the limbo map. - def fail_new_thread(*args): + def fail_new_thread(*args, **kwargs): raise threading.ThreadError() _start_joinable_thread = threading._start_joinable_thread threading._start_joinable_thread = fail_new_thread @@ -423,6 +426,10 @@ def test_finalize_running_thread(self): # Issue 1402: the PyGILState_Ensure / _Release functions may be called # very late on python exit: on deallocation of a running thread for # example. + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") import_module("ctypes") rc, out, err = assert_python_failure("-c", """if 1: @@ -455,6 +462,11 @@ def waitingThread(): def test_finalize_with_trace(self): # Issue1733757 # Avoid a deadlock when sys.settrace steps into threading._shutdown + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(2)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + assert_python_ok("-c", """if 1: import sys, threading @@ -504,7 +516,7 @@ def test_enumerate_after_join(self): old_interval = sys.getswitchinterval() try: for i in range(1, 100): - sys.setswitchinterval(i * 0.0002) + support.setswitchinterval(i * 0.0002) t = threading.Thread(target=lambda: None) t.start() t.join() @@ -769,8 +781,7 @@ def func(): "current is main True\n" ) - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - @support.requires_fork() + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork_from_foreign_thread(self, create_dummy=False): code = """if 1: @@ -901,41 +912,6 @@ def f(): rc, out, err = assert_python_ok("-c", code) self.assertEqual(err, b"") - def test_tstate_lock(self): - # Test an implementation detail of Thread objects. - started = _thread.allocate_lock() - finish = _thread.allocate_lock() - started.acquire() - finish.acquire() - def f(): - started.release() - finish.acquire() - time.sleep(0.01) - # The tstate lock is None until the thread is started - t = threading.Thread(target=f) - self.assertIs(t._tstate_lock, None) - t.start() - started.acquire() - self.assertTrue(t.is_alive()) - # The tstate lock can't be acquired when the thread is running - # (or suspended). - tstate_lock = t._tstate_lock - self.assertFalse(tstate_lock.acquire(timeout=0), False) - finish.release() - # When the thread ends, the state_lock can be successfully - # acquired. - self.assertTrue(tstate_lock.acquire(timeout=support.SHORT_TIMEOUT), False) - # But is_alive() is still True: we hold _tstate_lock now, which - # prevents is_alive() from knowing the thread's end-of-life C code - # is done. - self.assertTrue(t.is_alive()) - # Let is_alive() find out the C code is done. - tstate_lock.release() - self.assertFalse(t.is_alive()) - # And verify the thread disposed of _tstate_lock. - self.assertIsNone(t._tstate_lock) - t.join() - def test_repr_stopped(self): # Verify that "stopped" shows up in repr(Thread) appropriately. started = _thread.allocate_lock() @@ -983,6 +959,7 @@ def test_BoundedSemaphore_limit(self): @cpython_only def test_frame_tstate_tracing(self): + _testcapi = import_module("_testcapi") # Issue #14432: Crash when a generator is created in a C thread that is # destroyed while the generator is still used. The issue was that a # generator contains a frame, and the frame kept a reference to the @@ -1010,7 +987,6 @@ def callback(): threading.settrace(noop_trace) # Create a generator in a C thread which exits after the call - import _testcapi _testcapi.call_in_temporary_c_thread(callback) # Call the generator in a different Python thread, check that the @@ -1101,30 +1077,6 @@ def checker(): self.assertEqual(threading.getprofile(), old_profile) self.assertEqual(sys.getprofile(), old_profile) - @cpython_only - def test_shutdown_locks(self): - for daemon in (False, True): - with self.subTest(daemon=daemon): - event = threading.Event() - thread = threading.Thread(target=event.wait, daemon=daemon) - - # Thread.start() must add lock to _shutdown_locks, - # but only for non-daemon thread - thread.start() - tstate_lock = thread._tstate_lock - if not daemon: - self.assertIn(tstate_lock, threading._shutdown_locks) - else: - self.assertNotIn(tstate_lock, threading._shutdown_locks) - - # unblock the thread and join it - event.set() - thread.join() - - # Thread._stop() must remove tstate_lock from _shutdown_locks. - # Daemon threads must never add it to _shutdown_locks. - self.assertNotIn(tstate_lock, threading._shutdown_locks) - def test_locals_at_exit(self): # bpo-19466: thread locals must not be deleted before destructors # are called @@ -1202,23 +1154,93 @@ def import_threading(): self.assertEqual(out, b'') self.assertEqual(err, b'') - def test_start_new_thread_at_exit(self): + def test_start_new_thread_at_finalization(self): code = """if 1: - import atexit import _thread def f(): print("shouldn't be printed") - def exit_handler(): - _thread.start_new_thread(f, ()) - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + _thread.start_new_thread(f, ()) + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"can't create new thread at interpreter shutdown", err) + @cpython_only + def test_finalize_daemon_thread_hang(self): + if support.check_sanitizer(thread=True, memory=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest( + "https://github.com/python/cpython/issues/124878 - Known" + " race condition that TSAN identifies.") + # gh-87135: tests that daemon threads hang during finalization + script = textwrap.dedent(''' + import os + import sys + import threading + import time + import _testcapi + + lock = threading.Lock() + lock.acquire() + thread_started_event = threading.Event() + def thread_func(): + try: + thread_started_event.set() + _testcapi.finalize_thread_hang(lock.acquire) + finally: + # Control must not reach here. + os._exit(2) + + t = threading.Thread(target=thread_func) + t.daemon = True + t.start() + thread_started_event.wait() + # Sleep to ensure daemon thread is blocked on `lock.acquire` + # + # Note: This test is designed so that in the unlikely case that + # `0.1` seconds is not sufficient time for the thread to become + # blocked on `lock.acquire`, the test will still pass, it just + # won't be properly testing the thread behavior during + # finalization. + time.sleep(0.1) + + def run_during_finalization(): + # Wake up daemon thread + lock.release() + # Sleep to give the daemon thread time to crash if it is going + # to. + # + # Note: If due to an exceptionally slow execution this delay is + # insufficient, the test will still pass but will simply be + # ineffective as a test. + time.sleep(0.1) + # If control reaches here, the test succeeded. + os._exit(0) + + # Replace sys.stderr.flush as a way to run code during finalization + orig_flush = sys.stderr.flush + def do_flush(*args, **kwargs): + orig_flush(*args, **kwargs) + if not sys.is_finalizing: + return + sys.stderr.flush = orig_flush + run_during_finalization() + + sys.stderr.flush = do_flush + + # If the follow exit code is retained, `run_during_finalization` + # did not run. + sys.exit(1) + ''') + assert_python_ok("-c", script) + class ThreadJoinOnShutdown(BaseTestCase): def _run_and_join(self, script): @@ -1303,6 +1325,11 @@ def test_4_daemon_threads(self): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. + if support.check_sanitizer(thread=True): + # some of the threads running `random_io` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + script = """if True: import os import random @@ -1340,6 +1367,30 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) + def test_thread_from_thread(self): + script = """if True: + import threading + import time + + def thread2(): + time.sleep(0.05) + print("OK") + + def thread1(): + time.sleep(0.05) + t2 = threading.Thread(target=thread2) + t2.start() + + t = threading.Thread(target=thread1) + t.start() + # do not join() -- the interpreter waits for non-daemon threads to + # finish. + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"OK") + self.assertEqual(rc, 0) + @skip_unless_reliable_fork def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with @@ -1478,7 +1529,7 @@ def test_threads_join_with_no_main(self): DONE = b'D' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading import time @@ -1509,6 +1560,7 @@ def task(): @cpython_only def test_daemon_threads_fatal_error(self): + import_module("_testcapi") subinterp_code = f"""if 1: import os import threading @@ -1535,6 +1587,7 @@ def _check_allowed(self, before_start='', *, daemon_allowed=True, daemon=False, ): + import_module("_testinternalcapi") subinterp_code = textwrap.dedent(f""" import test.support import threading @@ -1544,6 +1597,7 @@ def func(): {before_start} t.start() """) + check_multi_interp_extensions = bool(support.Py_GIL_DISABLED) script = textwrap.dedent(f""" import test.support test.support.run_in_subinterp_with_config( @@ -1553,7 +1607,7 @@ def func(): allow_exec=True, allow_threads={allowed}, allow_daemon_threads={daemon_allowed}, - check_multi_interp_extensions=False, + check_multi_interp_extensions={check_multi_interp_extensions}, own_gil=False, ) """) @@ -1809,6 +1863,7 @@ def setUp(self): restore_default_excepthook(self) super().setUp() + @force_not_colorized def test_excepthook(self): with support.captured_output("stderr") as stderr: thread = ThreadRunFail(name="excepthook thread") @@ -1822,6 +1877,7 @@ def test_excepthook(self): self.assertIn('ValueError: run failed', stderr) @support.cpython_only + @force_not_colorized def test_excepthook_thread_None(self): # threading.excepthook called with thread=None: log the thread # identifier in this case. @@ -2038,6 +2094,7 @@ def check_interrupt_main_noerror(self, signum): # Restore original handler signal.signal(signum, handler) + @requires_gil_enabled("gh-118433: Flaky due to a longstanding bug") def test_interrupt_main_subthread(self): # Calling start_new_thread with a function that executes interrupt_main # should raise KeyboardInterrupt upon completion. diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py index 6a53d655015cdb..bf241ada90ea35 100644 --- a/Lib/test/test_threadsignals.py +++ b/Lib/test/test_threadsignals.py @@ -32,39 +32,28 @@ def handle_signals(sig,frame): # a function that will be spawned as a separate thread. def send_signals(): - os.kill(process_pid, signal.SIGUSR1) - os.kill(process_pid, signal.SIGUSR2) + # We use `raise_signal` rather than `kill` because: + # * It verifies that a signal delivered to a background thread still has + # its Python-level handler called on the main thread. + # * It ensures the signal is handled before the thread exits. + signal.raise_signal(signal.SIGUSR1) + signal.raise_signal(signal.SIGUSR2) signalled_all.release() @threading_helper.requires_working_threading() -@unittest.skipUnless(hasattr(signal, "alarm"), "test requires signal.alarm") class ThreadSignals(unittest.TestCase): def test_signals(self): with threading_helper.wait_threads_exit(): # Test signal handling semantics of threads. - # We spawn a thread, have the thread send two signals, and + # We spawn a thread, have the thread send itself two signals, and # wait for it to finish. Check that we got both signals # and that they were run by the main thread. signalled_all.acquire() self.spawnSignallingThread() signalled_all.acquire() - # the signals that we asked the kernel to send - # will come back, but we don't know when. - # (it might even be after the thread exits - # and might be out of order.) If we haven't seen - # the signals yet, send yet another signal and - # wait for it return. - if signal_blackboard[signal.SIGUSR1]['tripped'] == 0 \ - or signal_blackboard[signal.SIGUSR2]['tripped'] == 0: - try: - signal.alarm(1) - signal.pause() - finally: - signal.alarm(0) - self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped'], 1) self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped_by'], thread.get_ident()) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 3b5640abdb6b89..d368f08b610870 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -2,7 +2,6 @@ from test.support import warnings_helper import decimal import enum -import locale import math import platform import sys @@ -19,7 +18,7 @@ except ImportError: _testinternalcapi = None -from test.support import skip_if_buggy_ucrt_strfptime +from test.support import skip_if_buggy_ucrt_strfptime, SuppressCrashReport # Max year is only limited by the size of C int. SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 @@ -43,8 +42,8 @@ class _PyTime(enum.IntEnum): ROUND_UP = 3 # _PyTime_t is int64_t -_PyTime_MIN = -2 ** 63 -_PyTime_MAX = 2 ** 63 - 1 +PyTime_MIN = -2 ** 63 +PyTime_MAX = 2 ** 63 - 1 # Rounding modes supported by PyTime ROUNDING_MODES = ( @@ -182,8 +181,44 @@ def test_strftime(self): self.fail('conversion specifier: %r failed.' % format) self.assertRaises(TypeError, time.strftime, b'%S', tt) - # embedded null character - self.assertRaises(ValueError, time.strftime, '%S\0', tt) + + def test_strftime_invalid_format(self): + tt = time.gmtime(self.t) + with SuppressCrashReport(): + for i in range(1, 128): + format = ' %' + chr(i) + with self.subTest(format=format): + try: + time.strftime(format, tt) + except ValueError as exc: + self.assertEqual(str(exc), 'Invalid format string') + + def test_strftime_special(self): + tt = time.gmtime(self.t) + s1 = time.strftime('%c', tt) + s2 = time.strftime('%B', tt) + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(time.strftime('\U0001f40d', tt), '\U0001f40d') + self.assertEqual(time.strftime('\U0001f4bb%c\U0001f40d%B', tt), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(time.strftime('%c\U0001f4bb%B\U0001f40d', tt), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(time.strftime('\ud83d', tt), '\ud83d') + self.assertEqual(time.strftime('\udc0d', tt), '\udc0d') + self.assertEqual(time.strftime('\ud83d%c\udc0d%B', tt), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(time.strftime('%c\ud83d%B\udc0d', tt), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(time.strftime('%c\udc0d%B\ud83d', tt), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(time.strftime('\ud83d\udc0d', tt), '\ud83d\udc0d') + self.assertEqual(time.strftime('%c\ud83d\udc0d%B', tt), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(time.strftime('\udcf0\udc9f\udc90\udc8d', tt), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(time.strftime('%c\udcf0\udc9f\udc90\udc8d%B', tt), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(time.strftime('\0', tt), '\0') + self.assertEqual(time.strftime('\0'*1000, tt), '\0'*1000) + self.assertEqual(time.strftime('\0%c\0%B', tt), f'\0{s1}\0{s2}') + self.assertEqual(time.strftime('%c\0%B\0', tt), f'{s1}\0{s2}\0') def _bounds_checking(self, func): # Make sure that strftime() checks the bounds of the various parts @@ -277,6 +312,8 @@ def test_strptime(self): 'j', 'm', 'M', 'p', 'S', 'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'): format = '%' + directive + if directive == 'd': + format += ',%Y' # Avoid GH-70647. strf_output = time.strftime(format, tt) try: time.strptime(strf_output, format) @@ -297,7 +334,13 @@ def test_strptime_exception_context(self): # additional check for IndexError branch (issue #19545) with self.assertRaises(ValueError) as e: time.strptime('19', '%Y %') - self.assertIs(e.exception.__suppress_context__, True) + self.assertIsNone(e.exception.__context__) + + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + time.strptime('02-07 18:28', '%m-%d %H:%M') def test_asctime(self): time.asctime(time.gmtime(self.t)) @@ -511,7 +554,7 @@ def test_process_time(self): def test_thread_time(self): if not hasattr(time, 'thread_time'): - if sys.platform.startswith(('linux', 'win')): + if sys.platform.startswith(('linux', 'android', 'win')): self.fail("time.thread_time() should be available on %r" % (sys.platform,)) else: @@ -592,17 +635,8 @@ def test_get_clock_info(self): class TestLocale(unittest.TestCase): - def setUp(self): - self.oldloc = locale.setlocale(locale.LC_ALL) - - def tearDown(self): - locale.setlocale(locale.LC_ALL, self.oldloc) - + @support.run_with_locale('LC_ALL', 'fr_FR', '') def test_bug_3061(self): - try: - tmp = locale.setlocale(locale.LC_ALL, "fr_FR") - except locale.Error: - self.skipTest('could not set locale.LC_ALL to fr_FR') # This should not cause an exception time.strftime("%B", (2009,2,1,0,0,0,0,0,0)) @@ -646,8 +680,7 @@ def year4d(y): self.test_year('%04d', func=year4d) def skip_if_not_supported(y): - msg = "strftime() is limited to [1; 9999] with Visual Studio" - # Check that it doesn't crash for year > 9999 + msg = f"strftime() does not support year {y} on this platform" try: time.strftime('%Y', (y,) + (0,) * 8) except ValueError: @@ -934,7 +967,7 @@ def test_FromSecondsObject(self): _PyTime_FromSecondsObject(float('nan'), time_rnd) def test_AsSecondsDouble(self): - from _testinternalcapi import _PyTime_AsSecondsDouble + from _testcapi import PyTime_AsSecondsDouble def float_converter(ns): if abs(ns) % SEC_TO_NS == 0: @@ -942,15 +975,10 @@ def float_converter(ns): else: return float(ns) / SEC_TO_NS - self.check_int_rounding(lambda ns, rnd: _PyTime_AsSecondsDouble(ns), + self.check_int_rounding(lambda ns, rnd: PyTime_AsSecondsDouble(ns), float_converter, NS_TO_SEC) - # test nan - for time_rnd, _ in ROUNDING_MODES: - with self.assertRaises(TypeError): - _PyTime_AsSecondsDouble(float('nan')) - def create_decimal_converter(self, denominator): denom = decimal.Decimal(denominator) @@ -1009,7 +1037,7 @@ def test_AsTimeval_clamp(self): tv_sec_max = self.time_t_max tv_sec_min = self.time_t_min - for t in (_PyTime_MIN, _PyTime_MAX): + for t in (PyTime_MIN, PyTime_MAX): ts = _PyTime_AsTimeval_clamp(t, _PyTime.ROUND_CEILING) with decimal.localcontext() as context: context.rounding = decimal.ROUND_CEILING @@ -1028,7 +1056,7 @@ def test_AsTimeval_clamp(self): def test_AsTimespec_clamp(self): from _testinternalcapi import _PyTime_AsTimespec_clamp - for t in (_PyTime_MIN, _PyTime_MAX): + for t in (PyTime_MIN, PyTime_MAX): ts = _PyTime_AsTimespec_clamp(t) tv_sec, tv_nsec = divmod(t, NS_TO_SEC) if self.time_t_max < tv_sec: diff --git a/Lib/test/test_tkinter/test_geometry_managers.py b/Lib/test/test_tkinter/test_geometry_managers.py index 59fe592b492adc..d71a634a767310 100644 --- a/Lib/test/test_tkinter/test_geometry_managers.py +++ b/Lib/test/test_tkinter/test_geometry_managers.py @@ -10,6 +10,11 @@ requires('gui') +EXPECTED_FLOAT_ERRMSG = 'expected floating-point number but got "{}"' +EXPECTED_FLOAT_OR_EMPTY_ERRMSG = 'expected floating-point number (or "" )?but got "{}"' +EXPECTED_SCREEN_DISTANCE_ERRMSG = '(bad|expected) screen distance (but got )?"{}"' +EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG = '(bad|expected) screen distance (or "" but got )?"{}"' + class PackTest(AbstractWidgetTest, unittest.TestCase): test_keys = None @@ -317,7 +322,8 @@ def test_place_configure_x(self): self.assertEqual(f2.place_info()['x'], '-10') self.root.update() self.assertEqual(f2.winfo_x(), 190) - with self.assertRaisesRegex(TclError, 'bad screen distance "spam"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('spam')): f2.place_configure(in_=f, x='spam') def test_place_configure_y(self): @@ -334,7 +340,8 @@ def test_place_configure_y(self): self.assertEqual(f2.place_info()['y'], '-10') self.root.update() self.assertEqual(f2.winfo_y(), 110) - with self.assertRaisesRegex(TclError, 'bad screen distance "spam"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('spam')): f2.place_configure(in_=f, y='spam') def test_place_configure_relx(self): @@ -351,8 +358,7 @@ def test_place_configure_relx(self): self.assertEqual(f2.place_info()['relx'], '1') self.root.update() self.assertEqual(f2.winfo_x(), 200) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "spam"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_ERRMSG.format('spam')): f2.place_configure(in_=f, relx='spam') def test_place_configure_rely(self): @@ -369,8 +375,7 @@ def test_place_configure_rely(self): self.assertEqual(f2.place_info()['rely'], '1') self.root.update() self.assertEqual(f2.winfo_y(), 120) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "spam"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_ERRMSG.format('spam')): f2.place_configure(in_=f, rely='spam') def test_place_configure_anchor(self): @@ -391,7 +396,8 @@ def test_place_configure_width(self): f2.place_configure(width='') self.root.update() self.assertEqual(f2.winfo_width(), 30) - with self.assertRaisesRegex(TclError, 'bad screen distance "abcd"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(width='abcd') def test_place_configure_height(self): @@ -402,7 +408,8 @@ def test_place_configure_height(self): f2.place_configure(height='') self.root.update() self.assertEqual(f2.winfo_height(), 60) - with self.assertRaisesRegex(TclError, 'bad screen distance "abcd"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(height='abcd') def test_place_configure_relwidth(self): @@ -413,8 +420,7 @@ def test_place_configure_relwidth(self): f2.place_configure(relwidth='') self.root.update() self.assertEqual(f2.winfo_width(), 30) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "abcd"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(relwidth='abcd') def test_place_configure_relheight(self): @@ -425,8 +431,7 @@ def test_place_configure_relheight(self): f2.place_configure(relheight='') self.root.update() self.assertEqual(f2.winfo_height(), 60) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "abcd"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(relheight='abcd') def test_place_configure_bordermode(self): @@ -629,7 +634,8 @@ def test_grid_columnconfigure(self): self.assertEqual(self.root.grid_columnconfigure(0, 'weight'), 4) def test_grid_columnconfigure_minsize(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_columnconfigure(0, minsize='foo') self.root.grid_columnconfigure(0, minsize=10) self.assertEqual(self.root.grid_columnconfigure(0, 'minsize'), 10) @@ -646,7 +652,8 @@ def test_grid_columnconfigure_weight(self): self.assertEqual(self.root.grid_columnconfigure(0)['weight'], 3) def test_grid_columnconfigure_pad(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_columnconfigure(0, pad='foo') with self.assertRaisesRegex(TclError, 'invalid arg "-pad": ' 'should be non-negative'): @@ -683,7 +690,8 @@ def test_grid_rowconfigure(self): self.assertEqual(self.root.grid_rowconfigure(0, 'weight'), 4) def test_grid_rowconfigure_minsize(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_rowconfigure(0, minsize='foo') self.root.grid_rowconfigure(0, minsize=10) self.assertEqual(self.root.grid_rowconfigure(0, 'minsize'), 10) @@ -700,7 +708,8 @@ def test_grid_rowconfigure_weight(self): self.assertEqual(self.root.grid_rowconfigure(0)['weight'], 3) def test_grid_rowconfigure_pad(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_rowconfigure(0, pad='foo') with self.assertRaisesRegex(TclError, 'invalid arg "-pad": ' 'should be non-negative'): @@ -818,9 +827,11 @@ def test_grid_location(self): self.root.grid_location(0) with self.assertRaises(TypeError): self.root.grid_location(0, 0, 0) - with self.assertRaisesRegex(TclError, 'bad screen distance "x"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('x')): self.root.grid_location('x', 'y') - with self.assertRaisesRegex(TclError, 'bad screen distance "y"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('y')): self.root.grid_location('1c', 'y') t = self.root # de-maximize @@ -893,9 +904,5 @@ def test_grid_slaves(self): self.assertEqual(self.root.grid_slaves(row=1, column=1), [d, c]) -tests_gui = ( - PackTest, PlaceTest, GridTest, -) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index ef1c99f57c6f47..38371fe00d6eb5 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -302,7 +302,37 @@ def test_copy(self): image2 = image.copy() self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) - self.assertEqual(image.get(4, 6), image.get(4, 6)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = image.copy(from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(3, 2), image.get(3*2+2, 2*2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) def test_subsample(self): image = self.create() @@ -316,6 +346,13 @@ def test_subsample(self): self.assertEqual(image2.height(), 8) self.assertEqual(image2.get(2, 3), image.get(4, 6)) + image2 = image.subsample(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + def test_zoom(self): image = self.create() image2 = image.zoom(2, 3) @@ -330,6 +367,118 @@ def test_zoom(self): self.assertEqual(image2.get(8, 12), image.get(4, 6)) self.assertEqual(image2.get(9, 13), image.get(4, 6)) + image2 = image.zoom(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + def test_copy_replace(self): + image = self.create() + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(0, 0, 100, 50)) + self.assertEqual(image2.width(), 100) + self.assertEqual(image2.height(), 50) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12, 4+8), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12*2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2, 4+8*3), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(3*3, 2*3), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(3*3+2, 2*3+2), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) + + def checkImgTrans(self, image, expected): + actual = {(x, y) + for x in range(image.width()) + for y in range(image.height()) + if image.transparency_get(x, y)} + self.assertEqual(actual, expected) + + def test_copy_replace_compositingrule(self): + image1 = tkinter.PhotoImage(master=self.root, width=2, height=2) + image1.blank() + image1.put('black', to=(0, 0, 2, 2)) + image1.transparency_set(0, 0, True) + + # default compositingrule + image2 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image2.blank() + image2.put('white', to=(0, 0, 2, 2)) + image2.copy_replace(image1, to=(1, 1)) + self.checkImgTrans(image2, {(0, 2), (2, 0)}) + + image3 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image3.blank() + image3.put('white', to=(0, 0, 2, 2)) + image3.copy_replace(image1, to=(1, 1), compositingrule='overlay') + self.checkImgTrans(image3, {(0, 2), (2, 0)}) + + image4 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image4.blank() + image4.put('white', to=(0, 0, 2, 2)) + image4.copy_replace(image1, to=(1, 1), compositingrule='set') + self.checkImgTrans(image4, {(0, 2), (1, 1), (2, 0)}) + def test_put(self): image = self.create() image.put('{red green} {blue yellow}', to=(4, 6)) @@ -356,6 +505,50 @@ def test_get(self): self.assertRaises(tkinter.TclError, image.get, 16, 15) self.assertRaises(tkinter.TclError, image.get, 15, 16) + def test_read(self): + # Due to the Tk bug https://core.tcl-lang.org/tk/tktview/1576528 + # the -from option does not work correctly for GIF and PNG files. + # Use the PPM file for this test. + testfile = support.findfile('python.ppm', subdir='tkinterdata') + image = tkinter.PhotoImage(master=self.root, file=testfile) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + self.assertRaises(tkinter.TclError, image2.read, self.testfile, 'ppm') + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root, file=testfile) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + image2.read(testfile, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + def test_write(self): filename = os_helper.TESTFN import locale @@ -367,19 +560,17 @@ def test_write(self): image.write(filename) image2 = tkinter.PhotoImage('::img::test2', master=self.root, - format='ppm', - file=filename) + format='ppm', file=filename) self.assertEqual(str(image2), '::img::test2') self.assertEqual(image2.type(), 'photo') self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) self.assertEqual(image2.get(0, 0), image.get(0, 0)) - self.assertEqual(image2.get(15, 8), image.get(15, 8)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) image.write(filename, format='gif', from_coords=(4, 6, 6, 9)) image3 = tkinter.PhotoImage('::img::test3', master=self.root, - format='gif', - file=filename) + format='gif', file=filename) self.assertEqual(str(image3), '::img::test3') self.assertEqual(image3.type(), 'photo') self.assertEqual(image3.width(), 2) @@ -387,6 +578,67 @@ def test_write(self): self.assertEqual(image3.get(0, 0), image.get(4, 6)) self.assertEqual(image3.get(1, 2), image.get(5, 8)) + image.write(filename, background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', file=filename) + self.assertEqual(image4.get(0, 0), (255, 0, 0) if self.wantobjects else '255 0 0') + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + image.write(filename, grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', file=filename) + c = image5.get(4, 6) + if not self.wantobjects: + c = c.split() + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_data(self): + image = self.create() + + data = image.data() + self.assertIsInstance(data, tuple) + for row in data: + self.assertIsInstance(row, str) + c = image.get(4, 6) + if not self.wantobjects: + c = tuple(map(int, c.split())) + self.assertEqual(data[6].split()[4], '#%02x%02x%02x' % c) + + data = image.data('ppm') + image2 = tkinter.PhotoImage('::img::test2', master=self.root, + format='ppm', data=data) + self.assertEqual(str(image2), '::img::test2') + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + data = image.data(format='gif', from_coords=(4, 6, 6, 9)) + image3 = tkinter.PhotoImage('::img::test3', master=self.root, + format='gif', data=data) + self.assertEqual(str(image3), '::img::test3') + self.assertEqual(image3.type(), 'photo') + self.assertEqual(image3.width(), 2) + self.assertEqual(image3.height(), 3) + self.assertEqual(image3.get(0, 0), image.get(4, 6)) + self.assertEqual(image3.get(1, 2), image.get(5, 8)) + + data = image.data('ppm', background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', data=data) + self.assertEqual(image4.get(0, 0), (255, 0, 0) if self.wantobjects else '255 0 0') + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + data = image.data('ppm', grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', data=data) + c = image5.get(4, 6) + if not self.wantobjects: + c = c.split() + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_transparency(self): image = self.create() self.assertEqual(image.transparency_get(0, 0), True) diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 81a20b698a72eb..b0b9ed60040443 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -232,6 +232,46 @@ def callback(): with self.assertRaises(tkinter.TclError): root.tk.call('after', 'info', idle1) + def test_after_info(self): + root = self.root + + # No events. + self.assertEqual(root.after_info(), ()) + + # Add timer. + timer = root.after(1, lambda: 'break') + + # With no parameter, it returns a tuple of the event handler ids. + self.assertEqual(root.after_info(), (timer, )) + root.after_cancel(timer) + + timer1 = root.after(5000, lambda: 'break') + timer2 = root.after(5000, lambda: 'break') + idle1 = root.after_idle(lambda: 'break') + # Only contains new events and not 'timer'. + self.assertEqual(root.after_info(), (idle1, timer2, timer1)) + + # With a parameter returns a tuple of (script, type). + timer1_info = root.after_info(timer1) + self.assertEqual(len(timer1_info), 2) + self.assertEqual(timer1_info[1], 'timer') + idle1_info = root.after_info(idle1) + self.assertEqual(len(idle1_info), 2) + self.assertEqual(idle1_info[1], 'idle') + + root.after_cancel(timer1) + with self.assertRaises(tkinter.TclError): + root.after_info(timer1) + root.after_cancel(timer2) + with self.assertRaises(tkinter.TclError): + root.after_info(timer2) + root.after_cancel(idle1) + with self.assertRaises(tkinter.TclError): + root.after_info(idle1) + + # No events. + self.assertEqual(root.after_info(), ()) + def test_clipboard(self): root = self.root root.clipboard_clear() @@ -436,6 +476,15 @@ def test_info_patchlevel(self): self.assertEqual(vi.micro, 0) self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) + def test_embedded_null(self): + widget = tkinter.Entry(self.root) + widget.insert(0, 'abc\0def') # ASCII-only + widget.selection_range(0, 'end') + self.assertEqual(widget.selection_get(), 'abc\x00def') + widget.insert(0, '\u20ac\0') # non-ASCII + widget.selection_range(0, 'end') + self.assertEqual(widget.selection_get(), '\u20ac\0abc\x00def') + class WmTest(AbstractTkTest, unittest.TestCase): @@ -492,6 +541,284 @@ def test_wm_attribute(self): 1.0 if self.wantobjects else '1.0') +class EventTest(AbstractTkTest, unittest.TestCase): + + def test_focus(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.focus_force() + self.root.update() + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.FocusIn) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, '??') + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, '??') + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, '??') + self.assertEqual(e.y, '??') + self.assertEqual(e.x_root, '??') + self.assertEqual(e.y_root, '??') + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_configure(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.configure(height=120, borderwidth=10) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Configure) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, '??') + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, '??') + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, 150) + self.assertEqual(e.height, 100) + self.assertEqual(e.x, 0) + self.assertEqual(e.y, 0) + self.assertEqual(e.x_root, '??') + self.assertEqual(e.y_root, '??') + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_key_press(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('') + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.KeyPress) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertIsInstance(e.state, int) + self.assertNotEqual(e.state, 0) + self.assertEqual(e.char, 'z') + self.assertIsInstance(e.keycode, int) + self.assertNotEqual(e.keycode, 0) + self.assertEqual(e.keysym, 'z') + self.assertEqual(e.keysym_num, ord('z')) + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, -1 - f.winfo_rootx()) + self.assertEqual(e.y, -1 - f.winfo_rooty()) + self.assertEqual(e.x_root, -1) + self.assertEqual(e.y_root, -1) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), + f"") + + def test_event_generate_enter(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Enter) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertIs(e.focus, False) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, 100 + f.winfo_rootx()) + self.assertEqual(e.y_root, 50 + f.winfo_rooty()) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_button_press(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.ButtonPress) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, 1) + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_motion(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Motion) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0x100) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_mouse_wheel(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50, delta=-5) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.MouseWheel) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.time, 0) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, -5) + self.assertEqual(repr(e), '') + + def test_generate_event_virtual_event(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('<>', events.append) + f.focus_force() + + f.event_generate('<>', x=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.VirtualEvent) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 50) + self.assertEqual(e.y, 0) + self.assertEqual(e.x_root, f.winfo_rootx() + 50) + self.assertEqual(e.y_root, -1) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), + f"") + + class BindTest(AbstractTkTest, unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index f809c4510e3a1f..b26956930d3402 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -52,27 +52,47 @@ def test_count(self): options = ('chars', 'indices', 'lines', 'displaychars', 'displayindices', 'displaylines', 'xpixels', 'ypixels') + self.assertEqual(len(text.count('1.0', 'end', *options, return_ints=True)), 8) self.assertEqual(len(text.count('1.0', 'end', *options)), 8) - self.assertEqual(text.count('1.0', 'end', 'chars', 'lines'), (124, 4)) + self.assertEqual(text.count('1.0', 'end', 'chars', 'lines', return_ints=True), + (124, 4)) self.assertEqual(text.count('1.3', '4.5', 'chars', 'lines'), (92, 3)) + self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines', return_ints=True), + (-92, -3)) self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines'), (-92, -3)) + self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines', return_ints=True), + (0, 0)) self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines'), (0, 0)) - self.assertEqual(text.count('1.0', 'end', 'lines'), 4) - self.assertEqual(text.count('end', '1.0', 'lines'), -4) - self.assertEqual(text.count('1.3', '1.5', 'lines'), 0) - self.assertEqual(text.count('1.3', '1.3', 'lines'), 0) - self.assertEqual(text.count('1.0', 'end'), 124) # 'indices' by default - self.assertEqual(text.count('1.0', 'end', 'indices'), 124) + self.assertEqual(text.count('1.0', 'end', 'lines', return_ints=True), 4) + self.assertEqual(text.count('1.0', 'end', 'lines'), (4,)) + self.assertEqual(text.count('end', '1.0', 'lines', return_ints=True), -4) + self.assertEqual(text.count('end', '1.0', 'lines'), (-4,)) + self.assertEqual(text.count('1.3', '1.5', 'lines', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.5', 'lines'), None) + self.assertEqual(text.count('1.3', '1.3', 'lines', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'lines'), None) + # Count 'indices' by default. + self.assertEqual(text.count('1.0', 'end', return_ints=True), 124) + self.assertEqual(text.count('1.0', 'end'), (124,)) + self.assertEqual(text.count('1.0', 'end', 'indices', return_ints=True), 124) + self.assertEqual(text.count('1.0', 'end', 'indices'), (124,)) self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', 'spam') self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', '-lines') - self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), int) + self.assertIsInstance(text.count('1.3', '1.5', 'ypixels', return_ints=True), int) + self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), tuple) + self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels', return_ints=True), int) self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels'), int) - self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels'), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels'), None) + self.assertEqual(text.count('1.3', '1.5', 'update', 'indices', return_ints=True), 2) self.assertEqual(text.count('1.3', '1.5', 'update', 'indices'), 2) - self.assertEqual(text.count('1.3', '1.3', 'update', 'indices'), 0) - self.assertEqual(text.count('1.3', '1.5', 'update'), 2) - self.assertEqual(text.count('1.3', '1.3', 'update'), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'indices', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'indices'), None) + self.assertEqual(text.count('1.3', '1.5', 'update', return_ints=True), 2) + self.assertEqual(text.count('1.3', '1.5', 'update'), (2,)) + self.assertEqual(text.count('1.3', '1.3', 'update', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'update'), None) if __name__ == "__main__": diff --git a/Lib/test/test_tkinter/test_variables.py b/Lib/test/test_tkinter/test_variables.py index c1d232e2febc7a..def7aec077e800 100644 --- a/Lib/test/test_tkinter/test_variables.py +++ b/Lib/test/test_tkinter/test_variables.py @@ -6,7 +6,7 @@ from tkinter import (Variable, StringVar, IntVar, DoubleVar, BooleanVar, Tcl, TclError) from test.support import ALWAYS_EQ -from test.test_tkinter.support import AbstractDefaultRootTest +from test.test_tkinter.support import AbstractDefaultRootTest, tcl_version class Var(Variable): @@ -112,6 +112,8 @@ def test_initialize(self): self.assertTrue(v.side_effect) def test_trace_old(self): + if tcl_version >= (9, 0): + self.skipTest('requires Tcl version < 9.0') # Old interface v = Variable(self.root) vname = str(v) diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index d3f942db7baf9a..9ea764ca2a39d8 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -4,7 +4,7 @@ import os from test.support import requires -from test.test_tkinter.support import (requires_tk, +from test.test_tkinter.support import (requires_tk, tk_version, get_tk_patchlevel, widget_eq, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import ( @@ -14,6 +14,9 @@ requires('gui') +EXPECTED_SCREEN_DISTANCE_ERRMSG = '(bad|expected) screen distance (but got )?"{}"' +EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG = '(bad|expected) screen distance (or "" but got )?"{}"' + def float_round(x): return float(round(x)) @@ -58,11 +61,11 @@ def test_configure_visual(self): @add_standard_options(StandardOptionsTests) class ToplevelTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( - 'background', 'borderwidth', + 'background', 'backgroundimage', 'borderwidth', 'class', 'colormap', 'container', 'cursor', 'height', 'highlightbackground', 'highlightcolor', 'highlightthickness', 'menu', 'padx', 'pady', 'relief', 'screen', - 'takefocus', 'use', 'visual', 'width', + 'takefocus', 'tile', 'use', 'visual', 'width', ) def create(self, **kwargs): @@ -101,10 +104,10 @@ def test_configure_use(self): @add_standard_options(StandardOptionsTests) class FrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( - 'background', 'borderwidth', + 'background', 'backgroundimage', 'borderwidth', 'class', 'colormap', 'container', 'cursor', 'height', 'highlightbackground', 'highlightcolor', 'highlightthickness', - 'padx', 'pady', 'relief', 'takefocus', 'visual', 'width', + 'padx', 'pady', 'relief', 'takefocus', 'tile', 'visual', 'width', ) def create(self, **kwargs): @@ -141,11 +144,9 @@ def test_configure_labelwidget(self): class AbstractLabelTest(AbstractWidgetTest, IntegerSizeTests): _conv_pixels = False - - def test_configure_highlightthickness(self): - widget = self.create() - self.checkPixelsParam(widget, 'highlightthickness', - 0, 1.3, 2.6, 6, -2, '10p') + _clip_highlightthickness = tk_version >= (8, 7) + _clip_pad = tk_version >= (8, 7) + _clip_borderwidth = tk_version >= (8, 7) @add_standard_options(StandardOptionsTests) @@ -277,6 +278,9 @@ class MenubuttonTest(AbstractLabelTest, unittest.TestCase): 'underline', 'width', 'wraplength', ) _conv_pixels = round + _clip_highlightthickness = True + _clip_pad = True + _clip_borderwidth = False def create(self, **kwargs): return tkinter.Menubutton(self.root, **kwargs) @@ -290,9 +294,6 @@ def test_configure_height(self): widget = self.create() self.checkIntegerParam(widget, 'height', 100, -100, 0, conv=str) - test_configure_highlightthickness = \ - StandardOptionsTests.test_configure_highlightthickness - def test_configure_image(self): widget = self.create() image = tkinter.PhotoImage(master=self.root, name='image1') @@ -313,16 +314,6 @@ def test_configure_menu(self): self.checkParam(widget, 'menu', menu, eq=widget_eq) menu.destroy() - def test_configure_padx(self): - widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'padx', -2, expected=0) - - def test_configure_pady(self): - widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'pady', -2, expected=0) - def test_configure_width(self): widget = self.create() self.checkIntegerParam(widget, 'width', 402, -402, 0, conv=str) @@ -347,7 +338,8 @@ class EntryTest(AbstractWidgetTest, unittest.TestCase): 'highlightbackground', 'highlightcolor', 'highlightthickness', 'insertbackground', 'insertborderwidth', 'insertofftime', 'insertontime', 'insertwidth', - 'invalidcommand', 'justify', 'readonlybackground', 'relief', + 'invalidcommand', 'justify', 'placeholder', 'placeholderforeground', + 'readonlybackground', 'relief', 'selectbackground', 'selectborderwidth', 'selectforeground', 'show', 'state', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'width', 'xscrollcommand', @@ -441,8 +433,8 @@ class SpinboxTest(EntryTest, unittest.TestCase): 'increment', 'insertbackground', 'insertborderwidth', 'insertofftime', 'insertontime', 'insertwidth', - 'invalidcommand', 'justify', 'relief', 'readonlybackground', - 'repeatdelay', 'repeatinterval', + 'invalidcommand', 'justify', 'placeholder', 'placeholderforeground', + 'relief', 'readonlybackground', 'repeatdelay', 'repeatinterval', 'selectbackground', 'selectborderwidth', 'selectforeground', 'state', 'takefocus', 'textvariable', 'to', 'validate', 'validatecommand', 'values', @@ -489,8 +481,12 @@ def test_configure_from(self): widget = self.create() self.checkParam(widget, 'to', 100.0) self.checkFloatParam(widget, 'from', -10, 10.2, 11.7) - self.checkInvalidParam(widget, 'from', 200, - errmsg='-to value must be greater than -from value') + if tk_version >= (8, 7): + self.checkFloatParam(widget, 'from', 200, expected=100) + else: + self.checkInvalidParam( + widget, 'from', 200, + errmsg='-to value must be greater than -from value') def test_configure_increment(self): widget = self.create() @@ -500,8 +496,12 @@ def test_configure_to(self): widget = self.create() self.checkParam(widget, 'from', -100.0) self.checkFloatParam(widget, 'to', -10, 10.2, 11.7) - self.checkInvalidParam(widget, 'to', -200, - errmsg='-to value must be greater than -from value') + if tk_version >= (8, 7): + self.checkFloatParam(widget, 'to', -200, expected=-100) + else: + self.checkInvalidParam( + widget, 'to', -200, + errmsg='-to value must be greater than -from value') def test_configure_values(self): # XXX @@ -660,11 +660,13 @@ def test_configure_tabs(self): widget = self.create() self.checkParam(widget, 'tabs', (10.2, 20.7, '1i', '2i')) self.checkParam(widget, 'tabs', '10.2 20.7 1i 2i', - expected=('10.2', '20.7', '1i', '2i')) + expected=(10.2, 20.7, '1i', '2i') + if get_tk_patchlevel(self.root) >= (8, 6, 14) + else ('10.2', '20.7', '1i', '2i')) self.checkParam(widget, 'tabs', '2c left 4c 6c center', expected=('2c', 'left', '4c', '6c', 'center')) self.checkInvalidParam(widget, 'tabs', 'spam', - errmsg='bad screen distance "spam"') + errmsg=EXPECTED_SCREEN_DISTANCE_ERRMSG.format('spam')) def test_configure_tabstyle(self): widget = self.create() @@ -858,24 +860,27 @@ def test_create_line(self): def test_create_polygon(self): c = self.create() - i1 = c.create_polygon(20, 30, 40, 50, 60, 10) + tk87 = tk_version >= (8, 7) + # In Tk < 8.7 polygons are filled, but has no outline by default. + # This affects its size, so always explicitly specify outline. + i1 = c.create_polygon(20, 30, 40, 50, 60, 10, outline='red') self.assertEqual(c.coords(i1), [20.0, 30.0, 40.0, 50.0, 60.0, 10.0]) - self.assertEqual(c.bbox(i1), (19, 9, 61, 51)) + self.assertEqual(c.bbox(i1), (18, 8, 62, 52)) self.assertEqual(c.itemcget(i1, 'joinstyle'), 'round') self.assertEqual(c.itemcget(i1, 'smooth'), '0') self.assertEqual(c.itemcget(i1, 'splinestep'), '12') - i2 = c.create_polygon([21, 31, 41, 51, 61, 11]) + i2 = c.create_polygon([21, 31, 41, 51, 61, 11], outline='red') self.assertEqual(c.coords(i2), [21.0, 31.0, 41.0, 51.0, 61.0, 11.0]) - self.assertEqual(c.bbox(i2), (20, 10, 62, 52)) + self.assertEqual(c.bbox(i2), (19, 9, 63, 53)) - i3 = c.create_polygon((22, 32), (42, 52), (62, 12)) + i3 = c.create_polygon((22, 32), (42, 52), (62, 12), outline='red') self.assertEqual(c.coords(i3), [22.0, 32.0, 42.0, 52.0, 62.0, 12.0]) - self.assertEqual(c.bbox(i3), (21, 11, 63, 53)) + self.assertEqual(c.bbox(i3), (20, 10, 64, 54)) - i4 = c.create_polygon([(23, 33), (43, 53), (63, 13)]) + i4 = c.create_polygon([(23, 33), (43, 53), (63, 13)], outline='red') self.assertEqual(c.coords(i4), [23.0, 33.0, 43.0, 53.0, 63.0, 13.0]) - self.assertEqual(c.bbox(i4), (22, 12, 64, 54)) + self.assertEqual(c.bbox(i4), (21, 11, 65, 55)) self.assertRaises(TclError, c.create_polygon, 20, 30, 60) self.assertRaises(TclError, c.create_polygon, [20, 30, 60]) @@ -999,12 +1004,16 @@ def test_itemconfigure(self): widget.itemconfigure() with self.assertRaisesRegex(TclError, 'bad listbox index "red"'): widget.itemconfigure('red') + if get_tk_patchlevel(self.root) >= (8, 6, 14): + prefix = ('background', '', '', '') + else: + prefix = ('background', 'background', 'Background', '') self.assertEqual(widget.itemconfigure(0, 'background'), - ('background', 'background', 'Background', '', 'red')) + (*prefix, 'red')) self.assertEqual(widget.itemconfigure('end', 'background'), - ('background', 'background', 'Background', '', 'violet')) + (*prefix, 'violet')) self.assertEqual(widget.itemconfigure('@0,0', 'background'), - ('background', 'background', 'Background', '', 'red')) + (*prefix, 'red')) d = widget.itemconfigure(0) self.assertIsInstance(d, dict) @@ -1168,18 +1177,16 @@ class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): def create(self, **kwargs): return tkinter.Scrollbar(self.root, **kwargs) - def test_configure_activerelief(self): - widget = self.create() - self.checkReliefParam(widget, 'activerelief') - def test_configure_elementborderwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'elementborderwidth', 4.3, 5.6, -2, '1m') + self.checkPixelsParam(widget, 'elementborderwidth', 4.3, 5.6, '1m') + expected = self._default_pixels if tk_version >= (8, 7) else -2 + self.checkParam(widget, 'elementborderwidth', -2, expected=expected) def test_configure_orient(self): widget = self.create() self.checkEnumParam(widget, 'orient', 'vertical', 'horizontal', - errmsg='bad orientation "{}": must be vertical or horizontal') + fullname='orientation', allow_empty=True) def test_activate(self): sb = self.create() @@ -1250,7 +1257,8 @@ def test_configure_proxyborderwidth(self): @requires_tk(8, 6, 5) def test_configure_proxyrelief(self): widget = self.create() - self.checkReliefParam(widget, 'proxyrelief') + self.checkReliefParam(widget, 'proxyrelief', + allow_empty=(tk_version >= (8, 7))) def test_configure_sashcursor(self): widget = self.create() @@ -1323,7 +1331,7 @@ def test_paneconfigure_height(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'height', 10, 10) self.check_paneconfigure_bad(p, b, 'height', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('badValue')) def test_paneconfigure_hide(self): p, b, c = self.create2() @@ -1335,19 +1343,19 @@ def test_paneconfigure_minsize(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'minsize', 10, 10) self.check_paneconfigure_bad(p, b, 'minsize', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_padx(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'padx', 1.3, 1) self.check_paneconfigure_bad(p, b, 'padx', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_pady(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'pady', 1.3, 1) self.check_paneconfigure_bad(p, b, 'pady', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_sticky(self): p, b, c = self.create2() @@ -1368,13 +1376,14 @@ def test_paneconfigure_width(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'width', 10, 10) self.check_paneconfigure_bad(p, b, 'width', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('badValue')) @add_standard_options(StandardOptionsTests) class MenuTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeborderwidth', 'activeforeground', + 'activerelief', 'background', 'borderwidth', 'cursor', 'disabledforeground', 'font', 'foreground', 'postcommand', 'relief', 'selectcolor', 'takefocus', @@ -1390,6 +1399,8 @@ def test_indexcommand_none(self): i = widget.index('none') self.assertIsNone(i) + test_configure_activerelief = requires_tk(8, 7)(StandardOptionsTests.test_configure_activerelief) + def test_configure_postcommand(self): widget = self.create() self.checkCommandParam(widget, 'postcommand') @@ -1408,14 +1419,10 @@ def test_configure_title(self): def test_configure_type(self): widget = self.create() - opts = ('normal, tearoff, or menubar' - if widget.info_patchlevel() < (8, 7) else - 'menubar, normal, or tearoff') - self.checkEnumParam( - widget, 'type', - 'normal', 'tearoff', 'menubar', - errmsg='bad type "{}": must be ' + opts, - ) + values = ('normal', 'tearoff', 'menubar') + self.checkEnumParam(widget, 'type', *values, + allow_empty=tk_version < (8, 7), + sort=tk_version >= (8, 7)) def test_entryconfigure(self): m1 = self.create() @@ -1461,6 +1468,10 @@ class MessageTest(AbstractWidgetTest, unittest.TestCase): 'takefocus', 'text', 'textvariable', 'width', ) _conv_pad_pixels = False + if tk_version >= (8, 7): + _conv_pixels = False + _clip_pad = tk_version >= (8, 7) + _clip_borderwidth = tk_version >= (8, 7) def create(self, **kwargs): return tkinter.Message(self.root, **kwargs) @@ -1469,6 +1480,26 @@ def test_configure_aspect(self): widget = self.create() self.checkIntegerParam(widget, 'aspect', 250, 0, -300) + def test_configure_padx(self): + widget = self.create() + self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m', + conv=self._conv_pad_pixels) + expected = self._default_pixels if self._clip_pad else -2 + self.checkParam(widget, 'padx', -2, expected=expected) + + def test_configure_pady(self): + widget = self.create() + self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m', + conv=self._conv_pad_pixels) + expected = self._default_pixels if self._clip_pad else -2 + self.checkParam(widget, 'pady', -2, expected=expected) + + def test_configure_width(self): + widget = self.create() + self.checkPixelsParam(widget, 'width', 402, 403.4, 404.6, 0, '5i') + expected = 0 if tk_version >= (8, 7) else -402 + self.checkParam(widget, 'width', -402, expected=expected) + class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): @@ -1479,13 +1510,5 @@ def test_label(self): self._test_widget(tkinter.Label) -tests_gui = ( - ButtonTest, CanvasTest, CheckbuttonTest, EntryTest, - FrameTest, LabelFrameTest,LabelTest, ListboxTest, - MenubuttonTest, MenuTest, MessageTest, OptionMenuTest, - PanedWindowTest, RadiobuttonTest, ScaleTest, ScrollbarTest, - SpinboxTest, TextTest, ToplevelTest, DefaultRootTest, -) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tkinter/widget_tests.py b/Lib/test/test_tkinter/widget_tests.py index 31f82f459beefd..8ab2f74245095d 100644 --- a/Lib/test/test_tkinter/widget_tests.py +++ b/Lib/test/test_tkinter/widget_tests.py @@ -1,7 +1,8 @@ # Common tests for test_tkinter/test_widgets.py and test_ttk/test_widgets.py +import re import tkinter -from test.test_tkinter.support import (AbstractTkTest, tk_version, +from test.test_tkinter.support import (AbstractTkTest, requires_tk, tk_version, pixels_conv, tcl_obj_eq) import test.support @@ -9,9 +10,14 @@ _sentinel = object() class AbstractWidgetTest(AbstractTkTest): + _default_pixels = '' if tk_version >= (9, 0) else -1 if tk_version >= (8, 7) else '' _conv_pixels = round _conv_pad_pixels = None _stringify = False + _clip_highlightthickness = True + _clip_pad = False + _clip_borderwidth = False + _allow_empty_justify = False @property def scaling(self): @@ -56,16 +62,13 @@ def checkParam(self, widget, name, value, *, expected=_sentinel, def checkInvalidParam(self, widget, name, value, errmsg=None): orig = widget[name] if errmsg is not None: - errmsg = errmsg.format(value) - with self.assertRaises(tkinter.TclError) as cm: + errmsg = errmsg.format(re.escape(str(value))) + errmsg = fr'\A{errmsg}\Z' + with self.assertRaisesRegex(tkinter.TclError, errmsg or ''): widget[name] = value - if errmsg is not None: - self.assertEqual(str(cm.exception), errmsg) self.assertEqual(widget[name], orig) - with self.assertRaises(tkinter.TclError) as cm: + with self.assertRaisesRegex(tkinter.TclError, errmsg or ''): widget.configure({name: value}) - if errmsg is not None: - self.assertEqual(str(cm.exception), errmsg) self.assertEqual(widget[name], orig) def checkParams(self, widget, name, *values, **kwargs): @@ -74,30 +77,26 @@ def checkParams(self, widget, name, *values, **kwargs): def checkIntegerParam(self, widget, name, *values, **kwargs): self.checkParams(widget, name, *values, **kwargs) - self.checkInvalidParam(widget, name, '', - errmsg='expected integer but got ""') - self.checkInvalidParam(widget, name, '10p', - errmsg='expected integer but got "10p"') - self.checkInvalidParam(widget, name, 3.2, - errmsg='expected integer but got "3.2"') + errmsg = 'expected integer but got "{}"' + self.checkInvalidParam(widget, name, '', errmsg=errmsg) + self.checkInvalidParam(widget, name, '10p', errmsg=errmsg) + self.checkInvalidParam(widget, name, 3.2, errmsg=errmsg) def checkFloatParam(self, widget, name, *values, conv=float, **kwargs): for value in values: self.checkParam(widget, name, value, conv=conv, **kwargs) - self.checkInvalidParam(widget, name, '', - errmsg='expected floating-point number but got ""') - self.checkInvalidParam(widget, name, 'spam', - errmsg='expected floating-point number but got "spam"') + errmsg = 'expected floating-point number but got "{}"' + self.checkInvalidParam(widget, name, '', errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkBooleanParam(self, widget, name): for value in (False, 0, 'false', 'no', 'off'): self.checkParam(widget, name, value, expected=0) for value in (True, 1, 'true', 'yes', 'on'): self.checkParam(widget, name, value, expected=1) - self.checkInvalidParam(widget, name, '', - errmsg='expected boolean value but got ""') - self.checkInvalidParam(widget, name, 'spam', - errmsg='expected boolean value but got "spam"') + errmsg = 'expected boolean value but got "{}"' + self.checkInvalidParam(widget, name, '', errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkColorParam(self, widget, name, *, allow_empty=None, **kwargs): self.checkParams(widget, name, @@ -120,16 +119,24 @@ def command(*args): self.assertTrue(widget[name]) self.checkParams(widget, name, '') - def checkEnumParam(self, widget, name, *values, errmsg=None, **kwargs): + def checkEnumParam(self, widget, name, *values, + errmsg=None, allow_empty=False, fullname=None, + sort=False, **kwargs): self.checkParams(widget, name, *values, **kwargs) if errmsg is None: + if sort: + if values[-1]: + values = tuple(sorted(values)) + else: + values = tuple(sorted(values[:-1])) + ('',) errmsg2 = ' %s "{}": must be %s%s or %s' % ( - name, + fullname or name, ', '.join(values[:-1]), ',' if len(values) > 2 else '', - values[-1]) - self.checkInvalidParam(widget, name, '', - errmsg='ambiguous' + errmsg2) + values[-1] or '""') + if '' not in values and not allow_empty: + self.checkInvalidParam(widget, name, '', + errmsg='ambiguous' + errmsg2) errmsg = 'bad' + errmsg2 self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) @@ -146,20 +153,21 @@ def checkPixelsParam(self, widget, name, *values, conv1 = round self.checkParam(widget, name, value, expected=expected, conv=conv1, **kwargs) - self.checkInvalidParam(widget, name, '6x', - errmsg='bad screen distance "6x"') - self.checkInvalidParam(widget, name, 'spam', - errmsg='bad screen distance "spam"') + errmsg = '(bad|expected) screen distance ((or "" )?but got )?"{}"' + self.checkInvalidParam(widget, name, '6x', errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) - def checkReliefParam(self, widget, name): - self.checkParams(widget, name, - 'flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') - errmsg='bad relief "spam": must be '\ - 'flat, groove, raised, ridge, solid, or sunken' + def checkReliefParam(self, widget, name, *, allow_empty=False): + values = ('flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') + if allow_empty: + values += ('',) + self.checkParams(widget, name, *values) + errmsg = 'bad relief "{}": must be %s, or %s' % ( + ', '.join(values[:-1]), + values[-1] or '""') if tk_version < (8, 6): errmsg = None - self.checkInvalidParam(widget, name, 'spam', - errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkImageParam(self, widget, name): image = tkinter.PhotoImage(master=self.root, name='image1') @@ -193,6 +201,7 @@ def test_keys(self): aliases = { 'bd': 'borderwidth', 'bg': 'background', + 'bgimg': 'backgroundimage', 'fg': 'foreground', 'invcmd': 'invalidcommand', 'vcmd': 'validatecommand', @@ -235,6 +244,10 @@ def test_configure_activeforeground(self): widget = self.create() self.checkColorParam(widget, 'activeforeground') + def test_configure_activerelief(self): + widget = self.create() + self.checkReliefParam(widget, 'activerelief') + def test_configure_anchor(self): widget = self.create() self.checkEnumParam(widget, 'anchor', @@ -246,6 +259,11 @@ def test_configure_background(self): if 'bg' in self.OPTIONS: self.checkColorParam(widget, 'bg') + @requires_tk(8, 7) + def test_configure_backgroundimage(self): + widget = self.create() + self.checkImageParam(widget, 'backgroundimage') + def test_configure_bitmap(self): widget = self.create() self.checkParam(widget, 'bitmap', 'questhead') @@ -262,9 +280,14 @@ def test_configure_bitmap(self): def test_configure_borderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'borderwidth', - 0, 1.3, 2.6, 6, -2, '10p') + 0, 1.3, 2.6, 6, '10p') + expected = 0 if self._clip_borderwidth else -2 + self.checkParam(widget, 'borderwidth', -2, expected=expected, + conv=self._conv_pixels) if 'bd' in self.OPTIONS: - self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, -2, '10p') + self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'bd', -2, expected=expected, + conv=self._conv_pixels) def test_configure_compound(self): widget = self.create() @@ -287,8 +310,10 @@ def test_configure_font(self): widget = self.create() self.checkParam(widget, 'font', '-Adobe-Helvetica-Medium-R-Normal--*-120-*-*-*-*-*-*') - self.checkInvalidParam(widget, 'font', '', - errmsg='font "" doesn\'t exist') + is_ttk = widget.__class__.__module__ == 'tkinter.ttk' + if not is_ttk: + self.checkInvalidParam(widget, 'font', '', + errmsg='font "" doesn\'t exist') def test_configure_foreground(self): widget = self.create() @@ -308,7 +333,8 @@ def test_configure_highlightthickness(self): widget = self.create() self.checkPixelsParam(widget, 'highlightthickness', 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'highlightthickness', -2, expected=0, + expected = 0 if self._clip_highlightthickness else -2 + self.checkParam(widget, 'highlightthickness', -2, expected=expected, conv=self._conv_pixels) def test_configure_image(self): @@ -342,12 +368,11 @@ def test_configure_jump(self): def test_configure_justify(self): widget = self.create() - self.checkEnumParam(widget, 'justify', 'left', 'right', 'center', - errmsg='bad justification "{}": must be ' - 'left, right, or center') - self.checkInvalidParam(widget, 'justify', '', - errmsg='ambiguous justification "": must be ' - 'left, right, or center') + values = ('left', 'right', 'center') + if self._allow_empty_justify: + values += ('',) + self.checkEnumParam(widget, 'justify', *values, + fullname='justification') def test_configure_orient(self): widget = self.create() @@ -356,13 +381,29 @@ def test_configure_orient(self): def test_configure_padx(self): widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, -2, '12m', + self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m', conv=self._conv_pad_pixels) + expected = 0 if self._clip_pad else -2 + self.checkParam(widget, 'padx', -2, expected=expected, + conv=self._conv_pad_pixels) def test_configure_pady(self): widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, -2, '12m', + self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m', conv=self._conv_pad_pixels) + expected = 0 if self._clip_pad else -2 + self.checkParam(widget, 'pady', -2, expected=expected, + conv=self._conv_pad_pixels) + + @requires_tk(8, 7) + def test_configure_placeholder(self): + widget = self.create() + self.checkParam(widget, 'placeholder', 'xxx') + + @requires_tk(8, 7) + def test_configure_placeholderforeground(self): + widget = self.create() + self.checkColorParam(widget, 'placeholderforeground') def test_configure_relief(self): widget = self.create() @@ -409,13 +450,35 @@ def test_configure_textvariable(self): var = tkinter.StringVar(self.root) self.checkVariableParam(widget, 'textvariable', var) + @requires_tk(8, 7) + def test_configure_tile(self): + widget = self.create() + self.checkBooleanParam(widget, 'tile') + def test_configure_troughcolor(self): widget = self.create() self.checkColorParam(widget, 'troughcolor') def test_configure_underline(self): widget = self.create() - self.checkIntegerParam(widget, 'underline', 0, 1, 10) + self.checkParams(widget, 'underline', 0, 1, 10) + if tk_version >= (8, 7): + is_ttk = widget.__class__.__module__ == 'tkinter.ttk' + self.checkParam(widget, 'underline', '', + expected='' if is_ttk else self._default_pixels) + self.checkParam(widget, 'underline', '5+2', + expected='5+2' if is_ttk else 7) + self.checkParam(widget, 'underline', '5-2', + expected='5-2' if is_ttk else 3) + self.checkParam(widget, 'underline', 'end', expected='end') + self.checkParam(widget, 'underline', 'end-2', expected='end-2') + errmsg = (r'bad index "{}": must be integer\?\[\+-\]integer\?, ' + r'end\?\[\+-\]integer\?, or ""') + else: + errmsg = 'expected integer but got "{}"' + self.checkInvalidParam(widget, 'underline', '', errmsg=errmsg) + self.checkInvalidParam(widget, 'underline', '10p', errmsg=errmsg) + self.checkInvalidParam(widget, 'underline', 3.2, errmsg=errmsg) def test_configure_wraplength(self): widget = self.create() @@ -445,7 +508,8 @@ def test_configure_offrelief(self): def test_configure_overrelief(self): widget = self.create() - self.checkReliefParam(widget, 'overrelief') + self.checkReliefParam(widget, 'overrelief', + allow_empty=(tk_version >= (8, 7))) def test_configure_selectcolor(self): widget = self.create() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 21e8637a7ca905..75710db7d05375 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -228,7 +228,7 @@ def test_long(self): """) def test_float(self): - # Floating point numbers + # Floating-point numbers self.check_tokenize("x = 3.14159", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) @@ -1199,6 +1199,31 @@ def test_closing_parenthesis_from_different_line(self): NAME 'x' (1, 3) (1, 4) """) + def test_multiline_non_ascii_fstring(self): + self.check_tokenize("""\ +a = f''' + Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''""", """\ + NAME 'a' (1, 0) (1, 1) + OP '=' (1, 2) (1, 3) + FSTRING_START "f\'\'\'" (1, 4) (1, 8) + FSTRING_MIDDLE '\\n Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68) + FSTRING_END "\'\'\'" (2, 68) (2, 71) + """) + + def test_multiline_non_ascii_fstring_with_expr(self): + self.check_tokenize("""\ +f''' + 🔗 This is a test {test_arg1}🔗 +🔗'''""", """\ + FSTRING_START "f\'\'\'" (1, 0) (1, 4) + FSTRING_MIDDLE '\\n 🔗 This is a test ' (1, 4) (2, 21) + OP '{' (2, 21) (2, 22) + NAME 'test_arg1' (2, 22) (2, 31) + OP '}' (2, 31) (2, 32) + FSTRING_MIDDLE '🔗\\n🔗' (2, 32) (3, 1) + FSTRING_END "\'\'\'" (3, 1) (3, 4) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. @@ -1877,6 +1902,63 @@ def test_roundtrip(self): " print('Can not import' # comment2\n)" "else: print('Loaded')\n") + self.check_roundtrip("f'\\N{EXCLAMATION MARK}'") + self.check_roundtrip(r"f'\\N{SNAKE}'") + self.check_roundtrip(r"f'\\N{{SNAKE}}'") + self.check_roundtrip(r"f'\N{SNAKE}'") + self.check_roundtrip(r"f'\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\\\N{SNAKE}'") + + self.check_roundtrip(r"f'\\N{1}'") + self.check_roundtrip(r"f'\\\\N{2}'") + self.check_roundtrip(r"f'\\\\\\N{3}'") + self.check_roundtrip(r"f'\\\\\\\\N{4}'") + + self.check_roundtrip(r"f'\\N{{'") + self.check_roundtrip(r"f'\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\\\N{{'") + + self.check_roundtrip(r"f'\n{{foo}}'") + self.check_roundtrip(r"f'\\n{{foo}}'") + self.check_roundtrip(r"f'\\\n{{foo}}'") + self.check_roundtrip(r"f'\\\\n{{foo}}'") + + self.check_roundtrip(r"f'\t{{foo}}'") + self.check_roundtrip(r"f'\\t{{foo}}'") + self.check_roundtrip(r"f'\\\t{{foo}}'") + self.check_roundtrip(r"f'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\t{{foo}}'") + self.check_roundtrip(r"rf'\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\{{foo}}'") + self.check_roundtrip(r"f'\\{{foo}}'") + self.check_roundtrip(r"rf'\\\{{foo}}'") + self.check_roundtrip(r"f'\\\\{{foo}}'") + cases = [ + """ +if 1: + "foo" +"bar" +""", + """ +if 1: + ("foo" + "bar") +""", + """ +if 1: + "foo" + "bar" +""" ] + for case in cases: + self.check_roundtrip(case) + + def test_continuation(self): # Balancing continuation self.check_roundtrip("a = (3,4, \n" @@ -1911,9 +1993,6 @@ def test_random_files(self): tempdir = os.path.dirname(__file__) or os.curdir testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py")) - # TODO: Remove this once we can untokenize PEP 701 syntax - testfiles.remove(os.path.join(tempdir, "test_fstring.py")) - if not support.is_resource_enabled("cpu"): testfiles = random.sample(testfiles, 10) diff --git a/Lib/test/test_tomllib/test_error.py b/Lib/test/test_tomllib/test_error.py index 72446267f04759..d2ef59a29ca350 100644 --- a/Lib/test/test_tomllib/test_error.py +++ b/Lib/test/test_tomllib/test_error.py @@ -39,6 +39,15 @@ def test_invalid_char_quotes(self): tomllib.loads("v = '\n'") self.assertTrue(" '\\n' " in str(exc_info.exception)) + def test_type_error(self): + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(b"v = 1") # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bytes'") + + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(False) # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bool'") + def test_module_name(self): self.assertEqual(tomllib.TOMLDecodeError().__module__, tomllib.__name__) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py new file mode 100644 index 00000000000000..4c7588d4d93fc3 --- /dev/null +++ b/Lib/test/test_tools/test_makefile.py @@ -0,0 +1,81 @@ +""" +Tests for `Makefile`. +""" + +import os +import unittest +from test import support +import sysconfig + +MAKEFILE = sysconfig.get_makefile_filename() + +if not support.check_impl_detail(cpython=True): + raise unittest.SkipTest('cpython only') +if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE): + raise unittest.SkipTest('Makefile could not be found') + + +class TestMakefile(unittest.TestCase): + def list_test_dirs(self): + result = [] + found_testsubdirs = False + with open(MAKEFILE, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('TESTSUBDIRS='): + found_testsubdirs = True + result.append( + line.removeprefix('TESTSUBDIRS=').replace( + '\\', '', + ).strip(), + ) + continue + if found_testsubdirs: + if '\t' not in line: + break + result.append(line.replace('\\', '').strip()) + return result + + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") + def test_makefile_test_folders(self): + test_dirs = self.list_test_dirs() + idle_test = 'idlelib/idle_test' + self.assertIn(idle_test, test_dirs) + + used = set([idle_test]) + for dirpath, dirs, files in os.walk(support.TEST_HOME_DIR): + dirname = os.path.basename(dirpath) + # Skip temporary dirs: + if dirname == '__pycache__' or dirname.startswith('.'): + dirs.clear() # do not process subfolders + continue + # Skip empty dirs: + if not dirs and not files: + continue + # Skip dirs with hidden-only files: + if files and all( + filename.startswith('.') or filename == '__pycache__' + for filename in files + ): + continue + + relpath = os.path.relpath(dirpath, support.STDLIB_DIR) + with self.subTest(relpath=relpath): + self.assertIn( + relpath, + test_dirs, + msg=( + f"{relpath!r} is not included in the Makefile's list " + "of test directories to install" + ) + ) + used.add(relpath) + + # Don't check the wheel dir when Python is built --with-wheel-pkg-dir + if sysconfig.get_config_var('WHEEL_PKG_DIR'): + test_dirs.remove('test/wheeldata') + used.discard('test/wheeldata') + + # Check that there are no extra entries: + unique_test_dirs = set(test_dirs) + self.assertSetEqual(unique_test_dirs, used) + self.assertEqual(len(test_dirs), len(unique_test_dirs)) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index c1e289bcaff9e5..93966ee31d0a01 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -6,6 +6,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap import unittest +from types import FunctionType import trace from trace import Trace @@ -559,5 +560,29 @@ def test_run_as_module(self): assert_python_failure('-m', 'trace', '-l', '--module', 'not_a_module_zzz') +class TestTrace(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + self.tracer = Trace(count=0, trace=1) + self.filemod = my_file_and_modname() + + def test_no_source_file(self): + filename = "" + co = traced_func_linear.__code__ + co = co.replace(co_filename=filename) + f = FunctionType(co, globals()) + + with captured_stdout() as out: + self.tracer.runfunc(f, 2, 3) + + out = out.getvalue().splitlines() + firstlineno = get_firstlineno(f) + self.assertIn(f" --- modulename: {self.filemod[1]}, funcname: {f.__code__.co_name}", out[0]) + self.assertIn(f"{filename}({firstlineno + 1})", out[1]) + self.assertIn(f"{filename}({firstlineno + 2})", out[2]) + self.assertIn(f"{filename}({firstlineno + 3})", out[3]) + self.assertIn(f"{filename}({firstlineno + 4})", out[4]) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 372fc48bf81a6a..ec69412f5511eb 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -21,13 +21,14 @@ from test.support.os_helper import TESTFN, unlink from test.support.script_helper import assert_python_ok, assert_python_failure from test.support.import_helper import forget +from test.support import force_not_colorized import json import textwrap import traceback -import contextlib from functools import partial from pathlib import Path +import _colorize MODULE_PREFIX = f'{__name__}.' if __name__ == '__main__' else '' @@ -45,12 +46,12 @@ class TracebackCases(unittest.TestCase): # formatting of SyntaxErrors works based on changes for 2.1. def setUp(self): super().setUp() - self.colorize = traceback._COLORIZE - traceback._COLORIZE = False + self.colorize = _colorize.COLORIZE + _colorize.COLORIZE = False def tearDown(self): super().tearDown() - traceback._COLORIZE = self.colorize + _colorize.COLORIZE = self.colorize def get_exception_format(self, func, exc): try: @@ -124,6 +125,7 @@ def test_nocaret(self): self.assertEqual(len(err), 3) self.assertEqual(err[1].strip(), "bad syntax") + @force_not_colorized def test_no_caret_with_no_debug_ranges_flag(self): # Make sure that if `-X no_debug_ranges` is used, there are no carets # in the traceback. @@ -148,7 +150,7 @@ def test_no_caret_with_no_debug_ranges_flag_python_traceback(self): import traceback try: x = 1 / 0 - except: + except ZeroDivisionError: traceback.print_exc() """) try: @@ -375,6 +377,7 @@ def f(): ]) @requires_subprocess() + @force_not_colorized def test_encoded_file(self): # Test that tracebacks are correctly printed for encoded source files: # - correct line number (Issue2384) @@ -497,7 +500,7 @@ def test_format_exception_exc(self): traceback.format_exception(e.__class__, e) with self.assertRaisesRegex(ValueError, 'Both or neither'): traceback.format_exception(e.__class__, tb=e.__traceback__) - with self.assertRaisesRegex(TypeError, 'positional-only'): + with self.assertRaisesRegex(TypeError, 'required positional argument'): traceback.format_exception(exc=e) def test_format_exception_only_exc(self): @@ -536,20 +539,21 @@ def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.format_exception)), ('(exc, /, value=, tb=, limit=None, ' - 'chain=True)')) + 'chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception_only)), - '(exc, /, value=, *, show_group=False)') + '(exc, /, value=, *, show_group=False, **kwargs)') class PurePythonExceptionFormattingMixin: def get_exception(self, callable, slice_start=0, slice_end=-1): try: callable() - self.fail("No exception thrown.") - except: + except BaseException: return traceback.format_exc().splitlines()[slice_start:slice_end] + else: + self.fail("No exception thrown.") callable_line = get_exception.__code__.co_firstlineno + 2 @@ -619,6 +623,7 @@ def test_caret_in_type_annotation(self): def f_with_type(): def foo(a: THIS_DOES_NOT_EXIST ) -> int: return 0 + foo.__annotations__ lineno_f = f_with_type.__code__.co_firstlineno expected_f = ( @@ -626,7 +631,9 @@ def foo(a: THIS_DOES_NOT_EXIST ) -> int: f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' ' ~~~~~~~~^^\n' - f' File "{__file__}", line {lineno_f+1}, in f_with_type\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_type\n' + ' foo.__annotations__\n' + f' File "{__file__}", line {lineno_f+1}, in __annotate__\n' ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' ' ^^^^^^^^^^^^^^^^^^^\n' ) @@ -685,7 +692,6 @@ def f_with_multiline(): ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' ' return compile(code, "?", "exec")\n' - ' ~~~~~~~^^^^^^^^^^^^^^^^^^^\n' ' File "?", line 7\n' ' foo(a, z\n' ' ^' @@ -694,6 +700,35 @@ def f_with_multiline(): result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) + # Check custom error messages covering multiple lines + code = textwrap.dedent(""" + dummy_call( + "dummy value" + foo="bar", + ) + """) + + def f_with_multiline(): + # Need to defer the compilation until in self.get_exception(..) + return compile(code, "?", "exec") + + lineno_f = f_with_multiline.__code__.co_firstlineno + + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' return compile(code, "?", "exec")\n' + ' File "?", line 3\n' + ' "dummy value"\n' + ' ^^^^^^^^^^^^^' + ) + + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + def test_caret_multiline_expression_bin_op(self): # Make sure no carets are printed for expressions spanning multiple # lines. @@ -775,8 +810,8 @@ def f_with_binary_operator(): def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): def f_with_binary_operator(): a = 1 - b = "" - return ( a ) +b + b = c = "" + return ( a ) +b + c lineno_f = f_with_binary_operator.__code__.co_firstlineno expected_error = ( @@ -785,7 +820,7 @@ def f_with_binary_operator(): ' callable()\n' ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' - ' return ( a ) +b\n' + ' return ( a ) +b + c\n' ' ~~~~~~~~~~^~\n' ) result_lines = self.get_exception(f_with_binary_operator) @@ -973,7 +1008,7 @@ def f1(a): def f2(b): raise RuntimeError("fail") return f2 - return f1("x")("y") + return f1("x")("y")("z") lineno_f = f_with_call.__code__.co_firstlineno expected_error = ( @@ -982,7 +1017,7 @@ def f2(b): ' callable()\n' ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' - ' return f1("x")("y")\n' + ' return f1("x")("y")("z")\n' ' ~~~~~~~^^^^^\n' f' File "{__file__}", line {lineno_f+3}, in f2\n' ' raise RuntimeError("fail")\n' @@ -1497,6 +1532,184 @@ def f(): ' raise MemoryError()'] self.assertEqual(actual, expected) + def test_anchors_for_simple_return_statements_are_elided(self): + def g(): + 1/0 + + def f(): + return g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + return g() + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g() + 1", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " 2, 4,", + " 5)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " ~^^^", + " 2, 4,", + " ^^^^^", + " 5) + 1", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def test_anchors_for_simple_assign_statements_are_elided(self): + def g(): + 1/0 + + def f(): + x = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g(1,", + " 2, 3,", + " 4)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + x = y = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g()", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = y = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g(1,", + " ~^^^", + " 2, 3,", + " ^^^^^", + " 4)", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + @requires_debug_ranges() class PurePythonTracebackErrorCaretTests( @@ -1691,7 +1904,7 @@ def f(): # Check a known (limited) number of recursive invocations def g(count=10): if count: - return g(count-1) + return g(count-1) + 1 raise ValueError with captured_output("stderr") as stderr_g: @@ -1705,13 +1918,13 @@ def g(count=10): lineno_g = g.__code__.co_firstlineno result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' @@ -1750,13 +1963,10 @@ def h(count=10): ' ~^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_h+3}, in h\n' ' g()\n' @@ -1776,13 +1986,13 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -1790,7 +2000,7 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+80}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+77}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF)\n' ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) @@ -1808,13 +2018,13 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' ' [Previous line repeated 1 more time]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' @@ -1823,7 +2033,7 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+112}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+109}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF + 1)\n' ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) @@ -2028,7 +2238,7 @@ def test_context_suppression(self): try: try: raise Exception - except: + except Exception: raise ZeroDivisionError from None except ZeroDivisionError as _: e = _ @@ -2132,19 +2342,22 @@ def test_message_none(self): def test_syntax_error_various_offsets(self): for offset in range(-5, 10): for add in [0, 2]: - text = " "*add + "text%d" % offset + text = " " * add + "text%d" % offset expected = [' File "file.py", line 1'] if offset < 1: expected.append(" %s" % text.lstrip()) elif offset <= 6: expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*(offset-1))) + # Set the caret length to match the length of the text minus the offset. + caret_length = max(1, len(text.lstrip()) - offset + 1) + expected.append(" %s%s" % (" " * (offset - 1), "^" * caret_length)) else: + caret_length = max(1, len(text.lstrip()) - 4) expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*5)) + expected.append(" %s%s" % (" " * 5, "^" * caret_length)) expected.append("SyntaxError: msg") expected.append("") - err = self.get_report(SyntaxError("msg", ("file.py", 1, offset+add, text))) + err = self.get_report(SyntaxError("msg", ("file.py", 1, offset + add, text))) exp = "\n".join(expected) self.assertEqual(exp, err) @@ -2377,9 +2590,9 @@ def exc(): try: try: raise EG("eg1", [ValueError(1), TypeError(2)]) - except: + except EG: raise EG("eg2", [ValueError(3), TypeError(4)]) - except: + except EG: raise ImportError(5) expected = ( @@ -2429,7 +2642,7 @@ def exc(): except Exception as e: exc = e raise EG("eg", [VE(1), exc, VE(4)]) - except: + except EG: raise EG("top", [VE(5)]) expected = (f' + Exception Group Traceback (most recent call last):\n' @@ -3095,6 +3308,41 @@ def format_frame_summary(self, frame_summary, colorize=False): f' File "{__file__}", line {lno}, in f\n 1/0\n' ) + def test_summary_should_show_carets(self): + # See: https://github.com/python/cpython/issues/122353 + + # statement to execute and to get a ZeroDivisionError for a traceback + statement = "abcdef = 1 / 0 and 2.0" + colno = statement.index('1 / 0') + end_colno = colno + len('1 / 0') + + # Actual line to use when rendering the traceback + # and whose AST will be extracted (it will be empty). + cached_line = '# this line will be used during rendering' + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as file: + file.write(cached_line) + linecache.updatecache(TESTFN, {}) + + try: + exec(compile(statement, TESTFN, "exec")) + except ZeroDivisionError as exc: + # This is the simplest way to create a StackSummary + # whose FrameSummary items have their column offsets. + s = traceback.TracebackException.from_exception(exc).stack + self.assertIsInstance(s, traceback.StackSummary) + with unittest.mock.patch.object(s, '_should_show_carets', + wraps=s._should_show_carets) as ff: + self.assertEqual(len(s), 2) + self.assertListEqual( + s.format_frame_summary(s[1]).splitlines(), + [ + f' File "{TESTFN}", line 1, in ', + f' {cached_line}' + ] + ) + ff.assert_called_with(colno, end_colno, [cached_line], None) + class Unrepresentable: def __repr__(self) -> str: raise Exception("Unrepresentable") @@ -3124,10 +3372,13 @@ def test_smoke_user_exception(self): class MyException(Exception): pass - self.do_test_smoke( - MyException('bad things happened'), - ('test.test_traceback.TestTracebackException.' - 'test_smoke_user_exception..MyException')) + if __name__ == '__main__': + expected = ('TestTracebackException.' + 'test_smoke_user_exception..MyException') + else: + expected = ('test.test_traceback.TestTracebackException.' + 'test_smoke_user_exception..MyException') + self.do_test_smoke(MyException('bad things happened'), expected) def test_from_exception(self): # Check all the parameters are accepted. @@ -3204,7 +3455,7 @@ def test_long_context_chain(self): def f(): try: 1/0 - except: + except ZeroDivisionError: f() try: @@ -3308,7 +3559,7 @@ def test_comparison_params_variations(self): def raise_exc(): try: raise ValueError('bad value') - except: + except ValueError: raise def raise_with_locals(): @@ -3702,6 +3953,27 @@ class CaseChangeOverSubstitution: actual = self.get_suggestion(cls(), 'bluch') self.assertIn(suggestion, actual) + def test_getattr_suggestions_underscored(self): + class A: + bluch = None + + self.assertIn("'bluch'", self.get_suggestion(A(), 'blach')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_luch')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_bluch')) + + class B: + _bluch = None + def method(self, name): + getattr(self, name) + + self.assertIn("'_bluch'", self.get_suggestion(B(), '_blach')) + self.assertIn("'_bluch'", self.get_suggestion(B(), '_luch')) + self.assertNotIn("'_bluch'", self.get_suggestion(B(), 'bluch')) + + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_blach'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): class A: blech = None @@ -3894,6 +4166,17 @@ def test_import_from_suggestions(self): actual = self.get_import_from_suggestion(code, 'bluch') self.assertIn(suggestion, actual) + def test_import_from_suggestions_underscored(self): + code = "bluch = None" + self.assertIn("'bluch'", self.get_import_from_suggestion(code, 'blach')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_bluch')) + + code = "_bluch = None" + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_blach')) + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): code = "blech = None" @@ -4274,11 +4557,14 @@ def foo(*args): x = {'a':{'b': None}} y = x['a']['b']['c'] - def baz(*args): - return foo(1,2,3,4) + def baz2(*args): + return (lambda *args: foo(*args))(1,2,3,4) + + def baz1(*args): + return baz2(1,2,3,4) def bar(): - return baz(1, + return baz1(1, 2,3 ,4) try: @@ -4288,14 +4574,14 @@ def bar(): e, capture_locals=True ) lines = "".join(exc.format(colorize=True)) - red = traceback._ANSIColors.RED - boldr = traceback._ANSIColors.BOLD_RED - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) - self.assertIn("return " + red + "foo" + reset + boldr + "(1,2,3,4)" + reset, lines) - self.assertIn("return " + red + "baz" + reset + boldr + "(1," + reset, lines) - self.assertIn(boldr + "2,3" + reset, lines) - self.assertIn(boldr + ",4)" + reset, lines) + self.assertIn("return " + red + "(lambda *args: foo(*args))" + reset + boldr + "(1,2,3,4)" + reset, lines) + self.assertIn("return (lambda *args: " + red + "foo" + reset + boldr + "(*args)" + reset + ")(1,2,3,4)", lines) + self.assertIn("return baz2(1,2,3,4)", lines) + self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) def test_colorized_syntax_error(self): @@ -4306,11 +4592,11 @@ def test_colorized_syntax_error(self): e, capture_locals=True ) actual = "".join(exc.format(colorize=True)) - red = traceback._ANSIColors.RED - magenta = traceback._ANSIColors.MAGENTA - boldm = traceback._ANSIColors.BOLD_MAGENTA - boldr = traceback._ANSIColors.BOLD_RED - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET expected = "".join([ f' File {magenta}""{reset}, line {magenta}1{reset}\n', f' a {boldr}${reset} b\n', @@ -4329,15 +4615,15 @@ def foo(): self.fail("No exception thrown.") except Exception as e: with captured_output("stderr") as tbstderr: - with unittest.mock.patch('traceback._can_colorize', return_value=True): + with unittest.mock.patch('_colorize.can_colorize', return_value=True): exception_print(e) actual = tbstderr.getvalue().splitlines() - red = traceback._ANSIColors.RED - boldr = traceback._ANSIColors.BOLD_RED - magenta = traceback._ANSIColors.MAGENTA - boldm = traceback._ANSIColors.BOLD_MAGENTA - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + reset = _colorize.ANSIColors.RESET lno_foo = foo.__code__.co_firstlineno expected = ['Traceback (most recent call last):', f' File {magenta}"{__file__}"{reset}, ' @@ -4351,32 +4637,49 @@ def foo(): f'{boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}'] self.assertEqual(actual, expected) - def test_colorized_detection_checks_for_environment_variables(self): - if sys.platform == "win32": - virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", return_value=True) - else: - virtual_patching = contextlib.nullcontext() - with virtual_patching: - with unittest.mock.patch("os.isatty") as isatty_mock: - isatty_mock.return_value = True - with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): - self.assertEqual(traceback._can_colorize(), False) - isatty_mock.return_value = False - self.assertEqual(traceback._can_colorize(), False) + def test_colorized_traceback_from_exception_group(self): + def foo(): + exceptions = [] + try: + 1 / 0 + except ZeroDivisionError as inner_exc: + exceptions.append(inner_exc) + raise ExceptionGroup("test", exceptions) + + try: + foo() + except Exception as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + reset = _colorize.ANSIColors.RESET + lno_foo = foo.__code__.co_firstlineno + actual = "".join(exc.format(colorize=True)).splitlines() + expected = [f" + Exception Group Traceback (most recent call last):", + f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+9}{reset}, in {magenta}test_colorized_traceback_from_exception_group{reset}', + f' | {red}foo{reset}{boldr}(){reset}', + f' | {red}~~~{reset}{boldr}^^{reset}', + f" | e = ExceptionGroup('test', [ZeroDivisionError('division by zero')])", + f" | foo = {foo}", + f' | self = <{__name__}.TestColorizedTraceback testMethod=test_colorized_traceback_from_exception_group>', + f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+6}{reset}, in {magenta}foo{reset}', + f' | raise ExceptionGroup("test", exceptions)', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {boldm}ExceptionGroup{reset}: {magenta}test (1 sub-exception){reset}', + f' +-+---------------- 1 ----------------', + f' | Traceback (most recent call last):', + f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+3}{reset}, in {magenta}foo{reset}', + f' | {red}1 {reset}{boldr}/{reset}{red} 0{reset}', + f' | {red}~~{reset}{boldr}^{reset}{red}~~{reset}', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}', + f' +------------------------------------'] + self.assertEqual(actual, expected) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index bea124521032d1..5755f7697de91a 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -8,6 +8,7 @@ interpreter_requires_environment) from test import support from test.support import os_helper +from test.support import force_not_colorized try: import _testcapi @@ -938,6 +939,7 @@ def test_env_limit(self): stdout = stdout.rstrip() self.assertEqual(stdout, b'10') + @force_not_colorized def check_env_var_invalid(self, nframe): with support.SuppressCrashReport(): ok, stdout, stderr = assert_python_failure( diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 9a04a95dc40d65..eeaf5de2e303f6 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -227,13 +227,13 @@ def test_element_create_image(self): foreground='blue', background='yellow') img3 = tkinter.BitmapImage(master=self.root, file=imgfile, foreground='white', background='black') - style.element_create('Button.button', 'image', + style.element_create('TestButton.button', 'image', img1, ('pressed', img2), ('active', img3), border=(2, 4), sticky='we') - self.assertIn('Button.button', style.element_names()) + self.assertIn('TestButton.button', style.element_names()) - style.layout('Button', [('Button.button', {'sticky': 'news'})]) - b = ttk.Button(self.root, style='Button') + style.layout('TestButton', [('TestButton.button', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='TestButton') b.pack(expand=True, fill='both') self.assertEqual(b.winfo_reqwidth(), 16) self.assertEqual(b.winfo_reqheight(), 16) diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index fd1a748a498ac5..10bec33be617a1 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -5,8 +5,9 @@ import sys from test.test_ttk_textonly import MockTclObj -from test.test_tkinter.support import (AbstractTkTest, tk_version, get_tk_patchlevel, - simulate_mouse_click, AbstractDefaultRootTest) +from test.test_tkinter.support import ( + AbstractTkTest, requires_tk, tk_version, get_tk_patchlevel, + simulate_mouse_click, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import (add_standard_options, AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) @@ -27,16 +28,27 @@ def test_configure_class(self): def test_configure_padding(self): widget = self.create() - self.checkParam(widget, 'padding', 0, expected=('0',)) - self.checkParam(widget, 'padding', 5, expected=('5',)) - self.checkParam(widget, 'padding', (5, 6), expected=('5', '6')) + if get_tk_patchlevel(self.root) < (8, 6, 14): + def padding_conv(value): + self.assertIsInstance(value, tuple) + return tuple(map(str, value)) + else: + padding_conv = None + self.checkParam(widget, 'padding', 0, expected=(0,), conv=padding_conv) + self.checkParam(widget, 'padding', 5, expected=(5,), conv=padding_conv) + self.checkParam(widget, 'padding', (5, 6), + expected=(5, 6), conv=padding_conv) self.checkParam(widget, 'padding', (5, 6, 7), - expected=('5', '6', '7')) + expected=(5, 6, 7), conv=padding_conv) self.checkParam(widget, 'padding', (5, 6, 7, 8), - expected=('5', '6', '7', '8')) + expected=(5, 6, 7, 8), conv=padding_conv) self.checkParam(widget, 'padding', ('5p', '6p', '7p', '8p')) self.checkParam(widget, 'padding', (), expected='') + def test_configure_state(self): + widget = self.create() + self.checkParams(widget, 'state', 'active', 'disabled', 'readonly') + def test_configure_style(self): widget = self.create() self.assertEqual(widget['style'], '') @@ -50,6 +62,11 @@ def test_configure_style(self): self.assertEqual(widget2['class'], 'Foo') # XXX + def test_configure_relief(self): + widget = self.create() + self.checkReliefParam(widget, 'relief', + allow_empty=(tk_version >= (8, 7))) + class WidgetTest(AbstractTkTest, unittest.TestCase): """Tests methods available in every ttk widget.""" @@ -150,6 +167,7 @@ def test_configure_labelwidget(self): class AbstractLabelTest(AbstractWidgetTest): + _allow_empty_justify = True def checkImageParam(self, widget, name): image = tkinter.PhotoImage(master=self.root, name='image1') @@ -165,17 +183,13 @@ def checkImageParam(self, widget, name): errmsg='image "spam" doesn\'t exist') def test_configure_compound(self): - options = 'none text image center top bottom left right'.split() - errmsg = ( - 'bad compound "{}": must be' - f' {", ".join(options[:-1])}, or {options[-1]}' - ) + values = ('none', 'text', 'image', 'center', 'top', 'bottom', 'left', 'right') + if tk_version >= (8, 7): + values += ('',) widget = self.create() - self.checkEnumParam(widget, 'compound', *options, errmsg=errmsg) + self.checkEnumParam(widget, 'compound', *values, allow_empty=True) - def test_configure_state(self): - widget = self.create() - self.checkParams(widget, 'state', 'active', 'disabled', 'normal') + test_configure_justify = requires_tk(8, 7)(StandardOptionsTests.test_configure_justify) def test_configure_width(self): widget = self.create() @@ -192,21 +206,19 @@ class LabelTest(AbstractLabelTest, unittest.TestCase): 'underline', 'width', 'wraplength', ) _conv_pixels = False + _allow_empty_justify = tk_version >= (8, 7) def create(self, **kwargs): return ttk.Label(self.root, **kwargs) - def test_configure_font(self): - widget = self.create() - self.checkParam(widget, 'font', - '-Adobe-Helvetica-Medium-R-Normal--*-120-*-*-*-*-*-*') + test_configure_justify = StandardOptionsTests.test_configure_justify @add_standard_options(StandardTtkOptionsTests) class ButtonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', 'default', - 'image', 'padding', 'state', 'style', + 'image', 'justify', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', 'underline', 'width', ) @@ -216,7 +228,9 @@ def create(self, **kwargs): def test_configure_default(self): widget = self.create() - self.checkEnumParam(widget, 'default', 'normal', 'active', 'disabled') + values = ('normal', 'active', 'disabled') + self.checkEnumParam(widget, 'default', *values, + sort=tk_version >= (8, 7)) def test_invoke(self): success = [] @@ -229,7 +243,7 @@ def test_invoke(self): class CheckbuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', - 'image', + 'image', 'justify', 'offvalue', 'onvalue', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', @@ -268,7 +282,10 @@ def cb_test(): cbtn['command'] = '' res = cbtn.invoke() - self.assertFalse(str(res)) + if tk_version >= (8, 7) and self.wantobjects: + self.assertEqual(res, ()) + else: + self.assertEqual(str(res), '') self.assertLessEqual(len(success), 1) self.assertEqual(cbtn['offvalue'], cbtn.tk.globalgetvar(cbtn['variable'])) @@ -285,9 +302,29 @@ def test_unique_variables(self): b.pack() buttons.append(b) variables = [str(b['variable']) for b in buttons] - print(variables) self.assertEqual(len(set(variables)), 4, variables) + def test_unique_variables2(self): + buttons = [] + f = ttk.Frame(self.root) + f.pack() + f = ttk.Frame(self.root) + f.pack() + for j in 'AB': + b = tkinter.Checkbutton(f, text=j) + b.pack() + buttons.append(b) + # Should be larger than the number of all previously created + # tkinter.Checkbutton widgets: + for j in range(100): + b = ttk.Checkbutton(f, text=str(j)) + b.pack() + buttons.append(b) + names = [str(b) for b in buttons] + self.assertEqual(len(set(names)), len(buttons), names) + variables = [str(b['variable']) for b in buttons] + self.assertEqual(len(set(variables)), len(buttons), variables) + @add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) class EntryTest(AbstractWidgetTest, unittest.TestCase): @@ -295,10 +332,12 @@ class EntryTest(AbstractWidgetTest, unittest.TestCase): 'background', 'class', 'cursor', 'exportselection', 'font', 'foreground', 'invalidcommand', 'justify', + 'placeholder', 'placeholderforeground', 'show', 'state', 'style', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'width', 'xscrollcommand', ) - IDENTIFY_AS = 'Entry.field' if sys.platform == 'darwin' else 'textarea' + # bpo-27313: macOS Tk/Tcl may or may not report 'Entry.field'. + IDENTIFY_AS = {'Entry.field', 'textarea'} def setUp(self): super().setUp() @@ -317,11 +356,6 @@ def test_configure_show(self): self.checkParam(widget, 'show', '') self.checkParam(widget, 'show', ' ') - def test_configure_state(self): - widget = self.create() - self.checkParams(widget, 'state', - 'disabled', 'normal', 'readonly') - def test_configure_validate(self): widget = self.create() self.checkEnumParam(widget, 'validate', @@ -340,8 +374,7 @@ def test_identify(self): self.entry.pack() self.entry.update() - # bpo-27313: macOS Cocoa widget differs from X, allow either - self.assertEqual(self.entry.identify(5, 5), self.IDENTIFY_AS) + self.assertIn(self.entry.identify(5, 5), self.IDENTIFY_AS) self.assertEqual(self.entry.identify(-1, -1), "") self.assertRaises(tkinter.TclError, self.entry.identify, None, 5) @@ -422,12 +455,13 @@ class ComboboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'cursor', 'exportselection', 'font', 'foreground', 'height', 'invalidcommand', - 'justify', 'postcommand', 'show', 'state', 'style', + 'justify', 'placeholder', 'placeholderforeground', 'postcommand', + 'show', 'state', 'style', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'values', 'width', 'xscrollcommand', ) - IDENTIFY_AS = 'Combobox.button' if sys.platform == 'darwin' else 'textarea' + IDENTIFY_AS = {'Combobox.button', 'textarea'} def setUp(self): super().setUp() @@ -486,7 +520,7 @@ def check_get_current(getval, currval): self.assertEqual(self.combo.get(), getval) self.assertEqual(self.combo.current(), currval) - self.assertEqual(self.combo['values'], '') + self.assertIn(self.combo['values'], ((), '')) check_get_current('', -1) self.checkParam(self.combo, 'values', 'mon tue wed thur', @@ -611,8 +645,14 @@ def test_insert(self): child2 = ttk.Label(self.root) child3 = ttk.Label(self.root) - self.assertRaises(tkinter.TclError, self.paned.insert, 0, child) + if tk_version >= (8, 7): + self.paned.insert(0, child) + self.assertEqual(self.paned.panes(), (str(child),)) + self.paned.forget(0) + else: + self.assertRaises(tkinter.TclError, self.paned.insert, 0, child) + self.assertEqual(self.paned.panes(), ()) self.paned.insert('end', child2) self.paned.insert(0, child) self.assertEqual(self.paned.panes(), (str(child), str(child2))) @@ -676,7 +716,7 @@ def test_sashpos(self): class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', - 'image', + 'image', 'justify', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', 'underline', 'value', 'variable', 'width', @@ -715,7 +755,10 @@ def cb_test(): cbtn2['command'] = '' res = cbtn2.invoke() - self.assertEqual(str(res), '') + if tk_version >= (8, 7) and self.wantobjects: + self.assertEqual(res, ()) + else: + self.assertEqual(str(res), '') self.assertLessEqual(len(success), 1) self.assertEqual(conv(cbtn2['value']), myvar.get()) self.assertEqual(myvar.get(), @@ -727,7 +770,7 @@ def cb_test(): class MenubuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'compound', 'cursor', 'direction', - 'image', 'menu', 'padding', 'state', 'style', + 'image', 'justify', 'menu', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', 'underline', 'width', ) @@ -735,10 +778,11 @@ class MenubuttonTest(AbstractLabelTest, unittest.TestCase): def create(self, **kwargs): return ttk.Menubutton(self.root, **kwargs) - def test_direction(self): + def test_configure_direction(self): widget = self.create() - self.checkEnumParam(widget, 'direction', - 'above', 'below', 'left', 'right', 'flush') + values = ('above', 'below', 'left', 'right', 'flush') + self.checkEnumParam(widget, 'direction', *values, + sort=tk_version >= (8, 7)) def test_configure_menu(self): widget = self.create() @@ -751,7 +795,7 @@ def test_configure_menu(self): class ScaleTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'cursor', 'from', 'length', - 'orient', 'style', 'takefocus', 'to', 'value', 'variable', + 'orient', 'state', 'style', 'takefocus', 'to', 'value', 'variable', ) _conv_pixels = False default_orient = 'horizontal' @@ -773,6 +817,8 @@ def test_configure_length(self): widget = self.create() self.checkPixelsParam(widget, 'length', 130, 131.2, 135.6, '5i') + test_configure_state = requires_tk(8, 6, 9)(StandardTtkOptionsTests.test_configure_state) + def test_configure_to(self): widget = self.create() self.checkFloatParam(widget, 'to', 300, 14.9, 15.1, -10, conv=False) @@ -856,16 +902,28 @@ def test_set(self): @add_standard_options(StandardTtkOptionsTests) class ProgressbarTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( - 'class', 'cursor', 'orient', 'length', - 'mode', 'maximum', 'phase', + 'anchor', 'class', 'cursor', 'font', 'foreground', 'justify', + 'orient', 'length', + 'mode', 'maximum', 'phase', 'text', 'wraplength', 'style', 'takefocus', 'value', 'variable', ) _conv_pixels = False + _allow_empty_justify = True default_orient = 'horizontal' def create(self, **kwargs): return ttk.Progressbar(self.root, **kwargs) + @requires_tk(8, 7) + def test_configure_anchor(self): + widget = self.create() + self.checkEnumParam(widget, 'anchor', + 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center', '') + + test_configure_font = requires_tk(8, 7)(StandardOptionsTests.test_configure_font) + test_configure_foreground = requires_tk(8, 7)(StandardOptionsTests.test_configure_foreground) + test_configure_justify = requires_tk(8, 7)(StandardTtkOptionsTests.test_configure_justify) + def test_configure_length(self): widget = self.create() self.checkPixelsParam(widget, 'length', 100.1, 56.7, '2i') @@ -882,11 +940,15 @@ def test_configure_phase(self): # XXX pass + test_configure_text = requires_tk(8, 7)(StandardOptionsTests.test_configure_text) + def test_configure_value(self): widget = self.create() self.checkFloatParam(widget, 'value', 150.2, 77.7, 0, -10, conv=False) + test_configure_wraplength = requires_tk(8, 7)(StandardOptionsTests.test_configure_wraplength) + @unittest.skipIf(sys.platform == 'darwin', 'ttk.Scrollbar is special on MacOSX') @@ -901,11 +963,13 @@ def create(self, **kwargs): return ttk.Scrollbar(self.root, **kwargs) -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_standard_options(StandardTtkOptionsTests) class NotebookTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'height', 'padding', 'style', 'takefocus', 'width', ) + if tk_version >= (8, 7): + _conv_pixels = False def setUp(self): super().setUp() @@ -918,6 +982,20 @@ def setUp(self): def create(self, **kwargs): return ttk.Notebook(self.root, **kwargs) + def test_configure_height(self): + widget = self.create() + if get_tk_patchlevel(self.root) < (8, 6, 15): + self.checkIntegerParam(widget, 'height', 402, -402, 0) + else: + self.checkPixelsParam(widget, 'height', '10c', 402, -402, 0, conv=False) + + def test_configure_width(self): + widget = self.create() + if get_tk_patchlevel(self.root) < (8, 6, 15): + self.checkIntegerParam(widget, 'width', 402, -402, 0) + else: + self.checkPixelsParam(widget, 'width', '10c', 402, -402, 0, conv=False) + def test_tab_identifiers(self): self.nb.forget(0) self.nb.hide(self.child2) @@ -1024,7 +1102,11 @@ def test_insert(self): self.nb.insert(self.child1, child3) self.assertEqual(self.nb.tabs(), (str(child3), ) + tabs) self.nb.forget(child3) - self.assertRaises(tkinter.TclError, self.nb.insert, 2, child3) + if tk_version >= (8, 7): + self.nb.insert(2, child3) + self.assertEqual(self.nb.tabs(), (*tabs, str(child3))) + else: + self.assertRaises(tkinter.TclError, self.nb.insert, 2, child3) self.assertRaises(tkinter.TclError, self.nb.insert, -1, child3) # bad inserts @@ -1116,11 +1198,13 @@ class SpinboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'command', 'cursor', 'exportselection', 'font', 'foreground', 'format', 'from', 'increment', - 'invalidcommand', 'justify', 'show', 'state', 'style', + 'invalidcommand', 'justify', + 'placeholder', 'placeholderforeground', + 'show', 'state', 'style', 'takefocus', 'textvariable', 'to', 'validate', 'validatecommand', 'values', 'width', 'wrap', 'xscrollcommand', ) - IDENTIFY_AS = 'Spinbox.field' if sys.platform == 'darwin' else 'textarea' + IDENTIFY_AS = {'Spinbox.field', 'textarea'} def setUp(self): super().setUp() @@ -1290,8 +1374,9 @@ def test_configure_values(self): class TreeviewTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'columns', 'cursor', 'displaycolumns', - 'height', 'padding', 'selectmode', 'show', - 'style', 'takefocus', 'xscrollcommand', 'yscrollcommand', + 'height', 'padding', 'selectmode', 'selecttype', 'show', 'striped', + 'style', 'takefocus', 'titlecolumns', 'titleitems', + 'xscrollcommand', 'yscrollcommand', ) def setUp(self): @@ -1306,7 +1391,8 @@ def test_configure_columns(self): self.checkParam(widget, 'columns', 'a b c', expected=('a', 'b', 'c')) self.checkParam(widget, 'columns', ('a', 'b', 'c')) - self.checkParam(widget, 'columns', '') + self.checkParam(widget, 'columns', '', + expected=() if tk_version >= (8, 7) else '') def test_configure_displaycolumns(self): widget = self.create() @@ -1318,11 +1404,12 @@ def test_configure_displaycolumns(self): expected=('#all',)) self.checkParam(widget, 'displaycolumns', (2, 1, 0)) self.checkInvalidParam(widget, 'displaycolumns', ('a', 'b', 'd'), - errmsg='Invalid column index d') + errmsg='Invalid column index "?d"?') + errmsg = 'Column index "?{}"? out of bounds' self.checkInvalidParam(widget, 'displaycolumns', (1, 2, 3), - errmsg='Column index 3 out of bounds') + errmsg=errmsg.format(3)) self.checkInvalidParam(widget, 'displaycolumns', (1, -2), - errmsg='Column index -2 out of bounds') + errmsg=errmsg.format(-2)) def test_configure_height(self): widget = self.create() @@ -1334,6 +1421,11 @@ def test_configure_selectmode(self): self.checkEnumParam(widget, 'selectmode', 'none', 'browse', 'extended') + @requires_tk(8, 7) + def test_configure_selecttype(self): + widget = self.create() + self.checkEnumParam(widget, 'selecttype', 'item', 'cell') + def test_configure_show(self): widget = self.create() self.checkParam(widget, 'show', 'tree headings', @@ -1343,6 +1435,23 @@ def test_configure_show(self): self.checkParam(widget, 'show', 'tree', expected=('tree',)) self.checkParam(widget, 'show', 'headings', expected=('headings',)) + @requires_tk(8, 7) + def test_configure_striped(self): + widget = self.create() + self.checkBooleanParam(widget, 'striped') + + @requires_tk(8, 7) + def test_configure_titlecolumns(self): + widget = self.create() + self.checkIntegerParam(widget, 'titlecolumns', 0, 1, 5) + self.checkInvalidParam(widget, 'titlecolumns', -2) + + @requires_tk(8, 7) + def test_configure_titleitems(self): + widget = self.create() + self.checkIntegerParam(widget, 'titleitems', 0, 1, 5) + self.checkInvalidParam(widget, 'titleitems', -2) + def test_bbox(self): self.tv.pack() self.assertEqual(self.tv.bbox(''), '') @@ -1859,13 +1968,5 @@ def test_label(self): self._test_widget(ttk.Label) -tests_gui = ( - ButtonTest, CheckbuttonTest, ComboboxTest, EntryTest, - FrameTest, LabelFrameTest, LabelTest, MenubuttonTest, - NotebookTest, PanedWindowTest, ProgressbarTest, - RadiobuttonTest, ScaleTest, ScrollbarTest, SeparatorTest, - SizegripTest, SpinboxTest, TreeviewTest, WidgetTest, DefaultRootTest, - ) - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 14121a590a5026..c75a002a89b4c4 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -1,5 +1,9 @@ +import os import pickle +import re import unittest +import unittest.mock +import tempfile from test import support from test.support import import_helper from test.support import os_helper @@ -130,6 +134,7 @@ def assertVectorsAlmostEqual(self, vec1, vec2): self.assertAlmostEqual( i, j, msg='values at index {} do not match'.format(idx)) + class Multiplier: def __mul__(self, other): @@ -461,6 +466,67 @@ def test_teleport(self): self.assertTrue(tpen.isdown()) +class TestTurtleScreen(unittest.TestCase): + def test_save_raises_if_wrong_extension(self) -> None: + screen = unittest.mock.Mock() + + msg = "Unknown file extension: '.png', must be one of {'.ps', '.eps'}" + with ( + tempfile.TemporaryDirectory() as tmpdir, + self.assertRaisesRegex(ValueError, re.escape(msg)) + ): + turtle.TurtleScreen.save(screen, os.path.join(tmpdir, "file.png")) + + def test_save_raises_if_parent_not_found(self) -> None: + screen = unittest.mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir: + parent = os.path.join(tmpdir, "unknown_parent") + msg = f"The directory '{parent}' does not exist. Cannot save to it" + + with self.assertRaisesRegex(FileNotFoundError, re.escape(msg)): + turtle.TurtleScreen.save(screen, os.path.join(parent, "a.ps")) + + def test_save_raises_if_file_found(self) -> None: + screen = unittest.mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + with open(file_path, "w") as f: + f.write("some text") + + msg = ( + f"The file '{file_path}' already exists. To overwrite it use" + " the 'overwrite=True' argument of the save function." + ) + with self.assertRaisesRegex(FileExistsError, re.escape(msg)): + turtle.TurtleScreen.save(screen, file_path) + + def test_save_overwrites_if_specified(self) -> None: + screen = unittest.mock.Mock() + screen.cv.postscript.return_value = "postscript" + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + with open(file_path, "w") as f: + f.write("some text") + + turtle.TurtleScreen.save(screen, file_path, overwrite=True) + with open(file_path) as f: + assert f.read() == "postscript" + + def test_save(self) -> None: + screen = unittest.mock.Mock() + screen.cv.postscript.return_value = "postscript" + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + + turtle.TurtleScreen.save(screen, file_path) + with open(file_path) as f: + assert f.read() == "postscript" + + class TestModuleLevel(unittest.TestCase): def test_all_signatures(self): import inspect diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 9c325bc595f585..230bbe646baf28 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -4,7 +4,9 @@ from test.support import check_syntax_error, run_code from test.typinganndata import mod_generics_cache -from typing import Callable, TypeAliasType, TypeVar, get_args +from typing import ( + Callable, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, get_args, +) class TypeParamsInvalidTest(unittest.TestCase): @@ -211,6 +213,59 @@ def test_generic(self): self.assertEqual(TA.__value__, list[T]) self.assertEqual(TA.__type_params__, (T,)) self.assertEqual(TA.__module__, __name__) + self.assertIs(type(TA[int]), types.GenericAlias) + + def test_not_generic(self): + TA = TypeAliasType("TA", list[int], type_params=()) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[int]) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + with self.assertRaisesRegex( + TypeError, + "Only generic type aliases are subscriptable", + ): + TA[int] + + def test_type_params_order_with_defaults(self): + HasNoDefaultT = TypeVar("HasNoDefaultT") + WithDefaultT = TypeVar("WithDefaultT", default=int) + + HasNoDefaultP = ParamSpec("HasNoDefaultP") + WithDefaultP = ParamSpec("WithDefaultP", default=HasNoDefaultP) + + HasNoDefaultTT = TypeVarTuple("HasNoDefaultTT") + WithDefaultTT = TypeVarTuple("WithDefaultTT", default=HasNoDefaultTT) + + for type_params in [ + (HasNoDefaultT, WithDefaultT), + (HasNoDefaultP, WithDefaultP), + (HasNoDefaultTT, WithDefaultTT), + ]: + with self.subTest(type_params=type_params): + TypeAliasType("A", int, type_params=type_params) # ok + + msg = "follows default type parameter" + for type_params in [ + (WithDefaultT, HasNoDefaultT), + (WithDefaultP, HasNoDefaultP), + (WithDefaultTT, HasNoDefaultTT), + (WithDefaultT, HasNoDefaultP), # different types + ]: + with self.subTest(type_params=type_params): + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=type_params) + + def test_expects_type_like(self): + T = TypeVar("T") + + msg = "Expected a type param" + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1,)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1, 2)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(T, 2)) def test_keywords(self): TA = TypeAliasType(name="TA", value=int) @@ -328,3 +383,22 @@ def test_pickling_local(self): with self.subTest(thing=thing, proto=proto): with self.assertRaises(pickle.PickleError): pickle.dumps(thing, protocol=proto) + + +class TypeParamsExoticGlobalsTest(unittest.TestCase): + def test_exec_with_unusual_globals(self): + class customdict(dict): + def __missing__(self, key): + return key + + code = compile("type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["Alias"] + self.assertEqual(Alias.__value__, "undefined") + + code = compile("class A: type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["A"].Alias + self.assertEqual(Alias.__value__, "undefined") diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 3dbb35afcb620f..257b7fa95dcb76 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,6 +1,9 @@ +import annotationlib import textwrap +import types import unittest -from test.support import run_code +from test.support import run_code, check_syntax_error + class TypeAnnotationTests(unittest.TestCase): @@ -48,6 +51,7 @@ def test_annotations_are_created_correctly(self): class C: a:int=3 b:str=4 + self.assertEqual(C.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in C.__dict__) del C.__annotations__ self.assertFalse("__annotations__" in C.__dict__) @@ -105,6 +109,13 @@ class D(metaclass=C): self.assertEqual(D.__annotations__, {}) +def build_module(code: str, name: str = "top") -> types.ModuleType: + ns = run_code(code) + mod = types.ModuleType(name) + mod.__dict__.update(ns) + return mod + + class TestSetupAnnotations(unittest.TestCase): def check(self, code: str): code = textwrap.dedent(code) @@ -112,11 +123,10 @@ def check(self, code: str): with self.subTest(scope=scope): if scope == "class": code = f"class C:\n{textwrap.indent(code, ' ')}" - ns = run_code(code) - if scope == "class": + ns = run_code(code) annotations = ns["C"].__annotations__ else: - annotations = ns["__annotations__"] + annotations = build_module(code).__annotations__ self.assertEqual(annotations, {"x": int}) def test_top_level(self): @@ -212,3 +222,189 @@ def test_match(self): case 0: x: int = 1 """) + + +class AnnotateTests(unittest.TestCase): + """See PEP 649.""" + def test_manual_annotate(self): + def f(): + pass + mod = types.ModuleType("mod") + class X: + pass + + for obj in (f, mod, X): + with self.subTest(obj=obj): + self.check_annotations(obj) + + def check_annotations(self, f): + self.assertEqual(f.__annotations__, {}) + self.assertIs(f.__annotate__, None) + + with self.assertRaisesRegex(TypeError, "__annotate__ must be callable or None"): + f.__annotate__ = 42 + f.__annotate__ = lambda: 42 + with self.assertRaisesRegex(TypeError, r"takes 0 positional arguments but 1 was given"): + print(f.__annotations__) + + f.__annotate__ = lambda x: 42 + with self.assertRaisesRegex(TypeError, r"__annotate__ returned non-dict of type 'int'"): + print(f.__annotations__) + + f.__annotate__ = lambda x: {"x": x} + self.assertEqual(f.__annotations__, {"x": 1}) + + # Setting annotate to None does not invalidate the cached __annotations__ + f.__annotate__ = None + self.assertEqual(f.__annotations__, {"x": 1}) + + # But setting it to a new callable does + f.__annotate__ = lambda x: {"y": x} + self.assertEqual(f.__annotations__, {"y": 1}) + + # Setting f.__annotations__ also clears __annotate__ + f.__annotations__ = {"z": 43} + self.assertIs(f.__annotate__, None) + + +class DeferredEvaluationTests(unittest.TestCase): + def test_function(self): + def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_async_function(self): + async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_class(self): + class X: + a: undefined + + with self.assertRaises(NameError): + X.__annotations__ + + undefined = 1 + self.assertEqual(X.__annotations__, {"a": 1}) + + def test_module(self): + ns = run_code("x: undefined = 1") + anno = ns["__annotate__"] + with self.assertRaises(NotImplementedError): + anno(2) + + with self.assertRaises(NameError): + anno(1) + + ns["undefined"] = 1 + self.assertEqual(anno(1), {"x": 1}) + + def test_class_scoping(self): + class Outer: + def meth(self, x: Nested): ... + x: Nested + class Nested: ... + + self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested}) + self.assertEqual(Outer.__annotations__, {"x": Outer.Nested}) + + def test_no_exotic_expressions(self): + check_syntax_error(self, "def func(x: (yield)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (await 42)): ...", "await expression cannot be used within an annotation") + + def test_no_exotic_expressions_in_unevaluated_annotations(self): + preludes = [ + "", + "class X: ", + "def f(): ", + "async def f(): ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation") + + def test_ignore_non_simple_annotations(self): + ns = run_code("class X: (y): int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int.b: int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int[str]: int") + self.assertEqual(ns["X"].__annotations__, {}) + + def test_generated_annotate(self): + def func(x: int): + pass + class X: + x: int + mod = build_module("x: int") + for obj in (func, X, mod): + with self.subTest(obj=obj): + annotate = obj.__annotate__ + self.assertIsInstance(annotate, types.FunctionType) + self.assertEqual(annotate.__name__, "__annotate__") + with self.assertRaises(NotImplementedError): + annotate(annotationlib.Format.FORWARDREF) + with self.assertRaises(NotImplementedError): + annotate(annotationlib.Format.STRING) + with self.assertRaises(NotImplementedError): + annotate(None) + self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) + + def test_comprehension_in_annotation(self): + # This crashed in an earlier version of the code + ns = run_code("x: [y for y in range(10)]") + self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + + def f(x: int) -> int: pass + """ + ns = run_code(code) + f = ns["f"] + self.assertIsInstance(f.__annotate__, types.FunctionType) + annos = {"x": "int", "return": "int"} + self.assertEqual(f.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(f.__annotations__, annos) + + def test_name_clash_with_format(self): + # this test would fail if __annotate__'s parameter was called "format" + code = """ + class format: pass + + def f(x: format): pass + """ + ns = run_code(code) + f = ns["f"] + self.assertEqual(f.__annotations__, {"x": ns["format"]}) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 58572c6f4d3157..66abe73f8d766d 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -2,7 +2,7 @@ import unittest import dis from test import support -from test.support import import_helper +from test.support import import_helper, requires_specialization try: from sys import _clear_type_cache except ImportError: @@ -10,8 +10,9 @@ # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module("_testcapi") +_testinternalcapi = import_helper.import_module("_testinternalcapi") type_get_version = _testcapi.type_get_version -type_assign_specific_version_unsafe = _testcapi.type_assign_specific_version_unsafe +type_assign_specific_version_unsafe = _testinternalcapi.type_assign_specific_version_unsafe type_assign_version = _testcapi.type_assign_version type_modified = _testcapi.type_modified @@ -92,8 +93,24 @@ class C: new_version = type_get_version(C) self.assertEqual(new_version, 0) + def test_119462(self): + + class Holder: + value = None + + @classmethod + def set_value(cls): + cls.value = object() + + class HolderSub(Holder): + pass + + for _ in range(1050): + Holder.set_value() + HolderSub.value @support.cpython_only +@requires_specialization class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): _clear_type_cache() @@ -104,8 +121,10 @@ def _assign_valid_version_or_skip(self, type_): if type_get_version(type_) == 0: self.skipTest("Could not assign valid type version") - def _assign_and_check_version_0(self, user_type): + def _no_more_versions(self, user_type): type_modified(user_type) + for _ in range(1001): + type_assign_specific_version_unsafe(user_type, 1000_000_000) type_assign_specific_version_unsafe(user_type, 0) self.assertEqual(type_get_version(user_type), 0) @@ -134,7 +153,7 @@ def load_foo_1(type_): self._check_specialization(load_foo_1, A, "LOAD_ATTR", should_specialize=True) del load_foo_1 - self._assign_and_check_version_0(A) + self._no_more_versions(A) def load_foo_2(type_): return type_.foo @@ -142,8 +161,8 @@ def load_foo_2(type_): self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False) def test_class_load_attr_specialization_static_type(self): - self._assign_valid_version_or_skip(str) - self._assign_valid_version_or_skip(bytes) + self.assertNotEqual(type_get_version(str), 0) + self.assertNotEqual(type_get_version(bytes), 0) def get_capitalize_1(type_): return type_.capitalize @@ -151,25 +170,6 @@ def get_capitalize_1(type_): self._check_specialization(get_capitalize_1, str, "LOAD_ATTR", should_specialize=True) self.assertEqual(get_capitalize_1(str)('hello'), 'Hello') self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello') - del get_capitalize_1 - - # Permanently overflow the static type version counter, and force str and bytes - # to have tp_version_tag == 0 - for _ in range(2**16): - type_modified(str) - type_assign_version(str) - type_modified(bytes) - type_assign_version(bytes) - - self.assertEqual(type_get_version(str), 0) - self.assertEqual(type_get_version(bytes), 0) - - def get_capitalize_2(type_): - return type_.capitalize - - self._check_specialization(get_capitalize_2, str, "LOAD_ATTR", should_specialize=False) - self.assertEqual(get_capitalize_2(str)('hello'), 'Hello') - self.assertEqual(get_capitalize_2(bytes)(b'hello'), b'Hello') def test_property_load_attr_specialization_user_type(self): class G: @@ -185,7 +185,7 @@ def load_x_1(instance): self._check_specialization(load_x_1, G(), "LOAD_ATTR", should_specialize=True) del load_x_1 - self._assign_and_check_version_0(G) + self._no_more_versions(G) def load_x_2(instance): instance.x @@ -204,7 +204,7 @@ def store_bar_1(type_): self._check_specialization(store_bar_1, B(), "STORE_ATTR", should_specialize=True) del store_bar_1 - self._assign_and_check_version_0(B) + self._no_more_versions(B) def store_bar_2(type_): type_.bar = 10 @@ -224,7 +224,7 @@ def call_class_1(type_): self._check_specialization(call_class_1, F, "CALL", should_specialize=True) del call_class_1 - self._assign_and_check_version_0(F) + self._no_more_versions(F) def call_class_2(type_): type_() @@ -243,7 +243,7 @@ def to_bool_1(instance): self._check_specialization(to_bool_1, H(), "TO_BOOL", should_specialize=True) del to_bool_1 - self._assign_and_check_version_0(H) + self._no_more_versions(H) def to_bool_2(instance): not instance diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index 5a911da56f8f8a..ee8939f62d082c 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -309,7 +309,7 @@ def test_withstmt(self): self.assertEqual(tree.body[0].type_comment, None) def test_parenthesized_withstmt(self): - for tree in self.parse_all(parenthesized_withstmt, minver=9): + for tree in self.parse_all(parenthesized_withstmt): self.assertEqual(tree.body[0].type_comment, "int") self.assertEqual(tree.body[1].type_comment, "int") tree = self.classic_parse(parenthesized_withstmt) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 25ee188731f31f..433b19593bdd04 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1,3 +1,4 @@ +import annotationlib import asyncio import textwrap import types @@ -6,7 +7,7 @@ import weakref from test.support import requires_working_socket, check_syntax_error, run_code -from typing import Generic, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args +from typing import Generic, NoDefault, Sequence, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, get_args class TypeParamsInvalidTest(unittest.TestCase): @@ -412,6 +413,14 @@ def test_comprehension_02(self): func, = T.__bound__ self.assertEqual(func(), 1) + def test_comprehension_03(self): + def F[T: [lambda: T for T in (T, [1])[1]]](): return [lambda: T for T in T.__name__] + func, = F() + self.assertEqual(func(), "T") + T, = F.__type_params__ + func, = T.__bound__ + self.assertEqual(func(), 1) + def test_gen_exp_in_nested_class(self): code = """ from test.test_type_params import make_base @@ -436,9 +445,11 @@ class C[T]: class Inner[U](make_base(T for _ in (1,)), make_base(T)): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(list(base1.__arg__), [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") def test_listcomp_in_nested_class(self): code = """ @@ -464,9 +475,11 @@ class C[T]: class Inner[U](make_base([T for _ in (1,)]), make_base(T)): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(base1.__arg__, [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") def test_gen_exp_in_generic_method(self): code = """ @@ -475,29 +488,86 @@ class C[T]: def meth[U](x: (T for _ in (1,)), y: T): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + meth = ns["C"].meth + self.assertEqual(list(meth.__annotations__["x"]), [ns["C"].__type_params__[0]]) + self.assertEqual(meth.__annotations__["y"], "class") def test_nested_scope_in_generic_alias(self): code = """ - class C[T]: + T = "global" + class C: T = "class" {} """ - error_cases = [ - "type Alias1[T] = lambda: T", - "type Alias2 = lambda: T", - "type Alias3[T] = (T for _ in (1,))", - "type Alias4 = (T for _ in (1,))", - "type Alias5[T] = [T for _ in (1,)]", - "type Alias6 = [T for _ in (1,)]", + cases = [ + "type Alias[T] = (T for _ in (1,))", + "type Alias = (T for _ in (1,))", + "type Alias[T] = [T for _ in (1,)]", + "type Alias = [T for _ in (1,)]", ] - for case in error_cases: + for case in cases: with self.subTest(case=case): - with self.assertRaisesRegex(SyntaxError, - r"Cannot use [a-z]+ in annotation scope within class scope"): - run_code(code.format(case)) + ns = run_code(code.format(case)) + alias = ns["C"].Alias + value = list(alias.__value__)[0] + if alias.__type_params__: + self.assertIs(value, alias.__type_params__[0]) + else: + self.assertEqual(value, "global") + + def test_lambda_in_alias_in_class(self): + code = """ + T = "global" + class C: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertEqual(C.Alias.__value__(), "global") + + def test_lambda_in_alias_in_generic_class(self): + code = """ + class C[T]: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertIs(C.Alias.__value__(), C.__type_params__[0]) + + def test_lambda_in_generic_alias_in_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + T = U = "global" + class C: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertEqual(U, "global") + + def test_lambda_in_generic_alias_in_generic_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + class C[T, U]: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertIs(U, C.__type_params__[1]) + + def test_type_special_case(self): + # https://github.com/python/cpython/issues/119011 + self.assertEqual(type.__type_params__, ()) + self.assertEqual(object.__type_params__, ()) def make_base(arg): @@ -535,10 +605,12 @@ class Foo[T: Foo, U: (Foo, Foo)]: self.assertEqual(type_params[0].__name__, "T") self.assertIs(type_params[0].__bound__, Foo) self.assertEqual(type_params[0].__constraints__, ()) + self.assertIs(type_params[0].__default__, NoDefault) self.assertEqual(type_params[1].__name__, "U") self.assertIs(type_params[1].__bound__, None) self.assertEqual(type_params[1].__constraints__, (Foo, Foo)) + self.assertIs(type_params[1].__default__, NoDefault) def test_evaluation_error(self): class Foo[T: Undefined, U: (Undefined,)]: @@ -549,6 +621,8 @@ class Foo[T: Undefined, U: (Undefined,)]: type_params[0].__bound__ self.assertEqual(type_params[0].__constraints__, ()) self.assertIs(type_params[1].__bound__, None) + self.assertIs(type_params[0].__default__, NoDefault) + self.assertIs(type_params[1].__default__, NoDefault) with self.assertRaises(NameError): type_params[1].__constraints__ @@ -708,6 +782,31 @@ class D[U](T): self.assertIn(int, C.D.__bases__) self.assertIs(C.D.x, str) + +class DynamicClassTest(unittest.TestCase): + def _set_type_params(self, ns, params): + ns['__type_params__'] = params + + def test_types_new_class_with_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}, + lambda ns: self._set_type_params(ns, (T,))) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, (T,)) + self.assertEqual(Klass.__parameters__, (T,)) + + def test_types_new_class_no_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, ()) # must be explicitly set + self.assertEqual(Klass.__parameters__, (T,)) + + class TypeParamsManglingTest(unittest.TestCase): def test_mangling(self): class Foo[__T]: @@ -730,6 +829,100 @@ def meth[__U](self, arg: __T, arg2: __U): self.assertEqual(Foo.Alias.__value__, (T, V)) + def test_no_leaky_mangling_in_module(self): + ns = run_code(""" + __before = "before" + class X[T]: pass + __after = "after" + """) + self.assertEqual(ns["__before"], "before") + self.assertEqual(ns["__after"], "after") + + def test_no_leaky_mangling_in_function(self): + ns = run_code(""" + def f(): + class X[T]: pass + _X_foo = 2 + __foo = 1 + assert locals()['__foo'] == 1 + return __foo + """) + self.assertEqual(ns["f"](), 1) + + def test_no_leaky_mangling_in_class(self): + ns = run_code(""" + class Outer: + __before = "before" + class Inner[T]: + __x = "inner" + __after = "after" + """) + Outer = ns["Outer"] + self.assertEqual(Outer._Outer__before, "before") + self.assertEqual(Outer.Inner._Inner__x, "inner") + self.assertEqual(Outer._Outer__after, "after") + + def test_no_mangling_in_bases(self): + ns = run_code(""" + class __Base: + def __init_subclass__(self, **kwargs): + self.kwargs = kwargs + + class Derived[T](__Base, __kwarg=1): + pass + """) + Derived = ns["Derived"] + self.assertEqual(Derived.__bases__, (ns["__Base"], Generic)) + self.assertEqual(Derived.kwargs, {"__kwarg": 1}) + + def test_no_mangling_in_nested_scopes(self): + ns = run_code(""" + from test.test_type_params import make_base + + class __X: + pass + + class Y[T: __X]( + make_base(lambda: __X), + # doubly nested scope + make_base(lambda: (lambda: __X)), + # list comprehension + make_base([__X for _ in (1,)]), + # genexp + make_base(__X for _ in (1,)), + ): + pass + """) + Y = ns["Y"] + T, = Y.__type_params__ + self.assertIs(T.__bound__, ns["__X"]) + base0 = Y.__bases__[0] + self.assertIs(base0.__arg__(), ns["__X"]) + base1 = Y.__bases__[1] + self.assertIs(base1.__arg__()(), ns["__X"]) + base2 = Y.__bases__[2] + self.assertEqual(base2.__arg__, [ns["__X"]]) + base3 = Y.__bases__[3] + self.assertEqual(list(base3.__arg__), [ns["__X"]]) + + def test_type_params_are_mangled(self): + ns = run_code(""" + from test.test_type_params import make_base + + class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)): + param = __T + """) + Foo = ns["Foo"] + T, U = Foo.__type_params__ + self.assertEqual(T.__name__, "__T") + self.assertEqual(U.__name__, "__U") + self.assertIs(U.__bound__, T) + self.assertIs(Foo.param, T) + + base1, base2, *_ = Foo.__bases__ + self.assertIs(base1.__arg__, T) + self.assertIs(base2.__arg__(), T) + class TypeParamsComplexCallsTest(unittest.TestCase): def test_defaults(self): @@ -758,6 +951,7 @@ class C[T](Base, a=1, b=2, **kwargs): T, = C.__type_params__ self.assertEqual(T.__name__, "T") self.assertEqual(C.kwargs, {"a": 1, "b": 2, "c": 3}) + self.assertEqual(C.__bases__, (Base, Generic)) bases = (Base,) class C2[T](*bases, **kwargs): @@ -766,6 +960,22 @@ class C2[T](*bases, **kwargs): T, = C2.__type_params__ self.assertEqual(T.__name__, "T") self.assertEqual(C2.kwargs, {"c": 3}) + self.assertEqual(C2.__bases__, (Base, Generic)) + + def test_starargs_base(self): + class C1[T](*()): pass + + T, = C1.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C1.__bases__, (Generic,)) + + class Base: pass + bases = [Base] + class C2[T](*bases): pass + + T, = C2.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C2.__bases__, (Base, Generic)) class TypeParamsTraditionalTypeVarsTest(unittest.TestCase): @@ -1102,3 +1312,154 @@ class Inner[U](T): """ with self.assertRaises(RuntimeError): run_code(code) + + +class DefaultsTest(unittest.TestCase): + def test_defaults_on_func(self): + ns = run_code(""" + def func[T=int, **U=float, *V=None](): + pass + """) + + T, U, V = ns["func"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_class(self): + ns = run_code(""" + class C[T=int, **U=float, *V=None]: + pass + """) + + T, U, V = ns["C"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_type_alias(self): + ns = run_code(""" + type Alias[T = int, **U = float, *V = None] = int + """) + + T, U, V = ns["Alias"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_starred_invalid(self): + check_syntax_error(self, "type Alias[T = *int] = int") + check_syntax_error(self, "type Alias[**P = *int] = int") + + def test_starred_typevartuple(self): + ns = run_code(""" + default = tuple[int, str] + type Alias[*Ts = *default] = Ts + """) + + Ts, = ns["Alias"].__type_params__ + self.assertEqual(Ts.__default__, next(iter(ns["default"]))) + + def test_nondefault_after_default(self): + check_syntax_error(self, "def func[T=int, U](): pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "class C[T=int, U]: pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "type A[T=int, U] = int", "non-default type parameter 'U' follows default type parameter") + + def test_lazy_evaluation(self): + ns = run_code(""" + type Alias[T = Undefined, *U = Undefined, **V = Undefined] = int + """) + + T, U, V = ns["Alias"].__type_params__ + + with self.assertRaises(NameError): + T.__default__ + with self.assertRaises(NameError): + U.__default__ + with self.assertRaises(NameError): + V.__default__ + + ns["Undefined"] = "defined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + # Now it is cached + ns["Undefined"] = "redefined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + def test_symtable_key_regression_default(self): + # Test against the bugs that would happen if we used .default_ + # as the key in the symtable. + ns = run_code(""" + type X[T = [T for T in [T]]] = T + """) + + T, = ns["X"].__type_params__ + self.assertEqual(T.__default__, [T]) + + def test_symtable_key_regression_name(self): + # Test against the bugs that would happen if we used .name + # as the key in the symtable. + ns = run_code(""" + type X1[T = A] = T + type X2[T = B] = T + A = "A" + B = "B" + """) + + self.assertEqual(ns["X1"].__type_params__[0].__default__, "A") + self.assertEqual(ns["X2"].__type_params__[0].__default__, "B") + + +class TestEvaluateFunctions(unittest.TestCase): + def test_general(self): + type Alias = int + Alias2 = TypeAliasType("Alias2", int) + def f[T: int = int, **P = int, *Ts = int](): pass + T, P, Ts = f.__type_params__ + T2 = TypeVar("T2", bound=int, default=int) + P2 = ParamSpec("P2", default=int) + Ts2 = TypeVarTuple("Ts2", default=int) + cases = [ + Alias.evaluate_value, + Alias2.evaluate_value, + T.evaluate_bound, + T.evaluate_default, + P.evaluate_default, + Ts.evaluate_default, + T2.evaluate_bound, + T2.evaluate_default, + P2.evaluate_default, + Ts2.evaluate_default, + ] + for case in cases: + with self.subTest(case=case): + self.assertIs(case(1), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.VALUE), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) + self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') + + def test_constraints(self): + def f[T: (int, str)](): pass + T, = f.__type_params__ + T2 = TypeVar("T2", int, str) + for case in [T, T2]: + with self.subTest(case=case): + self.assertEqual(case.evaluate_constraints(1), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.VALUE), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.FORWARDREF), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.STRING), '(int, str)') + + def test_const_evaluator(self): + T = TypeVar("T", bound=int) + self.assertEqual(repr(T.evaluate_bound), ">") + + ConstEvaluator = type(T.evaluate_bound) + + with self.assertRaisesRegex(TypeError, r"cannot create '_typing\._ConstEvaluator' instances"): + ConstEvaluator() # This used to segfault. + with self.assertRaisesRegex(TypeError, r"cannot set 'attribute' attribute of immutable type '_typing\._ConstEvaluator'"): + ConstEvaluator.attribute = 1 diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 1acb2a4d81adf3..d1161719d98040 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1,8 +1,12 @@ # Python test set -- part 6, built-in types -from test.support import run_with_locale, cpython_only, MISSING_C_DOCSTRINGS +from test.support import ( + run_with_locale, is_apple_mobile, cpython_only, no_rerun, + iter_builtin_types, iter_slot_wrappers, + MISSING_C_DOCSTRINGS, +) import collections.abc -from collections import namedtuple +from collections import namedtuple, UserDict import copy import _datetime import gc @@ -10,6 +14,7 @@ import pickle import locale import sys +import textwrap import types import unittest.mock import weakref @@ -393,7 +398,7 @@ def test(i, format_spec, result): test(123456, "1=20", '11111111111111123456') test(123456, "*=20", '**************123456') - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_float__format__locale(self): # test locale support for __format__ code 'n' @@ -402,7 +407,7 @@ def test_float__format__locale(self): self.assertEqual(locale.format_string('%g', x, grouping=True), format(x, 'n')) self.assertEqual(locale.format_string('%.10g', x, grouping=True), format(x, '.10n')) - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_int__format__locale(self): # test locale support for __format__ code 'n' for integers @@ -713,6 +718,26 @@ def test_hash(self): self.assertEqual(hash(int | str), hash(str | int)) self.assertEqual(hash(int | str), hash(typing.Union[int, str])) + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual((A | B).__args__, (A, B)) + union1 = A | B + with self.assertRaises(TypeError): + hash(union1) + + union2 = int | B + with self.assertRaises(TypeError): + hash(union2) + + union3 = A | int + with self.assertRaises(TypeError): + hash(union3) + def test_instancecheck_and_subclasscheck(self): for x in (int | str, typing.Union[int, str]): with self.subTest(x=x): @@ -1735,21 +1760,50 @@ class Model(metaclass=ModelBase): class SimpleNamespaceTests(unittest.TestCase): def test_constructor(self): - ns1 = types.SimpleNamespace() - ns2 = types.SimpleNamespace(x=1, y=2) - ns3 = types.SimpleNamespace(**dict(x=1, y=2)) + def check(ns, expected): + self.assertEqual(len(ns.__dict__), len(expected)) + self.assertEqual(vars(ns), expected) + # check order + self.assertEqual(list(vars(ns).items()), list(expected.items())) + for name in expected: + self.assertEqual(getattr(ns, name), expected[name]) + + check(types.SimpleNamespace(), {}) + check(types.SimpleNamespace(x=1, y=2), {'x': 1, 'y': 2}) + check(types.SimpleNamespace(**dict(x=1, y=2)), {'x': 1, 'y': 2}) + check(types.SimpleNamespace({'x': 1, 'y': 2}, x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace([['x', 1], ['y', 2]], x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace(UserDict({'x': 1, 'y': 2}), x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace({'x': 1, 'y': 2}), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([['x', 1], ['y', 2]]), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([], x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace({}, x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace([]), {}) + check(types.SimpleNamespace({}), {}) with self.assertRaises(TypeError): - types.SimpleNamespace(1, 2, 3) + types.SimpleNamespace([], []) # too many positional arguments with self.assertRaises(TypeError): - types.SimpleNamespace(**{1: 2}) - - self.assertEqual(len(ns1.__dict__), 0) - self.assertEqual(vars(ns1), {}) - self.assertEqual(len(ns2.__dict__), 2) - self.assertEqual(vars(ns2), {'y': 2, 'x': 1}) - self.assertEqual(len(ns3.__dict__), 2) - self.assertEqual(vars(ns3), {'y': 2, 'x': 1}) + types.SimpleNamespace(1) # not a mapping or iterable + with self.assertRaises(TypeError): + types.SimpleNamespace([1]) # non-iterable + with self.assertRaises(ValueError): + types.SimpleNamespace([['x']]) # not a pair + with self.assertRaises(ValueError): + types.SimpleNamespace([['x', 'y', 'z']]) + with self.assertRaises(TypeError): + types.SimpleNamespace(**{1: 2}) # non-string key + with self.assertRaises(TypeError): + types.SimpleNamespace({1: 2}) + with self.assertRaises(TypeError): + types.SimpleNamespace([[1, 2]]) + with self.assertRaises(TypeError): + types.SimpleNamespace(UserDict({1: 2})) + with self.assertRaises(TypeError): + types.SimpleNamespace([[[], 2]]) # non-hashable key def test_unbound(self): ns1 = vars(types.SimpleNamespace()) @@ -2296,5 +2350,90 @@ def ex(a, /, b, *, c): ) +class SubinterpreterTests(unittest.TestCase): + + NUMERIC_METHODS = { + '__abs__', + '__add__', + '__bool__', + '__divmod__', + '__float__', + '__floordiv__', + '__index__', + '__int__', + '__lshift__', + '__mod__', + '__mul__', + '__neg__', + '__pos__', + '__pow__', + '__radd__', + '__rdivmod__', + '__rfloordiv__', + '__rlshift__', + '__rmod__', + '__rmul__', + '__rpow__', + '__rrshift__', + '__rshift__', + '__rsub__', + '__rtruediv__', + '__sub__', + '__truediv__', + } + + @classmethod + def setUpClass(cls): + global interpreters + try: + from test.support import interpreters + except ModuleNotFoundError: + raise unittest.SkipTest('subinterpreters required') + import test.support.interpreters.channels + + @cpython_only + @no_rerun('channels (and queues) might have a refleak; see gh-122199') + def test_static_types_inherited_slots(self): + rch, sch = interpreters.channels.create() + + script = textwrap.dedent(""" + import test.support + results = [] + for cls in test.support.iter_builtin_types(): + for attr, _ in test.support.iter_slot_wrappers(cls): + wrapper = getattr(cls, attr) + res = (cls, attr, wrapper) + results.append(res) + results = tuple((repr(c), a, repr(w)) for c, a, w in results) + sch.send_nowait(results) + """) + def collate_results(raw): + results = {} + for cls, attr, wrapper in raw: + key = cls, attr + assert key not in results, (results, key, wrapper) + results[key] = wrapper + return results + + exec(script) + raw = rch.recv_nowait() + main_results = collate_results(raw) + + interp = interpreters.create() + interp.exec('from test.support import interpreters') + interp.prepare_main(sch=sch) + interp.exec(script) + raw = rch.recv_nowait() + interp_results = collate_results(raw) + + for key, expected in main_results.items(): + cls, attr = key + with self.subTest(cls=cls, slotattr=attr): + actual = interp_results.pop(key) + self.assertEqual(actual, expected) + self.maxDiff = None + self.assertEqual(interp_results, {}) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c3a092f3af3009..2f1f9e86a0bce4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1,11 +1,13 @@ +import annotationlib import contextlib import collections import collections.abc from collections import defaultdict -from functools import lru_cache, wraps +from functools import lru_cache, wraps, reduce import gc import inspect import itertools +import operator import pickle import re import sys @@ -30,22 +32,22 @@ from typing import dataclass_transform from typing import no_type_check, no_type_check_decorator from typing import Type -from typing import NamedTuple, NotRequired, Required, TypedDict +from typing import NamedTuple, NotRequired, Required, ReadOnly, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef from typing import Self, LiteralString from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs -from typing import TypeGuard +from typing import TypeGuard, TypeIs, NoDefault import abc import textwrap import typing import weakref import types -from test.support import captured_stderr, cpython_only, infinite_recursion -from test.typinganndata import mod_generics_cache, _typed_dict_helper +from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper, run_code +from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper CANNOT_SUBCLASS_TYPE = 'Cannot subclass special typing classes' @@ -139,6 +141,26 @@ class MockSomething(Something, Mock): pass self.assertIsInstance(ms, Something) self.assertIsInstance(ms, Mock) + def test_subclassing_with_custom_constructor(self): + class Sub(Any): + def __init__(self, *args, **kwargs): pass + # The instantiation must not fail. + Sub(0, s="") + + def test_multiple_inheritance_with_custom_constructors(self): + class Foo: + def __init__(self, x): + self.x = x + + class Bar(Any, Foo): + def __init__(self, x, y): + self.y = y + super().__init__(x) + + b = Bar(1, 2) + self.assertEqual(b.x, 1) + self.assertEqual(b.y, 2) + def test_cannot_instantiate(self): with self.assertRaises(TypeError): Any() @@ -557,6 +579,240 @@ def test_constructor(self): self.assertEqual(T.__name__, "T") self.assertEqual(T.__constraints__, ()) self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", bound=type) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, type) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", default=()) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, ()) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", covariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, True) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", contravariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, True) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", infer_variance=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, True) + + +class TypeParameterDefaultsTests(BaseTestCase): + def test_typevar(self): + T = TypeVar('T', default=int) + self.assertEqual(T.__default__, int) + self.assertTrue(T.has_default()) + self.assertIsInstance(T, TypeVar) + + class A(Generic[T]): ... + Alias = Optional[T] + + def test_typevar_none(self): + U = TypeVar('U') + U_None = TypeVar('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[T]: ... + T, = X.__type_params__ + self.assertIs(T.__default__, NoDefault) + self.assertFalse(T.has_default()) + + def test_paramspec(self): + P = ParamSpec('P', default=(str, int)) + self.assertEqual(P.__default__, (str, int)) + self.assertTrue(P.has_default()) + self.assertIsInstance(P, ParamSpec) + + class A(Generic[P]): ... + Alias = typing.Callable[P, None] + + P_default = ParamSpec('P_default', default=...) + self.assertIs(P_default.__default__, ...) + + def test_paramspec_none(self): + U = ParamSpec('U') + U_None = ParamSpec('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**P]: ... + P, = X.__type_params__ + self.assertIs(P.__default__, NoDefault) + self.assertFalse(P.has_default()) + + def test_typevartuple(self): + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + self.assertTrue(Ts.has_default()) + self.assertIsInstance(Ts, TypeVarTuple) + + class A(Generic[Unpack[Ts]]): ... + Alias = Optional[Unpack[Ts]] + + def test_typevartuple_specialization(self): + T = TypeVar("T") + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, Unpack[Ts]]): ... + self.assertEqual(A[float].__args__, (float, str, int)) + self.assertEqual(A[float, range].__args__, (float, range)) + self.assertEqual(A[float, *tuple[int, ...]].__args__, (float, *tuple[int, ...])) + + def test_typevar_and_typevartuple_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, U, Unpack[Ts]]): ... + self.assertEqual(A[int].__args__, (int, float, str, int)) + self.assertEqual(A[int, str].__args__, (int, str, str, int)) + self.assertEqual(A[int, str, range].__args__, (int, str, range)) + self.assertEqual(A[int, str, *tuple[int, ...]].__args__, (int, str, *tuple[int, ...])) + + def test_no_default_after_typevar_tuple(self): + T = TypeVar("T", default=int) + Ts = TypeVarTuple("Ts") + Ts_default = TypeVarTuple("Ts_default", default=Unpack[Tuple[str, int]]) + + with self.assertRaises(TypeError): + class X(Generic[*Ts, T]): ... + + with self.assertRaises(TypeError): + class Y(Generic[*Ts_default, T]): ... + + def test_allow_default_after_non_default_in_alias(self): + T_default = TypeVar('T_default', default=int) + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + a1 = Callable[[T_default], T] + self.assertEqual(a1.__args__, (T_default, T)) + + a2 = dict[T_default, T] + self.assertEqual(a2.__args__, (T_default, T)) + + a3 = typing.Dict[T_default, T] + self.assertEqual(a3.__args__, (T_default, T)) + + a4 = Callable[*Ts, T] + self.assertEqual(a4.__args__, (*Ts, T)) + + def test_paramspec_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P]): ... + self.assertEqual(A[float].__args__, (float, (str, int))) + self.assertEqual(A[float, [range]].__args__, (float, (range,))) + + def test_typevar_and_paramspec_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, U, P]): ... + self.assertEqual(A[float].__args__, (float, float, (str, int))) + self.assertEqual(A[float, int].__args__, (float, int, (str, int))) + self.assertEqual(A[float, int, [range]].__args__, (float, int, (range,))) + + def test_paramspec_and_typevar_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, (str, int), float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + + def test_typevartuple_none(self): + U = TypeVarTuple('U') + U_None = TypeVarTuple('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**Ts]: ... + Ts, = X.__type_params__ + self.assertIs(Ts.__default__, NoDefault) + self.assertFalse(Ts.has_default()) + + def test_no_default_after_non_default(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + + with self.assertRaisesRegex( + TypeError, r"Type parameter ~T without a default follows type parameter with a default" + ): + Test = Generic[DefaultStrT, T] + + def test_need_more_params(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + U = TypeVar('U') + + class A(Generic[T, U, DefaultStrT]): ... + A[int, bool] + A[int, bool, str] + + with self.assertRaisesRegex( + TypeError, r"Too few arguments for .+; actual 1, expected at least 2" + ): + Test = A[int] + + def test_pickle(self): + global U, U_co, U_contra, U_default # pickle wants to reference the class by name + U = TypeVar('U') + U_co = TypeVar('U_co', covariant=True) + U_contra = TypeVar('U_contra', contravariant=True) + U_default = TypeVar('U_default', default=int) + for proto in range(pickle.HIGHEST_PROTOCOL): + for typevar in (U, U_co, U_contra, U_default): + z = pickle.loads(pickle.dumps(typevar, proto)) + self.assertEqual(z.__name__, typevar.__name__) + self.assertEqual(z.__covariant__, typevar.__covariant__) + self.assertEqual(z.__contravariant__, typevar.__contravariant__) + self.assertEqual(z.__bound__, typevar.__bound__) + self.assertEqual(z.__default__, typevar.__default__) def template_replace(templates: list[str], replacements: dict[str, list[str]]) -> list[tuple[str]]: @@ -957,6 +1213,38 @@ def foo(**kwargs: Unpack[Movie]): ... self.assertEqual(repr(foo.__annotations__['kwargs']), f"typing.Unpack[{__name__}.Movie]") + def test_builtin_tuple(self): + Ts = TypeVarTuple("Ts") + + class Old(Generic[*Ts]): ... + class New[*Ts]: ... + + PartOld = Old[int, *Ts] + self.assertEqual(PartOld[str].__args__, (int, str)) + self.assertEqual(PartOld[*tuple[str]].__args__, (int, str)) + self.assertEqual(PartOld[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[Tuple[str]]].__args__, (int, str)) + + PartNew = New[int, *Ts] + self.assertEqual(PartNew[str].__args__, (int, str)) + self.assertEqual(PartNew[*tuple[str]].__args__, (int, str)) + self.assertEqual(PartNew[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[Tuple[str]]].__args__, (int, str)) + + def test_unpack_wrong_type(self): + Ts = TypeVarTuple("Ts") + class Gen[*Ts]: ... + PartGen = Gen[int, *Ts] + + bad_unpack_param = re.escape("Unpack[...] must be used with a tuple type") + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[list[int]]] + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[List[int]]] + + class TypeVarTupleTests(BaseTestCase): def assertEndsWith(self, string, tail): @@ -1769,6 +2057,26 @@ def test_union_union(self): v = Union[u, Employee] self.assertEqual(v, Union[int, float, Employee]) + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual(Union[A, B].__args__, (A, B)) + union1 = Union[A, B] + with self.assertRaises(TypeError): + hash(union1) + + union2 = Union[int, B] + with self.assertRaises(TypeError): + hash(union2) + + union3 = Union[A, int] + with self.assertRaises(TypeError): + hash(union3) + def test_repr(self): self.assertEqual(repr(Union), 'typing.Union') u = Union[Employee, int] @@ -2192,7 +2500,7 @@ def test_concatenate(self): def test_nested_paramspec(self): # Since Callable has some special treatment, we want to be sure - # that substituion works correctly, see gh-103054 + # that substitution works correctly, see gh-103054 Callable = self.Callable P = ParamSpec('P') P2 = ParamSpec('P2') @@ -3465,7 +3773,7 @@ def meth(self): pass acceptable_extra_attrs = { '_is_protocol', '_is_runtime_protocol', '__parameters__', - '__init__', '__annotations__', '__subclasshook__', + '__init__', '__annotations__', '__subclasshook__', '__annotate__', } self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) self.assertLessEqual( @@ -4600,6 +4908,88 @@ def f(x: X): ... {'x': list[list[ForwardRef('X')]]} ) + def test_pep695_generic_class_with_future_annotations(self): + original_globals = dict(ann_module695.__dict__) + + hints_for_A = get_type_hints(ann_module695.A) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(hints_for_A["x"], A_type_params[0]) + self.assertEqual(hints_for_A["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(hints_for_A["z"].__args__[0], A_type_params[2]) + + # should not have changed as a result of the get_type_hints() calls! + self.assertEqual(ann_module695.__dict__, original_globals) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + hints_for_B = get_type_hints(ann_module695.B) + self.assertEqual(hints_for_B, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + hints_for_C = get_type_hints(ann_module695.C) + self.assertEqual( + set(hints_for_C.values()), + set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + hints_for_generic_function = get_type_hints(ann_module695.generic_function) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + hints_for_generic_function.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(hints_for_generic_function["x"], func_t_params[0]) + self.assertEqual(hints_for_generic_function["y"], Unpack[func_t_params[1]]) + self.assertIs(hints_for_generic_function["z"].__origin__, func_t_params[2]) + self.assertIs(hints_for_generic_function["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.generic_function_2).values()), + set(ann_module695.generic_function_2.__type_params__) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + hints_for_generic_method = get_type_hints(ann_module695.D.generic_method) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + hints_for_generic_method, + {"x": params["Foo"], "y": params["Bar"], "return": types.NoneType} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.D.generic_method_2).values()), + set(ann_module695.D.generic_method_2.__type_params__) + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = ann_module695.nested() + + self.assertEqual( + set(results.hints_for_E.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.hints_for_E_meth.values()), + set(results.E.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.hints_for_E_meth.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.hints_for_E_meth.values()).intersection(results.E.__type_params__), + set() + ) + + self.assertEqual( + set(results.hints_for_generic_func.values()), + set(results.generic_func.__type_params__) + ) + def test_extended_generic_rules_subclassing(self): class T1(Tuple[T, KT]): ... class T2(Tuple[T, ...]): ... @@ -4634,8 +5024,6 @@ def test_fail_with_bare_union(self): List[Union] with self.assertRaises(TypeError): Tuple[Optional] - with self.assertRaises(TypeError): - ClassVar[ClassVar[int]] with self.assertRaises(TypeError): List[ClassVar[int]] @@ -5168,6 +5556,7 @@ def test_subclass_special_form(self): Literal[1, 2], Concatenate[int, ParamSpec("P")], TypeGuard[int], + TypeIs[range], ): with self.subTest(msg=obj): with self.assertRaisesRegex( @@ -5506,10 +5895,8 @@ def some(self): self.assertFalse(hasattr(WithOverride.some, "__override__")) def test_multiple_decorators(self): - import functools - def with_wraps(f): # similar to `lru_cache` definition - @functools.wraps(f) + @wraps(f) def wrapper(*args, **kwargs): return f(*args, **kwargs) return wrapper @@ -5848,6 +6235,12 @@ def foo(a: 'Node[T'): with self.assertRaises(SyntaxError): get_type_hints(foo) + def test_syntax_error_empty_string(self): + for form in [typing.List, typing.Set, typing.Type, typing.Deque]: + with self.subTest(form=form): + with self.assertRaises(SyntaxError): + form[''] + def test_name_error(self): def foo(a: 'Noode[T]'): @@ -5989,16 +6382,6 @@ class F: for clazz in [C, D, E, F]: self.assertEqual(get_type_hints(clazz), expected_result) - def test_nested_classvar_fails_forward_ref_check(self): - class E: - foo: 'typing.ClassVar[typing.ClassVar[int]]' = 7 - class F: - foo: ClassVar['ClassVar[int]'] = 7 - - for clazz in [E, F]: - with self.assertRaises(TypeError): - get_type_hints(clazz) - def test_meta_no_type_check(self): depr_msg = ( "'typing.no_type_check_decorator' is deprecated " @@ -6051,6 +6434,42 @@ def test_or(self): self.assertEqual("x" | X, Union["x", X]) +class InternalsTests(BaseTestCase): + def test_deprecation_for_no_type_params_passed_to__evaluate(self): + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing._eval_type' is deprecated" + ) + ) as cm: + self.assertEqual(typing._eval_type(list["int"], globals(), {}), list[int]) + + self.assertEqual(cm.filename, __file__) + + f = ForwardRef("int") + + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing.ForwardRef._evaluate' is deprecated" + ) + ) as cm: + self.assertIs(f._evaluate(globals(), {}, recursive_guard=frozenset()), int) + + self.assertEqual(cm.filename, __file__) + + def test_collect_parameters(self): + typing = import_helper.import_fresh_module("typing") + with self.assertWarnsRegex( + DeprecationWarning, + "The private _collect_parameters function is deprecated" + ) as cm: + typing._collect_parameters + self.assertEqual(cm.filename, __file__) + + @lru_cache() def cached_func(x, y): return 3 * x + y @@ -6156,8 +6575,6 @@ def test_overload_registry_repeated(self): self.assertEqual(list(get_overloads(impl)), overloads) -# Definitions needed for features introduced in Python 3.6 - from test.typinganndata import ( ann_module, ann_module2, ann_module3, ann_module5, ann_module6, ) @@ -6325,7 +6742,7 @@ def test_get_type_hints_from_various_objects(self): gth(None) def test_get_type_hints_modules(self): - ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} + ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} self.assertEqual(gth(ann_module), ann_module_type_hints) self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) @@ -6343,7 +6760,7 @@ def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) self.assertIsInstance(gth(ann_module.j_class), dict) - self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type}) + self.assertEqual(gth(ann_module.M), {'o': type}) self.assertEqual(gth(ann_module.D), {'j': str, 'k': str, 'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.Y), {'z': int}) @@ -6626,6 +7043,25 @@ def h(x: collections.abc.Callable[P, int]): ... self.assertEqual(get_type_hints(g), {'x': collections.abc.Callable[..., int]}) self.assertEqual(get_type_hints(h), {'x': collections.abc.Callable[P, int]}) + def test_get_type_hints_format(self): + class C: + x: undefined + + with self.assertRaises(NameError): + get_type_hints(C) + + with self.assertRaises(NameError): + get_type_hints(C, format=annotationlib.Format.VALUE) + + annos = get_type_hints(C, format=annotationlib.Format.FORWARDREF) + self.assertIsInstance(annos, dict) + self.assertEqual(list(annos), ['x']) + self.assertIsInstance(annos['x'], annotationlib.ForwardRef) + self.assertEqual(annos['x'].__arg__, 'undefined') + + self.assertEqual(get_type_hints(C, format=annotationlib.Format.STRING), + {'x': 'undefined'}) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -6717,6 +7153,7 @@ class C(Generic[T]): pass self.assertEqual(get_args(NotRequired[int]), (int,)) self.assertEqual(get_args(TypeAlias), ()) self.assertEqual(get_args(TypeGuard[int]), (int,)) + self.assertEqual(get_args(TypeIs[range]), (range,)) Ts = TypeVarTuple('Ts') self.assertEqual(get_args(Ts), ()) self.assertEqual(get_args((*Ts,)[0]), (Ts,)) @@ -6750,24 +7187,16 @@ def test_iterator(self): self.assertNotIsInstance(42, typing.Iterator) def test_awaitable(self): - ns = {} - exec( - "async def foo() -> typing.Awaitable[int]:\n" - " return await AwaitableWrapper(42)\n", - globals(), ns) - foo = ns['foo'] + async def foo() -> typing.Awaitable[int]: + return await AwaitableWrapper(42) g = foo() self.assertIsInstance(g, typing.Awaitable) self.assertNotIsInstance(foo, typing.Awaitable) g.send(None) # Run foo() till completion, to avoid warning. def test_coroutine(self): - ns = {} - exec( - "async def foo():\n" - " return\n", - globals(), ns) - foo = ns['foo'] + async def foo(): + return g = foo() self.assertIsInstance(g, typing.Coroutine) with self.assertRaises(TypeError): @@ -6829,16 +7258,6 @@ def test_mutablesequence(self): self.assertIsInstance([], typing.MutableSequence) self.assertNotIsInstance((), typing.MutableSequence) - def test_bytestring(self): - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(b'', typing.ByteString) - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(bytearray(b''), typing.ByteString) - with self.assertWarns(DeprecationWarning): - class Foo(typing.ByteString): ... - with self.assertWarns(DeprecationWarning): - class Bar(typing.ByteString, typing.Awaitable): ... - def test_list(self): self.assertIsSubclass(list, typing.List) @@ -7032,6 +7451,17 @@ def foo(): g = foo() self.assertIsSubclass(type(g), typing.Generator) + def test_generator_default(self): + g1 = typing.Generator[int] + g2 = typing.Generator[int, None, None] + self.assertEqual(get_args(g1), (int, type(None), type(None))) + self.assertEqual(get_args(g1), get_args(g2)) + + g3 = typing.Generator[int, float] + g4 = typing.Generator[int, float, None] + self.assertEqual(get_args(g3), (int, float, type(None))) + self.assertEqual(get_args(g3), get_args(g4)) + def test_no_generator_instantiation(self): with self.assertRaises(TypeError): typing.Generator() @@ -7041,10 +7471,9 @@ def test_no_generator_instantiation(self): typing.Generator[int, int, int]() def test_async_generator(self): - ns = {} - exec("async def f():\n" - " yield 42\n", globals(), ns) - g = ns['f']() + async def f(): + yield 42 + g = f() self.assertIsSubclass(type(g), typing.AsyncGenerator) def test_no_async_generator_instantiation(self): @@ -7131,9 +7560,8 @@ def asend(self, value): def athrow(self, typ, val=None, tb=None): pass - ns = {} - exec('async def g(): yield 0', globals(), ns) - g = ns['g'] + async def g(): yield 0 + self.assertIsSubclass(G, typing.AsyncGenerator) self.assertIsSubclass(G, typing.AsyncIterable) self.assertIsSubclass(G, collections.abc.AsyncGenerator) @@ -7218,6 +7646,15 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) + def test_contextmanager_type_params(self): + cm1 = typing.ContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.ContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) + + type gen_cm[T1, T2] = typing.ContextManager[T1, T2] + self.assertEqual(get_args(gen_cm.__value__[int, None]), (int, types.NoneType)) + def test_async_contextmanager(self): class NotACM: pass @@ -7229,11 +7666,17 @@ def manager(): cm = manager() self.assertNotIsInstance(cm, typing.AsyncContextManager) - self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int, bool | None)) with self.assertRaises(TypeError): isinstance(42, typing.AsyncContextManager[int]) with self.assertRaises(TypeError): - typing.AsyncContextManager[int, str] + typing.AsyncContextManager[int, str, float] + + def test_asynccontextmanager_type_params(self): + cm1 = typing.AsyncContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.AsyncContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) class TypeTests(BaseTestCase): @@ -7438,6 +7881,48 @@ class XMethBad2(NamedTuple): def _source(self): return 'no chance for this as well' + def test_annotation_type_check(self): + # These are rejected by _type_check + with self.assertRaises(TypeError): + class X(NamedTuple): + a: Final + with self.assertRaises(TypeError): + class Y(NamedTuple): + a: (1, 2) + + # Conversion by _type_convert + class Z(NamedTuple): + a: None + b: "str" + annos = {'a': type(None), 'b': ForwardRef("str")} + self.assertEqual(Z.__annotations__, annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.STRING), {"a": "None", "b": "str"}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + from typing import NamedTuple + class X(NamedTuple): + a: int + b: None + """ + ns = run_code(textwrap.dedent(code)) + X = ns['X'] + self.assertEqual(X.__annotations__, {'a': ForwardRef("int"), 'b': ForwardRef("None")}) + + def test_deferred_annotations(self): + class X(NamedTuple): + y: undefined + + self.assertEqual(X._fields, ('y',)) + with self.assertRaises(NameError): + X.__annotations__ + + undefined = int + self.assertEqual(X.__annotations__, {'y': int}) + def test_multiple_inheritance(self): class A: pass @@ -7752,7 +8237,11 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__name__, 'Emp') self.assertEqual(Emp.__module__, __name__) self.assertEqual(Emp.__bases__, (dict,)) - self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) + annos = {'name': str, 'id': int} + self.assertEqual(Emp.__annotations__, annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.STRING), {'name': 'str', 'id': 'int'}) self.assertEqual(Emp.__total__, True) self.assertEqual(Emp.__required_keys__, {'name', 'id'}) self.assertIsInstance(Emp.__required_keys__, frozenset) @@ -8113,6 +8602,8 @@ class A[T](TypedDict): self.assertEqual(A.__bases__, (Generic, dict)) self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8124,6 +8615,8 @@ class A(TypedDict, Generic[T]): self.assertEqual(A.__bases__, (Generic, dict)) self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8134,6 +8627,8 @@ class A2(Generic[T], TypedDict): self.assertEqual(A2.__bases__, (Generic, dict)) self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) + self.assertEqual(A2.__annotations__, {'a': T}) + self.assertEqual(A2.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A2.__parameters__, (T,)) self.assertEqual(A2[str].__parameters__, ()) self.assertEqual(A2[str].__args__, (str,)) @@ -8144,6 +8639,8 @@ class B(A[KT], total=False): self.assertEqual(B.__bases__, (Generic, dict)) self.assertEqual(B.__orig_bases__, (A[KT],)) self.assertEqual(B.__mro__, (B, Generic, dict, object)) + self.assertEqual(B.__annotations__, {'a': T, 'b': KT}) + self.assertEqual(B.__annotate__(annotationlib.Format.STRING), {'a': 'T', 'b': 'KT'}) self.assertEqual(B.__parameters__, (KT,)) self.assertEqual(B.__total__, False) self.assertEqual(B.__optional_keys__, frozenset(['b'])) @@ -8168,6 +8665,11 @@ class C(B[int]): 'b': KT, 'c': int, }) + self.assertEqual(C.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) with self.assertRaises(TypeError): C[str] @@ -8187,6 +8689,11 @@ class Point3D(Point2DGeneric[T], Generic[T, KT]): 'b': T, 'c': KT, }) + self.assertEqual(Point3D.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'T', + 'c': 'KT', + }) self.assertEqual(Point3D[int, str].__origin__, Point3D) with self.assertRaises(TypeError): @@ -8218,6 +8725,11 @@ class WithImplicitAny(B): 'b': KT, 'c': int, }) + self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) with self.assertRaises(TypeError): WithImplicitAny[str] @@ -8311,6 +8823,117 @@ class T4(TypedDict, Generic[S]): pass self.assertEqual(klass.__optional_keys__, set()) self.assertIsInstance(klass(), dict) + def test_readonly_inheritance(self): + class Base1(TypedDict): + a: ReadOnly[int] + + class Child1(Base1): + b: str + + self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) + self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + + class Base2(TypedDict): + a: ReadOnly[int] + + class Child2(Base2): + b: str + + self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) + self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + + def test_cannot_make_mutable_key_readonly(self): + class Base(TypedDict): + a: int + + with self.assertRaises(TypeError): + class Child(Base): + a: ReadOnly[int] + + def test_can_make_readonly_key_mutable(self): + class Base(TypedDict): + a: ReadOnly[int] + + class Child(Base): + a: int + + self.assertEqual(Child.__readonly_keys__, frozenset()) + self.assertEqual(Child.__mutable_keys__, frozenset({'a'})) + + def test_combine_qualifiers(self): + class AllTheThings(TypedDict): + a: Annotated[Required[ReadOnly[int]], "why not"] + b: Required[Annotated[ReadOnly[int], "why not"]] + c: ReadOnly[NotRequired[Annotated[int, "why not"]]] + d: NotRequired[Annotated[int, "why not"]] + + self.assertEqual(AllTheThings.__required_keys__, frozenset({'a', 'b'})) + self.assertEqual(AllTheThings.__optional_keys__, frozenset({'c', 'd'})) + self.assertEqual(AllTheThings.__readonly_keys__, frozenset({'a', 'b', 'c'})) + self.assertEqual(AllTheThings.__mutable_keys__, frozenset({'d'})) + + self.assertEqual( + get_type_hints(AllTheThings, include_extras=False), + {'a': int, 'b': int, 'c': int, 'd': int}, + ) + self.assertEqual( + get_type_hints(AllTheThings, include_extras=True), + { + 'a': Annotated[Required[ReadOnly[int]], 'why not'], + 'b': Required[Annotated[ReadOnly[int], 'why not']], + 'c': ReadOnly[NotRequired[Annotated[int, 'why not']]], + 'd': NotRequired[Annotated[int, 'why not']], + }, + ) + + def test_annotations(self): + # _type_check is applied + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + class X(TypedDict): + a: Final + + # _type_convert is applied + class Y(TypedDict): + a: None + b: "int" + fwdref = ForwardRef('int', module=__name__) + self.assertEqual(Y.__annotations__, {'a': type(None), 'b': fwdref}) + self.assertEqual(Y.__annotate__(annotationlib.Format.FORWARDREF), {'a': type(None), 'b': fwdref}) + + # _type_check is also applied later + class Z(TypedDict): + a: undefined + + with self.assertRaises(NameError): + Z.__annotations__ + + undefined = Final + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + Z.__annotations__ + + undefined = None + self.assertEqual(Z.__annotations__, {'a': type(None)}) + + def test_deferred_evaluation(self): + class A(TypedDict): + x: NotRequired[undefined] + y: ReadOnly[undefined] + z: Required[undefined] + + self.assertEqual(A.__required_keys__, frozenset({'y', 'z'})) + self.assertEqual(A.__optional_keys__, frozenset({'x'})) + self.assertEqual(A.__readonly_keys__, frozenset({'y'})) + self.assertEqual(A.__mutable_keys__, frozenset({'x', 'z'})) + + with self.assertRaises(NameError): + A.__annotations__ + + self.assertEqual( + A.__annotate__(annotationlib.Format.STRING), + {'x': 'NotRequired[undefined]', 'y': 'ReadOnly[undefined]', + 'z': 'Required[undefined]'}, + ) + class RequiredTests(BaseTestCase): @@ -8526,6 +9149,76 @@ def test_flatten(self): self.assertEqual(A.__metadata__, (4, 5)) self.assertEqual(A.__origin__, int) + def test_deduplicate_from_union(self): + # Regular: + self.assertEqual(get_args(Annotated[int, 1] | int), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], int]), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[int, 2] | int), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[int, 2], int]), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[str, 1] | int), + (Annotated[int, 1], Annotated[str, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[str, 1], int]), + (Annotated[int, 1], Annotated[str, 1], int)) + + # Duplicates: + self.assertEqual(Annotated[int, 1] | Annotated[int, 1] | int, + Annotated[int, 1] | int) + self.assertEqual(Union[Annotated[int, 1], Annotated[int, 1], int], + Union[Annotated[int, 1], int]) + + # Unhashable metadata: + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[int, set()] | int), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[int, set()], int]), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[str, {}] | int), + (str, Annotated[int, {}], Annotated[str, {}], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[str, {}], int]), + (str, Annotated[int, {}], Annotated[str, {}], int)) + + self.assertEqual(get_args(Annotated[int, 1] | str | Annotated[str, {}] | int), + (Annotated[int, 1], str, Annotated[str, {}], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], str, Annotated[str, {}], int]), + (Annotated[int, 1], str, Annotated[str, {}], int)) + + import dataclasses + @dataclasses.dataclass + class ValueRange: + lo: int + hi: int + v = ValueRange(1, 2) + self.assertEqual(get_args(Annotated[int, v] | None), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Union[Annotated[int, v], None]), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Optional[Annotated[int, v]]), + (Annotated[int, v], types.NoneType)) + + # Unhashable metadata duplicated: + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + Annotated[int, {}] | int) + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + int | Annotated[int, {}]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[Annotated[int, {}], int]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[int, Annotated[int, {}]]) + + def test_order_in_union(self): + expr1 = Annotated[int, 1] | str | Annotated[str, {}] | int + for args in itertools.permutations(get_args(expr1)): + with self.subTest(args=args): + self.assertEqual(expr1, reduce(operator.or_, args)) + + expr2 = Union[Annotated[int, 1], str, Annotated[str, {}], int] + for args in itertools.permutations(get_args(expr2)): + with self.subTest(args=args): + self.assertEqual(expr2, Union[args]) + def test_specialize(self): L = Annotated[List[T], "my decoration"] LI = Annotated[List[int], "my decoration"] @@ -8546,6 +9239,16 @@ def test_hash_eq(self): {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, {Annotated[int, 4, 5], Annotated[T, 4, 5]} ) + # Unhashable `metadata` raises `TypeError`: + a1 = Annotated[int, []] + with self.assertRaises(TypeError): + hash(a1) + + class A: + __hash__ = None + a2 = Annotated[int, A()] + with self.assertRaises(TypeError): + hash(a2) def test_instantiate(self): class C: @@ -8613,6 +9316,34 @@ class C: self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) + def test_special_forms_nesting(self): + # These are uncommon types and are to ensure runtime + # is lax on validation. See gh-89547 for more context. + class CF: + x: ClassVar[Final[int]] + + class FC: + x: Final[ClassVar[int]] + + class ACF: + x: Annotated[ClassVar[Final[int]], "a decoration"] + + class CAF: + x: ClassVar[Annotated[Final[int], "a decoration"]] + + class AFC: + x: Annotated[Final[ClassVar[int]], "a decoration"] + + class FAC: + x: Final[Annotated[ClassVar[int], "a decoration"]] + + self.assertEqual(get_type_hints(CF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(FC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(ACF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(CAF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(AFC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(FAC, globals())['x'], Final[ClassVar[int]]) + def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, "Cannot subclass .*Annotated"): class C(Annotated): @@ -9390,6 +10121,56 @@ def test_no_isinstance(self): issubclass(int, TypeGuard) +class TypeIsTests(BaseTestCase): + def test_basics(self): + TypeIs[int] # OK + + def foo(arg) -> TypeIs[int]: ... + self.assertEqual(gth(foo), {'return': TypeIs[int]}) + + with self.assertRaises(TypeError): + TypeIs[int, str] + + def test_repr(self): + self.assertEqual(repr(TypeIs), 'typing.TypeIs') + cv = TypeIs[int] + self.assertEqual(repr(cv), 'typing.TypeIs[int]') + cv = TypeIs[Employee] + self.assertEqual(repr(cv), 'typing.TypeIs[%s.Employee]' % __name__) + cv = TypeIs[tuple[int]] + self.assertEqual(repr(cv), 'typing.TypeIs[tuple[int]]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(TypeIs)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(TypeIs[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs'): + class E(TypeIs): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs\[int\]'): + class F(TypeIs[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + TypeIs() + with self.assertRaises(TypeError): + type(TypeIs)() + with self.assertRaises(TypeError): + type(TypeIs[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, TypeIs[int]) + with self.assertRaises(TypeError): + issubclass(int, TypeIs) + + SpecialAttrsP = typing.ParamSpec('SpecialAttrsP') SpecialAttrsT = typing.TypeVar('SpecialAttrsT', int, float, complex) @@ -9405,7 +10186,6 @@ def test_special_attrs(self): typing.AsyncIterable: 'AsyncIterable', typing.AsyncIterator: 'AsyncIterator', typing.Awaitable: 'Awaitable', - typing.ByteString: 'ByteString', typing.Callable: 'Callable', typing.ChainMap: 'ChainMap', typing.Collection: 'Collection', @@ -9439,7 +10219,7 @@ def test_special_attrs(self): typing.ValuesView: 'ValuesView', # Subscribed ABC classes typing.AbstractSet[Any]: 'AbstractSet', - typing.AsyncContextManager[Any]: 'AsyncContextManager', + typing.AsyncContextManager[Any, Any]: 'AsyncContextManager', typing.AsyncGenerator[Any, Any]: 'AsyncGenerator', typing.AsyncIterable[Any]: 'AsyncIterable', typing.AsyncIterator[Any]: 'AsyncIterator', @@ -9449,7 +10229,7 @@ def test_special_attrs(self): typing.ChainMap[Any, Any]: 'ChainMap', typing.Collection[Any]: 'Collection', typing.Container[Any]: 'Container', - typing.ContextManager[Any]: 'ContextManager', + typing.ContextManager[Any, Any]: 'ContextManager', typing.Coroutine[Any, Any, Any]: 'Coroutine', typing.Counter[Any]: 'Counter', typing.DefaultDict[Any, Any]: 'DefaultDict', @@ -9481,7 +10261,6 @@ def test_special_attrs(self): typing.ClassVar: 'ClassVar', typing.Concatenate: 'Concatenate', typing.Final: 'Final', - typing.ForwardRef: 'ForwardRef', typing.Literal: 'Literal', typing.NewType: 'NewType', typing.NoReturn: 'NoReturn', @@ -9489,10 +10268,11 @@ def test_special_attrs(self): typing.Optional: 'Optional', typing.TypeAlias: 'TypeAlias', typing.TypeGuard: 'TypeGuard', + typing.TypeIs: 'TypeIs', typing.TypeVar: 'TypeVar', typing.Union: 'Union', typing.Self: 'Self', - # Subscribed special forms + # Subscripted special forms typing.Annotated[Any, "Annotation"]: 'Annotated', typing.Annotated[int, 'Annotation']: 'Annotated', typing.ClassVar[Any]: 'ClassVar', @@ -9503,10 +10283,10 @@ def test_special_attrs(self): typing.Literal[True, 2]: 'Literal', typing.Optional[Any]: 'Optional', typing.TypeGuard[Any]: 'TypeGuard', + typing.TypeIs[Any]: 'TypeIs', typing.Union[Any]: 'Any', typing.Union[int, float]: 'Union', # Incompatible special forms (tested in test_special_attrs2) - # - typing.ForwardRef('set[Any]') # - typing.NewType('TypeName', Any) # - typing.ParamSpec('SpecialAttrsP') # - typing.TypeVar('T') @@ -9525,18 +10305,6 @@ def test_special_attrs(self): TypeName = typing.NewType('SpecialAttrsTests.TypeName', Any) def test_special_attrs2(self): - # Forward refs provide a different introspection API. __name__ and - # __qualname__ make little sense for forward refs as they can store - # complex typing expressions. - fr = typing.ForwardRef('set[Any]') - self.assertFalse(hasattr(fr, '__name__')) - self.assertFalse(hasattr(fr, '__qualname__')) - self.assertEqual(fr.__module__, 'typing') - # Forward refs are currently unpicklable. - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError): - pickle.dumps(fr, proto) - self.assertEqual(SpecialAttrsTests.TypeName.__name__, 'TypeName') self.assertEqual( SpecialAttrsTests.TypeName.__qualname__, @@ -9689,6 +10457,45 @@ class CustomerModel(ModelBase, init=False): self.assertIsInstance(CustomerModel, Decorated) +class NoDefaultTests(BaseTestCase): + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(NoDefault, proto) + loaded = pickle.loads(s) + self.assertIs(NoDefault, loaded) + + def test_constructor(self): + self.assertIs(NoDefault, type(NoDefault)()) + with self.assertRaises(TypeError): + type(NoDefault)(1) + + def test_repr(self): + self.assertEqual(repr(NoDefault), 'typing.NoDefault') + + @requires_docstrings + def test_doc(self): + self.assertIsInstance(NoDefault.__doc__, str) + + def test_class(self): + self.assertIs(NoDefault.__class__, type(NoDefault)) + + def test_no_call(self): + with self.assertRaises(TypeError): + NoDefault() + + def test_no_attributes(self): + with self.assertRaises(AttributeError): + NoDefault.foo = 3 + with self.assertRaises(AttributeError): + NoDefault.foo + + # TypeError is consistent with the behavior of NoneType + with self.assertRaises(TypeError): + type(NoDefault).foo = 3 + with self.assertRaises(AttributeError): + type(NoDefault).foo + + class AllTests(BaseTestCase): """Tests for __all__.""" diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index 63c6c055824b20..3680072d643792 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -19,7 +19,7 @@ def test_non_bmp_normalized(self): def test_invalid(self): try: - from test.tokenizedata import badsyntax_3131 + from test.tokenizedata import badsyntax_3131 # noqa: F401 except SyntaxError as err: self.assertEqual(str(err), "invalid character '€' (U+20AC) (badsyntax_3131.py, line 2)") diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index d3bf4ea7c7d437..c7d09a6b460c19 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -18,7 +18,7 @@ class UnicodeMethodsTest(unittest.TestCase): # update this, if the database changes - expectedchecksum = '63aa77dcb36b0e1df082ee2a6071caeda7f0955e' + expectedchecksum = '9e43ee3929471739680c0e705482b4ae1c4122e4' @requires_resource('cpu') def test_method_checksum(self): @@ -71,7 +71,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): # Update this if the database changes. Make sure to do a full rebuild # (e.g. 'make distclean && make') to get the correct checksum. - expectedchecksum = '232affd2a50ec4bd69d2482aa0291385cbdefaba' + expectedchecksum = '23ab09ed4abdf93db23b97359108ed630dd8311d' @requires_resource('cpu') def test_function_checksum(self): @@ -114,7 +114,7 @@ def test_no_names_in_pua(self): def test_lookup_nonexistant(self): # just make sure that lookup can fail - for nonexistant in [ + for nonexistent in [ "LATIN SMLL LETR A", "OPEN HANDS SIGHS", "DREGS", @@ -122,7 +122,7 @@ def test_lookup_nonexistant(self): "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", "???", ]: - self.assertRaises(KeyError, self.db.lookup, nonexistant) + self.assertRaises(KeyError, self.db.lookup, nonexistent) def test_digit(self): self.assertEqual(self.db.digit('A', None), None) diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/__init__.py b/Lib/test/test_unittest/namespace_test_pkg/bar/__init__.py similarity index 100% rename from Lib/test/test_importlib/resources/data01/subdirectory/__init__.py rename to Lib/test/test_unittest/namespace_test_pkg/bar/__init__.py diff --git a/Lib/test/test_unittest/namespace_test_pkg/bar/test_bar.py b/Lib/test/test_unittest/namespace_test_pkg/bar/test_bar.py new file mode 100644 index 00000000000000..05b184d9eba685 --- /dev/null +++ b/Lib/test/test_unittest/namespace_test_pkg/bar/test_bar.py @@ -0,0 +1,5 @@ +import unittest + +class PassingTest(unittest.TestCase): + def test_true(self): + self.assertTrue(True) diff --git a/Lib/test/test_importlib/resources/data02/__init__.py b/Lib/test/test_unittest/namespace_test_pkg/noop/no2/__init__.py similarity index 100% rename from Lib/test/test_importlib/resources/data02/__init__.py rename to Lib/test/test_unittest/namespace_test_pkg/noop/no2/__init__.py diff --git a/Lib/test/test_unittest/namespace_test_pkg/noop/no2/test_no2.py b/Lib/test/test_unittest/namespace_test_pkg/noop/no2/test_no2.py new file mode 100644 index 00000000000000..05b184d9eba685 --- /dev/null +++ b/Lib/test/test_unittest/namespace_test_pkg/noop/no2/test_no2.py @@ -0,0 +1,5 @@ +import unittest + +class PassingTest(unittest.TestCase): + def test_true(self): + self.assertTrue(True) diff --git a/Lib/test/test_unittest/namespace_test_pkg/noop/test_noop.py b/Lib/test/test_unittest/namespace_test_pkg/noop/test_noop.py new file mode 100644 index 00000000000000..05b184d9eba685 --- /dev/null +++ b/Lib/test/test_unittest/namespace_test_pkg/noop/test_noop.py @@ -0,0 +1,5 @@ +import unittest + +class PassingTest(unittest.TestCase): + def test_true(self): + self.assertTrue(True) diff --git a/Lib/test/test_unittest/namespace_test_pkg/test_foo.py b/Lib/test/test_unittest/namespace_test_pkg/test_foo.py new file mode 100644 index 00000000000000..05b184d9eba685 --- /dev/null +++ b/Lib/test/test_unittest/namespace_test_pkg/test_foo.py @@ -0,0 +1,5 @@ +import unittest + +class PassingTest(unittest.TestCase): + def test_true(self): + self.assertTrue(True) diff --git a/Lib/test/test_unittest/test_assertions.py b/Lib/test/test_unittest/test_assertions.py index 5c1a28ecda5b49..1dec947ea76d23 100644 --- a/Lib/test/test_unittest/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -386,6 +386,16 @@ def testAssertWarns(self): '^UserWarning not triggered$', '^UserWarning not triggered : oops$']) + def test_assertNotWarns(self): + def warn_future(): + warnings.warn('xyz', FutureWarning, stacklevel=2) + self.assertMessagesCM('_assertNotWarns', (FutureWarning,), + warn_future, + ['^FutureWarning triggered$', + '^oops$', + '^FutureWarning triggered$', + '^FutureWarning triggered : oops$']) + def testAssertWarnsRegex(self): # test error not raised self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'unused regex'), diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index ba1ab838cd4a22..00ef55bdf9bc83 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -312,18 +312,21 @@ async def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'async_generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) def test_cleanups_interleave_order(self): events = [] diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index ed5eb5609a5dd1..b4b2194a09cf9f 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -325,18 +325,37 @@ def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) + + def test_deprecation_of_return_val_from_test_async_method(self): + class Foo(unittest.TestCase): + async def test1(self): + return 1 + + with self.assertWarns(DeprecationWarning) as w: + Foo('test1').run() + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) + self.assertIn('test1', str(w.warning)) + self.assertEqual(w.filename, __file__) + self.assertIn("returned 'coroutine'", str(w.warning)) + self.assertIn( + 'Maybe you forgot to use IsolatedAsyncioTestCase as the base class?', + str(w.warning), + ) def _check_call_order__subtests(self, result, events, expected_events): class Foo(Test.LoggingTestCase): @@ -1132,6 +1151,8 @@ def testAssertMultiLineEqual(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualSingleLine(self): sample_text = "laden swallows fly slowly" @@ -1148,6 +1169,8 @@ def testAssertEqualSingleLine(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualwithEmptyString(self): '''Verify when there is an empty string involved, the diff output @@ -1165,6 +1188,8 @@ def testAssertEqualwithEmptyString(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMissingNewlineTerminator(self): '''Verifying format of diff output from assertEqual involving strings @@ -1185,6 +1210,8 @@ def testAssertEqualMultipleLinesMissingNewlineTerminator(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): '''Verifying format of diff output from assertEqual involving strings @@ -1208,6 +1235,8 @@ def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testEqualityBytesWarning(self): if sys.flags.bytes_warning: diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index 004898ed431834..38c9779daaf87d 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -4,12 +4,14 @@ import sys import types import pickle +from importlib._bootstrap_external import NamespaceLoader from test import support from test.support import import_helper import unittest import unittest.mock import test.test_unittest +from test.test_importlib import util as test_util class TestableTestProgram(unittest.TestProgram): @@ -395,7 +397,7 @@ def restore_isdir(): self.addCleanup(restore_isdir) _find_tests_args = [] - def _find_tests(start_dir, pattern): + def _find_tests(start_dir, pattern, namespace=None): _find_tests_args.append((start_dir, pattern)) return ['tests'] loader._find_tests = _find_tests @@ -406,10 +408,34 @@ def _find_tests(start_dir, pattern): top_level_dir = os.path.abspath('/foo/bar') start_dir = os.path.abspath('/foo/bar/baz') self.assertEqual(suite, "['tests']") - self.assertEqual(loader._top_level_dir, top_level_dir) + self.assertEqual(loader._top_level_dir, os.path.abspath('/foo')) self.assertEqual(_find_tests_args, [(start_dir, 'pattern')]) self.assertIn(top_level_dir, sys.path) + def test_discover_should_not_persist_top_level_dir_between_calls(self): + original_isfile = os.path.isfile + original_isdir = os.path.isdir + original_sys_path = sys.path[:] + def restore(): + os.path.isfile = original_isfile + os.path.isdir = original_isdir + sys.path[:] = original_sys_path + self.addCleanup(restore) + + os.path.isfile = lambda path: True + os.path.isdir = lambda path: True + loader = unittest.TestLoader() + loader.suiteClass = str + dir = '/foo/bar' + top_level_dir = '/foo' + + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, None) + + loader._top_level_dir = dir2 = '/previous/dir' + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, dir2) + def test_discover_start_dir_is_package_calls_package_load_tests(self): # This test verifies that the package load_tests in a package is indeed # invoked when the start_dir is a package (and not the top level). @@ -791,7 +817,7 @@ def test_discovery_from_dotted_path(self): expectedPath = os.path.abspath(os.path.dirname(test.test_unittest.__file__)) self.wasRun = False - def _find_tests(start_dir, pattern): + def _find_tests(start_dir, pattern, namespace=None): self.wasRun = True self.assertEqual(start_dir, expectedPath) return tests @@ -824,6 +850,54 @@ def restore(): 'Can not use builtin modules ' 'as dotted module names') + def test_discovery_from_dotted_namespace_packages(self): + loader = unittest.TestLoader() + + package = types.ModuleType('package') + package.__name__ = "tests" + package.__path__ = ['/a', '/b'] + package.__file__ = None + package.__spec__ = types.SimpleNamespace( + name=package.__name__, + loader=NamespaceLoader(package.__name__, package.__path__, None), + submodule_search_locations=['/a', '/b'] + ) + + def _import(packagename, *args, **kwargs): + sys.modules[packagename] = package + return package + + _find_tests_args = [] + def _find_tests(start_dir, pattern, namespace=None): + _find_tests_args.append((start_dir, pattern)) + return ['%s/tests' % start_dir] + + loader._find_tests = _find_tests + loader.suiteClass = list + + with unittest.mock.patch('builtins.__import__', _import): + # Since loader.discover() can modify sys.path, restore it when done. + with import_helper.DirsOnSysPath(): + # Make sure to remove 'package' from sys.modules when done. + with test_util.uncache('package'): + suite = loader.discover('package') + + self.assertEqual(suite, ['/a/tests', '/b/tests']) + + def test_discovery_start_dir_is_namespace(self): + """Subdirectory discovery not affected if start_dir is a namespace pkg.""" + loader = unittest.TestLoader() + with ( + import_helper.DirsOnSysPath(os.path.join(os.path.dirname(__file__))), + test_util.uncache('namespace_test_pkg') + ): + suite = loader.discover('namespace_test_pkg') + self.assertEqual( + {list(suite)[0]._tests[0].__module__ for suite in suite._tests if list(suite)}, + # files under namespace_test_pkg.noop not discovered. + {'namespace_test_pkg.test_foo', 'namespace_test_pkg.bar.test_bar'}, + ) + def test_discovery_failed_discovery(self): from test.test_importlib import util diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 15e3f62ef66a4b..4e5ec54e9c892a 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -186,7 +186,7 @@ def test_1(self): test = Foo('test_1') try: test.fail("foo") - except: + except AssertionError: exc_info_tuple = sys.exc_info() result = unittest.TestResult() @@ -214,7 +214,7 @@ def test_1(self): def get_exc_info(): try: test.fail("foo") - except: + except AssertionError: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -241,9 +241,9 @@ def get_exc_info(): try: try: test.fail("foo") - except: + except AssertionError: raise ValueError(42) - except: + except ValueError: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -271,7 +271,7 @@ def get_exc_info(): loop.__cause__ = loop loop.__context__ = loop raise loop - except: + except Exception: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -300,7 +300,7 @@ def get_exc_info(): ex1.__cause__ = ex2 ex2.__context__ = ex1 raise C - except: + except Exception: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -345,7 +345,7 @@ def test_1(self): test = Foo('test_1') try: raise TypeError() - except: + except TypeError: exc_info_tuple = sys.exc_info() result = unittest.TestResult() diff --git a/Lib/test/test_unittest/test_util.py b/Lib/test/test_unittest/test_util.py new file mode 100644 index 00000000000000..d590a333930278 --- /dev/null +++ b/Lib/test/test_unittest/test_util.py @@ -0,0 +1,33 @@ +import unittest +from unittest.util import safe_repr, sorted_list_difference, unorderable_list_difference + + +class TestUtil(unittest.TestCase): + def test_safe_repr(self): + class RaisingRepr: + def __repr__(self): + raise ValueError("Invalid repr()") + + class LongRepr: + def __repr__(self): + return 'x' * 100 + + safe_repr(RaisingRepr()) + self.assertEqual(safe_repr('foo'), "'foo'") + self.assertEqual(safe_repr(LongRepr(), short=True), 'x'*80 + ' [truncated]...') + + def test_sorted_list_difference(self): + self.assertEqual(sorted_list_difference([], []), ([], [])) + self.assertEqual(sorted_list_difference([1, 2], [2, 3]), ([1], [3])) + self.assertEqual(sorted_list_difference([1, 2], [1, 3]), ([2], [3])) + self.assertEqual(sorted_list_difference([1, 1, 1], [1, 2, 3]), ([], [2, 3])) + self.assertEqual(sorted_list_difference([4], [1, 2, 3, 4]), ([], [1, 2, 3])) + self.assertEqual(sorted_list_difference([1, 1], [2]), ([1], [2])) + self.assertEqual(sorted_list_difference([2], [1, 1]), ([2], [1])) + self.assertEqual(sorted_list_difference([1, 2], [1, 1]), ([2], [])) + + def test_unorderable_list_difference(self): + self.assertEqual(unorderable_list_difference([], []), ([], [])) + self.assertEqual(unorderable_list_difference([1, 2], []), ([2, 1], [])) + self.assertEqual(unorderable_list_difference([], [1, 2]), ([], [1, 2])) + self.assertEqual(unorderable_list_difference([1, 2], [1, 3]), ([2], [3])) diff --git a/Lib/test/test_unittest/testmock/support.py b/Lib/test/test_unittest/testmock/support.py index 49986d65dc47af..6c535b7944f261 100644 --- a/Lib/test/test_unittest/testmock/support.py +++ b/Lib/test/test_unittest/testmock/support.py @@ -14,3 +14,14 @@ def wibble(self): pass class X(object): pass + +# A standin for weurkzeug.local.LocalProxy - issue 119600 +def _inaccessible(*args, **kwargs): + raise AttributeError + + +class OpaqueProxy: + __getattribute__ = _inaccessible + + +g = OpaqueProxy() diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index f57b83f457f279..73f04291373f91 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -8,7 +8,7 @@ support.requires_working_socket(module=True) -from asyncio import run, iscoroutinefunction +from asyncio import run from unittest import IsolatedAsyncioTestCase from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock, create_autospec, sentinel, _CallList, seal) @@ -60,7 +60,7 @@ class AsyncPatchDecoratorTest(unittest.TestCase): def test_is_coroutine_function_patch(self): @patch.object(AsyncClass, 'async_method') def test_async(mock_method): - self.assertTrue(iscoroutinefunction(mock_method)) + self.assertTrue(inspect.iscoroutinefunction(mock_method)) test_async() def test_is_async_patch(self): @@ -121,7 +121,7 @@ class AsyncPatchCMTest(unittest.TestCase): def test_is_async_function_cm(self): def test_async(): with patch.object(AsyncClass, 'async_method') as mock_method: - self.assertTrue(iscoroutinefunction(mock_method)) + self.assertTrue(inspect.iscoroutinefunction(mock_method)) test_async() @@ -155,7 +155,7 @@ def test_patch_dict_async_def(self): async def test_async(): self.assertEqual(foo['a'], 'b') - self.assertTrue(iscoroutinefunction(test_async)) + self.assertTrue(inspect.iscoroutinefunction(test_async)) run(test_async()) def test_patch_dict_async_def_context(self): @@ -170,12 +170,11 @@ async def test_async(): class AsyncMockTest(unittest.TestCase): def test_iscoroutinefunction_default(self): mock = AsyncMock() - self.assertTrue(iscoroutinefunction(mock)) + self.assertTrue(inspect.iscoroutinefunction(mock)) def test_iscoroutinefunction_function(self): async def foo(): pass mock = AsyncMock(foo) - self.assertTrue(iscoroutinefunction(mock)) self.assertTrue(inspect.iscoroutinefunction(mock)) def test_isawaitable(self): @@ -188,7 +187,6 @@ def test_isawaitable(self): def test_iscoroutinefunction_normal_function(self): def foo(): pass mock = AsyncMock(foo) - self.assertTrue(iscoroutinefunction(mock)) self.assertTrue(inspect.iscoroutinefunction(mock)) def test_future_isfuture(self): @@ -231,7 +229,6 @@ async def main(): run(main()) - self.assertTrue(iscoroutinefunction(spec)) self.assertTrue(inspect.iscoroutinefunction(spec)) self.assertTrue(asyncio.iscoroutine(awaitable)) self.assertTrue(inspect.iscoroutine(awaitable)) @@ -273,7 +270,6 @@ async def test_async(): awaitable = mock_method(1, 2, c=3) self.assertIsInstance(mock_method.mock, AsyncMock) - self.assertTrue(iscoroutinefunction(mock_method)) self.assertTrue(inspect.iscoroutinefunction(mock_method)) self.assertTrue(asyncio.iscoroutine(awaitable)) self.assertTrue(inspect.iscoroutine(awaitable)) @@ -430,13 +426,13 @@ def test_async(async_method): def test_is_async_AsyncMock(self): mock = AsyncMock(spec_set=AsyncClass.async_method) - self.assertTrue(iscoroutinefunction(mock)) + self.assertTrue(inspect.iscoroutinefunction(mock)) self.assertIsInstance(mock, AsyncMock) def test_is_child_AsyncMock(self): mock = MagicMock(spec_set=AsyncClass) - self.assertTrue(iscoroutinefunction(mock.async_method)) - self.assertFalse(iscoroutinefunction(mock.normal_method)) + self.assertTrue(inspect.iscoroutinefunction(mock.async_method)) + self.assertFalse(inspect.iscoroutinefunction(mock.normal_method)) self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, MagicMock) self.assertIsInstance(mock, MagicMock) @@ -606,8 +602,8 @@ def test_magic_methods_are_async_functions(self): self.assertIsInstance(m_mock.__aenter__, AsyncMock) self.assertIsInstance(m_mock.__aexit__, AsyncMock) # AsyncMocks are also coroutine functions - self.assertTrue(iscoroutinefunction(m_mock.__aenter__)) - self.assertTrue(iscoroutinefunction(m_mock.__aexit__)) + self.assertTrue(inspect.iscoroutinefunction(m_mock.__aenter__)) + self.assertTrue(inspect.iscoroutinefunction(m_mock.__aexit__)) class AsyncContextManagerTest(unittest.TestCase): @@ -746,11 +742,11 @@ def inner_test(mock_type): mock_instance = mock_type(instance) # Check that the mock and the real thing bahave the same # __aiter__ is not actually async, so not a coroutinefunction - self.assertFalse(iscoroutinefunction(instance.__aiter__)) - self.assertFalse(iscoroutinefunction(mock_instance.__aiter__)) + self.assertFalse(inspect.iscoroutinefunction(instance.__aiter__)) + self.assertFalse(inspect.iscoroutinefunction(mock_instance.__aiter__)) # __anext__ is async - self.assertTrue(iscoroutinefunction(instance.__anext__)) - self.assertTrue(iscoroutinefunction(mock_instance.__anext__)) + self.assertTrue(inspect.iscoroutinefunction(instance.__anext__)) + self.assertTrue(inspect.iscoroutinefunction(mock_instance.__anext__)) for mock_type in [AsyncMock, MagicMock]: with self.subTest(f"test aiter and anext corourtine with {mock_type}"): @@ -806,7 +802,7 @@ def test_assert_called_but_not_awaited(self): mock = AsyncMock(AsyncClass) with assertNeverAwaited(self): mock.async_method() - self.assertTrue(iscoroutinefunction(mock.async_method)) + self.assertTrue(inspect.iscoroutinefunction(mock.async_method)) mock.async_method.assert_called() mock.async_method.assert_called_once() mock.async_method.assert_called_once_with() diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index 74785a83757a92..f260769eb8c35e 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -8,8 +8,10 @@ Mock, ANY, _CallList, patch, PropertyMock, _callable ) +from dataclasses import dataclass, field, InitVar from datetime import datetime from functools import partial +from typing import ClassVar class SomeClass(object): def one(self, a, b): pass @@ -1034,6 +1036,76 @@ def f(a): pass self.assertEqual(mock.mock_calls, []) self.assertEqual(rv.mock_calls, []) + def test_dataclass_post_init(self): + @dataclass + class WithPostInit: + a: int = field(init=False) + b: int = field(init=False) + def __post_init__(self): + self.a = 1 + self.b = 2 + + for mock in [ + create_autospec(WithPostInit, instance=True), + create_autospec(WithPostInit()), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + # Classes do not have these fields: + mock = create_autospec(WithPostInit) + msg = "Mock object has no attribute" + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b + + def test_dataclass_default(self): + @dataclass + class WithDefault: + a: int + b: int = 0 + + for mock in [ + create_autospec(WithDefault, instance=True), + create_autospec(WithDefault(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + def test_dataclass_with_method(self): + @dataclass + class WithMethod: + a: int + def b(self) -> int: + return 1 + + for mock in [ + create_autospec(WithMethod, instance=True), + create_autospec(WithMethod(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + mock.b.assert_not_called() + + def test_dataclass_with_non_fields(self): + @dataclass + class WithNonFields: + a: ClassVar[int] + b: InitVar[int] + + msg = "Mock object has no attribute" + for mock in [ + create_autospec(WithNonFields, instance=True), + create_autospec(WithNonFields(1)), + ]: + with self.subTest(mock=mock): + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b class TestCallList(unittest.TestCase): @@ -1127,6 +1199,14 @@ def test_propertymock_side_effect(self): p.assert_called_once_with() + def test_propertymock_attach(self): + m = Mock() + p = PropertyMock() + type(m).foo = p + m.attach_mock(p, 'foo') + self.assertEqual(m.mock_calls, []) + + class TestCallablePredicate(unittest.TestCase): def test_type(self): diff --git a/Lib/test/test_unittest/testmock/testmagicmethods.py b/Lib/test/test_unittest/testmock/testmagicmethods.py index a4feae7e9d3b73..2a8aa11b3284f6 100644 --- a/Lib/test/test_unittest/testmock/testmagicmethods.py +++ b/Lib/test/test_unittest/testmock/testmagicmethods.py @@ -1,7 +1,7 @@ import math import unittest import os -from asyncio import iscoroutinefunction +from inspect import iscoroutinefunction from unittest.mock import AsyncMock, Mock, MagicMock, _magics @@ -331,6 +331,45 @@ def test_magic_methods_fspath(self): self.assertEqual(os.fspath(mock), expected_path) mock.__fspath__.assert_called_once() + def test_magic_mock_does_not_reset_magic_returns(self): + # https://github.com/python/cpython/issues/123934 + for reset in (True, False): + with self.subTest(reset=reset): + mm = MagicMock() + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + for _ in range(3): + # Repeat reset several times to be sure: + mm.reset_mock(return_value=reset) + + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + def test_magic_mock_resets_manual_mocks(self): + mm = MagicMock() + mm.__iter__ = MagicMock(return_value=iter([1])) + mm.custom = MagicMock(return_value=2) + self.assertEqual(list(iter(mm)), [1]) + self.assertEqual(mm.custom(), 2) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) + self.assertIsInstance(mm.custom(), MagicMock) + + def test_magic_mock_resets_manual_mocks_empty_iter(self): + mm = MagicMock() + mm.__iter__.return_value = [] + self.assertEqual(list(iter(mm)), []) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) def test_magic_methods_and_spec(self): class Iterable(object): diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index 1725406bcfb9e4..e1b108f81e513c 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -115,6 +115,24 @@ def f(): pass with self.assertRaises(TypeError): mock() + def test_create_autospec_should_be_configurable_by_kwargs(self): + """If kwargs are given to configure mock, the function must configure + the parent mock during initialization.""" + mocked_result = 'mocked value' + class_mock = create_autospec(spec=Something, **{ + 'return_value.meth.side_effect': [ValueError, DEFAULT], + 'return_value.meth.return_value': mocked_result}) + with self.assertRaises(ValueError): + class_mock().meth(a=None, b=None, c=None) + self.assertEqual(class_mock().meth(a=None, b=None, c=None), mocked_result) + # Only the parent mock should be configurable because the user will + # pass kwargs with respect to the parent mock. + self.assertEqual(class_mock().return_value.meth.side_effect, None) + + def test_create_autospec_correctly_handles_name(self): + class X: ... + mock = create_autospec(X, spec_set=True, name="Y") + self.assertEqual(mock._mock_name, "Y") def test_repr(self): mock = Mock(name='foo') @@ -245,6 +263,65 @@ class B(object): with mock.patch('builtins.open', mock.mock_open()): mock.mock_open() # should still be valid with open() mocked + def test_create_autospec_wraps_class(self): + """Autospec a class with wraps & test if the call is passed to the + wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + class_mock = create_autospec(spec=Result, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result (actual instance of "Result") when the mock is called. + class_mock.return_value = mock.DEFAULT + self.assertEqual(class_mock().get_result(), result) + # Autospec should also wrap child attributes of parent. + self.assertEqual(class_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_instance_wraps_class(self): + """Autospec a class instance with wraps & test if the call is passed + to the wrapped object.""" + result = "real result" + + class Result: + @staticmethod + def get_result(): + """This is a static method because when the mocked instance of + 'Result' will call this method, it won't be able to consume + 'self' argument.""" + return result + instance_mock = create_autospec(spec=Result, instance=True, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result from "Result.get_result" when the mocked instance of "Result" + # calls "get_result". + instance_mock.get_result.return_value = mock.DEFAULT + self.assertEqual(instance_mock.get_result(), result) + # Autospec should also wrap child attributes of the instance. + self.assertEqual(instance_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_wraps_function_type(self): + """Autospec a function or a method with wraps & test if the call is + passed to the wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + func_mock = create_autospec(spec=Result.get_result, wraps=Result.get_result) + self.assertEqual(func_mock(Result()), result) + + def test_explicit_return_value_even_if_mock_wraps_object(self): + """If the mock has an explicit return_value set then calls are not + passed to the wrapped object and the return_value is returned instead. + """ + def my_func(): + return None + func_mock = create_autospec(spec=my_func, wraps=my_func) + return_value = "explicit return value" + func_mock.return_value = return_value + self.assertEqual(func_mock(), return_value) + def test_explicit_parent(self): parent = Mock() mock1 = Mock(parent=parent, return_value=None) @@ -622,6 +699,14 @@ def test_wraps_calls(self): real = Mock() mock = Mock(wraps=real) + # If "Mock" wraps an object, just accessing its + # "return_value" ("NonCallableMock.__get_return_value") should not + # trigger its descriptor ("NonCallableMock.__set_return_value") so + # the default "return_value" should always be "sentinel.DEFAULT". + self.assertEqual(mock.return_value, DEFAULT) + # It will not be "sentinel.DEFAULT" if the mock is not wrapping any + # object. + self.assertNotEqual(real.return_value, DEFAULT) self.assertEqual(mock(), real()) real.reset_mock() diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index 833d7da1f31a20..f26e74ce0bc1ba 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -7,9 +7,11 @@ from collections import OrderedDict import unittest +import test from test.test_unittest.testmock import support from test.test_unittest.testmock.support import SomeClass, is_instance +from test.support.import_helper import DirsOnSysPath from test.test_importlib.util import uncache from unittest.mock import ( NonCallableMock, CallableMixin, sentinel, @@ -1728,6 +1730,71 @@ def test(mock): 'exception traceback not propagated') + def test_name_resolution_import_rebinding(self): + # Currently mock.patch uses pkgutil.resolve_name(), but repeat + # similar tests just for the case. + # The same data is also used for testing import in test_import and + # pkgutil.resolve_name() in test_pkgutil. + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + + def test_create_and_specs(self): for kwarg in ('spec', 'spec_set', 'autospec'): p = patch('%s.doesnotexist' % __name__, create=True, @@ -1912,7 +1979,7 @@ def foo(x=0): with patch.object(foo, '__module__', "testpatch2"): self.assertEqual(foo.__module__, "testpatch2") - self.assertEqual(foo.__module__, 'test.test_unittest.testmock.testpatch') + self.assertEqual(foo.__module__, __name__) with patch.object(foo, '__annotations__', dict([('s', 1, )])): self.assertEqual(foo.__annotations__, dict([('s', 1, )])) @@ -1978,6 +2045,13 @@ def test(): pass with self.assertRaises(TypeError): test() + def test_patch_proxy_object(self): + @patch("test.test_unittest.testmock.support.g", new_callable=MagicMock()) + def test(_): + pass + + test() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_unpack.py b/Lib/test/test_unpack.py index 515ec128a08a9c..adb30c0febbb6f 100644 --- a/Lib/test/test_unpack.py +++ b/Lib/test/test_unpack.py @@ -18,6 +18,13 @@ >>> a == 4 and b == 5 and c == 6 True +Unpack dict + + >>> d = {4: 'four', 5: 'five', 6: 'six'} + >>> a, b, c = d + >>> a == 4 and b == 5 and c == 6 + True + Unpack implied tuple >>> a, b, c = 7, 8, 9 @@ -66,14 +73,14 @@ >>> a, b = t Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 2) + ValueError: too many values to unpack (expected 2, got 3) Unpacking tuple of wrong size >>> a, b = l Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 2) + ValueError: too many values to unpack (expected 2, got 3) Unpacking sequence too short @@ -140,8 +147,52 @@ >>> () = [42] Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 0) + ValueError: too many values to unpack (expected 0, got 1) + +Unpacking a larger iterable should raise ValuleError, but it +should not entirely consume the iterable + >>> it = iter(range(100)) + >>> x, y, z = it + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) + >>> next(it) + 4 + +Unpacking unbalanced dict + + >>> d = {4: 'four', 5: 'five', 6: 'six', 7: 'seven'} + >>> a, b, c = d + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3, got 4) + +Ensure that custom `__len__()` is NOT called when showing the error message + + >>> class LengthTooLong: + ... def __len__(self): + ... return 5 + ... def __getitem__(self, i): + ... return i*2 + ... + >>> x, y, z = LengthTooLong() + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) + +For evil cases like these as well, no actual count to be shown + + >>> class BadLength: + ... def __len__(self): + ... return 1 + ... def __getitem__(self, i): + ... return i*2 + ... + >>> x, y, z = BadLength() + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) """ __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index c201d08f61b8cd..9e2d54bd3a8c4e 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -26,6 +26,12 @@ >>> a == [7, 8, 9] True +Unpack nested implied tuple + + >>> [*[*a]] = [[7,8,9]] + >>> a == [[7,8,9]] + True + Unpack string... fun! >>> a, *b = 'one' diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 6f698a8d891815..35394f29fbe49d 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -370,13 +370,13 @@ def test_slices(self): self.check_ast_roundtrip("a[i:j, k]") def test_invalid_raise(self): - self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X"))) + self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X", ctx=ast.Load()))) def test_invalid_fstring_value(self): self.check_invalid( ast.JoinedStr( values=[ - ast.Name(id="test"), + ast.Name(id="test", ctx=ast.Load()), ast.Constant(value="test") ] ) @@ -649,6 +649,44 @@ def test_multiquote_joined_string(self): self.check_ast_roundtrip("""f'''""\"''\\'{"\\n\\"'"}''' """) self.check_ast_roundtrip("""f'''""\"''\\'{""\"\\n\\"'''""\" '''\\n'''}''' """) + def test_backslash_in_format_spec(self): + import re + msg = re.escape("invalid escape sequence '\\ '") + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\ }" """) + self.check_ast_roundtrip("""f"{x:\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\ }" """) + + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\\\\\ }" """) + self.check_ast_roundtrip("""f"{x:\\\\\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\\\\\ }" """) + + def test_quote_in_format_spec(self): + self.check_ast_roundtrip("""f"{x:'}" """) + self.check_ast_roundtrip("""f"{x:\\'}" """) + self.check_ast_roundtrip("""f"{x:\\\\'}" """) + + self.check_ast_roundtrip("""f'\\'{x:"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + + def test_type_params(self): + self.check_ast_roundtrip("type A = int") + self.check_ast_roundtrip("type A[T] = int") + self.check_ast_roundtrip("type A[T: int] = int") + self.check_ast_roundtrip("type A[T = int] = int") + self.check_ast_roundtrip("type A[T: int = int] = int") + self.check_ast_roundtrip("type A[**P] = int") + self.check_ast_roundtrip("type A[**P = int] = int") + self.check_ast_roundtrip("type A[*Ts] = int") + self.check_ast_roundtrip("type A[*Ts = int] = int") + self.check_ast_roundtrip("type A[*Ts = *int] = int") + self.check_ast_roundtrip("def f[T: int = int, **P = int, *Ts = *int]():\n pass") + self.check_ast_roundtrip("class C[T: int = int, **P = int, *Ts = *int]():\n pass") + class ManualASTCreationTestCase(unittest.TestCase): """Test that AST nodes created without a type_params field unparse correctly.""" @@ -694,11 +732,25 @@ def test_function_with_type_params_and_bound(self): body=[ast.Pass()], decorator_list=[], returns=None, - type_params=[ast.TypeVar("T", bound=ast.Name("int"))], + type_params=[ast.TypeVar("T", bound=ast.Name("int", ctx=ast.Load()))], ) ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") + def test_function_with_type_params_and_default(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T = 1, *Ts = *1, **P = 1]():\n pass") + def test_async_function(self): node = ast.AsyncFunctionDef( name="f", @@ -722,6 +774,20 @@ def test_async_function_with_type_params(self): ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "async def f[T]():\n pass") + def test_async_function_with_type_params_and_default(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T = 1, *Ts = *1, **P = 1]():\n pass") + class DirectoryTestCase(ASTTestCase): """Test roundtrip behaviour on all files in Lib and Lib/test.""" diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 6bb0fb303fa202..3ee17f96b817e1 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -19,7 +19,6 @@ ssl = None import sys import tempfile -from nturl2path import url2pathname, pathname2url from base64 import b64encode import collections @@ -1520,39 +1519,86 @@ def test_quoting(self): (expect, result)) @unittest.skipUnless(sys.platform == 'win32', - 'test specific to the nturl2path functions.') - def test_prefixes(self): + 'test specific to Windows pathnames.') + def test_pathname2url_win(self): # Test special prefixes are correctly handled in pathname2url() - given = '\\\\?\\C:\\dir' - expect = '///C:/dir' - result = urllib.request.pathname2url(given) - self.assertEqual(expect, result, - "pathname2url() failed; %s != %s" % - (expect, result)) - given = '\\\\?\\unc\\server\\share\\dir' - expect = '/server/share/dir' - result = urllib.request.pathname2url(given) - self.assertEqual(expect, result, - "pathname2url() failed; %s != %s" % - (expect, result)) - + fn = urllib.request.pathname2url + self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir') + self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir') + self.assertEqual(fn("C:"), '///C:') + self.assertEqual(fn("C:\\"), '///C:') + self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c') + self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c') + self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9') + self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo") + # Long drive letter + self.assertRaises(IOError, fn, "XX:\\") + # No drive letter + self.assertEqual(fn("\\folder\\test\\"), '/folder/test/') + self.assertEqual(fn("\\\\folder\\test\\"), '//folder/test/') + self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn('\\\\some\\share\\'), '//some/share/') + self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c') + self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9') + # Round-tripping + urls = ['///C:', + '///folder/test/', + '///C:/foo/bar/spam.foo'] + for url in urls: + self.assertEqual(fn(urllib.request.url2pathname(url)), url) + + @unittest.skipIf(sys.platform == 'win32', + 'test specific to POSIX pathnames') + def test_pathname2url_posix(self): + fn = urllib.request.pathname2url + self.assertEqual(fn('/'), '/') + self.assertEqual(fn('/a/b.c'), '/a/b.c') + self.assertEqual(fn('/a/b%#c'), '/a/b%25%23c') @unittest.skipUnless(sys.platform == 'win32', - 'test specific to the urllib.url2path function.') - def test_ntpath(self): - given = ('/C:/', '///C:/', '/C|//') - expect = 'C:\\' - for url in given: - result = urllib.request.url2pathname(url) - self.assertEqual(expect, result, - 'urllib.request..url2pathname() failed; %s != %s' % - (expect, result)) - given = '///C|/path' - expect = 'C:\\path' - result = urllib.request.url2pathname(given) - self.assertEqual(expect, result, - 'urllib.request.url2pathname() failed; %s != %s' % - (expect, result)) + 'test specific to Windows pathnames.') + def test_url2pathname_win(self): + fn = urllib.request.url2pathname + self.assertEqual(fn('/C:/'), 'C:\\') + self.assertEqual(fn("///C|"), 'C:') + self.assertEqual(fn("///C:"), 'C:') + self.assertEqual(fn('///C:/'), 'C:\\') + self.assertEqual(fn('/C|//'), 'C:\\') + self.assertEqual(fn('///C|/path'), 'C:\\path') + # No DOS drive + self.assertEqual(fn("///C/test/"), '\\\\\\C\\test\\') + self.assertEqual(fn("////C/test/"), '\\\\C\\test\\') + # DOS drive paths + self.assertEqual(fn('C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('///C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn("///C|/foo/bar/spam.foo"), 'C:\\foo\\bar\\spam.foo') + # Non-ASCII drive letter + self.assertRaises(IOError, fn, "///\u00e8|/") + # UNC paths + self.assertEqual(fn('//server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('////server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('/////server/path/to/file'), '\\\\\\server\\path\\to\\file') + # Localhost paths + self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file') + # Round-tripping + paths = ['C:', + r'\\\C\test\\', + r'C:\foo\bar\spam.foo'] + for path in paths: + self.assertEqual(fn(urllib.request.pathname2url(path)), path) + + @unittest.skipIf(sys.platform == 'win32', + 'test specific to POSIX pathnames') + def test_url2pathname_posix(self): + fn = urllib.request.url2pathname + self.assertEqual(fn('/foo/bar'), '/foo/bar') + self.assertEqual(fn('//foo/bar'), '//foo/bar') + self.assertEqual(fn('///foo/bar'), '///foo/bar') + self.assertEqual(fn('////foo/bar'), '////foo/bar') + self.assertEqual(fn('//localhost/foo/bar'), '//localhost/foo/bar') class Utility_Tests(unittest.TestCase): """Testcase to test the various utility functions in the urllib.""" @@ -1636,60 +1682,5 @@ def test_with_method_arg(self): self.assertEqual(request.get_method(), 'HEAD') -class URL2PathNameTests(unittest.TestCase): - - def test_converting_drive_letter(self): - self.assertEqual(url2pathname("///C|"), 'C:') - self.assertEqual(url2pathname("///C:"), 'C:') - self.assertEqual(url2pathname("///C|/"), 'C:\\') - - def test_converting_when_no_drive_letter(self): - # cannot end a raw string in \ - self.assertEqual(url2pathname("///C/test/"), r'\\\C\test' '\\') - self.assertEqual(url2pathname("////C/test/"), r'\\C\test' '\\') - - def test_simple_compare(self): - self.assertEqual(url2pathname("///C|/foo/bar/spam.foo"), - r'C:\foo\bar\spam.foo') - - def test_non_ascii_drive_letter(self): - self.assertRaises(IOError, url2pathname, "///\u00e8|/") - - def test_roundtrip_url2pathname(self): - list_of_paths = ['C:', - r'\\\C\test\\', - r'C:\foo\bar\spam.foo' - ] - for path in list_of_paths: - self.assertEqual(url2pathname(pathname2url(path)), path) - -class PathName2URLTests(unittest.TestCase): - - def test_converting_drive_letter(self): - self.assertEqual(pathname2url("C:"), '///C:') - self.assertEqual(pathname2url("C:\\"), '///C:') - - def test_converting_when_no_drive_letter(self): - self.assertEqual(pathname2url(r"\\\folder\test" "\\"), - '/////folder/test/') - self.assertEqual(pathname2url(r"\\folder\test" "\\"), - '////folder/test/') - self.assertEqual(pathname2url(r"\folder\test" "\\"), - '/folder/test/') - - def test_simple_compare(self): - self.assertEqual(pathname2url(r'C:\foo\bar\spam.foo'), - "///C:/foo/bar/spam.foo" ) - - def test_long_drive_letter(self): - self.assertRaises(IOError, pathname2url, "XX:\\") - - def test_roundtrip_pathname2url(self): - list_of_paths = ['///C:', - '/////folder/test/', - '///C:/foo/bar/spam.foo'] - for path in list_of_paths: - self.assertEqual(pathname2url(url2pathname(path)), path) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index fa528a675892b5..b90ccc2f125b93 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -8,6 +8,7 @@ import os import io +import ftplib import socket import array import sys @@ -15,10 +16,11 @@ import subprocess import urllib.request -# The proxy bypass method imported below has logic specific to the OSX -# proxy config data structure but is testable on all platforms. +# The proxy bypass method imported below has logic specific to the +# corresponding system but is testable on all platforms. from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, HTTPPasswordMgrWithPriorAuth, _parse_proxy, + _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) from urllib.parse import urlparse @@ -753,7 +755,6 @@ def connect_ftp(self, user, passwd, host, port, dirs, self.ftpwrapper = MockFTPWrapper(self.data) return self.ftpwrapper - import ftplib data = "rheum rhaponicum" h = NullFTPHandler(data) h.parent = MockOpener() @@ -776,7 +777,7 @@ def connect_ftp(self, user, passwd, host, port, dirs, ["foo", "bar"], "", None), ("ftp://localhost/baz.gif;type=a", "localhost", ftplib.FTP_PORT, "", "", "A", - [], "baz.gif", None), # XXX really this should guess image/gif + [], "baz.gif", "image/gif"), ]: req = Request(url) req.timeout = None @@ -793,6 +794,28 @@ def connect_ftp(self, user, passwd, host, port, dirs, self.assertEqual(headers.get("Content-type"), mimetype) self.assertEqual(int(headers["Content-length"]), len(data)) + @support.requires_resource("network") + def test_ftp_error(self): + class ErrorFTPHandler(urllib.request.FTPHandler): + def __init__(self, exception): + self._exception = exception + + def connect_ftp(self, user, passwd, host, port, dirs, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + raise self._exception + + exception = ftplib.error_perm( + "500 OOPS: cannot change directory:/nonexistent") + h = ErrorFTPHandler(exception) + urlopen = urllib.request.build_opener(h).open + try: + urlopen("ftp://www.pythontest.net/") + except urllib.error.URLError as raised: + self.assertEqual(raised.reason, + f"ftp error: {exception.args[0]}") + else: + self.fail("Did not raise ftplib exception") + def test_file(self): import email.utils h = urllib.request.FileHandler() @@ -1401,6 +1424,15 @@ def http_open(self, req): request = handler.last_buf self.assertTrue(request.startswith(expected), repr(request)) + def test_redirect_head_request(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + req = Request(from_url, method="HEAD") + fp = MockFile() + new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url) + self.assertEqual(new_req.get_method(), "HEAD") + def test_proxy(self): u = "proxy.example.com:3128" for d in dict(http=u), dict(HTTP=u): @@ -1485,6 +1517,30 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + @unittest.skipUnless(os.name == "nt", "only relevant for Windows") + def test_winreg_proxy_bypass(self): + proxy_override = "www.example.com;*.example.net; 192.168.0.1" + proxy_bypass = _proxy_bypass_winreg_override + for host in ("www.example.com", "www.example.net", "192.168.0.1"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expected bypass of %s to be true" % host) + + for host in ("example.com", "www.example.org", "example.net", + "192.168.0.2"): + self.assertFalse(proxy_bypass(host, proxy_override), + "expected bypass of %s to be False" % host) + + # check intranet address bypass + proxy_override = "example.com; " + self.assertTrue(proxy_bypass("example.com", proxy_override), + "expected bypass of %s to be true" % host) + self.assertFalse(proxy_bypass("example.net", proxy_override), + "expected bypass of %s to be False" % host) + for host in ("test", "localhost"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expect to bypass intranet address '%s'" + % host) + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 625c6dc88796b6..d49e4388696ab4 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -19,6 +19,10 @@ ("=a", [('', 'a')]), ("a", [('a', '')]), ("a=", [('a', '')]), + ("a=b=c", [('a', 'b=c')]), + ("a%3Db=c", [('a=b', 'c')]), + ("a=b&c=d", [('a', 'b'), ('c', 'd')]), + ("a=b%26c=d", [('a', 'b&c=d')]), ("&a=b", [('a', 'b')]), ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]), ("a=1&a=2", [('a', '1'), ('a', '2')]), @@ -29,6 +33,10 @@ (b"=a", [(b'', b'a')]), (b"a", [(b'a', b'')]), (b"a=", [(b'a', b'')]), + (b"a=b=c", [(b'a', b'b=c')]), + (b"a%3Db=c", [(b'a=b', b'c')]), + (b"a=b&c=d", [(b'a', b'b'), (b'c', b'd')]), + (b"a=b%26c=d", [(b'a', b'b&c=d')]), (b"&a=b", [(b'a', b'b')]), (b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]), @@ -36,6 +44,14 @@ ("a=a+b;b=b+c", [('a', 'a b;b=b c')]), (b";a=b", [(b';a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]), + + ("\u0141=\xE9", [('\u0141', '\xE9')]), + ("%C5%81=%C3%A9", [('\u0141', '\xE9')]), + ("%81=%A9", [('\ufffd', '\ufffd')]), + (b"\xc5\x81=\xc3\xa9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"%C5%81=%C3%A9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"\x81=\xA9", [(b'\x81', b'\xa9')]), + (b"%81=%A9", [(b'\x81', b'\xa9')]), ] # Each parse_qs testcase is a two-tuple that contains @@ -49,6 +65,10 @@ ("=a", {'': ['a']}), ("a", {'a': ['']}), ("a=", {'a': ['']}), + ("a=b=c", {'a': ['b=c']}), + ("a%3Db=c", {'a=b': ['c']}), + ("a=b&c=d", {'a': ['b'], 'c': ['d']}), + ("a=b%26c=d", {'a': ['b&c=d']}), ("&a=b", {'a': ['b']}), ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), ("a=1&a=2", {'a': ['1', '2']}), @@ -59,6 +79,10 @@ (b"=a", {b'': [b'a']}), (b"a", {b'a': [b'']}), (b"a=", {b'a': [b'']}), + (b"a=b=c", {b'a': [b'b=c']}), + (b"a%3Db=c", {b'a=b': [b'c']}), + (b"a=b&c=d", {b'a': [b'b'], b'c': [b'd']}), + (b"a=b%26c=d", {b'a': [b'b&c=d']}), (b"&a=b", {b'a': [b'b']}), (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1&a=2", {b'a': [b'1', b'2']}), @@ -66,11 +90,22 @@ ("a=a+b;b=b+c", {'a': ['a b;b=b c']}), (b";a=b", {b';a': [b'b']}), (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}), + (b"a=a%E2%80%99b", {b'a': [b'a\xe2\x80\x99b']}), + + ("\u0141=\xE9", {'\u0141': ['\xE9']}), + ("%C5%81=%C3%A9", {'\u0141': ['\xE9']}), + ("%81=%A9", {'\ufffd': ['\ufffd']}), + (b"\xc5\x81=\xc3\xa9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"%C5%81=%C3%A9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"\x81=\xA9", {b'\x81': [b'\xa9']}), + (b"%81=%A9", {b'\x81': [b'\xa9']}), ] class UrlParseTestCase(unittest.TestCase): - def checkRoundtrips(self, url, parsed, split): + def checkRoundtrips(self, url, parsed, split, url2=None): + if url2 is None: + url2 = url result = urllib.parse.urlparse(url) self.assertSequenceEqual(result, parsed) t = (result.scheme, result.netloc, result.path, @@ -78,7 +113,7 @@ def checkRoundtrips(self, url, parsed, split): self.assertSequenceEqual(t, parsed) # put it back together and it should be the same result2 = urllib.parse.urlunparse(result) - self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, url2) self.assertSequenceEqual(result2, result.geturl()) # the result of geturl() is a fixpoint; we can always parse it @@ -104,7 +139,7 @@ def checkRoundtrips(self, url, parsed, split): result.query, result.fragment) self.assertSequenceEqual(t, split) result2 = urllib.parse.urlunsplit(result) - self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, url2) self.assertSequenceEqual(result2, result.geturl()) # check the fixpoint property of re-parsing the result of geturl() @@ -142,9 +177,60 @@ def test_qs(self): def test_roundtrips(self): str_cases = [ + ('path/to/file', + ('', '', 'path/to/file', '', '', ''), + ('', '', 'path/to/file', '', '')), + ('/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('//path/to/file', + ('', 'path', '/to/file', '', '', ''), + ('', 'path', '/to/file', '', '')), + ('////path/to/file', + ('', '', '//path/to/file', '', '', ''), + ('', '', '//path/to/file', '', '')), + ('/////path/to/file', + ('', '', '///path/to/file', '', '', ''), + ('', '', '///path/to/file', '', '')), + ('scheme:path/to/file', + ('scheme', '', 'path/to/file', '', '', ''), + ('scheme', '', 'path/to/file', '', '')), + ('scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('scheme://path/to/file', + ('scheme', 'path', '/to/file', '', '', ''), + ('scheme', 'path', '/to/file', '', '')), + ('scheme:////path/to/file', + ('scheme', '', '//path/to/file', '', '', ''), + ('scheme', '', '//path/to/file', '', '')), + ('scheme://///path/to/file', + ('scheme', '', '///path/to/file', '', '', ''), + ('scheme', '', '///path/to/file', '', '')), + ('file:tmp/junk.txt', + ('file', '', 'tmp/junk.txt', '', '', ''), + ('file', '', 'tmp/junk.txt', '', '')), ('file:///tmp/junk.txt', ('file', '', '/tmp/junk.txt', '', '', ''), ('file', '', '/tmp/junk.txt', '', '')), + ('file:////tmp/junk.txt', + ('file', '', '//tmp/junk.txt', '', '', ''), + ('file', '', '//tmp/junk.txt', '', '')), + ('file://///tmp/junk.txt', + ('file', '', '///tmp/junk.txt', '', '', ''), + ('file', '', '///tmp/junk.txt', '', '')), + ('http:tmp/junk.txt', + ('http', '', 'tmp/junk.txt', '', '', ''), + ('http', '', 'tmp/junk.txt', '', '')), + ('http://example.com/tmp/junk.txt', + ('http', 'example.com', '/tmp/junk.txt', '', '', ''), + ('http', 'example.com', '/tmp/junk.txt', '', '')), + ('http:///example.com/tmp/junk.txt', + ('http', '', '/example.com/tmp/junk.txt', '', '', ''), + ('http', '', '/example.com/tmp/junk.txt', '', '')), + ('http:////example.com/tmp/junk.txt', + ('http', '', '//example.com/tmp/junk.txt', '', '', ''), + ('http', '', '//example.com/tmp/junk.txt', '', '')), ('imap://mail.python.org/mbox1', ('imap', 'mail.python.org', '/mbox1', '', '', ''), ('imap', 'mail.python.org', '/mbox1', '', '')), @@ -171,14 +257,59 @@ def test_roundtrips(self): 'action=download-manifest&url=https://example.com/app', ''), ('itms-services', '', '', 'action=download-manifest&url=https://example.com/app', '')), + ('+scheme:path/to/file', + ('', '', '+scheme:path/to/file', '', '', ''), + ('', '', '+scheme:path/to/file', '', '')), + ('sch_me:path/to/file', + ('', '', 'sch_me:path/to/file', '', '', ''), + ('', '', 'sch_me:path/to/file', '', '')), ] def _encode(t): return (t[0].encode('ascii'), tuple(x.encode('ascii') for x in t[1]), tuple(x.encode('ascii') for x in t[2])) bytes_cases = [_encode(x) for x in str_cases] + str_cases += [ + ('schème:path/to/file', + ('', '', 'schème:path/to/file', '', '', ''), + ('', '', 'schème:path/to/file', '', '')), + ] for url, parsed, split in str_cases + bytes_cases: - self.checkRoundtrips(url, parsed, split) + with self.subTest(url): + self.checkRoundtrips(url, parsed, split) + + def test_roundtrips_normalization(self): + str_cases = [ + ('///path/to/file', + '/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('scheme:///path/to/file', + 'scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('file:/tmp/junk.txt', + 'file:///tmp/junk.txt', + ('file', '', '/tmp/junk.txt', '', '', ''), + ('file', '', '/tmp/junk.txt', '', '')), + ('http:/tmp/junk.txt', + 'http:///tmp/junk.txt', + ('http', '', '/tmp/junk.txt', '', '', ''), + ('http', '', '/tmp/junk.txt', '', '')), + ('https:/tmp/junk.txt', + 'https:///tmp/junk.txt', + ('https', '', '/tmp/junk.txt', '', '', ''), + ('https', '', '/tmp/junk.txt', '', '')), + ] + def _encode(t): + return (t[0].encode('ascii'), + t[1].encode('ascii'), + tuple(x.encode('ascii') for x in t[2]), + tuple(x.encode('ascii') for x in t[3])) + bytes_cases = [_encode(x) for x in str_cases] + for url, url2, parsed, split in str_cases + bytes_cases: + with self.subTest(url): + self.checkRoundtrips(url, parsed, split, url2) def test_http_roundtrips(self): # urllib.parse.urlsplit treats 'http:' as an optimized special case, @@ -218,12 +349,19 @@ def _encode(t): split = (scheme,) + split self.checkRoundtrips(url, parsed, split) - def checkJoin(self, base, relurl, expected): - str_components = (base, relurl, expected) - self.assertEqual(urllib.parse.urljoin(base, relurl), expected) - bytes_components = baseb, relurlb, expectedb = [ - x.encode('ascii') for x in str_components] - self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + def checkJoin(self, base, relurl, expected, *, relroundtrip=True): + with self.subTest(base=base, relurl=relurl): + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + baseb = base.encode('ascii') + relurlb = relurl.encode('ascii') + expectedb = expected.encode('ascii') + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + + if relroundtrip: + relurl = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurl)) + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) def test_unparse_parse(self): str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] @@ -389,8 +527,6 @@ def test_RFC3986(self): def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g:h','g:h') - self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') self.checkJoin(SIMPLE_BASE, 'g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, './g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, 'g/','http://a/b/c/g/') @@ -411,8 +547,6 @@ def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g/./h','http://a/b/c/g/h') self.checkJoin(SIMPLE_BASE, 'g/../h','http://a/b/c/h') self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') - self.checkJoin(SIMPLE_BASE, 'http:?y','http://a/b/c/d?y') self.checkJoin(SIMPLE_BASE, 'http:g?y','http://a/b/c/g?y') self.checkJoin(SIMPLE_BASE, 'http:g?y/./x','http://a/b/c/g?y/./x') self.checkJoin('http:///', '..','http:///') @@ -442,6 +576,53 @@ def test_urljoins(self): # issue 23703: don't duplicate filename self.checkJoin('a', 'b', 'b') + # Test with empty (but defined) components. + self.checkJoin(RFC1808_BASE, '', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, ';', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, ';?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, ';#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, ';x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, '/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '//#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '//?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, '//;x', 'http://;x') + self.checkJoin(RFC1808_BASE, '///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//v', 'http://v') + # For backward compatibility with RFC1630, the scheme name is allowed + # to be present in a relative reference if it is the same as the base + # URI scheme. + self.checkJoin(RFC1808_BASE, 'http:', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http:#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http:?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http:;', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, 'http:;?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, 'http:;#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, 'http:;x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, 'http:/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http://#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http://?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http://;x', 'http://;x') + self.checkJoin(RFC1808_BASE, 'http:///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://v', 'http://v') + # Different scheme is not ignored. + self.checkJoin(RFC1808_BASE, 'https:', 'https:', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#', 'https:#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#z', 'https:#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?', 'https:?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?y', 'https:?y', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:;', 'https:;') + self.checkJoin(RFC1808_BASE, 'https:;x', 'https:;x') + def test_RFC2732(self): str_cases = [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), @@ -504,16 +685,31 @@ def test_urldefrag(self): ('http://python.org/p?q', 'http://python.org/p?q', ''), (RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'), (RFC2396_BASE, 'http://a/b/c/d;p?q', ''), + ('http://a/b/c;p?q#f', 'http://a/b/c;p?q', 'f'), + ('http://a/b/c;p?q#', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?q', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?#f', 'http://a/b/c;p?', 'f'), + ('http://a/b/c;p#f', 'http://a/b/c;p', 'f'), + ('http://a/b/c;?q#f', 'http://a/b/c;?q', 'f'), + ('http://a/b/c?q#f', 'http://a/b/c?q', 'f'), + ('http:///b/c;p?q#f', 'http:///b/c;p?q', 'f'), + ('http:b/c;p?q#f', 'http:b/c;p?q', 'f'), + ('http:;?q#f', 'http:;?q', 'f'), + ('http:?q#f', 'http:?q', 'f'), + ('//a/b/c;p?q#f', '//a/b/c;p?q', 'f'), + ('://a/b/c;p?q#f', '://a/b/c;p?q', 'f'), ] def _encode(t): return type(t)(x.encode('ascii') for x in t) bytes_cases = [_encode(x) for x in str_cases] for url, defrag, frag in str_cases + bytes_cases: - result = urllib.parse.urldefrag(url) - self.assertEqual(result.geturl(), url) - self.assertEqual(result, (defrag, frag)) - self.assertEqual(result.url, defrag) - self.assertEqual(result.fragment, frag) + with self.subTest(url): + result = urllib.parse.urldefrag(url) + hash = '#' if isinstance(url, str) else b'#' + self.assertEqual(result.geturl(), url.rstrip(hash)) + self.assertEqual(result, (defrag, frag)) + self.assertEqual(result.url, defrag) + self.assertEqual(result.fragment, frag) def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') @@ -995,8 +1191,8 @@ def test_parse_qsl_encoding(self): def test_parse_qsl_max_num_fields(self): with self.assertRaises(ValueError): - urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) - urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*11), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*10), max_num_fields=10) def test_parse_qs_separator(self): parse_qs_semicolon_cases = [ @@ -1039,6 +1235,30 @@ def test_parse_qsl_separator(self): result_bytes = urllib.parse.parse_qsl(orig, separator=b';') self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) + def test_parse_qsl_bytes(self): + self.assertEqual(urllib.parse.parse_qsl(b'a=b'), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(bytearray(b'a=b')), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(memoryview(b'a=b')), [(b'a', b'b')]) + + def test_parse_qsl_false_value(self): + kwargs = dict(keep_blank_values=True, strict_parsing=True) + for x in '', b'', None, 0, 0.0, [], {}, memoryview(b''): + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + + def test_parse_qsl_errors(self): + self.assertRaises(TypeError, urllib.parse.parse_qsl, list(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, iter(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, 1) + self.assertRaises(TypeError, urllib.parse.parse_qsl, object()) + + for separator in '', b'', None, 0, 1, 0.0, 1.5: + with self.assertRaises(ValueError): + urllib.parse.parse_qsl('a=b', separator=separator) + with self.assertRaises(UnicodeEncodeError): + urllib.parse.parse_qsl(b'a=b', separator='\xa6') + with self.assertRaises(UnicodeDecodeError): + urllib.parse.parse_qsl('a=b', separator=b'\xa6') def test_urlencode_sequences(self): # Other tests incidentally urlencode things; test non-covered cases: @@ -1369,13 +1589,6 @@ def test_unwrap(self): class DeprecationTest(unittest.TestCase): - - def test_Quoter_deprecation(self): - with self.assertWarns(DeprecationWarning) as cm: - old_class = urllib.parse.Quoter - self.assertIs(old_class, urllib.parse._Quoter) - self.assertIn('Quoter will be removed', str(cm.warning)) - def test_splittype_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittype('') diff --git a/Lib/test/test_utf8source.py b/Lib/test/test_utf8source.py index c42b6aaaab579d..7336cf00a71183 100644 --- a/Lib/test/test_utf8source.py +++ b/Lib/test/test_utf8source.py @@ -14,7 +14,7 @@ def test_pep3120(self): def test_badsyntax(self): try: - import test.tokenizedata.badsyntax_pep3120 + import test.tokenizedata.badsyntax_pep3120 # noqa: F401 except SyntaxError as msg: msg = str(msg).lower() self.assertTrue('utf-8' in msg) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 9cec1e87fd3c2d..e177464c00f7a6 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -530,7 +530,14 @@ def test_uuid1(self): @support.requires_mac_ver(10, 5) @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') def test_uuid1_safe(self): - if not self.uuid._has_uuid_generate_time_safe: + try: + import _uuid + except ImportError: + has_uuid_generate_time_safe = False + else: + has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe + + if not has_uuid_generate_time_safe or not self.uuid._generate_time_safe: self.skipTest('requires uuid_generate_time_safe(3)') u = self.uuid.uuid1() @@ -546,7 +553,6 @@ def mock_generate_time_safe(self, safe_value): """ if os.name != 'posix': self.skipTest('POSIX-only test') - self.uuid._load_system_functions() f = self.uuid._generate_time_safe if f is None: self.skipTest('need uuid._generate_time_safe') @@ -581,8 +587,7 @@ def test_uuid1_bogus_return_value(self): self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) def test_uuid1_time(self): - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch.object(self.uuid, 'getnode', return_value=93328246233727), \ mock.patch('time.time_ns', return_value=1545052026752910643), \ @@ -590,8 +595,7 @@ def test_uuid1_time(self): u = self.uuid.uuid1() self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch('time.time_ns', return_value=1545052026752910643): u = self.uuid.uuid1(node=93328246233727, clock_seq=5317) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index ba31beb81e80b0..0b09010c69d4ea 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -17,12 +17,15 @@ import sys import sysconfig import tempfile +import shlex from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess, is_apple_mobile, is_emscripten, - is_wasi, requires_venv_with_pip, TEST_HOME_DIR, + requires_subprocess, is_android, is_apple_mobile, + is_emscripten, is_wasi, + requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) -from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) +from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, + TESTFN, FakePath) import unittest import venv from unittest.mock import patch, Mock @@ -39,10 +42,8 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -# Skip tests on WASM platforms, plus iOS/tvOS/watchOS -if is_apple_mobile or is_emscripten or is_wasi: - raise unittest.SkipTest(f"venv tests not required on {sys.platform}") - +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("venv is not available on this platform") @requires_subprocess() def check_output(cmd, encoding=None): @@ -75,7 +76,7 @@ def setUp(self): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) + self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}') self.include = 'include' executable = sys._base_executable self.exe = os.path.split(executable)[-1] @@ -110,6 +111,10 @@ def get_text_file_contents(self, *args, encoding='utf-8'): result = f.read() return result + def assertEndsWith(self, string, tail): + if not string.endswith(tail): + self.fail(f"String {string!r} does not end with {tail!r}") + class BasicTest(BaseTest): """Test venv module functionality.""" @@ -125,12 +130,12 @@ def test_defaults_with_str_path(self): self.run_with_capture(venv.create, self.env_dir) self._check_output_of_default_create() - def test_defaults_with_pathlib_path(self): + def test_defaults_with_pathlike(self): """ - Test the create function with default arguments and a pathlib.Path path. + Test the create function with default arguments and a path-like path. """ rmtree(self.env_dir) - self.run_with_capture(venv.create, pathlib.Path(self.env_dir)) + self.run_with_capture(venv.create, FakePath(self.env_dir)) self._check_output_of_default_create() def _check_output_of_default_create(self): @@ -274,7 +279,8 @@ def test_prefixes(self): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), prefix) + self.assertEqual(pathlib.Path(out.strip().decode()), + pathlib.Path(expected), prefix) @requireVenvCreate def test_sysconfig(self): @@ -487,6 +493,82 @@ def test_executable_symlinks(self): 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_bash(self): + """ + Test that the template strings are quoted properly (bash) + """ + rmtree(self.env_dir) + bash = shutil.which('bash') + if bash is None: + self.skipTest('bash required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate') + test_script = os.path.join(self.env_dir, 'test_special_chars.sh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([bash, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_csh(self): + """ + Test that the template strings are quoted properly (csh) + """ + rmtree(self.env_dir) + csh = shutil.which('tcsh') or shutil.which('csh') + if csh is None: + self.skipTest('csh required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.csh') + test_script = os.path.join(self.env_dir, 'test_special_chars.csh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([csh, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings on Windows + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_special_chars_windows(self): + """ + Test that the template strings are quoted properly on Windows + """ + rmtree(self.env_dir) + env_name = "'&&^$e" + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_special_chars.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + f'"{activate}" & ' + f'{self.exe} -c "import sys; print(sys.executable)" & ' + f'{self.exe} -c "import os; print(os.environ[\'VIRTUAL_ENV\'])" & ' + 'deactivate') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') def test_unicode_in_batch_file(self): """ @@ -503,6 +585,21 @@ def test_unicode_in_batch_file(self): ) self.assertEqual(out.strip(), '0') + @unittest.skipUnless(os.name == 'nt' and can_symlink(), + 'symlinks on Windows') + def test_failed_symlink(self): + """ + Test handling of failed symlinks on Windows. + """ + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + with patch('os.symlink') as mock_symlink: + mock_symlink.side_effect = OSError() + builder = venv.EnvBuilder(clear=True, symlinks=True) + _, err = self.run_with_capture(builder.create, env_dir) + filepath_regex = r"'[A-Z]:\\\\(?:[^\\\\]+\\\\)*[^\\\\]+'" + self.assertRegex(err, rf"Unable to symlink {filepath_regex} to {filepath_regex}") + @requireVenvCreate def test_multiprocessing(self): """ @@ -532,7 +629,7 @@ def test_multiprocessing_recursion(self): rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py') - subprocess.check_call([self.envpy(real_env_dir=True), script]) + subprocess.check_call([self.envpy(real_env_dir=True), "-I", script]) @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') def test_deactivate_with_strict_bash_opts(self): @@ -571,7 +668,7 @@ def test_pathsep_error(self): rmtree(self.env_dir) bad_itempath = self.env_dir + os.pathsep self.assertRaises(ValueError, venv.create, bad_itempath) - self.assertRaises(ValueError, venv.create, pathlib.Path(bad_itempath)) + self.assertRaises(ValueError, venv.create, FakePath(bad_itempath)) @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') @requireVenvCreate @@ -592,7 +689,8 @@ def test_zippath_from_non_installed_posix(self): libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) os.makedirs(libdir) landmark = os.path.join(libdir, "os.py") - stdlib_zip = "python%d%d.zip" % sys.version_info[:2] + abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "" + stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}" zip_landmark = os.path.join(non_installed_dir, platlibdir, stdlib_zip) @@ -744,6 +842,57 @@ def test_cli_without_scm_ignore_files(self): with self.assertRaises(FileNotFoundError): self.get_text_file_contents('.gitignore') + def test_venv_same_path(self): + same_path = venv.EnvBuilder._same_path + if sys.platform == 'win32': + # Case-insensitive, and handles short/long names + tests = [ + (True, TESTFN, TESTFN), + (True, TESTFN.lower(), TESTFN.upper()), + ] + import _winapi + # ProgramFiles is the most reliable path that will have short/long + progfiles = os.getenv('ProgramFiles') + if progfiles: + tests = [ + *tests, + (True, progfiles, progfiles), + (True, _winapi.GetShortPathName(progfiles), _winapi.GetLongPathName(progfiles)), + ] + else: + # Just a simple case-sensitive comparison + tests = [ + (True, TESTFN, TESTFN), + (False, TESTFN.lower(), TESTFN.upper()), + ] + for r, path1, path2 in tests: + with self.subTest(f"{path1}-{path2}"): + if r: + self.assertTrue(same_path(path1, path2)) + else: + self.assertFalse(same_path(path1, path2)) + + # gh-126084: venvwlauncher should run pythonw, not python + @requireVenvCreate + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_venvwlauncher(self): + """ + Test that the GUI launcher runs the GUI python. + """ + rmtree(self.env_dir) + venv.create(self.env_dir) + exename = self.exe + # Retain the debug suffix if present + if "python" in exename and not "pythonw" in exename: + exename = exename.replace("python", "pythonw") + envpyw = os.path.join(self.env_dir, self.bindir, exename) + try: + subprocess.check_call([envpyw, "-c", "import sys; " + "assert sys._base_executable.endswith('%s')" % exename]) + except subprocess.CalledProcessError: + self.fail("venvwlauncher.exe did not run %s" % exename) + + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" @@ -854,6 +1003,14 @@ def do_test_with_pip(self, system_site_packages): err = re.sub("^(WARNING: )?The directory .* or its parent directory " "is not owned or is not writable by the current user.*$", "", err, flags=re.MULTILINE) + # Ignore warning about missing optional module: + try: + import ssl + except ImportError: + err = re.sub( + "^WARNING: Disabling truststore since ssl support is missing$", + "", + err, flags=re.MULTILINE) self.assertEqual(err.rstrip(), "") # Being fairly specific regarding the expected behaviour for the # initial bundling phase in Python 3.4. If the output changes in diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 50b0f3fff04c57..8b59630717e790 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1,6 +1,8 @@ from contextlib import contextmanager import linecache import os +import importlib +import inspect from io import StringIO import re import sys @@ -12,6 +14,7 @@ from test.support import import_helper from test.support import os_helper from test.support import warnings_helper +from test.support import force_not_colorized from test.support.script_helper import assert_python_ok, assert_python_failure from test.test_warnings.data import package_helper @@ -154,40 +157,42 @@ def f(): f() self.assertEqual(len(w), 1) - def test_always(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: - self.module.resetwarnings() - self.module.filterwarnings("always", category=UserWarning) - message = "FilterTests.test_always" - def f(): - self.module.warn(message, UserWarning) - f() - self.assertEqual(len(w), 1) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 2) - self.assertEqual(w[-1].message.args[0], message) + def test_always_and_all(self): + for mode in {"always", "all"}: + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings(mode, category=UserWarning) + message = "FilterTests.test_always_and_all" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) - def test_always_after_default(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: - self.module.resetwarnings() - message = "FilterTests.test_always_after_ignore" - def f(): - self.module.warn(message, UserWarning) - f() - self.assertEqual(len(w), 1) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 1) - self.module.filterwarnings("always", category=UserWarning) - f() - self.assertEqual(len(w), 2) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 3) - self.assertEqual(w[-1].message.args[0], message) + def test_always_and_all_after_default(self): + for mode in {"always", "all"}: + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + message = "FilterTests.test_always_and_all_after_ignore" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 1) + self.module.filterwarnings(mode, category=UserWarning) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 3) + self.assertEqual(w[-1].message.args[0], message) def test_default(self): with original_warnings.catch_warnings(record=True, @@ -489,7 +494,7 @@ def test_stacklevel(self): warning_tests.inner("spam7", stacklevel=9999) self.assertEqual(os.path.basename(w[-1].filename), - "sys") + "") def test_stacklevel_import(self): # Issue #24305: With stacklevel=2, module-level warnings should work. @@ -498,7 +503,7 @@ def test_stacklevel_import(self): with original_warnings.catch_warnings(record=True, module=self.module) as w: self.module.simplefilter('always') - import test.test_warnings.data.import_warning + import test.test_warnings.data.import_warning # noqa: F401 self.assertEqual(len(w), 1) self.assertEqual(w[0].filename, __file__) @@ -635,6 +640,97 @@ class NonWarningSubclass: self.module.warn('good warning category', MyWarningClass) self.assertIsInstance(cm.warning, Warning) + def check_module_globals(self, module_globals): + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('default') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, UserWarning) + self.assertEqual(str(w[0].message), 'eggs') + + def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('always') + with self.assertRaisesRegex(errtype, re.escape(errmsg)): + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 0) + + def check_module_globals_deprecated(self, module_globals, msg): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('always') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 2) + self.assertEqual(w[0].category, DeprecationWarning) + self.assertEqual(str(w[0].message), msg) + self.assertEqual(w[1].category, UserWarning) + self.assertEqual(str(w[1].message), 'eggs') + + def test_gh86298_no_loader_and_no_spec(self): + self.check_module_globals({'__name__': 'bar'}) + + def test_gh86298_loader_is_none_and_no_spec(self): + self.check_module_globals({'__name__': 'bar', '__loader__': None}) + + def test_gh86298_no_loader_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_loader_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, + '__spec__': types.SimpleNamespace(loader=None)}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_spec_is_none(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec_loader(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_and_spec_loader_disagree(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace(loader=object())}, + 'Module globals; __loader__ != __spec__.loader') + + def test_gh86298_no_loader_and_no_spec_loader(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader', AttributeError) + + def test_gh86298_no_loader_with_spec_loader_okay(self): + self.check_module_globals( + {'__name__': 'bar', + '__spec__': types.SimpleNamespace(loader=object())}) + class CWarnTests(WarnTests, unittest.TestCase): module = c_warnings @@ -883,37 +979,46 @@ def test_issue31285(self): # warn_explicit() should neither raise a SystemError nor cause an # assertion failure, in case the return value of get_source() has a # bad splitlines() method. - def get_bad_loader(splitlines_ret_val): + get_source_called = [] + def get_module_globals(*, splitlines_ret_val): + class BadSource(str): + def splitlines(self): + return splitlines_ret_val + class BadLoader: def get_source(self, fullname): - class BadSource(str): - def splitlines(self): - return splitlines_ret_val + get_source_called.append(splitlines_ret_val) return BadSource('spam') - return BadLoader() + + loader = BadLoader() + spec = importlib.machinery.ModuleSpec('foobar', loader) + return {'__loader__': loader, + '__spec__': spec, + '__name__': 'foobar'} + wmod = self.module with original_warnings.catch_warnings(module=wmod): wmod.filterwarnings('default', category=UserWarning) + linecache.clearcache() with support.captured_stderr() as stderr: wmod.warn_explicit( 'foo', UserWarning, 'bar', 1, - module_globals={'__loader__': get_bad_loader(42), - '__name__': 'foobar'}) + module_globals=get_module_globals(splitlines_ret_val=42)) self.assertIn('UserWarning: foo', stderr.getvalue()) + self.assertEqual(get_source_called, [42]) - show = wmod._showwarnmsg - try: + linecache.clearcache() + with support.swap_attr(wmod, '_showwarnmsg', None): del wmod._showwarnmsg with support.captured_stderr() as stderr: wmod.warn_explicit( 'eggs', UserWarning, 'bar', 1, - module_globals={'__loader__': get_bad_loader([42]), - '__name__': 'foobar'}) + module_globals=get_module_globals(splitlines_ret_val=[42])) self.assertIn('UserWarning: eggs', stderr.getvalue()) - finally: - wmod._showwarnmsg = show + self.assertEqual(get_source_called, [42, [42]]) + linecache.clearcache() @support.cpython_only def test_issue31411(self): @@ -1239,6 +1344,7 @@ def test_comma_separated_warnings(self): self.assertEqual(stdout, b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + @force_not_colorized def test_envvar_and_command_line(self): rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c", "import sys; sys.stdout.write(str(sys.warnoptions))", @@ -1247,6 +1353,7 @@ def test_envvar_and_command_line(self): self.assertEqual(stdout, b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + @force_not_colorized def test_conflicting_envvar_and_command_line(self): rc, stdout, stderr = assert_python_failure("-Werror::DeprecationWarning", "-c", "import sys, warnings; sys.stdout.write(str(sys.warnoptions)); " @@ -1388,7 +1495,7 @@ def test_late_resource_warning(self): # Issue #21925: Emitting a ResourceWarning late during the Python # shutdown must be logged. - expected = b"sys:1: ResourceWarning: unclosed file " + expected = b":0: ResourceWarning: unclosed file " # don't import the warnings module # (_warnings will try to import it) @@ -1679,6 +1786,29 @@ def d(): pass isinstance(cell.cell_contents, deprecated) for cell in d.__closure__ )) + def test_inspect(self): + @deprecated("depr") + def sync(): + pass + + @deprecated("depr") + async def coro(): + pass + + class Cls: + @deprecated("depr") + def sync(self): + pass + + @deprecated("depr") + async def coro(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(sync)) + self.assertTrue(inspect.iscoroutinefunction(coro)) + self.assertFalse(inspect.iscoroutinefunction(Cls.sync)) + self.assertTrue(inspect.iscoroutinefunction(Cls.coro)) + def setUpModule(): py_warnings.onceregistry.clear() c_warnings.onceregistry.clear() diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 4cdd66d3769e0c..4faad6629fe23c 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -10,11 +10,14 @@ import threading import time import random +import textwrap from test import support from test.support import script_helper, ALWAYS_EQ from test.support import gc_collect +from test.support import import_helper from test.support import threading_helper +from test.support import is_wasi, Py_DEBUG # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None @@ -79,7 +82,7 @@ def callback(self, ref): @contextlib.contextmanager -def collect_in_thread(period=0.0001): +def collect_in_thread(period=0.005): """ Ensure GC collections happen in a different thread, at a high frequency. """ @@ -116,6 +119,38 @@ def test_basic_ref(self): del o repr(wr) + @support.cpython_only + def test_ref_repr(self): + obj = C() + ref = weakref.ref(obj) + regex = ( + rf"" + ) + self.assertRegex(repr(ref), regex) + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'') + + # test type with __name__ + class WithName: + @property + def __name__(self): + return "custom_name" + + obj2 = WithName() + ref2 = weakref.ref(obj2) + regex = ( + rf"" + ) + self.assertRegex(repr(ref2), regex) + def test_repr_failure_gh99184(self): class MyConfig(dict): def __getattr__(self, x): @@ -134,7 +169,7 @@ def test_basic_callback(self): @support.cpython_only def test_cfunction(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") create_cfunction = _testcapi.create_cfunction f = create_cfunction() wr = weakref.ref(f) @@ -195,6 +230,22 @@ def check(proxy): self.assertRaises(ReferenceError, bool, ref3) self.assertEqual(self.cbcalled, 2) + @support.cpython_only + def test_proxy_repr(self): + obj = C() + ref = weakref.proxy(obj, self.callback) + regex = ( + rf"" + ) + self.assertRegex(repr(ref), regex) + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'') + def check_basic_ref(self, factory): o = factory() ref = weakref.ref(o) @@ -914,6 +965,7 @@ def test_hashing(self): self.assertEqual(hash(a), hash(42)) self.assertRaises(TypeError, hash, b) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_trashcan_16602(self): # Issue #16602: when a weakref's target was part of a long # deallocation chain, the trashcan mechanism could delay clearing @@ -967,6 +1019,31 @@ def __del__(self): pass del x support.gc_collect() + @support.cpython_only + def test_no_memory_when_clearing(self): + # gh-118331: Make sure we do not raise an exception from the destructor + # when clearing weakrefs if allocating the intermediate tuple fails. + code = textwrap.dedent(""" + import _testcapi + import weakref + + class TestObj: + pass + + def callback(obj): + pass + + obj = TestObj() + # The choice of 50 is arbitrary, but must be large enough to ensure + # the allocation won't be serviced by the free list. + wrs = [weakref.ref(obj, callback) for _ in range(50)] + _testcapi.set_nomemory(0) + del obj + """).strip() + res, _ = script_helper.run_python_until_end("-c", code) + stderr = res.err.decode("ascii", "backslashreplace") + self.assertNotRegex(stderr, "_Py_Dealloc: Deallocator of type 'TestObj'") + class SubclassableWeakrefTestCase(TestBase): @@ -1213,6 +1290,12 @@ class MappingTestCase(TestBase): COUNT = 10 + if support.check_sanitizer(thread=True) and support.Py_GIL_DISABLED: + # Reduce iteration count to get acceptable latency + NUM_THREADED_ITERATIONS = 1000 + else: + NUM_THREADED_ITERATIONS = 100000 + def check_len_cycles(self, dict_type, cons): N = 20 items = [RefCycle() for i in range(N)] @@ -1838,7 +1921,7 @@ def test_make_weak_keyed_dict_repr(self): def test_threaded_weak_valued_setdefault(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): x = d.setdefault(10, RefCycle()) self.assertIsNot(x, None) # we never put None in there! del x @@ -1847,7 +1930,7 @@ def test_threaded_weak_valued_setdefault(self): def test_threaded_weak_valued_pop(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): d[10] = RefCycle() x = d.pop(10, 10) self.assertIsNot(x, None) # we never put None in there! @@ -1858,13 +1941,32 @@ def test_threaded_weak_valued_consistency(self): # WeakValueDictionary when collecting from another thread. d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(200000): + for i in range(2 * self.NUM_THREADED_ITERATIONS): o = RefCycle() d[10] = o # o is still alive, so the dict can't be empty self.assertEqual(len(d), 1) o = None # lose ref + @support.cpython_only + def test_weak_valued_consistency(self): + # A single-threaded, deterministic repro for issue #28427: old keys + # should not remove new values from WeakValueDictionary. This relies on + # an implementation detail of CPython's WeakValueDictionary (its + # underlying dictionary of KeyedRefs) to reproduce the issue. + d = weakref.WeakValueDictionary() + with support.disable_gc(): + d[10] = RefCycle() + # Keep the KeyedRef alive after it's replaced so that GC will invoke + # the callback. + wr = d.data[10] + # Replace the value with something that isn't cyclic garbage + o = RefCycle() + d[10] = o + # Trigger GC, which will invoke the callback for `wr` + gc.collect() + self.assertEqual(len(d), 1) + def check_threaded_weak_dict_copy(self, type_, deepcopy): # `type_` should be either WeakKeyDictionary or WeakValueDictionary. # `deepcopy` should be either True or False. @@ -1927,6 +2029,7 @@ def pop_and_collect(lst): raise exc[0] @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_key_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. @@ -1940,6 +2043,7 @@ def test_threaded_weak_key_dict_deepcopy(self): self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True) @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_value_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 8c074cb28a87e3..4fcbc5c2e59ea3 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -1,15 +1,20 @@ -import webbrowser -import unittest import os -import sys +import re +import shlex import subprocess -from unittest import mock +import sys +import unittest +import webbrowser from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper +from test.support import requires_subprocess +from test.support import threading_helper +from unittest import mock -if not support.has_subprocess_support: - raise unittest.SkipTest("test webserver requires subprocess") +# The webbrowser module uses threading locks +threading_helper.requires_working_threading(module=True) URL = 'https://www.example.com' CMD_NAME = 'test' @@ -24,6 +29,7 @@ def wait(self, seconds=None): return 0 +@requires_subprocess() class CommandTestMixin: def _test(self, meth, *, args=[URL], kw={}, options, arguments): @@ -94,6 +100,15 @@ def test_open_new_tab(self): options=[], arguments=[URL]) + def test_open_bad_new_parameter(self): + with self.assertRaisesRegex(webbrowser.Error, + re.escape("Bad 'new' parameter to open(); " + "expected 0, 1, or 2, got 999")): + self._test('open', + options=[], + arguments=[URL], + kw=dict(new=999)) + class EdgeCommandTest(CommandTestMixin, unittest.TestCase): @@ -201,22 +216,89 @@ class ELinksCommandTest(CommandTestMixin, unittest.TestCase): def test_open(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_with_autoraise_false(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_new(self): self._test('open_new', options=['-remote'], - arguments=['openURL({},new-window)'.format(URL)]) + arguments=[f'openURL({URL},new-window)']) def test_open_new_tab(self): self._test('open_new_tab', options=['-remote'], - arguments=['openURL({},new-tab)'.format(URL)]) + arguments=[f'openURL({URL},new-tab)']) + + +@unittest.skipUnless(sys.platform == "ios", "Test only applicable to iOS") +class IOSBrowserTest(unittest.TestCase): + def _obj_ref(self, *args): + # Construct a string representation of the arguments that can be used + # as a proxy for object instance references + return "|".join(str(a) for a in args) + + @unittest.skipIf(getattr(webbrowser, "objc", None) is None, + "iOS Webbrowser tests require ctypes") + def setUp(self): + # Intercept the objc library. Wrap the calls to get the + # references to classes and selectors to return strings, and + # wrap msgSend to return stringified object references + self.orig_objc = webbrowser.objc + + webbrowser.objc = mock.Mock() + webbrowser.objc.objc_getClass = lambda cls: f"C#{cls.decode()}" + webbrowser.objc.sel_registerName = lambda sel: f"S#{sel.decode()}" + webbrowser.objc.objc_msgSend.side_effect = self._obj_ref + + def tearDown(self): + webbrowser.objc = self.orig_objc + + def _test(self, meth, **kwargs): + # The browser always gets focus, there's no concept of separate browser + # windows, and there's no API-level control over creating a new tab. + # Therefore, all calls to webbrowser are effectively the same. + getattr(webbrowser, meth)(URL, **kwargs) + + # The ObjC String version of the URL is created with UTF-8 encoding + url_string_args = [ + "C#NSString", + "S#stringWithCString:encoding:", + b'https://www.example.com', + 4, + ] + # The NSURL version of the URL is created from that string + url_obj_args = [ + "C#NSURL", + "S#URLWithString:", + self._obj_ref(*url_string_args), + ] + # The openURL call is invoked on the shared application + shared_app_args = ["C#UIApplication", "S#sharedApplication"] + + # Verify that the last call is the one that opens the URL. + webbrowser.objc.objc_msgSend.assert_called_with( + self._obj_ref(*shared_app_args), + "S#openURL:options:completionHandler:", + self._obj_ref(*url_obj_args), + None, + None + ) + + def test_open(self): + self._test('open') + + def test_open_with_autoraise_false(self): + self._test('open', autoraise=False) + + def test_open_new(self): + self._test('open_new') + + def test_open_new_tab(self): + self._test('open_new_tab') class BrowserRegistrationTest(unittest.TestCase): @@ -271,7 +353,6 @@ def test_register_default(self): def test_register_preferred(self): self._check_registration(preferred=True) - @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") def test_no_xdg_settings_on_macOS(self): # On macOS webbrowser should not use xdg-settings to @@ -314,6 +395,10 @@ def test_synthesize(self): webbrowser.register(name, None, webbrowser.GenericBrowser(name)) webbrowser.get(sys.executable) + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: @@ -325,6 +410,10 @@ def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') webbrowser.get() + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment_preferred(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: @@ -344,5 +433,74 @@ def test_environment_preferred(self): self.assertEqual(webbrowser.get().name, sys.executable) -if __name__=='__main__': +class CliTest(unittest.TestCase): + def test_parse_args(self): + for command, url, new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + args = webbrowser.parse_args(shlex.split(command)) + + self.assertEqual(args.url, url) + self.assertEqual(args.new_win, new_win) + + def test_parse_args_error(self): + for command in [ + # Arguments must not both be given + "https://example.com -n -t", + "https://example.com --new-window --new-tab", + "https://example.com -n --new-tab", + "https://example.com --new-window -t", + ]: + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split(command)) + self.assertIn( + 'error: argument -t/--new-tab: not allowed with argument -n/--new-window', + stderr.getvalue(), + ) + + # Ensure ambiguous shortening fails + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split("https://example.com --new")) + self.assertIn( + 'error: ambiguous option: --new could match --new-window, --new-tab', + stderr.getvalue() + ) + + def test_main(self): + for command, expected_url, expected_new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + with ( + mock.patch("webbrowser.open", return_value=None) as mock_open, + mock.patch("builtins.print", return_value=None), + ): + webbrowser.main(shlex.split(command)) + mock_open.assert_called_once_with(expected_url, expected_new_win) + + +if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py new file mode 100644 index 00000000000000..e64208330ad2f9 --- /dev/null +++ b/Lib/test/test_winapi.py @@ -0,0 +1,158 @@ +# Test the Windows-only _winapi module + +import os +import pathlib +import re +import unittest +from test.support import import_helper, os_helper + +_winapi = import_helper.import_module('_winapi', required_on=['win']) + +MAXIMUM_WAIT_OBJECTS = 64 +MAXIMUM_BATCHED_WAIT_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) ** 2 + +class WinAPIBatchedWaitForMultipleObjectsTests(unittest.TestCase): + def _events_waitall_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + + # Ensure no errors raised when all are triggered + for e in evts: + _winapi.SetEvent(e) + try: + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + except TimeoutError: + self.fail("expected wait to complete immediately") + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Replace events with invalid handles to make sure we fail + for i in chosen: + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + evts[i] = old_evt + + + def _events_waitany_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Trigger one by one. They are auto-reset events, so will only trigger once + for i in chosen: + with self.subTest(f"trigger event {i} of {len(evts)}"): + _winapi.SetEvent(evts[i]) + triggered = _winapi.BatchedWaitForMultipleObjects(evts, False, 10000) + self.assertSetEqual(set(triggered), {i}) + + # Trigger all at once. This may require multiple calls + for i in chosen: + _winapi.SetEvent(evts[i]) + triggered = set() + while len(triggered) < len(chosen): + triggered.update(_winapi.BatchedWaitForMultipleObjects(evts, False, 10000)) + self.assertSetEqual(triggered, set(chosen)) + + # Replace events with invalid handles to make sure we fail + for i in chosen: + with self.subTest(f"corrupt event {i} of {len(evts)}"): + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + evts[i] = old_evt + + + def test_few_events_waitall(self): + self._events_waitall_test(16) + + def test_many_events_waitall(self): + self._events_waitall_test(256) + + def test_max_events_waitall(self): + self._events_waitall_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + + def test_few_events_waitany(self): + self._events_waitany_test(16) + + def test_many_events_waitany(self): + self._events_waitany_test(256) + + def test_max_events_waitany(self): + self._events_waitany_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + +class WinAPITests(unittest.TestCase): + def test_getlongpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")).parents[-1] / "PROGRA~1" + if not os.path.isdir(testfn): + raise unittest.SkipTest("require x:\\PROGRA~1 to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetLongPathName(testfn) + + actual = _winapi.GetLongPathName(os.fsdecode(testfn)) + + # Can't assume that PROGRA~1 expands to any particular variation, so + # ensure it matches any one of them. + candidates = set(testfn.parent.glob("Progra*")) + self.assertIn(pathlib.Path(actual), candidates) + + def test_getshortpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")) + if not os.path.isdir(testfn): + raise unittest.SkipTest("require '%ProgramFiles%' to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetShortPathName(testfn) + + actual = _winapi.GetShortPathName(os.fsdecode(testfn)) + + # Should contain "PROGRA~" but we can't predict the number + self.assertIsNotNone(re.match(r".\:\\PROGRA~\d", actual.upper()), actual) + + def test_namedpipe(self): + pipe_name = rf"\\.\pipe\LOCAL\{os_helper.TESTFN}" + + # Pipe does not exist, so this raises + with self.assertRaises(FileNotFoundError): + _winapi.WaitNamedPipe(pipe_name, 0) + + pipe = _winapi.CreateNamedPipe( + pipe_name, + _winapi.PIPE_ACCESS_DUPLEX, + 8, # 8=PIPE_REJECT_REMOTE_CLIENTS + 2, # two instances available + 32, 32, 0, 0) + self.addCleanup(_winapi.CloseHandle, pipe) + + # Pipe instance is available, so this passes + _winapi.WaitNamedPipe(pipe_name, 0) + + with open(pipe_name, 'w+b') as pipe2: + # No instances available, so this times out + # (WinError 121 does not get mapped to TimeoutError) + with self.assertRaises(OSError): + _winapi.WaitNamedPipe(pipe_name, 0) + + _winapi.WriteFile(pipe, b'testdata') + self.assertEqual(b'testdata', pipe2.read(8)) + + self.assertEqual((b'', 0), _winapi.PeekNamedPipe(pipe, 8)[:2]) + pipe2.write(b'testdata') + pipe2.flush() + self.assertEqual((b'testdata', 8), _winapi.PeekNamedPipe(pipe, 8)[:2]) diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index 209e4464e1a5c0..a10d63dfdc9753 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -43,6 +43,9 @@ def test_open_fd(self): self.assertEqual(0, f.fileno()) f.close() # multiple close should not crash f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(1, 'w') @@ -55,6 +58,9 @@ def test_open_fd(self): self.assertEqual(1, f.fileno()) f.close() f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(2, 'w') diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py index a59d0d24f5db48..870ab7bd41d8ce 100644 --- a/Lib/test/test_winsound.py +++ b/Lib/test/test_winsound.py @@ -1,12 +1,13 @@ # Ridiculously simple test of the winsound module for Windows. import functools -import pathlib +import os import time import unittest from test import support from test.support import import_helper +from test.support import os_helper support.requires('audio') @@ -85,13 +86,6 @@ def test_keyword_args(self): safe_MessageBeep(type=winsound.MB_OK) -# A class for testing winsound when the given path resolves -# to bytes rather than str. -class BytesPath(pathlib.WindowsPath): - def __fspath__(self): - return bytes(super().__fspath__(), 'UTF-8') - - class PlaySoundTest(unittest.TestCase): def test_errors(self): @@ -126,7 +120,7 @@ def test_snd_filename(self): def test_snd_filepath(self): fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') - path = pathlib.Path(fn) + path = os_helper.FakePath(fn) safe_PlaySound(path, winsound.SND_FILENAME | winsound.SND_NODEFAULT) def test_snd_filepath_as_bytes(self): @@ -134,7 +128,7 @@ def test_snd_filepath_as_bytes(self): self.assertRaises( TypeError, winsound.PlaySound, - BytesPath(fn), + os_helper.FakePath(os.fsencode(fn)), winsound.SND_FILENAME | winsound.SND_NODEFAULT ) diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index d81902327a7e0a..e3e2de09496728 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -5,6 +5,7 @@ __email__ = "mbland at acm dot org" import sys +import traceback import unittest from collections import deque from contextlib import _GeneratorContextManager, contextmanager, nullcontext @@ -170,7 +171,10 @@ def __exit__(self, *args): def shouldThrow(): ct = EnterThrows() self.foo = None - with ct as self.foo: + # Ruff complains that we're redefining `self.foo` here, + # but the whole point of the test is to check that `self.foo` + # is *not* redefined (because `__enter__` raises) + with ct as self.foo: # ruff: noqa: F811 pass self.assertRaises(RuntimeError, shouldThrow) self.assertEqual(self.foo, None) @@ -251,7 +255,6 @@ def testInlineGeneratorBoundSyntax(self): self.assertAfterWithGeneratorInvariantsNoError(foo) def testInlineGeneratorBoundToExistingVariable(self): - foo = None with mock_contextmanager_generator() as foo: self.assertInWithGeneratorInvariants(foo) self.assertAfterWithGeneratorInvariantsNoError(foo) @@ -716,7 +719,7 @@ def testExceptionInExprList(self): try: with self.Dummy() as a, self.InitRaises(): pass - except: + except RuntimeError: pass self.assertTrue(a.enter_called) self.assertTrue(a.exit_called) @@ -749,5 +752,48 @@ def testEnterReturnsTuple(self): self.assertEqual(10, b1) self.assertEqual(20, b2) + def testExceptionLocation(self): + # The location of an exception raised from + # __init__, __enter__ or __exit__ of a context + # manager should be just the context manager expression, + # pinpointing the precise context manager in case there + # is more than one. + + def init_raises(): + try: + with self.Dummy(), self.InitRaises() as cm, self.Dummy() as d: + pass + except Exception as e: + return e + + def enter_raises(): + try: + with self.EnterRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + def exit_raises(): + try: + with self.ExitRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "self.InitRaises()"), + (enter_raises, "self.EnterRaises()"), + (exit_raises, "self.ExitRaises()"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index bf8c52e646dc18..f667926d1f8ddf 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -14,11 +14,13 @@ def wmi_exec_query(query): # gh-112278: WMI maybe slow response when first call. try: return _wmi.exec_query(query) + except BrokenPipeError: + pass except WindowsError as e: if e.winerror != 258: raise - time.sleep(LOOPBACK_TIMEOUT) - return _wmi.exec_query(query) + time.sleep(LOOPBACK_TIMEOUT) + return _wmi.exec_query(query) class WmiTests(unittest.TestCase): diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index 9316d0ecbcf1ae..b047f7b06f85d3 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -137,7 +137,7 @@ def test_environ(self): def test_request_length(self): out, err = run_amock(data=b"GET " + (b"x" * 65537) + b" HTTP/1.0\n\n") self.assertEqual(out.splitlines()[0], - b"HTTP/1.0 414 Request-URI Too Long") + b"HTTP/1.0 414 URI Too Long") def test_validated_hello(self): out, err = run_amock(validator(hello_app)) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index a435ec7822ea0c..ae06a9cc11855f 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -13,6 +13,7 @@ import operator import os import pickle +import pyexpat import sys import textwrap import types @@ -137,9 +138,9 @@ class ModuleTest(unittest.TestCase): def test_sanity(self): # Import sanity. - from xml.etree import ElementTree - from xml.etree import ElementInclude - from xml.etree import ElementPath + from xml.etree import ElementTree # noqa: F401 + from xml.etree import ElementInclude # noqa: F401 + from xml.etree import ElementPath # noqa: F401 def test_all(self): names = ("xml.etree.ElementTree", "_elementtree") @@ -328,7 +329,7 @@ def test_simpleops(self): self.serialize_check(element, '') # 5 with self.assertRaises(ValueError) as cm: element.remove(subelement) - self.assertIn('not in list', str(cm.exception)) + self.assertEqual(str(cm.exception), 'list.remove(x): x not in list') self.serialize_check(element, '') # 6 element[0:0] = [subelement, subelement, subelement] self.serialize_check(element[1], '') @@ -1457,12 +1458,14 @@ def test_attlist_default(self): class XMLPullParserTest(unittest.TestCase): - def _feed(self, parser, data, chunk_size=None): + def _feed(self, parser, data, chunk_size=None, flush=False): if chunk_size is None: parser.feed(data) else: for i in range(0, len(data), chunk_size): parser.feed(data[i:i+chunk_size]) + if flush: + parser.flush() def assert_events(self, parser, expected, max_events=None): self.assertEqual( @@ -1480,28 +1483,35 @@ def assert_event_tags(self, parser, expected, max_events=None): self.assertEqual([(action, elem.tag) for action, elem in events], expected) - def test_simple_xml(self): - for chunk_size in (None, 1, 5): - with self.subTest(chunk_size=chunk_size): - parser = ET.XMLPullParser() - self.assert_event_tags(parser, []) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, []) - self._feed(parser, - "\n text\n", chunk_size) - self.assert_event_tags(parser, [('end', 'element')]) - self._feed(parser, "texttail\n", chunk_size) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, [ - ('end', 'element'), - ('end', 'empty-element'), - ]) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, [('end', 'root')]) - self.assertIsNone(parser.close()) + def test_simple_xml(self, chunk_size=None, flush=False): + parser = ET.XMLPullParser() + self.assert_event_tags(parser, []) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, []) + self._feed(parser, + "\n text\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'element')]) + self._feed(parser, "texttail\n", chunk_size, flush) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, [ + ('end', 'element'), + ('end', 'empty-element'), + ]) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'root')]) + self.assertIsNone(parser.close()) + + def test_simple_xml_chunk_1(self): + self.test_simple_xml(chunk_size=1, flush=True) + + def test_simple_xml_chunk_5(self): + self.test_simple_xml(chunk_size=5, flush=True) + + def test_simple_xml_chunk_22(self): + self.test_simple_xml(chunk_size=22) def test_feed_while_iterating(self): parser = ET.XMLPullParser() @@ -1697,6 +1707,56 @@ def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + def test_flush_reparse_deferral_enabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) + + def test_flush_reparse_deferral_disabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + if not ET is pyET: + self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled ' + 'methods not available in C') + parser._parser._parser.SetReparseDeferralEnabled(False) + self.assert_event_tags(parser, []) # i.e. no elements started + + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) # # xinclude tests (samples from appendix C of the xinclude specification) @@ -2363,6 +2423,22 @@ def test_39495_treebuilder_start(self): self.assertRaises(TypeError, ET.TreeBuilder().start, "tag") self.assertRaises(TypeError, ET.TreeBuilder().start, "tag", None) + def test_issue123213_correct_extend_exception(self): + # Does not hide the internal exception when extending the element + self.assertRaises(ZeroDivisionError, ET.Element('tag').extend, + (1/0 for i in range(2))) + + # Still raises the TypeError when extending with a non-iterable + self.assertRaises(TypeError, ET.Element('tag').extend, None) + + # Preserves the TypeError message when extending with a generator + def f(): + raise TypeError("mymessage") + + self.assertRaisesRegex( + TypeError, 'mymessage', + ET.Element('tag').extend, (f() for i in range(2))) + # -------------------------------------------------------------------- @@ -3688,6 +3764,22 @@ def test_setslice_negative_steps(self): e[1::-sys.maxsize<<64] = [ET.Element('d')] self.assertEqual(self._subelem_tags(e), ['a0', 'd', 'a2', 'a3']) + def test_issue123213_setslice_exception(self): + e = ET.Element('tag') + # Does not hide the internal exception when assigning to the element + with self.assertRaises(ZeroDivisionError): + e[:1] = (1/0 for i in range(2)) + + # Still raises the TypeError when assigning with a non-iterable + with self.assertRaises(TypeError): + e[:1] = None + + # Preserve the original TypeError message when assigning. + def f(): + raise TypeError("mymessage") + + with self.assertRaisesRegex(TypeError, 'mymessage'): + e[:1] = (f() for i in range(2)) class IOTest(unittest.TestCase): def test_encoding(self): @@ -4028,7 +4120,7 @@ class BoolTest(unittest.TestCase): def test_warning(self): e = ET.fromstring('') msg = ( - r"Testing an element's truth value will raise an exception in " + r"Testing an element's truth value will always return True in " r"future versions. " r"Use specific 'len\(elem\)' or 'elem is not None' test instead.") with self.assertWarnsRegex(DeprecationWarning, msg): diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 6c4b8384a3202e..2803c6d45c27bf 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -308,7 +308,7 @@ def test_get_host_info(self): def test_ssl_presence(self): try: - import ssl + import ssl # noqa: F401 except ImportError: has_ssl = False else: diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index f1c6b2d97621ee..00a5ed6626ddc5 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -265,14 +265,15 @@ def test_write_shebang_to_fileobj(self): zipapp.create_archive(str(target), new_target, interpreter='python2.7') self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) - def test_read_from_pathobj(self): - # Test that we can copy an archive using a pathlib.Path object + def test_read_from_pathlike_obj(self): + # Test that we can copy an archive using a path-like object # for the source. source = self.tmpdir / 'source' source.mkdir() (source / '__main__.py').touch() - target1 = self.tmpdir / 'target1.pyz' - target2 = self.tmpdir / 'target2.pyz' + source = os_helper.FakePath(str(source)) + target1 = os_helper.FakePath(str(self.tmpdir / 'target1.pyz')) + target2 = os_helper.FakePath(str(self.tmpdir / 'target2.pyz')) zipapp.create_archive(source, target1, interpreter='python') zipapp.create_archive(target1, target2, interpreter='python2.7') self.assertEqual(zipapp.get_interpreter(target2), 'python2.7') diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index 7050937738af18..b505dd7c376462 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -20,7 +20,7 @@ class TestComplexity(unittest.TestCase): @pytest.mark.flaky def test_implied_dirs_performance(self): best, others = big_o.big_o( - compose(consume, zipfile.CompleteDirs._implied_dirs), + compose(consume, zipfile._path.CompleteDirs._implied_dirs), lambda size: [ '/'.join(string.ascii_lowercase + str(n)) for n in range(size) ], @@ -43,13 +43,17 @@ def make_zip_path(self, depth=1, width=1) -> zipfile.Path: @classmethod def make_names(cls, width, letters=string.ascii_lowercase): """ + >>> list(TestComplexity.make_names(1)) + ['a'] >>> list(TestComplexity.make_names(2)) ['a', 'b'] >>> list(TestComplexity.make_names(30)) ['aa', 'ab', ..., 'bd'] + >>> list(TestComplexity.make_names(17124)) + ['aaa', 'aab', ..., 'zip'] """ # determine how many products are needed to produce width - n_products = math.ceil(math.log(width, len(letters))) + n_products = max(1, math.ceil(math.log(width, len(letters)))) inputs = (letters,) * n_products combinations = itertools.product(*inputs) names = map(''.join, combinations) @@ -80,7 +84,7 @@ def test_glob_depth(self): max_n=100, min_n=1, ) - assert best <= big_o.complexities.Quadratic + assert best <= big_o.complexities.Linear @pytest.mark.flaky def test_glob_width(self): diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index 171ab6fdb5fc28..aba515536f0c1a 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -3,31 +3,28 @@ import contextlib import pathlib import pickle +import stat import sys +import time import unittest import zipfile +import zipfile._path + +from test.support.os_helper import temp_dir, FakePath from ._functools import compose from ._itertools import Counter from ._test_params import parameterize, Invoked -from test.support.os_helper import temp_dir - class jaraco: class itertools: Counter = Counter -def add_dirs(zf): - """ - Given a writable zip file zf, inject directory entries for - any directories implied by the presence of children. - """ - for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): - zf.writestr(name, b"") - return zf +def _make_link(info: zipfile.ZipInfo): # type: ignore[name-defined] + info.external_attr |= stat.S_IFLNK << 16 def build_alpharep_fixture(): @@ -36,6 +33,7 @@ def build_alpharep_fixture(): . ├── a.txt + ├── n.txt (-> a.txt) ├── b │ ├── c.txt │ ├── d @@ -56,6 +54,7 @@ def build_alpharep_fixture(): - multiple files in a directory (b/c, b/f) - a directory containing only a directory (g/h) - a directory with files of different extensions (j/klm) + - a symlink (n) pointing to (a) "alpha" because it uses alphabet "rep" because it's a representative example @@ -70,13 +69,16 @@ def build_alpharep_fixture(): zf.writestr("j/k.bin", b"content of k") zf.writestr("j/l.baz", b"content of l") zf.writestr("j/m.bar", b"content of m") + zf.writestr("n.txt", b"a.txt") + _make_link(zf.infolist()[-1]) + zf.filename = "alpharep.zip" return zf alpharep_generators = [ Invoked.wrap(build_alpharep_fixture), - Invoked.wrap(compose(add_dirs, build_alpharep_fixture)), + Invoked.wrap(compose(zipfile._path.CompleteDirs.inject, build_alpharep_fixture)), ] pass_alpharep = parameterize(['alpharep'], alpharep_generators) @@ -100,7 +102,7 @@ def zipfile_ondisk(self, alpharep): def test_iterdir_and_types(self, alpharep): root = zipfile.Path(alpharep) assert root.is_dir() - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() assert a.is_file() assert b.is_dir() assert g.is_dir() @@ -120,7 +122,7 @@ def test_is_file_missing(self, alpharep): @pass_alpharep def test_iterdir_on_file(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() with self.assertRaises(ValueError): a.iterdir() @@ -135,7 +137,7 @@ def test_subdir_is_dir(self, alpharep): @pass_alpharep def test_open(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() with a.open(encoding="utf-8") as strm: data = strm.read() self.assertEqual(data, "content of a") @@ -210,11 +212,12 @@ def test_open_write(self): with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: strm.write('text file') - def test_open_extant_directory(self): + @pass_alpharep + def test_open_extant_directory(self, alpharep): """ Attempting to open a directory raises IsADirectoryError. """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + zf = zipfile.Path(alpharep) with self.assertRaises(IsADirectoryError): zf.joinpath('b').open() @@ -226,18 +229,19 @@ def test_open_binary_invalid_args(self, alpharep): with self.assertRaises(ValueError): root.joinpath('a.txt').open('rb', 'utf-8') - def test_open_missing_directory(self): + @pass_alpharep + def test_open_missing_directory(self, alpharep): """ Attempting to open a missing directory raises FileNotFoundError. """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + zf = zipfile.Path(alpharep) with self.assertRaises(FileNotFoundError): zf.joinpath('z').open() @pass_alpharep def test_read(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() assert a.read_text(encoding="utf-8") == "content of a" # Also check positional encoding arg (gh-101144). assert a.read_text("utf-8") == "content of a" @@ -271,13 +275,13 @@ def test_pathlike_construction(self, alpharep): zipfile.Path should be constructable from a path-like object """ zipfile_ondisk = self.zipfile_ondisk(alpharep) - pathlike = pathlib.Path(str(zipfile_ondisk)) + pathlike = FakePath(str(zipfile_ondisk)) zipfile.Path(pathlike) @pass_alpharep def test_traverse_pathlike(self, alpharep): root = zipfile.Path(alpharep) - root / pathlib.Path("a") + root / FakePath("a") @pass_alpharep def test_parent(self, alpharep): @@ -303,7 +307,7 @@ def test_mutability(self, alpharep): reflect that change. """ root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() alpharep.writestr('foo.txt', 'foo') alpharep.writestr('bar/baz.txt', 'baz') assert any(child.name == 'foo.txt' for child in root.iterdir()) @@ -472,6 +476,18 @@ def test_glob_recursive(self, alpharep): assert list(root.glob("**/*.txt")) == list(root.rglob("*.txt")) + @pass_alpharep + def test_glob_dirs(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('b')) == [zipfile.Path(alpharep, "b/")] + assert list(root.glob('b*')) == [zipfile.Path(alpharep, "b/")] + + @pass_alpharep + def test_glob_subdir(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('g/h')) == [zipfile.Path(alpharep, "g/h/")] + assert list(root.glob('g*/h*')) == [zipfile.Path(alpharep, "g/h/")] + @pass_alpharep def test_glob_subdirs(self, alpharep): root = zipfile.Path(alpharep) @@ -520,12 +536,9 @@ def test_eq_hash(self, alpharep): @pass_alpharep def test_is_symlink(self, alpharep): - """ - See python/cpython#82102 for symlink support beyond this object. - """ - root = zipfile.Path(alpharep) - assert not root.is_symlink() + assert not root.joinpath('a.txt').is_symlink() + assert root.joinpath('n.txt').is_symlink() @pass_alpharep def test_relative_to(self, alpharep): @@ -546,12 +559,12 @@ def test_inheritance(self, alpharep): ['alpharep', 'path_type', 'subpath'], itertools.product( alpharep_generators, - [str, pathlib.Path], + [str, FakePath], ['', 'b/'], ), ) def test_pickle(self, alpharep, path_type, subpath): - zipfile_ondisk = path_type(self.zipfile_ondisk(alpharep)) + zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) restored_1 = pickle.loads(saved_1) @@ -578,14 +591,86 @@ def test_getinfo_missing(self, alpharep): with self.assertRaises(KeyError): alpharep.getinfo('does-not-exist') - def test_root_folder_in_zipfile(self): + def test_malformed_paths(self): """ - gh-112795: Some tools or self constructed codes will add '/' folder to - the zip file, this is a strange behavior, but we should support it. + Path should handle malformed paths gracefully. + + Paths with leading slashes are not visible. + + Paths with dots are treated like regular files. """ - in_memory_file = io.BytesIO() - zf = zipfile.ZipFile(in_memory_file, "w") - zf.mkdir('/') - zf.writestr('./a.txt', 'aaa') - tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) - zf.extractall(tmpdir) + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("/one-slash.txt", b"content") + zf.writestr("//two-slash.txt", b"content") + zf.writestr("../parent.txt", b"content") + zf.filename = '' + root = zipfile.Path(zf) + assert list(map(str, root.iterdir())) == ['../'] + assert root.joinpath('..').joinpath('parent.txt').read_bytes() == b'content' + + def test_unsupported_names(self): + """ + Path segments with special characters are readable. + + On some platforms or file systems, characters like + ``:`` and ``?`` are not allowed, but they are valid + in the zip file. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("path?", b"content") + zf.writestr("V: NMS.flac", b"fLaC...") + zf.filename = '' + root = zipfile.Path(zf) + contents = root.iterdir() + assert next(contents).name == 'path?' + assert next(contents).name == 'V: NMS.flac' + assert root.joinpath('V: NMS.flac').read_bytes() == b"fLaC..." + + def test_backslash_not_separator(self): + """ + In a zip file, backslashes are not separators. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr(DirtyZipInfo.for_name("foo\\bar", zf), b"content") + zf.filename = '' + root = zipfile.Path(zf) + (first,) = root.iterdir() + assert not first.is_dir() + assert first.name == 'foo\\bar' + + @pass_alpharep + def test_interface(self, alpharep): + from importlib.resources.abc import Traversable + + zf = zipfile.Path(alpharep) + assert isinstance(zf, Traversable) + + +class DirtyZipInfo(zipfile.ZipInfo): + """ + Bypass name sanitization. + """ + + def __init__(self, filename, *args, **kwargs): + super().__init__(filename, *args, **kwargs) + self.filename = filename + + @classmethod + def for_name(cls, name, archive): + """ + Construct the same way that ZipFile.writestr does. + + TODO: extract this functionality and re-use + """ + self = cls(filename=name, date_time=time.localtime(time.time())[:6]) + self.compress_type = archive.compression + self.compress_level = archive.compresslevel + if self.filename.endswith('/'): # pragma: no cover + self.external_attr = 0o40775 << 16 # drwxrwxr-x + self.external_attr |= 0x10 # MS-DOS directory flag + else: + self.external_attr = 0o600 << 16 # ?rw------- + return self diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 087fa8d65cc336..36f7f542872897 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -4,7 +4,6 @@ import io import itertools import os -import pathlib import posixpath import struct import subprocess @@ -25,7 +24,7 @@ captured_stdout, captured_stderr, requires_subprocess ) from test.support.os_helper import ( - TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count + TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath ) @@ -160,7 +159,7 @@ def test_open(self): self.zip_open_test(f, self.compression) def test_open_with_pathlike(self): - path = pathlib.Path(TESTFN2) + path = FakePath(TESTFN2) self.zip_open_test(path, self.compression) with zipfile.ZipFile(path, "r", self.compression) as zipfp: self.assertIsInstance(zipfp.filename, str) @@ -390,7 +389,6 @@ def test_repr(self): with zipfp.open(fname) as zipopen: r = repr(zipopen) self.assertIn('name=%r' % fname, r) - self.assertIn("mode='r'", r) if self.compression != zipfile.ZIP_STORED: self.assertIn('compress_type=', r) self.assertIn('[closed]', repr(zipopen)) @@ -447,6 +445,27 @@ def write(self, data): self.assertEqual(zipfp.read('file1'), b'data1') self.assertEqual(zipfp.read('file2'), b'data2') + def test_zipextfile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'rb') + self.assertIs(fid.readable(), True) + self.assertIs(fid.writable(), False) + self.assertIs(fid.seekable(), True) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'rb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertRaises(ValueError, fid.readable) + self.assertIs(fid.writable(), False) + self.assertRaises(ValueError, fid.seekable) def tearDown(self): unlink(TESTFN) @@ -578,17 +597,16 @@ def test_write_default_name(self): def test_io_on_closed_zipextfile(self): fname = "somefile.txt" - with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: zipfp.writestr(fname, "bogus") with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: with zipfp.open(fname) as fid: fid.close() + self.assertIs(fid.closed, True) self.assertRaises(ValueError, fid.read) self.assertRaises(ValueError, fid.seek, 0) self.assertRaises(ValueError, fid.tell) - self.assertRaises(ValueError, fid.readable) - self.assertRaises(ValueError, fid.seekable) def test_write_to_readonly(self): """Check that trying to call write() on a readonly ZipFile object @@ -1098,7 +1116,7 @@ def test_force_zip64(self): # Because this is hard to verify by parsing the data as a zip, the raw # bytes are checked to ensure that they line up with the zip spec. # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT - # The relevent sections for this test are: + # The relevant sections for this test are: # - 4.3.7 for local file header # - 4.5.3 for zip64 extra field @@ -1169,7 +1187,7 @@ def test_unseekable_zip_known_filesize(self): # in as a zip, this test looks at the raw bytes created to ensure that # the correct data has been generated. # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT - # The relevent sections for this test are: + # The relevant sections for this test are: # - 4.3.7 for local file header # - 4.3.9 for the data descriptor # - 4.5.3 for zip64 extra field @@ -1285,6 +1303,25 @@ def test_issue44439(self): self.assertEqual(data.write(q), LENGTH) self.assertEqual(zip.getinfo('data').file_size, LENGTH) + def test_zipwritefile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: + with zipfp.open(fname, 'w') as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'wb') + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'wb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + class StoredWriterTests(AbstractWriterTests, unittest.TestCase): compression = zipfile.ZIP_STORED @@ -1487,7 +1524,7 @@ def test_write_pathlike(self): fp.write("print(42)\n") with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + zipfp.writepy(FakePath(os.path.join(TESTFN2, "mod1.py"))) names = zipfp.namelist() self.assertCompiledIn('mod1.py', names) finally: @@ -1545,7 +1582,7 @@ def test_extract_with_target(self): def test_extract_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_with_target(pathlib.Path(extdir)) + self._test_extract_with_target(FakePath(extdir)) def test_extract_all(self): with temp_cwd(): @@ -1580,7 +1617,7 @@ def test_extract_all_with_target(self): def test_extract_all_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_all_with_target(pathlib.Path(extdir)) + self._test_extract_all_with_target(FakePath(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1893,7 +1930,7 @@ def test_is_zip_erroneous_file(self): fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) # - passing a path-like object - self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) + self.assertFalse(zipfile.is_zipfile(FakePath(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -2903,6 +2940,22 @@ def test_bug_6050(self): os.mkdir(os.path.join(TESTFN2, "a")) self.test_extract_dir() + def test_extract_dir_backslash(self): + zfname = findfile("zipdir_backslash.zip", subdir="archivetestdata") + with zipfile.ZipFile(zfname) as zipf: + zipf.extractall(TESTFN2) + if os.name == 'nt': + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a", "b"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a", "b", "c"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d", "e"))) + else: + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a\\b\\c"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "d\\e\\"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "a"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "d"))) + def test_write_dir(self): dirpath = os.path.join(TESTFN2, "x") os.mkdir(dirpath) @@ -2999,6 +3052,17 @@ def test_create_directory_with_write(self): self.assertEqual(set(os.listdir(target)), {"directory", "directory2"}) + def test_root_folder_in_zipfile(self): + """ + gh-112795: Some tools or self constructed codes will add '/' folder to + the zip file, this is a strange behavior, but we should support it. + """ + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.mkdir('/') + zf.writestr('./a.txt', 'aaa') + zf.extractall(TESTFN2) + def tearDown(self): rmtree(TESTFN2) if os.path.exists(TESTFN): @@ -3013,7 +3077,7 @@ def test_from_file(self): self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_file_pathlike(self): - zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + zi = zipfile.ZipInfo.from_file(FakePath(__file__)) self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') self.assertFalse(zi.is_dir()) self.assertEqual(zi.file_size, os.path.getsize(__file__)) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index c12798d221e9b7..1f288c8b45d589 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -1,8 +1,10 @@ import sys import os import marshal +import glob import importlib import importlib.util +import re import struct import time import unittest @@ -50,10 +52,14 @@ def module_path_to_dotted_name(path): TESTMOD = "ziptestmodule" +TESTMOD2 = "ziptestmodule2" +TESTMOD3 = "ziptestmodule3" TESTPACK = "ziptestpackage" TESTPACK2 = "ziptestpackage2" +TESTPACK3 = "ziptestpackage3" TEMP_DIR = os.path.abspath("junk95142") TEMP_ZIP = os.path.abspath("junk95142.zip") +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "zipimport_data") pyc_file = importlib.util.cache_from_source(TESTMOD + '.py') pyc_ext = '.pyc' @@ -92,8 +98,10 @@ def makeTree(self, files, dirName=TEMP_DIR): # defined by files under the directory dirName. self.addCleanup(os_helper.rmtree, dirName) - for name, (mtime, data) in files.items(): - path = os.path.join(dirName, name) + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + path = os.path.join(dirName, *name.split('/')) if path[-1] == os.sep: if not os.path.isdir(path): os.makedirs(path) @@ -104,22 +112,18 @@ def makeTree(self, files, dirName=TEMP_DIR): with open(path, 'wb') as fp: fp.write(data) - def makeZip(self, files, zipName=TEMP_ZIP, **kw): + def makeZip(self, files, zipName=TEMP_ZIP, *, + comment=None, file_comment=None, stuff=None, prefix='', **kw): # Create a zip archive based set of modules/packages - # defined by files in the zip file zipName. If the - # key 'stuff' exists in kw it is prepended to the archive. + # defined by files in the zip file zipName. + # If stuff is not None, it is prepended to the archive. self.addCleanup(os_helper.unlink, zipName) - with ZipFile(zipName, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - z.writestr(zinfo, data) - comment = kw.get("comment", None) + with ZipFile(zipName, "w", compression=self.compression) as z: + self.writeZip(z, files, file_comment=file_comment, prefix=prefix) if comment is not None: z.comment = comment - stuff = kw.get("stuff", None) if stuff is not None: # Prepend 'stuff' to the start of the zipfile with open(zipName, "rb") as f: @@ -128,20 +132,47 @@ def makeZip(self, files, zipName=TEMP_ZIP, **kw): f.write(stuff) f.write(data) + def writeZip(self, z, files, *, file_comment=None, prefix=''): + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + else: + mtime = NOW + name = name.replace(os.sep, '/') + zinfo = ZipInfo(prefix + name, time.localtime(mtime)) + zinfo.compress_type = self.compression + if file_comment is not None: + zinfo.comment = file_comment + if data is None: + zinfo.CRC = 0 + z.mkdir(zinfo) + else: + assert name[-1] != '/' + z.writestr(zinfo, data) + + def getZip64Files(self): + # This is the simplest way to make zipfile generate the zip64 EOCD block + return {f"f{n}.py": test_src for n in range(65537)} + def doTest(self, expected_ext, files, *modules, **kw): + if 'prefix' not in kw: + kw['prefix'] = 'pre/fix/' self.makeZip(files, **kw) + self.doTestWithPreBuiltZip(expected_ext, *modules, **kw) - sys.path.insert(0, TEMP_ZIP) + def doTestWithPreBuiltZip(self, expected_ext, *modules, + call=None, prefix='', **kw): + zip_path = os.path.join(TEMP_ZIP, *prefix.split('/')[:-1]) + sys.path.insert(0, zip_path) mod = importlib.import_module(".".join(modules)) - call = kw.get('call') if call is not None: call(mod) if expected_ext: file = mod.get_file() - self.assertEqual(file, os.path.join(TEMP_ZIP, + self.assertEqual(file, os.path.join(zip_path, *modules) + expected_ext) def testAFakeZlib(self): @@ -167,7 +198,7 @@ def testAFakeZlib(self): self.skipTest('zlib is a builtin module') if "zlib" in sys.modules: del sys.modules["zlib"] - files = {"zlib.py": (NOW, test_src)} + files = {"zlib.py": test_src} try: self.doTest(".py", files, "zlib") except ImportError: @@ -178,16 +209,16 @@ def testAFakeZlib(self): self.fail("expected test to raise ImportError") def testPy(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD) def testPyc(self): - files = {TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testBoth(self): - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testUncheckedHashBasedPyc(self): @@ -220,22 +251,22 @@ def check(mod): self.doTest(None, files, TESTMOD, call=check) def testEmptyPy(self): - files = {TESTMOD + ".py": (NOW, "")} + files = {TESTMOD + ".py": ""} self.doTest(None, files, TESTMOD) def testBadMagic(self): # make pyc magic word invalid, forcing loading from .py badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badmagic_pyc} self.doTest(".py", files, TESTMOD) def testBadMagic2(self): # make pyc magic word invalid, causing an ImportError badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + pyc_ext: badmagic_pyc} try: self.doTest(".py", files, TESTMOD) self.fail("This should not be reached") @@ -248,22 +279,22 @@ def testBadMTime(self): # flip the second bit -- not the first as that one isn't stored in the # .py's mtime in the zip archive. badtime_pyc[11] ^= 0x02 - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badtime_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badtime_pyc} self.doTest(".py", files, TESTMOD) def test2038MTime(self): # Make sure we can handle mtimes larger than what a 32-bit signed number # can hold. twenty_thirty_eight_pyc = make_pyc(test_co, 2**32 - 1, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: twenty_thirty_eight_pyc} self.doTest(".py", files, TESTMOD) def testPackage(self): packdir = TESTPACK + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTMOD) def testSubPackage(self): @@ -271,9 +302,9 @@ def testSubPackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) def testSubNamespacePackage(self): @@ -282,29 +313,104 @@ def testSubNamespacePackage(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep # The first two files are just directory entries (so have no data). - files = {packdir: (NOW, ""), - packdir2: (NOW, ""), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir: None, + packdir2: None, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) + def testPackageExplicitDirectories(self): + # Test explicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.writestr('a/__init__.py', test_src) + z.mkdir('a/b') + z.writestr('a/b/__init__.py', test_src) + z.mkdir('a/b/c') + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testPackageImplicitDirectories(self): + # Test explicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/__init__.py', test_src) + z.writestr('a/b/__init__.py', test_src) + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testNamespacePackageExplicitDirectories(self): + # Test implicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def testNamespacePackageImplicitDirectories(self): + # Test implicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def _testPackage(self, initfile): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'a')) + if initfile is None: + # XXX Should it work? + self.assertRaises(zipimport.ZipImportError, zi.is_package, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_source, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_code, 'b') + else: + self.assertTrue(zi.is_package('b')) + self.assertEqual(zi.get_source('b'), test_src) + self.assertEqual(zi.get_code('b').co_filename, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + + sys.path.insert(0, TEMP_ZIP) + self.assertNotIn('a', sys.modules) + + mod = importlib.import_module(f'a.b') + self.assertIn('a', sys.modules) + self.assertIs(sys.modules['a.b'], mod) + if initfile is None: + self.assertIsNone(mod.__file__) + else: + self.assertEqual(mod.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + self.assertEqual(len(mod.__path__), 1, mod.__path__) + self.assertEqual(mod.__path__[0], os.path.join(TEMP_ZIP, 'a', 'b')) + + mod2 = importlib.import_module(f'a.b.c.d') + self.assertIn('a.b.c', sys.modules) + self.assertIn('a.b.c.d', sys.modules) + self.assertIs(sys.modules['a.b.c.d'], mod2) + self.assertIs(mod.c.d, mod2) + self.assertEqual(mod2.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', 'c', 'd.py')) + def testMixedNamespacePackage(self): # Test implicit namespace packages spread between a # real filesystem and a zip archive. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) @@ -337,8 +443,8 @@ def testMixedNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-3]) @@ -355,13 +461,13 @@ def testMixedNamespacePackage(self): self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testNamespacePackage(self): @@ -369,22 +475,22 @@ def testNamespacePackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip2 = os.path.abspath("path2.zip") self.makeZip(files2, zip2) @@ -413,8 +519,8 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual("path2.zip", mod.__file__.split(os.sep)[-3]) # One level deeper... @@ -429,29 +535,22 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((subpkg, TESTMOD))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testZipImporterMethods(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(zi.archive, TEMP_ZIP) @@ -507,35 +606,26 @@ def testZipImporterMethods(self): def testInvalidateCaches(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Check that the file information remains accurate after reloading zi.invalidate_caches() - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Check that we can detect the new file after invalidating the cache zi.invalidate_caches() - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) spec = zi.find_spec('spam2') self.assertIsNotNone(spec) self.assertIsInstance(spec.loader, zipimport.zipimporter) @@ -549,36 +639,27 @@ def testInvalidateCaches(self): def testInvalidateCachesWithMultipleZipimports(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Zipimporter for the same path. zi2 = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi2._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive to make the cache wrong. - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Invalidate the cache of the first zipimporter. zi.invalidate_caches() # Check that the second zipimporter detects the new file and isn't using a stale cache. - self.assertEqual(zi2._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) spec = zi2.find_spec('spam2') self.assertIsNotNone(spec) self.assertIsInstance(spec.loader, zipimport.zipimporter) @@ -586,16 +667,9 @@ def testInvalidateCachesWithMultipleZipimports(self): def testZipImporterMethodsInSubDirectory(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"eggs" - z.writestr(zinfo, data) + files = {packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"eggs") zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir) self.assertEqual(zi.archive, TEMP_ZIP) @@ -641,17 +715,33 @@ def testZipImporterMethodsInSubDirectory(self): self.assertIsNone(loader.get_source(mod_name)) self.assertEqual(loader.get_filename(mod_name), mod.__file__) - def testGetData(self): + def testGetDataExplicitDirectories(self): self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - z.compression = self.compression - name = "testdata.dat" - data = bytes(x for x in range(256)) - z.writestr(name, data) - - zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(data, zi.get_data(name)) - self.assertIn('zipimporter object', repr(zi)) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def testGetDataImplicitDirectories(self): + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def _testGetData(self): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'ignored')) + pathname = os.path.join('a', 'b', 'c', 'testdata.dat') + data = bytes(range(256)) + self.assertEqual(zi.get_data(pathname), data) + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, pathname)), data) + self.assertEqual(zi.get_data(os.path.join('a', 'b', '')), b'') + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, 'a', 'b', '')), b'') + self.assertRaises(OSError, zi.get_data, os.path.join('a', 'b')) + self.assertRaises(OSError, zi.get_data, os.path.join(TEMP_ZIP, 'a', 'b')) def testImporterAttr(self): src = """if 1: # indent hack @@ -660,9 +750,9 @@ def get_file(): if __loader__.get_data("some.data") != b"some data": raise AssertionError("bad data")\n""" pyc = make_pyc(compile(src, "", "exec"), NOW, len(src)) - files = {TESTMOD + pyc_ext: (NOW, pyc), - "some.data": (NOW, "some data")} - self.doTest(pyc_ext, files, TESTMOD) + files = {TESTMOD + pyc_ext: pyc, + "some.data": "some data"} + self.doTest(pyc_ext, files, TESTMOD, prefix='') def testDefaultOptimizationLevel(self): # zipimport should use the default optimization level (#28131) @@ -670,7 +760,7 @@ def testDefaultOptimizationLevel(self): def test(val): assert(val) return val\n""" - files = {TESTMOD + '.py': (NOW, src)} + files = {TESTMOD + '.py': src} self.makeZip(files) sys.path.insert(0, TEMP_ZIP) mod = importlib.import_module(TESTMOD) @@ -683,7 +773,7 @@ def test(val): def testImport_WithStuff(self): # try importing from a zipfile which contains additional # stuff at the beginning of the file - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"Some Stuff"*31) @@ -691,18 +781,18 @@ def assertModuleSource(self, module): self.assertEqual(inspect.getsource(module), test_src) def testGetSource(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, call=self.assertModuleSource) def testGetCompiledSource(self): pyc = make_pyc(compile(test_src, "", "exec"), NOW, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: pyc} self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource) def runDoctest(self, callback): - files = {TESTMOD + ".py": (NOW, test_src), - "xyz.txt": (NOW, ">>> log.append(True)\n")} + files = {TESTMOD + ".py": test_src, + "xyz.txt": ">>> log.append(True)\n"} self.doTest(".py", files, TESTMOD, call=callback) def doDoctestFile(self, module): @@ -754,29 +844,21 @@ def doTraceback(self, module): raise AssertionError("This ought to be impossible") def testTraceback(self): - files = {TESTMOD + ".py": (NOW, raise_src)} + files = {TESTMOD + ".py": raise_src} self.doTest(None, files, TESTMOD, call=self.doTraceback) @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, "need an unencodable filename") def testUnencodable(self): filename = os_helper.TESTFN_UNENCODABLE + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) spec = zipimport.zipimporter(filename).find_spec(TESTMOD) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) def testBytesPath(self): filename = os_helper.TESTFN + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) zipimport.zipimporter(filename) with self.assertRaises(TypeError): @@ -787,17 +869,143 @@ def testBytesPath(self): zipimport.zipimporter(memoryview(os.fsencode(filename))) def testComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"comment") def testBeginningCruftAndComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"cruft" * 64, comment=b"hi") def testLargestPossibleComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"c" * ((1 << 16) - 1)) + @support.requires_resource('cpu') + def testZip64(self): + files = self.getZip64Files() + self.doTest(".py", files, "f6") + + @support.requires_resource('cpu') + def testZip64CruftAndComment(self): + files = self.getZip64Files() + self.doTest(".py", files, "f65536", comment=b"c" * ((1 << 16) - 1)) + + def testZip64LargeFile(self): + support.requires( + "largefile", + f"test generates files >{0xFFFFFFFF} bytes and takes a long time " + "to run" + ) + + # N.B.: We do a lot of gymnastics below in the ZIP_STORED case to save + # and reconstruct a sparse zip on systems that support sparse files. + # Instead of creating a ~8GB zip file mainly consisting of null bytes + # for every run of the test, we create the zip once and save off the + # non-null portions of the resulting file as data blobs with offsets + # that allow re-creating the zip file sparsely. This drops disk space + # usage to ~9KB for the ZIP_STORED case and drops that test time by ~2 + # orders of magnitude. For the ZIP_DEFLATED case, however, we bite the + # bullet. The resulting zip file is ~8MB of non-null data; so the sparse + # trick doesn't work and would result in that full ~8MB zip data file + # being checked in to source control. + parts_glob = f"sparse-zip64-c{self.compression:d}-0x*.part" + full_parts_glob = os.path.join(TEST_DATA_DIR, parts_glob) + pre_built_zip_parts = glob.glob(full_parts_glob) + + self.addCleanup(os_helper.unlink, TEMP_ZIP) + if not pre_built_zip_parts: + if self.compression != ZIP_STORED: + support.requires( + "cpu", + "test requires a lot of CPU for compression." + ) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, "wb") as f: + f.write(b"data") + f.write(os.linesep.encode()) + f.seek(0xffff_ffff, os.SEEK_CUR) + f.write(os.linesep.encode()) + os.utime(os_helper.TESTFN, (0.0, 0.0)) + with ZipFile( + TEMP_ZIP, + "w", + compression=self.compression, + strict_timestamps=False + ) as z: + z.write(os_helper.TESTFN, "data1") + z.writestr( + ZipInfo("module.py", (1980, 1, 1, 0, 0, 0)), test_src + ) + z.write(os_helper.TESTFN, "data2") + + # This "works" but relies on the zip format having a non-empty + # final page due to the trailing central directory to wind up with + # the correct length file. + def make_sparse_zip_parts(name): + empty_page = b"\0" * 4096 + with open(name, "rb") as f: + part = None + try: + while True: + offset = f.tell() + data = f.read(len(empty_page)) + if not data: + break + if data != empty_page: + if not part: + part_fullname = os.path.join( + TEST_DATA_DIR, + f"sparse-zip64-c{self.compression:d}-" + f"{offset:#011x}.part", + ) + os.makedirs( + os.path.dirname(part_fullname), + exist_ok=True + ) + part = open(part_fullname, "wb") + print("Created", part_fullname) + part.write(data) + else: + if part: + part.close() + part = None + finally: + if part: + part.close() + + if self.compression == ZIP_STORED: + print(f"Creating sparse parts to check in into {TEST_DATA_DIR}:") + make_sparse_zip_parts(TEMP_ZIP) + + else: + def extract_offset(name): + if m := re.search(r"-(0x[0-9a-f]{9})\.part$", name): + return int(m.group(1), base=16) + raise ValueError(f"{name=} does not fit expected pattern.") + offset_parts = [(extract_offset(n), n) for n in pre_built_zip_parts] + with open(TEMP_ZIP, "wb") as f: + for offset, part_fn in sorted(offset_parts): + with open(part_fn, "rb") as part: + f.seek(offset, os.SEEK_SET) + f.write(part.read()) + # Confirm that the reconstructed zip file works and looks right. + with ZipFile(TEMP_ZIP, "r") as z: + self.assertEqual( + z.getinfo("module.py").date_time, (1980, 1, 1, 0, 0, 0) + ) + self.assertEqual( + z.read("module.py"), test_src.encode(), + msg=f"Recreate {full_parts_glob}, unexpected contents." + ) + def assertDataEntry(name): + zinfo = z.getinfo(name) + self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0)) + self.assertGreater(zinfo.file_size, 0xffff_ffff) + assertDataEntry("data1") + assertDataEntry("data2") + + self.doTestWithPreBuiltZip(".py", "module") + @support.requires_zlib() class CompressedZipImportTestCase(UncompressedZipImportTestCase): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 71039d2a8e7ab9..ae8a8c99762313 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -31,7 +31,7 @@ # Retrieve some helpers from other test cases from test.test_doctest import (test_doctest, sample_doctest, sample_doctest_no_doctests, - sample_doctest_no_docstrings) + sample_doctest_no_docstrings, sample_doctest_skip) def _run_object_doctest(obj, module): @@ -110,7 +110,7 @@ def test_doctest_issue4197(self): # The sample doctest files rewritten to include in the zipped version. sample_sources = {} for mod in [sample_doctest, sample_doctest_no_doctests, - sample_doctest_no_docstrings]: + sample_doctest_no_docstrings, sample_doctest_skip]: src = inspect.getsource(mod) src = src.replace("test.test_doctest.test_doctest", "test_zipped_doctest") # Rewrite the module name so that, for example, diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index ef02c64f886f8a..4d97fe56f3a094 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -6,7 +6,7 @@ import pickle import random import sys -from test.support import bigmemtest, _1G, _4G, skip_on_s390x +from test.support import bigmemtest, _1G, _4G, is_s390x zlib = import_helper.import_module('zlib') @@ -33,8 +33,9 @@ def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() -# bpo-46623: On s390x, when a hardware accelerator is used, using different -# ways to compress data with zlib can produce different compressed data. +# bpo-46623: When a hardware accelerator is used (currently only on s390x), +# using different ways to compress data with zlib can produce different +# compressed data. # Simplified test_pair() code: # # def func1(data): @@ -57,8 +58,10 @@ def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): # # zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data # -# Make the assumption that s390x always has an accelerator to simplify the skip -# condition. +# To simplify the skip condition, make the assumption that s390x always has an +# accelerator, and nothing else has it. +HW_ACCELERATED = is_s390x + class VersionTestCase(unittest.TestCase): @@ -223,12 +226,14 @@ def test_keywords(self): bufsize=zlib.DEF_BUF_SIZE), HAMLET_SCENE) - @skip_on_s390x def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 x = zlib.compress(data) - self.assertEqual(zlib.compress(bytearray(data)), x) + # With hardware acceleration, the compressed bytes + # might not be identical. + if not HW_ACCELERATED: + self.assertEqual(zlib.compress(bytearray(data)), x) for ob in x, bytearray(x): self.assertEqual(zlib.decompress(ob), data) @@ -275,7 +280,6 @@ def test_64bit_compress(self, size): class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # Test compression object - @skip_on_s390x def test_pair(self): # straightforward compress/decompress objects datasrc = HAMLET_SCENE * 128 @@ -286,7 +290,10 @@ def test_pair(self): x1 = co.compress(data) x2 = co.flush() self.assertRaises(zlib.error, co.flush) # second flush should not work - self.assertEqual(x1 + x2, datazip) + # With hardware acceleration, the compressed bytes might not + # be identical. + if not HW_ACCELERATED: + self.assertEqual(x1 + x2, datazip) for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))): dco = zlib.decompressobj() y1 = dco.decompress(v1 + v2) @@ -498,20 +505,16 @@ def test_flushes(self): for sync in sync_opt: for level in range(10): - try: + with self.subTest(sync=sync, level=level): obj = zlib.compressobj( level ) a = obj.compress( data[:3000] ) b = obj.flush( sync ) c = obj.compress( data[3000:] ) d = obj.flush() - except: - print("Error for flush mode={}, level={}" - .format(sync, level)) - raise - self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), - data, ("Decompress failed: flush " - "mode=%i, level=%i") % (sync, level)) - del obj + self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), + data, ("Decompress failed: flush " + "mode=%i, level=%i") % (sync, level)) + del obj @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'), 'requires zlib.Z_SYNC_FLUSH') diff --git a/Lib/test/translationdata/argparse/msgids.txt b/Lib/test/translationdata/argparse/msgids.txt new file mode 100644 index 00000000000000..2b012906436e85 --- /dev/null +++ b/Lib/test/translationdata/argparse/msgids.txt @@ -0,0 +1,30 @@ + (default: %(default)s) +%(heading)s: +%(prog)s: error: %(message)s\n +%(prog)s: warning: %(message)s\n +ambiguous option: %(option)s could match %(matches)s +argument "-" with mode %r +argument %(argument_name)s: %(message)s +argument '%(argument_name)s' is deprecated +can't open '%(filename)s': %(error)s +command '%(parser_name)s' is deprecated +expected at least one argument +expected at most one argument +expected one argument +ignored explicit argument %r +invalid %(type)s value: %(value)r +invalid choice: %(value)r (choose from %(choices)s) +invalid choice: %(value)r, maybe you meant %(closest)r? (choose from %(choices)s) +not allowed with argument %s +one of the arguments %s is required +option '%(option)s' is deprecated +options +positional arguments +show program's version number and exit +show this help message and exit +subcommands +the following arguments are required: %s +unexpected option string: %s +unknown parser %(parser_name)r (choices: %(choices)s) +unrecognized arguments: %s +usage: \ No newline at end of file diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b58345a9..e1a1792cb4a867 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/test/typinganndata/ann_module695.py b/Lib/test/typinganndata/ann_module695.py new file mode 100644 index 00000000000000..b6f3b06bd5065f --- /dev/null +++ b/Lib/test/typinganndata/ann_module695.py @@ -0,0 +1,72 @@ +from __future__ import annotations +from typing import Callable + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: *Ts, z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +def nested(): + from types import SimpleNamespace + from typing import get_type_hints + + Eggs = bytes + Spam = memoryview + + + class E[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + return SimpleNamespace( + E=E, + hints_for_E=get_type_hints(E), + hints_for_E_meth=get_type_hints(E.generic_method), + generic_func=generic_function, + hints_for_generic_func=get_type_hints(generic_function) + ) diff --git a/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl deleted file mode 100644 index 410132385bba4d..00000000000000 Binary files a/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl new file mode 100644 index 00000000000000..67e2308717d675 Binary files /dev/null and b/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part new file mode 100644 index 00000000000000..c6beae8e2552d6 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part new file mode 100644 index 00000000000000..74ab03b4648948 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part new file mode 100644 index 00000000000000..9769a404f675d4 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part differ diff --git a/Lib/threading.py b/Lib/threading.py index b6ff00acadd58f..94ea2f08178369 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -36,8 +36,11 @@ _daemon_threads_allowed = _thread.daemon_threads_allowed _allocate_lock = _thread.allocate_lock _LockType = _thread.LockType -_set_sentinel = _thread._set_sentinel +_thread_shutdown = _thread._shutdown +_make_thread_handle = _thread._make_thread_handle +_ThreadHandle = _thread._ThreadHandle get_ident = _thread.get_ident +_get_main_thread_ident = _thread._get_main_thread_ident _is_main_interpreter = _thread._is_main_interpreter try: get_native_id = _thread.get_native_id @@ -333,7 +336,7 @@ def wait(self, timeout=None): awakened or timed out, it re-acquires the lock and returns. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an RLock, it is not released using its @@ -643,7 +646,7 @@ def wait(self, timeout=None): the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). This method returns the internal flag on exit, so it will always return @@ -686,6 +689,8 @@ def __init__(self, parties, action=None, timeout=None): default for all subsequent 'wait()' calls. """ + if parties < 1: + raise ValueError("parties must be > 0") self._cond = Condition(Lock()) self._action = action self._timeout = timeout @@ -847,25 +852,6 @@ def _newname(name_template): _limbo = {} _dangling = WeakSet() -# Set of Thread._tstate_lock locks of non-daemon threads used by _shutdown() -# to wait until all Python thread states get deleted: -# see Thread._set_tstate_lock(). -_shutdown_locks_lock = _allocate_lock() -_shutdown_locks = set() - -def _maintain_shutdown_locks(): - """ - Drop any shutdown locks that don't correspond to running threads anymore. - - Calling this from time to time avoids an ever-growing _shutdown_locks - set when Thread objects are not joined explicitly. See bpo-37788. - - This must be called with _shutdown_locks_lock acquired. - """ - # If a lock was released, the corresponding thread has exited - to_remove = [lock for lock in _shutdown_locks if not lock.locked()] - _shutdown_locks.difference_update(to_remove) - # Main class for threads @@ -930,11 +916,8 @@ class is implemented. self._ident = None if _HAVE_THREAD_NATIVE_ID: self._native_id = None - self._tstate_lock = None - self._join_lock = None - self._handle = None + self._handle = _ThreadHandle() self._started = Event() - self._is_stopped = False self._initialized = True # Copy of sys.stderr used by self._invoke_excepthook() self._stderr = _sys.stderr @@ -948,31 +931,18 @@ def _after_fork(self, new_ident=None): if new_ident is not None: # This thread is alive. self._ident = new_ident - if self._handle is not None: - assert self._handle.ident == new_ident - # bpo-42350: If the fork happens when the thread is already stopped - # (ex: after threading._shutdown() has been called), _tstate_lock - # is None. Do nothing in this case. - if self._tstate_lock is not None: - self._tstate_lock._at_fork_reinit() - self._tstate_lock.acquire() - if self._join_lock is not None: - self._join_lock._at_fork_reinit() + assert self._handle.ident == new_ident else: - # This thread isn't alive after fork: it doesn't have a tstate - # anymore. - self._is_stopped = True - self._tstate_lock = None - self._join_lock = None - self._handle = None + # Otherwise, the thread is dead, Jim. _PyThread_AfterFork() + # already marked our handle done. + pass def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" - self.is_alive() # easy way to get ._is_stopped set when appropriate - if self._is_stopped: + if self._handle.is_done(): status = "stopped" if self._daemonic: status += " daemon" @@ -996,13 +966,12 @@ def start(self): if self._started.is_set(): raise RuntimeError("threads can only be started once") - self._join_lock = _allocate_lock() - with _active_limbo_lock: _limbo[self] = self try: # Start joinable thread - self._handle = _start_joinable_thread(self._bootstrap) + _start_joinable_thread(self._bootstrap, handle=self._handle, + daemon=self.daemon) except Exception: with _active_limbo_lock: del _limbo[self] @@ -1053,23 +1022,9 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() - def _set_tstate_lock(self): - """ - Set a lock object which will be released by the interpreter when - the underlying thread state (see pystate.h) gets deleted. - """ - self._tstate_lock = _set_sentinel() - self._tstate_lock.acquire() - - if not self.daemon: - with _shutdown_locks_lock: - _maintain_shutdown_locks() - _shutdown_locks.add(self._tstate_lock) - def _bootstrap_inner(self): try: self._set_ident() - self._set_tstate_lock() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() self._started.set() @@ -1089,33 +1044,6 @@ def _bootstrap_inner(self): finally: self._delete() - def _stop(self): - # After calling ._stop(), .is_alive() returns False and .join() returns - # immediately. ._tstate_lock must be released before calling ._stop(). - # - # Normal case: C code at the end of the thread's life - # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and - # that's detected by our ._wait_for_tstate_lock(), called by .join() - # and .is_alive(). Any number of threads _may_ call ._stop() - # simultaneously (for example, if multiple threads are blocked in - # .join() calls), and they're not serialized. That's harmless - - # they'll just make redundant rebindings of ._is_stopped and - # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the - # "assert self._is_stopped" in ._wait_for_tstate_lock() always works - # (the assert is executed only if ._tstate_lock is None). - # - # Special case: _main_thread releases ._tstate_lock via this - # module's _shutdown() function. - lock = self._tstate_lock - if lock is not None: - assert not lock.locked() - self._is_stopped = True - self._tstate_lock = None - if not self.daemon: - with _shutdown_locks_lock: - # Remove our lock and other released locks from _shutdown_locks - _maintain_shutdown_locks() - def _delete(self): "Remove current thread from the dict of currently running threads." with _active_limbo_lock: @@ -1133,7 +1061,7 @@ def join(self, timeout=None): or until the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). As join() always returns None, you must call is_alive() after join() to decide whether a timeout happened -- if the thread is still alive, the join() call timed out. @@ -1156,55 +1084,12 @@ def join(self, timeout=None): if self is current_thread(): raise RuntimeError("cannot join current thread") - if timeout is None: - self._wait_for_tstate_lock() - else: - # the behavior of a negative timeout isn't documented, but - # historically .join(timeout=x) for x<0 has acted as if timeout=0 - self._wait_for_tstate_lock(timeout=max(timeout, 0)) - - if self._is_stopped: - self._join_os_thread() + # the behavior of a negative timeout isn't documented, but + # historically .join(timeout=x) for x<0 has acted as if timeout=0 + if timeout is not None: + timeout = max(timeout, 0) - def _join_os_thread(self): - join_lock = self._join_lock - if join_lock is None: - return - with join_lock: - # Calling join() multiple times would raise an exception - # in one of the callers. - if self._handle is not None: - self._handle.join() - self._handle = None - # No need to keep this around - self._join_lock = None - - def _wait_for_tstate_lock(self, block=True, timeout=-1): - # Issue #18808: wait for the thread state to be gone. - # At the end of the thread's life, after all knowledge of the thread - # is removed from C data structures, C code releases our _tstate_lock. - # This method passes its arguments to _tstate_lock.acquire(). - # If the lock is acquired, the C code is done, and self._stop() is - # called. That sets ._is_stopped to True, and ._tstate_lock to None. - lock = self._tstate_lock - if lock is None: - # already determined that the C code is done - assert self._is_stopped - return - - try: - if lock.acquire(block, timeout): - lock.release() - self._stop() - except: - if lock.locked(): - # bpo-45274: lock.acquire() acquired the lock, but the function - # was interrupted with an exception before reaching the - # lock.release(). It can happen if a signal handler raises an - # exception, like CTRL+C which raises KeyboardInterrupt. - lock.release() - self._stop() - raise + self._handle.join(timeout) @property def name(self): @@ -1255,13 +1140,7 @@ def is_alive(self): """ assert self._initialized, "Thread.__init__() not called" - if self._is_stopped or not self._started.is_set(): - return False - self._wait_for_tstate_lock(False) - if not self._is_stopped: - return True - self._join_os_thread() - return False + return self._started.is_set() and not self._handle.is_done() @property def daemon(self): @@ -1470,9 +1349,9 @@ class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) - self._set_tstate_lock() self._started.set() - self._set_ident() + self._ident = _get_main_thread_ident() + self._handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: @@ -1520,17 +1399,15 @@ def __init__(self): daemon=_daemon_threads_allowed()) self._started.set() self._set_ident() + self._handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self _DeleteDummyThreadOnDel(self) - def _stop(self): - pass - def is_alive(self): - if not self._is_stopped and self._started.is_set(): + if not self._handle.is_done() and self._started.is_set(): return True raise RuntimeError("thread is not alive") @@ -1542,7 +1419,6 @@ def _after_fork(self, new_ident=None): self.__class__ = _MainThread self._name = 'MainThread' self._daemonic = False - self._set_tstate_lock() Thread._after_fork(self, new_ident=new_ident) @@ -1641,12 +1517,11 @@ def _shutdown(): """ Wait until the Python thread state of all non-daemon threads get deleted. """ - # Obscure: other threads may be waiting to join _main_thread. That's - # dubious, but some code does it. We can't wait for C code to release - # the main thread's tstate_lock - that won't happen until the interpreter - # is nearly dead. So we release it here. Note that just calling _stop() - # isn't enough: other threads may already be waiting on _tstate_lock. - if _main_thread._is_stopped and _is_main_interpreter(): + # Obscure: other threads may be waiting to join _main_thread. That's + # dubious, but some code does it. We can't wait for it to be marked as done + # normally - that won't happen until the interpreter is nearly dead. So + # mark it done here. + if _main_thread._handle.is_done() and _is_main_interpreter(): # _shutdown() was already called return @@ -1658,42 +1533,11 @@ def _shutdown(): for atexit_call in reversed(_threading_atexits): atexit_call() - # Main thread - if _main_thread.ident == get_ident(): - tlock = _main_thread._tstate_lock - # The main thread isn't finished yet, so its thread state lock can't - # have been released. - assert tlock is not None - if tlock.locked(): - # It should have been released already by - # _PyInterpreterState_SetNotRunningMain(), but there may be - # embedders that aren't calling that yet. - tlock.release() - _main_thread._stop() - else: - # bpo-1596321: _shutdown() must be called in the main thread. - # If the threading module was not imported by the main thread, - # _main_thread is the thread which imported the threading module. - # In this case, ignore _main_thread, similar behavior than for threads - # spawned by C libraries or using _thread.start_new_thread(). - pass - - # Join all non-deamon threads - while True: - with _shutdown_locks_lock: - locks = list(_shutdown_locks) - _shutdown_locks.clear() - - if not locks: - break - - for lock in locks: - # mimic Thread.join() - lock.acquire() - lock.release() - - # new threads can be spawned while we were waiting for the other - # threads to complete + if _is_main_interpreter(): + _main_thread._handle._set_done() + + # Wait for all non-daemon threads to exit. + _thread_shutdown() def main_thread(): @@ -1713,7 +1557,6 @@ def _after_fork(): # Reset _active_limbo_lock, in case we forked while the lock was held # by another (non-forked) thread. http://bugs.python.org/issue874900 global _active_limbo_lock, _main_thread - global _shutdown_locks_lock, _shutdown_locks _active_limbo_lock = RLock() # fork() only copied the current thread; clear references to others. @@ -1729,10 +1572,6 @@ def _after_fork(): _main_thread = current - # reset _shutdown() locks: threads re-register their _tstate_lock below - _shutdown_locks_lock = _allocate_lock() - _shutdown_locks = set() - with _active_limbo_lock: # Dangling thread instances must still have their locks reset, # because someone may join() them. @@ -1749,7 +1588,6 @@ def _after_fork(): else: # All the others are already stopped. thread._after_fork() - thread._stop() _limbo.clear() _active.clear() diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100755 new mode 100644 index 02cfafaf36e5d1..c106e0f67356da --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Tool for measuring execution time of small code snippets. This module avoids a number of common traps for measuring execution diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 2be9da2cfb9299..dd7b3e138f4236 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -41,6 +41,7 @@ import re wantobjects = 1 +_debug = False # set to True to print executed Tcl/Tk commands TkVersion = float(_tkinter.TK_VERSION) TclVersion = float(_tkinter.TCL_VERSION) @@ -69,7 +70,10 @@ def _stringify(value): else: value = '{%s}' % _join(value) else: - value = str(value) + if isinstance(value, bytes): + value = str(value, 'latin1') + else: + value = str(value) if not value: value = '{}' elif _magic_re.search(value): @@ -291,6 +295,8 @@ def __repr__(self): ''.join(' %s=%s' % (k, attrs[k]) for k in keys if k in attrs) ) + __class_getitem__ = classmethod(types.GenericAlias) + _support_default_root = True _default_root = None @@ -411,7 +417,6 @@ def __del__(self): self._tk.globalunsetvar(self._name) if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self._tk.deletecommand(name) self._tclCommands = None @@ -683,7 +688,6 @@ def destroy(self): this widget in the Tcl interpreter.""" if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) self._tclCommands = None @@ -691,7 +695,6 @@ def deletecommand(self, name): """Internal function. Delete the Tcl command provided in NAME.""" - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) try: self._tclCommands.remove(name) @@ -897,6 +900,21 @@ def after_cancel(self, id): pass self.tk.call('after', 'cancel', id) + def after_info(self, id=None): + """Return information about existing event handlers. + + With no argument, return a tuple of the identifiers for all existing + event handlers created by the after and after_idle commands for this + interpreter. If id is supplied, it specifies an existing handler; id + must have been the return value from some previous call to after or + after_idle and it must not have triggered yet or been canceled. If the + id doesn't exist, a TclError is raised. Otherwise, the return value is + a tuple containing (script, type) where script is a reference to the + function to be called by the event handler and type is either 'idle' + or 'timer' to indicate what kind of event handler it is. + """ + return self.tk.splitlist(self.tk.call('after', 'info', id)) + def bell(self, displayof=0): """Ring a display's bell.""" self.tk.call(('bell',) + self._displayof(displayof)) @@ -1711,6 +1729,9 @@ def getint_event(s): except (ValueError, TclError): return s + if any(isinstance(s, tuple) for s in args): + args = [s[0] if isinstance(s, tuple) and len(s) == 1 else s + for s in args] nsign, b, f, h, k, s, t, w, x, y, A, E, K, N, W, T, X, Y, D = args # Missing: (a, c, d, m, o, v, B, R) e = Event() @@ -1746,7 +1767,10 @@ def getint_event(s): try: e.type = EventType(T) except ValueError: - e.type = T + try: + e.type = EventType(str(T)) # can be int + except ValueError: + e.type = T try: e.widget = self._nametowidget(W) except KeyError: @@ -2435,6 +2459,8 @@ def __init__(self, screenName=None, baseName=None, className='Tk', baseName = baseName + ext interactive = False self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use) + if _debug: + self.tk.settrace(_print_command) if useTk: self._loadtk() if not sys.flags.ignore_environment: @@ -2521,6 +2547,14 @@ def __getattr__(self, attr): "Delegate attribute access to the interpreter object" return getattr(self.tk, attr) + +def _print_command(cmd, *, file=sys.stderr): + # Print executed Tcl/Tk commands. + assert isinstance(cmd, tuple) + cmd = _join(cmd) + print(cmd, file=file) + + # Ideally, the classes Pack, Place and Grid disappear, the # pack/place/grid methods are defined on the Widget class, and # everybody uses w.pack_whatever(...) instead of Pack.whatever(w, @@ -3166,11 +3200,16 @@ def __init__(self, master=None, cnf={}, **kw): Widget.__init__(self, master, 'checkbutton', cnf, kw) def _setup(self, master, cnf): + # Because Checkbutton defaults to a variable with the same name as + # the widget, Checkbutton default names must be globally unique, + # not just unique within the parent widget. if not cnf.get('name'): global _checkbutton_count name = self.__class__.__name__.lower() _checkbutton_count += 1 - cnf['name'] = f'!{name}{_checkbutton_count}' + # To avoid collisions with ttk.Checkbutton, use the different + # name template. + cnf['name'] = f'!{name}-{_checkbutton_count}' super()._setup(master, cnf) def deselect(self): @@ -3745,7 +3784,7 @@ def compare(self, index1, op, index2): return self.tk.getboolean(self.tk.call( self._w, 'compare', index1, op, index2)) - def count(self, index1, index2, *options): # new in Tk 8.5 + def count(self, index1, index2, *options, return_ints=False): # new in Tk 8.5 """Counts the number of relevant things between the two indices. If INDEX1 is after INDEX2, the result will be a negative number @@ -3753,19 +3792,26 @@ def count(self, index1, index2, *options): # new in Tk 8.5 The actual items which are counted depends on the options given. The result is a tuple of integers, one for the result of each - counting option given, if more than one option is specified, - otherwise it is an integer. Valid counting options are "chars", - "displaychars", "displayindices", "displaylines", "indices", - "lines", "xpixels" and "ypixels". The default value, if no - option is specified, is "indices". There is an additional possible - option "update", which if given then all subsequent options ensure - that any possible out of date information is recalculated.""" + counting option given, if more than one option is specified or + return_ints is false (default), otherwise it is an integer. + Valid counting options are "chars", "displaychars", + "displayindices", "displaylines", "indices", "lines", "xpixels" + and "ypixels". The default value, if no option is specified, is + "indices". There is an additional possible option "update", + which if given then all subsequent options ensure that any + possible out of date information is recalculated. + """ options = ['-%s' % arg for arg in options] res = self.tk.call(self._w, 'count', *options, index1, index2) if not isinstance(res, int): res = self._getints(res) if len(res) == 1: res, = res + if not return_ints: + if not res: + res = None + elif len(options) <= 1: + res = (res,) return res def debug(self, boolean=None): @@ -4251,33 +4297,112 @@ def cget(self, option): def __getitem__(self, key): return self.tk.call(self.name, 'cget', '-' + key) - # XXX copy -from, -to, ...? - def copy(self): - """Return a new PhotoImage with the same image as this widget.""" + def copy(self, *, from_coords=None, zoom=None, subsample=None): + """Return a new PhotoImage with the same image as this widget. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + """ destImage = PhotoImage(master=self.tk) - self.tk.call(destImage, 'copy', self.name) + destImage.copy_replace(self, from_coords=from_coords, + zoom=zoom, subsample=subsample) return destImage - def zoom(self, x, y=''): + def zoom(self, x, y='', *, from_coords=None): """Return a new PhotoImage with the same image as this widget - but zoom it with a factor of x in the X direction and y in the Y - direction. If y is not given, the default value is the same as x. + but zoom it with a factor of X in the X direction and Y in the Y + direction. If Y is not given, the default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-zoom',x,y) - return destImage + return self.copy(zoom=(x, y), from_coords=from_coords) - def subsample(self, x, y=''): + def subsample(self, x, y='', *, from_coords=None): """Return a new PhotoImage based on the same image as this widget - but use only every Xth or Yth pixel. If y is not given, the - default value is the same as x. + but use only every Xth or Yth pixel. If Y is not given, the + default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-subsample',x,y) - return destImage + return self.copy(subsample=(x, y), from_coords=from_coords) + + def copy_replace(self, sourceImage, *, from_coords=None, to=None, shrink=False, + zoom=None, subsample=None, compositingrule=None): + """Copy a region from the source image (which must be a PhotoImage) to + this image, possibly with pixel zooming and/or subsampling. If no + options are specified, this command copies the whole of the source + image into this image, starting at coordinates (0, 0). + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + The TO option specifies a rectangular sub-region of the destination + image to be affected. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is (x1,y1) plus the size of the source region + (after subsampling and zooming, if specified). If x2 and y2 are + specified, the source region will be replicated if necessary to fill + the destination region in a tiled fashion. + + If SHRINK is true, the size of the destination image should be + reduced, if necessary, so that the region being copied into is at + the bottom-right corner of the image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + + The COMPOSITINGRULE option specifies how transparent pixels in the + source image are combined with the destination image. When a + compositing rule of 'overlay' is set, the old contents of the + destination image are visible, as if the source image were printed + on a piece of transparent film and placed over the top of the + destination. When a compositing rule of 'set' is set, the old + contents of the destination image are discarded and the source image + is used as-is. The default compositing rule is 'overlay'. + """ + options = [] + if from_coords is not None: + options.extend(('-from', *from_coords)) + if to is not None: + options.extend(('-to', *to)) + if shrink: + options.append('-shrink') + if zoom is not None: + if not isinstance(zoom, (tuple, list)): + zoom = (zoom,) + options.extend(('-zoom', *zoom)) + if subsample is not None: + if not isinstance(subsample, (tuple, list)): + subsample = (subsample,) + options.extend(('-subsample', *subsample)) + if compositingrule: + options.extend(('-compositingrule', compositingrule)) + self.tk.call(self.name, 'copy', sourceImage, *options) def get(self, x, y): """Return the color (red, green, blue) of the pixel at X,Y.""" @@ -4292,17 +4417,117 @@ def put(self, data, to=None): to = to[1:] args = args + ('-to',) + tuple(to) self.tk.call(args) - # XXX read - - def write(self, filename, format=None, from_coords=None): - """Write image to file FILENAME in FORMAT starting from - position FROM_COORDS.""" - args = (self.name, 'write', filename) - if format: - args = args + ('-format', format) - if from_coords: - args = args + ('-from',) + tuple(from_coords) - self.tk.call(args) + + def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False): + """Reads image data from the file named FILENAME into the image. + + The FORMAT option specifies the format of the image data in the + file. + + The FROM_COORDS option specifies a rectangular sub-region of the image + file data to be copied to the destination image. It must be a tuple + or a list of 1 to 4 integers (x1, y1, x2, y2). (x1, y1) and + (x2, y2) specify diagonally opposite corners of the rectangle. If + x2 and y2 are not specified, the default value is the bottom-right + corner of the source image. The default, if this option is not + specified, is the whole of the image in the image file. + + The TO option specifies the coordinates of the top-left corner of + the region of the image into which data from filename are to be + read. The default is (0, 0). + + If SHRINK is true, the size of the destination image will be + reduced, if necessary, so that the region into which the image file + data are read is at the bottom-right corner of the image. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if shrink: + options += ('-shrink',) + if to is not None: + options += ('-to', *to) + self.tk.call(self.name, 'read', filename, *options) + + def write(self, filename, format=None, from_coords=None, *, + background=None, grayscale=False): + """Writes image data from the image to a file named FILENAME. + + The FORMAT option specifies the name of the image file format + handler to be used to write the data to the file. If this option + is not given, the format is guessed from the file extension. + + The FROM_COORDS option specifies a rectangular region of the image + to be written to the image file. It must be a tuple or a list of 1 + to 4 integers (x1, y1, x2, y2). If only x1 and y1 are specified, + the region extends from (x1,y1) to the bottom-right corner of the + image. If all four coordinates are given, they specify diagonally + opposite corners of the rectangular region. The default, if this + option is not given, is the whole image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + self.tk.call(self.name, 'write', filename, *options) + + def data(self, format=None, *, from_coords=None, + background=None, grayscale=False): + """Returns image data. + + The FORMAT option specifies the name of the image file format + handler to be used. If this option is not given, this method uses + a format that consists of a tuple (one element per row) of strings + containing space-separated (one element per pixel/column) colors + in “#RRGGBB” format (where RR is a pair of hexadecimal digits for + the red channel, GG for green, and BB for blue). + + The FROM_COORDS option specifies a rectangular region of the image + to be returned. It must be a tuple or a list of 1 to 4 integers + (x1, y1, x2, y2). If only x1 and y1 are specified, the region + extends from (x1,y1) to the bottom-right corner of the image. If + all four coordinates are given, they specify diagonally opposite + corners of the rectangular region, including (x1, y1) and excluding + (x2, y2). The default, if this option is not given, is the whole + image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + data = self.tk.call(self.name, 'data', *options) + if isinstance(data, str): # For wantobjects = 0. + if format is None: + data = self.tk.splitlist(data) + else: + data = bytes(data, 'latin1') + return data def transparency_get(self, x, y): """Return True if the pixel at x,y is transparent.""" diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py index 0f0dc66460f798..6e5b025a9f9d7d 100644 --- a/Lib/tkinter/simpledialog.py +++ b/Lib/tkinter/simpledialog.py @@ -357,7 +357,7 @@ def askinteger(title, prompt, **kw): class _QueryFloat(_QueryDialog): - errormessage = "Not a floating point value." + errormessage = "Not a floating-point value." def getresult(self): return self.getdouble(self.entry.get()) diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index 5ca938a670831a..073b3ae20797c3 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -690,7 +690,10 @@ def current(self, newindex=None): returns the index of the current value in the list of values or -1 if the current value does not appear in the list.""" if newindex is None: - return self.tk.getint(self.tk.call(self._w, "current")) + res = self.tk.call(self._w, "current") + if res == '': + return -1 + return self.tk.getint(res) return self.tk.call(self._w, "current", newindex) @@ -1522,7 +1525,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw): self.label.place(anchor='n' if label_side == 'top' else 's') # update the label as scale or variable changes - self.__tracecb = self._variable.trace_variable('w', self._adjust) + self.__tracecb = self._variable.trace_add('write', self._adjust) self.bind('', self._adjust) self.bind('', self._adjust) @@ -1530,7 +1533,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw): def destroy(self): """Destroy this widget and possibly its associated variable.""" try: - self._variable.trace_vdelete('w', self.__tracecb) + self._variable.trace_remove('write', self.__tracecb) except AttributeError: pass else: diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 0ab1893d42f72f..7ece4e9b70d31b 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -168,6 +168,7 @@ def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 + self.prev_type = None self.encoding = None def add_whitespace(self, start): @@ -183,6 +184,29 @@ def add_whitespace(self, start): if col_offset: self.tokens.append(" " * col_offset) + def escape_brackets(self, token): + characters = [] + consume_until_next_bracket = False + for character in token: + if character == "}": + if consume_until_next_bracket: + consume_until_next_bracket = False + else: + characters.append(character) + if character == "{": + n_backslashes = sum( + 1 for char in _itertools.takewhile( + "\\".__eq__, + characters[-2::-1] + ) + ) + if n_backslashes % 2 == 0 or characters[-1] != "N": + characters.append(character) + else: + consume_until_next_bracket = True + characters.append(character) + return "".join(characters) + def untokenize(self, iterable): it = iter(iterable) indents = [] @@ -214,11 +238,13 @@ def untokenize(self, iterable): startline = False elif tok_type == FSTRING_MIDDLE: if '{' in token or '}' in token: + token = self.escape_brackets(token) + last_line = token.splitlines()[-1] end_line, end_col = end - end = (end_line, end_col + token.count('{') + token.count('}')) - token = re.sub('{', '{{', token) - token = re.sub('}', '}}', token) - + extra_chars = last_line.count("{{") + last_line.count("}}") + end = (end_line, end_col + extra_chars) + elif tok_type in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") self.add_whitespace(start) self.tokens.append(token) @@ -226,6 +252,7 @@ def untokenize(self, iterable): if tok_type in (NEWLINE, NL): self.prev_row += 1 self.prev_col = 0 + self.prev_type = tok_type return "".join(self.tokens) def compat(self, token, iterable): @@ -233,6 +260,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False + in_fstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -251,6 +279,10 @@ def compat(self, token, iterable): else: prevstring = False + if toknum == FSTRING_START: + in_fstring += 1 + elif toknum == FSTRING_END: + in_fstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -263,11 +295,18 @@ def compat(self, token, iterable): toks_append(indents[-1]) startline = False elif toknum == FSTRING_MIDDLE: - if '{' in tokval or '}' in tokval: - tokval = re.sub('{', '{{', tokval) - tokval = re.sub('}', '}}', tokval) + tokval = self.escape_brackets(tokval) + + # Insert a space between two consecutive brackets if we are in an f-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring: + tokval = ' ' + tokval + + # Insert a space between two consecutive f-strings + if toknum in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") toks_append(tokval) + self.prev_type = toknum def untokenize(iterable): @@ -471,7 +510,7 @@ def error(message, filename=None, location=None): sys.exit(1) # Parse the arguments and options - parser = argparse.ArgumentParser(prog='python -m tokenize') + parser = argparse.ArgumentParser() parser.add_argument(dest='filename', nargs='?', metavar='filename.py', help='the file to tokenize; defaults to stdin') diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 45ca7a89630f0e..5671326646ca5a 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -71,7 +71,12 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n # The spec allows converting "\r\n" to "\n", even in string # literals. Let's do so to simplify parsing. - src = s.replace("\r\n", "\n") + try: + src = s.replace("\r\n", "\n") + except (AttributeError, TypeError): + raise TypeError( + f"Expected str object, not '{type(s).__qualname__}'" + ) from None pos = 0 out = Output(NestedDict(), Flags()) header: Key = () diff --git a/Lib/tomllib/_re.py b/Lib/tomllib/_re.py index 994bb7493fd928..9eacefc729544e 100644 --- a/Lib/tomllib/_re.py +++ b/Lib/tomllib/_re.py @@ -84,6 +84,9 @@ def match_to_datetime(match: re.Match) -> datetime | date: return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) +# No need to limit cache size. This is only ever called on input +# that matched RE_DATETIME, so there is an implicit bound of +# 24 (hours) * 60 (minutes) * 2 (offset direction) = 2880. @lru_cache(maxsize=None) def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: sign = 1 if sign_str == "+" else -1 diff --git a/Lib/trace.py b/Lib/trace.py old mode 100755 new mode 100644 index 7886959fa64f68..bb3d34fd8d6550 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # portions copyright 2001, Autonomous Zones Industries, Inc., all rights... # err... reserved and offered to the public under the terms of the # Python 2.2 license. @@ -402,7 +400,7 @@ def __init__(self, count=1, trace=1, countfuncs=0, countcallers=0, @param countfuncs true iff it should just output a list of (filename, modulename, funcname,) for functions that were called at least once; This overrides - `count' and `trace' + 'count' and 'trace' @param ignoremods a list of the names of modules to ignore @param ignoredirs a list of the names of directories to ignore all of the (recursive) contents of @@ -534,7 +532,7 @@ def globaltrace_countfuncs(self, frame, why, arg): def globaltrace_lt(self, frame, why, arg): """Handler for call events. - If the code block being entered is to be ignored, returns `None', + If the code block being entered is to be ignored, returns 'None', else returns self.localtrace. """ if why == 'call': @@ -565,8 +563,12 @@ def localtrace_trace_and_count(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_trace(self, frame, why, arg): @@ -578,8 +580,12 @@ def localtrace_trace(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_count(self, frame, why, arg): diff --git a/Lib/traceback.py b/Lib/traceback.py index d27c7a726d2bb6..f73149271b9bc9 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1,7 +1,5 @@ """Extract, format and print information about Python stack traces.""" -import os -import io import collections.abc import itertools import linecache @@ -9,6 +7,8 @@ import textwrap import warnings from contextlib import suppress +import _colorize +from _colorize import ANSIColors __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', @@ -21,7 +21,6 @@ # Formatting and printing lists of traceback lines. # -_COLORIZE = True def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or @@ -133,40 +132,15 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ BUILTIN_EXCEPTION_LIMIT = object() -def _can_colorize(): - if sys.platform == "win32": - try: - import nt - if not nt._supports_virtual_terminal(): - return False - except (ImportError, AttributeError): - return False - - if os.environ.get("PYTHON_COLORS") == "0": - return False - if os.environ.get("PYTHON_COLORS") == "1": - return True - if "NO_COLOR" in os.environ: - return False - if not _COLORIZE: - return False - if "FORCE_COLOR" in os.environ: - return True - if os.environ.get("TERM") == "dumb": - return False - try: - return os.isatty(sys.stderr.fileno()) - except io.UnsupportedOperation: - return sys.stderr.isatty() def _print_exception_bltin(exc, /): file = sys.stderr if sys.stderr is not None else sys.__stderr__ - colorize = _can_colorize() + colorize = _colorize.can_colorize() return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - chain=True): + chain=True, **kwargs): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -175,12 +149,13 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - return list(te.format(chain=chain)) + return list(te.format(chain=chain, colorize=colorize)) -def format_exception_only(exc, /, value=_sentinel, *, show_group=False): +def format_exception_only(exc, /, value=_sentinel, *, show_group=False, **kwargs): """Format the exception part of a traceback. The return value is a list of strings, each ending in a newline. @@ -195,10 +170,11 @@ def format_exception_only(exc, /, value=_sentinel, *, show_group=False): :exc:`BaseExceptionGroup`, the nested exceptions are included as well, recursively, with indentation relative to their nesting depth. """ + colorize = kwargs.get("colorize", False) if value is _sentinel: value = exc te = TracebackException(type(value), value, None, compact=True) - return list(te.format_exception_only(show_group=show_group)) + return list(te.format_exception_only(show_group=show_group, colorize=colorize)) # -- not official API but folk probably use these two functions. @@ -208,9 +184,9 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= end_char = "\n" if insert_final_newline else "" if colorize: if value is None or not valuestr: - line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}{end_char}" + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}{end_char}" else: - line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}: {_ANSIColors.MAGENTA}{valuestr}{_ANSIColors.RESET}{end_char}" + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}: {ANSIColors.MAGENTA}{valuestr}{ANSIColors.RESET}{end_char}" else: if value is None or not valuestr: line = f"{etype}{end_char}" @@ -218,6 +194,7 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= line = f"{etype}: {valuestr}{end_char}" return line + def _safe_string(value, what, func=str): try: return func(value) @@ -443,13 +420,6 @@ def _get_code_position(code, instruction_index): _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. -class _ANSIColors: - RED = '\x1b[31m' - BOLD_RED = '\x1b[1;31m' - MAGENTA = '\x1b[35m' - BOLD_MAGENTA = '\x1b[1;35m' - GREY = '\x1b[90m' - RESET = '\x1b[0m' class StackSummary(list): """A list of FrameSummary objects, representing a stack of frames.""" @@ -554,15 +524,15 @@ def format_frame_summary(self, frame_summary, **kwargs): filename = "" if colorize: row.append(' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( - _ANSIColors.MAGENTA, + ANSIColors.MAGENTA, filename, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, frame_summary.lineno, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, frame_summary.name, - _ANSIColors.RESET, + ANSIColors.RESET, ) ) else: @@ -607,13 +577,10 @@ def format_frame_summary(self, frame_summary, **kwargs): # attempt to parse for anchors anchors = None + show_carets = False with suppress(Exception): anchors = _extract_caret_anchors_from_line_segment(segment) - - # only use carets if there are anchors or the carets do not span all lines - show_carets = False - if anchors or all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): - show_carets = True + show_carets = self._should_show_carets(start_offset, end_offset, all_lines, anchors) result = [] @@ -689,11 +656,11 @@ def output_line(lineno): for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): caret_group = list(group) if color == "^": - colorized_line_parts.append(_ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) - colorized_carets_parts.append(_ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + colorized_line_parts.append(ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) elif color == "~": - colorized_line_parts.append(_ANSIColors.RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) - colorized_carets_parts.append(_ANSIColors.RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + colorized_line_parts.append(ANSIColors.RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) else: colorized_line_parts.append("".join(char for char, _ in caret_group)) colorized_carets_parts.append("".join(caret for _, caret in caret_group)) @@ -727,6 +694,39 @@ def output_line(lineno): return ''.join(row) + def _should_show_carets(self, start_offset, end_offset, all_lines, anchors): + with suppress(SyntaxError, ImportError): + import ast + tree = ast.parse('\n'.join(all_lines)) + if not tree.body: + return False + statement = tree.body[0] + value = None + def _spawns_full_line(value): + return ( + value.lineno == 1 + and value.end_lineno == len(all_lines) + and value.col_offset == start_offset + and value.end_col_offset == end_offset + ) + match statement: + case ast.Return(value=ast.Call()): + if isinstance(statement.value.func, ast.Name): + value = statement.value + case ast.Assign(value=ast.Call()): + if ( + len(statement.targets) == 1 and + isinstance(statement.targets[0], ast.Name) + ): + value = statement.value + if value is not None and _spawns_full_line(value): + return False + if anchors: + return True + if all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): + return True + return False + def format(self, **kwargs): """Format the stack ready for printing. @@ -1269,12 +1269,12 @@ def _format_syntax_error(self, stype, **kwargs): if self.lineno is not None: if colorize: yield ' File {}"{}"{}, line {}{}{}\n'.format( - _ANSIColors.MAGENTA, + ANSIColors.MAGENTA, self.filename or "", - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, self.lineno, - _ANSIColors.RESET, + ANSIColors.RESET, ) else: yield ' File "{}", line {}\n'.format( @@ -1294,11 +1294,15 @@ def _format_syntax_error(self, stype, **kwargs): yield ' {}\n'.format(ltext) else: offset = self.offset - end_offset = self.end_offset if self.end_offset not in {None, 0} else offset + if self.lineno == self.end_lineno: + end_offset = self.end_offset if self.end_offset not in {None, 0} else offset + else: + end_offset = len(rtext) + 1 + if self.text and offset > len(self.text): - offset = len(self.text) + 1 + offset = len(rtext) + 1 if self.text and end_offset > len(self.text): - end_offset = len(self.text) + 1 + end_offset = len(rtext) + 1 if offset >= end_offset or end_offset < 0: end_offset = offset + 1 @@ -1314,11 +1318,11 @@ def _format_syntax_error(self, stype, **kwargs): # colorize from colno to end_colno ltext = ( ltext[:colno] + - _ANSIColors.BOLD_RED + ltext[colno:end_colno] + _ANSIColors.RESET + + ANSIColors.BOLD_RED + ltext[colno:end_colno] + ANSIColors.RESET + ltext[end_colno:] ) - start_color = _ANSIColors.BOLD_RED - end_color = _ANSIColors.RESET + start_color = ANSIColors.BOLD_RED + end_color = ANSIColors.RESET yield ' {}\n'.format(ltext) yield ' {}{}{}{}\n'.format( "".join(caretspace), @@ -1331,12 +1335,12 @@ def _format_syntax_error(self, stype, **kwargs): msg = self.msg or "" if colorize: yield "{}{}{}: {}{}{}{}\n".format( - _ANSIColors.BOLD_MAGENTA, + ANSIColors.BOLD_MAGENTA, stype, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, msg, - _ANSIColors.RESET, + ANSIColors.RESET, filename_suffix) else: yield "{}: {}{}\n".format(stype, msg, filename_suffix) @@ -1424,7 +1428,7 @@ def format(self, *, chain=True, _ctx=None, **kwargs): f'+---------------- {title} ----------------\n') _ctx.exception_group_depth += 1 if not truncated: - yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx) + yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx, colorize=colorize) else: remaining = num_excs - self.max_group_width plural = 's' if remaining > 1 else '' @@ -1472,12 +1476,23 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): obj = exc_value.obj try: d = dir(obj) + hide_underscored = (wrong_name[:1] != '_') + if hide_underscored and tb is not None: + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + if 'self' in frame.f_locals and frame.f_locals['self'] is obj: + hide_underscored = False + if hide_underscored: + d = [x for x in d if x[:1] != '_'] except Exception: return None elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) d = dir(mod) + if wrong_name[:1] != '_': + d = [x for x in d if x[:1] != '_'] except Exception: return None else: diff --git a/Lib/turtle.py b/Lib/turtle.py index 7bfe81351b0b34..8a5801f2efe625 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -106,6 +106,7 @@ import sys from os.path import isfile, split, join +from pathlib import Path from copy import deepcopy from tkinter import simpledialog @@ -115,7 +116,7 @@ 'clearscreen', 'colormode', 'delay', 'exitonclick', 'getcanvas', 'getshapes', 'listen', 'mainloop', 'mode', 'numinput', 'onkey', 'onkeypress', 'onkeyrelease', 'onscreenclick', 'ontimer', - 'register_shape', 'resetscreen', 'screensize', 'setup', + 'register_shape', 'resetscreen', 'screensize', 'save', 'setup', 'setworldcoordinates', 'textinput', 'title', 'tracer', 'turtles', 'update', 'window_height', 'window_width'] _tg_turtle_functions = ['back', 'backward', 'begin_fill', 'begin_poly', 'bk', @@ -1492,6 +1493,39 @@ def screensize(self, canvwidth=None, canvheight=None, bg=None): """ return self._resize(canvwidth, canvheight, bg) + def save(self, filename, *, overwrite=False): + """Save the drawing as a PostScript file + + Arguments: + filename -- a string, the path of the created file. + Must end with '.ps' or '.eps'. + + Optional arguments: + overwrite -- boolean, if true, then existing files will be overwritten + + Example (for a TurtleScreen instance named screen): + >>> screen.save('my_drawing.eps') + """ + filename = Path(filename) + if not filename.parent.exists(): + raise FileNotFoundError( + f"The directory '{filename.parent}' does not exist." + " Cannot save to it." + ) + if not overwrite and filename.exists(): + raise FileExistsError( + f"The file '{filename}' already exists. To overwrite it use" + " the 'overwrite=True' argument of the save function." + ) + if (ext := filename.suffix) not in {".ps", ".eps"}: + raise ValueError( + f"Unknown file extension: '{ext}'," + " must be one of {'.ps', '.eps'}" + ) + + postscript = self.cv.postscript() + filename.write_text(postscript) + onscreenclick = onclick resetscreen = reset clearscreen = clear @@ -1718,7 +1752,7 @@ def xcor(self): >>> reset() >>> turtle.left(60) >>> turtle.forward(100) - >>> print turtle.xcor() + >>> print(turtle.xcor()) 50.0 """ return self._position[0] @@ -1732,7 +1766,7 @@ def ycor(self): >>> reset() >>> turtle.left(60) >>> turtle.forward(100) - >>> print turtle.ycor() + >>> print(turtle.ycor()) 86.6025403784 """ return self._position[1] @@ -2335,7 +2369,7 @@ def isvisible(self): Example (for a Turtle instance named turtle): >>> turtle.hideturtle() - >>> print turtle.isvisible(): + >>> print(turtle.isvisible()) False """ return self._shown diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py old mode 100755 new mode 100644 index 2ab6c15e2c079e..9c15916fb6672e --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ ---------------------------------------------- turtleDemo - Help @@ -92,13 +90,15 @@ from idlelib.colorizer import ColorDelegator, color_config from idlelib.percolator import Percolator from idlelib.textview import view_text +import turtle from turtledemo import __doc__ as about_turtledemo -import turtle +if sys.platform == 'win32': + from idlelib.util import fix_win_hidpi + fix_win_hidpi() demo_dir = os.path.dirname(os.path.abspath(__file__)) darwin = sys.platform == 'darwin' - STARTUP = 1 READY = 2 RUNNING = 3 @@ -216,7 +216,7 @@ def makeTextFrame(self, root): self.vbar = vbar = Scrollbar(text_frame, name='vbar') vbar['command'] = text.yview - vbar.pack(side=LEFT, fill=Y) + vbar.pack(side=RIGHT, fill=Y) self.hbar = hbar = Scrollbar(text_frame, name='hbar', orient=HORIZONTAL) hbar['command'] = text.xview hbar.pack(side=BOTTOM, fill=X) @@ -292,7 +292,7 @@ def configGUI(self, start, stop, clear, txt="", color="blue"): self.output_lbl.config(text=txt, fg=color) def makeLoadDemoMenu(self, master): - menu = Menu(master) + menu = Menu(master, tearoff=1) # TJR: leave this one. for entry in getExampleEntries(): def load(entry=entry): @@ -302,7 +302,7 @@ def load(entry=entry): return menu def makeFontMenu(self, master): - menu = Menu(master) + menu = Menu(master, tearoff=0) menu.add_command(label="Decrease (C-'-')", command=self.decrease_size, font=menufont) menu.add_command(label="Increase (C-'+')", command=self.increase_size, @@ -317,7 +317,7 @@ def resize(size=size): return menu def makeHelpMenu(self, master): - menu = Menu(master) + menu = Menu(master, tearoff=0) for help_label, help_file in help_entries: def show(help_label=help_label, help_file=help_file): diff --git a/Lib/turtledemo/bytedesign.py b/Lib/turtledemo/bytedesign.py old mode 100755 new mode 100644 index 1b7452b512c6eb..a5d76a6b6ff295 --- a/Lib/turtledemo/bytedesign.py +++ b/Lib/turtledemo/bytedesign.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_bytedesign.py +"""turtledemo/bytedesign.py An example adapted from the example-suite of PythonCard's turtle graphics. diff --git a/Lib/turtledemo/chaos.py b/Lib/turtledemo/chaos.py index 6a45d0d807ef0b..b25f0fa42c901d 100644 --- a/Lib/turtledemo/chaos.py +++ b/Lib/turtledemo/chaos.py @@ -1,9 +1,7 @@ -# File: tdemo_chaos.py -# Author: Gregor Lingl -# Date: 2009-06-24 - -# A demonstration of chaos +"""turtledemo/chaos.py +A demonstration of chaos. +""" from turtle import * N = 80 diff --git a/Lib/turtledemo/clock.py b/Lib/turtledemo/clock.py old mode 100755 new mode 100644 index 9f8585bd11e053..8b639066c4f440 --- a/Lib/turtledemo/clock.py +++ b/Lib/turtledemo/clock.py @@ -1,18 +1,14 @@ -#!/usr/bin/env python3 -# -*- coding: cp1252 -*- -""" turtle-example-suite: - - tdemo_clock.py +"""turtledemo/clock.py Enhanced clock-program, showing date -and time - ------------------------------------ - Press STOP to exit the program! - ------------------------------------ +and time. """ from turtle import * from datetime import datetime +dtfont = "TkFixedFont", 14, "bold" +current_day = None + def jump(distanz, winkel=0): penup() right(winkel) @@ -53,11 +49,23 @@ def clockface(radius): jump(-radius) rt(6) +def display_date_time(): + global current_day + writer.clear() + now = datetime.now() + current_day = now.day + writer.home() + writer.forward(distance=65) + writer.write(wochentag(now), align="center", font=dtfont) + writer.back(distance=150) + writer.write(datum(now), align="center", font=dtfont) + writer.forward(distance=85) + def setup(): global second_hand, minute_hand, hour_hand, writer mode("logo") make_hand_shape("second_hand", 125, 25) - make_hand_shape("minute_hand", 130, 25) + make_hand_shape("minute_hand", 115, 25) make_hand_shape("hour_hand", 90, 25) clockface(160) second_hand = Turtle() @@ -75,10 +83,10 @@ def setup(): hand.speed(0) ht() writer = Turtle() - #writer.mode("logo") writer.ht() writer.pu() writer.bk(85) + display_date_time() def wochentag(t): wochentag = ["Monday", "Tuesday", "Wednesday", @@ -100,18 +108,11 @@ def tick(): stunde = t.hour + minute/60.0 try: tracer(False) # Terminator can occur here - writer.clear() - writer.home() - writer.forward(65) - writer.write(wochentag(t), - align="center", font=("Courier", 14, "bold")) - writer.back(150) - writer.write(datum(t), - align="center", font=("Courier", 14, "bold")) - writer.forward(85) second_hand.setheading(6*sekunde) # or here minute_hand.setheading(6*minute) hour_hand.setheading(30*stunde) + if t.day != current_day: + display_date_time() tracer(True) ontimer(tick, 100) except Terminator: diff --git a/Lib/turtledemo/colormixer.py b/Lib/turtledemo/colormixer.py index 448db83361a649..f66012c8154317 100644 --- a/Lib/turtledemo/colormixer.py +++ b/Lib/turtledemo/colormixer.py @@ -1,5 +1,4 @@ -# colormixer - +"""turtledemo/colormixer.py""" from turtle import Screen, Turtle, mainloop class ColorTurtle(Turtle): diff --git a/Lib/turtledemo/forest.py b/Lib/turtledemo/forest.py old mode 100755 new mode 100644 index 55b7da947d2476..e1fa85a577ffce --- a/Lib/turtledemo/forest.py +++ b/Lib/turtledemo/forest.py @@ -1,15 +1,11 @@ -#!/usr/bin/env python3 -""" turtlegraphics-example-suite: +"""turtledemo/forest.py - tdemo_forest.py +Displays a 'forest' of 3 breadth-first trees, +similar to the one in tree.py. +For further details, see tree.py. -Displays a 'forest' of 3 breadth-first-trees -similar to the one in tree. -For further remarks see tree.py - -This example is a 'breadth-first'-rewrite of -a Logo program written by Erich Neuwirth. See -http://homepage.univie.ac.at/erich.neuwirth/ +This example is a breadth-first rewrite of +a Logo program by Erich Neuwirth. """ from turtle import Turtle, colormode, tracer, mainloop from random import randrange diff --git a/Lib/turtledemo/fractalcurves.py b/Lib/turtledemo/fractalcurves.py old mode 100755 new mode 100644 index 54ade96a0ad05e..2d0a506a4f5b9f --- a/Lib/turtledemo/fractalcurves.py +++ b/Lib/turtledemo/fractalcurves.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_fractalCurves.py +"""turtledemo/fractalcurves.py This program draws two fractal-curve-designs: (1) A hilbert curve (in a box) diff --git a/Lib/turtledemo/lindenmayer.py b/Lib/turtledemo/lindenmayer.py old mode 100755 new mode 100644 index 3925f25da61870..eb309afb9381b1 --- a/Lib/turtledemo/lindenmayer.py +++ b/Lib/turtledemo/lindenmayer.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - xtx_lindenmayer_indian.py +"""turtledemo/lindenmayer.py Each morning women in Tamil Nadu, in southern India, place designs, created by using rice diff --git a/Lib/turtledemo/minimal_hanoi.py b/Lib/turtledemo/minimal_hanoi.py old mode 100755 new mode 100644 index 4a432f2b2908d5..e44330eaaf7f18 --- a/Lib/turtledemo/minimal_hanoi.py +++ b/Lib/turtledemo/minimal_hanoi.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_minimal_hanoi.py +"""turtledemo/minimal_hanoi.py A minimal 'Towers of Hanoi' animation: A tower of 6 discs is transferred from the @@ -13,9 +10,6 @@ Discs are turtles with shape "square", but stretched to rectangles by shapesize() - --------------------------------------- - To exit press STOP button - --------------------------------------- """ from turtle import * diff --git a/Lib/turtledemo/nim.py b/Lib/turtledemo/nim.py index 9ae6cc5c01b903..f87c479714d662 100644 --- a/Lib/turtledemo/nim.py +++ b/Lib/turtledemo/nim.py @@ -1,6 +1,4 @@ -""" turtle-example-suite: - - tdemo_nim.py +"""turtledemo/nim.py Play nim against the computer. The player who takes the last stick is the winner. diff --git a/Lib/turtledemo/paint.py b/Lib/turtledemo/paint.py old mode 100755 new mode 100644 index fc6852a20082f5..780300fb2da9d1 --- a/Lib/turtledemo/paint.py +++ b/Lib/turtledemo/paint.py @@ -1,13 +1,9 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: +"""turtledemo/paint.py - tdemo_paint.py - -A simple event-driven paint program - -- left mouse button moves turtle -- middle mouse button changes color -- right mouse button toggles between pen up +A simple event-driven paint program. +- Left mouse button moves turtle. +- Middle mouse button changes color. +- Right mouse button toggles between pen up (no line drawn when the turtle moves) and pen down (line is drawn). If pen up follows at least two pen-down moves, the polygon that @@ -15,8 +11,6 @@ ------------------------------------------- Play around by clicking into the canvas using all three mouse buttons. - ------------------------------------------- - To exit press STOP button ------------------------------------------- """ from turtle import * diff --git a/Lib/turtledemo/peace.py b/Lib/turtledemo/peace.py old mode 100755 new mode 100644 index e2ba9288d9e42e..d86c94a48a2472 --- a/Lib/turtledemo/peace.py +++ b/Lib/turtledemo/peace.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_peace.py +"""turtledemo/peace.py A simple drawing suitable as a beginner's programming example. Aside from the diff --git a/Lib/turtledemo/penrose.py b/Lib/turtledemo/penrose.py old mode 100755 new mode 100644 index 045722a2286061..ceaefedac24a67 --- a/Lib/turtledemo/penrose.py +++ b/Lib/turtledemo/penrose.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" xturtle-example-suite: - - xtx_kites_and_darts.py +"""turtledemo/penrose.py Constructs two aperiodic penrose-tilings, consisting of kites and darts, by the method @@ -12,7 +9,7 @@ consisting of five darts. For more information see: - http://en.wikipedia.org/wiki/Penrose_tiling + https://en.wikipedia.org/wiki/Penrose_tiling ------------------------------------------- """ from turtle import * diff --git a/Lib/turtledemo/planet_and_moon.py b/Lib/turtledemo/planet_and_moon.py old mode 100755 new mode 100644 index 021ff99383aa65..571afcf922103f --- a/Lib/turtledemo/planet_and_moon.py +++ b/Lib/turtledemo/planet_and_moon.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_planets_and_moon.py +"""turtledemo/planets_and_moon.py Gravitational system simulation using the approximation method from Feynman-lectures, diff --git a/Lib/turtledemo/rosette.py b/Lib/turtledemo/rosette.py index 47d0f00e9da9d1..48897a620f9d8b 100644 --- a/Lib/turtledemo/rosette.py +++ b/Lib/turtledemo/rosette.py @@ -1,6 +1,4 @@ -""" turtle-example-suite: - - tdemo_wikipedia3.py +"""turtledemo/rosette.py This example is inspired by the Wikipedia article on turtle diff --git a/Lib/turtledemo/round_dance.py b/Lib/turtledemo/round_dance.py index 10383614c6e974..9da6389b213207 100644 --- a/Lib/turtledemo/round_dance.py +++ b/Lib/turtledemo/round_dance.py @@ -1,9 +1,4 @@ -""" turtle-example-suite: - - tdemo_round_dance.py - -(Needs version 1.1 of the turtle module that -comes with Python 3.1) +"""turtledemo/round_dance.py Dancing turtles have a compound shape consisting of a series of triangles of diff --git a/Lib/turtledemo/sorting_animate.py b/Lib/turtledemo/sorting_animate.py old mode 100755 new mode 100644 index d25a0ab6cebdc0..e0a2877cd5d621 --- a/Lib/turtledemo/sorting_animate.py +++ b/Lib/turtledemo/sorting_animate.py @@ -1,19 +1,13 @@ -#!/usr/bin/env python3 -""" - - sorting_animation.py +"""turtledemo/sorting_animation.py A minimal sorting algorithm animation: Sorts a shelf of 10 blocks using insertion sort, selection sort and quicksort. -Shelfs are implemented using builtin lists. +Shelves are implemented using builtin lists. Blocks are turtles with shape "square", but stretched to rectangles by shapesize() - --------------------------------------- - To exit press space button - --------------------------------------- """ from turtle import * import random diff --git a/Lib/turtledemo/tree.py b/Lib/turtledemo/tree.py old mode 100755 new mode 100644 index 98a20da7f15c11..6ad8fcc854a155 --- a/Lib/turtledemo/tree.py +++ b/Lib/turtledemo/tree.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_tree.py +"""turtledemo/tree.py Displays a 'breadth-first-tree' - in contrast to the classical Logo tree drawing programs, diff --git a/Lib/turtledemo/two_canvases.py b/Lib/turtledemo/two_canvases.py index f3602585ab0592..2c8020378edf1b 100644 --- a/Lib/turtledemo/two_canvases.py +++ b/Lib/turtledemo/two_canvases.py @@ -1,4 +1,4 @@ -"""turtledemo.two_canvases +"""turtledemo/two_canvases.py Use TurtleScreen and RawTurtle to draw on two distinct canvases in a separate window. The diff --git a/Lib/turtledemo/yinyang.py b/Lib/turtledemo/yinyang.py old mode 100755 new mode 100644 index 11d1f47cae2549..6e92d4bf739194 --- a/Lib/turtledemo/yinyang.py +++ b/Lib/turtledemo/yinyang.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_yinyang.py +"""turtledemo/yinyang.py Another drawing suitable as a beginner's programming example. diff --git a/Lib/typing.py b/Lib/typing.py index 914ddeaf504cd0..c924c767042552 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -19,6 +19,8 @@ """ from abc import abstractmethod, ABCMeta +import annotationlib +from annotationlib import ForwardRef import collections from collections import defaultdict import collections.abc @@ -27,7 +29,7 @@ import operator import sys import types -from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias +from types import GenericAlias from _typing import ( _idfunc, @@ -38,6 +40,7 @@ ParamSpecKwargs, TypeAliasType, Generic, + NoDefault, ) # Please keep __all__ alphabetized within each category. @@ -63,7 +66,6 @@ # ABCs (from collections.abc). 'AbstractSet', # collections.abc.Set. - 'ByteString', 'Container', 'ContextManager', 'Hashable', @@ -125,6 +127,7 @@ 'cast', 'clear_overloads', 'dataclass_transform', + 'evaluate_forward_ref', 'final', 'get_args', 'get_origin', @@ -138,12 +141,14 @@ 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoDefault', 'NoReturn', 'NotRequired', 'overload', 'override', 'ParamSpecArgs', 'ParamSpecKwargs', + 'ReadOnly', 'Required', 'reveal_type', 'runtime_checkable', @@ -152,6 +157,7 @@ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TypeIs', 'TypeAliasType', 'Unpack', ] @@ -162,7 +168,7 @@ def _type_convert(arg, module=None, *, allow_special_forms=False): if arg is None: return type(None) if isinstance(arg, str): - return ForwardRef(arg, module=module, is_class=allow_special_forms) + return _make_forward_ref(arg, module=module, is_class=allow_special_forms) return arg @@ -236,34 +242,27 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - # When changing this function, don't forget about - # `_collections_abc._type_repr`, which does the same thing - # and must be consistent with this one. - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is ...: - return '...' - if isinstance(obj, types.FunctionType): - return obj.__name__ if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return repr(obj) + return annotationlib.value_to_string(obj) -def _collect_parameters(args): - """Collect all type variables and parameter specifications in args +def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): + """Collect all type parameters in args in order of first appearance (lexicographic order). For example:: >>> P = ParamSpec('P') >>> T = TypeVar('T') - >>> _collect_parameters((T, Callable[P, T])) + >>> _collect_type_parameters((T, Callable[P, T])) (~T, ~P) """ + # required type parameter cannot appear after parameter with default + default_encountered = False + # or after TypeVarTuple + type_var_tuple_encountered = False parameters = [] for t in args: if isinstance(t, type): @@ -273,32 +272,64 @@ def _collect_parameters(args): # `t` might be a tuple, when `ParamSpec` is substituted with # `[T, int]`, or `[int, *Ts]`, etc. for x in t: - for collected in _collect_parameters([x]): + for collected in _collect_type_parameters([x]): if collected not in parameters: parameters.append(collected) elif hasattr(t, '__typing_subst__'): if t not in parameters: + if enforce_default_ordering: + if type_var_tuple_encountered and t.has_default(): + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') + + if t.has_default(): + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') + parameters.append(t) else: + if _is_unpacked_typevartuple(t): + type_var_tuple_encountered = True for x in getattr(t, '__parameters__', ()): if x not in parameters: parameters.append(x) return tuple(parameters) -def _check_generic(cls, parameters, elen): +def _check_generic_specialization(cls, arguments): """Check correct count for parameters of a generic cls (internal helper). This gives a nice error message in case of count mismatch. """ - if not elen: + expected_len = len(cls.__parameters__) + if not expected_len: raise TypeError(f"{cls} is not a generic class") - alen = len(parameters) - if alen != elen: - raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" - f" actual {alen}, expected {elen}") + actual_len = len(arguments) + if actual_len != expected_len: + # deal with defaults + if actual_len < expected_len: + # If the parameter at index `actual_len` in the parameters list + # has a default, then all parameters after it must also have + # one, because we validated as much in _collect_type_parameters(). + # That means that no error needs to be raised here, despite + # the number of arguments being passed not matching the number + # of parameters: all parameters that aren't explicitly + # specialized in this call are parameters with default values. + if cls.__parameters__[actual_len].has_default(): + return + + expected_len -= sum(p.has_default() for p in cls.__parameters__) + expect_val = f"at least {expected_len}" + else: + expect_val = expected_len + + raise TypeError(f"Too {'many' if actual_len > expected_len else 'few'} arguments" + f" for {cls}; actual {actual_len}, expected {expect_val}") -def _unpack_args(args): + +def _unpack_args(*args): newargs = [] for arg in args: subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) @@ -308,19 +339,33 @@ def _unpack_args(args): newargs.append(arg) return newargs -def _deduplicate(params): +def _deduplicate(params, *, unhashable_fallback=False): # Weed out strict duplicates, preserving the first of each occurrence. - all_params = set(params) - if len(all_params) < len(params): - new_params = [] - for t in params: - if t in all_params: - new_params.append(t) - all_params.remove(t) - params = new_params - assert not all_params, all_params - return params - + try: + return dict.fromkeys(params) + except TypeError: + if not unhashable_fallback: + raise + # Happens for cases like `Annotated[dict, {'x': IntValidator()}]` + return _deduplicate_unhashable(params) + +def _deduplicate_unhashable(unhashable_params): + new_unhashable = [] + for t in unhashable_params: + if t not in new_unhashable: + new_unhashable.append(t) + return new_unhashable + +def _compare_args_orderless(first_args, second_args): + first_unhashable = _deduplicate_unhashable(first_args) + second_unhashable = _deduplicate_unhashable(second_args) + t = list(second_unhashable) + try: + for elem in first_unhashable: + t.remove(elem) + except ValueError: + return False + return not t def _remove_dups_flatten(parameters): """Internal helper for Union creation and substitution. @@ -335,7 +380,7 @@ def _remove_dups_flatten(parameters): else: params.append(p) - return tuple(_deduplicate(params)) + return tuple(_deduplicate(params, unhashable_fallback=True)) def _flatten_literal_params(parameters): @@ -383,19 +428,52 @@ def inner(*args, **kwds): return decorator -def _eval_type(t, globalns, localns, recursive_guard=frozenset()): + +def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: + import warnings + + depr_message = ( + f"Failing to pass a value to the 'type_params' parameter " + f"of {funcname!r} is deprecated, as it leads to incorrect behaviour " + f"when calling {funcname} on a stringified annotation " + f"that references a PEP 695 type parameter. " + f"It will be disallowed in Python 3.15." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=3) + + +class _Sentinel: + __slots__ = () + def __repr__(self): + return '' + + +_sentinel = _Sentinel() + + +def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset(), + format=annotationlib.Format.VALUE, owner=None): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing._eval_type") + type_params = () if isinstance(t, ForwardRef): - return t._evaluate(globalns, localns, recursive_guard) + # If the forward_ref has __forward_module__ set, evaluate() infers the globals + # from the module, and it will probably pick better than the globals we have here. + if t.__forward_module__ is not None: + globalns = None + return evaluate_forward_ref(t, globals=globalns, locals=localns, + type_params=type_params, owner=owner, + _recursive_guard=recursive_guard, format=format) if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): if isinstance(t, GenericAlias): args = tuple( - ForwardRef(arg) if isinstance(arg, str) else arg + _make_forward_ref(arg) if isinstance(arg, str) else arg for arg in t.__args__ ) is_unpacked = t.__unpacked__ @@ -405,7 +483,14 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): t = t.__origin__[args] if is_unpacked: t = Unpack[t] - ev_args = tuple(_eval_type(a, globalns, localns, recursive_guard) for a in t.__args__) + + ev_args = tuple( + _eval_type( + a, globalns, localns, type_params, recursive_guard=recursive_guard, + format=format, owner=owner, + ) + for a in t.__args__ + ) if ev_args == t.__args__: return t if isinstance(t, GenericAlias): @@ -524,7 +609,7 @@ class Any(metaclass=_AnyMeta): def __new__(cls, *args, **kwargs): if cls is Any: raise TypeError("Any cannot be instantiated") - return super().__new__(cls, *args, **kwargs) + return super().__new__(cls) @_SpecialForm @@ -639,7 +724,7 @@ class Starship: Note that ClassVar is not a class itself, and should not be used with isinstance() or issubclass(). """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm @@ -661,7 +746,7 @@ class FastConnector(Connection): There is no runtime checking of these properties. """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm @@ -803,46 +888,52 @@ def Concatenate(self, parameters): @_SpecialForm def TypeGuard(self, parameters): - """Special typing construct for marking user-defined type guard functions. + """Special typing construct for marking user-defined type predicate functions. ``TypeGuard`` can be used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. + type predicate function. ``TypeGuard`` only accepts a single type argument. At runtime, functions marked this way should return a boolean. ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard". + conditional expression here is sometimes referred to as a "type predicate". Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeGuard[...]`` or + ``TypeIs[...]`` as its return type to alert static type checkers to + this intention. ``TypeGuard`` should be used over ``TypeIs`` when narrowing + from an incompatible type (e.g., ``list[object]`` to ``list[int]``) or when + the function does not return ``True`` for all instances of the narrowed type. - Using ``-> TypeGuard`` tells the static type checker that for a given - function: + Using ``-> TypeGuard[NarrowedType]`` tells the static type checker that + for a given function: 1. The return value is a boolean. 2. If the return value is ``True``, the type of its argument - is the type inside ``TypeGuard``. + is ``NarrowedType``. + + For example:: - For example:: + def is_str_list(val: list[object]) -> TypeGuard[list[str]]: + '''Determines whether all objects in the list are strings''' + return all(isinstance(x, str) for x in val) - def is_str(val: Union[str, float]): - # "isinstance" type guard - if isinstance(val, str): - # Type of ``val`` is narrowed to ``str`` - ... - else: - # Else, type of ``val`` is narrowed to ``float``. - ... + def func1(val: list[object]): + if is_str_list(val): + # Type of ``val`` is narrowed to ``list[str]``. + print(" ".join(val)) + else: + # Type of ``val`` remains as ``list[object]``. + print("Not a list of strings!") Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower form of ``TypeA`` (it can even be a wider form) and this may lead to type-unsafe results. The main reason is to allow for things like - narrowing ``List[object]`` to ``List[str]`` even though the latter is not - a subtype of the former, since ``List`` is invariant. The responsibility of - writing type-safe type guards is left to the user. + narrowing ``list[object]`` to ``list[str]`` even though the latter is not + a subtype of the former, since ``list`` is invariant. The responsibility of + writing type-safe type predicates is left to the user. ``TypeGuard`` also works with type variables. For more information, see PEP 647 (User-Defined Type Guards). @@ -851,88 +942,146 @@ def is_str(val: Union[str, float]): return _GenericAlias(self, (item,)) -class ForwardRef(_Final, _root=True): - """Internal wrapper to hold a forward reference.""" +@_SpecialForm +def TypeIs(self, parameters): + """Special typing construct for marking user-defined type predicate functions. + + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and accept + at least one argument. + + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type predicate". + + Sometimes it would be convenient to use a user-defined boolean function + as a type predicate. Such a function should use ``TypeIs[...]`` or + ``TypeGuard[...]`` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. - __slots__ = ('__forward_arg__', '__forward_code__', - '__forward_evaluated__', '__forward_value__', - '__forward_is_argument__', '__forward_is_class__', - '__forward_module__') + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for + a given function: - def __init__(self, arg, is_argument=True, module=None, *, is_class=False): - if not isinstance(arg, str): - raise TypeError(f"Forward reference must be a string -- got {arg!r}") + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and + ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. - # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. - # Unfortunately, this isn't a valid expression on its own, so we - # do the unpacking manually. - if arg[0] == '*': - arg_to_compile = f'({arg},)[0]' # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] - else: - arg_to_compile = arg - try: - code = compile(arg_to_compile, '', 'eval') - except SyntaxError: - raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") - - self.__forward_arg__ = arg - self.__forward_code__ = code - self.__forward_evaluated__ = False - self.__forward_value__ = None - self.__forward_is_argument__ = is_argument - self.__forward_is_class__ = is_class - self.__forward_module__ = module - - def _evaluate(self, globalns, localns, recursive_guard): - if self.__forward_arg__ in recursive_guard: - return self - if not self.__forward_evaluated__ or localns is not globalns: - if globalns is None and localns is None: - globalns = localns = {} - elif globalns is None: - globalns = localns - elif localns is None: - localns = globalns - if self.__forward_module__ is not None: - globalns = getattr( - sys.modules.get(self.__forward_module__, None), '__dict__', globalns - ) - type_ = _type_check( - eval(self.__forward_code__, globalns, localns), - "Forward references must evaluate to types.", - is_argument=self.__forward_is_argument__, - allow_special_forms=self.__forward_is_class__, - ) - self.__forward_value__ = _eval_type( - type_, globalns, localns, recursive_guard | {self.__forward_arg__} - ) - self.__forward_evaluated__ = True - return self.__forward_value__ + For example:: - def __eq__(self, other): - if not isinstance(other, ForwardRef): - return NotImplemented - if self.__forward_evaluated__ and other.__forward_evaluated__: - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_value__ == other.__forward_value__) - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_module__ == other.__forward_module__) + from typing import assert_type, final, TypeIs - def __hash__(self): - return hash((self.__forward_arg__, self.__forward_module__)) + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass - def __or__(self, other): - return Union[self, other] + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) - def __ror__(self, other): - return Union[other, self] + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + ``TypeIs`` also works with type variables. For more information, see + PEP 742 (Narrowing types with ``TypeIs``). + """ + item = _type_check(parameters, f'{self} accepts only single type.') + return _GenericAlias(self, (item,)) - def __repr__(self): - if self.__forward_module__ is None: - module_repr = '' + +def _make_forward_ref(code, **kwargs): + forward_ref = ForwardRef(code, **kwargs) + # For compatibility, eagerly compile the forwardref's code. + forward_ref.__forward_code__ + return forward_ref + + +def evaluate_forward_ref( + forward_ref, + *, + owner=None, + globals=None, + locals=None, + type_params=None, + format=annotationlib.Format.VALUE, + _recursive_guard=frozenset(), +): + """Evaluate a forward reference as a type hint. + + This is similar to calling the ForwardRef.evaluate() method, + but unlike that method, evaluate_forward_ref() also: + + * Recursively evaluates forward references nested within the type hint. + * Rejects certain objects that are not valid type hints. + * Replaces type hints that evaluate to None with types.NoneType. + * Supports the *FORWARDREF* and *STRING* formats. + + *forward_ref* must be an instance of ForwardRef. *owner*, if given, + should be the object that holds the annotations that the forward reference + derived from, such as a module, class object, or function. It is used to + infer the namespaces to use for looking up names. *globals* and *locals* + can also be explicitly given to provide the global and local namespaces. + *type_params* is a tuple of type parameters that are in scope when + evaluating the forward reference. This parameter must be provided (though + it may be an empty tuple) if *owner* is not given and the forward reference + does not already have an owner set. *format* specifies the format of the + annotation and is a member of the annotationlib.Format enum. + + """ + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing.evaluate_forward_ref") + type_params = () + if format == annotationlib.Format.STRING: + return forward_ref.__forward_arg__ + if forward_ref.__forward_arg__ in _recursive_guard: + return forward_ref + + try: + value = forward_ref.evaluate(globals=globals, locals=locals, + type_params=type_params, owner=owner) + except NameError: + if format == annotationlib.Format.FORWARDREF: + return forward_ref else: - module_repr = f', module={self.__forward_module__!r}' - return f'ForwardRef({self.__forward_arg__!r}{module_repr})' + raise + + type_ = _type_check( + value, + "Forward references must evaluate to types.", + is_argument=forward_ref.__forward_is_argument__, + allow_special_forms=forward_ref.__forward_is_class__, + ) + return _eval_type( + type_, + globals, + locals, + type_params, + recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, + format=format, + owner=owner, + ) def _is_unpacked_typevartuple(x: Any) -> bool: @@ -980,11 +1129,15 @@ def _typevartuple_prepare_subst(self, alias, args): elif left + right > alen: raise TypeError(f"Too few arguments for {alias};" f" actual {alen}, expected at least {plen-1}") + if left == alen - right and self.has_default(): + replacement = _unpack_args(self.__default__) + else: + replacement = args[left: alen - right] return ( *args[:left], *([fillarg]*(typevartuple_index - left)), - tuple(args[left: alen - right]), + replacement, *([fillarg]*(plen - right - left - typevartuple_index - 1)), *args[alen - right:], ) @@ -1002,6 +1155,8 @@ def _paramspec_subst(self, arg): def _paramspec_prepare_subst(self, alias, args): params = alias.__parameters__ i = params.index(self) + if i == len(args) and self.has_default(): + args = [*args, self.__default__] if i >= len(args): raise TypeError(f"Too few arguments for {alias}") # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. @@ -1015,33 +1170,33 @@ def _paramspec_prepare_subst(self, alias, args): @_tp_cache -def _generic_class_getitem(cls, params): +def _generic_class_getitem(cls, args): """Parameterizes a generic class. At least, parameterizing a generic class is the *main* thing this method does. For example, for some generic class `Foo`, this is called when we - do `Foo[int]` - there, with `cls=Foo` and `params=int`. + do `Foo[int]` - there, with `cls=Foo` and `args=int`. However, note that this method is also called when defining generic classes in the first place with `class Foo(Generic[T]): ...`. """ - if not isinstance(params, tuple): - params = (params,) + if not isinstance(args, tuple): + args = (args,) - params = tuple(_type_convert(p) for p in params) + args = tuple(_type_convert(p) for p in args) is_generic_or_protocol = cls in (Generic, Protocol) if is_generic_or_protocol: # Generic and Protocol can only be subscripted with unique type variables. - if not params: + if not args: raise TypeError( f"Parameter list to {cls.__qualname__}[...] cannot be empty" ) - if not all(_is_typevar_like(p) for p in params): + if not all(_is_typevar_like(p) for p in args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be type variables " f"or parameter specification variables.") - if len(set(params)) != len(params): + if len(set(args)) != len(args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be unique") else: @@ -1049,18 +1204,18 @@ def _generic_class_getitem(cls, params): for param in cls.__parameters__: prepare = getattr(param, '__typing_prepare_subst__', None) if prepare is not None: - params = prepare(cls, params) - _check_generic(cls, params, len(cls.__parameters__)) + args = prepare(cls, args) + _check_generic_specialization(cls, args) new_args = [] - for param, new_arg in zip(cls.__parameters__, params): + for param, new_arg in zip(cls.__parameters__, args): if isinstance(param, TypeVarTuple): new_args.extend(new_arg) else: new_args.append(new_arg) - params = tuple(new_args) + args = tuple(new_args) - return _GenericAlias(cls, params) + return _GenericAlias(cls, args) def _generic_init_subclass(cls, *args, **kwargs): @@ -1075,7 +1230,7 @@ def _generic_init_subclass(cls, *args, **kwargs): if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_parameters(cls.__orig_bases__) + tvars = _collect_type_parameters(cls.__orig_bases__) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. @@ -1176,7 +1331,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', '_defaults'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1223,11 +1378,12 @@ class _GenericAlias(_BaseGenericAlias, _root=True): # A = Callable[[], None] # _CallableGenericAlias # B = Callable[[T], None] # _CallableGenericAlias # C = B[int] # _CallableGenericAlias - # * Parameterized `Final`, `ClassVar` and `TypeGuard`: + # * Parameterized `Final`, `ClassVar`, `TypeGuard`, and `TypeIs`: # # All _GenericAlias # Final[int] # ClassVar[float] - # TypeVar[bool] + # TypeGuard[bool] + # TypeIs[range] def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) @@ -1235,7 +1391,11 @@ def __init__(self, origin, args, *, inst=True, name=None): args = (args,) self.__args__ = tuple(... if a is _TypingEllipsis else a for a in args) - self.__parameters__ = _collect_parameters(args) + enforce_default_ordering = origin in (Generic, Protocol) + self.__parameters__ = _collect_type_parameters( + args, + enforce_default_ordering=enforce_default_ordering, + ) if not name: self.__module__ = origin.__module__ @@ -1280,8 +1440,7 @@ def __getitem__(self, args): # Preprocess `args`. if not isinstance(args, tuple): args = (args,) - args = tuple(_type_convert(p) for p in args) - args = _unpack_args(args) + args = _unpack_args(*(_type_convert(p) for p in args)) new_args = self._determine_new_args(args) r = self.copy_with(new_args) return r @@ -1426,11 +1585,12 @@ def __iter__(self): # parameters are accepted (needs custom __getitem__). class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True): - def __init__(self, origin, nparams, *, inst=True, name=None): + def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): if name is None: name = origin.__name__ super().__init__(origin, inst=inst, name=name) self._nparams = nparams + self._defaults = defaults if origin.__module__ == 'builtins': self.__doc__ = f'A generic version of {origin.__qualname__}.' else: @@ -1442,7 +1602,22 @@ def __getitem__(self, params): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) - _check_generic(self, params, self._nparams) + if (self._defaults + and len(params) < self._nparams + and len(params) + len(self._defaults) >= self._nparams + ): + params = (*params, *self._defaults[len(params) - self._nparams:]) + actual_len = len(params) + + if actual_len != self._nparams: + if self._defaults: + expected = f"at least {self._nparams - len(self._defaults)}" + else: + expected = str(self._nparams) + if not self._nparams: + raise TypeError(f"{self} is not a generic class") + raise TypeError(f"Too {'many' if actual_len > self._nparams else 'few'} arguments for {self};" + f" actual {actual_len}, expected {expected}") return self.copy_with(params) def copy_with(self, params): @@ -1469,21 +1644,6 @@ def __ror__(self, left): return Union[left, self] -class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): - def __init__( - self, origin, nparams, *, removal_version, inst=True, name=None - ): - super().__init__(origin, nparams, inst=inst, name=name) - self._removal_version = removal_version - - def __instancecheck__(self, inst): - import warnings - warnings._deprecated( - f"{self.__module__}.{self._name}", remove=self._removal_version - ) - return super().__instancecheck__(inst) - - class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' @@ -1555,7 +1715,10 @@ def copy_with(self, params): def __eq__(self, other): if not isinstance(other, (_UnionGenericAlias, types.UnionType)): return NotImplemented - return set(self.__args__) == set(other.__args__) + try: # fast path + return set(self.__args__) == set(other.__args__) + except TypeError: # not hashable, slow path + return _compare_args_orderless(self.__args__, other.__args__) def __hash__(self): return hash(frozenset(self.__args__)) @@ -1673,8 +1836,9 @@ def __typing_unpacked_tuple_args__(self): assert self.__origin__ is Unpack assert len(self.__args__) == 1 arg, = self.__args__ - if isinstance(arg, _GenericAlias): - assert arg.__origin__ is tuple + if isinstance(arg, (_GenericAlias, types.GenericAlias)): + if arg.__origin__ is not tuple: + raise TypeError("Unpack[...] must be used with a tuple type") return arg.__args__ return None @@ -1699,7 +1863,8 @@ class _TypingEllipsis: '__abstractmethods__', '__annotations__', '__dict__', '__doc__', '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', - '__match_args__', + '__match_args__', '__static_attributes__', '__firstlineno__', + '__annotate__', }) # These special attributes will be not collected as protocol members. @@ -1997,7 +2162,7 @@ class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True): """Runtime representation of an annotated type. At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't' - with extra annotations. The alias behaves like a normal typing alias. + with extra metadata. The alias behaves like a normal typing alias. Instantiating is the same as instantiating the underlying type; binding it to types is also the same. @@ -2176,12 +2341,8 @@ def greet(name: str) -> None: return val -_allowed_types = (types.FunctionType, types.BuiltinFunctionType, - types.MethodType, types.ModuleType, - WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) - - -def get_type_hints(obj, globalns=None, localns=None, include_extras=False): +def get_type_hints(obj, globalns=None, localns=None, include_extras=False, + *, format=annotationlib.Format.VALUE): """Return type hints for an object. This is often the same as obj.__annotations__, but it handles @@ -2218,13 +2379,14 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): if isinstance(obj, type): hints = {} for base in reversed(obj.__mro__): + ann = annotationlib.get_annotations(base, format=format) + if format is annotationlib.Format.STRING: + hints.update(ann) + continue if globalns is None: base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) - if isinstance(ann, types.GetSetDescriptorType): - ann = {} base_locals = dict(vars(base)) if localns is None else localns if localns is None and globalns is None: # This is surprising, but required. Before Python 3.10, @@ -2238,10 +2400,26 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): if value is None: value = type(None) if isinstance(value, str): - value = ForwardRef(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals) + value = _make_forward_ref(value, is_argument=False, is_class=True) + value = _eval_type(value, base_globals, base_locals, base.__type_params__, + format=format, owner=obj) hints[name] = value - return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} + if include_extras or format is annotationlib.Format.STRING: + return hints + else: + return {k: _strip_annotations(t) for k, t in hints.items()} + + hints = annotationlib.get_annotations(obj, format=format) + if ( + not hints + and not isinstance(obj, types.ModuleType) + and not callable(obj) + and not hasattr(obj, '__annotations__') + and not hasattr(obj, '__annotate__') + ): + raise TypeError(f"{obj!r} is not a module, class, or callable.") + if format is annotationlib.Format.STRING: + return hints if globalns is None: if isinstance(obj, types.ModuleType): @@ -2256,27 +2434,19 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): localns = globalns elif localns is None: localns = globalns - hints = getattr(obj, '__annotations__', None) - if hints is None: - # Return empty annotations for something that _could_ have them. - if isinstance(obj, _allowed_types): - return {} - else: - raise TypeError('{!r} is not a module, class, method, ' - 'or function.'.format(obj)) - hints = dict(hints) + type_params = getattr(obj, "__type_params__", ()) for name, value in hints.items(): if value is None: value = type(None) if isinstance(value, str): # class-level forward refs were handled above, this must be either # a module-level annotation or a function argument annotation - value = ForwardRef( + value = _make_forward_ref( value, is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - hints[name] = _eval_type(value, globalns, localns) + hints[name] = _eval_type(value, globalns, localns, type_params, format=format, owner=obj) return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} @@ -2284,7 +2454,7 @@ def _strip_annotations(t): """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): return _strip_annotations(t.__origin__) - if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired): + if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired, ReadOnly): return _strip_annotations(t.__args__[0]) if isinstance(t, _GenericAlias): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) @@ -2622,9 +2792,6 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) -ByteString = _DeprecatedGenericAlias( - collections.abc.ByteString, 0, removal_version=(3, 14) # Not generic. -) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ @@ -2651,8 +2818,8 @@ class Other(Leaf): # Error reported by type checker OrderedDict = _alias(collections.OrderedDict, 2) Counter = _alias(collections.Counter, 1) ChainMap = _alias(collections.ChainMap, 2) -Generator = _alias(collections.abc.Generator, 3) -AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2) +Generator = _alias(collections.abc.Generator, 3, defaults=(types.NoneType, types.NoneType)) +AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2, defaults=(types.NoneType,)) Type = _alias(type, 1, inst=False, name='Type') Type.__doc__ = \ """Deprecated alias to builtins.type. @@ -2756,22 +2923,30 @@ def __round__(self, ndigits: int = 0) -> T: pass -def _make_nmtuple(name, types, module, defaults = ()): - fields = [n for n, t in types] - types = {n: _type_check(t, f"field {n} annotation must be a type") - for n, t in types} +def _make_nmtuple(name, fields, annotate_func, module, defaults = ()): nm_tpl = collections.namedtuple(name, fields, defaults=defaults, module=module) - nm_tpl.__annotations__ = nm_tpl.__new__.__annotations__ = types + nm_tpl.__annotate__ = nm_tpl.__new__.__annotate__ = annotate_func return nm_tpl +def _make_eager_annotate(types): + checked_types = {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in types.items()} + def annotate(format): + if format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): + return checked_types + else: + return annotationlib.annotations_to_string(types) + return annotate + + # attributes prohibited to set in NamedTuple class syntax _prohibited = frozenset({'__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_make', '_replace', '_asdict', '_source'}) -_special = frozenset({'__module__', '__name__', '__annotations__'}) +_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__'}) class NamedTupleMeta(type): @@ -2782,9 +2957,31 @@ def __new__(cls, typename, bases, ns): raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + field_names = list(types) + annotate = _make_eager_annotate(types) + elif "__annotate__" in ns: + original_annotate = ns["__annotate__"] + types = annotationlib.call_annotate_function(original_annotate, annotationlib.Format.FORWARDREF) + field_names = list(types) + + # For backward compatibility, type-check all the types at creation time + for typ in types.values(): + _type_check(typ, "field annotation must be a type") + + def annotate(format): + annos = annotationlib.call_annotate_function(original_annotate, format) + if format != annotationlib.Format.STRING: + return {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in annos.items()} + return annos + else: + # Empty NamedTuple + field_names = [] + annotate = lambda format: {} default_names = [] - for field_name in types: + for field_name in field_names: if field_name in ns: default_names.append(field_name) elif default_names: @@ -2792,7 +2989,7 @@ def __new__(cls, typename, bases, ns): f"cannot follow default field" f"{'s' if len(default_names) > 1 else ''} " f"{', '.join(default_names)}") - nm_tpl = _make_nmtuple(typename, types.items(), + nm_tpl = _make_nmtuple(typename, field_names, annotate, defaults=[ns[n] for n in default_names], module=ns['__module__']) nm_tpl.__bases__ = bases @@ -2825,15 +3022,6 @@ def __new__(cls, typename, bases, ns): return nm_tpl -class _Sentinel: - __slots__ = () - def __repr__(self): - return '' - - -_sentinel = _Sentinel() - - def NamedTuple(typename, fields=_sentinel, /, **kwargs): """Typed version of namedtuple. @@ -2892,7 +3080,11 @@ class Employee(NamedTuple): import warnings warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) fields = kwargs.items() - nt = _make_nmtuple(typename, fields, module=_caller()) + types = {n: _type_check(t, f"field {n} annotation must be a type") + for n, t in fields} + field_names = [n for n, _ in fields] + + nt = _make_nmtuple(typename, field_names, _make_eager_annotate(types), module=_caller()) nt.__orig_bases__ = (NamedTuple,) return nt @@ -2905,6 +3097,28 @@ def _namedtuple_mro_entries(bases): NamedTuple.__mro_entries__ = _namedtuple_mro_entries +def _get_typeddict_qualifiers(annotation_type): + while True: + annotation_origin = get_origin(annotation_type) + if annotation_origin is Annotated: + annotation_args = get_args(annotation_type) + if annotation_args: + annotation_type = annotation_args[0] + else: + break + elif annotation_origin is Required: + yield Required + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is NotRequired: + yield NotRequired + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is ReadOnly: + yield ReadOnly + (annotation_type,) = get_args(annotation_type) + else: + break + + class _TypedDictMeta(type): def __new__(cls, name, bases, ns, total=True): """Create a new typed dict class object. @@ -2929,19 +3143,28 @@ def __new__(cls, name, bases, ns, total=True): if not hasattr(tp_dict, '__orig_bases__'): tp_dict.__orig_bases__ = bases - annotations = {} - own_annotations = ns.get('__annotations__', {}) + if "__annotations__" in ns: + own_annotate = None + own_annotations = ns["__annotations__"] + elif "__annotate__" in ns: + own_annotate = ns["__annotate__"] + own_annotations = annotationlib.call_annotate_function( + own_annotate, annotationlib.Format.FORWARDREF, owner=tp_dict + ) + else: + own_annotate = None + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" - own_annotations = { + own_checked_annotations = { n: _type_check(tp, msg, module=tp_dict.__module__) for n, tp in own_annotations.items() } required_keys = set() optional_keys = set() + readonly_keys = set() + mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) - base_required = base.__dict__.get('__required_keys__', set()) required_keys |= base_required optional_keys -= base_required @@ -2950,18 +3173,14 @@ def __new__(cls, name, bases, ns, total=True): required_keys -= base_optional optional_keys |= base_optional - annotations.update(own_annotations) - for annotation_key, annotation_type in own_annotations.items(): - annotation_origin = get_origin(annotation_type) - if annotation_origin is Annotated: - annotation_args = get_args(annotation_type) - if annotation_args: - annotation_type = annotation_args[0] - annotation_origin = get_origin(annotation_type) + readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) + mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) - if annotation_origin is Required: + for annotation_key, annotation_type in own_checked_annotations.items(): + qualifiers = set(_get_typeddict_qualifiers(annotation_type)) + if Required in qualifiers: is_required = True - elif annotation_origin is NotRequired: + elif NotRequired in qualifiers: is_required = False else: is_required = total @@ -2973,13 +3192,51 @@ def __new__(cls, name, bases, ns, total=True): optional_keys.add(annotation_key) required_keys.discard(annotation_key) + if ReadOnly in qualifiers: + if annotation_key in mutable_keys: + raise TypeError( + f"Cannot override mutable key {annotation_key!r}" + " with read-only key" + ) + readonly_keys.add(annotation_key) + else: + mutable_keys.add(annotation_key) + readonly_keys.discard(annotation_key) + assert required_keys.isdisjoint(optional_keys), ( f"Required keys overlap with optional keys in {name}:" f" {required_keys=}, {optional_keys=}" ) - tp_dict.__annotations__ = annotations + + def __annotate__(format): + annos = {} + for base in bases: + if base is Generic: + continue + base_annotate = base.__annotate__ + if base_annotate is None: + continue + base_annos = annotationlib.call_annotate_function(base.__annotate__, format, owner=base) + annos.update(base_annos) + if own_annotate is not None: + own = annotationlib.call_annotate_function(own_annotate, format, owner=tp_dict) + if format != annotationlib.Format.STRING: + own = { + n: _type_check(tp, msg, module=tp_dict.__module__) + for n, tp in own.items() + } + elif format == annotationlib.Format.STRING: + own = annotationlib.annotations_to_string(own_annotations) + else: + own = own_checked_annotations + annos.update(own) + return annos + + tp_dict.__annotate__ = __annotate__ tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) + tp_dict.__readonly_keys__ = frozenset(readonly_keys) + tp_dict.__mutable_keys__ = frozenset(mutable_keys) tp_dict.__total__ = total return tp_dict @@ -3038,6 +3295,14 @@ class Point2D(TypedDict): y: NotRequired[int] # the "y" key can be omitted See PEP 655 for more details on Required and NotRequired. + + The ReadOnly special form can be used + to mark individual keys as immutable for type checkers:: + + class DatabaseUser(TypedDict): + id: ReadOnly[int] # the "id" key must not be modified + username: str # the "username" key can be changed + """ if fields is _sentinel or fields is None: import warnings @@ -3114,6 +3379,26 @@ class Movie(TypedDict): return _GenericAlias(self, (item,)) +@_SpecialForm +def ReadOnly(self, parameters): + """A special typing construct to mark an item of a TypedDict as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1992 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + class NewType: """NewType creates simple unique types with almost zero runtime overhead. @@ -3529,7 +3814,17 @@ def __getattr__(attr): obj = _alias(getattr(re, attr), 1) elif attr in {"ContextManager", "AsyncContextManager"}: import contextlib - obj = _alias(getattr(contextlib, f"Abstract{attr}"), 1, name=attr) + obj = _alias(getattr(contextlib, f"Abstract{attr}"), 2, name=attr, defaults=(bool | None,)) + elif attr == "_collect_parameters": + import warnings + + depr_message = ( + "The private _collect_parameters function is deprecated and will be" + " removed in a future version of Python. Any use of private functions" + " is discouraged and may break in the future." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=2) + obj = _collect_type_parameters else: raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") globals()[attr] = obj diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index f1f6c911ef17d9..324e5d038aef03 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -57,9 +57,9 @@ def testMultiply(self): from .case import (addModuleCleanup, TestCase, FunctionTestCase, SkipTest, skip, skipIf, skipUnless, expectedFailure, doModuleCleanups, enterModuleContext) -from .suite import BaseTestSuite, TestSuite +from .suite import BaseTestSuite, TestSuite # noqa: F401 from .loader import TestLoader, defaultTestLoader -from .main import TestProgram, main +from .main import TestProgram, main # noqa: F401 from .runner import TextTestRunner, TextTestResult from .signals import installHandler, registerResult, removeResult, removeHandler # IsolatedAsyncioTestCase will be imported lazily. diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index 63ff6a5d1f8b61..bd06eb3207697a 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -90,9 +90,13 @@ def _callSetUp(self): self._callAsync(self.asyncSetUp) def _callTestMethod(self, method): - if self._callMaybeAsync(method) is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=4) + result = self._callMaybeAsync(method) + if result is not None: + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})', + ) + warnings.warn(msg, DeprecationWarning, stacklevel=4) def _callTearDown(self): self._callAsync(self.asyncTearDown) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 001b640dc43ad6..55c79d353539ca 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -332,6 +332,23 @@ def __exit__(self, exc_type, exc_value, tb): self._raiseFailure("{} not triggered".format(exc_name)) +class _AssertNotWarnsContext(_AssertWarnsContext): + + def __exit__(self, exc_type, exc_value, tb): + self.warnings_manager.__exit__(exc_type, exc_value, tb) + if exc_type is not None: + # let unexpected exceptions pass through + return + try: + exc_name = self.expected.__name__ + except AttributeError: + exc_name = str(self.expected) + for m in self.warnings: + w = m.message + if isinstance(w, self.expected): + self._raiseFailure(f"{exc_name} triggered") + + class _OrderedChainMap(collections.ChainMap): def __iter__(self): seen = set() @@ -586,9 +603,18 @@ def _callSetUp(self): self.setUp() def _callTestMethod(self, method): - if method() is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=3) + result = method() + if result is not None: + import inspect + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})' + ) + if inspect.iscoroutine(result): + msg += ( + '. Maybe you forgot to use IsolatedAsyncioTestCase as the base class?' + ) + warnings.warn(msg, DeprecationWarning, stacklevel=3) def _callTearDown(self): self.tearDown() @@ -811,6 +837,11 @@ def assertWarns(self, expected_warning, *args, **kwargs): context = _AssertWarnsContext(expected_warning, self) return context.handle('assertWarns', args, kwargs) + def _assertNotWarns(self, expected_warning, *args, **kwargs): + """The opposite of assertWarns. Private due to low demand.""" + context = _AssertNotWarnsContext(expected_warning, self) + return context.handle('_assertNotWarns', args, kwargs) + def assertLogs(self, logger=None, level=None): """Fail unless a log message of level *level* or higher is emitted on *logger_name* or its children. If omitted, *level* defaults to diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index 9a3e5cc4bf30e5..a52950dad224ee 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -254,6 +254,7 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): Paths are sorted before being imported to ensure reproducible execution order even on filesystems with non-alphabetical ordering like ext3/4. """ + original_top_level_dir = self._top_level_dir set_implicit_top = False if top_level_dir is None and self._top_level_dir is not None: # make top_level_dir optional if called from load_tests in a package @@ -273,6 +274,8 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): self._top_level_dir = top_level_dir is_not_importable = False + is_namespace = False + tests = [] if os.path.isdir(os.path.abspath(start_dir)): start_dir = os.path.abspath(start_dir) if start_dir != top_level_dir: @@ -285,12 +288,25 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): is_not_importable = True else: the_module = sys.modules[start_dir] - top_part = start_dir.split('.')[0] - try: - start_dir = os.path.abspath( - os.path.dirname((the_module.__file__))) - except AttributeError: - if the_module.__name__ in sys.builtin_module_names: + if not hasattr(the_module, "__file__") or the_module.__file__ is None: + # look for namespace packages + try: + spec = the_module.__spec__ + except AttributeError: + spec = None + + if spec and spec.submodule_search_locations is not None: + is_namespace = True + + for path in the_module.__path__: + if (not set_implicit_top and + not path.startswith(top_level_dir)): + continue + self._top_level_dir = \ + (path.split(the_module.__name__ + .replace(".", os.path.sep))[0]) + tests.extend(self._find_tests(path, pattern, namespace=True)) + elif the_module.__name__ in sys.builtin_module_names: # builtin module raise TypeError('Can not use builtin modules ' 'as dotted module names') from None @@ -299,14 +315,28 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): f"don't know how to discover from {the_module!r}" ) from None + else: + top_part = start_dir.split('.')[0] + start_dir = os.path.abspath(os.path.dirname((the_module.__file__))) + if set_implicit_top: - self._top_level_dir = self._get_directory_containing_module(top_part) + if not is_namespace: + if sys.modules[top_part].__file__ is None: + self._top_level_dir = os.path.dirname(the_module.__file__) + if self._top_level_dir not in sys.path: + sys.path.insert(0, self._top_level_dir) + else: + self._top_level_dir = \ + self._get_directory_containing_module(top_part) sys.path.remove(top_level_dir) if is_not_importable: raise ImportError('Start directory is not importable: %r' % start_dir) - tests = list(self._find_tests(start_dir, pattern)) + if not is_namespace: + tests = list(self._find_tests(start_dir, pattern)) + + self._top_level_dir = original_top_level_dir return self.suiteClass(tests) def _get_directory_containing_module(self, module_name): @@ -341,7 +371,7 @@ def _match_path(self, path, full_path, pattern): # override this method to use alternative matching strategy return fnmatch(path, pattern) - def _find_tests(self, start_dir, pattern): + def _find_tests(self, start_dir, pattern, namespace=False): """Used by discovery. Yields test suites it loads.""" # Handle the __init__ in this package name = self._get_name_from_path(start_dir) @@ -350,7 +380,8 @@ def _find_tests(self, start_dir, pattern): if name != '.' and name not in self._loading_packages: # name is in self._loading_packages while we have called into # loadTestsFromModule with name. - tests, should_recurse = self._find_test_path(start_dir, pattern) + tests, should_recurse = self._find_test_path( + start_dir, pattern, namespace) if tests is not None: yield tests if not should_recurse: @@ -361,7 +392,8 @@ def _find_tests(self, start_dir, pattern): paths = sorted(os.listdir(start_dir)) for path in paths: full_path = os.path.join(start_dir, path) - tests, should_recurse = self._find_test_path(full_path, pattern) + tests, should_recurse = self._find_test_path( + full_path, pattern, False) if tests is not None: yield tests if should_recurse: @@ -369,11 +401,11 @@ def _find_tests(self, start_dir, pattern): name = self._get_name_from_path(full_path) self._loading_packages.add(name) try: - yield from self._find_tests(full_path, pattern) + yield from self._find_tests(full_path, pattern, False) finally: self._loading_packages.discard(name) - def _find_test_path(self, full_path, pattern): + def _find_test_path(self, full_path, pattern, namespace=False): """Used by discovery. Loads tests from a single file, or a directories' __init__.py when @@ -417,7 +449,8 @@ def _find_test_path(self, full_path, pattern): msg % (mod_name, module_dir, expected_dir)) return self.loadTestsFromModule(module, pattern=pattern), False elif os.path.isdir(full_path): - if not os.path.isfile(os.path.join(full_path, '__init__.py')): + if (not namespace and + not os.path.isfile(os.path.join(full_path, '__init__.py'))): return None, False load_tests = None diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 93f4d9743ed2fa..21ca061a77c26f 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -32,8 +32,9 @@ import sys import builtins import pkgutil -from asyncio import iscoroutinefunction +from inspect import iscoroutinefunction import threading +from dataclasses import fields, is_dataclass from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr from functools import wraps, partial @@ -573,7 +574,7 @@ def __get_return_value(self): if self._mock_delegate is not None: ret = self._mock_delegate.return_value - if ret is DEFAULT: + if ret is DEFAULT and self._mock_wraps is None: ret = self._get_child_mock( _new_parent=self, _new_name='()' ) @@ -628,7 +629,9 @@ def __set_side_effect(self, value): side_effect = property(__get_side_effect, __set_side_effect) - def reset_mock(self, visited=None,*, return_value=False, side_effect=False): + def reset_mock(self, visited=None, *, + return_value: bool = False, + side_effect: bool = False): "Restore the mock object to its initial state." if visited is None: visited = [] @@ -830,6 +833,9 @@ def __setattr__(self, name, value): mock_name = f'{self._extract_mock_name()}.{name}' raise AttributeError(f'Cannot set {mock_name}') + if isinstance(value, PropertyMock): + self.__dict__[name] = value + return return object.__setattr__(self, name, value) @@ -1234,6 +1240,9 @@ def _execute_mock_call(self, /, *args, **kwargs): if self._mock_return_value is not DEFAULT: return self.return_value + if self._mock_delegate and self._mock_delegate.return_value is not DEFAULT: + return self.return_value + if self._mock_wraps is not None: return self._mock_wraps(*args, **kwargs) @@ -1505,13 +1514,12 @@ def __enter__(self): if isinstance(original, type): # If we're patching out a class and there is a spec inherit = True - if spec is None and _is_async_obj(original): - Klass = AsyncMock - else: - Klass = MagicMock - _kwargs = {} + + # Determine the Klass to use if new_callable is not None: Klass = new_callable + elif spec is None and _is_async_obj(original): + Klass = AsyncMock elif spec is not None or spec_set is not None: this_spec = spec if spec_set is not None: @@ -1524,7 +1532,12 @@ def __enter__(self): Klass = AsyncMock elif not_callable: Klass = NonCallableMagicMock + else: + Klass = MagicMock + else: + Klass = MagicMock + _kwargs = {} if spec is not None: _kwargs['spec'] = spec if spec_set is not None: @@ -1745,7 +1758,7 @@ def patch( the patch is undone. If `new` is omitted, then the target is replaced with an - `AsyncMock if the patched object is an async function or a + `AsyncMock` if the patched object is an async function or a `MagicMock` otherwise. If `patch` is used as a decorator and `new` is omitted, the created mock is passed in as an extra argument to the decorated function. If `patch` is used as a context manager the created @@ -1819,7 +1832,8 @@ def patch( class _patch_dict(object): """ Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is + a copy of the dictionary as it was before the test. `in_dict` can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items @@ -2155,8 +2169,6 @@ def _mock_set_magics(self): if getattr(self, "_mock_methods", None) is not None: these_magics = orig_magics.intersection(self._mock_methods) - - remove_magics = set() remove_magics = orig_magics - these_magics for entry in remove_magics: @@ -2209,6 +2221,17 @@ def mock_add_spec(self, spec, spec_set=False): self._mock_add_spec(spec, spec_set) self._mock_set_magics() + def reset_mock(self, /, *args, return_value: bool = False, **kwargs): + if ( + return_value + and self._mock_name + and _is_magic(self._mock_name) + ): + # Don't reset return values for magic methods, + # otherwise `m.__str__` will start + # to return `MagicMock` instances, instead of `str` instances. + return_value = False + super().reset_mock(*args, return_value=return_value, **kwargs) class MagicProxy(Base): @@ -2445,7 +2468,7 @@ class AsyncMock(AsyncMockMixin, AsyncMagicMixin, Mock): recognized as an async function, and the result of a call is an awaitable: >>> mock = AsyncMock() - >>> iscoroutinefunction(mock) + >>> inspect.iscoroutinefunction(mock) True >>> inspect.isawaitable(mock()) True @@ -2734,7 +2757,15 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, raise InvalidSpecError(f'Cannot autospec a Mock object. ' f'[object={spec!r}]') is_async_func = _is_async_func(spec) - _kwargs = {'spec': spec} + + entries = [(entry, _missing) for entry in dir(spec)] + if is_type and instance and is_dataclass(spec): + dataclass_fields = fields(spec) + entries.extend((f.name, f.type) for f in dataclass_fields) + _kwargs = {'spec': [f.name for f in dataclass_fields]} + else: + _kwargs = {'spec': spec} + if spec_set: _kwargs = {'spec_set': spec} elif spec is None: @@ -2745,6 +2776,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if not unsafe: _check_spec_arg_typos(kwargs) + _name = kwargs.pop('name', _name) + _new_name = _name + if _parent is None: + # for a top level object no _new_name should be set + _new_name = '' + _kwargs.update(kwargs) Klass = MagicMock @@ -2762,13 +2799,6 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, elif is_type and instance and not _instance_callable(spec): Klass = NonCallableMagicMock - _name = _kwargs.pop('name', _name) - - _new_name = _name - if _parent is None: - # for a top level object no _new_name should be set - _new_name = '' - mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name, name=_name, **_kwargs) @@ -2785,11 +2815,14 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if _parent is not None and not instance: _parent._mock_children[_name] = mock + # Pop wraps from kwargs because it must not be passed to configure_mock. + wrapped = kwargs.pop('wraps', None) if is_type and not instance and 'return_value' not in kwargs: mock.return_value = create_autospec(spec, spec_set, instance=True, - _name='()', _parent=mock) + _name='()', _parent=mock, + wraps=wrapped) - for entry in dir(spec): + for entry, original in entries: if _is_magic(entry): # MagicMock already does the useful magic methods for us continue @@ -2803,14 +2836,18 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # AttributeError on being fetched? # we could be resilient against it, or catch and propagate the # exception when the attribute is fetched from the mock - try: - original = getattr(spec, entry) - except AttributeError: - continue + if original is _missing: + try: + original = getattr(spec, entry) + except AttributeError: + continue - kwargs = {'spec': original} + child_kwargs = {'spec': original} + # Wrap child attributes also. + if wrapped and hasattr(wrapped, entry): + child_kwargs.update(wraps=original) if spec_set: - kwargs = {'spec_set': original} + child_kwargs = {'spec_set': original} if not isinstance(original, FunctionTypes): new = _SpecState(original, spec_set, mock, entry, instance) @@ -2821,14 +2858,13 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, parent = mock.mock skipfirst = _must_skip(spec, entry, is_type) - kwargs['_eat_self'] = skipfirst + child_kwargs['_eat_self'] = skipfirst if iscoroutinefunction(original): child_klass = AsyncMock else: child_klass = MagicMock new = child_klass(parent=parent, name=entry, _new_name=entry, - _new_parent=parent, - **kwargs) + _new_parent=parent, **child_kwargs) mock._mock_children[entry] = new new.return_value = child_klass() _check_signature(original, new, skipfirst=skipfirst) @@ -2839,6 +2875,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # setting as an instance attribute? if isinstance(new, FunctionTypes): setattr(mock, entry, new) + # kwargs are passed with respect to the parent mock so, they are not used + # for creating return_value of the parent mock. So, this condition + # should be true only for the parent mock if kwargs are given. + if _is_instance_mock(mock) and kwargs: + mock.configure_mock(**kwargs) return mock diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index c129b0d7971d71..5b00ab25c6b4ca 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -392,20 +392,23 @@ def urlparse(url, scheme='', allow_fragments=True): Note that % escapes are not expanded. """ url, scheme, _coerce_result = _coerce_args(url, scheme) - splitresult = urlsplit(url, scheme, allow_fragments) - scheme, netloc, url, query, fragment = splitresult - if scheme in uses_params and ';' in url: - url, params = _splitparams(url) - else: - params = '' - result = ParseResult(scheme, netloc, url, params, query, fragment) + scheme, netloc, url, params, query, fragment = _urlparse(url, scheme, allow_fragments) + result = ParseResult(scheme or '', netloc or '', url, params or '', query or '', fragment or '') return _coerce_result(result) -def _splitparams(url): +def _urlparse(url, scheme=None, allow_fragments=True): + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + if (scheme or '') in uses_params and ';' in url: + url, params = _splitparams(url, allow_none=True) + else: + params = None + return (scheme, netloc, url, params, query, fragment) + +def _splitparams(url, allow_none=False): if '/' in url: i = url.find(';', url.rfind('/')) if i < 0: - return url, '' + return url, None if allow_none else '' else: i = url.find(';') return url[:i], url[i+1:] @@ -472,17 +475,23 @@ def urlsplit(url, scheme='', allow_fragments=True): """ url, scheme, _coerce_result = _coerce_args(url, scheme) + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + v = SplitResult(scheme or '', netloc or '', url, query or '', fragment or '') + return _coerce_result(v) + +def _urlsplit(url, scheme=None, allow_fragments=True): # Only lstrip url as some applications rely on preserving trailing space. # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) - scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) - for b in _UNSAFE_URL_BYTES_TO_REMOVE: url = url.replace(b, "") - scheme = scheme.replace(b, "") + if scheme is not None: + scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + scheme = scheme.replace(b, "") allow_fragments = bool(allow_fragments) - netloc = query = fragment = '' + netloc = query = fragment = None i = url.find(':') if i > 0 and url[0].isascii() and url[0].isalpha(): for c in url[:i]: @@ -503,8 +512,7 @@ def urlsplit(url, scheme='', allow_fragments=True): if '?' in url: url, query = url.split('?', 1) _checknetloc(netloc) - v = SplitResult(scheme, netloc, url, query, fragment) - return _coerce_result(v) + return (scheme, netloc, url, query, fragment) def urlunparse(components): """Put a parsed URL back together again. This may result in a @@ -513,9 +521,15 @@ def urlunparse(components): (the draft states that these are equivalent).""" scheme, netloc, url, params, query, fragment, _coerce_result = ( _coerce_args(*components)) + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None if params: url = "%s;%s" % (url, params) - return _coerce_result(urlunsplit((scheme, netloc, url, query, fragment))) + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) def urlunsplit(components): """Combine the elements of a tuple as returned by urlsplit() into a @@ -525,16 +539,27 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) + +def _urlunsplit(scheme, netloc, url, query, fragment): + if netloc is not None: if url and url[:1] != '/': url = '/' + url - url = '//' + (netloc or '') + url + url = '//' + netloc + url + elif url[:2] == '//': + url = '//' + url if scheme: url = scheme + ':' + url - if query: + if query is not None: url = url + '?' + query - if fragment: + if fragment is not None: url = url + '#' + fragment - return _coerce_result(url) + return url def urljoin(base, url, allow_fragments=True): """Join a base URL and a possibly relative URL to form an absolute @@ -545,26 +570,29 @@ def urljoin(base, url, allow_fragments=True): return base base, url, _coerce_result = _coerce_args(base, url) - bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments) - scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments) + bscheme, bnetloc, bpath, bquery, bfragment = \ + _urlsplit(base, None, allow_fragments) + scheme, netloc, path, query, fragment = \ + _urlsplit(url, None, allow_fragments) + if scheme is None: + scheme = bscheme if scheme != bscheme or scheme not in uses_relative: return _coerce_result(url) if scheme in uses_netloc: if netloc: - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) netloc = bnetloc - if not path and not params: + if not path: path = bpath - params = bparams - if not query: + if query is None: query = bquery - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + if fragment is None: + fragment = bfragment + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) base_parts = bpath.split('/') if base_parts[-1] != '': @@ -601,8 +629,8 @@ def urljoin(base, url, allow_fragments=True): # then we need to append the trailing '/' resolved_path.append('') - return _coerce_result(urlunparse((scheme, netloc, '/'.join( - resolved_path) or '/', params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, '/'.join( + resolved_path) or '/', query, fragment)) def urldefrag(url): @@ -614,12 +642,12 @@ def urldefrag(url): """ url, _coerce_result = _coerce_args(url) if '#' in url: - s, n, p, a, q, frag = urlparse(url) - defrag = urlunparse((s, n, p, a, q, '')) + s, n, p, q, frag = _urlsplit(url) + defrag = _urlunsplit(s, n, p, q, None) else: frag = '' defrag = url - return _coerce_result(DefragResult(defrag, frag)) + return _coerce_result(DefragResult(defrag, frag or '')) _hexdig = '0123456789ABCDEFabcdef' _hextobyte = None @@ -763,42 +791,48 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, Returns a list, as G-d intended. """ - qs, _coerce_result = _coerce_args(qs) - separator, _ = _coerce_args(separator) - if not separator or (not isinstance(separator, (str, bytes))): + if not separator or not isinstance(separator, (str, bytes)): raise ValueError("Separator must be of type string or bytes.") + if isinstance(qs, str): + if not isinstance(separator, str): + separator = str(separator, 'ascii') + eq = '=' + def _unquote(s): + return unquote_plus(s, encoding=encoding, errors=errors) + else: + if not qs: + return [] + # Use memoryview() to reject integers and iterables, + # acceptable by the bytes constructor. + qs = bytes(memoryview(qs)) + if isinstance(separator, str): + separator = bytes(separator, 'ascii') + eq = b'=' + def _unquote(s): + return unquote_to_bytes(s.replace(b'+', b' ')) + + if not qs: + return [] # If max_num_fields is defined then check that the number of fields # is less than max_num_fields. This prevents a memory exhaustion DOS # attack via post bodies with many fields. if max_num_fields is not None: - num_fields = 1 + qs.count(separator) if qs else 0 + num_fields = 1 + qs.count(separator) if max_num_fields < num_fields: raise ValueError('Max number of fields exceeded') r = [] - query_args = qs.split(separator) if qs else [] - for name_value in query_args: - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: + for name_value in qs.split(separator): + if name_value or strict_parsing: + name, has_eq, value = name_value.partition(eq) + if not has_eq and strict_parsing: raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = nv[0].replace('+', ' ') - name = unquote(name, encoding=encoding, errors=errors) - name = _coerce_result(name) - value = nv[1].replace('+', ' ') - value = unquote(value, encoding=encoding, errors=errors) - value = _coerce_result(value) - r.append((name, value)) + if value or keep_blank_values: + name = _unquote(name) + value = _unquote(value) + r.append((name, value)) return r def unquote_plus(string, encoding='utf-8', errors='replace'): @@ -816,14 +850,6 @@ def unquote_plus(string, encoding='utf-8', errors='replace'): b'_.-~') _ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) -def __getattr__(name): - if name == 'Quoter': - warnings.warn('Deprecated in 3.11. ' - 'urllib.parse.Quoter will be removed in Python 3.14. ' - 'It was not intended to be a public API.', - DeprecationWarning, stacklevel=2) - return _Quoter - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') class _Quoter(dict): """A mapping from bytes numbers (in range(0,256)) to strings. diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index bca594420f6d9d..bc35d8a80e5d03 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -108,7 +108,7 @@ # check for SSL try: - import ssl + import ssl # noqa: F401 except ImportError: _have_ssl = False else: @@ -650,6 +650,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): newheaders = {k: v for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS} return Request(newurl, + method="HEAD" if m == "HEAD" else "GET", headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True) @@ -1554,7 +1555,7 @@ def ftp_open(self, req): headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) except ftplib.all_errors as exp: - raise URLError(exp) from exp + raise URLError(f"ftp error: {exp}") from exp def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, @@ -2563,6 +2564,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): } """ from fnmatch import fnmatch + from ipaddress import AddressValueError, IPv4Address hostonly, port = _splitport(host) @@ -2579,20 +2581,17 @@ def ip2num(ipAddr): return True hostIP = None + try: + hostIP = int(IPv4Address(hostonly)) + except AddressValueError: + pass for value in proxy_settings.get('exceptions', ()): # Items in the list are strings like these: *.local, 169.254/16 if not value: continue m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) - if m is not None: - if hostIP is None: - try: - hostIP = socket.gethostbyname(hostonly) - hostIP = ip2num(hostIP) - except OSError: - continue - + if m is not None and hostIP is not None: base = ip2num(m.group(1)) mask = m.group(2) if mask is None: @@ -2615,6 +2614,31 @@ def ip2num(ipAddr): return False +# Same as _proxy_bypass_macosx_sysconf, testable on all platforms +def _proxy_bypass_winreg_override(host, override): + """Return True if the host should bypass the proxy server. + + The proxy override list is obtained from the Windows + Internet settings proxy override registry value. + + An example of a proxy override value is: + "www.example.com;*.example.net; 192.168.0.1" + """ + from fnmatch import fnmatch + + host, _ = _splitport(host) + proxy_override = override.split(';') + for test in proxy_override: + test = test.strip() + # "" should bypass the proxy server for all intranet addresses + if test == '': + if '.' not in host: + return True + elif fnmatch(host, test): + return True + return False + + if sys.platform == 'darwin': from _scproxy import _get_proxy_settings, _get_proxies @@ -2713,7 +2737,7 @@ def proxy_bypass_registry(host): import winreg except ImportError: # Std modules, so should be around - but you never know! - return 0 + return False try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') @@ -2723,40 +2747,10 @@ def proxy_bypass_registry(host): 'ProxyOverride')[0]) # ^^^^ Returned as Unicode but problems if not converted to ASCII except OSError: - return 0 + return False if not proxyEnable or not proxyOverride: - return 0 - # try to make a host list from name and IP address. - rawHost, port = _splitport(host) - host = [rawHost] - try: - addr = socket.gethostbyname(rawHost) - if addr != rawHost: - host.append(addr) - except OSError: - pass - try: - fqdn = socket.getfqdn(rawHost) - if fqdn != rawHost: - host.append(fqdn) - except OSError: - pass - # make a check value list from the registry entry: replace the - # '' string by the localhost entry and the corresponding - # canonical entry. - proxyOverride = proxyOverride.split(';') - # now check if we match one of the registry values. - for test in proxyOverride: - if test == '': - if '.' not in rawHost: - return 1 - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char - for val in host: - if re.match(test, val, re.I): - return 1 - return 0 + return False + return _proxy_bypass_winreg_override(host, proxyOverride) def proxy_bypass(host): """Return True, if host should be bypassed. diff --git a/Lib/uuid.py b/Lib/uuid.py index 470bc0d68597ab..4d4f06cfc9ebbe 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -53,13 +53,16 @@ __author__ = 'Ka-Ping Yee ' # The recognized platforms - known behaviors -if sys.platform in ('win32', 'darwin', 'emscripten', 'wasi'): +if sys.platform in {'win32', 'darwin', 'emscripten', 'wasi'}: _AIX = _LINUX = False +elif sys.platform == 'linux': + _LINUX = True + _AIX = False else: import platform _platform_system = platform.system() _AIX = _platform_system == 'AIX' - _LINUX = _platform_system == 'Linux' + _LINUX = _platform_system in ('Linux', 'Android') _MAC_DELIM = b':' _MAC_OMITS_LEADING_ZEROES = False @@ -371,7 +374,7 @@ def _get_command_stdout(command, *args): # for are actually localized, but in theory some system could do so.) env = dict(os.environ) env['LC_ALL'] = 'C' - # Empty strings will be quoted by popen so we should just ommit it + # Empty strings will be quoted by popen so we should just omit it if args != ('',): command = (executable, *args) else: @@ -564,32 +567,16 @@ def _netstat_getnode(): # This works on AIX and might work on Tru64 UNIX. return _find_mac_under_heading('netstat', '-ian', b'Address') -def _ipconfig_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - -def _netbios_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - # Import optional C extension at toplevel, to help disabling it when testing try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) _UuidCreate = getattr(_uuid, "UuidCreate", None) - _has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe except ImportError: _uuid = None _generate_time_safe = None _UuidCreate = None - _has_uuid_generate_time_safe = None - - -def _load_system_functions(): - """[DEPRECATED] Platform-specific functions loaded at import time""" def _unix_getnode(): diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 4856594755ae57..ca1af84e6705fe 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -11,6 +11,7 @@ import sys import sysconfig import types +import shlex CORE_VENV_DEPS = ('pip',) @@ -107,6 +108,33 @@ def _venv_path(self, env_dir, name): } return sysconfig.get_path(name, scheme='venv', vars=vars) + @classmethod + def _same_path(cls, path1, path2): + """Check whether two paths appear the same. + + Whether they refer to the same file is irrelevant; we're testing for + whether a human reader would look at the path string and easily tell + that they're the same file. + """ + if sys.platform == 'win32': + if os.path.normcase(path1) == os.path.normcase(path2): + return True + # gh-90329: Don't display a warning for short/long names + import _winapi + try: + path1 = _winapi.GetLongPathName(os.fsdecode(path1)) + except OSError: + pass + try: + path2 = _winapi.GetLongPathName(os.fsdecode(path2)) + except OSError: + pass + if os.path.normcase(path1) == os.path.normcase(path2): + return True + return False + else: + return path1 == path2 + def ensure_directories(self, env_dir): """ Create the directories for the environment. @@ -171,7 +199,7 @@ def create_if_needed(d): # bpo-45337: Fix up env_exec_cmd to account for file system redirections. # Some redirects only apply to CreateFile and not CreateProcess real_env_exe = os.path.realpath(context.env_exe) - if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): + if not self._same_path(real_env_exe, context.env_exe): logger.warning('Actual environment location may have moved due to ' 'redirects, links or junctions.\n' ' Requested location: "%s"\n' @@ -276,8 +304,11 @@ def setup_python(self, context): copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - for suffix in ('python', 'python3', - f'python3.{sys.version_info[1]}'): + + suffixes = ['python', 'python3', f'python3.{sys.version_info[1]}'] + if sys.version_info[:2] == (3, 14): + suffixes.append('𝜋thon') + for suffix in suffixes: path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if @@ -366,7 +397,7 @@ def setup_python(self, context): os.symlink(src, dest) to_unlink.append(dest) except OSError: - logger.warning('Unable to symlink %r to %r', src, dst) + logger.warning('Unable to symlink %r to %r', src, dest) do_copies = True for f in to_unlink: try: @@ -454,11 +485,41 @@ def replace_variables(self, text, context): :param context: The information for the environment creation request being processed. """ - text = text.replace('__VENV_DIR__', context.env_dir) - text = text.replace('__VENV_NAME__', context.env_name) - text = text.replace('__VENV_PROMPT__', context.prompt) - text = text.replace('__VENV_BIN_NAME__', context.bin_name) - text = text.replace('__VENV_PYTHON__', context.env_exe) + replacements = { + '__VENV_DIR__': context.env_dir, + '__VENV_NAME__': context.env_name, + '__VENV_PROMPT__': context.prompt, + '__VENV_BIN_NAME__': context.bin_name, + '__VENV_PYTHON__': context.env_exe, + } + + def quote_ps1(s): + """ + This should satisfy PowerShell quoting rules [1], unless the quoted + string is passed directly to Windows native commands [2]. + [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules + [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters + """ + s = s.replace("'", "''") + return f"'{s}'" + + def quote_bat(s): + return s + + # gh-124651: need to quote the template strings properly + quote = shlex.quote + script_path = context.script_path + if script_path.endswith('.ps1'): + quote = quote_ps1 + elif script_path.endswith('.bat'): + quote = quote_bat + else: + # fallbacks to POSIX shell compliant quote + quote = shlex.quote + + replacements = {key: quote(s) for key, s in replacements.items()} + for key, quoted in replacements.items(): + text = text.replace(key, quoted) return text def install_scripts(self, context, path): @@ -508,6 +569,7 @@ def skip_file(f): with open(srcfile, 'rb') as f: data = f.read() try: + context.script_path = srcfile new_data = ( self.replace_variables(data.decode('utf-8'), context) .encode('utf-8') @@ -545,8 +607,7 @@ def create(env_dir, system_site_packages=False, clear=False, def main(args=None): import argparse - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' + parser = argparse.ArgumentParser(description='Creates virtual Python ' 'environments in one or ' 'more target ' 'directories.', diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index cbd4873f012246..70673a265d41f8 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -14,8 +14,9 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi - # Call hash to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected + # Call hash to forget past locations. Without forgetting + # past locations the $PATH changes we made may not be respected. + # See "man bash" for more details. hash is usually a builtin of your shell hash -r 2> /dev/null if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then @@ -37,23 +38,23 @@ deactivate nondestructive # on Windows, a path can contain colons and backslashes and has to be converted: case "$(uname)" in - CYGWIN*|MSYS*) - # transform D:\path\to\venv to /d/path/to/venv on MSYS + CYGWIN*|MSYS*|MINGW*) + # transform D:\path\to\venv to /d/path/to/venv on MSYS and MINGW # and to /cygdrive/d/path/to/venv on Cygwin - VIRTUAL_ENV=$(cygpath "__VENV_DIR__") + VIRTUAL_ENV=$(cygpath __VENV_DIR__) export VIRTUAL_ENV ;; *) # use the path as-is - export VIRTUAL_ENV="__VENV_DIR__" + export VIRTUAL_ENV=__VENV_DIR__ ;; esac _OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +PATH="$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" export PATH -VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" +VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ export VIRTUAL_ENV_PROMPT # unset PYTHONHOME if set @@ -66,7 +67,7 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="(__VENV_PROMPT__) ${PS1:-}" + PS1="("__VENV_PROMPT__") ${PS1:-}" export PS1 fi diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/common/activate.fish similarity index 90% rename from Lib/venv/scripts/posix/activate.fish rename to Lib/venv/scripts/common/activate.fish index 25c42756789bbc..284a7469c99b57 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/common/activate.fish @@ -33,11 +33,11 @@ end # Unset irrelevant variables. deactivate nondestructive -set -gx VIRTUAL_ENV "__VENV_DIR__" +set -gx VIRTUAL_ENV __VENV_DIR__ set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH -set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" +set -gx PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__ $PATH +set -gx VIRTUAL_ENV_PROMPT __VENV_PROMPT__ # Unset PYTHONHOME if set. if set -q PYTHONHOME @@ -57,7 +57,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s(%s)%s " (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + printf "%s(%s)%s " (set_color 4B8BBE) __VENV_PROMPT__ (set_color normal) # Restore the return status of the previous command. echo "exit $old_status" | . diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index dd5ea8eb67b90a..35533e4b551155 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -8,7 +8,7 @@ if defined _OLD_CODEPAGE ( "%SystemRoot%\System32\chcp.com" 65001 > nul ) -set VIRTUAL_ENV=__VENV_DIR__ +set "VIRTUAL_ENV=__VENV_DIR__" if not defined PROMPT set PROMPT=$P$G @@ -24,8 +24,8 @@ set PYTHONHOME= if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% -set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% -set VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" +set "VIRTUAL_ENV_PROMPT=__VENV_PROMPT__" :END if defined _OLD_CODEPAGE ( diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index c707f1988b0acc..2a3fa835476ab9 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -9,17 +9,17 @@ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PA # Unset irrelevant variables. deactivate nondestructive -setenv VIRTUAL_ENV "__VENV_DIR__" +setenv VIRTUAL_ENV __VENV_DIR__ set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" -setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" +setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" +setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__ set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "(__VENV_PROMPT__) $prompt" + set prompt = "("__VENV_PROMPT__") $prompt:q" endif alias pydoc python -m pydoc diff --git a/Lib/warnings.py b/Lib/warnings.py index 4ad6ad027192e8..e83cde37ab2d1a 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -132,7 +132,7 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, append=False): """Insert an entry into the list of warnings filters (at the front). - 'action' -- one of "error", "ignore", "always", "default", "module", + 'action' -- one of "error", "ignore", "always", "all", "default", "module", or "once" 'message' -- a regex that the warning message must match 'category' -- a class that the warning must be a subclass of @@ -140,7 +140,7 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - if action not in {"error", "ignore", "always", "default", "module", "once"}: + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: raise ValueError(f"invalid action: {action!r}") if not isinstance(message, str): raise TypeError("message must be a string") @@ -171,13 +171,13 @@ def simplefilter(action, category=Warning, lineno=0, append=False): """Insert a simple entry into the list of warnings filters (at the front). A simple filter matches all modules and messages. - 'action' -- one of "error", "ignore", "always", "default", "module", + 'action' -- one of "error", "ignore", "always", "all", "default", "module", or "once" 'category' -- a class that the warning must be a subclass of 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - if action not in {"error", "ignore", "always", "default", "module", "once"}: + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: raise ValueError(f"invalid action: {action!r}") if not isinstance(lineno, int): raise TypeError("lineno must be an int") @@ -248,8 +248,7 @@ def _setoption(arg): def _getaction(action): if not action: return "default" - if action == "all": return "always" # Alias - for a in ('default', 'always', 'ignore', 'module', 'once', 'error'): + for a in ('default', 'always', 'all', 'ignore', 'module', 'once', 'error'): if a.startswith(action): return a raise _OptionError("invalid action: %r" % (action,)) @@ -332,8 +331,8 @@ def warn(message, category=None, stacklevel=1, source=None, raise ValueError except ValueError: globals = sys.__dict__ - filename = "sys" - lineno = 1 + filename = "" + lineno = 0 else: globals = frame.f_globals filename = frame.f_code.co_filename @@ -397,7 +396,7 @@ def warn_explicit(message, category, filename, lineno, if onceregistry.get(oncekey): return onceregistry[oncekey] = 1 - elif action == "always": + elif action in {"always", "all"}: pass elif action == "module": registry[key] = 1 @@ -629,12 +628,16 @@ def __init_subclass__(*args, **kwargs): return arg elif callable(arg): import functools + import inspect @functools.wraps(arg) def wrapper(*args, **kwargs): warn(msg, category=category, stacklevel=stacklevel + 1) return arg(*args, **kwargs) + if inspect.iscoroutinefunction(arg): + wrapper = inspect.markcoroutinefunction(wrapper) + arg.__deprecated__ = wrapper.__deprecated__ = msg return wrapper else: @@ -690,7 +693,7 @@ def extract(): # filters contains a sequence of filter 5-tuples # The components of the 5-tuple are: -# - an action: error, ignore, always, default, module, or once +# - an action: error, ignore, always, all, default, module, or once # - a compiled regex that must match the warning message # - a class representing the warning category # - a compiled regex that must match the module that is being warned diff --git a/Lib/weakref.py b/Lib/weakref.py index 25b70927e29c31..94e4278143c987 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -19,7 +19,7 @@ ReferenceType, _remove_dead_weakref) -from _weakrefset import WeakSet, _IterationGuard +from _weakrefset import WeakSet import _collections_abc # Import after _weakref to avoid circular import. import sys @@ -105,34 +105,14 @@ def __init__(self, other=(), /, **kw): def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(wr.key) - else: - # Atomic removal is necessary since this function - # can be called asynchronously by the GC - _atomic_removal(self.data, wr.key) + # Atomic removal is necessary since this function + # can be called asynchronously by the GC + _atomic_removal(self.data, wr.key) self._remove = remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() self.data = {} self.update(other, **kw) - def _commit_removals(self, _atomic_removal=_remove_dead_weakref): - pop = self._pending_removals.pop - d = self.data - # We shouldn't encounter any KeyError, because this method should - # always be called *before* mutating the dict. - while True: - try: - key = pop() - except IndexError: - return - _atomic_removal(d, key) - def __getitem__(self, key): - if self._pending_removals: - self._commit_removals() o = self.data[key]() if o is None: raise KeyError(key) @@ -140,18 +120,12 @@ def __getitem__(self, key): return o def __delitem__(self, key): - if self._pending_removals: - self._commit_removals() del self.data[key] def __len__(self): - if self._pending_removals: - self._commit_removals() return len(self.data) def __contains__(self, key): - if self._pending_removals: - self._commit_removals() try: o = self.data[key]() except KeyError: @@ -162,38 +136,28 @@ def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) def __setitem__(self, key, value): - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(value, self._remove, key) def copy(self): - if self._pending_removals: - self._commit_removals() new = WeakValueDictionary() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[key] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[key] = o return new __copy__ = copy def __deepcopy__(self, memo): from copy import deepcopy - if self._pending_removals: - self._commit_removals() new = self.__class__() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[deepcopy(key, memo)] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o return new def get(self, key, default=None): - if self._pending_removals: - self._commit_removals() try: wr = self.data[key] except KeyError: @@ -207,21 +171,15 @@ def get(self, key, default=None): return o def items(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - v = wr() - if v is not None: - yield k, v + for k, wr in self.data.copy().items(): + v = wr() + if v is not None: + yield k, v def keys(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - if wr() is not None: - yield k + for k, wr in self.data.copy().items(): + if wr() is not None: + yield k __iter__ = keys @@ -235,23 +193,15 @@ def itervaluerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - yield from self.data.values() + yield from self.data.copy().values() def values(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for wr in self.data.values(): - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy().values(): + obj = wr() + if obj is not None: + yield obj def popitem(self): - if self._pending_removals: - self._commit_removals() while True: key, wr = self.data.popitem() o = wr() @@ -259,8 +209,6 @@ def popitem(self): return key, o def pop(self, key, *args): - if self._pending_removals: - self._commit_removals() try: o = self.data.pop(key)() except KeyError: @@ -279,16 +227,12 @@ def setdefault(self, key, default=None): except KeyError: o = None if o is None: - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(default, self._remove, key) return default else: return o def update(self, other=None, /, **kwargs): - if self._pending_removals: - self._commit_removals() d = self.data if other is not None: if not hasattr(other, "items"): @@ -308,9 +252,7 @@ def valuerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - return list(self.data.values()) + return list(self.data.copy().values()) def __ior__(self, other): self.update(other) @@ -369,57 +311,22 @@ def __init__(self, dict=None): def remove(k, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(k) - else: - try: - del self.data[k] - except KeyError: - pass + try: + del self.data[k] + except KeyError: + pass self._remove = remove - # A list of dead weakrefs (keys to be removed) - self._pending_removals = [] - self._iterating = set() - self._dirty_len = False if dict is not None: self.update(dict) - def _commit_removals(self): - # NOTE: We don't need to call this method before mutating the dict, - # because a dead weakref never compares equal to a live weakref, - # even if they happened to refer to equal objects. - # However, it means keys may already have been removed. - pop = self._pending_removals.pop - d = self.data - while True: - try: - key = pop() - except IndexError: - return - - try: - del d[key] - except KeyError: - pass - - def _scrub_removals(self): - d = self.data - self._pending_removals = [k for k in self._pending_removals if k in d] - self._dirty_len = False - def __delitem__(self, key): - self._dirty_len = True del self.data[ref(key)] def __getitem__(self, key): return self.data[ref(key)] def __len__(self): - if self._dirty_len and self._pending_removals: - # self._pending_removals may still contain keys which were - # explicitly removed, we have to scrub them (see issue #21173). - self._scrub_removals() - return len(self.data) - len(self._pending_removals) + return len(self.data) def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) @@ -429,11 +336,10 @@ def __setitem__(self, key, value): def copy(self): new = WeakKeyDictionary() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = value + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = value return new __copy__ = copy @@ -441,11 +347,10 @@ def copy(self): def __deepcopy__(self, memo): from copy import deepcopy new = self.__class__() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = deepcopy(value, memo) + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) return new def get(self, key, default=None): @@ -459,26 +364,23 @@ def __contains__(self, key): return wr in self.data def items(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - key = wr() - if key is not None: - yield key, value + for wr, value in self.data.copy().items(): + key = wr() + if key is not None: + yield key, value def keys(self): - with _IterationGuard(self): - for wr in self.data: - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy(): + obj = wr() + if obj is not None: + yield obj __iter__ = keys def values(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - if wr() is not None: - yield value + for wr, value in self.data.copy().items(): + if wr() is not None: + yield value def keyrefs(self): """Return a list of weak references to the keys. @@ -493,7 +395,6 @@ def keyrefs(self): return list(self.data) def popitem(self): - self._dirty_len = True while True: key, value = self.data.popitem() o = key() @@ -501,7 +402,6 @@ def popitem(self): return o, value def pop(self, key, *args): - self._dirty_len = True return self.data.pop(ref(key), *args) def setdefault(self, key, default=None): diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py old mode 100755 new mode 100644 index 636e8ca459d109..d2efc72113a917 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python3 """Interfaces for launching and remotely controlling web browsers.""" # Maintained by Georg Brandl. @@ -11,14 +10,17 @@ __all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"] + class Error(Exception): pass + _lock = threading.RLock() _browsers = {} # Dictionary of available browser controllers _tryorder = None # Preference order of available browsers _os_preferred_browser = None # The preferred browser + def register(name, klass, instance=None, *, preferred=False): """Register a browser connector.""" with _lock: @@ -29,11 +31,12 @@ def register(name, klass, instance=None, *, preferred=False): # Preferred browsers go to the front of the list. # Need to match to the default browser returned by xdg-settings, which # may be of the form e.g. "firefox.desktop". - if preferred or (_os_preferred_browser and name in _os_preferred_browser): + if preferred or (_os_preferred_browser and f'{name}.desktop' == _os_preferred_browser): _tryorder.insert(0, name) else: _tryorder.append(name) + def get(using=None): """Return a browser launcher instance appropriate for the environment.""" if _tryorder is None: @@ -64,6 +67,7 @@ def get(using=None): return command[0]() raise Error("could not locate runnable browser") + # Please note: the following definition hides a builtin function. # It is recommended one does "import webbrowser" and uses webbrowser.open(url) # instead of "from webbrowser import *". @@ -76,6 +80,9 @@ def open(url, new=0, autoraise=True): - 1: a new browser window. - 2: a new browser page ("tab"). If possible, autoraise raises the window (the default) or not. + + If opening the browser succeeds, return True. + If there is a problem, return False. """ if _tryorder is None: with _lock: @@ -87,6 +94,7 @@ def open(url, new=0, autoraise=True): return True return False + def open_new(url): """Open url in a new window of the default browser. @@ -94,6 +102,7 @@ def open_new(url): """ return open(url, 1) + def open_new_tab(url): """Open url in a new page ("tab") of the default browser. @@ -136,7 +145,7 @@ def _synthesize(browser, *, preferred=False): # General parent classes -class BaseBrowser(object): +class BaseBrowser: """Parent class for all browsers. Do not use directly.""" args = ['%s'] @@ -197,7 +206,7 @@ def open(self, url, new=0, autoraise=True): else: p = subprocess.Popen(cmdline, close_fds=True, start_new_session=True) - return (p.poll() is None) + return p.poll() is None except OSError: return False @@ -225,7 +234,8 @@ def _invoke(self, args, remote, autoraise, url=None): # use autoraise argument only for remote invocation autoraise = int(autoraise) opt = self.raise_opts[autoraise] - if opt: raise_opt = [opt] + if opt: + raise_opt = [opt] cmdline = [self.name] + raise_opt + args @@ -266,8 +276,8 @@ def open(self, url, new=0, autoraise=True): else: action = self.remote_action_newtab else: - raise Error("Bad 'new' parameter to open(); " + - "expected 0, 1, or 2, got %s" % new) + raise Error("Bad 'new' parameter to open(); " + f"expected 0, 1, or 2, got {new}") args = [arg.replace("%s", url).replace("%action", action) for arg in self.remote_args] @@ -302,7 +312,7 @@ class Epiphany(UnixBrowser): class Chrome(UnixBrowser): - "Launcher class for Google Chrome browser." + """Launcher class for Google Chrome browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -310,11 +320,12 @@ class Chrome(UnixBrowser): remote_action_newtab = "" background = True + Chromium = Chrome class Opera(UnixBrowser): - "Launcher class for Opera browser." + """Launcher class for Opera browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -324,7 +335,7 @@ class Opera(UnixBrowser): class Elinks(UnixBrowser): - "Launcher class for Elinks browsers." + """Launcher class for Elinks browsers.""" remote_args = ['-remote', 'openURL(%s%action)'] remote_action = "" @@ -387,11 +398,11 @@ def open(self, url, new=0, autoraise=True): except OSError: return False else: - return (p.poll() is None) + return p.poll() is None class Edge(UnixBrowser): - "Launcher class for Microsoft Edge browser." + """Launcher class for Microsoft Edge browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -418,12 +429,18 @@ def register_X_browsers(): if shutil.which("gio"): register("gio", None, BackgroundBrowser(["gio", "open", "--", "%s"])) - # Equivalent of gio open before 2015 - if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gvfs-open"): + xdg_desktop = os.getenv("XDG_CURRENT_DESKTOP", "").split(":") + + # The default GNOME3 browser + if (("GNOME" in xdg_desktop or + "GNOME_DESKTOP_SESSION_ID" in os.environ) and + shutil.which("gvfs-open")): register("gvfs-open", None, BackgroundBrowser("gvfs-open")) # The default KDE browser - if "KDE_FULL_SESSION" in os.environ and shutil.which("kfmclient"): + if (("KDE" in xdg_desktop or + "KDE_FULL_SESSION" in os.environ) and + shutil.which("kfmclient")): register("kfmclient", Konqueror, Konqueror("kfmclient")) # Common symbolic link for the default X11 browser @@ -455,7 +472,6 @@ def register_X_browsers(): if shutil.which("opera"): register("opera", None, Opera("opera")) - if shutil.which("microsoft-edge"): register("microsoft-edge", None, Edge("microsoft-edge")) @@ -472,6 +488,9 @@ def register_standard_browsers(): # OS X can use below Unix support (but we prefer using the OS X # specific stuff) + if sys.platform == "ios": + register("iosbrowser", None, IOSBrowser(), preferred=True) + if sys.platform == "serenityos": # SerenityOS webbrowser, simply called "Browser". register("Browser", None, BackgroundBrowser("Browser")) @@ -505,7 +524,8 @@ def register_standard_browsers(): cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) result = raw_result.decode().strip() - except (FileNotFoundError, subprocess.CalledProcessError, PermissionError, NotADirectoryError) : + except (FileNotFoundError, subprocess.CalledProcessError, + PermissionError, NotADirectoryError): pass else: global _os_preferred_browser @@ -575,15 +595,16 @@ def __init__(self, name='default'): def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) + url = url.replace('"', '%22') if self.name == 'default': - script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser + script = f'open location "{url}"' # opens in default browser else: script = f''' - tell application "%s" + tell application "{self.name}" activate - open location "%s" + open location "{url}" end - '''%(self.name, url.replace('"', '%22')) + ''' osapipe = os.popen("osascript", "w") if osapipe is None: @@ -593,34 +614,96 @@ def open(self, url, new=0, autoraise=True): rc = osapipe.close() return not rc +# +# Platform support for iOS +# +if sys.platform == "ios": + from _ios_support import objc + if objc: + # If objc exists, we know ctypes is also importable. + from ctypes import c_void_p, c_char_p, c_ulong -def main(): - import getopt - usage = """Usage: %s [-n | -t | -h] url - -n: open new window - -t: open new tab - -h, --help: show help""" % sys.argv[0] - try: - opts, args = getopt.getopt(sys.argv[1:], 'ntdh',['help']) - except getopt.error as msg: - print(msg, file=sys.stderr) - print(usage, file=sys.stderr) - sys.exit(1) - new_win = 0 - for o, a in opts: - if o == '-n': new_win = 1 - elif o == '-t': new_win = 2 - elif o == '-h' or o == '--help': - print(usage, file=sys.stderr) - sys.exit() - if len(args) != 1: - print(usage, file=sys.stderr) - sys.exit(1) - - url = args[0] - open(url, new_win) + class IOSBrowser(BaseBrowser): + def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) + # If ctypes isn't available, we can't open a browser + if objc is None: + return False + + # All the messages in this call return object references. + objc.objc_msgSend.restype = c_void_p + + # This is the equivalent of: + # NSString url_string = + # [NSString stringWithCString:url.encode("utf-8") + # encoding:NSUTF8StringEncoding]; + NSString = objc.objc_getClass(b"NSString") + constructor = objc.sel_registerName(b"stringWithCString:encoding:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_char_p, c_ulong] + url_string = objc.objc_msgSend( + NSString, + constructor, + url.encode("utf-8"), + 4, # NSUTF8StringEncoding = 4 + ) + + # Create an NSURL object representing the URL + # This is the equivalent of: + # NSURL *nsurl = [NSURL URLWithString:url]; + NSURL = objc.objc_getClass(b"NSURL") + urlWithString_ = objc.sel_registerName(b"URLWithString:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_void_p] + ns_url = objc.objc_msgSend(NSURL, urlWithString_, url_string) + + # Get the shared UIApplication instance + # This code is the equivalent of: + # UIApplication shared_app = [UIApplication sharedApplication] + UIApplication = objc.objc_getClass(b"UIApplication") + sharedApplication = objc.sel_registerName(b"sharedApplication") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + shared_app = objc.objc_msgSend(UIApplication, sharedApplication) + + # Open the URL on the shared application + # This code is the equivalent of: + # [shared_app openURL:ns_url + # options:NIL + # completionHandler:NIL]; + openURL_ = objc.sel_registerName(b"openURL:options:completionHandler:") + objc.objc_msgSend.argtypes = [ + c_void_p, c_void_p, c_void_p, c_void_p, c_void_p + ] + # Method returns void + objc.objc_msgSend.restype = None + objc.objc_msgSend(shared_app, openURL_, ns_url, None, None) + + return True + + +def parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser(description="Open URL in a web browser.") + parser.add_argument("url", help="URL to open") + + group = parser.add_mutually_exclusive_group() + group.add_argument("-n", "--new-window", action="store_const", + const=1, default=0, dest="new_win", + help="open new window") + group.add_argument("-t", "--new-tab", action="store_const", + const=2, default=0, dest="new_win", + help="open new tab") + + args = parser.parse_args(arg_list) + + return args + + +def main(arg_list: list[str] | None = None): + args = parse_args(arg_list) + + open(args.url, args.new_win) print("\a") + if __name__ == "__main__": main() diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py index fab851c5a44430..05d2ba4c664e5e 100644 --- a/Lib/wsgiref/headers.py +++ b/Lib/wsgiref/headers.py @@ -5,7 +5,7 @@ written by Barry Warsaw. """ -# Regular expression that matches `special' characters in parameters, the +# Regular expression that matches 'special' characters in parameters, the # existence of which force quoting of the parameter value. import re tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') diff --git a/Lib/xml/dom/__init__.py b/Lib/xml/dom/__init__.py index 97cf9a6429993d..dd7fb996afd616 100644 --- a/Lib/xml/dom/__init__.py +++ b/Lib/xml/dom/__init__.py @@ -137,4 +137,4 @@ class UserDataHandler: EMPTY_NAMESPACE = None EMPTY_PREFIX = None -from .domreg import getDOMImplementation, registerDOMImplementation +from .domreg import getDOMImplementation, registerDOMImplementation # noqa: F401 diff --git a/Lib/xml/etree/ElementInclude.py b/Lib/xml/etree/ElementInclude.py index 40a9b22292479f..986e6c3bbe90f6 100644 --- a/Lib/xml/etree/ElementInclude.py +++ b/Lib/xml/etree/ElementInclude.py @@ -79,8 +79,8 @@ class LimitedRecursiveIncludeError(FatalIncludeError): # @param parse Parse mode. Either "xml" or "text". # @param encoding Optional text encoding (UTF-8 by default for "text"). # @return The expanded resource. If the parse mode is "xml", this -# is an ElementTree instance. If the parse mode is "text", this -# is a Unicode string. If the loader fails, it can return None +# is an Element instance. If the parse mode is "text", this +# is a string. If the loader fails, it can return None # or raise an OSError exception. # @throws OSError If the loader fails to load the resource. @@ -98,7 +98,7 @@ def default_loader(href, parse, encoding=None): ## # Expand XInclude directives. # -# @param elem Root element. +# @param elem Root Element or any ElementTree of a tree to be expanded # @param loader Optional resource loader. If omitted, it defaults # to {@link default_loader}. If given, it should be a callable # that implements the same interface as default_loader. @@ -106,12 +106,13 @@ def default_loader(href, parse, encoding=None): # relative include file references. # @param max_depth The maximum number of recursive inclusions. # Limited to reduce the risk of malicious content explosion. -# Pass a negative value to disable the limitation. +# Pass None to disable the limitation. # @throws LimitedRecursiveIncludeError If the {@link max_depth} was exceeded. # @throws FatalIncludeError If the function fails to include a given # resource, or if the tree contains malformed XInclude elements. -# @throws IOError If the function fails to load a given resource. -# @returns the node or its replacement if it was an XInclude node +# @throws OSError If the function fails to load a given resource. +# @throws ValueError If negative {@link max_depth} is passed. +# @returns None. Modifies tree pointed by {@link elem} def include(elem, loader=None, base_url=None, max_depth=DEFAULT_MAX_INCLUSION_DEPTH): diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index a37fead41b750e..ce67d7d7d54748 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -201,7 +201,7 @@ def __len__(self): def __bool__(self): warnings.warn( - "Testing an element's truth value will raise an exception in " + "Testing an element's truth value will always return True in " "future versions. " "Use specific 'len(elem)' or 'elem is not None' test instead.", DeprecationWarning, stacklevel=2 @@ -1320,6 +1320,11 @@ def read_events(self): else: yield event + def flush(self): + if self._parser is None: + raise ValueError("flush() called after end of stream") + self._parser.flush() + def XML(text, parser=None): """Parse XML document from string constant. @@ -1726,6 +1731,15 @@ def close(self): del self.parser, self._parser del self.target, self._target + def flush(self): + was_enabled = self.parser.GetReparseDeferralEnabled() + try: + self.parser.SetReparseDeferralEnabled(False) + self.parser.Parse(b"", False) + except self._error as v: + self._raiseerror(v) + finally: + self.parser.SetReparseDeferralEnabled(was_enabled) # -------------------------------------------------------------------- # C14N 2.0 diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index b657310207cfe5..fe4582c6f8b758 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -21,9 +21,9 @@ from .xmlreader import InputSource from .handler import ContentHandler, ErrorHandler -from ._exceptions import SAXException, SAXNotRecognizedException, \ - SAXParseException, SAXNotSupportedException, \ - SAXReaderNotAvailable +from ._exceptions import (SAXException, SAXNotRecognizedException, + SAXParseException, SAXNotSupportedException, + SAXReaderNotAvailable) def parse(source, handler, errorHandler=ErrorHandler()): @@ -55,7 +55,7 @@ def parseString(string, handler, errorHandler=ErrorHandler()): # tell modulefinder that importing sax potentially imports expatreader _false = 0 if _false: - import xml.sax.expatreader + import xml.sax.expatreader # noqa: F401 import os, sys if not sys.flags.ignore_environment and "PY_SAX_PARSER" in os.environ: @@ -92,3 +92,9 @@ def make_parser(parser_list=()): def _create_parser(parser_name): drv_module = __import__(parser_name,{},{},['create_parser']) return drv_module.create_parser() + + +__all__ = ['ContentHandler', 'ErrorHandler', 'InputSource', 'SAXException', + 'SAXNotRecognizedException', 'SAXNotSupportedException', + 'SAXParseException', 'SAXReaderNotAvailable', + 'default_parser_list', 'make_parser', 'parse', 'parseString'] diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index b9ad52692db8dd..ba3c1e98517429 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -214,6 +214,20 @@ def feed(self, data, isFinal=False): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def flush(self): + if self._parser is None: + return + + was_enabled = self._parser.GetReparseDeferralEnabled() + try: + self._parser.SetReparseDeferralEnabled(False) + self._parser.Parse(b"", False) + except expat.error as e: + exc = SAXParseException(expat.ErrorString(e.code), e, self) + self._err_handler.fatalError(exc) + finally: + self._parser.SetReparseDeferralEnabled(was_enabled) + def _close_source(self): source = self._source try: diff --git a/Lib/xml/sax/handler.py b/Lib/xml/sax/handler.py index e8d417e5194232..3183c3fe96d74f 100644 --- a/Lib/xml/sax/handler.py +++ b/Lib/xml/sax/handler.py @@ -371,7 +371,7 @@ def startDTD(self, name, public_id, system_id): name is the name of the document element type, public_id the public identifier of the DTD (or None if none were supplied) - and system_id the system identfier of the external subset (or + and system_id the system identifier of the external subset (or None if none were supplied).""" def endDTD(self): diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 4dddb1d10e08bd..90a356fbb8eae4 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -578,6 +578,7 @@ class SimpleXMLRPCServer(socketserver.TCPServer, """ allow_reuse_address = True + allow_reuse_port = True # Warning: this is for debugging purposes only! Never set this to True in # production code, as will be sending out sensitive information (exception diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 8005b4b34ccf76..e2aaf8bab4913d 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -605,7 +605,15 @@ def from_file(cls, filename, arcname=None, *, strict_timestamps=True): def is_dir(self): """Return True if this archive member is a directory.""" - return self.filename.endswith('/') + if self.filename.endswith('/'): + return True + # The ZIP format specification requires to use forward slashes + # as the directory separator, but in practice some ZIP files + # created on Windows can use backward slashes. For compatibility + # with the extraction code which already handles this: + if os.path.altsep: + return self.filename.endswith((os.path.sep, os.path.altsep)) + return False # ZIP encryption uses the CRC32 one-byte primitive for scrambling some @@ -932,7 +940,7 @@ def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] if not self.closed: - result.append(' name=%r mode=%r' % (self.name, self.mode)) + result.append(' name=%r' % (self.name,)) if self._compress_type != ZIP_STORED: result.append(' compress_type=%s' % compressor_names.get(self._compress_type, @@ -1209,6 +1217,14 @@ def __init__(self, zf, zinfo, zip64): def _fileobj(self): return self._zipfile.fp + @property + def name(self): + return self._zinfo.filename + + @property + def mode(self): + return 'wb' + def writable(self): return True @@ -1577,7 +1593,8 @@ def comment(self, comment): self._didModify = True def read(self, name, pwd=None): - """Return file bytes for name.""" + """Return file bytes for name. 'pwd' is the password to decrypt + encrypted files.""" with self.open(name, "r", pwd) as fp: return fp.read() @@ -1678,7 +1695,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): else: pwd = None - return ZipExtFile(zef_file, mode, zinfo, pwd, True) + return ZipExtFile(zef_file, mode + 'b', zinfo, pwd, True) except: zef_file.close() raise @@ -1728,8 +1745,9 @@ def _open_to_write(self, zinfo, force_zip64=False): def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. + as possible. 'member' may be a filename or a ZipInfo object. You can + specify a different directory using 'path'. You can specify the + password to decrypt the file using 'pwd'. """ if path is None: path = os.getcwd() @@ -1740,9 +1758,10 @@ def extract(self, member, path=None, pwd=None): def extractall(self, path=None, members=None, pwd=None): """Extract all members from the archive to the current working - directory. `path' specifies a different directory to extract to. - `members' is optional and must be a subset of the list returned - by namelist(). + directory. 'path' specifies a different directory to extract to. + 'members' is optional and must be a subset of the list returned + by namelist(). You can specify the password to decrypt all files + using 'pwd'. """ if members is None: members = self.namelist() @@ -1802,11 +1821,15 @@ def _extract_member(self, member, targetpath, pwd): # Create all upper directories if necessary. upperdirs = os.path.dirname(targetpath) if upperdirs and not os.path.exists(upperdirs): - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if member.is_dir(): if not os.path.isdir(targetpath): - os.mkdir(targetpath) + try: + os.mkdir(targetpath) + except FileExistsError: + if not os.path.isdir(targetpath): + raise return targetpath with self.open(member, pwd=pwd) as source, \ diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 78c413563bb2b1..c0e53e273cfaac 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -1,3 +1,12 @@ +""" +A Path-like interface for zipfiles. + +This codebase is shared between zipfile.Path in the stdlib +and zipp in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" + import io import posixpath import zipfile @@ -5,8 +14,10 @@ import contextlib import pathlib import re +import stat +import sys -from .glob import translate +from .glob import Translator __all__ = ['Path'] @@ -34,7 +45,7 @@ def _parents(path): def _ancestry(path): """ Given a path with elements separated by - posixpath.sep, generate all elements of that path + posixpath.sep, generate all elements of that path. >>> list(_ancestry('b/d')) ['b/d', 'b'] @@ -46,9 +57,14 @@ def _ancestry(path): ['b'] >>> list(_ancestry('')) [] + + Multiple separators are treated like a single. + + >>> list(_ancestry('//b//d///f//')) + ['//b//d///f', '//b//d', '//b'] """ path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: + while path.rstrip(posixpath.sep): yield path path, tail = posixpath.split(path) @@ -147,6 +163,16 @@ def make(cls, source): source.__class__ = cls return source + @classmethod + def inject(cls, zf: zipfile.ZipFile) -> zipfile.ZipFile: + """ + Given a writable zip file zf, inject directory entries for + any directories implied by the presence of children. + """ + for name in cls._implied_dirs(zf.namelist()): + zf.writestr(name, b"") + return zf + class FastLookup(CompleteDirs): """ @@ -168,13 +194,18 @@ def _name_set(self): def _extract_text_encoding(encoding=None, *args, **kwargs): - # stacklevel=3 so that the caller of the caller see any warning. - return io.text_encoding(encoding, 3), args, kwargs + # compute stack level so that the caller of the caller sees any warning. + is_pypy = sys.implementation.name == 'pypy' + stack_level = 3 + is_pypy + return io.text_encoding(encoding, stack_level), args, kwargs class Path: """ - A pathlib-compatible interface for zip files. + A :class:`importlib.resources.abc.Traversable` interface for zip files. + + Implements many of the features users enjoy from + :class:`pathlib.Path`. Consider a zip file with this structure:: @@ -194,13 +225,13 @@ class Path: Path accepts the zipfile object itself or a filename - >>> root = Path(zf) + >>> path = Path(zf) From there, several path operations are available. Directory iteration (including the zip file itself): - >>> a, b = root.iterdir() + >>> a, b = path.iterdir() >>> a Path('mem/abcde.zip', 'a.txt') >>> b @@ -238,16 +269,38 @@ class Path: 'mem/abcde.zip/b/c.txt' At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. + resolve to the zipfile. - >>> root.name + >>> str(path) + 'mem/abcde.zip/' + >>> path.name 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) + >>> path.filename == pathlib.Path('mem/abcde.zip') + True + >>> str(path.parent) 'mem' + + If the zipfile has no filename, such attributes are not + valid and accessing them will raise an Exception. + + >>> zf.filename = None + >>> path.name + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.filename + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.parent + Traceback (most recent call last): + ... + TypeError: ... + + # workaround python/cpython#106763 + >>> pass """ __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" @@ -355,16 +408,19 @@ def match(self, path_pattern): def is_symlink(self): """ - Return whether this path is a symlink. Always false (python/cpython#82102). + Return whether this path is a symlink. """ - return False + info = self.root.getinfo(self.at) + mode = info.external_attr >> 16 + return stat.S_ISLNK(mode) def glob(self, pattern): if not pattern: raise ValueError(f"Unacceptable pattern: {pattern!r}") prefix = re.escape(self.at) - matches = re.compile(prefix + translate(pattern)).fullmatch + tr = Translator(seps='/') + matches = re.compile(prefix + tr.translate(pattern)).fullmatch return map(self._next, filter(matches, self.root.namelist())) def rglob(self, pattern): diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py index 4a2e665e27078a..4320f1c0badcf9 100644 --- a/Lib/zipfile/_path/glob.py +++ b/Lib/zipfile/_path/glob.py @@ -1,18 +1,105 @@ +import os import re -def translate(pattern): - r""" - Given a glob pattern, produce a regex that matches it. +_default_seps = os.sep + str(os.altsep) * bool(os.altsep) - >>> translate('*.txt') - '[^/]*\\.txt' - >>> translate('a?txt') - 'a.txt' - >>> translate('**/*') - '.*/[^/]*' + +class Translator: + """ + >>> Translator('xyz') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + + >>> Translator('') + Traceback (most recent call last): + ... + AssertionError: Invalid separators """ - return ''.join(map(replace, separate(pattern))) + + seps: str + + def __init__(self, seps: str = _default_seps): + assert seps and set(seps) <= set(_default_seps), "Invalid separators" + self.seps = seps + + def translate(self, pattern): + """ + Given a glob pattern, produce a regex that matches it. + """ + return self.extend(self.match_dirs(self.translate_core(pattern))) + + def extend(self, pattern): + r""" + Extend regex for pattern-wide concerns. + + Apply '(?s:)' to create a non-matching group that + matches newlines (valid on Unix). + + Append '\Z' to imply fullmatch even when match is used. + """ + return rf'(?s:{pattern})\Z' + + def match_dirs(self, pattern): + """ + Ensure that zipfile.Path directory names are matched. + + zipfile.Path directory names always end in a slash. + """ + return rf'{pattern}[/]?' + + def translate_core(self, pattern): + r""" + Given a glob pattern, produce a regex that matches it. + + >>> t = Translator() + >>> t.translate_core('*.txt').replace('\\\\', '') + '[^/]*\\.txt' + >>> t.translate_core('a?txt') + 'a[^/]txt' + >>> t.translate_core('**/*').replace('\\\\', '') + '.*/[^/][^/]*' + """ + self.restrict_rglob(pattern) + return ''.join(map(self.replace, separate(self.star_not_empty(pattern)))) + + def replace(self, match): + """ + Perform the replacements for a match from :func:`separate`. + """ + return match.group('set') or ( + re.escape(match.group(0)) + .replace('\\*\\*', r'.*') + .replace('\\*', rf'[^{re.escape(self.seps)}]*') + .replace('\\?', r'[^/]') + ) + + def restrict_rglob(self, pattern): + """ + Raise ValueError if ** appears in anything but a full path segment. + + >>> Translator().translate('**foo') + Traceback (most recent call last): + ... + ValueError: ** must appear alone in a path segment + """ + seps_pattern = rf'[{re.escape(self.seps)}]+' + segments = re.split(seps_pattern, pattern) + if any('**' in segment and segment != '**' for segment in segments): + raise ValueError("** must appear alone in a path segment") + + def star_not_empty(self, pattern): + """ + Ensure that * will not match an empty segment. + """ + + def handle_segment(match): + segment = match.group(0) + return '?*' if segment == '*' else segment + + not_seps_pattern = rf'[^{re.escape(self.seps)}]+' + return re.sub(not_seps_pattern, handle_segment, pattern) def separate(pattern): @@ -25,16 +112,3 @@ def separate(pattern): ['a', '[?]', 'txt'] """ return re.finditer(r'([^\[]+)|(?P[\[].*?[\]])|([\[][^\]]*$)', pattern) - - -def replace(match): - """ - Perform the replacements for a match from :func:`separate`. - """ - - return match.group('set') or ( - re.escape(match.group(0)) - .replace('\\*\\*', r'.*') - .replace('\\*', r'[^/]*') - .replace('\\?', r'.') - ) diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 823a82ee830465..e5192c4d074c4b 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -1,11 +1,9 @@ """zipimport provides support for importing Python modules from Zip archives. -This module exports three objects: +This module exports two objects: - zipimporter: a class; its constructor takes a path to a Zip archive. - ZipImportError: exception raised by zipimporter objects. It's a subclass of ImportError, so it can be caught as ImportError, too. -- _zip_directory_cache: a dict, mapping archive paths to zip directory - info dicts, as used in zipimporter._files. It is usually not needed to use the zipimport module explicitly; it is used by the builtin import mechanism for sys.path items that are paths @@ -15,7 +13,7 @@ #from importlib import _bootstrap_external #from importlib import _bootstrap # for _verbose_message import _frozen_importlib_external as _bootstrap_external -from _frozen_importlib_external import _unpack_uint16, _unpack_uint32 +from _frozen_importlib_external import _unpack_uint16, _unpack_uint32, _unpack_uint64 import _frozen_importlib as _bootstrap # for _verbose_message import _imp # for check_hash_based_pycs import _io # for open @@ -40,8 +38,14 @@ class ZipImportError(ImportError): _module_type = type(sys) END_CENTRAL_DIR_SIZE = 22 -STRING_END_ARCHIVE = b'PK\x05\x06' +END_CENTRAL_DIR_SIZE_64 = 56 +END_CENTRAL_DIR_LOCATOR_SIZE_64 = 20 +STRING_END_ARCHIVE = b'PK\x05\x06' # standard EOCD signature +STRING_END_LOCATOR_64 = b'PK\x06\x07' # Zip64 EOCD Locator signature +STRING_END_ZIP_64 = b'PK\x06\x06' # Zip64 EOCD signature MAX_COMMENT_LEN = (1 << 16) - 1 +MAX_UINT32 = 0xffffffff +ZIP64_EXTRA_TAG = 0x1 class zipimporter(_bootstrap_external._LoaderBasics): """zipimporter(archivepath) -> zipimporter object @@ -151,6 +155,8 @@ def get_data(self, pathname): toc_entry = self._get_files()[key] except KeyError: raise OSError(0, '', key) + if toc_entry is None: + return b'' return _get_data(self.archive, toc_entry) @@ -250,17 +256,9 @@ def load_module(self, fullname): def get_resource_reader(self, fullname): - """Return the ResourceReader for a package in a zip file. - - If 'fullname' is a package within the zip file, return the - 'ResourceReader' object for the package. Otherwise return None. - """ - try: - if not self.is_package(fullname): - return None - except ZipImportError: - return None + """Return the ResourceReader for a module in a zip file.""" from importlib.readers import ZipReader + return ZipReader(self, fullname) @@ -356,49 +354,72 @@ def _read_directory(archive): # to not cause problems when some runs 'python3 /dev/fd/9 9= 0 and pos64+END_CENTRAL_DIR_SIZE_64+END_CENTRAL_DIR_LOCATOR_SIZE_64==pos): + # Zip64 at "correct" offset from standard EOCD + buffer = data[pos64:pos64 + END_CENTRAL_DIR_SIZE_64] + if len(buffer) != END_CENTRAL_DIR_SIZE_64: + raise ZipImportError( + f"corrupt Zip64 file: Expected {END_CENTRAL_DIR_SIZE_64} byte " + f"zip64 central directory, but read {len(buffer)} bytes.", + path=archive) + header_position = file_size - len(data) + pos64 + + central_directory_size = _unpack_uint64(buffer[40:48]) + central_directory_position = _unpack_uint64(buffer[48:56]) + num_entries = _unpack_uint64(buffer[24:32]) + elif pos >= 0: buffer = data[pos:pos+END_CENTRAL_DIR_SIZE] if len(buffer) != END_CENTRAL_DIR_SIZE: raise ZipImportError(f"corrupt Zip file: {archive!r}", path=archive) + header_position = file_size - len(data) + pos - header_size = _unpack_uint32(buffer[12:16]) - header_offset = _unpack_uint32(buffer[16:20]) - if header_position < header_size: + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + central_directory_size = _unpack_uint32(buffer[12:16]) + central_directory_position = _unpack_uint32(buffer[16:20]) + num_entries = _unpack_uint16(buffer[8:10]) + + # N.b. if someday you want to prefer the standard (non-zip64) EOCD, + # you need to adjust position by 76 for arc to be 0. + else: + raise ZipImportError(f'not a Zip file: {archive!r}', + path=archive) + + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + # XXX: These are cursory checks but are not as exact or strict as they + # could be. Checking the arc-adjusted value is probably good too. + if header_position < central_directory_size: raise ZipImportError(f'bad central directory size: {archive!r}', path=archive) - if header_position < header_offset: + if header_position < central_directory_position: raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive) - header_position -= header_size - arc_offset = header_position - header_offset + header_position -= central_directory_size + # On just-a-zipfile these values are the same and arc_offset is zero; if + # the file has some bytes prepended, `arc_offset` is the number of such + # bytes. This is used for pex as well as self-extracting .exe. + arc_offset = header_position - central_directory_position if arc_offset < 0: raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive) @@ -415,6 +436,11 @@ def _read_directory(archive): raise EOFError('EOF read where not expected') # Start of file header if buffer[:4] != b'PK\x01\x02': + if count != num_entries: + raise ZipImportError( + f"mismatched num_entries: {count} should be {num_entries} in {archive!r}", + path=archive, + ) break # Bad: Central Dir File Header if len(buffer) != 46: raise EOFError('EOF read where not expected') @@ -430,9 +456,6 @@ def _read_directory(archive): comment_size = _unpack_uint16(buffer[32:34]) file_offset = _unpack_uint32(buffer[42:46]) header_size = name_size + extra_size + comment_size - if file_offset > header_offset: - raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) - file_offset += arc_offset try: name = fp.read(name_size) @@ -444,7 +467,10 @@ def _read_directory(archive): # slower than reading the data because fseek flushes stdio's # internal buffers. See issue #8745. try: - if len(fp.read(header_size - name_size)) != header_size - name_size: + extra_data_len = header_size - name_size + extra_data = memoryview(fp.read(extra_data_len)) + + if len(extra_data) != extra_data_len: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) @@ -461,12 +487,83 @@ def _read_directory(archive): name = name.replace('/', path_sep) path = _bootstrap_external._path_join(archive, name) + + # Ordering matches unpacking below. + if ( + file_size == MAX_UINT32 or + data_size == MAX_UINT32 or + file_offset == MAX_UINT32 + ): + # need to decode extra_data looking for a zip64 extra (which might not + # be present) + while extra_data: + if len(extra_data) < 4: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + tag = _unpack_uint16(extra_data[:2]) + size = _unpack_uint16(extra_data[2:4]) + if len(extra_data) < 4 + size: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + if tag == ZIP64_EXTRA_TAG: + if (len(extra_data) - 4) % 8 != 0: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + num_extra_values = (len(extra_data) - 4) // 8 + if num_extra_values > 3: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + import struct + values = list(struct.unpack_from(f"<{min(num_extra_values, 3)}Q", + extra_data, offset=4)) + + # N.b. Here be dragons: the ordering of these is different than + # the header fields, and it's really easy to get it wrong since + # naturally-occurring zips that use all 3 are >4GB + if file_size == MAX_UINT32: + file_size = values.pop(0) + if data_size == MAX_UINT32: + data_size = values.pop(0) + if file_offset == MAX_UINT32: + file_offset = values.pop(0) + + break + + # For a typical zip, this bytes-slicing only happens 2-3 times, on + # small data like timestamps and filesizes. + extra_data = extra_data[4+size:] + else: + _bootstrap._verbose_message( + "zipimport: suspected zip64 but no zip64 extra for {!r}", + path, + ) + # XXX These two statements seem swapped because `central_directory_position` + # is a position within the actual file, but `file_offset` (when compared) is + # as encoded in the entry, not adjusted for this file. + # N.b. this must be after we've potentially read the zip64 extra which can + # change `file_offset`. + if file_offset > central_directory_position: + raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) + file_offset += arc_offset + t = (path, compress, data_size, file_size, file_offset, time, date, crc) files[name] = t count += 1 finally: fp.seek(start_offset) _bootstrap._verbose_message('zipimport: found {} names in {!r}', count, archive) + + # Add implicit directories. + count = 0 + for name in list(files): + while True: + i = name.rstrip(path_sep).rfind(path_sep) + if i < 0: + break + name = name[:i + 1] + if name in files: + break + files[name] = None + count += 1 + if count: + _bootstrap._verbose_message('zipimport: added {} implicit directories in {!r}', + count, archive) return files # During bootstrap, we may need to load the encodings @@ -600,7 +697,7 @@ def _unmarshal_code(self, pathname, fullpath, fullname, data): source_bytes = _get_pyc_source(self, fullpath) if source_bytes is not None: source_hash = _imp.source_hash( - _bootstrap_external._RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) diff --git a/Mac/BuildScript/README.rst b/Mac/BuildScript/README.rst index 4f74e7dc00520a..e44e74f3a49234 100644 --- a/Mac/BuildScript/README.rst +++ b/Mac/BuildScript/README.rst @@ -17,7 +17,7 @@ Gatekeeper download quarantine, the final package must be signed with a valid Apple Developer ID certificate using productsign. Starting with macOS 10.15 Catalina, Gatekeeper now also requires that installer packages are submitted to and pass Apple's automated -notarization service using the altool command. To pass notarization, +notarization service using the ``notarytool`` command. To pass notarization, the binaries included in the package must be built with at least the macOS 10.9 SDK, must now be signed with the codesign utility, and executables must opt in to the hardened run time option with @@ -27,7 +27,7 @@ available in the on-line Apple Developer Documentation and man pages. A goal of PSF-provided (python.org) Python binaries for macOS is to support a wide-range of operating system releases with one set of binaries. Currently, the oldest release supported by python.org -binaries is macOS 10.9; it is still possible to build Python and +binaries is macOS 10.9; it should still be possible to build Python and Python installers on older versions of macOS but we not regularly test on those systems nor provide binaries for them. @@ -49,20 +49,17 @@ Starting with 3.9.1, Python fully supports macOS "weaklinking", meaning it is now possible to build a Python on a current macOS version with a deployment target of an earlier macOS system. For 3.9.1 and later systems, we provide a "macOS 64-bit universal2 installer" -variant, currently build on macOS 11 Big Sur with fat binaries +variant, currently built on macOS 11 Big Sur with fat binaries natively supporting both Apple Silicon (arm64) and Intel-64 (x86_64) Macs running macOS 10.9 or later. -The legacy "macOS 64-bit Intel installer" variant is expected to -be retired prior to the end of 3.9.x support. - build-installer.py requires Apple Developer tools, either from the Command Line Tools package or from a full Xcode installation. You should use the most recent version of either for the operating system version in use. (One notable exception: on macOS 10.6, Snow Leopard, use Xcode 3, not Xcode 4 which was released later in the 10.6 support cycle.) build-installer.py also must be run -with recent versions of Python 3.x or 2.7. On older systems, +with recent versions of Python 3.x. On older systems, due to changes in TLS practices, it may be easier to manually download and cache third-party source distributions used by build-installer.py rather than have it attempt to automatically @@ -76,12 +73,12 @@ download them. - builds the following third-party libraries - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 + * OpenSSL 3.0.x + * Tcl/Tk 8.6.x * NCurses * SQLite * XZ - * libffi + * mpdecimal - uses system-supplied versions of third-party libraries @@ -97,35 +94,6 @@ download them. * ``MACOSX_DEPLOYMENT_TARGET=10.9`` * Apple ``clang`` -2. legacy Intel 64-bit, x86_64, for OS X 10.9 (and later):: - - /path/to/bootstrap/python3 build-installer.py \ - --universal-archs=intel-64 \ - --dep-target=10.9 - - - builds the following third-party libraries - - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 - * NCurses - * SQLite - * XZ - * libffi - - - uses system-supplied versions of third-party libraries - - * readline module links with Apple BSD editline (libedit) - * zlib - * bz2 - - - recommended build environment: - - * Mac OS X 10.9.5 - * Xcode Command Line Tools 6.2 - * ``MacOSX10.9`` SDK - * ``MACOSX_DEPLOYMENT_TARGET=10.9`` - * Apple ``clang`` - General Prerequisites --------------------- diff --git a/Mac/BuildScript/backport_gh92603_fix.patch b/Mac/BuildScript/backport_gh92603_fix.patch deleted file mode 100644 index 9a37b029650340..00000000000000 --- a/Mac/BuildScript/backport_gh92603_fix.patch +++ /dev/null @@ -1,82 +0,0 @@ -Accepted upstream for release in Tk 8.6.14: -https://core.tcl-lang.org/tk/info/cf3830280b - ---- tk8.6.13/macosx/tkMacOSXWindowEvent.c.orig -+++ tk8.6.13-patched/macosx/tkMacOSXWindowEvent.c -@@ -239,8 +239,8 @@ extern NSString *NSWindowDidOrderOffScreenNotification; - if (winPtr) { - TKContentView *view = [window contentView]; - --#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101500 -- if (@available(macOS 10.15, *)) { -+#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101400 -+ if (@available(macOS 10.14, *)) { - [view viewDidChangeEffectiveAppearance]; - } - #endif -@@ -1237,29 +1237,8 @@ static const char *const accentNames[] = { - } else if (effectiveAppearanceName == NSAppearanceNameDarkAqua) { - TkSendVirtualEvent(tkwin, "DarkAqua", NULL); - } -- if ([NSApp macOSVersion] < 101500) { -- -- /* -- * Mojave cannot handle the KVO shenanigans that we need for the -- * highlight and accent color notifications. -- */ -- -- return; -- } - if (!defaultColor) { - defaultColor = [NSApp macOSVersion] < 110000 ? "Blue" : "Multicolor"; -- preferences = [[NSUserDefaults standardUserDefaults] retain]; -- -- /* -- * AppKit calls this method when the user changes the Accent Color -- * but not when the user changes the Highlight Color. So we register -- * to receive KVO notifications for Highlight Color as well. -- */ -- -- [preferences addObserver:self -- forKeyPath:@"AppleHighlightColor" -- options:NSKeyValueObservingOptionNew -- context:NULL]; - } - NSString *accent = [preferences stringForKey:@"AppleAccentColor"]; - NSArray *words = [[preferences stringForKey:@"AppleHighlightColor"] ---- tk8.6.13/macosx/tkMacOSXWm.c.orig -+++ tk8.6.13-patched/macosx/tkMacOSXWm.c -@@ -1289,6 +1289,11 @@ TkWmDeadWindow( - [NSApp _setMainWindow:nil]; - } - [deadNSWindow close]; -+#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101400 -+ NSUserDefaults *preferences = [NSUserDefaults standardUserDefaults]; -+ [preferences removeObserver:deadNSWindow.contentView -+ forKeyPath:@"AppleHighlightColor"]; -+#endif - [deadNSWindow release]; - - #if DEBUG_ZOMBIES > 1 -@@ -6763,6 +6768,21 @@ TkMacOSXMakeRealWindowExist( - } - TKContentView *contentView = [[TKContentView alloc] - initWithFrame:NSZeroRect]; -+#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101400 -+ NSUserDefaults *preferences = [NSUserDefaults standardUserDefaults]; -+ -+ /* -+ * AppKit calls the viewDidChangeEffectiveAppearance method when the -+ * user changes the Accent Color but not when the user changes the -+ * Highlight Color. So we register to receive KVO notifications for -+ * Highlight Color as well. -+ */ -+ -+ [preferences addObserver:contentView -+ forKeyPath:@"AppleHighlightColor" -+ options:NSKeyValueObservingOptionNew -+ context:NULL]; -+#endif - [window setContentView:contentView]; - [contentView release]; - [window setDelegate:NSApp]; diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 0af90563cbbb2b..f5f0ed44884142 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.13", - url="https://www.openssl.org/source/openssl-3.0.13.tar.gz", - checksum='88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313', + name="OpenSSL 3.0.15", + url="https://github.com/openssl/openssl/releases/download/openssl-3.0.15/openssl-3.0.15.tar.gz", + checksum='23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533', buildrecipe=build_universal_openssl, configure=None, install=None, @@ -264,11 +264,11 @@ def library_recipes(): tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.13' - tcl_checksum='43a1fae7412f61ff11de2cfd05d28cfc3a73762f354a417c62370a54e2caf066' + tcl_tk_ver='8.6.15' + tcl_checksum='861e159753f2e2fbd6ec1484103715b0be56be3357522b858d3cbb5f893ffef1' - tk_checksum='2e65fa069a23365440a3c56c556b8673b5e32a283800d8d9b257e3f584ce0675' - tk_patches = ['backport_gh92603_fix.patch', 'backport_gh71383_fix.patch', 'backport_gh110950_fix.patch'] + tk_checksum='550969f35379f952b3020f3ab7b9dd5bfd11c1ef7c9b7c6a75f5c49aca793fec' + tk_patches = [] base_url = "https://prdownloads.sourceforge.net/tcl/{what}{version}-src.tar.gz" @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.45.1", - url="https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz", - checksum="cd9c27841b7a5932c9897651e20b86c701dd740556989b01ca596fcfa3d49a0a", + name="SQLite 3.45.3", + url="https://sqlite.org/2024/sqlite-autoconf-3450300.tar.gz", + checksum="b2809ca53124c19c60f42bf627736eae011afdcc205bb48270a5ee9a38191531", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -378,6 +378,15 @@ def library_recipes(): '--disable-dependency-tracking', ] ), + dict( + name="libmpdec 4.0.0", + url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz", + checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c", + configure_pre=[ + "--disable-cxx", + "MACHINE=universal", + ] + ), ]) if not PYTHON_3: @@ -1150,6 +1159,7 @@ def buildPython(): print(" NOTE: --with-mimalloc=no pending resolution of weak linking issues") runCommand("%s -C --enable-framework --enable-universalsdk=/ " "--with-mimalloc=no " + "--with-system-libmpdec " "--with-universal-archs=%s " "%s " "%s " diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index efd76b9b1ae64b..ee5ba4707dfea4 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fswiss\fcharset0 Helvetica-Oblique; \f3\fmodern\fcharset0 CourierNewPSMT;\f4\fmodern\fcharset0 Courier;} {\colortbl;\red255\green255\blue255;} @@ -11,7 +11,7 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f1\b \cf0 NOTE: -\f0\b0 This is an alpha preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 @@ -69,7 +69,7 @@ Due to new security checks on macOS 10.15 Catalina, when launching IDLE macOS ma \f1\b \ul Apple Silicon Mac support\ \f0\b0 \ulnone \ -On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the +On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the \f4 python3-intel64 \f0 command instead of just \f4 python3 diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 79851e1f4a69cc..49d6e22286be26 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; } {\colortbl;\red255\green255\blue255;} @@ -26,5 +26,5 @@ At the end of this install, click on \ \f1\b NOTE: -\f0\b0 This is an alpha test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file diff --git a/Mac/BuildScript/scripts/postflight.patch-profile b/Mac/BuildScript/scripts/postflight.patch-profile index 68b8e4bb044e10..9caf62211ddd16 100755 --- a/Mac/BuildScript/scripts/postflight.patch-profile +++ b/Mac/BuildScript/scripts/postflight.patch-profile @@ -77,16 +77,17 @@ bash) fi ;; fish) - CONFIG_DIR="${HOME}/.config/fish" - RC="${CONFIG_DIR}/config.fish" + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" mkdir -p "$CONFIG_DIR" if [ -f "${RC}" ]; then cp -fp "${RC}" "${RC}.pysave" fi - echo "" >> "${RC}" - echo "# Setting PATH for Python ${PYVER}" >> "${RC}" - echo "# The original version is saved in ${RC}.pysave" >> "${RC}" - echo "set -x PATH \"${PYTHON_ROOT}/bin\" \"\$PATH\"" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" if [ `id -ur` = 0 ]; then chown "${USER}" "${RC}" fi diff --git a/Mac/Resources/app-store-compliance.patch b/Mac/Resources/app-store-compliance.patch new file mode 100644 index 00000000000000..f4b7decc01cf1f --- /dev/null +++ b/Mac/Resources/app-store-compliance.patch @@ -0,0 +1,29 @@ +diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py +index d6c83a75c1c..19ed4e01091 100644 +--- a/Lib/test/test_urlparse.py ++++ b/Lib/test/test_urlparse.py +@@ -237,11 +237,6 @@ def test_roundtrips(self): + '','',''), + ('git+ssh', 'git@github.com','/user/project.git', + '', '')), +- ('itms-services://?action=download-manifest&url=https://example.com/app', +- ('itms-services', '', '', '', +- 'action=download-manifest&url=https://example.com/app', ''), +- ('itms-services', '', '', +- 'action=download-manifest&url=https://example.com/app', '')), + ('+scheme:path/to/file', + ('', '', '+scheme:path/to/file', '', '', ''), + ('', '', '+scheme:path/to/file', '', '')), +diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py +index 8f724f907d4..148caf742c9 100644 +--- a/Lib/urllib/parse.py ++++ b/Lib/urllib/parse.py +@@ -59,7 +59,7 @@ + 'imap', 'wais', 'file', 'mms', 'https', 'shttp', + 'snews', 'prospero', 'rtsp', 'rtsps', 'rtspu', 'rsync', + 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', +- 'ws', 'wss', 'itms-services'] ++ 'ws', 'wss'] + + uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', + 'https', 'shttp', 'rtsp', 'rtsps', 'rtspu', 'sip', diff --git a/Makefile.pre.in b/Makefile.pre.in index 4dabe328ce0362..b0263f9f4c21da 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -40,8 +40,10 @@ LINKCC= @LINKCC@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ +ABIFLAGS= @ABIFLAGS@ +ABI_THREAD= @ABI_THREAD@ LDVERSION= @LDVERSION@ -LIBPYTHON= @LIBPYTHON@ +LIBPYTHON=@LIBPYTHON@ GITVERSION= @GITVERSION@ GITTAG= @GITTAG@ GITBRANCH= @GITBRANCH@ @@ -139,6 +141,13 @@ prefix= @prefix@ # Install prefix for architecture-dependent files exec_prefix= @exec_prefix@ +# For cross compilation, we distinguish between "prefix" (where we install the +# files) and "host_prefix" (where getpath.c expects to find the files at +# runtime) +host_prefix= @host_prefix@ +host_exec_prefix= @host_exec_prefix@ + + # Install prefix for data files datarootdir= @datarootdir@ @@ -150,7 +159,6 @@ INCLUDEDIR= @includedir@ CONFINCLUDEDIR= $(exec_prefix)/include PLATLIBDIR= @PLATLIBDIR@ SCRIPTDIR= $(prefix)/$(PLATLIBDIR) -ABIFLAGS= @ABIFLAGS@ # executable name for shebangs EXENAME= $(BINDIR)/python$(LDVERSION)$(EXE) # Variable used by ensurepip @@ -158,7 +166,7 @@ WHEEL_PKG_DIR= @WHEEL_PKG_DIR@ # Detailed destination directories BINLIBDEST= @BINLIBDEST@ -LIBDEST= $(SCRIPTDIR)/python$(VERSION) +LIBDEST= $(SCRIPTDIR)/python$(VERSION)$(ABI_THREAD) INCLUDEPY= $(INCLUDEDIR)/python$(LDVERSION) CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) @@ -167,7 +175,7 @@ SHLIB_SUFFIX= @SHLIB_SUFFIX@ EXT_SUFFIX= @EXT_SUFFIX@ LDSHARED= @LDSHARED@ $(PY_LDFLAGS) BLDSHARED= @BLDSHARED@ $(PY_CORE_LDFLAGS) -LDCXXSHARED= @LDCXXSHARED@ +LDCXXSHARED= @LDCXXSHARED@ $(PY_LDFLAGS) DESTSHARED= $(BINLIBDEST)/lib-dynload # List of exported symbols for AIX @@ -178,18 +186,29 @@ EXPORTSFROM= @EXPORTSFROM@ EXE= @EXEEXT@ BUILDEXE= @BUILDEXEEXT@ +# Name of the patch file to apply for app store compliance +APP_STORE_COMPLIANCE_PATCH=@APP_STORE_COMPLIANCE_PATCH@ + # Short name and location for Mac OS X Python framework UNIVERSALSDK=@UNIVERSALSDK@ PYTHONFRAMEWORK= @PYTHONFRAMEWORK@ PYTHONFRAMEWORKDIR= @PYTHONFRAMEWORKDIR@ PYTHONFRAMEWORKPREFIX= @PYTHONFRAMEWORKPREFIX@ PYTHONFRAMEWORKINSTALLDIR= @PYTHONFRAMEWORKINSTALLDIR@ -# Deployment target selected during configure, to be checked +PYTHONFRAMEWORKINSTALLNAMEPREFIX= @PYTHONFRAMEWORKINSTALLNAMEPREFIX@ +RESSRCDIR= @RESSRCDIR@ +# macOS deployment target selected during configure, to be checked # by distutils. The export statement is needed to ensure that the # deployment target is active during build. MACOSX_DEPLOYMENT_TARGET=@CONFIGURE_MACOSX_DEPLOYMENT_TARGET@ @EXPORT_MACOSX_DEPLOYMENT_TARGET@export MACOSX_DEPLOYMENT_TARGET +# iOS Deployment target selected during configure. Unlike macOS, the iOS +# deployment target is controlled using `-mios-version-min` arguments added to +# CFLAGS and LDFLAGS by the configure script. This variable is not used during +# the build, and is only listed here so it will be included in sysconfigdata. +IPHONEOS_DEPLOYMENT_TARGET=@IPHONEOS_DEPLOYMENT_TARGET@ + # Option to install to strip binaries STRIPFLAG=-s @@ -209,6 +228,12 @@ ENSUREPIP= @ENSUREPIP@ LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a LIBHACL_SHA2_A= Modules/_hacl/libHacl_Hash_SHA2.a +LIBHACL_BLAKE2_A= Modules/_hacl/libHacl_Hash_Blake2.a +LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ +LIBHACL_SIMD128_FLAGS=@LIBHACL_SIMD128_FLAGS@ +LIBHACL_SIMD256_FLAGS=@LIBHACL_SIMD256_FLAGS@ +LIBHACL_SIMD128_OBJS=@LIBHACL_SIMD128_OBJS@ +LIBHACL_SIMD256_OBJS=@LIBHACL_SIMD256_OBJS@ # Module state, compiler flags and linker flags # Empty CFLAGS and LDFLAGS are omitted. @@ -225,6 +250,9 @@ LIBHACL_SHA2_A= Modules/_hacl/libHacl_Hash_SHA2.a # Default zoneinfo.TZPATH. Added here to expose it in sysconfig.get_config_var TZPATH=@TZPATH@ +# If to install mimalloc headers +INSTALL_MIMALLOC=@INSTALL_MIMALLOC@ + # Modes for directories, executables and data files created by the # install process. Default to user-only-writable for all file types. DIRMODE= 755 @@ -408,6 +436,7 @@ PYTHON_OBJS= \ Python/brc.o \ Python/ceval.o \ Python/codecs.o \ + Python/codegen.o \ Python/compile.o \ Python/context.o \ Python/critical_section.o \ @@ -432,7 +461,9 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/interpconfig.o \ Python/instrumentation.o \ + Python/instruction_sequence.o \ Python/intrinsics.o \ Python/jit.o \ Python/legacy_tracing.o \ @@ -444,6 +475,7 @@ PYTHON_OBJS= \ Python/object_stack.o \ Python/optimizer.o \ Python/optimizer_analysis.o \ + Python/optimizer_symbols.o \ Python/parking_lot.o \ Python/pathconfig.o \ Python/preconfig.o \ @@ -456,6 +488,7 @@ PYTHON_OBJS= \ Python/pystate.o \ Python/pythonrun.o \ Python/pytime.o \ + Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ Python/structmember.o \ @@ -464,6 +497,7 @@ PYTHON_OBJS= \ Python/thread.o \ Python/traceback.o \ Python/tracemalloc.o \ + Python/uniqueid.o \ Python/getopt.o \ Python/pystrcmp.o \ Python/pystrtod.o \ @@ -473,6 +507,7 @@ PYTHON_OBJS= \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ + Python/perf_jit_trampoline.o \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ @@ -503,7 +538,6 @@ OBJECT_OBJS= \ Objects/floatobject.o \ Objects/frameobject.o \ Objects/funcobject.o \ - Objects/interpreteridobject.o \ Objects/iterobject.o \ Objects/listobject.o \ Objects/longobject.o \ @@ -616,7 +650,9 @@ LIBEXPAT_HEADERS= \ Modules/expat/utf8tab.h \ Modules/expat/xmlrole.h \ Modules/expat/xmltok.h \ - Modules/expat/xmltok_impl.h + Modules/expat/xmltok_impl.h \ + Modules/expat/xmltok_impl.c \ + Modules/expat/xmltok_ns.c ########################################################################## # hashlib's HACL* library @@ -624,6 +660,13 @@ LIBEXPAT_HEADERS= \ LIBHACL_SHA2_OBJS= \ Modules/_hacl/Hacl_Hash_SHA2.o +LIBHACL_BLAKE2_OBJS= \ + Modules/_hacl/Hacl_Hash_Blake2s.o \ + Modules/_hacl/Hacl_Hash_Blake2b.o \ + Modules/_hacl/Lib_Memzero0.o \ + $(LIBHACL_SIMD128_OBJS) \ + $(LIBHACL_SIMD256_OBJS) + LIBHACL_HEADERS= \ Modules/_hacl/include/krml/FStar_UInt128_Verified.h \ Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \ @@ -639,6 +682,18 @@ LIBHACL_SHA2_HEADERS= \ Modules/_hacl/internal/Hacl_Hash_SHA2.h \ $(LIBHACL_HEADERS) +LIBHACL_BLAKE2_HEADERS= \ + Modules/_hacl/Hacl_Hash_Blake2b.h \ + Modules/_hacl/Hacl_Hash_Blake2s.h \ + Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h \ + Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h \ + Modules/_hacl/internal/Hacl_Hash_Blake2b.h \ + Modules/_hacl/internal/Hacl_Hash_Blake2s.h \ + Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h \ + Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h \ + Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h \ + $(LIBHACL_HEADERS) + ######################################################################### # Rules @@ -649,8 +704,32 @@ all: @DEF_MAKE_ALL_RULE@ # all. .PHONY: all +# Provide quick help for common Makefile targets. +.PHONY: help +help: + @echo "Run 'make' to build the Python executable and extension modules" + @echo "" + @echo "or 'make ' where is one of:" + @echo " test run the test suite" + @echo " install install built files" + @echo " regen-all regenerate a number of generated source files" + @echo " clinic run Argument Clinic over source files" + @echo "" + @echo " clean to remove build files" + @echo " distclean 'clean' + remove other generated files (patch, exe, etc)" + @echo "" + @echo " recheck rerun configure with last cmdline options" + @echo " reindent reindent .py files in Lib directory" + @echo " tags build a tags file (useful for Emacs and other editors)" + @echo " list-targets list all targets in the Makefile" + +# Display a full list of Makefile targets +.PHONY: list-targets +list-targets: + @grep -E '^[A-Za-z][-A-Za-z0-9]+:' Makefile | awk -F : '{print $$1}' + .PHONY: build_all -build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ +build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \ gdbhooks Programs/_testembed scripts checksharedmods rundsymutil .PHONY: build_wasm @@ -673,6 +752,16 @@ check-clean-src: exit 1; \ fi +# Check that the app store compliance patch can be applied (if configured). +# This is checked as a dry-run against the original library sources; +# the patch will be actually applied during the install phase. +.PHONY: check-app-store-compliance +check-app-store-compliance: + @if [ "$(APP_STORE_COMPLIANCE_PATCH)" != "" ]; then \ + patch --dry-run --quiet --force --strip 1 --directory "$(abs_srcdir)" --input "$(abs_srcdir)/$(APP_STORE_COMPLIANCE_PATCH)"; \ + echo "App store compliance patch can be applied."; \ + fi + # Profile generation build must start from a clean tree. profile-clean-stamp: $(MAKE) clean @@ -784,7 +873,7 @@ coverage-lcov: @ # remove 3rd party modules, system headers and internal files with @ # debug, test or dummy functions. @lcov $(COVERAGE_LCOV_OPTIONS) --remove $(COVERAGE_INFO) \ - '*/Modules/_blake2/impl/*' \ + '*/Modules/_hacl/*' \ '*/Modules/_ctypes/libffi*/*' \ '*/Modules/_decimal/libmpdec/*' \ '*/Modules/expat/*' \ @@ -814,8 +903,8 @@ coverage-report: regen-token regen-frozen # Run "Argument Clinic" over all source files .PHONY: clinic -clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) +clinic: check-clean-src + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --force --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) .PHONY: clinic-tests clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c @@ -844,29 +933,22 @@ pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS) exit 1 ; \ fi -# blake2s is auto-generated from blake2b -$(srcdir)/Modules/_blake2/blake2s_impl.c: $(srcdir)/Modules/_blake2/blake2b_impl.c $(srcdir)/Modules/_blake2/blake2b2s.py - $(PYTHON_FOR_REGEN) $(srcdir)/Modules/_blake2/blake2b2s.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py -f $@ - # Build static library $(LIBRARY): $(LIBRARY_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) - if test $(INSTSONAME) != $(LDLIBRARY); then \ - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + if test $(INSTSONAME) != $@; then \ $(LN) -f $(INSTSONAME) $@; \ - else \ - $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ fi libpython3.so: libpython$(LDVERSION).so $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) - $(CC) -dynamiclib -Wl,-single_module $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ libpython$(VERSION).sl: $(LIBRARY_OBJS) @@ -891,14 +973,13 @@ $(BUILDPYTHON)-gdb.py: $(SRC_GDB_HOOKS) # This rule is here for OPENSTEP/Rhapsody/MacOSX. It builds a temporary # minimal framework (not including the Lib directory and such) in the current # directory. -RESSRCDIR=Mac/Resources/framework $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(LIBRARY) \ $(RESSRCDIR)/Info.plist $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION) $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ - -all_load $(LIBRARY) -Wl,-single_module \ - -install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK) \ + -all_load $(LIBRARY) \ + -install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ -compatibility_version $(VERSION) \ -current_version $(VERSION) \ -framework CoreFoundation $(LIBS); @@ -910,6 +991,21 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK) $(LN) -fsn Versions/Current/Resources $(PYTHONFRAMEWORKDIR)/Resources +# This rule is for iOS, which requires an annoyingly just slightly different +# format for frameworks to macOS. It *doesn't* use a versioned framework, and +# the Info.plist must be in the root of the framework. +$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK): \ + $(LIBRARY) \ + $(RESSRCDIR)/Info.plist + $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR) + $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ + -all_load $(LIBRARY) \ + -install_name $(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ + -compatibility_version $(VERSION) \ + -current_version $(VERSION) \ + -framework CoreFoundation $(LIBS); + $(INSTALL_DATA) $(RESSRCDIR)/Info.plist $(PYTHONFRAMEWORKDIR)/Info.plist + # This rule builds the Cygwin Python DLL and import library if configured # for a shared core library; otherwise, this rule is a noop. $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) @@ -943,6 +1039,7 @@ python.worker.js: $(srcdir)/Tools/wasm/python.worker.js PYTHON_HEADERS= \ $(srcdir)/Include/Python.h \ $(srcdir)/Include/abstract.h \ + $(srcdir)/Include/audit.h \ $(srcdir)/Include/bltinmodule.h \ $(srcdir)/Include/boolobject.h \ $(srcdir)/Include/bytearrayobject.h \ @@ -951,6 +1048,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/codecs.h \ $(srcdir)/Include/compile.h \ $(srcdir)/Include/complexobject.h \ + $(srcdir)/Include/critical_section.h \ $(srcdir)/Include/descrobject.h \ $(srcdir)/Include/dictobject.h \ $(srcdir)/Include/dynamic_annotations.h \ @@ -963,16 +1061,17 @@ PYTHON_HEADERS= \ $(srcdir)/Include/frameobject.h \ $(srcdir)/Include/genericaliasobject.h \ $(srcdir)/Include/import.h \ - $(srcdir)/Include/interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ + $(srcdir)/Include/lock.h \ $(srcdir)/Include/longobject.h \ $(srcdir)/Include/marshal.h \ $(srcdir)/Include/memoryobject.h \ $(srcdir)/Include/methodobject.h \ $(srcdir)/Include/modsupport.h \ $(srcdir)/Include/moduleobject.h \ + $(srcdir)/Include/monitoring.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ @@ -1002,6 +1101,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/pythread.h \ $(srcdir)/Include/pytypedefs.h \ $(srcdir)/Include/rangeobject.h \ + $(srcdir)/Include/refcount.h \ $(srcdir)/Include/setobject.h \ $(srcdir)/Include/sliceobject.h \ $(srcdir)/Include/structmember.h \ @@ -1018,6 +1118,7 @@ PYTHON_HEADERS= \ $(PARSER_HEADERS) \ \ $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/audit.h \ $(srcdir)/Include/cpython/bytearrayobject.h \ $(srcdir)/Include/cpython/bytesobject.h \ $(srcdir)/Include/cpython/cellobject.h \ @@ -1027,6 +1128,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/compile.h \ $(srcdir)/Include/cpython/complexobject.h \ $(srcdir)/Include/cpython/context.h \ + $(srcdir)/Include/cpython/critical_section.h \ $(srcdir)/Include/cpython/descrobject.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/fileobject.h \ @@ -1037,16 +1139,17 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/genobject.h \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ - $(srcdir)/Include/cpython/interpreteridobject.h \ $(srcdir)/Include/cpython/listobject.h \ + $(srcdir)/Include/cpython/lock.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ $(srcdir)/Include/cpython/memoryobject.h \ $(srcdir)/Include/cpython/methodobject.h \ + $(srcdir)/Include/cpython/modsupport.h \ + $(srcdir)/Include/cpython/monitoring.h \ $(srcdir)/Include/cpython/object.h \ $(srcdir)/Include/cpython/objimpl.h \ $(srcdir)/Include/cpython/odictobject.h \ - $(srcdir)/Include/cpython/optimizer.h \ $(srcdir)/Include/cpython/picklebufobject.h \ $(srcdir)/Include/cpython/pthread_stubs.h \ $(srcdir)/Include/cpython/pyatomic.h \ @@ -1065,7 +1168,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ $(srcdir)/Include/cpython/setobject.h \ - $(srcdir)/Include/cpython/sysmodule.h \ $(srcdir)/Include/cpython/traceback.h \ $(srcdir)/Include/cpython/tracemalloc.h \ $(srcdir)/Include/cpython/tupleobject.h \ @@ -1080,6 +1182,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_ast.h \ $(srcdir)/Include/internal/pycore_ast_state.h \ $(srcdir)/Include/internal/pycore_atexit.h \ + $(srcdir)/Include/internal/pycore_audit.h \ + $(srcdir)/Include/internal/pycore_backoff.h \ $(srcdir)/Include/internal/pycore_bitutils.h \ $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ $(srcdir)/Include/internal/pycore_brc.h \ @@ -1087,6 +1191,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_bytesobject.h \ $(srcdir)/Include/internal/pycore_call.h \ $(srcdir)/Include/internal/pycore_capsule.h \ + $(srcdir)/Include/internal/pycore_cell.h \ $(srcdir)/Include/internal/pycore_ceval.h \ $(srcdir)/Include/internal/pycore_ceval_state.h \ $(srcdir)/Include/internal/pycore_code.h \ @@ -1097,6 +1202,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_context.h \ $(srcdir)/Include/internal/pycore_critical_section.h \ $(srcdir)/Include/internal/pycore_crossinterp.h \ + $(srcdir)/Include/internal/pycore_debug_offsets.h \ $(srcdir)/Include/internal/pycore_descrobject.h \ $(srcdir)/Include/internal/pycore_dict.h \ $(srcdir)/Include/internal/pycore_dict_state.h \ @@ -1109,6 +1215,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_format.h \ $(srcdir)/Include/internal/pycore_frame.h \ $(srcdir)/Include/internal/pycore_freelist.h \ + $(srcdir)/Include/internal/pycore_freelist_state.h \ $(srcdir)/Include/internal/pycore_function.h \ $(srcdir)/Include/internal/pycore_gc.h \ $(srcdir)/Include/internal/pycore_genobject.h \ @@ -1119,11 +1226,11 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_global_strings.h \ $(srcdir)/Include/internal/pycore_hamt.h \ $(srcdir)/Include/internal/pycore_hashtable.h \ - $(srcdir)/Include/internal/pycore_identifier.h \ $(srcdir)/Include/internal/pycore_import.h \ $(srcdir)/Include/internal/pycore_importdl.h \ $(srcdir)/Include/internal/pycore_initconfig.h \ $(srcdir)/Include/internal/pycore_instruments.h \ + $(srcdir)/Include/internal/pycore_instruction_sequence.h \ $(srcdir)/Include/internal/pycore_interp.h \ $(srcdir)/Include/internal/pycore_intrinsics.h \ $(srcdir)/Include/internal/pycore_jit.h \ @@ -1138,6 +1245,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_namespace.h \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_object_alloc.h \ + $(srcdir)/Include/internal/pycore_object_deferred.h \ $(srcdir)/Include/internal/pycore_object_stack.h \ $(srcdir)/Include/internal/pycore_object_state.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ @@ -1149,6 +1257,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_parser.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ + $(srcdir)/Include/internal/pycore_pyatomic_ft_wrappers.h \ $(srcdir)/Include/internal/pycore_pybuffer.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ $(srcdir)/Include/internal/pycore_pyhash.h \ @@ -1160,6 +1269,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pystats.h \ $(srcdir)/Include/internal/pycore_pythonrun.h \ $(srcdir)/Include/internal/pycore_pythread.h \ + $(srcdir)/Include/internal/pycore_qsbr.h \ $(srcdir)/Include/internal/pycore_range.h \ $(srcdir)/Include/internal/pycore_runtime.h \ $(srcdir)/Include/internal/pycore_runtime_init.h \ @@ -1172,12 +1282,14 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_stackref.h \ $(srcdir)/Include/internal/pycore_time.h \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ $(srcdir)/Include/internal/pycore_tracemalloc.h \ $(srcdir)/Include/internal/pycore_tstate.h \ $(srcdir)/Include/internal/pycore_tuple.h \ + $(srcdir)/Include/internal/pycore_uniqueid.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_typevarobject.h \ $(srcdir)/Include/internal/pycore_ucnhash.h \ @@ -1265,8 +1377,9 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) ########################################################################## -# Build HACL* static libraries for hashlib: libHacl_Hash_SHA2.a -LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED) +# Build HACL* static libraries for hashlib: libHacl_Hash_SHA2.a, and +# libHacl_Blake2.a -- the contents of the latter vary depending on whether we +# have the ability to compile vectorized versions Modules/_hacl/Hacl_Hash_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c $(LIBHACL_SHA2_HEADERS) $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c @@ -1275,6 +1388,31 @@ $(LIBHACL_SHA2_A): $(LIBHACL_SHA2_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBHACL_SHA2_OBJS) +Modules/_hacl/Hacl_Hash_Blake2s.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c + +Modules/_hacl/Hacl_Hash_Blake2b.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b.c + +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD128_FLAGS) -DHACL_CAN_COMPILE_VEC128 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c + +Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD128_FLAGS) -DHACL_CAN_COMPILE_VEC128 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c + +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD256_FLAGS) -DHACL_CAN_COMPILE_VEC256 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c + +Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD256_FLAGS) -DHACL_CAN_COMPILE_VEC256 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c + +Modules/_hacl/Lib_Memzero0.o: $(srcdir)/Modules/_hacl/Lib_Memzero0.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Lib_Memzero0.c + +$(LIBHACL_BLAKE2_A): $(LIBHACL_BLAKE2_OBJS) + -rm -f $@ + $(AR) $(ARFLAGS) $@ $(LIBHACL_BLAKE2_OBJS) + # create relative links from build/lib.platform/egg.so to Modules/egg.so # pybuilddir.txt is created too late. We cannot use it in Makefile # targets. ln --relative is not portable. @@ -1358,7 +1496,7 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ -# "Bootstrap Python" used to run deepfreeze.py +# "Bootstrap Python" used to run Programs/_freeze_module.py BOOTSTRAP_HEADERS = \ Python/frozen_modules/importlib._bootstrap.h \ @@ -1377,7 +1515,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # # Freezing is a multi step process. It works differently for standard builds # and cross builds. Standard builds use Programs/_freeze_module and -# _bootstrap_python for freezing and deepfreezing, so users can build Python +# _bootstrap_python for freezing, so users can build Python # without an existing Python installation. Cross builds cannot execute # compiled binaries and therefore rely on an external build Python # interpreter. The build interpreter must have same version and same bytecode @@ -1391,12 +1529,10 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # 5) create remaining frozen module headers with # ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python # script is used to test the cross compile code path. -# 6) deepfreeze modules with _bootstrap_python # # Cross compile process: # 1) create all frozen module headers with external build Python and # Programs/_freeze_module.py script. -# 2) deepfreeze modules with external build Python. # # FROZEN_FILES_* are auto-generated by Tools/build/freeze_modules.py. @@ -1542,41 +1678,6 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py --frozen-modules @echo "The Makefile was updated, you may need to re-run make." -############################################################################ -# Deepfreeze targets - -DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c -DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) - -# BEGIN: deepfreeze modules -$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \ - Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \ - Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \ - Python/frozen_modules/zipimport.h:zipimport \ - Python/frozen_modules/abc.h:abc \ - Python/frozen_modules/codecs.h:codecs \ - Python/frozen_modules/io.h:io \ - Python/frozen_modules/_collections_abc.h:_collections_abc \ - Python/frozen_modules/_sitebuiltins.h:_sitebuiltins \ - Python/frozen_modules/genericpath.h:genericpath \ - Python/frozen_modules/ntpath.h:ntpath \ - Python/frozen_modules/posixpath.h:posixpath \ - Python/frozen_modules/os.h:os \ - Python/frozen_modules/site.h:site \ - Python/frozen_modules/stat.h:stat \ - Python/frozen_modules/importlib.util.h:importlib.util \ - Python/frozen_modules/importlib.machinery.h:importlib.machinery \ - Python/frozen_modules/runpy.h:runpy \ - Python/frozen_modules/__hello__.h:__hello__ \ - Python/frozen_modules/__phello__.h:__phello__ \ - Python/frozen_modules/__phello__.ham.h:__phello__.ham \ - Python/frozen_modules/__phello__.ham.eggs.h:__phello__.ham.eggs \ - Python/frozen_modules/__phello__.spam.h:__phello__.spam \ - Python/frozen_modules/frozen_only.h:frozen_only \ - -o Python/deepfreeze/deepfreeze.c -# END: deepfreeze modules - # We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib regen-importlib: regen-frozen @@ -1605,7 +1706,7 @@ check-abidump: all .PHONY: regen-limited-abi regen-limited-abi: all - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --generate-all ############################################################################ # Regenerate Unicode Data @@ -1623,10 +1724,10 @@ regen-unicodedata: regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects regen-sbom regen-jit + regen-test-levenshtein regen-global-objects @echo @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " - @echo "make regen-configure and make regen-unicodedata should be run manually" + @echo "make regen-configure, make regen-sbom, and make regen-unicodedata should be run manually" ############################################################################ # Special rules for object files @@ -1646,8 +1747,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h Makefile $(PYTHON_HEADERS) $(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \ - -DPREFIX='"$(prefix)"' \ - -DEXEC_PREFIX='"$(exec_prefix)"' \ + -DPREFIX='"$(host_prefix)"' \ + -DEXEC_PREFIX='"$(host_exec_prefix)"' \ -DVERSION='"$(VERSION)"' \ -DVPATH='"$(VPATH)"' \ -DPLATLIBDIR='"$(PLATLIBDIR)"' \ @@ -1670,6 +1771,18 @@ Modules/pwdmodule.o: $(srcdir)/Modules/pwdmodule.c $(srcdir)/Modules/posixmodule Modules/signalmodule.o: $(srcdir)/Modules/signalmodule.c $(srcdir)/Modules/posixmodule.h +Modules/_interpretersmodule.o: $(srcdir)/Modules/_interpretersmodule.c $(srcdir)/Modules/_interpreters_common.h + +Modules/_interpqueuesmodule.o: $(srcdir)/Modules/_interpqueuesmodule.c $(srcdir)/Modules/_interpreters_common.h + +Modules/_interpchannelsmodule.o: $(srcdir)/Modules/_interpchannelsmodule.c $(srcdir)/Modules/_interpreters_common.h + +Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h + +Python/initconfig.o: $(srcdir)/Python/initconfig.c $(srcdir)/Python/config_common.h + +Python/interpconfig.o: $(srcdir)/Python/interpconfig.c $(srcdir)/Python/config_common.h + Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile $(CC) -c $(PY_CORE_CFLAGS) \ -DSOABI='"$(SOABI)"' \ @@ -1777,7 +1890,7 @@ regen-sre: $(srcdir)/Modules/_sre/sre_constants.h \ $(srcdir)/Modules/_sre/sre_targets.h -Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h +Python/compile.o Python/codegen.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h Python/getplatform.o: $(srcdir)/Python/getplatform.c $(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c @@ -1809,6 +1922,7 @@ UNICODE_DEPS = \ $(srcdir)/Objects/stringlib/localeutil.h \ $(srcdir)/Objects/stringlib/partition.h \ $(srcdir)/Objects/stringlib/replace.h \ + $(srcdir)/Objects/stringlib/repr.h \ $(srcdir)/Objects/stringlib/split.h \ $(srcdir)/Objects/stringlib/ucs1lib.h \ $(srcdir)/Objects/stringlib/ucs2lib.h \ @@ -1846,36 +1960,78 @@ Objects/obmalloc.o: $(srcdir)/Objects/mimalloc/alloc.c \ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c + +# Regenerate various files from Python/bytecodes.c +# Pass CASESFLAG=-l to insert #line directives in the output + .PHONY: regen-cases -regen-cases: - # Regenerate various files from Python/bytecodes.c - # Pass CASESFLAG=-l to insert #line directives in the output +regen-cases: \ + regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ + regen-generated-cases regen-executor-cases regen-optimizer-cases \ + regen-opcode-metadata regen-uop-metadata + +.PHONY: regen-opcode-ids +regen-opcode-ids: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_id_generator.py \ -o $(srcdir)/Include/opcode_ids.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new + +.PHONY: regen-opcode-targets +regen-opcode-targets: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/target_generator.py \ -o $(srcdir)/Python/opcode_targets.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new + +.PHONY: regen-uop-ids +regen-uop-ids: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/uop_id_generator.py \ -o $(srcdir)/Include/internal/pycore_uop_ids.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new + +.PHONY: regen-opcode-metadata-py +regen-opcode-metadata-py: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/py_metadata_generator.py \ -o $(srcdir)/Lib/_opcode_metadata.py.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new + +.PHONY: regen-generated-cases +regen-generated-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_generator.py \ -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new + +.PHONY: regen-executor-cases +regen-executor-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new + +.PHONY: regen-optimizer-cases +regen-optimizer-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/optimizer_generator.py \ + -o $(srcdir)/Python/optimizer_cases.c.h.new \ + $(srcdir)/Python/optimizer_bytecodes.c \ + $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new + +.PHONY: regen-opcode-metadata +regen-opcode-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ -o $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new + +.PHONY: regen-uop-metadata +regen-uop-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/uop_metadata_generator.py -o \ $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(srcdir)/Python/bytecodes.c - $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new - $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new - $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new - $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new - $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new - $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new - $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new -Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h +Python/compile.o Python/codegen.o Python/assemble.o Python/flowgraph.o Python/instruction_sequence.o: \ + $(srcdir)/Include/internal/pycore_compile.h \ + $(srcdir)/Include/internal/pycore_flowgraph.h \ + $(srcdir)/Include/internal/pycore_instruction_sequence.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_opcode_utils.h Python/ceval.o: \ $(srcdir)/Python/ceval_macros.h \ @@ -1894,7 +2050,8 @@ Python/optimizer.o: \ Python/optimizer_analysis.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ - $(srcdir)/Include/internal/pycore_optimizer.h + $(srcdir)/Include/internal/pycore_optimizer.h \ + $(srcdir)/Python/optimizer_cases.c.h Python/frozen.o: $(FROZEN_FILES_OUT) @@ -1947,6 +2104,12 @@ cleantest: all # Similar to buildbottest, but use --fast-ci option, instead of --slow-ci. .PHONY: test test: all + $(TESTRUNNER) --fast-ci -u-gui --timeout=$(TESTTIMEOUT) $(TESTOPTS) + +# Run a basic set of regression tests inside the CI. +# This excludes some tests that are particularly resource-intensive. +# Similar to test, but also runs GUI tests. +ci: all $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Run the test suite for both architectures in a Universal build on OSX. @@ -1961,6 +2124,54 @@ testuniversal: all $(RUNSHARED) /usr/libexec/oah/translate \ ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS) +# Run the test suite on the iOS simulator. Must be run on a macOS machine with +# a full Xcode install that has an iPhone SE (3rd edition) simulator available. +# This must be run *after* a `make install` has completed the build. The +# `--with-framework-name` argument *cannot* be used when configuring the build. +XCFOLDER:=iOSTestbed.$(MULTIARCH).$(shell date +%s) +XCRESULT=$(XCFOLDER)/$(MULTIARCH).xcresult +.PHONY: testios +testios: + @if test "$(MACHDEP)" != "ios"; then \ + echo "Cannot run the iOS testbed for a non-iOS build."; \ + exit 1;\ + fi + @if test "$(findstring -iphonesimulator,$(MULTIARCH))" != "-iphonesimulator"; then \ + echo "Cannot run the iOS testbed for non-simulator builds."; \ + exit 1;\ + fi + @if test $(PYTHONFRAMEWORK) != "Python"; then \ + echo "Cannot run the iOS testbed with a non-default framework name."; \ + exit 1;\ + fi + @if ! test -d $(PYTHONFRAMEWORKPREFIX); then \ + echo "Cannot find a finalized iOS Python.framework. Have you run 'make install' to finalize the framework build?"; \ + exit 1;\ + fi + # Copy the testbed project into the build folder + cp -r $(srcdir)/iOS/testbed $(XCFOLDER) + # Copy the framework from the install location to the testbed project. + cp -r $(PYTHONFRAMEWORKPREFIX)/* $(XCFOLDER)/Python.xcframework/ios-arm64_x86_64-simulator + + # Run the test suite for the Xcode project, targeting the iOS simulator. + # If the suite fails, touch a file in the test folder as a marker + if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) -derivedDataPath $(XCFOLDER)/DerivedData ; then \ + touch $(XCFOLDER)/failed; \ + fi + + # Regardless of success or failure, extract and print the test output + xcrun xcresulttool get --path $(XCRESULT) \ + --id $$( \ + xcrun xcresulttool get --path $(XCRESULT) --format json | \ + $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['actions']['_values'][0]['actionResult']['logRef']['id']['_value'])" \ + ) \ + --format json | \ + $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['subsections']['_values'][1]['subsections']['_values'][0]['emittedOutput']['_value'])" + + @if test -e $(XCFOLDER)/failed ; then \ + exit 1; \ + fi + # Like test, but using --slow-ci which enables all test resources and use # longer timeout. Run an optional pybuildbot.identify script to include # information about the build environment. @@ -2000,7 +2211,7 @@ multissltest: all # which can lead to two parallel `./python setup.py build` processes that # step on each others toes. .PHONY: install -install: @FRAMEWORKINSTALLFIRST@ commoninstall bininstall maninstall @FRAMEWORKINSTALLLAST@ +install: @FRAMEWORKINSTALLFIRST@ @INSTALLTARGETS@ @FRAMEWORKINSTALLLAST@ if test "x$(ENSUREPIP)" != "xno" ; then \ case $(ENSUREPIP) in \ upgrade) ensurepip="--upgrade" ;; \ @@ -2136,10 +2347,10 @@ bininstall: commoninstall altbininstall -if test "$(VERSION)" != "$(LDVERSION)"; then \ rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)-config; \ (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(LDVERSION)-config python$(VERSION)-config); \ - rm -f $(DESTDIR)$(LIBPC)/python-$(LDVERSION).pc; \ - (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION).pc python-$(LDVERSION).pc); \ - rm -f $(DESTDIR)$(LIBPC)/python-$(LDVERSION)-embed.pc; \ - (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION)-embed.pc python-$(LDVERSION)-embed.pc); \ + rm -f $(DESTDIR)$(LIBPC)/python-$(VERSION).pc; \ + (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(LDVERSION).pc python-$(VERSION).pc); \ + rm -f $(DESTDIR)$(LIBPC)/python-$(VERSION)-embed.pc; \ + (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(LDVERSION)-embed.pc python-$(VERSION)-embed.pc); \ fi -rm -f $(DESTDIR)$(BINDIR)/python3-config (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-config python3-config) @@ -2216,16 +2427,19 @@ LIBSUBDIRS= asyncio \ xmlrpc \ zipfile zipfile/_path \ zoneinfo \ + _pyrepl \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ - test/audiodata \ + test/test_ast \ + test/test_ast/data \ test/archivetestdata \ + test/audiodata \ test/certdata \ test/certdata/capath \ test/cjkencodings \ - test/crashers \ test/configdata \ + test/crashers \ test/data \ test/decimaltestdata \ test/dtracedata \ @@ -2239,17 +2453,20 @@ TESTSUBDIRS= idlelib/idle_test \ test/subprocessdata \ test/support \ test/support/_hypothesis_stubs \ + test/support/interpreters \ test/test_asyncio \ test/test_capi \ + test/test_cext \ + test/test_concurrent_futures \ test/test_cppext \ test/test_ctypes \ test/test_dataclasses \ test/test_doctest \ test/test_email \ test/test_email/data \ + test/test_free_threading \ test/test_future_stmt \ test/test_gdb \ - test/test_inspect \ test/test_import \ test/test_import/data \ test/test_import/data/circular_imports \ @@ -2258,13 +2475,21 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_import/data/circular_imports/subpkg2/parent \ test/test_import/data/package \ test/test_import/data/package2 \ + test/test_import/data/package3 \ + test/test_import/data/package4 \ test/test_import/data/unwritable \ test/test_importlib \ test/test_importlib/builtin \ - test/test_importlib/data \ test/test_importlib/extension \ test/test_importlib/frozen \ test/test_importlib/import_ \ + test/test_importlib/metadata \ + test/test_importlib/metadata/data \ + test/test_importlib/metadata/data/sources \ + test/test_importlib/metadata/data/sources/example \ + test/test_importlib/metadata/data/sources/example/example \ + test/test_importlib/metadata/data/sources/example2 \ + test/test_importlib/metadata/data/sources/example2/example2 \ test/test_importlib/namespace_pkgs \ test/test_importlib/namespace_pkgs/both_portions \ test/test_importlib/namespace_pkgs/both_portions/foo \ @@ -2287,25 +2512,18 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/namespace_pkgs/project3/parent/child \ test/test_importlib/partial \ test/test_importlib/resources \ - test/test_importlib/resources/data01 \ - test/test_importlib/resources/data01/subdirectory \ - test/test_importlib/resources/data02 \ - test/test_importlib/resources/data02/one \ - test/test_importlib/resources/data02/subdirectory \ - test/test_importlib/resources/data02/subdirectory/subsubdir \ - test/test_importlib/resources/data02/two \ - test/test_importlib/resources/data03 \ - test/test_importlib/resources/data03/namespace \ - test/test_importlib/resources/data03/namespace/portion1 \ - test/test_importlib/resources/data03/namespace/portion2 \ - test/test_importlib/resources/namespacedata01 \ - test/test_importlib/resources/zipdata01 \ - test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ + test/test_inspect \ + test/test_interpreters \ test/test_json \ test/test_module \ + test/test_multiprocessing_fork \ + test/test_multiprocessing_forkserver \ + test/test_multiprocessing_spawn \ test/test_pathlib \ test/test_peg_generator \ + test/test_pydoc \ + test/test_pyrepl \ test/test_sqlite3 \ test/test_tkinter \ test/test_tomllib \ @@ -2329,6 +2547,10 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_tools \ test/test_ttk \ test/test_unittest \ + test/test_unittest/namespace_test_pkg \ + test/test_unittest/namespace_test_pkg/bar \ + test/test_unittest/namespace_test_pkg/noop \ + test/test_unittest/namespace_test_pkg/noop/no2 \ test/test_unittest/testmock \ test/test_warnings \ test/test_warnings/data \ @@ -2339,10 +2561,13 @@ TESTSUBDIRS= idlelib/idle_test \ test/tkinterdata \ test/tokenizedata \ test/tracedmodules \ + test/translationdata \ + test/translationdata/argparse \ test/typinganndata \ test/wheeldata \ test/xmltestdata \ - test/xmltestdata/c14n-20 + test/xmltestdata/c14n-20 \ + test/zipimport_data COMPILEALL_OPTS=-j0 @@ -2417,6 +2642,14 @@ libinstall: all $(srcdir)/Modules/xxmodule.c $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt + @ # If app store compliance has been configured, apply the patch to the + @ # installed library code. The patch has been previously validated against + @ # the original source tree, so we can ignore any errors that are raised + @ # due to files that are missing because of --disable-test-modules etc. + @if [ "$(APP_STORE_COMPLIANCE_PATCH)" != "" ]; then \ + echo "Applying app store compliance patch"; \ + patch --force --reject-file "$(abs_builddir)/app-store-compliance.rej" --strip 2 --directory "$(DESTDIR)$(LIBDEST)" --input "$(abs_srcdir)/$(APP_STORE_COMPLIANCE_PATCH)" || true ; \ + fi @ # Build PYC files for the 3 optimization levels (0, 1, 2) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ @@ -2485,6 +2718,12 @@ inclinstall: $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(INCLUDEPY)/internal; \ else true; \ fi + @if test "$(INSTALL_MIMALLOC)" = "yes"; then \ + if test ! -d $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; then \ + echo "Creating directory $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc"; \ + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; \ + fi; \ + fi @for i in $(srcdir)/Include/*.h; \ do \ echo $(INSTALL_DATA) $$i $(INCLUDEPY); \ @@ -2500,6 +2739,16 @@ inclinstall: echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal; \ $(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal; \ done + @if test "$(INSTALL_MIMALLOC)" = "yes"; then \ + echo $(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \ + $(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \ + for i in $(srcdir)/Include/internal/mimalloc/mimalloc/*.h; \ + do \ + echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal/mimalloc/mimalloc; \ + $(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; \ + done; \ + fi + echo $(INSTALL_DATA) pyconfig.h $(DESTDIR)$(CONFINCLUDEPY)/pyconfig.h $(INSTALL_DATA) pyconfig.h $(DESTDIR)$(CONFINCLUDEPY)/pyconfig.h # Install the library and miscellaneous stuff needed for extending/embedding @@ -2540,8 +2789,8 @@ libainstall: all scripts $(INSTALL_DATA) Modules/Setup.bootstrap $(DESTDIR)$(LIBPL)/Setup.bootstrap $(INSTALL_DATA) Modules/Setup.stdlib $(DESTDIR)$(LIBPL)/Setup.stdlib $(INSTALL_DATA) Modules/Setup.local $(DESTDIR)$(LIBPL)/Setup.local - $(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(VERSION).pc - $(INSTALL_DATA) Misc/python-embed.pc $(DESTDIR)$(LIBPC)/python-$(VERSION)-embed.pc + $(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(LDVERSION).pc + $(INSTALL_DATA) Misc/python-embed.pc $(DESTDIR)$(LIBPC)/python-$(LDVERSION)-embed.pc $(INSTALL_SCRIPT) $(srcdir)/Modules/makesetup $(DESTDIR)$(LIBPL)/makesetup $(INSTALL_SCRIPT) $(srcdir)/install-sh $(DESTDIR)$(LIBPL)/install-sh $(INSTALL_SCRIPT) python-config.py $(DESTDIR)$(LIBPL)/python-config.py @@ -2582,10 +2831,11 @@ frameworkinstall: install # only have to cater for the structural bits of the framework. .PHONY: frameworkinstallframework -frameworkinstallframework: frameworkinstallstructure install frameworkinstallmaclib +frameworkinstallframework: @FRAMEWORKINSTALLFIRST@ install frameworkinstallmaclib -.PHONY: frameworkinstallstructure -frameworkinstallstructure: $(LDLIBRARY) +# macOS uses a versioned frameworks structure that includes a full install +.PHONY: frameworkinstallversionedstructure +frameworkinstallversionedstructure: $(LDLIBRARY) @if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ echo Not configured with --enable-framework; \ exit 1; \ @@ -2606,6 +2856,27 @@ frameworkinstallstructure: $(LDLIBRARY) $(LN) -fsn Versions/Current/Resources $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Resources $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) +# iOS/tvOS/watchOS uses a non-versioned framework with Info.plist in the +# framework root, no .lproj data, and only stub compilation assistance binaries +.PHONY: frameworkinstallunversionedstructure +frameworkinstallunversionedstructure: $(LDLIBRARY) + @if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ + echo Not configured with --enable-framework; \ + exit 1; \ + else true; \ + fi + if test -d $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include; then \ + echo "Clearing stale header symlink directory"; \ + rm -rf $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include; \ + fi + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) + sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist + $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(BINDIR) + for file in $(srcdir)/$(RESSRCDIR)/bin/* ; do \ + $(INSTALL) -m $(EXEMODE) $$file $(DESTDIR)$(BINDIR); \ + done + # This installs Mac/Lib into the framework # Install a number of symlinks to keep software that expects a normal unix # install (which includes python-config) happy. @@ -2646,6 +2917,19 @@ frameworkaltinstallunixtools: frameworkinstallextras: cd Mac && $(MAKE) installextras DESTDIR="$(DESTDIR)" +# On iOS, bin/lib can't live inside the framework; include needs to be called +# "Headers", but *must* be in the framework, and *not* include the `python3.X` +# subdirectory. The install has put these folders in the same folder as +# Python.framework; Move the headers to their final framework-compatible home. +.PHONY: frameworkinstallmobileheaders +frameworkinstallmobileheaders: frameworkinstallunversionedstructure inclinstall + if test -d $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; then \ + echo "Removing old framework headers"; \ + rm -rf $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; \ + fi + mv "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(LDVERSION)" "$(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers" + $(LN) -fs "../$(PYTHONFRAMEWORKDIR)/Headers" "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(LDVERSION)" + # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status CONFIG_FILES=Makefile.pre CONFIG_HEADERS= ./config.status @@ -2766,13 +3050,17 @@ clean-retain-profile: pycremoval -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) -rm -f Programs/_testembed Programs/_freeze_module - -rm -f Python/deepfreeze/*.[co] + -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST -rm -f jit_stencils.h -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp + -rm -rf iOS/testbed/Python.xcframework/ios-*/bin + -rm -rf iOS/testbed/Python.xcframework/ios-*/lib + -rm -rf iOS/testbed/Python.xcframework/ios-*/include + -rm -rf iOS/testbed/Python.xcframework/ios-*/Python.framework .PHONY: profile-removal profile-removal: @@ -2798,6 +3086,8 @@ clobber: clean config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) + -rm -rf iOS/Frameworks + -rm -rf iOSTestbed.* -rm -f python-config.py python-config -rm -rf cross-build @@ -2875,7 +3165,7 @@ patchcheck: all .PHONY: check-limited-abi check-limited-abi: all - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --all $(srcdir)/Misc/stable_abi.toml + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --all .PHONY: update-config update-config: @@ -2892,7 +3182,7 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h # force rebuild when header file or module build flavor (static/shared) is changed MODULE_DEPS_STATIC=Modules/config.c -MODULE_DEPS_SHARED=$(MODULE_DEPS_STATIC) $(EXPORTSYMS) +MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@ MODULE__CURSES_DEPS=$(srcdir)/Include/py_curses.h MODULE__CURSES_PANEL_DEPS=$(srcdir)/Include/py_curses.h @@ -2901,20 +3191,22 @@ MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h -MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/blake2module.h $(srcdir)/Modules/hashlib.h -MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h +MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h $(srcdir)/Modules/_complex.h +MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h -MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c -MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c +MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/internal/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c +MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/internal/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) -MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c +MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/internal/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c +MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_A) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h -MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h +MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h +MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h diff --git a/Misc/ACKS b/Misc/ACKS index 8a80e02ecba26a..a1769d9601a2ea 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -191,6 +191,7 @@ Finn Bock Paul Boddie Matthew Boedicker Robin Boerdijk +Wannes Boeykens Andra Bogildea Matt Bogosian Nikolay Bogoychev @@ -314,6 +315,7 @@ Greg Chapman Mitch Chapman Matt Chaput William Chargin +Ben Chatterton Yogesh Chaudhari Gautam Chaudhuri David Chaum @@ -470,6 +472,7 @@ Allen Downey Cesar Douady Dean Draayer Fred L. Drake, Jr. +Mehdi Drissi Derk Drukker John DuBois Paul Dubois @@ -494,6 +497,7 @@ David Edelsohn John Edmonds Benjamin Edwards Grant Edwards +Vlad Efanov Zvi Effron John Ehresman Tal Einat @@ -517,6 +521,7 @@ Michael Ernst Ben Escoto Andy Eskilsson André Espaze +Lucas Esposito Stefan Esser Nicolas Estibals Jonathan Eunice @@ -606,6 +611,7 @@ Nitin Ganatra Soumendra Ganguly (गङ्गोपाध्याय) Fred Gansevles Paul Ganssle +Tian Gao Lars Marius Garshol Jake Garver Dan Gass @@ -639,6 +645,7 @@ Neil Girdhar Matt Giuca Andrea Giudiceandrea Franz Glasner +Jeff Glass Wim Glenn Michael Goderbauer Karan Goel @@ -747,6 +754,7 @@ Kasun Herath Chris Herborth Ivan Herman Jürgen Hermann +Joshua Jay Herman Gary Herron Ernie Hershey Thomas Herve @@ -756,6 +764,7 @@ Raymond Hettinger Lisa Hewus Fresh Kevan Heydon Wouter van Heyst +Derek Higgins Kelsey Hightower Jason Hildebrand Ryan Hileman @@ -925,6 +934,7 @@ Hiroaki Kawai Dmitry Kazakov Brian Kearns Sebastien Keim +Russell Keith-Magee Ryan Kelly Hugo van Kemenade Dan Kenigsberg @@ -1090,6 +1100,7 @@ Ivan Levkivskyi Ben Lewis William Lewis Akira Li +Jiahao Li Robert Li Xuanji Li Zekun Li @@ -1308,6 +1319,7 @@ Hrvoje Nikšić Gregory Nofi Jesse Noller Bill Noon +Janek Nouvertné Stefan Norberg Tim Northover Joe Norton @@ -1540,7 +1552,9 @@ Lisa Roach Carl Robben Ben Roberts Mark Roberts +Tony Roberts Andy Robinson +Izan "TizzySaurus" Robinson Jim Robinson Yolanda Robla Daniel Rocco @@ -1641,6 +1655,7 @@ Scott Schram Robin Schreiber Chad J. Schroeder Simon-Martin Schroeder +Brian Schubert Christian Schubert Sam Schulenburg Andreas Schwab @@ -1659,6 +1674,7 @@ Fred Sells Jiwon Seo Iñigo Serna Joakim Sernbrant +Rodrigo Girão Serrão Roger D. Serwy Jerry Seutter Pete Sevander @@ -1731,6 +1747,7 @@ Christopher Smith Eric V. Smith Ethan H. Smith Gregory P. Smith +Malcolm Smith Mark Smith Nathaniel J. Smith Roy Smith @@ -1797,6 +1814,7 @@ Reuben Sumner Eryk Sun Sanjay Sundaresan Marek Šuppa +Danica J. Sutherland Hisao Suzuki Kalle Svensson Andrew Svetlov @@ -2048,6 +2066,7 @@ Doug Wyatt Xiang Zhang Robert Xiao Florent Xicluna +Yanbo, Xie Xinhang Xu Arnon Yaari Alakshendra Yadav diff --git a/Misc/HISTORY b/Misc/HISTORY index b66413277259dc..d68aaa066771fb 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -607,7 +607,7 @@ Library MemoryError. - Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed - ambigious reverse mappings. Added many new mappings. Import mapping is no + ambiguous reverse mappings. Added many new mappings. Import mapping is no longer applied to modules already mapped with full name mapping. - Issue #23745: The new email header parser now handles duplicate MIME @@ -2030,7 +2030,7 @@ Library initialization of the unquote_to_bytes() table of the urllib.parse module, to not waste memory if these modules are not used. -- Issue #19157: Include the broadcast address in the usuable hosts for IPv6 +- Issue #19157: Include the broadcast address in the usable hosts for IPv6 in ipaddress. - Issue #11599: When an external command (e.g. compiler) fails, distutils now @@ -2620,7 +2620,7 @@ Library - asyncio: Various improvements and small changes not all covered by issues listed below. E.g. wait_for() now cancels the inner task if - the timeout occcurs; tweaked the set of exported symbols; renamed + the timeout occurs; tweaked the set of exported symbols; renamed Empty/Full to QueueEmpty/QueueFull; "with (yield from lock)" now uses a separate context manager; readexactly() raises if not enough data was read; PTY support tweaks. @@ -3944,7 +3944,7 @@ Library - Issue #18996: TestCase.assertEqual() now more cleverly shorten differing strings in error report. -- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string reperesentation. +- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string representation. - Issue #18978: ``urllib.request.Request`` now allows the method to be indicated on the class and no longer sets it to None in ``__init__``. @@ -3952,7 +3952,7 @@ Library - Issue #18626: the inspect module now offers a basic command line introspection interface (Initial patch by Claudiu Popa) -- Issue #3015: Fixed tkinter with wantobject=False. Any Tcl command call +- Issue #3015: Fixed tkinter with ``wantobjects=False``. Any Tcl command call returned empty string. - Issue #19037: The mailbox module now makes all changes to maildir files @@ -4191,7 +4191,7 @@ Library - Issue #18532: Change the builtin hash algorithms' names to lower case names as promised by hashlib's documentation. -- Issue #8713: add new spwan and forkserver start methods, and new functions +- Issue #8713: add new spawn and forkserver start methods, and new functions get_all_start_methods, get_start_method, and set_start_method, to multiprocessing. @@ -4524,7 +4524,7 @@ Core and Builtins - Issue #16613: Add *m* argument to ``collections.Chainmap.new_child`` to allow the new child map to be specified explicitly. -- Issue #16730: importlib.machinery.FileFinder now no longers raises an +- Issue #16730: importlib.machinery.FileFinder now no longer raises an exception when trying to populate its cache and it finds out the directory is unreadable or has turned into a file. Reported and diagnosed by David Pritchard. @@ -4832,7 +4832,7 @@ Library on Windows and adds no value over and above python -m pydoc ... - Issue #18155: The csv module now correctly handles csv files that use - a delimter character that has a special meaning in regexes, instead of + a delimiter character that has a special meaning in regexes, instead of throwing an exception. - Issue #14360: encode_quopri can now be successfully used as an encoder @@ -5590,7 +5590,7 @@ Library - Issue #16248: Disable code execution from the user's home directory by tkinter when the -E flag is passed to Python. Patch by Zachary Ware. -- Issue #13390: New function :func:`sys.getallocatedblocks()` returns the +- Issue #13390: New function :func:`sys.getallocatedblocks` returns the number of memory blocks currently allocated. - Issue #16628: Fix a memory leak in ctypes.resize(). @@ -6157,7 +6157,7 @@ Tests starting with a ".". Patch by Sebastian Kreft. - Issue #13390: The ``-R`` option to regrtest now also checks for memory - allocation leaks, using :func:`sys.getallocatedblocks()`. + allocation leaks, using :func:`sys.getallocatedblocks`. - Issue #16559: Add more tests for the json module, including some from the official test suite at json.org. Patch by Serhiy Storchaka. @@ -6329,7 +6329,7 @@ Documentation - Issue #15940: Specify effect of locale on time functions. -- Issue #17538: Document XML vulnerabilties +- Issue #17538: Document XML vulnerabilities - Issue #16642: sched.scheduler timefunc initial default is time.monotonic. Patch by Ramchandra Apte @@ -6676,7 +6676,7 @@ Library - Issue #14669: Fix pickling of connections and sockets on Mac OS X by sending/receiving an acknowledgment after file descriptor transfer. - TestPicklingConnection has been reenabled for Mac OS X. + TestPicklingConnection has been re-enabled for Mac OS X. - Issue #11062: Fix adding a message from file to Babyl mailbox. @@ -7114,7 +7114,7 @@ Build - Issue #14330: For cross builds, don't use host python, use host search paths for host compiler. -- Issue #15235: Allow Berkley DB versions up to 5.3 to build the dbm module. +- Issue #15235: Allow Berkeley DB versions up to 5.3 to build the dbm module. - Issue #15268: Search curses.h in /usr/include/ncursesw. @@ -7264,7 +7264,7 @@ Library called with no arguments. - Issue #14653: email.utils.mktime_tz() no longer relies on system - mktime() when timezone offest is supplied. + mktime() when timezone offset is supplied. - Issue #14684: zlib.compressobj() and zlib.decompressobj() now support the use of predefined compression dictionaries. Original patch by Sam Rushing. @@ -7606,7 +7606,7 @@ Library - Issue #14773: Fix os.fwalk() failing on dangling symlinks. - Issue #12541: Be lenient with quotes around Realm field of HTTP Basic - Authentation in urllib2. + Authentication in urllib2. - Issue #14807: move undocumented tarfile.filemode() to stat.filemode() and add doc entry. Add tarfile.filemode alias with deprecation warning. @@ -7673,7 +7673,7 @@ Library IDLE ---- -- Issue #14958: Change IDLE systax highlighting to recognize all string and +- Issue #14958: Change IDLE syntax highlighting to recognize all string and byte literals supported in Python 3.3. - Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. @@ -10176,7 +10176,7 @@ IDLE - Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. (Patch by Roger Serwy) -- Issue #9871: Prevent IDLE 3 crash when given byte stings +- Issue #9871: Prevent IDLE 3 crash when given byte strings with invalid hex escape sequences, like b'\x0'. (Original patch by Claudiu Popa.) @@ -12098,7 +12098,7 @@ Library - Issue #9632: Remove sys.setfilesystemencoding() function: use PYTHONFSENCODING environment variable to set the filesystem encoding at Python startup. sys.setfilesystemencoding() creates inconsistencies because it is unable to - reencode all filenames in all objects. + re-encode all filenames in all objects. - Issue #9410: Various optimizations to the pickle module, leading to speedups up to 4x (depending on the benchmark). Mostly ported from Unladen Swallow; @@ -12509,7 +12509,7 @@ Library - Issue #9605: posix.getlogin() decodes the username with file filesystem encoding and surrogateescape error handler. Patch written by David Watson. -- Issue #9604: posix.initgroups() encodes the username using the fileystem +- Issue #9604: posix.initgroups() encodes the username using the filesystem encoding and surrogateescape error handler. Patch written by David Watson. - Issue #9603: posix.ttyname() and posix.ctermid() decode the terminal name @@ -12667,7 +12667,7 @@ What's New in Python 3.2 Alpha 1? Core and Builtins ----------------- -- Issue #8991: convertbuffer() rejects discontigious buffers. +- Issue #8991: convertbuffer() rejects discontiguous buffers. - Issue #7616: Fix copying of overlapping memoryview slices with the Intel compiler. @@ -13211,7 +13211,7 @@ Library - Issue #7989: Added pure python implementation of the `datetime` module. The C module is renamed to `_datetime` and if available, overrides all classes - defined in datetime with fast C impementation. Python implementation is based + defined in datetime with fast C implementation. Python implementation is based on the original python prototype for the datetime module by Tim Peters with minor modifications by the PyPy project. The test suite now tests `datetime` module with and without `_datetime` acceleration using the same test cases. @@ -15049,7 +15049,7 @@ Extension Modules an error. The _PY_STRUCT_FLOAT_COERCE constant has been removed. The version number has been bumped to 0.3. -- Issue #5359: Readd the Berkeley DB detection code to allow _dbm be built +- Issue #5359: Re-add the Berkeley DB detection code to allow _dbm be built using Berkeley DB. Tests @@ -17028,7 +17028,7 @@ Extension Modules and renamed to filter(), map(), and zip(). Also, renamed izip_longest() to zip_longest() and ifilterfalse() to filterfalse(). -- Issue #1762972: Readded the reload() function as imp.reload(). +- Issue #1762972: Re-added the reload() function as imp.reload(). - Bug #2111: mmap segfaults when trying to write a block opened with PROT_READ. @@ -18448,7 +18448,7 @@ Core and builtins - Fixed bug #1459029 - unicode reprs were double-escaped. -- Patch #1396919: The system scope threads are reenabled on FreeBSD +- Patch #1396919: The system scope threads are re-enabled on FreeBSD 5.4 and later versions. - Bug #1115379: Compiling a Unicode string with an encoding declaration @@ -21803,7 +21803,7 @@ Library - New csv package makes it easy to read/write CSV files. - Module shlex has been extended to allow posix-like shell parsings, - including a split() function for easy spliting of quoted strings and + including a split() function for easy splitting of quoted strings and commands. An iterator interface was also implemented. Tools/Demos @@ -27751,7 +27751,7 @@ Fri Mar 12 22:15:43 1999 Guido van Rossum The filename to URL conversion didn't properly quote special characters. - The URL to filename didn't properly unquote special chatacters. + The URL to filename didn't properly unquote special characters. * Objects/floatobject.c: OK, try again. Vladimir gave me a fix for the alignment bus error, @@ -27807,7 +27807,7 @@ Wed Mar 10 22:55:47 1999 Guido van Rossum classes in selected module methods of selected class - Sinlge clicking in a directory, module or class item updates the next + Single clicking in a directory, module or class item updates the next column with info about the selected item. Double clicking in a module, class or method item opens the file (and selects the clicked item if it is a class or method). @@ -28130,7 +28130,7 @@ webchecker and other ftp retrieves. - ConfigParser's get() method now accepts an optional keyword argument (vars) that is substituted on top of the defaults that were setup in -__init__. You can now also have recusive references in your +__init__. You can now also have recursive references in your configuration file. - Some improvements to the Queue module, including a put_nowait() @@ -28209,7 +28209,7 @@ core. not. - The curses module implements an optional nlines argument to -w.scroll(). (It then calls wscrl(win, nlines) instead of scoll(win).) +w.scroll(). (It then calls wscrl(win, nlines) instead of scroll(win).) Changes to tools ---------------- @@ -28504,7 +28504,7 @@ PyEval_GetGlobals. - glmodule.c: check in the changed version after running the stubber again -- this solves the conflict with curses over the 'clear' entry point much nicer. (Jack Jansen had checked in the changes to cstubs -eons ago, but I never regenrated glmodule.c :-( ) +eons ago, but I never regenerated glmodule.c :-( ) - frameobject.c: fix reference count bug in PyFrame_New. Vladimir Marangozov. @@ -28581,7 +28581,7 @@ idiom L1[len(L1):] = L2. - Better error messages when a sequence is indexed with a non-integer. -- Bettter error message when calling a non-callable object (include +- Better error message when calling a non-callable object (include the type in the message). Python services @@ -28656,7 +28656,7 @@ Internet Protocols and Support - imaplib.py: new version from Piers Lauder. - smtplib.py: change sendmail() method to accept a single string or a -list or strings as the destination (commom newbie mistake). +list or strings as the destination (common newbie mistake). - poplib.py: LIST with a msg argument fixed. @@ -31109,7 +31109,7 @@ encoding/decoding CGI form arguments. Catch all errors from the ftp module. HTTP requests now add the Host: header line. The proxy variable names are now mapped to lower case, for Windows. The spliturl() function no longer erroneously throws away all data past -the first newline. The basejoin() function now intereprets "../" +the first newline. The basejoin() function now interprets "../" correctly. I *believe* that the problems with "exception raised in __del__" under certain circumstances have been fixed (mostly by changes elsewher in the interpreter). @@ -31397,7 +31397,7 @@ changes and fixes. - Added a bunch of new winfo options to Tkinter.py; we should now be up to date with Tk 4.2. The new winfo options supported are: -mananger, pointerx, pointerxy, pointery, server, viewable, visualid, +manager, pointerx, pointerxy, pointery, server, viewable, visualid, visualsavailable. - The broken bind() method on Canvas objects defined in the Canvas.py @@ -32552,7 +32552,7 @@ The same applies to posixfile.open() and the socket method makefile(). is being maintained and distributed separately. - Improved support for the Apple Macintosh, in part by Jack Jansen, -e.g. interfaces to (a few) resource mananger functions, get/set file +e.g. interfaces to (a few) resource manager functions, get/set file type and creator, gestalt, sound manager, speech manager, MacTCP, comm toolbox, and the think C console library. This is being maintained and distributed separately. @@ -33229,7 +33229,7 @@ sys.argv[0]; it can simply do "if __name__ == '__main__': main()". * When an object is printed by the print statement, its implementation of str() is used. This means that classes can define __str__(self) to direct how their instances are printed. This is different from -__repr__(self), which should define an unambigous string +__repr__(self), which should define an unambiguous string representation of the instance. (If __str__() is not defined, it defaults to __repr__().) @@ -34366,7 +34366,7 @@ eval_code) and ceval.h (which doesn't need compile.hand declares the rest) ceval.h defines macros BGN_SAVE / END_SAVE for use with threads (to -improve the parallellism of multi-threaded programs by letting other +improve the parallelism of multi-threaded programs by letting other Python code run when a blocking system call or something similar is made) @@ -34514,7 +34514,7 @@ names listed in a 'global' statement must not be used in the function before the statement is reached. Remember that you don't need to use 'global' if you only want to *use* -a global variable in a function; nor do you need ot for assignments to +a global variable in a function; nor do you need to for assignments to parts of global variables (e.g., list or dictionary items or attributes of class instances). This has not changed; in fact assignment to part of a global variable was the standard workaround. diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index 4842a026aa49f7..f09842f1e77dea 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -5,7 +5,7 @@ .. section: Security Fixes ``python3x._pth`` being ignored on Windows, caused by the fix for -:issue:`29778` (CVE-2020-15801). +:issue:`29778` (:cve:`2020-15801`). .. @@ -25,7 +25,7 @@ events. .. section: Security Ensure :file:`python3.dll` is loaded from correct locations when Python is -embedded (CVE-2020-15523). +embedded (:cve:`2020-15523`). .. @@ -97,7 +97,7 @@ convention. Patch by Donghee Na. .. nonce: aJS9B3 .. section: Core and Builtins -Port the :mod:`_bisect` module to the multi-phase initialization API +Port the :mod:`!_bisect` module to the multi-phase initialization API (:pep:`489`). .. @@ -128,7 +128,7 @@ Taskaya. .. nonce: lh335O .. section: Core and Builtins -Port the :mod:`_lsprof` extension module to multi-phase initialization +Port the :mod:`!_lsprof` extension module to multi-phase initialization (:pep:`489`). .. @@ -148,7 +148,7 @@ Port the :mod:`cmath` extension module to multi-phase initialization .. nonce: jiXmyT .. section: Core and Builtins -Port the :mod:`_scproxy` extension module to multi-phase initialization +Port the :mod:`!_scproxy` extension module to multi-phase initialization (:pep:`489`). .. @@ -168,7 +168,7 @@ Port the :mod:`termios` extension module to multi-phase initialization .. nonce: QuDIut .. section: Core and Builtins -Convert the :mod:`_sha256` extension module types to heap types. +Convert the :mod:`!_sha256` extension module types to heap types. .. @@ -187,7 +187,7 @@ classes with a huge amount of arguments. Patch by Pablo Galindo. .. nonce: CnRME3 .. section: Core and Builtins -Port the :mod:`_overlapped` extension module to multi-phase initialization +Port the :mod:`!_overlapped` extension module to multi-phase initialization (:pep:`489`). .. @@ -197,7 +197,7 @@ Port the :mod:`_overlapped` extension module to multi-phase initialization .. nonce: X9CZgo .. section: Core and Builtins -Port the :mod:`_curses_panel` extension module to multi-phase initialization +Port the :mod:`!_curses_panel` extension module to multi-phase initialization (:pep:`489`). .. @@ -207,7 +207,7 @@ Port the :mod:`_curses_panel` extension module to multi-phase initialization .. nonce: 5jZymK .. section: Core and Builtins -Port the :mod:`_opcode` extension module to multi-phase initialization +Port the :mod:`!_opcode` extension module to multi-phase initialization (:pep:`489`). .. @@ -282,7 +282,7 @@ initialized ``_ast`` module. .. nonce: vcxSUa .. section: Core and Builtins -Convert :mod:`_operator` to use :c:func:`PyType_FromSpec`. +Convert :mod:`!_operator` to use :c:func:`PyType_FromSpec`. .. @@ -291,7 +291,7 @@ Convert :mod:`_operator` to use :c:func:`PyType_FromSpec`. .. nonce: fubBkb .. section: Core and Builtins -Port :mod:`_sha3` to multi-phase init. Convert static types to heap types. +Port :mod:`!_sha3` to multi-phase init. Convert static types to heap types. .. @@ -300,7 +300,7 @@ Port :mod:`_sha3` to multi-phase init. Convert static types to heap types. .. nonce: FC13e7 .. section: Core and Builtins -Port the :mod:`_blake2` extension module to the multi-phase initialization +Port the :mod:`!_blake2` extension module to the multi-phase initialization API (:pep:`489`). .. @@ -339,7 +339,7 @@ The output of ``python --help`` contains now only ASCII characters. .. nonce: O0d3ym .. section: Core and Builtins -Port the :mod:`_sha1`, :mod:`_sha512`, and :mod:`_md5` extension modules to +Port the :mod:`!_sha1`, :mod:`!_sha512`, and :mod:`!_md5` extension modules to multi-phase initialization API (:pep:`489`). .. @@ -636,7 +636,7 @@ Remove the remaining files from the old parser and the :mod:`symbol` module. .. nonce: _yI-ax .. section: Core and Builtins -Convert :mod:`_bz2` to use :c:func:`PyType_FromSpec`. +Convert :mod:`!_bz2` to use :c:func:`PyType_FromSpec`. .. @@ -666,7 +666,7 @@ by Brandt Bucher. .. nonce: 61iyYh .. section: Core and Builtins -Port :mod:`_gdbm` to multiphase initialization. +Port :mod:`!_gdbm` to multiphase initialization. .. @@ -696,7 +696,7 @@ for emitting syntax errors. Patch by Pablo Galindo. .. nonce: mmlp3Q .. section: Core and Builtins -Port :mod:`_dbm` to multiphase initialization. +Port :mod:`!_dbm` to multiphase initialization. .. @@ -1010,7 +1010,7 @@ Port :mod:`mmap` to multiphase initialization. .. nonce: Kfe9fT .. section: Core and Builtins -Port :mod:`_lzma` to multiphase initialization. +Port :mod:`!_lzma` to multiphase initialization. .. @@ -1032,7 +1032,7 @@ the :meth:`~object.__int__` method but do not have the .. nonce: AkRzjb .. section: Core and Builtins -Add :meth:`int.bit_count()`, counting the number of ones in the binary +Add :meth:`int.bit_count`, counting the number of ones in the binary representation of an integer. Patch by Niklas Fiekas. .. @@ -1499,7 +1499,7 @@ used to cause ZeroDivisionError now cause an OverflowError instead. .. nonce: rju34k .. section: Library -Add :func:`os.cpu_count()` support for VxWorks RTOS. +Add :func:`os.cpu_count` support for VxWorks RTOS. .. @@ -1596,7 +1596,7 @@ UnpicklingError instead of crashing. .. section: Library Avoid infinite loop when reading specially crafted TAR files using the -tarfile module (CVE-2019-20907). +tarfile module (:cve:`2019-20907`). .. @@ -1861,8 +1861,8 @@ bundled versions of ``pip`` and ``setuptools``. Patch by Krzysztof Konopko. .. nonce: _dx3OO .. section: Library -Removed :meth:`asyncio.Task.current_task` and -:meth:`asyncio.Task.all_tasks`. Patch contributed by Rémi Lapeyre. +Removed :meth:`!asyncio.Task.current_task` and +:meth:`!asyncio.Task.all_tasks`. Patch contributed by Rémi Lapeyre. .. @@ -2452,7 +2452,7 @@ Added the *root_dir* and *dir_fd* parameters in :func:`glob.glob`. .. nonce: X-TJZO .. section: Library -Fix :meth:`IMAP4.noop()` when debug mode is enabled (ex: ``imaplib.Debug = +Fix :meth:`IMAP4.noop` when debug mode is enabled (ex: ``imaplib.Debug = 3``). .. diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst index 79f570439b52b8..3e82de9ef266d6 100644 --- a/Misc/NEWS.d/3.10.0a2.rst +++ b/Misc/NEWS.d/3.10.0a2.rst @@ -226,8 +226,8 @@ thread at the time the function is called. .. section: Core and Builtins Enable ``from __future__ import annotations`` (:pep:`563`) by default. The -values found in :attr:`__annotations__` dicts are now strings, e.g. ``{"x": -"int"}`` instead of ``{"x": int}``. +values found in :attr:`~object.__annotations__` dicts are now strings, for +example ``{"x": "int"}`` instead of ``{"x": int}``. .. @@ -362,7 +362,7 @@ plistlib: fix parsing XML plists with hexadecimal integer values .. nonce: 85BsRA .. section: Library -Fix an incorrectly formatted error from :meth:`_codecs.charmap_decode` when +Fix an incorrectly formatted error from :meth:`!_codecs.charmap_decode` when called with a mapped value outside the range of valid Unicode code points. PR by Max Bernstein. @@ -383,7 +383,7 @@ Inwood. .. nonce: jd_gkA .. section: Library -:meth:`sched.scheduler.cancel()` will now cancel the correct event, if two +:meth:`sched.scheduler.cancel` will now cancel the correct event, if two events with same priority are scheduled for the same time. Patch by Bar Harel. diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 179cf3e9cfb08c..33c3e14b7a4bcf 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -477,7 +477,7 @@ object belongs to, potentially breaking the unpickling of those objects. Simplify the :mod:`importlib` external bootstrap code: ``importlib._bootstrap_external`` now uses regular imports to import builtin -modules. When it is imported, the builtin :func:`__import__()` function is +modules. When it is imported, the builtin :func:`__import__` function is already fully working and so can be used to import builtin modules like :mod:`sys`. Patch by Victor Stinner. @@ -517,8 +517,8 @@ Port the ``_signal`` extension module to the multi-phase initialization API .. nonce: Wh5svI .. section: Library -:func:`time.time()`, :func:`time.perf_counter()` and -:func:`time.monotonic()` functions can no longer fail with a Python fatal +:func:`time.time`, :func:`time.perf_counter` and +:func:`time.monotonic` functions can no longer fail with a Python fatal error, instead raise a regular Python exception on failure. .. @@ -550,10 +550,10 @@ deduplicate, use type to cache key). Patch provided by Yurii Karabas. .. nonce: iDbHrw .. section: Library -:func:`time.perf_counter()` on Windows and :func:`time.monotonic()` on macOS +:func:`time.perf_counter` on Windows and :func:`time.monotonic` on macOS are now system-wide. Previously, they used an offset computed at startup to reduce the precision loss caused by the float type. Use -:func:`time.perf_counter_ns()` and :func:`time.monotonic_ns()` added in +:func:`time.perf_counter_ns` and :func:`time.monotonic_ns` added in Python 3.7 to avoid this precision loss. .. @@ -1386,7 +1386,7 @@ Python already implicitly installs signal handlers: see The ``Py_TRASHCAN_BEGIN`` macro no longer accesses PyTypeObject attributes, but now can get the condition by calling the new private -:c:func:`_PyTrash_cond()` function which hides implementation details. +:c:func:`!_PyTrash_cond()` function which hides implementation details. .. diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index 398f7e5d3422cb..19f0db9a6be5e9 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -193,7 +193,7 @@ subinterpreters. Patch by Victor Stinner. .. nonce: j7nl6A .. section: Core and Builtins -Make :c:func:`_PyUnicode_FromId` function compatible with subinterpreters. +Make :c:func:`!_PyUnicode_FromId` function compatible with subinterpreters. Each interpreter now has an array of identifier objects (interned strings decoded from UTF-8). Patch by Victor Stinner. @@ -367,7 +367,7 @@ uses "options" instead. .. nonce: Quy3zn .. section: Library -Port the :mod:`_thread` extension module to the multiphase initialization +Port the :mod:`!_thread` extension module to the multiphase initialization API (:pep:`489`) and convert its static types to heap types. .. @@ -412,7 +412,7 @@ be created automatically. ``logging.disable`` will now validate the types and value of its parameter. It also now accepts strings representing the levels (as does -``loging.setLevel``) instead of only the numerical values. +``logging.setLevel``) instead of only the numerical values. .. @@ -709,7 +709,7 @@ directories. .. nonce: ek38d_ .. section: Library -Add :func:`os.set_blocking()` support for VxWorks RTOS. +Add :func:`os.set_blocking` support for VxWorks RTOS. .. @@ -960,8 +960,8 @@ explicitly and so not exported. .. nonce: Je08Ny .. section: C API -Remove the private :c:func:`_Py_fopen` function which is no longer needed. -Use :c:func:`_Py_wfopen` or :c:func:`_Py_fopen_obj` instead. Patch by Victor +Remove the private :c:func:`!_Py_fopen` function which is no longer needed. +Use :c:func:`!_Py_wfopen` or :c:func:`!_Py_fopen_obj` instead. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.10.0a5.rst b/Misc/NEWS.d/3.10.0a5.rst index dc95e8ce072fd9..a85ea1ff1c2817 100644 --- a/Misc/NEWS.d/3.10.0a5.rst +++ b/Misc/NEWS.d/3.10.0a5.rst @@ -108,7 +108,7 @@ a slice at the start of the ``bytearray`` to a shorter byte string). .. nonce: WfTdfg .. section: Core and Builtins -Fix the :c:func:`_PyUnicode_FromId` function (_Py_IDENTIFIER(var) API) when +Fix the :c:func:`!_PyUnicode_FromId` function (_Py_IDENTIFIER(var) API) when :c:func:`Py_Initialize` / :c:func:`Py_Finalize` is called multiple times: preserve ``_PyRuntime.unicode_ids.next_index`` value. diff --git a/Misc/NEWS.d/3.10.0a6.rst b/Misc/NEWS.d/3.10.0a6.rst index c379b968c9885b..31b7df2c61158e 100644 --- a/Misc/NEWS.d/3.10.0a6.rst +++ b/Misc/NEWS.d/3.10.0a6.rst @@ -158,7 +158,7 @@ tests that are unrelated to :class:`ProcessPoolExecutor` on those platforms. .. nonce: hsCNgX .. section: Core and Builtins -If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator +If :func:`object.__ipow__` returns :data:`NotImplemented`, the operator will correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. @@ -315,7 +315,7 @@ Adds :const:`resource.RLIMIT_KQUEUES` constant from FreeBSD to the .. section: Library Make the pure Python implementation of :mod:`xml.etree.ElementTree` behave -the same as the C implementation (:mod:`_elementree`) regarding default +the same as the C implementation (:mod:`!_elementree`) regarding default attribute values (by not setting ``specified_attributes=1``). .. diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index 74120a3b40c012..d866e805fd3a7e 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -4,7 +4,7 @@ .. release date: 2021-04-05 .. section: Security -CVE-2021-3426: Remove the ``getfile`` feature of the :mod:`pydoc` module +:cve:`2021-3426`: Remove the ``getfile`` feature of the :mod:`pydoc` module which could be abused to read arbitrary files on the disk (directory traversal vulnerability). Moreover, even source code of Python modules can contain sensitive data like passwords. Vulnerability reported by David @@ -83,7 +83,7 @@ instruction dispatch a bit. .. nonce: PhaT-B .. section: Core and Builtins -Fix reference leak in the :mod:`_hashopenssl` extension. Patch by Pablo +Fix reference leak in the :mod:`!_hashopenssl` extension. Patch by Pablo Galindo. .. @@ -654,7 +654,7 @@ support importlib.invalidate_caches(). Patch by Desmond Cheong. .. nonce: 3r0HFY .. section: Library -Fail fast in :func:`shutil.move()` to avoid creating destination directories +Fail fast in :func:`shutil.move` to avoid creating destination directories on failure. .. @@ -701,8 +701,8 @@ sessions in :mod:`pdb`'s interactive mode. When the :data:`tempfile.tempdir` global variable is set to a value of type bytes, it is now handled consistently. Previously exceptions could be raised from some tempfile APIs when the directory did not already exist in -this situation. Also ensures that the :func:`tempfile.gettempdir()` and -:func:`tempfile.gettempdirb()` functions *always* return ``str`` and +this situation. Also ensures that the :func:`tempfile.gettempdir` and +:func:`tempfile.gettempdirb` functions *always* return ``str`` and ``bytes`` respectively. .. @@ -715,7 +715,7 @@ this situation. Also ensures that the :func:`tempfile.gettempdir()` and Expose ``X509_V_FLAG_ALLOW_PROXY_CERTS`` as :const:`~ssl.VERIFY_ALLOW_PROXY_CERTS` to allow proxy certificate validation as explained in -https://www.openssl.org/docs/man1.1.1/man7/proxy-certificates.html. +https://docs.openssl.org/1.1.1/man7/proxy-certificates/. .. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index 640f3ee58adbae..25c6b827146e82 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -182,7 +182,7 @@ normally be possible, but might occur in some unusual circumstances. .. nonce: u5Y6bS .. section: Core and Builtins -Importing the :mod:`_signal` module in a subinterpreter has no longer side +Importing the :mod:`!_signal` module in a subinterpreter has no longer side effects. .. @@ -277,7 +277,7 @@ cause any runtime effects with ``from __future__ import annotations``. .. section: Core and Builtins :exc:`SyntaxError` exceptions raised by the interpreter will highlight the -full error range of the expression that consistutes the syntax error itself, +full error range of the expression that constitutes the syntax error itself, instead of just where the problem is detected. Patch by Pablo Galindo. .. @@ -776,11 +776,11 @@ builtins.open() is now io.open(). .. nonce: o1zEk_ .. section: Library -The Python :func:`_pyio.open` function becomes a static method to behave as +The Python :func:`!_pyio.open` function becomes a static method to behave as :func:`io.open` built-in function: don't become a bound method when stored as a class variable. It becomes possible since static methods are now -callable in Python 3.10. Moreover, :func:`_pyio.OpenWrapper` becomes a -simple alias to :func:`_pyio.open`. Patch by Victor Stinner. +callable in Python 3.10. Moreover, :func:`!_pyio.OpenWrapper` becomes a +simple alias to :func:`!_pyio.open`. Patch by Victor Stinner. .. @@ -958,7 +958,7 @@ Patch by Jelle Zijlstra. .. nonce: nnVd3h .. section: Library -Add an ``encoding`` parameter :func:`logging.fileConfig()`. +Add an ``encoding`` parameter :func:`logging.fileConfig`. .. @@ -1270,7 +1270,7 @@ Fix thread locks in zlib module may go wrong in rare case. Patch by Ma Lin. .. nonce: oi6Kdb .. section: Library -Fix dataclasses with ``InitVar``\s and :func:`~dataclasses.replace()`. Patch +Fix dataclasses with ``InitVar``\s and :func:`~dataclasses.replace`. Patch by Claudiu Popa. .. @@ -1310,11 +1310,11 @@ functions in the :mod:`os` module. .. nonce: 9adF3E .. section: Library -:func:`os.path.expanduser()` now refuses to guess Windows home directories +:func:`os.path.expanduser` now refuses to guess Windows home directories if the basename of current user's home directory does not match their username. -:meth:`pathlib.Path.expanduser()` and :meth:`~pathlib.Path.home()` now +:meth:`pathlib.Path.expanduser` and :meth:`~pathlib.Path.home` now consistently raise :exc:`RuntimeError` exception when a home directory cannot be resolved. Previously a :exc:`KeyError` exception could be raised on Windows when the ``"USERNAME"`` environment variable was unset. @@ -1516,7 +1516,7 @@ Adds additional arguments to :func:`os.startfile` function. .. nonce: F0Cg6X .. section: Windows -Avoid raising errors from :meth:`pathlib.Path.exists()` when passed an +Avoid raising errors from :meth:`pathlib.Path.exists` when passed an invalid filename. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index e8d4a02a11e0f9..0b49c2a78771d2 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -38,7 +38,7 @@ significant performance overhead when loading from ``.pyc`` files. .. section: Security Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix -for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used +for the :cve:`2013-0340` "Billion Laughs" vulnerability. This copy is most used on Windows and macOS. .. @@ -613,7 +613,7 @@ Rename ``types.Union`` to ``types.UnionType``. .. section: Core and Builtins Expose specialization stats in python via -:func:`_opcode.get_specialization_stats`. +:func:`!_opcode.get_specialization_stats`. .. @@ -660,7 +660,7 @@ Karabas. .. section: Core and Builtins Parameter substitution of the union type with wrong types now raises -``TypeError`` instead of returning ``NotImplemented``. +``TypeError`` instead of returning :data:`NotImplemented`. .. @@ -972,7 +972,7 @@ manager` protocols correspondingly. .. section: Core and Builtins Make sure that the line number is set when entering a comprehension scope. -Ensures that backtraces inclusing generator expressions show the correct +This ensures that backtraces including generator expressions show the correct line number. .. @@ -1701,7 +1701,7 @@ Remove many old deprecated :mod:`unittest` features: .. nonce: y1kEfP .. section: Library -Remove the deprecated ``split()`` method of :class:`_tkinter.TkappType`. +Remove the deprecated ``split()`` method of :class:`!_tkinter.TkappType`. Patch by Erlend E. Aasland. .. @@ -2000,7 +2000,7 @@ during file extraction. .. nonce: roUl0G .. section: Library -:mod:`subprocess` on Solaris now also uses :func:`os.posix_spawn()` for +:mod:`subprocess` on Solaris now also uses :func:`os.posix_spawn` for better performance. .. @@ -2298,9 +2298,9 @@ Adopt *binacii.a2b_base64*'s strict mode in *base64.b64decode*. .. nonce: ThuDMI .. section: Library -Fixed a bug in the :mod:`_ssl` module that was throwing :exc:`OverflowError` -when using :meth:`_ssl._SSLSocket.write` and :meth:`_ssl._SSLSocket.read` -for a big value of the ``len`` parameter. Patch by Pablo Galindo +Fixed a bug in the :mod:`!_ssl` module that was throwing :exc:`OverflowError` +when using :meth:`!_ssl._SSLSocket.write` and :meth:`!_ssl._SSLSocket.read` +for a big value of the ``len`` parameter. Patch by Pablo Galindo. .. @@ -2398,7 +2398,7 @@ class in the interactive session. Instead of :exc:`TypeError`, it should be .. nonce: R3IcM1 .. section: Library -Fix memory leak in :func:`_tkinter._flatten` if it is called with a sequence +Fix memory leak in :func:`!_tkinter._flatten` if it is called with a sequence or set, but not list or tuple. .. @@ -2722,7 +2722,7 @@ Importing typing.io or typing.re now prints a ``DeprecationWarning``. .. section: Library argparse actions store_const and append_const each receive a default value -of None when the ``const`` kwarg is not provided. Previously, this raised a +of ``None`` when the ``const`` kwarg is not provided. Previously, this raised a :exc:`TypeError`. .. @@ -3995,7 +3995,7 @@ operator expressions. .. section: Documentation Document that :class:`collections.defaultdict` parameter ``default_factory`` -defaults to None and is positional-only. +defaults to ``None`` and is positional-only. .. @@ -4187,7 +4187,7 @@ Add calls of :func:`gc.collect` in tests to support PyPy. .. nonce: mQZdXU .. section: Tests -Made tests relying on the :mod:`_asyncio` C extension module optional to +Made tests relying on the :mod:`!_asyncio` C extension module optional to allow running on alternative Python implementations. Patch by Serhiy Storchaka. @@ -4238,7 +4238,7 @@ harmless "malloc can't allocate region" messages spewed by test_decimal. .. nonce: KKsNOV .. section: Tests -Fixed floating point precision issue in turtle tests. +Fixed floating-point precision issue in turtle tests. .. diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index a6b5fe54b391c5..48cf2c1e428d87 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -15,7 +15,7 @@ Improve the :exc:`SyntaxError` message when using ``True``, ``None`` or .. section: Core and Builtins :data:`sys.stdlib_module_names` now contains the macOS-specific module -:mod:`_scproxy`. +:mod:`!_scproxy`. .. @@ -34,7 +34,7 @@ module but frozen modules are disabled. .. nonce: veL4lJ .. section: Core and Builtins -Specialize simple calls to Python functions (no starargs, keyowrd dict, or +Specialize simple calls to Python functions (no starargs, keyword dict, or closure) .. @@ -331,8 +331,8 @@ underlying SQLite API signals memory error. Patch by Erlend E. Aasland. .. nonce: 4MQt4r .. section: Library -pprint.pprint() now handles underscore_numbers correctly. Previously it was -always setting it to False. +:func:`pprint.pprint` now handles *underscore_numbers* correctly. +Previously it was always setting it to ``False``. .. @@ -1023,7 +1023,7 @@ compile shared modules. .. nonce: 61gM2A .. section: Build -:mod:`pyexpat` and :mod:`_elementtree` no longer define obsolete macros +:mod:`pyexpat` and :mod:`!_elementtree` no longer define obsolete macros ``HAVE_EXPAT_CONFIG_H`` and ``USE_PYEXPAT_CAPI``. ``XML_POOR_ENTROPY`` is now defined in ``expat_config.h``. diff --git a/Misc/NEWS.d/3.11.0a3.rst b/Misc/NEWS.d/3.11.0a3.rst index 2842aad0e163d6..6a0ae20d1fb5ed 100644 --- a/Misc/NEWS.d/3.11.0a3.rst +++ b/Misc/NEWS.d/3.11.0a3.rst @@ -27,7 +27,7 @@ invalid targets. Patch by Pablo Galindo .. nonce: 3TmTSw .. section: Core and Builtins -:c:func:`_PyErr_ChainStackItem` no longer normalizes ``exc_info`` (including +:c:func:`!_PyErr_ChainStackItem` no longer normalizes ``exc_info`` (including setting the traceback on the exception instance) because ``exc_info`` is always normalized. diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst index 78b682f7a22cc6..64e2f39ad9db18 100644 --- a/Misc/NEWS.d/3.11.0a4.rst +++ b/Misc/NEWS.d/3.11.0a4.rst @@ -7,7 +7,7 @@ :c:func:`Py_EndInterpreter` now explicitly untracks all objects currently tracked by the GC. Previously, if an object was used later by another interpreter, calling :c:func:`PyObject_GC_UnTrack` on the object crashed if -the previous or the next object of the :c:type:`PyGC_Head` structure became +the previous or the next object of the :c:type:`!PyGC_Head` structure became a dangling pointer. Patch by Victor Stinner. .. @@ -258,7 +258,7 @@ instruction which performs the same operation, but without the loop. .. nonce: ADVaPT .. section: Core and Builtins -The code called from :c:func:`_PyErr_Display` was refactored to improve +The code called from :c:func:`!_PyErr_Display` was refactored to improve error handling. It now exits immediately upon an unrecoverable error. .. diff --git a/Misc/NEWS.d/3.11.0a5.rst b/Misc/NEWS.d/3.11.0a5.rst index 30a462e9bfdcbf..5418d5d59dd583 100644 --- a/Misc/NEWS.d/3.11.0a5.rst +++ b/Misc/NEWS.d/3.11.0a5.rst @@ -486,8 +486,8 @@ Use ``dis.Positions`` in ``dis.Instruction`` instead of a regular ``tuple``. .. nonce: geS-aP .. section: Library -:mod:`pdb` now gracefully handles ``help`` when :attr:`__doc__` is missing, -for example when run with pregenerated optimized ``.pyc`` files. +:mod:`pdb` now gracefully handles ``help`` when :attr:`~module.__doc__` is +missing, for example when run with pregenerated optimized ``.pyc`` files. .. @@ -748,7 +748,7 @@ tests to use ``support.infinite_recursion()``. Patch by Victor Stinner. Skip test_builtin PTY tests on non-ASCII characters if the readline module is loaded. The readline module changes input() behavior, but test_builtin is -not intented to test the readline module. Patch by Victor Stinner. +not intended to test the readline module. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 2fdceef7746d4e..e88142e641f040 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -1054,7 +1054,7 @@ Patch by Victor Stinner. .. nonce: ajJjkh .. section: Build -Building Python now requires support for floating point Not-a-Number (NaN): +Building Python now requires support for floating-point Not-a-Number (NaN): remove the ``Py_NO_NAN`` macro. Patch by Victor Stinner. .. @@ -1088,7 +1088,7 @@ Patch by Kumar Aditya. Fix wasm32-emscripten test failures and platform issues. - Disable syscalls that are not supported or don't work, e.g. wait, getrusage, prlimit, -mkfifo, mknod, setres[gu]id, setgroups. - Use fd_count to cound open fds. - +mkfifo, mknod, setres[gu]id, setgroups. - Use fd_count to count open fds. - Add more checks for subprocess and fork. - Add workarounds for missing _multiprocessing and failing socket.accept(). - Enable bzip2. - Disable large file support. - Disable signal.alarm. @@ -1162,7 +1162,7 @@ Terry Jan Reedy. .. section: C API Python's public headers no longer import ````, leaving code that -embedd/extends Python free to define ``bool``, ``true`` and ``false``. +embeds/extends Python free to define ``bool``, ``true`` and ``false``. .. @@ -1182,7 +1182,7 @@ internal C API ``pycore_frame.h`` header file. Patch by Victor Stinner. .. section: C API Rename ``Include/buffer.h`` header file to ``Include/pybuffer.h`` to avoid -conflits with projects having an existing ``buffer.h`` header file. Patch by +conflicts with projects having an existing ``buffer.h`` header file. Patch by Victor Stinner. .. @@ -1202,5 +1202,5 @@ API). Patch by Victor Stinner. .. nonce: __ZdpH .. section: C API -Added function :c:func:`PyType_GetModuleByDef`, which allows accesss to +Added function :c:func:`PyType_GetModuleByDef`, which allows access to module state when a method's defining class is not available. diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst index ec99bd0294ceca..eff2ea2dac13f8 100644 --- a/Misc/NEWS.d/3.11.0a7.rst +++ b/Misc/NEWS.d/3.11.0a7.rst @@ -224,7 +224,7 @@ Kumar Aditya. .. nonce: ZI05b5 .. section: Core and Builtins -Improved the performance of :meth:`list.append()` and list comprehensions by +Improved the performance of :meth:`list.append` and list comprehensions by optimizing for the common case, where no resize is needed. Patch by Dennis Sweeney. @@ -1127,7 +1127,7 @@ Raise more accurate and :pep:`249` compatible exceptions in :mod:`sqlite3`. * Don't overwrite :exc:`BufferError` with :exc:`ValueError` when conversion to BLOB fails. * Raise :exc:`~sqlite3.ProgrammingError` instead of :exc:`~sqlite3.Warning` if - user tries to :meth:`~sqlite3.Cursor.execute()` more than one SQL statement. + user tries to :meth:`~sqlite3.Cursor.execute` more than one SQL statement. * Raise :exc:`~sqlite3.ProgrammingError` instead of :exc:`ValueError` if an SQL query contains null characters. @@ -1173,7 +1173,7 @@ implemented. .. section: Library Add an Barrier object in synchronization primitives of *asyncio* Lib in -order to be consistant with Barrier from *threading* and *multiprocessing* +order to be consistent with Barrier from *threading* and *multiprocessing* libs* .. @@ -1211,7 +1211,7 @@ Update PEP URLs to :pep:`676`'s new canonical form. .. nonce: 4Dn48U .. section: Documentation -Clarified the old Python versions compatiblity note of +Clarified the old Python versions compatibility note of :func:`binascii.crc32` / :func:`zlib.adler32` / :func:`zlib.crc32` functions. @@ -1401,7 +1401,7 @@ Christian's container image ``quay.io/tiran/cpython_autoconf:269``. .. nonce: fry4aK .. section: Build -Building Python now requires support of IEEE 754 floating point numbers. +Building Python now requires support of IEEE 754 floating-point numbers. Patch by Victor Stinner. .. @@ -1421,7 +1421,7 @@ Patch by Victor Stinner. .. nonce: IB0XL4 .. section: Windows -Update ``zlib`` to v1.2.12 to resolve CVE-2018-25032. +Update ``zlib`` to v1.2.12 to resolve :cve:`2018-25032`. .. @@ -1472,8 +1472,8 @@ Update Windows installer to use SQLite 3.38.1. .. nonce: SPrGS9 .. section: Windows -Update bzip2 to 1.0.8 in Windows builds to mitigate CVE-2016-3189 and -CVE-2019-12900 +Update bzip2 to 1.0.8 in Windows builds to mitigate :cve:`2016-3189` and +:cve:`2019-12900`. .. @@ -1482,7 +1482,7 @@ CVE-2019-12900 .. nonce: Ufd4tG .. section: Windows -Prevent CVE-2022-26488 by ensuring the Add to PATH option in the Windows +Prevent :cve:`2022-26488` by ensuring the Add to PATH option in the Windows installer uses the correct path when being repaired. .. diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index f9296679655573..85cb0f1b5cffbd 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -58,10 +58,10 @@ may have prevented Python-to-Python calls respecting PEP 523. .. nonce: -igcjS .. section: Core and Builtins -Add a closure keyword-only parameter to exec(). It can only be specified +Add a closure keyword-only parameter to :func:`exec`. It can only be specified when exec-ing a code object that uses free variables. When specified, it must be a tuple, with exactly the number of cell variables referenced by the -code object. closure has a default value of None, and it must be None if the +code object. closure has a default value of ``None``, and it must be ``None`` if the code object doesn't refer to any free variables. .. @@ -285,7 +285,7 @@ macros. .. nonce: 11YXHQ .. section: Core and Builtins -Add a new :c:func:`_PyFrame_IsEntryFrame` API function, to check if a +Add a new :c:func:`!_PyFrame_IsEntryFrame` API function, to check if a :c:type:`PyFrameObject` is an entry frame. Patch by Pablo Galindo. .. @@ -664,8 +664,9 @@ for :func:`os.fcopyfile` available in macOs. .. nonce: l1p7CJ .. section: Library -For @dataclass, add weakref_slot. Default is False. If True, and if -slots=True, add a slot named "__weakref__", which will allow instances to be +For :func:`@dataclass `, add *weakref_slot*. +The new parameter defaults to ``False``. If true, and if +``slots=True``, add a slot named ``"__weakref__"``, which will allow instances to be weakref'd. Contributed by Eric V. Smith .. @@ -1800,8 +1801,8 @@ The documentation now lists which members of C structs are part of the .. section: Documentation All docstrings in code snippets are now wrapped into :c:macro:`PyDoc_STR` to -follow the guideline of `PEP 7's Documentation Strings paragraph -`_. Patch +follow the guideline of :pep:`PEP 7's Documentation Strings paragraph +<0007#documentation-strings>`. Patch by Oleg Iarygin. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index f192bf086ed259..f2668e99a6299b 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -29,8 +29,7 @@ process. This was a potential privilege escalation. Filesystem based socket permissions restrict this to the *forkserver* process user as was the default in Python 3.8 and earlier. -This prevents Linux `CVE-2022-42919 -`_. +This prevents Linux :cve:`2022-42919`. .. @@ -103,7 +102,7 @@ well as generator expressions. .. section: Core and Builtins Added unicode check for ``name`` attribute of ``spec`` argument passed in -:func:`_imp.create_builtin` function. +:func:`!_imp.create_builtin` function. .. @@ -484,7 +483,7 @@ Fix case of undefined behavior in ceval.c .. nonce: AfCi36 .. section: Core and Builtins -Convert :mod:`_functools` to argument clinic. +Convert :mod:`!_functools` to argument clinic. .. @@ -493,7 +492,7 @@ Convert :mod:`_functools` to argument clinic. .. nonce: wky0Fc .. section: Core and Builtins -Do not expose ``KeyWrapper`` in :mod:`_functools`. +Do not expose ``KeyWrapper`` in :mod:`!_functools`. .. @@ -561,7 +560,7 @@ versions prior to 3.11 .. nonce: 9lmTCC .. section: Core and Builtins -Remove two cases of undefined behavoir, by adding NULL checks. +Remove two cases of undefined behavior, by adding NULL checks. .. @@ -1732,7 +1731,7 @@ tracing functions implemented in C. .. nonce: lenv9h .. section: Core and Builtins -:meth:`_warnings.warn_explicit` is ported to Argument Clinic. +:meth:`!_warnings.warn_explicit` is ported to Argument Clinic. .. @@ -2592,7 +2591,7 @@ Update bundled pip to 22.2.2. Fix :class:`asyncio.TaskGroup` to propagate exception when :exc:`asyncio.CancelledError` was replaced with another exception by a -context manger. Patch by Kumar Aditya and Guido van Rossum. +context manager. Patch by Kumar Aditya and Guido van Rossum. .. @@ -2722,7 +2721,7 @@ on future on an error - e.g. TimeoutError or KeyboardInterrupt. Fix a :mod:`sqlite3` regression where ``*args`` and ``**kwds`` were incorrectly relayed from :py:func:`~sqlite3.connect` to the :class:`~sqlite3.Connection` factory. The regression was introduced in -3.11a1 with PR 24421 (:gh:`85128`). Patch by Erlend E. Aasland.` +3.11a1 with PR 24421 (:gh:`85128`). Patch by Erlend E. Aasland. .. @@ -2988,7 +2987,7 @@ Kumar Aditya. .. section: Library Fix crash in :class:`struct.Struct` when it was not completely initialized -by initializing it in :meth:`~object.__new__``. Patch by Kumar Aditya. +by initializing it in :meth:`~object.__new__`. Patch by Kumar Aditya. .. @@ -3143,8 +3142,8 @@ test.test_codecs.EncodedFileTest`` instead. .. nonce: VhS1eS .. section: Library -Made :class:`_struct.Struct` GC-tracked in order to fix a reference leak in -the :mod:`_struct` module. +Made :class:`!_struct.Struct` GC-tracked in order to fix a reference leak in +the :mod:`!_struct` module. .. @@ -3200,9 +3199,8 @@ Remove the :func:`ssl.wrap_socket` function, deprecated in Python 3.7: instead, create a :class:`ssl.SSLContext` object and call its :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses :func:`ssl.wrap_socket` is broken and insecure. The function neither sends a -SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 -`_: Improper Certificate -Validation. Patch by Victor Stinner. +SNI TLS extension nor validates server hostname. Code is subject to :cwe:`295` +Improper Certificate Validation. Patch by Victor Stinner. .. @@ -3223,9 +3221,9 @@ Stinner. .. section: Library :mod:`hashlib`: Remove the pure Python implementation of -:func:`hashlib.pbkdf2_hmac()`, deprecated in Python 3.10. Python 3.10 and +:func:`hashlib.pbkdf2_hmac`, deprecated in Python 3.10. Python 3.10 and newer requires OpenSSL 1.1.1 (:pep:`644`): this OpenSSL version provides a C -implementation of :func:`~hashlib.pbkdf2_hmac()` which is faster. Patch by +implementation of :func:`~hashlib.pbkdf2_hmac` which is faster. Patch by Victor Stinner. .. @@ -3260,7 +3258,7 @@ on the main thread Remove ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python 3.10: just use :func:`open` instead. The :func:`open` (:func:`io.open`) -function is a built-in function. Since Python 3.10, :func:`_pyio.open` is +function is a built-in function. Since Python 3.10, :func:`!_pyio.open` is also a static method. Patch by Victor Stinner. .. @@ -3500,7 +3498,7 @@ Illia Volochii. .. nonce: tjfu9L .. section: Library -Deprecate :data:`sqlite3.version` and :data:`sqlite3.version_info`. +Deprecate :data:`!version` and :data:`!version_info`. .. @@ -3563,8 +3561,8 @@ with :func:`os.pidfd_open` in non-blocking mode. Patch by Kumar Aditya. .. nonce: mkYl5q .. section: Library -Implement Enum __contains__ that returns True or False to replace the -deprecated behaviour that would sometimes raise a TypeError. +Implement ``Enum.__contains__`` that returns ``True`` or ``False`` to replace the +deprecated behaviour that would sometimes raise a :exc:`TypeError`. .. @@ -3731,7 +3729,7 @@ In a very special case, the email package tried to append the nonexistent .. nonce: e6uKxj .. section: Library -Fix :func:`ast.unparse` when ``ImportFrom.level`` is None +Fix :func:`ast.unparse` when ``ImportFrom.level`` is ``None`` .. @@ -3793,7 +3791,7 @@ the :c:type:`time_t` type in C. .. section: Library Fixed crash resulting from calling bisect.insort() or bisect.insort_left() -with the key argument not equal to None. +with the key argument not equal to ``None``. .. @@ -4082,7 +4080,7 @@ replacement strings containing group references by 2--3 times. .. section: Library Fix findtext in the xml module to only give an empty string when the text -attribute is set to None. +attribute is set to ``None``. .. @@ -4239,8 +4237,8 @@ by :mod:`asyncio` to AIX platform only. .. nonce: 4dzB80 .. section: Library -Set :attr:`doctest.DocTest.lineno` to ``None`` when object does not have -:attr:`__doc__`. +Set :attr:`doctest.DocTest.lineno` to ``None`` when an object does not have +:attr:`~definition.__doc__`. .. @@ -4404,8 +4402,7 @@ Remove extra row .. section: Documentation Deprecated tools ``make suspicious`` and ``rstlint.py`` are now removed. -They have been replaced by `spinx-lint -`_. +They have been replaced by :pypi:`sphinx-lint`. .. @@ -4479,7 +4476,7 @@ they are deprecated. Contributed by C.A.M. Gerlach. .. nonce: we7AFm .. section: Documentation -Replaced incorrectly written true/false values in documentiation. Patch by +Replaced incorrectly written true/false values in documentation. Patch by Robert O'Shea .. @@ -5613,7 +5610,7 @@ Accept os.PathLike for the argument to winsound.PlaySound Support native Windows case-insensitive path comparisons by using ``LCMapStringEx`` instead of :func:`str.lower` in :func:`ntpath.normcase`. -Add ``LCMapStringEx`` to the :mod:`_winapi` module. +Add ``LCMapStringEx`` to the :mod:`!_winapi` module. .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index a9c5038fa489bb..bc028f30636bf7 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -527,7 +527,7 @@ Stinner. .. nonce: Ai2KDh .. section: Library -Now :mod:`_pyio` is consistent with :mod:`_io` in raising ``ValueError`` +Now :mod:`!_pyio` is consistent with :mod:`!_io` in raising ``ValueError`` when executing methods over closed buffers. .. @@ -537,7 +537,7 @@ when executing methods over closed buffers. .. nonce: 0v8iyw .. section: Library -Clean up refleak on failed module initialisation in :mod:`_zoneinfo` +Clean up refleak on failed module initialisation in :mod:`!_zoneinfo` .. @@ -546,7 +546,7 @@ Clean up refleak on failed module initialisation in :mod:`_zoneinfo` .. nonce: qc_KHr .. section: Library -Clean up refleaks on failed module initialisation in :mod:`_pickle` +Clean up refleaks on failed module initialisation in :mod:`!_pickle` .. @@ -555,7 +555,7 @@ Clean up refleaks on failed module initialisation in :mod:`_pickle` .. nonce: LBl79O .. section: Library -Clean up refleak on failed module initialisation in :mod:`_io`. +Clean up refleak on failed module initialisation in :mod:`!_io`. .. @@ -706,7 +706,7 @@ Remove modules :mod:`!asyncore` and :mod:`!asynchat`, which were deprecated by .. section: Library Fix handling of ``bytes`` :term:`path-like objects ` in -:func:`os.ismount()`. +:func:`os.ismount`. .. @@ -822,7 +822,7 @@ and to indicate when it became late-bound. .. nonce: 7KinCV .. section: Tests -The Python test suite now fails wit exit code 4 if no tests ran. It should +The Python test suite now fails with exit code 4 if no tests ran. It should help detecting typos in test names and test methods. .. @@ -968,7 +968,7 @@ if :option:`--with-system-expat` is passed to :program:`configure`. .. nonce: 0f6e_N .. section: Windows -Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but the +Update Windows builds to zlib v1.2.13. v1.2.12 has :cve:`2022-37434`, but the vulnerable ``inflateGetHeader`` API is not used by Python. .. diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst index ce128fd5f80c77..04a2bf9fb916b7 100644 --- a/Misc/NEWS.d/3.12.0a3.rst +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -70,7 +70,7 @@ Fix bug where compiler crashes on an if expression with an empty body block. .. nonce: DcKoBJ .. section: Core and Builtins -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +Fix a reference bug in :func:`!_imp.create_builtin` after the creation of the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor Stinner. @@ -82,7 +82,7 @@ Victor Stinner. .. section: Core and Builtins Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the +line missing the newline character from a file that is as long as the available tokenizer buffer. Patch by Pablo galindo .. @@ -221,7 +221,7 @@ returns an invalid file descriptor. Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so that it is technically possible to parse the line and reconstruct what the -original data was. Without this a \xHH is ambiguious as to if it is a hex +original data was. Without this a \xHH is ambiguous as to if it is a hex replacement we put in or the characters r"\x" came through in the original request line. @@ -399,7 +399,7 @@ Fix refcount error when arguments are packed to tuple in Argument Clinic. .. nonce: 7uCiIB .. section: Library -:meth:`pathlib.PurePath.relative_to()` now treats naked Windows drive paths +:meth:`pathlib.PurePath.relative_to` now treats naked Windows drive paths as relative. This brings its behaviour in line with other parts of pathlib. .. @@ -496,7 +496,7 @@ Created packages from zipfile and test_zipfile modules, separating Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would -return True from this method; now they correctly return False. +return ``True`` from this method; now they correctly return ``False``. .. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index 82faa5ad0b2031..57fb2052764b6f 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -65,8 +65,8 @@ redundant. .. nonce: M2n6Kg .. section: Core and Builtins -Fix :func:`int.__sizeof__` calculation to include the 1 element ob_digit -array for 0 and False. +Fix :func:`int.__sizeof__` calculation to include the 1-element ``ob_digit`` +array for ``0`` and ``False``. .. @@ -241,7 +241,7 @@ are now always dumped, even if switched off. Improve ``BUILD_LIST`` opcode so that it works similarly to the ``BUILD_TUPLE`` opcode, by stealing references from the stack rather than repeatedly using stack operations to set list elements. Implementation -details are in a new private API :c:func:`_PyList_FromArraySteal`. +details are in a new private API :c:func:`!_PyList_FromArraySteal`. .. @@ -611,8 +611,8 @@ random.expovariate(). .. nonce: bgtzMV .. section: Library -A :exc:`DeprecationWarning` may be raised when :func:`os.fork()` or -:func:`os.forkpty()` is called from multi-threaded processes. Forking with +A :exc:`DeprecationWarning` may be raised when :func:`os.fork` or +:func:`os.forkpty` is called from multi-threaded processes. Forking with threads is unsafe and can cause deadlocks, crashes and subtle problems. Lack of a warning does not indicate that the fork call was actually safe, as Python may not be aware of all threads. @@ -830,7 +830,7 @@ Reduced the memory usage of :func:`urllib.parse.unquote` and .. section: Library ``inspect.signature`` was raising ``TypeError`` on call with mock objects. -Now it correctly returns ``(*args, **kwargs)`` as infered signature. +Now it correctly returns ``(*args, **kwargs)`` as inferred signature. .. diff --git a/Misc/NEWS.d/3.12.0a5.rst b/Misc/NEWS.d/3.12.0a5.rst index 8cf90b0e9cde46..5dc443bb55b617 100644 --- a/Misc/NEWS.d/3.12.0a5.rst +++ b/Misc/NEWS.d/3.12.0a5.rst @@ -287,7 +287,7 @@ a positional argument would lead to a :exc:`TypeError`. .. section: Library Group-related variables of ``_posixsubprocess`` module are renamed to stress -that supplimentary group affinity is added to a fork, not replace the +that supplementary group affinity is added to a fork, not replace the inherited ones. Patch by Oleg Iarygin. .. @@ -307,7 +307,7 @@ It must not drop the ``Unpack`` part. .. nonce: wz4Xgc .. section: Library -Add :func:`os.path.splitroot()`, which splits a path into a 3-item tuple +Add :func:`os.path.splitroot`, which splits a path into a 3-item tuple ``(drive, root, tail)``. This new function is used by :mod:`pathlib` to improve the performance of path construction by up to a third. @@ -506,7 +506,7 @@ inheritance. .. nonce: 7sQz5l .. section: Build -Update BOLT configration not to use depreacted usage of ``--split +Update BOLT configuration not to use deprecated usage of ``--split functions``. Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst index cf28bdb9258820..bc708d163ce0e9 100644 --- a/Misc/NEWS.d/3.12.0a6.rst +++ b/Misc/NEWS.d/3.12.0a6.rst @@ -15,9 +15,9 @@ from the HACL* project. .. section: Security Updated the OpenSSL version used in Windows and macOS binary release builds -to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per +to 1.1.1t to address :cve:`2023-0286`, :cve:`2022-4303`, and :cve:`2022-4303` per `the OpenSSL 2023-02-07 security advisory -`_. +`_. .. @@ -453,7 +453,7 @@ E. Aasland. .. section: Library Change repr of :class:`collections.OrderedDict` to use regular dictionary -formating instead of pairs of keys and values. +formatting instead of pairs of keys and values. .. diff --git a/Misc/NEWS.d/3.12.0a7.rst b/Misc/NEWS.d/3.12.0a7.rst index a859be8a047456..f48b9ce0550440 100644 --- a/Misc/NEWS.d/3.12.0a7.rst +++ b/Misc/NEWS.d/3.12.0a7.rst @@ -219,7 +219,7 @@ Aasland. .. nonce: DqNehf .. section: Library -Pure python :func:`locale.getencoding()` will not warn deprecation. +Pure python :func:`locale.getencoding` will not warn deprecation. .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 211513d05d0040..3a3870ac9fe621 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -37,7 +37,7 @@ or lacks SHA3. :func:`urllib.parse.urlsplit` now strips leading C0 control and space characters following the specification for URLs defined by WHATWG in -response to CVE-2023-24329. Patch by Illia Volochii. +response to :cve:`2023-24329`. Patch by Illia Volochii. .. @@ -395,7 +395,7 @@ Fix bug in line numbers of instructions emitted for :keyword:`except* .. section: Core and Builtins Clarify :exc:`SyntaxWarning` with literal ``is`` comparison by specifying -which literal is problematic, since comparisons using ``is`` with e.g. None +which literal is problematic, since comparisons using ``is`` with e.g. ``None`` and bool literals are idiomatic. .. @@ -563,10 +563,10 @@ Complex function calls are now faster and consume no C stack space. .. nonce: fvgsCl .. section: Core and Builtins -``len()`` for 0-dimensional :class:`memoryview`` objects (such as +``len()`` for 0-dimensional :class:`memoryview` objects (such as ``memoryview(ctypes.c_uint8(42))``) now raises a :exc:`TypeError`. Previously this returned ``1``, which was not consistent with ``mem_0d[0]`` -raising an :exc:`IndexError``. +raising an :exc:`IndexError`. .. @@ -1446,7 +1446,7 @@ Adapt the :mod:`winsound` extension module to :pep:`687`. .. nonce: jurMzv .. section: Library -Remove deprecation of enum ``memmber.member`` access. +Remove deprecation of enum ``member.member`` access. .. @@ -1750,7 +1750,7 @@ Remove the long-deprecated ``imp`` module. .. nonce: N46coo .. section: Library -Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` in +Deprecate :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` in favor of :func:`importlib.util.find_spec`. .. @@ -1828,7 +1828,7 @@ is relative. .. nonce: 511Tbh .. section: Library -Convert private :meth:`_posixsubprocess.fork_exec` to use Argument Clinic. +Convert private :meth:`!_posixsubprocess.fork_exec` to use Argument Clinic. .. @@ -2371,7 +2371,7 @@ Add a new C-API function to eagerly assign a version tag to a PyTypeObject: .. nonce: _paFIF .. section: C API -:c:func:`PyObject_GC_Resize` should calculate preheader size if needed. +:c:macro:`PyObject_GC_Resize` should calculate preheader size if needed. Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index d385b6a4504f97..c32c9a537d7b53 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -8,9 +8,7 @@ Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. Security issue reported as -`CVE-2023-40217 -`_ by Aapo -Oksman. Patch by Gregory P. Smith. +:cve:`2023-40217` by Aapo Oksman. Patch by Gregory P. Smith. .. @@ -565,7 +563,7 @@ on deallocation. Fix :meth:`multiprocessing.synchronize.SemLock.__setstate__` to properly initialize :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx`. This -fixes a regression when passing a SemLock accross nested processes. +fixes a regression when passing a SemLock across nested processes. Rename :attr:`multiprocessing.synchronize.SemLock.is_fork_ctx` to :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx` to avoid exposing @@ -710,7 +708,7 @@ Fixes crash when tracing in recursive calls to Python classes. .. section: Core and Builtins Remove the ``_PyCFrame`` struct, moving the pointer to the current -intepreter frame back to the threadstate, as it was for 3.10 and earlier. +interpreter frame back to the threadstate, as it was for 3.10 and earlier. The ``_PyCFrame`` existed as a performance optimization for tracing. Since PEP 669 has been implemented, this optimization no longer applies. @@ -928,7 +926,7 @@ Isolate :mod:`!_decimal` (apply :pep:`687`). Patch by Charlie Zhao. Add the exception as the third argument to ``PY_UNIND`` callbacks in ``sys.monitoring``. This makes the ``PY_UNWIND`` callback consistent with -the other exception hanlding callbacks. +the other exception handling callbacks. .. @@ -937,7 +935,7 @@ the other exception hanlding callbacks. .. nonce: DdEwV8 .. section: Core and Builtins -Raise a ``ValueError`` when a monitoring callback funtion returns +Raise a ``ValueError`` when a monitoring callback function returns ``DISABLE`` for events that cannot be disabled locally. .. @@ -1008,7 +1006,7 @@ Add :meth:`dbm.gnu.gdbm.clear` to :mod:`dbm.gnu`. Patch By Donghee Na. .. section: Core and Builtins The ASYNC and AWAIT tokens are removed from the Grammar, which removes the -posibility of making ``async`` and ``await`` soft keywords when using +possibility of making ``async`` and ``await`` soft keywords when using ``feature_version<7`` in :func:`ast.parse`. .. @@ -1030,7 +1028,7 @@ the call is not a classmethod. .. nonce: DdqHFg .. section: Core and Builtins -Python no longer crashes due an infrequent race when initialzing +Python no longer crashes due an infrequent race when initializing per-interpreter interned strings. The crash would manifest when the interpreter was finalized. @@ -1924,7 +1922,7 @@ objects .. nonce: RDGe8- .. section: Library -Deprecation warning about non-integer number in :mod:`gettext` now alwais +Deprecation warning about non-integer number in :mod:`gettext` now always refers to the line in the user code where gettext function or method is used. Previously it could refer to a line in ``gettext`` code. @@ -1966,7 +1964,7 @@ debugging. .. nonce: LCxiau .. section: Library -Fix :func:`termios.tcsetattr` bug that was overwritting existing errors +Fix :func:`termios.tcsetattr` bug that was overwriting existing errors during parsing integers from ``term`` list. .. @@ -2049,7 +2047,7 @@ point. .. nonce: fECxTj .. section: Library -On Windows, multiprocessing ``Popen.terminate()`` now catchs +On Windows, multiprocessing ``Popen.terminate()`` now catches :exc:`PermissionError` and get the process exit code. If the process is still running, raise again the :exc:`PermissionError`. Otherwise, the process terminated as expected: store its exit code. Patch by Victor @@ -2859,7 +2857,7 @@ Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` instead of ``SSL_ERROR_SSL`` when a certification verification has failed, but the error parameters will still contain ``ERR_LIB_SSL`` and ``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and -raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +raising the appropriate ``ssl.SSLCertVerificationError``. Patch by Pablo Galindo .. @@ -2890,9 +2888,9 @@ documented and were not intended to be used externally. .. nonce: vMbmj_ .. section: Library -:data:`opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never +:data:`!opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never documented or intended for external usage) is moved to -:data:`_opcode.ENABLE_SPECIALIZATION` where tests can access it. +:data:`!_opcode.ENABLE_SPECIALIZATION` where tests can access it. .. @@ -2981,7 +2979,7 @@ method. Patch by James Cave. .. section: Library Fix overflow on 32-bit systems with :mod:`asyncio` :func:`os.sendfile` -implemention. +implementation. .. @@ -3055,7 +3053,7 @@ Donghee Na. .. nonce: U9nD_B .. section: Library -Optimize :meth:`_PollLikeSelector.select` for many iteration case. +Optimize :meth:`!_PollLikeSelector.select` for many iteration case. .. @@ -3175,7 +3173,7 @@ Disable tab completion in multiline mode of :mod:`pdb` .. nonce: pYSwMj .. section: Library -Expose opcode metadata through :mod:`_opcode`. +Expose opcode metadata through :mod:`!_opcode`. .. @@ -3253,7 +3251,7 @@ Eliseev. .. nonce: NN35-U .. section: Library -Optimize ``(?!)`` (pattern which alwais fails) in regular expressions. +Optimize ``(?!)`` (pattern which always fails) in regular expressions. .. @@ -3438,7 +3436,8 @@ added support for this decorator. Patch by Alex Waygood. .. nonce: C1ahtk .. section: Library -Make pydoc.doc catch bad module ImportError when output stream is not None. +Make :func:`pydoc.doc` catch bad module :exc:`ImportError` +when output stream is not ``None``. .. @@ -3496,7 +3495,7 @@ star imports. .. nonce: TJEUkd .. section: Library -Zipapp will now skip over apending an archive to itself. +Zipapp will now skip over appending an archive to itself. .. @@ -3736,7 +3735,7 @@ overwritten. .. nonce: _sZilh .. section: Library -Fix bugs in :mod:`_ctypes` where exceptions could end up being overwritten. +Fix bugs in :mod:`!_ctypes` where exceptions could end up being overwritten. .. @@ -4184,8 +4183,7 @@ Hugo van Kemenade. .. section: Library :pep:`594`: Remove the :mod:`!spwd` module, deprecated in Python 3.11: the -`python-pam project `_ can be used -instead. Patch by Victor Stinner. +:pypi:`python-pam` project can be used instead. Patch by Victor Stinner. .. @@ -4380,7 +4378,7 @@ Patch by Victor Stinner. .. nonce: I6MQhb .. section: Library -:pep:`594`: Remove the :mod:`!cgi`` and :mod:`!cgitb` modules, deprecated in +:pep:`594`: Remove the :mod:`!cgi` and :mod:`!cgitb` modules, deprecated in Python 3.11. Patch by Victor Stinner. .. @@ -4566,7 +4564,7 @@ Deprecate passing any arguments to :func:`threading.RLock`. .. nonce: o5Zb0t .. section: Library -Refactored ``zipfile._strip_extra`` to use higher level abstactions for +Refactored ``zipfile._strip_extra`` to use higher level abstractions for extras instead of a heavy-state loop. .. @@ -5016,7 +5014,7 @@ by Victor Stinner. Fix test_timeout() of test_concurrent_futures.test_wait. Remove the future which may or may not complete depending if it takes longer than the timeout -ot not. Keep the second future which does not complete before wait() +or not. Keep the second future which does not complete before wait() timeout. Patch by Victor Stinner. .. @@ -5106,7 +5104,7 @@ Victor Stinner. regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests). -``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all +``--slow-ci`` uses a default timeout of 20 minutes and ``-u all`` (run all tests). Patch by Victor Stinner. .. @@ -5150,7 +5148,7 @@ frame did not save the PC`` is found. Patch by Victor Stinner. .. nonce: qxI4OG .. section: Tests -libregrtest now calls :func:`random.seed()` before running each test file +libregrtest now calls :func:`random.seed` before running each test file when ``-r/--randomize`` command line option is used. Moreover, it's also called in worker processes. It should help to make tests more deterministic. Previously, it was only called once in the main process @@ -5234,7 +5232,7 @@ and ``sysctl net.inet.udp.blackhole=1``). Patch by Victor Stinner. Skip ``test_gdb`` if gdb is unable to retrieve Python frame objects: if a frame is ````. When Python is built with "clang -Og", gdb can -fail to retrive the *frame* parameter of ``_PyEval_EvalFrameDefault()``. In +fail to retrieve the *frame* parameter of ``_PyEval_EvalFrameDefault()``. In this case, tests like ``py_bt()`` are likely to fail. Without getting access to Python frames, ``python-gdb.py`` is mostly clueless on retrieving the Python traceback. Moreover, ``test_gdb`` is no longer skipped on macOS if diff --git a/Misc/NEWS.d/3.13.0a2.rst b/Misc/NEWS.d/3.13.0a2.rst index e5841e14c02efb..a1a2d8a42ec03b 100644 --- a/Misc/NEWS.d/3.13.0a2.rst +++ b/Misc/NEWS.d/3.13.0a2.rst @@ -228,7 +228,7 @@ cross-interpreter API. Patch by Anthony Shaw. .. nonce: ageUWQ .. section: Core and Builtins -Add support for sharing of True and False between interpreters using the +Add support for sharing of ``True`` and ``False`` between interpreters using the cross-interpreter API. Patch by Anthony Shaw. .. @@ -632,8 +632,8 @@ Add extra argument validation for ``alias`` command in :mod:`pdb` .. nonce: FWqZIU .. section: Library -:mod:`time`: Make :func:`time.clock_gettime()` and -:func:`time.clock_gettime_ns()` functions up to 2x faster by faster calling +:mod:`time`: Make :func:`time.clock_gettime` and +:func:`time.clock_gettime_ns` functions up to 2x faster by faster calling convention. Patch by Victor Stinner. .. @@ -674,7 +674,7 @@ Unix socket when the server is closed. .. nonce: Bc8LvA .. section: Library -Added :func:`io.text_encoding()`, :data:`io.DEFAULT_BUFFER_SIZE`, and +Added :func:`io.text_encoding`, :data:`io.DEFAULT_BUFFER_SIZE`, and :class:`io.IncrementalNewlineDecoder` to ``io.__all__``. .. @@ -736,7 +736,7 @@ which allows to format :exc:`ExceptionGroup` instances. .. nonce: 5ePgFl .. section: Library -Another attempt at fixing :func:`asyncio.Server.wait_closed()`. It now +Another attempt at fixing :func:`asyncio.Server.wait_closed`. It now blocks until both conditions are true: the server is closed, *and* there are no more active connections. (This means that in some cases where in 3.12.0 this function would *incorrectly* have returned immediately, it will now @@ -777,7 +777,7 @@ Add error checking during :mod:`!_socket` module init. .. nonce: urFYtn .. section: Library -Fix :mod:`_blake2` not checking for errors when initializing. +Fix :mod:`!_blake2` not checking for errors when initializing. .. @@ -880,7 +880,7 @@ Make :mod:`pdb` enter post-mortem mode even for :exc:`SyntaxError` .. nonce: _M-cQC .. section: Library -Set ``f_trace_lines = True`` on all frames upon :func:`pdb.set_trace()` +Set ``f_trace_lines = True`` on all frames upon :func:`pdb.set_trace` .. @@ -974,7 +974,7 @@ pattern. .. nonce: 6ah-aw .. section: Library -Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which retuns the +Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. .. @@ -1354,8 +1354,8 @@ crash encountered after the first :meth:`tkinter.Tk` instance is destroyed. .. section: IDLE Add docstrings to the IDLE debugger module. Fix two bugs: initialize -Idb.botframe (should be in Bdb); in Idb.in_rpc_code, check whether -prev_frame is None before trying to use it. Greatly expand test_debugger. +``Idb.botframe`` (should be in Bdb); in ``Idb.in_rpc_code``, check whether +``prev_frame`` is ``None`` before trying to use it. Greatly expand test_debugger. .. diff --git a/Misc/NEWS.d/3.13.0a3.rst b/Misc/NEWS.d/3.13.0a3.rst index 95aa66603de7cb..0f8dee261c6589 100644 --- a/Misc/NEWS.d/3.13.0a3.rst +++ b/Misc/NEWS.d/3.13.0a3.rst @@ -269,7 +269,7 @@ Correctly compute end column offsets for multiline tokens in the .. nonce: 4ADN7i .. section: Core and Builtins -Fix None.__ne__(None) returning NotImplemented instead of False +Fix ``None.__ne__(None)`` returning ``NotImplemented`` instead of ``False``. .. @@ -449,8 +449,8 @@ well-formed for surrogateescape encoding. Patch by Sidney Markowitz. .. nonce: N8E1zw .. section: Core and Builtins -Use the object's actual class name in :meth:`_io.FileIO.__repr__`, -:meth:`_io._WindowsConsoleIO` and :meth:`_io.TextIOWrapper.__repr__`, to +Use the object's actual class name in :meth:`!_io.FileIO.__repr__`, +:meth:`!_io._WindowsConsoleIO` and :meth:`!_io.TextIOWrapper.__repr__`, to make these methods subclass friendly. .. @@ -555,7 +555,7 @@ Added :data:`mmap.MAP_NORESERVE`, :data:`mmap.MAP_NOEXTEND`, .. nonce: kXoCy0 .. section: Library -:func:`asyncio.TaskGroup()` and :func:`asyncio.timeout()` context managers +:func:`asyncio.TaskGroup` and :func:`asyncio.timeout` context managers now handle :exc:`~asyncio.CancelledError` subclasses as well as exact :exc:`!CancelledError`. @@ -607,9 +607,9 @@ with the documentation) .. nonce: xN2LuL .. section: Library -:func:`asyncio.Condition.wait()` now re-raises the same +:func:`asyncio.Condition.wait` now re-raises the same :exc:`CancelledError` instance that may have caused it to be interrupted. -Fixed race condition in :func:`asyncio.Semaphore.aquire` when interrupted +Fixed race condition in :func:`asyncio.Semaphore.acquire` when interrupted with a :exc:`CancelledError`. .. @@ -863,7 +863,7 @@ Fixed tarfile list() method to show file type. .. nonce: jLWGlr .. section: Library -:meth:`asyncio.futures.Future.set_exception()` now transforms +:meth:`asyncio.futures.Future.set_exception` now transforms :exc:`StopIteration` into :exc:`RuntimeError` instead of hanging or other misbehavior. Patch contributed by Jamie Phan. @@ -928,7 +928,7 @@ on Windows. .. section: Library Fix :func:`shutil.copymode` and :func:`shutil.copystat` on Windows. -Previously they worked differenly if *dst* is a symbolic link: they modified +Previously they worked differently if *dst* is a symbolic link: they modified the permission bits of *dst* itself rather than the file it points to if *follow_symlinks* is true or *src* is not a symbolic link, and did not modify the permission bits if *follow_symlinks* is false and *src* is a @@ -1550,8 +1550,8 @@ addresses are encountered instead of potentially inaccurate values. Add optional *strict* parameter to these two functions: use ``strict=False`` to get the old behavior, accept malformed inputs. ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check if the *strict* -paramater is available. Patch by Thomas Dwyer and Victor Stinner to improve -the CVE-2023-27043 fix. +parameter is available. Patch by Thomas Dwyer and Victor Stinner to improve +the :cve:`2023-27043` fix. .. @@ -1615,7 +1615,7 @@ method of :class:`itertools.pairwise`. .. section: Library Small (10 - 20%) and trivial performance improvement of -:func:`urrlib.request.getproxies_environment`, typically useful when there +:func:`urllib.request.getproxies_environment`, typically useful when there are many environment variables to go over. .. @@ -1805,7 +1805,7 @@ size. .. nonce: xPOBBY .. section: Library -:func:`warnings.filterwarnings()` and :func:`warnings.simplefilter()` now +:func:`warnings.filterwarnings` and :func:`warnings.simplefilter` now raise appropriate exceptions instead of ``AssertionError``. Patch contributed by Rémi Lapeyre. diff --git a/Misc/NEWS.d/3.13.0a4.rst b/Misc/NEWS.d/3.13.0a4.rst new file mode 100644 index 00000000000000..1b971113173e0a --- /dev/null +++ b/Misc/NEWS.d/3.13.0a4.rst @@ -0,0 +1,1433 @@ +.. date: 2024-02-13-15-14-39 +.. gh-issue: 115399 +.. nonce: xT-scP +.. release date: 2024-02-15 +.. section: Security + +Update bundled libexpat to 2.6.0 + +.. + +.. date: 2024-02-12-00-33-01 +.. gh-issue: 115243 +.. nonce: e1oGX8 +.. section: Security + +Fix possible crashes in :meth:`collections.deque.index` when the deque is +concurrently modified. + +.. + +.. date: 2024-02-14-23-50-55 +.. gh-issue: 112087 +.. nonce: H_4W_v +.. section: Core and Builtins + +For an empty reverse iterator for list will be reduced to :func:`reversed`. +Patch by Donghee Na + +.. + +.. date: 2024-02-12-17-18-26 +.. gh-issue: 114570 +.. nonce: BzwMlJ +.. section: Core and Builtins + +Add :exc:`PythonFinalizationError` exception. This exception derived from +:exc:`RuntimeError` is raised when an operation is blocked during the +:term:`Python finalization `. Patch by Victor Stinner. + +.. + +.. date: 2024-02-07-18-04-36 +.. gh-issue: 114695 +.. nonce: o9wP5P +.. section: Core and Builtins + +Add :func:`sys._clear_internal_caches`, which clears all internal +performance-related caches (and deprecate the less-general +:func:`sys._clear_type_cache` function). + +.. + +.. date: 2024-02-07-07-50-12 +.. gh-issue: 114828 +.. nonce: nSXwMi +.. section: Core and Builtins + +Fix compilation crashes in uncommon code examples using :func:`super` inside +a comprehension in a class body. + +.. + +.. date: 2024-02-07-00-18-42 +.. gh-issue: 112069 +.. nonce: jRDRR5 +.. section: Core and Builtins + +Adapt :class:`set` and :class:`frozenset` methods to Argument Clinic. + +.. + +.. date: 2024-02-05-12-40-26 +.. gh-issue: 115011 +.. nonce: L1AKF5 +.. section: Core and Builtins + +Setters for members with an unsigned integer type now support the same range +of valid values for objects that has a :meth:`~object.__index__` method as +for :class:`int`. + +.. + +.. date: 2024-02-03-04-07-18 +.. gh-issue: 114887 +.. nonce: uLSFmN +.. section: Core and Builtins + +Changed socket type validation in +:meth:`~asyncio.loop.create_datagram_endpoint` to accept all non-stream +sockets. This fixes a regression in compatibility with raw sockets. + +.. + +.. date: 2024-02-03-01-48-38 +.. gh-issue: 114944 +.. nonce: 4J5ELD +.. section: Core and Builtins + +Fixes a race between ``PyParkingLot_Park`` and ``_PyParkingLot_UnparkAll``. + +.. + +.. date: 2024-02-02-05-27-48 +.. gh-issue: 113462 +.. nonce: VMml8q +.. section: Core and Builtins + +Limit the number of versions that a single class can use. Prevents a few +wayward classes using up all the version numbers. + +.. + +.. date: 2024-02-01-23-43-49 +.. gh-issue: 76763 +.. nonce: o_2J6i +.. section: Core and Builtins + +The :func:`chr` builtin function now always raises :exc:`ValueError` for +values outside the valid range. Previously it raised :exc:`OverflowError` +for very large or small values. + +.. + +.. date: 2024-02-01-18-16-52 +.. gh-issue: 114806 +.. nonce: wrH2J6 +.. section: Core and Builtins + +No longer specialize calls to classes, if those classes have metaclasses. +Fixes bug where the ``__call__`` method of the metaclass was not being +called. + +.. + +.. date: 2024-01-31-09-10-10 +.. gh-issue: 107944 +.. nonce: XWm1B- +.. section: Core and Builtins + +Improve error message for function calls with bad keyword arguments via +getargs + +.. + +.. date: 2024-01-25-18-50-49 +.. gh-issue: 112529 +.. nonce: IbbApA +.. section: Core and Builtins + +The free-threaded build no longer allocates space for the ``PyGC_Head`` +structure in objects that support cyclic garbage collection. A number of +other fields and data structures are used as replacements, including +``ob_gc_bits``, ``ob_tid``, and mimalloc internal data structures. + +.. + +.. date: 2024-01-22-15-10-01 +.. gh-issue: 114456 +.. nonce: fBFEJF +.. section: Core and Builtins + +Lower the recursion limit under a debug build of WASI. + +.. + +.. date: 2024-01-22-09-49-02 +.. gh-issue: 114083 +.. nonce: hf1-ku +.. section: Core and Builtins + +Compiler applies folding of LOAD_CONST with following instruction in a +separate pass before other optimisations. This enables jump threading in +certain circumstances. + +.. + +.. date: 2024-01-21-17-29-32 +.. gh-issue: 114388 +.. nonce: UVGO4K +.. section: Core and Builtins + +Fix a :exc:`RuntimeWarning` emitted when assign an integer-like value that +is not an instance of :class:`int` to an attribute that corresponds to a C +struct member of :ref:`type ` T_UINT and T_ULONG. Fix a +double :exc:`RuntimeWarning` emitted when assign a negative integer value to +an attribute that corresponds to a C struct member of type T_UINT. + +.. + +.. date: 2024-01-19-13-18-13 +.. gh-issue: 114265 +.. nonce: 7HAi-- +.. section: Core and Builtins + +Compiler propagates line numbers before optimization, leading to more +optimization opportunities and removing the need for the +``guarantee_lineno_for_exits`` hack. + +.. + +.. date: 2024-01-18-20-20-37 +.. gh-issue: 112529 +.. nonce: oVNvDG +.. section: Core and Builtins + +The free-threaded build now has its own thread-safe GC implementation that +uses mimalloc to find GC tracked objects. It is non-generational, unlike the +existing GC implementation. + +.. + +.. date: 2024-01-17-23-39-20 +.. gh-issue: 114050 +.. nonce: Lnv1oq +.. section: Core and Builtins + +Fix segmentation fault caused by an incorrect format string in ``TypeError`` +exception when more than two arguments are passed to ``int``. + +.. + +.. date: 2024-01-17-05-09-32 +.. gh-issue: 112354 +.. nonce: Run9ko +.. section: Core and Builtins + +The ``END_FOR`` instruction now pops only one value. This is to better +support side exits in loops. + +.. + +.. date: 2024-01-17-00-52-57 +.. gh-issue: 113884 +.. nonce: CvEjUE +.. section: Core and Builtins + +Make :class:`queue.SimpleQueue` thread safe when the GIL is disabled. + +.. + +.. date: 2024-01-16-14-41-54 +.. gh-issue: 114058 +.. nonce: Cb2b8h +.. section: Core and Builtins + +Implement the foundations of the Tier 2 redundancy eliminator. + +.. + +.. date: 2024-01-12-16-40-07 +.. gh-issue: 113939 +.. nonce: Yi3L-e +.. section: Core and Builtins + +frame.clear(): Clear frame.f_locals as well, and not only the fast locals. +This is relevant once frame.f_locals was accessed, which would contain also +references to all the locals. + +.. + +.. date: 2024-01-11-22-58-45 +.. gh-issue: 112050 +.. nonce: hDuvDW +.. section: Core and Builtins + +Convert :class:`collections.deque` to use Argument Clinic. + +.. + +.. date: 2024-01-08-21-57-41 +.. gh-issue: 112050 +.. nonce: qwgjx1 +.. section: Core and Builtins + +Make methods on :class:`collections.deque` thread-safe when the GIL is +disabled. + +.. + +.. date: 2023-12-24-03-25-28 +.. gh-issue: 113464 +.. nonce: dvjQmA +.. section: Core and Builtins + +Add an option (``--enable-experimental-jit`` for ``configure``-based builds +or ``--experimental-jit`` for ``PCbuild``-based ones) to build an +*experimental* just-in-time compiler, based on `copy-and-patch +`_ + +.. + +.. date: 2023-12-22-13-21-39 +.. gh-issue: 113055 +.. nonce: 47xBMF +.. section: Core and Builtins + +Make interp->obmalloc a pointer. For interpreters that share state with the +main interpreter, this points to the same static memory structure. For +interpreters with their own obmalloc state, it is heap allocated. Add +free_obmalloc_arenas() which will free the obmalloc arenas and radix tree +structures for interpreters with their own obmalloc state. + +.. + +.. date: 2023-06-06-19-09-00 +.. gh-issue: 55664 +.. nonce: vYYl0V +.. section: Core and Builtins + +Add warning when creating :class:`type` using a namespace dictionary with +non-string keys. Patched by Daniel Urban and Furkan Onder. + +.. + +.. date: 2023-05-16-06-52-34 +.. gh-issue: 104530 +.. nonce: mJnA0W +.. section: Core and Builtins + +Use native Win32 condition variables. + +.. + +.. date: 2024-02-13-18-27-03 +.. gh-issue: 115392 +.. nonce: gle5tp +.. section: Library + +Fix a bug in :mod:`doctest` where incorrect line numbers would be reported +for decorated functions. + +.. + +.. date: 2024-02-11-20-23-36 +.. gh-issue: 114563 +.. nonce: RzxNYT +.. section: Library + +Fix several :func:`format` bugs when using the C implementation of +:class:`~decimal.Decimal`: * memory leak in some rare cases when using the +``z`` format option (coerce negative 0) * incorrect output when applying the +``z`` format option to type ``F`` (fixed-point with capital ``NAN`` / +``INF``) * incorrect output when applying the ``#`` format option (alternate +form) + +.. + +.. date: 2024-02-10-15-24-20 +.. gh-issue: 102840 +.. nonce: 4mnDq1 +.. section: Library + +Fix confused traceback when floordiv, mod, or divmod operations happens +between instances of :class:`fractions.Fraction` and :class:`complex`. + +.. + +.. date: 2024-02-09-07-20-16 +.. gh-issue: 115165 +.. nonce: yfJLXA +.. section: Library + +Most exceptions are now ignored when attempting to set the +``__orig_class__`` attribute on objects returned when calling :mod:`typing` +generic aliases (including generic aliases created using +:data:`typing.Annotated`). Previously only :exc:`AttributeError` was +ignored. Patch by Dave Shawley. + +.. + +.. date: 2024-02-08-17-04-58 +.. gh-issue: 112903 +.. nonce: SN_vUs +.. section: Library + +Fix "issubclass() arg 1 must be a class" errors in certain cases of multiple +inheritance with generic aliases (regression in early 3.13 alpha releases). + +.. + +.. date: 2024-02-08-14-21-28 +.. gh-issue: 115133 +.. nonce: ycl4ko +.. section: Library + +Fix tests for :class:`~xml.etree.ElementTree.XMLPullParser` with Expat +2.6.0. + +.. + +.. date: 2024-02-08-13-26-14 +.. gh-issue: 115059 +.. nonce: DqP9dr +.. section: Library + +:meth:`io.BufferedRandom.read1` now flushes the underlying write buffer. + +.. + +.. date: 2024-02-07-12-37-52 +.. gh-issue: 79382 +.. nonce: Yz_5WB +.. section: Library + +Trailing ``**`` no longer allows to match files and non-existing paths in +recursive :func:`~glob.glob`. + +.. + +.. date: 2024-02-06-15-16-28 +.. gh-issue: 67837 +.. nonce: _JKa73 +.. section: Library + +Avoid race conditions in the creation of directories during concurrent +extraction in :mod:`tarfile` and :mod:`zipfile`. + +.. + +.. date: 2024-02-06-03-55-46 +.. gh-issue: 115060 +.. nonce: EkWRpP +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by removing redundant regex matching. + +.. + +.. date: 2024-02-05-16-48-06 +.. gh-issue: 97928 +.. nonce: JZCies +.. section: Library + +Partially revert the behavior of :meth:`tkinter.Text.count`. By default it +preserves the behavior of older Python versions, except that setting +``wantobjects`` to 0 no longer has effect. Add a new parameter +*return_ints*: specifying ``return_ints=True`` makes ``Text.count()`` always +returning the single count as an integer instead of a 1-tuple or ``None``. + +.. + +.. date: 2024-02-04-13-17-33 +.. gh-issue: 114628 +.. nonce: WJpqqS +.. section: Library + +When csv.Error is raised when handling TypeError, do not print the TypeError +traceback. + +.. + +.. date: 2024-02-04-02-28-37 +.. gh-issue: 85984 +.. nonce: NHZVTQ +.. section: Library + +Added ``_POSIX_VDISABLE`` from C's ```` to :mod:`termios`. + +.. + +.. date: 2024-02-03-17-54-17 +.. gh-issue: 114965 +.. nonce: gHksCK +.. section: Library + +Update bundled pip to 24.0 + +.. + +.. date: 2024-02-03-16-59-25 +.. gh-issue: 114959 +.. nonce: dCfAG2 +.. section: Library + +:mod:`tarfile` no longer ignores errors when trying to extract a directory +on top of a file. + +.. + +.. date: 2024-02-02-15-50-13 +.. gh-issue: 114894 +.. nonce: DF-dSd +.. section: Library + +Add :meth:`array.array.clear`. + +.. + +.. date: 2024-02-01-10-19-11 +.. gh-issue: 114071 +.. nonce: vkm2G_ +.. section: Library + +Support tuple subclasses using auto() for enum member value. + +.. + +.. date: 2024-01-31-20-07-11 +.. gh-issue: 109475 +.. nonce: lmTb9S +.. section: Library + +Fix support of explicit option value "--" in :mod:`argparse` (e.g. +``--option=--``). + +.. + +.. date: 2024-01-30-22-10-50 +.. gh-issue: 49766 +.. nonce: yulJL_ +.. section: Library + +Fix :class:`~datetime.date`-:class:`~datetime.datetime` comparison. Now the +special comparison methods like ``__eq__`` and ``__lt__`` return +:data:`NotImplemented` if one of comparands is :class:`!date` and other is +:class:`!datetime` instead of ignoring the time part and the time zone or +forcefully return "not equal" or raise :exc:`TypeError`. It makes comparison +of :class:`!date` and :class:`!datetime` subclasses more symmetric and +allows to change the default behavior by overriding the special comparison +methods in subclasses. + +.. + +.. date: 2024-01-30-15-34-08 +.. gh-issue: 110190 +.. nonce: Z5PQQX +.. section: Library + +Fix ctypes structs with array on Windows ARM64 platform by setting +``MAX_STRUCT_SIZE`` to 32 in stgdict. Patch by Diego Russo + +.. + +.. date: 2024-01-28-19-40-40 +.. gh-issue: 114678 +.. nonce: kYKcJw +.. section: Library + +Ensure that deprecation warning for 'N' specifier in +:class:`~decimal.Decimal` format is not raised for cases where 'N' appears +in other places in the format specifier. Based on patch by Stefan Krah. + +.. + +.. date: 2024-01-28-18-38-18 +.. gh-issue: 70303 +.. nonce: _Lt_pj +.. section: Library + +Return both files and directories from :meth:`pathlib.Path.glob` if a +pattern ends with "``**``". Previously only directories were returned. + +.. + +.. date: 2024-01-28-00-48-12 +.. gh-issue: 109653 +.. nonce: vF4exe +.. section: Library + +Improve import time of :mod:`importlib.metadata` and :mod:`email.utils`. + +.. + +.. date: 2024-01-27-20-11-24 +.. gh-issue: 113280 +.. nonce: CZPQMf +.. section: Library + +Fix a leak of open socket in rare cases when error occurred in +:class:`ssl.SSLSocket` creation. + +.. + +.. date: 2024-01-26-16-46-21 +.. gh-issue: 77749 +.. nonce: NY_7TS +.. section: Library + +:meth:`email.policy.EmailPolicy.fold` now always encodes non-ASCII +characters in headers if :attr:`~email.policy.EmailPolicy.utf8` is false. + +.. + +.. date: 2024-01-25-19-22-17 +.. gh-issue: 83383 +.. nonce: 3GwO9v +.. section: Library + +Synchronization of the :mod:`dbm.dumb` database is now no-op if there was no +modification since opening or last synchronization. The directory file for a +newly created empty :mod:`dbm.dumb` database is now created immediately +after opening instead of deferring this until synchronizing or closing. + +.. + +.. date: 2024-01-24-20-51-49 +.. gh-issue: 91602 +.. nonce: 8fOH8l +.. section: Library + +Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump` +for filtering database objects to dump. Patch by Mariusz Felisiak. + +.. + +.. date: 2024-01-24-20-11-46 +.. gh-issue: 112451 +.. nonce: 7YrG4p +.. section: Library + +Prohibit subclassing pure-Python :class:`datetime.timezone`. This is +consistent with C-extension implementation. Patch by Mariusz Felisiak. + +.. + +.. date: 2024-01-24-17-25-18 +.. gh-issue: 69893 +.. nonce: PQq5fR +.. section: Library + +Add the :meth:`!close` method for the iterator returned by +:func:`xml.etree.ElementTree.iterparse`. + +.. + +.. date: 2024-01-23-23-13-47 +.. gh-issue: 109653 +.. nonce: KLBHmT +.. section: Library + +Reduce the import time of :mod:`threading` module by ~50%. Patch by Daniel +Hollas. + +.. + +.. date: 2024-01-23-21-20-40 +.. gh-issue: 114492 +.. nonce: vKxl5o +.. section: Library + +Make the result of :func:`termios.tcgetattr` reproducible on Alpine Linux. +Previously it could leave a random garbage in some fields. + +.. + +.. date: 2024-01-23-14-11-49 +.. gh-issue: 114315 +.. nonce: KeVdzl +.. section: Library + +Make :class:`threading.Lock` a real class, not a factory function. Add +``__new__`` to ``_thread.lock`` type. + +.. + +.. date: 2024-01-23-13-03-22 +.. gh-issue: 100414 +.. nonce: 5kTdU5 +.. section: Library + +Add :mod:`dbm.sqlite3` as a backend to :mod:`dbm`, and make it the new +default :mod:`!dbm` backend. Patch by Raymond Hettinger and Erlend E. +Aasland. + +.. + +.. date: 2024-01-23-11-04-21 +.. gh-issue: 113267 +.. nonce: xe_Pxe +.. section: Library + +Revert changes in :gh:`106584` which made calls of ``TestResult`` methods +``startTest()`` and ``stopTest()`` unbalanced. + +.. + +.. date: 2024-01-22-12-10-34 +.. gh-issue: 75128 +.. nonce: 4FGlRS +.. section: Library + +Ignore an :exc:`OSError` in :meth:`asyncio.BaseEventLoop.create_server` when +IPv6 is available but the interface cannot actually support it. + +.. + +.. date: 2024-01-22-11-43-38 +.. gh-issue: 114423 +.. nonce: 6mMoPH +.. section: Library + +``_DummyThread`` entries in ``threading._active`` are now automatically +removed when the related thread dies. + +.. + +.. date: 2024-01-21-16-32-55 +.. gh-issue: 114257 +.. nonce: bCFld5 +.. section: Library + +Dismiss the :exc:`FileNotFound` error in :func:`ctypes.util.find_library` +and just return ``None`` on Linux. + +.. + +.. date: 2024-01-19-18-41-02 +.. gh-issue: 114321 +.. nonce: yj_Xw3 +.. section: Library + +Expose more platform specific constants in the :mod:`fcntl` module on Linux, +macOS, FreeBSD and NetBSD. + +.. + +.. date: 2024-01-19-15-48-06 +.. gh-issue: 114328 +.. nonce: hixxW3 +.. section: Library + +The :func:`tty.setcbreak` and new :func:`tty.cfmakecbreak` no longer clears +the terminal input ICRLF flag. This fixes a regression introduced in 3.12 +that no longer matched how OSes define cbreak mode in their ``stty(1)`` +manual pages. + +.. + +.. date: 2024-01-19-12-05-22 +.. gh-issue: 114281 +.. nonce: H5JQe4 +.. section: Library + +Remove type hints from ``Lib/asyncio/staggered.py``. The annotations in the +`typeshed `__ project should be used +instead. + +.. + +.. date: 2024-01-18-22-29-28 +.. gh-issue: 101438 +.. nonce: 1-uUi_ +.. section: Library + +Avoid reference cycle in ElementTree.iterparse. The iterator returned by +``ElementTree.iterparse`` may hold on to a file descriptor. The reference +cycle prevented prompt clean-up of the file descriptor if the returned +iterator was not exhausted. + +.. + +.. date: 2024-01-18-10-07-52 +.. gh-issue: 114198 +.. nonce: lK4Iif +.. section: Library + +The signature for the ``__replace__`` method on :mod:`dataclasses` now has +the first argument named ``self``, rather than ``obj``. + +.. + +.. date: 2024-01-17-18-53-51 +.. gh-issue: 104522 +.. nonce: 3NyDf4 +.. section: Library + +:exc:`OSError` raised when run a subprocess now only has *filename* +attribute set to *cwd* if the error was caused by a failed attempt to change +the current directory. + +.. + +.. date: 2024-01-16-15-59-06 +.. gh-issue: 114149 +.. nonce: LJ8IPm +.. section: Library + +Enum: correctly handle tuple subclasses in custom ``__new__``. + +.. + +.. date: 2024-01-15-20-21-33 +.. gh-issue: 83648 +.. nonce: HzD_fY +.. section: Library + +Support deprecation of options, positional arguments and subcommands in +:mod:`argparse`. + +.. + +.. date: 2024-01-15-19-54-41 +.. gh-issue: 114087 +.. nonce: Xic5vY +.. section: Library + +Speed up ``dataclasses.asdict`` up to 1.35x. + +.. + +.. date: 2024-01-15-18-42-44 +.. gh-issue: 109534 +.. nonce: wYaLMZ +.. section: Library + +Fix a reference leak in +:class:`asyncio.selector_events.BaseSelectorEventLoop` when SSL handshakes +fail. Patch contributed by Jamie Phan. + +.. + +.. date: 2024-01-12-17-32-36 +.. gh-issue: 79634 +.. nonce: uTSTRI +.. section: Library + +Accept :term:`path-like objects ` as patterns in +:meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob`. + +.. + +.. date: 2024-01-12-09-35-07 +.. gh-issue: 112202 +.. nonce: t_0V1m +.. section: Library + +Ensure that a :func:`asyncio.Condition.notify` call does not get lost if the +awakened ``Task`` is simultaneously cancelled or encounters any other error. + +.. + +.. date: 2024-01-11-20-47-49 +.. gh-issue: 113951 +.. nonce: AzlqFK +.. section: Library + +Fix the behavior of ``tag_unbind()`` methods of :class:`tkinter.Text` and +:class:`tkinter.Canvas` classes with three arguments. Previously, +``widget.tag_unbind(tag, sequence, funcid)`` destroyed the current binding +for *sequence*, leaving *sequence* unbound, and deleted the *funcid* +command. Now it removes only *funcid* from the binding for *sequence*, +keeping other commands, and deletes the *funcid* command. It leaves +*sequence* unbound only if *funcid* was the last bound command. + +.. + +.. date: 2024-01-11-15-10-53 +.. gh-issue: 97959 +.. nonce: UOj6d4 +.. section: Library + +Fix rendering class methods, bound methods, method and function aliases in +:mod:`pydoc`. Class methods no longer have "method of builtins.type +instance" note. Corresponding notes are now added for class and unbound +methods. Method and function aliases now have references to the module or +the class where the origin was defined if it differs from the current. Bound +methods are now listed in the static methods section. Methods of builtin +classes are now supported as well as methods of Python classes. + +.. + +.. date: 2024-01-07-21-04-24 +.. gh-issue: 113796 +.. nonce: 6iNsCR +.. section: Library + +Add more validation checks in the :class:`csv.Dialect` constructor. +:exc:`ValueError` is now raised if the same character is used in different +roles. + +.. + +.. date: 2024-01-05-16-27-34 +.. gh-issue: 113732 +.. nonce: fgDRXA +.. section: Library + +Fix support of :data:`~csv.QUOTE_NOTNULL` and :data:`~csv.QUOTE_STRINGS` in +:func:`csv.reader`. + +.. + +.. date: 2024-01-04-20-58-17 +.. gh-issue: 113225 +.. nonce: -nyJM4 +.. section: Library + +Speed up :meth:`pathlib.Path.walk` by using :attr:`os.DirEntry.path` where +possible. + +.. + +.. date: 2023-12-18-20-10-50 +.. gh-issue: 89039 +.. nonce: gqFdtU +.. section: Library + +When replace() method is called on a subclass of datetime, date or time, +properly call derived constructor. Previously, only the base class's +constructor was called. + +Also, make sure to pass non-zero fold values when creating subclasses in +various methods. Previously, fold was silently ignored. + +.. + +.. date: 2023-12-09-23-31-17 +.. gh-issue: 112919 +.. nonce: S5k9QN +.. section: Library + +Speed-up :func:`datetime.datetime.replace`, :func:`datetime.date.replace` +and :func:`datetime.time.replace`. + +.. + +.. date: 2023-11-27-19-54-43 +.. gh-issue: 59013 +.. nonce: chpQ0e +.. section: Library + +Set breakpoint on the first executable line of the function, instead of the +line of function definition when the user do ``break func`` using :mod:`pdb` + +.. + +.. date: 2023-11-24-19-08-50 +.. gh-issue: 112343 +.. nonce: RarGFC +.. section: Library + +Improve handling of pdb convenience variables to avoid replacing string +contents. + +.. + +.. date: 2023-11-18-16-30-21 +.. gh-issue: 112240 +.. nonce: YXS0tj +.. section: Library + +Add option to calendar module CLI to specify the weekday to start each week. +Patch by Steven Ward. + +.. + +.. date: 2023-11-04-22-32-27 +.. gh-issue: 111741 +.. nonce: f1ufr8 +.. section: Library + +Recognise ``image/webp`` as a standard format in the :mod:`mimetypes` +module. + +.. + +.. date: 2023-10-27-19-24-58 +.. gh-issue: 43457 +.. nonce: 84lx9H +.. section: Library + +Fix the :mod:`tkinter` widget method :meth:`!wm_attributes`. It now accepts +the attribute name without the minus prefix to get window attributes and +allows to specify attributes and values to set as keyword arguments. Add new +optional keyword argument *return_python_dict*: calling +``w.wm_attributes(return_python_dict=True)`` returns the attributes as a +dict instead of a tuple. Calling ``w.wm_attributes()`` now returns a tuple +instead of string if *wantobjects* was set to 0. + +.. + +.. date: 2023-10-24-19-19-54 +.. gh-issue: 82626 +.. nonce: _hfLRf +.. section: Library + +Many functions now emit a warning if a boolean value is passed as a file +descriptor argument. + +.. + +.. date: 2023-10-19-02-08-12 +.. gh-issue: 111051 +.. nonce: 8h1Dpk +.. section: Library + +Added check for file modification during debugging with :mod:`pdb` + +.. + +.. date: 2023-10-04-11-09-30 +.. gh-issue: 110345 +.. nonce: fZU1ud +.. section: Library + +Show the Tcl/Tk patchlevel (rather than version) in :meth:`tkinter._test`. + +.. + +.. date: 2023-09-22-22-17-45 +.. gh-issue: 38807 +.. nonce: m9McRN +.. section: Library + +Fix race condition in :mod:`trace`. Instead of checking if a directory +exists and creating it, directly call :func:`os.makedirs` with the kwarg +``exist_ok=True``. + +.. + +.. date: 2023-07-23-12-28-26 +.. gh-issue: 75705 +.. nonce: aB2-Ww +.. section: Library + +Set unixfrom envelope in :class:`mailbox.mbox` and :class:`mailbox.MMDF`. + +.. + +.. date: 2023-06-29-14-26-56 +.. gh-issue: 106233 +.. nonce: Aqw2HI +.. section: Library + +Fix stacklevel in ``InvalidTZPathWarning`` during :mod:`zoneinfo` module +import. + +.. + +.. date: 2023-05-30-18-30-11 +.. gh-issue: 105102 +.. nonce: SnpK04 +.. section: Library + +Allow :class:`ctypes.Union` to be nested in :class:`ctypes.Structure` when +the system endianness is the opposite of the classes. + +.. + +.. date: 2023-05-08-09-30-00 +.. gh-issue: 104282 +.. nonce: h4c6Eb +.. section: Library + +Fix null pointer dereference in :func:`lzma._decode_filter_properties` due +to improper handling of BCJ filters with properties of zero length. Patch by +Radislav Chugunov. + +.. + +.. date: 2023-05-06-04-57-10 +.. gh-issue: 96471 +.. nonce: C9wAU7 +.. section: Library + +Add :py:class:`queue.Queue` termination with +:py:meth:`~queue.Queue.shutdown`. + +.. + +.. date: 2023-04-08-11-41-07 +.. gh-issue: 101599 +.. nonce: PaWNFh +.. section: Library + +Changed argparse flag options formatting to remove redundancy. + +.. + +.. date: 2023-03-15-03-21-18 +.. gh-issue: 85984 +.. nonce: Xaq6ZN +.. section: Library + +Add POSIX pseudo-terminal functions :func:`os.posix_openpt`, +:func:`os.grantpt`, :func:`os.unlockpt`, and :func:`os.ptsname`. + +.. + +.. date: 2023-03-08-00-02-30 +.. gh-issue: 102512 +.. nonce: LiugDr +.. section: Library + +When :func:`os.fork` is called from a foreign thread (aka ``_DummyThread``), +the type of the thread in a child process is changed to ``_MainThread``. +Also changed its name and daemonic status, it can be now joined. + +.. + +.. date: 2022-07-31-01-24-40 +.. gh-issue: 88569 +.. nonce: eU0--b +.. section: Library + +Add :func:`os.path.isreserved`, which identifies reserved pathnames such as +"NUL", "AUX" and "CON". This function is only available on Windows. + +Deprecate :meth:`pathlib.PurePath.is_reserved`. + +.. + +.. bpo: 38364 +.. date: 2019-10-05-22-56-50 +.. nonce: sYTCWF +.. section: Library + +The ``inspect`` functions ``isgeneratorfunction``, ``iscoroutinefunction``, +``isasyncgenfunction`` now support ``functools.partialmethod`` wrapped +functions the same way they support ``functools.partial``. + +.. + +.. date: 2024-02-12-12-26-17 +.. gh-issue: 115233 +.. nonce: aug6r9 +.. section: Documentation + +Fix an example for :class:`~logging.LoggerAdapter` in the Logging Cookbook. + +.. + +.. date: 2024-01-17-11-40-03 +.. gh-issue: 114123 +.. nonce: LuueXf +.. section: Documentation + +Move the :mod:`csv` module docstring to the :mod:`!csv` module instead of +reexporting it from the internal :mod:`!_csv` module, and remove ``__doc__`` +from ``csv.__all__``. + +Move :attr:`!csv.__version__` to the :mod:`!csv` module instead of +reexporting it from the internal :mod:`!_csv` module, and remove +``__version__`` from ``csv.__all__``. + +.. + +.. date: 2024-02-02-13-18-55 +.. gh-issue: 114099 +.. nonce: C_ycWg +.. section: Tests + +Added test exclusions required to run the test suite on iOS. + +.. + +.. date: 2023-06-02-05-04-15 +.. gh-issue: 105089 +.. nonce: KaZFtU +.. section: Tests + +Fix +``test.test_zipfile.test_core.TestWithDirectory.test_create_directory_with_write`` +test in AIX by doing a bitwise AND of 0xFFFF on mode , so that it will be in +sync with ``zinfo.external_attr`` + +.. + +.. date: 2024-02-08-19-36-20 +.. gh-issue: 115167 +.. nonce: LB9nDK +.. section: Build + +Avoid vendoring ``vcruntime140_threads.dll`` when building with Visual +Studio 2022 version 17.8. + +.. + +.. date: 2024-02-08-17-38-56 +.. gh-issue: 113632 +.. nonce: y9KIGb +.. section: Build + +Promote WASI to a tier 2 platform and drop Emscripten from tier 3 in +configure.ac. + +.. + +.. date: 2024-02-07-08-23-48 +.. gh-issue: 114099 +.. nonce: XcEXEZ +.. section: Build + +configure and Makefile were refactored to accommodate framework builds on +Apple platforms other than macOS. + +.. + +.. date: 2024-02-01-20-08-11 +.. gh-issue: 114875 +.. nonce: x_2iZ9 +.. section: Build + +Add :c:func:`!getgrent` as a prerequisite for building the :mod:`grp` +module. + +.. + +.. date: 2024-02-08-21-37-22 +.. gh-issue: 115049 +.. nonce: X1ObpJ +.. section: Windows + +Fixes ``py.exe`` launcher failing when run as users without user profiles. + +.. + +.. date: 2024-02-06-09-05-13 +.. gh-issue: 115009 +.. nonce: ShMjZs +.. section: Windows + +Update Windows installer to use SQLite 3.45.1. + +.. + +.. date: 2024-02-05-16-53-12 +.. gh-issue: 109991 +.. nonce: YqjnDz +.. section: Windows + +Update Windows build to use OpenSSL 3.0.13. + +.. + +.. date: 2024-02-01-14-35-05 +.. gh-issue: 111239 +.. nonce: SO7SUF +.. section: Windows + +Update Windows builds to use zlib v1.3.1. + +.. + +.. date: 2024-01-23-00-05-05 +.. gh-issue: 100107 +.. nonce: lkbP_Q +.. section: Windows + +The ``py.exe`` launcher will no longer attempt to run the Microsoft Store +redirector when launching a script containing a ``/usr/bin/env`` shebang + +.. + +.. date: 2023-12-19-22-32-28 +.. gh-issue: 112984 +.. nonce: F7kFMl +.. section: Windows + +Adds free-threaded binaries to Windows installer as an optional component. + +.. + +.. date: 2023-08-11-18-21-38 +.. gh-issue: 89240 +.. nonce: dtSOLG +.. section: Windows + +Allows :mod:`multiprocessing` to create pools of greater than 62 processes. + +.. + +.. date: 2024-02-06-09-01-10 +.. gh-issue: 115009 +.. nonce: ysau7e +.. section: macOS + +Update macOS installer to use SQLite 3.45.1. + +.. + +.. date: 2024-02-05-18-30-27 +.. gh-issue: 109991 +.. nonce: tun6Yu +.. section: macOS + +Update macOS installer to use OpenSSL 3.0.13. + +.. + +.. date: 2024-01-23-11-35-26 +.. gh-issue: 114490 +.. nonce: FrQOQ0 +.. section: macOS + +Add Mach-O linkage support for :func:`platform.architecture`. + +.. + +.. date: 2022-11-18-10-05-35 +.. gh-issue: 87804 +.. nonce: rhlDmD +.. section: macOS + +On macOS the result of ``os.statvfs`` and ``os.fstatvfs`` now correctly +report the size of very large disks, in previous versions the reported +number of blocks was wrong for disks with at least 2**32 blocks. + +.. + +.. date: 2024-01-17-23-18-15 +.. gh-issue: 96905 +.. nonce: UYaxoU +.. section: IDLE + +In idlelib code, stop redefining built-ins 'dict' and 'object'. + +.. + +.. date: 2023-04-25-03-01-23 +.. gh-issue: 103820 +.. nonce: LCSpza +.. section: IDLE + +Revise IDLE bindings so that events from mouse button 4/5 on non-X11 +windowing systems (i.e. Win32 and Aqua) are not mistaken for scrolling. + +.. + +.. date: 2024-02-14-15-58-13 +.. gh-issue: 113516 +.. nonce: TyIHWx +.. section: Tools/Demos + +Don't set ``LDSHARED`` when building for WASI. + +.. + +.. date: 2024-02-05-19-00-32 +.. gh-issue: 109991 +.. nonce: yJSEkw +.. section: Tools/Demos + +Update GitHub CI workflows to use OpenSSL 3.0.13 and multissltests to use +1.1.1w, 3.0.13, 3.1.5, and 3.2.1. + +.. + +.. date: 2024-02-05-02-45-51 +.. gh-issue: 115015 +.. nonce: rgtiDB +.. section: Tools/Demos + +Fix a bug in Argument Clinic that generated incorrect code for methods with +no parameters that use the :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS +` calling convention. Only the +positional parameter count was checked; any keyword argument passed would be +silently accepted. + +.. + +.. date: 2024-02-05-17-11-15 +.. gh-issue: 111140 +.. nonce: WMEjid +.. section: C API + +Adds :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes` and +:c:func:`PyLong_FromUnsignedNativeBytes` functions. + +.. + +.. date: 2024-01-31-15-43-35 +.. gh-issue: 114685 +.. nonce: n7aRmX +.. section: C API + +:c:func:`PyBuffer_FillInfo` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. + +.. + +.. date: 2024-01-29-12-13-24 +.. gh-issue: 114685 +.. nonce: B07RME +.. section: C API + +:c:func:`PyObject_GetBuffer` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. + +.. + +.. date: 2024-01-26-21-54-42 +.. gh-issue: 114626 +.. nonce: SKhbh_ +.. section: C API + +Add ``PyCFunctionFast`` and ``PyCFunctionFastWithKeywords`` typedefs +(identical to the existing ``_PyCFunctionFast`` and +``_PyCFunctionFastWithKeywords`` typedefs, just without a leading ``_`` +prefix). + +.. + +.. date: 2024-01-23-21-45-02 +.. gh-issue: 114329 +.. nonce: YRaBoe +.. section: C API + +Add :c:func:`PyList_GetItemRef`, which is similar to +:c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of a +:term:`borrowed reference`. + +.. + +.. date: 2023-11-16-02-07-48 +.. gh-issue: 110850 +.. nonce: DQGNfF +.. section: C API + +Add PyTime C API: + +* :c:type:`PyTime_t` type. +* :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX` constants. +* :c:func:`PyTime_AsSecondsDouble`, + :c:func:`PyTime_Monotonic`, :c:func:`PyTime_PerfCounter`, and + :c:func:`PyTime_Time` functions. + +Patch by Victor Stinner. + +.. + +.. date: 2023-11-15-13-47-48 +.. gh-issue: 112066 +.. nonce: 22WsqR +.. section: C API + +Add :c:func:`PyDict_SetDefaultRef`: insert a key and value into a dictionary +if the key is not already present. This is similar to +:meth:`dict.setdefault`, but returns an integer value indicating if the key +was already present. It is also similar to :c:func:`PyDict_SetDefault`, but +returns a strong reference instead of a borrowed reference. diff --git a/Misc/NEWS.d/3.13.0a5.rst b/Misc/NEWS.d/3.13.0a5.rst new file mode 100644 index 00000000000000..d56b1542b01823 --- /dev/null +++ b/Misc/NEWS.d/3.13.0a5.rst @@ -0,0 +1,1163 @@ +.. date: 2024-02-18-03-14-40 +.. gh-issue: 115398 +.. nonce: tzvxH8 +.. release date: 2024-03-12 +.. section: Security + +Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) by adding +five new methods: + +* :meth:`xml.etree.ElementTree.XMLParser.flush` +* :meth:`xml.etree.ElementTree.XMLPullParser.flush` +* :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` +* :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` +* :meth:`xml.sax.expatreader.ExpatParser.flush` + +.. + +.. date: 2024-01-26-22-14-09 +.. gh-issue: 114572 +.. nonce: t1QMQD +.. section: Security + +:meth:`ssl.SSLContext.cert_store_stats` and +:meth:`ssl.SSLContext.get_ca_certs` now correctly lock access to the +certificate store, when the :class:`ssl.SSLContext` is shared across +multiple threads. + +.. + +.. date: 2024-03-11-22-24-59 +.. gh-issue: 116604 +.. nonce: LCEzAT +.. section: Core and Builtins + +Respect the status of the garbage collector when indirect calls are made via +:c:func:`PyErr_CheckSignals` and the evaluation breaker. Patch by Pablo +Galindo + +.. + +.. date: 2024-03-09-11-10-53 +.. gh-issue: 112087 +.. nonce: nbI0Pw +.. section: Core and Builtins + +:class:`list` is now compatible with the implementation of :pep:`703`. + +.. + +.. date: 2024-03-05-22-00-58 +.. gh-issue: 116381 +.. nonce: 0Nq9iO +.. section: Core and Builtins + +Add specialization for ``CONTAINS_OP``. + +.. + +.. date: 2024-03-04-10-19-51 +.. gh-issue: 116296 +.. nonce: gvtxyU +.. section: Core and Builtins + +Fix possible refleak in :meth:`!object.__reduce__` internal error handling. + +.. + +.. date: 2024-02-22-16-17-53 +.. gh-issue: 115823 +.. nonce: c1TreJ +.. section: Core and Builtins + +Properly calculate error ranges in the parser when raising +:exc:`SyntaxError` exceptions caused by invalid byte sequences. Patch by +Pablo Galindo + +.. + +.. date: 2024-02-22-11-33-20 +.. gh-issue: 115778 +.. nonce: jksd1D +.. section: Core and Builtins + +Add ``tierN`` annotation for instruction definition in interpreter DSL. + +.. + +.. date: 2024-02-20-18-49-02 +.. gh-issue: 115733 +.. nonce: 51Zb85 +.. section: Core and Builtins + +Fix crash when calling ``next()`` on exhausted list iterators. + +.. + +.. date: 2024-02-20-12-46-20 +.. gh-issue: 115700 +.. nonce: KLJ5r4 +.. section: Core and Builtins + +The regen-cases build stage now works on Windows. + +.. + +.. date: 2024-02-14-23-50-43 +.. gh-issue: 115347 +.. nonce: VkHvQC +.. section: Core and Builtins + +Fix bug where docstring was replaced by a redundant NOP when Python is run +with ``-OO``. + +.. + +.. date: 2024-02-12-23-29-17 +.. gh-issue: 115323 +.. nonce: 3t6687 +.. section: Core and Builtins + +Make error message more meaningful for when :meth:`bytearray.extend` is +called with a :class:`str` object. + +.. + +.. date: 2024-02-09-18-59-22 +.. gh-issue: 112175 +.. nonce: qglugr +.. section: Core and Builtins + +Every ``PyThreadState`` now has its own ``eval_breaker``, allowing specific +threads to be interrupted. + +.. + +.. date: 2024-02-08-16-01-18 +.. gh-issue: 115154 +.. nonce: ji96FV +.. section: Core and Builtins + +Fix a bug that was causing the :func:`tokenize.untokenize` function to +handle unicode named literals incorrectly. Patch by Pablo Galindo + +.. + +.. date: 2024-01-28-02-46-12 +.. gh-issue: 112433 +.. nonce: FUX-nT +.. section: Core and Builtins + +Add ability to force alignment of :mod:`ctypes.Structure` by way of the new +``_align_`` attribute on the class. + +.. + +.. date: 2023-07-16-15-02-47 +.. gh-issue: 104090 +.. nonce: oMjNa9 +.. section: Core and Builtins + +The multiprocessing resource tracker now exits with non-zero status code if +a resource leak was detected. It still exits with status code 0 otherwise. + +.. + +.. date: 2023-06-16-21-29-06 +.. gh-issue: 105858 +.. nonce: Q7h0EV +.. section: Core and Builtins + +Improve the constructors for :mod:`ast` nodes. Arguments of list types now +default to an empty list if omitted, and optional fields default to +``None``. AST nodes now have an ``__annotations__`` attribute with the +expected types of their attributes. Passing unrecognized extra arguments to +AST nodes is deprecated and will become an error in Python 3.15. Omitting a +required argument to an AST node is deprecated and will become an error in +Python 3.15. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-02-13-11-36-50 +.. gh-issue: 101860 +.. nonce: CKCMbC +.. section: Core and Builtins + +Expose ``__name__`` attribute on property. + +.. + +.. date: 2022-09-04-16-51-56 +.. gh-issue: 96497 +.. nonce: HTBuIL +.. section: Core and Builtins + +Fix incorrect resolution of mangled class variables used in assignment +expressions in comprehensions. + +.. + +.. date: 2024-03-11-12-11-10 +.. gh-issue: 116600 +.. nonce: FcNBy_ +.. section: Library + +Fix :func:`repr` for global :class:`~enum.Flag` members. + +.. + +.. date: 2024-03-07-21-57-50 +.. gh-issue: 116349 +.. nonce: fD2pbP +.. section: Library + +:func:`platform.java_ver` is deprecated and will be removed in 3.15. It was +largely untested, had a confusing API, and was only useful for Jython +support. + +.. + +.. date: 2024-03-05-20-53-34 +.. gh-issue: 116143 +.. nonce: sww6Zl +.. section: Library + +Fix a race in pydoc ``_start_server``, eliminating a window in which +``_start_server`` can return a thread that is "serving" but without a +``docserver`` set. + +.. + +.. date: 2024-03-05-14-34-22 +.. gh-issue: 116127 +.. nonce: 5uktu3 +.. section: Library + +:mod:`typing`: implement :pep:`705` which adds :data:`typing.ReadOnly` +support to :class:`typing.TypedDict`. + +.. + +.. date: 2024-03-05-02-09-18 +.. gh-issue: 116325 +.. nonce: FmlBYv +.. section: Library + +:mod:`typing`: raise :exc:`SyntaxError` instead of :exc:`AttributeError` on +forward references as empty strings. + +.. + +.. date: 2024-03-02-11-31-49 +.. gh-issue: 115957 +.. nonce: C-3Z_U +.. section: Library + +When ``asyncio.TaskGroup.create_task`` is called on an inactive +``asyncio.TaskGroup``, the given coroutine will be closed (which prevents a +``RuntimeWarning``). + +.. + +.. date: 2024-03-01-14-22-08 +.. gh-issue: 115978 +.. nonce: r2ePTo +.. section: Library + +Disable preadv(), readv(), pwritev(), and writev() on WASI. + +Under wasmtime for WASI 0.2, these functions don't pass test_posix +(https://github.com/bytecodealliance/wasmtime/issues/7830). + +.. + +.. date: 2024-03-01-11-57-32 +.. gh-issue: 88352 +.. nonce: bZ68rw +.. section: Library + +Fix the computation of the next rollover time in the +:class:`logging.TimedRotatingFileHandler` handler. :meth:`!computeRollover` +now always returns a timestamp larger than the specified time and works +correctly during the DST change. :meth:`!doRollover` no longer overwrite the +already rolled over file, saving from data loss when run at midnight or +during repeated time at the DST change. + +.. + +.. date: 2024-02-29-20-06-06 +.. gh-issue: 87115 +.. nonce: FVMiOR +.. section: Library + +Set ``__main__.__spec__`` to ``None`` when running a script with :mod:`pdb` + +.. + +.. date: 2024-02-29-17-06-54 +.. gh-issue: 76511 +.. nonce: WqjRLP +.. section: Library + +Fix UnicodeEncodeError in :meth:`email.Message.as_string` that results when +a message that claims to be in the ascii character set actually has +non-ascii characters. Non-ascii characters are now replaced with the U+FFFD +replacement character, like in the ``replace`` error handler. + +.. + +.. date: 2024-02-28-17-50-42 +.. gh-issue: 89547 +.. nonce: GetF38 +.. section: Library + +Add support for nested typing special forms like Final[ClassVar[int]]. + +.. + +.. date: 2024-02-28-17-04-28 +.. gh-issue: 65824 +.. nonce: gG8KR1 +.. section: Library + +Improve the ``less`` prompt in :mod:`pydoc`. + +.. + +.. date: 2024-02-28-13-10-17 +.. gh-issue: 116040 +.. nonce: wDidHd +.. section: Library + +[Enum] fix by-value calls when second value is falsey; e.g. Cardinal(1, 0) + +.. + +.. date: 2024-02-28-12-14-31 +.. gh-issue: 115821 +.. nonce: YO2vKA +.. section: Library + +[Enum] Improve error message when calling super().__new__() in custom +__new__. + +.. + +.. date: 2024-02-27-20-11-29 +.. gh-issue: 85644 +.. nonce: 3rgcBm +.. section: Library + +Use the ``XDG_CURRENT_DESKTOP`` environment variable in :mod:`webbrowser` to +check desktop. Prefer it to the deprecated ``GNOME_DESKTOP_SESSION_ID`` for +GNOME detection. + +.. + +.. date: 2024-02-27-13-05-51 +.. gh-issue: 75988 +.. nonce: In6LlB +.. section: Library + +Fixed :func:`unittest.mock.create_autospec` to pass the call through to the +wrapped object to return the real result. + +.. + +.. date: 2024-02-25-19-20-05 +.. gh-issue: 115881 +.. nonce: ro_Kuw +.. section: Library + +Fix issue where :func:`ast.parse` would incorrectly flag conditional context +managers (such as ``with (x() if y else z()): ...``) as invalid syntax if +``feature_version=(3, 8)`` was passed. This reverts changes to the grammar +made as part of gh-94949. + +.. + +.. date: 2024-02-24-18-48-14 +.. gh-issue: 115886 +.. nonce: rgM6AF +.. section: Library + +Fix silent truncation of the name with an embedded null character in +:class:`multiprocessing.shared_memory.SharedMemory`. + +.. + +.. date: 2024-02-23-11-08-31 +.. gh-issue: 115532 +.. nonce: zVd3gK +.. section: Library + +Add kernel density estimation to the statistics module. + +.. + +.. date: 2024-02-22-12-10-18 +.. gh-issue: 115714 +.. nonce: P2JsU1 +.. section: Library + +On WASI, the :mod:`time` module no longer get process time using ``times()`` +or ``CLOCK_PROCESS_CPUTIME_ID``, system API is that is unreliable and is +likely to be removed from WASI. The affected clock functions fall back to +calling ``clock()``. + +.. + +.. date: 2024-02-22-11-29-27 +.. gh-issue: 115809 +.. nonce: 9H1DhB +.. section: Library + +Improve algorithm for computing which rolled-over log files to delete in +:class:`logging.TimedRotatingFileHandler`. It is now reliable for handlers +without ``namer`` and with arbitrary deterministic ``namer`` that leaves the +datetime part in the file name unmodified. + +.. + +.. date: 2024-02-21-17-54-59 +.. gh-issue: 74668 +.. nonce: JT-Q8W +.. section: Library + +:mod:`urllib.parse` functions :func:`~urllib.parse.parse_qs` and +:func:`~urllib.parse.parse_qsl` now support bytes arguments containing raw +and percent-encoded non-ASCII data. + +.. + +.. date: 2024-02-20-22-02-34 +.. gh-issue: 67044 +.. nonce: QF9_Ru +.. section: Library + +:func:`csv.writer` now always quotes or escapes ``'\r'`` and ``'\n'``, +regardless of *lineterminator* value. + +.. + +.. date: 2024-02-20-16-42-54 +.. gh-issue: 115712 +.. nonce: EXVMXw +.. section: Library + +Restore support of space delimiter with ``skipinitialspace=True`` in +:mod:`csv`. :func:`csv.writer` now quotes empty fields if delimiter is a +space and skipinitialspace is true and raises exception if quoting is not +possible. + +.. + +.. date: 2024-02-20-07-38-15 +.. gh-issue: 112364 +.. nonce: EX7uGI +.. section: Library + +Fixed :func:`ast.unparse` to handle format_spec with ``"``, ``'`` or ``\\``. +Patched by Frank Hoffmann. + +.. + +.. date: 2024-02-19-16-53-48 +.. gh-issue: 112997 +.. nonce: sYBXRZ +.. section: Library + +Stop logging potentially sensitive callback arguments in :mod:`asyncio` +unless debug mode is active. + +.. + +.. date: 2024-02-19-15-52-30 +.. gh-issue: 114914 +.. nonce: M5-1d8 +.. section: Library + +Fix an issue where an abandoned :class:`StreamWriter` would not be garbage +collected. + +.. + +.. date: 2024-02-18-12-18-12 +.. gh-issue: 111358 +.. nonce: 9yJUMD +.. section: Library + +Fix a bug in :meth:`asyncio.BaseEventLoop.shutdown_default_executor` to +ensure the timeout passed to the coroutine behaves as expected. + +.. + +.. date: 2024-02-17-18-47-12 +.. gh-issue: 115618 +.. nonce: napiNp +.. section: Library + +Fix improper decreasing the reference count for ``None`` argument in +:class:`property` methods :meth:`~property.getter`, :meth:`~property.setter` +and :meth:`~property.deleter`. + +.. + +.. date: 2024-02-16-16-40-10 +.. gh-issue: 112720 +.. nonce: io6_Ac +.. section: Library + +Refactor :class:`dis.ArgResolver` to make it possible to subclass and change +the way jump args are interpreted. + +.. + +.. date: 2024-02-15-23-42-54 +.. gh-issue: 112006 +.. nonce: 4wxcK- +.. section: Library + +Fix :func:`inspect.unwrap` for types with the ``__wrapper__`` data +descriptor. Fix :meth:`inspect.Signature.from_callable` for builtins +:func:`classmethod` and :func:`staticmethod`. + +.. + +.. date: 2024-02-15-19-11-49 +.. gh-issue: 101293 +.. nonce: 898b8l +.. section: Library + +Support callables with the ``__call__()`` method and types with +``__new__()`` and ``__init__()`` methods set to class methods, static +methods, bound methods, partial functions, and other types of methods and +descriptors in :meth:`inspect.Signature.from_callable`. + +.. + +.. date: 2024-02-12-11-42-48 +.. gh-issue: 103092 +.. nonce: sGMKr0 +.. section: Library + +Isolate :mod:`!_lsprof` (apply :pep:`687`). + +.. + +.. date: 2024-02-11-20-12-39 +.. gh-issue: 113942 +.. nonce: i72sMJ +.. section: Library + +:mod:`pydoc` no longer skips global functions implemented as builtin +methods, such as :class:`~type.MethodDescriptorType` and +:class:`~type.WrapperDescriptorType`. + +.. + +.. date: 2024-02-10-17-18-49 +.. gh-issue: 115256 +.. nonce: 41Fy9P +.. section: Library + +Added DeprecationWarning when accessing the tarfile attribute of TarInfo +objects. The attribute is never used internally and is only attached to +TarInfos when the tarfile is opened in write-mode, not read-mode. The +attribute creates an unnecessary reference cycle which may cause corruption +when not closing the handle after writing a tarfile. + +.. + +.. date: 2024-02-09-19-41-48 +.. gh-issue: 115197 +.. nonce: 20wkWH +.. section: Library + +``urllib.request`` no longer resolves the hostname before checking it +against the system's proxy bypass list on macOS and Windows. + +.. + +.. date: 2024-02-09-12-22-47 +.. gh-issue: 113812 +.. nonce: wOraaG +.. section: Library + +:meth:`DatagramTransport.sendto` will now send zero-length datagrams if +called with an empty bytes object. The transport flow control also now +accounts for the datagram header when calculating the buffer size. + +.. + +.. date: 2024-01-30-23-28-29 +.. gh-issue: 114763 +.. nonce: BRjKkg +.. section: Library + +Protect modules loaded with :class:`importlib.util.LazyLoader` from race +conditions when multiple threads try to access attributes before the loading +is complete. + +.. + +.. date: 2024-01-29-13-46-41 +.. gh-issue: 114709 +.. nonce: SQ998l +.. section: Library + +:func:`posixpath.commonpath` now raises a :exc:`ValueError` exception when +passed an empty iterable. Previously, :exc:`IndexError` was raised. + +:func:`posixpath.commonpath` now raises a :exc:`TypeError` exception when +passed ``None``. Previously, :exc:`ValueError` was raised. + +.. + +.. date: 2024-01-26-16-42-31 +.. gh-issue: 114610 +.. nonce: S18Vuz +.. section: Library + +Fix bug where :meth:`pathlib.PurePath.with_stem` converted a non-empty path +suffix to a stem when given an empty *stem* argument. It now raises +:exc:`ValueError`, just like :meth:`pathlib.PurePath.with_suffix` does when +called on a path with an empty stem, given a non-empty *suffix* argument. + +.. + +.. date: 2023-11-24-23-40-00 +.. gh-issue: 107361 +.. nonce: v54gh46 +.. section: Library + +Add :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT` to +the default SSL context created with :func:`ssl.create_default_context`. + +.. + +.. date: 2023-11-20-16-15-44 +.. gh-issue: 112281 +.. nonce: gH4EVk +.. section: Library + +Allow creating :ref:`union of types` for +:class:`typing.Annotated` with unhashable metadata. + +.. + +.. date: 2023-11-07-10-22-06 +.. gh-issue: 111775 +.. nonce: IoVxfX +.. section: Library + +Fix :meth:`importlib.resources.simple.ResourceHandle.open` for text mode, +added missed ``stream`` argument. + +.. + +.. date: 2023-10-07-06-15-13 +.. gh-issue: 90095 +.. nonce: gWn1ka +.. section: Library + +Make .pdbrc and -c work with any valid pdb commands. + +.. + +.. date: 2023-08-05-08-41-58 +.. gh-issue: 107625 +.. nonce: cVSHCT +.. section: Library + +Raise :exc:`configparser.ParsingError` from +:meth:`~configparser.ConfigParser.read` and +:meth:`~configparser.ConfigParser.read_file` methods of +:class:`configparser.ConfigParser` if a key without a corresponding value is +continued (that is, followed by an indented line). + +.. + +.. date: 2023-08-02-01-17-32 +.. gh-issue: 107155 +.. nonce: Mj1K9L +.. section: Library + +Fix incorrect output of ``help(x)`` where ``x`` is a :keyword:`lambda` +function, which has an ``__annotations__`` dictionary attribute with a +``"return"`` key. + +.. + +.. date: 2023-07-12-14-52-04 +.. gh-issue: 57141 +.. nonce: L2k8Xb +.. section: Library + +Add option for *non-shallow* comparisons to :class:`filecmp.dircmp` like +:func:`filecmp.cmp`. Original patch by Steven Ward. Enhanced by Tobias +Rautenkranz + +.. + +.. date: 2023-05-17-21-33-21 +.. gh-issue: 69990 +.. nonce: Blvz9G +.. section: Library + +:meth:`Profile.print_stats` has been improved to accept multiple sort +arguments. Patched by Chiu-Hsiang Hsu and Furkan Onder. + +.. + +.. date: 2023-05-01-22-28-57 +.. gh-issue: 104061 +.. nonce: vxfBXf +.. section: Library + +Add :data:`socket.SO_BINDTOIFINDEX` constant. + +.. + +.. date: 2023-04-02-21-20-35 +.. gh-issue: 60346 +.. nonce: 7mjgua +.. section: Library + +Fix ArgumentParser inconsistent with parse_known_args. + +.. + +.. date: 2023-03-03-09-05-42 +.. gh-issue: 102389 +.. nonce: ucmo0_ +.. section: Library + +Add ``windows_31j`` to aliases for ``cp932`` codec + +.. + +.. date: 2023-02-14-17-19-59 +.. gh-issue: 72249 +.. nonce: fv35wU +.. section: Library + +:func:`functools.partial`s of :func:`repr` has been improved to include the +:term:`module` name. Patched by Furkan Onder and Anilyka Barry. + +.. + +.. date: 2023-01-12-14-16-01 +.. gh-issue: 100985 +.. nonce: GT5Fvd +.. section: Library + +Update HTTPSConnection to consistently wrap IPv6 Addresses when using a +proxy. + +.. + +.. date: 2023-01-09-14-08-02 +.. gh-issue: 100884 +.. nonce: DcmdLl +.. section: Library + +email: fix misfolding of comma in address-lists over multiple lines in +combination with unicode encoding. + +.. + +.. date: 2022-11-22-23-17-43 +.. gh-issue: 95782 +.. nonce: an_and +.. section: Library + +Fix :func:`io.BufferedReader.tell`, :func:`io.BufferedReader.seek`, +:func:`!_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, +:func:`io.BufferedRandom.seek` and :func:`!_pyio.BufferedRandom.tell` being +able to return negative offsets. + +.. + +.. date: 2022-08-26-15-50-53 +.. gh-issue: 96310 +.. nonce: 0NssDh +.. section: Library + +Fix a traceback in :mod:`argparse` when all options in a mutually exclusive +group are suppressed. + +.. + +.. date: 2022-05-25-17-49-04 +.. gh-issue: 93205 +.. nonce: DjhFVR +.. section: Library + +Fixed a bug in :class:`logging.handlers.TimedRotatingFileHandler` where +multiple rotating handler instances pointing to files with the same name but +different extensions would conflict and not delete the correct files. + +.. + +.. bpo: 31116 +.. date: 2022-01-14-10-50-17 +.. nonce: 0bduV9 +.. section: Library + +Add Z85 encoding to ``base64``. + +.. + +.. bpo: 44865 +.. date: 2021-08-24-20-47-37 +.. nonce: c3BhZS +.. section: Library + +Add missing call to localization function in :mod:`argparse`. + +.. + +.. bpo: 43952 +.. date: 2021-05-03-11-04-12 +.. nonce: Me7fJe +.. section: Library + +Fix :meth:`multiprocessing.connection.Listener.accept` to accept empty +bytes as authkey. Not accepting empty bytes as key causes it to hang +indefinitely. + +.. + +.. bpo: 42125 +.. date: 2020-12-15-22-30-49 +.. nonce: UGyseY +.. section: Library + +linecache: get module name from ``__spec__`` if available. This allows +getting source code for the ``__main__`` module when a custom loader is +used. + +.. + +.. bpo: 41122 +.. date: 2020-07-13-23-59-42 +.. nonce: 8P_Brh +.. section: Library + +Failing to pass arguments properly to :func:`functools.singledispatchmethod` +now throws a TypeError instead of hitting an index out of bounds internally. + +.. + +.. bpo: 40818 +.. date: 2020-05-29-18-08-54 +.. nonce: Ij8ffq +.. section: Library + +The asyncio REPL now runs :data:`sys.__interactivehook__` on startup. The +default implementation of :data:`sys.__interactivehook__` provides +auto-completion to the asyncio REPL. Patch contributed by Rémi Lapeyre. + +.. + +.. bpo: 33775 +.. date: 2019-04-06-23-50-59 +.. nonce: 0yhMDc +.. section: Library + +Add 'default' and 'version' help text for localization in argparse. + +.. + +.. date: 2024-02-14-20-17-04 +.. gh-issue: 115399 +.. nonce: fb9a0R +.. section: Documentation + +Document :cve:`2023-52425` of Expat <2.6.0 under "XML vulnerabilities". + +.. + +.. date: 2024-02-08-08-51-37 +.. gh-issue: 109653 +.. nonce: QHLW4w +.. section: Documentation + +Improve import time of :mod:`uuid` on Linux. + +.. + +.. date: 2024-02-25-16-28-26 +.. gh-issue: 71052 +.. nonce: lSb9EC +.. section: Tests + +Add test exclusions to support running the test suite on Android. + +.. + +.. date: 2024-02-25-15-58-28 +.. gh-issue: 71052 +.. nonce: lxBjqY +.. section: Tests + +Enable ``test_concurrent_futures`` on platforms that support threading but +not multiprocessing. + +.. + +.. date: 2024-02-22-00-17-06 +.. gh-issue: 115796 +.. nonce: d4hpKy +.. section: Tests + +Make '_testinternalcapi.assemble_code_object' construct the exception table +for the code object. + +.. + +.. date: 2024-02-20-15-47-41 +.. gh-issue: 115720 +.. nonce: w8i8UG +.. section: Tests + +Leak tests (``-R``, ``--huntrleaks``) now show a summary of the number of +leaks found in each iteration. + +.. + +.. date: 2024-02-18-14-20-52 +.. gh-issue: 115122 +.. nonce: 3rGNo9 +.. section: Tests + +Add ``--bisect`` option to regrtest test runner: run failed tests with +``test.bisect_cmd`` to identify failing tests. Patch by Victor Stinner. + +.. + +.. date: 2024-02-17-08-25-01 +.. gh-issue: 115596 +.. nonce: RGPCrR +.. section: Tests + +Fix ``ProgramPriorityTests`` in ``test_os`` permanently changing the process +priority. + +.. + +.. date: 2024-02-16-13-04-28 +.. gh-issue: 115556 +.. nonce: rjaQ9w +.. section: Tests + +On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and +``PCbuild\\rt.bat`` are now properly handled. + +.. + +.. date: 2024-02-13-18-24-04 +.. gh-issue: 115420 +.. nonce: -dlzfI +.. section: Tests + +Fix translation of exception handler targets by +``_testinternalcapi.optimize_cfg``. + +.. + +.. date: 2024-02-12-22-35-01 +.. gh-issue: 115376 +.. nonce: n9vubZ +.. section: Tests + +Fix segfault in ``_testinternalcapi.compiler_codegen`` on bad input. + +.. + +.. date: 2024-03-04-12-43-42 +.. gh-issue: 116313 +.. nonce: cLLb8S +.. section: Build + +Get WASI builds to work under wasmtime 18 w/ WASI 0.2/preview2 primitives. + +.. + +.. date: 2024-03-01-16-44-19 +.. gh-issue: 71052 +.. nonce: Hs-9EP +.. section: Build + +Change Android's :data:`sys.platform` from ``"linux"`` to ``"android"``. + +.. + +.. date: 2024-02-29-15-12-31 +.. gh-issue: 116117 +.. nonce: eENkQK +.. section: Build + +Backport ``libb2``'s PR #42 to fix compiling CPython on 32-bit Windows with +``clang-cl``. + +.. + +.. date: 2024-02-26-14-54-58 +.. gh-issue: 71052 +.. nonce: XvFay1 +.. section: Build + +Fix several Android build issues + +.. + +.. date: 2024-02-26-13-13-53 +.. gh-issue: 114099 +.. nonce: 8lpX-7 +.. section: Build + +A testbed project was added to run the test suite on iOS. + +.. + +.. date: 2024-02-24-12-50-43 +.. gh-issue: 115350 +.. nonce: naQA6y +.. section: Build + +Fix building ctypes module with -DWIN32_LEAN_AND_MEAN defined + +.. + +.. date: 2024-02-21-18-22-49 +.. gh-issue: 111225 +.. nonce: Z8C3av +.. section: Build + +Link extension modules against libpython on Android. + +.. + +.. date: 2024-02-21-11-58-30 +.. gh-issue: 115737 +.. nonce: dpNl2T +.. section: Build + +The install name for libPython is now correctly set for non-framework macOS +builds. + +.. + +.. date: 2024-02-13-14-52-59 +.. gh-issue: 114099 +.. nonce: zjXsQr +.. section: Build + +Makefile targets were added to support compiling an iOS-compatible framework +build. + +.. + +.. date: 2024-02-27-23-21-55 +.. gh-issue: 116012 +.. nonce: B9_IwM +.. section: Windows + +Ensure the value of ``GetLastError()`` is preserved across GIL operations. + +.. + +.. date: 2024-02-23-11-43-43 +.. gh-issue: 115582 +.. nonce: sk1XPi +.. section: Windows + +Building extensions intended for free-threaded builds of CPython now require +compiling with ``/DPy_GIL_DISABLED`` manually when using a regular install. +This is expected to change in future releases. + +.. + +.. date: 2024-02-21-23-48-59 +.. gh-issue: 115554 +.. nonce: 02mpQC +.. section: Windows + +The installer now has more strict rules about updating the :ref:`launcher`. +In general, most users only have a single launcher installed and will see no +difference. When multiple launchers have been installed, the option to +install the launcher is disabled until all but one have been removed. +Downgrading the launcher (which was never allowed) is now more obviously +blocked. + +.. + +.. date: 2024-02-15-23-16-31 +.. gh-issue: 115543 +.. nonce: otrWnw +.. section: Windows + +:ref:`launcher` can now detect Python 3.13 when installed from the Microsoft +Store, and will install Python 3.12 by default when +:envvar:`PYLAUNCHER_ALLOW_INSTALL` is set. + +.. + +.. date: 2024-02-29-20-52-23 +.. gh-issue: 116145 +.. nonce: ygafim +.. section: macOS + +Update macOS installer to Tcl/Tk 8.6.14. + +.. + +.. date: 2023-12-09-11-04-26 +.. gh-issue: 88516 +.. nonce: SIIvfs +.. section: IDLE + +On macOS show a proxy icon in the title bar of editor windows to match +platform behaviour. + +.. + +.. date: 2023-02-12-19-28-08 +.. gh-issue: 100176 +.. nonce: Kzs4Zw +.. section: Tools/Demos + +Remove outdated Tools/{io,cc,string}bench + +.. + +.. bpo: 45101 +.. date: 2021-09-05-02-47-48 +.. nonce: 60Zqmt +.. section: Tools/Demos + +Add consistency in usage message IO between 2 versions of python-config. + +.. + +.. date: 2024-02-16-15-56-53 +.. gh-issue: 114626 +.. nonce: ie2esA +.. section: C API + +Add again ``_PyCFunctionFastWithKeywords`` name, removed in Python 3.13 +alpha 4 by mistake. Keep the old private ``_PyCFunctionFastWithKeywords`` +name (Python 3.7) as an alias to the new public name +``PyCFunctionFastWithKeywords`` (Python 3.13a4). Patch by Victor Stinner. + +.. + +.. date: 2023-11-15-09-24-51 +.. gh-issue: 111418 +.. nonce: FYYetY +.. section: C API + +Add :c:macro:`PyHASH_MODULUS`, :c:macro:`PyHASH_BITS`, :c:macro:`PyHASH_INF` +and :c:macro:`PyHASH_IMAG` C macros. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst new file mode 100644 index 00000000000000..b9cdbc4e146d5a --- /dev/null +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -0,0 +1,1270 @@ +.. date: 2024-04-04-13-42-59 +.. gh-issue: 117494 +.. nonce: GPQH64 +.. release date: 2024-04-09 +.. section: Core and Builtins + +Refactored the instruction sequence data structure out of compile.c into +instruction_sequence.c. + +.. + +.. date: 2024-04-03-13-44-04 +.. gh-issue: 116968 +.. nonce: zgcdG2 +.. section: Core and Builtins + +Introduce a unified 16-bit backoff counter type (``_Py_BackoffCounter``), +shared between the Tier 1 adaptive specializer and the Tier 2 optimizer. The +API used for adaptive specialization counters is changed but the behavior is +(supposed to be) identical. + +The behavior of the Tier 2 counters is changed: + +* There are no longer dynamic thresholds (we never varied these). +* All counters now use the same exponential backoff. +* The counter for ``JUMP_BACKWARD`` starts counting down from 16. +* The ``temperature`` in side exits starts counting down from 64. + +.. + +.. date: 2024-04-03-09-49-15 +.. gh-issue: 117431 +.. nonce: WAqRgc +.. section: Core and Builtins + +Improve the performance of the following :class:`bytes` and +:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` +calling convention: + +* :meth:`!endswith` +* :meth:`!startswith` + +.. + +.. date: 2024-04-02-17-37-35 +.. gh-issue: 117431 +.. nonce: vDKAOn +.. section: Core and Builtins + +Improve the performance of the following :class:`str` methods by adapting +them to the :c:macro:`METH_FASTCALL` calling convention: + +* :meth:`~str.count` +* :meth:`~str.endswith` +* :meth:`~str.find` +* :meth:`~str.index` +* :meth:`~str.rfind` +* :meth:`~str.rindex` +* :meth:`~str.startswith` + +.. + +.. date: 2024-04-02-10-04-57 +.. gh-issue: 117411 +.. nonce: YdyVmG +.. section: Core and Builtins + +Move ``PyFutureFeatures`` to an internal header and make it private. + +.. + +.. date: 2024-04-02-06-16-49 +.. gh-issue: 109120 +.. nonce: X485oN +.. section: Core and Builtins + +Added handle of incorrect star expressions, e.g ``f(3, *)``. Patch by +Grigoryev Semyon + +.. + +.. date: 2024-03-26-17-22-38 +.. gh-issue: 117266 +.. nonce: Kwh79O +.. section: Core and Builtins + +Fix crashes for certain user-created subclasses of :class:`ast.AST`. Such +classes are now expected to set the ``_field_types`` attribute. + +.. + +.. date: 2024-03-25-17-04-54 +.. gh-issue: 99108 +.. nonce: 8bjdO6 +.. section: Core and Builtins + +Updated the :mod:`hashlib` built-in `HACL\* project`_ C code from upstream +that we use for many implementations when they are not present via OpenSSL +in a given build. This also avoids the rare potential for a C symbol name +one definition rule linking issue. + +.. _HACL\* project: https://github.com/hacl-star/hacl-star + +.. + +.. date: 2024-03-25-12-51-12 +.. gh-issue: 117108 +.. nonce: tNqDEo +.. section: Core and Builtins + +Change the old space bit of objects in the young generation from 0 to +gcstate->visited, so that any objects created during GC will have the old +bit set correctly if they get moved into the old generation. + +.. + +.. date: 2024-03-21-12-10-11 +.. gh-issue: 117108 +.. nonce: _6jIrB +.. section: Core and Builtins + +The cycle GC now chooses the size of increments based on the total heap +size, instead of the rate of object creation. This ensures that it can keep +up with growing heaps. + +.. + +.. date: 2024-03-13-16-55-25 +.. gh-issue: 116735 +.. nonce: o3w6y8 +.. section: Core and Builtins + +For ``INSTRUMENTED_CALL_FUNCTION_EX``, set ``arg0`` to +``sys.monitoring.MISSING`` instead of ``None`` for :monitoring-event:`CALL` +event. + +.. + +.. date: 2024-03-12-20-31-57 +.. gh-issue: 113964 +.. nonce: bJppzg +.. section: Core and Builtins + +Starting new threads and process creation through :func:`os.fork` are now +only prevented once all non-daemon threads exit. + +.. + +.. date: 2024-03-11-22-05-56 +.. gh-issue: 116626 +.. nonce: GsyczB +.. section: Core and Builtins + +Ensure ``INSTRUMENTED_CALL_FUNCTION_EX`` always emits +:monitoring-event:`CALL` + +.. + +.. date: 2024-03-11-00-45-39 +.. gh-issue: 116554 +.. nonce: gYumG5 +.. section: Core and Builtins + +``list.sort()`` now exploits more cases of partial ordering, particularly +those with long descending runs with sub-runs of equal values. Those are +recognized as single runs now (previously, each block of repeated values +caused a new run to be created). + +.. + +.. date: 2024-03-07-16-12-39 +.. gh-issue: 114099 +.. nonce: ujdjn2 +.. section: Core and Builtins + +Added a Loader that can discover extension modules in an iOS-style +Frameworks folder. + +.. + +.. date: 2024-02-25-14-17-25 +.. gh-issue: 115775 +.. nonce: CNbGbJ +.. section: Core and Builtins + +Compiler populates the new ``__static_attributes__`` field on a class with +the names of attributes of this class which are accessed through self.X from +any function in its body. + +.. + +.. date: 2024-02-24-03-39-09 +.. gh-issue: 115776 +.. nonce: THJXqg +.. section: Core and Builtins + +The array of values, the ``PyDictValues`` struct is now embedded in the +object during allocation. This provides better performance in the common +case, and does not degrade as much when the object's ``__dict__`` is +materialized. + +.. + +.. date: 2024-01-07-04-22-51 +.. gh-issue: 108362 +.. nonce: oB9Gcf +.. section: Core and Builtins + +Implement an incremental cyclic garbage collector. By collecting the old +generation in increments, there is no need for a full heap scan. This can +hugely reduce maximum pause time for programs with large heaps. + +Reduce the number of generations from three to two. The old generation is +split into two spaces, "visited" and "pending". + +Collection happens in two steps:: * An increment is formed from the young +generation and a small part of the pending space. * This increment is +scanned and the survivors moved to the end of the visited space. + +When the collecting space becomes empty, the two spaces are swapped. + +.. + +.. date: 2023-10-14-00-05-17 +.. gh-issue: 109870 +.. nonce: oKpJ3P +.. section: Core and Builtins + +Dataclasses now calls :func:`exec` once per dataclass, instead of once per +method being added. This can speed up dataclass creation by up to 20%. + +.. + +.. date: 2022-10-05-09-33-48 +.. gh-issue: 97901 +.. nonce: BOLluU +.. section: Core and Builtins + +Mime type ``text/rtf`` is now supported by :mod:`mimetypes`. + +.. + +.. bpo: 24612 +.. date: 2021-09-04-22-33-01 +.. nonce: SsTuUX +.. section: Core and Builtins + +Improve the :exc:`SyntaxError` that happens when 'not' appears after an +operator. Patch by Pablo Galindo + +.. + +.. date: 2024-04-08-20-26-15 +.. gh-issue: 117648 +.. nonce: NzVEa7 +.. section: Library + +Improve performance of :func:`os.path.join` and :func:`os.path.expanduser`. + +.. + +.. date: 2024-04-06-16-42-34 +.. gh-issue: 117584 +.. nonce: hqk9Hn +.. section: Library + +Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath`. + +.. + +.. date: 2024-04-03-18-36-53 +.. gh-issue: 117467 +.. nonce: l6rWlj +.. section: Library + +Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush`. +Patch by Tony Mountifield. + +.. + +.. date: 2024-04-02-20-30-12 +.. gh-issue: 114848 +.. nonce: YX4pEc +.. section: Library + +Raise :exc:`FileNotFoundError` when ``getcwd()`` returns '(unreachable)', +which can happen on Linux >= 2.6.36 with glibc < 2.27. + +.. + +.. date: 2024-04-02-13-13-46 +.. gh-issue: 117459 +.. nonce: jiIZmH +.. section: Library + +:meth:`asyncio.asyncio.run_coroutine_threadsafe` now keeps the traceback of +:class:`CancelledError`, :class:`TimeoutError` and +:class:`InvalidStateError` which are raised in the coroutine. + +.. + +.. date: 2024-03-29-21-43-19 +.. gh-issue: 117381 +.. nonce: fT0JFM +.. section: Library + +Fix error message for :func:`ntpath.commonpath`. + +.. + +.. date: 2024-03-29-15-58-01 +.. gh-issue: 117337 +.. nonce: 7w3Qwp +.. section: Library + +Deprecate undocumented :func:`!glob.glob0` and :func:`!glob.glob1` +functions. Use :func:`glob.glob` and pass a directory to its *root_dir* +argument instead. + +.. + +.. date: 2024-03-29-15-04-13 +.. gh-issue: 117349 +.. nonce: OB9kQQ +.. section: Library + +Optimise several functions in :mod:`os.path`. + +.. + +.. date: 2024-03-29-12-07-26 +.. gh-issue: 117348 +.. nonce: WjCYvK +.. section: Library + +Refactored :meth:`configparser.RawConfigParser._read` to reduce cyclometric +complexity and improve comprehensibility. + +.. + +.. date: 2024-03-28-19-13-20 +.. gh-issue: 117335 +.. nonce: d6uKJu +.. section: Library + +Raise TypeError for non-sequences for :func:`ntpath.commonpath`. + +.. + +.. date: 2024-03-28-17-55-22 +.. gh-issue: 66449 +.. nonce: 4jhuEV +.. section: Library + +:class:`configparser.ConfigParser` now accepts unnamed sections before named +ones, if configured to do so. + +.. + +.. date: 2024-03-28-13-54-20 +.. gh-issue: 88014 +.. nonce: zJz31I +.. section: Library + +In documentation of :class:`gzip.GzipFile` in module gzip, explain data type +of optional constructor argument *mtime*, and recommend ``mtime = 0`` for +generating deterministic streams. + +.. + +.. date: 2024-03-27-21-05-52 +.. gh-issue: 117310 +.. nonce: Bt2wox +.. section: Library + +Fixed an unlikely early & extra ``Py_DECREF`` triggered crash in :mod:`ssl` +when creating a new ``_ssl._SSLContext`` if CPython was built implausibly +such that the default cipher list is empty **or** the SSL library it was +linked against reports a failure from its C ``SSL_CTX_set_cipher_list()`` +API. + +.. + +.. date: 2024-03-27-16-43-42 +.. gh-issue: 117294 +.. nonce: wbXNFv +.. section: Library + +A ``DocTestCase`` now reports as skipped if all examples in the doctest are +skipped. + +.. + +.. date: 2024-03-26-11-48-39 +.. gh-issue: 98966 +.. nonce: SayV9y +.. section: Library + +In :mod:`subprocess`, raise a more informative message when +``stdout=STDOUT``. + +.. + +.. date: 2024-03-25-21-15-56 +.. gh-issue: 117225 +.. nonce: oOaZXb +.. section: Library + +doctest: only print "and X failed" when non-zero, don't pluralise "1 items". +Patch by Hugo van Kemenade. + +.. + +.. date: 2024-03-25-00-20-16 +.. gh-issue: 117205 +.. nonce: yV7xGb +.. section: Library + +Speed up :func:`compileall.compile_dir` by 20% when using multiprocessing by +increasing ``chunksize``. + +.. + +.. date: 2024-03-23-14-26-18 +.. gh-issue: 117178 +.. nonce: vTisTG +.. section: Library + +Fix regression in lazy loading of self-referential modules, introduced in +gh-114781. + +.. + +.. date: 2024-03-23-13-40-13 +.. gh-issue: 112383 +.. nonce: XuHf3G +.. section: Library + +Fix :mod:`dis` module's handling of ``ENTER_EXECUTOR`` instructions. + +.. + +.. date: 2024-03-23-12-28-05 +.. gh-issue: 117182 +.. nonce: a0KANW +.. section: Library + +Lazy-loading of modules that modify their own ``__class__`` no longer +reverts the ``__class__`` to :class:`types.ModuleType`. + +.. + +.. date: 2024-03-21-17-07-38 +.. gh-issue: 117084 +.. nonce: w1mTpT +.. section: Library + +Fix :mod:`zipfile` extraction for directory entries with the name containing +backslashes on Windows. + +.. + +.. date: 2024-03-21-09-57-57 +.. gh-issue: 117114 +.. nonce: Qu-p55 +.. section: Library + +Make :func:`os.path.isdevdrive` available on all platforms. For those that +do not offer Dev Drives, it will always return ``False``. + +.. + +.. date: 2024-03-21-07-27-36 +.. gh-issue: 117110 +.. nonce: 9K1InX +.. section: Library + +Fix a bug that prevents subclasses of :class:`typing.Any` to be instantiated +with arguments. Patch by Chris Fu. + +.. + +.. date: 2024-03-20-23-07-58 +.. gh-issue: 109653 +.. nonce: uu3lrX +.. section: Library + +Deferred select imports in importlib.metadata and importlib.resources for a +14% speedup. + +.. + +.. date: 2024-03-20-16-10-29 +.. gh-issue: 70647 +.. nonce: FpD6Ar +.. section: Library + +Start the deprecation period for the current behavior of +:func:`datetime.datetime.strptime` and :func:`time.strptime` which always +fails to parse a date string with a :exc:`ValueError` involving a day of +month such as ``strptime("02-29", "%m-%d")`` when a year is **not** +specified and the date happen to be February 29th. This should help avoid +users finding new bugs every four years due to a natural mistaken assumption +about the API when parsing partial date values. + +.. + +.. date: 2024-03-19-19-42-25 +.. gh-issue: 116987 +.. nonce: ZVKUH1 +.. section: Library + +Fixed :func:`inspect.findsource` for class code objects. + +.. + +.. date: 2024-03-19-14-35-57 +.. gh-issue: 114099 +.. nonce: siNSpK +.. section: Library + +Modify standard library to allow for iOS platform differences. + +.. + +.. date: 2024-03-19-11-08-26 +.. gh-issue: 90872 +.. nonce: ghys95 +.. section: Library + +On Windows, :meth:`subprocess.Popen.wait` no longer calls +``WaitForSingleObject()`` with a negative timeout: pass ``0`` ms if the +timeout is negative. Patch by Victor Stinner. + +.. + +.. date: 2024-03-18-14-36-50 +.. gh-issue: 116957 +.. nonce: dTCs4f +.. section: Library + +configparser: Don't leave ConfigParser values in an invalid state (stored as +a list instead of a str) after an earlier read raised DuplicateSectionError +or DuplicateOptionError. + +.. + +.. date: 2024-03-17-18-12-39 +.. gh-issue: 115538 +.. nonce: PBiRQB +.. section: Library + +:class:`!_io.WindowsConsoleIO` now emit a warning if a boolean value is +passed as a filedescriptor argument. + +.. + +.. date: 2024-03-14-20-59-28 +.. gh-issue: 90095 +.. nonce: 7UaJ1U +.. section: Library + +Ignore empty lines and comments in ``.pdbrc`` + +.. + +.. date: 2024-03-14-17-24-59 +.. gh-issue: 106531 +.. nonce: 9ehywi +.. section: Library + +Refreshed zipfile._path from `zipp 3.18 +`_, providing +better compatibility for PyPy, better glob performance for deeply nested +zipfiles, and providing internal access to ``CompleteDirs.inject`` for use +in other tests (like importlib.resources). + +.. + +.. date: 2024-03-14-17-21-25 +.. gh-issue: 63207 +.. nonce: LV16SL +.. section: Library + +On Windows, :func:`time.time` now uses the +``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better than +1 us, instead of the ``GetSystemTimeAsFileTime()`` clock which has a +resolution of 15.6 ms. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-14-01-46 +.. gh-issue: 116764 +.. nonce: moB3Lc +.. section: Library + +Restore support of ``None`` and other false values in :mod:`urllib.parse` +functions :func:`~urllib.parse.parse_qs` and +:func:`~urllib.parse.parse_qsl`. Also, they now raise a TypeError for +non-zero integers and non-empty sequences. + +.. + +.. date: 2024-03-14-10-01-23 +.. gh-issue: 116811 +.. nonce: _h5iKP +.. section: Library + +In ``PathFinder.invalidate_caches``, delegate to +``MetadataPathFinder.invalidate_caches``. + +.. + +.. date: 2024-03-14-09-38-51 +.. gh-issue: 116647 +.. nonce: h0d_zj +.. section: Library + +Fix recursive child in dataclasses + +.. + +.. date: 2024-03-14-01-38-44 +.. gh-issue: 113171 +.. nonce: VFnObz +.. section: Library + +Fixed various false positives and false negatives in + +* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details) +* :attr:`ipaddress.IPv4Address.is_global` +* :attr:`ipaddress.IPv6Address.is_private` +* :attr:`ipaddress.IPv6Address.is_global` + +Also in the corresponding :class:`ipaddress.IPv4Network` and +:class:`ipaddress.IPv6Network` attributes. + +.. + +.. date: 2024-03-13-15-45-54 +.. gh-issue: 63283 +.. nonce: OToJnG +.. section: Library + +In :mod:`encodings.idna`, any capitalization of the the ACE prefix +(``xn--``) is now acceptable. Patch by Pepijn de Vos and Zackery Spytz. + +.. + +.. date: 2024-03-12-19-32-17 +.. gh-issue: 71042 +.. nonce: oI0Ron +.. section: Library + +Add :func:`platform.android_ver`, which provides device and OS information +on Android. + +.. + +.. date: 2024-03-12-17-53-14 +.. gh-issue: 73468 +.. nonce: z4ZzvJ +.. section: Library + +Added new :func:`math.fma` function, wrapping C99's ``fma()`` operation: +fused multiply-add function. Patch by Mark Dickinson and Victor Stinner. + +.. + +.. date: 2024-03-11-17-04-55 +.. gh-issue: 116608 +.. nonce: 30f58- +.. section: Library + +The :mod:`importlib.resources` functions +:func:`~importlib.resources.is_resource`, +:func:`~importlib.resources.open_binary`, +:func:`~importlib.resources.open_text`, +:func:`~importlib.resources.path`, +:func:`~importlib.resources.read_binary`, and +:func:`~importlib.resources.read_text` are un-deprecated, and support +subdirectories via multiple positional arguments. The +:func:`~importlib.resources.contents` function also allows subdirectories, +but remains deprecated. + +.. + +.. date: 2024-03-08-11-31-49 +.. gh-issue: 116484 +.. nonce: VMAsU7 +.. section: Library + +Change automatically generated :class:`tkinter.Checkbutton` widget names to +avoid collisions with automatically generated +:class:`tkinter.ttk.Checkbutton` widget names within the same parent widget. + +.. + +.. date: 2024-03-07-11-10-27 +.. gh-issue: 114314 +.. nonce: iEhAMH +.. section: Library + +In :mod:`ctypes`, ctype data is now stored in type objects directly rather +than in a dict subclass. This is an internal change that should not affect +usage. + +.. + +.. date: 2024-03-06-18-30-37 +.. gh-issue: 116401 +.. nonce: 3Wcda2 +.. section: Library + +Fix blocking :func:`os.fwalk` and :func:`shutil.rmtree` on opening named +pipe. + +.. + +.. date: 2024-03-05-19-56-29 +.. gh-issue: 71052 +.. nonce: PMDK-- +.. section: Library + +Implement :func:`ctypes.util.find_library` on Android. + +.. + +.. date: 2024-03-01-20-23-57 +.. gh-issue: 90535 +.. nonce: wXm-jC +.. section: Library + +Fix support of *interval* values > 1 in +:class:`logging.TimedRotatingFileHandler` for ``when='MIDNIGHT'`` and +``when='Wx'``. + +.. + +.. date: 2024-02-26-10-06-50 +.. gh-issue: 113308 +.. nonce: MbvOFt +.. section: Library + +Remove some internal protected parts from :mod:`uuid`: +``_has_uuid_generate_time_safe``, ``_netbios_getnode``, +``_ipconfig_getnode``, and ``_load_system_functions``. They were unused. + +.. + +.. date: 2024-02-18-09-50-31 +.. gh-issue: 115627 +.. nonce: HGchj0 +.. section: Library + +Fix the :mod:`ssl` module error handling of connection terminate by peer. It +now throws an OSError with the appropriate error code instead of an +EOFError. + +.. + +.. date: 2024-02-01-08-09-20 +.. gh-issue: 114847 +.. nonce: -JrWrR +.. section: Library + +Speed up :func:`os.path.realpath` on non-Windows platforms. + +.. + +.. date: 2024-02-01-03-09-38 +.. gh-issue: 114271 +.. nonce: raCkt5 +.. section: Library + +Fix a race in ``threading.Thread.join()``. + +``threading._MainThread`` now always represents the main thread of the main +interpreter. + +``PyThreadState.on_delete`` and ``PyThreadState.on_delete_data`` have been +removed. + +.. + +.. date: 2024-01-22-15-50-58 +.. gh-issue: 113538 +.. nonce: v2wrwg +.. section: Library + +Add :meth:`asyncio.Server.close_clients` and +:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully +close an asyncio server. + +.. + +.. date: 2024-01-02-22-47-12 +.. gh-issue: 85287 +.. nonce: ZC5DLj +.. section: Library + +Changes Unicode codecs to return UnicodeEncodeError or UnicodeDecodeError, +rather than just UnicodeError. + +.. + +.. date: 2023-12-28-22-52-45 +.. gh-issue: 113548 +.. nonce: j6TJ7O +.. section: Library + +:mod:`pdb` now allows CLI arguments to ``pdb -m``. + +.. + +.. date: 2023-12-11-00-51-51 +.. gh-issue: 112948 +.. nonce: k-OKp5 +.. section: Library + +Make completion of :mod:`pdb` similar to Python REPL + +.. + +.. date: 2023-06-16-19-17-06 +.. gh-issue: 105866 +.. nonce: 0NBveV +.. section: Library + +Fixed ``_get_slots`` bug which caused error when defining dataclasses with +slots and a weakref_slot. + +.. + +.. date: 2023-05-06-05-00-42 +.. gh-issue: 96471 +.. nonce: S3X5I- +.. section: Library + +Add :py:class:`asyncio.Queue` termination with +:py:meth:`~asyncio.Queue.shutdown` method. + +.. + +.. date: 2022-06-22-14-45-32 +.. gh-issue: 89739 +.. nonce: CqZcRL +.. section: Library + +The :mod:`zipimport` module can now read ZIP64 files. + +.. + +.. bpo: 33533 +.. date: 2020-10-02-17-35-19 +.. nonce: GLIhM5 +.. section: Library + +:func:`asyncio.as_completed` now returns an object that is both an +asynchronous iterator and plain iterator. The new asynchronous iteration +pattern allows for easier correlation between prior tasks and their +completed results. This is a closer match to +:func:`concurrent.futures.as_completed`'s iteration pattern. Patch by Justin +Arthur. + +.. + +.. bpo: 27578 +.. date: 2020-06-11-16-20-33 +.. nonce: CIA-fu +.. section: Library + +:func:`inspect.getsource` (and related functions) work with empty module +files, returning ``'\n'`` (or reasonable equivalent) instead of raising +``OSError``. Patch by Kernc. + +.. + +.. bpo: 37141 +.. date: 2019-09-26-17-52-52 +.. nonce: onYY2- +.. section: Library + +Accept an iterable of separators in :meth:`asyncio.StreamReader.readuntil`, +stopping when one of them is encountered. + +.. + +.. date: 2019-08-27-01-03-26 +.. gh-issue: 66543 +.. nonce: _TRpYr +.. section: Library + +Make :func:`mimetypes.guess_type` properly parsing of URLs with only a host +name, URLs containing fragment or query, and filenames with only a UNC +sharepoint on Windows. Based on patch by Dong-hee Na. + +.. + +.. bpo: 15010 +.. date: 2019-08-12-19-08-06 +.. nonce: 3bY2CF +.. section: Library + +:meth:`unittest.TestLoader.discover` now saves the original value of +``unittest.TestLoader._top_level_dir`` and restores it at the end of the +call. + +.. + +.. date: 2024-03-20-15-12-37 +.. gh-issue: 115977 +.. nonce: IMLi6K +.. section: Documentation + +Remove compatibility references to Emscripten. + +.. + +.. date: 2024-03-20-12-41-47 +.. gh-issue: 114099 +.. nonce: ad_Ck9 +.. section: Documentation + +Add an iOS platform guide, and flag modules not available on iOS. + +.. + +.. date: 2022-04-15-13-15-23 +.. gh-issue: 91565 +.. nonce: OznXwC +.. section: Documentation + +Changes to documentation files and config outputs to reflect the new +location for reporting bugs - i.e. GitHub rather than bugs.python.org. + +.. + +.. date: 2024-03-25-21-31-49 +.. gh-issue: 83434 +.. nonce: U7Z8cY +.. section: Tests + +Disable JUnit XML output (``--junit-xml=FILE`` command line option) in +regrtest when hunting for reference leaks (``-R`` option). Patch by Victor +Stinner. + +.. + +.. date: 2024-03-24-23-49-25 +.. gh-issue: 117187 +.. nonce: eMLT5n +.. section: Tests + +Fix XML tests for vanilla Expat <2.6.0. + +.. + +.. date: 2024-03-21-11-32-29 +.. gh-issue: 116333 +.. nonce: F-9Ram +.. section: Tests + +Tests of TLS related things (error codes, etc) were updated to be more +lenient about specific error message strings and behaviors as seen in the +BoringSSL and AWS-LC forks of OpenSSL. + +.. + +.. date: 2024-03-20-14-19-32 +.. gh-issue: 117089 +.. nonce: WwR1Z1 +.. section: Tests + +Consolidated tests for importlib.metadata in their own ``metadata`` package. + +.. + +.. date: 2024-03-13-12-06-49 +.. gh-issue: 115979 +.. nonce: zsNpQD +.. section: Tests + +Update test_importlib so that it passes under WASI SDK 21. + +.. + +.. date: 2024-03-11-23-20-28 +.. gh-issue: 112536 +.. nonce: Qv1RrX +.. section: Tests + +Add --tsan to test.regrtest for running TSAN tests in reasonable execution +times. Patch by Donghee Na. + +.. + +.. date: 2024-03-06-11-00-36 +.. gh-issue: 116307 +.. nonce: Uij0t_ +.. section: Tests + +Added import helper ``isolated_modules`` as ``CleanImport`` does not remove +modules imported during the context. Use it in importlib.resources tests to +avoid leaving ``mod`` around to impede importlib.metadata tests. + +.. + +.. date: 2024-03-13-16-16-43 +.. gh-issue: 114736 +.. nonce: ZhmauG +.. section: Build + +Have WASI builds use WASI SDK 21. + +.. + +.. date: 2024-03-08-17-05-15 +.. gh-issue: 115983 +.. nonce: ZQqk0Q +.. section: Build + +Skip building test modules that must be built as shared under WASI. + +.. + +.. date: 2024-03-06-17-26-55 +.. gh-issue: 71052 +.. nonce: vLbu9u +.. section: Build + +Add Android build script and instructions. + +.. + +.. date: 2024-03-28-22-12-00 +.. gh-issue: 117267 +.. nonce: K_tki1 +.. section: Windows + +Ensure ``DirEntry.stat().st_ctime`` behaves consistently with +:func:`os.stat` during the deprecation period of ``st_ctime`` by containing +the same value as ``st_birthtime``. After the deprecation period, +``st_ctime`` will be the metadata change time (or unavailable through +``DirEntry``), and only ``st_birthtime`` will contain the creation time. + +.. + +.. date: 2024-03-14-20-46-23 +.. gh-issue: 116195 +.. nonce: Cu_rYs +.. section: Windows + +Improves performance of :func:`os.getppid` by using an alternate system API +when available. Contributed by vxiiduu. + +.. + +.. date: 2024-03-14-09-14-21 +.. gh-issue: 88494 +.. nonce: Bwfmp7 +.. section: Windows + +On Windows, :func:`time.monotonic` now uses the +``QueryPerformanceCounter()`` clock to have a resolution better than 1 us, +instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. +Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-01-58-22 +.. gh-issue: 116773 +.. nonce: H2UldY +.. section: Windows + +Fix instances of ``<_overlapped.Overlapped object at 0xXXX> still has +pending operation at deallocation, the process may crash``. + +.. + +.. date: 2024-02-24-23-03-43 +.. gh-issue: 91227 +.. nonce: sL4zWC +.. section: Windows + +Fix the asyncio ProactorEventLoop implementation so that sending a datagram +to an address that is not listening does not prevent receiving any more +datagrams. + +.. + +.. date: 2024-02-08-14-48-15 +.. gh-issue: 115119 +.. nonce: qMt32O +.. section: Windows + +Switched from vendored ``libmpdecimal`` code to a separately-hosted external +package in the ``cpython-source-deps`` repository when building the +``_decimal`` module. + +.. + +.. date: 2024-04-08-18-53-33 +.. gh-issue: 117642 +.. nonce: _-tYH_ +.. section: C API + +Fix :pep:`737` implementation for ``%#T`` and ``%#N``. + +.. + +.. date: 2024-03-22-19-29-24 +.. gh-issue: 87193 +.. nonce: u7O-jY +.. section: C API + +:c:func:`_PyBytes_Resize` can now be called for bytes objects with reference +count > 1, including 1-byte bytes objects. It creates a new bytes object and +destroys the old one if it has reference count > 1. + +.. + +.. date: 2024-03-20-13-13-22 +.. gh-issue: 117021 +.. nonce: 0Q5jBx +.. section: C API + +Fix integer overflow in :c:func:`PyLong_AsPid` on non-Windows 64-bit +platforms. + +.. + +.. date: 2024-03-19-09-49-04 +.. gh-issue: 115756 +.. nonce: 4Ls_Tl +.. section: C API + +:c:func:`!PyCode_GetFirstFree` is an ustable API now and has been renamed to +:c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in +:gh:`115781`) + +.. + +.. date: 2024-03-18-10-58-47 +.. gh-issue: 116869 +.. nonce: lN0GBl +.. section: C API + +Add ``test_cext`` test: build a C extension to check if the Python C API +emits C compiler warnings. Patch by Victor Stinner. + +.. + +.. date: 2024-03-18-09-58-46 +.. gh-issue: 116869 +.. nonce: LFDVKM +.. section: C API + +Make the C API compatible with ``-Werror=declaration-after-statement`` +compiler flag again. Patch by Victor Stinner. + +.. + +.. date: 2024-03-17-22-42-21 +.. gh-issue: 116936 +.. nonce: tNrzfm +.. section: C API + +Add :c:func:`PyType_GetModuleByDef` to the limited C API. Patch by Victor +Stinner. + +.. + +.. date: 2024-03-16-12-21-00 +.. gh-issue: 116809 +.. nonce: JL786L +.. section: C API + +Restore removed private ``_PyErr_ChainExceptions1()`` function. Patch by +Victor Stinner. + +.. + +.. date: 2024-03-15-23-57-33 +.. gh-issue: 115754 +.. nonce: zLdv82 +.. section: C API + +In the limited C API version 3.13, getting ``Py_None``, ``Py_False``, +``Py_True``, ``Py_Ellipsis`` and ``Py_NotImplemented`` singletons is now +implemented as function calls at the stable ABI level to hide implementation +details. Getting these constants still return borrowed references. Patch by +Victor Stinner. + +.. + +.. date: 2024-03-15-23-55-24 +.. gh-issue: 115754 +.. nonce: xnzc__ +.. section: C API + +Add :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` functions +to get constants. For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns +a :term:`strong reference` to the constant zero. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-22-30-07 +.. gh-issue: 111696 +.. nonce: 76UMKi +.. section: C API + +Add support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats to +:c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object +type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for +more information. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-18-00-32 +.. gh-issue: 111696 +.. nonce: L6oIPq +.. section: C API + +Add :c:func:`PyType_GetModuleName` function to get the type's module name. +Equivalent to getting the ``type.__module__`` attribute. Patch by Eric Snow +and Victor Stinner. + +.. + +.. date: 2024-03-14-15-17-11 +.. gh-issue: 111696 +.. nonce: YmnvAi +.. section: C API + +Add :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully +qualified name. Equivalent to ``f"{type.__module__}.{type.__qualname__}"``, +or ``type.__qualname__`` if ``type.__module__`` is not a string or is equal +to ``"builtins"``. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-10-33-58 +.. gh-issue: 85283 +.. nonce: LOgmdU +.. section: C API + +The ``fcntl``, ``grp``, ``pwd``, ``termios``, ``_statistics`` and +``_testconsole`` C extensions are now built with the :ref:`limited C API +`. Patch by Victor Stinner. + +.. + +.. date: 2024-02-28-15-50-01 +.. gh-issue: 111140 +.. nonce: mpwcUg +.. section: C API + +Add additional flags to :c:func:`PyLong_AsNativeBytes` and +:c:func:`PyLong_FromNativeBytes` to allow the caller to determine how to +handle edge cases around values that fill the entire buffer. + +.. + +.. date: 2023-12-12-19-48-31 +.. gh-issue: 113024 +.. nonce: rXcQs7 +.. section: C API + +Add :c:func:`PyObject_GenericHash` function. diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst new file mode 100644 index 00000000000000..97731276679ba6 --- /dev/null +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -0,0 +1,1671 @@ +.. date: 2024-03-27-13-50-02 +.. gh-issue: 116741 +.. nonce: ZoGryG +.. release date: 2024-05-08 +.. section: Security + +Update bundled libexpat to 2.6.2 + +.. + +.. date: 2024-03-25-21-25-28 +.. gh-issue: 117233 +.. nonce: E4CyI_ +.. section: Security + +Detect BLAKE2, SHA3, Shake, & truncated SHA512 support in the OpenSSL-ish +libcrypto library at build time. This allows :mod:`hashlib` to be used with +libraries that do not to support every algorithm that upstream OpenSSL does. + +.. + +.. date: 2024-05-07-01-39-24 +.. gh-issue: 118414 +.. nonce: G5GG7l +.. section: Core and Builtins + +Add instrumented opcodes to YIELD_VALUE assertion for tracing cases. + +.. + +.. date: 2024-05-06-10-57-54 +.. gh-issue: 117953 +.. nonce: DqCzIs +.. section: Core and Builtins + +When a builtin or extension module is imported for the first time, while a +subinterpreter is active, the module's init function is now run by the main +interpreter first before import continues in the subinterpreter. +Consequently, single-phase init modules now fail in an isolated +subinterpreter without the init function running under that interpreter, +whereas before it would run under the subinterpreter *before* failing, +potentially leaving behind global state and callbacks and otherwise leaving +the module in an inconsistent state. + +.. + +.. date: 2024-05-05-12-04-02 +.. gh-issue: 117549 +.. nonce: kITawD +.. section: Core and Builtins + +Don't use designated initializer syntax in inline functions in internal +headers. They cause problems for C++ or MSVC users who aren't yet using the +latest C++ standard (C++20). While internal, pycore_backoff.h, is included +(indirectly, via pycore_code.h) by some key 3rd party software that does so +for speed. + +.. + +.. date: 2024-05-03-18-01-26 +.. gh-issue: 95382 +.. nonce: 73FSEv +.. section: Core and Builtins + +Improve performance of :func:`json.dumps` and :func:`json.dump` when using +the argument *indent*. Depending on the data the encoding using +:func:`json.dumps` with *indent* can be up to 2 to 3 times faster. + +.. + +.. date: 2024-05-03-17-49-37 +.. gh-issue: 116322 +.. nonce: Gy6M4j +.. section: Core and Builtins + +In ``--disable-gil`` builds, the GIL will be enabled while loading C +extension modules. If the module indicates that it supports running without +the GIL, the GIL will be disabled once loading is complete. Otherwise, the +GIL will remain enabled for the remainder of the interpreter's lifetime. +This behavior does not apply if the GIL has been explicitly enabled or +disabled with ``PYTHON_GIL`` or ``-Xgil``. + +.. + +.. date: 2024-05-02-21-19-35 +.. gh-issue: 118513 +.. nonce: qHODjb +.. section: Core and Builtins + +Fix incorrect :exc:`UnboundLocalError` when two comprehensions in the same +function both reference the same name, and in one comprehension the name is +bound while in the other it's an implicit global. + +.. + +.. date: 2024-05-02-20-32-42 +.. gh-issue: 118518 +.. nonce: m-JbTi +.. section: Core and Builtins + +Allow the Linux perf support to work without frame pointers using perf's +advanced JIT support. The feature is activated when using the +``PYTHON_PERF_JIT_SUPPORT`` environment variable or when running Python with +``-Xperf_jit``. Patch by Pablo Galindo. + +.. + +.. date: 2024-05-02-16-04-51 +.. gh-issue: 117514 +.. nonce: CJiuC0 +.. section: Core and Builtins + +Add ``sys._is_gil_enabled()`` function that returns whether the GIL is +currently enabled. In the default build it always returns ``True`` because +the GIL is always enabled. In the free-threaded build, it may return +``True`` or ``False``. + +.. + +.. date: 2024-05-02-15-57-07 +.. gh-issue: 118164 +.. nonce: AF6kwI +.. section: Core and Builtins + +Break a loop between the Python implementation of the :mod:`decimal` module +and the Python code for integer to string conversion. Also optimize integer +to string conversion for values in the range from 9_000 to 135_000 decimal +digits. + +.. + +.. date: 2024-05-01-22-43-54 +.. gh-issue: 118473 +.. nonce: QIvq9R +.. section: Core and Builtins + +Fix :func:`sys.set_asyncgen_hooks` not to be partially set when raising +:exc:`TypeError`. + +.. + +.. date: 2024-05-01-17-12-36 +.. gh-issue: 118465 +.. nonce: g3Q8iE +.. section: Core and Builtins + +Compiler populates the new ``__firstlineno__`` field on a class with the +line number of the first line of the class definition. + +.. + +.. date: 2024-05-01-14-20-28 +.. gh-issue: 118492 +.. nonce: VUsSfn +.. section: Core and Builtins + +Fix an issue where the type cache can expose a previously accessed attribute +when a finalizer is run. + +.. + +.. date: 2024-05-01-07-06-48 +.. gh-issue: 117714 +.. nonce: Ip_dm5 +.. section: Core and Builtins + +update ``async_generator.athrow().close()`` and +``async_generator.asend().close()`` to close their section of the underlying +async generator + +.. + +.. date: 2024-04-28-00-41-17 +.. gh-issue: 111201 +.. nonce: cQsh5U +.. section: Core and Builtins + +The :term:`interactive` interpreter is now implemented in Python, which +allows for a number of new features like colors, multiline input, history +viewing, and paste mode. Contributed by Pablo Galindo, Łukasz Langa and +Lysandros Nikolaou based on code from the PyPy project. + +.. + +.. date: 2024-04-27-21-44-40 +.. gh-issue: 74929 +.. nonce: C2nESp +.. section: Core and Builtins + +Implement PEP 667: converted :attr:`FrameType.f_locals ` and +:c:func:`PyFrame_GetLocals` to return a write-through proxy object when the +frame refers to a function or comprehension. + +.. + +.. date: 2024-04-27-16-23-29 +.. gh-issue: 116767 +.. nonce: z9UFpr +.. section: Core and Builtins + +Fix crash in compiler on 'async with' that has many context managers. + +.. + +.. date: 2024-04-26-14-06-18 +.. gh-issue: 118335 +.. nonce: SRFsxO +.. section: Core and Builtins + +Change how to use the tier 2 interpreter. Instead of running Python with +``-X uops`` or setting the environment variable ``PYTHON_UOPS=1``, this +choice is now made at build time by configuring with +``--enable-experimental-jit=interpreter``. + +**Beware!** This changes the environment variable to enable or disable +micro-ops to ``PYTHON_JIT``. The old ``PYTHON_UOPS`` is no longer used. + +.. + +.. date: 2024-04-26-05-38-18 +.. gh-issue: 118306 +.. nonce: vRUEOU +.. section: Core and Builtins + +Update JIT compilation to use LLVM 18 + +.. + +.. date: 2024-04-25-21-18-19 +.. gh-issue: 118160 +.. nonce: GH5SMc +.. section: Core and Builtins + +:ref:`Annotation scopes ` within classes can now contain +comprehensions. However, such comprehensions are not inlined into their +parent scope at runtime. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-04-25-12-55-47 +.. gh-issue: 118272 +.. nonce: 5ptjk_ +.. section: Core and Builtins + +Fix bug where ``generator.close`` does not free the generator frame's +locals. + +.. + +.. date: 2024-04-25-11-48-28 +.. gh-issue: 118216 +.. nonce: SVg700 +.. section: Core and Builtins + +Don't consider :mod:`__future__` imports with dots before the module name. + +.. + +.. date: 2024-04-22-08-34-28 +.. gh-issue: 118074 +.. nonce: 5_JnIa +.. section: Core and Builtins + +Make sure that the Executor objects in the COLD_EXITS array aren't assumed +to be GC-able (which would access bytes outside the object). + +.. + +.. date: 2024-04-20-20-30-15 +.. gh-issue: 107674 +.. nonce: GZPOP7 +.. section: Core and Builtins + +Lazy load frame line number to improve performance of tracing + +.. + +.. date: 2024-04-19-11-59-57 +.. gh-issue: 118082 +.. nonce: _FLuOT +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for imports without names, like in ``from +x import`` and ``import`` cases. It now points out to users that +:keyword:`import` expects at least one name after it. + +.. + +.. date: 2024-04-19-11-57-46 +.. gh-issue: 118090 +.. nonce: eGAQ0B +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for empty type param brackets. + +.. + +.. date: 2024-04-18-03-49-41 +.. gh-issue: 117958 +.. nonce: -EsfUs +.. section: Core and Builtins + +Added a ``get_jit_code()`` method to access JIT compiled machine code from +the UOp Executor when the experimental JIT is enabled. Patch by Anthony +Shaw. + +.. + +.. date: 2024-04-17-22-53-52 +.. gh-issue: 117901 +.. nonce: SsEcVJ +.. section: Core and Builtins + +Add option for compiler's codegen to save nested instruction sequences for +introspection. + +.. + +.. date: 2024-04-17-22-49-15 +.. gh-issue: 116622 +.. nonce: tthNUF +.. section: Core and Builtins + +Redirect stdout and stderr to system log when embedded in an Android app. + +.. + +.. date: 2024-04-17-17-52-32 +.. gh-issue: 109118 +.. nonce: q9iPEI +.. section: Core and Builtins + +:ref:`annotation scope ` within class scopes can now +contain lambdas. + +.. + +.. date: 2024-04-15-13-53-59 +.. gh-issue: 117894 +.. nonce: 8LpZ6m +.. section: Core and Builtins + +Prevent ``agen.aclose()`` objects being re-used after ``.throw()``. + +.. + +.. date: 2024-04-15-07-37-09 +.. gh-issue: 117881 +.. nonce: 07H0wI +.. section: Core and Builtins + +prevent concurrent access to an async generator via athrow().throw() or +asend().throw() + +.. + +.. date: 2024-04-13-16-55-53 +.. gh-issue: 117536 +.. nonce: xkVbfv +.. section: Core and Builtins + +Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)``. + +.. + +.. date: 2024-04-12-12-28-49 +.. gh-issue: 117755 +.. nonce: 6ct8kU +.. section: Core and Builtins + +Fix mimalloc allocator for huge memory allocation (around 8,589,934,592 GiB) +on s390x. Patch by Victor Stinner. + +.. + +.. date: 2024-04-12-11-19-18 +.. gh-issue: 117750 +.. nonce: YttK6h +.. section: Core and Builtins + +Fix issue where an object's dict would get out of sync with the object's +internal values when being cleared. ``obj.__dict__.clear()`` now clears the +internal values, but leaves the dict attached to the object. + +.. + +.. date: 2024-04-12-09-09-11 +.. gh-issue: 117431 +.. nonce: lxFEeJ +.. section: Core and Builtins + +Improve the performance of the following :class:`bytes` and +:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` +calling convention: + +* :meth:`!count` +* :meth:`!find` +* :meth:`!index` +* :meth:`!rfind` +* :meth:`!rindex` + +.. + +.. date: 2024-04-10-22-16-18 +.. gh-issue: 117709 +.. nonce: -_1YL0 +.. section: Core and Builtins + +Speed up calls to :func:`str` with positional-only argument, by using the +:pep:`590` ``vectorcall`` calling convention. Patch by Erlend Aasland. + +.. + +.. date: 2024-04-09-16-07-00 +.. gh-issue: 117680 +.. nonce: MRZ78K +.. section: Core and Builtins + +Give ``_PyInstructionSequence`` a Python interface and use it in tests. + +.. + +.. date: 2024-04-09-11-31-25 +.. gh-issue: 115776 +.. nonce: 5Nthd0 +.. section: Core and Builtins + +Statically allocated objects are, by definition, immortal so must be marked +as such regardless of whether they are in extension modules or not. + +.. + +.. date: 2024-03-30-00-37-53 +.. gh-issue: 117385 +.. nonce: h0OJti +.. section: Core and Builtins + +Remove unhandled ``PY_MONITORING_EVENT_BRANCH`` and +``PY_MONITORING_EVENT_EXCEPTION_HANDLED`` events from :func:`sys.settrace`. + +.. + +.. date: 2024-03-12-13-51-09 +.. gh-issue: 116322 +.. nonce: q8TcDQ +.. section: Core and Builtins + +Extension modules may indicate to the runtime that they can run without the +GIL. Multi-phase init modules do so by calling providing +``Py_MOD_GIL_NOT_USED`` for the ``Py_mod_gil`` slot, while single-phase init +modules call ``PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED)`` from +their init function. + +.. + +.. date: 2024-02-29-18-55-45 +.. gh-issue: 116129 +.. nonce: wsFnIq +.. section: Core and Builtins + +Implement :pep:`696`, adding support for defaults on type parameters. Patch +by Jelle Zijlstra. + +.. + +.. date: 2024-02-26-13-14-52 +.. gh-issue: 93502 +.. nonce: JMWRvA +.. section: Core and Builtins + +Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and +:c:func:`PyRefTracer_GetTracer`, that allows to track object creation and +destruction the same way the :mod:`tracemalloc` module does. Patch by Pablo +Galindo + +.. + +.. date: 2024-02-04-07-45-29 +.. gh-issue: 107674 +.. nonce: q8mCmi +.. section: Core and Builtins + +Improved the performance of :func:`sys.settrace` significantly + +.. + +.. date: 2024-01-07-03-38-34 +.. gh-issue: 95754 +.. nonce: aPjEBG +.. section: Core and Builtins + +Improve the error message when a script shadowing a module from the standard +library causes :exc:`AttributeError` to be raised. Similarly, improve the +error message when a script shadowing a third party module attempts to +access an attribute from that third party module while still initialising. + +.. + +.. date: 2023-12-03-18-21-59 +.. gh-issue: 99180 +.. nonce: 5m0V0q +.. section: Core and Builtins + +Elide uninformative traceback indicators in ``return`` and simple +``assignment`` statements. Patch by Pablo Galindo. + +.. + +.. date: 2023-06-18-00-27-57 +.. gh-issue: 105879 +.. nonce: dPw78k +.. section: Core and Builtins + +Allow the *globals* and *locals* arguments to :func:`exec` and :func:`eval` +to be passed as keywords. + +.. + +.. date: 2024-05-07-11-23-11 +.. gh-issue: 118418 +.. nonce: QPMdJm +.. section: Library + +A :exc:`DeprecationWarning` is now emitted if you fail to pass a value to +the new *type_params* parameter of ``typing._eval_type()`` or +``typing.ForwardRef._evaluate()``. (Using either of these private and +undocumented functions is discouraged to begin with, but failing to pass a +value to the ``type_params`` parameter may lead to incorrect behaviour on +Python 3.12 or newer.) + +.. + +.. date: 2024-05-06-18-13-02 +.. gh-issue: 118660 +.. nonce: n01Vb7 +.. section: Library + +Add an optional second type parameter to :class:`typing.ContextManager` and +:class:`typing.AsyncContextManager`, representing the return types of +:meth:`~object.__exit__` and :meth:`~object.__aexit__` respectively. This +parameter defaults to ``bool | None``. + +.. + +.. date: 2024-05-06-16-52-40 +.. gh-issue: 118650 +.. nonce: qKz5lp +.. section: Library + +The ``enum`` module allows method named ``_repr_*`` to be defined on +``Enum`` types. + +.. + +.. date: 2024-05-06-08-23-01 +.. gh-issue: 118648 +.. nonce: OVA3jJ +.. section: Library + +Add type parameter defaults to :class:`typing.Generator` and +:class:`typing.AsyncGenerator`. + +.. + +.. date: 2024-05-05-16-08-03 +.. gh-issue: 101137 +.. nonce: 71ECXu +.. section: Library + +Mime type ``text/x-rst`` is now supported by :mod:`mimetypes`. + +.. + +.. date: 2024-05-04-20-22-59 +.. gh-issue: 118164 +.. nonce: 9D02MQ +.. section: Library + +The Python implementation of the ``decimal`` module could appear to hang in +relatively small power cases (like ``2**117``) if context precision was set +to a very high value. A different method to check for exactly representable +results is used now that doesn't rely on computing ``10**precision`` (which +could be effectively too large to compute). + +.. + +.. date: 2024-05-04-18-40-43 +.. gh-issue: 111744 +.. nonce: nuCtwN +.. section: Library + +``breakpoint()`` and ``pdb.set_trace()`` now enter the debugger immediately +after the call rather than before the next line is executed. + +.. + +.. date: 2024-05-02-04-27-12 +.. gh-issue: 118500 +.. nonce: pBGGtQ +.. section: Library + +Add :mod:`pdb` support for zipapps + +.. + +.. date: 2024-04-30-15-18-19 +.. gh-issue: 118406 +.. nonce: y-GnMo +.. section: Library + +Add signature for :class:`sqlite3.Connection` objects. + +.. + +.. date: 2024-04-30-12-59-04 +.. gh-issue: 101732 +.. nonce: 29zUDu +.. section: Library + +Use a Y2038 compatible openssl time function when available. + +.. + +.. date: 2024-04-29-22-11-54 +.. gh-issue: 118404 +.. nonce: GYfMaD +.. section: Library + +Fix :func:`inspect.signature` for non-comparable callables. + +.. + +.. date: 2024-04-29-21-51-28 +.. gh-issue: 118402 +.. nonce: Z_06Th +.. section: Library + +Fix :func:`inspect.signature` for the result of the +:func:`functools.cmp_to_key` call. + +.. + +.. date: 2024-04-27-20-34-56 +.. gh-issue: 116622 +.. nonce: YlQgXv +.. section: Library + +On Android, :any:`sysconfig.get_platform` now returns the format specified +by :pep:`738`. + +.. + +.. date: 2024-04-26-14-53-28 +.. gh-issue: 118285 +.. nonce: A0_pte +.. section: Library + +Allow to specify the signature of custom callable instances of extension +type by the ``__text_signature__`` attribute. Specify signatures of +:class:`operator.attrgetter`, :class:`operator.itemgetter`, and +:class:`operator.methodcaller` instances. + +.. + +.. date: 2024-04-26-12-42-29 +.. gh-issue: 118314 +.. nonce: Z7reGc +.. section: Library + +Fix an edge case in :func:`binascii.a2b_base64` strict mode, where excessive +padding is not detected when no padding is necessary. + +.. + +.. date: 2024-04-25-11-49-11 +.. gh-issue: 118271 +.. nonce: 5N2Xcy +.. section: Library + +Add the :class:`!PhotoImage` methods :meth:`!read` to +read an image from a file and :meth:`!data` to get the +image data. Add *background* and *grayscale* parameters to +:class:`!PhotoImage` method :meth:`!write`. + +.. + +.. date: 2024-04-24-16-07-26 +.. gh-issue: 118225 +.. nonce: KdrcgL +.. section: Library + +Add the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region +from one image to other image, possibly with pixel zooming and/or +subsampling. Add *from_coords* parameter to :class:`!PhotoImage` methods +:meth:`!copy`, :meth:`!zoom` and :meth:`!subsample`. Add *zoom* and +*subsample* parameters to :class:`!PhotoImage` method :meth:`!copy`. + +.. + +.. date: 2024-04-24-12-29-33 +.. gh-issue: 118221 +.. nonce: 2k_bac +.. section: Library + +Fix a bug where :meth:`sqlite3.Connection.iterdump` could fail if a custom +:attr:`row factory ` was used. Patch by +Erlend Aasland. + +.. + +.. date: 2024-04-24-12-20-48 +.. gh-issue: 118013 +.. nonce: TKn_kZ +.. section: Library + +Fix regression introduced in gh-103193 that meant that calling +:func:`inspect.getattr_static` on an instance would cause a strong reference +to that instance's class to persist in an internal cache in the +:mod:`inspect` module. This caused unexpected memory consumption if the +class was dynamically created, the class held strong references to other +objects which took up a significant amount of memory, and the cache +contained the sole strong reference to the class. The fix for the regression +leads to a slowdown in :func:`!getattr_static`, but the function should +still be significantly faster than it was in Python 3.11. Patch by Alex +Waygood. + +.. + +.. date: 2024-04-24-07-45-08 +.. gh-issue: 118218 +.. nonce: m1OHbN +.. section: Library + +Speed up :func:`itertools.pairwise` in the common case by up to 1.8x. + +.. + +.. date: 2024-04-23-21-17-00 +.. gh-issue: 117486 +.. nonce: ea3KYD +.. section: Library + +Improve the behavior of user-defined subclasses of :class:`ast.AST`. Such +classes will now require no changes in the usual case to conform with the +behavior changes of the :mod:`ast` module in Python 3.13. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-04-22-21-54-12 +.. gh-issue: 90848 +.. nonce: 5jHEEc +.. section: Library + +Fixed :func:`unittest.mock.create_autospec` to configure parent mock with +keyword arguments. + +.. + +.. date: 2024-04-22-20-42-29 +.. gh-issue: 118168 +.. nonce: Igni7h +.. section: Library + +Fix incorrect argument substitution when :data:`typing.Unpack` is used with +the builtin :class:`tuple`. :data:`!typing.Unpack` now raises +:exc:`TypeError` when used with certain invalid types. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-04-21-18-55-42 +.. gh-issue: 118131 +.. nonce: eAT0is +.. section: Library + +Add command-line interface for the :mod:`random` module. Patch by Hugo van +Kemenade. + +.. + +.. date: 2024-04-19-09-28-43 +.. gh-issue: 118107 +.. nonce: Mdsr1J +.. section: Library + +Fix :mod:`zipimport` reading of ZIP64 files with file entries that are too +big or offset too far. + +.. + +.. date: 2024-04-19-08-50-48 +.. gh-issue: 102511 +.. nonce: qDEB66 +.. section: Library + +Fix :func:`os.path.normpath` for UNC paths on Windows. +Speed up :func:`os.path.splitroot` with a native implementation. + +.. + +.. date: 2024-04-18-00-35-11 +.. gh-issue: 117535 +.. nonce: 0m6SIM +.. section: Library + +Change the unknown filename of :mod:`warnings` from ``sys`` to ```` to +clarify that it's not a real filename. + +.. + +.. date: 2024-04-17-22-00-15 +.. gh-issue: 114053 +.. nonce: _JBV4D +.. section: Library + +Fix erroneous :exc:`NameError` when calling :func:`typing.get_type_hints` on +a class that made use of :pep:`695` type parameters in a module that had +``from __future__ import annotations`` at the top of the file. Patch by Alex +Waygood. + +.. + +.. date: 2024-04-17-21-28-24 +.. gh-issue: 116931 +.. nonce: _AS09h +.. section: Library + +Add parameter *fileobj* check for :func:`tarfile.TarFile.addfile` + +.. + +.. date: 2024-04-17-19-41-59 +.. gh-issue: 117995 +.. nonce: Vt76Rv +.. section: Library + +Don't raise :exc:`DeprecationWarning` when a :term:`sequence` of parameters +is used to bind indexed, nameless placeholders. See also :gh:`100668`. + +.. + +.. date: 2024-04-17-18-00-30 +.. gh-issue: 80361 +.. nonce: RstWg- +.. section: Library + +Fix TypeError in :func:`email.message.Message.get_payload` when the charset is +:rfc:`2231` encoded. + +.. + +.. date: 2024-04-16-18-34-11 +.. gh-issue: 86650 +.. nonce: Zeydyg +.. section: Library + +Fix IndexError when parse some emails with invalid Message-ID (including +one-off addresses generated by Microsoft Outlook). + +.. + +.. date: 2024-04-14-15-59-28 +.. gh-issue: 117691 +.. nonce: 1mtREE +.. section: Library + +Improve the error messages emitted by :mod:`tarfile` deprecation warnings +relating to PEP 706. If a ``filter`` argument is not provided to +``extract()`` or ``extractall``, the deprecation warning now points to the +line in the user's code where the relevant function was called. Patch by +Alex Waygood. + +.. + +.. date: 2024-04-13-18-59-25 +.. gh-issue: 115874 +.. nonce: c3xG-E +.. section: Library + +Fixed a possible segfault during garbage collection of +``_asyncio.FutureIter`` objects. Patch by Savannah Ostrowski. + +.. + +.. date: 2024-04-13-01-45-15 +.. gh-issue: 115060 +.. nonce: IxoM03 +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by omitting an initial +:meth:`~pathlib.Path.is_dir` call. As a result of this change, +:meth:`~pathlib.Path.glob` can no longer raise :exc:`OSError`. + +.. + +.. date: 2024-04-12-17-37-11 +.. gh-issue: 77102 +.. nonce: Mk6X_E +.. section: Library + +:mod:`site` module now parses ``.pth`` file with UTF-8 first, and +:term:`locale encoding` if ``UnicodeDecodeError`` happened. It supported +only locale encoding before. + +.. + +.. date: 2024-04-11-18-11-37 +.. gh-issue: 76785 +.. nonce: BWNkhC +.. section: Library + +We've exposed the low-level :mod:`!_interpreters` module for the sake of the +PyPI implementation of :pep:`734`. It was sometimes available as the +:mod:`!_xxsubinterpreters` module and was formerly used only for testing. +For the most part, it should be considered an internal module, like +:mod:`!_thread` and :mod:`!_imp`. See +https://discuss.python.org/t/pep-734-multiple-interpreters-in-the-stdlib/41147/26. + +.. + +.. date: 2024-04-10-22-35-24 +.. gh-issue: 115060 +.. nonce: XEVuOb +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by not scanning directories for +non-wildcard pattern segments. + +.. + +.. date: 2024-04-10-21-30-37 +.. gh-issue: 117727 +.. nonce: uAYNVS +.. section: Library + +Speed up :meth:`pathlib.Path.iterdir` by using :func:`os.scandir` +internally. + +.. + +.. date: 2024-04-10-21-08-32 +.. gh-issue: 117586 +.. nonce: UCL__1 +.. section: Library + +Speed up :meth:`pathlib.Path.walk` by working with strings internally. + +.. + +.. date: 2024-04-10-20-59-10 +.. gh-issue: 117722 +.. nonce: oxIUEI +.. section: Library + +Change the new multi-separator support in :meth:`asyncio.StreamReader.readuntil` +to only accept tuples of separators rather than arbitrary iterables. + +.. + +.. date: 2024-04-09-23-22-21 +.. gh-issue: 117692 +.. nonce: EciInD +.. section: Library + +Fixes a bug when :class:`doctest.DocTestFinder` was failing on wrapped +``builtin_function_or_method``. + +.. + +.. date: 2024-04-09-20-14-44 +.. gh-issue: 117348 +.. nonce: A2NAAz +.. section: Library + +Largely restored import time performance of configparser by avoiding +dataclasses. + +.. + +.. date: 2024-04-08-19-30-38 +.. gh-issue: 117641 +.. nonce: oaBGSJ +.. section: Library + +Speedup :func:`os.path.commonpath` on Unix. + +.. + +.. date: 2024-04-08-19-12-26 +.. gh-issue: 117663 +.. nonce: CPfc_p +.. section: Library + +Fix ``_simple_enum`` to detect aliases when multiple arguments are present +but only one is the member value. + +.. + +.. date: 2024-04-08-14-33-38 +.. gh-issue: 117636 +.. nonce: exnRKd +.. section: Library + +Speedup :func:`os.path.join`. + +.. + +.. date: 2024-04-08-03-23-22 +.. gh-issue: 117618 +.. nonce: -4DCUw +.. section: Library + +Support ``package.module`` as ``filename`` for ``break`` command of +:mod:`pdb` + +.. + +.. date: 2024-04-07-19-39-20 +.. gh-issue: 102247 +.. nonce: h8rqiX +.. section: Library + +the status codes enum with constants in http.HTTPStatus are updated to +include the names from RFC9110. This RFC includes some HTTP statuses +previously only used for WEBDAV and assigns more generic names to them. + +The old constants are preserved for backwards compatibility. + +.. + +.. date: 2024-04-07-18-42-09 +.. gh-issue: 117607 +.. nonce: C978BD +.. section: Library + +Speedup :func:`os.path.relpath`. + +.. + +.. date: 2024-04-06-20-31-09 +.. gh-issue: 117586 +.. nonce: UgWdRK +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by working with strings internally. + +.. + +.. date: 2024-04-06-18-41-36 +.. gh-issue: 117225 +.. nonce: tJh1Hw +.. section: Library + +Add colour to doctest output. Patch by Hugo van Kemenade. + +.. + +.. date: 2024-04-05-15-51-01 +.. gh-issue: 117566 +.. nonce: 54nABf +.. section: Library + +:meth:`ipaddress.IPv6Address.is_loopback` will now return ``True`` for +IPv4-mapped loopback addresses, i.e. addresses in the +``::ffff:127.0.0.0/104`` address space. + +.. + +.. date: 2024-04-05-13-38-53 +.. gh-issue: 117546 +.. nonce: lWjhHE +.. section: Library + +Fix issue where :func:`os.path.realpath` stopped resolving symlinks after +encountering a symlink loop on POSIX. + +.. + +.. date: 2024-04-04-15-28-12 +.. gh-issue: 116720 +.. nonce: aGhXns +.. section: Library + +Improved behavior of :class:`asyncio.TaskGroup` when an external +cancellation collides with an internal cancellation. For example, when two +task groups are nested and both experience an exception in a child task +simultaneously, it was possible that the outer task group would misbehave, +because its internal cancellation was swallowed by the inner task group. + +In the case where a task group is cancelled externally and also must raise +an :exc:`ExceptionGroup`, it will now call the parent task's +:meth:`~asyncio.Task.cancel` method. This ensures that a +:exc:`asyncio.CancelledError` will be raised at the next :keyword:`await`, +so the cancellation is not lost. + +An added benefit of these changes is that task groups now preserve the +cancellation count (:meth:`asyncio.Task.cancelling`). + +In order to handle some corner cases, :meth:`asyncio.Task.uncancel` may now +reset the undocumented ``_must_cancel`` flag when the cancellation count +reaches zero. + +.. + +.. date: 2024-04-03-16-01-31 +.. gh-issue: 117516 +.. nonce: 7DlHje +.. section: Library + +Add :data:`typing.TypeIs`, implementing :pep:`742`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-04-03-15-04-23 +.. gh-issue: 117503 +.. nonce: NMfwup +.. section: Library + +Fix support of non-ASCII user names in bytes paths in +:func:`os.path.expanduser` on Posix. + +.. + +.. date: 2024-04-02-11-17-44 +.. gh-issue: 117394 +.. nonce: 2aoSlb +.. section: Library + +:func:`os.path.ismount` is now 2-3 times faster if the user has permissions. + +.. + +.. date: 2024-03-29-15-14-51 +.. gh-issue: 117313 +.. nonce: ks_ONu +.. section: Library + +Only treat ``'\n'``, ``'\r'`` and ``'\r\n'`` as line separators in +re-folding the :mod:`email` messages. Preserve control characters ``'\v'``, +``'\f'``, ``'\x1c'``, ``'\x1d'`` and ``'\x1e'`` and Unicode line separators +``'\x85'``, ``'\u2028'`` and ``'\u2029'`` as is. + +.. + +.. date: 2024-03-29-12-21-40 +.. gh-issue: 117142 +.. nonce: U0agfh +.. section: Library + +Convert :mod:`!_ctypes` to multi-phase initialisation (:pep:`489`). + +.. + +.. date: 2024-03-26-15-29-39 +.. gh-issue: 66543 +.. nonce: OZBhU5 +.. section: Library + +Add the :func:`mimetypes.guess_file_type` function which works with file +path. Passing file path instead of URL in :func:`~mimetypes.guess_type` is +:term:`soft deprecated`. + +.. + +.. date: 2024-03-20-00-11-39 +.. gh-issue: 68583 +.. nonce: mIlxxb +.. section: Library + +webbrowser CLI: replace getopt with argparse, add long options. Patch by +Hugo van Kemenade. + +.. + +.. date: 2024-03-17-18-24-23 +.. gh-issue: 116871 +.. nonce: 9uSl8M +.. section: Library + +Name suggestions for :exc:`AttributeError` and :exc:`ImportError` now only +include underscored names if the original name was underscored. + +.. + +.. date: 2024-02-28-11-51-51 +.. gh-issue: 116023 +.. nonce: CGYhFh +.. section: Library + +Don't show empty fields (value ``None`` or ``[]``) in :func:`ast.dump` by +default. Add ``show_empty=False`` parameter to optionally show them. + +.. + +.. date: 2024-02-28-10-41-24 +.. gh-issue: 115961 +.. nonce: P-_DU0 +.. section: Library + +Added :attr:`!name` and :attr:`!mode` attributes for compressed and archived +file-like objects in modules :mod:`bz2`, :mod:`lzma`, :mod:`tarfile` and +:mod:`zipfile`. The value of the :attr:`!mode` attribute of +:class:`gzip.GzipFile` was changed from integer (``1`` or ``2``) to string +(``'rb'`` or ``'wb'``). The value of the :attr:`!mode` attribute of the +readable file-like object returned by :meth:`zipfile.ZipFile.open` was +changed from ``'r'`` to ``'rb'``. + +.. + +.. date: 2024-02-11-07-31-43 +.. gh-issue: 82062 +.. nonce: eeS6w7 +.. section: Library + +Fix :func:`inspect.signature` to correctly handle parameter defaults on +methods in extension modules that use names defined in the module namespace. + +.. + +.. date: 2024-01-19-05-40-46 +.. gh-issue: 83856 +.. nonce: jN5M80 +.. section: Library + +Honor :mod:`atexit` for all :mod:`multiprocessing` start methods + +.. + +.. date: 2023-12-14-02-51-38 +.. gh-issue: 113081 +.. nonce: S-9Qyn +.. section: Library + +Print colorized exception just like built-in traceback in :mod:`pdb` + +.. + +.. date: 2023-12-07-20-05-54 +.. gh-issue: 112855 +.. nonce: ph4ehh +.. section: Library + +Speed up pickling of :class:`pathlib.PurePath` objects. Patch by Barney +Gale. + +.. + +.. date: 2023-11-07-22-41-42 +.. gh-issue: 111744 +.. nonce: TbLxF0 +.. section: Library + +Support opcode events in :mod:`bdb` + +.. + +.. date: 2023-10-24-12-39-04 +.. gh-issue: 109617 +.. nonce: YoI8TV +.. section: Library + +:mod:`!ncurses`: fixed a crash that could occur on macOS 13 or earlier when +Python was built with Apple Xcode 15's SDK. + +.. + +.. date: 2023-10-20-03-50-17 +.. gh-issue: 83151 +.. nonce: bcsD40 +.. section: Library + +Enabled arbitrary statements and evaluations in :mod:`pdb` shell to access +the local variables of the current frame, which made it possible for +multi-scope code like generators or nested function to work. + +.. + +.. date: 2023-10-02-10-35-58 +.. gh-issue: 110209 +.. nonce: b5zfIz +.. section: Library + +Add :meth:`~object.__class_getitem__` to :class:`types.GeneratorType` and +:class:`types.CoroutineType` for type hinting purposes. Patch by James +Hilton-Balfe. + +.. + +.. date: 2023-08-21-10-34-43 +.. gh-issue: 108191 +.. nonce: GZM3mv +.. section: Library + +The :class:`types.SimpleNamespace` now accepts an optional positional +argument which specifies initial values of attributes as a dict or an +iterable of key-value pairs. + +.. + +.. date: 2023-05-28-11-25-18 +.. gh-issue: 62090 +.. nonce: opAhDn +.. section: Library + +Fix assertion errors caused by whitespace in metavars or ``SUPPRESS``-ed +groups in :mod:`argparse` by simplifying usage formatting. Patch by Ali +Hamdan. + +.. + +.. date: 2023-03-03-21-13-08 +.. gh-issue: 102402 +.. nonce: fpkRO1 +.. section: Library + +Adjust ``logging.LogRecord`` to use ``time.time_ns()`` and fix minor bug +related to floating-point math. + +.. + +.. date: 2022-12-14-15-53-38 +.. gh-issue: 100242 +.. nonce: Ny7VUO +.. section: Library + +Bring pure Python implementation ``functools.partial.__new__`` more in line +with the C-implementation by not just always checking for the presence of +the attribute ``'func'`` on the first argument of ``partial``. Instead, both +the Python version and the C version perform an ``isinstance(func, +partial)`` check on the first argument of ``partial``. + +.. + +.. date: 2022-11-23-17-16-31 +.. gh-issue: 99730 +.. nonce: bDQdaX +.. section: Library + +HEAD requests are no longer upgraded to GET request during redirects in +urllib. + +.. + +.. date: 2022-10-24-12-05-19 +.. gh-issue: 66410 +.. nonce: du4UKW +.. section: Library + +Setting the :mod:`!tkinter` module global :data:`!wantobjects` to ``2`` +before creating the :class:`~tkinter.Tk` object or call the +:meth:`!wantobjects` method of the :class:`!Tk` object with argument +``2`` makes now arguments to callbacks registered in the :mod:`tkinter` module +to be passed as various Python objects (``int``, ``float``, ``bytes``, ``tuple``), +depending on their internal representation in Tcl, instead of always ``str``. +:data:`!tkinter.wantobjects` is now set to ``2`` by default. + +.. + +.. bpo: 40943 +.. date: 2020-06-10-19-24-17 +.. nonce: vjiiN_ +.. section: Library + +Fix several IndexError when parse emails with truncated Message-ID, address, +routes, etc, e.g. ``example@``. + +.. + +.. bpo: 39324 +.. date: 2020-01-14-09-46-51 +.. nonce: qUcDrM +.. section: Library + +Add mime type mapping for .md <-> text/markdown + +.. + +.. bpo: 18108 +.. date: 2019-09-09-18-18-34 +.. nonce: ajPLAO +.. section: Library + +:func:`shutil.chown` now supports *dir_fd* and *follow_symlinks* keyword +arguments. + +.. + +.. bpo: 30988 +.. date: 2019-08-29-20-26-08 +.. nonce: b-_h5O +.. section: Library + +Fix parsing of emails with invalid address headers having a leading or +trailing dot. Patch by tsufeki. + +.. + +.. bpo: 32839 +.. date: 2018-02-13-10-02-54 +.. nonce: McbVz3 +.. section: Library + +Add the :meth:`!after_info` method for Tkinter widgets. + +.. + +.. date: 2024-04-25-22-12-20 +.. gh-issue: 117928 +.. nonce: LKdTno +.. section: Documentation + +The minimum Sphinx version required for the documentation is now 6.2.1. + +.. + +.. date: 2024-05-07-21-15-47 +.. gh-issue: 118734 +.. nonce: --GHiS +.. section: Build + +Fixes Windows build when invoked directly (not through the :file:`build.bat` +script) without specifying a value for ``UseTIER2``. + +.. + +.. date: 2024-05-06-00-39-06 +.. gh-issue: 115119 +.. nonce: LT27pF +.. section: Build + +The :file:`configure` option :option:`--with-system-libmpdec` now defaults +to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python +3.15. + +.. + +.. date: 2024-04-15-08-35-06 +.. gh-issue: 117845 +.. nonce: IowzyW +.. section: Build + +Fix building against recent libedit versions by detecting readline hook +signatures in :program:`configure`. + +.. + +.. date: 2024-04-14-19-35-35 +.. gh-issue: 116622 +.. nonce: 8lpX-7 +.. section: Build + +A testbed project was added to run the test suite on Android. + +.. + +.. date: 2024-04-09-12-59-06 +.. gh-issue: 117645 +.. nonce: 0oEVAa +.. section: Build + +Increase WASI stack size from 512 KiB to 8 MiB and the initial memory from +10 MiB to 20 MiB. Patch by Victor Stinner. + +.. + +.. date: 2024-02-13-15-31-28 +.. gh-issue: 115119 +.. nonce: FnQzAW +.. section: Build + +:program:`configure` now uses :program:`pkg-config` to detect :mod:`decimal` +dependencies if the :option:`--with-system-libmpdec` option is given. + +.. + +.. date: 2024-05-02-09-28-04 +.. gh-issue: 115119 +.. nonce: cUKMXo +.. section: Windows + +Update Windows installer to use libmpdecimal 4.0.0. + +.. + +.. date: 2024-05-01-20-57-09 +.. gh-issue: 118486 +.. nonce: K44KJG +.. section: Windows + +:func:`os.mkdir` now accepts *mode* of ``0o700`` to restrict the new +directory to the current user. + +.. + +.. date: 2024-04-29-13-53-25 +.. gh-issue: 118347 +.. nonce: U5ZRm_ +.. section: Windows + +Fixes launcher updates not being installed. + +.. + +.. date: 2024-04-26-14-23-07 +.. gh-issue: 118293 +.. nonce: ohhPtW +.. section: Windows + +The ``multiprocessing`` module now passes the ``STARTF_FORCEOFFFEEDBACK`` +flag when spawning processes to tell Windows not to change the mouse cursor. + +.. + +.. date: 2024-04-15-21-23-34 +.. gh-issue: 115009 +.. nonce: uhisHP +.. section: Windows + +Update Windows installer to use SQLite 3.45.3. + +.. + +.. date: 2024-04-12-14-02-58 +.. gh-issue: 90329 +.. nonce: YpEeaO +.. section: Windows + +Suppress the warning displayed on virtual environment creation when the +requested and created paths differ only by a short (8.3 style) name. +Warnings will continue to be shown if a junction or symlink in the path +caused the venv to be created in a different location than originally +requested. + +.. + +.. date: 2024-04-12-13-18-42 +.. gh-issue: 117786 +.. nonce: LpI01s +.. section: Windows + +Fixes virtual environments not correctly launching when created from a Store +install. + +.. + +.. date: 2024-05-03-12-13-27 +.. gh-issue: 115119 +.. nonce: ltDtoR +.. section: macOS + +Update macOS installer to use libmpdecimal 4.0.0. + +.. + +.. date: 2024-04-19-08-40-00 +.. gh-issue: 114099 +.. nonce: _iDfrQ +.. section: macOS + +iOS preprocessor symbol usage was made compatible with older macOS SDKs. + +.. + +.. date: 2024-04-15-21-19-39 +.. gh-issue: 115009 +.. nonce: IdxH9N +.. section: macOS + +Update macOS installer to use SQLite 3.45.3. + +.. + +.. date: 2022-04-17-01-07-42 +.. gh-issue: 91629 +.. nonce: YBGAAt +.. section: macOS + +Use :file:`~/.config/fish/conf.d` configs and :program:`fish_add_path` to +set :envvar:`PATH` when installing for the Fish shell. + +.. + +.. bpo: 34774 +.. date: 2018-09-23-01-36-39 +.. nonce: VeM-X- +.. section: IDLE + +Use user-selected color theme for Help => IDLE Doc. + +.. + +.. date: 2024-04-29-17-44-15 +.. gh-issue: 118124 +.. nonce: czQQ9G +.. section: C API + +Fix :c:macro:`Py_BUILD_ASSERT` and :c:macro:`Py_BUILD_ASSERT_EXPR` for +non-constant expressions: use ``static_assert()`` on C11 and newer. Patch by +Victor Stinner. + +.. + +.. date: 2024-04-29-17-19-07 +.. gh-issue: 110850 +.. nonce: vcpLn1 +.. section: C API + +Add "Raw" variant of PyTime functions + +* :c:func:`PyTime_MonotonicRaw` +* :c:func:`PyTime_PerfCounterRaw` +* :c:func:`PyTime_TimeRaw` + +Patch by Victor Stinner. + +.. + +.. date: 2024-04-17-16-48-17 +.. gh-issue: 117987 +.. nonce: zsvNL1 +.. section: C API + +Restore functions removed in Python 3.13 alpha 1: + +* :c:func:`Py_SetPythonHome` +* :c:func:`Py_SetProgramName` +* :c:func:`PySys_SetArgvEx` +* :c:func:`PySys_SetArgv` + +Patch by Victor Stinner. + +.. + +.. date: 2024-04-16-13-34-01 +.. gh-issue: 117929 +.. nonce: HSr419 +.. section: C API + +Restore removed :c:func:`PyEval_InitThreads` function. Patch by Victor +Stinner. + +.. + +.. date: 2024-04-08-09-44-29 +.. gh-issue: 117534 +.. nonce: 54ZE_n +.. section: C API + +Improve validation logic in the C implementation of +:meth:`datetime.datetime.fromisoformat` to better handle invalid years. +Patch by Vlad Efanov. + +.. + +.. date: 2024-03-18-17-29-52 +.. gh-issue: 68114 +.. nonce: W7R_lI +.. section: C API + +Fixed skipitem()'s handling of the old 'w' and 'w#' formatters. These are +no longer supported and now raise an exception if used. + +.. + +.. date: 2024-03-13-17-48-24 +.. gh-issue: 111997 +.. nonce: 8ZbHlA +.. section: C API + +Add a C-API for firing monitoring events. diff --git a/Misc/NEWS.d/3.14.0a1.rst b/Misc/NEWS.d/3.14.0a1.rst new file mode 100644 index 00000000000000..dfc33812611185 --- /dev/null +++ b/Misc/NEWS.d/3.14.0a1.rst @@ -0,0 +1,6604 @@ +.. date: 2024-09-24-10-48-46 +.. gh-issue: 124448 +.. nonce: bFMrS6 +.. release date: 2024-10-15 +.. section: macOS + +Update bundled Tcl/Tk in macOS installer to 8.6.15. + +.. + +.. date: 2024-09-07-12-14-54 +.. gh-issue: 123797 +.. nonce: yFDeug +.. section: macOS + +Check for runtime availability of ``ptsname_r`` function on macos. + +.. + +.. date: 2024-09-04-11-55-29 +.. gh-issue: 123418 +.. nonce: 8P4bmN +.. section: macOS + +Updated macOS installer build to use OpenSSL 3.0.15. + +.. + +.. date: 2024-09-27-15-07-30 +.. gh-issue: 124487 +.. nonce: 7LrwHC +.. section: Windows + +Increases Windows required OS and API level to Windows 10. + +.. + +.. date: 2024-09-27-13-40-25 +.. gh-issue: 124609 +.. nonce: WaKk8G +.. section: Windows + +Fix ``_Py_ThreadId`` for Windows builds using MinGW. Patch by Tony Roberts. + +.. + +.. date: 2024-09-24-19-04-56 +.. gh-issue: 124448 +.. nonce: srVT3d +.. section: Windows + +Updated bundled Tcl/Tk to 8.6.15. + +.. + +.. date: 2024-09-20-11-18-50 +.. gh-issue: 124254 +.. nonce: iPin-L +.. section: Windows + +Ensures experimental free-threaded binaries remain installed when updating. + +.. + +.. date: 2024-09-10-19-23-00 +.. gh-issue: 123915 +.. nonce: yZMEDO +.. section: Windows + +Ensure that ``Tools\msi\buildrelease.bat`` uses different directories for +AMD64 and ARM64 builds. + +.. + +.. date: 2024-09-04-09-59-18 +.. gh-issue: 123418 +.. nonce: QaMC12 +.. section: Windows + +Updated Windows build to use OpenSSL 3.0.15. + +.. + +.. date: 2024-08-29-16-13-45 +.. gh-issue: 123476 +.. nonce: m2DFS4 +.. section: Windows + +Add support for ``socket.TCP_QUICKACK`` on Windows platforms. + +.. + +.. date: 2024-08-01-10-55-15 +.. gh-issue: 122573 +.. nonce: 4-UCFY +.. section: Windows + +The Windows build of CPython now requires 3.10 or newer. + +.. + +.. date: 2024-07-19-21-50-54 +.. gh-issue: 100256 +.. nonce: GDrKba +.. section: Windows + +:mod:`mimetypes` no longer fails when it encounters an inaccessible registry +key. + +.. + +.. date: 2024-05-30-17-39-25 +.. gh-issue: 119679 +.. nonce: mZC87w +.. section: Windows + +Ensures correct import libraries are included in Windows installs. + +.. + +.. date: 2024-05-29-11-06-12 +.. gh-issue: 119690 +.. nonce: 8q6e1p +.. section: Windows + +Adds Unicode support and fixes audit events for ``_winapi.CreateNamedPipe``. + +.. + +.. date: 2024-05-25-18-43-10 +.. gh-issue: 111201 +.. nonce: SLPJIx +.. section: Windows + +Add support for new pyrepl on Windows + +.. + +.. date: 2024-05-22-19-43-29 +.. gh-issue: 119070 +.. nonce: _enton +.. section: Windows + +Fixes ``py.exe`` handling of shebangs like ``/usr/bin/env python3.12``, +which were previously interpreted as ``python3.exe`` instead of +``python3.12.exe``. + +.. + +.. date: 2024-04-24-22-50-33 +.. gh-issue: 117505 +.. nonce: gcTb_p +.. section: Windows + +Fixes an issue with the Windows installer not running ensurepip in a fully +isolated environment. This could cause unexpected interactions with the user +site-packages. + +.. + +.. date: 2024-04-24-05-16-32 +.. gh-issue: 118209 +.. nonce: Ryyzlz +.. section: Windows + +Avoid crashing in :mod:`mmap` on Windows when the mapped memory is +inaccessible due to file system errors or access violations. + +.. + +.. date: 2022-04-20-18-32-30 +.. gh-issue: 79846 +.. nonce: Vggv3f +.. section: Windows + +Makes :code:`ssl.create_default_context()` ignore invalid certificates in +the Windows certificate store + +.. + +.. date: 2024-09-04-10-07-51 +.. gh-issue: 123418 +.. nonce: 1eIFZb +.. section: Tools/Demos + +Update GitHub CI workflows to use OpenSSL 3.0.15 and multissltests to use +3.0.15, 3.1.7, and 3.2.3. + +.. + +.. date: 2024-10-07-14-13-38 +.. gh-issue: 125041 +.. nonce: PKLWDf +.. section: Tests + +Re-enable skipped tests for :mod:`zlib` on the s390x architecture: only skip +checks of the compressed bytes, which can be different between zlib's +software implementation and the hardware-accelerated implementation. + +.. + +.. date: 2024-09-25-12-39-34 +.. gh-issue: 124378 +.. nonce: Ywwgia +.. section: Tests + +Updated ``test_ttk`` to pass with Tcl/Tk 8.6.15. + +.. + +.. date: 2024-09-18-18-39-21 +.. gh-issue: 124213 +.. nonce: AQq_xg +.. section: Tests + +Detect whether the test suite is running inside a systemd-nspawn container +with ``--suppress-sync=true`` option, and skip the ``test_os`` and +``test_mmap`` tests that are failing in this scenario. + +.. + +.. date: 2024-09-17-22-21-58 +.. gh-issue: 124190 +.. nonce: 3fWhiX +.. section: Tests + +Add capability to ignore entire files or directories in check warning CI +tool + +.. + +.. date: 2024-07-17-08-25-06 +.. gh-issue: 121921 +.. nonce: HW8CIS +.. section: Tests + +Update ``Lib/test/crashers/bogus_code_obj.py`` so that it crashes properly +again. + +.. + +.. date: 2024-07-13-21-55-58 +.. gh-issue: 112301 +.. nonce: YJS1dl +.. section: Tests + +Add tooling to check for changes in compiler warnings. Patch by Nate Ohlson. + +.. + +.. date: 2024-07-13-11-48-20 +.. gh-issue: 59022 +.. nonce: fYNbQ8 +.. section: Tests + +Add tests for :func:`pkgutil.extend_path`. Patch by Andreas Stocker. + +.. + +.. date: 2024-07-13-11-04-44 +.. gh-issue: 99242 +.. nonce: aGxnwz +.. section: Tests + +:func:`os.getloadavg` may throw :exc:`OSError` when running regression tests +under certain conditions (e.g. chroot). This error is now caught and +ignored, since reporting load average is optional. + +.. + +.. date: 2024-07-04-15-10-29 +.. gh-issue: 121084 +.. nonce: qxcd5d +.. section: Tests + +Fix test_typing random leaks. Clear typing ABC caches when running tests for +refleaks (``-R`` option): call ``_abc_caches_clear()`` on typing abstract +classes and their subclasses. Patch by Victor Stinner. + +.. + +.. date: 2024-07-03-14-41-00 +.. gh-issue: 121160 +.. nonce: LEtiTd +.. section: Tests + +Add a test for :func:`readline.set_history_length`. Note that this test may +fail on readline libraries. + +.. + +.. date: 2024-07-01-16-15-06 +.. gh-issue: 121200 +.. nonce: 4Pc-gc +.. section: Tests + +Fix ``test_expanduser_pwd2()`` of ``test_posixpath``. Call ``getpwnam()`` +to get ``pw_dir``, since it can be different than ``getpwall()`` ``pw_dir``. +Patch by Victor Stinner. + +.. + +.. date: 2024-07-01-09-04-32 +.. gh-issue: 121188 +.. nonce: XbuTVa +.. section: Tests + +When creating the JUnit XML file, regrtest now escapes characters which are +invalid in XML, such as the chr(27) control character used in ANSI escape +sequences. Patch by Victor Stinner. + +.. + +.. date: 2024-06-20-12-51-26 +.. gh-issue: 120801 +.. nonce: lMVXC9 +.. section: Tests + +Cleaned up fixtures for importlib.metadata tests and consolidated behavior +with 'test.support.os_helper'. + +.. + +.. date: 2024-05-29-15-28-08 +.. gh-issue: 119727 +.. nonce: dVkaZM +.. section: Tests + +Add ``--single-process`` command line option to Python test runner +(regrtest). Patch by Victor Stinner. + +.. + +.. date: 2024-05-20-18-06-31 +.. gh-issue: 119273 +.. nonce: hf-yhX +.. section: Tests + +Python test runner no longer runs tests using TTY (ex: test_ioctl) in a +process group (using ``setsid()``). Previously, tests using TTY were +skipped. Patch by Victor Stinner. + +.. + +.. date: 2024-05-18-10-59-27 +.. gh-issue: 119050 +.. nonce: g4qiH7 +.. section: Tests + +regrtest test runner: Add XML support to the refleak checker (-R option). +Patch by Victor Stinner. + +.. + +.. date: 2024-05-04-22-56-41 +.. gh-issue: 101525 +.. nonce: LHK166 +.. section: Tests + +Skip ``test_gdb`` if the binary is relocated by BOLT. Patch by Donghee Na. + +.. + +.. date: 2023-08-03-17-26-55 +.. gh-issue: 107562 +.. nonce: ZnbscS +.. section: Tests + +Test certificates have been updated to expire far in the future. This allows +testing Y2038 with system time set to after that, so that actual Y2038 +issues can be exposed, and not masked by expired certificate errors. + +.. + +.. date: 2024-10-09-20-08-13 +.. gh-issue: 125140 +.. nonce: YgNWRB +.. section: Security + +Remove the current directory from ``sys.path`` when using PyREPL. + +.. + +.. date: 2024-09-04-12-41-35 +.. gh-issue: 123678 +.. nonce: N41y9n +.. section: Security + +Upgrade libexpat to 2.6.3 + +.. + +.. date: 2024-08-14-19-43-57 +.. gh-issue: 112301 +.. nonce: IQUcOy +.. section: Security + +Enable compiler options that warn of potential security vulnerabilities. + +.. + +.. date: 2024-08-07-10-42-13 +.. gh-issue: 122792 +.. nonce: oiTMo9 +.. section: Security + +Changed IPv4-mapped ``ipaddress.IPv6Address`` to consistently use the mapped +IPv4 address value for deciding properties. Properties which have their +behavior fixed are ``is_multicast``, ``is_reserved``, ``is_link_local``, +``is_global``, and ``is_unspecified``. + +.. + +.. date: 2024-08-06-00-06-23 +.. gh-issue: 112301 +.. nonce: 4k4lw6 +.. section: Security + +Add ability to ignore warnings per file with warning count in warning +checking tooling. Patch by Nate Ohlson. + +.. + +.. date: 2024-07-24-05-18-25 +.. gh-issue: 112301 +.. nonce: lfINgZ +.. section: Security + +Add macOS warning tracking to warning check tooling. Patch by Nate Ohlson. + +.. + +.. date: 2024-07-22-13-11-28 +.. gh-issue: 122133 +.. nonce: 0mPeta +.. section: Security + +Authenticate the socket connection for the ``socket.socketpair()`` fallback +on platforms where ``AF_UNIX`` is not available like Windows. + +Patch by Gregory P. Smith and Seth Larson +. Reported by Ellie + +.. + +.. date: 2024-07-18-13-17-47 +.. gh-issue: 121957 +.. nonce: QemKLU +.. section: Security + +Fixed missing audit events around interactive use of Python, now also +properly firing for ``python -i``, as well as for ``python -m asyncio``. The +events in question are ``cpython.run_stdin`` and ``cpython.run_startup``. + +.. + +.. date: 2024-07-08-23-39-04 +.. gh-issue: 112301 +.. nonce: TD8G01 +.. section: Security + +Enable runtime protections for glibc to abort execution when unsafe behavior +is encountered, for all platforms except Windows. + +.. + +.. date: 2024-07-02-13-39-20 +.. gh-issue: 121285 +.. nonce: hrl-yI +.. section: Security + +Remove backtracking from tarfile header parsing for ``hdrcharset``, PAX, and +GNU sparse headers. + +.. + +.. date: 2024-06-25-04-42-43 +.. gh-issue: 112301 +.. nonce: god4IC +.. section: Security + +Add default compiler options to improve security. Enable +-Wimplicit-fallthrough, -fstack-protector-strong, -Wtrampolines. + +.. + +.. date: 2024-05-08-21-59-38 +.. gh-issue: 118773 +.. nonce: 7dFRJY +.. section: Security + +Fixes creation of ACLs in :func:`os.mkdir` on Windows to work correctly on +non-English machines. + +.. + +.. date: 2024-05-01-20-57-09 +.. gh-issue: 118486 +.. nonce: K44KJG +.. section: Security + +:func:`os.mkdir` on Windows now accepts *mode* of ``0o700`` to restrict the +new directory to the current user. This fixes :cve:`2024-4030` affecting +:func:`tempfile.mkdtemp` in scenarios where the base temporary directory is +more permissive than the default. + +.. + +.. date: 2024-10-14-04-44-12 +.. gh-issue: 125422 +.. nonce: MlVuC6 +.. section: Library + +Fixed the bug where :mod:`pdb` and :mod:`bdb` can step into the bottom +caller frame. + +.. + +.. date: 2024-10-14-02-27-03 +.. gh-issue: 100141 +.. nonce: NuAcwa +.. section: Library + +Fixed the bug where :mod:`pdb` will be stuck in an infinite loop when +debugging an empty file. + +.. + +.. date: 2024-10-13-20-21-35 +.. gh-issue: 53203 +.. nonce: Rz1c8A +.. section: Library + +Fix :func:`time.strptime` for ``%c``, ``%x`` and ``%X`` formats in many +locales that use non-ASCII digits, like Persian, Burmese, Odia and Shan. + +.. + +.. date: 2024-10-11-18-03-05 +.. gh-issue: 125206 +.. nonce: pWRRK6 +.. section: Library + +Workaround for old libffi versions is added. Module ctypes supports +:c:expr:`double complex` only with libffi >= 3.3.0. Patch by Mikhail Efimov. + +.. + +.. date: 2024-10-11-16-19-46 +.. gh-issue: 89967 +.. nonce: vhWUOR +.. section: Library + +Make :class:`~weakref.WeakKeyDictionary` and +:class:`~weakref.WeakValueDictionary` safe against concurrent mutations from +other threads. Patch by Kumar Aditya. + +.. + +.. date: 2024-10-11-04-04-38 +.. gh-issue: 125260 +.. nonce: PeZ0Mb +.. section: Library + +The :func:`gzip.compress` *mtime* parameter now defaults to 0 for +reproducible output. Patch by Bernhard M. Wiedemann and Adam Turner. + +.. + +.. date: 2024-10-10-20-39-57 +.. gh-issue: 125243 +.. nonce: eUbbtu +.. section: Library + +Fix data race when creating :class:`zoneinfo.ZoneInfo` objects in the free +threading build. + +.. + +.. date: 2024-10-10-19-57-35 +.. gh-issue: 125254 +.. nonce: RtZxXS +.. section: Library + +Fix a bug where ArgumentError includes the incorrect ambiguous option in +:mod:`argparse`. + +.. + +.. date: 2024-10-10-18-33-31 +.. gh-issue: 125235 +.. nonce: 0kOB5I +.. section: Library + +Keep :mod:`tkinter` TCL paths in venv pointing to base installation on +Windows. + +.. + +.. date: 2024-10-09-21-42-43 +.. gh-issue: 61011 +.. nonce: pQXZb1 +.. section: Library + +Fix inheritance of nested mutually exclusive groups from parent parser in +:class:`argparse.ArgumentParser`. Previously, all nested mutually exclusive +groups lost their connection to the group containing them and were displayed +as belonging directly to the parser. + +.. + +.. date: 2024-10-09-07-09-00 +.. gh-issue: 125118 +.. nonce: J9rQ1S +.. section: Library + +Don't copy arbitrary values to :c:expr:`_Bool` in the :mod:`struct` module. + +.. + +.. date: 2024-10-08-21-17-16 +.. gh-issue: 125069 +.. nonce: 0RP0Mx +.. section: Library + +Fix an issue where providing a :class:`pathlib.PurePath` object as an +initializer argument to a second :class:`~pathlib.PurePath` object with a +different :attr:`~pathlib.PurePath.parser` resulted in arguments to the +former object's initializer being joined by the latter object's parser. + +.. + +.. date: 2024-10-08-13-28-22 +.. gh-issue: 125096 +.. nonce: Vz0W5g +.. section: Library + +If the :envvar:`PYTHON_BASIC_REPL` environment variable is set, the +:mod:`site` module no longer imports the :mod:`!_pyrepl` module. Moreover, +the :mod:`site` module now respects :option:`-E` and :option:`-I` command +line options: ignore :envvar:`PYTHON_BASIC_REPL` in this case. Patch by +Victor Stinner. + +.. + +.. date: 2024-10-08-12-09-09 +.. gh-issue: 124969 +.. nonce: _VBQLq +.. section: Library + +Fix ``locale.nl_langinfo(locale.ALT_DIGITS)``. Now it returns a tuple of up +to 100 strings (an empty tuple on most locales). Previously it returned the +first item of that tuple or an empty string. + +.. + +.. date: 2024-10-05-15-49-53 +.. gh-issue: 124960 +.. nonce: Bol9hT +.. section: Library + +Fix support for the ``barry_as_FLUFL`` future flag in the new REPL. + +.. + +.. date: 2024-10-04-12-43-03 +.. gh-issue: 69998 +.. nonce: DVqOXX +.. section: Library + +Fix :func:`locale.nl_langinfo` in case when different categories have +different locales. The function now sets temporarily the ``LC_CTYPE`` locale +in some cases. This temporary change affects other threads. + +.. + +.. date: 2024-10-04-08-46-00 +.. gh-issue: 124958 +.. nonce: rea9-x +.. section: Library + +Fix refcycles in exceptions raised from :class:`asyncio.TaskGroup` and the +python implementation of :class:`asyncio.Future` + +.. + +.. date: 2024-10-03-20-45-57 +.. gh-issue: 53203 +.. nonce: 3Sk4Ia +.. section: Library + +Fix :func:`time.strptime` for ``%c`` and ``%x`` formats in many locales: +Arabic, Bislama, Breton, Bodo, Kashubian, Chuvash, Estonian, French, Irish, +Ge'ez, Gurajati, Manx Gaelic, Hebrew, Hindi, Chhattisgarhi, Haitian Kreyol, +Japanese, Kannada, Korean, Marathi, Malay, Norwegian, Nynorsk, Punjabi, +Rajasthani, Tok Pisin, Yoruba, Yue Chinese, Yau/Nungon and Chinese. + +.. + +.. date: 2024-10-03-19-16-38 +.. gh-issue: 123961 +.. nonce: ik1Dgs +.. section: Library + +Convert :mod:`curses` to multi-phase initialization (:pep:`489`), thereby +fixing reference leaks at interpreter shutdown. Patch by Bénédikt Tran. + +.. + +.. date: 2024-10-03-17-13-22 +.. gh-issue: 124944 +.. nonce: YyLAzf +.. section: Library + +Add ``SO_ORIGINAL_DST`` to the :mod:`socket` module. + +.. + +.. date: 2024-10-03-05-00-25 +.. gh-issue: 117151 +.. nonce: Prdw_W +.. section: Library + +The default buffer size used by :func:`shutil.copyfileobj` has been +increased from 64k to 256k on non-Windows platforms. It was already larger +on Windows. + +.. + +.. date: 2024-10-02-22-53-48 +.. gh-issue: 90102 +.. nonce: 4qX52R +.. section: Library + +Skip the ``isatty`` system call during open() when the file is known to not +be a character device. This provides a slight performance improvement when +reading whole files. + +.. + +.. date: 2024-10-02-21-11-18 +.. gh-issue: 124917 +.. nonce: Lnwh5b +.. section: Library + +Allow calling :func:`os.path.exists` and :func:`os.path.lexists` with +keyword arguments on Windows. Fixes a regression in 3.13.0. + +.. + +.. date: 2024-10-02-16-35-07 +.. gh-issue: 65865 +.. nonce: S2D4wq +.. section: Library + +:mod:`argparse` now raises early error for invalid ``help`` arguments to +:meth:`~argparse.ArgumentParser.add_argument`, +:meth:`~argparse.ArgumentParser.add_subparsers` and :meth:`!add_parser`. + +.. + +.. date: 2024-10-02-15-05-45 +.. gh-issue: 124653 +.. nonce: tqsTu9 +.. section: Library + +Fix detection of the minimal Queue API needed by the :mod:`logging` module. +Patch by Bénédikt Tran. + +.. + +.. date: 2024-10-01-23-29-09 +.. gh-issue: 91818 +.. nonce: Kz8cPI +.. section: Library + +The CLI of many modules (:mod:`ast`, :mod:`ensurepip`, :mod:`json`, +:mod:`pdb`, :mod:`sqlite3`, :mod:`tokenize`, :mod:`venv`) now uses the +actual executable name instead of simply "python" to display in the usage +message. + +.. + +.. date: 2024-10-01-17-12-20 +.. gh-issue: 124858 +.. nonce: Zy0tvT +.. section: Library + +Fix reference cycles left in tracebacks in :func:`asyncio.open_connection` +when used with ``happy_eyeballs_delay`` + +.. + +.. date: 2024-10-01-13-46-58 +.. gh-issue: 124390 +.. nonce: dK1Zcm +.. section: Library + +Fixed :exc:`AssertionError` when using +:func:`!asyncio.staggered.staggered_race` with +:attr:`asyncio.eager_task_factory`. + +.. + +.. date: 2024-10-01-13-11-53 +.. gh-issue: 85935 +.. nonce: CTwJUy +.. section: Library + +:meth:`argparse.ArgumentParser.add_argument` now raises an exception if an +:ref:`action` that does not consume arguments (like 'store_const' or +'store_true') or explicit ``nargs=0`` are specified for positional +arguments. + +.. + +.. date: 2024-10-01-12-43-42 +.. gh-issue: 124835 +.. nonce: SVyp3K +.. section: Library + +Make :func:`tomllib.loads` raise :exc:`TypeError` not :exc:`AttributeError` +on bad input types that do not have the ``replace`` attribute. Improve error +message when :class:`bytes` is received. + +.. + +.. date: 2024-10-01-02-31-13 +.. gh-issue: 124693 +.. nonce: qzbXKB +.. section: Library + +Fix a bug where :mod:`argparse` doesn't recognize negative complex numbers +or negative numbers using scientific notation. + +.. + +.. date: 2024-09-30-20-46-32 +.. gh-issue: 124787 +.. nonce: 3FnJnP +.. section: Library + +Fix :class:`typing.TypeAliasType` with incorrect ``type_params`` argument. +Now it raises a :exc:`TypeError` when a type parameter without a default +follows one with a default, and when an entry in the ``type_params`` tuple +is not a type parameter object. + +.. + +.. date: 2024-09-30-19-59-28 +.. gh-issue: 66436 +.. nonce: 4gYN_n +.. section: Library + +Improved :ref:`prog` default value for :class:`argparse.ArgumentParser`. It +will now include the name of the Python executable along with the module or +package name, or the path to a directory, ZIP file, or directory within a +ZIP file if the code was run that way. + +.. + +.. date: 2024-09-27-15-16-04 +.. gh-issue: 116850 +.. nonce: dBkR0- +.. section: Library + +Fix :mod:`argparse` for namespaces with not directly writable dict (e.g. +classes). + +.. + +.. date: 2024-09-27-06-39-32 +.. gh-issue: 101552 +.. nonce: xYkzag +.. section: Library + +Add an *annoation_format* parameter to :func:`inspect.signature`. Add an +*quote_annotation_strings* parameter to :meth:`inspect.Signature.format`. +Use the new functionality to improve the display of annotations in +signatures in :mod:`pydoc`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-09-26-22-14-12 +.. gh-issue: 58573 +.. nonce: hozbm9 +.. section: Library + +Fix conflicts between abbreviated long options in the parent parser and +subparsers in :mod:`argparse`. + +.. + +.. date: 2024-09-26-13-43-39 +.. gh-issue: 124594 +.. nonce: peYhsP +.. section: Library + +All :mod:`asyncio` REPL prompts run in the same :class:`context +`. Contributed by Bartosz Sławecki. + +.. + +.. date: 2024-09-26-09-18-09 +.. gh-issue: 61181 +.. nonce: dwjmch +.. section: Library + +Fix support of :ref:`choices` with string value in :mod:`argparse`. +Substrings of the specified string no longer considered valid values. + +.. + +.. date: 2024-09-26-00-35-24 +.. gh-issue: 116750 +.. nonce: X1aMHI +.. section: Library + +Provide :func:`sys.monitoring.clear_tool_id` to unregister all events and +callbacks set by the tool. + +.. + +.. date: 2024-09-25-22-06-52 +.. gh-issue: 124552 +.. nonce: 1nQKNM +.. section: Library + +Improve the accuracy of :mod:`bdb`'s check for the possibility of breakpoint +in a frame. This makes it possible to disable unnecessary events in +functions. + +.. + +.. date: 2024-09-25-18-34-48 +.. gh-issue: 124538 +.. nonce: nXZk4R +.. section: Library + +Fixed crash when using :func:`gc.get_referents` on a capsule object. + +.. + +.. date: 2024-09-25-18-08-29 +.. gh-issue: 80259 +.. nonce: kO5Tw7 +.. section: Library + +Fix :mod:`argparse` support of positional arguments with ``nargs='?'``, +``default=argparse.SUPPRESS`` and specified ``type``. + +.. + +.. date: 2024-09-25-18-07-51 +.. gh-issue: 120378 +.. nonce: NlBSz_ +.. section: Library + +Fix a crash related to an integer overflow in :func:`curses.resizeterm` and +:func:`curses.resize_term`. + +.. + +.. date: 2024-09-25-12-14-58 +.. gh-issue: 124498 +.. nonce: Ozxs55 +.. section: Library + +Fix :class:`typing.TypeAliasType` not to be generic, when ``type_params`` is +an empty tuple. + +.. + +.. date: 2024-09-25-10-25-57 +.. gh-issue: 53834 +.. nonce: uyIckw +.. section: Library + +Fix support of arguments with :ref:`choices` in :mod:`argparse`. Positional +arguments with :ref:`nargs` equal to ``'?'`` or ``'*'`` no longer check +:ref:`default` against ``choices``. Optional arguments with ``nargs`` equal +to ``'?'`` no longer check :ref:`const` against ``choices``. + +.. + +.. date: 2024-09-24-22-38-51 +.. gh-issue: 123884 +.. nonce: iEPTK4 +.. section: Library + +Fixed bug in itertools.tee() handling of other tee inputs (a tee in a tee). +The output now has the promised *n* independent new iterators. Formerly, +the first iterator was identical (not independent) to the input iterator. +This would sometimes give surprising results. + +.. + +.. date: 2024-09-24-21-15-27 +.. gh-issue: 123017 +.. nonce: dSAr2f +.. section: Library + +Due to unreliable results on some devices, :func:`time.strftime` no longer +accepts negative years on Android. + +.. + +.. date: 2024-09-24-19-32-14 +.. gh-issue: 123014 +.. nonce: zVcfkZ +.. section: Library + +:func:`os.pidfd_open` and :func:`signal.pidfd_send_signal` are now +unavailable when building against Android API levels older than 31, since +the underlying system calls may cause a crash. + +.. + +.. date: 2024-09-24-13-32-16 +.. gh-issue: 124176 +.. nonce: 6hmOPz +.. section: Library + +Add support for :func:`dataclasses.dataclass` in +:func:`unittest.mock.create_autospec`. Now ``create_autospec`` will check +for potential dataclasses and use :func:`dataclasses.fields` function to +retrieve the spec information. + +.. + +.. date: 2024-09-24-12-34-48 +.. gh-issue: 124345 +.. nonce: s3vKql +.. section: Library + +:mod:`argparse` vim supports abbreviated single-dash long options separated +by ``=`` from its value. + +.. + +.. date: 2024-09-24-00-01-24 +.. gh-issue: 124400 +.. nonce: 0XCgfe +.. section: Library + +Fixed a :mod:`pdb` bug where ``until`` has no effect when it appears in a +``commands`` sequence. Also avoid printing the frame information at a +breakpoint that has a command list containing a command that resumes +execution. + +.. + +.. date: 2024-09-23-18-26-17 +.. gh-issue: 90562 +.. nonce: Yj566G +.. section: Library + +Modify dataclasses to support zero-argument super() when ``slots=True`` is +specified. This works by modifying all references to ``__class__`` to point +to the newly created class. + +.. + +.. date: 2024-09-23-17-33-47 +.. gh-issue: 104860 +.. nonce: O86OSc +.. section: Library + +Fix disallowing abbreviation of single-dash long options in :mod:`argparse` +with ``allow_abbrev=False``. + +.. + +.. date: 2024-09-21-23-56-41 +.. gh-issue: 63143 +.. nonce: YKu-LQ +.. section: Library + +Fix parsing mutually exclusive arguments in :mod:`argparse`. Arguments with +the value identical to the default value (e.g. booleans, small integers, +empty or 1-character strings) are no longer considered "not present". + +.. + +.. date: 2024-09-21-22-32-21 +.. gh-issue: 72795 +.. nonce: naLmkX +.. section: Library + +Positional arguments with :ref:`nargs` equal to ``'*'`` or +:data:`!argparse.REMAINDER` are no longer required. This allows to use +positional argument with ``nargs='*'`` and without ``default`` in mutually +exclusive group and improves error message about required arguments. + +.. + +.. date: 2024-09-21-19-02-37 +.. gh-issue: 59317 +.. nonce: OAhNZZ +.. section: Library + +Fix parsing positional argument with :ref:`nargs` equal to ``'?'`` or +``'*'`` if it is preceded by an option and another positional argument. + +.. + +.. date: 2024-09-20-18-23-19 +.. gh-issue: 100980 +.. nonce: 8nVAB6 +.. section: Library + +The :attr:`~ctypes.Structure._fields_` attribute of +:class:`ctypes.Structure` and :class:`~ctypes.Union` is no longer set if the +setattr operation raises an error. + +.. + +.. date: 2024-09-20-12-23-11 +.. gh-issue: 53780 +.. nonce: mrV1zi +.. section: Library + +:mod:`argparse` now ignores the first ``"--"`` (double dash) between an +option and command. + +.. + +.. date: 2024-09-19-20-15-00 +.. gh-issue: 124217 +.. nonce: j0KlQB +.. section: Library + +Add RFC 9637 reserved IPv6 block ``3fff::/20`` in :mod:`ipaddress` module. + +.. + +.. date: 2024-09-19-16-00-22 +.. gh-issue: 111513 +.. nonce: 6jHm02 +.. section: Library + +Improve the error message that may be raised by +:meth:`datetime.date.fromtimestamp`. + +.. + +.. date: 2024-09-19-11-47-39 +.. gh-issue: 124248 +.. nonce: g7rufd +.. section: Library + +Fixed potential crash when using :mod:`struct` to process zero-width 'Pascal +string' fields (``0p``). + +.. + +.. date: 2024-09-19-10-36-18 +.. gh-issue: 81691 +.. nonce: Hyhp_U +.. section: Library + +Fix handling of multiple ``"--"`` (double dashes) in :mod:`argparse`. Only +the first one has now been removed, all subsequent ones are now taken +literally. + +.. + +.. date: 2024-09-19-03-46-59 +.. gh-issue: 87041 +.. nonce: 9Ox7Bv +.. section: Library + +Fix a bug in :mod:`argparse` where lengthy subparser argument help is +incorrectly indented. + +.. + +.. date: 2024-09-19-00-09-48 +.. gh-issue: 84559 +.. nonce: IrxvQe +.. section: Library + +The default :mod:`multiprocessing` start method on Linux and other POSIX +systems has been changed away from often unsafe ``"fork"`` to +``"forkserver"`` (when the platform supports sending file handles over pipes +as most do) or ``"spawn"``. Mac and Windows are unchanged as they already +default to ``"spawn"``. + +.. + +.. date: 2024-09-18-17-45-52 +.. gh-issue: 124212 +.. nonce: n6kIby +.. section: Library + +Fix invalid variable in :mod:`venv` handling of failed symlink on Windows + +.. + +.. date: 2024-09-17-18-06-42 +.. gh-issue: 124171 +.. nonce: PHCvRJ +.. section: Library + +Add workaround for broken :c:func:`!fmod()` implementations on Windows, that +loose zero sign (e.g. ``fmod(-10, 1)`` returns ``0.0``). Patch by Sergey B +Kirpichev. + +.. + +.. date: 2024-09-16-12-31-48 +.. gh-issue: 123978 +.. nonce: z3smEu +.. section: Library + +Remove broken :func:`time.thread_time` and :func:`time.thread_time_ns` on +NetBSD. + +.. + +.. date: 2024-09-13-10-34-19 +.. gh-issue: 123934 +.. nonce: yMe7mL +.. section: Library + +Fix :class:`unittest.mock.MagicMock` resetting magic methods return values +after ``.reset_mock(return_value=True)`` was called. + +.. + +.. date: 2024-09-12-10-55-19 +.. gh-issue: 124016 +.. nonce: ncs0hd +.. section: Library + +Update :mod:`unicodedata` database to Unicode 16.0.0. + +.. + +.. date: 2024-09-11-19-12-23 +.. gh-issue: 123968 +.. nonce: OwHON_ +.. section: Library + +Fix the command-line interface for the :mod:`random` module to select floats +between 0 and N, not 1 and N. + +.. + +.. date: 2024-09-11-19-05-32 +.. gh-issue: 123945 +.. nonce: jLwybB +.. section: Library + +Fix a bug where :mod:`argparse` doesn't recognize negative numbers with +underscores + +.. + +.. date: 2024-09-11-13-33-19 +.. gh-issue: 123935 +.. nonce: fRZ_56 +.. section: Library + +Fix parent slots detection for dataclasses that inherit from classes with +``__dictoffset__``. + +.. + +.. date: 2024-09-10-11-26-14 +.. gh-issue: 123892 +.. nonce: 2gzIrz +.. section: Library + +Add ``"_wmi"`` to :data:`sys.stdlib_module_names`. Patch by Victor Stinner. + +.. + +.. date: 2024-09-06-10-17-54 +.. gh-issue: 84808 +.. nonce: ION67Z +.. section: Library + +Fix error handling in :py:class:`~socket.socket` method +:py:func:`~socket.socket.connect_ex` on platforms where :c:data:`errno` can +be negative. + +.. + +.. date: 2024-09-06-01-35-11 +.. gh-issue: 123756 +.. nonce: Ozbhke +.. section: Library + +Added a new argument ``mode`` to :class:`pdb.Pdb`. Only allow :mod:`pdb` +from command line to use ``restart`` command. + +.. + +.. date: 2024-09-06-00-00-43 +.. gh-issue: 122765 +.. nonce: tx4hsr +.. section: Library + +Fix unbalanced quote errors occurring when activate.csh in :mod:`venv` was +sourced with a custom prompt containing unpaired quotes or newlines. + +.. + +.. date: 2024-09-04-18-23-43 +.. gh-issue: 123657 +.. nonce: Oks4So +.. section: Library + +Fix crash and memory leak in :func:`decimal.getcontext`. It crashed when +using a thread-local context by ``--with-decimal-contextvar=no``. + +.. + +.. date: 2024-09-02-20-34-04 +.. gh-issue: 123339 +.. nonce: czgcSu +.. section: Library + +Fix :func:`inspect.getsource` for classes in :mod:`collections.abc` and +:mod:`decimal` (for pure Python implementation) modules. +:func:`inspect.getcomments` now raises OSError instead of IndexError if the +``__firstlineno__`` value for a class is out of bound. + +.. + +.. date: 2024-08-31-12-34-44 +.. gh-issue: 123374 +.. nonce: 3kE7rb +.. section: Library + +Remove check for redefined memo entry in :func:`pickletools.dis`. + +.. + +.. date: 2024-08-30-09-01-35 +.. gh-issue: 123504 +.. nonce: lJ9_BB +.. section: Library + +Fixed reference leak in the finalization of :mod:`tkinter`. + +.. + +.. date: 2024-08-29-14-51-36 +.. gh-issue: 123430 +.. nonce: M7wXl9 +.. section: Library + +Pages generated by the :mod:`http.server` module allow the browser to apply +its default dark mode. + +.. + +.. date: 2024-08-29-09-27-12 +.. gh-issue: 123446 +.. nonce: _I_mMr +.. section: Library + +Fix empty function name in :exc:`TypeError` when :func:`csv.reader`, +:func:`csv.writer`, or :func:`csv.register_dialect` are used without the +required args. + +.. + +.. date: 2024-08-28-20-08-19 +.. gh-issue: 123448 +.. nonce: tItJlp +.. section: Library + +Fixed memory leak of :class:`typing.NoDefault` by moving it to the static +types array. + +.. + +.. date: 2024-08-28-13-03-36 +.. gh-issue: 123409 +.. nonce: lW0YF- +.. section: Library + +Fix :attr:`ipaddress.IPv6Address.reverse_pointer` output according to +:rfc:`RFC 3596, §2.5 <3596#section-2.5>`. Patch by Bénédikt Tran. + +.. + +.. date: 2024-08-27-12-38-42 +.. gh-issue: 123089 +.. nonce: vA7iFR +.. section: Library + +Make :class:`weakref.WeakSet` safe against concurrent mutations while it is +being iterated. Patch by Kumar Aditya. + +.. + +.. date: 2024-08-27-12-11-00 +.. gh-issue: 123363 +.. nonce: gKuJp6 +.. section: Library + +Show string value of :opcode:`CONTAINS_OP` oparg in :mod:`dis` output. Patch +by Alexandr153. + +.. + +.. date: 2024-08-27-10-30-37 +.. gh-issue: 123341 +.. nonce: 5e-fjt +.. section: Library + +Add :meth:`~object.__class_getitem__` to :class:`!tkinter.Event` for type +subscript support at runtime. Patch by Adonis Rakateli. + +.. + +.. date: 2024-08-26-19-36-00 +.. gh-issue: 123340 +.. nonce: mQKI1H +.. section: Library + +Show string value of :opcode:`IS_OP` oparg in :mod:`dis` output. + +.. + +.. date: 2024-08-26-18-48-13 +.. gh-issue: 119518 +.. nonce: QFYH9q +.. section: Library + +Speed up normalization of :class:`pathlib.PurePath` and +:class:`~pathlib.Path` objects by not interning string parts. + +.. + +.. date: 2024-08-26-13-45-20 +.. gh-issue: 123270 +.. nonce: gXHvNJ +.. section: Library + +Applied a more surgical fix for malformed payloads in :class:`zipfile.Path` +causing infinite loops (gh-122905) without breaking contents using +legitimate characters. + +.. + +.. date: 2024-08-25-16-59-20 +.. gh-issue: 73991 +.. nonce: 1w8u3K +.. section: Library + +Add :meth:`pathlib.Path.copy_into` and :meth:`~pathlib.Path.move_into`, +which copy and move files and directories into *existing* directories. + +.. + +.. date: 2024-08-24-06-05-41 +.. gh-issue: 123228 +.. nonce: jR_5O5 +.. section: Library + +Fix return type for +:func:`!_pyrepl.readline._ReadlineWrapper.get_line_buffer` to be +:func:`str`. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-08-24-00-03-01 +.. gh-issue: 123240 +.. nonce: uFPG3l +.. section: Library + +Raise audit events for the :func:`input` in the new REPL. + +.. + +.. date: 2024-08-23-22-01-30 +.. gh-issue: 76960 +.. nonce: vsANPu +.. section: Library + +Fix :func:`urllib.parse.urljoin` and :func:`urllib.parse.urldefrag` for URIs +containing empty components. For example, :func:`!urljoin` with relative +reference "?" now sets empty query and removes fragment. Preserve empty +components (authority, params, query, fragment) in :func:`!urljoin`. +Preserve empty components (authority, params, query) in :func:`!urldefrag`. + +.. + +.. date: 2024-08-23-15-49-10 +.. gh-issue: 116810 +.. nonce: QLBUU8 +.. section: Library + +Resolve a memory leak introduced in CPython 3.10's :mod:`ssl` when the +:attr:`ssl.SSLSocket.session` property was accessed. Speeds up read and +write access to said property by no longer unnecessarily cloning session +objects via serialization. + +.. + +.. date: 2024-08-22-20-10-13 +.. gh-issue: 123243 +.. nonce: Kifj1L +.. section: Library + +Fix memory leak in :mod:`!_decimal`. + +.. + +.. date: 2024-08-22-11-25-19 +.. gh-issue: 122546 +.. nonce: BSmeE7 +.. section: Library + +Consistently use same file name for different exceptions in the new repl. +Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-08-22-09-37-48 +.. gh-issue: 123213 +.. nonce: owmXnP +.. section: Library + +:meth:`xml.etree.ElementTree.Element.extend` and +:class:`~xml.etree.ElementTree.Element` assignment no longer hide the +internal exception if an erroneous generator is passed. Patch by Bar Harel. + +.. + +.. date: 2024-08-20-18-02-27 +.. gh-issue: 85110 +.. nonce: 8_iDQy +.. section: Library + +Preserve relative path in URL without netloc in +:func:`urllib.parse.urlunsplit` and :func:`urllib.parse.urlunparse`. + +.. + +.. date: 2024-08-20-14-22-49 +.. gh-issue: 123165 +.. nonce: vOZZOA +.. section: Library + +Add support for rendering :class:`~dis.Positions` in :mod:`dis`. + +.. + +.. date: 2024-08-19-17-37-18 +.. gh-issue: 122909 +.. nonce: kP12SK +.. section: Library + +In urllib.request when URLError is raised opening an ftp URL, the exception +argument is now consistently a string. Earlier versions passed either a +string or an ftplib exception instance as the argument to URLError. + +.. + +.. date: 2024-08-18-08-25-32 +.. gh-issue: 123084 +.. nonce: rf8izX +.. section: Library + +Deprecate :class:`!shutil.ExecError`, which hasn't been raised by any +:mod:`shutil` function since Python 3.4. It's now an alias for +:exc:`RuntimeError`. + +.. + +.. date: 2024-08-17-08-17-20 +.. gh-issue: 123085 +.. nonce: 7Io2yH +.. section: Library + +In a bare call to :func:`importlib.resources.files`, ensure the caller's +frame is properly detected when ``importlib.resources`` is itself available +as a compiled module only (no source). + +.. + +.. date: 2024-08-16-19-13-21 +.. gh-issue: 123067 +.. nonce: Nx9O4R +.. section: Library + +Fix quadratic complexity in parsing ``"``-quoted cookie values with +backslashes by :mod:`http.cookies`. + +.. + +.. date: 2024-08-16-16-53-52 +.. gh-issue: 123049 +.. nonce: izx_fH +.. section: Library + +Add support for :const:`~configparser.UNNAMED_SECTION` in +:meth:`configparser.ConfigParser.add_section`. + +.. + +.. date: 2024-08-15-09-45-34 +.. gh-issue: 121735 +.. nonce: _1q0qf +.. section: Library + +When working with zip archives, importlib.resources now properly honors +module-adjacent references (e.g. ``files(pkg.mod)`` and not just +``files(pkg)``). + +.. + +.. date: 2024-08-14-10-41-11 +.. gh-issue: 122981 +.. nonce: BHV0Z9 +.. section: Library + +Fix :func:`inspect.getsource` for generated classes with Python base classes +(e.g. enums). + +.. + +.. date: 2024-08-11-14-23-07 +.. gh-issue: 122903 +.. nonce: xktZta +.. section: Library + +``zipfile.Path.glob`` now correctly matches directories instead of silently +omitting them. + +.. + +.. date: 2024-08-11-14-08-04 +.. gh-issue: 122905 +.. nonce: 7tDsxA +.. section: Library + +:class:`zipfile.Path` objects now sanitize names from the zipfile. + +.. + +.. date: 2024-08-10-14-16-59 +.. gh-issue: 122873 +.. nonce: XlHaUn +.. section: Library + +Enable :mod:`json` module to work as a script using the :option:`-m` switch: +``python -m json``. See the :ref:`JSON command-line interface +` documentation. Patch by Trey Hunner. + +.. + +.. date: 2024-08-10-10-21-44 +.. gh-issue: 122858 +.. nonce: ZC1rJD +.. section: Library + +Deprecate :func:`!asyncio.iscoroutinefunction` in favor of +:func:`inspect.iscoroutinefunction`. + +.. + +.. date: 2024-08-07-17-41-16 +.. gh-issue: 116263 +.. nonce: EcXir0 +.. section: Library + +:class:`logging.handlers.RotatingFileHandler` no longer rolls over empty log +files. + +.. + +.. date: 2024-08-07-14-12-19 +.. gh-issue: 105376 +.. nonce: QbGPdE +.. section: Library + +Restore the deprecated :mod:`logging` ``warn()`` method. It was removed in +Python 3.13 alpha 1. Keep the deprecated ``warn()`` method in Python 3.13. +Patch by Victor Stinner. + +.. + +.. date: 2024-08-07-11-57-41 +.. gh-issue: 122311 +.. nonce: LDExnJ +.. section: Library + +Improve errors in the :mod:`pickle` module. :exc:`~pickle.PicklingError` is +now raised more often instead of :exc:`UnicodeEncodeError`, +:exc:`ValueError` and :exc:`AttributeError`, and the original exception is +chained to it. Improve and unify error messages in Python and C +implementations. + +.. + +.. date: 2024-08-06-18-07-19 +.. gh-issue: 122744 +.. nonce: kCzNDI +.. section: Library + +Bump the version of pip bundled in ensurepip to version 24.2. + +.. + +.. date: 2024-08-06-10-36-55 +.. gh-issue: 118761 +.. nonce: q_x_1A +.. section: Library + +Improve import time of :mod:`pprint` by around seven times. Patch by Hugo +van Kemenade. + +.. + +.. date: 2024-08-06-07-24-00 +.. gh-issue: 118974 +.. nonce: qamsCQ +.. section: Library + +Add ``decorator`` parameter to :func:`dataclasses.make_dataclass` to +customize the functional creation of dataclasses. + +.. + +.. date: 2024-08-04-14-07-18 +.. gh-issue: 118814 +.. nonce: uiyks1 +.. section: Library + +Fix the :class:`typing.TypeVar` constructor when name is passed by keyword. + +.. + +.. date: 2024-08-03-06-51-08 +.. gh-issue: 122637 +.. nonce: gpas8J +.. section: Library + +Adjust ``cmath.tanh(nanj)`` and ``cmath.tanh(infj)`` for recent C standards. + +.. + +.. date: 2024-07-31-20-43-21 +.. gh-issue: 122478 +.. nonce: sCU2Le +.. section: Library + +Remove internal frames from tracebacks shown in +:class:`code.InteractiveInterpreter` with non-default +:func:`sys.excepthook`. Save correct tracebacks in +:attr:`sys.last_traceback` and update ``__traceback__`` attribute of +:attr:`sys.last_value` and :attr:`sys.last_exc`. + +.. + +.. date: 2024-07-31-15-08-42 +.. gh-issue: 116622 +.. nonce: aKxIQA +.. section: Library + +On Android, the ``FICLONE`` and ``FICLONERANGE`` constants are no longer +exposed by :mod:`fcntl`, as these ioctls are blocked by SELinux. + +.. + +.. date: 2024-07-31-14-55-41 +.. gh-issue: 82378 +.. nonce: eZvYmR +.. section: Library + +Make sure that the new :term:`REPL` interprets :data:`sys.tracebacklimit` in +the same way that the classic REPL did. + +.. + +.. date: 2024-07-30-21-29-30 +.. gh-issue: 122334 +.. nonce: LeoE1x +.. section: Library + +Fix crash when importing :mod:`ssl` after the main interpreter restarts. + +.. + +.. date: 2024-07-30-15-57-07 +.. gh-issue: 122459 +.. nonce: AYIoeN +.. section: Library + +Optimize :mod:`pickling ` by name objects without the ``__module__`` +attribute. + +.. + +.. date: 2024-07-30-14-46-16 +.. gh-issue: 87320 +.. nonce: -Yk1wb +.. section: Library + +In :class:`code.InteractiveInterpreter`, handle exceptions caused by calling +a non-default :func:`sys.excepthook`. Before, the exception bubbled up to +the caller, ending the :term:`REPL`. + +.. + +.. date: 2024-07-30-04-27-55 +.. gh-issue: 122272 +.. nonce: 6Wwa1V +.. section: Library + +On some platforms such as Linux, year with century was not 0-padded when +formatted by :meth:`~.datetime.strftime` with C99-specific specifiers +``'%C'`` or ``'%F'``. The 0-padding behavior is now guaranteed when the +format specifiers ``'%C'`` and ``'%F'`` are supported by the C library. +Patch by Ben Hsing + +.. + +.. date: 2024-07-29-16-47-08 +.. gh-issue: 122400 +.. nonce: fM0YSv +.. section: Library + +Handle :exc:`ValueError`\s raised by :func:`os.stat` in +:class:`filecmp.dircmp` and :func:`filecmp.cmpfiles`. Patch by Bénédikt +Tran. + +.. + +.. date: 2024-07-29-10-24-48 +.. gh-issue: 122311 +.. nonce: xChV1b +.. section: Library + +Fix some error messages in :mod:`pickle`. + +.. + +.. date: 2024-07-27-16-10-41 +.. gh-issue: 121650 +.. nonce: nf6oc9 +.. section: Library + +:mod:`email` headers with embedded newlines are now quoted on output. The +:mod:`~email.generator` will now refuse to serialize (write) headers that +are unsafely folded or delimited; see +:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas +Bloemsaat and Petr Viktorin in :gh:`121650`.) + +.. + +.. date: 2024-07-26-21-21-13 +.. gh-issue: 122332 +.. nonce: fvw88r +.. section: Library + +Fixed segfault with :meth:`asyncio.Task.get_coro` when using an eager task +factory. + +.. + +.. date: 2024-07-25-15-41-14 +.. gh-issue: 105733 +.. nonce: o3koJA +.. section: Library + +:func:`ctypes.ARRAY` is now :term:`soft deprecated`: it no longer emits +deprecation warnings and is not scheduled for removal. + +.. + +.. date: 2024-07-24-08-48-22 +.. gh-issue: 122213 +.. nonce: o3pdgA +.. section: Library + +Add notes for pickle serialization errors that allow to identify the source +of the error. + +.. + +.. date: 2024-07-23-22-26-00 +.. gh-issue: 119180 +.. nonce: B2IVT8 +.. section: Library + +As part of :pep:`749`, add the following attributes for customizing +evaluation of annotation scopes: + +* ``evaluate_value`` on :class:`typing.TypeAliasType` +* ``evaluate_bound``, ``evaluate_constraints``, and ``evaluate_default`` on :class:`typing.TypeVar` +* ``evaluate_default`` on :class:`typing.ParamSpec` +* ``evaluate_default`` on :class:`typing.TypeVarTuple` + +.. + +.. date: 2024-07-23-17-13-10 +.. gh-issue: 119180 +.. nonce: 5PZELo +.. section: Library + +Fix handling of classes with custom metaclasses in +``annotationlib.get_annotations``. + +.. + +.. date: 2024-07-23-15-30-23 +.. gh-issue: 122170 +.. nonce: Z9gi3Y +.. section: Library + +Handle :exc:`ValueError`\s raised by :func:`os.stat` in :mod:`linecache`. +Patch by Bénédikt Tran. + +.. + +.. date: 2024-07-23-15-11-13 +.. gh-issue: 122163 +.. nonce: 4wRUuM +.. section: Library + +Add notes for JSON serialization errors that allow to identify the source of +the error. + +.. + +.. date: 2024-07-23-13-07-12 +.. gh-issue: 122129 +.. nonce: PwbC8q +.. section: Library + +Improve support of method descriptors and wrappers in the help title. + +.. + +.. date: 2024-07-23-12-38-14 +.. gh-issue: 122145 +.. nonce: sTO8nX +.. section: Library + +Fix an issue when reporting tracebacks corresponding to Python code emitting +an empty AST body. Patch by Nikita Sobolev and Bénédikt Tran. + +.. + +.. date: 2024-07-23-10-59-38 +.. gh-issue: 121723 +.. nonce: iJEf7e +.. section: Library + +Make :func:`logging.config.dictConfig` accept any object implementing the +Queue public API. See the :ref:`queue configuration ` +section for details. Patch by Bénédikt Tran. + +.. + +.. date: 2024-07-23-09-14-44 +.. gh-issue: 82951 +.. nonce: -F5p5A +.. section: Library + +Serializing objects with complex ``__qualname__`` (such as unbound methods +and nested classes) by name no longer involves serializing parent objects by +value in pickle protocols < 4. + +.. + +.. date: 2024-07-22-08-57-28 +.. gh-issue: 120754 +.. nonce: Eo5puP +.. section: Library + +``Pathlib.read_bytes`` no longer opens the file in Python's buffered I/O +mode. This reduces overheads as the code reads a file in whole leading to a +modest speedup. + +.. + +.. date: 2024-07-22-08-14-04 +.. gh-issue: 113785 +.. nonce: 6B_KNB +.. section: Library + +:mod:`csv` now correctly parses numeric fields (when used with +:const:`csv.QUOTE_NONNUMERIC` or :const:`csv.QUOTE_STRINGS`) which start +with an escape character. + +.. + +.. date: 2024-07-21-18-03-30 +.. gh-issue: 122088 +.. nonce: vi2bP- +.. section: Library + +:func:`@warnings.deprecated ` now copies the coroutine +status of functions and methods so that :func:`inspect.iscoroutinefunction` +returns the correct result. + +.. + +.. date: 2024-07-21-10-45-24 +.. gh-issue: 122081 +.. nonce: dNrYMq +.. section: Library + +Fix a crash in the :func:`!decimal.IEEEContext` optional function available +via the ``EXTRA_FUNCTIONALITY`` configuration flag. + +.. + +.. date: 2024-07-21-02-00-46 +.. gh-issue: 73991 +.. nonce: pLxdtJ +.. section: Library + +Add :meth:`pathlib.Path.move`, which moves a file or directory tree. + +.. + +.. date: 2024-07-17-12-55-22 +.. gh-issue: 121268 +.. nonce: 41RmjR +.. section: Library + +Remove workarounds for non-IEEE 754 systems in :mod:`cmath`. + +.. + +.. date: 2024-07-17-09-44-35 +.. gh-issue: 119698 +.. nonce: WlygzR +.. section: Library + +Due to the lack of interest for :meth:`symtable.Class.get_methods`, the +method is marked as deprecated and will be removed in Python 3.16. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-07-17-09-23-03 +.. gh-issue: 121889 +.. nonce: 6se9jS +.. section: Library + +Adjusts ``cmath.acosh(complex('0+nanj'))`` for recent C standards. + +.. + +.. date: 2024-07-16-20-49-07 +.. gh-issue: 121804 +.. nonce: gYN-In +.. section: Library + +Correctly show error locations, when :exc:`SyntaxError` raised in new repl. +Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-07-15-19-34-56 +.. gh-issue: 121797 +.. nonce: qDqj59 +.. section: Library + +Add alternative :class:`~fractions.Fraction` constructor +:meth:`Fraction.from_number() `. + +.. + +.. date: 2024-07-15-19-25-25 +.. gh-issue: 121798 +.. nonce: GmuBDu +.. section: Library + +Add alternative :class:`~decimal.Decimal` constructor +:meth:`Decimal.from_number() `. + +.. + +.. date: 2024-07-14-11-18-28 +.. gh-issue: 120930 +.. nonce: Kuo4L0 +.. section: Library + +Fixed a bug introduced by gh-92081 that added an incorrect extra blank to +encoded words occurring in wrapped headers. + +.. + +.. date: 2024-07-14-06-24-02 +.. gh-issue: 57141 +.. nonce: C3jhDh +.. section: Library + +The *shallow* argument to :class:`filecmp.dircmp` (new in Python 3.13) is +now keyword-only. + +.. + +.. date: 2024-07-13-06-23-24 +.. gh-issue: 121245 +.. nonce: RfOgf4 +.. section: Library + +Simplify handling of the history file in ``site.register_readline()`` +helper. The ``CAN_USE_PYREPL`` variable now will be initialized, when +imported. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-07-10-08-13-34 +.. gh-issue: 121249 +.. nonce: W9Gd09 +.. section: Library + +Support the :c:expr:`float complex` and :c:expr:`double complex` C types in +the :mod:`struct` module if the compiler has C11 complex arithmetic. Patch +by Sergey B Kirpichev. + +.. + +.. date: 2024-07-09-12-23-32 +.. gh-issue: 121486 +.. nonce: Iultjh +.. section: Library + +:mod:`math` functions :func:`~math.isqrt`, :func:`~math.log`, +:func:`~math.log2` and :func:`~math.log10` now support integers larger than +``2**2**32`` on 32-bit platforms. + +.. + +.. date: 2024-07-08-03-45-34 +.. gh-issue: 121474 +.. nonce: NsvrUN +.. section: Library + +Fix missing sanity check for ``parties`` arg in :class:`threading.Barrier` +constructor. Patch by Clinton Christian (pygeek). + +.. + +.. date: 2024-07-06-23-39-38 +.. gh-issue: 121450 +.. nonce: vGqb3c +.. section: Library + +Hard-coded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now +reuse the most recent ``Pdb`` instance that calls ``Pdb.set_trace()``, +instead of creating a new one each time. As a result, all the instance +specific data like ``display`` and ``commands`` are preserved across +Hard-coded breakpoints. + +.. + +.. date: 2024-07-06-16-08-39 +.. gh-issue: 119169 +.. nonce: o0YymL +.. section: Library + +Slightly speed up :func:`os.walk` by simplifying exception handling. + +.. + +.. date: 2024-07-06-12-37-10 +.. gh-issue: 121423 +.. nonce: vnxrl4 +.. section: Library + +Improve import time of :mod:`socket` by lazy importing modules and writing +:data:`!socket.errorTab` as a constant. + +.. + +.. date: 2024-07-04-17-36-03 +.. gh-issue: 59110 +.. nonce: IlI9Fz +.. section: Library + +:mod:`zipimport` supports now namespace packages when no directory entry +exists. + +.. + +.. date: 2024-07-03-14-23-04 +.. gh-issue: 119004 +.. nonce: L5MoUu +.. section: Library + +Fix a crash in :ref:`OrderedDict.__eq__ ` +when operands are mutated during the check. Patch by Bénédikt Tran. + +.. + +.. date: 2024-07-03-10-11-53 +.. gh-issue: 121313 +.. nonce: D7gARW +.. section: Library + +Limit the reading size in the :class:`multiprocessing.connection.Connection` +class to 64 KiB to prevent memory overallocation and unnecessary memory +management system calls. + +.. + +.. date: 2024-07-03-07-25-21 +.. gh-issue: 121332 +.. nonce: Iz6FEq +.. section: Library + +Fix constructor of :mod:`ast` nodes with custom ``_attributes``. Previously, +passing custom attributes would raise a :py:exc:`DeprecationWarning`. +Passing arguments to the constructor that are not in ``_fields`` or +``_attributes`` remains deprecated. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-07-02-11-34-06 +.. gh-issue: 121245 +.. nonce: sSkDAr +.. section: Library + +Fix a bug in the handling of the command history of the new :term:`REPL` +that caused the history file to be wiped at REPL exit. + +.. + +.. date: 2024-07-01-11-23-18 +.. gh-issue: 121210 +.. nonce: cD0zfn +.. section: Library + +Handle AST nodes with missing runtime fields or attributes in +:func:`ast.compare`. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-29-19-30-15 +.. gh-issue: 121163 +.. nonce: SJKDFq +.. section: Library + +Add support for ``all`` as an valid ``action`` for +:func:`warnings.simplefilter` and :func:`warnings.filterwarnings`. + +.. + +.. date: 2024-06-29-15-23-26 +.. gh-issue: 121151 +.. nonce: HeLEvq +.. section: Library + +Fix wrapping of long usage text of arguments inside a mutually exclusive +group in :mod:`argparse`. + +.. + +.. date: 2024-06-29-15-21-12 +.. gh-issue: 121141 +.. nonce: 4evD6q +.. section: Library + +Add support for :func:`copy.replace` to AST nodes. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-29-05-08-59 +.. gh-issue: 87744 +.. nonce: rpF6Jw +.. section: Library + +Fix waitpid race while calling +:meth:`~asyncio.subprocess.Process.send_signal` in asyncio. Patch by Kumar +Aditya. + +.. + +.. date: 2024-06-27-13-47-14 +.. gh-issue: 121027 +.. nonce: jh55EC +.. section: Library + +Add a future warning in :meth:`!functools.partial.__get__`. In future Python +versions :class:`functools.partial` will be a method descriptor. + +.. + +.. date: 2024-06-27-12-27-52 +.. gh-issue: 121027 +.. nonce: D4K1OX +.. section: Library + +Make the :class:`functools.partial` object a method descriptor. + +.. + +.. date: 2024-06-26-17-00-39 +.. gh-issue: 117784 +.. nonce: inCtAV +.. section: Library + +CPython now detects whether its linked TLS library supports TLSv1.3 +post-handshake authentication and disables that feature if support is +lacking. + +.. + +.. date: 2024-06-26-10-13-40 +.. gh-issue: 121025 +.. nonce: M-XXlV +.. section: Library + +Improve the :meth:`~object.__repr__` of :class:`functools.partialmethod`. +Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-26-03-04-24 +.. gh-issue: 121018 +.. nonce: clVSc4 +.. section: Library + +Fixed issues where :meth:`!argparse.ArgumentParser.parse_args` did not honor +``exit_on_error=False``. Based on patch by Ben Hsing. + +.. + +.. date: 2024-06-23-17-50-40 +.. gh-issue: 119614 +.. nonce: vwPGLB +.. section: Library + +Fix truncation of strings with embedded null characters in some internal +operations in :mod:`tkinter`. + +.. + +.. date: 2024-06-23-11-21-27 +.. gh-issue: 120910 +.. nonce: t0QXdB +.. section: Library + +When reading installed files from an egg, use ``relative_to(walk_up=True)`` +to honor files installed outside of the installation root. + +.. + +.. date: 2024-06-23-07-23-08 +.. gh-issue: 61103 +.. nonce: ca_U_l +.. section: Library + +Support :c:expr:`float complex`, :c:expr:`double complex` and :c:expr:`long +double complex` C types in :mod:`ctypes` as +:class:`~ctypes.c_float_complex`, :class:`~ctypes.c_double_complex` and +:class:`~ctypes.c_longdouble_complex` if the compiler has C11 complex +arithmetic. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-06-22-22-52-24 +.. gh-issue: 120888 +.. nonce: sd8I3N +.. section: Library + +Upgrade pip wheel bundled with ensurepip (pip 24.1.1) + +.. + +.. date: 2024-06-22-22-23-56 +.. gh-issue: 101830 +.. nonce: 1BAoxH +.. section: Library + +Accessing the :mod:`tkinter` object's string representation no longer +converts the underlying Tcl object to a string on Windows. + +.. + +.. date: 2024-06-22-17-01-56 +.. gh-issue: 120678 +.. nonce: Ik8dCg +.. section: Library + +Fix regression in the new REPL that meant that globals from files passed +using the ``-i`` argument would not be included in the REPL's global +namespace. Patch by Alex Waygood. + +.. + +.. date: 2024-06-21-14-32-56 +.. gh-issue: 120811 +.. nonce: eBmVTV +.. section: Library + +Fix possible memory leak in :meth:`contextvars.Context.run`. + +.. + +.. date: 2024-06-21-12-00-16 +.. gh-issue: 120782 +.. nonce: LOE8tj +.. section: Library + +Fix wrong references of the :mod:`datetime` types after reloading the +module. + +.. + +.. date: 2024-06-21-06-37-46 +.. gh-issue: 120713 +.. nonce: WBbQx4 +.. section: Library + +:meth:`datetime.datetime.strftime` now 0-pads years with less than four +digits for the format specifiers ``%Y`` and ``%G`` on Linux. Patch by Ben +Hsing + +.. + +.. date: 2024-06-20-01-31-24 +.. gh-issue: 120769 +.. nonce: PfiMrc +.. section: Library + +Make empty line in :mod:`pdb` repeats the last command even when the command +is from ``cmdqueue``. + +.. + +.. date: 2024-06-19-23-08-25 +.. gh-issue: 120780 +.. nonce: 0Omopb +.. section: Library + +Show string value of LOAD_SPECIAL oparg in :mod:`dis` output. + +.. + +.. date: 2024-06-19-19-53-42 +.. gh-issue: 41431 +.. nonce: gnkUc5 +.. section: Library + +Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. +Contributed by Wannes Boeykens. + +.. + +.. date: 2024-06-19-15-43-04 +.. gh-issue: 120743 +.. nonce: CMMl2P +.. section: Library + +:term:`Soft deprecate ` :func:`os.popen` and +:func:`os.spawn* ` functions. They should no longer be used to +write new code. The :mod:`subprocess` module is recommended instead. Patch +by Victor Stinner. + +.. + +.. date: 2024-06-19-15-06-58 +.. gh-issue: 120732 +.. nonce: OvYV9b +.. section: Library + +Fix ``name`` passing to :class:`unittest.mock.Mock` object when using +:func:`unittest.mock.create_autospec`. + +.. + +.. date: 2024-06-19-13-20-01 +.. gh-issue: 111259 +.. nonce: Wki5PV +.. section: Library + +:mod:`re` now handles patterns like ``"[\s\S]"`` or ``"\s|\S"`` which match +any character as effectively as a dot with the ``DOTALL`` modifier +(``"(?s:.)"``). + +.. + +.. date: 2024-06-18-19-18-10 +.. gh-issue: 120683 +.. nonce: xmRez7 +.. section: Library + +Fix an error in :class:`logging.LogRecord`, when the integer part of the +timestamp is rounded up, while the millisecond calculation truncates, +causing the log timestamp to be wrong by up to 999 ms (affected roughly 1 in +8 million timestamps). + +.. + +.. date: 2024-06-18-14-45-38 +.. gh-issue: 118710 +.. nonce: 5GZZPX +.. section: Library + +:class:`ipaddress.IPv4Address` and :class:`ipaddress.IPv6Address` attributes +``version`` and ``max_prefixlen`` are now available on the class. + +.. + +.. date: 2024-06-17-20-04-13 +.. gh-issue: 120633 +.. nonce: kZC5wt +.. section: Library + +Move scrollbar and remove tear-off menus in turtledemo. + +.. + +.. date: 2024-06-16-21-33-56 +.. gh-issue: 120606 +.. nonce: kugbwR +.. section: Library + +Allow users to use EOF to exit ``commands`` definition in :mod:`pdb` + +.. + +.. date: 2024-06-15-23-38-36 +.. gh-issue: 120284 +.. nonce: HwsAtY +.. section: Library + +Allow :meth:`asyncio.Runner.run` to accept :term:`awaitable` objects instead +of simply :term:`coroutine`\s. + +.. + +.. date: 2024-06-15-12-04-46 +.. gh-issue: 120541 +.. nonce: d3cc5y +.. section: Library + +Improve the prompt in the "less" pager when :func:`help` is called with +non-string argument. + +.. + +.. date: 2024-06-14-20-05-25 +.. gh-issue: 120495 +.. nonce: OxgZKB +.. section: Library + +Fix incorrect exception handling in Tab Nanny. Patch by Wulian233. + +.. + +.. date: 2024-06-12-15-07-58 +.. gh-issue: 120388 +.. nonce: VuTQMT +.. section: Library + +Improve a warning message when a test method in :mod:`unittest` returns +something other than ``None``. Now we show the returned object type and +optional asyncio-related tip. + +.. + +.. date: 2024-06-12-11-54-05 +.. gh-issue: 120381 +.. nonce: O-BNLs +.. section: Library + +Correct :func:`inspect.ismethoddescriptor` to check also for the lack of +:meth:`~object.__delete__`. Patch by Jan Kaliszewski. + +.. + +.. date: 2024-06-12-10-00-31 +.. gh-issue: 90425 +.. nonce: 5CfkKG +.. section: Library + +The OS byte in gzip headers is now always set to 255 when using +:func:`gzip.compress`. + +.. + +.. date: 2024-06-11-16-34-41 +.. gh-issue: 120343 +.. nonce: hdiXeU +.. section: Library + +Fix column offset reporting for tokens that come after multiline f-strings +in the :mod:`tokenize` module. + +.. + +.. date: 2024-06-11-07-17-25 +.. gh-issue: 119180 +.. nonce: iH-2zy +.. section: Library + +As part of implementing :pep:`649` and :pep:`749`, add a new module +``annotationlib``. Add support for unresolved forward references in +annotations to :mod:`dataclasses`, :class:`typing.TypedDict`, and +:class:`typing.NamedTuple`. + +.. + +.. date: 2024-06-10-14-00-40 +.. gh-issue: 119600 +.. nonce: jJMf4C +.. section: Library + +Fix :func:`unittest.mock.patch` to not read attributes of the target when +``new_callable`` is set. Patch by Robert Collins. + +.. + +.. date: 2024-06-09-19-53-11 +.. gh-issue: 120289 +.. nonce: s4HXR0 +.. section: Library + +Fixed the use-after-free issue in :mod:`cProfile` by disallowing +``disable()`` and ``clear()`` in external timers. + +.. + +.. date: 2024-06-08-17-41-11 +.. gh-issue: 82017 +.. nonce: WpSTGi +.. section: Library + +Added support for converting any objects that have the +:meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. + +.. + +.. date: 2024-06-08-15-46-35 +.. gh-issue: 114053 +.. nonce: Ub2XgJ +.. section: Library + +Fix edge-case bug where :func:`typing.get_type_hints` would produce +incorrect results if type parameters in a class scope were overridden by +assignments in a class scope and ``from __future__ import annotations`` +semantics were enabled. Patch by Alex Waygood. + +.. + +.. date: 2024-06-08-15-15-29 +.. gh-issue: 114053 +.. nonce: WQLAFG +.. section: Library + +Fix erroneous :exc:`NameError` when calling :func:`inspect.get_annotations` +with ``eval_str=True``` on a class that made use of :pep:`695` type +parameters in a module that had ``from __future__ import annotations`` at +the top of the file. Patch by Alex Waygood. + +.. + +.. date: 2024-06-08-14-36-40 +.. gh-issue: 120268 +.. nonce: MNpd1q +.. section: Library + +Prohibit passing ``None`` to pure-Python :meth:`datetime.date.fromtimestamp` +to achieve consistency with C-extension implementation. + +.. + +.. date: 2024-06-08-09-45-31 +.. gh-issue: 120244 +.. nonce: 8o9Dzr +.. section: Library + +Fix memory leak in :func:`re.sub` when the replacement string contains +backreferences. + +.. + +.. date: 2024-06-08-03-29-01 +.. gh-issue: 120254 +.. nonce: h682ke +.. section: Library + +Added ``commands`` argument to :func:`pdb.set_trace` which allows users to +send debugger commands from the source file. + +.. + +.. date: 2024-06-07-13-21-11 +.. gh-issue: 120211 +.. nonce: Rws_gf +.. section: Library + +Fix :mod:`tkinter.ttk` with Tcl/Tk 9.0. + +.. + +.. date: 2024-06-07-11-23-31 +.. gh-issue: 71587 +.. nonce: IjFajE +.. section: Library + +Fix crash in C version of :meth:`datetime.datetime.strptime` when called +again on the restarted interpreter. + +.. + +.. date: 2024-06-07-10-10-32 +.. gh-issue: 117983 +.. nonce: NeMR9n +.. section: Library + +Defer the ``threading`` import in ``importlib.util`` until lazy loading is +used. + +.. + +.. date: 2024-06-07-02-00-31 +.. gh-issue: 120157 +.. nonce: HnWcF9 +.. section: Library + +Remove unused constant ``concurrent.futures._base._FUTURE_STATES`` in +:mod:`concurrent.futures`. Patch by Clinton Christian (pygeek). + +.. + +.. date: 2024-06-06-17-24-43 +.. gh-issue: 120161 +.. nonce: DahNXV +.. section: Library + +:mod:`datetime` no longer crashes in certain complex reference cycle +situations. + +.. + +.. date: 2024-06-06-12-07-57 +.. gh-issue: 119698 +.. nonce: rRrprk +.. section: Library + +Fix :meth:`symtable.Class.get_methods` and document its behaviour. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-06-05-16-30-28 +.. gh-issue: 120121 +.. nonce: 9dz8i7 +.. section: Library + +Add :exc:`concurrent.futures.InvalidStateError` to module's ``__all__``. + +.. + +.. date: 2024-06-05-11-39-21 +.. gh-issue: 119933 +.. nonce: ooJXQV +.. section: Library + +Add the :class:`symtable.SymbolTableType` enumeration to represent the +possible outputs of the :class:`symtable.SymbolTable.get_type` method. Patch +by Bénédikt Tran. + +.. + +.. date: 2024-06-05-11-03-10 +.. gh-issue: 120029 +.. nonce: QBsw47 +.. section: Library + +Expose :class:`symtable.Symbol` methods +:meth:`~symtable.Symbol.is_free_class`, +:meth:`~symtable.Symbol.is_comp_iter` and +:meth:`~symtable.Symbol.is_comp_cell`. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-05-08-02-46 +.. gh-issue: 120108 +.. nonce: 4U9BL8 +.. section: Library + +Fix calling :func:`copy.deepcopy` on :mod:`ast` trees that have been +modified to have references to parent nodes. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-06-04-19-49-16 +.. gh-issue: 120056 +.. nonce: 5aqozw +.. section: Library + +Add :data:`!socket.IP_RECVERR` and :data:`!socket.IP_RECVTTL` constants +(both available since Linux 2.2). And :data:`!socket.IP_RECVORIGDSTADDR` +constant (available since Linux 2.6.29). + +.. + +.. date: 2024-06-04-19-03-25 +.. gh-issue: 112672 +.. nonce: K2XfZH +.. section: Library + +Support building :mod:`tkinter` with Tcl 9.0. + +.. + +.. date: 2024-06-04-18-53-10 +.. gh-issue: 120057 +.. nonce: RSD9_Z +.. section: Library + +Added the :data:`os.environ.refresh() ` method to update +:data:`os.environ` with changes to the environment made by +:func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the +same process. Patch by Victor Stinner. + +.. + +.. date: 2024-06-04-14-54-46 +.. gh-issue: 120029 +.. nonce: _1YdTf +.. section: Library + +Expose :meth:`symtable.Symbol.is_type_parameter` in the :mod:`symtable` +module. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-04-12-23-01 +.. gh-issue: 119819 +.. nonce: WKKrYh +.. section: Library + +Fix regression to allow logging configuration with multiprocessing queue +types. + +.. + +.. date: 2024-06-04-08-57-02 +.. gh-issue: 65454 +.. nonce: o9j4wF +.. section: Library + +:func:`unittest.mock.Mock.attach_mock` no longer triggers a call to a +``PropertyMock`` being attached. + +.. + +.. date: 2024-06-03-11-18-16 +.. gh-issue: 117142 +.. nonce: kWTXQo +.. section: Library + +The :mod:`ctypes` module may now be imported in all subinterpreters, +including those that have their own GIL. + +.. + +.. date: 2024-06-02-15-09-17 +.. gh-issue: 118835 +.. nonce: KUAuz6 +.. section: Library + +Fix _pyrepl crash when using custom prompt with ANSI escape codes. + +.. + +.. date: 2024-06-02-13-35-11 +.. gh-issue: 81936 +.. nonce: ETeW9x +.. section: Library + +:meth:`!help` and :meth:`!showtopic` methods now respect a configured +*output* argument to :class:`!pydoc.Helper` and not use the pager in such +cases. Patch by Enrico Tröger. + +.. + +.. date: 2024-06-01-16-58-43 +.. gh-issue: 117398 +.. nonce: kR0RW7 +.. section: Library + +The ``_datetime`` module (C implementation for :mod:`datetime`) now supports +being imported in multiple interpreters. + +.. + +.. date: 2024-05-31-21-17-43 +.. gh-issue: 119824 +.. nonce: CQlxWV +.. section: Library + +Print stack entry in :mod:`pdb` when and only when user input is needed. + +.. + +.. date: 2024-05-31-13-56-21 +.. gh-issue: 119838 +.. nonce: H6XHlE +.. section: Library + +In mixed arithmetic operations with :class:`~fractions.Fraction` and +complex, the fraction is now converted to :class:`float` instead of +:class:`complex`. + +.. + +.. date: 2024-05-31-12-57-31 +.. gh-issue: 119770 +.. nonce: NCtels +.. section: Library + +Make :mod:`termios` ``ioctl()`` constants positive. Patch by Victor Stinner. + +.. + +.. date: 2024-05-30-21-37-05 +.. gh-issue: 89727 +.. nonce: D6S9ig +.. section: Library + +Fix issue with :func:`shutil.rmtree` where a :exc:`RecursionError` is raised +on deep directory trees. + +.. + +.. date: 2024-05-29-21-50-05 +.. gh-issue: 119577 +.. nonce: S3BlKJ +.. section: Library + +The :exc:`DeprecationWarning` emitted when testing the truth value of an +:class:`xml.etree.ElementTree.Element` now describes unconditionally +returning ``True`` in a future version rather than raising an exception in +Python 3.14. + +.. + +.. date: 2024-05-29-20-42-17 +.. gh-issue: 89727 +.. nonce: 5lPTTW +.. section: Library + +Partially fix issue with :func:`shutil.rmtree` where a :exc:`RecursionError` +is raised on deep directory trees. A recursion error is no longer raised +when :data:`!rmtree.avoids_symlink_attacks` is false. + +.. + +.. date: 2024-05-29-12-42-40 +.. gh-issue: 93963 +.. nonce: cb1oJS +.. section: Library + +Remove deprecated names from ``importlib.abc`` as found in +``importlib.resources.abc``. + +.. + +.. date: 2024-05-28-12-15-03 +.. gh-issue: 119118 +.. nonce: FMKz1F +.. section: Library + +Fix performance regression in the :mod:`tokenize` module by caching the +``line`` token attribute and calculating the column offset more efficiently. + +.. + +.. date: 2024-05-28-00-56-59 +.. gh-issue: 89727 +.. nonce: _bxoL3 +.. section: Library + +Fix issue with :func:`os.fwalk` where a :exc:`RecursionError` was raised on +deep directory trees by adjusting the implementation to be iterative instead +of recursive. + +.. + +.. date: 2024-05-26-22-22-51 +.. gh-issue: 119594 +.. nonce: fnQNM8 +.. section: Library + +If one calls pow(fractions.Fraction, x, module) with modulo not None, the +error message now says that the types are incompatible rather than saying +pow only takes 2 arguments. Patch by Wim Jeantine-Glenn and Mark Dickinson. + +.. + +.. date: 2024-05-26-21-28-11 +.. gh-issue: 119588 +.. nonce: wlLBK5 +.. section: Library + +``zipfile.Path.is_symlink`` now assesses if the given path is a symlink. + +.. + +.. date: 2024-05-25-20-20-42 +.. gh-issue: 119562 +.. nonce: DyplWc +.. section: Library + +Remove :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, +:class:`!ast.NameConstant` and :class:`!ast.Ellipsis`. They had all emitted +deprecation warnings since Python 3.12. Patch by Alex Waygood. + +.. + +.. date: 2024-05-25-20-15-26 +.. gh-issue: 119555 +.. nonce: mvHbEL +.. section: Library + +Catch :exc:`SyntaxError` from :func:`compile` in the runsource() method of +the InteractiveColoredConsole. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-05-25-10-40-38 +.. gh-issue: 118908 +.. nonce: XcZiq4 +.. section: Library + +Limit exposed globals from internal imports and definitions on new REPL +startup. Patch by Eugene Triguba and Pablo Galindo. + +.. + +.. date: 2024-05-25-07-25-07 +.. gh-issue: 117865 +.. nonce: 1A0Xpi +.. section: Library + +Improve the import time of the :mod:`ast` module by deferring the import of +:mod:`re`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-25-00-54-26 +.. gh-issue: 119127 +.. nonce: LpPvag +.. section: Library + +Positional arguments of :func:`functools.partial` objects now support +placeholders via :data:`functools.Placeholder`. + +.. + +.. date: 2024-05-24-21-54-55 +.. gh-issue: 113892 +.. nonce: JKDFqq +.. section: Library + +Now, the method ``sock_connect`` of :class:`asyncio.ProactorEventLoop` +raises a :exc:`ValueError` if given socket is not in non-blocking mode, as +well as in other loop implementations. + +.. + +.. date: 2024-05-24-14-32-24 +.. gh-issue: 119506 +.. nonce: -nMNqq +.. section: Library + +Fix :meth:`!io.TextIOWrapper.write` method breaks internal buffer when the +method is called again during flushing internal buffer. + +.. + +.. date: 2024-05-24-11-47-08 +.. gh-issue: 69214 +.. nonce: Grl6zF +.. section: Library + +Fix ``fcntl.ioctl()`` *request* parameter: use an ``unsigned long`` instead +of an ``unsigned int`` for the *request* parameter of :func:`fcntl.ioctl` to +support requests larger than ``UINT_MAX``. Patch by Victor Stinner. + +.. + +.. date: 2024-05-24-04-05-37 +.. gh-issue: 119105 +.. nonce: aDSRFn +.. section: Library + +``difflib``'s ``DIffer.compare()`` (and so also ``ndiff``) can no longer be +provoked into cubic-time behavior, or into unbounded recursion, and should +generally be faster in ordinary cases too. Results may change in some cases, +although that should be rare. Correctness of diffs is not affected. Some +similar lines far apart may be reported as deleting one and adding the +other, where before they were displayed on adjacent output lines with markup +showing the intraline differences. + +.. + +.. date: 2024-05-23-22-29-59 +.. gh-issue: 119443 +.. nonce: KAGz6S +.. section: Library + +The interactive REPL no longer runs with ``from __future__ import +annotations`` enabled. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-23-15-48-17 +.. gh-issue: 119461 +.. nonce: 82KqUW +.. section: Library + +Add ``socket.VMADDR_CID_LOCAL`` constant. Patch by Victor Stinner. + +.. + +.. date: 2024-05-23-11-52-36 +.. gh-issue: 117398 +.. nonce: 2FG1Mk +.. section: Library + +Objects in the datetime C-API are now all statically allocated, which means +better memory safety, especially when the module is reloaded. This should be +transparent to users. + +.. + +.. date: 2024-05-22-21-20-43 +.. gh-issue: 118894 +.. nonce: xHdxR_ +.. section: Library + +:mod:`asyncio` REPL now has the same capabilities as PyREPL. + +.. + +.. date: 2024-05-21-23-39-22 +.. gh-issue: 118830 +.. nonce: YTqvEo +.. section: Library + +Bump :mod:`pickle` default protocol to ``5``. + +.. + +.. date: 2024-05-21-20-13-23 +.. gh-issue: 118911 +.. nonce: iG8nMq +.. section: Library + +In PyREPL, updated ``maybe-accept``'s logic so that if the user hits +:kbd:`Enter` twice, they are able to terminate the block even if there's +trailing whitespace. Also, now when the user hits arrow up, the cursor is on +the last functional line. This matches IPython's behavior. Patch by Aya +Elsayed. + +.. + +.. date: 2024-05-21-19-10-30 +.. gh-issue: 115225 +.. nonce: eRmfJH +.. section: Library + +Raise error on certain technically valid but pathological ISO 8601 strings +passed to :meth:`datetime.time.fromisoformat` that were previously parsed +incorrectly. + +.. + +.. date: 2024-05-20-20-30-57 +.. gh-issue: 111201 +.. nonce: DAA5lC +.. section: Library + +Remove dependency to :mod:`readline` from the new Python REPL. + +.. + +.. date: 2024-05-20-13-48-37 +.. gh-issue: 119189 +.. nonce: dhJVs5 +.. section: Library + +When using the ``**`` operator or :func:`pow` with +:class:`~fractions.Fraction` as the base and an exponent that is not +rational, a float, or a complex, the fraction is no longer converted to a +float. + +.. + +.. date: 2024-05-19-18-49-04 +.. gh-issue: 119174 +.. nonce: 5GTv7d +.. section: Library + +Fix high DPI causes turtledemo(turtle-graphics examples) windows blurry +Patch by Wulian233 and Terry Jan Reedy + +.. + +.. date: 2024-05-19-13-05-59 +.. gh-issue: 119121 +.. nonce: P1gnh1 +.. section: Library + +Fix a NameError happening in ``asyncio.staggered.staggered_race``. This +function is now tested. + +.. + +.. date: 2024-05-19-12-25-36 +.. gh-issue: 119105 +.. nonce: VcR4ig +.. section: Library + +``difflib.Differ`` is much faster for some cases of diffs where many pairs +of lines are equally similar. + +.. + +.. date: 2024-05-17-17-32-12 +.. gh-issue: 119113 +.. nonce: kEv1Ll +.. section: Library + +Fix issue where :meth:`pathlib.PurePath.with_suffix` didn't raise +:exc:`TypeError` when given ``None`` as a suffix. + +.. + +.. date: 2024-05-16-17-31-46 +.. gh-issue: 118643 +.. nonce: hAWH4C +.. section: Library + +Fix an AttributeError in the :mod:`email` module when re-fold a long address +list. Also fix more cases of incorrect encoding of the address separator in +the address list. + +.. + +.. date: 2024-05-15-01-36-08 +.. gh-issue: 73991 +.. nonce: CGknDf +.. section: Library + +Add :meth:`pathlib.Path.copy`, which copies a file or directory to another. + +.. + +.. date: 2024-05-12-21-38-42 +.. gh-issue: 58933 +.. nonce: 0kgU2l +.. section: Library + +Make :mod:`pdb` return to caller frame correctly when ``f_trace`` of the +caller frame is not set + +.. + +.. date: 2024-05-11-20-23-45 +.. gh-issue: 82805 +.. nonce: F9bz4J +.. section: Library + +Support single-dot file extensions in :attr:`pathlib.PurePath.suffix` and +related attributes and methods. For example, the +:attr:`~pathlib.PurePath.suffixes` of ``PurePath('foo.bar.')`` are now +``['.bar', '.']`` rather than ``[]``. This brings file extension splitting +in line with :func:`os.path.splitext`. + +.. + +.. date: 2024-05-10-22-59-01 +.. gh-issue: 118924 +.. nonce: 9nyvSH +.. section: Library + +Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. Patch +by Hugo van Kemenade. + +.. + +.. date: 2024-05-10-22-36-01 +.. gh-issue: 118928 +.. nonce: IW7Ukv +.. section: Library + +Disallow using a sequence of parameters with named placeholders in +:mod:`sqlite3` queries. Patch by Erlend E. Aasland. + +.. + +.. date: 2024-05-10-05-24-32 +.. gh-issue: 118895 +.. nonce: wUm5r2 +.. section: Library + +Setting attributes on :data:`typing.NoDefault` now raises +:exc:`AttributeError` instead of :exc:`TypeError`. + +.. + +.. date: 2024-05-09-21-36-11 +.. gh-issue: 118868 +.. nonce: uckxxP +.. section: Library + +Fixed issue where kwargs were no longer passed to the logging handler +QueueHandler + +.. + +.. date: 2024-05-09-12-33-25 +.. gh-issue: 118827 +.. nonce: JrzHz1 +.. section: Library + +Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. It had +previously raised a :exc:`DeprecationWarning` since Python 3.11. Patch by +Nikita Sobolev. + +.. + +.. date: 2024-05-09-11-50-26 +.. gh-issue: 118824 +.. nonce: -jBJQC +.. section: Library + +Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. Use +:func:`pty.openpty` instead. Patch by Nikita Sobolev. + +.. + +.. date: 2024-05-09-08-46-12 +.. gh-issue: 118851 +.. nonce: aPAoJw +.. section: Library + +``ctx`` arguments to the constructors of :mod:`ast` node classes now default +to :class:`ast.Load() `. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-09-02-43-37 +.. gh-issue: 101588 +.. nonce: 30bNAr +.. section: Library + +Remove copy, deepcopy, and pickle from itertools. These had previously +raised a DeprecationWarning since Python 3.12. + +.. + +.. date: 2024-05-09-01-05-52 +.. gh-issue: 118805 +.. nonce: N7dm07 +.. section: Library + +Remove *type*, *choices*, and *metavar* parameters of +:class:`!argparse.BooleanOptionalAction`. They were deprecated since Python +3.12. + +.. + +.. date: 2024-05-09-00-52-30 +.. gh-issue: 118803 +.. nonce: Wv3AvU +.. section: Library + +:class:`!typing.ByteString` and :class:`!collections.abc.ByteString` are +removed. They had previously raised a :exc:`DeprecationWarning` since Python +3.12. + +.. + +.. date: 2024-05-08-23-16-50 +.. gh-issue: 118798 +.. nonce: Q_ybqP +.. section: Library + +The *isdst* parameter has been removed from :func:`email.utils.localtime`. +Patch by Hugo van Kemenade. + +.. + +.. date: 2024-05-08-21-30-33 +.. gh-issue: 118760 +.. nonce: XvyMHn +.. section: Library + +Restore the default value of ``tkiter.wantobjects`` to ``1``. + +.. + +.. date: 2024-05-08-21-13-56 +.. gh-issue: 118760 +.. nonce: mdmH3T +.. section: Library + +Fix errors in calling Tkinter bindings on Windows. + +.. + +.. date: 2024-05-08-20-41-48 +.. gh-issue: 74033 +.. nonce: YebHZj +.. section: Library + +Drop support for passing keyword arguments to :class:`pathlib.Path`. + +.. + +.. date: 2024-05-08-19-47-34 +.. gh-issue: 101357 +.. nonce: e4R_9x +.. section: Library + +Suppress all :exc:`OSError` exceptions from :meth:`pathlib.Path.exists` and +``is_*()`` methods, rather than a selection of more common errors. The new +behaviour is consistent with :func:`os.path.exists`, :func:`os.path.isdir`, +etc. Use :meth:`pathlib.Path.stat` to retrieve the file status without +suppressing exceptions. + +.. + +.. date: 2024-05-08-18-59-19 +.. gh-issue: 78707 +.. nonce: _Lz1sw +.. section: Library + +Drop support for passing additional positional arguments to +:meth:`pathlib.PurePath.relative_to` and +:meth:`~pathlib.PurePath.is_relative_to`. + +.. + +.. date: 2024-05-08-18-33-07 +.. gh-issue: 118507 +.. nonce: OCQsAY +.. section: Library + +Fix :func:`os.path.isfile` on Windows for pipes. Speedup +:func:`os.path.isjunction` and :func:`os.path.lexists` on Windows with a +native implementation. + +.. + +.. date: 2024-05-08-09-21-49 +.. gh-issue: 118772 +.. nonce: c16E8X +.. section: Library + +Allow :class:`typing.TypeVar` instances without a default to follow +instances without a default in some cases. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-07-17-38-53 +.. gh-issue: 118714 +.. nonce: XXKpVZ +.. section: Library + +Allow ``restart`` in post-mortem debugging of :mod:`pdb`. Removed restart +message when the user quits pdb from post-mortem mode. + +.. + +.. date: 2024-05-06-17-39-52 +.. gh-issue: 118673 +.. nonce: sTXBit +.. section: Library + +Removed executable bits and shebang from stdlib modules. + +.. + +.. date: 2024-05-01-22-24-05 +.. gh-issue: 110863 +.. nonce: GjYBbq +.. section: Library + +:func:`os.path.realpath` now suppresses any :exc:`OSError` from +:func:`os.readlink` when *strict* mode is disabled (the default). + +.. + +.. date: 2024-04-28-19-51-00 +.. gh-issue: 118263 +.. nonce: Gaap3S +.. section: Library + +Speed up :func:`os.path.splitroot` & :func:`os.path.normpath` with a direct +C call. + +.. + +.. date: 2024-04-24-16-23-04 +.. gh-issue: 110190 +.. nonce: TGd5qx +.. section: Library + +Fix ctypes structs with array on SPARC by setting ``MAX_STRUCT_SIZE`` to 32 +in stgdict. Patch by Jakub Kulik + +.. + +.. date: 2024-04-19-14-59-53 +.. gh-issue: 118033 +.. nonce: amS4Gw +.. section: Library + +Fix :func:`dataclasses.dataclass` not creating a ``__weakref__`` slot when +subclassing :class:`typing.Generic`. + +.. + +.. date: 2024-04-19-05-58-50 +.. gh-issue: 117766 +.. nonce: J3xepp +.. section: Library + +Always use :func:`str` to print ``choices`` in :mod:`argparse`. + +.. + +.. date: 2024-03-19-21-41-31 +.. gh-issue: 106531 +.. nonce: Mgd--6 +.. section: Library + +In :mod:`importlib.resources`, sync with `importlib_resources 6.3.2 +`_, +including: ``MultiplexedPath`` now expects ``Traversable`` paths, +deprecating string arguments to ``MultiplexedPath``; Enabled support for +resources in namespace packages in zip files; Fixed ``NotADirectoryError`` +when calling files on a subdirectory of a namespace package. + +.. + +.. date: 2024-02-27-10-22-15 +.. gh-issue: 115937 +.. nonce: 0cVNur +.. section: Library + +Removed extra preprocessing for the ``__signature__`` attribute: the code +just check if it's a :class:`inspect.Signature` instance. Patch by Sergey B +Kirpichev. + +.. + +.. date: 2024-02-22-10-12-59 +.. gh-issue: 115808 +.. nonce: F2g2Ku +.. section: Library + +Add :func:`operator.is_none` and :func:`operator.is_not_none` functions. + +.. + +.. date: 2024-01-18-21-44-23 +.. gh-issue: 114264 +.. nonce: DBKn29 +.. section: Library + +Improve performance of :func:`copy.deepcopy` by adding a fast path for +atomic types. + +.. + +.. date: 2024-01-14-11-43-31 +.. gh-issue: 113878 +.. nonce: dmEIN3 +.. section: Library + +Add *doc* parameter to :func:`dataclasses.field`, so it can be stored and +shown as a documentation / metadata. If ``@dataclass(slots=True)`` is used, +then the supplied string is availabl in the :attr:`~object.__slots__` dict. +Otherwise, the supplied string is only available in the corresponding +:class:`dataclasses.Field` object. + +In order to support this feature we are changing the ``__slots__`` format in +dataclasses from :class:`tuple` to :class:`dict` when documentation / +metadata is present. + +.. + +.. date: 2024-01-12-08-51-03 +.. gh-issue: 113978 +.. nonce: MqTgB0 +.. section: Library + +Ignore warnings on text completion inside REPL. + +.. + +.. date: 2023-12-14-13-43-27 +.. gh-issue: 113008 +.. nonce: jWYn8T +.. section: Library + +Correct argparse usage output for required, mutually exclusive groups +containing a positional argument + +.. + +.. date: 2023-12-12-15-19-58 +.. gh-issue: 108172 +.. nonce: KyDPuG +.. section: Library + +``webbrowser`` honors OS preferred browser on Linux when its desktop entry +name contains the text of a known browser name. + +.. + +.. date: 2023-09-19-17-56-24 +.. gh-issue: 109109 +.. nonce: WJvvX2 +.. section: Library + +You can now get the raw TLS certificate chains from TLS connections via +:meth:`ssl.SSLSocket.get_verified_chain` and +:meth:`ssl.SSLSocket.get_unverified_chain` methods. + +Contributed by Mateusz Nowak. + +.. + +.. date: 2023-06-17-09-07-06 +.. gh-issue: 105623 +.. nonce: 5G06od +.. section: Library + +Fix performance degradation in +:class:`logging.handlers.RotatingFileHandler`. Patch by Craig Robson. + +.. + +.. date: 2023-06-16-14-52-00 +.. gh-issue: 102450 +.. nonce: MfeR6A +.. section: Library + +Add missing ISO-8601 24:00 alternative to midnight of next day to +:meth:`datetime.datetime.fromisoformat` and +:meth:`datetime.time.fromisoformat`. Patch by Izan "TizzySaurus" Robinson +(tizzysaurus@gmail.com) + +.. + +.. date: 2023-04-28-09-54-15 +.. gh-issue: 103956 +.. nonce: EyLDPS +.. section: Library + +Fix lack of newline characters in :mod:`trace` module output when line +tracing is enabled but source code line for current frame is not available. + +.. + +.. date: 2023-04-26-22-24-17 +.. gh-issue: 92081 +.. nonce: V8xMot +.. section: Library + +Fix missing spaces in email headers when the spaces are mixed with encoded +8-bit characters. + +.. + +.. date: 2023-04-24-05-34-23 +.. gh-issue: 103194 +.. nonce: GwBwWL +.. section: Library + +Prepare Tkinter for C API changes in Tcl 8.7/9.0 to avoid +:class:`!_tkinter.Tcl_Obj` being unexpectedly returned instead of +:class:`bool`, :class:`str`, :class:`bytearray`, or :class:`int`. + +.. + +.. date: 2023-04-10-00-04-37 +.. gh-issue: 87106 +.. nonce: UyBnPQ +.. section: Library + +Fixed handling in :meth:`inspect.Signature.bind` of keyword arguments having +the same name as positional-only arguments when a variadic keyword argument +(e.g. ``**kwargs``) is present. + +.. + +.. date: 2023-03-30-18-19-53 +.. gh-issue: 103134 +.. nonce: bHrn91 +.. section: Library + +Add additional methods to :ref:`proxy objects +` in the :mod:`!multiprocessing` module: + +* :meth:`!clear` and :meth:`!copy` for proxies of :class:`list` +* :meth:`~dict.fromkeys`, ``reversed(d)``, ``d | {}``, ``{} | d``, + ``d |= {'b': 2}`` for proxies of :class:`dict` + +.. + +.. bpo: 45767 +.. date: 2022-03-10-16-47-57 +.. nonce: ywmyo1 +.. section: Library + +Fix integer conversion in :func:`os.major`, :func:`os.minor`, and +:func:`os.makedev`. Support device numbers larger than ``2**63-1``. Support +non-existent device number (``NODEV``). + +.. + +.. bpo: 44864 +.. date: 2021-08-24-19-37-46 +.. nonce: KzxaDh +.. section: Library + +Do not translate user-provided strings in :class:`argparse.ArgumentParser`. + +.. + +.. bpo: 41843 +.. date: 2020-12-22-18-08-12 +.. nonce: q9Nh2r +.. section: Library + +Solaris now uses :func:`os.sendfile` fast-copy syscall for more efficient +:mod:`shutil` file copy related functions. + +.. + +.. bpo: 15987 +.. date: 2020-03-28-21-00-54 +.. nonce: aBL8XS +.. section: Library + +Implemented :func:`ast.compare` for comparing two ASTs. Patch by Batuhan +Taskaya with some help from Jeremy Hylton. + +.. + +.. date: 2019-08-27-01-16-50 +.. gh-issue: 67693 +.. nonce: 4NIAiy +.. section: Library + +Fix :func:`urllib.parse.urlunparse` and :func:`urllib.parse.urlunsplit` for +URIs with path starting with multiple slashes and no authority. Based on +patch by Ashwin Ramaswami. + +.. + +.. bpo: 14074 +.. date: 2018-12-04-07-36-27 +.. nonce: fMLKCu +.. section: Library + +Fix :mod:`argparse` metavar processing to allow positional arguments to have +a tuple metavar. + +.. + +.. date: 2024-10-04-15-34-34 +.. gh-issue: 122392 +.. nonce: V8K3w2 +.. section: IDLE + +Increase currently inadequate vertical spacing for the IDLE browsers (path, +module, and stack) on high-resolution monitors. + +.. + +.. date: 2024-09-21-23-12-18 +.. gh-issue: 112938 +.. nonce: OeiDru +.. section: IDLE + +Fix uninteruptable hang when Shell gets rapid continuous output. + +.. + +.. date: 2024-07-30-18-02-55 +.. gh-issue: 122482 +.. nonce: TerE0g +.. section: IDLE + +Change About IDLE to direct users to discuss.python.org instead of the now +unused idle-dev email and mailing list. + +.. + +.. date: 2024-07-16-16-57-03 +.. gh-issue: 78889 +.. nonce: U7ghFD +.. section: IDLE + +Stop Shell freezes by blocking user access to non-method sys.stdout.shell +attributes, which are all private. + +.. + +.. date: 2024-06-16-21-42-45 +.. gh-issue: 120083 +.. nonce: nczuyv +.. section: IDLE + +Add explicit black IDLE Hovertip foreground color needed for recent macOS. +Fixes Sonoma showing unreadable white on pale yellow. Patch by John +Riggles. + +.. + +.. date: 2024-06-05-14-54-24 +.. gh-issue: 120104 +.. nonce: j_thj4 +.. section: IDLE + +Fix padding in config and search dialog windows in IDLE. + +.. + +.. date: 2024-10-10-02-56-24 +.. gh-issue: 124872 +.. nonce: 0mDDOq +.. section: Documentation + +Added definitions for :term:`context`, :term:`current context`, and +:term:`context management protocol`, updated related definitions to be +consistent, and expanded the documentation for :class:`contextvars.Context`. + +.. + +.. date: 2024-10-07-00-31-17 +.. gh-issue: 125018 +.. nonce: yKnymn +.. section: Documentation + +The :mod:`importlib.metadata` documentation now includes semantic +cross-reference targets for the significant documented APIs. This means +intersphinx references like :func:`importlib.metadata.version` will now work +as expected. + +.. + +.. date: 2024-09-27-16-47-48 +.. gh-issue: 124720 +.. nonce: nVSTVb +.. section: Documentation + +Update "Using Python on a Mac" section of the "Python Setup and Usage" +document and include information on installing free-threading support. + +.. + +.. date: 2024-09-24-11-52-36 +.. gh-issue: 124457 +.. nonce: yrCjSV +.. section: Documentation + +Remove coverity scan from the CPython repo. It has not been used since 2020 +and is currently unmaintained. + +.. + +.. date: 2024-09-19-19-33-25 +.. gh-issue: 116622 +.. nonce: M65UZ6 +.. section: Documentation + +Add an Android platform guide, and flag modules not available on Android. + +.. + +.. date: 2024-09-11-16-52-08 +.. gh-issue: 123976 +.. nonce: jhOfNR +.. section: Documentation + +Refresh docs around custom providers. + +.. + +.. date: 2024-08-01-17-18-21 +.. gh-issue: 70870 +.. nonce: fZnBM9 +.. section: Documentation + +Clarified the dual usage of the term "free variable" (both the formal +meaning of any reference to names defined outside the local scope, and the +narrower pragmatic meaning of nonlocal variables named in ``co_freevars``). + +.. + +.. date: 2024-07-19-12-22-48 +.. gh-issue: 121277 +.. nonce: wF_zKd +.. section: Documentation + +Writers of CPython's documentation can now use ``next`` as the version for +the ``versionchanged``, ``versionadded``, ``deprecated`` directives. + +.. + +.. date: 2024-07-14-12-25-53 +.. gh-issue: 117765 +.. nonce: YFMOUv +.. section: Documentation + +Improved documentation for :func:`unittest.mock.patch.dict` + +.. + +.. date: 2024-07-14-11-48-10 +.. gh-issue: 121749 +.. nonce: nxHoTk +.. section: Documentation + +Fix documentation for :c:func:`PyModule_AddObjectRef`. + +.. + +.. date: 2024-06-05-12-36-18 +.. gh-issue: 120012 +.. nonce: f14DbQ +.. section: Documentation + +Clarify the behaviours of :meth:`multiprocessing.Queue.empty` and +:meth:`multiprocessing.SimpleQueue.empty` on closed queues. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-06-03-22-06-26 +.. gh-issue: 119574 +.. nonce: Ik9kOO +.. section: Documentation + +Added some missing environment variables to the output of +:option:`--help-env`. + +.. + +.. bpo: 34008 +.. date: 2018-07-04-20-35-25 +.. nonce: bqecIb +.. section: Documentation + +The :c:func:`Py_Main` documentation moved from the "Very High Level API" +section to the "Initialization and Finalization" section. + +Also make it explicit that we expect ``Py_Main`` to typically be called +instead of ``Py_Initialize`` rather than after it (since ``Py_Main`` makes +its own call to ``Py_Initialize``). Document that calling both is supported +but is version dependent on which settings will be applied correctly. + +.. + +.. date: 2024-10-15-15-29-41 +.. gh-issue: 124375 +.. nonce: wNrWVa +.. section: Core and Builtins + +Fix a crash in the free threading build when the GC runs concurrently with a +new thread starting. + +.. + +.. date: 2024-10-10-14-47-13 +.. gh-issue: 125221 +.. nonce: nfSQzT +.. section: Core and Builtins + +Fix possible race condition when calling :meth:`~object.__reduce_ex__` for +the first time in the free threading build. + +.. + +.. date: 2024-10-10-12-04-56 +.. gh-issue: 125174 +.. nonce: _8h6T7 +.. section: Core and Builtins + +Make the handling of reference counts of immortal objects more robust. +Immortal objects with reference counts that deviate from their original +reference count by up to a billion (half a billion on 32 bit builds) are +still counted as immortal. + +.. + +.. date: 2024-10-07-23-33-18 +.. gh-issue: 125039 +.. nonce: MKTyNI +.. section: Core and Builtins + +Make ``this_instr`` and ``prev_instr`` const in cases generator. + +.. + +.. date: 2024-10-05-23-53-06 +.. gh-issue: 125008 +.. nonce: ETANpd +.. section: Core and Builtins + +Fix :func:`tokenize.untokenize` producing invalid syntax for double braces +preceded by certain escape characters. + +.. + +.. date: 2024-10-03-22-26-39 +.. gh-issue: 124871 +.. nonce: tAMF47 +.. section: Core and Builtins + +Fix compiler bug (in some versions of 3.13) where an assertion fails during +reachability analysis. + +.. + +.. date: 2024-10-03-14-39-41 +.. gh-issue: 123378 +.. nonce: dCxANf +.. section: Core and Builtins + +Fix a crash in the :meth:`~object.__str__` method of :exc:`UnicodeError` +objects when the :attr:`UnicodeError.start` and :attr:`UnicodeError.end` +values are invalid or out-of-range. Patch by Bénédikt Tran. + +.. + +.. date: 2024-09-30-16-39-37 +.. gh-issue: 118093 +.. nonce: J2A3gz +.. section: Core and Builtins + +Improve the experimental JIT compiler's ability to stay "on trace" when +encountering highly-biased branches. + +.. + +.. date: 2024-09-27-17-18-53 +.. gh-issue: 124642 +.. nonce: OCjhBJ +.. section: Core and Builtins + +Fixed scalability issue in free-threaded builds for lock-free reads from +dictionaries in multi-threaded scenarios + +.. + +.. date: 2024-09-26-18-21-06 +.. gh-issue: 116510 +.. nonce: FacUWO +.. section: Core and Builtins + +Fix a crash caused by immortal interned strings being shared between +sub-interpreters that use basic single-phase init. In that case, the string +can be used by an interpreter that outlives the interpreter that created and +interned it. For interpreters that share obmalloc state, also share the +interned dict with the main interpreter. + +.. + +.. date: 2024-09-26-17-55-34 +.. gh-issue: 116510 +.. nonce: dhn8w8 +.. section: Core and Builtins + +Fix a bug that can cause a crash when sub-interpreters use "basic" +single-phase extension modules. Shared objects could refer to PyGC_Head +nodes that had been freed as part of interpreter cleanup. + +.. + +.. date: 2024-09-26-13-25-01 +.. gh-issue: 119180 +.. nonce: k_JCX0 +.. section: Core and Builtins + +The ``__main__`` module no longer always contains an ``__annotations__`` +dictionary in its global namespace. + +.. + +.. date: 2024-09-26-12-19-13 +.. gh-issue: 124547 +.. nonce: P_SHfU +.. section: Core and Builtins + +When deallocating an object with inline values whose ``__dict__`` is still +live: if memory allocation for the inline values fails, clear the +dictionary. Prevents an interpreter crash. + +.. + +.. date: 2024-09-25-14-45-56 +.. gh-issue: 124513 +.. nonce: ywiXtr +.. section: Core and Builtins + +Fix a crash in FrameLocalsProxy constructor: check the number of arguments. +Patch by Victor Stinner. + +.. + +.. date: 2024-09-25-11-53-22 +.. gh-issue: 124442 +.. nonce: EXC1Ve +.. section: Core and Builtins + +Fix nondeterminism in compilation by sorting the value of +:attr:`~type.__static_attributes__`. Patch by kp2pml30. + +.. + +.. date: 2024-09-23-23-06-19 +.. gh-issue: 124285 +.. nonce: mahGTg +.. section: Core and Builtins + +Fix bug where ``bool(a)`` can be invoked more than once during the +evaluation of a compound boolean expression. + +.. + +.. date: 2024-09-23-15-23-14 +.. gh-issue: 123856 +.. nonce: yrgJ9m +.. section: Core and Builtins + +Fix PyREPL failure when a keyboard interrupt is triggered after using a +history search + +.. + +.. date: 2024-09-23-13-25-27 +.. gh-issue: 65961 +.. nonce: LDqXV2 +.. section: Core and Builtins + +Deprecate the setting and using ``__package__`` and ``__cached__``. + +.. + +.. date: 2024-09-19-16-57-34 +.. gh-issue: 119726 +.. nonce: DseseK +.. section: Core and Builtins + +The JIT now generates more efficient code for calls to C functions resulting +in up to 0.8% memory savings and 1.5% speed improvement on AArch64. Patch by +Diego Russo. + +.. + +.. date: 2024-09-19-13-17-31 +.. gh-issue: 122878 +.. nonce: 4iFpsB +.. section: Core and Builtins + +Use the ``pager`` binary, if available (e.g. on Debian and derivatives), to +display REPL ``help()``. + +.. + +.. date: 2024-09-17-22-06-01 +.. gh-issue: 124188 +.. nonce: aFqNAB +.. section: Core and Builtins + +Fix reading and decoding a line from the source file witn non-UTF-8 encoding +for syntax errors raised in the compiler. + +.. + +.. date: 2024-09-13-02-25-06 +.. gh-issue: 124027 +.. nonce: to_9DY +.. section: Core and Builtins + +Support ````, ````, and ```` keys in the Python +REPL when ``$TERM`` is set to ``vt100``. + +.. + +.. date: 2024-09-12-21-53-26 +.. gh-issue: 124022 +.. nonce: fQzUiW +.. section: Core and Builtins + +Fix bug where docstring is removed from classes in interactive mode. + +.. + +.. date: 2024-09-11-15-48-36 +.. gh-issue: 123958 +.. nonce: 5VW2r0 +.. section: Core and Builtins + +docstrings are now removed from the optimized AST in optimization level 2. + +.. + +.. date: 2024-09-10-20-25-00 +.. gh-issue: 123923 +.. nonce: A7uxqa +.. section: Core and Builtins + +The ``f_executable`` field in the internal :c:struct:`_PyInterpreterFrame` +struct now uses a tagged pointer. Profilers and debuggers that uses this +field should clear the least significant bit to recover the +:c:expr:`PyObject*` pointer. + +.. + +.. date: 2024-09-10-13-27-16 +.. gh-issue: 77894 +.. nonce: ZC-Olu +.. section: Core and Builtins + +Fix possible crash in the garbage collector when it tries to break a +reference loop containing a :class:`memoryview` object. Now a +:class:`!memoryview` object can only be cleared if there are no buffers that +refer it. + +.. + +.. date: 2024-09-06-19-23-44 +.. gh-issue: 120221 +.. nonce: giJEDT +.. section: Core and Builtins + +asyncio REPL is now again properly recognizing KeyboardInterrupts. Display +of exceptions raised in secondary threads is fixed. + +.. + +.. date: 2024-09-06-14-13-01 +.. gh-issue: 119310 +.. nonce: WQxyDF +.. section: Core and Builtins + +Allow the new interactive shell to read history files written with the +editline library that use unicode-escaped entries. Patch by aorcajo and +Łukasz Langa. + +.. + +.. date: 2024-09-03-13-34-35 +.. gh-issue: 123572 +.. nonce: uuqoYV +.. section: Core and Builtins + +Fix key mappings for various F-keys in Windows for the new REPL. Patch by +devdanzin + +.. + +.. date: 2024-09-02-20-39-10 +.. gh-issue: 123614 +.. nonce: 26TMHp +.. section: Core and Builtins + +Add :func:`turtle.save` to easily save Turtle drawings as PostScript files. +Patch by Marie Roald and Yngve Mardal Moe. + +.. + +.. date: 2024-09-02-20-36-45 +.. gh-issue: 123339 +.. nonce: QcmpSs +.. section: Core and Builtins + +Setting the :attr:`!__module__` attribute for a class now removes the +``__firstlineno__`` item from the type's dict, so they will no longer be +inconsistent. + +.. + +.. date: 2024-09-02-17-32-15 +.. gh-issue: 119034 +.. nonce: HYh5Vj +.. section: Core and Builtins + +Change ```` and ```` keys of the Python REPL to history +search forward/backward. Patch by Victor Stinner. + +.. + +.. date: 2024-09-01-12-08-39 +.. gh-issue: 123562 +.. nonce: aJPKVu +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for using ``case ... as ...`` with not a +name. + +.. + +.. date: 2024-09-01-00-02-05 +.. gh-issue: 123545 +.. nonce: 8nQNbL +.. section: Core and Builtins + +Fix a double decref in rare cases on experimental JIT builds. + +.. + +.. date: 2024-08-29-19-46-07 +.. gh-issue: 123484 +.. nonce: rjUn_F +.. section: Core and Builtins + +Fix ``_Py_DebugOffsets`` for long objects to be relative to the start of the +object rather than the start of a subobject. + +.. + +.. date: 2024-08-29-13-18-18 +.. gh-issue: 123446 +.. nonce: KWDrgq +.. section: Core and Builtins + +Fix empty function name in :exc:`TypeError` when builtin magic methods are +used without the required args. + +.. + +.. date: 2024-08-28-22-42-51 +.. gh-issue: 123440 +.. nonce: yOFB0N +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for using ``except as`` with not a name. + +.. + +.. date: 2024-08-27-21-44-14 +.. gh-issue: 116017 +.. nonce: ZY3yBY +.. section: Core and Builtins + +Improved JIT memory consumption by periodically freeing memory used by +infrequently-executed code. This change is especially likely to improve the +memory footprint of long-running programs. + +.. + +.. date: 2024-08-27-13-16-40 +.. gh-issue: 123344 +.. nonce: 56Or78 +.. section: Core and Builtins + +Add AST optimizations for type parameter defaults. + +.. + +.. date: 2024-08-26-00-58-26 +.. gh-issue: 123321 +.. nonce: ApxcnE +.. section: Core and Builtins + +Prevent Parser/myreadline race condition from segfaulting on multi-threaded +use. Patch by Bar Harel and Amit Wienner. + +.. + +.. date: 2024-08-25-18-27-49 +.. gh-issue: 123177 +.. nonce: yLuyqE +.. section: Core and Builtins + +Fix a bug causing stray prompts to appear in the middle of wrapped lines in +the new REPL. + +.. + +.. date: 2024-08-25-10-54-22 +.. gh-issue: 122982 +.. nonce: KLD91q +.. section: Core and Builtins + +Extend the deprecation period for bool inversion (``~``) by two years. + +.. + +.. date: 2024-08-23-21-20-34 +.. gh-issue: 123271 +.. nonce: xeVViR +.. section: Core and Builtins + +Make concurrent iterations over the same :func:`zip` iterator safe under +free-threading. + +.. + +.. date: 2024-08-23-18-31-10 +.. gh-issue: 123275 +.. nonce: DprIrj +.. section: Core and Builtins + +Support :option:`-X gil=1 <-X>` and :envvar:`PYTHON_GIL=1 ` on +non-free-threaded builds. + +.. + +.. date: 2024-08-23-15-59-54 +.. gh-issue: 123177 +.. nonce: OLcaC5 +.. section: Core and Builtins + +Deactivate line wrap in the Apple Terminal via a ANSI escape code. Patch by +Pablo Galindo + +.. + +.. date: 2024-08-23-13-08-27 +.. gh-issue: 123229 +.. nonce: aHm-dw +.. section: Core and Builtins + +Fix valgrind warning by initializing the f-string buffers to 0 in the +tokenizer. Patch by Pablo Galindo + +.. + +.. date: 2024-08-23-11-26-54 +.. gh-issue: 122298 +.. nonce: ZMyln4 +.. section: Core and Builtins + +Restore printout of GC stats when ``gc.set_debug(gc.DEBUG_STATS)`` is +called. This feature was accidentally removed when implementing incremental +GC. + +.. + +.. date: 2024-08-21-15-22-53 +.. gh-issue: 121804 +.. nonce: r5K3PS +.. section: Core and Builtins + +Correctly show error locations when a :exc:`SyntaxError` is raised in the +basic REPL. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-08-21-08-53-00 +.. gh-issue: 115776 +.. nonce: 9A7Dv_ +.. section: Core and Builtins + +Enables inline values (Python's equivalent of hidden classes) on any class +who's instances are of a fixed size. + +.. + +.. date: 2024-08-20-12-29-52 +.. gh-issue: 123142 +.. nonce: 3PXiNb +.. section: Core and Builtins + +Fix too-wide source location in exception tracebacks coming from broken +iterables in comprehensions. + +.. + +.. date: 2024-08-20-11-09-16 +.. gh-issue: 123048 +.. nonce: 2TISpv +.. section: Core and Builtins + +Fix a bug where pattern matching code could emit a :opcode:`JUMP_FORWARD` +with no source location. + +.. + +.. date: 2024-08-19-15-13-13 +.. gh-issue: 118093 +.. nonce: dLZ8qS +.. section: Core and Builtins + +Break up ``CALL_ALLOC_AND_ENTER_INIT`` into micro-ops and relax requirement +for exact args, in order to increase the amount of code supported by tier 2. + +.. + +.. date: 2024-08-18-18-25-54 +.. gh-issue: 123123 +.. nonce: 0ZcaEB +.. section: Core and Builtins + +Fix displaying :exc:`SyntaxError` exceptions covering multiple lines. Patch +by Pablo Galindo + +.. + +.. date: 2024-08-17-17-26-25 +.. gh-issue: 123083 +.. nonce: 9xWLJ- +.. section: Core and Builtins + +Fix a potential use-after-free in ``STORE_ATTR_WITH_HINT``. + +.. + +.. date: 2024-08-15-19-28-43 +.. gh-issue: 123022 +.. nonce: m3EF9E +.. section: Core and Builtins + +Fix crash in free-threaded build when calling :c:func:`Py_Initialize` from a +non-main thread. + +.. + +.. date: 2024-08-14-11-38-56 +.. gh-issue: 118093 +.. nonce: 3BywDP +.. section: Core and Builtins + +Add three specializations for :opcode:`CALL_KW`: + +* :opcode:`!CALL_KW_PY` for calls to Python functions +* :opcode:`!CALL_KW_BOUND_METHOD` for calls to bound methods +* :opcode:`!CALL_KW_NON_PY` for all other calls + +.. + +.. date: 2024-08-12-11-45-47 +.. gh-issue: 122821 +.. nonce: WnAzTK +.. section: Core and Builtins + +Make sure that branches in :keyword:`while` statements have consistent +offsets for :mod:`sys.monitoring`. :keyword:`!while` statements are now +compiled with a simple jump at the end of the body, instead of duplicating +the test. + +.. + +.. date: 2024-08-12-11-19-37 +.. gh-issue: 122907 +.. nonce: q68096 +.. section: Core and Builtins + +Building with ``HAVE_DYNAMIC_LOADING`` now works as well as it did in 3.12. +Existing deficiences will be addressed separately. (See +https://github.com/python/cpython/issues/122950.) + +.. + +.. date: 2024-08-10-12-44-03 +.. gh-issue: 122888 +.. nonce: TUyu9r +.. section: Core and Builtins + +Fix crash on certain calls to ``str()`` with positional arguments of the +wrong type. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-08-08-16-02-28 +.. gh-issue: 118093 +.. nonce: m6Mrvy +.. section: Core and Builtins + +Improve the experimental JIT's handling of polymorphic code. + +.. + +.. date: 2024-08-05-19-28-12 +.. gh-issue: 122697 +.. nonce: 17MvYl +.. section: Core and Builtins + +Fixed memory leaks at interpreter shutdown in the free-threaded build, and +also reporting of leaked memory blocks via :option:`-X showrefcount <-X>`. + +.. + +.. date: 2024-08-05-19-04-06 +.. gh-issue: 116622 +.. nonce: 3LWUzE +.. section: Core and Builtins + +Fix Android stdout and stderr messages being truncated or lost. + +.. + +.. date: 2024-08-01-19-13-58 +.. gh-issue: 122527 +.. nonce: eztso6 +.. section: Core and Builtins + +Fix a crash that occurred when a ``PyStructSequence`` was deallocated after +its type's dictionary was cleared by the GC. The type's +:c:member:`~PyTypeObject.tp_basicsize` now accounts for non-sequence fields +that aren't included in the :c:macro:`Py_SIZE` of the sequence. + +.. + +.. date: 2024-07-30-11-41-35 +.. gh-issue: 122445 +.. nonce: Rq0bjS +.. section: Core and Builtins + +Add only fields which are modified via self.* to +:attr:`~type.__static_attributes__`. + +.. + +.. date: 2024-07-29-19-20-25 +.. gh-issue: 122417 +.. nonce: NVgs0a +.. section: Core and Builtins + +In the free-threaded build, the reference counts for heap type objects are +now partially stored in a distributed manner in per-thread arrays. This +reduces contention on the heap type's reference count fields when creating +or destroying instances of the same type from multiple threads concurrently. + +.. + +.. date: 2024-07-29-10-55-46 +.. gh-issue: 116090 +.. nonce: p1MhU0 +.. section: Core and Builtins + +Fix an issue in JIT builds that prevented some :keyword:`for` loops from +correctly firing :monitoring-event:`RAISE` monitoring events. + +.. + +.. date: 2024-07-26-14-05-51 +.. gh-issue: 122300 +.. nonce: SVIF-l +.. section: Core and Builtins + +Preserve AST nodes for f-string with single-element format specifiers. Patch +by Pablo Galindo + +.. + +.. date: 2024-07-26-13-56-32 +.. gh-issue: 120906 +.. nonce: qBh2I9 +.. section: Core and Builtins + +:attr:`frame.f_locals` now supports arbitrary hashable objects as keys. + +.. + +.. date: 2024-07-25-01-45-21 +.. gh-issue: 122239 +.. nonce: 7zh-sW +.. section: Core and Builtins + +When a :class:`list`, :class:`tuple` or :class:`dict` with too many elements +is unpacked, show the actual length in the error message. + +.. + +.. date: 2024-07-24-22-39-07 +.. gh-issue: 122245 +.. nonce: LVa9v8 +.. section: Core and Builtins + +Detection of writes to ``__debug__`` is moved from the compiler's codegen +stage to the symtable. This means that these errors are now detected even in +code that is optimized away before codegen (such as assertions with the +:option:`-O` command line option). + +.. + +.. date: 2024-07-24-17-11-51 +.. gh-issue: 122234 +.. nonce: VxsP_F +.. section: Core and Builtins + +Specializations for sums with float and complex inputs in :func:`sum` now +always use compensated summation. Also, for integer items in above +specializations: :c:func:`PyLong_AsDouble` is used, instead of +:c:func:`PyLong_AsLongAndOverflow`. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-07-23-23-59-04 +.. gh-issue: 122208 +.. nonce: z8KHsY +.. section: Core and Builtins + +Dictionary watchers now only deliver the PyDict_EVENT_ADDED event when the +insertion is in a known good state to succeed. + +.. + +.. date: 2024-07-23-11-57-36 +.. gh-issue: 122160 +.. nonce: HSnrAP +.. section: Core and Builtins + +Remove the ``BUILD_CONST_KEY_MAP`` opcode. Use :opcode:`BUILD_MAP` instead. + +.. + +.. date: 2024-07-21-01-23-54 +.. gh-issue: 122029 +.. nonce: gKv-e2 +.. section: Core and Builtins + +Emit ``c_call`` events in :func:`sys.setprofile` when a ``PyMethodObject`` +pointing to a ``PyCFunction`` is called. + +.. + +.. date: 2024-07-19-15-28-05 +.. gh-issue: 122026 +.. nonce: sta2Ca +.. section: Core and Builtins + +Fix a bug that caused the tokenizer to not correctly identify mismatched +parentheses inside f-strings in some situations. Patch by Pablo Galindo + +.. + +.. date: 2024-07-19-09-38-01 +.. gh-issue: 99108 +.. nonce: qzM6gl +.. section: Core and Builtins + +Python's hashlib now unconditionally uses the vendored HACL* library for +Blake2. Python no longer accepts libb2 as an optional dependency for Blake2. + +We refreshed HACL* to the latest version, and now vendor HACL*'s 128-bit and +256-bit wide vector implementations for Blake2, which are used on x86/x64 +toolchains when the required CPU features are available at runtime. + +HACL*'s 128-bit wide vector implementation of Blake2 can also run on ARM +NEON and Power8, but lacking evidence of a performance gain, these are not +enabled (yet). + +.. + +.. date: 2024-07-18-21-19-04 +.. gh-issue: 121999 +.. nonce: 8IBbTK +.. section: Core and Builtins + +The default extraction filter for the :mod:`tarfile` module is now set to +:func:`'data' `. + +.. + +.. date: 2024-07-16-18-23-22 +.. gh-issue: 121860 +.. nonce: -FTauD +.. section: Core and Builtins + +Fix crash when rematerializing a managed dictionary after it was deleted. + +.. + +.. date: 2024-07-16-15-11-51 +.. gh-issue: 121795 +.. nonce: xkIHrI +.. section: Core and Builtins + +Improve performance of set membership testing, ``set.remove()`` and +``set.discard()`` when the argument is a set. + +.. + +.. date: 2024-07-15-20-41-06 +.. gh-issue: 121814 +.. nonce: oR2ixR +.. section: Core and Builtins + +Fixed the SegFault when :c:func:`PyEval_SetTrace` is used with no Python +frame on stack. + +.. + +.. date: 2024-07-15-20-03-29 +.. gh-issue: 121295 +.. nonce: w53ucI +.. section: Core and Builtins + +Fix PyREPL console getting into a blocked state after interrupting a long +paste + +.. + +.. date: 2024-07-15-16-26-32 +.. gh-issue: 121794 +.. nonce: fhBtiQ +.. section: Core and Builtins + +Fix bug in free-threaded Python where a resurrected object could lead to a +negative ref count assertion failure. + +.. + +.. date: 2024-07-13-12-27-31 +.. gh-issue: 121657 +.. nonce: wgOYLw +.. section: Core and Builtins + +Improve the :exc:`SyntaxError` message if the user tries to use +:keyword:`yield from ` outside a function. + +.. + +.. date: 2024-07-13-09-51-44 +.. gh-issue: 121609 +.. nonce: jWsE5t +.. section: Core and Builtins + +Fix pasting of characters containing unicode character joiners in the new +REPL. Patch by Marta Gomez Macias + +.. + +.. date: 2024-07-12-18-18-44 +.. gh-issue: 121297 +.. nonce: 67VE7b +.. section: Core and Builtins + +Previously, incorrect usage of :keyword:`await` or asynchronous +comprehensions in code removed by the :option:`-O` option was not flagged by +the Python compiler. Now, such code raises :exc:`SyntaxError`. Patch by +Jelle Zijlstra. + +.. + +.. date: 2024-07-10-15-43-54 +.. gh-issue: 117482 +.. nonce: 5WYaXR +.. section: Core and Builtins + +Unexpected slot wrappers are no longer created for builtin static types in +subinterpreters. + +.. + +.. date: 2024-07-10-02-02-32 +.. gh-issue: 121562 +.. nonce: 8beIMi +.. section: Core and Builtins + +Optimized performance of hex_from_char by replacing switch-case with a +lookup table + +.. + +.. date: 2024-07-09-13-53-18 +.. gh-issue: 121499 +.. nonce: rpp7il +.. section: Core and Builtins + +Fix a bug affecting how multi-line history was being rendered in the new +REPL after interacting with the new screen cache. Patch by Pablo Galindo + +.. + +.. date: 2024-07-08-17-15-14 +.. gh-issue: 121497 +.. nonce: I8hMDC +.. section: Core and Builtins + +Fix a bug that was preventing the REPL to correctly respect the history when +an input hook was set. Patch by Pablo Galindo + +.. + +.. date: 2024-07-08-10-31-08 +.. gh-issue: 121012 +.. nonce: M5hHk- +.. section: Core and Builtins + +Tier 2 execution now ensures that list iterators remain exhausted, once they +become exhausted. + +.. + +.. date: 2024-07-08-02-24-55 +.. gh-issue: 121439 +.. nonce: jDHod3 +.. section: Core and Builtins + +Allow tuples of length 20 in the freelist to be reused. + +.. + +.. date: 2024-07-05-11-29-27 +.. gh-issue: 121288 +.. nonce: lYKYYP +.. section: Core and Builtins + +:exc:`ValueError` messages for :meth:`!list.index`, :meth:`!range.index`, +:meth:`!deque.index`, :meth:`!deque.remove` and :meth:`!ShareableList.index` +no longer contain the repr of the searched value (which can be arbitrary +large) and are consistent with error messages for other :meth:`!index` and +:meth:`!remove` methods. + +.. + +.. date: 2024-07-04-23-38-30 +.. gh-issue: 121368 +.. nonce: m3EF9E +.. section: Core and Builtins + +Fix race condition in ``_PyType_Lookup`` in the free-threaded build due to a +missing memory fence. This could lead to ``_PyType_Lookup`` returning +incorrect results on arm64. + +.. + +.. date: 2024-06-30-03-48-10 +.. gh-issue: 121149 +.. nonce: lLBMKe +.. section: Core and Builtins + +Added specialization for summation of complexes, this also improves accuracy +of builtin :func:`sum` for such inputs. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-06-29-10-46-14 +.. gh-issue: 121130 +.. nonce: Rj66Xs +.. section: Core and Builtins + +Fix f-strings with debug expressions in format specifiers. Patch by Pablo +Galindo + +.. + +.. date: 2024-06-28-23-17-22 +.. gh-issue: 121381 +.. nonce: i2xL7P +.. section: Core and Builtins + +Remove ``subprocess._USE_VFORK`` escape hatch code and documentation. It was +added just in case, and doesn't have any known cases that require it. + +.. + +.. date: 2024-06-28-18-34-49 +.. gh-issue: 119726 +.. nonce: Fjv_Ab +.. section: Core and Builtins + +Optimize code layout for calls to C functions from the JIT on AArch64. Patch +by Diego Russo. + +.. + +.. date: 2024-06-28-10-02-58 +.. gh-issue: 121115 +.. nonce: EeSLfc +.. section: Core and Builtins + +:c:func:`PyLong_AsNativeBytes` no longer uses :meth:`~object.__index__` +methods by default. The ``Py_ASNATIVEBYTES_ALLOW_INDEX`` flag has been added +to allow it. + +.. + +.. date: 2024-06-26-14-09-31 +.. gh-issue: 120838 +.. nonce: nFeTL9 +.. section: Core and Builtins + +:c:func:`Py_Finalize()` and :c:func:`Py_FinalizeEx()` now always run with +the main interpreter active. + +.. + +.. date: 2024-06-26-13-42-36 +.. gh-issue: 113433 +.. nonce: xKAtLB +.. section: Core and Builtins + +Subinterpreters now get cleaned up automatically during runtime +finalization. + +.. + +.. date: 2024-06-25-16-26-44 +.. gh-issue: 119726 +.. nonce: WqvHxB +.. section: Core and Builtins + +Improve the speed and memory use of C function calls from JIT code on +AArch64. Patch by Diego Russo + +.. + +.. date: 2024-06-24-08-39-23 +.. gh-issue: 116017 +.. nonce: -Bw2UY +.. section: Core and Builtins + +Simplify the warmup mechanism used for "side exits" in JIT code, resulting +in slightly better performance and slightly lower memory usage for most +platforms. + +.. + +.. date: 2024-06-19-21-34-21 +.. gh-issue: 98442 +.. nonce: cqhjkN +.. section: Core and Builtins + +Fix too wide source locations of the cleanup instructions of a with +statement. + +.. + +.. date: 2024-06-19-19-54-35 +.. gh-issue: 120754 +.. nonce: uF29sj +.. section: Core and Builtins + +Reduce the number of system calls invoked when reading a whole file (ex. +``open('a.txt').read()``). For a sample program that reads the contents of +the 400+ ``.rst`` files in the cpython repository ``Doc`` folder, there is +an over 10% reduction in system call count. + +.. + +.. date: 2024-06-19-11-10-50 +.. gh-issue: 119462 +.. nonce: DpcqSe +.. section: Core and Builtins + +Make sure that invariants of type versioning are maintained: * Superclasses +always have their version number assigned before subclasses * The version +tag is always zero if the tag is not valid. * The version tag is always +non-if the tag is valid. + +.. + +.. date: 2024-06-19-01-58-54 +.. gh-issue: 120437 +.. nonce: nCkIoI +.. section: Core and Builtins + +Fix ``_CHECK_STACK_SPACE`` optimization problems introduced in :gh:`118322`. + +.. + +.. date: 2024-06-18-22-41-05 +.. gh-issue: 120722 +.. nonce: rS7tkE +.. section: Core and Builtins + +Correctly set the bytecode position on return instructions within lambdas. +Patch by Jelle Zijlstra. + +.. + +.. date: 2024-06-18-21-34-30 +.. gh-issue: 120367 +.. nonce: zDwffP +.. section: Core and Builtins + +Fix bug where compiler creates a redundant jump during pseudo-op +replacement. Can only happen with a synthetic AST that has a try on the same +line as the instruction following the exception handler. + +.. + +.. date: 2024-06-17-12-20-20 +.. gh-issue: 120507 +.. nonce: 94lz2J +.. section: Core and Builtins + +Remove the ``BEFORE_WITH`` and ``BEFORE_ASYNC_WITH`` instructions. Add the +new :opcode:`LOAD_SPECIAL` instruction. Generate code for ``with`` and +``async with`` statements using the new instruction. + +.. + +.. date: 2024-06-14-22-02-25 +.. gh-issue: 113993 +.. nonce: MiA0vX +.. section: Core and Builtins + +Strings interned with :func:`sys.intern` are again garbage-collected when no +longer used, as per the documentation. Strings interned with the C function +:c:func:`PyUnicode_InternInPlace` are still immortal. Internals of the +string interning mechanism have been changed. This may affect performance +and identities of :class:`str` objects. + +.. + +.. date: 2024-06-14-07-52-00 +.. gh-issue: 120485 +.. nonce: yy4K4b +.. section: Core and Builtins + +Add an override of ``allow_reuse_port`` on classes subclassing +``socketserver.TCPServer`` where ``allow_reuse_address`` is also overridden. + +.. + +.. date: 2024-06-13-12-17-52 +.. gh-issue: 120384 +.. nonce: w1UBGl +.. section: Core and Builtins + +Fix an array out of bounds crash in ``list_ass_subscript``, which could be +invoked via some specifically tailored input: including concurrent +modification of a list object, where one thread assigns a slice and another +clears it. + +.. + +.. date: 2024-06-12-18-50-29 +.. gh-issue: 120367 +.. nonce: LmXx2y +.. section: Core and Builtins + +Fix crash in compiler on code with redundant NOPs and JUMPs which show up +after exception handlers are moved to the end of the code. + +.. + +.. date: 2024-06-12-18-23-15 +.. gh-issue: 120380 +.. nonce: edtqjq +.. section: Core and Builtins + +Fix Python implementation of :class:`pickle.Pickler` for :class:`bytes` and +:class:`bytearray` objects when using protocol version 5. Patch by Bénédikt +Tran. + +.. + +.. date: 2024-06-12-13-47-25 +.. gh-issue: 120397 +.. nonce: n-I_cc +.. section: Core and Builtins + +Improve the throughput by up to two times for the :meth:`str.count`, +:meth:`bytes.count` and :meth:`bytearray.count` methods for counting single +characters. + +.. + +.. date: 2024-06-11-17-56-12 +.. gh-issue: 120221 +.. nonce: si9hM9 +.. section: Core and Builtins + +Deliver real signals on Ctrl-C and Ctrl-Z in the new REPL. Patch by Pablo +Galindo + +.. + +.. date: 2024-06-11-12-47-54 +.. gh-issue: 120346 +.. nonce: hhn_6X +.. section: Core and Builtins + +Respect :envvar:`PYTHON_BASIC_REPL` when running in interactive inspect mode +(``python -i``). Patch by Pablo Galindo + +.. + +.. date: 2024-06-10-22-30-26 +.. gh-issue: 93691 +.. nonce: 68WOTS +.. section: Core and Builtins + +Fix source locations of instructions generated for the iterator of a for +statement. + +.. + +.. date: 2024-06-10-15-07-16 +.. gh-issue: 120198 +.. nonce: WW_pjO +.. section: Core and Builtins + +Fix a crash when multiple threads read and write to the same ``__class__`` +of an object concurrently. + +.. + +.. date: 2024-06-10-10-42-48 +.. gh-issue: 120298 +.. nonce: napREA +.. section: Core and Builtins + +Fix use-after free in ``list_richcompare_impl`` which can be invoked via +some specifically tailored evil input. + +.. + +.. date: 2024-06-09-19-13-38 +.. gh-issue: 119666 +.. nonce: S0G4rZ +.. section: Core and Builtins + +Fix a compiler crash in the case where two comprehensions in class scope +both reference ``__class__``. + +.. + +.. date: 2024-06-07-22-54-15 +.. gh-issue: 119726 +.. nonce: D9EE-o +.. section: Core and Builtins + +JIT: Re-use trampolines on AArch64 when creating stencils. Patch by Diego +Russo + +.. + +.. date: 2024-06-07-16-09-04 +.. gh-issue: 120225 +.. nonce: kuYf9t +.. section: Core and Builtins + +Fix crash in compiler on empty block at end of exception handler. + +.. + +.. date: 2024-06-05-18-29-18 +.. gh-issue: 93691 +.. nonce: 6OautB +.. section: Core and Builtins + +Fix source locations of instructions generated for with statements. + +.. + +.. date: 2024-06-05-10-32-44 +.. gh-issue: 120097 +.. nonce: 9S2klk +.. section: Core and Builtins + +``FrameLocalsProxy`` now subclasses ``collections.abc.Mapping`` and can be +matched as a mapping in ``match`` statements + +.. + +.. date: 2024-06-05-08-39-40 +.. gh-issue: 120080 +.. nonce: DJFK11 +.. section: Core and Builtins + +Direct call to the :meth:`!int.__round__` now accepts ``None`` as a valid +argument. + +.. + +.. date: 2024-06-03-13-48-44 +.. gh-issue: 119933 +.. nonce: Kc0HG5 +.. section: Core and Builtins + +Improve :exc:`SyntaxError` messages for invalid expressions in a type +parameters bound, a type parameter constraint tuple or a default type +parameter. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-03-13-25-04 +.. gh-issue: 119724 +.. nonce: EH1dkA +.. section: Core and Builtins + +Reverted improvements to error messages for ``elif``/``else`` statements not +matching any valid statements, which made in hard to locate the syntax +errors inside those ``elif``/``else`` blocks. + +.. + +.. date: 2024-06-02-06-12-35 +.. gh-issue: 119879 +.. nonce: Jit951 +.. section: Core and Builtins + +String search is now slightly faster for certain cases. It now utilizes last +character gap (good suffix rule) for two-way periodic needles. + +.. + +.. date: 2024-05-31-12-06-11 +.. gh-issue: 119842 +.. nonce: tCGVsv +.. section: Core and Builtins + +Honor :c:func:`PyOS_InputHook` in the new REPL. Patch by Pablo Galindo + +.. + +.. date: 2024-05-31-08-23-41 +.. gh-issue: 119180 +.. nonce: KL4VxZ +.. section: Core and Builtins + +:func:`classmethod` and :func:`staticmethod` now wrap the +:attr:`~object.__annotations__` and :attr:`~object.__annotate__` attributes +of their underlying callable lazily. See :pep:`649`. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-05-30-23-01-00 +.. gh-issue: 119821 +.. nonce: jPGfvt +.. section: Core and Builtins + +Fix execution of :ref:`annotation scopes ` within classes +when ``globals`` is set to a non-dict. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-30-04-11-36 +.. gh-issue: 118934 +.. nonce: fbDqve +.. section: Core and Builtins + +Make ``PyEval_GetLocals`` return borrowed reference + +.. + +.. date: 2024-05-29-18-53-43 +.. gh-issue: 119740 +.. nonce: zP2JNM +.. section: Core and Builtins + +Remove the previously-deprecated delegation of :func:`int` to +:meth:`~object.__trunc__`. + +.. + +.. date: 2024-05-28-22-49-56 +.. gh-issue: 119689 +.. nonce: VwLFD5 +.. section: Core and Builtins + +Generate stack effect metadata for pseudo instructions from bytecodes.c. + +.. + +.. date: 2024-05-27-19-13-49 +.. gh-issue: 109218 +.. nonce: -sdDg0 +.. section: Core and Builtins + +:func:`complex` accepts now a string only as a positional argument. Passing +a complex number as the "real" or "imag" argument is deprecated; it should +only be passed as a single positional argument. + +.. + +.. date: 2024-05-25-16-45-27 +.. gh-issue: 119548 +.. nonce: pqF9Y6 +.. section: Core and Builtins + +Add a ``clear`` command to the REPL. Patch by Pablo Galindo + +.. + +.. date: 2024-05-25-13-51-48 +.. gh-issue: 111999 +.. nonce: L0q1gh +.. section: Core and Builtins + +Fix the signature of :meth:`str.format_map`. + +.. + +.. date: 2024-05-25-12-52-25 +.. gh-issue: 119560 +.. nonce: wSlm8q +.. section: Core and Builtins + +An invalid assert in beta 1 has been removed. The assert would fail if +``PyState_FindModule()`` was used in an extension module's init function +before the module def had been initialized. + +.. + +.. date: 2024-05-24-21-16-52 +.. gh-issue: 119369 +.. nonce: qBThho +.. section: Core and Builtins + +Fix deadlock during thread deletion in free-threaded build, which could +occur when the GIL was enabled at runtime. + +.. + +.. date: 2024-05-24-21-04-00 +.. gh-issue: 119525 +.. nonce: zLFLf1 +.. section: Core and Builtins + +Fix deadlock involving ``_PyType_Lookup()`` cache in the free-threaded build +when the GIL is dynamically enabled at runtime. + +.. + +.. date: 2024-05-23-20-17-37 +.. gh-issue: 119258 +.. nonce: wZFIpt +.. section: Core and Builtins + +Eliminate type version guards in the tier two interpreter. + +Note that setting the ``tp_version_tag`` manually (which has never been +supported) may result in crashes. + +.. + +.. date: 2024-05-23-06-34-45 +.. gh-issue: 119311 +.. nonce: 2DBwKR +.. section: Core and Builtins + +Fix bug where names are unexpectedly mangled in the bases of generic +classes. + +.. + +.. date: 2024-05-23-06-34-14 +.. gh-issue: 119395 +.. nonce: z-Hsqb +.. section: Core and Builtins + +Fix bug where names appearing after a generic class are mangled as if they +are in the generic class. + +.. + +.. date: 2024-05-22-12-49-03 +.. gh-issue: 119372 +.. nonce: PXig1R +.. section: Core and Builtins + +Correct invalid corner cases in complex division (resulted in ``(nan+nanj)`` +output), e.g. ``1/complex('(inf+infj)')``. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-05-22-06-22-47 +.. gh-issue: 119180 +.. nonce: vZMiXm +.. section: Core and Builtins + +Evaluation of annotations is now deferred. See :pep:`649` for details. + +.. + +.. date: 2024-05-21-12-17-02 +.. gh-issue: 119180 +.. nonce: UNDUb9 +.. section: Core and Builtins + +Replace :opcode:`!LOAD_ASSERTION_ERROR` opcode with +:opcode:`LOAD_COMMON_CONSTANT` and add support for +:exc:`NotImplementedError`. + +.. + +.. date: 2024-05-21-11-27-14 +.. gh-issue: 119213 +.. nonce: nxjxrt +.. section: Core and Builtins + +Non-builtin modules built with argument clinic were crashing if used in a +subinterpreter before the main interpreter. The objects that were causing +the problem by leaking between interpreters carelessly have been fixed. + +.. + +.. date: 2024-05-21-09-46-51 +.. gh-issue: 119011 +.. nonce: WOe3bu +.. section: Core and Builtins + +Fixes ``type.__type_params__`` to return an empty tuple instead of a +descriptor. + +.. + +.. date: 2024-05-20-14-57-39 +.. gh-issue: 118692 +.. nonce: Qadm7F +.. section: Core and Builtins + +Avoid creating unnecessary :exc:`StopIteration` instances for monitoring. + +.. + +.. date: 2024-05-20-10-10-51 +.. gh-issue: 119180 +.. nonce: 35xqpu +.. section: Core and Builtins + +Add an ``__annotate__`` attribute to functions, classes, and modules as part +of :pep:`649`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-16-23-02-03 +.. gh-issue: 119049 +.. nonce: qpd_S- +.. section: Core and Builtins + +Fix displaying the source line for warnings created by the C API if the +:mod:`warnings` module had not yet been imported. + +.. + +.. date: 2024-05-15-12-15-58 +.. gh-issue: 119057 +.. nonce: P3G9G2 +.. section: Core and Builtins + +Improve :exc:`ZeroDivisionError` error message. Now, all error messages are +harmonized: all ``/``, ``//``, and ``%`` operations just use "division by +zero" message. And ``0 ** -1`` operation uses "zero to a negative power". + +.. + +.. date: 2024-05-11-21-44-17 +.. gh-issue: 118844 +.. nonce: q2H_km +.. section: Core and Builtins + +Fix build failures when configuring with both ``--disable-gil`` and +``--enable-experimental-jit``. + +.. + +.. date: 2024-05-10-19-54-18 +.. gh-issue: 118921 +.. nonce: O4ztZG +.. section: Core and Builtins + +Add ``copy()`` method for ``FrameLocalsProxy`` which returns a snapshot +``dict`` for local variables. + +.. + +.. date: 2024-05-09-19-47-12 +.. gh-issue: 117657 +.. nonce: Vn0Yey +.. section: Core and Builtins + +Fix data races on the field that stores a pointer to the interpreter's main +thread that occur in free-threaded builds. + +.. + +.. date: 2024-05-09-02-37-25 +.. gh-issue: 118750 +.. nonce: 7aLfT- +.. section: Core and Builtins + +If the C version of the ``decimal`` module is available, ``int(str)`` now +uses it to supply an asymptotically much faster conversion. However, this +only applies if the string contains over about 2 million digits. + +.. + +.. date: 2024-05-08-09-44-15 +.. gh-issue: 118767 +.. nonce: iFF5F5 +.. section: Core and Builtins + +Using :data:`NotImplemented` in a boolean context now raises +:exc:`TypeError`. Contributed by Jelle Zijlstra. + +.. + +.. date: 2024-05-07-16-57-56 +.. gh-issue: 118561 +.. nonce: wNMKVd +.. section: Core and Builtins + +Fix race condition in free-threaded build where :meth:`!list.extend` could +expose uninitialised memory to concurrent readers. + +.. + +.. date: 2024-05-01-05-09-16 +.. gh-issue: 117139 +.. nonce: t41w_D +.. section: Core and Builtins + +Convert the Python evaluation stack to use internal stack references. The +purpose is to support tagged pointers. In :pep:`703`, this will allow for +its form of deferred reference counting. For both the default and +free-threaded builds, this sets up the infrastructure for unboxed integers +in the future. + +.. + +.. date: 2024-04-27-18-36-46 +.. gh-issue: 115801 +.. nonce: SVeHSy +.. section: Core and Builtins + +Raise ``TypeError`` when passing a string to :func:`difflib.unified_diff` +and :func:`difflib.context_diff`. + +.. + +.. date: 2024-04-08-13-49-02 +.. gh-issue: 117558 +.. nonce: 9lSEpR +.. section: Core and Builtins + +Improve error messages when a string, bytes or bytearray object of length 1 +is expected. + +.. + +.. date: 2024-03-25-15-07-01 +.. gh-issue: 117195 +.. nonce: OWakgD +.. section: Core and Builtins + +Avoid assertion failure for debug builds when calling +``object.__sizeof__(1)`` + +.. + +.. date: 2024-03-19-22-21-22 +.. gh-issue: 116022 +.. nonce: iyHENN +.. section: Core and Builtins + +Improve the :meth:`~object.__repr__` output of :class:`~ast.AST` nodes. + +.. + +.. date: 2024-01-15-21-51-26 +.. gh-issue: 114091 +.. nonce: VOtSJl +.. section: Core and Builtins + +Changed the error message for awaiting something that can't be awaited from +"object can't be used in an await expression" to "'' object +can't be awaited". + +.. + +.. date: 2024-01-15-18-11-48 +.. gh-issue: 113190 +.. nonce: OwQX64 +.. section: Core and Builtins + +:c:func:`Py_Finalize` now deletes all interned strings. + +.. + +.. date: 2023-10-14-23-05-40 +.. gh-issue: 84978 +.. nonce: Z0t6dg +.. section: Core and Builtins + +Add class methods :meth:`float.from_number` and :meth:`complex.from_number`. + +.. + +.. date: 2022-07-22-15-56-35 +.. gh-issue: 95144 +.. nonce: FZYWX- +.. section: Core and Builtins + +Improve the error message from ``a in b`` when ``b`` is not a container to +mention the term "container". + +.. + +.. bpo: 24766 +.. date: 2018-10-09-15-14-53 +.. nonce: c_C1Wc +.. section: Core and Builtins + +Fix handling of ``doc`` argument to subclasses of ``property``. + +.. + +.. date: 2024-10-03-14-06-08 +.. gh-issue: 123961 +.. nonce: uwJQTY +.. section: C API + +Convert the :ref:`curses.window ` static type exposed +by the :c:macro:`!PyCursesWindow_Type` macro in ``Include/py_curses.h`` to a +:ref:`heap type `. Patch by Bénédikt Tran. + +.. + +.. date: 2024-09-25-11-44-02 +.. gh-issue: 124502 +.. nonce: qWuDjT +.. section: C API + +Add :c:func:`PyUnicode_Equal` function to the limited C API: test if two +strings are equal. Patch by Victor Stinner. + +.. + +.. date: 2024-09-24-20-34-21 +.. gh-issue: 124296 +.. nonce: S4QoS1 +.. section: C API + +:c:type:`PyDictObject` no longer maintains a private version tag field +``ma_version_tag`` per :pep:`699`. This field was originally added in +Python 3.6 (:pep:`509`) and deprecated in Python 3.12. + +.. + +.. date: 2024-09-18-18-40-30 +.. gh-issue: 124160 +.. nonce: Zy-VKi +.. section: C API + +Fix crash when importing modules containing state and single-phase +initialization in a subinterpreter. + +.. + +.. date: 2024-09-17-05-23-35 +.. gh-issue: 124153 +.. nonce: L8TWmx +.. section: C API + +Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for +easier type checking, related to :pep:`489` and :pep:`630`. + +.. + +.. date: 2024-09-16-16-21-39 +.. gh-issue: 124127 +.. nonce: LB8DBU +.. section: C API + +In the limited C API 3.14 and newer, :c:func:`Py_REFCNT` is now implemented +as an opaque function call to hide implementation details. Patch by Victor +Stinner. + +.. + +.. date: 2024-09-12-16-16-24 +.. gh-issue: 123880 +.. nonce: 2-8vcj +.. section: C API + +Fixed a bug that prevented circular imports of extension modules that use +single-phase initialization. + +.. + +.. date: 2024-09-10-16-54-27 +.. gh-issue: 123909 +.. nonce: CTGxDR +.. section: C API + +:c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases` and +:c:func:`PyType_FromModuleAndSpec` will now fail if the metaclass of the new +type has custom :c:member:`~PyTypeObject.tp_new`. + +.. + +.. date: 2024-08-30-14-02-17 +.. gh-issue: 107954 +.. nonce: TPvj4u +.. section: C API + +Add functions to configure the Python initialization (:pep:`741`): + +* :c:func:`PyInitConfig_Create` +* :c:func:`PyInitConfig_Free` +* :c:func:`PyInitConfig_GetError` +* :c:func:`PyInitConfig_GetExitCode` +* :c:func:`PyInitConfig_HasOption` +* :c:func:`PyInitConfig_GetInt` +* :c:func:`PyInitConfig_GetStr` +* :c:func:`PyInitConfig_GetStrList` +* :c:func:`PyInitConfig_FreeStrList` +* :c:func:`PyInitConfig_SetInt` +* :c:func:`PyInitConfig_SetStr` +* :c:func:`PyInitConfig_SetStrList` +* :c:func:`PyInitConfig_AddModule` +* :c:func:`Py_InitializeFromInitConfig` + +Patch by Victor Stinner. + +.. + +.. date: 2024-08-29-15-55-55 +.. gh-issue: 107954 +.. nonce: pr2O50 +.. section: C API + +Add functions to get and set the current runtime Python configuration: + +* :c:func:`PyConfig_Get` +* :c:func:`PyConfig_GetInt` +* :c:func:`PyConfig_Set` +* :c:func:`PyConfig_Names` + +Patch by Victor Stinner. + +.. + +.. date: 2024-08-29-15-05-19 +.. gh-issue: 123465 +.. nonce: eqwNWq +.. section: C API + +:c:macro:`Py_RELATIVE_OFFSET` is now allowed in :c:type:`PyMemberDef` for +the special offset member ``"__vectorcalloffset__"``, as well as the +discouraged special offset members ``"__dictoffset__"`` and +``"__weaklistoffset__"`` + +.. + +.. date: 2024-08-26-13-01-20 +.. gh-issue: 100554 +.. nonce: 0ku85o +.. section: C API + +Added a slot ``Py_tp_vectorcall`` to set +:c:member:`~PyTypeObject.tp_vectorcall` via the :c:func:`PyType_FromSpec` +function family. Limited API extensions can use this feature to provide more +efficient vector call-based implementation of ``__new__`` and ``__init__``. + +.. + +.. date: 2024-08-09-13-12-20 +.. gh-issue: 122854 +.. nonce: -1OgvU +.. section: C API + +Add :c:func:`Py_HashBuffer` to compute and return the hash value of a +buffer. Patch by Antoine Pitrou and Victor Stinner. + +.. + +.. date: 2024-08-06-14-23-11 +.. gh-issue: 122728 +.. nonce: l-fQ-v +.. section: C API + +Fix :c:func:`PyEval_GetLocals` to avoid :exc:`SystemError` ("bad argument to +internal function"). Patch by Victor Stinner. + +.. + +.. date: 2024-07-30-23-48-26 +.. gh-issue: 116622 +.. nonce: yTTtil +.. section: C API + +Make :any:`PyObject_Print` work around a bug in Android and OpenBSD which +prevented it from throwing an exception when trying to write to a read-only +stream. + +.. + +.. date: 2024-07-27-00-28-35 +.. gh-issue: 105201 +.. nonce: 0-xUWq +.. section: C API + +Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, which has an +ambiguous return value. Patch by Irit Katriel and Erlend Aasland. + +.. + +.. date: 2024-07-21-17-40-07 +.. gh-issue: 121489 +.. nonce: SUMFCr +.. section: C API + +Export private :c:func:`!_PyBytes_Join` again. + +.. + +.. date: 2024-07-12-13-40-59 +.. gh-issue: 121645 +.. nonce: 96QvD3 +.. section: C API + +Add :c:func:`PyBytes_Join(sep, iterable) ` function, similar +to ``sep.join(iterable)`` in Python. Patch by Victor Stinner. + +.. + +.. date: 2024-07-09-15-55-20 +.. gh-issue: 89364 +.. nonce: yYYroI +.. section: C API + +Export the :c:func:`PySignal_SetWakeupFd` function. Previously, the function +was documented but it couldn't be used in 3rd party code. Patch by Victor +Stinner. + +.. + +.. date: 2024-07-04-15-41-10 +.. gh-issue: 113993 +.. nonce: cLSiWV +.. section: C API + +:c:func:`PyUnicode_InternInPlace` no longer prevents its argument from being +garbage collected. + +Several functions that take ``char *`` are now documented as possibly +preventing string objects from being garbage collected; refer to their +documentation for details: :c:func:`PyUnicode_InternFromString`, +:c:func:`PyDict_SetItemString`, :c:func:`PyObject_SetAttrString`, +:c:func:`PyObject_DelAttrString`, :c:func:`PyUnicode_InternFromString`, and +``PyModule_Add*`` convenience functions. + +.. + +.. date: 2024-07-04-13-23-27 +.. gh-issue: 113601 +.. nonce: K3RLqp +.. section: C API + +Removed debug build assertions related to interning strings, which were +falsely triggered by stable ABI extensions. + +.. + +.. date: 2024-07-02-11-03-40 +.. gh-issue: 112136 +.. nonce: f3fiY8 +.. section: C API + +Restore the private ``_PyArg_Parser`` structure and the private +``_PyArg_ParseTupleAndKeywordsFast()`` function, previously removed in +Python 3.13 alpha 1. Patch by Victor Stinner. + +.. + +.. date: 2024-06-26-11-29-01 +.. gh-issue: 120642 +.. nonce: H7P9qK +.. section: C API + +Remove the private ``_Py_CODEUNIT`` type from the public C API. The internal +``pycore_code.h`` header should now be used to get this internal type. Patch +by Victor Stinner. + +.. + +.. date: 2024-06-21-16-41-21 +.. gh-issue: 120858 +.. nonce: Z5_-Mn +.. section: C API + +:c:func:`PyDict_Next` no longer locks the dictionary in the free-threaded +build. The locking needs to be done by the caller around the entire +iteration loop. + +.. + +.. date: 2024-06-19-21-27-42 +.. gh-issue: 120642 +.. nonce: UlKClN +.. section: C API + +Remove the following unstable functions: + +* ``PyUnstable_Replace_Executor()`` +* ``PyUnstable_SetOptimizer()`` +* ``PyUnstable_GetOptimizer()`` +* ``PyUnstable_GetExecutor()`` +* ``PyUnstable_Optimizer_NewCounter()`` +* ``PyUnstable_Optimizer_NewUOpOptimizer()`` + +Patch by Victor Stinner. + +.. + +.. date: 2024-06-19-17-27-22 +.. gh-issue: 120389 +.. nonce: GSZeHF +.. section: C API + +Add new functions to convert C ```` numbers from/to Python +:class:`int`: + +* :c:func:`PyLong_FromInt32` +* :c:func:`PyLong_FromUInt32` +* :c:func:`PyLong_FromInt64` +* :c:func:`PyLong_FromUInt64` +* :c:func:`PyLong_AsInt32` +* :c:func:`PyLong_AsUInt32` +* :c:func:`PyLong_AsInt64` +* :c:func:`PyLong_AsUInt64` + +Patch by Victor Stinner. + +.. + +.. date: 2024-06-16-22-58-47 +.. gh-issue: 120600 +.. nonce: TJdf0w +.. section: C API + +In the limited C API 3.14 and newer, :c:func:`Py_TYPE` is now implemented as +an opaque function call to hide implementation details. Patch by Victor +Stinner. + +.. + +.. date: 2024-06-11-21-38-32 +.. gh-issue: 70278 +.. nonce: WDE4zM +.. section: C API + +:c:func:`PyUnicode_FromFormat` no longer produces the ending ``\ufffd`` +character for truncated C string when use precision with ``%s`` and ``%V``. +It now truncates the string before the start of truncated multibyte +sequences. + +.. + +.. date: 2024-06-08-08-33-40 +.. gh-issue: 119771 +.. nonce: Oip2dL +.. section: C API + +Set :data:`errno` in :c:func:`_Py_c_pow` on overflows. Patch by Sergey B +Kirpichev. + +.. + +.. date: 2024-06-07-22-12-30 +.. gh-issue: 119182 +.. nonce: yt8Ar7 +.. section: C API + +Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` +object: + +* :c:func:`PyUnicodeWriter_Create`. +* :c:func:`PyUnicodeWriter_Discard`. +* :c:func:`PyUnicodeWriter_Finish`. +* :c:func:`PyUnicodeWriter_WriteChar`. +* :c:func:`PyUnicodeWriter_WriteUTF8`. +* :c:func:`PyUnicodeWriter_WriteUCS4`. +* :c:func:`PyUnicodeWriter_WriteWideChar`. +* :c:func:`PyUnicodeWriter_WriteStr`. +* :c:func:`PyUnicodeWriter_WriteRepr`. +* :c:func:`PyUnicodeWriter_WriteSubstring`. +* :c:func:`PyUnicodeWriter_Format`. +* :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + +Patch by Victor Stinner. + +.. + +.. date: 2024-06-04-10-58-20 +.. gh-issue: 119613 +.. nonce: qOr9GF +.. section: C API + +Soft deprecate the :c:macro:`!Py_MEMCPY` macro: use directly ``memcpy()`` +instead. Patch by Victor Stinner. + +.. + +.. date: 2024-05-30-12-51-21 +.. gh-issue: 119775 +.. nonce: CBq9IG +.. section: C API + +Creating :c:data:`immutable types ` with mutable +bases was deprecated since 3.12 and now raises a :exc:`TypeError`. + +.. + +.. date: 2024-05-29-21-05-59 +.. gh-issue: 119585 +.. nonce: Sn7JL3 +.. section: C API + +Fix crash when a thread state that was created by +:c:func:`PyGILState_Ensure` calls a destructor that during +:c:func:`PyThreadState_Clear` that calls back into +:c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`. This might +occur when in the free-threaded build or when using thread-local variables +whose destructors call :c:func:`PyGILState_Ensure`. + +.. + +.. date: 2024-05-29-09-21-37 +.. gh-issue: 119613 +.. nonce: J2xfrC +.. section: C API + +Macros ``Py_IS_NAN``, ``Py_IS_INFINITY`` and ``Py_IS_FINITE`` are +:term:`soft deprecated`. + +.. + +.. date: 2024-05-22-17-50-48 +.. gh-issue: 119336 +.. nonce: ff3qnS +.. section: C API + +Restore the removed ``_PyLong_NumBits()`` function. It is used by the +pywin32 project. Patch by Ethan Smith + +.. + +.. date: 2024-05-21-19-41-41 +.. gh-issue: 119344 +.. nonce: QKvzQb +.. section: C API + +The critical section API is now public as part of the non-limited C API. + +.. + +.. date: 2024-05-21-18-28-44 +.. gh-issue: 119333 +.. nonce: OTsYVX +.. section: C API + +Add :c:func:`PyContext_AddWatcher` and :c:func:`PyContext_ClearWatcher` APIs +to register callbacks to receive notification on enter and exit of context +objects. + +.. + +.. date: 2024-05-21-11-35-11 +.. gh-issue: 119247 +.. nonce: U6n6mh +.. section: C API + +Added ``Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST`` and +``Py_END_CRITICAL_SECTION_SEQUENCE_FAST`` macros to make it possible to use +PySequence_Fast APIs safely when free-threaded, and update str.join to work +without the GIL using them. + +.. + +.. date: 2024-05-20-10-35-22 +.. gh-issue: 111389 +.. nonce: a6axBk +.. section: C API + +Add :c:macro:`PyHASH_MULTIPLIER` constant: prime multiplier used in string +and various other hashes. Patch by Victor Stinner. + +.. + +.. date: 2024-05-10-15-43-14 +.. gh-issue: 118771 +.. nonce: 5KVglT +.. section: C API + +Several C declarations with names that didn't start with the ``Py`` or +``_Py`` prefixes, which were added by mistake in 3.13 alpha and beta +releases, were moved to internal headers. + +.. + +.. date: 2024-05-08-23-14-06 +.. gh-issue: 116984 +.. nonce: 5sgcDo +.. section: C API + +Make mimalloc includes relative to the current file to avoid embedders or +extensions needing to include ``Internal/mimalloc`` if they are already +including internal CPython headers. + +.. + +.. date: 2024-05-08-21-57-50 +.. gh-issue: 118789 +.. nonce: Ni4UQx +.. section: C API + +Add :c:func:`PyUnstable_Object_ClearWeakRefsNoCallbacks`, which clears +weakrefs without calling their callbacks. + +.. + +.. date: 2024-05-08-20-13-00 +.. gh-issue: 118789 +.. nonce: m88uUa +.. section: C API + +Restore ``_PyWeakref_ClearRef`` that was previously removed in Python 3.13 +alpha 1. + +.. + +.. date: 2024-04-10-16-48-04 +.. gh-issue: 117511 +.. nonce: RZtBRK +.. section: C API + +Make the :c:type:`PyMutex` public in the non-limited C API. + +.. + +.. date: 2024-03-10-14-55-51 +.. gh-issue: 116560 +.. nonce: x2mZaO +.. section: C API + +Add :c:func:`PyLong_GetSign` function. Patch by Sergey B Kirpichev. + +.. + +.. date: 2022-10-01-09-56-27 +.. gh-issue: 97588 +.. nonce: Gvg54o +.. section: C API + +Fix creating bitfields in :mod:`ctypes` structures and unions. Fields no +longer overlap. + +.. + +.. date: 2022-08-05-19-41-20 +.. gh-issue: 87135 +.. nonce: SCNBYj +.. section: C API + +Attempting to acquire the GIL after runtime finalization has begun in a +different thread now causes the thread to hang rather than terminate, which +avoids potential crashes or memory corruption caused by attempting to +terminate a thread that is running code not specifically designed to support +termination. In most cases this hanging is harmless since the process will +soon exit anyway. + +The ``PyThread_exit_thread`` function is now deprecated. Its behavior is +inconsistent across platforms, and it can only be used safely in the +unlikely case that every function in the entire call stack has been designed +to support the platform-dependent termination mechanism. It is recommended +that users of this function change their design to not require thread +termination. In the unlikely case that thread termination is needed and can +be done safely, users may migrate to calling platform-specific APIs such as +``pthread_exit`` (POSIX) or ``_endthreadex`` (Windows) directly. + +.. + +.. bpo: 34008 +.. date: 2018-06-30-21-48-16 +.. nonce: 2Wjtm0 +.. section: C API + +Added ``Py_IsInitialized`` to the list of APIs that are safe to call before +the interpreter is initialized, and updated the embedding tests to cover it. + +.. + +.. date: 2024-10-13-21-11-30 +.. gh-issue: 125269 +.. nonce: BC-fdo +.. section: Build + +Fix detection of whether ``-latomic`` is needed when cross-compiling CPython +using the configure script. + +.. + +.. date: 2024-09-23-11-27-25 +.. gh-issue: 123990 +.. nonce: d6HrYC +.. section: Build + +Remove ``WITH_FREELISTS`` macro and ``--without-freelists`` build +configuration + +.. + +.. date: 2024-09-16-09-42-05 +.. gh-issue: 124102 +.. nonce: Ow254j +.. section: Build + +Update internal documentation under PCbuild, so it now correctly states that +Windows requires VS2017 or later and Python 3.10 or later + +.. + +.. date: 2024-09-13-17-48-37 +.. gh-issue: 124043 +.. nonce: Bruxpq +.. section: Build + +Building using :option:`--with-trace-refs` is (temporarily) disallowed when +the GIL is disabled. + +.. + +.. date: 2024-09-04-12-01-43 +.. gh-issue: 123418 +.. nonce: ynzspB +.. section: Build + +Updated Android build to use OpenSSL 3.0.15. + +.. + +.. date: 2024-08-24-19-09-31 +.. gh-issue: 123297 +.. nonce: fdtXoe +.. section: Build + +Propagate the value of ``LDFLAGS`` to ``LDCXXSHARED`` in :mod:`sysconfig`. +Patch by Pablo Galindo + +.. + +.. date: 2024-08-14-19-17-34 +.. gh-issue: 121634 +.. nonce: eOMfHG +.. section: Build + +Allow for specifying the target compile triple for WASI. + +.. + +.. date: 2024-08-12-15-48-49 +.. gh-issue: 122578 +.. nonce: YJ3xEa +.. section: Build + +Use WASI SDK 24 for testing. + +.. + +.. date: 2024-08-07-00-20-18 +.. gh-issue: 116622 +.. nonce: U9cxHM +.. section: Build + +Rename build variable ``MODULE_LDFLAGS`` back to ``LIBPYTHON``, as it's used +by package build systems (e.g. Meson). + +.. + +.. date: 2024-08-02-12-59-15 +.. gh-issue: 118943 +.. nonce: vZQtET +.. section: Build + +Fix an issue where the experimental JIT could be built several times by the +``make regen-all`` target, leading to possible race conditions on heavily +parallelized builds. + +.. + +.. date: 2024-07-19-10-14-31 +.. gh-issue: 121996 +.. nonce: IEb2sz +.. section: Build + +Introduce ./configure --disable-safety and --enable-slower-safety options. +Patch by Donghee Na. + +.. + +.. date: 2024-07-18-07-53-07 +.. gh-issue: 120522 +.. nonce: dg3o5A +.. section: Build + +Added a :option:`--with-app-store-compliance` option to patch out known +issues with macOS/iOS App Store review processes. + +.. + +.. date: 2024-07-16-12-29-54 +.. gh-issue: 120371 +.. nonce: E7x858 +.. section: Build + +Support WASI SDK 22 by explicitly skipping functions that are just stubs in +wasi-libc. + +.. + +.. date: 2024-07-14-01-29-47 +.. gh-issue: 121731 +.. nonce: RMPGP3 +.. section: Build + +Fix mimalloc compile error on GNU/Hurd + +.. + +.. date: 2024-07-08-14-01-17 +.. gh-issue: 121487 +.. nonce: ekHmpR +.. section: Build + +Fix deprecation warning for ATOMIC_VAR_INIT in mimalloc. + +.. + +.. date: 2024-07-08-01-11-54 +.. gh-issue: 121467 +.. nonce: 3qWRQj +.. section: Build + +Fix a Makefile bug that prevented mimalloc header files from being +installed. + +.. + +.. date: 2024-07-02-20-16-09 +.. gh-issue: 121103 +.. nonce: TMef9j +.. section: Build + +On POSIX systems, excluding macOS framework installs, the lib directory for +the free-threaded build now includes a "t" suffix to avoid conflicts with a +co-located default build installation. + +.. + +.. date: 2024-07-02-12-42-25 +.. gh-issue: 120831 +.. nonce: i3eIjg +.. section: Build + +The default minimum iOS version was increased to 13.0. + +.. + +.. date: 2024-06-27-18-03-20 +.. gh-issue: 121082 +.. nonce: w3AfRx +.. section: Build + +Fix build failure when the developer use ``--enable-pystats`` arguments in +configuration command after #118450. + +.. + +.. date: 2024-06-21-09-24-03 +.. gh-issue: 120671 +.. nonce: Z8sBQB +.. section: Build + +Fix failing configure tests due to a missing space when appending to CFLAGS. + +.. + +.. date: 2024-06-19-21-05-15 +.. gh-issue: 120602 +.. nonce: UyDARz +.. section: Build + +Correctly handle LLVM installs with ``LLVM_VERSION_SUFFIX`` when building +with ``--enable-experimental-jit``. + +.. + +.. date: 2024-06-18-15-32-36 +.. gh-issue: 120688 +.. nonce: tjIPLD +.. section: Build + +On WASI in debug mode, Python is now built with compiler flag ``-O3`` +instead of ``-Og``, to support more recursive calls. Patch by Victor +Stinner. + +.. + +.. date: 2024-06-18-15-28-25 +.. gh-issue: 118943 +.. nonce: aie7nn +.. section: Build + +Fix a possible race condition affecting parallel builds configured with +``--enable-experimental-jit``, in which :exc:`FileNotFoundError` could be +caused by another process already moving ``jit_stencils.h.new`` to +``jit_stencils.h``. + +.. + +.. date: 2024-06-11-00-38-05 +.. gh-issue: 120326 +.. nonce: JHSDF1 +.. section: Build + +On Windows, fix build error when ``--disable-gil`` and +``--experimental-jit`` options are combined. + +.. + +.. date: 2024-06-09-15-54-22 +.. gh-issue: 120291 +.. nonce: IpfHzE +.. section: Build + +Make the ``python-config`` shell script compatible with non-bash shells. + +.. + +.. date: 2024-06-02-13-23-26 +.. gh-issue: 113565 +.. nonce: 8xBlId +.. section: Build + +Improve :mod:`curses` and :mod:`curses.panel` dependency checks in +:program:`configure`. + +.. + +.. date: 2024-05-29-17-40-50 +.. gh-issue: 119729 +.. nonce: k0xJ5U +.. section: Build + +On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI +flags, which may include debug ("d") and free-threaded ("t"). For example: +* ``python-3.14.pc`` (default, non-debug build) * ``python-3.14d.pc`` +(default, debug build) * ``python-3.14t.pc`` (free-threaded build) + +.. + +.. date: 2024-05-22-13-18-02 +.. gh-issue: 119400 +.. nonce: WEt83v +.. section: Build + +``make_ssl_certs``, the script that prepares certificate data for the test +suite, now allows specifying expiration dates. + +.. + +.. date: 2024-05-19-22-54-55 +.. gh-issue: 115119 +.. nonce: DwMwev +.. section: Build + +Fall back to the bundled libmpdec if a system version cannot be found. + +.. + +.. date: 2024-05-17-19-53-27 +.. gh-issue: 119132 +.. nonce: wepPgM +.. section: Build + +Update :data:`sys.version` to identify whether the build is default build or +free-threading build. Patch By Donghee Na. + +.. + +.. date: 2024-05-13-15-57-58 +.. gh-issue: 118836 +.. nonce: 7yN1iB +.. section: Build + +Fix an ``AssertionError`` when building with ``--enable-experimental-jit`` +and the compiler emits a ``SHT_NOTE`` section. + +.. + +.. date: 2024-05-11-15-11-30 +.. gh-issue: 118943 +.. nonce: VI_MnY +.. section: Build + +Fix a possible race condition affecting parallel builds configured with +``--enable-experimental-jit``, in which compilation errors could be caused +by an incompletely-generated header file. diff --git a/Misc/NEWS.d/3.5.0a1.rst b/Misc/NEWS.d/3.5.0a1.rst index 26b3d8253dbdee..8f9d8ce57caa05 100644 --- a/Misc/NEWS.d/3.5.0a1.rst +++ b/Misc/NEWS.d/3.5.0a1.rst @@ -1284,7 +1284,7 @@ Add function :func:`sys.is_finalizing` to know about interpreter shutdown. .. section: Library Add a default limit for the amount of data xmlrpclib.gzip_decode will -return. This resolves CVE-2013-1753. +return. This resolves :cve:`2013-1753`. .. @@ -1345,7 +1345,7 @@ newer worked. .. section: Library The "ip" command is now used on Linux to determine MAC address in -uuid.getnode(). Pach by Bruno Cauet. +uuid.getnode(). Patch by Bruno Cauet. .. @@ -3447,7 +3447,8 @@ tkinter.ttk now works when default root window is not set. .. nonce: FE_PII .. section: Library -_tkinter.create() now creates tkapp object with wantobject=1 by default. +``_tkinter.create()`` now creates ``tkapp`` object with ``wantobjects=1`` by +default. .. @@ -3930,7 +3931,7 @@ has been called. .. nonce: 5CDoox .. section: Library -New keyword argument ``unsafe`` to Mock. It raises ``AttributeError`` incase of +New keyword argument ``unsafe`` to Mock. It raises ``AttributeError`` in case of an attribute startswith assert or assret. .. @@ -4030,7 +4031,7 @@ unittest.mock.MagicMock now supports division. Patch by Johannes Baiter. .. section: Library Fix arbitrary memory access in JSONDecoder.raw_decode with a negative second -parameter. Bug reported by Guido Vranken. (See also: CVE-2014-4616) +parameter. Bug reported by Guido Vranken. (See also: :cve:`2014-4616`) .. @@ -4339,7 +4340,7 @@ these modules are not used. .. nonce: V1-XhC .. section: Library -Include the broadcast address in the usuable hosts for IPv6 in ipaddress. +Include the broadcast address in the usable hosts for IPv6 in ipaddress. .. @@ -5467,7 +5468,7 @@ All resources are now allowed when tests are not run by regrtest.py. .. section: Tests Fix pystone micro-benchmark: use floor division instead of true division to -benchmark integers instead of floating point numbers. Set pystone version to +benchmark integers instead of floating-point numbers. Set pystone version to 1.2. Patch written by Lennart Regebro. .. diff --git a/Misc/NEWS.d/3.5.2rc1.rst b/Misc/NEWS.d/3.5.2rc1.rst index a7e5c1b130f9e9..f9409b62e352ac 100644 --- a/Misc/NEWS.d/3.5.2rc1.rst +++ b/Misc/NEWS.d/3.5.2rc1.rst @@ -5,7 +5,7 @@ .. original section: Library .. section: Security -Update expat to 2.1.1, fixes CVE-2015-1283. +Update expat to 2.1.1, fixes :cve:`2015-1283`. .. @@ -15,8 +15,8 @@ Update expat to 2.1.1, fixes CVE-2015-1283. .. original section: Library .. section: Security -Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. Reported by Team -Oststrom +Fix TLS stripping vulnerability in smtplib, :cve:`2016-0772`. Reported by Team +Oststrom. .. diff --git a/Misc/NEWS.d/3.5.3.rst b/Misc/NEWS.d/3.5.3.rst index c3fcb67a4563f9..25db389ba5734f 100644 --- a/Misc/NEWS.d/3.5.3.rst +++ b/Misc/NEWS.d/3.5.3.rst @@ -3,5 +3,6 @@ .. no changes: True .. nonce: zYPqUK .. release date: 2017-01-17 +.. section: Library There were no code changes between 3.5.3rc1 and 3.5.3 final. diff --git a/Misc/NEWS.d/3.5.3rc1.rst b/Misc/NEWS.d/3.5.3rc1.rst index bf4ef9302c9d1d..2424604249a65c 100644 --- a/Misc/NEWS.d/3.5.3rc1.rst +++ b/Misc/NEWS.d/3.5.3rc1.rst @@ -1048,7 +1048,7 @@ certs. .. section: Library Remove 3DES from ssl module's default cipher list to counter measure sweet32 -attack (CVE-2016-2183). +attack (:cve:`2016-2183`). .. @@ -1251,7 +1251,7 @@ Fix possible integer overflow in the _csv module for large record lengths. .. nonce: OnuO9s .. section: Library -Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the HTTP_PROXY variable +Prevent HTTPoxy attack (:cve:`2016-1000110`). Ignore the HTTP_PROXY variable when REQUEST_METHOD environment is set, which indicates that the script is in CGI mode. diff --git a/Misc/NEWS.d/3.5.4rc1.rst b/Misc/NEWS.d/3.5.4rc1.rst index d65d5d14ee78bb..d5a85b3a2d8666 100644 --- a/Misc/NEWS.d/3.5.4rc1.rst +++ b/Misc/NEWS.d/3.5.4rc1.rst @@ -17,10 +17,10 @@ passing other environment variables and command arguments. .. section: Security Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security -vulnerabilities including: CVE-2017-9233 (External entity infinite loop -DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix -regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 -(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use +vulnerabilities including: :cve:`2017-9233` (External entity infinite loop +DoS), :cve:`2016-9063` (Integer overflow, re-fix), :cve:`2016-0718` (Fix +regression bugs from 2.2.0's fix to :cve:`2016-0718`) and :cve:`2012-0876` +(Counter hash flooding with SipHash). Note: the :cve:`2016-5300` (Use os-specific entropy sources like getrandom) doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. @@ -46,8 +46,8 @@ authentication (``login@host``). .. original section: Library .. section: Security -Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and -CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +Update expat copy from 2.1.1 to 2.2.0 to get fixes of :cve:`2016-0718` and +:cve:`2016-4472`. See https://sourceforge.net/p/expat/bugs/537/ for more information. .. diff --git a/Misc/NEWS.d/3.5.5rc1.rst b/Misc/NEWS.d/3.5.5rc1.rst index 9ccbf7b8060cd4..4a44840039e388 100644 --- a/Misc/NEWS.d/3.5.5rc1.rst +++ b/Misc/NEWS.d/3.5.5rc1.rst @@ -24,7 +24,7 @@ also be affected) .. nonce: Fd8kId .. section: Security -Fixed possible integer overflow in PyBytes_DecodeEscape, CVE-2017-1000158. +Fixed possible integer overflow in PyBytes_DecodeEscape, :cve:`2017-1000158`. Original patch by Jay Bosamiya; rebased to Python 3 by Miro Hrončok. .. diff --git a/Misc/NEWS.d/3.6.0.rst b/Misc/NEWS.d/3.6.0.rst index f9805cab28615e..d5c41f38838d93 100644 --- a/Misc/NEWS.d/3.6.0.rst +++ b/Misc/NEWS.d/3.6.0.rst @@ -3,5 +3,6 @@ .. no changes: True .. nonce: F9ENBV .. release date: 2016-12-23 +.. section: Library No changes since release candidate 2 diff --git a/Misc/NEWS.d/3.6.0a1.rst b/Misc/NEWS.d/3.6.0a1.rst index 144d217f6098a1..803c9fc5925fa6 100644 --- a/Misc/NEWS.d/3.6.0a1.rst +++ b/Misc/NEWS.d/3.6.0a1.rst @@ -1401,7 +1401,7 @@ array is extended. .. section: Library doctest.DocFileTest and doctest.testfile() now support packages (module -splitted into multiple directories) for the package parameter. +split into multiple directories) for the package parameter. .. @@ -1484,9 +1484,9 @@ on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). .. nonce: RWN1jR .. section: Library -Add C functions :c:func:`_PyTraceMalloc_Track` and -:c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the -:mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get +Add C functions :c:func:`!_PyTraceMalloc_Track` and +:c:func:`!_PyTraceMalloc_Untrack` to track memory blocks using the +:mod:`tracemalloc` module. Add :c:func:`!_PyTraceMalloc_GetTraceback` to get the traceback of an object. .. diff --git a/Misc/NEWS.d/3.6.0a2.rst b/Misc/NEWS.d/3.6.0a2.rst index 05b3d9f0463c1c..89d68ab3f8078f 100644 --- a/Misc/NEWS.d/3.6.0a2.rst +++ b/Misc/NEWS.d/3.6.0a2.rst @@ -5,7 +5,7 @@ .. original section: Library .. section: Security -Update expat to 2.1.1, fixes CVE-2015-1283. +Update expat to 2.1.1, fixes :cve:`2015-1283`. .. @@ -15,7 +15,7 @@ Update expat to 2.1.1, fixes CVE-2015-1283. .. original section: Library .. section: Security -Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. Reported by Team +Fix TLS stripping vulnerability in smtplib, :cve:`2016-0772`. Reported by Team Oststrom. .. diff --git a/Misc/NEWS.d/3.6.0a4.rst b/Misc/NEWS.d/3.6.0a4.rst index d613fd5d928b65..3abbdecb57038b 100644 --- a/Misc/NEWS.d/3.6.0a4.rst +++ b/Misc/NEWS.d/3.6.0a4.rst @@ -359,7 +359,7 @@ Fix possible integer overflow in the _csv module for large record lengths. .. nonce: OnuO9s .. section: Library -Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the HTTP_PROXY variable +Prevent HTTPoxy attack (:cve:`2016-1000110`). Ignore the HTTP_PROXY variable when REQUEST_METHOD environment is set, which indicates that the script is in CGI mode. diff --git a/Misc/NEWS.d/3.6.0b1.rst b/Misc/NEWS.d/3.6.0b1.rst index 4fb6bdd6f89c9b..bd54cf601d053b 100644 --- a/Misc/NEWS.d/3.6.0b1.rst +++ b/Misc/NEWS.d/3.6.0b1.rst @@ -949,7 +949,7 @@ Add scrypt (password-based key derivation function) to hashlib module .. section: Library Remove 3DES from ssl module's default cipher list to counter measure sweet32 -attack (CVE-2016-2183). +attack (:cve:`2016-2183`). .. diff --git a/Misc/NEWS.d/3.6.0b2.rst b/Misc/NEWS.d/3.6.0b2.rst index 9413c6e01917d5..23dd69efb23b88 100644 --- a/Misc/NEWS.d/3.6.0b2.rst +++ b/Misc/NEWS.d/3.6.0b2.rst @@ -215,7 +215,7 @@ memcpy(). .. nonce: e5xc1i .. section: Core and Builtins -Fix dict.pop() for splitted dictionary when trying to remove a "pending key" +Fix dict.pop() for split dictionary when trying to remove a "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. .. diff --git a/Misc/NEWS.d/3.6.2.rst b/Misc/NEWS.d/3.6.2.rst index dba43d146df954..ee50670bd9f442 100644 --- a/Misc/NEWS.d/3.6.2.rst +++ b/Misc/NEWS.d/3.6.2.rst @@ -3,5 +3,6 @@ .. no changes: True .. nonce: F9ENBV .. release date: 2017-07-17 +.. section: Library No changes since release candidate 2 diff --git a/Misc/NEWS.d/3.6.2rc1.rst b/Misc/NEWS.d/3.6.2rc1.rst index 28eb88f79130c5..8e28bc9691921b 100644 --- a/Misc/NEWS.d/3.6.2rc1.rst +++ b/Misc/NEWS.d/3.6.2rc1.rst @@ -5,8 +5,8 @@ .. original section: Library .. section: Security -Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and -CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +Update expat copy from 2.1.1 to 2.2.0 to get fixes of :cve:`2016-0718` and +:cve:`2016-4472`. See https://sourceforge.net/p/expat/bugs/537/ for more information. .. diff --git a/Misc/NEWS.d/3.6.2rc2.rst b/Misc/NEWS.d/3.6.2rc2.rst index 8c6545f6dbbeec..5ae7425828b692 100644 --- a/Misc/NEWS.d/3.6.2rc2.rst +++ b/Misc/NEWS.d/3.6.2rc2.rst @@ -17,10 +17,10 @@ passing other environment variables and command arguments. .. section: Security Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security -vulnerabilities including: CVE-2017-9233 (External entity infinite loop -DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix -regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 -(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use +vulnerabilities including: :cve:`2017-9233` (External entity infinite loop +DoS), :cve:`2016-9063` (Integer overflow, re-fix), :cve:`2016-0718` (Fix +regression bugs from 2.2.0's fix to :cve:`2016-0718`) and :cve:`2012-0876` +(Counter hash flooding with SipHash). Note: the :cve:`2016-5300` (Use os-specific entropy sources like getrandom) doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. diff --git a/Misc/NEWS.d/3.6.3rc1.rst b/Misc/NEWS.d/3.6.3rc1.rst index ebda7665e2b6ea..6a20e07f05956c 100644 --- a/Misc/NEWS.d/3.6.3rc1.rst +++ b/Misc/NEWS.d/3.6.3rc1.rst @@ -85,7 +85,7 @@ wrong line (typically the first line of the file). .. nonce: Kl_fS5 .. section: Core and Builtins -Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C libray +Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C library plans to remove the functions from sys/types.h. .. diff --git a/Misc/NEWS.d/3.6.5rc1.rst b/Misc/NEWS.d/3.6.5rc1.rst index 056bacb5267c41..a45e97fb29045b 100644 --- a/Misc/NEWS.d/3.6.5rc1.rst +++ b/Misc/NEWS.d/3.6.5rc1.rst @@ -15,7 +15,7 @@ Minimal fix to prevent buffer overrun in os.symlink on Windows Regexes in difflib and poplib were vulnerable to catastrophic backtracking. These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +refactored. This resolves :cve:`2018-1060` and :cve:`2018-1061`. Patch by Jamie Davis. .. @@ -474,7 +474,7 @@ platforms with OpenSSL 1.0.2+ or inet_pton. .. nonce: ideco .. section: Library -:func:`urllib.parse.urlsplit()` does not convert zone-id (scope) to lower +:func:`urllib.parse.urlsplit` does not convert zone-id (scope) to lower case for scoped IPv6 addresses in hostnames now. .. diff --git a/Misc/NEWS.d/3.7.0a1.rst b/Misc/NEWS.d/3.7.0a1.rst index aca79c4cc8c1b8..fd6ba07b53a617 100644 --- a/Misc/NEWS.d/3.7.0a1.rst +++ b/Misc/NEWS.d/3.7.0a1.rst @@ -46,10 +46,10 @@ passing other environment variables and command arguments. .. section: Security Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security -vulnerabilities including: CVE-2017-9233 (External entity infinite loop -DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix -regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 -(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use +vulnerabilities including: :cve:`2017-9233` (External entity infinite loop +DoS), :cve:`2016-9063` (Integer overflow, re-fix), :cve:`2016-0718` (Fix +regression bugs from 2.2.0's fix to :cve:`2016-0718`) and :cve:`2012-0876` +(Counter hash flooding with SipHash). Note: the :cve:`2016-5300` (Use os-specific entropy sources like getrandom) doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. @@ -75,8 +75,8 @@ authentication (``login@host``). .. original section: Library .. section: Security -Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and -CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +Update expat copy from 2.1.1 to 2.2.0 to get fixes of :cve:`2016-0718` and +:cve:`2016-4472`. See https://sourceforge.net/p/expat/bugs/537/ for more information. .. @@ -214,7 +214,7 @@ Fix possible undefined behavior in _PyObject_FastCall_Prepend. .. nonce: Kl_fS5 .. section: Core and Builtins -Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C libray +Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C library plans to remove the functions from sys/types.h. .. @@ -1479,7 +1479,7 @@ memcpy(). .. nonce: e5xc1i .. section: Core and Builtins -Fix dict.pop() for splitted dictionary when trying to remove a "pending key" +Fix dict.pop() for split dictionary when trying to remove a "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. .. @@ -2516,7 +2516,7 @@ stdin.write() if the child process is still running but closed the pipe. .. nonce: CdOuSl .. section: Library -Addded empty __slots__ to abc.ABC. This allows subclassers to deny __dict__ +Added empty __slots__ to abc.ABC. This allows subclassers to deny __dict__ and __weakref__ creation. Patch by Aaron Hall. .. diff --git a/Misc/NEWS.d/3.7.0a4.rst b/Misc/NEWS.d/3.7.0a4.rst index f2c6559037d84f..2ceb9e78e0421b 100644 --- a/Misc/NEWS.d/3.7.0a4.rst +++ b/Misc/NEWS.d/3.7.0a4.rst @@ -434,7 +434,7 @@ loop.getaddrinfo, loop.getnameinfo. .. nonce: ideco .. section: Library -:func:`urllib.parse.urlsplit()` does not convert zone-id (scope) to lower +:func:`urllib.parse.urlsplit` does not convert zone-id (scope) to lower case for scoped IPv6 addresses in hostnames now. .. @@ -463,7 +463,7 @@ Fix ``stop_serving`` in asyncio proactor loop kill all listening servers .. nonce: CUbsb2 .. section: Library -:func:`re.sub()` now replaces empty matches adjacent to a previous non-empty +:func:`re.sub` now replaces empty matches adjacent to a previous non-empty match. .. @@ -595,7 +595,7 @@ Add asyncio.get_running_loop() function. .. section: Library All class and static methods of builtin types now are correctly classified -by inspect.classify_class_attrs() and grouped in pydoc ouput. Added +by inspect.classify_class_attrs() and grouped in pydoc output. Added types.ClassMethodDescriptorType for unbound class methods of builtin types. .. diff --git a/Misc/NEWS.d/3.7.0b1.rst b/Misc/NEWS.d/3.7.0b1.rst index d1beec9cdcc33a..c9786e55c20739 100644 --- a/Misc/NEWS.d/3.7.0b1.rst +++ b/Misc/NEWS.d/3.7.0b1.rst @@ -414,7 +414,7 @@ callback. .. section: Library Fix memory leak in asyncio.Queue, when the queue has limited size and it is -full, the cancelation of queue.put() can cause a memory leak. Patch by: José +full, the cancellation of queue.put() can cause a memory leak. Patch by: José Melero. .. @@ -601,7 +601,7 @@ Add socket.getblocking() method. Add :mod:`importlib.resources` and :class:`importlib.abc.ResourceReader` as the unified API for reading resources contained within packages. Loaders wishing to support resource reading must implement the -:meth:`get_resource_reader()` method. File-based and zipimport-based +:meth:`get_resource_reader` method. File-based and zipimport-based loaders both implement these APIs. :class:`importlib.abc.ResourceLoader` is deprecated in favor of these new APIs. @@ -875,4 +875,4 @@ by Stéphane Wirtel .. section: C API Add C API access to the ``datetime.timezone`` constructor and -``datetime.timzone.UTC`` singleton. +``datetime.timezone.UTC`` singleton. diff --git a/Misc/NEWS.d/3.7.0b2.rst b/Misc/NEWS.d/3.7.0b2.rst index 702dbc960c018d..10cd57ea7edfce 100644 --- a/Misc/NEWS.d/3.7.0b2.rst +++ b/Misc/NEWS.d/3.7.0b2.rst @@ -274,7 +274,7 @@ collections.ChainMap() preserves the order of the underlying mappings. .. nonce: -T77_c .. section: Library -:func:`fnmatch.translate()` no longer produces patterns which contain set +:func:`fnmatch.translate` no longer produces patterns which contain set operations. Sets starting with '[' or containing '--', '&&', '~~' or '||' will be interpreted differently in regular expressions in future versions. Currently they emit warnings. fnmatch.translate() now avoids producing diff --git a/Misc/NEWS.d/3.7.0b3.rst b/Misc/NEWS.d/3.7.0b3.rst index c86963b7e42daf..a0c4cb15dc2b40 100644 --- a/Misc/NEWS.d/3.7.0b3.rst +++ b/Misc/NEWS.d/3.7.0b3.rst @@ -4,7 +4,7 @@ .. release date: 2018-03-29 .. section: Security -Harden ssl module against LibreSSL CVE-2018-8970. +Harden ssl module against LibreSSL :cve:`2018-8970`. X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test ensures that NULL bytes are not allowed. @@ -26,7 +26,7 @@ Minimal fix to prevent buffer overrun in os.symlink on Windows Regexes in difflib and poplib were vulnerable to catastrophic backtracking. These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +refactored. This resolves :cve:`2018-1060` and :cve:`2018-1061`. Patch by Jamie Davis. .. diff --git a/Misc/NEWS.d/3.7.0b4.rst b/Misc/NEWS.d/3.7.0b4.rst index b17c7e08d1d408..93627f54900ddd 100644 --- a/Misc/NEWS.d/3.7.0b4.rst +++ b/Misc/NEWS.d/3.7.0b4.rst @@ -46,8 +46,8 @@ Fix potential memory leak in ``normalizestring()``. Change dict growth function from ``round_up_to_power_2(used*2+hashtable_size/2)`` to -``round_up_to_power_2(used*3)``. Previously, dict is shrinked only when -``used == 0``. Now dict has more chance to be shrinked. +``round_up_to_power_2(used*3)``. Previously, dict is shrunk only when +``used == 0``. Now dict has more chance to be shrunk. .. @@ -235,7 +235,7 @@ End framing at the end of C implementation of :func:`pickle.Pickler.dump`. .. section: Library Improved error handling and fixed a reference leak in -:func:`os.posix_spawn()`. +:func:`os.posix_spawn`. .. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index bd9061601fe190..93995bc8feaad7 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -4,7 +4,7 @@ .. release date: 2019-02-03 .. section: Security -[CVE-2019-5010] Fix a NULL pointer deref in ssl module. The cert parser did +:cve:`2019-5010`: Fix a NULL pointer deref in ssl module. The cert parser did not handle CRL distribution points with empty DP or URI correctly. A malicious or buggy certificate can result into segfault. Vulnerability (TALOS-2018-0758) reported by Colin Read and Nicolas Edet of Cisco. @@ -50,7 +50,7 @@ files or create network connections. .. nonce: Ua9jMv .. section: Security -CVE-2018-14647: The C accelerated _elementtree module now initializes hash +:cve:`2018-14647`: The C accelerated _elementtree module now initializes hash randomization salt from _Py_HashSecret instead of libexpat's default CSPRNG. .. @@ -89,7 +89,7 @@ Fixed thread-safety of error handling in _ssl. .. nonce: TzSN4x .. section: Security -Harden ssl module against LibreSSL CVE-2018-8970. +Harden ssl module against LibreSSL :cve:`2018-8970`. X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test ensures that NULL bytes are not allowed. @@ -111,7 +111,7 @@ Minimal fix to prevent buffer overrun in os.symlink on Windows Regexes in difflib and poplib were vulnerable to catastrophic backtracking. These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +refactored. This resolves :cve:`2018-1060` and :cve:`2018-1061`. Patch by Jamie Davis. .. @@ -224,7 +224,7 @@ positives from posix, socket, time, test_io, and test_faulthandler. .. nonce: 9vMWSP .. section: Core and Builtins -Fix an assertion error in :func:`format` in debug build for floating point +Fix an assertion error in :func:`format` in debug build for floating-point formatting with "n" format, zero padding and small width. Release build is not impacted. Patch by Karthikeyan Singaravelan. @@ -625,7 +625,7 @@ Spytz. The C function ``property_descr_get()`` uses a "cached" tuple to optimize function calls. But this tuple can be discovered in debug mode with -:func:`sys.getobjects()`. Remove the optimization, it's not really worth it +:func:`sys.getobjects`. Remove the optimization, it's not really worth it and it causes 3 different crashes last years. .. @@ -1406,8 +1406,8 @@ Fix potential memory leak in ``normalizestring()``. Change dict growth function from ``round_up_to_power_2(used*2+hashtable_size/2)`` to -``round_up_to_power_2(used*3)``. Previously, dict is shrinked only when -``used == 0``. Now dict has more chance to be shrinked. +``round_up_to_power_2(used*3)``. Previously, dict is shrunk only when +``used == 0``. Now dict has more chance to be shrunk. .. @@ -2519,7 +2519,7 @@ non-Windows systems. .. nonce: dQS1ng .. section: Library -Fix incorrect parsing of :class:`_io.IncrementalNewlineDecoder`'s +Fix incorrect parsing of :class:`io.IncrementalNewlineDecoder`'s *translate* argument. .. @@ -2850,8 +2850,8 @@ allow for tweaking of protocols and also to add support by default for .. nonce: 37IdsA .. section: Library -Fixed integer overflow in the :meth:`~hashlib.shake.digest()` and -:meth:`~hashlib.shake.hexdigest()` methods for the SHAKE algorithm in the +Fixed integer overflow in the :meth:`~hashlib.shake.digest` and +:meth:`~hashlib.shake.hexdigest` methods for the SHAKE algorithm in the :mod:`hashlib` module. .. @@ -3211,10 +3211,10 @@ bytes objects. (microoptimization) .. nonce: i-F_E5 .. section: Library -Add :func:`~unittest.addModuleCleanup()` and -:meth:`~unittest.TestCase.addClassCleanup()` to unittest to support cleanups -for :func:`~unittest.setUpModule()` and -:meth:`~unittest.TestCase.setUpClass()`. Patch by Lisa Roach. +Add :func:`~unittest.addModuleCleanup` and +:meth:`~unittest.TestCase.addClassCleanup` to unittest to support cleanups +for :func:`~unittest.setUpModule` and +:meth:`~unittest.TestCase.setUpClass`. Patch by Lisa Roach. .. @@ -3458,7 +3458,7 @@ Running the :mod:`trace` module no longer creates the ``trace.cover`` file. .. section: Library Fix crash when an ``ABC``-derived class with invalid ``__subclasses__`` is -passed as the second argument to :func:`issubclass()`. Patch by Alexey +passed as the second argument to :func:`issubclass`. Patch by Alexey Izbyshev. .. @@ -3664,7 +3664,7 @@ Add pure Python fallback for functools.reduce. Patch by Robert Wright. .. section: Library The default asyncio task class now always has a name which can be get or set -using two new methods (:meth:`~asyncio.Task.get_name()` and +using two new methods (:meth:`~asyncio.Task.get_name` and :meth:`~asyncio.Task.set_name`) and is visible in the :func:`repr` output. An initial name can also be set using the new ``name`` keyword argument to :func:`asyncio.create_task` or the @@ -4152,12 +4152,12 @@ Convert content length to string before putting to header. :func:`~os.path.exists`, :func:`~os.path.lexists`, :func:`~os.path.isdir`, :func:`~os.path.isfile`, :func:`~os.path.islink`, and :func:`~os.path.ismount`, and :mod:`pathlib.Path` methods that return a -boolean result like :meth:`~pathlib.Path.exists()`, -:meth:`~pathlib.Path.is_dir()`, :meth:`~pathlib.Path.is_file()`, -:meth:`~pathlib.Path.is_mount()`, :meth:`~pathlib.Path.is_symlink()`, -:meth:`~pathlib.Path.is_block_device()`, -:meth:`~pathlib.Path.is_char_device()`, :meth:`~pathlib.Path.is_fifo()`, -:meth:`~pathlib.Path.is_socket()` now return ``False`` instead of raising +boolean result like :meth:`~pathlib.Path.exists`, +:meth:`~pathlib.Path.is_dir`, :meth:`~pathlib.Path.is_file`, +:meth:`~pathlib.Path.is_mount`, :meth:`~pathlib.Path.is_symlink`, +:meth:`~pathlib.Path.is_block_device`, +:meth:`~pathlib.Path.is_char_device`, :meth:`~pathlib.Path.is_fifo`, +:meth:`~pathlib.Path.is_socket` now return ``False`` instead of raising :exc:`ValueError` or its subclasses :exc:`UnicodeEncodeError` and :exc:`UnicodeDecodeError` for paths that contain characters or bytes unrepresentable at the OS level. @@ -5269,7 +5269,7 @@ performance and smaller size compared to protocol 3 introduced in Python .. section: Library Improved error handling and fixed a reference leak in -:func:`os.posix_spawn()`. +:func:`os.posix_spawn`. .. @@ -5857,7 +5857,7 @@ collections.ChainMap() preserves the order of the underlying mappings. .. nonce: -T77_c .. section: Library -:func:`fnmatch.translate()` no longer produces patterns which contain set +:func:`fnmatch.translate` no longer produces patterns which contain set operations. Sets starting with '[' or containing '--', '&&', '~~' or '||' will be interpreted differently in regular expressions in future versions. Currently they emit warnings. fnmatch.translate() now avoids producing @@ -6282,7 +6282,7 @@ Add documentation about the new command line interface of the gzip module. .. nonce: YO9CYm .. section: Documentation -chm document displays non-ASCII charaters properly on some MBCS Windows +chm document displays non-ASCII characters properly on some MBCS Windows systems. .. @@ -8051,7 +8051,7 @@ Update macOS 10.9+ installer to Tcl/Tk 8.6.8. .. nonce: K6jCVG .. section: macOS -In :mod:`_scproxy`, drop the GIL when calling into ``SystemConfiguration`` +In :mod:`!_scproxy`, drop the GIL when calling into ``SystemConfiguration`` to avoid deadlocks. .. diff --git a/Misc/NEWS.d/3.8.0a2.rst b/Misc/NEWS.d/3.8.0a2.rst index c8620aeea7f133..0dbfa2758fe601 100644 --- a/Misc/NEWS.d/3.8.0a2.rst +++ b/Misc/NEWS.d/3.8.0a2.rst @@ -202,7 +202,7 @@ the mean and standard deviation of measurement data as single entity. .. nonce: V88MCD .. section: Library -Added statistics.fmean() as a faster, floating point variant of the existing +Added statistics.fmean() as a faster, floating-point variant of the existing mean() function. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index fa5eb697d9202d..41eea7303bd899 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -13,7 +13,7 @@ Fixes mishandling of pre-normalization characters in urlsplit(). .. nonce: 51E-DA .. section: Security -Address CVE-2019-9740 by disallowing URL paths with embedded whitespace or +Address :cve:`2019-9740` by disallowing URL paths with embedded whitespace or control characters through into the underlying http client request. Such potentially malicious header injection URLs now cause an http.client.InvalidURL exception to be raised. @@ -600,7 +600,7 @@ exceptions. .. nonce: 9sjd38 .. section: Library -Add time module support and fix test_time faiures for VxWorks. +Add time module support and fix test_time failures for VxWorks. .. @@ -663,8 +663,8 @@ followed imports. Patch by Brandt Bucher. .. nonce: QmfNmY .. section: Library -Added :meth:`~socket.create_server()` and -:meth:`~socket.has_dualstack_ipv6()` convenience functions to automate the +Added :meth:`~socket.create_server` and +:meth:`~socket.has_dualstack_ipv6` convenience functions to automate the necessary tasks usually involved when creating a server socket, including accepting both IPv4 and IPv6 connections on the same socket. (Contributed by Giampaolo Rodola in :issue:`17561`.) @@ -843,7 +843,7 @@ Using the code of the ``Tools/scripts/serve.py`` script as an example in the .. nonce: nF1pP1 .. section: Documentation -Added Documention for PyInterpreterState_Main(). +Added documentation for PyInterpreterState_Main(). .. @@ -945,7 +945,7 @@ P. Hemsley. .. nonce: __FTq9 .. section: Tests -Add a new :mod:`_testinternalcapi` module to test the internal C API. +Add a new :mod:`!_testinternalcapi` module to test the internal C API. .. @@ -1383,7 +1383,7 @@ Since Python 3.7.0, calling :c:func:`Py_DecodeLocale` before coerced and/or if the UTF-8 Mode is enabled by the user configuration. The LC_CTYPE coercion and UTF-8 Mode are now disabled by default to fix the mojibake issue. They must now be enabled explicitly (opt-in) using the new -:c:func:`_Py_PreInitialize` API with ``_PyPreConfig``. +:c:func:`!_Py_PreInitialize` API with ``_PyPreConfig``. .. diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst index 4eb0c0451e97b5..5010473269d92f 100644 --- a/Misc/NEWS.d/3.8.0b1.rst +++ b/Misc/NEWS.d/3.8.0b1.rst @@ -4,7 +4,7 @@ .. release date: 2019-06-04 .. section: Security -CVE-2019-9948: Avoid file reading by disallowing ``local-file://`` and +:cve:`2019-9948`: Avoid file reading by disallowing ``local-file://`` and ``local_file://`` URL schemes in ``URLopener().open()`` and ``URLopener().retrieve()`` of :mod:`urllib.request`. @@ -176,8 +176,8 @@ Added new ``replace()`` method to the code type (:class:`types.CodeType`). .. nonce: d1SOtF .. section: Core and Builtins -Implement :func:`socket.if_nameindex()`, :func:`socket.if_nametoindex()`, -and :func:`socket.if_indextoname()` on Windows. +Implement :func:`socket.if_nameindex`, :func:`socket.if_nametoindex`, +and :func:`socket.if_indextoname` on Windows. .. @@ -538,7 +538,7 @@ module. .. nonce: TQFOR4 .. section: Library -:meth:`!msilib.Directory.start_component()` no longer fails if *keyfile* is +:meth:`!msilib.Directory.start_component` no longer fails if *keyfile* is not ``None``. .. @@ -600,7 +600,7 @@ default. .. nonce: sLULGQ .. section: Library -Fix destructor :class:`_pyio.BytesIO` and :class:`_pyio.TextIOWrapper`: +Fix destructor :class:`!_pyio.BytesIO` and :class:`!_pyio.TextIOWrapper`: initialize their ``_buffer`` attribute as soon as possible (in the class body), because it's used by ``__del__()`` which calls ``close()``. @@ -1371,7 +1371,7 @@ Asyncio: Remove inner callback on outer cancellation in shield .. nonce: d8djAJ .. section: Library -Fix :meth:`asyncio.SelectorEventLoop.subprocess_exec()` leaks file +Fix :meth:`asyncio.SelectorEventLoop.subprocess_exec` leaks file descriptors if ``Popen`` fails and called with ``stdin=subprocess.PIPE``. Patch by Niklas Fiekas. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 66d7fc1f32e705..d75132b0aac040 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -44,7 +44,7 @@ rendering the document page as HTML. (Contributed by Donghee Na in .. section: Security Update vendorized expat library version to 2.2.8, which resolves -CVE-2019-15903. +:cve:`2019-15903`. .. @@ -149,7 +149,7 @@ exception in :meth:`float.__getformat__`. .. nonce: 9-vKtO .. section: Core and Builtins -Optimized :func:`math.floor()`, :func:`math.ceil()` and :func:`math.trunc()` +Optimized :func:`math.floor`, :func:`math.ceil` and :func:`math.trunc` for floats. .. @@ -299,7 +299,7 @@ Check the error from the system's underlying ``crypt`` or ``crypt_r``. .. section: Core and Builtins On FreeBSD, Python no longer calls ``fedisableexcept()`` at startup to -control the floating point control mode. The call became useless since +control the floating-point control mode. The call became useless since FreeBSD 6: it became the default mode. .. @@ -1384,7 +1384,7 @@ Nested subclasses of :class:`typing.NamedTuple` are now pickleable. .. nonce: hwrPN7 .. section: Library -Prevent :exc:`KeyError` thrown by :func:`_encoded_words.decode` when given +Prevent :exc:`KeyError` thrown by :func:`!_encoded_words.decode` when given an encoded-word with invalid content-type encoding from propagating all the way to :func:`email.message.get`. @@ -1395,8 +1395,8 @@ way to :func:`email.message.get`. .. nonce: S6Klvm .. section: Library -Deprecated the ``split()`` method in :class:`_tkinter.TkappType` in favour -of the ``splitlist()`` method which has more consistent and predicable +Deprecated the ``split()`` method in :class:`!_tkinter.TkappType` in favour +of the ``splitlist()`` method which has more consistent and predictable behavior. .. @@ -2990,7 +2990,7 @@ mode. .. nonce: FRGH4I .. section: Library -:func:`ctypes.create_unicode_buffer()` now also supports non-BMP characters +:func:`ctypes.create_unicode_buffer` now also supports non-BMP characters on platforms with 16-bit :c:type:`wchar_t` (for example, Windows and AIX). .. @@ -3013,7 +3013,7 @@ thread was still running. .. section: Library Allow pure Python implementation of :mod:`pickle` to work even when the C -:mod:`_pickle` module is unavailable. +:mod:`!_pickle` module is unavailable. .. @@ -3054,7 +3054,7 @@ Change the format of feature_version to be a (major, minor) tuple. .. nonce: 5_mJkQ .. section: Library -Eliminate :exc:`RuntimeError` raised by :func:`asyncio.all_tasks()` if +Eliminate :exc:`RuntimeError` raised by :func:`asyncio.all_tasks` if internal tasks weak set is changed by another thread during iteration. .. @@ -3064,8 +3064,8 @@ internal tasks weak set is changed by another thread during iteration. .. nonce: ADqCkq .. section: Library -:class:`_pyio.IOBase` destructor now does nothing if getting the ``closed`` -attribute fails to better mimic :class:`_io.IOBase` finalizer. +:class:`!_pyio.IOBase` destructor now does nothing if getting the ``closed`` +attribute fails to better mimic :class:`!_io.IOBase` finalizer. .. @@ -3536,7 +3536,7 @@ Add :meth:`~pathlib.Path.readlink`. Patch by Girts Folkmanis. .. nonce: La3TZz .. section: Library -Made :func:`urllib.parse.unquote()` accept bytes in addition to strings. +Made :func:`urllib.parse.unquote` accept bytes in addition to strings. Patch by Stein Karlsen. .. @@ -3839,7 +3839,7 @@ Added possible exceptions to the description of os.chdir(). .. nonce: r_wGRc .. section: Documentation -Documented that in :meth:`datetime.datetime.strptime()`, the leading zero in +Documented that in :meth:`datetime.datetime.strptime`, the leading zero in some two-digit formats is optional. Patch by Mike Gleen. .. @@ -4993,7 +4993,7 @@ Make :const:`winreg.REG_MULTI_SZ` support zero-length strings. .. section: Windows Replace use of :c:func:`strcasecmp` for the system function -:c:func:`_stricmp`. Patch by Minmin Gong. +:c:func:`!_stricmp`. Patch by Minmin Gong. .. @@ -5616,7 +5616,7 @@ heap type .. nonce: 4DcUaI .. section: C API -Add :c:func:`_PyObject_FunctionStr` to get a user-friendly string +Add :c:func:`!_PyObject_FunctionStr` to get a user-friendly string representation of a function-like object. Patch by Jeroen Demeyer. .. @@ -5696,8 +5696,8 @@ Add :c:func:`PyConfig_SetWideStringList` function. .. section: C API Add fast functions for calling methods: -:c:func:`_PyObject_VectorcallMethod`, :c:func:`_PyObject_CallMethodNoArgs` -and :c:func:`_PyObject_CallMethodOneArg`. +:c:func:`!_PyObject_VectorcallMethod`, :c:func:`!_PyObject_CallMethodNoArgs` +and :c:func:`!_PyObject_CallMethodOneArg`. .. diff --git a/Misc/NEWS.d/3.9.0a2.rst b/Misc/NEWS.d/3.9.0a2.rst index 7d878cfe227552..39b1c308312aa4 100644 --- a/Misc/NEWS.d/3.9.0a2.rst +++ b/Misc/NEWS.d/3.9.0a2.rst @@ -844,7 +844,7 @@ test.regrtest now can receive a list of test patterns to ignore (using the .. nonce: cNsA7S .. section: Build -:mod:`asyncio` now raises :exc:`TyperError` when calling incompatible +:mod:`asyncio` now raises :exc:`TypeError` when calling incompatible methods with an :class:`ssl.SSLSocket` socket. Patch by Ido Michael. .. diff --git a/Misc/NEWS.d/3.9.0a4.rst b/Misc/NEWS.d/3.9.0a4.rst index ca0eb2abf1d654..cce0c4c9acdf1b 100644 --- a/Misc/NEWS.d/3.9.0a4.rst +++ b/Misc/NEWS.d/3.9.0a4.rst @@ -755,7 +755,7 @@ dependencies. .. nonce: X7FRaN .. section: Windows -:meth:`~pathlib.Path.home()` and :meth:`~pathlib.Path.expanduser()` on +:meth:`~pathlib.Path.home` and :meth:`~pathlib.Path.expanduser` on Windows now prefer :envvar:`USERPROFILE` and no longer use :envvar:`HOME`, which is not normally set for regular user accounts. This makes them again behave like :func:`os.path.expanduser`, which was changed to ignore diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index f0015ac54df307..9402e5077c2e77 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -5,7 +5,7 @@ .. section: Security Disallow control characters in hostnames in http.client, addressing -CVE-2019-18348. Such potentially malicious header injection URLs now cause a +:cve:`2019-18348`. Such potentially malicious header injection URLs now cause a InvalidURL to be raised. .. @@ -548,7 +548,7 @@ large for an AF_UNIX socket address. Patch by Pablo Galindo. .. nonce: mxr5m8 .. section: Library -:func:`ast.dump()` no longer outputs optional fields and attributes with +:func:`ast.dump` no longer outputs optional fields and attributes with default values. The default values for optional fields and attributes of AST nodes are now set as class attributes (e.g. ``Constant.kind`` is set to ``None``). diff --git a/Misc/NEWS.d/3.9.0a6.rst b/Misc/NEWS.d/3.9.0a6.rst index 366a260172efb8..11309fccc0a20c 100644 --- a/Misc/NEWS.d/3.9.0a6.rst +++ b/Misc/NEWS.d/3.9.0a6.rst @@ -23,7 +23,7 @@ header injection attacks. .. nonce: B299Yq .. section: Security -CVE-2020-8492: The :class:`~urllib.request.AbstractBasicAuthHandler` class +:cve:`2020-8492`: The :class:`~urllib.request.AbstractBasicAuthHandler` class of the :mod:`urllib.request` module uses an inefficient regular expression which can be exploited by an attacker to cause a denial of service. Fix the regex to prevent the catastrophic backtracking. Vulnerability reported by @@ -111,7 +111,7 @@ str.decode(). .. nonce: m15TTX .. section: Core and Builtins -Fix possible refleaks in :mod:`_json`, memo of PyScannerObject should be +Fix possible refleaks in :mod:`!_json`, memo of PyScannerObject should be traversed. .. @@ -403,7 +403,7 @@ after encoding it to utf-8, not before. .. nonce: pDZR6V .. section: Library -Added :meth:`pathlib.Path.with_stem()` to create a new Path with the stem +Added :meth:`pathlib.Path.with_stem` to create a new Path with the stem replaced. .. @@ -564,7 +564,7 @@ Implement traverse and clear slots in _abc._abc_data type. .. nonce: 3rO_q7 .. section: Library -Remove deprecated :meth:`symtable.SymbolTable.has_exec`. +Remove deprecated :meth:`!symtable.SymbolTable.has_exec`. .. @@ -635,7 +635,7 @@ script is killed by signal 11, it now logs: "CGI script exit code -11." .. section: Library Improve the error message when triying to import a module using :mod:`runpy` -and incorrently use the ".py" extension at the end of the module name. Patch +and incorrectly using the ".py" extension at the end of the module name. Patch by Pablo Galindo. .. @@ -666,8 +666,8 @@ for _main_thread, instead of a _DummyThread instance. .. nonce: VTq_8s .. section: Library -Add a private ``_at_fork_reinit()`` method to :class:`_thread.Lock`, -:class:`_thread.RLock`, :class:`threading.RLock` and +Add a private ``_at_fork_reinit()`` method to :class:`!_thread.Lock`, +:class:`!_thread.RLock`, :class:`threading.RLock` and :class:`threading.Condition` classes: reinitialize the lock at fork in the child process, reset the lock to the unlocked state. Rename also the private ``_reset_internal_locks()`` method of :class:`threading.Event` to @@ -866,7 +866,7 @@ of source of the class. Patch by Karthikeyan Singaravelan. .. nonce: vHC7YQ .. section: Library -Deprecate passing None as an argument for :func:`shlex.split()`'s ``s`` +Deprecate passing None as an argument for :func:`shlex.split`'s ``s`` parameter. Patch by Zackery Spytz. .. @@ -1118,7 +1118,7 @@ into an exit code. .. nonce: _FOf7E .. section: C API -Move the :c:type:`PyGC_Head` structure to the internal C API. +Move the :c:type:`!PyGC_Head` structure to the internal C API. .. @@ -1149,8 +1149,8 @@ the garbage collector respectively. Patch by Pablo Galindo. .. nonce: Seuh3D .. section: C API -The :c:func:`PyObject_NEW` macro becomes an alias to the -:c:func:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro +The :c:func:`!PyObject_NEW` macro becomes an alias to the +:c:func:`PyObject_New` macro, and the :c:func:`!PyObject_NEW_VAR` macro becomes an alias to the :c:func:`PyObject_NewVar` macro, to hide implementation details. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. @@ -1174,7 +1174,7 @@ used. .. nonce: 6nFYbY .. section: C API -Convert the :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro to a function to +Convert the :c:func:`!PyObject_GET_WEAKREFS_LISTPTR` macro to a function to hide implementation details: the macro accessed directly to the :c:member:`PyTypeObject.tp_weaklistoffset` member. diff --git a/Misc/NEWS.d/3.9.0b1.rst b/Misc/NEWS.d/3.9.0b1.rst index 40fb8474bf9364..9a3630ddf216a8 100644 --- a/Misc/NEWS.d/3.9.0b1.rst +++ b/Misc/NEWS.d/3.9.0b1.rst @@ -532,7 +532,7 @@ Remove ``_random.Random.randbytes()``: the C implementation of .. section: Library Added default arguments to -:meth:`difflib.SequenceMatcher.find_longest_match()`. +:meth:`difflib.SequenceMatcher.find_longest_match`. .. diff --git a/Misc/NEWS.d/next/Build/2024-02-01-20-08-11.gh-issue-114875.x_2iZ9.rst b/Misc/NEWS.d/next/Build/2024-02-01-20-08-11.gh-issue-114875.x_2iZ9.rst deleted file mode 100644 index 20e9d6376b973c..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-01-20-08-11.gh-issue-114875.x_2iZ9.rst +++ /dev/null @@ -1 +0,0 @@ -Add :c:func:`!getgrent` as a prerequisite for building the :mod:`grp` module. diff --git a/Misc/NEWS.d/next/Build/2024-02-08-17-38-56.gh-issue-113632.y9KIGb.rst b/Misc/NEWS.d/next/Build/2024-02-08-17-38-56.gh-issue-113632.y9KIGb.rst deleted file mode 100644 index 8b02b1b2cd08c9..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-08-17-38-56.gh-issue-113632.y9KIGb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Promote WASI to a tier 2 platform and drop Emscripten from tier 3 in -configure.ac. diff --git a/Misc/NEWS.d/next/Build/2024-02-08-19-36-20.gh-issue-115167.LB9nDK.rst b/Misc/NEWS.d/next/Build/2024-02-08-19-36-20.gh-issue-115167.LB9nDK.rst deleted file mode 100644 index c60c4a93fe8906..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-08-19-36-20.gh-issue-115167.LB9nDK.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid vendoring ``vcruntime140_threads.dll`` when building with Visual Studio 2022 version 17.8. diff --git a/Misc/NEWS.d/next/Build/2024-03-03-20-28-23.gh-issue-115382.97hJFE.rst b/Misc/NEWS.d/next/Build/2024-03-03-20-28-23.gh-issue-115382.97hJFE.rst new file mode 100644 index 00000000000000..f8d19651fc5854 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-03-03-20-28-23.gh-issue-115382.97hJFE.rst @@ -0,0 +1 @@ +Fix cross compile failures when the host and target SOABIs match. diff --git a/Misc/NEWS.d/next/Build/2024-10-04-17-29-23.gh-issue-124928.FsGffe.rst b/Misc/NEWS.d/next/Build/2024-10-04-17-29-23.gh-issue-124928.FsGffe.rst new file mode 100644 index 00000000000000..291bf336ef225e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-04-17-29-23.gh-issue-124928.FsGffe.rst @@ -0,0 +1 @@ +Emscripten builds now require node >= 18. diff --git a/Misc/NEWS.d/next/Build/2024-10-16-09-37-51.gh-issue-89640.UDsW-j.rst b/Misc/NEWS.d/next/Build/2024-10-16-09-37-51.gh-issue-89640.UDsW-j.rst new file mode 100644 index 00000000000000..5aba2c789b6842 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-16-09-37-51.gh-issue-89640.UDsW-j.rst @@ -0,0 +1,2 @@ +Improve detection of float word ordering on Linux when link-time optimizations +are enabled. diff --git a/Misc/NEWS.d/next/Build/2024-10-24-22-14-35.gh-issue-125940.2wMtTA.rst b/Misc/NEWS.d/next/Build/2024-10-24-22-14-35.gh-issue-125940.2wMtTA.rst new file mode 100644 index 00000000000000..2b4c1c95db8806 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-24-22-14-35.gh-issue-125940.2wMtTA.rst @@ -0,0 +1,2 @@ +The Android build now supports `16 KB page sizes +`__. diff --git a/Misc/NEWS.d/next/Build/2024-10-25-00-29-15.gh-issue-125946.KPA3g0.rst b/Misc/NEWS.d/next/Build/2024-10-25-00-29-15.gh-issue-125946.KPA3g0.rst new file mode 100644 index 00000000000000..ecab57c8111d45 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-25-00-29-15.gh-issue-125946.KPA3g0.rst @@ -0,0 +1 @@ +The minimum supported Android version is now 7.0 (API level 24). diff --git a/Misc/NEWS.d/next/Build/2024-10-25-17-20-50.gh-issue-124932.F-aNuS.rst b/Misc/NEWS.d/next/Build/2024-10-25-17-20-50.gh-issue-124932.F-aNuS.rst new file mode 100644 index 00000000000000..10c4171dc14a64 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-25-17-20-50.gh-issue-124932.F-aNuS.rst @@ -0,0 +1,4 @@ +For cross builds, there is now support for having a different install +``prefix`` than the ``host_prefix`` used by ``getpath.py``. This is set to ``/`` by +default for Emscripten, on other platforms the default behavior is the same +as before. diff --git a/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst b/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst new file mode 100644 index 00000000000000..24b172e1747403 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst @@ -0,0 +1,2 @@ +``make clinic`` now runs Argument Clinic using the ``--force`` option, +thus forcefully regenerating generated code. diff --git a/Misc/NEWS.d/next/C API/2023-11-15-13-47-48.gh-issue-112066.22WsqR.rst b/Misc/NEWS.d/next/C API/2023-11-15-13-47-48.gh-issue-112066.22WsqR.rst deleted file mode 100644 index ae2b8b2444de97..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-11-15-13-47-48.gh-issue-112066.22WsqR.rst +++ /dev/null @@ -1,5 +0,0 @@ -Add :c:func:`PyDict_SetDefaultRef`: insert a key and value into a dictionary -if the key is not already present. This is similar to -:meth:`dict.setdefault`, but returns an integer value indicating if the key -was already present. It is also similar to :c:func:`PyDict_SetDefault`, but -returns a strong reference instead of a borrowed reference. diff --git a/Misc/NEWS.d/next/C API/2024-01-23-21-45-02.gh-issue-114329.YRaBoe.rst b/Misc/NEWS.d/next/C API/2024-01-23-21-45-02.gh-issue-114329.YRaBoe.rst deleted file mode 100644 index 62d4ce0cfb8de5..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-01-23-21-45-02.gh-issue-114329.YRaBoe.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :c:func:`PyList_GetItemRef`, which is similar to -:c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of a -:term:`borrowed reference`. diff --git a/Misc/NEWS.d/next/C API/2024-01-29-12-13-24.gh-issue-114685.B07RME.rst b/Misc/NEWS.d/next/C API/2024-01-29-12-13-24.gh-issue-114685.B07RME.rst deleted file mode 100644 index 55b02d1d8e1e9f..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-01-29-12-13-24.gh-issue-114685.B07RME.rst +++ /dev/null @@ -1,3 +0,0 @@ -:c:func:`PyObject_GetBuffer` now raises a :exc:`SystemError` if called with -:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should -only be used with the ``PyMemoryView_*`` C API. diff --git a/Misc/NEWS.d/next/C API/2024-01-31-15-43-35.gh-issue-114685.n7aRmX.rst b/Misc/NEWS.d/next/C API/2024-01-31-15-43-35.gh-issue-114685.n7aRmX.rst deleted file mode 100644 index 76ff00645fe57d..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-01-31-15-43-35.gh-issue-114685.n7aRmX.rst +++ /dev/null @@ -1,3 +0,0 @@ -:c:func:`PyBuffer_FillInfo` now raises a :exc:`SystemError` if called with -:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should -only be used with the ``PyMemoryView_*`` C API. diff --git a/Misc/NEWS.d/next/C_API/2024-06-04-13-38-44.gh-issue-120026.uhEvJ9.rst b/Misc/NEWS.d/next/C_API/2024-06-04-13-38-44.gh-issue-120026.uhEvJ9.rst new file mode 100644 index 00000000000000..d43a138724ad33 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-06-04-13-38-44.gh-issue-120026.uhEvJ9.rst @@ -0,0 +1 @@ +The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`. diff --git a/Misc/NEWS.d/next/C_API/2024-07-30-14-40-08.gh-issue-121654.tgGeAl.rst b/Misc/NEWS.d/next/C_API/2024-07-30-14-40-08.gh-issue-121654.tgGeAl.rst new file mode 100644 index 00000000000000..134d36c281ab21 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-07-30-14-40-08.gh-issue-121654.tgGeAl.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyType_Freeze` function to make a type immutable. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2024-10-16-19-28-23.gh-issue-125608.gTsU2g.rst b/Misc/NEWS.d/next/C_API/2024-10-16-19-28-23.gh-issue-125608.gTsU2g.rst new file mode 100644 index 00000000000000..e70f9f173957a2 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-10-16-19-28-23.gh-issue-125608.gTsU2g.rst @@ -0,0 +1,3 @@ +Fix a bug where dictionary watchers (e.g., :c:func:`PyDict_Watch`) on an +object's attribute dictionary (:attr:`~object.__dict__`) were not triggered +when the object's attributes were modified. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-16-06-52-34.gh-issue-104530.mJnA0W.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-16-06-52-34.gh-issue-104530.mJnA0W.rst deleted file mode 100644 index 8643a25ae51b13..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-16-06-52-34.gh-issue-104530.mJnA0W.rst +++ /dev/null @@ -1 +0,0 @@ -Use native Win32 condition variables. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst deleted file mode 100644 index 438be985496650..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst +++ /dev/null @@ -1 +0,0 @@ -Add warning when creating :class:`type` using a namespace dictionary with non-string keys. Patched by Daniel Urban and Furkan Onder. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst deleted file mode 100644 index 90f49272218c96..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst +++ /dev/null @@ -1,5 +0,0 @@ -Make interp->obmalloc a pointer. For interpreters that share state with the -main interpreter, this points to the same static memory structure. For -interpreters with their own obmalloc state, it is heap allocated. Add -free_obmalloc_arenas() which will free the obmalloc arenas and radix tree -structures for interpreters with their own obmalloc state. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst deleted file mode 100644 index bdee4d645f61c8..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add an option (``--enable-experimental-jit`` for ``configure``-based builds -or ``--experimental-jit`` for ``PCbuild``-based ones) to build an -*experimental* just-in-time compiler, based on `copy-and-patch -`_ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst deleted file mode 100644 index e5f3d5ea0cea25..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst +++ /dev/null @@ -1 +0,0 @@ -Convert :class:`collections.deque` to use Argument Clinic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-12-16-40-07.gh-issue-113939.Yi3L-e.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-12-16-40-07.gh-issue-113939.Yi3L-e.rst deleted file mode 100644 index 28b8e4bdda6be4..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-12-16-40-07.gh-issue-113939.Yi3L-e.rst +++ /dev/null @@ -1,4 +0,0 @@ -frame.clear(): -Clear frame.f_locals as well, and not only the fast locals. -This is relevant once frame.f_locals was accessed, -which would contain also references to all the locals. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst deleted file mode 100644 index 6a39fd2f60ab81..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst +++ /dev/null @@ -1 +0,0 @@ -Make :class:`queue.SimpleQueue` thread safe when the GIL is disabled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst deleted file mode 100644 index ed45ba49c3ad42..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst +++ /dev/null @@ -1,2 +0,0 @@ -The ``END_FOR`` instruction now pops only one value. This is to better -support side exits in loops. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-23-39-20.gh-issue-114050.Lnv1oq.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-23-39-20.gh-issue-114050.Lnv1oq.rst deleted file mode 100644 index c35d2508e6bdda..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-23-39-20.gh-issue-114050.Lnv1oq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix segmentation fault caused by an incorrect format string -in ``TypeError`` exception when more than two arguments are passed to ``int``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst deleted file mode 100644 index b3aa43801da488..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst +++ /dev/null @@ -1,3 +0,0 @@ -The free-threaded build now has its own thread-safe GC implementation that -uses mimalloc to find GC tracked objects. It is non-generational, unlike the -existing GC implementation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-19-13-18-13.gh-issue-114265.7HAi--.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-19-13-18-13.gh-issue-114265.7HAi--.rst deleted file mode 100644 index 74affbbd09ffb4..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-19-13-18-13.gh-issue-114265.7HAi--.rst +++ /dev/null @@ -1 +0,0 @@ -Compiler propagates line numbers before optimization, leading to more optimization opportunities and removing the need for the ``guarantee_lineno_for_exits`` hack. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-21-17-29-32.gh-issue-114388.UVGO4K.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-21-17-29-32.gh-issue-114388.UVGO4K.rst deleted file mode 100644 index 52c2742001d9ca..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-21-17-29-32.gh-issue-114388.UVGO4K.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix a :exc:`RuntimeWarning` emitted when assign an integer-like value that -is not an instance of :class:`int` to an attribute that corresponds to a C -struct member of :ref:`type ` T_UINT and T_ULONG. Fix a -double :exc:`RuntimeWarning` emitted when assign a negative integer value to -an attribute that corresponds to a C struct member of type T_UINT. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-09-49-02.gh-issue-114083.hf1-ku.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-09-49-02.gh-issue-114083.hf1-ku.rst deleted file mode 100644 index 79be45e87b90d3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-09-49-02.gh-issue-114083.hf1-ku.rst +++ /dev/null @@ -1 +0,0 @@ -Compiler applies folding of LOAD_CONST with following instruction in a separate pass before other optimisations. This enables jump threading in certain circumstances. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst deleted file mode 100644 index 2b30ad98fb5c79..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst +++ /dev/null @@ -1 +0,0 @@ -Lower the recursion limit under a debug build of WASI. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-25-18-50-49.gh-issue-112529.IbbApA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-25-18-50-49.gh-issue-112529.IbbApA.rst deleted file mode 100644 index 2a6d74fb222702..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-25-18-50-49.gh-issue-112529.IbbApA.rst +++ /dev/null @@ -1,4 +0,0 @@ -The free-threaded build no longer allocates space for the ``PyGC_Head`` -structure in objects that support cyclic garbage collection. A number of -other fields and data structures are used as replacements, including -``ob_gc_bits``, ``ob_tid``, and mimalloc internal data structures. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-31-09-10-10.gh-issue-107944.XWm1B-.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-31-09-10-10.gh-issue-107944.XWm1B-.rst deleted file mode 100644 index 8e3fb786c11055..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-31-09-10-10.gh-issue-107944.XWm1B-.rst +++ /dev/null @@ -1 +0,0 @@ -Improve error message for function calls with bad keyword arguments via getargs diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-01-18-16-52.gh-issue-114806.wrH2J6.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-01-18-16-52.gh-issue-114806.wrH2J6.rst deleted file mode 100644 index 795f2529df8207..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-01-18-16-52.gh-issue-114806.wrH2J6.rst +++ /dev/null @@ -1,3 +0,0 @@ -No longer specialize calls to classes, if those classes have metaclasses. -Fixes bug where the ``__call__`` method of the metaclass was not being -called. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-01-23-43-49.gh-issue-76763.o_2J6i.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-01-23-43-49.gh-issue-76763.o_2J6i.rst deleted file mode 100644 index d35d3d87073ddd..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-01-23-43-49.gh-issue-76763.o_2J6i.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :func:`chr` builtin function now always raises :exc:`ValueError` for -values outside the valid range. Previously it raised :exc:`OverflowError` for -very large or small values. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-02-05-27-48.gh-issue-113462.VMml8q.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-02-05-27-48.gh-issue-113462.VMml8q.rst deleted file mode 100644 index 1a401ecebf019a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-02-05-27-48.gh-issue-113462.VMml8q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Limit the number of versions that a single class can use. Prevents a few -wayward classes using up all the version numbers. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-03-01-48-38.gh-issue-114944.4J5ELD.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-01-48-38.gh-issue-114944.4J5ELD.rst deleted file mode 100644 index fb41caf7c5f4fa..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-03-01-48-38.gh-issue-114944.4J5ELD.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes a race between ``PyParkingLot_Park`` and ``_PyParkingLot_UnparkAll``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-03-04-07-18.gh-issue-114887.uLSFmN.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-04-07-18.gh-issue-114887.uLSFmN.rst deleted file mode 100644 index b4d8cf4089d723..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-03-04-07-18.gh-issue-114887.uLSFmN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Changed socket type validation in :meth:`~asyncio.loop.create_datagram_endpoint` to accept all non-stream sockets. -This fixes a regression in compatibility with raw sockets. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-00-18-42.gh-issue-112069.jRDRR5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-00-18-42.gh-issue-112069.jRDRR5.rst deleted file mode 100644 index 51ba6bd1ddaac3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-00-18-42.gh-issue-112069.jRDRR5.rst +++ /dev/null @@ -1 +0,0 @@ -Adapt :class:`set` and :class:`frozenset` methods to Argument Clinic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-07-50-12.gh-issue-114828.nSXwMi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-07-50-12.gh-issue-114828.nSXwMi.rst deleted file mode 100644 index b1c63e0a1518fd..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-07-50-12.gh-issue-114828.nSXwMi.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix compilation crashes in uncommon code examples using :func:`super` inside -a comprehension in a class body. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst new file mode 100644 index 00000000000000..6e0f0afe05369b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst @@ -0,0 +1 @@ +Fixed a bug in ``reprlib.repr`` where it incorrectly called the repr method on shadowed Python built-in types. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst new file mode 100644 index 00000000000000..82be975f4d808d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst @@ -0,0 +1 @@ +Fix bug where SSLProtocol.connection_lost wasn't getting called when OSError was thrown on writing to socket. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-11-01-32-07.gh-issue-123930.BkPfB6.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-11-01-32-07.gh-issue-123930.BkPfB6.rst new file mode 100644 index 00000000000000..3c8eb02b2dc2d6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-11-01-32-07.gh-issue-123930.BkPfB6.rst @@ -0,0 +1,4 @@ +Improve the error message when a script shadowing a module from the standard +library causes :exc:`ImportError` to be raised during a "from" import. +Similarly, improve the error message when a script shadowing a third party module +attempts to "from" import an attribute from that third party module while still initialising. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-14-20-09-39.gh-issue-123714.o1mbe4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-14-20-09-39.gh-issue-123714.o1mbe4.rst new file mode 100644 index 00000000000000..03a2ef63238073 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-14-20-09-39.gh-issue-123714.o1mbe4.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 19 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-01-17-31-32.gh-issue-124855.sdsv_H.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-01-17-31-32.gh-issue-124855.sdsv_H.rst new file mode 100644 index 00000000000000..b65a5e6ac11c76 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-01-17-31-32.gh-issue-124855.sdsv_H.rst @@ -0,0 +1,2 @@ +Don't allow the JIT and perf support to be active at the same time. Patch by +Pablo Galindo diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-09-13-53-50.gh-issue-125038.ffSLCz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-09-13-53-50.gh-issue-125038.ffSLCz.rst new file mode 100644 index 00000000000000..15de48ec0e4450 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-09-13-53-50.gh-issue-125038.ffSLCz.rst @@ -0,0 +1,2 @@ +Fix crash when iterating over a generator expression after direct changes on ``gi_frame.f_locals``. +Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-17-13-12.gh-issue-118423.SkBoda.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-17-13-12.gh-issue-118423.SkBoda.rst new file mode 100644 index 00000000000000..8511a8de5530d6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-17-13-12.gh-issue-118423.SkBoda.rst @@ -0,0 +1,2 @@ +Add a new ``INSTRUCTION_SIZE`` macro to the cases generator which returns +the current instruction size. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-12-12-39.gh-issue-125444.9tG2X6.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-12-12-39.gh-issue-125444.9tG2X6.rst new file mode 100644 index 00000000000000..13c1e745edf8d5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-12-12-39.gh-issue-125444.9tG2X6.rst @@ -0,0 +1 @@ +Fix illegal instruction for older Arm architectures. Patch by Diego Russo, testing by Ross Burton. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst new file mode 100644 index 00000000000000..1d59a9c3c205b8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst @@ -0,0 +1,2 @@ +The Python PEG generator can now use f-strings in the grammar actions. Patch +by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-23-06-06.gh-issue-125017.fcltj0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-23-06-06.gh-issue-125017.fcltj0.rst new file mode 100644 index 00000000000000..11c526643c3122 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-23-06-06.gh-issue-125017.fcltj0.rst @@ -0,0 +1,2 @@ +Fix crash on certain accesses to the ``__annotations__`` of +:class:`staticmethod` and :class:`classmethod` objects. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-10-11-43.gh-issue-125593.Q97m3A.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-10-11-43.gh-issue-125593.Q97m3A.rst new file mode 100644 index 00000000000000..220e94467af849 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-10-11-43.gh-issue-125593.Q97m3A.rst @@ -0,0 +1 @@ +Use color to highlight error locations in traceback from exception group diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-16-00-10.gh-issue-125703.QRoqMo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-16-00-10.gh-issue-125703.QRoqMo.rst new file mode 100644 index 00000000000000..7cbfa725e78cef --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-16-00-10.gh-issue-125703.QRoqMo.rst @@ -0,0 +1,2 @@ +Correctly honour :mod:`tracemalloc` hooks in specialized ``Py_DECREF`` +paths. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-22-04-18-53.gh-issue-125498.cFjPIn.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-22-04-18-53.gh-issue-125498.cFjPIn.rst new file mode 100644 index 00000000000000..807c2e07210e7f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-22-04-18-53.gh-issue-125498.cFjPIn.rst @@ -0,0 +1,4 @@ +The JIT has been updated to leverage Clang 19’s new ``preserve_none`` attribute, +which supports more platforms and is more useful than LLVM's existing ``ghccc`` +calling convention. This also removes the need to manually patch the calling +convention in LLVM IR, simplifying the JIT compilation process. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-23-14-05-47.gh-issue-125868.uLfXYB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-23-14-05-47.gh-issue-125868.uLfXYB.rst new file mode 100644 index 00000000000000..dea250e7166ec6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-23-14-05-47.gh-issue-125868.uLfXYB.rst @@ -0,0 +1,3 @@ +It was possible in 3.14.0a1 only for attribute lookup to give the wrong +value. This was due to an incorrect specialization in very specific +circumstances. This is fixed in 3.14.0a2. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-23-14-42-27.gh-issue-125859.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-23-14-42-27.gh-issue-125859.m3EF9E.rst new file mode 100644 index 00000000000000..d36aa8fbe7482f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-23-14-42-27.gh-issue-125859.m3EF9E.rst @@ -0,0 +1,2 @@ +Fix a crash in the free threading build when :func:`gc.get_objects` or +:func:`gc.get_referrers` is called during an in-progress garbage collection. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-24-22-43-03.gh-issue-125942.3UQht1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-24-22-43-03.gh-issue-125942.3UQht1.rst new file mode 100644 index 00000000000000..d1b1ecd2a724ab --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-24-22-43-03.gh-issue-125942.3UQht1.rst @@ -0,0 +1,2 @@ +On Android, the ``errors`` setting of :any:`sys.stdout` was changed from +``surrogateescape`` to ``backslashreplace``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-25-15-56-14.gh-issue-125837.KlCdgD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-25-15-56-14.gh-issue-125837.KlCdgD.rst new file mode 100644 index 00000000000000..9538f34f969377 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-25-15-56-14.gh-issue-125837.KlCdgD.rst @@ -0,0 +1,5 @@ +Adds :opcode:`LOAD_SMALL_INT` and :opcode:`LOAD_CONST_IMMORTAL` instructions. +``LOAD_SMALL_INT`` pushes a small integer equal to the ``oparg`` to the stack. +``LOAD_CONST_IMMORTAL`` does the same as ``LOAD_CONST`` but is more +efficient for immortal objects. +Removes ``RETURN_CONST`` instruction. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst new file mode 100644 index 00000000000000..5307920ddcf200 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst @@ -0,0 +1,2 @@ +The :class:`memoryview` type now supports subscription, making it a +:term:`generic type`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst new file mode 100644 index 00000000000000..e019408638997b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst @@ -0,0 +1,2 @@ +Fix a crash in :func:`sys.audit` when passing a non-string as first argument +and Python was compiled in debug mode. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-20-31-43.gh-issue-90370.IP_W3a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-20-31-43.gh-issue-90370.IP_W3a.rst new file mode 100644 index 00000000000000..b6a19c06a228ca --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-20-31-43.gh-issue-90370.IP_W3a.rst @@ -0,0 +1,2 @@ +Avoid temporary tuple creation for vararg in argument passing with Argument +Clinic generated code (if arguments either vararg or positional-only). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-10-37-39.gh-issue-126072.XLKlxv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-10-37-39.gh-issue-126072.XLKlxv.rst new file mode 100644 index 00000000000000..4ad30e9f954ecf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-10-37-39.gh-issue-126072.XLKlxv.rst @@ -0,0 +1,3 @@ +Add a new attribute in :attr:`~codeobject.co_flags` to indicate whether the +first item in :attr:`~codeobject.co_consts` is the docstring. If a code +object has no docstring, ``None`` will **NOT** be inserted. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-15-17-31.gh-issue-126139.B4OQ8a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-15-17-31.gh-issue-126139.B4OQ8a.rst new file mode 100644 index 00000000000000..278971b46d18ab --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-15-17-31.gh-issue-126139.B4OQ8a.rst @@ -0,0 +1,2 @@ +Provide better error location when attempting to use a :term:`future +statement <__future__>` with an unknown future feature. diff --git a/Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst b/Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst new file mode 100644 index 00000000000000..e401cc2535e389 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst @@ -0,0 +1,2 @@ +Include the :class:`object` type in the lists of documented types. +Change by Furkan Onder and Martin Panter. diff --git a/Misc/NEWS.d/next/Documentation/2024-01-17-11-40-03.gh-issue-114123.LuueXf.rst b/Misc/NEWS.d/next/Documentation/2024-01-17-11-40-03.gh-issue-114123.LuueXf.rst deleted file mode 100644 index 1d93a422840077..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2024-01-17-11-40-03.gh-issue-114123.LuueXf.rst +++ /dev/null @@ -1,7 +0,0 @@ -Move the :mod:`csv` module docstring to the :mod:`!csv` module -instead of reexporting it from the internal :mod:`!_csv` module, -and remove ``__doc__`` from ``csv.__all__``. - -Move :attr:`!csv.__version__` to the :mod:`!csv` module -instead of reexporting it from the internal :mod:`!_csv` module, -and remove ``__version__`` from ``csv.__all__``. diff --git a/Misc/NEWS.d/next/Documentation/2024-10-10-23-46-54.gh-issue-125277.QAby09.rst b/Misc/NEWS.d/next/Documentation/2024-10-10-23-46-54.gh-issue-125277.QAby09.rst new file mode 100644 index 00000000000000..fcd6e22c27b5f4 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-10-10-23-46-54.gh-issue-125277.QAby09.rst @@ -0,0 +1,2 @@ +Require Sphinx 7.2.6 or later to build the Python documentation. +Patch by Adam Turner. diff --git a/Misc/NEWS.d/next/IDLE/2023-04-25-03-01-23.gh-issue-103820.LCSpza.rst b/Misc/NEWS.d/next/IDLE/2023-04-25-03-01-23.gh-issue-103820.LCSpza.rst deleted file mode 100644 index b9d7faf047b28e..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-04-25-03-01-23.gh-issue-103820.LCSpza.rst +++ /dev/null @@ -1,2 +0,0 @@ -Revise IDLE bindings so that events from mouse button 4/5 on non-X11 -windowing systems (i.e. Win32 and Aqua) are not mistaken for scrolling. diff --git a/Misc/NEWS.d/next/IDLE/2024-01-17-23-18-15.gh-issue-96905.UYaxoU.rst b/Misc/NEWS.d/next/IDLE/2024-01-17-23-18-15.gh-issue-96905.UYaxoU.rst deleted file mode 100644 index fe7dde64c7c7d5..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2024-01-17-23-18-15.gh-issue-96905.UYaxoU.rst +++ /dev/null @@ -1 +0,0 @@ -In idlelib code, stop redefining built-ins 'dict' and 'object'. diff --git a/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst b/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst deleted file mode 100644 index 31dd985bb5c3b6..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :func:`os.path.isreserved`, which identifies reserved pathnames such -as "NUL", "AUX" and "CON". This function is only available on Windows. - -Deprecate :meth:`pathlib.PurePath.is_reserved`. diff --git a/Misc/NEWS.d/next/Library/2023-03-08-00-02-30.gh-issue-102512.LiugDr.rst b/Misc/NEWS.d/next/Library/2023-03-08-00-02-30.gh-issue-102512.LiugDr.rst deleted file mode 100644 index 659cba73cbf34e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-08-00-02-30.gh-issue-102512.LiugDr.rst +++ /dev/null @@ -1,3 +0,0 @@ -When :func:`os.fork` is called from a foreign thread (aka ``_DummyThread``), -the type of the thread in a child process is changed to ``_MainThread``. -Also changed its name and daemonic status, it can be now joined. diff --git a/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst b/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst deleted file mode 100644 index 0e54a1fe3c8a1c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add POSIX pseudo-terminal functions :func:`os.posix_openpt`, -:func:`os.grantpt`, :func:`os.unlockpt`, and :func:`os.ptsname`. diff --git a/Misc/NEWS.d/next/Library/2023-04-08-11-41-07.gh-issue-101599.PaWNFh.rst b/Misc/NEWS.d/next/Library/2023-04-08-11-41-07.gh-issue-101599.PaWNFh.rst deleted file mode 100644 index a1608a1ae0d2fa..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-08-11-41-07.gh-issue-101599.PaWNFh.rst +++ /dev/null @@ -1 +0,0 @@ -Changed argparse flag options formatting to remove redundancy. diff --git a/Misc/NEWS.d/next/Library/2023-05-06-04-57-10.gh-issue-96471.C9wAU7.rst b/Misc/NEWS.d/next/Library/2023-05-06-04-57-10.gh-issue-96471.C9wAU7.rst deleted file mode 100644 index 0bace8d8bd425c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-06-04-57-10.gh-issue-96471.C9wAU7.rst +++ /dev/null @@ -1 +0,0 @@ -Add :py:class:`queue.Queue` termination with :py:meth:`~queue.Queue.shutdown`. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-09-30-00.gh-issue-104282.h4c6Eb.rst b/Misc/NEWS.d/next/Library/2023-05-08-09-30-00.gh-issue-104282.h4c6Eb.rst deleted file mode 100644 index 569ce66a5b9d5f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-09-30-00.gh-issue-104282.h4c6Eb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix null pointer dereference in :func:`lzma._decode_filter_properties` -due to improper handling of BCJ filters with properties of zero length. -Patch by Radislav Chugunov. diff --git a/Misc/NEWS.d/next/Library/2023-05-30-18-30-11.gh-issue-105102.SnpK04.rst b/Misc/NEWS.d/next/Library/2023-05-30-18-30-11.gh-issue-105102.SnpK04.rst deleted file mode 100644 index 7ca21afefa3132..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-30-18-30-11.gh-issue-105102.SnpK04.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow :class:`ctypes.Union` to be nested in :class:`ctypes.Structure` when -the system endianness is the opposite of the classes. diff --git a/Misc/NEWS.d/next/Library/2023-06-29-14-26-56.gh-issue-106233.Aqw2HI.rst b/Misc/NEWS.d/next/Library/2023-06-29-14-26-56.gh-issue-106233.Aqw2HI.rst deleted file mode 100644 index 345c8b20815c95..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-29-14-26-56.gh-issue-106233.Aqw2HI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix stacklevel in ``InvalidTZPathWarning`` during :mod:`zoneinfo` module -import. diff --git a/Misc/NEWS.d/next/Library/2023-07-23-12-28-26.gh-issue-75705.aB2-Ww.rst b/Misc/NEWS.d/next/Library/2023-07-23-12-28-26.gh-issue-75705.aB2-Ww.rst deleted file mode 100644 index 272e31d64cfbd9..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-23-12-28-26.gh-issue-75705.aB2-Ww.rst +++ /dev/null @@ -1 +0,0 @@ -Set unixfrom envelope in :class:`mailbox.mbox` and :class:`mailbox.MMDF`. diff --git a/Misc/NEWS.d/next/Library/2023-09-22-22-17-45.gh-issue-38807.m9McRN.rst b/Misc/NEWS.d/next/Library/2023-09-22-22-17-45.gh-issue-38807.m9McRN.rst deleted file mode 100644 index 4219723d15b9e6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-09-22-22-17-45.gh-issue-38807.m9McRN.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix race condition in :mod:`trace`. Instead of checking if a directory -exists and creating it, directly call :func:`os.makedirs` with the kwarg -``exist_ok=True``. diff --git a/Misc/NEWS.d/next/Library/2023-10-04-11-09-30.gh-issue-110345.fZU1ud.rst b/Misc/NEWS.d/next/Library/2023-10-04-11-09-30.gh-issue-110345.fZU1ud.rst deleted file mode 100644 index d9ccc0f12de47c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-04-11-09-30.gh-issue-110345.fZU1ud.rst +++ /dev/null @@ -1 +0,0 @@ -Show the Tcl/Tk patchlevel (rather than version) in :meth:`tkinter._test`. diff --git a/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst b/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst deleted file mode 100644 index adb3241b89ae3e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst +++ /dev/null @@ -1 +0,0 @@ -Added check for file modification during debugging with :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2023-10-24-19-19-54.gh-issue-82626._hfLRf.rst b/Misc/NEWS.d/next/Library/2023-10-24-19-19-54.gh-issue-82626._hfLRf.rst deleted file mode 100644 index 92a66b5bf0f635..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-24-19-19-54.gh-issue-82626._hfLRf.rst +++ /dev/null @@ -1,2 +0,0 @@ -Many functions now emit a warning if a boolean value is passed as a file -descriptor argument. diff --git a/Misc/NEWS.d/next/Library/2023-10-27-19-24-58.gh-issue-43457.84lx9H.rst b/Misc/NEWS.d/next/Library/2023-10-27-19-24-58.gh-issue-43457.84lx9H.rst deleted file mode 100644 index 401a532ce03e77..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-27-19-24-58.gh-issue-43457.84lx9H.rst +++ /dev/null @@ -1,8 +0,0 @@ -Fix the :mod:`tkinter` widget method :meth:`!wm_attributes`. It now -accepts the attribute name without the minus prefix to get window attributes -and allows to specify attributes and values to set as keyword arguments. -Add new optional keyword argument *return_python_dict*: calling -``w.wm_attributes(return_python_dict=True)`` returns the attributes as -a dict instead of a tuple. -Calling ``w.wm_attributes()`` now returns a tuple instead of string if -*wantobjects* was set to 0. diff --git a/Misc/NEWS.d/next/Library/2023-11-04-22-32-27.gh-issue-111741.f1ufr8.rst b/Misc/NEWS.d/next/Library/2023-11-04-22-32-27.gh-issue-111741.f1ufr8.rst deleted file mode 100644 index e43f93a270ce9c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-04-22-32-27.gh-issue-111741.f1ufr8.rst +++ /dev/null @@ -1 +0,0 @@ -Recognise ``image/webp`` as a standard format in the :mod:`mimetypes` module. diff --git a/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst b/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst deleted file mode 100644 index 686f0311e80dcb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add option to calendar module CLI to specify the weekday to start each week. -Patch by Steven Ward. diff --git a/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst deleted file mode 100644 index aaa50fce3ac962..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst +++ /dev/null @@ -1 +0,0 @@ -Improve handling of pdb convenience variables to avoid replacing string contents. diff --git a/Misc/NEWS.d/next/Library/2023-11-27-19-54-43.gh-issue-59013.chpQ0e.rst b/Misc/NEWS.d/next/Library/2023-11-27-19-54-43.gh-issue-59013.chpQ0e.rst deleted file mode 100644 index a2be2fb8eacf17..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-27-19-54-43.gh-issue-59013.chpQ0e.rst +++ /dev/null @@ -1 +0,0 @@ -Set breakpoint on the first executable line of the function, instead of the line of function definition when the user do ``break func`` using :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst b/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst deleted file mode 100644 index 3e99d480139cbe..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed-up :func:`datetime.datetime.replace`, :func:`datetime.date.replace` and -:func:`datetime.time.replace`. diff --git a/Misc/NEWS.d/next/Library/2024-01-04-20-58-17.gh-issue-113225.-nyJM4.rst b/Misc/NEWS.d/next/Library/2024-01-04-20-58-17.gh-issue-113225.-nyJM4.rst deleted file mode 100644 index 0c07f42fd065d2..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-04-20-58-17.gh-issue-113225.-nyJM4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :meth:`pathlib.Path.walk` by using :attr:`os.DirEntry.path` where -possible. diff --git a/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst b/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst deleted file mode 100644 index 7582603dcf95f5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix support of :data:`~csv.QUOTE_NOTNULL` and :data:`~csv.QUOTE_STRINGS` in -:func:`csv.reader`. diff --git a/Misc/NEWS.d/next/Library/2024-01-07-21-04-24.gh-issue-113796.6iNsCR.rst b/Misc/NEWS.d/next/Library/2024-01-07-21-04-24.gh-issue-113796.6iNsCR.rst deleted file mode 100644 index e9d4aba9906677..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-07-21-04-24.gh-issue-113796.6iNsCR.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add more validation checks in the :class:`csv.Dialect` constructor. -:exc:`ValueError` is now raised if the same character is used in different -roles. diff --git a/Misc/NEWS.d/next/Library/2024-01-11-20-47-49.gh-issue-113951.AzlqFK.rst b/Misc/NEWS.d/next/Library/2024-01-11-20-47-49.gh-issue-113951.AzlqFK.rst deleted file mode 100644 index e683472e59b8a4..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-11-20-47-49.gh-issue-113951.AzlqFK.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fix the behavior of ``tag_unbind()`` methods of :class:`tkinter.Text` and -:class:`tkinter.Canvas` classes with three arguments. Previously, -``widget.tag_unbind(tag, sequence, funcid)`` destroyed the current binding -for *sequence*, leaving *sequence* unbound, and deleted the *funcid* -command. Now it removes only *funcid* from the binding for *sequence*, -keeping other commands, and deletes the *funcid* command. It leaves -*sequence* unbound only if *funcid* was the last bound command. diff --git a/Misc/NEWS.d/next/Library/2024-01-12-09-35-07.gh-issue-112202.t_0V1m.rst b/Misc/NEWS.d/next/Library/2024-01-12-09-35-07.gh-issue-112202.t_0V1m.rst deleted file mode 100644 index 9abde13bbf8571..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-12-09-35-07.gh-issue-112202.t_0V1m.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure that a :func:`asyncio.Condition.notify` call does not get lost if the awakened ``Task`` is simultaneously cancelled or encounters any other error. diff --git a/Misc/NEWS.d/next/Library/2024-01-12-17-32-36.gh-issue-79634.uTSTRI.rst b/Misc/NEWS.d/next/Library/2024-01-12-17-32-36.gh-issue-79634.uTSTRI.rst deleted file mode 100644 index ba19b5209e648e..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-12-17-32-36.gh-issue-79634.uTSTRI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Accept :term:`path-like objects ` as patterns in -:meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob`. diff --git a/Misc/NEWS.d/next/Library/2024-01-15-18-42-44.gh-issue-109534.wYaLMZ.rst b/Misc/NEWS.d/next/Library/2024-01-15-18-42-44.gh-issue-109534.wYaLMZ.rst deleted file mode 100644 index fc9a765a230037..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-15-18-42-44.gh-issue-109534.wYaLMZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a reference leak in -:class:`asyncio.selector_events.BaseSelectorEventLoop` when SSL handshakes -fail. Patch contributed by Jamie Phan. diff --git a/Misc/NEWS.d/next/Library/2024-01-15-19-54-41.gh-issue-114087.Xic5vY.rst b/Misc/NEWS.d/next/Library/2024-01-15-19-54-41.gh-issue-114087.Xic5vY.rst deleted file mode 100644 index 68b27a7b0c9f6c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-15-19-54-41.gh-issue-114087.Xic5vY.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up ``dataclasses.asdict`` up to 1.35x. diff --git a/Misc/NEWS.d/next/Library/2024-01-15-20-21-33.gh-issue-83648.HzD_fY.rst b/Misc/NEWS.d/next/Library/2024-01-15-20-21-33.gh-issue-83648.HzD_fY.rst deleted file mode 100644 index bd3e27b4be0cf5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-15-20-21-33.gh-issue-83648.HzD_fY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Support deprecation of options, positional arguments and subcommands in -:mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2024-01-16-15-59-06.gh-issue-114149.LJ8IPm.rst b/Misc/NEWS.d/next/Library/2024-01-16-15-59-06.gh-issue-114149.LJ8IPm.rst deleted file mode 100644 index 1403d78d0d4905..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-16-15-59-06.gh-issue-114149.LJ8IPm.rst +++ /dev/null @@ -1 +0,0 @@ -Enum: correctly handle tuple subclasses in custom ``__new__``. diff --git a/Misc/NEWS.d/next/Library/2024-01-17-18-53-51.gh-issue-104522.3NyDf4.rst b/Misc/NEWS.d/next/Library/2024-01-17-18-53-51.gh-issue-104522.3NyDf4.rst deleted file mode 100644 index ca980945ea12d3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-17-18-53-51.gh-issue-104522.3NyDf4.rst +++ /dev/null @@ -1,3 +0,0 @@ -:exc:`OSError` raised when run a subprocess now only has *filename* -attribute set to *cwd* if the error was caused by a failed attempt to change -the current directory. diff --git a/Misc/NEWS.d/next/Library/2024-01-18-10-07-52.gh-issue-114198.lK4Iif.rst b/Misc/NEWS.d/next/Library/2024-01-18-10-07-52.gh-issue-114198.lK4Iif.rst deleted file mode 100644 index fa047e288f807e..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-18-10-07-52.gh-issue-114198.lK4Iif.rst +++ /dev/null @@ -1,2 +0,0 @@ -The signature for the ``__replace__`` method on :mod:`dataclasses` now has -the first argument named ``self``, rather than ``obj``. diff --git a/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst b/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst deleted file mode 100644 index 9b69b5deb1b5a0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst +++ /dev/null @@ -1,4 +0,0 @@ -Avoid reference cycle in ElementTree.iterparse. The iterator returned by -``ElementTree.iterparse`` may hold on to a file descriptor. The reference -cycle prevented prompt clean-up of the file descriptor if the returned -iterator was not exhausted. diff --git a/Misc/NEWS.d/next/Library/2024-01-19-12-05-22.gh-issue-114281.H5JQe4.rst b/Misc/NEWS.d/next/Library/2024-01-19-12-05-22.gh-issue-114281.H5JQe4.rst deleted file mode 100644 index 36c54e8faf214c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-19-12-05-22.gh-issue-114281.H5JQe4.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove type hints from ``Lib/asyncio/staggered.py``. -The annotations in the `typeshed `__ -project should be used instead. diff --git a/Misc/NEWS.d/next/Library/2024-01-19-15-48-06.gh-issue-114328.hixxW3.rst b/Misc/NEWS.d/next/Library/2024-01-19-15-48-06.gh-issue-114328.hixxW3.rst deleted file mode 100644 index 42262c05fd1fbf..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-19-15-48-06.gh-issue-114328.hixxW3.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :func:`tty.setcbreak` and new :func:`tty.cfmakecbreak` no longer clears -the terminal input ICRLF flag. This fixes a regression introduced in 3.12 -that no longer matched how OSes define cbreak mode in their ``stty(1)`` -manual pages. diff --git a/Misc/NEWS.d/next/Library/2024-01-19-18-41-02.gh-issue-114321.yj_Xw3.rst b/Misc/NEWS.d/next/Library/2024-01-19-18-41-02.gh-issue-114321.yj_Xw3.rst deleted file mode 100644 index dc2934bd81a42a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-19-18-41-02.gh-issue-114321.yj_Xw3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Expose more platform specific constants in the :mod:`fcntl` module on Linux, -macOS, FreeBSD and NetBSD. diff --git a/Misc/NEWS.d/next/Library/2024-01-21-16-32-55.gh-issue-114257.bCFld5.rst b/Misc/NEWS.d/next/Library/2024-01-21-16-32-55.gh-issue-114257.bCFld5.rst deleted file mode 100644 index 6f02ff9e62617d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-21-16-32-55.gh-issue-114257.bCFld5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Dismiss the :exc:`FileNotFound` error in :func:`ctypes.util.find_library` and -just return ``None`` on Linux. diff --git a/Misc/NEWS.d/next/Library/2024-01-22-11-43-38.gh-issue-114423.6mMoPH.rst b/Misc/NEWS.d/next/Library/2024-01-22-11-43-38.gh-issue-114423.6mMoPH.rst deleted file mode 100644 index 7b77b73295d948..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-22-11-43-38.gh-issue-114423.6mMoPH.rst +++ /dev/null @@ -1 +0,0 @@ -``_DummyThread`` entries in ``threading._active`` are now automatically removed when the related thread dies. diff --git a/Misc/NEWS.d/next/Library/2024-01-22-12-10-34.gh-issue-75128.4FGlRS.rst b/Misc/NEWS.d/next/Library/2024-01-22-12-10-34.gh-issue-75128.4FGlRS.rst deleted file mode 100644 index d875148e89b41b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-22-12-10-34.gh-issue-75128.4FGlRS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ignore an :exc:`OSError` in :meth:`asyncio.BaseEventLoop.create_server` when -IPv6 is available but the interface cannot actually support it. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-11-04-21.gh-issue-113267.xe_Pxe.rst b/Misc/NEWS.d/next/Library/2024-01-23-11-04-21.gh-issue-113267.xe_Pxe.rst deleted file mode 100644 index ad8aaf9250f6d8..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-23-11-04-21.gh-issue-113267.xe_Pxe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Revert changes in :gh:`106584` which made calls of ``TestResult`` methods -``startTest()`` and ``stopTest()`` unbalanced. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst b/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst deleted file mode 100644 index a8a19fc525d019..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make :class:`threading.Lock` a real class, not a factory function. Add -``__new__`` to ``_thread.lock`` type. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst b/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst deleted file mode 100644 index 8df8299d0dffcd..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make the result of :func:`termios.tcgetattr` reproducible on Alpine Linux. -Previously it could leave a random garbage in some fields. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-23-13-47.gh-issue-109653.KLBHmT.rst b/Misc/NEWS.d/next/Library/2024-01-23-23-13-47.gh-issue-109653.KLBHmT.rst deleted file mode 100644 index 76074df9c76fa6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-23-23-13-47.gh-issue-109653.KLBHmT.rst +++ /dev/null @@ -1 +0,0 @@ -Reduce the import time of :mod:`threading` module by ~50%. Patch by Daniel Hollas. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-17-25-18.gh-issue-69893.PQq5fR.rst b/Misc/NEWS.d/next/Library/2024-01-24-17-25-18.gh-issue-69893.PQq5fR.rst deleted file mode 100644 index 1ebf434c33187b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-24-17-25-18.gh-issue-69893.PQq5fR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add the :meth:`!close` method for the iterator returned by -:func:`xml.etree.ElementTree.iterparse`. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst b/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst deleted file mode 100644 index 126ca36a3b7cb1..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst +++ /dev/null @@ -1,2 +0,0 @@ -Prohibit subclassing pure-Python :class:`datetime.timezone`. This is consistent -with C-extension implementation. Patch by Mariusz Felisiak. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst b/Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst deleted file mode 100644 index 21d39df43e035b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add *filter* keyword-only parameter to -:meth:`sqlite3.Connection.iterdump` for filtering database objects to dump. -Patch by Mariusz Felisiak. diff --git a/Misc/NEWS.d/next/Library/2024-01-25-19-22-17.gh-issue-83383.3GwO9v.rst b/Misc/NEWS.d/next/Library/2024-01-25-19-22-17.gh-issue-83383.3GwO9v.rst deleted file mode 100644 index e6336204dfa236..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-25-19-22-17.gh-issue-83383.3GwO9v.rst +++ /dev/null @@ -1,5 +0,0 @@ -Synchronization of the :mod:`dbm.dumb` database is now no-op if there was no -modification since opening or last synchronization. -The directory file for a newly created empty :mod:`dbm.dumb` database is now -created immediately after opening instead of deferring this until -synchronizing or closing. diff --git a/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst b/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst deleted file mode 100644 index f1c99c09d2dfe1..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`email.policy.EmailPolicy.fold` now always encodes non-ASCII characters -in headers if :attr:`~email.policy.EmailPolicy.utf8` is false. diff --git a/Misc/NEWS.d/next/Library/2024-01-27-20-11-24.gh-issue-113280.CZPQMf.rst b/Misc/NEWS.d/next/Library/2024-01-27-20-11-24.gh-issue-113280.CZPQMf.rst deleted file mode 100644 index 3dcdbcf0995616..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-27-20-11-24.gh-issue-113280.CZPQMf.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a leak of open socket in rare cases when error occurred in -:class:`ssl.SSLSocket` creation. diff --git a/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst b/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst deleted file mode 100644 index fb3382098853b3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst +++ /dev/null @@ -1 +0,0 @@ -Improve import time of :mod:`importlib.metadata` and :mod:`email.utils`. diff --git a/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst b/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst deleted file mode 100644 index dedda24b481241..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Return both files and directories from :meth:`pathlib.Path.glob` if a -pattern ends with "``**``". Previously only directories were returned. diff --git a/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst b/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst deleted file mode 100644 index 2306af4a39dcf6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst +++ /dev/null @@ -1,3 +0,0 @@ -Ensure that deprecation warning for 'N' specifier in :class:`~decimal.Decimal` -format is not raised for cases where 'N' appears in other places -in the format specifier. Based on patch by Stefan Krah. diff --git a/Misc/NEWS.d/next/Library/2024-01-30-15-34-08.gh-issue-110190.Z5PQQX.rst b/Misc/NEWS.d/next/Library/2024-01-30-15-34-08.gh-issue-110190.Z5PQQX.rst deleted file mode 100644 index af77e409963e04..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-30-15-34-08.gh-issue-110190.Z5PQQX.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ctypes structs with array on Windows ARM64 platform by setting ``MAX_STRUCT_SIZE`` to 32 in stgdict. Patch by Diego Russo diff --git a/Misc/NEWS.d/next/Library/2024-01-31-20-07-11.gh-issue-109475.lmTb9S.rst b/Misc/NEWS.d/next/Library/2024-01-31-20-07-11.gh-issue-109475.lmTb9S.rst deleted file mode 100644 index 7582cb2bcd7629..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-31-20-07-11.gh-issue-109475.lmTb9S.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix support of explicit option value "--" in :mod:`argparse` (e.g. -``--option=--``). diff --git a/Misc/NEWS.d/next/Library/2024-02-01-10-19-11.gh-issue-114071.vkm2G_.rst b/Misc/NEWS.d/next/Library/2024-02-01-10-19-11.gh-issue-114071.vkm2G_.rst deleted file mode 100644 index 587ce4d2157637..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-01-10-19-11.gh-issue-114071.vkm2G_.rst +++ /dev/null @@ -1 +0,0 @@ -Support tuple subclasses using auto() for enum member value. diff --git a/Misc/NEWS.d/next/Library/2024-02-03-16-59-25.gh-issue-114959.dCfAG2.rst b/Misc/NEWS.d/next/Library/2024-02-03-16-59-25.gh-issue-114959.dCfAG2.rst deleted file mode 100644 index 5c6eaa7525e3b0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-03-16-59-25.gh-issue-114959.dCfAG2.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`tarfile` no longer ignores errors when trying to extract a directory on -top of a file. diff --git a/Misc/NEWS.d/next/Library/2024-02-03-17-54-17.gh-issue-114965.gHksCK.rst b/Misc/NEWS.d/next/Library/2024-02-03-17-54-17.gh-issue-114965.gHksCK.rst deleted file mode 100644 index d59ff991993792..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-03-17-54-17.gh-issue-114965.gHksCK.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled pip to 24.0 diff --git a/Misc/NEWS.d/next/Library/2024-02-04-13-17-33.gh-issue-114628.WJpqqS.rst b/Misc/NEWS.d/next/Library/2024-02-04-13-17-33.gh-issue-114628.WJpqqS.rst deleted file mode 100644 index 8138adc62c95f3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-04-13-17-33.gh-issue-114628.WJpqqS.rst +++ /dev/null @@ -1,2 +0,0 @@ -When csv.Error is raised when handling TypeError, do not print the TypeError -traceback. diff --git a/Misc/NEWS.d/next/Library/2024-02-08-13-26-14.gh-issue-115059.DqP9dr.rst b/Misc/NEWS.d/next/Library/2024-02-08-13-26-14.gh-issue-115059.DqP9dr.rst deleted file mode 100644 index 331baedd3b24c5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-08-13-26-14.gh-issue-115059.DqP9dr.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`io.BufferedRandom.read1` now flushes the underlying write buffer. diff --git a/Misc/NEWS.d/next/Library/2024-02-08-17-04-58.gh-issue-112903.SN_vUs.rst b/Misc/NEWS.d/next/Library/2024-02-08-17-04-58.gh-issue-112903.SN_vUs.rst deleted file mode 100644 index e27f5832553c13..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-08-17-04-58.gh-issue-112903.SN_vUs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix "issubclass() arg 1 must be a class" errors in certain cases of multiple -inheritance with generic aliases (regression in early 3.13 alpha releases). diff --git a/Misc/NEWS.d/next/Library/2024-02-09-07-20-16.gh-issue-115165.yfJLXA.rst b/Misc/NEWS.d/next/Library/2024-02-09-07-20-16.gh-issue-115165.yfJLXA.rst deleted file mode 100644 index 73d3d001f07f3f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-09-07-20-16.gh-issue-115165.yfJLXA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Most exceptions are now ignored when attempting to set the ``__orig_class__`` -attribute on objects returned when calling :mod:`typing` generic aliases -(including generic aliases created using :data:`typing.Annotated`). -Previously only :exc:`AttributeError`` was ignored. Patch by Dave Shawley. diff --git a/Misc/NEWS.d/next/Library/2024-05-13-10-09-41.gh-issue-118986.-r4W9h.rst b/Misc/NEWS.d/next/Library/2024-05-13-10-09-41.gh-issue-118986.-r4W9h.rst new file mode 100644 index 00000000000000..196da60a950bfb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-13-10-09-41.gh-issue-118986.-r4W9h.rst @@ -0,0 +1 @@ +Add :data:`!socket.IPV6_RECVERR` constant (available since Linux 2.2). diff --git a/Misc/NEWS.d/next/Library/2024-05-28-14-35-23.gh-issue-97850.dCtjel.rst b/Misc/NEWS.d/next/Library/2024-05-28-14-35-23.gh-issue-97850.dCtjel.rst new file mode 100644 index 00000000000000..bb94f7d8ad124d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-28-14-35-23.gh-issue-97850.dCtjel.rst @@ -0,0 +1 @@ +Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. diff --git a/Misc/NEWS.d/next/Library/2024-07-02-15-56-42.gh-issue-121267.yFBWkh.rst b/Misc/NEWS.d/next/Library/2024-07-02-15-56-42.gh-issue-121267.yFBWkh.rst new file mode 100644 index 00000000000000..9e52405c15a82d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-02-15-56-42.gh-issue-121267.yFBWkh.rst @@ -0,0 +1,2 @@ +Improve the performance of :mod:`tarfile` when writing files, by caching user names +and group names. diff --git a/Misc/NEWS.d/next/Library/2024-08-28-19-27-35.gh-issue-123370.SPZ9Ux.rst b/Misc/NEWS.d/next/Library/2024-08-28-19-27-35.gh-issue-123370.SPZ9Ux.rst new file mode 100644 index 00000000000000..1fd5cc54eaf3e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-28-19-27-35.gh-issue-123370.SPZ9Ux.rst @@ -0,0 +1 @@ +Fix the canvas not clearing after running turtledemo clock. diff --git a/Misc/NEWS.d/next/Library/2024-09-07-13-57-49.gh-issue-80958.fVYnqV.rst b/Misc/NEWS.d/next/Library/2024-09-07-13-57-49.gh-issue-80958.fVYnqV.rst new file mode 100644 index 00000000000000..f0edd7b1ac6e8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-07-13-57-49.gh-issue-80958.fVYnqV.rst @@ -0,0 +1 @@ +unittest discovery supports PEP 420 namespace packages as start directory again. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-18-16-59.gh-issue-58956.0wFrBR.rst b/Misc/NEWS.d/next/Library/2024-09-24-18-16-59.gh-issue-58956.0wFrBR.rst new file mode 100644 index 00000000000000..a882a632fddf1b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-18-16-59.gh-issue-58956.0wFrBR.rst @@ -0,0 +1 @@ +Fixed a bug in :mod:`pdb` where sometimes the breakpoint won't trigger if it was set on a function which is already in the call stack. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-18-49-16.gh-issue-99749.gBDJX7.rst b/Misc/NEWS.d/next/Library/2024-09-24-18-49-16.gh-issue-99749.gBDJX7.rst new file mode 100644 index 00000000000000..3ecd75c5b551b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-18-49-16.gh-issue-99749.gBDJX7.rst @@ -0,0 +1 @@ +Adds a feature to optionally enable suggestions for argument choices and subparser names if mistyped by the user. diff --git a/Misc/NEWS.d/next/Library/2024-09-27-13-10-17.gh-issue-58032.0aNAQ0.rst b/Misc/NEWS.d/next/Library/2024-09-27-13-10-17.gh-issue-58032.0aNAQ0.rst new file mode 100644 index 00000000000000..278512b22a8d3f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-27-13-10-17.gh-issue-58032.0aNAQ0.rst @@ -0,0 +1 @@ +Deprecate the :class:`argparse.FileType` type converter. diff --git a/Misc/NEWS.d/next/Library/2024-09-27-15-42-55.gh-issue-124694.uUy32y.rst b/Misc/NEWS.d/next/Library/2024-09-27-15-42-55.gh-issue-124694.uUy32y.rst new file mode 100644 index 00000000000000..1aa1a463b0c63a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-27-15-42-55.gh-issue-124694.uUy32y.rst @@ -0,0 +1,6 @@ +We've added :class:`concurrent.futures.InterpreterPoolExecutor`, which +allows you to run code in multiple isolated interpreters. This allows you +to circumvent the limitations of CPU-bound threads (due to the GIL). Patch +by Eric Snow. + +This addition is unrelated to :pep:`734`. diff --git a/Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst b/Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst new file mode 100644 index 00000000000000..17fc9171390dd9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst @@ -0,0 +1 @@ +Properly quote template strings in :mod:`venv` activation scripts. diff --git a/Misc/NEWS.d/next/Library/2024-10-04-22-43-48.gh-issue-124984.xjMv9b.rst b/Misc/NEWS.d/next/Library/2024-10-04-22-43-48.gh-issue-124984.xjMv9b.rst new file mode 100644 index 00000000000000..dd0a55a6854c25 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-04-22-43-48.gh-issue-124984.xjMv9b.rst @@ -0,0 +1 @@ +Fixed thread safety in :mod:`ssl` in the free-threaded build. OpenSSL operations are now protected by a per-object lock. diff --git a/Misc/NEWS.d/next/Library/2024-10-09-17-07-33.gh-issue-52551.PBakSY.rst b/Misc/NEWS.d/next/Library/2024-10-09-17-07-33.gh-issue-52551.PBakSY.rst new file mode 100644 index 00000000000000..edc9ac5bb23117 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-09-17-07-33.gh-issue-52551.PBakSY.rst @@ -0,0 +1,8 @@ +Fix encoding issues in :func:`time.strftime`, the +:meth:`~datetime.datetime.strftime` method of the :mod:`datetime` classes +:class:`~datetime.datetime`, :class:`~datetime.date` and +:class:`~datetime.time` and formatting of these classes. Characters not +encodable in the current locale are now acceptable in the format string. +Surrogate pairs and sequence of surrogatescape-encoded bytes are no longer +recombinated. Embedded null character no longer terminates the format +string. diff --git a/Misc/NEWS.d/next/Library/2024-10-11-00-40-13.gh-issue-125245.8vReM-.rst b/Misc/NEWS.d/next/Library/2024-10-11-00-40-13.gh-issue-125245.8vReM-.rst new file mode 100644 index 00000000000000..c880efe73d06b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-11-00-40-13.gh-issue-125245.8vReM-.rst @@ -0,0 +1,2 @@ +Fix race condition when importing :mod:`collections.abc`, which could +incorrectly return an empty module. diff --git a/Misc/NEWS.d/next/Library/2024-10-13-15-04-58.gh-issue-125398.UW7Ndv.rst b/Misc/NEWS.d/next/Library/2024-10-13-15-04-58.gh-issue-125398.UW7Ndv.rst new file mode 100644 index 00000000000000..a188b35e1fbdbc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-13-15-04-58.gh-issue-125398.UW7Ndv.rst @@ -0,0 +1 @@ +Fix the conversion of the :envvar:`!VIRTUAL_ENV` path in the activate script in :mod:`venv` when running in Git Bash for Windows. diff --git a/Misc/NEWS.d/next/Library/2024-10-14-02-07-44.gh-issue-125115.IOf3ON.rst b/Misc/NEWS.d/next/Library/2024-10-14-02-07-44.gh-issue-125115.IOf3ON.rst new file mode 100644 index 00000000000000..3583d537a6ec61 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-14-02-07-44.gh-issue-125115.IOf3ON.rst @@ -0,0 +1 @@ +Fixed a bug in :mod:`pdb` where arguments starting with ``-`` can't be passed to the debugged script. diff --git a/Misc/NEWS.d/next/Library/2024-10-14-17-29-34.gh-issue-125451.fmP3T9.rst b/Misc/NEWS.d/next/Library/2024-10-14-17-29-34.gh-issue-125451.fmP3T9.rst new file mode 100644 index 00000000000000..589988d4d6273f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-14-17-29-34.gh-issue-125451.fmP3T9.rst @@ -0,0 +1,2 @@ +Fix deadlock when :class:`concurrent.futures.ProcessPoolExecutor` shuts down +concurrently with an error when feeding a job to a worker process. diff --git a/Misc/NEWS.d/next/Library/2024-10-15-14-01-03.gh-issue-125519.TqGh6a.rst b/Misc/NEWS.d/next/Library/2024-10-15-14-01-03.gh-issue-125519.TqGh6a.rst new file mode 100644 index 00000000000000..e6062625104590 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-15-14-01-03.gh-issue-125519.TqGh6a.rst @@ -0,0 +1,2 @@ +Improve traceback if :func:`importlib.reload` is called with an object that +is not a module. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2024-10-15-16-50-03.gh-issue-125541.FfhmWo.rst b/Misc/NEWS.d/next/Library/2024-10-15-16-50-03.gh-issue-125541.FfhmWo.rst new file mode 100644 index 00000000000000..7a20bca1739869 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-15-16-50-03.gh-issue-125541.FfhmWo.rst @@ -0,0 +1,4 @@ +Pressing :kbd:`Ctrl-C` while blocked in :meth:`threading.Lock.acquire`, +:meth:`threading.RLock.acquire`, and :meth:`threading.Thread.join` now +interrupts the function call and raises a :exc:`KeyboardInterrupt` exception +on Windows, similar to how those functions behave on macOS and Linux. diff --git a/Misc/NEWS.d/next/Library/2024-10-16-04-50-53.gh-issue-125542.vZJ-Ns.rst b/Misc/NEWS.d/next/Library/2024-10-16-04-50-53.gh-issue-125542.vZJ-Ns.rst new file mode 100644 index 00000000000000..777920cc54ff9b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-16-04-50-53.gh-issue-125542.vZJ-Ns.rst @@ -0,0 +1,2 @@ +Deprecate passing keyword-only *prefix_chars* argument to +:meth:`argparse.ArgumentParser.add_argument_group`. diff --git a/Misc/NEWS.d/next/Library/2024-10-16-15-55-50.gh-issue-125600.yMsJx0.rst b/Misc/NEWS.d/next/Library/2024-10-16-15-55-50.gh-issue-125600.yMsJx0.rst new file mode 100644 index 00000000000000..19bf4fbefb601b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-16-15-55-50.gh-issue-125600.yMsJx0.rst @@ -0,0 +1 @@ +Only show stale code warning in :mod:`pdb` when we display source code. diff --git a/Misc/NEWS.d/next/Library/2024-10-16-20-32-40.gh-issue-125590.stHzOP.rst b/Misc/NEWS.d/next/Library/2024-10-16-20-32-40.gh-issue-125590.stHzOP.rst new file mode 100644 index 00000000000000..dc6765ada641a9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-16-20-32-40.gh-issue-125590.stHzOP.rst @@ -0,0 +1 @@ +Allow ``FrameLocalsProxy`` to delete and pop if the key is not a fast variable. diff --git a/Misc/NEWS.d/next/Library/2024-10-16-22-45-50.gh-issue-125614.3OEo_Q.rst b/Misc/NEWS.d/next/Library/2024-10-16-22-45-50.gh-issue-125614.3OEo_Q.rst new file mode 100644 index 00000000000000..5f4803c9b74578 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-16-22-45-50.gh-issue-125614.3OEo_Q.rst @@ -0,0 +1,3 @@ +In the :data:`~annotationlib.Format.FORWARDREF` format of +:mod:`annotationlib`, fix bug where nested expressions were not returned as +:class:`annotationlib.ForwardRef` format. diff --git a/Misc/NEWS.d/next/Library/2024-10-17-04-52-00.gh-issue-125633.lMck06.rst b/Misc/NEWS.d/next/Library/2024-10-17-04-52-00.gh-issue-125633.lMck06.rst new file mode 100644 index 00000000000000..e816a13b75e0c7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-17-04-52-00.gh-issue-125633.lMck06.rst @@ -0,0 +1,2 @@ +Add function :func:`inspect.ispackage` to determine whether an object is a +:term:`package` or not. diff --git a/Misc/NEWS.d/next/Library/2024-10-17-16-10-29.gh-issue-125259.oMew0c.rst b/Misc/NEWS.d/next/Library/2024-10-17-16-10-29.gh-issue-125259.oMew0c.rst new file mode 100644 index 00000000000000..4fa6330abea512 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-17-16-10-29.gh-issue-125259.oMew0c.rst @@ -0,0 +1 @@ +Fix the notes removal logic for errors thrown in enum initialization. diff --git a/Misc/NEWS.d/next/Library/2024-10-17-20-36-06.gh-issue-52551.EIVNYY.rst b/Misc/NEWS.d/next/Library/2024-10-17-20-36-06.gh-issue-52551.EIVNYY.rst new file mode 100644 index 00000000000000..48d3d93c3d72af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-17-20-36-06.gh-issue-52551.EIVNYY.rst @@ -0,0 +1 @@ +Use :c:func:`!wcsftime` to implement :func:`time.strftime` on Windows. diff --git a/Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst b/Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst new file mode 100644 index 00000000000000..74d76c7bddae7d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst @@ -0,0 +1 @@ +Reject invalid unicode escapes for Python implementation of :func:`json.loads`. diff --git a/Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst b/Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst new file mode 100644 index 00000000000000..3eb2905ad8d810 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst @@ -0,0 +1,2 @@ +Reject non-ASCII digits in the Python implementation of :func:`json.loads` +conforming to the JSON specification. diff --git a/Misc/NEWS.d/next/Library/2024-10-19-01-30-40.gh-issue-125378.WTosxX.rst b/Misc/NEWS.d/next/Library/2024-10-19-01-30-40.gh-issue-125378.WTosxX.rst new file mode 100644 index 00000000000000..dc76889d3b210a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-19-01-30-40.gh-issue-125378.WTosxX.rst @@ -0,0 +1 @@ +Fixed the bug in :mod:`pdb` where after a multi-line command, an empty line repeats the first line of the multi-line command, instead of the full command. diff --git a/Misc/NEWS.d/next/Library/2024-10-19-13-37-37.gh-issue-125710.FyFAAr.rst b/Misc/NEWS.d/next/Library/2024-10-19-13-37-37.gh-issue-125710.FyFAAr.rst new file mode 100644 index 00000000000000..8d5220e9889c3a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-19-13-37-37.gh-issue-125710.FyFAAr.rst @@ -0,0 +1 @@ +[Enum] fix hashable<->nonhashable comparisons for member values diff --git a/Misc/NEWS.d/next/Library/2024-10-19-16-06-52.gh-issue-125666.jGfdCP.rst b/Misc/NEWS.d/next/Library/2024-10-19-16-06-52.gh-issue-125666.jGfdCP.rst new file mode 100644 index 00000000000000..3b4488815cced6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-19-16-06-52.gh-issue-125666.jGfdCP.rst @@ -0,0 +1 @@ +Avoid the exiting the interpreter if a null byte is given as input in the new REPL. diff --git a/Misc/NEWS.d/next/Library/2024-10-21-12-06-55.gh-issue-124969.xiY8UP.rst b/Misc/NEWS.d/next/Library/2024-10-21-12-06-55.gh-issue-124969.xiY8UP.rst new file mode 100644 index 00000000000000..c44550184e0000 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-21-12-06-55.gh-issue-124969.xiY8UP.rst @@ -0,0 +1,2 @@ +``locale.nl_langinfo(locale.ALT_DIGITS)`` now returns a string again. The +returned value consists of up to 100 semicolon-separated symbols. diff --git a/Misc/NEWS.d/next/Library/2024-10-21-13-52-37.gh-issue-125767.0kK4lX.rst b/Misc/NEWS.d/next/Library/2024-10-21-13-52-37.gh-issue-125767.0kK4lX.rst new file mode 100644 index 00000000000000..bfda740a79d10e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-21-13-52-37.gh-issue-125767.0kK4lX.rst @@ -0,0 +1,2 @@ +:class:`super` objects are now :mod:`pickleable ` and +:mod:`copyable `. diff --git a/Misc/NEWS.d/next/Library/2024-10-22-13-28-00.gh-issue-125355.zssHm_.rst b/Misc/NEWS.d/next/Library/2024-10-22-13-28-00.gh-issue-125355.zssHm_.rst new file mode 100644 index 00000000000000..fd67f697641d92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-22-13-28-00.gh-issue-125355.zssHm_.rst @@ -0,0 +1,7 @@ +Fix several bugs in :meth:`argparse.ArgumentParser.parse_intermixed_args`. + +* The parser no longer changes temporarily during parsing. +* Default values are not processed twice. +* Required mutually exclusive groups containing positional arguments are now supported. +* The missing arguments report now includes the names of all required optional and positional arguments. +* Unknown options can be intermixed with positional arguments in parse_known_intermixed_args(). diff --git a/Misc/NEWS.d/next/Library/2024-10-23-17-45-40.gh-issue-125884.41E_PD.rst b/Misc/NEWS.d/next/Library/2024-10-23-17-45-40.gh-issue-125884.41E_PD.rst new file mode 100644 index 00000000000000..684b1f282b143e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-23-17-45-40.gh-issue-125884.41E_PD.rst @@ -0,0 +1 @@ +Fixed the bug for :mod:`pdb` where it can't set breakpoints on functions with certain annotations. diff --git a/Misc/NEWS.d/next/Library/2024-10-24-14-08-10.gh-issue-125789.eaiAMw.rst b/Misc/NEWS.d/next/Library/2024-10-24-14-08-10.gh-issue-125789.eaiAMw.rst new file mode 100644 index 00000000000000..964a006bb47b7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-24-14-08-10.gh-issue-125789.eaiAMw.rst @@ -0,0 +1 @@ +Fix possible crash when mutating list of callbacks returned by :attr:`!asyncio.Future._callbacks`. It now always returns a new copy in C implementation :mod:`!_asyncio`. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst new file mode 100644 index 00000000000000..9fe8795de18003 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst @@ -0,0 +1,2 @@ +Fix a use-after-free crash in :meth:`asyncio.Future.remove_done_callback`. +Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-10-25-11-13-24.gh-issue-125969.YvbrTr.rst b/Misc/NEWS.d/next/Library/2024-10-25-11-13-24.gh-issue-125969.YvbrTr.rst new file mode 100644 index 00000000000000..dc99adff7416c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-25-11-13-24.gh-issue-125969.YvbrTr.rst @@ -0,0 +1,2 @@ +Fix an out-of-bounds crash when an evil :meth:`asyncio.loop.call_soon` +mutates the length of the internal callbacks list. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst b/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst new file mode 100644 index 00000000000000..7a1d7b53b11301 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst @@ -0,0 +1,3 @@ +Fix use-after-free crashes on :class:`asyncio.Future` objects for which the +underlying event loop implements an evil :meth:`~object.__getattribute__`. +Reported by Nico-Posada. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst b/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst new file mode 100644 index 00000000000000..ddf1f9725d9695 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst @@ -0,0 +1,3 @@ +Add :meth:`pathlib.Path.scandir` method to efficiently fetch directory +children and their file attributes. This is a trivial wrapper of +:func:`os.scandir`. diff --git a/Misc/NEWS.d/next/Library/2024-10-28-11-33-59.gh-issue-126068.Pdznm_.rst b/Misc/NEWS.d/next/Library/2024-10-28-11-33-59.gh-issue-126068.Pdznm_.rst new file mode 100644 index 00000000000000..a0faf61890da17 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-28-11-33-59.gh-issue-126068.Pdznm_.rst @@ -0,0 +1,5 @@ +Fix exceptions in the :mod:`argparse` module so that only error messages for +ArgumentError and ArgumentTypeError are now translated. +ArgumentError is now only used for command line errors, not for logical +errors in the program. TypeError is now raised instead of ValueError for +some logical errors. diff --git a/Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst b/Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst new file mode 100644 index 00000000000000..d64b7dd2fedbd6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst @@ -0,0 +1 @@ +Fixed a reference leak in :class:`asyncio.Task` objects when reinitializing the same object with a non-``None`` context. Patch by Nico Posada. diff --git a/Misc/NEWS.d/next/Library/2024-10-29-07-24-52.gh-issue-125322.sstOM-.rst b/Misc/NEWS.d/next/Library/2024-10-29-07-24-52.gh-issue-125322.sstOM-.rst new file mode 100644 index 00000000000000..e0cc91d287f89d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-29-07-24-52.gh-issue-125322.sstOM-.rst @@ -0,0 +1 @@ +Correct detection of complex numbers support in libffi. diff --git a/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst b/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst new file mode 100644 index 00000000000000..e54ac17b217c92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst @@ -0,0 +1,3 @@ +Fix a use-after-free crash on :class:`asyncio.Task` objects for which the +underlying event loop implements an evil :meth:`~object.__getattribute__`. +Reported by Nico-Posada. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-10-29-10-58-52.gh-issue-126106.rlF798.rst b/Misc/NEWS.d/next/Library/2024-10-29-10-58-52.gh-issue-126106.rlF798.rst new file mode 100644 index 00000000000000..de989007b4c35a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-29-10-58-52.gh-issue-126106.rlF798.rst @@ -0,0 +1 @@ +Fixes a possible ``NULL`` pointer dereference in :mod:`ssl`. diff --git a/Misc/NEWS.d/next/Library/2024-10-29-11-45-44.gh-issue-126105.cOL-R6.rst b/Misc/NEWS.d/next/Library/2024-10-29-11-45-44.gh-issue-126105.cOL-R6.rst new file mode 100644 index 00000000000000..547eb3af1ca064 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-29-11-45-44.gh-issue-126105.cOL-R6.rst @@ -0,0 +1 @@ +Fix a crash in :mod:`ast` when the :attr:`ast.AST._fields` attribute is deleted. diff --git a/Misc/NEWS.d/next/Library/2024-10-30-00-12-22.gh-issue-126156.BOSqv0.rst b/Misc/NEWS.d/next/Library/2024-10-30-00-12-22.gh-issue-126156.BOSqv0.rst new file mode 100644 index 00000000000000..4fe18275ab9384 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-30-00-12-22.gh-issue-126156.BOSqv0.rst @@ -0,0 +1 @@ +Improved performances of creating :py:class:`~http.cookies.Morsel` objects by a factor of 3.8x. diff --git a/Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst b/Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst new file mode 100644 index 00000000000000..c92ffb75056606 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst @@ -0,0 +1,2 @@ +Fix issue where :func:`urllib.request.pathname2url` generated URLs beginning +with four slashes (rather than two) when given a Windows UNC path. diff --git a/Misc/NEWS.d/next/Library/2024-10-30-23-42-44.gh-issue-126223.k2qooc.rst b/Misc/NEWS.d/next/Library/2024-10-30-23-42-44.gh-issue-126223.k2qooc.rst new file mode 100644 index 00000000000000..fee391c030b941 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-30-23-42-44.gh-issue-126223.k2qooc.rst @@ -0,0 +1,2 @@ +Raise a :exc:`UnicodeEncodeError` instead of a :exc:`SystemError` upon +calling :func:`!_interpreters.create` with an invalid Unicode character. diff --git a/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst b/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst new file mode 100644 index 00000000000000..555f2f3bafbf33 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst @@ -0,0 +1,2 @@ +Fix crash in :class:`!cProfile.Profile` and :class:`!_lsprof.Profiler` when their +callbacks were directly called with 0 arguments. diff --git a/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst b/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst new file mode 100644 index 00000000000000..459eebc82bd42a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst @@ -0,0 +1,3 @@ +Fix a use-after-free crash on :class:`asyncio.Task` objects +whose underlying coroutine yields an object that implements +an evil :meth:`~object.__getattribute__`. Patch by Nico Posada. diff --git a/Misc/NEWS.d/next/Tests/2023-06-02-05-04-15.gh-issue-105089.KaZFtU.rst b/Misc/NEWS.d/next/Tests/2023-06-02-05-04-15.gh-issue-105089.KaZFtU.rst deleted file mode 100644 index d04ef435dd572d..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-06-02-05-04-15.gh-issue-105089.KaZFtU.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix -``test.test_zipfile.test_core.TestWithDirectory.test_create_directory_with_write`` -test in AIX by doing a bitwise AND of 0xFFFF on mode , so that it will be in -sync with ``zinfo.external_attr`` diff --git a/Misc/NEWS.d/next/Tests/2024-02-02-13-18-55.gh-issue-114099.C_ycWg.rst b/Misc/NEWS.d/next/Tests/2024-02-02-13-18-55.gh-issue-114099.C_ycWg.rst deleted file mode 100644 index 487cd5062fc75b..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-02-13-18-55.gh-issue-114099.C_ycWg.rst +++ /dev/null @@ -1 +0,0 @@ -Added test exclusions required to run the test suite on iOS. diff --git a/Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst b/Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst new file mode 100644 index 00000000000000..3c2455cfc8c530 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst @@ -0,0 +1 @@ +Add translation tests to the :mod:`argparse` module. diff --git a/Misc/NEWS.d/next/Tests/2024-10-21-14-10-56.gh-issue-125730.kcWbvI.rst b/Misc/NEWS.d/next/Tests/2024-10-21-14-10-56.gh-issue-125730.kcWbvI.rst new file mode 100644 index 00000000000000..061a1f9f27599c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-10-21-14-10-56.gh-issue-125730.kcWbvI.rst @@ -0,0 +1,2 @@ +Change ``make test`` to not run GUI tests by default. Use ``make ci`` to run +tests with GUI tests instead. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-02-05-02-45-51.gh-issue-115015.rgtiDB.rst b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-02-45-51.gh-issue-115015.rgtiDB.rst deleted file mode 100644 index d8739d28eb2b73..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2024-02-05-02-45-51.gh-issue-115015.rgtiDB.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix a bug in Argument Clinic that generated incorrect code for methods with -no parameters that use the :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS -` calling convention. Only the -positional parameter count was checked; any keyword argument passed would be -silently accepted. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-02-05-19-00-32.gh-issue-109991.yJSEkw.rst b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-19-00-32.gh-issue-109991.yJSEkw.rst deleted file mode 100644 index 4eb4d39629b9bc..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2024-02-05-19-00-32.gh-issue-109991.yJSEkw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update GitHub CI workflows to use OpenSSL 3.0.13 and multissltests to use -1.1.1w, 3.0.13, 3.1.5, and 3.2.1. diff --git a/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst b/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst deleted file mode 100644 index 429cd5bc923e09..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst +++ /dev/null @@ -1 +0,0 @@ -Adds free-threaded binaries to Windows installer as an optional component. diff --git a/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst b/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst deleted file mode 100644 index 388d61a2b3bd6d..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst +++ /dev/null @@ -1 +0,0 @@ -The ``py.exe`` launcher will no longer attempt to run the Microsoft Store redirector when launching a script containing a ``/usr/bin/env`` shebang diff --git a/Misc/NEWS.d/next/Windows/2024-02-01-14-35-05.gh-issue-111239.SO7SUF.rst b/Misc/NEWS.d/next/Windows/2024-02-01-14-35-05.gh-issue-111239.SO7SUF.rst deleted file mode 100644 index ea82c3b941f802..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-01-14-35-05.gh-issue-111239.SO7SUF.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows builds to use zlib v1.3.1. diff --git a/Misc/NEWS.d/next/Windows/2024-02-05-16-53-12.gh-issue-109991.YqjnDz.rst b/Misc/NEWS.d/next/Windows/2024-02-05-16-53-12.gh-issue-109991.YqjnDz.rst deleted file mode 100644 index d9923c35c2726e..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-05-16-53-12.gh-issue-109991.YqjnDz.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows build to use OpenSSL 3.0.13. diff --git a/Misc/NEWS.d/next/Windows/2024-02-06-09-05-13.gh-issue-115009.ShMjZs.rst b/Misc/NEWS.d/next/Windows/2024-02-06-09-05-13.gh-issue-115009.ShMjZs.rst deleted file mode 100644 index 5bdb6963a24311..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-06-09-05-13.gh-issue-115009.ShMjZs.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.45.1. diff --git a/Misc/NEWS.d/next/Windows/2024-10-15-21-28-43.gh-issue-125550.hmGWCP.rst b/Misc/NEWS.d/next/Windows/2024-10-15-21-28-43.gh-issue-125550.hmGWCP.rst new file mode 100644 index 00000000000000..c3ae00c74b3d91 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-10-15-21-28-43.gh-issue-125550.hmGWCP.rst @@ -0,0 +1,2 @@ +Enable the :ref:`launcher` to detect Python 3.14 installs from the Windows +Store. diff --git a/Misc/NEWS.d/next/Windows/2024-10-23-17-24-23.gh-issue-125842.m3EF9E.rst b/Misc/NEWS.d/next/Windows/2024-10-23-17-24-23.gh-issue-125842.m3EF9E.rst new file mode 100644 index 00000000000000..63644721d57f5b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-10-23-17-24-23.gh-issue-125842.m3EF9E.rst @@ -0,0 +1,2 @@ +Fix a :exc:`SystemError` when :func:`sys.exit` is called with ``0xffffffff`` +on Windows. diff --git a/Misc/NEWS.d/next/Windows/2024-10-29-09-39-06.gh-issue-126084.3wAL8o.rst b/Misc/NEWS.d/next/Windows/2024-10-29-09-39-06.gh-issue-126084.3wAL8o.rst new file mode 100644 index 00000000000000..319053d95db19e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-10-29-09-39-06.gh-issue-126084.3wAL8o.rst @@ -0,0 +1,2 @@ +Fix venvwlauncher to launch pythonw instead of python so no extra console +window is created. diff --git a/Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst b/Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst new file mode 100644 index 00000000000000..3d813248766a5b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst @@ -0,0 +1,2 @@ +Avoid crashing in :mod:`platform` due to slow WMI calls on some Windows +machines. diff --git a/Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst b/Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst deleted file mode 100644 index e6554d5c9f1e1e..00000000000000 --- a/Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst +++ /dev/null @@ -1 +0,0 @@ -On macOS the result of ``os.statvfs`` and ``os.fstatvfs`` now correctly report the size of very large disks, in previous versions the reported number of blocks was wrong for disks with at least 2**32 blocks. diff --git a/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst b/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst deleted file mode 100644 index abd296f8608518..00000000000000 --- a/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst +++ /dev/null @@ -1 +0,0 @@ -Add Mach-O linkage support for :func:`platform.architecture()`. diff --git a/Misc/NEWS.d/next/macOS/2024-02-05-18-30-27.gh-issue-109991.tun6Yu.rst b/Misc/NEWS.d/next/macOS/2024-02-05-18-30-27.gh-issue-109991.tun6Yu.rst deleted file mode 100644 index 79b45e7d51da3f..00000000000000 --- a/Misc/NEWS.d/next/macOS/2024-02-05-18-30-27.gh-issue-109991.tun6Yu.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use OpenSSL 3.0.13. diff --git a/Misc/NEWS.d/next/macOS/2024-02-06-09-01-10.gh-issue-115009.ysau7e.rst b/Misc/NEWS.d/next/macOS/2024-02-06-09-01-10.gh-issue-115009.ysau7e.rst deleted file mode 100644 index 47ec488c3cced2..00000000000000 --- a/Misc/NEWS.d/next/macOS/2024-02-06-09-01-10.gh-issue-115009.ysau7e.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use SQLite 3.45.1. diff --git a/Misc/README b/Misc/README index 3dab768ba1a7a4..cbad9b72dc713c 100644 --- a/Misc/README +++ b/Misc/README @@ -17,7 +17,6 @@ python.man UNIX man page for the python interpreter python.pc.in Package configuration info template for pkg-config README The file you're reading now README.AIX Information about using Python on AIX -README.coverity Information about running Coverity's Prevent on Python README.valgrind Information for Valgrind users, see valgrind-python.supp SpecialBuilds.txt Describes extra symbols you can set for debug builds svnmap.txt Map of old SVN revs and branches to hg changeset ids, diff --git a/Misc/README.coverity b/Misc/README.coverity deleted file mode 100644 index f5e1bf6f28d245..00000000000000 --- a/Misc/README.coverity +++ /dev/null @@ -1,22 +0,0 @@ - -Coverity has a static analysis tool (Prevent) which is similar to Klocwork. -They run their tool on the Python source code (SVN head) on a daily basis. -The results are available at: - - http://scan.coverity.com/ - -About 20 people have access to the analysis reports. Other -people can be added by request. - -Prevent was first run on the Python 2.5 source code in March 2006. -There were originally about 100 defects reported. Some of these -were false positives. Over 70 issues were uncovered. - -Each warning has a unique id and comments that can be made on it. -When checking in changes due to a warning, the unique id -as reported by the tool was added to the SVN commit message. - -False positives were annotated so that the comments can -be reviewed and reversed if the analysis was incorrect. - -Contact python-dev@python.org for more information. diff --git a/Misc/coverity_model.c b/Misc/coverity_model.c deleted file mode 100644 index 8960362a6d74a0..00000000000000 --- a/Misc/coverity_model.c +++ /dev/null @@ -1,179 +0,0 @@ -/* Coverity Scan model - * - * This is a modeling file for Coverity Scan. Modeling helps to avoid false - * positives. - * - * - A model file can't import any header files. - * - Therefore only some built-in primitives like int, char and void are - * available but not wchar_t, NULL etc. - * - Modeling doesn't need full structs and typedefs. Rudimentary structs - * and similar types are sufficient. - * - An uninitialized local pointer is not an error. It signifies that the - * variable could be either NULL or have some data. - * - * Coverity Scan doesn't pick up modifications automatically. The model file - * must be uploaded by an admin in the analysis settings of - * http://scan.coverity.com/projects/200 - */ - -/* dummy definitions, in most cases struct fields aren't required. */ - -#define NULL (void *)0 -#define assert(op) /* empty */ -typedef int sdigit; -typedef long Py_ssize_t; -typedef unsigned short wchar_t; -typedef struct {} PyObject; -typedef struct {} grammar; -typedef struct {} DIR; -typedef struct {} RFILE; - -/* Python/pythonrun.c - * resource leak false positive */ - -void Py_FatalError(const char *msg) { - __coverity_panic__(); -} - -/* Objects/longobject.c - * NEGATIVE_RETURNS false positive */ - -static PyObject *get_small_int(sdigit ival) -{ - /* Never returns NULL */ - PyObject *p; - assert(p != NULL); - return p; -} - -PyObject *PyLong_FromLong(long ival) -{ - PyObject *p; - int maybe; - - if ((ival >= -5) && (ival < 257 + 5)) { - p = get_small_int(ival); - assert(p != NULL); - return p; - } - if (maybe) - return p; - else - return NULL; -} - -PyObject *PyLong_FromLongLong(long long ival) -{ - return PyLong_FromLong((long)ival); -} - -PyObject *PyLong_FromSsize_t(Py_ssize_t ival) -{ - return PyLong_FromLong((long)ival); -} - -/* tainted sinks - * - * Coverity considers argv, environ, read() data etc as tained. - */ - -PyObject *PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) -{ - __coverity_tainted_data_sink__(filename); - return NULL; -} - -/* Python/fileutils.c */ -wchar_t *Py_DecodeLocale(const char* arg, size_t *size) -{ - wchar_t *w; - __coverity_tainted_data_sink__(arg); - __coverity_tainted_data_sink__(size); - return w; -} - -/* Python/marshal.c */ - -static Py_ssize_t r_string(char *s, Py_ssize_t n, RFILE *p) -{ - __coverity_tainted_string_argument__(s); - return 0; -} - -static long r_long(RFILE *p) -{ - long l; - unsigned char buffer[4]; - - r_string((char *)buffer, 4, p); - __coverity_tainted_string_sanitize_content__(buffer); - l = (long)buffer; - return l; -} - -/* Coverity doesn't understand that fdopendir() may take ownership of fd. */ - -DIR *fdopendir(int fd) -{ - DIR *d; - if (d) { - __coverity_close__(fd); - } - return d; -} - -/* Modules/_datetime.c - * - * Coverity thinks that the input values for these function come from a - * tainted source PyDateTime_DATE_GET_* macros use bit shifting. - */ -static PyObject * -build_struct_time(int y, int m, int d, int hh, int mm, int ss, int dstflag) -{ - PyObject *result; - - __coverity_tainted_data_sanitize__(y); - __coverity_tainted_data_sanitize__(m); - __coverity_tainted_data_sanitize__(d); - __coverity_tainted_data_sanitize__(hh); - __coverity_tainted_data_sanitize__(mm); - __coverity_tainted_data_sanitize__(ss); - __coverity_tainted_data_sanitize__(dstflag); - - return result; -} - -static int -ymd_to_ord(int year, int month, int day) -{ - int ord = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return ord; -} - -static int -normalize_date(int *year, int *month, int *day) -{ - __coverity_tainted_data_sanitize__(*year); - __coverity_tainted_data_sanitize__(*month); - __coverity_tainted_data_sanitize__(*day); - - return 0; -} - -static int -weekday(int year, int month, int day) -{ - int w = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return w; -} - diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json new file mode 100644 index 00000000000000..d54b1fbe251378 --- /dev/null +++ b/Misc/externals.spdx.json @@ -0,0 +1,196 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "packages": [ + { + "SPDXID": "SPDXRef-PACKAGE-bzip2", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ab8d1b0cc087c20d4c32c0e4fcf7d0c733a95da12cedc6d63b3f0a9af07427e2" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/bzip2-1.0.8.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bzip:bzip2:1.0.8:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "bzip2", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.8" + }, + { + "SPDXID": "SPDXRef-PACKAGE-libffi", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "9d802681adfea27d84cae0487a785fb9caa925bdad44c401b364c59ab2b8edda" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/libffi-3.4.4.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:libffi_project:libffi:3.4.4:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "libffi", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.4.4" + }, + { + "SPDXID": "SPDXRef-PACKAGE-mpdecimal", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "338fac3fb8cdd60f406b6326431338756f58a8af94229ffd9bf1e7c2b1ad71ca" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-4.0.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:4.0.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "mpdecimal", + "originator": "Organization: bytereef.org", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "4.0.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-openssl", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "1550c87996a0858474a9dd179deab2c55eb73726b9a140b32865b02fd3d8a86b" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.15.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.15:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "openssl", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.0.15" + }, + { + "SPDXID": "SPDXRef-PACKAGE-sqlite", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "730e4a3efd6a63828bee499940fb13acc2a32c182502ce8a1d970387895d0504" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.45.3.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.45.3.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "sqlite", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.45.3.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tcl-core", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "4c23f0dd3efcbe6f3a22c503a68d147617bb30c4f5290f1eb3eaacf0b460440b" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.15.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tcl-core", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.6.15.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tk", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "0ae56d39bca92865f338529557a1e56d110594184b6dc5a91339c5675751e264" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.15.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tk", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.6.15.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-xz", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.2.5.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tukaani:xz:5.2.5:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "xz", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "5.2.5" + }, + { + "SPDXID": "SPDXRef-PACKAGE-zlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e3f3fb32564952006eb18b091ca8464740e5eca29d328cfb0b2da22768e0b638" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-1.3.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:zlib:zlib:1.3.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "zlib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.3.1" + } + ], + "spdxVersion": "SPDX-2.3" +} \ No newline at end of file diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index 3307260544e8a6..ec0857a4a998c0 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -12,8 +12,20 @@ #undef powerpc #undef sparc #undef unix + #if defined(__ANDROID__) - # Android is not a multiarch system. +# if defined(__x86_64__) +PLATFORM_TRIPLET=x86_64-linux-android +# elif defined(__i386__) +PLATFORM_TRIPLET=i686-linux-android +# elif defined(__aarch64__) +PLATFORM_TRIPLET=aarch64-linux-android +# elif defined(__arm__) +PLATFORM_TRIPLET=arm-linux-androideabi +# else +# error unknown Android platform +# endif + #elif defined(__linux__) /* * BEGIN of Linux block @@ -233,7 +245,24 @@ PLATFORM_TRIPLET=i386-gnu # error unknown platform triplet # endif #elif defined(__APPLE__) +# include "TargetConditionals.h" +// Older macOS SDKs do not define TARGET_OS_* +# if defined(TARGET_OS_IOS) && TARGET_OS_IOS +# if defined(TARGET_OS_SIMULATOR) && TARGET_OS_SIMULATOR +# if __x86_64__ +PLATFORM_TRIPLET=x86_64-iphonesimulator +# else +PLATFORM_TRIPLET=arm64-iphonesimulator +# endif +# else +PLATFORM_TRIPLET=arm64-iphoneos +# endif +// Older macOS SDKs do not define TARGET_OS_OSX +# elif !defined(TARGET_OS_OSX) || TARGET_OS_OSX PLATFORM_TRIPLET=darwin +# else +# error unknown Apple platform +# endif #elif defined(__VXWORKS__) PLATFORM_TRIPLET=vxworks #elif defined(__wasm32__) diff --git a/Misc/python-config.in b/Misc/python-config.in index 81c3316e334a48..dd5d161ab2286f 100644 --- a/Misc/python-config.in +++ b/Misc/python-config.in @@ -13,7 +13,8 @@ valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags', def exit_with_usage(code=1): print("Usage: {0} [{1}]".format( - sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), file=sys.stderr) + sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), + file=sys.stdout if code == 0 else sys.stderr) sys.exit(code) try: diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 2602fe24c0402e..555b0cb6ba2a48 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -4,7 +4,13 @@ exit_with_usage () { - echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + local usage + usage="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + if [ "$1" -eq 0 ]; then + echo "$usage" + else + echo "$usage" >&2 + fi exit $1 } diff --git a/Misc/python.man b/Misc/python.man index 0f5dfa2e2289f7..4076b8d3d1ba30 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -251,6 +251,7 @@ emitted by a process (even those that are otherwise ignored by default): -Wdefault # Warn once per call location -Werror # Convert to exceptions -Walways # Warn every time + -Wall # Same as -Walways -Wmodule # Warn once per calling module -Wonce # Warn once per Python process -Wignore # Never warn @@ -607,6 +608,10 @@ output. Setting it to 0 deactivates this behavior. .IP PYTHON_HISTORY This environment variable can be used to set the location of a history file (on Unix, it is \fI~/.python_history\fP by default). +.IP PYTHON_GIL +If this variable is set to 1, the global interpreter lock (GIL) will be forced +on. Setting it to 0 forces the GIL off. Only available in builds configured +with \fB--disable-gil\fP. .SS Debug-mode variables Setting these variables only has an effect in a debug build of Python, that is, if Python was configured with the diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index d783d14255e66f..cc73e93009b43f 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,29 +48,15 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ab7bb32514d170592dfb3f76e41bbdc075a4e7e0" + "checksumValue": "6aaee1b194bea30f0a60d1cce71eada8b14d3526" }, { "algorithm": "SHA256", - "checksumValue": "f521acdad222644365b0e81a33bcd6939a98c91b225c47582cc84bd73d96febc" + "checksumValue": "7bd4e53a8015534b5bbb58afe1a131b3989d3d4fca29bca685c44d34bcaa2555" } ], "fileName": "Modules/expat/expat.h" }, - { - "SPDXID": "SPDXRef-FILE-Modules-expat-expat-config.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "73627287302ee3e84347c4fe21f37a9cb828bc3b" - }, - { - "algorithm": "SHA256", - "checksumValue": "f17e59f9d95eeb05694c02508aa284d332616c22cbe2e6a802d8a0710310eaab" - } - ], - "fileName": "Modules/expat/expat_config.h" - }, { "SPDXID": "SPDXRef-FILE-Modules-expat-expat-external.h", "checksums": [ @@ -104,11 +90,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2790d37e7de2f13dccc4f4fb352cbdf9ed6abaa2" + "checksumValue": "e23d160cc33cc2c25a4b48f7b242f906444418e0" }, { "algorithm": "SHA256", - "checksumValue": "d2efe5a1018449968a689f444cca432e3d5875aba6ad08ee18ca235d64f41bb9" + "checksumValue": "f7523357d8009749e7dba94b0bd7d0fa60e011cc254e55c4ebccd6313f031122" } ], "fileName": "Modules/expat/internal.h" @@ -146,11 +132,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "baa44fe4581895d42e8d5e83d8ce6a69b1c34dbe" + "checksumValue": "f50c899172acd93fc539007bfb43315b83d407e4" }, { "algorithm": "SHA256", - "checksumValue": "33a7b9ac8bf4571e23272cdf644c6f9808bd44c66b149e3c41ab3870d1888609" + "checksumValue": "d571b8258cfaa067a20adef553e5fcedd6671ca4a8841483496de031bd904567" } ], "fileName": "Modules/expat/pyexpatns.h" @@ -160,11 +146,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2b984f806f10fbfbf72d8d1b7ba2992413c15299" + "checksumValue": "aca27f46d9fd387b63ce7ff2e4f172cad130b39b" }, { "algorithm": "SHA256", - "checksumValue": "fbce56cd680e690043bbf572188cc2d0a25dbfc0d47ac8cb98eb3de768d4e694" + "checksumValue": "f537add526ecda8389503b7ef45fb52b6217e4dc171dcc3a8dc6903ff6134726" } ], "fileName": "Modules/expat/siphash.h" @@ -188,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e774ae6ee9391aa6ffb8f775fb74e48f4b428959" + "checksumValue": "a3a8c44efd55dbf2cfea8fcee009ec63120ec0a3" }, { "algorithm": "SHA256", - "checksumValue": "3c71cea9a6174718542331971a35db317902b2433be9d8dd1cb24239b635c0cc" + "checksumValue": "e70948500d34dfcba4e9f0b305319dfe2a937c7cbfb687905128b56e1a6f8b33" } ], "fileName": "Modules/expat/winconfig.h" @@ -202,11 +188,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "b580e827e16baa6b035586ffcd4d90301e5a353f" + "checksumValue": "b2ec0ad170ccc21e63fbcfc8d7404cdd756eedd3" }, { "algorithm": "SHA256", - "checksumValue": "483518bbd69338eefc706cd7fc0b6039df2d3e347f64097989059ed6d2385a1e" + "checksumValue": "92159d4e17393e56ee85f47d9fb31348695a58589899aa01e7536cdc88f60b85" } ], "fileName": "Modules/expat/xmlparse.c" @@ -216,11 +202,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5ef21312af73deb2428be3fe97a65244608e76de" + "checksumValue": "ef767128d2dda99436712dcf3465dde5dbaab876" }, { "algorithm": "SHA256", - "checksumValue": "6fcf8c72ac0112c1b98bd2039c632a66b4c3dc516ce7c1f981390951121ef3c0" + "checksumValue": "71fb52aa302cf6f56e41943009965804f49ff2210d9bd15b258f70aaf70db772" } ], "fileName": "Modules/expat/xmlrole.c" @@ -230,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c1a4ea6356643d0820edb9c024c20ad2aaf562dc" + "checksumValue": "c961fb1a80f7b0601a63e69fba793fe5f6dff157" }, { "algorithm": "SHA256", - "checksumValue": "2b5d674be6ef20c7e3f69295176d75e68c5616e4dfce0a186fdd5e2ed8315f7a" + "checksumValue": "228470eb9181a9a7575b63137edcb61b817ee4e0923faffdbeba29e07c939713" } ], "fileName": "Modules/expat/xmlrole.h" @@ -244,11 +230,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e6d66ae9fd61d7950c62c5d87693c30a707e8577" + "checksumValue": "8394790c0199c8f88108542ad78f23095d28a3fe" }, { "algorithm": "SHA256", - "checksumValue": "1110f651bdccfa765ad3d6f3857a35887ab35fc0fe7f3f3488fde2b238b482e3" + "checksumValue": "5b16c671ccc42496374762768e4bf48f614aecfd2025a07925b8d94244aec645" } ], "fileName": "Modules/expat/xmltok.c" @@ -258,11 +244,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9c2a544875fd08ba9c2397296c97263518a410aa" + "checksumValue": "7d2943a0128094455004b1a98007b98734221bae" }, { "algorithm": "SHA256", - "checksumValue": "4299a03828b98bfe47ec6809f6e279252954a9a911dc7e0f19551bd74e3af971" + "checksumValue": "6b8919dc951606dc6f2b0175f8955a9ced901ce8bd08db47f291b6c04227ae7f" } ], "fileName": "Modules/expat/xmltok.h" @@ -272,11 +258,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "aa96882de8e3d1d3083124b595aa911efe44e5ad" + "checksumValue": "7756f7c0d3625ae7dde6cf7d386685ffacb57c7e" }, { "algorithm": "SHA256", - "checksumValue": "0fbcba7931707c60301305dab78d2298d96447d0a5513926d8b18135228c0818" + "checksumValue": "a3fe18ff32b21fbcb7c190895c68158404e1b9fb449db6431bc08b261dc03938" } ], "fileName": "Modules/expat/xmltok_impl.c" @@ -310,480 +296,536 @@ "fileName": "Modules/expat/xmltok_ns.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f77449b2b4eb99f1da0938633cc558baf9c444fb" + "checksumValue": "a34e821b68ef5334eccf4f729b28bb7bb65b965e" }, { "algorithm": "SHA256", - "checksumValue": "0f252967debca5b35362ca53951ea16ca8bb97a19a1d24f6695f44d50010859e" + "checksumValue": "4582db9143c0810b98838a5357c577e0b32ae77f3018486159df4e0dfd3fce3c" } ], - "fileName": "Modules/_hacl/Hacl_Hash_MD5.c" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c24e6779a91c840f3d65d24abbce225b608b676e" + "checksumValue": "b0b3ae92d6aee7b52bacfdf02409d8d7e23701ee" }, { "algorithm": "SHA256", - "checksumValue": "9cd062e782801013e3cacaba583e44e1b5e682e217d20208d5323354d42011f1" + "checksumValue": "95d1dd4097a706b0719610da674297fa253b30d03a6ead4685ed648e20cb51a2" } ], - "fileName": "Modules/_hacl/Hacl_Hash_MD5.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "560f6ff541b5eff480ea047b147f4212bb0db7ed" + "checksumValue": "0ffe60c6d5eed5dd222515e820d461d319d16b1f" }, { "algorithm": "SHA256", - "checksumValue": "0ade3ab264e912d7b4e5cdcf773db8c63e4440540d295922d74b06bcfc74c77a" + "checksumValue": "4804cb3ce68bfdcf98853d6f1d77b4a844a3c2796f776b39770ba327e400d402" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA1.c" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "853b77d45379146faaeac5fe899b28db386ad13c" + "checksumValue": "a5011646670c4f51368aca661e458e4c7f1d88e0" }, { "algorithm": "SHA256", - "checksumValue": "b13eb14f91582703819235ea7c8f807bb93e4f1e6b695499dc1d86021dc39e72" + "checksumValue": "f00c1fe8e774c7ec65f6c5a8efa43ce180a17fc80ed6119ada8c4022d058b6e2" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA1.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256-universal2.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "667120b6100c946cdaa442f1173c723339923071" + "checksumValue": "5afc433179d71abd6649596797a7e8953e89172d" }, { "algorithm": "SHA256", - "checksumValue": "b189459b863341a3a9c5c78c0208b6554a2f2ac26e0748fbd4432a91db21fae6" + "checksumValue": "db42da82d18641d68d3670e6201e0cbb43415daaa84f29770b8f0ebf33562975" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "81db38b0b920e63ec33c7109d1144c35cf091da0" + "checksumValue": "cf035ffeff875bc74345a47373ce25dc408ea9dc" }, { "algorithm": "SHA256", - "checksumValue": "631c9ba19c1c2c835bb63d3f2f22b8d76fb535edfed3c254ff2a52f12af3fe61" + "checksumValue": "579059b002c45fab0fed6381e85c3f5eaf1d959400ca64b103542ac6c35bade3" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA2.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9c832b98a2f2a68202d2da016fb718965d7b7602" + "checksumValue": "0328172a62507a051cd60ff9603710ed5aea1bc8" }, { "algorithm": "SHA256", - "checksumValue": "38d350d1184238966cfa821a59ae00343f362182b6c2fbea7f2651763d757fb7" + "checksumValue": "9f3c8ef615c9fbc59ef796d0ad2a7a76a7e55dc8939077b44ca538cbf8889a8c" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ecc766fb6f7ee85e902b593b61b41e5a728fca34" + "checksumValue": "9bb53022d158a9c349edb52a8def8aac7d098a4e" }, { "algorithm": "SHA256", - "checksumValue": "bae290a94366a2460f51e8468144baaade91d9048db111e10d2e2ffddc3f98cf" + "checksumValue": "2abde0c6b5da0402e91b4bedfe786c24b908fbdc04e08e74651c7624729254d9" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA3.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ab7b4d9465a2765a07f8d5bccace7182b28ed1b8" + "checksumValue": "32f35c173c10a2c49ac53c839cfbccd8a147274d" }, { "algorithm": "SHA256", - "checksumValue": "26913613f3b4f8ffff0a3e211a5ebc849159094e5e11de0a31fcb95b6105b74c" + "checksumValue": "8734879b551f0fa860002ae81c0d0cfbade561007d9c26ad18c5a221e239237e" } ], - "fileName": "Modules/_hacl/Hacl_Streaming_Types.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128-universal2.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2ea61d6a236147462045f65c20311819d74db80c" + "checksumValue": "d70c6dbcb91d56bbd80f7bf860e508a748042d0d" }, { "algorithm": "SHA256", - "checksumValue": "2c22b4d49ba06d6a3053cdc66405bd5ae953a28fcfed1ab164e8f5e0f6e2fb8b" + "checksumValue": "5b132ab850a5e0fe6f27e08a955f8989ea3aae8e5b3115f0195039034ece8c04" } ], - "fileName": "Modules/_hacl/include/krml/FStar_UInt128_Verified.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "1a647d841180ac8ca667afa968c353425e81ad0d" + "checksumValue": "f8ba39b46ebdfa7d031d9c33130c6ded680a8120" }, { "algorithm": "SHA256", - "checksumValue": "e5d1c5854833bec7ea02e227ec35bd7b49c5fb9e0f339efa0dd83e1595f722d4" + "checksumValue": "f71cf6a0e8f09354c2af2c785a1d36e0cba7613a589be01ca8a3d8478f4c8874" } ], - "fileName": "Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h" + "fileName": "Modules/_hacl/Hacl_Hash_MD5.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "1987119a563a8fdc5966286e274f716dbcea77ee" + "checksumValue": "eaaab54cea2b0bb8ec0eedf0b373d42f1a0f8f6c" }, { "algorithm": "SHA256", - "checksumValue": "fe57e1bc5ce3224d106e36cb8829b5399c63a68a70b0ccd0c91d82a4565c8869" + "checksumValue": "9a02e2a6e163515ea0228a859d5e55c1f57b11fae5908c42f9f9814ce9bca230" } ], - "fileName": "Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h" + "fileName": "Modules/_hacl/Hacl_Hash_MD5.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "903c9eb76b01f3a95c04c3bc841c2fb71dea5403" + "checksumValue": "f4f42faf8da78a230199f649c0f2a1b865799a31" }, { "algorithm": "SHA256", - "checksumValue": "08ec602c7f90a1540389c0cfc95769fa7fec251e7ca143ef83c0b9f7afcf89a7" + "checksumValue": "5b29bd9951646861e0e19427be5d923a5bab7a4516824ccc068f696469195eec" } ], - "fileName": "Modules/_hacl/include/krml/internal/target.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA1.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "964e09bd99ff2366afd6193b59863fc925e7fb05" + "checksumValue": "722b57139737ceeb88e41d3839e6f7d70578741b" }, { "algorithm": "SHA256", - "checksumValue": "3734c7942bec9a434e16df069fa45bdcb84b130f14417bc5f7bfe8546272d9f5" + "checksumValue": "5640295c790d56b1b4df147d6a6c58803b1845cd7d93365bf7cc7b75ba3cacd5" } ], - "fileName": "Modules/_hacl/include/krml/lowstar_endianness.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA1.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "df8e0ed74a5970d09d3cc4c6e7c6c7a4c4e5015c" + "checksumValue": "f2aa3ed6acce621c162bc3a0592780ce5aa3bc4d" }, { "algorithm": "SHA256", - "checksumValue": "de7444c345caa4c47902c4380500356a3ee7e199d2aab84fd8c4960410154f3d" + "checksumValue": "30638efb75c8b185bb09c3df6977e3f3c5d21a1e696218cf7ade6bc4d5201b31" } ], - "fileName": "Modules/_hacl/include/krml/types.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5dd4ee3c835a0d176a6e9fecbe9752fd1474ff41" + "checksumValue": "4903e10291d07367be3bc283935bc52926e57ba1" }, { "algorithm": "SHA256", - "checksumValue": "d82ef594cba44203576d67b047240316bb3c542912ebb7034afa1e07888cec56" + "checksumValue": "093d7693084af0999d2a13d207311d74b5bdfdc9c08447ed4a979e3f7505ae6b" } ], - "fileName": "Modules/_hacl/internal/Hacl_Hash_MD5.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA2.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "515b3082eb7c30597773e1c63ec46688f6da3634" + "checksumValue": "fc2c3ef83a71bef42eb3f73b78e4ef6642a4634e" }, { "algorithm": "SHA256", - "checksumValue": "10aacf847006b8e0dfb64d5c327443f954db6718b4aec712fb3268230df6a752" + "checksumValue": "e4f3ed9d1e8f661482cbd2d04b197e15cc3b698c5ef2ddedf0eb65df320dbbc4" } ], - "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA1.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "a044ec12b70ba97b67e9a312827d6270452a20ca" + "checksumValue": "7d78e6844dde1f9b5e68f58ca105a4c330461ff6" }, { "algorithm": "SHA256", - "checksumValue": "a1426b54fa7273ba5b50817c25b2b26fc85c4d1befb14092cd27dc4c99439463" + "checksumValue": "231d9bc13190be4b6821acb518194f32f4a3c04f1c034b3118f6db0bab2debe3" } ], - "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA2.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA3.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "cfb7b520c39a73cb84c541d370455f92b998781f" + "checksumValue": "ab7b4d9465a2765a07f8d5bccace7182b28ed1b8" }, { "algorithm": "SHA256", - "checksumValue": "fd41997f9e96b3c9a3337b1b51fab965a1e21b0c16f353d156f1a1fa00709fbf" + "checksumValue": "26913613f3b4f8ffff0a3e211a5ebc849159094e5e11de0a31fcb95b6105b74c" } ], - "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA3.h" + "fileName": "Modules/_hacl/Hacl_Streaming_Types.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Lib-Memzero0.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f5c7b3ed911af6c8d582e8b3714b0c36195dc994" + "checksumValue": "47ce34375d43a27312e1fffb96b8965610b05855" }, { "algorithm": "SHA256", - "checksumValue": "07de72398b12957e014e97b9ac197bceef12d6d6505c2bfe8b23ee17b94ec5fa" + "checksumValue": "8affd767d7644150064d8bccd05d7bf4c4ae41fd4bb5bf5b8e943eabf09f3d74" } ], - "fileName": "Modules/_hacl/python_hacl_namespaces.h" + "fileName": "Modules/_hacl/Lib_Memzero0.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "7665829b9396f72e7f8098080d6d6773565468e9" + }, + { + "algorithm": "SHA256", + "checksumValue": "ca7357ee70365c690664a44f6522e526636151d9ed2da8d0d29da15bb8556530" + } + ], + "fileName": "Modules/_hacl/include/krml/FStar_UInt128_Verified.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "a2db924d0e8f7df3139e9a20355ffa520aded479" + }, + { + "algorithm": "SHA256", + "checksumValue": "f1de79fb4c763b215c823f44471bbae6b65e6bb533eb52a5863d551d5e2e6748" + } + ], + "fileName": "Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "7f23693151d5409623cbe886e5b45a0e4f0d3c72" + }, + { + "algorithm": "SHA256", + "checksumValue": "1c9bee7ac4b987c73cc3aba6b7ceed8ec7e75c9a741810e4411f35602490e0d8" + } + ], + "fileName": "Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "9881567f43deb32bae77a84b2d349858a24b6685" + }, + { + "algorithm": "SHA256", + "checksumValue": "3382156e32fcb376009177d3d2dc9712ff7c8c02afb97b3e16d98b41a2114f84" + } + ], + "fileName": "Modules/_hacl/include/krml/internal/target.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2-config.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ff5e3ae2360adf7279a9c54d12a1d32e16a1f223" + "checksumValue": "e18efc9239a5df0f222b5f7b0a65f72509d7e304" }, { "algorithm": "SHA256", - "checksumValue": "1eb919e885244e43cdf7b2104ad30dc9271513478c0026f6bfb4bad6e2f0ab42" + "checksumValue": "47dd5a7d21b5302255f9fff28884f65d3056fc3f54471ed62ec85fa1904f8aa5" } ], - "fileName": "Modules/_blake2/impl/blake2-config.h" + "fileName": "Modules/_hacl/include/krml/lowstar_endianness.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2-impl.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "28b947b43bdc680b9f4335712bb2a5f2d5d32623" + "checksumValue": "df8e0ed74a5970d09d3cc4c6e7c6c7a4c4e5015c" }, { "algorithm": "SHA256", - "checksumValue": "4277092643b289f1d36d32cf0fd2efc30ead8bdd99342e5da3b3609dd8ea7d86" + "checksumValue": "de7444c345caa4c47902c4380500356a3ee7e199d2aab84fd8c4960410154f3d" } ], - "fileName": "Modules/_blake2/impl/blake2-impl.h" + "fileName": "Modules/_hacl/include/krml/types.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2b.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "caa3da7953109d0d2961e3b686d2d285c484b901" + "checksumValue": "31b329bd39ff72ed25086e2afe7875949003c140" }, { "algorithm": "SHA256", - "checksumValue": "2f6c9d0ecf70be474f2853b52394993625a32960e0a64eae147ef97a3a5c1460" + "checksumValue": "16df6cf240ee99aade0fd11d5cc7573c201c7589d8325a5c95c7670c531e1518" } ], - "fileName": "Modules/_blake2/impl/blake2.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2b.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2b-Simd256.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "029a98f87a178936d9e5211c7798b3e0fc622f94" + "checksumValue": "3f4fdfdaef97a2cbac5ec091c91ede18d4b33f92" }, { "algorithm": "SHA256", - "checksumValue": "b392a6e7b43813a05609e994db5fc3552c5912bd482efc781daa0778eb56ab4e" + "checksumValue": "96b1c77860f12bcadad0caca77a5a1649a840ad9989d97984a3b51bb98c80e2f" } ], - "fileName": "Modules/_blake2/impl/blake2b-load-sse2.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse41.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2s.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "fb466dd72344170d09e311e5ea12de99ce071357" + "checksumValue": "9efd61f6ba8d126e98abd83679a5ed5954278c31" }, { "algorithm": "SHA256", - "checksumValue": "cc3072c92164142bf2f9dda4e6c08db61be68ec15a95442415e861090d08f6a2" + "checksumValue": "143f58f033786173501a72ac302e435963fdce6c2cc38eef6d6adeb3cdc1bb9c" } ], - "fileName": "Modules/_blake2/impl/blake2b-load-sse41.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2s.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-ref.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2s-Simd128.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4c0d79128cf891a95b1f668031d55c0c6d2e0270" + "checksumValue": "3f984829465285283b03b1111b4918cfb48b8031" }, { "algorithm": "SHA256", - "checksumValue": "07b257d44e9cc2d95d4911629c92138feafd16d63fef0a5fa7b38914dfd82349" + "checksumValue": "cd24038fdd617edc65e472496b0d58f23ff312f81f9244c3e7893fdc9a1b2977" } ], - "fileName": "Modules/_blake2/impl/blake2b-ref.c" + "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-round.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4c7418e2026417c9c6736fcd305a31f23e05a661" + "checksumValue": "60f02d21f045c8a4c2b6b84a8f7e023d9490c8e5" }, { "algorithm": "SHA256", - "checksumValue": "fa34a60c2d198a0585033f43fd4003f4ba279c9ebcabdf5d6650def0e6d1e914" + "checksumValue": "370d8ef9c48cb55472ece11e12eaf94c58118de3f5515b6df1c130b696597828" } ], - "fileName": "Modules/_blake2/impl/blake2b-round.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_MD5.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "6fa074693aa7305018dfa8db48010a8ef1050ad4" + "checksumValue": "6346c30a140e7d3010c98fe19d14fa229a54eb16" }, { "algorithm": "SHA256", - "checksumValue": "c8c6dd861ac193d4a0e836242ff44900f83423f86d2c2940c8c4c1e41fbd5812" + "checksumValue": "ab52c6092bdbbfc9884f841bf4824016792ffa96167577cbe0df00dd96f56a34" } ], - "fileName": "Modules/_blake2/impl/blake2b.c" + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA1.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ad3f79b6cbe3fd812722114a0d5d08064e69e4d0" + "checksumValue": "0018e084339058dd454b4e49d10d236b4f896bf8" }, { "algorithm": "SHA256", - "checksumValue": "57f1ac6c09f4a50d95811529062220eab4f29cec3805bc6081dec00426c6df62" + "checksumValue": "10e959a92b3288a6165a404c8fae2bbcd7fb00a9abbae2b7809fa55d6fe9068d" } ], - "fileName": "Modules/_blake2/impl/blake2s-load-sse2.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA2.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse41.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "51c32d79f419f3d2eb9875cd9a7f5c0d7892f8a8" + "checksumValue": "39ba6e8959e44ae956a640d3a1fb3ef60de8a9e5" }, { "algorithm": "SHA256", - "checksumValue": "ecc9e09adcbe098629eafd305596bed8d7004be1d83f326995def42bbde93b23" + "checksumValue": "dbf4b86a04b4d8716976f8c023cccbfe174435dbec3bc00fc1f066fb52c4e341" } ], - "fileName": "Modules/_blake2/impl/blake2s-load-sse41.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA3.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-xop.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Impl-Blake2-Constants.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2749a7ba0104b765d4f56f13faf70b6eb89cf203" + "checksumValue": "c3ae35ed5bf70cf011b2732df011231528b9111c" }, { "algorithm": "SHA256", - "checksumValue": "8bc95595cec4c50f5d70f2b330d3798de07cc784e8890791b3328890e602d5c5" + "checksumValue": "c381fea7b8b505a7c7ce27231a36751add6b184b204132935c5faaba4fce8ba1" } ], - "fileName": "Modules/_blake2/impl/blake2s-load-xop.h" + "fileName": "Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-ref.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-lib-memzero0.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "883fcfe85f9063819f21b1100296d1f9eb55bac1" + "checksumValue": "3d65f95f6f4bbfe980a89b82c55d02d7694a5a79" }, { "algorithm": "SHA256", - "checksumValue": "9715c00d0f11587a139b07fa26678e6d26e44d3d4910b96158d158da2b022bfb" + "checksumValue": "0f8d744620cf5f6b8450da187484b418d24dec7d8cf72b757b7080e84cb3ae5e" } ], - "fileName": "Modules/_blake2/impl/blake2s-ref.c" + "fileName": "Modules/_hacl/lib_memzero0.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-round.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-libintvector.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5d9f69adda40ed163b287b9ed4cedb35b88f2daa" + "checksumValue": "f4a33ad535768b860362ab0bd033a70da0b524b7" }, { "algorithm": "SHA256", - "checksumValue": "65d90111c89c43bb18a9e1d1a4fdbd9f85bebd1ff00129335b85995d0f30ee8b" + "checksumValue": "433cdf4ba80bc72e0cea5d4b420ff18676baeafdb5ba19adf5b7fb33e90b424b" } ], - "fileName": "Modules/_blake2/impl/blake2s-round.h" + "fileName": "Modules/_hacl/libintvector.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "d2691353fa54ac6ffcd7c0a294984dc9d7968ef7" + "checksumValue": "37e3eb63c5c6f8ae671748bfde642c180b96d2de" }, { "algorithm": "SHA256", - "checksumValue": "cfd7948c9fd50e9f9c62f8a93b20a254d1d510a862d1092af4f187b7c1a859a3" + "checksumValue": "0b5c7892cc25a2b3467936c1f346a6186d9d0a257d1bd5671beda253b66e0f68" } ], - "fileName": "Modules/_blake2/impl/blake2s.c" + "fileName": "Modules/_hacl/python_hacl_namespaces.h" }, { "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-init-.py", @@ -1238,11 +1280,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "12402bcf7f0161adb83f78163f41cc10a5e5de5f" + "checksumValue": "9dcb50e3f9c3245972731be5da0b28e7583198d9" }, { "algorithm": "SHA256", - "checksumValue": "cba044c76b6bc3ae6cfa49df1121cad7552140157b9e61e11cbb6580cc5d74cf" + "checksumValue": "7cac49fef5e9d952ec9390bf81c54d83f1b5da32fdf76091c2f0770ed943b7fe" } ], "fileName": "Modules/_decimal/libmpdec/io.c" @@ -1568,20 +1610,6 @@ } ], "fileName": "Modules/_decimal/libmpdec/vcdiv64.asm" - }, - { - "SPDXID": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-24.0-py3-none-any.whl", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "e44313ae1e6af3c2bd3b60ab2fa8c34308d00555" - }, - { - "algorithm": "SHA256", - "checksumValue": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc" - } - ], - "fileName": "Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl" } ], "packages": [ @@ -1590,14 +1618,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "6b902ab103843592be5e99504f846ec109c1abb692e85347587f237a4ffa1033" + "checksumValue": "17aa6cfc5c4c219c09287abfc10bc13f0c06f30bb654b28bfe6f567ca646eb79" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_6_3/expat-2.6.3.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.5.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.6.3:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1605,21 +1633,21 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.5.0" + "versionInfo": "2.6.3" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "c23ac158b238c368389dc86bfc315263e5c0e57785da74144aea2cab9a3d51a2" + "checksumValue": "935ae51d0ff0bf1403f0ecc1ff02b8f685d09053618558c07fbe4bd2abbc5dd1" } ], - "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/521af282fdf6d60227335120f18ae9309a4b8e8c.zip", + "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/315a9e491d2bc347b9dae99e0ea506995ea84d9d.zip", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:521af282fdf6d60227335120f18ae9309a4b8e8c:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:315a9e491d2bc347b9dae99e0ea506995ea84d9d:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1627,29 +1655,7 @@ "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", - "versionInfo": "521af282fdf6d60227335120f18ae9309a4b8e8c" - }, - { - "SPDXID": "SPDXRef-PACKAGE-libb2", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "53626fddce753c454a3fea581cbbc7fe9bbcf0bc70416d48fdbbf5d87ef6c72e" - } - ], - "downloadLocation": "https://github.com/BLAKE2/libb2/releases/download/v0.98.1/libb2-0.98.1.tar.gz", - "externalRefs": [ - { - "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:blake2:libb2:0.98.1:*:*:*:*:*:*:*", - "referenceType": "cpe23Type" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "libb2", - "originator": "Organization: BLAKE2 - fast secure hashing", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.98.1" + "versionInfo": "315a9e491d2bc347b9dae99e0ea506995ea84d9d" }, { "SPDXID": "SPDXRef-PACKAGE-macholib", @@ -1694,769 +1700,163 @@ "originator": "Organization: bytereef.org", "primaryPackagePurpose": "SOURCE", "versionInfo": "2.5.1" + } + ], + "relationships": [ + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "SPDXID": "SPDXRef-PACKAGE-cachecontrol", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "95dedbec849f46dda3137866dc28b9d133fc9af55f5b805ab1291833e4457aa4" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/1d/e3/a22348e6226dcd585d5a4b5f0175b3a16dabfd3912cbeb02f321d00e56c7/cachecontrol-0.13.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/cachecontrol@0.13.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "cachecontrol", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.13.1" + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-ascii.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "SPDXID": "SPDXRef-PACKAGE-colorama", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/colorama@0.4.6", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "colorama", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.4.6" + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-asciitab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "SPDXID": "SPDXRef-PACKAGE-distlib", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/8e/41/9307e4f5f9976bc8b7fea0b66367734e8faf3ec84bc0d412d8cfabbb66cd/distlib-0.3.8-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/distlib@0.3.8", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "distlib", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.3.8" + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "SPDXID": "SPDXRef-PACKAGE-distro", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "99522ca3e365cac527b44bde033f64c6945d90eb9f769703caaec52b09bbd3ff" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/f4/2c/c90a3adaf0ddb70afe193f5ebfb539612af57cffe677c3126be533df3098/distro-1.8.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/distro@1.8.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "distro", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.8.0" + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat-external.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "SPDXID": "SPDXRef-PACKAGE-msgpack", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/9f/4a/36d936e54cf71e23ad276564465f6a54fb129e3d61520b76e13e0bb29167/msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/msgpack@1.0.5", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "msgpack", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.0.5" + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-iasciitab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "SPDXID": "SPDXRef-PACKAGE-packaging", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/05/8e/8de486cbd03baba4deef4142bd643a3e7bbe954a784dc1bb17142572d127/packaging-21.3-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/packaging@21.3", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "packaging", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "21.3" - }, - { - "SPDXID": "SPDXRef-PACKAGE-platformdirs", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/9e/d8/563a9fc17153c588c8c2042d2f0f84a89057cdb1c30270f589c88b42d62c/platformdirs-3.8.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/platformdirs@3.8.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "platformdirs", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.8.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-pyparsing", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/a4/24/6ae4c9c45cf99d96b06b5d99e25526c060303171fb0aea9da2bfd7dbde93/pyparsing-3.1.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pyparsing@3.1.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "pyparsing", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.1.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-pyproject-hooks", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pyproject-hooks@1.0.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "pyproject-hooks", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.0.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-requests", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/requests@2.31.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "requests", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.31.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-certifi", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/certifi@2023.7.22", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "certifi", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "2023.7.22" - }, - { - "SPDXID": "SPDXRef-PACKAGE-chardet", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/74/8f/8fc49109009e8d2169d94d72e6b1f4cd45c13d147ba7d6170fb41f22b08f/chardet-5.1.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/chardet@5.1.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "chardet", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "5.1.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-idna", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/idna@3.4", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "idna", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.4" - }, - { - "SPDXID": "SPDXRef-PACKAGE-rich", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/fc/1e/482e5eec0b89b593e81d78f819a9412849814e22225842b598908e7ac560/rich-13.4.2-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/rich@13.4.2", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "rich", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "13.4.2" - }, - { - "SPDXID": "SPDXRef-PACKAGE-pygments", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/34/a7/37c8d68532ba71549db4212cb036dbd6161b40e463aba336770e80c72f84/Pygments-2.15.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pygments@2.15.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "pygments", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.15.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-typing-extensions", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/ec/6b/63cc3df74987c36fe26157ee12e09e8f9db4de771e0f3404263117e75b95/typing_extensions-4.7.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/typing_extensions@4.7.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "typing_extensions", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "4.7.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-resolvelib", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "d2da45d1a8dfee81bdd591647783e340ef3bcb104b54c383f70d422ef5cc7dbf" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/d2/fc/e9ccf0521607bcd244aa0b3fbd574f71b65e9ce6a112c83af988bbbe2e23/resolvelib-1.0.1-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/resolvelib@1.0.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "resolvelib", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.0.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-setuptools", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/c7/42/be1c7bbdd83e1bfb160c94b9cafd8e25efc7400346cf7ccdbdb452c467fa/setuptools-68.0.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/setuptools@68.0.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "setuptools", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "68.0.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-six", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/six@1.16.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "six", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.16.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-tenacity", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/e7/b0/c23bd61e1b32c9b96fbca996c87784e196a812da8d621d8d04851f6c8181/tenacity-8.2.2-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/tenacity@8.2.2", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "tenacity", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.2.2" - }, - { - "SPDXID": "SPDXRef-PACKAGE-tomli", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/tomli@2.0.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "tomli", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.0.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-truststore", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "e37a5642ae9fc48caa8f120b6283d77225d600d224965a672c9e8ef49ce4bb4c" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/20/56/7811d5439b6a56374f274a8672d8f18b4deadadeb3a9f0c86424b98b6f96/truststore-0.8.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/truststore@0.8.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "truststore", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.8.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-webencodings", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/webencodings@0.5.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "webencodings", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.5.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-urllib3", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/48/fe/a5c6cc46e9fe9171d7ecf0f33ee7aae14642f8d74baa7af4d7840f9358be/urllib3-1.26.17-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/urllib3@1.26.17", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "urllib3", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.26.17" - }, - { - "SPDXID": "SPDXRef-PACKAGE-pip", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:pypa:pip:24.0:*:*:*:*:*:*:*", - "referenceType": "cpe23Type" - }, - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pip@24.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "pip", - "originator": "Organization: Python Packaging Authority", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "24.0" - } - ], - "relationships": [ - { - "relatedSpdxElement": "SPDXRef-PACKAGE-cachecontrol", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-certifi", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-chardet", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-colorama", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-distlib", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-distro", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-idna", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-msgpack", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-packaging", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-platformdirs", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-pygments", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-pyparsing", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-pyproject-hooks", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-requests", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-resolvelib", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-rich", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-setuptools", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-six", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-tenacity", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-tomli", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-truststore", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-typing-extensions", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-internal.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-PACKAGE-urllib3", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-latin1tab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-PACKAGE-webencodings", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-nametab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-pyexpatns.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-ascii.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-siphash.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-asciitab.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-utf8tab.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-winconfig.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat-config.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlparse.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat-external.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlrole.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-iasciitab.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlrole.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-internal.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-latin1tab.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-nametab.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-impl.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-pyexpatns.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-impl.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-siphash.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-ns.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-utf8tab.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b.c", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-winconfig.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlparse.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256.c", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlrole.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlrole.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256-universal2.c", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s.c", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-impl.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128.c", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-impl.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-ns.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128-universal2.c", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", @@ -2503,6 +1903,11 @@ "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Lib-Memzero0.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, { "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", "relationshipType": "CONTAINS", @@ -2534,99 +1939,64 @@ "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2b.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2b-Simd256.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2s.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2s-Simd128.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2-config.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2-impl.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse2.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse41.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-ref.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-round.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse2.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse41.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-xop.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Impl-Blake2-Constants.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-ref.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-lib-memzero0.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-round.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-libintvector.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-init-.py", @@ -2907,11 +2277,6 @@ "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-vcdiv64.asm", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-24.0-py3-none-any.whl", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-pip" } ], "spdxVersion": "SPDX-2.3" diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index a9875f6ffd1a56..f9e51f0683c965 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -40,7 +40,7 @@ # - struct_abi_kind: for `struct`, defines how much of the struct is exposed: # - 'full-abi': All of the struct is part of the ABI, including the size # (users may define arrays of these structs). -# Typically used for initalization, rather than at runtime. +# Typically used for initialization, rather than at runtime. # - 'opaque': No members are part of the ABI, nor is the size. The Limited # API only handles these via pointers. The C definition should be # incomplete (opaque). @@ -702,7 +702,6 @@ added = '3.2' [function.PyEval_InitThreads] added = '3.2' - abi_only = true [function.PyEval_ReleaseLock] added = '3.2' abi_only = true @@ -1337,10 +1336,8 @@ added = '3.2' [function.PySys_SetArgv] added = '3.2' - abi_only = true [function.PySys_SetArgvEx] added = '3.2' - abi_only = true [function.PySys_SetObject] added = '3.2' [function.PySys_SetPath] @@ -1673,10 +1670,8 @@ added = '3.2' [function.Py_SetProgramName] added = '3.2' - abi_only = true [function.Py_SetPythonHome] added = '3.2' - abi_only = true [function.Py_SetRecursionLimit] added = '3.2' [function.Py_VaBuildValue] @@ -2485,7 +2480,63 @@ [function._Py_SetRefcnt] added = '3.13' abi_only = true -[data.PyExc_IncompleteInputError] - added = '3.13' [function.PyList_GetItemRef] added = '3.13' +[typedef.PyCFunctionFast] + added = '3.13' + # "abi-only" since 3.10. (Callback type names aren't used in C code, + # but this function signature was expected with METH_FASTCALL.) +[typedef.PyCFunctionFastWithKeywords] + added = '3.13' + # "abi-only" since 3.10. (Same story as PyCFunctionFast.) +[function.PyType_GetFullyQualifiedName] + added = '3.13' +[function.PyType_GetModuleName] + added = '3.13' +[function.Py_GetConstant] + added = '3.13' +[function.Py_GetConstantBorrowed] + added = '3.13' +[function.PyType_GetModuleByDef] + added = '3.13' +[function.PyEval_GetFrameBuiltins] + added = '3.13' +[function.PyEval_GetFrameGlobals] + added = '3.13' +[function.PyEval_GetFrameLocals] + added = '3.13' + +[function.Py_TYPE] + added = '3.14' +[function.Py_REFCNT] + added = '3.14' +[function.PyIter_NextItem] + added = '3.14' +[function.PyLong_FromInt32] + added = '3.14' +[function.PyLong_FromUInt32] + added = '3.14' +[function.PyLong_AsInt32] + added = '3.14' +[function.PyLong_AsUInt32] + added = '3.14' +[function.PyLong_FromInt64] + added = '3.14' +[function.PyLong_FromUInt64] + added = '3.14' +[function.PyLong_AsInt64] + added = '3.14' +[function.PyLong_AsUInt64] + added = '3.14' +[const.Py_tp_vectorcall] + added = '3.14' +[function.PyType_GetBaseByToken] + added = '3.14' +[const.Py_tp_token] + added = '3.14' +[const.Py_TP_USE_SPEC] + added = '3.14' +[function.PyUnicode_Equal] + added = '3.14' +[function.PyType_Freeze] + added = '3.14' diff --git a/Misc/valgrind-python.supp b/Misc/valgrind-python.supp index 29954c1bbad350..8b2027cd452767 100644 --- a/Misc/valgrind-python.supp +++ b/Misc/valgrind-python.supp @@ -95,6 +95,49 @@ fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING } +# +# Leaks: dlopen() called without dlclose() +# + +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:malloc + fun:malloc + fun:strdup + fun:_dl_load_cache_lookup +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:malloc + fun:malloc + fun:strdup + fun:_dl_map_object +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:malloc + fun:* + fun:_dl_new_object +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:calloc + fun:* + fun:_dl_new_object +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:calloc + fun:* + fun:_dl_check_map_versions +} + + # # Non-python specific leaks # diff --git a/Modules/Setup b/Modules/Setup index 8ad9a5aebbfcaa..ddf39e0b966610 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -137,6 +137,9 @@ PYTHONPATH=$(COREPYTHONPATH) #_datetime _datetimemodule.c #_decimal _decimal/_decimal.c #_heapq _heapqmodule.c +#_interpchannels _interpchannelsmodule.c +#_interpqueues _interpqueuesmodule.c +#_interpreters _interpretersmodule.c #_json _json.c #_lsprof _lsprof.c rotatingtree.c #_multiprocessing -I$(srcdir)/Modules/_multiprocessing _multiprocessing/multiprocessing.c _multiprocessing/semaphore.c @@ -162,7 +165,7 @@ PYTHONPATH=$(COREPYTHONPATH) #pyexpat pyexpat.c # hashing builtins -#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c +#_blake2 blake2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_Blake2.a #_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_SHA2.a @@ -271,9 +274,6 @@ PYTHONPATH=$(COREPYTHONPATH) # Testing -#_xxsubinterpreters _xxsubinterpretersmodule.c -#_xxinterpchannels _xxinterpchannelsmodule.c -#_xxinterpqueues _xxinterpqueuesmodule.c #_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c #_testbuffer _testbuffer.c #_testinternalcapi _testinternalcapi.c @@ -285,6 +285,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c +#_testexternalinspection _testexternalinspection.c #_testsinglephase _testsinglephase.c # --- diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index aa4e60e272653b..4dcc0f55176d0e 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -30,6 +30,7 @@ _weakref _weakref.c _abc _abc.c _functools _functoolsmodule.c _locale _localemodule.c +_opcode _opcode.c _operator _operator.c _stat _stat.c _symtable symtablemodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 8a65a9cffb1b9d..52c0f883d383db 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -36,16 +36,16 @@ @MODULE__HEAPQ_TRUE@_heapq _heapqmodule.c @MODULE__JSON_TRUE@_json _json.c @MODULE__LSPROF_TRUE@_lsprof _lsprof.c rotatingtree.c -@MODULE__OPCODE_TRUE@_opcode _opcode.c @MODULE__PICKLE_TRUE@_pickle _pickle.c @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c @MODULE__STRUCT_TRUE@_struct _struct.c # build supports subinterpreters -@MODULE__XXSUBINTERPRETERS_TRUE@_xxsubinterpreters _xxsubinterpretersmodule.c -@MODULE__XXINTERPCHANNELS_TRUE@_xxinterpchannels _xxinterpchannelsmodule.c -@MODULE__XXINTERPQUEUES_TRUE@_xxinterpqueues _xxinterpqueuesmodule.c +@MODULE__INTERPRETERS_TRUE@_interpreters _interpretersmodule.c +@MODULE__INTERPCHANNELS_TRUE@_interpchannels _interpchannelsmodule.c +@MODULE__INTERPQUEUES_TRUE@_interpqueues _interpqueuesmodule.c + @MODULE__ZONEINFO_TRUE@_zoneinfo _zoneinfo.c # needs libm @@ -82,7 +82,7 @@ @MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_SHA2.a @MODULE__SHA3_TRUE@_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE -@MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c +@MODULE__BLAKE2_TRUE@_blake2 blake2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_Blake2.a ############################################################################ # XML and text @@ -162,7 +162,8 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c _testcapi/hash.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c @@ -170,7 +171,8 @@ *shared* @MODULE__TESTIMPORTMULTIPLE_TRUE@_testimportmultiple _testimportmultiple.c @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c -@MODULE__TESTMULTIPHASE_TRUE@_testsinglephase _testsinglephase.c +@MODULE__TESTSINGLEPHASE_TRUE@_testsinglephase _testsinglephase.c +@MODULE__TESTEXTERNALINSPECTION_TRUE@_testexternalinspection _testexternalinspection.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c # Limited API template modules; must be built as shared modules. diff --git a/Modules/_abc.c b/Modules/_abc.c index 9473905243d438..4f4b24b035db4a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -8,7 +8,6 @@ #include "pycore_object.h" // _PyType_GetSubclasses() #include "pycore_runtime.h" // _Py_ID() #include "pycore_setobject.h" // _PySet_NextEntry() -#include "pycore_typeobject.h" // _PyType_GetMRO() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "clinic/_abc.c.h" @@ -22,7 +21,7 @@ PyDoc_STRVAR(_abc__doc__, typedef struct { PyTypeObject *_abc_data_type; - unsigned long long abc_invalidation_counter; + uint64_t abc_invalidation_counter; } _abcmodule_state; static inline _abcmodule_state* @@ -33,17 +32,61 @@ get_abc_state(PyObject *module) return (_abcmodule_state *)state; } +static inline uint64_t +get_invalidation_counter(_abcmodule_state *state) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_uint64(&state->abc_invalidation_counter); +#else + return state->abc_invalidation_counter; +#endif +} + +static inline void +increment_invalidation_counter(_abcmodule_state *state) +{ +#ifdef Py_GIL_DISABLED + _Py_atomic_add_uint64(&state->abc_invalidation_counter, 1); +#else + state->abc_invalidation_counter++; +#endif +} + /* This object stores internal state for ABCs. Note that we can use normal sets for caches, since they are never iterated over. */ typedef struct { PyObject_HEAD + /* These sets of weak references are lazily created. Once created, they + will point to the same sets until the ABCMeta object is destroyed or + cleared, both of which will only happen while the object is visible to a + single thread. */ PyObject *_abc_registry; - PyObject *_abc_cache; /* Normal set of weak references. */ - PyObject *_abc_negative_cache; /* Normal set of weak references. */ - unsigned long long _abc_negative_cache_version; + PyObject *_abc_cache; + PyObject *_abc_negative_cache; + uint64_t _abc_negative_cache_version; } _abc_data; +static inline uint64_t +get_cache_version(_abc_data *impl) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_uint64(&impl->_abc_negative_cache_version); +#else + return impl->_abc_negative_cache_version; +#endif +} + +static inline void +set_cache_version(_abc_data *impl, uint64_t version) +{ +#ifdef Py_GIL_DISABLED + _Py_atomic_store_uint64(&impl->_abc_negative_cache_version, version); +#else + impl->_abc_negative_cache_version = version; +#endif +} + static int abc_data_traverse(_abc_data *self, visitproc visit, void *arg) { @@ -91,7 +134,7 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->_abc_registry = NULL; self->_abc_cache = NULL; self->_abc_negative_cache = NULL; - self->_abc_negative_cache_version = state->abc_invalidation_counter; + self->_abc_negative_cache_version = get_invalidation_counter(state); return (PyObject *) self; } @@ -131,8 +174,12 @@ _get_impl(PyObject *module, PyObject *self) } static int -_in_weak_set(PyObject *set, PyObject *obj) +_in_weak_set(_abc_data *impl, PyObject **pset, PyObject *obj) { + PyObject *set; + Py_BEGIN_CRITICAL_SECTION(impl); + set = *pset; + Py_END_CRITICAL_SECTION(); if (set == NULL || PySet_GET_SIZE(set) == 0) { return 0; } @@ -169,16 +216,19 @@ static PyMethodDef _destroy_def = { }; static int -_add_to_weak_set(PyObject **pset, PyObject *obj) +_add_to_weak_set(_abc_data *impl, PyObject **pset, PyObject *obj) { - if (*pset == NULL) { - *pset = PySet_New(NULL); - if (*pset == NULL) { - return -1; - } + PyObject *set; + Py_BEGIN_CRITICAL_SECTION(impl); + set = *pset; + if (set == NULL) { + set = *pset = PySet_New(NULL); + } + Py_END_CRITICAL_SECTION(); + if (set == NULL) { + return -1; } - PyObject *set = *pset; PyObject *ref, *wr; PyObject *destroy_cb; wr = PyWeakref_NewRef(set, NULL); @@ -221,7 +271,11 @@ _abc__reset_registry(PyObject *module, PyObject *self) if (impl == NULL) { return NULL; } - if (impl->_abc_registry != NULL && PySet_Clear(impl->_abc_registry) < 0) { + PyObject *registry; + Py_BEGIN_CRITICAL_SECTION(impl); + registry = impl->_abc_registry; + Py_END_CRITICAL_SECTION(); + if (registry != NULL && PySet_Clear(registry) < 0) { Py_DECREF(impl); return NULL; } @@ -248,13 +302,17 @@ _abc__reset_caches(PyObject *module, PyObject *self) if (impl == NULL) { return NULL; } - if (impl->_abc_cache != NULL && PySet_Clear(impl->_abc_cache) < 0) { + PyObject *cache, *negative_cache; + Py_BEGIN_CRITICAL_SECTION(impl); + cache = impl->_abc_cache; + negative_cache = impl->_abc_negative_cache; + Py_END_CRITICAL_SECTION(); + if (cache != NULL && PySet_Clear(cache) < 0) { Py_DECREF(impl); return NULL; } /* also the second cache */ - if (impl->_abc_negative_cache != NULL && - PySet_Clear(impl->_abc_negative_cache) < 0) { + if (negative_cache != NULL && PySet_Clear(negative_cache) < 0) { Py_DECREF(impl); return NULL; } @@ -283,11 +341,14 @@ _abc__get_dump(PyObject *module, PyObject *self) if (impl == NULL) { return NULL; } - PyObject *res = Py_BuildValue("NNNK", - PySet_New(impl->_abc_registry), - PySet_New(impl->_abc_cache), - PySet_New(impl->_abc_negative_cache), - impl->_abc_negative_cache_version); + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(impl); + res = Py_BuildValue("NNNK", + PySet_New(impl->_abc_registry), + PySet_New(impl->_abc_cache), + PySet_New(impl->_abc_negative_cache), + get_cache_version(impl)); + Py_END_CRITICAL_SECTION(); Py_DECREF(impl); return res; } @@ -454,56 +515,27 @@ _abc__abc_init(PyObject *module, PyObject *self) if (PyType_Check(self)) { PyTypeObject *cls = (PyTypeObject *)self; PyObject *dict = _PyType_GetDict(cls); - PyObject *flags = PyDict_GetItemWithError(dict, - &_Py_ID(__abc_tpflags__)); - if (flags == NULL) { - if (PyErr_Occurred()) { - return NULL; - } + PyObject *flags = NULL; + if (PyDict_Pop(dict, &_Py_ID(__abc_tpflags__), &flags) < 0) { + return NULL; } - else { - if (PyLong_CheckExact(flags)) { - long val = PyLong_AsLong(flags); - if (val == -1 && PyErr_Occurred()) { - return NULL; - } - if ((val & COLLECTION_FLAGS) == COLLECTION_FLAGS) { - PyErr_SetString(PyExc_TypeError, "__abc_tpflags__ cannot be both Py_TPFLAGS_SEQUENCE and Py_TPFLAGS_MAPPING"); - return NULL; - } - ((PyTypeObject *)self)->tp_flags |= (val & COLLECTION_FLAGS); - } - if (PyDict_DelItem(dict, &_Py_ID(__abc_tpflags__)) < 0) { - return NULL; - } + if (flags == NULL || !PyLong_CheckExact(flags)) { + Py_XDECREF(flags); + Py_RETURN_NONE; } - } - Py_RETURN_NONE; -} - -static void -set_collection_flag_recursive(PyTypeObject *child, unsigned long flag) -{ - assert(flag == Py_TPFLAGS_MAPPING || flag == Py_TPFLAGS_SEQUENCE); - if (PyType_HasFeature(child, Py_TPFLAGS_IMMUTABLETYPE) || - (child->tp_flags & COLLECTION_FLAGS) == flag) - { - return; - } - - child->tp_flags &= ~COLLECTION_FLAGS; - child->tp_flags |= flag; - PyObject *grandchildren = _PyType_GetSubclasses(child); - if (grandchildren == NULL) { - return; - } - - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(grandchildren); i++) { - PyObject *grandchild = PyList_GET_ITEM(grandchildren, i); - set_collection_flag_recursive((PyTypeObject *)grandchild, flag); + long val = PyLong_AsLong(flags); + Py_DECREF(flags); + if (val == -1 && PyErr_Occurred()) { + return NULL; + } + if ((val & COLLECTION_FLAGS) == COLLECTION_FLAGS) { + PyErr_SetString(PyExc_TypeError, "__abc_tpflags__ cannot be both Py_TPFLAGS_SEQUENCE and Py_TPFLAGS_MAPPING"); + return NULL; + } + _PyType_SetFlags((PyTypeObject *)self, 0, val & COLLECTION_FLAGS); } - Py_DECREF(grandchildren); + Py_RETURN_NONE; } /*[clinic input] @@ -546,20 +578,23 @@ _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) if (impl == NULL) { return NULL; } - if (_add_to_weak_set(&impl->_abc_registry, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_registry, subclass) < 0) { Py_DECREF(impl); return NULL; } Py_DECREF(impl); /* Invalidate negative cache */ - get_abc_state(module)->abc_invalidation_counter++; + increment_invalidation_counter(get_abc_state(module)); - /* Set Py_TPFLAGS_SEQUENCE or Py_TPFLAGS_MAPPING flag */ + /* Set Py_TPFLAGS_SEQUENCE or Py_TPFLAGS_MAPPING flag */ if (PyType_Check(self)) { - unsigned long collection_flag = ((PyTypeObject *)self)->tp_flags & COLLECTION_FLAGS; + unsigned long collection_flag = + PyType_GetFlags((PyTypeObject *)self) & COLLECTION_FLAGS; if (collection_flag) { - set_collection_flag_recursive((PyTypeObject *)subclass, collection_flag); + _PyType_SetFlagsRecursive((PyTypeObject *)subclass, + COLLECTION_FLAGS, + collection_flag); } } return Py_NewRef(subclass); @@ -593,7 +628,7 @@ _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, return NULL; } /* Inline the cache checking. */ - int incache = _in_weak_set(impl->_abc_cache, subclass); + int incache = _in_weak_set(impl, &impl->_abc_cache, subclass); if (incache < 0) { goto end; } @@ -603,8 +638,8 @@ _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, } subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { - if (impl->_abc_negative_cache_version == get_abc_state(module)->abc_invalidation_counter) { - incache = _in_weak_set(impl->_abc_negative_cache, subclass); + if (get_cache_version(impl) == get_invalidation_counter(get_abc_state(module))) { + incache = _in_weak_set(impl, &impl->_abc_negative_cache, subclass); if (incache < 0) { goto end; } @@ -682,7 +717,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } /* 1. Check cache. */ - incache = _in_weak_set(impl->_abc_cache, subclass); + incache = _in_weak_set(impl, &impl->_abc_cache, subclass); if (incache < 0) { goto end; } @@ -693,17 +728,20 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, state = get_abc_state(module); /* 2. Check negative cache; may have to invalidate. */ - if (impl->_abc_negative_cache_version < state->abc_invalidation_counter) { + uint64_t invalidation_counter = get_invalidation_counter(state); + if (get_cache_version(impl) < invalidation_counter) { /* Invalidate the negative cache. */ - if (impl->_abc_negative_cache != NULL && - PySet_Clear(impl->_abc_negative_cache) < 0) - { + PyObject *negative_cache; + Py_BEGIN_CRITICAL_SECTION(impl); + negative_cache = impl->_abc_negative_cache; + Py_END_CRITICAL_SECTION(); + if (negative_cache != NULL && PySet_Clear(negative_cache) < 0) { goto end; } - impl->_abc_negative_cache_version = state->abc_invalidation_counter; + set_cache_version(impl, invalidation_counter); } else { - incache = _in_weak_set(impl->_abc_negative_cache, subclass); + incache = _in_weak_set(impl, &impl->_abc_negative_cache, subclass); if (incache < 0) { goto end; } @@ -721,7 +759,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } if (ok == Py_True) { Py_DECREF(ok); - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -729,7 +767,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } if (ok == Py_False) { Py_DECREF(ok); - if (_add_to_weak_set(&impl->_abc_negative_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -744,18 +782,12 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, Py_DECREF(ok); /* 4. Check if it's a direct subclass. */ - PyObject *mro = _PyType_GetMRO((PyTypeObject *)subclass); - assert(PyTuple_Check(mro)); - for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { - PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - assert(mro_item != NULL); - if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { - goto end; - } - result = Py_True; + if (PyType_IsSubtype((PyTypeObject *)subclass, (PyTypeObject *)self)) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { goto end; } + result = Py_True; + goto end; } /* 5. Check if it's a subclass of a registered class (recursive). */ @@ -774,12 +806,14 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, goto end; } for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { - PyObject *scls = PyList_GET_ITEM(subclasses, pos); - Py_INCREF(scls); + PyObject *scls = PyList_GetItemRef(subclasses, pos); + if (scls == NULL) { + goto end; + } int r = PyObject_IsSubclass(subclass, scls); Py_DECREF(scls); if (r > 0) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -791,7 +825,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } /* No dice; update negative cache. */ - if (_add_to_weak_set(&impl->_abc_negative_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -808,7 +842,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, PyObject **result) { // Fast path: check subclass is in weakref directly. - int ret = _in_weak_set(impl->_abc_registry, subclass); + int ret = _in_weak_set(impl, &impl->_abc_registry, subclass); if (ret < 0) { *result = NULL; return -1; @@ -818,33 +852,27 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, return 1; } - if (impl->_abc_registry == NULL) { + PyObject *registry_shared; + Py_BEGIN_CRITICAL_SECTION(impl); + registry_shared = impl->_abc_registry; + Py_END_CRITICAL_SECTION(); + if (registry_shared == NULL) { return 0; } - Py_ssize_t registry_size = PySet_Size(impl->_abc_registry); - if (registry_size == 0) { - return 0; - } - // Weakref callback may remove entry from set. - // So we take snapshot of registry first. - PyObject **copy = PyMem_Malloc(sizeof(PyObject*) * registry_size); - if (copy == NULL) { - PyErr_NoMemory(); + + // Make a local copy of the registry to protect against concurrent + // modifications of _abc_registry. + PyObject *registry = PyFrozenSet_New(registry_shared); + if (registry == NULL) { return -1; } PyObject *key; Py_ssize_t pos = 0; Py_hash_t hash; - Py_ssize_t i = 0; - while (_PySet_NextEntry(impl->_abc_registry, &pos, &key, &hash)) { - copy[i++] = Py_NewRef(key); - } - assert(i == registry_size); - - for (i = 0; i < registry_size; i++) { + while (_PySet_NextEntry(registry, &pos, &key, &hash)) { PyObject *rkey; - if (PyWeakref_GetRef(copy[i], &rkey) < 0) { + if (PyWeakref_GetRef(key, &rkey) < 0) { // Someone inject non-weakref type in the registry. ret = -1; break; @@ -860,7 +888,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, break; } if (r > 0) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { ret = -1; break; } @@ -870,10 +898,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, } } - for (i = 0; i < registry_size; i++) { - Py_DECREF(copy[i]); - } - PyMem_Free(copy); + Py_DECREF(registry); return ret; } @@ -892,7 +917,7 @@ _abc_get_cache_token_impl(PyObject *module) /*[clinic end generated code: output=c7d87841e033dacc input=70413d1c423ad9f9]*/ { _abcmodule_state *state = get_abc_state(module); - return PyLong_FromUnsignedLongLong(state->abc_invalidation_counter); + return PyLong_FromUnsignedLongLong(get_invalidation_counter(state)); } static struct PyMethodDef _abcmodule_methods[] = { @@ -945,6 +970,7 @@ _abcmodule_free(void *module) static PyModuleDef_Slot _abcmodule_slots[] = { {Py_mod_exec, _abcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index c1aa849ecf1aad..214c18e966c4c1 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3,9 +3,12 @@ #endif #include "Python.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION_MUT() #include "pycore_dict.h" // _PyDict_GetItem_KnownHash() +#include "pycore_freelist.h" // _Py_FREELIST_POP() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _Py_SetImmortalUntracked #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -19,13 +22,76 @@ module _asyncio [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=8fd17862aa989c69]*/ +typedef enum { + STATE_PENDING, + STATE_CANCELLED, + STATE_FINISHED +} fut_state; + +#define FutureObj_HEAD(prefix) \ + PyObject_HEAD \ + PyObject *prefix##_loop; \ + PyObject *prefix##_callback0; \ + PyObject *prefix##_context0; \ + PyObject *prefix##_callbacks; \ + PyObject *prefix##_exception; \ + PyObject *prefix##_exception_tb; \ + PyObject *prefix##_result; \ + PyObject *prefix##_source_tb; \ + PyObject *prefix##_cancel_msg; \ + PyObject *prefix##_cancelled_exc; \ + fut_state prefix##_state; \ + /* These bitfields need to be at the end of the struct + so that these and bitfields from TaskObj are contiguous. + */ \ + unsigned prefix##_log_tb: 1; \ + unsigned prefix##_blocking: 1; + +typedef struct { + FutureObj_HEAD(fut) +} FutureObj; + +typedef struct TaskObj { + FutureObj_HEAD(task) + unsigned task_must_cancel: 1; + unsigned task_log_destroy_pending: 1; + int task_num_cancels_requested; + PyObject *task_fut_waiter; + PyObject *task_coro; + PyObject *task_name; + PyObject *task_context; + struct TaskObj *next; + struct TaskObj *prev; +} TaskObj; + +typedef struct { + PyObject_HEAD + TaskObj *sw_task; + PyObject *sw_arg; +} TaskStepMethWrapper; + -#define FI_FREELIST_MAXLEN 255 +#define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) +#define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) + +#define Future_Check(state, obj) PyObject_TypeCheck(obj, state->FutureType) +#define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) + +#ifdef Py_GIL_DISABLED +# define ASYNCIO_STATE_LOCK(state) Py_BEGIN_CRITICAL_SECTION_MUT(&state->mutex) +# define ASYNCIO_STATE_UNLOCK(state) Py_END_CRITICAL_SECTION() +#else +# define ASYNCIO_STATE_LOCK(state) ((void)state) +# define ASYNCIO_STATE_UNLOCK(state) ((void)state) +#endif typedef struct futureiterobject futureiterobject; /* State of the _asyncio module */ typedef struct { +#ifdef Py_GIL_DISABLED + PyMutex mutex; +#endif PyTypeObject *FutureIterType; PyTypeObject *TaskStepMethWrapper_Type; PyTypeObject *FutureType; @@ -38,8 +104,9 @@ typedef struct { all running event loops. {EventLoop: Task} */ PyObject *current_tasks; - /* WeakSet containing all tasks scheduled to run on event loops. */ - PyObject *scheduled_tasks; + /* WeakSet containing scheduled 3rd party tasks which don't + inherit from native asyncio.Task */ + PyObject *non_asyncio_tasks; /* Set containing all eagerly executing tasks. */ PyObject *eager_tasks; @@ -68,14 +135,53 @@ typedef struct { /* Imports from traceback. */ PyObject *traceback_extract_stack; - PyObject *cached_running_loop; // Borrowed reference - volatile uint64_t cached_running_loop_tsid; - /* Counter for autogenerated Task names */ uint64_t task_name_counter; - futureiterobject *fi_freelist; - Py_ssize_t fi_freelist_len; + /* Linked-list of all tasks which are instances of asyncio.Task or subclasses + of it. Third party tasks implementations which don't inherit from + asyncio.Task are tracked separately using the 'non_asyncio_tasks' WeakSet. + `tail` is used as a sentinel to mark the end of the linked-list. It avoids one + branch in checking for empty list when adding a new task, the list is + initialized with `head` pointing to `tail` to mark an empty list. + + Invariants: + * When the list is empty: + - asyncio_tasks.head == &asyncio_tasks.tail + - asyncio_tasks.head->prev == NULL + - asyncio_tasks.head->next == NULL + + * After adding the first task 'task1': + - asyncio_tasks.head == task1 + - task1->next == &asyncio_tasks.tail + - task1->prev == NULL + - asyncio_tasks.tail.prev == task1 + + * After adding a second task 'task2': + - asyncio_tasks.head == task2 + - task2->next == task1 + - task2->prev == NULL + - task1->prev == task2 + - asyncio_tasks.tail.prev == task1 + + * After removing task 'task1': + - asyncio_tasks.head == task2 + - task2->next == &asyncio_tasks.tail + - task2->prev == NULL + - asyncio_tasks.tail.prev == task2 + + * After removing task 'task2', the list is empty: + - asyncio_tasks.head == &asyncio_tasks.tail + - asyncio_tasks.head->prev == NULL + - asyncio_tasks.tail.prev == NULL + - asyncio_tasks.tail.next == NULL + */ + + struct { + TaskObj tail; + TaskObj *head; + } asyncio_tasks; + } asyncio_state; static inline asyncio_state * @@ -105,59 +211,6 @@ get_asyncio_state_by_def(PyObject *self) return get_asyncio_state(mod); } -typedef enum { - STATE_PENDING, - STATE_CANCELLED, - STATE_FINISHED -} fut_state; - -#define FutureObj_HEAD(prefix) \ - PyObject_HEAD \ - PyObject *prefix##_loop; \ - PyObject *prefix##_callback0; \ - PyObject *prefix##_context0; \ - PyObject *prefix##_callbacks; \ - PyObject *prefix##_exception; \ - PyObject *prefix##_exception_tb; \ - PyObject *prefix##_result; \ - PyObject *prefix##_source_tb; \ - PyObject *prefix##_cancel_msg; \ - PyObject *prefix##_cancelled_exc; \ - fut_state prefix##_state; \ - /* These bitfields need to be at the end of the struct - so that these and bitfields from TaskObj are contiguous. - */ \ - unsigned prefix##_log_tb: 1; \ - unsigned prefix##_blocking: 1; - -typedef struct { - FutureObj_HEAD(fut) -} FutureObj; - -typedef struct { - FutureObj_HEAD(task) - unsigned task_must_cancel: 1; - unsigned task_log_destroy_pending: 1; - int task_num_cancels_requested; - PyObject *task_fut_waiter; - PyObject *task_coro; - PyObject *task_name; - PyObject *task_context; -} TaskObj; - -typedef struct { - PyObject_HEAD - TaskObj *sw_task; - PyObject *sw_arg; -} TaskStepMethWrapper; - - -#define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) -#define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) - -#define Future_Check(state, obj) PyObject_TypeCheck(obj, state->FutureType) -#define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) - #include "clinic/_asynciomodule.c.h" @@ -262,96 +315,15 @@ get_future_loop(asyncio_state *state, PyObject *fut) return PyObject_GetAttr(fut, &_Py_ID(_loop)); } - -static int -get_running_loop(asyncio_state *state, PyObject **loop) -{ - PyObject *rl; - - PyThreadState *ts = _PyThreadState_GET(); - uint64_t ts_id = PyThreadState_GetID(ts); - if (state->cached_running_loop_tsid == ts_id && - state->cached_running_loop != NULL) - { - // Fast path, check the cache. - rl = state->cached_running_loop; - } - else { - PyObject *ts_dict = _PyThreadState_GetDict(ts); // borrowed - if (ts_dict == NULL) { - goto not_found; - } - - rl = PyDict_GetItemWithError( - ts_dict, &_Py_ID(__asyncio_running_event_loop__)); // borrowed - if (rl == NULL) { - if (PyErr_Occurred()) { - goto error; - } - else { - goto not_found; - } - } - - state->cached_running_loop = rl; - state->cached_running_loop_tsid = ts_id; - } - - - if (rl == Py_None) { - goto not_found; - } - - *loop = Py_NewRef(rl); - return 0; - -not_found: - *loop = NULL; - return 0; - -error: - *loop = NULL; - return -1; -} - - -static int -set_running_loop(asyncio_state *state, PyObject *loop) -{ - PyObject *ts_dict = NULL; - - PyThreadState *tstate = _PyThreadState_GET(); - if (tstate != NULL) { - ts_dict = _PyThreadState_GetDict(tstate); // borrowed - } - - if (ts_dict == NULL) { - PyErr_SetString( - PyExc_RuntimeError, "thread-local storage is not available"); - return -1; - } - if (PyDict_SetItem( - ts_dict, &_Py_ID(__asyncio_running_event_loop__), loop) < 0) - { - return -1; - } - - state->cached_running_loop = loop; // borrowed, kept alive by ts_dict - state->cached_running_loop_tsid = PyThreadState_GetID(tstate); - - return 0; -} - - static PyObject * get_event_loop(asyncio_state *state) { PyObject *loop; PyObject *policy; - if (get_running_loop(state, &loop)) { - return NULL; - } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + loop = Py_XNewRef(ts->asyncio_running_loop); + if (loop != NULL) { return loop; } @@ -434,18 +406,22 @@ future_ensure_alive(FutureObj *fut) static int future_schedule_callbacks(asyncio_state *state, FutureObj *fut) { - Py_ssize_t len; - Py_ssize_t i; - if (fut->fut_callback0 != NULL) { /* There's a 1st callback */ - int ret = call_soon(state, - fut->fut_loop, fut->fut_callback0, - (PyObject *)fut, fut->fut_context0); - - Py_CLEAR(fut->fut_callback0); - Py_CLEAR(fut->fut_context0); + // Beware: An evil call_soon could alter fut_callback0 or fut_context0. + // Since we are anyway clearing them after the call, whether call_soon + // succeeds or not, the idea is to transfer ownership so that external + // code is not able to alter them during the call. + PyObject *fut_callback0 = fut->fut_callback0; + fut->fut_callback0 = NULL; + PyObject *fut_context0 = fut->fut_context0; + fut->fut_context0 = NULL; + + int ret = call_soon(state, fut->fut_loop, fut_callback0, + (PyObject *)fut, fut_context0); + Py_CLEAR(fut_callback0); + Py_CLEAR(fut_context0); if (ret) { /* If an error occurs in pure-Python implementation, all callbacks are cleared. */ @@ -462,27 +438,25 @@ future_schedule_callbacks(asyncio_state *state, FutureObj *fut) return 0; } - len = PyList_GET_SIZE(fut->fut_callbacks); - if (len == 0) { - /* The list of callbacks was empty; clear it and return. */ - Py_CLEAR(fut->fut_callbacks); - return 0; - } - - for (i = 0; i < len; i++) { - PyObject *cb_tup = PyList_GET_ITEM(fut->fut_callbacks, i); + // Beware: An evil call_soon could change fut->fut_callbacks. + // The idea is to transfer the ownership of the callbacks list + // so that external code is not able to mutate the list during + // the iteration. + PyObject *callbacks = fut->fut_callbacks; + fut->fut_callbacks = NULL; + Py_ssize_t n = PyList_GET_SIZE(callbacks); + for (Py_ssize_t i = 0; i < n; i++) { + assert(PyList_GET_SIZE(callbacks) == n); + PyObject *cb_tup = PyList_GET_ITEM(callbacks, i); PyObject *cb = PyTuple_GET_ITEM(cb_tup, 0); PyObject *ctx = PyTuple_GET_ITEM(cb_tup, 1); if (call_soon(state, fut->fut_loop, cb, (PyObject *)fut, ctx)) { - /* If an error occurs in pure-Python implementation, - all callbacks are cleared. */ - Py_CLEAR(fut->fut_callbacks); + Py_DECREF(callbacks); return -1; } } - - Py_CLEAR(fut->fut_callbacks); + Py_DECREF(callbacks); return 0; } @@ -1045,7 +1019,12 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, ENSURE_FUTURE_ALIVE(state, self) if (self->fut_callback0 != NULL) { - int cmp = PyObject_RichCompareBool(self->fut_callback0, fn, Py_EQ); + // Beware: An evil PyObject_RichCompareBool could free fut_callback0 + // before a recursive call is made with that same arg. For details, see + // https://github.com/python/cpython/pull/125967#discussion_r1816593340. + PyObject *fut_callback0 = Py_NewRef(self->fut_callback0); + int cmp = PyObject_RichCompareBool(fut_callback0, fn, Py_EQ); + Py_DECREF(fut_callback0); if (cmp == -1) { return NULL; } @@ -1293,52 +1272,49 @@ static PyObject * FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) { asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - Py_ssize_t i; - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_callback0 == NULL) { - if (fut->fut_callbacks == NULL) { - Py_RETURN_NONE; - } - - return Py_NewRef(fut->fut_callbacks); + Py_ssize_t len = 0; + if (fut->fut_callback0 != NULL) { + len++; } - - Py_ssize_t len = 1; if (fut->fut_callbacks != NULL) { len += PyList_GET_SIZE(fut->fut_callbacks); } - - PyObject *new_list = PyList_New(len); - if (new_list == NULL) { - return NULL; + if (len == 0) { + Py_RETURN_NONE; } - PyObject *tup0 = PyTuple_New(2); - if (tup0 == NULL) { - Py_DECREF(new_list); + PyObject *callbacks = PyList_New(len); + if (callbacks == NULL) { return NULL; } - Py_INCREF(fut->fut_callback0); - PyTuple_SET_ITEM(tup0, 0, fut->fut_callback0); - assert(fut->fut_context0 != NULL); - Py_INCREF(fut->fut_context0); - PyTuple_SET_ITEM(tup0, 1, (PyObject *)fut->fut_context0); - - PyList_SET_ITEM(new_list, 0, tup0); + Py_ssize_t i = 0; + if (fut->fut_callback0 != NULL) { + PyObject *tup0 = PyTuple_New(2); + if (tup0 == NULL) { + Py_DECREF(callbacks); + return NULL; + } + PyTuple_SET_ITEM(tup0, 0, Py_NewRef(fut->fut_callback0)); + assert(fut->fut_context0 != NULL); + PyTuple_SET_ITEM(tup0, 1, Py_NewRef(fut->fut_context0)); + PyList_SET_ITEM(callbacks, i, tup0); + i++; + } if (fut->fut_callbacks != NULL) { - for (i = 0; i < PyList_GET_SIZE(fut->fut_callbacks); i++) { - PyObject *cb = PyList_GET_ITEM(fut->fut_callbacks, i); + for (Py_ssize_t j = 0; j < PyList_GET_SIZE(fut->fut_callbacks); j++) { + PyObject *cb = PyList_GET_ITEM(fut->fut_callbacks, j); Py_INCREF(cb); - PyList_SET_ITEM(new_list, i + 1, cb); + PyList_SET_ITEM(callbacks, i, cb); + i++; } } - return new_list; + return callbacks; } static PyObject * @@ -1601,16 +1577,13 @@ static void FutureIter_dealloc(futureiterobject *it) { PyTypeObject *tp = Py_TYPE(it); - asyncio_state *state = get_asyncio_state_by_def((PyObject *)it); + + assert(_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); + PyObject_GC_UnTrack(it); tp->tp_clear((PyObject *)it); - if (state->fi_freelist_len < FI_FREELIST_MAXLEN) { - state->fi_freelist_len++; - it->future = (FutureObj*) state->fi_freelist; - state->fi_freelist = it; - } - else { + if (!_Py_FREELIST_PUSH(futureiters, it, Py_futureiters_MAXFREELIST)) { PyObject_GC_Del(it); Py_DECREF(tp); } @@ -1814,14 +1787,8 @@ future_new_iter(PyObject *fut) asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); ENSURE_FUTURE_ALIVE(state, fut) - if (state->fi_freelist_len) { - state->fi_freelist_len--; - it = state->fi_freelist; - state->fi_freelist = (futureiterobject*) it->future; - it->future = NULL; - _Py_NewReference((PyObject*) it); - } - else { + it = _Py_FREELIST_POP(futureiterobject, futureiters); + if (it == NULL) { it = PyObject_GC_New(futureiterobject, state->FutureIterType); if (it == NULL) { return NULL; @@ -1953,16 +1920,24 @@ static PyMethodDef TaskWakeupDef = { /* ----- Task introspection helpers */ -static int -register_task(asyncio_state *state, PyObject *task) +static void +register_task(asyncio_state *state, TaskObj *task) { - PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, - &_Py_ID(add), task); - if (res == NULL) { - return -1; + ASYNCIO_STATE_LOCK(state); + assert(Task_Check(state, task)); + assert(task != &state->asyncio_tasks.tail); + if (task->next != NULL) { + // already registered + goto exit; } - Py_DECREF(res); - return 0; + assert(task->prev == NULL); + assert(state->asyncio_tasks.head != NULL); + + task->next = state->asyncio_tasks.head; + state->asyncio_tasks.head->prev = task; + state->asyncio_tasks.head = task; +exit: + ASYNCIO_STATE_UNLOCK(state); } static int @@ -1971,16 +1946,30 @@ register_eager_task(asyncio_state *state, PyObject *task) return PySet_Add(state->eager_tasks, task); } -static int -unregister_task(asyncio_state *state, PyObject *task) -{ - PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, - &_Py_ID(discard), task); - if (res == NULL) { - return -1; +static void +unregister_task(asyncio_state *state, TaskObj *task) +{ + ASYNCIO_STATE_LOCK(state); + assert(Task_Check(state, task)); + assert(task != &state->asyncio_tasks.tail); + if (task->next == NULL) { + // not registered + assert(task->prev == NULL); + assert(state->asyncio_tasks.head != task); + goto exit; + } + task->next->prev = task->prev; + if (task->prev == NULL) { + assert(state->asyncio_tasks.head == task); + state->asyncio_tasks.head = task->next; + } else { + task->prev->next = task->next; } - Py_DECREF(res); - return 0; + task->next = NULL; + task->prev = NULL; + assert(state->asyncio_tasks.head != task); +exit: + ASYNCIO_STATE_UNLOCK(state); } static int @@ -1993,14 +1982,11 @@ static int enter_task(asyncio_state *state, PyObject *loop, PyObject *task) { PyObject *item; - Py_hash_t hash; - hash = PyObject_Hash(loop); - if (hash == -1) { + int res = PyDict_SetDefaultRef(state->current_tasks, loop, task, &item); + if (res < 0) { return -1; } - item = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); - if (item != NULL) { - Py_INCREF(item); + else if (res == 1) { PyErr_Format( PyExc_RuntimeError, "Cannot enter into task %R while another " \ @@ -2009,72 +1995,85 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) Py_DECREF(item); return -1; } - if (PyErr_Occurred()) { - return -1; - } - return _PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash); + Py_DECREF(item); + return 0; } +static int +err_leave_task(PyObject *item, PyObject *task) +{ + PyErr_Format( + PyExc_RuntimeError, + "Leaving task %R does not match the current task %R.", + task, item); + return -1; +} + +static int +leave_task_predicate(PyObject *item, void *task) +{ + if (item != task) { + return err_leave_task(item, (PyObject *)task); + } + return 1; +} static int leave_task(asyncio_state *state, PyObject *loop, PyObject *task) /*[clinic end generated code: output=0ebf6db4b858fb41 input=51296a46313d1ad8]*/ { - PyObject *item; - Py_hash_t hash; - hash = PyObject_Hash(loop); - if (hash == -1) { - return -1; - } - item = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); - if (item != task) { - if (item == NULL) { - /* Not entered, replace with None */ - item = Py_None; - } - PyErr_Format( - PyExc_RuntimeError, - "Leaving task %R does not match the current task %R.", - task, item, NULL); - return -1; + int res = _PyDict_DelItemIf(state->current_tasks, loop, + leave_task_predicate, task); + if (res == 0) { + // task was not found + return err_leave_task(Py_None, task); } - return _PyDict_DelItem_KnownHash(state->current_tasks, loop, hash); + return res; } static PyObject * -swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) +swap_current_task_lock_held(PyDictObject *current_tasks, PyObject *loop, + Py_hash_t hash, PyObject *task) { PyObject *prev_task; - Py_hash_t hash; - hash = PyObject_Hash(loop); - if (hash == -1) { + if (_PyDict_GetItemRef_KnownHash_LockHeld(current_tasks, loop, hash, &prev_task) < 0) { + return NULL; + } + if (_PyDict_SetItem_KnownHash_LockHeld(current_tasks, loop, task, hash) < 0) { + Py_XDECREF(prev_task); return NULL; } - - prev_task = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); if (prev_task == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - prev_task = Py_None; + Py_RETURN_NONE; } - Py_INCREF(prev_task); + return prev_task; +} + +static PyObject * +swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) +{ + PyObject *prev_task; if (task == Py_None) { - if (_PyDict_DelItem_KnownHash(state->current_tasks, loop, hash) == -1) { - goto error; + if (PyDict_Pop(state->current_tasks, loop, &prev_task) < 0) { + return NULL; } - } else { - if (_PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash) == -1) { - goto error; + if (prev_task == NULL) { + Py_RETURN_NONE; } + return prev_task; } - return prev_task; + Py_hash_t hash = PyObject_Hash(loop); + if (hash == -1) { + return NULL; + } -error: - Py_DECREF(prev_task); - return NULL; + PyDictObject *current_tasks = (PyDictObject *)state->current_tasks; + Py_BEGIN_CRITICAL_SECTION(current_tasks); + prev_task = swap_current_task_lock_held(current_tasks, loop, hash, task); + Py_END_CRITICAL_SECTION(); + return prev_task; } /* ----- Task */ @@ -2121,7 +2120,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, return -1; } } else { - self->task_context = Py_NewRef(context); + Py_XSETREF(self->task_context, Py_NewRef(context)); } Py_CLEAR(self->task_fut_waiter); @@ -2135,7 +2134,12 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, // optimization: defer task name formatting // store the task counter as PyLong in the name // for deferred formatting in get_name - name = PyLong_FromUnsignedLongLong(++state->task_name_counter); +#ifdef Py_GIL_DISABLED + unsigned long long counter = _Py_atomic_add_uint64(&state->task_name_counter, 1) + 1; +#else + unsigned long long counter = ++state->task_name_counter; +#endif + name = PyLong_FromUnsignedLongLong(counter); } else if (!PyUnicode_CheckExact(name)) { name = PyObject_Str(name); } else { @@ -2164,7 +2168,8 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, if (task_call_step_soon(state, self, NULL)) { return -1; } - return register_task(state, (PyObject*)self); + register_task(state, self); + return 0; } static int @@ -2393,6 +2398,9 @@ _asyncio_Task_uncancel_impl(TaskObj *self) { if (self->task_num_cancels_requested > 0) { self->task_num_cancels_requested -= 1; + if (self->task_num_cancels_requested == 0) { + self->task_must_cancel = 0; + } } return PyLong_FromLong(self->task_num_cancels_requested); } @@ -2506,7 +2514,11 @@ static PyObject * _asyncio_Task_get_coro_impl(TaskObj *self) /*[clinic end generated code: output=bcac27c8cc6c8073 input=d2e8606c42a7b403]*/ { - return Py_NewRef(self->task_coro); + if (self->task_coro) { + return Py_NewRef(self->task_coro); + } + + Py_RETURN_NONE; } /*[clinic input] @@ -2569,6 +2581,15 @@ _asyncio_Task_set_name(TaskObj *self, PyObject *value) static void TaskObj_finalize(TaskObj *task) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); + // Unregister the task from the linked list of tasks. + // Since task is a native task, we directly call the + // unregister_task function. Third party event loops + // should use the asyncio._unregister_task function. + // See https://docs.python.org/3/library/asyncio-extending.html#task-lifetime-support + + unregister_task(state, task); + PyObject *context; PyObject *message = NULL; PyObject *func; @@ -2717,7 +2738,11 @@ task_call_step_soon(asyncio_state *state, TaskObj *task, PyObject *arg) return -1; } - int ret = call_soon(state, task->task_loop, cb, NULL, task->task_context); + // Beware: An evil call_soon could alter task_context. + // See: https://github.com/python/cpython/issues/126080. + PyObject *task_context = Py_NewRef(task->task_context); + int ret = call_soon(state, task->task_loop, cb, NULL, task_context); + Py_DECREF(task_context); Py_DECREF(cb); return ret; } @@ -2776,7 +2801,7 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) if (task->task_state != STATE_PENDING) { PyErr_Format(state->asyncio_InvalidStateError, - "_step(): already done: %R %R", + "__step(): already done: %R %R", task, exc ? exc : Py_None); goto fail; @@ -2942,8 +2967,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (task->task_must_cancel) { PyObject *r; int is_true; + + // Beware: An evil `__getattribute__` could + // prematurely delete task->task_cancel_msg before the + // task is cancelled, thereby causing a UAF crash. + // + // See https://github.com/python/cpython/issues/126138 + PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg); r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel), - task->task_cancel_msg); + task_cancel_msg); + Py_DECREF(task_cancel_msg); + if (r == NULL) { return NULL; } @@ -3035,8 +3069,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (task->task_must_cancel) { PyObject *r; int is_true; + + // Beware: An evil `__getattribute__` could + // prematurely delete task->task_cancel_msg before the + // task is cancelled, thereby causing a UAF crash. + // + // See https://github.com/python/cpython/issues/126138 + PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg); r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel), - task->task_cancel_msg); + task_cancel_msg); + Py_DECREF(task_cancel_msg); + if (r == NULL) { return NULL; } @@ -3180,9 +3223,7 @@ task_eager_start(asyncio_state *state, TaskObj *task) } if (task->task_state == STATE_PENDING) { - if (register_task(state, (PyObject *)task) == -1) { - retval = -1; - } + register_task(state, task); } else { // This seems to really help performance on pyperformance benchmarks Py_CLEAR(task->task_coro); @@ -3253,11 +3294,8 @@ static PyObject * _asyncio__get_running_loop_impl(PyObject *module) /*[clinic end generated code: output=b4390af721411a0a input=0a21627e25a4bd43]*/ { - PyObject *loop; - asyncio_state *state = get_asyncio_state(module); - if (get_running_loop(state, &loop)) { - return NULL; - } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + PyObject *loop = Py_XNewRef(ts->asyncio_running_loop); if (loop == NULL) { /* There's no currently running event loop */ Py_RETURN_NONE; @@ -3280,10 +3318,11 @@ static PyObject * _asyncio__set_running_loop(PyObject *module, PyObject *loop) /*[clinic end generated code: output=ae56bf7a28ca189a input=4c9720233d606604]*/ { - asyncio_state *state = get_asyncio_state(module); - if (set_running_loop(state, loop)) { - return NULL; + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (loop == Py_None) { + loop = NULL; } + Py_XSETREF(ts->asyncio_running_loop, Py_XNewRef(loop)); Py_RETURN_NONE; } @@ -3321,14 +3360,13 @@ _asyncio_get_running_loop_impl(PyObject *module) /*[clinic end generated code: output=c247b5f9e529530e input=2a3bf02ba39f173d]*/ { PyObject *loop; - asyncio_state *state = get_asyncio_state(module); - if (get_running_loop(state, &loop)) { - return NULL; - } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + loop = Py_XNewRef(ts->asyncio_running_loop); if (loop == NULL) { /* There's no currently running event loop */ PyErr_SetString( PyExc_RuntimeError, "no running event loop"); + return NULL; } return loop; } @@ -3348,9 +3386,20 @@ _asyncio__register_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=8672dadd69a7d4e2 input=21075aaea14dfbad]*/ { asyncio_state *state = get_asyncio_state(module); - if (register_task(state, task) < 0) { + if (Task_Check(state, task)) { + // task is an asyncio.Task instance or subclass, use efficient + // linked-list implementation. + register_task(state, (TaskObj *)task); + Py_RETURN_NONE; + } + // As task does not inherit from asyncio.Task, fallback to less efficient + // weakset implementation. + PyObject *res = PyObject_CallMethodOneArg(state->non_asyncio_tasks, + &_Py_ID(add), task); + if (res == NULL) { return NULL; } + Py_DECREF(res); Py_RETURN_NONE; } @@ -3391,9 +3440,16 @@ _asyncio__unregister_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=6e5585706d568a46 input=28fb98c3975f7bdc]*/ { asyncio_state *state = get_asyncio_state(module); - if (unregister_task(state, task) < 0) { + if (Task_Check(state, task)) { + unregister_task(state, (TaskObj *)task); + Py_RETURN_NONE; + } + PyObject *res = PyObject_CallMethodOneArg(state->non_asyncio_tasks, + &_Py_ID(discard), task); + if (res == NULL) { return NULL; } + Py_DECREF(res); Py_RETURN_NONE; } @@ -3524,26 +3580,123 @@ _asyncio_current_task_impl(PyObject *module, PyObject *loop) } +static inline int +add_one_task(asyncio_state *state, PyObject *tasks, PyObject *task, PyObject *loop) +{ + PyObject *done = PyObject_CallMethodNoArgs(task, &_Py_ID(done)); + if (done == NULL) { + return -1; + } + if (Py_IsTrue(done)) { + return 0; + } + Py_DECREF(done); + PyObject *task_loop = get_future_loop(state, task); + if (task_loop == NULL) { + return -1; + } + if (task_loop == loop) { + if (PySet_Add(tasks, task) < 0) { + Py_DECREF(task_loop); + return -1; + } + } + Py_DECREF(task_loop); + return 0; +} + /*********************** Module **************************/ +/*[clinic input] +_asyncio.all_tasks -static void -module_free_freelists(asyncio_state *state) -{ - PyObject *next; - PyObject *current; + loop: object = None - next = (PyObject*) state->fi_freelist; - while (next != NULL) { - assert(state->fi_freelist_len > 0); - state->fi_freelist_len--; +Return a set of all tasks for the loop. - current = next; - next = (PyObject*) ((futureiterobject*) current)->future; - PyObject_GC_Del(current); +[clinic start generated code]*/ + +static PyObject * +_asyncio_all_tasks_impl(PyObject *module, PyObject *loop) +/*[clinic end generated code: output=0e107cbb7f72aa7b input=43a1b423c2d95bfa]*/ +{ + + asyncio_state *state = get_asyncio_state(module); + PyObject *tasks = PySet_New(NULL); + if (tasks == NULL) { + return NULL; + } + if (loop == Py_None) { + loop = _asyncio_get_running_loop_impl(module); + if (loop == NULL) { + Py_DECREF(tasks); + return NULL; + } + } else { + Py_INCREF(loop); + } + // First add eager tasks to the set so that we don't miss + // any tasks which graduates from eager to non-eager + PyObject *eager_iter = PyObject_GetIter(state->eager_tasks); + if (eager_iter == NULL) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; } - assert(state->fi_freelist_len == 0); - state->fi_freelist = NULL; + PyObject *item; + while ((item = PyIter_Next(eager_iter)) != NULL) { + if (add_one_task(state, tasks, item, loop) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + Py_DECREF(item); + Py_DECREF(eager_iter); + return NULL; + } + Py_DECREF(item); + } + Py_DECREF(eager_iter); + int err = 0; + ASYNCIO_STATE_LOCK(state); + TaskObj *head = state->asyncio_tasks.head; + Py_INCREF(head); + assert(head != NULL); + assert(head->prev == NULL); + TaskObj *tail = &state->asyncio_tasks.tail; + while (head != tail) + { + if (add_one_task(state, tasks, (PyObject *)head, loop) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + Py_DECREF(head); + err = 1; + break; + } + Py_INCREF(head->next); + Py_SETREF(head, head->next); + } + ASYNCIO_STATE_UNLOCK(state); + if (err) { + return NULL; + } + PyObject *scheduled_iter = PyObject_GetIter(state->non_asyncio_tasks); + if (scheduled_iter == NULL) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + while ((item = PyIter_Next(scheduled_iter)) != NULL) { + if (add_one_task(state, tasks, item, loop) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + Py_DECREF(item); + Py_DECREF(scheduled_iter); + return NULL; + } + Py_DECREF(item); + } + Py_DECREF(scheduled_iter); + Py_DECREF(loop); + return tasks; } static int @@ -3567,20 +3720,13 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->asyncio_InvalidStateError); Py_VISIT(state->asyncio_CancelledError); - Py_VISIT(state->scheduled_tasks); + Py_VISIT(state->non_asyncio_tasks); Py_VISIT(state->eager_tasks); Py_VISIT(state->current_tasks); Py_VISIT(state->iscoroutine_typecache); Py_VISIT(state->context_kwname); - // Visit freelist. - PyObject *next = (PyObject*) state->fi_freelist; - while (next != NULL) { - PyObject *current = next; - Py_VISIT(current); - next = (PyObject*) ((futureiterobject*) current)->future; - } return 0; } @@ -3605,15 +3751,13 @@ module_clear(PyObject *mod) Py_CLEAR(state->asyncio_InvalidStateError); Py_CLEAR(state->asyncio_CancelledError); - Py_CLEAR(state->scheduled_tasks); + Py_CLEAR(state->non_asyncio_tasks); Py_CLEAR(state->eager_tasks); Py_CLEAR(state->current_tasks); Py_CLEAR(state->iscoroutine_typecache); Py_CLEAR(state->context_kwname); - module_free_freelists(state); - return 0; } @@ -3686,9 +3830,9 @@ module_init(asyncio_state *state) PyObject *weak_set; WITH_MOD("weakref") GET_MOD_ATTR(weak_set, "WeakSet"); - state->scheduled_tasks = PyObject_CallNoArgs(weak_set); + state->non_asyncio_tasks = PyObject_CallNoArgs(weak_set); Py_CLEAR(weak_set); - if (state->scheduled_tasks == NULL) { + if (state->non_asyncio_tasks == NULL) { goto fail; } @@ -3723,6 +3867,7 @@ static PyMethodDef asyncio_methods[] = { _ASYNCIO__ENTER_TASK_METHODDEF _ASYNCIO__LEAVE_TASK_METHODDEF _ASYNCIO__SWAP_CURRENT_TASK_METHODDEF + _ASYNCIO_ALL_TASKS_METHODDEF {NULL, NULL} }; @@ -3730,6 +3875,9 @@ static int module_exec(PyObject *mod) { asyncio_state *state = get_asyncio_state(mod); + Py_SET_TYPE(&state->asyncio_tasks.tail, state->TaskType); + _Py_SetImmortalUntracked((PyObject *)&state->asyncio_tasks.tail); + state->asyncio_tasks.head = &state->asyncio_tasks.tail; #define CREATE_TYPE(m, tp, spec, base) \ do { \ @@ -3759,7 +3907,7 @@ module_exec(PyObject *mod) return -1; } - if (PyModule_AddObjectRef(mod, "_scheduled_tasks", state->scheduled_tasks) < 0) { + if (PyModule_AddObjectRef(mod, "_scheduled_tasks", state->non_asyncio_tasks) < 0) { return -1; } @@ -3778,6 +3926,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9e0fd336419b44..9b146265445d9a 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -70,7 +70,7 @@ internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t if (sq_item == NULL) { return -1; } - if (Py_EnterRecursiveCall("in _bisect.bisect_right")) { + if (Py_EnterRecursiveCall(" in _bisect.bisect_right")) { return -1; } PyTypeObject *tp = Py_TYPE(item); @@ -254,7 +254,7 @@ internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t h if (sq_item == NULL) { return -1; } - if (Py_EnterRecursiveCall("in _bisect.bisect_left")) { + if (Py_EnterRecursiveCall(" in _bisect.bisect_left")) { return -1; } PyTypeObject *tp = Py_TYPE(item); @@ -462,6 +462,7 @@ bisect_modexec(PyObject *m) static PyModuleDef_Slot bisect_slots[] = { {Py_mod_exec, bisect_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_blake2/blake2b2s.py b/Modules/_blake2/blake2b2s.py deleted file mode 100755 index 01cf26521b3779..00000000000000 --- a/Modules/_blake2/blake2b2s.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/python3 - -import os -import re - -HERE = os.path.dirname(os.path.abspath(__file__)) -BLAKE2 = os.path.join(HERE, 'impl') - -PUBLIC_SEARCH = re.compile(r'\ int (blake2[bs]p?[a-z_]*)\(') - - -def getfiles(): - for name in os.listdir(BLAKE2): - name = os.path.join(BLAKE2, name) - if os.path.isfile(name): - yield name - - -def find_public(): - public_funcs = set() - for name in getfiles(): - with open(name) as f: - for line in f: - # find public functions - mo = PUBLIC_SEARCH.search(line) - if mo: - public_funcs.add(mo.group(1)) - - for f in sorted(public_funcs): - print('#define {0:<18} PyBlake2_{0}'.format(f)) - - return public_funcs - - -def main(): - lines = [] - with open(os.path.join(HERE, 'blake2b_impl.c')) as f: - for line in f: - line = line.replace('blake2b', 'blake2s') - line = line.replace('BLAKE2b', 'BLAKE2s') - line = line.replace('BLAKE2B', 'BLAKE2S') - lines.append(line) - with open(os.path.join(HERE, 'blake2s_impl.c'), 'w') as f: - f.write(''.join(lines)) - # find_public() - - -if __name__ == '__main__': - main() diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c deleted file mode 100644 index 0c3ae5a2fac275..00000000000000 --- a/Modules/_blake2/blake2b_impl.c +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Written in 2013 by Dmitry Chestnykh - * Modified for CPython by Christian Heimes - * - * To the extent possible under law, the author have dedicated all - * copyright and related and neighboring rights to this software to - * the public domain worldwide. This software is distributed without - * any warranty. http://creativecommons.org/publicdomain/zero/1.0/ - */ - -/* WARNING: autogenerated file! - * - * The blake2s_impl.c is autogenerated from blake2b_impl.c. - */ - -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include -#include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() - -#include "../hashlib.h" -#include "blake2module.h" - -#ifndef HAVE_LIBB2 -/* pure SSE2 implementation is very slow, so only use the more optimized SSSE3+ - * https://bugs.python.org/issue31834 */ -#if defined(__SSSE3__) || defined(__SSE4_1__) || defined(__AVX__) || defined(__XOP__) -#include "impl/blake2b.c" -#else -#include "impl/blake2b-ref.c" -#endif -#endif // !HAVE_LIBB2 - -#define HAVE_BLAKE2B 1 - -extern PyType_Spec blake2b_type_spec; - - -typedef struct { - PyObject_HEAD - blake2b_param param; - blake2b_state state; - bool use_mutex; - PyMutex mutex; -} BLAKE2bObject; - -#include "clinic/blake2b_impl.c.h" - -/*[clinic input] -module _blake2 -class _blake2.blake2b "BLAKE2bObject *" "&PyBlake2_BLAKE2bType" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d47b0527b39c673f]*/ - - -static BLAKE2bObject * -new_BLAKE2bObject(PyTypeObject *type) -{ - BLAKE2bObject *self; - self = (BLAKE2bObject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - HASHLIB_INIT_MUTEX(self); - - return self; -} - -/*[clinic input] -@classmethod -_blake2.blake2b.__new__ as py_blake2b_new - data: object(c_default="NULL") = b'' - / - * - digest_size: int(c_default="BLAKE2B_OUTBYTES") = _blake2.blake2b.MAX_DIGEST_SIZE - key: Py_buffer(c_default="NULL", py_default="b''") = None - salt: Py_buffer(c_default="NULL", py_default="b''") = None - person: Py_buffer(c_default="NULL", py_default="b''") = None - fanout: int = 1 - depth: int = 1 - leaf_size: unsigned_long = 0 - node_offset: unsigned_long_long = 0 - node_depth: int = 0 - inner_size: int = 0 - last_node: bool = False - usedforsecurity: bool = True - -Return a new BLAKE2b hash object. -[clinic start generated code]*/ - -static PyObject * -py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, - Py_buffer *key, Py_buffer *salt, Py_buffer *person, - int fanout, int depth, unsigned long leaf_size, - unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=32bfd8f043c6896f input=b947312abff46977]*/ -{ - BLAKE2bObject *self = NULL; - Py_buffer buf; - - self = new_BLAKE2bObject(type); - if (self == NULL) { - goto error; - } - - /* Zero parameter block. */ - memset(&self->param, 0, sizeof(self->param)); - - /* Set digest size. */ - if (digest_size <= 0 || digest_size > BLAKE2B_OUTBYTES) { - PyErr_Format(PyExc_ValueError, - "digest_size must be between 1 and %d bytes", - BLAKE2B_OUTBYTES); - goto error; - } - self->param.digest_length = digest_size; - - /* Set salt parameter. */ - if ((salt->obj != NULL) && salt->len) { - if (salt->len > BLAKE2B_SALTBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum salt length is %d bytes", - BLAKE2B_SALTBYTES); - goto error; - } - memcpy(self->param.salt, salt->buf, salt->len); - } - - /* Set personalization parameter. */ - if ((person->obj != NULL) && person->len) { - if (person->len > BLAKE2B_PERSONALBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum person length is %d bytes", - BLAKE2B_PERSONALBYTES); - goto error; - } - memcpy(self->param.personal, person->buf, person->len); - } - - /* Set tree parameters. */ - if (fanout < 0 || fanout > 255) { - PyErr_SetString(PyExc_ValueError, - "fanout must be between 0 and 255"); - goto error; - } - self->param.fanout = (uint8_t)fanout; - - if (depth <= 0 || depth > 255) { - PyErr_SetString(PyExc_ValueError, - "depth must be between 1 and 255"); - goto error; - } - self->param.depth = (uint8_t)depth; - - if (leaf_size > 0xFFFFFFFFU) { - PyErr_SetString(PyExc_OverflowError, "leaf_size is too large"); - goto error; - } - // NB: Simple assignment here would be incorrect on big endian platforms. - store32(&(self->param.leaf_length), leaf_size); - -#ifdef HAVE_BLAKE2S - if (node_offset > 0xFFFFFFFFFFFFULL) { - /* maximum 2**48 - 1 */ - PyErr_SetString(PyExc_OverflowError, "node_offset is too large"); - goto error; - } - store48(&(self->param.node_offset), node_offset); -#else - // NB: Simple assignment here would be incorrect on big endian platforms. - store64(&(self->param.node_offset), node_offset); -#endif - - if (node_depth < 0 || node_depth > 255) { - PyErr_SetString(PyExc_ValueError, - "node_depth must be between 0 and 255"); - goto error; - } - self->param.node_depth = node_depth; - - if (inner_size < 0 || inner_size > BLAKE2B_OUTBYTES) { - PyErr_Format(PyExc_ValueError, - "inner_size must be between 0 and is %d", - BLAKE2B_OUTBYTES); - goto error; - } - self->param.inner_length = inner_size; - - /* Set key length. */ - if ((key->obj != NULL) && key->len) { - if (key->len > BLAKE2B_KEYBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum key length is %d bytes", - BLAKE2B_KEYBYTES); - goto error; - } - self->param.key_length = (uint8_t)key->len; - } - - /* Initialize hash state. */ - if (blake2b_init_param(&self->state, &self->param) < 0) { - PyErr_SetString(PyExc_RuntimeError, - "error initializing hash state"); - goto error; - } - - /* Set last node flag (must come after initialization). */ - self->state.last_node = last_node; - - /* Process key block if any. */ - if (self->param.key_length) { - uint8_t block[BLAKE2B_BLOCKBYTES]; - memset(block, 0, sizeof(block)); - memcpy(block, key->buf, key->len); - blake2b_update(&self->state, block, sizeof(block)); - secure_zero_memory(block, sizeof(block)); - } - - /* Process initial data if any. */ - if (data != NULL) { - GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - - if (buf.len >= HASHLIB_GIL_MINSIZE) { - Py_BEGIN_ALLOW_THREADS - blake2b_update(&self->state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } else { - blake2b_update(&self->state, buf.buf, buf.len); - } - PyBuffer_Release(&buf); - } - - return (PyObject *)self; - - error: - if (self != NULL) { - Py_DECREF(self); - } - return NULL; -} - -/*[clinic input] -_blake2.blake2b.copy - -Return a copy of the hash object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2b_copy_impl(BLAKE2bObject *self) -/*[clinic end generated code: output=ff6acee5f93656ae input=e383c2d199fd8a2e]*/ -{ - BLAKE2bObject *cpy; - - if ((cpy = new_BLAKE2bObject(Py_TYPE(self))) == NULL) - return NULL; - - ENTER_HASHLIB(self); - cpy->param = self->param; - cpy->state = self->state; - LEAVE_HASHLIB(self); - return (PyObject *)cpy; -} - -/*[clinic input] -_blake2.blake2b.update - - data: object - / - -Update this hash object's state with the provided bytes-like object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) -/*[clinic end generated code: output=010dfcbe22654359 input=ffc4aa6a6a225d31]*/ -{ - Py_buffer buf; - - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - blake2b_update(&self->state, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - blake2b_update(&self->state, buf.buf, buf.len); - } - - PyBuffer_Release(&buf); - - Py_RETURN_NONE; -} - -/*[clinic input] -_blake2.blake2b.digest - -Return the digest value as a bytes object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2b_digest_impl(BLAKE2bObject *self) -/*[clinic end generated code: output=a5864660f4bfc61a input=7d21659e9c5fff02]*/ -{ - uint8_t digest[BLAKE2B_OUTBYTES]; - blake2b_state state_cpy; - - ENTER_HASHLIB(self); - state_cpy = self->state; - blake2b_final(&state_cpy, digest, self->param.digest_length); - LEAVE_HASHLIB(self); - return PyBytes_FromStringAndSize((const char *)digest, - self->param.digest_length); -} - -/*[clinic input] -_blake2.blake2b.hexdigest - -Return the digest value as a string of hexadecimal digits. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2b_hexdigest_impl(BLAKE2bObject *self) -/*[clinic end generated code: output=b5598a87d8794a60 input=76930f6946351f56]*/ -{ - uint8_t digest[BLAKE2B_OUTBYTES]; - blake2b_state state_cpy; - - ENTER_HASHLIB(self); - state_cpy = self->state; - blake2b_final(&state_cpy, digest, self->param.digest_length); - LEAVE_HASHLIB(self); - return _Py_strhex((const char *)digest, self->param.digest_length); -} - - -static PyMethodDef py_blake2b_methods[] = { - _BLAKE2_BLAKE2B_COPY_METHODDEF - _BLAKE2_BLAKE2B_DIGEST_METHODDEF - _BLAKE2_BLAKE2B_HEXDIGEST_METHODDEF - _BLAKE2_BLAKE2B_UPDATE_METHODDEF - {NULL, NULL} -}; - - - -static PyObject * -py_blake2b_get_name(BLAKE2bObject *self, void *closure) -{ - return PyUnicode_FromString("blake2b"); -} - - - -static PyObject * -py_blake2b_get_block_size(BLAKE2bObject *self, void *closure) -{ - return PyLong_FromLong(BLAKE2B_BLOCKBYTES); -} - - - -static PyObject * -py_blake2b_get_digest_size(BLAKE2bObject *self, void *closure) -{ - return PyLong_FromLong(self->param.digest_length); -} - - -static PyGetSetDef py_blake2b_getsetters[] = { - {"name", (getter)py_blake2b_get_name, - NULL, NULL, NULL}, - {"block_size", (getter)py_blake2b_get_block_size, - NULL, NULL, NULL}, - {"digest_size", (getter)py_blake2b_get_digest_size, - NULL, NULL, NULL}, - {NULL} -}; - - -static void -py_blake2b_dealloc(PyObject *self) -{ - BLAKE2bObject *obj = (BLAKE2bObject *)self; - - /* Try not to leave state in memory. */ - secure_zero_memory(&obj->param, sizeof(obj->param)); - secure_zero_memory(&obj->state, sizeof(obj->state)); - - PyTypeObject *type = Py_TYPE(self); - PyObject_Free(self); - Py_DECREF(type); -} - -static PyType_Slot blake2b_type_slots[] = { - {Py_tp_dealloc, py_blake2b_dealloc}, - {Py_tp_doc, (char *)py_blake2b_new__doc__}, - {Py_tp_methods, py_blake2b_methods}, - {Py_tp_getset, py_blake2b_getsetters}, - {Py_tp_new, py_blake2b_new}, - {0,0} -}; - -PyType_Spec blake2b_type_spec = { - .name = "_blake2.blake2b", - .basicsize = sizeof(BLAKE2bObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, - .slots = blake2b_type_slots -}; diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c deleted file mode 100644 index 5df9fd3df493ee..00000000000000 --- a/Modules/_blake2/blake2module.c +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Written in 2013 by Dmitry Chestnykh - * Modified for CPython by Christian Heimes - * - * To the extent possible under law, the author have dedicated all - * copyright and related and neighboring rights to this software to - * the public domain worldwide. This software is distributed without - * any warranty. http://creativecommons.org/publicdomain/zero/1.0/ - */ - -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include "Python.h" -#include "blake2module.h" - -extern PyType_Spec blake2b_type_spec; -extern PyType_Spec blake2s_type_spec; - -PyDoc_STRVAR(blake2mod__doc__, -"_blake2b provides BLAKE2b for hashlib\n" -); - -typedef struct { - PyTypeObject* blake2b_type; - PyTypeObject* blake2s_type; -} Blake2State; - -static inline Blake2State* -blake2_get_state(PyObject *module) -{ - void *state = PyModule_GetState(module); - assert(state != NULL); - return (Blake2State *)state; -} - -static struct PyMethodDef blake2mod_functions[] = { - {NULL, NULL} -}; - -static int -_blake2_traverse(PyObject *module, visitproc visit, void *arg) -{ - Blake2State *state = blake2_get_state(module); - Py_VISIT(state->blake2b_type); - Py_VISIT(state->blake2s_type); - return 0; -} - -static int -_blake2_clear(PyObject *module) -{ - Blake2State *state = blake2_get_state(module); - Py_CLEAR(state->blake2b_type); - Py_CLEAR(state->blake2s_type); - return 0; -} - -static void -_blake2_free(void *module) -{ - _blake2_clear((PyObject *)module); -} - -#define ADD_INT(d, name, value) do { \ - PyObject *x = PyLong_FromLong(value); \ - if (!x) \ - return -1; \ - if (PyDict_SetItemString(d, name, x) < 0) { \ - Py_DECREF(x); \ - return -1; \ - } \ - Py_DECREF(x); \ -} while(0) - -#define ADD_INT_CONST(NAME, VALUE) do { \ - if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ - return -1; \ - } \ -} while (0) - -static int -blake2_exec(PyObject *m) -{ - Blake2State* st = blake2_get_state(m); - - st->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec( - m, &blake2b_type_spec, NULL); - - if (NULL == st->blake2b_type) - return -1; - /* BLAKE2b */ - if (PyModule_AddType(m, st->blake2b_type) < 0) { - return -1; - } - - PyObject *d = st->blake2b_type->tp_dict; - ADD_INT(d, "SALT_SIZE", BLAKE2B_SALTBYTES); - ADD_INT(d, "PERSON_SIZE", BLAKE2B_PERSONALBYTES); - ADD_INT(d, "MAX_KEY_SIZE", BLAKE2B_KEYBYTES); - ADD_INT(d, "MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); - - ADD_INT_CONST("BLAKE2B_SALT_SIZE", BLAKE2B_SALTBYTES); - ADD_INT_CONST("BLAKE2B_PERSON_SIZE", BLAKE2B_PERSONALBYTES); - ADD_INT_CONST("BLAKE2B_MAX_KEY_SIZE", BLAKE2B_KEYBYTES); - ADD_INT_CONST("BLAKE2B_MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); - - /* BLAKE2s */ - st->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec( - m, &blake2s_type_spec, NULL); - - if (NULL == st->blake2s_type) - return -1; - - if (PyModule_AddType(m, st->blake2s_type) < 0) { - return -1; - } - - d = st->blake2s_type->tp_dict; - ADD_INT(d, "SALT_SIZE", BLAKE2S_SALTBYTES); - ADD_INT(d, "PERSON_SIZE", BLAKE2S_PERSONALBYTES); - ADD_INT(d, "MAX_KEY_SIZE", BLAKE2S_KEYBYTES); - ADD_INT(d, "MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); - - ADD_INT_CONST("BLAKE2S_SALT_SIZE", BLAKE2S_SALTBYTES); - ADD_INT_CONST("BLAKE2S_PERSON_SIZE", BLAKE2S_PERSONALBYTES); - ADD_INT_CONST("BLAKE2S_MAX_KEY_SIZE", BLAKE2S_KEYBYTES); - ADD_INT_CONST("BLAKE2S_MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); - - return 0; -} - -#undef ADD_INT -#undef ADD_INT_CONST - -static PyModuleDef_Slot _blake2_slots[] = { - {Py_mod_exec, blake2_exec}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {0, NULL} -}; - -static struct PyModuleDef blake2_module = { - PyModuleDef_HEAD_INIT, - "_blake2", - .m_doc = blake2mod__doc__, - .m_size = sizeof(Blake2State), - .m_methods = blake2mod_functions, - .m_slots = _blake2_slots, - .m_traverse = _blake2_traverse, - .m_clear = _blake2_clear, - .m_free = _blake2_free, -}; - -PyMODINIT_FUNC -PyInit__blake2(void) -{ - return PyModuleDef_Init(&blake2_module); -} diff --git a/Modules/_blake2/blake2module.h b/Modules/_blake2/blake2module.h deleted file mode 100644 index c8144ec9d48d29..00000000000000 --- a/Modules/_blake2/blake2module.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef Py_BLAKE2MODULE_H -#define Py_BLAKE2MODULE_H - -#ifdef HAVE_LIBB2 -#include - -#else -// use vendored copy of blake2 - -// Prefix all public blake2 symbols with PyBlake2_ -#define blake2b PyBlake2_blake2b -#define blake2b_compress PyBlake2_blake2b_compress -#define blake2b_final PyBlake2_blake2b_final -#define blake2b_init PyBlake2_blake2b_init -#define blake2b_init_key PyBlake2_blake2b_init_key -#define blake2b_init_param PyBlake2_blake2b_init_param -#define blake2b_update PyBlake2_blake2b_update -#define blake2bp PyBlake2_blake2bp -#define blake2bp_final PyBlake2_blake2bp_final -#define blake2bp_init PyBlake2_blake2bp_init -#define blake2bp_init_key PyBlake2_blake2bp_init_key -#define blake2bp_update PyBlake2_blake2bp_update -#define blake2s PyBlake2_blake2s -#define blake2s_compress PyBlake2_blake2s_compress -#define blake2s_final PyBlake2_blake2s_final -#define blake2s_init PyBlake2_blake2s_init -#define blake2s_init_key PyBlake2_blake2s_init_key -#define blake2s_init_param PyBlake2_blake2s_init_param -#define blake2s_update PyBlake2_blake2s_update -#define blake2sp PyBlake2_blake2sp -#define blake2sp_final PyBlake2_blake2sp_final -#define blake2sp_init PyBlake2_blake2sp_init -#define blake2sp_init_key PyBlake2_blake2sp_init_key -#define blake2sp_update PyBlake2_blake2sp_update - -#include "impl/blake2.h" - -#endif // HAVE_LIBB2 - -// for secure_zero_memory(), store32(), store48(), and store64() -#include "impl/blake2-impl.h" - -#endif // Py_BLAKE2MODULE_H diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c deleted file mode 100644 index 3014773ab52331..00000000000000 --- a/Modules/_blake2/blake2s_impl.c +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Written in 2013 by Dmitry Chestnykh - * Modified for CPython by Christian Heimes - * - * To the extent possible under law, the author have dedicated all - * copyright and related and neighboring rights to this software to - * the public domain worldwide. This software is distributed without - * any warranty. http://creativecommons.org/publicdomain/zero/1.0/ - */ - -/* WARNING: autogenerated file! - * - * The blake2s_impl.c is autogenerated from blake2s_impl.c. - */ - -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include -#include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() - -#include "../hashlib.h" -#include "blake2module.h" - -#ifndef HAVE_LIBB2 -/* pure SSE2 implementation is very slow, so only use the more optimized SSSE3+ - * https://bugs.python.org/issue31834 */ -#if defined(__SSSE3__) || defined(__SSE4_1__) || defined(__AVX__) || defined(__XOP__) -#include "impl/blake2s.c" -#else -#include "impl/blake2s-ref.c" -#endif -#endif // !HAVE_LIBB2 - -#define HAVE_BLAKE2S 1 - -extern PyType_Spec blake2s_type_spec; - - -typedef struct { - PyObject_HEAD - blake2s_param param; - blake2s_state state; - bool use_mutex; - PyMutex mutex; -} BLAKE2sObject; - -#include "clinic/blake2s_impl.c.h" - -/*[clinic input] -module _blake2 -class _blake2.blake2s "BLAKE2sObject *" "&PyBlake2_BLAKE2sType" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b79d7ffe07286ce]*/ - - -static BLAKE2sObject * -new_BLAKE2sObject(PyTypeObject *type) -{ - BLAKE2sObject *self; - self = (BLAKE2sObject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - HASHLIB_INIT_MUTEX(self); - - return self; -} - -/*[clinic input] -@classmethod -_blake2.blake2s.__new__ as py_blake2s_new - data: object(c_default="NULL") = b'' - / - * - digest_size: int(c_default="BLAKE2S_OUTBYTES") = _blake2.blake2s.MAX_DIGEST_SIZE - key: Py_buffer(c_default="NULL", py_default="b''") = None - salt: Py_buffer(c_default="NULL", py_default="b''") = None - person: Py_buffer(c_default="NULL", py_default="b''") = None - fanout: int = 1 - depth: int = 1 - leaf_size: unsigned_long = 0 - node_offset: unsigned_long_long = 0 - node_depth: int = 0 - inner_size: int = 0 - last_node: bool = False - usedforsecurity: bool = True - -Return a new BLAKE2s hash object. -[clinic start generated code]*/ - -static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, - Py_buffer *key, Py_buffer *salt, Py_buffer *person, - int fanout, int depth, unsigned long leaf_size, - unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=556181f73905c686 input=4dda87723f23abb0]*/ -{ - BLAKE2sObject *self = NULL; - Py_buffer buf; - - self = new_BLAKE2sObject(type); - if (self == NULL) { - goto error; - } - - /* Zero parameter block. */ - memset(&self->param, 0, sizeof(self->param)); - - /* Set digest size. */ - if (digest_size <= 0 || digest_size > BLAKE2S_OUTBYTES) { - PyErr_Format(PyExc_ValueError, - "digest_size must be between 1 and %d bytes", - BLAKE2S_OUTBYTES); - goto error; - } - self->param.digest_length = digest_size; - - /* Set salt parameter. */ - if ((salt->obj != NULL) && salt->len) { - if (salt->len > BLAKE2S_SALTBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum salt length is %d bytes", - BLAKE2S_SALTBYTES); - goto error; - } - memcpy(self->param.salt, salt->buf, salt->len); - } - - /* Set personalization parameter. */ - if ((person->obj != NULL) && person->len) { - if (person->len > BLAKE2S_PERSONALBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum person length is %d bytes", - BLAKE2S_PERSONALBYTES); - goto error; - } - memcpy(self->param.personal, person->buf, person->len); - } - - /* Set tree parameters. */ - if (fanout < 0 || fanout > 255) { - PyErr_SetString(PyExc_ValueError, - "fanout must be between 0 and 255"); - goto error; - } - self->param.fanout = (uint8_t)fanout; - - if (depth <= 0 || depth > 255) { - PyErr_SetString(PyExc_ValueError, - "depth must be between 1 and 255"); - goto error; - } - self->param.depth = (uint8_t)depth; - - if (leaf_size > 0xFFFFFFFFU) { - PyErr_SetString(PyExc_OverflowError, "leaf_size is too large"); - goto error; - } - // NB: Simple assignment here would be incorrect on big endian platforms. - store32(&(self->param.leaf_length), leaf_size); - -#ifdef HAVE_BLAKE2S - if (node_offset > 0xFFFFFFFFFFFFULL) { - /* maximum 2**48 - 1 */ - PyErr_SetString(PyExc_OverflowError, "node_offset is too large"); - goto error; - } - store48(&(self->param.node_offset), node_offset); -#else - // NB: Simple assignment here would be incorrect on big endian platforms. - store64(&(self->param.node_offset), node_offset); -#endif - - if (node_depth < 0 || node_depth > 255) { - PyErr_SetString(PyExc_ValueError, - "node_depth must be between 0 and 255"); - goto error; - } - self->param.node_depth = node_depth; - - if (inner_size < 0 || inner_size > BLAKE2S_OUTBYTES) { - PyErr_Format(PyExc_ValueError, - "inner_size must be between 0 and is %d", - BLAKE2S_OUTBYTES); - goto error; - } - self->param.inner_length = inner_size; - - /* Set key length. */ - if ((key->obj != NULL) && key->len) { - if (key->len > BLAKE2S_KEYBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum key length is %d bytes", - BLAKE2S_KEYBYTES); - goto error; - } - self->param.key_length = (uint8_t)key->len; - } - - /* Initialize hash state. */ - if (blake2s_init_param(&self->state, &self->param) < 0) { - PyErr_SetString(PyExc_RuntimeError, - "error initializing hash state"); - goto error; - } - - /* Set last node flag (must come after initialization). */ - self->state.last_node = last_node; - - /* Process key block if any. */ - if (self->param.key_length) { - uint8_t block[BLAKE2S_BLOCKBYTES]; - memset(block, 0, sizeof(block)); - memcpy(block, key->buf, key->len); - blake2s_update(&self->state, block, sizeof(block)); - secure_zero_memory(block, sizeof(block)); - } - - /* Process initial data if any. */ - if (data != NULL) { - GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - - if (buf.len >= HASHLIB_GIL_MINSIZE) { - Py_BEGIN_ALLOW_THREADS - blake2s_update(&self->state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } else { - blake2s_update(&self->state, buf.buf, buf.len); - } - PyBuffer_Release(&buf); - } - - return (PyObject *)self; - - error: - if (self != NULL) { - Py_DECREF(self); - } - return NULL; -} - -/*[clinic input] -_blake2.blake2s.copy - -Return a copy of the hash object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2s_copy_impl(BLAKE2sObject *self) -/*[clinic end generated code: output=5b90131c4eae275e input=0b9d44942f0fe4b2]*/ -{ - BLAKE2sObject *cpy; - - if ((cpy = new_BLAKE2sObject(Py_TYPE(self))) == NULL) - return NULL; - - ENTER_HASHLIB(self); - cpy->param = self->param; - cpy->state = self->state; - LEAVE_HASHLIB(self); - return (PyObject *)cpy; -} - -/*[clinic input] -_blake2.blake2s.update - - data: object - / - -Update this hash object's state with the provided bytes-like object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) -/*[clinic end generated code: output=757dc087fec37815 input=97500db2f9de4aaa]*/ -{ - Py_buffer buf; - - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - blake2s_update(&self->state, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - blake2s_update(&self->state, buf.buf, buf.len); - } - - PyBuffer_Release(&buf); - - Py_RETURN_NONE; -} - -/*[clinic input] -_blake2.blake2s.digest - -Return the digest value as a bytes object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2s_digest_impl(BLAKE2sObject *self) -/*[clinic end generated code: output=40c566ca4bc6bc51 input=f41e0b8d6d937454]*/ -{ - uint8_t digest[BLAKE2S_OUTBYTES]; - blake2s_state state_cpy; - - ENTER_HASHLIB(self); - state_cpy = self->state; - blake2s_final(&state_cpy, digest, self->param.digest_length); - LEAVE_HASHLIB(self); - return PyBytes_FromStringAndSize((const char *)digest, - self->param.digest_length); -} - -/*[clinic input] -_blake2.blake2s.hexdigest - -Return the digest value as a string of hexadecimal digits. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2s_hexdigest_impl(BLAKE2sObject *self) -/*[clinic end generated code: output=15153eb5e59c52eb input=c77a1321567e8952]*/ -{ - uint8_t digest[BLAKE2S_OUTBYTES]; - blake2s_state state_cpy; - - ENTER_HASHLIB(self); - state_cpy = self->state; - blake2s_final(&state_cpy, digest, self->param.digest_length); - LEAVE_HASHLIB(self); - return _Py_strhex((const char *)digest, self->param.digest_length); -} - - -static PyMethodDef py_blake2s_methods[] = { - _BLAKE2_BLAKE2S_COPY_METHODDEF - _BLAKE2_BLAKE2S_DIGEST_METHODDEF - _BLAKE2_BLAKE2S_HEXDIGEST_METHODDEF - _BLAKE2_BLAKE2S_UPDATE_METHODDEF - {NULL, NULL} -}; - - - -static PyObject * -py_blake2s_get_name(BLAKE2sObject *self, void *closure) -{ - return PyUnicode_FromString("blake2s"); -} - - - -static PyObject * -py_blake2s_get_block_size(BLAKE2sObject *self, void *closure) -{ - return PyLong_FromLong(BLAKE2S_BLOCKBYTES); -} - - - -static PyObject * -py_blake2s_get_digest_size(BLAKE2sObject *self, void *closure) -{ - return PyLong_FromLong(self->param.digest_length); -} - - -static PyGetSetDef py_blake2s_getsetters[] = { - {"name", (getter)py_blake2s_get_name, - NULL, NULL, NULL}, - {"block_size", (getter)py_blake2s_get_block_size, - NULL, NULL, NULL}, - {"digest_size", (getter)py_blake2s_get_digest_size, - NULL, NULL, NULL}, - {NULL} -}; - - -static void -py_blake2s_dealloc(PyObject *self) -{ - BLAKE2sObject *obj = (BLAKE2sObject *)self; - - /* Try not to leave state in memory. */ - secure_zero_memory(&obj->param, sizeof(obj->param)); - secure_zero_memory(&obj->state, sizeof(obj->state)); - - PyTypeObject *type = Py_TYPE(self); - PyObject_Free(self); - Py_DECREF(type); -} - -static PyType_Slot blake2s_type_slots[] = { - {Py_tp_dealloc, py_blake2s_dealloc}, - {Py_tp_doc, (char *)py_blake2s_new__doc__}, - {Py_tp_methods, py_blake2s_methods}, - {Py_tp_getset, py_blake2s_getsetters}, - {Py_tp_new, py_blake2s_new}, - {0,0} -}; - -PyType_Spec blake2s_type_spec = { - .name = "_blake2.blake2s", - .basicsize = sizeof(BLAKE2sObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, - .slots = blake2s_type_slots -}; diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h deleted file mode 100644 index 7a0f6eeff5b5b5..00000000000000 --- a/Modules/_blake2/clinic/blake2s_impl.c.h +++ /dev/null @@ -1,268 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif -#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() - -PyDoc_STRVAR(py_blake2s_new__doc__, -"blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" -" key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" -" node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" -" usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new BLAKE2s hash object."); - -static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, - Py_buffer *key, Py_buffer *salt, Py_buffer *person, - int fanout, int depth, unsigned long leaf_size, - unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity); - -static PyObject * -py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 12 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "blake2s", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[13]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; - int digest_size = BLAKE2S_OUTBYTES; - Py_buffer key = {NULL, NULL}; - Py_buffer salt = {NULL, NULL}; - Py_buffer person = {NULL, NULL}; - int fanout = 1; - int depth = 1; - unsigned long leaf_size = 0; - unsigned long long node_offset = 0; - int node_depth = 0; - int inner_size = 0; - int last_node = 0; - int usedforsecurity = 1; - - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - if (nargs < 1) { - goto skip_optional_posonly; - } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: - if (!noptargs) { - goto skip_optional_kwonly; - } - if (fastargs[1]) { - digest_size = PyLong_AsInt(fastargs[1]); - if (digest_size == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[2]) { - if (PyObject_GetBuffer(fastargs[2], &key, PyBUF_SIMPLE) != 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[3]) { - if (PyObject_GetBuffer(fastargs[3], &salt, PyBUF_SIMPLE) != 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[4]) { - if (PyObject_GetBuffer(fastargs[4], &person, PyBUF_SIMPLE) != 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[5]) { - fanout = PyLong_AsInt(fastargs[5]); - if (fanout == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[6]) { - depth = PyLong_AsInt(fastargs[6]); - if (depth == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[7]) { - if (!_PyLong_UnsignedLong_Converter(fastargs[7], &leaf_size)) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[8]) { - if (!_PyLong_UnsignedLongLong_Converter(fastargs[8], &node_offset)) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[9]) { - node_depth = PyLong_AsInt(fastargs[9]); - if (node_depth == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[10]) { - inner_size = PyLong_AsInt(fastargs[10]); - if (inner_size == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[11]) { - last_node = PyObject_IsTrue(fastargs[11]); - if (last_node < 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - usedforsecurity = PyObject_IsTrue(fastargs[12]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); - -exit: - /* Cleanup for key */ - if (key.obj) { - PyBuffer_Release(&key); - } - /* Cleanup for salt */ - if (salt.obj) { - PyBuffer_Release(&salt); - } - /* Cleanup for person */ - if (person.obj) { - PyBuffer_Release(&person); - } - - return return_value; -} - -PyDoc_STRVAR(_blake2_blake2s_copy__doc__, -"copy($self, /)\n" -"--\n" -"\n" -"Return a copy of the hash object."); - -#define _BLAKE2_BLAKE2S_COPY_METHODDEF \ - {"copy", (PyCFunction)_blake2_blake2s_copy, METH_NOARGS, _blake2_blake2s_copy__doc__}, - -static PyObject * -_blake2_blake2s_copy_impl(BLAKE2sObject *self); - -static PyObject * -_blake2_blake2s_copy(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) -{ - return _blake2_blake2s_copy_impl(self); -} - -PyDoc_STRVAR(_blake2_blake2s_update__doc__, -"update($self, data, /)\n" -"--\n" -"\n" -"Update this hash object\'s state with the provided bytes-like object."); - -#define _BLAKE2_BLAKE2S_UPDATE_METHODDEF \ - {"update", (PyCFunction)_blake2_blake2s_update, METH_O, _blake2_blake2s_update__doc__}, - -PyDoc_STRVAR(_blake2_blake2s_digest__doc__, -"digest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a bytes object."); - -#define _BLAKE2_BLAKE2S_DIGEST_METHODDEF \ - {"digest", (PyCFunction)_blake2_blake2s_digest, METH_NOARGS, _blake2_blake2s_digest__doc__}, - -static PyObject * -_blake2_blake2s_digest_impl(BLAKE2sObject *self); - -static PyObject * -_blake2_blake2s_digest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) -{ - return _blake2_blake2s_digest_impl(self); -} - -PyDoc_STRVAR(_blake2_blake2s_hexdigest__doc__, -"hexdigest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a string of hexadecimal digits."); - -#define _BLAKE2_BLAKE2S_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)_blake2_blake2s_hexdigest, METH_NOARGS, _blake2_blake2s_hexdigest__doc__}, - -static PyObject * -_blake2_blake2s_hexdigest_impl(BLAKE2sObject *self); - -static PyObject * -_blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) -{ - return _blake2_blake2s_hexdigest_impl(self); -} -/*[clinic end generated code: output=24690e4e2586cafd input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/impl/blake2-config.h b/Modules/_blake2/impl/blake2-config.h deleted file mode 100644 index c09cb4bcf06723..00000000000000 --- a/Modules/_blake2/impl/blake2-config.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2_CONFIG_H__ -#define __BLAKE2_CONFIG_H__ - -#if defined(__SSE2__) -#define HAVE_SSE2 -#endif - -#if defined(__SSSE3__) -#define HAVE_SSSE3 -#endif - -#if defined(__SSE4_1__) -#define HAVE_SSE4_1 -#endif - -#if defined(__AVX__) -#define HAVE_AVX -#endif - -#if defined(__XOP__) -#define HAVE_XOP -#endif - - -#ifdef HAVE_AVX2 -#ifndef HAVE_AVX -#define HAVE_AVX -#endif -#endif - -#ifdef HAVE_XOP -#ifndef HAVE_AVX -#define HAVE_AVX -#endif -#endif - -#ifdef HAVE_AVX -#ifndef HAVE_SSE4_1 -#define HAVE_SSE4_1 -#endif -#endif - -#ifdef HAVE_SSE4_1 -#ifndef HAVE_SSSE3 -#define HAVE_SSSE3 -#endif -#endif - -#ifdef HAVE_SSSE3 -#define HAVE_SSE2 -#endif - -#if !defined(HAVE_SSE2) -#error "This code requires at least SSE2." -#endif - -#endif - diff --git a/Modules/_blake2/impl/blake2-impl.h b/Modules/_blake2/impl/blake2-impl.h deleted file mode 100644 index 9d2fbb72fc1c03..00000000000000 --- a/Modules/_blake2/impl/blake2-impl.h +++ /dev/null @@ -1,162 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2_IMPL_H__ -#define __BLAKE2_IMPL_H__ - -#if defined(_WIN32) || defined(WIN32) -#include -#endif - -#include -#include -#include - -#define BLAKE2_IMPL_CAT(x,y) x ## y -#define BLAKE2_IMPL_EVAL(x,y) BLAKE2_IMPL_CAT(x,y) -#define BLAKE2_IMPL_NAME(fun) BLAKE2_IMPL_EVAL(fun, SUFFIX) - -static inline uint32_t load32( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint32_t w; - memcpy( &w, src, sizeof( w ) ); - return w; -#else - const uint8_t *p = ( uint8_t * )src; - uint32_t w = *p++; - w |= ( uint32_t )( *p++ ) << 8; - w |= ( uint32_t )( *p++ ) << 16; - w |= ( uint32_t )( *p++ ) << 24; - return w; -#endif -} - -static inline uint64_t load64( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint64_t w; - memcpy( &w, src, sizeof( w ) ); - return w; -#else - const uint8_t *p = ( uint8_t * )src; - uint64_t w = *p++; - w |= ( uint64_t )( *p++ ) << 8; - w |= ( uint64_t )( *p++ ) << 16; - w |= ( uint64_t )( *p++ ) << 24; - w |= ( uint64_t )( *p++ ) << 32; - w |= ( uint64_t )( *p++ ) << 40; - w |= ( uint64_t )( *p++ ) << 48; - w |= ( uint64_t )( *p++ ) << 56; - return w; -#endif -} - -static inline void store32( void *dst, uint32_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy( dst, &w, sizeof( w ) ); -#else - uint8_t *p = ( uint8_t * )dst; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; -#endif -} - -static inline void store64( void *dst, uint64_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy( dst, &w, sizeof( w ) ); -#else - uint8_t *p = ( uint8_t * )dst; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; -#endif -} - -static inline uint64_t load48( const void *src ) -{ - const uint8_t *p = ( const uint8_t * )src; - uint64_t w = *p++; - w |= ( uint64_t )( *p++ ) << 8; - w |= ( uint64_t )( *p++ ) << 16; - w |= ( uint64_t )( *p++ ) << 24; - w |= ( uint64_t )( *p++ ) << 32; - w |= ( uint64_t )( *p++ ) << 40; - return w; -} - -static inline void store48( void *dst, uint64_t w ) -{ - uint8_t *p = ( uint8_t * )dst; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; -} - -static inline uint32_t rotl32( const uint32_t w, const unsigned c ) -{ - return ( w << c ) | ( w >> ( 32 - c ) ); -} - -static inline uint64_t rotl64( const uint64_t w, const unsigned c ) -{ - return ( w << c ) | ( w >> ( 64 - c ) ); -} - -static inline uint32_t rotr32( const uint32_t w, const unsigned c ) -{ - return ( w >> c ) | ( w << ( 32 - c ) ); -} - -static inline uint64_t rotr64( const uint64_t w, const unsigned c ) -{ - return ( w >> c ) | ( w << ( 64 - c ) ); -} - -/* prevents compiler optimizing out memset() */ -static inline void secure_zero_memory(void *v, size_t n) -{ -#if defined(_WIN32) || defined(WIN32) - SecureZeroMemory(v, n); -#elif defined(__hpux) - static void *(*const volatile memset_v)(void *, int, size_t) = &memset; - memset_v(v, 0, n); -#else -// prioritize first the general C11 call -#if defined(HAVE_MEMSET_S) - memset_s(v, n, 0, n); -#elif defined(HAVE_EXPLICIT_BZERO) - explicit_bzero(v, n); -#elif defined(HAVE_EXPLICIT_MEMSET) - explicit_memset(v, 0, n); -#else - memset(v, 0, n); - __asm__ __volatile__("" :: "r"(v) : "memory"); -#endif -#endif -} - -#endif - diff --git a/Modules/_blake2/impl/blake2.h b/Modules/_blake2/impl/blake2.h deleted file mode 100644 index a08d82efefe09f..00000000000000 --- a/Modules/_blake2/impl/blake2.h +++ /dev/null @@ -1,177 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2_H__ -#define __BLAKE2_H__ - -#include -#include - -#if defined(_WIN32) || defined(__CYGWIN__) - #define BLAKE2_DLL_IMPORT __declspec(dllimport) - #define BLAKE2_DLL_EXPORT __declspec(dllexport) - #define BLAKE2_DLL_PRIVATE -#elif __GNUC__ >= 4 - #define BLAKE2_DLL_IMPORT __attribute__ ((visibility ("default"))) - #define BLAKE2_DLL_EXPORT __attribute__ ((visibility ("default"))) - #define BLAKE2_DLL_PRIVATE __attribute__ ((visibility ("hidden"))) -#else - #define BLAKE2_DLL_IMPORT - #define BLAKE2_DLL_EXPORT - #define BLAKE2_DLL_PRIVATE -#endif - -#if defined(BLAKE2_DLL) - #if defined(BLAKE2_DLL_EXPORTS) // defined if we are building the DLL - #define BLAKE2_API BLAKE2_DLL_EXPORT - #else - #define BLAKE2_API BLAKE2_DLL_IMPORT - #endif - #define BLAKE2_PRIVATE BLAKE2_DLL_PRIVATE // must only be used by hidden logic -#else - #define BLAKE2_API - #define BLAKE2_PRIVATE -#endif - -#if defined(__cplusplus) -extern "C" { -#elif defined(_MSC_VER) && !defined(inline) -#define inline __inline -#endif - - enum blake2s_constant - { - BLAKE2S_BLOCKBYTES = 64, - BLAKE2S_OUTBYTES = 32, - BLAKE2S_KEYBYTES = 32, - BLAKE2S_SALTBYTES = 8, - BLAKE2S_PERSONALBYTES = 8 - }; - - enum blake2b_constant - { - BLAKE2B_BLOCKBYTES = 128, - BLAKE2B_OUTBYTES = 64, - BLAKE2B_KEYBYTES = 64, - BLAKE2B_SALTBYTES = 16, - BLAKE2B_PERSONALBYTES = 16 - }; - -#pragma pack(push, 1) - typedef struct __blake2s_param - { - uint8_t digest_length; // 1 - uint8_t key_length; // 2 - uint8_t fanout; // 3 - uint8_t depth; // 4 - uint32_t leaf_length; // 8 - uint8_t node_offset[6];// 14 - uint8_t node_depth; // 15 - uint8_t inner_length; // 16 - // uint8_t reserved[0]; - uint8_t salt[BLAKE2S_SALTBYTES]; // 24 - uint8_t personal[BLAKE2S_PERSONALBYTES]; // 32 - } blake2s_param; - - typedef struct __blake2s_state - { - uint32_t h[8]; - uint32_t t[2]; - uint32_t f[2]; - uint8_t buf[2 * BLAKE2S_BLOCKBYTES]; - uint32_t buflen; - uint8_t outlen; - uint8_t last_node; - } blake2s_state; - - typedef struct __blake2b_param - { - uint8_t digest_length; // 1 - uint8_t key_length; // 2 - uint8_t fanout; // 3 - uint8_t depth; // 4 - uint32_t leaf_length; // 8 - uint64_t node_offset; // 16 - uint8_t node_depth; // 17 - uint8_t inner_length; // 18 - uint8_t reserved[14]; // 32 - uint8_t salt[BLAKE2B_SALTBYTES]; // 48 - uint8_t personal[BLAKE2B_PERSONALBYTES]; // 64 - } blake2b_param; - - typedef struct __blake2b_state - { - uint64_t h[8]; - uint64_t t[2]; - uint64_t f[2]; - uint8_t buf[2 * BLAKE2B_BLOCKBYTES]; - uint32_t buflen; - uint8_t outlen; - uint8_t last_node; - } blake2b_state; - - typedef struct __blake2sp_state - { - blake2s_state S[8][1]; - blake2s_state R[1]; - uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; - uint32_t buflen; - uint8_t outlen; - } blake2sp_state; - - typedef struct __blake2bp_state - { - blake2b_state S[4][1]; - blake2b_state R[1]; - uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; - uint32_t buflen; - uint8_t outlen; - } blake2bp_state; -#pragma pack(pop) - - // Streaming API - BLAKE2_API int blake2s_init( blake2s_state *S, size_t outlen ); - BLAKE2_API int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); - BLAKE2_API int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); - BLAKE2_API int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ); - BLAKE2_API int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ); - - BLAKE2_API int blake2b_init( blake2b_state *S, size_t outlen ); - BLAKE2_API int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); - BLAKE2_API int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); - BLAKE2_API int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ); - BLAKE2_API int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ); - - BLAKE2_API int blake2sp_init( blake2sp_state *S, size_t outlen ); - BLAKE2_API int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); - BLAKE2_API int blake2sp_update( blake2sp_state *S, const uint8_t *in, size_t inlen ); - BLAKE2_API int blake2sp_final( blake2sp_state *S, uint8_t *out, size_t outlen ); - - BLAKE2_API int blake2bp_init( blake2bp_state *S, size_t outlen ); - BLAKE2_API int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); - BLAKE2_API int blake2bp_update( blake2bp_state *S, const uint8_t *in, size_t inlen ); - BLAKE2_API int blake2bp_final( blake2bp_state *S, uint8_t *out, size_t outlen ); - - // Simple API - BLAKE2_API int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); - BLAKE2_API int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); - - BLAKE2_API int blake2sp( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); - BLAKE2_API int blake2bp( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); - -#if defined(__cplusplus) -} -#endif - -#endif - diff --git a/Modules/_blake2/impl/blake2b-load-sse2.h b/Modules/_blake2/impl/blake2b-load-sse2.h deleted file mode 100644 index 1ba153c87d7352..00000000000000 --- a/Modules/_blake2/impl/blake2b-load-sse2.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2B_LOAD_SSE2_H__ -#define __BLAKE2B_LOAD_SSE2_H__ - -#define LOAD_MSG_0_1(b0, b1) b0 = _mm_set_epi64x(m2, m0); b1 = _mm_set_epi64x(m6, m4) -#define LOAD_MSG_0_2(b0, b1) b0 = _mm_set_epi64x(m3, m1); b1 = _mm_set_epi64x(m7, m5) -#define LOAD_MSG_0_3(b0, b1) b0 = _mm_set_epi64x(m10, m8); b1 = _mm_set_epi64x(m14, m12) -#define LOAD_MSG_0_4(b0, b1) b0 = _mm_set_epi64x(m11, m9); b1 = _mm_set_epi64x(m15, m13) -#define LOAD_MSG_1_1(b0, b1) b0 = _mm_set_epi64x(m4, m14); b1 = _mm_set_epi64x(m13, m9) -#define LOAD_MSG_1_2(b0, b1) b0 = _mm_set_epi64x(m8, m10); b1 = _mm_set_epi64x(m6, m15) -#define LOAD_MSG_1_3(b0, b1) b0 = _mm_set_epi64x(m0, m1); b1 = _mm_set_epi64x(m5, m11) -#define LOAD_MSG_1_4(b0, b1) b0 = _mm_set_epi64x(m2, m12); b1 = _mm_set_epi64x(m3, m7) -#define LOAD_MSG_2_1(b0, b1) b0 = _mm_set_epi64x(m12, m11); b1 = _mm_set_epi64x(m15, m5) -#define LOAD_MSG_2_2(b0, b1) b0 = _mm_set_epi64x(m0, m8); b1 = _mm_set_epi64x(m13, m2) -#define LOAD_MSG_2_3(b0, b1) b0 = _mm_set_epi64x(m3, m10); b1 = _mm_set_epi64x(m9, m7) -#define LOAD_MSG_2_4(b0, b1) b0 = _mm_set_epi64x(m6, m14); b1 = _mm_set_epi64x(m4, m1) -#define LOAD_MSG_3_1(b0, b1) b0 = _mm_set_epi64x(m3, m7); b1 = _mm_set_epi64x(m11, m13) -#define LOAD_MSG_3_2(b0, b1) b0 = _mm_set_epi64x(m1, m9); b1 = _mm_set_epi64x(m14, m12) -#define LOAD_MSG_3_3(b0, b1) b0 = _mm_set_epi64x(m5, m2); b1 = _mm_set_epi64x(m15, m4) -#define LOAD_MSG_3_4(b0, b1) b0 = _mm_set_epi64x(m10, m6); b1 = _mm_set_epi64x(m8, m0) -#define LOAD_MSG_4_1(b0, b1) b0 = _mm_set_epi64x(m5, m9); b1 = _mm_set_epi64x(m10, m2) -#define LOAD_MSG_4_2(b0, b1) b0 = _mm_set_epi64x(m7, m0); b1 = _mm_set_epi64x(m15, m4) -#define LOAD_MSG_4_3(b0, b1) b0 = _mm_set_epi64x(m11, m14); b1 = _mm_set_epi64x(m3, m6) -#define LOAD_MSG_4_4(b0, b1) b0 = _mm_set_epi64x(m12, m1); b1 = _mm_set_epi64x(m13, m8) -#define LOAD_MSG_5_1(b0, b1) b0 = _mm_set_epi64x(m6, m2); b1 = _mm_set_epi64x(m8, m0) -#define LOAD_MSG_5_2(b0, b1) b0 = _mm_set_epi64x(m10, m12); b1 = _mm_set_epi64x(m3, m11) -#define LOAD_MSG_5_3(b0, b1) b0 = _mm_set_epi64x(m7, m4); b1 = _mm_set_epi64x(m1, m15) -#define LOAD_MSG_5_4(b0, b1) b0 = _mm_set_epi64x(m5, m13); b1 = _mm_set_epi64x(m9, m14) -#define LOAD_MSG_6_1(b0, b1) b0 = _mm_set_epi64x(m1, m12); b1 = _mm_set_epi64x(m4, m14) -#define LOAD_MSG_6_2(b0, b1) b0 = _mm_set_epi64x(m15, m5); b1 = _mm_set_epi64x(m10, m13) -#define LOAD_MSG_6_3(b0, b1) b0 = _mm_set_epi64x(m6, m0); b1 = _mm_set_epi64x(m8, m9) -#define LOAD_MSG_6_4(b0, b1) b0 = _mm_set_epi64x(m3, m7); b1 = _mm_set_epi64x(m11, m2) -#define LOAD_MSG_7_1(b0, b1) b0 = _mm_set_epi64x(m7, m13); b1 = _mm_set_epi64x(m3, m12) -#define LOAD_MSG_7_2(b0, b1) b0 = _mm_set_epi64x(m14, m11); b1 = _mm_set_epi64x(m9, m1) -#define LOAD_MSG_7_3(b0, b1) b0 = _mm_set_epi64x(m15, m5); b1 = _mm_set_epi64x(m2, m8) -#define LOAD_MSG_7_4(b0, b1) b0 = _mm_set_epi64x(m4, m0); b1 = _mm_set_epi64x(m10, m6) -#define LOAD_MSG_8_1(b0, b1) b0 = _mm_set_epi64x(m14, m6); b1 = _mm_set_epi64x(m0, m11) -#define LOAD_MSG_8_2(b0, b1) b0 = _mm_set_epi64x(m9, m15); b1 = _mm_set_epi64x(m8, m3) -#define LOAD_MSG_8_3(b0, b1) b0 = _mm_set_epi64x(m13, m12); b1 = _mm_set_epi64x(m10, m1) -#define LOAD_MSG_8_4(b0, b1) b0 = _mm_set_epi64x(m7, m2); b1 = _mm_set_epi64x(m5, m4) -#define LOAD_MSG_9_1(b0, b1) b0 = _mm_set_epi64x(m8, m10); b1 = _mm_set_epi64x(m1, m7) -#define LOAD_MSG_9_2(b0, b1) b0 = _mm_set_epi64x(m4, m2); b1 = _mm_set_epi64x(m5, m6) -#define LOAD_MSG_9_3(b0, b1) b0 = _mm_set_epi64x(m9, m15); b1 = _mm_set_epi64x(m13, m3) -#define LOAD_MSG_9_4(b0, b1) b0 = _mm_set_epi64x(m14, m11); b1 = _mm_set_epi64x(m0, m12) -#define LOAD_MSG_10_1(b0, b1) b0 = _mm_set_epi64x(m2, m0); b1 = _mm_set_epi64x(m6, m4) -#define LOAD_MSG_10_2(b0, b1) b0 = _mm_set_epi64x(m3, m1); b1 = _mm_set_epi64x(m7, m5) -#define LOAD_MSG_10_3(b0, b1) b0 = _mm_set_epi64x(m10, m8); b1 = _mm_set_epi64x(m14, m12) -#define LOAD_MSG_10_4(b0, b1) b0 = _mm_set_epi64x(m11, m9); b1 = _mm_set_epi64x(m15, m13) -#define LOAD_MSG_11_1(b0, b1) b0 = _mm_set_epi64x(m4, m14); b1 = _mm_set_epi64x(m13, m9) -#define LOAD_MSG_11_2(b0, b1) b0 = _mm_set_epi64x(m8, m10); b1 = _mm_set_epi64x(m6, m15) -#define LOAD_MSG_11_3(b0, b1) b0 = _mm_set_epi64x(m0, m1); b1 = _mm_set_epi64x(m5, m11) -#define LOAD_MSG_11_4(b0, b1) b0 = _mm_set_epi64x(m2, m12); b1 = _mm_set_epi64x(m3, m7) - - -#endif - diff --git a/Modules/_blake2/impl/blake2b-load-sse41.h b/Modules/_blake2/impl/blake2b-load-sse41.h deleted file mode 100644 index f6c1bc8393f167..00000000000000 --- a/Modules/_blake2/impl/blake2b-load-sse41.h +++ /dev/null @@ -1,402 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2B_LOAD_SSE41_H__ -#define __BLAKE2B_LOAD_SSE41_H__ - -#define LOAD_MSG_0_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m0, m1); \ -b1 = _mm_unpacklo_epi64(m2, m3); \ -} while(0) - - -#define LOAD_MSG_0_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m0, m1); \ -b1 = _mm_unpackhi_epi64(m2, m3); \ -} while(0) - - -#define LOAD_MSG_0_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m4, m5); \ -b1 = _mm_unpacklo_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_0_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m4, m5); \ -b1 = _mm_unpackhi_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_1_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m7, m2); \ -b1 = _mm_unpackhi_epi64(m4, m6); \ -} while(0) - - -#define LOAD_MSG_1_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m5, m4); \ -b1 = _mm_alignr_epi8(m3, m7, 8); \ -} while(0) - - -#define LOAD_MSG_1_3(b0, b1) \ -do \ -{ \ -b0 = _mm_shuffle_epi32(m0, _MM_SHUFFLE(1,0,3,2)); \ -b1 = _mm_unpackhi_epi64(m5, m2); \ -} while(0) - - -#define LOAD_MSG_1_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m6, m1); \ -b1 = _mm_unpackhi_epi64(m3, m1); \ -} while(0) - - -#define LOAD_MSG_2_1(b0, b1) \ -do \ -{ \ -b0 = _mm_alignr_epi8(m6, m5, 8); \ -b1 = _mm_unpackhi_epi64(m2, m7); \ -} while(0) - - -#define LOAD_MSG_2_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m4, m0); \ -b1 = _mm_blend_epi16(m1, m6, 0xF0); \ -} while(0) - - -#define LOAD_MSG_2_3(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m5, m1, 0xF0); \ -b1 = _mm_unpackhi_epi64(m3, m4); \ -} while(0) - - -#define LOAD_MSG_2_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m7, m3); \ -b1 = _mm_alignr_epi8(m2, m0, 8); \ -} while(0) - - -#define LOAD_MSG_3_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m3, m1); \ -b1 = _mm_unpackhi_epi64(m6, m5); \ -} while(0) - - -#define LOAD_MSG_3_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m4, m0); \ -b1 = _mm_unpacklo_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_3_3(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m1, m2, 0xF0); \ -b1 = _mm_blend_epi16(m2, m7, 0xF0); \ -} while(0) - - -#define LOAD_MSG_3_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m3, m5); \ -b1 = _mm_unpacklo_epi64(m0, m4); \ -} while(0) - - -#define LOAD_MSG_4_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m4, m2); \ -b1 = _mm_unpacklo_epi64(m1, m5); \ -} while(0) - - -#define LOAD_MSG_4_2(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m0, m3, 0xF0); \ -b1 = _mm_blend_epi16(m2, m7, 0xF0); \ -} while(0) - - -#define LOAD_MSG_4_3(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m7, m5, 0xF0); \ -b1 = _mm_blend_epi16(m3, m1, 0xF0); \ -} while(0) - - -#define LOAD_MSG_4_4(b0, b1) \ -do \ -{ \ -b0 = _mm_alignr_epi8(m6, m0, 8); \ -b1 = _mm_blend_epi16(m4, m6, 0xF0); \ -} while(0) - - -#define LOAD_MSG_5_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m1, m3); \ -b1 = _mm_unpacklo_epi64(m0, m4); \ -} while(0) - - -#define LOAD_MSG_5_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m6, m5); \ -b1 = _mm_unpackhi_epi64(m5, m1); \ -} while(0) - - -#define LOAD_MSG_5_3(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m2, m3, 0xF0); \ -b1 = _mm_unpackhi_epi64(m7, m0); \ -} while(0) - - -#define LOAD_MSG_5_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m6, m2); \ -b1 = _mm_blend_epi16(m7, m4, 0xF0); \ -} while(0) - - -#define LOAD_MSG_6_1(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m6, m0, 0xF0); \ -b1 = _mm_unpacklo_epi64(m7, m2); \ -} while(0) - - -#define LOAD_MSG_6_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m2, m7); \ -b1 = _mm_alignr_epi8(m5, m6, 8); \ -} while(0) - - -#define LOAD_MSG_6_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m0, m3); \ -b1 = _mm_shuffle_epi32(m4, _MM_SHUFFLE(1,0,3,2)); \ -} while(0) - - -#define LOAD_MSG_6_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m3, m1); \ -b1 = _mm_blend_epi16(m1, m5, 0xF0); \ -} while(0) - - -#define LOAD_MSG_7_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m6, m3); \ -b1 = _mm_blend_epi16(m6, m1, 0xF0); \ -} while(0) - - -#define LOAD_MSG_7_2(b0, b1) \ -do \ -{ \ -b0 = _mm_alignr_epi8(m7, m5, 8); \ -b1 = _mm_unpackhi_epi64(m0, m4); \ -} while(0) - - -#define LOAD_MSG_7_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m2, m7); \ -b1 = _mm_unpacklo_epi64(m4, m1); \ -} while(0) - - -#define LOAD_MSG_7_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m0, m2); \ -b1 = _mm_unpacklo_epi64(m3, m5); \ -} while(0) - - -#define LOAD_MSG_8_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m3, m7); \ -b1 = _mm_alignr_epi8(m0, m5, 8); \ -} while(0) - - -#define LOAD_MSG_8_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m7, m4); \ -b1 = _mm_alignr_epi8(m4, m1, 8); \ -} while(0) - - -#define LOAD_MSG_8_3(b0, b1) \ -do \ -{ \ -b0 = m6; \ -b1 = _mm_alignr_epi8(m5, m0, 8); \ -} while(0) - - -#define LOAD_MSG_8_4(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m1, m3, 0xF0); \ -b1 = m2; \ -} while(0) - - -#define LOAD_MSG_9_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m5, m4); \ -b1 = _mm_unpackhi_epi64(m3, m0); \ -} while(0) - - -#define LOAD_MSG_9_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m1, m2); \ -b1 = _mm_blend_epi16(m3, m2, 0xF0); \ -} while(0) - - -#define LOAD_MSG_9_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m7, m4); \ -b1 = _mm_unpackhi_epi64(m1, m6); \ -} while(0) - - -#define LOAD_MSG_9_4(b0, b1) \ -do \ -{ \ -b0 = _mm_alignr_epi8(m7, m5, 8); \ -b1 = _mm_unpacklo_epi64(m6, m0); \ -} while(0) - - -#define LOAD_MSG_10_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m0, m1); \ -b1 = _mm_unpacklo_epi64(m2, m3); \ -} while(0) - - -#define LOAD_MSG_10_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m0, m1); \ -b1 = _mm_unpackhi_epi64(m2, m3); \ -} while(0) - - -#define LOAD_MSG_10_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m4, m5); \ -b1 = _mm_unpacklo_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_10_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m4, m5); \ -b1 = _mm_unpackhi_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_11_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m7, m2); \ -b1 = _mm_unpackhi_epi64(m4, m6); \ -} while(0) - - -#define LOAD_MSG_11_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m5, m4); \ -b1 = _mm_alignr_epi8(m3, m7, 8); \ -} while(0) - - -#define LOAD_MSG_11_3(b0, b1) \ -do \ -{ \ -b0 = _mm_shuffle_epi32(m0, _MM_SHUFFLE(1,0,3,2)); \ -b1 = _mm_unpackhi_epi64(m5, m2); \ -} while(0) - - -#define LOAD_MSG_11_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m6, m1); \ -b1 = _mm_unpackhi_epi64(m3, m1); \ -} while(0) - - -#endif - diff --git a/Modules/_blake2/impl/blake2b-ref.c b/Modules/_blake2/impl/blake2b-ref.c deleted file mode 100644 index e58c43659d9cc8..00000000000000 --- a/Modules/_blake2/impl/blake2b-ref.c +++ /dev/null @@ -1,379 +0,0 @@ -/* - BLAKE2 reference source code package - reference C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ - -#include -#include -#include - -#include "blake2.h" -#include "blake2-impl.h" - -static const uint64_t blake2b_IV[8] = -{ - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - -static const uint8_t blake2b_sigma[12][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } -}; - - -static inline int blake2b_set_lastnode( blake2b_state *S ) -{ - S->f[1] = ~0ULL; - return 0; -} - -static inline int blake2b_clear_lastnode( blake2b_state *S ) -{ - S->f[1] = 0ULL; - return 0; -} - -/* Some helper functions, not necessarily useful */ -static inline int blake2b_set_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_set_lastnode( S ); - - S->f[0] = ~0ULL; - return 0; -} - -static inline int blake2b_clear_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_clear_lastnode( S ); - - S->f[0] = 0ULL; - return 0; -} - -static inline int blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) -{ - S->t[0] += inc; - S->t[1] += ( S->t[0] < inc ); - return 0; -} - - - -// Parameter-related functions -static inline int blake2b_param_set_digest_length( blake2b_param *P, const uint8_t digest_length ) -{ - P->digest_length = digest_length; - return 0; -} - -static inline int blake2b_param_set_fanout( blake2b_param *P, const uint8_t fanout ) -{ - P->fanout = fanout; - return 0; -} - -static inline int blake2b_param_set_max_depth( blake2b_param *P, const uint8_t depth ) -{ - P->depth = depth; - return 0; -} - -static inline int blake2b_param_set_leaf_length( blake2b_param *P, const uint32_t leaf_length ) -{ - store32( &P->leaf_length, leaf_length ); - return 0; -} - -static inline int blake2b_param_set_node_offset( blake2b_param *P, const uint64_t node_offset ) -{ - store64( &P->node_offset, node_offset ); - return 0; -} - -static inline int blake2b_param_set_node_depth( blake2b_param *P, const uint8_t node_depth ) -{ - P->node_depth = node_depth; - return 0; -} - -static inline int blake2b_param_set_inner_length( blake2b_param *P, const uint8_t inner_length ) -{ - P->inner_length = inner_length; - return 0; -} - -static inline int blake2b_param_set_salt( blake2b_param *P, const uint8_t salt[BLAKE2B_SALTBYTES] ) -{ - memcpy( P->salt, salt, BLAKE2B_SALTBYTES ); - return 0; -} - -static inline int blake2b_param_set_personal( blake2b_param *P, const uint8_t personal[BLAKE2B_PERSONALBYTES] ) -{ - memcpy( P->personal, personal, BLAKE2B_PERSONALBYTES ); - return 0; -} - -static inline int blake2b_init0( blake2b_state *S ) -{ - memset( S, 0, sizeof( blake2b_state ) ); - - for( int i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; - - return 0; -} - -#if defined(__cplusplus) -extern "C" { -#endif - int blake2b_init( blake2b_state *S, size_t outlen ); - int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); - int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ); - int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ); - int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); -#if defined(__cplusplus) -} -#endif - -/* init xors IV with input parameter block */ -int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) -{ - blake2b_init0( S ); - uint8_t *p = ( uint8_t * )( P ); - - /* IV XOR ParamBlock */ - for( size_t i = 0; i < 8; ++i ) - S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); - - S->outlen = P->digest_length; - return 0; -} - - - -int blake2b_init( blake2b_state *S, size_t outlen ) -{ - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - P->digest_length = ( uint8_t ) outlen; - P->key_length = 0; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store64( &P->node_offset, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - return blake2b_init_param( S, P ); -} - - -int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) -{ - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; - - P->digest_length = ( uint8_t ) outlen; - P->key_length = ( uint8_t ) keylen; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store64( &P->node_offset, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - - if( blake2b_init_param( S, P ) < 0 ) return -1; - - { - uint8_t block[BLAKE2B_BLOCKBYTES]; - memset( block, 0, BLAKE2B_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - -static int blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) -{ - uint64_t m[16]; - uint64_t v[16]; - size_t i; - - for( i = 0; i < 16; ++i ) - m[i] = load64( block + i * sizeof( m[i] ) ); - - for( i = 0; i < 8; ++i ) - v[i] = S->h[i]; - - v[ 8] = blake2b_IV[0]; - v[ 9] = blake2b_IV[1]; - v[10] = blake2b_IV[2]; - v[11] = blake2b_IV[3]; - v[12] = S->t[0] ^ blake2b_IV[4]; - v[13] = S->t[1] ^ blake2b_IV[5]; - v[14] = S->f[0] ^ blake2b_IV[6]; - v[15] = S->f[1] ^ blake2b_IV[7]; -#define G(r,i,a,b,c,d) \ - do { \ - a = a + b + m[blake2b_sigma[r][2*i+0]]; \ - d = rotr64(d ^ a, 32); \ - c = c + d; \ - b = rotr64(b ^ c, 24); \ - a = a + b + m[blake2b_sigma[r][2*i+1]]; \ - d = rotr64(d ^ a, 16); \ - c = c + d; \ - b = rotr64(b ^ c, 63); \ - } while(0) -#define ROUND(r) \ - do { \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ - } while(0) - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - ROUND( 10 ); - ROUND( 11 ); - - for( i = 0; i < 8; ++i ) - S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; - -#undef G -#undef ROUND - return 0; -} - - -int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ) -{ - while( inlen > 0 ) - { - uint32_t left = S->buflen; - uint32_t fill = 2 * BLAKE2B_BLOCKBYTES - left; - - if( inlen > fill ) - { - memcpy( S->buf + left, in, fill ); // Fill buffer - S->buflen += fill; - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); // Compress - memcpy( S->buf, S->buf + BLAKE2B_BLOCKBYTES, BLAKE2B_BLOCKBYTES ); // Shift buffer left - S->buflen -= BLAKE2B_BLOCKBYTES; - in += fill; - inlen -= fill; - } - else // inlen <= fill - { - memcpy( S->buf + left, in, inlen ); - S->buflen += ( uint32_t ) inlen; // Be lazy, do not compress - in += inlen; - inlen -= inlen; - } - } - - return 0; -} - -int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ) -{ - uint8_t buffer[BLAKE2B_OUTBYTES]; - size_t i; - - if(S->outlen != outlen) return -1; - - if( S->buflen > BLAKE2B_BLOCKBYTES ) - { - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); - S->buflen -= BLAKE2B_BLOCKBYTES; - memmove( S->buf, S->buf + BLAKE2B_BLOCKBYTES, S->buflen ); - } - - blake2b_increment_counter( S, S->buflen ); - blake2b_set_lastblock( S ); - memset( S->buf + S->buflen, 0, 2 * BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ - blake2b_compress( S, S->buf ); - - for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ - store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); - - memcpy( out, buffer, outlen ); - return 0; -} - -int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) -{ - blake2b_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if( NULL == key && keylen > 0 ) return -1; - - if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; - - if( keylen > BLAKE2B_KEYBYTES ) return -1; - - if( keylen > 0 ) - { - if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2b_init( S, outlen ) < 0 ) return -1; - } - - if( blake2b_update( S, ( uint8_t * )in, inlen ) < 0 ) return -1; - return blake2b_final( S, out, outlen ); -} - - diff --git a/Modules/_blake2/impl/blake2b-round.h b/Modules/_blake2/impl/blake2b-round.h deleted file mode 100644 index 5b452c4d63babe..00000000000000 --- a/Modules/_blake2/impl/blake2b-round.h +++ /dev/null @@ -1,160 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2B_ROUND_H__ -#define __BLAKE2B_ROUND_H__ - -#define LOAD(p) _mm_load_si128( (__m128i *)(p) ) -#define STORE(p,r) _mm_store_si128((__m128i *)(p), r) - -#define LOADU(p) _mm_loadu_si128( (__m128i *)(p) ) -#define STOREU(p,r) _mm_storeu_si128((__m128i *)(p), r) - -#define TOF(reg) _mm_castsi128_ps((reg)) -#define TOI(reg) _mm_castps_si128((reg)) - -#define LIKELY(x) __builtin_expect((x),1) - - -/* Microarchitecture-specific macros */ -#ifndef HAVE_XOP -#ifdef HAVE_SSSE3 -#define _mm_roti_epi64(x, c) \ - (-(c) == 32) ? _mm_shuffle_epi32((x), _MM_SHUFFLE(2,3,0,1)) \ - : (-(c) == 24) ? _mm_shuffle_epi8((x), r24) \ - : (-(c) == 16) ? _mm_shuffle_epi8((x), r16) \ - : (-(c) == 63) ? _mm_xor_si128(_mm_srli_epi64((x), -(c)), _mm_add_epi64((x), (x))) \ - : _mm_xor_si128(_mm_srli_epi64((x), -(c)), _mm_slli_epi64((x), 64-(-(c)))) -#else -#define _mm_roti_epi64(r, c) _mm_xor_si128(_mm_srli_epi64( (r), -(c) ),_mm_slli_epi64( (r), 64-(-(c)) )) -#endif -#else -/* ... */ -#endif - - - -#define G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ - row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ - row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ - \ - row4l = _mm_xor_si128(row4l, row1l); \ - row4h = _mm_xor_si128(row4h, row1h); \ - \ - row4l = _mm_roti_epi64(row4l, -32); \ - row4h = _mm_roti_epi64(row4h, -32); \ - \ - row3l = _mm_add_epi64(row3l, row4l); \ - row3h = _mm_add_epi64(row3h, row4h); \ - \ - row2l = _mm_xor_si128(row2l, row3l); \ - row2h = _mm_xor_si128(row2h, row3h); \ - \ - row2l = _mm_roti_epi64(row2l, -24); \ - row2h = _mm_roti_epi64(row2h, -24); \ - -#define G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ - row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ - row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ - \ - row4l = _mm_xor_si128(row4l, row1l); \ - row4h = _mm_xor_si128(row4h, row1h); \ - \ - row4l = _mm_roti_epi64(row4l, -16); \ - row4h = _mm_roti_epi64(row4h, -16); \ - \ - row3l = _mm_add_epi64(row3l, row4l); \ - row3h = _mm_add_epi64(row3h, row4h); \ - \ - row2l = _mm_xor_si128(row2l, row3l); \ - row2h = _mm_xor_si128(row2h, row3h); \ - \ - row2l = _mm_roti_epi64(row2l, -63); \ - row2h = _mm_roti_epi64(row2h, -63); \ - -#if defined(HAVE_SSSE3) -#define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ - t0 = _mm_alignr_epi8(row2h, row2l, 8); \ - t1 = _mm_alignr_epi8(row2l, row2h, 8); \ - row2l = t0; \ - row2h = t1; \ - \ - t0 = row3l; \ - row3l = row3h; \ - row3h = t0; \ - \ - t0 = _mm_alignr_epi8(row4h, row4l, 8); \ - t1 = _mm_alignr_epi8(row4l, row4h, 8); \ - row4l = t1; \ - row4h = t0; - -#define UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ - t0 = _mm_alignr_epi8(row2l, row2h, 8); \ - t1 = _mm_alignr_epi8(row2h, row2l, 8); \ - row2l = t0; \ - row2h = t1; \ - \ - t0 = row3l; \ - row3l = row3h; \ - row3h = t0; \ - \ - t0 = _mm_alignr_epi8(row4l, row4h, 8); \ - t1 = _mm_alignr_epi8(row4h, row4l, 8); \ - row4l = t1; \ - row4h = t0; -#else - -#define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ - t0 = row4l;\ - t1 = row2l;\ - row4l = row3l;\ - row3l = row3h;\ - row3h = row4l;\ - row4l = _mm_unpackhi_epi64(row4h, _mm_unpacklo_epi64(t0, t0)); \ - row4h = _mm_unpackhi_epi64(t0, _mm_unpacklo_epi64(row4h, row4h)); \ - row2l = _mm_unpackhi_epi64(row2l, _mm_unpacklo_epi64(row2h, row2h)); \ - row2h = _mm_unpackhi_epi64(row2h, _mm_unpacklo_epi64(t1, t1)) - -#define UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ - t0 = row3l;\ - row3l = row3h;\ - row3h = t0;\ - t0 = row2l;\ - t1 = row4l;\ - row2l = _mm_unpackhi_epi64(row2h, _mm_unpacklo_epi64(row2l, row2l)); \ - row2h = _mm_unpackhi_epi64(t0, _mm_unpacklo_epi64(row2h, row2h)); \ - row4l = _mm_unpackhi_epi64(row4l, _mm_unpacklo_epi64(row4h, row4h)); \ - row4h = _mm_unpackhi_epi64(row4h, _mm_unpacklo_epi64(t1, t1)) - -#endif - -#if defined(HAVE_SSE4_1) -#include "blake2b-load-sse41.h" -#else -#include "blake2b-load-sse2.h" -#endif - -#define ROUND(r) \ - LOAD_MSG_ ##r ##_1(b0, b1); \ - G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ - LOAD_MSG_ ##r ##_2(b0, b1); \ - G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ - DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h); \ - LOAD_MSG_ ##r ##_3(b0, b1); \ - G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ - LOAD_MSG_ ##r ##_4(b0, b1); \ - G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ - UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h); - -#endif - diff --git a/Modules/_blake2/impl/blake2b.c b/Modules/_blake2/impl/blake2b.c deleted file mode 100644 index c1068e8640546a..00000000000000 --- a/Modules/_blake2/impl/blake2b.c +++ /dev/null @@ -1,436 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ - -#include -#include -#include - -#include "blake2.h" -#include "blake2-impl.h" - -#include "blake2-config.h" - -#if defined(_MSC_VER) -#include -#endif - -#if defined(HAVE_SSE2) -#include -// MSVC only defines _mm_set_epi64x for x86_64... -#if defined(_MSC_VER) && !defined(_M_X64) -static inline __m128i _mm_set_epi64x( const uint64_t u1, const uint64_t u0 ) -{ - return _mm_set_epi32( u1 >> 32, u1, u0 >> 32, u0 ); -} -#endif -#endif - -#if defined(HAVE_SSSE3) -#include -#endif -#if defined(HAVE_SSE4_1) -#include -#endif -#if defined(HAVE_AVX) -#include -#endif -#if defined(HAVE_XOP) && !defined(_MSC_VER) -#include -#endif - - - -#include "blake2b-round.h" - -static const uint64_t blake2b_IV[8] = -{ - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - -static const uint8_t blake2b_sigma[12][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } -}; - - -/* Some helper functions, not necessarily useful */ -static inline int blake2b_set_lastnode( blake2b_state *S ) -{ - S->f[1] = ~0ULL; - return 0; -} - -static inline int blake2b_clear_lastnode( blake2b_state *S ) -{ - S->f[1] = 0ULL; - return 0; -} - -static inline int blake2b_set_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_set_lastnode( S ); - - S->f[0] = ~0ULL; - return 0; -} - -static inline int blake2b_clear_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_clear_lastnode( S ); - - S->f[0] = 0ULL; - return 0; -} - - -static inline int blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) -{ -#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) - // ADD/ADC chain - __uint128_t t = ( ( __uint128_t )S->t[1] << 64 ) | S->t[0]; - t += inc; - S->t[0] = ( uint64_t )( t >> 0 ); - S->t[1] = ( uint64_t )( t >> 64 ); -#else - S->t[0] += inc; - S->t[1] += ( S->t[0] < inc ); -#endif - return 0; -} - - -// Parameter-related functions -static inline int blake2b_param_set_digest_length( blake2b_param *P, const uint8_t digest_length ) -{ - P->digest_length = digest_length; - return 0; -} - -static inline int blake2b_param_set_fanout( blake2b_param *P, const uint8_t fanout ) -{ - P->fanout = fanout; - return 0; -} - -static inline int blake2b_param_set_max_depth( blake2b_param *P, const uint8_t depth ) -{ - P->depth = depth; - return 0; -} - -static inline int blake2b_param_set_leaf_length( blake2b_param *P, const uint32_t leaf_length ) -{ - P->leaf_length = leaf_length; - return 0; -} - -static inline int blake2b_param_set_node_offset( blake2b_param *P, const uint64_t node_offset ) -{ - P->node_offset = node_offset; - return 0; -} - -static inline int blake2b_param_set_node_depth( blake2b_param *P, const uint8_t node_depth ) -{ - P->node_depth = node_depth; - return 0; -} - -static inline int blake2b_param_set_inner_length( blake2b_param *P, const uint8_t inner_length ) -{ - P->inner_length = inner_length; - return 0; -} - -static inline int blake2b_param_set_salt( blake2b_param *P, const uint8_t salt[BLAKE2B_SALTBYTES] ) -{ - memcpy( P->salt, salt, BLAKE2B_SALTBYTES ); - return 0; -} - -static inline int blake2b_param_set_personal( blake2b_param *P, const uint8_t personal[BLAKE2B_PERSONALBYTES] ) -{ - memcpy( P->personal, personal, BLAKE2B_PERSONALBYTES ); - return 0; -} - -static inline int blake2b_init0( blake2b_state *S ) -{ - memset( S, 0, sizeof( blake2b_state ) ); - - for( int i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; - - return 0; -} - - - -#if defined(__cplusplus) -extern "C" { -#endif - int blake2b_init( blake2b_state *S, size_t outlen ); - int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); - int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ); - int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ); - int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); -#if defined(__cplusplus) -} -#endif - -/* init xors IV with input parameter block */ -int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) -{ - uint8_t *p, *h, *v; - //blake2b_init0( S ); - v = ( uint8_t * )( blake2b_IV ); - h = ( uint8_t * )( S->h ); - p = ( uint8_t * )( P ); - /* IV XOR ParamBlock */ - memset( S, 0, sizeof( blake2b_state ) ); - - for( int i = 0; i < BLAKE2B_OUTBYTES; ++i ) h[i] = v[i] ^ p[i]; - - S->outlen = P->digest_length; - return 0; -} - - -/* Some sort of default parameter block initialization, for sequential blake2b */ - -int blake2b_init( blake2b_state *S, size_t outlen ) -{ - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - const blake2b_param P = - { - ( uint8_t ) outlen, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - {0}, - {0}, - {0} - }; - return blake2b_init_param( S, &P ); -} - -int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) -{ - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - if ( ( !keylen ) || keylen > BLAKE2B_KEYBYTES ) return -1; - - const blake2b_param P = - { - ( uint8_t ) outlen, - ( uint8_t ) keylen, - 1, - 1, - 0, - 0, - 0, - 0, - {0}, - {0}, - {0} - }; - - if( blake2b_init_param( S, &P ) < 0 ) - return 0; - - { - uint8_t block[BLAKE2B_BLOCKBYTES]; - memset( block, 0, BLAKE2B_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - -static inline int blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) -{ - __m128i row1l, row1h; - __m128i row2l, row2h; - __m128i row3l, row3h; - __m128i row4l, row4h; - __m128i b0, b1; - __m128i t0, t1; -#if defined(HAVE_SSSE3) && !defined(HAVE_XOP) - const __m128i r16 = _mm_setr_epi8( 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9 ); - const __m128i r24 = _mm_setr_epi8( 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10 ); -#endif -#if defined(HAVE_SSE4_1) - const __m128i m0 = LOADU( block + 00 ); - const __m128i m1 = LOADU( block + 16 ); - const __m128i m2 = LOADU( block + 32 ); - const __m128i m3 = LOADU( block + 48 ); - const __m128i m4 = LOADU( block + 64 ); - const __m128i m5 = LOADU( block + 80 ); - const __m128i m6 = LOADU( block + 96 ); - const __m128i m7 = LOADU( block + 112 ); -#else - const uint64_t m0 = ( ( uint64_t * )block )[ 0]; - const uint64_t m1 = ( ( uint64_t * )block )[ 1]; - const uint64_t m2 = ( ( uint64_t * )block )[ 2]; - const uint64_t m3 = ( ( uint64_t * )block )[ 3]; - const uint64_t m4 = ( ( uint64_t * )block )[ 4]; - const uint64_t m5 = ( ( uint64_t * )block )[ 5]; - const uint64_t m6 = ( ( uint64_t * )block )[ 6]; - const uint64_t m7 = ( ( uint64_t * )block )[ 7]; - const uint64_t m8 = ( ( uint64_t * )block )[ 8]; - const uint64_t m9 = ( ( uint64_t * )block )[ 9]; - const uint64_t m10 = ( ( uint64_t * )block )[10]; - const uint64_t m11 = ( ( uint64_t * )block )[11]; - const uint64_t m12 = ( ( uint64_t * )block )[12]; - const uint64_t m13 = ( ( uint64_t * )block )[13]; - const uint64_t m14 = ( ( uint64_t * )block )[14]; - const uint64_t m15 = ( ( uint64_t * )block )[15]; -#endif - row1l = LOADU( &S->h[0] ); - row1h = LOADU( &S->h[2] ); - row2l = LOADU( &S->h[4] ); - row2h = LOADU( &S->h[6] ); - row3l = LOADU( &blake2b_IV[0] ); - row3h = LOADU( &blake2b_IV[2] ); - row4l = _mm_xor_si128( LOADU( &blake2b_IV[4] ), LOADU( &S->t[0] ) ); - row4h = _mm_xor_si128( LOADU( &blake2b_IV[6] ), LOADU( &S->f[0] ) ); - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - ROUND( 10 ); - ROUND( 11 ); - row1l = _mm_xor_si128( row3l, row1l ); - row1h = _mm_xor_si128( row3h, row1h ); - STOREU( &S->h[0], _mm_xor_si128( LOADU( &S->h[0] ), row1l ) ); - STOREU( &S->h[2], _mm_xor_si128( LOADU( &S->h[2] ), row1h ) ); - row2l = _mm_xor_si128( row4l, row2l ); - row2h = _mm_xor_si128( row4h, row2h ); - STOREU( &S->h[4], _mm_xor_si128( LOADU( &S->h[4] ), row2l ) ); - STOREU( &S->h[6], _mm_xor_si128( LOADU( &S->h[6] ), row2h ) ); - return 0; -} - - -int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ) -{ - while( inlen > 0 ) - { - uint32_t left = S->buflen; - uint32_t fill = 2 * BLAKE2B_BLOCKBYTES - left; - - if( inlen > fill ) - { - memcpy( S->buf + left, in, fill ); // Fill buffer - S->buflen += fill; - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); // Compress - memcpy( S->buf, S->buf + BLAKE2B_BLOCKBYTES, BLAKE2B_BLOCKBYTES ); // Shift buffer left - S->buflen -= BLAKE2B_BLOCKBYTES; - in += fill; - inlen -= fill; - } - else // inlen <= fill - { - memcpy( S->buf + left, in, inlen ); - S->buflen += ( uint32_t ) inlen; // Be lazy, do not compress - in += inlen; - inlen -= inlen; - } - } - - return 0; -} - - -int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ) -{ - if(S->outlen != outlen) return -1; - - if( S->buflen > BLAKE2B_BLOCKBYTES ) - { - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); - S->buflen -= BLAKE2B_BLOCKBYTES; - memmove( S->buf, S->buf + BLAKE2B_BLOCKBYTES, S->buflen ); - } - - blake2b_increment_counter( S, S->buflen ); - blake2b_set_lastblock( S ); - memset( S->buf + S->buflen, 0, 2 * BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ - blake2b_compress( S, S->buf ); - memcpy( out, &S->h[0], outlen ); - return 0; -} - - -int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) -{ - blake2b_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if( NULL == key && keylen > 0 ) return -1; - - if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; - - if( keylen > BLAKE2B_KEYBYTES ) return -1; - - if( keylen ) - { - if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2b_init( S, outlen ) < 0 ) return -1; - } - - if( blake2b_update( S, ( uint8_t * )in, inlen ) < 0) return -1; - return blake2b_final( S, out, outlen ); -} - -#if defined(SUPERCOP) -int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) -{ - return blake2b( out, in, NULL, BLAKE2B_OUTBYTES, inlen, 0 ); -} -#endif diff --git a/Modules/_blake2/impl/blake2s-load-sse2.h b/Modules/_blake2/impl/blake2s-load-sse2.h deleted file mode 100644 index b24483cf931c1f..00000000000000 --- a/Modules/_blake2/impl/blake2s-load-sse2.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2S_LOAD_SSE2_H__ -#define __BLAKE2S_LOAD_SSE2_H__ - -#define LOAD_MSG_0_1(buf) buf = _mm_set_epi32(m6,m4,m2,m0) -#define LOAD_MSG_0_2(buf) buf = _mm_set_epi32(m7,m5,m3,m1) -#define LOAD_MSG_0_3(buf) buf = _mm_set_epi32(m14,m12,m10,m8) -#define LOAD_MSG_0_4(buf) buf = _mm_set_epi32(m15,m13,m11,m9) -#define LOAD_MSG_1_1(buf) buf = _mm_set_epi32(m13,m9,m4,m14) -#define LOAD_MSG_1_2(buf) buf = _mm_set_epi32(m6,m15,m8,m10) -#define LOAD_MSG_1_3(buf) buf = _mm_set_epi32(m5,m11,m0,m1) -#define LOAD_MSG_1_4(buf) buf = _mm_set_epi32(m3,m7,m2,m12) -#define LOAD_MSG_2_1(buf) buf = _mm_set_epi32(m15,m5,m12,m11) -#define LOAD_MSG_2_2(buf) buf = _mm_set_epi32(m13,m2,m0,m8) -#define LOAD_MSG_2_3(buf) buf = _mm_set_epi32(m9,m7,m3,m10) -#define LOAD_MSG_2_4(buf) buf = _mm_set_epi32(m4,m1,m6,m14) -#define LOAD_MSG_3_1(buf) buf = _mm_set_epi32(m11,m13,m3,m7) -#define LOAD_MSG_3_2(buf) buf = _mm_set_epi32(m14,m12,m1,m9) -#define LOAD_MSG_3_3(buf) buf = _mm_set_epi32(m15,m4,m5,m2) -#define LOAD_MSG_3_4(buf) buf = _mm_set_epi32(m8,m0,m10,m6) -#define LOAD_MSG_4_1(buf) buf = _mm_set_epi32(m10,m2,m5,m9) -#define LOAD_MSG_4_2(buf) buf = _mm_set_epi32(m15,m4,m7,m0) -#define LOAD_MSG_4_3(buf) buf = _mm_set_epi32(m3,m6,m11,m14) -#define LOAD_MSG_4_4(buf) buf = _mm_set_epi32(m13,m8,m12,m1) -#define LOAD_MSG_5_1(buf) buf = _mm_set_epi32(m8,m0,m6,m2) -#define LOAD_MSG_5_2(buf) buf = _mm_set_epi32(m3,m11,m10,m12) -#define LOAD_MSG_5_3(buf) buf = _mm_set_epi32(m1,m15,m7,m4) -#define LOAD_MSG_5_4(buf) buf = _mm_set_epi32(m9,m14,m5,m13) -#define LOAD_MSG_6_1(buf) buf = _mm_set_epi32(m4,m14,m1,m12) -#define LOAD_MSG_6_2(buf) buf = _mm_set_epi32(m10,m13,m15,m5) -#define LOAD_MSG_6_3(buf) buf = _mm_set_epi32(m8,m9,m6,m0) -#define LOAD_MSG_6_4(buf) buf = _mm_set_epi32(m11,m2,m3,m7) -#define LOAD_MSG_7_1(buf) buf = _mm_set_epi32(m3,m12,m7,m13) -#define LOAD_MSG_7_2(buf) buf = _mm_set_epi32(m9,m1,m14,m11) -#define LOAD_MSG_7_3(buf) buf = _mm_set_epi32(m2,m8,m15,m5) -#define LOAD_MSG_7_4(buf) buf = _mm_set_epi32(m10,m6,m4,m0) -#define LOAD_MSG_8_1(buf) buf = _mm_set_epi32(m0,m11,m14,m6) -#define LOAD_MSG_8_2(buf) buf = _mm_set_epi32(m8,m3,m9,m15) -#define LOAD_MSG_8_3(buf) buf = _mm_set_epi32(m10,m1,m13,m12) -#define LOAD_MSG_8_4(buf) buf = _mm_set_epi32(m5,m4,m7,m2) -#define LOAD_MSG_9_1(buf) buf = _mm_set_epi32(m1,m7,m8,m10) -#define LOAD_MSG_9_2(buf) buf = _mm_set_epi32(m5,m6,m4,m2) -#define LOAD_MSG_9_3(buf) buf = _mm_set_epi32(m13,m3,m9,m15) -#define LOAD_MSG_9_4(buf) buf = _mm_set_epi32(m0,m12,m14,m11) - - -#endif diff --git a/Modules/_blake2/impl/blake2s-load-sse41.h b/Modules/_blake2/impl/blake2s-load-sse41.h deleted file mode 100644 index 3ac12eb6f5d082..00000000000000 --- a/Modules/_blake2/impl/blake2s-load-sse41.h +++ /dev/null @@ -1,229 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2S_LOAD_SSE41_H__ -#define __BLAKE2S_LOAD_SSE41_H__ - -#define LOAD_MSG_0_1(buf) \ -buf = TOI(_mm_shuffle_ps(TOF(m0), TOF(m1), _MM_SHUFFLE(2,0,2,0))); - -#define LOAD_MSG_0_2(buf) \ -buf = TOI(_mm_shuffle_ps(TOF(m0), TOF(m1), _MM_SHUFFLE(3,1,3,1))); - -#define LOAD_MSG_0_3(buf) \ -buf = TOI(_mm_shuffle_ps(TOF(m2), TOF(m3), _MM_SHUFFLE(2,0,2,0))); - -#define LOAD_MSG_0_4(buf) \ -buf = TOI(_mm_shuffle_ps(TOF(m2), TOF(m3), _MM_SHUFFLE(3,1,3,1))); - -#define LOAD_MSG_1_1(buf) \ -t0 = _mm_blend_epi16(m1, m2, 0x0C); \ -t1 = _mm_slli_si128(m3, 4); \ -t2 = _mm_blend_epi16(t0, t1, 0xF0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,1,0,3)); - -#define LOAD_MSG_1_2(buf) \ -t0 = _mm_shuffle_epi32(m2,_MM_SHUFFLE(0,0,2,0)); \ -t1 = _mm_blend_epi16(m1,m3,0xC0); \ -t2 = _mm_blend_epi16(t0, t1, 0xF0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,3,0,1)); - -#define LOAD_MSG_1_3(buf) \ -t0 = _mm_slli_si128(m1, 4); \ -t1 = _mm_blend_epi16(m2, t0, 0x30); \ -t2 = _mm_blend_epi16(m0, t1, 0xF0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,3,0,1)); - -#define LOAD_MSG_1_4(buf) \ -t0 = _mm_unpackhi_epi32(m0,m1); \ -t1 = _mm_slli_si128(m3, 4); \ -t2 = _mm_blend_epi16(t0, t1, 0x0C); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,3,0,1)); - -#define LOAD_MSG_2_1(buf) \ -t0 = _mm_unpackhi_epi32(m2,m3); \ -t1 = _mm_blend_epi16(m3,m1,0x0C); \ -t2 = _mm_blend_epi16(t0, t1, 0x0F); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(3,1,0,2)); - -#define LOAD_MSG_2_2(buf) \ -t0 = _mm_unpacklo_epi32(m2,m0); \ -t1 = _mm_blend_epi16(t0, m0, 0xF0); \ -t2 = _mm_slli_si128(m3, 8); \ -buf = _mm_blend_epi16(t1, t2, 0xC0); - -#define LOAD_MSG_2_3(buf) \ -t0 = _mm_blend_epi16(m0, m2, 0x3C); \ -t1 = _mm_srli_si128(m1, 12); \ -t2 = _mm_blend_epi16(t0,t1,0x03); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1,0,3,2)); - -#define LOAD_MSG_2_4(buf) \ -t0 = _mm_slli_si128(m3, 4); \ -t1 = _mm_blend_epi16(m0, m1, 0x33); \ -t2 = _mm_blend_epi16(t1, t0, 0xC0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(0,1,2,3)); - -#define LOAD_MSG_3_1(buf) \ -t0 = _mm_unpackhi_epi32(m0,m1); \ -t1 = _mm_unpackhi_epi32(t0, m2); \ -t2 = _mm_blend_epi16(t1, m3, 0x0C); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(3,1,0,2)); - -#define LOAD_MSG_3_2(buf) \ -t0 = _mm_slli_si128(m2, 8); \ -t1 = _mm_blend_epi16(m3,m0,0x0C); \ -t2 = _mm_blend_epi16(t1, t0, 0xC0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,0,1,3)); - -#define LOAD_MSG_3_3(buf) \ -t0 = _mm_blend_epi16(m0,m1,0x0F); \ -t1 = _mm_blend_epi16(t0, m3, 0xC0); \ -buf = _mm_shuffle_epi32(t1, _MM_SHUFFLE(3,0,1,2)); - -#define LOAD_MSG_3_4(buf) \ -t0 = _mm_unpacklo_epi32(m0,m2); \ -t1 = _mm_unpackhi_epi32(m1,m2); \ -buf = _mm_unpacklo_epi64(t1,t0); - -#define LOAD_MSG_4_1(buf) \ -t0 = _mm_unpacklo_epi64(m1,m2); \ -t1 = _mm_unpackhi_epi64(m0,m2); \ -t2 = _mm_blend_epi16(t0,t1,0x33); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,0,1,3)); - -#define LOAD_MSG_4_2(buf) \ -t0 = _mm_unpackhi_epi64(m1,m3); \ -t1 = _mm_unpacklo_epi64(m0,m1); \ -buf = _mm_blend_epi16(t0,t1,0x33); - -#define LOAD_MSG_4_3(buf) \ -t0 = _mm_unpackhi_epi64(m3,m1); \ -t1 = _mm_unpackhi_epi64(m2,m0); \ -buf = _mm_blend_epi16(t1,t0,0x33); - -#define LOAD_MSG_4_4(buf) \ -t0 = _mm_blend_epi16(m0,m2,0x03); \ -t1 = _mm_slli_si128(t0, 8); \ -t2 = _mm_blend_epi16(t1,m3,0x0F); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1,2,0,3)); - -#define LOAD_MSG_5_1(buf) \ -t0 = _mm_unpackhi_epi32(m0,m1); \ -t1 = _mm_unpacklo_epi32(m0,m2); \ -buf = _mm_unpacklo_epi64(t0,t1); - -#define LOAD_MSG_5_2(buf) \ -t0 = _mm_srli_si128(m2, 4); \ -t1 = _mm_blend_epi16(m0,m3,0x03); \ -buf = _mm_blend_epi16(t1,t0,0x3C); - -#define LOAD_MSG_5_3(buf) \ -t0 = _mm_blend_epi16(m1,m0,0x0C); \ -t1 = _mm_srli_si128(m3, 4); \ -t2 = _mm_blend_epi16(t0,t1,0x30); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1,2,3,0)); - -#define LOAD_MSG_5_4(buf) \ -t0 = _mm_unpacklo_epi64(m1,m2); \ -t1= _mm_shuffle_epi32(m3, _MM_SHUFFLE(0,2,0,1)); \ -buf = _mm_blend_epi16(t0,t1,0x33); - -#define LOAD_MSG_6_1(buf) \ -t0 = _mm_slli_si128(m1, 12); \ -t1 = _mm_blend_epi16(m0,m3,0x33); \ -buf = _mm_blend_epi16(t1,t0,0xC0); - -#define LOAD_MSG_6_2(buf) \ -t0 = _mm_blend_epi16(m3,m2,0x30); \ -t1 = _mm_srli_si128(m1, 4); \ -t2 = _mm_blend_epi16(t0,t1,0x03); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,1,3,0)); - -#define LOAD_MSG_6_3(buf) \ -t0 = _mm_unpacklo_epi64(m0,m2); \ -t1 = _mm_srli_si128(m1, 4); \ -buf = _mm_shuffle_epi32(_mm_blend_epi16(t0,t1,0x0C), _MM_SHUFFLE(2,3,1,0)); - -#define LOAD_MSG_6_4(buf) \ -t0 = _mm_unpackhi_epi32(m1,m2); \ -t1 = _mm_unpackhi_epi64(m0,t0); \ -buf = _mm_shuffle_epi32(t1, _MM_SHUFFLE(3,0,1,2)); - -#define LOAD_MSG_7_1(buf) \ -t0 = _mm_unpackhi_epi32(m0,m1); \ -t1 = _mm_blend_epi16(t0,m3,0x0F); \ -buf = _mm_shuffle_epi32(t1,_MM_SHUFFLE(2,0,3,1)); - -#define LOAD_MSG_7_2(buf) \ -t0 = _mm_blend_epi16(m2,m3,0x30); \ -t1 = _mm_srli_si128(m0,4); \ -t2 = _mm_blend_epi16(t0,t1,0x03); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1,0,2,3)); - -#define LOAD_MSG_7_3(buf) \ -t0 = _mm_unpackhi_epi64(m0,m3); \ -t1 = _mm_unpacklo_epi64(m1,m2); \ -t2 = _mm_blend_epi16(t0,t1,0x3C); \ -buf = _mm_shuffle_epi32(t2,_MM_SHUFFLE(0,2,3,1)); - -#define LOAD_MSG_7_4(buf) \ -t0 = _mm_unpacklo_epi32(m0,m1); \ -t1 = _mm_unpackhi_epi32(m1,m2); \ -buf = _mm_unpacklo_epi64(t0,t1); - -#define LOAD_MSG_8_1(buf) \ -t0 = _mm_unpackhi_epi32(m1,m3); \ -t1 = _mm_unpacklo_epi64(t0,m0); \ -t2 = _mm_blend_epi16(t1,m2,0xC0); \ -buf = _mm_shufflehi_epi16(t2,_MM_SHUFFLE(1,0,3,2)); - -#define LOAD_MSG_8_2(buf) \ -t0 = _mm_unpackhi_epi32(m0,m3); \ -t1 = _mm_blend_epi16(m2,t0,0xF0); \ -buf = _mm_shuffle_epi32(t1,_MM_SHUFFLE(0,2,1,3)); - -#define LOAD_MSG_8_3(buf) \ -t0 = _mm_blend_epi16(m2,m0,0x0C); \ -t1 = _mm_slli_si128(t0,4); \ -buf = _mm_blend_epi16(t1,m3,0x0F); - -#define LOAD_MSG_8_4(buf) \ -t0 = _mm_blend_epi16(m1,m0,0x30); \ -buf = _mm_shuffle_epi32(t0,_MM_SHUFFLE(1,0,3,2)); - -#define LOAD_MSG_9_1(buf) \ -t0 = _mm_blend_epi16(m0,m2,0x03); \ -t1 = _mm_blend_epi16(m1,m2,0x30); \ -t2 = _mm_blend_epi16(t1,t0,0x0F); \ -buf = _mm_shuffle_epi32(t2,_MM_SHUFFLE(1,3,0,2)); - -#define LOAD_MSG_9_2(buf) \ -t0 = _mm_slli_si128(m0,4); \ -t1 = _mm_blend_epi16(m1,t0,0xC0); \ -buf = _mm_shuffle_epi32(t1,_MM_SHUFFLE(1,2,0,3)); - -#define LOAD_MSG_9_3(buf) \ -t0 = _mm_unpackhi_epi32(m0,m3); \ -t1 = _mm_unpacklo_epi32(m2,m3); \ -t2 = _mm_unpackhi_epi64(t0,t1); \ -buf = _mm_shuffle_epi32(t2,_MM_SHUFFLE(3,0,2,1)); - -#define LOAD_MSG_9_4(buf) \ -t0 = _mm_blend_epi16(m3,m2,0xC0); \ -t1 = _mm_unpacklo_epi32(m0,m3); \ -t2 = _mm_blend_epi16(t0,t1,0x0F); \ -buf = _mm_shuffle_epi32(t2,_MM_SHUFFLE(0,1,2,3)); - -#endif - diff --git a/Modules/_blake2/impl/blake2s-load-xop.h b/Modules/_blake2/impl/blake2s-load-xop.h deleted file mode 100644 index 14d9e7f7640672..00000000000000 --- a/Modules/_blake2/impl/blake2s-load-xop.h +++ /dev/null @@ -1,189 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2S_LOAD_XOP_H__ -#define __BLAKE2S_LOAD_XOP_H__ - -#define TOB(x) ((x)*4*0x01010101 + 0x03020100) // ..or not TOB - -/* Basic VPPERM emulation, for testing purposes */ -/*static __m128i _mm_perm_epi8(const __m128i src1, const __m128i src2, const __m128i sel) -{ - const __m128i sixteen = _mm_set1_epi8(16); - const __m128i t0 = _mm_shuffle_epi8(src1, sel); - const __m128i s1 = _mm_shuffle_epi8(src2, _mm_sub_epi8(sel, sixteen)); - const __m128i mask = _mm_or_si128(_mm_cmpeq_epi8(sel, sixteen), - _mm_cmpgt_epi8(sel, sixteen)); // (>=16) = 0xff : 00 - return _mm_blendv_epi8(t0, s1, mask); -}*/ - -#define LOAD_MSG_0_1(buf) \ -buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(6),TOB(4),TOB(2),TOB(0)) ); - -#define LOAD_MSG_0_2(buf) \ -buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(7),TOB(5),TOB(3),TOB(1)) ); - -#define LOAD_MSG_0_3(buf) \ -buf = _mm_perm_epi8(m2, m3, _mm_set_epi32(TOB(6),TOB(4),TOB(2),TOB(0)) ); - -#define LOAD_MSG_0_4(buf) \ -buf = _mm_perm_epi8(m2, m3, _mm_set_epi32(TOB(7),TOB(5),TOB(3),TOB(1)) ); - -#define LOAD_MSG_1_1(buf) \ -t0 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(0),TOB(5),TOB(0),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(5),TOB(2),TOB(1),TOB(6)) ); - -#define LOAD_MSG_1_2(buf) \ -t1 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(2),TOB(0),TOB(4),TOB(6)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(7),TOB(1),TOB(0)) ); - -#define LOAD_MSG_1_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(0),TOB(0),TOB(1)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(7),TOB(1),TOB(0)) ); - -#define LOAD_MSG_1_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(3),TOB(7),TOB(2),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(4)) ); - -#define LOAD_MSG_2_1(buf) \ -t0 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(0),TOB(1),TOB(0),TOB(7)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(7),TOB(2),TOB(4),TOB(0)) ); - -#define LOAD_MSG_2_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(2),TOB(0),TOB(4)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(5),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_2_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(7),TOB(3),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(5),TOB(2),TOB(1),TOB(6)) ); - -#define LOAD_MSG_2_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(4),TOB(1),TOB(6),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(6)) ); - -#define LOAD_MSG_3_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(3),TOB(7)) ); \ -t0 = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(7),TOB(2),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(5),TOB(1),TOB(0)) ); - -#define LOAD_MSG_3_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(0),TOB(1),TOB(5)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(6),TOB(4),TOB(1),TOB(0)) ); - -#define LOAD_MSG_3_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(4),TOB(5),TOB(2)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(7),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_3_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(0),TOB(6)) ); \ -buf = _mm_perm_epi8(t1, m2, _mm_set_epi32(TOB(4),TOB(2),TOB(6),TOB(0)) ); - -#define LOAD_MSG_4_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(2),TOB(5),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(6),TOB(2),TOB(1),TOB(5)) ); - -#define LOAD_MSG_4_2(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(4),TOB(7),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(7),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_4_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(3),TOB(6),TOB(0),TOB(0)) ); \ -t0 = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(2),TOB(7),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(6)) ); - -#define LOAD_MSG_4_4(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(4),TOB(0),TOB(1)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(5),TOB(2),TOB(4),TOB(0)) ); - -#define LOAD_MSG_5_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(6),TOB(2)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(4),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_5_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(3),TOB(7),TOB(6),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(4)) ); - -#define LOAD_MSG_5_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(1),TOB(0),TOB(7),TOB(4)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(7),TOB(1),TOB(0)) ); - -#define LOAD_MSG_5_4(buf) \ -t1 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(5),TOB(0),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(6),TOB(1),TOB(5)) ); - -#define LOAD_MSG_6_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(4),TOB(0),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(6),TOB(1),TOB(4)) ); - -#define LOAD_MSG_6_2(buf) \ -t1 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(6),TOB(0),TOB(0),TOB(1)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(5),TOB(7),TOB(0)) ); - -#define LOAD_MSG_6_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(6),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(4),TOB(5),TOB(1),TOB(0)) ); - -#define LOAD_MSG_6_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(2),TOB(3),TOB(7)) ); \ -buf = _mm_perm_epi8(t1, m2, _mm_set_epi32(TOB(7),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_7_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(3),TOB(0),TOB(7),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(4),TOB(1),TOB(5)) ); - -#define LOAD_MSG_7_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(5),TOB(1),TOB(0),TOB(7)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(6),TOB(0)) ); - -#define LOAD_MSG_7_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(2),TOB(0),TOB(0),TOB(5)) ); \ -t0 = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(4),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(7),TOB(0)) ); - -#define LOAD_MSG_7_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(6),TOB(4),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m2, _mm_set_epi32(TOB(6),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_8_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(0),TOB(6)) ); \ -t0 = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(7),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(6),TOB(0)) ); - -#define LOAD_MSG_8_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(4),TOB(3),TOB(5),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(7)) ); - -#define LOAD_MSG_8_3(buf) \ -t0 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(6),TOB(1),TOB(0),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(5),TOB(4)) ); \ - -#define LOAD_MSG_8_4(buf) \ -buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(4),TOB(7),TOB(2)) ); - -#define LOAD_MSG_9_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(1),TOB(7),TOB(0),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(2),TOB(4),TOB(6)) ); - -#define LOAD_MSG_9_2(buf) \ -buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(6),TOB(4),TOB(2)) ); - -#define LOAD_MSG_9_3(buf) \ -t0 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(3),TOB(5),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(5),TOB(2),TOB(1),TOB(7)) ); - -#define LOAD_MSG_9_4(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(0),TOB(0),TOB(7)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(4),TOB(6),TOB(0)) ); - -#endif - diff --git a/Modules/_blake2/impl/blake2s-ref.c b/Modules/_blake2/impl/blake2s-ref.c deleted file mode 100644 index ab86cc1b34e67d..00000000000000 --- a/Modules/_blake2/impl/blake2s-ref.c +++ /dev/null @@ -1,368 +0,0 @@ -/* - BLAKE2 reference source code package - reference C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ - -#include -#include -#include - -#include "blake2.h" -#include "blake2-impl.h" - -static const uint32_t blake2s_IV[8] = -{ - 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, - 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL -}; - -static const uint8_t blake2s_sigma[10][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , -}; - -static inline int blake2s_set_lastnode( blake2s_state *S ) -{ - S->f[1] = ~0U; - return 0; -} - -static inline int blake2s_clear_lastnode( blake2s_state *S ) -{ - S->f[1] = 0U; - return 0; -} - -/* Some helper functions, not necessarily useful */ -static inline int blake2s_set_lastblock( blake2s_state *S ) -{ - if( S->last_node ) blake2s_set_lastnode( S ); - - S->f[0] = ~0U; - return 0; -} - -static inline int blake2s_clear_lastblock( blake2s_state *S ) -{ - if( S->last_node ) blake2s_clear_lastnode( S ); - - S->f[0] = 0U; - return 0; -} - -static inline int blake2s_increment_counter( blake2s_state *S, const uint32_t inc ) -{ - S->t[0] += inc; - S->t[1] += ( S->t[0] < inc ); - return 0; -} - -// Parameter-related functions -static inline int blake2s_param_set_digest_length( blake2s_param *P, const uint8_t digest_length ) -{ - P->digest_length = digest_length; - return 0; -} - -static inline int blake2s_param_set_fanout( blake2s_param *P, const uint8_t fanout ) -{ - P->fanout = fanout; - return 0; -} - -static inline int blake2s_param_set_max_depth( blake2s_param *P, const uint8_t depth ) -{ - P->depth = depth; - return 0; -} - -static inline int blake2s_param_set_leaf_length( blake2s_param *P, const uint32_t leaf_length ) -{ - store32( &P->leaf_length, leaf_length ); - return 0; -} - -static inline int blake2s_param_set_node_offset( blake2s_param *P, const uint64_t node_offset ) -{ - store48( P->node_offset, node_offset ); - return 0; -} - -static inline int blake2s_param_set_node_depth( blake2s_param *P, const uint8_t node_depth ) -{ - P->node_depth = node_depth; - return 0; -} - -static inline int blake2s_param_set_inner_length( blake2s_param *P, const uint8_t inner_length ) -{ - P->inner_length = inner_length; - return 0; -} - -static inline int blake2s_param_set_salt( blake2s_param *P, const uint8_t salt[BLAKE2S_SALTBYTES] ) -{ - memcpy( P->salt, salt, BLAKE2S_SALTBYTES ); - return 0; -} - -static inline int blake2s_param_set_personal( blake2s_param *P, const uint8_t personal[BLAKE2S_PERSONALBYTES] ) -{ - memcpy( P->personal, personal, BLAKE2S_PERSONALBYTES ); - return 0; -} - -static inline int blake2s_init0( blake2s_state *S ) -{ - memset( S, 0, sizeof( blake2s_state ) ); - - for( int i = 0; i < 8; ++i ) S->h[i] = blake2s_IV[i]; - - return 0; -} - -#if defined(__cplusplus) -extern "C" { -#endif - int blake2s_init( blake2s_state *S, size_t outlen ); - int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); - int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ); - int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ); - int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); -#if defined(__cplusplus) -} -#endif - -/* init2 xors IV with input parameter block */ -int blake2s_init_param( blake2s_state *S, const blake2s_param *P ) -{ - blake2s_init0( S ); - uint32_t *p = ( uint32_t * )( P ); - - /* IV XOR ParamBlock */ - for( size_t i = 0; i < 8; ++i ) - S->h[i] ^= load32( &p[i] ); - - S->outlen = P->digest_length; - return 0; -} - - -// Sequential blake2s initialization -int blake2s_init( blake2s_state *S, size_t outlen ) -{ - blake2s_param P[1]; - - /* Move interval verification here? */ - if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; - - P->digest_length = ( uint8_t) outlen; - P->key_length = 0; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store48( &P->node_offset, 0 ); - P->node_depth = 0; - P->inner_length = 0; - // memset(P->reserved, 0, sizeof(P->reserved) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - return blake2s_init_param( S, P ); -} - -int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ) -{ - blake2s_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; - - if ( !key || !keylen || keylen > BLAKE2S_KEYBYTES ) return -1; - - P->digest_length = ( uint8_t ) outlen; - P->key_length = ( uint8_t ) keylen; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store48( &P->node_offset, 0 ); - P->node_depth = 0; - P->inner_length = 0; - // memset(P->reserved, 0, sizeof(P->reserved) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - - if( blake2s_init_param( S, P ) < 0 ) return -1; - - { - uint8_t block[BLAKE2S_BLOCKBYTES]; - memset( block, 0, BLAKE2S_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2s_update( S, block, BLAKE2S_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2S_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - -static int blake2s_compress( blake2s_state *S, const uint8_t block[BLAKE2S_BLOCKBYTES] ) -{ - uint32_t m[16]; - uint32_t v[16]; - - for( size_t i = 0; i < 16; ++i ) - m[i] = load32( block + i * sizeof( m[i] ) ); - - for( size_t i = 0; i < 8; ++i ) - v[i] = S->h[i]; - - v[ 8] = blake2s_IV[0]; - v[ 9] = blake2s_IV[1]; - v[10] = blake2s_IV[2]; - v[11] = blake2s_IV[3]; - v[12] = S->t[0] ^ blake2s_IV[4]; - v[13] = S->t[1] ^ blake2s_IV[5]; - v[14] = S->f[0] ^ blake2s_IV[6]; - v[15] = S->f[1] ^ blake2s_IV[7]; -#define G(r,i,a,b,c,d) \ - do { \ - a = a + b + m[blake2s_sigma[r][2*i+0]]; \ - d = rotr32(d ^ a, 16); \ - c = c + d; \ - b = rotr32(b ^ c, 12); \ - a = a + b + m[blake2s_sigma[r][2*i+1]]; \ - d = rotr32(d ^ a, 8); \ - c = c + d; \ - b = rotr32(b ^ c, 7); \ - } while(0) -#define ROUND(r) \ - do { \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ - } while(0) - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - - for( size_t i = 0; i < 8; ++i ) - S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; - -#undef G -#undef ROUND - return 0; -} - - -int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ) -{ - while( inlen > 0 ) - { - uint32_t left = S->buflen; - uint32_t fill = 2 * BLAKE2S_BLOCKBYTES - left; - - if( inlen > fill ) - { - memcpy( S->buf + left, in, fill ); // Fill buffer - S->buflen += fill; - blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); - blake2s_compress( S, S->buf ); // Compress - memcpy( S->buf, S->buf + BLAKE2S_BLOCKBYTES, BLAKE2S_BLOCKBYTES ); // Shift buffer left - S->buflen -= BLAKE2S_BLOCKBYTES; - in += fill; - inlen -= fill; - } - else // inlen <= fill - { - memcpy( S->buf + left, in, inlen ); - S->buflen += ( uint32_t ) inlen; // Be lazy, do not compress - in += inlen; - inlen -= inlen; - } - } - - return 0; -} - -int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ) -{ - uint8_t buffer[BLAKE2S_OUTBYTES]; - size_t i; - - if(S->outlen != outlen) return -1; - - if( S->buflen > BLAKE2S_BLOCKBYTES ) - { - blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); - blake2s_compress( S, S->buf ); - S->buflen -= BLAKE2S_BLOCKBYTES; - memmove( S->buf, S->buf + BLAKE2S_BLOCKBYTES, S->buflen ); - } - - blake2s_increment_counter( S, ( uint32_t )S->buflen ); - blake2s_set_lastblock( S ); - memset( S->buf + S->buflen, 0, 2 * BLAKE2S_BLOCKBYTES - S->buflen ); /* Padding */ - blake2s_compress( S, S->buf ); - - for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ - store32( buffer + sizeof( S->h[i] ) * i, S->h[i] ); - - memcpy( out, buffer, outlen ); - return 0; -} - -int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) -{ - blake2s_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if ( NULL == key && keylen > 0 ) return -1; - - if( !outlen || outlen > BLAKE2S_OUTBYTES ) return -1; - - if( keylen > BLAKE2S_KEYBYTES ) return -1; - - if( keylen > 0 ) - { - if( blake2s_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2s_init( S, outlen ) < 0 ) return -1; - } - - if( blake2s_update( S, ( uint8_t * )in, inlen ) < 0) return -1; - return blake2s_final( S, out, outlen ); -} - diff --git a/Modules/_blake2/impl/blake2s-round.h b/Modules/_blake2/impl/blake2s-round.h deleted file mode 100644 index 3af4be35bee5d4..00000000000000 --- a/Modules/_blake2/impl/blake2s-round.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2S_ROUND_H__ -#define __BLAKE2S_ROUND_H__ - -#define LOAD(p) _mm_load_si128( (__m128i *)(p) ) -#define STORE(p,r) _mm_store_si128((__m128i *)(p), r) - -#define LOADU(p) _mm_loadu_si128( (__m128i *)(p) ) -#define STOREU(p,r) _mm_storeu_si128((__m128i *)(p), r) - -#define TOF(reg) _mm_castsi128_ps((reg)) -#define TOI(reg) _mm_castps_si128((reg)) - -#define LIKELY(x) __builtin_expect((x),1) - - -/* Microarchitecture-specific macros */ -#ifndef HAVE_XOP -#ifdef HAVE_SSSE3 -#define _mm_roti_epi32(r, c) ( \ - (8==-(c)) ? _mm_shuffle_epi8(r,r8) \ - : (16==-(c)) ? _mm_shuffle_epi8(r,r16) \ - : _mm_xor_si128(_mm_srli_epi32( (r), -(c) ),_mm_slli_epi32( (r), 32-(-(c)) )) ) -#else -#define _mm_roti_epi32(r, c) _mm_xor_si128(_mm_srli_epi32( (r), -(c) ),_mm_slli_epi32( (r), 32-(-(c)) )) -#endif -#else -/* ... */ -#endif - - -#define G1(row1,row2,row3,row4,buf) \ - row1 = _mm_add_epi32( _mm_add_epi32( row1, buf), row2 ); \ - row4 = _mm_xor_si128( row4, row1 ); \ - row4 = _mm_roti_epi32(row4, -16); \ - row3 = _mm_add_epi32( row3, row4 ); \ - row2 = _mm_xor_si128( row2, row3 ); \ - row2 = _mm_roti_epi32(row2, -12); - -#define G2(row1,row2,row3,row4,buf) \ - row1 = _mm_add_epi32( _mm_add_epi32( row1, buf), row2 ); \ - row4 = _mm_xor_si128( row4, row1 ); \ - row4 = _mm_roti_epi32(row4, -8); \ - row3 = _mm_add_epi32( row3, row4 ); \ - row2 = _mm_xor_si128( row2, row3 ); \ - row2 = _mm_roti_epi32(row2, -7); - -#define DIAGONALIZE(row1,row2,row3,row4) \ - row4 = _mm_shuffle_epi32( row4, _MM_SHUFFLE(2,1,0,3) ); \ - row3 = _mm_shuffle_epi32( row3, _MM_SHUFFLE(1,0,3,2) ); \ - row2 = _mm_shuffle_epi32( row2, _MM_SHUFFLE(0,3,2,1) ); - -#define UNDIAGONALIZE(row1,row2,row3,row4) \ - row4 = _mm_shuffle_epi32( row4, _MM_SHUFFLE(0,3,2,1) ); \ - row3 = _mm_shuffle_epi32( row3, _MM_SHUFFLE(1,0,3,2) ); \ - row2 = _mm_shuffle_epi32( row2, _MM_SHUFFLE(2,1,0,3) ); - -#if defined(HAVE_XOP) -#include "blake2s-load-xop.h" -#elif defined(HAVE_SSE4_1) -#include "blake2s-load-sse41.h" -#else -#include "blake2s-load-sse2.h" -#endif - -#define ROUND(r) \ - LOAD_MSG_ ##r ##_1(buf1); \ - G1(row1,row2,row3,row4,buf1); \ - LOAD_MSG_ ##r ##_2(buf2); \ - G2(row1,row2,row3,row4,buf2); \ - DIAGONALIZE(row1,row2,row3,row4); \ - LOAD_MSG_ ##r ##_3(buf3); \ - G1(row1,row2,row3,row4,buf3); \ - LOAD_MSG_ ##r ##_4(buf4); \ - G2(row1,row2,row3,row4,buf4); \ - UNDIAGONALIZE(row1,row2,row3,row4); \ - -#endif - diff --git a/Modules/_blake2/impl/blake2s.c b/Modules/_blake2/impl/blake2s.c deleted file mode 100644 index 47514685b8f30b..00000000000000 --- a/Modules/_blake2/impl/blake2s.c +++ /dev/null @@ -1,415 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ - -#include -#include -#include - -#include "blake2.h" -#include "blake2-impl.h" - -#include "blake2-config.h" - -#if defined(_MSC_VER) -#include -#endif - -#if defined(HAVE_SSE2) -#include -// MSVC only defines _mm_set_epi64x for x86_64... -#if defined(_MSC_VER) && !defined(_M_X64) -static inline __m128i _mm_set_epi64x( const uint64_t u1, const uint64_t u0 ) -{ - return _mm_set_epi32( u1 >> 32, u1, u0 >> 32, u0 ); -} -#endif -#endif - - -#if defined(HAVE_SSSE3) -#include -#endif -#if defined(HAVE_SSE4_1) -#include -#endif -#if defined(HAVE_AVX) -#include -#endif -#if defined(HAVE_XOP) && !defined(_MSC_VER) -#include -#endif - -#include "blake2s-round.h" - -static const uint32_t blake2s_IV[8] = -{ - 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, - 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL -}; - -static const uint8_t blake2s_sigma[10][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , -}; - - -/* Some helper functions, not necessarily useful */ -static inline int blake2s_set_lastnode( blake2s_state *S ) -{ - S->f[1] = ~0U; - return 0; -} - -static inline int blake2s_clear_lastnode( blake2s_state *S ) -{ - S->f[1] = 0U; - return 0; -} - -static inline int blake2s_set_lastblock( blake2s_state *S ) -{ - if( S->last_node ) blake2s_set_lastnode( S ); - - S->f[0] = ~0U; - return 0; -} - -static inline int blake2s_clear_lastblock( blake2s_state *S ) -{ - if( S->last_node ) blake2s_clear_lastnode( S ); - - S->f[0] = 0U; - return 0; -} - -static inline int blake2s_increment_counter( blake2s_state *S, const uint32_t inc ) -{ - uint64_t t = ( ( uint64_t )S->t[1] << 32 ) | S->t[0]; - t += inc; - S->t[0] = ( uint32_t )( t >> 0 ); - S->t[1] = ( uint32_t )( t >> 32 ); - return 0; -} - - -// Parameter-related functions -static inline int blake2s_param_set_digest_length( blake2s_param *P, const uint8_t digest_length ) -{ - P->digest_length = digest_length; - return 0; -} - -static inline int blake2s_param_set_fanout( blake2s_param *P, const uint8_t fanout ) -{ - P->fanout = fanout; - return 0; -} - -static inline int blake2s_param_set_max_depth( blake2s_param *P, const uint8_t depth ) -{ - P->depth = depth; - return 0; -} - -static inline int blake2s_param_set_leaf_length( blake2s_param *P, const uint32_t leaf_length ) -{ - P->leaf_length = leaf_length; - return 0; -} - -static inline int blake2s_param_set_node_offset( blake2s_param *P, const uint64_t node_offset ) -{ - store48( P->node_offset, node_offset ); - return 0; -} - -static inline int blake2s_param_set_node_depth( blake2s_param *P, const uint8_t node_depth ) -{ - P->node_depth = node_depth; - return 0; -} - -static inline int blake2s_param_set_inner_length( blake2s_param *P, const uint8_t inner_length ) -{ - P->inner_length = inner_length; - return 0; -} - -static inline int blake2s_param_set_salt( blake2s_param *P, const uint8_t salt[BLAKE2S_SALTBYTES] ) -{ - memcpy( P->salt, salt, BLAKE2S_SALTBYTES ); - return 0; -} - -static inline int blake2s_param_set_personal( blake2s_param *P, const uint8_t personal[BLAKE2S_PERSONALBYTES] ) -{ - memcpy( P->personal, personal, BLAKE2S_PERSONALBYTES ); - return 0; -} - -static inline int blake2s_init0( blake2s_state *S ) -{ - memset( S, 0, sizeof( blake2s_state ) ); - - for( int i = 0; i < 8; ++i ) S->h[i] = blake2s_IV[i]; - - return 0; -} - -#if defined(__cplusplus) -extern "C" { -#endif - int blake2s_init( blake2s_state *S, size_t outlen ); - int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); - int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ); - int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ); - int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); -#if defined(__cplusplus) -} -#endif - - -/* init2 xors IV with input parameter block */ -int blake2s_init_param( blake2s_state *S, const blake2s_param *P ) -{ - uint8_t *p, *h, *v; - //blake2s_init0( S ); - v = ( uint8_t * )( blake2s_IV ); - h = ( uint8_t * )( S->h ); - p = ( uint8_t * )( P ); - /* IV XOR ParamBlock */ - memset( S, 0, sizeof( blake2s_state ) ); - - for( int i = 0; i < BLAKE2S_OUTBYTES; ++i ) h[i] = v[i] ^ p[i]; - - S->outlen = P->digest_length; - return 0; -} - - -/* Some sort of default parameter block initialization, for sequential blake2s */ -int blake2s_init( blake2s_state *S, size_t outlen ) -{ - if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; - - const blake2s_param P = - { - outlen, - 0, - 1, - 1, - 0, - {0}, - 0, - 0, - {0}, - {0} - }; - return blake2s_init_param( S, &P ); -} - - -int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ) -{ - if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; - - if ( ( !key ) || ( !keylen ) || keylen > BLAKE2S_KEYBYTES ) return -1; - - const blake2s_param P = - { - outlen, - keylen, - 1, - 1, - 0, - {0}, - 0, - 0, - {0}, - {0} - }; - - if( blake2s_init_param( S, &P ) < 0 ) - return -1; - - { - uint8_t block[BLAKE2S_BLOCKBYTES]; - memset( block, 0, BLAKE2S_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2s_update( S, block, BLAKE2S_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2S_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - - -static inline int blake2s_compress( blake2s_state *S, const uint8_t block[BLAKE2S_BLOCKBYTES] ) -{ - __m128i row1, row2, row3, row4; - __m128i buf1, buf2, buf3, buf4; -#if defined(HAVE_SSE4_1) - __m128i t0, t1; -#if !defined(HAVE_XOP) - __m128i t2; -#endif -#endif - __m128i ff0, ff1; -#if defined(HAVE_SSSE3) && !defined(HAVE_XOP) - const __m128i r8 = _mm_set_epi8( 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1 ); - const __m128i r16 = _mm_set_epi8( 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 ); -#endif -#if defined(HAVE_SSE4_1) - const __m128i m0 = LOADU( block + 00 ); - const __m128i m1 = LOADU( block + 16 ); - const __m128i m2 = LOADU( block + 32 ); - const __m128i m3 = LOADU( block + 48 ); -#else - const uint32_t m0 = ( ( uint32_t * )block )[ 0]; - const uint32_t m1 = ( ( uint32_t * )block )[ 1]; - const uint32_t m2 = ( ( uint32_t * )block )[ 2]; - const uint32_t m3 = ( ( uint32_t * )block )[ 3]; - const uint32_t m4 = ( ( uint32_t * )block )[ 4]; - const uint32_t m5 = ( ( uint32_t * )block )[ 5]; - const uint32_t m6 = ( ( uint32_t * )block )[ 6]; - const uint32_t m7 = ( ( uint32_t * )block )[ 7]; - const uint32_t m8 = ( ( uint32_t * )block )[ 8]; - const uint32_t m9 = ( ( uint32_t * )block )[ 9]; - const uint32_t m10 = ( ( uint32_t * )block )[10]; - const uint32_t m11 = ( ( uint32_t * )block )[11]; - const uint32_t m12 = ( ( uint32_t * )block )[12]; - const uint32_t m13 = ( ( uint32_t * )block )[13]; - const uint32_t m14 = ( ( uint32_t * )block )[14]; - const uint32_t m15 = ( ( uint32_t * )block )[15]; -#endif - row1 = ff0 = LOADU( &S->h[0] ); - row2 = ff1 = LOADU( &S->h[4] ); - row3 = _mm_setr_epi32( 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A ); - row4 = _mm_xor_si128( _mm_setr_epi32( 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 ), LOADU( &S->t[0] ) ); - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - STOREU( &S->h[0], _mm_xor_si128( ff0, _mm_xor_si128( row1, row3 ) ) ); - STOREU( &S->h[4], _mm_xor_si128( ff1, _mm_xor_si128( row2, row4 ) ) ); - return 0; -} - - -int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ) -{ - while( inlen > 0 ) - { - size_t left = S->buflen; - size_t fill = 2 * BLAKE2S_BLOCKBYTES - left; - - if( inlen > fill ) - { - memcpy( S->buf + left, in, fill ); // Fill buffer - S->buflen += fill; - blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); - blake2s_compress( S, S->buf ); // Compress - memcpy( S->buf, S->buf + BLAKE2S_BLOCKBYTES, BLAKE2S_BLOCKBYTES ); // Shift buffer left - S->buflen -= BLAKE2S_BLOCKBYTES; - in += fill; - inlen -= fill; - } - else /* inlen <= fill */ - { - memcpy( S->buf + left, in, inlen ); - S->buflen += inlen; // Be lazy, do not compress - in += inlen; - inlen -= inlen; - } - } - - return 0; -} - - -int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ) -{ - uint8_t buffer[BLAKE2S_OUTBYTES]; - - if(outlen != S->outlen ) return -1; - - if( S->buflen > BLAKE2S_BLOCKBYTES ) - { - blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); - blake2s_compress( S, S->buf ); - S->buflen -= BLAKE2S_BLOCKBYTES; - memmove( S->buf, S->buf + BLAKE2S_BLOCKBYTES, S->buflen ); - } - - blake2s_increment_counter( S, ( uint32_t )S->buflen ); - blake2s_set_lastblock( S ); - memset( S->buf + S->buflen, 0, 2 * BLAKE2S_BLOCKBYTES - S->buflen ); /* Padding */ - blake2s_compress( S, S->buf ); - - for( int i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ - store32( buffer + sizeof( S->h[i] ) * i, S->h[i] ); - - memcpy( out, buffer, outlen ); - return 0; -} - -int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) -{ - blake2s_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if ( NULL == key && keylen > 0) return -1; - - if( !outlen || outlen > BLAKE2S_OUTBYTES ) return -1; - - if( keylen > BLAKE2S_KEYBYTES ) return -1; - - if( keylen > 0 ) - { - if( blake2s_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2s_init( S, outlen ) < 0 ) return -1; - } - - if( blake2s_update( S, ( uint8_t * )in, inlen ) < 0) return -1; - return blake2s_final( S, out, outlen ); -} - -#if defined(SUPERCOP) -int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) -{ - return blake2s( out, in, NULL, BLAKE2S_OUTBYTES, (size_t)inlen, 0 ); -} -#endif - diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 3d0d4ee5e79c2b..661847ad26702e 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -802,6 +802,7 @@ _bz2_free(void *module) static struct PyModuleDef_Slot _bz2_slots[] = { {Py_mod_exec, _bz2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index c31c1b6d6f2bbc..471b42badc8e8c 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -979,6 +979,30 @@ _codecs_register_error_impl(PyObject *module, const char *errors, Py_RETURN_NONE; } +/*[clinic input] +_codecs._unregister_error -> bool + errors: str + / + +Un-register the specified error handler for the error handling `errors'. + +Only custom error handlers can be un-registered. An exception is raised +if the error handling is a built-in one (e.g., 'strict'), or if an error +occurs. + +Otherwise, this returns True if a custom handler has been successfully +un-registered, and False if no custom handler for the specified error +handling exists. + +[clinic start generated code]*/ + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors) +/*[clinic end generated code: output=28c22be667465503 input=a63ab9e9ce1686d4]*/ +{ + return _PyCodec_UnregisterError(errors); +} + /*[clinic input] _codecs.lookup_error name: str @@ -1044,12 +1068,14 @@ static PyMethodDef _codecs_functions[] = { _CODECS_CODE_PAGE_ENCODE_METHODDEF _CODECS_CODE_PAGE_DECODE_METHODDEF _CODECS_REGISTER_ERROR_METHODDEF + _CODECS__UNREGISTER_ERROR_METHODDEF _CODECS_LOOKUP_ERROR_METHODDEF {NULL, NULL} /* sentinel */ }; static PyModuleDef_Slot _codecs_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ef77d34b10e47b..aef04248c7e73c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -3,6 +3,7 @@ #include "pycore_dict.h" // _PyDict_GetItem_KnownHash() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_typeobject.h" // _PyType_GetModuleState() #include @@ -229,6 +230,7 @@ deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } /*[clinic input] +@critical_section _collections.deque.pop as deque_pop deque: dequeobject @@ -238,7 +240,7 @@ Remove and return the rightmost element. static PyObject * deque_pop_impl(dequeobject *deque) -/*[clinic end generated code: output=2e5f7890c4251f07 input=eb6e6d020f877dec]*/ +/*[clinic end generated code: output=2e5f7890c4251f07 input=55c5b6a8ad51d72f]*/ { PyObject *item; block *prevblock; @@ -273,6 +275,7 @@ deque_pop_impl(dequeobject *deque) } /*[clinic input] +@critical_section _collections.deque.popleft as deque_popleft deque: dequeobject @@ -282,7 +285,7 @@ Remove and return the leftmost element. static PyObject * deque_popleft_impl(dequeobject *deque) -/*[clinic end generated code: output=62b154897097ff68 input=acb41b9af50a9d9b]*/ +/*[clinic end generated code: output=62b154897097ff68 input=1571ce88fe3053de]*/ { PyObject *item; block *prevblock; @@ -332,7 +335,7 @@ deque_popleft_impl(dequeobject *deque) #define NEEDS_TRIM(deque, maxlen) ((size_t)(maxlen) < (size_t)(Py_SIZE(deque))) static inline int -deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) +deque_append_lock_held(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(deque); @@ -358,6 +361,7 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) } /*[clinic input] +@critical_section _collections.deque.append as deque_append deque: dequeobject @@ -368,16 +372,17 @@ Add an element to the right side of the deque. [clinic start generated code]*/ static PyObject * -deque_append(dequeobject *deque, PyObject *item) -/*[clinic end generated code: output=507b13efc4853ecc input=f112b83c380528e3]*/ +deque_append_impl(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=9c7bcb8b599c6362 input=b0eeeb09b9f5cf18]*/ { - if (deque_append_internal(deque, Py_NewRef(item), deque->maxlen) < 0) + if (deque_append_lock_held(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } static inline int -deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) +deque_appendleft_lock_held(dequeobject *deque, PyObject *item, + Py_ssize_t maxlen) { if (deque->leftindex == 0) { block *b = newblock(deque); @@ -393,7 +398,7 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) Py_SET_SIZE(deque, Py_SIZE(deque) + 1); deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - if (NEEDS_TRIM(deque, deque->maxlen)) { + if (NEEDS_TRIM(deque, maxlen)) { PyObject *olditem = deque_pop_impl(deque); Py_DECREF(olditem); } else { @@ -403,6 +408,7 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) } /*[clinic input] +@critical_section _collections.deque.appendleft as deque_appendleft deque: dequeobject @@ -413,10 +419,10 @@ Add an element to the left side of the deque. [clinic start generated code]*/ static PyObject * -deque_appendleft(dequeobject *deque, PyObject *item) -/*[clinic end generated code: output=de0335a64800ffd8 input=bbdaa60a3e956062]*/ +deque_appendleft_impl(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=9a192edbcd0f20db input=236c2fbceaf08e14]*/ { - if (deque_appendleft_internal(deque, Py_NewRef(item), deque->maxlen) < 0) + if (deque_appendleft_lock_held(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } @@ -452,6 +458,7 @@ consume_iterator(PyObject *it) } /*[clinic input] +@critical_section _collections.deque.extend as deque_extend deque: dequeobject @@ -462,8 +469,8 @@ Extend the right side of the deque with elements from the iterable. [clinic start generated code]*/ static PyObject * -deque_extend(dequeobject *deque, PyObject *iterable) -/*[clinic end generated code: output=a3a6e74d17063f8d input=cfebfd34d5383339]*/ +deque_extend_impl(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=8b5ffa57ce82d980 input=85861954127c81da]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -497,7 +504,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque_append_internal(deque, item, maxlen) == -1) { + if (deque_append_lock_held(deque, item, maxlen) == -1) { Py_DECREF(item); Py_DECREF(it); return NULL; @@ -507,6 +514,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) } /*[clinic input] +@critical_section _collections.deque.extendleft as deque_extendleft deque: dequeobject @@ -517,8 +525,8 @@ Extend the left side of the deque with elements from the iterable. [clinic start generated code]*/ static PyObject * -deque_extendleft(dequeobject *deque, PyObject *iterable) -/*[clinic end generated code: output=2dba946c50498c67 input=f4820e695a6f9416]*/ +deque_extendleft_impl(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=ba44191aa8e35a26 input=640dabd086115689]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -530,7 +538,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) PyObject *s = PySequence_List(iterable); if (s == NULL) return NULL; - result = deque_extendleft(deque, s); + result = deque_extendleft_impl(deque, s); Py_DECREF(s); return result; } @@ -552,7 +560,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque_appendleft_internal(deque, item, maxlen) == -1) { + if (deque_appendleft_lock_held(deque, item, maxlen) == -1) { Py_DECREF(item); Py_DECREF(it); return NULL; @@ -566,6 +574,7 @@ deque_inplace_concat(dequeobject *deque, PyObject *other) { PyObject *result; + // deque_extend is thread-safe result = deque_extend(deque, other); if (result == NULL) return result; @@ -575,6 +584,7 @@ deque_inplace_concat(dequeobject *deque, PyObject *other) } /*[clinic input] +@critical_section _collections.deque.copy as deque_copy deque: dequeobject @@ -584,7 +594,7 @@ Return a shallow copy of a deque. static PyObject * deque_copy_impl(dequeobject *deque) -/*[clinic end generated code: output=6409b3d1ad2898b5 input=0e22f138bc1fcbee]*/ +/*[clinic end generated code: output=6409b3d1ad2898b5 input=51d2ed1a23bab5e2]*/ { PyObject *result; dequeobject *old_deque = (dequeobject *)deque; @@ -598,12 +608,16 @@ deque_copy_impl(dequeobject *deque) if (new_deque == NULL) return NULL; new_deque->maxlen = old_deque->maxlen; - /* Fast path for the deque_repeat() common case where len(deque) == 1 */ + /* Fast path for the deque_repeat() common case where len(deque) == 1 + * + * It's safe to not acquire the per-object lock for new_deque; it's + * invisible to other threads. + */ if (Py_SIZE(deque) == 1) { PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; - rv = deque_append(new_deque, item); + rv = deque_append_impl(new_deque, item); } else { - rv = deque_extend(new_deque, (PyObject *)deque); + rv = deque_extend_impl(new_deque, (PyObject *)deque); } if (rv != NULL) { Py_DECREF(rv); @@ -629,6 +643,7 @@ deque_copy_impl(dequeobject *deque) } /*[clinic input] +@critical_section _collections.deque.__copy__ as deque___copy__ = _collections.deque.copy Return a shallow copy of a deque. @@ -636,13 +651,13 @@ Return a shallow copy of a deque. static PyObject * deque___copy___impl(dequeobject *deque) -/*[clinic end generated code: output=7c5821504342bf23 input=fce05df783e7912b]*/ +/*[clinic end generated code: output=7c5821504342bf23 input=f5464036f9686a55]*/ { return deque_copy_impl(deque); } static PyObject * -deque_concat(dequeobject *deque, PyObject *other) +deque_concat_lock_held(dequeobject *deque, PyObject *other) { PyObject *new_deque, *result; int rv; @@ -661,7 +676,10 @@ deque_concat(dequeobject *deque, PyObject *other) new_deque = deque_copy_impl(deque); if (new_deque == NULL) return NULL; - result = deque_extend((dequeobject *)new_deque, other); + + // It's safe to not acquire the per-object lock for new_deque; it's + // invisible to other threads. + result = deque_extend_impl((dequeobject *)new_deque, other); if (result == NULL) { Py_DECREF(new_deque); return NULL; @@ -670,6 +688,16 @@ deque_concat(dequeobject *deque, PyObject *other) return new_deque; } +static PyObject * +deque_concat(dequeobject *deque, PyObject *other) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_concat_lock_held(deque, other); + Py_END_CRITICAL_SECTION(); + return result; +} + static int deque_clear(dequeobject *deque) { @@ -755,6 +783,7 @@ deque_clear(dequeobject *deque) } /*[clinic input] +@critical_section _collections.deque.clear as deque_clearmethod deque: dequeobject @@ -764,14 +793,14 @@ Remove all elements from the deque. static PyObject * deque_clearmethod_impl(dequeobject *deque) -/*[clinic end generated code: output=79b2513e097615c1 input=20488eb932f89f9e]*/ +/*[clinic end generated code: output=79b2513e097615c1 input=3a22e9605d20c5e9]*/ { deque_clear(deque); Py_RETURN_NONE; } static PyObject * -deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) +deque_inplace_repeat_lock_held(dequeobject *deque, Py_ssize_t n) { Py_ssize_t i, m, size; PyObject *seq; @@ -835,7 +864,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) n = (deque->maxlen + size - 1) / size; for (i = 0 ; i < n-1 ; i++) { - rv = deque_extend(deque, seq); + rv = deque_extend_impl(deque, seq); if (rv == NULL) { Py_DECREF(seq); return NULL; @@ -847,16 +876,30 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } +static PyObject * +deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_inplace_repeat_lock_held(deque, n); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * deque_repeat(dequeobject *deque, Py_ssize_t n) { dequeobject *new_deque; PyObject *rv; + Py_BEGIN_CRITICAL_SECTION(deque); new_deque = (dequeobject *)deque_copy_impl(deque); + Py_END_CRITICAL_SECTION(); if (new_deque == NULL) return NULL; - rv = deque_inplace_repeat(new_deque, n); + // It's safe to not acquire the per-object lock for new_deque; it's + // invisible to other threads. + rv = deque_inplace_repeat_lock_held(new_deque, n); Py_DECREF(new_deque); return rv; } @@ -1011,6 +1054,7 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) } /*[clinic input] +@critical_section _collections.deque.rotate as deque_rotate deque: dequeobject @@ -1022,7 +1066,7 @@ Rotate the deque n steps to the right. If n is negative, rotates left. static PyObject * deque_rotate_impl(dequeobject *deque, Py_ssize_t n) -/*[clinic end generated code: output=96c2402a371eb15d input=d22070f49cc06c76]*/ +/*[clinic end generated code: output=96c2402a371eb15d input=5bf834296246e002]*/ { if (!_deque_rotate(deque, n)) Py_RETURN_NONE; @@ -1030,6 +1074,7 @@ deque_rotate_impl(dequeobject *deque, Py_ssize_t n) } /*[clinic input] +@critical_section _collections.deque.reverse as deque_reverse deque: dequeobject @@ -1039,7 +1084,7 @@ Reverse *IN PLACE*. static PyObject * deque_reverse_impl(dequeobject *deque) -/*[clinic end generated code: output=bdeebc2cf8c1f064 input=f139787f406101c9]*/ +/*[clinic end generated code: output=bdeebc2cf8c1f064 input=26f4167fd623027f]*/ { block *leftblock = deque->leftblock; block *rightblock = deque->rightblock; @@ -1077,6 +1122,7 @@ deque_reverse_impl(dequeobject *deque) } /*[clinic input] +@critical_section _collections.deque.count as deque_count deque: dequeobject @@ -1087,8 +1133,8 @@ Return number of occurrences of value. [clinic start generated code]*/ static PyObject * -deque_count(dequeobject *deque, PyObject *v) -/*[clinic end generated code: output=7405d289d94d7b9b input=1892925260ff5d78]*/ +deque_count_impl(dequeobject *deque, PyObject *v) +/*[clinic end generated code: output=2ca26c49b6ab0400 input=4ef67ef2b34dc1fc]*/ { block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; @@ -1124,7 +1170,7 @@ deque_count(dequeobject *deque, PyObject *v) } static int -deque_contains(dequeobject *deque, PyObject *v) +deque_contains_lock_held(dequeobject *deque, PyObject *v) { block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; @@ -1155,13 +1201,24 @@ deque_contains(dequeobject *deque, PyObject *v) return 0; } +static int +deque_contains(dequeobject *deque, PyObject *v) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_contains_lock_held(deque, v); + Py_END_CRITICAL_SECTION(); + return result; +} + static Py_ssize_t deque_len(dequeobject *deque) { - return Py_SIZE(deque); + return FT_ATOMIC_LOAD_SSIZE(((PyVarObject *)deque)->ob_size); } /*[clinic input] +@critical_section @text_signature "($self, value, [start, [stop]])" _collections.deque.index as deque_index @@ -1179,7 +1236,7 @@ Raises ValueError if the value is not present. static PyObject * deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, Py_ssize_t stop) -/*[clinic end generated code: output=df45132753175ef9 input=140210c099830f64]*/ +/*[clinic end generated code: output=df45132753175ef9 input=90f48833a91e1743]*/ { Py_ssize_t i, n; PyObject *item; @@ -1218,8 +1275,9 @@ deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, n = stop - i; while (--n >= 0) { CHECK_NOT_END(b); - item = b->data[index]; + item = Py_NewRef(b->data[index]); cmp = PyObject_RichCompareBool(item, v, Py_EQ); + Py_DECREF(item); if (cmp > 0) return PyLong_FromSsize_t(stop - n - 1); if (cmp < 0) @@ -1235,7 +1293,7 @@ deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, index = 0; } } - PyErr_Format(PyExc_ValueError, "%R is not in deque", v); + PyErr_SetString(PyExc_ValueError, "deque.index(x): x not in deque"); return NULL; } @@ -1248,6 +1306,7 @@ deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, */ /*[clinic input] +@critical_section _collections.deque.insert as deque_insert deque: dequeobject @@ -1260,7 +1319,7 @@ Insert value before index. static PyObject * deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value) -/*[clinic end generated code: output=ef4d2c15d5532b80 input=3e5c1c120d70c0e6]*/ +/*[clinic end generated code: output=ef4d2c15d5532b80 input=dbee706586cc9cde]*/ { Py_ssize_t n = Py_SIZE(deque); PyObject *rv; @@ -1270,15 +1329,15 @@ deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value) return NULL; } if (index >= n) - return deque_append(deque, value); + return deque_append_impl(deque, value); if (index <= -n || index == 0) - return deque_appendleft(deque, value); + return deque_appendleft_impl(deque, value); if (_deque_rotate(deque, -index)) return NULL; if (index < 0) - rv = deque_append(deque, value); + rv = deque_append_impl(deque, value); else - rv = deque_appendleft(deque, value); + rv = deque_appendleft_impl(deque, value); if (rv == NULL) return NULL; Py_DECREF(rv); @@ -1296,7 +1355,7 @@ valid_index(Py_ssize_t i, Py_ssize_t limit) } static PyObject * -deque_item(dequeobject *deque, Py_ssize_t i) +deque_item_lock_held(dequeobject *deque, Py_ssize_t i) { block *b; PyObject *item; @@ -1334,6 +1393,16 @@ deque_item(dequeobject *deque, Py_ssize_t i) return Py_NewRef(item); } +static PyObject * +deque_item(dequeobject *deque, Py_ssize_t i) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_item_lock_held(deque, i); + Py_END_CRITICAL_SECTION(); + return result; +} + static int deque_del_item(dequeobject *deque, Py_ssize_t i) { @@ -1351,6 +1420,7 @@ deque_del_item(dequeobject *deque, Py_ssize_t i) } /*[clinic input] +@critical_section _collections.deque.remove as deque_remove deque: dequeobject @@ -1361,8 +1431,8 @@ Remove first occurrence of value. [clinic start generated code]*/ static PyObject * -deque_remove(dequeobject *deque, PyObject *value) -/*[clinic end generated code: output=49e1666d612fe911 input=d972f32d15990880]*/ +deque_remove_impl(dequeobject *deque, PyObject *value) +/*[clinic end generated code: output=54cff28b8ef78c5b input=60eb3f8aa4de532a]*/ { PyObject *item; block *b = deque->leftblock; @@ -1392,7 +1462,7 @@ deque_remove(dequeobject *deque, PyObject *value) } } if (i == n) { - PyErr_Format(PyExc_ValueError, "%R is not in deque", value); + PyErr_SetString(PyExc_ValueError, "deque.remove(x): x not in deque"); return NULL; } rv = deque_del_item(deque, i); @@ -1403,7 +1473,7 @@ deque_remove(dequeobject *deque, PyObject *value) } static int -deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) +deque_ass_item_lock_held(dequeobject *deque, Py_ssize_t i, PyObject *v) { block *b; Py_ssize_t n, len=Py_SIZE(deque), halflen=(len+1)>>1, index=i; @@ -1434,6 +1504,16 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) return 0; } +static int +deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_ass_item_lock_held(deque, i, v); + Py_END_CRITICAL_SECTION(); + return result; +} + static void deque_dealloc(dequeobject *deque) { @@ -1508,6 +1588,8 @@ deque___reduce___impl(dequeobject *deque) return NULL; } + // It's safe to access deque->maxlen here without holding the per object + // lock for deque; deque->maxlen is only assigned during construction. if (deque->maxlen < 0) { return Py_BuildValue("O()NN", Py_TYPE(deque), state, it); } @@ -1628,6 +1710,7 @@ deque_richcompare(PyObject *v, PyObject *w, int op) } /*[clinic input] +@critical_section @text_signature "([iterable[, maxlen]])" _collections.deque.__init__ as deque_init @@ -1640,8 +1723,7 @@ A list-like sequence optimized for data accesses near its endpoints. static int deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj) -/*[clinic end generated code: output=7084a39d71218dcd input=5ebdffc48a2d27ae]*/ - +/*[clinic end generated code: output=7084a39d71218dcd input=2b9e37af1fd73143]*/ { Py_ssize_t maxlen = -1; if (maxlenobj != NULL && maxlenobj != Py_None) { @@ -1657,7 +1739,7 @@ deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj) if (Py_SIZE(deque) > 0) deque_clear(deque); if (iterable != NULL) { - PyObject *rv = deque_extend(deque, iterable); + PyObject *rv = deque_extend_impl(deque, iterable); if (rv == NULL) return -1; Py_DECREF(rv); @@ -1666,6 +1748,7 @@ deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj) } /*[clinic input] +@critical_section _collections.deque.__sizeof__ as deque___sizeof__ deque: dequeobject @@ -1675,7 +1758,7 @@ Return the size of the deque in memory, in bytes. static PyObject * deque___sizeof___impl(dequeobject *deque) -/*[clinic end generated code: output=4d36e9fb4f30bbaf input=4e7c9a00c03c3290]*/ +/*[clinic end generated code: output=4d36e9fb4f30bbaf input=762312f2d4813535]*/ { size_t res = _PyObject_SIZE(Py_TYPE(deque)); size_t blocks; @@ -1809,11 +1892,13 @@ deque_iter(dequeobject *deque) it = PyObject_GC_New(dequeiterobject, state->dequeiter_type); if (it == NULL) return NULL; + Py_BEGIN_CRITICAL_SECTION(deque); it->b = deque->leftblock; it->index = deque->leftindex; it->deque = (dequeobject*)Py_NewRef(deque); it->state = deque->state; it->counter = Py_SIZE(deque); + Py_END_CRITICAL_SECTION(); PyObject_GC_Track(it); return (PyObject *)it; } @@ -1845,7 +1930,7 @@ dequeiter_dealloc(dequeiterobject *dio) } static PyObject * -dequeiter_next(dequeiterobject *it) +dequeiter_next_lock_held(dequeiterobject *it, dequeobject *deque) { PyObject *item; @@ -1871,6 +1956,20 @@ dequeiter_next(dequeiterobject *it) return Py_NewRef(item); } +static PyObject * +dequeiter_next(dequeiterobject *it) +{ + PyObject *result; + // It's safe to access it->deque without holding the per-object lock for it + // here; it->deque is only assigned during construction of it. + dequeobject *deque = it->deque; + Py_BEGIN_CRITICAL_SECTION2(it, deque); + result = dequeiter_next_lock_held(it, deque); + Py_END_CRITICAL_SECTION2(); + + return result; +} + static PyObject * dequeiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1891,6 +1990,11 @@ dequeiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (item) { Py_DECREF(item); } else { + /* + * It's safe to read directly from it without acquiring the + * per-object lock; the iterator isn't visible to any other threads + * yet. + */ if (it->counter) { Py_DECREF(it); return NULL; @@ -1904,7 +2008,8 @@ dequeiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * dequeiter_len(dequeiterobject *it, PyObject *Py_UNUSED(ignored)) { - return PyLong_FromSsize_t(it->counter); + Py_ssize_t len = FT_ATOMIC_LOAD_SSIZE(it->counter); + return PyLong_FromSsize_t(len); } PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); @@ -1912,7 +2017,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list( static PyObject * dequeiter_reduce(dequeiterobject *it, PyObject *Py_UNUSED(ignored)) { - return Py_BuildValue("O(On)", Py_TYPE(it), it->deque, Py_SIZE(it->deque) - it->counter); + PyTypeObject *ty = Py_TYPE(it); + // It's safe to access it->deque without holding the per-object lock for it + // here; it->deque is only assigned during construction of it. + dequeobject *deque = it->deque; + Py_ssize_t size, counter; + Py_BEGIN_CRITICAL_SECTION2(it, deque); + size = Py_SIZE(deque); + counter = it->counter; + Py_END_CRITICAL_SECTION2(); + return Py_BuildValue("O(On)", ty, deque, size - counter); } static PyMethodDef dequeiter_methods[] = { @@ -1952,17 +2066,19 @@ deque_reviter(dequeobject *deque) it = PyObject_GC_New(dequeiterobject, state->dequereviter_type); if (it == NULL) return NULL; + Py_BEGIN_CRITICAL_SECTION(deque); it->b = deque->rightblock; it->index = deque->rightindex; it->deque = (dequeobject*)Py_NewRef(deque); it->state = deque->state; it->counter = Py_SIZE(deque); + Py_END_CRITICAL_SECTION(); PyObject_GC_Track(it); return (PyObject *)it; } static PyObject * -dequereviter_next(dequeiterobject *it) +dequereviter_next_lock_held(dequeiterobject *it, dequeobject *deque) { PyObject *item; if (it->counter == 0) @@ -1988,6 +2104,19 @@ dequereviter_next(dequeiterobject *it) return Py_NewRef(item); } +static PyObject * +dequereviter_next(dequeiterobject *it) +{ + PyObject *item; + // It's safe to access it->deque without holding the per-object lock for it + // here; it->deque is only assigned during construction of it. + dequeobject *deque = it->deque; + Py_BEGIN_CRITICAL_SECTION2(it, deque); + item = dequereviter_next_lock_held(it, deque); + Py_END_CRITICAL_SECTION2(); + return item; +} + static PyObject * dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -2008,6 +2137,11 @@ dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (item) { Py_DECREF(item); } else { + /* + * It's safe to read directly from it without acquiring the + * per-object lock; the iterator isn't visible to any other threads + * yet. + */ if (it->counter) { Py_DECREF(it); return NULL; @@ -2045,6 +2179,8 @@ typedef struct { PyObject *default_factory; } defdictobject; +static PyType_Spec defdict_spec; + PyDoc_STRVAR(defdict_missing_doc, "__missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n\ if self.default_factory is None: raise KeyError((key,))\n\ @@ -2224,23 +2360,16 @@ defdict_or(PyObject* left, PyObject* right) { PyObject *self, *other; - // Find module state - PyTypeObject *tp = Py_TYPE(left); - PyObject *mod = PyType_GetModuleByDef(tp, &_collectionsmodule); - if (mod == NULL) { - PyErr_Clear(); - tp = Py_TYPE(right); - mod = PyType_GetModuleByDef(tp, &_collectionsmodule); + int ret = PyType_GetBaseByToken(Py_TYPE(left), &defdict_spec, NULL); + if (ret < 0) { + return NULL; } - assert(mod != NULL); - collections_state *state = get_module_state(mod); - - if (PyObject_TypeCheck(left, state->defdict_type)) { + if (ret) { self = left; other = right; } else { - assert(PyObject_TypeCheck(right, state->defdict_type)); + assert(PyType_GetBaseByToken(Py_TYPE(right), &defdict_spec, NULL) == 1); self = right; other = left; } @@ -2320,6 +2449,7 @@ passed to the dict constructor, including keyword arguments.\n\ #define DEFERRED_ADDRESS(ADDR) 0 static PyType_Slot defdict_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, defdict_dealloc}, {Py_tp_repr, defdict_repr}, {Py_nb_or, defdict_or}, @@ -2377,9 +2507,9 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, /* Only take the fast path when get() and __setitem__() * have not been overridden. */ - mapping_get = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(get)); + mapping_get = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(get)); dict_get = _PyType_Lookup(&PyDict_Type, &_Py_ID(get)); - mapping_setitem = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(__setitem__)); + mapping_setitem = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(__setitem__)); dict_setitem = _PyType_Lookup(&PyDict_Type, &_Py_ID(__setitem__)); if (mapping_get != NULL && mapping_get == dict_get && @@ -2403,12 +2533,9 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, if (key == NULL) break; - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) - { - hash = PyObject_Hash(key); - if (hash == -1) - goto done; + hash = _PyObject_HashFast(key); + if (hash == -1) { + goto done; } oldval = _PyDict_GetItem_KnownHash(mapping, key, hash); @@ -2441,7 +2568,11 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, oldval = PyObject_CallFunctionObjArgs(bound_get, key, zero, NULL); if (oldval == NULL) break; - newval = PyNumber_Add(oldval, one); + if (oldval == zero) { + newval = Py_NewRef(one); + } else { + newval = PyNumber_Add(oldval, one); + } Py_DECREF(oldval); if (newval == NULL) break; @@ -2453,6 +2584,8 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, } done: + Py_XDECREF(mapping_get); + Py_XDECREF(mapping_setitem); Py_DECREF(it); Py_XDECREF(key); Py_XDECREF(newval); @@ -2683,6 +2816,7 @@ collections_exec(PyObject *module) { static struct PyModuleDef_Slot collections_slots[] = { {Py_mod_exec, collections_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_complex.h b/Modules/_complex.h new file mode 100644 index 00000000000000..28d4a32794b97c --- /dev/null +++ b/Modules/_complex.h @@ -0,0 +1,54 @@ +/* Workarounds for buggy complex number arithmetic implementations. */ + +#ifndef Py_HAVE_C_COMPLEX +# error "this header file should only be included if Py_HAVE_C_COMPLEX is defined" +#endif + +#include + +/* Other compilers (than clang), that claims to + implement C11 *and* define __STDC_IEC_559_COMPLEX__ - don't have + issue with CMPLX(). This is specific to glibc & clang combination: + https://sourceware.org/bugzilla/show_bug.cgi?id=26287 + + Here we fallback to using __builtin_complex(), available in clang + v12+. Else CMPLX implemented following C11 6.2.5p13: "Each complex type + has the same representation and alignment requirements as an array + type containing exactly two elements of the corresponding real type; + the first element is equal to the real part, and the second element + to the imaginary part, of the complex number. + */ +#if !defined(CMPLX) +# if defined(__clang__) && __has_builtin(__builtin_complex) +# define CMPLX(x, y) __builtin_complex ((double) (x), (double) (y)) +# define CMPLXF(x, y) __builtin_complex ((float) (x), (float) (y)) +# define CMPLXL(x, y) __builtin_complex ((long double) (x), (long double) (y)) +# else +static inline double complex +CMPLX(double real, double imag) +{ + double complex z; + ((double *)(&z))[0] = real; + ((double *)(&z))[1] = imag; + return z; +} + +static inline float complex +CMPLXF(float real, float imag) +{ + float complex z; + ((float *)(&z))[0] = real; + ((float *)(&z))[1] = imag; + return z; +} + +static inline long double complex +CMPLXL(long double real, long double imag) +{ + long double complex z; + ((long double *)(&z))[0] = real; + ((long double *)(&z))[1] = imag; + return z; +} +# endif +#endif diff --git a/Modules/_contextvarsmodule.c b/Modules/_contextvarsmodule.c index f621c1de6d42d6..3f96f07909b69a 100644 --- a/Modules/_contextvarsmodule.c +++ b/Modules/_contextvarsmodule.c @@ -45,6 +45,7 @@ _contextvars_exec(PyObject *m) static struct PyModuleDef_Slot _contextvars_slots[] = { {Py_mod_exec, _contextvars_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_csv.c b/Modules/_csv.c index 3aa648b8e9cec4..1a4dc3f1f55ace 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -14,6 +14,7 @@ module instead. #endif #include "Python.h" +#include "pycore_pyatomic_ft_wrappers.h" #include // offsetof() #include @@ -34,7 +35,7 @@ typedef struct { PyTypeObject *dialect_type; PyTypeObject *reader_type; PyTypeObject *writer_type; - long field_limit; /* max parsed field size */ + Py_ssize_t field_limit; /* max parsed field size */ PyObject *str_write; } _csvstate; @@ -332,9 +333,9 @@ dialect_check_quoting(int quoting) } static int -dialect_check_char(const char *name, Py_UCS4 c, DialectObj *dialect) +dialect_check_char(const char *name, Py_UCS4 c, DialectObj *dialect, bool allowspace) { - if (c == '\r' || c == '\n' || (dialect->skipinitialspace && c == ' ')) { + if (c == '\r' || c == '\n' || (c == ' ' && !allowspace)) { PyErr_Format(PyExc_ValueError, "bad %s value", name); return -1; } @@ -367,6 +368,8 @@ static struct PyMemberDef Dialect_memberlist[] = { { NULL } }; +#undef D_OFF + static PyGetSetDef Dialect_getsetlist[] = { { "delimiter", (getter)Dialect_get_delimiter}, { "escapechar", (getter)Dialect_get_escapechar}, @@ -502,6 +505,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) DIALECT_GETATTR(skipinitialspace, "skipinitialspace"); DIALECT_GETATTR(strict, "strict"); } +#undef DIALECT_GETATTR /* check types and convert to C values */ #define DIASET(meth, name, target, src, dflt) \ @@ -515,6 +519,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) DIASET(_set_int, "quoting", &self->quoting, quoting, QUOTE_MINIMAL); DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, false); DIASET(_set_bool, "strict", &self->strict, strict, false); +#undef DIASET /* validate options */ if (dialect_check_quoting(self->quoting)) @@ -535,9 +540,11 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "lineterminator must be set"); goto err; } - if (dialect_check_char("delimiter", self->delimiter, self) || - dialect_check_char("escapechar", self->escapechar, self) || - dialect_check_char("quotechar", self->quotechar, self) || + if (dialect_check_char("delimiter", self->delimiter, self, true) || + dialect_check_char("escapechar", self->escapechar, self, + !self->skipinitialspace) || + dialect_check_char("quotechar", self->quotechar, self, + !self->skipinitialspace) || dialect_check_chars("delimiter", "escapechar", self->delimiter, self->escapechar) || dialect_check_chars("delimiter", "quotechar", @@ -700,10 +707,11 @@ parse_grow_buff(ReaderObj *self) static int parse_add_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) { - if (self->field_len >= module_state->field_limit) { + Py_ssize_t field_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); + if (self->field_len >= field_limit) { PyErr_Format(module_state->error_obj, - "field larger than field limit (%ld)", - module_state->field_limit); + "field larger than field limit (%zd)", + field_limit); return -1; } if (self->field_len == self->field_size && !parse_grow_buff(self)) @@ -729,7 +737,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } /* normal character - handle as START_FIELD */ self->state = START_FIELD; - /* fallthru */ + _Py_FALLTHROUGH; case START_FIELD: /* expecting field */ self->unquoted_field = true; @@ -747,7 +755,6 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } else if (c == dialect->escapechar) { /* possible escaped character */ - self->unquoted_field = false; self->state = ESCAPED_CHAR; } else if (c == ' ' && dialect->skipinitialspace) @@ -783,7 +790,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) case AFTER_ESCAPED_CRNL: if (c == EOL) break; - /*fallthru*/ + _Py_FALLTHROUGH; case IN_FIELD: /* in unquoted field */ @@ -1025,6 +1032,8 @@ static struct PyMemberDef Reader_memberlist[] = { { NULL } }; +#undef R_OFF + static PyType_Slot Reader_Type_slots[] = { {Py_tp_doc, (char*)Reader_Type_doc}, @@ -1071,7 +1080,7 @@ csv_reader(PyObject *module, PyObject *args, PyObject *keyword_args) return NULL; } - if (!PyArg_UnpackTuple(args, "", 1, 2, &iterator, &dialect)) { + if (!PyArg_UnpackTuple(args, "reader", 1, 2, &iterator, &dialect)) { Py_DECREF(self); return NULL; } @@ -1150,6 +1159,8 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, if (c == dialect->delimiter || c == dialect->escapechar || c == dialect->quotechar || + c == '\n' || + c == '\r' || PyUnicode_FindChar( dialect->lineterminator, c, 0, PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) { @@ -1221,6 +1232,7 @@ join_check_rec_size(WriterObj *self, Py_ssize_t rec_len) static int join_append(WriterObj *self, PyObject *field, int quoted) { + DialectObj *dialect = self->dialect; int field_kind = -1; const void *field_data = NULL; Py_ssize_t field_len = 0; @@ -1231,6 +1243,19 @@ join_append(WriterObj *self, PyObject *field, int quoted) field_data = PyUnicode_DATA(field); field_len = PyUnicode_GET_LENGTH(field); } + if (!field_len && dialect->delimiter == ' ' && dialect->skipinitialspace) { + if (dialect->quoting == QUOTE_NONE || + (field == NULL && + (dialect->quoting == QUOTE_STRINGS || + dialect->quoting == QUOTE_NOTNULL))) + { + PyErr_Format(self->error_obj, + "empty field must be quoted if delimiter is a space " + "and skipinitialspace is true"); + return 0; + } + quoted = 1; + } rec_len = join_append_data(self, field_kind, field_data, field_len, "ed, 0); if (rec_len < 0) @@ -1282,6 +1307,7 @@ csv_writerow(WriterObj *self, PyObject *seq) { DialectObj *dialect = self->dialect; PyObject *iter, *field, *line, *result; + bool null_field = false; iter = PyObject_GetIter(seq); if (iter == NULL) { @@ -1318,11 +1344,12 @@ csv_writerow(WriterObj *self, PyObject *seq) break; } + null_field = (field == Py_None); if (PyUnicode_Check(field)) { append_ok = join_append(self, field, quoted); Py_DECREF(field); } - else if (field == Py_None) { + else if (null_field) { append_ok = join_append(self, NULL, quoted); Py_DECREF(field); } @@ -1348,7 +1375,11 @@ csv_writerow(WriterObj *self, PyObject *seq) return NULL; if (self->num_fields > 0 && self->rec_len == 0) { - if (dialect->quoting == QUOTE_NONE) { + if (dialect->quoting == QUOTE_NONE || + (null_field && + (dialect->quoting == QUOTE_STRINGS || + dialect->quoting == QUOTE_NOTNULL))) + { PyErr_Format(self->error_obj, "single empty field record must be quoted"); return NULL; @@ -1418,6 +1449,8 @@ static struct PyMemberDef Writer_memberlist[] = { { NULL } }; +#undef W_OFF + static int Writer_traverse(WriterObj *self, visitproc visit, void *arg) { @@ -1496,7 +1529,7 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args) self->error_obj = Py_NewRef(module_state->error_obj); - if (!PyArg_UnpackTuple(args, "", 1, 2, &output_file, &dialect)) { + if (!PyArg_UnpackTuple(args, "writer", 1, 2, &output_file, &dialect)) { Py_DECREF(self); return NULL; } @@ -1548,7 +1581,7 @@ csv_register_dialect(PyObject *module, PyObject *args, PyObject *kwargs) _csvstate *module_state = get_csv_state(module); PyObject *dialect; - if (!PyArg_UnpackTuple(args, "", 1, 2, &name_obj, &dialect_obj)) + if (!PyArg_UnpackTuple(args, "register_dialect", 1, 2, &name_obj, &dialect_obj)) return NULL; if (!PyUnicode_Check(name_obj)) { PyErr_SetString(PyExc_TypeError, @@ -1582,10 +1615,12 @@ _csv_unregister_dialect_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=0813ebca6c058df4 input=6b5c1557bf60c7e7]*/ { _csvstate *module_state = get_csv_state(module); - if (PyDict_DelItem(module_state->dialects, name) < 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Format(module_state->error_obj, "unknown dialect"); - } + int rc = PyDict_Pop(module_state->dialects, name, NULL); + if (rc < 0) { + return NULL; + } + if (rc == 0) { + PyErr_Format(module_state->error_obj, "unknown dialect"); return NULL; } Py_RETURN_NONE; @@ -1626,20 +1661,20 @@ _csv_field_size_limit_impl(PyObject *module, PyObject *new_limit) /*[clinic end generated code: output=f2799ecd908e250b input=cec70e9226406435]*/ { _csvstate *module_state = get_csv_state(module); - long old_limit = module_state->field_limit; + Py_ssize_t old_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); if (new_limit != NULL) { if (!PyLong_CheckExact(new_limit)) { PyErr_Format(PyExc_TypeError, "limit must be an integer"); return NULL; } - module_state->field_limit = PyLong_AsLong(new_limit); - if (module_state->field_limit == -1 && PyErr_Occurred()) { - module_state->field_limit = old_limit; + Py_ssize_t new_limit_value = PyLong_AsSsize_t(new_limit); + if (new_limit_value == -1 && PyErr_Occurred()) { return NULL; } + FT_ATOMIC_STORE_SSIZE_RELAXED(module_state->field_limit, new_limit_value); } - return PyLong_FromLong(old_limit); + return PyLong_FromSsize_t(old_limit); } static PyType_Slot error_slots[] = { @@ -1770,6 +1805,7 @@ csv_exec(PyObject *module) { static PyModuleDef_Slot csv_slots[] = { {Py_mod_exec, csv_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 94245ae41afffc..eae69e484e1660 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2,7 +2,7 @@ ToDo: Get rid of the checker (and also the converters) field in PyCFuncPtrObject and - StgDictObject, and replace them by slot functions in StgDictObject. + StgInfo, and replace them by slot functions in StgInfo. think about a buffer-like object (memory? bytes?) @@ -36,7 +36,6 @@ PyCData_Type Simple_Type __new__(), __init__(), _as_parameter_ PyCField_Type -PyCStgDict_Type ============================================================================== @@ -82,7 +81,6 @@ bytes(cdata) */ /* - * PyCStgDict_Type * PyCStructType_Type * UnionType_Type * PyCPointerType_Type @@ -128,30 +126,16 @@ bytes(cdata) #include "pycore_long.h" // _PyLong_GetZero() -static PyTypeObject Union_Type; -static PyTypeObject Struct_Type; -static PyTypeObject Simple_Type; - -ctypes_state global_state = { - .PyCStgDict_Type = &PyCStgDict_Type, - .PyCData_Type = &PyCData_Type, - .Struct_Type = &Struct_Type, - .Union_Type = &Union_Type, - .PyCArray_Type = &PyCArray_Type, - .Simple_Type = &Simple_Type, - .PyCPointer_Type = &PyCPointer_Type, - .PyCFuncPtr_Type = &PyCFuncPtr_Type, -}; - -PyObject *PyExc_ArgError = NULL; - -/* This dict maps ctypes types to POINTER types */ -PyObject *_ctypes_ptrtype_cache = NULL; - -/* a callable object used for unpickling: - strong reference to _ctypes._unpickle() function */ -static PyObject *_unpickle; +/*[clinic input] +module _ctypes +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ +#define clinic_state() (get_module_state_by_class(cls)) +#define clinic_state_sub() (get_module_state_by_class(cls->tp_base)) +#include "clinic/_ctypes.c.h" +#undef clinic_state +#undef clinic_state_sub /****************************************************************/ @@ -223,14 +207,13 @@ static PyType_Spec dictremover_spec = { }; int -PyDict_SetItemProxy(PyObject *dict, PyObject *key, PyObject *item) +PyDict_SetItemProxy(ctypes_state *st, PyObject *dict, PyObject *key, PyObject *item) { PyObject *obj; DictRemoverObject *remover; PyObject *proxy; int result; - ctypes_state *st = GLOBAL_STATE(); obj = _PyObject_CallNoArgs((PyObject *)st->DictRemover_Type); if (obj == NULL) return -1; @@ -337,7 +320,7 @@ _ctypes_alloc_format_string_for_type(char code, int big_endian) indicator set. If called with a suffix of NULL the error indicator must already be set. */ -char * +static char * _ctypes_alloc_format_string(const char *prefix, const char *suffix) { size_t len; @@ -369,7 +352,7 @@ _ctypes_alloc_format_string(const char *prefix, const char *suffix) Returns NULL on failure, with the error indicator set. If called with a suffix of NULL the error indicator must already be set. */ -char * +static char * _ctypes_alloc_format_string_with_shape(int ndim, const Py_ssize_t *shape, const char *prefix, const char *suffix) { @@ -459,20 +442,154 @@ static PyType_Spec structparam_spec = { .slots = structparam_slots, }; +/* + CType_Type - a base metaclass. Its instances (classes) have a StgInfo. + */ + +/*[clinic input] +class _ctypes.CType_Type "PyObject *" "clinic_state()->CType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8389fc5b74a84f2a]*/ + +static int +CType_Type_traverse(PyObject *self, visitproc visit, void *arg) +{ + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + Py_VISIT(info->proto); + Py_VISIT(info->argtypes); + Py_VISIT(info->converters); + Py_VISIT(info->restype); + Py_VISIT(info->checker); + Py_VISIT(info->module); + } + Py_VISIT(Py_TYPE(self)); + return PyType_Type.tp_traverse(self, visit, arg); +} + +void +ctype_clear_stginfo(StgInfo *info) +{ + assert(info); + Py_CLEAR(info->proto); + Py_CLEAR(info->argtypes); + Py_CLEAR(info->converters); + Py_CLEAR(info->restype); + Py_CLEAR(info->checker); + Py_CLEAR(info->module); +} + +static int +CType_Type_clear(PyObject *self) +{ + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + ctype_clear_stginfo(info); + } + return PyType_Type.tp_clear(self); +} + +static void +CType_Type_dealloc(PyObject *self) +{ + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(NULL); // NULL avoids segfault here + } + if (info) { + PyMem_Free(info->ffi_type_pointer.elements); + info->ffi_type_pointer.elements = NULL; + PyMem_Free(info->format); + info->format = NULL; + PyMem_Free(info->shape); + info->shape = NULL; + ctype_clear_stginfo(info); + } + + PyTypeObject *tp = Py_TYPE(self); + PyType_Type.tp_dealloc(self); + Py_DECREF(tp); +} + +/*[clinic input] +_ctypes.CType_Type.__sizeof__ + + cls: defining_class + / +Return memory consumption of the type object. +[clinic start generated code]*/ + +static PyObject * +_ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=c68c235be84d03f3 input=d064433b6110d1ce]*/ +{ + Py_ssize_t size = Py_TYPE(self)->tp_basicsize; + size += Py_TYPE(self)->tp_itemsize * Py_SIZE(self); + + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, self, &info) < 0) { + return NULL; + } + if (info) { + if (info->format) { + size += strlen(info->format) + 1; + } + if (info->ffi_type_pointer.elements) { + size += (info->length + 1) * sizeof(ffi_type *); + } + size += info->ndim * sizeof(Py_ssize_t); + } + + return PyLong_FromSsize_t(size); +} + +static PyObject * +CType_Type_repeat(PyObject *self, Py_ssize_t length); + + +static PyMethodDef ctype_methods[] = { + _CTYPES_CTYPE_TYPE___SIZEOF___METHODDEF + {0}, +}; + +static PyType_Slot ctype_type_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + {Py_tp_traverse, CType_Type_traverse}, + {Py_tp_clear, CType_Type_clear}, + {Py_tp_dealloc, CType_Type_dealloc}, + {Py_tp_methods, ctype_methods}, + // Sequence protocol. + {Py_sq_repeat, CType_Type_repeat}, + {0, NULL}, +}; + +PyType_Spec pyctype_type_spec = { + .name = "_ctypes.CType_Type", + .basicsize = -(Py_ssize_t)sizeof(StgInfo), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE ), + .slots = ctype_type_slots, +}; /* PyCStructType_Type - a meta type/class. Creating a new class using this one as - __metaclass__ will call the constructor StructUnionType_new. It replaces the - tp_dict member with a new instance of StgDict, and initializes the C - accessible fields somehow. + __metaclass__ will call the constructor StructUnionType_new. + It initializes the C accessible fields somehow. */ static PyCArgObject * -StructUnionType_paramfunc(CDataObject *self) +StructUnionType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; PyObject *obj; - StgDictObject *stgdict; void *ptr; if ((size_t)self->b_size > sizeof(void*)) { @@ -485,7 +602,6 @@ StructUnionType_paramfunc(CDataObject *self) /* Create a Python object which calls PyMem_Free(ptr) in its deallocator. The object will be destroyed at _ctypes_callproc() cleanup. */ - ctypes_state *st = GLOBAL_STATE(); PyTypeObject *tp = st->StructParam_Type; obj = tp->tp_alloc(tp, 0); if (obj == NULL) { @@ -501,117 +617,131 @@ StructUnionType_paramfunc(CDataObject *self) obj = Py_NewRef(self); } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) { Py_DECREF(obj); return NULL; } + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + Py_DECREF(obj); + return NULL; + } + assert(stginfo); /* Cannot be NULL for structure/union instances */ + parg->tag = 'V'; - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for structure/union instances */ - parg->pffi_type = &stgdict->ffi_type_pointer; + parg->pffi_type = &stginfo->ffi_type_pointer; parg->value.p = ptr; parg->size = self->b_size; parg->obj = obj; return parg; } -static PyObject * -StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isStruct) +static int +StructUnionType_init(PyObject *self, PyObject *args, PyObject *kwds, int isStruct) { - PyTypeObject *result; PyObject *fields; - StgDictObject *dict; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (!result) - return NULL; + PyObject *attrdict = PyType_GetDict((PyTypeObject *)self); + if (!attrdict) { + return -1; + } /* keep this for bw compatibility */ - int r = PyDict_Contains(result->tp_dict, &_Py_ID(_abstract_)); + int r = PyDict_Contains(attrdict, &_Py_ID(_abstract_)); if (r > 0) { - return (PyObject *)result; + Py_DECREF(attrdict); + return 0; } if (r < 0) { - Py_DECREF(result); - return NULL; + Py_DECREF(attrdict); + return -1; } - ctypes_state *st = GLOBAL_STATE(); - dict = (StgDictObject *)_PyObject_CallNoArgs((PyObject *)st->PyCStgDict_Type); - if (!dict) { - Py_DECREF(result); - return NULL; - } - if (!isStruct) { - dict->flags |= TYPEFLAG_HASUNION; - } - /* replace the class dict by our updated stgdict, which holds info - about storage requirements of the instances */ - if (-1 == PyDict_Update((PyObject *)dict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)dict); - return NULL; + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *info = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!info) { + Py_DECREF(attrdict); + return -1; } - Py_SETREF(result->tp_dict, (PyObject *)dict); - dict->format = _ctypes_alloc_format_string(NULL, "B"); - if (dict->format == NULL) { - Py_DECREF(result); - return NULL; + + info->format = _ctypes_alloc_format_string(NULL, "B"); + if (info->format == NULL) { + Py_DECREF(attrdict); + return -1; } - dict->paramfunc = StructUnionType_paramfunc; + info->paramfunc = StructUnionType_paramfunc; - if (PyDict_GetItemRef((PyObject *)dict, &_Py_ID(_fields_), &fields) < 0) { - Py_DECREF(result); - return NULL; + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_fields_), &fields) < 0) { + Py_DECREF(attrdict); + return -1; } + Py_CLEAR(attrdict); if (fields) { - if (PyObject_SetAttr((PyObject *)result, &_Py_ID(_fields_), fields) < 0) { - Py_DECREF(result); + if (PyObject_SetAttr(self, &_Py_ID(_fields_), fields) < 0) { Py_DECREF(fields); - return NULL; + return -1; } Py_DECREF(fields); - return (PyObject *)result; + return 0; } else { - StgDictObject *basedict = PyType_stgdict((PyObject *)result->tp_base); - - if (basedict == NULL) { - return (PyObject *)result; + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)((PyTypeObject *)self)->tp_base, + &baseinfo) < 0) { + return -1; } - /* copy base dict */ - if (-1 == PyCStgDict_clone(dict, basedict)) { - Py_DECREF(result); - return NULL; + if (baseinfo == NULL) { + return 0; + } + + /* copy base info */ + if (PyCStgInfo_clone(info, baseinfo) < 0) { + return -1; } - dict->flags &= ~DICTFLAG_FINAL; /* clear the 'final' flag in the subclass dict */ - basedict->flags |= DICTFLAG_FINAL; /* set the 'final' flag in the baseclass dict */ - return (PyObject *)result; + info->flags &= ~DICTFLAG_FINAL; /* clear the 'final' flag in the subclass info */ + baseinfo->flags |= DICTFLAG_FINAL; /* set the 'final' flag in the baseclass info */ } + return 0; } -static PyObject * -PyCStructType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCStructType_init(PyObject *self, PyObject *args, PyObject *kwds) { - return StructUnionType_new(type, args, kwds, 1); + return StructUnionType_init(self, args, kwds, 1); } -static PyObject * -UnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +UnionType_init(PyObject *self, PyObject *args, PyObject *kwds) { - return StructUnionType_new(type, args, kwds, 0); + return StructUnionType_init(self, args, kwds, 0); } -PyDoc_STRVAR(from_address_doc, -"C.from_address(integer) -> C instance\naccess a C instance at the specified address"); +/*[clinic input] +class _ctypes.CDataType "PyObject *" "clinic_state()->CType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=466a505a93d73156]*/ + + +/*[clinic input] +_ctypes.CDataType.from_address as CDataType_from_address + + type: self + cls: defining_class + value: object + / + +C.from_address(integer) -> C instance + +Access a C instance at the specified address. +[clinic start generated code]*/ static PyObject * -CDataType_from_address(PyObject *type, PyObject *value) +CDataType_from_address_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=5be4a7c0d9aa6c74 input=827a22cefe380c01]*/ { void *buf; if (!PyLong_Check(value)) { @@ -622,32 +752,45 @@ CDataType_from_address(PyObject *type, PyObject *value) buf = (void *)PyLong_AsVoidPtr(value); if (PyErr_Occurred()) return NULL; - return PyCData_AtAddress(type, buf); + ctypes_state *st = get_module_state_by_class(cls); + return PyCData_AtAddress(st, type, buf); } -PyDoc_STRVAR(from_buffer_doc, -"C.from_buffer(object, offset=0) -> C instance\ncreate a C instance from a writeable buffer"); - static int KeepRef(CDataObject *target, Py_ssize_t index, PyObject *keep); +/*[clinic input] +_ctypes.CDataType.from_buffer as CDataType_from_buffer + + type: self + cls: defining_class + obj: object + offset: Py_ssize_t = 0 + / + +C.from_buffer(object, offset=0) -> C instance + +Create a C instance from a writeable buffer. +[clinic start generated code]*/ + static PyObject * -CDataType_from_buffer(PyObject *type, PyObject *args) +CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, + Py_ssize_t offset) +/*[clinic end generated code: output=57604e99635abd31 input=0f36cedd105ca28d]*/ { - PyObject *obj; PyObject *mv; PyObject *result; Py_buffer *buffer; - Py_ssize_t offset = 0; - StgDictObject *dict = PyType_stgdict(type); - if (!dict) { - PyErr_SetString(PyExc_TypeError, "abstract class"); + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; } - - if (!PyArg_ParseTuple(args, "O|n:from_buffer", &obj, &offset)) + if (!info) { + PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; + } mv = PyMemoryView_FromObject(obj); if (mv == NULL) @@ -676,11 +819,11 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return NULL; } - if (dict->size > buffer->len - offset) { + if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small " "(%zd instead of at least %zd bytes)", - buffer->len, dict->size + offset); + buffer->len, info->size + offset); Py_DECREF(mv); return NULL; } @@ -691,7 +834,7 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return NULL; } - result = PyCData_AtAddress(type, (char *)buffer->buf + offset); + result = PyCData_AtAddress(st, type, (char *)buffer->buf + offset); if (result == NULL) { Py_DECREF(mv); return NULL; @@ -705,72 +848,94 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return result; } -PyDoc_STRVAR(from_buffer_copy_doc, -"C.from_buffer_copy(object, offset=0) -> C instance\ncreate a C instance from a readable buffer"); +static inline PyObject * +generic_pycdata_new(ctypes_state *st, + PyTypeObject *type, PyObject *args, PyObject *kwds); static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +/*[clinic input] +_ctypes.CDataType.from_buffer_copy as CDataType_from_buffer_copy + + type: self + cls: defining_class + buffer: Py_buffer + offset: Py_ssize_t = 0 + / + +C.from_buffer_copy(object, offset=0) -> C instance + +Create a C instance from a readable buffer. +[clinic start generated code]*/ + static PyObject * -CDataType_from_buffer_copy(PyObject *type, PyObject *args) +CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, + Py_buffer *buffer, Py_ssize_t offset) +/*[clinic end generated code: output=c8fc62b03e5cc6fa input=2a81e11b765a6253]*/ { - Py_buffer buffer; - Py_ssize_t offset = 0; PyObject *result; - StgDictObject *dict = PyType_stgdict(type); - if (!dict) { - PyErr_SetString(PyExc_TypeError, "abstract class"); + + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; } - - if (!PyArg_ParseTuple(args, "y*|n:from_buffer_copy", &buffer, &offset)) + if (!info) { + PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; + } if (offset < 0) { PyErr_SetString(PyExc_ValueError, "offset cannot be negative"); - PyBuffer_Release(&buffer); return NULL; } - if (dict->size > buffer.len - offset) { + if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small (%zd instead of at least %zd bytes)", - buffer.len, dict->size + offset); - PyBuffer_Release(&buffer); + buffer->len, info->size + offset); return NULL; } if (PySys_Audit("ctypes.cdata/buffer", "nnn", - (Py_ssize_t)buffer.buf, buffer.len, offset) < 0) { - PyBuffer_Release(&buffer); + (Py_ssize_t)buffer->buf, buffer->len, offset) < 0) { return NULL; } - result = GenericPyCData_new((PyTypeObject *)type, NULL, NULL); + result = generic_pycdata_new(st, (PyTypeObject *)type, NULL, NULL); if (result != NULL) { memcpy(((CDataObject *)result)->b_ptr, - (char *)buffer.buf + offset, dict->size); + (char *)buffer->buf + offset, info->size); } - PyBuffer_Release(&buffer); return result; } -PyDoc_STRVAR(in_dll_doc, -"C.in_dll(dll, name) -> C instance\naccess a C instance in a dll"); +/*[clinic input] +_ctypes.CDataType.in_dll as CDataType_in_dll + + type: self + cls: defining_class + dll: object + name: str + / + +C.in_dll(dll, name) -> C instance + +Access a C instance in a dll. +[clinic start generated code]*/ static PyObject * -CDataType_in_dll(PyObject *type, PyObject *args) +CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, + const char *name) +/*[clinic end generated code: output=d0e5c43b66bfa21f input=f85bf281477042b4]*/ { - PyObject *dll; - char *name; PyObject *obj; void *handle; void *address; - if (!PyArg_ParseTuple(args, "Os:in_dll", &dll, &name)) - return NULL; - if (PySys_Audit("ctypes.dlsym", "O", args) < 0) { + if (PySys_Audit("ctypes.dlsym", "Os", dll, name) < 0) { return NULL; } @@ -815,14 +980,24 @@ CDataType_in_dll(PyObject *type, PyObject *args) return NULL; } #endif - return PyCData_AtAddress(type, address); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + return PyCData_AtAddress(st, type, address); } -PyDoc_STRVAR(from_param_doc, -"Convert a Python object into a function call parameter."); +/*[clinic input] +_ctypes.CDataType.from_param as CDataType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ static PyObject * -CDataType_from_param(PyObject *type, PyObject *value) +CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=8da9e34263309f9e input=275a52c4899ddff0]*/ { PyObject *as_parameter; int res = PyObject_IsInstance(value, type); @@ -831,18 +1006,19 @@ CDataType_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); if (PyCArg_CheckExact(st, value)) { PyCArgObject *p = (PyCArgObject *)value; PyObject *ob = p->obj; const char *ob_name; - StgDictObject *dict; - dict = PyType_stgdict(type); - + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } /* If we got a PyCArgObject, we must check if the object packed in it - is an instance of the type's dict->proto */ - if(dict && ob) { - res = PyObject_IsInstance(ob, dict->proto); + is an instance of the type's info->proto */ + if(info && ob) { + res = PyObject_IsInstance(ob, info->proto); if (res == -1) return NULL; if (res) { @@ -860,7 +1036,7 @@ CDataType_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = CDataType_from_param(type, as_parameter); + value = CDataType_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -872,87 +1048,63 @@ CDataType_from_param(PyObject *type, PyObject *value) } static PyMethodDef CDataType_methods[] = { - { "from_param", CDataType_from_param, METH_O, from_param_doc }, - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc }, + CDATATYPE_FROM_PARAM_METHODDEF + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF { NULL, NULL }, }; static PyObject * -CDataType_repeat(PyObject *self, Py_ssize_t length) +CType_Type_repeat(PyObject *self, Py_ssize_t length) { if (length < 0) return PyErr_Format(PyExc_ValueError, "Array length must be >= 0, not %zd", length); - return PyCArrayType_from_ctype(self, length); + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + return PyCArrayType_from_ctype(st, self, length); } static int -CDataType_clear(PyTypeObject *self) +_structunion_setattro(PyObject *self, PyObject *key, PyObject *value, int is_struct) { - StgDictObject *dict = PyType_stgdict((PyObject *)self); - if (dict) - Py_CLEAR(dict->proto); - return PyType_Type.tp_clear((PyObject *)self); -} - -static int -CDataType_traverse(PyTypeObject *self, visitproc visit, void *arg) -{ - StgDictObject *dict = PyType_stgdict((PyObject *)self); - if (dict) { - Py_VISIT(dict->proto); + /* XXX Should we disallow deleting _fields_? */ + if (PyUnicode_Check(key) + && _PyUnicode_EqualToASCIIString(key, "_fields_")) + { + if (PyCStructUnionType_update_stginfo(self, value, is_struct) < 0) { + return -1; + } } - Py_VISIT(Py_TYPE(self)); - return PyType_Type.tp_traverse((PyObject *)self, visit, arg); + + return PyType_Type.tp_setattro(self, key, value); } static int PyCStructType_setattro(PyObject *self, PyObject *key, PyObject *value) { - /* XXX Should we disallow deleting _fields_? */ - if (-1 == PyType_Type.tp_setattro(self, key, value)) - return -1; - - if (value && PyUnicode_Check(key) && - _PyUnicode_EqualToASCIIString(key, "_fields_")) - return PyCStructUnionType_update_stgdict(self, value, 1); - return 0; + return _structunion_setattro(self, key, value, 1); } - static int UnionType_setattro(PyObject *self, PyObject *key, PyObject *value) { - /* XXX Should we disallow deleting _fields_? */ - if (-1 == PyObject_GenericSetAttr(self, key, value)) - return -1; - - if (PyUnicode_Check(key) && - _PyUnicode_EqualToASCIIString(key, "_fields_")) - return PyCStructUnionType_update_stgdict(self, value, 0); - return 0; + return _structunion_setattro(self, key, value, 0); } static PyType_Slot pycstruct_type_slots[] = { {Py_tp_setattro, PyCStructType_setattro}, {Py_tp_doc, PyDoc_STR("metatype for the CData Objects")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, PyCStructType_new}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCStructType_init}, {0, NULL}, }; -PyType_Spec pycstruct_type_spec = { +static PyType_Spec pycstruct_type_spec = { .name = "_ctypes.PyCStructType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycstruct_type_slots, }; @@ -960,19 +1112,14 @@ PyType_Spec pycstruct_type_spec = { static PyType_Slot union_type_slots[] = { {Py_tp_setattro, UnionType_setattro}, {Py_tp_doc, PyDoc_STR("metatype for the Union Objects")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, UnionType_new}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, UnionType_init}, {0, NULL}, }; static PyType_Spec union_type_spec = { .name = "_ctypes.UnionType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = union_type_slots, }; @@ -993,30 +1140,40 @@ size property/method, and the sequence protocol. */ +/*[clinic input] +class _ctypes.PyCPointerType "PyObject *" "clinic_state()->PyCPointerType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c45e96c1f7645ab7]*/ + + static int -PyCPointerType_SetProto(StgDictObject *stgdict, PyObject *proto) +PyCPointerType_SetProto(ctypes_state *st, StgInfo *stginfo, PyObject *proto) { if (!proto || !PyType_Check(proto)) { PyErr_SetString(PyExc_TypeError, "_type_ must be a type"); return -1; } - if (!PyType_stgdict(proto)) { + StgInfo *info; + if (PyStgInfo_FromType(st, proto, &info) < 0) { + return -1; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "_type_ must have storage info"); return -1; } Py_INCREF(proto); - Py_XSETREF(stgdict->proto, proto); + Py_XSETREF(stginfo->proto, proto); return 0; } static PyCArgObject * -PyCPointerType_paramfunc(CDataObject *self) +PyCPointerType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -1027,127 +1184,145 @@ PyCPointerType_paramfunc(CDataObject *self) return parg; } -static PyObject * -PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCPointerType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; PyObject *proto; PyObject *typedict; - typedict = PyTuple_GetItem(args, 2); if (!typedict) { - return NULL; + return -1; } + /* - stgdict items size, align, length contain info about pointers itself, - stgdict->proto has info about the pointed to type! + stginfo items size, align, length contain info about pointers itself, + stginfo->proto has info about the pointed to type! */ - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { - return NULL; + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { + return -1; } - stgdict->size = sizeof(void *); - stgdict->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; - stgdict->length = 1; - stgdict->ffi_type_pointer = ffi_type_pointer; - stgdict->paramfunc = PyCPointerType_paramfunc; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->size = sizeof(void *); + stginfo->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; + stginfo->length = 1; + stginfo->ffi_type_pointer = ffi_type_pointer; + stginfo->paramfunc = PyCPointerType_paramfunc; + stginfo->flags |= TYPEFLAG_ISPOINTER; if (PyDict_GetItemRef(typedict, &_Py_ID(_type_), &proto) < 0) { - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; } if (proto) { - StgDictObject *itemdict; const char *current_format; - if (-1 == PyCPointerType_SetProto(stgdict, proto)) { + if (PyCPointerType_SetProto(st, stginfo, proto) < 0) { Py_DECREF(proto); - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; + } + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + Py_DECREF(proto); + return -1; } - itemdict = PyType_stgdict(proto); - /* PyCPointerType_SetProto has verified proto has a stgdict. */ - assert(itemdict); - /* If itemdict->format is NULL, then this is a pointer to an + /* PyCPointerType_SetProto has verified proto has a stginfo. */ + assert(iteminfo); + /* If iteminfo->format is NULL, then this is a pointer to an incomplete type. We create a generic format string 'pointer to bytes' in this case. XXX Better would be to fix the format string later... */ - current_format = itemdict->format ? itemdict->format : "B"; - if (itemdict->shape != NULL) { + current_format = iteminfo->format ? iteminfo->format : "B"; + if (iteminfo->shape != NULL) { /* pointer to an array: the shape needs to be prefixed */ - stgdict->format = _ctypes_alloc_format_string_with_shape( - itemdict->ndim, itemdict->shape, "&", current_format); + stginfo->format = _ctypes_alloc_format_string_with_shape( + iteminfo->ndim, iteminfo->shape, "&", current_format); } else { - stgdict->format = _ctypes_alloc_format_string("&", current_format); + stginfo->format = _ctypes_alloc_format_string("&", current_format); } Py_DECREF(proto); - if (stgdict->format == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; + if (stginfo->format == NULL) { + return -1; } } - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); - - return (PyObject *)result; + return 0; } +/*[clinic input] +_ctypes.PyCPointerType.set_type as PyCPointerType_set_type + + self: self(type="PyTypeObject *") + cls: defining_class + type: object + / +[clinic start generated code]*/ static PyObject * -PyCPointerType_set_type(PyTypeObject *self, PyObject *type) +PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, + PyObject *type) +/*[clinic end generated code: output=51459d8f429a70ac input=67e1e8df921f123e]*/ { - StgDictObject *dict; - - - dict = PyType_stgdict((PyObject *)self); - if (!dict) { + PyObject *attrdict = PyType_GetDict(self); + if (!attrdict) { + return NULL; + } + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)self, &info) < 0) { + Py_DECREF(attrdict); + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); + Py_DECREF(attrdict); return NULL; } - if (-1 == PyCPointerType_SetProto(dict, type)) + if (PyCPointerType_SetProto(st, info, type) < 0) { + Py_DECREF(attrdict); return NULL; + } - if (-1 == PyDict_SetItem((PyObject *)dict, &_Py_ID(_type_), type)) + if (-1 == PyDict_SetItem(attrdict, &_Py_ID(_type_), type)) { + Py_DECREF(attrdict); return NULL; + } + Py_DECREF(attrdict); Py_RETURN_NONE; } -static PyObject *_byref(PyObject *); +static PyObject *_byref(ctypes_state *, PyObject *); + +/*[clinic input] +_ctypes.PyCPointerType.from_param as PyCPointerType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ static PyObject * -PyCPointerType_from_param(PyObject *type, PyObject *value) +PyCPointerType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=a4b32d929aabaf64 input=6c231276e3997884]*/ { - StgDictObject *typedict; - if (value == Py_None) { /* ConvParam will convert to a NULL pointer later */ return Py_NewRef(value); } - typedict = PyType_stgdict(type); - if (!typedict) { + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *typeinfo; + if (PyStgInfo_FromType(st, type, &typeinfo) < 0) { + return NULL; + } + if (!typeinfo) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; @@ -1156,24 +1331,27 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) /* If we expect POINTER(), but receive a instance, accept it by calling byref(). */ - switch (PyObject_IsInstance(value, typedict->proto)) { + assert(typeinfo->proto); + switch (PyObject_IsInstance(value, typeinfo->proto)) { case 1: Py_INCREF(value); /* _byref steals a refcount */ - return _byref(value); + return _byref(st, value); case -1: return NULL; default: break; } - ctypes_state *st = GLOBAL_STATE(); if (PointerObject_Check(st, value) || ArrayObject_Check(st, value)) { /* Array instances are also pointers when the item types are the same. */ - StgDictObject *v = PyObject_stgdict(value); + StgInfo *v; + if (PyStgInfo_FromObject(st, value, &v) < 0) { + return NULL; + } assert(v); /* Cannot be NULL for pointer or array objects */ - int ret = PyObject_IsSubclass(v->proto, typedict->proto); + int ret = PyObject_IsSubclass(v->proto, typeinfo->proto); if (ret < 0) { return NULL; } @@ -1181,34 +1359,29 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } } - return CDataType_from_param(type, value); + return CDataType_from_param_impl(type, cls, value); } static PyMethodDef PyCPointerType_methods[] = { - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc}, - { "from_param", (PyCFunction)PyCPointerType_from_param, METH_O, from_param_doc}, - { "set_type", (PyCFunction)PyCPointerType_set_type, METH_O }, + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF + PYCPOINTERTYPE_FROM_PARAM_METHODDEF + PYCPOINTERTYPE_SET_TYPE_METHODDEF { NULL, NULL }, }; static PyType_Slot pycpointer_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the Pointer Objects")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, PyCPointerType_methods}, - {Py_tp_new, PyCPointerType_new}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCPointerType_init}, {0, NULL}, }; static PyType_Spec pycpointer_type_spec = { .name = "_ctypes.PyCPointerType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycpointer_type_slots, }; @@ -1219,7 +1392,7 @@ static PyType_Spec pycpointer_type_spec = { PyCArrayType_Type */ /* - PyCArrayType_new ensures that the new Array subclass created has a _length_ + PyCArrayType_init ensures that the new Array subclass created has a _length_ attribute, and a _type_ attribute. */ @@ -1391,9 +1564,9 @@ add_getset(PyTypeObject *type, PyGetSetDef *gsp) } static PyCArgObject * -PyCArrayType_paramfunc(CDataObject *self) +PyCArrayType_paramfunc(ctypes_state *st, CDataObject *self) { - PyCArgObject *p = PyCArgObject_new(); + PyCArgObject *p = PyCArgObject_new(st); if (p == NULL) return NULL; p->tag = 'P'; @@ -1403,28 +1576,18 @@ PyCArrayType_paramfunc(CDataObject *self) return p; } -static PyObject * -PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCArrayType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; - StgDictObject *itemdict; PyObject *length_attr, *type_attr; Py_ssize_t length; Py_ssize_t itemsize, itemalign; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) - return NULL; - /* Initialize these variables to NULL so that we can simplify error handling by using Py_XDECREF. */ - stgdict = NULL; type_attr = NULL; - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_length_), &length_attr) < 0) { + if (PyObject_GetOptionalAttr(self, &_Py_ID(_length_), &length_attr) < 0) { goto error; } if (!length_attr) { @@ -1440,7 +1603,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - if (_PyLong_Sign(length_attr) == -1) { + if (_PyLong_IsNegative((PyLongObject *)length_attr)) { Py_DECREF(length_attr); PyErr_SetString(PyExc_ValueError, "The '_length_' attribute must not be negative"); @@ -1457,7 +1620,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_type_), &type_attr) < 0) { + if (PyObject_GetOptionalAttr(self, &_Py_ID(_type_), &type_attr) < 0) { goto error; } if (!type_attr) { @@ -1466,99 +1629,89 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject*)self); + if (!stginfo) { + goto error; + } + + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, type_attr, &iteminfo) < 0) { goto error; } - itemdict = PyType_stgdict(type_attr); - if (!itemdict) { + if (!iteminfo) { PyErr_SetString(PyExc_TypeError, "_type_ must have storage info"); goto error; } - assert(itemdict->format); - stgdict->format = _ctypes_alloc_format_string(NULL, itemdict->format); - if (stgdict->format == NULL) + assert(iteminfo->format); + stginfo->format = _ctypes_alloc_format_string(NULL, iteminfo->format); + if (stginfo->format == NULL) goto error; - stgdict->ndim = itemdict->ndim + 1; - stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stgdict->ndim); - if (stgdict->shape == NULL) { + stginfo->ndim = iteminfo->ndim + 1; + stginfo->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stginfo->ndim); + if (stginfo->shape == NULL) { PyErr_NoMemory(); goto error; } - stgdict->shape[0] = length; - if (stgdict->ndim > 1) { - memmove(&stgdict->shape[1], itemdict->shape, - sizeof(Py_ssize_t) * (stgdict->ndim - 1)); + stginfo->shape[0] = length; + if (stginfo->ndim > 1) { + memmove(&stginfo->shape[1], iteminfo->shape, + sizeof(Py_ssize_t) * (stginfo->ndim - 1)); } - itemsize = itemdict->size; + itemsize = iteminfo->size; if (itemsize != 0 && length > PY_SSIZE_T_MAX / itemsize) { PyErr_SetString(PyExc_OverflowError, "array too large"); goto error; } - itemalign = itemdict->align; + itemalign = iteminfo->align; - if (itemdict->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) - stgdict->flags |= TYPEFLAG_HASPOINTER; + if (iteminfo->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + stginfo->flags |= TYPEFLAG_HASPOINTER; - stgdict->size = itemsize * length; - stgdict->align = itemalign; - stgdict->length = length; - stgdict->proto = type_attr; + stginfo->size = itemsize * length; + stginfo->align = itemalign; + stginfo->length = length; + stginfo->proto = type_attr; type_attr = NULL; - stgdict->paramfunc = &PyCArrayType_paramfunc; + stginfo->paramfunc = &PyCArrayType_paramfunc; /* Arrays are passed as pointers to function calls. */ - stgdict->ffi_type_pointer = ffi_type_pointer; - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) - goto error; - Py_SETREF(result->tp_dict, (PyObject *)stgdict); /* steal the reference */ - stgdict = NULL; + stginfo->ffi_type_pointer = ffi_type_pointer; /* Special case for character arrays. A permanent annoyance: char arrays are also strings! */ - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - if (-1 == add_getset(result, CharArray_getsets)) + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (-1 == add_getset((PyTypeObject*)self, CharArray_getsets)) goto error; } - else if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { - if (-1 == add_getset(result, WCharArray_getsets)) + else if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (-1 == add_getset((PyTypeObject*)self, WCharArray_getsets)) goto error; } - return (PyObject *)result; + return 0; error: - Py_XDECREF((PyObject*)stgdict); Py_XDECREF(type_attr); - Py_DECREF(result); - return NULL; + return -1; } static PyType_Slot pycarray_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the Array Objects")}, - {Py_tp_traverse, CDataType_traverse}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, PyCArrayType_new}, - {Py_tp_clear, CDataType_clear}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCArrayType_init}, {0, NULL}, }; static PyType_Spec pycarray_type_spec = { .name = "_ctypes.PyCArrayType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycarray_type_slots, }; @@ -1569,26 +1722,61 @@ static PyType_Spec pycarray_type_spec = { */ /* -PyCSimpleType_new ensures that the new Simple_Type subclass created has a valid +PyCSimpleType_init ensures that the new Simple_Type subclass created has a valid _type_ attribute. */ +/*[clinic input] +class _ctypes.PyCSimpleType "PyObject *" "clinic_state()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d5a45772668e7f49]*/ + +/*[clinic input] +class _ctypes.c_wchar_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=468de7283d622d47]*/ + +/*[clinic input] +class _ctypes.c_char_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e750865616e7dcea]*/ + +/*[clinic input] +class _ctypes.c_void_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=dd4d9646c56f43a9]*/ + +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +static const char SIMPLE_TYPE_CHARS[] = "cbBhHiIlLdCEFfuzZqQPXOv?g"; +#else static const char SIMPLE_TYPE_CHARS[] = "cbBhHiIlLdfuzZqQPXOv?g"; +#endif + +/*[clinic input] +_ctypes.c_wchar_p.from_param as c_wchar_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ static PyObject * -c_wchar_p_from_param(PyObject *type, PyObject *value) +c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=e453949a2f725a4c input=d322c7237a319607]*/ { PyObject *as_parameter; int res; if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = get_module_state_by_class(cls->tp_base); if (PyUnicode_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("Z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1606,22 +1794,31 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_wchar array instance or pointer(c_wchar(...)) */ - StgDictObject *dt = PyObject_stgdict(value); - StgDictObject *dict; - assert(dt); /* Cannot be NULL for pointer or array objects */ - dict = dt && dt->proto ? PyType_stgdict(dt->proto) : NULL; - if (dict && (dict->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { + StgInfo *it; + if (PyStgInfo_FromObject(st, value, &it) < 0) { + return NULL; + } + assert(it); /* Cannot be NULL for pointer or array objects */ + StgInfo *info = NULL; + if (it && it->proto) { + if (PyStgInfo_FromType(st, it->proto, &info) < 0) { + return NULL; + } + } + if (info && (info->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { return Py_NewRef(value); } } if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; - StgDictObject *dict = PyObject_stgdict(a->obj); - if (dict && (dict->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { + StgInfo *info; + if (PyStgInfo_FromObject(st, a->obj, &info) < 0) { + return NULL; + } + if (info && (info->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { return Py_NewRef(value); } } @@ -1630,7 +1827,7 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_wchar_p_from_param(type, as_parameter); + value = c_wchar_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -1640,19 +1837,30 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) return NULL; } +/*[clinic input] +_ctypes.c_char_p.from_param as c_char_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ + static PyObject * -c_char_p_from_param(PyObject *type, PyObject *value) +c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=219652ab7c174aa1 input=6cf0d1b6bb4ede11]*/ { PyObject *as_parameter; int res; if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = get_module_state_by_class(cls->tp_base); if (PyBytes_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1670,22 +1878,31 @@ c_char_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_char array instance or pointer(c_char(...)) */ - StgDictObject *dt = PyObject_stgdict(value); - StgDictObject *dict; - assert(dt); /* Cannot be NULL for pointer or array objects */ - dict = dt && dt->proto ? PyType_stgdict(dt->proto) : NULL; - if (dict && (dict->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { + StgInfo *it; + if (PyStgInfo_FromObject(st, value, &it) < 0) { + return NULL; + } + assert(it); /* Cannot be NULL for pointer or array objects */ + StgInfo *info = NULL; + if (it && it->proto) { + if (PyStgInfo_FromType(st, it->proto, &info) < 0) { + return NULL; + } + } + if (info && (info->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { return Py_NewRef(value); } } if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; - StgDictObject *dict = PyObject_stgdict(a->obj); - if (dict && (dict->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { + StgInfo *info; + if (PyStgInfo_FromObject(st, a->obj, &info) < 0) { + return NULL; + } + if (info && (info->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { return Py_NewRef(value); } } @@ -1694,7 +1911,7 @@ c_char_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_char_p_from_param(type, as_parameter); + value = c_char_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -1704,10 +1921,19 @@ c_char_p_from_param(PyObject *type, PyObject *value) return NULL; } +/*[clinic input] +_ctypes.c_void_p.from_param as c_void_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ + static PyObject * -c_void_p_from_param(PyObject *type, PyObject *value) +c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=984d0075b6038cc7 input=0e8b343fc19c77d4]*/ { - StgDictObject *stgd; PyObject *as_parameter; int res; @@ -1715,13 +1941,15 @@ c_void_p_from_param(PyObject *type, PyObject *value) if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = get_module_state_by_class(cls->tp_base); + /* Should probably allow buffer interface as well */ /* int, long */ if (PyLong_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("P"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1739,7 +1967,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1756,7 +1984,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("Z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1776,7 +2004,6 @@ c_void_p_from_param(PyObject *type, PyObject *value) /* c_void_p instances */ return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); /* ctypes array or pointer instance */ if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* Any array or pointer is accepted */ @@ -1795,7 +2022,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; PyCFuncPtrObject *func; func = (PyCFuncPtrObject *)value; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1806,18 +2033,21 @@ c_void_p_from_param(PyObject *type, PyObject *value) return (PyObject *)parg; } /* c_char_p, c_wchar_p */ - stgd = PyObject_stgdict(value); - if (stgd + StgInfo *stgi; + if (PyStgInfo_FromObject(st, value, &stgi) < 0) { + return NULL; + } + if (stgi && CDataObject_Check(st, value) - && stgd->proto - && PyUnicode_Check(stgd->proto)) + && stgi->proto + && PyUnicode_Check(stgi->proto)) { PyCArgObject *parg; - switch (PyUnicode_AsUTF8(stgd->proto)[0]) { + switch (PyUnicode_AsUTF8(stgi->proto)[0]) { case 'z': /* c_char_p */ case 'Z': /* c_wchar_p */ - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1833,7 +2063,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_void_p_from_param(type, as_parameter); + value = c_void_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -1843,37 +2073,37 @@ c_void_p_from_param(PyObject *type, PyObject *value) return NULL; } -static PyMethodDef c_void_p_method = { "from_param", c_void_p_from_param, METH_O }; -static PyMethodDef c_char_p_method = { "from_param", c_char_p_from_param, METH_O }; -static PyMethodDef c_wchar_p_method = { "from_param", c_wchar_p_from_param, METH_O }; +static PyMethodDef c_void_p_methods[] = {C_VOID_P_FROM_PARAM_METHODDEF {0}}; +static PyMethodDef c_char_p_methods[] = {C_CHAR_P_FROM_PARAM_METHODDEF {0}}; +static PyMethodDef c_wchar_p_methods[] = {C_WCHAR_P_FROM_PARAM_METHODDEF {0}}; -static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject *kwds, +static PyObject *CreateSwappedType(ctypes_state *st, PyTypeObject *type, + PyObject *args, PyObject *kwds, PyObject *proto, struct fielddesc *fmt) { PyTypeObject *result; - StgDictObject *stgdict; PyObject *name = PyTuple_GET_ITEM(args, 0); PyObject *newname; PyObject *swapped_args; - static PyObject *suffix; Py_ssize_t i; swapped_args = PyTuple_New(PyTuple_GET_SIZE(args)); if (!swapped_args) return NULL; - if (suffix == NULL) + if (st->swapped_suffix == NULL) { #ifdef WORDS_BIGENDIAN - suffix = PyUnicode_InternFromString("_le"); + st->swapped_suffix = PyUnicode_InternFromString("_le"); #else - suffix = PyUnicode_InternFromString("_be"); + st->swapped_suffix = PyUnicode_InternFromString("_be"); #endif - if (suffix == NULL) { + } + if (st->swapped_suffix == NULL) { Py_DECREF(swapped_args); return NULL; } - newname = PyUnicode_Concat(name, suffix); + newname = PyUnicode_Concat(name, st->swapped_suffix); if (newname == NULL) { Py_DECREF(swapped_args); return NULL; @@ -1893,51 +2123,43 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject if (result == NULL) return NULL; - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { + StgInfo *stginfo = PyStgInfo_Init(st, result); + if (!stginfo) { Py_DECREF(result); return NULL; } - stgdict->ffi_type_pointer = *fmt->pffi_type; - stgdict->align = fmt->pffi_type->alignment; - stgdict->length = 0; - stgdict->size = fmt->pffi_type->size; - stgdict->setfunc = fmt->setfunc_swapped; - stgdict->getfunc = fmt->getfunc_swapped; + stginfo->ffi_type_pointer = *fmt->pffi_type; + stginfo->align = fmt->pffi_type->alignment; + stginfo->length = 0; + stginfo->size = fmt->pffi_type->size; + stginfo->setfunc = fmt->setfunc_swapped; + stginfo->getfunc = fmt->getfunc_swapped; - stgdict->proto = Py_NewRef(proto); - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->proto = Py_NewRef(proto); return (PyObject *)result; } static PyCArgObject * -PyCSimpleType_paramfunc(CDataObject *self) +PyCSimpleType_paramfunc(ctypes_state *st, CDataObject *self) { - StgDictObject *dict; const char *fmt; PyCArgObject *parg; struct fielddesc *fd; - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for CDataObject instances */ - fmt = PyUnicode_AsUTF8(dict->proto); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + fmt = PyUnicode_AsUTF8(info->proto); assert(fmt); fd = _ctypes_get_fielddesc(fmt); assert(fd); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -1948,33 +2170,27 @@ PyCSimpleType_paramfunc(CDataObject *self) return parg; } -static PyObject * -PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; PyObject *proto; const char *proto_str; Py_ssize_t proto_len; PyMethodDef *ml; struct fielddesc *fmt; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) - return NULL; - - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_type_), &proto) < 0) { - return NULL; + if (PyType_Type.tp_init(self, args, kwds) < 0) { + return -1; + } + if (PyObject_GetOptionalAttr(self, &_Py_ID(_type_), &proto) < 0) { + return -1; } if (!proto) { PyErr_SetString(PyExc_AttributeError, "class must define a '_type_' attribute"); error: Py_XDECREF(proto); - Py_DECREF(result); - return NULL; + return -1; } if (PyUnicode_Check(proto)) { proto_str = PyUnicode_AsUTF8AndSize(proto, &proto_len); @@ -2005,71 +2221,72 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { goto error; } - stgdict->ffi_type_pointer = *fmt->pffi_type; - stgdict->align = fmt->pffi_type->alignment; - stgdict->length = 0; - stgdict->size = fmt->pffi_type->size; - stgdict->setfunc = fmt->setfunc; - stgdict->getfunc = fmt->getfunc; + + if (!fmt->pffi_type->elements) { + stginfo->ffi_type_pointer = *fmt->pffi_type; + } + else { + const size_t els_size = sizeof(fmt->pffi_type->elements); + stginfo->ffi_type_pointer.size = fmt->pffi_type->size; + stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; + stginfo->ffi_type_pointer.type = fmt->pffi_type->type; + stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); + memcpy(stginfo->ffi_type_pointer.elements, + fmt->pffi_type->elements, els_size); + } + stginfo->align = fmt->pffi_type->alignment; + stginfo->length = 0; + stginfo->size = fmt->pffi_type->size; + stginfo->setfunc = fmt->setfunc; + stginfo->getfunc = fmt->getfunc; #ifdef WORDS_BIGENDIAN - stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1); + stginfo->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1); #else - stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0); + stginfo->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0); #endif - if (stgdict->format == NULL) { - Py_DECREF(result); + if (stginfo->format == NULL) { Py_DECREF(proto); - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; } - stgdict->paramfunc = PyCSimpleType_paramfunc; + stginfo->paramfunc = PyCSimpleType_paramfunc; /* - if (result->tp_base != st->Simple_Type) { - stgdict->setfunc = NULL; - stgdict->getfunc = NULL; + if (self->tp_base != st->Simple_Type) { + stginfo->setfunc = NULL; + stginfo->getfunc = NULL; } */ /* This consumes the refcount on proto which we have */ - stgdict->proto = proto; - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->proto = proto; /* Install from_param class methods in ctypes base classes. Overrides the PyCSimpleType_from_param generic method. */ - if (result->tp_base == st->Simple_Type) { + if (((PyTypeObject *)self)->tp_base == st->Simple_Type) { switch (*proto_str) { case 'z': /* c_char_p */ - ml = &c_char_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + ml = c_char_p_methods; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'Z': /* c_wchar_p */ - ml = &c_wchar_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + ml = c_wchar_p_methods; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'P': /* c_void_p */ - ml = &c_void_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + ml = c_void_p_methods; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 's': case 'X': case 'O': ml = NULL; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; default: ml = NULL; @@ -2079,67 +2296,85 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (ml) { PyObject *meth; int x; - meth = PyDescr_NewClassMethod(result, ml); + meth = PyDescr_NewClassMethod((PyTypeObject*)self, ml); if (!meth) { - Py_DECREF(result); - return NULL; + return -1; } - x = PyDict_SetItemString(result->tp_dict, - ml->ml_name, - meth); + PyObject *name = PyUnicode_FromString(ml->ml_name); + if (name == NULL) { + Py_DECREF(meth); + return -1; + } + PyUnicode_InternInPlace(&name); + x = PyDict_SetItem(((PyTypeObject*)self)->tp_dict, name, meth); + Py_DECREF(name); Py_DECREF(meth); if (x == -1) { - Py_DECREF(result); - return NULL; + return -1; } } } + PyTypeObject *type = Py_TYPE(self); if (type == st->PyCSimpleType_Type && fmt->setfunc_swapped && fmt->getfunc_swapped) { - PyObject *swapped = CreateSwappedType(type, args, kwds, + PyObject *swapped = CreateSwappedType(st, type, args, kwds, proto, fmt); - StgDictObject *sw_dict; if (swapped == NULL) { - Py_DECREF(result); - return NULL; + return -1; + } + StgInfo *sw_info; + if (PyStgInfo_FromType(st, swapped, &sw_info) < 0) { + return -1; } - sw_dict = PyType_stgdict(swapped); + assert(sw_info); #ifdef WORDS_BIGENDIAN - PyObject_SetAttrString((PyObject *)result, "__ctype_le__", swapped); - PyObject_SetAttrString((PyObject *)result, "__ctype_be__", (PyObject *)result); - PyObject_SetAttrString(swapped, "__ctype_be__", (PyObject *)result); + PyObject_SetAttrString(self, "__ctype_le__", swapped); + PyObject_SetAttrString(self, "__ctype_be__", self); + PyObject_SetAttrString(swapped, "__ctype_be__", self); PyObject_SetAttrString(swapped, "__ctype_le__", swapped); /* We are creating the type for the OTHER endian */ - sw_dict->format = _ctypes_alloc_format_string("<", stgdict->format+1); + sw_info->format = _ctypes_alloc_format_string("<", stginfo->format+1); #else - PyObject_SetAttrString((PyObject *)result, "__ctype_be__", swapped); - PyObject_SetAttrString((PyObject *)result, "__ctype_le__", (PyObject *)result); - PyObject_SetAttrString(swapped, "__ctype_le__", (PyObject *)result); + PyObject_SetAttrString(self, "__ctype_be__", swapped); + PyObject_SetAttrString(self, "__ctype_le__", self); + PyObject_SetAttrString(swapped, "__ctype_le__", self); PyObject_SetAttrString(swapped, "__ctype_be__", swapped); /* We are creating the type for the OTHER endian */ - sw_dict->format = _ctypes_alloc_format_string(">", stgdict->format+1); + sw_info->format = _ctypes_alloc_format_string(">", stginfo->format+1); #endif Py_DECREF(swapped); if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; + return -1; } }; - return (PyObject *)result; + return 0; } /* * This is a *class method*. * Convert a parameter into something that ConvParam can handle. */ + +/*[clinic input] +_ctypes.PyCSimpleType.from_param as PyCSimpleType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ + static PyObject * -PyCSimpleType_from_param(PyObject *type, PyObject *value) +PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=8a8453d9663e3a2e input=61cc48ce3a87a570]*/ { - StgDictObject *dict; const char *fmt; PyCArgObject *parg; struct fielddesc *fd; @@ -2155,21 +2390,25 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } - dict = PyType_stgdict(type); - if (!dict) { + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } /* I think we can rely on this being a one-character string */ - fmt = PyUnicode_AsUTF8(dict->proto); + fmt = PyUnicode_AsUTF8(info->proto); assert(fmt); fd = _ctypes_get_fielddesc(fmt); assert(fd); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -2186,12 +2425,12 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - if (_Py_EnterRecursiveCall("while processing _as_parameter_")) { + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { Py_DECREF(as_parameter); Py_XDECREF(exc); return NULL; } - value = PyCSimpleType_from_param(type, as_parameter); + value = PyCSimpleType_from_param_impl(type, cls, as_parameter); _Py_LeaveRecursiveCall(); Py_DECREF(as_parameter); Py_XDECREF(exc); @@ -2207,29 +2446,24 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) } static PyMethodDef PyCSimpleType_methods[] = { - { "from_param", PyCSimpleType_from_param, METH_O, from_param_doc }, - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc}, + PYCSIMPLETYPE_FROM_PARAM_METHODDEF + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF { NULL, NULL }, }; static PyType_Slot pycsimple_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the PyCSimpleType Objects")}, {Py_tp_methods, PyCSimpleType_methods}, - {Py_tp_new, PyCSimpleType_new}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCSimpleType_init}, {0, NULL}, }; -PyType_Spec pycsimple_type_spec = { +static PyType_Spec pycsimple_type_spec = { .name = "_ctypes.PyCSimpleType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycsimple_type_slots, }; @@ -2240,7 +2474,7 @@ PyType_Spec pycsimple_type_spec = { */ static PyObject * -converters_from_argtypes(PyObject *ob) +converters_from_argtypes(ctypes_state *st, PyObject *ob) { PyObject *converters; Py_ssize_t i; @@ -2255,7 +2489,7 @@ converters_from_argtypes(PyObject *ob) Py_ssize_t nArgs = PyTuple_GET_SIZE(ob); if (nArgs > CTYPES_MAX_ARGCOUNT) { Py_DECREF(ob); - PyErr_Format(PyExc_ArgError, + PyErr_Format(st->PyExc_ArgError, "_argtypes_ has too many arguments (%zi), maximum is %i", nArgs, CTYPES_MAX_ARGCOUNT); return NULL; @@ -2292,10 +2526,17 @@ converters_from_argtypes(PyObject *ob) * not bitfields, the bitfields check is also being disabled as a * precaution. - StgDictObject *stgdict = PyType_stgdict(tp); + StgInfo *stginfo; + if (PyStgInfo_FromType(st, tp, &stginfo) < 0) { + return -1; + } - if (stgdict != NULL) { - if (stgdict->flags & TYPEFLAG_HASUNION) { + // TYPEFLAG_HASUNION and TYPEFLAG_HASBITFIELD used to be set + // if there were any unions/bitfields; + // if the check is re-enabled we either need to loop here or + // restore the flag + if (stginfo != NULL) { + if (stginfo->flags & TYPEFLAG_HASUNION) { Py_DECREF(converters); Py_DECREF(ob); if (!PyErr_Occurred()) { @@ -2306,7 +2547,7 @@ converters_from_argtypes(PyObject *ob) } return NULL; } - if (stgdict->flags & TYPEFLAG_HASBITFIELD) { + if (stginfo->flags & TYPEFLAG_HASBITFIELD) { Py_DECREF(converters); Py_DECREF(ob); if (!PyErr_Occurred()) { @@ -2338,19 +2579,19 @@ converters_from_argtypes(PyObject *ob) } static int -make_funcptrtype_dict(StgDictObject *stgdict) +make_funcptrtype_dict(ctypes_state *st, PyObject *attrdict, StgInfo *stginfo) { PyObject *ob; PyObject *converters = NULL; - stgdict->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; - stgdict->length = 1; - stgdict->size = sizeof(void *); - stgdict->setfunc = NULL; - stgdict->getfunc = NULL; - stgdict->ffi_type_pointer = ffi_type_pointer; + stginfo->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; + stginfo->length = 1; + stginfo->size = sizeof(void *); + stginfo->setfunc = NULL; + stginfo->getfunc = NULL; + stginfo->ffi_type_pointer = ffi_type_pointer; - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_flags_), &ob) < 0) { + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_flags_), &ob) < 0) { return -1; } if (!ob || !PyLong_Check(ob)) { @@ -2359,42 +2600,46 @@ make_funcptrtype_dict(StgDictObject *stgdict) Py_XDECREF(ob); return -1; } - stgdict->flags = PyLong_AsUnsignedLongMask(ob) | TYPEFLAG_ISPOINTER; + stginfo->flags = PyLong_AsUnsignedLongMask(ob) | TYPEFLAG_ISPOINTER; Py_DECREF(ob); /* _argtypes_ is optional... */ - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_argtypes_), &ob) < 0) { + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_argtypes_), &ob) < 0) { return -1; } if (ob) { - converters = converters_from_argtypes(ob); + converters = converters_from_argtypes(st, ob); if (!converters) { Py_DECREF(ob); return -1; } - stgdict->argtypes = ob; - stgdict->converters = converters; + stginfo->argtypes = ob; + stginfo->converters = converters; } - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_restype_), &ob) < 0) { + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_restype_), &ob) < 0) { return -1; } if (ob) { - if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + return -1; + } + if (ob != Py_None && !info && !PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "_restype_ must be a type, a callable, or None"); Py_DECREF(ob); return -1; } - stgdict->restype = ob; + stginfo->restype = ob; if (PyObject_GetOptionalAttr(ob, &_Py_ID(_check_retval_), - &stgdict->checker) < 0) + &stginfo->checker) < 0) { return -1; } } /* XXX later, maybe. - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py _ID(_errcheck_), &ob) < 0) { + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py _ID(_errcheck_), &ob) < 0) { return -1; } if (ob) { @@ -2404,18 +2649,18 @@ make_funcptrtype_dict(StgDictObject *stgdict) Py_DECREF(ob); return -1; } - stgdict->errcheck = ob; + stginfo->errcheck = ob; } */ return 0; } static PyCArgObject * -PyCFuncPtrType_paramfunc(CDataObject *self) +PyCFuncPtrType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -2426,71 +2671,55 @@ PyCFuncPtrType_paramfunc(CDataObject *self) return parg; } -static PyObject * -PyCFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCFuncPtrType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; + PyObject *attrdict = PyType_GetDict((PyTypeObject *)self); + if (!attrdict) { + return -1; + } - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { - return NULL; + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { + Py_DECREF(attrdict); + return -1; } - stgdict->paramfunc = PyCFuncPtrType_paramfunc; + + stginfo->paramfunc = PyCFuncPtrType_paramfunc; + /* We do NOT expose the function signature in the format string. It is impossible, generally, because the only requirement for the argtypes items is that they have a .from_param method - we do not know the types of the arguments (although, in practice, most argtypes would be a ctypes type). */ - stgdict->format = _ctypes_alloc_format_string(NULL, "X{}"); - if (stgdict->format == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - stgdict->flags |= TYPEFLAG_ISPOINTER; - - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - - /* replace the class dict by our updated storage dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; + stginfo->format = _ctypes_alloc_format_string(NULL, "X{}"); + if (stginfo->format == NULL) { + Py_DECREF(attrdict); + return -1; } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->flags |= TYPEFLAG_ISPOINTER; - if (-1 == make_funcptrtype_dict(stgdict)) { - Py_DECREF(result); - return NULL; + if (make_funcptrtype_dict(st, attrdict, stginfo) < 0) { + Py_DECREF(attrdict); + return -1; } - return (PyObject *)result; + Py_DECREF(attrdict); + return 0; } static PyType_Slot pycfuncptr_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for C function pointers")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, PyCFuncPtrType_new}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCFuncPtrType_init}, {0, NULL}, }; static PyType_Spec pycfuncptr_type_spec = { .name = "_ctypes.PyCFuncPtrType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycfuncptr_type_slots, }; @@ -2604,11 +2833,20 @@ KeepRef(CDataObject *target, Py_ssize_t index, PyObject *keep) /* PyCData_Type */ + +/*[clinic input] +class _ctypes.PyCData "PyObject *" "clinic_state()->PyCData_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ac13df38dee3c22c]*/ + + static int PyCData_traverse(CDataObject *self, visitproc visit, void *arg) { Py_VISIT(self->b_objects); Py_VISIT((PyObject *)self->b_base); + PyTypeObject *type = Py_TYPE(self); + Py_VISIT(type); return 0; } @@ -2627,8 +2865,11 @@ PyCData_clear(CDataObject *self) static void PyCData_dealloc(PyObject *self) { + PyTypeObject *type = Py_TYPE(self); + PyObject_GC_UnTrack(self); PyCData_clear((CDataObject *)self); - Py_TYPE(self)->tp_free(self); + type->tp_free(self); + Py_DECREF(type); } static PyMemberDef PyCData_members[] = { @@ -2646,19 +2887,20 @@ static PyMemberDef PyCData_members[] = { /* Find the innermost type of an array type, returning a borrowed reference */ static PyObject * -PyCData_item_type(PyObject *type) +PyCData_item_type(ctypes_state *st, PyObject *type) { - ctypes_state *st = GLOBAL_STATE(); if (PyCArrayTypeObject_Check(st, type)) { - StgDictObject *stg_dict; PyObject *elem_type; /* asserts used here as these are all guaranteed by construction */ - stg_dict = PyType_stgdict(type); - assert(stg_dict); - elem_type = stg_dict->proto; + StgInfo *stg_info; + if (PyStgInfo_FromType(st, type, &stg_info) < 0) { + return NULL; + } + assert(stg_info); + elem_type = stg_info->proto; assert(elem_type); - return PyCData_item_type(elem_type); + return PyCData_item_type(st, elem_type); } else { return type; @@ -2669,32 +2911,42 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; - StgDictObject *dict = PyObject_stgdict(myself); - PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); - StgDictObject *item_dict = PyType_stgdict(item_type); + + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *info; + if (PyStgInfo_FromObject(st, myself, &info) < 0) { + return -1; + } + assert(info); + + PyObject *item_type = PyCData_item_type(st, (PyObject*)Py_TYPE(myself)); + if (item_type == NULL) { + return 0; + } if (view == NULL) return 0; + StgInfo *item_info; + if (PyStgInfo_FromType(st, item_type, &item_info) < 0) { + return -1; + } + assert(item_info); + view->buf = self->b_ptr; view->obj = Py_NewRef(myself); view->len = self->b_size; view->readonly = 0; /* use default format character if not set */ - view->format = dict->format ? dict->format : "B"; - view->ndim = dict->ndim; - view->shape = dict->shape; - view->itemsize = item_dict->size; + view->format = info->format ? info->format : "B"; + view->ndim = info->ndim; + view->shape = info->shape; + view->itemsize = item_info->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; return 0; } -static PyBufferProcs PyCData_as_buffer = { - PyCData_NewGetBuffer, - NULL, -}; - /* * CData objects are mutable, so they cannot be hashable! */ @@ -2705,12 +2957,28 @@ PyCData_nohash(PyObject *self) return -1; } +/*[clinic input] +_ctypes.PyCData.__reduce__ as PyCData_reduce + + myself: self + cls: defining_class + / +[clinic start generated code]*/ + static PyObject * -PyCData_reduce(PyObject *myself, PyObject *args) +PyCData_reduce_impl(PyObject *myself, PyTypeObject *cls) +/*[clinic end generated code: output=1a025ccfdd8c935d input=34097a5226ea63c1]*/ { CDataObject *self = (CDataObject *)myself; - if (PyObject_stgdict(myself)->flags & (TYPEFLAG_ISPOINTER|TYPEFLAG_HASPOINTER)) { + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromObject(st, myself, &info) < 0) { + return NULL; + } + assert(info); + + if (info->flags & (TYPEFLAG_ISPOINTER|TYPEFLAG_HASPOINTER)) { PyErr_SetString(PyExc_ValueError, "ctypes objects containing pointers cannot be pickled"); return NULL; @@ -2719,7 +2987,7 @@ PyCData_reduce(PyObject *myself, PyObject *args) if (dict == NULL) { return NULL; } - return Py_BuildValue("O(O(NN))", _unpickle, Py_TYPE(myself), dict, + return Py_BuildValue("O(O(NN))", st->_unpickle, Py_TYPE(myself), dict, PyBytes_FromStringAndSize(self->b_ptr, self->b_size)); } @@ -2768,56 +3036,35 @@ PyCData_from_outparam(PyObject *self, PyObject *args) static PyMethodDef PyCData_methods[] = { { "__ctypes_from_outparam__", PyCData_from_outparam, METH_NOARGS, }, - { "__reduce__", PyCData_reduce, METH_NOARGS, }, + PYCDATA_REDUCE_METHODDEF { "__setstate__", PyCData_setstate, METH_VARARGS, }, { NULL, NULL }, }; -PyTypeObject PyCData_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._CData", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - PyCData_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - PyCData_nohash, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCData_methods, /* tp_methods */ - PyCData_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycdata_slots[] = { + {Py_tp_dealloc, PyCData_dealloc}, + {Py_tp_hash, PyCData_nohash}, + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_traverse, PyCData_traverse}, + {Py_tp_clear, PyCData_clear}, + {Py_tp_methods, PyCData_methods}, + {Py_tp_members, PyCData_members}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, +}; + +static PyType_Spec pycdata_spec = { + .name = "_ctypes._CData", + .basicsize = sizeof(CDataObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + .slots = pycdata_slots, }; -static int PyCData_MallocBuffer(CDataObject *obj, StgDictObject *dict) +static int +PyCData_MallocBuffer(CDataObject *obj, StgInfo *info) { - if ((size_t)dict->size <= sizeof(obj->b_value)) { + if ((size_t)info->size <= sizeof(obj->b_value)) { /* No need to call malloc, can use the default buffer */ obj->b_ptr = (char *)&obj->b_value; /* The b_needsfree flag does not mean that we actually did @@ -2831,51 +3078,56 @@ static int PyCData_MallocBuffer(CDataObject *obj, StgDictObject *dict) /* In python 2.4, and ctypes 0.9.6, the malloc call took about 33% of the creation time for c_int(). */ - obj->b_ptr = (char *)PyMem_Malloc(dict->size); + obj->b_ptr = (char *)PyMem_Malloc(info->size); if (obj->b_ptr == NULL) { PyErr_NoMemory(); return -1; } obj->b_needsfree = 1; - memset(obj->b_ptr, 0, dict->size); + memset(obj->b_ptr, 0, info->size); } - obj->b_size = dict->size; + obj->b_size = info->size; return 0; } PyObject * -PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) +PyCData_FromBaseObj(ctypes_state *st, + PyObject *type, PyObject *base, Py_ssize_t index, char *adr) { CDataObject *cmem; - StgDictObject *dict; assert(PyType_Check(type)); - dict = PyType_stgdict(type); - if (!dict) { + + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; cmem = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); if (cmem == NULL) { return NULL; } - assert(CDataObject_Check(GLOBAL_STATE(), cmem)); - cmem->b_length = dict->length; - cmem->b_size = dict->size; + assert(CDataObject_Check(st, cmem)); + cmem->b_length = info->length; + cmem->b_size = info->size; if (base) { /* use base's buffer */ - assert(CDataObject_Check(GLOBAL_STATE(), base)); + assert(CDataObject_Check(st, base)); cmem->b_ptr = adr; cmem->b_needsfree = 0; cmem->b_base = (CDataObject *)Py_NewRef(base); cmem->b_index = index; } else { /* copy contents of adr */ - if (-1 == PyCData_MallocBuffer(cmem, dict)) { + if (-1 == PyCData_MallocBuffer(cmem, info)) { Py_DECREF(cmem); return NULL; } - memcpy(cmem->b_ptr, adr, dict->size); + memcpy(cmem->b_ptr, adr, info->size); cmem->b_index = index; } return (PyObject *)cmem; @@ -2885,32 +3137,36 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) Box a memory block into a CData instance. */ PyObject * -PyCData_AtAddress(PyObject *type, void *buf) +PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf) { CDataObject *pd; - StgDictObject *dict; if (PySys_Audit("ctypes.cdata", "n", (Py_ssize_t)buf) < 0) { return NULL; } assert(PyType_Check(type)); - dict = PyType_stgdict(type); - if (!dict) { + + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; pd = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); if (!pd) { return NULL; } - assert(CDataObject_Check(GLOBAL_STATE(), pd)); + assert(CDataObject_Check(st, pd)); pd->b_ptr = (char *)buf; - pd->b_length = dict->length; - pd->b_size = dict->size; + pd->b_length = info->length; + pd->b_size = info->size; return (PyObject *)pd; } @@ -2919,10 +3175,9 @@ PyCData_AtAddress(PyObject *type, void *buf) classes. FALSE otherwise FALSE also for subclasses of c_int and such. */ -int _ctypes_simple_instance(PyObject *obj) +int _ctypes_simple_instance(ctypes_state *st, PyObject *obj) { PyTypeObject *type = (PyTypeObject *)obj; - ctypes_state *st = GLOBAL_STATE(); if (PyCSimpleTypeObject_Check(st, type)) { return type->tp_base != st->Simple_Type; @@ -2931,24 +3186,28 @@ int _ctypes_simple_instance(PyObject *obj) } PyObject * -PyCData_get(PyObject *type, GETFUNC getfunc, PyObject *src, +PyCData_get(ctypes_state *st, PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *adr) { - StgDictObject *dict; if (getfunc) return getfunc(adr, size); assert(type); - dict = PyType_stgdict(type); - if (dict && dict->getfunc && !_ctypes_simple_instance(type)) - return dict->getfunc(adr, size); - return PyCData_FromBaseObj(type, src, index, adr); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (info && info->getfunc && !_ctypes_simple_instance(st, type)) { + return info->getfunc(adr, size); + } + return PyCData_FromBaseObj(st, type, src, index, adr); } /* Helper function for PyCData_set below. */ static PyObject * -_PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +_PyCData_set(ctypes_state *st, + CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t size, char *ptr) { CDataObject *src; @@ -2957,11 +3216,13 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, if (setfunc) { return setfunc(ptr, value, size); } - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, value)) { - StgDictObject *dict = PyType_stgdict(type); - if (dict && dict->setfunc) - return dict->setfunc(ptr, value, size); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (info && info->setfunc) + return info->setfunc(ptr, value, size); /* If value is a tuple, we try to call the type with the tuple and use the result! @@ -2976,7 +3237,7 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, ((PyTypeObject *)type)->tp_name); return NULL; } - result = _PyCData_set(dst, type, setfunc, ob, + result = _PyCData_set(st, dst, type, setfunc, ob, size, ptr); Py_DECREF(ob); return result; @@ -3014,11 +3275,16 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, if (PyCPointerTypeObject_Check(st, type) && ArrayObject_Check(st, value)) { - StgDictObject *p1, *p2; PyObject *keep; - p1 = PyObject_stgdict(value); + + StgInfo *p1, *p2; + if (PyStgInfo_FromObject(st, value, &p1) < 0) { + return NULL; + } assert(p1); /* Cannot be NULL for array instances */ - p2 = PyType_stgdict(type); + if (PyStgInfo_FromType(st, type, &p2) < 0) { + return NULL; + } assert(p2); /* Cannot be NULL for pointer types */ if (p1->proto != p2->proto) { @@ -3056,12 +3322,12 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, * to the value 'value'. */ int -PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +PyCData_set(ctypes_state *st, + PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr) { CDataObject *mem = (CDataObject *)dst; PyObject *result; - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, dst)) { PyErr_SetString(PyExc_TypeError, @@ -3069,7 +3335,7 @@ PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, return -1; } - result = _PyCData_set(mem, type, setfunc, value, + result = _PyCData_set(st, mem, type, setfunc, value, size, ptr); if (result == NULL) return -1; @@ -3084,17 +3350,28 @@ PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, /******************************************************************/ static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + return generic_pycdata_new(st, type, args, kwds); +} + +static inline PyObject * +generic_pycdata_new(ctypes_state *st, + PyTypeObject *type, PyObject *args, PyObject *kwds) { CDataObject *obj; - StgDictObject *dict; - dict = PyType_stgdict((PyObject *)type); - if (!dict) { + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; obj = (CDataObject *)type->tp_alloc(type, 0); if (!obj) @@ -3103,9 +3380,9 @@ GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) obj->b_base = NULL; obj->b_index = 0; obj->b_objects = NULL; - obj->b_length = dict->length; + obj->b_length = info->length; - if (-1 == PyCData_MallocBuffer(obj, dict)) { + if (-1 == PyCData_MallocBuffer(obj, info)) { Py_DECREF(obj); return NULL; } @@ -3149,7 +3426,12 @@ PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ign Py_XDECREF(oldchecker); return 0; } - if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + return -1; + } + if (ob != Py_None && !info && !PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "restype must be a type, a callable, or None"); return -1; @@ -3168,14 +3450,17 @@ PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ign static PyObject * PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) { - StgDictObject *dict; if (self->restype) { return Py_NewRef(self->restype); } - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - if (dict->restype) { - return Py_NewRef(dict->restype); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + if (info->restype) { + return Py_NewRef(info->restype); } else { Py_RETURN_NONE; } @@ -3190,7 +3475,8 @@ PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ig Py_CLEAR(self->converters); Py_CLEAR(self->argtypes); } else { - converters = converters_from_argtypes(ob); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + converters = converters_from_argtypes(st, ob); if (!converters) return -1; Py_XSETREF(self->converters, converters); @@ -3203,14 +3489,17 @@ PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ig static PyObject * PyCFuncPtr_get_argtypes(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) { - StgDictObject *dict; if (self->argtypes) { return Py_NewRef(self->argtypes); } - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - if (dict->argtypes) { - return Py_NewRef(dict->argtypes); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + if (info->argtypes) { + return Py_NewRef(info->argtypes); } else { Py_RETURN_NONE; } @@ -3242,7 +3531,6 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) #else char *mangled_name; int i; - StgDictObject *dict; Py_BEGIN_ALLOW_THREADS address = (PPROC)GetProcAddress(handle, name); @@ -3253,9 +3541,13 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) return NULL; } - dict = PyType_stgdict((PyObject *)type); - /* It should not happen that dict is NULL, but better be safe */ - if (dict==NULL || dict->flags & FUNCFLAG_CDECL) + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + /* It should not happen that info is NULL, but better be safe */ + if (info==NULL || info->flags & FUNCFLAG_CDECL) return address; /* for stdcall, try mangled names: @@ -3280,23 +3572,23 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) /* Return 1 if usable, 0 else and exception set. */ static int -_check_outarg_type(PyObject *arg, Py_ssize_t index) +_check_outarg_type(ctypes_state *st, PyObject *arg, Py_ssize_t index) { - StgDictObject *dict; - ctypes_state *st = GLOBAL_STATE(); - if (PyCPointerTypeObject_Check(st, arg)) { return 1; } if (PyCArrayTypeObject_Check(st, arg)) { return 1; } - dict = PyType_stgdict(arg); - if (dict + StgInfo *info; + if (PyStgInfo_FromType(st, arg, &info) < 0) { + return -1; + } + if (info /* simple pointer types, c_void_p, c_wchar_p, BSTR, ... */ - && PyUnicode_Check(dict->proto) + && PyUnicode_Check(info->proto) /* We only allow c_void_p, c_char_p and c_wchar_p as a simple output parameter type */ - && (strchr("PzZ", PyUnicode_AsUTF8(dict->proto)[0]))) { + && (strchr("PzZ", PyUnicode_AsUTF8(info->proto)[0]))) { return 1; } @@ -3311,21 +3603,23 @@ _check_outarg_type(PyObject *arg, Py_ssize_t index) /* Returns 1 on success, 0 on error */ static int -_validate_paramflags(PyTypeObject *type, PyObject *paramflags) +_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) { Py_ssize_t i, len; - StgDictObject *dict; PyObject *argtypes; - dict = PyType_stgdict((PyObject *)type); - if (!dict) { + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return -1; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return 0; } - argtypes = dict->argtypes; + argtypes = info->argtypes; - if (paramflags == NULL || dict->argtypes == NULL) + if (paramflags == NULL || info->argtypes == NULL) return 1; if (!PyTuple_Check(paramflags)) { @@ -3335,7 +3629,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) } len = PyTuple_GET_SIZE(paramflags); - if (len != PyTuple_GET_SIZE(dict->argtypes)) { + if (len != PyTuple_GET_SIZE(info->argtypes)) { PyErr_SetString(PyExc_ValueError, "paramflags must have the same length as argtypes"); return 0; @@ -3362,7 +3656,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) case PARAMFLAG_FIN | PARAMFLAG_FOUT: break; case PARAMFLAG_FOUT: - if (!_check_outarg_type(typ, i+1)) + if (!_check_outarg_type(st, typ, i+1)) return 0; break; default: @@ -3494,12 +3788,13 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #endif - if (!_validate_paramflags(type, paramflags)) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); return NULL; } - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); if (!self) { Py_DECREF(ftuple); return NULL; @@ -3535,10 +3830,11 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) if (paramflags == Py_None) paramflags = NULL; - if (!_validate_paramflags(type, paramflags)) + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + if (!_validate_paramflags(st, type, paramflags)) { return NULL; - - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + } + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); self->index = index + 0x1000; self->paramflags = Py_XNewRef(paramflags); if (iid_len == sizeof(GUID)) @@ -3565,7 +3861,6 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyCFuncPtrObject *self; PyObject *callable; - StgDictObject *dict; CThunkObject *thunk; if (PyTuple_GET_SIZE(args) == 0) @@ -3616,23 +3911,28 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } */ - dict = PyType_stgdict((PyObject *)type); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } /* XXXX Fails if we do: 'PyCFuncPtr(lambda x: x)' */ - if (!dict || !dict->argtypes) { + if (!info || !info->argtypes) { PyErr_SetString(PyExc_TypeError, "cannot construct instance of this class:" " no argtypes"); return NULL; } - thunk = _ctypes_alloc_callback(callable, - dict->argtypes, - dict->restype, - dict->flags); + thunk = _ctypes_alloc_callback(st, + callable, + info->argtypes, + info->restype, + info->flags); if (!thunk) return NULL; - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); if (self == NULL) { Py_DECREF(thunk); return NULL; @@ -3656,10 +3956,9 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _byref consumes a refcount to its argument */ static PyObject * -_byref(PyObject *obj) +_byref(ctypes_state *st, PyObject *obj) { PyCArgObject *parg; - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, @@ -3667,7 +3966,7 @@ _byref(PyObject *obj) return NULL; } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) { Py_DECREF(obj); return NULL; @@ -3731,13 +4030,12 @@ _get_arg(int *pindex, PyObject *name, PyObject *defval, PyObject *inargs, PyObje function. */ static PyObject * -_build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, +_build_callargs(ctypes_state *st, PyCFuncPtrObject *self, PyObject *argtypes, PyObject *inargs, PyObject *kwds, int *poutmask, int *pinoutmask, unsigned int *pnumretvals) { PyObject *paramflags = self->paramflags; PyObject *callargs; - StgDictObject *dict; Py_ssize_t i, len; int inargs_index = 0; /* It's a little bit difficult to determine how many arguments the @@ -3769,7 +4067,6 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, inargs_index = 1; } #endif - ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < len; ++i) { PyObject *item = PyTuple_GET_ITEM(paramflags, i); PyObject *ob; @@ -3798,7 +4095,7 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, case (PARAMFLAG_FIN | PARAMFLAG_FOUT): *pinoutmask |= (1 << i); /* mark as inout arg */ (*pnumretvals)++; - /* fall through */ + _Py_FALLTHROUGH; case 0: case PARAMFLAG_FIN: /* 'in' parameter. Copy it from inargs. */ @@ -3826,15 +4123,18 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, break; } ob = PyTuple_GET_ITEM(argtypes, i); - dict = PyType_stgdict(ob); - if (dict == NULL) { + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + goto error; + } + if (info == NULL) { /* Cannot happen: _validate_paramflags() would not accept such an object */ PyErr_Format(PyExc_RuntimeError, - "NULL stgdict unexpected"); + "NULL stginfo unexpected"); goto error; } - if (PyUnicode_Check(dict->proto)) { + if (PyUnicode_Check(info->proto)) { PyErr_Format( PyExc_TypeError, "%s 'out' parameter must be passed as default value", @@ -3846,7 +4146,7 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, } else { /* Create an instance of the pointed-to type */ - ob = _PyObject_CallNoArgs(dict->proto); + ob = _PyObject_CallNoArgs(info->proto); } /* XXX Is the following correct any longer? @@ -3966,7 +4266,6 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) PyObject *converters; PyObject *checker; PyObject *argtypes; - StgDictObject *dict = PyObject_stgdict((PyObject *)self); PyObject *result; PyObject *callargs; PyObject *errcheck; @@ -3979,13 +4278,19 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) int outmask; unsigned int numretvals; - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - restype = self->restype ? self->restype : dict->restype; - converters = self->converters ? self->converters : dict->converters; - checker = self->checker ? self->checker : dict->checker; - argtypes = self->argtypes ? self->argtypes : dict->argtypes; -/* later, we probably want to have an errcheck field in stgdict */ - errcheck = self->errcheck /* ? self->errcheck : dict->errcheck */; + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + + restype = self->restype ? self->restype : info->restype; + converters = self->converters ? self->converters : info->converters; + checker = self->checker ? self->checker : info->checker; + argtypes = self->argtypes ? self->argtypes : info->argtypes; +/* later, we probably want to have an errcheck field in stginfo */ + errcheck = self->errcheck /* ? self->errcheck : info->errcheck */; pProc = *(void **)self->b_ptr; @@ -3999,7 +4304,6 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) "native com method call without 'this' parameter"); return NULL; } - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, this)) { PyErr_SetString(PyExc_TypeError, "Expected a COM this pointer as first argument"); @@ -4021,7 +4325,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) pProc = ((void **)piunk->lpVtbl)[self->index - 0x1000]; } #endif - callargs = _build_callargs(self, argtypes, + callargs = _build_callargs(st, self, argtypes, inargs, kwds, &outmask, &inoutmask, &numretvals); if (callargs == NULL) @@ -4033,7 +4337,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) int actual = Py_SAFE_DOWNCAST(PyTuple_GET_SIZE(callargs), Py_ssize_t, int); - if ((dict->flags & FUNCFLAG_CDECL) == FUNCFLAG_CDECL) { + if ((info->flags & FUNCFLAG_CDECL) == FUNCFLAG_CDECL) { /* For cdecl functions, we allow more actual arguments than the length of the argtypes tuple. */ @@ -4057,13 +4361,14 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) } } - result = _ctypes_callproc(pProc, + result = _ctypes_callproc(st, + pProc, callargs, #ifdef MS_WIN32 piunk, self->iid, #endif - dict->flags, + info->flags, converters, restype, checker); @@ -4123,8 +4428,11 @@ PyCFuncPtr_clear(PyCFuncPtrObject *self) static void PyCFuncPtr_dealloc(PyCFuncPtrObject *self) { + PyObject_GC_UnTrack(self); PyCFuncPtr_clear(self); - Py_TYPE(self)->tp_free((PyObject *)self); + PyTypeObject *type = Py_TYPE(self); + type->tp_free((PyObject *)self); + Py_DECREF(type); } static PyObject * @@ -4152,59 +4460,26 @@ PyCFuncPtr_bool(PyCFuncPtrObject *self) ); } -static PyNumberMethods PyCFuncPtr_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)PyCFuncPtr_bool, /* nb_bool */ +static PyType_Slot pycfuncptr_slots[] = { + {Py_tp_dealloc, PyCFuncPtr_dealloc}, + {Py_tp_repr, PyCFuncPtr_repr}, + {Py_tp_call, PyCFuncPtr_call}, + {Py_tp_doc, PyDoc_STR("Function Pointer")}, + {Py_tp_traverse, PyCFuncPtr_traverse}, + {Py_tp_clear, PyCFuncPtr_clear}, + {Py_tp_getset, PyCFuncPtr_getsets}, + {Py_tp_new, PyCFuncPtr_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, PyCFuncPtr_bool}, + {0, NULL}, }; -PyTypeObject PyCFuncPtr_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.CFuncPtr", - sizeof(PyCFuncPtrObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)PyCFuncPtr_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)PyCFuncPtr_repr, /* tp_repr */ - &PyCFuncPtr_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc)PyCFuncPtr_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Function Pointer"), /* tp_doc */ - (traverseproc)PyCFuncPtr_traverse, /* tp_traverse */ - (inquiry)PyCFuncPtr_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - PyCFuncPtr_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCFuncPtr_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycfuncptr_spec = { + .name = "_ctypes.CFuncPtr", + .basicsize = sizeof(PyCFuncPtrObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycfuncptr_slots, }; /*****************************************************************/ @@ -4224,11 +4499,15 @@ _init_pos_args(PyObject *self, PyTypeObject *type, PyObject *args, PyObject *kwds, Py_ssize_t index) { - StgDictObject *dict; PyObject *fields; Py_ssize_t i; - if (PyType_stgdict((PyObject *)type->tp_base)) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)type->tp_base, &baseinfo) < 0) { + return -1; + } + if (baseinfo) { index = _init_pos_args(self, type->tp_base, args, kwds, index); @@ -4236,8 +4515,17 @@ _init_pos_args(PyObject *self, PyTypeObject *type, return -1; } - dict = PyType_stgdict((PyObject *)type); - fields = PyDict_GetItemWithError((PyObject *)dict, &_Py_ID(_fields_)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return -1; + } + assert(info); + + PyObject *attrdict = PyType_GetDict(type); + assert(attrdict); + + fields = PyDict_GetItemWithError((PyObject *)attrdict, &_Py_ID(_fields_)); + Py_CLEAR(attrdict); if (fields == NULL) { if (PyErr_Occurred()) { return -1; @@ -4246,7 +4534,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, } for (i = index; - i < dict->length && i < PyTuple_GET_SIZE(args); + i < info->length && i < PyTuple_GET_SIZE(args); ++i) { PyObject *pair = PySequence_GetItem(fields, i - index); PyObject *name, *val; @@ -4279,7 +4567,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, if (res == -1) return -1; } - return dict->length; + return info->length; } static int @@ -4316,88 +4604,34 @@ Struct_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } -static PyTypeObject Struct_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Structure", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Structure base class"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - Struct_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycstruct_slots[] = { + {Py_tp_doc, PyDoc_STR("Structure base class")}, + {Py_tp_init, Struct_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, +}; + +static PyType_Spec pycstruct_spec = { + .name = "_ctypes.Structure", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycstruct_slots, +}; + +static PyType_Slot pycunion_slots[] = { + {Py_tp_doc, PyDoc_STR("Union base class")}, + {Py_tp_init, Struct_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, }; -static PyTypeObject Union_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Union", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Union base class"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - Struct_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycunion_spec = { + .name = "_ctypes.Union", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycunion_slots, }; @@ -4431,8 +4665,6 @@ Array_item(PyObject *myself, Py_ssize_t index) { CDataObject *self = (CDataObject *)myself; Py_ssize_t offset, size; - StgDictObject *stgdict; - if (index < 0 || index >= self->b_length) { PyErr_SetString(PyExc_IndexError, @@ -4440,15 +4672,19 @@ Array_item(PyObject *myself, Py_ssize_t index) return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array instances */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + /* Would it be clearer if we got the item size from - stgdict->proto's stgdict? + stginfo->proto's stginfo? */ - size = stgdict->size / stgdict->length; + size = stginfo->size / stginfo->length; offset = index * size; - return PyCData_get(stgdict->proto, stgdict->getfunc, (PyObject *)self, + return PyCData_get(st, stginfo->proto, stginfo->getfunc, (PyObject *)self, index, size, self->b_ptr + offset); } @@ -4467,7 +4703,6 @@ Array_subscript(PyObject *myself, PyObject *item) return Array_item(myself, i); } else if (PySlice_Check(item)) { - StgDictObject *stgdict, *itemdict; PyObject *proto; PyObject *np; Py_ssize_t start, stop, step, slicelen, i; @@ -4478,19 +4713,26 @@ Array_subscript(PyObject *myself, PyObject *item) } slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array object instances */ - proto = stgdict->proto; - itemdict = PyType_stgdict(proto); - assert(itemdict); /* proto is the item type of the array, a + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for array object instances */ + proto = stginfo->proto; + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); /* proto is the item type of the array, a ctypes type, so this cannot be NULL */ - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { char *ptr = (char *)self->b_ptr; char *dest; if (slicelen <= 0) - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (step == 1) { return PyBytes_FromStringAndSize(ptr + start, slicelen); @@ -4509,12 +4751,12 @@ Array_subscript(PyObject *myself, PyObject *item) PyMem_Free(dest); return np; } - if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { wchar_t *ptr = (wchar_t *)self->b_ptr; wchar_t *dest; if (slicelen <= 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (step == 1) { return PyUnicode_FromWideChar(ptr + start, slicelen); @@ -4564,7 +4806,6 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) { CDataObject *self = (CDataObject *)myself; Py_ssize_t size, offset; - StgDictObject *stgdict; char *ptr; if (value == NULL) { @@ -4573,18 +4814,23 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array object instances */ - if (index < 0 || index >= stgdict->length) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for array object instances */ + + if (index < 0 || index >= stginfo->length) { PyErr_SetString(PyExc_IndexError, "invalid index"); return -1; } - size = stgdict->size / stgdict->length; + size = stginfo->size / stginfo->length; offset = index * size; ptr = self->b_ptr + offset; - return PyCData_set((PyObject *)self, stgdict->proto, stgdict->setfunc, value, + return PyCData_set(st, (PyObject *)self, stginfo->proto, stginfo->setfunc, value, index, size, ptr); } @@ -4658,26 +4904,6 @@ static PyMethodDef Array_methods[] = { { NULL, NULL } }; -static PySequenceMethods Array_as_sequence = { - Array_length, /* sq_length; */ - 0, /* sq_concat; */ - 0, /* sq_repeat; */ - Array_item, /* sq_item; */ - 0, /* sq_slice; */ - Array_ass_item, /* sq_ass_item; */ - 0, /* sq_ass_slice; */ - 0, /* sq_contains; */ - - 0, /* sq_inplace_concat; */ - 0, /* sq_inplace_repeat; */ -}; - -static PyMappingMethods Array_as_mapping = { - Array_length, - Array_subscript, - Array_ass_subscript, -}; - PyDoc_STRVAR(array_doc, "Abstract base class for arrays.\n" "\n" @@ -4688,60 +4914,40 @@ PyDoc_STRVAR(array_doc, "reads, the resulting object is not itself an Array." ); -PyTypeObject PyCArray_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Array", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &Array_as_sequence, /* tp_as_sequence */ - &Array_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - array_doc, /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Array_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Array_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycarray_slots[] = { + {Py_tp_doc, (char*)array_doc}, + {Py_tp_methods, Array_methods}, + {Py_tp_init, Array_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_sq_length, Array_length}, + {Py_sq_item, Array_item}, + {Py_sq_ass_item, Array_ass_item}, + {Py_mp_length, Array_length}, + {Py_mp_subscript, Array_subscript}, + {Py_mp_ass_subscript, Array_ass_subscript}, + {0, NULL}, +}; + +static PyType_Spec pycarray_spec = { + .name = "_ctypes.Array", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycarray_slots, }; PyObject * -PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) +PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length) { - static PyObject *cache; PyObject *key; char name[256]; PyObject *len; - if (cache == NULL) { - cache = PyDict_New(); - if (cache == NULL) + if (st->array_cache == NULL) { + st->array_cache = PyDict_New(); + if (st->array_cache == NULL) { return NULL; + } } len = PyLong_FromSsize_t(length); if (len == NULL) @@ -4752,7 +4958,7 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) return NULL; PyObject *result; - if (_PyDict_GetItemProxy(cache, key, &result) != 0) { + if (_PyDict_GetItemProxy(st->array_cache, key, &result) != 0) { // found or error Py_DECREF(key); return result; @@ -4771,7 +4977,6 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) sprintf(name, "%.200s_Array_%ld", ((PyTypeObject *)itemtype)->tp_name, (long)length); #endif - ctypes_state *st = GLOBAL_STATE(); result = PyObject_CallFunction((PyObject *)st->PyCArrayType_Type, "s(O){s:n,s:O}", name, @@ -4785,7 +4990,7 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) Py_DECREF(key); return NULL; } - if (-1 == PyDict_SetItemProxy(cache, key, result)) { + if (PyDict_SetItemProxy(st, st->array_cache, key, result) < 0) { Py_DECREF(key); Py_DECREF(result); return NULL; @@ -4800,20 +5005,32 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) Simple_Type */ +/*[clinic input] +class _ctypes.Simple "PyObject *" "clinic_state()->Simple_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=016c476c7aa8b8a8]*/ + + static int Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) { PyObject *result; - StgDictObject *dict = PyObject_stgdict((PyObject *)self); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "can't delete attribute"); return -1; } - assert(dict); /* Cannot be NULL for CDataObject instances */ - assert(dict->setfunc); - result = dict->setfunc(self->b_ptr, value, dict->size); + + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return -1; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + assert(info->setfunc); + + result = info->setfunc(self->b_ptr, value, info->size); if (!result) return -1; @@ -4835,11 +5052,14 @@ Simple_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) { - StgDictObject *dict; - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for CDataObject instances */ - assert(dict->getfunc); - return dict->getfunc(self->b_ptr, self->b_size); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + assert(info->getfunc); + return info->getfunc(self->b_ptr, self->b_size); } static PyGetSetDef Simple_getsets[] = { @@ -4848,18 +5068,28 @@ static PyGetSetDef Simple_getsets[] = { { NULL, NULL } }; +/*[clinic input] +_ctypes.Simple.__ctypes_from_outparam__ as Simple_from_outparm + + self: self + cls: defining_class + / +[clinic start generated code]*/ + static PyObject * -Simple_from_outparm(PyObject *self, PyObject *args) +Simple_from_outparm_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=6c61d90da8aa9b4f input=0f362803fb4629d5]*/ { - if (_ctypes_simple_instance((PyObject *)Py_TYPE(self))) { + ctypes_state *st = get_module_state_by_class(cls); + if (_ctypes_simple_instance(st, (PyObject *)Py_TYPE(self))) { return Py_NewRef(self); } - /* call stgdict->getfunc */ + /* call stginfo->getfunc */ return Simple_get_value((CDataObject *)self, NULL); } static PyMethodDef Simple_methods[] = { - { "__ctypes_from_outparam__", Simple_from_outparm, METH_NOARGS, }, + SIMPLE_FROM_OUTPARM_METHODDEF { NULL, NULL }, }; @@ -4868,25 +5098,12 @@ static int Simple_bool(CDataObject *self) return memcmp(self->b_ptr, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", self->b_size); } -static PyNumberMethods Simple_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)Simple_bool, /* nb_bool */ -}; - /* "%s(%s)" % (self.__class__.__name__, self.value) */ static PyObject * Simple_repr(CDataObject *self) { PyObject *val, *result; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); if (Py_TYPE(self)->tp_base != st->Simple_Type) { return PyUnicode_FromFormat("<%s object at %p>", @@ -4903,48 +5120,26 @@ Simple_repr(CDataObject *self) return result; } -static PyTypeObject Simple_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._SimpleCData", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)&Simple_repr, /* tp_repr */ - &Simple_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Simple_methods, /* tp_methods */ - 0, /* tp_members */ - Simple_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Simple_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycsimple_slots[] = { + {Py_tp_repr, &Simple_repr}, + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_methods, Simple_methods}, + {Py_tp_getset, Simple_getsets}, + {Py_tp_init, Simple_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, Simple_bool}, + {0, NULL}, +}; + +static PyType_Spec pycsimple_spec = { + .name = "_ctypes._SimpleCData", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycsimple_slots, }; + /******************************************************************/ /* PyCPointer_Type @@ -4955,7 +5150,6 @@ Pointer_item(PyObject *myself, Py_ssize_t index) CDataObject *self = (CDataObject *)myself; Py_ssize_t size; Py_ssize_t offset; - StgDictObject *stgdict, *itemdict; PyObject *proto; if (*(void **)self->b_ptr == NULL) { @@ -4964,19 +5158,27 @@ Pointer_item(PyObject *myself, Py_ssize_t index) return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer object instances */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer object instances */ - proto = stgdict->proto; + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); /* proto is the item type of the pointer, a ctypes + + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); /* proto is the item type of the pointer, a ctypes type, so this cannot be NULL */ - size = itemdict->size; - offset = index * itemdict->size; + size = iteminfo->size; + offset = index * iteminfo->size; - return PyCData_get(proto, stgdict->getfunc, (PyObject *)self, + return PyCData_get(st, proto, stginfo->getfunc, (PyObject *)self, index, size, (*(char **)self->b_ptr) + offset); } @@ -4986,7 +5188,6 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) CDataObject *self = (CDataObject *)myself; Py_ssize_t size; Py_ssize_t offset; - StgDictObject *stgdict, *itemdict; PyObject *proto; if (value == NULL) { @@ -5001,37 +5202,47 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ - proto = stgdict->proto; + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); /* Cannot be NULL because the itemtype of a pointer + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return -1; + } + assert(iteminfo); /* Cannot be NULL because the itemtype of a pointer is always a ctypes type */ - size = itemdict->size; - offset = index * itemdict->size; + size = iteminfo->size; + offset = index * iteminfo->size; - return PyCData_set((PyObject *)self, proto, stgdict->setfunc, value, + return PyCData_set(st, (PyObject *)self, proto, stginfo->setfunc, value, index, size, (*(char **)self->b_ptr) + offset); } static PyObject * Pointer_get_contents(CDataObject *self, void *closure) { - StgDictObject *stgdict; - if (*(void **)self->b_ptr == NULL) { PyErr_SetString(PyExc_ValueError, "NULL pointer access"); return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - return PyCData_FromBaseObj(stgdict->proto, + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + + return PyCData_FromBaseObj(st, stginfo->proto, (PyObject *)self, 0, *(void **)self->b_ptr); } @@ -5039,7 +5250,6 @@ Pointer_get_contents(CDataObject *self, void *closure) static int Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) { - StgDictObject *stgdict; CDataObject *dst; PyObject *keep; @@ -5048,18 +5258,21 @@ Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) "Pointer does not support item deletion"); return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - assert(stgdict->proto); - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + assert(stginfo->proto); if (!CDataObject_Check(st, value)) { - int res = PyObject_IsInstance(value, stgdict->proto); + int res = PyObject_IsInstance(value, stginfo->proto); if (res == -1) return -1; if (!res) { PyErr_Format(PyExc_TypeError, "expected %s instead of %s", - ((PyTypeObject *)(stgdict->proto))->tp_name, + ((PyTypeObject *)(stginfo->proto))->tp_name, Py_TYPE(value)->tp_name); return -1; } @@ -5107,13 +5320,17 @@ Pointer_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Pointer_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - StgDictObject *dict = PyType_stgdict((PyObject *)type); - if (!dict || !dict->proto) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + if (!info || !info->proto) { PyErr_SetString(PyExc_TypeError, "Cannot create instance: has no _type_"); return NULL; } - return GenericPyCData_new(type, args, kw); + return generic_pycdata_new(st, type, args, kw); } static PyObject * @@ -5130,7 +5347,6 @@ Pointer_subscript(PyObject *myself, PyObject *item) PySliceObject *slice = (PySliceObject *)item; Py_ssize_t start, stop, step; PyObject *np; - StgDictObject *stgdict, *itemdict; PyObject *proto; Py_ssize_t i, len; size_t cur; @@ -5184,18 +5400,25 @@ Pointer_subscript(PyObject *myself, PyObject *item) else len = (stop - start + 1) / step + 1; - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - proto = stgdict->proto; + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { char *ptr = *(char **)self->b_ptr; char *dest; if (len <= 0) - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (step == 1) { return PyBytes_FromStringAndSize(ptr + start, len); @@ -5210,12 +5433,12 @@ Pointer_subscript(PyObject *myself, PyObject *item) PyMem_Free(dest); return np; } - if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { wchar_t *ptr = *(wchar_t **)self->b_ptr; wchar_t *dest; if (len <= 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (step == 1) { return PyUnicode_FromWideChar(ptr + start, len); @@ -5248,87 +5471,32 @@ Pointer_subscript(PyObject *myself, PyObject *item) } } -static PySequenceMethods Pointer_as_sequence = { - 0, /* inquiry sq_length; */ - 0, /* binaryfunc sq_concat; */ - 0, /* intargfunc sq_repeat; */ - Pointer_item, /* intargfunc sq_item; */ - 0, /* intintargfunc sq_slice; */ - Pointer_ass_item, /* intobjargproc sq_ass_item; */ - 0, /* intintobjargproc sq_ass_slice; */ - 0, /* objobjproc sq_contains; */ - /* Added in release 2.0 */ - 0, /* binaryfunc sq_inplace_concat; */ - 0, /* intargfunc sq_inplace_repeat; */ -}; - -static PyMappingMethods Pointer_as_mapping = { - 0, - Pointer_subscript, -}; - static int Pointer_bool(CDataObject *self) { return (*(void **)self->b_ptr != NULL); } -static PyNumberMethods Pointer_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)Pointer_bool, /* nb_bool */ +static PyType_Slot pycpointer_slots[] = { + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_getset, Pointer_getsets}, + {Py_tp_init, Pointer_init}, + {Py_tp_new, Pointer_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, Pointer_bool}, + {Py_mp_subscript, Pointer_subscript}, + {Py_sq_item, Pointer_item}, + {Py_sq_ass_item, Pointer_ass_item}, + {0, NULL}, }; -PyTypeObject PyCPointer_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._Pointer", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - &Pointer_as_number, /* tp_as_number */ - &Pointer_as_sequence, /* tp_as_sequence */ - &Pointer_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - Pointer_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Pointer_init, /* tp_init */ - 0, /* tp_alloc */ - Pointer_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycpointer_spec = { + .name = "_ctypes._Pointer", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycpointer_slots, }; - /******************************************************************/ /* * Module initialization. @@ -5431,21 +5599,21 @@ string_at(const char *ptr, int size) } static int -cast_check_pointertype(PyObject *arg) +cast_check_pointertype(ctypes_state *st, PyObject *arg) { - StgDictObject *dict; - ctypes_state *st = GLOBAL_STATE(); - if (PyCPointerTypeObject_Check(st, arg)) { return 1; } if (PyCFuncPtrTypeObject_Check(st, arg)) { return 1; } - dict = PyType_stgdict(arg); - if (dict != NULL && dict->proto != NULL) { - if (PyUnicode_Check(dict->proto) - && (strchr("sPzUZXO", PyUnicode_AsUTF8(dict->proto)[0]))) { + StgInfo *info; + if (PyStgInfo_FromType(st, arg, &info) < 0) { + return 0; + } + if (info != NULL && info->proto != NULL) { + if (PyUnicode_Check(info->proto) + && (strchr("sPzUZXO", PyUnicode_AsUTF8(info->proto)[0]))) { /* simple pointer types, c_void_p, c_wchar_p, BSTR, ... */ return 1; } @@ -5461,9 +5629,18 @@ cast_check_pointertype(PyObject *arg) static PyObject * cast(void *ptr, PyObject *src, PyObject *ctype) { + PyObject *mod = PyType_GetModuleByDef(Py_TYPE(ctype), &_ctypesmodule); + if (!mod) { + PyErr_SetString(PyExc_TypeError, + "cast() argument 2 must be a pointer type"); + return NULL; + } + ctypes_state *st = get_module_state(mod); + CDataObject *result; - if (0 == cast_check_pointertype(ctype)) + if (cast_check_pointertype(st, ctype) == 0) { return NULL; + } result = (CDataObject *)_PyObject_CallNoArgs(ctype); if (result == NULL) return NULL; @@ -5474,7 +5651,6 @@ cast(void *ptr, PyObject *src, PyObject *ctype) It must certainly contain the source objects one. It must contain the source object itself. */ - ctypes_state *st = GLOBAL_STATE(); if (CDataObject_Check(st, src)) { CDataObject *obj = (CDataObject *)src; CDataObject *container; @@ -5528,15 +5704,6 @@ wstring_at(const wchar_t *ptr, int size) } -static struct PyModuleDef _ctypesmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "_ctypes", - .m_doc = _ctypes__doc__, - .m_size = -1, - .m_methods = _ctypes_module_methods, -}; - - static int _ctypes_add_types(PyObject *mod) { @@ -5544,26 +5711,8 @@ _ctypes_add_types(PyObject *mod) if (PyType_Ready(TYPE) < 0) { \ return -1; \ } - -#define TYPE_READY_BASE(TYPE_EXPR, TP_BASE) \ - do { \ - PyTypeObject *type = (TYPE_EXPR); \ - type->tp_base = (TP_BASE); \ - TYPE_READY(type); \ - } while (0) - -#define MOD_ADD_TYPE(TYPE_EXPR, TP_TYPE, TP_BASE) \ - do { \ - PyTypeObject *type = (TYPE_EXPR); \ - Py_SET_TYPE(type, TP_TYPE); \ - type->tp_base = TP_BASE; \ - if (PyModule_AddType(mod, type) < 0) { \ - return -1; \ - } \ - } while (0) - -#define CREATE_TYPE(MOD, TP, SPEC, BASE) do { \ - PyObject *type = PyType_FromMetaclass(NULL, MOD, SPEC, \ +#define CREATE_TYPE(TP, SPEC, META, BASE) do { \ + PyObject *type = PyType_FromMetaclass(META, mod, SPEC, \ (PyObject *)BASE); \ if (type == NULL) { \ return -1; \ @@ -5571,67 +5720,82 @@ _ctypes_add_types(PyObject *mod) TP = (PyTypeObject *)type; \ } while (0) - ctypes_state *st = GLOBAL_STATE(); +#define MOD_ADD_TYPE(TP, SPEC, META, BASE) do { \ + CREATE_TYPE(TP, SPEC, META, BASE); \ + if (PyModule_AddType(mod, (PyTypeObject *)(TP)) < 0) { \ + return -1; \ + } \ +} while (0) + + ctypes_state *st = get_module_state(mod); /* Note: ob_type is the metatype (the 'type'), defaults to PyType_Type, tp_base is the base type, defaults to 'object' aka PyBaseObject_Type. */ - CREATE_TYPE(mod, st->PyCArg_Type, &carg_spec, NULL); - CREATE_TYPE(mod, st->PyCThunk_Type, &cthunk_spec, NULL); - TYPE_READY(st->PyCData_Type); - /* StgDict is derived from PyDict_Type */ - TYPE_READY_BASE(st->PyCStgDict_Type, &PyDict_Type); + CREATE_TYPE(st->PyCArg_Type, &carg_spec, NULL, NULL); + CREATE_TYPE(st->PyCThunk_Type, &cthunk_spec, NULL, NULL); + CREATE_TYPE(st->PyCData_Type, &pycdata_spec, NULL, NULL); + + // Common Metaclass + CREATE_TYPE(st->PyCType_Type, &pyctype_type_spec, + NULL, &PyType_Type); /************************************************* * * Metaclasses */ - CREATE_TYPE(mod, st->PyCStructType_Type, &pycstruct_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->UnionType_Type, &union_type_spec, &PyType_Type); - CREATE_TYPE(mod, st->PyCPointerType_Type, &pycpointer_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->PyCArrayType_Type, &pycarray_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->PyCSimpleType_Type, &pycsimple_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->PyCFuncPtrType_Type, &pycfuncptr_type_spec, - &PyType_Type); + CREATE_TYPE(st->PyCStructType_Type, &pycstruct_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->UnionType_Type, &union_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCPointerType_Type, &pycpointer_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCArrayType_Type, &pycarray_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCSimpleType_Type, &pycsimple_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCFuncPtrType_Type, &pycfuncptr_type_spec, + NULL, st->PyCType_Type); /************************************************* * * Classes using a custom metaclass */ - MOD_ADD_TYPE(st->Struct_Type, st->PyCStructType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->Union_Type, st->UnionType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->PyCPointer_Type, st->PyCPointerType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->PyCArray_Type, st->PyCArrayType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->Simple_Type, st->PyCSimpleType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->PyCFuncPtr_Type, st->PyCFuncPtrType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Struct_Type, &pycstruct_spec, + st->PyCStructType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Union_Type, &pycunion_spec, + st->UnionType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCPointer_Type, &pycpointer_spec, + st->PyCPointerType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCArray_Type, &pycarray_spec, + st->PyCArrayType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Simple_Type, &pycsimple_spec, + st->PyCSimpleType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCFuncPtr_Type, &pycfuncptr_spec, + st->PyCFuncPtrType_Type, st->PyCData_Type); /************************************************* * * Simple classes */ - CREATE_TYPE(mod, st->PyCField_Type, &cfield_spec, NULL); + MOD_ADD_TYPE(st->PyCField_Type, &cfield_spec, NULL, NULL); /************************************************* * * Other stuff */ - CREATE_TYPE(mod, st->DictRemover_Type, &dictremover_spec, NULL); - CREATE_TYPE(mod, st->StructParam_Type, &structparam_spec, NULL); + CREATE_TYPE(st->DictRemover_Type, &dictremover_spec, NULL, NULL); + CREATE_TYPE(st->StructParam_Type, &structparam_spec, NULL, NULL); #ifdef MS_WIN32 - CREATE_TYPE(mod, st->PyComError_Type, &comerror_spec, PyExc_Exception); + CREATE_TYPE(st->PyComError_Type, &comerror_spec, NULL, PyExc_Exception); #endif #undef TYPE_READY -#undef TYPE_READY_BASE #undef MOD_ADD_TYPE #undef CREATE_TYPE return 0; @@ -5648,10 +5812,10 @@ _ctypes_add_objects(PyObject *mod) } \ } while (0) - MOD_ADD("_pointer_type_cache", Py_NewRef(_ctypes_ptrtype_cache)); + ctypes_state *st = get_module_state(mod); + MOD_ADD("_pointer_type_cache", Py_NewRef(st->_ctypes_ptrtype_cache)); #ifdef MS_WIN32 - ctypes_state *st = GLOBAL_STATE(); MOD_ADD("COMError", Py_NewRef(st->PyComError_Type)); MOD_ADD("FUNCFLAG_HRESULT", PyLong_FromLong(FUNCFLAG_HRESULT)); MOD_ADD("FUNCFLAG_STDCALL", PyLong_FromLong(FUNCFLAG_STDCALL)); @@ -5681,7 +5845,7 @@ _ctypes_add_objects(PyObject *mod) MOD_ADD("RTLD_LOCAL", PyLong_FromLong(RTLD_LOCAL)); MOD_ADD("RTLD_GLOBAL", PyLong_FromLong(RTLD_GLOBAL)); MOD_ADD("CTYPES_MAX_ARGCOUNT", PyLong_FromLong(CTYPES_MAX_ARGCOUNT)); - MOD_ADD("ArgumentError", Py_NewRef(PyExc_ArgError)); + MOD_ADD("ArgumentError", Py_NewRef(st->PyExc_ArgError)); MOD_ADD("SIZEOF_TIME_T", PyLong_FromSsize_t(SIZEOF_TIME_T)); return 0; #undef MOD_ADD @@ -5691,18 +5855,19 @@ _ctypes_add_objects(PyObject *mod) static int _ctypes_mod_exec(PyObject *mod) { - _unpickle = PyObject_GetAttrString(mod, "_unpickle"); - if (_unpickle == NULL) { + ctypes_state *st = get_module_state(mod); + st->_unpickle = PyObject_GetAttrString(mod, "_unpickle"); + if (st->_unpickle == NULL) { return -1; } - _ctypes_ptrtype_cache = PyDict_New(); - if (_ctypes_ptrtype_cache == NULL) { + st->_ctypes_ptrtype_cache = PyDict_New(); + if (st->_ctypes_ptrtype_cache == NULL) { return -1; } - PyExc_ArgError = PyErr_NewException("ctypes.ArgumentError", NULL, NULL); - if (!PyExc_ArgError) { + st->PyExc_ArgError = PyErr_NewException("ctypes.ArgumentError", NULL, NULL); + if (!st->PyExc_ArgError) { return -1; } @@ -5717,19 +5882,105 @@ _ctypes_mod_exec(PyObject *mod) } +static int +module_traverse(PyObject *module, visitproc visit, void *arg) { + ctypes_state *st = get_module_state(module); + Py_VISIT(st->_ctypes_ptrtype_cache); + Py_VISIT(st->_unpickle); + Py_VISIT(st->array_cache); + Py_VISIT(st->error_object_name); + Py_VISIT(st->PyExc_ArgError); + Py_VISIT(st->swapped_suffix); + + Py_VISIT(st->DictRemover_Type); + Py_VISIT(st->PyCArg_Type); + Py_VISIT(st->PyCField_Type); + Py_VISIT(st->PyCThunk_Type); + Py_VISIT(st->StructParam_Type); + Py_VISIT(st->PyCStructType_Type); + Py_VISIT(st->UnionType_Type); + Py_VISIT(st->PyCPointerType_Type); + Py_VISIT(st->PyCArrayType_Type); + Py_VISIT(st->PyCSimpleType_Type); + Py_VISIT(st->PyCFuncPtrType_Type); + Py_VISIT(st->PyCData_Type); + Py_VISIT(st->Struct_Type); + Py_VISIT(st->Union_Type); + Py_VISIT(st->PyCArray_Type); + Py_VISIT(st->Simple_Type); + Py_VISIT(st->PyCPointer_Type); + Py_VISIT(st->PyCFuncPtr_Type); +#ifdef MS_WIN32 + Py_VISIT(st->PyComError_Type); +#endif + Py_VISIT(st->PyCType_Type); + return 0; +} + +static int +module_clear(PyObject *module) { + ctypes_state *st = get_module_state(module); + Py_CLEAR(st->_ctypes_ptrtype_cache); + Py_CLEAR(st->_unpickle); + Py_CLEAR(st->array_cache); + Py_CLEAR(st->error_object_name); + Py_CLEAR(st->PyExc_ArgError); + Py_CLEAR(st->swapped_suffix); + + Py_CLEAR(st->DictRemover_Type); + Py_CLEAR(st->PyCArg_Type); + Py_CLEAR(st->PyCField_Type); + Py_CLEAR(st->PyCThunk_Type); + Py_CLEAR(st->StructParam_Type); + Py_CLEAR(st->PyCStructType_Type); + Py_CLEAR(st->UnionType_Type); + Py_CLEAR(st->PyCPointerType_Type); + Py_CLEAR(st->PyCArrayType_Type); + Py_CLEAR(st->PyCSimpleType_Type); + Py_CLEAR(st->PyCFuncPtrType_Type); + Py_CLEAR(st->PyCData_Type); + Py_CLEAR(st->Struct_Type); + Py_CLEAR(st->Union_Type); + Py_CLEAR(st->PyCArray_Type); + Py_CLEAR(st->Simple_Type); + Py_CLEAR(st->PyCPointer_Type); + Py_CLEAR(st->PyCFuncPtr_Type); +#ifdef MS_WIN32 + Py_CLEAR(st->PyComError_Type); +#endif + Py_CLEAR(st->PyCType_Type); + return 0; +} + +static void +module_free(void *module) +{ + (void)module_clear((PyObject *)module); +} + +static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _ctypes_mod_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +struct PyModuleDef _ctypesmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_ctypes", + .m_doc = _ctypes__doc__, + .m_size = sizeof(ctypes_state), + .m_methods = _ctypes_module_methods, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, +}; + PyMODINIT_FUNC PyInit__ctypes(void) { - PyObject *mod = PyModule_Create(&_ctypesmodule); - if (!mod) { - return NULL; - } - - if (_ctypes_mod_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - return mod; + return PyModuleDef_Init(&_ctypesmodule); } /* diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index ecc60417790417..7bac592fd38fb9 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,8 +1,7 @@ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif // gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt @@ -14,6 +13,10 @@ #include +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +# include "../_complex.h" // csqrt() +# undef I // for _ctypes_test_generated.c.h +#endif #include // printf() #include // qsort() #include // memset() @@ -23,6 +26,8 @@ #define EXPORT(x) Py_EXPORTED_SYMBOL x +#include "_ctypes_test_generated.c.h" + /* some functions handy for testing */ EXPORT(int) @@ -179,9 +184,9 @@ _testfunc_array_in_struct3B_set_defaults(void) /* * Test3C struct tests the MAX_STRUCT_SIZE 32. Structs containing arrays of up - * to four floating point types are passed in registers on Arm platforms. - * This struct is used for within bounds test on Arm platfroms and for an - * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 32. + * to four floating-point types are passed in registers on Arm platforms. + * This struct is used for within-bounds tests on Arm platforms and for an + * out-of-bounds test for platforms where MAX_STRUCT_SIZE is less than 32. * See gh-110190. */ typedef struct { @@ -203,9 +208,9 @@ _testfunc_array_in_struct3C_set_defaults(void) /* * Test3D struct tests the MAX_STRUCT_SIZE 64. Structs containing arrays of up - * to eight floating point types are passed in registers on PPC64LE platforms. - * This struct is used for within bounds test on PPC64LE platfroms and for an - * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 64. + * to eight floating-point types are passed in registers on PPC64LE platforms. + * This struct is used for within bounds test on PPC64LE platforms and for an + * out-of-bounds tests for platforms where MAX_STRUCT_SIZE is less than 64. * See gh-110190. */ typedef struct { @@ -344,6 +349,31 @@ _testfunc_bitfield_by_reference2(Test7 *in) { return result; } +typedef struct{ + uint16_t A ; + uint16_t B : 9; + uint16_t C : 1; + uint16_t D : 1; + uint16_t E : 1; + uint16_t F : 1; + uint16_t G : 3; + uint32_t H : 10; + uint32_t I : 20; + uint32_t J : 2; +} Test9; + +EXPORT(long) +_testfunc_bitfield_by_reference3(Test9 *in, long pos) { + long data[] = {in->A , in->B , in->C , in->D , in->E , in->F , in->G , in->H , in->I , in->J}; + long data_length = (long) (sizeof(data)/sizeof(data[0])); + if(pos < 0) + return -1; + if(pos >= data_length) + return -1; + + return data[pos]; +} + typedef union { signed int A: 1, B:2, C:3, D:2; } Test8; @@ -417,6 +447,23 @@ EXPORT(double) my_sqrt(double a) return sqrt(a); } +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +EXPORT(double complex) my_csqrt(double complex a) +{ + return csqrt(a); +} + +EXPORT(float complex) my_csqrtf(float complex a) +{ + return csqrtf(a); +} + +EXPORT(long double complex) my_csqrtl(long double complex a) +{ + return csqrtl(a); +} +#endif + EXPORT(void) my_qsort(void *base, size_t num, size_t width, int(*compare)(const void*, const void*)) { qsort(base, num, width, compare); @@ -705,7 +752,7 @@ struct BITS { */ #ifndef __xlc__ #define SIGNED_SHORT_BITFIELDS - short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; #endif }; @@ -735,12 +782,58 @@ EXPORT(int) unpack_bitfields(struct BITS *bits, char name) return 999; } +#if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) +struct +#ifndef MS_WIN32 +__attribute__ ((ms_struct)) +#endif +BITS_msvc +{ + signed int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9; +/* + * The test case needs/uses "signed short" bitfields, but the + * IBM XLC compiler does not support this + */ +#ifndef __xlc__ +#define SIGNED_SHORT_BITFIELDS + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; +#endif +}; + +EXPORT(int) unpack_bitfields_msvc(struct BITS_msvc *bits, char name) +{ + switch (name) { + case 'A': return bits->A; + case 'B': return bits->B; + case 'C': return bits->C; + case 'D': return bits->D; + case 'E': return bits->E; + case 'F': return bits->F; + case 'G': return bits->G; + case 'H': return bits->H; + case 'I': return bits->I; + +#ifdef SIGNED_SHORT_BITFIELDS + case 'M': return bits->M; + case 'N': return bits->N; + case 'O': return bits->O; + case 'P': return bits->P; + case 'Q': return bits->Q; + case 'R': return bits->R; + case 'S': return bits->S; +#endif + } + return 999; +} +#endif + static PyMethodDef module_methods[] = { /* {"get_last_tf_arg_s", get_last_tf_arg_s, METH_NOARGS}, {"get_last_tf_arg_u", get_last_tf_arg_u, METH_NOARGS}, */ {"func_si", py_func_si, METH_VARARGS}, {"func", py_func, METH_NOARGS}, + {"get_generated_test_data", get_generated_test_data, METH_O}, { NULL, NULL, 0, NULL}, }; @@ -1168,6 +1261,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item) static struct PyModuleDef_Slot _ctypes_test_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h new file mode 100644 index 00000000000000..46a3e4b01e2259 --- /dev/null +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -0,0 +1,1885 @@ + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + + // Append VALUE to the result. + #define APPEND(ITEM) { \ + PyObject *item = ITEM; \ + if (!item) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + int rv = PyList_Append(result, item); \ + Py_DECREF(item); \ + if (rv < 0) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \ + TYPE v = VAL; \ + TARGET = v; \ + APPEND(PyBytes_FromStringAndSize( \ + (char*)&value, sizeof(value))); \ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \ + SET_AND_APPEND(TYPE, TARGET, -1) \ + SET_AND_APPEND(TYPE, TARGET, 1) \ + SET_AND_APPEND(TYPE, TARGET, 0) \ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleInt") == 0) { + + struct SingleInt { + int a; + }; + struct SingleInt value = {0}; + APPEND(PyUnicode_FromString("SingleInt")); + APPEND(PyLong_FromLong(sizeof(struct SingleInt))); + APPEND(PyLong_FromLong(_Alignof(struct SingleInt))); + TEST_FIELD(int, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleInt_Union") == 0) { + + union SingleInt_Union { + int a; + }; + union SingleInt_Union value = {0}; + APPEND(PyUnicode_FromString("SingleInt_Union")); + APPEND(PyLong_FromLong(sizeof(union SingleInt_Union))); + APPEND(PyLong_FromLong(_Alignof(union SingleInt_Union))); + TEST_FIELD(int, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleU32") == 0) { + + struct SingleU32 { + uint32_t a; + }; + struct SingleU32 value = {0}; + APPEND(PyUnicode_FromString("SingleU32")); + APPEND(PyLong_FromLong(sizeof(struct SingleU32))); + APPEND(PyLong_FromLong(_Alignof(struct SingleU32))); + TEST_FIELD(uint32_t, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SimpleStruct") == 0) { + + struct SimpleStruct { + int32_t x; + int8_t y; + uint16_t z; + }; + struct SimpleStruct value = {0}; + APPEND(PyUnicode_FromString("SimpleStruct")); + APPEND(PyLong_FromLong(sizeof(struct SimpleStruct))); + APPEND(PyLong_FromLong(_Alignof(struct SimpleStruct))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SimpleUnion") == 0) { + + union SimpleUnion { + int32_t x; + int8_t y; + uint16_t z; + }; + union SimpleUnion value = {0}; + APPEND(PyUnicode_FromString("SimpleUnion")); + APPEND(PyLong_FromLong(sizeof(union SimpleUnion))); + APPEND(PyLong_FromLong(_Alignof(union SimpleUnion))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "ManyTypes") == 0) { + + struct ManyTypes { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + struct ManyTypes value = {0}; + APPEND(PyUnicode_FromString("ManyTypes")); + APPEND(PyLong_FromLong(sizeof(struct ManyTypes))); + APPEND(PyLong_FromLong(_Alignof(struct ManyTypes))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "ManyTypesU") == 0) { + + union ManyTypesU { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + union ManyTypesU value = {0}; + APPEND(PyUnicode_FromString("ManyTypesU")); + APPEND(PyLong_FromLong(sizeof(union ManyTypesU))); + APPEND(PyLong_FromLong(_Alignof(union ManyTypesU))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Nested") == 0) { + + struct Nested { + struct { + int32_t x; + int8_t y; + uint16_t z; + } a; + union { + int32_t x; + int8_t y; + uint16_t z; + } b; + struct { + int32_t x; + int8_t y; + uint16_t z; + }; + }; + struct Nested value = {0}; + APPEND(PyUnicode_FromString("Nested")); + APPEND(PyLong_FromLong(sizeof(struct Nested))); + APPEND(PyLong_FromLong(_Alignof(struct Nested))); + TEST_FIELD(int32_t, value.a.x); + TEST_FIELD(int8_t, value.a.y); + TEST_FIELD(uint16_t, value.a.z); + TEST_FIELD(int32_t, value.b.x); + TEST_FIELD(int8_t, value.b.y); + TEST_FIELD(uint16_t, value.b.z); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed1") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Packed1 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed1 value = {0}; + APPEND(PyUnicode_FromString("Packed1")); + APPEND(PyLong_FromLong(sizeof(struct Packed1))); + APPEND(PyLong_FromLong(_Alignof(struct Packed1))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed2") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 2) + struct GCC_ATTR(ms_struct) Packed2 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed2 value = {0}; + APPEND(PyUnicode_FromString("Packed2")); + APPEND(PyLong_FromLong(sizeof(struct Packed2))); + APPEND(PyLong_FromLong(_Alignof(struct Packed2))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed3") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 4) + struct GCC_ATTR(ms_struct) Packed3 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed3 value = {0}; + APPEND(PyUnicode_FromString("Packed3")); + APPEND(PyLong_FromLong(sizeof(struct Packed3))); + APPEND(PyLong_FromLong(_Alignof(struct Packed3))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed4") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 8) + struct GCC_ATTR(ms_struct) Packed4 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed4 value = {0}; + APPEND(PyUnicode_FromString("Packed4")); + APPEND(PyLong_FromLong(sizeof(struct Packed4))); + APPEND(PyLong_FromLong(_Alignof(struct Packed4))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "X86_32EdgeCase") == 0) { + + struct X86_32EdgeCase { + int32_t a; + int64_t b; + int32_t c; + }; + struct X86_32EdgeCase value = {0}; + APPEND(PyUnicode_FromString("X86_32EdgeCase")); + APPEND(PyLong_FromLong(sizeof(struct X86_32EdgeCase))); + APPEND(PyLong_FromLong(_Alignof(struct X86_32EdgeCase))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int32_t, value.c); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "MSBitFieldExample") == 0) { + + struct MSBitFieldExample { + unsigned int a :4; + unsigned int b :5; + unsigned int c :7; + }; + struct MSBitFieldExample value = {0}; + APPEND(PyUnicode_FromString("MSBitFieldExample")); + APPEND(PyLong_FromLong(sizeof(struct MSBitFieldExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSBitFieldExample))); + TEST_FIELD(unsigned int, value.a); + TEST_FIELD(unsigned int, value.b); + TEST_FIELD(unsigned int, value.c); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "MSStraddlingExample") == 0) { + + struct MSStraddlingExample { + unsigned int first :9; + unsigned int second :7; + unsigned int may_straddle :30; + unsigned int last :18; + }; + struct MSStraddlingExample value = {0}; + APPEND(PyUnicode_FromString("MSStraddlingExample")); + APPEND(PyLong_FromLong(sizeof(struct MSStraddlingExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSStraddlingExample))); + TEST_FIELD(unsigned int, value.first); + TEST_FIELD(unsigned int, value.second); + TEST_FIELD(unsigned int, value.may_straddle); + TEST_FIELD(unsigned int, value.last); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "IntBits") == 0) { + + struct IntBits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + struct IntBits value = {0}; + APPEND(PyUnicode_FromString("IntBits")); + APPEND(PyLong_FromLong(sizeof(struct IntBits))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Bits") == 0) { + + #if (!defined(__xlc__)) + + struct Bits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + struct Bits value = {0}; + APPEND(PyUnicode_FromString("Bits")); + APPEND(PyLong_FromLong(sizeof(struct Bits))); + APPEND(PyLong_FromLong(_Alignof(struct Bits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "IntBits_MSVC") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + struct GCC_ATTR(ms_struct) IntBits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + struct IntBits_MSVC value = {0}; + APPEND(PyUnicode_FromString("IntBits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct IntBits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Bits_MSVC") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + struct GCC_ATTR(ms_struct) Bits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + struct Bits_MSVC value = {0}; + APPEND(PyUnicode_FromString("Bits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct Bits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct Bits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "I64Bits") == 0) { + + #if (!defined(__xlc__)) + + struct I64Bits { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + struct I64Bits value = {0}; + APPEND(PyUnicode_FromString("I64Bits")); + APPEND(PyLong_FromLong(sizeof(struct I64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct I64Bits))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "U64Bits") == 0) { + + #if (!defined(__xlc__)) + + struct U64Bits { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + struct U64Bits value = {0}; + APPEND(PyUnicode_FromString("U64Bits")); + APPEND(PyLong_FromLong(sizeof(struct U64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct U64Bits))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_8 { + int8_t a :3; + int8_t b :3; + int8_t c :1; + }; + struct Struct331_8 value = {0}; + APPEND(PyUnicode_FromString("Struct331_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_8 { + int8_t a :1; + int8_t b :6; + int8_t c :1; + }; + struct Struct1x1_8 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_8 { + int8_t a :1; + int8_t full; + int8_t b :6; + int8_t c :1; + }; + struct Struct1nx1_8 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.full); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_8 { + int8_t a :3; + int8_t b :6; + int8_t c :6; + }; + struct Struct3xx_8 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u8 { + uint8_t a :3; + uint8_t b :3; + uint8_t c :1; + }; + struct Struct331_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u8 { + uint8_t a :1; + uint8_t b :6; + uint8_t c :1; + }; + struct Struct1x1_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u8 { + uint8_t a :1; + uint8_t full; + uint8_t b :6; + uint8_t c :1; + }; + struct Struct1nx1_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.full); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u8 { + uint8_t a :3; + uint8_t b :6; + uint8_t c :6; + }; + struct Struct3xx_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_16 { + int16_t a :3; + int16_t b :3; + int16_t c :1; + }; + struct Struct331_16 value = {0}; + APPEND(PyUnicode_FromString("Struct331_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_16 { + int16_t a :1; + int16_t b :14; + int16_t c :1; + }; + struct Struct1x1_16 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_16 { + int16_t a :1; + int16_t full; + int16_t b :14; + int16_t c :1; + }; + struct Struct1nx1_16 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.full); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_16 { + int16_t a :3; + int16_t b :14; + int16_t c :14; + }; + struct Struct3xx_16 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u16 { + uint16_t a :3; + uint16_t b :3; + uint16_t c :1; + }; + struct Struct331_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u16 { + uint16_t a :1; + uint16_t b :14; + uint16_t c :1; + }; + struct Struct1x1_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u16 { + uint16_t a :1; + uint16_t full; + uint16_t b :14; + uint16_t c :1; + }; + struct Struct1nx1_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.full); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u16 { + uint16_t a :3; + uint16_t b :14; + uint16_t c :14; + }; + struct Struct3xx_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_32 { + int32_t a :3; + int32_t b :3; + int32_t c :1; + }; + struct Struct331_32 value = {0}; + APPEND(PyUnicode_FromString("Struct331_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_32 { + int32_t a :1; + int32_t b :30; + int32_t c :1; + }; + struct Struct1x1_32 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_32 { + int32_t a :1; + int32_t full; + int32_t b :30; + int32_t c :1; + }; + struct Struct1nx1_32 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.full); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_32 { + int32_t a :3; + int32_t b :30; + int32_t c :30; + }; + struct Struct3xx_32 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u32 { + uint32_t a :3; + uint32_t b :3; + uint32_t c :1; + }; + struct Struct331_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u32 { + uint32_t a :1; + uint32_t b :30; + uint32_t c :1; + }; + struct Struct1x1_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u32 { + uint32_t a :1; + uint32_t full; + uint32_t b :30; + uint32_t c :1; + }; + struct Struct1nx1_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.full); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u32 { + uint32_t a :3; + uint32_t b :30; + uint32_t c :30; + }; + struct Struct3xx_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_64 { + int64_t a :3; + int64_t b :3; + int64_t c :1; + }; + struct Struct331_64 value = {0}; + APPEND(PyUnicode_FromString("Struct331_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_64 { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + struct Struct1x1_64 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_64 { + int64_t a :1; + int64_t full; + int64_t b :62; + int64_t c :1; + }; + struct Struct1nx1_64 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.full); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_64 { + int64_t a :3; + int64_t b :62; + int64_t c :62; + }; + struct Struct3xx_64 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u64 { + uint64_t a :3; + uint64_t b :3; + uint64_t c :1; + }; + struct Struct331_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u64 { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + struct Struct1x1_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u64 { + uint64_t a :1; + uint64_t full; + uint64_t b :62; + uint64_t c :1; + }; + struct Struct1nx1_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.full); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u64 { + uint64_t a :3; + uint64_t b :62; + uint64_t c :62; + }; + struct Struct3xx_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed1") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed1 { + signed char a :4; + int b :4; + }; + struct Mixed1 value = {0}; + APPEND(PyUnicode_FromString("Mixed1")); + APPEND(PyLong_FromLong(sizeof(struct Mixed1))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed1))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed2") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed2 { + signed char a :4; + int32_t b :32; + }; + struct Mixed2 value = {0}; + APPEND(PyUnicode_FromString("Mixed2")); + APPEND(PyLong_FromLong(sizeof(struct Mixed2))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed2))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int32_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed3") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed3 { + signed char a :4; + unsigned char b :4; + }; + struct Mixed3 value = {0}; + APPEND(PyUnicode_FromString("Mixed3")); + APPEND(PyLong_FromLong(sizeof(struct Mixed3))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed3))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed4") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed4 { + short a :4; + short b :4; + int c :24; + short d :4; + short e :4; + int f :24; + }; + struct Mixed4 value = {0}; + APPEND(PyUnicode_FromString("Mixed4")); + APPEND(PyLong_FromLong(sizeof(struct Mixed4))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed4))); + TEST_FIELD(short, value.a); + TEST_FIELD(short, value.b); + TEST_FIELD(int, value.c); + TEST_FIELD(short, value.d); + TEST_FIELD(short, value.e); + TEST_FIELD(int, value.f); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed5") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed5 { + unsigned int A :1; + unsigned short B :16; + }; + struct Mixed5 value = {0}; + APPEND(PyUnicode_FromString("Mixed5")); + APPEND(PyLong_FromLong(sizeof(struct Mixed5))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed5))); + TEST_FIELD(unsigned int, value.A); + TEST_FIELD(unsigned short, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed6") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed6 { + unsigned long long A :1; + unsigned int B :32; + }; + struct Mixed6 value = {0}; + APPEND(PyUnicode_FromString("Mixed6")); + APPEND(PyLong_FromLong(sizeof(struct Mixed6))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed6))); + TEST_FIELD(unsigned long long, value.A); + TEST_FIELD(unsigned int, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed7") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed7 { + uint32_t A; + uint32_t B :20; + uint64_t C :24; + }; + struct Mixed7 value = {0}; + APPEND(PyUnicode_FromString("Mixed7")); + APPEND(PyLong_FromLong(sizeof(struct Mixed7))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed7))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(uint64_t, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_a") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed8_a { + uint32_t A; + uint32_t B :32; + unsigned long long C :1; + }; + struct Mixed8_a value = {0}; + APPEND(PyUnicode_FromString("Mixed8_a")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_a))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_a))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_b") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed8_b { + uint32_t A; + uint32_t B; + unsigned long long C :1; + }; + struct Mixed8_b value = {0}; + APPEND(PyUnicode_FromString("Mixed8_b")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_b))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_b))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed9") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed9 { + uint8_t A; + uint32_t B :1; + }; + struct Mixed9 value = {0}; + APPEND(PyUnicode_FromString("Mixed9")); + APPEND(PyLong_FromLong(sizeof(struct Mixed9))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed9))); + TEST_FIELD(uint8_t, value.A); + TEST_FIELD(uint32_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed10") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed10 { + uint32_t A :1; + uint64_t B :1; + }; + struct Mixed10 value = {0}; + APPEND(PyUnicode_FromString("Mixed10")); + APPEND(PyLong_FromLong(sizeof(struct Mixed10))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed10))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_95496") == 0) { + + #if (!defined(__xlc__)) + + struct Example_gh_95496 { + uint32_t A :1; + uint64_t B :1; + }; + struct Example_gh_95496 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_95496")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_95496))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_95496))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_bad") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_bad { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + struct Example_gh_84039_bad value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_bad")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_bad))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_bad))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good_a") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good_a { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + }; + #pragma pack(pop) + struct Example_gh_84039_good_a value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_good_a")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good_a))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good_a))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good { + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + } a; + #pragma pack(pop) + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + struct Example_gh_84039_good value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_good")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good))); + TEST_FIELD(uint8_t, value.a.a0); + TEST_FIELD(uint8_t, value.a.a1); + TEST_FIELD(uint8_t, value.a.a2); + TEST_FIELD(uint8_t, value.a.a3); + TEST_FIELD(uint8_t, value.a.a4); + TEST_FIELD(uint8_t, value.a.a5); + TEST_FIELD(uint8_t, value.a.a6); + TEST_FIELD(uint8_t, value.a.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_73939") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_73939 { + uint16_t P; + uint16_t L :9; + uint16_t Pro :1; + uint16_t G :1; + uint16_t IB :1; + uint16_t IR :1; + uint16_t R :3; + uint32_t T :10; + uint32_t C :20; + uint32_t R2 :2; + }; + #pragma pack(pop) + struct Example_gh_73939 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_73939")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_73939))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_73939))); + TEST_FIELD(uint16_t, value.P); + TEST_FIELD(uint16_t, value.L); + TEST_FIELD(uint16_t, value.Pro); + TEST_FIELD(uint16_t, value.G); + TEST_FIELD(uint16_t, value.IB); + TEST_FIELD(uint16_t, value.IR); + TEST_FIELD(uint16_t, value.R); + TEST_FIELD(uint32_t, value.T); + TEST_FIELD(uint32_t, value.C); + TEST_FIELD(uint32_t, value.R2); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098") == 0) { + + #if (!defined(__xlc__)) + + struct Example_gh_86098 { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + struct Example_gh_86098 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_86098")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098_pack") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_86098_pack { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + #pragma pack(pop) + struct Example_gh_86098_pack value = {0}; + APPEND(PyUnicode_FromString("Example_gh_86098_pack")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098_pack))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098_pack))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "AnonBitfields") == 0) { + + #if (!defined(__xlc__)) + + struct AnonBitfields { + struct { + signed char a :4; + unsigned char b :4; + }; + signed char y; + }; + struct AnonBitfields value = {0}; + APPEND(PyUnicode_FromString("AnonBitfields")); + APPEND(PyLong_FromLong(sizeof(struct AnonBitfields))); + APPEND(PyLong_FromLong(_Alignof(struct AnonBitfields))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + TEST_FIELD(signed char, value.y); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index f70479435915ff..7b9f6437c7d55f 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -109,10 +109,14 @@ PrintError(const char *msg, ...) * slower. */ static void -TryAddRef(StgDictObject *dict, CDataObject *obj) +TryAddRef(PyObject *cnv, CDataObject *obj) { IUnknown *punk; - int r = PyDict_Contains((PyObject *)dict, &_Py_ID(_needs_com_addref_)); + PyObject *attrdict = _PyType_GetDict((PyTypeObject *)cnv); + if (!attrdict) { + return; + } + int r = PyDict_Contains(attrdict, &_Py_ID(_needs_com_addref_)); if (r <= 0) { if (r < 0) { PrintError("getting _needs_com_addref_"); @@ -132,7 +136,10 @@ TryAddRef(StgDictObject *dict, CDataObject *obj) * Call the python object with all arguments * */ -static void _CallPythonObject(void *mem, + +// BEWARE: The GIL needs to be held throughout the function +static void _CallPythonObject(ctypes_state *st, + void *mem, ffi_type *restype, SETFUNC setfunc, PyObject *callable, @@ -144,32 +151,34 @@ static void _CallPythonObject(void *mem, Py_ssize_t i = 0, j = 0, nargs = 0; PyObject *error_object = NULL; int *space; - PyGILState_STATE state = PyGILState_Ensure(); assert(PyTuple_Check(converters)); nargs = PyTuple_GET_SIZE(converters); assert(nargs <= CTYPES_MAX_ARGCOUNT); PyObject **args = alloca(nargs * sizeof(PyObject *)); PyObject **cnvs = PySequence_Fast_ITEMS(converters); - ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < nargs; i++) { PyObject *cnv = cnvs[i]; // borrowed ref - StgDictObject *dict; - dict = PyType_stgdict(cnv); - if (dict && dict->getfunc && !_ctypes_simple_instance(cnv)) { - PyObject *v = dict->getfunc(*pArgs, dict->size); + StgInfo *info; + if (PyStgInfo_FromType(st, cnv, &info) < 0) { + goto Done; + } + + if (info && info->getfunc && !_ctypes_simple_instance(st, cnv)) { + PyObject *v = info->getfunc(*pArgs, info->size); if (!v) { PrintError("create argument %zd:\n", i); goto Done; } args[i] = v; /* XXX XXX XX - We have the problem that c_byte or c_short have dict->size of + We have the problem that c_byte or c_short have info->size of 1 resp. 4, but these parameters are pushed as sizeof(int) bytes. BTW, the same problem occurs when they are pushed as parameters */ - } else if (dict) { + } + else if (info) { /* Hm, shouldn't we use PyCData_AtAddress() or something like that instead? */ CDataObject *obj = (CDataObject *)_PyObject_CallNoArgs(cnv); if (!obj) { @@ -181,10 +190,10 @@ static void _CallPythonObject(void *mem, PrintError("unexpected result of create argument %zd:\n", i); goto Done; } - memcpy(obj->b_ptr, *pArgs, dict->size); + memcpy(obj->b_ptr, *pArgs, info->size); args[i] = (PyObject *)obj; #ifdef MS_WIN32 - TryAddRef(dict, obj); + TryAddRef(cnv, obj); #endif } else { PyErr_SetString(PyExc_TypeError, @@ -197,7 +206,7 @@ static void _CallPythonObject(void *mem, } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { - error_object = _ctypes_get_errobj(&space); + error_object = _ctypes_get_errobj(st, &space); if (error_object == NULL) goto Done; if (flags & FUNCFLAG_USE_ERRNO) { @@ -286,7 +295,6 @@ static void _CallPythonObject(void *mem, for (j = 0; j < i; j++) { Py_DECREF(args[j]); } - PyGILState_Release(state); } static void closure_fcn(ffi_cif *cif, @@ -294,23 +302,28 @@ static void closure_fcn(ffi_cif *cif, void **args, void *userdata) { + PyGILState_STATE state = PyGILState_Ensure(); + CThunkObject *p = (CThunkObject *)userdata; + ctypes_state *st = get_module_state_by_class(Py_TYPE(p)); - _CallPythonObject(resp, + _CallPythonObject(st, + resp, p->ffi_restype, p->setfunc, p->callable, p->converters, p->flags, args); + + PyGILState_Release(state); } -static CThunkObject* CThunkObject_new(Py_ssize_t nargs) +static CThunkObject* CThunkObject_new(ctypes_state *st, Py_ssize_t nargs) { CThunkObject *p; Py_ssize_t i; - ctypes_state *st = GLOBAL_STATE(); p = PyObject_GC_NewVar(CThunkObject, st->PyCThunk_Type, nargs); if (p == NULL) { return NULL; @@ -332,7 +345,8 @@ static CThunkObject* CThunkObject_new(Py_ssize_t nargs) return p; } -CThunkObject *_ctypes_alloc_callback(PyObject *callable, +CThunkObject *_ctypes_alloc_callback(ctypes_state *st, + PyObject *callable, PyObject *converters, PyObject *restype, int flags) @@ -344,14 +358,11 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(PyTuple_Check(converters)); nargs = PyTuple_GET_SIZE(converters); - p = CThunkObject_new(nargs); + p = CThunkObject_new(st, nargs); if (p == NULL) return NULL; -#ifdef Py_DEBUG - ctypes_state *st = GLOBAL_STATE(); assert(CThunk_CheckExact(st, (PyObject *)p)); -#endif p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { @@ -363,7 +374,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, PyObject **cnvs = PySequence_Fast_ITEMS(converters); for (i = 0; i < nargs; ++i) { PyObject *cnv = cnvs[i]; // borrowed ref - p->atypes[i] = _ctypes_get_ffi_type(cnv); + p->atypes[i] = _ctypes_get_ffi_type(st, cnv); } p->atypes[i] = NULL; @@ -372,14 +383,18 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, p->setfunc = NULL; p->ffi_restype = &ffi_type_void; } else { - StgDictObject *dict = PyType_stgdict(restype); - if (dict == NULL || dict->setfunc == NULL) { + StgInfo *info; + if (PyStgInfo_FromType(st, restype, &info) < 0) { + goto error; + } + + if (info == NULL || info->setfunc == NULL) { PyErr_SetString(PyExc_TypeError, "invalid result type for callback function"); goto error; } - p->setfunc = dict->setfunc; - p->ffi_restype = &dict->ffi_type_pointer; + p->setfunc = info->setfunc; + p->ffi_restype = &info->ffi_type_pointer; } cc = FFI_DEFAULT_ABI; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 97d1dbaae03d4f..5ac9cf16681645 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -105,6 +105,10 @@ module _ctypes #include "pycore_global_objects.h"// _Py_ID() #include "pycore_traceback.h" // _PyTraceback_Add() +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +#include "../_complex.h" // complex +#endif + #include "clinic/callproc.c.h" #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem" @@ -153,22 +157,22 @@ static void pymem_destructor(PyObject *ptr) kept alive in the thread state dictionary as long as the thread itself. */ PyObject * -_ctypes_get_errobj(int **pspace) +_ctypes_get_errobj(ctypes_state *st, int **pspace) { PyObject *dict = PyThreadState_GetDict(); PyObject *errobj; - static PyObject *error_object_name; if (dict == NULL) { PyErr_SetString(PyExc_RuntimeError, "cannot get thread state"); return NULL; } - if (error_object_name == NULL) { - error_object_name = PyUnicode_InternFromString("ctypes.error_object"); - if (error_object_name == NULL) + if (st->error_object_name == NULL) { + st->error_object_name = PyUnicode_InternFromString("ctypes.error_object"); + if (st->error_object_name == NULL) { return NULL; + } } - if (PyDict_GetItemRef(dict, error_object_name, &errobj) < 0) { + if (PyDict_GetItemRef(dict, st->error_object_name, &errobj) < 0) { return NULL; } if (errobj) { @@ -188,8 +192,7 @@ _ctypes_get_errobj(int **pspace) PyMem_Free(space); return NULL; } - if (-1 == PyDict_SetItem(dict, error_object_name, - errobj)) { + if (PyDict_SetItem(dict, st->error_object_name, errobj) < 0) { Py_DECREF(errobj); return NULL; } @@ -202,7 +205,8 @@ static PyObject * get_error_internal(PyObject *self, PyObject *args, int index) { int *space; - PyObject *errobj = _ctypes_get_errobj(&space); + ctypes_state *st = get_module_state(self); + PyObject *errobj = _ctypes_get_errobj(st, &space); PyObject *result; if (errobj == NULL) @@ -222,7 +226,8 @@ set_error_internal(PyObject *self, PyObject *args, int index) if (!PyArg_ParseTuple(args, "i", &new_errno)) { return NULL; } - errobj = _ctypes_get_errobj(&space); + ctypes_state *st = get_module_state(self); + errobj = _ctypes_get_errobj(st, &space); if (errobj == NULL) return NULL; old_errno = space[index]; @@ -473,10 +478,9 @@ check_hresult(PyObject *self, PyObject *args) /**************************************************************/ PyCArgObject * -PyCArgObject_new(void) +PyCArgObject_new(ctypes_state *st) { PyCArgObject *p; - ctypes_state *st = GLOBAL_STATE(); p = PyObject_GC_New(PyCArgObject, st->PyCArg_Type); if (p == NULL) return NULL; @@ -651,6 +655,11 @@ union result { double d; float f; void *p; +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) + double complex C; + float complex E; + long double complex F; +#endif }; struct argument { @@ -662,17 +671,22 @@ struct argument { /* * Convert a single Python object into a PyCArgObject and return it. */ -static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) +static int ConvParam(ctypes_state *st, + PyObject *obj, Py_ssize_t index, struct argument *pa) { - StgDictObject *dict; pa->keep = NULL; /* so we cannot forget it later */ - dict = PyObject_stgdict(obj); - if (dict) { + StgInfo *info; + int result = PyStgInfo_FromObject(st, obj, &info); + if (result < 0) { + return -1; + } + if (info) { + assert(info); PyCArgObject *carg; - assert(dict->paramfunc); - /* If it has an stgdict, it is a CDataObject */ - carg = dict->paramfunc((CDataObject *)obj); + assert(info->paramfunc); + /* If it has an stginfo, it is a CDataObject */ + carg = info->paramfunc(st, (CDataObject *)obj); if (carg == NULL) return -1; pa->ffi_type = carg->pffi_type; @@ -681,7 +695,6 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) return 0; } - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, obj)) { PyCArgObject *carg = (PyCArgObject *)obj; pa->ffi_type = carg->pffi_type; @@ -744,7 +757,7 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) */ if (arg) { int result; - result = ConvParam(arg, index, pa); + result = ConvParam(st, arg, index, pa); Py_DECREF(arg); return result; } @@ -778,26 +791,33 @@ int can_return_struct_as_sint64(size_t s) #endif -ffi_type *_ctypes_get_ffi_type(PyObject *obj) +// returns NULL with exception set on error +ffi_type *_ctypes_get_ffi_type(ctypes_state *st, PyObject *obj) { - StgDictObject *dict; - if (obj == NULL) + if (obj == NULL) { return &ffi_type_sint; - dict = PyType_stgdict(obj); - if (dict == NULL) + } + + StgInfo *info; + if (PyStgInfo_FromType(st, obj, &info) < 0) { + return NULL; + } + + if (info == NULL) { return &ffi_type_sint; + } #if defined(MS_WIN32) && !defined(_WIN32_WCE) /* This little trick works correctly with MSVC. It returns small structures in registers */ - if (dict->ffi_type_pointer.type == FFI_TYPE_STRUCT) { - if (can_return_struct_as_int(dict->ffi_type_pointer.size)) + if (info->ffi_type_pointer.type == FFI_TYPE_STRUCT) { + if (can_return_struct_as_int(info->ffi_type_pointer.size)) return &ffi_type_sint32; - else if (can_return_struct_as_sint64 (dict->ffi_type_pointer.size)) + else if (can_return_struct_as_sint64 (info->ffi_type_pointer.size)) return &ffi_type_sint64; } #endif - return &dict->ffi_type_pointer; + return &info->ffi_type_pointer; } @@ -813,7 +833,8 @@ ffi_type *_ctypes_get_ffi_type(PyObject *obj) * * void ffi_call(ffi_cif *cif, void *fn, void *rvalue, void **avalues); */ -static int _call_function_pointer(int flags, +static int _call_function_pointer(ctypes_state *st, + int flags, PPROC pProc, void **avalues, ffi_type **atypes, @@ -914,7 +935,7 @@ static int _call_function_pointer(int flags, } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { - error_object = _ctypes_get_errobj(&space); + error_object = _ctypes_get_errobj(st, &space); if (error_object == NULL) return -1; } @@ -981,9 +1002,9 @@ static int _call_function_pointer(int flags, * - If restype is another ctypes type, return an instance of that. * - Otherwise, call restype and return the result. */ -static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) +static PyObject *GetResult(ctypes_state *st, + PyObject *restype, void *result, PyObject *checker) { - StgDictObject *dict; PyObject *retval, *v; if (restype == NULL) @@ -993,22 +1014,27 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) Py_RETURN_NONE; } - dict = PyType_stgdict(restype); - if (dict == NULL) + StgInfo *info; + if (PyStgInfo_FromType(st, restype, &info) < 0) { + return NULL; + } + if (info == NULL) { return PyObject_CallFunction(restype, "i", *(int *)result); + } - if (dict->getfunc && !_ctypes_simple_instance(restype)) { - retval = dict->getfunc(result, dict->size); + if (info->getfunc && !_ctypes_simple_instance(st, restype)) { + retval = info->getfunc(result, info->size); /* If restype is py_object (detected by comparing getfunc with O_get), we have to call Py_DECREF because O_get has already called Py_INCREF. */ - if (dict->getfunc == _ctypes_get_fielddesc("O")->getfunc) { + if (info->getfunc == _ctypes_get_fielddesc("O")->getfunc) { Py_DECREF(retval); } - } else - retval = PyCData_FromBaseObj(restype, NULL, 0, result); - + } + else { + retval = PyCData_FromBaseObj(st, restype, NULL, 0, result); + } if (!checker || !retval) return retval; @@ -1070,7 +1096,7 @@ void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) #ifdef MS_WIN32 static PyObject * -GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) +GetComError(ctypes_state *st, HRESULT errcode, GUID *riid, IUnknown *pIunk) { HRESULT hr; ISupportErrorInfo *psei = NULL; @@ -1122,7 +1148,6 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) descr, source, helpfile, helpcontext, progid); if (obj) { - ctypes_state *st = GLOBAL_STATE(); PyErr_SetObject((PyObject *)st->PyComError_Type, obj); Py_DECREF(obj); } @@ -1153,7 +1178,8 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) * * - XXX various requirements for restype, not yet collected */ -PyObject *_ctypes_callproc(PPROC pProc, +PyObject *_ctypes_callproc(ctypes_state *st, + PPROC pProc, PyObject *argtuple, #ifdef MS_WIN32 IUnknown *pIunk, @@ -1183,7 +1209,7 @@ PyObject *_ctypes_callproc(PPROC pProc, if (argcount > CTYPES_MAX_ARGCOUNT) { - PyErr_Format(PyExc_ArgError, "too many arguments (%zi), maximum is %i", + PyErr_Format(st->PyExc_ArgError, "too many arguments (%zi), maximum is %i", argcount, CTYPES_MAX_ARGCOUNT); return NULL; } @@ -1216,20 +1242,20 @@ PyObject *_ctypes_callproc(PPROC pProc, converter = PyTuple_GET_ITEM(argtypes, i); v = PyObject_CallOneArg(converter, arg); if (v == NULL) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; } - err = ConvParam(v, i+1, pa); + err = ConvParam(st, v, i+1, pa); Py_DECREF(v); if (-1 == err) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; } } else { - err = ConvParam(arg, i+1, pa); + err = ConvParam(st, arg, i+1, pa); if (-1 == err) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; /* leaking ? */ } } @@ -1238,7 +1264,10 @@ PyObject *_ctypes_callproc(PPROC pProc, if (restype == Py_None) { rtype = &ffi_type_void; } else { - rtype = _ctypes_get_ffi_type(restype); + rtype = _ctypes_get_ffi_type(st, restype); + } + if (!rtype) { + goto cleanup; } resbuf = alloca(max(rtype->size, sizeof(ffi_arg))); @@ -1277,7 +1306,7 @@ PyObject *_ctypes_callproc(PPROC pProc, avalues[i] = (void *)&args[i].value; } - if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, + if (-1 == _call_function_pointer(st, flags, pProc, avalues, atypes, rtype, resbuf, Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) @@ -1305,7 +1334,7 @@ PyObject *_ctypes_callproc(PPROC pProc, #ifdef MS_WIN32 if (iid && pIunk) { if (*(int *)resbuf & 0x80000000) - retval = GetComError(*(HRESULT *)resbuf, iid, pIunk); + retval = GetComError(st, *(HRESULT *)resbuf, iid, pIunk); else retval = PyLong_FromLong(*(int *)resbuf); } else if (flags & FUNCFLAG_HRESULT) { @@ -1315,7 +1344,7 @@ PyObject *_ctypes_callproc(PPROC pProc, retval = PyLong_FromLong(*(int *)resbuf); } else #endif - retval = GetResult(restype, resbuf, checker); + retval = GetResult(st, restype, resbuf, checker); cleanup: for (i = 0; i < argcount; ++i) Py_XDECREF(args[i].keep); @@ -1444,8 +1473,10 @@ copy_com_pointer(PyObject *self, PyObject *args) return NULL; a.keep = b.keep = NULL; - if (-1 == ConvParam(p1, 0, &a) || -1 == ConvParam(p2, 1, &b)) + ctypes_state *st = get_module_state(self); + if (ConvParam(st, p1, 0, &a) < 0 || ConvParam(st, p2, 1, &b) < 0) { goto done; + } src = (IUnknown *)a.value.p; pdst = (IUnknown **)b.value.p; @@ -1624,7 +1655,9 @@ call_function(PyObject *self, PyObject *args) return NULL; } - result = _ctypes_callproc((PPROC)func, + ctypes_state *st = get_module_state(self); + result = _ctypes_callproc(st, + (PPROC)func, arguments, #ifdef MS_WIN32 NULL, @@ -1659,7 +1692,9 @@ call_cdeclfunction(PyObject *self, PyObject *args) return NULL; } - result = _ctypes_callproc((PPROC)func, + ctypes_state *st = get_module_state(self); + result = _ctypes_callproc(st, + (PPROC)func, arguments, #ifdef MS_WIN32 NULL, @@ -1683,13 +1718,16 @@ PyDoc_STRVAR(sizeof_doc, static PyObject * sizeof_func(PyObject *self, PyObject *obj) { - StgDictObject *dict; + ctypes_state *st = get_module_state(self); - dict = PyType_stgdict(obj); - if (dict) { - return PyLong_FromSsize_t(dict->size); + StgInfo *info; + if (PyStgInfo_FromType(st, obj, &info) < 0) { + return NULL; } - ctypes_state *st = GLOBAL_STATE(); + if (info) { + return PyLong_FromSsize_t(info->size); + } + if (CDataObject_Check(st, obj)) { return PyLong_FromSsize_t(((CDataObject *)obj)->b_size); } @@ -1706,16 +1744,14 @@ PyDoc_STRVAR(alignment_doc, static PyObject * align_func(PyObject *self, PyObject *obj) { - StgDictObject *dict; - - dict = PyType_stgdict(obj); - if (dict) - return PyLong_FromSsize_t(dict->align); - - dict = PyObject_stgdict(obj); - if (dict) - return PyLong_FromSsize_t(dict->align); - + ctypes_state *st = get_module_state(self); + StgInfo *info; + if (PyStgInfo_FromAny(st, obj, &info) < 0) { + return NULL; + } + if (info) { + return PyLong_FromSsize_t(info->align); + } PyErr_SetString(PyExc_TypeError, "no alignment info"); return NULL; @@ -1746,7 +1782,7 @@ byref(PyObject *self, PyObject *args) if (offset == -1 && PyErr_Occurred()) return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); if (!CDataObject_Check(st, obj)) { PyErr_Format(PyExc_TypeError, "byref() argument must be a ctypes instance, not '%s'", @@ -1754,7 +1790,7 @@ byref(PyObject *self, PyObject *args) return NULL; } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -1772,7 +1808,7 @@ PyDoc_STRVAR(addressof_doc, static PyObject * addressof(PyObject *self, PyObject *obj) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, "invalid type"); @@ -1824,7 +1860,6 @@ static PyObject * resize(PyObject *self, PyObject *args) { CDataObject *obj; - StgDictObject *dict; Py_ssize_t size; if (!PyArg_ParseTuple(args, @@ -1832,16 +1867,21 @@ resize(PyObject *self, PyObject *args) &obj, &size)) return NULL; - dict = PyObject_stgdict((PyObject *)obj); - if (dict == NULL) { + ctypes_state *st = get_module_state(self); + StgInfo *info; + int result = PyStgInfo_FromObject(st, (PyObject *)obj, &info); + if (result < 0) { + return NULL; + } + if (info == NULL) { PyErr_SetString(PyExc_TypeError, "expected ctypes instance"); return NULL; } - if (size < dict->size) { + if (size < info->size) { PyErr_Format(PyExc_ValueError, "minimum size is %zd", - dict->size); + info->size); return NULL; } if (obj->b_needsfree == 0) { @@ -1925,11 +1965,12 @@ create_pointer_type(PyObject *module, PyObject *cls) PyTypeObject *typ; PyObject *key; - if (PyDict_GetItemRef(_ctypes_ptrtype_cache, cls, &result) != 0) { + assert(module); + ctypes_state *st = get_module_state(module); + if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, cls, &result) != 0) { // found or error return result; } - ctypes_state *st = GLOBAL_STATE(); // not found if (PyUnicode_CheckExact(cls)) { PyObject *name = PyUnicode_FromFormat("LP_%U", cls); @@ -1959,7 +2000,7 @@ create_pointer_type(PyObject *module, PyObject *cls) PyErr_SetString(PyExc_TypeError, "must be a ctypes type"); return NULL; } - if (-1 == PyDict_SetItem(_ctypes_ptrtype_cache, key, result)) { + if (PyDict_SetItem(st->_ctypes_ptrtype_cache, key, result) < 0) { Py_DECREF(result); Py_DECREF(key); return NULL; @@ -1988,11 +2029,12 @@ create_pointer_inst(PyObject *module, PyObject *arg) PyObject *result; PyObject *typ; - if (PyDict_GetItemRef(_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { + ctypes_state *st = get_module_state(module); + if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { return NULL; } if (typ == NULL) { - typ = create_pointer_type(NULL, (PyObject *)Py_TYPE(arg)); + typ = create_pointer_type(module, (PyObject *)Py_TYPE(arg)); if (typ == NULL) return NULL; } @@ -2004,28 +2046,30 @@ create_pointer_inst(PyObject *module, PyObject *arg) static PyObject * buffer_info(PyObject *self, PyObject *arg) { - StgDictObject *dict = PyType_stgdict(arg); PyObject *shape; Py_ssize_t i; - if (dict == NULL) - dict = PyObject_stgdict(arg); - if (dict == NULL) { + ctypes_state *st = get_module_state(self); + StgInfo *info; + if (PyStgInfo_FromAny(st, arg, &info) < 0) { + return NULL; + } + if (info == NULL) { PyErr_SetString(PyExc_TypeError, "not a ctypes type or object"); return NULL; } - shape = PyTuple_New(dict->ndim); + shape = PyTuple_New(info->ndim); if (shape == NULL) return NULL; - for (i = 0; i < (int)dict->ndim; ++i) - PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(dict->shape[i])); + for (i = 0; i < (int)info->ndim; ++i) + PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(info->shape[i])); if (PyErr_Occurred()) { Py_DECREF(shape); return NULL; } - return Py_BuildValue("siN", dict->format, dict->ndim, shape); + return Py_BuildValue("siN", info->format, info->ndim, shape); } diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 1d5b0b14bc39e5..3220852c8398e0 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -14,9 +14,19 @@ #include #include "ctypes.h" +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +# include "../_complex.h" // complex +#endif #define CTYPES_CFIELD_CAPSULE_NAME_PYMEM "_ctypes/cfield.c pymem" +/*[clinic input] +module _ctypes +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ + +#include "clinic/cfield.c.h" + static void pymem_destructor(PyObject *ptr) { void *p = PyCapsule_GetPointer(ptr, CTYPES_CFIELD_CAPSULE_NAME_PYMEM); @@ -30,181 +40,158 @@ static void pymem_destructor(PyObject *ptr) /* PyCField_Type */ +/*[clinic input] +class _ctypes.CField "PyObject *" "PyObject" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=602817ea3ffc709c]*/ -/* - * Expects the size, index and offset for the current field in *psize and - * *poffset, stores the total size so far in *psize, the offset for the next - * field in *poffset, the alignment requirements for the current field in - * *palign, and returns a field descriptor for this field. - */ -/* - * bitfields extension: - * bitsize != 0: this is a bit field. - * pbitofs points to the current bit offset, this will be updated. - * prev_desc points to the type of the previous bitfield, if any. - */ -PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian) -{ - CFieldObject *self; - PyObject *proto; - Py_ssize_t size, align; - SETFUNC setfunc = NULL; - GETFUNC getfunc = NULL; - StgDictObject *dict; - int fieldtype; -#define NO_BITFIELD 0 -#define NEW_BITFIELD 1 -#define CONT_BITFIELD 2 -#define EXPAND_BITFIELD 3 - - ctypes_state *st = GLOBAL_STATE(); - PyTypeObject *tp = st->PyCField_Type; +static inline +Py_ssize_t NUM_BITS(Py_ssize_t bitsize); +static inline +Py_ssize_t LOW_BIT(Py_ssize_t offset); + + +/*[clinic input] +@classmethod +_ctypes.CField.__new__ as PyCField_new + + name: object(subclass_of='&PyUnicode_Type') + type as proto: object + size: Py_ssize_t + offset: Py_ssize_t + index: Py_ssize_t + bit_size as bit_size_obj: object = None + +[clinic start generated code]*/ + +static PyObject * +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, + PyObject *bit_size_obj) +/*[clinic end generated code: output=43649ef9157c5f58 input=3d813f56373c4caa]*/ +{ + CFieldObject* self = NULL; + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "size of field %R must not be negative, got %zd", + name, size); + goto error; + } + // assert: no overflow; + if ((unsigned long long int) size + >= (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8) { + PyErr_Format(PyExc_ValueError, + "size of field %R is too big: %zd", name, size); + goto error; + } + + PyTypeObject *tp = type; + ctypes_state *st = get_module_state_by_class(tp); self = (CFieldObject *)tp->tp_alloc(tp, 0); - if (self == NULL) - return NULL; - dict = PyType_stgdict(desc); - if (!dict) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - Py_DECREF(self); + if (!self) { return NULL; } - if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ -#ifdef MS_WIN32 - /* MSVC, GCC with -mms-bitfields */ - && dict->size * 8 == *pfield_size -#else - /* GCC */ - && dict->size * 8 <= *pfield_size -#endif - && (*pbitofs + bitsize) <= *pfield_size) { - /* continue bit field */ - fieldtype = CONT_BITFIELD; -#ifndef MS_WIN32 - } else if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ - && dict->size * 8 >= *pfield_size - && (*pbitofs + bitsize) <= dict->size * 8) { - /* expand bit field */ - fieldtype = EXPAND_BITFIELD; -#endif - } else if (bitsize) { - /* start new bitfield */ - fieldtype = NEW_BITFIELD; - *pbitofs = 0; - *pfield_size = dict->size * 8; + if (PyUnicode_CheckExact(name)) { + self->name = Py_NewRef(name); } else { - /* not a bit field */ - fieldtype = NO_BITFIELD; - *pbitofs = 0; - *pfield_size = 0; + self->name = PyObject_Str(name); + if (!self->name) { + goto error; + } } - size = dict->size; - proto = desc; + StgInfo *info; + if (PyStgInfo_FromType(st, proto, &info) < 0) { + goto error; + } + if (info == NULL) { + PyErr_Format(PyExc_TypeError, + "type of field %R must be a C type", self->name); + goto error; + } - /* Field descriptors for 'c_char * n' are be scpecial cased to + Py_ssize_t bit_size = NUM_BITS(size); + if (bit_size) { + assert(bit_size > 0); + assert(bit_size <= info->size * 8); + switch(info->ffi_type_pointer.type) { + case FFI_TYPE_UINT8: + case FFI_TYPE_UINT16: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + break; + + case FFI_TYPE_SINT8: + case FFI_TYPE_SINT16: + case FFI_TYPE_SINT32: + if (info->getfunc != _ctypes_get_fielddesc("c")->getfunc + && info->getfunc != _ctypes_get_fielddesc("u")->getfunc) + { + break; + } + _Py_FALLTHROUGH; /* else fall through */ + default: + PyErr_Format(PyExc_TypeError, + "bit fields not allowed for type %s", + ((PyTypeObject*)proto)->tp_name); + goto error; + } + } + + self->proto = Py_NewRef(proto); + self->size = size; + self->offset = offset; + + self->index = index; + + /* Field descriptors for 'c_char * n' are be special cased to return a Python string instead of an Array object instance... */ + self->setfunc = NULL; + self->getfunc = NULL; if (PyCArrayTypeObject_Check(st, proto)) { - StgDictObject *adict = PyType_stgdict(proto); - StgDictObject *idict; - if (adict && adict->proto) { - idict = PyType_stgdict(adict->proto); - if (!idict) { + StgInfo *ainfo; + if (PyStgInfo_FromType(st, proto, &ainfo) < 0) { + goto error; + } + + if (ainfo && ainfo->proto) { + StgInfo *iinfo; + if (PyStgInfo_FromType(st, ainfo->proto, &iinfo) < 0) { + goto error; + } + if (!iinfo) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); - Py_DECREF(self); - return NULL; + goto error; } - if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (iinfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("s"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; + self->getfunc = fd->getfunc; + self->setfunc = fd->setfunc; } - if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iinfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("U"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; + self->getfunc = fd->getfunc; + self->setfunc = fd->setfunc; } } } - self->setfunc = setfunc; - self->getfunc = getfunc; - self->index = index; - - self->proto = Py_NewRef(proto); - - switch (fieldtype) { - case NEW_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - *pbitofs = bitsize; - /* fall through */ - case NO_BITFIELD: - if (pack) - align = min(pack, dict->align); - else - align = dict->align; - if (align && *poffset % align) { - Py_ssize_t delta = align - (*poffset % align); - *psize += delta; - *poffset += delta; - } - - if (bitsize == 0) - self->size = size; - *psize += size; - - self->offset = *poffset; - *poffset += size; - - *palign = align; - break; - - case EXPAND_BITFIELD: - *poffset += dict->size - *pfield_size/8; - *psize += dict->size - *pfield_size/8; - - *pfield_size = dict->size * 8; - - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; - - case CONT_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; - } - return (PyObject *)self; +error: + Py_XDECREF(self); + return NULL; } + static int PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) { CDataObject *dst; char *ptr; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(self)); if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); @@ -217,7 +204,7 @@ PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) "can't delete attribute"); return -1; } - return PyCData_set(inst, self->proto, self->setfunc, value, + return PyCData_set(st, inst, self->proto, self->setfunc, value, self->index, self->size, ptr); } @@ -228,14 +215,14 @@ PyCField_get(CFieldObject *self, PyObject *inst, PyTypeObject *type) if (inst == NULL) { return Py_NewRef(self); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(self)); if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); return NULL; } src = (CDataObject *)inst; - return PyCData_get(self->proto, self->getfunc, inst, + return PyCData_get(st, self->proto, self->getfunc, inst, self->index, self->size, src->b_ptr + self->offset); } @@ -277,8 +264,10 @@ PyCField_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - (void)PyCField_clear((CFieldObject *)self); - Py_TYPE(self)->tp_free((PyObject *)self); + CFieldObject *self_cf = (CFieldObject *)self; + (void)PyCField_clear(self_cf); + Py_CLEAR(self_cf->name); + Py_TYPE(self)->tp_free(self); Py_DECREF(tp); } @@ -286,8 +275,8 @@ static PyObject * PyCField_repr(CFieldObject *self) { PyObject *result; - Py_ssize_t bits = self->size >> 16; - Py_ssize_t size = self->size & 0xFFFF; + Py_ssize_t bits = NUM_BITS(self->size); + Py_ssize_t size = LOW_BIT(self->size); const char *name; name = ((PyTypeObject *)self->proto)->tp_name; @@ -304,6 +293,7 @@ PyCField_repr(CFieldObject *self) } static PyType_Slot cfield_slots[] = { + {Py_tp_new, PyCField_new}, {Py_tp_dealloc, PyCField_dealloc}, {Py_tp_repr, PyCField_repr}, {Py_tp_doc, (void *)PyDoc_STR("Structure/Union member")}, @@ -319,7 +309,7 @@ PyType_Spec cfield_spec = { .name = "_ctypes.CField", .basicsize = sizeof(CFieldObject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + Py_TPFLAGS_IMMUTABLETYPE), .slots = cfield_slots, }; @@ -384,8 +374,14 @@ get_ulonglong(PyObject *v, unsigned long long *p) */ /* how to decode the size field, for integer get/set functions */ -#define LOW_BIT(x) ((x) & 0xFFFF) -#define NUM_BITS(x) ((x) >> 16) +static inline +Py_ssize_t LOW_BIT(Py_ssize_t offset) { + return offset & 0xFFFF; +} +static inline +Py_ssize_t NUM_BITS(Py_ssize_t bitsize) { + return bitsize >> 16; +} /* Doesn't work if NUM_BITS(size) == 0, but it never happens in SET() call. */ #define BIT_MASK(type, size) (((((type)1 << (NUM_BITS(size) - 1)) - 1) << 1) + 1) @@ -976,6 +972,74 @@ d_get(void *ptr, Py_ssize_t size) return PyFloat_FromDouble(val); } +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +static PyObject * +C_set(void *ptr, PyObject *value, Py_ssize_t size) +{ + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } + double complex x = CMPLX(c.real, c.imag); + memcpy(ptr, &x, sizeof(x)); + _RET(value); +} + +static PyObject * +C_get(void *ptr, Py_ssize_t size) +{ + double complex x; + + memcpy(&x, ptr, sizeof(x)); + return PyComplex_FromDoubles(creal(x), cimag(x)); +} + +static PyObject * +E_set(void *ptr, PyObject *value, Py_ssize_t size) +{ + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } + float complex x = CMPLXF((float)c.real, (float)c.imag); + memcpy(ptr, &x, sizeof(x)); + _RET(value); +} + +static PyObject * +E_get(void *ptr, Py_ssize_t size) +{ + float complex x; + + memcpy(&x, ptr, sizeof(x)); + return PyComplex_FromDoubles(crealf(x), cimagf(x)); +} + +static PyObject * +F_set(void *ptr, PyObject *value, Py_ssize_t size) +{ + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } + long double complex x = CMPLXL(c.real, c.imag); + memcpy(ptr, &x, sizeof(x)); + _RET(value); +} + +static PyObject * +F_get(void *ptr, Py_ssize_t size) +{ + long double complex x; + + memcpy(&x, ptr, sizeof(x)); + return PyComplex_FromDoubles((double)creall(x), (double)cimagl(x)); +} +#endif + static PyObject * d_set_sw(void *ptr, PyObject *value, Py_ssize_t size) { @@ -1088,25 +1152,45 @@ O_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * c_set(void *ptr, PyObject *value, Py_ssize_t size) { - if (PyBytes_Check(value) && PyBytes_GET_SIZE(value) == 1) { + if (PyBytes_Check(value)) { + if (PyBytes_GET_SIZE(value) != 1) { + PyErr_Format(PyExc_TypeError, + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length %zd", + PyBytes_GET_SIZE(value)); + return NULL; + } *(char *)ptr = PyBytes_AS_STRING(value)[0]; _RET(value); } - if (PyByteArray_Check(value) && PyByteArray_GET_SIZE(value) == 1) { + if (PyByteArray_Check(value)) { + if (PyByteArray_GET_SIZE(value) != 1) { + PyErr_Format(PyExc_TypeError, + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytearray of length %zd", + PyByteArray_GET_SIZE(value)); + return NULL; + } *(char *)ptr = PyByteArray_AS_STRING(value)[0]; _RET(value); } - if (PyLong_Check(value)) - { - long longval = PyLong_AsLong(value); - if (longval < 0 || longval >= 256) - goto error; + if (PyLong_Check(value)) { + int overflow; + long longval = PyLong_AsLongAndOverflow(value, &overflow); + if (longval == -1 && PyErr_Occurred()) { + return NULL; + } + if (overflow || longval < 0 || longval >= 256) { + PyErr_SetString(PyExc_TypeError, "integer not in range(256)"); + return NULL; + } *(char *)ptr = (char)longval; _RET(value); } - error: PyErr_Format(PyExc_TypeError, - "one character bytes, bytearray or integer expected"); + "one character bytes, bytearray, or an integer " + "in range(256) expected, not %T", + value); return NULL; } @@ -1125,22 +1209,27 @@ u_set(void *ptr, PyObject *value, Py_ssize_t size) wchar_t chars[2]; if (!PyUnicode_Check(value)) { PyErr_Format(PyExc_TypeError, - "unicode string expected instead of %s instance", - Py_TYPE(value)->tp_name); + "a unicode character expected, not instance of %T", + value); return NULL; - } else - Py_INCREF(value); + } len = PyUnicode_AsWideChar(value, chars, 2); if (len != 1) { - Py_DECREF(value); - PyErr_SetString(PyExc_TypeError, - "one character unicode string expected"); + if (PyUnicode_GET_LENGTH(value) != 1) { + PyErr_Format(PyExc_TypeError, + "a unicode character expected, not a string of length %zd", + PyUnicode_GET_LENGTH(value)); + } + else { + PyErr_Format(PyExc_TypeError, + "the string %A cannot be converted to a single wchar_t character", + value); + } return NULL; } *(wchar_t *)ptr = chars[0]; - Py_DECREF(value); _RET(value); } @@ -1456,6 +1545,11 @@ static struct fielddesc formattable[] = { { 'B', B_set, B_get, NULL}, { 'c', c_set, c_get, NULL}, { 'd', d_set, d_get, NULL, d_set_sw, d_get_sw}, +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) + { 'C', C_set, C_get, NULL}, + { 'E', E_set, E_get, NULL}, + { 'F', F_set, F_get, NULL}, +#endif { 'g', g_set, g_get, NULL}, { 'f', f_set, f_get, NULL, f_set_sw, f_get_sw}, { 'h', h_set, h_get, NULL, h_set_sw, h_get_sw}, @@ -1506,6 +1600,11 @@ _ctypes_init_fielddesc(void) case 'B': fd->pffi_type = &ffi_type_uchar; break; case 'c': fd->pffi_type = &ffi_type_schar; break; case 'd': fd->pffi_type = &ffi_type_double; break; +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) + case 'C': fd->pffi_type = &ffi_type_complex_double; break; + case 'E': fd->pffi_type = &ffi_type_complex_float; break; + case 'F': fd->pffi_type = &ffi_type_complex_longdouble; break; +#endif case 'g': fd->pffi_type = &ffi_type_longdouble; break; case 'f': fd->pffi_type = &ffi_type_float; break; case 'h': fd->pffi_type = &ffi_type_sshort; break; diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h new file mode 100644 index 00000000000000..e1d5a17cbe7d68 --- /dev/null +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -0,0 +1,613 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_ctypes_CType_Type___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return memory consumption of the type object."); + +#define _CTYPES_CTYPE_TYPE___SIZEOF___METHODDEF \ + {"__sizeof__", _PyCFunction_CAST(_ctypes_CType_Type___sizeof__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _ctypes_CType_Type___sizeof____doc__}, + +static PyObject * +_ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_ctypes_CType_Type___sizeof__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__sizeof__() takes no arguments"); + return NULL; + } + return _ctypes_CType_Type___sizeof___impl(self, cls); +} + +PyDoc_STRVAR(CDataType_from_address__doc__, +"from_address($self, value, /)\n" +"--\n" +"\n" +"C.from_address(integer) -> C instance\n" +"\n" +"Access a C instance at the specified address."); + +#define CDATATYPE_FROM_ADDRESS_METHODDEF \ + {"from_address", _PyCFunction_CAST(CDataType_from_address), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_address__doc__}, + +static PyObject * +CDataType_from_address_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +CDataType_from_address(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_address", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = CDataType_from_address_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_buffer__doc__, +"from_buffer($self, obj, offset=0, /)\n" +"--\n" +"\n" +"C.from_buffer(object, offset=0) -> C instance\n" +"\n" +"Create a C instance from a writeable buffer."); + +#define CDATATYPE_FROM_BUFFER_METHODDEF \ + {"from_buffer", _PyCFunction_CAST(CDataType_from_buffer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_buffer__doc__}, + +static PyObject * +CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, + Py_ssize_t offset); + +static PyObject * +CDataType_from_buffer(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_buffer", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *obj; + Py_ssize_t offset = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + obj = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } +skip_optional_posonly: + return_value = CDataType_from_buffer_impl(type, cls, obj, offset); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_buffer_copy__doc__, +"from_buffer_copy($self, buffer, offset=0, /)\n" +"--\n" +"\n" +"C.from_buffer_copy(object, offset=0) -> C instance\n" +"\n" +"Create a C instance from a readable buffer."); + +#define CDATATYPE_FROM_BUFFER_COPY_METHODDEF \ + {"from_buffer_copy", _PyCFunction_CAST(CDataType_from_buffer_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_buffer_copy__doc__}, + +static PyObject * +CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, + Py_buffer *buffer, Py_ssize_t offset); + +static PyObject * +CDataType_from_buffer_copy(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_buffer_copy", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_buffer buffer = {NULL, NULL}; + Py_ssize_t offset = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &buffer, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional_posonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } +skip_optional_posonly: + return_value = CDataType_from_buffer_copy_impl(type, cls, &buffer, offset); + +exit: + /* Cleanup for buffer */ + if (buffer.obj) { + PyBuffer_Release(&buffer); + } + + return return_value; +} + +PyDoc_STRVAR(CDataType_in_dll__doc__, +"in_dll($self, dll, name, /)\n" +"--\n" +"\n" +"C.in_dll(dll, name) -> C instance\n" +"\n" +"Access a C instance in a dll."); + +#define CDATATYPE_IN_DLL_METHODDEF \ + {"in_dll", _PyCFunction_CAST(CDataType_in_dll), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_in_dll__doc__}, + +static PyObject * +CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, + const char *name); + +static PyObject * +CDataType_in_dll(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "in_dll", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *dll; + const char *name; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + dll = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("in_dll", "argument 2", "str", args[1]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[1], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = CDataType_in_dll_impl(type, cls, dll, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define CDATATYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(CDataType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_param__doc__}, + +static PyObject * +CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +CDataType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = CDataType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCPointerType_set_type__doc__, +"set_type($self, type, /)\n" +"--\n" +"\n"); + +#define PYCPOINTERTYPE_SET_TYPE_METHODDEF \ + {"set_type", _PyCFunction_CAST(PyCPointerType_set_type), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCPointerType_set_type__doc__}, + +static PyObject * +PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, + PyObject *type); + +static PyObject * +PyCPointerType_set_type(PyTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_type", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *type; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + type = args[0]; + return_value = PyCPointerType_set_type_impl(self, cls, type); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCPointerType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define PYCPOINTERTYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(PyCPointerType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCPointerType_from_param__doc__}, + +static PyObject * +PyCPointerType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +PyCPointerType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = PyCPointerType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_wchar_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_WCHAR_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_wchar_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_wchar_p_from_param__doc__}, + +static PyObject * +c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_wchar_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_wchar_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_char_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_CHAR_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_char_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_char_p_from_param__doc__}, + +static PyObject * +c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_char_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_char_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_void_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_VOID_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_void_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_void_p_from_param__doc__}, + +static PyObject * +c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_void_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_void_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCSimpleType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define PYCSIMPLETYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(PyCSimpleType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCSimpleType_from_param__doc__}, + +static PyObject * +PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +PyCSimpleType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = PyCSimpleType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCData_reduce__doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define PYCDATA_REDUCE_METHODDEF \ + {"__reduce__", _PyCFunction_CAST(PyCData_reduce), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCData_reduce__doc__}, + +static PyObject * +PyCData_reduce_impl(PyObject *myself, PyTypeObject *cls); + +static PyObject * +PyCData_reduce(PyObject *myself, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); + return NULL; + } + return PyCData_reduce_impl(myself, cls); +} + +PyDoc_STRVAR(Simple_from_outparm__doc__, +"__ctypes_from_outparam__($self, /)\n" +"--\n" +"\n"); + +#define SIMPLE_FROM_OUTPARM_METHODDEF \ + {"__ctypes_from_outparam__", _PyCFunction_CAST(Simple_from_outparm), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Simple_from_outparm__doc__}, + +static PyObject * +Simple_from_outparm_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__ctypes_from_outparam__() takes no arguments"); + return NULL; + } + return Simple_from_outparm_impl(self, cls); +} +/*[clinic end generated code: output=a90886be2a294ee6 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h new file mode 100644 index 00000000000000..df5da783e050ed --- /dev/null +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -0,0 +1,113 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +static PyObject * +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, + PyObject *bit_size_obj); + +static PyObject * +PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "CField", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; + PyObject *name; + PyObject *proto; + Py_ssize_t size; + Py_ssize_t offset; + Py_ssize_t index; + PyObject *bit_size_obj = Py_None; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 6, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!PyUnicode_Check(fastargs[0])) { + _PyArg_BadArgument("CField", "argument 'name'", "str", fastargs[0]); + goto exit; + } + name = fastargs[0]; + proto = fastargs[1]; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + size = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[3]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[4]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + index = ival; + } + if (!noptargs) { + goto skip_optional_pos; + } + bit_size_obj = fastargs[5]; +skip_optional_pos: + return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj); + +exit: + return return_value; +} +/*[clinic end generated code: output=27c010bae9be7213 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1989723f6f3dbb..7e0804054cded4 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -2,6 +2,13 @@ # include #endif +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() + +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +# include "../_complex.h" // complex +#endif + #ifndef MS_WIN32 #define max(a, b) ((a) > (b) ? (a) : (b)) #define min(a, b) ((a) < (b) ? (a) : (b)) @@ -32,13 +39,17 @@ #endif #endif +#ifdef MS_WIN32 +#include // for IUnknown interface +#endif + typedef struct { PyTypeObject *DictRemover_Type; PyTypeObject *PyCArg_Type; PyTypeObject *PyCField_Type; PyTypeObject *PyCThunk_Type; - PyTypeObject *PyCStgDict_Type; PyTypeObject *StructParam_Type; + PyTypeObject *PyCType_Type; PyTypeObject *PyCStructType_Type; PyTypeObject *UnionType_Type; PyTypeObject *PyCPointerType_Type; @@ -55,12 +66,47 @@ typedef struct { #ifdef MS_WIN32 PyTypeObject *PyComError_Type; #endif + /* This dict maps ctypes types to POINTER types */ + PyObject *_ctypes_ptrtype_cache; + /* a callable object used for unpickling: + strong reference to _ctypes._unpickle() function */ + PyObject *_unpickle; + PyObject *array_cache; + PyObject *error_object_name; // callproc.c + PyObject *PyExc_ArgError; + PyObject *swapped_suffix; } ctypes_state; -extern ctypes_state global_state; -#define GLOBAL_STATE() (&global_state) +extern struct PyModuleDef _ctypesmodule; + + +static inline ctypes_state * +get_module_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (ctypes_state *)state; +} + +static inline ctypes_state * +get_module_state_by_class(PyTypeObject *cls) +{ + ctypes_state *state = (ctypes_state *)_PyType_GetModuleState(cls); + assert(state != NULL); + return state; +} + +static inline ctypes_state * +get_module_state_by_def(PyTypeObject *cls) +{ + PyObject *mod = PyType_GetModuleByDef(cls, &_ctypesmodule); + assert(mod != NULL); + return get_module_state(mod); +} + +extern PyType_Spec pyctype_type_spec; extern PyType_Spec carg_spec; extern PyType_Spec cfield_spec; extern PyType_Spec cthunk_spec; @@ -69,7 +115,7 @@ typedef struct tagPyCArgObject PyCArgObject; typedef struct tagCDataObject CDataObject; typedef PyObject *(* GETFUNC)(void *, Py_ssize_t size); typedef PyObject *(* SETFUNC)(void *, PyObject *value, Py_ssize_t size); -typedef PyCArgObject *(* PARAMFUNC)(CDataObject *obj); +typedef PyCArgObject *(* PARAMFUNC)(ctypes_state *st, CDataObject *obj); /* A default buffer in CDataObject, which can be used for small C types. If this buffer is too small, PyMem_Malloc will be called to create a larger one, @@ -140,7 +186,7 @@ typedef struct { CThunkObject *thunk; PyObject *callable; - /* These two fields will override the ones in the type's stgdict if + /* These two fields will override the ones in the type's stginfo if they are set */ PyObject *converters; PyObject *argtypes; @@ -154,17 +200,12 @@ typedef struct { PyObject *paramflags; } PyCFuncPtrObject; -extern PyTypeObject PyCStgDict_Type; -#define PyCStgDict_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCStgDict_Type) -#define PyCStgDict_Check(st, v) PyObject_TypeCheck((v), (st)->PyCStgDict_Type) - -extern int PyCStructUnionType_update_stgdict(PyObject *fields, PyObject *type, int isStruct); +extern int PyCStructUnionType_update_stginfo(PyObject *fields, PyObject *type, int isStruct); extern int PyType_stginfo(PyTypeObject *self, Py_ssize_t *psize, Py_ssize_t *palign, Py_ssize_t *plength); extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palign, Py_ssize_t *plength); -extern PyTypeObject PyCData_Type; #define CDataObject_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCData_Type) #define CDataObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCData_Type) #define _CDataObject_HasExternalBuffer(v) ((v)->b_ptr != (char *)&(v)->b_value) @@ -174,19 +215,8 @@ extern PyTypeObject PyCData_Type; extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); - -extern PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int is_big_endian); - -extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); -extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length); - -extern PyTypeObject PyCArray_Type; -extern PyTypeObject PyCPointer_Type; -extern PyTypeObject PyCFuncPtr_Type; +extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); +extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); #define PyCArrayTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArrayType_Type) #define ArrayObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArray_Type) @@ -197,11 +227,12 @@ extern PyTypeObject PyCFuncPtr_Type; #define PyCStructTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCStructType_Type) extern PyObject * -PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length); +PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length); extern PyMethodDef _ctypes_module_methods[]; -extern CThunkObject *_ctypes_alloc_callback(PyObject *callable, +extern CThunkObject *_ctypes_alloc_callback(ctypes_state *st, + PyObject *callable, PyObject *converters, PyObject *restype, int flags); @@ -215,57 +246,36 @@ struct fielddesc { GETFUNC getfunc_swapped; }; -typedef struct { +typedef struct CFieldObject { PyObject_HEAD Py_ssize_t offset; Py_ssize_t size; Py_ssize_t index; /* Index into CDataObject's object array */ - PyObject *proto; /* a type or NULL */ + PyObject *proto; /* underlying ctype; must have StgInfo */ GETFUNC getfunc; /* getter function if proto is NULL */ SETFUNC setfunc; /* setter function if proto is NULL */ int anonymous; + + PyObject *name; /* exact PyUnicode */ } CFieldObject; -/* A subclass of PyDictObject, used as the instance dictionary of ctypes - metatypes */ -typedef struct { - PyDictObject dict; /* first part identical to PyDictObject */ -/* The size and align fields are unneeded, they are in ffi_type as well. As - an experiment shows, it's trivial to get rid of them, the only thing to - remember is that in PyCArrayType_new the ffi_type fields must be filled in - - so far it was unneeded because libffi doesn't support arrays at all - (because they are passed as pointers to function calls anyway). But it's - too much risk to change that now, and there are other fields which doesn't - belong into this structure anyway. Maybe in ctypes 2.0... (ctypes 2000?) -*/ - Py_ssize_t size; /* number of bytes */ - Py_ssize_t align; /* alignment requirements */ - Py_ssize_t length; /* number of fields */ - ffi_type ffi_type_pointer; - PyObject *proto; /* Only for Pointer/ArrayObject */ - SETFUNC setfunc; /* Only for simple objects */ - GETFUNC getfunc; /* Only for simple objects */ - PARAMFUNC paramfunc; +/**************************************************************** + StgInfo - /* Following fields only used by PyCFuncPtrType_Type instances */ - PyObject *argtypes; /* tuple of CDataObjects */ - PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ - PyObject *restype; /* CDataObject or NULL */ - PyObject *checker; - int flags; /* calling convention and such */ + Since Python 3.13, ctypes-specific type information is stored in the + corresponding type object, in a `StgInfo` struct accessed by the helpers + below. + Before that, each type's `tp_dict` was set to a dict *subclass* that included + the fields that are now in StgInfo. The mechanism was called "StgDict"; a few + references to that name might remain. - /* pep3118 fields, pointers need PyMem_Free */ - char *format; - int ndim; - Py_ssize_t *shape; -/* Py_ssize_t *strides; */ /* unused in ctypes */ -/* Py_ssize_t *suboffsets; */ /* unused in ctypes */ + Functions for accessing StgInfo are `static inline` for performance; + see later in this file. -} StgDictObject; + **************************************************************** -/**************************************************************** - StgDictObject fields + StgInfo fields setfunc and getfunc is only set for simple data types, it is copied from the corresponding fielddesc entry. These are functions to set and get the value @@ -276,11 +286,11 @@ typedef struct { object. Probably all the magic ctypes methods (like from_param) should have C - callable wrappers in the StgDictObject. For simple data type, for example, + callable wrappers in the StgInfo. For simple data type, for example, the fielddesc table could have entries for C codec from_param functions or other methods as well, if a subtype overrides this method in Python at construction time, or assigns to it later, tp_setattro should update the - StgDictObject function to a generic one. + StgInfo function to a generic one. Currently, PyCFuncPtr types have 'converters' and 'checker' entries in their type dict. They are only used to cache attributes from other entries, which @@ -304,17 +314,40 @@ typedef struct { *****************************************************************/ -/* May return NULL, but does not set an exception! */ -extern StgDictObject *PyType_stgdict(PyObject *obj); +typedef struct { + int initialized; + Py_ssize_t size; /* number of bytes */ + Py_ssize_t align; /* alignment requirements */ + Py_ssize_t length; /* number of fields */ + ffi_type ffi_type_pointer; + PyObject *proto; /* Only for Pointer/ArrayObject */ + SETFUNC setfunc; /* Only for simple objects */ + GETFUNC getfunc; /* Only for simple objects */ + PARAMFUNC paramfunc; + + /* Following fields only used by PyCFuncPtrType_Type instances */ + PyObject *argtypes; /* tuple of CDataObjects */ + PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ + PyObject *restype; /* CDataObject or NULL */ + PyObject *checker; + PyObject *module; + int flags; /* calling convention and such */ -/* May return NULL, but does not set an exception! */ -extern StgDictObject *PyObject_stgdict(PyObject *self); + /* pep3118 fields, pointers need PyMem_Free */ + char *format; + int ndim; + Py_ssize_t *shape; +/* Py_ssize_t *strides; */ /* unused in ctypes */ +/* Py_ssize_t *suboffsets; */ /* unused in ctypes */ +} StgInfo; -extern int PyCStgDict_clone(StgDictObject *src, StgDictObject *dst); +extern int PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info); +extern void ctype_clear_stginfo(StgInfo *info); typedef int(* PPROC)(void); -PyObject *_ctypes_callproc(PPROC pProc, +PyObject *_ctypes_callproc(ctypes_state *st, + PPROC pProc, PyObject *arguments, #ifdef MS_WIN32 IUnknown *pIUnk, @@ -335,8 +368,6 @@ PyObject *_ctypes_callproc(PPROC pProc, #define TYPEFLAG_ISPOINTER 0x100 #define TYPEFLAG_HASPOINTER 0x200 -#define TYPEFLAG_HASUNION 0x400 -#define TYPEFLAG_HASBITFIELD 0x800 #define DICTFLAG_FINAL 0x1000 @@ -355,20 +386,26 @@ struct tagPyCArgObject { double d; float f; void *p; +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) + double complex C; + float complex E; + long double complex F; +#endif } value; PyObject *obj; Py_ssize_t size; /* for the 'V' tag */ }; #define PyCArg_CheckExact(st, v) Py_IS_TYPE(v, st->PyCArg_Type) -extern PyCArgObject *PyCArgObject_new(void); +extern PyCArgObject *PyCArgObject_new(ctypes_state *st); extern PyObject * -PyCData_get(PyObject *type, GETFUNC getfunc, PyObject *src, +PyCData_get(ctypes_state *st, PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *ptr); extern int -PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +PyCData_set(ctypes_state *st, + PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr); extern void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...); @@ -379,30 +416,17 @@ struct basespec { char *adr; }; -extern char basespec_string[]; - -extern ffi_type *_ctypes_get_ffi_type(PyObject *obj); - -/* exception classes */ -extern PyObject *PyExc_ArgError; - -extern char *_ctypes_conversion_encoding; -extern char *_ctypes_conversion_errors; - +extern ffi_type *_ctypes_get_ffi_type(ctypes_state *st, PyObject *obj); extern void _ctypes_free_closure(void *); extern void *_ctypes_alloc_closure(void); -extern PyObject *PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr); -extern char *_ctypes_alloc_format_string(const char *prefix, const char *suffix); -extern char *_ctypes_alloc_format_string_with_shape(int ndim, - const Py_ssize_t *shape, - const char *prefix, const char *suffix); +extern PyObject *PyCData_FromBaseObj(ctypes_state *st, PyObject *type, + PyObject *base, Py_ssize_t index, char *adr); -extern int _ctypes_simple_instance(PyObject *obj); +extern int _ctypes_simple_instance(ctypes_state *st, PyObject *obj); -extern PyObject *_ctypes_ptrtype_cache; -PyObject *_ctypes_get_errobj(int **pspace); +PyObject *_ctypes_get_errobj(ctypes_state *st, int **pspace); #ifdef USING_MALLOC_CLOSURE_DOT_C void Py_ffi_closure_free(void *p); @@ -412,8 +436,101 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc); #define Py_ffi_closure_alloc ffi_closure_alloc #endif -/* - Local Variables: - compile-command: "python setup.py -q build install --home ~" - End: -*/ + +/**************************************************************** + * Accessing StgInfo -- these are inlined for performance reasons. + */ + +// `PyStgInfo_From**` functions get a PyCTypeDataObject. +// These return -1 on error, 0 if "not found", 1 on OK. +// (Currently, these do not return -1 in practice. This might change +// in the future.) + +// +// Common helper: +static inline int +_stginfo_from_type(ctypes_state *state, PyTypeObject *type, StgInfo **result) +{ + *result = NULL; + if (!PyObject_IsInstance((PyObject *)type, (PyObject *)state->PyCType_Type)) { + // not a ctypes class. + return 0; + } + StgInfo *info = PyObject_GetTypeData((PyObject *)type, state->PyCType_Type); + assert(info != NULL); + if (!info->initialized) { + // StgInfo is not initialized. This happens in abstract classes. + return 0; + } + *result = info; + return 1; +} +// from a type: +static inline int +PyStgInfo_FromType(ctypes_state *state, PyObject *type, StgInfo **result) +{ + return _stginfo_from_type(state, (PyTypeObject *)type, result); +} +// from an instance: +static inline int +PyStgInfo_FromObject(ctypes_state *state, PyObject *obj, StgInfo **result) +{ + return _stginfo_from_type(state, Py_TYPE(obj), result); +} +// from either a type or an instance: +static inline int +PyStgInfo_FromAny(ctypes_state *state, PyObject *obj, StgInfo **result) +{ + if (PyType_Check(obj)) { + return _stginfo_from_type(state, (PyTypeObject *)obj, result); + } + return _stginfo_from_type(state, Py_TYPE(obj), result); +} + +/* A variant of PyStgInfo_FromType that doesn't need the state, + * so it can be called from finalization functions when the module + * state is torn down. + */ +static inline StgInfo * +_PyStgInfo_FromType_NoState(PyObject *type) +{ + PyTypeObject *PyCType_Type; + if (PyType_GetBaseByToken(Py_TYPE(type), &pyctype_type_spec, &PyCType_Type) < 0) { + return NULL; + } + if (PyCType_Type == NULL) { + PyErr_Format(PyExc_TypeError, "expected a ctypes type, got '%N'", type); + return NULL; + } + + StgInfo *info = PyObject_GetTypeData(type, PyCType_Type); + Py_DECREF(PyCType_Type); + return info; +} + +// Initialize StgInfo on a newly created type +static inline StgInfo * +PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) +{ + if (!PyObject_IsInstance((PyObject *)type, (PyObject *)state->PyCType_Type)) { + PyErr_Format(PyExc_SystemError, + "'%s' is not a ctypes class.", + type->tp_name); + return NULL; + } + StgInfo *info = PyObject_GetTypeData((PyObject *)type, state->PyCType_Type); + if (info->initialized) { + PyErr_Format(PyExc_SystemError, + "StgInfo of '%s' is already initialized.", + type->tp_name); + return NULL; + } + PyObject *module = PyType_GetModule(state->PyCType_Type); + if (!module) { + return NULL; + } + info->module = Py_NewRef(module); + + info->initialized = 1; + return info; +} diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index deafa696fdd0d0..5dbbe0b3285d58 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -16,201 +16,64 @@ #endif #include "ctypes.h" -/******************************************************************/ -/* - StdDict - a dictionary subclass, containing additional C accessible fields - - XXX blabla more -*/ - -/* Seems we need this, otherwise we get problems when calling - * PyDict_SetItem() (ma_lookup is NULL) +/* This file relates to StgInfo -- type-specific information for ctypes. + * See ctypes.h for details. */ -static int -PyCStgDict_init(StgDictObject *self, PyObject *args, PyObject *kwds) -{ - if (PyDict_Type.tp_init((PyObject *)self, args, kwds) < 0) - return -1; - self->format = NULL; - self->ndim = 0; - self->shape = NULL; - return 0; -} - -static int -PyCStgDict_clear(StgDictObject *self) -{ - Py_CLEAR(self->proto); - Py_CLEAR(self->argtypes); - Py_CLEAR(self->converters); - Py_CLEAR(self->restype); - Py_CLEAR(self->checker); - return 0; -} - -static void -PyCStgDict_dealloc(StgDictObject *self) -{ - PyCStgDict_clear(self); - PyMem_Free(self->format); - PyMem_Free(self->shape); - PyMem_Free(self->ffi_type_pointer.elements); - PyDict_Type.tp_dealloc((PyObject *)self); -} - -static PyObject * -PyCStgDict_sizeof(StgDictObject *self, void *unused) -{ - Py_ssize_t res; - - res = _PyDict_SizeOf((PyDictObject *)self); - res += sizeof(StgDictObject) - sizeof(PyDictObject); - if (self->format) - res += strlen(self->format) + 1; - res += self->ndim * sizeof(Py_ssize_t); - if (self->ffi_type_pointer.elements) - res += (self->length + 1) * sizeof(ffi_type *); - return PyLong_FromSsize_t(res); -} int -PyCStgDict_clone(StgDictObject *dst, StgDictObject *src) +PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) { - char *d, *s; Py_ssize_t size; - PyCStgDict_clear(dst); - PyMem_Free(dst->ffi_type_pointer.elements); - PyMem_Free(dst->format); - dst->format = NULL; - PyMem_Free(dst->shape); - dst->shape = NULL; - dst->ffi_type_pointer.elements = NULL; - - d = (char *)dst; - s = (char *)src; - memcpy(d + sizeof(PyDictObject), - s + sizeof(PyDictObject), - sizeof(StgDictObject) - sizeof(PyDictObject)); - - Py_XINCREF(dst->proto); - Py_XINCREF(dst->argtypes); - Py_XINCREF(dst->converters); - Py_XINCREF(dst->restype); - Py_XINCREF(dst->checker); - - if (src->format) { - dst->format = PyMem_Malloc(strlen(src->format) + 1); - if (dst->format == NULL) { + ctype_clear_stginfo(dst_info); + PyMem_Free(dst_info->ffi_type_pointer.elements); + PyMem_Free(dst_info->format); + dst_info->format = NULL; + PyMem_Free(dst_info->shape); + dst_info->shape = NULL; + dst_info->ffi_type_pointer.elements = NULL; + + memcpy(dst_info, src_info, sizeof(StgInfo)); + + Py_XINCREF(dst_info->proto); + Py_XINCREF(dst_info->argtypes); + Py_XINCREF(dst_info->converters); + Py_XINCREF(dst_info->restype); + Py_XINCREF(dst_info->checker); + Py_XINCREF(dst_info->module); + + if (src_info->format) { + dst_info->format = PyMem_Malloc(strlen(src_info->format) + 1); + if (dst_info->format == NULL) { PyErr_NoMemory(); return -1; } - strcpy(dst->format, src->format); + strcpy(dst_info->format, src_info->format); } - if (src->shape) { - dst->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src->ndim); - if (dst->shape == NULL) { + if (src_info->shape) { + dst_info->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src_info->ndim); + if (dst_info->shape == NULL) { PyErr_NoMemory(); return -1; } - memcpy(dst->shape, src->shape, - sizeof(Py_ssize_t) * src->ndim); + memcpy(dst_info->shape, src_info->shape, + sizeof(Py_ssize_t) * src_info->ndim); } - if (src->ffi_type_pointer.elements == NULL) + if (src_info->ffi_type_pointer.elements == NULL) return 0; - size = sizeof(ffi_type *) * (src->length + 1); - dst->ffi_type_pointer.elements = PyMem_Malloc(size); - if (dst->ffi_type_pointer.elements == NULL) { + size = sizeof(ffi_type *) * (src_info->length + 1); + dst_info->ffi_type_pointer.elements = PyMem_Malloc(size); + if (dst_info->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); return -1; } - memcpy(dst->ffi_type_pointer.elements, - src->ffi_type_pointer.elements, + memcpy(dst_info->ffi_type_pointer.elements, + src_info->ffi_type_pointer.elements, size); return 0; } -static struct PyMethodDef PyCStgDict_methods[] = { - {"__sizeof__", (PyCFunction)PyCStgDict_sizeof, METH_NOARGS}, - {NULL, NULL} /* sentinel */ -}; - -PyTypeObject PyCStgDict_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "StgDict", - sizeof(StgDictObject), - 0, - (destructor)PyCStgDict_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCStgDict_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)PyCStgDict_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ -}; - -/* May return NULL, but does not set an exception! */ -StgDictObject * -PyType_stgdict(PyObject *obj) -{ - PyTypeObject *type; - - if (!PyType_Check(obj)) { - return NULL; - } - ctypes_state *st = GLOBAL_STATE(); - type = (PyTypeObject *)obj; - if (!type->tp_dict || !PyCStgDict_CheckExact(st, type->tp_dict)) { - return NULL; - } - return (StgDictObject *)type->tp_dict; -} - -/* May return NULL, but does not set an exception! */ -/* - This function should be as fast as possible, so we don't call PyType_stgdict - above but inline the code, and avoid the PyType_Check(). -*/ -StgDictObject * -PyObject_stgdict(PyObject *self) -{ - PyTypeObject *type = Py_TYPE(self); - ctypes_state *st = GLOBAL_STATE(); - if (!type->tp_dict || !PyCStgDict_CheckExact(st, type->tp_dict)) { - return NULL; - } - return (StgDictObject *)type->tp_dict; -} - /* descr is the descriptor for a field marked as anonymous. Get all the _fields_ descriptors from descr->proto, create new descriptors with offset and index adjusted, and stuff them into type. @@ -231,7 +94,7 @@ MakeFields(PyObject *type, CFieldObject *descr, if (fieldlist == NULL) return -1; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(descr)); PyTypeObject *cfield_tp = st->PyCField_Type; for (i = 0; i < PySequence_Fast_GET_SIZE(fieldlist); ++i) { PyObject *pair = PySequence_Fast_GET_ITEM(fieldlist, i); /* borrowed */ @@ -312,7 +175,7 @@ MakeAnonFields(PyObject *type) if (anon_names == NULL) return -1; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); PyTypeObject *cfield_tp = st->PyCField_Type; for (i = 0; i < PySequence_Fast_GET_SIZE(anon_names); ++i) { PyObject *fname = PySequence_Fast_GET_ITEM(anon_names, i); /* borrowed */ @@ -347,367 +210,230 @@ MakeAnonFields(PyObject *type) return 0; } -/* - Allocate a memory block for a pep3118 format string, copy prefix (if - non-null) into it and append `{padding}x` to the end. - Returns NULL on failure, with the error indicator set. -*/ -char * -_ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) -{ - /* int64 decimal characters + x + null */ - char buf[19 + 1 + 1]; - - assert(padding > 0); - - if (padding == 1) { - /* Use x instead of 1x, for brevity */ - return _ctypes_alloc_format_string(prefix, "x"); - } - - int ret = PyOS_snprintf(buf, sizeof(buf), "%zdx", padding); (void)ret; - assert(0 <= ret && ret < (Py_ssize_t)sizeof(buf)); - return _ctypes_alloc_format_string(prefix, buf); -} - /* Retrieve the (optional) _pack_ attribute from a type, the _fields_ attribute, - and create an StgDictObject. Used for Structure and Union subclasses. + and initialize StgInfo. Used for Structure and Union subclasses. */ int -PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct) +PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - StgDictObject *stgdict, *basedict; - Py_ssize_t len, offset, size, align, i; - Py_ssize_t union_size, total_align, aligned_size; - Py_ssize_t field_size = 0; - int bitofs; PyObject *tmp; - int pack; Py_ssize_t ffi_ofs; - int big_endian; int arrays_seen = 0; - if (fields == NULL) + int retval = -1; + // The following are NULL or hold strong references. + // They're cleared on error. + PyObject *layout_fields = NULL; + PyObject *layout = NULL; + PyObject *format_spec_obj = NULL; + + if (fields == NULL) { return 0; + } - int rc = PyObject_HasAttrWithError(type, &_Py_ID(_swappedbytes_)); - if (rc < 0) { - return -1; + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *stginfo; + if (PyStgInfo_FromType(st, type, &stginfo) < 0) { + goto error; } - if (rc) { - big_endian = !PY_BIG_ENDIAN; + if (!stginfo) { + PyErr_SetString(PyExc_TypeError, + "ctypes state is not initialized"); + goto error; } - else { - big_endian = PY_BIG_ENDIAN; + PyObject *base = (PyObject *)((PyTypeObject *)type)->tp_base; + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, base, &baseinfo) < 0) { + goto error; } - if (PyObject_GetOptionalAttr(type, &_Py_ID(_pack_), &tmp) < 0) { - return -1; + /* If this structure/union is already marked final we cannot assign + _fields_ anymore. */ + + if (stginfo->flags & DICTFLAG_FINAL) {/* is final ? */ + PyErr_SetString(PyExc_AttributeError, + "_fields_ is final"); + goto error; } - if (tmp) { - pack = PyLong_AsInt(tmp); - Py_DECREF(tmp); - if (pack < 0) { - if (!PyErr_Occurred() || - PyErr_ExceptionMatches(PyExc_TypeError) || - PyErr_ExceptionMatches(PyExc_OverflowError)) - { - PyErr_SetString(PyExc_ValueError, - "_pack_ must be a non-negative integer"); - } - return -1; - } + + PyObject *layout_func = _PyImport_GetModuleAttrString("ctypes._layout", + "get_layout"); + if (!layout_func) { + goto error; } - else { - /* Setting `_pack_ = 0` amounts to using the default alignment */ - pack = 0; + PyObject *kwnames = PyTuple_Pack( + 2, + &_Py_ID(is_struct), + &_Py_ID(base)); + if (!kwnames) { + goto error; + } + layout = PyObject_Vectorcall( + layout_func, + 1 + (PyObject*[]){ + NULL, + /* positional args */ + type, + fields, + /* keyword args */ + isStruct ? Py_True : Py_False, + baseinfo ? base : Py_None}, + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + Py_DECREF(kwnames); + Py_DECREF(layout_func); + fields = NULL; // a borrowed reference we won't be using again + if (!layout) { + goto error; } - len = PySequence_Size(fields); - if (len == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of pairs"); + tmp = PyObject_GetAttr(layout, &_Py_ID(align)); + if (!tmp) { + goto error; + } + Py_ssize_t total_align = PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (total_align < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "align must be a non-negative integer"); } - return -1; + goto error; } - stgdict = PyType_stgdict(type); - if (!stgdict) { - PyErr_SetString(PyExc_TypeError, - "ctypes state is not initialized"); - return -1; + tmp = PyObject_GetAttr(layout, &_Py_ID(size)); + if (!tmp) { + goto error; } - /* If this structure/union is already marked final we cannot assign - _fields_ anymore. */ - - if (stgdict->flags & DICTFLAG_FINAL) {/* is final ? */ - PyErr_SetString(PyExc_AttributeError, - "_fields_ is final"); - return -1; + Py_ssize_t total_size = PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (total_size < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "size must be a non-negative integer"); + } + goto error; } - if (stgdict->format) { - PyMem_Free(stgdict->format); - stgdict->format = NULL; + format_spec_obj = PyObject_GetAttr(layout, &_Py_ID(format_spec)); + if (!format_spec_obj) { + goto error; + } + Py_ssize_t format_spec_size; + const char *format_spec = PyUnicode_AsUTF8AndSize(format_spec_obj, + &format_spec_size); + if (!format_spec) { + goto error; } - if (stgdict->ffi_type_pointer.elements) - PyMem_Free(stgdict->ffi_type_pointer.elements); + if (stginfo->format) { + PyMem_Free(stginfo->format); + stginfo->format = NULL; + } + stginfo->format = PyMem_Malloc(format_spec_size + 1); + if (!stginfo->format) { + PyErr_NoMemory(); + goto error; + } + memcpy(stginfo->format, format_spec, format_spec_size + 1); - basedict = PyType_stgdict((PyObject *)((PyTypeObject *)type)->tp_base); - if (basedict) { - stgdict->flags |= (basedict->flags & - (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD)); + PyObject *layout_fields_obj = PyObject_GetAttr(layout, &_Py_ID(fields)); + if (!layout_fields_obj) { + goto error; + } + layout_fields = PySequence_Tuple(layout_fields_obj); + Py_DECREF(layout_fields_obj); + if (!layout_fields) { + goto error; } - if (!isStruct) { - stgdict->flags |= TYPEFLAG_HASUNION; + Py_CLEAR(layout); + + Py_ssize_t len = PyTuple_GET_SIZE(layout_fields); + + if (stginfo->ffi_type_pointer.elements) { + PyMem_Free(stginfo->ffi_type_pointer.elements); + stginfo->ffi_type_pointer.elements = NULL; } - if (basedict) { - size = offset = basedict->size; - align = basedict->align; - union_size = 0; - total_align = align ? align : 1; - stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; - stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, basedict->length + len + 1); - if (stgdict->ffi_type_pointer.elements == NULL) { + + if (baseinfo) { + stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; + stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); + if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); - return -1; + goto error; } - memset(stgdict->ffi_type_pointer.elements, 0, - sizeof(ffi_type *) * (basedict->length + len + 1)); - if (basedict->length > 0) { - memcpy(stgdict->ffi_type_pointer.elements, - basedict->ffi_type_pointer.elements, - sizeof(ffi_type *) * (basedict->length)); + memset(stginfo->ffi_type_pointer.elements, 0, + sizeof(ffi_type *) * (baseinfo->length + len + 1)); + if (baseinfo->length > 0) { + memcpy(stginfo->ffi_type_pointer.elements, + baseinfo->ffi_type_pointer.elements, + sizeof(ffi_type *) * (baseinfo->length)); } - ffi_ofs = basedict->length; + ffi_ofs = baseinfo->length; } else { - offset = 0; - size = 0; - align = 0; - union_size = 0; - total_align = 1; - stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; - stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); - if (stgdict->ffi_type_pointer.elements == NULL) { + stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; + stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); + if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); - return -1; + goto error; } - memset(stgdict->ffi_type_pointer.elements, 0, + memset(stginfo->ffi_type_pointer.elements, 0, sizeof(ffi_type *) * (len + 1)); ffi_ofs = 0; } - assert(stgdict->format == NULL); - if (isStruct) { - stgdict->format = _ctypes_alloc_format_string(NULL, "T{"); - } else { - /* PEP3118 doesn't support union. Use 'B' for bytes. */ - stgdict->format = _ctypes_alloc_format_string(NULL, "B"); - } - if (stgdict->format == NULL) - return -1; - - ctypes_state *st = GLOBAL_STATE(); - for (i = 0; i < len; ++i) { - PyObject *name = NULL, *desc = NULL; - PyObject *pair = PySequence_GetItem(fields, i); - PyObject *prop; - StgDictObject *dict; - int bitsize = 0; - - if (!pair || !PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_XDECREF(pair); - return -1; - } - if (PyCArrayTypeObject_Check(st, desc)) { - arrays_seen = 1; - } - dict = PyType_stgdict(desc); - if (dict == NULL) { - Py_DECREF(pair); + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); + assert(prop_obj); + if (!PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)) { PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - return -1; - } - stgdict->ffi_type_pointer.elements[ffi_ofs + i] = &dict->ffi_type_pointer; - if (dict->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) - stgdict->flags |= TYPEFLAG_HASPOINTER; - stgdict->flags |= dict->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); - dict->flags |= DICTFLAG_FINAL; /* mark field type final */ - if (PyTuple_Size(pair) == 3) { /* bits specified */ - stgdict->flags |= TYPEFLAG_HASBITFIELD; - switch(dict->ffi_type_pointer.type) { - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT64: - case FFI_TYPE_UINT64: - break; - - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: - case FFI_TYPE_SINT32: - if (dict->getfunc != _ctypes_get_fielddesc("c")->getfunc - && dict->getfunc != _ctypes_get_fielddesc("u")->getfunc - ) - break; - /* else fall through */ - default: - PyErr_Format(PyExc_TypeError, - "bit fields not allowed for type %s", - ((PyTypeObject *)desc)->tp_name); - Py_DECREF(pair); - return -1; - } - if (bitsize <= 0 || bitsize > dict->size * 8) { - PyErr_SetString(PyExc_ValueError, - "number of bits invalid for bit field"); - Py_DECREF(pair); - return -1; - } - } else - bitsize = 0; - - if (isStruct) { - const char *fieldfmt = dict->format ? dict->format : "B"; - const char *fieldname = PyUnicode_AsUTF8(name); - char *ptr; - Py_ssize_t len; - char *buf; - Py_ssize_t last_size = size; - Py_ssize_t padding; - - if (fieldname == NULL) - { - Py_DECREF(pair); - return -1; - } - - /* construct the field now, as `prop->offset` is `offset` with - corrected alignment */ - prop = PyCField_FromDesc(desc, i, - &field_size, bitsize, &bitofs, - &size, &offset, &align, - pack, big_endian); - if (prop == NULL) { - Py_DECREF(pair); - return -1; - } + "fields must be of type CField, got %T", prop_obj); + goto error; - /* number of bytes between the end of the last field and the start - of this one */ - padding = ((CFieldObject *)prop)->offset - last_size; - - if (padding > 0) { - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_padding(ptr, padding); - PyMem_Free(ptr); - if (stgdict->format == NULL) { - Py_DECREF(pair); - Py_DECREF(prop); - return -1; - } - } - - len = strlen(fieldname) + strlen(fieldfmt); - - buf = PyMem_Malloc(len + 2 + 1); - if (buf == NULL) { - Py_DECREF(pair); - Py_DECREF(prop); - PyErr_NoMemory(); - return -1; - } - sprintf(buf, "%s:%s:", fieldfmt, fieldname); - - ptr = stgdict->format; - if (dict->shape != NULL) { - stgdict->format = _ctypes_alloc_format_string_with_shape( - dict->ndim, dict->shape, stgdict->format, buf); - } else { - stgdict->format = _ctypes_alloc_format_string(stgdict->format, buf); - } - PyMem_Free(ptr); - PyMem_Free(buf); - - if (stgdict->format == NULL) { - Py_DECREF(pair); - Py_DECREF(prop); - return -1; - } - } else /* union */ { - size = 0; - offset = 0; - align = 0; - prop = PyCField_FromDesc(desc, i, - &field_size, bitsize, &bitofs, - &size, &offset, &align, - pack, big_endian); - if (prop == NULL) { - Py_DECREF(pair); - return -1; - } - union_size = max(size, union_size); } - total_align = max(align, total_align); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrow from prop_obj - if (-1 == PyObject_SetAttr(type, name, prop)) { - Py_DECREF(prop); - Py_DECREF(pair); - return -1; + if (prop->index != i) { + PyErr_Format(PyExc_ValueError, + "field %R index mismatch (expected %zd, got %zd)", + prop->name, i, prop->index); + goto error; } - Py_DECREF(pair); - Py_DECREF(prop); - } - if (!isStruct) { - size = union_size; - } + if (PyCArrayTypeObject_Check(st, prop->proto)) { + arrays_seen = 1; + } - /* Adjust the size according to the alignment requirements */ - aligned_size = ((size + total_align - 1) / total_align) * total_align; + StgInfo *info; + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { + goto error; + } + assert(info); - if (isStruct) { - char *ptr; - Py_ssize_t padding; + stginfo->ffi_type_pointer.elements[ffi_ofs + i] = &info->ffi_type_pointer; + if (info->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + stginfo->flags |= TYPEFLAG_HASPOINTER; + info->flags |= DICTFLAG_FINAL; /* mark field type final */ - /* Pad up to the full size of the struct */ - padding = aligned_size - size; - if (padding > 0) { - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_padding(ptr, padding); - PyMem_Free(ptr); - if (stgdict->format == NULL) { - return -1; - } + if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { + goto error; } - - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_string(stgdict->format, "}"); - PyMem_Free(ptr); - if (stgdict->format == NULL) - return -1; } - stgdict->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, + stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, Py_ssize_t, unsigned short); - stgdict->ffi_type_pointer.size = aligned_size; + stginfo->ffi_type_pointer.size = total_size; - stgdict->size = aligned_size; - stgdict->align = total_align; - stgdict->length = ffi_ofs + len; + stginfo->size = total_size; + stginfo->align = total_align; + stginfo->length = ffi_ofs + len; /* * The value of MAX_STRUCT_SIZE depends on the platform Python is running on. */ -#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) +#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) || defined(__sparc__) # define MAX_STRUCT_SIZE 32 #elif defined(__powerpc64__) # define MAX_STRUCT_SIZE 64 @@ -715,7 +441,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct # define MAX_STRUCT_SIZE 16 #endif - if (arrays_seen && (size <= MAX_STRUCT_SIZE)) { + if (arrays_seen && (total_size <= MAX_STRUCT_SIZE)) { /* * See bpo-22273 and gh-110190. Arrays are normally treated as * pointers, which is fine when an array name is being passed as @@ -790,45 +516,35 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct Py_ssize_t struct_index = 0; /* index into dummy structs */ /* first pass to see how much memory to allocate */ - for (i = 0; i < len; ++i) { - PyObject *name, *desc; - PyObject *pair = PySequence_GetItem(fields, i); - StgDictObject *dict; - int bitsize = 0; - - if (pair == NULL) { - return -1; - } - if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_DECREF(pair); - return -1; - } - dict = PyType_stgdict(desc); - if (dict == NULL) { - Py_DECREF(pair); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - return -1; + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed + assert(prop_obj); + assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed + + StgInfo *info; + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { + goto error; } - if (!PyCArrayTypeObject_Check(st, desc)) { + assert(info); + + if (!PyCArrayTypeObject_Check(st, prop->proto)) { /* Not an array. Just need an ffi_type pointer. */ num_ffi_type_pointers++; } else { /* It's an array. */ - Py_ssize_t length = dict->length; - StgDictObject *edict; + Py_ssize_t length = info->length; - edict = PyType_stgdict(dict->proto); - if (edict == NULL) { - Py_DECREF(pair); + StgInfo *einfo; + if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { + goto error; + } + if (einfo == NULL) { PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } /* * We need one extra ffi_type to hold the struct, and one @@ -838,7 +554,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct num_ffi_types++; num_ffi_type_pointers += length + 1; } - Py_DECREF(pair); } /* @@ -855,7 +570,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if (type_block == NULL) { PyErr_NoMemory(); - return -1; + goto error; } /* * the first block takes up ffi_ofs + len + 1 which is the pointers * @@ -871,81 +586,61 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if (num_ffi_types > 0) { memset(structs, 0, num_ffi_types * sizeof(ffi_type)); } - if (ffi_ofs && (basedict != NULL)) { + if (ffi_ofs && (baseinfo != NULL)) { memcpy(element_types, - basedict->ffi_type_pointer.elements, + baseinfo->ffi_type_pointer.elements, ffi_ofs * sizeof(ffi_type *)); } element_index = ffi_ofs; /* second pass to actually set the type pointers */ - for (i = 0; i < len; ++i) { - PyObject *name, *desc; - PyObject *pair = PySequence_GetItem(fields, i); - StgDictObject *dict; - int bitsize = 0; - - if (pair == NULL) { - PyMem_Free(type_block); - return -1; - } - /* In theory, we made this call in the first pass, so it *shouldn't* - * fail. However, you never know, and the code above might change - * later - keeping the check in here is a tad defensive but it - * will affect program size only slightly and performance hardly at - * all. - */ - if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_DECREF(pair); - PyMem_Free(type_block); - return -1; - } - dict = PyType_stgdict(desc); - /* Possibly this check could be avoided, but see above comment. */ - if (dict == NULL) { - Py_DECREF(pair); + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed + assert(prop_obj); + assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed + + StgInfo *info; + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { PyMem_Free(type_block); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - return -1; + goto error; } + assert(info); + assert(element_index < (ffi_ofs + len)); /* will be used below */ - if (!PyCArrayTypeObject_Check(st, desc)) { + if (!PyCArrayTypeObject_Check(st, prop->proto)) { /* Not an array. Just copy over the element ffi_type. */ - element_types[element_index++] = &dict->ffi_type_pointer; + element_types[element_index++] = &info->ffi_type_pointer; } else { - Py_ssize_t length = dict->length; - StgDictObject *edict; - - edict = PyType_stgdict(dict->proto); - if (edict == NULL) { - Py_DECREF(pair); + Py_ssize_t length = info->length; + StgInfo *einfo; + if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { + PyMem_Free(type_block); + goto error; + } + if (einfo == NULL) { PyMem_Free(type_block); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } element_types[element_index++] = &structs[struct_index]; - structs[struct_index].size = length * edict->ffi_type_pointer.size; - structs[struct_index].alignment = edict->ffi_type_pointer.alignment; + structs[struct_index].size = length * einfo->ffi_type_pointer.size; + structs[struct_index].alignment = einfo->ffi_type_pointer.alignment; structs[struct_index].type = FFI_TYPE_STRUCT; structs[struct_index].elements = &dummy_types[dummy_index]; ++struct_index; /* Copy over the element's type, length times. */ while (length > 0) { assert(dummy_index < (num_ffi_type_pointers)); - dummy_types[dummy_index++] = &edict->ffi_type_pointer; + dummy_types[dummy_index++] = &einfo->ffi_type_pointer; length--; } assert(dummy_index < (num_ffi_type_pointers)); dummy_types[dummy_index++] = NULL; } - Py_DECREF(pair); } element_types[element_index] = NULL; @@ -953,19 +648,24 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct * Replace the old elements with the new, taking into account * base class elements where necessary. */ - assert(stgdict->ffi_type_pointer.elements); - PyMem_Free(stgdict->ffi_type_pointer.elements); - stgdict->ffi_type_pointer.elements = element_types; + assert(stginfo->ffi_type_pointer.elements); + PyMem_Free(stginfo->ffi_type_pointer.elements); + stginfo->ffi_type_pointer.elements = element_types; } /* We did check that this flag was NOT set above, it must not have been set until now. */ - if (stgdict->flags & DICTFLAG_FINAL) { + if (stginfo->flags & DICTFLAG_FINAL) { PyErr_SetString(PyExc_AttributeError, "Structure or union cannot contain itself"); - return -1; + goto error; } - stgdict->flags |= DICTFLAG_FINAL; - - return MakeAnonFields(type); + stginfo->flags |= DICTFLAG_FINAL; + + retval = MakeAnonFields(type); +error: + Py_XDECREF(layout_fields); + Py_XDECREF(layout); + Py_XDECREF(format_spec_obj); + return retval; } diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 2ec8f34c5c220b..bbbb62c9066df0 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -19,7 +19,13 @@ static const char PyCursesVersion[] = "2.1"; #include "py_curses.h" -#include +#if defined(HAVE_NCURSESW_PANEL_H) +# include +#elif defined(HAVE_NCURSES_PANEL_H) +# include +#elif defined(HAVE_PANEL_H) +# include +#endif typedef struct { PyObject *PyCursesError; @@ -697,6 +703,7 @@ static PyModuleDef_Slot _curses_slots[] = { // XXX gh-103092: fix isolation. {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index d04d1e973af030..27d5df08de933e 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -105,8 +105,9 @@ static const char PyCursesVersion[] = "2.2"; #endif #include "Python.h" -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_structseq.h" // _PyStructSequence_NewType() +#include "pycore_capsule.h" // _PyCapsule_SetTraverse() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_structseq.h" // _PyStructSequence_NewType() #ifdef __hpux #define STRICT_SYSV_CURSES @@ -128,7 +129,7 @@ static const char PyCursesVersion[] = "2.2"; #include #endif -#if !defined(HAVE_NCURSES_H) && (defined(sgi) || defined(__sun) || defined(SCO5)) +#if !defined(NCURSES_VERSION) && (defined(sgi) || defined(__sun) || defined(SCO5)) #define STRICT_SYSV_CURSES /* Don't use ncurses extensions */ typedef chtype attr_t; /* No attr_t type is available */ #endif @@ -159,65 +160,165 @@ typedef chtype attr_t; /* No attr_t type is available */ #define _CURSES_PAIR_CONTENT_FUNC pair_content #endif /* _NCURSES_EXTENDED_COLOR_FUNCS */ +typedef struct { + PyObject *error; // curses exception type + PyTypeObject *window_type; // exposed by PyCursesWindow_Type +} cursesmodule_state; + +static inline cursesmodule_state * +get_cursesmodule_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (cursesmodule_state *)state; +} + +static inline cursesmodule_state * +get_cursesmodule_state_by_cls(PyTypeObject *cls) +{ + void *state = PyType_GetModuleState(cls); + assert(state != NULL); + return (cursesmodule_state *)state; +} + +static inline cursesmodule_state * +get_cursesmodule_state_by_win(PyCursesWindowObject *win) +{ + return get_cursesmodule_state_by_cls(Py_TYPE(win)); +} + /*[clinic input] module _curses -class _curses.window "PyCursesWindowObject *" "&PyCursesWindow_Type" +class _curses.window "PyCursesWindowObject *" "clinic_state()->window_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=43265c372c2887d6]*/ - -/* Definition of exception curses.error */ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ae6cb623018f2cbc]*/ -static PyObject *PyCursesError; +/* Indicate whether the module has already been loaded or not. */ +static int curses_module_loaded = 0; /* Tells whether setupterm() has been called to initialise terminfo. */ -static int initialised_setupterm = FALSE; +static int curses_setupterm_called = FALSE; /* Tells whether initscr() has been called to initialise curses. */ -static int initialised = FALSE; +static int curses_initscr_called = FALSE; /* Tells whether start_color() has been called to initialise color usage. */ -static int initialisedcolors = FALSE; +static int curses_start_color_called = FALSE; + +static const char *curses_screen_encoding = NULL; + +/* Utility Checking Procedures */ + +/* + * Function to check that 'funcname' has been called by testing + * the 'called' boolean. If an error occurs, a PyCursesError is + * set and this returns 0. Otherwise, this returns 1. + * + * Since this function can be called in functions that do not + * have a direct access to the module's state, '_curses.error' + * is imported on demand. + */ +static inline int +_PyCursesCheckFunction(int called, const char *funcname) +{ + if (called == TRUE) { + return 1; + } + PyObject *exc = _PyImport_GetModuleAttrString("_curses", "error"); + if (exc != NULL) { + PyErr_Format(exc, "must call %s() first", funcname); + Py_DECREF(exc); + } + assert(PyErr_Occurred()); + return 0; +} -static char *screen_encoding = NULL; +/* + * Function to check that 'funcname' has been called by testing + * the 'called'' boolean. If an error occurs, a PyCursesError is + * set and this returns 0. Otherwise this returns 1. + * + * The exception type is obtained from the 'module' state. + */ +static inline int +_PyCursesStatefulCheckFunction(PyObject *module, int called, const char *funcname) +{ + if (called == TRUE) { + return 1; + } + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "must call %s() first", funcname); + return 0; +} -/* Utility Macros */ -#define PyCursesSetupTermCalled \ - if (initialised_setupterm != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call (at least) setupterm() first"); \ - return 0; } +#define PyCursesStatefulSetupTermCalled(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_setupterm_called, \ + "setupterm")) \ + { \ + return 0; \ + } \ + } while (0) -#define PyCursesInitialised \ - if (initialised != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call initscr() first"); \ - return 0; } +#define PyCursesStatefulInitialised(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_initscr_called, \ + "initscr")) \ + { \ + return 0; \ + } \ + } while (0) -#define PyCursesInitialisedColor \ - if (initialisedcolors != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call start_color() first"); \ - return 0; } +#define PyCursesStatefulInitialisedColor(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_start_color_called, \ + "start_color")) \ + { \ + return 0; \ + } \ + } while (0) /* Utility Functions */ +static inline void +_PyCursesSetError(cursesmodule_state *state, const char *funcname) +{ + if (funcname == NULL) { + PyErr_SetString(state->error, catchall_ERR); + } + else { + PyErr_Format(state->error, "%s() returned ERR", funcname); + } +} + /* * Check the return code from a curses function and return None - * or raise an exception as appropriate. These are exported using the - * capsule API. + * or raise an exception as appropriate. */ static PyObject * -PyCursesCheckERR(int code, const char *fname) +PyCursesCheckERR(PyObject *module, int code, const char *fname) { if (code != ERR) { Py_RETURN_NONE; } else { - if (fname == NULL) { - PyErr_SetString(PyCursesError, catchall_ERR); - } else { - PyErr_Format(PyCursesError, "%s() returned ERR", fname); - } + cursesmodule_state *state = get_cursesmodule_state(module); + _PyCursesSetError(state, fname); + return NULL; + } +} + +static PyObject * +PyCursesCheckERR_ForWin(PyCursesWindowObject *win, int code, const char *fname) +{ + if (code != ERR) { + Py_RETURN_NONE; + } else { + cursesmodule_state *state = get_cursesmodule_state_by_win(win); + _PyCursesSetError(state, fname); return NULL; } } @@ -233,13 +334,20 @@ static int PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) { long value; - if(PyBytes_Check(obj) && PyBytes_Size(obj) == 1) { + if (PyBytes_Check(obj)) { + if (PyBytes_GET_SIZE(obj) != 1) { + PyErr_Format(PyExc_TypeError, + "expect int or bytes or str of length 1, " + "got a bytes of length %zd", + PyBytes_GET_SIZE(obj)); + return 0; + } value = (unsigned char)PyBytes_AsString(obj)[0]; } else if (PyUnicode_Check(obj)) { - if (PyUnicode_GetLength(obj) != 1) { + if (PyUnicode_GET_LENGTH(obj) != 1) { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, " + "expect int or bytes or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -251,7 +359,7 @@ PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) if (win) encoding = win->encoding; else - encoding = screen_encoding; + encoding = curses_screen_encoding; bytes = PyUnicode_AsEncodedString(obj, encoding, NULL); if (bytes == NULL) return 0; @@ -272,7 +380,7 @@ PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) } else { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, got %s", + "expect int or bytes or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -315,7 +423,7 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, #ifdef HAVE_NCURSESW if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, " + "expect int or bytes or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -326,7 +434,14 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, return PyCurses_ConvertToChtype(win, obj, ch); #endif } - else if(PyBytes_Check(obj) && PyBytes_Size(obj) == 1) { + else if (PyBytes_Check(obj)) { + if (PyBytes_GET_SIZE(obj) != 1) { + PyErr_Format(PyExc_TypeError, + "expect int or bytes or str of length 1, " + "got a bytes of length %zd", + PyBytes_GET_SIZE(obj)); + return 0; + } value = (unsigned char)PyBytes_AsString(obj)[0]; } else if (PyLong_CheckExact(obj)) { @@ -340,7 +455,7 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, } else { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, got %s", + "expect int or bytes or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -527,35 +642,10 @@ class component_converter(CConverter): [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=38e9be01d33927fb]*/ -/* Function versions of the 3 functions for testing whether curses has been - initialised or not. */ - -static int func_PyCursesSetupTermCalled(void) -{ - PyCursesSetupTermCalled; - return 1; -} - -static int func_PyCursesInitialised(void) -{ - PyCursesInitialised; - return 1; -} - -static int func_PyCursesInitialisedColor(void) -{ - PyCursesInitialisedColor; - return 1; -} - /***************************************************************************** The Window Object ******************************************************************************/ -/* Definition of the window type */ - -PyTypeObject PyCursesWindow_Type; - /* Function prototype macros for Window object X - function name @@ -567,7 +657,7 @@ PyTypeObject PyCursesWindow_Type; #define Window_NoArgNoReturnFunction(X) \ static PyObject *PyCursesWindow_ ## X \ (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ - { return PyCursesCheckERR(X(self->win), # X); } + { return PyCursesCheckERR_ForWin(self, X(self->win), # X); } #define Window_NoArgTrueFalseFunction(X) \ static PyObject * PyCursesWindow_ ## X \ @@ -602,7 +692,7 @@ PyTypeObject PyCursesWindow_Type; { \ TYPE arg1; \ if (!PyArg_ParseTuple(args,PARSESTR, &arg1)) return NULL; \ - return PyCursesCheckERR(X(self->win, arg1), # X); } + return PyCursesCheckERR_ForWin(self, X(self->win, arg1), # X); } #define Window_TwoArgNoReturnFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ @@ -610,7 +700,7 @@ PyTypeObject PyCursesWindow_Type; { \ TYPE arg1, arg2; \ if (!PyArg_ParseTuple(args,PARSESTR, &arg1, &arg2)) return NULL; \ - return PyCursesCheckERR(X(self->win, arg1, arg2), # X); } + return PyCursesCheckERR_ForWin(self, X(self->win, arg1, arg2), # X); } /* ------------- WINDOW routines --------------- */ @@ -665,10 +755,9 @@ Window_TwoArgNoReturnFunction(wresize, int, "ii;lines,columns") /* Allocation and deallocation of Window Objects */ static PyObject * -PyCursesWindow_New(WINDOW *win, const char *encoding) +PyCursesWindow_New(cursesmodule_state *state, + WINDOW *win, const char *encoding) { - PyCursesWindowObject *wo; - if (encoding == NULL) { #if defined(MS_WINDOWS) char *buffer[100]; @@ -680,15 +769,20 @@ PyCursesWindow_New(WINDOW *win, const char *encoding) } #elif defined(CODESET) const char *codeset = nl_langinfo(CODESET); - if (codeset != NULL && codeset[0] != 0) + if (codeset != NULL && codeset[0] != 0) { encoding = codeset; + } #endif - if (encoding == NULL) + if (encoding == NULL) { encoding = "utf-8"; + } } - wo = PyObject_New(PyCursesWindowObject, &PyCursesWindow_Type); - if (wo == NULL) return NULL; + PyCursesWindowObject *wo = PyObject_GC_New(PyCursesWindowObject, + state->window_type); + if (wo == NULL) { + return NULL; + } wo->win = win; wo->encoding = _PyMem_Strdup(encoding); if (wo->encoding == NULL) { @@ -696,16 +790,32 @@ PyCursesWindow_New(WINDOW *win, const char *encoding) PyErr_NoMemory(); return NULL; } + PyObject_GC_Track((PyObject *)wo); return (PyObject *)wo; } static void -PyCursesWindow_Dealloc(PyCursesWindowObject *wo) +PyCursesWindow_dealloc(PyObject *self) { - if (wo->win != stdscr) delwin(wo->win); - if (wo->encoding != NULL) + PyTypeObject *window_type = Py_TYPE(self); + PyObject_GC_UnTrack(self); + PyCursesWindowObject *wo = (PyCursesWindowObject *)self; + if (wo->win != stdscr && wo->win != NULL) { + // silently ignore errors in delwin(3) + (void)delwin(wo->win); + } + if (wo->encoding != NULL) { PyMem_Free(wo->encoding); - PyObject_Free(wo); + } + window_type->tp_free(self); + Py_DECREF(window_type); +} + +static int +PyCursesWindow_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; } /* Addch, Addstr, Addnstr */ @@ -780,7 +890,7 @@ _curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, else { return NULL; } - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -860,7 +970,7 @@ _curses_window_addstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -943,7 +1053,7 @@ _curses_window_addnstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -967,7 +1077,7 @@ _curses_window_bkgd_impl(PyCursesWindowObject *self, PyObject *ch, long attr) if (!PyCurses_ConvertToChtype(self, ch, &bkgd)) return NULL; - return PyCursesCheckERR(wbkgd(self->win, bkgd | attr), "bkgd"); + return PyCursesCheckERR_ForWin(self, wbkgd(self->win, bkgd | attr), "bkgd"); } /*[clinic input] @@ -983,7 +1093,7 @@ static PyObject * _curses_window_attroff_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=8a2fcd4df682fc64 input=786beedf06a7befe]*/ { - return PyCursesCheckERR(wattroff(self->win, (attr_t)attr), "attroff"); + return PyCursesCheckERR_ForWin(self, wattroff(self->win, (attr_t)attr), "attroff"); } /*[clinic input] @@ -999,7 +1109,7 @@ static PyObject * _curses_window_attron_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=7afea43b237fa870 input=5a88fba7b1524f32]*/ { - return PyCursesCheckERR(wattron(self->win, (attr_t)attr), "attron"); + return PyCursesCheckERR_ForWin(self, wattron(self->win, (attr_t)attr), "attron"); } /*[clinic input] @@ -1015,7 +1125,7 @@ static PyObject * _curses_window_attrset_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=84e379bff20c0433 input=42e400c0d0154ab5]*/ { - return PyCursesCheckERR(wattrset(self->win, (attr_t)attr), "attrset"); + return PyCursesCheckERR_ForWin(self, wattrset(self->win, (attr_t)attr), "attrset"); } /*[clinic input] @@ -1041,7 +1151,7 @@ _curses_window_bkgdset_impl(PyCursesWindowObject *self, PyObject *ch, return NULL; wbkgdset(self->win, bkgd | attr); - return PyCursesCheckERR(0, "bkgdset"); + return PyCursesCheckERR_ForWin(self, 0, "bkgdset"); } /*[clinic input] @@ -1156,8 +1266,10 @@ int py_mvwdelch(WINDOW *w, int y, int x) #endif #if defined(HAVE_CURSES_IS_PAD) +// is_pad() is defined, either as a macro or as a function #define py_is_pad(win) is_pad(win) #elif defined(WINDOW_HAS_FLAGS) +// is_pad() is not defined, but we can inspect WINDOW structure members #define py_is_pad(win) ((win) ? ((win)->_flags & _ISPAD) != 0 : FALSE) #endif @@ -1239,7 +1351,7 @@ PyCursesWindow_ChgAt(PyCursesWindowObject *self, PyObject *args) rtn = wchgat(self->win,num,attr,color,NULL); touchline(self->win,y,1); } - return PyCursesCheckERR(rtn, "chgat"); + return PyCursesCheckERR_ForWin(self, rtn, "chgat"); } #endif @@ -1263,10 +1375,10 @@ _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, /*[clinic end generated code: output=22e77bb9fa11b461 input=d2f79e630a4fc6d0]*/ { if (!group_right_1) { - return PyCursesCheckERR(wdelch(self->win), "wdelch"); + return PyCursesCheckERR_ForWin(self, wdelch(self->win), "wdelch"); } else { - return PyCursesCheckERR(py_mvwdelch(self->win, y, x), "mvwdelch"); + return PyCursesCheckERR_ForWin(self, py_mvwdelch(self->win, y, x), "mvwdelch"); } } @@ -1302,11 +1414,13 @@ _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, win = derwin(self->win,nlines,ncols,begin_y,begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + return PyCursesWindow_New(state, win, NULL); } /*[clinic input] @@ -1334,13 +1448,15 @@ _curses_window_echochar_impl(PyCursesWindowObject *self, PyObject *ch, #ifdef py_is_pad if (py_is_pad(self->win)) { - return PyCursesCheckERR(pechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + return PyCursesCheckERR_ForWin(self, + pechochar(self->win, ch_ | (attr_t)attr), + "echochar"); } else #endif - return PyCursesCheckERR(wechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + return PyCursesCheckERR_ForWin(self, + wechochar(self->win, ch_ | (attr_t)attr), + "echochar"); } #ifdef NCURSES_MOUSE_VERSION @@ -1451,8 +1567,10 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, if (rtn == ERR) { /* getch() returns ERR in nodelay mode */ PyErr_CheckSignals(); - if (!PyErr_Occurred()) - PyErr_SetString(PyCursesError, "no input"); + if (!PyErr_Occurred()) { + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "no input"); + } return NULL; } else if (rtn <= 255) { #ifdef NCURSES_VERSION_MAJOR @@ -1510,7 +1628,8 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, return NULL; /* get_wch() returns ERR in nodelay mode */ - PyErr_SetString(PyCursesError, "no input"); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "no input"); return NULL; } if (ct == KEY_CODE_YES) @@ -1634,10 +1753,10 @@ _curses_window_hline_impl(PyCursesWindowObject *self, int group_left_1, return NULL; if (group_left_1) { if (wmove(self->win, y, x) == ERR) { - return PyCursesCheckERR(ERR, "wmove"); + return PyCursesCheckERR_ForWin(self, ERR, "wmove"); } } - return PyCursesCheckERR(whline(self->win, ch_ | (attr_t)attr, n), "hline"); + return PyCursesCheckERR_ForWin(self, whline(self->win, ch_ | (attr_t)attr, n), "hline"); } /*[clinic input] @@ -1684,7 +1803,7 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, rtn = mvwinsch(self->win, y, x, ch_ | (attr_t)attr); } - return PyCursesCheckERR(rtn, "insch"); + return PyCursesCheckERR_ForWin(self, rtn, "insch"); } /*[clinic input] @@ -1861,7 +1980,7 @@ _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -1946,7 +2065,7 @@ _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -2023,7 +2142,8 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - PyErr_SetString(PyCursesError, + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "noutrefresh() called for a pad " "requires 6 arguments"); return NULL; @@ -2032,7 +2152,7 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) rtn = pnoutrefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "pnoutrefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "pnoutrefresh"); } if (group_right_1) { PyErr_SetString(PyExc_TypeError, @@ -2043,13 +2163,13 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) Py_BEGIN_ALLOW_THREADS rtn = wnoutrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "wnoutrefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "wnoutrefresh"); } /*[clinic input] _curses.window.overlay - destwin: object(type="PyCursesWindowObject *", subclass_of="&PyCursesWindow_Type") + destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") [ sminrow: int @@ -2078,25 +2198,25 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, PyCursesWindowObject *destwin, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=82bb2c4cb443ca58 input=7edd23ad22cc1984]*/ +/*[clinic end generated code: output=82bb2c4cb443ca58 input=6e4b32a7c627a356]*/ { int rtn; if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, TRUE); - return PyCursesCheckERR(rtn, "copywin"); + return PyCursesCheckERR_ForWin(self, rtn, "copywin"); } else { rtn = overlay(self->win, destwin->win); - return PyCursesCheckERR(rtn, "overlay"); + return PyCursesCheckERR_ForWin(self, rtn, "overlay"); } } /*[clinic input] _curses.window.overwrite - destwin: object(type="PyCursesWindowObject *", subclass_of="&PyCursesWindow_Type") + destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") [ sminrow: int @@ -2126,18 +2246,18 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=12ae007d1681be28 input=ea5de1b35cd948e0]*/ +/*[clinic end generated code: output=12ae007d1681be28 input=d83dd8b24ff2bcc9]*/ { int rtn; if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, FALSE); - return PyCursesCheckERR(rtn, "copywin"); + return PyCursesCheckERR_ForWin(self, rtn, "copywin"); } else { rtn = overwrite(self->win, destwin->win); - return PyCursesCheckERR(rtn, "overwrite"); + return PyCursesCheckERR_ForWin(self, rtn, "overwrite"); } } @@ -2166,7 +2286,7 @@ _curses_window_putwin(PyCursesWindowObject *self, PyObject *file) return PyErr_SetFromErrno(PyExc_OSError); if (_Py_set_inheritable(fileno(fp), 0, NULL) < 0) goto exit; - res = PyCursesCheckERR(putwin(self->win, fp), "putwin"); + res = PyCursesCheckERR_ForWin(self, putwin(self->win, fp), "putwin"); if (res == NULL) goto exit; fseek(fp, 0, 0); @@ -2205,7 +2325,7 @@ static PyObject * _curses_window_redrawln_impl(PyCursesWindowObject *self, int beg, int num) /*[clinic end generated code: output=ea216e334f9ce1b4 input=152155e258a77a7a]*/ { - return PyCursesCheckERR(wredrawln(self->win,beg,num), "redrawln"); + return PyCursesCheckERR_ForWin(self, wredrawln(self->win,beg,num), "redrawln"); } /*[clinic input] @@ -2247,7 +2367,8 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - PyErr_SetString(PyCursesError, + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "refresh() for a pad requires 6 arguments"); return NULL; } @@ -2255,7 +2376,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, rtn = prefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "prefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); } #endif if (group_right_1) { @@ -2266,7 +2387,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, Py_BEGIN_ALLOW_THREADS rtn = wrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "prefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); } /*[clinic input] @@ -2288,7 +2409,7 @@ _curses_window_setscrreg_impl(PyCursesWindowObject *self, int top, int bottom) /*[clinic end generated code: output=486ab5db218d2b1a input=1b517b986838bf0e]*/ { - return PyCursesCheckERR(wsetscrreg(self->win, top, bottom), "wsetscrreg"); + return PyCursesCheckERR_ForWin(self, wsetscrreg(self->win, top, bottom), "wsetscrreg"); } /*[clinic input] @@ -2329,11 +2450,13 @@ _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, win = subwin(self->win, nlines, ncols, begin_y, begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, self->encoding); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + return PyCursesWindow_New(state, win, self->encoding); } /*[clinic input] @@ -2356,10 +2479,10 @@ _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, /*[clinic end generated code: output=4541a8a11852d360 input=c969ca0cfabbdbec]*/ { if (!group_right_1) { - return PyCursesCheckERR(scroll(self->win), "scroll"); + return PyCursesCheckERR_ForWin(self, scroll(self->win), "scroll"); } else { - return PyCursesCheckERR(wscrl(self->win, lines), "scroll"); + return PyCursesCheckERR_ForWin(self, wscrl(self->win, lines), "scroll"); } } @@ -2385,10 +2508,10 @@ _curses_window_touchline_impl(PyCursesWindowObject *self, int start, /*[clinic end generated code: output=65d05b3f7438c61d input=a98aa4f79b6be845]*/ { if (!group_right_1) { - return PyCursesCheckERR(touchline(self->win, start, count), "touchline"); + return PyCursesCheckERR_ForWin(self, touchline(self->win, start, count), "touchline"); } else { - return PyCursesCheckERR(wtouchln(self->win, start, count, changed), "touchline"); + return PyCursesCheckERR_ForWin(self, wtouchln(self->win, start, count, changed), "touchline"); } } @@ -2428,9 +2551,9 @@ _curses_window_vline_impl(PyCursesWindowObject *self, int group_left_1, return NULL; if (group_left_1) { if (wmove(self->win, y, x) == ERR) - return PyCursesCheckERR(ERR, "wmove"); + return PyCursesCheckERR_ForWin(self, ERR, "wmove"); } - return PyCursesCheckERR(wvline(self->win, ch_ | (attr_t)attr, n), "vline"); + return PyCursesCheckERR_ForWin(self, wvline(self->win, ch_ | (attr_t)attr, n), "vline"); } static PyObject * @@ -2471,9 +2594,11 @@ PyCursesWindow_set_encoding(PyCursesWindowObject *self, PyObject *value, void *P return 0; } +#define clinic_state() (get_cursesmodule_state_by_cls(Py_TYPE(self))) #include "clinic/_cursesmodule.c.h" +#undef clinic_state -static PyMethodDef PyCursesWindow_Methods[] = { +static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_ADDCH_METHODDEF _CURSES_WINDOW_ADDNSTR_METHODDEF _CURSES_WINDOW_ADDSTR_METHODDEF @@ -2567,43 +2692,28 @@ static PyGetSetDef PyCursesWindow_getsets[] = { {NULL, NULL, NULL, NULL } /* sentinel */ }; -/* -------------------------------------------------------*/ +static PyType_Slot PyCursesWindow_Type_slots[] = { + {Py_tp_methods, PyCursesWindow_methods}, + {Py_tp_getset, PyCursesWindow_getsets}, + {Py_tp_dealloc, PyCursesWindow_dealloc}, + {Py_tp_traverse, PyCursesWindow_traverse}, + {0, NULL} +}; -PyTypeObject PyCursesWindow_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_curses.window", /*tp_name*/ - sizeof(PyCursesWindowObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)PyCursesWindow_Dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - (getattrfunc)0, /*tp_getattr*/ - (setattrfunc)0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - PyCursesWindow_Methods, /*tp_methods*/ - 0, /* tp_members */ - PyCursesWindow_getsets, /* tp_getset */ +static PyType_Spec PyCursesWindow_Type_spec = { + .name = "_curses.window", + .basicsize = sizeof(PyCursesWindowObject), + .flags = Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HEAPTYPE + | Py_TPFLAGS_HAVE_GC, + .slots = PyCursesWindow_Type_slots }; -/* Function Prototype Macros - They are ugly but very, very useful. ;-) +/* -------------------------------------------------------*/ + +/* Function Body Macros - They are ugly but very, very useful. ;-) X - function name TYPE - parameter Type @@ -2613,37 +2723,37 @@ PyTypeObject PyCursesWindow_Type = { #define NoArgNoReturnFunctionBody(X) \ { \ - PyCursesInitialised \ - return PyCursesCheckERR(X(), # X); } + PyCursesStatefulInitialised(module); \ + return PyCursesCheckERR(module, X(), # X); } #define NoArgOrFlagNoReturnFunctionBody(X, flag) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ if (flag) \ - return PyCursesCheckERR(X(), # X); \ + return PyCursesCheckERR(module, X(), # X); \ else \ - return PyCursesCheckERR(no ## X(), # X); \ + return PyCursesCheckERR(module, no ## X(), # X); \ } #define NoArgReturnIntFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ return PyLong_FromLong((long) X()); } #define NoArgReturnStringFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ return PyBytes_FromString(X()); } #define NoArgTrueFalseFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ return PyBool_FromLong(X()); } #define NoArgNoReturnVoidFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ X(); \ Py_RETURN_NONE; } @@ -2741,12 +2851,13 @@ _curses_color_content_impl(PyObject *module, int color_number) { _CURSES_COLOR_VAL_TYPE r,g,b; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_COLOR_CONTENT_FUNC(color_number, &r, &g, &b) == ERR) { - PyErr_Format(PyCursesError, "%s() returned ERR", - Py_STRINGIFY(_COLOR_CONTENT_FUNC)); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", + Py_STRINGIFY(_COLOR_CONTENT_FUNC)); return NULL; } @@ -2770,8 +2881,8 @@ static PyObject * _curses_color_pair_impl(PyObject *module, int pair_number) /*[clinic end generated code: output=60718abb10ce9feb input=6034e9146f343802]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); return PyLong_FromLong(COLOR_PAIR(pair_number)); } @@ -2797,10 +2908,10 @@ _curses_curs_set_impl(PyObject *module, int visibility) { int erg; - PyCursesInitialised; + PyCursesStatefulInitialised(module); erg = curs_set(visibility); - if (erg == ERR) return PyCursesCheckERR(erg, "curs_set"); + if (erg == ERR) return PyCursesCheckERR(module, erg, "curs_set"); return PyLong_FromLong((long) erg); } @@ -2849,9 +2960,9 @@ static PyObject * _curses_delay_output_impl(PyObject *module, int ms) /*[clinic end generated code: output=b6613a67f17fa4f4 input=5316457f5f59196c]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(delay_output(ms), "delay_output"); + return PyCursesCheckERR(module, delay_output(ms), "delay_output"); } /*[clinic input] @@ -2905,7 +3016,7 @@ _curses_erasechar_impl(PyObject *module) { char ch; - PyCursesInitialised; + PyCursesStatefulInitialised(module); ch = erasechar(); @@ -2955,7 +3066,7 @@ _curses_getsyx_impl(PyObject *module) int x = 0; int y = 0; - PyCursesInitialised; + PyCursesStatefulInitialised(module); getsyx(y, x); @@ -2980,11 +3091,12 @@ _curses_getmouse_impl(PyObject *module) int rtn; MEVENT event; - PyCursesInitialised; + PyCursesStatefulInitialised(module); rtn = getmouse( &event ); if (rtn == ERR) { - PyErr_SetString(PyCursesError, "getmouse() returned ERR"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "getmouse() returned ERR"); return NULL; } return Py_BuildValue("(hiiik)", @@ -3015,14 +3127,14 @@ _curses_ungetmouse_impl(PyObject *module, short id, int x, int y, int z, { MEVENT event; - PyCursesInitialised; + PyCursesStatefulInitialised(module); event.id = id; event.x = x; event.y = y; event.z = z; event.bstate = bstate; - return PyCursesCheckERR(ungetmouse(&event), "ungetmouse"); + return PyCursesCheckERR(module, ungetmouse(&event), "ungetmouse"); } #endif @@ -3048,7 +3160,7 @@ _curses_getwin(PyObject *module, PyObject *file) WINDOW *win; PyObject *res = NULL; - PyCursesInitialised; + PyCursesStatefulInitialised(module); fp = tmpfile(); if (fp == NULL) @@ -3078,10 +3190,12 @@ _curses_getwin(PyObject *module, PyObject *file) fseek(fp, 0, 0); win = getwin(fp); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); goto error; } - res = PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + res = PyCursesWindow_New(state, win, NULL); error: fclose(fp); @@ -3104,9 +3218,9 @@ static PyObject * _curses_halfdelay_impl(PyObject *module, unsigned char tenths) /*[clinic end generated code: output=e92cdf0ef33c0663 input=e42dce7259c15100]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(halfdelay(tenths), "halfdelay"); + return PyCursesCheckERR(module, halfdelay(tenths), "halfdelay"); } /*[clinic input] @@ -3157,7 +3271,7 @@ static PyObject * _curses_has_key_impl(PyObject *module, int key) /*[clinic end generated code: output=19ad48319414d0b1 input=78bd44acf1a4997c]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return PyBool_FromLong(has_key(key)); } @@ -3188,10 +3302,11 @@ _curses_init_color_impl(PyObject *module, int color_number, short r, short g, short b) /*[clinic end generated code: output=d7ed71b2d818cdf2 input=ae2b8bea0f152c80]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); - return PyCursesCheckERR(_CURSES_INIT_COLOR_FUNC(color_number, r, g, b), + return PyCursesCheckERR(module, + _CURSES_INIT_COLOR_FUNC(color_number, r, g, b), Py_STRINGIFY(_CURSES_INIT_COLOR_FUNC)); } @@ -3216,8 +3331,8 @@ static PyObject * _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) /*[clinic end generated code: output=a0bba03d2bbc3ee6 input=54b421b44c12c389]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_CURSES_INIT_PAIR_FUNC(pair_number, fg, bg) == ERR) { if (pair_number >= COLOR_PAIRS) { @@ -3226,7 +3341,8 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) COLOR_PAIRS - 1); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", Py_STRINGIFY(_CURSES_INIT_PAIR_FUNC)); } return NULL; @@ -3235,8 +3351,6 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) Py_RETURN_NONE; } -static PyObject *ModDict; - /*[clinic input] _curses.initscr @@ -3250,30 +3364,40 @@ _curses_initscr_impl(PyObject *module) /*[clinic end generated code: output=619fb68443810b7b input=514f4bce1821f6b5]*/ { WINDOW *win; - PyCursesWindowObject *winobj; - if (initialised) { + if (curses_initscr_called) { wrefresh(stdscr); - return (PyObject *)PyCursesWindow_New(stdscr, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesWindow_New(state, stdscr, NULL); } win = initscr(); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - initialised = initialised_setupterm = TRUE; + curses_initscr_called = curses_setupterm_called = TRUE; -/* This was moved from initcurses() because it core dumped on SGI, - where they're not defined until you've called initscr() */ -#define SetDictInt(string,ch) \ - do { \ - PyObject *o = PyLong_FromLong((long) (ch)); \ - if (o && PyDict_SetItemString(ModDict, string, o) == 0) { \ - Py_DECREF(o); \ - } \ + PyObject *module_dict = PyModule_GetDict(module); // borrowed + if (module_dict == NULL) { + return NULL; + } + /* This was moved from initcurses() because it core dumped on SGI, + where they're not defined until you've called initscr() */ +#define SetDictInt(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ } while (0) /* Here are some graphic symbols you can use */ @@ -3345,10 +3469,15 @@ _curses_initscr_impl(PyObject *module) SetDictInt("LINES", LINES); SetDictInt("COLS", COLS); +#undef SetDictInt - winobj = (PyCursesWindowObject *)PyCursesWindow_New(win, NULL); - screen_encoding = winobj->encoding; - return (PyObject *)winobj; + cursesmodule_state *state = get_cursesmodule_state(module); + PyObject *winobj = PyCursesWindow_New(state, win, NULL); + if (winobj == NULL) { + return NULL; + } + curses_screen_encoding = ((PyCursesWindowObject *)winobj)->encoding; + return winobj; } /*[clinic input] @@ -3376,9 +3505,8 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) sys_stdout = PySys_GetObject("stdout"); if (sys_stdout == NULL || sys_stdout == Py_None) { - PyErr_SetString( - PyCursesError, - "lost sys.stdout"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "lost sys.stdout"); return NULL; } @@ -3389,7 +3517,7 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) } } - if (!initialised_setupterm && setupterm((char *)term, fd, &err) == ERR) { + if (!curses_setupterm_called && setupterm((char *)term, fd, &err) == ERR) { const char* s = "setupterm: unknown error"; if (err == 0) { @@ -3398,11 +3526,12 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) s = "setupterm: could not find terminfo database"; } - PyErr_SetString(PyCursesError,s); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, s); return NULL; } - initialised_setupterm = TRUE; + curses_setupterm_called = TRUE; Py_RETURN_NONE; } @@ -3448,7 +3577,7 @@ _curses_set_escdelay_impl(PyObject *module, int ms) return NULL; } - return PyCursesCheckERR(set_escdelay(ms), "set_escdelay"); + return PyCursesCheckERR(module, set_escdelay(ms), "set_escdelay"); } /*[clinic input] @@ -3487,7 +3616,7 @@ _curses_set_tabsize_impl(PyObject *module, int size) return NULL; } - return PyCursesCheckERR(set_tabsize(size), "set_tabsize"); + return PyCursesCheckERR(module, set_tabsize(size), "set_tabsize"); } #endif @@ -3503,9 +3632,9 @@ static PyObject * _curses_intrflush_impl(PyObject *module, int flag) /*[clinic end generated code: output=c1986df35e999a0f input=c65fe2ef973fe40a]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(intrflush(NULL, flag), "intrflush"); + return PyCursesCheckERR(module, intrflush(NULL, flag), "intrflush"); } /*[clinic input] @@ -3536,7 +3665,7 @@ static PyObject * _curses_is_term_resized_impl(PyObject *module, int nlines, int ncols) /*[clinic end generated code: output=aafe04afe50f1288 input=ca9c0bd0fb8ab444]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return PyBool_FromLong(is_term_resized(nlines, ncols)); } @@ -3558,7 +3687,7 @@ _curses_keyname_impl(PyObject *module, int key) { const char *knp; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (key < 0) { PyErr_SetString(PyExc_ValueError, "invalid key number"); @@ -3616,9 +3745,9 @@ static PyObject * _curses_meta_impl(PyObject *module, int yes) /*[clinic end generated code: output=22f5abda46a605d8 input=cfe7da79f51d0e30]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(meta(stdscr, yes), "meta"); + return PyCursesCheckERR(module, meta(stdscr, yes), "meta"); } #ifdef NCURSES_MOUSE_VERSION @@ -3640,9 +3769,9 @@ static PyObject * _curses_mouseinterval_impl(PyObject *module, int interval) /*[clinic end generated code: output=c4f5ff04354634c5 input=75aaa3f0db10ac4e]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(mouseinterval(interval), "mouseinterval"); + return PyCursesCheckERR(module, mouseinterval(interval), "mouseinterval"); } /*[clinic input] @@ -3665,7 +3794,7 @@ _curses_mousemask_impl(PyObject *module, unsigned long newmask) { mmask_t oldmask, availmask; - PyCursesInitialised; + PyCursesStatefulInitialised(module); availmask = mousemask((mmask_t)newmask, &oldmask); return Py_BuildValue("(kk)", (unsigned long)availmask, (unsigned long)oldmask); @@ -3686,7 +3815,7 @@ static int _curses_napms_impl(PyObject *module, int ms) /*[clinic end generated code: output=5f292a6a724491bd input=c6d6e01f2f1df9f7]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return napms(ms); } @@ -3710,16 +3839,18 @@ _curses_newpad_impl(PyObject *module, int nlines, int ncols) { WINDOW *win; - PyCursesInitialised; + PyCursesStatefulInitialised(module); win = newpad(nlines, ncols); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesWindow_New(state, win, NULL); } /*[clinic input] @@ -3750,15 +3881,17 @@ _curses_newwin_impl(PyObject *module, int nlines, int ncols, { WINDOW *win; - PyCursesInitialised; + PyCursesStatefulInitialised(module); win = newwin(nlines,ncols,begin_y,begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesWindow_New(state, win, NULL); } /*[clinic input] @@ -3862,8 +3995,8 @@ _curses_pair_content_impl(PyObject *module, int pair_number) { _CURSES_COLOR_NUM_TYPE f, b; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_CURSES_PAIR_CONTENT_FUNC(pair_number, &f, &b) == ERR) { if (pair_number >= COLOR_PAIRS) { @@ -3872,7 +4005,8 @@ _curses_pair_content_impl(PyObject *module, int pair_number) COLOR_PAIRS - 1); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", Py_STRINGIFY(_CURSES_PAIR_CONTENT_FUNC)); } return NULL; @@ -3896,8 +4030,8 @@ static PyObject * _curses_pair_number_impl(PyObject *module, int attr) /*[clinic end generated code: output=85bce7d65c0aa3f4 input=d478548e33f5e61a]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); return PyLong_FromLong(PAIR_NUMBER(attr)); } @@ -3917,7 +4051,7 @@ static PyObject * _curses_putp_impl(PyObject *module, const char *string) /*[clinic end generated code: output=e98081d1b8eb5816 input=1601faa828b44cb3]*/ { - return PyCursesCheckERR(putp(string), "putp"); + return PyCursesCheckERR(module, putp(string), "putp"); } /*[clinic input] @@ -3937,7 +4071,7 @@ static PyObject * _curses_qiflush_impl(PyObject *module, int flag) /*[clinic end generated code: output=9167e862f760ea30 input=6ec8b3e2b717ec40]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (flag) { qiflush(); @@ -3948,52 +4082,57 @@ _curses_qiflush_impl(PyObject *module, int flag) Py_RETURN_NONE; } -/* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES - * and _curses.COLS */ #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM) +/* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES + * and _curses.COLS. Returns 1 on success and 0 on failure. */ static int -update_lines_cols(void) +update_lines_cols(PyObject *private_module) { - PyObject *o; - PyObject *m = PyImport_ImportModule("curses"); + PyObject *exposed_module = NULL, *o = NULL; - if (!m) - return 0; + exposed_module = PyImport_ImportModule("curses"); + if (exposed_module == NULL) { + goto error; + } + PyObject *exposed_module_dict = PyModule_GetDict(exposed_module); // borrowed + if (exposed_module_dict == NULL) { + goto error; + } + PyObject *private_module_dict = PyModule_GetDict(private_module); // borrowed + if (private_module_dict == NULL) { + goto error; + } o = PyLong_FromLong(LINES); - if (!o) { - Py_DECREF(m); - return 0; + if (o == NULL) { + goto error; } - if (PyObject_SetAttrString(m, "LINES", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(exposed_module_dict, "LINES", o) < 0) { + goto error; } - if (PyDict_SetItemString(ModDict, "LINES", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(private_module_dict, "LINES", o) < 0) { + goto error; } Py_DECREF(o); + o = PyLong_FromLong(COLS); - if (!o) { - Py_DECREF(m); - return 0; + if (o == NULL) { + goto error; } - if (PyObject_SetAttrString(m, "COLS", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(exposed_module_dict, "COLS", o) < 0) { + goto error; } - if (PyDict_SetItemString(ModDict, "COLS", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(private_module_dict, "COLS", o) < 0) { + goto error; } Py_DECREF(o); - Py_DECREF(m); + Py_DECREF(exposed_module); return 1; + +error: + Py_XDECREF(o); + Py_XDECREF(exposed_module); + return 0; } /*[clinic input] @@ -4005,7 +4144,7 @@ static PyObject * _curses_update_lines_cols_impl(PyObject *module) /*[clinic end generated code: output=423f2b1e63ed0f75 input=5f065ab7a28a5d90]*/ { - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { return NULL; } Py_RETURN_NONE; @@ -4069,9 +4208,9 @@ NoArgNoReturnFunctionBody(resetty) /*[clinic input] _curses.resizeterm - nlines: int + nlines: short Height. - ncols: int + ncols: short Width. / @@ -4082,17 +4221,17 @@ window dimensions (in particular the SIGWINCH handler). [clinic start generated code]*/ static PyObject * -_curses_resizeterm_impl(PyObject *module, int nlines, int ncols) -/*[clinic end generated code: output=56d6bcc5194ad055 input=0fca02ebad5ffa82]*/ +_curses_resizeterm_impl(PyObject *module, short nlines, short ncols) +/*[clinic end generated code: output=4de3abab50c67f02 input=414e92a63e3e9899]*/ { PyObject *result; - PyCursesInitialised; + PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(resizeterm(nlines, ncols), "resizeterm"); + result = PyCursesCheckERR(module, resizeterm(nlines, ncols), "resizeterm"); if (!result) return NULL; - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { Py_DECREF(result); return NULL; } @@ -4105,9 +4244,9 @@ _curses_resizeterm_impl(PyObject *module, int nlines, int ncols) /*[clinic input] _curses.resize_term - nlines: int + nlines: short Height. - ncols: int + ncols: short Width. / @@ -4121,17 +4260,17 @@ without additional interaction with the application. [clinic start generated code]*/ static PyObject * -_curses_resize_term_impl(PyObject *module, int nlines, int ncols) -/*[clinic end generated code: output=9e26d8b9ea311ed2 input=2197edd05b049ed4]*/ +_curses_resize_term_impl(PyObject *module, short nlines, short ncols) +/*[clinic end generated code: output=46c6d749fa291dbd input=276afa43d8ea7091]*/ { PyObject *result; - PyCursesInitialised; + PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(resize_term(nlines, ncols), "resize_term"); + result = PyCursesCheckERR(module, resize_term(nlines, ncols), "resize_term"); if (!result) return NULL; - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { Py_DECREF(result); return NULL; } @@ -4169,7 +4308,7 @@ static PyObject * _curses_setsyx_impl(PyObject *module, int y, int x) /*[clinic end generated code: output=23dcf753511a2464 input=fa7f2b208e10a557]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); setsyx(y,x); @@ -4194,35 +4333,38 @@ static PyObject * _curses_start_color_impl(PyObject *module) /*[clinic end generated code: output=8b772b41d8090ede input=0ca0ecb2b77e1a12]*/ { - int code; - PyObject *c, *cp; + PyCursesStatefulInitialised(module); - PyCursesInitialised; + if (start_color() == ERR) { + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "start_color() returned ERR"); + return NULL; + } - code = start_color(); - if (code != ERR) { - initialisedcolors = TRUE; - c = PyLong_FromLong((long) COLORS); - if (c == NULL) - return NULL; - if (PyDict_SetItemString(ModDict, "COLORS", c) < 0) { - Py_DECREF(c); - return NULL; - } - Py_DECREF(c); - cp = PyLong_FromLong((long) COLOR_PAIRS); - if (cp == NULL) - return NULL; - if (PyDict_SetItemString(ModDict, "COLOR_PAIRS", cp) < 0) { - Py_DECREF(cp); - return NULL; - } - Py_DECREF(cp); - Py_RETURN_NONE; - } else { - PyErr_SetString(PyCursesError, "start_color() returned ERR"); + curses_start_color_called = TRUE; + + PyObject *module_dict = PyModule_GetDict(module); // borrowed + if (module_dict == NULL) { return NULL; } +#define DICT_ADD_INT_VALUE(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ + } while (0) + + DICT_ADD_INT_VALUE("COLORS", COLORS); + DICT_ADD_INT_VALUE("COLOR_PAIRS", COLOR_PAIRS); +#undef DICT_ADD_INT_VALUE + + Py_RETURN_NONE; } /*[clinic input] @@ -4264,7 +4406,7 @@ static PyObject * _curses_tigetflag_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=8853c0e55542195b input=b0787af9e3e9a6ce]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); return PyLong_FromLong( (long) tigetflag( (char *)capname ) ); } @@ -4286,7 +4428,7 @@ static PyObject * _curses_tigetnum_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=46f8b0a1b5dff42f input=5cdf2f410b109720]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); return PyLong_FromLong( (long) tigetnum( (char *)capname ) ); } @@ -4308,7 +4450,7 @@ static PyObject * _curses_tigetstr_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=f22b576ad60248f3 input=36644df25c73c0a7]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); capname = tigetstr( (char *)capname ); if (capname == NULL || capname == (char*) -1) { @@ -4343,11 +4485,12 @@ _curses_tparm_impl(PyObject *module, const char *str, int i1, int i2, int i3, { char* result = NULL; - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); result = tparm((char *)str,i1,i2,i3,i4,i5,i6,i7,i8,i9); if (!result) { - PyErr_SetString(PyCursesError, "tparm() returned NULL"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "tparm() returned NULL"); return NULL; } @@ -4371,9 +4514,9 @@ static PyObject * _curses_typeahead_impl(PyObject *module, int fd) /*[clinic end generated code: output=084bb649d7066583 input=f2968d8e1805051b]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(typeahead( fd ), "typeahead"); + return PyCursesCheckERR(module, typeahead( fd ), "typeahead"); } #endif @@ -4395,7 +4538,7 @@ _curses_unctrl(PyObject *module, PyObject *ch) { chtype ch_; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; @@ -4418,12 +4561,12 @@ _curses_ungetch(PyObject *module, PyObject *ch) { chtype ch_; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; - return PyCursesCheckERR(ungetch(ch_), "ungetch"); + return PyCursesCheckERR(module, ungetch(ch_), "ungetch"); } #ifdef HAVE_NCURSESW @@ -4441,7 +4584,7 @@ PyCurses_ConvertToWchar_t(PyObject *obj, wchar_t buffer[2]; if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { PyErr_Format(PyExc_TypeError, - "expect str of length 1 or int, " + "expect int or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -4468,7 +4611,7 @@ PyCurses_ConvertToWchar_t(PyObject *obj, } else { PyErr_Format(PyExc_TypeError, - "expect str of length 1 or int, got %s", + "expect int or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -4489,11 +4632,11 @@ _curses_unget_wch(PyObject *module, PyObject *ch) { wchar_t wch; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToWchar_t(ch, &wch)) return NULL; - return PyCursesCheckERR(unget_wch(wch), "unget_wch"); + return PyCursesCheckERR(module, unget_wch(wch), "unget_wch"); } #endif @@ -4541,14 +4684,15 @@ _curses_use_default_colors_impl(PyObject *module) { int code; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); code = use_default_colors(); if (code != ERR) { Py_RETURN_NONE; } else { - PyErr_SetString(PyCursesError, "use_default_colors() returned ERR"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "use_default_colors() returned ERR"); return NULL; } } @@ -4579,26 +4723,32 @@ static PyStructSequence_Desc ncurses_version_desc = { static PyObject * make_ncurses_version(PyTypeObject *type) { - PyObject *ncurses_version; - int pos = 0; - - ncurses_version = PyStructSequence_New(type); + PyObject *ncurses_version = PyStructSequence_New(type); if (ncurses_version == NULL) { return NULL; } - -#define SetIntItem(flag) \ - PyStructSequence_SET_ITEM(ncurses_version, pos++, PyLong_FromLong(flag)); \ - if (PyErr_Occurred()) { \ - Py_CLEAR(ncurses_version); \ - return NULL; \ + const char *str = curses_version(); + unsigned long major = 0, minor = 0, patch = 0; + if (!str || sscanf(str, "%*[^0-9]%lu.%lu.%lu", &major, &minor, &patch) < 3) { + // Fallback to header version, which cannot be that wrong + major = NCURSES_VERSION_MAJOR; + minor = NCURSES_VERSION_MINOR; + patch = NCURSES_VERSION_PATCH; } +#define SET_VERSION_COMPONENT(INDEX, VALUE) \ + do { \ + PyObject *o = PyLong_FromLong(VALUE); \ + if (o == NULL) { \ + Py_DECREF(ncurses_version); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(ncurses_version, INDEX, o); \ + } while (0) - SetIntItem(NCURSES_VERSION_MAJOR) - SetIntItem(NCURSES_VERSION_MINOR) - SetIntItem(NCURSES_VERSION_PATCH) -#undef SetIntItem - + SET_VERSION_COMPONENT(0, major); + SET_VERSION_COMPONENT(1, minor); + SET_VERSION_COMPONENT(2, patch); +#undef SET_VERSION_COMPONENT return ncurses_version; } @@ -4622,7 +4772,7 @@ _curses_has_extended_color_support_impl(PyObject *module) /* List of functions defined in the module */ -static PyMethodDef PyCurses_methods[] = { +static PyMethodDef cursesmodule_methods[] = { _CURSES_BAUDRATE_METHODDEF _CURSES_BEEP_METHODDEF _CURSES_CAN_CHANGE_COLOR_METHODDEF @@ -4707,83 +4857,197 @@ static PyMethodDef PyCurses_methods[] = { {NULL, NULL} /* sentinel */ }; -/* Initialization function for the module */ +/* Module C API */ +/* Function versions of the 3 functions for testing whether curses has been + initialised or not. */ -static struct PyModuleDef _cursesmodule = { - PyModuleDef_HEAD_INIT, - "_curses", - NULL, - -1, - PyCurses_methods, - NULL, - NULL, - NULL, - NULL -}; +static inline int +curses_capi_setupterm_called(void) +{ + return _PyCursesCheckFunction(curses_setupterm_called, "setupterm"); +} -static void -curses_destructor(PyObject *op) +static inline int +curses_capi_initscr_called(void) { - void *ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); - Py_DECREF(*(void **)ptr); - PyMem_Free(ptr); + return _PyCursesCheckFunction(curses_initscr_called, "initscr"); } -PyMODINIT_FUNC -PyInit__curses(void) +static inline int +curses_capi_start_color_called(void) { - PyObject *m, *d, *v, *c_api_object; + return _PyCursesCheckFunction(curses_start_color_called, "start_color"); +} - /* Initialize object type */ - if (PyType_Ready(&PyCursesWindow_Type) < 0) +static void * +curses_capi_new(cursesmodule_state *state) +{ + assert(state->window_type != NULL); + void **capi = (void **)PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); + if (capi == NULL) { + PyErr_NoMemory(); return NULL; + } + capi[0] = (void *)Py_NewRef(state->window_type); + capi[1] = curses_capi_setupterm_called; + capi[2] = curses_capi_initscr_called; + capi[3] = curses_capi_start_color_called; + return (void *)capi; +} - /* Create the module and add the functions */ - m = PyModule_Create(&_cursesmodule); - if (m == NULL) - return NULL; +static void +curses_capi_free(void *capi) +{ + assert(capi != NULL); + void **capi_ptr = (void **)capi; + // In free-threaded builds, capi_ptr[0] may have been already cleared + // by curses_capi_capsule_destructor(), hence the use of Py_XDECREF(). + Py_XDECREF(capi_ptr[0]); // decref curses window type + PyMem_Free(capi_ptr); +} - /* Add some symbolic constants to the module */ - d = PyModule_GetDict(m); - if (d == NULL) - return NULL; - ModDict = d; /* For PyCurses_InitScr to use later */ +/* Module C API Capsule */ - void **PyCurses_API = PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); - if (PyCurses_API == NULL) { - PyErr_NoMemory(); +static void +curses_capi_capsule_destructor(PyObject *op) +{ + void *capi = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + curses_capi_free(capi); +} + +static int +curses_capi_capsule_traverse(PyObject *op, visitproc visit, void *arg) +{ + void **capi_ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + assert(capi_ptr != NULL); + Py_VISIT(capi_ptr[0]); // visit curses window type + return 0; +} + +static int +curses_capi_capsule_clear(PyObject *op) +{ + void **capi_ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + assert(capi_ptr != NULL); + Py_CLEAR(capi_ptr[0]); // clear curses window type + return 0; +} + +static PyObject * +curses_capi_capsule_new(void *capi) +{ + PyObject *capsule = PyCapsule_New(capi, PyCurses_CAPSULE_NAME, + curses_capi_capsule_destructor); + if (capsule == NULL) { + return NULL; + } + if (_PyCapsule_SetTraverse(capsule, + curses_capi_capsule_traverse, + curses_capi_capsule_clear) < 0) + { + Py_DECREF(capsule); return NULL; } - /* Initialize the C API pointer array */ - PyCurses_API[0] = (void *)Py_NewRef(&PyCursesWindow_Type); - PyCurses_API[1] = (void *)func_PyCursesSetupTermCalled; - PyCurses_API[2] = (void *)func_PyCursesInitialised; - PyCurses_API[3] = (void *)func_PyCursesInitialisedColor; + return capsule; +} + +/* Module initialization and cleanup functions */ + +static int +cursesmodule_traverse(PyObject *mod, visitproc visit, void *arg) +{ + cursesmodule_state *state = get_cursesmodule_state(mod); + Py_VISIT(state->error); + Py_VISIT(state->window_type); + return 0; +} + +static int +cursesmodule_clear(PyObject *mod) +{ + cursesmodule_state *state = get_cursesmodule_state(mod); + Py_CLEAR(state->error); + Py_CLEAR(state->window_type); + return 0; +} + +static void +cursesmodule_free(void *mod) +{ + (void)cursesmodule_clear((PyObject *)mod); + curses_module_loaded = 0; // allow reloading once garbage-collected +} + +static int +cursesmodule_exec(PyObject *module) +{ + if (curses_module_loaded) { + PyErr_SetString(PyExc_ImportError, + "module 'curses' can only be loaded once per process"); + return -1; + } + curses_module_loaded = 1; + + cursesmodule_state *state = get_cursesmodule_state(module); + /* Initialize object type */ + state->window_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &PyCursesWindow_Type_spec, NULL); + if (state->window_type == NULL) { + return -1; + } + if (PyModule_AddType(module, state->window_type) < 0) { + return -1; + } + /* Add some symbolic constants to the module */ + PyObject *module_dict = PyModule_GetDict(module); + if (module_dict == NULL) { + return -1; + } + + /* Create the C API object */ + void *capi = curses_capi_new(state); + if (capi == NULL) { + return -1; + } /* Add a capsule for the C API */ - c_api_object = PyCapsule_New(PyCurses_API, PyCurses_CAPSULE_NAME, - curses_destructor); - if (c_api_object == NULL) { - Py_DECREF(PyCurses_API[0]); - PyMem_Free(PyCurses_API); - return NULL; + PyObject *capi_capsule = curses_capi_capsule_new(capi); + if (capi_capsule == NULL) { + curses_capi_free(capi); + return -1; } - if (PyDict_SetItemString(d, "_C_API", c_api_object) < 0) { - Py_DECREF(c_api_object); - return NULL; + int rc = PyDict_SetItemString(module_dict, "_C_API", capi_capsule); + Py_DECREF(capi_capsule); + if (rc < 0) { + return -1; } - Py_DECREF(c_api_object); /* For exception curses.error */ - PyCursesError = PyErr_NewException("_curses.error", NULL, NULL); - PyDict_SetItemString(d, "error", PyCursesError); + state->error = PyErr_NewException("_curses.error", NULL, NULL); + if (state->error == NULL) { + return -1; + } + rc = PyDict_SetItemString(module_dict, "error", state->error); + if (rc < 0) { + return -1; + } /* Make the version available */ - v = PyBytes_FromString(PyCursesVersion); - PyDict_SetItemString(d, "version", v); - PyDict_SetItemString(d, "__version__", v); - Py_DECREF(v); + PyObject *curses_version = PyBytes_FromString(PyCursesVersion); + if (curses_version == NULL) { + return -1; + } + rc = PyDict_SetItemString(module_dict, "version", curses_version); + if (rc < 0) { + Py_DECREF(curses_version); + return -1; + } + rc = PyDict_SetItemString(module_dict, "__version__", curses_version); + Py_CLEAR(curses_version); + if (rc < 0) { + return -1; + } #ifdef NCURSES_VERSION /* ncurses_version */ @@ -4791,17 +5055,33 @@ PyInit__curses(void) version_type = _PyStructSequence_NewType(&ncurses_version_desc, Py_TPFLAGS_DISALLOW_INSTANTIATION); if (version_type == NULL) { - return NULL; + return -1; } - v = make_ncurses_version(version_type); + PyObject *ncurses_version = make_ncurses_version(version_type); Py_DECREF(version_type); - if (v == NULL) { - return NULL; + if (ncurses_version == NULL) { + return -1; + } + rc = PyDict_SetItemString(module_dict, "ncurses_version", ncurses_version); + Py_CLEAR(ncurses_version); + if (rc < 0) { + return -1; } - PyDict_SetItemString(d, "ncurses_version", v); - Py_DECREF(v); #endif /* NCURSES_VERSION */ +#define SetDictInt(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return -1; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return -1; \ + } \ + } while (0) + SetDictInt("ERR", ERR); SetDictInt("OK", OK); @@ -4897,43 +5177,71 @@ PyInit__curses(void) SetDictInt("REPORT_MOUSE_POSITION", REPORT_MOUSE_POSITION); #endif /* Now set everything up for KEY_ variables */ - { - int key; - char *key_n; - char *key_n2; - for (key=KEY_MIN;key < KEY_MAX; key++) { - key_n = (char *)keyname(key); - if (key_n == NULL || strcmp(key_n,"UNKNOWN KEY")==0) - continue; - if (strncmp(key_n,"KEY_F(",6)==0) { - char *p1, *p2; - key_n2 = PyMem_Malloc(strlen(key_n)+1); - if (!key_n2) { - PyErr_NoMemory(); - break; - } - p1 = key_n; - p2 = key_n2; - while (*p1) { - if (*p1 != '(' && *p1 != ')') { - *p2 = *p1; - p2++; - } - p1++; + for (int keycode = KEY_MIN; keycode < KEY_MAX; keycode++) { + const char *key_name = keyname(keycode); + if (key_name == NULL || strcmp(key_name, "UNKNOWN KEY") == 0) { + continue; + } + if (strncmp(key_name, "KEY_F(", 6) == 0) { + char *fn_key_name = PyMem_Malloc(strlen(key_name) + 1); + if (!fn_key_name) { + PyErr_NoMemory(); + return -1; + } + const char *p1 = key_name; + char *p2 = fn_key_name; + while (*p1) { + if (*p1 != '(' && *p1 != ')') { + *p2 = *p1; + p2++; } - *p2 = (char)0; - } else - key_n2 = key_n; - SetDictInt(key_n2,key); - if (key_n2 != key_n) - PyMem_Free(key_n2); + p1++; + } + *p2 = (char)0; + PyObject *p_keycode = PyLong_FromLong((long)keycode); + if (p_keycode == NULL) { + PyMem_Free(fn_key_name); + return -1; + } + int rc = PyDict_SetItemString(module_dict, fn_key_name, p_keycode); + Py_DECREF(p_keycode); + PyMem_Free(fn_key_name); + if (rc < 0) { + return -1; + } + } + else { + SetDictInt(key_name, keycode); } - SetDictInt("KEY_MIN", KEY_MIN); - SetDictInt("KEY_MAX", KEY_MAX); } + SetDictInt("KEY_MIN", KEY_MIN); + SetDictInt("KEY_MAX", KEY_MAX); +#undef SetDictInt + return 0; +} - if (PyModule_AddType(m, &PyCursesWindow_Type) < 0) { - return NULL; - } - return m; +/* Initialization function for the module */ + +static PyModuleDef_Slot cursesmodule_slots[] = { + {Py_mod_exec, cursesmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +static struct PyModuleDef cursesmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_curses", + .m_size = sizeof(cursesmodule_state), + .m_methods = cursesmodule_methods, + .m_slots = cursesmodule_slots, + .m_traverse = cursesmodule_traverse, + .m_clear = cursesmodule_clear, + .m_free = cursesmodule_free +}; + +PyMODINIT_FUNC +PyInit__curses(void) +{ + return PyModuleDef_Init(&cursesmodule); } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9b8e0a719d9048..b1102984cb5e9e 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1,6 +1,4 @@ -/* C implementation for the date/time type documented at - * https://www.zope.dev/Members/fdrake/DateTimeWiki/FrontPage - */ +/* C implementation of the datetime module */ /* bpo-35081: Defining this prevents including the C API capsule; * internal versions of the Py*_Check macros which do not require @@ -14,6 +12,8 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() +#include "pycore_time.h" // _PyTime_ObjectToTime_t() + #include "datetime.h" @@ -23,24 +23,20 @@ # include /* struct timeval */ #endif -#define PyDate_Check(op) PyObject_TypeCheck(op, &PyDateTime_DateType) -#define PyDate_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DateType) - -#define PyDateTime_Check(op) PyObject_TypeCheck(op, &PyDateTime_DateTimeType) -#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DateTimeType) - -#define PyTime_Check(op) PyObject_TypeCheck(op, &PyDateTime_TimeType) -#define PyTime_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_TimeType) -#define PyDelta_Check(op) PyObject_TypeCheck(op, &PyDateTime_DeltaType) -#define PyDelta_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DeltaType) - -#define PyTZInfo_Check(op) PyObject_TypeCheck(op, &PyDateTime_TZInfoType) -#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_TZInfoType) +/* forward declarations */ +static PyTypeObject PyDateTime_DateType; +static PyTypeObject PyDateTime_DateTimeType; +static PyTypeObject PyDateTime_TimeType; +static PyTypeObject PyDateTime_DeltaType; +static PyTypeObject PyDateTime_TZInfoType; +static PyTypeObject PyDateTime_TimeZoneType; -#define PyTimezone_Check(op) PyObject_TypeCheck(op, &PyDateTime_TimeZoneType) typedef struct { + /* Module heap types. */ + PyTypeObject *isocalendar_date_type; + /* Conversion factors. */ PyObject *us_per_ms; // 1_000 PyObject *us_per_second; // 1_000_000 @@ -50,16 +46,192 @@ typedef struct { PyObject *us_per_week; // 1e6 * 3600 * 24 * 7 as Python int PyObject *seconds_per_day; // 3600 * 24 as Python int - /* The interned UTC timezone instance */ - PyObject *utc; - /* The interned Unix epoch datetime instance */ PyObject *epoch; } datetime_state; -static datetime_state _datetime_global_state; +/* The module has a fixed number of static objects, due to being exposed + * through the datetime C-API. There are five types exposed directly, + * one type exposed indirectly, and one singleton constant (UTC). + * + * Each of these objects is hidden behind a macro in the same way as + * the per-module objects stored in module state. The macros for the + * static objects don't need to be passed a state, but the consistency + * of doing so is more clear. We use a dedicated noop macro, NO_STATE, + * to make the special case obvious. */ + +#define NO_STATE NULL + +#define DATE_TYPE(st) &PyDateTime_DateType +#define DATETIME_TYPE(st) &PyDateTime_DateTimeType +#define TIME_TYPE(st) &PyDateTime_TimeType +#define DELTA_TYPE(st) &PyDateTime_DeltaType +#define TZINFO_TYPE(st) &PyDateTime_TZInfoType +#define TIMEZONE_TYPE(st) &PyDateTime_TimeZoneType +#define ISOCALENDAR_DATE_TYPE(st) st->isocalendar_date_type + +#define PyDate_Check(op) PyObject_TypeCheck(op, DATE_TYPE(NO_STATE)) +#define PyDate_CheckExact(op) Py_IS_TYPE(op, DATE_TYPE(NO_STATE)) + +#define PyDateTime_Check(op) PyObject_TypeCheck(op, DATETIME_TYPE(NO_STATE)) +#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, DATETIME_TYPE(NO_STATE)) + +#define PyTime_Check(op) PyObject_TypeCheck(op, TIME_TYPE(NO_STATE)) +#define PyTime_CheckExact(op) Py_IS_TYPE(op, TIME_TYPE(NO_STATE)) + +#define PyDelta_Check(op) PyObject_TypeCheck(op, DELTA_TYPE(NO_STATE)) +#define PyDelta_CheckExact(op) Py_IS_TYPE(op, DELTA_TYPE(NO_STATE)) + +#define PyTZInfo_Check(op) PyObject_TypeCheck(op, TZINFO_TYPE(NO_STATE)) +#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, TZINFO_TYPE(NO_STATE)) + +#define PyTimezone_Check(op) PyObject_TypeCheck(op, TIMEZONE_TYPE(NO_STATE)) + +#define CONST_US_PER_MS(st) st->us_per_ms +#define CONST_US_PER_SECOND(st) st->us_per_second +#define CONST_US_PER_MINUTE(st) st->us_per_minute +#define CONST_US_PER_HOUR(st) st->us_per_hour +#define CONST_US_PER_DAY(st) st->us_per_day +#define CONST_US_PER_WEEK(st) st->us_per_week +#define CONST_SEC_PER_DAY(st) st->seconds_per_day +#define CONST_EPOCH(st) st->epoch +#define CONST_UTC(st) ((PyObject *)&utc_timezone) + +static datetime_state * +get_module_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (datetime_state *)state; +} + + +#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) + +static PyObject * +get_current_module(PyInterpreterState *interp, int *p_reloading) +{ + PyObject *mod = NULL; + int reloading = 0; + + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + goto error; + } + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + goto error; + } + if (ref != NULL) { + reloading = 1; + if (ref != Py_None) { + (void)PyWeakref_GetRef(ref, &mod); + if (mod == Py_None) { + Py_CLEAR(mod); + } + Py_DECREF(ref); + } + } + if (p_reloading != NULL) { + *p_reloading = reloading; + } + return mod; + +error: + assert(PyErr_Occurred()); + return NULL; +} + +static PyModuleDef datetimemodule; + +static datetime_state * +_get_current_state(PyObject **p_mod) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + PyObject *mod = get_current_module(interp, NULL); + if (mod == NULL) { + assert(!PyErr_Occurred()); + if (PyErr_Occurred()) { + return NULL; + } + /* The static types can outlive the module, + * so we must re-import the module. */ + mod = PyImport_ImportModule("_datetime"); + if (mod == NULL) { + return NULL; + } + } + datetime_state *st = get_module_state(mod); + *p_mod = mod; + return st; +} + +#define GET_CURRENT_STATE(MOD_VAR) \ + _get_current_state(&MOD_VAR) +#define RELEASE_CURRENT_STATE(ST_VAR, MOD_VAR) \ + Py_DECREF(MOD_VAR) + +static int +set_current_module(PyInterpreterState *interp, PyObject *mod) +{ + assert(mod != NULL); + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + return -1; + } + PyObject *ref = PyWeakref_NewRef(mod, NULL); + if (ref == NULL) { + return -1; + } + int rc = PyDict_SetItem(dict, INTERP_KEY, ref); + Py_DECREF(ref); + return rc; +} + +static void +clear_current_module(PyInterpreterState *interp, PyObject *expected) +{ + PyObject *exc = PyErr_GetRaisedException(); + + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + goto error; + } + + if (expected != NULL) { + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + goto error; + } + if (ref != NULL) { + PyObject *current = NULL; + int rc = PyWeakref_GetRef(ref, ¤t); + /* We only need "current" for pointer comparison. */ + Py_XDECREF(current); + Py_DECREF(ref); + if (rc < 0) { + goto error; + } + if (current != expected) { + goto finally; + } + } + } + + /* We use None to identify that the module was previously loaded. */ + if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { + goto error; + } + + goto finally; + +error: + PyErr_WriteUnraisable(NULL); + +finally: + PyErr_SetRaisedException(exc); +} -#define STATIC_STATE() (&_datetime_global_state) /* We require that C int be at least 32 bits, and use int virtually * everywhere. In just a few cases we use a temp long, where a Python @@ -140,25 +312,16 @@ static datetime_state _datetime_global_state; */ #define MONTH_IS_SANE(M) ((unsigned int)(M) - 1 < 12) -/* Forward declarations. */ -static PyTypeObject PyDateTime_DateType; -static PyTypeObject PyDateTime_DateTimeType; -static PyTypeObject PyDateTime_DeltaType; -static PyTypeObject PyDateTime_IsoCalendarDateType; -static PyTypeObject PyDateTime_TimeType; -static PyTypeObject PyDateTime_TZInfoType; -static PyTypeObject PyDateTime_TimeZoneType; - static int check_tzinfo_subclass(PyObject *p); /*[clinic input] module datetime -class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" -class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" -class datetime.time "PyDateTime_Time *" "&PyDateTime_TimeType" -class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" +class datetime.datetime "PyDateTime_DateTime *" "get_datetime_state()->datetime_type" +class datetime.date "PyDateTime_Date *" "get_datetime_state()->date_type" +class datetime.time "PyDateTime_Time *" "get_datetime_state()->time_type" +class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "get_datetime_state()->isocalendar_date_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6f65a48dd22fa40f]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c8f3d834a860d50a]*/ #include "clinic/_datetimemodule.c.h" @@ -414,6 +577,10 @@ iso_week1_monday(int year) static int iso_to_ymd(const int iso_year, const int iso_week, const int iso_day, int *year, int *month, int *day) { + // Year is bounded to 0 < year < 10000 because 9999-12-31 is (9999, 52, 5) + if (iso_year < MINYEAR || iso_year > MAXYEAR) { + return -4; + } if (iso_week <= 0 || iso_week >= 53) { int out_of_range = 1; if (iso_week == 53) { @@ -760,7 +927,7 @@ parse_isoformat_date(const char *dtstr, const size_t len, int *year, int *month, * -2: Inconsistent date separator usage * -3: Failed to parse ISO week. * -4: Failed to parse ISO day. - * -5, -6: Failure in iso_to_ymd + * -5, -6, -7: Failure in iso_to_ymd */ const char *p = dtstr; p = parse_digits(p, year, 4); @@ -851,6 +1018,9 @@ parse_hh_mm_ss_ff(const char *tstr, const char *tstr_end, int *hour, continue; } else if (c == '.' || c == ',') { + if (i < 2) { + return -3; // Decimal mark on hour or minute + } break; } else if (!has_separator) { --p; @@ -973,7 +1143,7 @@ new_date_ex(int year, int month, int day, PyTypeObject *type) } #define new_date(year, month, day) \ - new_date_ex(year, month, day, &PyDateTime_DateType) + new_date_ex(year, month, day, DATE_TYPE(NO_STATE)) // Forward declaration static PyObject * @@ -985,10 +1155,10 @@ new_date_subclass_ex(int year, int month, int day, PyObject *cls) { PyObject *result; // We have "fast path" constructors for two subclasses: date and datetime - if ((PyTypeObject *)cls == &PyDateTime_DateType) { + if ((PyTypeObject *)cls == DATE_TYPE(NO_STATE)) { result = new_date_ex(year, month, day, (PyTypeObject *)cls); } - else if ((PyTypeObject *)cls == &PyDateTime_DateTimeType) { + else if ((PyTypeObject *)cls == DATETIME_TYPE(NO_STATE)) { result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, (PyTypeObject *)cls); } @@ -1042,29 +1212,57 @@ new_datetime_ex(int year, int month, int day, int hour, int minute, } #define new_datetime(y, m, d, hh, mm, ss, us, tzinfo, fold) \ - new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, \ - &PyDateTime_DateTimeType) + new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, DATETIME_TYPE(NO_STATE)) + +static PyObject * +call_subclass_fold(PyObject *cls, int fold, const char *format, ...) +{ + PyObject *kwargs = NULL, *res = NULL; + va_list va; + + va_start(va, format); + PyObject *args = Py_VaBuildValue(format, va); + va_end(va); + if (args == NULL) { + return NULL; + } + if (fold) { + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto Done; + } + PyObject *obj = PyLong_FromLong(fold); + if (obj == NULL) { + goto Done; + } + int err = PyDict_SetItemString(kwargs, "fold", obj); + Py_DECREF(obj); + if (err < 0) { + goto Done; + } + } + res = PyObject_Call(cls, args, kwargs); +Done: + Py_DECREF(args); + Py_XDECREF(kwargs); + return res; +} static PyObject * new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - int fold, PyObject *cls) { + int fold, PyObject *cls) +{ PyObject* dt; - if ((PyTypeObject*)cls == &PyDateTime_DateTimeType) { + if ((PyTypeObject*)cls == DATETIME_TYPE(NO_STATE)) { // Use the fast path constructor dt = new_datetime(year, month, day, hour, minute, second, usecond, tzinfo, fold); - } else { + } + else { // Subclass - dt = PyObject_CallFunction(cls, "iiiiiiiO", - year, - month, - day, - hour, - minute, - second, - usecond, - tzinfo); + dt = call_subclass_fold(cls, fold, "iiiiiiiO", year, month, day, + hour, minute, second, usecond, tzinfo); } return dt; @@ -1117,8 +1315,28 @@ new_time_ex(int hour, int minute, int second, int usecond, return new_time_ex2(hour, minute, second, usecond, tzinfo, 0, type); } -#define new_time(hh, mm, ss, us, tzinfo, fold) \ - new_time_ex2(hh, mm, ss, us, tzinfo, fold, &PyDateTime_TimeType) +#define new_time(hh, mm, ss, us, tzinfo, fold) \ + new_time_ex2(hh, mm, ss, us, tzinfo, fold, TIME_TYPE(NO_STATE)) + +static PyObject * +new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, + PyObject *tzinfo, int fold, PyObject *cls) +{ + PyObject *t; + if ((PyTypeObject*)cls == TIME_TYPE(NO_STATE)) { + // Use the fast path constructor + t = new_time(hour, minute, second, usecond, tzinfo, fold); + } + else { + // Subclass + t = call_subclass_fold(cls, fold, "iiiiO", hour, minute, second, + usecond, tzinfo); + } + + return t; +} + +static PyDateTime_Delta * look_up_delta(int, int, int, PyTypeObject *); /* Create a timedelta instance. Normalize the members iff normalize is * true. Passing false is a speed optimization, if you know for sure @@ -1140,6 +1358,12 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, if (check_delta_day_range(days) < 0) return NULL; + self = look_up_delta(days, seconds, microseconds, type); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_Delta *) (type->tp_alloc(type, 0)); if (self != NULL) { self->hashcode = -1; @@ -1151,7 +1375,7 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, } #define new_delta(d, s, us, normalize) \ - new_delta_ex(d, s, us, normalize, &PyDateTime_DeltaType) + new_delta_ex(d, s, us, normalize, DELTA_TYPE(NO_STATE)) typedef struct @@ -1161,6 +1385,8 @@ typedef struct PyObject *name; } PyDateTime_TimeZone; +static PyDateTime_TimeZone * look_up_timezone(PyObject *offset, PyObject *name); + /* Create new timezone instance checking offset range. This function does not check the name argument. Caller must assure that offset is a timedelta instance and name is either NULL @@ -1169,12 +1395,18 @@ static PyObject * create_timezone(PyObject *offset, PyObject *name) { PyDateTime_TimeZone *self; - PyTypeObject *type = &PyDateTime_TimeZoneType; + PyTypeObject *type = TIMEZONE_TYPE(NO_STATE); assert(offset != NULL); assert(PyDelta_Check(offset)); assert(name == NULL || PyUnicode_Check(name)); + self = look_up_timezone(offset, name); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_TimeZone *)(type->tp_alloc(type, 0)); if (self == NULL) { return NULL; @@ -1185,6 +1417,7 @@ create_timezone(PyObject *offset, PyObject *name) } static int delta_bool(PyDateTime_Delta *self); +static PyDateTime_TimeZone utc_timezone; static PyObject * new_timezone(PyObject *offset, PyObject *name) @@ -1194,8 +1427,7 @@ new_timezone(PyObject *offset, PyObject *name) assert(name == NULL || PyUnicode_Check(name)); if (name == NULL && delta_bool((PyDateTime_Delta *)offset) == 0) { - datetime_state *st = STATIC_STATE(); - return Py_NewRef(st->utc); + return Py_NewRef(CONST_UTC(NO_STATE)); } if ((GET_TD_DAYS(offset) == -1 && GET_TD_SECONDS(offset) == 0 && @@ -1408,8 +1640,7 @@ tzinfo_from_isoformat_results(int rv, int tzoffset, int tz_useconds) if (rv == 1) { // Create a timezone from offset in seconds (0 returns UTC) if (tzoffset == 0) { - datetime_state *st = STATIC_STATE(); - return Py_NewRef(st->utc); + return Py_NewRef(CONST_UTC(NO_STATE)); } PyObject *delta = new_delta(0, tzoffset, tz_useconds, 1); @@ -1516,7 +1747,7 @@ make_somezreplacement(PyObject *object, char *sep, PyObject *tzinfoarg) PyObject *tzinfo = get_tzinfo_member(object); if (tzinfo == Py_None || tzinfo == NULL) { - return PyBytes_FromStringAndSize(NULL, 0); + return PyUnicode_FromStringAndSize(NULL, 0); } assert(tzinfoarg != NULL); @@ -1527,7 +1758,7 @@ make_somezreplacement(PyObject *object, char *sep, PyObject *tzinfoarg) tzinfoarg) < 0) return NULL; - return PyBytes_FromStringAndSize(buf, strlen(buf)); + return PyUnicode_FromString(buf); } static PyObject * @@ -1535,7 +1766,7 @@ make_Zreplacement(PyObject *object, PyObject *tzinfoarg) { PyObject *temp; PyObject *tzinfo = get_tzinfo_member(object); - PyObject *Zreplacement = PyUnicode_FromStringAndSize(NULL, 0); + PyObject *Zreplacement = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (Zreplacement == NULL) return NULL; @@ -1584,7 +1815,7 @@ make_freplacement(PyObject *object) else sprintf(freplacement, "%06d", 0); - return PyBytes_FromStringAndSize(freplacement, strlen(freplacement)); + return PyUnicode_FromString(freplacement); } /* I sure don't want to reproduce the strftime code from the time module, @@ -1605,79 +1836,60 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, PyObject *Zreplacement = NULL; /* py string, replacement for %Z */ PyObject *freplacement = NULL; /* py string, replacement for %f */ - const char *pin; /* pointer to next char in input format */ - Py_ssize_t flen; /* length of input format */ - char ch; /* next char in input format */ - - PyObject *newfmt = NULL; /* py string, the output format */ - char *pnew; /* pointer to available byte in output format */ - size_t totalnew; /* number bytes total in output format buffer, - exclusive of trailing \0 */ - size_t usednew; /* number bytes used so far in output format buffer */ - - const char *ptoappend; /* ptr to string to append to output buffer */ - Py_ssize_t ntoappend; /* # of bytes to append to output buffer */ - assert(object && format && timetuple); assert(PyUnicode_Check(format)); - /* Convert the input format to a C string and size */ - pin = PyUnicode_AsUTF8AndSize(format, &flen); - if (!pin) + + PyObject *strftime = _PyImport_GetModuleAttrString("time", "strftime"); + if (strftime == NULL) { return NULL; + } /* Scan the input format, looking for %z/%Z/%f escapes, building * a new format. Since computing the replacements for those codes * is expensive, don't unless they're actually used. */ - if (flen > INT_MAX - 1) { - PyErr_NoMemory(); - goto Done; - } - - totalnew = flen + 1; /* realistic if no %z/%Z */ - newfmt = PyBytes_FromStringAndSize(NULL, totalnew); - if (newfmt == NULL) goto Done; - pnew = PyBytes_AsString(newfmt); - usednew = 0; - while ((ch = *pin++) != '\0') { - if (ch != '%') { - ptoappend = pin - 1; - ntoappend = 1; + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + writer.overallocate = 1; + + Py_ssize_t flen = PyUnicode_GET_LENGTH(format); + Py_ssize_t i = 0; + Py_ssize_t start = 0; + Py_ssize_t end = 0; + while (i != flen) { + i = PyUnicode_FindChar(format, '%', i, flen, 1); + if (i < 0) { + assert(!PyErr_Occurred()); + break; } - else if ((ch = *pin++) == '\0') { - /* Null byte follows %, copy only '%'. - * - * Back the pin up one char so that we catch the null check - * the next time through the loop.*/ - pin--; - ptoappend = pin - 1; - ntoappend = 1; + end = i; + i++; + if (i == flen) { + break; } + Py_UCS4 ch = PyUnicode_READ_CHAR(format, i); + i++; /* A % has been seen and ch is the character after it. */ - else if (ch == 'z') { + PyObject *replacement = NULL; + if (ch == 'z') { /* %z -> +HHMM */ if (zreplacement == NULL) { zreplacement = make_somezreplacement(object, "", tzinfoarg); if (zreplacement == NULL) - goto Done; + goto Error; } - assert(zreplacement != NULL); - assert(PyBytes_Check(zreplacement)); - ptoappend = PyBytes_AS_STRING(zreplacement); - ntoappend = PyBytes_GET_SIZE(zreplacement); + replacement = zreplacement; } - else if (ch == ':' && *pin == 'z' && pin++) { + else if (ch == ':' && i < flen && PyUnicode_READ_CHAR(format, i) == 'z') { /* %:z -> +HH:MM */ + i++; if (colonzreplacement == NULL) { colonzreplacement = make_somezreplacement(object, ":", tzinfoarg); if (colonzreplacement == NULL) - goto Done; + goto Error; } - assert(colonzreplacement != NULL); - assert(PyBytes_Check(colonzreplacement)); - ptoappend = PyBytes_AS_STRING(colonzreplacement); - ntoappend = PyBytes_GET_SIZE(colonzreplacement); + replacement = colonzreplacement; } else if (ch == 'Z') { /* format tzname */ @@ -1685,79 +1897,124 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, Zreplacement = make_Zreplacement(object, tzinfoarg); if (Zreplacement == NULL) - goto Done; + goto Error; } - assert(Zreplacement != NULL); - assert(PyUnicode_Check(Zreplacement)); - ptoappend = PyUnicode_AsUTF8AndSize(Zreplacement, - &ntoappend); - if (ptoappend == NULL) - goto Done; + replacement = Zreplacement; } else if (ch == 'f') { /* format microseconds */ if (freplacement == NULL) { freplacement = make_freplacement(object); if (freplacement == NULL) - goto Done; + goto Error; } - assert(freplacement != NULL); - assert(PyBytes_Check(freplacement)); - ptoappend = PyBytes_AS_STRING(freplacement); - ntoappend = PyBytes_GET_SIZE(freplacement); + replacement = freplacement; } +#ifdef Py_NORMALIZE_CENTURY + else if (ch == 'Y' || ch == 'G' +#ifdef Py_STRFTIME_C99_SUPPORT + || ch == 'F' || ch == 'C' +#endif + ) { + /* 0-pad year with century as necessary */ + PyObject *item = PySequence_GetItem(timetuple, 0); + if (item == NULL) { + goto Error; + } + long year_long = PyLong_AsLong(item); + Py_DECREF(item); + if (year_long == -1 && PyErr_Occurred()) { + goto Error; + } + /* Note that datetime(1000, 1, 1).strftime('%G') == '1000' so year + 1000 for %G can go on the fast path. */ + if (year_long >= 1000) { + continue; + } + if (ch == 'G') { + PyObject *year_str = PyObject_CallFunction(strftime, "sO", + "%G", timetuple); + if (year_str == NULL) { + goto Error; + } + PyObject *year = PyNumber_Long(year_str); + Py_DECREF(year_str); + if (year == NULL) { + goto Error; + } + year_long = PyLong_AsLong(year); + Py_DECREF(year); + if (year_long == -1 && PyErr_Occurred()) { + goto Error; + } + } + /* Buffer of maximum size of formatted year permitted by long. + * +6 to accommodate dashes, 2-digit month and day for %F. */ + char buf[SIZEOF_LONG * 5 / 2 + 2 + 6]; + Py_ssize_t n = PyOS_snprintf(buf, sizeof(buf), +#ifdef Py_STRFTIME_C99_SUPPORT + ch == 'F' ? "%04ld-%%m-%%d" : +#endif + "%04ld", year_long); +#ifdef Py_STRFTIME_C99_SUPPORT + if (ch == 'C') { + n -= 2; + } +#endif + if (_PyUnicodeWriter_WriteSubstring(&writer, format, start, end) < 0) { + goto Error; + } + start = i; + if (_PyUnicodeWriter_WriteASCIIString(&writer, buf, n) < 0) { + goto Error; + } + continue; + } +#endif else { /* percent followed by something else */ - ptoappend = pin - 2; - ntoappend = 2; - } - - /* Append the ntoappend chars starting at ptoappend to - * the new format. - */ - if (ntoappend == 0) continue; - assert(ptoappend != NULL); - assert(ntoappend > 0); - while (usednew + ntoappend > totalnew) { - if (totalnew > (PY_SSIZE_T_MAX >> 1)) { /* overflow */ - PyErr_NoMemory(); - goto Done; - } - totalnew <<= 1; - if (_PyBytes_Resize(&newfmt, totalnew) < 0) - goto Done; - pnew = PyBytes_AsString(newfmt) + usednew; } - memcpy(pnew, ptoappend, ntoappend); - pnew += ntoappend; - usednew += ntoappend; - assert(usednew <= totalnew); + assert(replacement != NULL); + assert(PyUnicode_Check(replacement)); + if (_PyUnicodeWriter_WriteSubstring(&writer, format, start, end) < 0) { + goto Error; + } + start = i; + if (_PyUnicodeWriter_WriteStr(&writer, replacement) < 0) { + goto Error; + } } /* end while() */ - if (_PyBytes_Resize(&newfmt, usednew) < 0) - goto Done; - { - PyObject *format; - PyObject *strftime = _PyImport_GetModuleAttrString("time", "strftime"); - - if (strftime == NULL) + PyObject *newformat; + if (start == 0) { + _PyUnicodeWriter_Dealloc(&writer); + newformat = Py_NewRef(format); + } + else { + if (_PyUnicodeWriter_WriteSubstring(&writer, format, start, flen) < 0) { + goto Error; + } + newformat = _PyUnicodeWriter_Finish(&writer); + if (newformat == NULL) { goto Done; - format = PyUnicode_FromString(PyBytes_AS_STRING(newfmt)); - if (format != NULL) { - result = PyObject_CallFunctionObjArgs(strftime, - format, timetuple, NULL); - Py_DECREF(format); } - Py_DECREF(strftime); } + result = PyObject_CallFunctionObjArgs(strftime, + newformat, timetuple, NULL); + Py_DECREF(newformat); + Done: Py_XDECREF(freplacement); Py_XDECREF(zreplacement); Py_XDECREF(colonzreplacement); Py_XDECREF(Zreplacement); - Py_XDECREF(newfmt); + Py_XDECREF(strftime); return result; + + Error: + _PyUnicodeWriter_Dealloc(&writer); + goto Done; } /* --------------------------------------------------------------------------- @@ -1816,16 +2073,6 @@ diff_to_bool(int diff, int op) Py_RETURN_RICHCOMPARE(diff, 0, op); } -/* Raises a "can't compare" TypeError and returns NULL. */ -static PyObject * -cmperror(PyObject *a, PyObject *b) -{ - PyErr_Format(PyExc_TypeError, - "can't compare %s to %s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); - return NULL; -} - /* --------------------------------------------------------------------------- * Class implementations. */ @@ -1848,11 +2095,13 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) goto Done; - datetime_state *st = STATIC_STATE(); - x2 = PyNumber_Multiply(x1, st->seconds_per_day); /* days in seconds */ + x2 = PyNumber_Multiply(x1, CONST_SEC_PER_DAY(st)); /* days in seconds */ if (x2 == NULL) goto Done; Py_SETREF(x1, NULL); @@ -1869,7 +2118,7 @@ delta_to_microseconds(PyDateTime_Delta *self) /* x1 = */ x2 = NULL; /* x3 has days+seconds in seconds */ - x1 = PyNumber_Multiply(x3, st->us_per_second); /* us */ + x1 = PyNumber_Multiply(x3, CONST_US_PER_SECOND(st)); /* us */ if (x1 == NULL) goto Done; Py_SETREF(x3, NULL); @@ -1885,6 +2134,7 @@ delta_to_microseconds(PyDateTime_Delta *self) Py_XDECREF(x1); Py_XDECREF(x2); Py_XDECREF(x3); + RELEASE_CURRENT_STATE(st, current_mod); return result; } @@ -1924,8 +2174,10 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - datetime_state *st = STATIC_STATE(); - tuple = checked_divmod(pyus, st->us_per_second); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { goto Done; } @@ -1943,7 +2195,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) num = Py_NewRef(PyTuple_GET_ITEM(tuple, 0)); /* leftover seconds */ Py_DECREF(tuple); - tuple = checked_divmod(num, st->seconds_per_day); + tuple = checked_divmod(num, CONST_SEC_PER_DAY(st)); if (tuple == NULL) goto Done; Py_DECREF(num); @@ -1968,6 +2220,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) Done: Py_XDECREF(tuple); Py_XDECREF(num); + RELEASE_CURRENT_STATE(st, current_mod); return result; BadDivmod: @@ -1977,7 +2230,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) } #define microseconds_to_delta(pymicros) \ - microseconds_to_delta_ex(pymicros, &PyDateTime_DeltaType) + microseconds_to_delta_ex(pymicros, DELTA_TYPE(NO_STATE)) static PyObject * multiply_int_timedelta(PyObject *intobj, PyDateTime_Delta *delta) @@ -2505,6 +2758,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *self = NULL; + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + /* Argument objects. */ PyObject *day = NULL; PyObject *second = NULL; @@ -2543,29 +2799,28 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) y = accum("microseconds", x, us, _PyLong_GetOne(), &leftover_us); CLEANUP; } - datetime_state *st = STATIC_STATE(); if (ms) { - y = accum("milliseconds", x, ms, st->us_per_ms, &leftover_us); + y = accum("milliseconds", x, ms, CONST_US_PER_MS(st), &leftover_us); CLEANUP; } if (second) { - y = accum("seconds", x, second, st->us_per_second, &leftover_us); + y = accum("seconds", x, second, CONST_US_PER_SECOND(st), &leftover_us); CLEANUP; } if (minute) { - y = accum("minutes", x, minute, st->us_per_minute, &leftover_us); + y = accum("minutes", x, minute, CONST_US_PER_MINUTE(st), &leftover_us); CLEANUP; } if (hour) { - y = accum("hours", x, hour, st->us_per_hour, &leftover_us); + y = accum("hours", x, hour, CONST_US_PER_HOUR(st), &leftover_us); CLEANUP; } if (day) { - y = accum("days", x, day, st->us_per_day, &leftover_us); + y = accum("days", x, day, CONST_US_PER_DAY(st), &leftover_us); CLEANUP; } if (week) { - y = accum("weeks", x, week, st->us_per_week, &leftover_us); + y = accum("weeks", x, week, CONST_US_PER_WEEK(st), &leftover_us); CLEANUP; } if (leftover_us) { @@ -2607,7 +2862,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) self = microseconds_to_delta_ex(x, type); Py_DECREF(x); + Done: + RELEASE_CURRENT_STATE(st, current_mod); return self; #undef CLEANUP @@ -2624,7 +2881,7 @@ delta_bool(PyDateTime_Delta *self) static PyObject * delta_repr(PyDateTime_Delta *self) { - PyObject *args = PyUnicode_FromString(""); + PyObject *args = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (args == NULL) { return NULL; @@ -2720,9 +2977,12 @@ delta_total_seconds(PyObject *self, PyObject *Py_UNUSED(ignored)) if (total_microseconds == NULL) return NULL; - datetime_state *st = STATIC_STATE(); - total_seconds = PyNumber_TrueDivide(total_microseconds, st->us_per_second); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); + RELEASE_CURRENT_STATE(st, current_mod); Py_DECREF(total_microseconds); return total_seconds; } @@ -2843,6 +3103,25 @@ static PyTypeObject PyDateTime_DeltaType = { 0, /* tp_free */ }; +// XXX Can we make this const? +static PyDateTime_Delta zero_delta = { + PyObject_HEAD_INIT(&PyDateTime_DeltaType) + /* Letting this be set lazily is a benign race. */ + .hashcode = -1, +}; + +static PyDateTime_Delta * +look_up_delta(int days, int seconds, int microseconds, PyTypeObject *type) +{ + if (days == 0 && seconds == 0 && microseconds == 0 + && type == Py_TYPE(&zero_delta)) + { + return &zero_delta; + } + return NULL; +} + + /* * PyDateTime_Date implementation. */ @@ -3104,15 +3383,13 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return NULL; } - // Year is bounded to 0 < year < 10000 because 9999-12-31 is (9999, 52, 5) - if (year < MINYEAR || year > MAXYEAR) { - PyErr_Format(PyExc_ValueError, "Year is out of range: %d", year); - return NULL; - } - int month; int rv = iso_to_ymd(year, week, day, &year, &month, &day); + if (rv == -4) { + PyErr_Format(PyExc_ValueError, "Year is out of range: %d", year); + return NULL; + } if (rv == -2) { PyErr_Format(PyExc_ValueError, "Invalid week: %d", week); @@ -3128,6 +3405,27 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return new_date_subclass_ex(year, month, day, cls); } +/* Return new date from _strptime.strptime_datetime_date(). */ +static PyObject * +date_strptime(PyObject *cls, PyObject *args) +{ + PyObject *string, *format, *result; + + if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { + return NULL; + } + + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); + if (module == NULL) { + return NULL; + } + result = PyObject_CallMethodObjArgs(module, + &_Py_ID(_strptime_datetime_date), cls, + string, format, NULL); + Py_DECREF(module); + return result; +} + /* * Date arithmetic. @@ -3375,17 +3673,40 @@ static PyMethodDef iso_calendar_date_methods[] = { {NULL, NULL}, }; -static PyTypeObject PyDateTime_IsoCalendarDateType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "datetime.IsoCalendarDate", - .tp_basicsize = sizeof(PyDateTime_IsoCalendarDate), - .tp_repr = (reprfunc) iso_calendar_date_repr, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = iso_calendar_date__doc__, - .tp_methods = iso_calendar_date_methods, - .tp_getset = iso_calendar_date_getset, - // .tp_base = &PyTuple_Type, // filled in PyInit__datetime - .tp_new = iso_calendar_date_new, +static int +iso_calendar_date_traverse(PyDateTime_IsoCalendarDate *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return PyTuple_Type.tp_traverse((PyObject *)self, visit, arg); +} + +static void +iso_calendar_date_dealloc(PyDateTime_IsoCalendarDate *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyTuple_Type.tp_dealloc((PyObject *)self); // delegate GC-untrack as well + Py_DECREF(tp); +} + +static PyType_Slot isocal_slots[] = { + {Py_tp_repr, iso_calendar_date_repr}, + {Py_tp_doc, (void *)iso_calendar_date__doc__}, + {Py_tp_methods, iso_calendar_date_methods}, + {Py_tp_getset, iso_calendar_date_getset}, + {Py_tp_new, iso_calendar_date_new}, + {Py_tp_dealloc, iso_calendar_date_dealloc}, + {Py_tp_traverse, iso_calendar_date_traverse}, + {0, NULL}, +}; + +static PyType_Spec isocal_spec = { + .name = "datetime.IsoCalendarDate", + .basicsize = sizeof(PyDateTime_IsoCalendarDate), + .flags = (Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = isocal_slots, }; /*[clinic input] @@ -3435,8 +3756,12 @@ date_isocalendar(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) week = 0; } - PyObject* v = iso_calendar_date_new_impl(&PyDateTime_IsoCalendarDateType, - year, week + 1, day + 1); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), + year, week + 1, day + 1); + RELEASE_CURRENT_STATE(st, current_mod); if (v == NULL) { return NULL; } @@ -3448,7 +3773,15 @@ date_isocalendar(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) static PyObject * date_richcompare(PyObject *self, PyObject *other, int op) { - if (PyDate_Check(other)) { + /* Since DateTime is a subclass of Date, if the other object is + * a DateTime, it would compute an equality testing or an ordering + * based on the date part alone, and we don't want that. + * So return NotImplemented here in that case. + * If a subclass wants to change this, it's up to the subclass to do so. + * The behavior is the same as if Date and DateTime were independent + * classes. + */ + if (PyDate_Check(other) && !PyDateTime_Check(other)) { int diff = memcmp(((PyDateTime_Date *)self)->data, ((PyDateTime_Date *)other)->data, _PyDateTime_DATE_DATASIZE); @@ -3482,13 +3815,13 @@ datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, int day) /*[clinic end generated code: output=2a9430d1e6318aeb input=0d1f02685b3e90f6]*/ { - return new_date_ex(year, month, day, Py_TYPE(self)); + return new_date_subclass_ex(year, month, day, (PyObject *)Py_TYPE(self)); } static Py_hash_t generic_hash(unsigned char *data, int len) { - return _Py_HashBytes(data, len); + return Py_HashBuffer(data, len); } @@ -3558,6 +3891,11 @@ static PyMethodDef date_methods[] = { "number and weekday.\n\n" "This is the inverse of the date.isocalendar() function")}, + {"strptime", (PyCFunction)date_strptime, + METH_VARARGS | METH_CLASS, + PyDoc_STR("string, format -> new date parsed from a string " + "(like time.strptime()).")}, + {"today", (PyCFunction)date_today, METH_NOARGS | METH_CLASS, PyDoc_STR("Current date or datetime: same as " "self.__class__.fromtimestamp(time.time()).")}, @@ -3597,7 +3935,8 @@ static PyMethodDef date_methods[] = { DATETIME_DATE_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {"__reduce__", (PyCFunction)date_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, @@ -3897,7 +4236,7 @@ timezone_new(PyTypeObject *type, PyObject *args, PyObject *kw) PyObject *offset; PyObject *name = NULL; if (PyArg_ParseTupleAndKeywords(args, kw, "O!|U:timezone", timezone_kws, - &PyDateTime_DeltaType, &offset, &name)) + DELTA_TYPE(NO_STATE), &offset, &name)) return new_timezone(offset, name); return NULL; @@ -3950,8 +4289,7 @@ timezone_repr(PyDateTime_TimeZone *self) to use Py_TYPE(self)->tp_name here. */ const char *type_name = Py_TYPE(self)->tp_name; - datetime_state *st = STATIC_STATE(); - if (((PyObject *)self) == st->utc) { + if ((PyObject *)self == CONST_UTC(NO_STATE)) { return PyUnicode_FromFormat("%s.utc", type_name); } @@ -3973,8 +4311,7 @@ timezone_str(PyDateTime_TimeZone *self) if (self->name != NULL) { return Py_NewRef(self->name); } - datetime_state *st = STATIC_STATE(); - if ((PyObject *)self == st->utc || + if ((PyObject *)self == CONST_UTC(NO_STATE) || (GET_TD_DAYS(self->offset) == 0 && GET_TD_SECONDS(self->offset) == 0 && GET_TD_MICROSECONDS(self->offset) == 0)) @@ -4126,6 +4463,23 @@ static PyTypeObject PyDateTime_TimeZoneType = { timezone_new, /* tp_new */ }; +// XXX Can we make this const? +static PyDateTime_TimeZone utc_timezone = { + PyObject_HEAD_INIT(&PyDateTime_TimeZoneType) + .offset = (PyObject *)&zero_delta, + .name = NULL, +}; + +static PyDateTime_TimeZone * +look_up_timezone(PyObject *offset, PyObject *name) +{ + if (offset == utc_timezone.offset && name == NULL) { + return (PyDateTime_TimeZone *)CONST_UTC(NO_STATE); + } + return NULL; +} + + /* * PyDateTime_Time implementation. */ @@ -4276,6 +4630,27 @@ time_new(PyTypeObject *type, PyObject *args, PyObject *kw) return self; } +/* Return new time from _strptime.strptime_datetime_time(). */ +static PyObject * +time_strptime(PyObject *cls, PyObject *args) +{ + PyObject *string, *format, *result; + + if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { + return NULL; + } + + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); + if (module == NULL) { + return NULL; + } + result = PyObject_CallMethodObjArgs(module, + &_Py_ID(_strptime_datetime_time), cls, + string, format, NULL); + Py_DECREF(module); + return result; +} + /* * Destructor. */ @@ -4591,8 +4966,8 @@ datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, int fold) /*[clinic end generated code: output=0b89a44c299e4f80 input=9b6a35b1e704b0ca]*/ { - return new_time_ex2(hour, minute, second, microsecond, tzinfo, fold, - Py_TYPE(self)); + return new_time_subclass_fold_ex(hour, minute, second, microsecond, tzinfo, + fold, (PyObject *)Py_TYPE(self)); } static PyObject * @@ -4629,6 +5004,14 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { goto invalid_string_error; } + if (hour == 24) { + if (minute == 0 && second == 0 && microsecond == 0) { + hour = 0; + } else { + goto invalid_iso_midnight; + } + } + PyObject *tzinfo = tzinfo_from_isoformat_results(rv, tzoffset, tzimicrosecond); @@ -4637,7 +5020,7 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { } PyObject *t; - if ( (PyTypeObject *)cls == &PyDateTime_TimeType ) { + if ( (PyTypeObject *)cls == TIME_TYPE(NO_STATE)) { t = new_time(hour, minute, second, microsecond, tzinfo, 0); } else { t = PyObject_CallFunction(cls, "iiiiO", @@ -4647,6 +5030,10 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { Py_DECREF(tzinfo); return t; +invalid_iso_midnight: + PyErr_SetString(PyExc_ValueError, "minute, second, and microsecond must be 0 when hour is 24"); + return NULL; + invalid_string_error: PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", tstr); return NULL; @@ -4699,6 +5086,15 @@ time_reduce(PyDateTime_Time *self, PyObject *arg) static PyMethodDef time_methods[] = { + /* Class method: */ + + {"strptime", (PyCFunction)time_strptime, + METH_VARARGS | METH_CLASS, + PyDoc_STR("string, format -> new time parsed from a string " + "(like time.strptime()).")}, + + /* Instance methods: */ + {"isoformat", _PyCFunction_CAST(time_isoformat), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]]" "[+HH:MM].\n\n" @@ -4724,7 +5120,8 @@ static PyMethodDef time_methods[] = { DATETIME_TIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {"fromisoformat", (PyCFunction)time_fromisoformat, METH_O | METH_CLASS, PyDoc_STR("string -> time from a string in ISO 8601 format")}, @@ -5087,7 +5484,11 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, static PyObject * datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) { - _PyTime_t ts = _PyTime_GetSystemClock(); + PyTime_t ts; + if (PyTime_Time(&ts) < 0) { + return NULL; + } + time_t secs; int us; @@ -5201,23 +5602,24 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) return result; } -/* Return new datetime from _strptime.strptime_datetime(). */ +/* Return new datetime from _strptime.strptime_datetime_datetime(). */ static PyObject * datetime_strptime(PyObject *cls, PyObject *args) { - static PyObject *module = NULL; - PyObject *string, *format; + PyObject *string, *format, *result; if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) return NULL; + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { - module = PyImport_ImportModule("_strptime"); - if (module == NULL) - return NULL; + return NULL; } - return PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime), - cls, string, format, NULL); + result = PyObject_CallMethodObjArgs(module, + &_Py_ID(_strptime_datetime_datetime), + cls, string, format, NULL); + Py_DECREF(module); + return result; } /* Return new datetime from date/datetime and time arguments. */ @@ -5231,8 +5633,8 @@ datetime_combine(PyObject *cls, PyObject *args, PyObject *kw) PyObject *result = NULL; if (PyArg_ParseTupleAndKeywords(args, kw, "O!O!|O:combine", keywords, - &PyDateTime_DateType, &date, - &PyDateTime_TimeType, &time, &tzinfo)) { + DATE_TYPE(NO_STATE), &date, + TIME_TYPE(NO_STATE), &time, &tzinfo)) { if (tzinfo == NULL) { if (HASTZINFO(time)) tzinfo = ((PyDateTime_Time *)time)->tzinfo; @@ -5488,6 +5890,26 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) goto error; } + if ((hour == 24) && (month <= 12)) { + int d_in_month = days_in_month(year, month); + if (day <= d_in_month) { + if (minute == 0 && second == 0 && microsecond == 0) { + // Calculate midnight of the next day + hour = 0; + day += 1; + if (day > d_in_month) { + day = 1; + month += 1; + if (month > 12) { + month = 1; + year += 1; + } + } + } else { + goto invalid_iso_midnight; + } + } + } PyObject *dt = new_datetime_subclass_ex(year, month, day, hour, minute, second, microsecond, tzinfo, cls); @@ -5495,6 +5917,12 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) Py_DECREF(dtstr_clean); return dt; +invalid_iso_midnight: + PyErr_SetString(PyExc_ValueError, "minute, second, and microsecond must be 0 when hour is 24"); + Py_DECREF(tzinfo); + Py_DECREF(dtstr_clean); + return NULL; + invalid_string_error: PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", dtstr); @@ -5880,21 +6308,7 @@ datetime_richcompare(PyObject *self, PyObject *other, int op) PyObject *offset1, *offset2; int diff; - if (! PyDateTime_Check(other)) { - if (PyDate_Check(other)) { - /* Prevent invocation of date_richcompare. We want to - return NotImplemented here to give the other object - a chance. But since DateTime is a subclass of - Date, if the other object is a Date, it would - compute an ordering based on the date part alone, - and we don't want that. So force unequal or - uncomparable here in that case. */ - if (op == Py_EQ) - Py_RETURN_FALSE; - if (op == Py_NE) - Py_RETURN_TRUE; - return cmperror(self, other); - } + if (!PyDateTime_Check(other)) { Py_RETURN_NOTIMPLEMENTED; } @@ -6055,8 +6469,9 @@ datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, int fold) /*[clinic end generated code: output=00bc96536833fddb input=9b38253d56d9bcad]*/ { - return new_datetime_ex2(year, month, day, hour, minute, second, - microsecond, tzinfo, fold, Py_TYPE(self)); + return new_datetime_subclass_fold_ex(year, month, day, hour, minute, + second, microsecond, tzinfo, fold, + (PyObject *)Py_TYPE(self)); } static PyObject * @@ -6129,10 +6544,14 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - datetime_state *st = STATIC_STATE(); - delta = datetime_subtract((PyObject *)utc_time, st->epoch); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; + one_second = new_delta(0, 1, 0, 0); if (one_second == NULL) { Py_DECREF(delta); @@ -6242,7 +6661,6 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) if (result == NULL) return NULL; - datetime_state *st = STATIC_STATE(); /* Make sure result is aware and UTC. */ if (!HASTZINFO(result)) { temp = (PyObject *)result; @@ -6254,7 +6672,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) DATE_GET_MINUTE(result), DATE_GET_SECOND(result), DATE_GET_MICROSECOND(result), - st->utc, + CONST_UTC(NO_STATE), DATE_GET_FOLD(result), Py_TYPE(result)); Py_DECREF(temp); @@ -6263,7 +6681,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) } else { /* Result is already aware - just replace tzinfo. */ - Py_SETREF(result->tzinfo, Py_NewRef(st->utc)); + Py_SETREF(result->tzinfo, Py_NewRef(CONST_UTC(NO_STATE))); } /* Attach new tzinfo and let fromutc() do the rest. */ @@ -6367,9 +6785,12 @@ datetime_timestamp(PyDateTime_DateTime *self, PyObject *Py_UNUSED(ignored)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - datetime_state *st = STATIC_STATE(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + PyObject *delta; - delta = datetime_subtract((PyObject *)self, st->epoch); + delta = datetime_subtract((PyObject *)self, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; result = delta_total_seconds(delta, NULL); @@ -6580,7 +7001,8 @@ static PyMethodDef datetime_methods[] = { DATETIME_DATETIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, @@ -6657,11 +7079,45 @@ static PyTypeObject PyDateTime_DateTimeType = { }; /* --------------------------------------------------------------------------- - * Module methods and initialization. + * datetime C-API. */ -static PyMethodDef module_methods[] = { - {NULL, NULL} +static PyTypeObject * const capi_types[] = { + &PyDateTime_DateType, + &PyDateTime_DateTimeType, + &PyDateTime_TimeType, + &PyDateTime_DeltaType, + &PyDateTime_TZInfoType, + /* Indirectly, via the utc object. */ + &PyDateTime_TimeZoneType, +}; + +/* The C-API is process-global. This violates interpreter isolation + * due to the objects stored here. Thus each of those objects must + * be managed carefully. */ +// XXX Can we make this const? +static PyDateTime_CAPI capi = { + /* The classes must be readied before used here. + * That will happen the first time the module is loaded. + * They aren't safe to be shared between interpreters, + * but that's okay as long as the module is single-phase init. */ + .DateType = &PyDateTime_DateType, + .DateTimeType = &PyDateTime_DateTimeType, + .TimeType = &PyDateTime_TimeType, + .DeltaType = &PyDateTime_DeltaType, + .TZInfoType = &PyDateTime_TZInfoType, + + .TimeZone_UTC = (PyObject *)&utc_timezone, + + .Date_FromDate = new_date_ex, + .DateTime_FromDateAndTime = new_datetime_ex, + .Time_FromTime = new_time_ex, + .Delta_FromDelta = new_delta_ex, + .TimeZone_FromTimeZone = new_timezone, + .DateTime_FromTimestamp = datetime_fromtimestamp, + .Date_FromTimestamp = datetime_date_fromtimestamp_capi, + .DateTime_FromDateAndTimeAndFold = new_datetime_ex2, + .Time_FromTimeAndFold = new_time_ex2, }; /* Get a new C API by calling this function. @@ -6670,55 +7126,7 @@ static PyMethodDef module_methods[] = { static inline PyDateTime_CAPI * get_datetime_capi(void) { - PyDateTime_CAPI *capi = PyMem_Malloc(sizeof(PyDateTime_CAPI)); - if (capi == NULL) { - PyErr_NoMemory(); - return NULL; - } - capi->DateType = &PyDateTime_DateType; - capi->DateTimeType = &PyDateTime_DateTimeType; - capi->TimeType = &PyDateTime_TimeType; - capi->DeltaType = &PyDateTime_DeltaType; - capi->TZInfoType = &PyDateTime_TZInfoType; - capi->Date_FromDate = new_date_ex; - capi->DateTime_FromDateAndTime = new_datetime_ex; - capi->Time_FromTime = new_time_ex; - capi->Delta_FromDelta = new_delta_ex; - capi->TimeZone_FromTimeZone = new_timezone; - capi->DateTime_FromTimestamp = datetime_fromtimestamp; - capi->Date_FromTimestamp = datetime_date_fromtimestamp_capi; - capi->DateTime_FromDateAndTimeAndFold = new_datetime_ex2; - capi->Time_FromTimeAndFold = new_time_ex2; - // Make sure this function is called after utc has - // been initialized. - datetime_state *st = STATIC_STATE(); - assert(st->utc != NULL); - capi->TimeZone_UTC = st->utc; // borrowed ref - return capi; -} - -static void -datetime_destructor(PyObject *op) -{ - void *ptr = PyCapsule_GetPointer(op, PyDateTime_CAPSULE_NAME); - PyMem_Free(ptr); -} - -static int -datetime_clear(PyObject *module) -{ - datetime_state *st = STATIC_STATE(); - - Py_CLEAR(st->us_per_ms); - Py_CLEAR(st->us_per_second); - Py_CLEAR(st->us_per_minute); - Py_CLEAR(st->us_per_hour); - Py_CLEAR(st->us_per_day); - Py_CLEAR(st->us_per_week); - Py_CLEAR(st->seconds_per_day); - Py_CLEAR(st->utc); - Py_CLEAR(st->epoch); - return 0; + return &capi; } static PyObject * @@ -6733,9 +7141,45 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) return tz; } + +/* --------------------------------------------------------------------------- + * Module state lifecycle. + */ + static int -init_state(datetime_state *st) -{ +init_state(datetime_state *st, PyObject *module, PyObject *old_module) +{ + /* Each module gets its own heap types. */ +#define ADD_TYPE(FIELD, SPEC, BASE) \ + do { \ + PyObject *cls = PyType_FromModuleAndSpec( \ + module, SPEC, (PyObject *)BASE); \ + if (cls == NULL) { \ + return -1; \ + } \ + st->FIELD = (PyTypeObject *)cls; \ + } while (0) + + ADD_TYPE(isocalendar_date_type, &isocal_spec, &PyTuple_Type); +#undef ADD_TYPE + + if (old_module != NULL) { + assert(old_module != module); + datetime_state *st_old = get_module_state(old_module); + *st = (datetime_state){ + .isocalendar_date_type = st->isocalendar_date_type, + .us_per_ms = Py_NewRef(st_old->us_per_ms), + .us_per_second = Py_NewRef(st_old->us_per_second), + .us_per_minute = Py_NewRef(st_old->us_per_minute), + .us_per_hour = Py_NewRef(st_old->us_per_hour), + .us_per_day = Py_NewRef(st_old->us_per_day), + .us_per_week = Py_NewRef(st_old->us_per_week), + .seconds_per_day = Py_NewRef(st_old->seconds_per_day), + .epoch = Py_NewRef(st_old->epoch), + }; + return 0; + } + st->us_per_ms = PyLong_FromLong(1000); if (st->us_per_ms == NULL) { return -1; @@ -6769,51 +7213,111 @@ init_state(datetime_state *st) return -1; } - /* Init UTC timezone */ - st->utc = create_timezone_from_delta(0, 0, 0, 0); - if (st->utc == NULL) { - return -1; - } - /* Init Unix epoch */ - st->epoch = new_datetime(1970, 1, 1, 0, 0, 0, 0, st->utc, 0); + st->epoch = new_datetime( + 1970, 1, 1, 0, 0, 0, 0, (PyObject *)&utc_timezone, 0); if (st->epoch == NULL) { return -1; } + return 0; } static int -_datetime_exec(PyObject *module) +traverse_state(datetime_state *st, visitproc visit, void *arg) { + /* heap types */ + Py_VISIT(st->isocalendar_date_type); + + return 0; +} + +static int +clear_state(datetime_state *st) +{ + Py_CLEAR(st->isocalendar_date_type); + Py_CLEAR(st->us_per_ms); + Py_CLEAR(st->us_per_second); + Py_CLEAR(st->us_per_minute); + Py_CLEAR(st->us_per_hour); + Py_CLEAR(st->us_per_day); + Py_CLEAR(st->us_per_week); + Py_CLEAR(st->seconds_per_day); + Py_CLEAR(st->epoch); + return 0; +} + + +static int +init_static_types(PyInterpreterState *interp, int reloading) +{ + if (reloading) { + return 0; + } + // `&...` is not a constant expression according to a strict reading // of C standards. Fill tp_base at run-time rather than statically. // See https://bugs.python.org/issue40777 - PyDateTime_IsoCalendarDateType.tp_base = &PyTuple_Type; PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - PyTypeObject *types[] = { - &PyDateTime_DateType, - &PyDateTime_DateTimeType, - &PyDateTime_TimeType, - &PyDateTime_DeltaType, - &PyDateTime_TZInfoType, - &PyDateTime_TimeZoneType, - }; + /* Bases classes must be initialized before subclasses, + * so capi_types must have the types in the appropriate order. */ + for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { + PyTypeObject *type = capi_types[i]; + if (_PyStaticType_InitForExtension(interp, type) < 0) { + return -1; + } + } + + return 0; +} + + +/* --------------------------------------------------------------------------- + * Module methods and initialization. + */ + +static PyMethodDef module_methods[] = { + {NULL, NULL} +}; + - for (size_t i = 0; i < Py_ARRAY_LENGTH(types); i++) { - if (PyModule_AddType(module, types[i]) < 0) { +static int +_datetime_exec(PyObject *module) +{ + int rc = -1; + datetime_state *st = get_module_state(module); + int reloading = 0; + + PyInterpreterState *interp = PyInterpreterState_Get(); + PyObject *old_module = get_current_module(interp, &reloading); + if (PyErr_Occurred()) { + assert(old_module == NULL); + goto error; + } + /* We actually set the "current" module right before a successful return. */ + + if (init_static_types(interp, reloading) < 0) { + goto error; + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { + PyTypeObject *type = capi_types[i]; + const char *name = _PyType_Name(type); + assert(name != NULL); + if (PyModule_AddObjectRef(module, name, (PyObject *)type) < 0) { goto error; } } - if (PyType_Ready(&PyDateTime_IsoCalendarDateType) < 0) { + if (init_state(st, module, old_module) < 0) { goto error; } #define DATETIME_ADD_MACRO(dict, c, value_expr) \ do { \ + assert(!PyErr_Occurred()); \ PyObject *value = (value_expr); \ if (value == NULL) { \ goto error; \ @@ -6825,55 +7329,53 @@ _datetime_exec(PyObject *module) Py_DECREF(value); \ } while(0) - /* timedelta values */ - PyObject *d = PyDateTime_DeltaType.tp_dict; - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); - DATETIME_ADD_MACRO(d, "max", - new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); - - /* date values */ - d = PyDateTime_DateType.tp_dict; - DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); - DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); - - /* time values */ - d = PyDateTime_TimeType.tp_dict; - DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* datetime values */ - d = PyDateTime_DateTimeType.tp_dict; - DATETIME_ADD_MACRO(d, "min", - new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, - 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - datetime_state *st = STATIC_STATE(); - if (init_state(st) < 0) { - goto error; - } + if (!reloading) { + /* timedelta values */ + PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); + DATETIME_ADD_MACRO(d, "max", + new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); + + /* date values */ + d = _PyType_GetDict(&PyDateTime_DateType); + DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); + DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); + + /* time values */ + d = _PyType_GetDict(&PyDateTime_TimeType); + DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* datetime values */ + d = _PyType_GetDict(&PyDateTime_DateTimeType); + DATETIME_ADD_MACRO(d, "min", + new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, + 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* timezone values */ + d = _PyType_GetDict(&PyDateTime_TimeZoneType); + if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { + goto error; + } - /* timezone values */ - d = PyDateTime_TimeZoneType.tp_dict; - if (PyDict_SetItemString(d, "utc", st->utc) < 0) { - goto error; - } + /* bpo-37642: These attributes are rounded to the nearest minute for backwards + * compatibility, even though the constructor will accept a wider range of + * values. This may change in the future.*/ - /* bpo-37642: These attributes are rounded to the nearest minute for backwards - * compatibility, even though the constructor will accept a wider range of - * values. This may change in the future.*/ + /* -23:59 */ + DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); - /* -23:59 */ - PyObject *min = create_timezone_from_delta(-1, 60, 0, 1); - DATETIME_ADD_MACRO(d, "min", min); + /* +23:59 */ + DATETIME_ADD_MACRO( + d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + } - /* +23:59 */ - PyObject *max = create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0); - DATETIME_ADD_MACRO(d, "max", max); +#undef DATETIME_ADD_MACRO /* Add module level attributes */ if (PyModule_AddIntMacro(module, MINYEAR) < 0) { @@ -6882,7 +7384,7 @@ _datetime_exec(PyObject *module) if (PyModule_AddIntMacro(module, MAXYEAR) < 0) { goto error; } - if (PyModule_AddObjectRef(module, "UTC", st->utc) < 0) { + if (PyModule_AddObjectRef(module, "UTC", (PyObject *)&utc_timezone) < 0) { goto error; } @@ -6891,14 +7393,9 @@ _datetime_exec(PyObject *module) if (capi == NULL) { goto error; } - PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, - datetime_destructor); - if (capsule == NULL) { - PyMem_Free(capi); - goto error; - } + PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, NULL); + // (capsule == NULL) is handled by PyModule_Add if (PyModule_Add(module, "datetime_CAPI", capsule) < 0) { - PyMem_Free(capi); goto error; } @@ -6920,35 +7417,73 @@ _datetime_exec(PyObject *module) static_assert(DI100Y == 25 * DI4Y - 1, "DI100Y"); assert(DI100Y == days_before_year(100+1)); - return 0; + if (set_current_module(interp, module) < 0) { + goto error; + } + + rc = 0; + goto finally; error: - datetime_clear(module); - return -1; + clear_state(st); + +finally: + Py_XDECREF(old_module); + return rc; } -#undef DATETIME_ADD_MACRO -static struct PyModuleDef datetimemodule = { +static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _datetime_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + datetime_state *st = get_module_state(mod); + traverse_state(st, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + datetime_state *st = get_module_state(mod); + clear_state(st); + + PyInterpreterState *interp = PyInterpreterState_Get(); + clear_current_module(interp, mod); + + // The runtime takes care of the static types for us. + // See _PyTypes_FiniExtTypes().. + + return 0; +} + +static void +module_free(void *mod) +{ + (void)module_clear((PyObject *)mod); +} + +static PyModuleDef datetimemodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_datetime", - .m_doc = "Fast implementation of the datetime type.", - .m_size = -1, + .m_doc = "Fast implementation of the datetime module.", + .m_size = sizeof(datetime_state), .m_methods = module_methods, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, }; PyMODINIT_FUNC PyInit__datetime(void) { - PyObject *mod = PyModule_Create(&datetimemodule); - if (mod == NULL) - return NULL; - - if (_datetime_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - - return mod; + return PyModuleDef_Init(&datetimemodule); } /* --------------------------------------------------------------------------- diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index ee33fe625be3d7..1be4234aad3291 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -616,6 +616,7 @@ _dbm_module_free(void *module) static PyModuleDef_Slot _dbmmodule_slots[] = { {Py_mod_exec, _dbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 127f5f2887d4cd..c564813036e504 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -32,8 +32,21 @@ #include #include "pycore_long.h" // _PyLong_IsZero() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_typeobject.h" #include "complexobject.h" -#include "mpdecimal.h" + +#include + +// Reuse config from mpdecimal.h if present. +#if defined(MPD_CONFIG_64) + #ifndef CONFIG_64 + #define CONFIG_64 MPD_CONFIG_64 + #endif +#elif defined(MPD_CONFIG_32) + #ifndef CONFIG_32 + #define CONFIG_32 MPD_CONFIG_32 + #endif +#endif #include // isascii() #include @@ -63,8 +76,9 @@ typedef struct { #ifndef WITH_DECIMAL_CONTEXTVAR /* Key for thread state dictionary */ PyObject *tls_context_key; - /* Invariant: NULL or the most recently accessed thread local context */ - struct PyDecContextObject *cached_context; + /* Invariant: NULL or a strong reference to the most recently accessed + thread local context. */ + struct PyDecContextObject *cached_context; /* Not borrowed */ #else PyObject *current_context_var; #endif @@ -82,6 +96,9 @@ typedef struct { /* Convert rationals for comparison */ PyObject *Rational; + /* Invariant: NULL or pointer to _pydecimal.Decimal */ + PyObject *PyDecimal; + PyObject *SignalTuple; struct DecCondMap *signal_map; @@ -105,6 +122,8 @@ get_module_state(PyObject *mod) } static struct PyModuleDef _decimal_module; +static PyType_Spec dec_spec; +static PyType_Spec context_spec; static inline decimal_state * get_module_state_by_def(PyTypeObject *tp) @@ -117,13 +136,16 @@ get_module_state_by_def(PyTypeObject *tp) static inline decimal_state * find_state_left_or_right(PyObject *left, PyObject *right) { - PyObject *mod = PyType_GetModuleByDef(Py_TYPE(left), &_decimal_module); - if (mod == NULL) { - PyErr_Clear(); - mod = PyType_GetModuleByDef(Py_TYPE(right), &_decimal_module); + PyTypeObject *base; + if (PyType_GetBaseByToken(Py_TYPE(left), &dec_spec, &base) != 1) { + assert(!PyErr_Occurred()); + PyType_GetBaseByToken(Py_TYPE(right), &dec_spec, &base); } - assert(mod != NULL); - return get_module_state(mod); + assert(base != NULL); + void *state = _PyType_GetModuleState(base); + assert(state != NULL); + Py_DECREF(base); + return (decimal_state *)state; } @@ -147,12 +169,6 @@ find_state_left_or_right(PyObject *left, PyObject *right) #define BOUNDS_CHECK(x, MIN, MAX) x = (x < MIN || MAX < x) ? MAX : x -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) - #define UNUSED __attribute__((unused)) -#else - #define UNUSED -#endif - /* _Py_DEC_MINALLOC >= MPD_MINALLOC */ #define _Py_DEC_MINALLOC 4 @@ -175,6 +191,7 @@ typedef struct PyDecContextObject { PyObject *flags; int capitals; PyThreadState *tstate; + decimal_state *modstate; } PyDecContextObject; typedef struct { @@ -195,6 +212,15 @@ typedef struct { #define CTX(v) (&((PyDecContextObject *)v)->ctx) #define CtxCaps(v) (((PyDecContextObject *)v)->capitals) +static inline decimal_state * +get_module_state_from_ctx(PyObject *v) +{ + assert(PyType_GetBaseByToken(Py_TYPE(v), &context_spec, NULL) == 1); + decimal_state *state = ((PyDecContextObject *)v)->modstate; + assert(state != NULL); + return state; +} + Py_LOCAL_INLINE(PyObject *) incr_true(void) @@ -348,7 +374,7 @@ runtime_error_ptr(const char *mesg) return runtime_error_ptr("internal error in " funcname) static void -dec_traphandler(mpd_context_t *ctx UNUSED) /* GCOV_NOT_REACHED */ +dec_traphandler(mpd_context_t *Py_UNUSED(ctx)) /* GCOV_NOT_REACHED */ { /* GCOV_NOT_REACHED */ return; /* GCOV_NOT_REACHED */ } @@ -549,7 +575,7 @@ static int dec_addstatus(PyObject *context, uint32_t status) { mpd_context_t *ctx = CTX(context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); ctx->status |= status; if (status & (ctx->traps|MPD_Malloc_error)) { @@ -610,7 +636,8 @@ getround(decimal_state *state, PyObject *v) static const char *INVALID_SIGNALDICT_ERROR_MSG = "invalid signal dict"; static int -signaldict_init(PyObject *self, PyObject *args UNUSED, PyObject *kwds UNUSED) +signaldict_init(PyObject *self, + PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { SdFlagAddr(self) = NULL; return 0; @@ -736,7 +763,7 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) { PyObject *res = Py_NotImplemented; - decimal_state *state = find_state_left_or_right(v, w); + decimal_state *state = get_module_state_by_def(Py_TYPE(v)); assert(PyDecSignalDict_Check(state, v)); if ((SdFlagAddr(v) == NULL) || (SdFlagAddr(w) == NULL)) { @@ -768,7 +795,7 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) } static PyObject * -signaldict_copy(PyObject *self, PyObject *args UNUSED) +signaldict_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { if (SdFlagAddr(self) == NULL) { return value_error_ptr(INVALID_SIGNALDICT_ERROR_MSG); @@ -779,7 +806,7 @@ signaldict_copy(PyObject *self, PyObject *args UNUSED) static PyMethodDef signaldict_methods[] = { - { "copy", (PyCFunction)signaldict_copy, METH_NOARGS, NULL}, + { "copy", signaldict_copy, METH_NOARGS, NULL}, {NULL, NULL} }; @@ -815,18 +842,18 @@ static PyType_Spec signaldict_spec = { /* Context Object, Part 1 */ /******************************************************************************/ -#define Dec_CONTEXT_GET_SSIZE(mem) \ -static PyObject * \ -context_get##mem(PyObject *self, void *closure UNUSED) \ -{ \ - return PyLong_FromSsize_t(mpd_get##mem(CTX(self))); \ +#define Dec_CONTEXT_GET_SSIZE(mem) \ +static PyObject * \ +context_get##mem(PyObject *self, void *Py_UNUSED(closure)) \ +{ \ + return PyLong_FromSsize_t(mpd_get##mem(CTX(self))); \ } -#define Dec_CONTEXT_GET_ULONG(mem) \ -static PyObject * \ -context_get##mem(PyObject *self, void *closure UNUSED) \ -{ \ - return PyLong_FromUnsignedLong(mpd_get##mem(CTX(self))); \ +#define Dec_CONTEXT_GET_ULONG(mem) \ +static PyObject * \ +context_get##mem(PyObject *self, void *Py_UNUSED(closure)) \ +{ \ + return PyLong_FromUnsignedLong(mpd_get##mem(CTX(self))); \ } Dec_CONTEXT_GET_SSIZE(prec) @@ -840,42 +867,42 @@ Dec_CONTEXT_GET_ULONG(status) #endif static PyObject * -context_getround(PyObject *self, void *closure UNUSED) +context_getround(PyObject *self, void *Py_UNUSED(closure)) { int i = mpd_getround(CTX(self)); - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); return Py_NewRef(state->round_map[i]); } static PyObject * -context_getcapitals(PyObject *self, void *closure UNUSED) +context_getcapitals(PyObject *self, void *Py_UNUSED(closure)) { return PyLong_FromLong(CtxCaps(self)); } #ifdef EXTRA_FUNCTIONALITY static PyObject * -context_getallcr(PyObject *self, void *closure UNUSED) +context_getallcr(PyObject *self, void *Py_UNUSED(closure)) { return PyLong_FromLong(mpd_getcr(CTX(self))); } #endif static PyObject * -context_getetiny(PyObject *self, PyObject *dummy UNUSED) +context_getetiny(PyObject *self, PyObject *Py_UNUSED(dummy)) { return PyLong_FromSsize_t(mpd_etiny(CTX(self))); } static PyObject * -context_getetop(PyObject *self, PyObject *dummy UNUSED) +context_getetop(PyObject *self, PyObject *Py_UNUSED(dummy)) { return PyLong_FromSsize_t(mpd_etop(CTX(self))); } static int -context_setprec(PyObject *self, PyObject *value, void *closure UNUSED) +context_setprec(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -895,7 +922,7 @@ context_setprec(PyObject *self, PyObject *value, void *closure UNUSED) } static int -context_setemin(PyObject *self, PyObject *value, void *closure UNUSED) +context_setemin(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -915,7 +942,7 @@ context_setemin(PyObject *self, PyObject *value, void *closure UNUSED) } static int -context_setemax(PyObject *self, PyObject *value, void *closure UNUSED) +context_setemax(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -997,12 +1024,12 @@ context_unsafe_setemax(PyObject *self, PyObject *value) #endif static int -context_setround(PyObject *self, PyObject *value, void *closure UNUSED) +context_setround(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; int x; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); x = getround(state, value); if (x == -1) { return -1; @@ -1017,7 +1044,7 @@ context_setround(PyObject *self, PyObject *value, void *closure UNUSED) } static int -context_setcapitals(PyObject *self, PyObject *value, void *closure UNUSED) +context_setcapitals(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_ssize_t x; @@ -1037,7 +1064,7 @@ context_setcapitals(PyObject *self, PyObject *value, void *closure UNUSED) #ifdef EXTRA_FUNCTIONALITY static int -context_settraps(PyObject *self, PyObject *value, void *closure UNUSED) +context_settraps(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; uint32_t flags; @@ -1061,7 +1088,7 @@ context_settraps_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { return -1; @@ -1081,7 +1108,7 @@ context_settraps_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1102,7 +1129,7 @@ context_settraps_dict(PyObject *self, PyObject *value) #ifdef EXTRA_FUNCTIONALITY static int -context_setstatus(PyObject *self, PyObject *value, void *closure UNUSED) +context_setstatus(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; uint32_t flags; @@ -1126,7 +1153,7 @@ context_setstatus_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { @@ -1147,7 +1174,7 @@ context_setstatus_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1167,7 +1194,7 @@ context_setstatus_dict(PyObject *self, PyObject *value) } static int -context_setclamp(PyObject *self, PyObject *value, void *closure UNUSED) +context_setclamp(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -1188,7 +1215,7 @@ context_setclamp(PyObject *self, PyObject *value, void *closure UNUSED) #ifdef EXTRA_FUNCTIONALITY static int -context_setallcr(PyObject *self, PyObject *value, void *closure UNUSED) +context_setallcr(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -1310,14 +1337,14 @@ context_setattrs(PyObject *self, PyObject *prec, PyObject *rounding, } static PyObject * -context_clear_traps(PyObject *self, PyObject *dummy UNUSED) +context_clear_traps(PyObject *self, PyObject *Py_UNUSED(dummy)) { CTX(self)->traps = 0; Py_RETURN_NONE; } static PyObject * -context_clear_flags(PyObject *self, PyObject *dummy UNUSED) +context_clear_flags(PyObject *self, PyObject *Py_UNUSED(dummy)) { CTX(self)->status = 0; Py_RETURN_NONE; @@ -1333,7 +1360,8 @@ static mpd_context_t dflt_ctx = { }; static PyObject * -context_new(PyTypeObject *type, PyObject *args UNUSED, PyObject *kwds UNUSED) +context_new(PyTypeObject *type, + PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { PyDecContextObject *self = NULL; mpd_context_t *ctx; @@ -1376,7 +1404,12 @@ context_new(PyTypeObject *type, PyObject *args UNUSED, PyObject *kwds UNUSED) CtxCaps(self) = 1; self->tstate = NULL; + self->modstate = state; + if (type == state->PyDecContext_Type) { + PyObject_GC_Track(self); + } + assert(PyObject_GC_IsTracked((PyObject *)self)); return (PyObject *)self; } @@ -1402,12 +1435,6 @@ context_dealloc(PyDecContextObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); -#ifndef WITH_DECIMAL_CONTEXTVAR - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); - if (self == state->cached_context) { - state->cached_context = NULL; - } -#endif (void)context_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -1455,7 +1482,7 @@ context_repr(PyDecContextObject *self) int n, mem; #ifdef Py_DEBUG - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx((PyObject *)self); assert(PyDecContext_Check(state, self)); #endif ctx = CTX(self); @@ -1506,7 +1533,7 @@ init_extended_context(PyObject *v) #ifdef EXTRA_FUNCTIONALITY /* Factory function for creating IEEE interchange format contexts */ static PyObject * -ieee_context(PyObject *dummy UNUSED, PyObject *v) +ieee_context(PyObject *module, PyObject *v) { PyObject *context; mpd_ssize_t bits; @@ -1523,7 +1550,7 @@ ieee_context(PyObject *dummy UNUSED, PyObject *v) goto error; } - decimal_state *state = get_module_state_by_def(Py_TYPE(v)); + decimal_state *state = get_module_state(module); context = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (context == NULL) { return NULL; @@ -1542,11 +1569,11 @@ ieee_context(PyObject *dummy UNUSED, PyObject *v) #endif static PyObject * -context_copy(PyObject *self, PyObject *args UNUSED) +context_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *copy; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); copy = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (copy == NULL) { return NULL; @@ -1560,13 +1587,13 @@ context_copy(PyObject *self, PyObject *args UNUSED) } static PyObject * -context_reduce(PyObject *self, PyObject *args UNUSED) +context_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *flags; PyObject *traps; PyObject *ret; mpd_context_t *ctx; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); ctx = CTX(self); @@ -1684,7 +1711,8 @@ current_context_from_dict(decimal_state *modstate) /* Cache the context of the current thread, assuming that it * will be accessed several times before a thread switch. */ - modstate->cached_context = (PyDecContextObject *)tl_context; + Py_XSETREF(modstate->cached_context, + (PyDecContextObject *)Py_NewRef(tl_context)); modstate->cached_context->tstate = tstate; /* Borrowed reference with refcount==1 */ @@ -1704,15 +1732,17 @@ current_context(decimal_state *modstate) } /* ctxobj := borrowed reference to the current context */ -#define CURRENT_CONTEXT(state, ctxobj) \ - ctxobj = current_context(state); \ - if (ctxobj == NULL) { \ - return NULL; \ - } +#define CURRENT_CONTEXT(STATE, CTXOBJ) \ + do { \ + CTXOBJ = current_context(STATE); \ + if (CTXOBJ == NULL) { \ + return NULL; \ + } \ + } while (0) /* Return a new reference to the current context */ static PyObject * -PyDec_GetCurrentContext(PyObject *self, PyObject *args UNUSED) +PyDec_GetCurrentContext(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *context; decimal_state *state = get_module_state(self); @@ -1752,7 +1782,7 @@ PyDec_SetCurrentContext(PyObject *self, PyObject *v) Py_INCREF(v); } - state->cached_context = NULL; + Py_CLEAR(state->cached_context); if (PyDict_SetItem(dict, state->tls_context_key, v) < 0) { Py_DECREF(v); return NULL; @@ -1797,16 +1827,18 @@ current_context(decimal_state *state) } /* ctxobj := borrowed reference to the current context */ -#define CURRENT_CONTEXT(state, ctxobj) \ - ctxobj = current_context(state); \ - if (ctxobj == NULL) { \ - return NULL; \ - } \ - Py_DECREF(ctxobj); +#define CURRENT_CONTEXT(STATE, CTXOBJ) \ + do { \ + CTXOBJ = current_context(STATE); \ + if (CTXOBJ == NULL) { \ + return NULL; \ + } \ + Py_DECREF(CTXOBJ); \ + } while (0) /* Return a new reference to the current context */ static PyObject * -PyDec_GetCurrentContext(PyObject *self, PyObject *args UNUSED) +PyDec_GetCurrentContext(PyObject *self, PyObject *Py_UNUSED(dummy)) { decimal_state *state = get_module_state(self); return current_context(state); @@ -1943,7 +1975,8 @@ ctxmanager_dealloc(PyDecContextManagerObject *self) } static PyObject * -ctxmanager_set_local(PyDecContextManagerObject *self, PyObject *args UNUSED) +ctxmanager_set_local(PyDecContextManagerObject *self, + PyObject *Py_UNUSED(dummy)) { PyObject *ret; @@ -1958,7 +1991,7 @@ ctxmanager_set_local(PyDecContextManagerObject *self, PyObject *args UNUSED) static PyObject * ctxmanager_restore_global(PyDecContextManagerObject *self, - PyObject *args UNUSED) + PyObject *Py_UNUSED(args)) { PyObject *ret; @@ -2001,11 +2034,10 @@ static PyType_Spec ctxmanager_spec = { /******************************************************************************/ static PyObject * -PyDecType_New(PyTypeObject *type) +PyDecType_New(decimal_state *state, PyTypeObject *type) { PyDecObject *dec; - decimal_state *state = get_module_state_by_def(type); if (type == state->PyDec_Type) { dec = PyObject_GC_New(PyDecObject, state->PyDec_Type); } @@ -2025,9 +2057,13 @@ PyDecType_New(PyTypeObject *type) MPD(dec)->alloc = _Py_DEC_MINALLOC; MPD(dec)->data = dec->data; + if (type == state->PyDec_Type) { + PyObject_GC_Track(dec); + } + assert(PyObject_GC_IsTracked((PyObject *)dec)); return (PyObject *)dec; } -#define dec_alloc(st) PyDecType_New((st)->PyDec_Type) +#define dec_alloc(st) PyDecType_New(st, (st)->PyDec_Type) static int dec_traverse(PyObject *dec, visitproc visit, void *arg) @@ -2130,7 +2166,8 @@ PyDecType_FromCString(PyTypeObject *type, const char *s, PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2154,7 +2191,8 @@ PyDecType_FromCStringExact(PyTypeObject *type, const char *s, uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2241,7 +2279,8 @@ PyDecType_FromSsize(PyTypeObject *type, mpd_ssize_t v, PyObject *context) PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2262,7 +2301,8 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2280,13 +2320,13 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) /* Convert from a PyLongObject. The context is not modified; flags set during conversion are accumulated in the status parameter. */ static PyObject * -dec_from_long(PyTypeObject *type, PyObject *v, +dec_from_long(decimal_state *state, PyTypeObject *type, PyObject *v, const mpd_context_t *ctx, uint32_t *status) { PyObject *dec; PyLongObject *l = (PyLongObject *)v; - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2331,7 +2371,8 @@ PyDecType_FromLong(PyTypeObject *type, PyObject *v, PyObject *context) return NULL; } - dec = dec_from_long(type, v, CTX(context), &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, CTX(context), &status); if (dec == NULL) { return NULL; } @@ -2360,7 +2401,8 @@ PyDecType_FromLongExact(PyTypeObject *type, PyObject *v, } mpd_maxcontext(&maxctx); - dec = dec_from_long(type, v, &maxctx, &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, &maxctx, &status); if (dec == NULL) { return NULL; } @@ -2392,7 +2434,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, mpd_t *d1, *d2; uint32_t status = 0; mpd_context_t maxctx; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); #ifdef Py_DEBUG assert(PyType_IsSubtype(type, state->PyDec_Type)); @@ -2412,12 +2454,12 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, } sign = (copysign(1.0, x) == 1.0) ? 0 : 1; - if (Py_IS_NAN(x) || Py_IS_INFINITY(x)) { - dec = PyDecType_New(type); + if (isnan(x) || isinf(x)) { + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } - if (Py_IS_NAN(x)) { + if (isnan(x)) { /* decimal.py calls repr(float(+-nan)), * which always gives a positive result. */ mpd_setspecial(MPD(dec), MPD_POS, MPD_NAN); @@ -2530,12 +2572,12 @@ PyDecType_FromDecimalExact(PyTypeObject *type, PyObject *v, PyObject *context) PyObject *dec; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (type == state->PyDec_Type && PyDec_CheckExact(state, v)) { return Py_NewRef(v); } - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2815,11 +2857,56 @@ dec_from_float(PyObject *type, PyObject *pyfloat) return result; } +/* 'v' can have any numeric type accepted by the Decimal constructor. Attempt + an exact conversion. If the result does not meet the restrictions + for an mpd_t, fail with InvalidOperation. */ +static PyObject * +PyDecType_FromNumberExact(PyTypeObject *type, PyObject *v, PyObject *context) +{ + decimal_state *state = get_module_state_by_def(type); + assert(v != NULL); + if (PyDec_Check(state, v)) { + return PyDecType_FromDecimalExact(type, v, context); + } + else if (PyLong_Check(v)) { + return PyDecType_FromLongExact(type, v, context); + } + else if (PyFloat_Check(v)) { + if (dec_addstatus(context, MPD_Float_operation)) { + return NULL; + } + return PyDecType_FromFloatExact(type, v, context); + } + else { + PyErr_Format(PyExc_TypeError, + "conversion from %s to Decimal is not supported", + Py_TYPE(v)->tp_name); + return NULL; + } +} + +/* class method */ +static PyObject * +dec_from_number(PyObject *type, PyObject *number) +{ + PyObject *context; + PyObject *result; + + decimal_state *state = get_module_state_by_def((PyTypeObject *)type); + CURRENT_CONTEXT(state, context); + result = PyDecType_FromNumberExact(state->PyDec_Type, number, context); + if (type != (PyObject *)state->PyDec_Type && result != NULL) { + Py_SETREF(result, PyObject_CallFunctionObjArgs(type, result, NULL)); + } + + return result; +} + /* create_decimal_from_float */ static PyObject * ctx_from_float(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); return PyDec_FromFloat(state, v, context); } @@ -2830,7 +2917,7 @@ dec_apply(PyObject *v, PyObject *context) PyObject *result; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -2857,7 +2944,7 @@ dec_apply(PyObject *v, PyObject *context) static PyObject * PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDecType_FromSsizeExact(type, 0, context); } @@ -2892,7 +2979,7 @@ PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) static PyObject * PyDec_FromObject(PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDec_FromSsize(state, 0, context); } @@ -2979,7 +3066,7 @@ ctx_create_decimal(PyObject *context, PyObject *args) Py_LOCAL_INLINE(int) convert_op(int type_err, PyObject **conv, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, v)) { *conv = Py_NewRef(v); return 1; @@ -3082,7 +3169,7 @@ multiply_by_denominator(PyObject *v, PyObject *r, PyObject *context) if (tmp == NULL) { return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); denom = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); if (denom == NULL) { @@ -3137,7 +3224,7 @@ numerator_as_decimal(PyObject *r, PyObject *context) return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); num = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); return num; @@ -3156,7 +3243,7 @@ convert_op_cmp(PyObject **vcmp, PyObject **wcmp, PyObject *v, PyObject *w, *vcmp = v; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, w)) { *wcmp = Py_NewRef(w); } @@ -3336,56 +3423,6 @@ dotsep_as_utf8(const char *s) return utf8; } -/* copy of libmpdec _mpd_round() */ -static void -_mpd_round(mpd_t *result, const mpd_t *a, mpd_ssize_t prec, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_ssize_t exp = a->exp + a->digits - prec; - - if (prec <= 0) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mpd_isspecial(a) || mpd_iszero(a)) { - mpd_qcopy(result, a, status); - return; - } - - mpd_qrescale_fmt(result, a, exp, ctx, status); - if (result->digits > prec) { - mpd_qrescale_fmt(result, result, exp+1, ctx, status); - } -} - -/* Locate negative zero "z" option within a UTF-8 format spec string. - * Returns pointer to "z", else NULL. - * The portion of the spec we're working with is [[fill]align][sign][z] */ -static const char * -format_spec_z_search(char const *fmt, Py_ssize_t size) { - char const *pos = fmt; - char const *fmt_end = fmt + size; - /* skip over [[fill]align] (fill may be multi-byte character) */ - pos += 1; - while (pos < fmt_end && *pos & 0x80) { - pos += 1; - } - if (pos < fmt_end && strchr("<>=^", *pos) != NULL) { - pos += 1; - } else { - /* fill not present-- skip over [align] */ - pos = fmt; - if (pos < fmt_end && strchr("<>=^", *pos) != NULL) { - pos += 1; - } - } - /* skip over [sign] */ - if (pos < fmt_end && strchr("+- ", *pos) != NULL) { - pos += 1; - } - return pos < fmt_end && *pos == 'z' ? pos : NULL; -} - static int dict_get_item_string(PyObject *dict, const char *key, PyObject **valueobj, const char **valuestr) { @@ -3411,6 +3448,48 @@ dict_get_item_string(PyObject *dict, const char *key, PyObject **valueobj, const return 0; } +/* + * Fallback _pydecimal formatting for new format specifiers that mpdecimal does + * not yet support. As documented, libmpdec follows the PEP-3101 format language: + * https://www.bytereef.org/mpdecimal/doc/libmpdec/assign-convert.html#to-string + */ +static PyObject * +pydec_format(PyObject *dec, PyObject *context, PyObject *fmt, decimal_state *state) +{ + PyObject *result; + PyObject *pydec; + PyObject *u; + + if (state->PyDecimal == NULL) { + state->PyDecimal = _PyImport_GetModuleAttrString("_pydecimal", "Decimal"); + if (state->PyDecimal == NULL) { + return NULL; + } + } + + u = dec_str(dec); + if (u == NULL) { + return NULL; + } + + pydec = PyObject_CallOneArg(state->PyDecimal, u); + Py_DECREF(u); + if (pydec == NULL) { + return NULL; + } + + result = PyObject_CallMethod(pydec, "__format__", "(OO)", fmt, context); + Py_DECREF(pydec); + + if (result == NULL && PyErr_ExceptionMatches(PyExc_ValueError)) { + /* Do not confuse users with the _pydecimal exception */ + PyErr_Clear(); + PyErr_SetString(PyExc_ValueError, "invalid format string"); + } + + return result; +} + /* Formatted representation of a PyDecObject. */ static PyObject * dec_format(PyObject *dec, PyObject *args) @@ -3423,16 +3502,11 @@ dec_format(PyObject *dec, PyObject *args) PyObject *fmtarg; PyObject *context; mpd_spec_t spec; - char const *fmt; - char *fmt_copy = NULL; + char *fmt; char *decstring = NULL; uint32_t status = 0; int replace_fillchar = 0; - int no_neg_0 = 0; Py_ssize_t size; - mpd_t *mpd = MPD(dec); - mpd_uint_t dt[MPD_MINALLOC_MAX]; - mpd_t tmp = {MPD_STATIC|MPD_STATIC_DATA,0,0,0,MPD_MINALLOC_MAX,dt}; decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); @@ -3442,7 +3516,7 @@ dec_format(PyObject *dec, PyObject *args) } if (PyUnicode_Check(fmtarg)) { - fmt = PyUnicode_AsUTF8AndSize(fmtarg, &size); + fmt = (char *)PyUnicode_AsUTF8AndSize(fmtarg, &size); if (fmt == NULL) { return NULL; } @@ -3454,35 +3528,15 @@ dec_format(PyObject *dec, PyObject *args) } } - /* NOTE: If https://github.com/python/cpython/pull/29438 lands, the - * format string manipulation below can be eliminated by enhancing - * the forked mpd_parse_fmt_str(). */ if (size > 0 && fmt[0] == '\0') { /* NUL fill character: must be replaced with a valid UTF-8 char before calling mpd_parse_fmt_str(). */ replace_fillchar = 1; - fmt = fmt_copy = dec_strdup(fmt, size); - if (fmt_copy == NULL) { + fmt = dec_strdup(fmt, size); + if (fmt == NULL) { return NULL; } - fmt_copy[0] = '_'; - } - /* Strip 'z' option, which isn't understood by mpd_parse_fmt_str(). - * NOTE: fmt is always null terminated by PyUnicode_AsUTF8AndSize() */ - char const *z_position = format_spec_z_search(fmt, size); - if (z_position != NULL) { - no_neg_0 = 1; - size_t z_index = z_position - fmt; - if (fmt_copy == NULL) { - fmt = fmt_copy = dec_strdup(fmt, size); - if (fmt_copy == NULL) { - return NULL; - } - } - /* Shift characters (including null terminator) left, - overwriting the 'z' option. */ - memmove(fmt_copy + z_index, fmt_copy + z_index + 1, size - z_index); - size -= 1; + fmt[0] = '_'; } } else { @@ -3492,10 +3546,13 @@ dec_format(PyObject *dec, PyObject *args) } if (!mpd_parse_fmt_str(&spec, fmt, CtxCaps(context))) { - PyErr_SetString(PyExc_ValueError, - "invalid format string"); - goto finish; + if (replace_fillchar) { + PyMem_Free(fmt); + } + + return pydec_format(dec, context, fmtarg, state); } + if (replace_fillchar) { /* In order to avoid clobbering parts of UTF-8 thousands separators or decimal points when the substitution is reversed later, the actual @@ -3548,45 +3605,8 @@ dec_format(PyObject *dec, PyObject *args) } } - if (no_neg_0 && mpd_isnegative(mpd) && !mpd_isspecial(mpd)) { - /* Round into a temporary (carefully mirroring the rounding - of mpd_qformat_spec()), and check if the result is negative zero. - If so, clear the sign and format the resulting positive zero. */ - mpd_ssize_t prec; - mpd_qcopy(&tmp, mpd, &status); - if (spec.prec >= 0) { - switch (spec.type) { - case 'f': - mpd_qrescale(&tmp, &tmp, -spec.prec, CTX(context), &status); - break; - case '%': - tmp.exp += 2; - mpd_qrescale(&tmp, &tmp, -spec.prec, CTX(context), &status); - break; - case 'g': - prec = (spec.prec == 0) ? 1 : spec.prec; - if (tmp.digits > prec) { - _mpd_round(&tmp, &tmp, prec, CTX(context), &status); - } - break; - case 'e': - if (!mpd_iszero(&tmp)) { - _mpd_round(&tmp, &tmp, spec.prec+1, CTX(context), &status); - } - break; - } - } - if (status & MPD_Errors) { - PyErr_SetString(PyExc_ValueError, "unexpected error when rounding"); - goto finish; - } - if (mpd_iszero(&tmp)) { - mpd_set_positive(&tmp); - mpd = &tmp; - } - } - decstring = mpd_qformat_spec(mpd, &spec, CTX(context), &status); + decstring = mpd_qformat_spec(MPD(dec), &spec, CTX(context), &status); if (decstring == NULL) { if (status & MPD_Malloc_error) { PyErr_NoMemory(); @@ -3609,7 +3629,7 @@ dec_format(PyObject *dec, PyObject *args) Py_XDECREF(grouping); Py_XDECREF(sep); Py_XDECREF(dot); - if (fmt_copy) PyMem_Free(fmt_copy); + if (replace_fillchar) PyMem_Free(fmt); if (decstring) mpd_free(decstring); return result; } @@ -3684,7 +3704,7 @@ dec_as_long(PyObject *dec, PyObject *context, int round) /* Convert a Decimal to its exact integer ratio representation. */ static PyObject * -dec_as_integer_ratio(PyObject *self, PyObject *args UNUSED) +dec_as_integer_ratio(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *numerator = NULL; PyObject *denominator = NULL; @@ -3952,7 +3972,7 @@ PyDec_Round(PyObject *dec, PyObject *args) /* Return the DecimalTuple representation of a PyDecObject. */ static PyObject * -PyDec_AsTuple(PyObject *dec, PyObject *dummy UNUSED) +PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) { PyObject *result = NULL; PyObject *sign = NULL; @@ -4108,7 +4128,7 @@ nm_##MPDFUNC(PyObject *self, PyObject *other) \ /* Boolean function without a context arg. */ #define Dec_BoolFunc(MPDFUNC) \ static PyObject * \ -dec_##MPDFUNC(PyObject *self, PyObject *dummy UNUSED) \ +dec_##MPDFUNC(PyObject *self, PyObject *Py_UNUSED(dummy)) \ { \ return MPDFUNC(MPD(self)) ? incr_true() : incr_false(); \ } @@ -4341,7 +4361,7 @@ nm_mpd_qdivmod(PyObject *v, PyObject *w) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; @@ -4437,7 +4457,7 @@ Dec_BoolFuncVA(mpd_issubnormal) /* Unary functions, no context arg */ static PyObject * -dec_mpd_adjexp(PyObject *self, PyObject *dummy UNUSED) +dec_mpd_adjexp(PyObject *self, PyObject *Py_UNUSED(dummy)) { mpd_ssize_t retval; @@ -4452,23 +4472,22 @@ dec_mpd_adjexp(PyObject *self, PyObject *dummy UNUSED) } static PyObject * -dec_canonical(PyObject *self, PyObject *dummy UNUSED) +dec_canonical(PyObject *self, PyObject *Py_UNUSED(dummy)) { return Py_NewRef(self); } static PyObject * -dec_conjugate(PyObject *self, PyObject *dummy UNUSED) +dec_conjugate(PyObject *self, PyObject *Py_UNUSED(dummy)) { return Py_NewRef(self); } -static PyObject * -dec_mpd_radix(PyObject *self, PyObject *dummy UNUSED) +static inline PyObject * +_dec_mpd_radix(decimal_state *state) { PyObject *result; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -4479,7 +4498,14 @@ dec_mpd_radix(PyObject *self, PyObject *dummy UNUSED) } static PyObject * -dec_mpd_qcopy_abs(PyObject *self, PyObject *dummy UNUSED) +dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +{ + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + return _dec_mpd_radix(state); +} + +static PyObject * +dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *result; uint32_t status = 0; @@ -4500,7 +4526,7 @@ dec_mpd_qcopy_abs(PyObject *self, PyObject *dummy UNUSED) } static PyObject * -dec_mpd_qcopy_negate(PyObject *self, PyObject *dummy UNUSED) +dec_mpd_qcopy_negate(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *result; uint32_t status = 0; @@ -4754,7 +4780,7 @@ dec_richcompare(PyObject *v, PyObject *w, int op) /* __ceil__ */ static PyObject * -dec_ceil(PyObject *self, PyObject *dummy UNUSED) +dec_ceil(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *context; @@ -4765,7 +4791,7 @@ dec_ceil(PyObject *self, PyObject *dummy UNUSED) /* __complex__ */ static PyObject * -dec_complex(PyObject *self, PyObject *dummy UNUSED) +dec_complex(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *f; double x; @@ -4784,16 +4810,16 @@ dec_complex(PyObject *self, PyObject *dummy UNUSED) return PyComplex_FromDoubles(x, 0); } -/* __copy__ and __deepcopy__ */ +/* __copy__ (METH_NOARGS) and __deepcopy__ (METH_O) */ static PyObject * -dec_copy(PyObject *self, PyObject *dummy UNUSED) +dec_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { return Py_NewRef(self); } /* __floor__ */ static PyObject * -dec_floor(PyObject *self, PyObject *dummy UNUSED) +dec_floor(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *context; @@ -4844,7 +4870,7 @@ _dec_hash(PyDecObject *v) return -1; } else if (mpd_isnan(MPD(v))) { - return _Py_HashPointer(v); + return PyObject_GenericHash((PyObject *)v); } else { return py_hash_inf * mpd_arith_sign(MPD(v)); @@ -4930,7 +4956,7 @@ dec_hash(PyDecObject *self) /* __reduce__ */ static PyObject * -dec_reduce(PyObject *self, PyObject *dummy UNUSED) +dec_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *result, *str; @@ -4947,7 +4973,7 @@ dec_reduce(PyObject *self, PyObject *dummy UNUSED) /* __sizeof__ */ static PyObject * -dec_sizeof(PyObject *v, PyObject *dummy UNUSED) +dec_sizeof(PyObject *v, PyObject *Py_UNUSED(dummy)) { size_t res = _PyObject_SIZE(Py_TYPE(v)); if (mpd_isdynamic_data(MPD(v))) { @@ -4958,7 +4984,7 @@ dec_sizeof(PyObject *v, PyObject *dummy UNUSED) /* __trunc__ */ static PyObject * -dec_trunc(PyObject *self, PyObject *dummy UNUSED) +dec_trunc(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *context; @@ -4969,13 +4995,13 @@ dec_trunc(PyObject *self, PyObject *dummy UNUSED) /* real and imag */ static PyObject * -dec_real(PyObject *self, void *closure UNUSED) +dec_real(PyObject *self, void *Py_UNUSED(closure)) { return Py_NewRef(self); } static PyObject * -dec_imag(PyObject *self UNUSED, void *closure UNUSED) +dec_imag(PyObject *self, void *Py_UNUSED(closure)) { PyObject *result; @@ -5071,6 +5097,7 @@ static PyMethodDef dec_methods [] = /* Miscellaneous */ { "from_float", dec_from_float, METH_O|METH_CLASS, doc_from_float }, + { "from_number", dec_from_number, METH_O|METH_CLASS, doc_from_number }, { "as_tuple", PyDec_AsTuple, METH_NOARGS, doc_as_tuple }, { "as_integer_ratio", dec_as_integer_ratio, METH_NOARGS, doc_as_integer_ratio }, @@ -5090,6 +5117,7 @@ static PyMethodDef dec_methods [] = }; static PyType_Slot dec_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, dec_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, dec_traverse}, @@ -5179,7 +5207,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *v) \ \ CONVERT_OP_RAISE(&a, v, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ return NULL; \ @@ -5211,7 +5239,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5247,7 +5275,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5276,7 +5304,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ } \ \ CONVERT_TERNOP_RAISE(&a, &b, &c, v, w, x, context); \ - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); \ + decimal_state *state = get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5342,7 +5370,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); q = dec_alloc(state); if (q == NULL) { Py_DECREF(a); @@ -5366,7 +5394,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; @@ -5397,7 +5425,7 @@ ctx_mpd_qpow(PyObject *context, PyObject *args, PyObject *kwds) } } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5432,7 +5460,8 @@ DecCtx_TernaryFunc(mpd_qfma) static PyObject * ctx_mpd_radix(PyObject *context, PyObject *dummy) { - return dec_mpd_radix(context, dummy); + decimal_state *state = get_module_state_from_ctx(context); + return _dec_mpd_radix(state); } /* Boolean functions: single decimal argument */ @@ -5449,7 +5478,7 @@ DecCtx_BoolFunc_NO_CTX(mpd_iszero) static PyObject * ctx_iscanonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5475,7 +5504,7 @@ PyDecContext_Apply(PyObject *context, PyObject *v) static PyObject * ctx_canonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5492,7 +5521,7 @@ ctx_mpd_qcopy_abs(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5525,7 +5554,7 @@ ctx_mpd_qcopy_negate(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5622,7 +5651,7 @@ ctx_mpd_qcopy_sign(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5767,9 +5796,9 @@ static PyMethodDef context_methods [] = #endif /* Miscellaneous */ - { "__copy__", (PyCFunction)context_copy, METH_NOARGS, NULL }, + { "__copy__", context_copy, METH_NOARGS, NULL }, { "__reduce__", context_reduce, METH_NOARGS, NULL }, - { "copy", (PyCFunction)context_copy, METH_NOARGS, doc_ctx_copy }, + { "copy", context_copy, METH_NOARGS, doc_ctx_copy }, { "create_decimal", ctx_create_decimal, METH_VARARGS, doc_ctx_create_decimal }, { "create_decimal_from_float", ctx_from_float, METH_O, doc_ctx_create_decimal_from_float }, @@ -5777,6 +5806,7 @@ static PyMethodDef context_methods [] = }; static PyType_Slot context_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, context_dealloc}, {Py_tp_traverse, context_traverse}, {Py_tp_clear, context_clear}, @@ -5802,11 +5832,11 @@ static PyType_Spec context_spec = { static PyMethodDef _decimal_methods [] = { - { "getcontext", (PyCFunction)PyDec_GetCurrentContext, METH_NOARGS, doc_getcontext}, - { "setcontext", (PyCFunction)PyDec_SetCurrentContext, METH_O, doc_setcontext}, + { "getcontext", PyDec_GetCurrentContext, METH_NOARGS, doc_getcontext}, + { "setcontext", PyDec_SetCurrentContext, METH_O, doc_setcontext}, { "localcontext", _PyCFunction_CAST(ctxmanager_new), METH_VARARGS|METH_KEYWORDS, doc_localcontext}, #ifdef EXTRA_FUNCTIONALITY - { "IEEEContext", (PyCFunction)ieee_context, METH_O, doc_ieee_context}, + { "IEEEContext", ieee_context, METH_O, doc_ieee_context}, #endif { NULL, NULL, 1, NULL } }; @@ -5987,6 +6017,9 @@ _decimal_exec(PyObject *m) Py_CLEAR(collections_abc); Py_CLEAR(MutableMapping); + /* For format specifiers not yet supported by libmpdec */ + state->PyDecimal = NULL; + /* Add types to the module */ CHECK_INT(PyModule_AddType(m, state->PyDec_Type)); CHECK_INT(PyModule_AddType(m, state->PyDecContext_Type)); @@ -6165,6 +6198,16 @@ decimal_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->Rational); Py_VISIT(state->SignalTuple); + if (state->signal_map != NULL) { + for (DecCondMap *cm = state->signal_map; cm->name != NULL; cm++) { + Py_VISIT(cm->ex); + } + } + if (state->cond_map != NULL) { + for (DecCondMap *cm = state->cond_map + 1; cm->name != NULL; cm++) { + Py_VISIT(cm->ex); + } + } return 0; } @@ -6192,9 +6235,24 @@ decimal_clear(PyObject *module) Py_CLEAR(state->extended_context_template); Py_CLEAR(state->Rational); Py_CLEAR(state->SignalTuple); + Py_CLEAR(state->PyDecimal); - PyMem_Free(state->signal_map); - PyMem_Free(state->cond_map); + if (state->signal_map != NULL) { + for (DecCondMap *cm = state->signal_map; cm->name != NULL; cm++) { + Py_DECREF(cm->ex); + } + PyMem_Free(state->signal_map); + state->signal_map = NULL; + } + + if (state->cond_map != NULL) { + // cond_map[0].ex has borrowed a reference from signal_map[0].ex + for (DecCondMap *cm = state->cond_map + 1; cm->name != NULL; cm++) { + Py_DECREF(cm->ex); + } + PyMem_Free(state->cond_map); + state->cond_map = NULL; + } return 0; } @@ -6207,6 +6265,7 @@ decimal_free(void *module) static struct PyModuleDef_Slot _decimal_slots[] = { {Py_mod_exec, _decimal_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_decimal/docstrings.h b/Modules/_decimal/docstrings.h index a1823cdd32b74c..b34bff83d3f4e9 100644 --- a/Modules/_decimal/docstrings.h +++ b/Modules/_decimal/docstrings.h @@ -189,6 +189,19 @@ Decimal.from_float(0.1) is not the same as Decimal('0.1').\n\ \n\ \n"); +PyDoc_STRVAR(doc_from_number, +"from_number($type, number, /)\n--\n\n\ +Class method that converts a real number to a decimal number, exactly.\n\ +\n\ + >>> Decimal.from_number(314) # int\n\ + Decimal('314')\n\ + >>> Decimal.from_number(0.1) # float\n\ + Decimal('0.1000000000000000055511151231257827021181583404541015625')\n\ + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance\n\ + Decimal('3.14')\n\ +\n\ +\n"); + PyDoc_STRVAR(doc_fma, "fma($self, /, other, third, context=None)\n--\n\n\ Fused multiply-add. Return self*other+third with no rounding of the\n\ diff --git a/Modules/_decimal/libmpdec/io.c b/Modules/_decimal/libmpdec/io.c index e7bd6aee170056..4e95b8964c8e5d 100644 --- a/Modules/_decimal/libmpdec/io.c +++ b/Modules/_decimal/libmpdec/io.c @@ -48,6 +48,7 @@ #if defined(__GNUC__) && !defined(__INTEL_COMPILER) && __GNUC__ >= 7 #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #pragma GCC diagnostic ignored "-Wmisleading-indentation" + #pragma GCC diagnostic ignored "-Warray-bounds" #endif diff --git a/Modules/_decimal/windows/mpdecimal.h b/Modules/_decimal/windows/mpdecimal.h new file mode 100644 index 00000000000000..77bc6229fbc119 --- /dev/null +++ b/Modules/_decimal/windows/mpdecimal.h @@ -0,0 +1,17 @@ +/* Windows mpdecimal.h shim + * + * Generally, the mpdecimal library build will copy the correct header into + * place named "mpdecimal.h", but since we're building it ourselves directly + * into _decimal.pyd, we need to pick the right one. + * + * */ + +#if defined(_MSC_VER) + #if defined(CONFIG_64) + #include + #elif defined(CONFIG_32) + #include + #else + #error "Unknown configuration!" + #endif +#endif diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 54451081211654..e134e096e044b7 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -196,7 +196,7 @@ list_join(PyObject* list) PyObject* joiner; PyObject* result; - joiner = PyUnicode_FromStringAndSize("", 0); + joiner = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (!joiner) return NULL; result = PyUnicode_Join(joiner, list); @@ -372,33 +372,27 @@ element_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject* get_attrib_from_keywords(PyObject *kwds) { - PyObject *attrib_str = PyUnicode_FromString("attrib"); - if (attrib_str == NULL) { + PyObject *attrib; + if (PyDict_PopString(kwds, "attrib", &attrib) < 0) { return NULL; } - PyObject *attrib = PyDict_GetItemWithError(kwds, attrib_str); if (attrib) { /* If attrib was found in kwds, copy its value and remove it from * kwds */ if (!PyDict_Check(attrib)) { - Py_DECREF(attrib_str); PyErr_Format(PyExc_TypeError, "attrib must be dict, not %.100s", Py_TYPE(attrib)->tp_name); + Py_DECREF(attrib); return NULL; } - attrib = PyDict_Copy(attrib); - if (attrib && PyDict_DelItem(kwds, attrib_str) < 0) { - Py_SETREF(attrib, NULL); - } + Py_SETREF(attrib, PyDict_Copy(attrib)); } - else if (!PyErr_Occurred()) { + else { attrib = PyDict_New(); } - Py_DECREF(attrib_str); - if (attrib != NULL && PyDict_Update(attrib, kwds) < 0) { Py_DECREF(attrib); return NULL; @@ -1219,12 +1213,8 @@ _elementtree_Element_extend_impl(ElementObject *self, PyTypeObject *cls, PyObject* seq; Py_ssize_t i; - seq = PySequence_Fast(elements, ""); + seq = PySequence_Fast(elements, "'elements' must be an iterable"); if (!seq) { - PyErr_Format( - PyExc_TypeError, - "expected sequence, not \"%.200s\"", Py_TYPE(elements)->tp_name - ); return NULL; } @@ -1327,7 +1317,7 @@ _elementtree_Element_findtext_impl(ElementObject *self, PyTypeObject *cls, PyObject* text = element_get_text((ElementObject*)item); if (text == Py_None) { Py_DECREF(item); - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } Py_XINCREF(text); Py_DECREF(item); @@ -1508,7 +1498,7 @@ element_bool(PyObject* self_) { ElementObject* self = (ElementObject*) self_; if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Testing an element's truth value will raise an exception " + "Testing an element's truth value will always return True " "in future versions. Use specific 'len(elem)' or " "'elem is not None' test instead.", 1) < 0) { @@ -1924,12 +1914,8 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) } /* A new slice is actually being assigned */ - seq = PySequence_Fast(value, ""); + seq = PySequence_Fast(value, "assignment expects an iterable"); if (!seq) { - PyErr_Format( - PyExc_TypeError, - "expected sequence, not \"%.200s\"", Py_TYPE(value)->tp_name - ); return -1; } newlen = PySequence_Fast_GET_SIZE(seq); @@ -3894,6 +3880,40 @@ _elementtree_XMLParser_close_impl(XMLParserObject *self) } } +/*[clinic input] +_elementtree.XMLParser.flush + +[clinic start generated code]*/ + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self) +/*[clinic end generated code: output=42fdb8795ca24509 input=effbecdb28715949]*/ +{ + if (!_check_xmlparser(self)) { + return NULL; + } + + elementtreestate *st = self->state; + + if (EXPAT(st, SetReparseDeferralEnabled) == NULL) { + Py_RETURN_NONE; + } + + // NOTE: The Expat parser in the C implementation of ElementTree is not + // exposed to the outside; as a result we known that reparse deferral + // is currently enabled, or we would not even have access to function + // XML_SetReparseDeferralEnabled in the first place (which we checked + // for, a few lines up). + + EXPAT(st, SetReparseDeferralEnabled)(self->parser, XML_FALSE); + + PyObject *res = expat_parse(st, self, "", 0, XML_FALSE); + + EXPAT(st, SetReparseDeferralEnabled)(self->parser, XML_TRUE); + + return res; +} + /*[clinic input] _elementtree.XMLParser.feed @@ -4288,6 +4308,7 @@ static PyType_Spec treebuilder_spec = { static PyMethodDef xmlparser_methods[] = { _ELEMENTTREE_XMLPARSER_FEED_METHODDEF _ELEMENTTREE_XMLPARSER_CLOSE_METHODDEF + _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF _ELEMENTTREE_XMLPARSER__PARSE_WHOLE_METHODDEF _ELEMENTTREE_XMLPARSER__SETEVENTS_METHODDEF {NULL, NULL} @@ -4434,6 +4455,7 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 9ab847165dc097..d2afe1a1bea018 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -25,6 +25,8 @@ class _functools._lru_cache_wrapper "PyObject *" "&lru_cache_type_spec" typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; + PyTypeObject *placeholder_type; + PyObject *placeholder; // strong reference (singleton) PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -41,6 +43,95 @@ get_functools_state(PyObject *module) /* partial object **********************************************************/ + +// The 'Placeholder' singleton indicates which formal positional +// parameters are to be bound first when using a 'partial' object. + +typedef struct { + PyObject_HEAD +} placeholderobject; + +static inline _functools_state * +get_functools_state_by_type(PyTypeObject *type); + +PyDoc_STRVAR(placeholder_doc, +"The type of the Placeholder singleton.\n\n" +"Used as a placeholder for partial arguments."); + +static PyObject * +placeholder_repr(PyObject *op) +{ + return PyUnicode_FromString("Placeholder"); +} + +static PyObject * +placeholder_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("Placeholder"); +} + +static PyMethodDef placeholder_methods[] = { + {"__reduce__", placeholder_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static void +placeholder_dealloc(PyObject* self) +{ + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject*)self); + Py_DECREF(tp); +} + +static PyObject * +placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) { + PyErr_SetString(PyExc_TypeError, "PlaceholderType takes no arguments"); + return NULL; + } + _functools_state *state = get_functools_state_by_type(type); + if (state->placeholder != NULL) { + return Py_NewRef(state->placeholder); + } + + PyObject *placeholder = PyType_GenericNew(type, NULL, NULL); + if (placeholder == NULL) { + return NULL; + } + + if (state->placeholder == NULL) { + state->placeholder = Py_NewRef(placeholder); + } + return placeholder; +} + +static int +placeholder_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static PyType_Slot placeholder_type_slots[] = { + {Py_tp_dealloc, placeholder_dealloc}, + {Py_tp_repr, placeholder_repr}, + {Py_tp_doc, (void *)placeholder_doc}, + {Py_tp_methods, placeholder_methods}, + {Py_tp_new, placeholder_new}, + {Py_tp_traverse, placeholder_traverse}, + {0, 0} +}; + +static PyType_Spec placeholder_type_spec = { + .name = "functools._PlaceholderType", + .basicsize = sizeof(placeholderobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, + .slots = placeholder_type_slots +}; + + typedef struct { PyObject_HEAD PyObject *fn; @@ -48,13 +139,18 @@ typedef struct { PyObject *kw; PyObject *dict; /* __dict__ */ PyObject *weakreflist; /* List of weak references */ + PyObject *placeholder; /* Placeholder for positional arguments */ + Py_ssize_t phcount; /* Number of placeholders */ vectorcallfunc vectorcall; } partialobject; +// cast a PyObject pointer PTR to a partialobject pointer (no type checks) +#define _PyPartialObject_CAST(PTR) ((partialobject *)(PTR)) + static void partial_setvectorcall(partialobject *pto); static struct PyModuleDef _functools_module; static PyObject * -partial_call(partialobject *pto, PyObject *args, PyObject *kwargs); +partial_call(PyObject *pto, PyObject *args, PyObject *kwargs); static inline _functools_state * get_functools_state_by_type(PyTypeObject *type) @@ -70,36 +166,54 @@ get_functools_state_by_type(PyTypeObject *type) static PyObject * partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *func, *pargs, *nargs, *pkw; + PyObject *func, *pto_args, *new_args, *pto_kw, *phold; partialobject *pto; + Py_ssize_t pto_phcount = 0; + Py_ssize_t new_nargs = PyTuple_GET_SIZE(args) - 1; - if (PyTuple_GET_SIZE(args) < 1) { + if (new_nargs < 0) { PyErr_SetString(PyExc_TypeError, "type 'partial' takes at least one argument"); return NULL; } - - pargs = pkw = NULL; func = PyTuple_GET_ITEM(args, 0); - if (Py_TYPE(func)->tp_call == (ternaryfunc)partial_call) { - // The type of "func" might not be exactly the same type object - // as "type", but if it is called using partial_call, it must have the - // same memory layout (fn, args and kw members). + if (!PyCallable_Check(func)) { + PyErr_SetString(PyExc_TypeError, + "the first argument must be callable"); + return NULL; + } + + _functools_state *state = get_functools_state_by_type(type); + if (state == NULL) { + return NULL; + } + phold = state->placeholder; + + /* Placeholder restrictions */ + if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == phold) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + + /* check wrapped function / object */ + pto_args = pto_kw = NULL; + int res = PyObject_TypeCheck(func, state->partial_type); + if (res == -1) { + return NULL; + } + if (res == 1) { // We can use its underlying function directly and merge the arguments. partialobject *part = (partialobject *)func; if (part->dict == NULL) { - pargs = part->args; - pkw = part->kw; + pto_args = part->args; + pto_kw = part->kw; func = part->fn; - assert(PyTuple_Check(pargs)); - assert(PyDict_Check(pkw)); + pto_phcount = part->phcount; + assert(PyTuple_Check(pto_args)); + assert(PyDict_Check(pto_kw)); } } - if (!PyCallable_Check(func)) { - PyErr_SetString(PyExc_TypeError, - "the first argument must be callable"); - return NULL; - } /* create partialobject structure */ pto = (partialobject *)type->tp_alloc(type, 0); @@ -107,18 +221,58 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; pto->fn = Py_NewRef(func); + pto->placeholder = phold; - nargs = PyTuple_GetSlice(args, 1, PY_SSIZE_T_MAX); - if (nargs == NULL) { + new_args = PyTuple_GetSlice(args, 1, new_nargs + 1); + if (new_args == NULL) { Py_DECREF(pto); return NULL; } - if (pargs == NULL) { - pto->args = nargs; + + /* Count placeholders */ + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { + if (PyTuple_GET_ITEM(new_args, i) == phold) { + phcount++; + } + } + /* merge args with args of `func` which is `partial` */ + if (pto_phcount > 0 && new_nargs > 0) { + Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); + Py_ssize_t tot_nargs = npargs; + if (new_nargs > pto_phcount) { + tot_nargs += new_nargs - pto_phcount; + } + PyObject *item; + PyObject *tot_args = PyTuple_New(tot_nargs); + for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { + if (i < npargs) { + item = PyTuple_GET_ITEM(pto_args, i); + if (j < new_nargs && item == phold) { + item = PyTuple_GET_ITEM(new_args, j); + j++; + pto_phcount--; + } + } + else { + item = PyTuple_GET_ITEM(new_args, j); + j++; + } + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + } + pto->args = tot_args; + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); + } + else if (pto_args == NULL) { + pto->args = new_args; + pto->phcount = phcount; } else { - pto->args = PySequence_Concat(pargs, nargs); - Py_DECREF(nargs); + pto->args = PySequence_Concat(pto_args, new_args); + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); if (pto->args == NULL) { Py_DECREF(pto); return NULL; @@ -126,7 +280,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) assert(PyTuple_Check(pto->args)); } - if (pkw == NULL || PyDict_GET_SIZE(pkw) == 0) { + if (pto_kw == NULL || PyDict_GET_SIZE(pto_kw) == 0) { if (kw == NULL) { pto->kw = PyDict_New(); } @@ -138,7 +292,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } } else { - pto->kw = PyDict_Copy(pkw); + pto->kw = PyDict_Copy(pto_kw); if (kw != NULL && pto->kw != NULL) { if (PyDict_Merge(pto->kw, kw, 1) != 0) { Py_DECREF(pto); @@ -156,8 +310,9 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } static int -partial_clear(partialobject *pto) +partial_clear(PyObject *self) { + partialobject *pto = _PyPartialObject_CAST(self); Py_CLEAR(pto->fn); Py_CLEAR(pto->args); Py_CLEAR(pto->kw); @@ -166,8 +321,9 @@ partial_clear(partialobject *pto) } static int -partial_traverse(partialobject *pto, visitproc visit, void *arg) +partial_traverse(PyObject *self, visitproc visit, void *arg) { + partialobject *pto = _PyPartialObject_CAST(self); Py_VISIT(Py_TYPE(pto)); Py_VISIT(pto->fn); Py_VISIT(pto->args); @@ -177,19 +333,27 @@ partial_traverse(partialobject *pto, visitproc visit, void *arg) } static void -partial_dealloc(partialobject *pto) +partial_dealloc(PyObject *self) { - PyTypeObject *tp = Py_TYPE(pto); + PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(pto); - if (pto->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) pto); + PyObject_GC_UnTrack(self); + if (_PyPartialObject_CAST(self)->weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } - (void)partial_clear(pto); - tp->tp_free(pto); + (void)partial_clear(self); + tp->tp_free(self); Py_DECREF(tp); } +static PyObject * +partial_descr_get(PyObject *self, PyObject *obj, PyObject *type) +{ + if (obj == Py_None || obj == NULL) { + return Py_NewRef(self); + } + return PyMethod_New(self, obj); +} /* Merging keyword arguments using the vectorcall convention is messy, so * if we would need to do that, we stop using vectorcall and fall back @@ -201,32 +365,39 @@ partial_vectorcall_fallback(PyThreadState *tstate, partialobject *pto, { pto->vectorcall = NULL; Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - return _PyObject_MakeTpCall(tstate, (PyObject *)pto, - args, nargs, kwnames); + return _PyObject_MakeTpCall(tstate, (PyObject *)pto, args, nargs, kwnames); } static PyObject * -partial_vectorcall(partialobject *pto, PyObject *const *args, +partial_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + partialobject *pto = _PyPartialObject_CAST(self);; PyThreadState *tstate = _PyThreadState_GET(); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); /* pto->kw is mutable, so need to check every time */ if (PyDict_GET_SIZE(pto->kw)) { return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames); } + Py_ssize_t pto_phcount = pto->phcount; + if (nargs < pto_phcount) { + PyErr_Format(PyExc_TypeError, + "missing positional arguments in 'partial' call; " + "expected at least %zd, got %zd", pto_phcount, nargs); + return NULL; + } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - Py_ssize_t nargs_total = nargs; + Py_ssize_t nargskw = nargs; if (kwnames != NULL) { - nargs_total += PyTuple_GET_SIZE(kwnames); + nargskw += PyTuple_GET_SIZE(kwnames); } PyObject **pto_args = _PyTuple_ITEMS(pto->args); Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); /* Fast path if we're called without arguments */ - if (nargs_total == 0) { + if (nargskw == 0) { return _PyObject_VectorcallTstate(tstate, pto->fn, pto_args, pto_nargs, NULL); } @@ -243,29 +414,47 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, return ret; } - Py_ssize_t newnargs_total = pto_nargs + nargs_total; - PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; - PyObject *ret; PyObject **stack; - if (newnargs_total <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + Py_ssize_t tot_nargskw = pto_nargs + nargskw - pto_phcount; + if (tot_nargskw <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { stack = small_stack; } else { - stack = PyMem_Malloc(newnargs_total * sizeof(PyObject *)); + stack = PyMem_Malloc(tot_nargskw * sizeof(PyObject *)); if (stack == NULL) { PyErr_NoMemory(); return NULL; } } - /* Copy to new stack, using borrowed references */ - memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); - memcpy(stack + pto_nargs, args, nargs_total * sizeof(PyObject*)); - - ret = _PyObject_VectorcallTstate(tstate, pto->fn, - stack, pto_nargs + nargs, kwnames); + Py_ssize_t tot_nargs; + if (pto_phcount) { + tot_nargs = pto_nargs + nargs - pto_phcount; + Py_ssize_t j = 0; // New args index + for (Py_ssize_t i = 0; i < pto_nargs; i++) { + if (pto_args[i] == pto->placeholder) { + stack[i] = args[j]; + j += 1; + } + else { + stack[i] = pto_args[i]; + } + } + assert(j == pto_phcount); + if (nargskw > pto_phcount) { + memcpy(stack + pto_nargs, args + j, (nargskw - j) * sizeof(PyObject*)); + } + } + else { + tot_nargs = pto_nargs + nargs; + /* Copy to new stack, using borrowed references */ + memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); + memcpy(stack + pto_nargs, args, nargskw * sizeof(PyObject*)); + } + PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, + stack, tot_nargs, kwnames); if (stack != small_stack) { PyMem_Free(stack); } @@ -284,59 +473,102 @@ partial_setvectorcall(partialobject *pto) * but that is unlikely (why use partial without arguments?), * so we don't optimize that */ else { - pto->vectorcall = (vectorcallfunc)partial_vectorcall; + pto->vectorcall = partial_vectorcall; } } // Not converted to argument clinic, because of `*args, **kwargs` arguments. static PyObject * -partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) +partial_call(PyObject *self, PyObject *args, PyObject *kwargs) { + partialobject *pto = _PyPartialObject_CAST(self); assert(PyCallable_Check(pto->fn)); assert(PyTuple_Check(pto->args)); assert(PyDict_Check(pto->kw)); + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t pto_phcount = pto->phcount; + if (nargs < pto_phcount) { + PyErr_Format(PyExc_TypeError, + "missing positional arguments in 'partial' call; " + "expected at least %zd, got %zd", pto_phcount, nargs); + return NULL; + } + /* Merge keywords */ - PyObject *kwargs2; + PyObject *tot_kw; if (PyDict_GET_SIZE(pto->kw) == 0) { /* kwargs can be NULL */ - kwargs2 = Py_XNewRef(kwargs); + tot_kw = Py_XNewRef(kwargs); } else { /* bpo-27840, bpo-29318: dictionary of keyword parameters must be copied, because a function using "**kwargs" can modify the dictionary. */ - kwargs2 = PyDict_Copy(pto->kw); - if (kwargs2 == NULL) { + tot_kw = PyDict_Copy(pto->kw); + if (tot_kw == NULL) { return NULL; } if (kwargs != NULL) { - if (PyDict_Merge(kwargs2, kwargs, 1) != 0) { - Py_DECREF(kwargs2); + if (PyDict_Merge(tot_kw, kwargs, 1) != 0) { + Py_DECREF(tot_kw); return NULL; } } } /* Merge positional arguments */ - /* Note: tupleconcat() is optimized for empty tuples */ - PyObject *args2 = PySequence_Concat(pto->args, args); - if (args2 == NULL) { - Py_XDECREF(kwargs2); - return NULL; + PyObject *tot_args; + if (pto_phcount) { + Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); + Py_ssize_t tot_nargs = pto_nargs + nargs - pto_phcount; + assert(tot_nargs >= 0); + tot_args = PyTuple_New(tot_nargs); + if (tot_args == NULL) { + Py_XDECREF(tot_kw); + return NULL; + } + PyObject *pto_args = pto->args; + PyObject *item; + Py_ssize_t j = 0; // New args index + for (Py_ssize_t i = 0; i < pto_nargs; i++) { + item = PyTuple_GET_ITEM(pto_args, i); + if (item == pto->placeholder) { + item = PyTuple_GET_ITEM(args, j); + j += 1; + } + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + } + assert(j == pto_phcount); + for (Py_ssize_t i = pto_nargs; i < tot_nargs; i++) { + item = PyTuple_GET_ITEM(args, j); + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + j += 1; + } + } + else { + /* Note: tupleconcat() is optimized for empty tuples */ + tot_args = PySequence_Concat(pto->args, args); + if (tot_args == NULL) { + Py_XDECREF(tot_kw); + return NULL; + } } - PyObject *res = PyObject_Call(pto->fn, args2, kwargs2); - Py_DECREF(args2); - Py_XDECREF(kwargs2); + PyObject *res = PyObject_Call(pto->fn, tot_args, tot_kw); + Py_DECREF(tot_args); + Py_XDECREF(tot_kw); return res; } PyDoc_STRVAR(partial_doc, -"partial(func, *args, **keywords) - new function with partial application\n\ - of the given arguments and keywords.\n"); +"partial(func, /, *args, **keywords)\n--\n\n\ +Create a new function with partial application of the given arguments\n\ +and keywords."); #define OFF(x) offsetof(partialobject, x) static PyMemberDef partial_memberlist[] = { @@ -361,26 +593,29 @@ static PyGetSetDef partial_getsetlist[] = { }; static PyObject * -partial_repr(partialobject *pto) +partial_repr(PyObject *self) { + partialobject *pto = _PyPartialObject_CAST(self); PyObject *result = NULL; PyObject *arglist; + PyObject *mod; + PyObject *name; Py_ssize_t i, n; PyObject *key, *value; int status; - status = Py_ReprEnter((PyObject *)pto); + status = Py_ReprEnter(self); if (status != 0) { if (status < 0) return NULL; return PyUnicode_FromString("..."); } - arglist = PyUnicode_FromString(""); + arglist = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (arglist == NULL) goto done; /* Pack positional arguments */ - assert (PyTuple_Check(pto->args)); + assert(PyTuple_Check(pto->args)); n = PyTuple_GET_SIZE(pto->args); for (i = 0; i < n; i++) { Py_SETREF(arglist, PyUnicode_FromFormat("%U, %R", arglist, @@ -399,13 +634,28 @@ partial_repr(partialobject *pto) if (arglist == NULL) goto done; } - result = PyUnicode_FromFormat("%s(%R%U)", Py_TYPE(pto)->tp_name, - pto->fn, arglist); + + mod = PyType_GetModuleName(Py_TYPE(pto)); + if (mod == NULL) { + goto error; + } + name = PyType_GetQualName(Py_TYPE(pto)); + if (name == NULL) { + Py_DECREF(mod); + goto error; + } + result = PyUnicode_FromFormat("%S.%S(%R%U)", mod, name, pto->fn, arglist); + Py_DECREF(mod); + Py_DECREF(name); Py_DECREF(arglist); done: - Py_ReprLeave((PyObject *)pto); + Py_ReprLeave(self); return result; + error: + Py_DECREF(arglist); + Py_ReprLeave(self); + return NULL; } /* Pickle strategy: @@ -416,20 +666,25 @@ partial_repr(partialobject *pto) */ static PyObject * -partial_reduce(partialobject *pto, PyObject *unused) +partial_reduce(PyObject *self, PyObject *Py_UNUSED(args)) { + partialobject *pto = _PyPartialObject_CAST(self); return Py_BuildValue("O(O)(OOOO)", Py_TYPE(pto), pto->fn, pto->fn, pto->args, pto->kw, pto->dict ? pto->dict : Py_None); } static PyObject * -partial_setstate(partialobject *pto, PyObject *state) +partial_setstate(PyObject *self, PyObject *state) { + partialobject *pto = _PyPartialObject_CAST(self); PyObject *fn, *fnargs, *kw, *dict; - if (!PyTuple_Check(state) || - !PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, "invalid partial state"); + return NULL; + } + if (!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || !PyCallable_Check(fn) || !PyTuple_Check(fnargs) || (kw != Py_None && !PyDict_Check(kw))) @@ -438,6 +693,20 @@ partial_setstate(partialobject *pto, PyObject *state) return NULL; } + Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs); + if (nargs && PyTuple_GET_ITEM(fnargs, nargs - 1) == pto->placeholder) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + /* Count placeholders */ + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < nargs - 1; i++) { + if (PyTuple_GET_ITEM(fnargs, i) == pto->placeholder) { + phcount++; + } + } + if(!PyTuple_CheckExact(fnargs)) fnargs = PySequence_Tuple(fnargs); else @@ -460,18 +729,18 @@ partial_setstate(partialobject *pto, PyObject *state) dict = NULL; else Py_INCREF(dict); - Py_SETREF(pto->fn, Py_NewRef(fn)); Py_SETREF(pto->args, fnargs); Py_SETREF(pto->kw, kw); + pto->phcount = phcount; Py_XSETREF(pto->dict, dict); partial_setvectorcall(pto); Py_RETURN_NONE; } static PyMethodDef partial_methods[] = { - {"__reduce__", (PyCFunction)partial_reduce, METH_NOARGS}, - {"__setstate__", (PyCFunction)partial_setstate, METH_O}, + {"__reduce__", partial_reduce, METH_NOARGS}, + {"__setstate__", partial_setstate, METH_O}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -489,6 +758,7 @@ static PyType_Slot partial_type_slots[] = { {Py_tp_methods, partial_methods}, {Py_tp_members, partial_memberlist}, {Py_tp_getset, partial_getsetlist}, + {Py_tp_descr_get, partial_descr_get}, {Py_tp_new, partial_new}, {Py_tp_free, PyObject_GC_Del}, {0, 0} @@ -546,6 +816,17 @@ static PyMemberDef keyobject_members[] = { {NULL} }; +static PyObject * +keyobject_text_signature(PyObject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("(obj)"); +} + +static PyGetSetDef keyobject_getset[] = { + {"__text_signature__", keyobject_text_signature, (setter)NULL}, + {NULL} +}; + static PyObject * keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds); @@ -560,6 +841,7 @@ static PyType_Slot keyobject_type_slots[] = { {Py_tp_clear, keyobject_clear}, {Py_tp_richcompare, keyobject_richcompare}, {Py_tp_members, keyobject_members}, + {Py_tp_getset, keyobject_getset}, {0, 0} }; @@ -650,15 +932,31 @@ _functools_cmp_to_key_impl(PyObject *module, PyObject *mycmp) /* reduce (used to be a builtin) ********************************************/ -// Not converted to argument clinic, because of `args` in-place modification. -// AC will affect performance. +/*[clinic input] +_functools.reduce + + function as func: object + iterable as seq: object + initial as result: object = NULL + / + +Apply a function of two arguments cumulatively to the items of an iterable, from left to right. + +This effectively reduces the iterable to a single value. If initial is present, +it is placed before the items of the iterable in the calculation, and serves as +a default when the iterable is empty. + +For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) +calculates ((((1 + 2) + 3) + 4) + 5). +[clinic start generated code]*/ + static PyObject * -functools_reduce(PyObject *self, PyObject *args) +_functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, + PyObject *result) +/*[clinic end generated code: output=30d898fe1267c79d input=d233c2670cba7f66]*/ { - PyObject *seq, *func, *result = NULL, *it; + PyObject *args, *it; - if (!PyArg_UnpackTuple(args, "reduce", 2, 3, &func, &seq, &result)) - return NULL; if (result != NULL) Py_INCREF(result); @@ -724,16 +1022,6 @@ functools_reduce(PyObject *self, PyObject *args) return NULL; } -PyDoc_STRVAR(functools_reduce_doc, -"reduce(function, iterable[, initial], /) -> value\n\ -\n\ -Apply a function of two arguments cumulatively to the items of a sequence\n\ -or iterable, from left to right, so as to reduce the iterable to a single\n\ -value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates\n\ -((((1+2)+3)+4)+5). If initial is present, it is placed before the items\n\ -of the iterable in the calculation, and serves as a default when the\n\ -iterable is empty."); - /* lru_cache object **********************************************************/ /* There are four principal algorithmic differences from the pure python version: @@ -1438,7 +1726,7 @@ PyDoc_STRVAR(_functools_doc, "Tools that operate on functions."); static PyMethodDef _functools_methods[] = { - {"reduce", functools_reduce, METH_VARARGS, functools_reduce_doc}, + _FUNCTOOLS_REDUCE_METHODDEF _FUNCTOOLS_CMP_TO_KEY_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1452,6 +1740,25 @@ _functools_exec(PyObject *module) return -1; } + state->placeholder_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &placeholder_type_spec, NULL); + if (state->placeholder_type == NULL) { + return -1; + } + if (PyModule_AddType(module, state->placeholder_type) < 0) { + return -1; + } + + PyObject *placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); + if (placeholder == NULL) { + return -1; + } + if (PyModule_AddObjectRef(module, "Placeholder", placeholder) < 0) { + Py_DECREF(placeholder); + return -1; + } + Py_DECREF(placeholder); + state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL); if (state->partial_type == NULL) { @@ -1496,6 +1803,8 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) { _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); + Py_VISIT(state->placeholder_type); + Py_VISIT(state->placeholder); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1507,6 +1816,8 @@ _functools_clear(PyObject *module) { _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); + Py_CLEAR(state->placeholder_type); + Py_CLEAR(state->placeholder); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); @@ -1522,6 +1833,7 @@ _functools_free(void *module) static struct PyModuleDef_Slot _functools_slots[] = { {Py_mod_exec, _functools_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index db868c18160fda..df7fba67810ed0 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -825,6 +825,7 @@ _gdbm_module_free(void *module) static PyModuleDef_Slot _gdbm_module_slots[] = { {Py_mod_exec, _gdbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.c b/Modules/_hacl/Hacl_Hash_Blake2b.c new file mode 100644 index 00000000000000..cd3b9777e09f6c --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b.c @@ -0,0 +1,1565 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_Blake2b.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "lib_memzero0.h" + +static void +update_block( + uint64_t *wv, + uint64_t *hash, + bool flag, + bool last_node, + FStar_UInt128_uint128 totlen, + uint8_t *d +) +{ + uint64_t m_w[16U] = { 0U }; + KRML_MAYBE_FOR16(i, + 0U, + 16U, + 1U, + uint64_t *os = m_w; + uint8_t *bj = d + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + uint64_t mask[4U] = { 0U }; + uint64_t wv_14; + if (flag) + { + wv_14 = 0xFFFFFFFFFFFFFFFFULL; + } + else + { + wv_14 = 0ULL; + } + uint64_t wv_15; + if (last_node) + { + wv_15 = 0xFFFFFFFFFFFFFFFFULL; + } + else + { + wv_15 = 0ULL; + } + mask[0U] = FStar_UInt128_uint128_to_uint64(totlen); + mask[1U] = FStar_UInt128_uint128_to_uint64(FStar_UInt128_shift_right(totlen, 64U)); + mask[2U] = wv_14; + mask[3U] = wv_15; + memcpy(wv, hash, 16U * sizeof (uint64_t)); + uint64_t *wv3 = wv + 12U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv3; + uint64_t x = wv3[i] ^ mask[i]; + os[i] = x;); + KRML_MAYBE_FOR12(i0, + 0U, + 12U, + 1U, + uint32_t start_idx = i0 % 10U * 16U; + uint64_t m_st[16U] = { 0U }; + uint64_t *r0 = m_st; + uint64_t *r1 = m_st + 4U; + uint64_t *r20 = m_st + 8U; + uint64_t *r30 = m_st + 12U; + uint32_t s0 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 0U]; + uint32_t s1 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 1U]; + uint32_t s2 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 2U]; + uint32_t s3 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 3U]; + uint32_t s4 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 4U]; + uint32_t s5 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 5U]; + uint32_t s6 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 6U]; + uint32_t s7 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 7U]; + uint32_t s8 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 8U]; + uint32_t s9 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 9U]; + uint32_t s10 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 10U]; + uint32_t s11 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 11U]; + uint32_t s12 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 12U]; + uint32_t s13 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 13U]; + uint32_t s14 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 14U]; + uint32_t s15 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 15U]; + uint64_t uu____0 = m_w[s2]; + uint64_t uu____1 = m_w[s4]; + uint64_t uu____2 = m_w[s6]; + r0[0U] = m_w[s0]; + r0[1U] = uu____0; + r0[2U] = uu____1; + r0[3U] = uu____2; + uint64_t uu____3 = m_w[s3]; + uint64_t uu____4 = m_w[s5]; + uint64_t uu____5 = m_w[s7]; + r1[0U] = m_w[s1]; + r1[1U] = uu____3; + r1[2U] = uu____4; + r1[3U] = uu____5; + uint64_t uu____6 = m_w[s10]; + uint64_t uu____7 = m_w[s12]; + uint64_t uu____8 = m_w[s14]; + r20[0U] = m_w[s8]; + r20[1U] = uu____6; + r20[2U] = uu____7; + r20[3U] = uu____8; + uint64_t uu____9 = m_w[s11]; + uint64_t uu____10 = m_w[s13]; + uint64_t uu____11 = m_w[s15]; + r30[0U] = m_w[s9]; + r30[1U] = uu____9; + r30[2U] = uu____10; + r30[3U] = uu____11; + uint64_t *x = m_st; + uint64_t *y = m_st + 4U; + uint64_t *z = m_st + 8U; + uint64_t *w = m_st + 12U; + uint32_t a = 0U; + uint32_t b0 = 1U; + uint32_t c0 = 2U; + uint32_t d10 = 3U; + uint64_t *wv_a0 = wv + a * 4U; + uint64_t *wv_b0 = wv + b0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a0; + uint64_t x1 = wv_a0[i] + wv_b0[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a0; + uint64_t x1 = wv_a0[i] + x[i]; + os[i] = x1;); + uint64_t *wv_a1 = wv + d10 * 4U; + uint64_t *wv_b1 = wv + a * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a1; + uint64_t x1 = wv_a1[i] ^ wv_b1[i]; + os[i] = x1;); + uint64_t *r10 = wv_a1; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r10; + uint64_t x1 = r10[i]; + uint64_t x10 = x1 >> 32U | x1 << 32U; + os[i] = x10;); + uint64_t *wv_a2 = wv + c0 * 4U; + uint64_t *wv_b2 = wv + d10 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a2; + uint64_t x1 = wv_a2[i] + wv_b2[i]; + os[i] = x1;); + uint64_t *wv_a3 = wv + b0 * 4U; + uint64_t *wv_b3 = wv + c0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a3; + uint64_t x1 = wv_a3[i] ^ wv_b3[i]; + os[i] = x1;); + uint64_t *r12 = wv_a3; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r12; + uint64_t x1 = r12[i]; + uint64_t x10 = x1 >> 24U | x1 << 40U; + os[i] = x10;); + uint64_t *wv_a4 = wv + a * 4U; + uint64_t *wv_b4 = wv + b0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a4; + uint64_t x1 = wv_a4[i] + wv_b4[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a4; + uint64_t x1 = wv_a4[i] + y[i]; + os[i] = x1;); + uint64_t *wv_a5 = wv + d10 * 4U; + uint64_t *wv_b5 = wv + a * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a5; + uint64_t x1 = wv_a5[i] ^ wv_b5[i]; + os[i] = x1;); + uint64_t *r13 = wv_a5; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r13; + uint64_t x1 = r13[i]; + uint64_t x10 = x1 >> 16U | x1 << 48U; + os[i] = x10;); + uint64_t *wv_a6 = wv + c0 * 4U; + uint64_t *wv_b6 = wv + d10 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a6; + uint64_t x1 = wv_a6[i] + wv_b6[i]; + os[i] = x1;); + uint64_t *wv_a7 = wv + b0 * 4U; + uint64_t *wv_b7 = wv + c0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a7; + uint64_t x1 = wv_a7[i] ^ wv_b7[i]; + os[i] = x1;); + uint64_t *r14 = wv_a7; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r14; + uint64_t x1 = r14[i]; + uint64_t x10 = x1 >> 63U | x1 << 1U; + os[i] = x10;); + uint64_t *r15 = wv + 4U; + uint64_t *r21 = wv + 8U; + uint64_t *r31 = wv + 12U; + uint64_t *r110 = r15; + uint64_t x00 = r110[1U]; + uint64_t x10 = r110[2U]; + uint64_t x20 = r110[3U]; + uint64_t x30 = r110[0U]; + r110[0U] = x00; + r110[1U] = x10; + r110[2U] = x20; + r110[3U] = x30; + uint64_t *r111 = r21; + uint64_t x01 = r111[2U]; + uint64_t x11 = r111[3U]; + uint64_t x21 = r111[0U]; + uint64_t x31 = r111[1U]; + r111[0U] = x01; + r111[1U] = x11; + r111[2U] = x21; + r111[3U] = x31; + uint64_t *r112 = r31; + uint64_t x02 = r112[3U]; + uint64_t x12 = r112[0U]; + uint64_t x22 = r112[1U]; + uint64_t x32 = r112[2U]; + r112[0U] = x02; + r112[1U] = x12; + r112[2U] = x22; + r112[3U] = x32; + uint32_t a0 = 0U; + uint32_t b = 1U; + uint32_t c = 2U; + uint32_t d1 = 3U; + uint64_t *wv_a = wv + a0 * 4U; + uint64_t *wv_b8 = wv + b * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a; + uint64_t x1 = wv_a[i] + wv_b8[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a; + uint64_t x1 = wv_a[i] + z[i]; + os[i] = x1;); + uint64_t *wv_a8 = wv + d1 * 4U; + uint64_t *wv_b9 = wv + a0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a8; + uint64_t x1 = wv_a8[i] ^ wv_b9[i]; + os[i] = x1;); + uint64_t *r16 = wv_a8; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r16; + uint64_t x1 = r16[i]; + uint64_t x13 = x1 >> 32U | x1 << 32U; + os[i] = x13;); + uint64_t *wv_a9 = wv + c * 4U; + uint64_t *wv_b10 = wv + d1 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a9; + uint64_t x1 = wv_a9[i] + wv_b10[i]; + os[i] = x1;); + uint64_t *wv_a10 = wv + b * 4U; + uint64_t *wv_b11 = wv + c * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a10; + uint64_t x1 = wv_a10[i] ^ wv_b11[i]; + os[i] = x1;); + uint64_t *r17 = wv_a10; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r17; + uint64_t x1 = r17[i]; + uint64_t x13 = x1 >> 24U | x1 << 40U; + os[i] = x13;); + uint64_t *wv_a11 = wv + a0 * 4U; + uint64_t *wv_b12 = wv + b * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a11; + uint64_t x1 = wv_a11[i] + wv_b12[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a11; + uint64_t x1 = wv_a11[i] + w[i]; + os[i] = x1;); + uint64_t *wv_a12 = wv + d1 * 4U; + uint64_t *wv_b13 = wv + a0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a12; + uint64_t x1 = wv_a12[i] ^ wv_b13[i]; + os[i] = x1;); + uint64_t *r18 = wv_a12; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r18; + uint64_t x1 = r18[i]; + uint64_t x13 = x1 >> 16U | x1 << 48U; + os[i] = x13;); + uint64_t *wv_a13 = wv + c * 4U; + uint64_t *wv_b14 = wv + d1 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a13; + uint64_t x1 = wv_a13[i] + wv_b14[i]; + os[i] = x1;); + uint64_t *wv_a14 = wv + b * 4U; + uint64_t *wv_b = wv + c * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a14; + uint64_t x1 = wv_a14[i] ^ wv_b[i]; + os[i] = x1;); + uint64_t *r19 = wv_a14; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r19; + uint64_t x1 = r19[i]; + uint64_t x13 = x1 >> 63U | x1 << 1U; + os[i] = x13;); + uint64_t *r113 = wv + 4U; + uint64_t *r2 = wv + 8U; + uint64_t *r3 = wv + 12U; + uint64_t *r11 = r113; + uint64_t x03 = r11[3U]; + uint64_t x13 = r11[0U]; + uint64_t x23 = r11[1U]; + uint64_t x33 = r11[2U]; + r11[0U] = x03; + r11[1U] = x13; + r11[2U] = x23; + r11[3U] = x33; + uint64_t *r114 = r2; + uint64_t x04 = r114[2U]; + uint64_t x14 = r114[3U]; + uint64_t x24 = r114[0U]; + uint64_t x34 = r114[1U]; + r114[0U] = x04; + r114[1U] = x14; + r114[2U] = x24; + r114[3U] = x34; + uint64_t *r115 = r3; + uint64_t x0 = r115[1U]; + uint64_t x1 = r115[2U]; + uint64_t x2 = r115[3U]; + uint64_t x3 = r115[0U]; + r115[0U] = x0; + r115[1U] = x1; + r115[2U] = x2; + r115[3U] = x3;); + uint64_t *s0 = hash; + uint64_t *s1 = hash + 4U; + uint64_t *r0 = wv; + uint64_t *r1 = wv + 4U; + uint64_t *r2 = wv + 8U; + uint64_t *r3 = wv + 12U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = s0; + uint64_t x = s0[i] ^ r0[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = s0; + uint64_t x = s0[i] ^ r2[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = s1; + uint64_t x = s1[i] ^ r1[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = s1; + uint64_t x = s1[i] ^ r3[i]; + os[i] = x;); +} + +void Hacl_Hash_Blake2b_init(uint64_t *hash, uint32_t kk, uint32_t nn) +{ + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = 64U, .key_length = 0U, .fanout = 1U, .depth = 1U, .leaf_length = 0U, + .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, .personal = personal + }; + uint64_t tmp[8U] = { 0U }; + uint64_t *r0 = hash; + uint64_t *r1 = hash + 4U; + uint64_t *r2 = hash + 8U; + uint64_t *r3 = hash + 12U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + uint8_t kk1 = (uint8_t)kk; + uint8_t nn1 = (uint8_t)nn; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = p.salt + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = p.personal + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk1 + << 8U + ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + tmp[1U] = p.node_offset; + tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; +} + +static void update_key(uint64_t *wv, uint64_t *hash, uint32_t kk, uint8_t *k, uint32_t ll) +{ + FStar_UInt128_uint128 lb = FStar_UInt128_uint64_to_uint128((uint64_t)128U); + uint8_t b[128U] = { 0U }; + memcpy(b, k, kk * sizeof (uint8_t)); + if (ll == 0U) + { + update_block(wv, hash, true, false, lb, b); + } + else + { + update_block(wv, hash, false, false, lb, b); + } + Lib_Memzero0_memzero(b, 128U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2b_update_multi( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks, + uint32_t nb +) +{ + KRML_MAYBE_UNUSED_VAR(len); + for (uint32_t i = 0U; i < nb; i++) + { + FStar_UInt128_uint128 + totlen = + FStar_UInt128_add_mod(prev, + FStar_UInt128_uint64_to_uint128((uint64_t)((i + 1U) * 128U))); + uint8_t *b = blocks + i * 128U; + update_block(wv, hash, false, false, totlen, b); + } +} + +void +Hacl_Hash_Blake2b_update_last( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + bool last_node, + FStar_UInt128_uint128 prev, + uint32_t rem, + uint8_t *d +) +{ + uint8_t b[128U] = { 0U }; + uint8_t *last = d + len - rem; + memcpy(b, last, rem * sizeof (uint8_t)); + FStar_UInt128_uint128 + totlen = FStar_UInt128_add_mod(prev, FStar_UInt128_uint64_to_uint128((uint64_t)len)); + update_block(wv, hash, true, last_node, totlen, b); + Lib_Memzero0_memzero(b, 128U, uint8_t, void *); +} + +static void +update_blocks( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks +) +{ + uint32_t nb0 = len / 128U; + uint32_t rem0 = len % 128U; + uint32_t nb; + if (rem0 == 0U && nb0 > 0U) + { + nb = nb0 - 1U; + } + else + { + nb = nb0; + } + uint32_t rem; + if (rem0 == 0U && nb0 > 0U) + { + rem = 128U; + } + else + { + rem = rem0; + } + Hacl_Hash_Blake2b_update_multi(len, wv, hash, prev, blocks, nb); + Hacl_Hash_Blake2b_update_last(len, wv, hash, false, prev, rem, blocks); +} + +static inline void +update(uint64_t *wv, uint64_t *hash, uint32_t kk, uint8_t *k, uint32_t ll, uint8_t *d) +{ + FStar_UInt128_uint128 lb = FStar_UInt128_uint64_to_uint128((uint64_t)128U); + if (kk > 0U) + { + update_key(wv, hash, kk, k, ll); + if (!(ll == 0U)) + { + update_blocks(ll, wv, hash, lb, d); + return; + } + return; + } + update_blocks(ll, wv, hash, FStar_UInt128_uint64_to_uint128((uint64_t)0U), d); +} + +void Hacl_Hash_Blake2b_finish(uint32_t nn, uint8_t *output, uint64_t *hash) +{ + uint8_t b[64U] = { 0U }; + uint8_t *first = b; + uint8_t *second = b + 32U; + uint64_t *row0 = hash; + uint64_t *row1 = hash + 4U; + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store64_le(first + i * 8U, row0[i]);); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store64_le(second + i * 8U, row1[i]);); + uint8_t *final = b; + memcpy(output, final, nn * sizeof (uint8_t)); + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +static Hacl_Hash_Blake2b_state_t +*malloc_raw(Hacl_Hash_Blake2b_index kk, Hacl_Hash_Blake2b_params_and_key key) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + uint64_t *wv = (uint64_t *)KRML_HOST_CALLOC(16U, sizeof (uint64_t)); + uint64_t *b = (uint64_t *)KRML_HOST_CALLOC(16U, sizeof (uint64_t)); + Hacl_Hash_Blake2b_block_state_t + block_state = + { + .fst = kk.key_length, + .snd = kk.digest_length, + .thd = kk.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint8_t kk10 = kk.key_length; + uint32_t ite; + if (kk10 != 0U) + { + ite = 128U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2b_state_t + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + Hacl_Hash_Blake2b_state_t + *p = (Hacl_Hash_Blake2b_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_Blake2b_state_t)); + p[0U] = s; + Hacl_Hash_Blake2b_blake2_params *p1 = key.fst; + uint8_t kk1 = p1->key_length; + uint8_t nn = p1->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint64_t *h = block_state.f3.snd; + uint32_t kk20 = (uint32_t)i.key_length; + uint8_t *k_1 = key.snd; + if (!(kk20 == 0U)) + { + uint8_t *sub_b = buf + kk20; + memset(sub_b, 0U, (128U - kk20) * sizeof (uint8_t)); + memcpy(buf, k_1, kk20 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p1[0U]; + uint64_t tmp[8U] = { 0U }; + uint64_t *r0 = h; + uint64_t *r1 = h + 4U; + uint64_t *r2 = h + 8U; + uint64_t *r3 = h + 12U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + uint8_t kk2 = pv.key_length; + uint8_t nn1 = pv.digest_length; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r4 = u; + uint64_t x = r4; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r4 = u; + uint64_t x = r4; + os[i0] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk2 + << 8U + ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + tmp[1U] = pv.node_offset; + tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + return p; +} + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 32 for S, 64 for B. +- The digest_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_state_t +*Hacl_Hash_Blake2b_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + Hacl_Hash_Blake2b_index + i1 = { .key_length = pv.key_length, .digest_length = pv.digest_length, .last_node = last_node }; + return malloc_raw(i1, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_malloc_with_key(uint8_t *k, uint8_t kk) +{ + uint8_t nn = 64U; + Hacl_Hash_Blake2b_index i = { .key_length = kk, .digest_length = nn, .last_node = false }; + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = i.digest_length, .key_length = i.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + Hacl_Hash_Blake2b_state_t *s = Hacl_Hash_Blake2b_malloc_with_params_and_key(&p0, false, k); + return s; +} + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_malloc(void) +{ + return Hacl_Hash_Blake2b_malloc_with_key(NULL, 0U); +} + +static Hacl_Hash_Blake2b_index index_of_state(Hacl_Hash_Blake2b_state_t *s) +{ + Hacl_Hash_Blake2b_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk1 = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk1, .digest_length = nn, .last_node = last_node }); +} + +static void reset_raw(Hacl_Hash_Blake2b_state_t *state, Hacl_Hash_Blake2b_params_and_key key) +{ + Hacl_Hash_Blake2b_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2b_block_state_t block_state = scrut.block_state; + bool last_node0 = block_state.thd; + uint8_t nn0 = block_state.snd; + uint8_t kk10 = block_state.fst; + Hacl_Hash_Blake2b_index + i = { .key_length = kk10, .digest_length = nn0, .last_node = last_node0 }; + KRML_MAYBE_UNUSED_VAR(i); + Hacl_Hash_Blake2b_blake2_params *p = key.fst; + uint8_t kk1 = p->key_length; + uint8_t nn = p->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint64_t *h = block_state.f3.snd; + uint32_t kk20 = (uint32_t)i1.key_length; + uint8_t *k_1 = key.snd; + if (!(kk20 == 0U)) + { + uint8_t *sub_b = buf + kk20; + memset(sub_b, 0U, (128U - kk20) * sizeof (uint8_t)); + memcpy(buf, k_1, kk20 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + uint64_t tmp[8U] = { 0U }; + uint64_t *r0 = h; + uint64_t *r1 = h + 4U; + uint64_t *r2 = h + 8U; + uint64_t *r3 = h + 12U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + uint8_t kk2 = pv.key_length; + uint8_t nn1 = pv.digest_length; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i0] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk2 + << 8U + ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + tmp[1U] = pv.node_offset; + tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + uint8_t kk11 = i.key_length; + uint32_t ite; + if (kk11 != 0U) + { + ite = 128U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2b_state_t + tmp8 = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + state[0U] = tmp8; +} + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_reset_with_key_and_params( + Hacl_Hash_Blake2b_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +) +{ + index_of_state(s); + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_reset_with_key(Hacl_Hash_Blake2b_state_t *s, uint8_t *k) +{ + Hacl_Hash_Blake2b_index idx = index_of_state(s); + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = idx.digest_length, .key_length = idx.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = &p0, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_reset(Hacl_Hash_Blake2b_state_t *s) +{ + Hacl_Hash_Blake2b_reset_with_key(s, NULL); +} + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint32_t chunk_len) +{ + Hacl_Hash_Blake2b_state_t s = *state; + uint64_t total_len = s.total_len; + if ((uint64_t)chunk_len > 0xffffffffffffffffULL - total_len) + { + return Hacl_Streaming_Types_MaximumLengthExceeded; + } + uint32_t sz; + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) + { + sz = 128U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)128U); + } + if (chunk_len <= 128U - sz) + { + Hacl_Hash_Blake2b_state_t s1 = *state; + Hacl_Hash_Blake2b_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state + = + ( + (Hacl_Hash_Blake2b_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == 0U) + { + Hacl_Hash_Blake2b_state_t s1 = *state; + Hacl_Hash_Blake2b_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____uint64_t___uint64_t_ acc = block_state1.f3; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2b_update_multi(128U, + wv, + hash, + FStar_UInt128_uint64_to_uint128(prevlen), + buf, + nb); + } + uint32_t ite; + if ((uint64_t)chunk_len % (uint64_t)128U == 0ULL && (uint64_t)chunk_len > 0ULL) + { + ite = 128U; + } + else + { + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)128U); + } + uint32_t n_blocks = (chunk_len - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + K____uint64_t___uint64_t_ acc = block_state1.f3; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + uint32_t nb = data1_len / 128U; + Hacl_Hash_Blake2b_update_multi(data1_len, + wv, + hash, + FStar_UInt128_uint64_to_uint128(total_len1), + data1, + nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2b_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)chunk_len + } + ); + } + else + { + uint32_t diff = 128U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_Blake2b_state_t s1 = *state; + Hacl_Hash_Blake2b_block_state_t block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)128U == 0ULL && total_len10 > 0ULL) + { + sz10 = 128U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)128U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *state + = + ( + (Hacl_Hash_Blake2b_state_t){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Hash_Blake2b_state_t s10 = *state; + Hacl_Hash_Blake2b_block_state_t block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____uint64_t___uint64_t_ acc = block_state1.f3; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2b_update_multi(128U, + wv, + hash, + FStar_UInt128_uint64_to_uint128(prevlen), + buf, + nb); + } + uint32_t ite; + if + ((uint64_t)(chunk_len - diff) % (uint64_t)128U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) + { + ite = 128U; + } + else + { + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)128U); + } + uint32_t n_blocks = (chunk_len - diff - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + K____uint64_t___uint64_t_ acc = block_state1.f3; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + uint32_t nb = data1_len / 128U; + Hacl_Hash_Blake2b_update_multi(data1_len, + wv, + hash, + FStar_UInt128_uint64_to_uint128(total_len1), + data1, + nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2b_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(chunk_len - diff) + } + ); + } + return Hacl_Streaming_Types_Success; +} + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 32 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2B_32_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2b_digest(Hacl_Hash_Blake2b_state_t *s, uint8_t *dst) +{ + Hacl_Hash_Blake2b_block_state_t block_state0 = (*s).block_state; + bool last_node0 = block_state0.thd; + uint8_t nn0 = block_state0.snd; + uint8_t kk0 = block_state0.fst; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk0, .digest_length = nn0, .last_node = last_node0 }; + Hacl_Hash_Blake2b_state_t scrut = *s; + Hacl_Hash_Blake2b_block_state_t block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) + { + r = 128U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)128U); + } + uint8_t *buf_1 = buf_; + uint64_t wv0[16U] = { 0U }; + uint64_t b[16U] = { 0U }; + Hacl_Hash_Blake2b_block_state_t + tmp_block_state = + { + .fst = i1.key_length, + .snd = i1.digest_length, + .thd = i1.last_node, + .f3 = { .fst = wv0, .snd = b } + }; + uint64_t *src_b = block_state.f3.snd; + uint64_t *dst_b = tmp_block_state.f3.snd; + memcpy(dst_b, src_b, 16U * sizeof (uint64_t)); + uint64_t prev_len = total_len - (uint64_t)r; + uint32_t ite; + if (r % 128U == 0U && r > 0U) + { + ite = 128U; + } + else + { + ite = r % 128U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + K____uint64_t___uint64_t_ acc0 = tmp_block_state.f3; + uint64_t *wv1 = acc0.fst; + uint64_t *hash0 = acc0.snd; + uint32_t nb = 0U; + Hacl_Hash_Blake2b_update_multi(0U, + wv1, + hash0, + FStar_UInt128_uint64_to_uint128(prev_len), + buf_multi, + nb); + uint64_t prev_len_last = total_len - (uint64_t)r; + K____uint64_t___uint64_t_ acc = tmp_block_state.f3; + bool last_node1 = tmp_block_state.thd; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + Hacl_Hash_Blake2b_update_last(r, + wv, + hash, + last_node1, + FStar_UInt128_uint64_to_uint128(prev_len_last), + r, + buf_last); + uint8_t nn1 = tmp_block_state.snd; + Hacl_Hash_Blake2b_finish((uint32_t)nn1, dst, tmp_block_state.f3.snd); + Hacl_Hash_Blake2b_block_state_t block_state1 = (*s).block_state; + bool last_node = block_state1.thd; + uint8_t nn = block_state1.snd; + uint8_t kk = block_state1.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }).digest_length; +} + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2b_info(Hacl_Hash_Blake2b_state_t *s) +{ + Hacl_Hash_Blake2b_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }); +} + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2b_free(Hacl_Hash_Blake2b_state_t *state) +{ + Hacl_Hash_Blake2b_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2b_block_state_t block_state = scrut.block_state; + uint64_t *b = block_state.f3.snd; + uint64_t *wv = block_state.f3.fst; + KRML_HOST_FREE(wv); + KRML_HOST_FREE(b); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); +} + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_copy(Hacl_Hash_Blake2b_state_t *state) +{ + Hacl_Hash_Blake2b_state_t scrut = *state; + Hacl_Hash_Blake2b_block_state_t block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + bool last_node = block_state0.thd; + uint8_t nn = block_state0.snd; + uint8_t kk1 = block_state0.fst; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + memcpy(buf, buf0, 128U * sizeof (uint8_t)); + uint64_t *wv = (uint64_t *)KRML_HOST_CALLOC(16U, sizeof (uint64_t)); + uint64_t *b = (uint64_t *)KRML_HOST_CALLOC(16U, sizeof (uint64_t)); + Hacl_Hash_Blake2b_block_state_t + block_state = + { + .fst = i.key_length, + .snd = i.digest_length, + .thd = i.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint64_t *src_b = block_state0.f3.snd; + uint64_t *dst_b = block_state.f3.snd; + memcpy(dst_b, src_b, 16U * sizeof (uint64_t)); + Hacl_Hash_Blake2b_state_t + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Hash_Blake2b_state_t + *p = (Hacl_Hash_Blake2b_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_Blake2b_state_t)); + p[0U] = s; + return p; +} + +/** +Write the BLAKE2b digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2b_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +) +{ + uint64_t b[16U] = { 0U }; + uint64_t b1[16U] = { 0U }; + Hacl_Hash_Blake2b_init(b, key_len, output_len); + update(b1, b, key_len, key, input_len, input); + Hacl_Hash_Blake2b_finish(output_len, output, b); + Lib_Memzero0_memzero(b1, 16U, uint64_t, void *); + Lib_Memzero0_memzero(b, 16U, uint64_t, void *); +} + +/** +Write the BLAKE2b digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2b_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +) +{ + uint64_t b[16U] = { 0U }; + uint64_t b1[16U] = { 0U }; + uint64_t tmp[8U] = { 0U }; + uint64_t *r0 = b; + uint64_t *r1 = b + 4U; + uint64_t *r2 = b + 8U; + uint64_t *r3 = b + 12U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + uint8_t kk = params.key_length; + uint8_t nn = params.digest_length; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = params.salt + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = params.personal + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + tmp[0U] = + (uint64_t)nn + ^ + ((uint64_t)kk + << 8U + ^ + ((uint64_t)params.fanout + << 16U + ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + tmp[1U] = params.node_offset; + tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + update(b1, b, (uint32_t)params.key_length, key, input_len, input); + Hacl_Hash_Blake2b_finish((uint32_t)params.digest_length, output, b); + Lib_Memzero0_memzero(b1, 16U, uint64_t, void *); + Lib_Memzero0_memzero(b, 16U, uint64_t, void *); +} + diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.h b/Modules/_hacl/Hacl_Hash_Blake2b.h new file mode 100644 index 00000000000000..5b5b037bcdc8a4 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b.h @@ -0,0 +1,245 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_Blake2b_H +#define __Hacl_Hash_Blake2b_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "python_hacl_namespaces.h" +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + + +typedef struct Hacl_Hash_Blake2b_blake2_params_s +{ + uint8_t digest_length; + uint8_t key_length; + uint8_t fanout; + uint8_t depth; + uint32_t leaf_length; + uint64_t node_offset; + uint8_t node_depth; + uint8_t inner_length; + uint8_t *salt; + uint8_t *personal; +} +Hacl_Hash_Blake2b_blake2_params; + +typedef struct Hacl_Hash_Blake2b_index_s +{ + uint8_t key_length; + uint8_t digest_length; + bool last_node; +} +Hacl_Hash_Blake2b_index; + +#define HACL_HASH_BLAKE2B_BLOCK_BYTES (128U) + +#define HACL_HASH_BLAKE2B_OUT_BYTES (64U) + +#define HACL_HASH_BLAKE2B_KEY_BYTES (64U) + +#define HACL_HASH_BLAKE2B_SALT_BYTES (16U) + +#define HACL_HASH_BLAKE2B_PERSONAL_BYTES (16U) + +typedef struct K____uint64_t___uint64_t__s +{ + uint64_t *fst; + uint64_t *snd; +} +K____uint64_t___uint64_t_; + +typedef struct Hacl_Hash_Blake2b_block_state_t_s +{ + uint8_t fst; + uint8_t snd; + bool thd; + K____uint64_t___uint64_t_ f3; +} +Hacl_Hash_Blake2b_block_state_t; + +typedef struct Hacl_Hash_Blake2b_state_t_s +{ + Hacl_Hash_Blake2b_block_state_t block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Hash_Blake2b_state_t; + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 32 for S, 64 for B. +- The digest_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_state_t +*Hacl_Hash_Blake2b_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +); + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_malloc_with_key(uint8_t *k, uint8_t kk); + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_malloc(void); + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_reset_with_key_and_params( + Hacl_Hash_Blake2b_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +); + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_reset_with_key(Hacl_Hash_Blake2b_state_t *s, uint8_t *k); + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_reset(Hacl_Hash_Blake2b_state_t *s); + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint32_t chunk_len); + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 32 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2B_32_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2b_digest(Hacl_Hash_Blake2b_state_t *s, uint8_t *dst); + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2b_info(Hacl_Hash_Blake2b_state_t *s); + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2b_free(Hacl_Hash_Blake2b_state_t *state); + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_copy(Hacl_Hash_Blake2b_state_t *state); + +/** +Write the BLAKE2b digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2b_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +); + +/** +Write the BLAKE2b digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2b_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_Blake2b_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c new file mode 100644 index 00000000000000..92b2e8f539041b --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c @@ -0,0 +1,1397 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_Blake2b_Simd256.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "lib_memzero0.h" + +static inline void +update_block( + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + bool flag, + bool last_node, + FStar_UInt128_uint128 totlen, + uint8_t *d +) +{ + uint64_t m_w[16U] = { 0U }; + KRML_MAYBE_FOR16(i, + 0U, + 16U, + 1U, + uint64_t *os = m_w; + uint8_t *bj = d + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + Lib_IntVector_Intrinsics_vec256 mask = Lib_IntVector_Intrinsics_vec256_zero; + uint64_t wv_14; + if (flag) + { + wv_14 = 0xFFFFFFFFFFFFFFFFULL; + } + else + { + wv_14 = 0ULL; + } + uint64_t wv_15; + if (last_node) + { + wv_15 = 0xFFFFFFFFFFFFFFFFULL; + } + else + { + wv_15 = 0ULL; + } + mask = + Lib_IntVector_Intrinsics_vec256_load64s(FStar_UInt128_uint128_to_uint64(totlen), + FStar_UInt128_uint128_to_uint64(FStar_UInt128_shift_right(totlen, 64U)), + wv_14, + wv_15); + memcpy(wv, hash, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Lib_IntVector_Intrinsics_vec256 *wv3 = wv + 3U; + wv3[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv3[0U], mask); + KRML_MAYBE_FOR12(i, + 0U, + 12U, + 1U, + uint32_t start_idx = i % 10U * 16U; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 m_st[4U] KRML_POST_ALIGN(32) = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = m_st; + Lib_IntVector_Intrinsics_vec256 *r1 = m_st + 1U; + Lib_IntVector_Intrinsics_vec256 *r20 = m_st + 2U; + Lib_IntVector_Intrinsics_vec256 *r30 = m_st + 3U; + uint32_t s0 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 0U]; + uint32_t s1 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 1U]; + uint32_t s2 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 2U]; + uint32_t s3 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 3U]; + uint32_t s4 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 4U]; + uint32_t s5 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 5U]; + uint32_t s6 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 6U]; + uint32_t s7 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 7U]; + uint32_t s8 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 8U]; + uint32_t s9 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 9U]; + uint32_t s10 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 10U]; + uint32_t s11 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 11U]; + uint32_t s12 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 12U]; + uint32_t s13 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 13U]; + uint32_t s14 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 14U]; + uint32_t s15 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 15U]; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(m_w[s0], m_w[s2], m_w[s4], m_w[s6]); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(m_w[s1], m_w[s3], m_w[s5], m_w[s7]); + r20[0U] = Lib_IntVector_Intrinsics_vec256_load64s(m_w[s8], m_w[s10], m_w[s12], m_w[s14]); + r30[0U] = Lib_IntVector_Intrinsics_vec256_load64s(m_w[s9], m_w[s11], m_w[s13], m_w[s15]); + Lib_IntVector_Intrinsics_vec256 *x = m_st; + Lib_IntVector_Intrinsics_vec256 *y = m_st + 1U; + Lib_IntVector_Intrinsics_vec256 *z = m_st + 2U; + Lib_IntVector_Intrinsics_vec256 *w = m_st + 3U; + uint32_t a = 0U; + uint32_t b0 = 1U; + uint32_t c0 = 2U; + uint32_t d10 = 3U; + Lib_IntVector_Intrinsics_vec256 *wv_a0 = wv + a * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b0 = wv + b0 * 1U; + wv_a0[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a0[0U], wv_b0[0U]); + wv_a0[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a0[0U], x[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a1 = wv + d10 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b1 = wv + a * 1U; + wv_a1[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a1[0U], wv_b1[0U]); + wv_a1[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a1[0U], 32U); + Lib_IntVector_Intrinsics_vec256 *wv_a2 = wv + c0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b2 = wv + d10 * 1U; + wv_a2[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a2[0U], wv_b2[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a3 = wv + b0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b3 = wv + c0 * 1U; + wv_a3[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a3[0U], wv_b3[0U]); + wv_a3[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a3[0U], 24U); + Lib_IntVector_Intrinsics_vec256 *wv_a4 = wv + a * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b4 = wv + b0 * 1U; + wv_a4[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a4[0U], wv_b4[0U]); + wv_a4[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a4[0U], y[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a5 = wv + d10 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b5 = wv + a * 1U; + wv_a5[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a5[0U], wv_b5[0U]); + wv_a5[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a5[0U], 16U); + Lib_IntVector_Intrinsics_vec256 *wv_a6 = wv + c0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b6 = wv + d10 * 1U; + wv_a6[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a6[0U], wv_b6[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a7 = wv + b0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b7 = wv + c0 * 1U; + wv_a7[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a7[0U], wv_b7[0U]); + wv_a7[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a7[0U], 63U); + Lib_IntVector_Intrinsics_vec256 *r10 = wv + 1U; + Lib_IntVector_Intrinsics_vec256 *r21 = wv + 2U; + Lib_IntVector_Intrinsics_vec256 *r31 = wv + 3U; + Lib_IntVector_Intrinsics_vec256 v00 = r10[0U]; + Lib_IntVector_Intrinsics_vec256 + v1 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v00, 1U); + r10[0U] = v1; + Lib_IntVector_Intrinsics_vec256 v01 = r21[0U]; + Lib_IntVector_Intrinsics_vec256 + v10 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v01, 2U); + r21[0U] = v10; + Lib_IntVector_Intrinsics_vec256 v02 = r31[0U]; + Lib_IntVector_Intrinsics_vec256 + v11 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v02, 3U); + r31[0U] = v11; + uint32_t a0 = 0U; + uint32_t b = 1U; + uint32_t c = 2U; + uint32_t d1 = 3U; + Lib_IntVector_Intrinsics_vec256 *wv_a = wv + a0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b8 = wv + b * 1U; + wv_a[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a[0U], wv_b8[0U]); + wv_a[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a[0U], z[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a8 = wv + d1 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b9 = wv + a0 * 1U; + wv_a8[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a8[0U], wv_b9[0U]); + wv_a8[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a8[0U], 32U); + Lib_IntVector_Intrinsics_vec256 *wv_a9 = wv + c * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b10 = wv + d1 * 1U; + wv_a9[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a9[0U], wv_b10[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a10 = wv + b * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b11 = wv + c * 1U; + wv_a10[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a10[0U], wv_b11[0U]); + wv_a10[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a10[0U], 24U); + Lib_IntVector_Intrinsics_vec256 *wv_a11 = wv + a0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b12 = wv + b * 1U; + wv_a11[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a11[0U], wv_b12[0U]); + wv_a11[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a11[0U], w[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a12 = wv + d1 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b13 = wv + a0 * 1U; + wv_a12[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a12[0U], wv_b13[0U]); + wv_a12[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a12[0U], 16U); + Lib_IntVector_Intrinsics_vec256 *wv_a13 = wv + c * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b14 = wv + d1 * 1U; + wv_a13[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a13[0U], wv_b14[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a14 = wv + b * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b = wv + c * 1U; + wv_a14[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a14[0U], wv_b[0U]); + wv_a14[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a14[0U], 63U); + Lib_IntVector_Intrinsics_vec256 *r11 = wv + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = wv + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = wv + 3U; + Lib_IntVector_Intrinsics_vec256 v0 = r11[0U]; + Lib_IntVector_Intrinsics_vec256 + v12 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v0, 3U); + r11[0U] = v12; + Lib_IntVector_Intrinsics_vec256 v03 = r2[0U]; + Lib_IntVector_Intrinsics_vec256 + v13 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v03, 2U); + r2[0U] = v13; + Lib_IntVector_Intrinsics_vec256 v04 = r3[0U]; + Lib_IntVector_Intrinsics_vec256 + v14 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v04, 1U); + r3[0U] = v14;); + Lib_IntVector_Intrinsics_vec256 *s0 = hash; + Lib_IntVector_Intrinsics_vec256 *s1 = hash + 1U; + Lib_IntVector_Intrinsics_vec256 *r0 = wv; + Lib_IntVector_Intrinsics_vec256 *r1 = wv + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = wv + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = wv + 3U; + s0[0U] = Lib_IntVector_Intrinsics_vec256_xor(s0[0U], r0[0U]); + s0[0U] = Lib_IntVector_Intrinsics_vec256_xor(s0[0U], r2[0U]); + s1[0U] = Lib_IntVector_Intrinsics_vec256_xor(s1[0U], r1[0U]); + s1[0U] = Lib_IntVector_Intrinsics_vec256_xor(s1[0U], r3[0U]); +} + +void +Hacl_Hash_Blake2b_Simd256_init(Lib_IntVector_Intrinsics_vec256 *hash, uint32_t kk, uint32_t nn) +{ + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = 64U, .key_length = 0U, .fanout = 1U, .depth = 1U, .leaf_length = 0U, + .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, .personal = personal + }; + uint64_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = hash; + Lib_IntVector_Intrinsics_vec256 *r1 = hash + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = hash + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = hash + 3U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4, iv5, iv6, iv7); + uint8_t kk1 = (uint8_t)kk; + uint8_t nn1 = (uint8_t)nn; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = p.salt + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = p.personal + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk1 + << 8U + ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + tmp[1U] = p.node_offset; + tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4_, iv5_, iv6_, iv7_); +} + +static void +update_key( + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + uint32_t kk, + uint8_t *k, + uint32_t ll +) +{ + FStar_UInt128_uint128 lb = FStar_UInt128_uint64_to_uint128((uint64_t)128U); + uint8_t b[128U] = { 0U }; + memcpy(b, k, kk * sizeof (uint8_t)); + if (ll == 0U) + { + update_block(wv, hash, true, false, lb, b); + } + else + { + update_block(wv, hash, false, false, lb, b); + } + Lib_Memzero0_memzero(b, 128U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2b_Simd256_update_multi( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks, + uint32_t nb +) +{ + KRML_MAYBE_UNUSED_VAR(len); + for (uint32_t i = 0U; i < nb; i++) + { + FStar_UInt128_uint128 + totlen = + FStar_UInt128_add_mod(prev, + FStar_UInt128_uint64_to_uint128((uint64_t)((i + 1U) * 128U))); + uint8_t *b = blocks + i * 128U; + update_block(wv, hash, false, false, totlen, b); + } +} + +void +Hacl_Hash_Blake2b_Simd256_update_last( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + bool last_node, + FStar_UInt128_uint128 prev, + uint32_t rem, + uint8_t *d +) +{ + uint8_t b[128U] = { 0U }; + uint8_t *last = d + len - rem; + memcpy(b, last, rem * sizeof (uint8_t)); + FStar_UInt128_uint128 + totlen = FStar_UInt128_add_mod(prev, FStar_UInt128_uint64_to_uint128((uint64_t)len)); + update_block(wv, hash, true, last_node, totlen, b); + Lib_Memzero0_memzero(b, 128U, uint8_t, void *); +} + +static inline void +update_blocks( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks +) +{ + uint32_t nb0 = len / 128U; + uint32_t rem0 = len % 128U; + uint32_t nb; + if (rem0 == 0U && nb0 > 0U) + { + nb = nb0 - 1U; + } + else + { + nb = nb0; + } + uint32_t rem; + if (rem0 == 0U && nb0 > 0U) + { + rem = 128U; + } + else + { + rem = rem0; + } + Hacl_Hash_Blake2b_Simd256_update_multi(len, wv, hash, prev, blocks, nb); + Hacl_Hash_Blake2b_Simd256_update_last(len, wv, hash, false, prev, rem, blocks); +} + +static inline void +update( + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + uint32_t kk, + uint8_t *k, + uint32_t ll, + uint8_t *d +) +{ + FStar_UInt128_uint128 lb = FStar_UInt128_uint64_to_uint128((uint64_t)128U); + if (kk > 0U) + { + update_key(wv, hash, kk, k, ll); + if (!(ll == 0U)) + { + update_blocks(ll, wv, hash, lb, d); + return; + } + return; + } + update_blocks(ll, wv, hash, FStar_UInt128_uint64_to_uint128((uint64_t)0U), d); +} + +void +Hacl_Hash_Blake2b_Simd256_finish( + uint32_t nn, + uint8_t *output, + Lib_IntVector_Intrinsics_vec256 *hash +) +{ + uint8_t b[64U] = { 0U }; + uint8_t *first = b; + uint8_t *second = b + 32U; + Lib_IntVector_Intrinsics_vec256 *row0 = hash; + Lib_IntVector_Intrinsics_vec256 *row1 = hash + 1U; + Lib_IntVector_Intrinsics_vec256_store64_le(first, row0[0U]); + Lib_IntVector_Intrinsics_vec256_store64_le(second, row1[0U]); + uint8_t *final = b; + memcpy(output, final, nn * sizeof (uint8_t)); + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32( + Lib_IntVector_Intrinsics_vec256 *st, + uint64_t *st32 +) +{ + Lib_IntVector_Intrinsics_vec256 *r0 = st; + Lib_IntVector_Intrinsics_vec256 *r1 = st + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = st + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = st + 3U; + uint64_t *b0 = st32; + uint64_t *b1 = st32 + 4U; + uint64_t *b2 = st32 + 8U; + uint64_t *b3 = st32 + 12U; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(b0[0U], b0[1U], b0[2U], b0[3U]); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(b1[0U], b1[1U], b1[2U], b1[3U]); + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(b2[0U], b2[1U], b2[2U], b2[3U]); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(b3[0U], b3[1U], b3[2U], b3[3U]); +} + +void +Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32( + uint64_t *st32, + Lib_IntVector_Intrinsics_vec256 *st +) +{ + Lib_IntVector_Intrinsics_vec256 *r0 = st; + Lib_IntVector_Intrinsics_vec256 *r1 = st + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = st + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = st + 3U; + uint64_t *b0 = st32; + uint64_t *b1 = st32 + 4U; + uint64_t *b2 = st32 + 8U; + uint64_t *b3 = st32 + 12U; + uint8_t b8[32U] = { 0U }; + Lib_IntVector_Intrinsics_vec256_store64_le(b8, r0[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = b0; + uint8_t *bj = b8 + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + uint8_t b80[32U] = { 0U }; + Lib_IntVector_Intrinsics_vec256_store64_le(b80, r1[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = b1; + uint8_t *bj = b80 + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + uint8_t b81[32U] = { 0U }; + Lib_IntVector_Intrinsics_vec256_store64_le(b81, r2[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = b2; + uint8_t *bj = b81 + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + uint8_t b82[32U] = { 0U }; + Lib_IntVector_Intrinsics_vec256_store64_le(b82, r3[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = b3; + uint8_t *bj = b82 + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); +} + +Lib_IntVector_Intrinsics_vec256 *Hacl_Hash_Blake2b_Simd256_malloc_with_key(void) +{ + Lib_IntVector_Intrinsics_vec256 + *buf = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(buf, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + return buf; +} + +static Hacl_Hash_Blake2b_Simd256_state_t +*malloc_raw(Hacl_Hash_Blake2b_index kk, Hacl_Hash_Blake2b_params_and_key key) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + Lib_IntVector_Intrinsics_vec256 + *wv = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(wv, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Lib_IntVector_Intrinsics_vec256 + *b = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(b, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Hacl_Hash_Blake2b_Simd256_block_state_t + block_state = + { + .fst = kk.key_length, + .snd = kk.digest_length, + .thd = kk.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint8_t kk10 = kk.key_length; + uint32_t ite; + if (kk10 != 0U) + { + ite = 128U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2b_Simd256_state_t + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + Hacl_Hash_Blake2b_Simd256_state_t + *p = + (Hacl_Hash_Blake2b_Simd256_state_t *)KRML_HOST_MALLOC(sizeof ( + Hacl_Hash_Blake2b_Simd256_state_t + )); + p[0U] = s; + Hacl_Hash_Blake2b_blake2_params *p1 = key.fst; + uint8_t kk1 = p1->key_length; + uint8_t nn = p1->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + Lib_IntVector_Intrinsics_vec256 *h = block_state.f3.snd; + uint32_t kk20 = (uint32_t)i.key_length; + uint8_t *k_1 = key.snd; + if (!(kk20 == 0U)) + { + uint8_t *sub_b = buf + kk20; + memset(sub_b, 0U, (128U - kk20) * sizeof (uint8_t)); + memcpy(buf, k_1, kk20 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p1[0U]; + uint64_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = h; + Lib_IntVector_Intrinsics_vec256 *r1 = h + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = h + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = h + 3U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4, iv5, iv6, iv7); + uint8_t kk2 = pv.key_length; + uint8_t nn1 = pv.digest_length; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r4 = u; + uint64_t x = r4; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r4 = u; + uint64_t x = r4; + os[i0] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk2 + << 8U + ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + tmp[1U] = pv.node_offset; + tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4_, iv5_, iv6_, iv7_); + return p; +} + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 256 for S, 64 for B. +- The digest_length must not exceed 256 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + Hacl_Hash_Blake2b_index + i1 = { .key_length = pv.key_length, .digest_length = pv.digest_length, .last_node = last_node }; + return malloc_raw(i1, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 256 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_malloc_with_key0(uint8_t *k, uint8_t kk) +{ + uint8_t nn = 64U; + Hacl_Hash_Blake2b_index i = { .key_length = kk, .digest_length = nn, .last_node = false }; + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = i.digest_length, .key_length = i.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + Hacl_Hash_Blake2b_Simd256_state_t + *s = Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key(&p0, false, k); + return s; +} + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2b_Simd256_state_t *Hacl_Hash_Blake2b_Simd256_malloc(void) +{ + return Hacl_Hash_Blake2b_Simd256_malloc_with_key0(NULL, 0U); +} + +static Hacl_Hash_Blake2b_index index_of_state(Hacl_Hash_Blake2b_Simd256_state_t *s) +{ + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk1 = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk1, .digest_length = nn, .last_node = last_node }); +} + +static void +reset_raw(Hacl_Hash_Blake2b_Simd256_state_t *state, Hacl_Hash_Blake2b_params_and_key key) +{ + Hacl_Hash_Blake2b_Simd256_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = scrut.block_state; + bool last_node0 = block_state.thd; + uint8_t nn0 = block_state.snd; + uint8_t kk10 = block_state.fst; + Hacl_Hash_Blake2b_index + i = { .key_length = kk10, .digest_length = nn0, .last_node = last_node0 }; + KRML_MAYBE_UNUSED_VAR(i); + Hacl_Hash_Blake2b_blake2_params *p = key.fst; + uint8_t kk1 = p->key_length; + uint8_t nn = p->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + Lib_IntVector_Intrinsics_vec256 *h = block_state.f3.snd; + uint32_t kk20 = (uint32_t)i1.key_length; + uint8_t *k_1 = key.snd; + if (!(kk20 == 0U)) + { + uint8_t *sub_b = buf + kk20; + memset(sub_b, 0U, (128U - kk20) * sizeof (uint8_t)); + memcpy(buf, k_1, kk20 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + uint64_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = h; + Lib_IntVector_Intrinsics_vec256 *r1 = h + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = h + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = h + 3U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4, iv5, iv6, iv7); + uint8_t kk2 = pv.key_length; + uint8_t nn1 = pv.digest_length; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i0] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk2 + << 8U + ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + tmp[1U] = pv.node_offset; + tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4_, iv5_, iv6_, iv7_); + uint8_t kk11 = i.key_length; + uint32_t ite; + if (kk11 != 0U) + { + ite = 128U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2b_Simd256_state_t + tmp8 = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + state[0U] = tmp8; +} + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params( + Hacl_Hash_Blake2b_Simd256_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +) +{ + index_of_state(s); + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_Simd256_reset_with_key(Hacl_Hash_Blake2b_Simd256_state_t *s, uint8_t *k) +{ + Hacl_Hash_Blake2b_index idx = index_of_state(s); + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = idx.digest_length, .key_length = idx.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = &p0, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_Simd256_reset(Hacl_Hash_Blake2b_Simd256_state_t *s) +{ + Hacl_Hash_Blake2b_Simd256_reset_with_key(s, NULL); +} + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2b_Simd256_update( + Hacl_Hash_Blake2b_Simd256_state_t *state, + uint8_t *chunk, + uint32_t chunk_len +) +{ + Hacl_Hash_Blake2b_Simd256_state_t s = *state; + uint64_t total_len = s.total_len; + if ((uint64_t)chunk_len > 0xffffffffffffffffULL - total_len) + { + return Hacl_Streaming_Types_MaximumLengthExceeded; + } + uint32_t sz; + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) + { + sz = 128U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)128U); + } + if (chunk_len <= 128U - sz) + { + Hacl_Hash_Blake2b_Simd256_state_t s1 = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state + = + ( + (Hacl_Hash_Blake2b_Simd256_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == 0U) + { + Hacl_Hash_Blake2b_Simd256_state_t s1 = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2b_Simd256_update_multi(128U, + wv, + hash, + FStar_UInt128_uint64_to_uint128(prevlen), + buf, + nb); + } + uint32_t ite; + if ((uint64_t)chunk_len % (uint64_t)128U == 0ULL && (uint64_t)chunk_len > 0ULL) + { + ite = 128U; + } + else + { + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)128U); + } + uint32_t n_blocks = (chunk_len - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + uint32_t nb = data1_len / 128U; + Hacl_Hash_Blake2b_Simd256_update_multi(data1_len, + wv, + hash, + FStar_UInt128_uint64_to_uint128(total_len1), + data1, + nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2b_Simd256_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)chunk_len + } + ); + } + else + { + uint32_t diff = 128U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_Blake2b_Simd256_state_t s1 = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)128U == 0ULL && total_len10 > 0ULL) + { + sz10 = 128U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)128U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *state + = + ( + (Hacl_Hash_Blake2b_Simd256_state_t){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Hash_Blake2b_Simd256_state_t s10 = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2b_Simd256_update_multi(128U, + wv, + hash, + FStar_UInt128_uint64_to_uint128(prevlen), + buf, + nb); + } + uint32_t ite; + if + ((uint64_t)(chunk_len - diff) % (uint64_t)128U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) + { + ite = 128U; + } + else + { + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)128U); + } + uint32_t n_blocks = (chunk_len - diff - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + uint32_t nb = data1_len / 128U; + Hacl_Hash_Blake2b_Simd256_update_multi(data1_len, + wv, + hash, + FStar_UInt128_uint64_to_uint128(total_len1), + data1, + nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2b_Simd256_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(chunk_len - diff) + } + ); + } + return Hacl_Streaming_Types_Success; +} + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 256 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2B_256_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2b_Simd256_digest(Hacl_Hash_Blake2b_Simd256_state_t *s, uint8_t *dst) +{ + Hacl_Hash_Blake2b_Simd256_block_state_t block_state0 = (*s).block_state; + bool last_node0 = block_state0.thd; + uint8_t nn0 = block_state0.snd; + uint8_t kk0 = block_state0.fst; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk0, .digest_length = nn0, .last_node = last_node0 }; + Hacl_Hash_Blake2b_Simd256_state_t scrut = *s; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) + { + r = 128U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)128U); + } + uint8_t *buf_1 = buf_; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 wv0[4U] KRML_POST_ALIGN(32) = { 0U }; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b[4U] KRML_POST_ALIGN(32) = { 0U }; + Hacl_Hash_Blake2b_Simd256_block_state_t + tmp_block_state = + { + .fst = i1.key_length, + .snd = i1.digest_length, + .thd = i1.last_node, + .f3 = { .fst = wv0, .snd = b } + }; + Lib_IntVector_Intrinsics_vec256 *src_b = block_state.f3.snd; + Lib_IntVector_Intrinsics_vec256 *dst_b = tmp_block_state.f3.snd; + memcpy(dst_b, src_b, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + uint64_t prev_len = total_len - (uint64_t)r; + uint32_t ite; + if (r % 128U == 0U && r > 0U) + { + ite = 128U; + } + else + { + ite = r % 128U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ + acc0 = tmp_block_state.f3; + Lib_IntVector_Intrinsics_vec256 *wv1 = acc0.fst; + Lib_IntVector_Intrinsics_vec256 *hash0 = acc0.snd; + uint32_t nb = 0U; + Hacl_Hash_Blake2b_Simd256_update_multi(0U, + wv1, + hash0, + FStar_UInt128_uint64_to_uint128(prev_len), + buf_multi, + nb); + uint64_t prev_len_last = total_len - (uint64_t)r; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ + acc = tmp_block_state.f3; + bool last_node1 = tmp_block_state.thd; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + Hacl_Hash_Blake2b_Simd256_update_last(r, + wv, + hash, + last_node1, + FStar_UInt128_uint64_to_uint128(prev_len_last), + r, + buf_last); + uint8_t nn1 = tmp_block_state.snd; + Hacl_Hash_Blake2b_Simd256_finish((uint32_t)nn1, dst, tmp_block_state.f3.snd); + Hacl_Hash_Blake2b_Simd256_block_state_t block_state1 = (*s).block_state; + bool last_node = block_state1.thd; + uint8_t nn = block_state1.snd; + uint8_t kk = block_state1.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }).digest_length; +} + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2b_Simd256_info(Hacl_Hash_Blake2b_Simd256_state_t *s) +{ + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }); +} + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2b_Simd256_free(Hacl_Hash_Blake2b_Simd256_state_t *state) +{ + Hacl_Hash_Blake2b_Simd256_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = scrut.block_state; + Lib_IntVector_Intrinsics_vec256 *b = block_state.f3.snd; + Lib_IntVector_Intrinsics_vec256 *wv = block_state.f3.fst; + KRML_ALIGNED_FREE(wv); + KRML_ALIGNED_FREE(b); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); +} + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_copy(Hacl_Hash_Blake2b_Simd256_state_t *state) +{ + Hacl_Hash_Blake2b_Simd256_state_t scrut = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + bool last_node = block_state0.thd; + uint8_t nn = block_state0.snd; + uint8_t kk1 = block_state0.fst; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + memcpy(buf, buf0, 128U * sizeof (uint8_t)); + Lib_IntVector_Intrinsics_vec256 + *wv = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(wv, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Lib_IntVector_Intrinsics_vec256 + *b = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(b, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Hacl_Hash_Blake2b_Simd256_block_state_t + block_state = + { + .fst = i.key_length, + .snd = i.digest_length, + .thd = i.last_node, + .f3 = { .fst = wv, .snd = b } + }; + Lib_IntVector_Intrinsics_vec256 *src_b = block_state0.f3.snd; + Lib_IntVector_Intrinsics_vec256 *dst_b = block_state.f3.snd; + memcpy(dst_b, src_b, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Hacl_Hash_Blake2b_Simd256_state_t + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Hash_Blake2b_Simd256_state_t + *p = + (Hacl_Hash_Blake2b_Simd256_state_t *)KRML_HOST_MALLOC(sizeof ( + Hacl_Hash_Blake2b_Simd256_state_t + )); + p[0U] = s; + return p; +} + +/** +Write the BLAKE2b digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2b_Simd256_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +) +{ + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b[4U] KRML_POST_ALIGN(32) = { 0U }; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b1[4U] KRML_POST_ALIGN(32) = { 0U }; + Hacl_Hash_Blake2b_Simd256_init(b, key_len, output_len); + update(b1, b, key_len, key, input_len, input); + Hacl_Hash_Blake2b_Simd256_finish(output_len, output, b); + Lib_Memzero0_memzero(b1, 4U, Lib_IntVector_Intrinsics_vec256, void *); + Lib_Memzero0_memzero(b, 4U, Lib_IntVector_Intrinsics_vec256, void *); +} + +/** +Write the BLAKE2b digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +) +{ + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b[4U] KRML_POST_ALIGN(32) = { 0U }; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b1[4U] KRML_POST_ALIGN(32) = { 0U }; + uint64_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = b; + Lib_IntVector_Intrinsics_vec256 *r1 = b + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = b + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = b + 3U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4, iv5, iv6, iv7); + uint8_t kk = params.key_length; + uint8_t nn = params.digest_length; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = params.salt + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = params.personal + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + tmp[0U] = + (uint64_t)nn + ^ + ((uint64_t)kk + << 8U + ^ + ((uint64_t)params.fanout + << 16U + ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + tmp[1U] = params.node_offset; + tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4_, iv5_, iv6_, iv7_); + update(b1, b, (uint32_t)params.key_length, key, input_len, input); + Hacl_Hash_Blake2b_Simd256_finish((uint32_t)params.digest_length, output, b); + Lib_Memzero0_memzero(b1, 4U, Lib_IntVector_Intrinsics_vec256, void *); + Lib_Memzero0_memzero(b, 4U, Lib_IntVector_Intrinsics_vec256, void *); +} + diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h new file mode 100644 index 00000000000000..6c11a4ba32134a --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h @@ -0,0 +1,231 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_Blake2b_Simd256_H +#define __Hacl_Hash_Blake2b_Simd256_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "python_hacl_namespaces.h" +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +#include "Hacl_Hash_Blake2b.h" +#include "libintvector.h" + +#define HACL_HASH_BLAKE2B_SIMD256_BLOCK_BYTES (128U) + +#define HACL_HASH_BLAKE2B_SIMD256_OUT_BYTES (64U) + +#define HACL_HASH_BLAKE2B_SIMD256_KEY_BYTES (64U) + +#define HACL_HASH_BLAKE2B_SIMD256_SALT_BYTES (16U) + +#define HACL_HASH_BLAKE2B_SIMD256_PERSONAL_BYTES (16U) + +typedef struct K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256__s +{ + Lib_IntVector_Intrinsics_vec256 *fst; + Lib_IntVector_Intrinsics_vec256 *snd; +} +K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_; + +typedef struct Hacl_Hash_Blake2b_Simd256_block_state_t_s +{ + uint8_t fst; + uint8_t snd; + bool thd; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ f3; +} +Hacl_Hash_Blake2b_Simd256_block_state_t; + +typedef struct Hacl_Hash_Blake2b_Simd256_state_t_s +{ + Hacl_Hash_Blake2b_Simd256_block_state_t block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Hash_Blake2b_Simd256_state_t; + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 256 for S, 64 for B. +- The digest_length must not exceed 256 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +); + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 256 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_malloc_with_key0(uint8_t *k, uint8_t kk); + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2b_Simd256_state_t *Hacl_Hash_Blake2b_Simd256_malloc(void); + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params( + Hacl_Hash_Blake2b_Simd256_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +); + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_Simd256_reset_with_key(Hacl_Hash_Blake2b_Simd256_state_t *s, uint8_t *k); + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_Simd256_reset(Hacl_Hash_Blake2b_Simd256_state_t *s); + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2b_Simd256_update( + Hacl_Hash_Blake2b_Simd256_state_t *state, + uint8_t *chunk, + uint32_t chunk_len +); + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 256 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2B_256_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2b_Simd256_digest(Hacl_Hash_Blake2b_Simd256_state_t *s, uint8_t *dst); + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2b_Simd256_info(Hacl_Hash_Blake2b_Simd256_state_t *s); + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2b_Simd256_free(Hacl_Hash_Blake2b_Simd256_state_t *state); + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_copy(Hacl_Hash_Blake2b_Simd256_state_t *state); + +/** +Write the BLAKE2b digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2b_Simd256_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +); + +/** +Write the BLAKE2b digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_Blake2b_Simd256_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c new file mode 100644 index 00000000000000..116499fedb3bd0 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c @@ -0,0 +1,15 @@ +// This file isn't part of a standard HACL source tree. +// +// It is required for compatibility with universal2 macOS builds. The code in +// Hacl_Hash_Blake2b_Simd256.c *will* compile on macOS x86_64, but *won't* +// compile on ARM64. However, because universal2 builds are compiled in a +// single pass, autoconf detects that the required compiler features *are* +// available, and tries to compile this file, which then fails because of the +// lack of support on ARM64. +// +// To compensate for this, autoconf will include *this* file instead of +// Hacl_Hash_Blake2b_Simd256.c when compiling for universal. This allows the +// underlying source code of HACL to remain unmodified. +#if !(defined(__APPLE__) && defined(__arm64__)) +#include "Hacl_Hash_Blake2b_Simd256.c" +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.c b/Modules/_hacl/Hacl_Hash_Blake2s.c new file mode 100644 index 00000000000000..e5e0ecd0bfde7e --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s.c @@ -0,0 +1,1513 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_Blake2s.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "lib_memzero0.h" + +static inline void +update_block( + uint32_t *wv, + uint32_t *hash, + bool flag, + bool last_node, + uint64_t totlen, + uint8_t *d +) +{ + uint32_t m_w[16U] = { 0U }; + KRML_MAYBE_FOR16(i, + 0U, + 16U, + 1U, + uint32_t *os = m_w; + uint8_t *bj = d + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + uint32_t mask[4U] = { 0U }; + uint32_t wv_14; + if (flag) + { + wv_14 = 0xFFFFFFFFU; + } + else + { + wv_14 = 0U; + } + uint32_t wv_15; + if (last_node) + { + wv_15 = 0xFFFFFFFFU; + } + else + { + wv_15 = 0U; + } + mask[0U] = (uint32_t)totlen; + mask[1U] = (uint32_t)(totlen >> 32U); + mask[2U] = wv_14; + mask[3U] = wv_15; + memcpy(wv, hash, 16U * sizeof (uint32_t)); + uint32_t *wv3 = wv + 12U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv3; + uint32_t x = wv3[i] ^ mask[i]; + os[i] = x;); + KRML_MAYBE_FOR10(i0, + 0U, + 10U, + 1U, + uint32_t start_idx = i0 % 10U * 16U; + uint32_t m_st[16U] = { 0U }; + uint32_t *r0 = m_st; + uint32_t *r1 = m_st + 4U; + uint32_t *r20 = m_st + 8U; + uint32_t *r30 = m_st + 12U; + uint32_t s0 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 0U]; + uint32_t s1 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 1U]; + uint32_t s2 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 2U]; + uint32_t s3 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 3U]; + uint32_t s4 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 4U]; + uint32_t s5 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 5U]; + uint32_t s6 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 6U]; + uint32_t s7 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 7U]; + uint32_t s8 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 8U]; + uint32_t s9 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 9U]; + uint32_t s10 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 10U]; + uint32_t s11 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 11U]; + uint32_t s12 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 12U]; + uint32_t s13 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 13U]; + uint32_t s14 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 14U]; + uint32_t s15 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 15U]; + uint32_t uu____0 = m_w[s2]; + uint32_t uu____1 = m_w[s4]; + uint32_t uu____2 = m_w[s6]; + r0[0U] = m_w[s0]; + r0[1U] = uu____0; + r0[2U] = uu____1; + r0[3U] = uu____2; + uint32_t uu____3 = m_w[s3]; + uint32_t uu____4 = m_w[s5]; + uint32_t uu____5 = m_w[s7]; + r1[0U] = m_w[s1]; + r1[1U] = uu____3; + r1[2U] = uu____4; + r1[3U] = uu____5; + uint32_t uu____6 = m_w[s10]; + uint32_t uu____7 = m_w[s12]; + uint32_t uu____8 = m_w[s14]; + r20[0U] = m_w[s8]; + r20[1U] = uu____6; + r20[2U] = uu____7; + r20[3U] = uu____8; + uint32_t uu____9 = m_w[s11]; + uint32_t uu____10 = m_w[s13]; + uint32_t uu____11 = m_w[s15]; + r30[0U] = m_w[s9]; + r30[1U] = uu____9; + r30[2U] = uu____10; + r30[3U] = uu____11; + uint32_t *x = m_st; + uint32_t *y = m_st + 4U; + uint32_t *z = m_st + 8U; + uint32_t *w = m_st + 12U; + uint32_t a = 0U; + uint32_t b0 = 1U; + uint32_t c0 = 2U; + uint32_t d10 = 3U; + uint32_t *wv_a0 = wv + a * 4U; + uint32_t *wv_b0 = wv + b0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a0; + uint32_t x1 = wv_a0[i] + wv_b0[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a0; + uint32_t x1 = wv_a0[i] + x[i]; + os[i] = x1;); + uint32_t *wv_a1 = wv + d10 * 4U; + uint32_t *wv_b1 = wv + a * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a1; + uint32_t x1 = wv_a1[i] ^ wv_b1[i]; + os[i] = x1;); + uint32_t *r10 = wv_a1; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r10; + uint32_t x1 = r10[i]; + uint32_t x10 = x1 >> 16U | x1 << 16U; + os[i] = x10;); + uint32_t *wv_a2 = wv + c0 * 4U; + uint32_t *wv_b2 = wv + d10 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a2; + uint32_t x1 = wv_a2[i] + wv_b2[i]; + os[i] = x1;); + uint32_t *wv_a3 = wv + b0 * 4U; + uint32_t *wv_b3 = wv + c0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a3; + uint32_t x1 = wv_a3[i] ^ wv_b3[i]; + os[i] = x1;); + uint32_t *r12 = wv_a3; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r12; + uint32_t x1 = r12[i]; + uint32_t x10 = x1 >> 12U | x1 << 20U; + os[i] = x10;); + uint32_t *wv_a4 = wv + a * 4U; + uint32_t *wv_b4 = wv + b0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a4; + uint32_t x1 = wv_a4[i] + wv_b4[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a4; + uint32_t x1 = wv_a4[i] + y[i]; + os[i] = x1;); + uint32_t *wv_a5 = wv + d10 * 4U; + uint32_t *wv_b5 = wv + a * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a5; + uint32_t x1 = wv_a5[i] ^ wv_b5[i]; + os[i] = x1;); + uint32_t *r13 = wv_a5; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r13; + uint32_t x1 = r13[i]; + uint32_t x10 = x1 >> 8U | x1 << 24U; + os[i] = x10;); + uint32_t *wv_a6 = wv + c0 * 4U; + uint32_t *wv_b6 = wv + d10 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a6; + uint32_t x1 = wv_a6[i] + wv_b6[i]; + os[i] = x1;); + uint32_t *wv_a7 = wv + b0 * 4U; + uint32_t *wv_b7 = wv + c0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a7; + uint32_t x1 = wv_a7[i] ^ wv_b7[i]; + os[i] = x1;); + uint32_t *r14 = wv_a7; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r14; + uint32_t x1 = r14[i]; + uint32_t x10 = x1 >> 7U | x1 << 25U; + os[i] = x10;); + uint32_t *r15 = wv + 4U; + uint32_t *r21 = wv + 8U; + uint32_t *r31 = wv + 12U; + uint32_t *r110 = r15; + uint32_t x00 = r110[1U]; + uint32_t x10 = r110[2U]; + uint32_t x20 = r110[3U]; + uint32_t x30 = r110[0U]; + r110[0U] = x00; + r110[1U] = x10; + r110[2U] = x20; + r110[3U] = x30; + uint32_t *r111 = r21; + uint32_t x01 = r111[2U]; + uint32_t x11 = r111[3U]; + uint32_t x21 = r111[0U]; + uint32_t x31 = r111[1U]; + r111[0U] = x01; + r111[1U] = x11; + r111[2U] = x21; + r111[3U] = x31; + uint32_t *r112 = r31; + uint32_t x02 = r112[3U]; + uint32_t x12 = r112[0U]; + uint32_t x22 = r112[1U]; + uint32_t x32 = r112[2U]; + r112[0U] = x02; + r112[1U] = x12; + r112[2U] = x22; + r112[3U] = x32; + uint32_t a0 = 0U; + uint32_t b = 1U; + uint32_t c = 2U; + uint32_t d1 = 3U; + uint32_t *wv_a = wv + a0 * 4U; + uint32_t *wv_b8 = wv + b * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a; + uint32_t x1 = wv_a[i] + wv_b8[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a; + uint32_t x1 = wv_a[i] + z[i]; + os[i] = x1;); + uint32_t *wv_a8 = wv + d1 * 4U; + uint32_t *wv_b9 = wv + a0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a8; + uint32_t x1 = wv_a8[i] ^ wv_b9[i]; + os[i] = x1;); + uint32_t *r16 = wv_a8; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r16; + uint32_t x1 = r16[i]; + uint32_t x13 = x1 >> 16U | x1 << 16U; + os[i] = x13;); + uint32_t *wv_a9 = wv + c * 4U; + uint32_t *wv_b10 = wv + d1 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a9; + uint32_t x1 = wv_a9[i] + wv_b10[i]; + os[i] = x1;); + uint32_t *wv_a10 = wv + b * 4U; + uint32_t *wv_b11 = wv + c * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a10; + uint32_t x1 = wv_a10[i] ^ wv_b11[i]; + os[i] = x1;); + uint32_t *r17 = wv_a10; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r17; + uint32_t x1 = r17[i]; + uint32_t x13 = x1 >> 12U | x1 << 20U; + os[i] = x13;); + uint32_t *wv_a11 = wv + a0 * 4U; + uint32_t *wv_b12 = wv + b * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a11; + uint32_t x1 = wv_a11[i] + wv_b12[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a11; + uint32_t x1 = wv_a11[i] + w[i]; + os[i] = x1;); + uint32_t *wv_a12 = wv + d1 * 4U; + uint32_t *wv_b13 = wv + a0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a12; + uint32_t x1 = wv_a12[i] ^ wv_b13[i]; + os[i] = x1;); + uint32_t *r18 = wv_a12; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r18; + uint32_t x1 = r18[i]; + uint32_t x13 = x1 >> 8U | x1 << 24U; + os[i] = x13;); + uint32_t *wv_a13 = wv + c * 4U; + uint32_t *wv_b14 = wv + d1 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a13; + uint32_t x1 = wv_a13[i] + wv_b14[i]; + os[i] = x1;); + uint32_t *wv_a14 = wv + b * 4U; + uint32_t *wv_b = wv + c * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a14; + uint32_t x1 = wv_a14[i] ^ wv_b[i]; + os[i] = x1;); + uint32_t *r19 = wv_a14; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r19; + uint32_t x1 = r19[i]; + uint32_t x13 = x1 >> 7U | x1 << 25U; + os[i] = x13;); + uint32_t *r113 = wv + 4U; + uint32_t *r2 = wv + 8U; + uint32_t *r3 = wv + 12U; + uint32_t *r11 = r113; + uint32_t x03 = r11[3U]; + uint32_t x13 = r11[0U]; + uint32_t x23 = r11[1U]; + uint32_t x33 = r11[2U]; + r11[0U] = x03; + r11[1U] = x13; + r11[2U] = x23; + r11[3U] = x33; + uint32_t *r114 = r2; + uint32_t x04 = r114[2U]; + uint32_t x14 = r114[3U]; + uint32_t x24 = r114[0U]; + uint32_t x34 = r114[1U]; + r114[0U] = x04; + r114[1U] = x14; + r114[2U] = x24; + r114[3U] = x34; + uint32_t *r115 = r3; + uint32_t x0 = r115[1U]; + uint32_t x1 = r115[2U]; + uint32_t x2 = r115[3U]; + uint32_t x3 = r115[0U]; + r115[0U] = x0; + r115[1U] = x1; + r115[2U] = x2; + r115[3U] = x3;); + uint32_t *s0 = hash; + uint32_t *s1 = hash + 4U; + uint32_t *r0 = wv; + uint32_t *r1 = wv + 4U; + uint32_t *r2 = wv + 8U; + uint32_t *r3 = wv + 12U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = s0; + uint32_t x = s0[i] ^ r0[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = s0; + uint32_t x = s0[i] ^ r2[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = s1; + uint32_t x = s1[i] ^ r1[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = s1; + uint32_t x = s1[i] ^ r3[i]; + os[i] = x;); +} + +void Hacl_Hash_Blake2s_init(uint32_t *hash, uint32_t kk, uint32_t nn) +{ + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = 32U, .key_length = 0U, .fanout = 1U, .depth = 1U, .leaf_length = 0U, + .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, .personal = personal + }; + uint32_t tmp[8U] = { 0U }; + uint32_t *r0 = hash; + uint32_t *r1 = hash + 4U; + uint32_t *r2 = hash + 8U; + uint32_t *r3 = hash + 12U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = p.salt + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = p.personal + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + tmp[0U] = + (uint32_t)(uint8_t)nn + ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + tmp[1U] = p.leaf_length; + tmp[2U] = (uint32_t)p.node_offset; + tmp[3U] = + (uint32_t)(p.node_offset >> 32U) + ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; +} + +static void update_key(uint32_t *wv, uint32_t *hash, uint32_t kk, uint8_t *k, uint32_t ll) +{ + uint64_t lb = (uint64_t)64U; + uint8_t b[64U] = { 0U }; + memcpy(b, k, kk * sizeof (uint8_t)); + if (ll == 0U) + { + update_block(wv, hash, true, false, lb, b); + } + else + { + update_block(wv, hash, false, false, lb, b); + } + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2s_update_multi( + uint32_t len, + uint32_t *wv, + uint32_t *hash, + uint64_t prev, + uint8_t *blocks, + uint32_t nb +) +{ + KRML_MAYBE_UNUSED_VAR(len); + for (uint32_t i = 0U; i < nb; i++) + { + uint64_t totlen = prev + (uint64_t)((i + 1U) * 64U); + uint8_t *b = blocks + i * 64U; + update_block(wv, hash, false, false, totlen, b); + } +} + +void +Hacl_Hash_Blake2s_update_last( + uint32_t len, + uint32_t *wv, + uint32_t *hash, + bool last_node, + uint64_t prev, + uint32_t rem, + uint8_t *d +) +{ + uint8_t b[64U] = { 0U }; + uint8_t *last = d + len - rem; + memcpy(b, last, rem * sizeof (uint8_t)); + uint64_t totlen = prev + (uint64_t)len; + update_block(wv, hash, true, last_node, totlen, b); + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +static void +update_blocks(uint32_t len, uint32_t *wv, uint32_t *hash, uint64_t prev, uint8_t *blocks) +{ + uint32_t nb0 = len / 64U; + uint32_t rem0 = len % 64U; + uint32_t nb; + if (rem0 == 0U && nb0 > 0U) + { + nb = nb0 - 1U; + } + else + { + nb = nb0; + } + uint32_t rem; + if (rem0 == 0U && nb0 > 0U) + { + rem = 64U; + } + else + { + rem = rem0; + } + Hacl_Hash_Blake2s_update_multi(len, wv, hash, prev, blocks, nb); + Hacl_Hash_Blake2s_update_last(len, wv, hash, false, prev, rem, blocks); +} + +static inline void +update(uint32_t *wv, uint32_t *hash, uint32_t kk, uint8_t *k, uint32_t ll, uint8_t *d) +{ + uint64_t lb = (uint64_t)64U; + if (kk > 0U) + { + update_key(wv, hash, kk, k, ll); + if (!(ll == 0U)) + { + update_blocks(ll, wv, hash, lb, d); + return; + } + return; + } + update_blocks(ll, wv, hash, (uint64_t)0U, d); +} + +void Hacl_Hash_Blake2s_finish(uint32_t nn, uint8_t *output, uint32_t *hash) +{ + uint8_t b[32U] = { 0U }; + uint8_t *first = b; + uint8_t *second = b + 16U; + uint32_t *row0 = hash; + uint32_t *row1 = hash + 4U; + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store32_le(first + i * 4U, row0[i]);); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store32_le(second + i * 4U, row1[i]);); + uint8_t *final = b; + memcpy(output, final, nn * sizeof (uint8_t)); + Lib_Memzero0_memzero(b, 32U, uint8_t, void *); +} + +static Hacl_Hash_Blake2s_state_t +*malloc_raw(Hacl_Hash_Blake2b_index kk, Hacl_Hash_Blake2b_params_and_key key) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *wv = (uint32_t *)KRML_HOST_CALLOC(16U, sizeof (uint32_t)); + uint32_t *b = (uint32_t *)KRML_HOST_CALLOC(16U, sizeof (uint32_t)); + Hacl_Hash_Blake2s_block_state_t + block_state = + { + .fst = kk.key_length, + .snd = kk.digest_length, + .thd = kk.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint8_t kk10 = kk.key_length; + uint32_t ite; + if (kk10 != 0U) + { + ite = 64U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2s_state_t + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + Hacl_Hash_Blake2s_state_t + *p = (Hacl_Hash_Blake2s_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_Blake2s_state_t)); + p[0U] = s; + Hacl_Hash_Blake2b_blake2_params *p1 = key.fst; + uint8_t kk1 = p1->key_length; + uint8_t nn = p1->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint32_t *h = block_state.f3.snd; + uint32_t kk2 = (uint32_t)i.key_length; + uint8_t *k_1 = key.snd; + if (!(kk2 == 0U)) + { + uint8_t *sub_b = buf + kk2; + memset(sub_b, 0U, (64U - kk2) * sizeof (uint8_t)); + memcpy(buf, k_1, kk2 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p1[0U]; + uint32_t tmp[8U] = { 0U }; + uint32_t *r0 = h; + uint32_t *r1 = h + 4U; + uint32_t *r2 = h + 8U; + uint32_t *r3 = h + 12U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r4 = u; + uint32_t x = r4; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r4 = u; + uint32_t x = r4; + os[i0] = x;); + tmp[0U] = + (uint32_t)pv.digest_length + ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + tmp[1U] = pv.leaf_length; + tmp[2U] = (uint32_t)pv.node_offset; + tmp[3U] = + (uint32_t)(pv.node_offset >> 32U) + ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + return p; +} + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 32 for S, 64 for B. +- The digest_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_state_t +*Hacl_Hash_Blake2s_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + Hacl_Hash_Blake2b_index + i1 = { .key_length = pv.key_length, .digest_length = pv.digest_length, .last_node = last_node }; + return malloc_raw(i1, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_malloc_with_key(uint8_t *k, uint8_t kk) +{ + uint8_t nn = 32U; + Hacl_Hash_Blake2b_index i = { .key_length = kk, .digest_length = nn, .last_node = false }; + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = i.digest_length, .key_length = i.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + Hacl_Hash_Blake2s_state_t *s = Hacl_Hash_Blake2s_malloc_with_params_and_key(&p0, false, k); + return s; +} + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_malloc(void) +{ + return Hacl_Hash_Blake2s_malloc_with_key(NULL, 0U); +} + +static Hacl_Hash_Blake2b_index index_of_state(Hacl_Hash_Blake2s_state_t *s) +{ + Hacl_Hash_Blake2s_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk1 = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk1, .digest_length = nn, .last_node = last_node }); +} + +static void reset_raw(Hacl_Hash_Blake2s_state_t *state, Hacl_Hash_Blake2b_params_and_key key) +{ + Hacl_Hash_Blake2s_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2s_block_state_t block_state = scrut.block_state; + bool last_node0 = block_state.thd; + uint8_t nn0 = block_state.snd; + uint8_t kk10 = block_state.fst; + Hacl_Hash_Blake2b_index + i = { .key_length = kk10, .digest_length = nn0, .last_node = last_node0 }; + KRML_MAYBE_UNUSED_VAR(i); + Hacl_Hash_Blake2b_blake2_params *p = key.fst; + uint8_t kk1 = p->key_length; + uint8_t nn = p->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint32_t *h = block_state.f3.snd; + uint32_t kk2 = (uint32_t)i1.key_length; + uint8_t *k_1 = key.snd; + if (!(kk2 == 0U)) + { + uint8_t *sub_b = buf + kk2; + memset(sub_b, 0U, (64U - kk2) * sizeof (uint8_t)); + memcpy(buf, k_1, kk2 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + uint32_t tmp[8U] = { 0U }; + uint32_t *r0 = h; + uint32_t *r1 = h + 4U; + uint32_t *r2 = h + 8U; + uint32_t *r3 = h + 12U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i0] = x;); + tmp[0U] = + (uint32_t)pv.digest_length + ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + tmp[1U] = pv.leaf_length; + tmp[2U] = (uint32_t)pv.node_offset; + tmp[3U] = + (uint32_t)(pv.node_offset >> 32U) + ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + uint8_t kk11 = i.key_length; + uint32_t ite; + if (kk11 != 0U) + { + ite = 64U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2s_state_t + tmp8 = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + state[0U] = tmp8; +} + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_reset_with_key_and_params( + Hacl_Hash_Blake2s_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +) +{ + index_of_state(s); + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_reset_with_key(Hacl_Hash_Blake2s_state_t *s, uint8_t *k) +{ + Hacl_Hash_Blake2b_index idx = index_of_state(s); + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = idx.digest_length, .key_length = idx.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = &p0, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_reset(Hacl_Hash_Blake2s_state_t *s) +{ + Hacl_Hash_Blake2s_reset_with_key(s, NULL); +} + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint32_t chunk_len) +{ + Hacl_Hash_Blake2s_state_t s = *state; + uint64_t total_len = s.total_len; + if ((uint64_t)chunk_len > 0xffffffffffffffffULL - total_len) + { + return Hacl_Streaming_Types_MaximumLengthExceeded; + } + uint32_t sz; + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) + { + sz = 64U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)64U); + } + if (chunk_len <= 64U - sz) + { + Hacl_Hash_Blake2s_state_t s1 = *state; + Hacl_Hash_Blake2s_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state + = + ( + (Hacl_Hash_Blake2s_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == 0U) + { + Hacl_Hash_Blake2s_state_t s1 = *state; + Hacl_Hash_Blake2s_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____uint32_t___uint32_t_ acc = block_state1.f3; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2s_update_multi(64U, wv, hash, prevlen, buf, nb); + } + uint32_t ite; + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) + { + ite = 64U; + } + else + { + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); + } + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + K____uint32_t___uint32_t_ acc = block_state1.f3; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + uint32_t nb = data1_len / 64U; + Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2s_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)chunk_len + } + ); + } + else + { + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_Blake2s_state_t s1 = *state; + Hacl_Hash_Blake2s_block_state_t block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) + { + sz10 = 64U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *state + = + ( + (Hacl_Hash_Blake2s_state_t){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Hash_Blake2s_state_t s10 = *state; + Hacl_Hash_Blake2s_block_state_t block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____uint32_t___uint32_t_ acc = block_state1.f3; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2s_update_multi(64U, wv, hash, prevlen, buf, nb); + } + uint32_t ite; + if + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) + { + ite = 64U; + } + else + { + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); + } + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + K____uint32_t___uint32_t_ acc = block_state1.f3; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + uint32_t nb = data1_len / 64U; + Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2s_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(chunk_len - diff) + } + ); + } + return Hacl_Streaming_Types_Success; +} + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 32 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2S_32_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2s_digest(Hacl_Hash_Blake2s_state_t *s, uint8_t *dst) +{ + Hacl_Hash_Blake2s_block_state_t block_state0 = (*s).block_state; + bool last_node0 = block_state0.thd; + uint8_t nn0 = block_state0.snd; + uint8_t kk0 = block_state0.fst; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk0, .digest_length = nn0, .last_node = last_node0 }; + Hacl_Hash_Blake2s_state_t scrut = *s; + Hacl_Hash_Blake2s_block_state_t block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) + { + r = 64U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)64U); + } + uint8_t *buf_1 = buf_; + uint32_t wv0[16U] = { 0U }; + uint32_t b[16U] = { 0U }; + Hacl_Hash_Blake2s_block_state_t + tmp_block_state = + { + .fst = i1.key_length, + .snd = i1.digest_length, + .thd = i1.last_node, + .f3 = { .fst = wv0, .snd = b } + }; + uint32_t *src_b = block_state.f3.snd; + uint32_t *dst_b = tmp_block_state.f3.snd; + memcpy(dst_b, src_b, 16U * sizeof (uint32_t)); + uint64_t prev_len = total_len - (uint64_t)r; + uint32_t ite; + if (r % 64U == 0U && r > 0U) + { + ite = 64U; + } + else + { + ite = r % 64U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + K____uint32_t___uint32_t_ acc0 = tmp_block_state.f3; + uint32_t *wv1 = acc0.fst; + uint32_t *hash0 = acc0.snd; + uint32_t nb = 0U; + Hacl_Hash_Blake2s_update_multi(0U, wv1, hash0, prev_len, buf_multi, nb); + uint64_t prev_len_last = total_len - (uint64_t)r; + K____uint32_t___uint32_t_ acc = tmp_block_state.f3; + bool last_node1 = tmp_block_state.thd; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + Hacl_Hash_Blake2s_update_last(r, wv, hash, last_node1, prev_len_last, r, buf_last); + uint8_t nn1 = tmp_block_state.snd; + Hacl_Hash_Blake2s_finish((uint32_t)nn1, dst, tmp_block_state.f3.snd); + Hacl_Hash_Blake2s_block_state_t block_state1 = (*s).block_state; + bool last_node = block_state1.thd; + uint8_t nn = block_state1.snd; + uint8_t kk = block_state1.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }).digest_length; +} + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2s_info(Hacl_Hash_Blake2s_state_t *s) +{ + Hacl_Hash_Blake2s_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }); +} + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2s_free(Hacl_Hash_Blake2s_state_t *state) +{ + Hacl_Hash_Blake2s_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2s_block_state_t block_state = scrut.block_state; + uint32_t *b = block_state.f3.snd; + uint32_t *wv = block_state.f3.fst; + KRML_HOST_FREE(wv); + KRML_HOST_FREE(b); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); +} + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_copy(Hacl_Hash_Blake2s_state_t *state) +{ + Hacl_Hash_Blake2s_state_t scrut = *state; + Hacl_Hash_Blake2s_block_state_t block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + bool last_node = block_state0.thd; + uint8_t nn = block_state0.snd; + uint8_t kk1 = block_state0.fst; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *wv = (uint32_t *)KRML_HOST_CALLOC(16U, sizeof (uint32_t)); + uint32_t *b = (uint32_t *)KRML_HOST_CALLOC(16U, sizeof (uint32_t)); + Hacl_Hash_Blake2s_block_state_t + block_state = + { + .fst = i.key_length, + .snd = i.digest_length, + .thd = i.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint32_t *src_b = block_state0.f3.snd; + uint32_t *dst_b = block_state.f3.snd; + memcpy(dst_b, src_b, 16U * sizeof (uint32_t)); + Hacl_Hash_Blake2s_state_t + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Hash_Blake2s_state_t + *p = (Hacl_Hash_Blake2s_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_Blake2s_state_t)); + p[0U] = s; + return p; +} + +/** +Write the BLAKE2s digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2s_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +) +{ + uint32_t b[16U] = { 0U }; + uint32_t b1[16U] = { 0U }; + Hacl_Hash_Blake2s_init(b, key_len, output_len); + update(b1, b, key_len, key, input_len, input); + Hacl_Hash_Blake2s_finish(output_len, output, b); + Lib_Memzero0_memzero(b1, 16U, uint32_t, void *); + Lib_Memzero0_memzero(b, 16U, uint32_t, void *); +} + +/** +Write the BLAKE2s digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2s_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +) +{ + uint32_t b[16U] = { 0U }; + uint32_t b1[16U] = { 0U }; + uint32_t tmp[8U] = { 0U }; + uint32_t *r0 = b; + uint32_t *r1 = b + 4U; + uint32_t *r2 = b + 8U; + uint32_t *r3 = b + 12U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = params.salt + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = params.personal + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + tmp[0U] = + (uint32_t)params.digest_length + ^ + ((uint32_t)params.key_length + << 8U + ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + tmp[1U] = params.leaf_length; + tmp[2U] = (uint32_t)params.node_offset; + tmp[3U] = + (uint32_t)(params.node_offset >> 32U) + ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + update(b1, b, (uint32_t)params.key_length, key, input_len, input); + Hacl_Hash_Blake2s_finish((uint32_t)params.digest_length, output, b); + Lib_Memzero0_memzero(b1, 16U, uint32_t, void *); + Lib_Memzero0_memzero(b, 16U, uint32_t, void *); +} + diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.h b/Modules/_hacl/Hacl_Hash_Blake2s.h new file mode 100644 index 00000000000000..5c01da144018e3 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s.h @@ -0,0 +1,222 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_Blake2s_H +#define __Hacl_Hash_Blake2s_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "python_hacl_namespaces.h" +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" +#include "Hacl_Hash_Blake2b.h" + +#define HACL_HASH_BLAKE2S_BLOCK_BYTES (64U) + +#define HACL_HASH_BLAKE2S_OUT_BYTES (32U) + +#define HACL_HASH_BLAKE2S_KEY_BYTES (32U) + +#define HACL_HASH_BLAKE2S_SALT_BYTES (8U) + +#define HACL_HASH_BLAKE2S_PERSONAL_BYTES (8U) + +typedef struct K____uint32_t___uint32_t__s +{ + uint32_t *fst; + uint32_t *snd; +} +K____uint32_t___uint32_t_; + +typedef struct Hacl_Hash_Blake2s_block_state_t_s +{ + uint8_t fst; + uint8_t snd; + bool thd; + K____uint32_t___uint32_t_ f3; +} +Hacl_Hash_Blake2s_block_state_t; + +typedef struct Hacl_Hash_Blake2s_state_t_s +{ + Hacl_Hash_Blake2s_block_state_t block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Hash_Blake2s_state_t; + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 32 for S, 64 for B. +- The digest_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_state_t +*Hacl_Hash_Blake2s_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +); + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_malloc_with_key(uint8_t *k, uint8_t kk); + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_malloc(void); + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_reset_with_key_and_params( + Hacl_Hash_Blake2s_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +); + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_reset_with_key(Hacl_Hash_Blake2s_state_t *s, uint8_t *k); + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_reset(Hacl_Hash_Blake2s_state_t *s); + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint32_t chunk_len); + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 32 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2S_32_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2s_digest(Hacl_Hash_Blake2s_state_t *s, uint8_t *dst); + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2s_info(Hacl_Hash_Blake2s_state_t *s); + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2s_free(Hacl_Hash_Blake2s_state_t *state); + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_copy(Hacl_Hash_Blake2s_state_t *state); + +/** +Write the BLAKE2s digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2s_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +); + +/** +Write the BLAKE2s digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2s_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_Blake2s_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c new file mode 100644 index 00000000000000..f675a7f14f192f --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c @@ -0,0 +1,1350 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_Blake2s_Simd128.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "lib_memzero0.h" + +static inline void +update_block( + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + bool flag, + bool last_node, + uint64_t totlen, + uint8_t *d +) +{ + uint32_t m_w[16U] = { 0U }; + KRML_MAYBE_FOR16(i, + 0U, + 16U, + 1U, + uint32_t *os = m_w; + uint8_t *bj = d + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + Lib_IntVector_Intrinsics_vec128 mask = Lib_IntVector_Intrinsics_vec128_zero; + uint32_t wv_14; + if (flag) + { + wv_14 = 0xFFFFFFFFU; + } + else + { + wv_14 = 0U; + } + uint32_t wv_15; + if (last_node) + { + wv_15 = 0xFFFFFFFFU; + } + else + { + wv_15 = 0U; + } + mask = + Lib_IntVector_Intrinsics_vec128_load32s((uint32_t)totlen, + (uint32_t)(totlen >> 32U), + wv_14, + wv_15); + memcpy(wv, hash, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Lib_IntVector_Intrinsics_vec128 *wv3 = wv + 3U; + wv3[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv3[0U], mask); + KRML_MAYBE_FOR10(i, + 0U, + 10U, + 1U, + uint32_t start_idx = i % 10U * 16U; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 m_st[4U] KRML_POST_ALIGN(16) = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = m_st; + Lib_IntVector_Intrinsics_vec128 *r1 = m_st + 1U; + Lib_IntVector_Intrinsics_vec128 *r20 = m_st + 2U; + Lib_IntVector_Intrinsics_vec128 *r30 = m_st + 3U; + uint32_t s0 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 0U]; + uint32_t s1 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 1U]; + uint32_t s2 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 2U]; + uint32_t s3 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 3U]; + uint32_t s4 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 4U]; + uint32_t s5 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 5U]; + uint32_t s6 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 6U]; + uint32_t s7 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 7U]; + uint32_t s8 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 8U]; + uint32_t s9 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 9U]; + uint32_t s10 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 10U]; + uint32_t s11 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 11U]; + uint32_t s12 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 12U]; + uint32_t s13 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 13U]; + uint32_t s14 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 14U]; + uint32_t s15 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 15U]; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(m_w[s0], m_w[s2], m_w[s4], m_w[s6]); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(m_w[s1], m_w[s3], m_w[s5], m_w[s7]); + r20[0U] = Lib_IntVector_Intrinsics_vec128_load32s(m_w[s8], m_w[s10], m_w[s12], m_w[s14]); + r30[0U] = Lib_IntVector_Intrinsics_vec128_load32s(m_w[s9], m_w[s11], m_w[s13], m_w[s15]); + Lib_IntVector_Intrinsics_vec128 *x = m_st; + Lib_IntVector_Intrinsics_vec128 *y = m_st + 1U; + Lib_IntVector_Intrinsics_vec128 *z = m_st + 2U; + Lib_IntVector_Intrinsics_vec128 *w = m_st + 3U; + uint32_t a = 0U; + uint32_t b0 = 1U; + uint32_t c0 = 2U; + uint32_t d10 = 3U; + Lib_IntVector_Intrinsics_vec128 *wv_a0 = wv + a * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b0 = wv + b0 * 1U; + wv_a0[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a0[0U], wv_b0[0U]); + wv_a0[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a0[0U], x[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a1 = wv + d10 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b1 = wv + a * 1U; + wv_a1[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a1[0U], wv_b1[0U]); + wv_a1[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a1[0U], 16U); + Lib_IntVector_Intrinsics_vec128 *wv_a2 = wv + c0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b2 = wv + d10 * 1U; + wv_a2[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a2[0U], wv_b2[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a3 = wv + b0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b3 = wv + c0 * 1U; + wv_a3[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a3[0U], wv_b3[0U]); + wv_a3[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a3[0U], 12U); + Lib_IntVector_Intrinsics_vec128 *wv_a4 = wv + a * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b4 = wv + b0 * 1U; + wv_a4[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a4[0U], wv_b4[0U]); + wv_a4[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a4[0U], y[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a5 = wv + d10 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b5 = wv + a * 1U; + wv_a5[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a5[0U], wv_b5[0U]); + wv_a5[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a5[0U], 8U); + Lib_IntVector_Intrinsics_vec128 *wv_a6 = wv + c0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b6 = wv + d10 * 1U; + wv_a6[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a6[0U], wv_b6[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a7 = wv + b0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b7 = wv + c0 * 1U; + wv_a7[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a7[0U], wv_b7[0U]); + wv_a7[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a7[0U], 7U); + Lib_IntVector_Intrinsics_vec128 *r10 = wv + 1U; + Lib_IntVector_Intrinsics_vec128 *r21 = wv + 2U; + Lib_IntVector_Intrinsics_vec128 *r31 = wv + 3U; + Lib_IntVector_Intrinsics_vec128 v00 = r10[0U]; + Lib_IntVector_Intrinsics_vec128 + v1 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v00, 1U); + r10[0U] = v1; + Lib_IntVector_Intrinsics_vec128 v01 = r21[0U]; + Lib_IntVector_Intrinsics_vec128 + v10 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v01, 2U); + r21[0U] = v10; + Lib_IntVector_Intrinsics_vec128 v02 = r31[0U]; + Lib_IntVector_Intrinsics_vec128 + v11 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v02, 3U); + r31[0U] = v11; + uint32_t a0 = 0U; + uint32_t b = 1U; + uint32_t c = 2U; + uint32_t d1 = 3U; + Lib_IntVector_Intrinsics_vec128 *wv_a = wv + a0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b8 = wv + b * 1U; + wv_a[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a[0U], wv_b8[0U]); + wv_a[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a[0U], z[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a8 = wv + d1 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b9 = wv + a0 * 1U; + wv_a8[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a8[0U], wv_b9[0U]); + wv_a8[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a8[0U], 16U); + Lib_IntVector_Intrinsics_vec128 *wv_a9 = wv + c * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b10 = wv + d1 * 1U; + wv_a9[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a9[0U], wv_b10[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a10 = wv + b * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b11 = wv + c * 1U; + wv_a10[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a10[0U], wv_b11[0U]); + wv_a10[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a10[0U], 12U); + Lib_IntVector_Intrinsics_vec128 *wv_a11 = wv + a0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b12 = wv + b * 1U; + wv_a11[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a11[0U], wv_b12[0U]); + wv_a11[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a11[0U], w[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a12 = wv + d1 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b13 = wv + a0 * 1U; + wv_a12[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a12[0U], wv_b13[0U]); + wv_a12[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a12[0U], 8U); + Lib_IntVector_Intrinsics_vec128 *wv_a13 = wv + c * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b14 = wv + d1 * 1U; + wv_a13[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a13[0U], wv_b14[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a14 = wv + b * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b = wv + c * 1U; + wv_a14[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a14[0U], wv_b[0U]); + wv_a14[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a14[0U], 7U); + Lib_IntVector_Intrinsics_vec128 *r11 = wv + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = wv + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = wv + 3U; + Lib_IntVector_Intrinsics_vec128 v0 = r11[0U]; + Lib_IntVector_Intrinsics_vec128 + v12 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v0, 3U); + r11[0U] = v12; + Lib_IntVector_Intrinsics_vec128 v03 = r2[0U]; + Lib_IntVector_Intrinsics_vec128 + v13 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v03, 2U); + r2[0U] = v13; + Lib_IntVector_Intrinsics_vec128 v04 = r3[0U]; + Lib_IntVector_Intrinsics_vec128 + v14 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v04, 1U); + r3[0U] = v14;); + Lib_IntVector_Intrinsics_vec128 *s0 = hash; + Lib_IntVector_Intrinsics_vec128 *s1 = hash + 1U; + Lib_IntVector_Intrinsics_vec128 *r0 = wv; + Lib_IntVector_Intrinsics_vec128 *r1 = wv + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = wv + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = wv + 3U; + s0[0U] = Lib_IntVector_Intrinsics_vec128_xor(s0[0U], r0[0U]); + s0[0U] = Lib_IntVector_Intrinsics_vec128_xor(s0[0U], r2[0U]); + s1[0U] = Lib_IntVector_Intrinsics_vec128_xor(s1[0U], r1[0U]); + s1[0U] = Lib_IntVector_Intrinsics_vec128_xor(s1[0U], r3[0U]); +} + +void +Hacl_Hash_Blake2s_Simd128_init(Lib_IntVector_Intrinsics_vec128 *hash, uint32_t kk, uint32_t nn) +{ + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = 32U, .key_length = 0U, .fanout = 1U, .depth = 1U, .leaf_length = 0U, + .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, .personal = personal + }; + uint32_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = hash; + Lib_IntVector_Intrinsics_vec128 *r1 = hash + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = hash + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = hash + 3U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4, iv5, iv6, iv7); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = p.salt + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = p.personal + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + tmp[0U] = + (uint32_t)(uint8_t)nn + ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + tmp[1U] = p.leaf_length; + tmp[2U] = (uint32_t)p.node_offset; + tmp[3U] = + (uint32_t)(p.node_offset >> 32U) + ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4_, iv5_, iv6_, iv7_); +} + +static void +update_key( + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint32_t kk, + uint8_t *k, + uint32_t ll +) +{ + uint64_t lb = (uint64_t)64U; + uint8_t b[64U] = { 0U }; + memcpy(b, k, kk * sizeof (uint8_t)); + if (ll == 0U) + { + update_block(wv, hash, true, false, lb, b); + } + else + { + update_block(wv, hash, false, false, lb, b); + } + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2s_Simd128_update_multi( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint64_t prev, + uint8_t *blocks, + uint32_t nb +) +{ + KRML_MAYBE_UNUSED_VAR(len); + for (uint32_t i = 0U; i < nb; i++) + { + uint64_t totlen = prev + (uint64_t)((i + 1U) * 64U); + uint8_t *b = blocks + i * 64U; + update_block(wv, hash, false, false, totlen, b); + } +} + +void +Hacl_Hash_Blake2s_Simd128_update_last( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + bool last_node, + uint64_t prev, + uint32_t rem, + uint8_t *d +) +{ + uint8_t b[64U] = { 0U }; + uint8_t *last = d + len - rem; + memcpy(b, last, rem * sizeof (uint8_t)); + uint64_t totlen = prev + (uint64_t)len; + update_block(wv, hash, true, last_node, totlen, b); + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +static inline void +update_blocks( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint64_t prev, + uint8_t *blocks +) +{ + uint32_t nb0 = len / 64U; + uint32_t rem0 = len % 64U; + uint32_t nb; + if (rem0 == 0U && nb0 > 0U) + { + nb = nb0 - 1U; + } + else + { + nb = nb0; + } + uint32_t rem; + if (rem0 == 0U && nb0 > 0U) + { + rem = 64U; + } + else + { + rem = rem0; + } + Hacl_Hash_Blake2s_Simd128_update_multi(len, wv, hash, prev, blocks, nb); + Hacl_Hash_Blake2s_Simd128_update_last(len, wv, hash, false, prev, rem, blocks); +} + +static inline void +update( + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint32_t kk, + uint8_t *k, + uint32_t ll, + uint8_t *d +) +{ + uint64_t lb = (uint64_t)64U; + if (kk > 0U) + { + update_key(wv, hash, kk, k, ll); + if (!(ll == 0U)) + { + update_blocks(ll, wv, hash, lb, d); + return; + } + return; + } + update_blocks(ll, wv, hash, (uint64_t)0U, d); +} + +void +Hacl_Hash_Blake2s_Simd128_finish( + uint32_t nn, + uint8_t *output, + Lib_IntVector_Intrinsics_vec128 *hash +) +{ + uint8_t b[32U] = { 0U }; + uint8_t *first = b; + uint8_t *second = b + 16U; + Lib_IntVector_Intrinsics_vec128 *row0 = hash; + Lib_IntVector_Intrinsics_vec128 *row1 = hash + 1U; + Lib_IntVector_Intrinsics_vec128_store32_le(first, row0[0U]); + Lib_IntVector_Intrinsics_vec128_store32_le(second, row1[0U]); + uint8_t *final = b; + memcpy(output, final, nn * sizeof (uint8_t)); + Lib_Memzero0_memzero(b, 32U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32( + uint32_t *st32, + Lib_IntVector_Intrinsics_vec128 *st +) +{ + Lib_IntVector_Intrinsics_vec128 *r0 = st; + Lib_IntVector_Intrinsics_vec128 *r1 = st + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = st + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = st + 3U; + uint32_t *b0 = st32; + uint32_t *b1 = st32 + 4U; + uint32_t *b2 = st32 + 8U; + uint32_t *b3 = st32 + 12U; + uint8_t b8[16U] = { 0U }; + Lib_IntVector_Intrinsics_vec128_store32_le(b8, r0[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = b0; + uint8_t *bj = b8 + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + uint8_t b80[16U] = { 0U }; + Lib_IntVector_Intrinsics_vec128_store32_le(b80, r1[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = b1; + uint8_t *bj = b80 + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + uint8_t b81[16U] = { 0U }; + Lib_IntVector_Intrinsics_vec128_store32_le(b81, r2[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = b2; + uint8_t *bj = b81 + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + uint8_t b82[16U] = { 0U }; + Lib_IntVector_Intrinsics_vec128_store32_le(b82, r3[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = b3; + uint8_t *bj = b82 + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); +} + +void +Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32( + Lib_IntVector_Intrinsics_vec128 *st, + uint32_t *st32 +) +{ + Lib_IntVector_Intrinsics_vec128 *r0 = st; + Lib_IntVector_Intrinsics_vec128 *r1 = st + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = st + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = st + 3U; + uint32_t *b0 = st32; + uint32_t *b1 = st32 + 4U; + uint32_t *b2 = st32 + 8U; + uint32_t *b3 = st32 + 12U; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(b0[0U], b0[1U], b0[2U], b0[3U]); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(b1[0U], b1[1U], b1[2U], b1[3U]); + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(b2[0U], b2[1U], b2[2U], b2[3U]); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(b3[0U], b3[1U], b3[2U], b3[3U]); +} + +Lib_IntVector_Intrinsics_vec128 *Hacl_Hash_Blake2s_Simd128_malloc_with_key(void) +{ + Lib_IntVector_Intrinsics_vec128 + *buf = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(buf, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + return buf; +} + +static Hacl_Hash_Blake2s_Simd128_state_t +*malloc_raw(Hacl_Hash_Blake2b_index kk, Hacl_Hash_Blake2b_params_and_key key) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + Lib_IntVector_Intrinsics_vec128 + *wv = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(wv, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Lib_IntVector_Intrinsics_vec128 + *b = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(b, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Hacl_Hash_Blake2s_Simd128_block_state_t + block_state = + { + .fst = kk.key_length, + .snd = kk.digest_length, + .thd = kk.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint8_t kk10 = kk.key_length; + uint32_t ite; + if (kk10 != 0U) + { + ite = 64U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2s_Simd128_state_t + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + Hacl_Hash_Blake2s_Simd128_state_t + *p = + (Hacl_Hash_Blake2s_Simd128_state_t *)KRML_HOST_MALLOC(sizeof ( + Hacl_Hash_Blake2s_Simd128_state_t + )); + p[0U] = s; + Hacl_Hash_Blake2b_blake2_params *p1 = key.fst; + uint8_t kk1 = p1->key_length; + uint8_t nn = p1->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + Lib_IntVector_Intrinsics_vec128 *h = block_state.f3.snd; + uint32_t kk2 = (uint32_t)i.key_length; + uint8_t *k_1 = key.snd; + if (!(kk2 == 0U)) + { + uint8_t *sub_b = buf + kk2; + memset(sub_b, 0U, (64U - kk2) * sizeof (uint8_t)); + memcpy(buf, k_1, kk2 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p1[0U]; + uint32_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = h; + Lib_IntVector_Intrinsics_vec128 *r1 = h + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = h + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = h + 3U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4, iv5, iv6, iv7); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r4 = u; + uint32_t x = r4; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r4 = u; + uint32_t x = r4; + os[i0] = x;); + tmp[0U] = + (uint32_t)pv.digest_length + ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + tmp[1U] = pv.leaf_length; + tmp[2U] = (uint32_t)pv.node_offset; + tmp[3U] = + (uint32_t)(pv.node_offset >> 32U) + ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4_, iv5_, iv6_, iv7_); + return p; +} + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 128 for S, 64 for B. +- The digest_length must not exceed 128 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + Hacl_Hash_Blake2b_index + i1 = { .key_length = pv.key_length, .digest_length = pv.digest_length, .last_node = last_node }; + return malloc_raw(i1, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 128 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_malloc_with_key0(uint8_t *k, uint8_t kk) +{ + uint8_t nn = 32U; + Hacl_Hash_Blake2b_index i = { .key_length = kk, .digest_length = nn, .last_node = false }; + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = i.digest_length, .key_length = i.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + Hacl_Hash_Blake2s_Simd128_state_t + *s = Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key(&p0, false, k); + return s; +} + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2s_Simd128_state_t *Hacl_Hash_Blake2s_Simd128_malloc(void) +{ + return Hacl_Hash_Blake2s_Simd128_malloc_with_key0(NULL, 0U); +} + +static Hacl_Hash_Blake2b_index index_of_state(Hacl_Hash_Blake2s_Simd128_state_t *s) +{ + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk1 = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk1, .digest_length = nn, .last_node = last_node }); +} + +static void +reset_raw(Hacl_Hash_Blake2s_Simd128_state_t *state, Hacl_Hash_Blake2b_params_and_key key) +{ + Hacl_Hash_Blake2s_Simd128_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = scrut.block_state; + bool last_node0 = block_state.thd; + uint8_t nn0 = block_state.snd; + uint8_t kk10 = block_state.fst; + Hacl_Hash_Blake2b_index + i = { .key_length = kk10, .digest_length = nn0, .last_node = last_node0 }; + KRML_MAYBE_UNUSED_VAR(i); + Hacl_Hash_Blake2b_blake2_params *p = key.fst; + uint8_t kk1 = p->key_length; + uint8_t nn = p->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + Lib_IntVector_Intrinsics_vec128 *h = block_state.f3.snd; + uint32_t kk2 = (uint32_t)i1.key_length; + uint8_t *k_1 = key.snd; + if (!(kk2 == 0U)) + { + uint8_t *sub_b = buf + kk2; + memset(sub_b, 0U, (64U - kk2) * sizeof (uint8_t)); + memcpy(buf, k_1, kk2 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + uint32_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = h; + Lib_IntVector_Intrinsics_vec128 *r1 = h + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = h + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = h + 3U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4, iv5, iv6, iv7); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i0] = x;); + tmp[0U] = + (uint32_t)pv.digest_length + ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + tmp[1U] = pv.leaf_length; + tmp[2U] = (uint32_t)pv.node_offset; + tmp[3U] = + (uint32_t)(pv.node_offset >> 32U) + ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4_, iv5_, iv6_, iv7_); + uint8_t kk11 = i.key_length; + uint32_t ite; + if (kk11 != 0U) + { + ite = 64U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2s_Simd128_state_t + tmp8 = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + state[0U] = tmp8; +} + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params( + Hacl_Hash_Blake2s_Simd128_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +) +{ + index_of_state(s); + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_Simd128_reset_with_key(Hacl_Hash_Blake2s_Simd128_state_t *s, uint8_t *k) +{ + Hacl_Hash_Blake2b_index idx = index_of_state(s); + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = idx.digest_length, .key_length = idx.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = &p0, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_Simd128_reset(Hacl_Hash_Blake2s_Simd128_state_t *s) +{ + Hacl_Hash_Blake2s_Simd128_reset_with_key(s, NULL); +} + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2s_Simd128_update( + Hacl_Hash_Blake2s_Simd128_state_t *state, + uint8_t *chunk, + uint32_t chunk_len +) +{ + Hacl_Hash_Blake2s_Simd128_state_t s = *state; + uint64_t total_len = s.total_len; + if ((uint64_t)chunk_len > 0xffffffffffffffffULL - total_len) + { + return Hacl_Streaming_Types_MaximumLengthExceeded; + } + uint32_t sz; + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) + { + sz = 64U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)64U); + } + if (chunk_len <= 64U - sz) + { + Hacl_Hash_Blake2s_Simd128_state_t s1 = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state + = + ( + (Hacl_Hash_Blake2s_Simd128_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == 0U) + { + Hacl_Hash_Blake2s_Simd128_state_t s1 = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2s_Simd128_update_multi(64U, wv, hash, prevlen, buf, nb); + } + uint32_t ite; + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) + { + ite = 64U; + } + else + { + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); + } + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + uint32_t nb = data1_len / 64U; + Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2s_Simd128_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)chunk_len + } + ); + } + else + { + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_Blake2s_Simd128_state_t s1 = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) + { + sz10 = 64U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *state + = + ( + (Hacl_Hash_Blake2s_Simd128_state_t){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Hash_Blake2s_Simd128_state_t s10 = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2s_Simd128_update_multi(64U, wv, hash, prevlen, buf, nb); + } + uint32_t ite; + if + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) + { + ite = 64U; + } + else + { + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); + } + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + uint32_t nb = data1_len / 64U; + Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2s_Simd128_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(chunk_len - diff) + } + ); + } + return Hacl_Streaming_Types_Success; +} + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 128 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2S_128_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2s_Simd128_digest(Hacl_Hash_Blake2s_Simd128_state_t *s, uint8_t *dst) +{ + Hacl_Hash_Blake2s_Simd128_block_state_t block_state0 = (*s).block_state; + bool last_node0 = block_state0.thd; + uint8_t nn0 = block_state0.snd; + uint8_t kk0 = block_state0.fst; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk0, .digest_length = nn0, .last_node = last_node0 }; + Hacl_Hash_Blake2s_Simd128_state_t scrut = *s; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) + { + r = 64U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)64U); + } + uint8_t *buf_1 = buf_; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 wv0[4U] KRML_POST_ALIGN(16) = { 0U }; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b[4U] KRML_POST_ALIGN(16) = { 0U }; + Hacl_Hash_Blake2s_Simd128_block_state_t + tmp_block_state = + { + .fst = i1.key_length, + .snd = i1.digest_length, + .thd = i1.last_node, + .f3 = { .fst = wv0, .snd = b } + }; + Lib_IntVector_Intrinsics_vec128 *src_b = block_state.f3.snd; + Lib_IntVector_Intrinsics_vec128 *dst_b = tmp_block_state.f3.snd; + memcpy(dst_b, src_b, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + uint64_t prev_len = total_len - (uint64_t)r; + uint32_t ite; + if (r % 64U == 0U && r > 0U) + { + ite = 64U; + } + else + { + ite = r % 64U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ + acc0 = tmp_block_state.f3; + Lib_IntVector_Intrinsics_vec128 *wv1 = acc0.fst; + Lib_IntVector_Intrinsics_vec128 *hash0 = acc0.snd; + uint32_t nb = 0U; + Hacl_Hash_Blake2s_Simd128_update_multi(0U, wv1, hash0, prev_len, buf_multi, nb); + uint64_t prev_len_last = total_len - (uint64_t)r; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ + acc = tmp_block_state.f3; + bool last_node1 = tmp_block_state.thd; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + Hacl_Hash_Blake2s_Simd128_update_last(r, wv, hash, last_node1, prev_len_last, r, buf_last); + uint8_t nn1 = tmp_block_state.snd; + Hacl_Hash_Blake2s_Simd128_finish((uint32_t)nn1, dst, tmp_block_state.f3.snd); + Hacl_Hash_Blake2s_Simd128_block_state_t block_state1 = (*s).block_state; + bool last_node = block_state1.thd; + uint8_t nn = block_state1.snd; + uint8_t kk = block_state1.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }).digest_length; +} + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2s_Simd128_info(Hacl_Hash_Blake2s_Simd128_state_t *s) +{ + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }); +} + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2s_Simd128_free(Hacl_Hash_Blake2s_Simd128_state_t *state) +{ + Hacl_Hash_Blake2s_Simd128_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = scrut.block_state; + Lib_IntVector_Intrinsics_vec128 *b = block_state.f3.snd; + Lib_IntVector_Intrinsics_vec128 *wv = block_state.f3.fst; + KRML_ALIGNED_FREE(wv); + KRML_ALIGNED_FREE(b); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); +} + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_copy(Hacl_Hash_Blake2s_Simd128_state_t *state) +{ + Hacl_Hash_Blake2s_Simd128_state_t scrut = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + bool last_node = block_state0.thd; + uint8_t nn = block_state0.snd; + uint8_t kk1 = block_state0.fst; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + Lib_IntVector_Intrinsics_vec128 + *wv = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(wv, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Lib_IntVector_Intrinsics_vec128 + *b = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(b, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Hacl_Hash_Blake2s_Simd128_block_state_t + block_state = + { + .fst = i.key_length, + .snd = i.digest_length, + .thd = i.last_node, + .f3 = { .fst = wv, .snd = b } + }; + Lib_IntVector_Intrinsics_vec128 *src_b = block_state0.f3.snd; + Lib_IntVector_Intrinsics_vec128 *dst_b = block_state.f3.snd; + memcpy(dst_b, src_b, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Hacl_Hash_Blake2s_Simd128_state_t + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Hash_Blake2s_Simd128_state_t + *p = + (Hacl_Hash_Blake2s_Simd128_state_t *)KRML_HOST_MALLOC(sizeof ( + Hacl_Hash_Blake2s_Simd128_state_t + )); + p[0U] = s; + return p; +} + +/** +Write the BLAKE2s digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2s_Simd128_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +) +{ + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b[4U] KRML_POST_ALIGN(16) = { 0U }; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b1[4U] KRML_POST_ALIGN(16) = { 0U }; + Hacl_Hash_Blake2s_Simd128_init(b, key_len, output_len); + update(b1, b, key_len, key, input_len, input); + Hacl_Hash_Blake2s_Simd128_finish(output_len, output, b); + Lib_Memzero0_memzero(b1, 4U, Lib_IntVector_Intrinsics_vec128, void *); + Lib_Memzero0_memzero(b, 4U, Lib_IntVector_Intrinsics_vec128, void *); +} + +/** +Write the BLAKE2s digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +) +{ + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b[4U] KRML_POST_ALIGN(16) = { 0U }; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b1[4U] KRML_POST_ALIGN(16) = { 0U }; + uint32_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = b; + Lib_IntVector_Intrinsics_vec128 *r1 = b + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = b + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = b + 3U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4, iv5, iv6, iv7); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = params.salt + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = params.personal + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + tmp[0U] = + (uint32_t)params.digest_length + ^ + ((uint32_t)params.key_length + << 8U + ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + tmp[1U] = params.leaf_length; + tmp[2U] = (uint32_t)params.node_offset; + tmp[3U] = + (uint32_t)(params.node_offset >> 32U) + ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4_, iv5_, iv6_, iv7_); + update(b1, b, (uint32_t)params.key_length, key, input_len, input); + Hacl_Hash_Blake2s_Simd128_finish((uint32_t)params.digest_length, output, b); + Lib_Memzero0_memzero(b1, 4U, Lib_IntVector_Intrinsics_vec128, void *); + Lib_Memzero0_memzero(b, 4U, Lib_IntVector_Intrinsics_vec128, void *); +} + diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h new file mode 100644 index 00000000000000..cd1654c9726dc0 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h @@ -0,0 +1,230 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_Blake2s_Simd128_H +#define __Hacl_Hash_Blake2s_Simd128_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "python_hacl_namespaces.h" +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" +#include "Hacl_Hash_Blake2b.h" +#include "libintvector.h" + +#define HACL_HASH_BLAKE2S_SIMD128_BLOCK_BYTES (64U) + +#define HACL_HASH_BLAKE2S_SIMD128_OUT_BYTES (32U) + +#define HACL_HASH_BLAKE2S_SIMD128_KEY_BYTES (32U) + +#define HACL_HASH_BLAKE2S_SIMD128_SALT_BYTES (8U) + +#define HACL_HASH_BLAKE2S_SIMD128_PERSONAL_BYTES (8U) + +typedef struct K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128__s +{ + Lib_IntVector_Intrinsics_vec128 *fst; + Lib_IntVector_Intrinsics_vec128 *snd; +} +K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_; + +typedef struct Hacl_Hash_Blake2s_Simd128_block_state_t_s +{ + uint8_t fst; + uint8_t snd; + bool thd; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ f3; +} +Hacl_Hash_Blake2s_Simd128_block_state_t; + +typedef struct Hacl_Hash_Blake2s_Simd128_state_t_s +{ + Hacl_Hash_Blake2s_Simd128_block_state_t block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Hash_Blake2s_Simd128_state_t; + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 128 for S, 64 for B. +- The digest_length must not exceed 128 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +); + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 128 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_malloc_with_key0(uint8_t *k, uint8_t kk); + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2s_Simd128_state_t *Hacl_Hash_Blake2s_Simd128_malloc(void); + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params( + Hacl_Hash_Blake2s_Simd128_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +); + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_Simd128_reset_with_key(Hacl_Hash_Blake2s_Simd128_state_t *s, uint8_t *k); + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_Simd128_reset(Hacl_Hash_Blake2s_Simd128_state_t *s); + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2s_Simd128_update( + Hacl_Hash_Blake2s_Simd128_state_t *state, + uint8_t *chunk, + uint32_t chunk_len +); + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 128 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2S_128_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2s_Simd128_digest(Hacl_Hash_Blake2s_Simd128_state_t *s, uint8_t *dst); + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2s_Simd128_info(Hacl_Hash_Blake2s_Simd128_state_t *s); + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2s_Simd128_free(Hacl_Hash_Blake2s_Simd128_state_t *state); + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_copy(Hacl_Hash_Blake2s_Simd128_state_t *state); + +/** +Write the BLAKE2s digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2s_Simd128_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +); + +/** +Write the BLAKE2s digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_Blake2s_Simd128_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c new file mode 100644 index 00000000000000..951306db494833 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c @@ -0,0 +1,14 @@ +// This file isn't part of a standard HACL source tree. +// +// It is required for compatibility with universal2 macOS builds. The code in +// Hacl_Hash_Blake2s_Simd128.c will compile on macOS ARM64, but performance +// isn't great, so it's disabled. However, because universal2 builds are +// compiled in a single pass, autoconf detects that the required compiler +// features *are* available, and tries to include this file. +// +// To compensate for this, autoconf will include *this* file instead of +// Hacl_Hash_Blake2s_Simd128.c when compiling for universal. This allows the +// underlying source code of HACL to remain unmodified. +#if !(defined(__APPLE__) && defined(__arm64__)) +#include "Hacl_Hash_Blake2s_Simd128.c" +#endif diff --git a/Modules/_hacl/Hacl_Hash_MD5.c b/Modules/_hacl/Hacl_Hash_MD5.c index 222ac824f01961..ed294839ed8dc0 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.c +++ b/Modules/_hacl/Hacl_Hash_MD5.c @@ -25,37 +25,29 @@ #include "internal/Hacl_Hash_MD5.h" -static uint32_t -_h0[4U] = - { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; +static uint32_t _h0[4U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U }; static uint32_t _t[64U] = { - (uint32_t)0xd76aa478U, (uint32_t)0xe8c7b756U, (uint32_t)0x242070dbU, (uint32_t)0xc1bdceeeU, - (uint32_t)0xf57c0fafU, (uint32_t)0x4787c62aU, (uint32_t)0xa8304613U, (uint32_t)0xfd469501U, - (uint32_t)0x698098d8U, (uint32_t)0x8b44f7afU, (uint32_t)0xffff5bb1U, (uint32_t)0x895cd7beU, - (uint32_t)0x6b901122U, (uint32_t)0xfd987193U, (uint32_t)0xa679438eU, (uint32_t)0x49b40821U, - (uint32_t)0xf61e2562U, (uint32_t)0xc040b340U, (uint32_t)0x265e5a51U, (uint32_t)0xe9b6c7aaU, - (uint32_t)0xd62f105dU, (uint32_t)0x02441453U, (uint32_t)0xd8a1e681U, (uint32_t)0xe7d3fbc8U, - (uint32_t)0x21e1cde6U, (uint32_t)0xc33707d6U, (uint32_t)0xf4d50d87U, (uint32_t)0x455a14edU, - (uint32_t)0xa9e3e905U, (uint32_t)0xfcefa3f8U, (uint32_t)0x676f02d9U, (uint32_t)0x8d2a4c8aU, - (uint32_t)0xfffa3942U, (uint32_t)0x8771f681U, (uint32_t)0x6d9d6122U, (uint32_t)0xfde5380cU, - (uint32_t)0xa4beea44U, (uint32_t)0x4bdecfa9U, (uint32_t)0xf6bb4b60U, (uint32_t)0xbebfbc70U, - (uint32_t)0x289b7ec6U, (uint32_t)0xeaa127faU, (uint32_t)0xd4ef3085U, (uint32_t)0x4881d05U, - (uint32_t)0xd9d4d039U, (uint32_t)0xe6db99e5U, (uint32_t)0x1fa27cf8U, (uint32_t)0xc4ac5665U, - (uint32_t)0xf4292244U, (uint32_t)0x432aff97U, (uint32_t)0xab9423a7U, (uint32_t)0xfc93a039U, - (uint32_t)0x655b59c3U, (uint32_t)0x8f0ccc92U, (uint32_t)0xffeff47dU, (uint32_t)0x85845dd1U, - (uint32_t)0x6fa87e4fU, (uint32_t)0xfe2ce6e0U, (uint32_t)0xa3014314U, (uint32_t)0x4e0811a1U, - (uint32_t)0xf7537e82U, (uint32_t)0xbd3af235U, (uint32_t)0x2ad7d2bbU, (uint32_t)0xeb86d391U + 0xd76aa478U, 0xe8c7b756U, 0x242070dbU, 0xc1bdceeeU, 0xf57c0fafU, 0x4787c62aU, 0xa8304613U, + 0xfd469501U, 0x698098d8U, 0x8b44f7afU, 0xffff5bb1U, 0x895cd7beU, 0x6b901122U, 0xfd987193U, + 0xa679438eU, 0x49b40821U, 0xf61e2562U, 0xc040b340U, 0x265e5a51U, 0xe9b6c7aaU, 0xd62f105dU, + 0x02441453U, 0xd8a1e681U, 0xe7d3fbc8U, 0x21e1cde6U, 0xc33707d6U, 0xf4d50d87U, 0x455a14edU, + 0xa9e3e905U, 0xfcefa3f8U, 0x676f02d9U, 0x8d2a4c8aU, 0xfffa3942U, 0x8771f681U, 0x6d9d6122U, + 0xfde5380cU, 0xa4beea44U, 0x4bdecfa9U, 0xf6bb4b60U, 0xbebfbc70U, 0x289b7ec6U, 0xeaa127faU, + 0xd4ef3085U, 0x4881d05U, 0xd9d4d039U, 0xe6db99e5U, 0x1fa27cf8U, 0xc4ac5665U, 0xf4292244U, + 0x432aff97U, 0xab9423a7U, 0xfc93a039U, 0x655b59c3U, 0x8f0ccc92U, 0xffeff47dU, 0x85845dd1U, + 0x6fa87e4fU, 0xfe2ce6e0U, 0xa3014314U, 0x4e0811a1U, 0xf7537e82U, 0xbd3af235U, 0x2ad7d2bbU, + 0xeb86d391U }; -void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s) +void Hacl_Hash_MD5_init(uint32_t *s) { - KRML_MAYBE_FOR4(i, (uint32_t)0U, (uint32_t)4U, (uint32_t)1U, s[i] = _h0[i];); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, s[i] = _h0[i];); } -static void legacy_update(uint32_t *abcd, uint8_t *x) +static void update(uint32_t *abcd, uint8_t *x) { uint32_t aa = abcd[0U]; uint32_t bb = abcd[1U]; @@ -74,14 +66,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb0 + ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) - << (uint32_t)7U - | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> (uint32_t)25U); + << 7U + | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> 25U); abcd[0U] = v; uint32_t va0 = abcd[3U]; uint32_t vb1 = abcd[0U]; uint32_t vc1 = abcd[1U]; uint32_t vd1 = abcd[2U]; - uint8_t *b1 = x + (uint32_t)4U; + uint8_t *b1 = x + 4U; uint32_t u0 = load32_le(b1); uint32_t xk0 = u0; uint32_t ti1 = _t[1U]; @@ -90,14 +82,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb1 + ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) - << (uint32_t)12U - | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> (uint32_t)20U); + << 12U + | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> 20U); abcd[3U] = v0; uint32_t va1 = abcd[2U]; uint32_t vb2 = abcd[3U]; uint32_t vc2 = abcd[0U]; uint32_t vd2 = abcd[1U]; - uint8_t *b2 = x + (uint32_t)8U; + uint8_t *b2 = x + 8U; uint32_t u1 = load32_le(b2); uint32_t xk1 = u1; uint32_t ti2 = _t[2U]; @@ -106,14 +98,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb2 + ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) - << (uint32_t)17U - | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> (uint32_t)15U); + << 17U + | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> 15U); abcd[2U] = v1; uint32_t va2 = abcd[1U]; uint32_t vb3 = abcd[2U]; uint32_t vc3 = abcd[3U]; uint32_t vd3 = abcd[0U]; - uint8_t *b3 = x + (uint32_t)12U; + uint8_t *b3 = x + 12U; uint32_t u2 = load32_le(b3); uint32_t xk2 = u2; uint32_t ti3 = _t[3U]; @@ -122,14 +114,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb3 + ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) - << (uint32_t)22U - | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> (uint32_t)10U); + << 22U + | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> 10U); abcd[1U] = v2; uint32_t va3 = abcd[0U]; uint32_t vb4 = abcd[1U]; uint32_t vc4 = abcd[2U]; uint32_t vd4 = abcd[3U]; - uint8_t *b4 = x + (uint32_t)16U; + uint8_t *b4 = x + 16U; uint32_t u3 = load32_le(b4); uint32_t xk3 = u3; uint32_t ti4 = _t[4U]; @@ -138,14 +130,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb4 + ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) - << (uint32_t)7U - | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> (uint32_t)25U); + << 7U + | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> 25U); abcd[0U] = v3; uint32_t va4 = abcd[3U]; uint32_t vb5 = abcd[0U]; uint32_t vc5 = abcd[1U]; uint32_t vd5 = abcd[2U]; - uint8_t *b5 = x + (uint32_t)20U; + uint8_t *b5 = x + 20U; uint32_t u4 = load32_le(b5); uint32_t xk4 = u4; uint32_t ti5 = _t[5U]; @@ -154,14 +146,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb5 + ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) - << (uint32_t)12U - | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> (uint32_t)20U); + << 12U + | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> 20U); abcd[3U] = v4; uint32_t va5 = abcd[2U]; uint32_t vb6 = abcd[3U]; uint32_t vc6 = abcd[0U]; uint32_t vd6 = abcd[1U]; - uint8_t *b6 = x + (uint32_t)24U; + uint8_t *b6 = x + 24U; uint32_t u5 = load32_le(b6); uint32_t xk5 = u5; uint32_t ti6 = _t[6U]; @@ -170,14 +162,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb6 + ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) - << (uint32_t)17U - | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> (uint32_t)15U); + << 17U + | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> 15U); abcd[2U] = v5; uint32_t va6 = abcd[1U]; uint32_t vb7 = abcd[2U]; uint32_t vc7 = abcd[3U]; uint32_t vd7 = abcd[0U]; - uint8_t *b7 = x + (uint32_t)28U; + uint8_t *b7 = x + 28U; uint32_t u6 = load32_le(b7); uint32_t xk6 = u6; uint32_t ti7 = _t[7U]; @@ -186,14 +178,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb7 + ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) - << (uint32_t)22U - | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> (uint32_t)10U); + << 22U + | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> 10U); abcd[1U] = v6; uint32_t va7 = abcd[0U]; uint32_t vb8 = abcd[1U]; uint32_t vc8 = abcd[2U]; uint32_t vd8 = abcd[3U]; - uint8_t *b8 = x + (uint32_t)32U; + uint8_t *b8 = x + 32U; uint32_t u7 = load32_le(b8); uint32_t xk7 = u7; uint32_t ti8 = _t[8U]; @@ -202,14 +194,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb8 + ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) - << (uint32_t)7U - | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> (uint32_t)25U); + << 7U + | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> 25U); abcd[0U] = v7; uint32_t va8 = abcd[3U]; uint32_t vb9 = abcd[0U]; uint32_t vc9 = abcd[1U]; uint32_t vd9 = abcd[2U]; - uint8_t *b9 = x + (uint32_t)36U; + uint8_t *b9 = x + 36U; uint32_t u8 = load32_le(b9); uint32_t xk8 = u8; uint32_t ti9 = _t[9U]; @@ -218,14 +210,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb9 + ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) - << (uint32_t)12U - | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> (uint32_t)20U); + << 12U + | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> 20U); abcd[3U] = v8; uint32_t va9 = abcd[2U]; uint32_t vb10 = abcd[3U]; uint32_t vc10 = abcd[0U]; uint32_t vd10 = abcd[1U]; - uint8_t *b10 = x + (uint32_t)40U; + uint8_t *b10 = x + 40U; uint32_t u9 = load32_le(b10); uint32_t xk9 = u9; uint32_t ti10 = _t[10U]; @@ -234,14 +226,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb10 + ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) - << (uint32_t)17U - | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> (uint32_t)15U); + << 17U + | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> 15U); abcd[2U] = v9; uint32_t va10 = abcd[1U]; uint32_t vb11 = abcd[2U]; uint32_t vc11 = abcd[3U]; uint32_t vd11 = abcd[0U]; - uint8_t *b11 = x + (uint32_t)44U; + uint8_t *b11 = x + 44U; uint32_t u10 = load32_le(b11); uint32_t xk10 = u10; uint32_t ti11 = _t[11U]; @@ -250,14 +242,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb11 + ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) - << (uint32_t)22U - | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> (uint32_t)10U); + << 22U + | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> 10U); abcd[1U] = v10; uint32_t va11 = abcd[0U]; uint32_t vb12 = abcd[1U]; uint32_t vc12 = abcd[2U]; uint32_t vd12 = abcd[3U]; - uint8_t *b12 = x + (uint32_t)48U; + uint8_t *b12 = x + 48U; uint32_t u11 = load32_le(b12); uint32_t xk11 = u11; uint32_t ti12 = _t[12U]; @@ -266,14 +258,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb12 + ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) - << (uint32_t)7U - | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> (uint32_t)25U); + << 7U + | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> 25U); abcd[0U] = v11; uint32_t va12 = abcd[3U]; uint32_t vb13 = abcd[0U]; uint32_t vc13 = abcd[1U]; uint32_t vd13 = abcd[2U]; - uint8_t *b13 = x + (uint32_t)52U; + uint8_t *b13 = x + 52U; uint32_t u12 = load32_le(b13); uint32_t xk12 = u12; uint32_t ti13 = _t[13U]; @@ -282,14 +274,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb13 + ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) - << (uint32_t)12U - | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> (uint32_t)20U); + << 12U + | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> 20U); abcd[3U] = v12; uint32_t va13 = abcd[2U]; uint32_t vb14 = abcd[3U]; uint32_t vc14 = abcd[0U]; uint32_t vd14 = abcd[1U]; - uint8_t *b14 = x + (uint32_t)56U; + uint8_t *b14 = x + 56U; uint32_t u13 = load32_le(b14); uint32_t xk13 = u13; uint32_t ti14 = _t[14U]; @@ -298,14 +290,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb14 + ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) - << (uint32_t)17U - | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> (uint32_t)15U); + << 17U + | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> 15U); abcd[2U] = v13; uint32_t va14 = abcd[1U]; uint32_t vb15 = abcd[2U]; uint32_t vc15 = abcd[3U]; uint32_t vd15 = abcd[0U]; - uint8_t *b15 = x + (uint32_t)60U; + uint8_t *b15 = x + 60U; uint32_t u14 = load32_le(b15); uint32_t xk14 = u14; uint32_t ti15 = _t[15U]; @@ -314,14 +306,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb15 + ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) - << (uint32_t)22U - | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> (uint32_t)10U); + << 22U + | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> 10U); abcd[1U] = v14; uint32_t va15 = abcd[0U]; uint32_t vb16 = abcd[1U]; uint32_t vc16 = abcd[2U]; uint32_t vd16 = abcd[3U]; - uint8_t *b16 = x + (uint32_t)4U; + uint8_t *b16 = x + 4U; uint32_t u15 = load32_le(b16); uint32_t xk15 = u15; uint32_t ti16 = _t[16U]; @@ -330,14 +322,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb16 + ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) - << (uint32_t)5U - | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> (uint32_t)27U); + << 5U + | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> 27U); abcd[0U] = v15; uint32_t va16 = abcd[3U]; uint32_t vb17 = abcd[0U]; uint32_t vc17 = abcd[1U]; uint32_t vd17 = abcd[2U]; - uint8_t *b17 = x + (uint32_t)24U; + uint8_t *b17 = x + 24U; uint32_t u16 = load32_le(b17); uint32_t xk16 = u16; uint32_t ti17 = _t[17U]; @@ -346,14 +338,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb17 + ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) - << (uint32_t)9U - | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> (uint32_t)23U); + << 9U + | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> 23U); abcd[3U] = v16; uint32_t va17 = abcd[2U]; uint32_t vb18 = abcd[3U]; uint32_t vc18 = abcd[0U]; uint32_t vd18 = abcd[1U]; - uint8_t *b18 = x + (uint32_t)44U; + uint8_t *b18 = x + 44U; uint32_t u17 = load32_le(b18); uint32_t xk17 = u17; uint32_t ti18 = _t[18U]; @@ -362,8 +354,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb18 + ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) - << (uint32_t)14U - | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> (uint32_t)18U); + << 14U + | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> 18U); abcd[2U] = v17; uint32_t va18 = abcd[1U]; uint32_t vb19 = abcd[2U]; @@ -378,14 +370,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb19 + ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) - << (uint32_t)20U - | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> (uint32_t)12U); + << 20U + | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> 12U); abcd[1U] = v18; uint32_t va19 = abcd[0U]; uint32_t vb20 = abcd[1U]; uint32_t vc20 = abcd[2U]; uint32_t vd20 = abcd[3U]; - uint8_t *b20 = x + (uint32_t)20U; + uint8_t *b20 = x + 20U; uint32_t u19 = load32_le(b20); uint32_t xk19 = u19; uint32_t ti20 = _t[20U]; @@ -394,14 +386,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb20 + ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) - << (uint32_t)5U - | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> (uint32_t)27U); + << 5U + | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> 27U); abcd[0U] = v19; uint32_t va20 = abcd[3U]; uint32_t vb21 = abcd[0U]; uint32_t vc21 = abcd[1U]; uint32_t vd21 = abcd[2U]; - uint8_t *b21 = x + (uint32_t)40U; + uint8_t *b21 = x + 40U; uint32_t u20 = load32_le(b21); uint32_t xk20 = u20; uint32_t ti21 = _t[21U]; @@ -410,14 +402,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb21 + ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) - << (uint32_t)9U - | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> (uint32_t)23U); + << 9U + | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> 23U); abcd[3U] = v20; uint32_t va21 = abcd[2U]; uint32_t vb22 = abcd[3U]; uint32_t vc22 = abcd[0U]; uint32_t vd22 = abcd[1U]; - uint8_t *b22 = x + (uint32_t)60U; + uint8_t *b22 = x + 60U; uint32_t u21 = load32_le(b22); uint32_t xk21 = u21; uint32_t ti22 = _t[22U]; @@ -426,14 +418,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb22 + ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) - << (uint32_t)14U - | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> (uint32_t)18U); + << 14U + | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> 18U); abcd[2U] = v21; uint32_t va22 = abcd[1U]; uint32_t vb23 = abcd[2U]; uint32_t vc23 = abcd[3U]; uint32_t vd23 = abcd[0U]; - uint8_t *b23 = x + (uint32_t)16U; + uint8_t *b23 = x + 16U; uint32_t u22 = load32_le(b23); uint32_t xk22 = u22; uint32_t ti23 = _t[23U]; @@ -442,14 +434,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb23 + ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) - << (uint32_t)20U - | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> (uint32_t)12U); + << 20U + | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> 12U); abcd[1U] = v22; uint32_t va23 = abcd[0U]; uint32_t vb24 = abcd[1U]; uint32_t vc24 = abcd[2U]; uint32_t vd24 = abcd[3U]; - uint8_t *b24 = x + (uint32_t)36U; + uint8_t *b24 = x + 36U; uint32_t u23 = load32_le(b24); uint32_t xk23 = u23; uint32_t ti24 = _t[24U]; @@ -458,14 +450,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb24 + ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) - << (uint32_t)5U - | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> (uint32_t)27U); + << 5U + | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> 27U); abcd[0U] = v23; uint32_t va24 = abcd[3U]; uint32_t vb25 = abcd[0U]; uint32_t vc25 = abcd[1U]; uint32_t vd25 = abcd[2U]; - uint8_t *b25 = x + (uint32_t)56U; + uint8_t *b25 = x + 56U; uint32_t u24 = load32_le(b25); uint32_t xk24 = u24; uint32_t ti25 = _t[25U]; @@ -474,14 +466,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb25 + ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) - << (uint32_t)9U - | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> (uint32_t)23U); + << 9U + | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> 23U); abcd[3U] = v24; uint32_t va25 = abcd[2U]; uint32_t vb26 = abcd[3U]; uint32_t vc26 = abcd[0U]; uint32_t vd26 = abcd[1U]; - uint8_t *b26 = x + (uint32_t)12U; + uint8_t *b26 = x + 12U; uint32_t u25 = load32_le(b26); uint32_t xk25 = u25; uint32_t ti26 = _t[26U]; @@ -490,14 +482,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb26 + ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) - << (uint32_t)14U - | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> (uint32_t)18U); + << 14U + | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> 18U); abcd[2U] = v25; uint32_t va26 = abcd[1U]; uint32_t vb27 = abcd[2U]; uint32_t vc27 = abcd[3U]; uint32_t vd27 = abcd[0U]; - uint8_t *b27 = x + (uint32_t)32U; + uint8_t *b27 = x + 32U; uint32_t u26 = load32_le(b27); uint32_t xk26 = u26; uint32_t ti27 = _t[27U]; @@ -506,14 +498,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb27 + ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) - << (uint32_t)20U - | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> (uint32_t)12U); + << 20U + | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> 12U); abcd[1U] = v26; uint32_t va27 = abcd[0U]; uint32_t vb28 = abcd[1U]; uint32_t vc28 = abcd[2U]; uint32_t vd28 = abcd[3U]; - uint8_t *b28 = x + (uint32_t)52U; + uint8_t *b28 = x + 52U; uint32_t u27 = load32_le(b28); uint32_t xk27 = u27; uint32_t ti28 = _t[28U]; @@ -522,14 +514,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb28 + ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) - << (uint32_t)5U - | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> (uint32_t)27U); + << 5U + | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> 27U); abcd[0U] = v27; uint32_t va28 = abcd[3U]; uint32_t vb29 = abcd[0U]; uint32_t vc29 = abcd[1U]; uint32_t vd29 = abcd[2U]; - uint8_t *b29 = x + (uint32_t)8U; + uint8_t *b29 = x + 8U; uint32_t u28 = load32_le(b29); uint32_t xk28 = u28; uint32_t ti29 = _t[29U]; @@ -538,14 +530,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb29 + ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) - << (uint32_t)9U - | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> (uint32_t)23U); + << 9U + | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> 23U); abcd[3U] = v28; uint32_t va29 = abcd[2U]; uint32_t vb30 = abcd[3U]; uint32_t vc30 = abcd[0U]; uint32_t vd30 = abcd[1U]; - uint8_t *b30 = x + (uint32_t)28U; + uint8_t *b30 = x + 28U; uint32_t u29 = load32_le(b30); uint32_t xk29 = u29; uint32_t ti30 = _t[30U]; @@ -554,14 +546,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb30 + ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) - << (uint32_t)14U - | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> (uint32_t)18U); + << 14U + | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> 18U); abcd[2U] = v29; uint32_t va30 = abcd[1U]; uint32_t vb31 = abcd[2U]; uint32_t vc31 = abcd[3U]; uint32_t vd31 = abcd[0U]; - uint8_t *b31 = x + (uint32_t)48U; + uint8_t *b31 = x + 48U; uint32_t u30 = load32_le(b31); uint32_t xk30 = u30; uint32_t ti31 = _t[31U]; @@ -570,14 +562,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb31 + ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) - << (uint32_t)20U - | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> (uint32_t)12U); + << 20U + | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> 12U); abcd[1U] = v30; uint32_t va31 = abcd[0U]; uint32_t vb32 = abcd[1U]; uint32_t vc32 = abcd[2U]; uint32_t vd32 = abcd[3U]; - uint8_t *b32 = x + (uint32_t)20U; + uint8_t *b32 = x + 20U; uint32_t u31 = load32_le(b32); uint32_t xk31 = u31; uint32_t ti32 = _t[32U]; @@ -586,14 +578,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb32 + ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) - << (uint32_t)4U - | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> (uint32_t)28U); + << 4U + | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> 28U); abcd[0U] = v31; uint32_t va32 = abcd[3U]; uint32_t vb33 = abcd[0U]; uint32_t vc33 = abcd[1U]; uint32_t vd33 = abcd[2U]; - uint8_t *b33 = x + (uint32_t)32U; + uint8_t *b33 = x + 32U; uint32_t u32 = load32_le(b33); uint32_t xk32 = u32; uint32_t ti33 = _t[33U]; @@ -602,14 +594,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb33 + ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) - << (uint32_t)11U - | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> (uint32_t)21U); + << 11U + | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> 21U); abcd[3U] = v32; uint32_t va33 = abcd[2U]; uint32_t vb34 = abcd[3U]; uint32_t vc34 = abcd[0U]; uint32_t vd34 = abcd[1U]; - uint8_t *b34 = x + (uint32_t)44U; + uint8_t *b34 = x + 44U; uint32_t u33 = load32_le(b34); uint32_t xk33 = u33; uint32_t ti34 = _t[34U]; @@ -618,14 +610,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb34 + ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) - << (uint32_t)16U - | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> (uint32_t)16U); + << 16U + | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> 16U); abcd[2U] = v33; uint32_t va34 = abcd[1U]; uint32_t vb35 = abcd[2U]; uint32_t vc35 = abcd[3U]; uint32_t vd35 = abcd[0U]; - uint8_t *b35 = x + (uint32_t)56U; + uint8_t *b35 = x + 56U; uint32_t u34 = load32_le(b35); uint32_t xk34 = u34; uint32_t ti35 = _t[35U]; @@ -634,14 +626,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb35 + ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) - << (uint32_t)23U - | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> (uint32_t)9U); + << 23U + | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> 9U); abcd[1U] = v34; uint32_t va35 = abcd[0U]; uint32_t vb36 = abcd[1U]; uint32_t vc36 = abcd[2U]; uint32_t vd36 = abcd[3U]; - uint8_t *b36 = x + (uint32_t)4U; + uint8_t *b36 = x + 4U; uint32_t u35 = load32_le(b36); uint32_t xk35 = u35; uint32_t ti36 = _t[36U]; @@ -650,14 +642,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb36 + ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) - << (uint32_t)4U - | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> (uint32_t)28U); + << 4U + | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> 28U); abcd[0U] = v35; uint32_t va36 = abcd[3U]; uint32_t vb37 = abcd[0U]; uint32_t vc37 = abcd[1U]; uint32_t vd37 = abcd[2U]; - uint8_t *b37 = x + (uint32_t)16U; + uint8_t *b37 = x + 16U; uint32_t u36 = load32_le(b37); uint32_t xk36 = u36; uint32_t ti37 = _t[37U]; @@ -666,14 +658,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb37 + ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) - << (uint32_t)11U - | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> (uint32_t)21U); + << 11U + | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> 21U); abcd[3U] = v36; uint32_t va37 = abcd[2U]; uint32_t vb38 = abcd[3U]; uint32_t vc38 = abcd[0U]; uint32_t vd38 = abcd[1U]; - uint8_t *b38 = x + (uint32_t)28U; + uint8_t *b38 = x + 28U; uint32_t u37 = load32_le(b38); uint32_t xk37 = u37; uint32_t ti38 = _t[38U]; @@ -682,14 +674,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb38 + ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) - << (uint32_t)16U - | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> (uint32_t)16U); + << 16U + | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> 16U); abcd[2U] = v37; uint32_t va38 = abcd[1U]; uint32_t vb39 = abcd[2U]; uint32_t vc39 = abcd[3U]; uint32_t vd39 = abcd[0U]; - uint8_t *b39 = x + (uint32_t)40U; + uint8_t *b39 = x + 40U; uint32_t u38 = load32_le(b39); uint32_t xk38 = u38; uint32_t ti39 = _t[39U]; @@ -698,14 +690,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb39 + ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) - << (uint32_t)23U - | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> (uint32_t)9U); + << 23U + | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> 9U); abcd[1U] = v38; uint32_t va39 = abcd[0U]; uint32_t vb40 = abcd[1U]; uint32_t vc40 = abcd[2U]; uint32_t vd40 = abcd[3U]; - uint8_t *b40 = x + (uint32_t)52U; + uint8_t *b40 = x + 52U; uint32_t u39 = load32_le(b40); uint32_t xk39 = u39; uint32_t ti40 = _t[40U]; @@ -714,8 +706,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb40 + ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) - << (uint32_t)4U - | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> (uint32_t)28U); + << 4U + | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> 28U); abcd[0U] = v39; uint32_t va40 = abcd[3U]; uint32_t vb41 = abcd[0U]; @@ -730,14 +722,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb41 + ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) - << (uint32_t)11U - | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> (uint32_t)21U); + << 11U + | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> 21U); abcd[3U] = v40; uint32_t va41 = abcd[2U]; uint32_t vb42 = abcd[3U]; uint32_t vc42 = abcd[0U]; uint32_t vd42 = abcd[1U]; - uint8_t *b42 = x + (uint32_t)12U; + uint8_t *b42 = x + 12U; uint32_t u41 = load32_le(b42); uint32_t xk41 = u41; uint32_t ti42 = _t[42U]; @@ -746,14 +738,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb42 + ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) - << (uint32_t)16U - | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> (uint32_t)16U); + << 16U + | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> 16U); abcd[2U] = v41; uint32_t va42 = abcd[1U]; uint32_t vb43 = abcd[2U]; uint32_t vc43 = abcd[3U]; uint32_t vd43 = abcd[0U]; - uint8_t *b43 = x + (uint32_t)24U; + uint8_t *b43 = x + 24U; uint32_t u42 = load32_le(b43); uint32_t xk42 = u42; uint32_t ti43 = _t[43U]; @@ -762,14 +754,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb43 + ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) - << (uint32_t)23U - | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> (uint32_t)9U); + << 23U + | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> 9U); abcd[1U] = v42; uint32_t va43 = abcd[0U]; uint32_t vb44 = abcd[1U]; uint32_t vc44 = abcd[2U]; uint32_t vd44 = abcd[3U]; - uint8_t *b44 = x + (uint32_t)36U; + uint8_t *b44 = x + 36U; uint32_t u43 = load32_le(b44); uint32_t xk43 = u43; uint32_t ti44 = _t[44U]; @@ -778,14 +770,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb44 + ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) - << (uint32_t)4U - | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> (uint32_t)28U); + << 4U + | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> 28U); abcd[0U] = v43; uint32_t va44 = abcd[3U]; uint32_t vb45 = abcd[0U]; uint32_t vc45 = abcd[1U]; uint32_t vd45 = abcd[2U]; - uint8_t *b45 = x + (uint32_t)48U; + uint8_t *b45 = x + 48U; uint32_t u44 = load32_le(b45); uint32_t xk44 = u44; uint32_t ti45 = _t[45U]; @@ -794,14 +786,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb45 + ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) - << (uint32_t)11U - | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> (uint32_t)21U); + << 11U + | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> 21U); abcd[3U] = v44; uint32_t va45 = abcd[2U]; uint32_t vb46 = abcd[3U]; uint32_t vc46 = abcd[0U]; uint32_t vd46 = abcd[1U]; - uint8_t *b46 = x + (uint32_t)60U; + uint8_t *b46 = x + 60U; uint32_t u45 = load32_le(b46); uint32_t xk45 = u45; uint32_t ti46 = _t[46U]; @@ -810,14 +802,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb46 + ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) - << (uint32_t)16U - | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> (uint32_t)16U); + << 16U + | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> 16U); abcd[2U] = v45; uint32_t va46 = abcd[1U]; uint32_t vb47 = abcd[2U]; uint32_t vc47 = abcd[3U]; uint32_t vd47 = abcd[0U]; - uint8_t *b47 = x + (uint32_t)8U; + uint8_t *b47 = x + 8U; uint32_t u46 = load32_le(b47); uint32_t xk46 = u46; uint32_t ti47 = _t[47U]; @@ -826,8 +818,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb47 + ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) - << (uint32_t)23U - | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> (uint32_t)9U); + << 23U + | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> 9U); abcd[1U] = v46; uint32_t va47 = abcd[0U]; uint32_t vb48 = abcd[1U]; @@ -842,14 +834,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb48 + ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) - << (uint32_t)6U - | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> (uint32_t)26U); + << 6U + | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> 26U); abcd[0U] = v47; uint32_t va48 = abcd[3U]; uint32_t vb49 = abcd[0U]; uint32_t vc49 = abcd[1U]; uint32_t vd49 = abcd[2U]; - uint8_t *b49 = x + (uint32_t)28U; + uint8_t *b49 = x + 28U; uint32_t u48 = load32_le(b49); uint32_t xk48 = u48; uint32_t ti49 = _t[49U]; @@ -858,14 +850,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb49 + ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) - << (uint32_t)10U - | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> (uint32_t)22U); + << 10U + | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> 22U); abcd[3U] = v48; uint32_t va49 = abcd[2U]; uint32_t vb50 = abcd[3U]; uint32_t vc50 = abcd[0U]; uint32_t vd50 = abcd[1U]; - uint8_t *b50 = x + (uint32_t)56U; + uint8_t *b50 = x + 56U; uint32_t u49 = load32_le(b50); uint32_t xk49 = u49; uint32_t ti50 = _t[50U]; @@ -874,14 +866,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb50 + ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) - << (uint32_t)15U - | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> (uint32_t)17U); + << 15U + | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> 17U); abcd[2U] = v49; uint32_t va50 = abcd[1U]; uint32_t vb51 = abcd[2U]; uint32_t vc51 = abcd[3U]; uint32_t vd51 = abcd[0U]; - uint8_t *b51 = x + (uint32_t)20U; + uint8_t *b51 = x + 20U; uint32_t u50 = load32_le(b51); uint32_t xk50 = u50; uint32_t ti51 = _t[51U]; @@ -890,14 +882,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb51 + ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) - << (uint32_t)21U - | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> (uint32_t)11U); + << 21U + | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> 11U); abcd[1U] = v50; uint32_t va51 = abcd[0U]; uint32_t vb52 = abcd[1U]; uint32_t vc52 = abcd[2U]; uint32_t vd52 = abcd[3U]; - uint8_t *b52 = x + (uint32_t)48U; + uint8_t *b52 = x + 48U; uint32_t u51 = load32_le(b52); uint32_t xk51 = u51; uint32_t ti52 = _t[52U]; @@ -906,14 +898,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb52 + ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) - << (uint32_t)6U - | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> (uint32_t)26U); + << 6U + | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> 26U); abcd[0U] = v51; uint32_t va52 = abcd[3U]; uint32_t vb53 = abcd[0U]; uint32_t vc53 = abcd[1U]; uint32_t vd53 = abcd[2U]; - uint8_t *b53 = x + (uint32_t)12U; + uint8_t *b53 = x + 12U; uint32_t u52 = load32_le(b53); uint32_t xk52 = u52; uint32_t ti53 = _t[53U]; @@ -922,14 +914,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb53 + ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) - << (uint32_t)10U - | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> (uint32_t)22U); + << 10U + | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> 22U); abcd[3U] = v52; uint32_t va53 = abcd[2U]; uint32_t vb54 = abcd[3U]; uint32_t vc54 = abcd[0U]; uint32_t vd54 = abcd[1U]; - uint8_t *b54 = x + (uint32_t)40U; + uint8_t *b54 = x + 40U; uint32_t u53 = load32_le(b54); uint32_t xk53 = u53; uint32_t ti54 = _t[54U]; @@ -938,14 +930,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb54 + ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) - << (uint32_t)15U - | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> (uint32_t)17U); + << 15U + | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> 17U); abcd[2U] = v53; uint32_t va54 = abcd[1U]; uint32_t vb55 = abcd[2U]; uint32_t vc55 = abcd[3U]; uint32_t vd55 = abcd[0U]; - uint8_t *b55 = x + (uint32_t)4U; + uint8_t *b55 = x + 4U; uint32_t u54 = load32_le(b55); uint32_t xk54 = u54; uint32_t ti55 = _t[55U]; @@ -954,14 +946,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb55 + ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) - << (uint32_t)21U - | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> (uint32_t)11U); + << 21U + | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> 11U); abcd[1U] = v54; uint32_t va55 = abcd[0U]; uint32_t vb56 = abcd[1U]; uint32_t vc56 = abcd[2U]; uint32_t vd56 = abcd[3U]; - uint8_t *b56 = x + (uint32_t)32U; + uint8_t *b56 = x + 32U; uint32_t u55 = load32_le(b56); uint32_t xk55 = u55; uint32_t ti56 = _t[56U]; @@ -970,14 +962,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb56 + ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) - << (uint32_t)6U - | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> (uint32_t)26U); + << 6U + | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> 26U); abcd[0U] = v55; uint32_t va56 = abcd[3U]; uint32_t vb57 = abcd[0U]; uint32_t vc57 = abcd[1U]; uint32_t vd57 = abcd[2U]; - uint8_t *b57 = x + (uint32_t)60U; + uint8_t *b57 = x + 60U; uint32_t u56 = load32_le(b57); uint32_t xk56 = u56; uint32_t ti57 = _t[57U]; @@ -986,14 +978,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb57 + ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) - << (uint32_t)10U - | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> (uint32_t)22U); + << 10U + | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> 22U); abcd[3U] = v56; uint32_t va57 = abcd[2U]; uint32_t vb58 = abcd[3U]; uint32_t vc58 = abcd[0U]; uint32_t vd58 = abcd[1U]; - uint8_t *b58 = x + (uint32_t)24U; + uint8_t *b58 = x + 24U; uint32_t u57 = load32_le(b58); uint32_t xk57 = u57; uint32_t ti58 = _t[58U]; @@ -1002,14 +994,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb58 + ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) - << (uint32_t)15U - | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> (uint32_t)17U); + << 15U + | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> 17U); abcd[2U] = v57; uint32_t va58 = abcd[1U]; uint32_t vb59 = abcd[2U]; uint32_t vc59 = abcd[3U]; uint32_t vd59 = abcd[0U]; - uint8_t *b59 = x + (uint32_t)52U; + uint8_t *b59 = x + 52U; uint32_t u58 = load32_le(b59); uint32_t xk58 = u58; uint32_t ti59 = _t[59U]; @@ -1018,14 +1010,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb59 + ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) - << (uint32_t)21U - | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> (uint32_t)11U); + << 21U + | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> 11U); abcd[1U] = v58; uint32_t va59 = abcd[0U]; uint32_t vb60 = abcd[1U]; uint32_t vc60 = abcd[2U]; uint32_t vd60 = abcd[3U]; - uint8_t *b60 = x + (uint32_t)16U; + uint8_t *b60 = x + 16U; uint32_t u59 = load32_le(b60); uint32_t xk59 = u59; uint32_t ti60 = _t[60U]; @@ -1034,14 +1026,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb60 + ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) - << (uint32_t)6U - | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> (uint32_t)26U); + << 6U + | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> 26U); abcd[0U] = v59; uint32_t va60 = abcd[3U]; uint32_t vb61 = abcd[0U]; uint32_t vc61 = abcd[1U]; uint32_t vd61 = abcd[2U]; - uint8_t *b61 = x + (uint32_t)44U; + uint8_t *b61 = x + 44U; uint32_t u60 = load32_le(b61); uint32_t xk60 = u60; uint32_t ti61 = _t[61U]; @@ -1050,14 +1042,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb61 + ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) - << (uint32_t)10U - | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> (uint32_t)22U); + << 10U + | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> 22U); abcd[3U] = v60; uint32_t va61 = abcd[2U]; uint32_t vb62 = abcd[3U]; uint32_t vc62 = abcd[0U]; uint32_t vd62 = abcd[1U]; - uint8_t *b62 = x + (uint32_t)8U; + uint8_t *b62 = x + 8U; uint32_t u61 = load32_le(b62); uint32_t xk61 = u61; uint32_t ti62 = _t[62U]; @@ -1066,14 +1058,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb62 + ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) - << (uint32_t)15U - | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> (uint32_t)17U); + << 15U + | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> 17U); abcd[2U] = v61; uint32_t va62 = abcd[1U]; uint32_t vb = abcd[2U]; uint32_t vc = abcd[3U]; uint32_t vd = abcd[0U]; - uint8_t *b63 = x + (uint32_t)36U; + uint8_t *b63 = x + 36U; uint32_t u62 = load32_le(b63); uint32_t xk62 = u62; uint32_t ti = _t[63U]; @@ -1082,8 +1074,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb + ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) - << (uint32_t)21U - | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> (uint32_t)11U); + << 21U + | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> 11U); abcd[1U] = v62; uint32_t a = abcd[0U]; uint32_t b = abcd[1U]; @@ -1095,98 +1087,69 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) abcd[3U] = d + dd; } -static void legacy_pad(uint64_t len, uint8_t *dst) +static void pad(uint64_t len, uint8_t *dst) { uint8_t *dst1 = dst; - dst1[0U] = (uint8_t)0x80U; - uint8_t *dst2 = dst + (uint32_t)1U; - for - (uint32_t - i = (uint32_t)0U; - i - < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; - i++) + dst1[0U] = 0x80U; + uint8_t *dst2 = dst + 1U; + for (uint32_t i = 0U; i < (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; i++) { - dst2[i] = (uint8_t)0U; + dst2[i] = 0U; } - uint8_t - *dst3 = - dst - + - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U; - store64_le(dst3, len << (uint32_t)3U); + uint8_t *dst3 = dst + 1U + (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; + store64_le(dst3, len << 3U); } -void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst) +void Hacl_Hash_MD5_finish(uint32_t *s, uint8_t *dst) { - KRML_MAYBE_FOR4(i, - (uint32_t)0U, - (uint32_t)4U, - (uint32_t)1U, - store32_le(dst + i * (uint32_t)4U, s[i]);); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store32_le(dst + i * 4U, s[i]);); } -void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +void Hacl_Hash_MD5_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) { - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + for (uint32_t i = 0U; i < n_blocks; i++) { - uint32_t sz = (uint32_t)64U; + uint32_t sz = 64U; uint8_t *block = blocks + sz * i; - legacy_update(s, block); + update(s, block); } } void -Hacl_Hash_MD5_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -) +Hacl_Hash_MD5_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len) { - uint32_t blocks_n = input_len / (uint32_t)64U; - uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint32_t blocks_n = input_len / 64U; + uint32_t blocks_len = blocks_n * 64U; uint8_t *blocks = input; uint32_t rest_len = input_len - blocks_len; uint8_t *rest = input + blocks_len; - Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_MD5_update_multi(s, blocks, blocks_n); uint64_t total_input_len = prev_len + (uint64_t)input_len; - uint32_t - pad_len = - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U - + (uint32_t)8U; + uint32_t pad_len = 1U + (128U - (9U + (uint32_t)(total_input_len % (uint64_t)64U))) % 64U + 8U; uint32_t tmp_len = rest_len + pad_len; uint8_t tmp_twoblocks[128U] = { 0U }; uint8_t *tmp = tmp_twoblocks; uint8_t *tmp_rest = tmp; uint8_t *tmp_pad = tmp + rest_len; memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); - legacy_pad(total_input_len, tmp_pad); - Hacl_Hash_MD5_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); + pad(total_input_len, tmp_pad); + Hacl_Hash_MD5_update_multi(s, tmp, tmp_len / 64U); } -void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_MD5_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len) { - uint32_t - s[4U] = - { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; - uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t s[4U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U }; + uint32_t blocks_n0 = input_len / 64U; uint32_t blocks_n1; - if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + if (input_len % 64U == 0U && blocks_n0 > 0U) { - blocks_n1 = blocks_n0 - (uint32_t)1U; + blocks_n1 = blocks_n0 - 1U; } else { blocks_n1 = blocks_n0; } - uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint32_t blocks_len0 = blocks_n1 * 64U; uint8_t *blocks0 = input; uint32_t rest_len0 = input_len - blocks_len0; uint8_t *rest0 = input + blocks_len0; @@ -1195,75 +1158,75 @@ void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) uint8_t *blocks = blocks0; uint32_t rest_len = rest_len0; uint8_t *rest = rest0; - Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); - Hacl_Hash_MD5_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); - Hacl_Hash_Core_MD5_legacy_finish(s, dst); + Hacl_Hash_MD5_update_multi(s, blocks, blocks_n); + Hacl_Hash_MD5_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_MD5_finish(s, output); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_malloc(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(4U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_Hash_Core_MD5_legacy_init(block_state); + Hacl_Hash_MD5_init(block_state); return p; } -void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_MD5_reset(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_Hash_Core_MD5_legacy_init(block_state); + Hacl_Hash_MD5_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_32 s = *p; + Hacl_Streaming_MD_state_32 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + if ((uint64_t)chunk_len > 2305843009213693951ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)64U; + sz = 64U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + sz = (uint32_t)(total_len % (uint64_t)64U); } - if (len <= (uint32_t)64U - sz) + if (chunk_len <= 64U - sz) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_32){ @@ -1273,74 +1236,74 @@ Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, u } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_MD5_update_multi(block_state1, buf, 1U); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); } - uint32_t n_blocks = (len - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_Hash_MD5_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)64U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)64U; + sz10 = 64U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_32){ @@ -1349,114 +1312,109 @@ Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, u .total_len = total_len2 } ); - Hacl_Streaming_MD_state_32 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *state; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_MD5_update_multi(block_state1, buf, 1U); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)64U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_Hash_MD5_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } return Hacl_Streaming_Types_Success; } -void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_MD5_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[4U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)4U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 4U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_Hash_MD5_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + Hacl_Hash_MD5_update_multi(tmp_block_state, buf_multi, 0U); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_Hash_MD5_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); - Hacl_Hash_Core_MD5_legacy_finish(tmp_block_state, dst); + Hacl_Hash_MD5_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_MD5_finish(tmp_block_state, output); } -void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_MD5_free(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_copy(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); - memcpy(block_state, block_state0, (uint32_t)4U * sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(4U, sizeof (uint32_t)); + memcpy(block_state, block_state0, 4U * sizeof (uint32_t)); Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_32 @@ -1465,8 +1423,8 @@ Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_sta return p; } -void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_MD5_hash(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Hash_MD5_legacy_hash(input, input_len, dst); + Hacl_Hash_MD5_hash_oneshot(output, input, input_len); } diff --git a/Modules/_hacl/Hacl_Hash_MD5.h b/Modules/_hacl/Hacl_Hash_MD5.h index 13c19fd40f4d12..f69d6e5a81d63a 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.h +++ b/Modules/_hacl/Hacl_Hash_MD5.h @@ -31,31 +31,32 @@ extern "C" { #endif #include +#include "python_hacl_namespaces.h" #include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" #include "Hacl_Streaming_Types.h" -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_MD5_state; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_MD5_state_t; -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_malloc(void); -void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_MD5_reset(Hacl_Streaming_MD_state_32 *state); /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); +Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len); -void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_MD5_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output); -void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_MD5_free(Hacl_Streaming_MD_state_32 *state); -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_copy(Hacl_Streaming_MD_state_32 *state); -void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_MD5_hash(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/Hacl_Hash_SHA1.c b/Modules/_hacl/Hacl_Hash_SHA1.c index 5ecb3c0b3a56e0..1a8b09b1711894 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.c +++ b/Modules/_hacl/Hacl_Hash_SHA1.c @@ -25,19 +25,14 @@ #include "internal/Hacl_Hash_SHA1.h" -static uint32_t -_h0[5U] = - { - (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, - (uint32_t)0xc3d2e1f0U - }; +static uint32_t _h0[5U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U, 0xc3d2e1f0U }; -void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s) +void Hacl_Hash_SHA1_init(uint32_t *s) { - KRML_MAYBE_FOR5(i, (uint32_t)0U, (uint32_t)5U, (uint32_t)1U, s[i] = _h0[i];); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i] = _h0[i];); } -static void legacy_update(uint32_t *h, uint8_t *l) +static void update(uint32_t *h, uint8_t *l) { uint32_t ha = h[0U]; uint32_t hb = h[1U]; @@ -45,29 +40,26 @@ static void legacy_update(uint32_t *h, uint8_t *l) uint32_t hd = h[3U]; uint32_t he = h[4U]; uint32_t _w[80U] = { 0U }; - for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + for (uint32_t i = 0U; i < 80U; i++) { uint32_t v; - if (i < (uint32_t)16U) + if (i < 16U) { - uint8_t *b = l + i * (uint32_t)4U; + uint8_t *b = l + i * 4U; uint32_t u = load32_be(b); v = u; } else { - uint32_t wmit3 = _w[i - (uint32_t)3U]; - uint32_t wmit8 = _w[i - (uint32_t)8U]; - uint32_t wmit14 = _w[i - (uint32_t)14U]; - uint32_t wmit16 = _w[i - (uint32_t)16U]; - v = - (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) - << (uint32_t)1U - | (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) >> (uint32_t)31U; + uint32_t wmit3 = _w[i - 3U]; + uint32_t wmit8 = _w[i - 8U]; + uint32_t wmit14 = _w[i - 14U]; + uint32_t wmit16 = _w[i - 16U]; + v = (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) << 1U | (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) >> 31U; } _w[i] = v; } - for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + for (uint32_t i = 0U; i < 80U; i++) { uint32_t _a = h[0U]; uint32_t _b = h[1U]; @@ -76,11 +68,11 @@ static void legacy_update(uint32_t *h, uint8_t *l) uint32_t _e = h[4U]; uint32_t wmit = _w[i]; uint32_t ite0; - if (i < (uint32_t)20U) + if (i < 20U) { ite0 = (_b & _c) ^ (~_b & _d); } - else if ((uint32_t)39U < i && i < (uint32_t)60U) + else if (39U < i && i < 60U) { ite0 = (_b & _c) ^ ((_b & _d) ^ (_c & _d)); } @@ -89,32 +81,32 @@ static void legacy_update(uint32_t *h, uint8_t *l) ite0 = _b ^ (_c ^ _d); } uint32_t ite; - if (i < (uint32_t)20U) + if (i < 20U) { - ite = (uint32_t)0x5a827999U; + ite = 0x5a827999U; } - else if (i < (uint32_t)40U) + else if (i < 40U) { - ite = (uint32_t)0x6ed9eba1U; + ite = 0x6ed9eba1U; } - else if (i < (uint32_t)60U) + else if (i < 60U) { - ite = (uint32_t)0x8f1bbcdcU; + ite = 0x8f1bbcdcU; } else { - ite = (uint32_t)0xca62c1d6U; + ite = 0xca62c1d6U; } - uint32_t _T = (_a << (uint32_t)5U | _a >> (uint32_t)27U) + ite0 + _e + ite + wmit; + uint32_t _T = (_a << 5U | _a >> 27U) + ite0 + _e + ite + wmit; h[0U] = _T; h[1U] = _a; - h[2U] = _b << (uint32_t)30U | _b >> (uint32_t)2U; + h[2U] = _b << 30U | _b >> 2U; h[3U] = _c; h[4U] = _d; } - for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + for (uint32_t i = 0U; i < 80U; i++) { - _w[i] = (uint32_t)0U; + _w[i] = 0U; } uint32_t sta = h[0U]; uint32_t stb = h[1U]; @@ -128,101 +120,69 @@ static void legacy_update(uint32_t *h, uint8_t *l) h[4U] = ste + he; } -static void legacy_pad(uint64_t len, uint8_t *dst) +static void pad(uint64_t len, uint8_t *dst) { uint8_t *dst1 = dst; - dst1[0U] = (uint8_t)0x80U; - uint8_t *dst2 = dst + (uint32_t)1U; - for - (uint32_t - i = (uint32_t)0U; - i - < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; - i++) + dst1[0U] = 0x80U; + uint8_t *dst2 = dst + 1U; + for (uint32_t i = 0U; i < (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; i++) { - dst2[i] = (uint8_t)0U; + dst2[i] = 0U; } - uint8_t - *dst3 = - dst - + - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U; - store64_be(dst3, len << (uint32_t)3U); + uint8_t *dst3 = dst + 1U + (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; + store64_be(dst3, len << 3U); } -void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst) +void Hacl_Hash_SHA1_finish(uint32_t *s, uint8_t *dst) { - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - store32_be(dst + i * (uint32_t)4U, s[i]);); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, store32_be(dst + i * 4U, s[i]);); } -void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +void Hacl_Hash_SHA1_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) { - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + for (uint32_t i = 0U; i < n_blocks; i++) { - uint32_t sz = (uint32_t)64U; + uint32_t sz = 64U; uint8_t *block = blocks + sz * i; - legacy_update(s, block); + update(s, block); } } void -Hacl_Hash_SHA1_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -) +Hacl_Hash_SHA1_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len) { - uint32_t blocks_n = input_len / (uint32_t)64U; - uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint32_t blocks_n = input_len / 64U; + uint32_t blocks_len = blocks_n * 64U; uint8_t *blocks = input; uint32_t rest_len = input_len - blocks_len; uint8_t *rest = input + blocks_len; - Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_SHA1_update_multi(s, blocks, blocks_n); uint64_t total_input_len = prev_len + (uint64_t)input_len; - uint32_t - pad_len = - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U - + (uint32_t)8U; + uint32_t pad_len = 1U + (128U - (9U + (uint32_t)(total_input_len % (uint64_t)64U))) % 64U + 8U; uint32_t tmp_len = rest_len + pad_len; uint8_t tmp_twoblocks[128U] = { 0U }; uint8_t *tmp = tmp_twoblocks; uint8_t *tmp_rest = tmp; uint8_t *tmp_pad = tmp + rest_len; memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); - legacy_pad(total_input_len, tmp_pad); - Hacl_Hash_SHA1_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); + pad(total_input_len, tmp_pad); + Hacl_Hash_SHA1_update_multi(s, tmp, tmp_len / 64U); } -void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA1_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len) { - uint32_t - s[5U] = - { - (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, - (uint32_t)0xc3d2e1f0U - }; - uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t s[5U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U, 0xc3d2e1f0U }; + uint32_t blocks_n0 = input_len / 64U; uint32_t blocks_n1; - if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + if (input_len % 64U == 0U && blocks_n0 > 0U) { - blocks_n1 = blocks_n0 - (uint32_t)1U; + blocks_n1 = blocks_n0 - 1U; } else { blocks_n1 = blocks_n0; } - uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint32_t blocks_len0 = blocks_n1 * 64U; uint8_t *blocks0 = input; uint32_t rest_len0 = input_len - blocks_len0; uint8_t *rest0 = input + blocks_len0; @@ -231,75 +191,75 @@ void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst uint8_t *blocks = blocks0; uint32_t rest_len = rest_len0; uint8_t *rest = rest0; - Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); - Hacl_Hash_SHA1_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); - Hacl_Hash_Core_SHA1_legacy_finish(s, dst); + Hacl_Hash_SHA1_update_multi(s, blocks, blocks_n); + Hacl_Hash_SHA1_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_SHA1_finish(s, output); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_malloc(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(5U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_Hash_Core_SHA1_legacy_init(block_state); + Hacl_Hash_SHA1_init(block_state); return p; } -void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA1_reset(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_Hash_Core_SHA1_legacy_init(block_state); + Hacl_Hash_SHA1_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_32 s = *p; + Hacl_Streaming_MD_state_32 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + if ((uint64_t)chunk_len > 2305843009213693951ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)64U; + sz = 64U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + sz = (uint32_t)(total_len % (uint64_t)64U); } - if (len <= (uint32_t)64U - sz) + if (chunk_len <= 64U - sz) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_32){ @@ -309,74 +269,74 @@ Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_SHA1_update_multi(block_state1, buf, 1U); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); } - uint32_t n_blocks = (len - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_Hash_SHA1_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)64U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)64U; + sz10 = 64U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_32){ @@ -385,114 +345,109 @@ Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, .total_len = total_len2 } ); - Hacl_Streaming_MD_state_32 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *state; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_SHA1_update_multi(block_state1, buf, 1U); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)64U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_Hash_SHA1_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } return Hacl_Streaming_Types_Success; } -void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_SHA1_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[5U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)5U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 5U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_Hash_SHA1_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + Hacl_Hash_SHA1_update_multi(tmp_block_state, buf_multi, 0U); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_Hash_SHA1_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); - Hacl_Hash_Core_SHA1_legacy_finish(tmp_block_state, dst); + Hacl_Hash_SHA1_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_SHA1_finish(tmp_block_state, output); } -void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA1_free(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_copy(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); - memcpy(block_state, block_state0, (uint32_t)5U * sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(5U, sizeof (uint32_t)); + memcpy(block_state, block_state0, 5U * sizeof (uint32_t)); Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_32 @@ -501,8 +456,8 @@ Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_st return p; } -void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA1_hash(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Hash_SHA1_legacy_hash(input, input_len, dst); + Hacl_Hash_SHA1_hash_oneshot(output, input, input_len); } diff --git a/Modules/_hacl/Hacl_Hash_SHA1.h b/Modules/_hacl/Hacl_Hash_SHA1.h index dc50aa6f6d3902..ad1e8e72a739ec 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/Hacl_Hash_SHA1.h @@ -31,31 +31,32 @@ extern "C" { #endif #include +#include "python_hacl_namespaces.h" #include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" #include "Hacl_Streaming_Types.h" -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA1_state; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_SHA1_state_t; -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_malloc(void); -void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA1_reset(Hacl_Streaming_MD_state_32 *state); /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); +Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len); -void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_SHA1_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output); -void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA1_free(Hacl_Streaming_MD_state_32 *state); -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_copy(Hacl_Streaming_MD_state_32 *state); -void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA1_hash(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/Hacl_Hash_SHA2.c b/Modules/_hacl/Hacl_Hash_SHA2.c index 08e3f7edbf4ede..4b6af5fc78c680 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.c +++ b/Modules/_hacl/Hacl_Hash_SHA2.c @@ -27,14 +27,14 @@ -void Hacl_SHA2_Scalar32_sha256_init(uint32_t *hash) +void Hacl_Hash_SHA2_sha256_init(uint32_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint32_t *os = hash; - uint32_t x = Hacl_Impl_SHA2_Generic_h256[i]; + uint32_t x = Hacl_Hash_SHA2_h256[i]; os[i] = x;); } @@ -42,49 +42,49 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) { uint32_t hash_old[8U] = { 0U }; uint32_t ws[16U] = { 0U }; - memcpy(hash_old, hash, (uint32_t)8U * sizeof (uint32_t)); + memcpy(hash_old, hash, 8U * sizeof (uint32_t)); uint8_t *b10 = b; uint32_t u = load32_be(b10); ws[0U] = u; - uint32_t u0 = load32_be(b10 + (uint32_t)4U); + uint32_t u0 = load32_be(b10 + 4U); ws[1U] = u0; - uint32_t u1 = load32_be(b10 + (uint32_t)8U); + uint32_t u1 = load32_be(b10 + 8U); ws[2U] = u1; - uint32_t u2 = load32_be(b10 + (uint32_t)12U); + uint32_t u2 = load32_be(b10 + 12U); ws[3U] = u2; - uint32_t u3 = load32_be(b10 + (uint32_t)16U); + uint32_t u3 = load32_be(b10 + 16U); ws[4U] = u3; - uint32_t u4 = load32_be(b10 + (uint32_t)20U); + uint32_t u4 = load32_be(b10 + 20U); ws[5U] = u4; - uint32_t u5 = load32_be(b10 + (uint32_t)24U); + uint32_t u5 = load32_be(b10 + 24U); ws[6U] = u5; - uint32_t u6 = load32_be(b10 + (uint32_t)28U); + uint32_t u6 = load32_be(b10 + 28U); ws[7U] = u6; - uint32_t u7 = load32_be(b10 + (uint32_t)32U); + uint32_t u7 = load32_be(b10 + 32U); ws[8U] = u7; - uint32_t u8 = load32_be(b10 + (uint32_t)36U); + uint32_t u8 = load32_be(b10 + 36U); ws[9U] = u8; - uint32_t u9 = load32_be(b10 + (uint32_t)40U); + uint32_t u9 = load32_be(b10 + 40U); ws[10U] = u9; - uint32_t u10 = load32_be(b10 + (uint32_t)44U); + uint32_t u10 = load32_be(b10 + 44U); ws[11U] = u10; - uint32_t u11 = load32_be(b10 + (uint32_t)48U); + uint32_t u11 = load32_be(b10 + 48U); ws[12U] = u11; - uint32_t u12 = load32_be(b10 + (uint32_t)52U); + uint32_t u12 = load32_be(b10 + 52U); ws[13U] = u12; - uint32_t u13 = load32_be(b10 + (uint32_t)56U); + uint32_t u13 = load32_be(b10 + 56U); ws[14U] = u13; - uint32_t u14 = load32_be(b10 + (uint32_t)60U); + uint32_t u14 = load32_be(b10 + 60U); ws[15U] = u14; KRML_MAYBE_FOR4(i0, - (uint32_t)0U, - (uint32_t)4U, - (uint32_t)1U, + 0U, + 4U, + 1U, KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, - uint32_t k_t = Hacl_Impl_SHA2_Generic_k224_256[(uint32_t)16U * i0 + i]; + 0U, + 16U, + 1U, + uint32_t k_t = Hacl_Hash_SHA2_k224_256[16U * i0 + i]; uint32_t ws_t = ws[i]; uint32_t a0 = hash[0U]; uint32_t b0 = hash[1U]; @@ -98,20 +98,13 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) uint32_t t1 = h02 - + - ((e0 << (uint32_t)26U | e0 >> (uint32_t)6U) - ^ - ((e0 << (uint32_t)21U | e0 >> (uint32_t)11U) - ^ (e0 << (uint32_t)7U | e0 >> (uint32_t)25U))) + + ((e0 << 26U | e0 >> 6U) ^ ((e0 << 21U | e0 >> 11U) ^ (e0 << 7U | e0 >> 25U))) + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint32_t t2 = - ((a0 << (uint32_t)30U | a0 >> (uint32_t)2U) - ^ - ((a0 << (uint32_t)19U | a0 >> (uint32_t)13U) - ^ (a0 << (uint32_t)10U | a0 >> (uint32_t)22U))) + ((a0 << 30U | a0 >> 2U) ^ ((a0 << 19U | a0 >> 13U) ^ (a0 << 10U | a0 >> 22U))) + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint32_t a1 = t1 + t2; uint32_t b1 = a0; @@ -129,74 +122,63 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) hash[5U] = f1; hash[6U] = g1; hash[7U] = h12;); - if (i0 < (uint32_t)3U) + if (i0 < 3U) { KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, + 0U, + 16U, + 1U, uint32_t t16 = ws[i]; - uint32_t t15 = ws[(i + (uint32_t)1U) % (uint32_t)16U]; - uint32_t t7 = ws[(i + (uint32_t)9U) % (uint32_t)16U]; - uint32_t t2 = ws[(i + (uint32_t)14U) % (uint32_t)16U]; - uint32_t - s1 = - (t2 << (uint32_t)15U | t2 >> (uint32_t)17U) - ^ ((t2 << (uint32_t)13U | t2 >> (uint32_t)19U) ^ t2 >> (uint32_t)10U); - uint32_t - s0 = - (t15 << (uint32_t)25U | t15 >> (uint32_t)7U) - ^ ((t15 << (uint32_t)14U | t15 >> (uint32_t)18U) ^ t15 >> (uint32_t)3U); + uint32_t t15 = ws[(i + 1U) % 16U]; + uint32_t t7 = ws[(i + 9U) % 16U]; + uint32_t t2 = ws[(i + 14U) % 16U]; + uint32_t s1 = (t2 << 15U | t2 >> 17U) ^ ((t2 << 13U | t2 >> 19U) ^ t2 >> 10U); + uint32_t s0 = (t15 << 25U | t15 >> 7U) ^ ((t15 << 14U | t15 >> 18U) ^ t15 >> 3U); ws[i] = s1 + t7 + s0 + t16;); }); KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint32_t *os = hash; uint32_t x = hash[i] + hash_old[i]; os[i] = x;); } -void Hacl_SHA2_Scalar32_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) +void Hacl_Hash_SHA2_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) { - uint32_t blocks = len / (uint32_t)64U; - for (uint32_t i = (uint32_t)0U; i < blocks; i++) + uint32_t blocks = len / 64U; + for (uint32_t i = 0U; i < blocks; i++) { uint8_t *b0 = b; - uint8_t *mb = b0 + i * (uint32_t)64U; + uint8_t *mb = b0 + i * 64U; sha256_update(mb, st); } } void -Hacl_SHA2_Scalar32_sha256_update_last( - uint64_t totlen, - uint32_t len, - uint8_t *b, - uint32_t *hash -) +Hacl_Hash_SHA2_sha256_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *hash) { uint32_t blocks; - if (len + (uint32_t)8U + (uint32_t)1U <= (uint32_t)64U) + if (len + 8U + 1U <= 64U) { - blocks = (uint32_t)1U; + blocks = 1U; } else { - blocks = (uint32_t)2U; + blocks = 2U; } - uint32_t fin = blocks * (uint32_t)64U; + uint32_t fin = blocks * 64U; uint8_t last[128U] = { 0U }; uint8_t totlen_buf[8U] = { 0U }; - uint64_t total_len_bits = totlen << (uint32_t)3U; + uint64_t total_len_bits = totlen << 3U; store64_be(totlen_buf, total_len_bits); uint8_t *b0 = b; memcpy(last, b0, len * sizeof (uint8_t)); - last[len] = (uint8_t)0x80U; - memcpy(last + fin - (uint32_t)8U, totlen_buf, (uint32_t)8U * sizeof (uint8_t)); + last[len] = 0x80U; + memcpy(last + fin - 8U, totlen_buf, 8U * sizeof (uint8_t)); uint8_t *last00 = last; - uint8_t *last10 = last + (uint32_t)64U; + uint8_t *last10 = last + 64U; uint8_t *l0 = last00; uint8_t *l1 = last10; uint8_t *lb0 = l0; @@ -204,65 +186,56 @@ Hacl_SHA2_Scalar32_sha256_update_last( uint8_t *last0 = lb0; uint8_t *last1 = lb1; sha256_update(last0, hash); - if (blocks > (uint32_t)1U) + if (blocks > 1U) { sha256_update(last1, hash); return; } } -void Hacl_SHA2_Scalar32_sha256_finish(uint32_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha256_finish(uint32_t *st, uint8_t *h) { uint8_t hbuf[32U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store32_be(hbuf + i * (uint32_t)4U, st[i]);); - memcpy(h, hbuf, (uint32_t)32U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store32_be(hbuf + i * 4U, st[i]);); + memcpy(h, hbuf, 32U * sizeof (uint8_t)); } -void Hacl_SHA2_Scalar32_sha224_init(uint32_t *hash) +void Hacl_Hash_SHA2_sha224_init(uint32_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint32_t *os = hash; - uint32_t x = Hacl_Impl_SHA2_Generic_h224[i]; + uint32_t x = Hacl_Hash_SHA2_h224[i]; os[i] = x;); } static inline void sha224_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) { - Hacl_SHA2_Scalar32_sha256_update_nblocks(len, b, st); + Hacl_Hash_SHA2_sha256_update_nblocks(len, b, st); } -void -Hacl_SHA2_Scalar32_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st) +void Hacl_Hash_SHA2_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st) { - Hacl_SHA2_Scalar32_sha256_update_last(totlen, len, b, st); + Hacl_Hash_SHA2_sha256_update_last(totlen, len, b, st); } -void Hacl_SHA2_Scalar32_sha224_finish(uint32_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha224_finish(uint32_t *st, uint8_t *h) { uint8_t hbuf[32U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store32_be(hbuf + i * (uint32_t)4U, st[i]);); - memcpy(h, hbuf, (uint32_t)28U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store32_be(hbuf + i * 4U, st[i]);); + memcpy(h, hbuf, 28U * sizeof (uint8_t)); } -void Hacl_SHA2_Scalar32_sha512_init(uint64_t *hash) +void Hacl_Hash_SHA2_sha512_init(uint64_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint64_t *os = hash; - uint64_t x = Hacl_Impl_SHA2_Generic_h512[i]; + uint64_t x = Hacl_Hash_SHA2_h512[i]; os[i] = x;); } @@ -270,49 +243,49 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) { uint64_t hash_old[8U] = { 0U }; uint64_t ws[16U] = { 0U }; - memcpy(hash_old, hash, (uint32_t)8U * sizeof (uint64_t)); + memcpy(hash_old, hash, 8U * sizeof (uint64_t)); uint8_t *b10 = b; uint64_t u = load64_be(b10); ws[0U] = u; - uint64_t u0 = load64_be(b10 + (uint32_t)8U); + uint64_t u0 = load64_be(b10 + 8U); ws[1U] = u0; - uint64_t u1 = load64_be(b10 + (uint32_t)16U); + uint64_t u1 = load64_be(b10 + 16U); ws[2U] = u1; - uint64_t u2 = load64_be(b10 + (uint32_t)24U); + uint64_t u2 = load64_be(b10 + 24U); ws[3U] = u2; - uint64_t u3 = load64_be(b10 + (uint32_t)32U); + uint64_t u3 = load64_be(b10 + 32U); ws[4U] = u3; - uint64_t u4 = load64_be(b10 + (uint32_t)40U); + uint64_t u4 = load64_be(b10 + 40U); ws[5U] = u4; - uint64_t u5 = load64_be(b10 + (uint32_t)48U); + uint64_t u5 = load64_be(b10 + 48U); ws[6U] = u5; - uint64_t u6 = load64_be(b10 + (uint32_t)56U); + uint64_t u6 = load64_be(b10 + 56U); ws[7U] = u6; - uint64_t u7 = load64_be(b10 + (uint32_t)64U); + uint64_t u7 = load64_be(b10 + 64U); ws[8U] = u7; - uint64_t u8 = load64_be(b10 + (uint32_t)72U); + uint64_t u8 = load64_be(b10 + 72U); ws[9U] = u8; - uint64_t u9 = load64_be(b10 + (uint32_t)80U); + uint64_t u9 = load64_be(b10 + 80U); ws[10U] = u9; - uint64_t u10 = load64_be(b10 + (uint32_t)88U); + uint64_t u10 = load64_be(b10 + 88U); ws[11U] = u10; - uint64_t u11 = load64_be(b10 + (uint32_t)96U); + uint64_t u11 = load64_be(b10 + 96U); ws[12U] = u11; - uint64_t u12 = load64_be(b10 + (uint32_t)104U); + uint64_t u12 = load64_be(b10 + 104U); ws[13U] = u12; - uint64_t u13 = load64_be(b10 + (uint32_t)112U); + uint64_t u13 = load64_be(b10 + 112U); ws[14U] = u13; - uint64_t u14 = load64_be(b10 + (uint32_t)120U); + uint64_t u14 = load64_be(b10 + 120U); ws[15U] = u14; KRML_MAYBE_FOR5(i0, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, + 0U, + 5U, + 1U, KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, - uint64_t k_t = Hacl_Impl_SHA2_Generic_k384_512[(uint32_t)16U * i0 + i]; + 0U, + 16U, + 1U, + uint64_t k_t = Hacl_Hash_SHA2_k384_512[16U * i0 + i]; uint64_t ws_t = ws[i]; uint64_t a0 = hash[0U]; uint64_t b0 = hash[1U]; @@ -326,20 +299,13 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) uint64_t t1 = h02 - + - ((e0 << (uint32_t)50U | e0 >> (uint32_t)14U) - ^ - ((e0 << (uint32_t)46U | e0 >> (uint32_t)18U) - ^ (e0 << (uint32_t)23U | e0 >> (uint32_t)41U))) + + ((e0 << 50U | e0 >> 14U) ^ ((e0 << 46U | e0 >> 18U) ^ (e0 << 23U | e0 >> 41U))) + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint64_t t2 = - ((a0 << (uint32_t)36U | a0 >> (uint32_t)28U) - ^ - ((a0 << (uint32_t)30U | a0 >> (uint32_t)34U) - ^ (a0 << (uint32_t)25U | a0 >> (uint32_t)39U))) + ((a0 << 36U | a0 >> 28U) ^ ((a0 << 30U | a0 >> 34U) ^ (a0 << 25U | a0 >> 39U))) + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint64_t a1 = t1 + t2; uint64_t b1 = a0; @@ -357,48 +323,42 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) hash[5U] = f1; hash[6U] = g1; hash[7U] = h12;); - if (i0 < (uint32_t)4U) + if (i0 < 4U) { KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, + 0U, + 16U, + 1U, uint64_t t16 = ws[i]; - uint64_t t15 = ws[(i + (uint32_t)1U) % (uint32_t)16U]; - uint64_t t7 = ws[(i + (uint32_t)9U) % (uint32_t)16U]; - uint64_t t2 = ws[(i + (uint32_t)14U) % (uint32_t)16U]; - uint64_t - s1 = - (t2 << (uint32_t)45U | t2 >> (uint32_t)19U) - ^ ((t2 << (uint32_t)3U | t2 >> (uint32_t)61U) ^ t2 >> (uint32_t)6U); - uint64_t - s0 = - (t15 << (uint32_t)63U | t15 >> (uint32_t)1U) - ^ ((t15 << (uint32_t)56U | t15 >> (uint32_t)8U) ^ t15 >> (uint32_t)7U); + uint64_t t15 = ws[(i + 1U) % 16U]; + uint64_t t7 = ws[(i + 9U) % 16U]; + uint64_t t2 = ws[(i + 14U) % 16U]; + uint64_t s1 = (t2 << 45U | t2 >> 19U) ^ ((t2 << 3U | t2 >> 61U) ^ t2 >> 6U); + uint64_t s0 = (t15 << 63U | t15 >> 1U) ^ ((t15 << 56U | t15 >> 8U) ^ t15 >> 7U); ws[i] = s1 + t7 + s0 + t16;); }); KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint64_t *os = hash; uint64_t x = hash[i] + hash_old[i]; os[i] = x;); } -void Hacl_SHA2_Scalar32_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +void Hacl_Hash_SHA2_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) { - uint32_t blocks = len / (uint32_t)128U; - for (uint32_t i = (uint32_t)0U; i < blocks; i++) + uint32_t blocks = len / 128U; + for (uint32_t i = 0U; i < blocks; i++) { uint8_t *b0 = b; - uint8_t *mb = b0 + i * (uint32_t)128U; + uint8_t *mb = b0 + i * 128U; sha512_update(mb, st); } } void -Hacl_SHA2_Scalar32_sha512_update_last( +Hacl_Hash_SHA2_sha512_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, @@ -406,25 +366,25 @@ Hacl_SHA2_Scalar32_sha512_update_last( ) { uint32_t blocks; - if (len + (uint32_t)16U + (uint32_t)1U <= (uint32_t)128U) + if (len + 16U + 1U <= 128U) { - blocks = (uint32_t)1U; + blocks = 1U; } else { - blocks = (uint32_t)2U; + blocks = 2U; } - uint32_t fin = blocks * (uint32_t)128U; + uint32_t fin = blocks * 128U; uint8_t last[256U] = { 0U }; uint8_t totlen_buf[16U] = { 0U }; - FStar_UInt128_uint128 total_len_bits = FStar_UInt128_shift_left(totlen, (uint32_t)3U); + FStar_UInt128_uint128 total_len_bits = FStar_UInt128_shift_left(totlen, 3U); store128_be(totlen_buf, total_len_bits); uint8_t *b0 = b; memcpy(last, b0, len * sizeof (uint8_t)); - last[len] = (uint8_t)0x80U; - memcpy(last + fin - (uint32_t)16U, totlen_buf, (uint32_t)16U * sizeof (uint8_t)); + last[len] = 0x80U; + memcpy(last + fin - 16U, totlen_buf, 16U * sizeof (uint8_t)); uint8_t *last00 = last; - uint8_t *last10 = last + (uint32_t)128U; + uint8_t *last10 = last + 128U; uint8_t *l0 = last00; uint8_t *l1 = last10; uint8_t *lb0 = l0; @@ -432,76 +392,68 @@ Hacl_SHA2_Scalar32_sha512_update_last( uint8_t *last0 = lb0; uint8_t *last1 = lb1; sha512_update(last0, hash); - if (blocks > (uint32_t)1U) + if (blocks > 1U) { sha512_update(last1, hash); return; } } -void Hacl_SHA2_Scalar32_sha512_finish(uint64_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha512_finish(uint64_t *st, uint8_t *h) { uint8_t hbuf[64U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store64_be(hbuf + i * (uint32_t)8U, st[i]);); - memcpy(h, hbuf, (uint32_t)64U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store64_be(hbuf + i * 8U, st[i]);); + memcpy(h, hbuf, 64U * sizeof (uint8_t)); } -void Hacl_SHA2_Scalar32_sha384_init(uint64_t *hash) +void Hacl_Hash_SHA2_sha384_init(uint64_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint64_t *os = hash; - uint64_t x = Hacl_Impl_SHA2_Generic_h384[i]; + uint64_t x = Hacl_Hash_SHA2_h384[i]; os[i] = x;); } -void Hacl_SHA2_Scalar32_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +void Hacl_Hash_SHA2_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) { - Hacl_SHA2_Scalar32_sha512_update_nblocks(len, b, st); + Hacl_Hash_SHA2_sha512_update_nblocks(len, b, st); } void -Hacl_SHA2_Scalar32_sha384_update_last( +Hacl_Hash_SHA2_sha384_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *st ) { - Hacl_SHA2_Scalar32_sha512_update_last(totlen, len, b, st); + Hacl_Hash_SHA2_sha512_update_last(totlen, len, b, st); } -void Hacl_SHA2_Scalar32_sha384_finish(uint64_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha384_finish(uint64_t *st, uint8_t *h) { uint8_t hbuf[64U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store64_be(hbuf + i * (uint32_t)8U, st[i]);); - memcpy(h, hbuf, (uint32_t)48U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store64_be(hbuf + i * 8U, st[i]);); + memcpy(h, hbuf, 48U * sizeof (uint8_t)); } /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_256(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(8U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_SHA2_Scalar32_sha256_init(block_state); + Hacl_Hash_SHA2_sha256_init(block_state); return p; } @@ -511,16 +463,16 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_copy_256(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); - memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(8U, sizeof (uint32_t)); + memcpy(block_state, block_state0, 8U * sizeof (uint32_t)); Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_32 @@ -532,54 +484,54 @@ Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA2_reset_256(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha256_init(block_state); + Hacl_Hash_SHA2_sha256_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } static inline Hacl_Streaming_Types_error_code -update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_32 s = *p; + Hacl_Streaming_MD_state_32 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + if ((uint64_t)chunk_len > 2305843009213693951ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)64U; + sz = 64U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + sz = (uint32_t)(total_len % (uint64_t)64U); } - if (len <= (uint32_t)64U - sz) + if (chunk_len <= 64U - sz) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_32){ @@ -589,76 +541,74 @@ update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha256_update_nblocks((uint32_t)64U, buf, block_state1); + Hacl_Hash_SHA2_sha256_update_nblocks(64U, buf, block_state1); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); } - uint32_t n_blocks = (len - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_SHA2_Scalar32_sha256_update_nblocks(data1_len / (uint32_t)64U * (uint32_t)64U, - data1, - block_state1); + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)64U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)64U; + sz10 = 64U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_32){ @@ -667,55 +617,48 @@ update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) .total_len = total_len2 } ); - Hacl_Streaming_MD_state_32 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *state; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha256_update_nblocks((uint32_t)64U, buf, block_state1); + Hacl_Hash_SHA2_sha256_update_nblocks(64U, buf, block_state1); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)64U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_SHA2_Scalar32_sha256_update_nblocks(data1_len / (uint32_t)64U * (uint32_t)64U, - data11, - block_state1); + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } @@ -725,209 +668,203 @@ update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_256` -(since the last call to `init_256`) exceeds 2^61-1 bytes. +(since the last call to `reset_256`) exceeds 2^61-1 bytes. This function is identical to the update function for SHA2_224. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_256( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ) { - return update_224_256(p, input, input_len); + return update_224_256(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 32 bytes. The state remains -valid after a call to `finish_256`, meaning the user may feed more data into -the hash via `update_256`. (The finish_256 function operates on an internal copy of +Write the resulting hash into `output`, an array of 32 bytes. The state remains +valid after a call to `digest_256`, meaning the user may feed more data into +the hash via `update_256`. (The digest_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_256(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_SHA2_Scalar32_sha256_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha256_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha256_update_last(prev_len_last + (uint64_t)r, - r, - buf_last, - tmp_block_state); - Hacl_SHA2_Scalar32_sha256_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha256_update_last(prev_len_last + (uint64_t)r, r, buf_last, tmp_block_state); + Hacl_Hash_SHA2_sha256_finish(tmp_block_state, output); } /** -Free a state allocated with `create_in_256`. +Free a state allocated with `malloc_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA2_free_256(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 32 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 32 bytes. */ -void Hacl_Streaming_SHA2_hash_256(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_256(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint32_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha256_init(st); - uint32_t rem = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha256_init(st); + uint32_t rem = input_len % 64U; uint64_t len_ = (uint64_t)input_len; - Hacl_SHA2_Scalar32_sha256_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha256_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 64U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha256_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha256_finish(st, rb); + Hacl_Hash_SHA2_sha256_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha256_finish(st, rb); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_224(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(8U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_SHA2_Scalar32_sha224_init(block_state); + Hacl_Hash_SHA2_sha224_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA2_reset_224(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha224_init(block_state); + Hacl_Hash_SHA2_sha224_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_224( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ) { - return update_224_256(p, input, input_len); + return update_224_256(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 28 bytes. The state remains -valid after a call to `finish_224`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 28 bytes. The state remains +valid after a call to `digest_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_224(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - sha224_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + sha224_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha224_update_last(prev_len_last + (uint64_t)r, - r, - buf_last, - tmp_block_state); - Hacl_SHA2_Scalar32_sha224_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha224_update_last(prev_len_last + (uint64_t)r, r, buf_last, tmp_block_state); + Hacl_Hash_SHA2_sha224_finish(tmp_block_state, output); } -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p) +void Hacl_Hash_SHA2_free_224(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_SHA2_free_256(p); + Hacl_Hash_SHA2_free_256(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 28 bytes. */ -void Hacl_Streaming_SHA2_hash_224(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_224(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint32_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha224_init(st); - uint32_t rem = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha224_init(st); + uint32_t rem = input_len % 64U; uint64_t len_ = (uint64_t)input_len; sha224_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)64U; + uint32_t rem1 = input_len % 64U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha224_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha224_finish(st, rb); + Hacl_Hash_SHA2_sha224_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha224_finish(st, rb); } -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void) +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_512(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); - uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC(8U, sizeof (uint64_t)); Hacl_Streaming_MD_state_64 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_64 *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; - Hacl_SHA2_Scalar32_sha512_init(block_state); + Hacl_Hash_SHA2_sha512_init(block_state); return p; } @@ -937,16 +874,16 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0) +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_copy_512(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s0; + Hacl_Streaming_MD_state_64 scrut = *state; uint64_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)128U * sizeof (uint8_t)); - uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); - memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint64_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + memcpy(buf, buf0, 128U * sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC(8U, sizeof (uint64_t)); + memcpy(block_state, block_state0, 8U * sizeof (uint64_t)); Hacl_Streaming_MD_state_64 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_64 @@ -955,54 +892,54 @@ Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state return p; } -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s) +void Hacl_Hash_SHA2_reset_512(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *state; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha512_init(block_state); + Hacl_Hash_SHA2_sha512_init(block_state); Hacl_Streaming_MD_state_64 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } static inline Hacl_Streaming_Types_error_code -update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) +update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_64 s = *p; + Hacl_Streaming_MD_state_64 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)18446744073709551615U - total_len) + if ((uint64_t)chunk_len > 18446744073709551615ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)128U; + sz = 128U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + sz = (uint32_t)(total_len % (uint64_t)128U); } - if (len <= (uint32_t)128U - sz) + if (chunk_len <= 128U - sz) { - Hacl_Streaming_MD_state_64 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *state; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)128U; + sz1 = 128U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_64){ @@ -1012,76 +949,74 @@ update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_64 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *state; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)128U; + sz1 = 128U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha512_update_nblocks((uint32_t)128U, buf, block_state1); + Hacl_Hash_SHA2_sha512_update_nblocks(128U, buf, block_state1); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)128U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)128U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)128U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)128U); } - uint32_t n_blocks = (len - ite) / (uint32_t)128U; - uint32_t data1_len = n_blocks * (uint32_t)128U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_SHA2_Scalar32_sha512_update_nblocks(data1_len / (uint32_t)128U * (uint32_t)128U, - data1, - block_state1); + uint32_t n_blocks = (chunk_len - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)128U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_64 s1 = *p; + uint32_t diff = 128U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_64 s1 = *state; uint64_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)128U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)128U; + sz10 = 128U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)128U); + sz10 = (uint32_t)(total_len10 % (uint64_t)128U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_64){ @@ -1090,55 +1025,48 @@ update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) .total_len = total_len2 } ); - Hacl_Streaming_MD_state_64 s10 = *p; + Hacl_Streaming_MD_state_64 s10 = *state; uint64_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)128U; + sz1 = 128U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha512_update_nblocks((uint32_t)128U, buf, block_state1); + Hacl_Hash_SHA2_sha512_update_nblocks(128U, buf, block_state1); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)128U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)128U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)128U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)128U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)128U; - uint32_t data1_len = n_blocks * (uint32_t)128U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_SHA2_Scalar32_sha512_update_nblocks(data1_len / (uint32_t)128U * (uint32_t)128U, - data11, - block_state1); + uint32_t n_blocks = (chunk_len - diff - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } @@ -1148,198 +1076,198 @@ update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_512` -(since the last call to `init_512`) exceeds 2^125-1 bytes. +(since the last call to `reset_512`) exceeds 2^125-1 bytes. This function is identical to the update function for SHA2_384. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_512( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ) { - return update_384_512(p, input, input_len); + return update_384_512(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 64 bytes. The state remains -valid after a call to `finish_512`, meaning the user may feed more data into -the hash via `update_512`. (The finish_512 function operates on an internal copy of +Write the resulting hash into `output`, an array of 64 bytes. The state remains +valid after a call to `digest_512`, meaning the user may feed more data into +the hash via `update_512`. (The digest_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_512(Hacl_Streaming_MD_state_64 *state, uint8_t *output) { - Hacl_Streaming_MD_state_64 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *state; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) { - r = (uint32_t)128U; + r = 128U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + r = (uint32_t)(total_len % (uint64_t)128U); } uint8_t *buf_1 = buf_; uint64_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint64_t)); uint32_t ite; - if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 128U == 0U && r > 0U) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = r % (uint32_t)128U; + ite = r % 128U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_SHA2_Scalar32_sha512_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha512_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha512_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + Hacl_Hash_SHA2_sha512_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), FStar_UInt128_uint64_to_uint128((uint64_t)r)), r, buf_last, tmp_block_state); - Hacl_SHA2_Scalar32_sha512_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha512_finish(tmp_block_state, output); } /** -Free a state allocated with `create_in_512`. +Free a state allocated with `malloc_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s) +void Hacl_Hash_SHA2_free_512(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *state; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 64 bytes. */ -void Hacl_Streaming_SHA2_hash_512(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_512(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint64_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha512_init(st); - uint32_t rem = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha512_init(st); + uint32_t rem = input_len % 128U; FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); - Hacl_SHA2_Scalar32_sha512_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha512_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 128U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha512_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha512_finish(st, rb); + Hacl_Hash_SHA2_sha512_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha512_finish(st, rb); } -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void) +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_384(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); - uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC(8U, sizeof (uint64_t)); Hacl_Streaming_MD_state_64 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_64 *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; - Hacl_SHA2_Scalar32_sha384_init(block_state); + Hacl_Hash_SHA2_sha384_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s) +void Hacl_Hash_SHA2_reset_384(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *state; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha384_init(block_state); + Hacl_Hash_SHA2_sha384_init(block_state); Hacl_Streaming_MD_state_64 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_384( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ) { - return update_384_512(p, input, input_len); + return update_384_512(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 48 bytes. The state remains -valid after a call to `finish_384`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 48 bytes. The state remains +valid after a call to `digest_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_384(Hacl_Streaming_MD_state_64 *state, uint8_t *output) { - Hacl_Streaming_MD_state_64 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *state; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) { - r = (uint32_t)128U; + r = 128U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + r = (uint32_t)(total_len % (uint64_t)128U); } uint8_t *buf_1 = buf_; uint64_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint64_t)); uint32_t ite; - if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 128U == 0U && r > 0U) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = r % (uint32_t)128U; + ite = r % 128U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_SHA2_Scalar32_sha384_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha384_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha384_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + Hacl_Hash_SHA2_sha384_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), FStar_UInt128_uint64_to_uint128((uint64_t)r)), r, buf_last, tmp_block_state); - Hacl_SHA2_Scalar32_sha384_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha384_finish(tmp_block_state, output); } -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p) +void Hacl_Hash_SHA2_free_384(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_SHA2_free_512(p); + Hacl_Hash_SHA2_free_512(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 48 bytes. */ -void Hacl_Streaming_SHA2_hash_384(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_384(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint64_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha384_init(st); - uint32_t rem = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha384_init(st); + uint32_t rem = input_len % 128U; FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); - Hacl_SHA2_Scalar32_sha384_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha384_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 128U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha384_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha384_finish(st, rb); + Hacl_Hash_SHA2_sha384_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha384_finish(st, rb); } diff --git a/Modules/_hacl/Hacl_Hash_SHA2.h b/Modules/_hacl/Hacl_Hash_SHA2.h index a0e731094dfaa5..d8204b504baf82 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/Hacl_Hash_SHA2.h @@ -39,19 +39,19 @@ extern "C" { #include "Hacl_Streaming_Types.h" -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_224; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_SHA2_state_t_224; -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_256; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_SHA2_state_t_256; -typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_384; +typedef Hacl_Streaming_MD_state_64 Hacl_Hash_SHA2_state_t_384; -typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_512; +typedef Hacl_Streaming_MD_state_64 Hacl_Hash_SHA2_state_t_512; /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_256(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -59,73 +59,73 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_copy_256(Hacl_Streaming_MD_state_32 *state); /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA2_reset_256(Hacl_Streaming_MD_state_32 *state); /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_256` -(since the last call to `init_256`) exceeds 2^61-1 bytes. +(since the last call to `reset_256`) exceeds 2^61-1 bytes. This function is identical to the update function for SHA2_224. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_256( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 32 bytes. The state remains -valid after a call to `finish_256`, meaning the user may feed more data into -the hash via `update_256`. (The finish_256 function operates on an internal copy of +Write the resulting hash into `output`, an array of 32 bytes. The state remains +valid after a call to `digest_256`, meaning the user may feed more data into +the hash via `update_256`. (The digest_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_256(Hacl_Streaming_MD_state_32 *state, uint8_t *output); /** -Free a state allocated with `create_in_256`. +Free a state allocated with `malloc_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA2_free_256(Hacl_Streaming_MD_state_32 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 32 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 32 bytes. */ -void Hacl_Streaming_SHA2_hash_256(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_256(uint8_t *output, uint8_t *input, uint32_t input_len); -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_224(void); -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA2_reset_224(Hacl_Streaming_MD_state_32 *state); Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_224( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 28 bytes. The state remains -valid after a call to `finish_224`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 28 bytes. The state remains +valid after a call to `digest_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_224(Hacl_Streaming_MD_state_32 *state, uint8_t *output); -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p); +void Hacl_Hash_SHA2_free_224(Hacl_Streaming_MD_state_32 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 28 bytes. */ -void Hacl_Streaming_SHA2_hash_224(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_224(uint8_t *output, uint8_t *input, uint32_t input_len); -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void); +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_512(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -133,68 +133,68 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0); +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_copy_512(Hacl_Streaming_MD_state_64 *state); -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s); +void Hacl_Hash_SHA2_reset_512(Hacl_Streaming_MD_state_64 *state); /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_512` -(since the last call to `init_512`) exceeds 2^125-1 bytes. +(since the last call to `reset_512`) exceeds 2^125-1 bytes. This function is identical to the update function for SHA2_384. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_512( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 64 bytes. The state remains -valid after a call to `finish_512`, meaning the user may feed more data into -the hash via `update_512`. (The finish_512 function operates on an internal copy of +Write the resulting hash into `output`, an array of 64 bytes. The state remains +valid after a call to `digest_512`, meaning the user may feed more data into +the hash via `update_512`. (The digest_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_512(Hacl_Streaming_MD_state_64 *state, uint8_t *output); /** -Free a state allocated with `create_in_512`. +Free a state allocated with `malloc_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s); +void Hacl_Hash_SHA2_free_512(Hacl_Streaming_MD_state_64 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 64 bytes. */ -void Hacl_Streaming_SHA2_hash_512(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_512(uint8_t *output, uint8_t *input, uint32_t input_len); -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void); +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_384(void); -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s); +void Hacl_Hash_SHA2_reset_384(Hacl_Streaming_MD_state_64 *state); Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_384( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 48 bytes. The state remains -valid after a call to `finish_384`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 48 bytes. The state remains +valid after a call to `digest_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_384(Hacl_Streaming_MD_state_64 *state, uint8_t *output); -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p); +void Hacl_Hash_SHA2_free_384(Hacl_Streaming_MD_state_64 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 48 bytes. */ -void Hacl_Streaming_SHA2_hash_384(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_384(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c index b3febdfeb2b221..9cf5abb330b180 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.c +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -25,33 +25,178 @@ #include "internal/Hacl_Hash_SHA3.h" +const +uint32_t +Hacl_Hash_SHA3_keccak_rotc[24U] = + { + 1U, 3U, 6U, 10U, 15U, 21U, 28U, 36U, 45U, 55U, 2U, 14U, 27U, 41U, 56U, 8U, 25U, 43U, 62U, 18U, + 39U, 61U, 20U, 44U + }; + +const +uint32_t +Hacl_Hash_SHA3_keccak_piln[24U] = + { + 10U, 7U, 11U, 17U, 18U, 3U, 5U, 16U, 8U, 21U, 24U, 4U, 15U, 23U, 19U, 13U, 12U, 2U, 20U, 14U, + 22U, 9U, 6U, 1U + }; + +const +uint64_t +Hacl_Hash_SHA3_keccak_rndc[24U] = + { + 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL + }; + +static void absorb_inner_32(uint8_t *b, uint64_t *s) +{ + uint64_t ws[32U] = { 0U }; + uint8_t *b1 = b; + uint64_t u = load64_le(b1); + ws[0U] = u; + uint64_t u0 = load64_le(b1 + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b1 + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b1 + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b1 + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b1 + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b1 + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b1 + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b1 + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b1 + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b1 + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b1 + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b1 + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b1 + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b1 + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b1 + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b1 + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b1 + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b1 + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b1 + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b1 + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b1 + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b1 + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b1 + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b1 + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b1 + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b1 + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b1 + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b1 + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b1 + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b1 + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b1 + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws[i]; + } + for (uint32_t i0 = 0U; i0 < 24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i1, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i1 + 1U) % 5U]; + uint64_t _D = _C[(i1 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i1 + 5U * i] = s[i1 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } +} + static uint32_t block_len(Spec_Hash_Definitions_hash_alg a) { switch (a) { case Spec_Hash_Definitions_SHA3_224: { - return (uint32_t)144U; + return 144U; } case Spec_Hash_Definitions_SHA3_256: { - return (uint32_t)136U; + return 136U; } case Spec_Hash_Definitions_SHA3_384: { - return (uint32_t)104U; + return 104U; } case Spec_Hash_Definitions_SHA3_512: { - return (uint32_t)72U; + return 72U; } case Spec_Hash_Definitions_Shake128: { - return (uint32_t)168U; + return 168U; } case Spec_Hash_Definitions_Shake256: { - return (uint32_t)136U; + return 136U; } default: { @@ -67,19 +212,19 @@ static uint32_t hash_len(Spec_Hash_Definitions_hash_alg a) { case Spec_Hash_Definitions_SHA3_224: { - return (uint32_t)28U; + return 28U; } case Spec_Hash_Definitions_SHA3_256: { - return (uint32_t)32U; + return 32U; } case Spec_Hash_Definitions_SHA3_384: { - return (uint32_t)48U; + return 48U; } case Spec_Hash_Definitions_SHA3_512: { - return (uint32_t)64U; + return 64U; } default: { @@ -97,10 +242,17 @@ Hacl_Hash_SHA3_update_multi_sha3( uint32_t n_blocks ) { - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + uint32_t l = block_len(a) * n_blocks; + for (uint32_t i = 0U; i < l / block_len(a); i++) { - uint8_t *block = blocks + i * block_len(a); - Hacl_Impl_SHA3_absorb_inner(block_len(a), block, s); + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = blocks; + uint8_t *bl0 = b_; + uint8_t *uu____0 = b0 + i * block_len(a); + memcpy(bl0, uu____0, block_len(a) * sizeof (uint8_t)); + block_len(a); + absorb_inner_32(b_, s); } } @@ -115,139 +267,374 @@ Hacl_Hash_SHA3_update_last_sha3( uint8_t suffix; if (a == Spec_Hash_Definitions_Shake128 || a == Spec_Hash_Definitions_Shake256) { - suffix = (uint8_t)0x1fU; + suffix = 0x1fU; } else { - suffix = (uint8_t)0x06U; + suffix = 0x06U; } uint32_t len = block_len(a); if (input_len == len) { - Hacl_Impl_SHA3_absorb_inner(len, input, s); - uint8_t *uu____0 = input + input_len; - uint8_t lastBlock_[200U] = { 0U }; - uint8_t *lastBlock = lastBlock_; - memcpy(lastBlock, uu____0, (uint32_t)0U * sizeof (uint8_t)); - lastBlock[0U] = suffix; - Hacl_Impl_SHA3_loadState(len, lastBlock, s); - if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && (uint32_t)0U == len - (uint32_t)1U) - { - Hacl_Impl_SHA3_state_permute(s); - } - uint8_t nextBlock_[200U] = { 0U }; - uint8_t *nextBlock = nextBlock_; - nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; - Hacl_Impl_SHA3_loadState(len, nextBlock, s); - Hacl_Impl_SHA3_state_permute(s); + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint8_t *b00 = input; + uint8_t *bl00 = b_; + memcpy(bl00, b00 + 0U * len, len * sizeof (uint8_t)); + absorb_inner_32(b_, s); + uint8_t b2[256U] = { 0U }; + uint8_t *b_0 = b2; + uint32_t rem = 0U % len; + uint8_t *b01 = input + input_len; + uint8_t *bl0 = b_0; + memcpy(bl0, b01 + 0U - rem, rem * sizeof (uint8_t)); + uint8_t *b02 = b_0; + b02[0U % len] = suffix; + uint64_t ws[32U] = { 0U }; + uint8_t *b = b_0; + uint64_t u = load64_le(b); + ws[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws[i]; + } + if (!(((uint32_t)suffix & 0x80U) == 0U) && 0U % len == len - 1U) + { + for (uint32_t i0 = 0U; i0 < 24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i1, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i1 + 1U) % 5U]; + uint64_t _D = _C[(i1 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i1 + 5U * i] = s[i1 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } + } + uint8_t b3[256U] = { 0U }; + uint8_t *b4 = b3; + uint8_t *b0 = b4; + b0[len - 1U] = 0x80U; + absorb_inner_32(b4, s); return; } - uint8_t lastBlock_[200U] = { 0U }; - uint8_t *lastBlock = lastBlock_; - memcpy(lastBlock, input, input_len * sizeof (uint8_t)); - lastBlock[input_len] = suffix; - Hacl_Impl_SHA3_loadState(len, lastBlock, s); - if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && input_len == len - (uint32_t)1U) + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = input_len % len; + uint8_t *b00 = input; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + input_len - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[input_len % len] = suffix; + uint64_t ws[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws[i]; + } + if (!(((uint32_t)suffix & 0x80U) == 0U) && input_len % len == len - 1U) { - Hacl_Impl_SHA3_state_permute(s); + for (uint32_t i0 = 0U; i0 < 24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i1, + 0U, + 5U, + 1U, + uint64_t uu____2 = _C[(i1 + 1U) % 5U]; + uint64_t _D = _C[(i1 + 4U) % 5U] ^ (uu____2 << 1U | uu____2 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i1 + 5U * i] = s[i1 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____3 = current; + s[_Y] = uu____3 << r | uu____3 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } } - uint8_t nextBlock_[200U] = { 0U }; - uint8_t *nextBlock = nextBlock_; - nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; - Hacl_Impl_SHA3_loadState(len, nextBlock, s); - Hacl_Impl_SHA3_state_permute(s); + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[len - 1U] = 0x80U; + absorb_inner_32(b3, s); } typedef struct hash_buf2_s { - Hacl_Streaming_Keccak_hash_buf fst; - Hacl_Streaming_Keccak_hash_buf snd; + Hacl_Hash_SHA3_hash_buf fst; + Hacl_Hash_SHA3_hash_buf snd; } hash_buf2; -Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s) +Spec_Hash_Definitions_hash_alg Hacl_Hash_SHA3_get_alg(Hacl_Hash_SHA3_state_t *s) { - Hacl_Streaming_Keccak_state scrut = *s; - Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + Hacl_Hash_SHA3_hash_buf block_state = (*s).block_state; return block_state.fst; } -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a) +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_hash_alg a) { KRML_CHECK_SIZE(sizeof (uint8_t), block_len(a)); uint8_t *buf0 = (uint8_t *)KRML_HOST_CALLOC(block_len(a), sizeof (uint8_t)); - uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); - Hacl_Streaming_Keccak_hash_buf block_state = { .fst = a, .snd = buf }; - Hacl_Streaming_Keccak_state - s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_Keccak_state - *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC(25U, sizeof (uint64_t)); + Hacl_Hash_SHA3_hash_buf block_state = { .fst = a, .snd = buf }; + Hacl_Hash_SHA3_state_t + s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)0U }; + Hacl_Hash_SHA3_state_t + *p = (Hacl_Hash_SHA3_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_SHA3_state_t)); p[0U] = s; uint64_t *s1 = block_state.snd; - memset(s1, 0U, (uint32_t)25U * sizeof (uint64_t)); + memset(s1, 0U, 25U * sizeof (uint64_t)); return p; } -void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s) +void Hacl_Hash_SHA3_free(Hacl_Hash_SHA3_state_t *state) { - Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Hash_SHA3_state_t scrut = *state; uint8_t *buf = scrut.buf; - Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; - uint64_t *s1 = block_state.snd; - KRML_HOST_FREE(s1); - KRML_HOST_FREE(buf); + Hacl_Hash_SHA3_hash_buf block_state = scrut.block_state; + uint64_t *s = block_state.snd; KRML_HOST_FREE(s); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); } -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0) +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_copy(Hacl_Hash_SHA3_state_t *state) { - Hacl_Streaming_Keccak_state scrut0 = *s0; - Hacl_Streaming_Keccak_hash_buf block_state0 = scrut0.block_state; + Hacl_Hash_SHA3_state_t scrut0 = *state; + Hacl_Hash_SHA3_hash_buf block_state0 = scrut0.block_state; uint8_t *buf0 = scrut0.buf; uint64_t total_len0 = scrut0.total_len; Spec_Hash_Definitions_hash_alg i = block_state0.fst; KRML_CHECK_SIZE(sizeof (uint8_t), block_len(i)); uint8_t *buf1 = (uint8_t *)KRML_HOST_CALLOC(block_len(i), sizeof (uint8_t)); memcpy(buf1, buf0, block_len(i) * sizeof (uint8_t)); - uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); - Hacl_Streaming_Keccak_hash_buf block_state = { .fst = i, .snd = buf }; + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC(25U, sizeof (uint64_t)); + Hacl_Hash_SHA3_hash_buf block_state = { .fst = i, .snd = buf }; hash_buf2 scrut = { .fst = block_state0, .snd = block_state }; uint64_t *s_dst = scrut.snd.snd; uint64_t *s_src = scrut.fst.snd; - memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); - Hacl_Streaming_Keccak_state + memcpy(s_dst, s_src, 25U * sizeof (uint64_t)); + Hacl_Hash_SHA3_state_t s = { .block_state = block_state, .buf = buf1, .total_len = total_len0 }; - Hacl_Streaming_Keccak_state - *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + Hacl_Hash_SHA3_state_t + *p = (Hacl_Hash_SHA3_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_SHA3_state_t)); p[0U] = s; return p; } -void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s) +void Hacl_Hash_SHA3_reset(Hacl_Hash_SHA3_state_t *state) { - Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Hash_SHA3_state_t scrut = *state; uint8_t *buf = scrut.buf; - Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; - uint64_t *s1 = block_state.snd; - memset(s1, 0U, (uint32_t)25U * sizeof (uint64_t)); - Hacl_Streaming_Keccak_state - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + Hacl_Hash_SHA3_hash_buf block_state = scrut.block_state; + Spec_Hash_Definitions_hash_alg i = block_state.fst; + KRML_MAYBE_UNUSED_VAR(i); + uint64_t *s = block_state.snd; + memset(s, 0U, 25U * sizeof (uint64_t)); + Hacl_Hash_SHA3_state_t + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len) +Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_Keccak_state s = *p; - Hacl_Streaming_Keccak_hash_buf block_state = s.block_state; + Hacl_Hash_SHA3_state_t s = *state; + Hacl_Hash_SHA3_hash_buf block_state = s.block_state; uint64_t total_len = s.total_len; Spec_Hash_Definitions_hash_alg i = block_state.fst; - if ((uint64_t)len > (uint64_t)0xFFFFFFFFFFFFFFFFU - total_len) + if ((uint64_t)chunk_len > 0xFFFFFFFFFFFFFFFFULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)block_len(i) == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)block_len(i) == 0ULL && total_len > 0ULL) { sz = block_len(i); } @@ -255,14 +642,14 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint { sz = (uint32_t)(total_len % (uint64_t)block_len(i)); } - if (len <= block_len(i) - sz) + if (chunk_len <= block_len(i) - sz) { - Hacl_Streaming_Keccak_state s1 = *p; - Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + Hacl_Hash_SHA3_state_t s1 = *state; + Hacl_Hash_SHA3_hash_buf block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)block_len(i) == 0ULL && total_len1 > 0ULL) { sz1 = block_len(i); } @@ -271,26 +658,20 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = - ( - (Hacl_Streaming_Keccak_state){ - .block_state = block_state1, - .buf = buf, - .total_len = total_len2 - } - ); + ((Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, .total_len = total_len2 }); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_Keccak_state s1 = *p; - Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + Hacl_Hash_SHA3_state_t s1 = *state; + Hacl_Hash_SHA3_hash_buf block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)block_len(i) == 0ULL && total_len1 > 0ULL) { sz1 = block_len(i); } @@ -298,52 +679,52 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint { sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); } uint32_t ite; - if ((uint64_t)len % (uint64_t)block_len(i) == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)block_len(i) == 0ULL && (uint64_t)chunk_len > 0ULL) { ite = block_len(i); } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)block_len(i)); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)block_len(i)); } - uint32_t n_blocks = (len - ite) / block_len(i); + uint32_t n_blocks = (chunk_len - ite) / block_len(i); uint32_t data1_len = n_blocks * block_len(i); - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( - (Hacl_Streaming_Keccak_state){ + (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { uint32_t diff = block_len(i) - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_Keccak_state s1 = *p; - Hacl_Streaming_Keccak_hash_buf block_state10 = s1.block_state; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_SHA3_state_t s1 = *state; + Hacl_Hash_SHA3_hash_buf block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)block_len(i) == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)block_len(i) == 0ULL && total_len10 > 0ULL) { sz10 = block_len(i); } @@ -352,23 +733,23 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint sz10 = (uint32_t)(total_len10 % (uint64_t)block_len(i)); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( - (Hacl_Streaming_Keccak_state){ + (Hacl_Hash_SHA3_state_t){ .block_state = block_state10, .buf = buf0, .total_len = total_len2 } ); - Hacl_Streaming_Keccak_state s10 = *p; - Hacl_Streaming_Keccak_hash_buf block_state1 = s10.block_state; + Hacl_Hash_SHA3_state_t s10 = *state; + Hacl_Hash_SHA3_hash_buf block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)block_len(i) == 0ULL && total_len1 > 0ULL) { sz1 = block_len(i); } @@ -376,7 +757,7 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint { sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; @@ -385,35 +766,35 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint uint32_t ite; if ( - (uint64_t)(len - diff) + (uint64_t)(chunk_len - diff) % (uint64_t)block_len(i) - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U + == 0ULL + && (uint64_t)(chunk_len - diff) > 0ULL ) { ite = block_len(i); } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)block_len(i)); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)block_len(i)); } - uint32_t n_blocks = (len - diff - ite) / block_len(i); + uint32_t n_blocks = (chunk_len - diff - ite) / block_len(i); uint32_t data1_len = n_blocks * block_len(i); - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; - Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data11, data1_len / block_len(a1)); + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( - (Hacl_Streaming_Keccak_state){ + (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } @@ -421,19 +802,19 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint } static void -finish_( +digest_( Spec_Hash_Definitions_hash_alg a, - Hacl_Streaming_Keccak_state *p, - uint8_t *dst, + Hacl_Hash_SHA3_state_t *state, + uint8_t *output, uint32_t l ) { - Hacl_Streaming_Keccak_state scrut0 = *p; - Hacl_Streaming_Keccak_hash_buf block_state = scrut0.block_state; + Hacl_Hash_SHA3_state_t scrut0 = *state; + Hacl_Hash_SHA3_hash_buf block_state = scrut0.block_state; uint8_t *buf_ = scrut0.buf; uint64_t total_len = scrut0.total_len; uint32_t r; - if (total_len % (uint64_t)block_len(a) == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)block_len(a) == 0ULL && total_len > 0ULL) { r = block_len(a); } @@ -443,25 +824,25 @@ finish_( } uint8_t *buf_1 = buf_; uint64_t buf[25U] = { 0U }; - Hacl_Streaming_Keccak_hash_buf tmp_block_state = { .fst = a, .snd = buf }; + Hacl_Hash_SHA3_hash_buf tmp_block_state = { .fst = a, .snd = buf }; hash_buf2 scrut = { .fst = block_state, .snd = tmp_block_state }; uint64_t *s_dst = scrut.snd.snd; uint64_t *s_src = scrut.fst.snd; - memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); - uint32_t ite0; - if (r % block_len(a) == (uint32_t)0U && r > (uint32_t)0U) + memcpy(s_dst, s_src, 25U * sizeof (uint64_t)); + uint32_t ite; + if (r % block_len(a) == 0U && r > 0U) { - ite0 = block_len(a); + ite = block_len(a); } else { - ite0 = r % block_len(a); + ite = r % block_len(a); } - uint8_t *buf_last = buf_1 + r - ite0; + uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; Spec_Hash_Definitions_hash_alg a1 = tmp_block_state.fst; uint64_t *s0 = tmp_block_state.snd; - Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, (uint32_t)0U / block_len(a1)); + Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, 0U / block_len(a1)); Spec_Hash_Definitions_hash_alg a10 = tmp_block_state.fst; uint64_t *s1 = tmp_block_state.snd; Hacl_Hash_SHA3_update_last_sha3(a10, s1, buf_last, r); @@ -469,356 +850,1521 @@ finish_( uint64_t *s = tmp_block_state.snd; if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) { - uint32_t ite; - if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + for (uint32_t i0 = 0U; i0 < l / block_len(a11); i0++) { - ite = l; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b0 = output; + uint8_t *uu____0 = hbuf; + memcpy(b0 + i0 * block_len(a11), uu____0, block_len(a11) * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____1 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____1 << 1U | uu____1 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r1 = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____2 = current; + s[_Y] = uu____2 << r1 | uu____2 >> (64U - r1); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } } - else + uint32_t remOut = l % block_len(a11); + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) { - ite = hash_len(a11); + store64_le(hbuf + i * 8U, ws[i]); } - Hacl_Impl_SHA3_squeeze(s, block_len(a11), ite, dst); + memcpy(output + l - remOut, hbuf, remOut * sizeof (uint8_t)); return; } - Hacl_Impl_SHA3_squeeze(s, block_len(a11), hash_len(a11), dst); + for (uint32_t i0 = 0U; i0 < hash_len(a11) / block_len(a11); i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b0 = output; + uint8_t *uu____3 = hbuf; + memcpy(b0 + i0 * block_len(a11), uu____3, block_len(a11) * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____4 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____4 << 1U | uu____4 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r1 = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____5 = current; + s[_Y] = uu____5 << r1 | uu____5 >> (64U - r1); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = hash_len(a11) % block_len(a11); + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *uu____6 = hbuf; + memcpy(output + hash_len(a11) - remOut, uu____6, remOut * sizeof (uint8_t)); } Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst) +Hacl_Hash_SHA3_digest(Hacl_Hash_SHA3_state_t *state, uint8_t *output) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(state); if (a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256) { return Hacl_Streaming_Types_InvalidAlgorithm; } - finish_(a1, s, dst, hash_len(a1)); + digest_(a1, state, output, hash_len(a1)); return Hacl_Streaming_Types_Success; } Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l) +Hacl_Hash_SHA3_squeeze(Hacl_Hash_SHA3_state_t *s, uint8_t *dst, uint32_t l) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(s); if (!(a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256)) { return Hacl_Streaming_Types_InvalidAlgorithm; } - if (l == (uint32_t)0U) + if (l == 0U) { return Hacl_Streaming_Types_InvalidLength; } - finish_(a1, s, dst, l); + digest_(a1, s, dst, l); return Hacl_Streaming_Types_Success; } -uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s) +uint32_t Hacl_Hash_SHA3_block_len(Hacl_Hash_SHA3_state_t *s) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(s); return block_len(a1); } -uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s) +uint32_t Hacl_Hash_SHA3_hash_len(Hacl_Hash_SHA3_state_t *s) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(s); return hash_len(a1); } -bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s) +bool Hacl_Hash_SHA3_is_shake(Hacl_Hash_SHA3_state_t *s) { - Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Hash_SHA3_get_alg(s); return uu____0 == Spec_Hash_Definitions_Shake128 || uu____0 == Spec_Hash_Definitions_Shake256; } -void -Hacl_SHA3_shake128_hacl( - uint32_t inputByteLen, - uint8_t *input, - uint32_t outputByteLen, - uint8_t *output -) +void Hacl_Hash_SHA3_absorb_inner_32(uint32_t rateInBytes, uint8_t *b, uint64_t *s) { - Hacl_Impl_SHA3_keccak((uint32_t)1344U, - (uint32_t)256U, - inputByteLen, - input, - (uint8_t)0x1FU, - outputByteLen, - output); + KRML_MAYBE_UNUSED_VAR(rateInBytes); + uint64_t ws[32U] = { 0U }; + uint8_t *b1 = b; + uint64_t u = load64_le(b1); + ws[0U] = u; + uint64_t u0 = load64_le(b1 + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b1 + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b1 + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b1 + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b1 + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b1 + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b1 + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b1 + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b1 + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b1 + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b1 + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b1 + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b1 + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b1 + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b1 + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b1 + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b1 + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b1 + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b1 + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b1 + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b1 + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b1 + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b1 + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b1 + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b1 + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b1 + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b1 + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b1 + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b1 + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b1 + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b1 + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws[i]; + } + for (uint32_t i0 = 0U; i0 < 24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i1, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i1 + 1U) % 5U]; + uint64_t _D = _C[(i1 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i1 + 5U * i] = s[i1 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } } void -Hacl_SHA3_shake256_hacl( - uint32_t inputByteLen, - uint8_t *input, +Hacl_Hash_SHA3_shake128( + uint8_t *output, uint32_t outputByteLen, - uint8_t *output + uint8_t *input, + uint32_t inputByteLen ) { - Hacl_Impl_SHA3_keccak((uint32_t)1088U, - (uint32_t)512U, - inputByteLen, - input, - (uint8_t)0x1FU, - outputByteLen, - output); + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 168U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x1FU; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < outputByteLen / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = outputByteLen % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + outputByteLen - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void +Hacl_Hash_SHA3_shake256( + uint8_t *output, + uint32_t outputByteLen, + uint8_t *input, + uint32_t inputByteLen +) { - Hacl_Impl_SHA3_keccak((uint32_t)1152U, - (uint32_t)448U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)28U, - output); + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 136U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x1FU; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < outputByteLen / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = outputByteLen % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + outputByteLen - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_224(uint8_t *output, uint8_t *input, uint32_t inputByteLen) { - Hacl_Impl_SHA3_keccak((uint32_t)1088U, - (uint32_t)512U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)32U, - output); + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 144U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x06U; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < 28U / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = 28U % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + 28U - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_256(uint8_t *output, uint8_t *input, uint32_t inputByteLen) { - Hacl_Impl_SHA3_keccak((uint32_t)832U, - (uint32_t)768U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)48U, - output); + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 136U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x06U; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < 32U / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = 32U % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + 32U - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_384(uint8_t *output, uint8_t *input, uint32_t inputByteLen) { - Hacl_Impl_SHA3_keccak((uint32_t)576U, - (uint32_t)1024U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)64U, - output); -} - -static const -uint32_t -keccak_rotc[24U] = + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 104U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) { - (uint32_t)1U, (uint32_t)3U, (uint32_t)6U, (uint32_t)10U, (uint32_t)15U, (uint32_t)21U, - (uint32_t)28U, (uint32_t)36U, (uint32_t)45U, (uint32_t)55U, (uint32_t)2U, (uint32_t)14U, - (uint32_t)27U, (uint32_t)41U, (uint32_t)56U, (uint32_t)8U, (uint32_t)25U, (uint32_t)43U, - (uint32_t)62U, (uint32_t)18U, (uint32_t)39U, (uint32_t)61U, (uint32_t)20U, (uint32_t)44U - }; - -static const -uint32_t -keccak_piln[24U] = + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x06U; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) { - (uint32_t)10U, (uint32_t)7U, (uint32_t)11U, (uint32_t)17U, (uint32_t)18U, (uint32_t)3U, - (uint32_t)5U, (uint32_t)16U, (uint32_t)8U, (uint32_t)21U, (uint32_t)24U, (uint32_t)4U, - (uint32_t)15U, (uint32_t)23U, (uint32_t)19U, (uint32_t)13U, (uint32_t)12U, (uint32_t)2U, - (uint32_t)20U, (uint32_t)14U, (uint32_t)22U, (uint32_t)9U, (uint32_t)6U, (uint32_t)1U - }; - -static const -uint64_t -keccak_rndc[24U] = - { - (uint64_t)0x0000000000000001U, (uint64_t)0x0000000000008082U, (uint64_t)0x800000000000808aU, - (uint64_t)0x8000000080008000U, (uint64_t)0x000000000000808bU, (uint64_t)0x0000000080000001U, - (uint64_t)0x8000000080008081U, (uint64_t)0x8000000000008009U, (uint64_t)0x000000000000008aU, - (uint64_t)0x0000000000000088U, (uint64_t)0x0000000080008009U, (uint64_t)0x000000008000000aU, - (uint64_t)0x000000008000808bU, (uint64_t)0x800000000000008bU, (uint64_t)0x8000000000008089U, - (uint64_t)0x8000000000008003U, (uint64_t)0x8000000000008002U, (uint64_t)0x8000000000000080U, - (uint64_t)0x000000000000800aU, (uint64_t)0x800000008000000aU, (uint64_t)0x8000000080008081U, - (uint64_t)0x8000000000008080U, (uint64_t)0x0000000080000001U, (uint64_t)0x8000000080008008U - }; - -void Hacl_Impl_SHA3_state_permute(uint64_t *s) -{ - for (uint32_t i0 = (uint32_t)0U; i0 < (uint32_t)24U; i0++) + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < 48U / rateInBytes1; i0++) { - uint64_t _C[5U] = { 0U }; - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - _C[i] = - s[i - + (uint32_t)0U] - ^ - (s[i - + (uint32_t)5U] - ^ (s[i + (uint32_t)10U] ^ (s[i + (uint32_t)15U] ^ s[i + (uint32_t)20U])));); - KRML_MAYBE_FOR5(i1, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - uint64_t uu____0 = _C[(i1 + (uint32_t)1U) % (uint32_t)5U]; - uint64_t - _D = - _C[(i1 + (uint32_t)4U) - % (uint32_t)5U] - ^ (uu____0 << (uint32_t)1U | uu____0 >> (uint32_t)63U); - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - s[i1 + (uint32_t)5U * i] = s[i1 + (uint32_t)5U * i] ^ _D;);); - uint64_t x = s[1U]; - uint64_t current = x; - for (uint32_t i = (uint32_t)0U; i < (uint32_t)24U; i++) + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) { - uint32_t _Y = keccak_piln[i]; - uint32_t r = keccak_rotc[i]; - uint64_t temp = s[_Y]; - uint64_t uu____1 = current; - s[_Y] = uu____1 << r | uu____1 >> ((uint32_t)64U - r); - current = temp; + store64_le(hbuf + i * 8U, ws[i]); } - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - uint64_t - v0 = - s[(uint32_t)0U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)1U + (uint32_t)5U * i] & s[(uint32_t)2U + (uint32_t)5U * i]); - uint64_t - v1 = - s[(uint32_t)1U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)2U + (uint32_t)5U * i] & s[(uint32_t)3U + (uint32_t)5U * i]); - uint64_t - v2 = - s[(uint32_t)2U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)3U + (uint32_t)5U * i] & s[(uint32_t)4U + (uint32_t)5U * i]); - uint64_t - v3 = - s[(uint32_t)3U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)4U + (uint32_t)5U * i] & s[(uint32_t)0U + (uint32_t)5U * i]); - uint64_t - v4 = - s[(uint32_t)4U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)0U + (uint32_t)5U * i] & s[(uint32_t)1U + (uint32_t)5U * i]); - s[(uint32_t)0U + (uint32_t)5U * i] = v0; - s[(uint32_t)1U + (uint32_t)5U * i] = v1; - s[(uint32_t)2U + (uint32_t)5U * i] = v2; - s[(uint32_t)3U + (uint32_t)5U * i] = v3; - s[(uint32_t)4U + (uint32_t)5U * i] = v4;); - uint64_t c = keccak_rndc[i0]; - s[0U] = s[0U] ^ c; + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = 48U % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); } + memcpy(rb + 48U - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s) +void Hacl_Hash_SHA3_sha3_512(uint8_t *output, uint8_t *input, uint32_t inputByteLen) { - uint8_t block[200U] = { 0U }; - memcpy(block, input, rateInBytes * sizeof (uint8_t)); - for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 72U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) { - uint64_t u = load64_le(block + i * (uint32_t)8U); - uint64_t x = u; - s[i] = s[i] ^ x; + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x06U; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < 64U / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = 64U % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + 64U - remOut, hbuf, remOut * sizeof (uint8_t)); } -static void storeState(uint32_t rateInBytes, uint64_t *s, uint8_t *res) +/** +Allocate state buffer of 200-bytes +*/ +uint64_t *Hacl_Hash_SHA3_state_malloc(void) { - uint8_t block[200U] = { 0U }; - for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) - { - uint64_t sj = s[i]; - store64_le(block + i * (uint32_t)8U, sj); - } - memcpy(res, block, rateInBytes * sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC(25U, sizeof (uint64_t)); + return buf; } -void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s) +/** +Free state buffer +*/ +void Hacl_Hash_SHA3_state_free(uint64_t *s) { - Hacl_Impl_SHA3_loadState(rateInBytes, block, s); - Hacl_Impl_SHA3_state_permute(s); + KRML_HOST_FREE(s); } -static void -absorb( - uint64_t *s, - uint32_t rateInBytes, - uint32_t inputByteLen, - uint8_t *input, - uint8_t delimitedSuffix -) +/** +Absorb number of input blocks and write the output state + + This function is intended to receive a hash state and input buffer. + It processes an input of multiple of 168-bytes (SHAKE128 block size), + any additional bytes of final partial block are ignored. + + The argument `state` (IN/OUT) points to hash state, i.e., uint64_t[25] + The argument `input` (IN) points to `inputByteLen` bytes of valid memory, + i.e., uint8_t[inputByteLen] +*/ +void +Hacl_Hash_SHA3_shake128_absorb_nblocks(uint64_t *state, uint8_t *input, uint32_t inputByteLen) { - uint32_t n_blocks = inputByteLen / rateInBytes; - uint32_t rem = inputByteLen % rateInBytes; - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) - { - uint8_t *block = input + i * rateInBytes; - Hacl_Impl_SHA3_absorb_inner(rateInBytes, block, s); - } - uint8_t *last = input + n_blocks * rateInBytes; - uint8_t lastBlock_[200U] = { 0U }; - uint8_t *lastBlock = lastBlock_; - memcpy(lastBlock, last, rem * sizeof (uint8_t)); - lastBlock[rem] = delimitedSuffix; - Hacl_Impl_SHA3_loadState(rateInBytes, lastBlock, s); - if (!((delimitedSuffix & (uint8_t)0x80U) == (uint8_t)0U) && rem == rateInBytes - (uint32_t)1U) - { - Hacl_Impl_SHA3_state_permute(s); - } - uint8_t nextBlock_[200U] = { 0U }; - uint8_t *nextBlock = nextBlock_; - nextBlock[rateInBytes - (uint32_t)1U] = (uint8_t)0x80U; - Hacl_Impl_SHA3_loadState(rateInBytes, nextBlock, s); - Hacl_Impl_SHA3_state_permute(s); + for (uint32_t i = 0U; i < inputByteLen / 168U; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = input; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * 168U, 168U * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(168U, b_, state); + } } +/** +Absorb a final partial block of input and write the output state + + This function is intended to receive a hash state and input buffer. + It processes a sequence of bytes at end of input buffer that is less + than 168-bytes (SHAKE128 block size), + any bytes of full blocks at start of input buffer are ignored. + + The argument `state` (IN/OUT) points to hash state, i.e., uint64_t[25] + The argument `input` (IN) points to `inputByteLen` bytes of valid memory, + i.e., uint8_t[inputByteLen] + + Note: Full size of input buffer must be passed to `inputByteLen` including + the number of full-block bytes at start of input buffer that are ignored +*/ void -Hacl_Impl_SHA3_squeeze( - uint64_t *s, - uint32_t rateInBytes, - uint32_t outputByteLen, - uint8_t *output -) +Hacl_Hash_SHA3_shake128_absorb_final(uint64_t *state, uint8_t *input, uint32_t inputByteLen) { - uint32_t outBlocks = outputByteLen / rateInBytes; - uint32_t remOut = outputByteLen % rateInBytes; - uint8_t *last = output + outputByteLen - remOut; - uint8_t *blocks = output; - for (uint32_t i = (uint32_t)0U; i < outBlocks; i++) + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % 168U; + uint8_t *b00 = input; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % 168U] = 0x1FU; + uint64_t ws[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) { - storeState(rateInBytes, s, blocks + i * rateInBytes); - Hacl_Impl_SHA3_state_permute(s); + state[i] = state[i] ^ ws[i]; } - storeState(remOut, s, last); + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[167U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(168U, b3, state); } +/** +Squeeze a hash state to output buffer + + This function is intended to receive a hash state and output buffer. + It produces an output of multiple of 168-bytes (SHAKE128 block size), + any additional bytes of final partial block are ignored. + + The argument `state` (IN) points to hash state, i.e., uint64_t[25] + The argument `output` (OUT) points to `outputByteLen` bytes of valid memory, + i.e., uint8_t[outputByteLen] +*/ void -Hacl_Impl_SHA3_keccak( - uint32_t rate, - uint32_t capacity, - uint32_t inputByteLen, - uint8_t *input, - uint8_t delimitedSuffix, - uint32_t outputByteLen, - uint8_t *output +Hacl_Hash_SHA3_shake128_squeeze_nblocks( + uint64_t *state, + uint8_t *output, + uint32_t outputByteLen ) { - uint32_t rateInBytes = rate / (uint32_t)8U; - uint64_t s[25U] = { 0U }; - absorb(s, rateInBytes, inputByteLen, input, delimitedSuffix); - Hacl_Impl_SHA3_squeeze(s, rateInBytes, outputByteLen, output); + for (uint32_t i0 = 0U; i0 < outputByteLen / 168U; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, state, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b0 = output; + memcpy(b0 + i0 * 168U, hbuf, 168U * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = + state[i + + 0U] + ^ (state[i + 5U] ^ (state[i + 10U] ^ (state[i + 15U] ^ state[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, state[i2 + 5U * i] = state[i2 + 5U * i] ^ _D;);); + uint64_t x = state[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = state[_Y]; + uint64_t uu____1 = current; + state[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = state[0U + 5U * i] ^ (~state[1U + 5U * i] & state[2U + 5U * i]); + uint64_t v1 = state[1U + 5U * i] ^ (~state[2U + 5U * i] & state[3U + 5U * i]); + uint64_t v2 = state[2U + 5U * i] ^ (~state[3U + 5U * i] & state[4U + 5U * i]); + uint64_t v3 = state[3U + 5U * i] ^ (~state[4U + 5U * i] & state[0U + 5U * i]); + uint64_t v4 = state[4U + 5U * i] ^ (~state[0U + 5U * i] & state[1U + 5U * i]); + state[0U + 5U * i] = v0; + state[1U + 5U * i] = v1; + state[2U + 5U * i] = v2; + state[3U + 5U * i] = v3; + state[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + state[0U] = state[0U] ^ c; + } + } } diff --git a/Modules/_hacl/Hacl_Hash_SHA3.h b/Modules/_hacl/Hacl_Hash_SHA3.h index 681b6af4a80e77..4d85bb6902cc5b 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/Hacl_Hash_SHA3.h @@ -31,95 +31,137 @@ extern "C" { #endif #include +#include "python_hacl_namespaces.h" #include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" #include "Hacl_Streaming_Types.h" -typedef struct Hacl_Streaming_Keccak_hash_buf_s +typedef struct Hacl_Hash_SHA3_hash_buf_s { Spec_Hash_Definitions_hash_alg fst; uint64_t *snd; } -Hacl_Streaming_Keccak_hash_buf; +Hacl_Hash_SHA3_hash_buf; -typedef struct Hacl_Streaming_Keccak_state_s +typedef struct Hacl_Hash_SHA3_state_t_s { - Hacl_Streaming_Keccak_hash_buf block_state; + Hacl_Hash_SHA3_hash_buf block_state; uint8_t *buf; uint64_t total_len; } -Hacl_Streaming_Keccak_state; +Hacl_Hash_SHA3_state_t; -Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s); +Spec_Hash_Definitions_hash_alg Hacl_Hash_SHA3_get_alg(Hacl_Hash_SHA3_state_t *s); -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a); +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_hash_alg a); -void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s); +void Hacl_Hash_SHA3_free(Hacl_Hash_SHA3_state_t *state); -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0); +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_copy(Hacl_Hash_SHA3_state_t *state); -void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s); +void Hacl_Hash_SHA3_reset(Hacl_Hash_SHA3_state_t *state); Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len); +Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t chunk_len); Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst); +Hacl_Hash_SHA3_digest(Hacl_Hash_SHA3_state_t *state, uint8_t *output); Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l); +Hacl_Hash_SHA3_squeeze(Hacl_Hash_SHA3_state_t *s, uint8_t *dst, uint32_t l); -uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s); +uint32_t Hacl_Hash_SHA3_block_len(Hacl_Hash_SHA3_state_t *s); -uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s); +uint32_t Hacl_Hash_SHA3_hash_len(Hacl_Hash_SHA3_state_t *s); -bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s); +bool Hacl_Hash_SHA3_is_shake(Hacl_Hash_SHA3_state_t *s); + +void Hacl_Hash_SHA3_absorb_inner_32(uint32_t rateInBytes, uint8_t *b, uint64_t *s); void -Hacl_SHA3_shake128_hacl( - uint32_t inputByteLen, - uint8_t *input, +Hacl_Hash_SHA3_shake128( + uint8_t *output, uint32_t outputByteLen, - uint8_t *output + uint8_t *input, + uint32_t inputByteLen ); void -Hacl_SHA3_shake256_hacl( - uint32_t inputByteLen, - uint8_t *input, +Hacl_Hash_SHA3_shake256( + uint8_t *output, uint32_t outputByteLen, - uint8_t *output + uint8_t *input, + uint32_t inputByteLen ); -void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_224(uint8_t *output, uint8_t *input, uint32_t inputByteLen); -void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_256(uint8_t *output, uint8_t *input, uint32_t inputByteLen); -void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_384(uint8_t *output, uint8_t *input, uint32_t inputByteLen); -void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_512(uint8_t *output, uint8_t *input, uint32_t inputByteLen); -void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s); +/** +Allocate state buffer of 200-bytes +*/ +uint64_t *Hacl_Hash_SHA3_state_malloc(void); +/** +Free state buffer +*/ +void Hacl_Hash_SHA3_state_free(uint64_t *s); + +/** +Absorb number of input blocks and write the output state + + This function is intended to receive a hash state and input buffer. + It processes an input of multiple of 168-bytes (SHAKE128 block size), + any additional bytes of final partial block are ignored. + + The argument `state` (IN/OUT) points to hash state, i.e., uint64_t[25] + The argument `input` (IN) points to `inputByteLen` bytes of valid memory, + i.e., uint8_t[inputByteLen] +*/ void -Hacl_Impl_SHA3_squeeze( - uint64_t *s, - uint32_t rateInBytes, - uint32_t outputByteLen, - uint8_t *output -); +Hacl_Hash_SHA3_shake128_absorb_nblocks(uint64_t *state, uint8_t *input, uint32_t inputByteLen); +/** +Absorb a final partial block of input and write the output state + + This function is intended to receive a hash state and input buffer. + It processes a sequence of bytes at end of input buffer that is less + than 168-bytes (SHAKE128 block size), + any bytes of full blocks at start of input buffer are ignored. + + The argument `state` (IN/OUT) points to hash state, i.e., uint64_t[25] + The argument `input` (IN) points to `inputByteLen` bytes of valid memory, + i.e., uint8_t[inputByteLen] + + Note: Full size of input buffer must be passed to `inputByteLen` including + the number of full-block bytes at start of input buffer that are ignored +*/ void -Hacl_Impl_SHA3_keccak( - uint32_t rate, - uint32_t capacity, - uint32_t inputByteLen, - uint8_t *input, - uint8_t delimitedSuffix, - uint32_t outputByteLen, - uint8_t *output +Hacl_Hash_SHA3_shake128_absorb_final(uint64_t *state, uint8_t *input, uint32_t inputByteLen); + +/** +Squeeze a hash state to output buffer + + This function is intended to receive a hash state and output buffer. + It produces an output of multiple of 168-bytes (SHAKE128 block size), + any additional bytes of final partial block are ignored. + + The argument `state` (IN) points to hash state, i.e., uint64_t[25] + The argument `output` (OUT) points to `outputByteLen` bytes of valid memory, + i.e., uint8_t[outputByteLen] +*/ +void +Hacl_Hash_SHA3_shake128_squeeze_nblocks( + uint64_t *state, + uint8_t *output, + uint32_t outputByteLen ); #if defined(__cplusplus) diff --git a/Modules/_hacl/Lib_Memzero0.c b/Modules/_hacl/Lib_Memzero0.c new file mode 100644 index 00000000000000..5b1a2f7797db76 --- /dev/null +++ b/Modules/_hacl/Lib_Memzero0.c @@ -0,0 +1,54 @@ +#if defined(__has_include) +#if __has_include("config.h") +#include "config.h" +#endif +#endif + +#ifdef _WIN32 +#include +#endif + +#if (defined(__APPLE__) && defined(__MACH__)) || defined(__linux__) +#define __STDC_WANT_LIB_EXT1__ 1 +#include +#endif + +#if defined(__FreeBSD__) || defined(__NetBSD__) +#include +#endif + +#include +#include +#include +#include + +/* This is now a hand-written header */ +#include "lib_memzero0.h" +#include "krml/internal/target.h" + +/* The F* formalization talks about the number of elements in the array. The C + implementation wants a number of bytes in the array. KaRaMeL is aware of this + and inserts a sizeof multiplication. */ +void Lib_Memzero0_memzero0(void *dst, uint64_t len) { + /* This is safe: karamel checks at run-time (if needed) that all object sizes + fit within a size_t, so the size we receive has been checked at + allocation-time, possibly via KRML_CHECK_SIZE, to fit in a size_t. */ + size_t len_ = (size_t) len; + + #ifdef _WIN32 + SecureZeroMemory(dst, len); + #elif defined(__APPLE__) && defined(__MACH__) + memset_s(dst, len_, 0, len_); + #elif (defined(__linux__) && !defined(LINUX_NO_EXPLICIT_BZERO)) || defined(__FreeBSD__) + explicit_bzero(dst, len_); + #elif defined(__NetBSD__) + explicit_memset(dst, 0, len_); + #else + /* Default implementation for platforms with no particular support. */ + #warning "Your platform does not support any safe implementation of memzero -- consider a pull request!" + volatile unsigned char *volatile dst_ = (volatile unsigned char *volatile) dst; + size_t i = 0U; + while (i < len) + dst_[i++] = 0U; + #endif +} diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h index 3d36d440735530..659745b24265cb 100644 --- a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -1,6 +1,6 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. + Licensed under the Apache 2.0 and MIT Licenses. */ @@ -15,7 +15,7 @@ static inline uint64_t FStar_UInt128_constant_time_carry(uint64_t a, uint64_t b) { - return (a ^ ((a ^ b) | ((a - b) ^ b))) >> (uint32_t)63U; + return (a ^ ((a ^ b) | ((a - b) ^ b))) >> 63U; } static inline uint64_t FStar_UInt128_carry(uint64_t a, uint64_t b) @@ -118,7 +118,7 @@ static inline FStar_UInt128_uint128 FStar_UInt128_lognot(FStar_UInt128_uint128 a return lit; } -static uint32_t FStar_UInt128_u32_64 = (uint32_t)64U; +static uint32_t FStar_UInt128_u32_64 = 64U; static inline uint64_t FStar_UInt128_add_u64_shift_left(uint64_t hi, uint64_t lo, uint32_t s) { @@ -134,7 +134,7 @@ FStar_UInt128_add_u64_shift_left_respec(uint64_t hi, uint64_t lo, uint32_t s) static inline FStar_UInt128_uint128 FStar_UInt128_shift_left_small(FStar_UInt128_uint128 a, uint32_t s) { - if (s == (uint32_t)0U) + if (s == 0U) { return a; } @@ -151,7 +151,7 @@ static inline FStar_UInt128_uint128 FStar_UInt128_shift_left_large(FStar_UInt128_uint128 a, uint32_t s) { FStar_UInt128_uint128 lit; - lit.low = (uint64_t)0U; + lit.low = 0ULL; lit.high = a.low << (s - FStar_UInt128_u32_64); return lit; } @@ -183,7 +183,7 @@ FStar_UInt128_add_u64_shift_right_respec(uint64_t hi, uint64_t lo, uint32_t s) static inline FStar_UInt128_uint128 FStar_UInt128_shift_right_small(FStar_UInt128_uint128 a, uint32_t s) { - if (s == (uint32_t)0U) + if (s == 0U) { return a; } @@ -201,7 +201,7 @@ FStar_UInt128_shift_right_large(FStar_UInt128_uint128 a, uint32_t s) { FStar_UInt128_uint128 lit; lit.low = a.high >> (s - FStar_UInt128_u32_64); - lit.high = (uint64_t)0U; + lit.high = 0ULL; return lit; } @@ -269,7 +269,7 @@ static inline FStar_UInt128_uint128 FStar_UInt128_uint64_to_uint128(uint64_t a) { FStar_UInt128_uint128 lit; lit.low = a; - lit.high = (uint64_t)0U; + lit.high = 0ULL; return lit; } @@ -280,10 +280,10 @@ static inline uint64_t FStar_UInt128_uint128_to_uint64(FStar_UInt128_uint128 a) static inline uint64_t FStar_UInt128_u64_mod_32(uint64_t a) { - return a & (uint64_t)0xffffffffU; + return a & 0xffffffffULL; } -static uint32_t FStar_UInt128_u32_32 = (uint32_t)32U; +static uint32_t FStar_UInt128_u32_32 = 32U; static inline uint64_t FStar_UInt128_u32_combine(uint64_t hi, uint64_t lo) { diff --git a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h index a56c7d613498b7..68bac0b3f0aab1 100644 --- a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h +++ b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h @@ -1,6 +1,6 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. + Licensed under the Apache 2.0 and MIT Licenses. */ @@ -14,16 +14,16 @@ #include "krml/types.h" #include "krml/internal/target.h" -static inline uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) +static KRML_NOINLINE uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) { uint64_t x = a ^ b; - uint64_t minus_x = ~x + (uint64_t)1U; + uint64_t minus_x = ~x + 1ULL; uint64_t x_or_minus_x = x | minus_x; - uint64_t xnx = x_or_minus_x >> (uint32_t)63U; - return xnx - (uint64_t)1U; + uint64_t xnx = x_or_minus_x >> 63U; + return xnx - 1ULL; } -static inline uint64_t FStar_UInt64_gte_mask(uint64_t a, uint64_t b) +static KRML_NOINLINE uint64_t FStar_UInt64_gte_mask(uint64_t a, uint64_t b) { uint64_t x = a; uint64_t y = b; @@ -32,20 +32,20 @@ static inline uint64_t FStar_UInt64_gte_mask(uint64_t a, uint64_t b) uint64_t x_sub_y_xor_y = x_sub_y ^ y; uint64_t q = x_xor_y | x_sub_y_xor_y; uint64_t x_xor_q = x ^ q; - uint64_t x_xor_q_ = x_xor_q >> (uint32_t)63U; - return x_xor_q_ - (uint64_t)1U; + uint64_t x_xor_q_ = x_xor_q >> 63U; + return x_xor_q_ - 1ULL; } -static inline uint32_t FStar_UInt32_eq_mask(uint32_t a, uint32_t b) +static KRML_NOINLINE uint32_t FStar_UInt32_eq_mask(uint32_t a, uint32_t b) { uint32_t x = a ^ b; - uint32_t minus_x = ~x + (uint32_t)1U; + uint32_t minus_x = ~x + 1U; uint32_t x_or_minus_x = x | minus_x; - uint32_t xnx = x_or_minus_x >> (uint32_t)31U; - return xnx - (uint32_t)1U; + uint32_t xnx = x_or_minus_x >> 31U; + return xnx - 1U; } -static inline uint32_t FStar_UInt32_gte_mask(uint32_t a, uint32_t b) +static KRML_NOINLINE uint32_t FStar_UInt32_gte_mask(uint32_t a, uint32_t b) { uint32_t x = a; uint32_t y = b; @@ -54,52 +54,52 @@ static inline uint32_t FStar_UInt32_gte_mask(uint32_t a, uint32_t b) uint32_t x_sub_y_xor_y = x_sub_y ^ y; uint32_t q = x_xor_y | x_sub_y_xor_y; uint32_t x_xor_q = x ^ q; - uint32_t x_xor_q_ = x_xor_q >> (uint32_t)31U; - return x_xor_q_ - (uint32_t)1U; + uint32_t x_xor_q_ = x_xor_q >> 31U; + return x_xor_q_ - 1U; } -static inline uint16_t FStar_UInt16_eq_mask(uint16_t a, uint16_t b) +static KRML_NOINLINE uint16_t FStar_UInt16_eq_mask(uint16_t a, uint16_t b) { - uint16_t x = a ^ b; - uint16_t minus_x = ~x + (uint16_t)1U; - uint16_t x_or_minus_x = x | minus_x; - uint16_t xnx = x_or_minus_x >> (uint32_t)15U; - return xnx - (uint16_t)1U; + uint16_t x = (uint32_t)a ^ (uint32_t)b; + uint16_t minus_x = (uint32_t)~x + 1U; + uint16_t x_or_minus_x = (uint32_t)x | (uint32_t)minus_x; + uint16_t xnx = (uint32_t)x_or_minus_x >> 15U; + return (uint32_t)xnx - 1U; } -static inline uint16_t FStar_UInt16_gte_mask(uint16_t a, uint16_t b) +static KRML_NOINLINE uint16_t FStar_UInt16_gte_mask(uint16_t a, uint16_t b) { uint16_t x = a; uint16_t y = b; - uint16_t x_xor_y = x ^ y; - uint16_t x_sub_y = x - y; - uint16_t x_sub_y_xor_y = x_sub_y ^ y; - uint16_t q = x_xor_y | x_sub_y_xor_y; - uint16_t x_xor_q = x ^ q; - uint16_t x_xor_q_ = x_xor_q >> (uint32_t)15U; - return x_xor_q_ - (uint16_t)1U; + uint16_t x_xor_y = (uint32_t)x ^ (uint32_t)y; + uint16_t x_sub_y = (uint32_t)x - (uint32_t)y; + uint16_t x_sub_y_xor_y = (uint32_t)x_sub_y ^ (uint32_t)y; + uint16_t q = (uint32_t)x_xor_y | (uint32_t)x_sub_y_xor_y; + uint16_t x_xor_q = (uint32_t)x ^ (uint32_t)q; + uint16_t x_xor_q_ = (uint32_t)x_xor_q >> 15U; + return (uint32_t)x_xor_q_ - 1U; } -static inline uint8_t FStar_UInt8_eq_mask(uint8_t a, uint8_t b) +static KRML_NOINLINE uint8_t FStar_UInt8_eq_mask(uint8_t a, uint8_t b) { - uint8_t x = a ^ b; - uint8_t minus_x = ~x + (uint8_t)1U; - uint8_t x_or_minus_x = x | minus_x; - uint8_t xnx = x_or_minus_x >> (uint32_t)7U; - return xnx - (uint8_t)1U; + uint8_t x = (uint32_t)a ^ (uint32_t)b; + uint8_t minus_x = (uint32_t)~x + 1U; + uint8_t x_or_minus_x = (uint32_t)x | (uint32_t)minus_x; + uint8_t xnx = (uint32_t)x_or_minus_x >> 7U; + return (uint32_t)xnx - 1U; } -static inline uint8_t FStar_UInt8_gte_mask(uint8_t a, uint8_t b) +static KRML_NOINLINE uint8_t FStar_UInt8_gte_mask(uint8_t a, uint8_t b) { uint8_t x = a; uint8_t y = b; - uint8_t x_xor_y = x ^ y; - uint8_t x_sub_y = x - y; - uint8_t x_sub_y_xor_y = x_sub_y ^ y; - uint8_t q = x_xor_y | x_sub_y_xor_y; - uint8_t x_xor_q = x ^ q; - uint8_t x_xor_q_ = x_xor_q >> (uint32_t)7U; - return x_xor_q_ - (uint8_t)1U; + uint8_t x_xor_y = (uint32_t)x ^ (uint32_t)y; + uint8_t x_sub_y = (uint32_t)x - (uint32_t)y; + uint8_t x_sub_y_xor_y = (uint32_t)x_sub_y ^ (uint32_t)y; + uint8_t q = (uint32_t)x_xor_y | (uint32_t)x_sub_y_xor_y; + uint8_t x_xor_q = (uint32_t)x ^ (uint32_t)q; + uint8_t x_xor_q_ = (uint32_t)x_xor_q >> 7U; + return (uint32_t)x_xor_q_ - 1U; } diff --git a/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h b/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h index e2b6d62859a5f1..bb736add318aa8 100644 --- a/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h +++ b/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h @@ -1,5 +1,5 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. */ + Licensed under the Apache 2.0 and MIT Licenses. */ #ifndef FSTAR_UINT128_STRUCT_ENDIANNESS_H #define FSTAR_UINT128_STRUCT_ENDIANNESS_H diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index 5a2f94eb2ec8da..fd74d3da684567 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -1,16 +1,16 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. */ + Licensed under the Apache 2.0 and MIT Licenses. */ #ifndef __KRML_TARGET_H #define __KRML_TARGET_H -#include -#include -#include -#include +#include #include #include -#include +#include +#include +#include +#include /* Since KaRaMeL emits the inline keyword unconditionally, we follow the * guidelines at https://gcc.gnu.org/onlinedocs/gcc/Inline.html and make this @@ -57,6 +57,104 @@ # define KRML_HOST_IGNORE(x) (void)(x) #endif +#ifndef KRML_MAYBE_UNUSED_VAR +# define KRML_MAYBE_UNUSED_VAR(x) KRML_HOST_IGNORE(x) +#endif + +#ifndef KRML_MAYBE_UNUSED +# if defined(__GNUC__) +# define KRML_MAYBE_UNUSED __attribute__((unused)) +# else +# define KRML_MAYBE_UNUSED +# endif +#endif + +#ifndef KRML_ATTRIBUTE_TARGET +# if defined(__GNUC__) +# define KRML_ATTRIBUTE_TARGET(x) __attribute__((target(x))) +# else +# define KRML_ATTRIBUTE_TARGET(x) +# endif +#endif + +#ifndef KRML_NOINLINE +# if defined(_MSC_VER) +# define KRML_NOINLINE __declspec(noinline) +# elif defined (__GNUC__) +# define KRML_NOINLINE __attribute__((noinline,unused)) +# elif defined (__SUNPRO_C) +# define KRML_NOINLINE __attribute__((noinline)) +# else +# define KRML_NOINLINE +# warning "The KRML_NOINLINE macro is not defined for this toolchain!" +# warning "The compiler may defeat side-channel resistance with optimizations." +# warning "Please locate target.h and try to fill it out with a suitable definition for this compiler." +# endif +#endif + +#ifndef KRML_MUSTINLINE +# if defined(_MSC_VER) +# define KRML_MUSTINLINE inline __forceinline +# elif defined (__GNUC__) +# define KRML_MUSTINLINE inline __attribute__((always_inline)) +# elif defined (__SUNPRO_C) +# define KRML_MUSTINLINE inline __attribute__((always_inline)) +# else +# define KRML_MUSTINLINE inline +# warning "The KRML_MUSTINLINE macro defaults to plain inline for this toolchain!" +# warning "Please locate target.h and try to fill it out with a suitable definition for this compiler." +# endif +#endif + +#ifndef KRML_PRE_ALIGN +# ifdef _MSC_VER +# define KRML_PRE_ALIGN(X) __declspec(align(X)) +# else +# define KRML_PRE_ALIGN(X) +# endif +#endif + +#ifndef KRML_POST_ALIGN +# ifdef _MSC_VER +# define KRML_POST_ALIGN(X) +# else +# define KRML_POST_ALIGN(X) __attribute__((aligned(X))) +# endif +#endif + +/* MinGW-W64 does not support C11 aligned_alloc, but it supports + * MSVC's _aligned_malloc. + */ +#ifndef KRML_ALIGNED_MALLOC +# ifdef __MINGW32__ +# include <_mingw.h> +# endif +# if ( \ + defined(_MSC_VER) || \ + (defined(__MINGW32__) && defined(__MINGW64_VERSION_MAJOR))) +# define KRML_ALIGNED_MALLOC(X, Y) _aligned_malloc(Y, X) +# else +# define KRML_ALIGNED_MALLOC(X, Y) aligned_alloc(X, Y) +# endif +#endif + +/* Since aligned allocations with MinGW-W64 are done with + * _aligned_malloc (see above), such pointers must be freed with + * _aligned_free. + */ +#ifndef KRML_ALIGNED_FREE +# ifdef __MINGW32__ +# include <_mingw.h> +# endif +# if ( \ + defined(_MSC_VER) || \ + (defined(__MINGW32__) && defined(__MINGW64_VERSION_MAJOR))) +# define KRML_ALIGNED_FREE(X) _aligned_free(X) +# else +# define KRML_ALIGNED_FREE(X) free(X) +# endif +#endif + /* In FStar.Buffer.fst, the size of arrays is uint32_t, but it's a number of * *elements*. Do an ugly, run-time check (some of which KaRaMeL can eliminate). */ @@ -83,184 +181,186 @@ #define KRML_LOOP1(i, n, x) { \ x \ i += n; \ + (void) i; \ } -#define KRML_LOOP2(i, n, x) \ - KRML_LOOP1(i, n, x) \ +#define KRML_LOOP2(i, n, x) \ + KRML_LOOP1(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP3(i, n, x) \ - KRML_LOOP2(i, n, x) \ +#define KRML_LOOP3(i, n, x) \ + KRML_LOOP2(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP4(i, n, x) \ - KRML_LOOP2(i, n, x) \ +#define KRML_LOOP4(i, n, x) \ + KRML_LOOP2(i, n, x) \ KRML_LOOP2(i, n, x) -#define KRML_LOOP5(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP5(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP6(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP6(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP2(i, n, x) -#define KRML_LOOP7(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP7(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP3(i, n, x) -#define KRML_LOOP8(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP8(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP4(i, n, x) -#define KRML_LOOP9(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP9(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP10(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP10(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP2(i, n, x) -#define KRML_LOOP11(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP11(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP3(i, n, x) -#define KRML_LOOP12(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP12(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP4(i, n, x) -#define KRML_LOOP13(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP13(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP5(i, n, x) -#define KRML_LOOP14(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP14(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP6(i, n, x) -#define KRML_LOOP15(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP15(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP7(i, n, x) -#define KRML_LOOP16(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP16(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP8(i, n, x) -#define KRML_UNROLL_FOR(i, z, n, k, x) do { \ - uint32_t i = z; \ - KRML_LOOP##n(i, k, x) \ -} while (0) +#define KRML_UNROLL_FOR(i, z, n, k, x) \ + do { \ + uint32_t i = z; \ + KRML_LOOP##n(i, k, x) \ + } while (0) -#define KRML_ACTUAL_FOR(i, z, n, k, x) \ - do { \ - for (uint32_t i = z; i < n; i += k) { \ - x \ - } \ +#define KRML_ACTUAL_FOR(i, z, n, k, x) \ + do { \ + for (uint32_t i = z; i < n; i += k) { \ + x \ + } \ } while (0) #ifndef KRML_UNROLL_MAX -#define KRML_UNROLL_MAX 16 +# define KRML_UNROLL_MAX 16 #endif /* 1 is the number of loop iterations, i.e. (n - z)/k as evaluated by krml */ #if 0 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR0(i, z, n, k, x) +# define KRML_MAYBE_FOR0(i, z, n, k, x) #else -#define KRML_MAYBE_FOR0(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR0(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 1 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 1, k, x) +# define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 1, k, x) #else -#define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 2 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 2, k, x) +# define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 2, k, x) #else -#define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 3 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 3, k, x) +# define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 3, k, x) #else -#define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 4 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 4, k, x) +# define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 4, k, x) #else -#define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 5 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 5, k, x) +# define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 5, k, x) #else -#define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 6 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 6, k, x) +# define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 6, k, x) #else -#define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 7 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 7, k, x) +# define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 7, k, x) #else -#define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 8 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 8, k, x) +# define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 8, k, x) #else -#define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 9 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 9, k, x) +# define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 9, k, x) #else -#define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 10 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 10, k, x) +# define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 10, k, x) #else -#define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 11 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 11, k, x) +# define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 11, k, x) #else -#define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 12 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 12, k, x) +# define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 12, k, x) #else -#define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 13 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 13, k, x) +# define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 13, k, x) #else -#define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 14 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 14, k, x) +# define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 14, k, x) #else -#define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 15 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 15, k, x) +# define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 15, k, x) #else -#define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 16 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 16, k, x) +# define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 16, k, x) #else -#define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #endif diff --git a/Modules/_hacl/include/krml/lowstar_endianness.h b/Modules/_hacl/include/krml/lowstar_endianness.h index 1aa2ccd644c06f..af6b882cf259cc 100644 --- a/Modules/_hacl/include/krml/lowstar_endianness.h +++ b/Modules/_hacl/include/krml/lowstar_endianness.h @@ -1,5 +1,5 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. */ + Licensed under the Apache 2.0 and MIT Licenses. */ #ifndef __LOWSTAR_ENDIANNESS_H #define __LOWSTAR_ENDIANNESS_H diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2b.h b/Modules/_hacl/internal/Hacl_Hash_Blake2b.h new file mode 100644 index 00000000000000..8ee70282f4e4de --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2b.h @@ -0,0 +1,78 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_Blake2b_H +#define __internal_Hacl_Hash_Blake2b_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "../Hacl_Hash_Blake2b.h" + +typedef struct Hacl_Hash_Blake2b_params_and_key_s +{ + Hacl_Hash_Blake2b_blake2_params *fst; + uint8_t *snd; +} +Hacl_Hash_Blake2b_params_and_key; + +void Hacl_Hash_Blake2b_init(uint64_t *hash, uint32_t kk, uint32_t nn); + +void +Hacl_Hash_Blake2b_update_multi( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks, + uint32_t nb +); + +void +Hacl_Hash_Blake2b_update_last( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + bool last_node, + FStar_UInt128_uint128 prev, + uint32_t rem, + uint8_t *d +); + +void Hacl_Hash_Blake2b_finish(uint32_t nn, uint8_t *output, uint64_t *hash); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_Blake2b_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h b/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h new file mode 100644 index 00000000000000..ab329b92c3630c --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h @@ -0,0 +1,93 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_Blake2b_Simd256_H +#define __internal_Hacl_Hash_Blake2b_Simd256_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "../Hacl_Hash_Blake2b_Simd256.h" +#include "libintvector.h" + +void +Hacl_Hash_Blake2b_Simd256_init(Lib_IntVector_Intrinsics_vec256 *hash, uint32_t kk, uint32_t nn); + +void +Hacl_Hash_Blake2b_Simd256_update_multi( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks, + uint32_t nb +); + +void +Hacl_Hash_Blake2b_Simd256_update_last( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + bool last_node, + FStar_UInt128_uint128 prev, + uint32_t rem, + uint8_t *d +); + +void +Hacl_Hash_Blake2b_Simd256_finish( + uint32_t nn, + uint8_t *output, + Lib_IntVector_Intrinsics_vec256 *hash +); + +void +Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32( + Lib_IntVector_Intrinsics_vec256 *st, + uint64_t *st32 +); + +void +Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32( + uint64_t *st32, + Lib_IntVector_Intrinsics_vec256 *st +); + +Lib_IntVector_Intrinsics_vec256 *Hacl_Hash_Blake2b_Simd256_malloc_with_key(void); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_Blake2b_Simd256_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2s.h b/Modules/_hacl/internal/Hacl_Hash_Blake2s.h new file mode 100644 index 00000000000000..6494075b60a25b --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2s.h @@ -0,0 +1,72 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_Blake2s_H +#define __internal_Hacl_Hash_Blake2s_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "../Hacl_Hash_Blake2s.h" + +void Hacl_Hash_Blake2s_init(uint32_t *hash, uint32_t kk, uint32_t nn); + +void +Hacl_Hash_Blake2s_update_multi( + uint32_t len, + uint32_t *wv, + uint32_t *hash, + uint64_t prev, + uint8_t *blocks, + uint32_t nb +); + +void +Hacl_Hash_Blake2s_update_last( + uint32_t len, + uint32_t *wv, + uint32_t *hash, + bool last_node, + uint64_t prev, + uint32_t rem, + uint8_t *d +); + +void Hacl_Hash_Blake2s_finish(uint32_t nn, uint8_t *output, uint32_t *hash); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_Blake2s_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h b/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h new file mode 100644 index 00000000000000..60c09a67b445b6 --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h @@ -0,0 +1,93 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_Blake2s_Simd128_H +#define __internal_Hacl_Hash_Blake2s_Simd128_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "../Hacl_Hash_Blake2s_Simd128.h" +#include "libintvector.h" + +void +Hacl_Hash_Blake2s_Simd128_init(Lib_IntVector_Intrinsics_vec128 *hash, uint32_t kk, uint32_t nn); + +void +Hacl_Hash_Blake2s_Simd128_update_multi( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint64_t prev, + uint8_t *blocks, + uint32_t nb +); + +void +Hacl_Hash_Blake2s_Simd128_update_last( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + bool last_node, + uint64_t prev, + uint32_t rem, + uint8_t *d +); + +void +Hacl_Hash_Blake2s_Simd128_finish( + uint32_t nn, + uint8_t *output, + Lib_IntVector_Intrinsics_vec128 *hash +); + +void +Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32( + uint32_t *st32, + Lib_IntVector_Intrinsics_vec128 *st +); + +void +Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32( + Lib_IntVector_Intrinsics_vec128 *st, + uint32_t *st32 +); + +Lib_IntVector_Intrinsics_vec128 *Hacl_Hash_Blake2s_Simd128_malloc_with_key(void); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_Blake2s_Simd128_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_MD5.h b/Modules/_hacl/internal/Hacl_Hash_MD5.h index 87ad4cf228d91b..a50ec407f53e39 100644 --- a/Modules/_hacl/internal/Hacl_Hash_MD5.h +++ b/Modules/_hacl/internal/Hacl_Hash_MD5.h @@ -37,21 +37,16 @@ extern "C" { #include "../Hacl_Hash_MD5.h" -void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s); +void Hacl_Hash_MD5_init(uint32_t *s); -void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst); +void Hacl_Hash_MD5_finish(uint32_t *s, uint8_t *dst); -void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); +void Hacl_Hash_MD5_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); void -Hacl_Hash_MD5_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -); - -void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +Hacl_Hash_MD5_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len); + +void Hacl_Hash_MD5_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA1.h b/Modules/_hacl/internal/Hacl_Hash_SHA1.h index d2d9df44c6c14c..b39bad3f3b93e8 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA1.h @@ -37,21 +37,16 @@ extern "C" { #include "../Hacl_Hash_SHA1.h" -void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s); +void Hacl_Hash_SHA1_init(uint32_t *s); -void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst); +void Hacl_Hash_SHA1_finish(uint32_t *s, uint8_t *dst); -void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); +void Hacl_Hash_SHA1_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); void -Hacl_Hash_SHA1_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -); - -void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +Hacl_Hash_SHA1_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len); + +void Hacl_Hash_SHA1_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA2.h b/Modules/_hacl/internal/Hacl_Hash_SHA2.h index 851f7dc60c94c2..0127f4373fb1a1 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA2.h @@ -40,141 +40,121 @@ extern "C" { static const uint32_t -Hacl_Impl_SHA2_Generic_h224[8U] = +Hacl_Hash_SHA2_h224[8U] = { - (uint32_t)0xc1059ed8U, (uint32_t)0x367cd507U, (uint32_t)0x3070dd17U, (uint32_t)0xf70e5939U, - (uint32_t)0xffc00b31U, (uint32_t)0x68581511U, (uint32_t)0x64f98fa7U, (uint32_t)0xbefa4fa4U + 0xc1059ed8U, 0x367cd507U, 0x3070dd17U, 0xf70e5939U, 0xffc00b31U, 0x68581511U, 0x64f98fa7U, + 0xbefa4fa4U }; static const uint32_t -Hacl_Impl_SHA2_Generic_h256[8U] = +Hacl_Hash_SHA2_h256[8U] = { - (uint32_t)0x6a09e667U, (uint32_t)0xbb67ae85U, (uint32_t)0x3c6ef372U, (uint32_t)0xa54ff53aU, - (uint32_t)0x510e527fU, (uint32_t)0x9b05688cU, (uint32_t)0x1f83d9abU, (uint32_t)0x5be0cd19U + 0x6a09e667U, 0xbb67ae85U, 0x3c6ef372U, 0xa54ff53aU, 0x510e527fU, 0x9b05688cU, 0x1f83d9abU, + 0x5be0cd19U }; static const uint64_t -Hacl_Impl_SHA2_Generic_h384[8U] = +Hacl_Hash_SHA2_h384[8U] = { - (uint64_t)0xcbbb9d5dc1059ed8U, (uint64_t)0x629a292a367cd507U, (uint64_t)0x9159015a3070dd17U, - (uint64_t)0x152fecd8f70e5939U, (uint64_t)0x67332667ffc00b31U, (uint64_t)0x8eb44a8768581511U, - (uint64_t)0xdb0c2e0d64f98fa7U, (uint64_t)0x47b5481dbefa4fa4U + 0xcbbb9d5dc1059ed8ULL, 0x629a292a367cd507ULL, 0x9159015a3070dd17ULL, 0x152fecd8f70e5939ULL, + 0x67332667ffc00b31ULL, 0x8eb44a8768581511ULL, 0xdb0c2e0d64f98fa7ULL, 0x47b5481dbefa4fa4ULL }; static const uint64_t -Hacl_Impl_SHA2_Generic_h512[8U] = +Hacl_Hash_SHA2_h512[8U] = { - (uint64_t)0x6a09e667f3bcc908U, (uint64_t)0xbb67ae8584caa73bU, (uint64_t)0x3c6ef372fe94f82bU, - (uint64_t)0xa54ff53a5f1d36f1U, (uint64_t)0x510e527fade682d1U, (uint64_t)0x9b05688c2b3e6c1fU, - (uint64_t)0x1f83d9abfb41bd6bU, (uint64_t)0x5be0cd19137e2179U + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL }; static const uint32_t -Hacl_Impl_SHA2_Generic_k224_256[64U] = +Hacl_Hash_SHA2_k224_256[64U] = { - (uint32_t)0x428a2f98U, (uint32_t)0x71374491U, (uint32_t)0xb5c0fbcfU, (uint32_t)0xe9b5dba5U, - (uint32_t)0x3956c25bU, (uint32_t)0x59f111f1U, (uint32_t)0x923f82a4U, (uint32_t)0xab1c5ed5U, - (uint32_t)0xd807aa98U, (uint32_t)0x12835b01U, (uint32_t)0x243185beU, (uint32_t)0x550c7dc3U, - (uint32_t)0x72be5d74U, (uint32_t)0x80deb1feU, (uint32_t)0x9bdc06a7U, (uint32_t)0xc19bf174U, - (uint32_t)0xe49b69c1U, (uint32_t)0xefbe4786U, (uint32_t)0x0fc19dc6U, (uint32_t)0x240ca1ccU, - (uint32_t)0x2de92c6fU, (uint32_t)0x4a7484aaU, (uint32_t)0x5cb0a9dcU, (uint32_t)0x76f988daU, - (uint32_t)0x983e5152U, (uint32_t)0xa831c66dU, (uint32_t)0xb00327c8U, (uint32_t)0xbf597fc7U, - (uint32_t)0xc6e00bf3U, (uint32_t)0xd5a79147U, (uint32_t)0x06ca6351U, (uint32_t)0x14292967U, - (uint32_t)0x27b70a85U, (uint32_t)0x2e1b2138U, (uint32_t)0x4d2c6dfcU, (uint32_t)0x53380d13U, - (uint32_t)0x650a7354U, (uint32_t)0x766a0abbU, (uint32_t)0x81c2c92eU, (uint32_t)0x92722c85U, - (uint32_t)0xa2bfe8a1U, (uint32_t)0xa81a664bU, (uint32_t)0xc24b8b70U, (uint32_t)0xc76c51a3U, - (uint32_t)0xd192e819U, (uint32_t)0xd6990624U, (uint32_t)0xf40e3585U, (uint32_t)0x106aa070U, - (uint32_t)0x19a4c116U, (uint32_t)0x1e376c08U, (uint32_t)0x2748774cU, (uint32_t)0x34b0bcb5U, - (uint32_t)0x391c0cb3U, (uint32_t)0x4ed8aa4aU, (uint32_t)0x5b9cca4fU, (uint32_t)0x682e6ff3U, - (uint32_t)0x748f82eeU, (uint32_t)0x78a5636fU, (uint32_t)0x84c87814U, (uint32_t)0x8cc70208U, - (uint32_t)0x90befffaU, (uint32_t)0xa4506cebU, (uint32_t)0xbef9a3f7U, (uint32_t)0xc67178f2U + 0x428a2f98U, 0x71374491U, 0xb5c0fbcfU, 0xe9b5dba5U, 0x3956c25bU, 0x59f111f1U, 0x923f82a4U, + 0xab1c5ed5U, 0xd807aa98U, 0x12835b01U, 0x243185beU, 0x550c7dc3U, 0x72be5d74U, 0x80deb1feU, + 0x9bdc06a7U, 0xc19bf174U, 0xe49b69c1U, 0xefbe4786U, 0x0fc19dc6U, 0x240ca1ccU, 0x2de92c6fU, + 0x4a7484aaU, 0x5cb0a9dcU, 0x76f988daU, 0x983e5152U, 0xa831c66dU, 0xb00327c8U, 0xbf597fc7U, + 0xc6e00bf3U, 0xd5a79147U, 0x06ca6351U, 0x14292967U, 0x27b70a85U, 0x2e1b2138U, 0x4d2c6dfcU, + 0x53380d13U, 0x650a7354U, 0x766a0abbU, 0x81c2c92eU, 0x92722c85U, 0xa2bfe8a1U, 0xa81a664bU, + 0xc24b8b70U, 0xc76c51a3U, 0xd192e819U, 0xd6990624U, 0xf40e3585U, 0x106aa070U, 0x19a4c116U, + 0x1e376c08U, 0x2748774cU, 0x34b0bcb5U, 0x391c0cb3U, 0x4ed8aa4aU, 0x5b9cca4fU, 0x682e6ff3U, + 0x748f82eeU, 0x78a5636fU, 0x84c87814U, 0x8cc70208U, 0x90befffaU, 0xa4506cebU, 0xbef9a3f7U, + 0xc67178f2U }; static const uint64_t -Hacl_Impl_SHA2_Generic_k384_512[80U] = +Hacl_Hash_SHA2_k384_512[80U] = { - (uint64_t)0x428a2f98d728ae22U, (uint64_t)0x7137449123ef65cdU, (uint64_t)0xb5c0fbcfec4d3b2fU, - (uint64_t)0xe9b5dba58189dbbcU, (uint64_t)0x3956c25bf348b538U, (uint64_t)0x59f111f1b605d019U, - (uint64_t)0x923f82a4af194f9bU, (uint64_t)0xab1c5ed5da6d8118U, (uint64_t)0xd807aa98a3030242U, - (uint64_t)0x12835b0145706fbeU, (uint64_t)0x243185be4ee4b28cU, (uint64_t)0x550c7dc3d5ffb4e2U, - (uint64_t)0x72be5d74f27b896fU, (uint64_t)0x80deb1fe3b1696b1U, (uint64_t)0x9bdc06a725c71235U, - (uint64_t)0xc19bf174cf692694U, (uint64_t)0xe49b69c19ef14ad2U, (uint64_t)0xefbe4786384f25e3U, - (uint64_t)0x0fc19dc68b8cd5b5U, (uint64_t)0x240ca1cc77ac9c65U, (uint64_t)0x2de92c6f592b0275U, - (uint64_t)0x4a7484aa6ea6e483U, (uint64_t)0x5cb0a9dcbd41fbd4U, (uint64_t)0x76f988da831153b5U, - (uint64_t)0x983e5152ee66dfabU, (uint64_t)0xa831c66d2db43210U, (uint64_t)0xb00327c898fb213fU, - (uint64_t)0xbf597fc7beef0ee4U, (uint64_t)0xc6e00bf33da88fc2U, (uint64_t)0xd5a79147930aa725U, - (uint64_t)0x06ca6351e003826fU, (uint64_t)0x142929670a0e6e70U, (uint64_t)0x27b70a8546d22ffcU, - (uint64_t)0x2e1b21385c26c926U, (uint64_t)0x4d2c6dfc5ac42aedU, (uint64_t)0x53380d139d95b3dfU, - (uint64_t)0x650a73548baf63deU, (uint64_t)0x766a0abb3c77b2a8U, (uint64_t)0x81c2c92e47edaee6U, - (uint64_t)0x92722c851482353bU, (uint64_t)0xa2bfe8a14cf10364U, (uint64_t)0xa81a664bbc423001U, - (uint64_t)0xc24b8b70d0f89791U, (uint64_t)0xc76c51a30654be30U, (uint64_t)0xd192e819d6ef5218U, - (uint64_t)0xd69906245565a910U, (uint64_t)0xf40e35855771202aU, (uint64_t)0x106aa07032bbd1b8U, - (uint64_t)0x19a4c116b8d2d0c8U, (uint64_t)0x1e376c085141ab53U, (uint64_t)0x2748774cdf8eeb99U, - (uint64_t)0x34b0bcb5e19b48a8U, (uint64_t)0x391c0cb3c5c95a63U, (uint64_t)0x4ed8aa4ae3418acbU, - (uint64_t)0x5b9cca4f7763e373U, (uint64_t)0x682e6ff3d6b2b8a3U, (uint64_t)0x748f82ee5defb2fcU, - (uint64_t)0x78a5636f43172f60U, (uint64_t)0x84c87814a1f0ab72U, (uint64_t)0x8cc702081a6439ecU, - (uint64_t)0x90befffa23631e28U, (uint64_t)0xa4506cebde82bde9U, (uint64_t)0xbef9a3f7b2c67915U, - (uint64_t)0xc67178f2e372532bU, (uint64_t)0xca273eceea26619cU, (uint64_t)0xd186b8c721c0c207U, - (uint64_t)0xeada7dd6cde0eb1eU, (uint64_t)0xf57d4f7fee6ed178U, (uint64_t)0x06f067aa72176fbaU, - (uint64_t)0x0a637dc5a2c898a6U, (uint64_t)0x113f9804bef90daeU, (uint64_t)0x1b710b35131c471bU, - (uint64_t)0x28db77f523047d84U, (uint64_t)0x32caab7b40c72493U, (uint64_t)0x3c9ebe0a15c9bebcU, - (uint64_t)0x431d67c49c100d4cU, (uint64_t)0x4cc5d4becb3e42b6U, (uint64_t)0x597f299cfc657e2aU, - (uint64_t)0x5fcb6fab3ad6faecU, (uint64_t)0x6c44198c4a475817U + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL }; -void Hacl_SHA2_Scalar32_sha256_init(uint32_t *hash); +void Hacl_Hash_SHA2_sha256_init(uint32_t *hash); -void Hacl_SHA2_Scalar32_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st); +void Hacl_Hash_SHA2_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st); void -Hacl_SHA2_Scalar32_sha256_update_last( - uint64_t totlen, - uint32_t len, - uint8_t *b, - uint32_t *hash -); +Hacl_Hash_SHA2_sha256_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *hash); -void Hacl_SHA2_Scalar32_sha256_finish(uint32_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha256_finish(uint32_t *st, uint8_t *h); -void Hacl_SHA2_Scalar32_sha224_init(uint32_t *hash); +void Hacl_Hash_SHA2_sha224_init(uint32_t *hash); void -Hacl_SHA2_Scalar32_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st); +Hacl_Hash_SHA2_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st); -void Hacl_SHA2_Scalar32_sha224_finish(uint32_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha224_finish(uint32_t *st, uint8_t *h); -void Hacl_SHA2_Scalar32_sha512_init(uint64_t *hash); +void Hacl_Hash_SHA2_sha512_init(uint64_t *hash); -void Hacl_SHA2_Scalar32_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); +void Hacl_Hash_SHA2_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); void -Hacl_SHA2_Scalar32_sha512_update_last( +Hacl_Hash_SHA2_sha512_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *hash ); -void Hacl_SHA2_Scalar32_sha512_finish(uint64_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha512_finish(uint64_t *st, uint8_t *h); -void Hacl_SHA2_Scalar32_sha384_init(uint64_t *hash); +void Hacl_Hash_SHA2_sha384_init(uint64_t *hash); -void Hacl_SHA2_Scalar32_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); +void Hacl_Hash_SHA2_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); void -Hacl_SHA2_Scalar32_sha384_update_last( +Hacl_Hash_SHA2_sha384_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *st ); -void Hacl_SHA2_Scalar32_sha384_finish(uint64_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha384_finish(uint64_t *st, uint8_t *h); #if defined(__cplusplus) } diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA3.h b/Modules/_hacl/internal/Hacl_Hash_SHA3.h index 1c9808b8dd497c..0a152b4c622533 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA3.h @@ -37,6 +37,12 @@ extern "C" { #include "../Hacl_Hash_SHA3.h" +extern const uint32_t Hacl_Hash_SHA3_keccak_rotc[24U]; + +extern const uint32_t Hacl_Hash_SHA3_keccak_piln[24U]; + +extern const uint64_t Hacl_Hash_SHA3_keccak_rndc[24U]; + void Hacl_Hash_SHA3_update_multi_sha3( Spec_Hash_Definitions_hash_alg a, @@ -53,10 +59,6 @@ Hacl_Hash_SHA3_update_last_sha3( uint32_t input_len ); -void Hacl_Impl_SHA3_state_permute(uint64_t *s); - -void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s); - #if defined(__cplusplus) } #endif diff --git a/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h b/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h new file mode 100644 index 00000000000000..f4cf516124aabb --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h @@ -0,0 +1,73 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Impl_Blake2_Constants_H +#define __internal_Hacl_Impl_Blake2_Constants_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +static const +uint32_t +Hacl_Hash_Blake2b_sigmaTable[160U] = + { + 0U, 1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U, 9U, 10U, 11U, 12U, 13U, 14U, 15U, 14U, 10U, 4U, 8U, 9U, 15U, + 13U, 6U, 1U, 12U, 0U, 2U, 11U, 7U, 5U, 3U, 11U, 8U, 12U, 0U, 5U, 2U, 15U, 13U, 10U, 14U, 3U, 6U, + 7U, 1U, 9U, 4U, 7U, 9U, 3U, 1U, 13U, 12U, 11U, 14U, 2U, 6U, 5U, 10U, 4U, 0U, 15U, 8U, 9U, 0U, + 5U, 7U, 2U, 4U, 10U, 15U, 14U, 1U, 11U, 12U, 6U, 8U, 3U, 13U, 2U, 12U, 6U, 10U, 0U, 11U, 8U, 3U, + 4U, 13U, 7U, 5U, 15U, 14U, 1U, 9U, 12U, 5U, 1U, 15U, 14U, 13U, 4U, 10U, 0U, 7U, 6U, 3U, 9U, 2U, + 8U, 11U, 13U, 11U, 7U, 14U, 12U, 1U, 3U, 9U, 5U, 0U, 15U, 4U, 8U, 6U, 2U, 10U, 6U, 15U, 14U, 9U, + 11U, 3U, 0U, 8U, 12U, 2U, 13U, 7U, 1U, 4U, 10U, 5U, 10U, 2U, 8U, 4U, 7U, 6U, 1U, 5U, 15U, 11U, + 9U, 14U, 3U, 12U, 13U + }; + +static const +uint32_t +Hacl_Hash_Blake2b_ivTable_S[8U] = + { + 0x6A09E667U, 0xBB67AE85U, 0x3C6EF372U, 0xA54FF53AU, 0x510E527FU, 0x9B05688CU, 0x1F83D9ABU, + 0x5BE0CD19U + }; + +static const +uint64_t +Hacl_Hash_Blake2b_ivTable_B[8U] = + { + 0x6A09E667F3BCC908ULL, 0xBB67AE8584CAA73BULL, 0x3C6EF372FE94F82BULL, 0xA54FF53A5F1D36F1ULL, + 0x510E527FADE682D1ULL, 0x9B05688C2B3E6C1FULL, 0x1F83D9ABFB41BD6BULL, 0x5BE0CD19137E2179ULL + }; + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Impl_Blake2_Constants_H_DEFINED +#endif diff --git a/Modules/_hacl/lib_memzero0.h b/Modules/_hacl/lib_memzero0.h new file mode 100644 index 00000000000000..fea3e41c907f44 --- /dev/null +++ b/Modules/_hacl/lib_memzero0.h @@ -0,0 +1,5 @@ +#include + +void Lib_Memzero0_memzero0(void *dst, uint64_t len); + +#define Lib_Memzero0_memzero(dst, len, t, _ret_t) Lib_Memzero0_memzero0(dst, len * sizeof(t)) diff --git a/Modules/_hacl/libintvector.h b/Modules/_hacl/libintvector.h new file mode 100644 index 00000000000000..11e914f7e1650a --- /dev/null +++ b/Modules/_hacl/libintvector.h @@ -0,0 +1,936 @@ +#ifndef __Vec_Intrin_H +#define __Vec_Intrin_H + +#include + +/* We include config.h here to ensure that the various feature-flags are + * properly brought into scope. Users can either run the configure script, or + * write a config.h themselves and put it under version control. */ +#if defined(__has_include) +#if __has_include("config.h") +#include "config.h" +#endif +#endif + +/* # DEBUGGING: + * ============ + * It is possible to debug the current definitions by using libintvector_debug.h + * See the include at the bottom of the file. */ + +#define Lib_IntVector_Intrinsics_bit_mask64(x) -((x) & 1) + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) + +#if defined(HACL_CAN_COMPILE_VEC128) + +#include +#include +#include + +typedef __m128i Lib_IntVector_Intrinsics_vec128; + +#define Lib_IntVector_Intrinsics_ni_aes_enc(x0, x1) \ + (_mm_aesenc_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_ni_aes_enc_last(x0, x1) \ + (_mm_aesenclast_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_ni_aes_keygen_assist(x0, x1) \ + (_mm_aeskeygenassist_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_ni_clmul(x0, x1, x2) \ + (_mm_clmulepi64_si128(x0, x1, x2)) + + +#define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \ + (_mm_xor_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ + (_mm_cmpeq_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ + (_mm_cmpeq_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ + (_mm_cmpgt_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ + (_mm_cmpgt_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ + (_mm_or_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ + (_mm_and_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ + (_mm_xor_si128(x0, _mm_set1_epi32(-1))) + + +#define Lib_IntVector_Intrinsics_vec128_shift_left(x0, x1) \ + (_mm_slli_si128(x0, (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right(x0, x1) \ + (_mm_srli_si128(x0, (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ + (_mm_slli_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ + (_mm_srli_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_shift_left32(x0, x1) \ + (_mm_slli_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \ + (_mm_srli_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32_8(x0) \ + (_mm_shuffle_epi8(x0, _mm_set_epi8(14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) \ + (_mm_shuffle_epi8(x0, _mm_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32_24(x0) \ + (_mm_shuffle_epi8(x0, _mm_set_epi8(12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,x1) \ + (((x1) == 8? Lib_IntVector_Intrinsics_vec128_rotate_left32_8(x0) : \ + ((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) : \ + ((x1) == 24? Lib_IntVector_Intrinsics_vec128_rotate_left32_24(x0) : \ + _mm_xor_si128(_mm_slli_epi32(x0,x1),_mm_srli_epi32(x0,32-(x1))))))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0,x1) \ + (Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,32-(x1))) + +#define Lib_IntVector_Intrinsics_vec128_shuffle32(x0, x1, x2, x3, x4) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE(x4,x3,x2,x1))) + +#define Lib_IntVector_Intrinsics_vec128_shuffle64(x0, x1, x2) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE(2*x1+1,2*x1,2*x2+1,2*x2))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE((x1+3)%4,(x1+2)%4,(x1+1)%4,x1%4))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64(x0, x1) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE((2*x1+3)%4,(2*x1+2)%4,(2*x1+1)%4,(2*x1)%4))) + +#define Lib_IntVector_Intrinsics_vec128_load32_le(x0) \ + (_mm_loadu_si128((__m128i*)(x0))) + +#define Lib_IntVector_Intrinsics_vec128_load64_le(x0) \ + (_mm_loadu_si128((__m128i*)(x0))) + +#define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), x1)) + +#define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), x1)) + +#define Lib_IntVector_Intrinsics_vec128_load_be(x0) \ + (_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15))) + +#define Lib_IntVector_Intrinsics_vec128_load32_be(x0) \ + (_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3))) + +#define Lib_IntVector_Intrinsics_vec128_load64_be(x0) \ + (_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7))) + +#define Lib_IntVector_Intrinsics_vec128_store_be(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)))) + + +#define Lib_IntVector_Intrinsics_vec128_store32_be(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)))) + +#define Lib_IntVector_Intrinsics_vec128_store64_be(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)))) + + + +#define Lib_IntVector_Intrinsics_vec128_insert8(x0, x1, x2) \ + (_mm_insert_epi8(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ + (_mm_insert_epi32(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ + (_mm_insert_epi64(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_extract8(x0, x1) \ + (_mm_extract_epi8(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ + (_mm_extract_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ + (_mm_extract_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_zero \ + (_mm_setzero_si128()) + + +#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ + (_mm_add_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ + (_mm_sub_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ + (_mm_mul_epu32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ + (_mm_mul_epu32(x0, _mm_set1_epi64x(x1))) + +#define Lib_IntVector_Intrinsics_vec128_add32(x0, x1) \ + (_mm_add_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_sub32(x0, x1) \ + (_mm_sub_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_mul32(x0, x1) \ + (_mm_mullo_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_smul32(x0, x1) \ + (_mm_mullo_epi32(x0, _mm_set1_epi32(x1))) + +#define Lib_IntVector_Intrinsics_vec128_load128(x) \ + ((__m128i)x) + +#define Lib_IntVector_Intrinsics_vec128_load64(x) \ + (_mm_set1_epi64x(x)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec128_load64s(x0, x1) \ + (_mm_set_epi64x(x1, x0)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec128_load32(x) \ + (_mm_set1_epi32(x)) + +#define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \ + (_mm_set_epi32(x3, x2, x1, x0)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x1, x2) \ + (_mm_unpacklo_epi32(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x1, x2) \ + (_mm_unpackhi_epi32(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x1, x2) \ + (_mm_unpacklo_epi64(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x1, x2) \ + (_mm_unpackhi_epi64(x1, x2)) + +#endif /* HACL_CAN_COMPILE_VEC128 */ + +#if defined(HACL_CAN_COMPILE_VEC256) + +#include + +typedef __m256i Lib_IntVector_Intrinsics_vec256; + + +#define Lib_IntVector_Intrinsics_vec256_eq64(x0, x1) \ + (_mm256_cmpeq_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_eq32(x0, x1) \ + (_mm256_cmpeq_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_gt64(x0, x1) \ + (_mm256_cmpgt_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_gt32(x0, x1) \ + (_mm256_cmpgt_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_xor(x0, x1) \ + (_mm256_xor_si256(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_or(x0, x1) \ + (_mm256_or_si256(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_and(x0, x1) \ + (_mm256_and_si256(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_lognot(x0) \ + (_mm256_xor_si256(x0, _mm256_set1_epi32(-1))) + +#define Lib_IntVector_Intrinsics_vec256_shift_left(x0, x1) \ + (_mm256_slli_si256(x0, (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec256_shift_right(x0, x1) \ + (_mm256_srli_si256(x0, (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec256_shift_left64(x0, x1) \ + (_mm256_slli_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_shift_right64(x0, x1) \ + (_mm256_srli_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_shift_left32(x0, x1) \ + (_mm256_slli_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_shift_right32(x0, x1) \ + (_mm256_srli_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left32_8(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3,14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left32_16(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left32_24(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1,12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left32(x0,x1) \ + ((x1 == 8? Lib_IntVector_Intrinsics_vec256_rotate_left32_8(x0) : \ + (x1 == 16? Lib_IntVector_Intrinsics_vec256_rotate_left32_16(x0) : \ + (x1 == 24? Lib_IntVector_Intrinsics_vec256_rotate_left32_24(x0) : \ + _mm256_or_si256(_mm256_slli_epi32(x0,x1),_mm256_srli_epi32(x0,32-(x1))))))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right32(x0,x1) \ + (Lib_IntVector_Intrinsics_vec256_rotate_left32(x0,32-(x1))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_8(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(8,15,14,13,12,11,10,9,0,7,6,5,4,3,2,1,8,15,14,13,12,11,10,9,0,7,6,5,4,3,2,1))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_16(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(9,8,15,14,13,12,11,10,1,0,7,6,5,4,3,2,9,8,15,14,13,12,11,10,1,0,7,6,5,4,3,2))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_24(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(10,9,8,15,14,13,12,11,2,1,0,7,6,5,4,3,10,9,8,15,14,13,12,11,2,1,0,7,6,5,4,3))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_32(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_40(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(12,11,10,9,8,15,14,13,4,3,2,1,0,7,6,5,12,11,10,9,8,15,14,13,4,3,2,1,0,7,6,5))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_48(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(13,12,11,10,9,8,15,14,5,4,3,2,1,0,7,6,13,12,11,10,9,8,15,14,5,4,3,2,1,0,7,6))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_56(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(14,13,12,11,10,9,8,15,6,5,4,3,2,1,0,7,14,13,12,11,10,9,8,15,6,5,4,3,2,1,0,7))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64(x0,x1) \ + ((x1 == 8? Lib_IntVector_Intrinsics_vec256_rotate_right64_8(x0) : \ + (x1 == 16? Lib_IntVector_Intrinsics_vec256_rotate_right64_16(x0) : \ + (x1 == 24? Lib_IntVector_Intrinsics_vec256_rotate_right64_24(x0) : \ + (x1 == 32? Lib_IntVector_Intrinsics_vec256_rotate_right64_32(x0) : \ + (x1 == 40? Lib_IntVector_Intrinsics_vec256_rotate_right64_40(x0) : \ + (x1 == 48? Lib_IntVector_Intrinsics_vec256_rotate_right64_48(x0) : \ + (x1 == 56? Lib_IntVector_Intrinsics_vec256_rotate_right64_56(x0) : \ + _mm256_xor_si256(_mm256_srli_epi64((x0),(x1)),_mm256_slli_epi64((x0),(64-(x1)))))))))))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left64(x0,x1) \ + (Lib_IntVector_Intrinsics_vec256_rotate_right64(x0,64-(x1))) + +#define Lib_IntVector_Intrinsics_vec256_shuffle64(x0, x1, x2, x3, x4) \ + (_mm256_permute4x64_epi64(x0, _MM_SHUFFLE(x4,x3,x2,x1))) + +#define Lib_IntVector_Intrinsics_vec256_shuffle32(x0, x1, x2, x3, x4, x5, x6, x7, x8) \ + (_mm256_permutevar8x32_epi32(x0, _mm256_set_epi32(x8,x7,x6,x5,x4,x3,x2,x1))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes32(x0, x1) \ + (_mm256_permutevar8x32_epi32(x0, _mm256_set_epi32((x1+7)%8,(x1+6)%8,(x1+5)%8,(x1+4)%8,(x1+3%8),(x1+2)%8,(x1+1)%8,x1%8))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(x0, x1) \ + (_mm256_permute4x64_epi64(x0, _MM_SHUFFLE((x1+3)%4,(x1+2)%4,(x1+1)%4,x1%4))) + +#define Lib_IntVector_Intrinsics_vec256_load32_le(x0) \ + (_mm256_loadu_si256((__m256i*)(x0))) + +#define Lib_IntVector_Intrinsics_vec256_load64_le(x0) \ + (_mm256_loadu_si256((__m256i*)(x0))) + +#define Lib_IntVector_Intrinsics_vec256_load32_be(x0) \ + (_mm256_shuffle_epi8(_mm256_loadu_si256((__m256i*)(x0)), _mm256_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3))) + +#define Lib_IntVector_Intrinsics_vec256_load64_be(x0) \ + (_mm256_shuffle_epi8(_mm256_loadu_si256((__m256i*)(x0)), _mm256_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7))) + + +#define Lib_IntVector_Intrinsics_vec256_store32_le(x0, x1) \ + (_mm256_storeu_si256((__m256i*)(x0), x1)) + +#define Lib_IntVector_Intrinsics_vec256_store64_le(x0, x1) \ + (_mm256_storeu_si256((__m256i*)(x0), x1)) + +#define Lib_IntVector_Intrinsics_vec256_store32_be(x0, x1) \ + (_mm256_storeu_si256((__m256i*)(x0), _mm256_shuffle_epi8(x1, _mm256_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)))) + +#define Lib_IntVector_Intrinsics_vec256_store64_be(x0, x1) \ + (_mm256_storeu_si256((__m256i*)(x0), _mm256_shuffle_epi8(x1, _mm256_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)))) + + +#define Lib_IntVector_Intrinsics_vec256_insert8(x0, x1, x2) \ + (_mm256_insert_epi8(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_insert32(x0, x1, x2) \ + (_mm256_insert_epi32(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_insert64(x0, x1, x2) \ + (_mm256_insert_epi64(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_extract8(x0, x1) \ + (_mm256_extract_epi8(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_extract32(x0, x1) \ + (_mm256_extract_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_extract64(x0, x1) \ + (_mm256_extract_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_zero \ + (_mm256_setzero_si256()) + +#define Lib_IntVector_Intrinsics_vec256_add64(x0, x1) \ + (_mm256_add_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_sub64(x0, x1) \ + (_mm256_sub_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_mul64(x0, x1) \ + (_mm256_mul_epu32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_smul64(x0, x1) \ + (_mm256_mul_epu32(x0, _mm256_set1_epi64x(x1))) + + +#define Lib_IntVector_Intrinsics_vec256_add32(x0, x1) \ + (_mm256_add_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_sub32(x0, x1) \ + (_mm256_sub_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_mul32(x0, x1) \ + (_mm256_mullo_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_smul32(x0, x1) \ + (_mm256_mullo_epi32(x0, _mm256_set1_epi32(x1))) + + +#define Lib_IntVector_Intrinsics_vec256_load64(x1) \ + (_mm256_set1_epi64x(x1)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec256_load64s(x0, x1, x2, x3) \ + (_mm256_set_epi64x(x3,x2,x1,x0)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec256_load32(x) \ + (_mm256_set1_epi32(x)) + +#define Lib_IntVector_Intrinsics_vec256_load32s(x0,x1,x2,x3,x4, x5, x6, x7) \ + (_mm256_set_epi32(x7, x6, x5, x4, x3, x2, x1, x0)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec256_load128(x) \ + (_mm256_set_m128i((__m128i)x)) + +#define Lib_IntVector_Intrinsics_vec256_load128s(x0,x1) \ + (_mm256_set_m128i((__m128i)x1,(__m128i)x0)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_low32(x1, x2) \ + (_mm256_unpacklo_epi32(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_high32(x1, x2) \ + (_mm256_unpackhi_epi32(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_low64(x1, x2) \ + (_mm256_unpacklo_epi64(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_high64(x1, x2) \ + (_mm256_unpackhi_epi64(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_low128(x1, x2) \ + (_mm256_permute2x128_si256(x1, x2, 0x20)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_high128(x1, x2) \ + (_mm256_permute2x128_si256(x1, x2, 0x31)) + +#endif /* HACL_CAN_COMPILE_VEC256 */ + +#elif (defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM)) \ + && !defined(__ARM_32BIT_STATE) + +#if defined(HACL_CAN_COMPILE_VEC128) + +#include + +typedef uint32x4_t Lib_IntVector_Intrinsics_vec128; + +#define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \ + (veorq_u32(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ + (vceqq_u32(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ + (vceqq_u32(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ + (vcgtq_u32(x0, x1)) + +#define high32(x0) \ + (vmovn_u64(vshrq_n_u64(vreinterpretq_u64_u32(x0),32))) + +#define low32(x0) \ + (vmovn_u64(vreinterpretq_u64_u32(x0))) + +#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ + (vreinterpretq_u32_u64(vmovl_u32(vorr_u32(vcgt_u32(high32(x0),high32(x1)),vand_u32(vceq_u32(high32(x0),high32(x1)),vcgt_u32(low32(x0),low32(x1))))))) + +#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ + (vorrq_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ + (vandq_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ + (vmvnq_u32(x0)) + + +#define Lib_IntVector_Intrinsics_vec128_shift_left(x0, x1) \ + (vextq_u32(x0, vdupq_n_u8(0), 16-(x1)/8)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right(x0, x1) \ + (vextq_u32(x0, vdupq_n_u8(0), (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ + (vreinterpretq_u32_u64(vshlq_n_u64(vreinterpretq_u64_u32(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ + (vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_shift_left32(x0, x1) \ + (vshlq_n_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \ + (vshrq_n_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x1) \ + (vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,x1) \ + (((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) : \ + vsriq_n_u32(vshlq_n_u32((x0),(x1)),(x0),32-(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32_16(x1) \ + (vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0,x1) \ + (((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_right32_16(x0) : \ + vsriq_n_u32(vshlq_n_u32((x0),32-(x1)),(x0),(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ + (vextq_u32(x0,x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64(x0, x1) \ + (vextq_u64(x0,x0,x1)) + + +/* +#define Lib_IntVector_Intrinsics_vec128_shuffle32(x0, x1, x2, x3, x4) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE(x1,x2,x3,x4))) + +#define Lib_IntVector_Intrinsics_vec128_shuffle64(x0, x1, x2) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE(2*x1+1,2*x1,2*x2+1,2*x2))) +*/ + +#define Lib_IntVector_Intrinsics_vec128_load32_le(x0) \ + (vld1q_u32((const uint32_t*) (x0))) + +#define Lib_IntVector_Intrinsics_vec128_load64_le(x0) \ + (vld1q_u32((const uint32_t*) (x0))) + +#define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \ + (vst1q_u32((uint32_t*)(x0),(x1))) + +#define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \ + (vst1q_u32((uint32_t*)(x0),(x1))) + +/* +#define Lib_IntVector_Intrinsics_vec128_load_be(x0) \ + ( Lib_IntVector_Intrinsics_vec128 l = vrev64q_u8(vld1q_u32((uint32_t*)(x0))); + +*/ + +#define Lib_IntVector_Intrinsics_vec128_load32_be(x0) \ + (vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(vld1q_u32((const uint32_t*)(x0)))))) + +#define Lib_IntVector_Intrinsics_vec128_load64_be(x0) \ + (vreinterpretq_u32_u8(vrev64q_u8(vreinterpretq_u8_u32(vld1q_u32((const uint32_t*)(x0)))))) + +/* +#define Lib_IntVector_Intrinsics_vec128_store_be(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)))) +*/ + +#define Lib_IntVector_Intrinsics_vec128_store32_be(x0, x1) \ + (vst1q_u32((uint32_t*)(x0),(vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(x1)))))) + +#define Lib_IntVector_Intrinsics_vec128_store64_be(x0, x1) \ + (vst1q_u32((uint32_t*)(x0),(vreinterpretq_u32_u8(vrev64q_u8(vreinterpretq_u8_u32(x1)))))) + +#define Lib_IntVector_Intrinsics_vec128_insert8(x0, x1, x2) \ + (vsetq_lane_u8(x1,x0,x2)) + +#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ + (vsetq_lane_u32(x1,x0,x2)) + +#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ + (vreinterpretq_u32_u64(vsetq_lane_u64(x1,vreinterpretq_u64_u32(x0),x2))) + +#define Lib_IntVector_Intrinsics_vec128_extract8(x0, x1) \ + (vgetq_lane_u8(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ + (vgetq_lane_u32(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ + (vgetq_lane_u64(vreinterpretq_u64_u32(x0),x1)) + +#define Lib_IntVector_Intrinsics_vec128_zero \ + (vdupq_n_u32(0)) + +#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ + (vreinterpretq_u32_u64(vaddq_u64(vreinterpretq_u64_u32(x0), vreinterpretq_u64_u32(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ + (vreinterpretq_u32_u64(vsubq_u64(vreinterpretq_u64_u32(x0), vreinterpretq_u64_u32(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ + (vreinterpretq_u32_u64(vmull_u32(vmovn_u64(vreinterpretq_u64_u32(x0)), vmovn_u64(vreinterpretq_u64_u32(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ + (vreinterpretq_u32_u64(vmull_n_u32(vmovn_u64(vreinterpretq_u64_u32(x0)), (uint32_t)x1))) + +#define Lib_IntVector_Intrinsics_vec128_add32(x0, x1) \ + (vaddq_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_sub32(x0, x1) \ + (vsubq_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_mul32(x0, x1) \ + (vmulq_lane_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_smul32(x0, x1) \ + (vmulq_lane_u32(x0, vdupq_n_u32(x1))) + +#define Lib_IntVector_Intrinsics_vec128_load128(x) \ + ((uint32x4_t)(x)) + +#define Lib_IntVector_Intrinsics_vec128_load64(x) \ + (vreinterpretq_u32_u64(vdupq_n_u64(x))) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec128_load32(x) \ + (vdupq_n_u32(x)) /* hi lo */ + +static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load64s(uint64_t x1, uint64_t x2){ + const uint64_t a[2] = {x1,x2}; + return vreinterpretq_u32_u64(vld1q_u64(a)); +} + +static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load32s(uint32_t x1, uint32_t x2, uint32_t x3, uint32_t x4){ + const uint32_t a[4] = {x1,x2,x3,x4}; + return vld1q_u32(a); +} + +#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x1, x2) \ + (vzip1q_u32(x1,x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x1, x2) \ + (vzip2q_u32(x1,x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x1,x2) \ + (vreinterpretq_u32_u64(vzip1q_u64(vreinterpretq_u64_u32(x1),vreinterpretq_u64_u32(x2)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x1,x2) \ + (vreinterpretq_u32_u64(vzip2q_u64(vreinterpretq_u64_u32(x1),vreinterpretq_u64_u32(x2)))) + +#endif /* HACL_CAN_COMPILE_VEC128 */ + +/* IBM z architecture */ +#elif defined(__s390x__) /* this flag is for GCC only */ + +#if defined(HACL_CAN_COMPILE_VEC128) + +#include +#include + +/* The main vector 128 type + * We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char, + * unsigned int, unsigned long long: the compiler complains that the parameter + * combination is invalid. */ +typedef unsigned char vector128_8 __attribute__ ((vector_size(16))); +typedef unsigned int vector128_32 __attribute__ ((vector_size(16))); +typedef unsigned long long vector128_64 __attribute__ ((vector_size(16))); + +typedef vector128_8 Lib_IntVector_Intrinsics_vec128; +typedef vector128_8 vector128; + +#define Lib_IntVector_Intrinsics_vec128_load32_le(x) \ + (vector128) ((vector128_32) vec_revb(*((vector128_32*) (const uint8_t*)(x)))) + +#define Lib_IntVector_Intrinsics_vec128_load32_be(x) \ + (vector128) (*((vector128_32*) (const uint8_t*)(x))) + +#define Lib_IntVector_Intrinsics_vec128_load64_le(x) \ + (vector128) ((vector128_64) vec_revb(*((vector128_64*) (const uint8_t*)(x)))) + +static inline +void Lib_IntVector_Intrinsics_vec128_store32_le(const uint8_t *x0, vector128 x1) { + *((vector128_32*)x0) = vec_revb((vector128_32) x1); +} + +static inline +void Lib_IntVector_Intrinsics_vec128_store32_be(const uint8_t *x0, vector128 x1) { + *((vector128_32*)x0) = (vector128_32) x1; +} + +static inline +void Lib_IntVector_Intrinsics_vec128_store64_le(const uint8_t *x0, vector128 x1) { + *((vector128_64*)x0) = vec_revb((vector128_64) x1); +} + +#define Lib_IntVector_Intrinsics_vec128_add32(x0,x1) \ + ((vector128)((vector128_32)(((vector128_32)(x0)) + ((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ + ((vector128)((vector128_64)(((vector128_64)(x0)) + ((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ + ((vector128)(vec_and((vector128)(x0),(vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ + ((vector128)(vec_cmpeq(((vector128_32)(x0)),((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ + ((vector128)(vec_cmpeq(((vector128_64)(x0)),((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ + ((unsigned int)(vec_extract((vector128_32)(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ + ((unsigned long long)(vec_extract((vector128_64)(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ + ((vector128)((vector128_32)(((vector128_32)(x0)) > ((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ + ((vector128)((vector128_64)(((vector128_64)(x0)) > ((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ + ((vector128)((vector128_32)vec_insert((unsigned int)(x1), (vector128_32)(x0), x2))) + +#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ + ((vector128)((vector128_64)vec_insert((unsigned long long)(x1), (vector128_64)(x0), x2))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x0, x1) \ + ((vector128)((vector128_32)vec_mergel((vector128_32)(x0), (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x0, x1) \ + ((vector128)((vector128_64)vec_mergel((vector128_64)(x0), (vector128_64)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x0, x1) \ + ((vector128)((vector128_32)vec_mergeh((vector128_32)(x0), (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x0, x1) \ + ((vector128)((vector128_64)vec_mergeh((vector128_64)(x0), (vector128_64)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_load32(x) \ + ((vector128)((vector128_32){(unsigned int)(x), (unsigned int)(x), \ + (unsigned int)(x), (unsigned int)(x)})) + +#define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \ + ((vector128)((vector128_32){(unsigned int)(x0),(unsigned int)(x1),(unsigned int)(x2),(unsigned int)(x3)})) + +#define Lib_IntVector_Intrinsics_vec128_load64(x) \ + ((vector128)((vector128_64)vec_load_pair((unsigned long long)(x),(unsigned long long)(x)))) + +#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ + ((vector128)(vec_xor((vector128)(x0), (vector128)vec_splat_u32(-1)))) + +#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ + ((vector128)(vec_mulo((vector128_32)(x0), \ + (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ + ((vector128)(vec_or((vector128)(x0),(vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0, x1) \ + ((vector128)(vec_rli((vector128_32)(x0), (unsigned long)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0, x1) \ + (Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,(uint32_t)(32-(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ + ((vector128)(vec_sld((vector128)(x0), (vector128)(x0), (x1%4)*4))) + +#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ + (((vector128)((vector128_64)vec_rli((vector128_64)(x0), (unsigned long)(x1)))) & \ + ((vector128)((vector128_64){0xffffffffffffffff << (x1), 0xffffffffffffffff << (x1)}))) + +#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ + (((vector128)((vector128_64)vec_rli((vector128_64)(x0), (unsigned long)(64-(x1))))) & \ + ((vector128)((vector128_64){0xffffffffffffffff >> (x1), 0xffffffffffffffff >> (x1)}))) + +#define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \ + (((vector128)((vector128_32)vec_rli((vector128_32)(x0), (unsigned int)(32-(x1))))) & \ + ((vector128)((vector128_32){0xffffffff >> (x1), 0xffffffff >> (x1), \ + 0xffffffff >> (x1), 0xffffffff >> (x1)}))) + +/* Doesn't work with vec_splat_u64 */ +#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ + ((vector128)(Lib_IntVector_Intrinsics_vec128_mul64(x0,((vector128_64){(unsigned long long)(x1),(unsigned long long)(x1)})))) + +#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ + ((vector128)((vector128_64)(x0) - (vector128_64)(x1))) + +static inline +vector128 Lib_IntVector_Intrinsics_vec128_xor(vector128 x0, vector128 x1) { + return ((vector128)(vec_xor((vector128)(x0), (vector128)(x1)))); +} + + +#define Lib_IntVector_Intrinsics_vec128_zero \ + ((vector128){}) + +#endif /* HACL_CAN_COMPILE_VEC128 */ + +#elif defined(__powerpc64__) // PowerPC 64 - this flag is for GCC only + +#if defined(HACL_CAN_COMPILE_VEC128) + +#include +#include // for memcpy +#include + +// The main vector 128 type +// We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char, +// unsigned int, unsigned long long: the compiler complains that the parameter +// combination is invalid. +typedef vector unsigned char vector128_8; +typedef vector unsigned int vector128_32; +typedef vector unsigned long long vector128_64; + +typedef vector128_8 Lib_IntVector_Intrinsics_vec128; +typedef vector128_8 vector128; + +#define Lib_IntVector_Intrinsics_vec128_load32_le(x) \ + ((vector128)((vector128_32)(vec_xl(0, (const unsigned int*) ((const uint8_t*)(x)))))) + +#define Lib_IntVector_Intrinsics_vec128_load64_le(x) \ + ((vector128)((vector128_64)(vec_xl(0, (const unsigned long long*) ((const uint8_t*)(x)))))) + +#define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \ + (vec_xst((vector128_32)(x1), 0, (unsigned int*) ((uint8_t*)(x0)))) + +#define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \ + (vec_xst((vector128_64)(x1), 0, (unsigned long long*) ((uint8_t*)(x0)))) + +#define Lib_IntVector_Intrinsics_vec128_add32(x0,x1) \ + ((vector128)((vector128_32)(((vector128_32)(x0)) + ((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ + ((vector128)((vector128_64)(((vector128_64)(x0)) + ((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ + ((vector128)(vec_and((vector128)(x0),(vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ + ((vector128)(vec_cmpeq(((vector128_32)(x0)),((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ + ((vector128)(vec_cmpeq(((vector128_64)(x0)),((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ + ((unsigned int)(vec_extract((vector128_32)(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ + ((unsigned long long)(vec_extract((vector128_64)(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ + ((vector128)((vector128_32)(((vector128_32)(x0)) > ((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ + ((vector128)((vector128_64)(((vector128_64)(x0)) > ((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ + ((vector128)((vector128_32)vec_insert((unsigned int)(x1), (vector128_32)(x0), x2))) + +#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ + ((vector128)((vector128_64)vec_insert((unsigned long long)(x1), (vector128_64)(x0), x2))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x0, x1) \ + ((vector128)((vector128_32)vec_mergel((vector128_32)(x0), (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x0, x1) \ + ((vector128)((vector128_64)vec_mergel((vector128_64)(x0), (vector128_64)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x0, x1) \ + ((vector128)((vector128_32)vec_mergeh((vector128_32)(x0), (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x0, x1) \ + ((vector128)((vector128_64)vec_mergeh((vector128_64)(x0), (vector128_64)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_load32(x) \ + ((vector128)((vector128_32){(unsigned int)(x), (unsigned int)(x), \ + (unsigned int)(x), (unsigned int)(x)})) + +#define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \ + ((vector128)((vector128_32){(unsigned int)(x0),(unsigned int)(x1),(unsigned int)(x2),(unsigned int)(x3)})) + +#define Lib_IntVector_Intrinsics_vec128_load64(x) \ + ((vector128)((vector128_64){(unsigned long long)(x),(unsigned long long)(x)})) + +#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ + ((vector128)(vec_xor((vector128)(x0), (vector128)vec_splat_u32(-1)))) + +#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ + ((vector128)(vec_mule((vector128_32)(x0), \ + (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ + ((vector128)(vec_or((vector128)(x0),(vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0, x1) \ + ((vector128)(vec_rl((vector128_32)(x0), (vector128_32){(unsigned int)(x1),(unsigned int)(x1),(unsigned int)(x1),(unsigned int)(x1)}))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0, x1) \ + (Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,(uint32_t)(32-(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ + ((vector128)(vec_sld((vector128)(x0), (vector128)(x0), ((4-(x1))%4)*4))) + +#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ + ((vector128)((vector128_64)vec_sl((vector128_64)(x0), (vector128_64){(unsigned long)(x1),(unsigned long)(x1)}))) + +#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ + ((vector128)((vector128_64)vec_sr((vector128_64)(x0), (vector128_64){(unsigned long)(x1),(unsigned long)(x1)}))) + +// Doesn't work with vec_splat_u64 +#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ + ((vector128)(Lib_IntVector_Intrinsics_vec128_mul64(x0,((vector128_64){(unsigned long long)(x1),(unsigned long long)(x1)})))) + +#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ + ((vector128)((vector128_64)(x0) - (vector128_64)(x1))) + +#define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \ + ((vector128)(vec_xor((vector128)(x0), (vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_zero \ + ((vector128){}) + +#endif /* HACL_CAN_COMPILE_VEC128 */ + +#endif // PowerPC64 + +// DEBUGGING: +// If libintvector_debug.h exists, use it to debug the current implementations. +// Note that some flags must be enabled for the debugging to be effective: +// see libintvector_debug.h for more details. +#if defined(__has_include) +#if __has_include("libintvector_debug.h") +#include "libintvector_debug.h" +#endif +#endif + +#endif // __Vec_Intrin_H diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index 0df236282ac509..8a1f4aef384d62 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -5,59 +5,61 @@ * C's excuse for namespaces: Use globally unique names to avoid linkage * conflicts with builds linking or dynamically loading other code potentially * using HACL* libraries. + * + * Something like this to generate new entries for the list: nm *.o | grep Hacl | cut -c 20- | sort | uniq | grep -v python_hashlib | egrep ^_ | gsed 's/_\(.*\)/#define \1 python_hashlib_\1/' */ -#define Hacl_Streaming_SHA2_state_sha2_224_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_224_s -#define Hacl_Streaming_SHA2_state_sha2_224 python_hashlib_Hacl_Streaming_SHA2_state_sha2_224 -#define Hacl_Streaming_SHA2_state_sha2_256 python_hashlib_Hacl_Streaming_SHA2_state_sha2_256 -#define Hacl_Streaming_SHA2_state_sha2_384_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_384_s -#define Hacl_Streaming_SHA2_state_sha2_384 python_hashlib_Hacl_Streaming_SHA2_state_sha2_384 -#define Hacl_Streaming_SHA2_state_sha2_512 python_hashlib_Hacl_Streaming_SHA2_state_sha2_512 -#define Hacl_Streaming_SHA2_create_in_256 python_hashlib_Hacl_Streaming_SHA2_create_in_256 -#define Hacl_Streaming_SHA2_create_in_224 python_hashlib_Hacl_Streaming_SHA2_create_in_224 -#define Hacl_Streaming_SHA2_create_in_512 python_hashlib_Hacl_Streaming_SHA2_create_in_512 -#define Hacl_Streaming_SHA2_create_in_384 python_hashlib_Hacl_Streaming_SHA2_create_in_384 -#define Hacl_Streaming_SHA2_copy_256 python_hashlib_Hacl_Streaming_SHA2_copy_256 -#define Hacl_Streaming_SHA2_copy_224 python_hashlib_Hacl_Streaming_SHA2_copy_224 -#define Hacl_Streaming_SHA2_copy_512 python_hashlib_Hacl_Streaming_SHA2_copy_512 -#define Hacl_Streaming_SHA2_copy_384 python_hashlib_Hacl_Streaming_SHA2_copy_384 -#define Hacl_Streaming_SHA2_init_256 python_hashlib_Hacl_Streaming_SHA2_init_256 -#define Hacl_Streaming_SHA2_init_224 python_hashlib_Hacl_Streaming_SHA2_init_224 -#define Hacl_Streaming_SHA2_init_512 python_hashlib_Hacl_Streaming_SHA2_init_512 -#define Hacl_Streaming_SHA2_init_384 python_hashlib_Hacl_Streaming_SHA2_init_384 +#define Hacl_Hash_SHA2_state_sha2_224_s python_hashlib_Hacl_Hash_SHA2_state_sha2_224_s +#define Hacl_Hash_SHA2_state_sha2_224 python_hashlib_Hacl_Hash_SHA2_state_sha2_224 +#define Hacl_Hash_SHA2_state_sha2_256 python_hashlib_Hacl_Hash_SHA2_state_sha2_256 +#define Hacl_Hash_SHA2_state_sha2_384_s python_hashlib_Hacl_Hash_SHA2_state_sha2_384_s +#define Hacl_Hash_SHA2_state_sha2_384 python_hashlib_Hacl_Hash_SHA2_state_sha2_384 +#define Hacl_Hash_SHA2_state_sha2_512 python_hashlib_Hacl_Hash_SHA2_state_sha2_512 +#define Hacl_Hash_SHA2_malloc_256 python_hashlib_Hacl_Hash_SHA2_malloc_256 +#define Hacl_Hash_SHA2_malloc_224 python_hashlib_Hacl_Hash_SHA2_malloc_224 +#define Hacl_Hash_SHA2_malloc_512 python_hashlib_Hacl_Hash_SHA2_malloc_512 +#define Hacl_Hash_SHA2_malloc_384 python_hashlib_Hacl_Hash_SHA2_malloc_384 +#define Hacl_Hash_SHA2_copy_256 python_hashlib_Hacl_Hash_SHA2_copy_256 +#define Hacl_Hash_SHA2_copy_224 python_hashlib_Hacl_Hash_SHA2_copy_224 +#define Hacl_Hash_SHA2_copy_512 python_hashlib_Hacl_Hash_SHA2_copy_512 +#define Hacl_Hash_SHA2_copy_384 python_hashlib_Hacl_Hash_SHA2_copy_384 +#define Hacl_Hash_SHA2_init_256 python_hashlib_Hacl_Hash_SHA2_init_256 +#define Hacl_Hash_SHA2_init_224 python_hashlib_Hacl_Hash_SHA2_init_224 +#define Hacl_Hash_SHA2_init_512 python_hashlib_Hacl_Hash_SHA2_init_512 +#define Hacl_Hash_SHA2_init_384 python_hashlib_Hacl_Hash_SHA2_init_384 #define Hacl_SHA2_Scalar32_sha512_init python_hashlib_Hacl_SHA2_Scalar32_sha512_init -#define Hacl_Streaming_SHA2_update_256 python_hashlib_Hacl_Streaming_SHA2_update_256 -#define Hacl_Streaming_SHA2_update_224 python_hashlib_Hacl_Streaming_SHA2_update_224 -#define Hacl_Streaming_SHA2_update_512 python_hashlib_Hacl_Streaming_SHA2_update_512 -#define Hacl_Streaming_SHA2_update_384 python_hashlib_Hacl_Streaming_SHA2_update_384 -#define Hacl_Streaming_SHA2_finish_256 python_hashlib_Hacl_Streaming_SHA2_finish_256 -#define Hacl_Streaming_SHA2_finish_224 python_hashlib_Hacl_Streaming_SHA2_finish_224 -#define Hacl_Streaming_SHA2_finish_512 python_hashlib_Hacl_Streaming_SHA2_finish_512 -#define Hacl_Streaming_SHA2_finish_384 python_hashlib_Hacl_Streaming_SHA2_finish_384 -#define Hacl_Streaming_SHA2_free_256 python_hashlib_Hacl_Streaming_SHA2_free_256 -#define Hacl_Streaming_SHA2_free_224 python_hashlib_Hacl_Streaming_SHA2_free_224 -#define Hacl_Streaming_SHA2_free_512 python_hashlib_Hacl_Streaming_SHA2_free_512 -#define Hacl_Streaming_SHA2_free_384 python_hashlib_Hacl_Streaming_SHA2_free_384 -#define Hacl_Streaming_SHA2_sha256 python_hashlib_Hacl_Streaming_SHA2_sha256 -#define Hacl_Streaming_SHA2_sha224 python_hashlib_Hacl_Streaming_SHA2_sha224 -#define Hacl_Streaming_SHA2_sha512 python_hashlib_Hacl_Streaming_SHA2_sha512 -#define Hacl_Streaming_SHA2_sha384 python_hashlib_Hacl_Streaming_SHA2_sha384 +#define Hacl_Hash_SHA2_update_256 python_hashlib_Hacl_Hash_SHA2_update_256 +#define Hacl_Hash_SHA2_update_224 python_hashlib_Hacl_Hash_SHA2_update_224 +#define Hacl_Hash_SHA2_update_512 python_hashlib_Hacl_Hash_SHA2_update_512 +#define Hacl_Hash_SHA2_update_384 python_hashlib_Hacl_Hash_SHA2_update_384 +#define Hacl_Hash_SHA2_digest_256 python_hashlib_Hacl_Hash_SHA2_digest_256 +#define Hacl_Hash_SHA2_digest_224 python_hashlib_Hacl_Hash_SHA2_digest_224 +#define Hacl_Hash_SHA2_digest_512 python_hashlib_Hacl_Hash_SHA2_digest_512 +#define Hacl_Hash_SHA2_digest_384 python_hashlib_Hacl_Hash_SHA2_digest_384 +#define Hacl_Hash_SHA2_free_256 python_hashlib_Hacl_Hash_SHA2_free_256 +#define Hacl_Hash_SHA2_free_224 python_hashlib_Hacl_Hash_SHA2_free_224 +#define Hacl_Hash_SHA2_free_512 python_hashlib_Hacl_Hash_SHA2_free_512 +#define Hacl_Hash_SHA2_free_384 python_hashlib_Hacl_Hash_SHA2_free_384 +#define Hacl_Hash_SHA2_sha256 python_hashlib_Hacl_Hash_SHA2_sha256 +#define Hacl_Hash_SHA2_sha224 python_hashlib_Hacl_Hash_SHA2_sha224 +#define Hacl_Hash_SHA2_sha512 python_hashlib_Hacl_Hash_SHA2_sha512 +#define Hacl_Hash_SHA2_sha384 python_hashlib_Hacl_Hash_SHA2_sha384 -#define Hacl_Streaming_MD5_legacy_create_in python_hashlib_Hacl_Streaming_MD5_legacy_create_in -#define Hacl_Streaming_MD5_legacy_init python_hashlib_Hacl_Streaming_MD5_legacy_init -#define Hacl_Streaming_MD5_legacy_update python_hashlib_Hacl_Streaming_MD5_legacy_update -#define Hacl_Streaming_MD5_legacy_finish python_hashlib_Hacl_Streaming_MD5_legacy_finish -#define Hacl_Streaming_MD5_legacy_free python_hashlib_Hacl_Streaming_MD5_legacy_free -#define Hacl_Streaming_MD5_legacy_copy python_hashlib_Hacl_Streaming_MD5_legacy_copy -#define Hacl_Streaming_MD5_legacy_hash python_hashlib_Hacl_Streaming_MD5_legacy_hash +#define Hacl_Hash_MD5_malloc python_hashlib_Hacl_Hash_MD5_malloc +#define Hacl_Hash_MD5_init python_hashlib_Hacl_Hash_MD5_init +#define Hacl_Hash_MD5_update python_hashlib_Hacl_Hash_MD5_update +#define Hacl_Hash_MD5_digest python_hashlib_Hacl_Hash_MD5_digest +#define Hacl_Hash_MD5_free python_hashlib_Hacl_Hash_MD5_free +#define Hacl_Hash_MD5_copy python_hashlib_Hacl_Hash_MD5_copy +#define Hacl_Hash_MD5_hash python_hashlib_Hacl_Hash_MD5_hash -#define Hacl_Streaming_SHA1_legacy_create_in python_hashlib_Hacl_Streaming_SHA1_legacy_create_in -#define Hacl_Streaming_SHA1_legacy_init python_hashlib_Hacl_Streaming_SHA1_legacy_init -#define Hacl_Streaming_SHA1_legacy_update python_hashlib_Hacl_Streaming_SHA1_legacy_update -#define Hacl_Streaming_SHA1_legacy_finish python_hashlib_Hacl_Streaming_SHA1_legacy_finish -#define Hacl_Streaming_SHA1_legacy_free python_hashlib_Hacl_Streaming_SHA1_legacy_free -#define Hacl_Streaming_SHA1_legacy_copy python_hashlib_Hacl_Streaming_SHA1_legacy_copy -#define Hacl_Streaming_SHA1_legacy_hash python_hashlib_Hacl_Streaming_SHA1_legacy_hash +#define Hacl_Hash_SHA1_malloc python_hashlib_Hacl_Hash_SHA1_malloc +#define Hacl_Hash_SHA1_init python_hashlib_Hacl_Hash_SHA1_init +#define Hacl_Hash_SHA1_update python_hashlib_Hacl_Hash_SHA1_update +#define Hacl_Hash_SHA1_digest python_hashlib_Hacl_Hash_SHA1_digest +#define Hacl_Hash_SHA1_free python_hashlib_Hacl_Hash_SHA1_free +#define Hacl_Hash_SHA1_copy python_hashlib_Hacl_Hash_SHA1_copy +#define Hacl_Hash_SHA1_hash python_hashlib_Hacl_Hash_SHA1_hash #define Hacl_Hash_SHA3_update_last_sha3 python_hashlib_Hacl_Hash_SHA3_update_last_sha3 #define Hacl_Hash_SHA3_update_multi_sha3 python_hashlib_Hacl_Hash_SHA3_update_multi_sha3 @@ -72,15 +74,139 @@ #define Hacl_SHA3_sha3_512 python_hashlib_Hacl_SHA3_sha3_512 #define Hacl_SHA3_shake128_hacl python_hashlib_Hacl_SHA3_shake128_hacl #define Hacl_SHA3_shake256_hacl python_hashlib_Hacl_SHA3_shake256_hacl -#define Hacl_Streaming_Keccak_block_len python_hashlib_Hacl_Streaming_Keccak_block_len -#define Hacl_Streaming_Keccak_copy python_hashlib_Hacl_Streaming_Keccak_copy -#define Hacl_Streaming_Keccak_finish python_hashlib_Hacl_Streaming_Keccak_finish -#define Hacl_Streaming_Keccak_free python_hashlib_Hacl_Streaming_Keccak_free -#define Hacl_Streaming_Keccak_get_alg python_hashlib_Hacl_Streaming_Keccak_get_alg -#define Hacl_Streaming_Keccak_hash_len python_hashlib_Hacl_Streaming_Keccak_hash_len -#define Hacl_Streaming_Keccak_is_shake python_hashlib_Hacl_Streaming_Keccak_is_shake -#define Hacl_Streaming_Keccak_malloc python_hashlib_Hacl_Streaming_Keccak_malloc -#define Hacl_Streaming_Keccak_reset python_hashlib_Hacl_Streaming_Keccak_reset -#define Hacl_Streaming_Keccak_update python_hashlib_Hacl_Streaming_Keccak_update +#define Hacl_Hash_SHA3_block_len python_hashlib_Hacl_Hash_SHA3_block_len +#define Hacl_Hash_SHA3_copy python_hashlib_Hacl_Hash_SHA3_copy +#define Hacl_Hash_SHA3_digest python_hashlib_Hacl_Hash_SHA3_digest +#define Hacl_Hash_SHA3_free python_hashlib_Hacl_Hash_SHA3_free +#define Hacl_Hash_SHA3_get_alg python_hashlib_Hacl_Hash_SHA3_get_alg +#define Hacl_Hash_SHA3_hash_len python_hashlib_Hacl_Hash_SHA3_hash_len +#define Hacl_Hash_SHA3_is_shake python_hashlib_Hacl_Hash_SHA3_is_shake +#define Hacl_Hash_SHA3_malloc python_hashlib_Hacl_Hash_SHA3_malloc +#define Hacl_Hash_SHA3_reset python_hashlib_Hacl_Hash_SHA3_reset +#define Hacl_Hash_SHA3_update python_hashlib_Hacl_Hash_SHA3_update +#define Hacl_Hash_SHA3_squeeze python_hashlib_Hacl_Hash_SHA3_squeeze + +#define Hacl_Hash_Blake2b_Simd256_copy python_hashlib_Hacl_Hash_Blake2b_Simd256_copy +#define Hacl_Hash_Blake2b_Simd256_digest python_hashlib_Hacl_Hash_Blake2b_Simd256_digest +#define Hacl_Hash_Blake2b_Simd256_finish python_hashlib_Hacl_Hash_Blake2b_Simd256_finish +#define Hacl_Hash_Blake2b_Simd256_free python_hashlib_Hacl_Hash_Blake2b_Simd256_free +#define Hacl_Hash_Blake2b_Simd256_hash_with_key python_hashlib_Hacl_Hash_Blake2b_Simd256_hash_with_key +#define Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params python_hashlib_Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params +#define Hacl_Hash_Blake2b_Simd256_info python_hashlib_Hacl_Hash_Blake2b_Simd256_info +#define Hacl_Hash_Blake2b_Simd256_init python_hashlib_Hacl_Hash_Blake2b_Simd256_init +#define Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32 python_hashlib_Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32 +#define Hacl_Hash_Blake2b_Simd256_malloc python_hashlib_Hacl_Hash_Blake2b_Simd256_malloc +#define Hacl_Hash_Blake2b_Simd256_malloc_with_key python_hashlib_Hacl_Hash_Blake2b_Simd256_malloc_with_key +#define Hacl_Hash_Blake2b_Simd256_malloc_with_key0 python_hashlib_Hacl_Hash_Blake2b_Simd256_malloc_with_key0 +#define Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key python_hashlib_Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key +#define Hacl_Hash_Blake2b_Simd256_reset python_hashlib_Hacl_Hash_Blake2b_Simd256_reset +#define Hacl_Hash_Blake2b_Simd256_reset_with_key python_hashlib_Hacl_Hash_Blake2b_Simd256_reset_with_key +#define Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params python_hashlib_Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params +#define Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32 python_hashlib_Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32 +#define Hacl_Hash_Blake2b_Simd256_update python_hashlib_Hacl_Hash_Blake2b_Simd256_update +#define Hacl_Hash_Blake2b_Simd256_update_last python_hashlib_Hacl_Hash_Blake2b_Simd256_update_last +#define Hacl_Hash_Blake2b_Simd256_update_multi python_hashlib_Hacl_Hash_Blake2b_Simd256_update_multi +#define Hacl_Hash_Blake2b_copy python_hashlib_Hacl_Hash_Blake2b_copy +#define Hacl_Hash_Blake2b_digest python_hashlib_Hacl_Hash_Blake2b_digest +#define Hacl_Hash_Blake2b_finish python_hashlib_Hacl_Hash_Blake2b_finish +#define Hacl_Hash_Blake2b_free python_hashlib_Hacl_Hash_Blake2b_free +#define Hacl_Hash_Blake2b_hash_with_key python_hashlib_Hacl_Hash_Blake2b_hash_with_key +#define Hacl_Hash_Blake2b_hash_with_key_and_params python_hashlib_Hacl_Hash_Blake2b_hash_with_key_and_params +#define Hacl_Hash_Blake2b_info python_hashlib_Hacl_Hash_Blake2b_info +#define Hacl_Hash_Blake2b_init python_hashlib_Hacl_Hash_Blake2b_init +#define Hacl_Hash_Blake2b_malloc python_hashlib_Hacl_Hash_Blake2b_malloc +#define Hacl_Hash_Blake2b_malloc_with_key python_hashlib_Hacl_Hash_Blake2b_malloc_with_key +#define Hacl_Hash_Blake2b_malloc_with_params_and_key python_hashlib_Hacl_Hash_Blake2b_malloc_with_params_and_key +#define Hacl_Hash_Blake2b_reset python_hashlib_Hacl_Hash_Blake2b_reset +#define Hacl_Hash_Blake2b_reset_with_key python_hashlib_Hacl_Hash_Blake2b_reset_with_key +#define Hacl_Hash_Blake2b_reset_with_key_and_params python_hashlib_Hacl_Hash_Blake2b_reset_with_key_and_params +#define Hacl_Hash_Blake2b_update python_hashlib_Hacl_Hash_Blake2b_update +#define Hacl_Hash_Blake2b_update_last python_hashlib_Hacl_Hash_Blake2b_update_last +#define Hacl_Hash_Blake2b_update_multi python_hashlib_Hacl_Hash_Blake2b_update_multi +#define Hacl_Hash_Blake2s_Simd128_copy python_hashlib_Hacl_Hash_Blake2s_Simd128_copy +#define Hacl_Hash_Blake2s_Simd128_digest python_hashlib_Hacl_Hash_Blake2s_Simd128_digest +#define Hacl_Hash_Blake2s_Simd128_finish python_hashlib_Hacl_Hash_Blake2s_Simd128_finish +#define Hacl_Hash_Blake2s_Simd128_free python_hashlib_Hacl_Hash_Blake2s_Simd128_free +#define Hacl_Hash_Blake2s_Simd128_hash_with_key python_hashlib_Hacl_Hash_Blake2s_Simd128_hash_with_key +#define Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params python_hashlib_Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params +#define Hacl_Hash_Blake2s_Simd128_info python_hashlib_Hacl_Hash_Blake2s_Simd128_info +#define Hacl_Hash_Blake2s_Simd128_init python_hashlib_Hacl_Hash_Blake2s_Simd128_init +#define Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32 python_hashlib_Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32 +#define Hacl_Hash_Blake2s_Simd128_malloc python_hashlib_Hacl_Hash_Blake2s_Simd128_malloc +#define Hacl_Hash_Blake2s_Simd128_malloc_with_key python_hashlib_Hacl_Hash_Blake2s_Simd128_malloc_with_key +#define Hacl_Hash_Blake2s_Simd128_malloc_with_key0 python_hashlib_Hacl_Hash_Blake2s_Simd128_malloc_with_key0 +#define Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key python_hashlib_Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key +#define Hacl_Hash_Blake2s_Simd128_reset python_hashlib_Hacl_Hash_Blake2s_Simd128_reset +#define Hacl_Hash_Blake2s_Simd128_reset_with_key python_hashlib_Hacl_Hash_Blake2s_Simd128_reset_with_key +#define Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params python_hashlib_Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params +#define Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32 python_hashlib_Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32 +#define Hacl_Hash_Blake2s_Simd128_update python_hashlib_Hacl_Hash_Blake2s_Simd128_update +#define Hacl_Hash_Blake2s_Simd128_update_last python_hashlib_Hacl_Hash_Blake2s_Simd128_update_last +#define Hacl_Hash_Blake2s_Simd128_update_multi python_hashlib_Hacl_Hash_Blake2s_Simd128_update_multi +#define Hacl_Hash_Blake2s_copy python_hashlib_Hacl_Hash_Blake2s_copy +#define Hacl_Hash_Blake2s_digest python_hashlib_Hacl_Hash_Blake2s_digest +#define Hacl_Hash_Blake2s_finish python_hashlib_Hacl_Hash_Blake2s_finish +#define Hacl_Hash_Blake2s_free python_hashlib_Hacl_Hash_Blake2s_free +#define Hacl_Hash_Blake2s_hash_with_key python_hashlib_Hacl_Hash_Blake2s_hash_with_key +#define Hacl_Hash_Blake2s_hash_with_key_and_params python_hashlib_Hacl_Hash_Blake2s_hash_with_key_and_params +#define Hacl_Hash_Blake2s_info python_hashlib_Hacl_Hash_Blake2s_info +#define Hacl_Hash_Blake2s_init python_hashlib_Hacl_Hash_Blake2s_init +#define Hacl_Hash_Blake2s_malloc python_hashlib_Hacl_Hash_Blake2s_malloc +#define Hacl_Hash_Blake2s_malloc_with_key python_hashlib_Hacl_Hash_Blake2s_malloc_with_key +#define Hacl_Hash_Blake2s_malloc_with_params_and_key python_hashlib_Hacl_Hash_Blake2s_malloc_with_params_and_key +#define Hacl_Hash_Blake2s_reset python_hashlib_Hacl_Hash_Blake2s_reset +#define Hacl_Hash_Blake2s_reset_with_key python_hashlib_Hacl_Hash_Blake2s_reset_with_key +#define Hacl_Hash_Blake2s_reset_with_key_and_params python_hashlib_Hacl_Hash_Blake2s_reset_with_key_and_params +#define Hacl_Hash_Blake2s_update python_hashlib_Hacl_Hash_Blake2s_update +#define Hacl_Hash_Blake2s_update_last python_hashlib_Hacl_Hash_Blake2s_update_last +#define Hacl_Hash_Blake2s_update_multi python_hashlib_Hacl_Hash_Blake2s_update_multi +#define Hacl_Hash_MD5_finish python_hashlib_Hacl_Hash_MD5_finish +#define Hacl_Hash_MD5_hash_oneshot python_hashlib_Hacl_Hash_MD5_hash_oneshot +#define Hacl_Hash_MD5_reset python_hashlib_Hacl_Hash_MD5_reset +#define Hacl_Hash_MD5_update_last python_hashlib_Hacl_Hash_MD5_update_last +#define Hacl_Hash_MD5_update_multi python_hashlib_Hacl_Hash_MD5_update_multi +#define Hacl_Hash_SHA1_finish python_hashlib_Hacl_Hash_SHA1_finish +#define Hacl_Hash_SHA1_hash_oneshot python_hashlib_Hacl_Hash_SHA1_hash_oneshot +#define Hacl_Hash_SHA1_reset python_hashlib_Hacl_Hash_SHA1_reset +#define Hacl_Hash_SHA1_update_last python_hashlib_Hacl_Hash_SHA1_update_last +#define Hacl_Hash_SHA1_update_multi python_hashlib_Hacl_Hash_SHA1_update_multi +#define Hacl_Hash_SHA2_hash_224 python_hashlib_Hacl_Hash_SHA2_hash_224 +#define Hacl_Hash_SHA2_hash_256 python_hashlib_Hacl_Hash_SHA2_hash_256 +#define Hacl_Hash_SHA2_hash_384 python_hashlib_Hacl_Hash_SHA2_hash_384 +#define Hacl_Hash_SHA2_hash_512 python_hashlib_Hacl_Hash_SHA2_hash_512 +#define Hacl_Hash_SHA2_reset_224 python_hashlib_Hacl_Hash_SHA2_reset_224 +#define Hacl_Hash_SHA2_reset_256 python_hashlib_Hacl_Hash_SHA2_reset_256 +#define Hacl_Hash_SHA2_reset_384 python_hashlib_Hacl_Hash_SHA2_reset_384 +#define Hacl_Hash_SHA2_reset_512 python_hashlib_Hacl_Hash_SHA2_reset_512 +#define Hacl_Hash_SHA2_sha224_finish python_hashlib_Hacl_Hash_SHA2_sha224_finish +#define Hacl_Hash_SHA2_sha224_init python_hashlib_Hacl_Hash_SHA2_sha224_init +#define Hacl_Hash_SHA2_sha224_update_last python_hashlib_Hacl_Hash_SHA2_sha224_update_last +#define Hacl_Hash_SHA2_sha256_finish python_hashlib_Hacl_Hash_SHA2_sha256_finish +#define Hacl_Hash_SHA2_sha256_init python_hashlib_Hacl_Hash_SHA2_sha256_init +#define Hacl_Hash_SHA2_sha256_update_last python_hashlib_Hacl_Hash_SHA2_sha256_update_last +#define Hacl_Hash_SHA2_sha256_update_nblocks python_hashlib_Hacl_Hash_SHA2_sha256_update_nblocks +#define Hacl_Hash_SHA2_sha384_finish python_hashlib_Hacl_Hash_SHA2_sha384_finish +#define Hacl_Hash_SHA2_sha384_init python_hashlib_Hacl_Hash_SHA2_sha384_init +#define Hacl_Hash_SHA2_sha384_update_last python_hashlib_Hacl_Hash_SHA2_sha384_update_last +#define Hacl_Hash_SHA2_sha384_update_nblocks python_hashlib_Hacl_Hash_SHA2_sha384_update_nblocks +#define Hacl_Hash_SHA2_sha512_finish python_hashlib_Hacl_Hash_SHA2_sha512_finish +#define Hacl_Hash_SHA2_sha512_init python_hashlib_Hacl_Hash_SHA2_sha512_init +#define Hacl_Hash_SHA2_sha512_update_last python_hashlib_Hacl_Hash_SHA2_sha512_update_last +#define Hacl_Hash_SHA2_sha512_update_nblocks python_hashlib_Hacl_Hash_SHA2_sha512_update_nblocks +#define Hacl_Hash_SHA3_absorb_inner_32 python_hashlib_Hacl_Hash_SHA3_absorb_inner_32 +#define Hacl_Hash_SHA3_keccak_piln python_hashlib_Hacl_Hash_SHA3_keccak_piln +#define Hacl_Hash_SHA3_keccak_rndc python_hashlib_Hacl_Hash_SHA3_keccak_rndc +#define Hacl_Hash_SHA3_keccak_rotc python_hashlib_Hacl_Hash_SHA3_keccak_rotc +#define Hacl_Hash_SHA3_sha3_224 python_hashlib_Hacl_Hash_SHA3_sha3_224 +#define Hacl_Hash_SHA3_sha3_256 python_hashlib_Hacl_Hash_SHA3_sha3_256 +#define Hacl_Hash_SHA3_sha3_384 python_hashlib_Hacl_Hash_SHA3_sha3_384 +#define Hacl_Hash_SHA3_sha3_512 python_hashlib_Hacl_Hash_SHA3_sha3_512 +#define Hacl_Hash_SHA3_shake128 python_hashlib_Hacl_Hash_SHA3_shake128 +#define Hacl_Hash_SHA3_shake128_absorb_final python_hashlib_Hacl_Hash_SHA3_shake128_absorb_final +#define Hacl_Hash_SHA3_shake128_absorb_nblocks python_hashlib_Hacl_Hash_SHA3_shake128_absorb_nblocks +#define Hacl_Hash_SHA3_shake128_squeeze_nblocks python_hashlib_Hacl_Hash_SHA3_shake128_squeeze_nblocks +#define Hacl_Hash_SHA3_shake256 python_hashlib_Hacl_Hash_SHA3_shake256 +#define Hacl_Hash_SHA3_state_free python_hashlib_Hacl_Hash_SHA3_state_free +#define Hacl_Hash_SHA3_state_malloc python_hashlib_Hacl_Hash_SHA3_state_malloc #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index c1b3e37f3afb9d..6234fea9f17bc7 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=521af282fdf6d60227335120f18ae9309a4b8e8c +expected_hacl_star_rev=315a9e491d2bc347b9dae99e0ea506995ea84d9d hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -40,19 +40,35 @@ fi declare -a dist_files dist_files=( - Hacl_Hash_SHA2.h Hacl_Streaming_Types.h - Hacl_Hash_SHA1.h - internal/Hacl_Hash_SHA1.h Hacl_Hash_MD5.h + Hacl_Hash_SHA1.h + Hacl_Hash_SHA2.h Hacl_Hash_SHA3.h + Hacl_Hash_Blake2b.h + Hacl_Hash_Blake2s.h + Hacl_Hash_Blake2b_Simd256.h + Hacl_Hash_Blake2s_Simd128.h internal/Hacl_Hash_MD5.h - internal/Hacl_Hash_SHA3.h - Hacl_Hash_SHA2.c + internal/Hacl_Hash_SHA1.h internal/Hacl_Hash_SHA2.h - Hacl_Hash_SHA1.c + internal/Hacl_Hash_SHA3.h + internal/Hacl_Hash_Blake2b.h + internal/Hacl_Hash_Blake2s.h + internal/Hacl_Hash_Blake2b_Simd256.h + internal/Hacl_Hash_Blake2s_Simd128.h + internal/Hacl_Impl_Blake2_Constants.h Hacl_Hash_MD5.c + Hacl_Hash_SHA1.c + Hacl_Hash_SHA2.c Hacl_Hash_SHA3.c + Hacl_Hash_Blake2b.c + Hacl_Hash_Blake2s.c + Hacl_Hash_Blake2b_Simd256.c + Hacl_Hash_Blake2s_Simd128.c + libintvector.h + lib_memzero0.h + Lib_Memzero0.c ) declare -a include_files @@ -127,13 +143,17 @@ $sed -i -z 's!\(extern\|typedef\)[^;]*;\n\n!!g' include/krml/FStar_UInt_8_16_32_ $sed -i 's!#include.*Hacl_Krmllib.h"!!g' "${all_files[@]}" # Use globally unique names for the Hacl_ C APIs to avoid linkage conflicts. -$sed -i -z 's!#include \n!#include \n#include "python_hacl_namespaces.h"\n!' Hacl_Hash_SHA2.h +$sed -i -z 's!#include \n!#include \n#include "python_hacl_namespaces.h"\n!' Hacl_Hash_*.h # Finally, we remove a bunch of ifdefs from target.h that are, again, useful in # the general case, but not exercised by the subset of HACL* that we vendor. -$sed -z -i 's!#ifndef KRML_\(PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h +$sed -z -i 's!#ifndef KRML_\(HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h +# Step 3: trim whitespace (for the linter) + +find . -name '*.c' -or -name '*.h' | xargs $sed -i 's![[:space:]]\+$!!' + echo "Updated; verify all is okay using git diff and git status." diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 0e230f332ff6cb..2c9a9feecc79f0 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -25,7 +25,6 @@ #include #include "Python.h" #include "pycore_hashtable.h" -#include "pycore_pyhash.h" // _Py_HashBytes() #include "pycore_strhex.h" // _Py_strhex() #include "hashlib.h" @@ -45,9 +44,15 @@ #define MUNCH_SIZE INT_MAX #define PY_OPENSSL_HAS_SCRYPT 1 +#if defined(NID_sha3_224) && defined(NID_sha3_256) && defined(NID_sha3_384) && defined(NID_sha3_512) #define PY_OPENSSL_HAS_SHA3 1 +#endif +#if defined(NID_shake128) || defined(NID_shake256) #define PY_OPENSSL_HAS_SHAKE 1 +#endif +#if defined(NID_blake2s256) || defined(NID_blake2b512) #define PY_OPENSSL_HAS_BLAKE2 1 +#endif #if OPENSSL_VERSION_NUMBER >= 0x30000000L #define PY_EVP_MD EVP_MD @@ -88,22 +93,45 @@ typedef struct { PY_EVP_MD *evp_nosecurity; } py_hashentry_t; +// Fundamental to TLS, assumed always present in any libcrypto: #define Py_hash_md5 "md5" #define Py_hash_sha1 "sha1" #define Py_hash_sha224 "sha224" #define Py_hash_sha256 "sha256" #define Py_hash_sha384 "sha384" #define Py_hash_sha512 "sha512" -#define Py_hash_sha512_224 "sha512_224" -#define Py_hash_sha512_256 "sha512_256" -#define Py_hash_sha3_224 "sha3_224" -#define Py_hash_sha3_256 "sha3_256" -#define Py_hash_sha3_384 "sha3_384" -#define Py_hash_sha3_512 "sha3_512" -#define Py_hash_shake_128 "shake_128" -#define Py_hash_shake_256 "shake_256" -#define Py_hash_blake2s "blake2s" -#define Py_hash_blake2b "blake2b" + +// Not all OpenSSL-like libcrypto libraries provide these: +#if defined(NID_sha512_224) +# define Py_hash_sha512_224 "sha512_224" +#endif +#if defined(NID_sha512_256) +# define Py_hash_sha512_256 "sha512_256" +#endif +#if defined(NID_sha3_224) +# define Py_hash_sha3_224 "sha3_224" +#endif +#if defined(NID_sha3_256) +# define Py_hash_sha3_256 "sha3_256" +#endif +#if defined(NID_sha3_384) +# define Py_hash_sha3_384 "sha3_384" +#endif +#if defined(NID_sha3_512) +# define Py_hash_sha3_512 "sha3_512" +#endif +#if defined(NID_shake128) +# define Py_hash_shake_128 "shake_128" +#endif +#if defined(NID_shake256) +# define Py_hash_shake_256 "shake_256" +#endif +#if defined(NID_blake2s256) +# define Py_hash_blake2s "blake2s" +#endif +#if defined(NID_blake2b512) +# define Py_hash_blake2b "blake2b" +#endif #define PY_HASH_ENTRY(py_name, py_alias, ossl_name, ossl_nid) \ {py_name, py_alias, ossl_name, ossl_nid, 0, NULL, NULL} @@ -119,25 +147,45 @@ static const py_hashentry_t py_hashes[] = { PY_HASH_ENTRY(Py_hash_sha384, "SHA384", SN_sha384, NID_sha384), PY_HASH_ENTRY(Py_hash_sha512, "SHA512", SN_sha512, NID_sha512), /* truncated sha2 */ +#ifdef Py_hash_sha512_224 PY_HASH_ENTRY(Py_hash_sha512_224, "SHA512_224", SN_sha512_224, NID_sha512_224), +#endif +#ifdef Py_hash_sha512_256 PY_HASH_ENTRY(Py_hash_sha512_256, "SHA512_256", SN_sha512_256, NID_sha512_256), +#endif /* sha3 */ +#ifdef Py_hash_sha3_224 PY_HASH_ENTRY(Py_hash_sha3_224, NULL, SN_sha3_224, NID_sha3_224), +#endif +#ifdef Py_hash_sha3_256 PY_HASH_ENTRY(Py_hash_sha3_256, NULL, SN_sha3_256, NID_sha3_256), +#endif +#ifdef Py_hash_sha3_384 PY_HASH_ENTRY(Py_hash_sha3_384, NULL, SN_sha3_384, NID_sha3_384), +#endif +#ifdef Py_hash_sha3_512 PY_HASH_ENTRY(Py_hash_sha3_512, NULL, SN_sha3_512, NID_sha3_512), +#endif /* sha3 shake */ +#ifdef Py_hash_shake_128 PY_HASH_ENTRY(Py_hash_shake_128, NULL, SN_shake128, NID_shake128), +#endif +#ifdef Py_hash_shake_256 PY_HASH_ENTRY(Py_hash_shake_256, NULL, SN_shake256, NID_shake256), +#endif /* blake2 digest */ +#ifdef Py_hash_blake2s PY_HASH_ENTRY(Py_hash_blake2s, "blake2s256", SN_blake2s256, NID_blake2s256), +#endif +#ifdef Py_hash_blake2b PY_HASH_ENTRY(Py_hash_blake2b, "blake2b512", SN_blake2b512, NID_blake2b512), +#endif PY_HASH_ENTRY(NULL, NULL, NULL, 0), }; static Py_uhash_t py_hashentry_t_hash_name(const void *key) { - return _Py_HashBytes(key, strlen((const char *)key)); + return Py_HashBuffer(key, strlen((const char *)key)); } static int @@ -2240,6 +2288,7 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_init_constructors}, {Py_mod_exec, hashlib_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 9d4ec256ee9e3e..80fe9cff98509d 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -585,7 +585,7 @@ non-existing elements are considered to be infinite. The interesting\n\ property of a heap is that a[0] is always its smallest element.\n" "\n\ The strange invariant above is meant to be an efficient memory\n\ -representation for a tournament. The numbers below are `k', not a[k]:\n\ +representation for a tournament. The numbers below are 'k', not a[k]:\n\ \n\ 0\n\ \n\ @@ -598,7 +598,7 @@ representation for a tournament. The numbers below are `k', not a[k]:\n\ 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30\n\ \n\ \n\ -In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In\n\ +In the tree above, each cell 'k' is topping '2*k+1' and '2*k+2'. In\n\ a usual binary tournament we see in sports, each cell is the winner\n\ over the two cells it tops, and we can trace the winner down the tree\n\ to see all opponents s/he had. However, in many computer applications\n\ @@ -653,7 +653,7 @@ vanishes, you switch heaps and start a new run. Clever and quite\n\ effective!\n\ \n\ In a word, heaps are useful memory structures to know. I use them in\n\ -a few applications, and I think it is good to keep a `heap' module\n\ +a few applications, and I think it is good to keep a 'heap' module\n\ around. :-)\n" "\n\ --------------------\n\ @@ -681,6 +681,7 @@ heapq_exec(PyObject *m) static struct PyModuleDef_Slot heapq_slots[] = { {Py_mod_exec, heapq_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_interpchannelsmodule.c similarity index 85% rename from Modules/_xxinterpchannelsmodule.c rename to Modules/_interpchannelsmodule.c index 4e9b8a82a3f630..8e6b21db76e01c 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -6,9 +6,9 @@ #endif #include "Python.h" -#include "interpreteridobject.h" #include "pycore_crossinterp.h" // struct _xid #include "pycore_interp.h" // _PyInterpreterState_LookUpID() +#include "pycore_pystate.h" // _PyInterpreterState_GetIDObject() #ifdef MS_WINDOWS #define WIN32_LEAN_AND_MEAN @@ -17,11 +17,18 @@ #include // sched_yield() #endif +#define REGISTERS_HEAP_TYPES +#define HAS_UNBOUND_ITEMS +#include "_interpreters_common.h" +#undef HAS_UNBOUND_ITEMS +#undef REGISTERS_HEAP_TYPES + /* This module has the following process-global state: _globals (static struct globals): + mutex (PyMutex) module_count (int) channels (struct _channels): numopen (int64_t) @@ -80,7 +87,9 @@ channel's queue, which are safely managed via the _PyCrossInterpreterData_*() API.. The module does not create any objects that are shared globally. */ -#define MODULE_NAME "_xxinterpchannels" +#define MODULE_NAME _interpchannels +#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) +#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) #define GLOBAL_MALLOC(TYPE) \ @@ -89,38 +98,6 @@ API.. The module does not create any objects that are shared globally. PyMem_RawFree(VAR) -struct xid_class_registry { - size_t count; -#define MAX_XID_CLASSES 5 - struct { - PyTypeObject *cls; - } added[MAX_XID_CLASSES]; -}; - -static int -register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared, - struct xid_class_registry *classes) -{ - int res = _PyCrossInterpreterData_RegisterClass(cls, shared); - if (res == 0) { - assert(classes->count < MAX_XID_CLASSES); - // The class has refs elsewhere, so we need to incref here. - classes->added[classes->count].cls = cls; - classes->count += 1; - } - return res; -} - -static void -clear_xid_class_registry(struct xid_class_registry *classes) -{ - while (classes->count > 0) { - classes->count -= 1; - PyTypeObject *cls = classes->added[classes->count].cls; - _PyCrossInterpreterData_UnregisterClass(cls); - } -} - #define XID_IGNORE_EXC 1 #define XID_FREE 2 @@ -167,7 +144,7 @@ _get_current_interp(void) static PyObject * _get_current_module(void) { - PyObject *name = PyUnicode_FromString(MODULE_NAME); + PyObject *name = PyUnicode_FromString(MODULE_NAME_STR); if (name == NULL) { return NULL; } @@ -217,29 +194,7 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) } #define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \ - add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE) - -static PyTypeObject * -add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared, - struct xid_class_registry *classes) -{ - PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, spec, NULL); - if (cls == NULL) { - return NULL; - } - if (PyModule_AddType(mod, cls) < 0) { - Py_DECREF(cls); - return NULL; - } - if (shared != NULL) { - if (register_xid_class(cls, shared, classes)) { - Py_DECREF(cls); - return NULL; - } - } - return cls; -} + add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE) static int wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) @@ -265,8 +220,6 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) /* module state *************************************************************/ typedef struct { - struct xid_class_registry xid_classes; - /* Added at runtime by interpreters module. */ PyTypeObject *send_channel_type; PyTypeObject *recv_channel_type; @@ -299,7 +252,7 @@ _get_current_module_state(void) if (mod == NULL) { // XXX import it? PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME " module not imported yet"); + MODULE_NAME_STR " module not imported yet"); return NULL; } module_state *state = get_module_state(mod); @@ -328,19 +281,33 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) return 0; } -static int -clear_module_state(module_state *state) +static void +clear_xid_types(module_state *state) { /* external types */ - Py_CLEAR(state->send_channel_type); - Py_CLEAR(state->recv_channel_type); + if (state->send_channel_type != NULL) { + (void)clear_xid_class(state->send_channel_type); + Py_CLEAR(state->send_channel_type); + } + if (state->recv_channel_type != NULL) { + (void)clear_xid_class(state->recv_channel_type); + Py_CLEAR(state->recv_channel_type); + } /* heap types */ - Py_CLEAR(state->ChannelInfoType); if (state->ChannelIDType != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); + (void)clear_xid_class(state->ChannelIDType); + Py_CLEAR(state->ChannelIDType); } - Py_CLEAR(state->ChannelIDType); +} + +static int +clear_module_state(module_state *state) +{ + clear_xid_types(state); + + /* heap types */ + Py_CLEAR(state->ChannelInfoType); /* exceptions */ Py_CLEAR(state->ChannelError); @@ -547,8 +514,14 @@ _waiting_finish_releasing(_waiting_t *waiting) struct _channelitem; typedef struct _channelitem { + /* The interpreter that added the item to the queue. + The actual bound interpid is found in item->data. + This is necessary because item->data might be NULL, + meaning the interpreter has been destroyed. */ + int64_t interpid; _PyCrossInterpreterData *data; _waiting_t *waiting; + int unboundop; struct _channelitem *next; } _channelitem; @@ -560,11 +533,22 @@ _channelitem_ID(_channelitem *item) static void _channelitem_init(_channelitem *item, - _PyCrossInterpreterData *data, _waiting_t *waiting) + int64_t interpid, _PyCrossInterpreterData *data, + _waiting_t *waiting, int unboundop) { + if (interpid < 0) { + interpid = _get_interpid(data); + } + else { + assert(data == NULL + || _PyCrossInterpreterData_INTERPID(data) < 0 + || interpid == _PyCrossInterpreterData_INTERPID(data)); + } *item = (_channelitem){ + .interpid = interpid, .data = data, .waiting = waiting, + .unboundop = unboundop, }; if (waiting != NULL) { waiting->itemid = _channelitem_ID(item); @@ -572,17 +556,15 @@ _channelitem_init(_channelitem *item, } static void -_channelitem_clear(_channelitem *item) +_channelitem_clear_data(_channelitem *item, int removed) { - item->next = NULL; - if (item->data != NULL) { // It was allocated in channel_send(). (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); item->data = NULL; } - if (item->waiting != NULL) { + if (item->waiting != NULL && removed) { if (item->waiting->status == WAITING_ACQUIRED) { _waiting_release(item->waiting, 0); } @@ -590,15 +572,23 @@ _channelitem_clear(_channelitem *item) } } +static void +_channelitem_clear(_channelitem *item) +{ + item->next = NULL; + _channelitem_clear_data(item, 1); +} + static _channelitem * -_channelitem_new(_PyCrossInterpreterData *data, _waiting_t *waiting) +_channelitem_new(int64_t interpid, _PyCrossInterpreterData *data, + _waiting_t *waiting, int unboundop) { _channelitem *item = GLOBAL_MALLOC(_channelitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _channelitem_init(item, data, waiting); + _channelitem_init(item, interpid, data, waiting, unboundop); return item; } @@ -621,17 +611,48 @@ _channelitem_free_all(_channelitem *item) static void _channelitem_popped(_channelitem *item, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting, + int *p_unboundop) { assert(item->waiting == NULL || item->waiting->status == WAITING_ACQUIRED); *p_data = item->data; *p_waiting = item->waiting; + *p_unboundop = item->unboundop; // We clear them here, so they won't be released in _channelitem_clear(). item->data = NULL; item->waiting = NULL; _channelitem_free(item); } +static int +_channelitem_clear_interpreter(_channelitem *item) +{ + assert(item->interpid >= 0); + if (item->data == NULL) { + // Its interpreter was already cleared (or it was never bound). + // For UNBOUND_REMOVE it should have been freed at that time. + assert(item->unboundop != UNBOUND_REMOVE); + return 0; + } + assert(_PyCrossInterpreterData_INTERPID(item->data) == item->interpid); + + switch (item->unboundop) { + case UNBOUND_REMOVE: + // The caller must free/clear it. + return 1; + case UNBOUND_ERROR: + case UNBOUND_REPLACE: + // We won't need the cross-interpreter data later + // so we completely throw it away. + _channelitem_clear_data(item, 0); + return 0; + default: + Py_FatalError("not reachable"); + return -1; + } +} + + typedef struct _channelqueue { int64_t count; _channelitem *first; @@ -670,9 +691,10 @@ _channelqueue_free(_channelqueue *queue) static int _channelqueue_put(_channelqueue *queue, - _PyCrossInterpreterData *data, _waiting_t *waiting) + int64_t interpid, _PyCrossInterpreterData *data, + _waiting_t *waiting, int unboundop) { - _channelitem *item = _channelitem_new(data, waiting); + _channelitem *item = _channelitem_new(interpid, data, waiting, unboundop); if (item == NULL) { return -1; } @@ -695,7 +717,8 @@ _channelqueue_put(_channelqueue *queue, static int _channelqueue_get(_channelqueue *queue, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting, + int *p_unboundop) { _channelitem *item = queue->first; if (item == NULL) { @@ -707,7 +730,7 @@ _channelqueue_get(_channelqueue *queue, } queue->count -= 1; - _channelitem_popped(item, p_data, p_waiting); + _channelitem_popped(item, p_data, p_waiting, p_unboundop); return 0; } @@ -773,7 +796,8 @@ _channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid, } queue->count -= 1; - _channelitem_popped(item, p_data, p_waiting); + int unboundop; + _channelitem_popped(item, p_data, p_waiting, &unboundop); } static void @@ -784,14 +808,17 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid) while (next != NULL) { _channelitem *item = next; next = item->next; - if (item->data->interpid == interpid) { + int remove = (item->interpid == interpid) + ? _channelitem_clear_interpreter(item) + : 0; + if (remove) { + _channelitem_free(item); if (prev == NULL) { - queue->first = item->next; + queue->first = next; } else { - prev->next = item->next; + prev->next = next; } - _channelitem_free(item); queue->count -= 1; } else { @@ -1054,12 +1081,15 @@ typedef struct _channel { PyThread_type_lock mutex; _channelqueue *queue; _channelends *ends; + struct { + int unboundop; + } defaults; int open; struct _channel_closing *closing; } _channel_state; static _channel_state * -_channel_new(PyThread_type_lock mutex) +_channel_new(PyThread_type_lock mutex, int unboundop) { _channel_state *chan = GLOBAL_MALLOC(_channel_state); if (chan == NULL) { @@ -1077,6 +1107,7 @@ _channel_new(PyThread_type_lock mutex) GLOBAL_FREE(chan); return NULL; } + chan->defaults.unboundop = unboundop; chan->open = 1; chan->closing = NULL; return chan; @@ -1097,7 +1128,8 @@ _channel_free(_channel_state *chan) static int _channel_add(_channel_state *chan, int64_t interpid, - _PyCrossInterpreterData *data, _waiting_t *waiting) + _PyCrossInterpreterData *data, _waiting_t *waiting, + int unboundop) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1111,7 +1143,7 @@ _channel_add(_channel_state *chan, int64_t interpid, goto done; } - if (_channelqueue_put(chan->queue, data, waiting) != 0) { + if (_channelqueue_put(chan->queue, interpid, data, waiting, unboundop) != 0) { goto done; } // Any errors past this point must cause a _waiting_release() call. @@ -1124,7 +1156,8 @@ _channel_add(_channel_state *chan, int64_t interpid, static int _channel_next(_channel_state *chan, int64_t interpid, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting, + int *p_unboundop) { int err = 0; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1138,11 +1171,15 @@ _channel_next(_channel_state *chan, int64_t interpid, goto done; } - int empty = _channelqueue_get(chan->queue, p_data, p_waiting); - assert(empty == 0 || empty == ERR_CHANNEL_EMPTY); + int empty = _channelqueue_get(chan->queue, p_data, p_waiting, p_unboundop); assert(!PyErr_Occurred()); - if (empty && chan->closing != NULL) { - chan->open = 0; + if (empty) { + assert(empty == ERR_CHANNEL_EMPTY); + if (chan->closing != NULL) { + chan->open = 0; + } + err = ERR_CHANNEL_EMPTY; + goto done; } done: @@ -1313,21 +1350,29 @@ typedef struct _channels { static void _channels_init(_channels *channels, PyThread_type_lock mutex) { - channels->mutex = mutex; - channels->head = NULL; - channels->numopen = 0; - channels->next_id = 0; + assert(mutex != NULL); + assert(channels->mutex == NULL); + *channels = (_channels){ + .mutex = mutex, + .head = NULL, + .numopen = 0, + .next_id = 0, + }; } static void -_channels_fini(_channels *channels) +_channels_fini(_channels *channels, PyThread_type_lock *p_mutex) { + PyThread_type_lock mutex = channels->mutex; + assert(mutex != NULL); + + PyThread_acquire_lock(mutex, WAIT_LOCK); assert(channels->numopen == 0); assert(channels->head == NULL); - if (channels->mutex != NULL) { - PyThread_free_lock(channels->mutex); - channels->mutex = NULL; - } + *channels = (_channels){0}; + PyThread_release_lock(mutex); + + *p_mutex = mutex; } static int64_t @@ -1564,18 +1609,27 @@ _channels_release_cid_object(_channels *channels, int64_t cid) PyThread_release_lock(channels->mutex); } -static int64_t * +struct channel_id_and_info { + int64_t id; + int unboundop; +}; + +static struct channel_id_and_info * _channels_list_all(_channels *channels, int64_t *count) { - int64_t *cids = NULL; + struct channel_id_and_info *cids = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(channels->numopen)); + struct channel_id_and_info *ids = + PyMem_NEW(struct channel_id_and_info, (Py_ssize_t)(channels->numopen)); if (ids == NULL) { goto done; } _channelref *ref = channels->head; for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = ref->cid; + ids[i] = (struct channel_id_and_info){ + .id = ref->cid, + .unboundop = ref->chan->defaults.unboundop, + }; } *count = channels->numopen; @@ -1660,13 +1714,13 @@ _channel_finish_closing(_channel_state *chan) { // Create a new channel. static int64_t -channel_create(_channels *channels) +channel_create(_channels *channels, int unboundop) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_CHANNEL_MUTEX_INIT; } - _channel_state *chan = _channel_new(mutex); + _channel_state *chan = _channel_new(mutex, unboundop); if (chan == NULL) { PyThread_free_lock(mutex); return -1; @@ -1698,7 +1752,7 @@ channel_destroy(_channels *channels, int64_t cid) // Optionally request to be notified when it is received. static int channel_send(_channels *channels, int64_t cid, PyObject *obj, - _waiting_t *waiting) + _waiting_t *waiting, int unboundop) { PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { @@ -1734,7 +1788,7 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, } // Add the data to the channel. - int res = _channel_add(chan, interpid, data, waiting); + int res = _channel_add(chan, interpid, data, waiting, unboundop); PyThread_release_lock(mutex); if (res != 0) { // We may chain an exception here: @@ -1771,7 +1825,7 @@ channel_clear_sent(_channels *channels, int64_t cid, _waiting_t *waiting) // Like channel_send(), but strictly wait for the object to be received. static int channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, - PY_TIMEOUT_T timeout) + int unboundop, PY_TIMEOUT_T timeout) { // We use a stack variable here, so we must ensure that &waiting // is not held by any channel item at the point this function exits. @@ -1782,7 +1836,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, } /* Queue up the object. */ - int res = channel_send(channels, cid, obj, &waiting); + int res = channel_send(channels, cid, obj, &waiting, unboundop); if (res < 0) { assert(waiting.status == WAITING_NO_STATUS); goto finally; @@ -1824,7 +1878,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, // The current interpreter gets associated with the recv end of the channel. // XXX Support a "wait" mutex? static int -channel_recv(_channels *channels, int64_t cid, PyObject **res) +channel_recv(_channels *channels, int64_t cid, PyObject **res, int *p_unboundop) { int err; *res = NULL; @@ -1852,13 +1906,15 @@ channel_recv(_channels *channels, int64_t cid, PyObject **res) // Pop off the next item from the channel. _PyCrossInterpreterData *data = NULL; _waiting_t *waiting = NULL; - err = _channel_next(chan, interpid, &data, &waiting); + err = _channel_next(chan, interpid, &data, &waiting, p_unboundop); PyThread_release_lock(mutex); if (err != 0) { return err; } else if (data == NULL) { + // The item was unbound. assert(!PyErr_Occurred()); + *res = NULL; return 0; } @@ -1951,6 +2007,23 @@ channel_is_associated(_channels *channels, int64_t cid, int64_t interpid, return (end != NULL && end->open); } +static int +_channel_get_count(_channels *channels, int64_t cid, Py_ssize_t *p_count) +{ + PyThread_type_lock mutex = NULL; + _channel_state *chan = NULL; + int err = _channels_lookup(channels, cid, &mutex, &chan); + if (err != 0) { + return err; + } + assert(chan != NULL); + int64_t count = chan->queue->count; + PyThread_release_lock(mutex); + + *p_count = (Py_ssize_t)count; + return 0; +} + /* channel info */ @@ -2126,7 +2199,7 @@ static PyStructSequence_Field channel_info_fields[] = { }; static PyStructSequence_Desc channel_info_desc = { - .name = MODULE_NAME ".ChannelInfo", + .name = MODULE_NAME_STR ".ChannelInfo", .doc = channel_info_doc, .fields = channel_info_fields, .n_in_sequence = 8, @@ -2154,7 +2227,7 @@ new_channel_info(PyObject *mod, struct channel_info *info) do { \ PyObject *obj = PyLong_FromLongLong(val); \ if (obj == NULL) { \ - Py_CLEAR(info); \ + Py_CLEAR(self); \ return NULL; \ } \ PyStructSequence_SET_ITEM(self, pos++, obj); \ @@ -2474,10 +2547,11 @@ struct _channelid_xid { static PyObject * _channelid_from_xid(_PyCrossInterpreterData *data) { - struct _channelid_xid *xid = (struct _channelid_xid *)data->data; + struct _channelid_xid *xid = \ + (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); // It might not be imported yet, so we can't use _get_current_module(). - PyObject *mod = PyImport_ImportModule(MODULE_NAME); + PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); if (mod == NULL) { return NULL; } @@ -2530,7 +2604,8 @@ _channelid_shared(PyThreadState *tstate, PyObject *obj, { return -1; } - struct _channelid_xid *xid = (struct _channelid_xid *)data->data; + struct _channelid_xid *xid = \ + (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); xid->cid = ((channelid *)obj)->cid; xid->end = ((channelid *)obj)->end; xid->resolve = ((channelid *)obj)->resolve; @@ -2601,13 +2676,32 @@ static PyType_Slot channelid_typeslots[] = { }; static PyType_Spec channelid_typespec = { - .name = MODULE_NAME ".ChannelID", + .name = MODULE_NAME_STR ".ChannelID", .basicsize = sizeof(channelid), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), .slots = channelid_typeslots, }; +static PyTypeObject * +add_channelid_type(PyObject *mod) +{ + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &channelid_typespec, NULL); + if (cls == NULL) { + return NULL; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); + return NULL; + } + if (ensure_xid_class(cls, _channelid_shared) < 0) { + Py_DECREF(cls); + return NULL; + } + return cls; +} + /* SendChannel and RecvChannel classes */ @@ -2630,10 +2724,10 @@ _get_current_channelend_type(int end) } if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.channel"); + PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); if (highlevel == NULL) { PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.channel"); + highlevel = PyImport_ImportModule("test.support.interpreters.channels"); if (highlevel == NULL) { return NULL; } @@ -2680,7 +2774,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj, if (res < 0) { return -1; } - data->new_object = _channelend_from_xid; + _PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid); return 0; } @@ -2691,21 +2785,29 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) if (state == NULL) { return -1; } - struct xid_class_registry *xid_classes = &state->xid_classes; - if (state->send_channel_type != NULL - || state->recv_channel_type != NULL) - { - PyErr_SetString(PyExc_TypeError, "already registered"); - return -1; + // Clear the old values if the .py module was reloaded. + if (state->send_channel_type != NULL) { + (void)clear_xid_class(state->send_channel_type); + Py_CLEAR(state->send_channel_type); } + if (state->recv_channel_type != NULL) { + (void)clear_xid_class(state->recv_channel_type); + Py_CLEAR(state->recv_channel_type); + } + + // Add and register the types. state->send_channel_type = (PyTypeObject *)Py_NewRef(send); state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - - if (register_xid_class(send, _channelend_shared, xid_classes)) { + if (ensure_xid_class(send, _channelend_shared) < 0) { + Py_CLEAR(state->send_channel_type); + Py_CLEAR(state->recv_channel_type); return -1; } - if (register_xid_class(recv, _channelend_shared, xid_classes)) { + if (ensure_xid_class(recv, _channelend_shared) < 0) { + (void)clear_xid_class(state->send_channel_type); + Py_CLEAR(state->send_channel_type); + Py_CLEAR(state->recv_channel_type); return -1; } @@ -2719,6 +2821,7 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) the data that we need to share between interpreters, so it cannot hold PyObject values. */ static struct globals { + PyMutex mutex; int module_count; _channels channels; } _globals = {0}; @@ -2726,32 +2829,36 @@ static struct globals { static int _globals_init(void) { - // XXX This isn't thread-safe. + PyMutex_Lock(&_globals.mutex); + assert(_globals.module_count >= 0); _globals.module_count++; - if (_globals.module_count > 1) { - // Already initialized. - return 0; - } - - assert(_globals.channels.mutex == NULL); - PyThread_type_lock mutex = PyThread_allocate_lock(); - if (mutex == NULL) { - return ERR_CHANNELS_MUTEX_INIT; + if (_globals.module_count == 1) { + // Called for the first time. + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + _globals.module_count--; + PyMutex_Unlock(&_globals.mutex); + return ERR_CHANNELS_MUTEX_INIT; + } + _channels_init(&_globals.channels, mutex); } - _channels_init(&_globals.channels, mutex); + PyMutex_Unlock(&_globals.mutex); return 0; } static void _globals_fini(void) { - // XXX This isn't thread-safe. + PyMutex_Lock(&_globals.mutex); + assert(_globals.module_count > 0); _globals.module_count--; - if (_globals.module_count > 0) { - return; + if (_globals.module_count == 0) { + PyThread_type_lock mutex; + _channels_fini(&_globals.channels, &mutex); + assert(mutex != NULL); + PyThread_free_lock(mutex); } - - _channels_fini(&_globals.channels); + PyMutex_Unlock(&_globals.mutex); } static _channels * @@ -2774,9 +2881,22 @@ clear_interpreter(void *data) static PyObject * -channelsmod_create(PyObject *self, PyObject *Py_UNUSED(ignored)) +channelsmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - int64_t cid = channel_create(&_globals.channels); + static char *kwlist[] = {"unboundop", NULL}; + int unboundop; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "i:create", kwlist, + &unboundop)) + { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); + return NULL; + } + + int64_t cid = channel_create(&_globals.channels, unboundop); if (cid < 0) { (void)handle_channel_error(-1, self, cid); return NULL; @@ -2803,7 +2923,7 @@ channelsmod_create(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(channelsmod_create_doc, -"channel_create() -> cid\n\ +"channel_create(unboundop) -> cid\n\ \n\ Create a new cross-interpreter channel and return a unique generated ID."); @@ -2838,7 +2958,8 @@ static PyObject * channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; - int64_t *cids = _channels_list_all(&_globals.channels, &count); + struct channel_id_and_info *cids = + _channels_list_all(&_globals.channels, &count); if (cids == NULL) { if (count == 0) { return PyList_New(0); @@ -2855,19 +2976,26 @@ channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) ids = NULL; goto finally; } - int64_t *cur = cids; + struct channel_id_and_info *cur = cids; for (int64_t i=0; i < count; cur++, i++) { PyObject *cidobj = NULL; - int err = newchannelid(state->ChannelIDType, *cur, 0, + int err = newchannelid(state->ChannelIDType, cur->id, 0, &_globals.channels, 0, 0, (channelid **)&cidobj); - if (handle_channel_error(err, self, *cur)) { + if (handle_channel_error(err, self, cur->id)) { assert(cidobj == NULL); Py_SETREF(ids, NULL); break; } assert(cidobj != NULL); - PyList_SET_ITEM(ids, (Py_ssize_t)i, cidobj); + + PyObject *item = Py_BuildValue("Oi", cidobj, cur->unboundop); + Py_DECREF(cidobj); + if (item == NULL) { + Py_SETREF(ids, NULL); + break; + } + PyList_SET_ITEM(ids, (Py_ssize_t)i, item); } finally: @@ -2915,7 +3043,7 @@ channelsmod_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) goto except; } if (res) { - interpid_obj = PyInterpreterState_GetIDObject(interp); + interpid_obj = _PyInterpreterState_GetIDObject(interp); if (interpid_obj == NULL) { goto except; } @@ -2949,16 +3077,24 @@ receive end."); static PyObject * channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "blocking", "timeout", NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", + NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; + int unboundop = UNBOUND_REPLACE; int blocking = 1; PyObject *timeout_obj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|$pO:channel_send", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|i$pO:channel_send", kwlist, channel_id_converter, &cid_data, &obj, - &blocking, &timeout_obj)) { + &unboundop, &blocking, &timeout_obj)) + { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); return NULL; } @@ -2971,10 +3107,10 @@ channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) /* Queue up the object. */ int err = 0; if (blocking) { - err = channel_send_wait(&_globals.channels, cid, obj, timeout); + err = channel_send_wait(&_globals.channels, cid, obj, unboundop, timeout); } else { - err = channel_send(&_globals.channels, cid, obj, NULL); + err = channel_send(&_globals.channels, cid, obj, NULL, unboundop); } if (handle_channel_error(err, self, cid)) { return NULL; @@ -2984,7 +3120,7 @@ channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(channelsmod_send_doc, -"channel_send(cid, obj, blocking=True)\n\ +"channel_send(cid, obj, *, blocking=True, timeout=None)\n\ \n\ Add the object's data to the channel's queue.\n\ By default this waits for the object to be received."); @@ -2992,17 +3128,24 @@ By default this waits for the object to be received."); static PyObject * channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "blocking", "timeout", NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", + NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; + int unboundop = UNBOUND_REPLACE; int blocking = 1; PyObject *timeout_obj = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&O|$pO:channel_send_buffer", kwlist, + "O&O|i$pO:channel_send_buffer", kwlist, channel_id_converter, &cid_data, &obj, - &blocking, &timeout_obj)) { + &unboundop, &blocking, &timeout_obj)) { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); return NULL; } @@ -3020,10 +3163,11 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) /* Queue up the object. */ int err = 0; if (blocking) { - err = channel_send_wait(&_globals.channels, cid, tempobj, timeout); + err = channel_send_wait( + &_globals.channels, cid, tempobj, unboundop, timeout); } else { - err = channel_send(&_globals.channels, cid, tempobj, NULL); + err = channel_send(&_globals.channels, cid, tempobj, NULL, unboundop); } Py_DECREF(tempobj); if (handle_channel_error(err, self, cid)) { @@ -3034,7 +3178,7 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(channelsmod_send_buffer_doc, -"channel_send_buffer(cid, obj, blocking=True)\n\ +"channel_send_buffer(cid, obj, *, blocking=True, timeout=None)\n\ \n\ Add the object's buffer to the channel's queue.\n\ By default this waits for the object to be received."); @@ -3055,25 +3199,28 @@ channelsmod_recv(PyObject *self, PyObject *args, PyObject *kwds) cid = cid_data.cid; PyObject *obj = NULL; - int err = channel_recv(&_globals.channels, cid, &obj); - if (handle_channel_error(err, self, cid)) { - return NULL; - } - Py_XINCREF(dflt); - if (obj == NULL) { + int unboundop = 0; + int err = channel_recv(&_globals.channels, cid, &obj, &unboundop); + if (err == ERR_CHANNEL_EMPTY && dflt != NULL) { // Use the default. - if (dflt == NULL) { - (void)handle_channel_error(ERR_CHANNEL_EMPTY, self, cid); - return NULL; - } obj = Py_NewRef(dflt); + err = 0; } - Py_XDECREF(dflt); - return obj; + else if (handle_channel_error(err, self, cid)) { + return NULL; + } + else if (obj == NULL) { + // The item was unbound. + return Py_BuildValue("Oi", Py_None, unboundop); + } + + PyObject *res = Py_BuildValue("OO", obj, Py_None); + Py_DECREF(obj); + return res; } PyDoc_STRVAR(channelsmod_recv_doc, -"channel_recv(cid, [default]) -> obj\n\ +"channel_recv(cid, [default]) -> (obj, unboundop)\n\ \n\ Return a new object from the data at the front of the channel's queue.\n\ \n\ @@ -3174,6 +3321,34 @@ Close the channel for the current interpreter. 'send' and 'recv'\n\ (bool) may be used to indicate the ends to close. By default both\n\ ends are closed. Closing an already closed end is a noop."); +static PyObject * +channelsmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", NULL}; + struct channel_id_converter_data cid_data = { + .module = self, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_count", kwlist, + channel_id_converter, &cid_data)) { + return NULL; + } + int64_t cid = cid_data.cid; + + Py_ssize_t count = -1; + int err = _channel_get_count(&_globals.channels, cid, &count); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + assert(count >= 0); + return PyLong_FromSsize_t(count); +} + +PyDoc_STRVAR(channelsmod_get_count_doc, +"get_count(cid)\n\ +\n\ +Return the number of items in the channel."); + static PyObject * channelsmod_get_info(PyObject *self, PyObject *args, PyObject *kwds) { @@ -3201,6 +3376,38 @@ PyDoc_STRVAR(channelsmod_get_info_doc, \n\ Return details about the channel."); +static PyObject * +channelsmod_get_channel_defaults(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", NULL}; + struct channel_id_converter_data cid_data = { + .module = self, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_channel_defaults", kwlist, + channel_id_converter, &cid_data)) { + return NULL; + } + int64_t cid = cid_data.cid; + + PyThread_type_lock mutex = NULL; + _channel_state *channel = NULL; + int err = _channels_lookup(&_globals.channels, cid, &mutex, &channel); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + int unboundop = channel->defaults.unboundop; + PyThread_release_lock(mutex); + + PyObject *defaults = Py_BuildValue("i", unboundop); + return defaults; +} + +PyDoc_STRVAR(channelsmod_get_channel_defaults_doc, +"get_channel_defaults(cid)\n\ +\n\ +Return the channel's default values, set when it was created."); + static PyObject * channelsmod__channel_id(PyObject *self, PyObject *args, PyObject *kwds) { @@ -3247,8 +3454,8 @@ channelsmod__register_end_types(PyObject *self, PyObject *args, PyObject *kwds) } static PyMethodDef module_functions[] = { - {"create", channelsmod_create, - METH_NOARGS, channelsmod_create_doc}, + {"create", _PyCFunction_CAST(channelsmod_create), + METH_VARARGS | METH_KEYWORDS, channelsmod_create_doc}, {"destroy", _PyCFunction_CAST(channelsmod_destroy), METH_VARARGS | METH_KEYWORDS, channelsmod_destroy_doc}, {"list_all", channelsmod_list_all, @@ -3265,8 +3472,12 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, channelsmod_close_doc}, {"release", _PyCFunction_CAST(channelsmod_release), METH_VARARGS | METH_KEYWORDS, channelsmod_release_doc}, + {"get_count", _PyCFunction_CAST(channelsmod_get_count), + METH_VARARGS | METH_KEYWORDS, channelsmod_get_count_doc}, {"get_info", _PyCFunction_CAST(channelsmod_get_info), METH_VARARGS | METH_KEYWORDS, channelsmod_get_info_doc}, + {"get_channel_defaults", _PyCFunction_CAST(channelsmod_get_channel_defaults), + METH_VARARGS | METH_KEYWORDS, channelsmod_get_channel_defaults_doc}, {"_channel_id", _PyCFunction_CAST(channelsmod__channel_id), METH_VARARGS | METH_KEYWORDS, NULL}, {"_register_end_types", _PyCFunction_CAST(channelsmod__register_end_types), @@ -3285,16 +3496,15 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { - if (_globals_init() != 0) { + int err = _globals_init(); + if (handle_channel_error(err, mod, -1)) { return -1; } - struct xid_class_registry *xid_classes = NULL; module_state *state = get_module_state(mod); if (state == NULL) { goto error; } - xid_classes = &state->xid_classes; /* Add exception types */ if (exceptions_init(mod) != 0) { @@ -3313,8 +3523,7 @@ module_exec(PyObject *mod) } // ChannelID - state->ChannelIDType = add_new_type( - mod, &channelid_typespec, _channelid_shared, xid_classes); + state->ChannelIDType = add_channelid_type(mod); if (state->ChannelIDType == NULL) { goto error; } @@ -3326,8 +3535,8 @@ module_exec(PyObject *mod) return 0; error: - if (xid_classes != NULL) { - clear_xid_class_registry(xid_classes); + if (state != NULL) { + clear_xid_types(state); } _globals_fini(); return -1; @@ -3336,6 +3545,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -3354,9 +3564,6 @@ module_clear(PyObject *mod) module_state *state = get_module_state(mod); assert(state != NULL); - // Before clearing anything, we unregister the various XID types. */ - clear_xid_class_registry(&state->xid_classes); - // Now we clear the module state. clear_module_state(state); return 0; @@ -3368,9 +3575,6 @@ module_free(void *mod) module_state *state = get_module_state(mod); assert(state != NULL); - // Before clearing anything, we unregister the various XID types. */ - clear_xid_class_registry(&state->xid_classes); - // Now we clear the module state. clear_module_state(state); @@ -3379,7 +3583,7 @@ module_free(void *mod) static struct PyModuleDef moduledef = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = MODULE_NAME, + .m_name = MODULE_NAME_STR, .m_doc = module_doc, .m_size = sizeof(module_state), .m_methods = module_functions, @@ -3390,7 +3594,7 @@ static struct PyModuleDef moduledef = { }; PyMODINIT_FUNC -PyInit__xxinterpchannels(void) +MODINIT_FUNC_NAME(void) { return PyModuleDef_Init(&moduledef); } diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_interpqueuesmodule.c similarity index 70% rename from Modules/_xxinterpqueuesmodule.c rename to Modules/_interpqueuesmodule.c index 537ba9188055dd..297a1763a98ce6 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -8,8 +8,16 @@ #include "Python.h" #include "pycore_crossinterp.h" // struct _xid +#define REGISTERS_HEAP_TYPES +#define HAS_UNBOUND_ITEMS +#include "_interpreters_common.h" +#undef HAS_UNBOUND_ITEMS +#undef REGISTERS_HEAP_TYPES -#define MODULE_NAME "_xxinterpqueues" + +#define MODULE_NAME _interpqueues +#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) +#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) #define GLOBAL_MALLOC(TYPE) \ @@ -52,7 +60,6 @@ _release_xid_data(_PyCrossInterpreterData *data, int flags) return res; } - static PyInterpreterState * _get_current_interp(void) { @@ -64,7 +71,7 @@ _get_current_interp(void) static PyObject * _get_current_module(void) { - PyObject *name = PyUnicode_FromString(MODULE_NAME); + PyObject *name = PyUnicode_FromString(MODULE_NAME_STR); if (name == NULL) { return NULL; } @@ -124,6 +131,22 @@ idarg_int64_converter(PyObject *arg, void *ptr) } +static int +ensure_highlevel_module_loaded(void) +{ + PyObject *highlevel = PyImport_ImportModule("interpreters.queues"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.interpreters.queues"); + if (highlevel == NULL) { + return -1; + } + } + Py_DECREF(highlevel); + return 0; +} + + /* module state *************************************************************/ typedef struct { @@ -165,6 +188,9 @@ static int clear_module_state(module_state *state) { /* external types */ + if (state->queue_type != NULL) { + (void)clear_xid_class(state->queue_type); + } Py_CLEAR(state->queue_type); /* QueueError */ @@ -188,6 +214,9 @@ clear_module_state(module_state *state) // single-queue errors #define ERR_QUEUE_EMPTY (-21) #define ERR_QUEUE_FULL (-22) +#define ERR_QUEUE_NEVER_BOUND (-23) + +static int ensure_external_exc_types(module_state *); static int resolve_module_errcode(module_state *state, int errcode, int64_t qid, @@ -205,13 +234,23 @@ resolve_module_errcode(module_state *state, int errcode, int64_t qid, msg = PyUnicode_FromFormat("queue %" PRId64 " not found", qid); break; case ERR_QUEUE_EMPTY: + if (ensure_external_exc_types(state) < 0) { + return -1; + } exctype = state->QueueEmpty; msg = PyUnicode_FromFormat("queue %" PRId64 " is empty", qid); break; case ERR_QUEUE_FULL: + if (ensure_external_exc_types(state) < 0) { + return -1; + } exctype = state->QueueFull; msg = PyUnicode_FromFormat("queue %" PRId64 " is full", qid); break; + case ERR_QUEUE_NEVER_BOUND: + exctype = state->QueueError; + msg = PyUnicode_FromFormat("queue %" PRId64 " never bound", qid); + break; default: PyErr_Format(PyExc_ValueError, "unsupported error code %d", errcode); @@ -260,20 +299,59 @@ add_QueueError(PyObject *mod) #define PREFIX "test.support.interpreters." #define ADD_EXCTYPE(NAME, BASE, DOC) \ + assert(state->NAME == NULL); \ if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ return -1; \ } ADD_EXCTYPE(QueueError, PyExc_RuntimeError, "Indicates that a queue-related error happened.") ADD_EXCTYPE(QueueNotFoundError, state->QueueError, NULL) - ADD_EXCTYPE(QueueEmpty, state->QueueError, NULL) - ADD_EXCTYPE(QueueFull, state->QueueError, NULL) + // QueueEmpty and QueueFull are set by set_external_exc_types(). + state->QueueEmpty = NULL; + state->QueueFull = NULL; #undef ADD_EXCTYPE #undef PREFIX return 0; } +static int +set_external_exc_types(module_state *state, + PyObject *emptyerror, PyObject *fullerror) +{ + if (state->QueueEmpty != NULL) { + assert(state->QueueFull != NULL); + Py_CLEAR(state->QueueEmpty); + Py_CLEAR(state->QueueFull); + } + else { + assert(state->QueueFull == NULL); + } + assert(PyObject_IsSubclass(emptyerror, state->QueueError)); + assert(PyObject_IsSubclass(fullerror, state->QueueError)); + state->QueueEmpty = Py_NewRef(emptyerror); + state->QueueFull = Py_NewRef(fullerror); + return 0; +} + +static int +ensure_external_exc_types(module_state *state) +{ + if (state->QueueEmpty != NULL) { + assert(state->QueueFull != NULL); + return 0; + } + assert(state->QueueFull == NULL); + + // Force the module to be loaded, to register the type. + if (ensure_highlevel_module_loaded() < 0) { + return -1; + } + assert(state->QueueEmpty != NULL); + assert(state->QueueFull != NULL); + return 0; +} + static int handle_queue_error(int err, PyObject *mod, int64_t qid) { @@ -286,10 +364,12 @@ handle_queue_error(int err, PyObject *mod, int64_t qid) module_state *state; switch (err) { - case ERR_QUEUE_ALLOC: // fall through + case ERR_QUEUE_ALLOC: _Py_FALLTHROUGH; case ERR_QUEUES_ALLOC: PyErr_NoMemory(); break; + case -1: + return -1; default: state = get_module_state(mod); assert(state->QueueError != NULL); @@ -315,39 +395,67 @@ handle_queue_error(int err, PyObject *mod, int64_t qid) struct _queueitem; typedef struct _queueitem { + /* The interpreter that added the item to the queue. + The actual bound interpid is found in item->data. + This is necessary because item->data might be NULL, + meaning the interpreter has been destroyed. */ + int64_t interpid; _PyCrossInterpreterData *data; + int fmt; + int unboundop; struct _queueitem *next; } _queueitem; static void -_queueitem_init(_queueitem *item, _PyCrossInterpreterData *data) +_queueitem_init(_queueitem *item, + int64_t interpid, _PyCrossInterpreterData *data, + int fmt, int unboundop) { + if (interpid < 0) { + interpid = _get_interpid(data); + } + else { + assert(data == NULL + || _PyCrossInterpreterData_INTERPID(data) < 0 + || interpid == _PyCrossInterpreterData_INTERPID(data)); + } + assert(check_unbound(unboundop)); *item = (_queueitem){ + .interpid = interpid, .data = data, + .fmt = fmt, + .unboundop = unboundop, }; } +static void +_queueitem_clear_data(_queueitem *item) +{ + if (item->data == NULL) { + return; + } + // It was allocated in queue_put(). + (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + item->data = NULL; +} + static void _queueitem_clear(_queueitem *item) { item->next = NULL; - - if (item->data != NULL) { - // It was allocated in queue_put(). - (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); - item->data = NULL; - } + _queueitem_clear_data(item); } static _queueitem * -_queueitem_new(_PyCrossInterpreterData *data) +_queueitem_new(int64_t interpid, _PyCrossInterpreterData *data, + int fmt, int unboundop) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _queueitem_init(item, data); + _queueitem_init(item, interpid, data, fmt, unboundop); return item; } @@ -369,16 +477,48 @@ _queueitem_free_all(_queueitem *item) } static void -_queueitem_popped(_queueitem *item, _PyCrossInterpreterData **p_data) +_queueitem_popped(_queueitem *item, + _PyCrossInterpreterData **p_data, int *p_fmt, int *p_unboundop) { *p_data = item->data; + *p_fmt = item->fmt; + *p_unboundop = item->unboundop; // We clear them here, so they won't be released in _queueitem_clear(). item->data = NULL; _queueitem_free(item); } +static int +_queueitem_clear_interpreter(_queueitem *item) +{ + assert(item->interpid >= 0); + if (item->data == NULL) { + // Its interpreter was already cleared (or it was never bound). + // For UNBOUND_REMOVE it should have been freed at that time. + assert(item->unboundop != UNBOUND_REMOVE); + return 0; + } + assert(_PyCrossInterpreterData_INTERPID(item->data) == item->interpid); + + switch (item->unboundop) { + case UNBOUND_REMOVE: + // The caller must free/clear it. + return 1; + case UNBOUND_ERROR: + case UNBOUND_REPLACE: + // We won't need the cross-interpreter data later + // so we completely throw it away. + _queueitem_clear_data(item); + return 0; + default: + Py_FatalError("not reachable"); + return -1; + } +} + /* the queue */ + typedef struct _queue { Py_ssize_t num_waiters; // protected by global lock PyThread_type_lock mutex; @@ -389,11 +529,16 @@ typedef struct _queue { _queueitem *first; _queueitem *last; } items; + struct { + int fmt; + int unboundop; + } defaults; } _queue; static int -_queue_init(_queue *queue, Py_ssize_t maxsize) +_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop) { + assert(check_unbound(unboundop)); PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_QUEUE_ALLOC; @@ -404,6 +549,10 @@ _queue_init(_queue *queue, Py_ssize_t maxsize) .items = { .maxsize = maxsize, }, + .defaults = { + .fmt = fmt, + .unboundop = unboundop, + }, }; return 0; } @@ -419,6 +568,8 @@ _queue_clear(_queue *queue) *queue = (_queue){0}; } +static void _queue_free(_queue *); + static void _queue_kill_and_wait(_queue *queue) { @@ -482,7 +633,8 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, _PyCrossInterpreterData *data) +_queue_add(_queue *queue, int64_t interpid, _PyCrossInterpreterData *data, + int fmt, int unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -498,7 +650,7 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data) return ERR_QUEUE_FULL; } - _queueitem *item = _queueitem_new(data); + _queueitem *item = _queueitem_new(interpid, data, fmt, unboundop); if (item == NULL) { _queue_unlock(queue); return -1; @@ -518,7 +670,8 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data) } static int -_queue_next(_queue *queue, _PyCrossInterpreterData **p_data) +_queue_next(_queue *queue, + _PyCrossInterpreterData **p_data, int *p_fmt, int *p_unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -537,7 +690,7 @@ _queue_next(_queue *queue, _PyCrossInterpreterData **p_data) } queue->items.count -= 1; - _queueitem_popped(item, p_data); + _queueitem_popped(item, p_data, p_fmt, p_unboundop); _queue_unlock(queue); return 0; @@ -602,14 +755,17 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid) while (next != NULL) { _queueitem *item = next; next = item->next; - if (item->data->interpid == interpid) { + int remove = (item->interpid == interpid) + ? _queueitem_clear_interpreter(item) + : 0; + if (remove) { + _queueitem_free(item); if (prev == NULL) { - queue->items.first = item->next; + queue->items.first = next; } else { - prev->next = item->next; + prev->next = next; } - _queueitem_free(item); queue->items.count -= 1; } else { @@ -650,6 +806,32 @@ _queuerefs_find(_queueref *first, int64_t qid, _queueref **pprev) return ref; } +static void +_queuerefs_clear(_queueref *head) +{ + _queueref *next = head; + while (next != NULL) { + _queueref *ref = next; + next = ref->next; + +#ifdef Py_DEBUG + int64_t qid = ref->qid; + fprintf(stderr, "queue %" PRId64 " still exists\n", qid); +#endif + _queue *queue = ref->queue; + GLOBAL_FREE(ref); + + _queue_kill_and_wait(queue); +#ifdef Py_DEBUG + if (queue->items.count > 0) { + fprintf(stderr, "queue %" PRId64 " still holds %zd items\n", + qid, queue->items.count); + } +#endif + _queue_free(queue); + } +} + /* a collection of queues ***************************************************/ @@ -663,21 +845,31 @@ typedef struct _queues { static void _queues_init(_queues *queues, PyThread_type_lock mutex) { - queues->mutex = mutex; - queues->head = NULL; - queues->count = 0; - queues->next_id = 1; + assert(mutex != NULL); + assert(queues->mutex == NULL); + *queues = (_queues){ + .mutex = mutex, + .head = NULL, + .count = 0, + .next_id = 1, + }; } static void -_queues_fini(_queues *queues) +_queues_fini(_queues *queues, PyThread_type_lock *p_mutex) { - assert(queues->count == 0); - assert(queues->head == NULL); - if (queues->mutex != NULL) { - PyThread_free_lock(queues->mutex); - queues->mutex = NULL; + PyThread_type_lock mutex = queues->mutex; + assert(mutex != NULL); + + PyThread_acquire_lock(mutex, WAIT_LOCK); + if (queues->count > 0) { + assert(queues->head != NULL); + _queuerefs_clear(queues->head); } + *queues = (_queues){0}; + PyThread_release_lock(mutex); + + *p_mutex = mutex; } static int64_t @@ -805,19 +997,21 @@ _queues_incref(_queues *queues, int64_t qid) return res; } -static void _queue_free(_queue *); - -static void +static int _queues_decref(_queues *queues, int64_t qid) { + int res = -1; PyThread_acquire_lock(queues->mutex, WAIT_LOCK); _queueref *prev = NULL; _queueref *ref = _queuerefs_find(queues->head, qid, &prev); if (ref == NULL) { assert(!PyErr_Occurred()); - // Already destroyed. - // XXX Warn? + res = ERR_QUEUE_NOT_FOUND; + goto finally; + } + if (ref->refcount == 0) { + res = ERR_QUEUE_NEVER_BOUND; goto finally; } assert(ref->refcount > 0); @@ -832,27 +1026,39 @@ _queues_decref(_queues *queues, int64_t qid) _queue_kill_and_wait(queue); _queue_free(queue); - return; + return 0; } + res = 0; finally: PyThread_release_lock(queues->mutex); + return res; } -static int64_t * -_queues_list_all(_queues *queues, int64_t *count) +struct queue_id_and_info { + int64_t id; + int fmt; + int unboundop; +}; + +static struct queue_id_and_info * +_queues_list_all(_queues *queues, int64_t *p_count) { - int64_t *qids = NULL; + struct queue_id_and_info *qids = NULL; PyThread_acquire_lock(queues->mutex, WAIT_LOCK); - int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(queues->count)); + struct queue_id_and_info *ids = PyMem_NEW(struct queue_id_and_info, + (Py_ssize_t)(queues->count)); if (ids == NULL) { goto done; } _queueref *ref = queues->head; for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = ref->qid; + ids[i].id = ref->qid; + assert(ref->queue != NULL); + ids[i].fmt = ref->queue->defaults.fmt; + ids[i].unboundop = ref->queue->defaults.unboundop; } - *count = queues->count; + *p_count = queues->count; qids = ids; done: @@ -886,13 +1092,13 @@ _queue_free(_queue *queue) // Create a new queue. static int64_t -queue_create(_queues *queues, Py_ssize_t maxsize) +queue_create(_queues *queues, Py_ssize_t maxsize, int fmt, int unboundop) { _queue *queue = GLOBAL_MALLOC(_queue); if (queue == NULL) { return ERR_QUEUE_ALLOC; } - int err = _queue_init(queue, maxsize); + int err = _queue_init(queue, maxsize, fmt, unboundop); if (err < 0) { GLOBAL_FREE(queue); return (int64_t)err; @@ -921,7 +1127,7 @@ queue_destroy(_queues *queues, int64_t qid) // Push an object onto the queue. static int -queue_put(_queues *queues, int64_t qid, PyObject *obj) +queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) { // Look up the queue. _queue *queue = NULL; @@ -942,9 +1148,12 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj) GLOBAL_FREE(data); return -1; } + assert(_PyCrossInterpreterData_INTERPID(data) == \ + PyInterpreterState_GetID(PyInterpreterState_Get())); // Add the data to the queue. - int res = _queue_add(queue, data); + int64_t interpid = -1; // _queueitem_init() will set it. + int res = _queue_add(queue, interpid, data, fmt, unboundop); _queue_unmark_waiter(queue, queues->mutex); if (res != 0) { // We may chain an exception here: @@ -959,7 +1168,8 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj) // Pop the next object off the queue. Fail if empty. // XXX Support a "wait" mutex? static int -queue_get(_queues *queues, int64_t qid, PyObject **res) +queue_get(_queues *queues, int64_t qid, + PyObject **res, int *p_fmt, int *p_unboundop) { int err; *res = NULL; @@ -975,7 +1185,7 @@ queue_get(_queues *queues, int64_t qid, PyObject **res) // Pop off the next item from the queue. _PyCrossInterpreterData *data = NULL; - err = _queue_next(queue, &data); + err = _queue_next(queue, &data, p_fmt, p_unboundop); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { return err; @@ -1052,19 +1262,19 @@ static int _queueobj_shared(PyThreadState *, PyObject *, _PyCrossInterpreterData *); static int -set_external_queue_type(PyObject *module, PyTypeObject *queue_type) +set_external_queue_type(module_state *state, PyTypeObject *queue_type) { - module_state *state = get_module_state(module); - + // Clear the old value if the .py module was reloaded. if (state->queue_type != NULL) { - PyErr_SetString(PyExc_TypeError, "already registered"); - return -1; + (void)clear_xid_class(state->queue_type); + Py_CLEAR(state->queue_type); } - state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); - if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0) { + // Add and register the new type. + if (ensure_xid_class(queue_type, _queueobj_shared) < 0) { return -1; } + state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); return 0; } @@ -1077,15 +1287,9 @@ get_external_queue_type(PyObject *module) PyTypeObject *cls = state->queue_type; if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.queue"); - if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.queue"); - if (highlevel == NULL) { - return NULL; - } + if (ensure_highlevel_module_loaded() < 0) { + return NULL; } - Py_DECREF(highlevel); cls = state->queue_type; assert(cls != NULL); } @@ -1111,7 +1315,7 @@ _queueid_xid_new(int64_t qid) struct _queueid_xid *data = PyMem_RawMalloc(sizeof(struct _queueid_xid)); if (data == NULL) { - _queues_incref(queues, qid); + _queues_decref(queues, qid); return NULL; } data->qid = qid; @@ -1124,13 +1328,20 @@ _queueid_xid_free(void *data) int64_t qid = ((struct _queueid_xid *)data)->qid; PyMem_RawFree(data); _queues *queues = _get_global_queues(); - _queues_decref(queues, qid); + int res = _queues_decref(queues, qid); + if (res == ERR_QUEUE_NOT_FOUND) { + // Already destroyed. + // XXX Warn? + } + else { + assert(res == 0); + } } static PyObject * _queueobj_from_xid(_PyCrossInterpreterData *data) { - int64_t qid = *(int64_t *)data->data; + int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data); PyObject *qidobj = PyLong_FromLongLong(qid); if (qidobj == NULL) { return NULL; @@ -1140,7 +1351,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data) if (mod == NULL) { // XXX import it? PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME " module not imported yet"); + MODULE_NAME_STR " module not imported yet"); return NULL; } @@ -1167,7 +1378,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, .label = "queue ID", }; int res = idarg_int64_converter(qidobj, &converted); - Py_DECREF(qidobj); + Py_CLEAR(qidobj); if (!res) { assert(PyErr_Occurred()); return -1; @@ -1175,13 +1386,11 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, void *raw = _queueid_xid_new(converted.id); if (raw == NULL) { - Py_DECREF(qidobj); return -1; } _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL, _queueobj_from_xid); - Py_DECREF(qidobj); - data->free = _queueid_xid_free; + _PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free); return 0; } @@ -1192,6 +1401,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, the data that we need to share between interpreters, so it cannot hold PyObject values. */ static struct globals { + PyMutex mutex; int module_count; _queues queues; } _globals = {0}; @@ -1199,32 +1409,36 @@ static struct globals { static int _globals_init(void) { - // XXX This isn't thread-safe. + PyMutex_Lock(&_globals.mutex); + assert(_globals.module_count >= 0); _globals.module_count++; - if (_globals.module_count > 1) { - // Already initialized. - return 0; - } - - assert(_globals.queues.mutex == NULL); - PyThread_type_lock mutex = PyThread_allocate_lock(); - if (mutex == NULL) { - return ERR_QUEUES_ALLOC; + if (_globals.module_count == 1) { + // Called for the first time. + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + _globals.module_count--; + PyMutex_Unlock(&_globals.mutex); + return ERR_QUEUES_ALLOC; + } + _queues_init(&_globals.queues, mutex); } - _queues_init(&_globals.queues, mutex); + PyMutex_Unlock(&_globals.mutex); return 0; } static void _globals_fini(void) { - // XXX This isn't thread-safe. + PyMutex_Lock(&_globals.mutex); + assert(_globals.module_count > 0); _globals.module_count--; - if (_globals.module_count > 0) { - return; + if (_globals.module_count == 0) { + PyThread_type_lock mutex; + _queues_fini(&_globals.queues, &mutex); + assert(mutex != NULL); + PyThread_free_lock(mutex); } - - _queues_fini(&_globals.queues); + PyMutex_Unlock(&_globals.mutex); } static _queues * @@ -1263,14 +1477,22 @@ qidarg_converter(PyObject *arg, void *ptr) static PyObject * queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"maxsize", NULL}; - Py_ssize_t maxsize = -1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n:create", kwlist, - &maxsize)) { + static char *kwlist[] = {"maxsize", "fmt", "unboundop", NULL}; + Py_ssize_t maxsize; + int fmt; + int unboundop; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "nii:create", kwlist, + &maxsize, &fmt, &unboundop)) + { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); return NULL; } - int64_t qid = queue_create(&_globals.queues, maxsize); + int64_t qid = queue_create(&_globals.queues, maxsize, fmt, unboundop); if (qid < 0) { (void)handle_queue_error((int)qid, self, qid); return NULL; @@ -1292,16 +1514,19 @@ queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_create_doc, -"create() -> qid\n\ +"create(maxsize, fmt, unboundop) -> qid\n\ \n\ Create a new cross-interpreter queue and return its unique generated ID.\n\ -It is a new reference as though bind() had been called on the queue."); +It is a new reference as though bind() had been called on the queue.\n\ +\n\ +The caller is responsible for calling destroy() for the new queue\n\ +before the runtime is finalized."); static PyObject * queuesmod_destroy(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:destroy", kwlist, qidarg_converter, &qidarg)) { return NULL; @@ -1325,9 +1550,9 @@ static PyObject * queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; - int64_t *qids = _queues_list_all(&_globals.queues, &count); + struct queue_id_and_info *qids = _queues_list_all(&_globals.queues, &count); if (qids == NULL) { - if (count == 0) { + if (!PyErr_Occurred() && count == 0) { return PyList_New(0); } return NULL; @@ -1336,14 +1561,15 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) if (ids == NULL) { goto finally; } - int64_t *cur = qids; + struct queue_id_and_info *cur = qids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *qidobj = PyLong_FromLongLong(*cur); - if (qidobj == NULL) { + PyObject *item = Py_BuildValue("Lii", cur->id, cur->fmt, + cur->unboundop); + if (item == NULL) { Py_SETREF(ids, NULL); break; } - PyList_SET_ITEM(ids, (Py_ssize_t)i, qidobj); + PyList_SET_ITEM(ids, (Py_ssize_t)i, item); } finally: @@ -1352,24 +1578,35 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(queuesmod_list_all_doc, -"list_all() -> [qid]\n\ +"list_all() -> [(qid, fmt)]\n\ \n\ -Return the list of IDs for all queues."); +Return the list of IDs for all queues.\n\ +Each corresponding default format is also included."); static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "obj", NULL}; - qidarg_converter_data qidarg; + static char *kwlist[] = {"qid", "obj", "fmt", "unboundop", NULL}; + qidarg_converter_data qidarg = {0}; PyObject *obj; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:put", kwlist, - qidarg_converter, &qidarg, &obj)) { + int fmt; + int unboundop; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oii:put", kwlist, + qidarg_converter, &qidarg, &obj, &fmt, + &unboundop)) + { return NULL; } int64_t qid = qidarg.id; + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); + return NULL; + } /* Queue up the object. */ - int err = queue_put(&_globals.queues, qid, obj); + int err = queue_put(&_globals.queues, qid, obj, fmt, unboundop); + // This is the only place that raises QueueFull. if (handle_queue_error(err, self, qid)) { return NULL; } @@ -1378,47 +1615,51 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_put_doc, -"put(qid, obj)\n\ +"put(qid, obj, fmt)\n\ \n\ Add the object's data to the queue."); static PyObject * queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "default", NULL}; - qidarg_converter_data qidarg; - PyObject *dflt = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O:get", kwlist, - qidarg_converter, &qidarg, &dflt)) { + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg = {0}; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get", kwlist, + qidarg_converter, &qidarg)) { return NULL; } int64_t qid = qidarg.id; PyObject *obj = NULL; - int err = queue_get(&_globals.queues, qid, &obj); - if (err == ERR_QUEUE_EMPTY && dflt != NULL) { - assert(obj == NULL); - obj = Py_NewRef(dflt); - } - else if (handle_queue_error(err, self, qid)) { + int fmt = 0; + int unboundop = 0; + int err = queue_get(&_globals.queues, qid, &obj, &fmt, &unboundop); + // This is the only place that raises QueueEmpty. + if (handle_queue_error(err, self, qid)) { return NULL; } - return obj; + + if (obj == NULL) { + return Py_BuildValue("Oii", Py_None, fmt, unboundop); + } + PyObject *res = Py_BuildValue("OiO", obj, fmt, Py_None); + Py_DECREF(obj); + return res; } PyDoc_STRVAR(queuesmod_get_doc, -"get(qid, [default]) -> obj\n\ +"get(qid) -> (obj, fmt)\n\ \n\ Return a new object from the data at the front of the queue.\n\ +The object's format is also returned.\n\ \n\ -If there is nothing to receive then raise QueueEmpty, unless\n\ -a default value is provided. In that case return it."); +If there is nothing to receive then raise QueueEmpty."); static PyObject * queuesmod_bind(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:bind", kwlist, qidarg_converter, &qidarg)) { return NULL; @@ -1448,7 +1689,7 @@ queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) { // Note that only the current interpreter is affected. static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:release", kwlist, qidarg_converter, &qidarg)) { @@ -1459,7 +1700,10 @@ queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) // XXX Check module state if bound already. // XXX Update module state. - _queues_decref(&_globals.queues, qid); + int err = _queues_decref(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } Py_RETURN_NONE; } @@ -1474,7 +1718,7 @@ static PyObject * queuesmod_get_maxsize(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get_maxsize", kwlist, qidarg_converter, &qidarg)) { @@ -1495,11 +1739,41 @@ PyDoc_STRVAR(queuesmod_get_maxsize_doc, \n\ Return the maximum number of items in the queue."); +static PyObject * +queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg = {0}; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_queue_defaults", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + _queue *queue = NULL; + int err = _queues_lookup(&_globals.queues, qid, &queue); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + int fmt = queue->defaults.fmt; + int unboundop = queue->defaults.unboundop; + _queue_unmark_waiter(queue, _globals.queues.mutex); + + PyObject *defaults = Py_BuildValue("ii", fmt, unboundop); + return defaults; +} + +PyDoc_STRVAR(queuesmod_get_queue_defaults_doc, +"get_queue_defaults(qid)\n\ +\n\ +Return the queue's default values, set when it was created."); + static PyObject * queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:is_full", kwlist, qidarg_converter, &qidarg)) { @@ -1527,7 +1801,7 @@ static PyObject * queuesmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get_count", kwlist, qidarg_converter, &qidarg)) { @@ -1550,22 +1824,39 @@ PyDoc_STRVAR(queuesmod_get_count_doc, Return the number of items in the queue."); static PyObject * -queuesmod__register_queue_type(PyObject *self, PyObject *args, PyObject *kwds) +queuesmod__register_heap_types(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"queuetype", NULL}; + static char *kwlist[] = {"queuetype", "emptyerror", "fullerror", NULL}; PyObject *queuetype; + PyObject *emptyerror; + PyObject *fullerror; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_register_queue_type", kwlist, - &queuetype)) { + "OOO:_register_heap_types", kwlist, + &queuetype, &emptyerror, &fullerror)) { return NULL; } if (!PyType_Check(queuetype)) { - PyErr_SetString(PyExc_TypeError, "expected a type for 'queuetype'"); + PyErr_SetString(PyExc_TypeError, + "expected a type for 'queuetype'"); + return NULL; + } + if (!PyExceptionClass_Check(emptyerror)) { + PyErr_SetString(PyExc_TypeError, + "expected an exception type for 'emptyerror'"); return NULL; } - PyTypeObject *cls_queue = (PyTypeObject *)queuetype; + if (!PyExceptionClass_Check(fullerror)) { + PyErr_SetString(PyExc_TypeError, + "expected an exception type for 'fullerror'"); + return NULL; + } + + module_state *state = get_module_state(self); - if (set_external_queue_type(self, cls_queue) < 0) { + if (set_external_queue_type(state, (PyTypeObject *)queuetype) < 0) { + return NULL; + } + if (set_external_exc_types(state, emptyerror, fullerror) < 0) { return NULL; } @@ -1579,21 +1870,23 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, queuesmod_destroy_doc}, {"list_all", queuesmod_list_all, METH_NOARGS, queuesmod_list_all_doc}, - {"put", _PyCFunction_CAST(queuesmod_put), + {"put", _PyCFunction_CAST(queuesmod_put), METH_VARARGS | METH_KEYWORDS, queuesmod_put_doc}, - {"get", _PyCFunction_CAST(queuesmod_get), + {"get", _PyCFunction_CAST(queuesmod_get), METH_VARARGS | METH_KEYWORDS, queuesmod_get_doc}, - {"bind", _PyCFunction_CAST(queuesmod_bind), + {"bind", _PyCFunction_CAST(queuesmod_bind), METH_VARARGS | METH_KEYWORDS, queuesmod_bind_doc}, {"release", _PyCFunction_CAST(queuesmod_release), METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, + {"get_queue_defaults", _PyCFunction_CAST(queuesmod_get_queue_defaults), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_queue_defaults_doc}, {"is_full", _PyCFunction_CAST(queuesmod_is_full), METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, {"get_count", _PyCFunction_CAST(queuesmod_get_count), METH_VARARGS | METH_KEYWORDS, queuesmod_get_count_doc}, - {"_register_queue_type", _PyCFunction_CAST(queuesmod__register_queue_type), + {"_register_heap_types", _PyCFunction_CAST(queuesmod__register_heap_types), METH_VARARGS | METH_KEYWORDS, NULL}, {NULL, NULL} /* sentinel */ @@ -1609,7 +1902,8 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { - if (_globals_init() != 0) { + int err = _globals_init(); + if (handle_queue_error(err, mod, -1)) { return -1; } @@ -1632,6 +1926,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -1648,10 +1943,6 @@ module_clear(PyObject *mod) { module_state *state = get_module_state(mod); - if (state->queue_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->queue_type); - } - // Now we clear the module state. clear_module_state(state); return 0; @@ -1670,7 +1961,7 @@ module_free(void *mod) static struct PyModuleDef moduledef = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = MODULE_NAME, + .m_name = MODULE_NAME_STR, .m_doc = module_doc, .m_size = sizeof(module_state), .m_methods = module_functions, @@ -1681,7 +1972,7 @@ static struct PyModuleDef moduledef = { }; PyMODINIT_FUNC -PyInit__xxinterpqueues(void) +MODINIT_FUNC_NAME(void) { return PyModuleDef_Init(&moduledef); } diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h new file mode 100644 index 00000000000000..0d2e0c9efd3837 --- /dev/null +++ b/Modules/_interpreters_common.h @@ -0,0 +1,66 @@ + +#define _RESOLVE_MODINIT_FUNC_NAME(NAME) \ + PyInit_ ## NAME +#define RESOLVE_MODINIT_FUNC_NAME(NAME) \ + _RESOLVE_MODINIT_FUNC_NAME(NAME) + + +static int +ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata) +{ + //assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE); + return _PyCrossInterpreterData_RegisterClass(cls, getdata); +} + +#ifdef REGISTERS_HEAP_TYPES +static int +clear_xid_class(PyTypeObject *cls) +{ + return _PyCrossInterpreterData_UnregisterClass(cls); +} +#endif + + +static inline int64_t +_get_interpid(_PyCrossInterpreterData *data) +{ + int64_t interpid; + if (data != NULL) { + interpid = _PyCrossInterpreterData_INTERPID(data); + assert(!PyErr_Occurred()); + } + else { + interpid = PyInterpreterState_GetID(PyInterpreterState_Get()); + } + return interpid; +} + + +/* unbound items ************************************************************/ + +#ifdef HAS_UNBOUND_ITEMS + +#define UNBOUND_REMOVE 1 +#define UNBOUND_ERROR 2 +#define UNBOUND_REPLACE 3 + +// It would also be possible to add UNBOUND_REPLACE where the replacement +// value is user-provided. There would be some limitations there, though. +// Another possibility would be something like UNBOUND_COPY, where the +// object is released but the underlying data is copied (with the "raw" +// allocator) and used when the item is popped off the queue. + +static int +check_unbound(int unboundop) +{ + switch (unboundop) { + case UNBOUND_REMOVE: + case UNBOUND_ERROR: + case UNBOUND_REPLACE: + return 1; + default: + return 0; + } +} + +#endif diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_interpretersmodule.c similarity index 55% rename from Modules/_xxsubinterpretersmodule.c rename to Modules/_interpretersmodule.c index 4e9e13457a9eb3..63f2bb38768511 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -10,17 +10,21 @@ #include "pycore_crossinterp.h" // struct _xid #include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() -#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() -#include "interpreteridobject.h" #include "marshal.h" // PyMarshal_ReadObjectFromString() +#include "_interpreters_common.h" -#define MODULE_NAME "_xxsubinterpreters" + +#define MODULE_NAME _interpreters +#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) +#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) static PyInterpreterState * @@ -31,101 +35,13 @@ _get_current_interp(void) return PyInterpreterState_Get(); } -static int64_t -pylong_to_interpid(PyObject *idobj) -{ - assert(PyLong_CheckExact(idobj)); - - if (_PyLong_IsNegative((PyLongObject *)idobj)) { - PyErr_Format(PyExc_ValueError, - "interpreter ID must be a non-negative int, got %R", - idobj); - return -1; - } - - int overflow; - long long id = PyLong_AsLongLongAndOverflow(idobj, &overflow); - if (id == -1) { - if (!overflow) { - assert(PyErr_Occurred()); - return -1; - } - assert(!PyErr_Occurred()); - // For now, we don't worry about if LLONG_MAX < INT64_MAX. - goto bad_id; - } -#if LLONG_MAX > INT64_MAX - if (id > INT64_MAX) { - goto bad_id; - } -#endif - return (int64_t)id; - -bad_id: - PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %O", idobj); - return -1; -} - -static int64_t -convert_interpid_obj(PyObject *arg) -{ - int64_t id = -1; - if (_PyIndex_Check(arg)) { - PyObject *idobj = PyNumber_Long(arg); - if (idobj == NULL) { - return -1; - } - id = pylong_to_interpid(idobj); - Py_DECREF(idobj); - if (id < 0) { - return -1; - } - } - else { - PyErr_Format(PyExc_TypeError, - "interpreter ID must be an int, got %.100s", - Py_TYPE(arg)->tp_name); - return -1; - } - return id; -} - -static PyInterpreterState * -look_up_interp(PyObject *arg) -{ - int64_t id = convert_interpid_obj(arg); - if (id < 0) { - return NULL; - } - return _PyInterpreterState_LookUpID(id); -} - +#define look_up_interp _PyInterpreterState_LookUpIDObject -static PyObject * -interpid_to_pylong(int64_t id) -{ - assert(id < LLONG_MAX); - return PyLong_FromLongLong(id); -} - -static PyObject * -get_interpid_obj(PyInterpreterState *interp) -{ - if (_PyInterpreterState_IDInitref(interp) != 0) { - return NULL; - }; - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - return interpid_to_pylong(id); -} static PyObject * _get_current_module(void) { - PyObject *name = PyUnicode_FromString(MODULE_NAME); + PyObject *name = PyUnicode_FromString(MODULE_NAME_STR); if (name == NULL) { return NULL; } @@ -139,6 +55,24 @@ _get_current_module(void) } +static int +is_running_main(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IsRunningMain(interp)) { + return 1; + } + // Unlike with the general C-API, we can be confident that someone + // using this module for the main interpreter is doing so through + // the main program. Thus we can make this extra check. This benefits + // applications that embed Python but haven't been updated yet + // to call_PyInterpreterState_SetRunningMain(). + if (_Py_IsMainInterpreter(interp)) { + return 1; + } + return 0; +} + + /* Cross-interpreter Buffer Views *******************************************/ // XXX Release when the original interpreter is destroyed. @@ -152,16 +86,16 @@ typedef struct { static PyObject * xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) { - assert(data->data != NULL); - assert(data->obj == NULL); - assert(data->interpid >= 0); + assert(_PyCrossInterpreterData_DATA(data) != NULL); + assert(_PyCrossInterpreterData_OBJ(data) == NULL); + assert(_PyCrossInterpreterData_INTERPID(data) >= 0); XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); if (self == NULL) { return NULL; } PyObject_Init((PyObject *)self, cls); - self->view = (Py_buffer *)data->data; - self->interpid = data->interpid; + self->view = (Py_buffer *)_PyCrossInterpreterData_DATA(data); + self->interpid = _PyCrossInterpreterData_INTERPID(data); return (PyObject *)self; } @@ -209,7 +143,7 @@ static PyType_Slot XIBufferViewType_slots[] = { }; static PyType_Spec XIBufferViewType_spec = { - .name = MODULE_NAME ".CrossInterpreterBufferView", + .name = MODULE_NAME_STR ".CrossInterpreterBufferView", .basicsize = sizeof(XIBufferViewObject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), @@ -267,8 +201,7 @@ register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) *p_state = cls; // Register XID for the builtin memoryview type. - if (_PyCrossInterpreterData_RegisterClass( - &PyMemoryView_Type, _memoryview_shared) < 0) { + if (ensure_xid_class(&PyMemoryView_Type, _memoryview_shared) < 0) { return -1; } // We don't ever bother un-registering memoryview. @@ -303,7 +236,7 @@ _get_current_module_state(void) if (mod == NULL) { // XXX import it? PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME " module not imported yet"); + MODULE_NAME_STR " module not imported yet"); return NULL; } module_state *state = get_module_state(mod); @@ -372,9 +305,7 @@ check_code_object(PyCodeObject *code) } // We trust that no code objects under co_consts have unbound cell vars. - if (code->co_executors != NULL - || code->_co_instrumentation_version > 0) - { + if (_PyCode_HAS_EXECUTORS(code) || _PyCode_HAS_INSTRUMENTATION(code)) { return "only basic functions are supported"; } if (code->_co_monitoring != NULL) { @@ -435,6 +366,66 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) /* interpreter-specific code ************************************************/ +static int +init_named_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL + || strcmp(name, "") == 0 + || strcmp(name, "default") == 0) + { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + +static int +config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + const char *utf8name = PyUnicode_AsUTF8(configobj); + if (utf8name == NULL) { + return -1; + } + if (init_named_config(config, utf8name) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -503,80 +494,195 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ +static long +get_whence(PyInterpreterState *interp) +{ + return _PyInterpreterState_GetWhence(interp); +} + + +static PyInterpreterState * +resolve_interp(PyObject *idobj, int restricted, int reqready, const char *op) +{ + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = look_up_interp(idobj); + if (interp == NULL) { + return NULL; + } + } + + if (reqready && !_PyInterpreterState_IsReady(interp)) { + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s current interpreter (not ready)", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s interpreter %R (not ready)", op, idobj); + } + return NULL; + } + + if (restricted && get_whence(interp) != _PyInterpreterState_WHENCE_STDLIB) { + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized current interpreter", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized interpreter %R", op, idobj); + } + return NULL; + } + + return interp; +} + + +static PyObject * +get_summary(PyInterpreterState *interp) +{ + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + return NULL; + } + PyObject *whenceobj = PyLong_FromLong( + get_whence(interp)); + if (whenceobj == NULL) { + Py_DECREF(idobj); + return NULL; + } + PyObject *res = PyTuple_Pack(2, idobj, whenceobj); + Py_DECREF(idobj); + Py_DECREF(whenceobj); + return res; +} + + +static PyObject * +interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", + &name)) + { + return NULL; + } + PyObject *overrides = kwds; + + PyInterpreterConfig config; + if (init_named_config(&config, name) < 0) { + return NULL; + } + + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(new_config_doc, +"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ +\n\ +Return a representation of a new PyInterpreterConfig.\n\ +\n\ +The name determines the initial values of the config. Supported named\n\ +configs are: default, isolated, legacy, and empty.\n\ +\n\ +Any keyword arguments are set on the corresponding config fields,\n\ +overriding the initial values."); + + static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"config", "reqrefs", NULL}; + PyObject *configobj = NULL; + int reqrefs = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, + &configobj, &reqrefs)) { + return NULL; + } - static char *kwlist[] = {"isolated", NULL}; - int isolated = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist, - &isolated)) { + PyInterpreterConfig config; + if (config_from_object(configobj, &config) < 0) { return NULL; } - // Create and initialize the new interpreter. - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - const PyInterpreterConfig config = isolated - ? (PyInterpreterConfig)_PyInterpreterConfig_INIT - : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - - // XXX Possible GILState issues? - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); + long whence = _PyInterpreterState_WHENCE_STDLIB; + PyInterpreterState *interp = \ + _PyXI_NewInterpreter(&config, &whence, NULL, NULL); + if (interp == NULL) { + // XXX Move the chained exception to interpreters.create()? PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); + assert(exc != NULL); + PyErr_SetString(PyExc_InterpreterError, "interpreter creation failed"); _PyErr_ChainExceptions1(exc); return NULL; } - assert(tstate != NULL); + assert(_PyInterpreterState_IsReady(interp)); - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = get_interpid_obj(interp); + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); if (idobj == NULL) { - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); + _PyXI_EndInterpreter(interp, NULL, NULL); return NULL; } - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); + if (reqrefs) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } - _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } + PyDoc_STRVAR(create_doc, -"create() -> ID\n\ +"create([config], *, reqrefs=False) -> ID\n\ \n\ Create a new interpreter and return a unique generated ID.\n\ \n\ -The caller is responsible for destroying the interpreter before exiting."); +The caller is responsible for destroying the interpreter before exiting,\n\ +typically by using _interpreters.destroy(). This can be managed \n\ +automatically by passing \"reqrefs=True\" and then using _incref() and\n\ +_decref() appropriately.\n\ +\n\ +\"config\" must be a valid interpreter config or the name of a\n\ +predefined config (\"isolated\" or \"legacy\"). The default\n\ +is \"isolated\"."); static PyObject * interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *id; + int restricted = 0; // XXX Use "L" for id? if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:destroy", kwlist, &id)) { + "O|$p:destroy", kwlist, &id, &restricted)) + { return NULL; } // Look up the interpreter. - PyInterpreterState *interp = look_up_interp(id); + int reqready = 0; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "destroy"); if (interp == NULL) { return NULL; } @@ -587,7 +693,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } if (interp == current) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_InterpreterError, "cannot destroy the current interpreter"); return NULL; } @@ -595,57 +701,60 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) // Ensure the interpreter isn't running. /* XXX We *could* support destroying a running interpreter but aren't going to worry about it for now. */ - if (_PyInterpreterState_IsRunningMain(interp)) { - PyErr_Format(PyExc_RuntimeError, "interpreter running"); + if (is_running_main(interp)) { + PyErr_Format(PyExc_InterpreterError, "interpreter running"); return NULL; } // Destroy the interpreter. - PyThreadState *tstate = PyThreadState_New(interp); - tstate->_whence = _PyThreadState_WHENCE_INTERP; - // XXX Possible GILState issues? - PyThreadState *save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); + _PyXI_EndInterpreter(interp, NULL, NULL); Py_RETURN_NONE; } PyDoc_STRVAR(destroy_doc, -"destroy(id)\n\ +"destroy(id, *, restrict=False)\n\ \n\ Destroy the identified interpreter.\n\ \n\ -Attempting to destroy the current interpreter results in a RuntimeError.\n\ +Attempting to destroy the current interpreter raises InterpreterError.\n\ So does an unrecognized ID."); static PyObject * -interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) +interp_list_all(PyObject *self, PyObject *args, PyObject *kwargs) { - PyObject *ids, *id; - PyInterpreterState *interp; + static char *kwlist[] = {"require_ready", NULL}; + int reqready = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|$p:" MODULE_NAME_STR ".list_all", + kwlist, &reqready)) + { + return NULL; + } - ids = PyList_New(0); + PyObject *ids = PyList_New(0); if (ids == NULL) { return NULL; } - interp = PyInterpreterState_Head(); + PyInterpreterState *interp = PyInterpreterState_Head(); while (interp != NULL) { - id = get_interpid_obj(interp); - if (id == NULL) { - Py_DECREF(ids); - return NULL; - } - // insert at front of list - int res = PyList_Insert(ids, 0, id); - Py_DECREF(id); - if (res < 0) { - Py_DECREF(ids); - return NULL; - } + if (!reqready || _PyInterpreterState_IsReady(interp)) { + PyObject *item = get_summary(interp); + if (item == NULL) { + Py_DECREF(ids); + return NULL; + } + // insert at front of list + int res = PyList_Insert(ids, 0, item); + Py_DECREF(item); + if (res < 0) { + Py_DECREF(ids); + return NULL; + } + } interp = PyInterpreterState_Next(interp); } @@ -653,7 +762,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(list_all_doc, -"list_all() -> [ID]\n\ +"list_all() -> [(ID, whence)]\n\ \n\ Return a list containing the ID of every existing interpreter."); @@ -665,11 +774,12 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) if (interp == NULL) { return NULL; } - return get_interpid_obj(interp); + assert(_PyInterpreterState_IsReady(interp)); + return get_summary(interp); } PyDoc_STRVAR(get_current_doc, -"get_current() -> ID\n\ +"get_current() -> (ID, whence)\n\ \n\ Return the ID of current interpreter."); @@ -677,28 +787,34 @@ Return the ID of current interpreter."); static PyObject * interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) { - // Currently, 0 is always the main interpreter. - int64_t id = 0; - return PyLong_FromLongLong(id); + PyInterpreterState *interp = _PyInterpreterState_Main(); + assert(_PyInterpreterState_IsReady(interp)); + return get_summary(interp); } PyDoc_STRVAR(get_main_doc, -"get_main() -> ID\n\ +"get_main() -> (ID, whence)\n\ \n\ Return the ID of main interpreter."); + static PyObject * -interp_set___main___attrs(PyObject *self, PyObject *args) +interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) { + static char *kwlist[] = {"id", "updates", "restrict", NULL}; PyObject *id, *updates; - if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME ".set___main___attrs", - &id, &updates)) + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "OO|$p:" MODULE_NAME_STR ".set___main___attrs", + kwlist, &id, &updates, &restricted)) { return NULL; } // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "update __main__ for"); if (interp == NULL) { return NULL; } @@ -737,10 +853,11 @@ interp_set___main___attrs(PyObject *self, PyObject *args) } PyDoc_STRVAR(set___main___attrs_doc, -"set___main___attrs(id, ns)\n\ +"set___main___attrs(id, ns, *, restrict=False)\n\ \n\ Bind the given attributes in the interpreter's __main__ module."); + static PyUnicodeObject * convert_script_arg(PyObject *arg, const char *fname, const char *displayname, const char *expected) @@ -818,16 +935,9 @@ convert_code_arg(PyObject *arg, const char *fname, const char *displayname, } static int -_interp_exec(PyObject *self, - PyObject *id_arg, PyObject *code_arg, PyObject *shared_arg, - PyObject **p_excinfo) +_interp_exec(PyObject *self, PyInterpreterState *interp, + PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) { - // Look up the interpreter. - PyInterpreterState *interp = look_up_interp(id_arg); - if (interp == NULL) { - return -1; - } - // Extract code. Py_ssize_t codestrlen = -1; PyObject *bytes_obj = NULL; @@ -852,22 +962,31 @@ _interp_exec(PyObject *self, static PyObject * interp_exec(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "code", "shared", NULL}; + static char *kwlist[] = {"id", "code", "shared", "restrict", NULL}; PyObject *id, *code; PyObject *shared = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME ".exec", kwlist, - &id, &code, &shared)) { + "OO|O$p:" MODULE_NAME_STR ".exec", kwlist, + &id, &code, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "exec code for"); + if (interp == NULL) { return NULL; } const char *expected = "a string, a function, or a code object"; if (PyUnicode_Check(code)) { - code = (PyObject *)convert_script_arg(code, MODULE_NAME ".exec", + code = (PyObject *)convert_script_arg(code, MODULE_NAME_STR ".exec", "argument 2", expected); } else { - code = (PyObject *)convert_code_arg(code, MODULE_NAME ".exec", + code = (PyObject *)convert_code_arg(code, MODULE_NAME_STR ".exec", "argument 2", expected); } if (code == NULL) { @@ -875,7 +994,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, code, shared, &excinfo); + int res = _interp_exec(self, interp, code, shared, &excinfo); Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -885,7 +1004,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(exec_doc, -"exec(id, code, shared=None)\n\ +"exec(id, code, shared=None, *, restrict=False)\n\ \n\ Execute the provided code in the identified interpreter.\n\ This is equivalent to running the builtin exec() under the target\n\ @@ -901,26 +1020,96 @@ The code/function must not take any arguments or be a closure\n\ If a function is provided, its code object is used and all its state\n\ is ignored, including its __globals__ dict."); +static PyObject * +interp_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "callable", "args", "kwargs", + "restrict", NULL}; + PyObject *id, *callable; + PyObject *args_obj = NULL; + PyObject *kwargs_obj = NULL; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|OO$p:" MODULE_NAME_STR ".call", kwlist, + &id, &callable, &args_obj, &kwargs_obj, + &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "make a call in"); + if (interp == NULL) { + return NULL; + } + + if (args_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected args"); + return NULL; + } + if (kwargs_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected kwargs"); + return NULL; + } + + PyObject *code = (PyObject *)convert_code_arg(callable, MODULE_NAME_STR ".call", + "argument 2", "a function"); + if (code == NULL) { + return NULL; + } + + PyObject *excinfo = NULL; + int res = _interp_exec(self, interp, code, NULL, &excinfo); + Py_DECREF(code); + if (res < 0) { + assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); + return excinfo; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(call_doc, +"call(id, callable, args=None, kwargs=None, *, restrict=False)\n\ +\n\ +Call the provided object in the identified interpreter.\n\ +Pass the given args and kwargs, if possible.\n\ +\n\ +\"callable\" may be a plain function with no free vars that takes\n\ +no arguments.\n\ +\n\ +The function's code object is used and all its state\n\ +is ignored, including its __globals__ dict."); + static PyObject * interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "script", "shared", NULL}; + static char *kwlist[] = {"id", "script", "shared", "restrict", NULL}; PyObject *id, *script; PyObject *shared = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O:" MODULE_NAME ".run_string", kwlist, - &id, &script, &shared)) { + "OU|O$p:" MODULE_NAME_STR ".run_string", + kwlist, &id, &script, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "run a string in"); + if (interp == NULL) { return NULL; } - script = (PyObject *)convert_script_arg(script, MODULE_NAME ".exec", + script = (PyObject *)convert_script_arg(script, MODULE_NAME_STR ".exec", "argument 2", "a string"); if (script == NULL) { return NULL; } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, script, shared, &excinfo); + int res = _interp_exec(self, interp, script, shared, &excinfo); Py_DECREF(script); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -930,25 +1119,34 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(run_string_doc, -"run_string(id, script, shared=None)\n\ +"run_string(id, script, shared=None, *, restrict=False)\n\ \n\ Execute the provided string in the identified interpreter.\n\ \n\ -(See " MODULE_NAME ".exec()."); +(See " MODULE_NAME_STR ".exec()."); static PyObject * interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "func", "shared", NULL}; + static char *kwlist[] = {"id", "func", "shared", "restrict", NULL}; PyObject *id, *func; PyObject *shared = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME ".run_func", kwlist, - &id, &func, &shared)) { + "OO|O$p:" MODULE_NAME_STR ".run_func", + kwlist, &id, &func, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "run a function in"); + if (interp == NULL) { return NULL; } - PyCodeObject *code = convert_code_arg(func, MODULE_NAME ".exec", + PyCodeObject *code = convert_code_arg(func, MODULE_NAME_STR ".exec", "argument 2", "a function or a code object"); if (code == NULL) { @@ -956,7 +1154,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, (PyObject *)code, shared, &excinfo); + int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo); Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -966,13 +1164,13 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(run_func_doc, -"run_func(id, func, shared=None)\n\ +"run_func(id, func, shared=None, *, restrict=False)\n\ \n\ Execute the body of the provided function in the identified interpreter.\n\ Code objects are also supported. In both cases, closures and args\n\ are not supported. Methods and other callables are not supported either.\n\ \n\ -(See " MODULE_NAME ".exec()."); +(See " MODULE_NAME_STR ".exec()."); static PyObject * @@ -1002,36 +1200,86 @@ False otherwise."); static PyObject * interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *id; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:is_running", kwlist, &id)) { + "O|$p:is_running", kwlist, + &id, &restricted)) + { return NULL; } - PyInterpreterState *interp = look_up_interp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "check if running for"); if (interp == NULL) { return NULL; } - if (_PyInterpreterState_IsRunningMain(interp)) { + + if (is_running_main(interp)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; } PyDoc_STRVAR(is_running_doc, -"is_running(id) -> bool\n\ +"is_running(id, *, restrict=False) -> bool\n\ \n\ Return whether or not the identified interpreter is running."); static PyObject * -interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "restrict", NULL}; + PyObject *idobj = NULL; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$p:get_config", kwlist, + &idobj, &restricted)) + { + return NULL; + } + if (idobj == Py_None) { + idobj = NULL; + } + + int reqready = 0; + PyInterpreterState *interp = \ + resolve_interp(idobj, restricted, reqready, "get the config of"); + if (interp == NULL) { + return NULL; + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(get_config_doc, +"get_config(id, *, restrict=False) -> types.SimpleNamespace\n\ +\n\ +Return a representation of the config used to initialize the interpreter."); + + +static PyObject * +interp_whence(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", NULL}; PyObject *id; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O:whence", kwlist, &id)) + { return NULL; } @@ -1039,9 +1287,42 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - if (_PyInterpreterState_IDInitref(interp) < 0) { + + long whence = get_whence(interp); + return PyLong_FromLong(whence); +} + +PyDoc_STRVAR(whence_doc, +"whence(id) -> int\n\ +\n\ +Return an identifier for where the interpreter was created."); + + +static PyObject * +interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "implieslink", "restrict", NULL}; + PyObject *id; + int implieslink = 0; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$pp:incref", kwlist, + &id, &implieslink, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "incref"); + if (interp == NULL) { return NULL; } + + if (implieslink) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } _PyInterpreterState_IDIncref(interp); Py_RETURN_NONE; @@ -1051,30 +1332,106 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * interp_decref(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *id; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O|$p:decref", kwlist, &id, &restricted)) + { return NULL; } - PyInterpreterState *interp = look_up_interp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "decref"); if (interp == NULL) { return NULL; } + _PyInterpreterState_IDDecref(interp); Py_RETURN_NONE; } +static PyObject * +capture_exception(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"exc", NULL}; + PyObject *exc_arg = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "|O:capture_exception", kwlist, + &exc_arg)) + { + return NULL; + } + + PyObject *exc = exc_arg; + if (exc == NULL || exc == Py_None) { + exc = PyErr_GetRaisedException(); + if (exc == NULL) { + Py_RETURN_NONE; + } + } + else if (!PyExceptionInstance_Check(exc)) { + PyErr_Format(PyExc_TypeError, "expected exception, got %R", exc); + return NULL; + } + PyObject *captured = NULL; + + _PyXI_excinfo info = {0}; + if (_PyXI_InitExcInfo(&info, exc) < 0) { + goto finally; + } + captured = _PyXI_ExcInfoAsObject(&info); + if (captured == NULL) { + goto finally; + } + + PyObject *formatted = _PyXI_FormatExcInfo(&info); + if (formatted == NULL) { + Py_CLEAR(captured); + goto finally; + } + int res = PyObject_SetAttrString(captured, "formatted", formatted); + Py_DECREF(formatted); + if (res < 0) { + Py_CLEAR(captured); + goto finally; + } + +finally: + _PyXI_ClearExcInfo(&info); + if (exc != exc_arg) { + if (PyErr_Occurred()) { + PyErr_SetRaisedException(exc); + } + else { + _PyErr_ChainExceptions1(exc); + } + } + return captured; +} + +PyDoc_STRVAR(capture_exception_doc, +"capture_exception(exc=None) -> types.SimpleNamespace\n\ +\n\ +Return a snapshot of an exception. If \"exc\" is None\n\ +then the current exception, if any, is used (but not cleared).\n\ +\n\ +The returned snapshot is the same as what _interpreters.exec() returns."); + + static PyMethodDef module_functions[] = { + {"new_config", _PyCFunction_CAST(interp_new_config), + METH_VARARGS | METH_KEYWORDS, new_config_doc}, + {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), METH_VARARGS | METH_KEYWORDS, destroy_doc}, - {"list_all", interp_list_all, - METH_NOARGS, list_all_doc}, + {"list_all", _PyCFunction_CAST(interp_list_all), + METH_VARARGS | METH_KEYWORDS, list_all_doc}, {"get_current", interp_get_current, METH_NOARGS, get_current_doc}, {"get_main", interp_get_main, @@ -1082,23 +1439,33 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, + {"get_config", _PyCFunction_CAST(interp_get_config), + METH_VARARGS | METH_KEYWORDS, get_config_doc}, + {"whence", _PyCFunction_CAST(interp_whence), + METH_VARARGS | METH_KEYWORDS, whence_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, + {"call", _PyCFunction_CAST(interp_call), + METH_VARARGS | METH_KEYWORDS, call_doc}, {"run_string", _PyCFunction_CAST(interp_run_string), METH_VARARGS | METH_KEYWORDS, run_string_doc}, {"run_func", _PyCFunction_CAST(interp_run_func), METH_VARARGS | METH_KEYWORDS, run_func_doc}, {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), - METH_VARARGS, set___main___attrs_doc}, - {"is_shareable", _PyCFunction_CAST(object_is_shareable), - METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + METH_VARARGS | METH_KEYWORDS, set___main___attrs_doc}, - {"_incref", _PyCFunction_CAST(interp_incref), + {"incref", _PyCFunction_CAST(interp_incref), METH_VARARGS | METH_KEYWORDS, NULL}, - {"_decref", _PyCFunction_CAST(interp_decref), + {"decref", _PyCFunction_CAST(interp_decref), METH_VARARGS | METH_KEYWORDS, NULL}, + {"is_shareable", _PyCFunction_CAST(object_is_shareable), + METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + + {"capture_exception", _PyCFunction_CAST(capture_exception), + METH_VARARGS | METH_KEYWORDS, capture_exception_doc}, + {NULL, NULL} /* sentinel */ }; @@ -1112,8 +1479,23 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { + PyInterpreterState *interp = PyInterpreterState_Get(); module_state *state = get_module_state(mod); +#define ADD_WHENCE(NAME) \ + if (PyModule_AddIntConstant(mod, "WHENCE_" #NAME, \ + _PyInterpreterState_WHENCE_##NAME) < 0) \ + { \ + goto error; \ + } + ADD_WHENCE(UNKNOWN) + ADD_WHENCE(RUNTIME) + ADD_WHENCE(LEGACY_CAPI) + ADD_WHENCE(CAPI) + ADD_WHENCE(XI) + ADD_WHENCE(STDLIB) +#undef ADD_WHENCE + // exceptions if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { goto error; @@ -1121,6 +1503,11 @@ module_exec(PyObject *mod) if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { goto error; } + PyObject *PyExc_NotShareableError = \ + _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_NotShareableError) < 0) { + goto error; + } if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) { goto error; @@ -1135,6 +1522,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -1166,7 +1554,7 @@ module_free(void *mod) static struct PyModuleDef moduledef = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = MODULE_NAME, + .m_name = MODULE_NAME_STR, .m_doc = module_doc, .m_size = sizeof(module_state), .m_methods = module_functions, @@ -1177,7 +1565,7 @@ static struct PyModuleDef moduledef = { }; PyMODINIT_FUNC -PyInit__xxsubinterpreters(void) +MODINIT_FUNC_NAME(void) { return PyModuleDef_Init(&moduledef); } diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 173f5b55e5f732..6622f2cabb908b 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -10,7 +10,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_long.h" // _PyLong_Sign() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -202,7 +202,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, const char *newline, int closefd, PyObject *opener) /*[clinic end generated code: output=aefafc4ce2b46dc0 input=cd034e7cdfbf4e78]*/ { - unsigned i; + size_t i; int creating = 0, reading = 0, writing = 0, appending = 0, updating = 0; int text = 0, binary = 0; @@ -346,7 +346,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, /* buffering */ if (buffering < 0) { - PyObject *res = PyObject_CallMethodNoArgs(raw, &_Py_ID(isatty)); + PyObject *res = PyObject_CallMethodNoArgs(raw, &_Py_ID(_isatty_open_only)); if (res == NULL) goto error; isatty = PyObject_IsTrue(res); @@ -544,10 +544,7 @@ PyNumber_AsOff_t(PyObject *item, PyObject *err) */ if (!err) { assert(PyLong_Check(value)); - /* Whether or not it is less than or equal to - zero is determined by the sign of ob_size - */ - if (_PyLong_Sign(value) < 0) + if (_PyLong_IsNegative((PyLongObject *)value)) result = PY_OFF_T_MIN; else result = PY_OFF_T_MAX; @@ -720,6 +717,7 @@ iomodule_exec(PyObject *m) static struct PyModuleDef_Slot iomodule_slots[] = { {Py_mod_exec, iomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 8ebe9ec7095586..bc5fff54a62b6d 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -8,7 +8,6 @@ */ #include "Python.h" -#include "pycore_bytesobject.h" // _PyBytes_Join() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() @@ -1284,7 +1283,7 @@ _buffered_readline(buffered *self, Py_ssize_t limit) Py_CLEAR(res); goto end; } - Py_XSETREF(res, _PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks)); + Py_XSETREF(res, PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks)); end: LEAVE_BUFFERED(self) @@ -1325,7 +1324,11 @@ _io__Buffered_tell_impl(buffered *self) if (pos == -1) return NULL; pos -= RAW_OFFSET(self); - /* TODO: sanity check (pos >= 0) */ + + // GH-95782 + if (pos < 0) + pos = 0; + return PyLong_FromOff_t(pos); } @@ -1395,6 +1398,11 @@ _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence) offset = target; if (offset >= -self->pos && offset <= avail) { self->pos += offset; + + // GH-95782 + if (current - avail + offset < 0) + return PyLong_FromOff_t(0); + return PyLong_FromOff_t(current - avail + offset); } } @@ -1728,7 +1736,7 @@ _bufferedreader_read_all(buffered *self) goto cleanup; } else { - tmp = _PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks); + tmp = PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks); res = tmp; goto cleanup; } @@ -2083,7 +2091,7 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) self->raw_pos = 0; } avail = Py_SAFE_DOWNCAST(self->buffer_size - self->pos, Py_off_t, Py_ssize_t); - if (buffer->len <= avail) { + if (buffer->len <= avail && buffer->len < self->buffer_size) { memcpy(self->buffer + self->pos, buffer->buf, buffer->len); if (!VALID_WRITE_BUFFER(self) || self->write_pos > self->pos) { self->write_pos = self->pos; @@ -2152,7 +2160,7 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) /* Then write buf itself. At this point the buffer has been emptied. */ remaining = buffer->len; written = 0; - while (remaining > self->buffer_size) { + while (remaining >= self->buffer_size) { Py_ssize_t n = _bufferedwriter_raw_write( self, (char *) buffer->buf + written, buffer->len - written); if (n == -1) { @@ -2522,8 +2530,8 @@ static PyMethodDef bufferedreader_methods[] = { _IO__BUFFERED_TRUNCATE_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; @@ -2582,8 +2590,8 @@ static PyMethodDef bufferedwriter_methods[] = { _IO__BUFFERED_TELL_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; @@ -2700,8 +2708,8 @@ static PyMethodDef bufferedrandom_methods[] = { _IO_BUFFEREDWRITER_WRITE_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 4a15c8e841f25f..fb66d3db0f7a1f 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -155,9 +155,6 @@ resize_buffer(bytesio *self, size_t size) alloc = size + 1; } - if (alloc > ((size_t)-1) / sizeof(char)) - goto overflow; - if (SHARED_BUF(self)) { if (unshare_buffer(self, alloc) < 0) return -1; diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 64eddcd314a803..708bef638887e2 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() @@ -1245,4 +1245,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=4249187a725a3b3e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8eead000083dc5fa input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index bae80a265fab07..a35cac7dc0b8d7 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -438,4 +441,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=e7326fbefc52bfba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dab5e9323d191e32 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index f04ee729abc9ed..669e2aa637ebbf 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() @@ -1292,4 +1292,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=93a5a91a22100a28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=04cb7c67791a9ec1 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 6bb156e41fe43c..cf0f1d671b507a 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -12,9 +12,6 @@ #ifdef HAVE_SYS_TYPES_H # include #endif -#ifdef HAVE_SYS_STAT_H -# include -#endif #ifdef HAVE_IO_H # include #endif @@ -54,6 +51,9 @@ # define SMALLCHUNK BUFSIZ #endif +/* Size at which a buffer is considered "large" and behavior should change to + avoid excessive memory allocation */ +#define LARGE_BUFFER_CUTOFF_SIZE 65536 /*[clinic input] module _io @@ -71,12 +71,19 @@ typedef struct { signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; char finalizing; - unsigned int blksize; + /* Stat result which was grabbed at file open, useful for optimizing common + File I/O patterns to be more efficient. This is only guidance / an + estimate, as it is subject to Time-Of-Check to Time-Of-Use (TOCTOU) + issues / bugs. Both the underlying file descriptor and file may be + modified outside of the fileio object / Python (ex. gh-90102, GH-121941, + gh-109523). */ + struct _Py_stat_struct *stat_atopen; PyObject *weakreflist; PyObject *dict; } fileio; #define PyFileIO_Check(state, op) (PyObject_TypeCheck((op), state->PyFileIO_Type)) +#define _PyFileIO_CAST(op) _Py_CAST(fileio*, (op)) /* Forward declarations */ static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_error); @@ -84,15 +91,16 @@ static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool int _PyFileIO_closed(PyObject *self) { - return ((fileio *)self)->fd < 0; + return (_PyFileIO_CAST(self)->fd < 0); } /* Because this can call arbitrary code, it shouldn't be called when the refcount is 0 (that is, not directly from tp_dealloc unless the refcount has been temporarily re-incremented). */ static PyObject * -fileio_dealloc_warn(fileio *self, PyObject *source) +fileio_dealloc_warn(PyObject *op, PyObject *source) { + fileio *self = _PyFileIO_CAST(op); if (self->fd >= 0 && self->closefd) { PyObject *exc = PyErr_GetRaisedException(); if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) { @@ -123,6 +131,8 @@ internal_close(fileio *self) _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; if (err < 0) { errno = save_errno; PyErr_SetFromErrno(PyExc_OSError); @@ -162,7 +172,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) exc = PyErr_GetRaisedException(); } if (self->finalizing) { - PyObject *r = fileio_dealloc_warn(self, (PyObject *) self); + PyObject *r = fileio_dealloc_warn((PyObject*)self, (PyObject *) self); if (r) { Py_DECREF(r); } @@ -183,23 +193,22 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) static PyObject * fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - fileio *self; - assert(type != NULL && type->tp_alloc != NULL); - self = (fileio *) type->tp_alloc(type, 0); - if (self != NULL) { - self->fd = -1; - self->created = 0; - self->readable = 0; - self->writable = 0; - self->appending = 0; - self->seekable = -1; - self->blksize = 0; - self->closefd = 1; - self->weakreflist = NULL; + fileio *self = (fileio *) type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; } + self->fd = -1; + self->created = 0; + self->readable = 0; + self->writable = 0; + self->appending = 0; + self->seekable = -1; + self->stat_atopen = NULL; + self->closefd = 1; + self->weakreflist = NULL; return (PyObject *) self; } @@ -251,7 +260,6 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #elif !defined(MS_WINDOWS) int *atomic_flag_works = NULL; #endif - struct _Py_stat_struct fdfstat; int fstat_result; int async_err = 0; @@ -262,8 +270,9 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ - if (internal_close(self) < 0) + if (internal_close(self) < 0) { return -1; + } } else self->fd = -1; @@ -449,9 +458,14 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #endif } - self->blksize = DEFAULT_BUFFER_SIZE; + PyMem_Free(self->stat_atopen); + self->stat_atopen = PyMem_New(struct _Py_stat_struct, 1); + if (self->stat_atopen == NULL) { + PyErr_NoMemory(); + goto error; + } Py_BEGIN_ALLOW_THREADS - fstat_result = _Py_fstat_noraise(self->fd, &fdfstat); + fstat_result = _Py_fstat_noraise(self->fd, self->stat_atopen); Py_END_ALLOW_THREADS if (fstat_result < 0) { /* Tolerate fstat() errors other than EBADF. See Issue #25717, where @@ -466,22 +480,21 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #endif goto error; } + + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; } else { #if defined(S_ISDIR) && defined(EISDIR) /* On Unix, open will succeed for directories. In Python, there should be no file objects referring to directories, so we need a check. */ - if (S_ISDIR(fdfstat.st_mode)) { + if (S_ISDIR(self->stat_atopen->st_mode)) { errno = EISDIR; PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); goto error; } #endif /* defined(S_ISDIR) */ -#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE - if (fdfstat.st_blksize > 1) - self->blksize = fdfstat.st_blksize; -#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */ } #if defined(MS_WINDOWS) || defined(__CYGWIN__) @@ -513,6 +526,8 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, internal_close(self); _PyErr_ChainExceptions1(exc); } + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; done: #ifdef MS_WINDOWS @@ -523,32 +538,43 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, } static int -fileio_traverse(fileio *self, visitproc visit, void *arg) +fileio_traverse(PyObject *op, visitproc visit, void *arg) { + fileio *self = _PyFileIO_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } static int -fileio_clear(fileio *self) +fileio_clear(PyObject *op) { + fileio *self = _PyFileIO_CAST(op); Py_CLEAR(self->dict); return 0; } static void -fileio_dealloc(fileio *self) +fileio_dealloc(PyObject *op) { - PyTypeObject *tp = Py_TYPE(self); + fileio *self = _PyFileIO_CAST(op); self->finalizing = 1; - if (_PyIOBase_finalize((PyObject *) self) < 0) + if (_PyIOBase_finalize(op) < 0) { return; + } + _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - (void)fileio_clear(self); - tp->tp_free((PyObject *)self); + if (self->stat_atopen != NULL) { + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; + } + if (self->weakreflist != NULL) { + PyObject_ClearWeakRefs(op); + } + (void)fileio_clear(op); + + PyTypeObject *tp = Py_TYPE(op); + tp->tp_free(op); Py_DECREF(tp); } @@ -684,7 +710,7 @@ new_buffersize(fileio *self, size_t currentsize) giving us amortized linear-time behavior. For bigger sizes, use a less-than-double growth factor to avoid excessive allocation. */ assert(currentsize <= PY_SSIZE_T_MAX); - if (currentsize > 65536) + if (currentsize > LARGE_BUFFER_CUTOFF_SIZE) addend = currentsize >> 3; else addend = 256 + currentsize; @@ -707,43 +733,63 @@ static PyObject * _io_FileIO_readall_impl(fileio *self) /*[clinic end generated code: output=faa0292b213b4022 input=dbdc137f55602834]*/ { - struct _Py_stat_struct status; Py_off_t pos, end; PyObject *result; Py_ssize_t bytes_read = 0; Py_ssize_t n; size_t bufsize; - int fstat_result; - if (self->fd < 0) + if (self->fd < 0) { return err_closed(); + } - Py_BEGIN_ALLOW_THREADS - _Py_BEGIN_SUPPRESS_IPH + if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) { + end = (Py_off_t)self->stat_atopen->st_size; + } + else { + end = -1; + } + if (end <= 0) { + /* Use a default size and resize as needed. */ + bufsize = SMALLCHUNK; + } + else { + /* This is probably a real file. */ + if (end > _PY_READ_MAX - 1) { + bufsize = _PY_READ_MAX; + } + else { + /* In order to detect end of file, need a read() of at + least 1 byte which returns size 0. Oversize the buffer + by 1 byte so the I/O can be completed with two read() + calls (one for all data, one for EOF) without needing + to resize the buffer. */ + bufsize = (size_t)end + 1; + } + + /* While a lot of code does open().read() to get the whole contents + of a file it is possible a caller seeks/reads a ways into the file + then calls readall() to get the rest, which would result in allocating + more than required. Guard against that for larger files where we expect + the I/O time to dominate anyways while keeping small files fast. */ + if (bufsize > LARGE_BUFFER_CUTOFF_SIZE) { + Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH #ifdef MS_WINDOWS - pos = _lseeki64(self->fd, 0L, SEEK_CUR); + pos = _lseeki64(self->fd, 0L, SEEK_CUR); #else - pos = lseek(self->fd, 0L, SEEK_CUR); + pos = lseek(self->fd, 0L, SEEK_CUR); #endif - _Py_END_SUPPRESS_IPH - fstat_result = _Py_fstat_noraise(self->fd, &status); - Py_END_ALLOW_THREADS + _Py_END_SUPPRESS_IPH + Py_END_ALLOW_THREADS - if (fstat_result == 0) - end = status.st_size; - else - end = (Py_off_t)-1; - - if (end > 0 && end >= pos && pos >= 0 && end - pos < PY_SSIZE_T_MAX) { - /* This is probably a real file, so we try to allocate a - buffer one byte larger than the rest of the file. If the - calculation is right then we should get EOF without having - to enlarge the buffer. */ - bufsize = (size_t)(end - pos + 1); - } else { - bufsize = SMALLCHUNK; + if (end >= pos && pos >= 0 && (end - pos) < (_PY_READ_MAX - 1)) { + bufsize = (size_t)(end - pos) + 1; + } + } } + result = PyBytes_FromStringAndSize(NULL, bufsize); if (result == NULL) return NULL; @@ -783,7 +829,6 @@ _io_FileIO_readall_impl(fileio *self) return NULL; } bytes_read += n; - pos += n; } if (PyBytes_GET_SIZE(result) > bytes_read) { @@ -1074,6 +1119,14 @@ _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj) return NULL; } + /* Since the file was truncated, its size at open is no longer accurate + as an estimate. Clear out the stat result, and rely on dynamic resize + code if a readall is requested. */ + if (self->stat_atopen != NULL) { + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; + } + return posobj; } #endif /* HAVE_FTRUNCATE */ @@ -1104,18 +1157,20 @@ mode_string(fileio *self) } static PyObject * -fileio_repr(fileio *self) +fileio_repr(PyObject *op) { - PyObject *nameobj, *res; - const char *type_name = Py_TYPE((PyObject *) self)->tp_name; + fileio *self = _PyFileIO_CAST(op); + const char *type_name = Py_TYPE(self)->tp_name; if (self->fd < 0) { return PyUnicode_FromFormat("<%.100s [closed]>", type_name); } + PyObject *nameobj; if (PyObject_GetOptionalAttr((PyObject *) self, &_Py_ID(name), &nameobj) < 0) { return NULL; } + PyObject *res; if (nameobj == NULL) { res = PyUnicode_FromFormat( "<%.100s fd=%d mode='%s' closefd=%s>", @@ -1161,6 +1216,24 @@ _io_FileIO_isatty_impl(fileio *self) return PyBool_FromLong(res); } +/* Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). */ +static PyObject * +_io_FileIO_isatty_open_only(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + fileio *self = _PyFileIO_CAST(op); + if (self->stat_atopen != NULL && !S_ISCHR(self->stat_atopen->st_mode)) { + Py_RETURN_FALSE; + } + return _io_FileIO_isatty_impl(self); +} + #include "clinic/fileio.c.h" static PyMethodDef fileio_methods[] = { @@ -1177,42 +1250,58 @@ static PyMethodDef fileio_methods[] = { _IO_FILEIO_WRITABLE_METHODDEF _IO_FILEIO_FILENO_METHODDEF _IO_FILEIO_ISATTY_METHODDEF - {"_dealloc_warn", (PyCFunction)fileio_dealloc_warn, METH_O, NULL}, - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"_isatty_open_only", _io_FileIO_isatty_open_only, METH_NOARGS}, + {"_dealloc_warn", fileio_dealloc_warn, METH_O, NULL}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} /* sentinel */ }; /* 'closed' and 'mode' are attributes for backwards compatibility reasons. */ static PyObject * -get_closed(fileio *self, void *closure) +fileio_get_closed(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); return PyBool_FromLong((long)(self->fd < 0)); } static PyObject * -get_closefd(fileio *self, void *closure) +fileio_get_closefd(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); return PyBool_FromLong((long)(self->closefd)); } static PyObject * -get_mode(fileio *self, void *closure) +fileio_get_mode(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); return PyUnicode_FromString(mode_string(self)); } +static PyObject * +fileio_get_blksize(PyObject *op, void *closure) +{ + fileio *self = _PyFileIO_CAST(op); +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE + if (self->stat_atopen != NULL && self->stat_atopen->st_blksize > 1) { + return PyLong_FromLong(self->stat_atopen->st_blksize); + } +#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */ + return PyLong_FromLong(DEFAULT_BUFFER_SIZE); +} + static PyGetSetDef fileio_getsetlist[] = { - {"closed", (getter)get_closed, NULL, "True if the file is closed"}, - {"closefd", (getter)get_closefd, NULL, + {"closed", fileio_get_closed, NULL, "True if the file is closed"}, + {"closefd", fileio_get_closefd, NULL, "True if the file descriptor will be closed by close()."}, - {"mode", (getter)get_mode, NULL, "String giving the file mode"}, + {"mode", fileio_get_mode, NULL, "String giving the file mode"}, + {"_blksize", fileio_get_blksize, NULL, "Stat st_blksize if available"}, {NULL}, }; static PyMemberDef fileio_members[] = { - {"_blksize", Py_T_UINT, offsetof(fileio, blksize), 0}, {"_finalizing", Py_T_BOOL, offsetof(fileio, finalizing), 0}, {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(fileio, weakreflist), Py_READONLY}, {"__dictoffset__", Py_T_PYSSIZET, offsetof(fileio, dict), Py_READONLY}, diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 184e0b7d1aa7f1..419e5516b5c11e 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -999,7 +999,7 @@ _io__RawIOBase_readall_impl(PyObject *self) return NULL; } } - result = _PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks); + result = PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks); Py_DECREF(chunks); return result; } diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 06bc2679e8e227..f558613dc6233c 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -196,7 +196,7 @@ write_str(stringio *self, PyObject *obj) } if (self->writenl) { PyObject *translated = PyUnicode_Replace( - decoded, &_Py_STR(newline), self->writenl, -1); + decoded, _Py_LATIN1_CHR('\n'), self->writenl, -1); Py_SETREF(decoded, translated); } if (decoded == NULL) @@ -353,7 +353,7 @@ _stringio_readline(stringio *self, Py_ssize_t limit) /* In case of overseek, return the empty string */ if (self->pos >= self->string_size) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); start = self->buf + self->pos; if (limit < 0 || limit > self->string_size - self->pos) diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index d794af8de2b8f0..0d851ee211511c 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -559,7 +559,7 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) Py_DECREF(state); } else { - buffer = PyBytes_FromString(""); + buffer = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); flag = 0; } flag <<= 1; @@ -1719,16 +1719,26 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) bytes_len = PyBytes_GET_SIZE(b); } - if (self->pending_bytes == NULL) { - self->pending_bytes_count = 0; - self->pending_bytes = b; - } - else if (self->pending_bytes_count + bytes_len > self->chunk_size) { - // Prevent to concatenate more than chunk_size data. - if (_textiowrapper_writeflush(self) < 0) { - Py_DECREF(b); - return NULL; + // We should avoid concatenating huge data. + // Flush the buffer before adding b to the buffer if b is not small. + // https://github.com/python/cpython/issues/87426 + if (bytes_len >= self->chunk_size) { + // _textiowrapper_writeflush() calls buffer.write(). + // self->pending_bytes can be appended during buffer->write() + // or other thread. + // We need to loop until buffer becomes empty. + // https://github.com/python/cpython/issues/118138 + // https://github.com/python/cpython/issues/119506 + while (self->pending_bytes != NULL) { + if (_textiowrapper_writeflush(self) < 0) { + Py_DECREF(b); + return NULL; + } } + } + + if (self->pending_bytes == NULL) { + assert(self->pending_bytes_count == 0); self->pending_bytes = b; } else if (!PyList_CheckExact(self->pending_bytes)) { @@ -1737,6 +1747,9 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) Py_DECREF(b); return NULL; } + // Since Python 3.12, allocating GC object won't trigger GC and release + // GIL. See https://github.com/python/cpython/issues/97922 + assert(!PyList_CheckExact(self->pending_bytes)); PyList_SET_ITEM(list, 0, self->pending_bytes); PyList_SET_ITEM(list, 1, b); self->pending_bytes = list; @@ -1793,7 +1806,7 @@ textiowrapper_get_decoded_chars(textio *self, Py_ssize_t n) Py_ssize_t avail; if (self->decoded_chars == NULL) - return PyUnicode_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* decoded_chars is guaranteed to be "ready". */ avail = (PyUnicode_GET_LENGTH(self->decoded_chars) @@ -2393,7 +2406,7 @@ textiowrapper_parse_cookie(cookie_type *cookie, PyObject *cookieObj) return -1; if (_PyLong_AsByteArray(cookieLong, buffer, sizeof(buffer), - PY_LITTLE_ENDIAN, 0) < 0) { + PY_LITTLE_ENDIAN, 0, 1) < 0) { Py_DECREF(cookieLong); return -1; } @@ -3337,8 +3350,8 @@ static PyMethodDef textiowrapper_methods[] = { _IO_TEXTIOWRAPPER_TELL_METHODDEF _IO_TEXTIOWRAPPER_TRUNCATE_METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 54e15555417287..d7cb5abfdc0abd 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -298,6 +298,13 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, self->fd = -1; } + if (PyBool_Check(nameobj)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { @@ -1121,6 +1128,7 @@ static PyMethodDef winconsoleio_methods[] = { _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF + {"_isatty_open_only", (PyCFunction)_io__WindowsConsoleIO_isatty, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_json.c b/Modules/_json.c index c55299899e77fe..ce0093ab431d05 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -11,6 +11,7 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_runtime.h" // _PyRuntime +#include "pycore_pyerrors.h" // _PyErr_FormatNote #include "pycore_global_strings.h" // _Py_ID() #include // bool @@ -85,11 +86,11 @@ encoder_dealloc(PyObject *self); static int encoder_clear(PyEncoderObject *self); static int -encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *seq, Py_ssize_t indent_level); +encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *seq, PyObject *newline_indent); static int -encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *obj, Py_ssize_t indent_level); +encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj, PyObject *newline_indent); static int -encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *dct, Py_ssize_t indent_level); +encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *dct, PyObject *newline_indent); static PyObject * _encoded_const(PyObject *obj); static void @@ -1251,33 +1252,55 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)s; } +static PyObject * +_create_newline_indent(PyObject *indent, Py_ssize_t indent_level) +{ + PyObject *newline_indent = PyUnicode_FromOrdinal('\n'); + if (newline_indent != NULL && indent_level) { + PyUnicode_AppendAndDel(&newline_indent, + PySequence_Repeat(indent, indent_level)); + } + return newline_indent; +} + static PyObject * encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) { /* Python callable interface to encode_listencode_obj */ static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj, *result; + PyObject *obj; Py_ssize_t indent_level; - _PyUnicodeWriter writer; if (!PyArg_ParseTupleAndKeywords(args, kwds, "On:_iterencode", kwlist, - &obj, &indent_level)) + &obj, &indent_level)) return NULL; - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } - if (encoder_listencode_obj(self, &writer, obj, indent_level)) { - _PyUnicodeWriter_Dealloc(&writer); + PyObject *newline_indent = NULL; + if (self->indent != Py_None) { + newline_indent = _create_newline_indent(self->indent, indent_level); + if (newline_indent == NULL) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + if (encoder_listencode_obj(self, writer, obj, newline_indent)) { + PyUnicodeWriter_Discard(writer); + Py_XDECREF(newline_indent); return NULL; } + Py_XDECREF(newline_indent); - result = PyTuple_New(1); - if (result == NULL || - PyTuple_SetItem(result, 0, _PyUnicodeWriter_Finish(&writer)) < 0) { - Py_XDECREF(result); + PyObject *str = PyUnicodeWriter_Finish(writer); + if (str == NULL) { return NULL; } + PyObject *result = PyTuple_Pack(1, str); + Py_DECREF(str); return result; } @@ -1305,7 +1328,7 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) { /* Return the JSON representation of a PyFloat. */ double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { + if (!isfinite(i)) { if (!s->allow_nan) { PyErr_Format( PyExc_ValueError, @@ -1348,30 +1371,30 @@ encoder_encode_string(PyEncoderObject *s, PyObject *obj) } static int -_steal_accumulate(_PyUnicodeWriter *writer, PyObject *stolen) +_steal_accumulate(PyUnicodeWriter *writer, PyObject *stolen) { /* Append stolen and then decrement its reference count */ - int rval = _PyUnicodeWriter_WriteStr(writer, stolen); + int rval = PyUnicodeWriter_WriteStr(writer, stolen); Py_DECREF(stolen); return rval; } static int -encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *obj, Py_ssize_t indent_level) +encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, + PyObject *obj, PyObject *newline_indent) { /* Encode Python object obj to a JSON term */ PyObject *newobj; int rv; if (obj == Py_None) { - return _PyUnicodeWriter_WriteASCIIString(writer, "null", 4); + return PyUnicodeWriter_WriteUTF8(writer, "null", 4); } else if (obj == Py_True) { - return _PyUnicodeWriter_WriteASCIIString(writer, "true", 4); + return PyUnicodeWriter_WriteUTF8(writer, "true", 4); } else if (obj == Py_False) { - return _PyUnicodeWriter_WriteASCIIString(writer, "false", 5); + return PyUnicodeWriter_WriteUTF8(writer, "false", 5); } else if (PyUnicode_Check(obj)) { PyObject *encoded = encoder_encode_string(s, obj); @@ -1380,6 +1403,10 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, return _steal_accumulate(writer, encoded); } else if (PyLong_Check(obj)) { + if (PyLong_CheckExact(obj)) { + // Fast-path for exact integers + return PyUnicodeWriter_WriteRepr(writer, obj); + } PyObject *encoded = PyLong_Type.tp_repr(obj); if (encoded == NULL) return -1; @@ -1394,14 +1421,14 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, else if (PyList_Check(obj) || PyTuple_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_list(s, writer, obj, indent_level); + rv = encoder_listencode_list(s, writer, obj, newline_indent); _Py_LeaveRecursiveCall(); return rv; } else if (PyDict_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_dict(s, writer, obj, indent_level); + rv = encoder_listencode_dict(s, writer, obj, newline_indent); _Py_LeaveRecursiveCall(); return rv; } @@ -1435,11 +1462,12 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_XDECREF(ident); return -1; } - rv = encoder_listencode_obj(s, writer, newobj, indent_level); + rv = encoder_listencode_obj(s, writer, newobj, newline_indent); _Py_LeaveRecursiveCall(); Py_DECREF(newobj); if (rv) { + _PyErr_FormatNote("when serializing %T object", obj); Py_XDECREF(ident); return -1; } @@ -1455,8 +1483,10 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, } static int -encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *first, - PyObject *key, PyObject *value, Py_ssize_t indent_level) +encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *first, + PyObject *dct, PyObject *key, PyObject *value, + PyObject *newline_indent, + PyObject *item_separator) { PyObject *keystr = NULL; PyObject *encoded; @@ -1493,7 +1523,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir *first = false; } else { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator) < 0) { + if (PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { Py_DECREF(keystr); return -1; } @@ -1508,27 +1538,32 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir if (_steal_accumulate(writer, encoded) < 0) { return -1; } - if (_PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { + if (PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { return -1; } - if (encoder_listencode_obj(s, writer, value, indent_level) < 0) { + if (encoder_listencode_obj(s, writer, value, newline_indent) < 0) { + _PyErr_FormatNote("when serializing %T item %R", dct, key); return -1; } return 0; } static int -encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *dct, Py_ssize_t indent_level) +encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, + PyObject *dct, PyObject *newline_indent) { /* Encode Python dict dct a JSON term */ PyObject *ident = NULL; PyObject *items = NULL; PyObject *key, *value; bool first = true; + PyObject *new_newline_indent = NULL; + PyObject *separator_indent = NULL; - if (PyDict_GET_SIZE(dct) == 0) /* Fast path */ - return _PyUnicodeWriter_WriteASCIIString(writer, "{}", 2); + if (PyDict_GET_SIZE(dct) == 0) { + /* Fast path */ + return PyUnicodeWriter_WriteUTF8(writer, "{}", 2); + } if (s->markers != Py_None) { int has_key; @@ -1546,17 +1581,25 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } } - if (_PyUnicodeWriter_WriteChar(writer, '{')) + if (PyUnicodeWriter_WriteChar(writer, '{')) { goto bail; + } + PyObject *current_item_separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (new_newline_indent == NULL) { + goto bail; + } + separator_indent = PyUnicode_Concat(current_item_separator, new_newline_indent); + if (separator_indent == NULL) { + goto bail; + } + // update item separator with a borrowed reference + current_item_separator = separator_indent; + if (PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + goto bail; + } } if (s->sort_keys || !PyDict_CheckExact(dct)) { @@ -1574,7 +1617,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, key = PyTuple_GET_ITEM(item, 0); value = PyTuple_GET_ITEM(item, 1); - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, dct, key, value, + new_newline_indent, + current_item_separator) < 0) goto bail; } Py_CLEAR(items); @@ -1582,7 +1627,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } else { Py_ssize_t pos = 0; while (PyDict_Next(dct, &pos, &key, &value)) { - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, dct, key, value, + new_newline_indent, + current_item_separator) < 0) goto bail; } } @@ -1592,29 +1639,37 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, goto bail; Py_CLEAR(ident); } - /* TODO DOES NOT RUN; dead code if (s->indent != Py_None) { - indent_level -= 1; + Py_CLEAR(new_newline_indent); + Py_CLEAR(separator_indent); + + if (PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ - if (_PyUnicodeWriter_WriteChar(writer, '}')) + if (PyUnicodeWriter_WriteChar(writer, '}')) { goto bail; + } return 0; bail: Py_XDECREF(items); Py_XDECREF(ident); + Py_XDECREF(separator_indent); + Py_XDECREF(new_newline_indent); return -1; } static int -encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *seq, Py_ssize_t indent_level) +encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, + PyObject *seq, PyObject *newline_indent) { PyObject *ident = NULL; PyObject *s_fast = NULL; Py_ssize_t i; + PyObject *new_newline_indent = NULL; + PyObject *separator_indent = NULL; ident = NULL; s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence"); @@ -1622,7 +1677,7 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, return -1; if (PySequence_Fast_GET_SIZE(s_fast) == 0) { Py_DECREF(s_fast); - return _PyUnicodeWriter_WriteASCIIString(writer, "[]", 2); + return PyUnicodeWriter_WriteUTF8(writer, "[]", 2); } if (s->markers != Py_None) { @@ -1641,25 +1696,37 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, } } - if (_PyUnicodeWriter_WriteChar(writer, '[')) + if (PyUnicodeWriter_WriteChar(writer, '[')) { goto bail; + } + + PyObject *separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (new_newline_indent == NULL) { + goto bail; + } + + if (PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + goto bail; + } + + separator_indent = PyUnicode_Concat(separator, new_newline_indent); + if (separator_indent == NULL) { + goto bail; + } + separator = separator_indent; // assign separator with borrowed reference } for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); if (i) { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator)) + if (PyUnicodeWriter_WriteStr(writer, separator) < 0) goto bail; } - if (encoder_listencode_obj(s, writer, obj, indent_level)) + if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) { + _PyErr_FormatNote("when serializing %T item %zd", seq, i); goto bail; + } } if (ident != NULL) { if (PyDict_DelItem(s->markers, ident)) @@ -1667,20 +1734,25 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_CLEAR(ident); } - /* TODO: DOES NOT RUN if (s->indent != Py_None) { - indent_level -= 1; + Py_CLEAR(new_newline_indent); + Py_CLEAR(separator_indent); + if (PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ - if (_PyUnicodeWriter_WriteChar(writer, ']')) + if (PyUnicodeWriter_WriteChar(writer, ']')) { goto bail; + } Py_DECREF(s_fast); return 0; bail: Py_XDECREF(ident); Py_DECREF(s_fast); + Py_XDECREF(separator_indent); + Py_XDECREF(new_newline_indent); return -1; } @@ -1721,7 +1793,7 @@ encoder_clear(PyEncoderObject *self) return 0; } -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); +PyDoc_STRVAR(encoder_doc, "Encoder(markers, default, encoder, indent, key_separator, item_separator, sort_keys, skipkeys, allow_nan)"); static PyType_Slot PyEncoderType_slots[] = { {Py_tp_doc, (void *)encoder_doc}, @@ -1780,6 +1852,7 @@ _json_exec(PyObject *module) static PyModuleDef_Slot _json_slots[] = { {Py_mod_exec, _json_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index fe8e4c5e30035b..2a789ea74d27da 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -52,7 +52,7 @@ module _locale [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=ed98569b726feada]*/ -/* support functions for formatting floating point numbers */ +/* support functions for formatting floating-point numbers */ /* the grouping is terminated by either 0 or CHAR_MAX */ static PyObject* @@ -144,6 +144,17 @@ locale_is_ascii(const char *str) return (strlen(str) == 1 && ((unsigned char)str[0]) <= 127); } +static int +is_all_ascii(const char *str) +{ + for (; *str; str++) { + if ((unsigned char)*str > 127) { + return 0; + } + } + return 1; +} + static int locale_decode_monetary(PyObject *dict, struct lconv *lc) { @@ -478,113 +489,153 @@ _locale__getdefaultlocale_impl(PyObject *module) #endif #ifdef HAVE_LANGINFO_H -#define LANGINFO(X) {#X, X} +#define LANGINFO(X, Y) {#X, X, Y} static struct langinfo_constant{ - char* name; + const char *name; int value; + int category; } langinfo_constants[] = { /* These constants should exist on any langinfo implementation */ - LANGINFO(DAY_1), - LANGINFO(DAY_2), - LANGINFO(DAY_3), - LANGINFO(DAY_4), - LANGINFO(DAY_5), - LANGINFO(DAY_6), - LANGINFO(DAY_7), - - LANGINFO(ABDAY_1), - LANGINFO(ABDAY_2), - LANGINFO(ABDAY_3), - LANGINFO(ABDAY_4), - LANGINFO(ABDAY_5), - LANGINFO(ABDAY_6), - LANGINFO(ABDAY_7), - - LANGINFO(MON_1), - LANGINFO(MON_2), - LANGINFO(MON_3), - LANGINFO(MON_4), - LANGINFO(MON_5), - LANGINFO(MON_6), - LANGINFO(MON_7), - LANGINFO(MON_8), - LANGINFO(MON_9), - LANGINFO(MON_10), - LANGINFO(MON_11), - LANGINFO(MON_12), - - LANGINFO(ABMON_1), - LANGINFO(ABMON_2), - LANGINFO(ABMON_3), - LANGINFO(ABMON_4), - LANGINFO(ABMON_5), - LANGINFO(ABMON_6), - LANGINFO(ABMON_7), - LANGINFO(ABMON_8), - LANGINFO(ABMON_9), - LANGINFO(ABMON_10), - LANGINFO(ABMON_11), - LANGINFO(ABMON_12), + LANGINFO(DAY_1, LC_TIME), + LANGINFO(DAY_2, LC_TIME), + LANGINFO(DAY_3, LC_TIME), + LANGINFO(DAY_4, LC_TIME), + LANGINFO(DAY_5, LC_TIME), + LANGINFO(DAY_6, LC_TIME), + LANGINFO(DAY_7, LC_TIME), + + LANGINFO(ABDAY_1, LC_TIME), + LANGINFO(ABDAY_2, LC_TIME), + LANGINFO(ABDAY_3, LC_TIME), + LANGINFO(ABDAY_4, LC_TIME), + LANGINFO(ABDAY_5, LC_TIME), + LANGINFO(ABDAY_6, LC_TIME), + LANGINFO(ABDAY_7, LC_TIME), + + LANGINFO(MON_1, LC_TIME), + LANGINFO(MON_2, LC_TIME), + LANGINFO(MON_3, LC_TIME), + LANGINFO(MON_4, LC_TIME), + LANGINFO(MON_5, LC_TIME), + LANGINFO(MON_6, LC_TIME), + LANGINFO(MON_7, LC_TIME), + LANGINFO(MON_8, LC_TIME), + LANGINFO(MON_9, LC_TIME), + LANGINFO(MON_10, LC_TIME), + LANGINFO(MON_11, LC_TIME), + LANGINFO(MON_12, LC_TIME), + + LANGINFO(ABMON_1, LC_TIME), + LANGINFO(ABMON_2, LC_TIME), + LANGINFO(ABMON_3, LC_TIME), + LANGINFO(ABMON_4, LC_TIME), + LANGINFO(ABMON_5, LC_TIME), + LANGINFO(ABMON_6, LC_TIME), + LANGINFO(ABMON_7, LC_TIME), + LANGINFO(ABMON_8, LC_TIME), + LANGINFO(ABMON_9, LC_TIME), + LANGINFO(ABMON_10, LC_TIME), + LANGINFO(ABMON_11, LC_TIME), + LANGINFO(ABMON_12, LC_TIME), #ifdef RADIXCHAR /* The following are not available with glibc 2.0 */ - LANGINFO(RADIXCHAR), - LANGINFO(THOUSEP), + LANGINFO(RADIXCHAR, LC_NUMERIC), + LANGINFO(THOUSEP, LC_NUMERIC), /* YESSTR and NOSTR are deprecated in glibc, since they are a special case of message translation, which should be rather done using gettext. So we don't expose it to Python in the first place. - LANGINFO(YESSTR), - LANGINFO(NOSTR), + LANGINFO(YESSTR, LC_MESSAGES), + LANGINFO(NOSTR, LC_MESSAGES), */ - LANGINFO(CRNCYSTR), + LANGINFO(CRNCYSTR, LC_MONETARY), #endif - LANGINFO(D_T_FMT), - LANGINFO(D_FMT), - LANGINFO(T_FMT), - LANGINFO(AM_STR), - LANGINFO(PM_STR), + LANGINFO(D_T_FMT, LC_TIME), + LANGINFO(D_FMT, LC_TIME), + LANGINFO(T_FMT, LC_TIME), + LANGINFO(AM_STR, LC_TIME), + LANGINFO(PM_STR, LC_TIME), /* The following constants are available only with XPG4, but... OpenBSD doesn't have CODESET but has T_FMT_AMPM, and doesn't have a few of the others. Solution: ifdef-test them all. */ #ifdef CODESET - LANGINFO(CODESET), + LANGINFO(CODESET, LC_CTYPE), #endif #ifdef T_FMT_AMPM - LANGINFO(T_FMT_AMPM), + LANGINFO(T_FMT_AMPM, LC_TIME), #endif #ifdef ERA - LANGINFO(ERA), + LANGINFO(ERA, LC_TIME), #endif #ifdef ERA_D_FMT - LANGINFO(ERA_D_FMT), + LANGINFO(ERA_D_FMT, LC_TIME), #endif #ifdef ERA_D_T_FMT - LANGINFO(ERA_D_T_FMT), + LANGINFO(ERA_D_T_FMT, LC_TIME), #endif #ifdef ERA_T_FMT - LANGINFO(ERA_T_FMT), + LANGINFO(ERA_T_FMT, LC_TIME), #endif #ifdef ALT_DIGITS - LANGINFO(ALT_DIGITS), + LANGINFO(ALT_DIGITS, LC_TIME), #endif #ifdef YESEXPR - LANGINFO(YESEXPR), + LANGINFO(YESEXPR, LC_MESSAGES), #endif #ifdef NOEXPR - LANGINFO(NOEXPR), + LANGINFO(NOEXPR, LC_MESSAGES), #endif #ifdef _DATE_FMT /* This is not available in all glibc versions that have CODESET. */ - LANGINFO(_DATE_FMT), + LANGINFO(_DATE_FMT, LC_TIME), #endif - {0, 0} + {0, 0, 0} }; +/* Temporary make the LC_CTYPE locale to be the same as + * the locale of the specified category. */ +static int +change_locale(int category, char **oldloc) +{ + /* Keep a copy of the LC_CTYPE locale */ + *oldloc = setlocale(LC_CTYPE, NULL); + if (!*oldloc) { + PyErr_SetString(PyExc_RuntimeError, "faild to get LC_CTYPE locale"); + return -1; + } + *oldloc = _PyMem_Strdup(*oldloc); + if (!*oldloc) { + PyErr_NoMemory(); + return -1; + } + + /* Set a new locale if it is different. */ + char *loc = setlocale(category, NULL); + if (loc == NULL || strcmp(loc, *oldloc) == 0) { + PyMem_Free(*oldloc); + *oldloc = NULL; + return 0; + } + + setlocale(LC_CTYPE, loc); + return 1; +} + +/* Restore the old LC_CTYPE locale. */ +static void +restore_locale(char *oldloc) +{ + if (oldloc != NULL) { + setlocale(LC_CTYPE, oldloc); + PyMem_Free(oldloc); + } +} + /*[clinic input] _locale.nl_langinfo @@ -602,14 +653,58 @@ _locale_nl_langinfo_impl(PyObject *module, int item) /* Check whether this is a supported constant. GNU libc sometimes returns numeric values in the char* return value, which would crash PyUnicode_FromString. */ - for (i = 0; langinfo_constants[i].name; i++) + for (i = 0; langinfo_constants[i].name; i++) { if (langinfo_constants[i].value == item) { /* Check NULL as a workaround for GNU libc's returning NULL instead of an empty string for nl_langinfo(ERA). */ const char *result = nl_langinfo(item); result = result != NULL ? result : ""; - return PyUnicode_DecodeLocale(result, NULL); + char *oldloc = NULL; + if (langinfo_constants[i].category != LC_CTYPE + && !is_all_ascii(result) + && change_locale(langinfo_constants[i].category, &oldloc) < 0) + { + return NULL; + } + PyObject *pyresult; +#ifdef __GLIBC__ +#ifdef ALT_DIGITS + if (item == ALT_DIGITS && *result) { + /* According to the POSIX specification the result must be + * a sequence of up to 100 semicolon-separated strings. + * But in Glibc they are NUL-separated. */ + Py_ssize_t i = 0; + int count = 0; + for (; count < 100 && result[i]; count++) { + i += strlen(result + i) + 1; + } + char *buf = PyMem_Malloc(i); + if (buf == NULL) { + PyErr_NoMemory(); + pyresult = NULL; + } + else { + memcpy(buf, result, i); + /* Replace all NULs with semicolons. */ + i = 0; + while (--count) { + i += strlen(buf + i); + buf[i++] = ';'; + } + pyresult = PyUnicode_DecodeLocale(buf, NULL); + PyMem_Free(buf); + } + } + else +#endif +#endif + { + pyresult = PyUnicode_DecodeLocale(result, NULL); + } + restore_locale(oldloc); + return pyresult; } + } PyErr_SetString(PyExc_ValueError, "unsupported langinfo constant"); return NULL; } @@ -860,6 +955,7 @@ _locale_exec(PyObject *module) static struct PyModuleDef_Slot _locale_slots[] = { {Py_mod_exec, _locale_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -880,9 +976,9 @@ locale_clear(PyObject *module) } static void -locale_free(PyObject *module) +locale_free(void *module) { - locale_clear(module); + locale_clear((PyObject*)module); } static struct PyModuleDef _localemodule = { @@ -894,7 +990,7 @@ static struct PyModuleDef _localemodule = { _locale_slots, locale_traverse, locale_clear, - (freefunc)locale_free, + locale_free, }; PyMODINIT_FUNC diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 8f09204097529f..06958a078509d9 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -6,6 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetProfile() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_time.h" // _PyTime_FromLong() #include "rotatingtree.h" @@ -17,8 +18,8 @@ struct _ProfilerEntry; /* represents a function called from another function */ typedef struct _ProfilerSubEntry { rotating_node_t header; - _PyTime_t tt; - _PyTime_t it; + PyTime_t tt; + PyTime_t it; long callcount; long recursivecallcount; long recursionLevel; @@ -28,8 +29,8 @@ typedef struct _ProfilerSubEntry { typedef struct _ProfilerEntry { rotating_node_t header; PyObject *userObj; /* PyCodeObject, or a descriptive str for builtins */ - _PyTime_t tt; /* total time in this entry */ - _PyTime_t it; /* inline time in this entry (not in subcalls) */ + PyTime_t tt; /* total time in this entry */ + PyTime_t it; /* inline time in this entry (not in subcalls) */ long callcount; /* how many times this was called */ long recursivecallcount; /* how many times called recursively */ long recursionLevel; @@ -37,8 +38,8 @@ typedef struct _ProfilerEntry { } ProfilerEntry; typedef struct _ProfilerContext { - _PyTime_t t0; - _PyTime_t subt; + PyTime_t t0; + PyTime_t subt; struct _ProfilerContext *previous; ProfilerEntry *ctxEntry; } ProfilerContext; @@ -58,6 +59,7 @@ typedef struct { #define POF_ENABLED 0x001 #define POF_SUBCALLS 0x002 #define POF_BUILTINS 0x004 +#define POF_EXT_TIMER 0x008 #define POF_NOMEMORY 0x100 /*[clinic input] @@ -84,24 +86,31 @@ _lsprof_get_state(PyObject *module) /*** External Timers ***/ -static _PyTime_t CallExternalTimer(ProfilerObject *pObj) +static PyTime_t CallExternalTimer(ProfilerObject *pObj) { - PyObject *o = _PyObject_CallNoArgs(pObj->externalTimer); + PyObject *o = NULL; + + // External timer can do arbitrary things so we need a flag to prevent + // horrible things to happen + pObj->flags |= POF_EXT_TIMER; + o = _PyObject_CallNoArgs(pObj->externalTimer); + pObj->flags &= ~POF_EXT_TIMER; + if (o == NULL) { PyErr_WriteUnraisable(pObj->externalTimer); return 0; } - _PyTime_t result; + PyTime_t result; int err; if (pObj->externalTimerUnit > 0.0) { /* interpret the result as an integer that will be scaled in profiler_getstats() */ - err = _PyTime_FromNanosecondsObject(&result, o); + err = _PyTime_FromLong(&result, o); } else { /* interpret the result as a double measured in seconds. - As the profiler works with _PyTime_t internally + As the profiler works with PyTime_t internally we convert it to a large integer */ err = _PyTime_FromSecondsObject(&result, o, _PyTime_ROUND_FLOOR); } @@ -113,14 +122,16 @@ static _PyTime_t CallExternalTimer(ProfilerObject *pObj) return result; } -static inline _PyTime_t +static inline PyTime_t call_timer(ProfilerObject *pObj) { if (pObj->externalTimer != NULL) { return CallExternalTimer(pObj); } else { - return _PyTime_GetPerfCounter(); + PyTime_t t; + (void)PyTime_PerfCounterRaw(&t); + return t; } } @@ -174,8 +185,7 @@ normalizeUserObj(PyObject *obj) PyObject *modname = fn->m_module; if (name != NULL) { - PyObject *mo = _PyType_Lookup(Py_TYPE(self), name); - Py_XINCREF(mo); + PyObject *mo = _PyType_LookupRef(Py_TYPE(self), name); Py_DECREF(name); if (mo != NULL) { PyObject *res = PyObject_Repr(mo); @@ -311,8 +321,8 @@ initContext(ProfilerObject *pObj, ProfilerContext *self, ProfilerEntry *entry) static void Stop(ProfilerObject *pObj, ProfilerContext *self, ProfilerEntry *entry) { - _PyTime_t tt = call_timer(pObj) - self->t0; - _PyTime_t it = tt - self->subt; + PyTime_t tt = call_timer(pObj) - self->t0; + PyTime_t it = tt - self->subt; if (self->previous) self->previous->subt += tt; pObj->currentProfilerContext = self->previous; @@ -557,7 +567,7 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls) return NULL; } if (!self->externalTimer || self->externalTimerUnit == 0.0) { - _PyTime_t onesec = _PyTime_FromSeconds(1); + PyTime_t onesec = _PyTime_FromSeconds(1); collect.factor = (double)1 / onesec; } else { @@ -598,6 +608,12 @@ setBuiltins(ProfilerObject *pObj, int nvalue) PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) { + if (size < 2) { + PyErr_Format(PyExc_TypeError, + "_pystart_callback expected 2 arguments, got %zd", + size); + return NULL; + } PyObject* code = args[0]; ptrace_enter_call((PyObject*)self, (void *)code, (PyObject *)code); @@ -606,6 +622,12 @@ PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize PyObject* pyreturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) { + if (size < 3) { + PyErr_Format(PyExc_TypeError, + "_pyreturn_callback expected 3 arguments, got %zd", + size); + return NULL; + } PyObject* code = args[0]; ptrace_leave_call((PyObject*)self, (void *)code); @@ -641,6 +663,12 @@ PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObje PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) { + if (size < 4) { + PyErr_Format(PyExc_TypeError, + "_ccall_callback expected 4 arguments, got %zd", + size); + return NULL; + } if (self->flags & POF_BUILTINS) { PyObject* callable = args[2]; PyObject* self_arg = args[3]; @@ -659,6 +687,12 @@ PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t PyObject* creturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) { + if (size < 4) { + PyErr_Format(PyExc_TypeError, + "_creturn_callback expected 4 arguments, got %zd", + size); + return NULL; + } if (self->flags & POF_BUILTINS) { PyObject* callable = args[2]; PyObject* self_arg = args[3]; @@ -775,6 +809,11 @@ Stop collecting profiling information.\n\ static PyObject* profiler_disable(ProfilerObject *self, PyObject* noarg) { + if (self->flags & POF_EXT_TIMER) { + PyErr_SetString(PyExc_RuntimeError, + "cannot disable profiler in external timer"); + return NULL; + } if (self->flags & POF_ENABLED) { PyObject* result = NULL; PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); @@ -828,6 +867,11 @@ Clear all profiling information collected so far.\n\ static PyObject* profiler_clear(ProfilerObject *pObj, PyObject* noarg) { + if (pObj->flags & POF_EXT_TIMER) { + PyErr_SetString(PyExc_RuntimeError, + "cannot clear profiler in external timer"); + return NULL; + } clearEntries(pObj); Py_RETURN_NONE; } @@ -836,6 +880,7 @@ static int profiler_traverse(ProfilerObject *op, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(op)); + Py_VISIT(op->externalTimer); return 0; } @@ -1004,9 +1049,8 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index f6bfbfa62687b8..97f3a8f03da9a8 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1604,6 +1604,7 @@ static PyMethodDef lzma_methods[] = { static PyModuleDef_Slot lzma_slots[] = { {Py_mod_exec, lzma_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 1b894ea4c67adc..a545ff4d80f067 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -45,7 +45,7 @@ _posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) #if defined(HAVE_SHM_UNLINK) PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, -"shm_unlink($module, /, path)\n" +"shm_unlink($module, path, /)\n" "--\n" "\n" "Remove a shared memory object (similar to unlink()).\n" @@ -55,21 +55,22 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ - {"shm_unlink", (PyCFunction)(void(*)(void))_posixshmem_shm_unlink, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, + {"shm_unlink", (PyCFunction)_posixshmem_shm_unlink, METH_O, _posixshmem_shm_unlink__doc__}, static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path); static PyObject * -_posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) +_posixshmem_shm_unlink(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - static char *_keywords[] = {"path", NULL}; PyObject *path; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "U:shm_unlink", _keywords, - &path)) + if (!PyUnicode_Check(arg)) { + PyErr_Format(PyExc_TypeError, "shm_unlink() argument must be str, not %T", arg); goto exit; + } + path = arg; return_value = _posixshmem_shm_unlink_impl(module, path); exit: @@ -85,4 +86,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=be0661dbed83ea23 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=74588a5abba6e36c input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index 7c855113113c20..512e5a016192fb 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() #if defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) @@ -75,7 +76,9 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ } timeout_obj = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock_acquire_impl(self, blocking, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -100,7 +103,13 @@ _multiprocessing_SemLock_release_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock_release(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock_release_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_release_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) */ @@ -172,7 +181,9 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ } timeout_obj = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock_acquire_impl(self, blocking, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -197,7 +208,13 @@ _multiprocessing_SemLock_release_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock_release(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock_release_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_release_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) && !defined(MS_WINDOWS) */ @@ -340,7 +357,13 @@ _multiprocessing_SemLock__count_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock__count(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__count_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock__count_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -450,7 +473,13 @@ _multiprocessing_SemLock___enter___impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock___enter__(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock___enter___impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock___enter___impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -495,7 +524,9 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py } exc_tb = args[2]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock___exit___impl(self, exc_type, exc_value, exc_tb); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -542,4 +573,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=d57992037e6770b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dea36482d23a355f input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 2e6d8eb68c0243..cee8cf7b9a83c0 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -181,7 +181,7 @@ static PyMethodDef module_methods[] = { _MULTIPROCESSING_RECV_METHODDEF _MULTIPROCESSING_SEND_METHODDEF #endif -#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) +#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) _MULTIPROCESSING_SEM_UNLINK_METHODDEF #endif {NULL} @@ -277,6 +277,7 @@ multiprocessing_exec(PyObject *module) static PyModuleDef_Slot multiprocessing_slots[] = { {Py_mod_exec, multiprocessing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 425ce10075c156..ab45e4136c7d46 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -2,15 +2,15 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include +#include // strlen() #include // EINTR #ifdef HAVE_SYS_MMAN_H # include // shm_open(), shm_unlink() @@ -48,10 +48,15 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, { int fd; int async_err = 0; - const char *name = PyUnicode_AsUTF8AndSize(path, NULL); + Py_ssize_t name_size; + const char *name = PyUnicode_AsUTF8AndSize(path, &name_size); if (name == NULL) { return -1; } + if (strlen(name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return -1; + } do { Py_BEGIN_ALLOW_THREADS fd = shm_open(name, flags, mode); @@ -72,6 +77,7 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, /*[clinic input] _posixshmem.shm_unlink path: unicode + / Remove a shared memory object (similar to unlink()). @@ -83,14 +89,19 @@ region. static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=42f8b23d134b9ff5 input=8dc0f87143e3b300]*/ +/*[clinic end generated code: output=42f8b23d134b9ff5 input=298369d013dcad63]*/ { int rv; int async_err = 0; - const char *name = PyUnicode_AsUTF8AndSize(path, NULL); + Py_ssize_t name_size; + const char *name = PyUnicode_AsUTF8AndSize(path, &name_size); if (name == NULL) { return NULL; } + if (strlen(name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } do { Py_BEGIN_ALLOW_THREADS rv = shm_unlink(name); @@ -118,6 +129,7 @@ static PyMethodDef module_methods[ ] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index f8f2afda28d06d..4de4ee6c78fbd1 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -81,6 +81,7 @@ _GetSemaphoreValue(HANDLE handle, long *value) } /*[clinic input] +@critical_section _multiprocessing.SemLock.acquire block as blocking: bool = True @@ -92,7 +93,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=079ca779975f3ad6]*/ { double timeout; DWORD res, full_msecs, nhandles; @@ -172,6 +173,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, } /*[clinic input] +@critical_section _multiprocessing.SemLock.release Release the semaphore/lock. @@ -179,7 +181,7 @@ Release the semaphore/lock. static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self) -/*[clinic end generated code: output=b22f53ba96b0d1db input=ba7e63a961885d3d]*/ +/*[clinic end generated code: output=b22f53ba96b0d1db input=9bd62d3645e7a531]*/ { if (self->kind == RECURSIVE_MUTEX) { if (!ISMINE(self)) { @@ -297,6 +299,7 @@ sem_timedwait_save(sem_t *sem, struct timespec *deadline, PyThreadState *_save) #endif /* !HAVE_SEM_TIMEDWAIT */ /*[clinic input] +@critical_section _multiprocessing.SemLock.acquire block as blocking: bool = True @@ -308,7 +311,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=079ca779975f3ad6]*/ { int res, err = 0; struct timespec deadline = {0}; @@ -382,6 +385,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, } /*[clinic input] +@critical_section _multiprocessing.SemLock.release Release the semaphore/lock. @@ -389,7 +393,7 @@ Release the semaphore/lock. static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self) -/*[clinic end generated code: output=b22f53ba96b0d1db input=ba7e63a961885d3d]*/ +/*[clinic end generated code: output=b22f53ba96b0d1db input=9bd62d3645e7a531]*/ { if (self->kind == RECURSIVE_MUTEX) { if (!ISMINE(self)) { @@ -583,6 +587,7 @@ semlock_dealloc(SemLockObject* self) } /*[clinic input] +@critical_section _multiprocessing.SemLock._count Num of `acquire()`s minus num of `release()`s for this process. @@ -590,7 +595,7 @@ Num of `acquire()`s minus num of `release()`s for this process. static PyObject * _multiprocessing_SemLock__count_impl(SemLockObject *self) -/*[clinic end generated code: output=5ba8213900e517bb input=36fc59b1cd1025ab]*/ +/*[clinic end generated code: output=5ba8213900e517bb input=9fa6e0b321b16935]*/ { return PyLong_FromLong((long)self->count); } @@ -677,6 +682,7 @@ _multiprocessing_SemLock__after_fork_impl(SemLockObject *self) } /*[clinic input] +@critical_section _multiprocessing.SemLock.__enter__ Enter the semaphore/lock. @@ -684,12 +690,13 @@ Enter the semaphore/lock. static PyObject * _multiprocessing_SemLock___enter___impl(SemLockObject *self) -/*[clinic end generated code: output=beeb2f07c858511f input=c5e27d594284690b]*/ +/*[clinic end generated code: output=beeb2f07c858511f input=d35c9860992ee790]*/ { return _multiprocessing_SemLock_acquire_impl(self, 1, Py_None); } /*[clinic input] +@critical_section _multiprocessing.SemLock.__exit__ exc_type: object = None @@ -704,7 +711,7 @@ static PyObject * _multiprocessing_SemLock___exit___impl(SemLockObject *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb) -/*[clinic end generated code: output=3b37c1a9f8b91a03 input=7d644b64a89903f8]*/ +/*[clinic end generated code: output=3b37c1a9f8b91a03 input=1610c8cc3e0e337e]*/ { return _multiprocessing_SemLock_release_impl(self); } diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 93c71377f03a76..dc93063aee7e54 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -5,9 +5,13 @@ #include "Python.h" #include "compile.h" #include "opcode.h" -#include "internal/pycore_code.h" -#include "internal/pycore_compile.h" -#include "internal/pycore_intrinsics.h" +#include "pycore_ceval.h" +#include "pycore_code.h" +#include "pycore_compile.h" +#include "pycore_intrinsics.h" +#include "pycore_optimizer.h" // _Py_GetExecutor() +#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, OPCODE_HAS_*, etc +#include "pycore_opcode_utils.h" /*[clinic input] module _opcode @@ -79,7 +83,7 @@ static int _opcode_is_valid_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b0d918ea1d073f65 input=fe23e0aa194ddae0]*/ { - return _PyCompile_OpcodeIsValid(opcode); + return IS_VALID_OPCODE(opcode); } /*[clinic input] @@ -95,8 +99,7 @@ static int _opcode_has_arg_impl(PyObject *module, int opcode) /*[clinic end generated code: output=7a062d3b2dcc0815 input=93d878ba6361db5f]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasArg(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_ARG(opcode); } /*[clinic input] @@ -112,8 +115,7 @@ static int _opcode_has_const_impl(PyObject *module, int opcode) /*[clinic end generated code: output=c646d5027c634120 input=a6999e4cf13f9410]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasConst(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_CONST(opcode); } /*[clinic input] @@ -129,8 +131,7 @@ static int _opcode_has_name_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasName(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_NAME(opcode); } /*[clinic input] @@ -146,9 +147,7 @@ static int _opcode_has_jump_impl(PyObject *module, int opcode) /*[clinic end generated code: output=e9c583c669f1c46a input=35f711274357a0c3]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasJump(opcode); - + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_JUMP(opcode); } /*[clinic input] @@ -169,9 +168,7 @@ static int _opcode_has_free_impl(PyObject *module, int opcode) /*[clinic end generated code: output=d81ae4d79af0ee26 input=117dcd5c19c1139b]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasFree(opcode); - + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_FREE(opcode); } /*[clinic input] @@ -187,8 +184,7 @@ static int _opcode_has_local_impl(PyObject *module, int opcode) /*[clinic end generated code: output=da5a8616b7a5097b input=9a798ee24aaef49d]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasLocal(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_LOCAL(opcode); } /*[clinic input] @@ -204,8 +200,7 @@ static int _opcode_has_exc_impl(PyObject *module, int opcode) /*[clinic end generated code: output=41b68dff0ec82a52 input=db0e4bdb9bf13fa5]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasExc(opcode); + return IS_VALID_OPCODE(opcode) && IS_BLOCK_PUSH_OPCODE(opcode); } /*[clinic input] @@ -347,6 +342,60 @@ _opcode_get_intrinsic2_descs_impl(PyObject *module) return list; } +/*[clinic input] + +_opcode.get_special_method_names + +Return a list of special method names. +[clinic start generated code]*/ + +static PyObject * +_opcode_get_special_method_names_impl(PyObject *module) +/*[clinic end generated code: output=fce72614cd988d17 input=25f2115560bdf163]*/ +{ + PyObject *list = PyList_New(SPECIAL_MAX + 1); + if (list == NULL) { + return NULL; + } + for (int i=0; i <= SPECIAL_MAX; i++) { + PyObject *name = _Py_SpecialMethods[i].name; + if (name == NULL) { + Py_DECREF(list); + return NULL; + } + PyList_SET_ITEM(list, i, name); + } + return list; +} + +/*[clinic input] + +_opcode.get_executor + + code: object + offset: int + +Return the executor object at offset in code if exists, None otherwise. +[clinic start generated code]*/ + +static PyObject * +_opcode_get_executor_impl(PyObject *module, PyObject *code, int offset) +/*[clinic end generated code: output=c035c7a47b16648f input=85eff93ea7aac282]*/ +{ + if (!PyCode_Check(code)) { + PyErr_Format(PyExc_TypeError, + "expected a code object, not '%.100s'", + Py_TYPE(code)->tp_name); + return NULL; + } +#ifdef _Py_TIER2 + return (PyObject *)_Py_GetExecutor((PyCodeObject *)code, offset); +#else + PyErr_Format(PyExc_RuntimeError, + "Executors are not available in this build"); + return NULL; +#endif +} static PyMethodDef opcode_functions[] = { @@ -363,10 +412,12 @@ opcode_functions[] = { _OPCODE_GET_NB_OPS_METHODDEF _OPCODE_GET_INTRINSIC1_DESCS_METHODDEF _OPCODE_GET_INTRINSIC2_DESCS_METHODDEF + _OPCODE_GET_EXECUTOR_METHODDEF + _OPCODE_GET_SPECIAL_METHOD_NAMES_METHODDEF {NULL, NULL, 0, NULL} }; -int +static int _opcode_exec(PyObject *m) { if (PyModule_AddIntMacro(m, ENABLE_SPECIALIZATION) < 0) { return -1; @@ -377,6 +428,7 @@ _opcode_exec(PyObject *m) { static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _opcode_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_operator.c b/Modules/_operator.c index 1f6496d381adac..7e0d1f3df87e4d 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -2,6 +2,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "clinic/_operator.c.h" @@ -727,6 +728,34 @@ _operator_is_not_impl(PyObject *module, PyObject *a, PyObject *b) return Py_NewRef(result); } +/*[clinic input] +_operator.is_none = _operator.neg + +Same as a is None. +[clinic start generated code]*/ + +static PyObject * +_operator_is_none(PyObject *module, PyObject *a) +/*[clinic end generated code: output=07159cc102261dec input=0448b38af7b8533d]*/ +{ + PyObject *result = Py_IsNone(a) ? Py_True : Py_False; + return Py_NewRef(result); +} + +/*[clinic input] +_operator.is_not_none = _operator.neg + +Same as a is not None. +[clinic start generated code]*/ + +static PyObject * +_operator_is_not_none(PyObject *module, PyObject *a) +/*[clinic end generated code: output=b0168a51451d9140 input=7587f38ebac51688]*/ +{ + PyObject *result = Py_IsNone(a) ? Py_False : Py_True; + return Py_NewRef(result); +} + /* compare_digest **********************************************************/ /* @@ -915,6 +944,8 @@ static struct PyMethodDef operator_methods[] = { _OPERATOR_COUNTOF_METHODDEF _OPERATOR_IS__METHODDEF _OPERATOR_IS_NOT_METHODDEF + _OPERATOR_IS_NONE_METHODDEF + _OPERATOR_IS_NOT_NONE_METHODDEF _OPERATOR_INDEX_METHODDEF _OPERATOR_ADD_METHODDEF _OPERATOR_SUB_METHODDEF @@ -966,6 +997,18 @@ static struct PyMethodDef operator_methods[] = { }; + +static PyObject * +text_signature(PyObject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("(obj, /)"); +} + +static PyGetSetDef common_getset[] = { + {"__text_signature__", text_signature, (setter)NULL}, + {NULL} +}; + /* itemgetter object **********************************************************/ typedef struct { @@ -1171,6 +1214,7 @@ static PyType_Slot itemgetter_type_slots[] = { {Py_tp_clear, itemgetter_clear}, {Py_tp_methods, itemgetter_methods}, {Py_tp_members, itemgetter_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, itemgetter_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, itemgetter_repr}, @@ -1223,6 +1267,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; /* prepare attr while checking args */ + PyInterpreterState *interp = _PyInterpreterState_GET(); for (idx = 0; idx < nattrs; ++idx) { PyObject *item = PyTuple_GET_ITEM(args, idx); int dot_count; @@ -1246,7 +1291,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (dot_count == 0) { Py_INCREF(item); - PyUnicode_InternInPlace(&item); + _PyUnicode_InternMortal(interp, &item); PyTuple_SET_ITEM(attr, idx, item); } else { /* make it a tuple of non-dotted attrnames */ PyObject *attr_chain = PyTuple_New(dot_count + 1); @@ -1272,7 +1317,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(attr); return NULL; } - PyUnicode_InternInPlace(&attr_chain_item); + _PyUnicode_InternMortal(interp, &attr_chain_item); PyTuple_SET_ITEM(attr_chain, attr_chain_idx, attr_chain_item); ++attr_chain_idx; unibuff_till = unibuff_from = unibuff_till + 1; @@ -1286,7 +1331,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(attr); return NULL; } - PyUnicode_InternInPlace(&attr_chain_item); + _PyUnicode_InternMortal(interp, &attr_chain_item); PyTuple_SET_ITEM(attr_chain, attr_chain_idx, attr_chain_item); PyTuple_SET_ITEM(attr, idx, attr_chain); @@ -1528,6 +1573,7 @@ static PyType_Slot attrgetter_type_slots[] = { {Py_tp_clear, attrgetter_clear}, {Py_tp_methods, attrgetter_methods}, {Py_tp_members, attrgetter_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, attrgetter_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, attrgetter_repr}, @@ -1648,7 +1694,8 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } Py_INCREF(name); - PyUnicode_InternInPlace(&name); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &name); mc->name = name; mc->xargs = Py_XNewRef(args); // allows us to use borrowed references @@ -1863,6 +1910,7 @@ static PyType_Slot methodcaller_type_slots[] = { {Py_tp_clear, methodcaller_clear}, {Py_tp_methods, methodcaller_methods}, {Py_tp_members, methodcaller_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, methodcaller_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, methodcaller_repr}, @@ -1913,6 +1961,7 @@ operator_exec(PyObject *module) static struct PyModuleDef_Slot operator_slots[] = { {Py_mod_exec, operator_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index f210c0ca205991..b2bd9545c1b130 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -9,15 +9,17 @@ #endif #include "Python.h" -#include "pycore_bytesobject.h" // _PyBytesWriter -#include "pycore_ceval.h" // _Py_EnterRecursiveCall() -#include "pycore_long.h" // _PyLong_AsByteArray() -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_object.h" // _PyNone_Type -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_runtime.h" // _Py_ID() -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_AsByteArray() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyNone_Type +#include "pycore_pyerrors.h" // _PyErr_FormatNote +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_runtime.h" // _Py_ID() +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include // strtol() @@ -39,7 +41,7 @@ class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "" already includes it. */ enum { HIGHEST_PROTOCOL = 5, - DEFAULT_PROTOCOL = 4 + DEFAULT_PROTOCOL = 5 }; #ifdef MS_WINDOWS @@ -1802,80 +1804,53 @@ memo_put(PickleState *st, PicklerObject *self, PyObject *obj) } static PyObject * -get_dotted_path(PyObject *obj, PyObject *name) +get_dotted_path(PyObject *name) +{ + return PyUnicode_Split(name, _Py_LATIN1_CHR('.'), -1); +} + +static int +check_dotted_path(PickleState *st, PyObject *obj, PyObject *dotted_path) { - PyObject *dotted_path; Py_ssize_t i, n; - _Py_DECLARE_STR(dot, "."); - dotted_path = PyUnicode_Split(name, &_Py_STR(dot), -1); - if (dotted_path == NULL) - return NULL; n = PyList_GET_SIZE(dotted_path); assert(n >= 1); for (i = 0; i < n; i++) { PyObject *subpath = PyList_GET_ITEM(dotted_path, i); if (_PyUnicode_EqualToASCIIString(subpath, "")) { - if (obj == NULL) - PyErr_Format(PyExc_AttributeError, - "Can't pickle local object %R", name); - else - PyErr_Format(PyExc_AttributeError, - "Can't pickle local attribute %R on %R", name, obj); - Py_DECREF(dotted_path); - return NULL; + PyErr_Format(st->PicklingError, + "Can't pickle local object %R", obj); + return -1; } } - return dotted_path; + return 0; } static PyObject * -get_deep_attribute(PyObject *obj, PyObject *names, PyObject **pparent) +getattribute(PyObject *obj, PyObject *names, int raises) { Py_ssize_t i, n; - PyObject *parent = NULL; assert(PyList_CheckExact(names)); Py_INCREF(obj); n = PyList_GET_SIZE(names); for (i = 0; i < n; i++) { PyObject *name = PyList_GET_ITEM(names, i); - Py_XSETREF(parent, obj); - (void)PyObject_GetOptionalAttr(parent, name, &obj); + PyObject *parent = obj; + if (raises) { + obj = PyObject_GetAttr(parent, name); + } + else { + (void)PyObject_GetOptionalAttr(parent, name, &obj); + } + Py_DECREF(parent); if (obj == NULL) { - Py_DECREF(parent); return NULL; } } - if (pparent != NULL) - *pparent = parent; - else - Py_XDECREF(parent); return obj; } - -static PyObject * -getattribute(PyObject *obj, PyObject *name, int allow_qualname) -{ - PyObject *dotted_path, *attr; - - if (allow_qualname) { - dotted_path = get_dotted_path(obj, name); - if (dotted_path == NULL) - return NULL; - attr = get_deep_attribute(obj, dotted_path, NULL); - Py_DECREF(dotted_path); - } - else { - (void)PyObject_GetOptionalAttr(obj, name, &attr); - } - if (attr == NULL && !PyErr_Occurred()) { - PyErr_Format(PyExc_AttributeError, - "Can't get attribute %R on %R", name, obj); - } - return attr; -} - static int _checkmodule(PyObject *module_name, PyObject *module, PyObject *global, PyObject *dotted_path) @@ -1888,7 +1863,7 @@ _checkmodule(PyObject *module_name, PyObject *module, return -1; } - PyObject *candidate = get_deep_attribute(module, dotted_path, NULL); + PyObject *candidate = getattribute(module, dotted_path, 0); if (candidate == NULL) { return -1; } @@ -1901,73 +1876,125 @@ _checkmodule(PyObject *module_name, PyObject *module, } static PyObject * -whichmodule(PyObject *global, PyObject *dotted_path) +whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject *dotted_path) { PyObject *module_name; PyObject *module = NULL; Py_ssize_t i; PyObject *modules; + if (check_dotted_path(st, global, dotted_path) < 0) { + return NULL; + } if (PyObject_GetOptionalAttr(global, &_Py_ID(__module__), &module_name) < 0) { return NULL; } - if (module_name) { + if (module_name == NULL || module_name == Py_None) { /* In some rare cases (e.g., bound methods of extension types), __module__ can be None. If it is so, then search sys.modules for the module of global. */ - if (module_name != Py_None) - return module_name; Py_CLEAR(module_name); - } - assert(module_name == NULL); - - /* Fallback on walking sys.modules */ - PyThreadState *tstate = _PyThreadState_GET(); - modules = _PySys_GetAttr(tstate, &_Py_ID(modules)); - if (modules == NULL) { - PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); - return NULL; - } - if (PyDict_CheckExact(modules)) { - i = 0; - while (PyDict_Next(modules, &i, &module_name, &module)) { - if (_checkmodule(module_name, module, global, dotted_path) == 0) { - return Py_NewRef(module_name); - } - if (PyErr_Occurred()) { - return NULL; - } - } - } - else { - PyObject *iterator = PyObject_GetIter(modules); - if (iterator == NULL) { + PyThreadState *tstate = _PyThreadState_GET(); + modules = _PySys_GetAttr(tstate, &_Py_ID(modules)); + if (modules == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); return NULL; } - while ((module_name = PyIter_Next(iterator))) { - module = PyObject_GetItem(modules, module_name); - if (module == NULL) { + if (PyDict_CheckExact(modules)) { + i = 0; + while (PyDict_Next(modules, &i, &module_name, &module)) { + Py_INCREF(module_name); + Py_INCREF(module); + if (_checkmodule(module_name, module, global, dotted_path) == 0) { + Py_DECREF(module); + return module_name; + } + Py_DECREF(module); Py_DECREF(module_name); - Py_DECREF(iterator); + if (PyErr_Occurred()) { + return NULL; + } + } + } + else { + PyObject *iterator = PyObject_GetIter(modules); + if (iterator == NULL) { return NULL; } - if (_checkmodule(module_name, module, global, dotted_path) == 0) { + while ((module_name = PyIter_Next(iterator))) { + module = PyObject_GetItem(modules, module_name); + if (module == NULL) { + Py_DECREF(module_name); + Py_DECREF(iterator); + return NULL; + } + if (_checkmodule(module_name, module, global, dotted_path) == 0) { + Py_DECREF(module); + Py_DECREF(iterator); + return module_name; + } Py_DECREF(module); - Py_DECREF(iterator); - return module_name; - } - Py_DECREF(module); - Py_DECREF(module_name); - if (PyErr_Occurred()) { - Py_DECREF(iterator); - return NULL; + Py_DECREF(module_name); + if (PyErr_Occurred()) { + Py_DECREF(iterator); + return NULL; + } } + Py_DECREF(iterator); + } + if (PyErr_Occurred()) { + return NULL; } - Py_DECREF(iterator); + + /* If no module is found, use __main__. */ + module_name = Py_NewRef(&_Py_ID(__main__)); } - /* If no module is found, use __main__. */ - return &_Py_ID(__main__); + /* XXX: Change to use the import C API directly with level=0 to disallow + relative imports. + + XXX: PyImport_ImportModuleLevel could be used. However, this bypasses + builtins.__import__. Therefore, _pickle, unlike pickle.py, will ignore + custom import functions (IMHO, this would be a nice security + feature). The import C API would need to be extended to support the + extra parameters of __import__ to fix that. */ + module = PyImport_Import(module_name); + if (module == NULL) { + if (PyErr_ExceptionMatches(PyExc_ImportError) || + PyErr_ExceptionMatches(PyExc_ValueError)) + { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(st->PicklingError, + "Can't pickle %R: %S", global, exc); + _PyErr_ChainExceptions1(exc); + } + Py_DECREF(module_name); + return NULL; + } + PyObject *actual = getattribute(module, dotted_path, 1); + Py_DECREF(module); + if (actual == NULL) { + assert(PyErr_Occurred()); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(st->PicklingError, + "Can't pickle %R: it's not found as %S.%S", + global, module_name, global_name); + _PyErr_ChainExceptions1(exc); + } + Py_DECREF(module_name); + return NULL; + } + if (actual != global) { + Py_DECREF(actual); + PyErr_Format(st->PicklingError, + "Can't pickle %R: it's not the same object as %S.%S", + global, module_name, global_name); + Py_DECREF(module_name); + return NULL; + } + Py_DECREF(actual); + return module_name; } /* fast_save_enter() and fast_save_leave() are guards against recursive @@ -2119,7 +2146,7 @@ save_long(PicklerObject *self, PyObject *obj) if (self->proto >= 2) { /* Linear-time pickling. */ - size_t nbits; + int64_t nbits; size_t nbytes; unsigned char *pdata; char header[5]; @@ -2134,8 +2161,8 @@ save_long(PicklerObject *self, PyObject *obj) return 0; } nbits = _PyLong_NumBits(obj); - if (nbits == (size_t)-1 && PyErr_Occurred()) - goto error; + assert(nbits >= 0); + assert(!PyErr_Occurred()); /* How many bytes do we need? There are nbits >> 3 full * bytes of data, and nbits & 7 leftover bits. If there * are any leftover bits, then we clearly need another @@ -2150,7 +2177,7 @@ save_long(PicklerObject *self, PyObject *obj) * for in advance, though, so we always grab an extra * byte at the start, and cut it back later if possible. */ - nbytes = (nbits >> 3) + 1; + nbytes = (size_t)((nbits >> 3) + 1); if (nbytes > 0x7fffffffL) { PyErr_SetString(PyExc_OverflowError, "int too large to pickle"); @@ -2162,7 +2189,8 @@ save_long(PicklerObject *self, PyObject *obj) pdata = (unsigned char *)PyBytes_AS_STRING(repr); i = _PyLong_AsByteArray((PyLongObject *)obj, pdata, nbytes, - 1 /* little endian */ , 1 /* signed */ ); + 1 /* little endian */ , 1 /* signed */ , + 1 /* with exceptions */); if (i < 0) goto error; /* If the int is negative, this may be a byte more than @@ -2506,7 +2534,7 @@ save_picklebuffer(PickleState *st, PicklerObject *self, PyObject *obj) { if (self->proto < 5) { PyErr_SetString(st->PicklingError, - "PickleBuffer can only pickled with protocol >= 5"); + "PickleBuffer can only be pickled with protocol >= 5"); return -1; } const Py_buffer* view = PyPickleBuffer_GetBuffer(obj); @@ -2737,8 +2765,10 @@ store_tuple_elements(PickleState *state, PicklerObject *self, PyObject *t, if (element == NULL) return -1; - if (save(state, self, element, 0) < 0) + if (save(state, self, element, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %zd", t, i); return -1; + } } return 0; @@ -2857,11 +2887,12 @@ save_tuple(PickleState *state, PicklerObject *self, PyObject *obj) * Returns 0 on success, <0 on error. */ static int -batch_list(PickleState *state, PicklerObject *self, PyObject *iter) +batch_list(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *origobj) { PyObject *obj = NULL; PyObject *firstitem = NULL; int i, n; + Py_ssize_t total = 0; const char mark_op = MARK; const char append_op = APPEND; @@ -2876,7 +2907,7 @@ batch_list(PickleState *state, PicklerObject *self, PyObject *iter) if (self->proto == 0) { /* APPENDS isn't available; do one at a time. */ - for (;;) { + for (;; total++) { obj = PyIter_Next(iter); if (obj == NULL) { if (PyErr_Occurred()) @@ -2885,8 +2916,10 @@ batch_list(PickleState *state, PicklerObject *self, PyObject *iter) } i = save(state, self, obj, 0); Py_DECREF(obj); - if (i < 0) + if (i < 0) { + _PyErr_FormatNote("when serializing %T item %zd", origobj, total); return -1; + } if (_Pickler_Write(self, &append_op, 1) < 0) return -1; } @@ -2912,8 +2945,10 @@ batch_list(PickleState *state, PicklerObject *self, PyObject *iter) goto error; /* Only one item to write */ - if (save(state, self, firstitem, 0) < 0) + if (save(state, self, firstitem, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %zd", origobj, total); goto error; + } if (_Pickler_Write(self, &append_op, 1) < 0) goto error; Py_CLEAR(firstitem); @@ -2926,16 +2961,22 @@ batch_list(PickleState *state, PicklerObject *self, PyObject *iter) if (_Pickler_Write(self, &mark_op, 1) < 0) goto error; - if (save(state, self, firstitem, 0) < 0) + if (save(state, self, firstitem, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %zd", origobj, total); goto error; + } Py_CLEAR(firstitem); + total++; n = 1; /* Fetch and save up to BATCHSIZE items */ while (obj) { - if (save(state, self, obj, 0) < 0) + if (save(state, self, obj, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %zd", origobj, total); goto error; + } Py_CLEAR(obj); + total++; n += 1; if (n == BATCHSIZE) @@ -2991,8 +3032,10 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) Py_INCREF(item); int err = save(state, self, item, 0); Py_DECREF(item); - if (err < 0) + if (err < 0) { + _PyErr_FormatNote("when serializing %T item 0", obj); return -1; + } if (_Pickler_Write(self, &append_op, 1) < 0) return -1; return 0; @@ -3009,8 +3052,10 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) Py_INCREF(item); int err = save(state, self, item, 0); Py_DECREF(item); - if (err < 0) + if (err < 0) { + _PyErr_FormatNote("when serializing %T item %zd", obj, total); return -1; + } total++; if (++this_batch == BATCHSIZE) break; @@ -3070,7 +3115,7 @@ save_list(PickleState *state, PicklerObject *self, PyObject *obj) Py_DECREF(iter); goto error; } - status = batch_list(state, self, iter); + status = batch_list(state, self, iter, obj); _Py_LeaveRecursiveCall(); Py_DECREF(iter); } @@ -3098,7 +3143,7 @@ save_list(PickleState *state, PicklerObject *self, PyObject *obj) * ugly to bear. */ static int -batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) +batch_dict(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *origobj) { PyObject *obj = NULL; PyObject *firstitem = NULL; @@ -3122,11 +3167,17 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) { PyErr_SetString(PyExc_TypeError, "dict items " "iterator must return 2-tuples"); + Py_DECREF(obj); return -1; } i = save(state, self, PyTuple_GET_ITEM(obj, 0), 0); - if (i >= 0) + if (i >= 0) { i = save(state, self, PyTuple_GET_ITEM(obj, 1), 0); + if (i < 0) { + _PyErr_FormatNote("when serializing %T item %R", + origobj, PyTuple_GET_ITEM(obj, 0)); + } + } Py_DECREF(obj); if (i < 0) return -1; @@ -3162,8 +3213,11 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) /* Only one item to write */ if (save(state, self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) goto error; - if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", + origobj, PyTuple_GET_ITEM(firstitem, 0)); goto error; + } if (_Pickler_Write(self, &setitem_op, 1) < 0) goto error; Py_CLEAR(firstitem); @@ -3178,8 +3232,11 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) if (save(state, self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) goto error; - if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", + origobj, PyTuple_GET_ITEM(firstitem, 0)); goto error; + } Py_CLEAR(firstitem); n = 1; @@ -3190,9 +3247,13 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) "iterator must return 2-tuples"); goto error; } - if (save(state, self, PyTuple_GET_ITEM(obj, 0), 0) < 0 || - save(state, self, PyTuple_GET_ITEM(obj, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(obj, 0), 0) < 0) + goto error; + if (save(state, self, PyTuple_GET_ITEM(obj, 1), 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", + origobj, PyTuple_GET_ITEM(obj, 0)); goto error; + } Py_CLEAR(obj); n += 1; @@ -3253,6 +3314,7 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) goto error; } if (save(state, self, value, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", obj, key); goto error; } Py_CLEAR(key); @@ -3274,6 +3336,7 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) goto error; } if (save(state, self, value, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", obj, key); goto error; } Py_CLEAR(key); @@ -3348,7 +3411,7 @@ save_dict(PickleState *state, PicklerObject *self, PyObject *obj) Py_DECREF(iter); goto error; } - status = batch_dict(state, self, iter); + status = batch_dict(state, self, iter, obj); _Py_LeaveRecursiveCall(); Py_DECREF(iter); } @@ -3412,15 +3475,23 @@ save_set(PickleState *state, PicklerObject *self, PyObject *obj) i = 0; if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; - while (_PySet_NextEntry(obj, &ppos, &item, &hash)) { - Py_INCREF(item); - int err = save(state, self, item, 0); + + int err = 0; + Py_BEGIN_CRITICAL_SECTION(obj); + while (_PySet_NextEntryRef(obj, &ppos, &item, &hash)) { + err = save(state, self, item, 0); Py_CLEAR(item); - if (err < 0) - return -1; + if (err < 0) { + _PyErr_FormatNote("when serializing %T element", obj); + break; + } if (++i == BATCHSIZE) break; } + Py_END_CRITICAL_SECTION(); + if (err < 0) { + return -1; + } if (_Pickler_Write(self, &additems_op, 1) < 0) return -1; if (PySet_GET_SIZE(obj) != set_size) { @@ -3485,6 +3556,7 @@ save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) break; } if (save(state, self, item, 0) < 0) { + _PyErr_FormatNote("when serializing %T element", obj); Py_DECREF(item); Py_DECREF(iter); return -1; @@ -3582,11 +3654,7 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, { PyObject *global_name = NULL; PyObject *module_name = NULL; - PyObject *module = NULL; - PyObject *parent = NULL; PyObject *dotted_path = NULL; - PyObject *lastname = NULL; - PyObject *cls; int status = 0; const char global_op = GLOBAL; @@ -3604,47 +3672,13 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, } } - dotted_path = get_dotted_path(module, global_name); + dotted_path = get_dotted_path(global_name); if (dotted_path == NULL) goto error; - module_name = whichmodule(obj, dotted_path); + module_name = whichmodule(st, obj, global_name, dotted_path); if (module_name == NULL) goto error; - /* XXX: Change to use the import C API directly with level=0 to disallow - relative imports. - - XXX: PyImport_ImportModuleLevel could be used. However, this bypasses - builtins.__import__. Therefore, _pickle, unlike pickle.py, will ignore - custom import functions (IMHO, this would be a nice security - feature). The import C API would need to be extended to support the - extra parameters of __import__ to fix that. */ - module = PyImport_Import(module_name); - if (module == NULL) { - PyErr_Format(st->PicklingError, - "Can't pickle %R: import of module %R failed", - obj, module_name); - goto error; - } - lastname = Py_NewRef(PyList_GET_ITEM(dotted_path, - PyList_GET_SIZE(dotted_path) - 1)); - cls = get_deep_attribute(module, dotted_path, &parent); - Py_CLEAR(dotted_path); - if (cls == NULL) { - PyErr_Format(st->PicklingError, - "Can't pickle %R: attribute lookup %S on %S failed", - obj, global_name, module_name); - goto error; - } - if (cls != obj) { - Py_DECREF(cls); - PyErr_Format(st->PicklingError, - "Can't pickle %R: it's not the same object as %S.%S", - obj, module_name, global_name); - goto error; - } - Py_DECREF(cls); - if (self->proto >= 2) { /* See whether this is in the extension registry, and if * so generate an EXT opcode. @@ -3659,34 +3693,24 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, if (extension_key == NULL) { goto error; } - code_obj = PyDict_GetItemWithError(st->extension_registry, - extension_key); + if (PyDict_GetItemRef(st->extension_registry, extension_key, &code_obj) < 0) { + Py_DECREF(extension_key); + goto error; + } Py_DECREF(extension_key); - /* The object is not registered in the extension registry. - This is the most likely code path. */ if (code_obj == NULL) { - if (PyErr_Occurred()) { - goto error; - } + /* The object is not registered in the extension registry. + This is the most likely code path. */ goto gen_global; } - /* XXX: pickle.py doesn't check neither the type, nor the range - of the value returned by the extension_registry. It should for - consistency. */ - - /* Verify code_obj has the right type and value. */ - if (!PyLong_Check(code_obj)) { - PyErr_Format(st->PicklingError, - "Can't pickle %R: extension code %R isn't an integer", - obj, code_obj); - goto error; - } - code = PyLong_AS_LONG(code_obj); + code = PyLong_AsLong(code_obj); + Py_DECREF(code_obj); if (code <= 0 || code > 0x7fffffffL) { + /* Should never happen in normal circumstances, since the type and + the value of the code are checked in copyreg.add_extension(). */ if (!PyErr_Occurred()) - PyErr_Format(st->PicklingError, "Can't pickle %R: extension " - "code %ld is out of range", obj, code); + PyErr_Format(PyExc_RuntimeError, "extension code %ld is out of range", code); goto error; } @@ -3716,9 +3740,6 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, } else { gen_global: - if (parent == module) { - Py_SETREF(global_name, Py_NewRef(lastname)); - } if (self->proto >= 4) { const char stack_global_op = STACK_GLOBAL; @@ -3730,20 +3751,30 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, if (_Pickler_Write(self, &stack_global_op, 1) < 0) goto error; } - else if (parent != module) { - PyObject *reduce_value = Py_BuildValue("(O(OO))", - st->getattr, parent, lastname); - if (reduce_value == NULL) - goto error; - status = save_reduce(st, self, reduce_value, NULL); - Py_DECREF(reduce_value); - if (status < 0) - goto error; - } else { /* Generate a normal global opcode if we are using a pickle protocol < 4, or if the object is not registered in the - extension registry. */ + extension registry. + + Objects with multi-part __qualname__ are represented as + getattr(getattr(..., attrname1), attrname2). */ + const char mark_op = MARK; + const char tupletwo_op = (self->proto < 2) ? TUPLE : TUPLE2; + const char reduce_op = REDUCE; + Py_ssize_t i; + if (dotted_path) { + if (PyList_GET_SIZE(dotted_path) > 1) { + Py_SETREF(global_name, Py_NewRef(PyList_GET_ITEM(dotted_path, 0))); + } + for (i = 1; i < PyList_GET_SIZE(dotted_path); i++) { + if (save(st, self, st->getattr, 0) < 0 || + (self->proto < 2 && _Pickler_Write(self, &mark_op, 1) < 0)) + { + goto error; + } + } + } + PyObject *encoded; PyObject *(*unicode_encoder)(PyObject *); @@ -3771,11 +3802,14 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, } encoded = unicode_encoder(module_name); if (encoded == NULL) { - if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) + if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { + PyObject *exc = PyErr_GetRaisedException(); PyErr_Format(st->PicklingError, - "can't pickle module identifier '%S' using " + "can't pickle module identifier %R using " "pickle protocol %i", module_name, self->proto); + _PyErr_ChainExceptions1(exc); + } goto error; } if (_Pickler_Write(self, PyBytes_AS_STRING(encoded), @@ -3790,11 +3824,14 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, /* Save the name of the module. */ encoded = unicode_encoder(global_name); if (encoded == NULL) { - if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) + if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { + PyObject *exc = PyErr_GetRaisedException(); PyErr_Format(st->PicklingError, - "can't pickle global identifier '%S' using " + "can't pickle global identifier %R using " "pickle protocol %i", global_name, self->proto); + _PyErr_ChainExceptions1(exc); + } goto error; } if (_Pickler_Write(self, PyBytes_AS_STRING(encoded), @@ -3805,6 +3842,17 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, Py_DECREF(encoded); if (_Pickler_Write(self, "\n", 1) < 0) goto error; + + if (dotted_path) { + for (i = 1; i < PyList_GET_SIZE(dotted_path); i++) { + if (save(st, self, PyList_GET_ITEM(dotted_path, i), 0) < 0 || + _Pickler_Write(self, &tupletwo_op, 1) < 0 || + _Pickler_Write(self, &reduce_op, 1) < 0) + { + goto error; + } + } + } } /* Memoize the object. */ if (memo_put(st, self, obj) < 0) @@ -3817,10 +3865,7 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, } Py_XDECREF(module_name); Py_XDECREF(global_name); - Py_XDECREF(module); - Py_XDECREF(parent); Py_XDECREF(dotted_path); - Py_XDECREF(lastname); return status; } @@ -3947,8 +3992,9 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, size = PyTuple_Size(args); if (size < 2 || size > 6) { - PyErr_SetString(st->PicklingError, "tuple returned by " - "__reduce__ must contain 2 through 6 elements"); + PyErr_SetString(st->PicklingError, + "tuple returned by __reduce__ " + "must contain 2 through 6 elements"); return -1; } @@ -3958,13 +4004,15 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, return -1; if (!PyCallable_Check(callable)) { - PyErr_SetString(st->PicklingError, "first item of the tuple " - "returned by __reduce__ must be callable"); + PyErr_Format(st->PicklingError, + "first item of the tuple returned by __reduce__ " + "must be callable, not %T", callable); return -1; } if (!PyTuple_Check(argtup)) { - PyErr_SetString(st->PicklingError, "second item of the tuple " - "returned by __reduce__ must be a tuple"); + PyErr_Format(st->PicklingError, + "second item of the tuple returned by __reduce__ " + "must be a tuple, not %T", argtup); return -1; } @@ -3974,27 +4022,27 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, if (listitems == Py_None) listitems = NULL; else if (!PyIter_Check(listitems)) { - PyErr_Format(st->PicklingError, "fourth element of the tuple " - "returned by __reduce__ must be an iterator, not %s", - Py_TYPE(listitems)->tp_name); + PyErr_Format(st->PicklingError, + "fourth item of the tuple returned by __reduce__ " + "must be an iterator, not %T", listitems); return -1; } if (dictitems == Py_None) dictitems = NULL; else if (!PyIter_Check(dictitems)) { - PyErr_Format(st->PicklingError, "fifth element of the tuple " - "returned by __reduce__ must be an iterator, not %s", - Py_TYPE(dictitems)->tp_name); + PyErr_Format(st->PicklingError, + "fifth item of the tuple returned by __reduce__ " + "must be an iterator, not %T", dictitems); return -1; } if (state_setter == Py_None) state_setter = NULL; else if (!PyCallable_Check(state_setter)) { - PyErr_Format(st->PicklingError, "sixth element of the tuple " - "returned by __reduce__ must be a function, not %s", - Py_TYPE(state_setter)->tp_name); + PyErr_Format(st->PicklingError, + "sixth item of the tuple returned by __reduce__ " + "must be callable, not %T", state_setter); return -1; } @@ -4020,38 +4068,45 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, if (PyTuple_GET_SIZE(argtup) != 3) { PyErr_Format(st->PicklingError, - "length of the NEWOBJ_EX argument tuple must be " - "exactly 3, not %zd", PyTuple_GET_SIZE(argtup)); + "__newobj_ex__ expected 3 arguments, got %zd", + PyTuple_GET_SIZE(argtup)); return -1; } cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { PyErr_Format(st->PicklingError, - "first item from NEWOBJ_EX argument tuple must " - "be a class, not %.200s", Py_TYPE(cls)->tp_name); + "first argument to __newobj_ex__() " + "must be a class, not %T", cls); return -1; } args = PyTuple_GET_ITEM(argtup, 1); if (!PyTuple_Check(args)) { PyErr_Format(st->PicklingError, - "second item from NEWOBJ_EX argument tuple must " - "be a tuple, not %.200s", Py_TYPE(args)->tp_name); + "second argument to __newobj_ex__() " + "must be a tuple, not %T", args); return -1; } kwargs = PyTuple_GET_ITEM(argtup, 2); if (!PyDict_Check(kwargs)) { PyErr_Format(st->PicklingError, - "third item from NEWOBJ_EX argument tuple must " - "be a dict, not %.200s", Py_TYPE(kwargs)->tp_name); + "third argument to __newobj_ex__() " + "must be a dict, not %T", kwargs); return -1; } if (self->proto >= 4) { - if (save(st, self, cls, 0) < 0 || - save(st, self, args, 0) < 0 || - save(st, self, kwargs, 0) < 0 || - _Pickler_Write(self, &newobj_ex_op, 1) < 0) { + if (save(st, self, cls, 0) < 0) { + _PyErr_FormatNote("when serializing %T class", obj); + return -1; + } + if (save(st, self, args, 0) < 0 || + save(st, self, kwargs, 0) < 0) + { + _PyErr_FormatNote("when serializing %T __new__ arguments", obj); + return -1; + } + if (_Pickler_Write(self, &newobj_ex_op, 1) < 0) { return -1; } } @@ -4088,14 +4143,18 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, } if (save(st, self, callable, 0) < 0 || - save(st, self, newargs, 0) < 0 || - _Pickler_Write(self, &reduce_op, 1) < 0) { + save(st, self, newargs, 0) < 0) + { + _PyErr_FormatNote("when serializing %T reconstructor", obj); Py_DECREF(newargs); Py_DECREF(callable); return -1; } Py_DECREF(newargs); Py_DECREF(callable); + if (_Pickler_Write(self, &reduce_op, 1) < 0) { + return -1; + } } } else if (use_newobj) { @@ -4106,14 +4165,17 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, /* Sanity checks. */ if (PyTuple_GET_SIZE(argtup) < 1) { - PyErr_SetString(st->PicklingError, "__newobj__ arglist is empty"); + PyErr_Format(st->PicklingError, + "__newobj__ expected at least 1 argument, got %zd", + PyTuple_GET_SIZE(argtup)); return -1; } cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { - PyErr_SetString(st->PicklingError, "args[0] from " - "__newobj__ args is not a type"); + PyErr_Format(st->PicklingError, + "first argument to __newobj__() " + "must be a class, not %T", cls); return -1; } @@ -4122,13 +4184,14 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, if (obj_class == NULL) { return -1; } - p = obj_class != cls; - Py_DECREF(obj_class); - if (p) { - PyErr_SetString(st->PicklingError, "args[0] from " - "__newobj__ args has the wrong class"); + if (obj_class != cls) { + PyErr_Format(st->PicklingError, + "first argument to __newobj__() " + "must be %R, not %R", obj_class, cls); + Py_DECREF(obj_class); return -1; } + Py_DECREF(obj_class); } /* XXX: These calls save() are prone to infinite recursion. Imagine what happen if the value returned by the __reduce__() method of @@ -4159,6 +4222,7 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, /* Save the class and its __new__ arguments. */ if (save(st, self, cls, 0) < 0) { + _PyErr_FormatNote("when serializing %T class", obj); return -1; } @@ -4168,18 +4232,27 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, p = save(st, self, newargtup, 0); Py_DECREF(newargtup); - if (p < 0) + if (p < 0) { + _PyErr_FormatNote("when serializing %T __new__ arguments", obj); return -1; + } /* Add NEWOBJ opcode. */ if (_Pickler_Write(self, &newobj_op, 1) < 0) return -1; } else { /* Not using NEWOBJ. */ - if (save(st, self, callable, 0) < 0 || - save(st, self, argtup, 0) < 0 || - _Pickler_Write(self, &reduce_op, 1) < 0) + if (save(st, self, callable, 0) < 0) { + _PyErr_FormatNote("when serializing %T reconstructor", obj); return -1; + } + if (save(st, self, argtup, 0) < 0) { + _PyErr_FormatNote("when serializing %T reconstructor arguments", obj); + return -1; + } + if (_Pickler_Write(self, &reduce_op, 1) < 0) { + return -1; + } } /* obj can be NULL when save_reduce() is used directly. A NULL obj means @@ -4204,16 +4277,19 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, return -1; } - if (listitems && batch_list(st, self, listitems) < 0) + if (listitems && batch_list(st, self, listitems, obj) < 0) return -1; - if (dictitems && batch_dict(st, self, dictitems) < 0) + if (dictitems && batch_dict(st, self, dictitems, obj) < 0) return -1; if (state) { if (state_setter == NULL) { - if (save(st, self, state, 0) < 0 || - _Pickler_Write(self, &build_op, 1) < 0) + if (save(st, self, state, 0) < 0) { + _PyErr_FormatNote("when serializing %T state", obj); + return -1; + } + if (_Pickler_Write(self, &build_op, 1) < 0) return -1; } else { @@ -4229,9 +4305,18 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, const char tupletwo_op = TUPLE2; const char pop_op = POP; - if (save(st, self, state_setter, 0) < 0 || - save(st, self, obj, 0) < 0 || save(st, self, state, 0) < 0 || - _Pickler_Write(self, &tupletwo_op, 1) < 0 || + if (save(st, self, state_setter, 0) < 0) { + _PyErr_FormatNote("when serializing %T state setter", obj); + return -1; + } + if (save(st, self, obj, 0) < 0) { + return -1; + } + if (save(st, self, state, 0) < 0) { + _PyErr_FormatNote("when serializing %T state", obj); + return -1; + } + if (_Pickler_Write(self, &tupletwo_op, 1) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0 || _Pickler_Write(self, &pop_op, 1) < 0) return -1; @@ -4421,8 +4506,7 @@ save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) } else { PyErr_Format(st->PicklingError, - "can't pickle '%.200s' object: %R", - type->tp_name, obj); + "Can't pickle %T object", obj); goto error; } } @@ -4438,12 +4522,16 @@ save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) } if (!PyTuple_Check(reduce_value)) { - PyErr_SetString(st->PicklingError, - "__reduce__ must return a string or tuple"); + PyErr_Format(st->PicklingError, + "__reduce__ must return a string or tuple, not %T", reduce_value); + _PyErr_FormatNote("when serializing %T object", obj); goto error; } status = save_reduce(st, self, reduce_value, obj); + if (status < 0) { + _PyErr_FormatNote("when serializing %T object", obj); + } if (0) { error: @@ -4685,7 +4773,7 @@ This takes a binary file for writing a pickle data stream. The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default -protocol is 4. It was introduced in Python 3.4, and is incompatible +protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest protocol @@ -4718,7 +4806,7 @@ static int _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, PyObject *protocol, int fix_imports, PyObject *buffer_callback) -/*[clinic end generated code: output=0abedc50590d259b input=a7c969699bf5dad3]*/ +/*[clinic end generated code: output=0abedc50590d259b input=cddc50f66b770002]*/ { /* In case of multiple __init__() calls, clear previous content. */ if (self->write != NULL) @@ -6517,11 +6605,13 @@ load_additems(PickleState *state, UnpicklerObject *self) if (result == NULL) { Pdata_clear(self->stack, i + 1); Py_SET_SIZE(self->stack, mark); + Py_DECREF(add_func); return -1; } Py_DECREF(result); } Py_SET_SIZE(self->stack, mark); + Py_DECREF(add_func); } return 0; @@ -6597,8 +6687,10 @@ load_build(PickleState *st, UnpicklerObject *self) /* normally the keys for instance attributes are interned. we should try to do that here. */ Py_INCREF(d_key); - if (PyUnicode_CheckExact(d_key)) - PyUnicode_InternInPlace(&d_key); + if (PyUnicode_CheckExact(d_key)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &d_key); + } if (PyObject_SetItem(dict, d_key, d_value) < 0) { Py_DECREF(d_key); goto error; @@ -7032,7 +7124,26 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, if (module == NULL) { return NULL; } - global = getattribute(module, global_name, self->proto >= 4); + if (self->proto >= 4) { + PyObject *dotted_path = get_dotted_path(global_name); + if (dotted_path == NULL) { + Py_DECREF(module); + return NULL; + } + global = getattribute(module, dotted_path, 1); + assert(global != NULL || PyErr_Occurred()); + if (global == NULL && PyList_GET_SIZE(dotted_path) > 1) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_AttributeError, + "Can't resolve path %R on module %R", + global_name, module_name); + _PyErr_ChainExceptions1(exc); + } + Py_DECREF(dotted_path); + } + else { + global = PyObject_GetAttr(module, global_name); + } Py_DECREF(module); return global; } @@ -7496,7 +7607,7 @@ be more efficient. The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default -protocol is 4. It was introduced in Python 3.4, and is incompatible +protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest protocol @@ -7522,7 +7633,7 @@ static PyObject * _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, PyObject *protocol, int fix_imports, PyObject *buffer_callback) -/*[clinic end generated code: output=706186dba996490c input=5ed6653da99cd97c]*/ +/*[clinic end generated code: output=706186dba996490c input=b89ce8d0e911fd46]*/ { PickleState *state = _Pickle_GetState(module); PicklerObject *pickler = _Pickler_New(state); @@ -7567,7 +7678,7 @@ Return the pickled representation of the object as a bytes object. The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default -protocol is 4. It was introduced in Python 3.4, and is incompatible +protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest protocol @@ -7587,7 +7698,7 @@ into *file* as part of the pickle stream. It is an error if static PyObject * _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, int fix_imports, PyObject *buffer_callback) -/*[clinic end generated code: output=fbab0093a5580fdf input=e543272436c6f987]*/ +/*[clinic end generated code: output=fbab0093a5580fdf input=139fc546886c63ac]*/ { PyObject *result; PickleState *state = _Pickle_GetState(module); @@ -7855,6 +7966,7 @@ _pickle_exec(PyObject *m) static PyModuleDef_Slot pickle_slots[] = { {Py_mod_exec, _pickle_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index aa1a300e4378dd..ad6d7ceda84e37 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -977,7 +977,6 @@ _posixsubprocess.fork_exec as subprocess_fork_exec uid as uid_object: object child_umask: int preexec_fn: object - allow_vfork: bool / Spawn a fresh new child process. @@ -1014,8 +1013,8 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, pid_t pgid_to_set, PyObject *gid_object, PyObject *extra_groups_packed, PyObject *uid_object, int child_umask, - PyObject *preexec_fn, int allow_vfork) -/*[clinic end generated code: output=7ee4f6ee5cf22b5b input=51757287ef266ffa]*/ + PyObject *preexec_fn) +/*[clinic end generated code: output=288464dc56e373c7 input=f311c3bcb5dd55c8]*/ { PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; @@ -1031,8 +1030,10 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep); PyInterpreterState *interp = _PyInterpreterState_GET(); - if ((preexec_fn != Py_None) && interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + if ((preexec_fn != Py_None) && + _PyInterpreterState_GetFinalizing(interp) != NULL) + { + PyErr_SetString(PyExc_PythonFinalizationError, "preexec_fn not supported at interpreter shutdown"); return NULL; } @@ -1216,7 +1217,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, #ifdef VFORK_USABLE /* Use vfork() only if it's safe. See the comment above child_exec(). */ sigset_t old_sigs; - if (preexec_fn == Py_None && allow_vfork && + if (preexec_fn == Py_None && uid == (uid_t)-1 && gid == (gid_t)-1 && extra_group_size < 0) { /* Block all signals to ensure that no signal handlers are run in the * child process while it shares memory with us. Note that signals @@ -1315,6 +1316,7 @@ static PyMethodDef module_methods[] = { static PyModuleDef_Slot _posixsubprocess_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 18b24855c52ad6..aee8db802d8c3f 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -6,7 +6,7 @@ #include "pycore_ceval.h" // Py_MakePendingCalls() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_parking_lot.h" -#include "pycore_time.h" // _PyTime_t +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include #include // offsetof() @@ -372,13 +372,13 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) /*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ { - _PyTime_t endtime = 0; + PyTime_t endtime = 0; // XXX Use PyThread_ParseTimeoutArg(). if (block != 0 && !Py_IsNone(timeout_obj)) { /* With timeout */ - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) { return NULL; @@ -594,6 +594,7 @@ queuemodule_exec(PyObject *module) static PyModuleDef_Slot queuemodule_slots[] = { {Py_mod_exec, queuemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 4403e1d132c057..ad66df47349db0 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -259,13 +259,15 @@ random_seed_urandom(RandomObject *self) return 0; } -static void +static int random_seed_time_pid(RandomObject *self) { - _PyTime_t now; - uint32_t key[5]; + PyTime_t now; + if (PyTime_Time(&now) < 0) { + return -1; + } - now = _PyTime_GetSystemClock(); + uint32_t key[5]; key[0] = (uint32_t)(now & 0xffffffffU); key[1] = (uint32_t)(now >> 32); @@ -277,11 +279,14 @@ random_seed_time_pid(RandomObject *self) key[2] = 0; #endif - now = _PyTime_GetMonotonicClock(); + if (PyTime_Monotonic(&now) < 0) { + return -1; + } key[3] = (uint32_t)(now & 0xffffffffU); key[4] = (uint32_t)(now >> 32); init_by_array(self, key, Py_ARRAY_LENGTH(key)); + return 0; } static int @@ -290,7 +295,8 @@ random_seed(RandomObject *self, PyObject *arg) int result = -1; /* guilty until proved innocent */ PyObject *n = NULL; uint32_t *key = NULL; - size_t bits, keyused; + int64_t bits; + size_t keyused; int res; if (arg == NULL || arg == Py_None) { @@ -299,7 +305,9 @@ random_seed(RandomObject *self, PyObject *arg) /* Reading system entropy failed, fall back on the worst entropy: use the current time and process identifier. */ - random_seed_time_pid(self); + if (random_seed_time_pid(self) < 0) { + return -1; + } } return 0; } @@ -327,11 +335,11 @@ random_seed(RandomObject *self, PyObject *arg) /* Now split n into 32-bit chunks, from the right. */ bits = _PyLong_NumBits(n); - if (bits == (size_t)-1 && PyErr_Occurred()) - goto Done; + assert(bits >= 0); + assert(!PyErr_Occurred()); /* Figure out how many 32-bit chunks this gives us. */ - keyused = bits == 0 ? 1 : (bits - 1) / 32 + 1; + keyused = bits == 0 ? 1 : (size_t)((bits - 1) / 32 + 1); /* Convert seed to byte sequence. */ key = (uint32_t *)PyMem_Malloc((size_t)4 * keyused); @@ -342,7 +350,8 @@ random_seed(RandomObject *self, PyObject *arg) res = _PyLong_AsByteArray((PyLongObject *)n, (unsigned char *)key, keyused * 4, PY_LITTLE_ENDIAN, - 0); /* unsigned */ + 0, /* unsigned */ + 1); /* with exceptions */ if (res == -1) { goto Done; } @@ -634,6 +643,7 @@ _random_exec(PyObject *module) static PyModuleDef_Slot _random_slots[] = { {Py_mod_exec, _random_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index fe82e918677f9a..e9170f2ce1ae87 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -3,11 +3,10 @@ * using the SystemConfiguration framework. */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -240,6 +239,7 @@ static PyMethodDef mod_methods[] = { static PyModuleDef_Slot _scproxy_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f099020c5f4e6f..d1a549a971c24a 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -4,7 +4,6 @@ #include "blob.h" #include "util.h" -#include "pycore_weakref.h" // _PyWeakref_GET_REF() #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/blob.c.h" @@ -100,10 +99,10 @@ blob_close_impl(pysqlite_Blob *self) void pysqlite_close_all_blobs(pysqlite_Connection *self) { - for (int i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(self->blobs); i++) { PyObject *weakref = PyList_GET_ITEM(self->blobs, i); - PyObject *blob = _PyWeakref_GET_REF(weakref); - if (blob == NULL) { + PyObject *blob; + if (!PyWeakref_GetRef(weakref, &blob)) { continue; } close_blob((pysqlite_Blob *)blob); diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 811314b5cd8aed..6cb610a12b59c6 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -744,7 +744,7 @@ pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyTypeObject *cls, PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -837,14 +837,14 @@ pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(progress_handler), &_Py_ID(n), }, + .ob_item = { &_Py_ID(progress_handler), _Py_LATIN1_CHR('n'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -925,7 +925,7 @@ pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyTypeObject * PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1866,4 +1866,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=3c6d0b748fac016f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5d4d7e4abe17b164 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index f97afcf5fcf16e..fc03e4a085c179 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -38,7 +38,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() -#include "pycore_weakref.h" // _PyWeakref_IS_DEAD() +#include "pycore_weakref.h" #include @@ -1065,7 +1065,7 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) for (Py_ssize_t i = 0; i < PyList_Size(self->cursors); i++) { PyObject* weakref = PyList_GetItem(self->cursors, i); - if (_PyWeakref_IS_DEAD(weakref)) { + if (_PyWeakref_IsDead(weakref)) { continue; } if (PyList_Append(new_list, weakref) != 0) { @@ -2561,6 +2561,12 @@ set_autocommit(pysqlite_Connection *self, PyObject *val, void *Py_UNUSED(ctx)) return 0; } +static PyObject * +get_sig(PyObject *self, void *Py_UNUSED(ctx)) +{ + return PyUnicode_FromString("(sql, /)"); +} + static const char connection_doc[] = PyDoc_STR("SQLite database connection object."); @@ -2570,6 +2576,7 @@ static PyGetSetDef connection_getset[] = { {"total_changes", (getter)pysqlite_connection_get_total_changes, (setter)0}, {"in_transaction", (getter)pysqlite_connection_get_in_transaction, (setter)0}, {"autocommit", (getter)get_autocommit, (setter)set_autocommit}, + {"__text_signature__", get_sig, (setter)0}, {NULL} }; diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index f95df612328e57..0fbd408f18cf6a 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -669,16 +669,13 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, } for (i = 0; i < num_params; i++) { const char *name = sqlite3_bind_parameter_name(self->st, i+1); - if (name != NULL) { - int ret = PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + if (name != NULL && name[0] != '?') { + PyErr_Format(state->ProgrammingError, "Binding %d ('%s') is a named parameter, but you " "supplied a sequence which requires nameless (qmark) " - "placeholders. Starting with Python 3.14 an " - "sqlite3.ProgrammingError will be raised.", + "placeholders.", i+1, name); - if (ret < 0) { - return; - } + return; } if (PyTuple_CheckExact(parameters)) { diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 46fed9f13281f3..698e81d9b897d0 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -714,10 +714,6 @@ module_exec(PyObject *module) goto error; } - if (PyModule_AddStringConstant(module, "_deprecated_version", PYSQLITE_VERSION) < 0) { - goto error; - } - if (PyModule_AddStringConstant(module, "sqlite_version", sqlite3_libversion())) { goto error; } @@ -758,6 +754,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 833a666301d8ff..9e8613ef67916e 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -162,7 +162,7 @@ _pysqlite_long_as_int64(PyObject * py_val) sqlite_int64 int64val; if (_PyLong_AsByteArray((PyLongObject *)py_val, (unsigned char *)&int64val, sizeof(int64val), - IS_LITTLE_ENDIAN, 1 /* signed */) >= 0) { + IS_LITTLE_ENDIAN, 1 /* signed */, 0) >= 0) { return int64val; } } diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 00fbd9674b8cdd..2c86f8869d8e58 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -530,7 +530,7 @@ state_fini(SRE_STATE* state) PyBuffer_Release(&state->buffer); Py_XDECREF(state->string); data_stack_dealloc(state); - /* See above PyMem_Del for why we explicitly cast here. */ + /* See above PyMem_Free() for why we explicitly cast here. */ PyMem_Free((void*) state->mark); state->mark = NULL; } @@ -1287,7 +1287,7 @@ pattern_subx(_sremodulestate* module_state, } else { if (state.isbytes) - item = _PyBytes_Join(joiner, list); + item = PyBytes_Join(joiner, list); else item = PyUnicode_Join(joiner, list); Py_DECREF(joiner); @@ -1622,6 +1622,7 @@ _sre_template_impl(PyObject *module, PyObject *pattern, PyObject *template) } self->items[i].literal = Py_XNewRef(literal); } + PyObject_GC_Track(self); return (PyObject*) self; bad_template: @@ -2216,6 +2217,8 @@ match_getindex(MatchObject* self, PyObject* index) return -1; } + // Check that i*2 cannot overflow to make static analyzers happy + assert(i <= SRE_MAXGROUPS); return i; } @@ -2368,7 +2371,7 @@ _sre_SRE_Match_groupdict_impl(MatchObject *self, PyObject *default_value) goto exit; } } -exit: +exit:; Py_END_CRITICAL_SECTION(); return result; @@ -2915,7 +2918,7 @@ expand_template(TemplateObject *self, MatchObject *match) } else { Py_SET_SIZE(list, count); - result = _PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), list); + result = PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), list); } cleanup: @@ -2941,7 +2944,7 @@ pattern_hash(PatternObject *self) return -1; } - hash2 = _Py_HashBytes(self->code, sizeof(self->code[0]) * self->codesize); + hash2 = Py_HashBuffer(self->code, sizeof(self->code[0]) * self->codesize); hash ^= hash2; hash ^= self->flags; @@ -3272,6 +3275,7 @@ sre_exec(PyObject *m) static PyModuleDef_Slot sre_slots[] = { {Py_mod_exec, sre_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index bc302909424227..54bac28e5beccf 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -28,7 +28,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "pycore_time.h" // _PyDeadline_Init() /* Include symbols from _socket module */ #include "socketmodule.h" @@ -51,7 +51,6 @@ PySSL_BEGIN_ALLOW_THREADS_S(_save); #define PySSL_END_ALLOW_THREADS PySSL_END_ALLOW_THREADS_S(_save); } - #if defined(HAVE_POLL_H) #include #elif defined(HAVE_SYS_POLL_H) @@ -187,6 +186,11 @@ extern const SSL_METHOD *TLSv1_2_method(void); #endif +#if defined(SSL_VERIFY_POST_HANDSHAKE) && defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + #define PySSL_HAVE_POST_HS_AUTH +#endif + + enum py_ssl_error { /* these mirror ssl.h */ PY_SSL_ERROR_NONE, @@ -231,7 +235,7 @@ enum py_proto_version { PY_PROTO_TLSv1 = TLS1_VERSION, PY_PROTO_TLSv1_1 = TLS1_1_VERSION, PY_PROTO_TLSv1_2 = TLS1_2_VERSION, -#ifdef TLS1_3_VERSION +#if defined(TLS1_3_VERSION) PY_PROTO_TLSv1_3 = TLS1_3_VERSION, #else PY_PROTO_TLSv1_3 = 0x304, @@ -293,7 +297,7 @@ typedef struct { */ unsigned int hostflags; int protocol; -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) int post_handshake_auth; #endif PyObject *msg_cb; @@ -369,10 +373,7 @@ class _ssl.SSLSession "PySSLSession *" "get_state_type(type)->PySSLSession_Type" #include "clinic/_ssl.c.h" -static int PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout); - -static int PySSL_set_owner(PySSLSocket *, PyObject *, void *); -static int PySSL_set_session(PySSLSocket *, PyObject *, void *); +static int PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout); typedef enum { SOCKET_IS_NONBLOCKING, @@ -391,8 +392,8 @@ typedef enum { // Return a borrowed reference. static inline PySocketSockObject* GET_SOCKET(PySSLSocket *obj) { if (obj->Socket) { - PyObject *sock = _PyWeakref_GET_REF(obj->Socket); - if (sock != NULL) { + PyObject *sock; + if (PyWeakref_GetRef(obj->Socket, &sock)) { // GET_SOCKET() returns a borrowed reference Py_DECREF(sock); } @@ -598,7 +599,7 @@ PySSL_ChainExceptions(PySSLSocket *sslsock) { } static PyObject * -PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) +PySSL_SetError(PySSLSocket *sslsock, const char *filename, int lineno) { PyObject *type; char *errstr = NULL; @@ -611,7 +612,7 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) _sslmodulestate *state = get_state_sock(sslsock); type = state->PySSLErrorObject; - assert(ret <= 0); + // ERR functions are thread local, no need to lock them. e = ERR_peek_last_error(); if (sslsock->ssl != NULL) { @@ -644,32 +645,21 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) case SSL_ERROR_SYSCALL: { if (e == 0) { - PySocketSockObject *s = GET_SOCKET(sslsock); - if (ret == 0 || (((PyObject *)s) == Py_None)) { + /* underlying BIO reported an I/O error */ + ERR_clear_error(); +#ifdef MS_WINDOWS + if (err.ws) { + return PyErr_SetFromWindowsErr(err.ws); + } +#endif + if (err.c) { + errno = err.c; + return PyErr_SetFromErrno(PyExc_OSError); + } + else { p = PY_SSL_ERROR_EOF; type = state->PySSLEOFErrorObject; errstr = "EOF occurred in violation of protocol"; - } else if (s && ret == -1) { - /* underlying BIO reported an I/O error */ - ERR_clear_error(); -#ifdef MS_WINDOWS - if (err.ws) { - return PyErr_SetFromWindowsErr(err.ws); - } -#endif - if (err.c) { - errno = err.c; - return PyErr_SetFromErrno(PyExc_OSError); - } - else { - p = PY_SSL_ERROR_EOF; - type = state->PySSLEOFErrorObject; - errstr = "EOF occurred in violation of protocol"; - } - } else { /* possible? */ - p = PY_SSL_ERROR_SYSCALL; - type = state->PySSLSyscallErrorObject; - errstr = "Some I/O error occurred"; } } else { if (ERR_GET_LIB(e) == ERR_LIB_SSL && @@ -885,7 +875,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, SSL_set_mode(self->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_AUTO_RETRY); -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) if (sslctx->post_handshake_auth == 1) { if (socket_type == PY_SSL_SERVER) { /* bpo-37428: OpenSSL does not ignore SSL_VERIFY_POST_HANDSHAKE. @@ -933,13 +923,13 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, } } if (owner && owner != Py_None) { - if (PySSL_set_owner(self, owner, NULL) == -1) { + if (_ssl__SSLSocket_owner_set(self, owner, NULL) == -1) { Py_DECREF(self); return NULL; } } if (session && session != Py_None) { - if (PySSL_set_session(self, session, NULL) == -1) { + if (_ssl__SSLSocket_session_set(self, session, NULL) == -1) { Py_DECREF(self); return NULL; } @@ -952,18 +942,19 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, /* SSL object methods */ /*[clinic input] +@critical_section _ssl._SSLSocket.do_handshake [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) -/*[clinic end generated code: output=6c0898a8936548f6 input=d2d737de3df018c8]*/ +/*[clinic end generated code: output=6c0898a8936548f6 input=65619a7a4bea3176]*/ { int ret; _PySSLError err; int sockstate, nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock) { @@ -1028,8 +1019,9 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) } while (err.ssl == SSL_ERROR_WANT_READ || err.ssl == SSL_ERROR_WANT_WRITE); Py_XDECREF(sock); + if (ret < 1) - return PySSL_SetError(self, ret, __FILE__, __LINE__); + return PySSL_SetError(self, __FILE__, __LINE__); if (PySSL_ChainExceptions(self) < 0) return NULL; Py_RETURN_NONE; @@ -1826,6 +1818,7 @@ _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) /*[clinic input] +@critical_section _ssl._SSLSocket.getpeercert der as binary_mode: bool = False / @@ -1843,7 +1836,7 @@ return the certificate even if it wasn't validated. static PyObject * _ssl__SSLSocket_getpeercert_impl(PySSLSocket *self, int binary_mode) -/*[clinic end generated code: output=1f0ab66dfb693c88 input=c0fbe802e57629b7]*/ +/*[clinic end generated code: output=1f0ab66dfb693c88 input=e35af55fa5f9bab8]*/ { int verification; X509 *peer_cert; @@ -1873,13 +1866,14 @@ _ssl__SSLSocket_getpeercert_impl(PySSLSocket *self, int binary_mode) } /*[clinic input] +@critical_section _ssl._SSLSocket.get_verified_chain [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_get_verified_chain_impl(PySSLSocket *self) -/*[clinic end generated code: output=802421163cdc3110 input=5fb0714f77e2bd51]*/ +/*[clinic end generated code: output=802421163cdc3110 input=83035fe238ec057b]*/ { /* borrowed reference */ STACK_OF(X509) *chain = SSL_get0_verified_chain(self->ssl); @@ -1890,13 +1884,14 @@ _ssl__SSLSocket_get_verified_chain_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.get_unverified_chain [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_get_unverified_chain_impl(PySSLSocket *self) -/*[clinic end generated code: output=5acdae414e13f913 input=78c33c360c635cb5]*/ +/*[clinic end generated code: output=5acdae414e13f913 input=079f8ff5c205cb3b]*/ { PyObject *retval; /* borrowed reference */ @@ -2026,12 +2021,13 @@ cipher_to_dict(const SSL_CIPHER *cipher) } /*[clinic input] +@critical_section _ssl._SSLSocket.shared_ciphers [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self) -/*[clinic end generated code: output=3d174ead2e42c4fd input=0bfe149da8fe6306]*/ +/*[clinic end generated code: output=3d174ead2e42c4fd input=869645271e3bc6d0]*/ { STACK_OF(SSL_CIPHER) *server_ciphers; STACK_OF(SSL_CIPHER) *client_ciphers; @@ -2075,12 +2071,13 @@ _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.cipher [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_cipher_impl(PySSLSocket *self) -/*[clinic end generated code: output=376417c16d0e5815 input=548fb0e27243796d]*/ +/*[clinic end generated code: output=376417c16d0e5815 input=39e180269a36f486]*/ { const SSL_CIPHER *current; @@ -2093,12 +2090,13 @@ _ssl__SSLSocket_cipher_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.version [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_version_impl(PySSLSocket *self) -/*[clinic end generated code: output=178aed33193b2cdb input=900186a503436fd6]*/ +/*[clinic end generated code: output=178aed33193b2cdb input=2732bc3f7f597d09]*/ { const char *version; @@ -2115,12 +2113,13 @@ _ssl__SSLSocket_version_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.selected_alpn_protocol [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_selected_alpn_protocol_impl(PySSLSocket *self) -/*[clinic end generated code: output=ec33688b303d250f input=442de30e35bc2913]*/ +/*[clinic end generated code: output=ec33688b303d250f input=f0b53506c9acdf8c]*/ { const unsigned char *out; unsigned int outlen; @@ -2158,13 +2157,29 @@ _ssl__SSLSocket_compression_impl(PySSLSocket *self) #endif } -static PySSLContext *PySSL_get_context(PySSLSocket *self, void *closure) { - return (PySSLContext*)Py_NewRef(self->ctx); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.context +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_context_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=d23e82f72f32e3d7 input=25aa82e4d9fa344a]*/ +{ + return Py_NewRef(self->ctx); } -static int PySSL_set_context(PySSLSocket *self, PyObject *value, - void *closure) { +/*[clinic input] +@critical_section +@setter +_ssl._SSLSocket.context +[clinic start generated code]*/ +static int +_ssl__SSLSocket_context_set_impl(PySSLSocket *self, PyObject *value) +/*[clinic end generated code: output=6b0a6cc5cf33d9fe input=48ece77724fd9dd4]*/ +{ if (PyObject_TypeCheck(value, self->ctx->state->PySSLContext_Type)) { Py_SETREF(self->ctx, (PySSLContext *)Py_NewRef(value)); SSL_set_SSL_CTX(self->ssl, self->ctx->ctx); @@ -2181,50 +2196,67 @@ static int PySSL_set_context(PySSLSocket *self, PyObject *value, return 0; } -PyDoc_STRVAR(PySSL_set_context_doc, -"_setter_context(ctx)\n\ -\ -This changes the context associated with the SSLSocket. This is typically\n\ -used from within a callback function set by the sni_callback\n\ -on the SSLContext to change the certificate information associated with the\n\ -SSLSocket before the cryptographic exchange handshake messages\n"); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.server_side +Whether this is a server-side socket. +[clinic start generated code]*/ static PyObject * -PySSL_get_server_side(PySSLSocket *self, void *c) +_ssl__SSLSocket_server_side_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=ae51e372489148e3 input=b09b320510bc7cae]*/ { return PyBool_FromLong(self->socket_type == PY_SSL_SERVER); } -PyDoc_STRVAR(PySSL_get_server_side_doc, -"Whether this is a server-side socket."); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.server_hostname + +The currently set server hostname (for SNI). +[clinic start generated code]*/ static PyObject * -PySSL_get_server_hostname(PySSLSocket *self, void *c) +_ssl__SSLSocket_server_hostname_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=1f40ea5a076de8e7 input=55d12a1dc6634b08]*/ { if (self->server_hostname == NULL) Py_RETURN_NONE; return Py_NewRef(self->server_hostname); } -PyDoc_STRVAR(PySSL_get_server_hostname_doc, -"The currently set server hostname (for SNI)."); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.owner +[clinic start generated code]*/ static PyObject * -PySSL_get_owner(PySSLSocket *self, void *c) +_ssl__SSLSocket_owner_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=1f278cb930382927 input=bc2861ff3cf1402e]*/ { if (self->owner == NULL) { Py_RETURN_NONE; } - PyObject *owner = _PyWeakref_GET_REF(self->owner); - if (owner == NULL) { + PyObject *owner; + if (!PyWeakref_GetRef(self->owner, &owner)) { Py_RETURN_NONE; } return owner; } +/*[clinic input] +@critical_section +@setter +_ssl._SSLSocket.owner +[clinic start generated code]*/ + static int -PySSL_set_owner(PySSLSocket *self, PyObject *value, void *c) +_ssl__SSLSocket_owner_set_impl(PySSLSocket *self, PyObject *value) +/*[clinic end generated code: output=2e3924498f2b6cde input=875666fd32367a73]*/ { Py_XSETREF(self->owner, PyWeakref_NewRef(value, NULL)); if (self->owner == NULL) @@ -2232,10 +2264,6 @@ PySSL_set_owner(PySSLSocket *self, PyObject *value, void *c) return 0; } -PyDoc_STRVAR(PySSL_get_owner_doc, -"The Python-level owner of this object.\ -Passed as \"self\" in servername callback."); - static int PySSL_traverse(PySSLSocket *self, visitproc visit, void *arg) { @@ -2257,6 +2285,17 @@ PySSL_dealloc(PySSLSocket *self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); if (self->ssl) { + // If we free the SSL socket object without having called SSL_shutdown, + // OpenSSL will invalidate the linked SSL session object. While this + // behavior is strictly RFC-compliant, it makes session reuse less + // likely and it would also break compatibility with older stdlib + // versions (which used an ugly workaround of duplicating the + // SSL_SESSION object). + // Therefore, we ensure the socket is marked as shutdown in any case. + // + // See elaborate explanation at + // https://github.com/python/cpython/pull/123249#discussion_r1766164530 + SSL_set_shutdown(self->ssl, SSL_SENT_SHUTDOWN | SSL_get_shutdown(self->ssl)); SSL_free(self->ssl); } Py_XDECREF(self->Socket); @@ -2273,12 +2312,12 @@ PySSL_dealloc(PySSLSocket *self) */ static int -PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout) +PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) { int rc; #ifdef HAVE_POLL struct pollfd pollfd; - _PyTime_t ms; + PyTime_t ms; #else int nfds; fd_set fds; @@ -2338,6 +2377,7 @@ PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout) } /*[clinic input] +@critical_section _ssl._SSLSocket.write b: Py_buffer / @@ -2349,7 +2389,7 @@ Returns the number of bytes written. static PyObject * _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) -/*[clinic end generated code: output=aa7a6be5527358d8 input=77262d994fe5100a]*/ +/*[clinic end generated code: output=aa7a6be5527358d8 input=967b5feeae641a26]*/ { size_t count = 0; int retval; @@ -2357,7 +2397,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) _PySSLError err; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock != NULL) { @@ -2436,7 +2476,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) Py_XDECREF(sock); if (retval == 0) - return PySSL_SetError(self, retval, __FILE__, __LINE__); + return PySSL_SetError(self, __FILE__, __LINE__); if (PySSL_ChainExceptions(self) < 0) return NULL; return PyLong_FromSize_t(count); @@ -2447,6 +2487,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) } /*[clinic input] +@critical_section _ssl._SSLSocket.pending Returns the number of already decrypted bytes available for read, pending on the connection. @@ -2454,7 +2495,7 @@ Returns the number of already decrypted bytes available for read, pending on the static PyObject * _ssl__SSLSocket_pending_impl(PySSLSocket *self) -/*[clinic end generated code: output=983d9fecdc308a83 input=2b77487d6dfd597f]*/ +/*[clinic end generated code: output=983d9fecdc308a83 input=32ab982a254e8866]*/ { int count = 0; _PySSLError err; @@ -2466,12 +2507,13 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) self->err = err; if (count < 0) - return PySSL_SetError(self, count, __FILE__, __LINE__); + return PySSL_SetError(self, __FILE__, __LINE__); else return PyLong_FromLong(count); } /*[clinic input] +@critical_section _ssl._SSLSocket.read size as len: Py_ssize_t [ @@ -2485,7 +2527,7 @@ Read up to size bytes from the SSL socket. static PyObject * _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, int group_right_1, Py_buffer *buffer) -/*[clinic end generated code: output=49b16e6406023734 input=ec48bf622be1c4a1]*/ +/*[clinic end generated code: output=49b16e6406023734 input=80ed30436df01a71]*/ { PyObject *dest = NULL; char *mem; @@ -2495,7 +2537,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, _PySSLError err; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (!group_right_1 && len < 0) { @@ -2589,7 +2631,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, err.ssl == SSL_ERROR_WANT_WRITE); if (retval == 0) { - PySSL_SetError(self, retval, __FILE__, __LINE__); + PySSL_SetError(self, __FILE__, __LINE__); goto error; } if (self->exc != NULL) @@ -2614,6 +2656,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, } /*[clinic input] +@critical_section _ssl._SSLSocket.shutdown Does the SSL shutdown handshake with the remote end. @@ -2621,13 +2664,13 @@ Does the SSL shutdown handshake with the remote end. static PyObject * _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) -/*[clinic end generated code: output=ca1aa7ed9d25ca42 input=11d39e69b0a2bf4a]*/ +/*[clinic end generated code: output=ca1aa7ed9d25ca42 input=98d9635cd4e16514]*/ { _PySSLError err; int sockstate, nonblocking, ret; int zeros = 0; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock != NULL) { @@ -2715,7 +2758,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } if (ret < 0) { Py_XDECREF(sock); - PySSL_SetError(self, ret, __FILE__, __LINE__); + PySSL_SetError(self, __FILE__, __LINE__); return NULL; } if (self->exc != NULL) @@ -2733,6 +2776,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.get_channel_binding cb_type: str = "tls-unique" @@ -2746,7 +2790,7 @@ Only 'tls-unique' channel binding data from RFC 5929 is supported. static PyObject * _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, const char *cb_type) -/*[clinic end generated code: output=34bac9acb6a61d31 input=08b7e43b99c17d41]*/ +/*[clinic end generated code: output=34bac9acb6a61d31 input=e008004fc08744db]*/ { char buf[PySSL_CB_MAXLEN]; size_t len; @@ -2778,6 +2822,7 @@ _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, } /*[clinic input] +@critical_section _ssl._SSLSocket.verify_client_post_handshake Initiate TLS 1.3 post-handshake authentication @@ -2785,9 +2830,9 @@ Initiate TLS 1.3 post-handshake authentication static PyObject * _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) -/*[clinic end generated code: output=532147f3b1341425 input=6bfa874810a3d889]*/ +/*[clinic end generated code: output=532147f3b1341425 input=42b5bb1f0981eda1]*/ { -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) int err = SSL_verify_client_post_handshake(self->ssl); if (err == 0) return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); @@ -2801,70 +2846,29 @@ _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) #endif } -static SSL_SESSION* -_ssl_session_dup(SSL_SESSION *session) { - SSL_SESSION *newsession = NULL; - int slen; - unsigned char *senc = NULL, *p; - const unsigned char *const_p; - - if (session == NULL) { - PyErr_SetString(PyExc_ValueError, "Invalid session"); - goto error; - } - - /* get length */ - slen = i2d_SSL_SESSION(session, NULL); - if (slen == 0 || slen > 0xFF00) { - PyErr_SetString(PyExc_ValueError, "i2d() failed"); - goto error; - } - if ((senc = PyMem_Malloc(slen)) == NULL) { - PyErr_NoMemory(); - goto error; - } - p = senc; - if (!i2d_SSL_SESSION(session, &p)) { - PyErr_SetString(PyExc_ValueError, "i2d() failed"); - goto error; - } - const_p = senc; - newsession = d2i_SSL_SESSION(NULL, &const_p, slen); - if (newsession == NULL) { - PyErr_SetString(PyExc_ValueError, "d2i() failed"); - goto error; - } - PyMem_Free(senc); - return newsession; - error: - if (senc != NULL) { - PyMem_Free(senc); - } - return NULL; -} +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.session +[clinic start generated code]*/ static PyObject * -PySSL_get_session(PySSLSocket *self, void *closure) { +_ssl__SSLSocket_session_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=a5cd5755b35da670 input=b9792df9255a9f63]*/ +{ /* get_session can return sessions from a server-side connection, * it does not check for handshake done or client socket. */ PySSLSession *pysess; SSL_SESSION *session; - /* duplicate session as workaround for session bug in OpenSSL 1.1.0, - * https://github.com/openssl/openssl/issues/1550 */ - session = SSL_get0_session(self->ssl); /* borrowed reference */ - if (session == NULL) { - Py_RETURN_NONE; - } - if ((session = _ssl_session_dup(session)) == NULL) { - return NULL; - } session = SSL_get1_session(self->ssl); if (session == NULL) { Py_RETURN_NONE; } pysess = PyObject_GC_New(PySSLSession, self->ctx->state->PySSLSession_Type); if (pysess == NULL) { + // It's not possible for another thread to access this, so + // we don't need to lock it. SSL_SESSION_free(session); return NULL; } @@ -2876,12 +2880,17 @@ PySSL_get_session(PySSLSocket *self, void *closure) { return (PyObject *)pysess; } -static int PySSL_set_session(PySSLSocket *self, PyObject *value, - void *closure) - { +/*[clinic input] +@critical_section +@setter +_ssl._SSLSocket.session +[clinic start generated code]*/ + +static int +_ssl__SSLSocket_session_set_impl(PySSLSocket *self, PyObject *value) +/*[clinic end generated code: output=a3fa2ddd7c2d54a2 input=5fa5f921640db98b]*/ +{ PySSLSession *pysess; - SSL_SESSION *session; - int result; if (!Py_IS_TYPE(value, get_state_sock(self)->PySSLSession_Type)) { PyErr_SetString(PyExc_TypeError, "Value is not a SSLSession."); @@ -2904,50 +2913,37 @@ static int PySSL_set_session(PySSLSocket *self, PyObject *value, "Cannot set session after handshake."); return -1; } - /* duplicate session */ - if ((session = _ssl_session_dup(pysess->session)) == NULL) { - return -1; - } - result = SSL_set_session(self->ssl, session); - /* free duplicate, SSL_set_session() bumps ref count */ - SSL_SESSION_free(session); - if (result == 0) { + + if (SSL_set_session(self->ssl, pysess->session) == 0) { _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); return -1; } return 0; } -PyDoc_STRVAR(PySSL_set_session_doc, -"_setter_session(session)\n\ -\ -Get / set SSLSession."); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.session_reused + +Was the client session reused during handshake? +[clinic start generated code]*/ static PyObject * -PySSL_get_session_reused(PySSLSocket *self, void *closure) { - if (SSL_session_reused(self->ssl)) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } +_ssl__SSLSocket_session_reused_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=c8916909bcb80893 input=cec8bfec73a4461e]*/ +{ + int res = SSL_session_reused(self->ssl); + return res ? Py_True : Py_False; } -PyDoc_STRVAR(PySSL_get_session_reused_doc, -"Was the client session reused during handshake?"); - static PyGetSetDef ssl_getsetlist[] = { - {"context", (getter) PySSL_get_context, - (setter) PySSL_set_context, PySSL_set_context_doc}, - {"server_side", (getter) PySSL_get_server_side, NULL, - PySSL_get_server_side_doc}, - {"server_hostname", (getter) PySSL_get_server_hostname, NULL, - PySSL_get_server_hostname_doc}, - {"owner", (getter) PySSL_get_owner, (setter) PySSL_set_owner, - PySSL_get_owner_doc}, - {"session", (getter) PySSL_get_session, - (setter) PySSL_set_session, PySSL_set_session_doc}, - {"session_reused", (getter) PySSL_get_session_reused, NULL, - PySSL_get_session_reused_doc}, + _SSL__SSLSOCKET_CONTEXT_GETSETDEF + _SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF + _SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF + _SSL__SSLSOCKET_OWNER_GETSETDEF + _SSL__SSLSOCKET_SESSION_GETSETDEF + _SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF {NULL}, /* sentinel */ }; @@ -3020,6 +3016,7 @@ _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) } /*[clinic input] +@critical_section @classmethod _ssl._SSLContext.__new__ protocol as proto_version: int @@ -3028,7 +3025,7 @@ _ssl._SSLContext.__new__ static PyObject * _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) -/*[clinic end generated code: output=2cf0d7a0741b6bd1 input=8d58a805b95fc534]*/ +/*[clinic end generated code: output=2cf0d7a0741b6bd1 input=6fc79e62ae9d143c]*/ { PySSLContext *self; uint64_t options; @@ -3098,6 +3095,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) return NULL; } + // This is the constructor, no need to lock it, because + // no other thread can be touching this object yet. + // (Technically, we can't even lock if we wanted to, as the + // lock hasn't been initialized yet.) PySSL_BEGIN_ALLOW_THREADS ctx = SSL_CTX_new(method); PySSL_END_ALLOW_THREADS @@ -3177,7 +3178,6 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) result = SSL_CTX_set_cipher_list(ctx, "HIGH:!aNULL:!eNULL"); } if (result == 0) { - Py_DECREF(self); ERR_clear_error(); PyErr_SetString(get_state_ctx(self)->PySSLErrorObject, "No cipher can be selected."); @@ -3211,7 +3211,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) self->post_handshake_auth = 0; SSL_CTX_set_post_handshake_auth(self->ctx, self->post_handshake_auth); #endif @@ -3265,6 +3265,7 @@ context_dealloc(PySSLContext *self) } /*[clinic input] +@critical_section _ssl._SSLContext.set_ciphers cipherlist: str / @@ -3272,7 +3273,7 @@ _ssl._SSLContext.set_ciphers static PyObject * _ssl__SSLContext_set_ciphers_impl(PySSLContext *self, const char *cipherlist) -/*[clinic end generated code: output=3a3162f3557c0f3f input=a7ac931b9f3ca7fc]*/ +/*[clinic end generated code: output=3a3162f3557c0f3f input=40b583cded5c6ff9]*/ { int ret = SSL_CTX_set_cipher_list(self->ctx, cipherlist); if (ret == 0) { @@ -3288,12 +3289,13 @@ _ssl__SSLContext_set_ciphers_impl(PySSLContext *self, const char *cipherlist) } /*[clinic input] +@critical_section _ssl._SSLContext.get_ciphers [clinic start generated code]*/ static PyObject * _ssl__SSLContext_get_ciphers_impl(PySSLContext *self) -/*[clinic end generated code: output=a56e4d68a406dfc4 input=a2aadc9af89b79c5]*/ +/*[clinic end generated code: output=a56e4d68a406dfc4 input=d7fff51631a260ae]*/ { SSL *ssl = NULL; STACK_OF(SSL_CIPHER) *sk = NULL; @@ -3368,6 +3370,7 @@ _selectALPN_cb(SSL *s, } /*[clinic input] +@critical_section _ssl._SSLContext._set_alpn_protocols protos: Py_buffer / @@ -3376,7 +3379,7 @@ _ssl._SSLContext._set_alpn_protocols static PyObject * _ssl__SSLContext__set_alpn_protocols_impl(PySSLContext *self, Py_buffer *protos) -/*[clinic end generated code: output=87599a7f76651a9b input=9bba964595d519be]*/ +/*[clinic end generated code: output=87599a7f76651a9b input=b5096b186e49287d]*/ { if ((size_t)protos->len > UINT_MAX) { PyErr_Format(PyExc_OverflowError, @@ -3386,25 +3389,34 @@ _ssl__SSLContext__set_alpn_protocols_impl(PySSLContext *self, PyMem_Free(self->alpn_protocols); self->alpn_protocols = PyMem_Malloc(protos->len); - if (!self->alpn_protocols) + if (!self->alpn_protocols) { return PyErr_NoMemory(); + } memcpy(self->alpn_protocols, protos->buf, protos->len); self->alpn_protocols_len = (unsigned int)protos->len; - - if (SSL_CTX_set_alpn_protos(self->ctx, self->alpn_protocols, self->alpn_protocols_len)) + if (SSL_CTX_set_alpn_protos(self->ctx, self->alpn_protocols, self->alpn_protocols_len)) { return PyErr_NoMemory(); + } SSL_CTX_set_alpn_select_cb(self->ctx, _selectALPN_cb, self); Py_RETURN_NONE; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.verify_mode +[clinic start generated code]*/ + static PyObject * -get_verify_mode(PySSLContext *self, void *c) +_ssl__SSLContext_verify_mode_get_impl(PySSLContext *self) +/*[clinic end generated code: output=3e788736cc7229bc input=7e3c7f4454121d0a]*/ { /* ignore SSL_VERIFY_CLIENT_ONCE and SSL_VERIFY_POST_HANDSHAKE */ int mask = (SSL_VERIFY_NONE | SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT); - switch (SSL_CTX_get_verify_mode(self->ctx) & mask) { + int verify_mode = SSL_CTX_get_verify_mode(self->ctx); + switch (verify_mode & mask) { case SSL_VERIFY_NONE: return PyLong_FromLong(PY_SSL_CERT_NONE); case SSL_VERIFY_PEER: @@ -3417,11 +3429,18 @@ get_verify_mode(PySSLContext *self, void *c) return NULL; } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.verify_mode +[clinic start generated code]*/ + static int -set_verify_mode(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_verify_mode_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=d698e16c58db3118 input=3ee60057c3a22378]*/ { int n; - if (!PyArg_Parse(arg, "i", &n)) + if (!PyArg_Parse(value, "i", &n)) return -1; if (n == PY_SSL_CERT_NONE && self->check_hostname) { PyErr_SetString(PyExc_ValueError, @@ -3432,8 +3451,15 @@ set_verify_mode(PySSLContext *self, PyObject *arg, void *c) return _set_verify_mode(self, n); } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.verify_flags +[clinic start generated code]*/ + static PyObject * -get_verify_flags(PySSLContext *self, void *c) +_ssl__SSLContext_verify_flags_get_impl(PySSLContext *self) +/*[clinic end generated code: output=fbbf8ba28ad6e56e input=c1ec36d610b3f391]*/ { X509_VERIFY_PARAM *param; unsigned long flags; @@ -3443,13 +3469,20 @@ get_verify_flags(PySSLContext *self, void *c) return PyLong_FromUnsignedLong(flags); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.verify_flags +[clinic start generated code]*/ + static int -set_verify_flags(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_verify_flags_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=a3e3b2a0ce6c2e99 input=b2a0c42583d4f34e]*/ { X509_VERIFY_PARAM *param; unsigned long new_flags, flags, set, clear; - if (!PyArg_Parse(arg, "k", &new_flags)) + if (!PyArg_Parse(value, "k", &new_flags)) return -1; param = SSL_CTX_get0_param(self->ctx); flags = X509_VERIFY_PARAM_get_flags(param); @@ -3485,8 +3518,8 @@ set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) } switch(self->protocol) { - case PY_SSL_VERSION_TLS_CLIENT: /* fall through */ - case PY_SSL_VERSION_TLS_SERVER: /* fall through */ + case PY_SSL_VERSION_TLS_CLIENT: _Py_FALLTHROUGH; + case PY_SSL_VERSION_TLS_SERVER: _Py_FALLTHROUGH; case PY_SSL_VERSION_TLS: break; default: @@ -3557,8 +3590,15 @@ set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) return 0; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.minimum_version +[clinic start generated code]*/ + static PyObject * -get_minimum_version(PySSLContext *self, void *c) +_ssl__SSLContext_minimum_version_get_impl(PySSLContext *self) +/*[clinic end generated code: output=27fa8382276635ed input=6832821e7e974d40]*/ { int v = SSL_CTX_get_min_proto_version(self->ctx); if (v == 0) { @@ -3567,14 +3607,29 @@ get_minimum_version(PySSLContext *self, void *c) return PyLong_FromLong(v); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.minimum_version +[clinic start generated code]*/ + static int -set_minimum_version(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_minimum_version_set_impl(PySSLContext *self, + PyObject *value) +/*[clinic end generated code: output=482e82f7372afb78 input=2c64724901a514b3]*/ { - return set_min_max_proto_version(self, arg, 0); + return set_min_max_proto_version(self, value, 0); } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.maximum_version +[clinic start generated code]*/ + static PyObject * -get_maximum_version(PySSLContext *self, void *c) +_ssl__SSLContext_maximum_version_get_impl(PySSLContext *self) +/*[clinic end generated code: output=889249475112826a input=2b9e4c2d45f16b14]*/ { int v = SSL_CTX_get_max_proto_version(self->ctx); if (v == 0) { @@ -3583,24 +3638,51 @@ get_maximum_version(PySSLContext *self, void *c) return PyLong_FromLong(v); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.maximum_version +[clinic start generated code]*/ + static int -set_maximum_version(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_maximum_version_set_impl(PySSLContext *self, + PyObject *value) +/*[clinic end generated code: output=4c0eed3042ca20d5 input=fe27e9fbbeb73c89]*/ { - return set_min_max_proto_version(self, arg, 1); + return set_min_max_proto_version(self, value, 1); } -#ifdef TLS1_3_VERSION +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.num_tickets +[clinic start generated code]*/ + static PyObject * -get_num_tickets(PySSLContext *self, void *c) +_ssl__SSLContext_num_tickets_get_impl(PySSLContext *self) +/*[clinic end generated code: output=3d06d016318846c9 input=1dee26d75163c073]*/ { - return PyLong_FromSize_t(SSL_CTX_get_num_tickets(self->ctx)); + // Clinic seems to be misbehaving when the comment is wrapped with in directive +#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + PyObject *res = PyLong_FromSize_t(SSL_CTX_get_num_tickets(self->ctx)); + return res; +#else + return 0; +#endif } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.num_tickets +[clinic start generated code]*/ + static int -set_num_tickets(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_num_tickets_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=ced81b46f3beab09 input=6ef8067ac55607e7]*/ { long num; - if (!PyArg_Parse(arg, "l", &num)) + if (!PyArg_Parse(value, "l", &num)) return -1; if (num < 0) { PyErr_SetString(PyExc_ValueError, "value must be non-negative"); @@ -3618,27 +3700,44 @@ set_num_tickets(PySSLContext *self, PyObject *arg, void *c) return 0; } -PyDoc_STRVAR(PySSLContext_num_tickets_doc, -"Control the number of TLSv1.3 session tickets"); -#endif /* TLS1_3_VERSION */ +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.security_level +[clinic start generated code]*/ static PyObject * -get_security_level(PySSLContext *self, void *c) +_ssl__SSLContext_security_level_get_impl(PySSLContext *self) +/*[clinic end generated code: output=56ece09e6a9572d0 input=a0416598e07c3183]*/ { - return PyLong_FromLong(SSL_CTX_get_security_level(self->ctx)); + PyObject *res = PyLong_FromLong(SSL_CTX_get_security_level(self->ctx)); + return res; } -PyDoc_STRVAR(PySSLContext_security_level_doc, "The current security level"); + +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.options +[clinic start generated code]*/ static PyObject * -get_options(PySSLContext *self, void *c) +_ssl__SSLContext_options_get_impl(PySSLContext *self) +/*[clinic end generated code: output=3dfa6a74837f525b input=f5a2805c7cda6f25]*/ { uint64_t options = SSL_CTX_get_options(self->ctx); Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(options)); return PyLong_FromUnsignedLongLong(options); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.options +[clinic start generated code]*/ + static int -set_options(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_options_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=92ca34731ece5dbb input=2b94bf789e9ae5dd]*/ { PyObject *new_opts_obj; unsigned long long new_opts_arg; @@ -3648,7 +3747,7 @@ set_options(PySSLContext *self, PyObject *arg, void *c) SSL_OP_NO_TLSv1_1 | SSL_OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3 ); - if (!PyArg_Parse(arg, "O!", &PyLong_Type, &new_opts_obj)) { + if (!PyArg_Parse(value, "O!", &PyLong_Type, &new_opts_obj)) { return -1; } new_opts_arg = PyLong_AsUnsignedLongLong(new_opts_obj); @@ -3677,19 +3776,33 @@ set_options(PySSLContext *self, PyObject *arg, void *c) return 0; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext._host_flags +[clinic start generated code]*/ + static PyObject * -get_host_flags(PySSLContext *self, void *c) +_ssl__SSLContext__host_flags_get_impl(PySSLContext *self) +/*[clinic end generated code: output=0f9db6654ce32582 input=8e3c49499eefd0e5]*/ { return PyLong_FromUnsignedLong(self->hostflags); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext._host_flags +[clinic start generated code]*/ + static int -set_host_flags(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext__host_flags_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=1ed6f4027aaf2e3e input=28caf1fb9c32f6cb]*/ { X509_VERIFY_PARAM *param; unsigned int new_flags = 0; - if (!PyArg_Parse(arg, "I", &new_flags)) + if (!PyArg_Parse(value, "I", &new_flags)) return -1; param = SSL_CTX_get0_param(self->ctx); @@ -3698,20 +3811,35 @@ set_host_flags(PySSLContext *self, PyObject *arg, void *c) return 0; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.check_hostname +[clinic start generated code]*/ + static PyObject * -get_check_hostname(PySSLContext *self, void *c) +_ssl__SSLContext_check_hostname_get_impl(PySSLContext *self) +/*[clinic end generated code: output=e046d6eeefc76063 input=1b8341e705f9ecf5]*/ { return PyBool_FromLong(self->check_hostname); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.check_hostname +[clinic start generated code]*/ + static int -set_check_hostname(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_check_hostname_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=0e767b4784e7dc3f input=e6a771cb5919f74d]*/ { int check_hostname; - if (!PyArg_Parse(arg, "p", &check_hostname)) + if (!PyArg_Parse(value, "p", &check_hostname)) return -1; + int verify_mode = check_hostname ? SSL_CTX_get_verify_mode(self->ctx) : 0; if (check_hostname && - SSL_CTX_get_verify_mode(self->ctx) == SSL_VERIFY_NONE) { + verify_mode == SSL_VERIFY_NONE) { /* check_hostname = True sets verify_mode = CERT_REQUIRED */ if (_set_verify_mode(self, PY_SSL_CERT_REQUIRED) == -1) { return -1; @@ -3723,14 +3851,14 @@ set_check_hostname(PySSLContext *self, PyObject *arg, void *c) static PyObject * get_post_handshake_auth(PySSLContext *self, void *c) { -#if TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) return PyBool_FromLong(self->post_handshake_auth); #else Py_RETURN_NONE; #endif } -#if TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) static int set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { if (arg == NULL) { @@ -3751,8 +3879,16 @@ set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { } #endif +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.protocol +[clinic start generated code]*/ + static PyObject * -get_protocol(PySSLContext *self, void *c) { +_ssl__SSLContext_protocol_get_impl(PySSLContext *self) +/*[clinic end generated code: output=a9a48febc16cee22 input=c9f5fa1a2bd4b8a8]*/ +{ return PyLong_FromLong(self->protocol); } @@ -3866,6 +4002,7 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) } /*[clinic input] +@critical_section _ssl._SSLContext.load_cert_chain certfile: object keyfile: object = None @@ -3876,7 +4013,7 @@ _ssl._SSLContext.load_cert_chain static PyObject * _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, PyObject *keyfile, PyObject *password) -/*[clinic end generated code: output=9480bc1c380e2095 input=30bc7e967ea01a58]*/ +/*[clinic end generated code: output=9480bc1c380e2095 input=6c7c5e8b73e4264b]*/ { PyObject *certfile_bytes = NULL, *keyfile_bytes = NULL; pem_password_cb *orig_passwd_cb = SSL_CTX_get_default_passwd_cb(self->ctx); @@ -4070,6 +4207,7 @@ _add_ca_certs(PySSLContext *self, const void *data, Py_ssize_t len, /*[clinic input] +@critical_section _ssl._SSLContext.load_verify_locations cafile: object = None capath: object = None @@ -4082,7 +4220,7 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, PyObject *cafile, PyObject *capath, PyObject *cadata) -/*[clinic end generated code: output=454c7e41230ca551 input=42ecfe258233e194]*/ +/*[clinic end generated code: output=454c7e41230ca551 input=b178852b41618414]*/ { PyObject *cafile_bytes = NULL, *capath_bytes = NULL; const char *cafile_buf = NULL, *capath_buf = NULL; @@ -4198,6 +4336,7 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, } /*[clinic input] +@critical_section _ssl._SSLContext.load_dh_params path as filepath: object / @@ -4205,8 +4344,8 @@ _ssl._SSLContext.load_dh_params [clinic start generated code]*/ static PyObject * -_ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) -/*[clinic end generated code: output=1c8e57a38e055af0 input=c8871f3c796ae1d6]*/ +_ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath) +/*[clinic end generated code: output=dd74b3c524dd2723 input=832769a0734b8c4d]*/ { FILE *f; DH *dh; @@ -4239,6 +4378,7 @@ _ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) } /*[clinic input] +@critical_section _ssl._SSLContext._wrap_socket sock: object(subclass_of="get_state_ctx(self)->Sock_Type") server_side: bool @@ -4253,7 +4393,7 @@ static PyObject * _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, int server_side, PyObject *hostname_obj, PyObject *owner, PyObject *session) -/*[clinic end generated code: output=f103f238633940b4 input=700ca8fedff53994]*/ +/*[clinic end generated code: output=f103f238633940b4 input=eceadcee4434a06f]*/ { char *hostname = NULL; PyObject *res; @@ -4275,6 +4415,7 @@ _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, } /*[clinic input] +@critical_section _ssl._SSLContext._wrap_bio incoming: object(subclass_of="get_state_ctx(self)->PySSLMemoryBIO_Type", type="PySSLMemoryBIO *") outgoing: object(subclass_of="get_state_ctx(self)->PySSLMemoryBIO_Type", type="PySSLMemoryBIO *") @@ -4291,7 +4432,7 @@ _ssl__SSLContext__wrap_bio_impl(PySSLContext *self, PySSLMemoryBIO *incoming, PySSLMemoryBIO *outgoing, int server_side, PyObject *hostname_obj, PyObject *owner, PyObject *session) -/*[clinic end generated code: output=5c5d6d9b41f99332 input=a9205d097fd45a82]*/ +/*[clinic end generated code: output=5c5d6d9b41f99332 input=58277fc962a60182]*/ { char *hostname = NULL; PyObject *res; @@ -4312,12 +4453,13 @@ _ssl__SSLContext__wrap_bio_impl(PySSLContext *self, PySSLMemoryBIO *incoming, } /*[clinic input] +@critical_section _ssl._SSLContext.session_stats [clinic start generated code]*/ static PyObject * _ssl__SSLContext_session_stats_impl(PySSLContext *self) -/*[clinic end generated code: output=0d96411c42893bfb input=7e0a81fb11102c8b]*/ +/*[clinic end generated code: output=0d96411c42893bfb input=db62af53004127a4]*/ { int r; PyObject *value, *stats = PyDict_New(); @@ -4356,12 +4498,13 @@ _ssl__SSLContext_session_stats_impl(PySSLContext *self) } /*[clinic input] +@critical_section _ssl._SSLContext.set_default_verify_paths [clinic start generated code]*/ static PyObject * _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self) -/*[clinic end generated code: output=0bee74e6e09deaaa input=35f3408021463d74]*/ +/*[clinic end generated code: output=0bee74e6e09deaaa input=939a88e78f634119]*/ { int rc; Py_BEGIN_ALLOW_THREADS @@ -4375,6 +4518,7 @@ _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self) } /*[clinic input] +@critical_section _ssl._SSLContext.set_ecdh_curve name: object / @@ -4382,8 +4526,8 @@ _ssl._SSLContext.set_ecdh_curve [clinic start generated code]*/ static PyObject * -_ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) -/*[clinic end generated code: output=23022c196e40d7d2 input=c2bafb6f6e34726b]*/ +_ssl__SSLContext_set_ecdh_curve_impl(PySSLContext *self, PyObject *name) +/*[clinic end generated code: output=01081151ce0ecc45 input=039df032e666870e]*/ { PyObject *name_bytes; int nid; @@ -4445,9 +4589,9 @@ _servername_callback(SSL *s, int *al, void *args) * will be passed. If both do not exist only then the C-level object is * passed. */ if (ssl->owner) - ssl_socket = _PyWeakref_GET_REF(ssl->owner); + PyWeakref_GetRef(ssl->owner, &ssl_socket); else if (ssl->Socket) - ssl_socket = _PyWeakref_GET_REF(ssl->Socket); + PyWeakref_GetRef(ssl->Socket, &ssl_socket); else ssl_socket = Py_NewRef(ssl); @@ -4517,8 +4661,15 @@ _servername_callback(SSL *s, int *al, void *args) return ret; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.sni_callback +[clinic start generated code]*/ + static PyObject * -get_sni_callback(PySSLContext *self, void *c) +_ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) +/*[clinic end generated code: output=961e6575cdfaf036 input=22dd28c31fdc4318]*/ { PyObject *cb = self->set_sni_cb; if (cb == NULL) { @@ -4527,8 +4678,15 @@ get_sni_callback(PySSLContext *self, void *c) return Py_NewRef(cb); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.sni_callback +[clinic start generated code]*/ + static int -set_sni_callback(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_sni_callback_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=b32736c6b891f61a input=c3c4ff33540b3c85]*/ { if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { PyErr_SetString(PyExc_ValueError, @@ -4536,31 +4694,69 @@ set_sni_callback(PySSLContext *self, PyObject *arg, void *c) return -1; } Py_CLEAR(self->set_sni_cb); - if (arg == Py_None) { + if (value == Py_None) { SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); } else { - if (!PyCallable_Check(arg)) { + if (!PyCallable_Check(value)) { SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); PyErr_SetString(PyExc_TypeError, "not a callable object"); return -1; } - self->set_sni_cb = Py_NewRef(arg); + self->set_sni_cb = Py_NewRef(value); SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); SSL_CTX_set_tlsext_servername_arg(self->ctx, self); } return 0; } -PyDoc_STRVAR(PySSLContext_sni_callback_doc, -"Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n\ -\n\ -If the argument is None then the callback is disabled. The method is called\n\ -with the SSLSocket, the server name as a string, and the SSLContext object.\n\ -See RFC 6066 for details of the SNI extension."); +#if OPENSSL_VERSION_NUMBER < 0x30300000L +static X509_OBJECT *x509_object_dup(const X509_OBJECT *obj) +{ + int ok; + X509_OBJECT *ret = X509_OBJECT_new(); + if (ret == NULL) { + return NULL; + } + switch (X509_OBJECT_get_type(obj)) { + case X509_LU_X509: + ok = X509_OBJECT_set1_X509(ret, X509_OBJECT_get0_X509(obj)); + break; + case X509_LU_CRL: + /* X509_OBJECT_get0_X509_CRL was not const-correct prior to 3.0.*/ + ok = X509_OBJECT_set1_X509_CRL( + ret, X509_OBJECT_get0_X509_CRL((X509_OBJECT *)obj)); + break; + default: + /* We cannot duplicate unrecognized types in a polyfill, but it is + * safe to leave an empty object. The caller will ignore it. */ + ok = 1; + break; + } + if (!ok) { + X509_OBJECT_free(ret); + return NULL; + } + return ret; +} + +static STACK_OF(X509_OBJECT) * +X509_STORE_get1_objects(X509_STORE *store) +{ + STACK_OF(X509_OBJECT) *ret; + if (!X509_STORE_lock(store)) { + return NULL; + } + ret = sk_X509_OBJECT_deep_copy(X509_STORE_get0_objects(store), + x509_object_dup, X509_OBJECT_free); + X509_STORE_unlock(store); + return ret; +} +#endif /*[clinic input] +@critical_section _ssl._SSLContext.cert_store_stats Returns quantities of loaded X.509 certificates. @@ -4574,7 +4770,7 @@ been used at least once. static PyObject * _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) -/*[clinic end generated code: output=5f356f4d9cca874d input=eb40dd0f6d0e40cf]*/ +/*[clinic end generated code: output=5f356f4d9cca874d input=d13c6e3f2b48539b]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -4582,7 +4778,12 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) int x509 = 0, crl = 0, ca = 0, i; store = SSL_CTX_get_cert_store(self->ctx); - objs = X509_STORE_get0_objects(store); + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + return NULL; + } + for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { obj = sk_X509_OBJECT_value(objs, i); switch (X509_OBJECT_get_type(obj)) { @@ -4596,17 +4797,17 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) crl++; break; default: - /* Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY. - * As far as I can tell they are internal states and never - * stored in a cert store */ + /* Ignore unrecognized types. */ break; } } + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); return Py_BuildValue("{sisisi}", "x509", x509, "crl", crl, "x509_ca", ca); } /*[clinic input] +@critical_section _ssl._SSLContext.get_ca_certs binary_form: bool = False @@ -4621,7 +4822,7 @@ been used at least once. static PyObject * _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) -/*[clinic end generated code: output=0d58f148f37e2938 input=6887b5a09b7f9076]*/ +/*[clinic end generated code: output=0d58f148f37e2938 input=eb0592909c9ad6e7]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -4633,7 +4834,12 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } store = SSL_CTX_get_cert_store(self->ctx); - objs = X509_STORE_get0_objects(store); + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + goto error; + } + for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { X509_OBJECT *obj; X509 *cert; @@ -4661,9 +4867,11 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } Py_CLEAR(ci); } + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); return rlist; error: + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); Py_XDECREF(ci); Py_XDECREF(rlist); return NULL; @@ -4740,6 +4948,7 @@ static unsigned int psk_client_callback(SSL *s, #endif /*[clinic input] +@critical_section _ssl._SSLContext.set_psk_client_callback callback: object @@ -4748,7 +4957,7 @@ _ssl._SSLContext.set_psk_client_callback static PyObject * _ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, PyObject *callback) -/*[clinic end generated code: output=0aba86f6ed75119e input=7627bae0e5ee7635]*/ +/*[clinic end generated code: output=0aba86f6ed75119e input=1e436eea625cfc35]*/ { #ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_SERVER) { @@ -4848,6 +5057,7 @@ static unsigned int psk_server_callback(SSL *s, #endif /*[clinic input] +@critical_section _ssl._SSLContext.set_psk_server_callback callback: object identity_hint: str(accept={str, NoneType}) = None @@ -4858,7 +5068,7 @@ static PyObject * _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, PyObject *callback, const char *identity_hint) -/*[clinic end generated code: output=1f4d6a4e09a92b03 input=65d4b6022aa85ea3]*/ +/*[clinic end generated code: output=1f4d6a4e09a92b03 input=5f79d932458284a7]*/ { #ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { @@ -4903,41 +5113,30 @@ _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, static PyGetSetDef context_getsetlist[] = { - {"check_hostname", (getter) get_check_hostname, - (setter) set_check_hostname, NULL}, - {"_host_flags", (getter) get_host_flags, - (setter) set_host_flags, NULL}, - {"minimum_version", (getter) get_minimum_version, - (setter) set_minimum_version, NULL}, - {"maximum_version", (getter) get_maximum_version, - (setter) set_maximum_version, NULL}, + _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF + _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF + _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF + _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"keylog_filename", (getter) _PySSLContext_get_keylog_filename, (setter) _PySSLContext_set_keylog_filename, NULL}, {"_msg_callback", (getter) _PySSLContext_get_msg_callback, (setter) _PySSLContext_set_msg_callback, NULL}, - {"sni_callback", (getter) get_sni_callback, - (setter) set_sni_callback, PySSLContext_sni_callback_doc}, -#ifdef TLS1_3_VERSION - {"num_tickets", (getter) get_num_tickets, - (setter) set_num_tickets, PySSLContext_num_tickets_doc}, + _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF +#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF #endif - {"options", (getter) get_options, - (setter) set_options, NULL}, + _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"post_handshake_auth", (getter) get_post_handshake_auth, -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) (setter) set_post_handshake_auth, #else NULL, #endif NULL}, - {"protocol", (getter) get_protocol, - NULL, NULL}, - {"verify_flags", (getter) get_verify_flags, - (setter) set_verify_flags, NULL}, - {"verify_mode", (getter) get_verify_mode, - (setter) set_verify_mode, NULL}, - {"security_level", (getter) get_security_level, - NULL, PySSLContext_security_level_doc}, + _SSL__SSLCONTEXT_PROTOCOL_GETSETDEF + _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF + _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF + _SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF {NULL}, /* sentinel */ }; @@ -4984,6 +5183,7 @@ static PyType_Spec PySSLContext_spec = { */ /*[clinic input] +@critical_section @classmethod _ssl.MemoryBIO.__new__ @@ -4991,7 +5191,7 @@ _ssl.MemoryBIO.__new__ static PyObject * _ssl_MemoryBIO_impl(PyTypeObject *type) -/*[clinic end generated code: output=8820a58db78330ac input=26d22e4909ecb1b5]*/ +/*[clinic end generated code: output=8820a58db78330ac input=87f146cf30af454e]*/ { BIO *bio; PySSLMemoryBIO *self; @@ -5036,26 +5236,36 @@ memory_bio_dealloc(PySSLMemoryBIO *self) Py_DECREF(tp); } +/*[clinic input] +@critical_section +@getter +_ssl.MemoryBIO.pending +[clinic start generated code]*/ + static PyObject * -memory_bio_get_pending(PySSLMemoryBIO *self, void *c) +_ssl_MemoryBIO_pending_get_impl(PySSLMemoryBIO *self) +/*[clinic end generated code: output=19236a32a51ac8ff input=c0b6d14eba107f6a]*/ { - return PyLong_FromSize_t(BIO_ctrl_pending(self->bio)); + size_t res = BIO_ctrl_pending(self->bio); + return PyLong_FromSize_t(res); } -PyDoc_STRVAR(PySSL_memory_bio_pending_doc, -"The number of bytes pending in the memory BIO."); +/*[clinic input] +@critical_section +@getter +_ssl.MemoryBIO.eof +[clinic start generated code]*/ static PyObject * -memory_bio_get_eof(PySSLMemoryBIO *self, void *c) +_ssl_MemoryBIO_eof_get_impl(PySSLMemoryBIO *self) +/*[clinic end generated code: output=c255a9ea16e31b92 input=0f5c6be69752e04c]*/ { - return PyBool_FromLong((BIO_ctrl_pending(self->bio) == 0) - && self->eof_written); + size_t pending = BIO_ctrl_pending(self->bio); + return PyBool_FromLong((pending == 0) && self->eof_written); } -PyDoc_STRVAR(PySSL_memory_bio_eof_doc, -"Whether the memory BIO is at EOF."); - /*[clinic input] +@critical_section _ssl.MemoryBIO.read size as len: int = -1 / @@ -5070,7 +5280,7 @@ distinguish between the two. static PyObject * _ssl_MemoryBIO_read_impl(PySSLMemoryBIO *self, int len) -/*[clinic end generated code: output=a657aa1e79cd01b3 input=574d7be06a902366]*/ +/*[clinic end generated code: output=a657aa1e79cd01b3 input=21046f2d7dac3a90]*/ { int avail, nbytes; PyObject *result; @@ -5100,6 +5310,7 @@ _ssl_MemoryBIO_read_impl(PySSLMemoryBIO *self, int len) } /*[clinic input] +@critical_section _ssl.MemoryBIO.write b: Py_buffer / @@ -5111,7 +5322,7 @@ Returns the number of bytes written. static PyObject * _ssl_MemoryBIO_write_impl(PySSLMemoryBIO *self, Py_buffer *b) -/*[clinic end generated code: output=156ec59110d75935 input=e45757b3e17c4808]*/ +/*[clinic end generated code: output=156ec59110d75935 input=107da3f5fba26b37]*/ { int nbytes; @@ -5141,6 +5352,7 @@ _ssl_MemoryBIO_write_impl(PySSLMemoryBIO *self, Py_buffer *b) } /*[clinic input] +@critical_section _ssl.MemoryBIO.write_eof Write an EOF marker to the memory BIO. @@ -5150,7 +5362,7 @@ When all data has been read, the "eof" property will be True. static PyObject * _ssl_MemoryBIO_write_eof_impl(PySSLMemoryBIO *self) -/*[clinic end generated code: output=d4106276ccd1ed34 input=56a945f1d29e8bd6]*/ +/*[clinic end generated code: output=d4106276ccd1ed34 input=1e914231b1c5900a]*/ { self->eof_written = 1; /* After an EOF is written, a zero return from read() should be a real EOF @@ -5162,10 +5374,8 @@ _ssl_MemoryBIO_write_eof_impl(PySSLMemoryBIO *self) } static PyGetSetDef memory_bio_getsetlist[] = { - {"pending", (getter) memory_bio_get_pending, NULL, - PySSL_memory_bio_pending_doc}, - {"eof", (getter) memory_bio_get_eof, NULL, - PySSL_memory_bio_eof_doc}, + _SSL_MEMORYBIO_PENDING_GETSETDEF + _SSL_MEMORYBIO_EOF_GETSETDEF {NULL}, /* sentinel */ }; @@ -5214,14 +5424,14 @@ PySSLSession_dealloc(PySSLSession *self) static PyObject * PySSLSession_richcompare(PyObject *left, PyObject *right, int op) { - int result; - PyTypeObject *sesstype = ((PySSLSession*)left)->ctx->state->PySSLSession_Type; - if (left == NULL || right == NULL) { PyErr_BadInternalCall(); return NULL; } + int result; + PyTypeObject *sesstype = ((PySSLSession*)left)->ctx->state->PySSLSession_Type; + if (!Py_IS_TYPE(left, sesstype) || !Py_IS_TYPE(right, sesstype)) { Py_RETURN_NOTIMPLEMENTED; } @@ -5285,70 +5495,89 @@ PySSLSession_clear(PySSLSession *self) } +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.time +[clinic start generated code]*/ + static PyObject * -PySSLSession_get_time(PySSLSession *self, void *closure) { +_ssl_SSLSession_time_get_impl(PySSLSession *self) +/*[clinic end generated code: output=4b887b9299de9be4 input=8d1e4afd09103279]*/ +{ +#if OPENSSL_VERSION_NUMBER >= 0x30300000L + return _PyLong_FromTime_t(SSL_SESSION_get_time_ex(self->session)); +#else return PyLong_FromLong(SSL_SESSION_get_time(self->session)); +#endif } -PyDoc_STRVAR(PySSLSession_get_time_doc, -"Session creation time (seconds since epoch)."); - +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.timeout +[clinic start generated code]*/ static PyObject * -PySSLSession_get_timeout(PySSLSession *self, void *closure) { - return PyLong_FromLong(SSL_SESSION_get_timeout(self->session)); +_ssl_SSLSession_timeout_get_impl(PySSLSession *self) +/*[clinic end generated code: output=82339c148ab2f7d1 input=ae5e84a9d85df60d]*/ +{ + long timeout = SSL_SESSION_get_timeout(self->session); + PyObject *res = PyLong_FromLong(timeout); + return res; } -PyDoc_STRVAR(PySSLSession_get_timeout_doc, -"Session timeout (delta in seconds)."); - +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.ticket_lifetime_hint +[clinic start generated code]*/ static PyObject * -PySSLSession_get_ticket_lifetime_hint(PySSLSession *self, void *closure) { +_ssl_SSLSession_ticket_lifetime_hint_get_impl(PySSLSession *self) +/*[clinic end generated code: output=c8b6db498136c275 input=d0e06942ddd8d07f]*/ +{ unsigned long hint = SSL_SESSION_get_ticket_lifetime_hint(self->session); return PyLong_FromUnsignedLong(hint); } -PyDoc_STRVAR(PySSLSession_get_ticket_lifetime_hint_doc, -"Ticket life time hint."); - +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.id +[clinic start generated code]*/ static PyObject * -PySSLSession_get_session_id(PySSLSession *self, void *closure) { +_ssl_SSLSession_id_get_impl(PySSLSession *self) +/*[clinic end generated code: output=c532fb96b10c5adf input=e7322372cf6325dd]*/ + +{ const unsigned char *id; unsigned int len; id = SSL_SESSION_get_id(self->session, &len); return PyBytes_FromStringAndSize((const char *)id, len); } -PyDoc_STRVAR(PySSLSession_get_session_id_doc, -"Session id"); - +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.has_ticket +[clinic start generated code]*/ static PyObject * -PySSLSession_get_has_ticket(PySSLSession *self, void *closure) { - if (SSL_SESSION_has_ticket(self->session)) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } +_ssl_SSLSession_has_ticket_get_impl(PySSLSession *self) +/*[clinic end generated code: output=aa3ccfc40b10b96d input=1a48ae8955fa9601]*/ +{ + int res = SSL_SESSION_has_ticket(self->session); + return res ? Py_True : Py_False; } -PyDoc_STRVAR(PySSLSession_get_has_ticket_doc, -"Does the session contain a ticket?"); - - static PyGetSetDef PySSLSession_getsetlist[] = { - {"has_ticket", (getter) PySSLSession_get_has_ticket, NULL, - PySSLSession_get_has_ticket_doc}, - {"id", (getter) PySSLSession_get_session_id, NULL, - PySSLSession_get_session_id_doc}, - {"ticket_lifetime_hint", (getter) PySSLSession_get_ticket_lifetime_hint, - NULL, PySSLSession_get_ticket_lifetime_hint_doc}, - {"time", (getter) PySSLSession_get_time, NULL, - PySSLSession_get_time_doc}, - {"timeout", (getter) PySSLSession_get_timeout, NULL, - PySSLSession_get_timeout_doc}, + _SSL_SSLSESSION_HAS_TICKET_GETSETDEF + _SSL_SSLSESSION_ID_GETSETDEF + _SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF + _SSL_SSLSESSION_TIME_GETSETDEF + _SSL_SSLSESSION_TIMEOUT_GETSETDEF {NULL}, /* sentinel */ }; @@ -5373,6 +5602,7 @@ static PyType_Spec PySSLSession_spec = { /* helper routines for seeding the SSL PRNG */ /*[clinic input] +@critical_section _ssl.RAND_add string as view: Py_buffer(accept={str, buffer}) entropy: double @@ -5386,7 +5616,7 @@ string. See RFC 4086. static PyObject * _ssl_RAND_add_impl(PyObject *module, Py_buffer *view, double entropy) -/*[clinic end generated code: output=e6dd48df9c9024e9 input=5c33017422828f5c]*/ +/*[clinic end generated code: output=e6dd48df9c9024e9 input=313cb73b34db31d5]*/ { const char *buf; Py_ssize_t len, written; @@ -5442,6 +5672,7 @@ PySSL_RAND(PyObject *module, int len, int pseudo) } /*[clinic input] +@critical_section _ssl.RAND_bytes n: int / @@ -5451,13 +5682,14 @@ Generate n cryptographically strong pseudo-random bytes. static PyObject * _ssl_RAND_bytes_impl(PyObject *module, int n) -/*[clinic end generated code: output=977da635e4838bc7 input=678ddf2872dfebfc]*/ +/*[clinic end generated code: output=977da635e4838bc7 input=2e78ce1e86336776]*/ { return PySSL_RAND(module, n, 0); } /*[clinic input] +@critical_section _ssl.RAND_status Returns True if the OpenSSL PRNG has been seeded with enough data and False if not. @@ -5468,12 +5700,13 @@ using the ssl() function. static PyObject * _ssl_RAND_status_impl(PyObject *module) -/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=d5ae5aea52f36e01]*/ +/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=636fb5659ea2e727]*/ { return PyBool_FromLong(RAND_status()); } /*[clinic input] +@critical_section _ssl.get_default_verify_paths Return search paths and environment vars that are used by SSLContext's set_default_verify_paths() to load default CAs. @@ -5483,7 +5716,7 @@ The values are 'cert_file_env', 'cert_file', 'cert_dir_env', 'cert_dir'. static PyObject * _ssl_get_default_verify_paths_impl(PyObject *module) -/*[clinic end generated code: output=e5b62a466271928b input=5210c953d98c3eb5]*/ +/*[clinic end generated code: output=e5b62a466271928b input=c6ae00bc04eb2b6e]*/ { PyObject *ofile_env = NULL; PyObject *ofile = NULL; @@ -5532,6 +5765,7 @@ asn1obj2py(_sslmodulestate *state, ASN1_OBJECT *obj) } /*[clinic input] +@critical_section _ssl.txt2obj txt: str name: bool = False @@ -5544,7 +5778,7 @@ long name are also matched. static PyObject * _ssl_txt2obj_impl(PyObject *module, const char *txt, int name) -/*[clinic end generated code: output=c38e3991347079c1 input=1c1e7d0aa7c48602]*/ +/*[clinic end generated code: output=c38e3991347079c1 input=c99b134d70173c5e]*/ { PyObject *result = NULL; ASN1_OBJECT *obj; @@ -5560,6 +5794,7 @@ _ssl_txt2obj_impl(PyObject *module, const char *txt, int name) } /*[clinic input] +@critical_section _ssl.nid2obj nid: int / @@ -5569,7 +5804,7 @@ Lookup NID, short name, long name and OID of an ASN1_OBJECT by NID. static PyObject * _ssl_nid2obj_impl(PyObject *module, int nid) -/*[clinic end generated code: output=4a98ab691cd4f84a input=51787a3bee7d8f98]*/ +/*[clinic end generated code: output=4a98ab691cd4f84a input=1b1170506fa83a53]*/ { PyObject *result = NULL; ASN1_OBJECT *obj; @@ -5721,6 +5956,7 @@ ssl_collect_certificates(const char *store_name) } /*[clinic input] +@critical_section _ssl.enum_certificates store_name: str @@ -5735,7 +5971,7 @@ a set of OIDs or the boolean True. static PyObject * _ssl_enum_certificates_impl(PyObject *module, const char *store_name) -/*[clinic end generated code: output=5134dc8bb3a3c893 input=915f60d70461ea4e]*/ +/*[clinic end generated code: output=5134dc8bb3a3c893 input=263c22e6c6988cf3]*/ { HCERTSTORE hCollectionStore = NULL; PCCERT_CONTEXT pCertCtx = NULL; @@ -5820,6 +6056,7 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name) } /*[clinic input] +@critical_section _ssl.enum_crls store_name: str @@ -5833,7 +6070,7 @@ X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. static PyObject * _ssl_enum_crls_impl(PyObject *module, const char *store_name) -/*[clinic end generated code: output=bce467f60ccd03b6 input=a1f1d7629f1c5d3d]*/ +/*[clinic end generated code: output=bce467f60ccd03b6 input=51a1b1059e55ce43]*/ { HCERTSTORE hCollectionStore = NULL; PCCRL_CONTEXT pCrlCtx = NULL; @@ -6469,6 +6706,7 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_strings}, {Py_mod_exec, sslmodule_init_lock}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 07e9ce7a6fce2d..9c87f8b4d21e68 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -28,12 +28,12 @@ _PySSL_msg_callback(int write_p, int version, int content_type, PyObject *ssl_socket; /* ssl.SSLSocket or ssl.SSLObject */ if (ssl_obj->owner) - ssl_socket = _PyWeakref_GET_REF(ssl_obj->owner); + PyWeakref_GetRef(ssl_obj->owner, &ssl_socket); else if (ssl_obj->Socket) - ssl_socket = _PyWeakref_GET_REF(ssl_obj->Socket); + PyWeakref_GetRef(ssl_obj->Socket, &ssl_socket); else ssl_socket = (PyObject *)Py_NewRef(ssl_obj); - assert(ssl_socket != NULL); // _PyWeakref_GET_REF() can return NULL + assert(ssl_socket != NULL); // PyWeakref_GetRef() can return NULL /* assume that OpenSSL verifies all payload and buf len is of sufficient length */ diff --git a/Modules/_stat.c b/Modules/_stat.c index b43e79453f5b2f..13a2bec252f448 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -11,11 +11,10 @@ * */ +// Need limited C API version 3.13 for PyModule_Add() on Windows #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.13 for PyModule_Add() on Windows -#define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -507,7 +506,7 @@ S_IWOTH: write by others\n\ S_IXOTH: execute by others\n\ \n" -"UF_SETTABLE: mask of owner changable flags\n\ +"UF_SETTABLE: mask of owner changeable flags\n\ UF_NODUMP: do not dump file\n\ UF_IMMUTABLE: file may not be changed\n\ UF_APPEND: file may only be appended to\n\ @@ -680,6 +679,7 @@ stat_exec(PyObject *module) static PyModuleDef_Slot stat_slots[] = { {Py_mod_exec, stat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c index a04a2a779a5d3d..b84f731ad6a1da 100644 --- a/Modules/_statisticsmodule.c +++ b/Modules/_statisticsmodule.c @@ -1,8 +1,9 @@ /* statistics accelerator C extension: _statistics module. */ -// clinic/_statisticsmodule.c.h uses internal pycore_modsupport.h API -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -135,6 +136,7 @@ PyDoc_STRVAR(statistics_doc, static struct PyModuleDef_Slot _statisticsmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_struct.c b/Modules/_struct.c index bd16fa89f18945..21582b945be23d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -12,6 +12,9 @@ #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() +#ifdef Py_HAVE_C_COMPLEX +# include "_complex.h" // complex +#endif #include // offsetof() /*[clinic input] @@ -80,6 +83,10 @@ typedef struct { char c; int x; } st_int; typedef struct { char c; long x; } st_long; typedef struct { char c; float x; } st_float; typedef struct { char c; double x; } st_double; +#ifdef Py_HAVE_C_COMPLEX +typedef struct { char c; float complex x; } st_float_complex; +typedef struct { char c; double complex x; } st_double_complex; +#endif typedef struct { char c; void *x; } st_void_p; typedef struct { char c; size_t x; } st_size_t; typedef struct { char c; _Bool x; } st_bool; @@ -89,6 +96,10 @@ typedef struct { char c; _Bool x; } st_bool; #define LONG_ALIGN (sizeof(st_long) - sizeof(long)) #define FLOAT_ALIGN (sizeof(st_float) - sizeof(float)) #define DOUBLE_ALIGN (sizeof(st_double) - sizeof(double)) +#ifdef Py_HAVE_C_COMPLEX +# define FLOAT_COMPLEX_ALIGN (sizeof(st_float_complex) - sizeof(float complex)) +# define DOUBLE_COMPLEX_ALIGN (sizeof(st_double_complex) - sizeof(double complex)) +#endif #define VOID_P_ALIGN (sizeof(st_void_p) - sizeof(void *)) #define SIZE_T_ALIGN (sizeof(st_size_t) - sizeof(size_t)) #define BOOL_ALIGN (sizeof(st_bool) - sizeof(_Bool)) @@ -281,7 +292,7 @@ get_size_t(_structmodulestate *state, PyObject *v, size_t *p) #define RANGE_ERROR(state, f, flag) return _range_error(state, f, flag) -/* Floating point helpers */ +/* Floating-point helpers */ static PyObject * unpack_halffloat(const char *p, /* start of 2-byte string */ @@ -407,7 +418,7 @@ static PyObject * nu_short(_structmodulestate *state, const char *p, const formatdef *f) { short x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong((long)x); } @@ -415,7 +426,7 @@ static PyObject * nu_ushort(_structmodulestate *state, const char *p, const formatdef *f) { unsigned short x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong((long)x); } @@ -423,7 +434,7 @@ static PyObject * nu_int(_structmodulestate *state, const char *p, const formatdef *f) { int x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong((long)x); } @@ -431,7 +442,7 @@ static PyObject * nu_uint(_structmodulestate *state, const char *p, const formatdef *f) { unsigned int x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromUnsignedLong((unsigned long)x); } @@ -439,7 +450,7 @@ static PyObject * nu_long(_structmodulestate *state, const char *p, const formatdef *f) { long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong(x); } @@ -447,7 +458,7 @@ static PyObject * nu_ulong(_structmodulestate *state, const char *p, const formatdef *f) { unsigned long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromUnsignedLong(x); } @@ -455,7 +466,7 @@ static PyObject * nu_ssize_t(_structmodulestate *state, const char *p, const formatdef *f) { Py_ssize_t x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromSsize_t(x); } @@ -463,7 +474,7 @@ static PyObject * nu_size_t(_structmodulestate *state, const char *p, const formatdef *f) { size_t x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromSize_t(x); } @@ -471,7 +482,7 @@ static PyObject * nu_longlong(_structmodulestate *state, const char *p, const formatdef *f) { long long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLongLong(x); } @@ -479,16 +490,15 @@ static PyObject * nu_ulonglong(_structmodulestate *state, const char *p, const formatdef *f) { unsigned long long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromUnsignedLongLong(x); } static PyObject * nu_bool(_structmodulestate *state, const char *p, const formatdef *f) { - _Bool x; - memcpy((char *)&x, p, sizeof x); - return PyBool_FromLong(x != 0); + const _Bool bool_false = 0; + return PyBool_FromLong(memcmp(p, &bool_false, sizeof(_Bool))); } @@ -506,7 +516,7 @@ static PyObject * nu_float(_structmodulestate *state, const char *p, const formatdef *f) { float x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyFloat_FromDouble((double)x); } @@ -514,15 +524,35 @@ static PyObject * nu_double(_structmodulestate *state, const char *p, const formatdef *f) { double x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyFloat_FromDouble(x); } +#ifdef Py_HAVE_C_COMPLEX +static PyObject * +nu_float_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + float complex x; + + memcpy(&x, p, sizeof(x)); + return PyComplex_FromDoubles(creal(x), cimag(x)); +} + +static PyObject * +nu_double_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double complex x; + + memcpy(&x, p, sizeof(x)); + return PyComplex_FromDoubles(creal(x), cimag(x)); +} +#endif + static PyObject * nu_void_p(_structmodulestate *state, const char *p, const formatdef *f) { void *x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromVoidPtr(x); } @@ -587,7 +617,7 @@ np_short(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) RANGE_ERROR(state, f, 0); } y = (short)x; - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -606,7 +636,7 @@ np_ushort(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) RANGE_ERROR(state, f, 1); } y = (unsigned short)x; - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -626,7 +656,7 @@ np_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) RANGE_ERROR(state, f, 0); #endif y = (int)x; - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -646,7 +676,7 @@ np_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) if (x > ((unsigned long)UINT_MAX)) RANGE_ERROR(state, f, 1); #endif - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -660,7 +690,7 @@ np_long(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -674,7 +704,7 @@ np_ulong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -688,7 +718,7 @@ np_ssize_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -702,7 +732,7 @@ np_size_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -720,7 +750,7 @@ np_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -737,7 +767,7 @@ np_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -751,7 +781,7 @@ np_bool(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) if (y < 0) return -1; x = y; - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -774,7 +804,7 @@ np_float(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) "required argument is not a float"); return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -787,10 +817,62 @@ np_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) "required argument is not a float"); return -1; } - memcpy(p, (char *)&x, sizeof(double)); + memcpy(p, &x, sizeof(double)); + return 0; +} + +#ifdef Py_HAVE_C_COMPLEX +static int +np_float_complex(_structmodulestate *state, char *p, PyObject *v, + const formatdef *f) +{ + Py_complex c = PyComplex_AsCComplex(v); + float complex x = CMPLXF((float)c.real, (float)c.imag); + + if (c.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + memcpy(p, &x, sizeof(x)); return 0; } +static int +np_double_complex(_structmodulestate *state, char *p, PyObject *v, + const formatdef *f) +{ + Py_complex c = PyComplex_AsCComplex(v); + double complex x = CMPLX(c.real, c.imag); + + if (c.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + memcpy(p, &x, sizeof(x)); + return 0; +} +#else +static int +np_complex_stub(_structmodulestate *state, char *p, PyObject *v, + const formatdef *f) +{ + PyErr_Format(state->StructError, + "'%c' format not supported on this system", + f->format); + return -1; +} +static PyObject * +nu_complex_stub(_structmodulestate *state, const char *p, const formatdef *f) +{ + PyErr_Format(state->StructError, + "'%c' format not supported on this system", + f->format); + return NULL; +} +#endif + static int np_void_p(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { @@ -804,7 +886,7 @@ np_void_p(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) Py_DECREF(v); if (x == NULL && PyErr_Occurred()) return -1; - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -829,6 +911,13 @@ static const formatdef native_table[] = { {'e', sizeof(short), SHORT_ALIGN, nu_halffloat, np_halffloat}, {'f', sizeof(float), FLOAT_ALIGN, nu_float, np_float}, {'d', sizeof(double), DOUBLE_ALIGN, nu_double, np_double}, +#ifdef Py_HAVE_C_COMPLEX + {'E', sizeof(float complex), FLOAT_COMPLEX_ALIGN, nu_float_complex, np_float_complex}, + {'C', sizeof(double complex), DOUBLE_COMPLEX_ALIGN, nu_double_complex, np_double_complex}, +#else + {'E', 1, 0, nu_complex_stub, np_complex_stub}, + {'C', 1, 0, nu_complex_stub, np_complex_stub}, +#endif {'P', sizeof(void *), VOID_P_ALIGN, nu_void_p, np_void_p}, {0} }; @@ -929,6 +1018,38 @@ bu_double(_structmodulestate *state, const char *p, const formatdef *f) return unpack_double(p, 0); } +#ifdef Py_HAVE_C_COMPLEX +static PyObject * +bu_float_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x = PyFloat_Unpack4(p, 0); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + double y = PyFloat_Unpack4(p + 4, 0); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} + +static PyObject * +bu_double_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x, y; + + x = PyFloat_Unpack8(p, 0); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + y = PyFloat_Unpack8(p + 8, 0); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} +#endif + static PyObject * bu_bool(_structmodulestate *state, const char *p, const formatdef *f) { @@ -1000,9 +1121,10 @@ bp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) (unsigned char *)p, 8, 0, /* little_endian */ - 1 /* signed */); + 1, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires %lld <= number <= %lld", f->format, @@ -1024,9 +1146,10 @@ bp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f (unsigned char *)p, 8, 0, /* little_endian */ - 0 /* signed */); + 0, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires 0 <= number <= %llu", f->format, @@ -1066,6 +1189,38 @@ bp_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) return PyFloat_Pack8(x, p, 0); } +#ifdef Py_HAVE_C_COMPLEX +static int +bp_float_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack4(x.real, p, 0)) { + return -1; + } + return PyFloat_Pack4(x.imag, p + 4, 0); +} + +static int +bp_double_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack8(x.real, p, 0)) { + return -1; + } + return PyFloat_Pack8(x.imag, p + 8, 0); +} +#endif + static int bp_bool(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { @@ -1096,6 +1251,13 @@ static formatdef bigendian_table[] = { {'e', 2, 0, bu_halffloat, bp_halffloat}, {'f', 4, 0, bu_float, bp_float}, {'d', 8, 0, bu_double, bp_double}, +#ifdef Py_HAVE_C_COMPLEX + {'E', 8, 0, bu_float_complex, bp_float_complex}, + {'C', 16, 0, bu_double_complex, bp_double_complex}, +#else + {'E', 1, 0, nu_complex_stub, np_complex_stub}, + {'C', 1, 0, nu_complex_stub, np_complex_stub}, +#endif {0} }; @@ -1195,6 +1357,38 @@ lu_double(_structmodulestate *state, const char *p, const formatdef *f) return unpack_double(p, 1); } +#ifdef Py_HAVE_C_COMPLEX +static PyObject * +lu_float_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x = PyFloat_Unpack4(p, 1); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + double y = PyFloat_Unpack4(p + 4, 1); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} + +static PyObject * +lu_double_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x, y; + + x = PyFloat_Unpack8(p, 1); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + y = PyFloat_Unpack8(p + 8, 1); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} +#endif + static int lp_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { @@ -1260,9 +1454,10 @@ lp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) (unsigned char *)p, 8, 1, /* little_endian */ - 1 /* signed */); + 1, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires %lld <= number <= %lld", f->format, @@ -1284,9 +1479,10 @@ lp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f (unsigned char *)p, 8, 1, /* little_endian */ - 0 /* signed */); + 0, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires 0 <= number <= %llu", f->format, @@ -1326,6 +1522,39 @@ lp_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) return PyFloat_Pack8(x, p, 1); } +#ifdef Py_HAVE_C_COMPLEX +static int +lp_float_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack4(x.real, p, 1)) { + return -1; + } + return PyFloat_Pack4(x.imag, p + 4, 1); + +} + +static int +lp_double_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack8(x.real, p, 1)) { + return -1; + } + return PyFloat_Pack8(x.imag, p + 8, 1); +} +#endif + static formatdef lilendian_table[] = { {'x', 1, 0, NULL}, {'b', 1, 0, nu_byte, np_byte}, @@ -1346,6 +1575,13 @@ static formatdef lilendian_table[] = { {'e', 2, 0, lu_halffloat, lp_halffloat}, {'f', 4, 0, lu_float, lp_float}, {'d', 8, 0, lu_double, lp_double}, +#ifdef Py_HAVE_C_COMPLEX + {'E', 8, 0, lu_float_complex, lp_float_complex}, + {'C', 16, 0, lu_double_complex, lp_double_complex}, +#else + {'E', 1, 0, nu_complex_stub, np_complex_stub}, + {'C', 1, 0, nu_complex_stub, np_complex_stub}, +#endif {0} }; @@ -1369,7 +1605,7 @@ whichtable(const char **pfmt) } default: --*pfmt; /* Back out of pointer increment */ - /* Fall through */ + _Py_FALLTHROUGH; case '@': return native_table; } @@ -1471,7 +1707,7 @@ prepare_s(PyStructObject *self) return -1; switch (c) { - case 's': /* fall through */ + case 's': _Py_FALLTHROUGH; case 'p': len++; ncodes++; break; case 'x': break; default: len += num; if (num) ncodes++; break; @@ -1665,9 +1901,16 @@ s_unpack_internal(PyStructObject *soself, const char *startfrom, if (e->format == 's') { v = PyBytes_FromStringAndSize(res, code->size); } else if (e->format == 'p') { - Py_ssize_t n = *(unsigned char*)res; - if (n >= code->size) - n = code->size - 1; + Py_ssize_t n; + if (code->size == 0) { + n = 0; + } + else { + n = *(unsigned char*)res; + if (n >= code->size) { + n = code->size - 1; + } + } v = PyBytes_FromStringAndSize(res + 1, n); } else { v = e->unpack(state, res, e); @@ -1978,8 +2221,12 @@ s_pack_internal(PyStructObject *soself, PyObject *const *args, int offset, n = PyByteArray_GET_SIZE(v); p = PyByteArray_AS_STRING(v); } - if (n > (code->size - 1)) + if (code->size == 0) { + n = 0; + } + else if (n > (code->size - 1)) { n = code->size - 1; + } if (n > 0) memcpy(res + 1, p, n); if (n > 255) @@ -2589,6 +2836,7 @@ _structmodule_exec(PyObject *m) static PyModuleDef_Slot _structmodule_slots[] = { {Py_mod_exec, _structmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_suggestions.c b/Modules/_suggestions.c index 30b524d70c1211..80c7179c4c251c 100644 --- a/Modules/_suggestions.c +++ b/Modules/_suggestions.c @@ -49,15 +49,21 @@ static PyMethodDef module_methods[] = { {NULL, NULL, 0, NULL} // Sentinel }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef suggestions_module = { PyModuleDef_HEAD_INIT, "_suggestions", NULL, - -1, - module_methods + 0, + module_methods, + module_slots, }; PyMODINIT_FUNC PyInit__suggestions(void) { - return PyModule_Create(&suggestions_module); + return PyModuleDef_Init(&suggestions_module); } - diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index c76b9e6b3ebafa..c50c5cfabc2f1f 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -80,6 +80,7 @@ static struct PyMethodDef sysconfig_methods[] = { static PyModuleDef_Slot sysconfig_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 5101834cfe1387..b47bc8a830cb5e 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -24,11 +24,13 @@ static PyTypeObject NDArray_Type; #define NDArray_Check(v) Py_IS_TYPE(v, &NDArray_Type) #define CHECK_LIST_OR_TUPLE(v) \ - if (!PyList_Check(v) && !PyTuple_Check(v)) { \ - PyErr_SetString(PyExc_TypeError, \ - #v " must be a list or a tuple"); \ - return NULL; \ - } \ + do { \ + if (!PyList_Check(v) && !PyTuple_Check(v)) { \ + PyErr_SetString(PyExc_TypeError, \ + #v " must be a list or a tuple"); \ + return NULL; \ + } \ + } while (0) #define PyMem_XFree(v) \ do { if (v) PyMem_Free(v); } while (0) @@ -1180,7 +1182,7 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, Py_ssize_t itemsize; /* ndim = len(shape) */ - CHECK_LIST_OR_TUPLE(shape) + CHECK_LIST_OR_TUPLE(shape); ndim = PySequence_Fast_GET_SIZE(shape); if (ndim > ND_MAX_NDIM) { PyErr_Format(PyExc_ValueError, @@ -1190,7 +1192,7 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, /* len(strides) = len(shape) */ if (strides) { - CHECK_LIST_OR_TUPLE(strides) + CHECK_LIST_OR_TUPLE(strides); if (PySequence_Fast_GET_SIZE(strides) == 0) strides = NULL; else if (flags & ND_FORTRAN) { @@ -1217,12 +1219,12 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, /* convert scalar to list */ if (ndim == 0) { - items = Py_BuildValue("(O)", items); + items = PyTuple_Pack(1, items); if (items == NULL) return NULL; } else { - CHECK_LIST_OR_TUPLE(items) + CHECK_LIST_OR_TUPLE(items); Py_INCREF(items); } @@ -2816,70 +2818,97 @@ static struct PyModuleDef _testbuffermodule = { NULL }; - -PyMODINIT_FUNC -PyInit__testbuffer(void) +static int +_testbuffer_exec(PyObject *mod) { - PyObject *m; - - m = PyModule_Create(&_testbuffermodule); - if (m == NULL) - return NULL; - Py_SET_TYPE(&NDArray_Type, &PyType_Type); - Py_INCREF(&NDArray_Type); - PyModule_AddObject(m, "ndarray", (PyObject *)&NDArray_Type); + if (PyType_Ready(&NDArray_Type)) { + return -1; + } + if (PyModule_AddType(mod, &NDArray_Type) < 0) { + return -1; + } Py_SET_TYPE(&StaticArray_Type, &PyType_Type); - Py_INCREF(&StaticArray_Type); - PyModule_AddObject(m, "staticarray", (PyObject *)&StaticArray_Type); + if (PyModule_AddType(mod, &StaticArray_Type) < 0) { + return -1; + } structmodule = PyImport_ImportModule("struct"); - if (structmodule == NULL) - return NULL; + if (structmodule == NULL) { + return -1; + } Struct = PyObject_GetAttrString(structmodule, "Struct"); + if (Struct == NULL) { + return -1; + } calcsize = PyObject_GetAttrString(structmodule, "calcsize"); - if (Struct == NULL || calcsize == NULL) - return NULL; + if (calcsize == NULL) { + return -1; + } simple_format = PyUnicode_FromString(simple_fmt); - if (simple_format == NULL) - return NULL; - - PyModule_AddIntMacro(m, ND_MAX_NDIM); - PyModule_AddIntMacro(m, ND_VAREXPORT); - PyModule_AddIntMacro(m, ND_WRITABLE); - PyModule_AddIntMacro(m, ND_FORTRAN); - PyModule_AddIntMacro(m, ND_SCALAR); - PyModule_AddIntMacro(m, ND_PIL); - PyModule_AddIntMacro(m, ND_GETBUF_FAIL); - PyModule_AddIntMacro(m, ND_GETBUF_UNDEFINED); - PyModule_AddIntMacro(m, ND_REDIRECT); - - PyModule_AddIntMacro(m, PyBUF_SIMPLE); - PyModule_AddIntMacro(m, PyBUF_WRITABLE); - PyModule_AddIntMacro(m, PyBUF_FORMAT); - PyModule_AddIntMacro(m, PyBUF_ND); - PyModule_AddIntMacro(m, PyBUF_STRIDES); - PyModule_AddIntMacro(m, PyBUF_INDIRECT); - PyModule_AddIntMacro(m, PyBUF_C_CONTIGUOUS); - PyModule_AddIntMacro(m, PyBUF_F_CONTIGUOUS); - PyModule_AddIntMacro(m, PyBUF_ANY_CONTIGUOUS); - PyModule_AddIntMacro(m, PyBUF_FULL); - PyModule_AddIntMacro(m, PyBUF_FULL_RO); - PyModule_AddIntMacro(m, PyBUF_RECORDS); - PyModule_AddIntMacro(m, PyBUF_RECORDS_RO); - PyModule_AddIntMacro(m, PyBUF_STRIDED); - PyModule_AddIntMacro(m, PyBUF_STRIDED_RO); - PyModule_AddIntMacro(m, PyBUF_CONTIG); - PyModule_AddIntMacro(m, PyBUF_CONTIG_RO); - - PyModule_AddIntMacro(m, PyBUF_READ); - PyModule_AddIntMacro(m, PyBUF_WRITE); - - return m; -} + if (simple_format == NULL) { + return -1; + } +#define ADD_INT_MACRO(mod, macro) \ + do { \ + if (PyModule_AddIntConstant(mod, #macro, macro) < 0) { \ + return -1; \ + } \ + } while (0) + + ADD_INT_MACRO(mod, ND_MAX_NDIM); + ADD_INT_MACRO(mod, ND_VAREXPORT); + ADD_INT_MACRO(mod, ND_WRITABLE); + ADD_INT_MACRO(mod, ND_FORTRAN); + ADD_INT_MACRO(mod, ND_SCALAR); + ADD_INT_MACRO(mod, ND_PIL); + ADD_INT_MACRO(mod, ND_GETBUF_FAIL); + ADD_INT_MACRO(mod, ND_GETBUF_UNDEFINED); + ADD_INT_MACRO(mod, ND_REDIRECT); + + ADD_INT_MACRO(mod, PyBUF_SIMPLE); + ADD_INT_MACRO(mod, PyBUF_WRITABLE); + ADD_INT_MACRO(mod, PyBUF_FORMAT); + ADD_INT_MACRO(mod, PyBUF_ND); + ADD_INT_MACRO(mod, PyBUF_STRIDES); + ADD_INT_MACRO(mod, PyBUF_INDIRECT); + ADD_INT_MACRO(mod, PyBUF_C_CONTIGUOUS); + ADD_INT_MACRO(mod, PyBUF_F_CONTIGUOUS); + ADD_INT_MACRO(mod, PyBUF_ANY_CONTIGUOUS); + ADD_INT_MACRO(mod, PyBUF_FULL); + ADD_INT_MACRO(mod, PyBUF_FULL_RO); + ADD_INT_MACRO(mod, PyBUF_RECORDS); + ADD_INT_MACRO(mod, PyBUF_RECORDS_RO); + ADD_INT_MACRO(mod, PyBUF_STRIDED); + ADD_INT_MACRO(mod, PyBUF_STRIDED_RO); + ADD_INT_MACRO(mod, PyBUF_CONTIG); + ADD_INT_MACRO(mod, PyBUF_CONTIG_RO); + + ADD_INT_MACRO(mod, PyBUF_READ); + ADD_INT_MACRO(mod, PyBUF_WRITE); + +#undef ADD_INT_MACRO + return 0; +} +PyMODINIT_FUNC +PyInit__testbuffer(void) +{ + PyObject *mod = PyModule_Create(&_testbuffermodule); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif + if (_testbuffer_exec(mod) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index a8ba009eb6a54b..8c2c7137cdce40 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -2,59 +2,6 @@ #include "util.h" -static PyObject * -object_repr(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_Repr(arg); -} - -static PyObject * -object_ascii(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_ASCII(arg); -} - -static PyObject * -object_str(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_Str(arg); -} - -static PyObject * -object_bytes(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_Bytes(arg); -} - -static PyObject * -object_getattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - return PyObject_GetAttr(obj, attr_name); -} - -static PyObject * -object_getattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { - return NULL; - } - NULLABLE(obj); - return PyObject_GetAttrString(obj, attr_name); -} - static PyObject * object_getoptionalattr(PyObject *self, PyObject *args) { @@ -106,31 +53,6 @@ object_getoptionalattrstring(PyObject *self, PyObject *args) } } -static PyObject * -object_hasattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); -} - -static PyObject * -object_hasattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { - return NULL; - } - NULLABLE(obj); - return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); -} - static PyObject * object_hasattrwitherror(PyObject *self, PyObject *args) { @@ -157,121 +79,17 @@ object_hasattrstringwitherror(PyObject *self, PyObject *args) } static PyObject * -object_setattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name, *value; - if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - NULLABLE(value); - RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); -} - -static PyObject * -object_setattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj, *value; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); -} - -static PyObject * -object_delattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; -if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - RETURN_INT(PyObject_DelAttr(obj, attr_name)); -} - -static PyObject * -object_delattrstring(PyObject *self, PyObject *args) +mapping_getoptionalitemstring(PyObject *self, PyObject *args) { - PyObject *obj; + PyObject *obj, *value = UNINITIALIZED_PTR; const char *attr_name; Py_ssize_t size; if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { return NULL; } NULLABLE(obj); - RETURN_INT(PyObject_DelAttrString(obj, attr_name)); -} -static PyObject * -number_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyBool_FromLong(PyNumber_Check(obj)); -} - -static PyObject * -mapping_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyMapping_Check(obj)); -} - -static PyObject * -mapping_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyMapping_Size(obj)); -} - -static PyObject * -mapping_length(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyMapping_Length(obj)); -} - -static PyObject * -object_getitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - return PyObject_GetItem(mapping, key); -} - -static PyObject * -mapping_getitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - return PyMapping_GetItemString(mapping, key); -} - -static PyObject * -mapping_getoptionalitem(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - - switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { + switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { case -1: assert(value == NULL); return NULL; @@ -281,23 +99,22 @@ mapping_getoptionalitem(PyObject *self, PyObject *args) case 1: return value; default: - Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); + Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); Py_UNREACHABLE(); } } static PyObject * -mapping_getoptionalitemstring(PyObject *self, PyObject *args) +mapping_getoptionalitem(PyObject *self, PyObject *args) { - PyObject *obj, *value = UNINITIALIZED_PTR; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { return NULL; } NULLABLE(obj); + NULLABLE(attr_name); - switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { + switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { case -1: assert(value == NULL); return NULL; @@ -307,400 +124,49 @@ mapping_getoptionalitemstring(PyObject *self, PyObject *args) case 1: return value; default: - Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); + Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); Py_UNREACHABLE(); } } static PyObject * -mapping_haskey(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - return PyLong_FromLong(PyMapping_HasKey(mapping, key)); -} - -static PyObject * -mapping_haskeystring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); -} - -static PyObject * -mapping_haskeywitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyMapping_HasKeyWithError(mapping, key)); -} - -static PyObject * -mapping_haskeystringwitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyMapping_HasKeyStringWithError(mapping, key)); -} - -static PyObject * -object_setitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key, *value; - if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - NULLABLE(value); - RETURN_INT(PyObject_SetItem(mapping, key, value)); -} - -static PyObject * -mapping_setitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping, *value; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(value); - RETURN_INT(PyMapping_SetItemString(mapping, key, value)); -} - -static PyObject * -object_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyObject_DelItem(mapping, key)); -} - -static PyObject * -mapping_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyMapping_DelItem(mapping, key)); -} - -static PyObject * -mapping_delitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyMapping_DelItemString(mapping, key)); -} - -static PyObject * -mapping_keys(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Keys(obj); -} - -static PyObject * -mapping_values(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Values(obj); -} - -static PyObject * -mapping_items(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Items(obj); -} - - -static PyObject * -sequence_check(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PySequence_Check(obj)); -} - -static PyObject * -sequence_size(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySequence_Size(obj)); -} - -static PyObject * -sequence_length(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySequence_Length(obj)); -} - -static PyObject * -sequence_concat(PyObject *self, PyObject *args) -{ - PyObject *seq1, *seq2; - if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { - return NULL; - } - NULLABLE(seq1); - NULLABLE(seq2); - - return PySequence_Concat(seq1, seq2); -} - -static PyObject * -sequence_repeat(PyObject *self, PyObject *args) +pyiter_next(PyObject *self, PyObject *iter) { - PyObject *seq; - Py_ssize_t count; - if (!PyArg_ParseTuple(args, "On", &seq, &count)) { - return NULL; + PyObject *item = PyIter_Next(iter); + if (item == NULL && !PyErr_Occurred()) { + Py_RETURN_NONE; } - NULLABLE(seq); - - return PySequence_Repeat(seq, count); + return item; } static PyObject * -sequence_inplaceconcat(PyObject *self, PyObject *args) +pyiter_nextitem(PyObject *self, PyObject *iter) { - PyObject *seq1, *seq2; - if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + PyObject *item; + int rc = PyIter_NextItem(iter, &item); + if (rc < 0) { + assert(PyErr_Occurred()); + assert(item == NULL); return NULL; } - NULLABLE(seq1); - NULLABLE(seq2); - - return PySequence_InPlaceConcat(seq1, seq2); -} - -static PyObject * -sequence_inplacerepeat(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t count; - if (!PyArg_ParseTuple(args, "On", &seq, &count)) { - return NULL; + assert(!PyErr_Occurred()); + if (item == NULL) { + Py_RETURN_NONE; } - NULLABLE(seq); - - return PySequence_InPlaceRepeat(seq, count); -} - -static PyObject * -sequence_getitem(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - NULLABLE(seq); - - return PySequence_GetItem(seq, i); -} - -static PyObject * -sequence_setitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq, *val; - if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(val); - - RETURN_INT(PySequence_SetItem(seq, i, val)); -} - - -static PyObject * -sequence_delitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - NULLABLE(seq); - - RETURN_INT(PySequence_DelItem(seq, i)); -} - -static PyObject * -sequence_setslice(PyObject* self, PyObject *args) -{ - PyObject *sequence, *obj; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { - return NULL; - } - NULLABLE(sequence); - NULLABLE(obj); - - RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); -} - -static PyObject * -sequence_delslice(PyObject *self, PyObject *args) -{ - PyObject *sequence; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { - return NULL; - } - NULLABLE(sequence); - - RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); -} - -static PyObject * -sequence_count(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_SIZE(PySequence_Count(seq, value)); -} - -static PyObject * -sequence_contains(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_INT(PySequence_Contains(seq, value)); -} - -static PyObject * -sequence_index(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_SIZE(PySequence_Index(seq, value)); -} - -static PyObject * -sequence_list(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySequence_List(obj); -} - -static PyObject * -sequence_tuple(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySequence_Tuple(obj); + return item; } static PyMethodDef test_methods[] = { - {"object_repr", object_repr, METH_O}, - {"object_ascii", object_ascii, METH_O}, - {"object_str", object_str, METH_O}, - {"object_bytes", object_bytes, METH_O}, - - {"object_getattr", object_getattr, METH_VARARGS}, - {"object_getattrstring", object_getattrstring, METH_VARARGS}, {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, {"object_getoptionalattrstring", object_getoptionalattrstring, METH_VARARGS}, - {"object_hasattr", object_hasattr, METH_VARARGS}, - {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, {"object_hasattrwitherror", object_hasattrwitherror, METH_VARARGS}, {"object_hasattrstringwitherror", object_hasattrstringwitherror, METH_VARARGS}, - {"object_setattr", object_setattr, METH_VARARGS}, - {"object_setattrstring", object_setattrstring, METH_VARARGS}, - {"object_delattr", object_delattr, METH_VARARGS}, - {"object_delattrstring", object_delattrstring, METH_VARARGS}, - - {"number_check", number_check, METH_O}, - {"mapping_check", mapping_check, METH_O}, - {"mapping_size", mapping_size, METH_O}, - {"mapping_length", mapping_length, METH_O}, - {"object_getitem", object_getitem, METH_VARARGS}, - {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, {"mapping_getoptionalitem", mapping_getoptionalitem, METH_VARARGS}, {"mapping_getoptionalitemstring", mapping_getoptionalitemstring, METH_VARARGS}, - {"mapping_haskey", mapping_haskey, METH_VARARGS}, - {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, - {"mapping_haskeywitherror", mapping_haskeywitherror, METH_VARARGS}, - {"mapping_haskeystringwitherror", mapping_haskeystringwitherror, METH_VARARGS}, - {"object_setitem", object_setitem, METH_VARARGS}, - {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, - {"object_delitem", object_delitem, METH_VARARGS}, - {"mapping_delitem", mapping_delitem, METH_VARARGS}, - {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, - {"mapping_keys", mapping_keys, METH_O}, - {"mapping_values", mapping_values, METH_O}, - {"mapping_items", mapping_items, METH_O}, - - {"sequence_check", sequence_check, METH_O}, - {"sequence_size", sequence_size, METH_O}, - {"sequence_length", sequence_length, METH_O}, - {"sequence_concat", sequence_concat, METH_VARARGS}, - {"sequence_repeat", sequence_repeat, METH_VARARGS}, - {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, - {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, - {"sequence_getitem", sequence_getitem, METH_VARARGS}, - {"sequence_setitem", sequence_setitem, METH_VARARGS}, - {"sequence_delitem", sequence_delitem, METH_VARARGS}, - {"sequence_setslice", sequence_setslice, METH_VARARGS}, - {"sequence_delslice", sequence_delslice, METH_VARARGS}, - {"sequence_count", sequence_count, METH_VARARGS}, - {"sequence_contains", sequence_contains, METH_VARARGS}, - {"sequence_index", sequence_index, METH_VARARGS}, - {"sequence_list", sequence_list, METH_O}, - {"sequence_tuple", sequence_tuple, METH_O}, + {"PyIter_Next", pyiter_next, METH_O}, + {"PyIter_NextItem", pyiter_nextitem, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index da10503f6f6856..33903de14ba68d 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -2,245 +2,58 @@ #include "util.h" -/* Test PyBytes_Check() */ +/* Test _PyBytes_Resize() */ static PyObject * -bytes_check(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyBytes_Check(obj)); -} - -/* Test PyBytes_CheckExact() */ -static PyObject * -bytes_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyBytes_CheckExact(obj)); -} - -/* Test PyBytes_FromStringAndSize() */ -static PyObject * -bytes_fromstringandsize(PyObject *Py_UNUSED(module), PyObject *args) -{ - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; - - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { - return NULL; - } - - if (size == -100) { - size = bsize; - } - return PyBytes_FromStringAndSize(s, size); -} - -/* Test PyBytes_FromString() */ -static PyObject * -bytes_fromstring(PyObject *Py_UNUSED(module), PyObject *arg) -{ - const char *s; - Py_ssize_t size; - - if (!PyArg_Parse(arg, "z#", &s, &size)) { - return NULL; - } - return PyBytes_FromString(s); -} - -/* Test PyBytes_FromObject() */ -static PyObject * -bytes_fromobject(PyObject *Py_UNUSED(module), PyObject *arg) -{ - NULLABLE(arg); - return PyBytes_FromObject(arg); -} - -/* Test PyBytes_Size() */ -static PyObject * -bytes_size(PyObject *Py_UNUSED(module), PyObject *arg) -{ - NULLABLE(arg); - RETURN_SIZE(PyBytes_Size(arg)); -} - -/* Test PyUnicode_AsString() */ -static PyObject * -bytes_asstring(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t buflen; - const char *s; - - if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) - return NULL; - - NULLABLE(obj); - s = PyBytes_AsString(obj); - if (s == NULL) - return NULL; - - return PyBytes_FromStringAndSize(s, buflen); -} - -/* Test PyBytes_AsStringAndSize() */ -static PyObject * -bytes_asstringandsize(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t buflen; - char *s = UNINITIALIZED_PTR; - Py_ssize_t size = UNINITIALIZED_SIZE; - - if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) - return NULL; - - NULLABLE(obj); - if (PyBytes_AsStringAndSize(obj, &s, &size) < 0) { - return NULL; - } - - if (s == NULL) { - return Py_BuildValue("(On)", Py_None, size); - } - else { - return Py_BuildValue("(y#n)", s, buflen, size); - } -} - -static PyObject * -bytes_asstringandsize_null(PyObject *Py_UNUSED(module), PyObject *args) +bytes_resize(PyObject *Py_UNUSED(module), PyObject *args) { PyObject *obj; - Py_ssize_t buflen; - char *s = UNINITIALIZED_PTR; + Py_ssize_t newsize; + int new; - if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) - return NULL; - - NULLABLE(obj); - if (PyBytes_AsStringAndSize(obj, &s, NULL) < 0) { - return NULL; - } - - if (s == NULL) { - Py_RETURN_NONE; - } - else { - return PyBytes_FromStringAndSize(s, buflen); - } -} - -/* Test PyBytes_Repr() */ -static PyObject * -bytes_repr(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - int smartquotes; - if (!PyArg_ParseTuple(args, "Oi", &obj, &smartquotes)) + if (!PyArg_ParseTuple(args, "Onp", &obj, &newsize, &new)) return NULL; NULLABLE(obj); - return PyBytes_Repr(obj, smartquotes); -} - -/* Test PyBytes_Concat() */ -static PyObject * -bytes_concat(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *left, *right; - int new = 0; - - if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) - return NULL; - - NULLABLE(left); - NULLABLE(right); if (new) { - assert(left != NULL); - assert(PyBytes_CheckExact(left)); - left = PyBytes_FromStringAndSize(PyBytes_AS_STRING(left), - PyBytes_GET_SIZE(left)); - if (left == NULL) { + assert(obj != NULL); + assert(PyBytes_CheckExact(obj)); + PyObject *newobj = PyBytes_FromStringAndSize(NULL, PyBytes_Size(obj)); + if (newobj == NULL) { return NULL; } + memcpy(PyBytes_AsString(newobj), PyBytes_AsString(obj), PyBytes_Size(obj)); + obj = newobj; } else { - Py_XINCREF(left); + Py_XINCREF(obj); } - PyBytes_Concat(&left, right); - if (left == NULL && !PyErr_Occurred()) { - Py_RETURN_NONE; - } - return left; -} - -/* Test PyBytes_ConcatAndDel() */ -static PyObject * -bytes_concatanddel(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *left, *right; - int new = 0; - - if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - if (new) { - assert(left != NULL); - assert(PyBytes_CheckExact(left)); - left = PyBytes_FromStringAndSize(PyBytes_AS_STRING(left), - PyBytes_GET_SIZE(left)); - if (left == NULL) { - return NULL; - } + if (_PyBytes_Resize(&obj, newsize) < 0) { + assert(obj == NULL); } else { - Py_XINCREF(left); - } - Py_XINCREF(right); - PyBytes_ConcatAndDel(&left, right); - if (left == NULL && !PyErr_Occurred()) { - Py_RETURN_NONE; + assert(obj != NULL); } - return left; + return obj; } -/* Test PyBytes_DecodeEscape() */ + +/* Test PyBytes_Join() */ static PyObject * -bytes_decodeescape(PyObject *Py_UNUSED(module), PyObject *args) +bytes_join(PyObject *Py_UNUSED(module), PyObject *args) { - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "z#|zn", &s, &bsize, &errors, &size)) + PyObject *sep, *iterable; + if (!PyArg_ParseTuple(args, "OO", &sep, &iterable)) { return NULL; - - if (size == -100) { - size = bsize; } - return PyBytes_DecodeEscape(s, size, errors, 0, NULL); + NULLABLE(sep); + NULLABLE(iterable); + return PyBytes_Join(sep, iterable); } static PyMethodDef test_methods[] = { - {"bytes_check", bytes_check, METH_O}, - {"bytes_checkexact", bytes_checkexact, METH_O}, - {"bytes_fromstringandsize", bytes_fromstringandsize, METH_VARARGS}, - {"bytes_fromstring", bytes_fromstring, METH_O}, - {"bytes_fromobject", bytes_fromobject, METH_O}, - {"bytes_size", bytes_size, METH_O}, - {"bytes_asstring", bytes_asstring, METH_VARARGS}, - {"bytes_asstringandsize", bytes_asstringandsize, METH_VARARGS}, - {"bytes_asstringandsize_null", bytes_asstringandsize_null, METH_VARARGS}, - {"bytes_repr", bytes_repr, METH_VARARGS}, - {"bytes_concat", bytes_concat, METH_VARARGS}, - {"bytes_concatanddel", bytes_concatanddel, METH_VARARGS}, - {"bytes_decodeescape", bytes_decodeescape, METH_VARARGS}, + {"bytes_resize", bytes_resize, METH_VARARGS}, + {"bytes_join", bytes_join, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/clinic/long.c.h b/Modules/_testcapi/clinic/long.c.h index e2f7042be12c48..767c671abb8eae 100644 --- a/Modules/_testcapi/clinic/long.c.h +++ b/Modules/_testcapi/clinic/long.c.h @@ -2,137 +2,6 @@ preserve [clinic start generated code]*/ -PyDoc_STRVAR(_testcapi_test_long_api__doc__, -"test_long_api($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_API_METHODDEF \ - {"test_long_api", (PyCFunction)_testcapi_test_long_api, METH_NOARGS, _testcapi_test_long_api__doc__}, - -static PyObject * -_testcapi_test_long_api_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_api_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_longlong_api__doc__, -"test_longlong_api($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONGLONG_API_METHODDEF \ - {"test_longlong_api", (PyCFunction)_testcapi_test_longlong_api, METH_NOARGS, _testcapi_test_longlong_api__doc__}, - -static PyObject * -_testcapi_test_longlong_api_impl(PyObject *module); - -static PyObject * -_testcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_longlong_api_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_and_overflow__doc__, -"test_long_and_overflow($module, /)\n" -"--\n" -"\n" -"Test the PyLong_AsLongAndOverflow API.\n" -"\n" -"General conversion to PY_LONG is tested by test_long_api_inner.\n" -"This test will concentrate on proper handling of overflow."); - -#define _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ - {"test_long_and_overflow", (PyCFunction)_testcapi_test_long_and_overflow, METH_NOARGS, _testcapi_test_long_and_overflow__doc__}, - -static PyObject * -_testcapi_test_long_and_overflow_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_and_overflow_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_long_and_overflow__doc__, -"test_long_long_and_overflow($module, /)\n" -"--\n" -"\n" -"Test the PyLong_AsLongLongAndOverflow API.\n" -"\n" -"General conversion to long long is tested by test_long_api_inner.\n" -"This test will concentrate on proper handling of overflow."); - -#define _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ - {"test_long_long_and_overflow", (PyCFunction)_testcapi_test_long_long_and_overflow, METH_NOARGS, _testcapi_test_long_long_and_overflow__doc__}, - -static PyObject * -_testcapi_test_long_long_and_overflow_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_long_and_overflow_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_size_t__doc__, -"test_long_as_size_t($module, /)\n" -"--\n" -"\n" -"Test the PyLong_As{Size,Ssize}_t API.\n" -"\n" -"At present this just tests that non-integer arguments are handled correctly.\n" -"It should be extended to test overflow handling."); - -#define _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ - {"test_long_as_size_t", (PyCFunction)_testcapi_test_long_as_size_t, METH_NOARGS, _testcapi_test_long_as_size_t__doc__}, - -static PyObject * -_testcapi_test_long_as_size_t_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_size_t_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_unsigned_long_long_mask__doc__, -"test_long_as_unsigned_long_long_mask($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ - {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testcapi_test_long_as_unsigned_long_long_mask__doc__}, - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_unsigned_long_long_mask_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_double__doc__, -"test_long_as_double($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ - {"test_long_as_double", (PyCFunction)_testcapi_test_long_as_double, METH_NOARGS, _testcapi_test_long_as_double__doc__}, - -static PyObject * -_testcapi_test_long_as_double_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_double_impl(module); -} - PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, "call_long_compact_api($module, arg, /)\n" "--\n" @@ -140,12 +9,4 @@ PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, #define _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF \ {"call_long_compact_api", (PyCFunction)_testcapi_call_long_compact_api, METH_O, _testcapi_call_long_compact_api__doc__}, - -PyDoc_STRVAR(_testcapi_PyLong_AsInt__doc__, -"PyLong_AsInt($module, arg, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_PYLONG_ASINT_METHODDEF \ - {"PyLong_AsInt", (PyCFunction)_testcapi_PyLong_AsInt, METH_O, _testcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=de762870526e241d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ddcbc6ea06e5e21 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/vectorcall_limited.c.h b/Modules/_testcapi/clinic/vectorcall_limited.c.h deleted file mode 100644 index a233aefec79ecd..00000000000000 --- a/Modules/_testcapi/clinic/vectorcall_limited.c.h +++ /dev/null @@ -1,20 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -PyDoc_STRVAR(_testcapi_call_vectorcall__doc__, -"call_vectorcall($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHODDEF \ - {"call_vectorcall", (PyCFunction)_testcapi_call_vectorcall, METH_O, _testcapi_call_vectorcall__doc__}, - -PyDoc_STRVAR(_testcapi_call_vectorcall_method__doc__, -"call_vectorcall_method($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ - {"call_vectorcall_method", (PyCFunction)_testcapi_call_vectorcall_method, METH_O, _testcapi_call_vectorcall_method__doc__}, -/*[clinic end generated code: output=e980906a39602528 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/codec.c b/Modules/_testcapi/codec.c index d13f51e20331a1..e27e64e066c458 100644 --- a/Modules/_testcapi/codec.c +++ b/Modules/_testcapi/codec.c @@ -1,15 +1,219 @@ #include "parts.h" #include "util.h" +// === Codecs registration and un-registration ================================ + +static PyObject * +codec_register(PyObject *Py_UNUSED(module), PyObject *search_function) +{ + if (PyCodec_Register(search_function) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_unregister(PyObject *Py_UNUSED(module), PyObject *search_function) +{ + if (PyCodec_Unregister(search_function) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_known_encoding(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_KnownEncoding(encoding) ? Py_True : Py_False; +} + +// === Codecs encoding and decoding interfaces ================================ + +static PyObject * +codec_encode(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *input; + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "O|zz", &input, &encoding, &errors)) { + return NULL; + } + return PyCodec_Encode(input, encoding, errors); +} + +static PyObject * +codec_decode(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *input; + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "O|zz", &input, &encoding, &errors)) { + return NULL; + } + return PyCodec_Decode(input, encoding, errors); +} + +static PyObject * +codec_encoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_Encoder(encoding); +} + +static PyObject * +codec_decoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_Decoder(encoding); +} + +static PyObject * +codec_incremental_encoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zz", &encoding, &errors)) { + return NULL; + } + return PyCodec_IncrementalEncoder(encoding, errors); +} + +static PyObject * +codec_incremental_decoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zz", &encoding, &errors)) { + return NULL; + } + return PyCodec_IncrementalDecoder(encoding, errors); +} + +static PyObject * +codec_stream_reader(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + PyObject *stream; + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zOz", &encoding, &stream, &errors)) { + return NULL; + } + return PyCodec_StreamReader(encoding, stream, errors); +} + +static PyObject * +codec_stream_writer(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + PyObject *stream; + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zOz", &encoding, &stream, &errors)) { + return NULL; + } + return PyCodec_StreamWriter(encoding, stream, errors); +} + +// === Codecs errors handlers ================================================= + +static PyObject * +codec_register_error(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *encoding; // must not be NULL + PyObject *error; + if (!PyArg_ParseTuple(args, "sO", &encoding, &error)) { + return NULL; + } + if (PyCodec_RegisterError(encoding, error) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_lookup_error(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_LookupError(encoding); +} + +static PyObject * +codec_strict_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_StrictErrors(exc); +} + +static PyObject * +codec_ignore_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_IgnoreErrors(exc); +} + +static PyObject * +codec_replace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_ReplaceErrors(exc); +} + +static PyObject * +codec_xmlcharrefreplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_XMLCharRefReplaceErrors(exc); +} + +static PyObject * +codec_backslashreplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_BackslashReplaceErrors(exc); +} static PyMethodDef test_methods[] = { - {NULL}, + /* codecs registration */ + {"codec_register", codec_register, METH_O}, + {"codec_unregister", codec_unregister, METH_O}, + {"codec_known_encoding", codec_known_encoding, METH_VARARGS}, + /* encoding and decoding interface */ + {"codec_encode", codec_encode, METH_VARARGS}, + {"codec_decode", codec_decode, METH_VARARGS}, + {"codec_encoder", codec_encoder, METH_VARARGS}, + {"codec_decoder", codec_decoder, METH_VARARGS}, + {"codec_incremental_encoder", codec_incremental_encoder, METH_VARARGS}, + {"codec_incremental_decoder", codec_incremental_decoder, METH_VARARGS}, + {"codec_stream_reader", codec_stream_reader, METH_VARARGS}, + {"codec_stream_writer", codec_stream_writer, METH_VARARGS}, + /* error handling */ + {"codec_register_error", codec_register_error, METH_VARARGS}, + {"codec_lookup_error", codec_lookup_error, METH_VARARGS}, + {"codec_strict_errors", codec_strict_errors, METH_O}, + {"codec_ignore_errors", codec_ignore_errors, METH_O}, + {"codec_replace_errors", codec_replace_errors, METH_O}, + {"codec_xmlcharrefreplace_errors", codec_xmlcharrefreplace_errors, METH_O}, + {"codec_backslashreplace_errors", codec_backslashreplace_errors, METH_O}, + // PyCodec_NameReplaceErrors() is tested in _testlimitedcapi/codec.c + {NULL, NULL, 0, NULL}, }; int _PyTestCapi_Init_Codec(PyObject *m) { - if (PyModule_AddFunctions(m, test_methods) < 0){ + if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; } diff --git a/Modules/_testcapi/complex.c b/Modules/_testcapi/complex.c index 4a70217eb90d62..eceb1310bfe874 100644 --- a/Modules/_testcapi/complex.c +++ b/Modules/_testcapi/complex.c @@ -2,20 +2,6 @@ #include "util.h" -static PyObject * -complex_check(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyComplex_Check(obj)); -} - -static PyObject * -complex_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyComplex_CheckExact(obj)); -} - static PyObject * complex_fromccomplex(PyObject *Py_UNUSED(module), PyObject *obj) { @@ -28,48 +14,6 @@ complex_fromccomplex(PyObject *Py_UNUSED(module), PyObject *obj) return PyComplex_FromCComplex(complex); } -static PyObject * -complex_fromdoubles(PyObject *Py_UNUSED(module), PyObject *args) -{ - double real, imag; - - if (!PyArg_ParseTuple(args, "dd", &real, &imag)) { - return NULL; - } - - return PyComplex_FromDoubles(real, imag); -} - -static PyObject * -complex_realasdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double real; - - NULLABLE(obj); - real = PyComplex_RealAsDouble(obj); - - if (real == -1. && PyErr_Occurred()) { - return NULL; - } - - return PyFloat_FromDouble(real); -} - -static PyObject * -complex_imagasdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double imag; - - NULLABLE(obj); - imag = PyComplex_ImagAsDouble(obj); - - if (imag == -1. && PyErr_Occurred()) { - return NULL; - } - - return PyFloat_FromDouble(imag); -} - static PyObject * complex_asccomplex(PyObject *Py_UNUSED(module), PyObject *obj) { @@ -139,12 +83,7 @@ _py_c_abs(PyObject *Py_UNUSED(module), PyObject* obj) static PyMethodDef test_methods[] = { - {"complex_check", complex_check, METH_O}, - {"complex_checkexact", complex_checkexact, METH_O}, {"complex_fromccomplex", complex_fromccomplex, METH_O}, - {"complex_fromdoubles", complex_fromdoubles, METH_VARARGS}, - {"complex_realasdouble", complex_realasdouble, METH_O}, - {"complex_imagasdouble", complex_imagasdouble, METH_O}, {"complex_asccomplex", complex_asccomplex, METH_O}, {"_py_c_sum", _py_c_sum, METH_VARARGS}, {"_py_c_diff", _py_c_diff, METH_VARARGS}, diff --git a/Modules/_testcapi/config.c b/Modules/_testcapi/config.c new file mode 100644 index 00000000000000..bb3b7e88953a53 --- /dev/null +++ b/Modules/_testcapi/config.c @@ -0,0 +1,68 @@ +#include "parts.h" + + +static PyObject * +_testcapi_config_get(PyObject *module, PyObject *name_obj) +{ + const char *name; + if (PyArg_Parse(name_obj, "s", &name) < 0) { + return NULL; + } + + return PyConfig_Get(name); +} + + +static PyObject * +_testcapi_config_getint(PyObject *module, PyObject *name_obj) +{ + const char *name; + if (PyArg_Parse(name_obj, "s", &name) < 0) { + return NULL; + } + + int value; + if (PyConfig_GetInt(name, &value) < 0) { + return NULL; + } + return PyLong_FromLong(value); +} + + +static PyObject * +_testcapi_config_names(PyObject *module, PyObject* Py_UNUSED(args)) +{ + return PyConfig_Names(); +} + + +static PyObject * +_testcapi_config_set(PyObject *module, PyObject *args) +{ + const char *name; + PyObject *value; + if (PyArg_ParseTuple(args, "sO", &name, &value) < 0) { + return NULL; + } + + int res = PyConfig_Set(name, value); + if (res < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyMethodDef test_methods[] = { + {"config_get", _testcapi_config_get, METH_O}, + {"config_getint", _testcapi_config_getint, METH_O}, + {"config_names", _testcapi_config_names, METH_NOARGS}, + {"config_set", _testcapi_config_set, METH_VARARGS}, + {NULL} +}; + +int +_PyTestCapi_Init_Config(PyObject *mod) +{ + return PyModule_AddFunctions(mod, test_methods); +} diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index b1796039f0d83a..b800f9b8eb3473 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -22,10 +22,17 @@ test_datetime_capi(PyObject *self, PyObject *args) test_run_counter++; PyDateTime_IMPORT; - if (PyDateTimeAPI) { - Py_RETURN_NONE; + if (PyDateTimeAPI == NULL) { + return NULL; } - return NULL; + // The following C API types need to outlive interpreters, since the + // borrowed references to them can be held by users without being updated. + assert(!PyType_HasFeature(PyDateTimeAPI->DateType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DateTimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DeltaType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TZInfoType, Py_TPFLAGS_HEAPTYPE)); + Py_RETURN_NONE; } /* Functions exposing the C API type checking for testing */ @@ -122,7 +129,7 @@ static PyObject * get_timezones_offset_zero(PyObject *self, PyObject *args) { PyObject *offset = PyDelta_FromDSU(0, 0, 0); - PyObject *name = PyUnicode_FromString(""); + PyObject *name = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (offset == NULL || name == NULL) { Py_XDECREF(offset); Py_XDECREF(name); @@ -479,3 +486,38 @@ _PyTestCapi_Init_DateTime(PyObject *mod) } return 0; } + + +/* --------------------------------------------------------------------------- + * Test module for subinterpreters. + */ + +static int +_testcapi_datetime_exec(PyObject *mod) +{ + if (test_datetime_capi(NULL, NULL) == NULL) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot _testcapi_datetime_slots[] = { + {Py_mod_exec, _testcapi_datetime_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static struct PyModuleDef _testcapi_datetime_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_testcapi_datetime", + .m_size = 0, + .m_methods = test_methods, + .m_slots = _testcapi_datetime_slots, +}; + +PyMODINIT_FUNC +PyInit__testcapi_datetime(void) +{ + return PyModuleDef_Init(&_testcapi_datetime_module); +} diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index fe03c24f75e196..307797f98f12ae 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -1,60 +1,6 @@ #include "parts.h" #include "util.h" - -static PyObject * -dict_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyDict_Check(obj)); -} - -static PyObject * -dict_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyDict_CheckExact(obj)); -} - -static PyObject * -dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyDict_New(); -} - -static PyObject * -dictproxy_new(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDictProxy_New(obj); -} - -static PyObject * -dict_clear(PyObject *self, PyObject *obj) -{ - PyDict_Clear(obj); - Py_RETURN_NONE; -} - -static PyObject * -dict_copy(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Copy(obj); -} - -static PyObject * -dict_contains(PyObject *self, PyObject *args) -{ - PyObject *obj, *key; - if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(key); - RETURN_INT(PyDict_Contains(obj, key)); -} - static PyObject * dict_containsstring(PyObject *self, PyObject *args) { @@ -68,72 +14,6 @@ dict_containsstring(PyObject *self, PyObject *args) RETURN_INT(PyDict_ContainsString(obj, key)); } -static PyObject * -dict_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyDict_Size(obj)); -} - -static PyObject * -dict_getitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - PyObject *value = PyDict_GetItem(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - -static PyObject * -dict_getitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - PyObject *value = PyDict_GetItemString(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - -static PyObject * -dict_getitemwitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - PyObject *value = PyDict_GetItemWithError(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - - static PyObject * dict_getitemref(PyObject *self, PyObject *args) { @@ -185,33 +65,6 @@ dict_getitemstringref(PyObject *self, PyObject *args) } } -static PyObject * -dict_setitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key, *value; - if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - NULLABLE(value); - RETURN_INT(PyDict_SetItem(mapping, key, value)); -} - -static PyObject * -dict_setitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping, *value; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(value); - RETURN_INT(PyDict_SetItemString(mapping, key, value)); -} - static PyObject * dict_setdefault(PyObject *self, PyObject *args) { @@ -250,112 +103,6 @@ dict_setdefaultref(PyObject *self, PyObject *args) } } -static PyObject * -dict_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyDict_DelItem(mapping, key)); -} - -static PyObject * -dict_delitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyDict_DelItemString(mapping, key)); -} - -static PyObject * -dict_keys(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Keys(obj); -} - -static PyObject * -dict_values(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Values(obj); -} - -static PyObject * -dict_items(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Items(obj); -} - -static PyObject * -dict_next(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR; - Py_ssize_t pos; - if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { - return NULL; - } - NULLABLE(mapping); - int rc = PyDict_Next(mapping, &pos, &key, &value); - if (rc != 0) { - return Py_BuildValue("inOO", rc, pos, key, value); - } - assert(key == UNINITIALIZED_PTR); - assert(value == UNINITIALIZED_PTR); - if (PyErr_Occurred()) { - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -dict_merge(PyObject *self, PyObject *args) -{ - PyObject *mapping, *mapping2; - int override; - if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(mapping2); - RETURN_INT(PyDict_Merge(mapping, mapping2, override)); -} - -static PyObject * -dict_update(PyObject *self, PyObject *args) -{ - PyObject *mapping, *mapping2; - if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(mapping2); - RETURN_INT(PyDict_Update(mapping, mapping2)); -} - -static PyObject * -dict_mergefromseq2(PyObject *self, PyObject *args) -{ - PyObject *mapping, *seq; - int override; - if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(seq); - RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); -} - - static PyObject * dict_pop(PyObject *self, PyObject *args) { @@ -382,7 +129,6 @@ dict_pop(PyObject *self, PyObject *args) return Py_BuildValue("iN", res, result); } - static PyObject * dict_pop_null(PyObject *self, PyObject *args) { @@ -396,7 +142,6 @@ dict_pop_null(PyObject *self, PyObject *args) RETURN_INT(PyDict_Pop(dict, key, NULL)); } - static PyObject * dict_popstring(PyObject *self, PyObject *args) { @@ -423,7 +168,6 @@ dict_popstring(PyObject *self, PyObject *args) return Py_BuildValue("iN", res, result); } - static PyObject * dict_popstring_null(PyObject *self, PyObject *args) { @@ -437,35 +181,12 @@ dict_popstring_null(PyObject *self, PyObject *args) RETURN_INT(PyDict_PopString(dict, key, NULL)); } - static PyMethodDef test_methods[] = { - {"dict_check", dict_check, METH_O}, - {"dict_checkexact", dict_checkexact, METH_O}, - {"dict_new", dict_new, METH_NOARGS}, - {"dictproxy_new", dictproxy_new, METH_O}, - {"dict_clear", dict_clear, METH_O}, - {"dict_copy", dict_copy, METH_O}, - {"dict_size", dict_size, METH_O}, - {"dict_getitem", dict_getitem, METH_VARARGS}, - {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, - {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_containsstring", dict_containsstring, METH_VARARGS}, {"dict_getitemref", dict_getitemref, METH_VARARGS}, {"dict_getitemstringref", dict_getitemstringref, METH_VARARGS}, - {"dict_contains", dict_contains, METH_VARARGS}, - {"dict_containsstring", dict_containsstring, METH_VARARGS}, - {"dict_setitem", dict_setitem, METH_VARARGS}, - {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, - {"dict_delitem", dict_delitem, METH_VARARGS}, - {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, {"dict_setdefault", dict_setdefault, METH_VARARGS}, {"dict_setdefaultref", dict_setdefaultref, METH_VARARGS}, - {"dict_keys", dict_keys, METH_O}, - {"dict_values", dict_values, METH_O}, - {"dict_items", dict_items, METH_O}, - {"dict_next", dict_next, METH_VARARGS}, - {"dict_merge", dict_merge, METH_VARARGS}, - {"dict_update", dict_update, METH_VARARGS}, - {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, {"dict_pop", dict_pop, METH_VARARGS}, {"dict_pop_null", dict_pop_null, METH_VARARGS}, {"dict_popstring", dict_popstring, METH_VARARGS}, diff --git a/Modules/_testcapi/docstring.c b/Modules/_testcapi/docstring.c index d99fbdd904b594..3f7acbae1b181b 100644 --- a/Modules/_testcapi/docstring.c +++ b/Modules/_testcapi/docstring.c @@ -169,6 +169,13 @@ static PyMethodDef DocStringUnrepresentableSignatureTest_methods[] = { "--\n\n" "This docstring has a signature with unrepresentable default." )}, + {"with_default", + (PyCFunction)test_with_docstring, METH_VARARGS, + PyDoc_STR( + "with_default($self, /, x=ONE)\n" + "--\n\n" + "This instance method has a default parameter value from the module scope." + )}, {NULL}, }; @@ -193,5 +200,8 @@ _PyTestCapi_Init_Docstring(PyObject *mod) if (PyModule_AddType(mod, &DocStringUnrepresentableSignatureTest) < 0) { return -1; } + if (PyModule_AddObject(mod, "ONE", PyLong_FromLong(1)) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 42a9915143e6fa..316ef0e7ad7e55 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -34,11 +34,11 @@ err_restore(PyObject *self, PyObject *args) { case 3: traceback = PyTuple_GetItem(args, 2); Py_INCREF(traceback); - /* fall through */ + _Py_FALLTHROUGH; case 2: value = PyTuple_GetItem(args, 1); Py_INCREF(value); - /* fall through */ + _Py_FALLTHROUGH; case 1: type = PyTuple_GetItem(args, 0); Py_INCREF(type); diff --git a/Modules/_testcapi/float.c b/Modules/_testcapi/float.c index 4fcbaf3bb2aa1e..15ea97ec4520b7 100644 --- a/Modules/_testcapi/float.c +++ b/Modules/_testcapi/float.c @@ -6,71 +6,6 @@ #include "clinic/float.c.h" -static PyObject * -float_check(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyFloat_Check(obj)); -} - -static PyObject * -float_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyFloat_CheckExact(obj)); -} - -static PyObject * -float_fromstring(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyFloat_FromString(obj); -} - -static PyObject * -float_fromdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double d; - - if (!PyArg_Parse(obj, "d", &d)) { - return NULL; - } - - return PyFloat_FromDouble(d); -} - -static PyObject * -float_asdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double d; - - NULLABLE(obj); - d = PyFloat_AsDouble(obj); - if (d == -1. && PyErr_Occurred()) { - return NULL; - } - - return PyFloat_FromDouble(d); -} - -static PyObject * -float_getinfo(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) -{ - return PyFloat_GetInfo(); -} - -static PyObject * -float_getmax(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) -{ - return PyFloat_FromDouble(PyFloat_GetMax()); -} - -static PyObject * -float_getmin(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) -{ - return PyFloat_FromDouble(PyFloat_GetMin()); -} - /*[clinic input] module _testcapi [clinic start generated code]*/ @@ -165,14 +100,6 @@ _testcapi_float_unpack_impl(PyObject *module, const char *data, } static PyMethodDef test_methods[] = { - {"float_check", float_check, METH_O}, - {"float_checkexact", float_checkexact, METH_O}, - {"float_fromstring", float_fromstring, METH_O}, - {"float_fromdouble", float_fromdouble, METH_O}, - {"float_asdouble", float_asdouble, METH_O}, - {"float_getinfo", float_getinfo, METH_NOARGS}, - {"float_getmax", float_getmax, METH_NOARGS}, - {"float_getmin", float_getmin, METH_NOARGS}, _TESTCAPI_FLOAT_PACK_METHODDEF _TESTCAPI_FLOAT_UNPACK_METHODDEF {NULL}, diff --git a/Modules/_testcapi/gc.c b/Modules/_testcapi/gc.c index f4feaaafbdc6cc..7e33e0d4861e84 100644 --- a/Modules/_testcapi/gc.c +++ b/Modules/_testcapi/gc.c @@ -72,7 +72,7 @@ without_gc(PyObject *Py_UNUSED(self), PyObject *obj) if (PyType_IS_GC(tp)) { // Don't try this at home, kids: tp->tp_flags -= Py_TPFLAGS_HAVE_GC; - tp->tp_free = PyObject_Del; + tp->tp_free = PyObject_Free; tp->tp_traverse = NULL; tp->tp_clear = NULL; } @@ -99,10 +99,11 @@ slot_tp_del(PyObject *self) return; } /* Execute __del__ method, if any. */ - del = _PyType_Lookup(Py_TYPE(self), tp_del); + del = _PyType_LookupRef(Py_TYPE(self), tp_del); Py_DECREF(tp_del); if (del != NULL) { res = PyObject_CallOneArg(del, self); + Py_DECREF(del); if (res == NULL) PyErr_WriteUnraisable(del); else diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index 0d61d8c8969f82..ee04c760d27213 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -141,6 +141,122 @@ getargs_w_star(PyObject *self, PyObject *args) return result; } +static PyObject * +getargs_w_star_opt(PyObject *self, PyObject *args) +{ + Py_buffer buffer; + Py_buffer buf2; + int number = 1; + + if (!PyArg_ParseTuple(args, "w*|w*i:getargs_w_star", + &buffer, &buf2, &number)) { + return NULL; + } + + if (2 <= buffer.len) { + char *str = buffer.buf; + str[0] = '['; + str[buffer.len-1] = ']'; + } + + PyObject *result = PyBytes_FromStringAndSize(buffer.buf, buffer.len); + PyBuffer_Release(&buffer); + return result; +} + +/* Test the old w and w# codes that no longer work */ +static PyObject * +test_w_code_invalid(PyObject *self, PyObject *arg) +{ + static const char * const keywords[] = {"a", "b", "c", "d", NULL}; + char *formats_3[] = {"O|w#$O", + "O|w$O", + "O|w#O", + "O|wO", + NULL}; + char *formats_4[] = {"O|w#O$O", + "O|wO$O", + "O|Ow#O", + "O|OwO", + "O|Ow#$O", + "O|Ow$O", + NULL}; + size_t n; + PyObject *args; + PyObject *kwargs; + PyObject *tmp; + + if (!(args = PyTuple_Pack(1, Py_None))) { + return NULL; + } + + kwargs = PyDict_New(); + if (!kwargs) { + Py_DECREF(args); + return NULL; + } + + if (PyDict_SetItemString(kwargs, "c", Py_None)) { + Py_DECREF(args); + Py_XDECREF(kwargs); + return NULL; + } + + for (n = 0; formats_3[n]; ++n) { + if (PyArg_ParseTupleAndKeywords(args, kwargs, formats_3[n], + (char**) keywords, + &tmp, &tmp, &tmp)) { + Py_DECREF(args); + Py_DECREF(kwargs); + PyErr_Format(PyExc_AssertionError, + "test_w_code_invalid_suffix: %s", + formats_3[n]); + return NULL; + } + else { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + Py_DECREF(args); + Py_DECREF(kwargs); + return NULL; + } + PyErr_Clear(); + } + } + + if (PyDict_DelItemString(kwargs, "c") || + PyDict_SetItemString(kwargs, "d", Py_None)) { + + Py_DECREF(kwargs); + Py_DECREF(args); + return NULL; + } + + for (n = 0; formats_4[n]; ++n) { + if (PyArg_ParseTupleAndKeywords(args, kwargs, formats_4[n], + (char**) keywords, + &tmp, &tmp, &tmp, &tmp)) { + Py_DECREF(args); + Py_DECREF(kwargs); + PyErr_Format(PyExc_AssertionError, + "test_w_code_invalid_suffix: %s", + formats_4[n]); + return NULL; + } + else { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + Py_DECREF(args); + Py_DECREF(kwargs); + return NULL; + } + PyErr_Clear(); + } + } + + Py_DECREF(args); + Py_DECREF(kwargs); + Py_RETURN_NONE; +} + static PyObject * getargs_empty(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -684,6 +800,7 @@ static PyMethodDef test_methods[] = { {"getargs_s_star", getargs_s_star, METH_VARARGS}, {"getargs_tuple", getargs_tuple, METH_VARARGS}, {"getargs_w_star", getargs_w_star, METH_VARARGS}, + {"getargs_w_star_opt", getargs_w_star_opt, METH_VARARGS}, {"getargs_empty", _PyCFunction_CAST(getargs_empty), METH_VARARGS|METH_KEYWORDS}, {"getargs_y", getargs_y, METH_VARARGS}, {"getargs_y_hash", getargs_y_hash, METH_VARARGS}, @@ -693,6 +810,7 @@ static PyMethodDef test_methods[] = { {"getargs_z_star", getargs_z_star, METH_VARARGS}, {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS}, {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, + {"test_w_code_invalid", test_w_code_invalid, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapi/hash.c b/Modules/_testcapi/hash.c index aee76787dcddb3..1525344a93fbcf 100644 --- a/Modules/_testcapi/hash.c +++ b/Modules/_testcapi/hash.c @@ -45,6 +45,14 @@ hash_getfuncdef(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } +static PyObject * +long_from_hash(Py_hash_t hash) +{ + Py_BUILD_ASSERT(sizeof(long long) >= sizeof(hash)); + return PyLong_FromLongLong(hash); +} + + static PyObject * hash_pointer(PyObject *Py_UNUSED(module), PyObject *arg) { @@ -54,14 +62,38 @@ hash_pointer(PyObject *Py_UNUSED(module), PyObject *arg) } Py_hash_t hash = Py_HashPointer(ptr); - Py_BUILD_ASSERT(sizeof(long long) >= sizeof(hash)); - return PyLong_FromLongLong(hash); + return long_from_hash(hash); +} + + +static PyObject * +hash_buffer(PyObject *Py_UNUSED(module), PyObject *args) +{ + char *ptr; + Py_ssize_t len; + if (!PyArg_ParseTuple(args, "y#", &ptr, &len)) { + return NULL; + } + + Py_hash_t hash = Py_HashBuffer(ptr, len); + return long_from_hash(hash); +} + + +static PyObject * +object_generichash(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + Py_hash_t hash = PyObject_GenericHash(arg); + return long_from_hash(hash); } static PyMethodDef test_methods[] = { {"hash_getfuncdef", hash_getfuncdef, METH_NOARGS}, {"hash_pointer", hash_pointer, METH_O}, + {"hash_buffer", hash_buffer, METH_VARARGS}, + {"object_generichash", object_generichash, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 4526583a8059d9..cc88147dfcd7fb 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -269,16 +269,16 @@ test_type_from_ephemeral_spec(PyObject *self, PyObject *Py_UNUSED(ignored)) // (Explicitly overwrite memory before freeing, // so bugs show themselves even without the debug allocator's help.) memset(spec, 0xdd, sizeof(PyType_Spec)); - PyMem_Del(spec); + PyMem_Free(spec); spec = NULL; memset(name, 0xdd, sizeof(NAME)); - PyMem_Del(name); + PyMem_Free(name); name = NULL; memset(doc, 0xdd, sizeof(DOC)); - PyMem_Del(doc); + PyMem_Free(doc); doc = NULL; memset(slots, 0xdd, 3 * sizeof(PyType_Slot)); - PyMem_Del(slots); + PyMem_Free(slots); slots = NULL; /* check that everything works */ @@ -304,10 +304,10 @@ test_type_from_ephemeral_spec(PyObject *self, PyObject *Py_UNUSED(ignored)) result = Py_NewRef(Py_None); finally: - PyMem_Del(spec); - PyMem_Del(name); - PyMem_Del(doc); - PyMem_Del(slots); + PyMem_Free(spec); + PyMem_Free(name); + PyMem_Free(doc); + PyMem_Free(slots); Py_XDECREF(class); Py_XDECREF(instance); Py_XDECREF(obj); @@ -410,6 +410,118 @@ pyobject_getitemdata(PyObject *self, PyObject *o) } +static PyObject * +create_type_with_token(PyObject *module, PyObject *args) +{ + const char *name; + PyObject *py_token; + if (!PyArg_ParseTuple(args, "sO", &name, &py_token)) { + return NULL; + } + void *token = PyLong_AsVoidPtr(py_token); + if (token == Py_TP_USE_SPEC) { + // Py_TP_USE_SPEC requires the spec that at least outlives the class + static PyType_Slot slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + {0}, + }; + static PyType_Spec spec = { + .name = "_testcapi.DefaultTokenTest", + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = slots, + }; + PyObject *type = PyType_FromMetaclass(NULL, NULL, &spec, NULL); + if (!type) { + return NULL; + } + token = PyType_GetSlot((PyTypeObject *)type, Py_tp_token); + assert(!PyErr_Occurred()); + Py_DECREF(type); + if (token != &spec) { + PyErr_SetString(PyExc_AssertionError, + "failed to convert token from Py_TP_USE_SPEC"); + return NULL; + } + } + // Test non-NULL token that must also outlive the class + PyType_Slot slots[] = { + {Py_tp_token, token}, + {0}, + }; + PyType_Spec spec = { + .name = name, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = slots, + }; + return PyType_FromMetaclass(NULL, module, &spec, NULL); +} + +static PyObject * +get_tp_token(PyObject *self, PyObject *type) +{ + void *token = PyType_GetSlot((PyTypeObject *)type, Py_tp_token); + if (PyErr_Occurred()) { + return NULL; + } + return PyLong_FromVoidPtr(token); +} + +static PyObject * +pytype_getbasebytoken(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + PyObject *py_token, *use_mro, *need_result; + if (!PyArg_ParseTuple(args, "OOOO", + &type, &py_token, &use_mro, &need_result)) { + return NULL; + } + + PyObject *mro_save = NULL; + if (use_mro != Py_True) { + // Test internal detail: PyType_GetBaseByToken works even with + // types that are only partially initialized (or torn down): + // if tp_mro=NULL we fall back to tp_bases. + assert(PyType_Check(type)); + mro_save = type->tp_mro; + type->tp_mro = NULL; + } + + void *token = PyLong_AsVoidPtr(py_token); + PyObject *result; + int ret; + if (need_result == Py_True) { + ret = PyType_GetBaseByToken(type, token, (PyTypeObject **)&result); + } + else { + result = NULL; + ret = PyType_GetBaseByToken(type, token, NULL); + } + + if (use_mro != Py_True) { + type->tp_mro = mro_save; + } + if (ret < 0) { + assert(result == NULL); + return NULL; + } + PyObject *py_ret = PyLong_FromLong(ret); + if (py_ret == NULL) { + goto error; + } + PyObject *tuple = PyTuple_New(2); + if (tuple == NULL) { + goto error; + } + PyTuple_SET_ITEM(tuple, 0, py_ret); + PyTuple_SET_ITEM(tuple, 1, result ? result : Py_None); + return tuple; +error: + Py_XDECREF(py_ret); + Py_XDECREF(result); + return NULL; +} + + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, {"test_type_from_ephemeral_spec", test_type_from_ephemeral_spec, METH_NOARGS}, @@ -423,6 +535,9 @@ static PyMethodDef TestMethods[] = { {"make_immutable_type_with_base", make_immutable_type_with_base, METH_O}, {"make_type_with_base", make_type_with_base, METH_O}, {"pyobject_getitemdata", pyobject_getitemdata, METH_O}, + {"create_type_with_token", create_type_with_token, METH_VARARGS}, + {"get_tp_token", get_tp_token, METH_O}, + {"pytype_getbasebytoken", pytype_getbasebytoken, METH_VARARGS}, {NULL}, }; @@ -1008,6 +1123,89 @@ static PyType_Spec HeapCTypeSetattr_spec = { HeapCTypeSetattr_slots }; +/* + * The code below is for a test that uses PyType_FromSpec API to create a heap + * type that simultaneously exposes + * + * - A regular __new__ / __init__ constructor pair + * - A vector call handler in the type object + * + * A general requirement of vector call implementations is that they should + * behave identically (except being potentially faster). The example below + * deviates from this rule by initializing the instance with a different value. + * This is only done here only so that we can see which path was taken and is + * strongly discouraged in other cases. + */ + +typedef struct { + PyObject_HEAD + long value; +} HeapCTypeVectorcallObject; + +static PyObject *heapctype_vectorcall_vectorcall(PyObject *self, + PyObject *const *args_in, + size_t nargsf, + PyObject *kwargs_in) +{ + if (kwargs_in || PyVectorcall_NARGS(nargsf)) { + return PyErr_Format(PyExc_IndexError, "HeapCTypeVectorcall() takes no arguments!"); + } + + HeapCTypeVectorcallObject *r = + PyObject_New(HeapCTypeVectorcallObject, (PyTypeObject *) self); + + if (!r) { + return NULL; + } + + r->value = 1; + + return (PyObject *) r; +} + +static PyObject * +heapctype_vectorcall_new(PyTypeObject* type, PyObject* args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || kwargs) { + return PyErr_Format(PyExc_IndexError, "HeapCTypeVectorcall() takes no arguments!"); + } + + return (PyObject *) PyObject_New(HeapCTypeVectorcallObject, type); +} + +static int +heapctype_vectorcall_init(PyObject *self, PyObject *args, PyObject *kwargs) { + if (PyTuple_GET_SIZE(args) || kwargs) { + PyErr_Format(PyExc_IndexError, "HeapCTypeVectorcall() takes no arguments!"); + return -1; + } + + HeapCTypeVectorcallObject *o = (HeapCTypeVectorcallObject *) self; + o->value = 2; + return 0; +} + +static struct PyMemberDef heapctype_vectorcall_members[] = { + {"value", Py_T_LONG, offsetof(HeapCTypeVectorcallObject, value), 0, NULL}, + {NULL} +}; + +static PyType_Slot HeapCTypeVectorcall_slots[] = { + {Py_tp_new, heapctype_vectorcall_new}, + {Py_tp_init, heapctype_vectorcall_init}, + {Py_tp_vectorcall, heapctype_vectorcall_vectorcall}, + {Py_tp_members, heapctype_vectorcall_members}, + {0, 0}, +}; + +static PyType_Spec HeapCTypeVectorcall_spec = { + "_testcapi.HeapCTypeVectorcall", + sizeof(HeapCTypeVectorcallObject), + 0, + Py_TPFLAGS_DEFAULT, + HeapCTypeVectorcall_slots +}; + PyDoc_STRVAR(HeapCCollection_doc, "Tuple-like heap type that uses PyObject_GetItemData for items."); @@ -1180,6 +1378,9 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { PyObject *HeapCTypeSetattr = PyType_FromSpec(&HeapCTypeSetattr_spec); ADD("HeapCTypeSetattr", HeapCTypeSetattr); + PyObject *HeapCTypeVectorcall = PyType_FromSpec(&HeapCTypeVectorcall_spec); + ADD("HeapCTypeVectorcall", HeapCTypeVectorcall); + PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass); if (subclass_with_finalizer_bases == NULL) { return -1; @@ -1201,6 +1402,8 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { &PyType_Type, m, &HeapCTypeMetaclassNullNew_spec, (PyObject *) &PyType_Type); ADD("HeapCTypeMetaclassNullNew", HeapCTypeMetaclassNullNew); + ADD("Py_TP_USE_SPEC", PyLong_FromVoidPtr(Py_TP_USE_SPEC)); + PyObject *HeapCCollection = PyType_FromMetaclass( NULL, m, &HeapCCollection_spec, NULL); if (HeapCCollection == NULL) { diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index 2cb6499e28336d..09cec4c30c8c36 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -1,32 +1,6 @@ #include "parts.h" #include "util.h" -static PyObject * -list_check(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyList_Check(obj)); -} - -static PyObject * -list_check_exact(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyList_CheckExact(obj)); -} - -static PyObject * -list_new(PyObject* Py_UNUSED(module), PyObject *obj) -{ - return PyList_New(PyLong_AsSsize_t(obj)); -} - -static PyObject * -list_size(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyList_Size(obj)); -} static PyObject * list_get_size(PyObject *Py_UNUSED(module), PyObject *obj) @@ -35,17 +9,6 @@ list_get_size(PyObject *Py_UNUSED(module), PyObject *obj) RETURN_SIZE(PyList_GET_SIZE(obj)); } -static PyObject * -list_getitem(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &obj, &i)) { - return NULL; - } - NULLABLE(obj); - return Py_XNewRef(PyList_GetItem(obj, i)); -} static PyObject * list_get_item(PyObject *Py_UNUSED(module), PyObject *args) @@ -59,31 +22,6 @@ list_get_item(PyObject *Py_UNUSED(module), PyObject *args) return Py_XNewRef(PyList_GET_ITEM(obj, i)); } -static PyObject * -list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &obj, &i)) { - return NULL; - } - NULLABLE(obj); - return PyList_GetItemRef(obj, i); -} - -static PyObject * -list_setitem(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_SetItem(obj, i, Py_XNewRef(value))); - -} static PyObject * list_set_item(PyObject *Py_UNUSED(module), PyObject *args) @@ -100,79 +38,6 @@ list_set_item(PyObject *Py_UNUSED(module), PyObject *args) } -static PyObject * -list_insert(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t where; - if (!PyArg_ParseTuple(args, "OnO", &obj, &where, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_Insert(obj, where, Py_XNewRef(value))); - -} - -static PyObject * -list_append(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - if (!PyArg_ParseTuple(args, "OO", &obj, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_Append(obj, value)); -} - -static PyObject * -list_getslice(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t ilow, ihigh; - if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { - return NULL; - } - NULLABLE(obj); - return PyList_GetSlice(obj, ilow, ihigh); - -} - -static PyObject * -list_setslice(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t ilow, ihigh; - if (!PyArg_ParseTuple(args, "OnnO", &obj, &ilow, &ihigh, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_SetSlice(obj, ilow, ihigh, value)); -} - -static PyObject * -list_sort(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyList_Sort(obj)); -} - -static PyObject * -list_reverse(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyList_Reverse(obj)); -} - -static PyObject * -list_astuple(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyList_AsTuple(obj); -} - static PyObject * list_clear(PyObject* Py_UNUSED(module), PyObject *obj) @@ -196,25 +61,12 @@ list_extend(PyObject* Py_UNUSED(module), PyObject *args) static PyMethodDef test_methods[] = { - {"list_check", list_check, METH_O}, - {"list_check_exact", list_check_exact, METH_O}, - {"list_new", list_new, METH_O}, - {"list_size", list_size, METH_O}, {"list_get_size", list_get_size, METH_O}, - {"list_getitem", list_getitem, METH_VARARGS}, {"list_get_item", list_get_item, METH_VARARGS}, - {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, - {"list_setitem", list_setitem, METH_VARARGS}, {"list_set_item", list_set_item, METH_VARARGS}, - {"list_insert", list_insert, METH_VARARGS}, - {"list_append", list_append, METH_VARARGS}, - {"list_getslice", list_getslice, METH_VARARGS}, - {"list_setslice", list_setslice, METH_VARARGS}, - {"list_sort", list_sort, METH_O}, - {"list_reverse", list_reverse, METH_O}, - {"list_astuple", list_astuple, METH_O}, {"list_clear", list_clear, METH_O}, {"list_extend", list_extend, METH_VARARGS}, + {NULL}, }; diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 32ad8d32ab8523..2b5e85d5707522 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -12,529 +12,6 @@ module _testcapi /*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ -static PyObject * -raiseTestError(const char* test_name, const char* msg) -{ - PyErr_Format(PyExc_AssertionError, "%s: %s", test_name, msg); - return NULL; -} - -/* Tests of PyLong_{As, From}{Unsigned,}Long(), and - PyLong_{As, From}{Unsigned,}LongLong(). - - Note that the meat of the test is contained in testcapi_long.h. - This is revolting, but delicate code duplication is worse: "almost - exactly the same" code is needed to test long long, but the ubiquitous - dependence on type names makes it impossible to use a parameterized - function. A giant macro would be even worse than this. A C++ template - would be perfect. - - The "report an error" functions are deliberately not part of the #include - file: if the test fails, you can set a breakpoint in the appropriate - error function directly, and crawl back from there in the debugger. -*/ - -#define UNBIND(X) Py_DECREF(X); (X) = NULL - -static PyObject * -raise_test_long_error(const char* msg) -{ - return raiseTestError("test_long_api", msg); -} - -// Test PyLong_FromLong()/PyLong_AsLong() -// and PyLong_FromUnsignedLong()/PyLong_AsUnsignedLong(). - -#define TESTNAME test_long_api_inner -#define TYPENAME long -#define F_S_TO_PY PyLong_FromLong -#define F_PY_TO_S PyLong_AsLong -#define F_U_TO_PY PyLong_FromUnsignedLong -#define F_PY_TO_U PyLong_AsUnsignedLong - -#include "testcapi_long.h" - -/*[clinic input] -_testcapi.test_long_api -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_api_impl(PyObject *module) -/*[clinic end generated code: output=4405798ca1e9f444 input=e9b8880d7331c688]*/ -{ - return TESTNAME(raise_test_long_error); -} - -#undef TESTNAME -#undef TYPENAME -#undef F_S_TO_PY -#undef F_PY_TO_S -#undef F_U_TO_PY -#undef F_PY_TO_U - -// Test PyLong_FromLongLong()/PyLong_AsLongLong() -// and PyLong_FromUnsignedLongLong()/PyLong_AsUnsignedLongLong(). - -static PyObject * -raise_test_longlong_error(const char* msg) -{ - return raiseTestError("test_longlong_api", msg); -} - -#define TESTNAME test_longlong_api_inner -#define TYPENAME long long -#define F_S_TO_PY PyLong_FromLongLong -#define F_PY_TO_S PyLong_AsLongLong -#define F_U_TO_PY PyLong_FromUnsignedLongLong -#define F_PY_TO_U PyLong_AsUnsignedLongLong - -#include "testcapi_long.h" - -/*[clinic input] -_testcapi.test_longlong_api -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_longlong_api_impl(PyObject *module) -/*[clinic end generated code: output=2b3414ba8c31dfe6 input=ccbb2a48c2b3c4a5]*/ -{ - return TESTNAME(raise_test_longlong_error); -} - -#undef TESTNAME -#undef TYPENAME -#undef F_S_TO_PY -#undef F_PY_TO_S -#undef F_U_TO_PY -#undef F_PY_TO_U - - -/*[clinic input] -_testcapi.test_long_and_overflow - -Test the PyLong_AsLongAndOverflow API. - -General conversion to PY_LONG is tested by test_long_api_inner. -This test will concentrate on proper handling of overflow. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_and_overflow_impl(PyObject *module) -/*[clinic end generated code: output=f8460ca115e31d8e input=762f6b62da0a3cdc]*/ -{ - PyObject *num, *one, *temp; - long value; - int overflow; - - /* Test that overflow is set properly for a large value. */ - /* num is a number larger than LONG_MAX even on 64-bit platforms */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to 1"); - - /* Same again, with num = LONG_MAX + 1 */ - num = PyLong_FromLong(LONG_MAX); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Add(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to 1"); - - /* Test that overflow is set properly for a large negative value. */ - /* num is a number smaller than LONG_MIN even on 64-bit platforms */ - num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to -1"); - - /* Same again, with num = LONG_MIN - 1 */ - num = PyLong_FromLong(LONG_MIN); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Subtract(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to -1"); - - /* Test that overflow is cleared properly for small values. */ - num = PyLong_FromString("FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != 0xFF) - return raiseTestError("test_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromString("-FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -0xFF) - return raiseTestError("test_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was set incorrectly"); - - num = PyLong_FromLong(LONG_MAX); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LONG_MAX) - return raiseTestError("test_long_and_overflow", - "expected return value LONG_MAX"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromLong(LONG_MIN); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LONG_MIN) - return raiseTestError("test_long_and_overflow", - "expected return value LONG_MIN"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_long_and_overflow - -Test the PyLong_AsLongLongAndOverflow API. - -General conversion to long long is tested by test_long_api_inner. -This test will concentrate on proper handling of overflow. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_long_and_overflow_impl(PyObject *module) -/*[clinic end generated code: output=0b92330786f45483 input=544bb0aefe5e8a9e]*/ -{ - PyObject *num, *one, *temp; - long long value; - int overflow; - - /* Test that overflow is set properly for a large value. */ - /* num is a number larger than LLONG_MAX on a typical machine. */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to 1"); - - /* Same again, with num = LLONG_MAX + 1 */ - num = PyLong_FromLongLong(LLONG_MAX); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Add(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to 1"); - - /* Test that overflow is set properly for a large negative value. */ - /* num is a number smaller than LLONG_MIN on a typical platform */ - num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to -1"); - - /* Same again, with num = LLONG_MIN - 1 */ - num = PyLong_FromLongLong(LLONG_MIN); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Subtract(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to -1"); - - /* Test that overflow is cleared properly for small values. */ - num = PyLong_FromString("FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != 0xFF) - return raiseTestError("test_long_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromString("-FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -0xFF) - return raiseTestError("test_long_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was set incorrectly"); - - num = PyLong_FromLongLong(LLONG_MAX); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LLONG_MAX) - return raiseTestError("test_long_long_and_overflow", - "expected return value LLONG_MAX"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromLongLong(LLONG_MIN); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LLONG_MIN) - return raiseTestError("test_long_long_and_overflow", - "expected return value LLONG_MIN"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_as_size_t - -Test the PyLong_As{Size,Ssize}_t API. - -At present this just tests that non-integer arguments are handled correctly. -It should be extended to test overflow handling. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_size_t_impl(PyObject *module) -/*[clinic end generated code: output=f6490ea2b41e6173 input=922990c4a3edfb0d]*/ -{ - size_t out_u; - Py_ssize_t out_s; - - Py_INCREF(Py_None); - - out_u = PyLong_AsSize_t(Py_None); - if (out_u != (size_t)-1 || !PyErr_Occurred()) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSize_t(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSize_t(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - out_s = PyLong_AsSsize_t(Py_None); - if (out_s != (Py_ssize_t)-1 || !PyErr_Occurred()) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSsize_t(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSsize_t(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ - return Py_None; -} - -/*[clinic input] -_testcapi.test_long_as_unsigned_long_long_mask -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) -/*[clinic end generated code: output=e3e16cd0189440cc input=eb2438493ae7b9af]*/ -{ - unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); - - if (res != (unsigned long long)-1 || !PyErr_Occurred()) { - return raiseTestError("test_long_as_unsigned_long_long_mask", - "PyLong_AsUnsignedLongLongMask(NULL) didn't " - "complain"); - } - if (!PyErr_ExceptionMatches(PyExc_SystemError)) { - return raiseTestError("test_long_as_unsigned_long_long_mask", - "PyLong_AsUnsignedLongLongMask(NULL) raised " - "something other than SystemError"); - } - PyErr_Clear(); - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_as_double -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_double_impl(PyObject *module) -/*[clinic end generated code: output=deca0898e15adde5 input=c77bc88ef5a1de76]*/ -{ - double out; - - Py_INCREF(Py_None); - - out = PyLong_AsDouble(Py_None); - if (out != -1.0 || !PyErr_Occurred()) - return raiseTestError("test_long_as_double", - "PyLong_AsDouble(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_double", - "PyLong_AsDouble(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ - return Py_None; -} - /*[clinic input] _testcapi.call_long_compact_api arg: object @@ -555,48 +32,6 @@ _testcapi_call_long_compact_api(PyObject *module, PyObject *arg) return Py_BuildValue("in", is_compact, value); } -static PyObject * -pylong_check(PyObject *module, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyLong_Check(obj)); -} - -static PyObject * -pylong_checkexact(PyObject *module, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyLong_CheckExact(obj)); -} - -static PyObject * -pylong_fromdouble(PyObject *module, PyObject *arg) -{ - double value; - if (!PyArg_Parse(arg, "d", &value)) { - return NULL; - } - return PyLong_FromDouble(value); -} - -static PyObject * -pylong_fromstring(PyObject *module, PyObject *args) -{ - const char *str; - Py_ssize_t len; - int base; - char *end = UNINITIALIZED_PTR; - if (!PyArg_ParseTuple(args, "z#i", &str, &len, &base)) { - return NULL; - } - - PyObject *result = PyLong_FromString(str, &end, base); - if (result == NULL) { - // XXX 'end' is not always set. - return NULL; - } - return Py_BuildValue("Nn", result, (Py_ssize_t)(end - str)); -} static PyObject * pylong_fromunicodeobject(PyObject *module, PyObject *args) @@ -611,199 +46,84 @@ pylong_fromunicodeobject(PyObject *module, PyObject *args) return PyLong_FromUnicodeObject(unicode, base); } -static PyObject * -pylong_fromvoidptr(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - return PyLong_FromVoidPtr((void *)arg); -} - -/*[clinic input] -_testcapi.PyLong_AsInt - arg: object - / -[clinic start generated code]*/ static PyObject * -_testcapi_PyLong_AsInt(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=0df9f19de5fa575b input=9561b97105493a67]*/ +pylong_asnativebytes(PyObject *module, PyObject *args) { - NULLABLE(arg); - assert(!PyErr_Occurred()); - int value = PyLong_AsInt(arg); - if (value == -1 && PyErr_Occurred()) { + PyObject *v; + Py_buffer buffer; + Py_ssize_t n, flags; + if (!PyArg_ParseTuple(args, "Ow*nn", &v, &buffer, &n, &flags)) { return NULL; } - return PyLong_FromLong(value); -} - -static PyObject * -pylong_aslong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - long value = PyLong_AsLong(arg); - if (value == -1 && PyErr_Occurred()) { + if (buffer.readonly) { + PyErr_SetString(PyExc_TypeError, "buffer must be writable"); + PyBuffer_Release(&buffer); return NULL; } - return PyLong_FromLong(value); -} - -static PyObject * -pylong_aslongandoverflow(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - int overflow = UNINITIALIZED_INT; - long value = PyLong_AsLongAndOverflow(arg, &overflow); - if (value == -1 && PyErr_Occurred()) { - assert(overflow == -1); + if (buffer.len < n) { + PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); + PyBuffer_Release(&buffer); return NULL; } - return Py_BuildValue("li", value, overflow); + Py_ssize_t res = PyLong_AsNativeBytes(v, buffer.buf, n, (int)flags); + PyBuffer_Release(&buffer); + return res >= 0 ? PyLong_FromSsize_t(res) : NULL; } -static PyObject * -pylong_asunsignedlong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - unsigned long value = PyLong_AsUnsignedLong(arg); - if (value == (unsigned long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLong(value); -} static PyObject * -pylong_asunsignedlongmask(PyObject *module, PyObject *arg) +pylong_fromnativebytes(PyObject *module, PyObject *args) { - NULLABLE(arg); - unsigned long value = PyLong_AsUnsignedLongMask(arg); - if (value == (unsigned long)-1 && PyErr_Occurred()) { + Py_buffer buffer; + Py_ssize_t n, flags, signed_; + if (!PyArg_ParseTuple(args, "y*nnn", &buffer, &n, &flags, &signed_)) { return NULL; } - return PyLong_FromUnsignedLong(value); -} - -static PyObject * -pylong_aslonglong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - long long value = PyLong_AsLongLong(arg); - if (value == -1 && PyErr_Occurred()) { + if (buffer.len < n) { + PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); + PyBuffer_Release(&buffer); return NULL; } - return PyLong_FromLongLong(value); + PyObject *res = signed_ + ? PyLong_FromNativeBytes(buffer.buf, n, (int)flags) + : PyLong_FromUnsignedNativeBytes(buffer.buf, n, (int)flags); + PyBuffer_Release(&buffer); + return res; } -static PyObject * -pylong_aslonglongandoverflow(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - int overflow = UNINITIALIZED_INT; - long long value = PyLong_AsLongLongAndOverflow(arg, &overflow); - if (value == -1 && PyErr_Occurred()) { - assert(overflow == -1); - return NULL; - } - return Py_BuildValue("Li", value, overflow); -} static PyObject * -pylong_asunsignedlonglong(PyObject *module, PyObject *arg) +pylong_getsign(PyObject *module, PyObject *arg) { + int sign; NULLABLE(arg); - unsigned long long value = PyLong_AsUnsignedLongLong(arg); - if (value == (unsigned long long)-1 && PyErr_Occurred()) { + if (PyLong_GetSign(arg, &sign) == -1) { return NULL; } - return PyLong_FromUnsignedLongLong(value); + return PyLong_FromLong(sign); } -static PyObject * -pylong_asunsignedlonglongmask(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - unsigned long long value = PyLong_AsUnsignedLongLongMask(arg); - if (value == (unsigned long long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLongLong(value); -} static PyObject * -pylong_as_ssize_t(PyObject *module, PyObject *arg) +pylong_aspid(PyObject *module, PyObject *arg) { NULLABLE(arg); - Py_ssize_t value = PyLong_AsSsize_t(arg); + pid_t value = PyLong_AsPid(arg); if (value == -1 && PyErr_Occurred()) { return NULL; } - return PyLong_FromSsize_t(value); -} - -static PyObject * -pylong_as_size_t(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - size_t value = PyLong_AsSize_t(arg); - if (value == (size_t)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromSize_t(value); + return PyLong_FromPid(value); } -static PyObject * -pylong_asdouble(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - double value = PyLong_AsDouble(arg); - if (value == -1.0 && PyErr_Occurred()) { - return NULL; - } - return PyFloat_FromDouble(value); -} - -static PyObject * -pylong_asvoidptr(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - void *value = PyLong_AsVoidPtr(arg); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - Py_RETURN_NONE; - } - return Py_NewRef((PyObject *)value); -} static PyMethodDef test_methods[] = { - _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF - _TESTCAPI_TEST_LONG_API_METHODDEF - _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF - _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF - _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF - _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF - _TESTCAPI_TEST_LONGLONG_API_METHODDEF _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF - {"pylong_check", pylong_check, METH_O}, - {"pylong_checkexact", pylong_checkexact, METH_O}, - {"pylong_fromdouble", pylong_fromdouble, METH_O}, - {"pylong_fromstring", pylong_fromstring, METH_VARARGS}, {"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS}, - {"pylong_fromvoidptr", pylong_fromvoidptr, METH_O}, - _TESTCAPI_PYLONG_ASINT_METHODDEF - {"pylong_aslong", pylong_aslong, METH_O}, - {"pylong_aslongandoverflow", pylong_aslongandoverflow, METH_O}, - {"pylong_asunsignedlong", pylong_asunsignedlong, METH_O}, - {"pylong_asunsignedlongmask", pylong_asunsignedlongmask, METH_O}, - {"pylong_aslonglong", pylong_aslonglong, METH_O}, - {"pylong_aslonglongandoverflow", pylong_aslonglongandoverflow, METH_O}, - {"pylong_asunsignedlonglong", pylong_asunsignedlonglong, METH_O}, - {"pylong_asunsignedlonglongmask", pylong_asunsignedlonglongmask, METH_O}, - {"pylong_as_ssize_t", pylong_as_ssize_t, METH_O}, - {"pylong_as_size_t", pylong_as_size_t, METH_O}, - {"pylong_asdouble", pylong_asdouble, METH_O}, - {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, + {"pylong_asnativebytes", pylong_asnativebytes, METH_VARARGS}, + {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, + {"pylong_getsign", pylong_getsign, METH_O}, + {"pylong_aspid", pylong_aspid, METH_O}, {NULL}, }; @@ -813,6 +133,5 @@ _PyTestCapi_Init_Long(PyObject *mod) if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; } - return 0; } diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c new file mode 100644 index 00000000000000..6fd4a405688f48 --- /dev/null +++ b/Modules/_testcapi/monitoring.c @@ -0,0 +1,508 @@ +#include "parts.h" +#include "util.h" + +#include "monitoring.h" + +#define Py_BUILD_CORE +#include "internal/pycore_instruments.h" + +typedef struct { + PyObject_HEAD + PyMonitoringState *monitoring_states; + uint64_t version; + int num_events; + /* Other fields */ +} PyCodeLikeObject; + + +static PyObject * +CodeLike_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + int num_events; + if (!PyArg_ParseTuple(args, "i", &num_events)) { + return NULL; + } + PyMonitoringState *states = (PyMonitoringState *)PyMem_Calloc( + num_events, sizeof(PyMonitoringState)); + if (states == NULL) { + return NULL; + } + PyCodeLikeObject *self = (PyCodeLikeObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->version = 0; + self->monitoring_states = states; + self->num_events = num_events; + } + else { + PyMem_Free(states); + } + return (PyObject *) self; +} + +static void +CodeLike_dealloc(PyCodeLikeObject *self) +{ + if (self->monitoring_states) { + PyMem_Free(self->monitoring_states); + } + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +CodeLike_str(PyCodeLikeObject *self) +{ + PyObject *res = NULL; + PyObject *sep = NULL; + PyObject *parts = NULL; + if (self->monitoring_states) { + parts = PyList_New(0); + if (parts == NULL) { + goto end; + } + + PyObject *heading = PyUnicode_FromString("PyCodeLikeObject"); + if (heading == NULL) { + goto end; + } + int err = PyList_Append(parts, heading); + Py_DECREF(heading); + if (err < 0) { + goto end; + } + + for (int i = 0; i < self->num_events; i++) { + PyObject *part = PyUnicode_FromFormat(" %d", self->monitoring_states[i].active); + if (part == NULL) { + goto end; + } + int err = PyList_Append(parts, part); + Py_XDECREF(part); + if (err < 0) { + goto end; + } + } + sep = PyUnicode_FromString(": "); + if (sep == NULL) { + goto end; + } + res = PyUnicode_Join(sep, parts); + } +end: + Py_XDECREF(sep); + Py_XDECREF(parts); + return res; +} + +static PyTypeObject PyCodeLike_Type = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "monitoring.CodeLike", + .tp_doc = PyDoc_STR("CodeLike objects"), + .tp_basicsize = sizeof(PyCodeLikeObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = CodeLike_new, + .tp_dealloc = (destructor) CodeLike_dealloc, + .tp_str = (reprfunc) CodeLike_str, +}; + +#define RAISE_UNLESS_CODELIKE(v) if (!Py_IS_TYPE((v), &PyCodeLike_Type)) { \ + PyErr_Format(PyExc_TypeError, "expected a code-like, got %s", Py_TYPE(v)->tp_name); \ + return NULL; \ + } + +/*******************************************************************/ + +static PyMonitoringState * +setup_fire(PyObject *codelike, int offset, PyObject *exc) +{ + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = ((PyCodeLikeObject *)codelike); + assert(offset >= 0 && offset < cl->num_events); + PyMonitoringState *state = &cl->monitoring_states[offset]; + + if (exc != NULL) { + PyErr_SetRaisedException(Py_NewRef(exc)); + } + return state; +} + +static int +teardown_fire(int res, PyMonitoringState *state, PyObject *exception) +{ + if (res == -1) { + return -1; + } + if (exception) { + assert(PyErr_Occurred()); + assert(((PyObject*)Py_TYPE(exception)) == PyErr_Occurred()); + } + + else { + assert(!PyErr_Occurred()); + } + PyErr_Clear(); + return state->active; +} + +static PyObject * +fire_event_py_start(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyStartEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_resume(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyResumeEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_yield(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_call(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *callable, *arg0; + if (!PyArg_ParseTuple(args, "OiOO", &codelike, &offset, &callable, &arg0)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_line(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset, lineno; + if (!PyArg_ParseTuple(args, "Oii", &codelike, &offset, &lineno)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireLineEvent(state, codelike, offset, lineno); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_jump(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_branch(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_throw(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyThrowEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_reraise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireReraiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_exception_handled(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireExceptionHandledEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_unwind(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyUnwindEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_stop_iteration(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *value; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &value)) { + return NULL; + } + NULLABLE(value); + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireStopIterationEvent(state, codelike, offset, value); + RETURN_INT(teardown_fire(res, state, exception)); +} + +/*******************************************************************/ + +static PyObject * +enter_scope(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int event1, event2=0; + Py_ssize_t num_events = PyTuple_Size(args) - 1; + if (num_events == 1) { + if (!PyArg_ParseTuple(args, "Oi", &codelike, &event1)) { + return NULL; + } + } + else { + assert(num_events == 2); + if (!PyArg_ParseTuple(args, "Oii", &codelike, &event1, &event2)) { + return NULL; + } + } + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = (PyCodeLikeObject *) codelike; + + uint8_t events[] = { event1, event2 }; + + PyMonitoring_EnterScope(cl->monitoring_states, + &cl->version, + events, + num_events); + + Py_RETURN_NONE; +} + +static PyObject * +exit_scope(PyObject *self, PyObject *args) +{ + PyMonitoring_ExitScope(); + Py_RETURN_NONE; +} + +static PyMethodDef TestMethods[] = { + {"fire_event_py_start", fire_event_py_start, METH_VARARGS}, + {"fire_event_py_resume", fire_event_py_resume, METH_VARARGS}, + {"fire_event_py_return", fire_event_py_return, METH_VARARGS}, + {"fire_event_c_return", fire_event_c_return, METH_VARARGS}, + {"fire_event_py_yield", fire_event_py_yield, METH_VARARGS}, + {"fire_event_call", fire_event_call, METH_VARARGS}, + {"fire_event_line", fire_event_line, METH_VARARGS}, + {"fire_event_jump", fire_event_jump, METH_VARARGS}, + {"fire_event_branch", fire_event_branch, METH_VARARGS}, + {"fire_event_py_throw", fire_event_py_throw, METH_VARARGS}, + {"fire_event_raise", fire_event_raise, METH_VARARGS}, + {"fire_event_c_raise", fire_event_c_raise, METH_VARARGS}, + {"fire_event_reraise", fire_event_reraise, METH_VARARGS}, + {"fire_event_exception_handled", fire_event_exception_handled, METH_VARARGS}, + {"fire_event_py_unwind", fire_event_py_unwind, METH_VARARGS}, + {"fire_event_stop_iteration", fire_event_stop_iteration, METH_VARARGS}, + {"monitoring_enter_scope", enter_scope, METH_VARARGS}, + {"monitoring_exit_scope", exit_scope, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Monitoring(PyObject *m) +{ + if (PyType_Ready(&PyCodeLike_Type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "CodeLike", (PyObject *) &PyCodeLike_Type) < 0) { + Py_DECREF(m); + return -1; + } + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/numbers.c b/Modules/_testcapi/numbers.c index 6f7fa3fa7a4186..e16ff73744067a 100644 --- a/Modules/_testcapi/numbers.c +++ b/Modules/_testcapi/numbers.c @@ -1,7 +1,168 @@ #include "parts.h" #include "util.h" + +static PyObject * +number_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyNumber_Check(obj)); +} + +#define BINARYFUNC(funcsuffix, methsuffix) \ + static PyObject * \ + number_##methsuffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + PyObject *o1, *o2; \ + \ + if (!PyArg_ParseTuple(args, "OO", &o1, &o2)) { \ + return NULL; \ + } \ + \ + NULLABLE(o1); \ + NULLABLE(o2); \ + return PyNumber_##funcsuffix(o1, o2); \ + }; + +BINARYFUNC(Add, add) +BINARYFUNC(Subtract, subtract) +BINARYFUNC(Multiply, multiply) +BINARYFUNC(MatrixMultiply, matrixmultiply) +BINARYFUNC(FloorDivide, floordivide) +BINARYFUNC(TrueDivide, truedivide) +BINARYFUNC(Remainder, remainder) +BINARYFUNC(Divmod, divmod) + +#define TERNARYFUNC(funcsuffix, methsuffix) \ + static PyObject * \ + number_##methsuffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + PyObject *o1, *o2, *o3 = Py_None; \ + \ + if (!PyArg_ParseTuple(args, "OO|O", &o1, &o2, &o3)) { \ + return NULL; \ + } \ + \ + NULLABLE(o1); \ + NULLABLE(o2); \ + return PyNumber_##funcsuffix(o1, o2, o3); \ + }; + +TERNARYFUNC(Power, power) + +#define UNARYFUNC(funcsuffix, methsuffix) \ + static PyObject * \ + number_##methsuffix(PyObject *Py_UNUSED(module), PyObject *obj) \ + { \ + NULLABLE(obj); \ + return PyNumber_##funcsuffix(obj); \ + }; + +UNARYFUNC(Negative, negative) +UNARYFUNC(Positive, positive) +UNARYFUNC(Absolute, absolute) +UNARYFUNC(Invert, invert) + +BINARYFUNC(Lshift, lshift) +BINARYFUNC(Rshift, rshift) +BINARYFUNC(And, and) +BINARYFUNC(Xor, xor) +BINARYFUNC(Or, or) + +BINARYFUNC(InPlaceAdd, inplaceadd) +BINARYFUNC(InPlaceSubtract, inplacesubtract) +BINARYFUNC(InPlaceMultiply, inplacemultiply) +BINARYFUNC(InPlaceMatrixMultiply, inplacematrixmultiply) +BINARYFUNC(InPlaceFloorDivide, inplacefloordivide) +BINARYFUNC(InPlaceTrueDivide, inplacetruedivide) +BINARYFUNC(InPlaceRemainder, inplaceremainder) + +TERNARYFUNC(InPlacePower, inplacepower) + +BINARYFUNC(InPlaceLshift, inplacelshift) +BINARYFUNC(InPlaceRshift, inplacershift) +BINARYFUNC(InPlaceAnd, inplaceand) +BINARYFUNC(InPlaceXor, inplacexor) +BINARYFUNC(InPlaceOr, inplaceor) + +UNARYFUNC(Long, long) +UNARYFUNC(Float, float) +UNARYFUNC(Index, index) + +static PyObject * +number_tobase(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *n; + int base; + + if (!PyArg_ParseTuple(args, "Oi", &n, &base)) { + return NULL; + } + + NULLABLE(n); + return PyNumber_ToBase(n, base); +} + +static PyObject * +number_asssizet(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *o, *exc; + Py_ssize_t ret; + + if (!PyArg_ParseTuple(args, "OO", &o, &exc)) { + return NULL; + } + + NULLABLE(o); + NULLABLE(exc); + ret = PyNumber_AsSsize_t(o, exc); + + if (ret == (Py_ssize_t)(-1) && PyErr_Occurred()) { + return NULL; + } + + return PyLong_FromSsize_t(ret); +} + + static PyMethodDef test_methods[] = { + {"number_check", number_check, METH_O}, + {"number_add", number_add, METH_VARARGS}, + {"number_subtract", number_subtract, METH_VARARGS}, + {"number_multiply", number_multiply, METH_VARARGS}, + {"number_matrixmultiply", number_matrixmultiply, METH_VARARGS}, + {"number_floordivide", number_floordivide, METH_VARARGS}, + {"number_truedivide", number_truedivide, METH_VARARGS}, + {"number_remainder", number_remainder, METH_VARARGS}, + {"number_divmod", number_divmod, METH_VARARGS}, + {"number_power", number_power, METH_VARARGS}, + {"number_negative", number_negative, METH_O}, + {"number_positive", number_positive, METH_O}, + {"number_absolute", number_absolute, METH_O}, + {"number_invert", number_invert, METH_O}, + {"number_lshift", number_lshift, METH_VARARGS}, + {"number_rshift", number_rshift, METH_VARARGS}, + {"number_and", number_and, METH_VARARGS}, + {"number_xor", number_xor, METH_VARARGS}, + {"number_or", number_or, METH_VARARGS}, + {"number_inplaceadd", number_inplaceadd, METH_VARARGS}, + {"number_inplacesubtract", number_inplacesubtract, METH_VARARGS}, + {"number_inplacemultiply", number_inplacemultiply, METH_VARARGS}, + {"number_inplacematrixmultiply", number_inplacematrixmultiply, METH_VARARGS}, + {"number_inplacefloordivide", number_inplacefloordivide, METH_VARARGS}, + {"number_inplacetruedivide", number_inplacetruedivide, METH_VARARGS}, + {"number_inplaceremainder", number_inplaceremainder, METH_VARARGS}, + {"number_inplacepower", number_inplacepower, METH_VARARGS}, + {"number_inplacelshift", number_inplacelshift, METH_VARARGS}, + {"number_inplacershift", number_inplacershift, METH_VARARGS}, + {"number_inplaceand", number_inplaceand, METH_VARARGS}, + {"number_inplacexor", number_inplacexor, METH_VARARGS}, + {"number_inplaceor", number_inplaceor, METH_VARARGS}, + {"number_long", number_long, METH_O}, + {"number_float", number_float, METH_O}, + {"number_index", number_index, METH_O}, + {"number_tobase", number_tobase, METH_VARARGS}, + {"number_asssizet", number_asssizet, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c new file mode 100644 index 00000000000000..1c76e766a790f0 --- /dev/null +++ b/Modules/_testcapi/object.c @@ -0,0 +1,145 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +call_pyobject_print(PyObject *self, PyObject * args) +{ + PyObject *object; + PyObject *filename; + PyObject *print_raw; + FILE *fp; + int flags = 0; + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 3, 3, + &object, &filename, &print_raw)) { + return NULL; + } + + fp = _Py_fopen_obj(filename, "w+"); + + if (Py_IsTrue(print_raw)) { + flags = Py_PRINT_RAW; + } + + if (PyObject_Print(object, fp, flags) < 0) { + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_print_null(PyObject *self, PyObject *args) +{ + PyObject *filename; + FILE *fp; + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + fp = _Py_fopen_obj(filename, "w+"); + + if (PyObject_Print(NULL, fp, 0) < 0) { + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_print_noref_object(PyObject *self, PyObject *args) +{ + PyObject *test_string; + PyObject *filename; + FILE *fp; + char correct_string[100]; + + test_string = PyUnicode_FromString("Spam spam spam"); + + Py_SET_REFCNT(test_string, 0); + + PyOS_snprintf(correct_string, 100, "", + Py_REFCNT(test_string), (void *)test_string); + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + fp = _Py_fopen_obj(filename, "w+"); + + if (PyObject_Print(test_string, fp, 0) < 0){ + fclose(fp); + Py_SET_REFCNT(test_string, 1); + Py_DECREF(test_string); + return NULL; + } + + fclose(fp); + + Py_SET_REFCNT(test_string, 1); + Py_DECREF(test_string); + + return PyUnicode_FromString(correct_string); +} + +static PyObject * +pyobject_print_os_error(PyObject *self, PyObject *args) +{ + PyObject *test_string; + PyObject *filename; + FILE *fp; + + test_string = PyUnicode_FromString("Spam spam spam"); + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + // open file in read mode to induce OSError + fp = _Py_fopen_obj(filename, "r"); + + if (PyObject_Print(test_string, fp, 0) < 0) { + fclose(fp); + Py_DECREF(test_string); + return NULL; + } + + fclose(fp); + Py_DECREF(test_string); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_clear_weakrefs_no_callbacks(PyObject *self, PyObject *obj) +{ + PyUnstable_Object_ClearWeakRefsNoCallbacks(obj); + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, + {"pyobject_print_null", pyobject_print_null, METH_VARARGS}, + {"pyobject_print_noref_object", pyobject_print_noref_object, METH_VARARGS}, + {"pyobject_print_os_error", pyobject_print_os_error, METH_VARARGS}, + {"pyobject_clear_weakrefs_no_callbacks", pyobject_clear_weakrefs_no_callbacks, METH_O}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Object(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 29817edd69b134..65ba77596c760e 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -31,7 +31,6 @@ int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); int _PyTestCapi_Init_Abstract(PyObject *module); -int _PyTestCapi_Init_ByteArray(PyObject *module); int _PyTestCapi_Init_Bytes(PyObject *module); int _PyTestCapi_Init_Unicode(PyObject *module); int _PyTestCapi_Init_GetArgs(PyObject *module); @@ -52,15 +51,15 @@ int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_Buffer(PyObject *module); int _PyTestCapi_Init_PyAtomic(PyObject *module); -int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Run(PyObject *module); int _PyTestCapi_Init_File(PyObject *module); int _PyTestCapi_Init_Codec(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); -int _PyTestCapi_Init_Sys(PyObject *module); int _PyTestCapi_Init_Hash(PyObject *module); - -int _PyTestCapi_Init_VectorcallLimited(PyObject *module); -int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); +int _PyTestCapi_Init_Time(PyObject *module); +int _PyTestCapi_Init_Monitoring(PyObject *module); +int _PyTestCapi_Init_Object(PyObject *module); +int _PyTestCapi_Init_Config(PyObject *mod); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index 4f72844535ebd6..850de6f9c3366b 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -125,6 +125,7 @@ test_atomic_fences(PyObject *self, PyObject *obj) { // Just make sure that the fences compile. We are not // testing any synchronizing ordering. _Py_atomic_fence_seq_cst(); + _Py_atomic_fence_acquire(); _Py_atomic_fence_release(); Py_RETURN_NONE; } diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c new file mode 100644 index 00000000000000..c9db7ccdd965a2 --- /dev/null +++ b/Modules/_testcapi/run.c @@ -0,0 +1,113 @@ +#define PYTESTCAPI_NEED_INTERNAL_API +#include "parts.h" +#include "util.h" +#include "pycore_fileutils.h" // _Py_IsValidFD() + +#include +#include + + +static PyObject * +run_stringflags(PyObject *mod, PyObject *pos_args) +{ + const char *str; + Py_ssize_t size; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + if (!PyArg_ParseTuple(pos_args, "z#iO|Oii", + &str, &size, &start, &globals, &locals, + &cf_flags, &cf_feature_version)) { + return NULL; + } + + NULLABLE(globals); + NULLABLE(locals); + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return PyRun_StringFlags(str, start, globals, locals, pflags); +} + +static PyObject * +run_fileexflags(PyObject *mod, PyObject *pos_args) +{ + PyObject *result = NULL; + const char *filename = NULL; + Py_ssize_t filename_size; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; + int closeit = 0; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + FILE *fp = NULL; + + if (!PyArg_ParseTuple(pos_args, "z#iO|Oiii", + &filename, &filename_size, &start, &globals, &locals, + &closeit, &cf_flags, &cf_feature_version)) { + return NULL; + } + + NULLABLE(globals); + NULLABLE(locals); + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + fp = fopen(filename, "r"); + if (fp == NULL) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); + return NULL; + } + int fd = fileno(fp); + + result = PyRun_FileExFlags(fp, filename, start, globals, locals, closeit, pflags); + + if (closeit && result && _Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "File was not closed after execution"); + Py_DECREF(result); + fclose(fp); + return NULL; + } + + if (!closeit && !_Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "Bad file descriptor after execution"); + Py_XDECREF(result); + return NULL; + } + + if (!closeit) { + fclose(fp); /* don't need open file any more*/ + } + + return result; +} + +static PyMethodDef test_methods[] = { + {"run_stringflags", run_stringflags, METH_VARARGS}, + {"run_fileexflags", run_fileexflags, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Run(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/set.c b/Modules/_testcapi/set.c index 2fbd0aeffcd9f9..31b52cee5e9623 100644 --- a/Modules/_testcapi/set.c +++ b/Modules/_testcapi/set.c @@ -1,75 +1,6 @@ #include "parts.h" #include "util.h" -static PyObject * -set_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Check(obj)); -} - -static PyObject * -set_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_CheckExact(obj)); -} - -static PyObject * -frozenset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_Check(obj)); -} - -static PyObject * -frozenset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_CheckExact(obj)); -} - -static PyObject * -anyset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_Check(obj)); -} - -static PyObject * -anyset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_CheckExact(obj)); -} - -static PyObject * -set_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PySet_New(iterable); -} - -static PyObject * -frozenset_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PyFrozenSet_New(iterable); -} - -static PyObject * -set_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySet_Size(obj)); -} - static PyObject * set_get_size(PyObject *self, PyObject *obj) { @@ -77,111 +8,8 @@ set_get_size(PyObject *self, PyObject *obj) RETURN_SIZE(PySet_GET_SIZE(obj)); } -static PyObject * -set_contains(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Contains(obj, item)); -} - -static PyObject * -set_add(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Add(obj, item)); -} - -static PyObject * -set_discard(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Discard(obj, item)); -} - -static PyObject * -set_pop(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySet_Pop(obj); -} - -static PyObject * -set_clear(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Clear(obj)); -} - -static PyObject * -test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) -{ - // Test that `frozenset` can be used with `PySet_Add`, - // when frozenset is just created in CAPI. - PyObject *fs = PyFrozenSet_New(NULL); - if (fs == NULL) { - return NULL; - } - PyObject *num = PyLong_FromLong(1); - if (num == NULL) { - goto error; - } - if (PySet_Add(fs, num) < 0) { - goto error; - } - int contains = PySet_Contains(fs, num); - if (contains < 0) { - goto error; - } - else if (contains == 0) { - goto unexpected; - } - Py_DECREF(fs); - Py_DECREF(num); - Py_RETURN_NONE; - -unexpected: - PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); -error: - Py_DECREF(fs); - Py_XDECREF(num); - return NULL; -} - static PyMethodDef test_methods[] = { - {"set_check", set_check, METH_O}, - {"set_checkexact", set_checkexact, METH_O}, - {"frozenset_check", frozenset_check, METH_O}, - {"frozenset_checkexact", frozenset_checkexact, METH_O}, - {"anyset_check", anyset_check, METH_O}, - {"anyset_checkexact", anyset_checkexact, METH_O}, - - {"set_new", set_new, METH_VARARGS}, - {"frozenset_new", frozenset_new, METH_VARARGS}, - - {"set_size", set_size, METH_O}, {"set_get_size", set_get_size, METH_O}, - {"set_contains", set_contains, METH_VARARGS}, - {"set_add", set_add, METH_VARARGS}, - {"set_discard", set_discard, METH_VARARGS}, - {"set_pop", set_pop, METH_O}, - {"set_clear", set_clear, METH_O}, - - {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapi/time.c b/Modules/_testcapi/time.c new file mode 100644 index 00000000000000..464cf5c3125012 --- /dev/null +++ b/Modules/_testcapi/time.c @@ -0,0 +1,169 @@ +#include "parts.h" + + +static int +pytime_from_nanoseconds(PyTime_t *tp, PyObject *obj) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expect int, got %s", + Py_TYPE(obj)->tp_name); + return -1; + } + + long long nsec = PyLong_AsLongLong(obj); + if (nsec == -1 && PyErr_Occurred()) { + return -1; + } + + Py_BUILD_ASSERT(sizeof(long long) == sizeof(PyTime_t)); + *tp = (PyTime_t)nsec; + return 0; +} + + +static PyObject * +test_pytime_assecondsdouble(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + PyTime_t ts; + if (pytime_from_nanoseconds(&ts, obj) < 0) { + return NULL; + } + double d = PyTime_AsSecondsDouble(ts); + return PyFloat_FromDouble(d); +} + + +static PyObject* +pytime_as_float(PyTime_t t) +{ + return PyFloat_FromDouble(PyTime_AsSecondsDouble(t)); +} + + + +static PyObject* +test_pytime_monotonic(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res = PyTime_Monotonic(&t); + if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_monotonic_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_MonotonicRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_MonotonicRaw() failed"); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_perf_counter(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res = PyTime_PerfCounter(&t); + if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_perf_counter_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_PerfCounterRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_PerfCounterRaw() failed"); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res = PyTime_Time(&t); + if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_time_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_TimeRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_TimeRaw() failed"); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyMethodDef test_methods[] = { + {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, + {"PyTime_Monotonic", test_pytime_monotonic, METH_NOARGS}, + {"PyTime_MonotonicRaw", test_pytime_monotonic_raw, METH_NOARGS}, + {"PyTime_PerfCounter", test_pytime_perf_counter, METH_NOARGS}, + {"PyTime_PerfCounterRaw", test_pytime_perf_counter_raw, METH_NOARGS}, + {"PyTime_Time", test_pytime_time, METH_NOARGS}, + {"PyTime_TimeRaw", test_pytime_time_raw, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Time(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + Py_BUILD_ASSERT(sizeof(long long) == sizeof(PyTime_t)); + if (PyModule_AddObject(m, "PyTime_MIN", PyLong_FromLongLong(PyTime_MIN)) < 0) { + return 1; + } + if (PyModule_AddObject(m, "PyTime_MAX", PyLong_FromLongLong(PyTime_MAX)) < 0) { + return 1; + } + return 0; +} diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index 95dde8c0edadbe..d9c02ba0ff04fe 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -2,14 +2,121 @@ #include "util.h" +static PyObject * +tuple_get_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyTuple_GET_SIZE(obj)); +} + +static PyObject * +tuple_get_item(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyTuple_GET_ITEM(obj, i)); +} + +static PyObject * +tuple_copy(PyObject *tuple) +{ + Py_ssize_t size = PyTuple_GET_SIZE(tuple); + PyObject *newtuple = PyTuple_New(size); + if (!newtuple) { + return NULL; + } + for (Py_ssize_t n = 0; n < size; n++) { + PyTuple_SET_ITEM(newtuple, n, Py_XNewRef(PyTuple_GET_ITEM(tuple, n))); + } + return newtuple; +} + +static PyObject * +tuple_set_item(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value, *newtuple; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(value); + if (PyTuple_CheckExact(obj)) { + newtuple = tuple_copy(obj); + if (!newtuple) { + return NULL; + } + + PyObject *val = PyTuple_GET_ITEM(newtuple, i); + PyTuple_SET_ITEM(newtuple, i, Py_XNewRef(value)); + Py_DECREF(val); + return newtuple; + } + else { + NULLABLE(obj); + + PyObject *val = PyTuple_GET_ITEM(obj, i); + PyTuple_SET_ITEM(obj, i, Py_XNewRef(value)); + Py_DECREF(val); + return Py_XNewRef(obj); + } +} + +static PyObject * +_tuple_resize(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *tup; + Py_ssize_t newsize; + int new = 1; + if (!PyArg_ParseTuple(args, "On|p", &tup, &newsize, &new)) { + return NULL; + } + if (new) { + tup = tuple_copy(tup); + if (!tup) { + return NULL; + } + } + else { + NULLABLE(tup); + Py_XINCREF(tup); + } + int r = _PyTuple_Resize(&tup, newsize); + if (r == -1) { + assert(tup == NULL); + return NULL; + } + return tup; +} + +static PyObject * +_check_tuple_item_is_NULL(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + return PyLong_FromLong(PyTuple_GET_ITEM(obj, i) == NULL); +} + + static PyMethodDef test_methods[] = { + {"tuple_get_size", tuple_get_size, METH_O}, + {"tuple_get_item", tuple_get_item, METH_VARARGS}, + {"tuple_set_item", tuple_set_item, METH_VARARGS}, + {"_tuple_resize", _tuple_resize, METH_VARARGS}, + {"_check_tuple_item_is_NULL", _check_tuple_item_is_NULL, METH_VARARGS}, {NULL}, }; int _PyTestCapi_Init_Tuple(PyObject *m) { - if (PyModule_AddFunctions(m, test_methods) < 0){ + if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; } diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index a10183dddeca98..b8ecf53f4f8b9c 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -3,103 +3,29 @@ #include "parts.h" #include "util.h" -static struct PyModuleDef *_testcapimodule = NULL; // set at initialization - -static PyObject * -codec_incrementalencoder(PyObject *self, PyObject *args) -{ - const char *encoding, *errors = NULL; - if (!PyArg_ParseTuple(args, "s|s:test_incrementalencoder", - &encoding, &errors)) - return NULL; - return PyCodec_IncrementalEncoder(encoding, errors); -} - -static PyObject * -codec_incrementaldecoder(PyObject *self, PyObject *args) -{ - const char *encoding, *errors = NULL; - if (!PyArg_ParseTuple(args, "s|s:test_incrementaldecoder", - &encoding, &errors)) - return NULL; - return PyCodec_IncrementalDecoder(encoding, errors); -} - -static PyObject * -test_unicode_compare_with_ascii(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *py_s = PyUnicode_FromStringAndSize("str\0", 4); - int result; - if (py_s == NULL) - return NULL; - result = PyUnicode_CompareWithASCIIString(py_s, "str"); - Py_DECREF(py_s); - if (!result) { - PyErr_SetString(PyExc_AssertionError, "Python string ending in NULL " - "should not compare equal to c string."); - return NULL; - } - Py_RETURN_NONE; -} - +/* Test PyUnicode_New() */ static PyObject * -test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) +unicode_new(PyObject *self, PyObject *args) { -#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) - const wchar_t wtext[2] = {(wchar_t)0x10ABCDu}; - size_t wtextlen = 1; - const wchar_t invalid[1] = {(wchar_t)0x110000u}; -#else - const wchar_t wtext[3] = {(wchar_t)0xDBEAu, (wchar_t)0xDFCDu}; - size_t wtextlen = 2; -#endif - PyObject *wide, *utf8; - - wide = PyUnicode_FromWideChar(wtext, wtextlen); - if (wide == NULL) - return NULL; + Py_ssize_t size; + unsigned int maxchar; + PyObject *result; - utf8 = PyUnicode_FromString("\xf4\x8a\xaf\x8d"); - if (utf8 == NULL) { - Py_DECREF(wide); + if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { return NULL; } - if (PyUnicode_GET_LENGTH(wide) != PyUnicode_GET_LENGTH(utf8)) { - Py_DECREF(wide); - Py_DECREF(utf8); - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "wide string and utf8 string " - "have different length"); - return NULL; - } - if (PyUnicode_Compare(wide, utf8)) { - Py_DECREF(wide); - Py_DECREF(utf8); - if (PyErr_Occurred()) - return NULL; - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "wide string and utf8 string " - "are different"); + result = PyUnicode_New(size, (Py_UCS4)maxchar); + if (!result) { return NULL; } - - Py_DECREF(wide); - Py_DECREF(utf8); - -#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) - wide = PyUnicode_FromWideChar(invalid, 1); - if (wide == NULL) - PyErr_Clear(); - else { - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail"); + if (size > 0 && maxchar <= 0x10ffff && + PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) + { + Py_DECREF(result); return NULL; } -#endif - Py_RETURN_NONE; + return result; } @@ -130,30 +56,6 @@ unicode_copy(PyObject *unicode) return copy; } -/* Test PyUnicode_New() */ -static PyObject * -unicode_new(PyObject *self, PyObject *args) -{ - Py_ssize_t size; - unsigned int maxchar; - PyObject *result; - - if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { - return NULL; - } - - result = PyUnicode_New(size, (Py_UCS4)maxchar); - if (!result) { - return NULL; - } - if (size > 0 && maxchar <= 0x10ffff && - PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) - { - Py_DECREF(result); - return NULL; - } - return result; -} /* Test PyUnicode_Fill() */ static PyObject * @@ -180,1929 +82,490 @@ unicode_fill(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", to_copy, filled); } -/* Test PyUnicode_WriteChar() */ + +/* Test PyUnicode_FromKindAndData() */ static PyObject * -unicode_writechar(PyObject *self, PyObject *args) +unicode_fromkindanddata(PyObject *self, PyObject *args) { - PyObject *to, *to_copy; - Py_ssize_t index; - unsigned int character; - int result; + int kind; + void *buffer; + Py_ssize_t bsize; + Py_ssize_t size = -100; - if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + if (!PyArg_ParseTuple(args, "iz#|n", &kind, &buffer, &bsize, &size)) { return NULL; } - NULLABLE(to); - if (!(to_copy = unicode_copy(to)) && to) { - return NULL; + if (size == -100) { + size = bsize; } - - result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); - if (result == -1 && PyErr_Occurred()) { - Py_DECREF(to_copy); + if (kind && size % kind) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromkindanddata()"); return NULL; } - return Py_BuildValue("(Ni)", to_copy, result); + return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0); } -/* Test PyUnicode_Resize() */ + +// Test PyUnicode_AsUCS4(). +// Part of the limited C API, but the test needs PyUnicode_FromKindAndData(). static PyObject * -unicode_resize(PyObject *self, PyObject *args) +unicode_asucs4(PyObject *self, PyObject *args) { - PyObject *obj, *copy; - Py_ssize_t length; - int result; + PyObject *unicode, *result; + Py_UCS4 *buffer; + int copy_null; + Py_ssize_t str_len, buf_len; - if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + if (!PyArg_ParseTuple(args, "Onp:unicode_asucs4", &unicode, &str_len, ©_null)) { return NULL; } - NULLABLE(obj); - if (!(copy = unicode_copy(obj)) && obj) { - return NULL; + NULLABLE(unicode); + buf_len = str_len + 1; + buffer = PyMem_NEW(Py_UCS4, buf_len); + if (buffer == NULL) { + return PyErr_NoMemory(); } - result = PyUnicode_Resize(©, length); - if (result == -1 && PyErr_Occurred()) { - Py_XDECREF(copy); + memset(buffer, 0, sizeof(Py_UCS4)*buf_len); + buffer[str_len] = 0xffffU; + + if (!PyUnicode_AsUCS4(unicode, buffer, buf_len, copy_null)) { + PyMem_Free(buffer); return NULL; } - if (obj && PyUnicode_Check(obj) && length > PyUnicode_GET_LENGTH(obj)) { - if (PyUnicode_Fill(copy, PyUnicode_GET_LENGTH(obj), length, 0U) < 0) { - Py_DECREF(copy); - return NULL; - } - } - return Py_BuildValue("(Ni)", copy, result); + + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, buf_len); + PyMem_Free(buffer); + return result; } -/* Test PyUnicode_Append() */ + +// Test PyUnicode_AsUCS4Copy(). +// Part of the limited C API, but the test needs PyUnicode_FromKindAndData(). static PyObject * -unicode_append(PyObject *self, PyObject *args) +unicode_asucs4copy(PyObject *self, PyObject *args) { - PyObject *left, *right, *left_copy; + PyObject *unicode; + Py_UCS4 *buffer; + PyObject *result; - if (!PyArg_ParseTuple(args, "OO", &left, &right)) + if (!PyArg_ParseTuple(args, "O", &unicode)) { return NULL; + } - NULLABLE(left); - NULLABLE(right); - if (!(left_copy = unicode_copy(left)) && left) { + NULLABLE(unicode); + buffer = PyUnicode_AsUCS4Copy(unicode); + if (buffer == NULL) { return NULL; } - PyUnicode_Append(&left_copy, right); - return left_copy; + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buffer, + PyUnicode_GET_LENGTH(unicode) + 1); + PyMem_FREE(buffer); + return result; } -/* Test PyUnicode_AppendAndDel() */ + +/* Test PyUnicode_AsUTF8() */ static PyObject * -unicode_appendanddel(PyObject *self, PyObject *args) +unicode_asutf8(PyObject *self, PyObject *args) { - PyObject *left, *right, *left_copy; + PyObject *unicode; + Py_ssize_t buflen; + const char *s; - if (!PyArg_ParseTuple(args, "OO", &left, &right)) + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; - NULLABLE(left); - NULLABLE(right); - if (!(left_copy = unicode_copy(left)) && left) { + NULLABLE(unicode); + s = PyUnicode_AsUTF8(unicode); + if (s == NULL) return NULL; - } - Py_XINCREF(right); - PyUnicode_AppendAndDel(&left_copy, right); - return left_copy; + + return PyBytes_FromStringAndSize(s, buflen); } -/* Test PyUnicode_FromStringAndSize() */ + +/* Test PyUnicode_CopyCharacters() */ static PyObject * -unicode_fromstringandsize(PyObject *self, PyObject *args) +unicode_copycharacters(PyObject *self, PyObject *args) { - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; + PyObject *from, *to, *to_copy; + Py_ssize_t from_start, to_start, how_many, copied; - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start, + &from, &from_start, &how_many)) { return NULL; } - if (size == -100) { - size = bsize; + NULLABLE(from); + if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to), + PyUnicode_MAX_CHAR_VALUE(to)))) { + return NULL; + } + if (PyUnicode_Fill(to_copy, 0, PyUnicode_GET_LENGTH(to_copy), 0U) < 0) { + Py_DECREF(to_copy); + return NULL; + } + + copied = PyUnicode_CopyCharacters(to_copy, to_start, from, + from_start, how_many); + if (copied == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; } - return PyUnicode_FromStringAndSize(s, size); + + return Py_BuildValue("(Nn)", to_copy, copied); } -/* Test PyUnicode_FromString() */ + +// --- PyUnicodeWriter type ------------------------------------------------- + +typedef struct { + PyObject_HEAD + PyUnicodeWriter *writer; +} WriterObject; + + static PyObject * -unicode_fromstring(PyObject *self, PyObject *arg) +writer_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - const char *s; - Py_ssize_t size; - - if (!PyArg_Parse(arg, "z#", &s, &size)) { + WriterObject *self = (WriterObject *)type->tp_alloc(type, 0); + if (!self) { return NULL; } - return PyUnicode_FromString(s); + self->writer = NULL; + return (PyObject*)self; } -/* Test PyUnicode_FromKindAndData() */ -static PyObject * -unicode_fromkindanddata(PyObject *self, PyObject *args) + +static int +writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) { - int kind; - void *buffer; - Py_ssize_t bsize; - Py_ssize_t size = -100; + WriterObject *self = (WriterObject *)self_raw; - if (!PyArg_ParseTuple(args, "iz#|n", &kind, &buffer, &bsize, &size)) { - return NULL; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "n", &size)) { + return -1; } - if (size == -100) { - size = bsize; + if (self->writer) { + PyUnicodeWriter_Discard(self->writer); } - if (kind && size % kind) { - PyErr_SetString(PyExc_AssertionError, - "invalid size in unicode_fromkindanddata()"); - return NULL; + + self->writer = PyUnicodeWriter_Create(size); + if (self->writer == NULL) { + return -1; } - return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0); + return 0; } -/* Test PyUnicode_Substring() */ -static PyObject * -unicode_substring(PyObject *self, PyObject *args) -{ - PyObject *str; - Py_ssize_t start, end; - if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { - return NULL; +static void +writer_dealloc(PyObject *self_raw) +{ + WriterObject *self = (WriterObject *)self_raw; + PyTypeObject *tp = Py_TYPE(self); + if (self->writer) { + PyUnicodeWriter_Discard(self->writer); } - - NULLABLE(str); - return PyUnicode_Substring(str, start, end); + tp->tp_free(self); + Py_DECREF(tp); } -/* Test PyUnicode_GetLength() */ -static PyObject * -unicode_getlength(PyObject *self, PyObject *arg) + +static inline int +writer_check(WriterObject *self) { - NULLABLE(arg); - RETURN_SIZE(PyUnicode_GetLength(arg)); + if (self->writer == NULL) { + PyErr_SetString(PyExc_ValueError, "operation on finished writer"); + return -1; + } + return 0; } -/* Test PyUnicode_ReadChar() */ -static PyObject * -unicode_readchar(PyObject *self, PyObject *args) + +static PyObject* +writer_write_char(PyObject *self_raw, PyObject *args) { - PyObject *unicode; - Py_ssize_t index; - Py_UCS4 result; + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } - if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { + PyObject *str; + if (!PyArg_ParseTuple(args, "U", &str)) { return NULL; } + if (PyUnicode_GET_LENGTH(str) != 1) { + PyErr_SetString(PyExc_ValueError, "expect a single character"); + } + Py_UCS4 ch = PyUnicode_READ_CHAR(str, 0); - NULLABLE(unicode); - result = PyUnicode_ReadChar(unicode, index); - if (result == (Py_UCS4)-1) + if (PyUnicodeWriter_WriteChar(self->writer, ch) < 0) { return NULL; - return PyLong_FromUnsignedLong(result); + } + Py_RETURN_NONE; } -/* Test PyUnicode_FromEncodedObject() */ -static PyObject * -unicode_fromencodedobject(PyObject *self, PyObject *args) + +static PyObject* +writer_write_utf8(PyObject *self_raw, PyObject *args) { - PyObject *obj; - const char *encoding; - const char *errors = NULL; + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } - if (!PyArg_ParseTuple(args, "Oz|z", &obj, &encoding, &errors)) { + char *str; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "yn", &str, &size)) { return NULL; } - NULLABLE(obj); - return PyUnicode_FromEncodedObject(obj, encoding, errors); + if (PyUnicodeWriter_WriteUTF8(self->writer, str, size) < 0) { + return NULL; + } + Py_RETURN_NONE; } -/* Test PyUnicode_FromObject() */ -static PyObject * -unicode_fromobject(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_FromObject(arg); -} -/* Test PyUnicode_InternInPlace() */ -static PyObject * -unicode_interninplace(PyObject *self, PyObject *arg) +static PyObject* +writer_write_widechar(PyObject *self_raw, PyObject *args) { - NULLABLE(arg); - Py_XINCREF(arg); - PyUnicode_InternInPlace(&arg); - return arg; -} + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyObject *str; + if (!PyArg_ParseTuple(args, "U", &str)) { + return NULL; + } -/* Test PyUnicode_InternFromString() */ -static PyObject * -unicode_internfromstring(PyObject *self, PyObject *arg) -{ - const char *s; Py_ssize_t size; + wchar_t *wstr = PyUnicode_AsWideCharString(str, &size); + if (wstr == NULL) { + return NULL; + } - if (!PyArg_Parse(arg, "z#", &s, &size)) { + int res = PyUnicodeWriter_WriteWideChar(self->writer, wstr, size); + PyMem_Free(wstr); + if (res < 0) { return NULL; } - return PyUnicode_InternFromString(s); + Py_RETURN_NONE; } -/* Test PyUnicode_FromWideChar() */ -static PyObject * -unicode_fromwidechar(PyObject *self, PyObject *args) -{ - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { +static PyObject* +writer_write_ucs4(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - if (size == -100) { - if (bsize % SIZEOF_WCHAR_T) { - PyErr_SetString(PyExc_AssertionError, - "invalid size in unicode_fromwidechar()"); - return NULL; - } - size = bsize / SIZEOF_WCHAR_T; - } - return PyUnicode_FromWideChar((const wchar_t *)s, size); -} -/* Test PyUnicode_AsWideChar() */ -static PyObject * -unicode_aswidechar(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_ssize_t buflen, size; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - NULLABLE(unicode); - buffer = PyMem_New(wchar_t, buflen); - if (buffer == NULL) - return PyErr_NoMemory(); - - size = PyUnicode_AsWideChar(unicode, buffer, buflen); - if (size == -1) { - PyMem_Free(buffer); - return NULL; - } - - if (size < buflen) - buflen = size + 1; - else - buflen = size; - result = PyUnicode_FromWideChar(buffer, buflen); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - - return Py_BuildValue("(Nn)", result, size); -} - -/* Test PyUnicode_AsWideCharString() with NULL as buffer */ -static PyObject * -unicode_aswidechar_null(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - NULLABLE(unicode); - RETURN_SIZE(PyUnicode_AsWideChar(unicode, NULL, buflen)); -} - -/* Test PyUnicode_AsWideCharString() */ -static PyObject * -unicode_aswidecharstring(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_ssize_t size = UNINITIALIZED_SIZE; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "O", &unicode)) - return NULL; - - NULLABLE(unicode); - buffer = PyUnicode_AsWideCharString(unicode, &size); - if (buffer == NULL) { - assert(size == UNINITIALIZED_SIZE); - return NULL; - } - - result = PyUnicode_FromWideChar(buffer, size + 1); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - return Py_BuildValue("(Nn)", result, size); -} - -/* Test PyUnicode_AsWideCharString() with NULL as the size address */ -static PyObject * -unicode_aswidecharstring_null(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "O", &unicode)) - return NULL; - - NULLABLE(unicode); - buffer = PyUnicode_AsWideCharString(unicode, NULL); - if (buffer == NULL) - return NULL; - - result = PyUnicode_FromWideChar(buffer, -1); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - return result; -} - -/* Test PyUnicode_AsUCS4() */ -static PyObject * -unicode_asucs4(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_UCS4 *buffer; - int copy_null; - Py_ssize_t str_len, buf_len; - - if (!PyArg_ParseTuple(args, "Onp:unicode_asucs4", &unicode, &str_len, ©_null)) { - return NULL; - } - - NULLABLE(unicode); - buf_len = str_len + 1; - buffer = PyMem_NEW(Py_UCS4, buf_len); - if (buffer == NULL) { - return PyErr_NoMemory(); - } - memset(buffer, 0, sizeof(Py_UCS4)*buf_len); - buffer[str_len] = 0xffffU; - - if (!PyUnicode_AsUCS4(unicode, buffer, buf_len, copy_null)) { - PyMem_Free(buffer); - return NULL; - } - - result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, buf_len); - PyMem_Free(buffer); - return result; -} - -/* Test PyUnicode_AsUCS4Copy() */ -static PyObject * -unicode_asucs4copy(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_UCS4 *buffer; - PyObject *result; - - if (!PyArg_ParseTuple(args, "O", &unicode)) { - return NULL; - } - - NULLABLE(unicode); - buffer = PyUnicode_AsUCS4Copy(unicode); - if (buffer == NULL) { - return NULL; - } - result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buffer, - PyUnicode_GET_LENGTH(unicode) + 1); - PyMem_FREE(buffer); - return result; -} - -/* Test PyUnicode_FromOrdinal() */ -static PyObject * -unicode_fromordinal(PyObject *self, PyObject *args) -{ - int ordinal; - - if (!PyArg_ParseTuple(args, "i", &ordinal)) - return NULL; - - return PyUnicode_FromOrdinal(ordinal); -} - -/* Test PyUnicode_AsUTF8() */ -static PyObject * -unicode_asutf8(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8(unicode); - if (s == NULL) - return NULL; - - return PyBytes_FromStringAndSize(s, buflen); -} - -/* Test PyUnicode_AsUTF8AndSize() */ -static PyObject * -unicode_asutf8andsize(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - Py_ssize_t size = UNINITIALIZED_SIZE; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8AndSize(unicode, &size); - if (s == NULL) { - assert(size == -1); - return NULL; - } - - return Py_BuildValue("(y#n)", s, buflen, size); -} - -/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ -static PyObject * -unicode_asutf8andsize_null(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8AndSize(unicode, NULL); - if (s == NULL) - return NULL; - - return PyBytes_FromStringAndSize(s, buflen); -} - -/* Test PyUnicode_GetDefaultEncoding() */ -static PyObject * -unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - const char *s = PyUnicode_GetDefaultEncoding(); - if (s == NULL) - return NULL; - - return PyBytes_FromString(s); -} - -/* Test PyUnicode_Decode() */ -static PyObject * -unicode_decode(PyObject *self, PyObject *args) -{ - const char *s; - Py_ssize_t size; - const char *encoding; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#z|z", &s, &size, &encoding, &errors)) - return NULL; - - return PyUnicode_Decode(s, size, encoding, errors); -} - -/* Test PyUnicode_AsEncodedString() */ -static PyObject * -unicode_asencodedstring(PyObject *self, PyObject *args) -{ - PyObject *unicode; - const char *encoding; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "Oz|z", &unicode, &encoding, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_AsEncodedString(unicode, encoding, errors); -} - -/* Test PyUnicode_BuildEncodingMap() */ -static PyObject * -unicode_buildencodingmap(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_BuildEncodingMap(arg); -} - -/* Test PyUnicode_DecodeUTF7() */ -static PyObject * -unicode_decodeutf7(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUTF7(data, size, errors); -} - -/* Test PyUnicode_DecodeUTF7Stateful() */ -static PyObject * -unicode_decodeutf7stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_DecodeUTF8() */ -static PyObject * -unicode_decodeutf8(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUTF8(data, size, errors); -} - -/* Test PyUnicode_DecodeUTF8Stateful() */ -static PyObject * -unicode_decodeutf8stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_AsUTF8String() */ -static PyObject * -unicode_asutf8string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF8String(arg); -} - -/* Test PyUnicode_DecodeUTF32() */ -static PyObject * -unicode_decodeutf32(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF32(data, size, errors, &byteorder); - if (!result) { - return NULL; - } - return Py_BuildValue("(iN)", byteorder, result); -} - -/* Test PyUnicode_DecodeUTF32Stateful() */ -static PyObject * -unicode_decodeutf32stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(iNn)", byteorder, result, consumed); -} - -/* Test PyUnicode_AsUTF32String() */ -static PyObject * -unicode_asutf32string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF32String(arg); -} - -/* Test PyUnicode_DecodeUTF16() */ -static PyObject * -unicode_decodeutf16(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF16(data, size, errors, &byteorder); - if (!result) { - return NULL; - } - return Py_BuildValue("(iN)", byteorder, result); -} - -/* Test PyUnicode_DecodeUTF16Stateful() */ -static PyObject * -unicode_decodeutf16stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(iNn)", byteorder, result, consumed); -} - -/* Test PyUnicode_AsUTF16String() */ -static PyObject * -unicode_asutf16string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF16String(arg); -} - -/* Test PyUnicode_DecodeUnicodeEscape() */ -static PyObject * -unicode_decodeunicodeescape(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUnicodeEscape(data, size, errors); -} - -/* Test PyUnicode_AsUnicodeEscapeString() */ -static PyObject * -unicode_asunicodeescapestring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUnicodeEscapeString(arg); -} - -static PyObject * -unicode_decoderawunicodeescape(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeRawUnicodeEscape(data, size, errors); -} - -/* Test PyUnicode_AsRawUnicodeEscapeString() */ -static PyObject * -unicode_asrawunicodeescapestring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsRawUnicodeEscapeString(arg); -} - -static PyObject * -unicode_decodelatin1(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLatin1(data, size, errors); -} - -/* Test PyUnicode_AsLatin1String() */ -static PyObject * -unicode_aslatin1string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsLatin1String(arg); -} - -/* Test PyUnicode_DecodeASCII() */ -static PyObject * -unicode_decodeascii(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeASCII(data, size, errors); -} - -/* Test PyUnicode_AsASCIIString() */ -static PyObject * -unicode_asasciistring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsASCIIString(arg); -} - -/* Test PyUnicode_DecodeCharmap() */ -static PyObject * -unicode_decodecharmap(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - PyObject *mapping; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#O|z", &data, &size, &mapping, &errors)) - return NULL; - - NULLABLE(mapping); - return PyUnicode_DecodeCharmap(data, size, mapping, errors); -} - -/* Test PyUnicode_AsCharmapString() */ -static PyObject * -unicode_ascharmapstring(PyObject *self, PyObject *args) -{ - PyObject *unicode; - PyObject *mapping; - - if (!PyArg_ParseTuple(args, "OO", &unicode, &mapping)) - return NULL; - - NULLABLE(unicode); - NULLABLE(mapping); - return PyUnicode_AsCharmapString(unicode, mapping); -} - -#ifdef MS_WINDOWS - -/* Test PyUnicode_DecodeMBCS() */ -static PyObject * -unicode_decodembcs(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeMBCS(data, size, errors); -} - -/* Test PyUnicode_DecodeMBCSStateful() */ -static PyObject * -unicode_decodembcsstateful(PyObject *self, PyObject *args) -{ - const char *data; + PyObject *str; Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); + if (!PyArg_ParseTuple(args, "Un", &str, &size)) { return NULL; } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_DecodeCodePageStateful() */ -static PyObject * -unicode_decodecodepagestateful(PyObject *self, PyObject *args) -{ - int code_page; - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) - return NULL; + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + size = Py_MIN(size, len); - result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); + Py_UCS4 *ucs4 = PyUnicode_AsUCS4Copy(str); + if (ucs4 == NULL) { return NULL; } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_AsMBCSString() */ -static PyObject * -unicode_asmbcsstring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsMBCSString(arg); -} - -/* Test PyUnicode_EncodeCodePage() */ -static PyObject * -unicode_encodecodepage(PyObject *self, PyObject *args) -{ - int code_page; - PyObject *unicode; - const char *errors; - - if (!PyArg_ParseTuple(args, "iO|z", &code_page, &unicode, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_EncodeCodePage(code_page, unicode, errors); -} - -#endif /* MS_WINDOWS */ - -/* Test PyUnicode_DecodeLocaleAndSize() */ -static PyObject * -unicode_decodelocaleandsize(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLocaleAndSize(data, size, errors); -} - -/* Test PyUnicode_DecodeLocale() */ -static PyObject * -unicode_decodelocale(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLocale(data, errors); -} - -/* Test PyUnicode_EncodeLocale() */ -static PyObject * -unicode_encodelocale(PyObject *self, PyObject *args) -{ - PyObject *unicode; - const char *errors; - - if (!PyArg_ParseTuple(args, "O|z", &unicode, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_EncodeLocale(unicode, errors); -} -/* Test PyUnicode_DecodeFSDefault() */ -static PyObject * -unicode_decodefsdefault(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - - if (!PyArg_ParseTuple(args, "y#", &data, &size)) - return NULL; - - return PyUnicode_DecodeFSDefault(data); -} - -/* Test PyUnicode_DecodeFSDefaultAndSize() */ -static PyObject * -unicode_decodefsdefaultandsize(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - - if (!PyArg_ParseTuple(args, "y#|n", &data, &size, &size)) + int res = PyUnicodeWriter_WriteUCS4(self->writer, ucs4, size); + PyMem_Free(ucs4); + if (res < 0) { return NULL; - - return PyUnicode_DecodeFSDefaultAndSize(data, size); -} - -/* Test PyUnicode_EncodeFSDefault() */ -static PyObject * -unicode_encodefsdefault(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_EncodeFSDefault(arg); + } + Py_RETURN_NONE; } -/* Test PyUnicode_Concat() */ -static PyObject * -unicode_concat(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - return PyUnicode_Concat(left, right); -} -/* Test PyUnicode_Split() */ -static PyObject * -unicode_split(PyObject *self, PyObject *args) +static PyObject* +writer_write_str(PyObject *self_raw, PyObject *args) { - PyObject *s; - PyObject *sep; - Py_ssize_t maxsplit = -1; - - if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; + } - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_Split(s, sep, maxsplit); -} - -/* Test PyUnicode_RSplit() */ -static PyObject * -unicode_rsplit(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - Py_ssize_t maxsplit = -1; - - if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; + } - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_RSplit(s, sep, maxsplit); -} - -/* Test PyUnicode_Splitlines() */ -static PyObject * -unicode_splitlines(PyObject *self, PyObject *args) -{ - PyObject *s; - int keepends = 0; - - if (!PyArg_ParseTuple(args, "O|i", &s, &keepends)) + if (PyUnicodeWriter_WriteStr(self->writer, obj) < 0) { return NULL; - - NULLABLE(s); - return PyUnicode_Splitlines(s, keepends); + } + Py_RETURN_NONE; } -/* Test PyUnicode_Partition() */ -static PyObject * -unicode_partition(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - - if (!PyArg_ParseTuple(args, "OO", &s, &sep)) - return NULL; - - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_Partition(s, sep); -} -/* Test PyUnicode_RPartition() */ -static PyObject * -unicode_rpartition(PyObject *self, PyObject *args) +static PyObject* +writer_write_repr(PyObject *self_raw, PyObject *args) { - PyObject *s; - PyObject *sep; - - if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; + } - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_RPartition(s, sep); -} - -/* Test PyUnicode_Translate() */ -static PyObject * -unicode_translate(PyObject *self, PyObject *args) -{ PyObject *obj; - PyObject *table; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "OO|z", &obj, &table, &errors)) + if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; + } - NULLABLE(obj); - NULLABLE(table); - return PyUnicode_Translate(obj, table, errors); -} - -/* Test PyUnicode_Join() */ -static PyObject * -unicode_join(PyObject *self, PyObject *args) -{ - PyObject *sep; - PyObject *seq; - - if (!PyArg_ParseTuple(args, "OO", &sep, &seq)) + if (PyUnicodeWriter_WriteRepr(self->writer, obj) < 0) { return NULL; - - NULLABLE(sep); - NULLABLE(seq); - return PyUnicode_Join(sep, seq); + } + Py_RETURN_NONE; } -/* Test PyUnicode_Count() */ -static PyObject * -unicode_count(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - - if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - RETURN_SIZE(PyUnicode_Count(str, substr, start, end)); -} -/* Test PyUnicode_Find() */ -static PyObject * -unicode_find(PyObject *self, PyObject *args) +static PyObject* +writer_write_substring(PyObject *self_raw, PyObject *args) { - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - int direction; - Py_ssize_t result; - - if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - result = PyUnicode_Find(str, substr, start, end, direction); - if (result == -2) { - assert(PyErr_Occurred()); + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - assert(!PyErr_Occurred()); - return PyLong_FromSsize_t(result); -} - -/* Test PyUnicode_Tailmatch() */ -static PyObject * -unicode_tailmatch(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - int direction; - - if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - RETURN_SIZE(PyUnicode_Tailmatch(str, substr, start, end, direction)); -} -/* Test PyUnicode_FindChar() */ -static PyObject * -unicode_findchar(PyObject *self, PyObject *args) -{ PyObject *str; - int direction; - unsigned int ch; - Py_ssize_t result; Py_ssize_t start, end; - - if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, - &start, &end, &direction)) { - return NULL; - } - NULLABLE(str); - result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); - if (result == -2) { - assert(PyErr_Occurred()); + if (!PyArg_ParseTuple(args, "Unn", &str, &start, &end)) { return NULL; } - assert(!PyErr_Occurred()); - return PyLong_FromSsize_t(result); -} - -/* Test PyUnicode_Replace() */ -static PyObject * -unicode_replace(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - PyObject *replstr; - Py_ssize_t maxcount = -1; - - if (!PyArg_ParseTuple(args, "OOO|n", &str, &substr, &replstr, &maxcount)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - NULLABLE(replstr); - return PyUnicode_Replace(str, substr, replstr, maxcount); -} - -/* Test PyUnicode_Compare() */ -static PyObject * -unicode_compare(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - int result; - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - result = PyUnicode_Compare(left, right); - if (result == -1 && PyErr_Occurred()) { + if (PyUnicodeWriter_WriteSubstring(self->writer, str, start, end) < 0) { return NULL; } - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); + Py_RETURN_NONE; } -/* Test PyUnicode_CompareWithASCIIString() */ -static PyObject * -unicode_comparewithasciistring(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - int result; - - if (!PyArg_ParseTuple(args, "O|y#", &left, &right, &right_len)) - return NULL; - NULLABLE(left); - result = PyUnicode_CompareWithASCIIString(left, right); - if (result == -1 && PyErr_Occurred()) { +static PyObject* +writer_decodeutf8stateful(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - return PyLong_FromLong(result); -} - -/* Test PyUnicode_EqualToUTF8() */ -static PyObject * -unicode_equaltoutf8(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - int result; - if (!PyArg_ParseTuple(args, "Oz#", &left, &right, &right_len)) { + const char *str; + Py_ssize_t len; + const char *errors; + int use_consumed = 0; + if (!PyArg_ParseTuple(args, "yny|i", &str, &len, &errors, &use_consumed)) { return NULL; } - NULLABLE(left); - result = PyUnicode_EqualToUTF8(left, right); - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); -} - -/* Test PyUnicode_EqualToUTF8AndSize() */ -static PyObject * -unicode_equaltoutf8andsize(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - Py_ssize_t size = -100; - int result; - - if (!PyArg_ParseTuple(args, "Oz#|n", &left, &right, &right_len, &size)) { + Py_ssize_t consumed = 12345; + Py_ssize_t *pconsumed = use_consumed ? &consumed : NULL; + if (PyUnicodeWriter_DecodeUTF8Stateful(self->writer, str, len, + errors, pconsumed) < 0) { + if (use_consumed) { + assert(consumed == 0); + } return NULL; } - NULLABLE(left); - if (size == -100) { - size = right_len; + if (use_consumed) { + return PyLong_FromSsize_t(consumed); } - result = PyUnicode_EqualToUTF8AndSize(left, right, size); - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); -} - -/* Test PyUnicode_RichCompare() */ -static PyObject * -unicode_richcompare(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - int op; - - if (!PyArg_ParseTuple(args, "OOi", &left, &right, &op)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - return PyUnicode_RichCompare(left, right, op); + Py_RETURN_NONE; } -/* Test PyUnicode_Format() */ -static PyObject * -unicode_format(PyObject *self, PyObject *args) -{ - PyObject *format; - PyObject *fargs; - - if (!PyArg_ParseTuple(args, "OO", &format, &fargs)) - return NULL; - - NULLABLE(format); - NULLABLE(fargs); - return PyUnicode_Format(format, fargs); -} -/* Test PyUnicode_Contains() */ -static PyObject * -unicode_contains(PyObject *self, PyObject *args) +static PyObject* +writer_get_pointer(PyObject *self_raw, PyObject *args) { - PyObject *container; - PyObject *element; - - if (!PyArg_ParseTuple(args, "OO", &container, &element)) + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; + } - NULLABLE(container); - NULLABLE(element); - RETURN_INT(PyUnicode_Contains(container, element)); + return PyLong_FromVoidPtr(self->writer); } -/* Test PyUnicode_IsIdentifier() */ -static PyObject * -unicode_isidentifier(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - RETURN_INT(PyUnicode_IsIdentifier(arg)); -} -/* Test PyUnicode_CopyCharacters() */ -static PyObject * -unicode_copycharacters(PyObject *self, PyObject *args) +static PyObject* +writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) { - PyObject *from, *to, *to_copy; - Py_ssize_t from_start, to_start, how_many, copied; - - if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start, - &from, &from_start, &how_many)) { - return NULL; - } - - NULLABLE(from); - if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to), - PyUnicode_MAX_CHAR_VALUE(to)))) { - return NULL; - } - if (PyUnicode_Fill(to_copy, 0, PyUnicode_GET_LENGTH(to_copy), 0U) < 0) { - Py_DECREF(to_copy); - return NULL; - } - - copied = PyUnicode_CopyCharacters(to_copy, to_start, from, - from_start, how_many); - if (copied == -1 && PyErr_Occurred()) { - Py_DECREF(to_copy); + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - return Py_BuildValue("(Nn)", to_copy, copied); + PyObject *str = PyUnicodeWriter_Finish(self->writer); + self->writer = NULL; + return str; } -static int -check_raised_systemerror(PyObject *result, char* msg) -{ - if (result) { - // no exception - PyErr_Format(PyExc_AssertionError, - "SystemError not raised: %s", - msg); - return 0; - } - if (PyErr_ExceptionMatches(PyExc_SystemError)) { - // expected exception - PyErr_Clear(); - return 1; - } - // unexpected exception - return 0; -} -static PyObject * -test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *result; - PyObject *unicode = PyUnicode_FromString("None"); - -#define CHECK_FORMAT_2(FORMAT, EXPECTED, ARG1, ARG2) \ - result = PyUnicode_FromFormat(FORMAT, ARG1, ARG2); \ - if (EXPECTED == NULL) { \ - if (!check_raised_systemerror(result, FORMAT)) { \ - goto Fail; \ - } \ - } \ - else if (result == NULL) \ - return NULL; \ - else if (PyUnicode_CompareWithASCIIString(result, EXPECTED) != 0) { \ - PyErr_Format(PyExc_AssertionError, \ - "test_string_from_format: failed at \"%s\" " \ - "expected \"%s\" got \"%s\"", \ - FORMAT, EXPECTED, PyUnicode_AsUTF8(result)); \ - goto Fail; \ - } \ - Py_XDECREF(result) - -#define CHECK_FORMAT_1(FORMAT, EXPECTED, ARG) \ - CHECK_FORMAT_2(FORMAT, EXPECTED, ARG, 0) - -#define CHECK_FORMAT_0(FORMAT, EXPECTED) \ - CHECK_FORMAT_2(FORMAT, EXPECTED, 0, 0) - - // Unrecognized - CHECK_FORMAT_2("%u %? %u", NULL, 1, 2); - - // "%%" (options are rejected) - CHECK_FORMAT_0( "%%", "%"); - CHECK_FORMAT_0( "%0%", NULL); - CHECK_FORMAT_0("%00%", NULL); - CHECK_FORMAT_0( "%2%", NULL); - CHECK_FORMAT_0("%02%", NULL); - CHECK_FORMAT_0("%.0%", NULL); - CHECK_FORMAT_0("%.2%", NULL); - - // "%c" - CHECK_FORMAT_1( "%c", "c", 'c'); - CHECK_FORMAT_1( "%0c", "c", 'c'); - CHECK_FORMAT_1("%00c", "c", 'c'); - CHECK_FORMAT_1( "%2c", NULL, 'c'); - CHECK_FORMAT_1("%02c", NULL, 'c'); - CHECK_FORMAT_1("%.0c", NULL, 'c'); - CHECK_FORMAT_1("%.2c", NULL, 'c'); - - // Integers - CHECK_FORMAT_1("%d", "123", (int)123); - CHECK_FORMAT_1("%i", "123", (int)123); - CHECK_FORMAT_1("%u", "123", (unsigned int)123); - CHECK_FORMAT_1("%x", "7b", (unsigned int)123); - CHECK_FORMAT_1("%X", "7B", (unsigned int)123); - CHECK_FORMAT_1("%o", "173", (unsigned int)123); - CHECK_FORMAT_1("%ld", "123", (long)123); - CHECK_FORMAT_1("%li", "123", (long)123); - CHECK_FORMAT_1("%lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); - CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); - CHECK_FORMAT_1("%lo", "173", (unsigned long)123); - CHECK_FORMAT_1("%lld", "123", (long long)123); - CHECK_FORMAT_1("%lli", "123", (long long)123); - CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); - CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); - CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); - CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%zu", "123", (size_t)123); - CHECK_FORMAT_1("%zx", "7b", (size_t)123); - CHECK_FORMAT_1("%zX", "7B", (size_t)123); - CHECK_FORMAT_1("%zo", "173", (size_t)123); - CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); - CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); - CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); - CHECK_FORMAT_1("%jd", "123", (intmax_t)123); - CHECK_FORMAT_1("%ji", "123", (intmax_t)123); - CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); - CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); - CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); - CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); - - CHECK_FORMAT_1("%d", "-123", (int)-123); - CHECK_FORMAT_1("%i", "-123", (int)-123); - CHECK_FORMAT_1("%ld", "-123", (long)-123); - CHECK_FORMAT_1("%li", "-123", (long)-123); - CHECK_FORMAT_1("%lld", "-123", (long long)-123); - CHECK_FORMAT_1("%lli", "-123", (long long)-123); - CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); - CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); - CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); - CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); - - // Integers: width < length - CHECK_FORMAT_1("%1d", "123", (int)123); - CHECK_FORMAT_1("%1i", "123", (int)123); - CHECK_FORMAT_1("%1u", "123", (unsigned int)123); - CHECK_FORMAT_1("%1ld", "123", (long)123); - CHECK_FORMAT_1("%1li", "123", (long)123); - CHECK_FORMAT_1("%1lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%1lld", "123", (long long)123); - CHECK_FORMAT_1("%1lli", "123", (long long)123); - CHECK_FORMAT_1("%1llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%1zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%1zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%1zu", "123", (size_t)123); - CHECK_FORMAT_1("%1x", "7b", (int)123); - - CHECK_FORMAT_1("%1d", "-123", (int)-123); - CHECK_FORMAT_1("%1i", "-123", (int)-123); - CHECK_FORMAT_1("%1ld", "-123", (long)-123); - CHECK_FORMAT_1("%1li", "-123", (long)-123); - CHECK_FORMAT_1("%1lld", "-123", (long long)-123); - CHECK_FORMAT_1("%1lli", "-123", (long long)-123); - CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); - - // Integers: width > length - CHECK_FORMAT_1("%5d", " 123", (int)123); - CHECK_FORMAT_1("%5i", " 123", (int)123); - CHECK_FORMAT_1("%5u", " 123", (unsigned int)123); - CHECK_FORMAT_1("%5ld", " 123", (long)123); - CHECK_FORMAT_1("%5li", " 123", (long)123); - CHECK_FORMAT_1("%5lu", " 123", (unsigned long)123); - CHECK_FORMAT_1("%5lld", " 123", (long long)123); - CHECK_FORMAT_1("%5lli", " 123", (long long)123); - CHECK_FORMAT_1("%5llu", " 123", (unsigned long long)123); - CHECK_FORMAT_1("%5zd", " 123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5zi", " 123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5zu", " 123", (size_t)123); - CHECK_FORMAT_1("%5x", " 7b", (int)123); - - CHECK_FORMAT_1("%5d", " -123", (int)-123); - CHECK_FORMAT_1("%5i", " -123", (int)-123); - CHECK_FORMAT_1("%5ld", " -123", (long)-123); - CHECK_FORMAT_1("%5li", " -123", (long)-123); - CHECK_FORMAT_1("%5lld", " -123", (long long)-123); - CHECK_FORMAT_1("%5lli", " -123", (long long)-123); - CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); - - // Integers: width > length, 0-flag - CHECK_FORMAT_1("%05d", "00123", (int)123); - CHECK_FORMAT_1("%05i", "00123", (int)123); - CHECK_FORMAT_1("%05u", "00123", (unsigned int)123); - CHECK_FORMAT_1("%05ld", "00123", (long)123); - CHECK_FORMAT_1("%05li", "00123", (long)123); - CHECK_FORMAT_1("%05lu", "00123", (unsigned long)123); - CHECK_FORMAT_1("%05lld", "00123", (long long)123); - CHECK_FORMAT_1("%05lli", "00123", (long long)123); - CHECK_FORMAT_1("%05llu", "00123", (unsigned long long)123); - CHECK_FORMAT_1("%05zd", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05zi", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05zu", "00123", (size_t)123); - CHECK_FORMAT_1("%05x", "0007b", (int)123); - - CHECK_FORMAT_1("%05d", "-0123", (int)-123); - CHECK_FORMAT_1("%05i", "-0123", (int)-123); - CHECK_FORMAT_1("%05ld", "-0123", (long)-123); - CHECK_FORMAT_1("%05li", "-0123", (long)-123); - CHECK_FORMAT_1("%05lld", "-0123", (long long)-123); - CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); - CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); - - // Integers: precision < length - CHECK_FORMAT_1("%.1d", "123", (int)123); - CHECK_FORMAT_1("%.1i", "123", (int)123); - CHECK_FORMAT_1("%.1u", "123", (unsigned int)123); - CHECK_FORMAT_1("%.1ld", "123", (long)123); - CHECK_FORMAT_1("%.1li", "123", (long)123); - CHECK_FORMAT_1("%.1lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%.1lld", "123", (long long)123); - CHECK_FORMAT_1("%.1lli", "123", (long long)123); - CHECK_FORMAT_1("%.1llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%.1zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.1zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.1zu", "123", (size_t)123); - CHECK_FORMAT_1("%.1x", "7b", (int)123); - - CHECK_FORMAT_1("%.1d", "-123", (int)-123); - CHECK_FORMAT_1("%.1i", "-123", (int)-123); - CHECK_FORMAT_1("%.1ld", "-123", (long)-123); - CHECK_FORMAT_1("%.1li", "-123", (long)-123); - CHECK_FORMAT_1("%.1lld", "-123", (long long)-123); - CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); - CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); - - // Integers: precision > length - CHECK_FORMAT_1("%.5d", "00123", (int)123); - CHECK_FORMAT_1("%.5i", "00123", (int)123); - CHECK_FORMAT_1("%.5u", "00123", (unsigned int)123); - CHECK_FORMAT_1("%.5ld", "00123", (long)123); - CHECK_FORMAT_1("%.5li", "00123", (long)123); - CHECK_FORMAT_1("%.5lu", "00123", (unsigned long)123); - CHECK_FORMAT_1("%.5lld", "00123", (long long)123); - CHECK_FORMAT_1("%.5lli", "00123", (long long)123); - CHECK_FORMAT_1("%.5llu", "00123", (unsigned long long)123); - CHECK_FORMAT_1("%.5zd", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.5zi", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.5zu", "00123", (size_t)123); - CHECK_FORMAT_1("%.5x", "0007b", (int)123); - - CHECK_FORMAT_1("%.5d", "-00123", (int)-123); - CHECK_FORMAT_1("%.5i", "-00123", (int)-123); - CHECK_FORMAT_1("%.5ld", "-00123", (long)-123); - CHECK_FORMAT_1("%.5li", "-00123", (long)-123); - CHECK_FORMAT_1("%.5lld", "-00123", (long long)-123); - CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); - CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); - - // Integers: width > precision > length - CHECK_FORMAT_1("%7.5d", " 00123", (int)123); - CHECK_FORMAT_1("%7.5i", " 00123", (int)123); - CHECK_FORMAT_1("%7.5u", " 00123", (unsigned int)123); - CHECK_FORMAT_1("%7.5ld", " 00123", (long)123); - CHECK_FORMAT_1("%7.5li", " 00123", (long)123); - CHECK_FORMAT_1("%7.5lu", " 00123", (unsigned long)123); - CHECK_FORMAT_1("%7.5lld", " 00123", (long long)123); - CHECK_FORMAT_1("%7.5lli", " 00123", (long long)123); - CHECK_FORMAT_1("%7.5llu", " 00123", (unsigned long long)123); - CHECK_FORMAT_1("%7.5zd", " 00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%7.5zi", " 00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%7.5zu", " 00123", (size_t)123); - CHECK_FORMAT_1("%7.5x", " 0007b", (int)123); - - CHECK_FORMAT_1("%7.5d", " -00123", (int)-123); - CHECK_FORMAT_1("%7.5i", " -00123", (int)-123); - CHECK_FORMAT_1("%7.5ld", " -00123", (long)-123); - CHECK_FORMAT_1("%7.5li", " -00123", (long)-123); - CHECK_FORMAT_1("%7.5lld", " -00123", (long long)-123); - CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); - CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); - - // Integers: width > precision > length, 0-flag - CHECK_FORMAT_1("%07.5d", "0000123", (int)123); - CHECK_FORMAT_1("%07.5i", "0000123", (int)123); - CHECK_FORMAT_1("%07.5u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%07.5ld", "0000123", (long)123); - CHECK_FORMAT_1("%07.5li", "0000123", (long)123); - CHECK_FORMAT_1("%07.5lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%07.5lld", "0000123", (long long)123); - CHECK_FORMAT_1("%07.5lli", "0000123", (long long)123); - CHECK_FORMAT_1("%07.5llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%07.5zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%07.5zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%07.5zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%07.5x", "000007b", (int)123); - - CHECK_FORMAT_1("%07.5d", "-000123", (int)-123); - CHECK_FORMAT_1("%07.5i", "-000123", (int)-123); - CHECK_FORMAT_1("%07.5ld", "-000123", (long)-123); - CHECK_FORMAT_1("%07.5li", "-000123", (long)-123); - CHECK_FORMAT_1("%07.5lld", "-000123", (long long)-123); - CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); - CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); - - // Integers: precision > width > length - CHECK_FORMAT_1("%5.7d", "0000123", (int)123); - CHECK_FORMAT_1("%5.7i", "0000123", (int)123); - CHECK_FORMAT_1("%5.7u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%5.7ld", "0000123", (long)123); - CHECK_FORMAT_1("%5.7li", "0000123", (long)123); - CHECK_FORMAT_1("%5.7lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%5.7lld", "0000123", (long long)123); - CHECK_FORMAT_1("%5.7lli", "0000123", (long long)123); - CHECK_FORMAT_1("%5.7llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%5.7zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5.7zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5.7zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%5.7x", "000007b", (int)123); - - CHECK_FORMAT_1("%5.7d", "-0000123", (int)-123); - CHECK_FORMAT_1("%5.7i", "-0000123", (int)-123); - CHECK_FORMAT_1("%5.7ld", "-0000123", (long)-123); - CHECK_FORMAT_1("%5.7li", "-0000123", (long)-123); - CHECK_FORMAT_1("%5.7lld", "-0000123", (long long)-123); - CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); - CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); - - // Integers: precision > width > length, 0-flag - CHECK_FORMAT_1("%05.7d", "0000123", (int)123); - CHECK_FORMAT_1("%05.7i", "0000123", (int)123); - CHECK_FORMAT_1("%05.7u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%05.7ld", "0000123", (long)123); - CHECK_FORMAT_1("%05.7li", "0000123", (long)123); - CHECK_FORMAT_1("%05.7lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%05.7lld", "0000123", (long long)123); - CHECK_FORMAT_1("%05.7lli", "0000123", (long long)123); - CHECK_FORMAT_1("%05.7llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%05.7zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05.7zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05.7zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%05.7x", "000007b", (int)123); - - CHECK_FORMAT_1("%05.7d", "-0000123", (int)-123); - CHECK_FORMAT_1("%05.7i", "-0000123", (int)-123); - CHECK_FORMAT_1("%05.7ld", "-0000123", (long)-123); - CHECK_FORMAT_1("%05.7li", "-0000123", (long)-123); - CHECK_FORMAT_1("%05.7lld", "-0000123", (long long)-123); - CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); - CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); - - // Integers: precision = 0, arg = 0 (empty string in C) - CHECK_FORMAT_1("%.0d", "0", (int)0); - CHECK_FORMAT_1("%.0i", "0", (int)0); - CHECK_FORMAT_1("%.0u", "0", (unsigned int)0); - CHECK_FORMAT_1("%.0ld", "0", (long)0); - CHECK_FORMAT_1("%.0li", "0", (long)0); - CHECK_FORMAT_1("%.0lu", "0", (unsigned long)0); - CHECK_FORMAT_1("%.0lld", "0", (long long)0); - CHECK_FORMAT_1("%.0lli", "0", (long long)0); - CHECK_FORMAT_1("%.0llu", "0", (unsigned long long)0); - CHECK_FORMAT_1("%.0zd", "0", (Py_ssize_t)0); - CHECK_FORMAT_1("%.0zi", "0", (Py_ssize_t)0); - CHECK_FORMAT_1("%.0zu", "0", (size_t)0); - CHECK_FORMAT_1("%.0x", "0", (int)0); - - // Strings - CHECK_FORMAT_1("%s", "None", "None"); - CHECK_FORMAT_1("%ls", "None", L"None"); - CHECK_FORMAT_1("%U", "None", unicode); - CHECK_FORMAT_1("%A", "None", Py_None); - CHECK_FORMAT_1("%S", "None", Py_None); - CHECK_FORMAT_1("%R", "None", Py_None); - CHECK_FORMAT_2("%V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%V", "None", NULL, "None"); - CHECK_FORMAT_2("%lV", "None", NULL, L"None"); - - // Strings: width < length - CHECK_FORMAT_1("%1s", "None", "None"); - CHECK_FORMAT_1("%1ls", "None", L"None"); - CHECK_FORMAT_1("%1U", "None", unicode); - CHECK_FORMAT_1("%1A", "None", Py_None); - CHECK_FORMAT_1("%1S", "None", Py_None); - CHECK_FORMAT_1("%1R", "None", Py_None); - CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%1V", "None", NULL, "None"); - CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); - - // Strings: width > length - CHECK_FORMAT_1("%5s", " None", "None"); - CHECK_FORMAT_1("%5ls", " None", L"None"); - CHECK_FORMAT_1("%5U", " None", unicode); - CHECK_FORMAT_1("%5A", " None", Py_None); - CHECK_FORMAT_1("%5S", " None", Py_None); - CHECK_FORMAT_1("%5R", " None", Py_None); - CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); - CHECK_FORMAT_2("%5V", " None", NULL, "None"); - CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); - - // Strings: precision < length - CHECK_FORMAT_1("%.1s", "N", "None"); - CHECK_FORMAT_1("%.1ls", "N", L"None"); - CHECK_FORMAT_1("%.1U", "N", unicode); - CHECK_FORMAT_1("%.1A", "N", Py_None); - CHECK_FORMAT_1("%.1S", "N", Py_None); - CHECK_FORMAT_1("%.1R", "N", Py_None); - CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); - CHECK_FORMAT_2("%.1V", "N", NULL, "None"); - CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); - - // Strings: precision > length - CHECK_FORMAT_1("%.5s", "None", "None"); - CHECK_FORMAT_1("%.5ls", "None", L"None"); - CHECK_FORMAT_1("%.5U", "None", unicode); - CHECK_FORMAT_1("%.5A", "None", Py_None); - CHECK_FORMAT_1("%.5S", "None", Py_None); - CHECK_FORMAT_1("%.5R", "None", Py_None); - CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%.5V", "None", NULL, "None"); - CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); - - // Strings: precision < length, width > length - CHECK_FORMAT_1("%5.1s", " N", "None"); - CHECK_FORMAT_1("%5.1ls"," N", L"None"); - CHECK_FORMAT_1("%5.1U", " N", unicode); - CHECK_FORMAT_1("%5.1A", " N", Py_None); - CHECK_FORMAT_1("%5.1S", " N", Py_None); - CHECK_FORMAT_1("%5.1R", " N", Py_None); - CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); - CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); - CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); - - // Strings: width < length, precision > length - CHECK_FORMAT_1("%1.5s", "None", "None"); - CHECK_FORMAT_1("%1.5ls", "None", L"None"); - CHECK_FORMAT_1("%1.5U", "None", unicode); - CHECK_FORMAT_1("%1.5A", "None", Py_None); - CHECK_FORMAT_1("%1.5S", "None", Py_None); - CHECK_FORMAT_1("%1.5R", "None", Py_None); - CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); - CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); - - Py_XDECREF(unicode); - Py_RETURN_NONE; +static PyMethodDef writer_methods[] = { + {"write_char", _PyCFunction_CAST(writer_write_char), METH_VARARGS}, + {"write_utf8", _PyCFunction_CAST(writer_write_utf8), METH_VARARGS}, + {"write_widechar", _PyCFunction_CAST(writer_write_widechar), METH_VARARGS}, + {"write_ucs4", _PyCFunction_CAST(writer_write_ucs4), METH_VARARGS}, + {"write_str", _PyCFunction_CAST(writer_write_str), METH_VARARGS}, + {"write_repr", _PyCFunction_CAST(writer_write_repr), METH_VARARGS}, + {"write_substring", _PyCFunction_CAST(writer_write_substring), METH_VARARGS}, + {"decodeutf8stateful", _PyCFunction_CAST(writer_decodeutf8stateful), METH_VARARGS}, + {"get_pointer", _PyCFunction_CAST(writer_get_pointer), METH_VARARGS}, + {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; - Fail: - Py_XDECREF(result); - Py_XDECREF(unicode); - return NULL; +static PyType_Slot Writer_Type_slots[] = { + {Py_tp_new, writer_new}, + {Py_tp_init, writer_init}, + {Py_tp_dealloc, writer_dealloc}, + {Py_tp_methods, writer_methods}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Writer_spec = { + .name = "_testcapi.PyUnicodeWriter", + .basicsize = sizeof(WriterObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = Writer_Type_slots, +}; -#undef CHECK_FORMAT_2 -#undef CHECK_FORMAT_1 -#undef CHECK_FORMAT_0 -} static PyMethodDef TestMethods[] = { - {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS}, - {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS}, - {"test_unicode_compare_with_ascii", - test_unicode_compare_with_ascii, METH_NOARGS}, - {"test_string_from_format", test_string_from_format, METH_NOARGS}, - {"test_widechar", test_widechar, METH_NOARGS}, {"unicode_new", unicode_new, METH_VARARGS}, {"unicode_fill", unicode_fill, METH_VARARGS}, - {"unicode_writechar", unicode_writechar, METH_VARARGS}, - {"unicode_resize", unicode_resize, METH_VARARGS}, - {"unicode_append", unicode_append, METH_VARARGS}, - {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, - {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, - {"unicode_fromstring", unicode_fromstring, METH_O}, {"unicode_fromkindanddata", unicode_fromkindanddata, METH_VARARGS}, - {"unicode_substring", unicode_substring, METH_VARARGS}, - {"unicode_getlength", unicode_getlength, METH_O}, - {"unicode_readchar", unicode_readchar, METH_VARARGS}, - {"unicode_fromencodedobject",unicode_fromencodedobject, METH_VARARGS}, - {"unicode_fromobject", unicode_fromobject, METH_O}, - {"unicode_interninplace", unicode_interninplace, METH_O}, - {"unicode_internfromstring", unicode_internfromstring, METH_O}, - {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, - {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, - {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, - {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, - {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS}, - {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, - {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, - {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, - {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, - {"unicode_decode", unicode_decode, METH_VARARGS}, - {"unicode_asencodedstring", unicode_asencodedstring, METH_VARARGS}, - {"unicode_buildencodingmap", unicode_buildencodingmap, METH_O}, - {"unicode_decodeutf7", unicode_decodeutf7, METH_VARARGS}, - {"unicode_decodeutf7stateful",unicode_decodeutf7stateful, METH_VARARGS}, - {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, - {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, - {"unicode_asutf8string", unicode_asutf8string, METH_O}, - {"unicode_decodeutf16", unicode_decodeutf16, METH_VARARGS}, - {"unicode_decodeutf16stateful",unicode_decodeutf16stateful, METH_VARARGS}, - {"unicode_asutf16string", unicode_asutf16string, METH_O}, - {"unicode_decodeutf32", unicode_decodeutf32, METH_VARARGS}, - {"unicode_decodeutf32stateful",unicode_decodeutf32stateful, METH_VARARGS}, - {"unicode_asutf32string", unicode_asutf32string, METH_O}, - {"unicode_decodeunicodeescape",unicode_decodeunicodeescape, METH_VARARGS}, - {"unicode_asunicodeescapestring",unicode_asunicodeescapestring,METH_O}, - {"unicode_decoderawunicodeescape",unicode_decoderawunicodeescape,METH_VARARGS}, - {"unicode_asrawunicodeescapestring",unicode_asrawunicodeescapestring,METH_O}, - {"unicode_decodelatin1", unicode_decodelatin1, METH_VARARGS}, - {"unicode_aslatin1string", unicode_aslatin1string, METH_O}, - {"unicode_decodeascii", unicode_decodeascii, METH_VARARGS}, - {"unicode_asasciistring", unicode_asasciistring, METH_O}, - {"unicode_decodecharmap", unicode_decodecharmap, METH_VARARGS}, - {"unicode_ascharmapstring", unicode_ascharmapstring, METH_VARARGS}, -#ifdef MS_WINDOWS - {"unicode_decodembcs", unicode_decodembcs, METH_VARARGS}, - {"unicode_decodembcsstateful",unicode_decodembcsstateful, METH_VARARGS}, - {"unicode_decodecodepagestateful",unicode_decodecodepagestateful,METH_VARARGS}, - {"unicode_asmbcsstring", unicode_asmbcsstring, METH_O}, - {"unicode_encodecodepage", unicode_encodecodepage, METH_VARARGS}, -#endif /* MS_WINDOWS */ - {"unicode_decodelocaleandsize",unicode_decodelocaleandsize, METH_VARARGS}, - {"unicode_decodelocale", unicode_decodelocale, METH_VARARGS}, - {"unicode_encodelocale", unicode_encodelocale, METH_VARARGS}, - {"unicode_decodefsdefault", unicode_decodefsdefault, METH_VARARGS}, - {"unicode_decodefsdefaultandsize",unicode_decodefsdefaultandsize,METH_VARARGS}, - {"unicode_encodefsdefault", unicode_encodefsdefault, METH_O}, - {"unicode_concat", unicode_concat, METH_VARARGS}, - {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, - {"unicode_split", unicode_split, METH_VARARGS}, - {"unicode_rsplit", unicode_rsplit, METH_VARARGS}, - {"unicode_partition", unicode_partition, METH_VARARGS}, - {"unicode_rpartition", unicode_rpartition, METH_VARARGS}, - {"unicode_translate", unicode_translate, METH_VARARGS}, - {"unicode_join", unicode_join, METH_VARARGS}, - {"unicode_count", unicode_count, METH_VARARGS}, - {"unicode_tailmatch", unicode_tailmatch, METH_VARARGS}, - {"unicode_find", unicode_find, METH_VARARGS}, - {"unicode_findchar", unicode_findchar, METH_VARARGS}, - {"unicode_replace", unicode_replace, METH_VARARGS}, - {"unicode_compare", unicode_compare, METH_VARARGS}, - {"unicode_comparewithasciistring",unicode_comparewithasciistring,METH_VARARGS}, - {"unicode_equaltoutf8", unicode_equaltoutf8, METH_VARARGS}, - {"unicode_equaltoutf8andsize",unicode_equaltoutf8andsize, METH_VARARGS}, - {"unicode_richcompare", unicode_richcompare, METH_VARARGS}, - {"unicode_format", unicode_format, METH_VARARGS}, - {"unicode_contains", unicode_contains, METH_VARARGS}, - {"unicode_isidentifier", unicode_isidentifier, METH_O}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {NULL}, }; int _PyTestCapi_Init_Unicode(PyObject *m) { - _testcapimodule = PyModule_GetDef(m); - if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } + PyTypeObject *writer_type = (PyTypeObject *)PyType_FromSpec(&Writer_spec); + if (writer_type == NULL) { + return -1; + } + if (PyModule_AddType(m, writer_type) < 0) { + Py_DECREF(writer_type); + return -1; + } + Py_DECREF(writer_type); + return 0; } diff --git a/Modules/_testcapi/util.h b/Modules/_testcapi/util.h index f26d7656a10138..042e522542eddb 100644 --- a/Modules/_testcapi/util.h +++ b/Modules/_testcapi/util.h @@ -31,3 +31,13 @@ static const char uninitialized[] = "uninitialized"; #define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) /* Marker to check that integer value was set. */ #define UNINITIALIZED_INT (63256717) +/* + * Marker to indicate that a NULL parameter would not be allowed + * at runtime but that the test interface will check that it is + * indeed the case. + * + * Use this macro only if passing NULL to the C API would raise + * a catchable exception (and not a fatal exception that would + * crash the interpreter). + */ + #define NULL_WOULD_RAISE(NAME) NAME diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index b30c5e8704c8af..03aaacb328e0b6 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -348,6 +348,9 @@ static PyObject * MethodDescriptor2_new(PyTypeObject* type, PyObject* args, PyObject *kw) { MethodDescriptor2Object *op = PyObject_New(MethodDescriptor2Object, type); + if (op == NULL) { + return NULL; + } op->base.vectorcall = NULL; op->vectorcall = MethodDescriptor_vectorcall; return (PyObject *)op; diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 1eb0db2c2e6576..321d3aeffb6ad1 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -8,6 +8,7 @@ #define Py_BUILD_CORE #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_code.h" // CODE_MAX_WATCHERS +#include "pycore_context.h" // CONTEXT_MAX_WATCHERS /*[clinic input] module _testcapi @@ -622,6 +623,152 @@ allocate_too_many_func_watchers(PyObject *self, PyObject *args) Py_RETURN_NONE; } +// Test contexct object watchers +#define NUM_CONTEXT_WATCHERS 2 +static int context_watcher_ids[NUM_CONTEXT_WATCHERS] = {-1, -1}; +static PyObject *context_switches[NUM_CONTEXT_WATCHERS]; + +static int +handle_context_watcher_event(int which_watcher, PyContextEvent event, PyObject *ctx) { + if (event == Py_CONTEXT_SWITCHED) { + PyList_Append(context_switches[which_watcher], ctx); + } + else { + return -1; + } + return 0; +} + +static int +first_context_watcher_callback(PyContextEvent event, PyObject *ctx) { + return handle_context_watcher_event(0, event, ctx); +} + +static int +second_context_watcher_callback(PyContextEvent event, PyObject *ctx) { + return handle_context_watcher_event(1, event, ctx); +} + +static int +noop_context_event_handler(PyContextEvent event, PyObject *ctx) { + return 0; +} + +static int +error_context_event_handler(PyContextEvent event, PyObject *ctx) { + PyErr_SetString(PyExc_RuntimeError, "boom!"); + return -1; +} + +static PyObject * +add_context_watcher(PyObject *self, PyObject *which_watcher) +{ + static const PyContext_WatchCallback callbacks[] = { + &first_context_watcher_callback, + &second_context_watcher_callback, + &error_context_event_handler, + }; + assert(PyLong_Check(which_watcher)); + long which_l = PyLong_AsLong(which_watcher); + if (which_l < 0 || which_l >= (long)Py_ARRAY_LENGTH(callbacks)) { + PyErr_Format(PyExc_ValueError, "invalid watcher %d", which_l); + return NULL; + } + int watcher_id = PyContext_AddWatcher(callbacks[which_l]); + if (watcher_id < 0) { + return NULL; + } + if (which_l >= 0 && which_l < NUM_CONTEXT_WATCHERS) { + context_watcher_ids[which_l] = watcher_id; + Py_XSETREF(context_switches[which_l], PyList_New(0)); + if (context_switches[which_l] == NULL) { + return NULL; + } + } + return PyLong_FromLong(watcher_id); +} + +static PyObject * +clear_context_watcher(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + if (PyContext_ClearWatcher(watcher_id_l) < 0) { + return NULL; + } + // reset static events counters + if (watcher_id_l >= 0) { + for (int i = 0; i < NUM_CONTEXT_WATCHERS; i++) { + if (watcher_id_l == context_watcher_ids[i]) { + context_watcher_ids[i] = -1; + Py_CLEAR(context_switches[i]); + } + } + } + Py_RETURN_NONE; +} + +static PyObject * +clear_context_stack(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = PyThreadState_Get(); + if (tstate->context == NULL) { + Py_RETURN_NONE; + } + if (((PyContext *)tstate->context)->ctx_prev != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "must first exit all non-base contexts"); + return NULL; + } + Py_CLEAR(tstate->context); + Py_RETURN_NONE; +} + +static PyObject * +get_context_switches(PyObject *Py_UNUSED(self), PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + if (watcher_id_l < 0 || watcher_id_l >= NUM_CONTEXT_WATCHERS) { + PyErr_Format(PyExc_ValueError, "invalid watcher %ld", watcher_id_l); + return NULL; + } + if (context_switches[watcher_id_l] == NULL) { + return PyList_New(0); + } + return Py_NewRef(context_switches[watcher_id_l]); +} + +static PyObject * +allocate_too_many_context_watchers(PyObject *self, PyObject *args) +{ + int watcher_ids[CONTEXT_MAX_WATCHERS + 1]; + int num_watchers = 0; + for (unsigned long i = 0; i < sizeof(watcher_ids) / sizeof(int); i++) { + int watcher_id = PyContext_AddWatcher(noop_context_event_handler); + if (watcher_id == -1) { + break; + } + watcher_ids[i] = watcher_id; + num_watchers++; + } + PyObject *exc = PyErr_GetRaisedException(); + for (int i = 0; i < num_watchers; i++) { + if (PyContext_ClearWatcher(watcher_ids[i]) < 0) { + PyErr_WriteUnraisable(Py_None); + break; + } + } + if (exc) { + PyErr_SetRaisedException(exc); + return NULL; + } + else if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] _testcapi.set_func_defaults_via_capi func: object @@ -689,6 +836,14 @@ static PyMethodDef test_methods[] = { _TESTCAPI_SET_FUNC_KWDEFAULTS_VIA_CAPI_METHODDEF {"allocate_too_many_func_watchers", allocate_too_many_func_watchers, METH_NOARGS, NULL}, + + // Code object watchers. + {"add_context_watcher", add_context_watcher, METH_O, NULL}, + {"clear_context_watcher", clear_context_watcher, METH_O, NULL}, + {"clear_context_stack", clear_context_stack, METH_NOARGS, NULL}, + {"get_context_switches", get_context_switches, METH_O, NULL}, + {"allocate_too_many_context_watchers", + (PyCFunction) allocate_too_many_context_watchers, METH_NOARGS, NULL}, {NULL}, }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index e67de3eeb6e17e..26f68691e44f83 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -13,7 +13,6 @@ #include "_testcapi/parts.h" #include "frameobject.h" // PyFrame_New() -#include "interpreteridobject.h" // PyInterpreterID_Type #include "marshal.h" // PyMarshal_WriteLongToFile() #include // FLT_MAX @@ -82,8 +81,11 @@ static PyObject* test_config(PyObject *self, PyObject *Py_UNUSED(ignored)) { #define CHECK_SIZEOF(FATNAME, TYPE) \ - if (FATNAME != sizeof(TYPE)) \ - return sizeof_error(self, #FATNAME, #TYPE, FATNAME, sizeof(TYPE)) + do { \ + if (FATNAME != sizeof(TYPE)) { \ + return sizeof_error(self, #FATNAME, #TYPE, FATNAME, sizeof(TYPE)); \ + } \ + } while (0) CHECK_SIZEOF(SIZEOF_SHORT, short); CHECK_SIZEOF(SIZEOF_INT, int); @@ -104,21 +106,25 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wtype-limits" #endif -#define CHECK_SIZEOF(TYPE, EXPECTED) \ - if (EXPECTED != sizeof(TYPE)) { \ - PyErr_Format(get_testerror(self), \ - "sizeof(%s) = %u instead of %u", \ - #TYPE, sizeof(TYPE), EXPECTED); \ - return (PyObject*)NULL; \ - } +#define CHECK_SIZEOF(TYPE, EXPECTED) \ + do { \ + if (EXPECTED != sizeof(TYPE)) { \ + PyErr_Format(get_testerror(self), \ + "sizeof(%s) = %u instead of %u", \ + #TYPE, sizeof(TYPE), EXPECTED); \ + return (PyObject*)NULL; \ + } \ + } while (0) #define IS_SIGNED(TYPE) (((TYPE)-1) < (TYPE)0) -#define CHECK_SIGNNESS(TYPE, SIGNED) \ - if (IS_SIGNED(TYPE) != SIGNED) { \ - PyErr_Format(get_testerror(self), \ - "%s signness is %i, instead of %i", \ - #TYPE, IS_SIGNED(TYPE), SIGNED); \ - return (PyObject*)NULL; \ - } +#define CHECK_SIGNNESS(TYPE, SIGNED) \ + do { \ + if (IS_SIGNED(TYPE) != SIGNED) { \ + PyErr_Format(get_testerror(self), \ + "%s signness is %i, instead of %i", \ + #TYPE, IS_SIGNED(TYPE), SIGNED); \ + return (PyObject*)NULL; \ + } \ + } while (0) /* integer types */ CHECK_SIZEOF(Py_UCS1, 1); @@ -283,7 +289,7 @@ static PyTypeObject _HashInheritanceTester_Type = { "hashinheritancetester", /* Name of this type */ sizeof(PyObject), /* Basic object size */ 0, /* Item size for varobject */ - (destructor)PyObject_Del, /* tp_dealloc */ + (destructor)PyObject_Free, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -597,83 +603,39 @@ get_heaptype_for_name(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyType_FromSpec(&HeapTypeNameType_Spec); } + static PyObject * -test_get_type_name(PyObject *self, PyObject *Py_UNUSED(ignored)) +get_type_name(PyObject *self, PyObject *type) { - PyObject *tp_name = PyType_GetName(&PyLong_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "int") == 0); - Py_DECREF(tp_name); - - tp_name = PyType_GetName(&PyModule_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "module") == 0); - Py_DECREF(tp_name); - - PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); - if (HeapTypeNameType == NULL) { - Py_RETURN_NONE; - } - tp_name = PyType_GetName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "HeapTypeNameType") == 0); - Py_DECREF(tp_name); - - PyObject *name = PyUnicode_FromString("test_name"); - if (name == NULL) { - goto done; - } - if (PyObject_SetAttrString(HeapTypeNameType, "__name__", name) < 0) { - Py_DECREF(name); - goto done; - } - tp_name = PyType_GetName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "test_name") == 0); - Py_DECREF(name); - Py_DECREF(tp_name); - - done: - Py_DECREF(HeapTypeNameType); - Py_RETURN_NONE; + assert(PyType_Check(type)); + return PyType_GetName((PyTypeObject *)type); } static PyObject * -test_get_type_qualname(PyObject *self, PyObject *Py_UNUSED(ignored)) +get_type_qualname(PyObject *self, PyObject *type) { - PyObject *tp_qualname = PyType_GetQualName(&PyLong_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "int") == 0); - Py_DECREF(tp_qualname); + assert(PyType_Check(type)); + return PyType_GetQualName((PyTypeObject *)type); +} - tp_qualname = PyType_GetQualName(&PyODict_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "OrderedDict") == 0); - Py_DECREF(tp_qualname); - PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); - if (HeapTypeNameType == NULL) { - Py_RETURN_NONE; - } - tp_qualname = PyType_GetQualName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "HeapTypeNameType") == 0); - Py_DECREF(tp_qualname); +static PyObject * +get_type_fullyqualname(PyObject *self, PyObject *type) +{ + assert(PyType_Check(type)); + return PyType_GetFullyQualifiedName((PyTypeObject *)type); +} - PyObject *spec_name = PyUnicode_FromString(HeapTypeNameType_Spec.name); - if (spec_name == NULL) { - goto done; - } - if (PyObject_SetAttrString(HeapTypeNameType, - "__qualname__", spec_name) < 0) { - Py_DECREF(spec_name); - goto done; - } - tp_qualname = PyType_GetQualName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), - "_testcapi.HeapTypeNameType") == 0); - Py_DECREF(spec_name); - Py_DECREF(tp_qualname); - done: - Py_DECREF(HeapTypeNameType); - Py_RETURN_NONE; +static PyObject * +get_type_module_name(PyObject *self, PyObject *type) +{ + assert(PyType_Check(type)); + return PyType_GetModuleName((PyTypeObject *)type); } + static PyObject * test_get_type_dict(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -809,6 +771,14 @@ test_thread_state(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +gilstate_ensure_release(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyGILState_Release(state); + Py_RETURN_NONE; +} + #ifndef MS_WINDOWS static PyThread_type_lock wait_done = NULL; @@ -864,25 +834,55 @@ static int _pending_callback(void *arg) * run from any python thread. */ static PyObject * -pending_threadfunc(PyObject *self, PyObject *arg) +pending_threadfunc(PyObject *self, PyObject *arg, PyObject *kwargs) { + static char *kwlist[] = {"callback", "num", + "blocking", "ensure_added", NULL}; PyObject *callable; - int r; - if (PyArg_ParseTuple(arg, "O", &callable) == 0) + unsigned int num = 1; + int blocking = 0; + int ensure_added = 0; + if (!PyArg_ParseTupleAndKeywords(arg, kwargs, + "O|I$pp:_pending_threadfunc", kwlist, + &callable, &num, &blocking, &ensure_added)) + { return NULL; + } /* create the reference for the callbackwhile we hold the lock */ - Py_INCREF(callable); + for (unsigned int i = 0; i < num; i++) { + Py_INCREF(callable); + } - Py_BEGIN_ALLOW_THREADS - r = Py_AddPendingCall(&_pending_callback, callable); - Py_END_ALLOW_THREADS + PyThreadState *save_tstate = NULL; + if (!blocking) { + save_tstate = PyEval_SaveThread(); + } + + unsigned int num_added = 0; + for (; num_added < num; num_added++) { + if (ensure_added) { + int r; + do { + r = Py_AddPendingCall(&_pending_callback, callable); + } while (r < 0); + } + else { + if (Py_AddPendingCall(&_pending_callback, callable) < 0) { + break; + } + } + } - if (r<0) { + if (!blocking) { + PyEval_RestoreThread(save_tstate); + } + + for (unsigned int i = num_added; i < num; i++) { Py_DECREF(callable); /* unsuccessful add, destroy the extra reference */ - Py_RETURN_FALSE; } - Py_RETURN_TRUE; + /* The callable is decref'ed above in each added _pending_callback(). */ + return PyLong_FromUnsignedLong((unsigned long)num_added); } /* Test PyOS_string_to_double. */ @@ -891,27 +891,34 @@ test_string_to_double(PyObject *self, PyObject *Py_UNUSED(ignored)) { double result; const char *msg; -#define CHECK_STRING(STR, expected) \ - result = PyOS_string_to_double(STR, NULL, NULL); \ - if (result == -1.0 && PyErr_Occurred()) \ - return NULL; \ - if (result != (double)expected) { \ - msg = "conversion of " STR " to float failed"; \ - goto fail; \ - } +#define CHECK_STRING(STR, expected) \ + do { \ + result = PyOS_string_to_double(STR, NULL, NULL); \ + if (result == -1.0 && PyErr_Occurred()) { \ + return NULL; \ + } \ + if (result != (double)expected) { \ + msg = "conversion of " STR " to float failed"; \ + goto fail; \ + } \ + } while (0) -#define CHECK_INVALID(STR) \ - result = PyOS_string_to_double(STR, NULL, NULL); \ - if (result == -1.0 && PyErr_Occurred()) { \ - if (PyErr_ExceptionMatches(PyExc_ValueError)) \ - PyErr_Clear(); \ - else \ - return NULL; \ - } \ - else { \ - msg = "conversion of " STR " didn't raise ValueError"; \ - goto fail; \ - } +#define CHECK_INVALID(STR) \ + do { \ + result = PyOS_string_to_double(STR, NULL, NULL); \ + if (result == -1.0 && PyErr_Occurred()) { \ + if (PyErr_ExceptionMatches(PyExc_ValueError)) { \ + PyErr_Clear(); \ + } \ + else { \ + return NULL; \ + } \ + } \ + else { \ + msg = "conversion of " STR " didn't raise ValueError"; \ + goto fail; \ + } \ + } while (0) CHECK_STRING("0.1", 0.1); CHECK_STRING("1.234", 1.234); @@ -978,16 +985,22 @@ test_capsule(PyObject *self, PyObject *Py_UNUSED(ignored)) }; known_capsule *known = &known_capsules[0]; -#define FAIL(x) { error = (x); goto exit; } +#define FAIL(x) \ + do { \ + error = (x); \ + goto exit; \ + } while (0) #define CHECK_DESTRUCTOR \ - if (capsule_error) { \ - FAIL(capsule_error); \ - } \ - else if (!capsule_destructor_call_count) { \ - FAIL("destructor not called!"); \ - } \ - capsule_destructor_call_count = 0; \ + do { \ + if (capsule_error) { \ + FAIL(capsule_error); \ + } \ + else if (!capsule_destructor_call_count) { \ + FAIL("destructor not called!"); \ + } \ + capsule_destructor_call_count = 0; \ + } while (0) object = PyCapsule_New(capsule_pointer, capsule_name, capsule_destructor); PyCapsule_SetContext(object, capsule_context); @@ -1031,12 +1044,12 @@ test_capsule(PyObject *self, PyObject *Py_UNUSED(ignored)) static char buffer[256]; #undef FAIL #define FAIL(x) \ - { \ - sprintf(buffer, "%s module: \"%s\" attribute: \"%s\"", \ - x, known->module, known->attribute); \ - error = buffer; \ - goto exit; \ - } \ + do { \ + sprintf(buffer, "%s module: \"%s\" attribute: \"%s\"", \ + x, known->module, known->attribute); \ + error = buffer; \ + goto exit; \ + } while (0) PyObject *module = PyImport_ImportModule(known->module); if (module) { @@ -1493,36 +1506,6 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } -static PyObject * -get_interpreterid_type(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - return Py_NewRef(&PyInterpreterID_Type); -} - -static PyObject * -link_interpreter_refcount(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); - if (interp == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - _PyInterpreterState_RequireIDRef(interp, 1); - Py_RETURN_NONE; -} - -static PyObject * -unlink_interpreter_refcount(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); - if (interp == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - _PyInterpreterState_RequireIDRef(interp, 0); - Py_RETURN_NONE; -} - static PyMethodDef ml; static PyObject * @@ -1926,25 +1909,6 @@ getitem_with_error(PyObject *self, PyObject *args) return PyObject_GetItem(map, key); } -static PyObject * -dict_get_version(PyObject *self, PyObject *args) -{ - PyDictObject *dict; - uint64_t version; - - if (!PyArg_ParseTuple(args, "O!", &PyDict_Type, &dict)) - return NULL; - - _Py_COMP_DIAG_PUSH - _Py_COMP_DIAG_IGNORE_DEPR_DECLS - version = dict->ma_version_tag; - _Py_COMP_DIAG_POP - - static_assert(sizeof(unsigned long long) >= sizeof(version), - "version is larger than unsigned long long"); - return PyLong_FromUnsignedLongLong((unsigned long long)version); -} - static PyObject * raise_SIGINT_then_send_None(PyObject *self, PyObject *args) @@ -2015,11 +1979,15 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) "an already initialized key"); } #define CHECK_TSS_API(expr) \ + do { \ (void)(expr); \ if (!PyThread_tss_is_created(&tss_key)) { \ return raiseTestError(self, "test_pythread_tss_key_state", \ "TSS key initialization state was not " \ - "preserved after calling " #expr); } + "preserved after calling " #expr); \ + } \ + } while (0) + CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); CHECK_TSS_API(PyThread_tss_get(&tss_key)); #undef CHECK_TSS_API @@ -2341,7 +2309,7 @@ test_py_setref(PyObject *self, PyObject *Py_UNUSED(ignored)) \ Py_DECREF(obj); \ Py_RETURN_NONE; \ - } while (0) \ + } while (0) // Test Py_NewRef() and Py_XNewRef() macros @@ -2440,21 +2408,6 @@ type_modified(PyObject *self, PyObject *type) Py_RETURN_NONE; } -// Circumvents standard version assignment machinery - use with caution and only on -// short-lived heap types -static PyObject * -type_assign_specific_version_unsafe(PyObject *self, PyObject *args) -{ - PyTypeObject *type; - unsigned int version; - if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { - return NULL; - } - assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); - type->tp_version_tag = version; - type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; - Py_RETURN_NONE; -} static PyObject * type_assign_version(PyObject *self, PyObject *type) @@ -2520,7 +2473,7 @@ get_basic_static_type(PyObject *self, PyObject *args) PyTypeObject *cls = &BasicStaticTypes[num_basic_static_types_used++]; if (base != NULL) { - cls->tp_bases = Py_BuildValue("(O)", base); + cls->tp_bases = PyTuple_Pack(1, base); if (cls->tp_bases == NULL) { return NULL; } @@ -2683,18 +2636,6 @@ test_frame_getvarstring(PyObject *self, PyObject *args) } -static PyObject * -eval_get_func_name(PyObject *self, PyObject *func) -{ - return PyUnicode_FromString(PyEval_GetFuncName(func)); -} - -static PyObject * -eval_get_func_desc(PyObject *self, PyObject *func) -{ - return PyUnicode_FromString(PyEval_GetFuncDesc(func)); -} - static PyObject * gen_get_code(PyObject *self, PyObject *gen) { @@ -2720,107 +2661,60 @@ eval_eval_code_ex(PyObject *mod, PyObject *pos_args) PyObject **c_kwargs = NULL; - if (!PyArg_UnpackTuple(pos_args, - "eval_code_ex", - 2, - 8, - &code, - &globals, - &locals, - &args, - &kwargs, - &defaults, - &kw_defaults, - &closure)) + if (!PyArg_ParseTuple(pos_args, + "OO|OO!O!O!OO:eval_code_ex", + &code, + &globals, + &locals, + &PyTuple_Type, &args, + &PyDict_Type, &kwargs, + &PyTuple_Type, &defaults, + &kw_defaults, + &closure)) { goto exit; } - if (!PyCode_Check(code)) { - PyErr_SetString(PyExc_TypeError, - "code must be a Python code object"); - goto exit; - } - - if (!PyDict_Check(globals)) { - PyErr_SetString(PyExc_TypeError, "globals must be a dict"); - goto exit; - } - - if (locals && !PyMapping_Check(locals)) { - PyErr_SetString(PyExc_TypeError, "locals must be a mapping"); - goto exit; - } - if (locals == Py_None) { - locals = NULL; - } + NULLABLE(code); + NULLABLE(globals); + NULLABLE(locals); + NULLABLE(kw_defaults); + NULLABLE(closure); PyObject **c_args = NULL; Py_ssize_t c_args_len = 0; - - if (args) - { - if (!PyTuple_Check(args)) { - PyErr_SetString(PyExc_TypeError, "args must be a tuple"); - goto exit; - } else { - c_args = &PyTuple_GET_ITEM(args, 0); - c_args_len = PyTuple_Size(args); - } + if (args) { + c_args = &PyTuple_GET_ITEM(args, 0); + c_args_len = PyTuple_Size(args); } Py_ssize_t c_kwargs_len = 0; + if (kwargs) { + c_kwargs_len = PyDict_Size(kwargs); + if (c_kwargs_len > 0) { + c_kwargs = PyMem_NEW(PyObject*, 2 * c_kwargs_len); + if (!c_kwargs) { + PyErr_NoMemory(); + goto exit; + } - if (kwargs) - { - if (!PyDict_Check(kwargs)) { - PyErr_SetString(PyExc_TypeError, "keywords must be a dict"); - goto exit; - } else { - c_kwargs_len = PyDict_Size(kwargs); - if (c_kwargs_len > 0) { - c_kwargs = PyMem_NEW(PyObject*, 2 * c_kwargs_len); - if (!c_kwargs) { - PyErr_NoMemory(); - goto exit; - } - - Py_ssize_t i = 0; - Py_ssize_t pos = 0; - - while (PyDict_Next(kwargs, - &pos, - &c_kwargs[i], - &c_kwargs[i + 1])) - { - i += 2; - } - c_kwargs_len = i / 2; - /* XXX This is broken if the caller deletes dict items! */ + Py_ssize_t i = 0; + Py_ssize_t pos = 0; + while (PyDict_Next(kwargs, &pos, &c_kwargs[i], &c_kwargs[i + 1])) { + i += 2; } + c_kwargs_len = i / 2; + /* XXX This is broken if the caller deletes dict items! */ } } - PyObject **c_defaults = NULL; Py_ssize_t c_defaults_len = 0; - - if (defaults && PyTuple_Check(defaults)) { + if (defaults) { c_defaults = &PyTuple_GET_ITEM(defaults, 0); c_defaults_len = PyTuple_Size(defaults); } - if (kw_defaults && !PyDict_Check(kw_defaults)) { - PyErr_SetString(PyExc_TypeError, "kw_defaults must be a dict"); - goto exit; - } - - if (closure && !PyTuple_Check(closure)) { - PyErr_SetString(PyExc_TypeError, "closure must be a tuple of cells"); - goto exit; - } - - result = PyEval_EvalCodeEx( code, globals, @@ -3138,6 +3032,33 @@ function_set_kw_defaults(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +function_get_closure(PyObject *self, PyObject *func) +{ + PyObject *closure = PyFunction_GetClosure(func); + if (closure != NULL) { + return Py_NewRef(closure); + } else if (PyErr_Occurred()) { + return NULL; + } else { + Py_RETURN_NONE; // This can happen when `closure` is set to `None` + } +} + +static PyObject * +function_set_closure(PyObject *self, PyObject *args) +{ + PyObject *func = NULL, *closure = NULL; + if (!PyArg_ParseTuple(args, "OO", &func, &closure)) { + return NULL; + } + int result = PyFunction_SetClosure(func, closure); + if (result == -1) { + return NULL; + } + Py_RETURN_NONE; +} + static PyObject * check_pyimport_addmodule(PyObject *self, PyObject *args) { @@ -3284,6 +3205,154 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) _Py_COMP_DIAG_POP } +struct simpletracer_data { + int create_count; + int destroy_count; + void* addresses[10]; +}; + +static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { + struct simpletracer_data* the_data = (struct simpletracer_data*)data; + assert(the_data->create_count + the_data->destroy_count < (int)Py_ARRAY_LENGTH(the_data->addresses)); + the_data->addresses[the_data->create_count + the_data->destroy_count] = obj; + if (event == PyRefTracer_CREATE) { + the_data->create_count++; + } else { + the_data->destroy_count++; + } + return 0; +} + +static PyObject * +test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + // Save the current tracer and data to restore it later + void* current_data; + PyRefTracer current_tracer = PyRefTracer_GetTracer(¤t_data); + + struct simpletracer_data tracer_data = {0}; + void* the_data = &tracer_data; + // Install a simple tracer function + if (PyRefTracer_SetTracer(_simpletracer, the_data) != 0) { + goto failed; + } + + // Check that the tracer was correctly installed + void* data; + if (PyRefTracer_GetTracer(&data) != _simpletracer || data != the_data) { + PyErr_SetString(PyExc_AssertionError, "The reftracer not correctly installed"); + (void)PyRefTracer_SetTracer(NULL, NULL); + goto failed; + } + + // Create a bunch of objects + PyObject* obj = PyList_New(0); + if (obj == NULL) { + goto failed; + } + PyObject* obj2 = PyDict_New(); + if (obj2 == NULL) { + Py_DECREF(obj); + goto failed; + } + + // Kill all objects + Py_DECREF(obj); + Py_DECREF(obj2); + + // Remove the tracer + (void)PyRefTracer_SetTracer(NULL, NULL); + + // Check that the tracer was removed + if (PyRefTracer_GetTracer(&data) != NULL || data != NULL) { + PyErr_SetString(PyExc_ValueError, "The reftracer was not correctly removed"); + goto failed; + } + + if (tracer_data.create_count != 2 || + tracer_data.addresses[0] != obj || + tracer_data.addresses[1] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object creation was not correctly traced"); + goto failed; + } + + if (tracer_data.destroy_count != 2 || + tracer_data.addresses[2] != obj || + tracer_data.addresses[3] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object destruction was not correctly traced"); + goto failed; + } + PyRefTracer_SetTracer(current_tracer, current_data); + Py_RETURN_NONE; +failed: + PyRefTracer_SetTracer(current_tracer, current_data); + return NULL; +} + +static PyObject * +function_set_warning(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + if (PyErr_WarnEx(PyExc_RuntimeWarning, "Testing PyErr_WarnEx", 2)) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +test_critical_sections(PyObject *module, PyObject *Py_UNUSED(args)) +{ + Py_BEGIN_CRITICAL_SECTION(module); + Py_END_CRITICAL_SECTION(); + + Py_BEGIN_CRITICAL_SECTION2(module, module); + Py_END_CRITICAL_SECTION2(); + + Py_RETURN_NONE; +} + + +// Used by `finalize_thread_hang`. +#ifdef _POSIX_THREADS +static void finalize_thread_hang_cleanup_callback(void *Py_UNUSED(arg)) { + // Should not reach here. + Py_FatalError("pthread thread termination was triggered unexpectedly"); +} +#endif + +// Tests that finalization does not trigger pthread cleanup. +// +// Must be called with a single nullary callable function that should block +// (with GIL released) until finalization is in progress. +static PyObject * +finalize_thread_hang(PyObject *self, PyObject *callback) +{ + // WASI builds some pthread stuff but doesn't have these APIs today? +#if defined(_POSIX_THREADS) && !defined(__wasi__) + pthread_cleanup_push(finalize_thread_hang_cleanup_callback, NULL); +#endif + PyObject_CallNoArgs(callback); + // Should not reach here. + Py_FatalError("thread unexpectedly did not hang"); +#if defined(_POSIX_THREADS) && !defined(__wasi__) + pthread_cleanup_pop(0); +#endif + Py_RETURN_NONE; +} + + +static PyObject * +type_freeze(PyObject *module, PyObject *args) +{ + PyTypeObject *type; + if (!PyArg_ParseTuple(args, "O!", &PyType_Type, &type)) { + return NULL; + } + if (PyType_Freeze(type) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, @@ -3317,15 +3386,20 @@ static PyMethodDef TestMethods[] = { {"test_buildvalue_N", test_buildvalue_N, METH_NOARGS}, {"test_get_statictype_slots", test_get_statictype_slots, METH_NOARGS}, {"get_heaptype_for_name", get_heaptype_for_name, METH_NOARGS}, - {"test_get_type_name", test_get_type_name, METH_NOARGS}, - {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, + {"get_type_name", get_type_name, METH_O}, + {"get_type_qualname", get_type_qualname, METH_O}, + {"get_type_fullyqualname", get_type_fullyqualname, METH_O}, + {"get_type_module_name", get_type_module_name, METH_O}, {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, + {"test_reftracer", test_reftracer, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, + {"gilstate_ensure_release", gilstate_ensure_release, METH_NOARGS}, #ifndef MS_WINDOWS {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, {"_end_spawned_pthread", end_spawned_pthread, METH_NOARGS}, #endif - {"_pending_threadfunc", pending_threadfunc, METH_VARARGS}, + {"_pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), + METH_VARARGS|METH_KEYWORDS}, #ifdef HAVE_GETTIMEOFDAY {"profile_int", profile_int, METH_NOARGS}, #endif @@ -3338,9 +3412,6 @@ static PyMethodDef TestMethods[] = { {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, - {"get_interpreterid_type", get_interpreterid_type, METH_NOARGS}, - {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, - {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, @@ -3361,7 +3432,6 @@ static PyMethodDef TestMethods[] = { {"return_result_with_error", return_result_with_error, METH_NOARGS}, {"getitem_with_error", getitem_with_error, METH_VARARGS}, {"Py_CompileString", pycompilestring, METH_O}, - {"dict_get_version", dict_get_version, METH_VARARGS}, {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, #ifdef W_STOPCODE @@ -3390,8 +3460,6 @@ static PyMethodDef TestMethods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, {"type_modified", type_modified, METH_O, PyDoc_STR("PyType_Modified")}, - {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, - PyDoc_STR("forcefully assign type->tp_version_tag")}, {"type_assign_version", type_assign_version, METH_O, PyDoc_STR("PyUnstable_Type_AssignVersionTag")}, {"type_get_tp_bases", type_get_tp_bases, METH_O}, {"type_get_tp_mro", type_get_tp_mro, METH_O}, @@ -3405,8 +3473,6 @@ static PyMethodDef TestMethods[] = { {"frame_new", frame_new, METH_VARARGS, NULL}, {"frame_getvar", test_frame_getvar, METH_VARARGS, NULL}, {"frame_getvarstring", test_frame_getvarstring, METH_VARARGS, NULL}, - {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, - {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, {"gen_get_code", gen_get_code, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, {"test_code_api", test_code_api, METH_NOARGS, NULL}, @@ -3421,8 +3487,14 @@ static PyMethodDef TestMethods[] = { {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, + {"function_get_closure", function_get_closure, METH_O, NULL}, + {"function_set_closure", function_set_closure, METH_VARARGS, NULL}, {"check_pyimport_addmodule", check_pyimport_addmodule, METH_VARARGS}, {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, + {"function_set_warning", function_set_warning, METH_NOARGS}, + {"test_critical_sections", test_critical_sections, METH_NOARGS}, + {"finalize_thread_hang", finalize_thread_hang, METH_O, NULL}, + {"type_freeze", type_freeze, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; @@ -3527,7 +3599,7 @@ static PyTypeObject matmulType = { 0, 0, PyType_GenericNew, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; typedef struct { @@ -3537,7 +3609,7 @@ typedef struct { static PyObject * ipowType_ipow(PyObject *self, PyObject *other, PyObject *mod) { - return Py_BuildValue("OO", other, mod); + return PyTuple_Pack(2, other, mod); } static PyNumberMethods ipowType_as_number = { @@ -3639,7 +3711,7 @@ static PyTypeObject awaitType = { 0, 0, awaitObject_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; @@ -3914,9 +3986,14 @@ PyInit__testcapi(void) m = PyModule_Create(&_testcapimodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type); - + if (PyType_Ready(&_HashInheritanceTester_Type) < 0) { + return NULL; + } if (PyType_Ready(&matmulType) < 0) return NULL; Py_INCREF(&matmulType); @@ -3988,12 +4065,29 @@ PyInit__testcapi(void) PyModule_AddObject(m, "SIZEOF_WCHAR_T", PyLong_FromSsize_t(sizeof(wchar_t))); PyModule_AddObject(m, "SIZEOF_VOID_P", PyLong_FromSsize_t(sizeof(void*))); PyModule_AddObject(m, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))); + PyModule_AddObject(m, "SIZEOF_PID_T", PyLong_FromSsize_t(sizeof(pid_t))); PyModule_AddObject(m, "Py_Version", PyLong_FromUnsignedLong(Py_Version)); Py_INCREF(&PyInstanceMethod_Type); PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); PyModule_AddIntConstant(m, "the_number_three", 3); PyModule_AddIntMacro(m, Py_C_RECURSION_LIMIT); + PyModule_AddObject(m, "INT32_MIN", PyLong_FromInt32(INT32_MIN)); + PyModule_AddObject(m, "INT32_MAX", PyLong_FromInt32(INT32_MAX)); + PyModule_AddObject(m, "UINT32_MAX", PyLong_FromUInt32(UINT32_MAX)); + PyModule_AddObject(m, "INT64_MIN", PyLong_FromInt64(INT64_MIN)); + PyModule_AddObject(m, "INT64_MAX", PyLong_FromInt64(INT64_MAX)); + PyModule_AddObject(m, "UINT64_MAX", PyLong_FromUInt64(UINT64_MAX)); + + if (PyModule_AddIntMacro(m, Py_single_input)) { + return NULL; + } + if (PyModule_AddIntMacro(m, Py_file_input)) { + return NULL; + } + if (PyModule_AddIntMacro(m, Py_eval_input)) { + return NULL; + } testcapistate_t *state = get_testcapi_state(m); state->error = PyErr_NewException("_testcapi.error", NULL, NULL); @@ -4017,9 +4111,6 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Abstract(m) < 0) { return NULL; } - if (_PyTestCapi_Init_ByteArray(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_Bytes(m) < 0) { return NULL; } @@ -4077,18 +4168,12 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Buffer(m) < 0) { return NULL; } - if (_PyTestCapi_Init_PyOS(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_File(m) < 0) { return NULL; } if (_PyTestCapi_Init_Codec(m) < 0) { return NULL; } - if (_PyTestCapi_Init_Sys(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_Immortal(m) < 0) { return NULL; } @@ -4098,13 +4183,22 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_PyAtomic(m) < 0) { return NULL; } - if (_PyTestCapi_Init_VectorcallLimited(m) < 0) { + if (_PyTestCapi_Init_Run(m) < 0) { return NULL; } - if (_PyTestCapi_Init_HeaptypeRelative(m) < 0) { + if (_PyTestCapi_Init_Hash(m) < 0) { return NULL; } - if (_PyTestCapi_Init_Hash(m) < 0) { + if (_PyTestCapi_Init_Time(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Monitoring(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Object(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Config(m) < 0) { return NULL; } diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index fb0936bbccd318..e3c8ba9b0b5074 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -58,6 +58,20 @@ pack_arguments_newref(int argc, ...) return tuple; } +static PyObject * +pack_varargs_to_tuple(Py_ssize_t varargssize, PyObject *const *args) +{ + assert(!PyErr_Occurred()); + PyObject *tuple = PyTuple_New(varargssize); + if (!tuple) { + return NULL; + } + for (Py_ssize_t i = 0; i < varargssize; i++) { + PyTuple_SET_ITEM(tuple, i, Py_NewRef(args[i])); + } + return tuple; +} + /* Pack arguments to a tuple. * `wrapper` is function which converts primitive type to PyObject. * `arg_type` is type that arguments should be converted to before wrapped. */ @@ -963,43 +977,72 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) /*[clinic input] -posonly_vararg +varpos + + *args: object + +[clinic start generated code]*/ + +static PyObject * +varpos_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=b65096f423fb5dcc input=f87cd674145d394c]*/ +{ + PyObject *vararg_tuple = pack_varargs_to_tuple(nargs, args); + if (!vararg_tuple) { + return NULL; + } + PyObject *result = pack_arguments_newref(1, vararg_tuple); + Py_DECREF(vararg_tuple); + return result; +} + + +/*[clinic input] +posonly_varpos a: object - / b: object + / *args: object [clinic start generated code]*/ static PyObject * -posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *args) -/*[clinic end generated code: output=ee6713acda6b954e input=783427fe7ec2b67a]*/ +posonly_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=d10d43d86d117ab3 input=c9fd7895cfbaabba]*/ { - return pack_arguments_newref(3, a, b, args); + PyObject *vararg_tuple = pack_varargs_to_tuple(nargs, args); + if (!vararg_tuple) { + return NULL; + } + PyObject *result = pack_arguments_newref(3, a, b, vararg_tuple); + Py_DECREF(vararg_tuple); + return result; } /*[clinic input] -vararg_and_posonly +posonly_poskw_varpos a: object - *args: object / + b: object + *args: object [clinic start generated code]*/ static PyObject * -vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=42792f799465a14d input=defe017b19ba52e8]*/ +posonly_poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=bffdb7649941c939 input=b3d7a734e0625f68]*/ { - return pack_arguments_newref(2, a, args); + return pack_arguments_newref(3, a, b, args); } /*[clinic input] -vararg +poskw_varpos a: object *args: object @@ -1007,15 +1050,15 @@ vararg [clinic start generated code]*/ static PyObject * -vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=91ab7a0efc52dd5e input=02c0f772d05f591e]*/ +poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=2413ddfb5ef22794 input=a1dff12d00422484]*/ { return pack_arguments_newref(2, a, args); } /*[clinic input] -vararg_with_default +poskw_varpos_kwonly_opt a: object *args: object @@ -1024,9 +1067,9 @@ vararg_with_default [clinic start generated code]*/ static PyObject * -vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, - int b) -/*[clinic end generated code: output=182c01035958ce92 input=68cafa6a79f89e36]*/ +poskw_varpos_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *args, + int b) +/*[clinic end generated code: output=f36d35ba6133463b input=1721d43dc5f6d856]*/ { PyObject *obj_b = b ? Py_True : Py_False; return pack_arguments_newref(3, a, args, obj_b); @@ -1034,21 +1077,59 @@ vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, /*[clinic input] -vararg_with_only_defaults +poskw_varpos_kwonly_opt2 + a: object *args: object - b: object = None + b: object = False + c: object = False + +[clinic start generated code]*/ + +static PyObject * +poskw_varpos_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *args, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=846cef62c6c40463 input=bb4b8d1577a8a408]*/ +{ + return pack_arguments_newref(4, a, args, b, c); +} + + +/*[clinic input] +varpos_kwonly_opt + + *args: object + b: object = False [clinic start generated code]*/ static PyObject * -vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b) -/*[clinic end generated code: output=c06b1826d91f2f7b input=678c069bc67550e1]*/ +varpos_kwonly_opt_impl(PyObject *module, PyObject *args, PyObject *b) +/*[clinic end generated code: output=3b7bf98b091f5731 input=380fb00deec847e8]*/ { return pack_arguments_newref(2, args, b); } +/*[clinic input] +varpos_kwonly_req_opt + + *args: object + a: object + b: object = False + c: object = False + +[clinic start generated code]*/ + +static PyObject * +varpos_kwonly_req_opt_impl(PyObject *module, PyObject *args, PyObject *a, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=165274e1fd037ae9 input=530794afd0690c22]*/ +{ + return pack_arguments_newref(4, args, a, b, c); +} + + /*[clinic input] gh_32092_oob @@ -1096,15 +1177,15 @@ gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, gh_99233_refcount *args: object - / Proof-of-concept of GH-99233 refcount error bug. [clinic start generated code]*/ static PyObject * -gh_99233_refcount_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=585855abfbca9a7f input=85f5fb47ac91a626]*/ +gh_99233_refcount_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args) +/*[clinic end generated code: output=b570007e61e5c670 input=eecfdc2092d90dc3]*/ { Py_RETURN_NONE; } @@ -1219,21 +1300,71 @@ class _testclinic.TestClass "PyObject *" "PyObject" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=668a591c65bec947]*/ /*[clinic input] -_testclinic.TestClass.meth_method_no_params +_testclinic.TestClass.get_defining_class + cls: defining_class +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_get_defining_class_impl(PyObject *self, + PyTypeObject *cls) +/*[clinic end generated code: output=94f9b0b5f7add930 input=537c59417471dee3]*/ +{ + return Py_NewRef(cls); +} + +/*[clinic input] +_testclinic.TestClass.get_defining_class_arg + cls: defining_class + arg: object +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_get_defining_class_arg_impl(PyObject *self, + PyTypeObject *cls, + PyObject *arg) +/*[clinic end generated code: output=fe7e49d96cbb7718 input=d1b83d3b853af6d9]*/ +{ + return PyTuple_Pack(2, cls, arg); +} + +/*[clinic input] +_testclinic.TestClass.defclass_varpos cls: defining_class + *args: object +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_defclass_varpos_impl(PyObject *self, PyTypeObject *cls, + PyObject *args) +/*[clinic end generated code: output=fad33f2d3a8d778d input=47071dcda393a7e1]*/ +{ + return PyTuple_Pack(2, cls, args); +} + +/*[clinic input] +_testclinic.TestClass.defclass_posonly_varpos + cls: defining_class + a: object + b: object / + *args: object [clinic start generated code]*/ static PyObject * -_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, - PyTypeObject *cls) -/*[clinic end generated code: output=c140f100080c2fc8 input=6bd34503d11c63c1]*/ +_testclinic_TestClass_defclass_posonly_varpos_impl(PyObject *self, + PyTypeObject *cls, + PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=1740fcf48d230b07 input=40f2e56286d4a7ef]*/ { - Py_RETURN_NONE; + return pack_arguments_newref(4, cls, a, b, args); } static struct PyMethodDef test_class_methods[] = { - _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF + _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_METHODDEF + _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_ARG_METHODDEF + _TESTCLINIC_TESTCLASS_DEFCLASS_VARPOS_METHODDEF + _TESTCLINIC_TESTCLASS_DEFCLASS_POSONLY_VARPOS_METHODDEF {NULL, NULL} }; @@ -1872,6 +2003,7 @@ static PyMethodDef tester_methods[] = { STR_CONVERTER_METHODDEF STR_CONVERTER_ENCODING_METHODDEF PY_BUFFER_CONVERTER_METHODDEF + KEYWORDS_METHODDEF KEYWORDS_KWONLY_METHODDEF KEYWORDS_OPT_METHODDEF @@ -1888,16 +2020,21 @@ static PyMethodDef tester_methods[] = { POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF - POSONLY_VARARG_METHODDEF - VARARG_AND_POSONLY_METHODDEF - VARARG_METHODDEF - VARARG_WITH_DEFAULT_METHODDEF - VARARG_WITH_ONLY_DEFAULTS_METHODDEF + + VARPOS_METHODDEF + POSONLY_VARPOS_METHODDEF + POSONLY_POSKW_VARPOS_METHODDEF + POSKW_VARPOS_METHODDEF + POSKW_VARPOS_KWONLY_OPT_METHODDEF + POSKW_VARPOS_KWONLY_OPT2_METHODDEF + VARPOS_KWONLY_OPT_METHODDEF + VARPOS_KWONLY_REQ_OPT_METHODDEF GH_32092_OOB_METHODDEF GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF GH_99240_DOUBLE_FREE_METHODDEF NULL_OR_TUPLE_FOR_VARARGS_METHODDEF + CLONE_F1_METHODDEF CLONE_F2_METHODDEF CLONE_WITH_CONV_F1_METHODDEF @@ -1940,6 +2077,9 @@ PyInit__testclinic(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddType(m, &TestClass) < 0) { goto error; } diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index ef595be0b626db..370433b3e2a0d9 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -4,11 +4,10 @@ #undef Py_BUILD_CORE_MODULE #undef Py_BUILD_CORE_BUILTIN +// For now, AC only supports the limited C API version 3.13 #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// For now, only limited C API 3.13 is supported -#define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030d0000 #endif /* Always enable assertions */ @@ -72,10 +71,64 @@ my_int_sum_impl(PyObject *module, int x, int y) } +/*[clinic input] +my_float_sum -> float + + x: float + y: float + / + +[clinic start generated code]*/ + +static float +my_float_sum_impl(PyObject *module, float x, float y) +/*[clinic end generated code: output=634f59a5a419cad7 input=d4b5313bdf4dc377]*/ +{ + return x + y; +} + + +/*[clinic input] +my_double_sum -> double + + x: double + y: double + / + +[clinic start generated code]*/ + +static double +my_double_sum_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=a75576d9e4d8557f input=16b11c8aba172801]*/ +{ + return x + y; +} + + +/*[clinic input] +get_file_descriptor -> int + + file as fd: fildes + / + +Get a file descriptor. +[clinic start generated code]*/ + +static int +get_file_descriptor_impl(PyObject *module, int fd) +/*[clinic end generated code: output=80051ebad54db8a8 input=82e2a1418848cd5b]*/ +{ + return fd; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF MY_INT_FUNC_METHODDEF MY_INT_SUM_METHODDEF + MY_FLOAT_SUM_METHODDEF + MY_DOUBLE_SUM_METHODDEF + GET_FILE_DESCRIPTOR_METHODDEF {NULL, NULL} }; @@ -93,5 +146,8 @@ PyInit__testclinic_limited(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif return m; } diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c new file mode 100644 index 00000000000000..0807d1e47b6736 --- /dev/null +++ b/Modules/_testexternalinspection.c @@ -0,0 +1,645 @@ +#define _GNU_SOURCE + +#ifdef __linux__ +# include +# include +# if INTPTR_MAX == INT64_MAX +# define Elf_Ehdr Elf64_Ehdr +# define Elf_Shdr Elf64_Shdr +# define Elf_Phdr Elf64_Phdr +# else +# define Elf_Ehdr Elf32_Ehdr +# define Elf_Shdr Elf32_Shdr +# define Elf_Phdr Elf32_Phdr +# endif +# include +#endif + +#if defined(__APPLE__) +# include +// Older macOS SDKs do not define TARGET_OS_OSX +# if !defined(TARGET_OS_OSX) +# define TARGET_OS_OSX 1 +# endif +# if TARGET_OS_OSX +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# endif +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif +#include "Python.h" +#include // _Py_DebugOffsets +#include // FRAME_OWNED_BY_CSTACK +#include // Py_TAG_BITS + +#ifndef HAVE_PROCESS_VM_READV +# define HAVE_PROCESS_VM_READV 0 +#endif + +#if defined(__APPLE__) && TARGET_OS_OSX +static void* +analyze_macho64(mach_port_t proc_ref, void* base, void* map) +{ + struct mach_header_64* hdr = (struct mach_header_64*)map; + int ncmds = hdr->ncmds; + + int cmd_cnt = 0; + struct segment_command_64* cmd = map + sizeof(struct mach_header_64); + + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + mach_vm_address_t address = (mach_vm_address_t)base; + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { + while (cmd->filesize != size) { + address += size; + if (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, // cppcheck-suppress [uninitvar] + &count, + &object_name) + != KERN_SUCCESS) + { + PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + return NULL; + } + } + base = (void*)address - cmd->vmaddr; + + int nsects = cmd->nsects; + struct section_64* sec = + (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); + for (int j = 0; j < nsects; j++) { + if (strcmp(sec[j].sectname, "PyRuntime") == 0) { + return base + sec[j].addr; + } + } + cmd_cnt++; + } + + cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); + } + return NULL; +} + +static void* +analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) +{ + int fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); + return NULL; + } + + struct stat fs; + if (fstat(fd, &fs) == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + close(fd); + return NULL; + } + + void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); + close(fd); + return NULL; + } + + void* result = NULL; + + struct mach_header_64* hdr = (struct mach_header_64*)map; + switch (hdr->magic) { + case MH_MAGIC: + case MH_CIGAM: + case FAT_MAGIC: + case FAT_CIGAM: + PyErr_SetString(PyExc_RuntimeError, "32-bit Mach-O binaries are not supported"); + break; + case MH_MAGIC_64: + case MH_CIGAM_64: + result = analyze_macho64(proc_ref, base, map); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); + break; + } + + munmap(map, fs.st_size); + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + return result; +} + +static mach_port_t +pid_to_task(pid_t pid) +{ + mach_port_t task; + kern_return_t result; + + result = task_for_pid(mach_task_self(), pid, &task); + if (result != KERN_SUCCESS) { + PyErr_Format(PyExc_PermissionError, "Cannot get task for PID %d", pid); + return 0; + } + return task; +} + +static void* +get_py_runtime_macos(pid_t pid) +{ + mach_vm_address_t address = 0; + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + mach_port_t proc_ref = pid_to_task(pid); + if (proc_ref == 0) { + PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); + return NULL; + } + + int match_found = 0; + char map_filename[MAXPATHLEN + 1]; + void* result_address = NULL; + while (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, + &count, + &object_name) + == KERN_SUCCESS) + { + int path_len = proc_regionfilename(pid, address, map_filename, MAXPATHLEN); + if (path_len == 0) { + address += size; + continue; + } + + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + break; + } + + address += size; + } + return result_address; +} +#endif + +#ifdef __linux__ +void* +find_python_map_start_address(pid_t pid, char* result_filename) +{ + char maps_file_path[64]; + sprintf(maps_file_path, "/proc/%d/maps", pid); + + FILE* maps_file = fopen(maps_file_path, "r"); + if (maps_file == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + int match_found = 0; + + char line[256]; + char map_filename[PATH_MAX]; + void* result_address = 0; + while (fgets(line, sizeof(line), maps_file) != NULL) { + unsigned long start_address = 0; + sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + break; + } + } + + fclose(maps_file); + + if (!match_found) { + map_filename[0] = '\0'; + } + + return result_address; +} + +void* +get_py_runtime_linux(pid_t pid) +{ + char elf_file[256]; + void* start_address = (void*)find_python_map_start_address(pid, elf_file); + + if (start_address == 0) { + PyErr_SetString(PyExc_RuntimeError, "No memory map associated with python or libpython found"); + return NULL; + } + + void* result = NULL; + void* file_memory = NULL; + + int fd = open(elf_file, O_RDONLY); + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + struct stat file_stats; + if (fstat(fd, &file_stats) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (file_memory == MAP_FAILED) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; + + Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); + + Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; + char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); + + Elf_Shdr* py_runtime_section = NULL; + for (int i = 0; i < elf_header->e_shnum; i++) { + if (strcmp(".PyRuntime", shstrtab + section_header_table[i].sh_name) == 0) { + py_runtime_section = §ion_header_table[i]; + break; + } + } + + Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); + // Find the first PT_LOAD segment + Elf_Phdr* first_load_segment = NULL; + for (int i = 0; i < elf_header->e_phnum; i++) { + if (program_header_table[i].p_type == PT_LOAD) { + first_load_segment = &program_header_table[i]; + break; + } + } + + if (py_runtime_section != NULL && first_load_segment != NULL) { + uintptr_t elf_load_addr = first_load_segment->p_vaddr + - (first_load_segment->p_vaddr % first_load_segment->p_align); + result = start_address + py_runtime_section->sh_addr - elf_load_addr; + } + +exit: + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + if (file_memory != NULL) { + munmap(file_memory, file_stats.st_size); + } + return result; +} +#endif + +ssize_t +read_memory(pid_t pid, void* remote_address, size_t len, void* dst) +{ + ssize_t total_bytes_read = 0; +#if defined(__linux__) && HAVE_PROCESS_VM_READV + struct iovec local[1]; + struct iovec remote[1]; + ssize_t result = 0; + ssize_t read = 0; + + do { + local[0].iov_base = dst + result; + local[0].iov_len = len - result; + remote[0].iov_base = (void*)(remote_address + result); + remote[0].iov_len = len - result; + + read = process_vm_readv(pid, local, 1, remote, 1, 0); + if (read < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += read; + } while ((size_t)read != local[0].iov_len); + total_bytes_read = result; +#elif defined(__APPLE__) && TARGET_OS_OSX + ssize_t result = -1; + kern_return_t kr = mach_vm_read_overwrite( + pid_to_task(pid), + (mach_vm_address_t)remote_address, + len, + (mach_vm_address_t)dst, + (mach_vm_size_t*)&result); + + if (kr != KERN_SUCCESS) { + switch (kr) { + case KERN_PROTECTION_FAILURE: + PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + break; + case KERN_INVALID_ARGUMENT: + PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + } + return -1; + } + total_bytes_read = len; +#else + return -1; +#endif + return total_bytes_read; +} + +int +read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buffer, Py_ssize_t size) +{ + Py_ssize_t len; + ssize_t bytes_read = + read_memory(pid, address + debug_offsets->unicode_object.length, sizeof(Py_ssize_t), &len); + if (bytes_read == -1) { + return -1; + } + if (len >= size) { + PyErr_SetString(PyExc_RuntimeError, "Buffer too small"); + return -1; + } + size_t offset = debug_offsets->unicode_object.asciiobject_size; + bytes_read = read_memory(pid, address + offset, len, buffer); + if (bytes_read == -1) { + return -1; + } + buffer[len] = '\0'; + return 0; +} + +void* +get_py_runtime(pid_t pid) +{ +#if defined(__linux__) + return get_py_runtime_linux(pid); +#elif defined(__APPLE__) && TARGET_OS_OSX + return get_py_runtime_macos(pid); +#else + return NULL; +#endif +} + +static int +parse_code_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + void* address_of_function_name; + read_memory( + pid, + (void*)(address + offsets->code_object.name), + sizeof(void*), + &address_of_function_name); + + if (address_of_function_name == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No function name found"); + return -1; + } + + char function_name[256]; + if (read_string(pid, offsets, address_of_function_name, function_name, sizeof(function_name)) != 0) { + return -1; + } + + PyObject* py_function_name = PyUnicode_FromString(function_name); + if (py_function_name == NULL) { + return -1; + } + + if (PyList_Append(result, py_function_name) == -1) { + Py_DECREF(py_function_name); + return -1; + } + Py_DECREF(py_function_name); + + return 0; +} + +static int +parse_frame_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + ssize_t bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.previous), + sizeof(void*), + previous_frame); + if (bytes_read == -1) { + return -1; + } + + char owner; + bytes_read = + read_memory(pid, (void*)(address + offsets->interpreter_frame.owner), sizeof(char), &owner); + if (bytes_read < 0) { + return -1; + } + + if (owner == FRAME_OWNED_BY_CSTACK) { + return 0; + } + + uintptr_t address_of_code_object; + bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.executable), + sizeof(void*), + &address_of_code_object); + if (bytes_read == -1) { + return -1; + } + + if (address_of_code_object == 0) { + return 0; + } + address_of_code_object &= ~Py_TAG_BITS; + return parse_code_object(pid, result, offsets, (void *)address_of_code_object, previous_frame); +} + +static PyObject* +get_stack_trace(PyObject* self, PyObject* args) +{ +#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); + return NULL; +#endif + int pid; + + if (!PyArg_ParseTuple(args, "i", &pid)) { + return NULL; + } + + void* runtime_start_address = get_py_runtime(pid); + if (runtime_start_address == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get .PyRuntime address"); + } + return NULL; + } + size_t size = sizeof(struct _Py_DebugOffsets); + struct _Py_DebugOffsets local_debug_offsets; + + ssize_t bytes_read = read_memory(pid, runtime_start_address, size, &local_debug_offsets); + if (bytes_read == -1) { + return NULL; + } + off_t interpreter_state_list_head = local_debug_offsets.runtime_state.interpreters_head; + + void* address_of_interpreter_state; + bytes_read = read_memory( + pid, + (void*)(runtime_start_address + interpreter_state_list_head), + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read == -1) { + return NULL; + } + + if (address_of_interpreter_state == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return NULL; + } + + void* address_of_thread; + bytes_read = read_memory( + pid, + (void*)(address_of_interpreter_state + local_debug_offsets.interpreter_state.threads_head), + sizeof(void*), + &address_of_thread); + if (bytes_read == -1) { + return NULL; + } + + PyObject* result = PyList_New(0); + if (result == NULL) { + return NULL; + } + + // No Python frames are available for us (can happen at tear-down). + if (address_of_thread != NULL) { + void* address_of_current_frame; + (void)read_memory( + pid, + (void*)(address_of_thread + local_debug_offsets.thread_state.current_frame), + sizeof(void*), + &address_of_current_frame); + while (address_of_current_frame != NULL) { + if (parse_frame_object( + pid, + result, + &local_debug_offsets, + address_of_current_frame, + &address_of_current_frame) + < 0) + { + Py_DECREF(result); + return NULL; + } + } + } + + return result; +} + +static PyMethodDef methods[] = { + {"get_stack_trace", get_stack_trace, METH_VARARGS, "Get the Python stack from a given PID"}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_testexternalinspection", + .m_size = -1, + .m_methods = methods, +}; + +PyMODINIT_FUNC +PyInit__testexternalinspection(void) +{ + PyObject* mod = PyModule_Create(&module); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif + int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + if (rc < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} diff --git a/Modules/_testimportmultiple.c b/Modules/_testimportmultiple.c index 7e6556ad400cde..c147596f88a3a8 100644 --- a/Modules/_testimportmultiple.c +++ b/Modules/_testimportmultiple.c @@ -5,20 +5,25 @@ */ #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x03020000 +# define Py_LIMITED_API 0x030d0000 #endif #include +static PyModuleDef_Slot shared_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef _testimportmultiple = { PyModuleDef_HEAD_INIT, "_testimportmultiple", "_testimportmultiple doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -26,16 +31,16 @@ static struct PyModuleDef _testimportmultiple = { PyMODINIT_FUNC PyInit__testimportmultiple(void) { - return PyModule_Create(&_testimportmultiple); + return PyModuleDef_Init(&_testimportmultiple); } static struct PyModuleDef _foomodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_foo", "_testimportmultiple_foo doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -43,21 +48,21 @@ static struct PyModuleDef _foomodule = { PyMODINIT_FUNC PyInit__testimportmultiple_foo(void) { - return PyModule_Create(&_foomodule); + return PyModuleDef_Init(&_foomodule); } static struct PyModuleDef _barmodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_bar", "_testimportmultiple_bar doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL }; PyMODINIT_FUNC PyInit__testimportmultiple_bar(void){ - return PyModule_Create(&_barmodule); + return PyModuleDef_Init(&_barmodule); } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 0bb739b5398b11..eb98b433c6c6af 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -10,26 +10,28 @@ #undef NDEBUG #include "Python.h" +#include "pycore_backoff.h" // JUMP_BACKWARD_INITIAL_VALUE #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_ceval.h" // _PyEval_AddPendingCall() #include "pycore_compile.h" // _PyCompile_CodeGen() #include "pycore_context.h" // _PyContext_NewHamtForTests() -#include "pycore_dict.h" // _PyDictOrValues_GetValues() +#include "pycore_dict.h" // _PyManagedDictPointer_GetValues() #include "pycore_fileutils.h" // _Py_normpath() +#include "pycore_flowgraph.h" // _PyCompile_OptimizeCfg() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() +#include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_typeobject.h" // _PyType_GetModuleName() - -#include "interpreteridobject.h" // PyInterpreterID_LookUp() #include "clinic/_testinternalcapi.c.h" @@ -679,13 +681,13 @@ set_eval_frame_default(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject * record_eval(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc) { - if (PyFunction_Check(f->f_funcobj)) { + if (PyStackRef_FunctionCheck(f->f_funcobj)) { + PyFunctionObject *func = _PyFrame_GetFunction(f); PyObject *module = _get_current_module(); assert(module != NULL); module_state *state = get_module_state(module); Py_DECREF(module); - int res = PyList_Append(state->record_list, - ((PyFunctionObject *)f->f_funcobj)->func_name); + int res = PyList_Append(state->record_list, func->func_name); if (res < 0) { return NULL; } @@ -723,6 +725,19 @@ _testinternalcapi_compiler_cleandoc_impl(PyObject *module, PyObject *doc) return _PyCompile_CleanDoc(doc); } +/*[clinic input] + +_testinternalcapi.new_instruction_sequence -> object + +Return a new, empty InstructionSequence. +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module) +/*[clinic end generated code: output=ea4243fddb9057fd input=1dec2591b173be83]*/ +{ + return _PyInstructionSequence_New(); +} /*[clinic input] @@ -831,6 +846,7 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, } +// Maybe this could be replaced by get_interpreter_config()? static PyObject * get_interp_settings(PyObject *self, PyObject *args) { @@ -960,15 +976,28 @@ iframe_getlasti(PyObject *self, PyObject *frame) } static PyObject * -get_counter_optimizer(PyObject *self, PyObject *arg) +get_co_framesize(PyObject *self, PyObject *arg) { - return PyUnstable_Optimizer_NewCounter(); + if (!PyCode_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "argument must be a code object"); + return NULL; + } + PyCodeObject *code = (PyCodeObject *)arg; + return PyLong_FromLong(code->co_framesize); +} + +#ifdef _Py_TIER2 + +static PyObject * +new_counter_optimizer(PyObject *self, PyObject *arg) +{ + return _PyOptimizer_NewCounter(); } static PyObject * -get_uop_optimizer(PyObject *self, PyObject *arg) +new_uop_optimizer(PyObject *self, PyObject *arg) { - return PyUnstable_Optimizer_NewUOpOptimizer(); + return _PyOptimizer_NewUOpOptimizer(); } static PyObject * @@ -977,40 +1006,25 @@ set_optimizer(PyObject *self, PyObject *opt) if (opt == Py_None) { opt = NULL; } - PyUnstable_SetOptimizer((_PyOptimizerObject*)opt); + if (_Py_SetTier2Optimizer((_PyOptimizerObject*)opt) < 0) { + return NULL; + } Py_RETURN_NONE; } static PyObject * get_optimizer(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *opt = (PyObject *)PyUnstable_GetOptimizer(); + PyObject *opt = NULL; +#ifdef _Py_TIER2 + opt = (PyObject *)_Py_GetOptimizer(); +#endif if (opt == NULL) { Py_RETURN_NONE; } return opt; } -static PyObject * -get_executor(PyObject *self, PyObject *const *args, Py_ssize_t nargs) -{ - - if (!_PyArg_CheckPositional("get_executor", nargs, 2, 2)) { - return NULL; - } - PyObject *code = args[0]; - PyObject *offset = args[1]; - long ioffset = PyLong_AsLong(offset); - if (ioffset == -1 && PyErr_Occurred()) { - return NULL; - } - if (!PyCode_Check(code)) { - PyErr_SetString(PyExc_TypeError, "first argument must be a code object"); - return NULL; - } - return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, ioffset); -} - static PyObject * add_executor_dependency(PyObject *self, PyObject *args) { @@ -1033,10 +1047,12 @@ static PyObject * invalidate_executors(PyObject *self, PyObject *obj) { PyInterpreterState *interp = PyInterpreterState_Get(); - _Py_Executors_InvalidateDependency(interp, obj); + _Py_Executors_InvalidateDependency(interp, obj, 1); Py_RETURN_NONE; } +#endif + static int _pending_callback(void *arg) { /* we assume the argument is callable object to which we own a reference */ @@ -1054,37 +1070,56 @@ static PyObject * pending_threadfunc(PyObject *self, PyObject *args, PyObject *kwargs) { PyObject *callable; + unsigned int num = 1; + int blocking = 0; int ensure_added = 0; - static char *kwlist[] = {"", "ensure_added", NULL}; + static char *kwlist[] = {"callback", "num", + "blocking", "ensure_added", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "O|$p:pending_threadfunc", kwlist, - &callable, &ensure_added)) + "O|I$pp:pending_threadfunc", kwlist, + &callable, &num, &blocking, &ensure_added)) { return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); /* create the reference for the callbackwhile we hold the lock */ - Py_INCREF(callable); + for (unsigned int i = 0; i < num; i++) { + Py_INCREF(callable); + } - int r; - Py_BEGIN_ALLOW_THREADS - r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); - Py_END_ALLOW_THREADS - if (r < 0) { - /* unsuccessful add */ - if (!ensure_added) { - Py_DECREF(callable); - Py_RETURN_FALSE; + PyThreadState *save_tstate = NULL; + if (!blocking) { + save_tstate = PyEval_SaveThread(); + } + + unsigned int num_added = 0; + for (; num_added < num; num_added++) { + if (ensure_added) { + _Py_add_pending_call_result r; + do { + r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); + assert(r == _Py_ADD_PENDING_SUCCESS + || r == _Py_ADD_PENDING_FULL); + } while (r == _Py_ADD_PENDING_FULL); + } + else { + if (_PyEval_AddPendingCall(interp, &_pending_callback, callable, 0) < 0) { + break; + } } - do { - Py_BEGIN_ALLOW_THREADS - r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); - Py_END_ALLOW_THREADS - } while (r < 0); } - Py_RETURN_TRUE; + if (!blocking) { + PyEval_RestoreThread(save_tstate); + } + + for (unsigned int i = num_added; i < num; i++) { + Py_DECREF(callable); /* unsuccessful add, destroy the extra reference */ + } + + /* The callable is decref'ed in _pending_callback() above. */ + return PyLong_FromUnsignedLong((unsigned long)num_added); } @@ -1110,7 +1145,7 @@ pending_identify(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pending_identify", &interpid)) { return NULL; } - PyInterpreterState *interp = PyInterpreterID_LookUp(interpid); + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(interpid); if (interp == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, "interpreter not found"); @@ -1127,14 +1162,16 @@ pending_identify(PyObject *self, PyObject *args) PyThread_acquire_lock(mutex, WAIT_LOCK); /* It gets released in _pending_identify_callback(). */ - int r; + _Py_add_pending_call_result r; do { Py_BEGIN_ALLOW_THREADS r = _PyEval_AddPendingCall(interp, &_pending_identify_callback, (void *)mutex, 0); Py_END_ALLOW_THREADS - } while (r < 0); + assert(r == _Py_ADD_PENDING_SUCCESS + || r == _Py_ADD_PENDING_FULL); + } while (r == _Py_ADD_PENDING_FULL); /* Wait for the pending call to complete. */ PyThread_acquire_lock(mutex, WAIT_LOCK); @@ -1285,8 +1322,8 @@ check_pyobject_forbidden_bytes_is_freed(PyObject *self, static PyObject * check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) { - /* This test would fail if run with the address sanitizer */ -#ifdef _Py_ADDRESS_SANITIZER + /* ASan or TSan would report an use-after-free error */ +#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) Py_RETURN_NONE; #else PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); @@ -1317,14 +1354,13 @@ static PyObject * get_object_dict_values(PyObject *self, PyObject *obj) { PyTypeObject *type = Py_TYPE(obj); - if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) { + if (!_PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) { Py_RETURN_NONE; } - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (!_PyDictOrValues_IsValues(dorv)) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (!values->valid) { Py_RETURN_NONE; } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(keys != NULL); int size = (int)keys->dk_nentries; @@ -1376,115 +1412,376 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } -/* To run some code in a sub-interpreter. */ -static PyObject * -run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) +static int +_init_interp_config_from_object(PyInterpreterConfig *config, PyObject *obj) { - const char *code; - int use_main_obmalloc = -1; - int allow_fork = -1; - int allow_exec = -1; - int allow_threads = -1; - int allow_daemon_threads = -1; - int check_multi_interp_extensions = -1; - int gil = -1; - int r; - PyThreadState *substate, *mainstate; - /* only initialise 'cflags.cf_flags' to test backwards compatibility */ - PyCompilerFlags cflags = {0}; + if (obj == NULL) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + return 0; + } + + PyObject *dict = PyObject_GetAttrString(obj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", obj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + return 0; +} - static char *kwlist[] = {"code", - "use_main_obmalloc", - "allow_fork", - "allow_exec", - "allow_threads", - "allow_daemon_threads", - "check_multi_interp_extensions", - "gil", - NULL}; +static PyInterpreterState * +_new_interpreter(PyInterpreterConfig *config, long whence) +{ + if (whence == _PyInterpreterState_WHENCE_XI) { + return _PyXI_NewInterpreter(config, &whence, NULL, NULL); + } + PyObject *exc = NULL; + PyInterpreterState *interp = NULL; + if (whence == _PyInterpreterState_WHENCE_UNKNOWN) { + assert(config == NULL); + interp = PyInterpreterState_New(); + } + else if (whence == _PyInterpreterState_WHENCE_CAPI + || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) + { + PyThreadState *tstate = NULL; + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + if (whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) { + assert(config == NULL); + tstate = Py_NewInterpreter(); + PyThreadState_Swap(save_tstate); + } + else { + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + assert(tstate == NULL); + _PyErr_SetFromPyStatus(status); + exc = PyErr_GetRaisedException(); + } + } + if (tstate != NULL) { + interp = PyThreadState_GetInterpreter(tstate); + // Throw away the initial tstate. + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + } + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported whence %ld", whence); + return NULL; + } + + if (interp == NULL) { + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + if (exc != NULL) { + _PyErr_ChainExceptions1(exc); + } + } + return interp; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.create() +static PyObject * +create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"config", "whence", NULL}; + PyObject *configobj = NULL; + long whence = _PyInterpreterState_WHENCE_XI; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$ppppppi:run_in_subinterp_with_config", kwlist, - &code, &use_main_obmalloc, - &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions, - &gil)) { + "|O$l:create_interpreter", kwlist, + &configobj, &whence)) + { + return NULL; + } + if (configobj == Py_None) { + configobj = NULL; + } + + // Resolve the config. + PyInterpreterConfig *config = NULL; + PyInterpreterConfig _config; + if (whence == _PyInterpreterState_WHENCE_UNKNOWN + || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) + { + if (configobj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected config"); + return NULL; + } + } + else { + config = &_config; + if (_init_interp_config_from_object(config, configobj) < 0) { + return NULL; + } + } + + // Create the interpreter. + PyInterpreterState *interp = _new_interpreter(config, whence); + if (interp == NULL) { return NULL; } - if (use_main_obmalloc < 0) { - PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc"); + + // Return the ID. + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + _PyXI_EndInterpreter(interp, NULL, NULL); return NULL; } - if (allow_fork < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_fork"); + + return idobj; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.destroy() +static PyObject * +destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O:destroy_interpreter", kwlist, + &idobj)) + { return NULL; } - if (allow_exec < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_exec"); + + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { return NULL; } - if (allow_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_threads"); + + _PyXI_EndInterpreter(interp, NULL, NULL); + Py_RETURN_NONE; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.destroy() +static PyObject * +exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"id", "code", "main", NULL}; + PyObject *idobj; + const char *code; + int runningmain = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "Os|$p:exec_interpreter", kwlist, + &idobj, &code, &runningmain)) + { return NULL; } - if (gil < 0) { - PyErr_SetString(PyExc_ValueError, "missing gil"); + + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { return NULL; } - if (allow_daemon_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); + + PyObject *res = NULL; + PyThreadState *tstate = + _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); + + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + if (runningmain) { + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + goto finally; + } + } + + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + int r = PyRun_SimpleStringFlags(code, &cflags); + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } + + if (runningmain) { + _PyInterpreterState_SetNotRunningMain(interp); + } + + res = PyLong_FromLong(r); + +finally: + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + return res; +} + + +/* To run some code in a sub-interpreter. + +Generally you can use test.support.interpreters, +but we keep this helper as a distinct implementation. +That's especially important for testing test.support.interpreters. +*/ +static PyObject * +run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *code; + PyObject *configobj; + int xi = 0; + static char *kwlist[] = {"code", "config", "xi", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "sO|$p:run_in_subinterp_with_config", kwlist, + &code, &configobj, &xi)) + { return NULL; } - if (check_multi_interp_extensions < 0) { - PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + + PyInterpreterConfig config; + if (_init_interp_config_from_object(&config, configobj) < 0) { return NULL; } - mainstate = PyThreadState_Get(); + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + + int r; + if (xi) { + PyThreadState *save_tstate; + PyThreadState *tstate; + + /* Create an interpreter, staying switched to it. */ + PyInterpreterState *interp = \ + _PyXI_NewInterpreter(&config, NULL, &tstate, &save_tstate); + if (interp == NULL) { + return NULL; + } + + /* Exec the code in the new interpreter. */ + r = PyRun_SimpleStringFlags(code, &cflags); + + /* clean up post-exec. */ + _PyXI_EndInterpreter(interp, tstate, &save_tstate); + } + else { + PyThreadState *substate; + PyThreadState *mainstate = PyThreadState_Swap(NULL); + + /* Create an interpreter, staying switched to it. */ + PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyThreadState_Swap(mainstate); + _PyErr_SetFromPyStatus(status); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + _PyErr_ChainExceptions1(exc); + return NULL; + } - PyThreadState_Swap(NULL); + /* Exec the code in the new interpreter. */ + r = PyRun_SimpleStringFlags(code, &cflags); - const PyInterpreterConfig config = { - .use_main_obmalloc = use_main_obmalloc, - .allow_fork = allow_fork, - .allow_exec = allow_exec, - .allow_threads = allow_threads, - .allow_daemon_threads = allow_daemon_threads, - .check_multi_interp_extensions = check_multi_interp_extensions, - .gil = gil, - }; - PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ + /* clean up post-exec. */ + Py_EndInterpreter(substate); PyThreadState_Swap(mainstate); - _PyErr_SetFromPyStatus(status); - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed"); - _PyErr_ChainExceptions1(exc); + } + + return PyLong_FromLong(r); +} + + +static PyObject * +normalize_interp_id(PyObject *self, PyObject *idobj) +{ + int64_t interpid = _PyInterpreterState_ObjectToID(idobj); + if (interpid < 0) { return NULL; } - assert(substate != NULL); - r = PyRun_SimpleStringFlags(code, &cflags); - Py_EndInterpreter(substate); + return PyLong_FromLongLong(interpid); +} - PyThreadState_Swap(mainstate); +static PyObject * +next_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t interpid = _PyRuntime.interpreters.next_id; + return PyLong_FromLongLong(interpid); +} - return PyLong_FromLong(r); +static PyObject * +unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t interpid = INT64_MAX; + assert(interpid > _PyRuntime.interpreters.next_id); + return PyLong_FromLongLong(interpid); } +static PyObject * +interpreter_exists(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + if (PyErr_ExceptionMatches(PyExc_InterpreterNotFoundError)) { + PyErr_Clear(); + Py_RETURN_FALSE; + } + assert(PyErr_Occurred()); + return NULL; + } + Py_RETURN_TRUE; +} static PyObject * get_interpreter_refcount(PyObject *self, PyObject *idobj) { - PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); if (interp == NULL) { return NULL; } return PyLong_FromLongLong(interp->id_refcount); } +static PyObject * +link_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 1); + Py_RETURN_NONE; +} + +static PyObject * +unlink_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 0); + Py_RETURN_NONE; +} + +static PyObject * +interpreter_refcount_linked(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + if (_PyInterpreterState_RequiresIDRef(interp)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static void _xid_capsule_destructor(PyObject *capsule) @@ -1556,7 +1853,7 @@ _testinternalcapi_test_long_numbits_impl(PyObject *module) { struct triple { long input; - size_t nbits; + uint64_t nbits; int sign; } testcases[] = {{0, 0, 0}, {1L, 1, 1}, @@ -1576,7 +1873,7 @@ _testinternalcapi_test_long_numbits_impl(PyObject *module) size_t i; for (i = 0; i < Py_ARRAY_LENGTH(testcases); ++i) { - size_t nbits; + uint64_t nbits; int sign; PyObject *plong; @@ -1628,13 +1925,6 @@ perf_trampoline_set_persist_after_fork(PyObject *self, PyObject *args) } -static PyObject * -get_type_module_name(PyObject *self, PyObject *type) -{ - assert(PyType_Check(type)); - return _PyType_GetModuleName((PyTypeObject *)type); -} - static PyObject * get_rare_event_counters(PyObject *self, PyObject *type) { @@ -1650,6 +1940,20 @@ get_rare_event_counters(PyObject *self, PyObject *type) ); } +static PyObject * +reset_rare_event_counters(PyObject *self, PyObject *Py_UNUSED(type)) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + + interp->rare_events.set_class = 0; + interp->rare_events.set_bases = 0; + interp->rare_events.set_eval_frame_func = 0; + interp->rare_events.builtin_dict = 0; + interp->rare_events.func_modification = 0; + + return Py_None; +} + #ifdef Py_GIL_DISABLED static PyObject * @@ -1661,6 +1965,63 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } #endif +static PyObject * +has_inline_values(PyObject *self, PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_InlineValues(obj)->valid) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + + +// Circumvents standard version assignment machinery - use with caution and only on +// short-lived heap types +static PyObject * +type_assign_specific_version_unsafe(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + unsigned int version; + if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { + return NULL; + } + assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); + _PyType_SetVersion(type, version); + Py_RETURN_NONE; +} + +/*[clinic input] +gh_119213_getargs + + spam: object = None + +Test _PyArg_Parser.kwtuple +[clinic start generated code]*/ + +static PyObject * +gh_119213_getargs_impl(PyObject *module, PyObject *spam) +/*[clinic end generated code: output=d8d9c95d5b446802 input=65ef47511da80fc2]*/ +{ + // It must never have been called in the main interprer + assert(!_Py_IsMainInterpreter(PyInterpreterState_Get())); + return Py_NewRef(spam); +} + + +static PyObject * +get_static_builtin_types(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _PyStaticType_GetBuiltins(); +} + + +static PyObject * +identify_type_slot_wrappers(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _PyType_GetSlotWrapperNames(); +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, @@ -1682,6 +2043,7 @@ static PyMethodDef module_functions[] = { {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF + _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF @@ -1692,13 +2054,15 @@ static PyMethodDef module_functions[] = { {"iframe_getcode", iframe_getcode, METH_O, NULL}, {"iframe_getline", iframe_getline, METH_O, NULL}, {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, + {"get_co_framesize", get_co_framesize, METH_O, NULL}, +#ifdef _Py_TIER2 {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, - {"get_executor", _PyCFunction_CAST(get_executor), METH_FASTCALL, NULL}, - {"get_counter_optimizer", get_counter_optimizer, METH_NOARGS, NULL}, - {"get_uop_optimizer", get_uop_optimizer, METH_NOARGS, NULL}, + {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, + {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, +#endif {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, @@ -1716,20 +2080,43 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"create_interpreter", _PyCFunction_CAST(create_interpreter), + METH_VARARGS | METH_KEYWORDS}, + {"destroy_interpreter", _PyCFunction_CAST(destroy_interpreter), + METH_VARARGS | METH_KEYWORDS}, + {"exec_interpreter", _PyCFunction_CAST(exec_interpreter), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, + {"normalize_interp_id", normalize_interp_id, METH_O}, + {"next_interpreter_id", next_interpreter_id, METH_NOARGS}, + {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, + {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, + {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, + {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, + {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF - {"get_type_module_name", get_type_module_name, METH_O}, {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, + {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, + {"has_inline_values", has_inline_values, METH_O}, + {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, + PyDoc_STR("forcefully assign type->tp_version_tag")}, + #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif +#ifdef _Py_TIER2 + {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, +#endif + GH_119213_GETARGS_METHODDEF + {"get_static_builtin_types", get_static_builtin_types, METH_NOARGS}, + {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -1777,12 +2164,18 @@ module_exec(PyObject *module) return 1; } + if (PyModule_Add(module, "TIER2_THRESHOLD", + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE)) < 0) { + return 1; + } + return 0; } static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testinternalcapi/pytime.c b/Modules/_testinternalcapi/pytime.c index 2b5f9eb0ef2851..2b0a205d158a96 100644 --- a/Modules/_testinternalcapi/pytime.c +++ b/Modules/_testinternalcapi/pytime.c @@ -2,10 +2,10 @@ #include "parts.h" -#include "pycore_time.h" +#include "pycore_time.h" // _PyTime_FromSeconds() #ifdef MS_WINDOWS -# include // struct timeval +# include // struct timeval #endif @@ -16,8 +16,8 @@ test_pytime_fromseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "i", &seconds)) { return NULL; } - _PyTime_t ts = _PyTime_FromSeconds(seconds); - return _PyTime_AsNanosecondsObject(ts); + PyTime_t ts = _PyTime_FromSeconds(seconds); + return _PyTime_AsLong(ts); } static int @@ -45,26 +45,11 @@ test_pytime_fromsecondsobject(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t ts; + PyTime_t ts; if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { return NULL; } - return _PyTime_AsNanosecondsObject(ts); -} - -static PyObject * -test_pytime_assecondsdouble(PyObject *self, PyObject *args) -{ - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { - return NULL; - } - _PyTime_t ts; - if (_PyTime_FromNanosecondsObject(&ts, obj) < 0) { - return NULL; - } - double d = _PyTime_AsSecondsDouble(ts); - return PyFloat_FromDouble(d); + return _PyTime_AsLong(ts); } static PyObject * @@ -78,8 +63,8 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timeval tv; @@ -105,8 +90,8 @@ test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timeval tv; @@ -127,8 +112,8 @@ test_PyTime_AsTimespec(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timespec ts; @@ -145,8 +130,8 @@ test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timespec ts; @@ -163,16 +148,15 @@ test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t ms = _PyTime_AsMilliseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(ms); - return _PyTime_AsNanosecondsObject(ns); + PyTime_t ms = _PyTime_AsMilliseconds(t, round); + return _PyTime_AsLong(ms); } static PyObject * @@ -183,16 +167,15 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t us = _PyTime_AsMicroseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(us); - return _PyTime_AsNanosecondsObject(ns); + PyTime_t us = _PyTime_AsMicroseconds(t, round); + return _PyTime_AsLong(us); } static PyObject * @@ -254,7 +237,6 @@ test_pytime_object_to_timespec(PyObject *self, PyObject *args) static PyMethodDef TestMethods[] = { {"_PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, {"_PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, - {"_PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, #ifdef HAVE_CLOCK_GETTIME {"_PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS}, {"_PyTime_AsTimespec_clamp", test_PyTime_AsTimespec_clamp, METH_VARARGS}, diff --git a/Modules/_testinternalcapi/set.c b/Modules/_testinternalcapi/set.c index 0305a7885d217c..01aab03cc109ed 100644 --- a/Modules/_testinternalcapi/set.c +++ b/Modules/_testinternalcapi/set.c @@ -1,6 +1,7 @@ #include "parts.h" #include "../_testcapi/util.h" // NULLABLE, RETURN_INT +#include "pycore_critical_section.h" #include "pycore_setobject.h" @@ -27,10 +28,13 @@ set_next_entry(PyObject *self, PyObject *args) return NULL; } NULLABLE(set); - - rc = _PySet_NextEntry(set, &pos, &item, &hash); + Py_BEGIN_CRITICAL_SECTION(set); + rc = _PySet_NextEntryRef(set, &pos, &item, &hash); + Py_END_CRITICAL_SECTION(); if (rc == 1) { - return Py_BuildValue("innO", rc, pos, hash, item); + PyObject *ret = Py_BuildValue("innO", rc, pos, hash, item); + Py_DECREF(item); + return ret; } assert(item == UNINITIALIZED_PTR); assert(hash == (Py_hash_t)UNINITIALIZED_SIZE); diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index 1f7e311558b27c..1df960f9881f70 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -130,6 +130,7 @@ test_critical_sections_suspend(PyObject *self, PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +#ifdef Py_CAN_START_THREADS struct test_data { PyObject *obj1; PyObject *obj2; @@ -170,7 +171,6 @@ thread_critical_sections(void *arg) } } -#ifdef Py_CAN_START_THREADS static PyObject * test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -185,7 +185,7 @@ test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) assert(test_data.obj2 != NULL); assert(test_data.obj3 != NULL); - for (int i = 0; i < NUM_THREADS; i++) { + for (Py_ssize_t i = 0; i < NUM_THREADS; i++) { PyThread_start_new_thread(&thread_critical_sections, &test_data); } PyEvent_Wait(&test_data.done_event); @@ -195,6 +195,90 @@ test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) Py_DECREF(test_data.obj1); Py_RETURN_NONE; } + +static void +pysleep(int ms) +{ +#ifdef MS_WINDOWS + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + +struct test_data_gc { + PyObject *obj; + Py_ssize_t num_threads; + Py_ssize_t id; + Py_ssize_t countdown; + PyEvent done_event; + PyEvent ready; +}; + +static void +thread_gc(void *arg) +{ + struct test_data_gc *test_data = arg; + PyGILState_STATE gil = PyGILState_Ensure(); + + Py_ssize_t id = _Py_atomic_add_ssize(&test_data->id, 1); + if (id == test_data->num_threads - 1) { + _PyEvent_Notify(&test_data->ready); + } + else { + // wait for all test threads to more reliably reproduce the issue. + PyEvent_Wait(&test_data->ready); + } + + if (id == 0) { + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + // pause long enough that the lock would be handed off directly to + // a waiting thread. + pysleep(5); + PyGC_Collect(); + Py_END_CRITICAL_SECTION(); + } + else if (id == 1) { + pysleep(1); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + else if (id == 2) { + // sleep long enough so that thread 0 is waiting to stop the world + pysleep(6); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + + PyGILState_Release(gil); + if (_Py_atomic_add_ssize(&test_data->countdown, -1) == 1) { + // last thread to finish sets done_event + _PyEvent_Notify(&test_data->done_event); + } +} + +static PyObject * +test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // gh-118332: Contended critical sections should not deadlock with GC + const Py_ssize_t NUM_THREADS = 3; + struct test_data_gc test_data = { + .obj = PyDict_New(), + .countdown = NUM_THREADS, + .num_threads = NUM_THREADS, + }; + assert(test_data.obj != NULL); + + for (Py_ssize_t i = 0; i < NUM_THREADS; i++) { + PyThread_start_new_thread(&thread_gc, &test_data); + } + PyEvent_Wait(&test_data.done_event); + Py_DECREF(test_data.obj); + Py_RETURN_NONE; +} + #endif static PyMethodDef test_methods[] = { @@ -203,6 +287,7 @@ static PyMethodDef test_methods[] = { {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, + {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, #endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 83081f73a72f64..8d678412fe7179 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -1,8 +1,9 @@ // C Extension module to test pycore_lock.h API #include "parts.h" - #include "pycore_lock.h" +#include "pycore_pythread.h" // PyThread_get_thread_ident_ex() + #include "clinic/test_lock.c.h" #ifdef MS_WINDOWS @@ -35,9 +36,9 @@ test_lock_basic(PyObject *self, PyObject *obj) // uncontended lock and unlock PyMutex_Lock(&m); - assert(m.v == 1); + assert(m._bits == 1); PyMutex_Unlock(&m); - assert(m.v == 0); + assert(m._bits == 0); Py_RETURN_NONE; } @@ -56,10 +57,10 @@ lock_thread(void *arg) _Py_atomic_store_int(&test_data->started, 1); PyMutex_Lock(m); - assert(m->v == 1); + assert(m->_bits == 1); PyMutex_Unlock(m); - assert(m->v == 0); + assert(m->_bits == 0); _PyEvent_Notify(&test_data->done); } @@ -72,7 +73,7 @@ test_lock_two_threads(PyObject *self, PyObject *obj) memset(&test_data, 0, sizeof(test_data)); PyMutex_Lock(&test_data.m); - assert(test_data.m.v == 1); + assert(test_data.m._bits == 1); PyThread_start_new_thread(lock_thread, &test_data); @@ -81,17 +82,17 @@ test_lock_two_threads(PyObject *self, PyObject *obj) uint8_t v; do { pysleep(10); // allow some time for the other thread to try to lock - v = _Py_atomic_load_uint8_relaxed(&test_data.m.v); + v = _Py_atomic_load_uint8_relaxed(&test_data.m._bits); assert(v == 1 || v == 3); iters++; } while (v != 3 && iters < 200); // both the "locked" and the "has parked" bits should be set - assert(test_data.m.v == 3); + assert(test_data.m._bits == 3); PyMutex_Unlock(&test_data.m); PyEvent_Wait(&test_data.done); - assert(test_data.m.v == 0); + assert(test_data.m._bits == 0); Py_RETURN_NONE; } @@ -289,7 +290,10 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, goto exit; } - _PyTime_t start = _PyTime_GetMonotonicClock(); + PyTime_t start, end; + if (PyTime_PerfCounter(&start) < 0) { + goto exit; + } for (Py_ssize_t i = 0; i < num_threads; i++) { thread_data[i].bench_data = &bench_data; @@ -306,7 +310,9 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, } Py_ssize_t total_iters = bench_data.total_iters; - _PyTime_t end = _PyTime_GetMonotonicClock(); + if (PyTime_PerfCounter(&end) < 0) { + goto exit; + } // Return the total number of acquisitions and the number of acquisitions // for each thread. @@ -318,7 +324,8 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, PyList_SET_ITEM(thread_iters, i, iter); } - double rate = total_iters * 1000000000.0 / (end - start); + assert(end != start); + double rate = total_iters * 1e9 / (end - start); res = Py_BuildValue("(dO)", rate, thread_iters); exit: @@ -470,6 +477,29 @@ test_lock_rwlock(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +static PyObject * +test_lock_recursive(PyObject *self, PyObject *obj) +{ + _PyRecursiveMutex m = (_PyRecursiveMutex){0}; + assert(!_PyRecursiveMutex_IsLockedByCurrentThread(&m)); + + _PyRecursiveMutex_Lock(&m); + assert(m.thread == PyThread_get_thread_ident_ex()); + assert(PyMutex_IsLocked(&m.mutex)); + assert(m.level == 0); + + _PyRecursiveMutex_Lock(&m); + assert(m.level == 1); + _PyRecursiveMutex_Unlock(&m); + + _PyRecursiveMutex_Unlock(&m); + assert(m.thread == 0); + assert(!PyMutex_IsLocked(&m.mutex)); + assert(m.level == 0); + + Py_RETURN_NONE; +} + static PyMethodDef test_methods[] = { {"test_lock_basic", test_lock_basic, METH_NOARGS}, {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS}, @@ -479,6 +509,7 @@ static PyMethodDef test_methods[] = { {"test_lock_benchmark", test_lock_benchmark, METH_NOARGS}, {"test_lock_once", test_lock_once, METH_NOARGS}, {"test_lock_rwlock", test_lock_rwlock, METH_NOARGS}, + {"test_lock_recursive", test_lock_recursive, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c new file mode 100644 index 00000000000000..ba83a23117b2a5 --- /dev/null +++ b/Modules/_testlimitedcapi.c @@ -0,0 +1,87 @@ +/* + * Test the limited C API. + * + * The 'test_*' functions exported by this module are run as part of the + * standard Python regression test, via Lib/test/test_capi.py. + */ + +#include "_testlimitedcapi/parts.h" + +static PyMethodDef TestMethods[] = { + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef _testlimitedcapimodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_testlimitedcapi", + .m_size = 0, + .m_methods = TestMethods, +}; + +PyMODINIT_FUNC +PyInit__testlimitedcapi(void) +{ + PyObject *mod = PyModule_Create(&_testlimitedcapimodule); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif + + if (_PyTestLimitedCAPI_Init_Abstract(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_ByteArray(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Codec(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Complex(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Dict(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Eval(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Float(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_HeaptypeRelative(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_List(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Long(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Object(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_PyOS(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Set(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Sys(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Tuple(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Unicode(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_VectorcallLimited(mod) < 0) { + return NULL; + } + return mod; +} diff --git a/Modules/_testlimitedcapi/abstract.c b/Modules/_testlimitedcapi/abstract.c new file mode 100644 index 00000000000000..6056dd100d6069 --- /dev/null +++ b/Modules/_testlimitedcapi/abstract.c @@ -0,0 +1,582 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +object_repr(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Repr(arg); +} + +static PyObject * +object_ascii(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_ASCII(arg); +} + +static PyObject * +object_str(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Str(arg); +} + +static PyObject * +object_bytes(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Bytes(arg); +} + +static PyObject * +object_getattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyObject_GetAttr(obj, attr_name); +} + +static PyObject * +object_getattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyObject_GetAttrString(obj, attr_name); +} + +static PyObject * +object_hasattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); +} + +static PyObject * +object_hasattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); +} + +static PyObject * +object_setattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + NULLABLE(value); + RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); +} + +static PyObject * +object_setattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); +} + +static PyObject * +object_delattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; +if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + RETURN_INT(PyObject_DelAttr(obj, attr_name)); +} + +static PyObject * +object_delattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + RETURN_INT(PyObject_DelAttrString(obj, attr_name)); +} + +static PyObject * +number_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyBool_FromLong(PyNumber_Check(obj)); +} + +static PyObject * +mapping_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyMapping_Check(obj)); +} + +static PyObject * +mapping_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Size(obj)); +} + +static PyObject * +mapping_length(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Length(obj)); +} + +static PyObject * +object_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyObject_GetItem(mapping, key); +} + +static PyObject * +mapping_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyMapping_GetItemString(mapping, key); +} + +static PyObject * +mapping_haskey(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyLong_FromLong(PyMapping_HasKey(mapping, key)); +} + +static PyObject * +mapping_haskeystring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); +} + +static PyObject * +mapping_haskeywitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_HasKeyWithError(mapping, key)); +} + +static PyObject * +mapping_haskeystringwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_HasKeyStringWithError(mapping, key)); +} + +static PyObject * +object_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyObject_SetItem(mapping, key, value)); +} + +static PyObject * +mapping_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyMapping_SetItemString(mapping, key, value)); +} + +static PyObject * +object_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyObject_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_DelItemString(mapping, key)); +} + +static PyObject * +mapping_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Keys(obj); +} + +static PyObject * +mapping_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Values(obj); +} + +static PyObject * +mapping_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Items(obj); +} + + +static PyObject * +sequence_check(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PySequence_Check(obj)); +} + +static PyObject * +sequence_size(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Size(obj)); +} + +static PyObject * +sequence_length(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Length(obj)); +} + +static PyObject * +sequence_concat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_Concat(seq1, seq2); +} + +static PyObject * +sequence_repeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_Repeat(seq, count); +} + +static PyObject * +sequence_inplaceconcat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_InPlaceConcat(seq1, seq2); +} + +static PyObject * +sequence_inplacerepeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_InPlaceRepeat(seq, count); +} + +static PyObject * +sequence_getitem(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_GetItem(seq, i); +} + +static PyObject * +sequence_setitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq, *val; + if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(val); + + RETURN_INT(PySequence_SetItem(seq, i, val)); +} + + +static PyObject * +sequence_delitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + RETURN_INT(PySequence_DelItem(seq, i)); +} + +static PyObject * +sequence_setslice(PyObject* self, PyObject *args) +{ + PyObject *sequence, *obj; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { + return NULL; + } + NULLABLE(sequence); + NULLABLE(obj); + + RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); +} + +static PyObject * +sequence_delslice(PyObject *self, PyObject *args) +{ + PyObject *sequence; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { + return NULL; + } + NULLABLE(sequence); + + RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); +} + +static PyObject * +sequence_count(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Count(seq, value)); +} + +static PyObject * +sequence_contains(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_INT(PySequence_Contains(seq, value)); +} + +static PyObject * +sequence_index(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Index(seq, value)); +} + +static PyObject * +sequence_list(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_List(obj); +} + +static PyObject * +sequence_tuple(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_Tuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"object_repr", object_repr, METH_O}, + {"object_ascii", object_ascii, METH_O}, + {"object_str", object_str, METH_O}, + {"object_bytes", object_bytes, METH_O}, + + {"object_getattr", object_getattr, METH_VARARGS}, + {"object_getattrstring", object_getattrstring, METH_VARARGS}, + {"object_hasattr", object_hasattr, METH_VARARGS}, + {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, + {"object_setattr", object_setattr, METH_VARARGS}, + {"object_setattrstring", object_setattrstring, METH_VARARGS}, + {"object_delattr", object_delattr, METH_VARARGS}, + {"object_delattrstring", object_delattrstring, METH_VARARGS}, + + {"number_check", number_check, METH_O}, + {"mapping_check", mapping_check, METH_O}, + {"mapping_size", mapping_size, METH_O}, + {"mapping_length", mapping_length, METH_O}, + {"object_getitem", object_getitem, METH_VARARGS}, + {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, + {"mapping_haskey", mapping_haskey, METH_VARARGS}, + {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, + {"mapping_haskeywitherror", mapping_haskeywitherror, METH_VARARGS}, + {"mapping_haskeystringwitherror", mapping_haskeystringwitherror, METH_VARARGS}, + {"object_setitem", object_setitem, METH_VARARGS}, + {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, + {"object_delitem", object_delitem, METH_VARARGS}, + {"mapping_delitem", mapping_delitem, METH_VARARGS}, + {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, + {"mapping_keys", mapping_keys, METH_O}, + {"mapping_values", mapping_values, METH_O}, + {"mapping_items", mapping_items, METH_O}, + + {"sequence_check", sequence_check, METH_O}, + {"sequence_size", sequence_size, METH_O}, + {"sequence_length", sequence_length, METH_O}, + {"sequence_concat", sequence_concat, METH_VARARGS}, + {"sequence_repeat", sequence_repeat, METH_VARARGS}, + {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, + {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, + {"sequence_getitem", sequence_getitem, METH_VARARGS}, + {"sequence_setitem", sequence_setitem, METH_VARARGS}, + {"sequence_delitem", sequence_delitem, METH_VARARGS}, + {"sequence_setslice", sequence_setslice, METH_VARARGS}, + {"sequence_delslice", sequence_delslice, METH_VARARGS}, + {"sequence_count", sequence_count, METH_VARARGS}, + {"sequence_contains", sequence_contains, METH_VARARGS}, + {"sequence_index", sequence_index, METH_VARARGS}, + {"sequence_list", sequence_list, METH_O}, + {"sequence_tuple", sequence_tuple, METH_O}, + + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Abstract(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/bytearray.c b/Modules/_testlimitedcapi/bytearray.c similarity index 98% rename from Modules/_testcapi/bytearray.c rename to Modules/_testlimitedcapi/bytearray.c index dc47ed2c306f40..68b029e25af677 100644 --- a/Modules/_testcapi/bytearray.c +++ b/Modules/_testlimitedcapi/bytearray.c @@ -113,7 +113,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_ByteArray(PyObject *m) +_PyTestLimitedCAPI_Init_ByteArray(PyObject *m) { if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/bytes.c b/Modules/_testlimitedcapi/bytes.c new file mode 100644 index 00000000000000..157d4089954af5 --- /dev/null +++ b/Modules/_testlimitedcapi/bytes.c @@ -0,0 +1,255 @@ +#include "parts.h" +#include "util.h" + + +/* Test PyBytes_Check() */ +static PyObject * +bytes_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyBytes_Check(obj)); +} + +/* Test PyBytes_CheckExact() */ +static PyObject * +bytes_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyBytes_CheckExact(obj)); +} + +/* Test PyBytes_FromStringAndSize() */ +static PyObject * +bytes_fromstringandsize(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyBytes_FromStringAndSize(s, size); +} + +/* Test PyBytes_FromString() */ +static PyObject * +bytes_fromstring(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyBytes_FromString(s); +} + +/* Test PyBytes_FromObject() */ +static PyObject * +bytes_fromobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + return PyBytes_FromObject(arg); +} + +/* Test PyBytes_Size() */ +static PyObject * +bytes_size(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + RETURN_SIZE(PyBytes_Size(arg)); +} + +/* Test PyUnicode_AsString() */ +static PyObject * +bytes_asstring(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + s = PyBytes_AsString(obj); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); +} + +/* Test PyBytes_AsStringAndSize() */ +static PyObject * +bytes_asstringandsize(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + char *s = UNINITIALIZED_PTR; + Py_ssize_t size = UNINITIALIZED_SIZE; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + if (PyBytes_AsStringAndSize(obj, &s, &size) < 0) { + return NULL; + } + + if (s == NULL) { + return Py_BuildValue("(On)", Py_None, size); + } + else { + return Py_BuildValue("(y#n)", s, buflen, size); + } +} + +static PyObject * +bytes_asstringandsize_null(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + char *s = UNINITIALIZED_PTR; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + if (PyBytes_AsStringAndSize(obj, &s, NULL) < 0) { + return NULL; + } + + if (s == NULL) { + Py_RETURN_NONE; + } + else { + return PyBytes_FromStringAndSize(s, buflen); + } +} + +/* Test PyBytes_Repr() */ +static PyObject * +bytes_repr(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + int smartquotes; + if (!PyArg_ParseTuple(args, "Oi", &obj, &smartquotes)) + return NULL; + + NULLABLE(obj); + return PyBytes_Repr(obj, smartquotes); +} + +/* Test PyBytes_Concat() */ +static PyObject * +bytes_concat(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *left, *right; + int new = 0; + + if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (new) { + assert(left != NULL); + assert(PyBytes_CheckExact(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); + if (left == NULL) { + return NULL; + } + } + else { + Py_XINCREF(left); + } + PyBytes_Concat(&left, right); + if (left == NULL && !PyErr_Occurred()) { + Py_RETURN_NONE; + } + return left; +} + +/* Test PyBytes_ConcatAndDel() */ +static PyObject * +bytes_concatanddel(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *left, *right; + int new = 0; + + if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (new) { + assert(left != NULL); + assert(PyBytes_CheckExact(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); + if (left == NULL) { + return NULL; + } + } + else { + Py_XINCREF(left); + } + Py_XINCREF(right); + PyBytes_ConcatAndDel(&left, right); + if (left == NULL && !PyErr_Occurred()) { + Py_RETURN_NONE; + } + return left; +} + +/* Test PyBytes_DecodeEscape() */ +static PyObject * +bytes_decodeescape(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "z#|zn", &s, &bsize, &errors, &size)) + return NULL; + + if (size == -100) { + size = bsize; + } + return PyBytes_DecodeEscape(s, size, errors, 0, NULL); +} + + +static PyMethodDef test_methods[] = { + {"bytes_check", bytes_check, METH_O}, + {"bytes_checkexact", bytes_checkexact, METH_O}, + {"bytes_fromstringandsize", bytes_fromstringandsize, METH_VARARGS}, + {"bytes_fromstring", bytes_fromstring, METH_O}, + {"bytes_fromobject", bytes_fromobject, METH_O}, + {"bytes_size", bytes_size, METH_O}, + {"bytes_asstring", bytes_asstring, METH_VARARGS}, + {"bytes_asstringandsize", bytes_asstringandsize, METH_VARARGS}, + {"bytes_asstringandsize_null", bytes_asstringandsize_null, METH_VARARGS}, + {"bytes_repr", bytes_repr, METH_VARARGS}, + {"bytes_concat", bytes_concat, METH_VARARGS}, + {"bytes_concatanddel", bytes_concatanddel, METH_VARARGS}, + {"bytes_decodeescape", bytes_decodeescape, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Bytes(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/clinic/heaptype_relative.c.h b/Modules/_testlimitedcapi/clinic/heaptype_relative.c.h new file mode 100644 index 00000000000000..994f83102adfb0 --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/heaptype_relative.c.h @@ -0,0 +1,44 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(make_heaptype_with_member__doc__, +"make_heaptype_with_member($module, /, extra_base_size=0, basicsize=0,\n" +" member_offset=0, add_relative_flag=False, *,\n" +" member_name=\'memb\', member_flags=0,\n" +" member_type=-1)\n" +"--\n" +"\n"); + +#define MAKE_HEAPTYPE_WITH_MEMBER_METHODDEF \ + {"make_heaptype_with_member", (PyCFunction)(void(*)(void))make_heaptype_with_member, METH_VARARGS|METH_KEYWORDS, make_heaptype_with_member__doc__}, + +static PyObject * +make_heaptype_with_member_impl(PyObject *module, int extra_base_size, + int basicsize, int member_offset, + int add_relative_flag, + const char *member_name, int member_flags, + int member_type); + +static PyObject * +make_heaptype_with_member(PyObject *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static char *_keywords[] = {"extra_base_size", "basicsize", "member_offset", "add_relative_flag", "member_name", "member_flags", "member_type", NULL}; + int extra_base_size = 0; + int basicsize = 0; + int member_offset = 0; + int add_relative_flag = 0; + const char *member_name = "memb"; + int member_flags = 0; + int member_type = Py_T_BYTE; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiip$sii:make_heaptype_with_member", _keywords, + &extra_base_size, &basicsize, &member_offset, &add_relative_flag, &member_name, &member_flags, &member_type)) + goto exit; + return_value = make_heaptype_with_member_impl(module, extra_base_size, basicsize, member_offset, add_relative_flag, member_name, member_flags, member_type); + +exit: + return return_value; +} +/*[clinic end generated code: output=01933185947faecc input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/clinic/long.c.h b/Modules/_testlimitedcapi/clinic/long.c.h new file mode 100644 index 00000000000000..ebaeb53921a82f --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/long.c.h @@ -0,0 +1,143 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_testlimitedcapi_test_long_api__doc__, +"test_long_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_API_METHODDEF \ + {"test_long_api", (PyCFunction)_testlimitedcapi_test_long_api, METH_NOARGS, _testlimitedcapi_test_long_api__doc__}, + +static PyObject * +_testlimitedcapi_test_long_api_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_api_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_longlong_api__doc__, +"test_longlong_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONGLONG_API_METHODDEF \ + {"test_longlong_api", (PyCFunction)_testlimitedcapi_test_longlong_api, METH_NOARGS, _testlimitedcapi_test_longlong_api__doc__}, + +static PyObject * +_testlimitedcapi_test_longlong_api_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_longlong_api_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_and_overflow__doc__, +"test_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongAndOverflow API.\n" +"\n" +"General conversion to PY_LONG is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTLIMITEDCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_and_overflow", (PyCFunction)_testlimitedcapi_test_long_and_overflow, METH_NOARGS, _testlimitedcapi_test_long_and_overflow__doc__}, + +static PyObject * +_testlimitedcapi_test_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_long_and_overflow__doc__, +"test_long_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongLongAndOverflow API.\n" +"\n" +"General conversion to long long is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTLIMITEDCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_long_and_overflow", (PyCFunction)_testlimitedcapi_test_long_long_and_overflow, METH_NOARGS, _testlimitedcapi_test_long_long_and_overflow__doc__}, + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_size_t__doc__, +"test_long_as_size_t($module, /)\n" +"--\n" +"\n" +"Test the PyLong_As{Size,Ssize}_t API.\n" +"\n" +"At present this just tests that non-integer arguments are handled correctly.\n" +"It should be extended to test overflow handling."); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ + {"test_long_as_size_t", (PyCFunction)_testlimitedcapi_test_long_as_size_t, METH_NOARGS, _testlimitedcapi_test_long_as_size_t__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_size_t_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_size_t_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_unsigned_long_long_mask__doc__, +"test_long_as_unsigned_long_long_mask($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ + {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testlimitedcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testlimitedcapi_test_long_as_unsigned_long_long_mask__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_double__doc__, +"test_long_as_double($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ + {"test_long_as_double", (PyCFunction)_testlimitedcapi_test_long_as_double, METH_NOARGS, _testlimitedcapi_test_long_as_double__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_double_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_double_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_PyLong_AsInt__doc__, +"PyLong_AsInt($module, arg, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF \ + {"PyLong_AsInt", (PyCFunction)_testlimitedcapi_PyLong_AsInt, METH_O, _testlimitedcapi_PyLong_AsInt__doc__}, +/*[clinic end generated code: output=bc52b73c599f96c2 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/clinic/vectorcall_limited.c.h b/Modules/_testlimitedcapi/clinic/vectorcall_limited.c.h new file mode 100644 index 00000000000000..6631a9c42deab0 --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/vectorcall_limited.c.h @@ -0,0 +1,20 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_testlimitedcapi_call_vectorcall__doc__, +"call_vectorcall($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_CALL_VECTORCALL_METHODDEF \ + {"call_vectorcall", (PyCFunction)_testlimitedcapi_call_vectorcall, METH_O, _testlimitedcapi_call_vectorcall__doc__}, + +PyDoc_STRVAR(_testlimitedcapi_call_vectorcall_method__doc__, +"call_vectorcall_method($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ + {"call_vectorcall_method", (PyCFunction)_testlimitedcapi_call_vectorcall_method, METH_O, _testlimitedcapi_call_vectorcall_method__doc__}, +/*[clinic end generated code: output=5976b9b360e1ff30 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/codec.c b/Modules/_testlimitedcapi/codec.c new file mode 100644 index 00000000000000..fdc18eedc2d288 --- /dev/null +++ b/Modules/_testlimitedcapi/codec.c @@ -0,0 +1,29 @@ +#include "pyconfig.h" // Py_GIL_DISABLED + +// Need limited C API version 3.5 for PyCodec_NameReplaceErrors() +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x03050000 +#endif + +#include "parts.h" + +static PyObject * +codec_namereplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_NameReplaceErrors(exc); +} + +static PyMethodDef test_methods[] = { + {"codec_namereplace_errors", codec_namereplace_errors, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Codec(PyObject *module) +{ + if (PyModule_AddFunctions(module, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testlimitedcapi/complex.c b/Modules/_testlimitedcapi/complex.c new file mode 100644 index 00000000000000..e4c244e5c88d06 --- /dev/null +++ b/Modules/_testlimitedcapi/complex.c @@ -0,0 +1,79 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +complex_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyComplex_Check(obj)); +} + +static PyObject * +complex_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyComplex_CheckExact(obj)); +} + +static PyObject * +complex_fromdoubles(PyObject *Py_UNUSED(module), PyObject *args) +{ + double real, imag; + + if (!PyArg_ParseTuple(args, "dd", &real, &imag)) { + return NULL; + } + + return PyComplex_FromDoubles(real, imag); +} + +static PyObject * +complex_realasdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double real; + + NULLABLE(obj); + real = PyComplex_RealAsDouble(obj); + + if (real == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(real); +} + +static PyObject * +complex_imagasdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double imag; + + NULLABLE(obj); + imag = PyComplex_ImagAsDouble(obj); + + if (imag == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(imag); +} + + +static PyMethodDef test_methods[] = { + {"complex_check", complex_check, METH_O}, + {"complex_checkexact", complex_checkexact, METH_O}, + {"complex_fromdoubles", complex_fromdoubles, METH_VARARGS}, + {"complex_realasdouble", complex_realasdouble, METH_O}, + {"complex_imagasdouble", complex_imagasdouble, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Complex(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/dict.c b/Modules/_testlimitedcapi/dict.c new file mode 100644 index 00000000000000..ec32712eef6434 --- /dev/null +++ b/Modules/_testlimitedcapi/dict.c @@ -0,0 +1,291 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +dict_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_Check(obj)); +} + +static PyObject * +dict_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_CheckExact(obj)); +} + +static PyObject * +dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyDict_New(); +} + +static PyObject * +dictproxy_new(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDictProxy_New(obj); +} + +static PyObject * +dict_clear(PyObject *self, PyObject *obj) +{ + PyDict_Clear(obj); + Py_RETURN_NONE; +} + +static PyObject * +dict_copy(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Copy(obj); +} + +static PyObject * +dict_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + RETURN_INT(PyDict_Contains(obj, key)); +} + +static PyObject * +dict_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyDict_Size(obj)); +} + +static PyObject * +dict_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItem(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + PyObject *value = PyDict_GetItemString(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItemWithError(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + + +static PyObject * +dict_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyDict_SetItem(mapping, key, value)); +} + +static PyObject * +dict_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyDict_SetItemString(mapping, key, value)); +} + +static PyObject * +dict_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyDict_DelItem(mapping, key)); +} + +static PyObject * +dict_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyDict_DelItemString(mapping, key)); +} + +static PyObject * +dict_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Keys(obj); +} + +static PyObject * +dict_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Values(obj); +} + +static PyObject * +dict_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Items(obj); +} + +static PyObject * +dict_next(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR; + Py_ssize_t pos; + if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { + return NULL; + } + NULLABLE(mapping); + int rc = PyDict_Next(mapping, &pos, &key, &value); + if (rc != 0) { + return Py_BuildValue("inOO", rc, pos, key, value); + } + assert(key == UNINITIALIZED_PTR); + assert(value == UNINITIALIZED_PTR); + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +dict_merge(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Merge(mapping, mapping2, override)); +} + +static PyObject * +dict_update(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Update(mapping, mapping2)); +} + +static PyObject * +dict_mergefromseq2(PyObject *self, PyObject *args) +{ + PyObject *mapping, *seq; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(seq); + RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); +} + + +static PyMethodDef test_methods[] = { + {"dict_check", dict_check, METH_O}, + {"dict_checkexact", dict_checkexact, METH_O}, + {"dict_new", dict_new, METH_NOARGS}, + {"dictproxy_new", dictproxy_new, METH_O}, + {"dict_clear", dict_clear, METH_O}, + {"dict_copy", dict_copy, METH_O}, + {"dict_size", dict_size, METH_O}, + {"dict_getitem", dict_getitem, METH_VARARGS}, + {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, + {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_contains", dict_contains, METH_VARARGS}, + {"dict_setitem", dict_setitem, METH_VARARGS}, + {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, + {"dict_delitem", dict_delitem, METH_VARARGS}, + {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, + {"dict_keys", dict_keys, METH_O}, + {"dict_values", dict_values, METH_O}, + {"dict_items", dict_items, METH_O}, + {"dict_next", dict_next, METH_VARARGS}, + {"dict_merge", dict_merge, METH_VARARGS}, + {"dict_update", dict_update, METH_VARARGS}, + {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Dict(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/eval.c b/Modules/_testlimitedcapi/eval.c new file mode 100644 index 00000000000000..28f5746dfb1783 --- /dev/null +++ b/Modules/_testlimitedcapi/eval.c @@ -0,0 +1,95 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +eval_get_func_name(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncName(func)); +} + +static PyObject * +eval_get_func_desc(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncDesc(func)); +} + +static PyObject * +eval_getlocals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetLocals()); +} + +static PyObject * +eval_getglobals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetGlobals()); +} + +static PyObject * +eval_getbuiltins(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetBuiltins()); +} + +static PyObject * +eval_getframe(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetFrame()); +} + +static PyObject * +eval_getframe_builtins(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return PyEval_GetFrameBuiltins(); +} + +static PyObject * +eval_getframe_globals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return PyEval_GetFrameGlobals(); +} + +static PyObject * +eval_getframe_locals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return PyEval_GetFrameLocals(); +} + +static PyObject * +eval_get_recursion_limit(PyObject *module, PyObject *Py_UNUSED(args)) +{ + int limit = Py_GetRecursionLimit(); + return PyLong_FromLong(limit); +} + +static PyObject * +eval_set_recursion_limit(PyObject *module, PyObject *args) +{ + int limit; + if (!PyArg_ParseTuple(args, "i", &limit)) { + return NULL; + } + Py_SetRecursionLimit(limit); + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, + {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, + {"eval_getlocals", eval_getlocals, METH_NOARGS}, + {"eval_getglobals", eval_getglobals, METH_NOARGS}, + {"eval_getbuiltins", eval_getbuiltins, METH_NOARGS}, + {"eval_getframe", eval_getframe, METH_NOARGS}, + {"eval_getframe_builtins", eval_getframe_builtins, METH_NOARGS}, + {"eval_getframe_globals", eval_getframe_globals, METH_NOARGS}, + {"eval_getframe_locals", eval_getframe_locals, METH_NOARGS}, + {"eval_get_recursion_limit", eval_get_recursion_limit, METH_NOARGS}, + {"eval_set_recursion_limit", eval_set_recursion_limit, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Eval(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testlimitedcapi/float.c b/Modules/_testlimitedcapi/float.c new file mode 100644 index 00000000000000..88dc91f682ff39 --- /dev/null +++ b/Modules/_testlimitedcapi/float.c @@ -0,0 +1,91 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +float_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyFloat_Check(obj)); +} + +static PyObject * +float_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyFloat_CheckExact(obj)); +} + +static PyObject * +float_fromstring(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyFloat_FromString(obj); +} + +static PyObject * +float_fromdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double d; + + if (!PyArg_Parse(obj, "d", &d)) { + return NULL; + } + + return PyFloat_FromDouble(d); +} + +static PyObject * +float_asdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double d; + + NULLABLE(obj); + d = PyFloat_AsDouble(obj); + if (d == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(d); +} + +static PyObject * +float_getinfo(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_GetInfo(); +} + +static PyObject * +float_getmax(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_FromDouble(PyFloat_GetMax()); +} + +static PyObject * +float_getmin(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_FromDouble(PyFloat_GetMin()); +} + + +static PyMethodDef test_methods[] = { + {"float_check", float_check, METH_O}, + {"float_checkexact", float_checkexact, METH_O}, + {"float_fromstring", float_fromstring, METH_O}, + {"float_fromdouble", float_fromdouble, METH_O}, + {"float_asdouble", float_asdouble, METH_O}, + {"float_getinfo", float_getinfo, METH_NOARGS}, + {"float_getmax", float_getmax, METH_NOARGS}, + {"float_getmin", float_getmin, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Float(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testlimitedcapi/heaptype_relative.c similarity index 65% rename from Modules/_testcapi/heaptype_relative.c rename to Modules/_testlimitedcapi/heaptype_relative.c index 52bda75736b316..45d65ee47349f9 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testlimitedcapi/heaptype_relative.c @@ -1,13 +1,15 @@ +// Need limited C API version 3.12 for PyType_FromMetaclass() #include "pyconfig.h" // Py_GIL_DISABLED - -#ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x030c0000 // 3.12 +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030c0000 #endif #include "parts.h" #include // max_align_t #include // memset +#include "clinic/heaptype_relative.c.h" + static PyType_Slot empty_slots[] = { {0, NULL}, }; @@ -247,6 +249,81 @@ heaptype_with_member_set_memb_relative(PyObject *self, PyObject *value) Py_RETURN_NONE; } +typedef struct { + int padding; // just so the offset isn't 0 + PyObject *dict; +} HeapCTypeWithDictStruct; + +static void +heapctypewithrelativedict_dealloc(PyObject* self) +{ + PyTypeObject *tp = Py_TYPE(self); + HeapCTypeWithDictStruct *data = PyObject_GetTypeData(self, tp); + Py_XDECREF(data->dict); + PyObject_Free(self); + Py_DECREF(tp); +} + +static PyType_Spec HeapCTypeWithRelativeDict_spec = { + .name = "_testcapi.HeapCTypeWithRelativeDict", + .basicsize = -(int)sizeof(HeapCTypeWithDictStruct), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = (PyType_Slot[]) { + {Py_tp_dealloc, heapctypewithrelativedict_dealloc}, + {Py_tp_getset, (PyGetSetDef[]) { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, + {NULL} /* Sentinel */ + }}, + {Py_tp_members, (PyMemberDef[]) { + {"dictobj", _Py_T_OBJECT, + offsetof(HeapCTypeWithDictStruct, dict), + Py_RELATIVE_OFFSET}, + {"__dictoffset__", Py_T_PYSSIZET, + offsetof(HeapCTypeWithDictStruct, dict), + Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL} /* Sentinel */ + }}, + {0, 0}, + } +}; + +typedef struct { + char padding; // just so the offset isn't 0 + PyObject *weakreflist; +} HeapCTypeWithWeakrefStruct; + +static void +heapctypewithrelativeweakref_dealloc(PyObject* self) +{ + PyTypeObject *tp = Py_TYPE(self); + HeapCTypeWithWeakrefStruct *data = PyObject_GetTypeData(self, tp); + if (data->weakreflist != NULL) { + PyObject_ClearWeakRefs(self); + } + Py_XDECREF(data->weakreflist); + PyObject_Free(self); + Py_DECREF(tp); +} + +static PyType_Spec HeapCTypeWithRelativeWeakref_spec = { + .name = "_testcapi.HeapCTypeWithRelativeWeakref", + .basicsize = -(int)sizeof(HeapCTypeWithWeakrefStruct), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = (PyType_Slot[]) { + {Py_tp_dealloc, heapctypewithrelativeweakref_dealloc}, + {Py_tp_members, (PyMemberDef[]) { + {"weakreflist", _Py_T_OBJECT, + offsetof(HeapCTypeWithWeakrefStruct, weakreflist), + Py_RELATIVE_OFFSET}, + {"__weaklistoffset__", Py_T_PYSSIZET, + offsetof(HeapCTypeWithWeakrefStruct, weakreflist), + Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL} /* Sentinel */ + }}, + {0, 0}, + } +}; + static PyMethodDef heaptype_with_member_methods[] = { {"get_memb", heaptype_with_member_get_memb, METH_NOARGS}, {"set_memb", heaptype_with_member_set_memb, METH_O}, @@ -256,19 +333,31 @@ static PyMethodDef heaptype_with_member_methods[] = { {NULL}, }; +/*[clinic input] +make_heaptype_with_member + + extra_base_size: int = 0 + basicsize: int = 0 + member_offset: int = 0 + add_relative_flag: bool = False + * + member_name: str = "memb" + member_flags: int = 0 + member_type: int(c_default="Py_T_BYTE") = -1 + +[clinic start generated code]*/ + static PyObject * -make_heaptype_with_member(PyObject *module, PyObject *args) +make_heaptype_with_member_impl(PyObject *module, int extra_base_size, + int basicsize, int member_offset, + int add_relative_flag, + const char *member_name, int member_flags, + int member_type) +/*[clinic end generated code: output=7005db9a07396997 input=007e29cdbe1d3390]*/ { PyObject *base = NULL; PyObject *result = NULL; - int extra_base_size, basicsize, offset, add_flag; - - int r = PyArg_ParseTuple(args, "iiip", &extra_base_size, &basicsize, &offset, &add_flag); - if (!r) { - goto finally; - } - PyType_Spec base_spec = { .name = "_testcapi.Base", .basicsize = sizeof(PyObject) + extra_base_size, @@ -281,7 +370,8 @@ make_heaptype_with_member(PyObject *module, PyObject *args) } PyMemberDef members[] = { - {"memb", Py_T_BYTE, offset, add_flag ? Py_RELATIVE_OFFSET : 0}, + {member_name, member_type, member_offset, + member_flags | (add_relative_flag ? Py_RELATIVE_OFFSET : 0)}, {0}, }; PyType_Slot slots[] = { @@ -325,13 +415,14 @@ static PyMethodDef TestMethods[] = { {"make_sized_heaptypes", make_sized_heaptypes, METH_VARARGS}, {"subclass_var_heaptype", subclass_var_heaptype, METH_VARARGS}, {"subclass_heaptype", subclass_heaptype, METH_VARARGS}, - {"make_heaptype_with_member", make_heaptype_with_member, METH_VARARGS}, + MAKE_HEAPTYPE_WITH_MEMBER_METHODDEF {"test_alignof_max_align_t", test_alignof_max_align_t, METH_NOARGS}, {NULL}, }; int -_PyTestCapi_Init_HeaptypeRelative(PyObject *m) { +_PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *m) +{ if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } @@ -340,5 +431,42 @@ _PyTestCapi_Init_HeaptypeRelative(PyObject *m) { return -1; } +#define ADD_FROM_SPEC(SPEC) do { \ + PyObject *tp = PyType_FromSpec(SPEC); \ + if (!tp) { \ + return -1; \ + } \ + if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { \ + return -1; \ + } \ + } while (0) + + PyObject *tp; + + tp = PyType_FromSpec(&HeapCTypeWithRelativeDict_spec); + if (!tp) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { + return -1; + } + Py_DECREF(tp); + + tp = PyType_FromSpec(&HeapCTypeWithRelativeWeakref_spec); + if (!tp) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { + return -1; + } + Py_DECREF(tp); + + if (PyModule_AddIntMacro(m, Py_T_PYSSIZET) < 0) { + return -1; + } + if (PyModule_AddIntMacro(m, Py_READONLY) < 0) { + return -1; + } + return 0; } diff --git a/Modules/_testlimitedcapi/list.c b/Modules/_testlimitedcapi/list.c new file mode 100644 index 00000000000000..ed492c3e719727 --- /dev/null +++ b/Modules/_testlimitedcapi/list.c @@ -0,0 +1,175 @@ +// Need limited C API version 3.13 for PyList_GetItemRef() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + +static PyObject * +list_check(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_Check(obj)); +} + +static PyObject * +list_check_exact(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_CheckExact(obj)); +} + +static PyObject * +list_new(PyObject* Py_UNUSED(module), PyObject *obj) +{ + return PyList_New(PyLong_AsSsize_t(obj)); +} + +static PyObject * +list_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyList_Size(obj)); +} + +static PyObject * +list_getitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyList_GetItem(obj, i)); +} + +static PyObject * +list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetItemRef(obj, i); +} + +static PyObject * +list_setitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetItem(obj, i, Py_XNewRef(value))); + +} + +static PyObject * +list_insert(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t where; + if (!PyArg_ParseTuple(args, "OnO", &obj, &where, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Insert(obj, where, Py_XNewRef(value))); + +} + +static PyObject * +list_append(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Append(obj, value)); +} + +static PyObject * +list_getslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetSlice(obj, ilow, ihigh); + +} + +static PyObject * +list_setslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "OnnO", &obj, &ilow, &ihigh, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetSlice(obj, ilow, ihigh, value)); +} + +static PyObject * +list_sort(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Sort(obj)); +} + +static PyObject * +list_reverse(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Reverse(obj)); +} + +static PyObject * +list_astuple(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyList_AsTuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"list_check", list_check, METH_O}, + {"list_check_exact", list_check_exact, METH_O}, + {"list_new", list_new, METH_O}, + {"list_size", list_size, METH_O}, + {"list_getitem", list_getitem, METH_VARARGS}, + {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, + {"list_setitem", list_setitem, METH_VARARGS}, + {"list_insert", list_insert, METH_VARARGS}, + {"list_append", list_append, METH_VARARGS}, + {"list_getslice", list_getslice, METH_VARARGS}, + {"list_setslice", list_setslice, METH_VARARGS}, + {"list_sort", list_sort, METH_O}, + {"list_reverse", list_reverse, METH_O}, + {"list_astuple", list_astuple, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_List(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c new file mode 100644 index 00000000000000..b9c35803b423c2 --- /dev/null +++ b/Modules/_testlimitedcapi/long.c @@ -0,0 +1,848 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.14 to test PyLong_AsInt64() +# define Py_LIMITED_API 0x030e0000 +#endif + +#include "parts.h" +#include "util.h" +#include "clinic/long.c.h" + +/*[clinic input] +module _testlimitedcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2700057f9c1135ba]*/ + + +static PyObject * +raiseTestError(const char* test_name, const char* msg) +{ + PyErr_Format(PyExc_AssertionError, "%s: %s", test_name, msg); + return NULL; +} + +/* Tests of PyLong_{As, From}{Unsigned,}Long(), and + PyLong_{As, From}{Unsigned,}LongLong(). + + Note that the meat of the test is contained in testcapi_long.h. + This is revolting, but delicate code duplication is worse: "almost + exactly the same" code is needed to test long long, but the ubiquitous + dependence on type names makes it impossible to use a parameterized + function. A giant macro would be even worse than this. A C++ template + would be perfect. + + The "report an error" functions are deliberately not part of the #include + file: if the test fails, you can set a breakpoint in the appropriate + error function directly, and crawl back from there in the debugger. +*/ + +#define UNBIND(X) Py_DECREF(X); (X) = NULL + +static PyObject * +raise_test_long_error(const char* msg) +{ + return raiseTestError("test_long_api", msg); +} + +// Test PyLong_FromLong()/PyLong_AsLong() +// and PyLong_FromUnsignedLong()/PyLong_AsUnsignedLong(). + +#define TESTNAME test_long_api_inner +#define TYPENAME long +#define F_S_TO_PY PyLong_FromLong +#define F_PY_TO_S PyLong_AsLong +#define F_U_TO_PY PyLong_FromUnsignedLong +#define F_PY_TO_U PyLong_AsUnsignedLong + +#include "testcapi_long.h" + +/*[clinic input] +_testlimitedcapi.test_long_api +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_api_impl(PyObject *module) +/*[clinic end generated code: output=06a2c02366d1853a input=9012b3d6a483df63]*/ +{ + return TESTNAME(raise_test_long_error); +} + +#undef TESTNAME +#undef TYPENAME +#undef F_S_TO_PY +#undef F_PY_TO_S +#undef F_U_TO_PY +#undef F_PY_TO_U + +// Test PyLong_FromLongLong()/PyLong_AsLongLong() +// and PyLong_FromUnsignedLongLong()/PyLong_AsUnsignedLongLong(). + +static PyObject * +raise_test_longlong_error(const char* msg) +{ + return raiseTestError("test_longlong_api", msg); +} + +#define TESTNAME test_longlong_api_inner +#define TYPENAME long long +#define F_S_TO_PY PyLong_FromLongLong +#define F_PY_TO_S PyLong_AsLongLong +#define F_U_TO_PY PyLong_FromUnsignedLongLong +#define F_PY_TO_U PyLong_AsUnsignedLongLong + +#include "testcapi_long.h" + +/*[clinic input] +_testlimitedcapi.test_longlong_api +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_longlong_api_impl(PyObject *module) +/*[clinic end generated code: output=8faa10e1c35214bf input=2b582a9d25bd68e7]*/ +{ + return TESTNAME(raise_test_longlong_error); +} + +#undef TESTNAME +#undef TYPENAME +#undef F_S_TO_PY +#undef F_PY_TO_S +#undef F_U_TO_PY +#undef F_PY_TO_U + + +/*[clinic input] +_testlimitedcapi.test_long_and_overflow + +Test the PyLong_AsLongAndOverflow API. + +General conversion to PY_LONG is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=fdfd3c1eeabb6d14 input=e3a18791de6519fe]*/ +{ + PyObject *num, *one, *temp; + long value; + int overflow; + + /* Test that overflow is set properly for a large value. */ + /* num is a number larger than LONG_MAX even on 64-bit platforms */ + num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to 1"); + + /* Same again, with num = LONG_MAX + 1 */ + num = PyLong_FromLong(LONG_MAX); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Add(num, one); + Py_DECREF(one); + Py_DECREF(num); + num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to 1"); + + /* Test that overflow is set properly for a large negative value. */ + /* num is a number smaller than LONG_MIN even on 64-bit platforms */ + num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to -1"); + + /* Same again, with num = LONG_MIN - 1 */ + num = PyLong_FromLong(LONG_MIN); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Subtract(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to -1"); + + /* Test that overflow is cleared properly for small values. */ + num = PyLong_FromString("FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != 0xFF) + return raiseTestError("test_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromString("-FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -0xFF) + return raiseTestError("test_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was set incorrectly"); + + num = PyLong_FromLong(LONG_MAX); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LONG_MAX) + return raiseTestError("test_long_and_overflow", + "expected return value LONG_MAX"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromLong(LONG_MIN); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LONG_MIN) + return raiseTestError("test_long_and_overflow", + "expected return value LONG_MIN"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_long_and_overflow + +Test the PyLong_AsLongLongAndOverflow API. + +General conversion to long long is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=3d2721a49c09a307 input=741c593b606cc6b3]*/ +{ + PyObject *num, *one, *temp; + long long value; + int overflow; + + /* Test that overflow is set properly for a large value. */ + /* num is a number larger than LLONG_MAX on a typical machine. */ + num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to 1"); + + /* Same again, with num = LLONG_MAX + 1 */ + num = PyLong_FromLongLong(LLONG_MAX); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Add(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to 1"); + + /* Test that overflow is set properly for a large negative value. */ + /* num is a number smaller than LLONG_MIN on a typical platform */ + num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to -1"); + + /* Same again, with num = LLONG_MIN - 1 */ + num = PyLong_FromLongLong(LLONG_MIN); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Subtract(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to -1"); + + /* Test that overflow is cleared properly for small values. */ + num = PyLong_FromString("FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != 0xFF) + return raiseTestError("test_long_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromString("-FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -0xFF) + return raiseTestError("test_long_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was set incorrectly"); + + num = PyLong_FromLongLong(LLONG_MAX); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LLONG_MAX) + return raiseTestError("test_long_long_and_overflow", + "expected return value LLONG_MAX"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromLongLong(LLONG_MIN); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LLONG_MIN) + return raiseTestError("test_long_long_and_overflow", + "expected return value LLONG_MIN"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_size_t + +Test the PyLong_As{Size,Ssize}_t API. + +At present this just tests that non-integer arguments are handled correctly. +It should be extended to test overflow handling. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_size_t_impl(PyObject *module) +/*[clinic end generated code: output=297a9f14a42f55af input=8923d8f2038c46f4]*/ +{ + size_t out_u; + Py_ssize_t out_s; + + Py_INCREF(Py_None); + + out_u = PyLong_AsSize_t(Py_None); + if (out_u != (size_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + out_s = PyLong_AsSsize_t(Py_None); + if (out_s != (Py_ssize_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_unsigned_long_long_mask +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) +/*[clinic end generated code: output=90be09ffeec8ecab input=17c660bd58becad5]*/ +{ + unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); + + if (res != (unsigned long long)-1 || !PyErr_Occurred()) { + return raiseTestError("test_long_as_unsigned_long_long_mask", + "PyLong_AsUnsignedLongLongMask(NULL) didn't " + "complain"); + } + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + return raiseTestError("test_long_as_unsigned_long_long_mask", + "PyLong_AsUnsignedLongLongMask(NULL) raised " + "something other than SystemError"); + } + PyErr_Clear(); + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_double +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_double_impl(PyObject *module) +/*[clinic end generated code: output=0e688c2acf224f88 input=e7b5712385064a48]*/ +{ + double out; + + Py_INCREF(Py_None); + + out = PyLong_AsDouble(Py_None); + if (out != -1.0 || !PyErr_Occurred()) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + +static PyObject * +pylong_check(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyLong_Check(obj)); +} + +static PyObject * +pylong_checkexact(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyLong_CheckExact(obj)); +} + +static PyObject * +pylong_fromdouble(PyObject *module, PyObject *arg) +{ + double value; + if (!PyArg_Parse(arg, "d", &value)) { + return NULL; + } + return PyLong_FromDouble(value); +} + +static PyObject * +pylong_fromstring(PyObject *module, PyObject *args) +{ + const char *str; + Py_ssize_t len; + int base; + char *end = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "z#i", &str, &len, &base)) { + return NULL; + } + + PyObject *result = PyLong_FromString(str, &end, base); + if (result == NULL) { + // XXX 'end' is not always set. + return NULL; + } + return Py_BuildValue("Nn", result, (Py_ssize_t)(end - str)); +} + +static PyObject * +pylong_fromvoidptr(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + return PyLong_FromVoidPtr((void *)arg); +} + +/*[clinic input] +_testlimitedcapi.PyLong_AsInt + arg: object + / +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_PyLong_AsInt(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=d91db4c1287f85fa input=32c66be86f3265a1]*/ +{ + NULLABLE(arg); + assert(!PyErr_Occurred()); + int value = PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(value); +} + +static PyObject * +pylong_aslong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + long value = PyLong_AsLong(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(value); +} + +static PyObject * +pylong_aslongandoverflow(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int overflow = UNINITIALIZED_INT; + long value = PyLong_AsLongAndOverflow(arg, &overflow); + if (value == -1 && PyErr_Occurred()) { + assert(overflow == -1); + return NULL; + } + return Py_BuildValue("li", value, overflow); +} + +static PyObject * +pylong_asunsignedlong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long value = PyLong_AsUnsignedLong(arg); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLong(value); +} + +static PyObject * +pylong_asunsignedlongmask(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long value = PyLong_AsUnsignedLongMask(arg); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLong(value); +} + +static PyObject * +pylong_aslonglong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + long long value = PyLong_AsLongLong(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLongLong(value); +} + +static PyObject * +pylong_aslonglongandoverflow(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int overflow = UNINITIALIZED_INT; + long long value = PyLong_AsLongLongAndOverflow(arg, &overflow); + if (value == -1 && PyErr_Occurred()) { + assert(overflow == -1); + return NULL; + } + return Py_BuildValue("Li", value, overflow); +} + +static PyObject * +pylong_asunsignedlonglong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long long value = PyLong_AsUnsignedLongLong(arg); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLongLong(value); +} + +static PyObject * +pylong_asunsignedlonglongmask(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long long value = PyLong_AsUnsignedLongLongMask(arg); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLongLong(value); +} + +static PyObject * +pylong_as_ssize_t(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + Py_ssize_t value = PyLong_AsSsize_t(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromSsize_t(value); +} + +static PyObject * +pylong_as_size_t(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + size_t value = PyLong_AsSize_t(arg); + if (value == (size_t)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromSize_t(value); +} + +static PyObject * +pylong_asdouble(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + double value = PyLong_AsDouble(arg); + if (value == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyFloat_FromDouble(value); +} + +static PyObject * +pylong_asvoidptr(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + void *value = PyLong_AsVoidPtr(arg); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef((PyObject *)value); +} + +static PyObject * +pylong_aspid(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + pid_t value = PyLong_AsPid(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromPid(value); +} + + +static PyObject * +pylong_asint32(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int32_t value; + if (PyLong_AsInt32(arg, &value) < 0) { + return NULL; + } + return PyLong_FromInt32(value); +} + +static PyObject * +pylong_asuint32(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + uint32_t value; + if (PyLong_AsUInt32(arg, &value) < 0) { + return NULL; + } + return PyLong_FromUInt32(value); +} + + +static PyObject * +pylong_asint64(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int64_t value; + if (PyLong_AsInt64(arg, &value) < 0) { + return NULL; + } + return PyLong_FromInt64(value); +} + +static PyObject * +pylong_asuint64(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + uint64_t value; + if (PyLong_AsUInt64(arg, &value) < 0) { + return NULL; + } + return PyLong_FromUInt64(value); +} + + +static PyMethodDef test_methods[] = { + _TESTLIMITEDCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_API_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_DOUBLE_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF + _TESTLIMITEDCAPI_TEST_LONGLONG_API_METHODDEF + {"pylong_check", pylong_check, METH_O}, + {"pylong_checkexact", pylong_checkexact, METH_O}, + {"pylong_fromdouble", pylong_fromdouble, METH_O}, + {"pylong_fromstring", pylong_fromstring, METH_VARARGS}, + {"pylong_fromvoidptr", pylong_fromvoidptr, METH_O}, + _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF + {"pylong_aslong", pylong_aslong, METH_O}, + {"pylong_aslongandoverflow", pylong_aslongandoverflow, METH_O}, + {"pylong_asunsignedlong", pylong_asunsignedlong, METH_O}, + {"pylong_asunsignedlongmask", pylong_asunsignedlongmask, METH_O}, + {"pylong_aslonglong", pylong_aslonglong, METH_O}, + {"pylong_aslonglongandoverflow", pylong_aslonglongandoverflow, METH_O}, + {"pylong_asunsignedlonglong", pylong_asunsignedlonglong, METH_O}, + {"pylong_asunsignedlonglongmask", pylong_asunsignedlonglongmask, METH_O}, + {"pylong_as_ssize_t", pylong_as_ssize_t, METH_O}, + {"pylong_as_size_t", pylong_as_size_t, METH_O}, + {"pylong_asdouble", pylong_asdouble, METH_O}, + {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, + {"pylong_aspid", pylong_aspid, METH_O}, + {"pylong_asint32", pylong_asint32, METH_O}, + {"pylong_asuint32", pylong_asuint32, METH_O}, + {"pylong_asint64", pylong_asint64, METH_O}, + {"pylong_asuint64", pylong_asuint64, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Long(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testlimitedcapi/object.c b/Modules/_testlimitedcapi/object.c new file mode 100644 index 00000000000000..da6fe3e4efa34c --- /dev/null +++ b/Modules/_testlimitedcapi/object.c @@ -0,0 +1,80 @@ +// Need limited C API version 3.13 for Py_GetConstant() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + + +/* Test Py_GetConstant() */ +static PyObject * +get_constant(PyObject *Py_UNUSED(module), PyObject *args) +{ + int constant_id; + if (!PyArg_ParseTuple(args, "i", &constant_id)) { + return NULL; + } + + PyObject *obj = Py_GetConstant(constant_id); + if (obj == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return obj; +} + + +/* Test Py_GetConstantBorrowed() */ +static PyObject * +get_constant_borrowed(PyObject *Py_UNUSED(module), PyObject *args) +{ + int constant_id; + if (!PyArg_ParseTuple(args, "i", &constant_id)) { + return NULL; + } + + PyObject *obj = Py_GetConstantBorrowed(constant_id); + if (obj == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return Py_NewRef(obj); +} + + +/* Test constants */ +static PyObject * +test_constants(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // Test that implementation of constants in the limited C API: + // check that the C code compiles. + // + // Test also that constants and Py_GetConstant() return the same + // objects. + assert(Py_None == Py_GetConstant(Py_CONSTANT_NONE)); + assert(Py_False == Py_GetConstant(Py_CONSTANT_FALSE)); + assert(Py_True == Py_GetConstant(Py_CONSTANT_TRUE)); + assert(Py_Ellipsis == Py_GetConstant(Py_CONSTANT_ELLIPSIS)); + assert(Py_NotImplemented == Py_GetConstant(Py_CONSTANT_NOT_IMPLEMENTED)); + // Other constants are tested in test_capi.test_object + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"get_constant", get_constant, METH_VARARGS}, + {"get_constant_borrowed", get_constant_borrowed, METH_VARARGS}, + {"test_constants", test_constants, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Object(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h new file mode 100644 index 00000000000000..4107b150c5b4e0 --- /dev/null +++ b/Modules/_testlimitedcapi/parts.h @@ -0,0 +1,44 @@ +#ifndef Py_TESTLIMITEDCAPI_PARTS_H +#define Py_TESTLIMITEDCAPI_PARTS_H + +// Always enable assertions +#undef NDEBUG + +#include "pyconfig.h" // Py_GIL_DISABLED + +// Use the limited C API +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) + // need limited C API version 3.5 for PyModule_AddFunctions() +# define Py_LIMITED_API 0x03050000 +#endif + +// Make sure that the internal C API cannot be used. +#undef Py_BUILD_CORE_MODULE +#undef Py_BUILD_CORE_BUILTIN + +#include "Python.h" + +#ifdef Py_BUILD_CORE +# error "Py_BUILD_CORE macro must not be defined" +#endif + +int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); +int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); +int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Codec(PyObject *module); +int _PyTestLimitedCAPI_Init_Complex(PyObject *module); +int _PyTestLimitedCAPI_Init_Dict(PyObject *module); +int _PyTestLimitedCAPI_Init_Eval(PyObject *module); +int _PyTestLimitedCAPI_Init_Float(PyObject *module); +int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); +int _PyTestLimitedCAPI_Init_Object(PyObject *module); +int _PyTestLimitedCAPI_Init_List(PyObject *module); +int _PyTestLimitedCAPI_Init_Long(PyObject *module); +int _PyTestLimitedCAPI_Init_PyOS(PyObject *module); +int _PyTestLimitedCAPI_Init_Set(PyObject *module); +int _PyTestLimitedCAPI_Init_Sys(PyObject *module); +int _PyTestLimitedCAPI_Init_Tuple(PyObject *module); +int _PyTestLimitedCAPI_Init_Unicode(PyObject *module); +int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); + +#endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testcapi/pyos.c b/Modules/_testlimitedcapi/pyos.c similarity index 97% rename from Modules/_testcapi/pyos.c rename to Modules/_testlimitedcapi/pyos.c index 63140e914875db..0f61801eae322a 100644 --- a/Modules/_testcapi/pyos.c +++ b/Modules/_testlimitedcapi/pyos.c @@ -50,7 +50,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_PyOS(PyObject *mod) +_PyTestLimitedCAPI_Init_PyOS(PyObject *mod) { if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c new file mode 100644 index 00000000000000..35da5fa5f008e1 --- /dev/null +++ b/Modules/_testlimitedcapi/set.c @@ -0,0 +1,189 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +set_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Check(obj)); +} + +static PyObject * +set_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_CheckExact(obj)); +} + +static PyObject * +frozenset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_Check(obj)); +} + +static PyObject * +frozenset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_CheckExact(obj)); +} + +static PyObject * +anyset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_Check(obj)); +} + +static PyObject * +anyset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_CheckExact(obj)); +} + +static PyObject * +set_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PySet_New(iterable); +} + +static PyObject * +frozenset_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PyFrozenSet_New(iterable); +} + +static PyObject * +set_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySet_Size(obj)); +} + +static PyObject * +set_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Contains(obj, item)); +} + +static PyObject * +set_add(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Add(obj, item)); +} + +static PyObject * +set_discard(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Discard(obj, item)); +} + +static PyObject * +set_pop(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySet_Pop(obj); +} + +static PyObject * +set_clear(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Clear(obj)); +} + +static PyObject * +test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // Test that `frozenset` can be used with `PySet_Add`, + // when frozenset is just created in CAPI. + PyObject *fs = PyFrozenSet_New(NULL); + if (fs == NULL) { + return NULL; + } + PyObject *num = PyLong_FromLong(1); + if (num == NULL) { + goto error; + } + if (PySet_Add(fs, num) < 0) { + goto error; + } + int contains = PySet_Contains(fs, num); + if (contains < 0) { + goto error; + } + else if (contains == 0) { + goto unexpected; + } + Py_DECREF(fs); + Py_DECREF(num); + Py_RETURN_NONE; + +unexpected: + PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); +error: + Py_DECREF(fs); + Py_XDECREF(num); + return NULL; +} + +static PyMethodDef test_methods[] = { + {"set_check", set_check, METH_O}, + {"set_checkexact", set_checkexact, METH_O}, + {"frozenset_check", frozenset_check, METH_O}, + {"frozenset_checkexact", frozenset_checkexact, METH_O}, + {"anyset_check", anyset_check, METH_O}, + {"anyset_checkexact", anyset_checkexact, METH_O}, + + {"set_new", set_new, METH_VARARGS}, + {"frozenset_new", frozenset_new, METH_VARARGS}, + + {"set_size", set_size, METH_O}, + {"set_contains", set_contains, METH_VARARGS}, + {"set_add", set_add, METH_VARARGS}, + {"set_discard", set_discard, METH_VARARGS}, + {"set_pop", set_pop, METH_O}, + {"set_clear", set_clear, METH_O}, + + {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Set(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/sys.c b/Modules/_testlimitedcapi/sys.c similarity index 96% rename from Modules/_testcapi/sys.c rename to Modules/_testlimitedcapi/sys.c index aa40e3cd5b9b29..7d8b7a8569e515 100644 --- a/Modules/_testcapi/sys.c +++ b/Modules/_testlimitedcapi/sys.c @@ -46,7 +46,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_Sys(PyObject *m) +_PyTestLimitedCAPI_Init_Sys(PyObject *m) { if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; diff --git a/Modules/_testcapi/testcapi_long.h b/Modules/_testlimitedcapi/testcapi_long.h similarity index 100% rename from Modules/_testcapi/testcapi_long.h rename to Modules/_testlimitedcapi/testcapi_long.h diff --git a/Modules/_testlimitedcapi/tuple.c b/Modules/_testlimitedcapi/tuple.c new file mode 100644 index 00000000000000..231ec12d517046 --- /dev/null +++ b/Modules/_testlimitedcapi/tuple.c @@ -0,0 +1,136 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +tuple_check(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyTuple_Check(obj)); +} + +static PyObject * +tuple_checkexact(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyTuple_CheckExact(obj)); +} + +static PyObject * +tuple_new(PyObject* Py_UNUSED(module), PyObject *len) +{ + return PyTuple_New(PyLong_AsSsize_t(len)); +} + +static PyObject * +tuple_pack(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *arg1 = NULL, *arg2 = NULL; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "n|OO", &size, &arg1, &arg2)) { + return NULL; + } + if (arg1) { + NULLABLE(arg1); + if (arg2) { + NULLABLE(arg2); + return PyTuple_Pack(size, arg1, arg2); + } + return PyTuple_Pack(size, arg1); + } + return PyTuple_Pack(size); +} + +static PyObject * +tuple_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyTuple_Size(obj)); +} + +static PyObject * +tuple_getitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyTuple_GetItem(obj, i)); +} + +static PyObject * +tuple_getslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { + return NULL; + } + NULLABLE(obj); + return PyTuple_GetSlice(obj, ilow, ihigh); +} + +static PyObject * +tuple_setitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value, *newtuple = NULL; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(value); + if (PyTuple_CheckExact(obj)) { + Py_ssize_t size = PyTuple_Size(obj); + newtuple = PyTuple_New(size); + if (!newtuple) { + return NULL; + } + for (Py_ssize_t n = 0; n < size; n++) { + if (PyTuple_SetItem(newtuple, n, + Py_XNewRef(PyTuple_GetItem(obj, n))) == -1) { + Py_DECREF(newtuple); + return NULL; + } + } + + if (PyTuple_SetItem(newtuple, i, Py_XNewRef(value)) == -1) { + Py_DECREF(newtuple); + return NULL; + } + return newtuple; + } + else { + NULLABLE(obj); + + if (PyTuple_SetItem(obj, i, Py_XNewRef(value)) == -1) { + return NULL; + } + return Py_XNewRef(obj); + } +} + + +static PyMethodDef test_methods[] = { + {"tuple_check", tuple_check, METH_O}, + {"tuple_checkexact", tuple_checkexact, METH_O}, + {"tuple_new", tuple_new, METH_O}, + {"tuple_pack", tuple_pack, METH_VARARGS}, + {"tuple_size", tuple_size, METH_O}, + {"tuple_getitem", tuple_getitem, METH_VARARGS}, + {"tuple_getslice", tuple_getslice, METH_VARARGS}, + {"tuple_setitem", tuple_setitem, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Tuple(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/unicode.c b/Modules/_testlimitedcapi/unicode.c new file mode 100644 index 00000000000000..c7a23d5d1cbd71 --- /dev/null +++ b/Modules/_testlimitedcapi/unicode.c @@ -0,0 +1,1956 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.14 to test PyUnicode_Equal() +# define Py_LIMITED_API 0x030e0000 +#endif + +#include "parts.h" +#include "util.h" + +#include // ptrdiff_t +#include // memset() + + +static PyObject * +codec_incrementalencoder(PyObject *self, PyObject *args) +{ + const char *encoding, *errors = NULL; + if (!PyArg_ParseTuple(args, "s|s:test_incrementalencoder", + &encoding, &errors)) + return NULL; + return PyCodec_IncrementalEncoder(encoding, errors); +} + +static PyObject * +codec_incrementaldecoder(PyObject *self, PyObject *args) +{ + const char *encoding, *errors = NULL; + if (!PyArg_ParseTuple(args, "s|s:test_incrementaldecoder", + &encoding, &errors)) + return NULL; + return PyCodec_IncrementalDecoder(encoding, errors); +} + +static PyObject * +test_unicode_compare_with_ascii(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *py_s = PyUnicode_FromStringAndSize("str\0", 4); + int result; + if (py_s == NULL) + return NULL; + result = PyUnicode_CompareWithASCIIString(py_s, "str"); + Py_DECREF(py_s); + if (!result) { + PyErr_SetString(PyExc_AssertionError, "Python string ending in NULL " + "should not compare equal to c string."); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) + const wchar_t wtext[2] = {(wchar_t)0x10ABCDu}; + size_t wtextlen = 1; + const wchar_t invalid[1] = {(wchar_t)0x110000u}; +#else + const wchar_t wtext[3] = {(wchar_t)0xDBEAu, (wchar_t)0xDFCDu}; + size_t wtextlen = 2; +#endif + PyObject *wide, *utf8; + + wide = PyUnicode_FromWideChar(wtext, wtextlen); + if (wide == NULL) + return NULL; + + utf8 = PyUnicode_FromString("\xf4\x8a\xaf\x8d"); + if (utf8 == NULL) { + Py_DECREF(wide); + return NULL; + } + + if (PyUnicode_GetLength(wide) != PyUnicode_GetLength(utf8)) { + Py_DECREF(wide); + Py_DECREF(utf8); + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "wide string and utf8 string " + "have different length"); + return NULL; + } + if (PyUnicode_Compare(wide, utf8)) { + Py_DECREF(wide); + Py_DECREF(utf8); + if (PyErr_Occurred()) + return NULL; + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "wide string and utf8 string " + "are different"); + return NULL; + } + + Py_DECREF(wide); + Py_DECREF(utf8); + +#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) + wide = PyUnicode_FromWideChar(invalid, 1); + if (wide == NULL) + PyErr_Clear(); + else { + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail"); + return NULL; + } +#endif + Py_RETURN_NONE; +} + + +static PyObject * +unicode_copy(PyObject *unicode) +{ + if (!unicode) { + return NULL; + } + if (!PyUnicode_Check(unicode)) { + Py_INCREF(unicode); + return unicode; + } + + // Create a new string by encoding to UTF-8 and then decode from UTF-8 + PyObject *utf8 = PyUnicode_AsUTF8String(unicode); + if (!utf8) { + return NULL; + } + + PyObject *copy = PyUnicode_DecodeUTF8( + PyBytes_AsString(utf8), + PyBytes_Size(utf8), + NULL); + Py_DECREF(utf8); + + return copy; +} + +/* Test PyUnicode_WriteChar() */ +static PyObject * +unicode_writechar(PyObject *self, PyObject *args) +{ + PyObject *to, *to_copy; + Py_ssize_t index; + unsigned int character; + int result; + + if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + return NULL; + } + + NULLABLE(to); + if (!(to_copy = unicode_copy(to)) && to) { + return NULL; + } + + result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); + if (result == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; + } + return Py_BuildValue("(Ni)", to_copy, result); +} + +static void +unicode_fill(PyObject *str, Py_ssize_t start, Py_ssize_t end, Py_UCS4 ch) +{ + assert(0 <= start); + assert(end <= PyUnicode_GetLength(str)); + for (Py_ssize_t i = start; i < end; i++) { + int res = PyUnicode_WriteChar(str, i, ch); + assert(res == 0); + } +} + + +/* Test PyUnicode_Resize() */ +static PyObject * +unicode_resize(PyObject *self, PyObject *args) +{ + PyObject *obj, *copy; + Py_ssize_t length; + int result; + + if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + return NULL; + } + + NULLABLE(obj); + if (!(copy = unicode_copy(obj)) && obj) { + return NULL; + } + result = PyUnicode_Resize(©, length); + if (result == -1 && PyErr_Occurred()) { + Py_XDECREF(copy); + return NULL; + } + if (obj && PyUnicode_Check(obj) && length > PyUnicode_GetLength(obj)) { + unicode_fill(copy, PyUnicode_GetLength(obj), length, 0U); + } + return Py_BuildValue("(Ni)", copy, result); +} + +/* Test PyUnicode_Append() */ +static PyObject * +unicode_append(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + PyUnicode_Append(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_AppendAndDel() */ +static PyObject * +unicode_appendanddel(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + Py_XINCREF(right); + PyUnicode_AppendAndDel(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_FromStringAndSize() */ +static PyObject * +unicode_fromstringandsize(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyUnicode_FromStringAndSize(s, size); +} + +/* Test PyUnicode_FromString() */ +static PyObject * +unicode_fromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_FromString(s); +} + +/* Test PyUnicode_Substring() */ +static PyObject * +unicode_substring(PyObject *self, PyObject *args) +{ + PyObject *str; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { + return NULL; + } + + NULLABLE(str); + return PyUnicode_Substring(str, start, end); +} + +/* Test PyUnicode_GetLength() */ +static PyObject * +unicode_getlength(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + RETURN_SIZE(PyUnicode_GetLength(arg)); +} + +/* Test PyUnicode_ReadChar() */ +static PyObject * +unicode_readchar(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t index; + Py_UCS4 result; + + if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { + return NULL; + } + + NULLABLE(unicode); + result = PyUnicode_ReadChar(unicode, index); + if (result == (Py_UCS4)-1) + return NULL; + return PyLong_FromUnsignedLong(result); +} + +/* Test PyUnicode_FromEncodedObject() */ +static PyObject * +unicode_fromencodedobject(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &obj, &encoding, &errors)) { + return NULL; + } + + NULLABLE(obj); + return PyUnicode_FromEncodedObject(obj, encoding, errors); +} + +/* Test PyUnicode_FromObject() */ +static PyObject * +unicode_fromobject(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_FromObject(arg); +} + +/* Test PyUnicode_InternInPlace() */ +static PyObject * +unicode_interninplace(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + Py_XINCREF(arg); + PyUnicode_InternInPlace(&arg); + return arg; +} + +/* Test PyUnicode_InternFromString() */ +static PyObject * +unicode_internfromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_InternFromString(s); +} + +/* Test PyUnicode_FromWideChar() */ +static PyObject * +unicode_fromwidechar(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + if (size == -100) { + if (bsize % SIZEOF_WCHAR_T) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromwidechar()"); + return NULL; + } + size = bsize / SIZEOF_WCHAR_T; + } + return PyUnicode_FromWideChar((const wchar_t *)s, size); +} + +/* Test PyUnicode_AsWideChar() */ +static PyObject * +unicode_aswidechar(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + Py_ssize_t buflen, size; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + buffer = PyMem_New(wchar_t, buflen); + if (buffer == NULL) + return PyErr_NoMemory(); + + size = PyUnicode_AsWideChar(unicode, buffer, buflen); + if (size == -1) { + PyMem_Free(buffer); + return NULL; + } + + if (size < buflen) + buflen = size + 1; + else + buflen = size; + result = PyUnicode_FromWideChar(buffer, buflen); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + + return Py_BuildValue("(Nn)", result, size); +} + +/* Test PyUnicode_AsWideCharString() with NULL as buffer */ +static PyObject * +unicode_aswidechar_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + RETURN_SIZE(PyUnicode_AsWideChar(unicode, NULL, buflen)); +} + +/* Test PyUnicode_AsWideCharString() */ +static PyObject * +unicode_aswidecharstring(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + Py_ssize_t size = UNINITIALIZED_SIZE; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, &size); + if (buffer == NULL) { + assert(size == UNINITIALIZED_SIZE); + return NULL; + } + + result = PyUnicode_FromWideChar(buffer, size + 1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return Py_BuildValue("(Nn)", result, size); +} + +/* Test PyUnicode_AsWideCharString() with NULL as the size address */ +static PyObject * +unicode_aswidecharstring_null(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, NULL); + if (buffer == NULL) + return NULL; + + result = PyUnicode_FromWideChar(buffer, -1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return result; +} + + +/* Test PyUnicode_FromOrdinal() */ +static PyObject * +unicode_fromordinal(PyObject *self, PyObject *args) +{ + int ordinal; + + if (!PyArg_ParseTuple(args, "i", &ordinal)) + return NULL; + + return PyUnicode_FromOrdinal(ordinal); +} + +/* Test PyUnicode_AsUTF8AndSize() */ +static PyObject * +unicode_asutf8andsize(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + Py_ssize_t size = UNINITIALIZED_SIZE; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, &size); + if (s == NULL) { + assert(size == -1); + return NULL; + } + + return Py_BuildValue("(y#n)", s, buflen, size); +} + +/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ +static PyObject * +unicode_asutf8andsize_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, NULL); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); +} + +/* Test PyUnicode_GetDefaultEncoding() */ +static PyObject * +unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + const char *s = PyUnicode_GetDefaultEncoding(); + if (s == NULL) + return NULL; + + return PyBytes_FromString(s); +} + +/* Test PyUnicode_Decode() */ +static PyObject * +unicode_decode(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t size; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#z|z", &s, &size, &encoding, &errors)) + return NULL; + + return PyUnicode_Decode(s, size, encoding, errors); +} + +/* Test PyUnicode_AsEncodedString() */ +static PyObject * +unicode_asencodedstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &unicode, &encoding, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_AsEncodedString(unicode, encoding, errors); +} + +/* Test PyUnicode_BuildEncodingMap() */ +static PyObject * +unicode_buildencodingmap(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_BuildEncodingMap(arg); +} + +/* Test PyUnicode_DecodeUTF7() */ +static PyObject * +unicode_decodeutf7(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF7(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF7Stateful() */ +static PyObject * +unicode_decodeutf7stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_DecodeUTF8() */ +static PyObject * +unicode_decodeutf8(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF8(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF8Stateful() */ +static PyObject * +unicode_decodeutf8stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_AsUTF8String() */ +static PyObject * +unicode_asutf8string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF8String(arg); +} + +/* Test PyUnicode_DecodeUTF32() */ +static PyObject * +unicode_decodeutf32(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF32Stateful() */ +static PyObject * +unicode_decodeutf32stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF32String() */ +static PyObject * +unicode_asutf32string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF32String(arg); +} + +/* Test PyUnicode_DecodeUTF16() */ +static PyObject * +unicode_decodeutf16(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF16Stateful() */ +static PyObject * +unicode_decodeutf16stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF16String() */ +static PyObject * +unicode_asutf16string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF16String(arg); +} + +/* Test PyUnicode_DecodeUnicodeEscape() */ +static PyObject * +unicode_decodeunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsUnicodeEscapeString() */ +static PyObject * +unicode_asunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decoderawunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeRawUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsRawUnicodeEscapeString() */ +static PyObject * +unicode_asrawunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsRawUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decodelatin1(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLatin1(data, size, errors); +} + +/* Test PyUnicode_AsLatin1String() */ +static PyObject * +unicode_aslatin1string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsLatin1String(arg); +} + +/* Test PyUnicode_DecodeASCII() */ +static PyObject * +unicode_decodeascii(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeASCII(data, size, errors); +} + +/* Test PyUnicode_AsASCIIString() */ +static PyObject * +unicode_asasciistring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsASCIIString(arg); +} + +/* Test PyUnicode_DecodeCharmap() */ +static PyObject * +unicode_decodecharmap(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + PyObject *mapping; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#O|z", &data, &size, &mapping, &errors)) + return NULL; + + NULLABLE(mapping); + return PyUnicode_DecodeCharmap(data, size, mapping, errors); +} + +/* Test PyUnicode_AsCharmapString() */ +static PyObject * +unicode_ascharmapstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + PyObject *mapping; + + if (!PyArg_ParseTuple(args, "OO", &unicode, &mapping)) + return NULL; + + NULLABLE(unicode); + NULLABLE(mapping); + return PyUnicode_AsCharmapString(unicode, mapping); +} + +#ifdef MS_WINDOWS + +/* Test PyUnicode_DecodeMBCS() */ +static PyObject * +unicode_decodembcs(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeMBCS(data, size, errors); +} + +/* Test PyUnicode_DecodeMBCSStateful() */ +static PyObject * +unicode_decodembcsstateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_DecodeCodePageStateful() */ +static PyObject * +unicode_decodecodepagestateful(PyObject *self, PyObject *args) +{ + int code_page; + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_AsMBCSString() */ +static PyObject * +unicode_asmbcsstring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsMBCSString(arg); +} + +/* Test PyUnicode_EncodeCodePage() */ +static PyObject * +unicode_encodecodepage(PyObject *self, PyObject *args) +{ + int code_page; + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "iO|z", &code_page, &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeCodePage(code_page, unicode, errors); +} + +#endif /* MS_WINDOWS */ + +/* Test PyUnicode_DecodeLocaleAndSize() */ +static PyObject * +unicode_decodelocaleandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocaleAndSize(data, size, errors); +} + +/* Test PyUnicode_DecodeLocale() */ +static PyObject * +unicode_decodelocale(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocale(data, errors); +} + +/* Test PyUnicode_EncodeLocale() */ +static PyObject * +unicode_encodelocale(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "O|z", &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeLocale(unicode, errors); +} + +/* Test PyUnicode_DecodeFSDefault() */ +static PyObject * +unicode_decodefsdefault(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#", &data, &size)) + return NULL; + + return PyUnicode_DecodeFSDefault(data); +} + +/* Test PyUnicode_DecodeFSDefaultAndSize() */ +static PyObject * +unicode_decodefsdefaultandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#|n", &data, &size, &size)) + return NULL; + + return PyUnicode_DecodeFSDefaultAndSize(data, size); +} + +/* Test PyUnicode_EncodeFSDefault() */ +static PyObject * +unicode_encodefsdefault(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_EncodeFSDefault(arg); +} + +/* Test PyUnicode_Concat() */ +static PyObject * +unicode_concat(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_Concat(left, right); +} + +/* Test PyUnicode_Split() */ +static PyObject * +unicode_split(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Split(s, sep, maxsplit); +} + +/* Test PyUnicode_RSplit() */ +static PyObject * +unicode_rsplit(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RSplit(s, sep, maxsplit); +} + +/* Test PyUnicode_Splitlines() */ +static PyObject * +unicode_splitlines(PyObject *self, PyObject *args) +{ + PyObject *s; + int keepends = 0; + + if (!PyArg_ParseTuple(args, "O|i", &s, &keepends)) + return NULL; + + NULLABLE(s); + return PyUnicode_Splitlines(s, keepends); +} + +/* Test PyUnicode_Partition() */ +static PyObject * +unicode_partition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Partition(s, sep); +} + +/* Test PyUnicode_RPartition() */ +static PyObject * +unicode_rpartition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RPartition(s, sep); +} + +/* Test PyUnicode_Translate() */ +static PyObject * +unicode_translate(PyObject *self, PyObject *args) +{ + PyObject *obj; + PyObject *table; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "OO|z", &obj, &table, &errors)) + return NULL; + + NULLABLE(obj); + NULLABLE(table); + return PyUnicode_Translate(obj, table, errors); +} + +/* Test PyUnicode_Join() */ +static PyObject * +unicode_join(PyObject *self, PyObject *args) +{ + PyObject *sep; + PyObject *seq; + + if (!PyArg_ParseTuple(args, "OO", &sep, &seq)) + return NULL; + + NULLABLE(sep); + NULLABLE(seq); + return PyUnicode_Join(sep, seq); +} + +/* Test PyUnicode_Count() */ +static PyObject * +unicode_count(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + + if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + RETURN_SIZE(PyUnicode_Count(str, substr, start, end)); +} + +/* Test PyUnicode_Find() */ +static PyObject * +unicode_find(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + Py_ssize_t result; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + result = PyUnicode_Find(str, substr, start, end, direction); + if (result == -2) { + assert(PyErr_Occurred()); + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_Tailmatch() */ +static PyObject * +unicode_tailmatch(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + RETURN_SIZE(PyUnicode_Tailmatch(str, substr, start, end, direction)); +} + +/* Test PyUnicode_FindChar() */ +static PyObject * +unicode_findchar(PyObject *self, PyObject *args) +{ + PyObject *str; + int direction; + unsigned int ch; + Py_ssize_t result; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, + &start, &end, &direction)) { + return NULL; + } + NULLABLE(str); + result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); + if (result == -2) { + assert(PyErr_Occurred()); + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_Replace() */ +static PyObject * +unicode_replace(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + PyObject *replstr; + Py_ssize_t maxcount = -1; + + if (!PyArg_ParseTuple(args, "OOO|n", &str, &substr, &replstr, &maxcount)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + NULLABLE(replstr); + return PyUnicode_Replace(str, substr, replstr, maxcount); +} + +/* Test PyUnicode_Compare() */ +static PyObject * +unicode_compare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int result; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + result = PyUnicode_Compare(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_CompareWithASCIIString() */ +static PyObject * +unicode_comparewithasciistring(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + int result; + + if (!PyArg_ParseTuple(args, "O|y#", &left, &right, &right_len)) + return NULL; + + NULLABLE(left); + result = PyUnicode_CompareWithASCIIString(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(result); +} + +/* Test PyUnicode_EqualToUTF8() */ +static PyObject * +unicode_equaltoutf8(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + int result; + + if (!PyArg_ParseTuple(args, "Oz#", &left, &right, &right_len)) { + return NULL; + } + + NULLABLE(left); + result = PyUnicode_EqualToUTF8(left, right); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_EqualToUTF8AndSize() */ +static PyObject * +unicode_equaltoutf8andsize(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + Py_ssize_t size = -100; + int result; + + if (!PyArg_ParseTuple(args, "Oz#|n", &left, &right, &right_len, &size)) { + return NULL; + } + + NULLABLE(left); + if (size == -100) { + size = right_len; + } + result = PyUnicode_EqualToUTF8AndSize(left, right, size); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_RichCompare() */ +static PyObject * +unicode_richcompare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int op; + + if (!PyArg_ParseTuple(args, "OOi", &left, &right, &op)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_RichCompare(left, right, op); +} + +/* Test PyUnicode_Format() */ +static PyObject * +unicode_format(PyObject *self, PyObject *args) +{ + PyObject *format; + PyObject *fargs; + + if (!PyArg_ParseTuple(args, "OO", &format, &fargs)) + return NULL; + + NULLABLE(format); + NULLABLE(fargs); + return PyUnicode_Format(format, fargs); +} + +/* Test PyUnicode_Contains() */ +static PyObject * +unicode_contains(PyObject *self, PyObject *args) +{ + PyObject *container; + PyObject *element; + + if (!PyArg_ParseTuple(args, "OO", &container, &element)) + return NULL; + + NULLABLE(container); + NULLABLE(element); + RETURN_INT(PyUnicode_Contains(container, element)); +} + +/* Test PyUnicode_IsIdentifier() */ +static PyObject * +unicode_isidentifier(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + RETURN_INT(PyUnicode_IsIdentifier(arg)); +} + + +static int +check_raised_systemerror(PyObject *result, char* msg) +{ + if (result) { + // no exception + PyErr_Format(PyExc_AssertionError, + "SystemError not raised: %s", + msg); + return 0; + } + if (PyErr_ExceptionMatches(PyExc_SystemError)) { + // expected exception + PyErr_Clear(); + return 1; + } + // unexpected exception + return 0; +} + +static PyObject * +test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *result; + PyObject *unicode = PyUnicode_FromString("None"); + +#define CHECK_FORMAT_2(FORMAT, EXPECTED, ARG1, ARG2) \ + result = PyUnicode_FromFormat(FORMAT, ARG1, ARG2); \ + if (EXPECTED == NULL) { \ + if (!check_raised_systemerror(result, FORMAT)) { \ + goto Fail; \ + } \ + } \ + else if (result == NULL) \ + return NULL; \ + else if (PyUnicode_CompareWithASCIIString(result, EXPECTED) != 0) { \ + PyObject *utf8 = PyUnicode_AsUTF8String(result); \ + PyErr_Format(PyExc_AssertionError, \ + "test_string_from_format: failed at \"%s\" " \ + "expected \"%s\" got \"%s\"", \ + FORMAT, EXPECTED, utf8); \ + Py_XDECREF(utf8); \ + goto Fail; \ + } \ + Py_XDECREF(result) + +#define CHECK_FORMAT_1(FORMAT, EXPECTED, ARG) \ + CHECK_FORMAT_2(FORMAT, EXPECTED, ARG, 0) + +#define CHECK_FORMAT_0(FORMAT, EXPECTED) \ + CHECK_FORMAT_2(FORMAT, EXPECTED, 0, 0) + + // Unrecognized + CHECK_FORMAT_2("%u %? %u", NULL, 1, 2); + + // "%%" (options are rejected) + CHECK_FORMAT_0( "%%", "%"); + CHECK_FORMAT_0( "%0%", NULL); + CHECK_FORMAT_0("%00%", NULL); + CHECK_FORMAT_0( "%2%", NULL); + CHECK_FORMAT_0("%02%", NULL); + CHECK_FORMAT_0("%.0%", NULL); + CHECK_FORMAT_0("%.2%", NULL); + + // "%c" + CHECK_FORMAT_1( "%c", "c", 'c'); + CHECK_FORMAT_1( "%0c", "c", 'c'); + CHECK_FORMAT_1("%00c", "c", 'c'); + CHECK_FORMAT_1( "%2c", NULL, 'c'); + CHECK_FORMAT_1("%02c", NULL, 'c'); + CHECK_FORMAT_1("%.0c", NULL, 'c'); + CHECK_FORMAT_1("%.2c", NULL, 'c'); + + // Integers + CHECK_FORMAT_1("%d", "123", (int)123); + CHECK_FORMAT_1("%i", "123", (int)123); + CHECK_FORMAT_1("%u", "123", (unsigned int)123); + CHECK_FORMAT_1("%x", "7b", (unsigned int)123); + CHECK_FORMAT_1("%X", "7B", (unsigned int)123); + CHECK_FORMAT_1("%o", "173", (unsigned int)123); + CHECK_FORMAT_1("%ld", "123", (long)123); + CHECK_FORMAT_1("%li", "123", (long)123); + CHECK_FORMAT_1("%lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); + CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); + CHECK_FORMAT_1("%lo", "173", (unsigned long)123); + CHECK_FORMAT_1("%lld", "123", (long long)123); + CHECK_FORMAT_1("%lli", "123", (long long)123); + CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); + CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); + CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); + CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%zu", "123", (size_t)123); + CHECK_FORMAT_1("%zx", "7b", (size_t)123); + CHECK_FORMAT_1("%zX", "7B", (size_t)123); + CHECK_FORMAT_1("%zo", "173", (size_t)123); + CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); + CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); + CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); + CHECK_FORMAT_1("%jd", "123", (intmax_t)123); + CHECK_FORMAT_1("%ji", "123", (intmax_t)123); + CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); + CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); + CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); + CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); + + CHECK_FORMAT_1("%d", "-123", (int)-123); + CHECK_FORMAT_1("%i", "-123", (int)-123); + CHECK_FORMAT_1("%ld", "-123", (long)-123); + CHECK_FORMAT_1("%li", "-123", (long)-123); + CHECK_FORMAT_1("%lld", "-123", (long long)-123); + CHECK_FORMAT_1("%lli", "-123", (long long)-123); + CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); + CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); + + // Integers: width < length + CHECK_FORMAT_1("%1d", "123", (int)123); + CHECK_FORMAT_1("%1i", "123", (int)123); + CHECK_FORMAT_1("%1u", "123", (unsigned int)123); + CHECK_FORMAT_1("%1ld", "123", (long)123); + CHECK_FORMAT_1("%1li", "123", (long)123); + CHECK_FORMAT_1("%1lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%1lld", "123", (long long)123); + CHECK_FORMAT_1("%1lli", "123", (long long)123); + CHECK_FORMAT_1("%1llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%1zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%1zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%1zu", "123", (size_t)123); + CHECK_FORMAT_1("%1x", "7b", (int)123); + + CHECK_FORMAT_1("%1d", "-123", (int)-123); + CHECK_FORMAT_1("%1i", "-123", (int)-123); + CHECK_FORMAT_1("%1ld", "-123", (long)-123); + CHECK_FORMAT_1("%1li", "-123", (long)-123); + CHECK_FORMAT_1("%1lld", "-123", (long long)-123); + CHECK_FORMAT_1("%1lli", "-123", (long long)-123); + CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); + + // Integers: width > length + CHECK_FORMAT_1("%5d", " 123", (int)123); + CHECK_FORMAT_1("%5i", " 123", (int)123); + CHECK_FORMAT_1("%5u", " 123", (unsigned int)123); + CHECK_FORMAT_1("%5ld", " 123", (long)123); + CHECK_FORMAT_1("%5li", " 123", (long)123); + CHECK_FORMAT_1("%5lu", " 123", (unsigned long)123); + CHECK_FORMAT_1("%5lld", " 123", (long long)123); + CHECK_FORMAT_1("%5lli", " 123", (long long)123); + CHECK_FORMAT_1("%5llu", " 123", (unsigned long long)123); + CHECK_FORMAT_1("%5zd", " 123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5zi", " 123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5zu", " 123", (size_t)123); + CHECK_FORMAT_1("%5x", " 7b", (int)123); + + CHECK_FORMAT_1("%5d", " -123", (int)-123); + CHECK_FORMAT_1("%5i", " -123", (int)-123); + CHECK_FORMAT_1("%5ld", " -123", (long)-123); + CHECK_FORMAT_1("%5li", " -123", (long)-123); + CHECK_FORMAT_1("%5lld", " -123", (long long)-123); + CHECK_FORMAT_1("%5lli", " -123", (long long)-123); + CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); + + // Integers: width > length, 0-flag + CHECK_FORMAT_1("%05d", "00123", (int)123); + CHECK_FORMAT_1("%05i", "00123", (int)123); + CHECK_FORMAT_1("%05u", "00123", (unsigned int)123); + CHECK_FORMAT_1("%05ld", "00123", (long)123); + CHECK_FORMAT_1("%05li", "00123", (long)123); + CHECK_FORMAT_1("%05lu", "00123", (unsigned long)123); + CHECK_FORMAT_1("%05lld", "00123", (long long)123); + CHECK_FORMAT_1("%05lli", "00123", (long long)123); + CHECK_FORMAT_1("%05llu", "00123", (unsigned long long)123); + CHECK_FORMAT_1("%05zd", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05zi", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05zu", "00123", (size_t)123); + CHECK_FORMAT_1("%05x", "0007b", (int)123); + + CHECK_FORMAT_1("%05d", "-0123", (int)-123); + CHECK_FORMAT_1("%05i", "-0123", (int)-123); + CHECK_FORMAT_1("%05ld", "-0123", (long)-123); + CHECK_FORMAT_1("%05li", "-0123", (long)-123); + CHECK_FORMAT_1("%05lld", "-0123", (long long)-123); + CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); + CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); + + // Integers: precision < length + CHECK_FORMAT_1("%.1d", "123", (int)123); + CHECK_FORMAT_1("%.1i", "123", (int)123); + CHECK_FORMAT_1("%.1u", "123", (unsigned int)123); + CHECK_FORMAT_1("%.1ld", "123", (long)123); + CHECK_FORMAT_1("%.1li", "123", (long)123); + CHECK_FORMAT_1("%.1lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%.1lld", "123", (long long)123); + CHECK_FORMAT_1("%.1lli", "123", (long long)123); + CHECK_FORMAT_1("%.1llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%.1zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.1zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.1zu", "123", (size_t)123); + CHECK_FORMAT_1("%.1x", "7b", (int)123); + + CHECK_FORMAT_1("%.1d", "-123", (int)-123); + CHECK_FORMAT_1("%.1i", "-123", (int)-123); + CHECK_FORMAT_1("%.1ld", "-123", (long)-123); + CHECK_FORMAT_1("%.1li", "-123", (long)-123); + CHECK_FORMAT_1("%.1lld", "-123", (long long)-123); + CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); + CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); + + // Integers: precision > length + CHECK_FORMAT_1("%.5d", "00123", (int)123); + CHECK_FORMAT_1("%.5i", "00123", (int)123); + CHECK_FORMAT_1("%.5u", "00123", (unsigned int)123); + CHECK_FORMAT_1("%.5ld", "00123", (long)123); + CHECK_FORMAT_1("%.5li", "00123", (long)123); + CHECK_FORMAT_1("%.5lu", "00123", (unsigned long)123); + CHECK_FORMAT_1("%.5lld", "00123", (long long)123); + CHECK_FORMAT_1("%.5lli", "00123", (long long)123); + CHECK_FORMAT_1("%.5llu", "00123", (unsigned long long)123); + CHECK_FORMAT_1("%.5zd", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.5zi", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.5zu", "00123", (size_t)123); + CHECK_FORMAT_1("%.5x", "0007b", (int)123); + + CHECK_FORMAT_1("%.5d", "-00123", (int)-123); + CHECK_FORMAT_1("%.5i", "-00123", (int)-123); + CHECK_FORMAT_1("%.5ld", "-00123", (long)-123); + CHECK_FORMAT_1("%.5li", "-00123", (long)-123); + CHECK_FORMAT_1("%.5lld", "-00123", (long long)-123); + CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); + CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); + + // Integers: width > precision > length + CHECK_FORMAT_1("%7.5d", " 00123", (int)123); + CHECK_FORMAT_1("%7.5i", " 00123", (int)123); + CHECK_FORMAT_1("%7.5u", " 00123", (unsigned int)123); + CHECK_FORMAT_1("%7.5ld", " 00123", (long)123); + CHECK_FORMAT_1("%7.5li", " 00123", (long)123); + CHECK_FORMAT_1("%7.5lu", " 00123", (unsigned long)123); + CHECK_FORMAT_1("%7.5lld", " 00123", (long long)123); + CHECK_FORMAT_1("%7.5lli", " 00123", (long long)123); + CHECK_FORMAT_1("%7.5llu", " 00123", (unsigned long long)123); + CHECK_FORMAT_1("%7.5zd", " 00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%7.5zi", " 00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%7.5zu", " 00123", (size_t)123); + CHECK_FORMAT_1("%7.5x", " 0007b", (int)123); + + CHECK_FORMAT_1("%7.5d", " -00123", (int)-123); + CHECK_FORMAT_1("%7.5i", " -00123", (int)-123); + CHECK_FORMAT_1("%7.5ld", " -00123", (long)-123); + CHECK_FORMAT_1("%7.5li", " -00123", (long)-123); + CHECK_FORMAT_1("%7.5lld", " -00123", (long long)-123); + CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); + CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); + + // Integers: width > precision > length, 0-flag + CHECK_FORMAT_1("%07.5d", "0000123", (int)123); + CHECK_FORMAT_1("%07.5i", "0000123", (int)123); + CHECK_FORMAT_1("%07.5u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%07.5ld", "0000123", (long)123); + CHECK_FORMAT_1("%07.5li", "0000123", (long)123); + CHECK_FORMAT_1("%07.5lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%07.5lld", "0000123", (long long)123); + CHECK_FORMAT_1("%07.5lli", "0000123", (long long)123); + CHECK_FORMAT_1("%07.5llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%07.5zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%07.5zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%07.5zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%07.5x", "000007b", (int)123); + + CHECK_FORMAT_1("%07.5d", "-000123", (int)-123); + CHECK_FORMAT_1("%07.5i", "-000123", (int)-123); + CHECK_FORMAT_1("%07.5ld", "-000123", (long)-123); + CHECK_FORMAT_1("%07.5li", "-000123", (long)-123); + CHECK_FORMAT_1("%07.5lld", "-000123", (long long)-123); + CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); + CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); + + // Integers: precision > width > length + CHECK_FORMAT_1("%5.7d", "0000123", (int)123); + CHECK_FORMAT_1("%5.7i", "0000123", (int)123); + CHECK_FORMAT_1("%5.7u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%5.7ld", "0000123", (long)123); + CHECK_FORMAT_1("%5.7li", "0000123", (long)123); + CHECK_FORMAT_1("%5.7lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%5.7lld", "0000123", (long long)123); + CHECK_FORMAT_1("%5.7lli", "0000123", (long long)123); + CHECK_FORMAT_1("%5.7llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%5.7zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5.7zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5.7zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%5.7x", "000007b", (int)123); + + CHECK_FORMAT_1("%5.7d", "-0000123", (int)-123); + CHECK_FORMAT_1("%5.7i", "-0000123", (int)-123); + CHECK_FORMAT_1("%5.7ld", "-0000123", (long)-123); + CHECK_FORMAT_1("%5.7li", "-0000123", (long)-123); + CHECK_FORMAT_1("%5.7lld", "-0000123", (long long)-123); + CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); + CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); + + // Integers: precision > width > length, 0-flag + CHECK_FORMAT_1("%05.7d", "0000123", (int)123); + CHECK_FORMAT_1("%05.7i", "0000123", (int)123); + CHECK_FORMAT_1("%05.7u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%05.7ld", "0000123", (long)123); + CHECK_FORMAT_1("%05.7li", "0000123", (long)123); + CHECK_FORMAT_1("%05.7lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%05.7lld", "0000123", (long long)123); + CHECK_FORMAT_1("%05.7lli", "0000123", (long long)123); + CHECK_FORMAT_1("%05.7llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%05.7zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05.7zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05.7zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%05.7x", "000007b", (int)123); + + CHECK_FORMAT_1("%05.7d", "-0000123", (int)-123); + CHECK_FORMAT_1("%05.7i", "-0000123", (int)-123); + CHECK_FORMAT_1("%05.7ld", "-0000123", (long)-123); + CHECK_FORMAT_1("%05.7li", "-0000123", (long)-123); + CHECK_FORMAT_1("%05.7lld", "-0000123", (long long)-123); + CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); + CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); + + // Integers: precision = 0, arg = 0 (empty string in C) + CHECK_FORMAT_1("%.0d", "0", (int)0); + CHECK_FORMAT_1("%.0i", "0", (int)0); + CHECK_FORMAT_1("%.0u", "0", (unsigned int)0); + CHECK_FORMAT_1("%.0ld", "0", (long)0); + CHECK_FORMAT_1("%.0li", "0", (long)0); + CHECK_FORMAT_1("%.0lu", "0", (unsigned long)0); + CHECK_FORMAT_1("%.0lld", "0", (long long)0); + CHECK_FORMAT_1("%.0lli", "0", (long long)0); + CHECK_FORMAT_1("%.0llu", "0", (unsigned long long)0); + CHECK_FORMAT_1("%.0zd", "0", (Py_ssize_t)0); + CHECK_FORMAT_1("%.0zi", "0", (Py_ssize_t)0); + CHECK_FORMAT_1("%.0zu", "0", (size_t)0); + CHECK_FORMAT_1("%.0x", "0", (int)0); + + // Strings + CHECK_FORMAT_1("%s", "None", "None"); + CHECK_FORMAT_1("%ls", "None", L"None"); + CHECK_FORMAT_1("%U", "None", unicode); + CHECK_FORMAT_1("%A", "None", Py_None); + CHECK_FORMAT_1("%S", "None", Py_None); + CHECK_FORMAT_1("%R", "None", Py_None); + CHECK_FORMAT_2("%V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%V", "None", NULL, "None"); + CHECK_FORMAT_2("%lV", "None", NULL, L"None"); + + // Strings: width < length + CHECK_FORMAT_1("%1s", "None", "None"); + CHECK_FORMAT_1("%1ls", "None", L"None"); + CHECK_FORMAT_1("%1U", "None", unicode); + CHECK_FORMAT_1("%1A", "None", Py_None); + CHECK_FORMAT_1("%1S", "None", Py_None); + CHECK_FORMAT_1("%1R", "None", Py_None); + CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%1V", "None", NULL, "None"); + CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); + + // Strings: width > length + CHECK_FORMAT_1("%5s", " None", "None"); + CHECK_FORMAT_1("%5ls", " None", L"None"); + CHECK_FORMAT_1("%5U", " None", unicode); + CHECK_FORMAT_1("%5A", " None", Py_None); + CHECK_FORMAT_1("%5S", " None", Py_None); + CHECK_FORMAT_1("%5R", " None", Py_None); + CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); + CHECK_FORMAT_2("%5V", " None", NULL, "None"); + CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); + + // Strings: precision < length + CHECK_FORMAT_1("%.1s", "N", "None"); + CHECK_FORMAT_1("%.1ls", "N", L"None"); + CHECK_FORMAT_1("%.1U", "N", unicode); + CHECK_FORMAT_1("%.1A", "N", Py_None); + CHECK_FORMAT_1("%.1S", "N", Py_None); + CHECK_FORMAT_1("%.1R", "N", Py_None); + CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); + CHECK_FORMAT_2("%.1V", "N", NULL, "None"); + CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); + + // Strings: precision > length + CHECK_FORMAT_1("%.5s", "None", "None"); + CHECK_FORMAT_1("%.5ls", "None", L"None"); + CHECK_FORMAT_1("%.5U", "None", unicode); + CHECK_FORMAT_1("%.5A", "None", Py_None); + CHECK_FORMAT_1("%.5S", "None", Py_None); + CHECK_FORMAT_1("%.5R", "None", Py_None); + CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); + + // Strings: precision < length, width > length + CHECK_FORMAT_1("%5.1s", " N", "None"); + CHECK_FORMAT_1("%5.1ls"," N", L"None"); + CHECK_FORMAT_1("%5.1U", " N", unicode); + CHECK_FORMAT_1("%5.1A", " N", Py_None); + CHECK_FORMAT_1("%5.1S", " N", Py_None); + CHECK_FORMAT_1("%5.1R", " N", Py_None); + CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); + CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); + CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); + + // Strings: width < length, precision > length + CHECK_FORMAT_1("%1.5s", "None", "None"); + CHECK_FORMAT_1("%1.5ls", "None", L"None"); + CHECK_FORMAT_1("%1.5U", "None", unicode); + CHECK_FORMAT_1("%1.5A", "None", Py_None); + CHECK_FORMAT_1("%1.5S", "None", Py_None); + CHECK_FORMAT_1("%1.5R", "None", Py_None); + CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); + + Py_XDECREF(unicode); + Py_RETURN_NONE; + + Fail: + Py_XDECREF(result); + Py_XDECREF(unicode); + return NULL; + +#undef CHECK_FORMAT_2 +#undef CHECK_FORMAT_1 +#undef CHECK_FORMAT_0 +} + + +/* Test PyUnicode_Equal() */ +static PyObject * +unicode_equal(PyObject *module, PyObject *args) +{ + PyObject *str1, *str2; + if (!PyArg_ParseTuple(args, "OO", &str1, &str2)) { + return NULL; + } + + NULLABLE(str1); + NULLABLE(str2); + RETURN_INT(PyUnicode_Equal(str1, str2)); +} + + + +static PyMethodDef TestMethods[] = { + {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS}, + {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS}, + {"test_unicode_compare_with_ascii", + test_unicode_compare_with_ascii, METH_NOARGS}, + {"test_string_from_format", test_string_from_format, METH_NOARGS}, + {"test_widechar", test_widechar, METH_NOARGS}, + {"unicode_writechar", unicode_writechar, METH_VARARGS}, + {"unicode_resize", unicode_resize, METH_VARARGS}, + {"unicode_append", unicode_append, METH_VARARGS}, + {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, + {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, + {"unicode_fromstring", unicode_fromstring, METH_O}, + {"unicode_substring", unicode_substring, METH_VARARGS}, + {"unicode_getlength", unicode_getlength, METH_O}, + {"unicode_readchar", unicode_readchar, METH_VARARGS}, + {"unicode_fromencodedobject",unicode_fromencodedobject, METH_VARARGS}, + {"unicode_fromobject", unicode_fromobject, METH_O}, + {"unicode_interninplace", unicode_interninplace, METH_O}, + {"unicode_internfromstring", unicode_internfromstring, METH_O}, + {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, + {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, + {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, + {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, + {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, + {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, + {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, + {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, + {"unicode_decode", unicode_decode, METH_VARARGS}, + {"unicode_asencodedstring", unicode_asencodedstring, METH_VARARGS}, + {"unicode_buildencodingmap", unicode_buildencodingmap, METH_O}, + {"unicode_decodeutf7", unicode_decodeutf7, METH_VARARGS}, + {"unicode_decodeutf7stateful",unicode_decodeutf7stateful, METH_VARARGS}, + {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, + {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, + {"unicode_asutf8string", unicode_asutf8string, METH_O}, + {"unicode_decodeutf16", unicode_decodeutf16, METH_VARARGS}, + {"unicode_decodeutf16stateful",unicode_decodeutf16stateful, METH_VARARGS}, + {"unicode_asutf16string", unicode_asutf16string, METH_O}, + {"unicode_decodeutf32", unicode_decodeutf32, METH_VARARGS}, + {"unicode_decodeutf32stateful",unicode_decodeutf32stateful, METH_VARARGS}, + {"unicode_asutf32string", unicode_asutf32string, METH_O}, + {"unicode_decodeunicodeescape",unicode_decodeunicodeescape, METH_VARARGS}, + {"unicode_asunicodeescapestring",unicode_asunicodeescapestring,METH_O}, + {"unicode_decoderawunicodeescape",unicode_decoderawunicodeescape,METH_VARARGS}, + {"unicode_asrawunicodeescapestring",unicode_asrawunicodeescapestring,METH_O}, + {"unicode_decodelatin1", unicode_decodelatin1, METH_VARARGS}, + {"unicode_aslatin1string", unicode_aslatin1string, METH_O}, + {"unicode_decodeascii", unicode_decodeascii, METH_VARARGS}, + {"unicode_asasciistring", unicode_asasciistring, METH_O}, + {"unicode_decodecharmap", unicode_decodecharmap, METH_VARARGS}, + {"unicode_ascharmapstring", unicode_ascharmapstring, METH_VARARGS}, +#ifdef MS_WINDOWS + {"unicode_decodembcs", unicode_decodembcs, METH_VARARGS}, + {"unicode_decodembcsstateful",unicode_decodembcsstateful, METH_VARARGS}, + {"unicode_decodecodepagestateful",unicode_decodecodepagestateful,METH_VARARGS}, + {"unicode_asmbcsstring", unicode_asmbcsstring, METH_O}, + {"unicode_encodecodepage", unicode_encodecodepage, METH_VARARGS}, +#endif /* MS_WINDOWS */ + {"unicode_decodelocaleandsize",unicode_decodelocaleandsize, METH_VARARGS}, + {"unicode_decodelocale", unicode_decodelocale, METH_VARARGS}, + {"unicode_encodelocale", unicode_encodelocale, METH_VARARGS}, + {"unicode_decodefsdefault", unicode_decodefsdefault, METH_VARARGS}, + {"unicode_decodefsdefaultandsize",unicode_decodefsdefaultandsize,METH_VARARGS}, + {"unicode_encodefsdefault", unicode_encodefsdefault, METH_O}, + {"unicode_concat", unicode_concat, METH_VARARGS}, + {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, + {"unicode_split", unicode_split, METH_VARARGS}, + {"unicode_rsplit", unicode_rsplit, METH_VARARGS}, + {"unicode_partition", unicode_partition, METH_VARARGS}, + {"unicode_rpartition", unicode_rpartition, METH_VARARGS}, + {"unicode_translate", unicode_translate, METH_VARARGS}, + {"unicode_join", unicode_join, METH_VARARGS}, + {"unicode_count", unicode_count, METH_VARARGS}, + {"unicode_tailmatch", unicode_tailmatch, METH_VARARGS}, + {"unicode_find", unicode_find, METH_VARARGS}, + {"unicode_findchar", unicode_findchar, METH_VARARGS}, + {"unicode_replace", unicode_replace, METH_VARARGS}, + {"unicode_compare", unicode_compare, METH_VARARGS}, + {"unicode_comparewithasciistring",unicode_comparewithasciistring,METH_VARARGS}, + {"unicode_equaltoutf8", unicode_equaltoutf8, METH_VARARGS}, + {"unicode_equaltoutf8andsize",unicode_equaltoutf8andsize, METH_VARARGS}, + {"unicode_richcompare", unicode_richcompare, METH_VARARGS}, + {"unicode_format", unicode_format, METH_VARARGS}, + {"unicode_contains", unicode_contains, METH_VARARGS}, + {"unicode_isidentifier", unicode_isidentifier, METH_O}, + {"unicode_equal", unicode_equal, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Unicode(PyObject *m) +{ + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/util.h b/Modules/_testlimitedcapi/util.h new file mode 100644 index 00000000000000..f26d7656a10138 --- /dev/null +++ b/Modules/_testlimitedcapi/util.h @@ -0,0 +1,33 @@ +#define NULLABLE(x) do { \ + if (x == Py_None) { \ + x = NULL; \ + } \ + } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + +/* Marker to check that pointer value was set. */ +static const char uninitialized[] = "uninitialized"; +#define UNINITIALIZED_PTR ((void *)uninitialized) +/* Marker to check that Py_ssize_t value was set. */ +#define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) +/* Marker to check that integer value was set. */ +#define UNINITIALIZED_INT (63256717) diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testlimitedcapi/vectorcall_limited.c similarity index 62% rename from Modules/_testcapi/vectorcall_limited.c rename to Modules/_testlimitedcapi/vectorcall_limited.c index d7070d37bb9e9b..4a7af965776470 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testlimitedcapi/vectorcall_limited.c @@ -1,18 +1,20 @@ /* Test Vectorcall in the limited API */ +// Need limited C API version 3.12 for PyObject_Vectorcall() #include "pyconfig.h" // Py_GIL_DISABLED - -#ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x030c0000 // 3.12 +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030c0000 #endif +#include // offsetof + #include "parts.h" #include "clinic/vectorcall_limited.c.h" /*[clinic input] -module _testcapi +module _testlimitedcapi [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2700057f9c1135ba]*/ static PyObject * LimitedVectorCallClass_tpcall(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -40,15 +42,15 @@ LimitedVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) } /*[clinic input] -_testcapi.call_vectorcall +_testlimitedcapi.call_vectorcall callable: object / [clinic start generated code]*/ static PyObject * -_testcapi_call_vectorcall(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=bae81eec97fcaad7 input=55d88f92240957ee]*/ +_testlimitedcapi_call_vectorcall(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=9cbb7832263a8eef input=0743636c12dccb28]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *kwname = NULL, *kwnames = NULL, *result = NULL; @@ -93,15 +95,15 @@ _testcapi_call_vectorcall(PyObject *module, PyObject *callable) } /*[clinic input] -_testcapi.call_vectorcall_method +_testlimitedcapi.call_vectorcall_method callable: object / [clinic start generated code]*/ static PyObject * -_testcapi_call_vectorcall_method(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=e661f48dda08b6fb input=5ba81c27511395b6]*/ +_testlimitedcapi_call_vectorcall_method(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=4558323a46cc09eb input=a736f7dbf15f1be5]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *name = NULL, *kwname = NULL, @@ -167,7 +169,7 @@ static PyType_Slot LimitedVectorallClass_slots[] = { }; static PyType_Spec LimitedVectorCallClass_spec = { - .name = "_testcapi.LimitedVectorCallClass", + .name = "_testlimitedcapi.LimitedVectorCallClass", .basicsize = (int)(sizeof(PyObject) + sizeof(vectorcallfunc)), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_VECTORCALL @@ -175,14 +177,50 @@ static PyType_Spec LimitedVectorCallClass_spec = { .slots = LimitedVectorallClass_slots, }; +typedef struct { + vectorcallfunc vfunc; +} LimitedRelativeVectorCallStruct; + +static PyObject * +LimitedRelativeVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) +{ + PyObject *self = ((allocfunc)PyType_GetSlot(tp, Py_tp_alloc))(tp, 0); + if (!self) { + return NULL; + } + LimitedRelativeVectorCallStruct *data = PyObject_GetTypeData(self, tp); + data->vfunc = LimitedVectorCallClass_vectorcall; + return self; +} + + +static PyType_Spec LimitedRelativeVectorCallClass_spec = { + .name = "_testlimitedcapi.LimitedRelativeVectorCallClass", + .basicsize = -(int)sizeof(LimitedRelativeVectorCallStruct), + .flags = Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_HAVE_VECTORCALL, + .slots = (PyType_Slot[]) { + {Py_tp_new, LimitedRelativeVectorCallClass_new}, + {Py_tp_call, LimitedVectorCallClass_tpcall}, + {Py_tp_members, (PyMemberDef[]){ + {"__vectorcalloffset__", Py_T_PYSSIZET, + offsetof(LimitedRelativeVectorCallStruct, vfunc), + Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL} + }}, + {0} + }, +}; + static PyMethodDef TestMethods[] = { - _TESTCAPI_CALL_VECTORCALL_METHODDEF - _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF + _TESTLIMITEDCAPI_CALL_VECTORCALL_METHODDEF + _TESTLIMITEDCAPI_CALL_VECTORCALL_METHOD_METHODDEF {NULL}, }; int -_PyTestCapi_Init_VectorcallLimited(PyObject *m) { +_PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *m) +{ if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } @@ -196,5 +234,16 @@ _PyTestCapi_Init_VectorcallLimited(PyObject *m) { return -1; } Py_DECREF(LimitedVectorCallClass); + + PyObject *LimitedRelativeVectorCallClass = PyType_FromModuleAndSpec( + m, &LimitedRelativeVectorCallClass_spec, NULL); + if (!LimitedRelativeVectorCallClass) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)LimitedRelativeVectorCallClass) < 0) { + return -1; + } + Py_DECREF(LimitedRelativeVectorCallClass); + return 0; } diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 21c5f696a4f2ec..886b260aceb20d 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -431,6 +431,7 @@ static int execfunc(PyObject *m) static PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -519,13 +520,18 @@ PyInit__testmultiphase_nonmodule_with_methods(void) /**** Non-ASCII-named modules ****/ +static PyModuleDef_Slot nonascii_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef def_nonascii_latin = { \ PyModuleDef_HEAD_INIT, /* m_base */ "_testmultiphase_nonascii_latin", /* m_name */ PyDoc_STR("Module named in Czech"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -543,7 +549,7 @@ static PyModuleDef def_nonascii_kana = { \ PyDoc_STR("Module named in Japanese"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -757,6 +763,7 @@ static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = { {Py_mod_create, createfunc_nonmodule}, {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -778,6 +785,7 @@ execfunc_err(PyObject *mod) static PyModuleDef_Slot slots_exec_err[] = { {Py_mod_exec, execfunc_err}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -800,6 +808,7 @@ execfunc_raise(PyObject *spec) static PyModuleDef_Slot slots_exec_raise[] = { {Py_mod_exec, execfunc_raise}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -822,6 +831,7 @@ execfunc_unreported_exception(PyObject *mod) static PyModuleDef_Slot slots_exec_unreported_exception[] = { {Py_mod_exec, execfunc_unreported_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -857,6 +867,7 @@ meth_state_access_exec(PyObject *m) static PyModuleDef_Slot meth_state_access_slots[] = { {Py_mod_exec, meth_state_access_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -889,6 +900,9 @@ PyInit__test_module_state_shared(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddObjectRef(module, "Error", PyExc_Exception) < 0) { Py_DECREF(module); @@ -903,6 +917,7 @@ PyInit__test_module_state_shared(void) static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -920,6 +935,7 @@ PyInit__testmultiphase_multiple_multiple_interpreters_slots(void) static PyModuleDef_Slot non_isolated_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -940,6 +956,7 @@ static PyModuleDef_Slot shared_gil_only_slots[] = { We put it here explicitly to draw attention to the contrast with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. */ {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index c42a15a0eff494..2c59085d15b5be 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -1,6 +1,207 @@ /* Testing module for single-phase initialization of extension modules - */ + +This file contains several distinct modules, meaning each as its own name +and its own init function (PyInit_...). The default import system will +only find the one matching the filename: _testsinglephase. To load the +others you must do so manually. For example: + +```python +name = '_testsinglephase_base_wrapper' +filename = _testsinglephase.__file__ +loader = importlib.machinery.ExtensionFileLoader(name, filename) +spec = importlib.util.spec_from_file_location(name, filename, loader=loader) +mod = importlib._bootstrap._load(spec) +loader.exec_module(module) +sys.modules[modname] = module +``` + +(The last two lines are just for completeness.) + +Here are the modules: + +* _testsinglephase + * def: _testsinglephase_basic, + * m_name: "_testsinglephase" + * m_size: -1 + * state + * process-global + * initialized_count (default to -1; will never be 0) + * module (see module state below) + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. clear .module + 3. initialize .module: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions + * (3 common, see below) + * initialized_count() - return .module.initialized_count + * import system + * caches + * global extensions cache: yes + * def.m_base.m_copy: yes + * def.m_base.m_init: no + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + 1. get init function from shared object file + 2. run init function + 3. copy __dict__ into def.m_base.m_copy + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. get module from sys.modules + 3. update module with contents of def.m_base.m_copy + * already loaded in other interpreter (already in global cache) + * same as reload, but create new module and update *it* + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * same as main interpreter + * unload: never (all single-phase init modules) +* _testsinglephase_basic_wrapper + * identical to _testsinglephase except module name +* _testsinglephase_basic_copy + * def: static local variable in init function + * m_name: "_testsinglephase_basic_copy" + * m_size: -1 + * state: same as _testsinglephase + * init function: same as _testsinglephase + * functions: same as _testsinglephase + * import system: same as _testsinglephase +* _testsinglephase_with_reinit + * def: _testsinglephase_with_reinit, + * m_name: "_testsinglephase_with_reinit" + * m_size: 0 + * state + * process-global state: no + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. initialize temporary module state (local var): see module state below + 3. initialize module: set initial __dict__ + * functions: see common functions below + * import system + * caches + * global extensions cache: only if loaded in main interpreter + * def.m_base.m_copy: no + * def.m_base.m_init: only if loaded in the main interpreter + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + * (same as _testsinglephase except step 3) + 1. get init function from shared object file + 2. run init function + 3. set def.m_base.m_init to the init function + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. call def->m_base.m_init to get a new module object + 3. replace the existing module in sys.modules + * already loaded in other interpreter (already in global cache) + * same as reload (since will only be in cache for main interp) + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * initial (not already in global cache) + * (same as main interpreter except skip steps 3 & 4 there) + 1. get init function from shared object file + 2. run init function + ... + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + * same as initial (load from scratch) + * already loaded in other interpreter (already in global cache) + * same as initial (load from scratch) + * not in any sys.modules, still in global cache + * same as initial (load from scratch) + * unload: never (all single-phase init modules) +* _testsinglephase_with_state + * def: _testsinglephase_with_state, + * m_name: "_testsinglephase_with_state" + * m_size: sizeof(module_state) + * state + * process-global: no + * module state: see module state below + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 3. initialize module state: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions: see common functions below + * import system: same as _testsinglephase_basic_copy +* _testsinglephase_check_cache_first + * def: _testsinglepahse_check_cache_first + * m_name: "_testsinglephase_check_cache_first" + * m_size: -1 + * state: none + * init function: + * tries PyState_FindModule() first + * otherwise creates empty module + * functions: none + * import system: same as _testsinglephase +* _testsinglephase_with_reinit_check_cache_first + * def: _testsinglepahse_with_reinit_check_cache_first + * m_name: "_testsinglephase_with_reinit_check_cache_first" + * m_size: 0 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_reinit +* _testsinglephase_with_state_check_cache_first + * def: _testsinglepahse_with_state_check_cache_first + * m_name: "_testsinglephase_with_state_check_cache_first" + * m_size: 42 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_state + +* _testsinglephase_circular + Regression test for gh-123880. + Does not have the common attributes & methods. + See test_singlephase_circular test.test_import.SinglephaseInitTests. + +Module state: + +* fields + * initialized - when the module was first initialized + * *error + * *int_const + * *str_const +* initialization + 1. set state.initialized to the current time + 2. set state.error to a new exception class + 3. set state->int_const to int(1969) + 4. set state->str_const to "something different" + +Common initial __dict__: + +* error: state.error +* int_const: state.int_const +* str_const: state.str_const +* _module_initialized: state.initialized + +Common functions: + +* look_up_self() - return the module from the per-interpreter "by-index" cache +* sum() - return a + b +* state_initialized() - return state->initialized (or None if m_size == 0) + +See Python/import.c, especially the long comments, for more about +single-phase init modules. +*/ + #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif @@ -8,11 +209,10 @@ //#include #include "Python.h" #include "pycore_namespace.h" // _PyNamespace_New() -#include "pycore_time.h" // _PyTime_t typedef struct { - _PyTime_t initialized; + PyTime_t initialized; PyObject *error; PyObject *int_const; PyObject *str_const; @@ -67,17 +267,17 @@ clear_state(module_state *state) } static int -_set_initialized(_PyTime_t *initialized) +_set_initialized(PyTime_t *initialized) { /* We go strictly monotonic to ensure each time is unique. */ - _PyTime_t prev; - if (_PyTime_GetMonotonicClockWithInfo(&prev, NULL) != 0) { + PyTime_t prev; + if (PyTime_Monotonic(&prev) != 0) { return -1; } /* We do a busy sleep since the interval should be super short. */ - _PyTime_t t; + PyTime_t t; do { - if (_PyTime_GetMonotonicClockWithInfo(&t, NULL) != 0) { + if (PyTime_Monotonic(&t) != 0) { return -1; } } while (t == prev); @@ -136,7 +336,7 @@ init_module(PyObject *module, module_state *state) return -1; } - double d = _PyTime_AsSecondsDouble(state->initialized); + double d = PyTime_AsSecondsDouble(state->initialized); if (PyModule_Add(module, "_module_initialized", PyFloat_FromDouble(d)) < 0) { return -1; } @@ -157,7 +357,7 @@ common_state_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) if (state == NULL) { Py_RETURN_NONE; } - double d = _PyTime_AsSecondsDouble(state->initialized); + double d = PyTime_AsSecondsDouble(state->initialized); return PyFloat_FromDouble(d); } @@ -306,6 +506,9 @@ init__testsinglephase_basic(PyModuleDef *def) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = &global_state.module; // It may have been set by a previous run or under a different name. @@ -397,6 +600,9 @@ PyInit__testsinglephase_with_reinit(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif assert(get_module_state(module) == NULL); @@ -459,6 +665,9 @@ PyInit__testsinglephase_with_state(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = get_module_state(module); assert(state != NULL); @@ -476,3 +685,117 @@ PyInit__testsinglephase_with_state(void) finally: return module; } + + +/****************************************************/ +/* the _testsinglephase_*_check_cache_first modules */ +/****************************************************/ + +/* Each of these modules should only be freshly loaded. That means + clearing the caches and each module def's m_base after each load. */ + +static struct PyModuleDef _testsinglephase_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_check_cache_first"), + .m_size = -1, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_check_cache_first(void) +{ + assert(_testsinglephase_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_reinit_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_reinit_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_reinit_check_cache_first"), + .m_size = 0, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_reinit_check_cache_first(void) +{ + assert(_testsinglephase_with_reinit_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_reinit_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_reinit_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_state_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_state_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_state_check_cache_first"), + .m_size = 42, // not used +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_state_check_cache_first(void) +{ + assert(_testsinglephase_with_state_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_state_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_state_check_cache_first); +} + + +/****************************************/ +/* the _testsinglephase_circular module */ +/****************************************/ + +static PyObject *static_module_circular; + +static PyObject * +circularmod_clear_static_var(PyObject *self, PyObject *arg) +{ + PyObject *result = static_module_circular; + static_module_circular = NULL; + return result; +} + +static struct PyModuleDef _testsinglephase_circular = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_circular", + .m_doc = PyDoc_STR("Test module _testsinglephase_circular"), + .m_methods = (PyMethodDef[]) { + {"clear_static_var", circularmod_clear_static_var, METH_NOARGS, + "Clear the static variable and return its previous value."}, + {NULL, NULL} /* sentinel */ + } +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_circular(void) +{ + if (!static_module_circular) { + static_module_circular = PyModule_Create(&_testsinglephase_circular); + if (!static_module_circular) { + return NULL; + } + } + static const char helper_mod_name[] = ( + "test.test_import.data.circular_imports.singlephase"); + PyObject *helper_mod = PyImport_ImportModule(helper_mod_name); + Py_XDECREF(helper_mod); + if (!helper_mod) { + return NULL; + } + if(PyModule_AddStringConstant(static_module_circular, + "helper_mod_name", + helper_mod_name) < 0) { + return NULL; + } + return Py_NewRef(static_module_circular); +} diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index df02b023012fbd..d4408aa9e42d9d 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1,14 +1,15 @@ - /* Thread module */ /* Interface to Sjoerd's portable C thread library */ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState.threads.count +#include "pycore_lock.h" #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_time.h" // _PyTime_FromSeconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include // offsetof() @@ -19,7 +20,6 @@ // ThreadError is just an alias to PyExc_RuntimeError #define ThreadError PyExc_RuntimeError - // Forward declarations static struct PyModuleDef thread_module; @@ -30,6 +30,10 @@ typedef struct { PyTypeObject *local_type; PyTypeObject *local_dummy_type; PyTypeObject *thread_handle_type; + + // Linked list of handles to all non-daemon threads created by the + // threading module. We wait for these to finish at shutdown. + struct llist_node shutdown_handles; } thread_module_state; static inline thread_module_state* @@ -42,24 +46,148 @@ get_thread_state(PyObject *module) // _ThreadHandle type +// Handles state transitions according to the following diagram: +// +// NOT_STARTED -> STARTING -> RUNNING -> DONE +// | ^ +// | | +// +----- error --------+ +typedef enum { + THREAD_HANDLE_NOT_STARTED = 1, + THREAD_HANDLE_STARTING = 2, + THREAD_HANDLE_RUNNING = 3, + THREAD_HANDLE_DONE = 4, +} ThreadHandleState; + +// A handle to wait for thread completion. +// +// This may be used to wait for threads that were spawned by the threading +// module as well as for the "main" thread of the threading module. In the +// former case an OS thread, identified by the `os_handle` field, will be +// associated with the handle. The handle "owns" this thread and ensures that +// the thread is either joined or detached after the handle is destroyed. +// +// Joining the handle is idempotent; the underlying OS thread, if any, is +// joined or detached only once. Concurrent join operations are serialized +// until it is their turn to execute or an earlier operation completes +// successfully. Once a join has completed successfully all future joins +// complete immediately. +// +// This must be separately reference counted because it may be destroyed +// in `thread_run()` after the PyThreadState has been destroyed. typedef struct { - PyObject_HEAD struct llist_node node; // linked list node (see _pythread_runtime_state) + + // linked list node (see thread_module_state) + struct llist_node shutdown_node; + + // The `ident`, `os_handle`, `has_os_handle`, and `state` fields are + // protected by `mutex`. PyThread_ident_t ident; - PyThread_handle_t handle; - char joinable; -} ThreadHandleObject; + PyThread_handle_t os_handle; + int has_os_handle; + + // Holds a value from the `ThreadHandleState` enum. + int state; + + PyMutex mutex; + + // Set immediately before `thread_run` returns to indicate that the OS + // thread is about to exit. This is used to avoid false positives when + // detecting self-join attempts. See the comment in `ThreadHandle_join()` + // for a more detailed explanation. + PyEvent thread_is_exiting; + + // Serializes calls to `join` and `set_done`. + _PyOnceFlag once; + + Py_ssize_t refcount; +} ThreadHandle; + +static inline int +get_thread_handle_state(ThreadHandle *handle) +{ + PyMutex_Lock(&handle->mutex); + int state = handle->state; + PyMutex_Unlock(&handle->mutex); + return state; +} + +static inline void +set_thread_handle_state(ThreadHandle *handle, ThreadHandleState state) +{ + PyMutex_Lock(&handle->mutex); + handle->state = state; + PyMutex_Unlock(&handle->mutex); +} + +static PyThread_ident_t +ThreadHandle_ident(ThreadHandle *handle) +{ + PyMutex_Lock(&handle->mutex); + PyThread_ident_t ident = handle->ident; + PyMutex_Unlock(&handle->mutex); + return ident; +} -static ThreadHandleObject* -new_thread_handle(thread_module_state* state) +static int +ThreadHandle_get_os_handle(ThreadHandle *handle, PyThread_handle_t *os_handle) { - ThreadHandleObject* self = PyObject_New(ThreadHandleObject, state->thread_handle_type); + PyMutex_Lock(&handle->mutex); + int has_os_handle = handle->has_os_handle; + if (has_os_handle) { + *os_handle = handle->os_handle; + } + PyMutex_Unlock(&handle->mutex); + return has_os_handle; +} + +static void +add_to_shutdown_handles(thread_module_state *state, ThreadHandle *handle) +{ + HEAD_LOCK(&_PyRuntime); + llist_insert_tail(&state->shutdown_handles, &handle->shutdown_node); + HEAD_UNLOCK(&_PyRuntime); +} + +static void +clear_shutdown_handles(thread_module_state *state) +{ + HEAD_LOCK(&_PyRuntime); + struct llist_node *node; + llist_for_each_safe(node, &state->shutdown_handles) { + llist_remove(node); + } + HEAD_UNLOCK(&_PyRuntime); +} + +static void +remove_from_shutdown_handles(ThreadHandle *handle) +{ + HEAD_LOCK(&_PyRuntime); + if (handle->shutdown_node.next != NULL) { + llist_remove(&handle->shutdown_node); + } + HEAD_UNLOCK(&_PyRuntime); +} + +static ThreadHandle * +ThreadHandle_new(void) +{ + ThreadHandle *self = + (ThreadHandle *)PyMem_RawCalloc(1, sizeof(ThreadHandle)); if (self == NULL) { + PyErr_NoMemory(); return NULL; } self->ident = 0; - self->handle = 0; - self->joinable = 0; + self->os_handle = 0; + self->has_os_handle = 0; + self->thread_is_exiting = (PyEvent){0}; + self->mutex = (PyMutex){_Py_UNLOCKED}; + self->once = (_PyOnceFlag){0}; + self->state = THREAD_HANDLE_NOT_STARTED; + self->refcount = 1; HEAD_LOCK(&_PyRuntime); llist_insert_tail(&_PyRuntime.threads.handles, &self->node); @@ -69,9 +197,34 @@ new_thread_handle(thread_module_state* state) } static void -ThreadHandle_dealloc(ThreadHandleObject *self) +ThreadHandle_incref(ThreadHandle *self) +{ + _Py_atomic_add_ssize(&self->refcount, 1); +} + +static int +detach_thread(ThreadHandle *self) +{ + if (!self->has_os_handle) { + return 0; + } + // This is typically short so no need to release the GIL + if (PyThread_detach_thread(self->os_handle)) { + fprintf(stderr, "detach_thread: failed detaching thread\n"); + return -1; + } + return 0; +} + +// NB: This may be called after the PyThreadState in `thread_run` has been +// deleted; it cannot call anything that relies on a valid PyThreadState +// existing. +static void +ThreadHandle_decref(ThreadHandle *self) { - PyObject *tp = (PyObject *) Py_TYPE(self); + if (_Py_atomic_add_ssize(&self->refcount, -1) > 1) { + return; + } // Remove ourself from the global list of handles HEAD_LOCK(&_PyRuntime); @@ -80,15 +233,17 @@ ThreadHandle_dealloc(ThreadHandleObject *self) } HEAD_UNLOCK(&_PyRuntime); - if (self->joinable) { - int ret = PyThread_detach_thread(self->handle); - if (ret) { - PyErr_SetString(ThreadError, "Failed detaching thread"); - PyErr_WriteUnraisable(tp); - } + assert(self->shutdown_node.next == NULL); + + // It's safe to access state non-atomically: + // 1. This is the destructor; nothing else holds a reference. + // 2. The refcount going to zero is a "synchronizes-with" event; all + // changes from other threads are visible. + if (self->state == THREAD_HANDLE_RUNNING && !detach_thread(self)) { + self->state = THREAD_HANDLE_DONE; } - PyObject_Free(self); - Py_DECREF(tp); + + PyMem_RawFree(self); } void @@ -102,100 +257,447 @@ _PyThread_AfterFork(struct _pythread_runtime_state *state) struct llist_node *node; llist_for_each_safe(node, &state->handles) { - ThreadHandleObject *hobj = llist_data(node, ThreadHandleObject, node); - if (hobj->ident == current) { + ThreadHandle *handle = llist_data(node, ThreadHandle, node); + if (handle->ident == current) { continue; } - // Disallow calls to detach() and join() as they could crash. - hobj->joinable = 0; + // Mark all threads as done. Any attempts to join or detach the + // underlying OS thread (if any) could crash. We are the only thread; + // it's safe to set this non-atomically. + handle->state = THREAD_HANDLE_DONE; + handle->once = (_PyOnceFlag){_Py_ONCE_INITIALIZED}; + handle->mutex = (PyMutex){_Py_UNLOCKED}; + _PyEvent_Notify(&handle->thread_is_exiting); llist_remove(node); + remove_from_shutdown_handles(handle); + } +} + +// bootstate is used to "bootstrap" new threads. Any arguments needed by +// `thread_run()`, which can only take a single argument due to platform +// limitations, are contained in bootstate. +struct bootstate { + PyThreadState *tstate; + PyObject *func; + PyObject *args; + PyObject *kwargs; + ThreadHandle *handle; + PyEvent handle_ready; +}; + +static void +thread_bootstate_free(struct bootstate *boot, int decref) +{ + if (decref) { + Py_DECREF(boot->func); + Py_DECREF(boot->args); + Py_XDECREF(boot->kwargs); } + ThreadHandle_decref(boot->handle); + PyMem_RawFree(boot); +} + +static void +thread_run(void *boot_raw) +{ + struct bootstate *boot = (struct bootstate *) boot_raw; + PyThreadState *tstate = boot->tstate; + + // Wait until the handle is marked as running + PyEvent_Wait(&boot->handle_ready); + + // `handle` needs to be manipulated after bootstate has been freed + ThreadHandle *handle = boot->handle; + ThreadHandle_incref(handle); + + // gh-108987: If _thread.start_new_thread() is called before or while + // Python is being finalized, thread_run() can called *after*. + // _PyRuntimeState_SetFinalizing() is called. At this point, all Python + // threads must exit, except of the thread calling Py_Finalize() which + // holds the GIL and must not exit. + // + // At this stage, tstate can be a dangling pointer (point to freed memory), + // it's ok to call _PyThreadState_MustExit() with a dangling pointer. + if (_PyThreadState_MustExit(tstate)) { + // Don't call PyThreadState_Clear() nor _PyThreadState_DeleteCurrent(). + // These functions are called on tstate indirectly by Py_Finalize() + // which calls _PyInterpreterState_Clear(). + // + // Py_DECREF() cannot be called because the GIL is not held: leak + // references on purpose. Python is being finalized anyway. + thread_bootstate_free(boot, 0); + goto exit; + } + + _PyThreadState_Bind(tstate); + PyEval_AcquireThread(tstate); + _Py_atomic_add_ssize(&tstate->interp->threads.count, 1); + + PyObject *res = PyObject_Call(boot->func, boot->args, boot->kwargs); + if (res == NULL) { + if (PyErr_ExceptionMatches(PyExc_SystemExit)) + /* SystemExit is ignored silently */ + PyErr_Clear(); + else { + PyErr_FormatUnraisable( + "Exception ignored in thread started by %R", boot->func); + } + } + else { + Py_DECREF(res); + } + + thread_bootstate_free(boot, 1); + + _Py_atomic_add_ssize(&tstate->interp->threads.count, -1); + PyThreadState_Clear(tstate); + _PyThreadState_DeleteCurrent(tstate); + +exit: + // Don't need to wait for this thread anymore + remove_from_shutdown_handles(handle); + + _PyEvent_Notify(&handle->thread_is_exiting); + ThreadHandle_decref(handle); + + // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with + // the glibc, pthread_exit() can abort the whole process if dlopen() fails + // to open the libgcc_s.so library (ex: EMFILE error). + return; +} + +static int +force_done(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_STARTING); + _PyEvent_Notify(&handle->thread_is_exiting); + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +ThreadHandle_start(ThreadHandle *self, PyObject *func, PyObject *args, + PyObject *kwargs) +{ + // Mark the handle as starting to prevent any other threads from doing so + PyMutex_Lock(&self->mutex); + if (self->state != THREAD_HANDLE_NOT_STARTED) { + PyMutex_Unlock(&self->mutex); + PyErr_SetString(ThreadError, "thread already started"); + return -1; + } + self->state = THREAD_HANDLE_STARTING; + PyMutex_Unlock(&self->mutex); + + // Do all the heavy lifting outside of the mutex. All other operations on + // the handle should fail since the handle is in the starting state. + + // gh-109795: Use PyMem_RawMalloc() instead of PyMem_Malloc(), + // because it should be possible to call thread_bootstate_free() + // without holding the GIL. + struct bootstate *boot = PyMem_RawMalloc(sizeof(struct bootstate)); + if (boot == NULL) { + PyErr_NoMemory(); + goto start_failed; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); + if (boot->tstate == NULL) { + PyMem_RawFree(boot); + if (!PyErr_Occurred()) { + PyErr_NoMemory(); + } + goto start_failed; + } + boot->func = Py_NewRef(func); + boot->args = Py_NewRef(args); + boot->kwargs = Py_XNewRef(kwargs); + boot->handle = self; + ThreadHandle_incref(self); + boot->handle_ready = (PyEvent){0}; + + PyThread_ident_t ident; + PyThread_handle_t os_handle; + if (PyThread_start_joinable_thread(thread_run, boot, &ident, &os_handle)) { + PyThreadState_Clear(boot->tstate); + thread_bootstate_free(boot, 1); + PyErr_SetString(ThreadError, "can't start new thread"); + goto start_failed; + } + + // Mark the handle running + PyMutex_Lock(&self->mutex); + assert(self->state == THREAD_HANDLE_STARTING); + self->ident = ident; + self->has_os_handle = 1; + self->os_handle = os_handle; + self->state = THREAD_HANDLE_RUNNING; + PyMutex_Unlock(&self->mutex); + + // Unblock the thread + _PyEvent_Notify(&boot->handle_ready); + + return 0; + +start_failed: + _PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)force_done, self); + return -1; +} + +static int +join_thread(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_RUNNING); + PyThread_handle_t os_handle; + if (ThreadHandle_get_os_handle(handle, &os_handle)) { + int err = 0; + Py_BEGIN_ALLOW_THREADS + err = PyThread_join_thread(os_handle); + Py_END_ALLOW_THREADS + if (err) { + PyErr_SetString(ThreadError, "Failed joining thread"); + return -1; + } + } + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +check_started(ThreadHandle *self) +{ + ThreadHandleState state = get_thread_handle_state(self); + if (state < THREAD_HANDLE_RUNNING) { + PyErr_SetString(ThreadError, "thread not started"); + return -1; + } + return 0; +} + +static int +ThreadHandle_join(ThreadHandle *self, PyTime_t timeout_ns) +{ + if (check_started(self) < 0) { + return -1; + } + + // We want to perform this check outside of the `_PyOnceFlag` to prevent + // deadlock in the scenario where another thread joins us and we then + // attempt to join ourselves. However, it's not safe to check thread + // identity once the handle's os thread has finished. We may end up reusing + // the identity stored in the handle and erroneously think we are + // attempting to join ourselves. + // + // To work around this, we set `thread_is_exiting` immediately before + // `thread_run` returns. We can be sure that we are not attempting to join + // ourselves if the handle's thread is about to exit. + if (!_PyEvent_IsSet(&self->thread_is_exiting) && + ThreadHandle_ident(self) == PyThread_get_thread_ident_ex()) { + // PyThread_join_thread() would deadlock or error out. + PyErr_SetString(ThreadError, "Cannot join current thread"); + return -1; + } + + // Wait until the deadline for the thread to exit. + PyTime_t deadline = timeout_ns != -1 ? _PyDeadline_Init(timeout_ns) : 0; + int detach = 1; + while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns, detach)) { + if (deadline) { + // _PyDeadline_Get will return a negative value if the deadline has + // been exceeded. + timeout_ns = Py_MAX(_PyDeadline_Get(deadline), 0); + } + + if (timeout_ns) { + // Interrupted + if (Py_MakePendingCalls() < 0) { + return -1; + } + } + else { + // Timed out + return 0; + } + } + + if (_PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)join_thread, + self) == -1) { + return -1; + } + assert(get_thread_handle_state(self) == THREAD_HANDLE_DONE); + return 0; +} + +static int +set_done(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_RUNNING); + if (detach_thread(handle) < 0) { + PyErr_SetString(ThreadError, "failed detaching handle"); + return -1; + } + _PyEvent_Notify(&handle->thread_is_exiting); + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +ThreadHandle_set_done(ThreadHandle *self) +{ + if (check_started(self) < 0) { + return -1; + } + + if (_PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)set_done, self) == + -1) { + return -1; + } + assert(get_thread_handle_state(self) == THREAD_HANDLE_DONE); + return 0; +} + +// A wrapper around a ThreadHandle. +typedef struct { + PyObject_HEAD + + ThreadHandle *handle; +} PyThreadHandleObject; + +static PyThreadHandleObject * +PyThreadHandleObject_new(PyTypeObject *type) +{ + ThreadHandle *handle = ThreadHandle_new(); + if (handle == NULL) { + return NULL; + } + + PyThreadHandleObject *self = + (PyThreadHandleObject *)type->tp_alloc(type, 0); + if (self == NULL) { + ThreadHandle_decref(handle); + return NULL; + } + + self->handle = handle; + + return self; } static PyObject * -ThreadHandle_repr(ThreadHandleObject *self) +PyThreadHandleObject_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - return PyUnicode_FromFormat("<%s object: ident=%" PY_FORMAT_THREAD_IDENT_T ">", - Py_TYPE(self)->tp_name, self->ident); + return (PyObject *)PyThreadHandleObject_new(type); +} + +static int +PyThreadHandleObject_traverse(PyThreadHandleObject *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +PyThreadHandleObject_dealloc(PyObject *op) +{ + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); + ThreadHandle_decref(self->handle); + tp->tp_free(self); + Py_DECREF(tp); } static PyObject * -ThreadHandle_get_ident(ThreadHandleObject *self, void *ignored) +PyThreadHandleObject_repr(PyObject *op) { - return PyLong_FromUnsignedLongLong(self->ident); + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + PyThread_ident_t ident = ThreadHandle_ident(self->handle); + return PyUnicode_FromFormat("<%s object: ident=%" PY_FORMAT_THREAD_IDENT_T ">", + Py_TYPE(self)->tp_name, ident); } +static PyObject * +PyThreadHandleObject_get_ident(PyObject *op, void *Py_UNUSED(ignored)) +{ + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + return PyLong_FromUnsignedLongLong(ThreadHandle_ident(self->handle)); +} static PyObject * -ThreadHandle_detach(ThreadHandleObject *self, void* ignored) +PyThreadHandleObject_join(PyObject *op, PyObject *args) { - if (!self->joinable) { - PyErr_SetString(PyExc_ValueError, - "the thread is not joinable and thus cannot be detached"); + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + + PyObject *timeout_obj = NULL; + if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { return NULL; } - self->joinable = 0; - // This is typically short so no need to release the GIL - int ret = PyThread_detach_thread(self->handle); - if (ret) { - PyErr_SetString(ThreadError, "Failed detaching thread"); + + PyTime_t timeout_ns = -1; + if (timeout_obj != NULL && timeout_obj != Py_None) { + if (_PyTime_FromSecondsObject(&timeout_ns, timeout_obj, + _PyTime_ROUND_TIMEOUT) < 0) { + return NULL; + } + } + + if (ThreadHandle_join(self->handle, timeout_ns) < 0) { return NULL; } Py_RETURN_NONE; } static PyObject * -ThreadHandle_join(ThreadHandleObject *self, void* ignored) +PyThreadHandleObject_is_done(PyObject *op, PyObject *Py_UNUSED(ignored)) { - if (!self->joinable) { - PyErr_SetString(PyExc_ValueError, "the thread is not joinable"); - return NULL; + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + if (_PyEvent_IsSet(&self->handle->thread_is_exiting)) { + Py_RETURN_TRUE; } - if (self->ident == PyThread_get_thread_ident_ex()) { - // PyThread_join_thread() would deadlock or error out. - PyErr_SetString(ThreadError, "Cannot join current thread"); - return NULL; + else { + Py_RETURN_FALSE; } - // Before actually joining, we must first mark the thread as non-joinable, - // as joining several times simultaneously or sequentially is undefined behavior. - self->joinable = 0; - int ret; - Py_BEGIN_ALLOW_THREADS - ret = PyThread_join_thread(self->handle); - Py_END_ALLOW_THREADS - if (ret) { - PyErr_SetString(ThreadError, "Failed joining thread"); +} + +static PyObject * +PyThreadHandleObject_set_done(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + if (ThreadHandle_set_done(self->handle) < 0) { return NULL; } Py_RETURN_NONE; } static PyGetSetDef ThreadHandle_getsetlist[] = { - {"ident", (getter)ThreadHandle_get_ident, NULL, NULL}, + {"ident", PyThreadHandleObject_get_ident, NULL, NULL}, {0}, }; -static PyMethodDef ThreadHandle_methods[] = -{ - {"detach", (PyCFunction)ThreadHandle_detach, METH_NOARGS}, - {"join", (PyCFunction)ThreadHandle_join, METH_NOARGS}, +static PyMethodDef ThreadHandle_methods[] = { + {"join", PyThreadHandleObject_join, METH_VARARGS, NULL}, + {"_set_done", PyThreadHandleObject_set_done, METH_NOARGS, NULL}, + {"is_done", PyThreadHandleObject_is_done, METH_NOARGS, NULL}, {0, 0} }; static PyType_Slot ThreadHandle_Type_slots[] = { - {Py_tp_dealloc, (destructor)ThreadHandle_dealloc}, - {Py_tp_repr, (reprfunc)ThreadHandle_repr}, + {Py_tp_dealloc, PyThreadHandleObject_dealloc}, + {Py_tp_repr, PyThreadHandleObject_repr}, {Py_tp_getset, ThreadHandle_getsetlist}, + {Py_tp_traverse, PyThreadHandleObject_traverse}, {Py_tp_methods, ThreadHandle_methods}, + {Py_tp_new, PyThreadHandleObject_tp_new}, {0, 0} }; static PyType_Spec ThreadHandle_Type_spec = { "_thread._ThreadHandle", - sizeof(ThreadHandleObject), + sizeof(PyThreadHandleObject), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, ThreadHandle_Type_slots, }; @@ -203,45 +705,31 @@ static PyType_Spec ThreadHandle_Type_spec = { typedef struct { PyObject_HEAD - PyThread_type_lock lock_lock; - PyObject *in_weakreflist; - char locked; /* for sanity checking */ + PyMutex lock; } lockobject; static int -lock_traverse(lockobject *self, visitproc visit, void *arg) +lock_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); return 0; } static void -lock_dealloc(lockobject *self) +lock_dealloc(PyObject *op) { + lockobject *self = (lockobject*)op; PyObject_GC_UnTrack(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) self); - } - if (self->lock_lock != NULL) { - /* Unlock the lock so it's safe to free it */ - if (self->locked) - PyThread_release_lock(self->lock_lock); - PyThread_free_lock(self->lock_lock); - } + PyObject_ClearWeakRefs((PyObject *) self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free((PyObject*)self); Py_DECREF(tp); } -static inline PyLockStatus -acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) -{ - return PyThread_acquire_lock_timed_with_retries(lock, timeout); -} static int lock_acquire_parse_args(PyObject *args, PyObject *kwds, - _PyTime_t *timeout) + PyTime_t *timeout) { char *kwlist[] = {"blocking", "timeout", NULL}; int blocking = 1; @@ -252,7 +740,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, // XXX Use PyThread_ParseTimeoutArg(). - const _PyTime_t unset_timeout = _PyTime_FromSeconds(-1); + const PyTime_t unset_timeout = _PyTime_FromSeconds(-1); *timeout = unset_timeout; if (timeout_obj @@ -273,7 +761,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, if (!blocking) *timeout = 0; else if (*timeout != unset_timeout) { - _PyTime_t microseconds; + PyTime_t microseconds; microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_TIMEOUT); if (microseconds > PY_TIMEOUT_MAX) { @@ -286,25 +774,27 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, } static PyObject * -lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds) +lock_PyThread_acquire_lock(PyObject *op, PyObject *args, PyObject *kwds) { - _PyTime_t timeout; - if (lock_acquire_parse_args(args, kwds, &timeout) < 0) + lockobject *self = (lockobject*)op; + + PyTime_t timeout; + if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { return NULL; + } - PyLockStatus r = acquire_timed(self->lock_lock, timeout); + PyLockStatus r = _PyMutex_LockTimed(&self->lock, timeout, + _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); if (r == PY_LOCK_INTR) { return NULL; } - if (r == PY_LOCK_ACQUIRED) - self->locked = 1; return PyBool_FromLong(r == PY_LOCK_ACQUIRED); } PyDoc_STRVAR(acquire_doc, -"acquire(blocking=True, timeout=-1) -> bool\n\ -(acquire_lock() is an obsolete synonym)\n\ +"acquire($self, /, blocking=True, timeout=-1)\n\ +--\n\ \n\ Lock the lock. Without argument, this blocks if the lock is already\n\ locked (even by the same thread), waiting for another thread to release\n\ @@ -313,58 +803,84 @@ With an argument, this will only block if the argument is true,\n\ and the return value reflects whether the lock is acquired.\n\ The blocking operation is interruptible."); +PyDoc_STRVAR(acquire_lock_doc, +"acquire_lock($self, /, blocking=True, timeout=-1)\n\ +--\n\ +\n\ +An obsolete synonym of acquire()."); + +PyDoc_STRVAR(enter_doc, +"__enter__($self, /)\n\ +--\n\ +\n\ +Lock the lock."); + static PyObject * -lock_PyThread_release_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) +lock_PyThread_release_lock(PyObject *op, PyObject *Py_UNUSED(ignored)) { + lockobject *self = (lockobject*)op; /* Sanity check: the lock must be locked */ - if (!self->locked) { + if (_PyMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(ThreadError, "release unlocked lock"); return NULL; } - PyThread_release_lock(self->lock_lock); - self->locked = 0; Py_RETURN_NONE; } PyDoc_STRVAR(release_doc, -"release()\n\ -(release_lock() is an obsolete synonym)\n\ +"release($self, /)\n\ +--\n\ \n\ Release the lock, allowing another thread that is blocked waiting for\n\ the lock to acquire the lock. The lock must be in the locked state,\n\ but it needn't be locked by the same thread that unlocks it."); +PyDoc_STRVAR(release_lock_doc, +"release_lock($self, /)\n\ +--\n\ +\n\ +An obsolete synonym of release()."); + +PyDoc_STRVAR(lock_exit_doc, +"__exit__($self, /, *exc_info)\n\ +--\n\ +\n\ +Release the lock."); + static PyObject * -lock_locked_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) +lock_locked_lock(PyObject *op, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong((long)self->locked); + lockobject *self = (lockobject*)op; + return PyBool_FromLong(PyMutex_IsLocked(&self->lock)); } PyDoc_STRVAR(locked_doc, -"locked() -> bool\n\ -(locked_lock() is an obsolete synonym)\n\ +"locked($self, /)\n\ +--\n\ \n\ Return whether the lock is in the locked state."); +PyDoc_STRVAR(locked_lock_doc, +"locked_lock($self, /)\n\ +--\n\ +\n\ +An obsolete synonym of locked()."); + static PyObject * -lock_repr(lockobject *self) +lock_repr(PyObject *op) { + lockobject *self = (lockobject*)op; return PyUnicode_FromFormat("<%s %s object at %p>", - self->locked ? "locked" : "unlocked", Py_TYPE(self)->tp_name, self); + PyMutex_IsLocked(&self->lock) ? "locked" : "unlocked", Py_TYPE(self)->tp_name, self); } #ifdef HAVE_FORK static PyObject * -lock__at_fork_reinit(lockobject *self, PyObject *Py_UNUSED(args)) +lock__at_fork_reinit(PyObject *op, PyObject *Py_UNUSED(args)) { - if (_PyThread_at_fork_reinit(&self->lock_lock) < 0) { - PyErr_SetString(ThreadError, "failed to reinitialize lock at fork"); - return NULL; - } - - self->locked = 0; - + lockobject *self = (lockobject *)op; + _PyMutex_at_fork_reinit(&self->lock); Py_RETURN_NONE; } #endif /* HAVE_FORK */ @@ -393,30 +909,33 @@ lock_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) static PyMethodDef lock_methods[] = { {"acquire_lock", _PyCFunction_CAST(lock_PyThread_acquire_lock), - METH_VARARGS | METH_KEYWORDS, acquire_doc}, + METH_VARARGS | METH_KEYWORDS, acquire_lock_doc}, {"acquire", _PyCFunction_CAST(lock_PyThread_acquire_lock), METH_VARARGS | METH_KEYWORDS, acquire_doc}, - {"release_lock", (PyCFunction)lock_PyThread_release_lock, + {"release_lock", lock_PyThread_release_lock, + METH_NOARGS, release_lock_doc}, + {"release", lock_PyThread_release_lock, METH_NOARGS, release_doc}, - {"release", (PyCFunction)lock_PyThread_release_lock, - METH_NOARGS, release_doc}, - {"locked_lock", (PyCFunction)lock_locked_lock, - METH_NOARGS, locked_doc}, - {"locked", (PyCFunction)lock_locked_lock, + {"locked_lock", lock_locked_lock, + METH_NOARGS, locked_lock_doc}, + {"locked", lock_locked_lock, METH_NOARGS, locked_doc}, {"__enter__", _PyCFunction_CAST(lock_PyThread_acquire_lock), - METH_VARARGS | METH_KEYWORDS, acquire_doc}, - {"__exit__", (PyCFunction)lock_PyThread_release_lock, - METH_VARARGS, release_doc}, + METH_VARARGS | METH_KEYWORDS, enter_doc}, + {"__exit__", lock_PyThread_release_lock, + METH_VARARGS, lock_exit_doc}, #ifdef HAVE_FORK - {"_at_fork_reinit", (PyCFunction)lock__at_fork_reinit, + {"_at_fork_reinit", lock__at_fork_reinit, METH_NOARGS, NULL}, #endif {NULL, NULL} /* sentinel */ }; PyDoc_STRVAR(lock_doc, -"A lock object is a synchronization primitive. To create a lock,\n\ +"lock()\n\ +--\n\ +\n\ +A lock object is a synchronization primitive. To create a lock,\n\ call threading.Lock(). Methods are:\n\ \n\ acquire() -- lock the lock, possibly blocking until it can be obtained\n\ @@ -427,18 +946,12 @@ A lock is not owned by the thread that locked it; another thread may\n\ unlock it. A thread attempting to lock a lock that it has already locked\n\ will block until another thread unlocks it. Deadlocks may ensue."); -static PyMemberDef lock_type_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, in_weakreflist), Py_READONLY}, - {NULL}, -}; - static PyType_Slot lock_type_slots[] = { - {Py_tp_dealloc, (destructor)lock_dealloc}, - {Py_tp_repr, (reprfunc)lock_repr}, + {Py_tp_dealloc, lock_dealloc}, + {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, {Py_tp_methods, lock_methods}, {Py_tp_traverse, lock_traverse}, - {Py_tp_members, lock_type_members}, {Py_tp_new, lock_new}, {0, 0} }; @@ -447,7 +960,7 @@ static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = lock_type_slots, }; @@ -455,10 +968,7 @@ static PyType_Spec lock_type_spec = { typedef struct { PyObject_HEAD - PyThread_type_lock rlock_lock; - PyThread_ident_t rlock_owner; - unsigned long rlock_count; - PyObject *in_weakreflist; + _PyRecursiveMutex lock; } rlockobject; static int @@ -470,53 +980,30 @@ rlock_traverse(rlockobject *self, visitproc visit, void *arg) static void -rlock_dealloc(rlockobject *self) +rlock_dealloc(PyObject *op) { + rlockobject *self = (rlockobject*)op; PyObject_GC_UnTrack(self); - if (self->in_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - /* self->rlock_lock can be NULL if PyThread_allocate_lock() failed - in rlock_new() */ - if (self->rlock_lock != NULL) { - /* Unlock the lock so it's safe to free it */ - if (self->rlock_count > 0) - PyThread_release_lock(self->rlock_lock); - - PyThread_free_lock(self->rlock_lock); - } + PyObject_ClearWeakRefs((PyObject *) self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); } + static PyObject * -rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) +rlock_acquire(PyObject *op, PyObject *args, PyObject *kwds) { - _PyTime_t timeout; - PyThread_ident_t tid; - PyLockStatus r = PY_LOCK_ACQUIRED; + rlockobject *self = (rlockobject*)op; + PyTime_t timeout; - if (lock_acquire_parse_args(args, kwds, &timeout) < 0) + if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { return NULL; - - tid = PyThread_get_thread_ident_ex(); - if (self->rlock_count > 0 && tid == self->rlock_owner) { - unsigned long count = self->rlock_count + 1; - if (count <= self->rlock_count) { - PyErr_SetString(PyExc_OverflowError, - "Internal lock count overflowed"); - return NULL; - } - self->rlock_count = count; - Py_RETURN_TRUE; } - r = acquire_timed(self->rlock_lock, timeout); - if (r == PY_LOCK_ACQUIRED) { - assert(self->rlock_count == 0); - self->rlock_owner = tid; - self->rlock_count = 1; - } - else if (r == PY_LOCK_INTR) { + + PyLockStatus r = _PyRecursiveMutex_LockTimed(&self->lock, timeout, + _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); + if (r == PY_LOCK_INTR) { return NULL; } @@ -524,7 +1011,8 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(rlock_acquire_doc, -"acquire(blocking=True) -> bool\n\ +"acquire($self, /, blocking=True, timeout=-1)\n\ +--\n\ \n\ Lock the lock. `blocking` indicates whether we should wait\n\ for the lock to be available or not. If `blocking` is False\n\ @@ -539,25 +1027,28 @@ Precisely, if the current thread already holds the lock, its\n\ internal counter is simply incremented. If nobody holds the lock,\n\ the lock is taken and its internal counter initialized to 1."); +PyDoc_STRVAR(rlock_enter_doc, +"__enter__($self, /)\n\ +--\n\ +\n\ +Lock the lock."); + static PyObject * -rlock_release(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_release(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t tid = PyThread_get_thread_ident_ex(); + rlockobject *self = (rlockobject*)op; - if (self->rlock_count == 0 || self->rlock_owner != tid) { + if (_PyRecursiveMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); return NULL; } - if (--self->rlock_count == 0) { - self->rlock_owner = 0; - PyThread_release_lock(self->rlock_lock); - } Py_RETURN_NONE; } PyDoc_STRVAR(rlock_release_doc, -"release()\n\ +"release($self, /)\n\ +--\n\ \n\ Release the lock, allowing another thread that is blocked waiting for\n\ the lock to acquire the lock. The lock must be in the locked state,\n\ @@ -568,88 +1059,86 @@ Do note that if the lock was acquire()d several times in a row by the\n\ current thread, release() needs to be called as many times for the lock\n\ to be available for other threads."); +PyDoc_STRVAR(rlock_exit_doc, +"__exit__($self, /, *exc_info)\n\ +--\n\ +\n\ +Release the lock."); + static PyObject * -rlock_acquire_restore(rlockobject *self, PyObject *args) +rlock_acquire_restore(PyObject *op, PyObject *args) { + rlockobject *self = (rlockobject*)op; PyThread_ident_t owner; - unsigned long count; - int r = 1; + Py_ssize_t count; - if (!PyArg_ParseTuple(args, "(k" Py_PARSE_THREAD_IDENT_T "):_acquire_restore", + if (!PyArg_ParseTuple(args, "(n" Py_PARSE_THREAD_IDENT_T "):_acquire_restore", &count, &owner)) return NULL; - if (!PyThread_acquire_lock(self->rlock_lock, 0)) { - Py_BEGIN_ALLOW_THREADS - r = PyThread_acquire_lock(self->rlock_lock, 1); - Py_END_ALLOW_THREADS - } - if (!r) { - PyErr_SetString(ThreadError, "couldn't acquire lock"); - return NULL; - } - assert(self->rlock_count == 0); - self->rlock_owner = owner; - self->rlock_count = count; + _PyRecursiveMutex_Lock(&self->lock); + _Py_atomic_store_ullong_relaxed(&self->lock.thread, owner); + self->lock.level = (size_t)count - 1; Py_RETURN_NONE; } PyDoc_STRVAR(rlock_acquire_restore_doc, -"_acquire_restore(state) -> None\n\ +"_acquire_restore($self, state, /)\n\ +--\n\ \n\ For internal use by `threading.Condition`."); static PyObject * -rlock_release_save(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_release_save(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t owner; - unsigned long count; + rlockobject *self = (rlockobject*)op; - if (self->rlock_count == 0) { + if (!_PyRecursiveMutex_IsLockedByCurrentThread(&self->lock)) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); return NULL; } - owner = self->rlock_owner; - count = self->rlock_count; - self->rlock_count = 0; - self->rlock_owner = 0; - PyThread_release_lock(self->rlock_lock); - return Py_BuildValue("k" Py_PARSE_THREAD_IDENT_T, count, owner); + PyThread_ident_t owner = self->lock.thread; + Py_ssize_t count = self->lock.level + 1; + self->lock.level = 0; // ensure the unlock releases the lock + _PyRecursiveMutex_Unlock(&self->lock); + return Py_BuildValue("n" Py_PARSE_THREAD_IDENT_T, count, owner); } PyDoc_STRVAR(rlock_release_save_doc, -"_release_save() -> tuple\n\ +"_release_save($self, /)\n\ +--\n\ \n\ For internal use by `threading.Condition`."); static PyObject * -rlock_recursion_count(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_recursion_count(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - return PyLong_FromUnsignedLong( - self->rlock_owner == tid ? self->rlock_count : 0UL); + rlockobject *self = (rlockobject*)op; + if (_PyRecursiveMutex_IsLockedByCurrentThread(&self->lock)) { + return PyLong_FromSize_t(self->lock.level + 1); + } + return PyLong_FromLong(0); } PyDoc_STRVAR(rlock_recursion_count_doc, -"_recursion_count() -> int\n\ +"_recursion_count($self, /)\n\ +--\n\ \n\ For internal use by reentrancy checks."); static PyObject * -rlock_is_owned(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_is_owned(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - - if (self->rlock_count > 0 && self->rlock_owner == tid) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + rlockobject *self = (rlockobject*)op; + long owned = _PyRecursiveMutex_IsLockedByCurrentThread(&self->lock); + return PyBool_FromLong(owned); } PyDoc_STRVAR(rlock_is_owned_doc, -"_is_owned() -> bool\n\ +"_is_owned($self, /)\n\ +--\n\ \n\ For internal use by `threading.Condition`."); @@ -660,27 +1149,21 @@ rlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (self == NULL) { return NULL; } - self->in_weakreflist = NULL; - self->rlock_owner = 0; - self->rlock_count = 0; - - self->rlock_lock = PyThread_allocate_lock(); - if (self->rlock_lock == NULL) { - Py_DECREF(self); - PyErr_SetString(ThreadError, "can't allocate lock"); - return NULL; - } + self->lock = (_PyRecursiveMutex){0}; return (PyObject *) self; } static PyObject * -rlock_repr(rlockobject *self) +rlock_repr(PyObject *op) { + rlockobject *self = (rlockobject*)op; + PyThread_ident_t owner = self->lock.thread; + size_t count = self->lock.level + 1; return PyUnicode_FromFormat( - "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%lu at %p>", - self->rlock_count ? "locked" : "unlocked", - Py_TYPE(self)->tp_name, self->rlock_owner, - self->rlock_count, self); + "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%zu at %p>", + owner ? "locked" : "unlocked", + Py_TYPE(self)->tp_name, owner, + count, self); } @@ -688,14 +1171,7 @@ rlock_repr(rlockobject *self) static PyObject * rlock__at_fork_reinit(rlockobject *self, PyObject *Py_UNUSED(args)) { - if (_PyThread_at_fork_reinit(&self->rlock_lock) < 0) { - PyErr_SetString(ThreadError, "failed to reinitialize lock at fork"); - return NULL; - } - - self->rlock_owner = 0; - self->rlock_count = 0; - + self->lock = (_PyRecursiveMutex){0}; Py_RETURN_NONE; } #endif /* HAVE_FORK */ @@ -704,20 +1180,20 @@ rlock__at_fork_reinit(rlockobject *self, PyObject *Py_UNUSED(args)) static PyMethodDef rlock_methods[] = { {"acquire", _PyCFunction_CAST(rlock_acquire), METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, - {"release", (PyCFunction)rlock_release, + {"release", rlock_release, METH_NOARGS, rlock_release_doc}, - {"_is_owned", (PyCFunction)rlock_is_owned, + {"_is_owned", rlock_is_owned, METH_NOARGS, rlock_is_owned_doc}, - {"_acquire_restore", (PyCFunction)rlock_acquire_restore, + {"_acquire_restore", rlock_acquire_restore, METH_VARARGS, rlock_acquire_restore_doc}, - {"_release_save", (PyCFunction)rlock_release_save, + {"_release_save", rlock_release_save, METH_NOARGS, rlock_release_save_doc}, - {"_recursion_count", (PyCFunction)rlock_recursion_count, + {"_recursion_count", rlock_recursion_count, METH_NOARGS, rlock_recursion_count_doc}, {"__enter__", _PyCFunction_CAST(rlock_acquire), - METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, - {"__exit__", (PyCFunction)rlock_release, - METH_VARARGS, rlock_release_doc}, + METH_VARARGS | METH_KEYWORDS, rlock_enter_doc}, + {"__exit__", rlock_release, + METH_VARARGS, rlock_exit_doc}, #ifdef HAVE_FORK {"_at_fork_reinit", (PyCFunction)rlock__at_fork_reinit, METH_NOARGS, NULL}, @@ -726,18 +1202,12 @@ static PyMethodDef rlock_methods[] = { }; -static PyMemberDef rlock_type_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, in_weakreflist), Py_READONLY}, - {NULL}, -}; - static PyType_Slot rlock_type_slots[] = { - {Py_tp_dealloc, (destructor)rlock_dealloc}, - {Py_tp_repr, (reprfunc)rlock_repr}, + {Py_tp_dealloc, rlock_dealloc}, + {Py_tp_repr, rlock_repr}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, - {Py_tp_members, rlock_type_members}, {Py_tp_traverse, rlock_traverse}, {0, 0}, }; @@ -746,7 +1216,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = rlock_type_slots, }; @@ -760,16 +1230,7 @@ newlockobject(PyObject *module) if (self == NULL) { return NULL; } - - self->lock_lock = PyThread_allocate_lock(); - self->locked = 0; - self->in_weakreflist = NULL; - - if (self->lock_lock == NULL) { - Py_DECREF(self); - PyErr_SetString(ThreadError, "can't allocate lock"); - return NULL; - } + self->lock = (PyMutex){0}; return self; } @@ -790,39 +1251,51 @@ newlockobject(PyObject *module) Our implementation uses small "localdummy" objects in order to break the reference chain. These trivial objects are hashable (using the default scheme of identity hashing) and weakrefable. - Each thread-state holds a separate localdummy for each local object - (as a /strong reference/), - and each thread-local object holds a dict mapping /weak references/ - of localdummies to local dicts. + + Each thread-state holds two separate localdummy objects: + + - `threading_local_key` is used as a key to retrieve the locals dictionary + for the thread in any `threading.local` object. + - `threading_local_sentinel` is used to signal when a thread is being + destroyed. Consequently, the associated thread-state must hold the only + reference. + + Each `threading.local` object contains a dict mapping localdummy keys to + locals dicts and a set containing weak references to localdummy + sentinels. Each sentinel weak reference has a callback that removes itself + and the locals dict for the key from the `threading.local` object when + called. Therefore: - - only the thread-state dict holds a strong reference to the dummies - - only the thread-local object holds a strong reference to the local dicts - - only outside objects (application- or library-level) hold strong - references to the thread-local objects - - as soon as a thread-state dict is destroyed, the weakref callbacks of all - dummies attached to that thread are called, and destroy the corresponding - local dicts from thread-local objects - - as soon as a thread-local object is destroyed, its local dicts are - destroyed and its dummies are manually removed from all thread states - - the GC can do its work correctly when a thread-local object is dangling, - without any interference from the thread-state dicts - - As an additional optimization, each localdummy holds a borrowed reference - to the corresponding localdict. This borrowed reference is only used - by the thread-local object which has created the localdummy, which should - guarantee that the localdict still exists when accessed. + - The thread-state only holds strong references to localdummy objects, which + cannot participate in cycles. + - Only outside objects (application- or library-level) hold strong + references to the thread-local objects. + - As soon as thread-state's sentinel dummy is destroyed the callbacks for + all weakrefs attached to the sentinel are called, and destroy the + corresponding local dicts from thread-local objects. + - As soon as a thread-local object is destroyed, its local dicts are + destroyed. + - The GC can do its work correctly when a thread-local object is dangling, + without any interference from the thread-state dicts. + + This dual key arrangement is necessary to ensure that `threading.local` + values can be retrieved from finalizers. If we were to only keep a mapping + of localdummy weakrefs to locals dicts it's possible that the weakrefs would + be cleared before finalizers were called (GC currently clears weakrefs that + are garbage before invoking finalizers), causing lookups in finalizers to + fail. */ typedef struct { PyObject_HEAD - PyObject *localdict; /* Borrowed reference! */ PyObject *weakreflist; /* List of weak references to self */ } localdummyobject; static void -localdummy_dealloc(localdummyobject *self) +localdummy_dealloc(PyObject *op) { + localdummyobject *self = (localdummyobject*)op; if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); PyTypeObject *tp = Py_TYPE(self); @@ -836,7 +1309,7 @@ static PyMemberDef local_dummy_type_members[] = { }; static PyType_Slot local_dummy_type_slots[] = { - {Py_tp_dealloc, (destructor)localdummy_dealloc}, + {Py_tp_dealloc, localdummy_dealloc}, {Py_tp_doc, "Thread-local dummy"}, {Py_tp_members, local_dummy_type_members}, {0, 0} @@ -853,80 +1326,60 @@ static PyType_Spec local_dummy_type_spec = { typedef struct { PyObject_HEAD - PyObject *key; PyObject *args; PyObject *kw; PyObject *weakreflist; /* List of weak references to self */ - /* A {localdummy weakref -> localdict} dict */ - PyObject *dummies; - /* The callback for weakrefs to localdummies */ - PyObject *wr_callback; + /* A {localdummy -> localdict} dict */ + PyObject *localdicts; + /* A set of weakrefs to thread sentinels localdummies*/ + PyObject *thread_watchdogs; } localobject; /* Forward declaration */ -static PyObject *_ldict(localobject *self, thread_module_state *state); -static PyObject *_localdummy_destroyed(PyObject *meth_self, PyObject *dummyweakref); +static int create_localsdict(localobject *self, thread_module_state *state, + PyObject **localsdict, PyObject **sentinel_wr); +static PyObject *clear_locals(PyObject *meth_self, PyObject *dummyweakref); -/* Create and register the dummy for the current thread. - Returns a borrowed reference of the corresponding local dict */ +/* Create a weakref to the sentinel localdummy for the current thread */ static PyObject * -_local_create_dummy(localobject *self, thread_module_state *state) +create_sentinel_wr(localobject *self) { - PyObject *ldict = NULL, *wr = NULL; - localdummyobject *dummy = NULL; - PyTypeObject *type = state->local_dummy_type; + static PyMethodDef wr_callback_def = { + "clear_locals", (PyCFunction) clear_locals, METH_O + }; - PyObject *tdict = PyThreadState_GetDict(); - if (tdict == NULL) { - PyErr_SetString(PyExc_SystemError, - "Couldn't get thread-state dictionary"); - goto err; - } + PyThreadState *tstate = PyThreadState_Get(); - ldict = PyDict_New(); - if (ldict == NULL) { - goto err; - } - dummy = (localdummyobject *) type->tp_alloc(type, 0); - if (dummy == NULL) { - goto err; - } - dummy->localdict = ldict; - wr = PyWeakref_NewRef((PyObject *) dummy, self->wr_callback); - if (wr == NULL) { - goto err; + /* We use a weak reference to self in the callback closure + in order to avoid spurious reference cycles */ + PyObject *self_wr = PyWeakref_NewRef((PyObject *) self, NULL); + if (self_wr == NULL) { + return NULL; } - /* As a side-effect, this will cache the weakref's hash before the - dummy gets deleted */ - int r = PyDict_SetItem(self->dummies, wr, ldict); - if (r < 0) { - goto err; + PyObject *args = PyTuple_New(2); + if (args == NULL) { + Py_DECREF(self_wr); + return NULL; } - Py_CLEAR(wr); - r = PyDict_SetItem(tdict, self->key, (PyObject *) dummy); - if (r < 0) { - goto err; + PyTuple_SET_ITEM(args, 0, self_wr); + PyTuple_SET_ITEM(args, 1, Py_NewRef(tstate->threading_local_key)); + + PyObject *cb = PyCFunction_New(&wr_callback_def, args); + Py_DECREF(args); + if (cb == NULL) { + return NULL; } - Py_CLEAR(dummy); - Py_DECREF(ldict); - return ldict; + PyObject *wr = PyWeakref_NewRef(tstate->threading_local_sentinel, cb); + Py_DECREF(cb); -err: - Py_XDECREF(ldict); - Py_XDECREF(wr); - Py_XDECREF(dummy); - return NULL; + return wr; } static PyObject * local_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - static PyMethodDef wr_callback_def = { - "_localdummy_destroyed", (PyCFunction) _localdummy_destroyed, METH_O - }; - if (type->tp_init == PyBaseObject_Type.tp_init) { int rc = 0; if (args != NULL) @@ -953,30 +1406,25 @@ local_new(PyTypeObject *type, PyObject *args, PyObject *kw) self->args = Py_XNewRef(args); self->kw = Py_XNewRef(kw); - self->key = PyUnicode_FromFormat("thread.local.%p", self); - if (self->key == NULL) { - goto err; - } - self->dummies = PyDict_New(); - if (self->dummies == NULL) { + self->localdicts = PyDict_New(); + if (self->localdicts == NULL) { goto err; } - /* We use a weak reference to self in the callback closure - in order to avoid spurious reference cycles */ - PyObject *wr = PyWeakref_NewRef((PyObject *) self, NULL); - if (wr == NULL) { + self->thread_watchdogs = PySet_New(NULL); + if (self->thread_watchdogs == NULL) { goto err; } - self->wr_callback = PyCFunction_NewEx(&wr_callback_def, wr, NULL); - Py_DECREF(wr); - if (self->wr_callback == NULL) { - goto err; - } - if (_local_create_dummy(self, state) == NULL) { + + PyObject *localsdict = NULL; + PyObject *sentinel_wr = NULL; + if (create_localsdict(self, state, &localsdict, &sentinel_wr) < 0) { goto err; } + Py_DECREF(localsdict); + Py_DECREF(sentinel_wr); + return (PyObject *)self; err: @@ -990,93 +1438,169 @@ local_traverse(localobject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); Py_VISIT(self->args); Py_VISIT(self->kw); - Py_VISIT(self->dummies); + Py_VISIT(self->localdicts); + Py_VISIT(self->thread_watchdogs); + return 0; +} + +static int +local_clear(localobject *self) +{ + Py_CLEAR(self->args); + Py_CLEAR(self->kw); + Py_CLEAR(self->localdicts); + Py_CLEAR(self->thread_watchdogs); + return 0; +} + +static void +local_dealloc(localobject *self) +{ + /* Weakrefs must be invalidated right now, otherwise they can be used + from code called below, which is very dangerous since Py_REFCNT(self) == 0 */ + if (self->weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject *) self); + } + + PyObject_GC_UnTrack(self); + + local_clear(self); + + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject*)self); + Py_DECREF(tp); +} + +/* Create the TLS key and sentinel if they don't exist */ +static int +create_localdummies(thread_module_state *state) +{ + PyThreadState *tstate = _PyThreadState_GET(); + + if (tstate->threading_local_key != NULL) { + return 0; + } + + PyTypeObject *ld_type = state->local_dummy_type; + tstate->threading_local_key = ld_type->tp_alloc(ld_type, 0); + if (tstate->threading_local_key == NULL) { + return -1; + } + + tstate->threading_local_sentinel = ld_type->tp_alloc(ld_type, 0); + if (tstate->threading_local_sentinel == NULL) { + Py_CLEAR(tstate->threading_local_key); + return -1; + } + return 0; } -static int -local_clear(localobject *self) -{ - Py_CLEAR(self->args); - Py_CLEAR(self->kw); - Py_CLEAR(self->dummies); - Py_CLEAR(self->wr_callback); - /* Remove all strong references to dummies from the thread states */ - if (self->key) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - while (tstate) { - if (tstate->dict) { - if (PyDict_Pop(tstate->dict, self->key, NULL) < 0) { - // Silently ignore error - PyErr_Clear(); - } - } - HEAD_LOCK(runtime); - tstate = PyThreadState_Next(tstate); - HEAD_UNLOCK(runtime); +/* Insert a localsdict and sentinel weakref for the current thread, placing + strong references in localsdict and sentinel_wr, respectively. +*/ +static int +create_localsdict(localobject *self, thread_module_state *state, + PyObject **localsdict, PyObject **sentinel_wr) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *ldict = NULL; + PyObject *wr = NULL; + + if (create_localdummies(state) < 0) { + goto err; + } + + /* Create and insert the locals dict and sentinel weakref */ + ldict = PyDict_New(); + if (ldict == NULL) { + goto err; + } + + if (PyDict_SetItem(self->localdicts, tstate->threading_local_key, ldict) < + 0) { + goto err; + } + + wr = create_sentinel_wr(self); + if (wr == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyDict_DelItem(self->localdicts, tstate->threading_local_key) < + 0) { + PyErr_WriteUnraisable((PyObject *)self); } + PyErr_SetRaisedException(exc); + goto err; } - return 0; -} -static void -local_dealloc(localobject *self) -{ - /* Weakrefs must be invalidated right now, otherwise they can be used - from code called below, which is very dangerous since Py_REFCNT(self) == 0 */ - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) self); + if (PySet_Add(self->thread_watchdogs, wr) < 0) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyDict_DelItem(self->localdicts, tstate->threading_local_key) < + 0) { + PyErr_WriteUnraisable((PyObject *)self); + } + PyErr_SetRaisedException(exc); + goto err; } - PyObject_GC_UnTrack(self); - - local_clear(self); - Py_XDECREF(self->key); + *localsdict = ldict; + *sentinel_wr = wr; + return 0; - PyTypeObject *tp = Py_TYPE(self); - tp->tp_free((PyObject*)self); - Py_DECREF(tp); +err: + Py_XDECREF(ldict); + Py_XDECREF(wr); + return -1; } -/* Returns a borrowed reference to the local dict, creating it if necessary */ +/* Return a strong reference to the locals dict for the current thread, + creating it if necessary. +*/ static PyObject * _ldict(localobject *self, thread_module_state *state) { - PyObject *tdict = PyThreadState_GetDict(); - if (tdict == NULL) { - PyErr_SetString(PyExc_SystemError, - "Couldn't get thread-state dictionary"); + if (create_localdummies(state) < 0) { return NULL; } + /* Check if a localsdict already exists */ PyObject *ldict; - PyObject *dummy = PyDict_GetItemWithError(tdict, self->key); - if (dummy == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - ldict = _local_create_dummy(self, state); - if (ldict == NULL) - return NULL; + PyThreadState *tstate = _PyThreadState_GET(); + if (PyDict_GetItemRef(self->localdicts, tstate->threading_local_key, + &ldict) < 0) { + return NULL; + } + if (ldict != NULL) { + return ldict; + } - if (Py_TYPE(self)->tp_init != PyBaseObject_Type.tp_init && - Py_TYPE(self)->tp_init((PyObject*)self, - self->args, self->kw) < 0) { - /* we need to get rid of ldict from thread so - we create a new one the next time we do an attr - access */ - PyDict_DelItem(tdict, self->key); - return NULL; - } + /* threading.local hasn't been instantiated for this thread */ + PyObject *wr; + if (create_localsdict(self, state, &ldict, &wr) < 0) { + return NULL; } - else { - assert(Py_IS_TYPE(dummy, state->local_dummy_type)); - ldict = ((localdummyobject *) dummy)->localdict; + + /* run __init__ if we're a subtype of `threading.local` */ + if (Py_TYPE(self)->tp_init != PyBaseObject_Type.tp_init && + Py_TYPE(self)->tp_init((PyObject *)self, self->args, self->kw) < 0) { + /* we need to get rid of ldict from thread so + we create a new one the next time we do an attr + access */ + PyObject *exc = PyErr_GetRaisedException(); + if (PyDict_DelItem(self->localdicts, tstate->threading_local_key) < + 0) { + PyErr_WriteUnraisable((PyObject *)self); + PyErr_Clear(); + } + if (PySet_Discard(self->thread_watchdogs, wr) < 0) { + PyErr_WriteUnraisable((PyObject *)self); + } + PyErr_SetRaisedException(exc); + Py_DECREF(ldict); + Py_DECREF(wr); + return NULL; } + Py_DECREF(wr); return ldict; } @@ -1090,21 +1614,28 @@ local_setattro(localobject *self, PyObject *name, PyObject *v) PyObject *ldict = _ldict(self, state); if (ldict == NULL) { - return -1; + goto err; } int r = PyObject_RichCompareBool(name, &_Py_ID(__dict__), Py_EQ); if (r == -1) { - return -1; + goto err; } if (r == 1) { PyErr_Format(PyExc_AttributeError, "'%.100s' object attribute '%U' is read-only", Py_TYPE(self)->tp_name, name); - return -1; + goto err; } - return _PyObject_GenericSetAttrWithDict((PyObject *)self, name, v, ldict); + int st = + _PyObject_GenericSetAttrWithDict((PyObject *)self, name, v, ldict); + Py_DECREF(ldict); + return st; + +err: + Py_XDECREF(ldict); + return -1; } static PyObject *local_getattro(localobject *, PyObject *); @@ -1118,7 +1649,7 @@ static PyType_Slot local_type_slots[] = { {Py_tp_dealloc, (destructor)local_dealloc}, {Py_tp_getattro, (getattrofunc)local_getattro}, {Py_tp_setattro, (setattrofunc)local_setattro}, - {Py_tp_doc, "Thread-local data"}, + {Py_tp_doc, "_local()\n--\n\nThread-local data"}, {Py_tp_traverse, (traverseproc)local_traverse}, {Py_tp_clear, (inquiry)local_clear}, {Py_tp_new, local_new}, @@ -1147,34 +1678,42 @@ local_getattro(localobject *self, PyObject *name) int r = PyObject_RichCompareBool(name, &_Py_ID(__dict__), Py_EQ); if (r == 1) { - return Py_NewRef(ldict); + return ldict; } if (r == -1) { + Py_DECREF(ldict); return NULL; } if (!Py_IS_TYPE(self, state->local_type)) { /* use generic lookup for subtypes */ - return _PyObject_GenericGetAttrWithDict((PyObject *)self, name, - ldict, 0); + PyObject *res = + _PyObject_GenericGetAttrWithDict((PyObject *)self, name, ldict, 0); + Py_DECREF(ldict); + return res; } /* Optimization: just look in dict ourselves */ PyObject *value; if (PyDict_GetItemRef(ldict, name, &value) != 0) { // found or error + Py_DECREF(ldict); return value; } /* Fall back on generic to get __class__ and __dict__ */ - return _PyObject_GenericGetAttrWithDict( - (PyObject *)self, name, ldict, 0); + PyObject *res = + _PyObject_GenericGetAttrWithDict((PyObject *)self, name, ldict, 0); + Py_DECREF(ldict); + return res; } -/* Called when a dummy is destroyed. */ +/* Called when a dummy is destroyed, indicating that the owning thread is being + * cleared. */ static PyObject * -_localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) +clear_locals(PyObject *locals_and_key, PyObject *dummyweakref) { + PyObject *localweakref = PyTuple_GetItem(locals_and_key, 0); localobject *self = (localobject *)_PyWeakref_GET_REF(localweakref); if (self == NULL) { Py_RETURN_NONE; @@ -1182,97 +1721,24 @@ _localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) /* If the thread-local object is still alive and not being cleared, remove the corresponding local dict */ - if (self->dummies != NULL) { - PyObject *ldict; - ldict = PyDict_GetItemWithError(self->dummies, dummyweakref); - if (ldict != NULL) { - PyDict_DelItem(self->dummies, dummyweakref); - } - if (PyErr_Occurred()) + if (self->localdicts != NULL) { + PyObject *key = PyTuple_GetItem(locals_and_key, 1); + if (PyDict_Pop(self->localdicts, key, NULL) < 0) { PyErr_WriteUnraisable((PyObject*)self); + } + } + if (self->thread_watchdogs != NULL) { + if (PySet_Discard(self->thread_watchdogs, dummyweakref) < 0) { + PyErr_WriteUnraisable((PyObject *)self); + } } + Py_DECREF(self); Py_RETURN_NONE; } /* Module functions */ -struct bootstate { - PyThreadState *tstate; - PyObject *func; - PyObject *args; - PyObject *kwargs; -}; - - -static void -thread_bootstate_free(struct bootstate *boot, int decref) -{ - if (decref) { - Py_DECREF(boot->func); - Py_DECREF(boot->args); - Py_XDECREF(boot->kwargs); - } - PyMem_RawFree(boot); -} - - -static void -thread_run(void *boot_raw) -{ - struct bootstate *boot = (struct bootstate *) boot_raw; - PyThreadState *tstate = boot->tstate; - - // gh-108987: If _thread.start_new_thread() is called before or while - // Python is being finalized, thread_run() can called *after*. - // _PyRuntimeState_SetFinalizing() is called. At this point, all Python - // threads must exit, except of the thread calling Py_Finalize() whch holds - // the GIL and must not exit. - // - // At this stage, tstate can be a dangling pointer (point to freed memory), - // it's ok to call _PyThreadState_MustExit() with a dangling pointer. - if (_PyThreadState_MustExit(tstate)) { - // Don't call PyThreadState_Clear() nor _PyThreadState_DeleteCurrent(). - // These functions are called on tstate indirectly by Py_Finalize() - // which calls _PyInterpreterState_Clear(). - // - // Py_DECREF() cannot be called because the GIL is not held: leak - // references on purpose. Python is being finalized anyway. - thread_bootstate_free(boot, 0); - goto exit; - } - - _PyThreadState_Bind(tstate); - PyEval_AcquireThread(tstate); - tstate->interp->threads.count++; - - PyObject *res = PyObject_Call(boot->func, boot->args, boot->kwargs); - if (res == NULL) { - if (PyErr_ExceptionMatches(PyExc_SystemExit)) - /* SystemExit is ignored silently */ - PyErr_Clear(); - else { - PyErr_FormatUnraisable( - "Exception ignored in thread started by %R", boot->func); - } - } - else { - Py_DECREF(res); - } - - thread_bootstate_free(boot, 1); - - tstate->interp->threads.count--; - PyThreadState_Clear(tstate); - _PyThreadState_DeleteCurrent(tstate); - -exit: - // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with - // the glibc, pthread_exit() can abort the whole process if dlopen() fails - // to open the libgcc_s.so library (ex: EMFILE error). - return; -} - static PyObject * thread_daemon_threads_allowed(PyObject *module, PyObject *Py_UNUSED(ignored)) { @@ -1286,16 +1752,15 @@ thread_daemon_threads_allowed(PyObject *module, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(daemon_threads_allowed_doc, -"daemon_threads_allowed()\n\ +"daemon_threads_allowed($module, /)\n\ +--\n\ \n\ Return True if daemon threads are allowed in the current interpreter,\n\ and False otherwise.\n"); static int -do_start_new_thread(thread_module_state* state, - PyObject *func, PyObject* args, PyObject* kwargs, - int joinable, - PyThread_ident_t* ident, PyThread_handle_t* handle) +do_start_new_thread(thread_module_state *state, PyObject *func, PyObject *args, + PyObject *kwargs, ThreadHandle *handle, int daemon) { PyInterpreterState *interp = _PyInterpreterState_GET(); if (!_PyInterpreterState_HasFeature(interp, Py_RTFLAGS_THREADS)) { @@ -1303,46 +1768,26 @@ do_start_new_thread(thread_module_state* state, "thread is not supported for isolated subinterpreters"); return -1; } - if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { + PyErr_SetString(PyExc_PythonFinalizationError, "can't create new thread at interpreter shutdown"); return -1; } - // gh-109795: Use PyMem_RawMalloc() instead of PyMem_Malloc(), - // because it should be possible to call thread_bootstate_free() - // without holding the GIL. - struct bootstate *boot = PyMem_RawMalloc(sizeof(struct bootstate)); - if (boot == NULL) { - PyErr_NoMemory(); - return -1; + if (!daemon) { + // Add the handle before starting the thread to avoid adding a handle + // to a thread that has already finished (i.e. if the thread finishes + // before the call to `ThreadHandle_start()` below returns). + add_to_shutdown_handles(state, handle); } - boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); - if (boot->tstate == NULL) { - PyMem_RawFree(boot); - if (!PyErr_Occurred()) { - PyErr_NoMemory(); + + if (ThreadHandle_start(handle, func, args, kwargs) < 0) { + if (!daemon) { + remove_from_shutdown_handles(handle); } return -1; } - boot->func = Py_NewRef(func); - boot->args = Py_NewRef(args); - boot->kwargs = Py_XNewRef(kwargs); - int err; - if (joinable) { - err = PyThread_start_joinable_thread(thread_run, (void*) boot, ident, handle); - } else { - *handle = 0; - *ident = PyThread_start_new_thread(thread_run, (void*) boot); - err = (*ident == PYTHREAD_INVALID_THREAD_ID); - } - if (err) { - PyErr_SetString(ThreadError, "can't start new thread"); - PyThreadState_Clear(boot->tstate); - thread_bootstate_free(boot, 1); - return -1; - } return 0; } @@ -1376,18 +1821,25 @@ thread_PyThread_start_new_thread(PyObject *module, PyObject *fargs) return NULL; } - PyThread_ident_t ident = 0; - PyThread_handle_t handle; - if (do_start_new_thread(state, func, args, kwargs, /*joinable=*/ 0, - &ident, &handle)) { + ThreadHandle *handle = ThreadHandle_new(); + if (handle == NULL) { return NULL; } + + int st = + do_start_new_thread(state, func, args, kwargs, handle, /*daemon=*/1); + if (st < 0) { + ThreadHandle_decref(handle); + return NULL; + } + PyThread_ident_t ident = ThreadHandle_ident(handle); + ThreadHandle_decref(handle); return PyLong_FromUnsignedLongLong(ident); } -PyDoc_STRVAR(start_new_doc, -"start_new_thread(function, args[, kwargs])\n\ -(start_new() is an obsolete synonym)\n\ +PyDoc_STRVAR(start_new_thread_doc, +"start_new_thread($module, function, args, kwargs={}, /)\n\ +--\n\ \n\ Start a new thread and return its identifier.\n\ \n\ @@ -1396,12 +1848,28 @@ tuple args and keyword arguments taken from the optional dictionary\n\ kwargs. The thread exits when the function returns; the return value\n\ is ignored. The thread will also exit when the function raises an\n\ unhandled exception; a stack trace will be printed unless the exception\n\ -is SystemExit.\n"); +is SystemExit."); + +PyDoc_STRVAR(start_new_doc, +"start_new($module, function, args, kwargs={}, /)\n\ +--\n\ +\n\ +An obsolete synonym of start_new_thread()."); static PyObject * -thread_PyThread_start_joinable_thread(PyObject *module, PyObject *func) +thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, + PyObject *fkwargs) { + static char *keywords[] = {"function", "handle", "daemon", NULL}; + PyObject *func = NULL; + int daemon = 1; thread_module_state *state = get_thread_state(module); + PyObject *hobj = NULL; + if (!PyArg_ParseTupleAndKeywords(fargs, fkwargs, + "O|Op:start_joinable_thread", keywords, + &func, &hobj, &daemon)) { + return NULL; + } if (!PyCallable_Check(func)) { PyErr_SetString(PyExc_TypeError, @@ -1409,32 +1877,46 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *func) return NULL; } - if (PySys_Audit("_thread.start_joinable_thread", "O", func) < 0) { + if (hobj == NULL) { + hobj = Py_None; + } + else if (hobj != Py_None && !Py_IS_TYPE(hobj, state->thread_handle_type)) { + PyErr_SetString(PyExc_TypeError, "'handle' must be a _ThreadHandle"); return NULL; } - PyObject* args = PyTuple_New(0); - if (args == NULL) { + if (PySys_Audit("_thread.start_joinable_thread", "OiO", func, daemon, + hobj) < 0) { return NULL; } - ThreadHandleObject* hobj = new_thread_handle(state); - if (hobj == NULL) { - Py_DECREF(args); + + if (hobj == Py_None) { + hobj = (PyObject *)PyThreadHandleObject_new(state->thread_handle_type); + if (hobj == NULL) { + return NULL; + } + } + else { + Py_INCREF(hobj); + } + + PyObject* args = PyTuple_New(0); + if (args == NULL) { return NULL; } - if (do_start_new_thread(state, func, args, /*kwargs=*/ NULL, /*joinable=*/ 1, - &hobj->ident, &hobj->handle)) { - Py_DECREF(args); + int st = do_start_new_thread(state, func, args, + /*kwargs=*/ NULL, ((PyThreadHandleObject*)hobj)->handle, daemon); + Py_DECREF(args); + if (st < 0) { Py_DECREF(hobj); return NULL; } - Py_DECREF(args); - hobj->joinable = 1; - return (PyObject*) hobj; + return (PyObject *) hobj; } PyDoc_STRVAR(start_joinable_doc, -"start_joinable_thread(function)\n\ +"start_joinable_thread($module, /, function, handle=None, daemon=True)\n\ +--\n\ \n\ *For internal use only*: start a new thread.\n\ \n\ @@ -1442,7 +1924,9 @@ Like start_new_thread(), this starts a new thread calling the given function.\n\ Unlike start_new_thread(), this returns a handle object with methods to join\n\ or detach the given thread.\n\ This function is not for third-party code, please use the\n\ -`threading` module instead.\n"); +`threading` module instead. During finalization the runtime will not wait for\n\ +the thread to exit if daemon is True. If handle is provided it must be a\n\ +newly created thread._ThreadHandle instance."); static PyObject * thread_PyThread_exit_thread(PyObject *self, PyObject *Py_UNUSED(ignored)) @@ -1452,12 +1936,18 @@ thread_PyThread_exit_thread(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(exit_doc, -"exit()\n\ -(exit_thread() is an obsolete synonym)\n\ +"exit($module, /)\n\ +--\n\ \n\ This is synonymous to ``raise SystemExit''. It will cause the current\n\ thread to exit silently unless the exception is caught."); +PyDoc_STRVAR(exit_thread_doc, +"exit_thread($module, /)\n\ +--\n\ +\n\ +An obsolete synonym of exit()."); + static PyObject * thread_PyThread_interrupt_main(PyObject *self, PyObject *args) { @@ -1474,7 +1964,8 @@ thread_PyThread_interrupt_main(PyObject *self, PyObject *args) } PyDoc_STRVAR(interrupt_doc, -"interrupt_main(signum=signal.SIGINT, /)\n\ +"interrupt_main($module, signum=signal.SIGINT, /)\n\ +--\n\ \n\ Simulate the arrival of the given signal in the main thread,\n\ where the corresponding signal handler will be executed.\n\ @@ -1490,13 +1981,19 @@ thread_PyThread_allocate_lock(PyObject *module, PyObject *Py_UNUSED(ignored)) return (PyObject *) newlockobject(module); } -PyDoc_STRVAR(allocate_doc, -"allocate_lock() -> lock object\n\ -(allocate() is an obsolete synonym)\n\ +PyDoc_STRVAR(allocate_lock_doc, +"allocate_lock($module, /)\n\ +--\n\ \n\ Create a new lock object. See help(type(threading.Lock())) for\n\ information about locks."); +PyDoc_STRVAR(allocate_doc, +"allocate($module, /)\n\ +--\n\ +\n\ +An obsolete synonym of allocate_lock()."); + static PyObject * thread_get_ident(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -1509,7 +2006,8 @@ thread_get_ident(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(get_ident_doc, -"get_ident() -> integer\n\ +"get_ident($module, /)\n\ +--\n\ \n\ Return a non-zero integer that uniquely identifies the current thread\n\ amongst other threads that exist simultaneously.\n\ @@ -1528,7 +2026,8 @@ thread_get_native_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(get_native_id_doc, -"get_native_id() -> integer\n\ +"get_native_id($module, /)\n\ +--\n\ \n\ Return a non-negative integer identifying the thread as reported\n\ by the OS (kernel). This may be used to uniquely identify a\n\ @@ -1539,13 +2038,13 @@ static PyObject * thread__count(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->threads.count); + return PyLong_FromSsize_t(_Py_atomic_load_ssize(&interp->threads.count)); } PyDoc_STRVAR(_count_doc, -"_count() -> integer\n\ +"_count($module, /)\n\ +--\n\ \n\ -\ Return the number of currently running Python threads, excluding\n\ the main thread. The returned number comprises all threads created\n\ through `start_new_thread()` as well as `threading.Thread`, and not\n\ @@ -1554,67 +2053,6 @@ yet finished.\n\ This function is meant for internal and specialized purposes only.\n\ In most applications `threading.enumerate()` should be used instead."); -static void -release_sentinel(void *weakref_raw) -{ - PyObject *weakref = _PyObject_CAST(weakref_raw); - - /* Tricky: this function is called when the current thread state - is being deleted. Therefore, only simple C code can safely - execute here. */ - lockobject *lock = (lockobject *)_PyWeakref_GET_REF(weakref); - if (lock != NULL) { - if (lock->locked) { - PyThread_release_lock(lock->lock_lock); - lock->locked = 0; - } - Py_DECREF(lock); - } - - /* Deallocating a weakref with a NULL callback only calls - PyObject_GC_Del(), which can't call any Python code. */ - Py_DECREF(weakref); -} - -static PyObject * -thread__set_sentinel(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - PyObject *wr; - PyThreadState *tstate = _PyThreadState_GET(); - lockobject *lock; - - if (tstate->on_delete_data != NULL) { - /* We must support the re-creation of the lock from a - fork()ed child. */ - assert(tstate->on_delete == &release_sentinel); - wr = (PyObject *) tstate->on_delete_data; - tstate->on_delete = NULL; - tstate->on_delete_data = NULL; - Py_DECREF(wr); - } - lock = newlockobject(module); - if (lock == NULL) - return NULL; - /* The lock is owned by whoever called _set_sentinel(), but the weakref - hangs to the thread state. */ - wr = PyWeakref_NewRef((PyObject *) lock, NULL); - if (wr == NULL) { - Py_DECREF(lock); - return NULL; - } - tstate->on_delete_data = (void *) wr; - tstate->on_delete = &release_sentinel; - return (PyObject *) lock; -} - -PyDoc_STRVAR(_set_sentinel_doc, -"_set_sentinel() -> lock\n\ -\n\ -Set a sentinel lock that will be released when the current thread\n\ -state is finalized (after it is untied from the interpreter).\n\ -\n\ -This is a private API for the threading module."); - static PyObject * thread_stack_size(PyObject *self, PyObject *args) { @@ -1650,7 +2088,8 @@ thread_stack_size(PyObject *self, PyObject *args) } PyDoc_STRVAR(stack_size_doc, -"stack_size([size]) -> size\n\ +"stack_size($module, size=0, /)\n\ +--\n\ \n\ Return the thread stack size used when creating new threads. The\n\ optional size argument specifies the stack size (in bytes) to be used\n\ @@ -1805,7 +2244,8 @@ thread_excepthook(PyObject *module, PyObject *args) } PyDoc_STRVAR(excepthook_doc, -"excepthook(exc_type, exc_value, exc_traceback, thread)\n\ +"_excepthook($module, (exc_type, exc_value, exc_traceback, thread), /)\n\ +--\n\ \n\ Handle uncaught Thread.run() exception."); @@ -1817,25 +2257,117 @@ thread__is_main_interpreter(PyObject *module, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(thread__is_main_interpreter_doc, -"_is_main_interpreter()\n\ +"_is_main_interpreter($module, /)\n\ +--\n\ \n\ Return True if the current interpreter is the main Python interpreter."); +static PyObject * +thread_shutdown(PyObject *self, PyObject *args) +{ + PyThread_ident_t ident = PyThread_get_thread_ident_ex(); + thread_module_state *state = get_thread_state(self); + + for (;;) { + ThreadHandle *handle = NULL; + + // Find a thread that's not yet finished. + HEAD_LOCK(&_PyRuntime); + struct llist_node *node; + llist_for_each_safe(node, &state->shutdown_handles) { + ThreadHandle *cur = llist_data(node, ThreadHandle, shutdown_node); + if (cur->ident != ident) { + ThreadHandle_incref(cur); + handle = cur; + break; + } + } + HEAD_UNLOCK(&_PyRuntime); + + if (!handle) { + // No more threads to wait on! + break; + } + + // Wait for the thread to finish. If we're interrupted, such + // as by a ctrl-c we print the error and exit early. + if (ThreadHandle_join(handle, -1) < 0) { + PyErr_WriteUnraisable(NULL); + ThreadHandle_decref(handle); + Py_RETURN_NONE; + } + + ThreadHandle_decref(handle); + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(shutdown_doc, +"_shutdown($module, /)\n\ +--\n\ +\n\ +Wait for all non-daemon threads (other than the calling thread) to stop."); + +static PyObject * +thread__make_thread_handle(PyObject *module, PyObject *identobj) +{ + thread_module_state *state = get_thread_state(module); + if (!PyLong_Check(identobj)) { + PyErr_SetString(PyExc_TypeError, "ident must be an integer"); + return NULL; + } + PyThread_ident_t ident = PyLong_AsUnsignedLongLong(identobj); + if (PyErr_Occurred()) { + return NULL; + } + PyThreadHandleObject *hobj = + PyThreadHandleObject_new(state->thread_handle_type); + if (hobj == NULL) { + return NULL; + } + PyMutex_Lock(&hobj->handle->mutex); + hobj->handle->ident = ident; + hobj->handle->state = THREAD_HANDLE_RUNNING; + PyMutex_Unlock(&hobj->handle->mutex); + return (PyObject*) hobj; +} + +PyDoc_STRVAR(thread__make_thread_handle_doc, +"_make_thread_handle($module, ident, /)\n\ +--\n\ +\n\ +Internal only. Make a thread handle for threads not spawned\n\ +by the _thread or threading module."); + +static PyObject * +thread__get_main_thread_ident(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLongLong(_PyRuntime.main_thread); +} + +PyDoc_STRVAR(thread__get_main_thread_ident_doc, +"_get_main_thread_ident($module, /)\n\ +--\n\ +\n\ +Internal only. Return a non-zero integer that uniquely identifies the main thread\n\ +of the main interpreter."); + static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, - METH_VARARGS, start_new_doc}, + METH_VARARGS, start_new_thread_doc}, {"start_new", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, start_new_doc}, - {"start_joinable_thread", (PyCFunction)thread_PyThread_start_joinable_thread, - METH_O, start_joinable_doc}, + {"start_joinable_thread", _PyCFunction_CAST(thread_PyThread_start_joinable_thread), + METH_VARARGS | METH_KEYWORDS, start_joinable_doc}, {"daemon_threads_allowed", (PyCFunction)thread_daemon_threads_allowed, METH_NOARGS, daemon_threads_allowed_doc}, {"allocate_lock", thread_PyThread_allocate_lock, - METH_NOARGS, allocate_doc}, + METH_NOARGS, allocate_lock_doc}, {"allocate", thread_PyThread_allocate_lock, METH_NOARGS, allocate_doc}, {"exit_thread", thread_PyThread_exit_thread, - METH_NOARGS, exit_doc}, + METH_NOARGS, exit_thread_doc}, {"exit", thread_PyThread_exit_thread, METH_NOARGS, exit_doc}, {"interrupt_main", (PyCFunction)thread_PyThread_interrupt_main, @@ -1850,12 +2382,16 @@ static PyMethodDef thread_methods[] = { METH_NOARGS, _count_doc}, {"stack_size", (PyCFunction)thread_stack_size, METH_VARARGS, stack_size_doc}, - {"_set_sentinel", thread__set_sentinel, - METH_NOARGS, _set_sentinel_doc}, {"_excepthook", thread_excepthook, METH_O, excepthook_doc}, {"_is_main_interpreter", thread__is_main_interpreter, METH_NOARGS, thread__is_main_interpreter_doc}, + {"_shutdown", thread_shutdown, + METH_NOARGS, shutdown_doc}, + {"_make_thread_handle", thread__make_thread_handle, + METH_O, thread__make_thread_handle_doc}, + {"_get_main_thread_ident", thread__get_main_thread_ident, + METH_NOARGS, thread__get_main_thread_ident_doc}, {NULL, NULL} /* sentinel */ }; @@ -1935,7 +2471,7 @@ thread_module_exec(PyObject *module) // TIMEOUT_MAX double timeout_max = (double)PY_TIMEOUT_MAX * 1e-6; - double time_max = _PyTime_AsSecondsDouble(_PyTime_MAX); + double time_max = PyTime_AsSecondsDouble(PyTime_MAX); timeout_max = Py_MIN(timeout_max, time_max); // Round towards minus infinity timeout_max = floor(timeout_max); @@ -1945,6 +2481,8 @@ thread_module_exec(PyObject *module) return -1; } + llist_init(&state->shutdown_handles); + return 0; } @@ -1970,6 +2508,10 @@ thread_module_clear(PyObject *module) Py_CLEAR(state->local_type); Py_CLEAR(state->local_dummy_type); Py_CLEAR(state->thread_handle_type); + // Remove any remaining handles (e.g. if shutdown exited early due to + // interrupt) so that attempts to unlink the handle after our module state + // is destroyed do not crash. + clear_shutdown_handles(state); return 0; } @@ -1988,6 +2530,7 @@ The 'threading' module provides a more convenient interface."); static PyModuleDef_Slot thread_module_slots[] = { {Py_mod_exec, thread_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index f6181168a85ae1..b0b70ccb8cc3d3 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -69,6 +69,12 @@ Copyright (C) 1994 Steen Lumholt. #define USE_DEPRECATED_TOMMATH_API 1 #endif +// As suggested by https://core.tcl-lang.org/tcl/wiki?name=Migrating+C+extensions+to+Tcl+9 +#ifndef TCL_SIZE_MAX +typedef int Tcl_Size; +#define TCL_SIZE_MAX INT_MAX +#endif + #if !(defined(MS_WINDOWS) || defined(__CYGWIN__)) #define HAVE_CREATEFILEHANDLER #endif @@ -137,7 +143,7 @@ _get_tcl_lib_path(void) struct stat stat_buf; int stat_return_value; - PyObject *prefix = PySys_GetObject("prefix"); // borrowed reference + PyObject *prefix = PySys_GetObject("base_prefix"); // borrowed reference if (prefix == NULL) { return NULL; } @@ -306,6 +312,7 @@ typedef struct { int threaded; /* True if tcl_platform[threaded] */ Tcl_ThreadId thread_id; int dispatching; + PyObject *trace; /* We cannot include tclInt.h, as this is internal. So we cache interesting types here. */ const Tcl_ObjType *OldBooleanType; @@ -317,6 +324,7 @@ typedef struct { const Tcl_ObjType *BignumType; const Tcl_ObjType *ListType; const Tcl_ObjType *StringType; + const Tcl_ObjType *UTF32StringType; } TkappObject; #define Tkapp_Interp(v) (((TkappObject *) (v))->interp) @@ -485,24 +493,28 @@ unicodeFromTclString(const char *s) } static PyObject * -unicodeFromTclObj(Tcl_Obj *value) +unicodeFromTclObj(TkappObject *tkapp, Tcl_Obj *value) { - int len; + Tcl_Size len; #if USE_TCL_UNICODE - int byteorder = NATIVE_BYTEORDER; - const Tcl_UniChar *u = Tcl_GetUnicodeFromObj(value, &len); - if (sizeof(Tcl_UniChar) == 2) - return PyUnicode_DecodeUTF16((const char *)u, len * 2, - "surrogatepass", &byteorder); - else if (sizeof(Tcl_UniChar) == 4) - return PyUnicode_DecodeUTF32((const char *)u, len * 4, - "surrogatepass", &byteorder); - else - Py_UNREACHABLE(); -#else + if (value->typePtr != NULL && tkapp != NULL && + (value->typePtr == tkapp->StringType || + value->typePtr == tkapp->UTF32StringType)) + { + int byteorder = NATIVE_BYTEORDER; + const Tcl_UniChar *u = Tcl_GetUnicodeFromObj(value, &len); + if (sizeof(Tcl_UniChar) == 2) + return PyUnicode_DecodeUTF16((const char *)u, len * 2, + "surrogatepass", &byteorder); + else if (sizeof(Tcl_UniChar) == 4) + return PyUnicode_DecodeUTF32((const char *)u, len * 4, + "surrogatepass", &byteorder); + else + Py_UNREACHABLE(); + } +#endif /* USE_TCL_UNICODE */ const char *s = Tcl_GetStringFromObj(value, &len); return unicodeFromTclStringAndSize(s, len); -#endif } /*[clinic input] @@ -515,6 +527,10 @@ class _tkinter.tktimertoken "TkttObject *" "&Tktt_Type_spec" /**** Tkapp Object ****/ +#if TK_MAJOR_VERSION >= 9 +int Tcl_AppInit(Tcl_Interp *); +#endif + #ifndef WITH_APPINIT int Tcl_AppInit(Tcl_Interp *interp) @@ -570,6 +586,7 @@ Tkapp_New(const char *screenName, const char *className, TCL_GLOBAL_ONLY) != NULL; v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; + v->trace = NULL; #ifndef TCL_THREADS if (v->threaded) { @@ -586,14 +603,40 @@ Tkapp_New(const char *screenName, const char *className, } v->OldBooleanType = Tcl_GetObjType("boolean"); - v->BooleanType = Tcl_GetObjType("booleanString"); - v->ByteArrayType = Tcl_GetObjType("bytearray"); + { + Tcl_Obj *value; + int boolValue; + + /* Tcl 8.5 "booleanString" type is not registered + and is renamed to "boolean" in Tcl 9.0. + Based on approach suggested at + https://core.tcl-lang.org/tcl/info/3bb3bcf2da5b */ + value = Tcl_NewStringObj("true", -1); + Tcl_GetBooleanFromObj(NULL, value, &boolValue); + v->BooleanType = value->typePtr; + Tcl_DecrRefCount(value); + + // "bytearray" type is not registered in Tcl 9.0 + value = Tcl_NewByteArrayObj(NULL, 0); + v->ByteArrayType = value->typePtr; + Tcl_DecrRefCount(value); + } v->DoubleType = Tcl_GetObjType("double"); + /* TIP 484 suggests retrieving the "int" type without Tcl_GetObjType("int") + since it is no longer registered in Tcl 9.0. But even though Tcl 8.7 + only uses the "wideInt" type on platforms with 32-bit long, it still has + a registered "int" type, which FromObj() should recognize just in case. */ v->IntType = Tcl_GetObjType("int"); + if (v->IntType == NULL) { + Tcl_Obj *value = Tcl_NewIntObj(0); + v->IntType = value->typePtr; + Tcl_DecrRefCount(value); + } v->WideIntType = Tcl_GetObjType("wideInt"); v->BignumType = Tcl_GetObjType("bignum"); v->ListType = Tcl_GetObjType("list"); v->StringType = Tcl_GetObjType("string"); + v->UTF32StringType = Tcl_GetObjType("utf32string"); /* Delete the 'exit' command, which can screw things up */ Tcl_DeleteCommand(v->interp, "exit"); @@ -754,7 +797,7 @@ PyTclObject_string(PyObject *_self, void *ignored) { PyTclObject *self = (PyTclObject *)_self; if (!self->string) { - self->string = unicodeFromTclObj(self->value); + self->string = unicodeFromTclObj(NULL, self->value); if (!self->string) return NULL; } @@ -769,7 +812,7 @@ PyTclObject_str(PyObject *_self) return Py_NewRef(self->string); } /* XXX Could cache result if it is non-ASCII. */ - return unicodeFromTclObj(self->value); + return unicodeFromTclObj(NULL, self->value); } static PyObject * @@ -926,7 +969,8 @@ AsObj(PyObject *value) (unsigned char *)(void *)&wideValue, sizeof(wideValue), PY_LITTLE_ENDIAN, - /* signed */ 1) == 0) { + /* signed */ 1, + /* with_exceptions */ 1) == 0) { return Tcl_NewWideIntObj(wideValue); } PyErr_Clear(); @@ -974,7 +1018,9 @@ AsObj(PyObject *value) PyErr_SetString(PyExc_OverflowError, "string is too long"); return NULL; } - if (PyUnicode_IS_ASCII(value)) { + if (PyUnicode_IS_ASCII(value) && + strlen(PyUnicode_DATA(value)) == (size_t)PyUnicode_GET_LENGTH(value)) + { return Tcl_NewStringObj((const char *)PyUnicode_DATA(value), (int)size); } @@ -989,9 +1035,6 @@ AsObj(PyObject *value) "surrogatepass", NATIVE_BYTEORDER); else Py_UNREACHABLE(); -#else - encoded = _PyUnicode_AsUTF8String(value, "surrogateescape"); -#endif if (!encoded) { return NULL; } @@ -1001,12 +1044,39 @@ AsObj(PyObject *value) PyErr_SetString(PyExc_OverflowError, "string is too long"); return NULL; } -#if USE_TCL_UNICODE result = Tcl_NewUnicodeObj((const Tcl_UniChar *)PyBytes_AS_STRING(encoded), (int)(size / sizeof(Tcl_UniChar))); #else + encoded = _PyUnicode_AsUTF8String(value, "surrogateescape"); + if (!encoded) { + return NULL; + } + size = PyBytes_GET_SIZE(encoded); + if (strlen(PyBytes_AS_STRING(encoded)) != (size_t)size) { + /* The string contains embedded null characters. + * Tcl needs a null character to be represented as \xc0\x80 in + * the Modified UTF-8 encoding. Otherwise the string can be + * truncated in some internal operations. + * + * NOTE: stringlib_replace() could be used here, but optimizing + * this obscure case isn't worth it unless stringlib_replace() + * was already exposed in the C API for other reasons. */ + Py_SETREF(encoded, + PyObject_CallMethod(encoded, "replace", "y#y#", + "\0", (Py_ssize_t)1, + "\xc0\x80", (Py_ssize_t)2)); + if (!encoded) { + return NULL; + } + size = PyBytes_GET_SIZE(encoded); + } + if (size > INT_MAX) { + Py_DECREF(encoded); + PyErr_SetString(PyExc_OverflowError, "string is too long"); + return NULL; + } result = Tcl_NewStringObj(PyBytes_AS_STRING(encoded), (int)size); -#endif +#endif /* USE_TCL_UNICODE */ Py_DECREF(encoded); return result; } @@ -1103,7 +1173,7 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) Tcl_Interp *interp = Tkapp_Interp(tkapp); if (value->typePtr == NULL) { - return unicodeFromTclObj(value); + return unicodeFromTclObj(tkapp, value); } if (value->typePtr == tkapp->BooleanType || @@ -1112,7 +1182,7 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) } if (value->typePtr == tkapp->ByteArrayType) { - int size; + Tcl_Size size; char *data = (char*)Tcl_GetByteArrayFromObj(value, &size); return PyBytes_FromStringAndSize(data, size); } @@ -1121,14 +1191,6 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) return PyFloat_FromDouble(value->internalRep.doubleValue); } - if (value->typePtr == tkapp->IntType) { - long longValue; - if (Tcl_GetLongFromObj(interp, value, &longValue) == TCL_OK) - return PyLong_FromLong(longValue); - /* If there is an error in the long conversion, - fall through to wideInt handling. */ - } - if (value->typePtr == tkapp->IntType || value->typePtr == tkapp->WideIntType) { result = fromWideIntObj(tkapp, value); @@ -1146,8 +1208,8 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) } if (value->typePtr == tkapp->ListType) { - int size; - int i, status; + Tcl_Size i, size; + int status; PyObject *elem; Tcl_Obj *tcl_elem; @@ -1173,15 +1235,10 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) return result; } - if (value->typePtr == tkapp->StringType) { - return unicodeFromTclObj(value); - } - - if (tkapp->BooleanType == NULL && - strcmp(value->typePtr->name, "booleanString") == 0) { - /* booleanString type is not registered in Tcl */ - tkapp->BooleanType = value->typePtr; - return fromBoolean(tkapp, value); + if (value->typePtr == tkapp->StringType || + value->typePtr == tkapp->UTF32StringType) + { + return unicodeFromTclObj(tkapp, value); } if (tkapp->BignumType == NULL && @@ -1208,9 +1265,9 @@ typedef struct Tkapp_CallEvent { } Tkapp_CallEvent; static void -Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, int objc) +Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, Tcl_Size objc) { - int i; + Tcl_Size i; for (i = 0; i < objc; i++) Tcl_DecrRefCount(objv[i]); if (objv != objStore) @@ -1221,7 +1278,7 @@ Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, int objc) interpreter thread, which may or may not be the calling thread. */ static Tcl_Obj** -Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) +Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, Tcl_Size *pobjc) { Tcl_Obj **objv = objStore; Py_ssize_t objc = 0, i; @@ -1269,10 +1326,10 @@ Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) Tcl_IncrRefCount(objv[i]); } } - *pobjc = (int)objc; + *pobjc = (Tcl_Size)objc; return objv; finally: - Tkapp_CallDeallocArgs(objv, objStore, (int)objc); + Tkapp_CallDeallocArgs(objv, objStore, (Tcl_Size)objc); return NULL; } @@ -1281,7 +1338,7 @@ Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) static PyObject * Tkapp_UnicodeResult(TkappObject *self) { - return unicodeFromTclObj(Tcl_GetObjResult(self->interp)); + return unicodeFromTclObj(self, Tcl_GetObjResult(self->interp)); } @@ -1300,11 +1357,34 @@ Tkapp_ObjectResult(TkappObject *self) res = FromObj(self, value); Tcl_DecrRefCount(value); } else { - res = unicodeFromTclObj(value); + res = unicodeFromTclObj(self, value); } return res; } +static int +Tkapp_Trace(TkappObject *self, PyObject *args) +{ + if (args == NULL) { + return 0; + } + if (self->trace) { + PyObject *res = PyObject_CallObject(self->trace, args); + if (res == NULL) { + Py_DECREF(args); + return 0; + } + Py_DECREF(res); + } + Py_DECREF(args); + return 1; +} + +#define TRACE(_self, ARGS) do { \ + if ((_self)->trace && !Tkapp_Trace((_self), Py_BuildValue ARGS)) { \ + return NULL; \ + } \ + } while (0) /* Tkapp_CallProc is the event procedure that is executed in the context of the Tcl interpreter thread. Initially, it holds the Tcl lock, and doesn't @@ -1316,10 +1396,15 @@ Tkapp_CallProc(Tcl_Event *evPtr, int flags) Tkapp_CallEvent *e = (Tkapp_CallEvent *)evPtr; Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv; - int objc; + Tcl_Size objc; int i; ENTER_PYTHON - objv = Tkapp_CallArgs(e->args, objStore, &objc); + if (e->self->trace && !Tkapp_Trace(e->self, PyTuple_Pack(1, e->args))) { + objv = NULL; + } + else { + objv = Tkapp_CallArgs(e->args, objStore, &objc); + } if (!objv) { *(e->exc) = PyErr_GetRaisedException(); *(e->res) = NULL; @@ -1367,7 +1452,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) { Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv = NULL; - int objc, i; + Tcl_Size objc; PyObject *res = NULL; TkappObject *self = (TkappObject*)selfptr; int flags = TCL_EVAL_DIRECT | TCL_EVAL_GLOBAL; @@ -1383,7 +1468,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) marshal the parameters to the interpreter thread. */ Tkapp_CallEvent *ev; Tcl_Condition cond = NULL; - PyObject *exc; + PyObject *exc = NULL; // init to make static analyzers happy if (!WaitForMainloop(self)) return NULL; ev = (Tkapp_CallEvent*)attemptckalloc(sizeof(Tkapp_CallEvent)); @@ -1412,7 +1497,9 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) } else { + TRACE(self, ("(O)", args)); + int i; objv = Tkapp_CallArgs(args, objStore, &objc); if (!objv) return NULL; @@ -1454,6 +1541,8 @@ _tkinter_tkapp_eval_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "eval", script)); + ENTER_TCL err = Tcl_Eval(Tkapp_Interp(self), script); ENTER_OVERLAP @@ -1483,6 +1572,8 @@ _tkinter_tkapp_evalfile_impl(TkappObject *self, const char *fileName) CHECK_STRING_LENGTH(fileName); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "source", fileName)); + ENTER_TCL err = Tcl_EvalFile(Tkapp_Interp(self), fileName); ENTER_OVERLAP @@ -1512,6 +1603,8 @@ _tkinter_tkapp_record_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ssss))", "history", "add", script, "exec")); + ENTER_TCL err = Tcl_RecordAndEval(Tkapp_Interp(self), script, TCL_NO_EVAL); ENTER_OVERLAP @@ -1649,7 +1742,8 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags) TkappObject *self = (TkappObject*)selfptr; if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { VarEvent *ev; - PyObject *res, *exc; + // init 'res' and 'exc' to make static analyzers happy + PyObject *res = NULL, *exc = NULL; Tcl_Condition cond = NULL; /* The current thread is not the interpreter thread. Marshal @@ -1701,6 +1795,15 @@ SetVar(TkappObject *self, PyObject *args, int flags) newval = AsObj(newValue); if (newval == NULL) return NULL; + + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((ssssO))", "uplevel", "#0", "set", + name1, newValue)); + } + else { + TRACE((TkappObject *)self, ("((ssO))", "set", name1, newValue)); + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, NULL, newval, flags); @@ -1718,8 +1821,22 @@ SetVar(TkappObject *self, PyObject *args, int flags) return NULL; CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + /* XXX must hold tcl lock already??? */ newval = AsObj(newValue); + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((sssNO))", "uplevel", "#0", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + else { + TRACE((TkappObject *)self, ("((sNO))", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, newval, flags); ENTER_OVERLAP @@ -1773,7 +1890,7 @@ GetVar(TkappObject *self, PyObject *args, int flags) res = FromObj(self, tres); } else { - res = unicodeFromTclObj(tres); + res = unicodeFromTclObj(self, tres); } } LEAVE_OVERLAP_TCL @@ -1806,6 +1923,28 @@ UnsetVar(TkappObject *self, PyObject *args, int flags) CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + if (name2) { + TRACE((TkappObject *)self, ("((sssN))", "uplevel", "#0", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ssss))", "uplevel", "#0", "unset", name1)); + } + } + else { + if (name2) { + TRACE((TkappObject *)self, ("((sN))", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ss))", "unset", name1)); + } + } + } + ENTER_TCL code = Tcl_UnsetVar2(Tkapp_Interp(self), name1, name2, flags); ENTER_OVERLAP @@ -1972,6 +2111,8 @@ _tkinter_tkapp_exprstring_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprString(Tkapp_Interp(self), s); ENTER_OVERLAP @@ -2002,6 +2143,8 @@ _tkinter_tkapp_exprlong_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprLong(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2031,6 +2174,9 @@ _tkinter_tkapp_exprdouble_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprDouble(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2060,6 +2206,9 @@ _tkinter_tkapp_exprboolean_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprBoolean(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2086,13 +2235,12 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) /*[clinic end generated code: output=13b51d34386d36fb input=2b2e13351e3c0b53]*/ { char *list; - int argc; + Tcl_Size argc, i; const char **argv; PyObject *v; - int i; if (PyTclObject_Check(arg)) { - int objc; + Tcl_Size objc; Tcl_Obj **objv; if (Tcl_ListObjGetElements(Tkapp_Interp(self), ((PyTclObject*)arg)->value, @@ -2155,7 +2303,7 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) /* Client data struct */ typedef struct { - PyObject *self; + TkappObject *self; PyObject *func; } PythonCmd_ClientData; @@ -2179,6 +2327,7 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, PyObject *args, *res; int i; Tcl_Obj *obj_res; + int objargs = data->self->wantobjects >= 2; ENTER_PYTHON @@ -2187,7 +2336,8 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, return PythonCmd_Error(interp); for (i = 0; i < (objc - 1); i++) { - PyObject *s = unicodeFromTclObj(objv[i + 1]); + PyObject *s = objargs ? FromObj(data->self, objv[i + 1]) + : unicodeFromTclObj(data->self, objv[i + 1]); if (!s) { Py_DECREF(args); return PythonCmd_Error(interp); @@ -2285,12 +2435,17 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, !WaitForMainloop(self)) return NULL; + TRACE(self, ("((ss()O))", "proc", name, func)); + data = PyMem_NEW(PythonCmd_ClientData, 1); if (!data) return PyErr_NoMemory(); - data->self = Py_NewRef(self); + Py_INCREF(self); + data->self = self; data->func = Py_NewRef(func); if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { + err = 0; // init to make static analyzers happy + Tcl_Condition cond = NULL; CommandEvent *ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent)); if (ev == NULL) { @@ -2343,7 +2498,11 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) CHECK_STRING_LENGTH(name); + TRACE(self, ("((sss))", "rename", name, "")); + if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { + err = 0; // init to make static analyzers happy + Tcl_Condition cond = NULL; CommandEvent *ev; ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent)); @@ -2468,6 +2627,8 @@ _tkinter_tkapp_createfilehandler_impl(TkappObject *self, PyObject *file, return NULL; } + TRACE(self, ("((ssiiO))", "#", "createfilehandler", tfile, mask, func)); + data = NewFHCD(func, file, tfile); if (data == NULL) return NULL; @@ -2499,6 +2660,8 @@ _tkinter_tkapp_deletefilehandler(TkappObject *self, PyObject *file) if (tfile < 0) return NULL; + TRACE(self, ("((ssi))", "#", "deletefilehandler", tfile)); + DeleteFHCD(tfile); /* Ought to check for null Tcl_File object... */ @@ -2533,6 +2696,7 @@ _tkinter_tktimertoken_deletetimerhandler_impl(TkttObject *self) PyObject *func = v->func; if (v->token != NULL) { + /* TRACE(...) */ Tcl_DeleteTimerHandler(v->token); v->token = NULL; } @@ -2635,6 +2799,8 @@ _tkinter_tkapp_createtimerhandler_impl(TkappObject *self, int milliseconds, CHECK_TCL_APPARTMENT; + TRACE(self, ("((siO))", "after", milliseconds, func)); + v = Tktt_New(func); if (v) { v->token = Tcl_CreateTimerHandler(milliseconds, TimerHandler, @@ -2793,15 +2959,56 @@ Tkapp_WantObjects(PyObject *self, PyObject *args) { int wantobjects = -1; - if (!PyArg_ParseTuple(args, "|p:wantobjects", &wantobjects)) + if (!PyArg_ParseTuple(args, "|i:wantobjects", &wantobjects)) return NULL; if (wantobjects == -1) - return PyBool_FromLong(((TkappObject*)self)->wantobjects); + return PyLong_FromLong(((TkappObject*)self)->wantobjects); ((TkappObject*)self)->wantobjects = wantobjects; Py_RETURN_NONE; } +/*[clinic input] +_tkinter.tkapp.settrace + + func: object + / + +Set the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_settrace(TkappObject *self, PyObject *func) +/*[clinic end generated code: output=847f6ebdf46e84fa input=31b260d46d3d018a]*/ +{ + if (func == Py_None) { + func = NULL; + } + else { + Py_INCREF(func); + } + Py_XSETREF(self->trace, func); + Py_RETURN_NONE; +} + +/*[clinic input] +_tkinter.tkapp.gettrace + +Get the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self) +/*[clinic end generated code: output=d4e2ba7d63e77bb5 input=ac2aea5be74e8c4c]*/ +{ + PyObject *func = self->trace; + if (!func) { + func = Py_None; + } + Py_INCREF(func); + return func; +} + /*[clinic input] _tkinter.tkapp.willdispatch @@ -2827,6 +3034,7 @@ Tkapp_Dealloc(PyObject *self) ENTER_TCL Tcl_DeleteInterp(Tkapp_Interp(self)); LEAVE_TCL + Py_XDECREF(((TkappObject *)self)->trace); PyObject_Free(self); Py_DECREF(tp); DisableEventHook(); @@ -2940,7 +3148,7 @@ _tkinter.create baseName: str = "" className: str = "Tk" interactive: bool = False - wantobjects: bool = False + wantobjects: int = 0 wantTk: bool = True if false, then Tk_Init() doesn't get called sync: bool = False @@ -2956,7 +3164,7 @@ _tkinter_create_impl(PyObject *module, const char *screenName, const char *baseName, const char *className, int interactive, int wantobjects, int wantTk, int sync, const char *use) -/*[clinic end generated code: output=e3315607648e6bb4 input=09afef9adea70a19]*/ +/*[clinic end generated code: output=e3315607648e6bb4 input=7e382ba431bed537]*/ { /* XXX baseName is not used anymore; * try getting rid of it. */ @@ -3037,6 +3245,8 @@ static PyMethodDef Tkapp_methods[] = { _TKINTER_TKAPP_WILLDISPATCH_METHODDEF {"wantobjects", Tkapp_WantObjects, METH_VARARGS}, + _TKINTER_TKAPP_SETTRACE_METHODDEF + _TKINTER_TKAPP_GETTRACE_METHODDEF {"call", Tkapp_Call, METH_VARARGS}, _TKINTER_TKAPP_EVAL_METHODDEF _TKINTER_TKAPP_EVALFILE_METHODDEF @@ -3179,17 +3389,40 @@ DisableEventHook(void) #endif } +static int +module_clear(PyObject *Py_UNUSED(mod)) +{ + Py_CLEAR(Tkinter_TclError); + Py_CLEAR(Tkapp_Type); + Py_CLEAR(Tktt_Type); + Py_CLEAR(PyTclObject_Type); + return 0; +} + +static int +module_traverse(PyObject *Py_UNUSED(module), visitproc visit, void *arg) +{ + Py_VISIT(Tkinter_TclError); + Py_VISIT(Tkapp_Type); + Py_VISIT(Tktt_Type); + Py_VISIT(PyTclObject_Type); + return 0; +} + +static void +module_free(void *mod) +{ + (void)module_clear((PyObject *)mod); +} static struct PyModuleDef _tkintermodule = { PyModuleDef_HEAD_INIT, - "_tkinter", - NULL, - -1, - moduleMethods, - NULL, - NULL, - NULL, - NULL + .m_name = "_tkinter", + .m_size = -1, + .m_methods = moduleMethods, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free }; PyMODINIT_FUNC @@ -3204,6 +3437,9 @@ PyInit__tkinter(void) m = PyModule_Create(&_tkintermodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Tkinter_TclError = PyErr_NewException("_tkinter.TclError", NULL, NULL); if (PyModule_AddObjectRef(m, "TclError", Tkinter_TclError)) { diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 6dba3cac01c1c8..887a1e820e250e 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -219,6 +219,9 @@ PyInit__tracemalloc(void) m = PyModule_Create(&module_def); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (_PyTraceMalloc_Init() < 0) { Py_DECREF(m); diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 9ea72bf89ce0b2..09fbb3c5e8b91d 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -63,12 +63,16 @@ _typing_exec(PyObject *m) if (PyModule_AddObjectRef(m, "TypeAliasType", (PyObject *)&_PyTypeAlias_Type) < 0) { return -1; } + if (PyModule_AddObjectRef(m, "NoDefault", (PyObject *)&_Py_NoDefaultStruct) < 0) { + return -1; + } return 0; } static struct PyModuleDef_Slot _typingmodule_slots[] = { {Py_mod_exec, _typing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index 4b6852c0d0ec73..c5e78b1510b5e3 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,11 +3,10 @@ * DCE compatible Universally Unique Identifier library. */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -112,6 +111,7 @@ static PyMethodDef uuid_methods[] = { static PyModuleDef_Slot uuid_slots[] = { {Py_mod_exec, uuid_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_weakref.c b/Modules/_weakref.c index 7225dbc9ce4a1b..ecaa08ff60f203 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -14,7 +14,6 @@ module _weakref #include "clinic/_weakref.c.h" /*[clinic input] -@critical_section object _weakref.getweakrefcount -> Py_ssize_t object: object @@ -25,19 +24,14 @@ Return the number of weak references to 'object'. static Py_ssize_t _weakref_getweakrefcount_impl(PyObject *module, PyObject *object) -/*[clinic end generated code: output=301806d59558ff3e input=6535a580f1d0ebdc]*/ +/*[clinic end generated code: output=301806d59558ff3e input=7d4d04fcaccf64d5]*/ { - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) { - return 0; - } - PyWeakReference **list = GET_WEAKREFS_LISTPTR(object); - Py_ssize_t count = _PyWeakref_GetWeakrefCount(*list); - return count; + return _PyWeakref_GetWeakrefCount(object); } static int -is_dead_weakref(PyObject *value) +is_dead_weakref(PyObject *value, void *unused) { if (!PyWeakref_Check(value)) { PyErr_SetString(PyExc_TypeError, "not a weakref"); @@ -62,22 +56,14 @@ _weakref__remove_dead_weakref_impl(PyObject *module, PyObject *dct, PyObject *key) /*[clinic end generated code: output=d9ff53061fcb875c input=19fc91f257f96a1d]*/ { - if (_PyDict_DelItemIf(dct, key, is_dead_weakref) < 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) - /* This function is meant to allow safe weak-value dicts - with GC in another thread (see issue #28427), so it's - ok if the key doesn't exist anymore. - */ - PyErr_Clear(); - else - return NULL; + if (_PyDict_DelItemIf(dct, key, is_dead_weakref, NULL) < 0) { + return NULL; } Py_RETURN_NONE; } /*[clinic input] -@critical_section object _weakref.getweakrefs object: object / @@ -86,26 +72,39 @@ Return a list of all weak reference objects pointing to 'object'. [clinic start generated code]*/ static PyObject * -_weakref_getweakrefs_impl(PyObject *module, PyObject *object) -/*[clinic end generated code: output=5ec268989fb8f035 input=3dea95b8f5b31bbb]*/ +_weakref_getweakrefs(PyObject *module, PyObject *object) +/*[clinic end generated code: output=25c7731d8e011824 input=00c6d0e5d3206693]*/ { if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) { return PyList_New(0); } - PyWeakReference **list = GET_WEAKREFS_LISTPTR(object); - Py_ssize_t count = _PyWeakref_GetWeakrefCount(*list); - - PyObject *result = PyList_New(count); + PyObject *result = PyList_New(0); if (result == NULL) { return NULL; } - PyWeakReference *current = *list; - for (Py_ssize_t i = 0; i < count; ++i) { - PyList_SET_ITEM(result, i, Py_NewRef(current)); + LOCK_WEAKREFS(object); + PyWeakReference *current = *GET_WEAKREFS_LISTPTR(object); + while (current != NULL) { + PyObject *curobj = (PyObject *) current; + if (_Py_TryIncref(curobj)) { + if (PyList_Append(result, curobj)) { + UNLOCK_WEAKREFS(object); + Py_DECREF(curobj); + Py_DECREF(result); + return NULL; + } + else { + // Undo our _Py_TryIncref. This is safe to do with the lock + // held in free-threaded builds; the list holds a reference to + // curobj so we're guaranteed not to invoke the destructor. + Py_DECREF(curobj); + } + } current = current->wr_next; } + UNLOCK_WEAKREFS(object); return result; } @@ -165,6 +164,7 @@ weakref_exec(PyObject *module) static struct PyModuleDef_Slot weakref_slots[] = { {Py_mod_exec, weakref_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 5e5eb123c4ccff..a330b3ff68db62 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -72,9 +72,45 @@ #ifndef STARTF_USESHOWWINDOW #define STARTF_USESHOWWINDOW 0x00000001 #endif +#ifndef STARTF_USESIZE +#define STARTF_USESIZE 0x00000002 +#endif +#ifndef STARTF_USEPOSITION +#define STARTF_USEPOSITION 0x00000004 +#endif +#ifndef STARTF_USECOUNTCHARS +#define STARTF_USECOUNTCHARS 0x00000008 +#endif +#ifndef STARTF_USEFILLATTRIBUTE +#define STARTF_USEFILLATTRIBUTE 0x00000010 +#endif +#ifndef STARTF_RUNFULLSCREEN +#define STARTF_RUNFULLSCREEN 0x00000020 +#endif +#ifndef STARTF_FORCEONFEEDBACK +#define STARTF_FORCEONFEEDBACK 0x00000040 +#endif +#ifndef STARTF_FORCEOFFFEEDBACK +#define STARTF_FORCEOFFFEEDBACK 0x00000080 +#endif #ifndef STARTF_USESTDHANDLES #define STARTF_USESTDHANDLES 0x00000100 #endif +#ifndef STARTF_USEHOTKEY +#define STARTF_USEHOTKEY 0x00000200 +#endif +#ifndef STARTF_TITLEISLINKNAME +#define STARTF_TITLEISLINKNAME 0x00000800 +#endif +#ifndef STARTF_TITLEISAPPID +#define STARTF_TITLEISAPPID 0x00001000 +#endif +#ifndef STARTF_PREVENTPINNING +#define STARTF_PREVENTPINNING 0x00002000 +#endif +#ifndef STARTF_UNTRUSTEDSOURCE +#define STARTF_UNTRUSTEDSOURCE 0x00008000 +#endif typedef struct { PyTypeObject *overlapped_type; @@ -139,7 +175,7 @@ overlapped_dealloc(OverlappedObject *self) { /* The operation is still pending -- give a warning. This will probably only happen on Windows XP. */ - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_PythonFinalizationError, "I/O operations still in flight while destroying " "Overlapped object, the process may crash"); PyErr_WriteUnraisable(NULL); @@ -188,7 +224,6 @@ create_converter('LPCVOID', '" F_POINTER "') create_converter('BOOL', 'i') # F_BOOL used previously (always 'i') create_converter('DWORD', 'k') # F_DWORD is always "k" (which is much shorter) -create_converter('LPCTSTR', 's') create_converter('UINT', 'I') # F_UINT used previously (always 'I') class LPCWSTR_converter(Py_UNICODE_converter): @@ -223,7 +258,7 @@ class LPVOID_return_converter(CReturnConverter): data.return_conversion.append( 'return_value = HANDLE_TO_PYNUM(_return_value);\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=ef52a757a1830d92]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=da0a4db751936ee7]*/ #include "clinic/_winapi.c.h" @@ -438,6 +473,39 @@ _winapi_ConnectNamedPipe_impl(PyObject *module, HANDLE handle, Py_RETURN_NONE; } +/*[clinic input] +_winapi.CreateEventW -> HANDLE + + security_attributes: LPSECURITY_ATTRIBUTES + manual_reset: BOOL + initial_state: BOOL + name: LPCWSTR(accept={str, NoneType}) +[clinic start generated code]*/ + +static HANDLE +_winapi_CreateEventW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL manual_reset, BOOL initial_state, + LPCWSTR name) +/*[clinic end generated code: output=2d4c7d5852ecb298 input=4187cee28ac763f8]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.CreateEventW", "bbu", manual_reset, initial_state, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = CreateEventW(security_attributes, manual_reset, initial_state, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + /*[clinic input] _winapi.CreateFile -> HANDLE @@ -461,7 +529,7 @@ _winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name, { HANDLE handle; - if (PySys_Audit("_winapi.CreateFile", "uIIII", + if (PySys_Audit("_winapi.CreateFile", "ukkkk", file_name, desired_access, share_mode, creation_disposition, flags_and_attributes) < 0) { return INVALID_HANDLE_VALUE; @@ -674,10 +742,41 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, Py_RETURN_NONE; } +/*[clinic input] +_winapi.CreateMutexW -> HANDLE + + security_attributes: LPSECURITY_ATTRIBUTES + initial_owner: BOOL + name: LPCWSTR(accept={str, NoneType}) +[clinic start generated code]*/ + +static HANDLE +_winapi_CreateMutexW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL initial_owner, LPCWSTR name) +/*[clinic end generated code: output=31b9ee8fc37e49a5 input=7d54b921e723254a]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.CreateMutexW", "bu", initial_owner, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = CreateMutexW(security_attributes, initial_owner, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + /*[clinic input] _winapi.CreateNamedPipe -> HANDLE - name: LPCTSTR + name: LPCWSTR open_mode: DWORD pipe_mode: DWORD max_instances: DWORD @@ -689,25 +788,25 @@ _winapi.CreateNamedPipe -> HANDLE [clinic start generated code]*/ static HANDLE -_winapi_CreateNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD open_mode, +_winapi_CreateNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD open_mode, DWORD pipe_mode, DWORD max_instances, DWORD out_buffer_size, DWORD in_buffer_size, DWORD default_timeout, LPSECURITY_ATTRIBUTES security_attributes) -/*[clinic end generated code: output=80f8c07346a94fbc input=5a73530b84d8bc37]*/ +/*[clinic end generated code: output=7d6fde93227680ba input=5bd4e4a55639ee02]*/ { HANDLE handle; - if (PySys_Audit("_winapi.CreateNamedPipe", "uII", + if (PySys_Audit("_winapi.CreateNamedPipe", "ukk", name, open_mode, pipe_mode) < 0) { return INVALID_HANDLE_VALUE; } Py_BEGIN_ALLOW_THREADS - handle = CreateNamedPipe(name, open_mode, pipe_mode, - max_instances, out_buffer_size, - in_buffer_size, default_timeout, - security_attributes); + handle = CreateNamedPipeW(name, open_mode, pipe_mode, + max_instances, out_buffer_size, + in_buffer_size, default_timeout, + security_attributes); Py_END_ALLOW_THREADS if (handle == INVALID_HANDLE_VALUE) @@ -1453,6 +1552,49 @@ _winapi_GetLastError_impl(PyObject *module) return GetLastError(); } + +/*[clinic input] +_winapi.GetLongPathName + + path: LPCWSTR + +Return the long version of the provided path. + +If the path is already in its long form, returns the same value. + +The path must already be a 'str'. If the type is not known, use +os.fsdecode before calling this function. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetLongPathName_impl(PyObject *module, LPCWSTR path) +/*[clinic end generated code: output=c4774b080275a2d0 input=9872e211e3a4a88f]*/ +{ + DWORD cchBuffer; + PyObject *result = NULL; + + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetLongPathNameW(path, NULL, 0); + Py_END_ALLOW_THREADS + if (cchBuffer) { + WCHAR *buffer = (WCHAR *)PyMem_Malloc(cchBuffer * sizeof(WCHAR)); + if (buffer) { + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetLongPathNameW(path, buffer, cchBuffer); + Py_END_ALLOW_THREADS + if (cchBuffer) { + result = PyUnicode_FromWideChar(buffer, cchBuffer); + } else { + PyErr_SetFromWindowsErr(0); + } + PyMem_Free((void *)buffer); + } + } else { + PyErr_SetFromWindowsErr(0); + } + return result; +} + /*[clinic input] _winapi.GetModuleFileName @@ -1487,6 +1629,48 @@ _winapi_GetModuleFileName_impl(PyObject *module, HMODULE module_handle) return PyUnicode_FromWideChar(filename, wcslen(filename)); } +/*[clinic input] +_winapi.GetShortPathName + + path: LPCWSTR + +Return the short version of the provided path. + +If the path is already in its short form, returns the same value. + +The path must already be a 'str'. If the type is not known, use +os.fsdecode before calling this function. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetShortPathName_impl(PyObject *module, LPCWSTR path) +/*[clinic end generated code: output=dab6ae494c621e81 input=43fa349aaf2ac718]*/ +{ + DWORD cchBuffer; + PyObject *result = NULL; + + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetShortPathNameW(path, NULL, 0); + Py_END_ALLOW_THREADS + if (cchBuffer) { + WCHAR *buffer = (WCHAR *)PyMem_Malloc(cchBuffer * sizeof(WCHAR)); + if (buffer) { + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetShortPathNameW(path, buffer, cchBuffer); + Py_END_ALLOW_THREADS + if (cchBuffer) { + result = PyUnicode_FromWideChar(buffer, cchBuffer); + } else { + PyErr_SetFromWindowsErr(0); + } + PyMem_Free((void *)buffer); + } + } else { + PyErr_SetFromWindowsErr(0); + } + return result; +} + /*[clinic input] _winapi.GetStdHandle -> HANDLE @@ -1590,6 +1774,67 @@ _winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address) Py_RETURN_NONE; } +/*[clinic input] +_winapi.OpenEventW -> HANDLE + + desired_access: DWORD + inherit_handle: BOOL + name: LPCWSTR +[clinic start generated code]*/ + +static HANDLE +_winapi_OpenEventW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name) +/*[clinic end generated code: output=c4a45e95545a4bd2 input=dec26598748d35aa]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.OpenEventW", "ku", desired_access, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = OpenEventW(desired_access, inherit_handle, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + + +/*[clinic input] +_winapi.OpenMutexW -> HANDLE + + desired_access: DWORD + inherit_handle: BOOL + name: LPCWSTR +[clinic start generated code]*/ + +static HANDLE +_winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name) +/*[clinic end generated code: output=dda39d7844397bf0 input=f3a7b466c5307712]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.OpenMutexW", "ku", desired_access, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = OpenMutexW(desired_access, inherit_handle, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + /*[clinic input] _winapi.OpenFileMapping -> HANDLE @@ -1636,7 +1881,7 @@ _winapi_OpenProcess_impl(PyObject *module, DWORD desired_access, { HANDLE handle; - if (PySys_Audit("_winapi.OpenProcess", "II", + if (PySys_Audit("_winapi.OpenProcess", "kk", process_id, desired_access) < 0) { return INVALID_HANDLE_VALUE; } @@ -1820,6 +2065,75 @@ _winapi_ReadFile_impl(PyObject *module, HANDLE handle, DWORD size, return Py_BuildValue("NI", buf, err); } +/*[clinic input] +_winapi.ReleaseMutex + + mutex: HANDLE +[clinic start generated code]*/ + +static PyObject * +_winapi_ReleaseMutex_impl(PyObject *module, HANDLE mutex) +/*[clinic end generated code: output=5b9001a72dd8af37 input=49e9d20de3559d84]*/ +{ + int err = 0; + + Py_BEGIN_ALLOW_THREADS + if (!ReleaseMutex(mutex)) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + if (err) { + return PyErr_SetFromWindowsErr(err); + } + Py_RETURN_NONE; +} + +/*[clinic input] +_winapi.ResetEvent + + event: HANDLE +[clinic start generated code]*/ + +static PyObject * +_winapi_ResetEvent_impl(PyObject *module, HANDLE event) +/*[clinic end generated code: output=81c8501d57c0530d input=e2d42d990322e87a]*/ +{ + int err = 0; + + Py_BEGIN_ALLOW_THREADS + if (!ResetEvent(event)) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + if (err) { + return PyErr_SetFromWindowsErr(err); + } + Py_RETURN_NONE; +} + +/*[clinic input] +_winapi.SetEvent + + event: HANDLE +[clinic start generated code]*/ + +static PyObject * +_winapi_SetEvent_impl(PyObject *module, HANDLE event) +/*[clinic end generated code: output=c18ba09eb9aa774d input=e660e830a37c09f8]*/ +{ + int err = 0; + + Py_BEGIN_ALLOW_THREADS + if (!SetEvent(event)) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + if (err) { + return PyErr_SetFromWindowsErr(err); + } + Py_RETURN_NONE; +} + /*[clinic input] _winapi.SetNamedPipeHandleState @@ -1921,19 +2235,19 @@ _winapi_VirtualQuerySize_impl(PyObject *module, LPCVOID address) /*[clinic input] _winapi.WaitNamedPipe - name: LPCTSTR + name: LPCWSTR timeout: DWORD / [clinic start generated code]*/ static PyObject * -_winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout) -/*[clinic end generated code: output=c2866f4439b1fe38 input=36fc781291b1862c]*/ +_winapi_WaitNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD timeout) +/*[clinic end generated code: output=e161e2e630b3e9c2 input=099a4746544488fa]*/ { BOOL success; Py_BEGIN_ALLOW_THREADS - success = WaitNamedPipe(name, timeout); + success = WaitNamedPipeW(name, timeout); Py_END_ALLOW_THREADS if (!success) @@ -1942,6 +2256,310 @@ _winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout) Py_RETURN_NONE; } + +typedef struct { + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + HANDLE cancel_event; + DWORD handle_base; + DWORD handle_count; + HANDLE thread; + volatile DWORD result; +} BatchedWaitData; + +static DWORD WINAPI +_batched_WaitForMultipleObjects_thread(LPVOID param) +{ + BatchedWaitData *data = (BatchedWaitData *)param; + data->result = WaitForMultipleObjects( + data->handle_count, + data->handles, + FALSE, + INFINITE + ); + if (data->result == WAIT_FAILED) { + DWORD err = GetLastError(); + SetEvent(data->cancel_event); + return err; + } else if (data->result >= WAIT_ABANDONED_0 && data->result < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) { + data->result = WAIT_FAILED; + SetEvent(data->cancel_event); + return ERROR_ABANDONED_WAIT_0; + } + return 0; +} + +/*[clinic input] +_winapi.BatchedWaitForMultipleObjects + + handle_seq: object + wait_all: BOOL + milliseconds: DWORD(c_default='INFINITE') = _winapi.INFINITE + +Supports a larger number of handles than WaitForMultipleObjects + +Note that the handles may be waited on other threads, which could cause +issues for objects like mutexes that become associated with the thread +that was waiting for them. Objects may also be left signalled, even if +the wait fails. + +It is recommended to use WaitForMultipleObjects whenever possible, and +only switch to BatchedWaitForMultipleObjects for scenarios where you +control all the handles involved, such as your own thread pool or +files, and all wait objects are left unmodified by a wait (for example, +manual reset events, threads, and files/pipes). + +Overlapped handles returned from this module use manual reset events. +[clinic start generated code]*/ + +static PyObject * +_winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, + PyObject *handle_seq, + BOOL wait_all, DWORD milliseconds) +/*[clinic end generated code: output=d21c1a4ad0a252fd input=7e196f29005dc77b]*/ +{ + Py_ssize_t thread_count = 0, handle_count = 0, i, j; + Py_ssize_t nhandles; + BatchedWaitData *thread_data[MAXIMUM_WAIT_OBJECTS]; + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + HANDLE sigint_event = NULL; + HANDLE cancel_event = NULL; + DWORD result; + + const Py_ssize_t _MAXIMUM_TOTAL_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) * (MAXIMUM_WAIT_OBJECTS - 1); + + if (!PySequence_Check(handle_seq)) { + PyErr_Format(PyExc_TypeError, + "sequence type expected, got '%s'", + Py_TYPE(handle_seq)->tp_name); + return NULL; + } + nhandles = PySequence_Length(handle_seq); + if (nhandles == -1) { + return NULL; + } + if (nhandles == 0) { + return wait_all ? Py_NewRef(Py_None) : PyList_New(0); + } + + /* If this is the main thread then make the wait interruptible + by Ctrl-C. When waiting for *all* handles, it is only checked + in between batches. */ + if (_PyOS_IsMainThread()) { + sigint_event = _PyOS_SigintEvent(); + assert(sigint_event != NULL); + } + + if (nhandles < 0 || nhandles > _MAXIMUM_TOTAL_OBJECTS) { + PyErr_Format(PyExc_ValueError, + "need at most %zd handles, got a sequence of length %zd", + _MAXIMUM_TOTAL_OBJECTS, nhandles); + return NULL; + } + + if (!wait_all) { + cancel_event = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!cancel_event) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, 0); + return NULL; + } + } + + i = 0; + while (i < nhandles) { + BatchedWaitData *data = (BatchedWaitData*)PyMem_Malloc(sizeof(BatchedWaitData)); + if (!data) { + goto error; + } + thread_data[thread_count++] = data; + data->thread = NULL; + data->cancel_event = cancel_event; + data->handle_base = Py_SAFE_DOWNCAST(i, Py_ssize_t, DWORD); + data->handle_count = Py_SAFE_DOWNCAST(nhandles - i, Py_ssize_t, DWORD); + if (data->handle_count > MAXIMUM_WAIT_OBJECTS - 1) { + data->handle_count = MAXIMUM_WAIT_OBJECTS - 1; + } + for (j = 0; j < data->handle_count; ++i, ++j) { + PyObject *v = PySequence_GetItem(handle_seq, i); + if (!v || !PyArg_Parse(v, F_HANDLE, &data->handles[j])) { + Py_XDECREF(v); + goto error; + } + Py_DECREF(v); + } + if (!wait_all) { + data->handles[data->handle_count++] = cancel_event; + } + } + + DWORD err = 0; + + /* We need to use different strategies when waiting for ALL handles + as opposed to ANY handle. This is because there is no way to + (safely) interrupt a thread that is waiting for all handles in a + group. So for ALL handles, we loop over each set and wait. For + ANY handle, we use threads and wait on them. */ + if (wait_all) { + Py_BEGIN_ALLOW_THREADS + long long deadline = 0; + if (milliseconds != INFINITE) { + deadline = (long long)GetTickCount64() + milliseconds; + } + + for (i = 0; !err && i < thread_count; ++i) { + DWORD timeout = milliseconds; + if (deadline) { + long long time_to_deadline = deadline - GetTickCount64(); + if (time_to_deadline <= 0) { + err = WAIT_TIMEOUT; + break; + } else if (time_to_deadline < UINT_MAX) { + timeout = (DWORD)time_to_deadline; + } + } + result = WaitForMultipleObjects(thread_data[i]->handle_count, + thread_data[i]->handles, TRUE, timeout); + // ABANDONED is not possible here because we own all the handles + if (result == WAIT_FAILED) { + err = GetLastError(); + } else if (result == WAIT_TIMEOUT) { + err = WAIT_TIMEOUT; + } + + if (!err && sigint_event) { + result = WaitForSingleObject(sigint_event, 0); + if (result == WAIT_OBJECT_0) { + err = ERROR_CONTROL_C_EXIT; + } else if (result == WAIT_FAILED) { + err = GetLastError(); + } + } + } + + CloseHandle(cancel_event); + + Py_END_ALLOW_THREADS + } else { + Py_BEGIN_ALLOW_THREADS + + for (i = 0; i < thread_count; ++i) { + BatchedWaitData *data = thread_data[i]; + data->thread = CreateThread( + NULL, + 1, // smallest possible initial stack + _batched_WaitForMultipleObjects_thread, + (LPVOID)data, + CREATE_SUSPENDED, + NULL + ); + if (!data->thread) { + err = GetLastError(); + break; + } + handles[handle_count++] = data->thread; + } + Py_END_ALLOW_THREADS + + if (err) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, err); + goto error; + } + if (handle_count > MAXIMUM_WAIT_OBJECTS - 1) { + // basically an assert, but stronger + PyErr_SetString(PyExc_SystemError, "allocated too many wait objects"); + goto error; + } + + Py_BEGIN_ALLOW_THREADS + + // Once we start resuming threads, can no longer "goto error" + for (i = 0; i < thread_count; ++i) { + ResumeThread(thread_data[i]->thread); + } + if (sigint_event) { + handles[handle_count++] = sigint_event; + } + result = WaitForMultipleObjects((DWORD)handle_count, handles, wait_all, milliseconds); + // ABANDONED is not possible here because we own all the handles + if (result == WAIT_FAILED) { + err = GetLastError(); + } else if (result == WAIT_TIMEOUT) { + err = WAIT_TIMEOUT; + } else if (sigint_event && result == WAIT_OBJECT_0 + handle_count) { + err = ERROR_CONTROL_C_EXIT; + } + + SetEvent(cancel_event); + + // Wait for all threads to finish before we start freeing their memory + if (sigint_event) { + handle_count -= 1; + } + WaitForMultipleObjects((DWORD)handle_count, handles, TRUE, INFINITE); + + for (i = 0; i < thread_count; ++i) { + if (!err && thread_data[i]->result == WAIT_FAILED) { + if (!GetExitCodeThread(thread_data[i]->thread, &err)) { + err = GetLastError(); + } + } + CloseHandle(thread_data[i]->thread); + } + + CloseHandle(cancel_event); + + Py_END_ALLOW_THREADS + + } + + PyObject *triggered_indices; + if (sigint_event != NULL && err == ERROR_CONTROL_C_EXIT) { + errno = EINTR; + PyErr_SetFromErrno(PyExc_OSError); + triggered_indices = NULL; + } else if (err) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, err); + triggered_indices = NULL; + } else if (wait_all) { + triggered_indices = Py_NewRef(Py_None); + } else { + triggered_indices = PyList_New(0); + if (triggered_indices) { + for (i = 0; i < thread_count; ++i) { + Py_ssize_t triggered = (Py_ssize_t)thread_data[i]->result - WAIT_OBJECT_0; + if (triggered >= 0 && triggered < thread_data[i]->handle_count - 1) { + PyObject *v = PyLong_FromSsize_t(thread_data[i]->handle_base + triggered); + if (!v || PyList_Append(triggered_indices, v) < 0) { + Py_XDECREF(v); + Py_CLEAR(triggered_indices); + break; + } + Py_DECREF(v); + } + } + } + } + + for (i = 0; i < thread_count; ++i) { + PyMem_Free((void *)thread_data[i]); + } + + return triggered_indices; + +error: + // We should only enter here before any threads start running. + // Once we start resuming threads, different cleanup is required + CloseHandle(cancel_event); + while (--thread_count >= 0) { + HANDLE t = thread_data[thread_count]->thread; + if (t) { + TerminateThread(t, WAIT_ABANDONED_0); + CloseHandle(t); + } + PyMem_Free((void *)thread_data[thread_count]); + } + return NULL; +} + /*[clinic input] _winapi.WaitForMultipleObjects @@ -2185,7 +2803,7 @@ _winapi__mimetypes_read_windows_registry_impl(PyObject *module, } err = RegOpenKeyExW(hkcr, ext, 0, KEY_READ, &subkey); - if (err == ERROR_FILE_NOT_FOUND) { + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_ACCESS_DENIED) { err = ERROR_SUCCESS; continue; } else if (err != ERROR_SUCCESS) { @@ -2298,7 +2916,7 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, HRESULT hr; COPYFILE2_EXTENDED_PARAMETERS params = { sizeof(COPYFILE2_EXTENDED_PARAMETERS) }; - if (PySys_Audit("_winapi.CopyFile2", "uuI", + if (PySys_Audit("_winapi.CopyFile2", "uuk", existing_file_name, new_file_name, flags) < 0) { return NULL; } @@ -2335,8 +2953,10 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF _WINAPI_CONNECTNAMEDPIPE_METHODDEF + _WINAPI_CREATEEVENTW_METHODDEF _WINAPI_CREATEFILE_METHODDEF _WINAPI_CREATEFILEMAPPING_METHODDEF + _WINAPI_CREATEMUTEXW_METHODDEF _WINAPI_CREATENAMEDPIPE_METHODDEF _WINAPI_CREATEPIPE_METHODDEF _WINAPI_CREATEPROCESS_METHODDEF @@ -2346,21 +2966,29 @@ static PyMethodDef winapi_functions[] = { _WINAPI_GETCURRENTPROCESS_METHODDEF _WINAPI_GETEXITCODEPROCESS_METHODDEF _WINAPI_GETLASTERROR_METHODDEF + _WINAPI_GETLONGPATHNAME_METHODDEF _WINAPI_GETMODULEFILENAME_METHODDEF + _WINAPI_GETSHORTPATHNAME_METHODDEF _WINAPI_GETSTDHANDLE_METHODDEF _WINAPI_GETVERSION_METHODDEF _WINAPI_MAPVIEWOFFILE_METHODDEF + _WINAPI_OPENEVENTW_METHODDEF _WINAPI_OPENFILEMAPPING_METHODDEF + _WINAPI_OPENMUTEXW_METHODDEF _WINAPI_OPENPROCESS_METHODDEF _WINAPI_PEEKNAMEDPIPE_METHODDEF _WINAPI_LCMAPSTRINGEX_METHODDEF _WINAPI_READFILE_METHODDEF + _WINAPI_RELEASEMUTEX_METHODDEF + _WINAPI_RESETEVENT_METHODDEF + _WINAPI_SETEVENT_METHODDEF _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF _WINAPI_TERMINATEPROCESS_METHODDEF _WINAPI_UNMAPVIEWOFFILE_METHODDEF _WINAPI_VIRTUALQUERYSIZE_METHODDEF _WINAPI_WAITNAMEDPIPE_METHODDEF _WINAPI_WAITFORMULTIPLEOBJECTS_METHODDEF + _WINAPI_BATCHEDWAITFORMULTIPLEOBJECTS_METHODDEF _WINAPI_WAITFORSINGLEOBJECT_METHODDEF _WINAPI_WRITEFILE_METHODDEF _WINAPI_GETACP_METHODDEF @@ -2468,7 +3096,19 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, SEC_RESERVE); WINAPI_CONSTANT(F_DWORD, SEC_WRITECOMBINE); WINAPI_CONSTANT(F_DWORD, STARTF_USESHOWWINDOW); + WINAPI_CONSTANT(F_DWORD, STARTF_USESIZE); + WINAPI_CONSTANT(F_DWORD, STARTF_USEPOSITION); + WINAPI_CONSTANT(F_DWORD, STARTF_USECOUNTCHARS); + WINAPI_CONSTANT(F_DWORD, STARTF_USEFILLATTRIBUTE); + WINAPI_CONSTANT(F_DWORD, STARTF_RUNFULLSCREEN); + WINAPI_CONSTANT(F_DWORD, STARTF_FORCEONFEEDBACK); + WINAPI_CONSTANT(F_DWORD, STARTF_FORCEOFFFEEDBACK); WINAPI_CONSTANT(F_DWORD, STARTF_USESTDHANDLES); + WINAPI_CONSTANT(F_DWORD, STARTF_USEHOTKEY); + WINAPI_CONSTANT(F_DWORD, STARTF_TITLEISLINKNAME); + WINAPI_CONSTANT(F_DWORD, STARTF_TITLEISAPPID); + WINAPI_CONSTANT(F_DWORD, STARTF_PREVENTPINNING); + WINAPI_CONSTANT(F_DWORD, STARTF_UNTRUSTEDSOURCE); WINAPI_CONSTANT(F_DWORD, STD_INPUT_HANDLE); WINAPI_CONSTANT(F_DWORD, STD_OUTPUT_HANDLE); WINAPI_CONSTANT(F_DWORD, STD_ERROR_HANDLE); @@ -2526,6 +3166,11 @@ static int winapi_exec(PyObject *m) #define COPY_FILE_REQUEST_COMPRESSED_TRAFFIC 0x10000000 #endif WINAPI_CONSTANT(F_DWORD, COPY_FILE_REQUEST_COMPRESSED_TRAFFIC); +#ifndef COPY_FILE_DIRECTORY + // Only defined in newer WinSDKs + #define COPY_FILE_DIRECTORY 0x00000080 +#endif + WINAPI_CONSTANT(F_DWORD, COPY_FILE_DIRECTORY); WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_CHUNK_STARTED); WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_CHUNK_FINISHED); @@ -2548,6 +3193,7 @@ static int winapi_exec(PyObject *m) static PyModuleDef_Slot winapi_slots[] = { {Py_mod_exec, winapi_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_xxtestfuzz/README.rst b/Modules/_xxtestfuzz/README.rst index b951858458c82f..68d5d589d2a551 100644 --- a/Modules/_xxtestfuzz/README.rst +++ b/Modules/_xxtestfuzz/README.rst @@ -23,7 +23,7 @@ Add the test name on a new line in ``fuzz_tests.txt``. In ``fuzzer.c``, add a function to be run:: - int $test_name (const char* data, size_t size) { + static int $fuzz_test_name(const char* data, size_t size) { ... return 0; } @@ -31,10 +31,12 @@ In ``fuzzer.c``, add a function to be run:: And invoke it from ``LLVMFuzzerTestOneInput``:: - #if _Py_FUZZ_YES(fuzz_builtin_float) - rv |= _run_fuzz(data, size, fuzz_builtin_float); + #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_$fuzz_test_name) + rv |= _run_fuzz(data, size, $fuzz_test_name); #endif +Don't forget to replace ``$fuzz_test_name`` with your actual test name. + ``LLVMFuzzerTestOneInput`` will run in oss-fuzz, with each test in ``fuzz_tests.txt`` run separately. diff --git a/Modules/_xxtestfuzz/_xxtestfuzz.c b/Modules/_xxtestfuzz/_xxtestfuzz.c index a2dbabce71ed67..2952d7043e01fe 100644 --- a/Modules/_xxtestfuzz/_xxtestfuzz.c +++ b/Modules/_xxtestfuzz/_xxtestfuzz.c @@ -28,13 +28,18 @@ static PyMethodDef module_methods[] = { {NULL}, }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef _fuzzmodule = { PyModuleDef_HEAD_INIT, "_fuzz", NULL, 0, module_methods, - NULL, + module_slots, NULL, NULL, NULL @@ -43,5 +48,5 @@ static struct PyModuleDef _fuzzmodule = { PyMODINIT_FUNC PyInit__xxtestfuzz(void) { - return PyModule_Create(&_fuzzmodule); + return PyModuleDef_Init(&_fuzzmodule); } diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index 6ea9f64d628530..a04f1412eefda1 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -15,7 +15,6 @@ #endif #include -#include "pycore_pyhash.h" // _Py_HashBytes() #include #include @@ -45,7 +44,7 @@ static int fuzz_builtin_int(const char* data, size_t size) { /* Pick a random valid base. (When the fuzzed function takes extra parameters, it's somewhat normal to hash the input to generate those parameters. We want to exercise all code paths, so we do so here.) */ - int base = _Py_HashBytes(data, size) % 37; + int base = Py_HashBuffer(data, size) % 37; if (base == 1) { // 1 is the only number between 0 and 36 that is not a valid base. base = 0; diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index fcd4af64df0be9..c5292575c22f23 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -3,6 +3,7 @@ #endif #include "Python.h" +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() @@ -298,15 +299,20 @@ get_weak_cache(zoneinfo_state *state, PyTypeObject *type) } } +/*[clinic input] +@critical_section +@classmethod +zoneinfo.ZoneInfo.__new__ + + key: object + +Create a new ZoneInfo instance. +[clinic start generated code]*/ + static PyObject * -zoneinfo_new(PyTypeObject *type, PyObject *args, PyObject *kw) +zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key) +/*[clinic end generated code: output=95e61dab86bb95c3 input=ef73d7a83bf8790e]*/ { - PyObject *key = NULL; - static char *kwlist[] = {"key", NULL}; - if (PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &key) == 0) { - return NULL; - } - zoneinfo_state *state = zoneinfo_get_state_by_self(type); PyObject *instance = zone_from_strong_cache(state, type, key); if (instance != NULL || PyErr_Occurred()) { @@ -467,6 +473,7 @@ zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyTypeObject *cls, } /*[clinic input] +@critical_section @classmethod zoneinfo.ZoneInfo.clear_cache @@ -481,7 +488,7 @@ Clear the ZoneInfo cache. static PyObject * zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, PyObject *only_keys) -/*[clinic end generated code: output=114d9b7c8a22e660 input=e32ca3bb396788ba]*/ +/*[clinic end generated code: output=114d9b7c8a22e660 input=35944715df26d24e]*/ { zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); PyObject *weak_cache = get_weak_cache(state, type); @@ -816,14 +823,10 @@ zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, /*[clinic end generated code: output=556712fc709deecb input=6ac8c73eed3de316]*/ { if (from_cache) { - PyObject *val_args = PyTuple_Pack(1, key); - if (val_args == NULL) { - return NULL; - } - - PyObject *rv = zoneinfo_new(type, val_args, NULL); - - Py_DECREF(val_args); + PyObject *rv; + Py_BEGIN_CRITICAL_SECTION(type); + rv = zoneinfo_ZoneInfo_impl(type, key); + Py_END_CRITICAL_SECTION(); return rv; } else { @@ -858,8 +861,7 @@ load_timedelta(zoneinfo_state *state, long seconds) 0, seconds, 0, 1, PyDateTimeAPI->DeltaType); if (tmp != NULL) { - rv = PyDict_SetDefault(state->TIMEDELTA_CACHE, pyoffset, tmp); - Py_XINCREF(rv); + PyDict_SetDefaultRef(state->TIMEDELTA_CACHE, pyoffset, tmp, &rv); Py_DECREF(tmp); } } @@ -944,6 +946,7 @@ ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1) static int load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) { + int rv = 0; PyObject *data_tuple = NULL; long *utcoff = NULL; @@ -1220,7 +1223,6 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } } - int rv = 0; goto cleanup; error: // These resources only need to be freed if we have failed, if we succeed @@ -2368,6 +2370,7 @@ strong_cache_free(StrongCacheNode *root) static void remove_from_strong_cache(zoneinfo_state *state, StrongCacheNode *node) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); if (state->ZONEINFO_STRONG_CACHE == node) { state->ZONEINFO_STRONG_CACHE = node->next; } @@ -2422,6 +2425,7 @@ eject_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, return 0; } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; StrongCacheNode *node = find_in_strong_cache(cache, key); if (node != NULL) { @@ -2478,6 +2482,7 @@ zone_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, return NULL; // Strong cache currently only implemented for base class } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; StrongCacheNode *node = find_in_strong_cache(cache, key); @@ -2504,6 +2509,7 @@ update_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, return; } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); StrongCacheNode *new_node = strong_cache_node_new(key, zone); if (new_node == NULL) { return; @@ -2631,7 +2637,7 @@ static PyType_Slot zoneinfo_slots[] = { {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_methods, zoneinfo_methods}, {Py_tp_members, zoneinfo_members}, - {Py_tp_new, zoneinfo_new}, + {Py_tp_new, zoneinfo_ZoneInfo}, {Py_tp_dealloc, zoneinfo_dealloc}, {Py_tp_traverse, zoneinfo_traverse}, {Py_tp_clear, zoneinfo_clear}, @@ -2760,6 +2766,7 @@ zoneinfomodule_exec(PyObject *m) static PyModuleDef_Slot zoneinfomodule_slots[] = { {Py_mod_exec, zoneinfomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index b97ade6126fa08..b80c964f20d65e 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -247,7 +247,7 @@ BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; if (i >= 0) - ((char *)ap->ob_item)[i] = x; + ((unsigned char *)ap->ob_item)[i] = x; return 0; } @@ -260,20 +260,32 @@ u_getitem(arrayobject *ap, Py_ssize_t i) static int u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - PyObject *u; - if (!PyArg_Parse(v, "U;array item must be unicode character", &u)) { + if (!PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, not %T", + v); return -1; } - Py_ssize_t len = PyUnicode_AsWideChar(u, NULL, 0); + Py_ssize_t len = PyUnicode_AsWideChar(v, NULL, 0); if (len != 2) { - PyErr_SetString(PyExc_TypeError, - "array item must be unicode character"); + if (PyUnicode_GET_LENGTH(v) != 1) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + } + else { + PyErr_Format(PyExc_TypeError, + "string %A cannot be converted to " + "a single wchar_t character", + v); + } return -1; } wchar_t w; - len = PyUnicode_AsWideChar(u, &w, 1); + len = PyUnicode_AsWideChar(v, &w, 1); assert(len == 1); if (i >= 0) { @@ -291,19 +303,23 @@ w_getitem(arrayobject *ap, Py_ssize_t i) static int w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - PyObject *u; - if (!PyArg_Parse(v, "U;array item must be unicode character", &u)) { + if (!PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, not %T", + v); return -1; } - if (PyUnicode_GetLength(u) != 1) { - PyErr_SetString(PyExc_TypeError, - "array item must be unicode character"); + if (PyUnicode_GET_LENGTH(v) != 1) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); return -1; } if (i >= 0) { - ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(u, 0); + ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } @@ -868,6 +884,21 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) return (PyObject *)np; } +/*[clinic input] +array.array.clear + +Remove all items from the array. +[clinic start generated code]*/ + +static PyObject * +array_array_clear_impl(arrayobject *self) +/*[clinic end generated code: output=5efe0417062210a9 input=5dffa30e94e717a4]*/ +{ + if (array_resize(self, 0) == -1) { + return NULL; + } + Py_RETURN_NONE; +} /*[clinic input] array.array.__copy__ @@ -2342,6 +2373,7 @@ static PyMethodDef array_methods[] = { ARRAY_ARRAY_APPEND_METHODDEF ARRAY_ARRAY_BUFFER_INFO_METHODDEF ARRAY_ARRAY_BYTESWAP_METHODDEF + ARRAY_ARRAY_CLEAR_METHODDEF ARRAY_ARRAY___COPY___METHODDEF ARRAY_ARRAY_COUNT_METHODDEF ARRAY_ARRAY___DEEPCOPY___METHODDEF @@ -2831,7 +2863,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ -an array of basic values: characters, integers, floating point\n\ +an array of basic values: characters, integers, floating-point\n\ numbers. Arrays are sequence types and behave very much like lists,\n\ except that the type of objects stored in them is constrained.\n"); @@ -2859,8 +2891,8 @@ The following type codes are defined:\n\ 'L' unsigned integer 4\n\ 'q' signed integer 8 (see note)\n\ 'Q' unsigned integer 8 (see note)\n\ - 'f' floating point 4\n\ - 'd' floating point 8\n\ + 'f' floating-point 4\n\ + 'd' floating-point 8\n\ \n\ NOTE: The 'u' typecode corresponds to Python's unicode character. On\n\ narrow builds this is 2-bytes on wide builds this is 4-bytes.\n\ @@ -3204,6 +3236,7 @@ array_modexec(PyObject *m) static PyModuleDef_Slot arrayslots[] = { {Py_mod_exec, array_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index b6f1bcbca67916..297a8d74ba3bf4 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -164,7 +164,8 @@ _PyAtExit_Call(PyInterpreterState *interp) PyDoc_STRVAR(atexit_register__doc__, -"register(func, *args, **kwargs) -> func\n\ +"register($module, func, /, *args, **kwargs)\n\ +--\n\ \n\ Register a function to be executed upon normal program termination\n\ \n\ @@ -221,7 +222,8 @@ atexit_register(PyObject *module, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(atexit_run_exitfuncs__doc__, -"_run_exitfuncs() -> None\n\ +"_run_exitfuncs($module, /)\n\ +--\n\ \n\ Run all registered exit functions.\n\ \n\ @@ -236,7 +238,8 @@ atexit_run_exitfuncs(PyObject *module, PyObject *unused) } PyDoc_STRVAR(atexit_clear__doc__, -"_clear() -> None\n\ +"_clear($module, /)\n\ +--\n\ \n\ Clear the list of previously registered exit functions."); @@ -248,7 +251,8 @@ atexit_clear(PyObject *module, PyObject *unused) } PyDoc_STRVAR(atexit_ncallbacks__doc__, -"_ncallbacks() -> int\n\ +"_ncallbacks($module, /)\n\ +--\n\ \n\ Return the number of registered exit functions."); @@ -260,7 +264,8 @@ atexit_ncallbacks(PyObject *module, PyObject *unused) } PyDoc_STRVAR(atexit_unregister__doc__, -"unregister(func) -> None\n\ +"unregister($module, func, /)\n\ +--\n\ \n\ Unregister an exit function which was previously registered using\n\ atexit.register\n\ @@ -317,6 +322,7 @@ Two public functions, register and unregister, are defined.\n\ static PyModuleDef_Slot atexitmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/binascii.c b/Modules/binascii.c index 86493241a1fb7e..6bb01d148b6faa 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -414,6 +414,13 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) if (this_ch == BASE64_PAD) { padding_started = 1; + if (strict_mode && quad_pos == 0) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Excess padding not allowed"); + } + goto error_end; + } if (quad_pos >= 2 && quad_pos + ++pads >= 4) { /* A pad sequence means we should not parse more input. ** We've already interpreted the data from the quad at this point. @@ -1278,6 +1285,7 @@ binascii_exec(PyObject *module) static PyModuleDef_Slot binascii_slots[] = { {Py_mod_exec, binascii_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/blake2module.c b/Modules/blake2module.c new file mode 100644 index 00000000000000..1ec676c34c6128 --- /dev/null +++ b/Modules/blake2module.c @@ -0,0 +1,948 @@ +/* + * Written in 2013 by Dmitry Chestnykh + * Modified for CPython by Christian Heimes + * Updated to use HACL* by Jonathan Protzenko + * + * To the extent possible under law, the author have dedicated all + * copyright and related and neighboring rights to this software to + * the public domain worldwide. This software is distributed without + * any warranty. http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "pyconfig.h" +#include "Python.h" +#include "hashlib.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" +#include "pycore_moduleobject.h" + +// QUICK CPU AUTODETECTION +// +// See https://github.com/python/cpython/pull/119316 -- we only enable +// vectorized versions for Intel CPUs, even though HACL*'s "vec128" modules also +// run on ARM NEON. (We could enable them on POWER -- but I don't have access to +// a test machine to see if that speeds anything up.) +// +// Note that configure.ac and the rest of the build are written in such a way +// that if the configure script finds suitable flags to compile HACL's SIMD128 +// (resp. SIMD256) files, then Hacl_Hash_Blake2b_Simd128.c (resp. ...) will be +// pulled into the build automatically, and then only the CPU autodetection will +// need to be updated here. + +#if defined(__x86_64__) && defined(__GNUC__) +#include +#elif defined(_M_X64) +#include +#endif + +#include + +// SIMD256 can't be compiled on macOS ARM64, and performance of SIMD128 isn't +// great; but when compiling a universal2 binary, autoconf will set +// HACL_CAN_COMPILE_SIMD128 and HACL_CAN_COMPILE_SIMD256 because they *can* be +// compiled on x86_64. If we're on macOS ARM64, disable these preprocessor +// symbols. +#if defined(__APPLE__) && defined(__arm64__) +# undef HACL_CAN_COMPILE_SIMD128 +# undef HACL_CAN_COMPILE_SIMD256 +#endif + +// ECX +#define ECX_SSE3 (1 << 0) +#define ECX_SSSE3 (1 << 9) +#define ECX_SSE4_1 (1 << 19) +#define ECX_SSE4_2 (1 << 20) +#define ECX_AVX (1 << 28) + +// EBX +#define EBX_AVX2 (1 << 5) + +// EDX +#define EDX_SSE (1 << 25) +#define EDX_SSE2 (1 << 26) +#define EDX_CMOV (1 << 15) + +// zero-initialized by default +typedef struct { + bool sse, sse2, sse3, sse41, sse42, cmov, avx, avx2; + bool done; +} cpu_flags; + +void detect_cpu_features(cpu_flags *flags) { + if (!flags->done) { + int eax1 = 0, ebx1 = 0, ecx1 = 0, edx1 = 0; + int eax7 = 0, ebx7 = 0, ecx7 = 0, edx7 = 0; +#if defined(__x86_64__) && defined(__GNUC__) + __cpuid_count(1, 0, eax1, ebx1, ecx1, edx1); + __cpuid_count(7, 0, eax7, ebx7, ecx7, edx7); +#elif defined(_M_X64) + int info1[4] = { 0 }; + int info7[4] = { 0 }; + __cpuidex(info1, 1, 0); + __cpuidex(info7, 7, 0); + eax1 = info1[0]; + ebx1 = info1[1]; + ecx1 = info1[2]; + edx1 = info1[3]; + eax7 = info7[0]; + ebx7 = info7[1]; + ecx7 = info7[2]; + edx7 = info7[3]; +#else + (void) eax1; (void) ebx1; (void) ecx1; (void) edx1; + (void) eax7; (void) ebx7; (void) ecx7; (void) edx7; +#endif + + flags->avx = (ecx1 & ECX_AVX) != 0; + + flags->avx2 = (ebx7 & EBX_AVX2) != 0; + + flags->sse = (edx1 & EDX_SSE) != 0; + flags->sse2 = (edx1 & EDX_SSE2) != 0; + flags->cmov = (edx1 & EDX_CMOV) != 0; + + flags->sse3 = (ecx1 & ECX_SSE3) != 0; + /* ssse3 = (ecx1 & ECX_SSSE3) != 0; */ + flags->sse41 = (ecx1 & ECX_SSE4_1) != 0; + flags->sse42 = (ecx1 & ECX_SSE4_2) != 0; + + flags->done = true; + } +} + +#ifdef HACL_CAN_COMPILE_SIMD128 +static inline bool has_simd128(cpu_flags *flags) { + // For now this is Intel-only, could conceivably be #ifdef'd to something + // else. + return flags->sse && flags->sse2 && flags->sse3 && flags->sse41 && flags->sse42 && flags->cmov; +} +#endif + +#ifdef HACL_CAN_COMPILE_SIMD256 +static inline bool has_simd256(cpu_flags *flags) { + return flags->avx && flags->avx2; +} +#endif + +// Small mismatch between the variable names Python defines as part of configure +// at the ones HACL* expects to be set in order to enable those headers. +#define HACL_CAN_COMPILE_VEC128 HACL_CAN_COMPILE_SIMD128 +#define HACL_CAN_COMPILE_VEC256 HACL_CAN_COMPILE_SIMD256 + +#include "_hacl/Hacl_Hash_Blake2b.h" +#include "_hacl/Hacl_Hash_Blake2s.h" +#if HACL_CAN_COMPILE_SIMD256 +#include "_hacl/Hacl_Hash_Blake2b_Simd256.h" +#endif +#if HACL_CAN_COMPILE_SIMD128 +#include "_hacl/Hacl_Hash_Blake2s_Simd128.h" +#endif + +// MODULE TYPE SLOTS + +static PyType_Spec blake2b_type_spec; +static PyType_Spec blake2s_type_spec; + +PyDoc_STRVAR(blake2mod__doc__, +"_blake2b provides BLAKE2b for hashlib\n" +); + +typedef struct { + PyTypeObject* blake2b_type; + PyTypeObject* blake2s_type; + cpu_flags flags; +} Blake2State; + +static inline Blake2State* +blake2_get_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (Blake2State *)state; +} + +#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) +static inline Blake2State* +blake2_get_state_from_type(PyTypeObject *module) +{ + void *state = _PyType_GetModuleState(module); + assert(state != NULL); + return (Blake2State *)state; +} +#endif + +static struct PyMethodDef blake2mod_functions[] = { + {NULL, NULL} +}; + +static int +_blake2_traverse(PyObject *module, visitproc visit, void *arg) +{ + Blake2State *state = blake2_get_state(module); + Py_VISIT(state->blake2b_type); + Py_VISIT(state->blake2s_type); + return 0; +} + +static int +_blake2_clear(PyObject *module) +{ + Blake2State *state = blake2_get_state(module); + Py_CLEAR(state->blake2b_type); + Py_CLEAR(state->blake2s_type); + return 0; +} + +static void +_blake2_free(void *module) +{ + (void)_blake2_clear((PyObject *)module); +} + +#define ADD_INT(d, name, value) do { \ + PyObject *x = PyLong_FromLong(value); \ + if (!x) \ + return -1; \ + if (PyDict_SetItemString(d, name, x) < 0) { \ + Py_DECREF(x); \ + return -1; \ + } \ + Py_DECREF(x); \ +} while(0) + +#define ADD_INT_CONST(NAME, VALUE) do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ +} while (0) + +static int +blake2_exec(PyObject *m) +{ + Blake2State* st = blake2_get_state(m); + + // This is called at module initialization-time, and so appears to be as + // good a place as any to probe the CPU flags. + detect_cpu_features(&st->flags); + + st->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec( + m, &blake2b_type_spec, NULL); + + if (st->blake2b_type == NULL) { + return -1; + } + /* BLAKE2b */ + if (PyModule_AddType(m, st->blake2b_type) < 0) { + return -1; + } + + PyObject *d = st->blake2b_type->tp_dict; + ADD_INT(d, "SALT_SIZE", HACL_HASH_BLAKE2B_SALT_BYTES); + ADD_INT(d, "PERSON_SIZE", HACL_HASH_BLAKE2B_PERSONAL_BYTES); + ADD_INT(d, "MAX_KEY_SIZE", HACL_HASH_BLAKE2B_KEY_BYTES); + ADD_INT(d, "MAX_DIGEST_SIZE", HACL_HASH_BLAKE2B_OUT_BYTES); + + ADD_INT_CONST("BLAKE2B_SALT_SIZE", HACL_HASH_BLAKE2B_SALT_BYTES); + ADD_INT_CONST("BLAKE2B_PERSON_SIZE", HACL_HASH_BLAKE2B_PERSONAL_BYTES); + ADD_INT_CONST("BLAKE2B_MAX_KEY_SIZE", HACL_HASH_BLAKE2B_KEY_BYTES); + ADD_INT_CONST("BLAKE2B_MAX_DIGEST_SIZE", HACL_HASH_BLAKE2B_OUT_BYTES); + + /* BLAKE2s */ + st->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec( + m, &blake2s_type_spec, NULL); + + if (NULL == st->blake2s_type) + return -1; + + if (PyModule_AddType(m, st->blake2s_type) < 0) { + return -1; + } + + d = st->blake2s_type->tp_dict; + ADD_INT(d, "SALT_SIZE", HACL_HASH_BLAKE2S_SALT_BYTES); + ADD_INT(d, "PERSON_SIZE", HACL_HASH_BLAKE2S_PERSONAL_BYTES); + ADD_INT(d, "MAX_KEY_SIZE", HACL_HASH_BLAKE2S_KEY_BYTES); + ADD_INT(d, "MAX_DIGEST_SIZE", HACL_HASH_BLAKE2S_OUT_BYTES); + + ADD_INT_CONST("BLAKE2S_SALT_SIZE", HACL_HASH_BLAKE2S_SALT_BYTES); + ADD_INT_CONST("BLAKE2S_PERSON_SIZE", HACL_HASH_BLAKE2S_PERSONAL_BYTES); + ADD_INT_CONST("BLAKE2S_MAX_KEY_SIZE", HACL_HASH_BLAKE2S_KEY_BYTES); + ADD_INT_CONST("BLAKE2S_MAX_DIGEST_SIZE", HACL_HASH_BLAKE2S_OUT_BYTES); + + return 0; +} + +#undef ADD_INT +#undef ADD_INT_CONST + +static PyModuleDef_Slot _blake2_slots[] = { + {Py_mod_exec, blake2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +static struct PyModuleDef blake2_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_blake2", + .m_doc = blake2mod__doc__, + .m_size = sizeof(Blake2State), + .m_methods = blake2mod_functions, + .m_slots = _blake2_slots, + .m_traverse = _blake2_traverse, + .m_clear = _blake2_clear, + .m_free = _blake2_free, +}; + +PyMODINIT_FUNC +PyInit__blake2(void) +{ + return PyModuleDef_Init(&blake2_module); +} + +// IMPLEMENTATION OF METHODS + +// The HACL* API does not offer an agile API that can deal with either Blake2S +// or Blake2B -- the reason is that the underlying states are optimized (uint32s +// for S, uint64s for B). Therefore, we use a tagged union in this module to +// correctly dispatch. Note that the previous incarnation of this code +// transformed the Blake2b implementation into the Blake2s one using a script, +// so this is an improvement. +// +// The 128 and 256 versions are only available if i) we were able to compile +// them, and ii) if the CPU we run on also happens to have the right instruction +// set. +typedef enum { Blake2s, Blake2b, Blake2s_128, Blake2b_256 } blake2_impl; + +static inline bool is_blake2b(blake2_impl impl) { + return impl == Blake2b || impl == Blake2b_256; +} + +static inline bool is_blake2s(blake2_impl impl) { + return !is_blake2b(impl); +} + +static inline blake2_impl type_to_impl(PyTypeObject *type) { +#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) + Blake2State* st = blake2_get_state_from_type(type); +#endif + if (!strcmp(type->tp_name, blake2b_type_spec.name)) { +#ifdef HACL_CAN_COMPILE_SIMD256 + if (has_simd256(&st->flags)) + return Blake2b_256; + else +#endif + return Blake2b; + } else if (!strcmp(type->tp_name, blake2s_type_spec.name)) { +#ifdef HACL_CAN_COMPILE_SIMD128 + if (has_simd128(&st->flags)) + return Blake2s_128; + else +#endif + return Blake2s; + } else { + Py_UNREACHABLE(); + } +} + +typedef struct { + PyObject_HEAD + union { + Hacl_Hash_Blake2s_state_t *blake2s_state; + Hacl_Hash_Blake2b_state_t *blake2b_state; +#ifdef HACL_CAN_COMPILE_SIMD128 + Hacl_Hash_Blake2s_Simd128_state_t *blake2s_128_state; +#endif +#ifdef HACL_CAN_COMPILE_SIMD256 + Hacl_Hash_Blake2b_Simd256_state_t *blake2b_256_state; +#endif + }; + blake2_impl impl; + bool use_mutex; + PyMutex mutex; +} Blake2Object; + +#include "clinic/blake2module.c.h" + +/*[clinic input] +module _blake2 +class _blake2.blake2b "Blake2Object *" "&PyBlake2_BLAKE2bType" +class _blake2.blake2s "Blake2Object *" "&PyBlake2_BLAKE2sType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b7526666bd18af83]*/ + + +static Blake2Object * +new_Blake2Object(PyTypeObject *type) +{ + Blake2Object *self; + self = (Blake2Object *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + HASHLIB_INIT_MUTEX(self); + + return self; +} + +/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be + * 64 bits so we loop in <4gig chunks when needed. */ + +#if PY_SSIZE_T_MAX > UINT32_MAX +#define HACL_UPDATE_LOOP(update,state,buf,len) \ + while (len > UINT32_MAX) { \ + update(state, buf, UINT32_MAX); \ + len -= UINT32_MAX; \ + buf += UINT32_MAX; \ + } +#else +#define HACL_UPDATE_LOOP(update,state,buf,len) +#endif + +#define HACL_UPDATE(update,state,buf,len) do { \ + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to overflow the maximum admissible length for SHA2-256 + * (namely, 2^61-1 bytes). */ \ + HACL_UPDATE_LOOP(update,state,buf,len) \ + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ \ + update(state, buf, (uint32_t) len); \ +} while (0) + +static void update(Blake2Object *self, uint8_t *buf, Py_ssize_t len) { + switch (self->impl) { + // These need to be ifdef'd out otherwise it's an unresolved symbol at + // link-time. +#ifdef HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + HACL_UPDATE(Hacl_Hash_Blake2b_Simd256_update,self->blake2b_256_state, buf, len); + return; +#endif +#ifdef HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + HACL_UPDATE(Hacl_Hash_Blake2s_Simd128_update,self->blake2s_128_state, buf, len); + return; +#endif + case Blake2b: + HACL_UPDATE(Hacl_Hash_Blake2b_update,self->blake2b_state, buf, len); + return; + case Blake2s: + HACL_UPDATE(Hacl_Hash_Blake2s_update,self->blake2s_state, buf, len); + return; + default: + Py_UNREACHABLE(); + } +} + +static PyObject * +py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity) + +{ + Blake2Object *self = NULL; + Py_buffer buf; + + self = new_Blake2Object(type); + if (self == NULL) { + goto error; + } + + self->impl = type_to_impl(type); + + // Using Blake2b because we statically know that these are greater than the + // Blake2s sizes -- this avoids a VLA. + uint8_t salt_[HACL_HASH_BLAKE2B_SALT_BYTES] = { 0 }; + uint8_t personal_[HACL_HASH_BLAKE2B_PERSONAL_BYTES] = { 0 }; + + /* Validate digest size. */ + if (digest_size <= 0 || + (unsigned) digest_size > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)) + { + PyErr_Format(PyExc_ValueError, + "digest_size for %s must be between 1 and %d bytes, here it is %d", + is_blake2b(self->impl) ? "Blake2b" : "Blake2s", + is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES, + digest_size); + goto error; + } + + /* Validate salt parameter. */ + if ((salt->obj != NULL) && salt->len) { + if (salt->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)) { + PyErr_Format(PyExc_ValueError, + "maximum salt length is %d bytes", + (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)); + goto error; + } + memcpy(salt_, salt->buf, salt->len); + } + + /* Validate personalization parameter. */ + if ((person->obj != NULL) && person->len) { + if (person->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)) { + PyErr_Format(PyExc_ValueError, + "maximum person length is %d bytes", + (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)); + goto error; + } + memcpy(personal_, person->buf, person->len); + } + + /* Validate tree parameters. */ + if (fanout < 0 || fanout > 255) { + PyErr_SetString(PyExc_ValueError, + "fanout must be between 0 and 255"); + goto error; + } + + if (depth <= 0 || depth > 255) { + PyErr_SetString(PyExc_ValueError, + "depth must be between 1 and 255"); + goto error; + } + + if (leaf_size > 0xFFFFFFFFU) { + PyErr_SetString(PyExc_OverflowError, "leaf_size is too large"); + goto error; + } + + if (is_blake2s(self->impl) && node_offset > 0xFFFFFFFFFFFFULL) { + /* maximum 2**48 - 1 */ + PyErr_SetString(PyExc_OverflowError, "node_offset is too large"); + goto error; + } + + if (node_depth < 0 || node_depth > 255) { + PyErr_SetString(PyExc_ValueError, + "node_depth must be between 0 and 255"); + goto error; + } + + if (inner_size < 0 || + (unsigned) inner_size > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)) { + PyErr_Format(PyExc_ValueError, + "inner_size must be between 0 and is %d", + (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)); + goto error; + } + + /* Set key length. */ + if ((key->obj != NULL) && key->len) { + if (key->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)) { + PyErr_Format(PyExc_ValueError, + "maximum key length is %d bytes", + (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)); + goto error; + } + } + + // Unlike the state types, the parameters share a single (client-friendly) + // structure. + + Hacl_Hash_Blake2b_blake2_params params = { + .digest_length = digest_size, + .key_length = (uint8_t)key->len, + .fanout = fanout, + .depth = depth, + .leaf_length = leaf_size, + .node_offset = node_offset, + .node_depth = node_depth, + .inner_length = inner_size, + .salt = salt_, + .personal = personal_ + }; + + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + self->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key(¶ms, last_node, key->buf); + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + self->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key(¶ms, last_node, key->buf); + break; +#endif + case Blake2b: + self->blake2b_state = Hacl_Hash_Blake2b_malloc_with_params_and_key(¶ms, last_node, key->buf); + break; + case Blake2s: + self->blake2s_state = Hacl_Hash_Blake2s_malloc_with_params_and_key(¶ms, last_node, key->buf); + break; + default: + Py_UNREACHABLE(); + } + + /* Process initial data if any. */ + if (data != NULL) { + GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); + + if (buf.len >= HASHLIB_GIL_MINSIZE) { + Py_BEGIN_ALLOW_THREADS + update(self, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update(self, buf.buf, buf.len); + } + PyBuffer_Release(&buf); + } + + return (PyObject *)self; +error: + Py_XDECREF(self); + return NULL; +} + +/*[clinic input] +@classmethod +_blake2.blake2b.__new__ as py_blake2b_new + data: object(c_default="NULL") = b'' + / + * + digest_size: int(c_default="HACL_HASH_BLAKE2B_OUT_BYTES") = _blake2.blake2b.MAX_DIGEST_SIZE + key: Py_buffer(c_default="NULL", py_default="b''") = None + salt: Py_buffer(c_default="NULL", py_default="b''") = None + person: Py_buffer(c_default="NULL", py_default="b''") = None + fanout: int = 1 + depth: int = 1 + leaf_size: unsigned_long = 0 + node_offset: unsigned_long_long = 0 + node_depth: int = 0 + inner_size: int = 0 + last_node: bool = False + usedforsecurity: bool = True + +Return a new BLAKE2b hash object. +[clinic start generated code]*/ + +static PyObject * +py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity) +/*[clinic end generated code: output=32bfd8f043c6896f input=8fee2b7b11428b2d]*/ +{ + return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); +} + +/*[clinic input] +@classmethod +_blake2.blake2s.__new__ as py_blake2s_new + data: object(c_default="NULL") = b'' + / + * + digest_size: int(c_default="HACL_HASH_BLAKE2S_OUT_BYTES") = _blake2.blake2s.MAX_DIGEST_SIZE + key: Py_buffer(c_default="NULL", py_default="b''") = None + salt: Py_buffer(c_default="NULL", py_default="b''") = None + person: Py_buffer(c_default="NULL", py_default="b''") = None + fanout: int = 1 + depth: int = 1 + leaf_size: unsigned_long = 0 + node_offset: unsigned_long_long = 0 + node_depth: int = 0 + inner_size: int = 0 + last_node: bool = False + usedforsecurity: bool = True + +Return a new BLAKE2s hash object. +[clinic start generated code]*/ + +static PyObject * +py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity) +/*[clinic end generated code: output=556181f73905c686 input=8165a11980eac7f3]*/ +{ + return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); +} + +/*[clinic input] +_blake2.blake2b.copy + +Return a copy of the hash object. +[clinic start generated code]*/ + +static PyObject * +_blake2_blake2b_copy_impl(Blake2Object *self) +/*[clinic end generated code: output=622d1c56b91c50d8 input=e383c2d199fd8a2e]*/ +{ + Blake2Object *cpy; + + if ((cpy = new_Blake2Object(Py_TYPE(self))) == NULL) + return NULL; + + ENTER_HASHLIB(self); + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + cpy->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_copy(self->blake2b_256_state); + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + cpy->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_copy(self->blake2s_128_state); + break; +#endif + case Blake2b: + cpy->blake2b_state = Hacl_Hash_Blake2b_copy(self->blake2b_state); + break; + case Blake2s: + cpy->blake2s_state = Hacl_Hash_Blake2s_copy(self->blake2s_state); + break; + default: + Py_UNREACHABLE(); + } + cpy->impl = self->impl; + LEAVE_HASHLIB(self); + return (PyObject *)cpy; +} + +/*[clinic input] +_blake2.blake2b.update + + data: object + / + +Update this hash object's state with the provided bytes-like object. +[clinic start generated code]*/ + +static PyObject * +_blake2_blake2b_update(Blake2Object *self, PyObject *data) +/*[clinic end generated code: output=e6d1ac88471df308 input=ffc4aa6a6a225d31]*/ +{ + Py_buffer buf; + + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); + + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; + } + if (self->use_mutex) { + Py_BEGIN_ALLOW_THREADS + PyMutex_Lock(&self->mutex); + update(self, buf.buf, buf.len); + PyMutex_Unlock(&self->mutex); + Py_END_ALLOW_THREADS + } else { + update(self, buf.buf, buf.len); + } + + PyBuffer_Release(&buf); + + Py_RETURN_NONE; +} + +/*[clinic input] +_blake2.blake2b.digest + +Return the digest value as a bytes object. +[clinic start generated code]*/ + +static PyObject * +_blake2_blake2b_digest_impl(Blake2Object *self) +/*[clinic end generated code: output=31ab8ad477f4a2f7 input=7d21659e9c5fff02]*/ +{ + uint8_t digest[HACL_HASH_BLAKE2B_OUT_BYTES]; + + ENTER_HASHLIB(self); + uint8_t digest_length = 0; + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); + break; +#endif + case Blake2b: + digest_length = Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); + break; + case Blake2s: + digest_length = Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); + break; + default: + Py_UNREACHABLE(); + } + LEAVE_HASHLIB(self); + return PyBytes_FromStringAndSize((const char *)digest, digest_length); +} + +/*[clinic input] +_blake2.blake2b.hexdigest + +Return the digest value as a string of hexadecimal digits. +[clinic start generated code]*/ + +static PyObject * +_blake2_blake2b_hexdigest_impl(Blake2Object *self) +/*[clinic end generated code: output=5ef54b138db6610a input=76930f6946351f56]*/ +{ + uint8_t digest[HACL_HASH_BLAKE2B_OUT_BYTES]; + + ENTER_HASHLIB(self); + uint8_t digest_length = 0; + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); + break; +#endif + case Blake2b: + digest_length = Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); + break; + case Blake2s: + digest_length = Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); + break; + default: + Py_UNREACHABLE(); + } + LEAVE_HASHLIB(self); + return _Py_strhex((const char *)digest, digest_length); +} + + +static PyMethodDef py_blake2b_methods[] = { + _BLAKE2_BLAKE2B_COPY_METHODDEF + _BLAKE2_BLAKE2B_DIGEST_METHODDEF + _BLAKE2_BLAKE2B_HEXDIGEST_METHODDEF + _BLAKE2_BLAKE2B_UPDATE_METHODDEF + {NULL, NULL} +}; + + +static PyObject * +py_blake2b_get_name(Blake2Object *self, void *closure) +{ + return PyUnicode_FromString(is_blake2b(self->impl) ? "blake2b" : "blake2s"); +} + + + +static PyObject * +py_blake2b_get_block_size(Blake2Object *self, void *closure) +{ + return PyLong_FromLong(is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_BLOCK_BYTES : HACL_HASH_BLAKE2S_BLOCK_BYTES); +} + + + +static PyObject * +py_blake2b_get_digest_size(Blake2Object *self, void *closure) +{ + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + return PyLong_FromLong(Hacl_Hash_Blake2b_Simd256_info(self->blake2b_256_state).digest_length); +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + return PyLong_FromLong(Hacl_Hash_Blake2s_Simd128_info(self->blake2s_128_state).digest_length); +#endif + case Blake2b: + return PyLong_FromLong(Hacl_Hash_Blake2b_info(self->blake2b_state).digest_length); + case Blake2s: + return PyLong_FromLong(Hacl_Hash_Blake2s_info(self->blake2s_state).digest_length); + default: + Py_UNREACHABLE(); + } +} + + +static PyGetSetDef py_blake2b_getsetters[] = { + {"name", (getter)py_blake2b_get_name, + NULL, NULL, NULL}, + {"block_size", (getter)py_blake2b_get_block_size, + NULL, NULL, NULL}, + {"digest_size", (getter)py_blake2b_get_digest_size, + NULL, NULL, NULL}, + {NULL} +}; + + +static void +py_blake2b_dealloc(Blake2Object *self) +{ + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + if (self->blake2b_256_state != NULL) + Hacl_Hash_Blake2b_Simd256_free(self->blake2b_256_state); + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + if (self->blake2s_128_state != NULL) + Hacl_Hash_Blake2s_Simd128_free(self->blake2s_128_state); + break; +#endif + case Blake2b: + // This happens if we hit "goto error" in the middle of the + // initialization function. We leverage the fact that tp_alloc + // guarantees that the contents of the object are NULL-initialized + // (see documentation for PyType_GenericAlloc) to detect this case. + if (self->blake2b_state != NULL) + Hacl_Hash_Blake2b_free(self->blake2b_state); + break; + case Blake2s: + if (self->blake2s_state != NULL) + Hacl_Hash_Blake2s_free(self->blake2s_state); + break; + default: + Py_UNREACHABLE(); + } + + PyTypeObject *type = Py_TYPE(self); + PyObject_Free(self); + Py_DECREF(type); +} + +static PyType_Slot blake2b_type_slots[] = { + {Py_tp_dealloc, py_blake2b_dealloc}, + {Py_tp_doc, (char *)py_blake2b_new__doc__}, + {Py_tp_methods, py_blake2b_methods}, + {Py_tp_getset, py_blake2b_getsetters}, + {Py_tp_new, py_blake2b_new}, + {0,0} +}; + +static PyType_Slot blake2s_type_slots[] = { + {Py_tp_dealloc, py_blake2b_dealloc}, + {Py_tp_doc, (char *)py_blake2s_new__doc__}, + {Py_tp_methods, py_blake2b_methods}, + {Py_tp_getset, py_blake2b_getsetters}, + // only the constructor differs, so that it can receive a clinic-generated + // default digest length suitable for blake2s + {Py_tp_new, py_blake2s_new}, + {0,0} +}; + +static PyType_Spec blake2b_type_spec = { + .name = "_blake2.blake2b", + .basicsize = sizeof(Blake2Object), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .slots = blake2b_type_slots +}; + +static PyType_Spec blake2s_type_spec = { + .name = "_blake2.blake2s", + .basicsize = sizeof(Blake2Object), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .slots = blake2s_type_slots +}; diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index e8835ad0909633..ef6faeb71274e1 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -806,7 +806,7 @@ jisx0213_encoder(const MultibyteCodec *codec, const Py_UCS4 *data, jisx0213_pair_encmap, JISX0213_ENCPAIRS); if (coded != DBCINV) return coded; - /* fall through */ + _Py_FALLTHROUGH; case -1: /* flush unterminated */ *length = 1; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index 766f82983025e4..2b446ba5226ac0 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -503,6 +503,7 @@ static struct PyMethodDef _cjk_methods[] = { static PyModuleDef_Slot _cjk_slots[] = { {Py_mod_exec, _cjk_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index b5639d5cf10a22..73edd5c3b25553 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -12,7 +12,7 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "encode($self, /, input, errors=None)\n" "--\n" "\n" -"Return an encoded string version of `input\'.\n" +"Return an encoded string version of \'input\'.\n" "\n" "\'errors\' may be given to set a different error handling scheme. Default is\n" "\'strict\' meaning that encoding errors raise a UnicodeEncodeError. Other possible\n" @@ -682,4 +682,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=ee767a6d93c7108a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f09052c5a28cc6e6 input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 5d3c16a98423ba..53135ae4aa7968 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -574,7 +574,7 @@ _multibytecodec.MultibyteCodec.encode input: object errors: str(accept={str, NoneType}) = None -Return an encoded string version of `input'. +Return an encoded string version of 'input'. 'errors' may be given to set a different error handling scheme. Default is 'strict' meaning that encoding errors raise a UnicodeEncodeError. Other possible @@ -586,7 +586,7 @@ static PyObject * _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=606d0e128a577bae]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=2841745b95ed338f]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; @@ -669,7 +669,7 @@ _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, if (datalen == 0) { ERROR_DECREF(errorcb); - return make_tuple(PyUnicode_New(0, 0), 0); + return make_tuple(Py_GetConstant(Py_CONSTANT_EMPTY_STR), 0); } _PyUnicodeWriter_Init(&buf.writer); @@ -825,8 +825,15 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx, if (inpos < datalen) { if (datalen - inpos > MAXENCPENDING) { /* normal codecs can't reach here */ - PyErr_SetString(PyExc_UnicodeError, - "pending buffer overflow"); + PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, + "sOnns", + ctx->codec->encoding, + inbuf, + inpos, datalen, + "pending buffer overflow"); + if (excobj == NULL) goto errorexit; + PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); + Py_DECREF(excobj); goto errorexit; } ctx->pending = PyUnicode_Substring(inbuf, inpos, datalen); @@ -857,7 +864,16 @@ decoder_append_pending(MultibyteStatefulDecoderContext *ctx, npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); if (npendings + ctx->pendingsize > MAXDECPENDING || npendings > PY_SSIZE_T_MAX - ctx->pendingsize) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow"); + Py_ssize_t bufsize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); + PyObject *excobj = PyUnicodeDecodeError_Create(ctx->codec->encoding, + (const char *)buf->inbuf_top, + bufsize, + 0, + bufsize, + "pending buffer overflow"); + if (excobj == NULL) return -1; + PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); + Py_DECREF(excobj); return -1; } memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings); @@ -938,7 +954,17 @@ _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEn return NULL; } if (pendingsize > MAXENCPENDING*4) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer too large"); + PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, + "sOnns", + self->codec->encoding, + self->pending, + 0, PyUnicode_GET_LENGTH(self->pending), + "pending buffer too large"); + if (excobj == NULL) { + return NULL; + } + PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); + Py_DECREF(excobj); return NULL; } statebytes[0] = (unsigned char)pendingsize; @@ -973,7 +999,8 @@ _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEn if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes), 1 /* little-endian */ , - 0 /* unsigned */ ) < 0) { + 0 /* unsigned */ , + 1 /* with_exceptions */) < 0) { goto errorexit; } @@ -1255,7 +1282,8 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes), 1 /* little-endian */ , - 0 /* unsigned */ ) < 0) { + 0 /* unsigned */ , + 1 /* with_exceptions */) < 0) { return NULL; } @@ -1265,7 +1293,13 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe } if (buffersize > MAXDECPENDING) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer too large"); + PyObject *excobj = PyUnicodeDecodeError_Create(self->codec->encoding, + PyBytes_AS_STRING(buffer), buffersize, + 0, buffersize, + "pending buffer too large"); + if (excobj == NULL) return NULL; + PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); + Py_DECREF(excobj); return NULL; } @@ -1400,7 +1434,7 @@ mbstreamreader_iread(MultibyteStreamReaderObject *self, Py_ssize_t rsize; if (sizehint == 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); _PyUnicodeWriter_Init(&buf.writer); buf.excobj = NULL; @@ -2060,6 +2094,7 @@ static struct PyMethodDef _multibytecodec_methods[] = { static PyModuleDef_Slot _multibytecodec_slots[] = { {Py_mod_exec, _multibytecodec_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 6a9c8ff6d8fdd9..d619a124ccead5 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -1487,4 +1487,64 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=b26155080c82c472 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_asyncio_all_tasks__doc__, +"all_tasks($module, /, loop=None)\n" +"--\n" +"\n" +"Return a set of all tasks for the loop."); + +#define _ASYNCIO_ALL_TASKS_METHODDEF \ + {"all_tasks", _PyCFunction_CAST(_asyncio_all_tasks), METH_FASTCALL|METH_KEYWORDS, _asyncio_all_tasks__doc__}, + +static PyObject * +_asyncio_all_tasks_impl(PyObject *module, PyObject *loop); + +static PyObject * +_asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(loop), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"loop", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "all_tasks", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *loop = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + loop = args[0]; +skip_optional_pos: + return_value = _asyncio_all_tasks_impl(module, loop); + +exit: + return return_value; +} +/*[clinic end generated code: output=ffe9b71bc65888b3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 9955d0edb2699f..528602c84a9b23 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -44,7 +44,7 @@ _bisect_bisect_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -152,7 +152,7 @@ _bisect_insort_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -257,7 +257,7 @@ _bisect_bisect_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -365,7 +365,7 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -434,4 +434,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=4af5bd405149bf5f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a8d5a32dd0a3f04 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 1c0f37442ab350..01855aec5e123e 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -2683,6 +2683,56 @@ _codecs_register_error(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_codecs__unregister_error__doc__, +"_unregister_error($module, errors, /)\n" +"--\n" +"\n" +"Un-register the specified error handler for the error handling `errors\'.\n" +"\n" +"Only custom error handlers can be un-registered. An exception is raised\n" +"if the error handling is a built-in one (e.g., \'strict\'), or if an error\n" +"occurs.\n" +"\n" +"Otherwise, this returns True if a custom handler has been successfully\n" +"un-registered, and False if no custom handler for the specified error\n" +"handling exists."); + +#define _CODECS__UNREGISTER_ERROR_METHODDEF \ + {"_unregister_error", (PyCFunction)_codecs__unregister_error, METH_O, _codecs__unregister_error__doc__}, + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors); + +static PyObject * +_codecs__unregister_error(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *errors; + int _return_value; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_unregister_error", "argument", "str", arg); + goto exit; + } + Py_ssize_t errors_length; + errors = PyUnicode_AsUTF8AndSize(arg, &errors_length); + if (errors == NULL) { + goto exit; + } + if (strlen(errors) != (size_t)errors_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + _return_value = _codecs__unregister_error_impl(module, errors); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_codecs_lookup_error__doc__, "lookup_error($module, name, /)\n" "--\n" @@ -2746,4 +2796,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b3013d4709d96ffe input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index 60fb12a2231619..fa0ff2133a05d2 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(deque_pop__doc__, @@ -24,7 +25,13 @@ deque_pop_impl(dequeobject *deque); static PyObject * deque_pop(dequeobject *deque, PyObject *Py_UNUSED(ignored)) { - return deque_pop_impl(deque); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_pop_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(deque_popleft__doc__, @@ -42,7 +49,13 @@ deque_popleft_impl(dequeobject *deque); static PyObject * deque_popleft(dequeobject *deque, PyObject *Py_UNUSED(ignored)) { - return deque_popleft_impl(deque); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_popleft_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(deque_append__doc__, @@ -54,6 +67,21 @@ PyDoc_STRVAR(deque_append__doc__, #define DEQUE_APPEND_METHODDEF \ {"append", (PyCFunction)deque_append, METH_O, deque_append__doc__}, +static PyObject * +deque_append_impl(dequeobject *deque, PyObject *item); + +static PyObject * +deque_append(dequeobject *deque, PyObject *item) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_append_impl(deque, item); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(deque_appendleft__doc__, "appendleft($self, item, /)\n" "--\n" @@ -63,6 +91,21 @@ PyDoc_STRVAR(deque_appendleft__doc__, #define DEQUE_APPENDLEFT_METHODDEF \ {"appendleft", (PyCFunction)deque_appendleft, METH_O, deque_appendleft__doc__}, +static PyObject * +deque_appendleft_impl(dequeobject *deque, PyObject *item); + +static PyObject * +deque_appendleft(dequeobject *deque, PyObject *item) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_appendleft_impl(deque, item); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(deque_extend__doc__, "extend($self, iterable, /)\n" "--\n" @@ -72,6 +115,21 @@ PyDoc_STRVAR(deque_extend__doc__, #define DEQUE_EXTEND_METHODDEF \ {"extend", (PyCFunction)deque_extend, METH_O, deque_extend__doc__}, +static PyObject * +deque_extend_impl(dequeobject *deque, PyObject *iterable); + +static PyObject * +deque_extend(dequeobject *deque, PyObject *iterable) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_extend_impl(deque, iterable); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(deque_extendleft__doc__, "extendleft($self, iterable, /)\n" "--\n" @@ -81,6 +139,21 @@ PyDoc_STRVAR(deque_extendleft__doc__, #define DEQUE_EXTENDLEFT_METHODDEF \ {"extendleft", (PyCFunction)deque_extendleft, METH_O, deque_extendleft__doc__}, +static PyObject * +deque_extendleft_impl(dequeobject *deque, PyObject *iterable); + +static PyObject * +deque_extendleft(dequeobject *deque, PyObject *iterable) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_extendleft_impl(deque, iterable); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(deque_copy__doc__, "copy($self, /)\n" "--\n" @@ -96,7 +169,13 @@ deque_copy_impl(dequeobject *deque); static PyObject * deque_copy(dequeobject *deque, PyObject *Py_UNUSED(ignored)) { - return deque_copy_impl(deque); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_copy_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(deque___copy____doc__, @@ -114,7 +193,13 @@ deque___copy___impl(dequeobject *deque); static PyObject * deque___copy__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) { - return deque___copy___impl(deque); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque___copy___impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(deque_clearmethod__doc__, @@ -132,7 +217,13 @@ deque_clearmethod_impl(dequeobject *deque); static PyObject * deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) { - return deque_clearmethod_impl(deque); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_clearmethod_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(deque_rotate__doc__, @@ -172,7 +263,9 @@ deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) n = ival; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(deque); return_value = deque_rotate_impl(deque, n); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -193,7 +286,13 @@ deque_reverse_impl(dequeobject *deque); static PyObject * deque_reverse(dequeobject *deque, PyObject *Py_UNUSED(ignored)) { - return deque_reverse_impl(deque); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_reverse_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(deque_count__doc__, @@ -205,6 +304,21 @@ PyDoc_STRVAR(deque_count__doc__, #define DEQUE_COUNT_METHODDEF \ {"count", (PyCFunction)deque_count, METH_O, deque_count__doc__}, +static PyObject * +deque_count_impl(dequeobject *deque, PyObject *v); + +static PyObject * +deque_count(dequeobject *deque, PyObject *v) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_count_impl(deque, v); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(deque_index__doc__, "index($self, value, [start, [stop]])\n" "--\n" @@ -245,7 +359,9 @@ deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(deque); return_value = deque_index_impl(deque, v, start, stop); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -286,7 +402,9 @@ deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) index = ival; } value = args[1]; + Py_BEGIN_CRITICAL_SECTION(deque); return_value = deque_insert_impl(deque, index, value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -301,6 +419,21 @@ PyDoc_STRVAR(deque_remove__doc__, #define DEQUE_REMOVE_METHODDEF \ {"remove", (PyCFunction)deque_remove, METH_O, deque_remove__doc__}, +static PyObject * +deque_remove_impl(dequeobject *deque, PyObject *value); + +static PyObject * +deque_remove(dequeobject *deque, PyObject *value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_remove_impl(deque, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(deque___reduce____doc__, "__reduce__($self, /)\n" "--\n" @@ -379,7 +512,9 @@ deque_init(PyObject *deque, PyObject *args, PyObject *kwargs) } maxlenobj = fastargs[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(deque); return_value = deque_init_impl((dequeobject *)deque, iterable, maxlenobj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -400,7 +535,13 @@ deque___sizeof___impl(dequeobject *deque); static PyObject * deque___sizeof__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) { - return deque___sizeof___impl(deque); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque___sizeof___impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(deque___reversed____doc__, @@ -488,4 +629,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=3633a5cbc23e8440 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=64c32b16df7be07a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 457f71370afda9..c8788c461f745c 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, @@ -418,4 +421,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=7bac14e9a1194c87 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=18dc5571174c7189 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index f7e0aaf7b23649..8399c5620f125b 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -1389,12 +1389,12 @@ _curses_window_overlay(PyCursesWindowObject *self, PyObject *args) switch (PyTuple_GET_SIZE(args)) { case 1: - if (!PyArg_ParseTuple(args, "O!:overlay", &PyCursesWindow_Type, &destwin)) { + if (!PyArg_ParseTuple(args, "O!:overlay", clinic_state()->window_type, &destwin)) { goto exit; } break; case 7: - if (!PyArg_ParseTuple(args, "O!iiiiii:overlay", &PyCursesWindow_Type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { + if (!PyArg_ParseTuple(args, "O!iiiiii:overlay", clinic_state()->window_type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { goto exit; } group_right_1 = 1; @@ -1448,12 +1448,12 @@ _curses_window_overwrite(PyCursesWindowObject *self, PyObject *args) switch (PyTuple_GET_SIZE(args)) { case 1: - if (!PyArg_ParseTuple(args, "O!:overwrite", &PyCursesWindow_Type, &destwin)) { + if (!PyArg_ParseTuple(args, "O!:overwrite", clinic_state()->window_type, &destwin)) { goto exit; } break; case 7: - if (!PyArg_ParseTuple(args, "O!iiiiii:overwrite", &PyCursesWindow_Type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { + if (!PyArg_ParseTuple(args, "O!iiiiii:overwrite", clinic_state()->window_type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { goto exit; } group_right_1 = 1; @@ -3693,25 +3693,55 @@ PyDoc_STRVAR(_curses_resizeterm__doc__, {"resizeterm", _PyCFunction_CAST(_curses_resizeterm), METH_FASTCALL, _curses_resizeterm__doc__}, static PyObject * -_curses_resizeterm_impl(PyObject *module, int nlines, int ncols); +_curses_resizeterm_impl(PyObject *module, short nlines, short ncols); static PyObject * _curses_resizeterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int nlines; - int ncols; + short nlines; + short ncols; if (!_PyArg_CheckPositional("resizeterm", nargs, 2, 2)) { goto exit; } - nlines = PyLong_AsInt(args[0]); - if (nlines == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + nlines = (short) ival; + } } - ncols = PyLong_AsInt(args[1]); - if (ncols == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + ncols = (short) ival; + } } return_value = _curses_resizeterm_impl(module, nlines, ncols); @@ -3744,25 +3774,55 @@ PyDoc_STRVAR(_curses_resize_term__doc__, {"resize_term", _PyCFunction_CAST(_curses_resize_term), METH_FASTCALL, _curses_resize_term__doc__}, static PyObject * -_curses_resize_term_impl(PyObject *module, int nlines, int ncols); +_curses_resize_term_impl(PyObject *module, short nlines, short ncols); static PyObject * _curses_resize_term(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int nlines; - int ncols; + short nlines; + short ncols; if (!_PyArg_CheckPositional("resize_term", nargs, 2, 2)) { goto exit; } - nlines = PyLong_AsInt(args[0]); - if (nlines == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + nlines = (short) ival; + } } - ncols = PyLong_AsInt(args[1]); - if (ncols == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + ncols = (short) ival; + } } return_value = _curses_resize_term_impl(module, nlines, ncols); @@ -4318,4 +4378,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=96887782374f070a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cd1273354b08948f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index d06271e18a49b2..4379b433db3738 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_dbm_dbm_close__doc__, @@ -218,4 +221,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=743ce0cea116747e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f7d9a87d80a64278 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 9622591a1aa855..1a5a820d1f00b5 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -1169,6 +1169,23 @@ _elementtree_XMLParser_close(XMLParserObject *self, PyObject *Py_UNUSED(ignored) return _elementtree_XMLParser_close_impl(self); } +PyDoc_STRVAR(_elementtree_XMLParser_flush__doc__, +"flush($self, /)\n" +"--\n" +"\n"); + +#define _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF \ + {"flush", (PyCFunction)_elementtree_XMLParser_flush, METH_NOARGS, _elementtree_XMLParser_flush__doc__}, + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self); + +static PyObject * +_elementtree_XMLParser_flush(XMLParserObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _elementtree_XMLParser_flush_impl(self); +} + PyDoc_STRVAR(_elementtree_XMLParser_feed__doc__, "feed($self, data, /)\n" "--\n" @@ -1219,4 +1236,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=218ec9e6a889f796 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bd28eba33d9c1f25 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index e98984dc4d3a09..0564921034be47 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -67,6 +67,50 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } +PyDoc_STRVAR(_functools_reduce__doc__, +"reduce($module, function, iterable, initial=, /)\n" +"--\n" +"\n" +"Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n" +"\n" +"This effectively reduces the iterable to a single value. If initial is present,\n" +"it is placed before the items of the iterable in the calculation, and serves as\n" +"a default when the iterable is empty.\n" +"\n" +"For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n" +"calculates ((((1 + 2) + 3) + 4) + 5)."); + +#define _FUNCTOOLS_REDUCE_METHODDEF \ + {"reduce", _PyCFunction_CAST(_functools_reduce), METH_FASTCALL, _functools_reduce__doc__}, + +static PyObject * +_functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, + PyObject *result); + +static PyObject * +_functools_reduce(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *func; + PyObject *seq; + PyObject *result = NULL; + + if (!_PyArg_CheckPositional("reduce", nargs, 2, 3)) { + goto exit; + } + func = args[0]; + seq = args[1]; + if (nargs < 3) { + goto skip_optional; + } + result = args[2]; +skip_optional: + return_value = _functools_reduce_impl(module, func, seq, result); + +exit: + return return_value; +} + PyDoc_STRVAR(_functools__lru_cache_wrapper_cache_info__doc__, "cache_info($self, /)\n" "--\n" @@ -114,4 +158,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig return return_value; } -/*[clinic end generated code: output=755265bb6d5ea751 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=214d6c6307cfcd91 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 626e4678809d4f..bbf4365114c0aa 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_gdbm_gdbm_get__doc__, @@ -340,4 +343,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=6b4c19905ac9967d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=07bdeb4a8ecb328e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 58650dff288444..1d85ab1c524082 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -1347,7 +1347,7 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(password), &_Py_ID(salt), &_Py_ID(n), &_Py_ID(r), &_Py_ID(p), &_Py_ID(maxmem), &_Py_ID(dklen), }, + .ob_item = { &_Py_ID(password), &_Py_ID(salt), _Py_LATIN1_CHR('n'), _Py_LATIN1_CHR('r'), _Py_LATIN1_CHR('p'), &_Py_ID(maxmem), &_Py_ID(dklen), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1824,4 +1824,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=b7eddeb3d6ccdeec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fef43fd9f4dbea49 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index c7fd0f9f8a7420..32ac9521a2b5cf 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -668,4 +668,82 @@ _opcode_get_intrinsic2_descs(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_intrinsic2_descs_impl(module); } -/*[clinic end generated code: output=a1052bb1deffb7f2 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_opcode_get_special_method_names__doc__, +"get_special_method_names($module, /)\n" +"--\n" +"\n" +"Return a list of special method names."); + +#define _OPCODE_GET_SPECIAL_METHOD_NAMES_METHODDEF \ + {"get_special_method_names", (PyCFunction)_opcode_get_special_method_names, METH_NOARGS, _opcode_get_special_method_names__doc__}, + +static PyObject * +_opcode_get_special_method_names_impl(PyObject *module); + +static PyObject * +_opcode_get_special_method_names(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _opcode_get_special_method_names_impl(module); +} + +PyDoc_STRVAR(_opcode_get_executor__doc__, +"get_executor($module, /, code, offset)\n" +"--\n" +"\n" +"Return the executor object at offset in code if exists, None otherwise."); + +#define _OPCODE_GET_EXECUTOR_METHODDEF \ + {"get_executor", _PyCFunction_CAST(_opcode_get_executor), METH_FASTCALL|METH_KEYWORDS, _opcode_get_executor__doc__}, + +static PyObject * +_opcode_get_executor_impl(PyObject *module, PyObject *code, int offset); + +static PyObject * +_opcode_get_executor(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(code), &_Py_ID(offset), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"code", "offset", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_executor", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *code; + int offset; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + code = args[0]; + offset = PyLong_AsInt(args[1]); + if (offset == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _opcode_get_executor_impl(module, code, offset); + +exit: + return return_value; +} +/*[clinic end generated code: output=3b4d4f32eedd636e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_operator.c.h b/Modules/clinic/_operator.c.h index 08615d690922a1..48a8ea8c3379ab 100644 --- a/Modules/clinic/_operator.c.h +++ b/Modules/clinic/_operator.c.h @@ -1393,6 +1393,24 @@ _operator_is_not(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(_operator_is_none__doc__, +"is_none($module, a, /)\n" +"--\n" +"\n" +"Same as a is None."); + +#define _OPERATOR_IS_NONE_METHODDEF \ + {"is_none", (PyCFunction)_operator_is_none, METH_O, _operator_is_none__doc__}, + +PyDoc_STRVAR(_operator_is_not_none__doc__, +"is_not_none($module, a, /)\n" +"--\n" +"\n" +"Same as a is not None."); + +#define _OPERATOR_IS_NOT_NONE_METHODDEF \ + {"is_not_none", (PyCFunction)_operator_is_not_none, METH_O, _operator_is_not_none__doc__}, + PyDoc_STRVAR(_operator_length_hint__doc__, "length_hint($module, obj, default=0, /)\n" "--\n" @@ -1489,4 +1507,4 @@ _operator__compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t na exit: return return_value; } -/*[clinic end generated code: output=ddbba2cd943571eb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=972e2543c4fcf1ba input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index 5a6ae7be6b6ea7..40f1309b6aa03c 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -111,7 +111,7 @@ PyDoc_STRVAR(_pickle_Pickler___init____doc__, "\n" "The optional *protocol* argument tells the pickler to use the given\n" "protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\n" -"protocol is 4. It was introduced in Python 3.4, and is incompatible\n" +"protocol is 5. It was introduced in Python 3.8, and is incompatible\n" "with previous versions.\n" "\n" "Specifying a negative protocol version selects the highest protocol\n" @@ -614,7 +614,7 @@ PyDoc_STRVAR(_pickle_dump__doc__, "\n" "The optional *protocol* argument tells the pickler to use the given\n" "protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\n" -"protocol is 4. It was introduced in Python 3.4, and is incompatible\n" +"protocol is 5. It was introduced in Python 3.8, and is incompatible\n" "with previous versions.\n" "\n" "Specifying a negative protocol version selects the highest protocol\n" @@ -724,7 +724,7 @@ PyDoc_STRVAR(_pickle_dumps__doc__, "\n" "The optional *protocol* argument tells the pickler to use the given\n" "protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\n" -"protocol is 4. It was introduced in Python 3.4, and is incompatible\n" +"protocol is 5. It was introduced in Python 3.8, and is incompatible\n" "with previous versions.\n" "\n" "Specifying a negative protocol version selects the highest protocol\n" @@ -1077,4 +1077,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=bd63c85a8737b0aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a9452cf1219f2e7a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index dd7644de6b7534..d52629cf6eaa5b 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -9,7 +9,7 @@ PyDoc_STRVAR(subprocess_fork_exec__doc__, " env, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite,\n" " errpipe_read, errpipe_write, restore_signals, call_setsid,\n" " pgid_to_set, gid, extra_groups, uid, child_umask, preexec_fn,\n" -" allow_vfork, /)\n" +" /)\n" "--\n" "\n" "Spawn a fresh new child process.\n" @@ -48,7 +48,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, pid_t pgid_to_set, PyObject *gid_object, PyObject *extra_groups_packed, PyObject *uid_object, int child_umask, - PyObject *preexec_fn, int allow_vfork); + PyObject *preexec_fn); static PyObject * subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -76,9 +76,8 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *uid_object; int child_umask; PyObject *preexec_fn; - int allow_vfork; - if (!_PyArg_CheckPositional("fork_exec", nargs, 23, 23)) { + if (!_PyArg_CheckPositional("fork_exec", nargs, 22, 22)) { goto exit; } process_args = args[0]; @@ -146,13 +145,9 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } preexec_fn = args[21]; - allow_vfork = PyObject_IsTrue(args[22]); - if (allow_vfork < 0) { - goto exit; - } - return_value = subprocess_fork_exec_impl(module, process_args, executable_list, close_fds, py_fds_to_keep, cwd_obj, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, call_setsid, pgid_to_set, gid_object, extra_groups_packed, uid_object, child_umask, preexec_fn, allow_vfork); + return_value = subprocess_fork_exec_impl(module, process_args, executable_list, close_fds, py_fds_to_keep, cwd_obj, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, call_setsid, pgid_to_set, gid_object, extra_groups_packed, uid_object, child_umask, preexec_fn); exit: return return_value; } -/*[clinic end generated code: output=48555f5965a871be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=942bc2748a9c2785 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 2940f16a2cb7f6..9d5b70dfad553d 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -6,7 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, @@ -23,7 +23,13 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_do_handshake(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_do_handshake_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_do_handshake_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__test_decode_cert__doc__, @@ -89,7 +95,9 @@ _ssl__SSLSocket_getpeercert(PySSLSocket *self, PyObject *const *args, Py_ssize_t goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLSocket_getpeercert_impl(self, binary_mode); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -109,7 +117,13 @@ _ssl__SSLSocket_get_verified_chain_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_get_verified_chain(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_get_verified_chain_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_get_verified_chain_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_get_unverified_chain__doc__, @@ -126,7 +140,13 @@ _ssl__SSLSocket_get_unverified_chain_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_get_unverified_chain(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_get_unverified_chain_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_get_unverified_chain_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_shared_ciphers__doc__, @@ -143,7 +163,13 @@ _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_shared_ciphers(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_shared_ciphers_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_shared_ciphers_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_cipher__doc__, @@ -160,7 +186,13 @@ _ssl__SSLSocket_cipher_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_cipher(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_cipher_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_cipher_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_version__doc__, @@ -177,7 +209,13 @@ _ssl__SSLSocket_version_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_version(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_version_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_version_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_selected_alpn_protocol__doc__, @@ -194,7 +232,13 @@ _ssl__SSLSocket_selected_alpn_protocol_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_selected_alpn_protocol(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_selected_alpn_protocol_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_selected_alpn_protocol_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_compression__doc__, @@ -214,6 +258,176 @@ _ssl__SSLSocket_compression(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) return _ssl__SSLSocket_compression_impl(self); } +#if defined(_ssl__SSLSocket_context_HAS_DOCSTR) +# define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ +#else +# define _ssl__SSLSocket_context_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_CONTEXT_GETSETDEF) +# undef _SSL__SSLSOCKET_CONTEXT_GETSETDEF +# define _SSL__SSLSOCKET_CONTEXT_GETSETDEF {"context", (getter)_ssl__SSLSocket_context_get, (setter)_ssl__SSLSocket_context_set, _ssl__SSLSocket_context_DOCSTR}, +#else +# define _SSL__SSLSOCKET_CONTEXT_GETSETDEF {"context", (getter)_ssl__SSLSocket_context_get, NULL, _ssl__SSLSocket_context_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_context_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_context_get(PySSLSocket *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_context_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLSOCKET_CONTEXT_HAS_DOCSTR) +# define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ +#else +# define _ssl__SSLSocket_context_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_CONTEXT_GETSETDEF) +# undef _SSL__SSLSOCKET_CONTEXT_GETSETDEF +# define _SSL__SSLSOCKET_CONTEXT_GETSETDEF {"context", (getter)_ssl__SSLSocket_context_get, (setter)_ssl__SSLSocket_context_set, _ssl__SSLSocket_context_DOCSTR}, +#else +# define _SSL__SSLSOCKET_CONTEXT_GETSETDEF {"context", NULL, (setter)_ssl__SSLSocket_context_set, NULL}, +#endif + +static int +_ssl__SSLSocket_context_set_impl(PySSLSocket *self, PyObject *value); + +static int +_ssl__SSLSocket_context_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_context_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_server_side__doc__, +"Whether this is a server-side socket."); +#define _ssl__SSLSocket_server_side_HAS_DOCSTR + +#if defined(_ssl__SSLSocket_server_side_HAS_DOCSTR) +# define _ssl__SSLSocket_server_side_DOCSTR _ssl__SSLSocket_server_side__doc__ +#else +# define _ssl__SSLSocket_server_side_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF) +# undef _SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF +# define _SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF {"server_side", (getter)_ssl__SSLSocket_server_side_get, (setter)_ssl__SSLSocket_server_side_set, _ssl__SSLSocket_server_side_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF {"server_side", (getter)_ssl__SSLSocket_server_side_get, NULL, _ssl__SSLSocket_server_side_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_server_side_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_server_side_get(PySSLSocket *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_server_side_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_server_hostname__doc__, +"The currently set server hostname (for SNI)."); +#define _ssl__SSLSocket_server_hostname_HAS_DOCSTR + +#if defined(_ssl__SSLSocket_server_hostname_HAS_DOCSTR) +# define _ssl__SSLSocket_server_hostname_DOCSTR _ssl__SSLSocket_server_hostname__doc__ +#else +# define _ssl__SSLSocket_server_hostname_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF) +# undef _SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF +# define _SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF {"server_hostname", (getter)_ssl__SSLSocket_server_hostname_get, (setter)_ssl__SSLSocket_server_hostname_set, _ssl__SSLSocket_server_hostname_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF {"server_hostname", (getter)_ssl__SSLSocket_server_hostname_get, NULL, _ssl__SSLSocket_server_hostname_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_server_hostname_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_server_hostname_get(PySSLSocket *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_server_hostname_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLSocket_owner_HAS_DOCSTR) +# define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ +#else +# define _ssl__SSLSocket_owner_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_OWNER_GETSETDEF) +# undef _SSL__SSLSOCKET_OWNER_GETSETDEF +# define _SSL__SSLSOCKET_OWNER_GETSETDEF {"owner", (getter)_ssl__SSLSocket_owner_get, (setter)_ssl__SSLSocket_owner_set, _ssl__SSLSocket_owner_DOCSTR}, +#else +# define _SSL__SSLSOCKET_OWNER_GETSETDEF {"owner", (getter)_ssl__SSLSocket_owner_get, NULL, _ssl__SSLSocket_owner_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_owner_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_owner_get(PySSLSocket *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_owner_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLSOCKET_OWNER_HAS_DOCSTR) +# define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ +#else +# define _ssl__SSLSocket_owner_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_OWNER_GETSETDEF) +# undef _SSL__SSLSOCKET_OWNER_GETSETDEF +# define _SSL__SSLSOCKET_OWNER_GETSETDEF {"owner", (getter)_ssl__SSLSocket_owner_get, (setter)_ssl__SSLSocket_owner_set, _ssl__SSLSocket_owner_DOCSTR}, +#else +# define _SSL__SSLSOCKET_OWNER_GETSETDEF {"owner", NULL, (setter)_ssl__SSLSocket_owner_set, NULL}, +#endif + +static int +_ssl__SSLSocket_owner_set_impl(PySSLSocket *self, PyObject *value); + +static int +_ssl__SSLSocket_owner_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_owner_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_ssl__SSLSocket_write__doc__, "write($self, b, /)\n" "--\n" @@ -237,7 +451,9 @@ _ssl__SSLSocket_write(PySSLSocket *self, PyObject *arg) if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLSocket_write_impl(self, &b); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for b */ @@ -263,7 +479,13 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_pending(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_pending_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_pending_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_read__doc__, @@ -301,7 +523,9 @@ _ssl__SSLSocket_read(PySSLSocket *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_ssl._SSLSocket.read requires 1 to 2 arguments"); goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLSocket_read_impl(self, len, group_right_1, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -327,7 +551,13 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_shutdown(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_shutdown_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_shutdown_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_get_channel_binding__doc__, @@ -401,7 +631,9 @@ _ssl__SSLSocket_get_channel_binding(PySSLSocket *self, PyObject *const *args, Py goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLSocket_get_channel_binding_impl(self, cb_type); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -422,117 +654,708 @@ _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self); static PyObject * _ssl__SSLSocket_verify_client_post_handshake(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_verify_client_post_handshake_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_verify_client_post_handshake_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLSocket_session_HAS_DOCSTR) +# define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ +#else +# define _ssl__SSLSocket_session_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SESSION_GETSETDEF) +# undef _SSL__SSLSOCKET_SESSION_GETSETDEF +# define _SSL__SSLSOCKET_SESSION_GETSETDEF {"session", (getter)_ssl__SSLSocket_session_get, (setter)_ssl__SSLSocket_session_set, _ssl__SSLSocket_session_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SESSION_GETSETDEF {"session", (getter)_ssl__SSLSocket_session_get, NULL, _ssl__SSLSocket_session_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_session_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_session_get(PySSLSocket *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_session_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLSOCKET_SESSION_HAS_DOCSTR) +# define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ +#else +# define _ssl__SSLSocket_session_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SESSION_GETSETDEF) +# undef _SSL__SSLSOCKET_SESSION_GETSETDEF +# define _SSL__SSLSOCKET_SESSION_GETSETDEF {"session", (getter)_ssl__SSLSocket_session_get, (setter)_ssl__SSLSocket_session_set, _ssl__SSLSocket_session_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SESSION_GETSETDEF {"session", NULL, (setter)_ssl__SSLSocket_session_set, NULL}, +#endif + +static int +_ssl__SSLSocket_session_set_impl(PySSLSocket *self, PyObject *value); + +static int +_ssl__SSLSocket_session_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_session_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_session_reused__doc__, +"Was the client session reused during handshake?"); +#define _ssl__SSLSocket_session_reused_HAS_DOCSTR + +#if defined(_ssl__SSLSocket_session_reused_HAS_DOCSTR) +# define _ssl__SSLSocket_session_reused_DOCSTR _ssl__SSLSocket_session_reused__doc__ +#else +# define _ssl__SSLSocket_session_reused_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF) +# undef _SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF +# define _SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF {"session_reused", (getter)_ssl__SSLSocket_session_reused_get, (setter)_ssl__SSLSocket_session_reused_set, _ssl__SSLSocket_session_reused_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF {"session_reused", (getter)_ssl__SSLSocket_session_reused_get, NULL, _ssl__SSLSocket_session_reused_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_session_reused_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_session_reused_get(PySSLSocket *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_session_reused_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +static PyObject * +_ssl__SSLContext_impl(PyTypeObject *type, int proto_version); + +static PyObject * +_ssl__SSLContext(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = get_state_type(type)->PySSLContext_Type; + int proto_version; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("_SSLContext", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("_SSLContext", PyTuple_GET_SIZE(args), 1, 1)) { + goto exit; + } + proto_version = PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); + if (proto_version == -1 && PyErr_Occurred()) { + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(type); + return_value = _ssl__SSLContext_impl(type, proto_version); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_set_ciphers__doc__, +"set_ciphers($self, cipherlist, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_CIPHERS_METHODDEF \ + {"set_ciphers", (PyCFunction)_ssl__SSLContext_set_ciphers, METH_O, _ssl__SSLContext_set_ciphers__doc__}, + +static PyObject * +_ssl__SSLContext_set_ciphers_impl(PySSLContext *self, const char *cipherlist); + +static PyObject * +_ssl__SSLContext_set_ciphers(PySSLContext *self, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *cipherlist; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("set_ciphers", "argument", "str", arg); + goto exit; + } + Py_ssize_t cipherlist_length; + cipherlist = PyUnicode_AsUTF8AndSize(arg, &cipherlist_length); + if (cipherlist == NULL) { + goto exit; + } + if (strlen(cipherlist) != (size_t)cipherlist_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_ciphers_impl(self, cipherlist); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_get_ciphers__doc__, +"get_ciphers($self, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_GET_CIPHERS_METHODDEF \ + {"get_ciphers", (PyCFunction)_ssl__SSLContext_get_ciphers, METH_NOARGS, _ssl__SSLContext_get_ciphers__doc__}, + +static PyObject * +_ssl__SSLContext_get_ciphers_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_get_ciphers(PySSLContext *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_get_ciphers_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext__set_alpn_protocols__doc__, +"_set_alpn_protocols($self, protos, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT__SET_ALPN_PROTOCOLS_METHODDEF \ + {"_set_alpn_protocols", (PyCFunction)_ssl__SSLContext__set_alpn_protocols, METH_O, _ssl__SSLContext__set_alpn_protocols__doc__}, + +static PyObject * +_ssl__SSLContext__set_alpn_protocols_impl(PySSLContext *self, + Py_buffer *protos); + +static PyObject * +_ssl__SSLContext__set_alpn_protocols(PySSLContext *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer protos = {NULL, NULL}; + + if (PyObject_GetBuffer(arg, &protos, PyBUF_SIMPLE) != 0) { + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext__set_alpn_protocols_impl(self, &protos); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for protos */ + if (protos.obj) { + PyBuffer_Release(&protos); + } + + return return_value; +} + +#if defined(_ssl__SSLContext_verify_mode_HAS_DOCSTR) +# define _ssl__SSLContext_verify_mode_DOCSTR _ssl__SSLContext_verify_mode__doc__ +#else +# define _ssl__SSLContext_verify_mode_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF) +# undef _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF +# define _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF {"verify_mode", (getter)_ssl__SSLContext_verify_mode_get, (setter)_ssl__SSLContext_verify_mode_set, _ssl__SSLContext_verify_mode_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF {"verify_mode", (getter)_ssl__SSLContext_verify_mode_get, NULL, _ssl__SSLContext_verify_mode_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_verify_mode_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_verify_mode_get(PySSLContext *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_verify_mode_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLCONTEXT_VERIFY_MODE_HAS_DOCSTR) +# define _ssl__SSLContext_verify_mode_DOCSTR _ssl__SSLContext_verify_mode__doc__ +#else +# define _ssl__SSLContext_verify_mode_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF) +# undef _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF +# define _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF {"verify_mode", (getter)_ssl__SSLContext_verify_mode_get, (setter)_ssl__SSLContext_verify_mode_set, _ssl__SSLContext_verify_mode_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF {"verify_mode", NULL, (setter)_ssl__SSLContext_verify_mode_set, NULL}, +#endif + +static int +_ssl__SSLContext_verify_mode_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_verify_mode_set(PySSLContext *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_verify_mode_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLContext_verify_flags_HAS_DOCSTR) +# define _ssl__SSLContext_verify_flags_DOCSTR _ssl__SSLContext_verify_flags__doc__ +#else +# define _ssl__SSLContext_verify_flags_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF) +# undef _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF +# define _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF {"verify_flags", (getter)_ssl__SSLContext_verify_flags_get, (setter)_ssl__SSLContext_verify_flags_set, _ssl__SSLContext_verify_flags_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF {"verify_flags", (getter)_ssl__SSLContext_verify_flags_get, NULL, _ssl__SSLContext_verify_flags_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_verify_flags_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_verify_flags_get(PySSLContext *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_verify_flags_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_HAS_DOCSTR) +# define _ssl__SSLContext_verify_flags_DOCSTR _ssl__SSLContext_verify_flags__doc__ +#else +# define _ssl__SSLContext_verify_flags_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF) +# undef _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF +# define _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF {"verify_flags", (getter)_ssl__SSLContext_verify_flags_get, (setter)_ssl__SSLContext_verify_flags_set, _ssl__SSLContext_verify_flags_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF {"verify_flags", NULL, (setter)_ssl__SSLContext_verify_flags_set, NULL}, +#endif + +static int +_ssl__SSLContext_verify_flags_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_verify_flags_set(PySSLContext *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_verify_flags_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLContext_minimum_version_HAS_DOCSTR) +# define _ssl__SSLContext_minimum_version_DOCSTR _ssl__SSLContext_minimum_version__doc__ +#else +# define _ssl__SSLContext_minimum_version_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF) +# undef _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF +# define _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF {"minimum_version", (getter)_ssl__SSLContext_minimum_version_get, (setter)_ssl__SSLContext_minimum_version_set, _ssl__SSLContext_minimum_version_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF {"minimum_version", (getter)_ssl__SSLContext_minimum_version_get, NULL, _ssl__SSLContext_minimum_version_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_minimum_version_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_minimum_version_get(PySSLContext *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_minimum_version_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_HAS_DOCSTR) +# define _ssl__SSLContext_minimum_version_DOCSTR _ssl__SSLContext_minimum_version__doc__ +#else +# define _ssl__SSLContext_minimum_version_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF) +# undef _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF +# define _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF {"minimum_version", (getter)_ssl__SSLContext_minimum_version_get, (setter)_ssl__SSLContext_minimum_version_set, _ssl__SSLContext_minimum_version_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF {"minimum_version", NULL, (setter)_ssl__SSLContext_minimum_version_set, NULL}, +#endif + +static int +_ssl__SSLContext_minimum_version_set_impl(PySSLContext *self, + PyObject *value); + +static int +_ssl__SSLContext_minimum_version_set(PySSLContext *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_minimum_version_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLContext_maximum_version_HAS_DOCSTR) +# define _ssl__SSLContext_maximum_version_DOCSTR _ssl__SSLContext_maximum_version__doc__ +#else +# define _ssl__SSLContext_maximum_version_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF) +# undef _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF +# define _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"maximum_version", (getter)_ssl__SSLContext_maximum_version_get, (setter)_ssl__SSLContext_maximum_version_set, _ssl__SSLContext_maximum_version_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"maximum_version", (getter)_ssl__SSLContext_maximum_version_get, NULL, _ssl__SSLContext_maximum_version_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_maximum_version_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_maximum_version_get(PySSLContext *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_maximum_version_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_HAS_DOCSTR) +# define _ssl__SSLContext_maximum_version_DOCSTR _ssl__SSLContext_maximum_version__doc__ +#else +# define _ssl__SSLContext_maximum_version_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF) +# undef _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF +# define _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"maximum_version", (getter)_ssl__SSLContext_maximum_version_get, (setter)_ssl__SSLContext_maximum_version_set, _ssl__SSLContext_maximum_version_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"maximum_version", NULL, (setter)_ssl__SSLContext_maximum_version_set, NULL}, +#endif + +static int +_ssl__SSLContext_maximum_version_set_impl(PySSLContext *self, + PyObject *value); + +static int +_ssl__SSLContext_maximum_version_set(PySSLContext *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_maximum_version_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLContext_num_tickets_HAS_DOCSTR) +# define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ +#else +# define _ssl__SSLContext_num_tickets_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF) +# undef _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF +# define _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF {"num_tickets", (getter)_ssl__SSLContext_num_tickets_get, (setter)_ssl__SSLContext_num_tickets_set, _ssl__SSLContext_num_tickets_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF {"num_tickets", (getter)_ssl__SSLContext_num_tickets_get, NULL, _ssl__SSLContext_num_tickets_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_num_tickets_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_num_tickets_get(PySSLContext *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_num_tickets_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLCONTEXT_NUM_TICKETS_HAS_DOCSTR) +# define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ +#else +# define _ssl__SSLContext_num_tickets_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF) +# undef _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF +# define _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF {"num_tickets", (getter)_ssl__SSLContext_num_tickets_get, (setter)_ssl__SSLContext_num_tickets_set, _ssl__SSLContext_num_tickets_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF {"num_tickets", NULL, (setter)_ssl__SSLContext_num_tickets_set, NULL}, +#endif + +static int +_ssl__SSLContext_num_tickets_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_num_tickets_set(PySSLContext *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_num_tickets_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLContext_security_level_HAS_DOCSTR) +# define _ssl__SSLContext_security_level_DOCSTR _ssl__SSLContext_security_level__doc__ +#else +# define _ssl__SSLContext_security_level_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF) +# undef _SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF +# define _SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF {"security_level", (getter)_ssl__SSLContext_security_level_get, (setter)_ssl__SSLContext_security_level_set, _ssl__SSLContext_security_level_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF {"security_level", (getter)_ssl__SSLContext_security_level_get, NULL, _ssl__SSLContext_security_level_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_security_level_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_security_level_get(PySSLContext *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_security_level_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLContext_options_HAS_DOCSTR) +# define _ssl__SSLContext_options_DOCSTR _ssl__SSLContext_options__doc__ +#else +# define _ssl__SSLContext_options_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_OPTIONS_GETSETDEF) +# undef _SSL__SSLCONTEXT_OPTIONS_GETSETDEF +# define _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"options", (getter)_ssl__SSLContext_options_get, (setter)_ssl__SSLContext_options_set, _ssl__SSLContext_options_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"options", (getter)_ssl__SSLContext_options_get, NULL, _ssl__SSLContext_options_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_options_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_options_get(PySSLContext *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_options_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLCONTEXT_OPTIONS_HAS_DOCSTR) +# define _ssl__SSLContext_options_DOCSTR _ssl__SSLContext_options__doc__ +#else +# define _ssl__SSLContext_options_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_OPTIONS_GETSETDEF) +# undef _SSL__SSLCONTEXT_OPTIONS_GETSETDEF +# define _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"options", (getter)_ssl__SSLContext_options_get, (setter)_ssl__SSLContext_options_set, _ssl__SSLContext_options_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"options", NULL, (setter)_ssl__SSLContext_options_set, NULL}, +#endif + +static int +_ssl__SSLContext_options_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_options_set(PySSLContext *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_options_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; } +#if defined(_ssl__SSLContext__host_flags_HAS_DOCSTR) +# define _ssl__SSLContext__host_flags_DOCSTR _ssl__SSLContext__host_flags__doc__ +#else +# define _ssl__SSLContext__host_flags_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF) +# undef _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF +# define _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF {"_host_flags", (getter)_ssl__SSLContext__host_flags_get, (setter)_ssl__SSLContext__host_flags_set, _ssl__SSLContext__host_flags_DOCSTR}, +#else +# define _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF {"_host_flags", (getter)_ssl__SSLContext__host_flags_get, NULL, _ssl__SSLContext__host_flags_DOCSTR}, +#endif + static PyObject * -_ssl__SSLContext_impl(PyTypeObject *type, int proto_version); +_ssl__SSLContext__host_flags_get_impl(PySSLContext *self); static PyObject * -_ssl__SSLContext(PyTypeObject *type, PyObject *args, PyObject *kwargs) +_ssl__SSLContext__host_flags_get(PySSLContext *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; - PyTypeObject *base_tp = get_state_type(type)->PySSLContext_Type; - int proto_version; - if ((type == base_tp || type->tp_init == base_tp->tp_init) && - !_PyArg_NoKeywords("_SSLContext", kwargs)) { - goto exit; - } - if (!_PyArg_CheckPositional("_SSLContext", PyTuple_GET_SIZE(args), 1, 1)) { - goto exit; - } - proto_version = PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); - if (proto_version == -1 && PyErr_Occurred()) { - goto exit; - } - return_value = _ssl__SSLContext_impl(type, proto_version); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext__host_flags_get_impl(self); + Py_END_CRITICAL_SECTION(); -exit: return return_value; } -PyDoc_STRVAR(_ssl__SSLContext_set_ciphers__doc__, -"set_ciphers($self, cipherlist, /)\n" -"--\n" -"\n"); +#if defined(_SSL__SSLCONTEXT__HOST_FLAGS_HAS_DOCSTR) +# define _ssl__SSLContext__host_flags_DOCSTR _ssl__SSLContext__host_flags__doc__ +#else +# define _ssl__SSLContext__host_flags_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF) +# undef _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF +# define _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF {"_host_flags", (getter)_ssl__SSLContext__host_flags_get, (setter)_ssl__SSLContext__host_flags_set, _ssl__SSLContext__host_flags_DOCSTR}, +#else +# define _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF {"_host_flags", NULL, (setter)_ssl__SSLContext__host_flags_set, NULL}, +#endif -#define _SSL__SSLCONTEXT_SET_CIPHERS_METHODDEF \ - {"set_ciphers", (PyCFunction)_ssl__SSLContext_set_ciphers, METH_O, _ssl__SSLContext_set_ciphers__doc__}, +static int +_ssl__SSLContext__host_flags_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext__host_flags_set(PySSLContext *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext__host_flags_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLContext_check_hostname_HAS_DOCSTR) +# define _ssl__SSLContext_check_hostname_DOCSTR _ssl__SSLContext_check_hostname__doc__ +#else +# define _ssl__SSLContext_check_hostname_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF) +# undef _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF +# define _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF {"check_hostname", (getter)_ssl__SSLContext_check_hostname_get, (setter)_ssl__SSLContext_check_hostname_set, _ssl__SSLContext_check_hostname_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF {"check_hostname", (getter)_ssl__SSLContext_check_hostname_get, NULL, _ssl__SSLContext_check_hostname_DOCSTR}, +#endif static PyObject * -_ssl__SSLContext_set_ciphers_impl(PySSLContext *self, const char *cipherlist); +_ssl__SSLContext_check_hostname_get_impl(PySSLContext *self); static PyObject * -_ssl__SSLContext_set_ciphers(PySSLContext *self, PyObject *arg) +_ssl__SSLContext_check_hostname_get(PySSLContext *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; - const char *cipherlist; - if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("set_ciphers", "argument", "str", arg); - goto exit; - } - Py_ssize_t cipherlist_length; - cipherlist = PyUnicode_AsUTF8AndSize(arg, &cipherlist_length); - if (cipherlist == NULL) { - goto exit; - } - if (strlen(cipherlist) != (size_t)cipherlist_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } - return_value = _ssl__SSLContext_set_ciphers_impl(self, cipherlist); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_check_hostname_get_impl(self); + Py_END_CRITICAL_SECTION(); -exit: return return_value; } -PyDoc_STRVAR(_ssl__SSLContext_get_ciphers__doc__, -"get_ciphers($self, /)\n" -"--\n" -"\n"); - -#define _SSL__SSLCONTEXT_GET_CIPHERS_METHODDEF \ - {"get_ciphers", (PyCFunction)_ssl__SSLContext_get_ciphers, METH_NOARGS, _ssl__SSLContext_get_ciphers__doc__}, +#if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_HAS_DOCSTR) +# define _ssl__SSLContext_check_hostname_DOCSTR _ssl__SSLContext_check_hostname__doc__ +#else +# define _ssl__SSLContext_check_hostname_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF) +# undef _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF +# define _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF {"check_hostname", (getter)_ssl__SSLContext_check_hostname_get, (setter)_ssl__SSLContext_check_hostname_set, _ssl__SSLContext_check_hostname_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF {"check_hostname", NULL, (setter)_ssl__SSLContext_check_hostname_set, NULL}, +#endif -static PyObject * -_ssl__SSLContext_get_ciphers_impl(PySSLContext *self); +static int +_ssl__SSLContext_check_hostname_set_impl(PySSLContext *self, PyObject *value); -static PyObject * -_ssl__SSLContext_get_ciphers(PySSLContext *self, PyObject *Py_UNUSED(ignored)) +static int +_ssl__SSLContext_check_hostname_set(PySSLContext *self, PyObject *value, void *Py_UNUSED(context)) { - return _ssl__SSLContext_get_ciphers_impl(self); -} + int return_value; -PyDoc_STRVAR(_ssl__SSLContext__set_alpn_protocols__doc__, -"_set_alpn_protocols($self, protos, /)\n" -"--\n" -"\n"); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_check_hostname_set_impl(self, value); + Py_END_CRITICAL_SECTION(); -#define _SSL__SSLCONTEXT__SET_ALPN_PROTOCOLS_METHODDEF \ - {"_set_alpn_protocols", (PyCFunction)_ssl__SSLContext__set_alpn_protocols, METH_O, _ssl__SSLContext__set_alpn_protocols__doc__}, + return return_value; +} + +#if defined(_ssl__SSLContext_protocol_HAS_DOCSTR) +# define _ssl__SSLContext_protocol_DOCSTR _ssl__SSLContext_protocol__doc__ +#else +# define _ssl__SSLContext_protocol_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_PROTOCOL_GETSETDEF) +# undef _SSL__SSLCONTEXT_PROTOCOL_GETSETDEF +# define _SSL__SSLCONTEXT_PROTOCOL_GETSETDEF {"protocol", (getter)_ssl__SSLContext_protocol_get, (setter)_ssl__SSLContext_protocol_set, _ssl__SSLContext_protocol_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_PROTOCOL_GETSETDEF {"protocol", (getter)_ssl__SSLContext_protocol_get, NULL, _ssl__SSLContext_protocol_DOCSTR}, +#endif static PyObject * -_ssl__SSLContext__set_alpn_protocols_impl(PySSLContext *self, - Py_buffer *protos); +_ssl__SSLContext_protocol_get_impl(PySSLContext *self); static PyObject * -_ssl__SSLContext__set_alpn_protocols(PySSLContext *self, PyObject *arg) +_ssl__SSLContext_protocol_get(PySSLContext *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; - Py_buffer protos = {NULL, NULL}; - if (PyObject_GetBuffer(arg, &protos, PyBUF_SIMPLE) != 0) { - goto exit; - } - return_value = _ssl__SSLContext__set_alpn_protocols_impl(self, &protos); - -exit: - /* Cleanup for protos */ - if (protos.obj) { - PyBuffer_Release(&protos); - } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_protocol_get_impl(self); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -600,7 +1423,9 @@ _ssl__SSLContext_load_cert_chain(PySSLContext *self, PyObject *const *args, Py_s } password = args[2]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_load_cert_chain_impl(self, certfile, keyfile, password); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -676,7 +1501,9 @@ _ssl__SSLContext_load_verify_locations(PySSLContext *self, PyObject *const *args } cadata = args[2]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_load_verify_locations_impl(self, cafile, capath, cadata); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -690,6 +1517,21 @@ PyDoc_STRVAR(_ssl__SSLContext_load_dh_params__doc__, #define _SSL__SSLCONTEXT_LOAD_DH_PARAMS_METHODDEF \ {"load_dh_params", (PyCFunction)_ssl__SSLContext_load_dh_params, METH_O, _ssl__SSLContext_load_dh_params__doc__}, +static PyObject * +_ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath); + +static PyObject * +_ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_load_dh_params_impl(self, filepath); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_ssl__SSLContext__wrap_socket__doc__, "_wrap_socket($self, /, sock, server_side, server_hostname=None, *,\n" " owner=None, session=None)\n" @@ -775,7 +1617,9 @@ _ssl__SSLContext__wrap_socket(PySSLContext *self, PyObject *const *args, Py_ssiz } session = args[4]; skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext__wrap_socket_impl(self, sock, server_side, hostname_obj, owner, session); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -873,7 +1717,9 @@ _ssl__SSLContext__wrap_bio(PySSLContext *self, PyObject *const *args, Py_ssize_t } session = args[5]; skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext__wrap_bio_impl(self, incoming, outgoing, server_side, hostname_obj, owner, session); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -893,7 +1739,13 @@ _ssl__SSLContext_session_stats_impl(PySSLContext *self); static PyObject * _ssl__SSLContext_session_stats(PySSLContext *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLContext_session_stats_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_session_stats_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLContext_set_default_verify_paths__doc__, @@ -910,7 +1762,13 @@ _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self); static PyObject * _ssl__SSLContext_set_default_verify_paths(PySSLContext *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLContext_set_default_verify_paths_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_default_verify_paths_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLContext_set_ecdh_curve__doc__, @@ -921,6 +1779,75 @@ PyDoc_STRVAR(_ssl__SSLContext_set_ecdh_curve__doc__, #define _SSL__SSLCONTEXT_SET_ECDH_CURVE_METHODDEF \ {"set_ecdh_curve", (PyCFunction)_ssl__SSLContext_set_ecdh_curve, METH_O, _ssl__SSLContext_set_ecdh_curve__doc__}, +static PyObject * +_ssl__SSLContext_set_ecdh_curve_impl(PySSLContext *self, PyObject *name); + +static PyObject * +_ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_ecdh_curve_impl(self, name); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl__SSLContext_sni_callback_HAS_DOCSTR) +# define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ +#else +# define _ssl__SSLContext_sni_callback_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF) +# undef _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF +# define _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF {"sni_callback", (getter)_ssl__SSLContext_sni_callback_get, (setter)_ssl__SSLContext_sni_callback_set, _ssl__SSLContext_sni_callback_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF {"sni_callback", (getter)_ssl__SSLContext_sni_callback_get, NULL, _ssl__SSLContext_sni_callback_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_sni_callback_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_sni_callback_get(PySSLContext *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_sni_callback_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_HAS_DOCSTR) +# define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ +#else +# define _ssl__SSLContext_sni_callback_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF) +# undef _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF +# define _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF {"sni_callback", (getter)_ssl__SSLContext_sni_callback_get, (setter)_ssl__SSLContext_sni_callback_set, _ssl__SSLContext_sni_callback_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF {"sni_callback", NULL, (setter)_ssl__SSLContext_sni_callback_set, NULL}, +#endif + +static int +_ssl__SSLContext_sni_callback_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_sni_callback_set(PySSLContext *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_sni_callback_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_ssl__SSLContext_cert_store_stats__doc__, "cert_store_stats($self, /)\n" "--\n" @@ -942,7 +1869,13 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self); static PyObject * _ssl__SSLContext_cert_store_stats(PySSLContext *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLContext_cert_store_stats_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_cert_store_stats_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLContext_get_ca_certs__doc__, @@ -1008,7 +1941,9 @@ _ssl__SSLContext_get_ca_certs(PySSLContext *self, PyObject *const *args, Py_ssiz goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_get_ca_certs_impl(self, binary_form); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1063,7 +1998,9 @@ _ssl__SSLContext_set_psk_client_callback(PySSLContext *self, PyObject *const *ar goto exit; } callback = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_set_psk_client_callback_impl(self, callback); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1143,7 +2080,9 @@ _ssl__SSLContext_set_psk_server_callback(PySSLContext *self, PyObject *const *ar goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_set_psk_server_callback_impl(self, callback, identity_hint); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1166,12 +2105,68 @@ _ssl_MemoryBIO(PyTypeObject *type, PyObject *args, PyObject *kwargs) !_PyArg_NoKeywords("MemoryBIO", kwargs)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(type); return_value = _ssl_MemoryBIO_impl(type); + Py_END_CRITICAL_SECTION(); exit: return return_value; } +#if defined(_ssl_MemoryBIO_pending_HAS_DOCSTR) +# define _ssl_MemoryBIO_pending_DOCSTR _ssl_MemoryBIO_pending__doc__ +#else +# define _ssl_MemoryBIO_pending_DOCSTR NULL +#endif +#if defined(_SSL_MEMORYBIO_PENDING_GETSETDEF) +# undef _SSL_MEMORYBIO_PENDING_GETSETDEF +# define _SSL_MEMORYBIO_PENDING_GETSETDEF {"pending", (getter)_ssl_MemoryBIO_pending_get, (setter)_ssl_MemoryBIO_pending_set, _ssl_MemoryBIO_pending_DOCSTR}, +#else +# define _SSL_MEMORYBIO_PENDING_GETSETDEF {"pending", (getter)_ssl_MemoryBIO_pending_get, NULL, _ssl_MemoryBIO_pending_DOCSTR}, +#endif + +static PyObject * +_ssl_MemoryBIO_pending_get_impl(PySSLMemoryBIO *self); + +static PyObject * +_ssl_MemoryBIO_pending_get(PySSLMemoryBIO *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_MemoryBIO_pending_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl_MemoryBIO_eof_HAS_DOCSTR) +# define _ssl_MemoryBIO_eof_DOCSTR _ssl_MemoryBIO_eof__doc__ +#else +# define _ssl_MemoryBIO_eof_DOCSTR NULL +#endif +#if defined(_SSL_MEMORYBIO_EOF_GETSETDEF) +# undef _SSL_MEMORYBIO_EOF_GETSETDEF +# define _SSL_MEMORYBIO_EOF_GETSETDEF {"eof", (getter)_ssl_MemoryBIO_eof_get, (setter)_ssl_MemoryBIO_eof_set, _ssl_MemoryBIO_eof_DOCSTR}, +#else +# define _SSL_MEMORYBIO_EOF_GETSETDEF {"eof", (getter)_ssl_MemoryBIO_eof_get, NULL, _ssl_MemoryBIO_eof_DOCSTR}, +#endif + +static PyObject * +_ssl_MemoryBIO_eof_get_impl(PySSLMemoryBIO *self); + +static PyObject * +_ssl_MemoryBIO_eof_get(PySSLMemoryBIO *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_MemoryBIO_eof_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_ssl_MemoryBIO_read__doc__, "read($self, size=-1, /)\n" "--\n" @@ -1206,7 +2201,9 @@ _ssl_MemoryBIO_read(PySSLMemoryBIO *self, PyObject *const *args, Py_ssize_t narg goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl_MemoryBIO_read_impl(self, len); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1235,7 +2232,9 @@ _ssl_MemoryBIO_write(PySSLMemoryBIO *self, PyObject *arg) if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl_MemoryBIO_write_impl(self, &b); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for b */ @@ -1263,7 +2262,148 @@ _ssl_MemoryBIO_write_eof_impl(PySSLMemoryBIO *self); static PyObject * _ssl_MemoryBIO_write_eof(PySSLMemoryBIO *self, PyObject *Py_UNUSED(ignored)) { - return _ssl_MemoryBIO_write_eof_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_MemoryBIO_write_eof_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl_SSLSession_time_HAS_DOCSTR) +# define _ssl_SSLSession_time_DOCSTR _ssl_SSLSession_time__doc__ +#else +# define _ssl_SSLSession_time_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_TIME_GETSETDEF) +# undef _SSL_SSLSESSION_TIME_GETSETDEF +# define _SSL_SSLSESSION_TIME_GETSETDEF {"time", (getter)_ssl_SSLSession_time_get, (setter)_ssl_SSLSession_time_set, _ssl_SSLSession_time_DOCSTR}, +#else +# define _SSL_SSLSESSION_TIME_GETSETDEF {"time", (getter)_ssl_SSLSession_time_get, NULL, _ssl_SSLSession_time_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_time_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_time_get(PySSLSession *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_time_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl_SSLSession_timeout_HAS_DOCSTR) +# define _ssl_SSLSession_timeout_DOCSTR _ssl_SSLSession_timeout__doc__ +#else +# define _ssl_SSLSession_timeout_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_TIMEOUT_GETSETDEF) +# undef _SSL_SSLSESSION_TIMEOUT_GETSETDEF +# define _SSL_SSLSESSION_TIMEOUT_GETSETDEF {"timeout", (getter)_ssl_SSLSession_timeout_get, (setter)_ssl_SSLSession_timeout_set, _ssl_SSLSession_timeout_DOCSTR}, +#else +# define _SSL_SSLSESSION_TIMEOUT_GETSETDEF {"timeout", (getter)_ssl_SSLSession_timeout_get, NULL, _ssl_SSLSession_timeout_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_timeout_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_timeout_get(PySSLSession *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_timeout_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl_SSLSession_ticket_lifetime_hint_HAS_DOCSTR) +# define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR _ssl_SSLSession_ticket_lifetime_hint__doc__ +#else +# define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF) +# undef _SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF +# define _SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF {"ticket_lifetime_hint", (getter)_ssl_SSLSession_ticket_lifetime_hint_get, (setter)_ssl_SSLSession_ticket_lifetime_hint_set, _ssl_SSLSession_ticket_lifetime_hint_DOCSTR}, +#else +# define _SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF {"ticket_lifetime_hint", (getter)_ssl_SSLSession_ticket_lifetime_hint_get, NULL, _ssl_SSLSession_ticket_lifetime_hint_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_ticket_lifetime_hint_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_ticket_lifetime_hint_get(PySSLSession *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_ticket_lifetime_hint_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl_SSLSession_id_HAS_DOCSTR) +# define _ssl_SSLSession_id_DOCSTR _ssl_SSLSession_id__doc__ +#else +# define _ssl_SSLSession_id_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_ID_GETSETDEF) +# undef _SSL_SSLSESSION_ID_GETSETDEF +# define _SSL_SSLSESSION_ID_GETSETDEF {"id", (getter)_ssl_SSLSession_id_get, (setter)_ssl_SSLSession_id_set, _ssl_SSLSession_id_DOCSTR}, +#else +# define _SSL_SSLSESSION_ID_GETSETDEF {"id", (getter)_ssl_SSLSession_id_get, NULL, _ssl_SSLSession_id_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_id_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_id_get(PySSLSession *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_id_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_ssl_SSLSession_has_ticket_HAS_DOCSTR) +# define _ssl_SSLSession_has_ticket_DOCSTR _ssl_SSLSession_has_ticket__doc__ +#else +# define _ssl_SSLSession_has_ticket_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_HAS_TICKET_GETSETDEF) +# undef _SSL_SSLSESSION_HAS_TICKET_GETSETDEF +# define _SSL_SSLSESSION_HAS_TICKET_GETSETDEF {"has_ticket", (getter)_ssl_SSLSession_has_ticket_get, (setter)_ssl_SSLSession_has_ticket_set, _ssl_SSLSession_has_ticket_DOCSTR}, +#else +# define _SSL_SSLSESSION_HAS_TICKET_GETSETDEF {"has_ticket", (getter)_ssl_SSLSession_has_ticket_get, NULL, _ssl_SSLSession_has_ticket_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_has_ticket_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_has_ticket_get(PySSLSession *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_has_ticket_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl_RAND_add__doc__, @@ -1316,7 +2456,9 @@ _ssl_RAND_add(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_RAND_add_impl(module, &view, entropy); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for view */ @@ -1349,7 +2491,9 @@ _ssl_RAND_bytes(PyObject *module, PyObject *arg) if (n == -1 && PyErr_Occurred()) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_RAND_bytes_impl(module, n); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1373,7 +2517,13 @@ _ssl_RAND_status_impl(PyObject *module); static PyObject * _ssl_RAND_status(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return _ssl_RAND_status_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = _ssl_RAND_status_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl_get_default_verify_paths__doc__, @@ -1393,7 +2543,13 @@ _ssl_get_default_verify_paths_impl(PyObject *module); static PyObject * _ssl_get_default_verify_paths(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return _ssl_get_default_verify_paths_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = _ssl_get_default_verify_paths_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl_txt2obj__doc__, @@ -1470,7 +2626,9 @@ _ssl_txt2obj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_txt2obj_impl(module, txt, name); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1498,7 +2656,9 @@ _ssl_nid2obj(PyObject *module, PyObject *arg) if (nid == -1 && PyErr_Occurred()) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_nid2obj_impl(module, nid); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1573,7 +2733,9 @@ _ssl_enum_certificates(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_enum_certificates_impl(module, store_name); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1649,7 +2811,9 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_enum_crls_impl(module, store_name); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1664,4 +2828,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=fd1c3378fbba5240 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c1489122072a9f5e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_statisticsmodule.c.h b/Modules/clinic/_statisticsmodule.c.h index 653a2138aaad70..cd5b9bea8db2f2 100644 --- a/Modules/clinic/_statisticsmodule.c.h +++ b/Modules/clinic/_statisticsmodule.c.h @@ -2,15 +2,13 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() - PyDoc_STRVAR(_statistics__normal_dist_inv_cdf__doc__, "_normal_dist_inv_cdf($module, p, mu, sigma, /)\n" "--\n" "\n"); #define _STATISTICS__NORMAL_DIST_INV_CDF_METHODDEF \ - {"_normal_dist_inv_cdf", _PyCFunction_CAST(_statistics__normal_dist_inv_cdf), METH_FASTCALL, _statistics__normal_dist_inv_cdf__doc__}, + {"_normal_dist_inv_cdf", (PyCFunction)(void(*)(void))_statistics__normal_dist_inv_cdf, METH_FASTCALL, _statistics__normal_dist_inv_cdf__doc__}, static double _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu, @@ -25,38 +23,21 @@ _statistics__normal_dist_inv_cdf(PyObject *module, PyObject *const *args, Py_ssi double sigma; double _return_value; - if (!_PyArg_CheckPositional("_normal_dist_inv_cdf", nargs, 3, 3)) { + if (nargs != 3) { + PyErr_Format(PyExc_TypeError, "_normal_dist_inv_cdf expected 3 arguments, got %zd", nargs); goto exit; } - if (PyFloat_CheckExact(args[0])) { - p = PyFloat_AS_DOUBLE(args[0]); - } - else - { - p = PyFloat_AsDouble(args[0]); - if (p == -1.0 && PyErr_Occurred()) { - goto exit; - } - } - if (PyFloat_CheckExact(args[1])) { - mu = PyFloat_AS_DOUBLE(args[1]); - } - else - { - mu = PyFloat_AsDouble(args[1]); - if (mu == -1.0 && PyErr_Occurred()) { - goto exit; - } + p = PyFloat_AsDouble(args[0]); + if (p == -1.0 && PyErr_Occurred()) { + goto exit; } - if (PyFloat_CheckExact(args[2])) { - sigma = PyFloat_AS_DOUBLE(args[2]); + mu = PyFloat_AsDouble(args[1]); + if (mu == -1.0 && PyErr_Occurred()) { + goto exit; } - else - { - sigma = PyFloat_AsDouble(args[2]); - if (sigma == -1.0 && PyErr_Occurred()) { - goto exit; - } + sigma = PyFloat_AsDouble(args[2]); + if (sigma == -1.0 && PyErr_Occurred()) { + goto exit; } _return_value = _statistics__normal_dist_inv_cdf_impl(module, p, mu, sigma); if ((_return_value == -1.0) && PyErr_Occurred()) { @@ -67,4 +48,4 @@ _statistics__normal_dist_inv_cdf(PyObject *module, PyObject *const *args, Py_ssi exit: return return_value; } -/*[clinic end generated code: output=e7cead17f9f3e19f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0f0c849d51f16f1b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index bb516be37ec3f0..7e29998c7db520 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -234,10 +234,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + if (PyBytes_Check(args[0])) { + if (PyBytes_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 1 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[0])); + goto exit; + } a = PyBytes_AS_STRING(args[0])[0]; } - else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + else if (PyByteArray_Check(args[0])) { + if (PyByteArray_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 1 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[0])); + goto exit; + } a = PyByteArray_AS_STRING(args[0])[0]; } else { @@ -247,10 +261,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } b = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } b = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -260,10 +288,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + if (PyBytes_Check(args[2])) { + if (PyBytes_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 3 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[2])); + goto exit; + } c = PyBytes_AS_STRING(args[2])[0]; } - else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + else if (PyByteArray_Check(args[2])) { + if (PyByteArray_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 3 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[2])); + goto exit; + } c = PyByteArray_AS_STRING(args[2])[0]; } else { @@ -273,10 +315,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + if (PyBytes_Check(args[3])) { + if (PyBytes_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 4 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[3])); + goto exit; + } d = PyBytes_AS_STRING(args[3])[0]; } - else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + else if (PyByteArray_Check(args[3])) { + if (PyByteArray_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 4 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[3])); + goto exit; + } d = PyByteArray_AS_STRING(args[3])[0]; } else { @@ -286,10 +342,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + if (PyBytes_Check(args[4])) { + if (PyBytes_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 5 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[4])); + goto exit; + } e = PyBytes_AS_STRING(args[4])[0]; } - else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + else if (PyByteArray_Check(args[4])) { + if (PyByteArray_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 5 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[4])); + goto exit; + } e = PyByteArray_AS_STRING(args[4])[0]; } else { @@ -299,10 +369,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 6) { goto skip_optional; } - if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + if (PyBytes_Check(args[5])) { + if (PyBytes_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 6 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[5])); + goto exit; + } f = PyBytes_AS_STRING(args[5])[0]; } - else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + else if (PyByteArray_Check(args[5])) { + if (PyByteArray_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 6 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[5])); + goto exit; + } f = PyByteArray_AS_STRING(args[5])[0]; } else { @@ -312,10 +396,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 7) { goto skip_optional; } - if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + if (PyBytes_Check(args[6])) { + if (PyBytes_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 7 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[6])); + goto exit; + } g = PyBytes_AS_STRING(args[6])[0]; } - else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + else if (PyByteArray_Check(args[6])) { + if (PyByteArray_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 7 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[6])); + goto exit; + } g = PyByteArray_AS_STRING(args[6])[0]; } else { @@ -325,10 +423,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 8) { goto skip_optional; } - if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + if (PyBytes_Check(args[7])) { + if (PyBytes_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 8 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[7])); + goto exit; + } h = PyBytes_AS_STRING(args[7])[0]; } - else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + else if (PyByteArray_Check(args[7])) { + if (PyByteArray_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 8 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[7])); + goto exit; + } h = PyByteArray_AS_STRING(args[7])[0]; } else { @@ -338,10 +450,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 9) { goto skip_optional; } - if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + if (PyBytes_Check(args[8])) { + if (PyBytes_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 9 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[8])); + goto exit; + } i = PyBytes_AS_STRING(args[8])[0]; } - else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + else if (PyByteArray_Check(args[8])) { + if (PyByteArray_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 9 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[8])); + goto exit; + } i = PyByteArray_AS_STRING(args[8])[0]; } else { @@ -351,10 +477,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 10) { goto skip_optional; } - if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + if (PyBytes_Check(args[9])) { + if (PyBytes_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 10 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[9])); + goto exit; + } j = PyBytes_AS_STRING(args[9])[0]; } - else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + else if (PyByteArray_Check(args[9])) { + if (PyByteArray_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 10 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[9])); + goto exit; + } j = PyByteArray_AS_STRING(args[9])[0]; } else { @@ -364,10 +504,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 11) { goto skip_optional; } - if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + if (PyBytes_Check(args[10])) { + if (PyBytes_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 11 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[10])); + goto exit; + } k = PyBytes_AS_STRING(args[10])[0]; } - else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + else if (PyByteArray_Check(args[10])) { + if (PyByteArray_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 11 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[10])); + goto exit; + } k = PyByteArray_AS_STRING(args[10])[0]; } else { @@ -377,10 +531,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 12) { goto skip_optional; } - if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + if (PyBytes_Check(args[11])) { + if (PyBytes_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 12 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[11])); + goto exit; + } l = PyBytes_AS_STRING(args[11])[0]; } - else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + else if (PyByteArray_Check(args[11])) { + if (PyByteArray_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 12 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[11])); + goto exit; + } l = PyByteArray_AS_STRING(args[11])[0]; } else { @@ -390,10 +558,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 13) { goto skip_optional; } - if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + if (PyBytes_Check(args[12])) { + if (PyBytes_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 13 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[12])); + goto exit; + } m = PyBytes_AS_STRING(args[12])[0]; } - else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + else if (PyByteArray_Check(args[12])) { + if (PyByteArray_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 13 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[12])); + goto exit; + } m = PyByteArray_AS_STRING(args[12])[0]; } else { @@ -403,10 +585,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 14) { goto skip_optional; } - if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + if (PyBytes_Check(args[13])) { + if (PyBytes_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 14 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[13])); + goto exit; + } n = PyBytes_AS_STRING(args[13])[0]; } - else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + else if (PyByteArray_Check(args[13])) { + if (PyByteArray_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 14 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[13])); + goto exit; + } n = PyByteArray_AS_STRING(args[13])[0]; } else { @@ -648,7 +844,10 @@ int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[2]) != 1) { - _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + PyErr_Format(PyExc_TypeError, + "int_converter(): argument 3 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[2])); goto exit; } c = PyUnicode_READ_CHAR(args[2], 0); @@ -1259,7 +1458,7 @@ keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1315,7 +1514,7 @@ keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1371,7 +1570,7 @@ keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1440,7 +1639,7 @@ keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1520,7 +1719,7 @@ keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1588,7 +1787,7 @@ posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1644,7 +1843,7 @@ posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1701,7 +1900,7 @@ posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1760,7 +1959,7 @@ posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1831,7 +2030,7 @@ posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1907,7 +2106,7 @@ posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1978,7 +2177,7 @@ posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2054,7 +2253,7 @@ posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2129,7 +2328,7 @@ posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssiz PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2213,7 +2412,7 @@ posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2292,7 +2491,7 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2322,20 +2521,81 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } -PyDoc_STRVAR(posonly_vararg__doc__, -"posonly_vararg($module, a, /, b, *args)\n" +PyDoc_STRVAR(varpos__doc__, +"varpos($module, /, *args)\n" +"--\n" +"\n"); + +#define VARPOS_METHODDEF \ + {"varpos", _PyCFunction_CAST(varpos), METH_FASTCALL, varpos__doc__}, + +static PyObject * +varpos_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("varpos", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = args + 0; + return_value = varpos_impl(module, nvararg, __clinic_args); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_varpos__doc__, +"posonly_varpos($module, a, b, /, *args)\n" +"--\n" +"\n"); + +#define POSONLY_VARPOS_METHODDEF \ + {"posonly_varpos", _PyCFunction_CAST(posonly_varpos), METH_FASTCALL, posonly_varpos__doc__}, + +static PyObject * +posonly_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +posonly_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 2; + PyObject *a; + PyObject *b; + PyObject *const *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("posonly_varpos", nargs, 2, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + b = args[1]; + __clinic_args = args + 2; + return_value = posonly_varpos_impl(module, a, b, nvararg, __clinic_args); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_poskw_varpos__doc__, +"posonly_poskw_varpos($module, a, /, b, *args)\n" "--\n" "\n"); -#define POSONLY_VARARG_METHODDEF \ - {"posonly_vararg", _PyCFunction_CAST(posonly_vararg), METH_FASTCALL|METH_KEYWORDS, posonly_vararg__doc__}, +#define POSONLY_POSKW_VARPOS_METHODDEF \ + {"posonly_poskw_varpos", _PyCFunction_CAST(posonly_poskw_varpos), METH_FASTCALL|METH_KEYWORDS, posonly_poskw_varpos__doc__}, static PyObject * -posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *args); +posonly_poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args); static PyObject * -posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +posonly_poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2347,7 +2607,7 @@ posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2359,7 +2619,7 @@ posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje static const char * const _keywords[] = {"", "b", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "posonly_vararg", + .fname = "posonly_poskw_varpos", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -2375,74 +2635,96 @@ posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje a = args[0]; b = args[1]; __clinic_args = args[2]; - return_value = posonly_vararg_impl(module, a, b, __clinic_args); + return_value = posonly_poskw_varpos_impl(module, a, b, __clinic_args); exit: Py_XDECREF(__clinic_args); return return_value; } -PyDoc_STRVAR(vararg_and_posonly__doc__, -"vararg_and_posonly($module, a, /, *args)\n" +PyDoc_STRVAR(poskw_varpos__doc__, +"poskw_varpos($module, /, a, *args)\n" "--\n" "\n"); -#define VARARG_AND_POSONLY_METHODDEF \ - {"vararg_and_posonly", _PyCFunction_CAST(vararg_and_posonly), METH_FASTCALL, vararg_and_posonly__doc__}, +#define POSKW_VARPOS_METHODDEF \ + {"poskw_varpos", _PyCFunction_CAST(poskw_varpos), METH_FASTCALL|METH_KEYWORDS, poskw_varpos__doc__}, static PyObject * -vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args); +poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *args); static PyObject * -vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('a'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "poskw_varpos", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; PyObject *a; PyObject *__clinic_args = NULL; - if (!_PyArg_CheckPositional("vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { goto exit; } a = args[0]; - __clinic_args = PyTuple_New(nargs - 1); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 1; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); - } - return_value = vararg_and_posonly_impl(module, a, __clinic_args); + __clinic_args = args[1]; + return_value = poskw_varpos_impl(module, a, __clinic_args); exit: Py_XDECREF(__clinic_args); return return_value; } -PyDoc_STRVAR(vararg__doc__, -"vararg($module, /, a, *args)\n" +PyDoc_STRVAR(poskw_varpos_kwonly_opt__doc__, +"poskw_varpos_kwonly_opt($module, /, a, *args, b=False)\n" "--\n" "\n"); -#define VARARG_METHODDEF \ - {"vararg", _PyCFunction_CAST(vararg), METH_FASTCALL|METH_KEYWORDS, vararg__doc__}, +#define POSKW_VARPOS_KWONLY_OPT_METHODDEF \ + {"poskw_varpos_kwonly_opt", _PyCFunction_CAST(poskw_varpos_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, poskw_varpos_kwonly_opt__doc__}, static PyObject * -vararg_impl(PyObject *module, PyObject *a, PyObject *args); +poskw_varpos_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *args, + int b); static PyObject * -vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +poskw_varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2451,16 +2733,18 @@ vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwna # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"a", NULL}; + static const char * const _keywords[] = {"a", "b", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "vararg", + .fname = "poskw_varpos_kwonly_opt", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; + int b = 0; args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); if (!args) { @@ -2468,39 +2752,47 @@ vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwna } a = args[0]; __clinic_args = args[1]; - return_value = vararg_impl(module, a, __clinic_args); + if (!noptargs) { + goto skip_optional_kwonly; + } + b = PyObject_IsTrue(args[2]); + if (b < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = poskw_varpos_kwonly_opt_impl(module, a, __clinic_args, b); exit: Py_XDECREF(__clinic_args); return return_value; } -PyDoc_STRVAR(vararg_with_default__doc__, -"vararg_with_default($module, /, a, *args, b=False)\n" +PyDoc_STRVAR(poskw_varpos_kwonly_opt2__doc__, +"poskw_varpos_kwonly_opt2($module, /, a, *args, b=False, c=False)\n" "--\n" "\n"); -#define VARARG_WITH_DEFAULT_METHODDEF \ - {"vararg_with_default", _PyCFunction_CAST(vararg_with_default), METH_FASTCALL|METH_KEYWORDS, vararg_with_default__doc__}, +#define POSKW_VARPOS_KWONLY_OPT2_METHODDEF \ + {"poskw_varpos_kwonly_opt2", _PyCFunction_CAST(poskw_varpos_kwonly_opt2), METH_FASTCALL|METH_KEYWORDS, poskw_varpos_kwonly_opt2__doc__}, static PyObject * -vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, - int b); +poskw_varpos_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *args, + PyObject *b, PyObject *c); static PyObject * -vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +poskw_varpos_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2509,18 +2801,19 @@ vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"a", "b", NULL}; + static const char * const _keywords[] = {"a", "b", "c", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "vararg_with_default", + .fname = "poskw_varpos_kwonly_opt2", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; - int b = 0; + PyObject *b = Py_False; + PyObject *c = Py_False; args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); if (!args) { @@ -2531,31 +2824,34 @@ vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_kwonly; } - b = PyObject_IsTrue(args[2]); - if (b < 0) { - goto exit; + if (args[2]) { + b = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } } + c = args[3]; skip_optional_kwonly: - return_value = vararg_with_default_impl(module, a, __clinic_args, b); + return_value = poskw_varpos_kwonly_opt2_impl(module, a, __clinic_args, b, c); exit: Py_XDECREF(__clinic_args); return return_value; } -PyDoc_STRVAR(vararg_with_only_defaults__doc__, -"vararg_with_only_defaults($module, /, *args, b=None)\n" +PyDoc_STRVAR(varpos_kwonly_opt__doc__, +"varpos_kwonly_opt($module, /, *args, b=False)\n" "--\n" "\n"); -#define VARARG_WITH_ONLY_DEFAULTS_METHODDEF \ - {"vararg_with_only_defaults", _PyCFunction_CAST(vararg_with_only_defaults), METH_FASTCALL|METH_KEYWORDS, vararg_with_only_defaults__doc__}, +#define VARPOS_KWONLY_OPT_METHODDEF \ + {"varpos_kwonly_opt", _PyCFunction_CAST(varpos_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, varpos_kwonly_opt__doc__}, static PyObject * -vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b); +varpos_kwonly_opt_impl(PyObject *module, PyObject *args, PyObject *b); static PyObject * -vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2567,7 +2863,7 @@ vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2579,14 +2875,14 @@ vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t na static const char * const _keywords[] = {"b", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "vararg_with_only_defaults", + .fname = "varpos_kwonly_opt", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *__clinic_args = NULL; - PyObject *b = Py_None; + PyObject *b = Py_False; args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); if (!args) { @@ -2598,7 +2894,79 @@ vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t na } b = args[1]; skip_optional_kwonly: - return_value = vararg_with_only_defaults_impl(module, __clinic_args, b); + return_value = varpos_kwonly_opt_impl(module, __clinic_args, b); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(varpos_kwonly_req_opt__doc__, +"varpos_kwonly_req_opt($module, /, *args, a, b=False, c=False)\n" +"--\n" +"\n"); + +#define VARPOS_KWONLY_REQ_OPT_METHODDEF \ + {"varpos_kwonly_req_opt", _PyCFunction_CAST(varpos_kwonly_req_opt), METH_FASTCALL|METH_KEYWORDS, varpos_kwonly_req_opt__doc__}, + +static PyObject * +varpos_kwonly_req_opt_impl(PyObject *module, PyObject *args, PyObject *a, + PyObject *b, PyObject *c); + +static PyObject * +varpos_kwonly_req_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "varpos_kwonly_req_opt", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *__clinic_args = NULL; + PyObject *a; + PyObject *b = Py_False; + PyObject *c = Py_False; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + a = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + b = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + c = args[3]; +skip_optional_kwonly: + return_value = varpos_kwonly_req_opt_impl(module, __clinic_args, a, b, c); exit: Py_XDECREF(__clinic_args); @@ -2756,28 +3124,23 @@ PyDoc_STRVAR(gh_99233_refcount__doc__, {"gh_99233_refcount", _PyCFunction_CAST(gh_99233_refcount), METH_FASTCALL, gh_99233_refcount__doc__}, static PyObject * -gh_99233_refcount_impl(PyObject *module, PyObject *args); +gh_99233_refcount_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args); static PyObject * gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("gh_99233_refcount", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = gh_99233_refcount_impl(module, __clinic_args); + __clinic_args = args + 0; + return_value = gh_99233_refcount_impl(module, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -3142,25 +3505,175 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } -PyDoc_STRVAR(_testclinic_TestClass_meth_method_no_params__doc__, -"meth_method_no_params($self, /)\n" +PyDoc_STRVAR(_testclinic_TestClass_get_defining_class__doc__, +"get_defining_class($self, /)\n" "--\n" "\n"); -#define _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF \ - {"meth_method_no_params", _PyCFunction_CAST(_testclinic_TestClass_meth_method_no_params), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_meth_method_no_params__doc__}, +#define _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_METHODDEF \ + {"get_defining_class", _PyCFunction_CAST(_testclinic_TestClass_get_defining_class), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_get_defining_class__doc__}, static PyObject * -_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, - PyTypeObject *cls); +_testclinic_TestClass_get_defining_class_impl(PyObject *self, + PyTypeObject *cls); static PyObject * -_testclinic_TestClass_meth_method_no_params(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testclinic_TestClass_get_defining_class(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "meth_method_no_params() takes no arguments"); + PyErr_SetString(PyExc_TypeError, "get_defining_class() takes no arguments"); return NULL; } - return _testclinic_TestClass_meth_method_no_params_impl(self, cls); + return _testclinic_TestClass_get_defining_class_impl(self, cls); +} + +PyDoc_STRVAR(_testclinic_TestClass_get_defining_class_arg__doc__, +"get_defining_class_arg($self, /, arg)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_ARG_METHODDEF \ + {"get_defining_class_arg", _PyCFunction_CAST(_testclinic_TestClass_get_defining_class_arg), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_get_defining_class_arg__doc__}, + +static PyObject * +_testclinic_TestClass_get_defining_class_arg_impl(PyObject *self, + PyTypeObject *cls, + PyObject *arg); + +static PyObject * +_testclinic_TestClass_get_defining_class_arg(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_defining_class_arg", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *arg; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + arg = args[0]; + return_value = _testclinic_TestClass_get_defining_class_arg_impl(self, cls, arg); + +exit: + return return_value; +} + +PyDoc_STRVAR(_testclinic_TestClass_defclass_varpos__doc__, +"defclass_varpos($self, /, *args)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_DEFCLASS_VARPOS_METHODDEF \ + {"defclass_varpos", _PyCFunction_CAST(_testclinic_TestClass_defclass_varpos), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_defclass_varpos__doc__}, + +static PyObject * +_testclinic_TestClass_defclass_varpos_impl(PyObject *self, PyTypeObject *cls, + PyObject *args); + +static PyObject * +_testclinic_TestClass_defclass_varpos(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "defclass_varpos", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + return_value = _testclinic_TestClass_defclass_varpos_impl(self, cls, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(_testclinic_TestClass_defclass_posonly_varpos__doc__, +"defclass_posonly_varpos($self, a, b, /, *args)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_DEFCLASS_POSONLY_VARPOS_METHODDEF \ + {"defclass_posonly_varpos", _PyCFunction_CAST(_testclinic_TestClass_defclass_posonly_varpos), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_defclass_posonly_varpos__doc__}, + +static PyObject * +_testclinic_TestClass_defclass_posonly_varpos_impl(PyObject *self, + PyTypeObject *cls, + PyObject *a, PyObject *b, + PyObject *args); + +static PyObject * +_testclinic_TestClass_defclass_posonly_varpos(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "defclass_posonly_varpos", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + __clinic_args = args[2]; + return_value = _testclinic_TestClass_defclass_posonly_varpos_impl(self, cls, a, b, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; } -/*[clinic end generated code: output=6520c1ca5392a3f0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7662d07e7d29cbeb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 732c3810408399..95a2cc4cb5ed6d 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -48,7 +48,7 @@ depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -133,7 +133,7 @@ depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -215,7 +215,7 @@ depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -300,7 +300,7 @@ depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -372,7 +372,7 @@ depr_star_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -451,7 +451,7 @@ depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -534,7 +534,7 @@ depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -608,7 +608,7 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -635,7 +635,7 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) &a, &b, &c, &d, &d_length)) { goto exit; } - if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, &_Py_ID(c))))) { + if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, _Py_LATIN1_CHR('c'))))) { if (PyErr_Occurred()) { // PyDict_Contains() above can fail goto exit; } @@ -692,7 +692,7 @@ depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -770,7 +770,7 @@ depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -851,7 +851,7 @@ depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -935,7 +935,7 @@ depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1020,7 +1020,7 @@ depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1101,7 +1101,7 @@ depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1186,7 +1186,7 @@ depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1269,7 +1269,7 @@ depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1354,7 +1354,7 @@ depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1441,7 +1441,7 @@ depr_star_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1525,7 +1525,7 @@ depr_star_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), _Py_LATIN1_CHR('h'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1618,7 +1618,7 @@ depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1699,7 +1699,7 @@ depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1780,7 +1780,7 @@ depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1866,7 +1866,7 @@ depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1959,7 +1959,7 @@ depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2057,7 +2057,7 @@ depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2145,7 +2145,7 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2171,7 +2171,7 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO &a, &b, &c, &d, &d_length)) { goto exit; } - if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, &_Py_ID(c))))) { + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, _Py_LATIN1_CHR('c'))))) { if (PyErr_Occurred()) { // PySequence_Contains() above can fail goto exit; } @@ -2232,7 +2232,7 @@ depr_kwd_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), _Py_LATIN1_CHR('h'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2333,7 +2333,7 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2393,4 +2393,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=2c19d1804ba6e53b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ca6da2c7137554be input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index eff72cee24975c..94897f4c6dc427 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -91,4 +91,119 @@ my_int_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5cf64baf978d2288 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(my_float_sum__doc__, +"my_float_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_FLOAT_SUM_METHODDEF \ + {"my_float_sum", (PyCFunction)(void(*)(void))my_float_sum, METH_FASTCALL, my_float_sum__doc__}, + +static float +my_float_sum_impl(PyObject *module, float x, float y); + +static PyObject * +my_float_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + float x; + float y; + float _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_float_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = (float) PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = (float) PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_float_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble((double)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(my_double_sum__doc__, +"my_double_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_DOUBLE_SUM_METHODDEF \ + {"my_double_sum", (PyCFunction)(void(*)(void))my_double_sum, METH_FASTCALL, my_double_sum__doc__}, + +static double +my_double_sum_impl(PyObject *module, double x, double y); + +static PyObject * +my_double_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_double_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_double_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(get_file_descriptor__doc__, +"get_file_descriptor($module, file, /)\n" +"--\n" +"\n" +"Get a file descriptor."); + +#define GET_FILE_DESCRIPTOR_METHODDEF \ + {"get_file_descriptor", (PyCFunction)get_file_descriptor, METH_O, get_file_descriptor__doc__}, + +static int +get_file_descriptor_impl(PyObject *module, int fd); + +static PyObject * +get_file_descriptor(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + int _return_value; + + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { + goto exit; + } + _return_value = get_file_descriptor_impl(module, fd); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=03fd7811c056dc74 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index cba2a943d03456..dcca9ecf735723 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -67,6 +67,24 @@ _testinternalcapi_compiler_cleandoc(PyObject *module, PyObject *const *args, Py_ return return_value; } +PyDoc_STRVAR(_testinternalcapi_new_instruction_sequence__doc__, +"new_instruction_sequence($module, /)\n" +"--\n" +"\n" +"Return a new, empty InstructionSequence."); + +#define _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF \ + {"new_instruction_sequence", (PyCFunction)_testinternalcapi_new_instruction_sequence, METH_NOARGS, _testinternalcapi_new_instruction_sequence__doc__}, + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module); + +static PyObject * +_testinternalcapi_new_instruction_sequence(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testinternalcapi_new_instruction_sequence_impl(module); +} + PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, "compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" @@ -282,4 +300,64 @@ _testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignore { return _testinternalcapi_test_long_numbits_impl(module); } -/*[clinic end generated code: output=679bf53bbae20085 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(gh_119213_getargs__doc__, +"gh_119213_getargs($module, /, spam=None)\n" +"--\n" +"\n" +"Test _PyArg_Parser.kwtuple"); + +#define GH_119213_GETARGS_METHODDEF \ + {"gh_119213_getargs", _PyCFunction_CAST(gh_119213_getargs), METH_FASTCALL|METH_KEYWORDS, gh_119213_getargs__doc__}, + +static PyObject * +gh_119213_getargs_impl(PyObject *module, PyObject *spam); + +static PyObject * +gh_119213_getargs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(spam), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"spam", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "gh_119213_getargs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *spam = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + spam = args[0]; +skip_optional_pos: + return_value = gh_119213_getargs_impl(module, spam); + +exit: + return return_value; +} +/*[clinic end generated code: output=4d0770a1c20fbf40 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index 7ac361ece1acc3..452897b3fae99e 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -88,7 +88,7 @@ _testmultiphase_StateAccessType_increment_count_clinic(StateAccessTypeObject *se PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(n), &_Py_ID(twice), }, + .ob_item = { _Py_LATIN1_CHR('n'), &_Py_ID(twice), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -162,4 +162,4 @@ _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObj } return _testmultiphase_StateAccessType_get_count_impl(self, cls); } -/*[clinic end generated code: output=2c199bad52e9cda7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=59cb50dae2d11dc1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 188bcc773cfc41..2b1ac954b4d570 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -622,6 +622,33 @@ _tkinter_tkapp_loadtk(TkappObject *self, PyObject *Py_UNUSED(ignored)) return _tkinter_tkapp_loadtk_impl(self); } +PyDoc_STRVAR(_tkinter_tkapp_settrace__doc__, +"settrace($self, func, /)\n" +"--\n" +"\n" +"Set the tracing function."); + +#define _TKINTER_TKAPP_SETTRACE_METHODDEF \ + {"settrace", (PyCFunction)_tkinter_tkapp_settrace, METH_O, _tkinter_tkapp_settrace__doc__}, + +PyDoc_STRVAR(_tkinter_tkapp_gettrace__doc__, +"gettrace($self, /)\n" +"--\n" +"\n" +"Get the tracing function."); + +#define _TKINTER_TKAPP_GETTRACE_METHODDEF \ + {"gettrace", (PyCFunction)_tkinter_tkapp_gettrace, METH_NOARGS, _tkinter_tkapp_gettrace__doc__}, + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self); + +static PyObject * +_tkinter_tkapp_gettrace(TkappObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _tkinter_tkapp_gettrace_impl(self); +} + PyDoc_STRVAR(_tkinter_tkapp_willdispatch__doc__, "willdispatch($self, /)\n" "--\n" @@ -649,7 +676,7 @@ PyDoc_STRVAR(_tkinter__flatten__doc__, PyDoc_STRVAR(_tkinter_create__doc__, "create($module, screenName=None, baseName=\'\', className=\'Tk\',\n" -" interactive=False, wantobjects=False, wantTk=True, sync=False,\n" +" interactive=False, wantobjects=0, wantTk=True, sync=False,\n" " use=None, /)\n" "--\n" "\n" @@ -750,8 +777,8 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - wantobjects = PyObject_IsTrue(args[4]); - if (wantobjects < 0) { + wantobjects = PyLong_AsInt(args[4]); + if (wantobjects == -1 && PyErr_Occurred()) { goto exit; } if (nargs < 6) { @@ -861,4 +888,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=d447501ec5aa9447 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d90c1a9850c63249 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_weakref.c.h b/Modules/clinic/_weakref.c.h index 550b6c4d71a015..8d7bc5dc936610 100644 --- a/Modules/clinic/_weakref.c.h +++ b/Modules/clinic/_weakref.c.h @@ -2,7 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_weakref_getweakrefcount__doc__, @@ -23,9 +22,7 @@ _weakref_getweakrefcount(PyObject *module, PyObject *object) PyObject *return_value = NULL; Py_ssize_t _return_value; - Py_BEGIN_CRITICAL_SECTION(object); _return_value = _weakref_getweakrefcount_impl(module, object); - Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -79,21 +76,6 @@ PyDoc_STRVAR(_weakref_getweakrefs__doc__, #define _WEAKREF_GETWEAKREFS_METHODDEF \ {"getweakrefs", (PyCFunction)_weakref_getweakrefs, METH_O, _weakref_getweakrefs__doc__}, -static PyObject * -_weakref_getweakrefs_impl(PyObject *module, PyObject *object); - -static PyObject * -_weakref_getweakrefs(PyObject *module, PyObject *object) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(object); - return_value = _weakref_getweakrefs_impl(module, object); - Py_END_CRITICAL_SECTION(); - - return return_value; -} - PyDoc_STRVAR(_weakref_proxy__doc__, "proxy($module, object, callback=None, /)\n" "--\n" @@ -130,4 +112,4 @@ _weakref_proxy(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=d5d30707212a9870 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=60f59adc1dc9eab8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index d1052f38919dde..b0c54fc809f4c1 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -151,6 +151,76 @@ _winapi_ConnectNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } +PyDoc_STRVAR(_winapi_CreateEventW__doc__, +"CreateEventW($module, /, security_attributes, manual_reset,\n" +" initial_state, name)\n" +"--\n" +"\n"); + +#define _WINAPI_CREATEEVENTW_METHODDEF \ + {"CreateEventW", _PyCFunction_CAST(_winapi_CreateEventW), METH_FASTCALL|METH_KEYWORDS, _winapi_CreateEventW__doc__}, + +static HANDLE +_winapi_CreateEventW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL manual_reset, BOOL initial_state, + LPCWSTR name); + +static PyObject * +_winapi_CreateEventW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(security_attributes), &_Py_ID(manual_reset), &_Py_ID(initial_state), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"security_attributes", "manual_reset", "initial_state", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_POINTER "iiO&:CreateEventW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + LPSECURITY_ATTRIBUTES security_attributes; + BOOL manual_reset; + BOOL initial_state; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &security_attributes, &manual_reset, &initial_state, _PyUnicode_WideCharString_Opt_Converter, &name)) { + goto exit; + } + _return_value = _winapi_CreateEventW_impl(module, security_attributes, manual_reset, initial_state, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + PyDoc_STRVAR(_winapi_CreateFile__doc__, "CreateFile($module, file_name, desired_access, share_mode,\n" " security_attributes, creation_disposition,\n" @@ -297,6 +367,73 @@ _winapi_CreateJunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_winapi_CreateMutexW__doc__, +"CreateMutexW($module, /, security_attributes, initial_owner, name)\n" +"--\n" +"\n"); + +#define _WINAPI_CREATEMUTEXW_METHODDEF \ + {"CreateMutexW", _PyCFunction_CAST(_winapi_CreateMutexW), METH_FASTCALL|METH_KEYWORDS, _winapi_CreateMutexW__doc__}, + +static HANDLE +_winapi_CreateMutexW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL initial_owner, LPCWSTR name); + +static PyObject * +_winapi_CreateMutexW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(security_attributes), &_Py_ID(initial_owner), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"security_attributes", "initial_owner", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_POINTER "iO&:CreateMutexW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + LPSECURITY_ATTRIBUTES security_attributes; + BOOL initial_owner; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &security_attributes, &initial_owner, _PyUnicode_WideCharString_Opt_Converter, &name)) { + goto exit; + } + _return_value = _winapi_CreateMutexW_impl(module, security_attributes, initial_owner, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + PyDoc_STRVAR(_winapi_CreateNamedPipe__doc__, "CreateNamedPipe($module, name, open_mode, pipe_mode, max_instances,\n" " out_buffer_size, in_buffer_size, default_timeout,\n" @@ -308,7 +445,7 @@ PyDoc_STRVAR(_winapi_CreateNamedPipe__doc__, {"CreateNamedPipe", _PyCFunction_CAST(_winapi_CreateNamedPipe), METH_FASTCALL, _winapi_CreateNamedPipe__doc__}, static HANDLE -_winapi_CreateNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD open_mode, +_winapi_CreateNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD open_mode, DWORD pipe_mode, DWORD max_instances, DWORD out_buffer_size, DWORD in_buffer_size, DWORD default_timeout, @@ -318,7 +455,7 @@ static PyObject * _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR name; + LPCWSTR name = NULL; DWORD open_mode; DWORD pipe_mode; DWORD max_instances; @@ -328,8 +465,8 @@ _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t narg LPSECURITY_ATTRIBUTES security_attributes; HANDLE _return_value; - if (!_PyArg_ParseStack(args, nargs, "skkkkkk" F_POINTER ":CreateNamedPipe", - &name, &open_mode, &pipe_mode, &max_instances, &out_buffer_size, &in_buffer_size, &default_timeout, &security_attributes)) { + if (!_PyArg_ParseStack(args, nargs, "O&kkkkkk" F_POINTER ":CreateNamedPipe", + _PyUnicode_WideCharString_Converter, &name, &open_mode, &pipe_mode, &max_instances, &out_buffer_size, &in_buffer_size, &default_timeout, &security_attributes)) { goto exit; } _return_value = _winapi_CreateNamedPipe_impl(module, name, open_mode, pipe_mode, max_instances, out_buffer_size, in_buffer_size, default_timeout, security_attributes); @@ -342,6 +479,9 @@ _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t narg return_value = HANDLE_TO_PYNUM(_return_value); exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + return return_value; } @@ -604,6 +744,76 @@ _winapi_GetLastError(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(_winapi_GetLongPathName__doc__, +"GetLongPathName($module, /, path)\n" +"--\n" +"\n" +"Return the long version of the provided path.\n" +"\n" +"If the path is already in its long form, returns the same value.\n" +"\n" +"The path must already be a \'str\'. If the type is not known, use\n" +"os.fsdecode before calling this function."); + +#define _WINAPI_GETLONGPATHNAME_METHODDEF \ + {"GetLongPathName", _PyCFunction_CAST(_winapi_GetLongPathName), METH_FASTCALL|METH_KEYWORDS, _winapi_GetLongPathName__doc__}, + +static PyObject * +_winapi_GetLongPathName_impl(PyObject *module, LPCWSTR path); + +static PyObject * +_winapi_GetLongPathName(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "GetLongPathName", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + LPCWSTR path = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("GetLongPathName", "argument 'path'", "str", args[0]); + goto exit; + } + path = PyUnicode_AsWideCharString(args[0], NULL); + if (path == NULL) { + goto exit; + } + return_value = _winapi_GetLongPathName_impl(module, path); + +exit: + /* Cleanup for path */ + PyMem_Free((void *)path); + + return return_value; +} + PyDoc_STRVAR(_winapi_GetModuleFileName__doc__, "GetModuleFileName($module, module_handle, /)\n" "--\n" @@ -638,6 +848,76 @@ _winapi_GetModuleFileName(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_winapi_GetShortPathName__doc__, +"GetShortPathName($module, /, path)\n" +"--\n" +"\n" +"Return the short version of the provided path.\n" +"\n" +"If the path is already in its short form, returns the same value.\n" +"\n" +"The path must already be a \'str\'. If the type is not known, use\n" +"os.fsdecode before calling this function."); + +#define _WINAPI_GETSHORTPATHNAME_METHODDEF \ + {"GetShortPathName", _PyCFunction_CAST(_winapi_GetShortPathName), METH_FASTCALL|METH_KEYWORDS, _winapi_GetShortPathName__doc__}, + +static PyObject * +_winapi_GetShortPathName_impl(PyObject *module, LPCWSTR path); + +static PyObject * +_winapi_GetShortPathName(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "GetShortPathName", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + LPCWSTR path = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("GetShortPathName", "argument 'path'", "str", args[0]); + goto exit; + } + path = PyUnicode_AsWideCharString(args[0], NULL); + if (path == NULL) { + goto exit; + } + return_value = _winapi_GetShortPathName_impl(module, path); + +exit: + /* Cleanup for path */ + PyMem_Free((void *)path); + + return return_value; +} + PyDoc_STRVAR(_winapi_GetStdHandle__doc__, "GetStdHandle($module, std_handle, /)\n" "--\n" @@ -771,6 +1051,138 @@ _winapi_UnmapViewOfFile(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_winapi_OpenEventW__doc__, +"OpenEventW($module, /, desired_access, inherit_handle, name)\n" +"--\n" +"\n"); + +#define _WINAPI_OPENEVENTW_METHODDEF \ + {"OpenEventW", _PyCFunction_CAST(_winapi_OpenEventW), METH_FASTCALL|METH_KEYWORDS, _winapi_OpenEventW__doc__}, + +static HANDLE +_winapi_OpenEventW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name); + +static PyObject * +_winapi_OpenEventW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(desired_access), &_Py_ID(inherit_handle), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"desired_access", "inherit_handle", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "kiO&:OpenEventW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + DWORD desired_access; + BOOL inherit_handle; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) { + goto exit; + } + _return_value = _winapi_OpenEventW_impl(module, desired_access, inherit_handle, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + +PyDoc_STRVAR(_winapi_OpenMutexW__doc__, +"OpenMutexW($module, /, desired_access, inherit_handle, name)\n" +"--\n" +"\n"); + +#define _WINAPI_OPENMUTEXW_METHODDEF \ + {"OpenMutexW", _PyCFunction_CAST(_winapi_OpenMutexW), METH_FASTCALL|METH_KEYWORDS, _winapi_OpenMutexW__doc__}, + +static HANDLE +_winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name); + +static PyObject * +_winapi_OpenMutexW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(desired_access), &_Py_ID(inherit_handle), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"desired_access", "inherit_handle", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "kiO&:OpenMutexW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + DWORD desired_access; + BOOL inherit_handle; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) { + goto exit; + } + _return_value = _winapi_OpenMutexW_impl(module, desired_access, inherit_handle, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + PyDoc_STRVAR(_winapi_OpenFileMapping__doc__, "OpenFileMapping($module, desired_access, inherit_handle, name, /)\n" "--\n" @@ -991,6 +1403,162 @@ _winapi_ReadFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(_winapi_ReleaseMutex__doc__, +"ReleaseMutex($module, /, mutex)\n" +"--\n" +"\n"); + +#define _WINAPI_RELEASEMUTEX_METHODDEF \ + {"ReleaseMutex", _PyCFunction_CAST(_winapi_ReleaseMutex), METH_FASTCALL|METH_KEYWORDS, _winapi_ReleaseMutex__doc__}, + +static PyObject * +_winapi_ReleaseMutex_impl(PyObject *module, HANDLE mutex); + +static PyObject * +_winapi_ReleaseMutex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(mutex), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"mutex", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_HANDLE ":ReleaseMutex", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + HANDLE mutex; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &mutex)) { + goto exit; + } + return_value = _winapi_ReleaseMutex_impl(module, mutex); + +exit: + return return_value; +} + +PyDoc_STRVAR(_winapi_ResetEvent__doc__, +"ResetEvent($module, /, event)\n" +"--\n" +"\n"); + +#define _WINAPI_RESETEVENT_METHODDEF \ + {"ResetEvent", _PyCFunction_CAST(_winapi_ResetEvent), METH_FASTCALL|METH_KEYWORDS, _winapi_ResetEvent__doc__}, + +static PyObject * +_winapi_ResetEvent_impl(PyObject *module, HANDLE event); + +static PyObject * +_winapi_ResetEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(event), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"event", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_HANDLE ":ResetEvent", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + HANDLE event; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &event)) { + goto exit; + } + return_value = _winapi_ResetEvent_impl(module, event); + +exit: + return return_value; +} + +PyDoc_STRVAR(_winapi_SetEvent__doc__, +"SetEvent($module, /, event)\n" +"--\n" +"\n"); + +#define _WINAPI_SETEVENT_METHODDEF \ + {"SetEvent", _PyCFunction_CAST(_winapi_SetEvent), METH_FASTCALL|METH_KEYWORDS, _winapi_SetEvent__doc__}, + +static PyObject * +_winapi_SetEvent_impl(PyObject *module, HANDLE event); + +static PyObject * +_winapi_SetEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(event), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"event", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_HANDLE ":SetEvent", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + HANDLE event; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &event)) { + goto exit; + } + return_value = _winapi_SetEvent_impl(module, event); + +exit: + return return_value; +} + PyDoc_STRVAR(_winapi_SetNamedPipeHandleState__doc__, "SetNamedPipeHandleState($module, named_pipe, mode,\n" " max_collection_count, collect_data_timeout, /)\n" @@ -1095,21 +1663,95 @@ PyDoc_STRVAR(_winapi_WaitNamedPipe__doc__, {"WaitNamedPipe", _PyCFunction_CAST(_winapi_WaitNamedPipe), METH_FASTCALL, _winapi_WaitNamedPipe__doc__}, static PyObject * -_winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout); +_winapi_WaitNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD timeout); static PyObject * _winapi_WaitNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR name; + LPCWSTR name = NULL; DWORD timeout; - if (!_PyArg_ParseStack(args, nargs, "sk:WaitNamedPipe", - &name, &timeout)) { + if (!_PyArg_ParseStack(args, nargs, "O&k:WaitNamedPipe", + _PyUnicode_WideCharString_Converter, &name, &timeout)) { goto exit; } return_value = _winapi_WaitNamedPipe_impl(module, name, timeout); +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + +PyDoc_STRVAR(_winapi_BatchedWaitForMultipleObjects__doc__, +"BatchedWaitForMultipleObjects($module, /, handle_seq, wait_all,\n" +" milliseconds=_winapi.INFINITE)\n" +"--\n" +"\n" +"Supports a larger number of handles than WaitForMultipleObjects\n" +"\n" +"Note that the handles may be waited on other threads, which could cause\n" +"issues for objects like mutexes that become associated with the thread\n" +"that was waiting for them. Objects may also be left signalled, even if\n" +"the wait fails.\n" +"\n" +"It is recommended to use WaitForMultipleObjects whenever possible, and\n" +"only switch to BatchedWaitForMultipleObjects for scenarios where you\n" +"control all the handles involved, such as your own thread pool or\n" +"files, and all wait objects are left unmodified by a wait (for example,\n" +"manual reset events, threads, and files/pipes).\n" +"\n" +"Overlapped handles returned from this module use manual reset events."); + +#define _WINAPI_BATCHEDWAITFORMULTIPLEOBJECTS_METHODDEF \ + {"BatchedWaitForMultipleObjects", _PyCFunction_CAST(_winapi_BatchedWaitForMultipleObjects), METH_FASTCALL|METH_KEYWORDS, _winapi_BatchedWaitForMultipleObjects__doc__}, + +static PyObject * +_winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, + PyObject *handle_seq, + BOOL wait_all, DWORD milliseconds); + +static PyObject * +_winapi_BatchedWaitForMultipleObjects(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(handle_seq), &_Py_ID(wait_all), &_Py_ID(milliseconds), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"handle_seq", "wait_all", "milliseconds", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "Oi|k:BatchedWaitForMultipleObjects", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *handle_seq; + BOOL wait_all; + DWORD milliseconds = INFINITE; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &handle_seq, &wait_all, &milliseconds)) { + goto exit; + } + return_value = _winapi_BatchedWaitForMultipleObjects_impl(module, handle_seq, wait_all, milliseconds); + exit: return return_value; } @@ -1482,4 +2124,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=2350d4f2275d3a6f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2304c62187a90140 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index 9905b6425e2f79..bde88b5c4fa65b 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -6,8 +6,65 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(zoneinfo_ZoneInfo__doc__, +"ZoneInfo(key)\n" +"--\n" +"\n" +"Create a new ZoneInfo instance."); + +static PyObject * +zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key); + +static PyObject * +zoneinfo_ZoneInfo(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(key), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"key", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "ZoneInfo", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *key; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + key = fastargs[0]; + Py_BEGIN_CRITICAL_SECTION(type); + return_value = zoneinfo_ZoneInfo_impl(type, key); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + PyDoc_STRVAR(zoneinfo_ZoneInfo_from_file__doc__, "from_file($type, file_obj, /, key=None)\n" "--\n" @@ -182,7 +239,9 @@ zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *c } only_keys = args[0]; skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(type); return_value = zoneinfo_ZoneInfo_clear_cache_impl(type, cls, only_keys); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -372,4 +431,4 @@ zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *con exit: return return_value; } -/*[clinic end generated code: output=2a15f32fdd2ab6cd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b4fdc0b30247110a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 0b764e43e19437..2ed7eaa6abf7af 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -2,9 +2,30 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(array_array_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from the array."); + +#define ARRAY_ARRAY_CLEAR_METHODDEF \ + {"clear", (PyCFunction)array_array_clear, METH_NOARGS, array_array_clear__doc__}, + +static PyObject * +array_array_clear_impl(arrayobject *self); + +static PyObject * +array_array_clear(arrayobject *self, PyObject *Py_UNUSED(ignored)) +{ + return array_array_clear_impl(self); +} + PyDoc_STRVAR(array_array___copy____doc__, "__copy__($self, /)\n" "--\n" @@ -578,7 +599,10 @@ array__array_reconstructor(PyObject *module, PyObject *const *args, Py_ssize_t n goto exit; } if (PyUnicode_GET_LENGTH(args[1]) != 1) { - _PyArg_BadArgument("_array_reconstructor", "argument 2", "a unicode character", args[1]); + PyErr_Format(PyExc_TypeError, + "_array_reconstructor(): argument 2 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[1])); goto exit; } typecode = PyUnicode_READ_CHAR(args[1], 0); @@ -667,4 +691,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=3be987238a4bb431 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ecd63acd7924c223 input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/clinic/blake2module.c.h similarity index 54% rename from Modules/_blake2/clinic/blake2b_impl.c.h rename to Modules/clinic/blake2module.c.h index 47d62717eb76e7..50478bcbecf8e3 100644 --- a/Modules/_blake2/clinic/blake2b_impl.c.h +++ b/Modules/clinic/blake2module.c.h @@ -59,7 +59,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *data = NULL; - int digest_size = BLAKE2B_OUTBYTES; + int digest_size = HACL_HASH_BLAKE2B_OUT_BYTES; Py_buffer key = {NULL, NULL}; Py_buffer salt = {NULL, NULL}; Py_buffer person = {NULL, NULL}; @@ -203,6 +203,200 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(py_blake2s_new__doc__, +"blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" +" key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" +" node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" +" usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new BLAKE2s hash object."); + +static PyObject * +py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity); + +static PyObject * +py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 12 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "blake2s", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[13]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *data = NULL; + int digest_size = HACL_HASH_BLAKE2S_OUT_BYTES; + Py_buffer key = {NULL, NULL}; + Py_buffer salt = {NULL, NULL}; + Py_buffer person = {NULL, NULL}; + int fanout = 1; + int depth = 1; + unsigned long leaf_size = 0; + unsigned long long node_offset = 0; + int node_depth = 0; + int inner_size = 0; + int last_node = 0; + int usedforsecurity = 1; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (nargs < 1) { + goto skip_optional_posonly; + } + noptargs--; + data = fastargs[0]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (fastargs[1]) { + digest_size = PyLong_AsInt(fastargs[1]); + if (digest_size == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[2]) { + if (PyObject_GetBuffer(fastargs[2], &key, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + if (PyObject_GetBuffer(fastargs[3], &salt, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + if (PyObject_GetBuffer(fastargs[4], &person, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[5]) { + fanout = PyLong_AsInt(fastargs[5]); + if (fanout == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[6]) { + depth = PyLong_AsInt(fastargs[6]); + if (depth == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[7]) { + if (!_PyLong_UnsignedLong_Converter(fastargs[7], &leaf_size)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[8]) { + if (!_PyLong_UnsignedLongLong_Converter(fastargs[8], &node_offset)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[9]) { + node_depth = PyLong_AsInt(fastargs[9]); + if (node_depth == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[10]) { + inner_size = PyLong_AsInt(fastargs[10]); + if (inner_size == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[11]) { + last_node = PyObject_IsTrue(fastargs[11]); + if (last_node < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + usedforsecurity = PyObject_IsTrue(fastargs[12]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + +exit: + /* Cleanup for key */ + if (key.obj) { + PyBuffer_Release(&key); + } + /* Cleanup for salt */ + if (salt.obj) { + PyBuffer_Release(&salt); + } + /* Cleanup for person */ + if (person.obj) { + PyBuffer_Release(&person); + } + + return return_value; +} + PyDoc_STRVAR(_blake2_blake2b_copy__doc__, "copy($self, /)\n" "--\n" @@ -213,10 +407,10 @@ PyDoc_STRVAR(_blake2_blake2b_copy__doc__, {"copy", (PyCFunction)_blake2_blake2b_copy, METH_NOARGS, _blake2_blake2b_copy__doc__}, static PyObject * -_blake2_blake2b_copy_impl(BLAKE2bObject *self); +_blake2_blake2b_copy_impl(Blake2Object *self); static PyObject * -_blake2_blake2b_copy(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_copy(Blake2Object *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_copy_impl(self); } @@ -240,10 +434,10 @@ PyDoc_STRVAR(_blake2_blake2b_digest__doc__, {"digest", (PyCFunction)_blake2_blake2b_digest, METH_NOARGS, _blake2_blake2b_digest__doc__}, static PyObject * -_blake2_blake2b_digest_impl(BLAKE2bObject *self); +_blake2_blake2b_digest_impl(Blake2Object *self); static PyObject * -_blake2_blake2b_digest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_digest(Blake2Object *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_digest_impl(self); } @@ -258,11 +452,11 @@ PyDoc_STRVAR(_blake2_blake2b_hexdigest__doc__, {"hexdigest", (PyCFunction)_blake2_blake2b_hexdigest, METH_NOARGS, _blake2_blake2b_hexdigest__doc__}, static PyObject * -_blake2_blake2b_hexdigest_impl(BLAKE2bObject *self); +_blake2_blake2b_hexdigest_impl(Blake2Object *self); static PyObject * -_blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_hexdigest(Blake2Object *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl(self); } -/*[clinic end generated code: output=e18eeaee40623bfc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d1a351f44e20e273 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index b709204af1d959..50745fd4f407a3 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -908,7 +908,7 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -982,4 +982,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=364093af55bfe53f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=454309b21cfa9bf6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 5dc2fc068d0f7e..53b139e09afdf1 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -2,9 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() -#include "pycore_modsupport.h" // _PyArg_CheckPositional() - PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" "--\n" @@ -22,7 +19,7 @@ PyDoc_STRVAR(fcntl_fcntl__doc__, "corresponding to the return value of the fcntl call in the C code."); #define FCNTL_FCNTL_METHODDEF \ - {"fcntl", _PyCFunction_CAST(fcntl_fcntl), METH_FASTCALL, fcntl_fcntl__doc__}, + {"fcntl", (PyCFunction)(void(*)(void))fcntl_fcntl, METH_FASTCALL, fcntl_fcntl__doc__}, static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg); @@ -35,10 +32,16 @@ fcntl_fcntl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int code; PyObject *arg = NULL; - if (!_PyArg_CheckPositional("fcntl", nargs, 2, 3)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "fcntl expected at least 2 arguments, got %zd", nargs); + goto exit; + } + if (nargs > 3) { + PyErr_Format(PyExc_TypeError, "fcntl expected at most 3 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -90,10 +93,10 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, "code."); #define FCNTL_IOCTL_METHODDEF \ - {"ioctl", _PyCFunction_CAST(fcntl_ioctl), METH_FASTCALL, fcntl_ioctl__doc__}, + {"ioctl", (PyCFunction)(void(*)(void))fcntl_ioctl, METH_FASTCALL, fcntl_ioctl__doc__}, static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg); static PyObject * @@ -101,20 +104,27 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; - unsigned int code; + unsigned long code; PyObject *ob_arg = NULL; int mutate_arg = 1; - if (!_PyArg_CheckPositional("ioctl", nargs, 2, 4)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "ioctl expected at least 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + if (nargs > 4) { + PyErr_Format(PyExc_TypeError, "ioctl expected at most 4 arguments, got %zd", nargs); goto exit; } - code = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (code == (unsigned int)-1 && PyErr_Occurred()) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } + if (!PyLong_Check(args[1])) { + PyErr_Format(PyExc_TypeError, "ioctl() argument 2 must be int, not %T", args[1]); + goto exit; + } + code = PyLong_AsUnsignedLongMask(args[1]); if (nargs < 3) { goto skip_optional; } @@ -143,7 +153,7 @@ PyDoc_STRVAR(fcntl_flock__doc__, "function is emulated using fcntl())."); #define FCNTL_FLOCK_METHODDEF \ - {"flock", _PyCFunction_CAST(fcntl_flock), METH_FASTCALL, fcntl_flock__doc__}, + {"flock", (PyCFunction)(void(*)(void))fcntl_flock, METH_FASTCALL, fcntl_flock__doc__}, static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code); @@ -155,10 +165,12 @@ fcntl_flock(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int code; - if (!_PyArg_CheckPositional("flock", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "flock expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -199,7 +211,7 @@ PyDoc_STRVAR(fcntl_lockf__doc__, " 2 - relative to the end of the file (SEEK_END)"); #define FCNTL_LOCKF_METHODDEF \ - {"lockf", _PyCFunction_CAST(fcntl_lockf), METH_FASTCALL, fcntl_lockf__doc__}, + {"lockf", (PyCFunction)(void(*)(void))fcntl_lockf, METH_FASTCALL, fcntl_lockf__doc__}, static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, @@ -215,10 +227,16 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *startobj = NULL; int whence = 0; - if (!_PyArg_CheckPositional("lockf", nargs, 2, 5)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "lockf expected at least 2 arguments, got %zd", nargs); + goto exit; + } + if (nargs > 5) { + PyErr_Format(PyExc_TypeError, "lockf expected at most 5 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -246,4 +264,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=732e33ba92042031 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=45a56f53fd17ff3c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index d50d170589a2cd..be3bd35b4ffd44 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -312,28 +312,23 @@ PyDoc_STRVAR(gc_get_referrers__doc__, {"get_referrers", _PyCFunction_CAST(gc_get_referrers), METH_FASTCALL, gc_get_referrers__doc__}, static PyObject * -gc_get_referrers_impl(PyObject *module, PyObject *args); +gc_get_referrers_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args); static PyObject * gc_get_referrers(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("get_referrers", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = gc_get_referrers_impl(module, __clinic_args); + __clinic_args = args + 0; + return_value = gc_get_referrers_impl(module, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -347,28 +342,23 @@ PyDoc_STRVAR(gc_get_referents__doc__, {"get_referents", _PyCFunction_CAST(gc_get_referents), METH_FASTCALL, gc_get_referents__doc__}, static PyObject * -gc_get_referents_impl(PyObject *module, PyObject *args); +gc_get_referents_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args); static PyObject * gc_get_referents(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("get_referents", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = gc_get_referents_impl(module, __clinic_args); + __clinic_args = args + 0; + return_value = gc_get_referents_impl(module, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -469,6 +459,25 @@ PyDoc_STRVAR(gc_is_tracked__doc__, #define GC_IS_TRACKED_METHODDEF \ {"is_tracked", (PyCFunction)gc_is_tracked, METH_O, gc_is_tracked__doc__}, +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_tracked(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_tracked_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_is_finalized__doc__, "is_finalized($module, obj, /)\n" "--\n" @@ -478,6 +487,25 @@ PyDoc_STRVAR(gc_is_finalized__doc__, #define GC_IS_FINALIZED_METHODDEF \ {"is_finalized", (PyCFunction)gc_is_finalized, METH_O, gc_is_finalized__doc__}, +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_finalized(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_finalized_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_freeze__doc__, "freeze($module, /)\n" "--\n" @@ -547,4 +575,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=258f92524c1141fc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f488a0d4d6bd3687 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/grpmodule.c.h b/Modules/clinic/grpmodule.c.h index 1da061f3fd8eb1..cc0ad210f42743 100644 --- a/Modules/clinic/grpmodule.c.h +++ b/Modules/clinic/grpmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() - PyDoc_STRVAR(grp_getgrgid__doc__, "getgrgid($module, /, id)\n" "--\n" @@ -17,48 +11,21 @@ PyDoc_STRVAR(grp_getgrgid__doc__, "If id is not valid, raise KeyError."); #define GRP_GETGRGID_METHODDEF \ - {"getgrgid", _PyCFunction_CAST(grp_getgrgid), METH_FASTCALL|METH_KEYWORDS, grp_getgrgid__doc__}, + {"getgrgid", (PyCFunction)(void(*)(void))grp_getgrgid, METH_VARARGS|METH_KEYWORDS, grp_getgrgid__doc__}, static PyObject * grp_getgrgid_impl(PyObject *module, PyObject *id); static PyObject * -grp_getgrgid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +grp_getgrgid(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(id), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"id", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "getgrgid", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"id", NULL}; PyObject *id; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:getgrgid", _keywords, + &id)) goto exit; - } - id = args[0]; return_value = grp_getgrgid_impl(module, id); exit: @@ -74,52 +41,21 @@ PyDoc_STRVAR(grp_getgrnam__doc__, "If name is not valid, raise KeyError."); #define GRP_GETGRNAM_METHODDEF \ - {"getgrnam", _PyCFunction_CAST(grp_getgrnam), METH_FASTCALL|METH_KEYWORDS, grp_getgrnam__doc__}, + {"getgrnam", (PyCFunction)(void(*)(void))grp_getgrnam, METH_VARARGS|METH_KEYWORDS, grp_getgrnam__doc__}, static PyObject * grp_getgrnam_impl(PyObject *module, PyObject *name); static PyObject * -grp_getgrnam(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +grp_getgrnam(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"name", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "getgrnam", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"name", NULL}; PyObject *name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("getgrnam", "argument 'name'", "str", args[0]); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "U:getgrnam", _keywords, + &name)) goto exit; - } - name = args[0]; return_value = grp_getgrnam_impl(module, name); exit: @@ -146,4 +82,4 @@ grp_getgrall(PyObject *module, PyObject *Py_UNUSED(ignored)) { return grp_getgrall_impl(module); } -/*[clinic end generated code: output=2f7011384604d38d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=81f180beb67fc585 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 3ec479943a83d4..050c21460d79d7 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -47,7 +47,7 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(n), &_Py_ID(strict), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('n'), &_Py_ID(strict), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -509,7 +509,7 @@ itertools_combinations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(r), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -580,7 +580,7 @@ itertools_combinations_with_replacement(PyTypeObject *type, PyObject *args, PyOb PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(r), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -650,7 +650,7 @@ itertools_permutations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(r), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -928,4 +928,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=c6a515f765da86b5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7b13be3075f2d6d3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index ca14c03f16f706..7d0b98d5502267 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -8,6 +8,64 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(math_gcd__doc__, +"gcd($module, /, *integers)\n" +"--\n" +"\n" +"Greatest Common Divisor."); + +#define MATH_GCD_METHODDEF \ + {"gcd", _PyCFunction_CAST(math_gcd), METH_FASTCALL, math_gcd__doc__}, + +static PyObject * +math_gcd_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +math_gcd(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("gcd", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = args + 0; + return_value = math_gcd_impl(module, nvararg, __clinic_args); + +exit: + return return_value; +} + +PyDoc_STRVAR(math_lcm__doc__, +"lcm($module, /, *integers)\n" +"--\n" +"\n" +"Least Common Multiple."); + +#define MATH_LCM_METHODDEF \ + {"lcm", _PyCFunction_CAST(math_lcm), METH_FASTCALL, math_lcm__doc__}, + +static PyObject * +math_lcm_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +math_lcm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("lcm", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = args + 0; + return_value = math_lcm_impl(module, nvararg, __clinic_args); + +exit: + return return_value; +} + PyDoc_STRVAR(math_ceil__doc__, "ceil($module, x, /)\n" "--\n" @@ -34,9 +92,9 @@ PyDoc_STRVAR(math_fsum__doc__, "fsum($module, seq, /)\n" "--\n" "\n" -"Return an accurate floating point sum of values in the iterable seq.\n" +"Return an accurate floating-point sum of values in the iterable seq.\n" "\n" -"Assumes IEEE-754 floating point arithmetic."); +"Assumes IEEE-754 floating-point arithmetic."); #define MATH_FSUM_METHODDEF \ {"fsum", (PyCFunction)math_fsum, METH_O, math_fsum__doc__}, @@ -204,6 +262,67 @@ PyDoc_STRVAR(math_log10__doc__, #define MATH_LOG10_METHODDEF \ {"log10", (PyCFunction)math_log10, METH_O, math_log10__doc__}, +PyDoc_STRVAR(math_fma__doc__, +"fma($module, x, y, z, /)\n" +"--\n" +"\n" +"Fused multiply-add operation.\n" +"\n" +"Compute (x * y) + z with a single round."); + +#define MATH_FMA_METHODDEF \ + {"fma", _PyCFunction_CAST(math_fma), METH_FASTCALL, math_fma__doc__}, + +static PyObject * +math_fma_impl(PyObject *module, double x, double y, double z); + +static PyObject * +math_fma(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double z; + + if (!_PyArg_CheckPositional("fma", nargs, 3, 3)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[2])) { + z = PyFloat_AS_DOUBLE(args[2]); + } + else + { + z = PyFloat_AsDouble(args[2]); + if (z == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = math_fma_impl(module, x, y, z); + +exit: + return return_value; +} + PyDoc_STRVAR(math_fmod__doc__, "fmod($module, x, y, /)\n" "--\n" @@ -290,6 +409,46 @@ math_dist(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(math_hypot__doc__, +"hypot($module, /, *coordinates)\n" +"--\n" +"\n" +"Multidimensional Euclidean distance from the origin to a point.\n" +"\n" +"Roughly equivalent to:\n" +" sqrt(sum(x**2 for x in coordinates))\n" +"\n" +"For a two dimensional point (x, y), gives the hypotenuse\n" +"using the Pythagorean theorem: sqrt(x*x + y*y).\n" +"\n" +"For example, the hypotenuse of a 3/4/5 right triangle is:\n" +"\n" +" >>> hypot(3.0, 4.0)\n" +" 5.0"); + +#define MATH_HYPOT_METHODDEF \ + {"hypot", _PyCFunction_CAST(math_hypot), METH_FASTCALL, math_hypot__doc__}, + +static PyObject * +math_hypot_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +math_hypot(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("hypot", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = args + 0; + return_value = math_hypot_impl(module, nvararg, __clinic_args); + +exit: + return return_value; +} + PyDoc_STRVAR(math_sumprod__doc__, "sumprod($module, p, q, /)\n" "--\n" @@ -549,7 +708,7 @@ PyDoc_STRVAR(math_isclose__doc__, "isclose($module, /, a, b, *, rel_tol=1e-09, abs_tol=0.0)\n" "--\n" "\n" -"Determine whether two floating point numbers are close in value.\n" +"Determine whether two floating-point numbers are close in value.\n" "\n" " rel_tol\n" " maximum difference for being considered \"close\", relative to the\n" @@ -587,7 +746,7 @@ math_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -950,4 +1109,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=6b2eeaed8d8a76d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ee0a2f6bd1220061 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index b49d64d4281889..1857fca736ef20 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7,7 +7,6 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() #include "pycore_long.h" // _PyLong_UnsignedInt_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -73,7 +72,7 @@ os_stat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("stat", "path", 0, 1); + path_t path = PATH_T_INITIALIZE_P("stat", "path", 0, 0, 0, 1); int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; @@ -155,7 +154,7 @@ os_lstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("lstat", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lstat", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -251,7 +250,7 @@ os_access(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("access", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("access", "path", 0, 0, 0, 0); int mode; int dir_fd = DEFAULT_DIR_FD; int effective_ids = 0; @@ -410,7 +409,7 @@ os_chdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chdir", "path", 0, PATH_HAVE_FCHDIR); + path_t path = PATH_T_INITIALIZE_P("chdir", "path", 0, 0, 0, PATH_HAVE_FCHDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -481,7 +480,8 @@ os_fchdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fchdir_impl(module, fd); @@ -560,7 +560,7 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chmod", "path", 0, PATH_HAVE_FCHMOD); + path_t path = PATH_T_INITIALIZE_P("chmod", "path", 0, 0, 0, PATH_HAVE_FCHMOD); int mode; int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = CHMOD_DEFAULT_FOLLOW_SYMLINKS; @@ -725,7 +725,7 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchmod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchmod", "path", 0, 0, 0, 0); int mode; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -802,7 +802,7 @@ os_chflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chflags", "path", 0, 0, 0, 0); unsigned long flags; int follow_symlinks = 1; @@ -884,7 +884,7 @@ os_lchflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchflags", "path", 0, 0, 0, 0); unsigned long flags; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -954,7 +954,7 @@ os_chroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chroot", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1024,7 +1024,8 @@ os_fsync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fsync_impl(module, fd); @@ -1107,7 +1108,8 @@ os_fdatasync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fdatasync_impl(module, fd); @@ -1188,7 +1190,7 @@ os_chown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("chown", "path", 0, PATH_HAVE_FCHOWN); + path_t path = PATH_T_INITIALIZE_P("chown", "path", 0, 0, 0, PATH_HAVE_FCHOWN); uid_t uid; gid_t gid; int dir_fd = DEFAULT_DIR_FD; @@ -1353,7 +1355,7 @@ os_lchown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("lchown", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchown", "path", 0, 0, 0, 0); uid_t uid; gid_t gid; @@ -1474,8 +1476,8 @@ os_link(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("link", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("link", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("link", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("link", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; @@ -1581,7 +1583,7 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listdir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("listdir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { @@ -1697,7 +1699,7 @@ os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t volume = PATH_T_INITIALIZE("listmounts", "volume", 0, 0); + path_t volume = PATH_T_INITIALIZE_P("listmounts", "volume", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1761,7 +1763,7 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_isdevdrive", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_isdevdrive", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1798,7 +1800,7 @@ static PyObject * os__getfullpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfullpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfullpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1832,7 +1834,7 @@ static PyObject * os__getfinalpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfinalpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfinalpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1866,7 +1868,7 @@ static PyObject * os__findfirstfile(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_findfirstfile", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_findfirstfile", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1926,7 +1928,7 @@ os__getvolumepathname(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getvolumepathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getvolumepathname", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1990,7 +1992,7 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_splitroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_splitroot", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -2012,6 +2014,144 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #if defined(MS_WINDOWS) +PyDoc_STRVAR(os__path_exists__doc__, +"_path_exists($module, /, path)\n" +"--\n" +"\n" +"Test whether a path exists. Returns False for broken symbolic links."); + +#define OS__PATH_EXISTS_METHODDEF \ + {"_path_exists", _PyCFunction_CAST(os__path_exists), METH_FASTCALL|METH_KEYWORDS, os__path_exists__doc__}, + +static int +os__path_exists_impl(PyObject *module, path_t *path); + +static PyObject * +os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_exists", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t path = PATH_T_INITIALIZE_P("_path_exists", "path", 0, 0, 1, 1); + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_exists_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os__path_lexists__doc__, +"_path_lexists($module, /, path)\n" +"--\n" +"\n" +"Test whether a path exists. Returns True for broken symbolic links."); + +#define OS__PATH_LEXISTS_METHODDEF \ + {"_path_lexists", _PyCFunction_CAST(os__path_lexists), METH_FASTCALL|METH_KEYWORDS, os__path_lexists__doc__}, + +static int +os__path_lexists_impl(PyObject *module, path_t *path); + +static PyObject * +os__path_lexists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_lexists", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t path = PATH_T_INITIALIZE_P("_path_lexists", "path", 0, 0, 1, 1); + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_lexists_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + PyDoc_STRVAR(os__path_isdir__doc__, "_path_isdir($module, /, s)\n" "--\n" @@ -2021,8 +2161,8 @@ PyDoc_STRVAR(os__path_isdir__doc__, #define OS__PATH_ISDIR_METHODDEF \ {"_path_isdir", _PyCFunction_CAST(os__path_isdir), METH_FASTCALL|METH_KEYWORDS, os__path_isdir__doc__}, -static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *s); +static int +os__path_isdir_impl(PyObject *module, path_t *path); static PyObject * os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2037,7 +2177,7 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(s), }, + .ob_item = { _Py_LATIN1_CHR('s'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2054,16 +2194,26 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *s; + path_t path = PATH_T_INITIALIZE_P("_path_isdir", "path", 0, 0, 1, 1); + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - s = args[0]; - return_value = os__path_isdir_impl(module, s); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isdir_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2080,8 +2230,8 @@ PyDoc_STRVAR(os__path_isfile__doc__, #define OS__PATH_ISFILE_METHODDEF \ {"_path_isfile", _PyCFunction_CAST(os__path_isfile), METH_FASTCALL|METH_KEYWORDS, os__path_isfile__doc__}, -static PyObject * -os__path_isfile_impl(PyObject *module, PyObject *path); +static int +os__path_isfile_impl(PyObject *module, path_t *path); static PyObject * os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2113,16 +2263,26 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isfile", "path", 0, 0, 1, 1); + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_isfile_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isfile_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2130,20 +2290,20 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #if defined(MS_WINDOWS) -PyDoc_STRVAR(os__path_exists__doc__, -"_path_exists($module, /, path)\n" +PyDoc_STRVAR(os__path_islink__doc__, +"_path_islink($module, /, path)\n" "--\n" "\n" -"Test whether a path exists. Returns False for broken symbolic links"); +"Test whether a path is a symbolic link"); -#define OS__PATH_EXISTS_METHODDEF \ - {"_path_exists", _PyCFunction_CAST(os__path_exists), METH_FASTCALL|METH_KEYWORDS, os__path_exists__doc__}, +#define OS__PATH_ISLINK_METHODDEF \ + {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, -static PyObject * -os__path_exists_impl(PyObject *module, PyObject *path); +static int +os__path_islink_impl(PyObject *module, path_t *path); static PyObject * -os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2167,21 +2327,31 @@ os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj static const char * const _keywords[] = {"path", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_path_exists", + .fname = "_path_islink", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_islink", "path", 0, 0, 1, 1); + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_exists_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_islink_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2189,20 +2359,20 @@ os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #if defined(MS_WINDOWS) -PyDoc_STRVAR(os__path_islink__doc__, -"_path_islink($module, /, path)\n" +PyDoc_STRVAR(os__path_isjunction__doc__, +"_path_isjunction($module, /, path)\n" "--\n" "\n" -"Test whether a path is a symbolic link"); +"Test whether a path is a junction"); -#define OS__PATH_ISLINK_METHODDEF \ - {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, +#define OS__PATH_ISJUNCTION_METHODDEF \ + {"_path_isjunction", _PyCFunction_CAST(os__path_isjunction), METH_FASTCALL|METH_KEYWORDS, os__path_isjunction__doc__}, -static PyObject * -os__path_islink_impl(PyObject *module, PyObject *path); +static int +os__path_isjunction_impl(PyObject *module, path_t *path); static PyObject * -os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2226,37 +2396,109 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj static const char * const _keywords[] = {"path", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_path_islink", + .fname = "_path_isjunction", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isjunction", "path", 0, 0, 1, 1); + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_islink_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isjunction_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } #endif /* defined(MS_WINDOWS) */ +PyDoc_STRVAR(os__path_splitroot_ex__doc__, +"_path_splitroot_ex($module, /, p)\n" +"--\n" +"\n" +"Split a pathname into drive, root and tail.\n" +"\n" +"The tail contains anything after the root."); + +#define OS__PATH_SPLITROOT_EX_METHODDEF \ + {"_path_splitroot_ex", _PyCFunction_CAST(os__path_splitroot_ex), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot_ex__doc__}, + +static PyObject * +os__path_splitroot_ex_impl(PyObject *module, path_t *path); + +static PyObject * +os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('p'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"p", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_splitroot_ex", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t path = PATH_T_INITIALIZE("_path_splitroot_ex", "path", 0, 1, 1, 0, 0); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_splitroot_ex_impl(module, &path); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + PyDoc_STRVAR(os__path_normpath__doc__, "_path_normpath($module, /, path)\n" "--\n" "\n" -"Basic path normalization."); +"Normalize path, eliminating double slashes, etc."); #define OS__PATH_NORMPATH_METHODDEF \ {"_path_normpath", _PyCFunction_CAST(os__path_normpath), METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__}, static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path); +os__path_normpath_impl(PyObject *module, path_t *path); static PyObject * os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2288,16 +2530,21 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_normpath", "path", 0, 1, 1, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_normpath_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_normpath_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2352,7 +2599,7 @@ os_mkdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkdir", "path", 0, 0, 0, 0); int mode = 511; int dir_fd = DEFAULT_DIR_FD; @@ -2613,8 +2860,8 @@ os_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("rename", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("rename", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("rename", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("rename", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; @@ -2704,8 +2951,8 @@ os_replace(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("replace", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("replace", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("replace", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("replace", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; @@ -2793,7 +3040,7 @@ os_rmdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("rmdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("rmdir", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3042,7 +3289,7 @@ os_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("unlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("unlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3116,7 +3363,7 @@ os_remove(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("remove", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("remove", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3234,7 +3481,7 @@ os_utime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("utime", "path", 0, PATH_UTIME_HAVE_FD); + path_t path = PATH_T_INITIALIZE_P("utime", "path", 0, 0, 0, PATH_UTIME_HAVE_FD); PyObject *times = Py_None; PyObject *ns = NULL; int dir_fd = DEFAULT_DIR_FD; @@ -3369,7 +3616,7 @@ static PyObject * os_execv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("execv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("execv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("execv", nargs, 2, 2)) { @@ -3441,7 +3688,7 @@ os_execve(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("execve", "path", 0, PATH_HAVE_FEXECVE); + path_t path = PATH_T_INITIALIZE_P("execve", "path", 0, 0, 0, PATH_HAVE_FEXECVE); PyObject *argv; PyObject *env; @@ -3537,7 +3784,7 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawn", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawn", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3687,7 +3934,7 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawnp", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawnp", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3791,7 +4038,7 @@ os_spawnv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("spawnv", nargs, 3, 3)) { @@ -3845,7 +4092,7 @@ os_spawnve(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnve", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnve", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; @@ -4157,7 +4404,8 @@ os_sched_getscheduler(PyObject *module, PyObject *arg) PyObject *return_value = NULL; pid_t pid; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":sched_getscheduler", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_sched_getscheduler_impl(module, pid); @@ -4255,10 +4503,18 @@ os_sched_setscheduler(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int policy; PyObject *param_obj; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "iO:sched_setscheduler", - &pid, &policy, ¶m_obj)) { + if (!_PyArg_CheckPositional("sched_setscheduler", nargs, 3, 3)) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } + policy = PyLong_AsInt(args[1]); + if (policy == -1 && PyErr_Occurred()) { + goto exit; + } + param_obj = args[2]; return_value = os_sched_setscheduler_impl(module, pid, policy, param_obj); exit: @@ -4290,7 +4546,8 @@ os_sched_getparam(PyObject *module, PyObject *arg) PyObject *return_value = NULL; pid_t pid; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":sched_getparam", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_sched_getparam_impl(module, pid); @@ -4325,10 +4582,14 @@ os_sched_setparam(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; PyObject *param_obj; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "O:sched_setparam", - &pid, ¶m_obj)) { + if (!_PyArg_CheckPositional("sched_setparam", nargs, 2, 2)) { goto exit; } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + param_obj = args[1]; return_value = os_sched_setparam_impl(module, pid, param_obj); exit: @@ -4360,7 +4621,8 @@ os_sched_rr_get_interval(PyObject *module, PyObject *arg) pid_t pid; double _return_value; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":sched_rr_get_interval", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } _return_value = os_sched_rr_get_interval_impl(module, pid); @@ -4420,10 +4682,14 @@ os_sched_setaffinity(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; PyObject *mask; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "O:sched_setaffinity", - &pid, &mask)) { + if (!_PyArg_CheckPositional("sched_setaffinity", nargs, 2, 2)) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } + mask = args[1]; return_value = os_sched_setaffinity_impl(module, pid, mask); exit: @@ -4454,7 +4720,8 @@ os_sched_getaffinity(PyObject *module, PyObject *arg) PyObject *return_value = NULL; pid_t pid; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":sched_getaffinity", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_sched_getaffinity_impl(module, pid); @@ -4531,7 +4798,8 @@ os_grantpt(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_grantpt_impl(module, fd); @@ -4567,7 +4835,8 @@ os_unlockpt(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_unlockpt_impl(module, fd); @@ -4604,7 +4873,8 @@ os_ptsname(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_ptsname_impl(module, fd); @@ -4664,7 +4934,8 @@ os_login_tty(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_login_tty_impl(module, fd); @@ -5049,14 +5320,19 @@ os_getpgid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * static const char * const _keywords[] = {"pid", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" _Py_PARSE_PID ":getpgid", + .fname = "getpgid", .kwtuple = KWTUPLE, }; #undef KWTUPLE + PyObject *argsbuf[1]; pid_t pid; - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &pid)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_getpgid_impl(module, pid); @@ -5201,10 +5477,25 @@ os_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; Py_ssize_t signal; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "n:kill", - &pid, &signal)) { + if (!_PyArg_CheckPositional("kill", nargs, 2, 2)) { goto exit; } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + signal = ival; + } return_value = os_kill_impl(module, pid, signal); exit: @@ -5234,8 +5525,15 @@ os_killpg(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pgid; int signal; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "i:killpg", - &pgid, &signal)) { + if (!_PyArg_CheckPositional("killpg", nargs, 2, 2)) { + goto exit; + } + pgid = PyLong_AsPid(args[0]); + if (pgid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + signal = PyLong_AsInt(args[1]); + if (signal == -1 && PyErr_Occurred()) { goto exit; } return_value = os_killpg_impl(module, pgid, signal); @@ -5598,15 +5896,24 @@ os_wait4(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw static const char * const _keywords[] = {"pid", "options", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" _Py_PARSE_PID "i:wait4", + .fname = "wait4", .kwtuple = KWTUPLE, }; #undef KWTUPLE + PyObject *argsbuf[2]; pid_t pid; int options; - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &pid, &options)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + options = PyLong_AsInt(args[1]); + if (options == -1 && PyErr_Occurred()) { goto exit; } return_value = os_wait4_impl(module, pid, options); @@ -5650,8 +5957,18 @@ os_waitid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) id_t id; int options; - if (!_PyArg_ParseStack(args, nargs, "i" _Py_PARSE_PID "i:waitid", - &idtype, &id, &options)) { + if (!_PyArg_CheckPositional("waitid", nargs, 3, 3)) { + goto exit; + } + if (!idtype_t_converter(args[0], &idtype)) { + goto exit; + } + id = (id_t)PyLong_AsPid(args[1]); + if (id == (id_t)(-1) && PyErr_Occurred()) { + goto exit; + } + options = PyLong_AsInt(args[2]); + if (options == -1 && PyErr_Occurred()) { goto exit; } return_value = os_waitid_impl(module, idtype, id, options); @@ -5688,8 +6005,15 @@ os_waitpid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; int options; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "i:waitpid", - &pid, &options)) { + if (!_PyArg_CheckPositional("waitpid", nargs, 2, 2)) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + options = PyLong_AsInt(args[1]); + if (options == -1 && PyErr_Occurred()) { goto exit; } return_value = os_waitpid_impl(module, pid, options); @@ -5726,8 +6050,15 @@ os_waitpid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) intptr_t pid; int options; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_INTPTR "i:waitpid", - &pid, &options)) { + if (!_PyArg_CheckPositional("waitpid", nargs, 2, 2)) { + goto exit; + } + pid = (intptr_t)PyLong_AsVoidPtr(args[0]); + if (!pid && PyErr_Occurred()) { + goto exit; + } + options = PyLong_AsInt(args[1]); + if (options == -1 && PyErr_Occurred()) { goto exit; } return_value = os_waitpid_impl(module, pid, options); @@ -5763,7 +6094,7 @@ os_wait(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_WAIT) */ -#if (defined(__linux__) && defined(__NR_pidfd_open)) +#if (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) PyDoc_STRVAR(os_pidfd_open__doc__, "pidfd_open($module, /, pid, flags=0)\n" @@ -5805,24 +6136,37 @@ os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec static const char * const _keywords[] = {"pid", "flags", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" _Py_PARSE_PID "|O&:pidfd_open", + .fname = "pidfd_open", .kwtuple = KWTUPLE, }; #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; pid_t pid; unsigned int flags = 0; - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &pid, _PyLong_UnsignedInt_Converter, &flags)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!_PyLong_UnsignedInt_Converter(args[1], &flags)) { goto exit; } +skip_optional_pos: return_value = os_pidfd_open_impl(module, pid, flags); exit: return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_open)) */ +#endif /* (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ #if defined(HAVE_SETNS) @@ -5881,7 +6225,8 @@ os_setns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6016,7 +6361,7 @@ os_readlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("readlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("readlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -6100,8 +6445,8 @@ os_symlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("symlink", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("symlink", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("symlink", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("symlink", "dst", 0, 0, 0, 0); int target_is_directory = 0; int dir_fd = DEFAULT_DIR_FD; @@ -6156,7 +6501,7 @@ PyDoc_STRVAR(os_times__doc__, "\n" "The object returned behaves like a named tuple with these fields:\n" " (utime, stime, cutime, cstime, elapsed_time)\n" -"All fields are floating point numbers."); +"All fields are floating-point numbers."); #define OS_TIMES_METHODDEF \ {"times", (PyCFunction)os_times, METH_NOARGS, os_times__doc__}, @@ -6279,8 +6624,8 @@ PyDoc_STRVAR(os_timerfd_settime__doc__, {"timerfd_settime", _PyCFunction_CAST(os_timerfd_settime), METH_FASTCALL|METH_KEYWORDS, os_timerfd_settime__doc__}, static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval); +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double); static PyObject * os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -6315,14 +6660,15 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; int fd; int flags = 0; - double initial = 0.0; - double interval = 0.0; + double initial_double = 0.0; + double interval_double = 0.0; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6339,12 +6685,12 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } if (args[2]) { if (PyFloat_CheckExact(args[2])) { - initial = PyFloat_AS_DOUBLE(args[2]); + initial_double = PyFloat_AS_DOUBLE(args[2]); } else { - initial = PyFloat_AsDouble(args[2]); - if (initial == -1.0 && PyErr_Occurred()) { + initial_double = PyFloat_AsDouble(args[2]); + if (initial_double == -1.0 && PyErr_Occurred()) { goto exit; } } @@ -6353,17 +6699,17 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } } if (PyFloat_CheckExact(args[3])) { - interval = PyFloat_AS_DOUBLE(args[3]); + interval_double = PyFloat_AS_DOUBLE(args[3]); } else { - interval = PyFloat_AsDouble(args[3]); - if (interval == -1.0 && PyErr_Occurred()) { + interval_double = PyFloat_AsDouble(args[3]); + if (interval_double == -1.0 && PyErr_Occurred()) { goto exit; } } skip_optional_kwonly: - return_value = os_timerfd_settime_impl(module, fd, flags, initial, interval); + return_value = os_timerfd_settime_impl(module, fd, flags, initial_double, interval_double); exit: return return_value; @@ -6435,7 +6781,8 @@ os_timerfd_settime_ns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6495,7 +6842,8 @@ os_timerfd_gettime(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_timerfd_gettime_impl(module, fd); @@ -6529,7 +6877,8 @@ os_timerfd_gettime_ns(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_timerfd_gettime_ns_impl(module, fd); @@ -6560,7 +6909,8 @@ os_getsid(PyObject *module, PyObject *arg) PyObject *return_value = NULL; pid_t pid; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":getsid", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_getsid_impl(module, pid); @@ -6614,8 +6964,15 @@ os_setpgid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; pid_t pgrp; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "" _Py_PARSE_PID ":setpgid", - &pid, &pgrp)) { + if (!_PyArg_CheckPositional("setpgid", nargs, 2, 2)) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + pgrp = PyLong_AsPid(args[1]); + if (pgrp == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_setpgid_impl(module, pid, pgrp); @@ -6679,8 +7036,15 @@ os_tcsetpgrp(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; pid_t pgid; - if (!_PyArg_ParseStack(args, nargs, "i" _Py_PARSE_PID ":tcsetpgrp", - &fd, &pgid)) { + if (!_PyArg_CheckPositional("tcsetpgrp", nargs, 2, 2)) { + goto exit; + } + fd = PyLong_AsInt(args[0]); + if (fd == -1 && PyErr_Occurred()) { + goto exit; + } + pgid = PyLong_AsPid(args[1]); + if (pgid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_tcsetpgrp_impl(module, fd, pgid); @@ -6739,7 +7103,7 @@ os_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("open", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("open", "path", 0, 0, 0, 0); int flags; int mode = 511; int dir_fd = DEFAULT_DIR_FD; @@ -8327,7 +8691,7 @@ os_mkfifo(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkfifo", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkfifo", "path", 0, 0, 0, 0); int mode = 438; int dir_fd = DEFAULT_DIR_FD; @@ -8427,7 +8791,7 @@ os_mknod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mknod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mknod", "path", 0, 0, 0, 0); int mode = 384; dev_t device = 0; int dir_fd = DEFAULT_DIR_FD; @@ -8489,7 +8853,7 @@ PyDoc_STRVAR(os_major__doc__, #define OS_MAJOR_METHODDEF \ {"major", (PyCFunction)os_major, METH_O, os_major__doc__}, -static unsigned int +static PyObject * os_major_impl(PyObject *module, dev_t device); static PyObject * @@ -8497,16 +8861,11 @@ os_major(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; dev_t device; - unsigned int _return_value; if (!_Py_Dev_Converter(arg, &device)) { goto exit; } - _return_value = os_major_impl(module, device); - if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + return_value = os_major_impl(module, device); exit: return return_value; @@ -8525,7 +8884,7 @@ PyDoc_STRVAR(os_minor__doc__, #define OS_MINOR_METHODDEF \ {"minor", (PyCFunction)os_minor, METH_O, os_minor__doc__}, -static unsigned int +static PyObject * os_minor_impl(PyObject *module, dev_t device); static PyObject * @@ -8533,16 +8892,11 @@ os_minor(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; dev_t device; - unsigned int _return_value; if (!_Py_Dev_Converter(arg, &device)) { goto exit; } - _return_value = os_minor_impl(module, device); - if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + return_value = os_minor_impl(module, device); exit: return return_value; @@ -8562,25 +8916,23 @@ PyDoc_STRVAR(os_makedev__doc__, {"makedev", _PyCFunction_CAST(os_makedev), METH_FASTCALL, os_makedev__doc__}, static dev_t -os_makedev_impl(PyObject *module, int major, int minor); +os_makedev_impl(PyObject *module, dev_t major, dev_t minor); static PyObject * os_makedev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int major; - int minor; + dev_t major; + dev_t minor; dev_t _return_value; if (!_PyArg_CheckPositional("makedev", nargs, 2, 2)) { goto exit; } - major = PyLong_AsInt(args[0]); - if (major == -1 && PyErr_Occurred()) { + if (!_Py_Dev_Converter(args[0], &major)) { goto exit; } - minor = PyLong_AsInt(args[1]); - if (minor == -1 && PyErr_Occurred()) { + if (!_Py_Dev_Converter(args[1], &minor)) { goto exit; } _return_value = os_makedev_impl(module, major, minor); @@ -8681,7 +9033,7 @@ os_truncate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("truncate", "path", 0, PATH_HAVE_FTRUNCATE); + path_t path = PATH_T_INITIALIZE_P("truncate", "path", 0, 0, 0, PATH_HAVE_FTRUNCATE); Py_off_t length; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -9580,7 +9932,7 @@ os_statvfs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("statvfs", "path", 0, PATH_HAVE_FSTATVFS); + path_t path = PATH_T_INITIALIZE_P("statvfs", "path", 0, 0, 0, PATH_HAVE_FSTATVFS); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -9644,7 +9996,7 @@ os__getdiskusage(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getdiskusage", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getdiskusage", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -9691,7 +10043,8 @@ os_fpathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("fpathconf", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!conv_path_confname(args[1], &name)) { @@ -9757,7 +10110,7 @@ os_pathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("pathconf", "path", 0, PATH_HAVE_FPATHCONF); + path_t path = PATH_T_INITIALIZE_P("pathconf", "path", 0, 0, 0, PATH_HAVE_FPATHCONF); int name; long _return_value; @@ -9947,10 +10300,10 @@ os_startfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t filepath = PATH_T_INITIALIZE("startfile", "filepath", 0, 0); + path_t filepath = PATH_T_INITIALIZE_P("startfile", "filepath", 0, 0, 0, 0); const wchar_t *operation = NULL; const wchar_t *arguments = NULL; - path_t cwd = PATH_T_INITIALIZE("startfile", "cwd", 1, 0); + path_t cwd = PATH_T_INITIALIZE_P("startfile", "cwd", 1, 0, 0, 0); int show_cmd = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 5, 0, argsbuf); @@ -10285,8 +10638,8 @@ os_getxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("getxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("getxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("getxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("getxattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -10372,8 +10725,8 @@ os_setxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("setxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("setxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("setxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("setxattr", "attribute", 0, 0, 0, 0); Py_buffer value = {NULL, NULL}; int flags = 0; int follow_symlinks = 1; @@ -10480,8 +10833,8 @@ os_removexattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("removexattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("removexattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("removexattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("removexattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -10566,7 +10919,7 @@ os_listxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listxattr", "path", 1, 1); + path_t path = PATH_T_INITIALIZE_P("listxattr", "path", 1, 0, 0, 1); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); @@ -10834,7 +11187,8 @@ os_eventfd_read(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_eventfd_read_impl(module, fd); @@ -10896,7 +11250,8 @@ os_eventfd_write(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!_PyLong_UnsignedLongLong_Converter(args[1], &value)) { @@ -11071,7 +11426,8 @@ os_get_handle_inheritable(PyObject *module, PyObject *arg) intptr_t handle; int _return_value; - if (!PyArg_Parse(arg, "" _Py_PARSE_INTPTR ":get_handle_inheritable", &handle)) { + handle = (intptr_t)PyLong_AsVoidPtr(arg); + if (!handle && PyErr_Occurred()) { goto exit; } _return_value = os_get_handle_inheritable_impl(module, handle); @@ -11108,8 +11464,15 @@ os_set_handle_inheritable(PyObject *module, PyObject *const *args, Py_ssize_t na intptr_t handle; int inheritable; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_INTPTR "p:set_handle_inheritable", - &handle, &inheritable)) { + if (!_PyArg_CheckPositional("set_handle_inheritable", nargs, 2, 2)) { + goto exit; + } + handle = (intptr_t)PyLong_AsVoidPtr(args[0]); + if (!handle && PyErr_Occurred()) { + goto exit; + } + inheritable = PyObject_IsTrue(args[1]); + if (inheritable < 0) { goto exit; } return_value = os_set_handle_inheritable_impl(module, handle, inheritable); @@ -11541,7 +11904,7 @@ os_scandir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("scandir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("scandir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { @@ -11753,7 +12116,7 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_add_dll_directory", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_add_dll_directory", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -11929,6 +12292,60 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(MS_WINDOWS) */ +PyDoc_STRVAR(os__inputhook__doc__, +"_inputhook($module, /)\n" +"--\n" +"\n" +"Calls PyOS_CallInputHook droppong the GIL first"); + +#define OS__INPUTHOOK_METHODDEF \ + {"_inputhook", (PyCFunction)os__inputhook, METH_NOARGS, os__inputhook__doc__}, + +static PyObject * +os__inputhook_impl(PyObject *module); + +static PyObject * +os__inputhook(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__inputhook_impl(module); +} + +PyDoc_STRVAR(os__is_inputhook_installed__doc__, +"_is_inputhook_installed($module, /)\n" +"--\n" +"\n" +"Checks if PyOS_CallInputHook is set"); + +#define OS__IS_INPUTHOOK_INSTALLED_METHODDEF \ + {"_is_inputhook_installed", (PyCFunction)os__is_inputhook_installed, METH_NOARGS, os__is_inputhook_installed__doc__}, + +static PyObject * +os__is_inputhook_installed_impl(PyObject *module); + +static PyObject * +os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__is_inputhook_installed_impl(module); +} + +PyDoc_STRVAR(os__create_environ__doc__, +"_create_environ($module, /)\n" +"--\n" +"\n" +"Create the environment dictionary."); + +#define OS__CREATE_ENVIRON_METHODDEF \ + {"_create_environ", (PyCFunction)os__create_environ, METH_NOARGS, os__create_environ__doc__}, + +static PyObject * +os__create_environ_impl(PyObject *module); + +static PyObject * +os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__create_environ_impl(module); +} + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -12025,6 +12442,14 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS__PATH_SPLITROOT_METHODDEF #endif /* !defined(OS__PATH_SPLITROOT_METHODDEF) */ +#ifndef OS__PATH_EXISTS_METHODDEF + #define OS__PATH_EXISTS_METHODDEF +#endif /* !defined(OS__PATH_EXISTS_METHODDEF) */ + +#ifndef OS__PATH_LEXISTS_METHODDEF + #define OS__PATH_LEXISTS_METHODDEF +#endif /* !defined(OS__PATH_LEXISTS_METHODDEF) */ + #ifndef OS__PATH_ISDIR_METHODDEF #define OS__PATH_ISDIR_METHODDEF #endif /* !defined(OS__PATH_ISDIR_METHODDEF) */ @@ -12033,14 +12458,14 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS__PATH_ISFILE_METHODDEF #endif /* !defined(OS__PATH_ISFILE_METHODDEF) */ -#ifndef OS__PATH_EXISTS_METHODDEF - #define OS__PATH_EXISTS_METHODDEF -#endif /* !defined(OS__PATH_EXISTS_METHODDEF) */ - #ifndef OS__PATH_ISLINK_METHODDEF #define OS__PATH_ISLINK_METHODDEF #endif /* !defined(OS__PATH_ISLINK_METHODDEF) */ +#ifndef OS__PATH_ISJUNCTION_METHODDEF + #define OS__PATH_ISJUNCTION_METHODDEF +#endif /* !defined(OS__PATH_ISJUNCTION_METHODDEF) */ + #ifndef OS_NICE_METHODDEF #define OS_NICE_METHODDEF #endif /* !defined(OS_NICE_METHODDEF) */ @@ -12588,4 +13013,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=268af5cbc8baa9d4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9756767bdbdabe94 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pwdmodule.c.h b/Modules/clinic/pwdmodule.c.h index 43d4825031c7e6..365d99aab1dd22 100644 --- a/Modules/clinic/pwdmodule.c.h +++ b/Modules/clinic/pwdmodule.c.h @@ -2,8 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_BadArgument() - PyDoc_STRVAR(pwd_getpwuid__doc__, "getpwuid($module, uidobj, /)\n" "--\n" @@ -36,7 +34,7 @@ pwd_getpwnam(PyObject *module, PyObject *arg) PyObject *name; if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("getpwnam", "argument", "str", arg); + PyErr_Format(PyExc_TypeError, "getpwnam() argument must be str, not %T", arg); goto exit; } name = arg; @@ -73,4 +71,4 @@ pwd_getpwall(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef PWD_GETPWALL_METHODDEF #define PWD_GETPWALL_METHODDEF #endif /* !defined(PWD_GETPWALL_METHODDEF) */ -/*[clinic end generated code: output=5a8fb12939ff4ea3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dac88d500f6d6f49 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index a5b93e68598204..682d8481a2a2f4 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -4,17 +4,64 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(pyexpat_xmlparser_SetReparseDeferralEnabled__doc__, +"SetReparseDeferralEnabled($self, enabled, /)\n" +"--\n" +"\n" +"Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0."); + +#define PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF \ + {"SetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_SetReparseDeferralEnabled, METH_O, pyexpat_xmlparser_SetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled); + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled(xmlparseobject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int enabled; + + enabled = PyObject_IsTrue(arg); + if (enabled < 0) { + goto exit; + } + return_value = pyexpat_xmlparser_SetReparseDeferralEnabled_impl(self, enabled); + +exit: + return return_value; +} + +PyDoc_STRVAR(pyexpat_xmlparser_GetReparseDeferralEnabled__doc__, +"GetReparseDeferralEnabled($self, /)\n" +"--\n" +"\n" +"Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0."); + +#define PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF \ + {"GetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_GetReparseDeferralEnabled, METH_NOARGS, pyexpat_xmlparser_GetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self); + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled(xmlparseobject *self, PyObject *Py_UNUSED(ignored)) +{ + return pyexpat_xmlparser_GetReparseDeferralEnabled_impl(self); +} + PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, "Parse($self, data, isfinal=False, /)\n" "--\n" "\n" "Parse XML data.\n" "\n" -"`isfinal\' should be true at end of input."); +"\'isfinal\' should be true at end of input."); #define PYEXPAT_XMLPARSER_PARSE_METHODDEF \ {"Parse", _PyCFunction_CAST(pyexpat_xmlparser_Parse), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_Parse__doc__}, @@ -498,4 +545,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=48c4296e43777df4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9f1e9a7192d29976 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 67e76d4c89f59a..49c0e48d2e0eac 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -6,7 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() @@ -26,7 +26,7 @@ PyDoc_STRVAR(select_select__doc__, "gotten from a fileno() method call on one of those.\n" "\n" "The optional 4th argument specifies a timeout in seconds; it may be\n" -"a floating point number to specify fractions of seconds. If it is absent\n" +"a floating-point number to specify fractions of seconds. If it is absent\n" "or None, the call will never time out.\n" "\n" "The return value is a tuple of three lists corresponding to the first three\n" @@ -100,7 +100,8 @@ select_poll_register(pollObject *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("register", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -110,7 +111,9 @@ select_poll_register(pollObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_register_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -148,13 +151,16 @@ select_poll_modify(pollObject *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("modify", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!_PyLong_UnsignedShort_Converter(args[1], &eventmask)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_modify_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -182,10 +188,13 @@ select_poll_unregister(pollObject *self, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_unregister_impl(self, fd); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -228,7 +237,9 @@ select_poll_poll(pollObject *self, PyObject *const *args, Py_ssize_t nargs) } timeout_obj = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_poll_impl(self, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -268,7 +279,8 @@ select_devpoll_register(devpollObject *self, PyObject *const *args, Py_ssize_t n if (!_PyArg_CheckPositional("register", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -278,7 +290,9 @@ select_devpoll_register(devpollObject *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_register_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -318,7 +332,8 @@ select_devpoll_modify(devpollObject *self, PyObject *const *args, Py_ssize_t nar if (!_PyArg_CheckPositional("modify", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -328,7 +343,9 @@ select_devpoll_modify(devpollObject *self, PyObject *const *args, Py_ssize_t nar goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_modify_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -356,10 +373,13 @@ select_devpoll_unregister(devpollObject *self, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_unregister_impl(self, fd); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -402,7 +422,9 @@ select_devpoll_poll(devpollObject *self, PyObject *const *args, Py_ssize_t nargs } timeout_obj = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_poll_impl(self, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -429,7 +451,13 @@ select_devpoll_close_impl(devpollObject *self); static PyObject * select_devpoll_close(devpollObject *self, PyObject *Py_UNUSED(ignored)) { - return select_devpoll_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) */ @@ -451,7 +479,13 @@ select_devpoll_fileno_impl(devpollObject *self); static PyObject * select_devpoll_fileno(devpollObject *self, PyObject *Py_UNUSED(ignored)) { - return select_devpoll_fileno_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_fileno_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) */ @@ -610,7 +644,13 @@ select_epoll_close_impl(pyEpoll_Object *self); static PyObject * select_epoll_close(pyEpoll_Object *self, PyObject *Py_UNUSED(ignored)) { - return select_epoll_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_epoll_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_EPOLL) */ @@ -730,7 +770,8 @@ select_epoll_register(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t na if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -806,7 +847,8 @@ select_epoll_modify(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t narg if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } eventmask = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); @@ -874,7 +916,8 @@ select_epoll_unregister(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = select_epoll_unregister_impl(self, fd); @@ -1100,7 +1143,13 @@ select_kqueue_close_impl(kqueue_queue_Object *self); static PyObject * select_kqueue_close(kqueue_queue_Object *self, PyObject *Py_UNUSED(ignored)) { - return select_kqueue_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_kqueue_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_KQUEUE) */ @@ -1311,4 +1360,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=4c2dcb31cb17c2c6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f99427b75cbe6d44 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index bc33e066654364..986c0289f2bfcb 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -2,6 +2,10 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(signal_default_int_handler__doc__, @@ -276,6 +280,77 @@ signal_siginterrupt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_SIGINTERRUPT) */ +PyDoc_STRVAR(signal_set_wakeup_fd__doc__, +"set_wakeup_fd($module, fd, /, *, warn_on_full_buffer=True)\n" +"--\n" +"\n" +"Sets the fd to be written to (with the signal number) when a signal comes in.\n" +"\n" +"A library can use this to wakeup select or poll.\n" +"The previous fd or -1 is returned.\n" +"\n" +"The fd must be non-blocking."); + +#define SIGNAL_SET_WAKEUP_FD_METHODDEF \ + {"set_wakeup_fd", _PyCFunction_CAST(signal_set_wakeup_fd), METH_FASTCALL|METH_KEYWORDS, signal_set_wakeup_fd__doc__}, + +static PyObject * +signal_set_wakeup_fd_impl(PyObject *module, PyObject *fdobj, + int warn_on_full_buffer); + +static PyObject * +signal_set_wakeup_fd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(warn_on_full_buffer), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "warn_on_full_buffer", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_wakeup_fd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *fdobj; + int warn_on_full_buffer = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + fdobj = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + warn_on_full_buffer = PyObject_IsTrue(args[1]); + if (warn_on_full_buffer < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = signal_set_wakeup_fd_impl(module, fdobj, warn_on_full_buffer); + +exit: + return return_value; +} + #if defined(HAVE_SETITIMER) PyDoc_STRVAR(signal_setitimer__doc__, @@ -522,7 +597,7 @@ PyDoc_STRVAR(signal_sigtimedwait__doc__, "\n" "Like sigwaitinfo(), but with a timeout.\n" "\n" -"The timeout is specified in seconds, with floating point numbers allowed."); +"The timeout is specified in seconds, with floating-point numbers allowed."); #define SIGNAL_SIGTIMEDWAIT_METHODDEF \ {"sigtimedwait", _PyCFunction_CAST(signal_sigtimedwait), METH_FASTCALL, signal_sigtimedwait__doc__}, @@ -595,7 +670,7 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PTHREAD_KILL) */ -#if (defined(__linux__) && defined(__NR_pidfd_send_signal)) +#if (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) PyDoc_STRVAR(signal_pidfd_send_signal__doc__, "pidfd_send_signal($module, pidfd, signalnum, siginfo=None, flags=0, /)\n" @@ -648,7 +723,7 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_send_signal)) */ +#endif /* (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ #ifndef SIGNAL_ALARM_METHODDEF #define SIGNAL_ALARM_METHODDEF @@ -701,4 +776,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=5a9928cb2dc75b5f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c57b4b98fad6f4b8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 3f4056efff2fec..7b0a3f8d4b1cc6 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -6,8 +6,35 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(_socket_socket_close__doc__, +"close($self, /)\n" +"--\n" +"\n" +"close()\n" +"\n" +"Close the socket. It cannot be used after this call."); + +#define _SOCKET_SOCKET_CLOSE_METHODDEF \ + {"close", (PyCFunction)_socket_socket_close, METH_NOARGS, _socket_socket_close__doc__}, + +static PyObject * +_socket_socket_close_impl(PySocketSockObject *s); + +static PyObject * +_socket_socket_close(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(s); + return_value = _socket_socket_close_impl(s); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + static int sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, PyObject *fdobj); @@ -259,4 +286,4 @@ _socket_socket_if_nametoindex(PySocketSockObject *self, PyObject *arg) #ifndef _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF #define _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF #endif /* !defined(_SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF) */ -/*[clinic end generated code: output=eb37b5d88a1e4661 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6037e47b012911c5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/syslogmodule.c.h b/Modules/clinic/syslogmodule.c.h index 58b0ea57b065b3..77cf24ea5ba9ca 100644 --- a/Modules/clinic/syslogmodule.c.h +++ b/Modules/clinic/syslogmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(syslog_openlog__doc__, @@ -88,7 +89,9 @@ syslog_openlog(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(module); return_value = syslog_openlog_impl(module, ident, logopt, facility); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -129,7 +132,9 @@ syslog_syslog(PyObject *module, PyObject *args) PyErr_SetString(PyExc_TypeError, "syslog.syslog requires 1 to 2 arguments"); goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = syslog_syslog_impl(module, group_left_1, priority, message); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -150,7 +155,13 @@ syslog_closelog_impl(PyObject *module); static PyObject * syslog_closelog(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return syslog_closelog_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = syslog_closelog_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(syslog_setlogmask__doc__, @@ -251,4 +262,4 @@ syslog_LOG_UPTO(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=86ca2fd84b2da98e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8d25899bd31969d3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 1c340681e5862f..83f5a4f6e9f882 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -2,9 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() -#include "pycore_modsupport.h" // _PyArg_CheckPositional() - PyDoc_STRVAR(termios_tcgetattr__doc__, "tcgetattr($module, fd, /)\n" "--\n" @@ -30,7 +27,8 @@ termios_tcgetattr(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcgetattr_impl(module, fd); @@ -53,7 +51,7 @@ PyDoc_STRVAR(termios_tcsetattr__doc__, "queued output and discarding all queued input."); #define TERMIOS_TCSETATTR_METHODDEF \ - {"tcsetattr", _PyCFunction_CAST(termios_tcsetattr), METH_FASTCALL, termios_tcsetattr__doc__}, + {"tcsetattr", (PyCFunction)(void(*)(void))termios_tcsetattr, METH_FASTCALL, termios_tcsetattr__doc__}, static PyObject * termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term); @@ -66,10 +64,12 @@ termios_tcsetattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int when; PyObject *term; - if (!_PyArg_CheckPositional("tcsetattr", nargs, 3, 3)) { + if (nargs != 3) { + PyErr_Format(PyExc_TypeError, "tcsetattr expected 3 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } when = PyLong_AsInt(args[1]); @@ -93,7 +93,7 @@ PyDoc_STRVAR(termios_tcsendbreak__doc__, "has a system dependent meaning."); #define TERMIOS_TCSENDBREAK_METHODDEF \ - {"tcsendbreak", _PyCFunction_CAST(termios_tcsendbreak), METH_FASTCALL, termios_tcsendbreak__doc__}, + {"tcsendbreak", (PyCFunction)(void(*)(void))termios_tcsendbreak, METH_FASTCALL, termios_tcsendbreak__doc__}, static PyObject * termios_tcsendbreak_impl(PyObject *module, int fd, int duration); @@ -105,10 +105,12 @@ termios_tcsendbreak(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int duration; - if (!_PyArg_CheckPositional("tcsendbreak", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcsendbreak expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } duration = PyLong_AsInt(args[1]); @@ -139,7 +141,8 @@ termios_tcdrain(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcdrain_impl(module, fd); @@ -159,7 +162,7 @@ PyDoc_STRVAR(termios_tcflush__doc__, "both queues."); #define TERMIOS_TCFLUSH_METHODDEF \ - {"tcflush", _PyCFunction_CAST(termios_tcflush), METH_FASTCALL, termios_tcflush__doc__}, + {"tcflush", (PyCFunction)(void(*)(void))termios_tcflush, METH_FASTCALL, termios_tcflush__doc__}, static PyObject * termios_tcflush_impl(PyObject *module, int fd, int queue); @@ -171,10 +174,12 @@ termios_tcflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int queue; - if (!_PyArg_CheckPositional("tcflush", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcflush expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } queue = PyLong_AsInt(args[1]); @@ -198,7 +203,7 @@ PyDoc_STRVAR(termios_tcflow__doc__, "or termios.TCION to restart input."); #define TERMIOS_TCFLOW_METHODDEF \ - {"tcflow", _PyCFunction_CAST(termios_tcflow), METH_FASTCALL, termios_tcflow__doc__}, + {"tcflow", (PyCFunction)(void(*)(void))termios_tcflow, METH_FASTCALL, termios_tcflow__doc__}, static PyObject * termios_tcflow_impl(PyObject *module, int fd, int action); @@ -210,10 +215,12 @@ termios_tcflow(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int action; - if (!_PyArg_CheckPositional("tcflow", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcflow expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } action = PyLong_AsInt(args[1]); @@ -246,7 +253,8 @@ termios_tcgetwinsize(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcgetwinsize_impl(module, fd); @@ -265,7 +273,7 @@ PyDoc_STRVAR(termios_tcsetwinsize__doc__, "is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize()."); #define TERMIOS_TCSETWINSIZE_METHODDEF \ - {"tcsetwinsize", _PyCFunction_CAST(termios_tcsetwinsize), METH_FASTCALL, termios_tcsetwinsize__doc__}, + {"tcsetwinsize", (PyCFunction)(void(*)(void))termios_tcsetwinsize, METH_FASTCALL, termios_tcsetwinsize__doc__}, static PyObject * termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz); @@ -277,10 +285,12 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; PyObject *winsz; - if (!_PyArg_CheckPositional("tcsetwinsize", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcsetwinsize expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } winsz = args[1]; @@ -289,4 +299,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f31382658135c774 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6c6192583b0da36 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/unicodedata.c.h b/Modules/clinic/unicodedata.c.h index 739f498f1d2672..345440eeee89a6 100644 --- a/Modules/clinic/unicodedata.c.h +++ b/Modules/clinic/unicodedata.c.h @@ -36,7 +36,10 @@ unicodedata_UCD_decimal(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("decimal", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "decimal(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -82,7 +85,10 @@ unicodedata_UCD_digit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("digit", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "digit(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -129,7 +135,10 @@ unicodedata_UCD_numeric(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("numeric", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "numeric(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -167,7 +176,10 @@ unicodedata_UCD_category(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("category", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "category(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -202,7 +214,10 @@ unicodedata_UCD_bidirectional(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("bidirectional", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "bidirectional(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -238,7 +253,10 @@ unicodedata_UCD_combining(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("combining", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "combining(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -279,7 +297,10 @@ unicodedata_UCD_mirrored(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("mirrored", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "mirrored(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -316,7 +337,10 @@ unicodedata_UCD_east_asian_width(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("east_asian_width", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "east_asian_width(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -351,7 +375,10 @@ unicodedata_UCD_decomposition(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("decomposition", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "decomposition(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -473,7 +500,10 @@ unicodedata_UCD_name(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("name", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "name(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -519,4 +549,4 @@ unicodedata_UCD_lookup(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=ea30f89007b2bfff input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8a59d430cee41058 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 57bc55632be485..81cbf0d554de3c 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -117,7 +117,7 @@ enum special_types { static enum special_types special_type(double d) { - if (Py_IS_FINITE(d)) { + if (isfinite(d)) { if (d != 0) { if (copysign(1., d) == 1.) return ST_POS; @@ -131,7 +131,7 @@ special_type(double d) return ST_NZERO; } } - if (Py_IS_NAN(d)) + if (isnan(d)) return ST_NAN; if (copysign(1., d) == 1.) return ST_PINF; @@ -139,18 +139,18 @@ special_type(double d) return ST_NINF; } -#define SPECIAL_VALUE(z, table) \ - if (!Py_IS_FINITE((z).real) || !Py_IS_FINITE((z).imag)) { \ - errno = 0; \ - return table[special_type((z).real)] \ - [special_type((z).imag)]; \ +#define SPECIAL_VALUE(z, table) \ + if (!isfinite((z).real) || !isfinite((z).imag)) { \ + errno = 0; \ + return table[special_type((z).real)] \ + [special_type((z).imag)]; \ } #define P Py_MATH_PI #define P14 0.25*Py_MATH_PI #define P12 0.5*Py_MATH_PI #define P34 0.75*Py_MATH_PI -#define INF Py_HUGE_VAL +#define INF Py_INFINITY #define N Py_NAN #define U -9.5426319407711027e33 /* unlikely value, used as placeholder */ @@ -185,15 +185,8 @@ cmath_acos_impl(PyObject *module, Py_complex z) if (fabs(z.real) > CM_LARGE_DOUBLE || fabs(z.imag) > CM_LARGE_DOUBLE) { /* avoid unnecessary overflow for large arguments */ r.real = atan2(fabs(z.imag), z.real); - /* split into cases to make sure that the branch cut has the - correct continuity on systems with unsigned zeros */ - if (z.real < 0.) { - r.imag = -copysign(log(hypot(z.real/2., z.imag/2.)) + - M_LN2*2., z.imag); - } else { - r.imag = copysign(log(hypot(z.real/2., z.imag/2.)) + - M_LN2*2., -z.imag); - } + r.imag = -copysign(log(hypot(z.real/2., z.imag/2.)) + + M_LN2*2., z.imag); } else { s1.real = 1.-z.real; s1.imag = -z.imag; @@ -324,36 +317,6 @@ cmath_atan_impl(PyObject *module, Py_complex z) return r; } -/* Windows screws up atan2 for inf and nan, and alpha Tru64 5.1 doesn't follow - C99 for atan2(0., 0.). */ -static double -c_atan2(Py_complex z) -{ - if (Py_IS_NAN(z.real) || Py_IS_NAN(z.imag)) - return Py_NAN; - if (Py_IS_INFINITY(z.imag)) { - if (Py_IS_INFINITY(z.real)) { - if (copysign(1., z.real) == 1.) - /* atan2(+-inf, +inf) == +-pi/4 */ - return copysign(0.25*Py_MATH_PI, z.imag); - else - /* atan2(+-inf, -inf) == +-pi*3/4 */ - return copysign(0.75*Py_MATH_PI, z.imag); - } - /* atan2(+-inf, x) == +-pi/2 for finite x */ - return copysign(0.5*Py_MATH_PI, z.imag); - } - if (Py_IS_INFINITY(z.real) || z.imag == 0.) { - if (copysign(1., z.real) == 1.) - /* atan2(+-y, +inf) = atan2(+-0, +x) = +-0. */ - return copysign(0., z.imag); - else - /* atan2(+-y, -inf) = atan2(+-0., -x) = +-pi. */ - return copysign(Py_MATH_PI, z.imag); - } - return atan2(z.imag, z.real); -} - static Py_complex atanh_special_values[7][7]; @@ -386,11 +349,7 @@ cmath_atanh_impl(PyObject *module, Py_complex z) */ h = hypot(z.real/2., z.imag/2.); /* safe from overflow */ r.real = z.real/4./h/h; - /* the two negations in the next line cancel each other out - except when working with unsigned zeros: they're there to - ensure that the branch cut has the correct continuity on - systems that don't support signed zeros */ - r.imag = -copysign(Py_MATH_PI/2., -z.imag); + r.imag = copysign(Py_MATH_PI/2., z.imag); errno = 0; } else if (z.real == 1. && ay < CM_SQRT_DBL_MIN) { /* C99 standard says: atanh(1+/-0.) should be inf +/- 0i */ @@ -448,8 +407,8 @@ cmath_cosh_impl(PyObject *module, Py_complex z) double x_minus_one; /* special treatment for cosh(+/-inf + iy) if y is not a NaN */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) && + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -466,7 +425,7 @@ cmath_cosh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN */ - if (Py_IS_INFINITY(z.imag) && !Py_IS_NAN(z.real)) + if (isinf(z.imag) && !isnan(z.real)) errno = EDOM; else errno = 0; @@ -484,7 +443,7 @@ cmath_cosh_impl(PyObject *module, Py_complex z) r.imag = sin(z.imag) * sinh(z.real); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -509,8 +468,8 @@ cmath_exp_impl(PyObject *module, Py_complex z) Py_complex r; double l; - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -527,9 +486,9 @@ cmath_exp_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN and not -infinity */ - if (Py_IS_INFINITY(z.imag) && - (Py_IS_FINITE(z.real) || - (Py_IS_INFINITY(z.real) && z.real > 0))) + if (isinf(z.imag) && + (isfinite(z.real) || + (isinf(z.real) && z.real > 0))) errno = EDOM; else errno = 0; @@ -546,7 +505,7 @@ cmath_exp_impl(PyObject *module, Py_complex z) r.imag = l*sin(z.imag); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -686,8 +645,8 @@ cmath_sinh_impl(PyObject *module, Py_complex z) /* special treatment for sinh(+/-inf + iy) if y is finite and nonzero */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -704,7 +663,7 @@ cmath_sinh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN */ - if (Py_IS_INFINITY(z.imag) && !Py_IS_NAN(z.real)) + if (isinf(z.imag) && !isnan(z.real)) errno = EDOM; else errno = 0; @@ -720,7 +679,7 @@ cmath_sinh_impl(PyObject *module, Py_complex z) r.imag = sin(z.imag) * cosh(z.real); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -856,8 +815,8 @@ cmath_tanh_impl(PyObject *module, Py_complex z) /* special treatment for tanh(+/-inf + iy) if y is finite and nonzero */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = 1.0; @@ -876,7 +835,7 @@ cmath_tanh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if z.imag is +/-infinity and z.real is finite */ - if (Py_IS_INFINITY(z.imag) && Py_IS_FINITE(z.real)) + if (isinf(z.imag) && isfinite(z.real)) errno = EDOM; else errno = 0; @@ -966,7 +925,7 @@ cmath_phase_impl(PyObject *module, Py_complex z) double phi; errno = 0; - phi = c_atan2(z); /* should not cause any exception */ + phi = atan2(z.imag, z.real); /* should not cause any exception */ if (errno != 0) return math_error(); else @@ -991,7 +950,7 @@ cmath_polar_impl(PyObject *module, Py_complex z) double r, phi; errno = 0; - phi = c_atan2(z); /* should not cause any exception */ + phi = atan2(z.imag, z.real); /* should not cause any exception */ r = _Py_c_abs(z); /* sets errno to ERANGE on overflow */ if (errno != 0) return math_error(); @@ -1030,11 +989,11 @@ cmath_rect_impl(PyObject *module, double r, double phi) errno = 0; /* deal with special values */ - if (!Py_IS_FINITE(r) || !Py_IS_FINITE(phi)) { + if (!isfinite(r) || !isfinite(phi)) { /* if r is +/-infinity and phi is finite but nonzero then result is (+-INF +-INF i), but we need to compute cos(phi) and sin(phi) to figure out the signs. */ - if (Py_IS_INFINITY(r) && (Py_IS_FINITE(phi) + if (isinf(r) && (isfinite(phi) && (phi != 0.))) { if (r > 0) { z.real = copysign(INF, cos(phi)); @@ -1051,7 +1010,7 @@ cmath_rect_impl(PyObject *module, double r, double phi) } /* need to set errno = EDOM if r is a nonzero number and phi is infinite */ - if (r != 0. && !Py_IS_NAN(r) && Py_IS_INFINITY(phi)) + if (r != 0. && !isnan(r) && isinf(phi)) errno = EDOM; else errno = 0; @@ -1085,7 +1044,7 @@ static PyObject * cmath_isfinite_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=ac76611e2c774a36 input=848e7ee701895815]*/ { - return PyBool_FromLong(Py_IS_FINITE(z.real) && Py_IS_FINITE(z.imag)); + return PyBool_FromLong(isfinite(z.real) && isfinite(z.imag)); } /*[clinic input] @@ -1098,7 +1057,7 @@ static PyObject * cmath_isnan_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=e7abf6e0b28beab7 input=71799f5d284c9baf]*/ { - return PyBool_FromLong(Py_IS_NAN(z.real) || Py_IS_NAN(z.imag)); + return PyBool_FromLong(isnan(z.real) || isnan(z.imag)); } /*[clinic input] @@ -1111,8 +1070,7 @@ static PyObject * cmath_isinf_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=502a75a79c773469 input=363df155c7181329]*/ { - return PyBool_FromLong(Py_IS_INFINITY(z.real) || - Py_IS_INFINITY(z.imag)); + return PyBool_FromLong(isinf(z.real) || isinf(z.imag)); } /*[clinic input] @@ -1167,8 +1125,7 @@ cmath_isclose_impl(PyObject *module, Py_complex a, Py_complex b, above. */ - if (Py_IS_INFINITY(a.real) || Py_IS_INFINITY(a.imag) || - Py_IS_INFINITY(b.real) || Py_IS_INFINITY(b.imag)) { + if (isinf(a.real) || isinf(a.imag) || isinf(b.real) || isinf(b.imag)) { return 0; } @@ -1261,8 +1218,8 @@ cmath_exec(PyObject *mod) INIT_SPECIAL_VALUES(acosh_special_values, { C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,N) + C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) + C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) @@ -1341,8 +1298,8 @@ cmath_exec(PyObject *mod) INIT_SPECIAL_VALUES(tanh_special_values, { C(-1.,0.) C(U,U) C(-1.,-0.) C(-1.,0.) C(U,U) C(-1.,0.) C(-1.,0.) C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(N,N) C(N,N) + C(-0.0,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.0,N) C(-0.,N) + C(0.0,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.0,N) C(0.,N) C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) C(1.,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(1.,0.) C(1.,0.) C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) @@ -1363,6 +1320,7 @@ cmath_exec(PyObject *mod) static PyModuleDef_Slot cmath_slots[] = { {Py_mod_exec, cmath_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 1100e9f6094352..3f96f2f846d612 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,10 +1,9 @@ /* Errno module */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -952,6 +951,7 @@ errno_exec(PyObject *module) static PyModuleDef_Slot errno_slots[] = { {Py_mod_exec, errno_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 1c83563cbf68e7..d0d6015a66283f 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -11,11 +11,14 @@ Copyright (c) 2000-2005 Fred L. Drake, Jr. Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2016 Cristian Rodríguez Copyright (c) 2016 Thomas Beutlich Copyright (c) 2017 Rhodri James Copyright (c) 2022 Thijs Schreijer + Copyright (c) 2023 Hanno Böck + Copyright (c) 2023 Sony Corporation / Snild Dolkow + Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -269,7 +272,7 @@ XML_ParserCreate_MM(const XML_Char *encoding, const XML_Memory_Handling_Suite *memsuite, const XML_Char *namespaceSeparator); -/* Prepare a parser object to be re-used. This is particularly +/* Prepare a parser object to be reused. This is particularly valuable when memory allocation overhead is disproportionately high, such as when a large number of small documnents need to be parsed. All handlers are cleared from the parser, except for the @@ -951,7 +954,7 @@ XMLPARSEAPI(XML_Index) XML_GetCurrentByteIndex(XML_Parser parser); XMLPARSEAPI(int) XML_GetCurrentByteCount(XML_Parser parser); -/* If XML_CONTEXT_BYTES is defined, returns the input buffer, sets +/* If XML_CONTEXT_BYTES is >=1, returns the input buffer, sets the integer pointed to by offset to the offset within this buffer of the current parse position, and sets the integer pointed to by size to the size of this buffer (the number of input bytes). Otherwise @@ -1025,7 +1028,9 @@ enum XML_FeatureEnum { XML_FEATURE_ATTR_INFO, /* Added in Expat 2.4.0. */ XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, - XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT + XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, + /* Added in Expat 2.6.0. */ + XML_FEATURE_GE /* Additional features must be added to the end of this enum. */ }; @@ -1038,24 +1043,30 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -#ifdef XML_DTD -/* Added in Expat 2.4.0. */ +#if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) +/* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionMaximumAmplification( XML_Parser parser, float maximumAmplificationFactor); -/* Added in Expat 2.4.0. */ +/* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes); #endif +/* Added in Expat 2.6.0. */ +XMLPARSEAPI(XML_Bool) +XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); + /* Expat follows the semantic versioning convention. - See http://semver.org. + See https://semver.org */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 5 -#define XML_MICRO_VERSION 0 +#define XML_MINOR_VERSION 6 +#define XML_MICRO_VERSION 3 #ifdef __cplusplus } diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h index 6671f7b689ba97..e7d9499d9078d9 100644 --- a/Modules/expat/expat_config.h +++ b/Modules/expat/expat_config.h @@ -16,6 +16,7 @@ #define XML_NS 1 #define XML_DTD 1 +#define XML_GE 1 #define XML_CONTEXT_BYTES 1024 // bpo-30947: Python uses best available entropy sources to diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index e09f533b23c9df..167ec36804a43b 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -28,9 +28,11 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2003 Greg Stein - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2018 Yury Gribov Copyright (c) 2019 David Loffredo + Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow + Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -154,12 +156,21 @@ extern "C" { void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); -#if defined(XML_DTD) +#if defined(XML_GE) && XML_GE == 1 unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); const char *unsignedCharToPrintable(unsigned char c); #endif +extern +#if ! defined(XML_TESTING) + const +#endif + XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c +#if defined(XML_TESTING) +extern unsigned int g_bytesScanned; // used for testing only +#endif + #ifdef __cplusplus } #endif diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index d45d9b6c457159..8ee03ef0792815 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -108,6 +108,7 @@ #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler #define XML_SetParamEntityParsing PyExpat_XML_SetParamEntityParsing #define XML_SetProcessingInstructionHandler PyExpat_XML_SetProcessingInstructionHandler +#define XML_SetReparseDeferralEnabled PyExpat_XML_SetReparseDeferralEnabled #define XML_SetReturnNSTriplet PyExpat_XML_SetReturnNSTriplet #define XML_SetSkippedEntityHandler PyExpat_XML_SetSkippedEntityHandler #define XML_SetStartCdataSectionHandler PyExpat_XML_SetStartCdataSectionHandler diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index 303283ad2de98d..04f6f74585b5a2 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -106,7 +106,7 @@ * if this code is included and compiled as C++; related GCC warning is: * warning: use of C++11 long long integer constant [-Wlong-long] */ -#define _SIP_ULL(high, low) ((((uint64_t)high) << 32) | (low)) +#define SIP_ULL(high, low) ((((uint64_t)high) << 32) | (low)) #define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) @@ -126,8 +126,7 @@ | ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) \ | ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) -#define SIPHASH_INITIALIZER \ - { 0, 0, 0, 0, {0}, 0, 0 } +#define SIPHASH_INITIALIZER {0, 0, 0, 0, {0}, 0, 0} struct siphash { uint64_t v0, v1, v2, v3; @@ -190,10 +189,10 @@ sip_round(struct siphash *H, const int rounds) { static struct siphash * sip24_init(struct siphash *H, const struct sipkey *key) { - H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; - H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; - H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; - H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; + H->v0 = SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; + H->v1 = SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; + H->v2 = SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; + H->v3 = SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; H->p = H->buf; H->c = 0; diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index 2ecd61b5b94820..05805514ec7fa2 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -9,7 +9,8 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Greg Stein Copyright (c) 2005 Karl Waclawek - Copyright (c) 2017-2021 Sebastian Pipping + Copyright (c) 2017-2023 Sebastian Pipping + Copyright (c) 2023 Orgad Shaneh Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -35,7 +36,9 @@ #ifndef WINCONFIG_H #define WINCONFIG_H -#define WIN32_LEAN_AND_MEAN +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif #include #undef WIN32_LEAN_AND_MEAN diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index b6c2eca97567ba..d9285b213b38bd 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* 5ab094ffadd6edfc94c3eee53af44a86951f9f1f0933ada3114bbce2bfb02c99 (2.5.0+) +/* ba4cdf9bdb534f355a9def4c9e25d20ee8e72f95b0a4d930be52e563f5080196 (2.6.3+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -13,7 +13,7 @@ Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie Copyright (c) 2016 Eric Rahm - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2016 Gaurav Copyright (c) 2016 Thomas Beutlich Copyright (c) 2016 Gustavo Grieco @@ -32,10 +32,14 @@ Copyright (c) 2019 David Loffredo Copyright (c) 2019-2020 Ben Wagner Copyright (c) 2019 Vadim Zeitlin - Copyright (c) 2021 Dong-hee Na + Copyright (c) 2021 Donghee Na Copyright (c) 2022 Samanta Navarro Copyright (c) 2022 Jeffrey Walton Copyright (c) 2022 Jann Horn + Copyright (c) 2022 Sean McBride + Copyright (c) 2023 Owain Davies + Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow + Copyright (c) 2024 Berkay Eren Ürün Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -60,10 +64,25 @@ #define XML_BUILDING_EXPAT 1 -#include +#include "expat_config.h" -#if ! defined(_GNU_SOURCE) -# define _GNU_SOURCE 1 /* syscall prototype */ +#if ! defined(XML_GE) || (1 - XML_GE - 1 == 2) || (XML_GE < 0) || (XML_GE > 1) +# error XML_GE (for general entities) must be defined, non-empty, either 1 or 0 (0 to disable, 1 to enable; 1 is a common default) +#endif + +#if defined(XML_DTD) && XML_GE == 0 +# error Either undefine XML_DTD or define XML_GE to 1. +#endif + +#if ! defined(XML_CONTEXT_BYTES) || (1 - XML_CONTEXT_BYTES - 1 == 2) \ + || (XML_CONTEXT_BYTES + 0 < 0) +# error XML_CONTEXT_BYTES must be defined, non-empty and >=0 (0 to disable, >=1 to enable; 1024 is a common default) +#endif + +#if defined(HAVE_SYSCALL_GETRANDOM) +# if ! defined(_GNU_SOURCE) +# define _GNU_SOURCE 1 /* syscall prototype */ +# endif #endif #ifdef _WIN32 @@ -73,6 +92,7 @@ # endif #endif +#include #include #include /* memset(), memcpy() */ #include @@ -131,8 +151,8 @@ Your options include: \ * Linux >=3.17 + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ - * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ - * BSD / macOS (including <10.7) (arc4random): HAVE_ARC4RANDOM, \ + * BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ + * BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \ * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ * Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \ @@ -191,11 +211,13 @@ typedef char ICHAR; #endif /* Round up n to be a multiple of sz, where sz is a power of 2. */ -#define ROUND_UP(n, sz) (((n) + ((sz)-1)) & ~((sz)-1)) +#define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1)) /* Do safe (NULL-aware) pointer arithmetic */ #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0) +#define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) + #include "internal.h" #include "xmltok.h" #include "xmlrole.h" @@ -227,7 +249,7 @@ static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); it odd, since odd numbers are always relative prime to a power of 2. */ #define SECOND_HASH(hash, mask, power) \ - ((((hash) & ~(mask)) >> ((power)-1)) & ((mask) >> 2)) + ((((hash) & ~(mask)) >> ((power) - 1)) & ((mask) >> 2)) #define PROBE_STEP(hash, mask, power) \ ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1)) @@ -273,13 +295,13 @@ typedef struct { The name of the element is stored in both the document and API encodings. The memory buffer 'buf' is a separately-allocated memory area which stores the name. During the XML_Parse()/ - XMLParseBuffer() when the element is open, the memory for the 'raw' + XML_ParseBuffer() when the element is open, the memory for the 'raw' version of the name (in the document encoding) is shared with the document buffer. If the element is open across calls to XML_Parse()/XML_ParseBuffer(), the buffer is re-allocated to contain the 'raw' name as well. - A parser re-uses these structures, maintaining a list of allocated + A parser reuses these structures, maintaining a list of allocated TAG objects in a free list. */ typedef struct tag { @@ -408,12 +430,12 @@ enum XML_Account { XML_ACCOUNT_NONE /* i.e. do not account, was accounted already */ }; -#ifdef XML_DTD +#if XML_GE == 1 typedef unsigned long long XmlBigCount; typedef struct accounting { XmlBigCount countBytesDirect; XmlBigCount countBytesIndirect; - int debugLevel; + unsigned long debugLevel; float maximumAmplificationFactor; // >=1.0 unsigned long long activationThresholdBytes; } ACCOUNTING; @@ -422,9 +444,9 @@ typedef struct entity_stats { unsigned int countEverOpened; unsigned int currentDepth; unsigned int maximumDepthSeen; - int debugLevel; + unsigned long debugLevel; } ENTITY_STATS; -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, const char *end, const char **endPtr); @@ -464,41 +486,47 @@ static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, XML_Bool haveMore, enum XML_Account account); -static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *, +static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore, enum XML_Account account); #ifdef XML_DTD -static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *, +static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore); #endif /* XML_DTD */ static void freeBindings(XML_Parser parser, BINDING *bindings); -static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, - const char *s, TAG_NAME *tagNamePtr, +static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *enc, + const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account); static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, const XML_Char *uri, BINDING **bindingsPtr); -static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata, - XML_Bool isId, const XML_Char *dfltValue, - XML_Parser parser); -static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *, - XML_Bool isCdata, const char *, - const char *, STRING_POOL *, +static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, + XML_Bool isCdata, XML_Bool isId, + const XML_Char *value, XML_Parser parser); +static enum XML_Error storeAttributeValue(XML_Parser parser, + const ENCODING *enc, XML_Bool isCdata, + const char *ptr, const char *end, + STRING_POOL *pool, enum XML_Account account); -static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *, - XML_Bool isCdata, const char *, - const char *, STRING_POOL *, +static enum XML_Error appendAttributeValue(XML_Parser parser, + const ENCODING *enc, + XML_Bool isCdata, const char *ptr, + const char *end, STRING_POOL *pool, enum XML_Account account); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); -static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *); +static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType); +#if XML_GE == 1 static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start, const char *end, enum XML_Account account); +#else +static enum XML_Error storeSelfEntityValue(XML_Parser parser, ENTITY *entity); +#endif static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int reportComment(XML_Parser parser, const ENCODING *enc, @@ -518,21 +546,22 @@ static void dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms); static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms); -static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *, STRING_POOL *, - const HASH_TABLE *); +static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, + STRING_POOL *newPool, const HASH_TABLE *oldTable); static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize); -static void FASTCALL hashTableInit(HASH_TABLE *, +static void FASTCALL hashTableInit(HASH_TABLE *table, const XML_Memory_Handling_Suite *ms); -static void FASTCALL hashTableClear(HASH_TABLE *); -static void FASTCALL hashTableDestroy(HASH_TABLE *); -static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *); -static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *); +static void FASTCALL hashTableClear(HASH_TABLE *table); +static void FASTCALL hashTableDestroy(HASH_TABLE *table); +static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, + const HASH_TABLE *table); +static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter); -static void FASTCALL poolInit(STRING_POOL *, +static void FASTCALL poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms); -static void FASTCALL poolClear(STRING_POOL *); -static void FASTCALL poolDestroy(STRING_POOL *); +static void FASTCALL poolClear(STRING_POOL *pool); +static void FASTCALL poolDestroy(STRING_POOL *pool); static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, @@ -562,7 +591,7 @@ static XML_Parser parserCreate(const XML_Char *encodingName, static void parserInit(XML_Parser parser, const XML_Char *encodingName); -#ifdef XML_DTD +#if XML_GE == 1 static float accountingGetCurrentAmplification(XML_Parser rootParser); static void accountingReportStats(XML_Parser originParser, const char *epilog); static void accountingOnAbort(XML_Parser originParser); @@ -585,13 +614,12 @@ static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); #define poolStart(pool) ((pool)->start) -#define poolEnd(pool) ((pool)->ptr) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) #define poolLastChar(pool) (((pool)->ptr)[-1]) @@ -602,21 +630,41 @@ static unsigned long getDebugLevel(const char *variableName, ? 0 \ : ((*((pool)->ptr)++ = c), 1)) +#if ! defined(XML_TESTING) +const +#endif + XML_Bool g_reparseDeferralEnabledDefault + = XML_TRUE; // write ONLY in runtests.c +#if defined(XML_TESTING) +unsigned int g_bytesScanned = 0; // used for testing only +#endif + struct XML_ParserStruct { /* The first member must be m_userData so that the XML_GetUserData macro works. */ void *m_userData; void *m_handlerArg; - char *m_buffer; + + // How the four parse buffer pointers below relate in time and space: + // + // m_buffer <= m_bufferPtr <= m_bufferEnd <= m_bufferLim + // | | | | + // <--parsed-->| | | + // <---parsing--->| | + // <--unoccupied-->| + // <---------total-malloced/realloced-------->| + + char *m_buffer; // malloc/realloc base pointer of parse buffer const XML_Memory_Handling_Suite m_mem; - /* first character to be parsed */ - const char *m_bufferPtr; - /* past last character to be parsed */ - char *m_bufferEnd; - /* allocated end of m_buffer */ - const char *m_bufferLim; + const char *m_bufferPtr; // first character to be parsed + char *m_bufferEnd; // past last character to be parsed + const char *m_bufferLim; // allocated end of m_buffer + XML_Index m_parseEndByteIndex; const char *m_parseEndPtr; + size_t m_partialTokenBytesBefore; /* used in heuristic to avoid O(n^2) */ + XML_Bool m_reparseDeferralEnabled; + int m_lastBufferRequestSize; XML_Char *m_dataBuf; XML_Char *m_dataBufEnd; XML_StartElementHandler m_startElementHandler; @@ -703,7 +751,7 @@ struct XML_ParserStruct { enum XML_ParamEntityParsing m_paramEntityParsing; #endif unsigned long m_hash_secret_salt; -#ifdef XML_DTD +#if XML_GE == 1 ACCOUNTING m_accounting; ENTITY_STATS m_entity_stats; #endif @@ -948,6 +996,49 @@ get_hash_secret_salt(XML_Parser parser) { return parser->m_hash_secret_salt; } +static enum XML_Error +callProcessor(XML_Parser parser, const char *start, const char *end, + const char **endPtr) { + const size_t have_now = EXPAT_SAFE_PTR_DIFF(end, start); + + if (parser->m_reparseDeferralEnabled + && ! parser->m_parsingStatus.finalBuffer) { + // Heuristic: don't try to parse a partial token again until the amount of + // available data has increased significantly. + const size_t had_before = parser->m_partialTokenBytesBefore; + // ...but *do* try anyway if we're close to causing a reallocation. + size_t available_buffer + = EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); +#if XML_CONTEXT_BYTES > 0 + available_buffer -= EXPAT_MIN(available_buffer, XML_CONTEXT_BYTES); +#endif + available_buffer + += EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd); + // m_lastBufferRequestSize is never assigned a value < 0, so the cast is ok + const bool enough + = (have_now >= 2 * had_before) + || ((size_t)parser->m_lastBufferRequestSize > available_buffer); + + if (! enough) { + *endPtr = start; // callers may expect this to be set + return XML_ERROR_NONE; + } + } +#if defined(XML_TESTING) + g_bytesScanned += (unsigned)have_now; +#endif + const enum XML_Error ret = parser->m_processor(parser, start, end, endPtr); + if (ret == XML_ERROR_NONE) { + // if we consumed nothing, remember what we had on this parse attempt. + if (*endPtr == start) { + parser->m_partialTokenBytesBefore = have_now; + } else { + parser->m_partialTokenBytesBefore = 0; + } + } + return ret; +} + static XML_Bool /* only valid for root parser */ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ @@ -1129,6 +1220,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_bufferEnd = parser->m_buffer; parser->m_parseEndByteIndex = 0; parser->m_parseEndPtr = NULL; + parser->m_partialTokenBytesBefore = 0; + parser->m_reparseDeferralEnabled = g_reparseDeferralEnabledDefault; + parser->m_lastBufferRequestSize = 0; parser->m_declElementType = NULL; parser->m_declAttributeId = NULL; parser->m_declEntity = NULL; @@ -1163,7 +1257,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { #endif parser->m_hash_secret_salt = 0; -#ifdef XML_DTD +#if XML_GE == 1 memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); parser->m_accounting.debugLevel = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); parser->m_accounting.maximumAmplificationFactor @@ -1298,6 +1392,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, to worry which hash secrets each table has. */ unsigned long oldhash_secret_salt; + XML_Bool oldReparseDeferralEnabled; /* Validate the oldParser parameter before we pull everything out of it */ if (oldParser == NULL) @@ -1342,6 +1437,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, to worry which hash secrets each table has. */ oldhash_secret_salt = parser->m_hash_secret_salt; + oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled; #ifdef XML_DTD if (! context) @@ -1394,6 +1490,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities; parser->m_ns_triplets = oldns_triplets; parser->m_hash_secret_salt = oldhash_secret_salt; + parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled; parser->m_parentParser = oldParser; #ifdef XML_DTD parser->m_paramEntityParsing = oldParamEntityParsing; @@ -1848,55 +1945,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_PARSING; } - if (len == 0) { - parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; - if (! isFinal) - return XML_STATUS_OK; - parser->m_positionPtr = parser->m_bufferPtr; - parser->m_parseEndPtr = parser->m_bufferEnd; - - /* If data are left over from last buffer, and we now know that these - data are the final chunk of input, then we have to check them again - to detect errors based on that fact. - */ - parser->m_errorCode - = parser->m_processor(parser, parser->m_bufferPtr, - parser->m_parseEndPtr, &parser->m_bufferPtr); - - if (parser->m_errorCode == XML_ERROR_NONE) { - switch (parser->m_parsingStatus.parsing) { - case XML_SUSPENDED: - /* It is hard to be certain, but it seems that this case - * cannot occur. This code is cleaning up a previous parse - * with no new data (since len == 0). Changing the parsing - * state requires getting to execute a handler function, and - * there doesn't seem to be an opportunity for that while in - * this circumstance. - * - * Given the uncertainty, we retain the code but exclude it - * from coverage tests. - * - * LCOV_EXCL_START - */ - XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, - parser->m_bufferPtr, &parser->m_position); - parser->m_positionPtr = parser->m_bufferPtr; - return XML_STATUS_SUSPENDED; - /* LCOV_EXCL_STOP */ - case XML_INITIALIZED: - case XML_PARSING: - parser->m_parsingStatus.parsing = XML_FINISHED; - /* fall through */ - default: - return XML_STATUS_OK; - } - } - parser->m_eventEndPtr = parser->m_eventPtr; - parser->m_processor = errorProcessor; - return XML_STATUS_ERROR; - } -#ifndef XML_CONTEXT_BYTES - else if (parser->m_bufferPtr == parser->m_bufferEnd) { +#if XML_CONTEXT_BYTES == 0 + if (parser->m_bufferPtr == parser->m_bufferEnd) { const char *end; int nLeftOver; enum XML_Status result; @@ -1907,12 +1957,15 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_processor = errorProcessor; return XML_STATUS_ERROR; } + // though this isn't a buffer request, we assume that `len` is the app's + // preferred buffer fill size, and therefore save it here. + parser->m_lastBufferRequestSize = len; parser->m_parseEndByteIndex += len; parser->m_positionPtr = s; parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; parser->m_errorCode - = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end); + = callProcessor(parser, s, parser->m_parseEndPtr = s + len, &end); if (parser->m_errorCode != XML_ERROR_NONE) { parser->m_eventEndPtr = parser->m_eventPtr; @@ -1939,23 +1992,25 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { &parser->m_position); nLeftOver = s + len - end; if (nLeftOver) { - if (parser->m_buffer == NULL - || nLeftOver > parser->m_bufferLim - parser->m_buffer) { - /* avoid _signed_ integer overflow */ - char *temp = NULL; - const int bytesToAllocate = (int)((unsigned)len * 2U); - if (bytesToAllocate > 0) { - temp = (char *)REALLOC(parser, parser->m_buffer, bytesToAllocate); - } - if (temp == NULL) { - parser->m_errorCode = XML_ERROR_NO_MEMORY; - parser->m_eventPtr = parser->m_eventEndPtr = NULL; - parser->m_processor = errorProcessor; - return XML_STATUS_ERROR; - } - parser->m_buffer = temp; - parser->m_bufferLim = parser->m_buffer + bytesToAllocate; + // Back up and restore the parsing status to avoid XML_ERROR_SUSPENDED + // (and XML_ERROR_FINISHED) from XML_GetBuffer. + const enum XML_Parsing originalStatus = parser->m_parsingStatus.parsing; + parser->m_parsingStatus.parsing = XML_PARSING; + void *const temp = XML_GetBuffer(parser, nLeftOver); + parser->m_parsingStatus.parsing = originalStatus; + // GetBuffer may have overwritten this, but we want to remember what the + // app requested, not how many bytes were left over after parsing. + parser->m_lastBufferRequestSize = len; + if (temp == NULL) { + // NOTE: parser->m_errorCode has already been set by XML_GetBuffer(). + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; } + // Since we know that the buffer was empty and XML_CONTEXT_BYTES is 0, we + // don't have any data to preserve, and can copy straight into the start + // of the buffer rather than the GetBuffer return pointer (which may be + // pointing further into the allocated buffer). memcpy(parser->m_buffer, end, nLeftOver); } parser->m_bufferPtr = parser->m_buffer; @@ -1966,16 +2021,15 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_eventEndPtr = parser->m_bufferPtr; return result; } -#endif /* not defined XML_CONTEXT_BYTES */ - else { - void *buff = XML_GetBuffer(parser, len); - if (buff == NULL) - return XML_STATUS_ERROR; - else { - memcpy(buff, s, len); - return XML_ParseBuffer(parser, len, isFinal); - } +#endif /* XML_CONTEXT_BYTES == 0 */ + void *buff = XML_GetBuffer(parser, len); + if (buff == NULL) + return XML_STATUS_ERROR; + if (len > 0) { + assert(s != NULL); // make sure s==NULL && len!=0 was rejected above + memcpy(buff, s, len); } + return XML_ParseBuffer(parser, len, isFinal); } enum XML_Status XMLCALL @@ -1985,6 +2039,12 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { if (parser == NULL) return XML_STATUS_ERROR; + + if (len < 0) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } + switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2015,8 +2075,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parseEndByteIndex += len; parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; - parser->m_errorCode = parser->m_processor( - parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr); + parser->m_errorCode = callProcessor(parser, start, parser->m_parseEndPtr, + &parser->m_bufferPtr); if (parser->m_errorCode != XML_ERROR_NONE) { parser->m_eventEndPtr = parser->m_eventPtr; @@ -2061,10 +2121,14 @@ XML_GetBuffer(XML_Parser parser, int len) { default:; } - if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) { -#ifdef XML_CONTEXT_BYTES + // whether or not the request succeeds, `len` seems to be the app's preferred + // buffer fill size; remember it. + parser->m_lastBufferRequestSize = len; + if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) + || parser->m_buffer == NULL) { +#if XML_CONTEXT_BYTES > 0 int keep; -#endif /* defined XML_CONTEXT_BYTES */ +#endif /* XML_CONTEXT_BYTES > 0 */ /* Do not invoke signed arithmetic overflow: */ int neededSize = (int)((unsigned)len + (unsigned)EXPAT_SAFE_PTR_DIFF( @@ -2073,7 +2137,7 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; @@ -2083,10 +2147,11 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } neededSize += keep; -#endif /* defined XML_CONTEXT_BYTES */ - if (neededSize - <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { -#ifdef XML_CONTEXT_BYTES +#endif /* XML_CONTEXT_BYTES > 0 */ + if (parser->m_buffer && parser->m_bufferPtr + && neededSize + <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { +#if XML_CONTEXT_BYTES > 0 if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { int offset = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) @@ -2099,19 +2164,17 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferPtr -= offset; } #else - if (parser->m_buffer && parser->m_bufferPtr) { - memmove(parser->m_buffer, parser->m_bufferPtr, - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - parser->m_bufferEnd - = parser->m_buffer - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); - parser->m_bufferPtr = parser->m_buffer; - } -#endif /* not defined XML_CONTEXT_BYTES */ + memmove(parser->m_buffer, parser->m_bufferPtr, + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); + parser->m_bufferEnd + = parser->m_buffer + + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); + parser->m_bufferPtr = parser->m_buffer; +#endif /* XML_CONTEXT_BYTES > 0 */ } else { char *newBuf; int bufferSize - = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr); + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); if (bufferSize == 0) bufferSize = INIT_BUFFER_SIZE; do { @@ -2128,7 +2191,7 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } parser->m_bufferLim = newBuf + bufferSize; -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 if (parser->m_bufferPtr) { memcpy(newBuf, &parser->m_bufferPtr[-keep], EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) @@ -2158,7 +2221,7 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferEnd = newBuf; } parser->m_bufferPtr = parser->m_buffer = newBuf; -#endif /* not defined XML_CONTEXT_BYTES */ +#endif /* XML_CONTEXT_BYTES > 0 */ } parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; @@ -2208,7 +2271,7 @@ XML_ResumeParser(XML_Parser parser) { } parser->m_parsingStatus.parsing = XML_PARSING; - parser->m_errorCode = parser->m_processor( + parser->m_errorCode = callProcessor( parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr); if (parser->m_errorCode != XML_ERROR_NONE) { @@ -2272,7 +2335,7 @@ XML_GetCurrentByteCount(XML_Parser parser) { const char *XMLCALL XML_GetInputContext(XML_Parser parser, int *offset, int *size) { -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 if (parser == NULL) return NULL; if (parser->m_eventPtr && parser->m_buffer) { @@ -2286,7 +2349,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)parser; (void)offset; (void)size; -#endif /* defined XML_CONTEXT_BYTES */ +#endif /* XML_CONTEXT_BYTES > 0 */ return (const char *)0; } @@ -2506,7 +2569,7 @@ XML_GetFeatureList(void) { #ifdef XML_DTD {XML_FEATURE_DTD, XML_L("XML_DTD"), 0}, #endif -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), XML_CONTEXT_BYTES}, #endif @@ -2522,8 +2585,9 @@ XML_GetFeatureList(void) { #ifdef XML_ATTR_INFO {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, #endif -#ifdef XML_DTD - /* Added in Expat 2.4.0. */ +#if XML_GE == 1 + /* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, XML_L("XML_BLAP_MAX_AMP"), (long int) @@ -2531,13 +2595,15 @@ XML_GetFeatureList(void) { {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, XML_L("XML_BLAP_ACT_THRES"), EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, + /* Added in Expat 2.6.0. */ + {XML_FEATURE_GE, XML_L("XML_GE"), 0}, #endif {XML_FEATURE_END, NULL, 0}}; return features; } -#ifdef XML_DTD +#if XML_GE == 1 XML_Bool XMLCALL XML_SetBillionLaughsAttackProtectionMaximumAmplification( XML_Parser parser, float maximumAmplificationFactor) { @@ -2559,7 +2625,16 @@ XML_SetBillionLaughsAttackProtectionActivationThreshold( parser->m_accounting.activationThresholdBytes = activationThresholdBytes; return XML_TRUE; } -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ + +XML_Bool XMLCALL +XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) { + parser->m_reparseDeferralEnabled = enabled; + return XML_TRUE; + } + return XML_FALSE; +} /* Initially tag->rawName always points into the parse buffer; for those TAG instances opened while the current parse buffer was @@ -2581,7 +2656,7 @@ storeRawNames(XML_Parser parser) { */ if (tag->rawName == rawNameBuf) break; - /* For re-use purposes we need to ensure that the + /* For reuse purposes we need to ensure that the size of tag->buf is a multiple of sizeof(XML_Char). */ rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char)); @@ -2645,13 +2720,13 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, int tok = XmlContentTok(parser->m_encoding, start, end, &next); switch (tok) { case XML_TOK_BOM: -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, start, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ /* If we are at the end of the buffer, this would cause the next stage, i.e. externalEntityInitProcessor3, to pass control directly to @@ -2765,7 +2840,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, for (;;) { const char *next = s; /* XmlContentTok doesn't always set the last arg */ int tok = XmlContentTok(enc, s, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 const char *accountAfter = ((tok == XML_TOK_TRAILING_RSQB) || (tok == XML_TOK_TRAILING_CR)) ? (haveMore ? s /* i.e. 0 bytes */ : end) @@ -2831,14 +2906,14 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { -#ifdef XML_DTD +#if XML_GE == 1 /* NOTE: We are replacing 4-6 characters original input for 1 character * so there is no amplification and hence recording without * protection. */ accountingDiffTolerated(parser, tok, (char *)&ch, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ if (parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); else if (parser->m_defaultHandler) @@ -3039,13 +3114,13 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (parser->m_ns && localPart) { /* localPart and prefix may have been overwritten in tag->name.str, since this points to the binding->uri - buffer which gets re-used; so we have to add them again + buffer which gets reused; so we have to add them again */ uri = (XML_Char *)tag->name.str + tag->name.uriLen; /* don't need to check for space - already done in storeAtts() */ while (*localPart) *uri++ = *localPart++; - prefix = (XML_Char *)tag->name.prefix; + prefix = tag->name.prefix; if (parser->m_ns_triplets && prefix) { *uri++ = parser->m_namespaceSeparator; while (*prefix) @@ -3112,7 +3187,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, However, now we have a start/endCdataSectionHandler, so it seems easier to let the user deal with this. */ - else if (0 && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); /* END disabled code */ @@ -3141,8 +3216,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); } else parser->m_characterDataHandler( - parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)end - (XML_Char *)s)); + parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)end - (const XML_Char *)s)); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3175,8 +3250,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventPP = s; } } else - charDataHandler(parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)next - (XML_Char *)s)); + charDataHandler(parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)next - (const XML_Char *)s)); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4040,7 +4115,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, for (;;) { const char *next = s; /* in case of XML_TOK_NONE or XML_TOK_PARTIAL */ int tok = XmlCdataSectionTok(enc, s, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { accountingOnAbort(parser); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; @@ -4055,7 +4130,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, parser->m_endCdataSectionHandler(parser->m_handlerArg); /* BEGIN disabled code */ /* see comment under XML_TOK_CDATA_SECT_OPEN */ - else if (0 && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); /* END disabled code */ @@ -4091,8 +4166,8 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventPP = s; } } else - charDataHandler(parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)next - (XML_Char *)s)); + charDataHandler(parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)next - (const XML_Char *)s)); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4192,7 +4267,7 @@ doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventPP = s; *startPtr = NULL; tok = XmlIgnoreSectionTok(enc, s, end, &next); -# ifdef XML_DTD +# if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -4284,7 +4359,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, const XML_Char *storedversion = NULL; int standalone = -1; -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, XML_TOK_XML_DECL, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -4482,16 +4557,16 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, parser->m_processor = entityValueProcessor; return entityValueProcessor(parser, next, end, nextPtr); } - /* If we are at the end of the buffer, this would cause XmlPrologTok to - return XML_TOK_NONE on the next call, which would then cause the - function to exit with *nextPtr set to s - that is what we want for other - tokens, but not for the BOM - we would rather like to skip it; - then, when this routine is entered the next time, XmlPrologTok will - return XML_TOK_INVALID, since the BOM is still in the buffer + /* XmlPrologTok has now set the encoding based on the BOM it found, and we + must move s and nextPtr forward to consume the BOM. + + If we didn't, and got XML_TOK_NONE from the next XmlPrologTok call, we + would leave the BOM in the buffer and return. On the next call to this + function, our XmlPrologTok call would return XML_TOK_INVALID, since it + is not valid to have multiple BOMs. */ - else if (tok == XML_TOK_BOM && next == end - && ! parser->m_parsingStatus.finalBuffer) { -# ifdef XML_DTD + else if (tok == XML_TOK_BOM) { +# if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -4500,7 +4575,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, # endif *nextPtr = next; - return XML_ERROR_NONE; + s = next; } /* If we get this token, we have the start of what might be a normal tag, but not a declaration (i.e. it doesn't begin with @@ -4707,11 +4782,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc); -#ifdef XML_DTD +#if XML_GE == 1 switch (role) { case XML_ROLE_INSTANCE_START: // bytes accounted in contentProcessor case XML_ROLE_XML_DECL: // bytes accounted in processXmlDecl - case XML_ROLE_TEXT_DECL: // bytes accounted in processXmlDecl +# ifdef XML_DTD + case XML_ROLE_TEXT_DECL: // bytes accounted in processXmlDecl +# endif break; default: if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { @@ -5029,6 +5106,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_ENTITY_VALUE: if (dtd->keepProcessing) { +#if XML_GE == 1 + // This will store the given replacement text in + // parser->m_declEntity->textPtr. enum XML_Error result = storeEntityValue(parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, XML_ACCOUNT_NONE); @@ -5049,6 +5129,25 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolDiscard(&dtd->entityValuePool); if (result != XML_ERROR_NONE) return result; +#else + // This will store "&entity123;" in parser->m_declEntity->textPtr + // to end up as "&entity123;" in the handler. + if (parser->m_declEntity != NULL) { + const enum XML_Error result + = storeSelfEntityValue(parser, parser->m_declEntity); + if (result != XML_ERROR_NONE) + return result; + + if (parser->m_entityDeclHandler) { + *eventEndPP = s; + parser->m_entityDeclHandler( + parser->m_handlerArg, parser->m_declEntity->name, + parser->m_declEntity->is_param, parser->m_declEntity->textPtr, + parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + handleDefault = XML_FALSE; + } + } +#endif } break; case XML_ROLE_DOCTYPE_SYSTEM_ID: @@ -5107,6 +5206,16 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } break; case XML_ROLE_ENTITY_COMPLETE: +#if XML_GE == 0 + // This will store "&entity123;" in entity->textPtr + // to end up as "&entity123;" in the handler. + if (parser->m_declEntity != NULL) { + const enum XML_Error result + = storeSelfEntityValue(parser, parser->m_declEntity); + if (result != XML_ERROR_NONE) + return result; + } +#endif if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) { *eventEndPP = s; @@ -5648,7 +5757,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { const char *next = NULL; int tok = XmlPrologTok(parser->m_encoding, s, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -5728,7 +5837,7 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { return XML_ERROR_NO_MEMORY; } entity->open = XML_TRUE; -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif entity->processed = 0; @@ -5744,27 +5853,26 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { /* Set a safe default value in case 'next' does not get set */ next = textStart; -#ifdef XML_DTD if (entity->is_param) { int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok, next, &next, XML_FALSE, XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); - } else -#endif /* XML_DTD */ + } else { result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, textStart, textEnd, &next, XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); + } if (result == XML_ERROR_NONE) { if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { entity->processed = (int)(next - textStart); parser->m_processor = internalEntityProcessor; - } else { -#ifdef XML_DTD + } else if (parser->m_openInternalEntities->entity == entity) { +#if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ entity->open = XML_FALSE; parser->m_openInternalEntities = openEntity->next; /* put openEntity back in list of free instances */ @@ -5792,18 +5900,17 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, /* Set a safe default value in case 'next' does not get set */ next = textStart; -#ifdef XML_DTD if (entity->is_param) { int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok, next, &next, XML_FALSE, XML_TRUE, XML_ACCOUNT_ENTITY_EXPANSION); - } else -#endif /* XML_DTD */ + } else { result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding, textStart, textEnd, &next, XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); + } if (result != XML_ERROR_NONE) return result; @@ -5813,7 +5920,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return result; } -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); #endif entity->open = XML_FALSE; @@ -5830,7 +5937,6 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_NONE; } -#ifdef XML_DTD if (entity->is_param) { int tok; parser->m_processor = prologProcessor; @@ -5838,9 +5944,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, XML_ACCOUNT_DIRECT); - } else -#endif /* XML_DTD */ - { + } else { parser->m_processor = contentProcessor; /* see externalEntityContentProcessor vs contentProcessor */ result = doContent(parser, parser->m_parentParser ? 1 : 0, @@ -5892,7 +5996,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, const char *next = ptr; /* XmlAttributeValueTok doesn't always set the last arg */ int tok = XmlAttributeValueTok(enc, ptr, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, ptr, next, __LINE__, account)) { accountingOnAbort(parser); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; @@ -5957,14 +6061,14 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { -#ifdef XML_DTD +#if XML_GE == 1 /* NOTE: We are replacing 4-6 characters original input for 1 character * so there is no amplification and hence recording without * protection. */ accountingDiffTolerated(parser, tok, (char *)&ch, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ if (! poolAppendChar(pool, ch)) return XML_ERROR_NO_MEMORY; break; @@ -6042,14 +6146,14 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, enum XML_Error result; const XML_Char *textEnd = entity->textPtr + entity->textLen; entity->open = XML_TRUE; -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif result = appendAttributeValue(parser, parser->m_internalEncoding, isCdata, (const char *)entity->textPtr, (const char *)textEnd, pool, XML_ACCOUNT_ENTITY_EXPANSION); -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); #endif entity->open = XML_FALSE; @@ -6079,6 +6183,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, /* not reached */ } +#if XML_GE == 1 static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *entityTextPtr, const char *entityTextEnd, @@ -6086,12 +6191,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, DTD *const dtd = parser->m_dtd; /* save one level of indirection */ STRING_POOL *pool = &(dtd->entityValuePool); enum XML_Error result = XML_ERROR_NONE; -#ifdef XML_DTD +# ifdef XML_DTD int oldInEntityValue = parser->m_prologState.inEntityValue; parser->m_prologState.inEntityValue = 1; -#else +# else UNUSED_P(account); -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* never return Null for the value argument in EntityDeclHandler, since this would indicate an external entity; therefore we have to make sure that entityValuePool.start is not null */ @@ -6105,18 +6210,16 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ int tok = XmlEntityValueTok(enc, entityTextPtr, entityTextEnd, &next); -#ifdef XML_DTD if (! accountingDiffTolerated(parser, tok, entityTextPtr, next, __LINE__, account)) { accountingOnAbort(parser); result = XML_ERROR_AMPLIFICATION_LIMIT_BREACH; goto endEntityValue; } -#endif switch (tok) { case XML_TOK_PARAM_ENTITY_REF: -#ifdef XML_DTD +# ifdef XML_DTD if (parser->m_isParamEntity || enc != parser->m_encoding) { const XML_Char *name; ENTITY *entity; @@ -6139,7 +6242,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->keepProcessing = dtd->standalone; goto endEntityValue; } - if (entity->open) { + if (entity->open || (entity == parser->m_declEntity)) { if (enc == parser->m_encoding) parser->m_eventPtr = entityTextPtr; result = XML_ERROR_RECURSIVE_ENTITY_REF; @@ -6178,7 +6281,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } break; } -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* In the internal subset, PE references are not legal within markup declarations, e.g entity values in this case. */ parser->m_eventPtr = entityTextPtr; @@ -6259,12 +6362,38 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, entityTextPtr = next; } endEntityValue: -#ifdef XML_DTD +# ifdef XML_DTD parser->m_prologState.inEntityValue = oldInEntityValue; -#endif /* XML_DTD */ +# endif /* XML_DTD */ return result; } +#else /* XML_GE == 0 */ + +static enum XML_Error +storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { + // This will store "&entity123;" in entity->textPtr + // to end up as "&entity123;" in the handler. + const char *const entity_start = "&"; + const char *const entity_end = ";"; + + STRING_POOL *const pool = &(parser->m_dtd->entityValuePool); + if (! poolAppendString(pool, entity_start) + || ! poolAppendString(pool, entity->name) + || ! poolAppendString(pool, entity_end)) { + poolDiscard(pool); + return XML_ERROR_NO_MEMORY; + } + + entity->textPtr = poolStart(pool); + entity->textLen = (int)(poolLength(pool)); + poolFinish(pool); + + return XML_ERROR_NONE; +} + +#endif /* XML_GE == 0 */ + static void FASTCALL normalizeLines(XML_Char *s) { XML_Char *p; @@ -6375,8 +6504,9 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); } else - parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)end - (XML_Char *)s)); + parser->m_defaultHandler( + parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)end - (const XML_Char *)s)); } static int @@ -6480,7 +6610,7 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, name = poolStoreString(&dtd->pool, enc, start, end); if (! name) return NULL; - /* skip quotation mark - its storage will be re-used (like in name[-1]) */ + /* skip quotation mark - its storage will be reused (like in name[-1]) */ ++name; id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name, sizeof(ATTRIBUTE_ID)); @@ -6630,6 +6760,10 @@ getContext(XML_Parser parser) { static XML_Bool setContext(XML_Parser parser, const XML_Char *context) { + if (context == NULL) { + return XML_FALSE; + } + DTD *const dtd = parser->m_dtd; /* save one level of indirection */ const XML_Char *s = context; @@ -6884,6 +7018,16 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (! newE) return 0; if (oldE->nDefaultAtts) { + /* Detect and prevent integer overflow. + * The preprocessor guard addresses the "always false" warning + * from -Wtype-limits on platforms where + * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ +#if UINT_MAX >= SIZE_MAX + if ((size_t)oldE->nDefaultAtts + > ((size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE))) { + return 0; + } +#endif newE->defaultAtts = ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { @@ -7220,7 +7364,7 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, return NULL; for (;;) { const enum XML_Convert_Result convert_res = XmlConvert( - enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end); + enc, &ptr, end, (ICHAR **)&(pool->ptr), (const ICHAR *)pool->end); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; @@ -7426,6 +7570,15 @@ nextScaffoldPart(XML_Parser parser) { int next; if (! dtd->scaffIndex) { + /* Detect and prevent integer overflow. + * The preprocessor guard addresses the "always false" warning + * from -Wtype-limits on platforms where + * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ +#if UINT_MAX >= SIZE_MAX + if (parser->m_groupSize > ((size_t)(-1) / sizeof(int))) { + return -1; + } +#endif dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; @@ -7651,10 +7804,12 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { return result; } -#ifdef XML_DTD +#if XML_GE == 1 static float accountingGetCurrentAmplification(XML_Parser rootParser) { + // 1.........1.........12 => 22 + const size_t lenOfShortestInclude = sizeof("") - 1; const XmlBigCount countBytesOutput = rootParser->m_accounting.countBytesDirect + rootParser->m_accounting.countBytesIndirect; @@ -7662,7 +7817,9 @@ accountingGetCurrentAmplification(XML_Parser rootParser) { = rootParser->m_accounting.countBytesDirect ? (countBytesOutput / (float)(rootParser->m_accounting.countBytesDirect)) - : 1.0f; + : ((lenOfShortestInclude + + rootParser->m_accounting.countBytesIndirect) + / (float)lenOfShortestInclude); assert(! rootParser->m_parentParser); return amplificationFactor; } @@ -7672,7 +7829,7 @@ accountingReportStats(XML_Parser originParser, const char *epilog) { const XML_Parser rootParser = getRootParserOf(originParser, NULL); assert(! rootParser->m_parentParser); - if (rootParser->m_accounting.debugLevel < 1) { + if (rootParser->m_accounting.debugLevel == 0u) { return; } @@ -7709,7 +7866,7 @@ accountingReportDiff(XML_Parser rootParser, /* Note: Performance is of no concern here */ const char *walker = before; - if ((rootParser->m_accounting.debugLevel >= 3) + if ((rootParser->m_accounting.debugLevel >= 3u) || (after - before) <= (ptrdiff_t)(contextLength + ellipsisLength + contextLength)) { for (; walker < after; walker++) { @@ -7774,7 +7931,7 @@ accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, || (amplificationFactor <= rootParser->m_accounting.maximumAmplificationFactor); - if (rootParser->m_accounting.debugLevel >= 2) { + if (rootParser->m_accounting.debugLevel >= 2u) { accountingReportStats(rootParser, ""); accountingReportDiff(rootParser, levelsAwayFromRootParser, before, after, bytesMore, source_line, account); @@ -7801,7 +7958,7 @@ static void entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, const char *action, int sourceLine) { assert(! rootParser->m_parentParser); - if (rootParser->m_entity_stats.debugLevel < 1) + if (rootParser->m_entity_stats.debugLevel == 0u) return; # if defined(XML_UNICODE) @@ -8382,7 +8539,7 @@ unsignedCharToPrintable(unsigned char c) { assert(0); /* never gets here */ } -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel) { @@ -8393,9 +8550,9 @@ getDebugLevel(const char *variableName, unsigned long defaultDebugLevel) { const char *const value = valueOrNull; errno = 0; - char *afterValue = (char *)value; + char *afterValue = NULL; unsigned long debugLevel = strtoul(value, &afterValue, 10); - if ((errno != 0) || (afterValue[0] != '\0')) { + if ((errno != 0) || (afterValue == value) || (afterValue[0] != '\0')) { errno = 0; return defaultDebugLevel; } diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 3f0f5c150c6278..2c48bf40867953 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -12,10 +12,10 @@ Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2021 Sebastian Pipping + Copyright (c) 2016-2023 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2019 David Loffredo - Copyright (c) 2021 Dong-hee Na + Copyright (c) 2021 Donghee Na Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -38,7 +38,7 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include +#include "expat_config.h" #include diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index d6e1fa150a108a..a7904274c91d4e 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Karl Waclawek Copyright (c) 2002 Fred L. Drake, Jr. - Copyright (c) 2017 Sebastian Pipping + Copyright (c) 2017-2024 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -127,9 +127,9 @@ typedef struct prolog_state { #endif /* XML_DTD */ } PROLOG_STATE; -void XmlPrologStateInit(PROLOG_STATE *); +void XmlPrologStateInit(PROLOG_STATE *state); #ifdef XML_DTD -void XmlPrologStateInitExternalEntity(PROLOG_STATE *); +void XmlPrologStateInitExternalEntity(PROLOG_STATE *state); #endif /* XML_DTD */ #define XmlTokenRole(state, tok, ptr, end, enc) \ diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 2b7012a58be419..29a66d72ceea5e 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2016 Pascal Cuoq Copyright (c) 2016 Don Lewis Copyright (c) 2017 Rhodri James @@ -20,8 +20,10 @@ Copyright (c) 2017 Benbuck Nason Copyright (c) 2017 José Gutiérrez de la Concha Copyright (c) 2019 David Loffredo - Copyright (c) 2021 Dong-hee Na + Copyright (c) 2021 Donghee Na Copyright (c) 2022 Martin Ettl + Copyright (c) 2022 Sean McBride + Copyright (c) 2023 Hanno Böck Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -44,7 +46,7 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include +#include "expat_config.h" #include #include /* memcpy */ @@ -76,7 +78,7 @@ #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16) #define UCS2_GET_NAMING(pages, hi, lo) \ - (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo)&0x1F))) + (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F))) /* A 2 byte UTF-8 representation splits the characters 11 bits between the bottom 5 and 6 bits of the bytes. We need 8 bits to index into @@ -100,7 +102,7 @@ & (1u << (((byte)[2]) & 0x1F))) /* Detection of invalid UTF-8 sequences is based on Table 3.1B - of Unicode 3.2: http://www.unicode.org/unicode/reports/tr28/ + of Unicode 3.2: https://www.unicode.org/unicode/reports/tr28/ with the additional restriction of not allowing the Unicode code points 0xFFFF and 0xFFFE (sequences EF,BF,BF and EF,BF,BE). Implementation details: @@ -225,7 +227,7 @@ struct normal_encoding { /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \ /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL -static int FASTCALL checkCharRefNumber(int); +static int FASTCALL checkCharRefNumber(int result); #include "xmltok_impl.h" #include "ascii.h" @@ -243,7 +245,7 @@ static int FASTCALL checkCharRefNumber(int); #endif #define SB_BYTE_TYPE(enc, p) \ - (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) + (((const struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) #ifdef XML_MIN_SIZE static int PTRFASTCALL @@ -407,7 +409,7 @@ utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short *to = *toP; const char *from = *fromP; while (from < fromLim && to < toLim) { - switch (((struct normal_encoding *)enc)->type[(unsigned char)*from]) { + switch (SB_BYTE_TYPE(enc, from)) { case BT_LEAD2: if (fromLim - from < 2) { res = XML_CONVERT_INPUT_INCOMPLETE; @@ -715,31 +717,26 @@ unicode_byte_type(char hi, char lo) { return res; \ } -#define SET2(ptr, ch) (((ptr)[0] = ((ch)&0xff)), ((ptr)[1] = ((ch) >> 8))) #define GET_LO(ptr) ((unsigned char)(ptr)[0]) #define GET_HI(ptr) ((unsigned char)(ptr)[1]) DEFINE_UTF16_TO_UTF8(little2_) DEFINE_UTF16_TO_UTF16(little2_) -#undef SET2 #undef GET_LO #undef GET_HI -#define SET2(ptr, ch) (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch)&0xFF))) #define GET_LO(ptr) ((unsigned char)(ptr)[1]) #define GET_HI(ptr) ((unsigned char)(ptr)[0]) DEFINE_UTF16_TO_UTF8(big2_) DEFINE_UTF16_TO_UTF16(big2_) -#undef SET2 #undef GET_LO #undef GET_HI #define LITTLE2_BYTE_TYPE(enc, p) \ - ((p)[1] == 0 ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \ - : unicode_byte_type((p)[1], (p)[0])) + ((p)[1] == 0 ? SB_BYTE_TYPE(enc, p) : unicode_byte_type((p)[1], (p)[0])) #define LITTLE2_BYTE_TO_ASCII(p) ((p)[1] == 0 ? (p)[0] : -1) #define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == (c)) #define LITTLE2_IS_NAME_CHAR_MINBPC(p) \ @@ -872,9 +869,7 @@ static const struct normal_encoding internal_little2_encoding #endif #define BIG2_BYTE_TYPE(enc, p) \ - ((p)[0] == 0 \ - ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \ - : unicode_byte_type((p)[0], (p)[1])) + ((p)[0] == 0 ? SB_BYTE_TYPE(enc, p + 1) : unicode_byte_type((p)[0], (p)[1])) #define BIG2_BYTE_TO_ASCII(p) ((p)[0] == 0 ? (p)[1] : -1) #define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == (c)) #define BIG2_IS_NAME_CHAR_MINBPC(p) \ diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index 6f630c2f9ba96d..c51fce1ec1518b 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2005 Karl Waclawek - Copyright (c) 2016-2017 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2017 Rhodri James Licensed under the MIT license: @@ -289,7 +289,8 @@ int XmlParseXmlDecl(int isGeneralTextEntity, const ENCODING *enc, const char **encodingNamePtr, const ENCODING **namedEncodingPtr, int *standalonePtr); -int XmlInitEncoding(INIT_ENCODING *, const ENCODING **, const char *name); +int XmlInitEncoding(INIT_ENCODING *p, const ENCODING **encPtr, + const char *name); const ENCODING *XmlGetUtf8InternalEncoding(void); const ENCODING *XmlGetUtf16InternalEncoding(void); int FASTCALL XmlUtf8Encode(int charNumber, char *buf); @@ -307,7 +308,8 @@ int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc, const char **encodingNamePtr, const ENCODING **namedEncodingPtr, int *standalonePtr); -int XmlInitEncodingNS(INIT_ENCODING *, const ENCODING **, const char *name); +int XmlInitEncodingNS(INIT_ENCODING *p, const ENCODING **encPtr, + const char *name); const ENCODING *XmlGetUtf8InternalEncodingNS(void); const ENCODING *XmlGetUtf16InternalEncodingNS(void); ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 1971d74bf8c91f..239a2d06c4512c 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -126,7 +126,7 @@ # endif # define HAS_CHARS(enc, ptr, end, count) \ - ((end) - (ptr) >= ((count)*MINBPC(enc))) + ((end) - (ptr) >= ((count) * MINBPC(enc))) # define HAS_CHAR(enc, ptr, end) HAS_CHARS(enc, ptr, end, 1) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 95d646c9c65b3c..b62362f277797e 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -4,6 +4,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // Py_NSIG #include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include "pycore_traceback.h" // _Py_DumpTracebackThreads #ifdef HAVE_UNISTD_H @@ -74,7 +75,7 @@ static fault_handler_t faulthandler_handlers[] = { #ifdef SIGILL {SIGILL, 0, "Illegal instruction", }, #endif - {SIGFPE, 0, "Floating point exception", }, + {SIGFPE, 0, "Floating-point exception", }, {SIGABRT, 0, "Aborted", }, /* define SIGSEGV at the end to make it the default choice if searching the handler fails in faulthandler_fatal_error() */ @@ -623,7 +624,7 @@ cancel_dump_traceback_later(void) #define SEC_TO_US (1000 * 1000) static char* -format_timeout(_PyTime_t us) +format_timeout(PyTime_t us) { unsigned long sec, min, hour; char buffer[100]; @@ -656,7 +657,7 @@ faulthandler_dump_traceback_later(PyObject *self, { static char *kwlist[] = {"timeout", "repeat", "file", "exit", NULL}; PyObject *timeout_obj; - _PyTime_t timeout, timeout_us; + PyTime_t timeout, timeout_us; int repeat = 0; PyObject *file = NULL; int fd; @@ -1191,58 +1192,67 @@ PyDoc_STRVAR(module_doc, static PyMethodDef module_methods[] = { {"enable", _PyCFunction_CAST(faulthandler_py_enable), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("enable(file=sys.stderr, all_threads=True): " - "enable the fault handler")}, + PyDoc_STR("enable($module, /, file=sys.stderr, all_threads=True)\n--\n\n" + "Enable the fault handler.")}, {"disable", faulthandler_disable_py, METH_NOARGS, - PyDoc_STR("disable(): disable the fault handler")}, + PyDoc_STR("disable($module, /)\n--\n\n" + "Disable the fault handler.")}, {"is_enabled", faulthandler_is_enabled, METH_NOARGS, - PyDoc_STR("is_enabled()->bool: check if the handler is enabled")}, + PyDoc_STR("is_enabled($module, /)\n--\n\n" + "Check if the handler is enabled.")}, {"dump_traceback", _PyCFunction_CAST(faulthandler_dump_traceback_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("dump_traceback(file=sys.stderr, all_threads=True): " - "dump the traceback of the current thread, or of all threads " - "if all_threads is True, into file")}, + PyDoc_STR("dump_traceback($module, /, file=sys.stderr, all_threads=True)\n--\n\n" + "Dump the traceback of the current thread, or of all threads " + "if all_threads is True, into file.")}, {"dump_traceback_later", _PyCFunction_CAST(faulthandler_dump_traceback_later), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False):\n" - "dump the traceback of all threads in timeout seconds,\n" + PyDoc_STR("dump_traceback_later($module, /, timeout, repeat=False, file=sys.stderr, exit=False)\n--\n\n" + "Dump the traceback of all threads in timeout seconds,\n" "or each timeout seconds if repeat is True. If exit is True, " "call _exit(1) which is not safe.")}, {"cancel_dump_traceback_later", faulthandler_cancel_dump_traceback_later_py, METH_NOARGS, - PyDoc_STR("cancel_dump_traceback_later():\ncancel the previous call " - "to dump_traceback_later().")}, + PyDoc_STR("cancel_dump_traceback_later($module, /)\n--\n\n" + "Cancel the previous call to dump_traceback_later().")}, #ifdef FAULTHANDLER_USER {"register", _PyCFunction_CAST(faulthandler_register_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("register(signum, file=sys.stderr, all_threads=True, chain=False): " - "register a handler for the signal 'signum': dump the " + PyDoc_STR("register($module, /, signum, file=sys.stderr, all_threads=True, chain=False)\n--\n\n" + "Register a handler for the signal 'signum': dump the " "traceback of the current thread, or of all threads if " - "all_threads is True, into file")}, + "all_threads is True, into file.")}, {"unregister", - _PyCFunction_CAST(faulthandler_unregister_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("unregister(signum): unregister the handler of the signal " - "'signum' registered by register()")}, + _PyCFunction_CAST(faulthandler_unregister_py), METH_VARARGS, + PyDoc_STR("unregister($module, signum, /)\n--\n\n" + "Unregister the handler of the signal " + "'signum' registered by register().")}, #endif {"_read_null", faulthandler_read_null, METH_NOARGS, - PyDoc_STR("_read_null(): read from NULL, raise " - "a SIGSEGV or SIGBUS signal depending on the platform")}, + PyDoc_STR("_read_null($module, /)\n--\n\n" + "Read from NULL, raise " + "a SIGSEGV or SIGBUS signal depending on the platform.")}, {"_sigsegv", faulthandler_sigsegv, METH_VARARGS, - PyDoc_STR("_sigsegv(release_gil=False): raise a SIGSEGV signal")}, + PyDoc_STR("_sigsegv($module, release_gil=False, /)\n--\n\n" + "Raise a SIGSEGV signal.")}, {"_fatal_error_c_thread", faulthandler_fatal_error_c_thread, METH_NOARGS, - PyDoc_STR("fatal_error_c_thread(): " - "call Py_FatalError() in a new C thread.")}, + PyDoc_STR("_fatal_error_c_thread($module, /)\n--\n\n" + "Call Py_FatalError() in a new C thread.")}, {"_sigabrt", faulthandler_sigabrt, METH_NOARGS, - PyDoc_STR("_sigabrt(): raise a SIGABRT signal")}, + PyDoc_STR("_sigabrt($module, /)\n--\n\n" + "Raise a SIGABRT signal.")}, {"_sigfpe", (PyCFunction)faulthandler_sigfpe, METH_NOARGS, - PyDoc_STR("_sigfpe(): raise a SIGFPE signal")}, + PyDoc_STR("_sigfpe($module, /)\n--\n\n" + "Raise a SIGFPE signal.")}, #ifdef FAULTHANDLER_STACK_OVERFLOW {"_stack_overflow", faulthandler_stack_overflow, METH_NOARGS, - PyDoc_STR("_stack_overflow(): recursive call to raise a stack overflow")}, + PyDoc_STR("_stack_overflow($module, /)\n--\n\n" + "Recursive call to raise a stack overflow.")}, #endif #ifdef MS_WINDOWS {"_raise_exception", faulthandler_raise_exception, METH_VARARGS, - PyDoc_STR("raise_exception(code, flags=0): Call RaiseException(code, flags).")}, + PyDoc_STR("_raise_exception($module, code, flags=0, /)\n--\n\n" + "Call RaiseException(code, flags).")}, #endif {NULL, NULL} /* sentinel */ }; @@ -1282,6 +1292,7 @@ static PyModuleDef_Slot faulthandler_slots[] = { {Py_mod_exec, PyExec_faulthandler}, // XXX gh-103092: fix isolation. //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 0d16602692b62d..90ebfd7e99a777 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -1,22 +1,25 @@ /* fcntl module */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyLong_AsInt() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" +#include // EINTR +#include // fcntl() +#include // memcpy() +#include // ioctl() #ifdef HAVE_SYS_FILE_H -#include +# include // flock() #endif #ifdef HAVE_LINUX_FS_H -#include +# include #endif - -#include -#include #ifdef HAVE_STROPTS_H -#include +# include // I_FLUSHBAND #endif /*[clinic input] @@ -109,7 +112,7 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) fcntl.ioctl fd: fildes - request as code: unsigned_int(bitwise=True) + request as code: unsigned_long(bitwise=True) arg as ob_arg: object(c_default='NULL') = 0 mutate_flag as mutate_arg: bool = True / @@ -145,9 +148,9 @@ code. [clinic start generated code]*/ static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg) -/*[clinic end generated code: output=7f7f5840c65991be input=967b4a4cbeceb0a8]*/ +/*[clinic end generated code: output=3d8eb6828666cea1 input=cee70f6a27311e58]*/ { #define IOCTL_BUFSZ 1024 /* We use the unsigned non-checked 'I' format for the 'code' parameter @@ -167,7 +170,7 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, Py_ssize_t len; char buf[IOCTL_BUFSZ+1]; /* argument plus NUL byte */ - if (PySys_Audit("fcntl.ioctl", "iIO", fd, code, + if (PySys_Audit("fcntl.ioctl", "ikO", fd, code, ob_arg ? ob_arg : Py_None) < 0) { return NULL; } @@ -577,12 +580,17 @@ all_ins(PyObject* m) #ifdef F_GETPIPE_SZ if (PyModule_AddIntMacro(m, F_GETPIPE_SZ)) return -1; #endif + +/* On Android, FICLONE is blocked by SELinux. */ +#ifndef __ANDROID__ #ifdef FICLONE if (PyModule_AddIntMacro(m, FICLONE)) return -1; #endif #ifdef FICLONERANGE if (PyModule_AddIntMacro(m, FICLONERANGE)) return -1; #endif +#endif + #ifdef F_GETOWN_EX // since Linux 2.6.32 if (PyModule_AddIntMacro(m, F_GETOWN_EX)) return -1; @@ -742,6 +750,7 @@ fcntl_exec(PyObject *module) static PyModuleDef_Slot fcntl_slots[] = { {Py_mod_exec, fcntl_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index a2b66b9b78c169..f5fea5aa4dde08 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -8,6 +8,7 @@ #include "pycore_gc.h" #include "pycore_object.h" // _PyObject_IS_GC() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_tuple.h" // _PyTuple_FromArray() typedef struct _gc_runtime_state GCState; @@ -158,17 +159,12 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, { GCState *gcstate = get_gc_state(); - gcstate->generations[0].threshold = threshold0; + gcstate->young.threshold = threshold0; if (group_right_1) { - gcstate->generations[1].threshold = threshold1; + gcstate->old[0].threshold = threshold1; } if (group_right_2) { - gcstate->generations[2].threshold = threshold2; - - /* generations higher than 2 get the same threshold */ - for (int i = 3; i < NUM_GENERATIONS; i++) { - gcstate->generations[i].threshold = gcstate->generations[2].threshold; - } + gcstate->old[1].threshold = threshold2; } Py_RETURN_NONE; } @@ -185,9 +181,9 @@ gc_get_threshold_impl(PyObject *module) { GCState *gcstate = get_gc_state(); return Py_BuildValue("(iii)", - gcstate->generations[0].threshold, - gcstate->generations[1].threshold, - gcstate->generations[2].threshold); + gcstate->young.threshold, + gcstate->old[0].threshold, + 0); } /*[clinic input] @@ -201,10 +197,20 @@ gc_get_count_impl(PyObject *module) /*[clinic end generated code: output=354012e67b16398f input=a392794a08251751]*/ { GCState *gcstate = get_gc_state(); + +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct _gc_thread_state *gc = &tstate->gc; + + // Flush the local allocation count to the global count + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + gc->alloc_count = 0; +#endif + return Py_BuildValue("(iii)", - gcstate->generations[0].count, - gcstate->generations[1].count, - gcstate->generations[2].count); + gcstate->young.count, + gcstate->old[gcstate->visited_space].count, + gcstate->old[gcstate->visited_space^1].count); } /*[clinic input] @@ -216,15 +222,25 @@ Return the list of objects that directly refer to any of 'objs'. [clinic start generated code]*/ static PyObject * -gc_get_referrers_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=296a09587f6a86b5 input=bae96961b14a0922]*/ +gc_get_referrers_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args) +/*[clinic end generated code: output=1d44a7695ea25c40 input=bae96961b14a0922]*/ { - if (PySys_Audit("gc.get_referrers", "(O)", args) < 0) { + PyObject *varargs = _PyTuple_FromArray(args, nargs); + + if (!varargs) { + return NULL; + } + if (PySys_Audit("gc.get_referrers", "(O)", varargs) < 0) { + Py_DECREF(varargs); return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); - return _PyGC_GetReferrers(interp, args); + PyObject *result = _PyGC_GetReferrers(interp, varargs); + + Py_DECREF(varargs); + return result; } /* Append obj to list; return true if error (out of memory), false if OK. */ @@ -264,27 +280,37 @@ Return the list of objects that are directly referred to by 'objs'. [clinic start generated code]*/ static PyObject * -gc_get_referents_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=d47dc02cefd06fe8 input=b3ceab0c34038cbf]*/ +gc_get_referents_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args) +/*[clinic end generated code: output=e459f3e8c0d19311 input=b3ceab0c34038cbf]*/ { - if (PySys_Audit("gc.get_referents", "(O)", args) < 0) { + PyObject *varargs = _PyTuple_FromArray(args, nargs); + + if (!varargs) { + return NULL; + } + if (PySys_Audit("gc.get_referents", "(O)", varargs) < 0) { + Py_DECREF(varargs); return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *result = PyList_New(0); - if (result == NULL) + if (result == NULL) { + Py_DECREF(varargs); return NULL; + } // NOTE: stop the world is a no-op in default build _PyEval_StopTheWorld(interp); - int err = append_referrents(result, args); + int err = append_referrents(result, varargs); _PyEval_StartTheWorld(interp); if (err < 0) { Py_CLEAR(result); } + Py_DECREF(varargs); return result; } @@ -321,7 +347,7 @@ gc_get_objects_impl(PyObject *module, Py_ssize_t generation) } PyInterpreterState *interp = _PyInterpreterState_GET(); - return _PyGC_GetObjects(interp, generation); + return _PyGC_GetObjects(interp, (int)generation); } /*[clinic input] @@ -373,7 +399,7 @@ gc_get_stats_impl(PyObject *module) /*[clinic input] -gc.is_tracked +gc.is_tracked -> bool obj: object / @@ -383,21 +409,15 @@ Returns true if the object is tracked by the garbage collector. Simple atomic objects will return false. [clinic start generated code]*/ -static PyObject * -gc_is_tracked(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=14f0103423b28e31 input=d83057f170ea2723]*/ +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=91c8d086b7f47a33 input=423b98ec680c3126]*/ { - PyObject *result; - - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) - result = Py_True; - else - result = Py_False; - return Py_NewRef(result); + return PyObject_GC_IsTracked(obj); } /*[clinic input] -gc.is_finalized +gc.is_finalized -> bool obj: object / @@ -405,14 +425,11 @@ gc.is_finalized Returns true if the object has been already finalized by the GC. [clinic start generated code]*/ -static PyObject * -gc_is_finalized(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=e1516ac119a918ed input=201d0c58f69ae390]*/ +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=401ff5d6fc660429 input=ca4d111c8f8c4e3a]*/ { - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return PyObject_GC_IsFinalized(obj); } /*[clinic input] @@ -539,6 +556,7 @@ gcmodule_exec(PyObject *module) static PyModuleDef_Slot gcmodule_slots[] = { {Py_mod_exec, gcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/getpath.c b/Modules/getpath.c index abed139028244a..18ddfaf8dbce1a 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -108,7 +108,7 @@ getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args) Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); if (pos < 0) { - return PyUnicode_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } return PyUnicode_Substring(path, 0, pos); } @@ -258,7 +258,7 @@ getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t n = PyTuple_GET_SIZE(args); if (n == 0) { - return PyUnicode_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } /* Convert all parts to wchar and accumulate max final length */ wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *)); @@ -302,7 +302,7 @@ getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) PyErr_NoMemory(); return NULL; } - return PyUnicode_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } final[0] = '\0'; @@ -951,6 +951,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !wchar_to_dict(dict, "executable_dir", NULL) || !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) || !funcs_to_dict(dict, config->pathconfig_warnings) || +#ifdef Py_GIL_DISABLED + !decode_to_dict(dict, "ABI_THREAD", "t") || +#else + !decode_to_dict(dict, "ABI_THREAD", "") || +#endif #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || #endif diff --git a/Modules/getpath.py b/Modules/getpath.py index 1410ffdbed8c70..1f1bfcb4f64dd4 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -40,6 +40,7 @@ # EXE_SUFFIX -- [in, opt] '.exe' on Windows/Cygwin/similar # VERSION_MAJOR -- [in] sys.version_info.major # VERSION_MINOR -- [in] sys.version_info.minor +# ABI_THREAD -- [in] either 't' for free-threaded builds or '' # PYWINVER -- [in] the Windows platform-specific version (e.g. 3.8-32) # ** Values read from the environment ** @@ -172,17 +173,18 @@ # ****************************************************************************** platlibdir = config.get('platlibdir') or PLATLIBDIR +ABI_THREAD = ABI_THREAD or '' if os_name == 'posix' or os_name == 'darwin': BUILDDIR_TXT = 'pybuilddir.txt' BUILD_LANDMARK = 'Modules/Setup.local' DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' - STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}' STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] - PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}/lib-dynload' BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] VENV_LANDMARK = 'pyvenv.cfg' - ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}{ABI_THREAD}.zip' DELIM = ':' SEP = '/' @@ -310,7 +312,10 @@ def search_up(prefix, *landmarks, test=isfile): # and should not affect base_executable. base_executable = f"{dirname(library)}/bin/python{VERSION_MAJOR}.{VERSION_MINOR}" else: - base_executable = executable + # Use the real executable as our base, or argv[0] otherwise + # (on Windows, argv[0] is likely to be ENV___PYVENV_LAUNCHER__; on + # other platforms, real_executable is likely to be empty) + base_executable = real_executable or executable if not real_executable: real_executable = base_executable @@ -408,13 +413,14 @@ def search_up(prefix, *landmarks, test=isfile): if not real_executable: real_executable = base_executable -try: - real_executable = realpath(real_executable) -except OSError as ex: - # Only warn if the file actually exists and was unresolvable - # Otherwise users who specify a fake executable may get spurious warnings. - if isfile(real_executable): - warn(f'Failed to find real location of {base_executable}') +if real_executable: + try: + real_executable = realpath(real_executable) + except OSError as ex: + # Only warn if the file actually exists and was unresolvable + # Otherwise users who specify a fake executable may get spurious warnings. + if isfile(real_executable): + warn(f'Failed to find real location of {base_executable}') if not executable_dir and os_name == 'darwin' and library: # QUIRK: macOS checks adjacent to its library early @@ -427,12 +433,12 @@ def search_up(prefix, *landmarks, test=isfile): # If we do not have the executable's directory, we can calculate it. # This is the directory used to find prefix/exec_prefix if necessary. -if not executable_dir: +if not executable_dir and real_executable: executable_dir = real_executable_dir = dirname(real_executable) # If we do not have the real executable's directory, we calculate it. # This is the directory used to detect build layouts. -if not real_executable_dir: +if not real_executable_dir and real_executable: real_executable_dir = dirname(real_executable) # ****************************************************************************** diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 9756f1c2fb15b1..f7d3e12f347ec2 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -1,15 +1,17 @@ - /* UNIX group file access module */ -// clinic/grpmodule.c.h uses internal pycore_modsupport.h API -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyMem_RawRealloc() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +#define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" #include "posixmodule.h" +#include // ERANGE #include // getgrgid_r() +#include // memcpy() #include // sysconf() #include "clinic/grpmodule.c.h" @@ -88,7 +90,7 @@ mkgrent(PyObject *module, struct group *p) Py_DECREF(x); } -#define SET(i,val) PyStructSequence_SET_ITEM(v, i, val) +#define SET(i,val) PyStructSequence_SetItem(v, i, val) SET(setIndex++, PyUnicode_DecodeFSDefault(p->gr_name)); if (p->gr_passwd) SET(setIndex++, PyUnicode_DecodeFSDefault(p->gr_passwd)); @@ -340,6 +342,7 @@ grpmodule_exec(PyObject *module) static PyModuleDef_Slot grpmodule_slots[] = { {Py_mod_exec, grpmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 164741495c7baf..1201fa094902d7 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,13 +1,14 @@ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_tuple.h" // _PyTuple_ITEMS() -#include // offsetof() +#include // offsetof() /* Itertools module written and maintained by Raymond D. Hettinger @@ -93,15 +94,6 @@ class itertools.pairwise "pairwiseobject *" "clinic_state()->pairwise_type" #undef clinic_state_by_cls #undef clinic_state -/* Deprecation of pickle support: GH-101588 *********************************/ - -#define ITERTOOL_PICKLE_DEPRECATION \ - if (PyErr_WarnEx( \ - PyExc_DeprecationWarning, \ - "Pickle, copy, and deepcopy support will be " \ - "removed from itertools in Python 3.14.", 1) < 0) { \ - return NULL; \ - } /* batched object ************************************************************/ @@ -270,6 +262,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *old; + PyObject *result; } pairwiseobject; /*[clinic input] @@ -301,6 +294,11 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) } po->it = it; po->old = NULL; + po->result = PyTuple_Pack(2, Py_None, Py_None); + if (po->result == NULL) { + Py_DECREF(po); + return NULL; + } return (PyObject *)po; } @@ -311,6 +309,7 @@ pairwise_dealloc(pairwiseobject *po) PyObject_GC_UnTrack(po); Py_XDECREF(po->it); Py_XDECREF(po->old); + Py_XDECREF(po->result); tp->tp_free(po); Py_DECREF(tp); } @@ -321,6 +320,7 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) Py_VISIT(Py_TYPE(po)); Py_VISIT(po->it); Py_VISIT(po->old); + Py_VISIT(po->result); return 0; } @@ -355,8 +355,30 @@ pairwise_next(pairwiseobject *po) Py_DECREF(old); return NULL; } - /* Future optimization: Reuse the result tuple as we do in enumerate() */ - result = PyTuple_Pack(2, old, new); + + result = po->result; + if (Py_REFCNT(result) == 1) { + Py_INCREF(result); + PyObject *last_old = PyTuple_GET_ITEM(result, 0); + PyObject *last_new = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + Py_DECREF(last_old); + Py_DECREF(last_new); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result != NULL) { + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + } + } + Py_XSETREF(po->old, new); Py_DECREF(old); return result; @@ -524,57 +546,6 @@ groupby_next(groupbyobject *gbo) return r; } -static PyObject * -groupby_reduce(groupbyobject *lz, PyObject *Py_UNUSED(ignored)) -{ - /* reduce as a 'new' call with an optional 'setstate' if groupby - * has started - */ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *value; - if (lz->tgtkey && lz->currkey && lz->currvalue) - value = Py_BuildValue("O(OO)(OOO)", Py_TYPE(lz), - lz->it, lz->keyfunc, lz->currkey, lz->currvalue, lz->tgtkey); - else - value = Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->it, lz->keyfunc); - - return value; -} - -PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); - -static PyObject * -groupby_setstate(groupbyobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *currkey, *currvalue, *tgtkey; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "OOO", &currkey, &currvalue, &tgtkey)) { - return NULL; - } - Py_INCREF(currkey); - Py_XSETREF(lz->currkey, currkey); - Py_INCREF(currvalue); - Py_XSETREF(lz->currvalue, currvalue); - Py_INCREF(tgtkey); - Py_XSETREF(lz->tgtkey, tgtkey); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); - -static PyMethodDef groupby_methods[] = { - {"__reduce__", (PyCFunction)groupby_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)groupby_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot groupby_slots[] = { {Py_tp_dealloc, groupby_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -582,7 +553,6 @@ static PyType_Slot groupby_slots[] = { {Py_tp_traverse, groupby_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, groupby_next}, - {Py_tp_methods, groupby_methods}, {Py_tp_new, itertools_groupby}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -683,29 +653,12 @@ _grouper_next(_grouperobject *igo) return r; } -static PyObject * -_grouper_reduce(_grouperobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (((groupbyobject *)lz->parent)->currgrouper != lz) { - return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter))); - } - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->parent, lz->tgtkey); -} - -static PyMethodDef _grouper_methods[] = { - {"__reduce__", (PyCFunction)_grouper_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot _grouper_slots[] = { {Py_tp_dealloc, _grouper_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, _grouper_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, _grouper_next}, - {Py_tp_methods, _grouper_methods}, {Py_tp_new, itertools__grouper}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -815,10 +768,9 @@ teedataobject_traverse(teedataobject *tdo, visitproc visit, void * arg) } static void -teedataobject_safe_decref(PyObject *obj, PyTypeObject *tdo_type) +teedataobject_safe_decref(PyObject *obj) { - while (obj && Py_IS_TYPE(obj, tdo_type) && - Py_REFCNT(obj) == 1) { + while (obj && Py_REFCNT(obj) == 1) { PyObject *nextlink = ((teedataobject *)obj)->nextlink; ((teedataobject *)obj)->nextlink = NULL; Py_SETREF(obj, nextlink); @@ -837,8 +789,7 @@ teedataobject_clear(teedataobject *tdo) Py_CLEAR(tdo->values[i]); tmp = tdo->nextlink; tdo->nextlink = NULL; - itertools_state *state = get_module_state_by_cls(Py_TYPE(tdo)); - teedataobject_safe_decref(tmp, state->teedataobject_type); + teedataobject_safe_decref(tmp); return 0; } @@ -852,25 +803,6 @@ teedataobject_dealloc(teedataobject *tdo) Py_DECREF(tp); } -static PyObject * -teedataobject_reduce(teedataobject *tdo, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - int i; - /* create a temporary list of already iterated values */ - PyObject *values = PyList_New(tdo->numread); - - if (!values) - return NULL; - for (i=0 ; inumread ; i++) { - Py_INCREF(tdo->values[i]); - PyList_SET_ITEM(values, i, tdo->values[i]); - } - return Py_BuildValue("O(ONO)", Py_TYPE(tdo), tdo->it, - values, - tdo->nextlink ? tdo->nextlink : Py_None); -} - /*[clinic input] @classmethod itertools.teedataobject.__new__ @@ -925,19 +857,12 @@ itertools_teedataobject_impl(PyTypeObject *type, PyObject *it, return NULL; } -static PyMethodDef teedataobject_methods[] = { - {"__reduce__", (PyCFunction)teedataobject_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot teedataobject_slots[] = { {Py_tp_dealloc, teedataobject_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_doc, (void *)itertools_teedataobject__doc__}, {Py_tp_traverse, teedataobject_traverse}, {Py_tp_clear, teedataobject_clear}, - {Py_tp_methods, teedataobject_methods}, {Py_tp_new, itertools_teedataobject}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1066,41 +991,8 @@ tee_dealloc(teeobject *to) Py_DECREF(tp); } -static PyObject * -tee_reduce(teeobject *to, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(())(Oi)", Py_TYPE(to), to->dataobj, to->index); -} - -static PyObject * -tee_setstate(teeobject *to, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - teedataobject *tdo; - int index; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - PyTypeObject *tdo_type = to->state->teedataobject_type; - if (!PyArg_ParseTuple(state, "O!i", tdo_type, &tdo, &index)) { - return NULL; - } - if (index < 0 || index > LINKCELLS) { - PyErr_SetString(PyExc_ValueError, "Index out of range"); - return NULL; - } - Py_INCREF(tdo); - Py_XSETREF(to->dataobj, tdo); - to->index = index; - Py_RETURN_NONE; -} - static PyMethodDef tee_methods[] = { {"__copy__", (PyCFunction)tee_copy, METH_NOARGS, teecopy_doc}, - {"__reduce__", (PyCFunction)tee_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)tee_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -1144,7 +1036,7 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n) /*[clinic end generated code: output=1c64519cd859c2f0 input=c99a1472c425d66d]*/ { Py_ssize_t i; - PyObject *it, *copyable, *copyfunc, *result; + PyObject *it, *to, *result; if (n < 0) { PyErr_SetString(PyExc_ValueError, "n must be >= 0"); @@ -1161,41 +1053,23 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n) return NULL; } - if (PyObject_GetOptionalAttr(it, &_Py_ID(__copy__), ©func) < 0) { - Py_DECREF(it); + itertools_state *state = get_module_state(module); + to = tee_fromiterable(state, it); + Py_DECREF(it); + if (to == NULL) { Py_DECREF(result); return NULL; } - if (copyfunc != NULL) { - copyable = it; - } - else { - itertools_state *state = get_module_state(module); - copyable = tee_fromiterable(state, it); - Py_DECREF(it); - if (copyable == NULL) { - Py_DECREF(result); - return NULL; - } - copyfunc = PyObject_GetAttr(copyable, &_Py_ID(__copy__)); - if (copyfunc == NULL) { - Py_DECREF(copyable); - Py_DECREF(result); - return NULL; - } - } - PyTuple_SET_ITEM(result, 0, copyable); + PyTuple_SET_ITEM(result, 0, to); for (i = 1; i < n; i++) { - copyable = _PyObject_CallNoArgs(copyfunc); - if (copyable == NULL) { - Py_DECREF(copyfunc); + to = tee_copy((teeobject *)to, NULL); + if (to == NULL) { Py_DECREF(result); return NULL; } - PyTuple_SET_ITEM(result, i, copyable); + PyTuple_SET_ITEM(result, i, to); } - Py_DECREF(copyfunc); return result; } @@ -1302,59 +1176,6 @@ cycle_next(cycleobject *lz) return Py_NewRef(item); } -static PyObject * -cycle_reduce(cycleobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Create a new cycle with the iterator tuple, then set the saved state */ - if (lz->it == NULL) { - PyObject *it = PyObject_GetIter(lz->saved); - if (it == NULL) - return NULL; - if (lz->index != 0) { - PyObject *res = _PyObject_CallMethod(it, &_Py_ID(__setstate__), - "n", lz->index); - if (res == NULL) { - Py_DECREF(it); - return NULL; - } - Py_DECREF(res); - } - return Py_BuildValue("O(N)(OO)", Py_TYPE(lz), it, lz->saved, Py_True); - } - return Py_BuildValue("O(O)(OO)", Py_TYPE(lz), lz->it, lz->saved, - lz->firstpass ? Py_True : Py_False); -} - -static PyObject * -cycle_setstate(cycleobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *saved=NULL; - int firstpass; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - // The second item can be 1/0 in old pickles and True/False in new pickles - if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { - return NULL; - } - Py_INCREF(saved); - Py_XSETREF(lz->saved, saved); - lz->firstpass = firstpass != 0; - lz->index = 0; - Py_RETURN_NONE; -} - -static PyMethodDef cycle_methods[] = { - {"__reduce__", (PyCFunction)cycle_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)cycle_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot cycle_slots[] = { {Py_tp_dealloc, cycle_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1362,7 +1183,6 @@ static PyType_Slot cycle_slots[] = { {Py_tp_traverse, cycle_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, cycle_next}, - {Py_tp_methods, cycle_methods}, {Py_tp_new, itertools_cycle}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1475,32 +1295,6 @@ dropwhile_next(dropwhileobject *lz) } } -static PyObject * -dropwhile_reduce(dropwhileobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->start); -} - -static PyObject * -dropwhile_setstate(dropwhileobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - int start = PyObject_IsTrue(state); - if (start < 0) - return NULL; - lz->start = start; - Py_RETURN_NONE; -} - -static PyMethodDef dropwhile_methods[] = { - {"__reduce__", (PyCFunction)dropwhile_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)dropwhile_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot dropwhile_slots[] = { {Py_tp_dealloc, dropwhile_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1508,7 +1302,6 @@ static PyType_Slot dropwhile_slots[] = { {Py_tp_traverse, dropwhile_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, dropwhile_next}, - {Py_tp_methods, dropwhile_methods}, {Py_tp_new, itertools_dropwhile}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1615,33 +1408,6 @@ takewhile_next(takewhileobject *lz) return NULL; } -static PyObject * -takewhile_reduce(takewhileobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->stop); -} - -static PyObject * -takewhile_reduce_setstate(takewhileobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - int stop = PyObject_IsTrue(state); - - if (stop < 0) - return NULL; - lz->stop = stop; - Py_RETURN_NONE; -} - -static PyMethodDef takewhile_reduce_methods[] = { - {"__reduce__", (PyCFunction)takewhile_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)takewhile_reduce_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot takewhile_slots[] = { {Py_tp_dealloc, takewhile_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1649,7 +1415,6 @@ static PyType_Slot takewhile_slots[] = { {Py_tp_traverse, takewhile_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, takewhile_next}, - {Py_tp_methods, takewhile_reduce_methods}, {Py_tp_new, itertools_takewhile}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1819,59 +1584,6 @@ islice_next(isliceobject *lz) return NULL; } -static PyObject * -islice_reduce(isliceobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* When unpickled, generate a new object with the same bounds, - * then 'setstate' with the next and count - */ - PyObject *stop; - - if (lz->it == NULL) { - PyObject *empty_list; - PyObject *empty_it; - empty_list = PyList_New(0); - if (empty_list == NULL) - return NULL; - empty_it = PyObject_GetIter(empty_list); - Py_DECREF(empty_list); - if (empty_it == NULL) - return NULL; - return Py_BuildValue("O(Nn)n", Py_TYPE(lz), empty_it, 0, 0); - } - if (lz->stop == -1) { - stop = Py_NewRef(Py_None); - } else { - stop = PyLong_FromSsize_t(lz->stop); - if (stop == NULL) - return NULL; - } - return Py_BuildValue("O(OnNn)n", Py_TYPE(lz), - lz->it, lz->next, stop, lz->step, - lz->cnt); -} - -static PyObject * -islice_setstate(isliceobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_ssize_t cnt = PyLong_AsSsize_t(state); - - if (cnt == -1 && PyErr_Occurred()) - return NULL; - lz->cnt = cnt; - Py_RETURN_NONE; -} - -static PyMethodDef islice_methods[] = { - {"__reduce__", (PyCFunction)islice_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)islice_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - PyDoc_STRVAR(islice_doc, "islice(iterable, stop) --> islice object\n\ islice(iterable, start, stop[, step]) --> islice object\n\ @@ -1890,7 +1602,6 @@ static PyType_Slot islice_slots[] = { {Py_tp_traverse, islice_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, islice_next}, - {Py_tp_methods, islice_methods}, {Py_tp_new, islice_new}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1988,20 +1699,6 @@ starmap_next(starmapobject *lz) return result; } -static PyObject * -starmap_reduce(starmapobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Just pickle the iterator */ - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); -} - -static PyMethodDef starmap_methods[] = { - {"__reduce__", (PyCFunction)starmap_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot starmap_slots[] = { {Py_tp_dealloc, starmap_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -2009,7 +1706,6 @@ static PyType_Slot starmap_slots[] = { {Py_tp_traverse, starmap_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, starmap_next}, - {Py_tp_methods, starmap_methods}, {Py_tp_new, itertools_starmap}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -2145,53 +1841,9 @@ chain_next(chainobject *lz) return NULL; } -static PyObject * -chain_reduce(chainobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->source) { - /* we can't pickle function objects (itertools.from_iterable) so - * we must use setstate to replace the iterable. One day we - * will fix pickling of functions - */ - if (lz->active) { - return Py_BuildValue("O()(OO)", Py_TYPE(lz), lz->source, lz->active); - } else { - return Py_BuildValue("O()(O)", Py_TYPE(lz), lz->source); - } - } else { - return Py_BuildValue("O()", Py_TYPE(lz)); /* exhausted */ - } - return NULL; -} - -static PyObject * -chain_setstate(chainobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *source, *active=NULL; - - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "O|O", &source, &active)) { - return NULL; - } - if (!PyIter_Check(source) || (active != NULL && !PyIter_Check(active))) { - PyErr_SetString(PyExc_TypeError, "Arguments must be iterators."); - return NULL; - } - - Py_INCREF(source); - Py_XSETREF(lz->source, source); - Py_XINCREF(active); - Py_XSETREF(lz->active, active); - Py_RETURN_NONE; -} - PyDoc_STRVAR(chain_doc, -"chain(*iterables) --> chain object\n\ +"chain(*iterables)\n\ +--\n\ \n\ Return a chain object whose .__next__() method returns elements from the\n\ first iterable until it is exhausted, then elements from the next\n\ @@ -2199,10 +1851,6 @@ iterable, until all of the iterables are exhausted."); static PyMethodDef chain_methods[] = { ITERTOOLS_CHAIN_FROM_ITERABLE_METHODDEF - {"__reduce__", (PyCFunction)chain_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)chain_setstate, METH_O, - setstate_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -2441,96 +2089,15 @@ product_next(productobject *lz) return NULL; } -static PyObject * -product_reduce(productobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->stopped) { - return Py_BuildValue("O(())", Py_TYPE(lz)); - } else if (lz->result == NULL) { - return Py_BuildValue("OO", Py_TYPE(lz), lz->pools); - } else { - PyObject *indices; - Py_ssize_t n, i; - - /* we must pickle the indices use them for setstate, and - * additionally indicate that the iterator has started - */ - n = PyTuple_GET_SIZE(lz->pools); - indices = PyTuple_New(n); - if (indices == NULL) - return NULL; - for (i=0; iindices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - return Py_BuildValue("OON", Py_TYPE(lz), lz->pools, indices); - } -} - -static PyObject * -product_setstate(productobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t n, i; - - n = PyTuple_GET_SIZE(lz->pools); - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != n) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - for (i=0; ipools, i); - poolsize = PyTuple_GET_SIZE(pool); - if (poolsize == 0) { - lz->stopped = 1; - Py_RETURN_NONE; - } - /* clamp the index */ - if (index < 0) - index = 0; - else if (index > poolsize-1) - index = poolsize-1; - lz->indices[i] = index; - } - - result = PyTuple_New(n); - if (!result) - return NULL; - for (i=0; ipools, i); - PyObject *element = PyTuple_GET_ITEM(pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef product_methods[] = { - {"__reduce__", (PyCFunction)product_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)product_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)product_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ }; PyDoc_STRVAR(product_doc, -"product(*iterables, repeat=1) --> product object\n\ +"product(*iterables, repeat=1)\n\ +--\n\ \n\ Cartesian product of input iterables. Equivalent to nested for-loops.\n\n\ For example, product(A, B) returns the same as: ((x,y) for x in A for y in B).\n\ @@ -2751,83 +2318,7 @@ combinations_next(combinationsobject *co) return NULL; } -static PyObject * -combinations_reduce(combinationsobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(lz), lz->pool, lz->r); - } else if (lz->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(lz), lz->r); - } else { - PyObject *indices; - Py_ssize_t i; - - /* we must pickle the indices and use them for setstate */ - indices = PyTuple_New(lz->r); - if (!indices) - return NULL; - for (i=0; ir; i++) - { - PyObject* index = PyLong_FromSsize_t(lz->indices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - - return Py_BuildValue("O(On)N", Py_TYPE(lz), lz->pool, lz->r, indices); - } -} - -static PyObject * -combinations_setstate(combinationsobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t i; - Py_ssize_t n = PyTuple_GET_SIZE(lz->pool); - - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - for (i=0; ir; i++) { - Py_ssize_t max; - PyObject* indexObject = PyTuple_GET_ITEM(state, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - - if (index == -1 && PyErr_Occurred()) - return NULL; /* not an integer */ - max = i + n - lz->r; - /* clamp the index (beware of negative max) */ - if (index > max) - index = max; - if (index < 0) - index = 0; - lz->indices[i] = index; - } - - result = PyTuple_New(lz->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(lz->pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef combinations_methods[] = { - {"__reduce__", (PyCFunction)combinations_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)combinations_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)combinations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3061,79 +2552,7 @@ cwr_next(cwrobject *co) return NULL; } -static PyObject * -cwr_reduce(cwrobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(lz), lz->pool, lz->r); - } else if (lz->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(lz), lz->r); - } else { - PyObject *indices; - Py_ssize_t i; - - /* we must pickle the indices and use them for setstate */ - indices = PyTuple_New(lz->r); - if (!indices) - return NULL; - for (i=0; ir; i++) { - PyObject* index = PyLong_FromSsize_t(lz->indices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - - return Py_BuildValue("O(On)N", Py_TYPE(lz), lz->pool, lz->r, indices); - } -} - -static PyObject * -cwr_setstate(cwrobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t n, i; - - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) - { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - n = PyTuple_GET_SIZE(lz->pool); - for (i=0; ir; i++) { - PyObject* indexObject = PyTuple_GET_ITEM(state, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - - if (index < 0 && PyErr_Occurred()) - return NULL; /* not an integer */ - /* clamp the index */ - if (index < 0) - index = 0; - else if (index > n-1) - index = n-1; - lz->indices[i] = index; - } - result = PyTuple_New(lz->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(lz->pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef cwr_methods[] = { - {"__reduce__", (PyCFunction)cwr_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)cwr_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)cwr_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3398,113 +2817,7 @@ permutations_next(permutationsobject *po) return NULL; } -static PyObject * -permutations_reduce(permutationsobject *po, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (po->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(po), po->pool, po->r); - } else if (po->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(po), po->r); - } else { - PyObject *indices=NULL, *cycles=NULL; - Py_ssize_t n, i; - - /* we must pickle the indices and cycles and use them for setstate */ - n = PyTuple_GET_SIZE(po->pool); - indices = PyTuple_New(n); - if (indices == NULL) - goto err; - for (i=0; iindices[i]); - if (!index) - goto err; - PyTuple_SET_ITEM(indices, i, index); - } - - cycles = PyTuple_New(po->r); - if (cycles == NULL) - goto err; - for (i=0 ; ir ; i++) { - PyObject* index = PyLong_FromSsize_t(po->cycles[i]); - if (!index) - goto err; - PyTuple_SET_ITEM(cycles, i, index); - } - return Py_BuildValue("O(On)(NN)", Py_TYPE(po), - po->pool, po->r, - indices, cycles); - err: - Py_XDECREF(indices); - Py_XDECREF(cycles); - return NULL; - } -} - -static PyObject * -permutations_setstate(permutationsobject *po, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *indices, *cycles, *result; - Py_ssize_t n, i; - - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "O!O!", - &PyTuple_Type, &indices, - &PyTuple_Type, &cycles)) { - return NULL; - } - - n = PyTuple_GET_SIZE(po->pool); - if (PyTuple_GET_SIZE(indices) != n || PyTuple_GET_SIZE(cycles) != po->r) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - for (i=0; i n-1) - index = n-1; - po->indices[i] = index; - } - - for (i=0; ir; i++) { - PyObject* indexObject = PyTuple_GET_ITEM(cycles, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - if (index < 0 && PyErr_Occurred()) - return NULL; /* not an integer */ - if (index < 1) - index = 1; - else if (index > n-i) - index = n-i; - po->cycles[i] = index; - } - result = PyTuple_New(po->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(po->pool, po->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(po->result, result); - Py_RETURN_NONE; -} - static PyMethodDef permuations_methods[] = { - {"__reduce__", (PyCFunction)permutations_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)permutations_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)permutations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3639,59 +2952,6 @@ accumulate_next(accumulateobject *lz) return newtotal; } -static PyObject * -accumulate_reduce(accumulateobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - itertools_state *state = lz->state; - - if (lz->initial != Py_None) { - PyObject *it; - - assert(lz->total == NULL); - it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", - lz->initial, lz->it); - if (it == NULL) - return NULL; - return Py_BuildValue("O(NO)O", Py_TYPE(lz), - it, lz->binop?lz->binop:Py_None, Py_None); - } - if (lz->total == Py_None) { - PyObject *it; - - it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", - lz->total, lz->it); - if (it == NULL) - return NULL; - it = PyObject_CallFunction((PyObject *)Py_TYPE(lz), "NO", - it, lz->binop ? lz->binop : Py_None); - if (it == NULL) - return NULL; - - return Py_BuildValue("O(NiO)", state->islice_type, it, 1, Py_None); - } - return Py_BuildValue("O(OO)O", Py_TYPE(lz), - lz->it, lz->binop?lz->binop:Py_None, - lz->total?lz->total:Py_None); -} - -static PyObject * -accumulate_setstate(accumulateobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_INCREF(state); - Py_XSETREF(lz->total, state); - Py_RETURN_NONE; -} - -static PyMethodDef accumulate_methods[] = { - {"__reduce__", (PyCFunction)accumulate_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)accumulate_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot accumulate_slots[] = { {Py_tp_dealloc, accumulate_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3699,7 +2959,6 @@ static PyType_Slot accumulate_slots[] = { {Py_tp_traverse, accumulate_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, accumulate_next}, - {Py_tp_methods, accumulate_methods}, {Py_tp_new, itertools_accumulate}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -3824,20 +3083,6 @@ compress_next(compressobject *lz) } } -static PyObject * -compress_reduce(compressobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->data, lz->selectors); -} - -static PyMethodDef compress_methods[] = { - {"__reduce__", (PyCFunction)compress_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot compress_slots[] = { {Py_tp_dealloc, compress_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3845,7 +3090,6 @@ static PyType_Slot compress_slots[] = { {Py_tp_traverse, compress_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, compress_next}, - {Py_tp_methods, compress_methods}, {Py_tp_new, itertools_compress}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -3957,19 +3201,6 @@ filterfalse_next(filterfalseobject *lz) } } -static PyObject * -filterfalse_reduce(filterfalseobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); -} - -static PyMethodDef filterfalse_methods[] = { - {"__reduce__", (PyCFunction)filterfalse_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot filterfalse_slots[] = { {Py_tp_dealloc, filterfalse_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3977,7 +3208,6 @@ static PyType_Slot filterfalse_slots[] = { {Py_tp_traverse, filterfalse_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, filterfalse_next}, - {Py_tp_methods, filterfalse_methods}, {Py_tp_new, itertools_filterfalse}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4007,7 +3237,7 @@ fast_mode: when cnt an integer < PY_SSIZE_T_MAX and no step is specified. assert(cnt != PY_SSIZE_T_MAX && long_cnt == NULL && long_step==PyLong(1)); Advances with: cnt += 1 - When count hits Y_SSIZE_T_MAX, switch to slow_mode. + When count hits PY_SSIZE_T_MAX, switch to slow_mode. slow_mode: when cnt == PY_SSIZE_T_MAX, step is not int(1), or cnt is a float. @@ -4156,9 +3386,30 @@ count_nextlong(countobject *lz) static PyObject * count_next(countobject *lz) { +#ifndef Py_GIL_DISABLED if (lz->cnt == PY_SSIZE_T_MAX) return count_nextlong(lz); return PyLong_FromSsize_t(lz->cnt++); +#else + // free-threading version + // fast mode uses compare-exchange loop + // slow mode uses a critical section + PyObject *returned; + Py_ssize_t cnt; + + cnt = _Py_atomic_load_ssize_relaxed(&lz->cnt); + for (;;) { + if (cnt == PY_SSIZE_T_MAX) { + Py_BEGIN_CRITICAL_SECTION(lz); + returned = count_nextlong(lz); + Py_END_CRITICAL_SECTION(); + return returned; + } + if (_Py_atomic_compare_exchange_ssize(&lz->cnt, &cnt, cnt + 1)) { + return PyLong_FromSsize_t(cnt); + } + } +#endif } static PyObject * @@ -4185,21 +3436,6 @@ count_repr(countobject *lz) lz->long_cnt, lz->long_step); } -static PyObject * -count_reduce(countobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->cnt == PY_SSIZE_T_MAX) - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->long_cnt, lz->long_step); - return Py_BuildValue("O(n)", Py_TYPE(lz), lz->cnt); -} - -static PyMethodDef count_methods[] = { - {"__reduce__", (PyCFunction)count_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot count_slots[] = { {Py_tp_dealloc, count_dealloc}, {Py_tp_repr, count_repr}, @@ -4208,7 +3444,6 @@ static PyType_Slot count_slots[] = { {Py_tp_traverse, count_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, count_next}, - {Py_tp_methods, count_methods}, {Py_tp_new, itertools_count}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4309,22 +3544,8 @@ repeat_len(repeatobject *ro, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); -static PyObject * -repeat_reduce(repeatobject *ro, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* unpickle this so that a new repeat iterator is constructed with an - * object, then call __setstate__ on it to set cnt - */ - if (ro->cnt >= 0) - return Py_BuildValue("O(On)", Py_TYPE(ro), ro->element, ro->cnt); - else - return Py_BuildValue("O(O)", Py_TYPE(ro), ro->element); -} - static PyMethodDef repeat_methods[] = { {"__length_hint__", (PyCFunction)repeat_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)repeat_reduce, METH_NOARGS, reduce_doc}, {NULL, NULL} /* sentinel */ }; @@ -4530,52 +3751,9 @@ zip_longest_next(ziplongestobject *lz) return result; } -static PyObject * -zip_longest_reduce(ziplongestobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Create a new tuple with empty sequences where appropriate to pickle. - * Then use setstate to set the fillvalue - */ - int i; - PyObject *args = PyTuple_New(PyTuple_GET_SIZE(lz->ittuple)); - - if (args == NULL) - return NULL; - for (i=0; iittuple); i++) { - PyObject *elem = PyTuple_GET_ITEM(lz->ittuple, i); - if (elem == NULL) { - elem = PyTuple_New(0); - if (elem == NULL) { - Py_DECREF(args); - return NULL; - } - } else - Py_INCREF(elem); - PyTuple_SET_ITEM(args, i, elem); - } - return Py_BuildValue("ONO", Py_TYPE(lz), args, lz->fillvalue); -} - -static PyObject * -zip_longest_setstate(ziplongestobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_INCREF(state); - Py_XSETREF(lz->fillvalue, state); - Py_RETURN_NONE; -} - -static PyMethodDef zip_longest_methods[] = { - {"__reduce__", (PyCFunction)zip_longest_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)zip_longest_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - PyDoc_STRVAR(zip_longest_doc, -"zip_longest(iter1 [,iter2 [...]], [fillvalue=None]) --> zip_longest object\n\ +"zip_longest(*iterables, fillvalue=None)\n\ +--\n\ \n\ Return a zip_longest object whose .__next__() method returns a tuple where\n\ the i-th element comes from the i-th iterable argument. The .__next__()\n\ @@ -4592,7 +3770,6 @@ static PyType_Slot ziplongest_slots[] = { {Py_tp_traverse, zip_longest_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, zip_longest_next}, - {Py_tp_methods, zip_longest_methods}, {Py_tp_new, zip_longest_new}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4623,15 +3800,15 @@ batched(p, n) --> [p0, p1, ..., p_n-1], [p_n, p_n+1, ..., p_2n-1], ...\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ...\n\ chain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ...\n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ -dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ +dropwhile(predicate, seq) --> seq[n], seq[n+1], starting when predicate fails\n\ groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\ -filterfalse(pred, seq) --> elements of seq where pred(elem) is False\n\ +filterfalse(predicate, seq) --> elements of seq where predicate(elem) is False\n\ islice(seq, [start,] stop [, step]) --> elements from\n\ seq[start:stop:step]\n\ pairwise(s) --> (s[0],s[1]), (s[1],s[2]), (s[2], s[3]), ...\n\ starmap(fun, seq) --> fun(*seq[0]), fun(*seq[1]), ...\n\ tee(it, n=2) --> (it1, it2 , ... itn) splits one iterator into n\n\ -takewhile(pred, seq) --> seq[0], seq[1], until pred fails\n\ +takewhile(predicate, seq) --> seq[0], seq[1], until predicate fails\n\ zip_longest(p, q, ...) --> (p[0], q[0]), (p[1], q[1]), ...\n\ \n\ Combinatoric generators:\n\ @@ -4750,6 +3927,7 @@ itertoolsmodule_exec(PyObject *mod) static struct PyModuleDef_Slot itertoolsmodule_slots[] = { {Py_mod_exec, itertoolsmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/main.c b/Modules/main.c index df2ce550245088..15ea49a1bad19e 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -4,6 +4,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_initconfig.h" // _PyArgv #include "pycore_interp.h" // _PyInterpreterState.sysdict +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -259,6 +260,57 @@ pymain_run_command(wchar_t *command) } +static int +pymain_start_pyrepl_no_main(void) +{ + int res = 0; + PyObject *console = NULL; + PyObject *empty_tuple = NULL; + PyObject *kwargs = NULL; + PyObject *console_result = NULL; + + PyObject *pyrepl = PyImport_ImportModule("_pyrepl.main"); + if (pyrepl == NULL) { + fprintf(stderr, "Could not import _pyrepl.main\n"); + res = pymain_exit_err_print(); + goto done; + } + console = PyObject_GetAttrString(pyrepl, "interactive_console"); + if (console == NULL) { + fprintf(stderr, "Could not access _pyrepl.main.interactive_console\n"); + res = pymain_exit_err_print(); + goto done; + } + empty_tuple = PyTuple_New(0); + if (empty_tuple == NULL) { + res = pymain_exit_err_print(); + goto done; + } + kwargs = PyDict_New(); + if (kwargs == NULL) { + res = pymain_exit_err_print(); + goto done; + } + if (!PyDict_SetItemString(kwargs, "pythonstartup", _PyLong_GetOne())) { + _PyRuntime.signals.unhandled_keyboard_interrupt = 0; + console_result = PyObject_Call(console, empty_tuple, kwargs); + if (!console_result && PyErr_Occurred() == PyExc_KeyboardInterrupt) { + _PyRuntime.signals.unhandled_keyboard_interrupt = 1; + } + if (console_result == NULL) { + res = pymain_exit_err_print(); + } + } +done: + Py_XDECREF(console_result); + Py_XDECREF(kwargs); + Py_XDECREF(empty_tuple); + Py_XDECREF(console); + Py_XDECREF(pyrepl); + return res; +} + + static int pymain_run_module(const wchar_t *modname, int set_argv0) { @@ -513,8 +565,13 @@ pymain_run_stdin(PyConfig *config) return pymain_exit_err_print(); } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + return (run != 0); + } + int run = pymain_run_module(L"_pyrepl", 0); return (run != 0); } @@ -537,9 +594,20 @@ pymain_repl(PyConfig *config, int *exitcode) return; } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int res = PyRun_AnyFileFlags(stdin, "", &cf); - *exitcode = (res != 0); + if (PySys_Audit("cpython.run_stdin", NULL) < 0) { + return; + } + + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + *exitcode = (run != 0); + return; + } + int run = pymain_start_pyrepl_no_main(); + *exitcode = (run != 0); + return; } diff --git a/Modules/makesetup b/Modules/makesetup index f000c9cd67310e..8bb971b152a522 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -87,18 +87,6 @@ esac NL='\ ' -# Setup to link with extra libraries when making shared extensions. -# Currently, only Cygwin needs this baggage. -case `uname -s` in -CYGWIN*) if test $libdir = . - then - ExtraLibDir=. - else - ExtraLibDir='$(LIBPL)' - fi - ExtraLibs="-L$ExtraLibDir -lpython\$(LDVERSION)";; -esac - # Main loop for i in ${*-Setup} do @@ -286,7 +274,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" - rule="$rule; \$(BLDSHARED) $objs $libs $ExtraLibs -o $file" + rule="$rule; \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file" echo "$rule" >>$rulesf done done diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index a877bfcd6afb68..7e8d8b3f5bafa2 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -107,7 +107,7 @@ typedef struct{ double hi; double lo; } DoubleLength; static DoubleLength dl_fast_sum(double a, double b) { - /* Algorithm 1.1. Compensated summation of two floating point numbers. */ + /* Algorithm 1.1. Compensated summation of two floating-point numbers. */ assert(fabs(a) >= fabs(b)); double x = a + b; double y = (a - x) + b; @@ -237,7 +237,7 @@ m_sinpi(double x) double y, r; int n; /* this function should only ever be called for finite arguments */ - assert(Py_IS_FINITE(x)); + assert(isfinite(x)); y = fmod(fabs(x), 2.0); n = (int)round(2.0*y); assert(0 <= n && n <= 4); @@ -396,8 +396,8 @@ m_tgamma(double x) double absx, r, y, z, sqrtpow; /* special cases */ - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x) || x > 0.0) + if (!isfinite(x)) { + if (isnan(x) || x > 0.0) return x; /* tgamma(nan) = nan, tgamma(inf) = inf */ else { errno = EDOM; @@ -424,7 +424,7 @@ m_tgamma(double x) /* tiny arguments: tgamma(x) ~ 1/x for x near 0 */ if (absx < 1e-20) { r = 1.0/x; - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -438,7 +438,7 @@ m_tgamma(double x) } else { errno = ERANGE; - return Py_HUGE_VAL; + return Py_INFINITY; } } @@ -481,7 +481,7 @@ m_tgamma(double x) r *= sqrtpow; } } - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -498,18 +498,18 @@ m_lgamma(double x) double absx; /* special cases */ - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x)) + if (!isfinite(x)) { + if (isnan(x)) return x; /* lgamma(nan) = nan */ else - return Py_HUGE_VAL; /* lgamma(+-inf) = +inf */ + return Py_INFINITY; /* lgamma(+-inf) = +inf */ } /* integer arguments */ if (x == floor(x) && x <= 2.0) { if (x <= 0.0) { errno = EDOM; /* lgamma(n) = inf, divide-by-zero for */ - return Py_HUGE_VAL; /* integers n <= 0 */ + return Py_INFINITY; /* integers n <= 0 */ } else { return 0.0; /* lgamma(1) = lgamma(2) = 0.0 */ @@ -530,48 +530,11 @@ m_lgamma(double x) if (x < 0.0) /* Use reflection formula to get value for negative x. */ r = logpi - log(fabs(m_sinpi(absx))) - log(absx) - r; - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } -/* - wrapper for atan2 that deals directly with special cases before - delegating to the platform libm for the remaining cases. This - is necessary to get consistent behaviour across platforms. - Windows, FreeBSD and alpha Tru64 are amongst platforms that don't - always follow C99. -*/ - -static double -m_atan2(double y, double x) -{ - if (Py_IS_NAN(x) || Py_IS_NAN(y)) - return Py_NAN; - if (Py_IS_INFINITY(y)) { - if (Py_IS_INFINITY(x)) { - if (copysign(1., x) == 1.) - /* atan2(+-inf, +inf) == +-pi/4 */ - return copysign(0.25*Py_MATH_PI, y); - else - /* atan2(+-inf, -inf) == +-pi*3/4 */ - return copysign(0.75*Py_MATH_PI, y); - } - /* atan2(+-inf, x) == +-pi/2 for finite x */ - return copysign(0.5*Py_MATH_PI, y); - } - if (Py_IS_INFINITY(x) || y == 0.) { - if (copysign(1., x) == 1.) - /* atan2(+-y, +inf) = atan2(+-0, +x) = +-0. */ - return copysign(0., y); - else - /* atan2(+-y, -inf) = atan2(+-0., -x) = +-pi. */ - return copysign(Py_MATH_PI, y); - } - return atan2(y, x); -} - - /* IEEE 754-style remainder operation: x - n*y where n*y is the nearest multiple of y to x, taking n even in the case of a tie. Assuming an IEEE 754 binary floating-point format, the result is always exact. */ @@ -580,7 +543,7 @@ static double m_remainder(double x, double y) { /* Deal with most common case first. */ - if (Py_IS_FINITE(x) && Py_IS_FINITE(y)) { + if (isfinite(x) && isfinite(y)) { double absx, absy, c, m, r; if (y == 0.0) { @@ -653,16 +616,16 @@ m_remainder(double x, double y) } /* Special values. */ - if (Py_IS_NAN(x)) { + if (isnan(x)) { return x; } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return y; } - if (Py_IS_INFINITY(x)) { + if (isinf(x)) { return Py_NAN; } - assert(Py_IS_INFINITY(y)); + assert(isinf(y)); return x; } @@ -677,16 +640,16 @@ m_remainder(double x, double y) static double m_log(double x) { - if (Py_IS_FINITE(x)) { + if (isfinite(x)) { if (x > 0.0) return log(x); errno = EDOM; if (x == 0.0) - return -Py_HUGE_VAL; /* log(0) = -inf */ + return -Py_INFINITY; /* log(0) = -inf */ else return Py_NAN; /* log(-ve) = nan */ } - else if (Py_IS_NAN(x)) + else if (isnan(x)) return x; /* log(nan) = nan */ else if (x > 0.0) return x; /* log(inf) = inf */ @@ -709,8 +672,8 @@ m_log(double x) static double m_log2(double x) { - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x)) + if (!isfinite(x)) { + if (isnan(x)) return x; /* log2(nan) = nan */ else if (x > 0.0) return x; /* log2(+inf) = +inf */ @@ -725,7 +688,7 @@ m_log2(double x) } else if (x == 0.0) { errno = EDOM; - return -Py_HUGE_VAL; /* log2(0) = -inf, divide-by-zero */ + return -Py_INFINITY; /* log2(0) = -inf, divide-by-zero */ } else { errno = EDOM; @@ -736,16 +699,16 @@ m_log2(double x) static double m_log10(double x) { - if (Py_IS_FINITE(x)) { + if (isfinite(x)) { if (x > 0.0) return log10(x); errno = EDOM; if (x == 0.0) - return -Py_HUGE_VAL; /* log10(0) = -inf */ + return -Py_INFINITY; /* log10(0) = -inf */ else return Py_NAN; /* log10(-ve) = nan */ } - else if (Py_IS_NAN(x)) + else if (isnan(x)) return x; /* log10(nan) = nan */ else if (x > 0.0) return x; /* log10(inf) = inf */ @@ -756,8 +719,17 @@ m_log10(double x) } +/*[clinic input] +math.gcd + + *integers as args: object + +Greatest Common Divisor. +[clinic start generated code]*/ + static PyObject * -math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) +math_gcd_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=b57687fcf431c1b8 input=94e675b7ceeaf0c9]*/ { // Fast-path for the common case: gcd(int, int) if (nargs == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) @@ -800,12 +772,6 @@ math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) return res; } -PyDoc_STRVAR(math_gcd_doc, -"gcd($module, *integers)\n" -"--\n" -"\n" -"Greatest Common Divisor."); - static PyObject * long_lcm(PyObject *a, PyObject *b) @@ -835,8 +801,17 @@ long_lcm(PyObject *a, PyObject *b) } +/*[clinic input] +math.lcm + + *integers as args: object + +Least Common Multiple. +[clinic start generated code]*/ + static PyObject * -math_lcm(PyObject *module, PyObject * const *args, Py_ssize_t nargs) +math_lcm_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=f3eff0c25e4d7030 input=e64c33e85f4c47c6]*/ { PyObject *res, *x; Py_ssize_t i; @@ -876,13 +851,6 @@ math_lcm(PyObject *module, PyObject * const *args, Py_ssize_t nargs) } -PyDoc_STRVAR(math_lcm_doc, -"lcm($module, *integers)\n" -"--\n" -"\n" -"Least Common Multiple."); - - /* Call is_error when errno != 0, and where x is the result libm * returned. is_error will usually set up an exception and return * true (1), but may return false (0) without setting up an exception. @@ -966,12 +934,12 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) return NULL; errno = 0; r = (*func)(x); - if (Py_IS_NAN(r) && !Py_IS_NAN(x)) { + if (isnan(r) && !isnan(x)) { PyErr_SetString(PyExc_ValueError, "math domain error"); /* invalid arg */ return NULL; } - if (Py_IS_INFINITY(r) && Py_IS_FINITE(x)) { + if (isinf(r) && isfinite(x)) { if (can_overflow) PyErr_SetString(PyExc_OverflowError, "math range error"); /* overflow */ @@ -980,7 +948,7 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) "math domain error"); /* singularity */ return NULL; } - if (Py_IS_FINITE(r) && errno && is_error(r)) + if (isfinite(r) && errno && is_error(r)) /* this branch unnecessary on most platforms */ return NULL; @@ -1049,14 +1017,14 @@ math_2(PyObject *const *args, Py_ssize_t nargs, } errno = 0; r = (*func)(x, y); - if (Py_IS_NAN(r)) { - if (!Py_IS_NAN(x) && !Py_IS_NAN(y)) + if (isnan(r)) { + if (!isnan(x) && !isnan(y)) errno = EDOM; else errno = 0; } - else if (Py_IS_INFINITY(r)) { - if (Py_IS_FINITE(x) && Py_IS_FINITE(y)) + else if (isinf(r)) { + if (isfinite(x) && isfinite(y)) errno = ERANGE; else errno = 0; @@ -1103,7 +1071,7 @@ FUNC1(atan, atan, 0, "atan($module, x, /)\n--\n\n" "Return the arc tangent (measured in radians) of x.\n\n" "The result is between -pi/2 and pi/2.") -FUNC2(atan2, m_atan2, +FUNC2(atan2, atan2, "atan2($module, y, x, /)\n--\n\n" "Return the arc tangent (measured in radians) of y/x.\n\n" "Unlike atan(y/x), the signs of both x and y are considered.") @@ -1253,7 +1221,7 @@ FUNC1(tanh, tanh, 0, "Return the hyperbolic tangent of x.") /* Precision summation function as msum() by Raymond Hettinger in - , + , enhanced with the exact partials sum and roundoff from Mark Dickinson's post at . See those links for more details, proofs and other references. @@ -1354,14 +1322,14 @@ math.fsum seq: object / -Return an accurate floating point sum of values in the iterable seq. +Return an accurate floating-point sum of values in the iterable seq. -Assumes IEEE-754 floating point arithmetic. +Assumes IEEE-754 floating-point arithmetic. [clinic start generated code]*/ static PyObject * math_fsum(PyObject *module, PyObject *seq) -/*[clinic end generated code: output=ba5c672b87fe34fc input=c51b7d8caf6f6e82]*/ +/*[clinic end generated code: output=ba5c672b87fe34fc input=4506244ded6057dc]*/ { PyObject *item, *iter, *sum = NULL; Py_ssize_t i, j, n = 0, m = NUM_PARTIALS; @@ -1403,17 +1371,17 @@ math_fsum(PyObject *module, PyObject *seq) n = i; /* ps[i:] = [x] */ if (x != 0.0) { - if (! Py_IS_FINITE(x)) { + if (! isfinite(x)) { /* a nonfinite x could arise either as a result of intermediate overflow, or as a result of a nan or inf in the summands */ - if (Py_IS_FINITE(xsave)) { + if (isfinite(xsave)) { PyErr_SetString(PyExc_OverflowError, "intermediate overflow in fsum"); goto _fsum_error; } - if (Py_IS_INFINITY(xsave)) + if (isinf(xsave)) inf_sum += xsave; special_sum += xsave; /* reset partials */ @@ -1427,7 +1395,7 @@ math_fsum(PyObject *module, PyObject *seq) } if (special_sum != 0.0) { - if (Py_IS_NAN(inf_sum)) + if (isnan(inf_sum)) PyErr_SetString(PyExc_ValueError, "-inf + inf in fsum"); else @@ -1694,7 +1662,7 @@ math_isqrt(PyObject *module, PyObject *n) /*[clinic end generated code: output=35a6f7f980beab26 input=5b6e7ae4fa6c43d6]*/ { int a_too_large, c_bit_length; - size_t c, d; + int64_t c, d; uint64_t m; uint32_t u; PyObject *a = NULL, *b; @@ -1717,14 +1685,13 @@ math_isqrt(PyObject *module, PyObject *n) /* c = (n.bit_length() - 1) // 2 */ c = _PyLong_NumBits(n); - if (c == (size_t)(-1)) { - goto error; - } - c = (c - 1U) / 2U; + assert(c > 0); + assert(!PyErr_Occurred()); + c = (c - 1) / 2; /* Fast path: if c <= 31 then n < 2**64 and we can compute directly with a fast, almost branch-free algorithm. */ - if (c <= 31U) { + if (c <= 31) { int shift = 31 - (int)c; m = (uint64_t)PyLong_AsUnsignedLongLong(n); Py_DECREF(n); @@ -1741,13 +1708,13 @@ math_isqrt(PyObject *module, PyObject *n) /* From n >= 2**64 it follows that c.bit_length() >= 6. */ c_bit_length = 6; - while ((c >> c_bit_length) > 0U) { + while ((c >> c_bit_length) > 0) { ++c_bit_length; } /* Initialise d and a. */ d = c >> (c_bit_length - 5); - b = _PyLong_Rshift(n, 2U*c - 62U); + b = _PyLong_Rshift(n, 2*c - 62); if (b == NULL) { goto error; } @@ -1764,12 +1731,12 @@ math_isqrt(PyObject *module, PyObject *n) for (int s = c_bit_length - 6; s >= 0; --s) { PyObject *q; - size_t e = d; + int64_t e = d; d = c >> s; /* q = (n >> 2*c - e - d + 1) // a */ - q = _PyLong_Rshift(n, 2U*c - d - e + 1U); + q = _PyLong_Rshift(n, 2*c - d - e + 1); if (q == NULL) { goto error; } @@ -1779,7 +1746,7 @@ math_isqrt(PyObject *module, PyObject *n) } /* a = (a << d - 1 - e) + q */ - Py_SETREF(a, _PyLong_Lshift(a, d - 1U - e)); + Py_SETREF(a, _PyLong_Lshift(a, d - 1 - e)); if (a == NULL) { Py_DECREF(q); goto error; @@ -2108,7 +2075,7 @@ math_frexp_impl(PyObject *module, double x) int i; /* deal with special cases directly, to sidestep platform differences */ - if (Py_IS_NAN(x) || Py_IS_INFINITY(x) || !x) { + if (isnan(x) || isinf(x) || !x) { i = 0; } else { @@ -2153,13 +2120,13 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) return NULL; } - if (x == 0. || !Py_IS_FINITE(x)) { + if (x == 0. || !isfinite(x)) { /* NaNs, zeros and infinities are returned unchanged */ r = x; errno = 0; } else if (exp > INT_MAX) { /* overflow */ - r = copysign(Py_HUGE_VAL, x); + r = copysign(Py_INFINITY, x); errno = ERANGE; } else if (exp < INT_MIN) { /* underflow to +-0 */ @@ -2168,7 +2135,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) } else { errno = 0; r = ldexp(x, (int)exp); - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; } @@ -2196,9 +2163,9 @@ math_modf_impl(PyObject *module, double x) double y; /* some platforms don't do the right thing for NaNs and infinities, so we take care of special cases directly. */ - if (Py_IS_INFINITY(x)) + if (isinf(x)) return Py_BuildValue("(dd)", copysign(0., x), x); - else if (Py_IS_NAN(x)) + else if (isnan(x)) return Py_BuildValue("(dd)", x, x); errno = 0; @@ -2222,7 +2189,7 @@ loghelper(PyObject* arg, double (*func)(double)) /* If it is int, do it ourselves. */ if (PyLong_Check(arg)) { double x, result; - Py_ssize_t e; + int64_t e; /* Negative or zero inputs give a ValueError. */ if (!_PyLong_IsPositive((PyLongObject *)arg)) { @@ -2239,8 +2206,8 @@ loghelper(PyObject* arg, double (*func)(double)) to compute the log anyway. Clear the exception and continue. */ PyErr_Clear(); x = _PyLong_Frexp((PyLongObject *)arg, &e); - if (x == -1.0 && PyErr_Occurred()) - return NULL; + assert(e >= 0); + assert(!PyErr_Occurred()); /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ result = func(x) + func(2.0) * e; } @@ -2321,6 +2288,48 @@ math_log10(PyObject *module, PyObject *x) } +/*[clinic input] +math.fma + + x: double + y: double + z: double + / + +Fused multiply-add operation. + +Compute (x * y) + z with a single round. +[clinic start generated code]*/ + +static PyObject * +math_fma_impl(PyObject *module, double x, double y, double z) +/*[clinic end generated code: output=4fc8626dbc278d17 input=e3ad1f4a4c89626e]*/ +{ + double r = fma(x, y, z); + + /* Fast path: if we got a finite result, we're done. */ + if (isfinite(r)) { + return PyFloat_FromDouble(r); + } + + /* Non-finite result. Raise an exception if appropriate, else return r. */ + if (isnan(r)) { + if (!isnan(x) && !isnan(y) && !isnan(z)) { + /* NaN result from non-NaN inputs. */ + PyErr_SetString(PyExc_ValueError, "invalid operation in fma"); + return NULL; + } + } + else if (isfinite(x) && isfinite(y) && isfinite(z)) { + /* Infinite result from finite inputs. */ + PyErr_SetString(PyExc_OverflowError, "overflow in fma"); + return NULL; + } + + return PyFloat_FromDouble(r); +} + + /*[clinic input] math.fmod @@ -2339,12 +2348,21 @@ math_fmod_impl(PyObject *module, double x, double y) { double r; /* fmod(x, +/-Inf) returns x for finite x. */ - if (Py_IS_INFINITY(y) && Py_IS_FINITE(x)) + if (isinf(y) && isfinite(x)) return PyFloat_FromDouble(x); errno = 0; r = fmod(x, y); - if (Py_IS_NAN(r)) { - if (!Py_IS_NAN(x) && !Py_IS_NAN(y)) +#ifdef _MSC_VER + /* Windows (e.g. Windows 10 with MSC v.1916) loose sign + for zero result. But C99+ says: "if y is nonzero, the result + has the same sign as x". + */ + if (r == 0.0 && y != 0.0) { + r = copysign(r, x); + } +#endif + if (isnan(r)) { + if (!isnan(x) && !isnan(y)) errno = EDOM; else errno = 0; @@ -2411,7 +2429,7 @@ Since lo**2 is less than 1/2 ulp(csum), we have csum+lo*lo == csum. To minimize loss of information during the accumulation of fractional values, each term has a separate accumulator. This also breaks up sequential dependencies in the inner loop so the CPU can maximize -floating point throughput. [4] On an Apple M1 Max, hypot(*vec) +floating-point throughput. [4] On an Apple M1 Max, hypot(*vec) takes only 3.33 µsec when len(vec) == 1000. The square root differential correction is needed because a @@ -2466,7 +2484,7 @@ vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) int max_e; Py_ssize_t i; - if (Py_IS_INFINITY(max)) { + if (isinf(max)) { return max; } if (found_nan) { @@ -2488,7 +2506,7 @@ vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) assert(max * scale < 1.0); for (i=0 ; i < n ; i++) { x = vec[i]; - assert(Py_IS_FINITE(x) && fabs(x) <= max); + assert(isfinite(x) && fabs(x) <= max); x *= scale; // lossless scaling assert(fabs(x) < 1.0); pr = dl_mul(x, x); // lossless squaring @@ -2578,7 +2596,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) ASSIGN_DOUBLE(qx, item, error_exit); x = fabs(px - qx); diffs[i] = x; - found_nan |= Py_IS_NAN(x); + found_nan |= isnan(x); if (x > max) { max = x; } @@ -2608,9 +2626,28 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) return NULL; } -/* AC: cannot convert yet, waiting for *args support */ +/*[clinic input] +math.hypot + + *coordinates as args: object + +Multidimensional Euclidean distance from the origin to a point. + +Roughly equivalent to: + sqrt(sum(x**2 for x in coordinates)) + +For a two dimensional point (x, y), gives the hypotenuse +using the Pythagorean theorem: sqrt(x*x + y*y). + +For example, the hypotenuse of a 3/4/5 right triangle is: + + >>> hypot(3.0, 4.0) + 5.0 +[clinic start generated code]*/ + static PyObject * -math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +math_hypot_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=dcb6d4b7a1102ee1 input=5c0061a2d11235ed]*/ { Py_ssize_t i; PyObject *item; @@ -2631,7 +2668,7 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) ASSIGN_DOUBLE(x, item, error_exit); x = fabs(x); coordinates[i] = x; - found_nan |= Py_IS_NAN(x); + found_nan |= isnan(x); if (x > max) { max = x; } @@ -2651,22 +2688,6 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #undef NUM_STACK_ELEMS -PyDoc_STRVAR(math_hypot_doc, - "hypot(*coordinates) -> value\n\n\ -Multidimensional Euclidean distance from the origin to a point.\n\ -\n\ -Roughly equivalent to:\n\ - sqrt(sum(x**2 for x in coordinates))\n\ -\n\ -For a two dimensional point (x, y), gives the hypotenuse\n\ -using the Pythagorean theorem: sqrt(x*x + y*y).\n\ -\n\ -For example, the hypotenuse of a 3/4/5 right triangle is:\n\ -\n\ - >>> hypot(3.0, 4.0)\n\ - 5.0\n\ -"); - /** sumprod() ***************************************************************/ /* Forward declaration */ @@ -2934,14 +2955,14 @@ math_pow_impl(PyObject *module, double x, double y) /* deal directly with IEEE specials, to cope with problems on various platforms whose semantics don't exactly match C99 */ r = 0.; /* silence compiler warning */ - if (!Py_IS_FINITE(x) || !Py_IS_FINITE(y)) { + if (!isfinite(x) || !isfinite(y)) { errno = 0; - if (Py_IS_NAN(x)) + if (isnan(x)) r = y == 0. ? 1. : x; /* NaN**0 = 1 */ - else if (Py_IS_NAN(y)) + else if (isnan(y)) r = x == 1. ? 1. : y; /* 1**NaN = 1 */ - else if (Py_IS_INFINITY(x)) { - odd_y = Py_IS_FINITE(y) && fmod(fabs(y), 2.0) == 1.0; + else if (isinf(x)) { + odd_y = isfinite(y) && fmod(fabs(y), 2.0) == 1.0; if (y > 0.) r = odd_y ? x : fabs(x); else if (y == 0.) @@ -2950,7 +2971,7 @@ math_pow_impl(PyObject *module, double x, double y) r = odd_y ? copysign(0., x) : 0.; } else { - assert(Py_IS_INFINITY(y)); + assert(isinf(y)); if (fabs(x) == 1.0) r = 1.; else if (y > 0. && fabs(x) > 1.0) @@ -2968,8 +2989,8 @@ math_pow_impl(PyObject *module, double x, double y) r = pow(x, y); /* a NaN result should arise only from (-ve)**(finite non-integer); in this case we want to raise ValueError. */ - if (!Py_IS_FINITE(r)) { - if (Py_IS_NAN(r)) { + if (!isfinite(r)) { + if (isnan(r)) { errno = EDOM; } /* @@ -2977,7 +2998,7 @@ math_pow_impl(PyObject *module, double x, double y) (A) (+/-0.)**negative (-> divide-by-zero) (B) overflow of x**y with x and y finite */ - else if (Py_IS_INFINITY(r)) { + else if (isinf(r)) { if (x == 0.) errno = EDOM; else @@ -3043,7 +3064,7 @@ static PyObject * math_isfinite_impl(PyObject *module, double x) /*[clinic end generated code: output=8ba1f396440c9901 input=46967d254812e54a]*/ { - return PyBool_FromLong((long)Py_IS_FINITE(x)); + return PyBool_FromLong((long)isfinite(x)); } @@ -3060,7 +3081,7 @@ static PyObject * math_isnan_impl(PyObject *module, double x) /*[clinic end generated code: output=f537b4d6df878c3e input=935891e66083f46a]*/ { - return PyBool_FromLong((long)Py_IS_NAN(x)); + return PyBool_FromLong((long)isnan(x)); } @@ -3077,7 +3098,7 @@ static PyObject * math_isinf_impl(PyObject *module, double x) /*[clinic end generated code: output=9f00cbec4de7b06b input=32630e4212cf961f]*/ { - return PyBool_FromLong((long)Py_IS_INFINITY(x)); + return PyBool_FromLong((long)isinf(x)); } @@ -3094,7 +3115,7 @@ math.isclose -> bool maximum difference for being considered "close", regardless of the magnitude of the input values -Determine whether two floating point numbers are close in value. +Determine whether two floating-point numbers are close in value. Return True if a is close in value to b, and False otherwise. @@ -3109,7 +3130,7 @@ only close to themselves. static int math_isclose_impl(PyObject *module, double a, double b, double rel_tol, double abs_tol) -/*[clinic end generated code: output=b73070207511952d input=f28671871ea5bfba]*/ +/*[clinic end generated code: output=b73070207511952d input=12d41764468bfdb8]*/ { double diff = 0.0; @@ -3134,7 +3155,7 @@ math_isclose_impl(PyObject *module, double a, double b, double rel_tol, above. */ - if (Py_IS_INFINITY(a) || Py_IS_INFINITY(b)) { + if (isinf(a) || isinf(b)) { return 0; } @@ -3884,10 +3905,10 @@ math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) Bug fixed in bos.adt.libm 7.2.2.0 by APAR IV95512. */ return PyFloat_FromDouble(y); } - if (Py_IS_NAN(x)) { + if (isnan(x)) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return PyFloat_FromDouble(y); } #endif @@ -3933,10 +3954,10 @@ math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) if (usteps == 0) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(x)) { + if (isnan(x)) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return PyFloat_FromDouble(y); } @@ -4002,16 +4023,16 @@ static double math_ulp_impl(PyObject *module, double x) /*[clinic end generated code: output=f5207867a9384dd4 input=31f9bfbbe373fcaa]*/ { - if (Py_IS_NAN(x)) { + if (isnan(x)) { return x; } x = fabs(x); - if (Py_IS_INFINITY(x)) { + if (isinf(x)) { return x; } double inf = Py_INFINITY; double x2 = nextafter(x, inf); - if (Py_IS_INFINITY(x2)) { + if (isinf(x2)) { /* special case: x is the largest positive representable float */ x2 = nextafter(x, -inf); return x - x2; @@ -4094,18 +4115,19 @@ static PyMethodDef math_methods[] = { {"fabs", math_fabs, METH_O, math_fabs_doc}, MATH_FACTORIAL_METHODDEF MATH_FLOOR_METHODDEF + MATH_FMA_METHODDEF MATH_FMOD_METHODDEF MATH_FREXP_METHODDEF MATH_FSUM_METHODDEF {"gamma", math_gamma, METH_O, math_gamma_doc}, - {"gcd", _PyCFunction_CAST(math_gcd), METH_FASTCALL, math_gcd_doc}, - {"hypot", _PyCFunction_CAST(math_hypot), METH_FASTCALL, math_hypot_doc}, + MATH_GCD_METHODDEF + MATH_HYPOT_METHODDEF MATH_ISCLOSE_METHODDEF MATH_ISFINITE_METHODDEF MATH_ISINF_METHODDEF MATH_ISNAN_METHODDEF MATH_ISQRT_METHODDEF - {"lcm", _PyCFunction_CAST(math_lcm), METH_FASTCALL, math_lcm_doc}, + MATH_LCM_METHODDEF MATH_LDEXP_METHODDEF {"lgamma", math_lgamma, METH_O, math_lgamma_doc}, {"log", _PyCFunction_CAST(math_log), METH_FASTCALL, math_log_doc}, @@ -4134,6 +4156,7 @@ static PyMethodDef math_methods[] = { static PyModuleDef_Slot math_slots[] = { {Py_mod_exec, math_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/md5module.c b/Modules/md5module.c index 7d2b3275f213fd..ef9163e8be5b6c 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -51,7 +51,7 @@ typedef struct { // Prevents undefined behavior via multiple threads entering the C API. bool use_mutex; PyMutex mutex; - Hacl_Streaming_MD5_state *hash_state; + Hacl_Hash_MD5_state_t *hash_state; } MD5object; #include "clinic/md5module.c.h" @@ -93,7 +93,7 @@ MD5_traverse(PyObject *ptr, visitproc visit, void *arg) static void MD5_dealloc(MD5object *ptr) { - Hacl_Streaming_MD5_legacy_free(ptr->hash_state); + Hacl_Hash_MD5_free(ptr->hash_state); PyTypeObject *tp = Py_TYPE((PyObject*)ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -122,7 +122,7 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls) return NULL; ENTER_HASHLIB(self); - newobj->hash_state = Hacl_Streaming_MD5_legacy_copy(self->hash_state); + newobj->hash_state = Hacl_Hash_MD5_copy(self->hash_state); LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -139,7 +139,7 @@ MD5Type_digest_impl(MD5object *self) { unsigned char digest[MD5_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + Hacl_Hash_MD5_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, MD5_DIGESTSIZE); } @@ -156,7 +156,7 @@ MD5Type_hexdigest_impl(MD5object *self) { unsigned char digest[MD5_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + Hacl_Hash_MD5_digest(self->hash_state, digest); LEAVE_HASHLIB(self); const char *hexdigits = "0123456789abcdef"; @@ -170,15 +170,15 @@ MD5Type_hexdigest_impl(MD5object *self) return PyUnicode_FromStringAndSize(digest_hex, sizeof(digest_hex)); } -static void update(Hacl_Streaming_MD5_state *state, uint8_t *buf, Py_ssize_t len) { +static void update(Hacl_Hash_MD5_state_t *state, uint8_t *buf, Py_ssize_t len) { #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_MD5_legacy_update(state, buf, UINT32_MAX); + Hacl_Hash_MD5_update(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif - Hacl_Streaming_MD5_legacy_update(state, buf, (uint32_t) len); + Hacl_Hash_MD5_update(state, buf, (uint32_t) len); } /*[clinic input] @@ -302,7 +302,7 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->hash_state = Hacl_Streaming_MD5_legacy_create_in(); + new->hash_state = Hacl_Hash_MD5_malloc(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -375,6 +375,7 @@ md5_exec(PyObject *m) static PyModuleDef_Slot _md5_slots[] = { {Py_mod_exec, md5_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 0cce7c27f9b16a..e1c26e19932664 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -41,6 +41,7 @@ #ifdef MS_WINDOWS #include +#include // LsaNtStatusToWinError static int my_getpagesize(void) { @@ -255,6 +256,208 @@ do { \ } while (0) #endif /* UNIX */ +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +static DWORD +filter_page_exception(EXCEPTION_POINTERS *ptrs, EXCEPTION_RECORD *record) +{ + *record = *ptrs->ExceptionRecord; + if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || + record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) + { + return EXCEPTION_EXECUTE_HANDLER; + } + return EXCEPTION_CONTINUE_SEARCH; +} + +static DWORD +filter_page_exception_method(mmap_object *self, EXCEPTION_POINTERS *ptrs, + EXCEPTION_RECORD *record) +{ + *record = *ptrs->ExceptionRecord; + if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || + record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) + { + + ULONG_PTR address = record->ExceptionInformation[1]; + if (address >= (ULONG_PTR) self->data && + address < (ULONG_PTR) self->data + (ULONG_PTR) self->size) + { + return EXCEPTION_EXECUTE_HANDLER; + } + } + return EXCEPTION_CONTINUE_SEARCH; +} +#endif + +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +#define HANDLE_INVALID_MEM(sourcecode) \ +do { \ + EXCEPTION_RECORD record; \ + __try { \ + sourcecode \ + } \ + __except (filter_page_exception(GetExceptionInformation(), &record)) { \ + assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ + record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ + if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ + NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ + ULONG code = LsaNtStatusToWinError(status); \ + PyErr_SetFromWindowsErr(code); \ + } \ + else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ + PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ + } \ + return -1; \ + } \ +} while (0) +#else +#define HANDLE_INVALID_MEM(sourcecode) \ +do { \ + sourcecode \ +} while (0) +#endif + +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +#define HANDLE_INVALID_MEM_METHOD(self, sourcecode) \ +do { \ + EXCEPTION_RECORD record; \ + __try { \ + sourcecode \ + } \ + __except (filter_page_exception_method(self, GetExceptionInformation(), \ + &record)) { \ + assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ + record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ + if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ + NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ + ULONG code = LsaNtStatusToWinError(status); \ + PyErr_SetFromWindowsErr(code); \ + } \ + else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ + PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ + } \ + return -1; \ + } \ +} while (0) +#else +#define HANDLE_INVALID_MEM_METHOD(self, sourcecode) \ +do { \ + sourcecode \ +} while (0) +#endif + +int +safe_memcpy(void *dest, const void *src, size_t count) +{ + HANDLE_INVALID_MEM( + memcpy(dest, src, count); + ); + return 0; +} + +int +safe_byte_copy(char *dest, const char *src) +{ + HANDLE_INVALID_MEM( + *dest = *src; + ); + return 0; +} + +int +safe_memchr(char **out, const void *ptr, int ch, size_t count) +{ + HANDLE_INVALID_MEM( + *out = (char *) memchr(ptr, ch, count); + ); + return 0; +} + +int +safe_memmove(void *dest, const void *src, size_t count) +{ + HANDLE_INVALID_MEM( + memmove(dest, src, count); + ); + return 0; +} + +int +safe_copy_from_slice(char *dest, const char *src, Py_ssize_t start, + Py_ssize_t step, Py_ssize_t slicelen) +{ + HANDLE_INVALID_MEM( + size_t cur; + Py_ssize_t i; + for (cur = start, i = 0; i < slicelen; cur += step, i++) { + dest[cur] = src[i]; + } + ); + return 0; +} + +int +safe_copy_to_slice(char *dest, const char *src, Py_ssize_t start, + Py_ssize_t step, Py_ssize_t slicelen) +{ + HANDLE_INVALID_MEM( + size_t cur; + Py_ssize_t i; + for (cur = start, i = 0; i < slicelen; cur += step, i++) { + dest[i] = src[cur]; + } + ); + return 0; +} + + +int +_safe_PyBytes_Find(Py_ssize_t *out, mmap_object *self, const char *haystack, + Py_ssize_t len_haystack, const char *needle, + Py_ssize_t len_needle, Py_ssize_t offset) +{ + HANDLE_INVALID_MEM_METHOD(self, + *out = _PyBytes_Find(haystack, len_haystack, needle, len_needle, offset); + ); + return 0; +} + +int +_safe_PyBytes_ReverseFind(Py_ssize_t *out, mmap_object *self, + const char *haystack, Py_ssize_t len_haystack, + const char *needle, Py_ssize_t len_needle, + Py_ssize_t offset) +{ + HANDLE_INVALID_MEM_METHOD(self, + *out = _PyBytes_ReverseFind(haystack, len_haystack, needle, len_needle, + offset); + ); + return 0; +} + +PyObject * +_safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { + if (num_bytes == 1) { + char dest; + if (safe_byte_copy(&dest, start) < 0) { + return NULL; + } + else { + return PyBytes_FromStringAndSize(&dest, 1); + } + } + else { + PyObject *result = PyBytes_FromStringAndSize(NULL, num_bytes); + if (result == NULL) { + return NULL; + } + if (safe_memcpy(PyBytes_AS_STRING(result), start, num_bytes) < 0) { + Py_CLEAR(result); + } + return result; + } +} + static PyObject * mmap_read_byte_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) @@ -264,7 +467,12 @@ mmap_read_byte_method(mmap_object *self, PyErr_SetString(PyExc_ValueError, "read byte out of range"); return NULL; } - return PyLong_FromLong((unsigned char)self->data[self->pos++]); + char dest; + if (safe_byte_copy(&dest, self->data + self->pos) < 0) { + return NULL; + } + self->pos++; + return PyLong_FromLong((unsigned char) dest); } static PyObject * @@ -273,21 +481,27 @@ mmap_read_line_method(mmap_object *self, { Py_ssize_t remaining; char *start, *eol; - PyObject *result; CHECK_VALID(NULL); remaining = (self->pos < self->size) ? self->size - self->pos : 0; if (!remaining) - return PyBytes_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); start = self->data + self->pos; - eol = memchr(start, '\n', remaining); + + if (safe_memchr(&eol, start, '\n', remaining) < 0) { + return NULL; + } + if (!eol) eol = self->data + self->size; else ++eol; /* advance past newline */ - result = PyBytes_FromStringAndSize(start, (eol - start)); - self->pos += (eol - start); + + PyObject *result = _safe_PyBytes_FromStringAndSize(start, eol - start); + if (result != NULL) { + self->pos += (eol - start); + } return result; } @@ -296,7 +510,6 @@ mmap_read_method(mmap_object *self, PyObject *args) { Py_ssize_t num_bytes = PY_SSIZE_T_MAX, remaining; - PyObject *result; CHECK_VALID(NULL); if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes)) @@ -307,8 +520,12 @@ mmap_read_method(mmap_object *self, remaining = (self->pos < self->size) ? self->size - self->pos : 0; if (num_bytes < 0 || num_bytes > remaining) num_bytes = remaining; - result = PyBytes_FromStringAndSize(&self->data[self->pos], num_bytes); - self->pos += num_bytes; + + PyObject *result = _safe_PyBytes_FromStringAndSize(self->data + self->pos, + num_bytes); + if (result != NULL) { + self->pos += num_bytes; + } return result; } @@ -341,25 +558,38 @@ mmap_gfind(mmap_object *self, else if (end > self->size) end = self->size; - Py_ssize_t res; + Py_ssize_t index; + PyObject *result; CHECK_VALID_OR_RELEASE(NULL, view); if (end < start) { - res = -1; + result = PyLong_FromSsize_t(-1); } else if (reverse) { assert(0 <= start && start <= end && end <= self->size); - res = _PyBytes_ReverseFind( + if (_safe_PyBytes_ReverseFind(&index, self, self->data + start, end - start, - view.buf, view.len, start); + view.buf, view.len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } } else { assert(0 <= start && start <= end && end <= self->size); - res = _PyBytes_Find( + if (_safe_PyBytes_Find(&index, self, self->data + start, end - start, - view.buf, view.len, start); + view.buf, view.len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } } PyBuffer_Release(&view); - return PyLong_FromSsize_t(res); + return result; } } @@ -432,10 +662,16 @@ mmap_write_method(mmap_object *self, } CHECK_VALID_OR_RELEASE(NULL, data); - memcpy(&self->data[self->pos], data.buf, data.len); - self->pos += data.len; + PyObject *result; + if (safe_memcpy(self->data + self->pos, data.buf, data.len) < 0) { + result = NULL; + } + else { + self->pos += data.len; + result = PyLong_FromSsize_t(data.len); + } PyBuffer_Release(&data); - return PyLong_FromSsize_t(data.len); + return result; } static PyObject * @@ -452,14 +688,16 @@ mmap_write_byte_method(mmap_object *self, return NULL; CHECK_VALID(NULL); - if (self->pos < self->size) { - self->data[self->pos++] = value; - Py_RETURN_NONE; - } - else { + if (self->pos >= self->size) { PyErr_SetString(PyExc_ValueError, "write byte out of range"); return NULL; } + + if (safe_byte_copy(self->data + self->pos, &value) < 0) { + return NULL; + } + self->pos++; + Py_RETURN_NONE; } static PyObject * @@ -763,8 +1001,9 @@ mmap_move_method(mmap_object *self, PyObject *args) goto bounds; CHECK_VALID(NULL); - memmove(&self->data[dest], &self->data[src], cnt); - + if (safe_memmove(self->data + dest, self->data + src, cnt) < 0) { + return NULL; + }; Py_RETURN_NONE; bounds: @@ -855,6 +1094,29 @@ mmap__sizeof__method(mmap_object *self, void *Py_UNUSED(ignored)) } #endif +#if defined(MS_WINDOWS) && defined(Py_DEBUG) +static PyObject * +mmap_protect_method(mmap_object *self, PyObject *args) { + DWORD flNewProtect, flOldProtect; + Py_ssize_t start, length; + + CHECK_VALID(NULL); + + if (!PyArg_ParseTuple(args, "Inn:protect", &flNewProtect, &start, &length)) { + return NULL; + } + + if (!VirtualProtect((void *) (self->data + start), length, flNewProtect, + &flOldProtect)) + { + PyErr_SetFromWindowsErr(GetLastError()); + return NULL; + } + + Py_RETURN_NONE; +} +#endif + #ifdef HAVE_MADVISE static PyObject * mmap_madvise_method(mmap_object *self, PyObject *args) @@ -924,7 +1186,10 @@ static struct PyMethodDef mmap_object_methods[] = { {"__exit__", (PyCFunction) mmap__exit__method, METH_VARARGS}, #ifdef MS_WINDOWS {"__sizeof__", (PyCFunction) mmap__sizeof__method, METH_NOARGS}, -#endif +#ifdef Py_DEBUG + {"_protect", (PyCFunction) mmap_protect_method, METH_VARARGS}, +#endif // Py_DEBUG +#endif // MS_WINDOWS {NULL, NULL} /* sentinel */ }; @@ -968,7 +1233,12 @@ mmap_item(mmap_object *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "mmap index out of range"); return NULL; } - return PyBytes_FromStringAndSize(self->data + i, 1); + + char dest; + if (safe_byte_copy(&dest, self->data + i) < 0) { + return NULL; + } + return PyBytes_FromStringAndSize(&dest, 1); } static PyObject * @@ -987,7 +1257,12 @@ mmap_subscript(mmap_object *self, PyObject *item) return NULL; } CHECK_VALID(NULL); - return PyLong_FromLong(Py_CHARMASK(self->data[i])); + + char dest; + if (safe_byte_copy(&dest, self->data + i) < 0) { + return NULL; + } + return PyLong_FromLong(Py_CHARMASK(dest)); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; @@ -999,25 +1274,24 @@ mmap_subscript(mmap_object *self, PyObject *item) CHECK_VALID(NULL); if (slicelen <= 0) - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); else if (step == 1) - return PyBytes_FromStringAndSize(self->data + start, - slicelen); + return _safe_PyBytes_FromStringAndSize(self->data + start, slicelen); else { char *result_buf = (char *)PyMem_Malloc(slicelen); - size_t cur; - Py_ssize_t i; PyObject *result; if (result_buf == NULL) return PyErr_NoMemory(); - for (cur = start, i = 0; i < slicelen; - cur += step, i++) { - result_buf[i] = self->data[cur]; + if (safe_copy_to_slice(result_buf, self->data, start, step, + slicelen) < 0) + { + result = NULL; + } + else { + result = PyBytes_FromStringAndSize(result_buf, slicelen); } - result = PyBytes_FromStringAndSize(result_buf, - slicelen); PyMem_Free(result_buf); return result; } @@ -1052,7 +1326,10 @@ mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v) if (!is_writable(self)) return -1; buf = PyBytes_AsString(v); - self->data[i] = buf[0]; + + if (safe_byte_copy(self->data + i, buf) < 0) { + return -1; + } return 0; } @@ -1097,7 +1374,11 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) return -1; } CHECK_VALID(-1); - self->data[i] = (char) v; + + char v_char = (char) v; + if (safe_byte_copy(self->data + i, &v_char) < 0) { + return -1; + } return 0; } else if (PySlice_Check(item)) { @@ -1123,24 +1404,23 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) } CHECK_VALID_OR_RELEASE(-1, vbuf); + int result = 0; if (slicelen == 0) { } else if (step == 1) { - memcpy(self->data + start, vbuf.buf, slicelen); + if (safe_memcpy(self->data + start, vbuf.buf, slicelen) < 0) { + result = -1; + } } else { - size_t cur; - Py_ssize_t i; - - for (cur = start, i = 0; - i < slicelen; - cur += step, i++) + if (safe_copy_from_slice(self->data, (char *)vbuf.buf, start, step, + slicelen) < 0) { - self->data[cur] = ((char *)vbuf.buf)[i]; + result = -1; } } PyBuffer_Release(&vbuf); - return 0; + return result; } else { PyErr_SetString(PyExc_TypeError, @@ -1801,6 +2081,7 @@ mmap_exec(PyObject *module) static PyModuleDef_Slot mmap_slots[] = { {Py_mod_exec, mmap_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index fd40e91d0f50c4..308a0dab7fab1a 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -723,6 +723,24 @@ Overlapped_dealloc(OverlappedObject *self) if (!HasOverlappedIoCompleted(&self->overlapped) && self->type != TYPE_NOT_STARTED) { + // NOTE: We should not get here, if we do then something is wrong in + // the IocpProactor or ProactorEventLoop. Since everything uses IOCP if + // the overlapped IO hasn't completed yet then we should not be + // deallocating! + // + // The problem is likely that this OverlappedObject was removed from + // the IocpProactor._cache before it was complete. The _cache holds a + // reference while IO is pending so that it does not get deallocated + // while the kernel has retained the OVERLAPPED structure. + // + // CancelIoEx (likely called from self.cancel()) may have successfully + // completed, but the OVERLAPPED is still in use until either + // HasOverlappedIoCompleted() is true or GetQueuedCompletionStatus has + // returned this OVERLAPPED object. + // + // NOTE: Waiting when IOCP is in use can hang indefinitely, but this + // CancelIoEx is superfluous in that self.cancel() was already called, + // so I've only ever seen this return FALSE with GLE=ERROR_NOT_FOUND Py_BEGIN_ALLOW_THREADS if (CancelIoEx(self->handle, &self->overlapped)) wait = TRUE; @@ -905,7 +923,7 @@ _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) { break; } - /* fall through */ + _Py_FALLTHROUGH; default: return SetFromWindowsErr(err); } @@ -2038,6 +2056,7 @@ overlapped_exec(PyObject *module) WINAPI_CONSTANT(F_DWORD, ERROR_OPERATION_ABORTED); WINAPI_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); WINAPI_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); + WINAPI_CONSTANT(F_DWORD, ERROR_PORT_UNREACHABLE); WINAPI_CONSTANT(F_DWORD, INFINITE); WINAPI_CONSTANT(F_HANDLE, INVALID_HANDLE_VALUE); WINAPI_CONSTANT(F_HANDLE, NULL); @@ -2051,6 +2070,7 @@ overlapped_exec(PyObject *module) static PyModuleDef_Slot overlapped_slots[] = { {Py_mod_exec, overlapped_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d05b4ba723ce8c..bb5077cc7f0f09 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16,13 +16,14 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_ReInitThreads() #include "pycore_fileutils.h" // _Py_closerange() -#include "pycore_import.h" // _PyImport_ReInitLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_LookupSpecial() #include "pycore_pylifecycle.h" // _PyOS_URandom() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG +#include "pycore_time.h" // _PyLong_FromTime_t() #ifdef HAVE_UNISTD_H # include // symlink() @@ -36,6 +37,8 @@ # include # include // UNLEN # include "osdefs.h" // SEP +# include // SetEntriesInAcl +# include // SDDL_REVISION_1 # if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SYMLINK # endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ @@ -122,6 +125,7 @@ # define HAVE_PWRITEV_RUNTIME __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) # define HAVE_MKFIFOAT_RUNTIME __builtin_available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) # define HAVE_MKNODAT_RUNTIME __builtin_available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) +# define HAVE_PTSNAME_R_RUNTIME __builtin_available(macOS 10.13.4, iOS 11.3, tvOS 11.3, watchOS 4.3, *) # define HAVE_POSIX_SPAWN_SETSID_RUNTIME __builtin_available(macOS 10.15, *) @@ -203,6 +207,10 @@ # define HAVE_MKNODAT_RUNTIME (mknodat != NULL) # endif +# ifdef HAVE_PTSNAME_R +# define HAVE_PTSNAME_R_RUNTIME (ptsname_r != NULL) +# endif + #endif #ifdef HAVE_FUTIMESAT @@ -228,6 +236,7 @@ # define HAVE_PWRITEV_RUNTIME 1 # define HAVE_MKFIFOAT_RUNTIME 1 # define HAVE_MKNODAT_RUNTIME 1 +# define HAVE_PTSNAME_R_RUNTIME 1 #endif @@ -612,16 +621,18 @@ PyOS_BeforeFork(void) run_at_forkers(interp->before_forkers, 1); _PyImport_AcquireLock(interp); + _PyEval_StopTheWorldAll(&_PyRuntime); + HEAD_LOCK(&_PyRuntime); } void PyOS_AfterFork_Parent(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_PyImport_ReleaseLock(interp) <= 0) { - Py_FatalError("failed releasing import lock after fork"); - } + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorldAll(&_PyRuntime); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyImport_ReleaseLock(interp); run_at_forkers(interp->after_forkers_parent, 0); } @@ -631,6 +642,7 @@ PyOS_AfterFork_Child(void) PyStatus status; _PyRuntimeState *runtime = &_PyRuntime; + // re-creates runtime->interpreters.mutex (HEAD_UNLOCK) status = _PyRuntimeState_ReInitThreads(runtime); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; @@ -639,23 +651,33 @@ PyOS_AfterFork_Child(void) PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); + assert(tstate->thread_id == PyThread_get_thread_ident()); #ifdef PY_HAVE_THREAD_NATIVE_ID tstate->native_thread_id = PyThread_get_thread_native_id(); #endif #ifdef Py_GIL_DISABLED _Py_brc_after_fork(tstate->interp); + _Py_qsbr_after_fork((_PyThreadStateImpl *)tstate); #endif + // Ideally we could guarantee tstate is running main. + _PyInterpreterState_ReinitRunningMain(tstate); + status = _PyEval_ReInitThreads(tstate); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; } - status = _PyImport_ReInitLock(tstate->interp); - if (_PyStatus_EXCEPTION(status)) { - goto fatal_error; - } + // Remove the dead thread states. We "start the world" once we are the only + // thread state left to undo the stop the world call in `PyOS_BeforeFork`. + // That needs to happen before `_PyThreadState_DeleteList`, because that + // may call destructors. + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + _PyEval_StartTheWorldAll(&_PyRuntime); + _PyThreadState_DeleteList(list); + + _PyImport_ReleaseLock(tstate->interp); _PySignal_AfterFork(); @@ -945,16 +967,46 @@ _Py_Gid_Converter(PyObject *obj, gid_t *p) #endif /* MS_WINDOWS */ -#define _PyLong_FromDev PyLong_FromLongLong +static PyObject * +_PyLong_FromDev(dev_t dev) +{ +#ifdef NODEV + if (dev == NODEV) { + return PyLong_FromLongLong((long long)dev); + } +#endif + return PyLong_FromUnsignedLongLong((unsigned long long)dev); +} #if (defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV)) || defined(HAVE_DEVICE_MACROS) static int _Py_Dev_Converter(PyObject *obj, void *p) { - *((dev_t *)p) = PyLong_AsUnsignedLongLong(obj); - if (PyErr_Occurred()) +#ifdef NODEV + if (PyLong_Check(obj) && _PyLong_IsNegative((PyLongObject *)obj)) { + int overflow; + long long result = PyLong_AsLongLongAndOverflow(obj, &overflow); + if (result == -1 && PyErr_Occurred()) { + return 0; + } + if (!overflow && result == (long long)NODEV) { + *((dev_t *)p) = NODEV; + return 1; + } + } +#endif + + unsigned long long result = PyLong_AsUnsignedLongLong(obj); + if (result == (unsigned long long)-1 && PyErr_Occurred()) { + return 0; + } + if ((unsigned long long)(dev_t)result != result) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C dev_t"); return 0; + } + *((dev_t *)p) = (dev_t)result; return 1; } #endif /* (HAVE_MKNOD && HAVE_MAKEDEV) || HAVE_DEVICE_MACROS */ @@ -1070,16 +1122,15 @@ get_posix_state(PyObject *module) * * path_converter accepts (Unicode) strings and their * subclasses, and bytes and their subclasses. What - * it does with the argument depends on the platform: + * it does with the argument depends on path.make_wide: * - * * On Windows, if we get a (Unicode) string we - * extract the wchar_t * and return it; if we get - * bytes we decode to wchar_t * and return that. + * * If path.make_wide is nonzero, if we get a (Unicode) + * string we extract the wchar_t * and return it; if we + * get bytes we decode to wchar_t * and return that. * - * * On all other platforms, strings are encoded - * to bytes using PyUnicode_FSConverter, then we - * extract the char * from the bytes object and - * return that. + * * If path.make_wide is zero, if we get bytes we extract + * the char_t * and return it; if we get a (Unicode) + * string we encode to char_t * and return that. * * path_converter also optionally accepts signed * integers (representing open file descriptors) instead @@ -1088,6 +1139,15 @@ get_posix_state(PyObject *module) * Input fields: * path.nullable * If nonzero, the path is permitted to be None. + * path.nonstrict + * If nonzero, the path is permitted to contain + * embedded null characters and have any length. + * path.make_wide + * If nonzero, the converter always uses wide, decoding if necessary, else + * it always uses narrow, encoding if necessary. The default value is + * nonzero on Windows, else zero. + * path.suppress_value_error + * If nonzero, raising ValueError is suppressed. * path.allow_fd * If nonzero, the path is permitted to be a file handle * (a signed int) instead of a string. @@ -1103,12 +1163,10 @@ get_posix_state(PyObject *module) * Output fields: * path.wide * Points to the path if it was expressed as Unicode - * and was not encoded. (Only used on Windows.) + * or if it was bytes and decoded to Unicode. * path.narrow * Points to the path if it was expressed as bytes, - * or it was Unicode and was encoded to bytes. (On Windows, - * is a non-zero integer if the path was expressed as bytes. - * The type is deliberately incompatible to prevent misuse.) + * or if it was Unicode and encoded to bytes. * path.fd * Contains a file descriptor if path.accept_fd was true * and the caller provided a signed integer instead of any @@ -1118,6 +1176,9 @@ get_posix_state(PyObject *module) * unspecified, path_converter will never get called. * So if you set allow_fd, you *MUST* initialize path.fd = -1 * yourself! + * path.value_error + * If nonzero, then suppress_value_error was specified and a ValueError + * occurred. * path.length * The length of the path in characters, if specified as * a string. @@ -1150,28 +1211,38 @@ get_posix_state(PyObject *module) * path_cleanup(). However it is safe to do so.) */ typedef struct { + // Input fields const char *function_name; const char *argument_name; int nullable; + int nonstrict; + int make_wide; + int suppress_value_error; int allow_fd; + // Output fields const wchar_t *wide; -#ifdef MS_WINDOWS - BOOL narrow; -#else const char *narrow; -#endif int fd; + int value_error; Py_ssize_t length; PyObject *object; PyObject *cleanup; } path_t; +#define PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, \ + make_wide, suppress_value_error, allow_fd) \ + {function_name, argument_name, nullable, nonstrict, make_wide, \ + suppress_value_error, allow_fd, NULL, NULL, -1, 0, 0, NULL, NULL} #ifdef MS_WINDOWS -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, FALSE, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 1, \ + suppress_value_error, allow_fd) #else -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, NULL, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 0, \ + suppress_value_error, allow_fd) #endif static void @@ -1192,10 +1263,8 @@ path_converter(PyObject *o, void *p) Py_ssize_t length = 0; int is_index, is_bytes, is_unicode; const char *narrow; -#ifdef MS_WINDOWS PyObject *wo = NULL; wchar_t *wide = NULL; -#endif #define FORMAT_EXCEPTION(exc, fmt) \ PyErr_Format(exc, "%s%s" fmt, \ @@ -1216,11 +1285,7 @@ path_converter(PyObject *o, void *p) if ((o == Py_None) && path->nullable) { path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif path->fd = -1; goto success_exit; } @@ -1264,30 +1329,33 @@ path_converter(PyObject *o, void *p) } if (is_unicode) { + if (path->make_wide) { + wide = PyUnicode_AsWideCharString(o, &length); + if (!wide) { + goto error_exit; + } #ifdef MS_WINDOWS - wide = PyUnicode_AsWideCharString(o, &length); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } - path->wide = wide; - path->narrow = FALSE; - path->fd = -1; - wide = NULL; - goto success_exit; -#else - if (!PyUnicode_FSConverter(o, &bytes)) { + path->wide = wide; + path->narrow = NULL; + path->fd = -1; + wide = NULL; + goto success_exit; + } + bytes = PyUnicode_EncodeFSDefault(o); + if (!bytes) { goto error_exit; } -#endif } else if (is_bytes) { bytes = Py_NewRef(o); @@ -1297,11 +1365,7 @@ path_converter(PyObject *o, void *p) goto error_exit; } path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif goto success_exit; } else { @@ -1321,52 +1385,54 @@ path_converter(PyObject *o, void *p) length = PyBytes_GET_SIZE(bytes); narrow = PyBytes_AS_STRING(bytes); - if ((size_t)length != strlen(narrow)) { + if (!path->nonstrict && strlen(narrow) != (size_t)length) { FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); goto error_exit; } -#ifdef MS_WINDOWS - wo = PyUnicode_DecodeFSDefaultAndSize( - narrow, - length - ); - if (!wo) { - goto error_exit; - } + if (path->make_wide) { + wo = PyUnicode_DecodeFSDefaultAndSize(narrow, length); + if (!wo) { + goto error_exit; + } - wide = PyUnicode_AsWideCharString(wo, &length); - Py_DECREF(wo); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } - path->wide = wide; - path->narrow = TRUE; - Py_DECREF(bytes); - wide = NULL; -#else - path->wide = NULL; - path->narrow = narrow; - if (bytes == o) { - /* Still a reference owned by path->object, don't have to - worry about path->narrow is used after free. */ + wide = PyUnicode_AsWideCharString(wo, &length); + Py_DECREF(wo); + if (!wide) { + goto error_exit; + } +#ifdef MS_WINDOWS + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } + path->wide = wide; + path->narrow = NULL; Py_DECREF(bytes); + wide = NULL; } else { - path->cleanup = bytes; + path->wide = NULL; + path->narrow = narrow; + if (bytes == o) { + /* Still a reference owned by path->object, don't have to + worry about path->narrow is used after free. */ + Py_DECREF(bytes); + } + else { + path->cleanup = bytes; + } } -#endif path->fd = -1; success_exit: + path->value_error = 0; path->length = length; path->object = o; return Py_CLEANUP_SUPPORTED; @@ -1374,10 +1440,20 @@ path_converter(PyObject *o, void *p) error_exit: Py_XDECREF(o); Py_XDECREF(bytes); -#ifdef MS_WINDOWS PyMem_Free(wide); -#endif - return 0; + if (!path->suppress_value_error || + !PyErr_ExceptionMatches(PyExc_ValueError)) + { + return 0; + } + PyErr_Clear(); + path->wide = NULL; + path->narrow = NULL; + path->fd = -1; + path->value_error = 1; + path->length = 0; + path->object = NULL; + return Py_CLEANUP_SUPPORTED; } static void @@ -1427,11 +1503,7 @@ follow_symlinks_specified(const char *function_name, int follow_symlinks) static int path_and_dir_fd_invalid(const char *function_name, path_t *path, int dir_fd) { - if (!path->wide && (dir_fd != DEFAULT_DIR_FD) -#ifndef MS_WINDOWS - && !path->narrow -#endif - ) { + if (!path->wide && (dir_fd != DEFAULT_DIR_FD) && !path->narrow) { PyErr_Format(PyExc_ValueError, "%s: can't specify dir_fd without matching path", function_name); @@ -1478,6 +1550,19 @@ dir_fd_and_follow_symlinks_invalid(const char *function_name, int dir_fd, return 0; } +#if defined(HAVE_WAITID) +static int +idtype_t_converter(PyObject *arg, void *addr) +{ + int value = PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) { + return 0; + } + *((idtype_t *)addr) = (idtype_t)(value); + return 1; +} +#endif + #ifdef MS_WINDOWS typedef long long Py_off_t; #else @@ -2891,7 +2976,9 @@ class path_t_converter(CConverter): converter = 'path_converter' - def converter_init(self, *, allow_fd=False, nullable=False): + def converter_init(self, *, allow_fd=False, make_wide=None, + nonstrict=False, nullable=False, + suppress_value_error=False): # right now path_t doesn't support default values. # to support a default value, you'll need to override initialize(). if self.default not in (unspecified, None): @@ -2901,6 +2988,9 @@ class path_t_converter(CConverter): raise RuntimeError("Can't specify a c_default to the path_t converter!") self.nullable = nullable + self.nonstrict = nonstrict + self.make_wide = make_wide + self.suppress_value_error = suppress_value_error self.allow_fd = allow_fd def pre_render(self): @@ -2910,11 +3000,24 @@ class path_t_converter(CConverter): return str(int(bool(value))) # add self.py_name here when merging with posixmodule conversion - self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {})'.format( - self.function.name, - self.name, - strify(self.nullable), - strify(self.allow_fd), + if self.make_wide is None: + self.c_default = 'PATH_T_INITIALIZE_P("{}", "{}", {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.suppress_value_error), + strify(self.allow_fd), + ) + else: + self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.make_wide), + strify(self.suppress_value_error), + strify(self.allow_fd), ) def cleanup(self): @@ -2964,17 +3067,42 @@ class pid_t_converter(CConverter): type = 'pid_t' format_unit = '" _Py_PARSE_PID "' -class idtype_t_converter(int_converter): + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" + {paramname} = PyLong_AsPid({argname}); + if ({paramname} == (pid_t)(-1) && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, argname=argname) + +class idtype_t_converter(CConverter): type = 'idtype_t' + converter = 'idtype_t_converter' class id_t_converter(CConverter): type = 'id_t' format_unit = '" _Py_PARSE_PID "' + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" + {paramname} = (id_t)PyLong_AsPid({argname}); + if ({paramname} == (id_t)(-1) && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, argname=argname) + class intptr_t_converter(CConverter): type = 'intptr_t' format_unit = '" _Py_PARSE_INTPTR "' + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" + {paramname} = (intptr_t)PyLong_AsVoidPtr({argname}); + if (!{paramname} && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, argname=argname) + class Py_off_t_converter(CConverter): type = 'Py_off_t' converter = 'Py_off_t_converter' @@ -2994,7 +3122,7 @@ class sysconf_confname_converter(path_confname_converter): converter="conv_sysconf_confname" [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3338733161aa7879]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=1860d32584c2a539]*/ /*[clinic input] @@ -4086,6 +4214,20 @@ posix_getcwd(int use_bytes) else { obj = PyUnicode_DecodeFSDefault(buf); } +#ifdef __linux__ + if (buf[0] != '/') { + /* + * On Linux >= 2.6.36 with glibc < 2.27, getcwd() can return a + * relative pathname starting with '(unreachable)'. We detect this + * and fail with ENOENT, matching newer glibc behaviour. + */ + errno = ENOENT; + path_object_error(obj); + PyMem_RawFree(buf); + return NULL; + } +#endif + assert(buf[0] == '/'); PyMem_RawFree(buf); return obj; @@ -4249,7 +4391,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) { PyObject *v; HANDLE hFindFile = INVALID_HANDLE_VALUE; - BOOL result; + BOOL result, return_bytes; wchar_t namebuf[MAX_PATH+4]; /* Overallocate for "\*.*" */ /* only claim to have space for MAX_PATH */ Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; @@ -4261,9 +4403,11 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) if (!path->wide) { /* Default arg: "." */ po_wchars = L"."; len = 1; + return_bytes = 0; } else { po_wchars = path->wide; len = wcslen(path->wide); + return_bytes = PyBytes_Check(path->object); } /* The +5 is so we can append "\\*.*\0" */ wnamebuf = PyMem_New(wchar_t, len + 5); @@ -4298,7 +4442,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) wcscmp(wFileData.cFileName, L"..") != 0) { v = PyUnicode_FromWideChar(wFileData.cFileName, wcslen(wFileData.cFileName)); - if (path->narrow && v) { + if (return_bytes && v) { Py_SETREF(v, PyUnicode_EncodeFSDefault(v)); } if (v == NULL) { @@ -4841,7 +4985,7 @@ os__getfullpathname_impl(PyObject *module, path_t *path) if (str == NULL) { return NULL; } - if (path->narrow) { + if (PyBytes_Check(path->object)) { Py_SETREF(str, PyUnicode_EncodeFSDefault(str)); } return str; @@ -4914,7 +5058,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) } result = PyUnicode_FromWideChar(target_path, result_length); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } @@ -4997,7 +5141,7 @@ os__getvolumepathname_impl(PyObject *module, path_t *path) goto exit; } result = PyUnicode_FromWideChar(mountpath, wcslen(mountpath)); - if (path->narrow) + if (PyBytes_Check(path->object)) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); exit: @@ -5052,413 +5196,426 @@ os__path_splitroot_impl(PyObject *module, path_t *path) } -/*[clinic input] -os._path_isdir - - s: 'O' +#define PY_IFREG 1 // Regular file +#define PY_IFDIR 2 // Directory +#define PY_IFLNK 4 // Symlink +#define PY_IFMNT 8 // Mount Point (junction) +#define PY_IFLRP 16 // Link Reparse Point (name-surrogate, symlink, junction) +#define PY_IFRRP 32 // Regular Reparse Point -Return true if the pathname refers to an existing directory. +static inline BOOL +_testInfo(DWORD attributes, DWORD reparseTag, BOOL diskDevice, int testedType) +{ + switch (testedType) { + case PY_IFREG: + return diskDevice && attributes && + !(attributes & FILE_ATTRIBUTE_DIRECTORY); + case PY_IFDIR: + return attributes & FILE_ATTRIBUTE_DIRECTORY; + case PY_IFLNK: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag == IO_REPARSE_TAG_SYMLINK; + case PY_IFMNT: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag == IO_REPARSE_TAG_MOUNT_POINT; + case PY_IFLRP: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + IsReparseTagNameSurrogate(reparseTag); + case PY_IFRRP: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag && !IsReparseTagNameSurrogate(reparseTag); + } -[clinic start generated code]*/ + return FALSE; +} -static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *s) -/*[clinic end generated code: output=9d87ab3c8b8a4e61 input=c17f7ef21d22d64e]*/ +static BOOL +_testFileTypeByHandle(HANDLE hfile, int testedType, BOOL diskOnly) { - HANDLE hfile; - BOOL close_file = TRUE; + assert(testedType == PY_IFREG || testedType == PY_IFDIR || + testedType == PY_IFLNK || testedType == PY_IFMNT || + testedType == PY_IFLRP || testedType == PY_IFRRP); + + BOOL diskDevice = GetFileType(hfile) == FILE_TYPE_DISK; + if (diskOnly && !diskDevice) { + return FALSE; + } + if (testedType != PY_IFREG && testedType != PY_IFDIR) { + FILE_ATTRIBUTE_TAG_INFO info; + return GetFileInformationByHandleEx(hfile, FileAttributeTagInfo, &info, + sizeof(info)) && + _testInfo(info.FileAttributes, info.ReparseTag, diskDevice, + testedType); + } FILE_BASIC_INFO info; - path_t _path = PATH_T_INITIALIZE("isdir", "s", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + return GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, + sizeof(info)) && + _testInfo(info.FileAttributes, 0, diskDevice, testedType); +} - if (!path_converter(s, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; +static BOOL +_testFileTypeByName(LPCWSTR path, int testedType) +{ + assert(testedType == PY_IFREG || testedType == PY_IFDIR || + testedType == PY_IFLNK || testedType == PY_IFMNT || + testedType == PY_IFLRP || testedType == PY_IFRRP); + + FILE_STAT_BASIC_INFORMATION info; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, &info, + sizeof(info))) + { + BOOL diskDevice = info.DeviceType == FILE_DEVICE_DISK || + info.DeviceType == FILE_DEVICE_VIRTUAL_DISK || + info.DeviceType == FILE_DEVICE_CD_ROM; + BOOL result = _testInfo(info.FileAttributes, info.ReparseTag, + diskDevice, testedType); + if (!result || (testedType != PY_IFREG && testedType != PY_IFDIR) || + !(info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) + { + return result; } - return NULL; + } + else if (_Py_GetFileInformationByName_ErrorIsTrustworthy( + GetLastError())) + { + return FALSE; } - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY; - } else if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY)) { - slow_path = FALSE; - result = 0; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (testedType != PY_IFREG && testedType != PY_IFDIR) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; + HANDLE hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, + OPEN_EXISTING, flags, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + BOOL result = _testFileTypeByHandle(hfile, testedType, FALSE); + CloseHandle(hfile); + return result; + } + + switch (GetLastError()) { + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + case ERROR_CANT_ACCESS_FILE: + case ERROR_INVALID_PARAMETER: + int rc; + STRUCT_STAT st; + if (testedType == PY_IFREG || testedType == PY_IFDIR) { + rc = STAT(path, &st); } else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - } - if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, - sizeof(info))) - { - result = info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY; - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } + // PY_IFRRP is not generally supported in this case, except for + // unhandled reparse points such as IO_REPARSE_TAG_APPEXECLINK. + rc = LSTAT(path, &st); } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISDIR(st.st_mode); - } - break; - default: - result = 0; - } + if (!rc) { + return _testInfo(st.st_file_attributes, st.st_reparse_tag, + st.st_mode & S_IFREG, testedType); } } - Py_END_ALLOW_THREADS - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return FALSE; } -/*[clinic input] -os._path_isfile +static BOOL +_testFileExistsByName(LPCWSTR path, BOOL followLinks) +{ + FILE_STAT_BASIC_INFORMATION info; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, &info, + sizeof(info))) + { + if (!(info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) || + !followLinks && IsReparseTagNameSurrogate(info.ReparseTag)) + { + return TRUE; + } + } + else if (_Py_GetFileInformationByName_ErrorIsTrustworthy( + GetLastError())) + { + return FALSE; + } - path: 'O' + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (!followLinks) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + HANDLE hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, + OPEN_EXISTING, flags, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + if (followLinks) { + CloseHandle(hfile); + return TRUE; + } + // Regular Reparse Points (PY_IFRRP) have to be traversed. + BOOL result = _testFileTypeByHandle(hfile, PY_IFRRP, FALSE); + CloseHandle(hfile); + if (!result) { + return TRUE; + } + hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + CloseHandle(hfile); + return TRUE; + } + } -Test whether a path is a regular file + switch (GetLastError()) { + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + case ERROR_CANT_ACCESS_FILE: + case ERROR_INVALID_PARAMETER: + STRUCT_STAT _st; + return followLinks ? !STAT(path, &_st): !LSTAT(path, &_st); + } -[clinic start generated code]*/ + return FALSE; +} -static PyObject * -os__path_isfile_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=2394ed7c4b5cfd85 input=de22d74960ade365]*/ -{ - HANDLE hfile; - BOOL close_file = TRUE; - FILE_BASIC_INFO info; - path_t _path = PATH_T_INITIALIZE("isfile", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; +static BOOL +_testFileExists(path_t *path, BOOL followLinks) +{ + BOOL result = FALSE; + if (path->value_error) { + return FALSE; } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = !(statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY); - } else if (statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - slow_path = FALSE; - result = 0; + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); + if (hfile != INVALID_HANDLE_VALUE) { + if (GetFileType(hfile) != FILE_TYPE_UNKNOWN || !GetLastError()) { + result = TRUE; } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; } } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - } + else if (path->wide) { + result = _testFileExistsByName(path->wide, followLinks); + } + Py_END_ALLOW_THREADS + + return result; +} + + +static BOOL +_testFileType(path_t *path, int testedType) +{ + BOOL result = FALSE; + if (path->value_error) { + return FALSE; + } + + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, - sizeof(info))) - { - result = !(info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY); - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISREG(st.st_mode); - } - break; - default: - result = 0; - } + result = _testFileTypeByHandle(hfile, testedType, TRUE); } } + else if (path->wide) { + result = _testFileTypeByName(path->wide, testedType); + } Py_END_ALLOW_THREADS - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return result; } /*[clinic input] -os._path_exists +os._path_exists -> bool - path: 'O' + path: path_t(allow_fd=True, suppress_value_error=True) -Test whether a path exists. Returns False for broken symbolic links +Test whether a path exists. Returns False for broken symbolic links. [clinic start generated code]*/ -static PyObject * -os__path_exists_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=f508c3b35e13a249 input=380f77cdfa0f7ae8]*/ +static int +os__path_exists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=8da13acf666e16ba input=142beabfc66783eb]*/ { - HANDLE hfile; - BOOL close_file = TRUE; - path_t _path = PATH_T_INITIALIZE("exists", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + return _testFileExists(path, TRUE); +} - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; - } - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = 1; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } - } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - } - if (hfile != INVALID_HANDLE_VALUE) { - result = 1; - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = 1; - } - break; - default: - result = 0; - } - } - } - Py_END_ALLOW_THREADS +/*[clinic input] +os._path_lexists -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) + +Test whether a path exists. Returns True for broken symbolic links. + +[clinic start generated code]*/ + +static int +os__path_lexists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e7240ed5fc45bff3 input=208205112a3cc1ed]*/ +{ + return _testFileExists(path, FALSE); +} + + +/*[clinic input] +os._path_isdir -> bool + + s as path: path_t(allow_fd=True, suppress_value_error=True) + +Return true if the pathname refers to an existing directory. + +[clinic start generated code]*/ + +static int +os__path_isdir_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d5786196f9e2fa7a input=132a3b5301aecf79]*/ +{ + return _testFileType(path, PY_IFDIR); +} + + +/*[clinic input] +os._path_isfile -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) + +Test whether a path is a regular file + +[clinic start generated code]*/ - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; +static int +os__path_isfile_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=5c3073bc212b9863 input=4ac1fd350b30a39e]*/ +{ + return _testFileType(path, PY_IFREG); } /*[clinic input] -os._path_islink +os._path_islink -> bool - path: 'O' + path: path_t(allow_fd=True, suppress_value_error=True) Test whether a path is a symbolic link [clinic start generated code]*/ +static int +os__path_islink_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=30da7bda8296adcc input=7510ce05b547debb]*/ +{ + return _testFileType(path, PY_IFLNK); +} + + +/*[clinic input] +os._path_isjunction -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) + +Test whether a path is a junction + +[clinic start generated code]*/ + +static int +os__path_isjunction_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e1d17a9dd18a9945 input=7dcb8bc4e972fcaf]*/ +{ + return _testFileType(path, PY_IFMNT); +} + +#undef PY_IFREG +#undef PY_IFDIR +#undef PY_IFLNK +#undef PY_IFMNT +#undef PY_IFLRP +#undef PY_IFRRP + +#endif /* MS_WINDOWS */ + + +/*[clinic input] +os._path_splitroot_ex + + p as path: path_t(make_wide=True, nonstrict=True) + +Split a pathname into drive, root and tail. + +The tail contains anything after the root. +[clinic start generated code]*/ + static PyObject * -os__path_islink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=6d8640b1a390c054 input=38a3cb937ccf59bf]*/ +os__path_splitroot_ex_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=4b0072b6cdf4b611 input=4556b615c7cc13f2]*/ { - HANDLE hfile; - BOOL close_file = TRUE; - FILE_ATTRIBUTE_TAG_INFO info; - path_t _path = PATH_T_INITIALIZE("islink", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + Py_ssize_t drvsize, rootsize; + PyObject *drv = NULL, *root = NULL, *tail = NULL, *result = NULL; - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; + const wchar_t *buffer = path->wide; + _Py_skiproot(buffer, path->length, &drvsize, &rootsize); + drv = PyUnicode_FromWideChar(buffer, drvsize); + if (drv == NULL) { + goto exit; } - - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - slow_path = FALSE; - if (statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - result = (statInfo.ReparseTag == IO_REPARSE_TAG_SYMLINK); - } - else { - result = 0; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } + root = PyUnicode_FromWideChar(&buffer[drvsize], rootsize); + if (root == NULL) { + goto exit; } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, - FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, - NULL); + tail = PyUnicode_FromWideChar(&buffer[drvsize + rootsize], + path->length - drvsize - rootsize); + if (tail == NULL) { + goto exit; + } + if (PyBytes_Check(path->object)) { + Py_SETREF(drv, PyUnicode_EncodeFSDefault(drv)); + if (drv == NULL) { + goto exit; } - if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileAttributeTagInfo, &info, - sizeof(info))) - { - result = (info.ReparseTag == IO_REPARSE_TAG_SYMLINK); - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } + Py_SETREF(root, PyUnicode_EncodeFSDefault(root)); + if (root == NULL) { + goto exit; } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (LSTAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISLNK(st.st_mode); - } - break; - default: - result = 0; - } + Py_SETREF(tail, PyUnicode_EncodeFSDefault(tail)); + if (tail == NULL) { + goto exit; } } - Py_END_ALLOW_THREADS - - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + result = PyTuple_Pack(3, drv, root, tail); +exit: + Py_XDECREF(drv); + Py_XDECREF(root); + Py_XDECREF(tail); + return result; } -#endif /* MS_WINDOWS */ - /*[clinic input] os._path_normpath - path: object + path: path_t(make_wide=True, nonstrict=True) -Basic path normalization. +Normalize path, eliminating double slashes, etc. [clinic start generated code]*/ static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/ +os__path_normpath_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d353e7ed9410c044 input=3d4ac23b06332dcb]*/ { - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", - Py_TYPE(path)->tp_name); - return NULL; + PyObject *result; + Py_ssize_t norm_len; + wchar_t *norm_path = _Py_normpath_and_size((wchar_t *)path->wide, + path->length, &norm_len); + if (!norm_len) { + result = PyUnicode_FromOrdinal('.'); } - Py_ssize_t len; - wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); - if (!buffer) { - return NULL; + else { + result = PyUnicode_FromWideChar(norm_path, norm_len); + } + if (PyBytes_Check(path->object)) { + Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } - Py_ssize_t norm_len; - wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); - PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); - PyMem_Free(buffer); return result; } @@ -5491,6 +5648,12 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) /*[clinic end generated code: output=a70446903abe821f input=a61722e1576fab03]*/ { int result; +#ifdef MS_WINDOWS + int error = 0; + int pathError = 0; + SECURITY_ATTRIBUTES secAttr = { sizeof(secAttr) }; + SECURITY_ATTRIBUTES *pSecAttr = NULL; +#endif #ifdef HAVE_MKDIRAT int mkdirat_unavailable = 0; #endif @@ -5502,11 +5665,38 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS - result = CreateDirectoryW(path->wide, NULL); + if (mode == 0700 /* 0o700 */) { + ULONG sdSize; + pSecAttr = &secAttr; + // Set a discretionary ACL (D) that is protected (P) and includes + // inheritable (OICI) entries that allow (A) full control (FA) to + // SYSTEM (SY), Administrators (BA), and the owner (OW). + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)", + SDDL_REVISION_1, + &secAttr.lpSecurityDescriptor, + &sdSize + )) { + error = GetLastError(); + } + } + if (!error) { + result = CreateDirectoryW(path->wide, pSecAttr); + if (secAttr.lpSecurityDescriptor && + // uncommonly, LocalFree returns non-zero on error, but still uses + // GetLastError() to see what the error code is + LocalFree(secAttr.lpSecurityDescriptor)) { + error = GetLastError(); + } + } Py_END_ALLOW_THREADS - if (!result) + if (error) { + return PyErr_SetFromWindowsErr(error); + } + if (!result) { return path_error(path); + } #else Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT @@ -7725,14 +7915,20 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, } #endif /* HAVE_FORK */ +#if defined(HAVE_FORK1) || defined(HAVE_FORKPTY) || defined(HAVE_FORK) // Common code to raise a warning if we detect there is more than one thread // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This code might do an import, thus acquiring the import lock, which -// PyOS_BeforeFork() also does. As this should only be called from -// the parent process, it is in the same thread so that works. -static void warn_about_fork_with_threads(const char* name) { +// This should only be called from the parent process after +// PyOS_AfterFork_Parent(). +static void +warn_about_fork_with_threads(const char* name) +{ + // It's not safe to issue the warning while the world is stopped, because + // other threads might be holding locks that we need, which would deadlock. + assert(!_PyRuntime.stoptheworld.world_stopped); + // TODO: Consider making an `os` module API to return the current number // of threads in the process. That'd presumably use this platform code but // raise an error rather than using the inaccurate fallback. @@ -7823,6 +8019,7 @@ static void warn_about_fork_with_threads(const char* name) { PyErr_Clear(); } } +#endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK #ifdef HAVE_FORK1 /*[clinic input] @@ -7840,8 +8037,8 @@ os_fork1_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { + PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; } @@ -7856,9 +8053,10 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("fork1"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("fork1"); } if (pid == -1) { errno = saved_errno; @@ -7884,8 +8082,8 @@ os_fork_impl(PyObject *module) { pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { + PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; } @@ -7904,9 +8102,10 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("fork"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("fork"); } if (pid == -1) { errno = saved_errno; @@ -8478,6 +8677,19 @@ os_unlockpt_impl(PyObject *module, int fd) #endif /* HAVE_UNLOCKPT */ #if defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) +static PyObject * +py_ptsname(int fd) +{ + // POSIX manpage: Upon failure, ptsname() shall return a null pointer + // and may set errno. Always initialize errno to avoid undefined behavior. + errno = 0; + char *name = ptsname(fd); + if (name == NULL) { + return posix_error(); + } + return PyUnicode_DecodeFSDefault(name); +} + /*[clinic input] os.ptsname @@ -8499,22 +8711,22 @@ os_ptsname_impl(PyObject *module, int fd) int ret; char name[MAXPATHLEN+1]; - ret = ptsname_r(fd, name, sizeof(name)); + if (HAVE_PTSNAME_R_RUNTIME) { + ret = ptsname_r(fd, name, sizeof(name)); + } + else { + // fallback to ptsname() if ptsname_r() is not available in runtime. + return py_ptsname(fd); + } if (ret != 0) { errno = ret; return posix_error(); } -#else - char *name; - - name = ptsname(fd); - /* POSIX manpage: Upon failure, ptsname() shall return a null pointer and may set errno. - *MAY* set errno? Hmm... */ - if (name == NULL) - return posix_error(); -#endif /* HAVE_PTSNAME_R */ return PyUnicode_DecodeFSDefault(name); +#else + return py_ptsname(fd); +#endif /* HAVE_PTSNAME_R */ } #endif /* defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) */ @@ -8717,8 +8929,8 @@ os_forkpty_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { + PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; } @@ -8735,9 +8947,10 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("forkpty"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("forkpty"); } if (pid == -1) { return posix_error(); @@ -8867,7 +9080,7 @@ os_getgrouplist_impl(PyObject *module, const char *user, gid_t basegid) /* * NGROUPS_MAX is defined by POSIX.1 as the maximum - * number of supplimental groups a users can belong to. + * number of supplemental groups a users can belong to. * We have to increment it by one because * getgrouplist() returns both the supplemental groups * and the primary group, i.e. all of the groups the @@ -9113,7 +9326,81 @@ os_setpgrp_impl(PyObject *module) #ifdef HAVE_GETPPID #ifdef MS_WINDOWS -#include +#include +#include + +// The structure definition in winternl.h may be incomplete. +// This structure is the full version from the MSDN documentation. +typedef struct _PROCESS_BASIC_INFORMATION_FULL { + NTSTATUS ExitStatus; + PVOID PebBaseAddress; + ULONG_PTR AffinityMask; + LONG BasePriority; + ULONG_PTR UniqueProcessId; + ULONG_PTR InheritedFromUniqueProcessId; +} PROCESS_BASIC_INFORMATION_FULL; + +typedef NTSTATUS (NTAPI *PNT_QUERY_INFORMATION_PROCESS) ( + IN HANDLE ProcessHandle, + IN PROCESSINFOCLASS ProcessInformationClass, + OUT PVOID ProcessInformation, + IN ULONG ProcessInformationLength, + OUT PULONG ReturnLength OPTIONAL); + +// This function returns the process ID of the parent process. +// Returns 0 on failure. +static ULONG +win32_getppid_fast(void) +{ + NTSTATUS status; + HMODULE ntdll; + PNT_QUERY_INFORMATION_PROCESS pNtQueryInformationProcess; + PROCESS_BASIC_INFORMATION_FULL basic_information; + static ULONG cached_ppid = 0; + + if (cached_ppid) { + // No need to query the kernel again. + return cached_ppid; + } + + ntdll = GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) { + return 0; + } + + pNtQueryInformationProcess = (PNT_QUERY_INFORMATION_PROCESS) GetProcAddress(ntdll, "NtQueryInformationProcess"); + if (!pNtQueryInformationProcess) { + return 0; + } + + status = pNtQueryInformationProcess(GetCurrentProcess(), + ProcessBasicInformation, + &basic_information, + sizeof(basic_information), + NULL); + + if (!NT_SUCCESS(status)) { + return 0; + } + + // Perform sanity check on the parent process ID we received from NtQueryInformationProcess. + // The check covers values which exceed the 32-bit range (if running on x64) as well as + // zero and (ULONG) -1. + + if (basic_information.InheritedFromUniqueProcessId == 0 || + basic_information.InheritedFromUniqueProcessId >= ULONG_MAX) + { + return 0; + } + + // Now that we have reached this point, the BasicInformation.InheritedFromUniqueProcessId + // structure member contains a ULONG_PTR which represents the process ID of our parent + // process. This process ID will be correctly returned even if the parent process has + // exited or been terminated. + + cached_ppid = (ULONG) basic_information.InheritedFromUniqueProcessId; + return cached_ppid; +} static PyObject* win32_getppid(void) @@ -9121,8 +9408,16 @@ win32_getppid(void) DWORD error; PyObject* result = NULL; HANDLE process = GetCurrentProcess(); - HPSS snapshot = NULL; + ULONG pid; + + pid = win32_getppid_fast(); + if (pid != 0) { + return PyLong_FromUnsignedLong(pid); + } + + // If failure occurs in win32_getppid_fast(), fall back to using the PSS API. + error = PssCaptureSnapshot(process, PSS_CAPTURE_NONE, 0, &snapshot); if (error != ERROR_SUCCESS) { return PyErr_SetFromWindowsErr(error); @@ -9575,36 +9870,39 @@ wait_helper(PyObject *module, pid_t pid, int status, struct rusage *ru) if (!result) return NULL; + int pos = 0; + #ifndef doubletime #define doubletime(TV) ((double)(TV).tv_sec + (TV).tv_usec * 0.000001) #endif - PyStructSequence_SET_ITEM(result, 0, - PyFloat_FromDouble(doubletime(ru->ru_utime))); - PyStructSequence_SET_ITEM(result, 1, - PyFloat_FromDouble(doubletime(ru->ru_stime))); -#define SET_INT(result, index, value)\ - PyStructSequence_SET_ITEM(result, index, PyLong_FromLong(value)) - SET_INT(result, 2, ru->ru_maxrss); - SET_INT(result, 3, ru->ru_ixrss); - SET_INT(result, 4, ru->ru_idrss); - SET_INT(result, 5, ru->ru_isrss); - SET_INT(result, 6, ru->ru_minflt); - SET_INT(result, 7, ru->ru_majflt); - SET_INT(result, 8, ru->ru_nswap); - SET_INT(result, 9, ru->ru_inblock); - SET_INT(result, 10, ru->ru_oublock); - SET_INT(result, 11, ru->ru_msgsnd); - SET_INT(result, 12, ru->ru_msgrcv); - SET_INT(result, 13, ru->ru_nsignals); - SET_INT(result, 14, ru->ru_nvcsw); - SET_INT(result, 15, ru->ru_nivcsw); -#undef SET_INT - - if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; - } +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(result, pos++, item); \ + } while(0) + + SET_RESULT(PyFloat_FromDouble(doubletime(ru->ru_utime))); + SET_RESULT(PyFloat_FromDouble(doubletime(ru->ru_stime))); + SET_RESULT(PyLong_FromLong(ru->ru_maxrss)); + SET_RESULT(PyLong_FromLong(ru->ru_ixrss)); + SET_RESULT(PyLong_FromLong(ru->ru_idrss)); + SET_RESULT(PyLong_FromLong(ru->ru_isrss)); + SET_RESULT(PyLong_FromLong(ru->ru_minflt)); + SET_RESULT(PyLong_FromLong(ru->ru_majflt)); + SET_RESULT(PyLong_FromLong(ru->ru_nswap)); + SET_RESULT(PyLong_FromLong(ru->ru_inblock)); + SET_RESULT(PyLong_FromLong(ru->ru_oublock)); + SET_RESULT(PyLong_FromLong(ru->ru_msgsnd)); + SET_RESULT(PyLong_FromLong(ru->ru_msgrcv)); + SET_RESULT(PyLong_FromLong(ru->ru_nsignals)); + SET_RESULT(PyLong_FromLong(ru->ru_nvcsw)); + SET_RESULT(PyLong_FromLong(ru->ru_nivcsw)); +#undef SET_RESULT return Py_BuildValue("NiN", PyLong_FromPid(pid), status, result); } @@ -9727,15 +10025,25 @@ os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options) if (!result) return NULL; - PyStructSequence_SET_ITEM(result, 0, PyLong_FromPid(si.si_pid)); - PyStructSequence_SET_ITEM(result, 1, _PyLong_FromUid(si.si_uid)); - PyStructSequence_SET_ITEM(result, 2, PyLong_FromLong((long)(si.si_signo))); - PyStructSequence_SET_ITEM(result, 3, PyLong_FromLong((long)(si.si_status))); - PyStructSequence_SET_ITEM(result, 4, PyLong_FromLong((long)(si.si_code))); - if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; - } + int pos = 0; + +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(result, pos++, item); \ + } while(0) + + SET_RESULT(PyLong_FromPid(si.si_pid)); + SET_RESULT(_PyLong_FromUid(si.si_uid)); + SET_RESULT(PyLong_FromLong((long)(si.si_signo))); + SET_RESULT(PyLong_FromLong((long)(si.si_status))); + SET_RESULT(PyLong_FromLong((long)(si.si_code))); + +#undef SET_RESULT return result; } @@ -9849,7 +10157,10 @@ os_wait_impl(PyObject *module) } #endif /* HAVE_WAIT */ -#if defined(__linux__) && defined(__NR_pidfd_open) + +// This system call always crashes on older Android versions. +#if defined(__linux__) && defined(__NR_pidfd_open) && \ + !(defined(__ANDROID__) && __ANDROID_API__ < 31) /*[clinic input] os.pidfd_open pid: pid_t @@ -10049,7 +10360,7 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) name[1] = L'\\'; } result = PyUnicode_FromWideChar(name, nameLen); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } } @@ -10332,12 +10643,12 @@ Return a collection containing process timing information. The object returned behaves like a named tuple with these fields: (utime, stime, cutime, cstime, elapsed_time) -All fields are floating point numbers. +All fields are floating-point numbers. [clinic start generated code]*/ static PyObject * os_times_impl(PyObject *module) -/*[clinic end generated code: output=35f640503557d32a input=2bf9df3d6ab2e48b]*/ +/*[clinic end generated code: output=35f640503557d32a input=8dbfe33a2dcc3df3]*/ { #ifdef MS_WINDOWS FILETIME create, exit, kernel, user; @@ -10409,7 +10720,7 @@ build_itimerspec(const struct itimerspec* curr_value) static PyObject * build_itimerspec_ns(const struct itimerspec* curr_value) { - _PyTime_t value, interval; + PyTime_t value, interval; if (_PyTime_FromTimespec(&value, &curr_value->it_value) < 0) { return NULL; } @@ -10468,30 +10779,40 @@ os.timerfd_settime * flags: int = 0 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET. - initial: double = 0.0 + initial as initial_double: double = 0.0 The initial expiration time, in seconds. - interval: double = 0.0 + interval as interval_double: double = 0.0 The timer's interval, in seconds. Alter a timer file descriptor's internal timer in seconds. [clinic start generated code]*/ static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval) -/*[clinic end generated code: output=0dda31115317adb9 input=6c24e47e7a4d799e]*/ +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double) +/*[clinic end generated code: output=df4c1bce6859224e input=81d2c0d7e936e8a7]*/ { - struct itimerspec new_value; - struct itimerspec old_value; - int result; - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(initial, _PyTime_ROUND_FLOOR), &new_value.it_value) < 0) { + PyTime_t initial, interval; + if (_PyTime_FromSecondsDouble(initial_double, _PyTime_ROUND_FLOOR, + &initial) < 0) { + return NULL; + } + if (_PyTime_FromSecondsDouble(interval_double, _PyTime_ROUND_FLOOR, + &interval) < 0) { + return NULL; + } + + struct itimerspec new_value, old_value; + if (_PyTime_AsTimespec(initial, &new_value.it_value) < 0) { PyErr_SetString(PyExc_ValueError, "invalid initial value"); return NULL; } - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(interval, _PyTime_ROUND_FLOOR), &new_value.it_interval) < 0) { + if (_PyTime_AsTimespec(interval, &new_value.it_interval) < 0) { PyErr_SetString(PyExc_ValueError, "invalid interval value"); return NULL; } + + int result; Py_BEGIN_ALLOW_THREADS result = timerfd_settime(fd, flags, &new_value, &old_value); Py_END_ALLOW_THREADS @@ -12281,8 +12602,30 @@ os_mknod_impl(PyObject *module, path_t *path, int mode, dev_t device, #ifdef HAVE_DEVICE_MACROS +static PyObject * +major_minor_conv(unsigned int value) +{ +#ifdef NODEV + if (value == (unsigned int)NODEV) { + return PyLong_FromLong((int)NODEV); + } +#endif + return PyLong_FromUnsignedLong(value); +} + +static int +major_minor_check(dev_t value) +{ +#ifdef NODEV + if (value == NODEV) { + return 1; + } +#endif + return (dev_t)(unsigned int)value == value; +} + /*[clinic input] -os.major -> unsigned_int +os.major device: dev_t / @@ -12290,16 +12633,16 @@ os.major -> unsigned_int Extracts a device major number from a raw device number. [clinic start generated code]*/ -static unsigned int +static PyObject * os_major_impl(PyObject *module, dev_t device) -/*[clinic end generated code: output=5b3b2589bafb498e input=1e16a4d30c4d4462]*/ +/*[clinic end generated code: output=4071ffee17647891 input=b1a0a14ec9448229]*/ { - return major(device); + return major_minor_conv(major(device)); } /*[clinic input] -os.minor -> unsigned_int +os.minor device: dev_t / @@ -12307,28 +12650,33 @@ os.minor -> unsigned_int Extracts a device minor number from a raw device number. [clinic start generated code]*/ -static unsigned int +static PyObject * os_minor_impl(PyObject *module, dev_t device) -/*[clinic end generated code: output=5e1a25e630b0157d input=0842c6d23f24c65e]*/ +/*[clinic end generated code: output=306cb78e3bc5004f input=2f686e463682a9da]*/ { - return minor(device); + return major_minor_conv(minor(device)); } /*[clinic input] os.makedev -> dev_t - major: int - minor: int + major: dev_t + minor: dev_t / Composes a raw device number from the major and minor device numbers. [clinic start generated code]*/ static dev_t -os_makedev_impl(PyObject *module, int major, int minor) -/*[clinic end generated code: output=881aaa4aba6f6a52 input=4b9fd8fc73cbe48f]*/ +os_makedev_impl(PyObject *module, dev_t major, dev_t minor) +/*[clinic end generated code: output=cad6125c51f5af80 input=2146126ec02e55c1]*/ { + if (!major_minor_check(major) || !major_minor_check(minor)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C unsigned int"); + return (dev_t)-1; + } return makedev(major, minor); } #endif /* HAVE_DEVICE_MACROS */ @@ -12894,7 +13242,7 @@ os_WSTOPSIG_impl(PyObject *module, int status) #ifdef __APPLE__ /* On macOS struct statvfs uses 32-bit integers for block counts, - * resulting in overflow when filesystems are larger tan 4TB. Therefore + * resulting in overflow when filesystems are larger than 4TB. Therefore * os.statvfs is implemented in terms of statfs(2). */ @@ -12902,42 +13250,45 @@ static PyObject* _pystatvfs_fromstructstatfs(PyObject *module, struct statfs st) { PyObject *StatVFSResultType = get_posix_state(module)->StatVFSResultType; PyObject *v = PyStructSequence_New((PyTypeObject *)StatVFSResultType); - if (v == NULL) + if (v == NULL) { return NULL; + } - long flags = 0; - if (st.f_flags & MNT_RDONLY) { - flags |= ST_RDONLY; - } - if (st.f_flags & MNT_NOSUID) { - flags |= ST_NOSUID; - } - - _Static_assert(sizeof(st.f_blocks) == sizeof(long long), "assuming large file"); - - PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_iosize)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_bsize)); - PyStructSequence_SET_ITEM(v, 2, - PyLong_FromLongLong((long long) st.f_blocks)); - PyStructSequence_SET_ITEM(v, 3, - PyLong_FromLongLong((long long) st.f_bfree)); - PyStructSequence_SET_ITEM(v, 4, - PyLong_FromLongLong((long long) st.f_bavail)); - PyStructSequence_SET_ITEM(v, 5, - PyLong_FromLongLong((long long) st.f_files)); - PyStructSequence_SET_ITEM(v, 6, - PyLong_FromLongLong((long long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 7, - PyLong_FromLongLong((long long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) flags)); - - PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) NAME_MAX)); - PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); - if (PyErr_Occurred()) { - Py_DECREF(v); - return NULL; + long flags = 0; + if (st.f_flags & MNT_RDONLY) { + flags |= ST_RDONLY; + } + if (st.f_flags & MNT_NOSUID) { + flags |= ST_NOSUID; } + _Static_assert(sizeof(st.f_blocks) == sizeof(long long), "assuming large file"); + +#define SET_ITEM(SEQ, INDEX, EXPR) \ + do { \ + PyObject *obj = (EXPR); \ + if (obj == NULL) { \ + Py_DECREF((SEQ)); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM((SEQ), (INDEX), obj); \ + } while (0) + + SET_ITEM(v, 0, PyLong_FromLong((long) st.f_iosize)); + SET_ITEM(v, 1, PyLong_FromLong((long) st.f_bsize)); + SET_ITEM(v, 2, PyLong_FromLongLong((long long) st.f_blocks)); + SET_ITEM(v, 3, PyLong_FromLongLong((long long) st.f_bfree)); + SET_ITEM(v, 4, PyLong_FromLongLong((long long) st.f_bavail)); + SET_ITEM(v, 5, PyLong_FromLongLong((long long) st.f_files)); + SET_ITEM(v, 6, PyLong_FromLongLong((long long) st.f_ffree)); + SET_ITEM(v, 7, PyLong_FromLongLong((long long) st.f_ffree)); + SET_ITEM(v, 8, PyLong_FromLong((long) flags)); + + SET_ITEM(v, 9, PyLong_FromLong((long) NAME_MAX)); + SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); + +#undef SET_ITEM + return v; } @@ -12952,46 +13303,50 @@ _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { if (v == NULL) return NULL; + int pos = 0; + +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(v); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(v, pos++, item); \ + } while(0) + #if !defined(HAVE_LARGEFILE_SUPPORT) - PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_bsize)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_frsize)); - PyStructSequence_SET_ITEM(v, 2, PyLong_FromLong((long) st.f_blocks)); - PyStructSequence_SET_ITEM(v, 3, PyLong_FromLong((long) st.f_bfree)); - PyStructSequence_SET_ITEM(v, 4, PyLong_FromLong((long) st.f_bavail)); - PyStructSequence_SET_ITEM(v, 5, PyLong_FromLong((long) st.f_files)); - PyStructSequence_SET_ITEM(v, 6, PyLong_FromLong((long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 7, PyLong_FromLong((long) st.f_favail)); - PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) st.f_flag)); - PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) st.f_namemax)); + SET_RESULT(PyLong_FromLong((long) st.f_bsize)); + SET_RESULT(PyLong_FromLong((long) st.f_frsize)); + SET_RESULT(PyLong_FromLong((long) st.f_blocks)); + SET_RESULT(PyLong_FromLong((long) st.f_bfree)); + SET_RESULT(PyLong_FromLong((long) st.f_bavail)); + SET_RESULT(PyLong_FromLong((long) st.f_files)); + SET_RESULT(PyLong_FromLong((long) st.f_ffree)); + SET_RESULT(PyLong_FromLong((long) st.f_favail)); + SET_RESULT(PyLong_FromLong((long) st.f_flag)); + SET_RESULT(PyLong_FromLong((long) st.f_namemax)); #else - PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_bsize)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_frsize)); - PyStructSequence_SET_ITEM(v, 2, - PyLong_FromLongLong((long long) st.f_blocks)); - PyStructSequence_SET_ITEM(v, 3, - PyLong_FromLongLong((long long) st.f_bfree)); - PyStructSequence_SET_ITEM(v, 4, - PyLong_FromLongLong((long long) st.f_bavail)); - PyStructSequence_SET_ITEM(v, 5, - PyLong_FromLongLong((long long) st.f_files)); - PyStructSequence_SET_ITEM(v, 6, - PyLong_FromLongLong((long long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 7, - PyLong_FromLongLong((long long) st.f_favail)); - PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) st.f_flag)); - PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) st.f_namemax)); + SET_RESULT(PyLong_FromLong((long) st.f_bsize)); + SET_RESULT(PyLong_FromLong((long) st.f_frsize)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_blocks)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_bfree)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_bavail)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_files)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_ffree)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_favail)); + SET_RESULT(PyLong_FromLong((long) st.f_flag)); + SET_RESULT(PyLong_FromLong((long) st.f_namemax)); #endif /* The _ALL_SOURCE feature test macro defines f_fsid as a structure * (issue #32390). */ #if defined(_AIX) && defined(_ALL_SOURCE) - PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); + SET_RESULT(PyLong_FromUnsignedLong(st.f_fsid.val[0])); #else - PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid)); + SET_RESULT(PyLong_FromUnsignedLong(st.f_fsid)); #endif - if (PyErr_Occurred()) { - Py_DECREF(v); - return NULL; - } + +#undef SET_RESULT return v; } @@ -14963,12 +15318,23 @@ os_get_terminal_size_impl(PyObject *module, int fd) termsize = PyStructSequence_New((PyTypeObject *)TerminalSizeType); if (termsize == NULL) return NULL; - PyStructSequence_SET_ITEM(termsize, 0, PyLong_FromLong(columns)); - PyStructSequence_SET_ITEM(termsize, 1, PyLong_FromLong(lines)); - if (PyErr_Occurred()) { - Py_DECREF(termsize); - return NULL; - } + + int pos = 0; + +#define SET_TERMSIZE(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(termsize); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(termsize, pos++, item); \ + } while(0) + + SET_TERMSIZE(PyLong_FromLong(columns)); + SET_TERMSIZE(PyLong_FromLong(lines)); +#undef SET_TERMSIZE + return termsize; } #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ @@ -15642,7 +16008,8 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) entry->name = PyUnicode_FromWideChar(dataW->cFileName, -1); if (!entry->name) goto error; - if (path->narrow) { + int return_bytes = path->wide && PyBytes_Check(path->object); + if (return_bytes) { Py_SETREF(entry->name, PyUnicode_EncodeFSDefault(entry->name)); if (!entry->name) goto error; @@ -15656,7 +16023,7 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) PyMem_Free(joined_path); if (!entry->path) goto error; - if (path->narrow) { + if (return_bytes) { Py_SETREF(entry->path, PyUnicode_EncodeFSDefault(entry->path)); if (!entry->path) goto error; @@ -15665,6 +16032,10 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) find_data_to_file_info(dataW, &file_info, &reparse_tag); _Py_attribute_data_to_stat(&file_info, reparse_tag, NULL, NULL, &entry->win32_lstat); + /* ctime is only deprecated from 3.12, so we copy birthtime across */ + entry->win32_lstat.st_ctime = entry->win32_lstat.st_birthtime; + entry->win32_lstat.st_ctime_nsec = entry->win32_lstat.st_birthtime_nsec; + return (PyObject *)entry; error: @@ -16466,6 +16837,51 @@ os__supports_virtual_terminal_impl(PyObject *module) } #endif +/*[clinic input] +os._inputhook + +Calls PyOS_CallInputHook droppong the GIL first +[clinic start generated code]*/ + +static PyObject * +os__inputhook_impl(PyObject *module) +/*[clinic end generated code: output=525aca4ef3c6149f input=fc531701930d064f]*/ +{ + int result = 0; + if (PyOS_InputHook) { + Py_BEGIN_ALLOW_THREADS; + result = PyOS_InputHook(); + Py_END_ALLOW_THREADS; + } + return PyLong_FromLong(result); +} + +/*[clinic input] +os._is_inputhook_installed + +Checks if PyOS_CallInputHook is set +[clinic start generated code]*/ + +static PyObject * +os__is_inputhook_installed_impl(PyObject *module) +/*[clinic end generated code: output=3b3eab4f672c689a input=ff177c9938dd76d8]*/ +{ + return PyBool_FromLong(PyOS_InputHook != NULL); +} + +/*[clinic input] +os._create_environ + +Create the environment dictionary. +[clinic start generated code]*/ + +static PyObject * +os__create_environ_impl(PyObject *module) +/*[clinic end generated code: output=19d9039ab14f8ad4 input=a4c05686b34635e8]*/ +{ + return convertenviron(); +} + static PyMethodDef posix_methods[] = { @@ -16630,6 +17046,7 @@ static PyMethodDef posix_methods[] = { OS__FINDFIRSTFILE_METHODDEF OS__GETVOLUMEPATHNAME_METHODDEF OS__PATH_SPLITROOT_METHODDEF + OS__PATH_SPLITROOT_EX_METHODDEF OS__PATH_NORMPATH_METHODDEF OS_GETLOADAVG_METHODDEF OS_URANDOM_METHODDEF @@ -16673,9 +17090,14 @@ static PyMethodDef posix_methods[] = { OS__PATH_ISDIR_METHODDEF OS__PATH_ISFILE_METHODDEF OS__PATH_ISLINK_METHODDEF + OS__PATH_ISJUNCTION_METHODDEF OS__PATH_EXISTS_METHODDEF + OS__PATH_LEXISTS_METHODDEF OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF + OS__INPUTHOOK_METHODDEF + OS__IS_INPUTHOOK_INSTALLED_METHODDEF + OS__CREATE_ENVIRON_METHODDEF {NULL, NULL} /* Sentinel */ }; @@ -17387,6 +17809,9 @@ PROBE(probe_futimens, HAVE_FUTIMENS_RUNTIME) PROBE(probe_utimensat, HAVE_UTIMENSAT_RUNTIME) #endif +#ifdef HAVE_PTSNAME_R +PROBE(probe_ptsname_r, HAVE_PTSNAME_R_RUNTIME) +#endif @@ -17527,6 +17952,10 @@ static const struct have_function { { "HAVE_UTIMENSAT", probe_utimensat }, #endif +#ifdef HAVE_PTSNAME_R + { "HAVE_PTSNAME_R", probe_ptsname_r }, +#endif + #ifdef MS_WINDOWS { "MS_WINDOWS", NULL }, #endif @@ -17548,11 +17977,11 @@ posixmodule_exec(PyObject *m) return -1; } - if (PyDict_DelItemString(dct, "pwritev") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "pwritev", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "preadv") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "preadv", NULL) < 0) { + return -1; } } #endif @@ -17691,6 +18120,7 @@ posixmodule_exec(PyObject *m) static PyModuleDef_Slot posixmodile_slots[] = { {Py_mod_exec, posixmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h index 8827ce153fed8c..be732db04faf94 100644 --- a/Modules/posixmodule.h +++ b/Modules/posixmodule.h @@ -2,7 +2,6 @@ #ifndef Py_POSIXMODULE_H #define Py_POSIXMODULE_H -#ifndef Py_LIMITED_API #ifdef __cplusplus extern "C" { #endif @@ -34,5 +33,4 @@ extern int _Py_Sigset_Converter(PyObject *, void *); #ifdef __cplusplus } #endif -#endif // !Py_LIMITED_API #endif // !Py_POSIXMODULE_H diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index b7034369c4731e..2240e2078b2d98 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -1,9 +1,16 @@ /* UNIX password file access module */ +// Need limited C API version 3.13 for PyMem_RawRealloc() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include "Python.h" #include "posixmodule.h" +#include // ERANGE #include // getpwuid() #include // sysconf() @@ -64,53 +71,52 @@ static struct PyModuleDef pwdmodule; #define DEFAULT_BUFFER_SIZE 1024 -static void -sets(PyObject *v, int i, const char* val) -{ - if (val) { - PyObject *o = PyUnicode_DecodeFSDefault(val); - PyStructSequence_SET_ITEM(v, i, o); - } - else { - PyStructSequence_SET_ITEM(v, i, Py_None); - Py_INCREF(Py_None); - } -} - static PyObject * mkpwent(PyObject *module, struct passwd *p) { - int setIndex = 0; PyObject *v = PyStructSequence_New(get_pwd_state(module)->StructPwdType); - if (v == NULL) + if (v == NULL) { return NULL; + } -#define SETS(i,val) sets(v, i, val) + int setIndex = 0; + +#define SET_STRING(VAL) \ + SET_RESULT((VAL) ? PyUnicode_DecodeFSDefault((VAL)) : Py_NewRef(Py_None)) - SETS(setIndex++, p->pw_name); +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + goto error; \ + } \ + PyStructSequence_SetItem(v, setIndex++, item); \ + } while(0) + + SET_STRING(p->pw_name); #if defined(HAVE_STRUCT_PASSWD_PW_PASSWD) && !defined(__ANDROID__) - SETS(setIndex++, p->pw_passwd); + SET_STRING(p->pw_passwd); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromUid(p->pw_uid)); - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromGid(p->pw_gid)); + SET_RESULT(_PyLong_FromUid(p->pw_uid)); + SET_RESULT(_PyLong_FromGid(p->pw_gid)); #if defined(HAVE_STRUCT_PASSWD_PW_GECOS) - SETS(setIndex++, p->pw_gecos); + SET_STRING(p->pw_gecos); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - SETS(setIndex++, p->pw_dir); - SETS(setIndex++, p->pw_shell); + SET_STRING(p->pw_dir); + SET_STRING(p->pw_shell); -#undef SETS - - if (PyErr_Occurred()) { - Py_XDECREF(v); - return NULL; - } +#undef SET_STRING +#undef SET_RESULT return v; + +error: + Py_DECREF(v); + return NULL; } /*[clinic input] @@ -338,6 +344,7 @@ pwdmodule_exec(PyObject *module) static PyModuleDef_Slot pwdmodule_slots[] = { {Py_mod_exec, pwdmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 62cd262a7885e9..9733bc34f7c80a 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -7,6 +7,7 @@ #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_traceback.h" // _PyTraceback_Add() +#include #include // offsetof() #include "expat.h" #include "pyexpat.h" @@ -81,6 +82,12 @@ typedef struct { /* NULL if not enabled */ int buffer_size; /* Size of buffer, in XML_Char units */ int buffer_used; /* Buffer units in use */ + bool reparse_deferral_enabled; /* Whether to defer reparsing of + unfinished XML tokens; a de-facto cache of + what Expat has the authority on, for lack + of a getter API function + "XML_GetReparseDeferralEnabled" in Expat + 2.6.0 */ PyObject *intern; /* Dictionary to intern strings */ PyObject **handlers; } xmlparseobject; @@ -703,6 +710,40 @@ get_parse_result(pyexpat_state *state, xmlparseobject *self, int rv) #define MAX_CHUNK_SIZE (1 << 20) +/*[clinic input] +pyexpat.xmlparser.SetReparseDeferralEnabled + + enabled: bool + / + +Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled) +/*[clinic end generated code: output=5ec539e3b63c8c49 input=021eb9e0bafc32c5]*/ +{ +#if XML_COMBINED_VERSION >= 20600 + XML_SetReparseDeferralEnabled(self->itself, enabled ? XML_TRUE : XML_FALSE); + self->reparse_deferral_enabled = (bool)enabled; +#endif + Py_RETURN_NONE; +} + +/*[clinic input] +pyexpat.xmlparser.GetReparseDeferralEnabled + +Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self) +/*[clinic end generated code: output=4e91312e88a595a8 input=54b5f11d32b20f3e]*/ +{ + return PyBool_FromLong(self->reparse_deferral_enabled); +} + /*[clinic input] pyexpat.xmlparser.Parse @@ -713,13 +754,13 @@ pyexpat.xmlparser.Parse Parse XML data. -`isfinal' should be true at end of input. +'isfinal' should be true at end of input. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_Parse_impl(xmlparseobject *self, PyTypeObject *cls, PyObject *data, int isfinal) -/*[clinic end generated code: output=8faffe07fe1f862a input=d0eb2a69fab3b9f1]*/ +/*[clinic end generated code: output=8faffe07fe1f862a input=053e0f047e55c05a]*/ { const char *s; Py_ssize_t slen; @@ -1063,6 +1104,8 @@ static struct PyMethodDef xmlparse_methods[] = { #if XML_COMBINED_VERSION >= 19505 PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif + PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF + PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1158,6 +1201,11 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, self->ns_prefixes = 0; self->handlers = NULL; self->intern = Py_XNewRef(intern); +#if XML_COMBINED_VERSION >= 20600 + self->reparse_deferral_enabled = true; +#else + self->reparse_deferral_enabled = false; +#endif /* namespace_separator is either NULL or contains one char + \0 */ self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler, @@ -1603,7 +1651,7 @@ PyDoc_STRVAR(pyexpat_module_documentation, static int init_handler_descrs(pyexpat_state *state) { int i; - assert(!PyType_HasFeature(state->xml_parse_type, Py_TPFLAGS_VALID_VERSION_TAG)); + assert(state->xml_parse_type->tp_version_tag == 0); for (i = 0; handler_info[i].name != NULL; i++) { struct HandlerInfo *hi = &handler_info[i]; hi->getset.name = hi->name; @@ -2019,6 +2067,11 @@ pyexpat_exec(PyObject *mod) #else capi->SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20600 + capi->SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled; +#else + capi->SetReparseDeferralEnabled = NULL; +#endif /* export using capsule */ PyObject *capi_object = PyCapsule_New(capi, PyExpat_CAPSULE_NAME, @@ -2064,6 +2117,7 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/readline.c b/Modules/readline.c index e29051c37f8827..35655c70a4618f 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1041,7 +1041,7 @@ on_hook(PyObject *func) } static int -#if defined(_RL_FUNCTION_TYPEDEF) +#if defined(_RL_FUNCTION_TYPEDEF) || !defined(Py_RL_STARTUP_HOOK_TAKES_ARGS) on_startup_hook(void) #else on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) @@ -1061,7 +1061,7 @@ on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #ifdef HAVE_RL_PRE_INPUT_HOOK static int -#if defined(_RL_FUNCTION_TYPEDEF) +#if defined(_RL_FUNCTION_TYPEDEF) || !defined(Py_RL_STARTUP_HOOK_TAKES_ARGS) on_pre_input_hook(void) #else on_pre_input_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) @@ -1552,6 +1552,9 @@ PyInit_readline(void) if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddIntConstant(m, "_READLINE_VERSION", RL_READLINE_VERSION) < 0) { diff --git a/Modules/resource.c b/Modules/resource.c index 19020b8cc1b6db..3fe18e7c98e3d8 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,8 +1,7 @@ +// Need limited C API version 3.13 for PySys_Audit() #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.13 for PySys_Audit() -#define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -514,6 +513,7 @@ resource_exec(PyObject *module) static struct PyModuleDef_Slot resource_slots[] = { {Py_mod_exec, resource_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/rotatingtree.c b/Modules/rotatingtree.c index 07e08bc3167c0a..5910e25bed6389 100644 --- a/Modules/rotatingtree.c +++ b/Modules/rotatingtree.c @@ -1,3 +1,4 @@ +#include "Python.h" #include "rotatingtree.h" #define KEY_LOWER_THAN(key1, key2) ((char*)(key1) < (char*)(key2)) @@ -10,17 +11,20 @@ static unsigned int random_value = 1; static unsigned int random_stream = 0; +static PyMutex random_mutex = {0}; static int randombits(int bits) { int result; + PyMutex_Lock(&random_mutex); if (random_stream < (1U << bits)) { random_value *= 1082527; random_stream = random_value; } result = random_stream & ((1<>= bits; + PyMutex_Unlock(&random_mutex); return result; } diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 1dbde3e9e6ca5d..6ced71cca73178 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -15,7 +15,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_import.h" // _PyImport_GetModuleAttrString() -#include "pycore_time.h" // _PyTime_t +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include #include // offsetof() @@ -262,7 +262,7 @@ A file descriptor is either a socket or file object, or a small integer gotten from a fileno() method call on one of those. The optional 4th argument specifies a timeout in seconds; it may be -a floating point number to specify fractions of seconds. If it is absent +a floating-point number to specify fractions of seconds. If it is absent or None, the call will never time out. The return value is a tuple of three lists corresponding to the first three @@ -277,7 +277,7 @@ descriptors can be used. static PyObject * select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, PyObject *xlist, PyObject *timeout_obj) -/*[clinic end generated code: output=2b3cfa824f7ae4cf input=e467f5d68033de00]*/ +/*[clinic end generated code: output=2b3cfa824f7ae4cf input=1199d5e101abca4a]*/ { #ifdef SELECT_USES_HEAP pylist *rfd2obj, *wfd2obj, *efd2obj; @@ -297,7 +297,7 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, struct timeval tv, *tvp; int imax, omax, emax, max; int n; - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; if (timeout_obj == Py_None) tvp = (struct timeval *)NULL; @@ -473,6 +473,7 @@ update_ufd_array(pollObject *self) } /*[clinic input] +@critical_section select.poll.register fd: fildes @@ -486,7 +487,7 @@ Register a file descriptor with the polling object. static PyObject * select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=0dc7173c800a4a65 input=34e16cfb28d3c900]*/ +/*[clinic end generated code: output=0dc7173c800a4a65 input=c475e029ce6c2830]*/ { PyObject *key, *value; int err; @@ -514,6 +515,7 @@ select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask) /*[clinic input] +@critical_section select.poll.modify fd: fildes @@ -528,7 +530,7 @@ Modify an already registered file descriptor. static PyObject * select_poll_modify_impl(pollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=1a7b88bf079eff17 input=a8e383df075c32cf]*/ +/*[clinic end generated code: output=1a7b88bf079eff17 input=38c9db5346711872]*/ { PyObject *key, *value; int err; @@ -566,6 +568,7 @@ select_poll_modify_impl(pollObject *self, int fd, unsigned short eventmask) /*[clinic input] +@critical_section select.poll.unregister fd: fildes @@ -576,7 +579,7 @@ Remove a file descriptor being tracked by the polling object. static PyObject * select_poll_unregister_impl(pollObject *self, int fd) -/*[clinic end generated code: output=8c9f42e75e7d291b input=4b4fccc1040e79cb]*/ +/*[clinic end generated code: output=8c9f42e75e7d291b input=ae6315d7f5243704]*/ { PyObject *key; @@ -599,6 +602,7 @@ select_poll_unregister_impl(pollObject *self, int fd) } /*[clinic input] +@critical_section select.poll.poll timeout as timeout_obj: object = None @@ -614,12 +618,12 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=876e837d193ed7e4 input=c2f6953ec45e5622]*/ +/*[clinic end generated code: output=876e837d193ed7e4 input=54310631457efdec]*/ { PyObject *result_list = NULL; int poll_result, i, j; PyObject *value = NULL, *num = NULL; - _PyTime_t timeout = -1, ms = -1, deadline = 0; + PyTime_t timeout = -1, ms = -1, deadline = 0; int async_err = 0; if (timeout_obj != Py_None) { @@ -813,14 +817,14 @@ static int devpoll_flush(devpollObject *self) if (n < size) { /* - ** Data writed to /dev/poll is a binary data structure. It is not + ** Data written to /dev/poll is a binary data structure. It is not ** clear what to do if a partial write occurred. For now, raise ** an exception and see if we actually found this problem in ** the wild. - ** See http://bugs.python.org/issue6397. + ** See https://github.com/python/cpython/issues/50646. */ PyErr_Format(PyExc_OSError, "failed to write all pollfds. " - "Please, report at http://bugs.python.org/. " + "Please, report at https://github.com/python/cpython/issues/. " "Data to report: Size tried: %d, actual size written: %d.", size, n); return -1; @@ -857,6 +861,7 @@ internal_devpoll_register(devpollObject *self, int fd, } /*[clinic input] +@critical_section select.devpoll.register fd: fildes @@ -872,12 +877,13 @@ Register a file descriptor with the polling object. static PyObject * select_devpoll_register_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=6e07fe8b74abba0c input=22006fabe9567522]*/ +/*[clinic end generated code: output=6e07fe8b74abba0c input=8d48bd2653a61c42]*/ { return internal_devpoll_register(self, fd, eventmask, 0); } /*[clinic input] +@critical_section select.devpoll.modify fd: fildes @@ -893,12 +899,13 @@ Modify a possible already registered file descriptor. static PyObject * select_devpoll_modify_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=bc2e6d23aaff98b4 input=09fa335db7cdc09e]*/ +/*[clinic end generated code: output=bc2e6d23aaff98b4 input=773b37e9abca2460]*/ { return internal_devpoll_register(self, fd, eventmask, 1); } /*[clinic input] +@critical_section select.devpoll.unregister fd: fildes @@ -909,7 +916,7 @@ Remove a file descriptor being tracked by the polling object. static PyObject * select_devpoll_unregister_impl(devpollObject *self, int fd) -/*[clinic end generated code: output=95519ffa0c7d43fe input=b4ea42a4442fd467]*/ +/*[clinic end generated code: output=95519ffa0c7d43fe input=6052d368368d4d05]*/ { if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -926,6 +933,7 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) } /*[clinic input] +@critical_section select.devpoll.poll timeout as timeout_obj: object = None The maximum time to wait in milliseconds, or else None (or a negative @@ -940,13 +948,13 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=2654e5457cca0b3c input=3c3f0a355ec2bedb]*/ +/*[clinic end generated code: output=2654e5457cca0b3c input=fe7a3f6dcbc118c5]*/ { struct dvpoll dvp; PyObject *result_list = NULL; int poll_result, i; PyObject *value, *num1, *num2; - _PyTime_t timeout, ms, deadline = 0; + PyTime_t timeout, ms, deadline = 0; if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -1059,6 +1067,7 @@ devpoll_internal_close(devpollObject *self) } /*[clinic input] +@critical_section select.devpoll.close Close the devpoll file descriptor. @@ -1068,7 +1077,7 @@ Further operations on the devpoll object will raise an exception. static PyObject * select_devpoll_close_impl(devpollObject *self) -/*[clinic end generated code: output=26b355bd6429f21b input=6273c30f5560a99b]*/ +/*[clinic end generated code: output=26b355bd6429f21b input=408fde21a377ccfb]*/ { errno = devpoll_internal_close(self); if (errno < 0) { @@ -1088,6 +1097,7 @@ devpoll_get_closed(devpollObject *self, void *Py_UNUSED(ignored)) } /*[clinic input] +@critical_section select.devpoll.fileno Return the file descriptor. @@ -1095,7 +1105,7 @@ Return the file descriptor. static PyObject * select_devpoll_fileno_impl(devpollObject *self) -/*[clinic end generated code: output=26920929f8d292f4 input=ef15331ebde6c368]*/ +/*[clinic end generated code: output=26920929f8d292f4 input=8c9db2efa1ade538]*/ { if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -1378,6 +1388,7 @@ pyepoll_dealloc(pyEpoll_Object *self) } /*[clinic input] +@critical_section select.epoll.close Close the epoll control file descriptor. @@ -1387,7 +1398,7 @@ Further operations on the epoll object will raise an exception. static PyObject * select_epoll_close_impl(pyEpoll_Object *self) -/*[clinic end generated code: output=ee2144c446a1a435 input=ca6c66ba5a736bfd]*/ +/*[clinic end generated code: output=ee2144c446a1a435 input=f626a769192e1dbe]*/ { errno = pyepoll_internal_close(self); if (errno < 0) { @@ -1559,7 +1570,7 @@ select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int nfds, i; PyObject *elist = NULL, *etuple = NULL; struct epoll_event *evs = NULL; - _PyTime_t timeout = -1, ms = -1, deadline = 0; + PyTime_t timeout = -1, ms = -1, deadline = 0; if (self->epfd < 0) return pyepoll_err_closed(); @@ -2023,10 +2034,8 @@ kqueue_tracking_init(PyObject *module) { } static int -kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) { - if (!state->kqueue_tracking_initialized) { - kqueue_tracking_init(PyType_GetModule(Py_TYPE(self))); - } +kqueue_tracking_add_lock_held(_selectstate *state, kqueue_queue_Object *self) +{ assert(self->kqfd >= 0); _kqueue_list_item *item = PyMem_New(_kqueue_list_item, 1); if (item == NULL) { @@ -2039,8 +2048,23 @@ kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) { return 0; } +static int +kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) +{ + int ret; + PyObject *module = PyType_GetModule(Py_TYPE(self)); + Py_BEGIN_CRITICAL_SECTION(module); + if (!state->kqueue_tracking_initialized) { + kqueue_tracking_init(module); + } + ret = kqueue_tracking_add_lock_held(state, self); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void -kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) { +kqueue_tracking_remove_lock_held(_selectstate *state, kqueue_queue_Object *self) +{ _kqueue_list *listptr = &state->kqueue_open_list; while (*listptr != NULL) { _kqueue_list_item *item = *listptr; @@ -2056,6 +2080,14 @@ kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) { assert(0); } +static void +kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) +{ + Py_BEGIN_CRITICAL_SECTION(PyType_GetModule(Py_TYPE(self))); + kqueue_tracking_remove_lock_held(state, self); + Py_END_CRITICAL_SECTION(); +} + static int kqueue_queue_internal_close(kqueue_queue_Object *self) { @@ -2150,6 +2182,7 @@ kqueue_queue_finalize(kqueue_queue_Object *self) } /*[clinic input] +@critical_section select.kqueue.close Close the kqueue control file descriptor. @@ -2159,7 +2192,7 @@ Further operations on the kqueue object will raise an exception. static PyObject * select_kqueue_close_impl(kqueue_queue_Object *self) -/*[clinic end generated code: output=d1c7df0b407a4bc1 input=0b12d95430e0634c]*/ +/*[clinic end generated code: output=d1c7df0b407a4bc1 input=6d763c858b17b690]*/ { errno = kqueue_queue_internal_close(self); if (errno < 0) { @@ -2242,7 +2275,7 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, struct kevent *chl = NULL; struct timespec timeoutspec; struct timespec *ptimeoutspec; - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; _selectstate *state = _selectstate_by_type(Py_TYPE(self)); if (self->kqfd < 0) @@ -2802,6 +2835,7 @@ _select_exec(PyObject *m) static PyModuleDef_Slot _select_slots[] = { {Py_mod_exec, _select_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index eda6b5608d52f7..34a427a39b5cf8 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -52,7 +52,7 @@ typedef struct { bool use_mutex; PyMutex mutex; PyThread_type_lock lock; - Hacl_Streaming_SHA1_state *hash_state; + Hacl_Hash_SHA1_state_t *hash_state; } SHA1object; #include "clinic/sha1module.c.h" @@ -95,7 +95,7 @@ SHA1_traverse(PyObject *ptr, visitproc visit, void *arg) static void SHA1_dealloc(SHA1object *ptr) { - Hacl_Streaming_SHA1_legacy_free(ptr->hash_state); + Hacl_Hash_SHA1_free(ptr->hash_state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -124,7 +124,7 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls) return NULL; ENTER_HASHLIB(self); - newobj->hash_state = Hacl_Streaming_SHA1_legacy_copy(self->hash_state); + newobj->hash_state = Hacl_Hash_SHA1_copy(self->hash_state); LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -141,7 +141,7 @@ SHA1Type_digest_impl(SHA1object *self) { unsigned char digest[SHA1_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + Hacl_Hash_SHA1_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, SHA1_DIGESTSIZE); } @@ -158,20 +158,20 @@ SHA1Type_hexdigest_impl(SHA1object *self) { unsigned char digest[SHA1_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + Hacl_Hash_SHA1_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, SHA1_DIGESTSIZE); } -static void update(Hacl_Streaming_SHA1_state *state, uint8_t *buf, Py_ssize_t len) { +static void update(Hacl_Hash_SHA1_state_t *state, uint8_t *buf, Py_ssize_t len) { #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_SHA1_legacy_update(state, buf, UINT32_MAX); + Hacl_Hash_SHA1_update(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif - Hacl_Streaming_SHA1_legacy_update(state, buf, (uint32_t) len); + Hacl_Hash_SHA1_update(state, buf, (uint32_t) len); } /*[clinic input] @@ -295,7 +295,7 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->hash_state = Hacl_Streaming_SHA1_legacy_create_in(); + new->hash_state = Hacl_Hash_SHA1_malloc(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -371,6 +371,7 @@ _sha1_exec(PyObject *module) static PyModuleDef_Slot _sha1_slots[] = { {Py_mod_exec, _sha1_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 968493ba51b50d..7d6a1e40243f9d 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -55,7 +55,7 @@ typedef struct { // Prevents undefined behavior via multiple threads entering the C API. bool use_mutex; PyMutex mutex; - Hacl_Streaming_SHA2_state_sha2_256 *state; + Hacl_Hash_SHA2_state_t_256 *state; } SHA256object; typedef struct { @@ -64,7 +64,7 @@ typedef struct { // Prevents undefined behavior via multiple threads entering the C API. bool use_mutex; PyMutex mutex; - Hacl_Streaming_SHA2_state_sha2_512 *state; + Hacl_Hash_SHA2_state_t_512 *state; } SHA512object; #include "clinic/sha2module.c.h" @@ -89,13 +89,13 @@ sha2_get_state(PyObject *module) static void SHA256copy(SHA256object *src, SHA256object *dest) { dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_256(src->state); + dest->state = Hacl_Hash_SHA2_copy_256(src->state); } static void SHA512copy(SHA512object *src, SHA512object *dest) { dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_512(src->state); + dest->state = Hacl_Hash_SHA2_copy_512(src->state); } static SHA256object * @@ -166,7 +166,7 @@ SHA2_traverse(PyObject *ptr, visitproc visit, void *arg) static void SHA256_dealloc(SHA256object *ptr) { - Hacl_Streaming_SHA2_free_256(ptr->state); + Hacl_Hash_SHA2_free_256(ptr->state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -176,7 +176,7 @@ SHA256_dealloc(SHA256object *ptr) static void SHA512_dealloc(SHA512object *ptr) { - Hacl_Streaming_SHA2_free_512(ptr->state); + Hacl_Hash_SHA2_free_512(ptr->state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -186,34 +186,34 @@ SHA512_dealloc(SHA512object *ptr) /* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be * 64 bits so we loop in <4gig chunks when needed. */ -static void update_256(Hacl_Streaming_SHA2_state_sha2_256 *state, uint8_t *buf, Py_ssize_t len) { +static void update_256(Hacl_Hash_SHA2_state_t_256 *state, uint8_t *buf, Py_ssize_t len) { /* Note: we explicitly ignore the error code on the basis that it would take > * 1 billion years to overflow the maximum admissible length for SHA2-256 * (namely, 2^61-1 bytes). */ #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_256(state, buf, UINT32_MAX); + Hacl_Hash_SHA2_update_256(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ - Hacl_Streaming_SHA2_update_256(state, buf, (uint32_t) len); + Hacl_Hash_SHA2_update_256(state, buf, (uint32_t) len); } -static void update_512(Hacl_Streaming_SHA2_state_sha2_512 *state, uint8_t *buf, Py_ssize_t len) { +static void update_512(Hacl_Hash_SHA2_state_t_512 *state, uint8_t *buf, Py_ssize_t len) { /* Note: we explicitly ignore the error code on the basis that it would take > * 1 billion years to overflow the maximum admissible length for this API * (namely, 2^64-1 bytes). */ #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_512(state, buf, UINT32_MAX); + Hacl_Hash_SHA2_update_512(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ - Hacl_Streaming_SHA2_update_512(state, buf, (uint32_t) len); + Hacl_Hash_SHA2_update_512(state, buf, (uint32_t) len); } @@ -296,7 +296,7 @@ SHA256Type_digest_impl(SHA256object *self) ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. - Hacl_Streaming_SHA2_finish_256(self->state, digest); + Hacl_Hash_SHA2_digest_256(self->state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -316,7 +316,7 @@ SHA512Type_digest_impl(SHA512object *self) ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. - Hacl_Streaming_SHA2_finish_512(self->state, digest); + Hacl_Hash_SHA2_digest_512(self->state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -334,7 +334,7 @@ SHA256Type_hexdigest_impl(SHA256object *self) uint8_t digest[SHA256_DIGESTSIZE]; assert(self->digestsize <= SHA256_DIGESTSIZE); ENTER_HASHLIB(self); - Hacl_Streaming_SHA2_finish_256(self->state, digest); + Hacl_Hash_SHA2_digest_256(self->state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -352,7 +352,7 @@ SHA512Type_hexdigest_impl(SHA512object *self) uint8_t digest[SHA512_DIGESTSIZE]; assert(self->digestsize <= SHA512_DIGESTSIZE); ENTER_HASHLIB(self); - Hacl_Streaming_SHA2_finish_512(self->state, digest); + Hacl_Hash_SHA2_digest_512(self->state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -597,7 +597,7 @@ _sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_256(); + new->state = Hacl_Hash_SHA2_malloc_256(); new->digestsize = 32; if (PyErr_Occurred()) { @@ -651,7 +651,7 @@ _sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_224(); + new->state = Hacl_Hash_SHA2_malloc_224(); new->digestsize = 28; if (PyErr_Occurred()) { @@ -705,7 +705,7 @@ _sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_512(); + new->state = Hacl_Hash_SHA2_malloc_512(); new->digestsize = 64; if (PyErr_Occurred()) { @@ -758,7 +758,7 @@ _sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_384(); + new->state = Hacl_Hash_SHA2_malloc_384(); new->digestsize = 48; if (PyErr_Occurred()) { @@ -866,6 +866,7 @@ static int sha2_exec(PyObject *module) static PyModuleDef_Slot _sha2_slots[] = { {Py_mod_exec, sha2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha3module.c b/Modules/sha3module.c index d9d2f6c385a68b..ca839dc55e0519 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -63,7 +63,7 @@ typedef struct { // Prevents undefined behavior via multiple threads entering the C API. bool use_mutex; PyMutex mutex; - Hacl_Streaming_Keccak_state *hash_state; + Hacl_Hash_SHA3_state_t *hash_state; } SHA3object; #include "clinic/sha3module.c.h" @@ -81,18 +81,18 @@ newSHA3object(PyTypeObject *type) return newobj; } -static void sha3_update(Hacl_Streaming_Keccak_state *state, uint8_t *buf, Py_ssize_t len) { +static void sha3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *buf, Py_ssize_t len) { /* Note: we explicitly ignore the error code on the basis that it would take > * 1 billion years to hash more than 2^64 bytes. */ #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_Keccak_update(state, buf, UINT32_MAX); + Hacl_Hash_SHA3_update(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ - Hacl_Streaming_Keccak_update(state, buf, (uint32_t) len); + Hacl_Hash_SHA3_update(state, buf, (uint32_t) len); } /*[clinic input] @@ -120,17 +120,17 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) assert(state != NULL); if (type == state->sha3_224_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_224); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_224); } else if (type == state->sha3_256_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_256); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_256); } else if (type == state->sha3_384_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_384); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_384); } else if (type == state->sha3_512_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_512); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_512); } else if (type == state->shake_128_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake128); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_Shake128); } else if (type == state->shake_256_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake256); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_Shake256); } else { PyErr_BadInternalCall(); goto error; @@ -169,7 +169,7 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) static void SHA3_dealloc(SHA3object *self) { - Hacl_Streaming_Keccak_free(self->hash_state); + Hacl_Hash_SHA3_free(self->hash_state); PyTypeObject *tp = Py_TYPE(self); PyObject_Free(self); Py_DECREF(tp); @@ -195,7 +195,7 @@ _sha3_sha3_224_copy_impl(SHA3object *self) return NULL; } ENTER_HASHLIB(self); - newobj->hash_state = Hacl_Streaming_Keccak_copy(self->hash_state); + newobj->hash_state = Hacl_Hash_SHA3_copy(self->hash_state); LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -215,10 +215,10 @@ _sha3_sha3_224_digest_impl(SHA3object *self) // This function errors out if the algorithm is Shake. Here, we know this // not to be the case, and therefore do not perform error checking. ENTER_HASHLIB(self); - Hacl_Streaming_Keccak_finish(self->hash_state, digest); + Hacl_Hash_SHA3_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, - Hacl_Streaming_Keccak_hash_len(self->hash_state)); + Hacl_Hash_SHA3_hash_len(self->hash_state)); } @@ -234,10 +234,10 @@ _sha3_sha3_224_hexdigest_impl(SHA3object *self) { unsigned char digest[SHA3_MAX_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_Keccak_finish(self->hash_state, digest); + Hacl_Hash_SHA3_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, - Hacl_Streaming_Keccak_hash_len(self->hash_state)); + Hacl_Hash_SHA3_hash_len(self->hash_state)); } @@ -288,7 +288,7 @@ static PyMethodDef SHA3_methods[] = { static PyObject * SHA3_get_block_size(SHA3object *self, void *closure) { - uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state); + uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state); return PyLong_FromLong(rate); } @@ -324,17 +324,17 @@ static PyObject * SHA3_get_digest_size(SHA3object *self, void *closure) { // Preserving previous behavior: variable-length algorithms return 0 - if (Hacl_Streaming_Keccak_is_shake(self->hash_state)) + if (Hacl_Hash_SHA3_is_shake(self->hash_state)) return PyLong_FromLong(0); else - return PyLong_FromLong(Hacl_Streaming_Keccak_hash_len(self->hash_state)); + return PyLong_FromLong(Hacl_Hash_SHA3_hash_len(self->hash_state)); } static PyObject * SHA3_get_capacity_bits(SHA3object *self, void *closure) { - uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state) * 8; int capacity = 1600 - rate; return PyLong_FromLong(capacity); } @@ -343,7 +343,7 @@ SHA3_get_capacity_bits(SHA3object *self, void *closure) static PyObject * SHA3_get_rate_bits(SHA3object *self, void *closure) { - uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state) * 8; return PyLong_FromLong(rate); } @@ -432,11 +432,11 @@ _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) } /* Get the raw (binary) digest value. The HACL functions errors out if: - * - the algorith is not shake -- not the case here + * - the algorithm is not shake -- not the case here * - the output length is zero -- we follow the existing behavior and return * an empty digest, without raising an error */ if (digestlen > 0) { - Hacl_Streaming_Keccak_squeeze(self->hash_state, digest, digestlen); + Hacl_Hash_SHA3_squeeze(self->hash_state, digest, digestlen); } if (hex) { result = _Py_strhex((const char *)digest, digestlen); @@ -602,6 +602,7 @@ _sha3_exec(PyObject *m) static PyModuleDef_Slot _sha3_slots[] = { {Py_mod_exec, _sha3_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 394a997b20c06d..0e53a36bca55f0 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -13,6 +13,7 @@ #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // _Py_RestoreSignals() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #ifndef MS_WINDOWS # include "posixmodule.h" // _PyLong_FromUid() @@ -173,7 +174,7 @@ timeval_from_double(PyObject *obj, struct timeval *tv) return 0; } - _PyTime_t t; + PyTime_t t; if (_PyTime_FromSecondsObject(&t, obj, _PyTime_ROUND_CEILING) < 0) { return -1; } @@ -276,11 +277,7 @@ trip_signal(int sig_num) cleared in PyErr_CheckSignals() before .tripped. */ _Py_atomic_store_int(&is_tripped, 1); - /* Signals are always handled by the main interpreter */ - PyInterpreterState *interp = _PyInterpreterState_Main(); - - /* Notify ceval.c */ - _PyEval_SignalReceived(interp); + _PyEval_SignalReceived(); /* And then write to the wakeup fd *after* setting all the globals and doing the _PyEval_SignalReceived. We used to write to the wakeup fd @@ -303,6 +300,7 @@ trip_signal(int sig_num) int fd = wakeup.fd; if (fd != INVALID_FD) { + PyInterpreterState *interp = _PyInterpreterState_Main(); unsigned char byte = (unsigned char)sig_num; #ifdef MS_WINDOWS if (wakeup.use_send) { @@ -640,7 +638,7 @@ signal_strsignal_impl(PyObject *module, int signalnum) res = "Aborted"; break; case SIGFPE: - res = "Floating point exception"; + res = "Floating-point exception"; break; case SIGSEGV: res = "Segmentation fault"; @@ -708,35 +706,43 @@ signal_siginterrupt_impl(PyObject *module, int signalnum, int flag) #endif -static PyObject* -signal_set_wakeup_fd(PyObject *self, PyObject *args, PyObject *kwds) +/*[clinic input] +signal.set_wakeup_fd + + fd as fdobj: object + / + * + warn_on_full_buffer: bool = True + +Sets the fd to be written to (with the signal number) when a signal comes in. + +A library can use this to wakeup select or poll. +The previous fd or -1 is returned. + +The fd must be non-blocking. +[clinic start generated code]*/ + +static PyObject * +signal_set_wakeup_fd_impl(PyObject *module, PyObject *fdobj, + int warn_on_full_buffer) +/*[clinic end generated code: output=2280d72dd2a54c4f input=5b545946a28b8339]*/ { struct _Py_stat_struct status; - static char *kwlist[] = { - "", "warn_on_full_buffer", NULL, - }; - int warn_on_full_buffer = 1; #ifdef MS_WINDOWS - PyObject *fdobj; SOCKET_T sockfd, old_sockfd; int res; int res_size = sizeof res; PyObject *mod; int is_socket; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$p:set_wakeup_fd", kwlist, - &fdobj, &warn_on_full_buffer)) - return NULL; - sockfd = PyLong_AsSocket_t(fdobj); if (sockfd == (SOCKET_T)(-1) && PyErr_Occurred()) return NULL; #else - int fd; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "i|$p:set_wakeup_fd", kwlist, - &fd, &warn_on_full_buffer)) + int fd = PyLong_AsInt(fdobj); + if (fd == -1 && PyErr_Occurred()) { return NULL; + } #endif PyThreadState *tstate = _PyThreadState_GET(); @@ -822,15 +828,6 @@ signal_set_wakeup_fd(PyObject *self, PyObject *args, PyObject *kwds) #endif } -PyDoc_STRVAR(set_wakeup_fd_doc, -"set_wakeup_fd(fd, *, warn_on_full_buffer=True) -> fd\n\ -\n\ -Sets the fd to be written to (with the signal number) when a signal\n\ -comes in. A library can use this to wakeup select or poll.\n\ -The previous fd or -1 is returned.\n\ -\n\ -The fd must be non-blocking."); - /* C API for the same, without all the error checking */ int PySignal_SetWakeupFd(int fd) @@ -1202,15 +1199,15 @@ signal.sigtimedwait Like sigwaitinfo(), but with a timeout. -The timeout is specified in seconds, with floating point numbers allowed. +The timeout is specified in seconds, with floating-point numbers allowed. [clinic start generated code]*/ static PyObject * signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, PyObject *timeout_obj) -/*[clinic end generated code: output=59c8971e8ae18a64 input=87fd39237cf0b7ba]*/ +/*[clinic end generated code: output=59c8971e8ae18a64 input=955773219c1596cd]*/ { - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) return NULL; @@ -1220,7 +1217,7 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, return NULL; } - _PyTime_t deadline = _PyDeadline_Init(timeout); + PyTime_t deadline = _PyDeadline_Init(timeout); siginfo_t si; do { @@ -1302,7 +1299,9 @@ signal_pthread_kill_impl(PyObject *module, unsigned long thread_id, #endif /* #if defined(HAVE_PTHREAD_KILL) */ -#if defined(__linux__) && defined(__NR_pidfd_send_signal) +// This system call always crashes on older Android versions. +#if defined(__linux__) && defined(__NR_pidfd_send_signal) && \ + !(defined(__ANDROID__) && __ANDROID_API__ < 31) /*[clinic input] signal.pidfd_send_signal @@ -1346,7 +1345,7 @@ static PyMethodDef signal_methods[] = { SIGNAL_RAISE_SIGNAL_METHODDEF SIGNAL_STRSIGNAL_METHODDEF SIGNAL_GETSIGNAL_METHODDEF - {"set_wakeup_fd", _PyCFunction_CAST(signal_set_wakeup_fd), METH_VARARGS | METH_KEYWORDS, set_wakeup_fd_doc}, + SIGNAL_SET_WAKEUP_FD_METHODDEF SIGNAL_SIGINTERRUPT_METHODDEF SIGNAL_PAUSE_METHODDEF SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF @@ -1701,6 +1700,7 @@ _signal_module_free(void *module) static PyModuleDef_Slot signal_slots[] = { {Py_mod_exec, signal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -1770,8 +1770,8 @@ PyErr_CheckSignals(void) Python code to ensure signals are handled. Checking for the GC here allows long running native code to clean cycles created using the C-API even if it doesn't run the evaluation loop */ - if (_Py_eval_breaker_bit_is_set(tstate->interp, _PY_GC_SCHEDULED_BIT)) { - _Py_set_eval_breaker_bit(tstate->interp, _PY_GC_SCHEDULED_BIT, 0); + if (_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) { + _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); _Py_RunGC(tstate); } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 0a0e0e78656f76..2764bd6e2b2a47 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -109,6 +109,7 @@ Local naming conventions: #include "pycore_capsule.h" // _PyCapsule_SetTraverse() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState +#include "pycore_time.h" // _PyTime_AsMilliseconds() #ifdef _Py_MEMORY_SANITIZER # include @@ -547,7 +548,7 @@ typedef struct _socket_state { PyObject *socket_gaierror; /* Default timeout for new sockets */ - _PyTime_t defaulttimeout; + PyTime_t defaulttimeout; #if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) @@ -772,13 +773,13 @@ internal_setblocking(PySocketSockObject *s, int block) } static int -internal_select(PySocketSockObject *s, int writing, _PyTime_t interval, +internal_select(PySocketSockObject *s, int writing, PyTime_t interval, int connect) { int n; #ifdef HAVE_POLL struct pollfd pollfd; - _PyTime_t ms; + PyTime_t ms; #else fd_set fds, efds; struct timeval tv, *tvp; @@ -888,10 +889,10 @@ sock_call_ex(PySocketSockObject *s, void *data, int connect, int *err, - _PyTime_t timeout) + PyTime_t timeout) { int has_timeout = (timeout > 0); - _PyTime_t deadline = 0; + PyTime_t deadline = 0; int deadline_initialized = 0; int res; @@ -905,7 +906,7 @@ sock_call_ex(PySocketSockObject *s, runs asynchronously. */ if (has_timeout || connect) { if (has_timeout) { - _PyTime_t interval; + PyTime_t interval; if (deadline_initialized) { /* recompute the timeout */ @@ -1053,8 +1054,8 @@ init_sockobject(socket_state *state, PySocketSockObject *s, else #endif { - s->sock_timeout = state->defaulttimeout; - if (state->defaulttimeout >= 0) { + s->sock_timeout = _Py_atomic_load_int64_relaxed(&state->defaulttimeout); + if (s->sock_timeout >= 0) { if (internal_setblocking(s, 0) == -1) { return -1; } @@ -1886,12 +1887,14 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, #ifdef AF_RDS case AF_RDS: - /* RDS sockets use sockaddr_in: fall-through */ + /* RDS sockets use sockaddr_in */ + _Py_FALLTHROUGH; #endif /* AF_RDS */ #ifdef AF_DIVERT case AF_DIVERT: - /* FreeBSD divert(4) sockets use sockaddr_in: fall-through */ + /* FreeBSD divert(4) sockets use sockaddr_in */ + _Py_FALLTHROUGH; #endif /* AF_DIVERT */ case AF_INET: @@ -2213,7 +2216,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, switch (s->sock_proto) { #ifdef CAN_RAW case CAN_RAW: - /* fall-through */ + _Py_FALLTHROUGH; #endif #ifdef CAN_BCM case CAN_BCM: @@ -2589,7 +2592,8 @@ getsockaddrlen(PySocketSockObject *s, socklen_t *len_ret) #ifdef AF_RDS case AF_RDS: - /* RDS sockets use sockaddr_in: fall-through */ + /* RDS sockets use sockaddr_in */ + _Py_FALLTHROUGH; #endif /* AF_RDS */ case AF_INET: @@ -3011,13 +3015,13 @@ Returns True if socket is in blocking mode, or False if it\n\ is in non-blocking mode."); static int -socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) +socket_parse_timeout(PyTime_t *timeout, PyObject *timeout_obj) { #ifdef MS_WINDOWS struct timeval tv; #endif #ifndef HAVE_POLL - _PyTime_t ms; + PyTime_t ms; #endif int overflow = 0; @@ -3060,7 +3064,7 @@ socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) static PyObject * sock_settimeout(PySocketSockObject *s, PyObject *arg) { - _PyTime_t timeout; + PyTime_t timeout; if (socket_parse_timeout(&timeout, arg) < 0) return NULL; @@ -3112,7 +3116,7 @@ sock_gettimeout(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } else { - double seconds = _PyTime_AsSecondsDouble(s->sock_timeout); + double seconds = PyTime_AsSecondsDouble(s->sock_timeout); return PyFloat_FromDouble(seconds); } } @@ -3162,6 +3166,17 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) /* setsockopt(level, opt, flag) */ if (PyArg_ParseTuple(args, "iii:setsockopt", &level, &optname, &flag)) { +#ifdef MS_WINDOWS + if (optname == SIO_TCP_SET_ACK_FREQUENCY) { + int dummy; + res = WSAIoctl(s->sock_fd, SIO_TCP_SET_ACK_FREQUENCY, &flag, + sizeof(flag), NULL, 0, &dummy, NULL, NULL); + if (res >= 0) { + s->quickack = flag; + } + goto done; + } +#endif res = setsockopt(s->sock_fd, level, optname, (char*)&flag, sizeof flag); goto done; @@ -3247,6 +3262,11 @@ sock_getsockopt(PySocketSockObject *s, PyObject *args) return s->errorhandler(); return PyLong_FromUnsignedLong(vflag); } +#endif +#ifdef MS_WINDOWS + if (optname == SIO_TCP_SET_ACK_FREQUENCY) { + return PyLong_FromLong(s->quickack); + } #endif flagsize = sizeof flag; res = getsockopt(s->sock_fd, level, optname, @@ -3327,8 +3347,19 @@ sockets the address is a tuple (ifname, proto [,pkttype [,hatype [,addr]]])"); Set the file descriptor to -1 so operations tried subsequently will surely fail. */ +/*[clinic input] +@critical_section +_socket.socket.close + self as s: self(type="PySocketSockObject *") + +close() + +Close the socket. It cannot be used after this call. +[clinic start generated code]*/ + static PyObject * -sock_close(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +_socket_socket_close_impl(PySocketSockObject *s) +/*[clinic end generated code: output=038b2418e07f6f6c input=9839a261e05bcb97]*/ { SOCKET_T fd; int res; @@ -3353,11 +3384,6 @@ sock_close(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -PyDoc_STRVAR(sock_close_doc, -"close()\n\ -\n\ -Close the socket. It cannot be used after this call."); - static PyObject * sock_detach(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) { @@ -3395,6 +3421,18 @@ sock_connect_impl(PySocketSockObject *s, void* Py_UNUSED(data)) return 1; } +/* Common functionality for socket.connect and socket.connect_ex. + * + * If *raise* is set: + * - On success, return 0. + * - On any failure, return -1 with an exception set. + * If *raise* is zero: + * - On success, return 0. + * - On connect() failure, return errno (without an exception set) + * - On other error, return -1 with an exception set. + * + * Note that -1 is a valid errno value on some systems. + */ static int internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, int raise) @@ -3479,8 +3517,10 @@ sock_connect(PySocketSockObject *s, PyObject *addro) } res = internal_connect(s, SAS2SA(&addrbuf), addrlen, 1); - if (res < 0) + if (res < 0) { + assert(PyErr_Occurred()); return NULL; + } Py_RETURN_NONE; } @@ -3510,8 +3550,9 @@ sock_connect_ex(PySocketSockObject *s, PyObject *addro) } res = internal_connect(s, SAS2SA(&addrbuf), addrlen, 0); - if (res < 0) + if (res == -1 && PyErr_Occurred()) { return NULL; + } return PyLong_FromLong((long) res); } @@ -4382,8 +4423,8 @@ sock_sendall(PySocketSockObject *s, PyObject *args) Py_buffer pbuf; struct sock_send ctx; int has_timeout = (s->sock_timeout > 0); - _PyTime_t timeout = s->sock_timeout; - _PyTime_t deadline = 0; + PyTime_t timeout = s->sock_timeout; + PyTime_t deadline = 0; int deadline_initialized = 0; PyObject *res = NULL; @@ -5114,8 +5155,7 @@ static PyMethodDef sock_methods[] = { {"bind", (PyCFunction)sock_bind, METH_O, bind_doc}, #endif - {"close", (PyCFunction)sock_close, METH_NOARGS, - sock_close_doc}, + _SOCKET_SOCKET_CLOSE_METHODDEF #ifdef HAVE_CONNECT {"connect", (PyCFunction)sock_connect, METH_O, connect_doc}, @@ -5307,6 +5347,9 @@ sock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) ((PySocketSockObject *)new)->sock_fd = INVALID_SOCKET; ((PySocketSockObject *)new)->sock_timeout = _PyTime_FromSeconds(-1); ((PySocketSockObject *)new)->errorhandler = &set_error; +#ifdef MS_WINDOWS + ((PySocketSockObject *)new)->quickack = 0; +#endif } return new; } @@ -5593,7 +5636,7 @@ socket_gethostname(PyObject *self, PyObject *unused) return PyErr_SetFromWindowsErr(0); if (size == 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* MSDN says ERROR_MORE_DATA may occur because DNS allows longer names */ @@ -6912,11 +6955,12 @@ static PyObject * socket_getdefaulttimeout(PyObject *self, PyObject *Py_UNUSED(ignored)) { socket_state *state = get_module_state(self); - if (state->defaulttimeout < 0) { + PyTime_t timeout = _Py_atomic_load_int64_relaxed(&state->defaulttimeout); + if (timeout < 0) { Py_RETURN_NONE; } else { - double seconds = _PyTime_AsSecondsDouble(state->defaulttimeout); + double seconds = PyTime_AsSecondsDouble(timeout); return PyFloat_FromDouble(seconds); } } @@ -6931,13 +6975,13 @@ When the socket module is first imported, the default is None."); static PyObject * socket_setdefaulttimeout(PyObject *self, PyObject *arg) { - _PyTime_t timeout; + PyTime_t timeout; if (socket_parse_timeout(&timeout, arg) < 0) return NULL; socket_state *state = get_module_state(self); - state->defaulttimeout = timeout; + _Py_atomic_store_int64_relaxed(&state->defaulttimeout, timeout); Py_RETURN_NONE; } @@ -7594,6 +7638,7 @@ socket_exec(PyObject *m) ADD_INT_CONST(m, "SO_VM_SOCKETS_BUFFER_MAX_SIZE", 2); ADD_INT_CONST(m, "VMADDR_CID_ANY", 0xffffffff); ADD_INT_CONST(m, "VMADDR_PORT_ANY", 0xffffffff); + ADD_INT_CONST(m, "VMADDR_CID_LOCAL", 1); ADD_INT_CONST(m, "VMADDR_CID_HOST", 2); ADD_INT_CONST(m, "VM_SOCKETS_INVALID_VERSION", 0xffffffff); ADD_INT_CONST(m, "IOCTL_VM_SOCKETS_GET_LOCAL_CID", _IO(7, 0xb9)); @@ -7876,6 +7921,9 @@ socket_exec(PyObject *m) #ifdef SO_OOBINLINE ADD_INT_MACRO(m, SO_OOBINLINE); #endif +#ifdef SO_ORIGINAL_DST + ADD_INT_MACRO(m, SO_ORIGINAL_DST); +#endif #ifndef __GNU__ #ifdef SO_REUSEPORT ADD_INT_MACRO(m, SO_REUSEPORT); @@ -7926,6 +7974,9 @@ socket_exec(PyObject *m) #ifdef SO_BINDTODEVICE ADD_INT_MACRO(m, SO_BINDTODEVICE); #endif +#ifdef SO_BINDTOIFINDEX + ADD_INT_MACRO(m, SO_BINDTOIFINDEX); +#endif #ifdef SO_PRIORITY ADD_INT_MACRO(m, SO_PRIORITY); #endif @@ -8406,15 +8457,24 @@ socket_exec(PyObject *m) #ifdef IP_TTL ADD_INT_MACRO(m, IP_TTL); #endif +#ifdef IP_RECVERR + ADD_INT_MACRO(m, IP_RECVERR); +#endif #ifdef IP_RECVOPTS ADD_INT_MACRO(m, IP_RECVOPTS); #endif +#ifdef IP_RECVORIGDSTADDR + ADD_INT_MACRO(m, IP_RECVORIGDSTADDR); +#endif #ifdef IP_RECVRETOPTS ADD_INT_MACRO(m, IP_RECVRETOPTS); #endif #ifdef IP_RECVTOS ADD_INT_MACRO(m, IP_RECVTOS); #endif +#ifdef IP_RECVTTL + ADD_INT_MACRO(m, IP_RECVTTL); +#endif #ifdef IP_RECVDSTADDR ADD_INT_MACRO(m, IP_RECVDSTADDR); #endif @@ -8518,6 +8578,9 @@ socket_exec(PyObject *m) #ifdef IPV6_RECVDSTOPTS ADD_INT_MACRO(m, IPV6_RECVDSTOPTS); #endif +#ifdef IPV6_RECVERR + ADD_INT_MACRO(m, IPV6_RECVERR); +#endif #ifdef IPV6_RECVHOPLIMIT ADD_INT_MACRO(m, IPV6_RECVHOPLIMIT); #endif @@ -8593,6 +8656,9 @@ socket_exec(PyObject *m) #ifdef TCP_CONNECTION_INFO ADD_INT_MACRO(m, TCP_CONNECTION_INFO); #endif +#ifdef SIO_TCP_SET_ACK_FREQUENCY +#define TCP_QUICKACK SIO_TCP_SET_ACK_FREQUENCY +#endif #ifdef TCP_QUICKACK ADD_INT_MACRO(m, TCP_QUICKACK); #endif @@ -8891,6 +8957,7 @@ socket_exec(PyObject *m) static struct PyModuleDef_Slot socket_slots[] = { {Py_mod_exec, socket_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 47146a28e02c8f..e1d96377728eb9 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -1,7 +1,5 @@ /* Socket module header file */ -#include "pycore_time.h" // _PyTime_t - /* Includes needed for the sockaddr_* symbols below */ #ifndef MS_WINDOWS #ifdef __VMS @@ -174,6 +172,10 @@ typedef int socklen_t; # undef AF_VSOCK #endif +#ifdef HAVE_LINUX_NETFILTER_IPV4_H +# include +#endif + #ifdef HAVE_SOCKADDR_ALG # include @@ -324,9 +326,12 @@ typedef struct { PyObject *(*errorhandler)(void); /* Error handler; checks errno, returns NULL and sets a Python exception */ - _PyTime_t sock_timeout; /* Operation timeout in seconds; + PyTime_t sock_timeout; /* Operation timeout in seconds; 0.0 means non-blocking */ struct _socket_state *state; +#ifdef MS_WINDOWS + int quickack; +#endif } PySocketSockObject; /* --- C API ----------------------------------------------------*/ diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index ddc9ac3324356d..d0d5223e5acea8 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -75,24 +75,27 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntMacro(m, DEF_NONLOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_LOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_PARAM) < 0) return -1; - if (PyModule_AddIntMacro(m, DEF_FREE) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_TYPE_PARAM) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_FREE_CLASS) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_IMPORT) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_BOUND) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_ANNOT) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_COMP_ITER) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_COMP_CELL) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock) < 0) return -1; + if (PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock) < 0) + return -1; if (PyModule_AddIntConstant(m, "TYPE_MODULE", ModuleBlock) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_ANNOTATION", AnnotationBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_TYPE_VAR_BOUND", TypeVarBoundBlock) < 0) - return -1; if (PyModule_AddIntConstant(m, "TYPE_TYPE_ALIAS", TypeAliasBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_TYPE_PARAM", TypeParamBlock) < 0) + if (PyModule_AddIntConstant(m, "TYPE_TYPE_PARAMETERS", TypeParametersBlock) < 0) + return -1; + if (PyModule_AddIntConstant(m, "TYPE_TYPE_VARIABLE", TypeVariableBlock) < 0) return -1; if (PyModule_AddIntMacro(m, LOCAL) < 0) return -1; @@ -110,6 +113,7 @@ symtable_init_constants(PyObject *m) static PyModuleDef_Slot symtable_slots[] = { {Py_mod_exec, symtable_init_constants}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 62c7816f891ee2..14e7ca591a076b 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -132,6 +132,7 @@ syslog_get_argv(void) /*[clinic input] +@critical_section syslog.openlog ident: unicode = NULL @@ -144,7 +145,7 @@ Set logging options of subsequent syslog() calls. static PyObject * syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, long facility) -/*[clinic end generated code: output=5476c12829b6eb75 input=8a987a96a586eee7]*/ +/*[clinic end generated code: output=5476c12829b6eb75 input=ee700b8786f81c23]*/ { // Since the sys.openlog changes the process level state of syslog library, // this operation is only allowed for the main interpreter. @@ -189,6 +190,7 @@ syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, /*[clinic input] +@critical_section syslog.syslog [ @@ -205,7 +207,7 @@ Send the string message to the system logger. static PyObject * syslog_syslog_impl(PyObject *module, int group_left_1, int priority, const char *message) -/*[clinic end generated code: output=c3dbc73445a0e078 input=ac83d92b12ea3d4e]*/ +/*[clinic end generated code: output=c3dbc73445a0e078 input=6588ddb0b113af8e]*/ { if (PySys_Audit("syslog.syslog", "is", priority, message) < 0) { return NULL; @@ -243,6 +245,7 @@ syslog_syslog_impl(PyObject *module, int group_left_1, int priority, /*[clinic input] +@critical_section syslog.closelog Reset the syslog module values and call the system library closelog(). @@ -250,7 +253,7 @@ Reset the syslog module values and call the system library closelog(). static PyObject * syslog_closelog_impl(PyObject *module) -/*[clinic end generated code: output=97890a80a24b1b84 input=fb77a54d447acf07]*/ +/*[clinic end generated code: output=97890a80a24b1b84 input=167f489868bd5a72]*/ { // Since the sys.closelog changes the process level state of syslog library, // this operation is only allowed for the main interpreter. @@ -436,6 +439,7 @@ syslog_exec(PyObject *module) static PyModuleDef_Slot syslog_slots[] = { {Py_mod_exec, syslog_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/termios.c b/Modules/termios.c index 69dbd88be5fcc2..efb5fcc17fa5ef 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1,11 +1,19 @@ /* termios.c -- POSIX terminal I/O module implementation. */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyLong_AsInt() +// in code generated by Argument Clinic. +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" +#include // memset() +#include +#include +#include // _POSIX_VDISABLE + // On QNX 6, struct termio must be declared by including sys/termio.h // if TCGETA, TCSETA, TCSETAW, or TCSETAF are used. sys/termio.h must // be included before termios.h or it will generate an error. @@ -19,16 +27,10 @@ # define CTRL(c) ((c)&037) #endif +// We could do better. Check bpo-32660 #if defined(__sun) -/* We could do better. Check issue-32660 */ -#include -#include -#endif - -#include -#include -#if defined(__sun) && defined(__SVR4) -# include // ioctl() +# include +# include #endif /* HP-UX requires that this be included to pick up MDCD, MCTS, MDSR, @@ -36,13 +38,15 @@ * defined as macros; these are not used here directly). */ #ifdef HAVE_SYS_MODEM_H -#include +# include #endif + /* HP-UX requires that this be included to pick up TIOCGPGRP and friends */ #ifdef HAVE_SYS_BSDTTY_H -#include +# include #endif + /*[clinic input] module termios [clinic start generated code]*/ @@ -122,7 +126,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) v = PyBytes_FromStringAndSize(&ch, 1); if (v == NULL) goto err; - PyList_SET_ITEM(cc, i, v); + PyList_SetItem(cc, i, v); } /* Convert the MIN and TIME slots to integer. On some systems, the @@ -156,7 +160,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) Py_DECREF(v); \ goto err; \ } \ - PyList_SET_ITEM(v, index, l); \ + PyList_SetItem(v, index, l); \ } while (0) ADD_LONG_ITEM(0, mode.c_iflag); @@ -167,7 +171,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) ADD_LONG_ITEM(5, ospeed); #undef ADD_LONG_ITEM - PyList_SET_ITEM(v, 6, cc); + PyList_SetItem(v, 6, cc); return v; err: Py_DECREF(cc); @@ -216,7 +220,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) speed_t ispeed, ospeed; #define SET_FROM_LIST(TYPE, VAR, LIST, N) do { \ - PyObject *item = PyList_GET_ITEM(LIST, N); \ + PyObject *item = PyList_GetItem(LIST, N); \ long num = PyLong_AsLong(item); \ if (num == -1 && PyErr_Occurred()) { \ return NULL; \ @@ -232,7 +236,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) SET_FROM_LIST(speed_t, ospeed, term, 5); #undef SET_FROM_LIST - PyObject *cc = PyList_GET_ITEM(term, 6); + PyObject *cc = PyList_GetItem(term, 6); if (!PyList_Check(cc) || PyList_Size(cc) != NCCS) { PyErr_Format(PyExc_TypeError, "tcsetattr: attributes[6] must be %d element list", @@ -256,7 +260,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) } else { PyErr_SetString(PyExc_TypeError, - "tcsetattr: elements of attributes must be characters or integers"); + "tcsetattr: elements of attributes must be bytes objects of length 1 or integers"); return NULL; } } @@ -1315,6 +1319,9 @@ static struct constant { #ifdef TIOCTTYGSTRUCT {"TIOCTTYGSTRUCT", TIOCTTYGSTRUCT}, #endif +#ifdef _POSIX_VDISABLE + {"_POSIX_VDISABLE", _POSIX_VDISABLE}, +#endif /* sentinel */ {NULL, 0} @@ -1345,9 +1352,21 @@ termios_exec(PyObject *mod) } while (constant->name != NULL) { - if (PyModule_AddIntConstant( - mod, constant->name, constant->value) < 0) { - return -1; + if (strncmp(constant->name, "TIO", 3) == 0) { + // gh-119770: Convert value to unsigned int for ioctl() constants, + // constants can be negative on macOS whereas ioctl() expects an + // unsigned long 'request'. + unsigned int value = constant->value & UINT_MAX; + if (PyModule_Add(mod, constant->name, + PyLong_FromUnsignedLong(value)) < 0) { + return -1; + } + } + else { + if (PyModule_AddIntConstant( + mod, constant->name, constant->value) < 0) { + return -1; + } } ++constant; } @@ -1357,6 +1376,7 @@ termios_exec(PyObject *mod) static PyModuleDef_Slot termios_slots[] = { {Py_mod_exec, termios_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 2b0d3900dbddd6..340011fc08b551 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -5,6 +5,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_time.h" // _PyTimeFraction #include // clock() #ifdef HAVE_SYS_TIMES_H @@ -70,12 +71,13 @@ module time /* Forward declarations */ -static int pysleep(_PyTime_t timeout); +static int pysleep(PyTime_t timeout); typedef struct { PyTypeObject *struct_time_type; -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) // times() clock frequency in hertz _PyTimeFraction times_base; #endif @@ -95,26 +97,18 @@ get_time_state(PyObject *module) static PyObject* -_PyFloat_FromPyTime(_PyTime_t t) +_PyFloat_FromPyTime(PyTime_t t) { - double d = _PyTime_AsSecondsDouble(t); + double d = PyTime_AsSecondsDouble(t); return PyFloat_FromDouble(d); } -static int -get_system_time(_PyTime_t *t) -{ - // Avoid _PyTime_GetSystemClock() which silently ignores errors. - return _PyTime_GetSystemClockWithInfo(t, NULL); -} - - static PyObject * time_time(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_system_time(&t) < 0) { + PyTime_t t; + if (PyTime_Time(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -122,7 +116,7 @@ time_time(PyObject *self, PyObject *unused) PyDoc_STRVAR(time_doc, -"time() -> floating point number\n\ +"time() -> floating-point number\n\ \n\ Return the current time in seconds since the Epoch.\n\ Fractions of a second may be present if the system clock provides them."); @@ -130,11 +124,11 @@ Fractions of a second may be present if the system clock provides them."); static PyObject * time_time_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_system_time(&t) < 0) { + PyTime_t t; + if (PyTime_Time(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(time_ns_doc, @@ -153,7 +147,7 @@ Return the current time in nanoseconds since the Epoch."); #endif static int -py_clock(time_module_state *state, _PyTime_t *tp, _Py_clock_info_t *info) +py_clock(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { _PyTimeFraction *base = &state->clock_base; @@ -171,8 +165,7 @@ py_clock(time_module_state *state, _PyTime_t *tp, _Py_clock_info_t *info) "or its value cannot be represented"); return -1; } - _PyTime_t ns = _PyTimeFraction_Mul(ticks, base); - *tp = _PyTime_FromNanoseconds(ns); + *tp = _PyTimeFraction_Mul(ticks, base); return 0; } #endif /* HAVE_CLOCK */ @@ -262,11 +255,11 @@ time_clock_gettime_ns_impl(PyObject *module, clockid_t clk_id) return NULL; } - _PyTime_t t; + PyTime_t t; if (_PyTime_FromTimespec(&t, &ts) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } #endif /* HAVE_CLOCK_GETTIME */ @@ -276,7 +269,7 @@ time_clock_settime(PyObject *self, PyObject *args) { int clk_id; PyObject *obj; - _PyTime_t t; + PyTime_t t; struct timespec tp; int ret; @@ -307,7 +300,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) { int clk_id; PyObject *obj; - _PyTime_t t; + PyTime_t t; struct timespec ts; int ret; @@ -315,7 +308,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) return NULL; } - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (_PyTime_AsTimespec(t, &ts) == -1) { @@ -357,7 +350,7 @@ time_clock_getres(PyObject *self, PyObject *args) } PyDoc_STRVAR(clock_getres_doc, -"clock_getres(clk_id) -> floating point number\n\ +"clock_getres(clk_id) -> floating-point number\n\ \n\ Return the resolution (precision) of the specified clock clk_id."); @@ -402,7 +395,7 @@ time_sleep(PyObject *self, PyObject *timeout_obj) return NULL; } - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT)) return NULL; if (timeout < 0) { @@ -420,7 +413,7 @@ PyDoc_STRVAR(sleep_doc, "sleep(seconds)\n\ \n\ Delay execution for a given number of seconds. The argument may be\n\ -a floating point number for subsecond precision."); +a floating-point number for subsecond precision."); static PyStructSequence_Field struct_time_type_fields[] = { {"tm_year", "year, for example, 1993"}, @@ -469,7 +462,18 @@ tmtotuple(time_module_state *state, struct tm *p if (v == NULL) return NULL; -#define SET(i,val) PyStructSequence_SET_ITEM(v, i, PyLong_FromLong((long) val)) +#define SET_ITEM(INDEX, CALL) \ + do { \ + PyObject *obj = (CALL); \ + if (obj == NULL) { \ + Py_DECREF(v); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(v, (INDEX), obj); \ + } while (0) + +#define SET(INDEX, VAL) \ + SET_ITEM((INDEX), PyLong_FromLong((long) (VAL))) SET(0, p->tm_year + 1900); SET(1, p->tm_mon + 1); /* Want January == 1 */ @@ -481,19 +485,15 @@ tmtotuple(time_module_state *state, struct tm *p SET(7, p->tm_yday + 1); /* Want January, 1 == 1 */ SET(8, p->tm_isdst); #ifdef HAVE_STRUCT_TM_TM_ZONE - PyStructSequence_SET_ITEM(v, 9, - PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); + SET_ITEM(9, PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); SET(10, p->tm_gmtoff); #else - PyStructSequence_SET_ITEM(v, 9, - PyUnicode_DecodeLocale(zone, "surrogateescape")); - PyStructSequence_SET_ITEM(v, 10, _PyLong_FromTime_t(gmtoff)); + SET_ITEM(9, PyUnicode_DecodeLocale(zone, "surrogateescape")); + SET_ITEM(10, _PyLong_FromTime_t(gmtoff)); #endif /* HAVE_STRUCT_TM_TM_ZONE */ + #undef SET - if (PyErr_Occurred()) { - Py_XDECREF(v); - return NULL; - } +#undef SET_ITEM return v; } @@ -739,10 +739,6 @@ checktm(struct tm* buf) return 1; } -#ifdef MS_WINDOWS - /* wcsftime() doesn't format correctly time zones, see issue #10653 */ -# undef HAVE_WCSFTIME -#endif #define STRFTIME_FORMAT_CODES \ "Commonly used format codes:\n\ \n\ @@ -776,27 +772,100 @@ the C library strftime function.\n" #endif static PyObject * -time_strftime(PyObject *module, PyObject *args) +time_strftime1(time_char **outbuf, size_t *bufsize, + time_char *format, size_t fmtlen, + struct tm *tm) { - PyObject *tup = NULL; - struct tm buf; - const time_char *fmt; + size_t buflen; +#if defined(MS_WINDOWS) && !defined(HAVE_WCSFTIME) + /* check that the format string contains only valid directives */ + for (const time_char *f = strchr(format, '%'); + f != NULL; + f = strchr(f + 2, '%')) + { + if (f[1] == '#') + ++f; /* not documented by python, */ + if (f[1] == '\0') + break; + if ((f[1] == 'y') && tm->tm_year < 0) { + PyErr_SetString(PyExc_ValueError, + "format %y requires year >= 1900 on Windows"); + return NULL; + } + } +#elif (defined(_AIX) || (defined(__sun) && defined(__SVR4))) && defined(HAVE_WCSFTIME) + for (const time_char *f = wcschr(format, '%'); + f != NULL; + f = wcschr(f + 2, '%')) + { + if (f[1] == L'\0') + break; + /* Issue #19634: On AIX, wcsftime("y", (1899, 1, 1, 0, 0, 0, 0, 0, 0)) + returns "0/" instead of "99" */ + if (f[1] == L'y' && tm->tm_year < 0) { + PyErr_SetString(PyExc_ValueError, + "format %y requires year >= 1900 on AIX"); + return NULL; + } + } +#endif + + /* I hate these functions that presume you know how big the output + * will be ahead of time... + */ + while (1) { + if (*bufsize > PY_SSIZE_T_MAX/sizeof(time_char)) { + PyErr_NoMemory(); + return NULL; + } + *outbuf = (time_char *)PyMem_Realloc(*outbuf, + *bufsize*sizeof(time_char)); + if (*outbuf == NULL) { + PyErr_NoMemory(); + return NULL; + } +#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) + errno = 0; +#endif + _Py_BEGIN_SUPPRESS_IPH + buflen = format_time(*outbuf, *bufsize, format, tm); + _Py_END_SUPPRESS_IPH +#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) + /* VisualStudio .NET 2005 does this properly */ + if (buflen == 0 && errno == EINVAL) { + PyErr_SetString(PyExc_ValueError, "Invalid format string"); + return NULL; + } +#endif + if (buflen == 0 && *bufsize < 256 * fmtlen) { + *bufsize += *bufsize; + continue; + } + /* If the buffer is 256 times as long as the format, + it's probably not failing for lack of room! + More likely, the format yields an empty result, + e.g. an empty format, or %Z when the timezone + is unknown. */ #ifdef HAVE_WCSFTIME - wchar_t *format; + return PyUnicode_FromWideChar(*outbuf, buflen); #else - PyObject *format; + return PyUnicode_DecodeLocaleAndSize(*outbuf, buflen, "surrogateescape"); #endif + } +} + +static PyObject * +time_strftime(PyObject *module, PyObject *args) +{ + PyObject *tup = NULL; + struct tm buf; PyObject *format_arg; - size_t fmtlen, buflen; - time_char *outbuf = NULL; - size_t i; - PyObject *ret = NULL; + Py_ssize_t format_size; + time_char *format, *outbuf = NULL; + size_t fmtlen, bufsize = 1024; memset((void *) &buf, '\0', sizeof(buf)); - /* Will always expect a unicode string to be passed as format. - Given that there's no str type anymore in py3k this seems safe. - */ if (!PyArg_ParseTuple(args, "U|O:strftime", &format_arg, &tup)) return NULL; @@ -813,7 +882,12 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } -#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) +// Some platforms only support a limited range of years. +// +// Android works with negative years on the emulator, but fails on some +// physical devices (#123017). +#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) \ + || defined(__VXWORKS__) || defined(__ANDROID__) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); @@ -829,101 +903,63 @@ time_strftime(PyObject *module, PyObject *args) else if (buf.tm_isdst > 1) buf.tm_isdst = 1; -#ifdef HAVE_WCSFTIME - format = PyUnicode_AsWideCharString(format_arg, NULL); - if (format == NULL) + format_size = PyUnicode_GET_LENGTH(format_arg); + if ((size_t)format_size > PY_SSIZE_T_MAX/sizeof(time_char) - 1) { + PyErr_NoMemory(); return NULL; - fmt = format; -#else - /* Convert the unicode string to an ascii one */ - format = PyUnicode_EncodeLocale(format_arg, "surrogateescape"); - if (format == NULL) + } + format = PyMem_Malloc((format_size + 1)*sizeof(time_char)); + if (format == NULL) { + PyErr_NoMemory(); return NULL; - fmt = PyBytes_AS_STRING(format); -#endif - -#if defined(MS_WINDOWS) && !defined(HAVE_WCSFTIME) - /* check that the format string contains only valid directives */ - for (outbuf = strchr(fmt, '%'); - outbuf != NULL; - outbuf = strchr(outbuf+2, '%')) - { - if (outbuf[1] == '#') - ++outbuf; /* not documented by python, */ - if (outbuf[1] == '\0') - break; - if ((outbuf[1] == 'y') && buf.tm_year < 0) { - PyErr_SetString(PyExc_ValueError, - "format %y requires year >= 1900 on Windows"); - Py_DECREF(format); - return NULL; - } } -#elif (defined(_AIX) || (defined(__sun) && defined(__SVR4))) && defined(HAVE_WCSFTIME) - for (outbuf = wcschr(fmt, '%'); - outbuf != NULL; - outbuf = wcschr(outbuf+2, '%')) - { - if (outbuf[1] == L'\0') - break; - /* Issue #19634: On AIX, wcsftime("y", (1899, 1, 1, 0, 0, 0, 0, 0, 0)) - returns "0/" instead of "99" */ - if (outbuf[1] == L'y' && buf.tm_year < 0) { - PyErr_SetString(PyExc_ValueError, - "format %y requires year >= 1900 on AIX"); - PyMem_Free(format); - return NULL; + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + writer.overallocate = 1; + Py_ssize_t i = 0; + while (i < format_size) { + fmtlen = 0; + for (; i < format_size; i++) { + Py_UCS4 c = PyUnicode_READ_CHAR(format_arg, i); + if (!c || c > 127) { + break; + } + format[fmtlen++] = (char)c; } - } -#endif - - fmtlen = time_strlen(fmt); - - /* I hate these functions that presume you know how big the output - * will be ahead of time... - */ - for (i = 1024; ; i += i) { - outbuf = (time_char *)PyMem_Malloc(i*sizeof(time_char)); - if (outbuf == NULL) { - PyErr_NoMemory(); - break; + if (fmtlen) { + format[fmtlen] = 0; + PyObject *unicode = time_strftime1(&outbuf, &bufsize, + format, fmtlen, &buf); + if (unicode == NULL) { + goto error; + } + if (_PyUnicodeWriter_WriteStr(&writer, unicode) < 0) { + Py_DECREF(unicode); + goto error; + } + Py_DECREF(unicode); } -#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) - errno = 0; -#endif - _Py_BEGIN_SUPPRESS_IPH - buflen = format_time(outbuf, i, fmt, &buf); - _Py_END_SUPPRESS_IPH -#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) - /* VisualStudio .NET 2005 does this properly */ - if (buflen == 0 && errno == EINVAL) { - PyErr_SetString(PyExc_ValueError, "Invalid format string"); - PyMem_Free(outbuf); - break; + + Py_ssize_t start = i; + for (; i < format_size; i++) { + Py_UCS4 c = PyUnicode_READ_CHAR(format_arg, i); + if (c == '%') { + break; + } } -#endif - if (buflen > 0 || i >= 256 * fmtlen) { - /* If the buffer is 256 times as long as the format, - it's probably not failing for lack of room! - More likely, the format yields an empty result, - e.g. an empty format, or %Z when the timezone - is unknown. */ -#ifdef HAVE_WCSFTIME - ret = PyUnicode_FromWideChar(outbuf, buflen); -#else - ret = PyUnicode_DecodeLocaleAndSize(outbuf, buflen, "surrogateescape"); -#endif - PyMem_Free(outbuf); - break; + if (_PyUnicodeWriter_WriteSubstring(&writer, format_arg, start, i) < 0) { + goto error; } - PyMem_Free(outbuf); } -#ifdef HAVE_WCSFTIME + + PyMem_Free(outbuf); PyMem_Free(format); -#else - Py_DECREF(format); -#endif - return ret; + return _PyUnicodeWriter_Finish(&writer); +error: + PyMem_Free(outbuf); + PyMem_Free(format); + _PyUnicodeWriter_Dealloc(&writer); + return NULL; } #undef time_char @@ -1104,7 +1140,7 @@ time_mktime(PyObject *module, PyObject *tm_tuple) } PyDoc_STRVAR(mktime_doc, -"mktime(tuple) -> floating point number\n\ +"mktime(tuple) -> floating-point number\n\ \n\ Convert a time tuple in local time to seconds since the Epoch.\n\ Note that mktime(gmtime(0)) will not generally return zero for most\n\ @@ -1155,19 +1191,11 @@ should not be relied on."); #endif /* HAVE_WORKING_TZSET */ -static int -get_monotonic(_PyTime_t *t) -{ - // Avoid _PyTime_GetMonotonicClock() which silently ignores errors. - return _PyTime_GetMonotonicClockWithInfo(t, NULL); -} - - static PyObject * time_monotonic(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_monotonic(&t) < 0) { + PyTime_t t; + if (PyTime_Monotonic(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1181,11 +1209,11 @@ Monotonic clock, cannot go backward."); static PyObject * time_monotonic_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_monotonic(&t) < 0) { + PyTime_t t; + if (PyTime_Monotonic(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(monotonic_ns_doc, @@ -1194,19 +1222,11 @@ PyDoc_STRVAR(monotonic_ns_doc, Monotonic clock, cannot go backward, as nanoseconds."); -static int -get_perf_counter(_PyTime_t *t) -{ - // Avoid _PyTime_GetPerfCounter() which silently ignores errors. - return _PyTime_GetPerfCounterWithInfo(t, NULL); -} - - static PyObject * time_perf_counter(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_perf_counter(&t) < 0) { + PyTime_t t; + if (PyTime_PerfCounter(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1221,11 +1241,11 @@ Performance counter for benchmarking."); static PyObject * time_perf_counter_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_perf_counter(&t) < 0) { + PyTime_t t; + if (PyTime_PerfCounter(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(perf_counter_ns_doc, @@ -1234,9 +1254,10 @@ PyDoc_STRVAR(perf_counter_ns_doc, Performance counter for benchmarking as nanoseconds."); -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) static int -process_time_times(time_module_state *state, _PyTime_t *tp, +process_time_times(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { _PyTimeFraction *base = &state->times_base; @@ -1253,24 +1274,24 @@ process_time_times(time_module_state *state, _PyTime_t *tp, info->adjustable = 0; } - _PyTime_t ns; + PyTime_t ns; ns = _PyTimeFraction_Mul(process.tms_utime, base); ns += _PyTimeFraction_Mul(process.tms_stime, base); - *tp = _PyTime_FromNanoseconds(ns); + *tp = ns; return 1; } #endif static int -py_process_time(time_module_state *state, _PyTime_t *tp, +py_process_time(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { #if defined(MS_WINDOWS) HANDLE process; FILETIME creation_time, exit_time, kernel_time, user_time; ULARGE_INTEGER large; - _PyTime_t ktime, utime, t; + PyTime_t ktime, utime; BOOL ok; process = GetCurrentProcess(); @@ -1297,14 +1318,20 @@ py_process_time(time_module_state *state, _PyTime_t *tp, utime = large.QuadPart; /* ktime and utime have a resolution of 100 nanoseconds */ - t = _PyTime_FromNanoseconds((ktime + utime) * 100); - *tp = t; + *tp = (ktime + utime) * 100; return 0; #else /* clock_gettime */ +// gh-115714: Don't use CLOCK_PROCESS_CPUTIME_ID on WASI. +/* CLOCK_PROF is defined on NetBSD, but not supported. + * CLOCK_PROCESS_CPUTIME_ID is broken on NetBSD for the same reason as + * CLOCK_THREAD_CPUTIME_ID (see comment below). + */ #if defined(HAVE_CLOCK_GETTIME) \ - && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) + && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) \ + && !defined(__wasi__) \ + && !defined(__NetBSD__) struct timespec ts; if (HAVE_CLOCK_GETTIME_RUNTIME) { @@ -1343,7 +1370,7 @@ py_process_time(time_module_state *state, _PyTime_t *tp, struct rusage ru; if (getrusage(RUSAGE_SELF, &ru) == 0) { - _PyTime_t utime, stime; + PyTime_t utime, stime; if (info) { info->implementation = "getrusage(RUSAGE_SELF)"; @@ -1359,14 +1386,15 @@ py_process_time(time_module_state *state, _PyTime_t *tp, return -1; } - _PyTime_t total = utime + stime; + PyTime_t total = utime + stime; *tp = total; return 0; } #endif /* times() */ -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) int res = process_time_times(state, tp, info); if (res < 0) { return -1; @@ -1386,7 +1414,7 @@ static PyObject * time_process_time(PyObject *module, PyObject *unused) { time_module_state *state = get_time_state(module); - _PyTime_t t; + PyTime_t t; if (py_process_time(state, &t, NULL) < 0) { return NULL; } @@ -1402,11 +1430,11 @@ static PyObject * time_process_time_ns(PyObject *module, PyObject *unused) { time_module_state *state = get_time_state(module); - _PyTime_t t; + PyTime_t t; if (py_process_time(state, &t, NULL) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(process_time_ns_doc, @@ -1419,12 +1447,12 @@ sum of the kernel and user-space CPU time."); #if defined(MS_WINDOWS) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { HANDLE thread; FILETIME creation_time, exit_time, kernel_time, user_time; ULARGE_INTEGER large; - _PyTime_t ktime, utime, t; + PyTime_t ktime, utime; BOOL ok; thread = GetCurrentThread(); @@ -1451,15 +1479,14 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) utime = large.QuadPart; /* ktime and utime have a resolution of 100 nanoseconds */ - t = _PyTime_FromNanoseconds((ktime + utime) * 100); - *tp = t; + *tp = (ktime + utime) * 100; return 0; } #elif defined(_AIX) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { /* bpo-40192: On AIX, thread_cputime() is preferred: it has nanosecond resolution, whereas clock_gettime(CLOCK_THREAD_CPUTIME_ID) @@ -1476,14 +1503,14 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) info->adjustable = 0; info->resolution = 1e-9; } - *tp = _PyTime_FromNanoseconds(tc.stime + tc.utime); + *tp = (tc.stime + tc.utime); return 0; } #elif defined(__sun) && defined(__SVR4) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { /* bpo-35455: On Solaris, CLOCK_THREAD_CPUTIME_ID clock is not always available; use gethrvtime() to substitute this functionality. */ @@ -1493,18 +1520,25 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) info->monotonic = 1; info->adjustable = 0; } - *tp = _PyTime_FromNanoseconds(gethrvtime()); + *tp = gethrvtime(); return 0; } +/* CLOCK_THREAD_CPUTIME_ID is broken on NetBSD: the result of clock_gettime() + * includes the sleeping time, that defeats the purpose of the clock. + * Also, clock_getres() does not support it. + * https://github.com/python/cpython/issues/123978 + * https://gnats.netbsd.org/57512 + */ #elif defined(HAVE_CLOCK_GETTIME) && \ - defined(CLOCK_PROCESS_CPUTIME_ID) && \ - !defined(__EMSCRIPTEN__) && !defined(__wasi__) + defined(CLOCK_THREAD_CPUTIME_ID) && \ + !defined(__EMSCRIPTEN__) && !defined(__wasi__) && \ + !defined(__NetBSD__) #define HAVE_THREAD_TIME -#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) +#if defined(__APPLE__) && _Py__has_attribute(availability) static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) __attribute__((availability(macos, introduced=10.12))) __attribute__((availability(ios, introduced=10.0))) __attribute__((availability(tvos, introduced=10.0))) @@ -1512,7 +1546,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #endif static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { struct timespec ts; const clockid_t clk_id = CLOCK_THREAD_CPUTIME_ID; @@ -1554,7 +1588,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) static PyObject * time_thread_time(PyObject *self, PyObject *unused) { - _PyTime_t t; + PyTime_t t; if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } @@ -1569,11 +1603,11 @@ Thread time for profiling: sum of the kernel and user-space CPU time."); static PyObject * time_thread_time_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; + PyTime_t t; if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(thread_time_ns_doc, @@ -1595,7 +1629,7 @@ time_get_clock_info(PyObject *module, PyObject *args) char *name; _Py_clock_info_t info; PyObject *obj = NULL, *dict, *ns; - _PyTime_t t; + PyTime_t t; if (!PyArg_ParseTuple(args, "s:get_clock_info", &name)) { return NULL; @@ -1614,17 +1648,17 @@ time_get_clock_info(PyObject *module, PyObject *args) #endif if (strcmp(name, "time") == 0) { - if (_PyTime_GetSystemClockWithInfo(&t, &info) < 0) { + if (_PyTime_TimeWithInfo(&t, &info) < 0) { return NULL; } } else if (strcmp(name, "monotonic") == 0) { - if (_PyTime_GetMonotonicClockWithInfo(&t, &info) < 0) { + if (_PyTime_MonotonicWithInfo(&t, &info) < 0) { return NULL; } } else if (strcmp(name, "perf_counter") == 0) { - if (_PyTime_GetPerfCounterWithInfo(&t, &info) < 0) { + if (_PyTime_PerfCounterWithInfo(&t, &info) < 0) { return NULL; } } @@ -1916,9 +1950,9 @@ PyDoc_STRVAR(module_doc, \n\ There are two standard representations of time. One is the number\n\ of seconds since the Epoch, in UTC (a.k.a. GMT). It may be an integer\n\ -or a floating point number (to represent fractions of seconds).\n\ -The Epoch is system-defined; on Unix, it is generally January 1st, 1970.\n\ -The actual value can be retrieved by calling gmtime(0).\n\ +or a floating-point number (to represent fractions of seconds).\n\ +The epoch is the point where the time starts, the return value of time.gmtime(0).\n\ +It is January 1, 1970, 00:00:00 (UTC) on all platforms.\n\ \n\ The other representation is a tuple of 9 integers giving local time.\n\ The tuple items are:\n\ @@ -1949,20 +1983,20 @@ time_exec(PyObject *module) return -1; } - if (PyDict_DelItemString(dct, "clock_gettime") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_gettime", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_gettime_ns") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_gettime_ns", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_settime") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_settime", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_settime_ns") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_settime_ns", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_getres") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_getres", NULL) < 0) { + return -1; } } #endif @@ -1972,11 +2006,11 @@ time_exec(PyObject *module) } else { PyObject* dct = PyModule_GetDict(module); - if (PyDict_DelItemString(dct, "thread_time") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "thread_time", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "thread_time_ns") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "thread_time_ns", NULL) < 0) { + return -1; } } #endif @@ -2094,7 +2128,8 @@ time_exec(PyObject *module) } #endif -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) long ticks_per_second; if (_Py_GetTicksPerSecond(&ticks_per_second) < 0) { PyErr_SetString(PyExc_RuntimeError, @@ -2148,6 +2183,7 @@ time_module_free(void *module) static struct PyModuleDef_Slot time_slots[] = { {Py_mod_exec, time_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -2174,7 +2210,7 @@ PyInit_time(void) // On error, raise an exception and return -1. // On success, return 0. static int -pysleep(_PyTime_t timeout) +pysleep(PyTime_t timeout) { assert(timeout >= 0); @@ -2186,10 +2222,10 @@ pysleep(_PyTime_t timeout) #else struct timeval timeout_tv; #endif - _PyTime_t deadline, monotonic; + PyTime_t deadline, monotonic; int err = 0; - if (get_monotonic(&monotonic) < 0) { + if (PyTime_Monotonic(&monotonic) < 0) { return -1; } deadline = monotonic + timeout; @@ -2242,7 +2278,7 @@ pysleep(_PyTime_t timeout) } #ifndef HAVE_CLOCK_NANOSLEEP - if (get_monotonic(&monotonic) < 0) { + if (PyTime_Monotonic(&monotonic) < 0) { return -1; } timeout = deadline - monotonic; @@ -2255,7 +2291,7 @@ pysleep(_PyTime_t timeout) return 0; #else // MS_WINDOWS - _PyTime_t timeout_100ns = _PyTime_As100Nanoseconds(timeout, + PyTime_t timeout_100ns = _PyTime_As100Nanoseconds(timeout, _PyTime_ROUND_CEILING); // Maintain Windows Sleep() semantics for time.sleep(0) diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 6ae35b9372b830..60bde755d24574 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -413,7 +413,7 @@ unicodedata_UCD_decomposition_impl(PyObject *self, int chr) if (UCD_Check(self)) { const change_record *old = get_old_record(self, c); if (old->category_changed == 0) - return PyUnicode_FromString(""); /* unassigned */ + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* unassigned */ } if (code < 0 || code >= 0x110000) @@ -1668,6 +1668,7 @@ unicodedata_exec(PyObject *module) static PyModuleDef_Slot unicodedata_slots[] = { {Py_mod_exec, unicodedata_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/unicodedata_db.h b/Modules/unicodedata_db.h index 3e210863448b78..610ee6b749cb0c 100644 --- a/Modules/unicodedata_db.h +++ b/Modules/unicodedata_db.h @@ -1,6 +1,6 @@ -/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ -#define UNIDATA_VERSION "15.1.0" +#define UNIDATA_VERSION "16.0.0" /* a list of unique database records */ const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = { {0, 0, 0, 0, 0, 0}, @@ -334,24 +334,28 @@ const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = { {30, 0, 4, 0, 0, 0}, {1, 0, 4, 0, 0, 0}, {2, 0, 4, 0, 0, 0}, + {27, 0, 4, 0, 0, 0}, {9, 0, 12, 0, 0, 0}, {9, 0, 5, 0, 0, 0}, + {5, 0, 1, 0, 0, 90}, {4, 9, 1, 0, 0, 0}, + {4, 0, 14, 0, 0, 90}, + {19, 0, 1, 0, 0, 90}, {4, 0, 14, 0, 2, 0}, {5, 6, 1, 0, 2, 0}, + {30, 0, 1, 0, 0, 136}, + {7, 0, 9, 0, 0, 136}, {30, 0, 1, 0, 0, 170}, {5, 216, 1, 0, 0, 0}, {5, 226, 1, 0, 0, 0}, - {27, 0, 1, 0, 0, 136}, - {7, 0, 9, 0, 0, 136}, - {30, 0, 1, 0, 0, 136}, + {9, 0, 1, 0, 2, 0}, {30, 0, 1, 0, 4, 0}, {29, 0, 19, 0, 2, 0}, }; /* Reindexing of NFC first characters. */ -#define TOTAL_FIRST 377 -#define TOTAL_LAST 63 +#define TOTAL_FIRST 391 +#define TOTAL_LAST 72 struct reindex{int start;short count,index;}; static struct reindex nfc_first[] = { { 60, 2, 0}, @@ -557,14 +561,27 @@ static struct reindex nfc_first[] = { { 12507, 0, 361}, { 12527, 3, 362}, { 12541, 0, 366}, - { 69785, 0, 367}, - { 69787, 0, 368}, - { 69797, 0, 369}, - { 69937, 1, 370}, - { 70471, 0, 372}, - { 70841, 0, 373}, - { 71096, 1, 374}, - { 71989, 0, 376}, + { 67026, 0, 367}, + { 67034, 0, 368}, + { 69785, 0, 369}, + { 69787, 0, 370}, + { 69797, 0, 371}, + { 69937, 1, 372}, + { 70471, 0, 374}, + { 70530, 0, 375}, + { 70532, 0, 376}, + { 70539, 0, 377}, + { 70544, 0, 378}, + { 70594, 0, 379}, + { 70841, 0, 380}, + { 71096, 1, 381}, + { 71989, 0, 383}, + { 90398, 0, 384}, + { 90401, 1, 385}, + { 90409, 0, 387}, + { 93539, 0, 388}, + { 93543, 0, 389}, + { 93545, 0, 390}, {0,0,0} }; @@ -604,11 +621,18 @@ static struct reindex nfc_last[] = { { 69927, 0, 55}, { 70462, 0, 56}, { 70487, 0, 57}, - { 70832, 0, 58}, - { 70842, 0, 59}, - { 70845, 0, 60}, - { 71087, 0, 61}, - { 71984, 0, 62}, + { 70584, 0, 58}, + { 70587, 0, 59}, + { 70594, 0, 60}, + { 70601, 0, 61}, + { 70832, 0, 62}, + { 70842, 0, 63}, + { 70845, 0, 64}, + { 71087, 0, 65}, + { 71984, 0, 66}, + { 90398, 2, 67}, + { 90409, 0, 70}, + { 93543, 0, 71}, {0,0,0} }; @@ -742,42 +766,41 @@ static const unsigned short index1[] = { 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 121, 121, 122, 123, 124, 125, 126, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 41, 41, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 137, 158, 159, 160, 161, 162, - 163, 164, 165, 166, 167, 168, 137, 169, 170, 137, 171, 172, 173, 174, - 137, 175, 176, 177, 178, 179, 180, 181, 137, 182, 183, 184, 185, 137, - 186, 187, 188, 41, 41, 41, 41, 41, 41, 41, 189, 190, 41, 191, 137, 137, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, + 164, 165, 166, 167, 168, 169, 170, 171, 172, 137, 173, 174, 175, 176, + 137, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 137, + 189, 190, 191, 41, 41, 41, 41, 41, 41, 41, 192, 193, 41, 194, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 192, 41, 41, 41, 41, 41, 41, 41, 41, 193, 137, 137, + 137, 137, 137, 137, 195, 41, 41, 41, 41, 41, 41, 41, 41, 196, 41, 41, 41, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 134, 41, 41, 41, 41, 197, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 41, 41, 41, 41, 194, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 41, 41, 41, 41, 195, 196, 197, 198, 137, 137, 137, 137, 199, - 200, 201, 202, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 198, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 41, 41, 41, 41, 199, 200, + 201, 202, 137, 137, 203, 137, 204, 205, 206, 207, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 203, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 204, 205, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 208, 101, 101, 101, 101, 101, 101, 101, 101, 101, 209, 210, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 206, 101, 101, 207, 101, 101, 208, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 209, 210, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 211, 101, 101, + 212, 101, 101, 213, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 214, 215, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 211, 212, 78, 213, - 214, 215, 216, 217, 218, 137, 219, 220, 221, 222, 223, 224, 225, 226, 78, - 78, 78, 78, 227, 228, 137, 137, 137, 137, 137, 137, 137, 137, 229, 137, - 230, 231, 232, 137, 137, 233, 137, 137, 137, 234, 137, 137, 137, 137, - 137, 235, 236, 237, 238, 137, 137, 137, 137, 137, 239, 240, 241, 137, - 242, 243, 137, 137, 244, 245, 246, 247, 248, 137, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 137, 137, 137, 137, 137, 137, 137, 137, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 216, 217, + 216, 216, 216, 218, 219, 220, 78, 221, 222, 223, 224, 225, 226, 137, 227, + 228, 229, 230, 231, 232, 233, 234, 78, 78, 78, 78, 235, 236, 137, 137, + 137, 137, 137, 137, 137, 137, 237, 137, 238, 239, 240, 137, 137, 241, + 137, 137, 137, 242, 137, 243, 137, 137, 137, 244, 245, 246, 247, 137, + 137, 137, 137, 137, 248, 249, 250, 137, 251, 252, 137, 137, 253, 254, + 255, 256, 257, 137, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + 268, 269, 270, 271, 272, 273, 216, 274, 137, 137, 137, 137, 137, 137, + 137, 137, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, @@ -800,57 +823,57 @@ static const unsigned short index1[] = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 267, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 275, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 268, 101, 269, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 276, 101, 277, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 270, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 278, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 271, 101, 101, 101, 101, 272, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 121, 121, 121, 121, 274, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 275, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 279, 101, 101, + 101, 101, 280, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 121, 121, 121, 121, 282, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 283, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 276, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 284, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 277, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 275, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 285, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 283, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, @@ -1215,8 +1238,8 @@ static const unsigned short index1[] = { 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 278, 137, 279, 280, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 286, 137, 287, 288, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, @@ -1252,6 +1275,7 @@ static const unsigned short index1[] = { 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, @@ -1288,7 +1312,7 @@ static const unsigned short index1[] = { 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 281, 120, 120, 120, 120, 120, 120, + 120, 289, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, @@ -1324,8 +1348,7 @@ static const unsigned short index1[] = { 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 281, + 120, 120, 120, 120, 120, 120, 120, 120, 120, 289, }; static const unsigned short index2[] = { @@ -1457,7 +1480,7 @@ static const unsigned short index2[] = { 118, 118, 118, 118, 118, 118, 118, 118, 0, 0, 0, 0, 0, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 140, 118, 118, 118, 118, 118, 118, 0, - 108, 108, 0, 0, 0, 0, 0, 0, 81, 86, 86, 86, 81, 81, 81, 81, 118, 118, + 108, 108, 0, 0, 0, 0, 0, 81, 81, 86, 86, 86, 81, 81, 81, 81, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 120, 81, 81, 81, @@ -1713,46 +1736,46 @@ static const unsigned short index2[] = { 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 148, 135, 135, 135, 135, 135, 149, 135, 149, 141, 141, 149, 149, 135, - 149, 176, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 146, 146, 146, 146, + 149, 176, 48, 48, 48, 48, 48, 48, 48, 48, 0, 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 83, 83, 83, 83, 83, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 81, 86, 81, 81, 81, 81, 81, 81, 81, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 83, 83, 0, 135, 135, 141, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 141, 135, 135, 135, 135, 141, 141, 135, 135, 176, - 144, 135, 135, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 80, 80, 80, 80, 80, 80, 83, 83, 83, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 141, 135, 135, 135, 135, 141, 141, 135, 135, + 176, 144, 135, 135, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, 135, 135, 141, 141, 141, 135, - 141, 135, 135, 135, 176, 176, 0, 0, 0, 0, 0, 0, 0, 0, 83, 83, 83, 83, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, 135, 135, 141, 141, 141, + 135, 141, 135, 135, 135, 176, 176, 0, 0, 0, 0, 0, 0, 0, 0, 83, 83, 83, + 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, - 141, 141, 141, 141, 141, 141, 141, 135, 135, 135, 135, 135, 135, 135, - 135, 141, 141, 135, 147, 0, 0, 0, 83, 83, 83, 83, 83, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 0, 0, 0, 48, 48, 48, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 141, 141, 141, 141, 141, 141, 141, 141, 135, 135, 135, 135, 135, 135, + 135, 135, 141, 141, 135, 147, 0, 0, 0, 83, 83, 83, 83, 83, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 48, 48, 48, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 53, 53, 53, 53, 53, 53, 83, 83, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 48, 48, 48, 53, 53, 53, 53, 53, 53, 83, 83, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 44, 47, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 44, 44, 44, 83, 83, - 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 81, 83, 178, 86, - 86, 86, 86, 86, 81, 81, 86, 86, 86, 86, 81, 141, 178, 178, 178, 178, 178, - 178, 178, 48, 48, 48, 48, 86, 48, 48, 48, 48, 48, 48, 81, 48, 48, 141, - 81, 81, 48, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 44, 44, + 44, 83, 83, 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 81, + 83, 178, 86, 86, 86, 86, 86, 81, 81, 86, 86, 86, 86, 81, 141, 178, 178, + 178, 178, 178, 178, 178, 48, 48, 48, 48, 86, 48, 48, 48, 48, 48, 48, 81, + 48, 48, 141, 81, 81, 48, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 51, 51, 51, - 53, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 51, 51, 51, 53, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 81, 81, 86, 81, 81, 81, 81, 81, 81, 81, 86, 81, 81, 179, 180, 86, - 181, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 51, 51, 51, 51, 81, 81, 86, 81, 81, 81, 81, 81, 81, 81, 86, 81, 81, 179, + 180, 86, 181, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 182, 88, 88, 86, 183, 81, 184, 86, 81, 86, 38, 43, 38, 43, 38, + 81, 81, 81, 81, 182, 88, 88, 86, 183, 81, 184, 86, 81, 86, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, @@ -1761,245 +1784,245 @@ static const unsigned short index2[] = { 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 43, 43, 43, 43, 35, 185, 47, 47, 44, 47, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 43, 43, 43, 43, 35, 185, 47, 47, 44, 47, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 44, 47, 44, 47, 44, 47, 43, - 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, - 43, 43, 43, 0, 0, 38, 38, 38, 38, 38, 38, 0, 0, 43, 43, 43, 43, 43, 43, - 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, 43, 43, - 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, 0, 0, 38, 38, 38, - 38, 38, 38, 0, 0, 43, 43, 43, 43, 43, 43, 43, 43, 0, 38, 0, 38, 0, 38, 0, - 38, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, - 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 0, 0, 43, 43, - 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, 43, 43, - 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, 43, 43, - 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, 43, 43, - 43, 43, 43, 0, 43, 43, 38, 38, 38, 188, 187, 58, 186, 58, 58, 76, 43, 43, - 43, 0, 43, 43, 38, 188, 38, 188, 187, 76, 76, 76, 43, 43, 43, 186, 0, 0, - 43, 43, 38, 38, 38, 188, 0, 76, 76, 76, 43, 43, 43, 186, 43, 43, 43, 43, - 38, 38, 38, 188, 38, 76, 189, 189, 0, 0, 43, 43, 43, 0, 43, 43, 38, 188, - 38, 188, 187, 189, 58, 0, 190, 190, 191, 191, 191, 191, 191, 191, 191, - 191, 191, 177, 177, 177, 192, 193, 194, 195, 84, 194, 194, 194, 22, 196, - 197, 198, 199, 200, 197, 198, 199, 200, 22, 22, 22, 138, 201, 201, 201, - 22, 202, 203, 204, 205, 206, 207, 208, 21, 209, 110, 209, 210, 211, 22, - 196, 196, 138, 28, 36, 22, 196, 138, 201, 212, 212, 138, 138, 138, 213, - 165, 166, 196, 196, 196, 138, 138, 138, 138, 138, 138, 138, 138, 78, 138, - 212, 138, 138, 196, 138, 138, 138, 138, 138, 138, 138, 191, 177, 177, - 177, 177, 177, 0, 214, 215, 216, 217, 177, 177, 177, 177, 177, 177, 218, - 51, 0, 0, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, 222, 223, 218, - 34, 34, 34, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, 222, 0, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, 0, 85, 85, 85, 85, - 85, 85, 85, 85, 224, 225, 85, 85, 23, 85, 85, 85, 85, 85, 85, 85, 85, 85, - 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 81, 81, 178, 178, 81, 81, 81, 81, 178, 178, 178, 81, 81, - 82, 82, 82, 82, 81, 82, 82, 82, 178, 178, 81, 86, 81, 178, 178, 86, 86, - 86, 86, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 226, 49, - 227, 26, 227, 226, 49, 26, 227, 35, 49, 49, 49, 35, 35, 49, 49, 49, 46, - 26, 49, 227, 26, 78, 49, 49, 49, 49, 49, 26, 26, 226, 227, 227, 26, 49, - 26, 228, 26, 49, 26, 188, 228, 49, 49, 229, 35, 49, 49, 44, 49, 35, 158, - 158, 158, 158, 35, 26, 226, 35, 35, 49, 49, 230, 78, 78, 78, 78, 49, 35, - 35, 35, 35, 26, 78, 26, 26, 47, 80, 231, 231, 231, 37, 37, 231, 231, 231, - 231, 231, 231, 37, 37, 37, 37, 231, 232, 232, 232, 232, 232, 232, 232, - 232, 232, 232, 232, 232, 233, 233, 233, 233, 232, 232, 232, 232, 232, - 232, 232, 232, 232, 232, 233, 233, 233, 233, 233, 233, 175, 175, 175, 44, - 47, 175, 175, 175, 175, 37, 26, 26, 0, 0, 0, 0, 40, 40, 40, 40, 40, 30, - 30, 30, 30, 30, 234, 234, 26, 26, 26, 26, 78, 26, 26, 78, 26, 26, 78, 26, - 26, 26, 26, 26, 26, 26, 234, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 235, 234, 234, 26, 26, 40, 26, 40, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 40, - 236, 237, 237, 238, 78, 78, 40, 237, 238, 236, 237, 238, 236, 78, 40, 78, - 237, 239, 240, 78, 237, 236, 78, 78, 78, 237, 236, 236, 237, 40, 237, - 237, 236, 236, 40, 238, 40, 238, 40, 40, 40, 40, 237, 241, 230, 237, 230, - 230, 236, 236, 236, 40, 40, 40, 40, 78, 236, 78, 236, 237, 237, 236, 236, - 236, 238, 236, 236, 238, 236, 236, 238, 237, 238, 236, 236, 237, 78, 78, - 78, 78, 78, 237, 236, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, - 242, 40, 238, 78, 237, 237, 237, 237, 236, 236, 237, 237, 78, 234, 242, - 242, 238, 238, 236, 236, 238, 238, 236, 236, 238, 238, 236, 236, 236, - 236, 236, 236, 238, 238, 237, 237, 238, 238, 237, 237, 238, 238, 236, - 236, 236, 78, 78, 236, 236, 236, 236, 78, 78, 40, 78, 78, 236, 40, 78, - 78, 78, 78, 78, 78, 78, 78, 236, 236, 78, 40, 236, 236, 236, 236, 236, - 236, 238, 238, 238, 238, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, - 78, 78, 78, 78, 236, 237, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, - 236, 236, 236, 78, 78, 236, 236, 78, 78, 78, 78, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 238, 238, 238, 238, 236, 236, 236, 236, 236, - 236, 238, 238, 238, 238, 78, 78, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 26, 26, 26, 26, 26, 26, 26, 26, - 165, 166, 165, 166, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, - 26, 243, 243, 26, 26, 26, 26, 236, 236, 26, 26, 26, 26, 26, 26, 26, 244, - 245, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 80, 80, 80, 80, 80, 80, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 44, 47, 44, 47, 44, + 47, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, + 43, 43, 43, 43, 43, 0, 0, 38, 38, 38, 38, 38, 38, 0, 0, 43, 43, 43, 43, + 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, + 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, 0, 0, 38, + 38, 38, 38, 38, 38, 0, 0, 43, 43, 43, 43, 43, 43, 43, 43, 0, 38, 0, 38, + 0, 38, 0, 38, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, + 38, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 0, 0, + 43, 43, 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, + 43, 43, 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, + 43, 43, 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, + 43, 43, 43, 43, 43, 0, 43, 43, 38, 38, 38, 188, 187, 58, 186, 58, 58, 76, + 43, 43, 43, 0, 43, 43, 38, 188, 38, 188, 187, 76, 76, 76, 43, 43, 43, + 186, 0, 0, 43, 43, 38, 38, 38, 188, 0, 76, 76, 76, 43, 43, 43, 186, 43, + 43, 43, 43, 38, 38, 38, 188, 38, 76, 189, 189, 0, 0, 43, 43, 43, 0, 43, + 43, 38, 188, 38, 188, 187, 189, 58, 0, 190, 190, 191, 191, 191, 191, 191, + 191, 191, 191, 191, 177, 177, 177, 192, 193, 194, 195, 84, 194, 194, 194, + 22, 196, 197, 198, 199, 200, 197, 198, 199, 200, 22, 22, 22, 138, 201, + 201, 201, 22, 202, 203, 204, 205, 206, 207, 208, 21, 209, 110, 209, 210, + 211, 22, 196, 196, 138, 28, 36, 22, 196, 138, 201, 212, 212, 138, 138, + 138, 213, 165, 166, 196, 196, 196, 138, 138, 138, 138, 138, 138, 138, + 138, 78, 138, 212, 138, 138, 196, 138, 138, 138, 138, 138, 138, 138, 191, + 177, 177, 177, 177, 177, 0, 214, 215, 216, 217, 177, 177, 177, 177, 177, + 177, 218, 51, 0, 0, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, 222, + 223, 218, 34, 34, 34, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, + 222, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, 0, 85, + 85, 85, 85, 85, 85, 85, 85, 224, 225, 85, 85, 23, 85, 85, 85, 85, 85, 85, + 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 178, 178, 81, 81, 81, 81, 178, 178, + 178, 81, 81, 82, 82, 82, 82, 81, 82, 82, 82, 178, 178, 81, 86, 81, 178, + 178, 86, 86, 86, 86, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 226, 226, 49, 227, 26, 227, 226, 49, 26, 227, 35, 49, 49, 49, 35, 35, 49, + 49, 49, 46, 26, 49, 227, 26, 78, 49, 49, 49, 49, 49, 26, 26, 226, 227, + 227, 26, 49, 26, 228, 26, 49, 26, 188, 228, 49, 49, 229, 35, 49, 49, 44, + 49, 35, 158, 158, 158, 158, 35, 26, 226, 35, 35, 49, 49, 230, 78, 78, 78, + 78, 49, 35, 35, 35, 35, 26, 78, 26, 26, 47, 80, 231, 231, 231, 37, 37, + 231, 231, 231, 231, 231, 231, 37, 37, 37, 37, 231, 232, 232, 232, 232, + 232, 232, 232, 232, 232, 232, 232, 232, 233, 233, 233, 233, 232, 232, + 232, 232, 232, 232, 232, 232, 232, 232, 233, 233, 233, 233, 233, 233, + 175, 175, 175, 44, 47, 175, 175, 175, 175, 37, 26, 26, 0, 0, 0, 0, 40, + 40, 40, 40, 40, 30, 30, 30, 30, 30, 234, 234, 26, 26, 26, 26, 78, 26, 26, + 78, 26, 26, 78, 26, 26, 26, 26, 26, 26, 26, 234, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 235, 234, 234, 26, 26, 40, 26, 40, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 40, 236, 237, 237, 238, 78, 78, 40, 237, 238, 236, 237, + 238, 236, 78, 40, 78, 237, 239, 240, 78, 237, 236, 78, 78, 78, 237, 236, + 236, 237, 40, 237, 237, 236, 236, 40, 238, 40, 238, 40, 40, 40, 40, 237, + 241, 230, 237, 230, 230, 236, 236, 236, 40, 40, 40, 40, 78, 236, 78, 236, + 237, 237, 236, 236, 236, 238, 236, 236, 238, 236, 236, 238, 237, 238, + 236, 236, 237, 78, 78, 78, 78, 78, 237, 236, 236, 236, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 236, 242, 40, 238, 78, 237, 237, 237, 237, 236, 236, + 237, 237, 78, 238, 242, 242, 238, 238, 236, 236, 238, 238, 236, 236, 238, + 238, 236, 236, 236, 236, 236, 236, 238, 238, 237, 237, 238, 238, 237, + 237, 238, 238, 236, 236, 236, 78, 78, 236, 236, 236, 236, 78, 78, 40, 78, + 78, 236, 40, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 78, 40, 236, 236, + 236, 236, 236, 236, 238, 238, 238, 238, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 78, 78, 78, 78, 78, 236, 237, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 236, 236, 236, 236, 236, 78, 78, 236, 236, 78, 78, 78, 78, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 238, 238, 238, 238, 236, + 236, 236, 236, 236, 236, 238, 238, 238, 238, 78, 78, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 26, 26, 26, + 26, 26, 26, 26, 26, 165, 166, 165, 166, 26, 26, 26, 26, 26, 26, 30, 26, + 26, 26, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 236, 236, 26, 26, 26, + 26, 26, 26, 26, 244, 245, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 26, 78, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 80, - 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 26, 78, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 80, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 78, 78, - 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 26, 26, - 26, 243, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 78, 78, 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, 26, 243, 243, + 243, 243, 26, 26, 26, 243, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 37, 37, 37, 37, 37, 37, 37, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, - 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 37, 37, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 231, 247, 247, 247, 247, 247, 247, 247, - 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 30, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 231, 247, + 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, + 247, 247, 247, 247, 247, 247, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 26, 26, 26, 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, - 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, - 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, - 26, 30, 40, 26, 26, 26, 26, 30, 30, 26, 26, 30, 40, 26, 26, 26, 26, 30, - 30, 30, 26, 26, 30, 26, 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, - 248, 248, 78, 26, 26, 26, 26, 26, 30, 30, 26, 26, 30, 26, 26, 26, 26, 30, - 30, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 30, 26, 30, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 30, 26, 26, - 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, - 26, 30, 30, 30, 30, 26, 30, 30, 26, 40, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 30, 30, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 80, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, - 30, 26, 26, 26, 26, 243, 243, 30, 30, 30, 30, 30, 30, 30, 30, 243, 30, - 30, 30, 30, 30, 243, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 26, 30, 26, 26, 26, 26, 30, 30, 243, 30, 30, 30, 30, 30, 30, 30, 243, - 243, 30, 243, 30, 30, 30, 30, 243, 30, 30, 243, 30, 30, 26, 26, 26, 26, - 26, 243, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 243, 26, 243, 26, 26, 26, 26, 243, 243, 243, 26, 243, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 165, 166, 165, 166, 165, - 166, 165, 166, 165, 166, 165, 166, 165, 166, 247, 247, 247, 247, 247, - 247, 247, 247, 247, 247, 155, 155, 155, 155, 155, 155, 155, 155, 155, - 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 26, 243, 243, 243, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 26, 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, + 26, 26, 26, 26, 26, 30, 30, 26, 26, 30, 40, 26, 26, 26, 26, 30, 30, 26, + 26, 30, 40, 26, 26, 26, 26, 30, 30, 30, 26, 26, 30, 26, 26, 30, 30, 30, + 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, + 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, + 26, 26, 26, 78, 78, 78, 78, 78, 248, 248, 78, 26, 26, 26, 26, 26, 30, 30, + 26, 26, 30, 26, 26, 26, 26, 30, 30, 26, 26, 26, 26, 243, 243, 26, 26, 26, + 26, 26, 26, 30, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, + 26, 26, 26, 26, 26, 30, 26, 30, 26, 26, 26, 26, 26, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, 26, 30, 30, 30, 30, 26, 30, 30, + 26, 40, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 26, + 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, 243, 26, + 26, 26, 26, 26, 26, 26, 26, 243, 243, 80, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 30, 26, 26, 26, 26, 243, 243, + 30, 30, 30, 30, 30, 30, 30, 30, 243, 30, 30, 30, 30, 30, 243, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 30, 26, 26, 26, 26, 30, 30, + 243, 30, 30, 30, 30, 30, 30, 30, 243, 243, 30, 243, 30, 30, 30, 30, 243, + 30, 30, 243, 30, 30, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 243, 236, 78, 78, 236, 236, 165, 166, 78, 236, 236, 78, 236, - 236, 236, 78, 78, 78, 78, 78, 236, 236, 236, 236, 78, 78, 78, 78, 78, - 236, 236, 236, 78, 78, 78, 236, 236, 236, 236, 9, 10, 9, 10, 9, 10, 9, - 10, 165, 166, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 243, 26, 26, 26, 26, + 243, 243, 243, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, + 166, 165, 166, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 26, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 236, 78, 78, + 236, 236, 165, 166, 78, 236, 236, 78, 236, 236, 236, 78, 78, 78, 78, 78, + 236, 236, 236, 236, 78, 78, 78, 78, 78, 236, 236, 236, 78, 78, 78, 236, + 236, 236, 236, 9, 10, 9, 10, 9, 10, 9, 10, 165, 166, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 165, 166, 9, 10, 165, 166, 165, 166, 165, - 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 78, 78, - 236, 236, 236, 236, 236, 236, 78, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 236, 78, - 78, 78, 78, 78, 78, 78, 236, 236, 236, 236, 236, 236, 78, 78, 78, 236, - 78, 78, 78, 78, 236, 236, 236, 236, 236, 78, 236, 236, 78, 78, 165, 166, - 165, 166, 236, 78, 78, 78, 78, 236, 78, 236, 236, 236, 78, 78, 236, 236, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 236, 236, 236, 236, 78, - 78, 165, 166, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, - 230, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 78, 236, 236, 236, 236, 78, 78, 236, 78, 236, 78, 78, 236, - 78, 236, 236, 236, 236, 78, 78, 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, - 78, 236, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 236, 236, 78, 78, 78, 78, 236, 236, 236, 236, 78, - 236, 236, 78, 78, 236, 230, 220, 220, 78, 78, 236, 236, 236, 236, 236, + 165, 166, 9, 10, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, + 166, 165, 166, 165, 166, 165, 166, 78, 78, 236, 236, 236, 236, 236, 236, + 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 78, 78, 78, 78, 78, 78, 78, 78, 236, 78, 78, 78, 78, 78, 78, 78, 236, + 236, 236, 236, 236, 236, 78, 78, 78, 236, 78, 78, 78, 78, 236, 236, 236, + 236, 236, 78, 236, 236, 78, 78, 165, 166, 165, 166, 236, 78, 78, 78, 78, + 236, 78, 236, 236, 236, 78, 78, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 236, 236, 236, 236, 236, 236, 78, 78, 165, 166, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 230, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, 236, 236, 236, + 236, 78, 78, 236, 78, 236, 78, 78, 236, 78, 236, 236, 236, 236, 78, 78, + 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, 78, 236, 236, 236, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, + 236, 78, 78, 78, 78, 236, 236, 236, 236, 78, 236, 236, 78, 78, 236, 230, + 220, 220, 78, 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, 78, 236, 236, 236, - 236, 236, 236, 236, 236, 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 78, 78, 236, 236, 236, 236, 236, 236, 236, 236, 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 78, 78, 78, 78, 78, 249, 78, 236, 78, 78, 78, 236, 236, - 236, 236, 236, 78, 78, 78, 78, 78, 236, 236, 236, 78, 78, 78, 78, 236, - 78, 78, 78, 236, 236, 236, 236, 236, 78, 236, 78, 78, 26, 26, 26, 26, 26, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, 78, 78, + 78, 78, 249, 78, 236, 78, 78, 78, 236, 236, 236, 236, 236, 78, 78, 78, + 78, 78, 236, 236, 236, 78, 78, 78, 78, 236, 78, 78, 78, 236, 236, 236, + 236, 236, 78, 236, 78, 78, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 26, 26, 78, 78, 78, 78, 78, 78, - 26, 26, 26, 243, 26, 26, 26, 26, 243, 30, 30, 30, 30, 26, 26, 26, 26, 26, + 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 26, 26, 78, 78, 78, 78, 78, 78, 26, 26, 26, 243, 26, 26, + 26, 26, 243, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 250, 26, 44, 44, + 26, 26, 26, 26, 26, 26, 26, 26, 250, 26, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 44, 47, 44, 44, 44, 47, 47, 44, 47, 44, 47, 44, 47, 44, - 44, 44, 44, 47, 44, 47, 47, 44, 47, 47, 47, 47, 47, 47, 51, 51, 44, 44, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 44, 47, + 44, 44, 44, 47, 47, 44, 47, 44, 47, 44, 47, 44, 44, 44, 44, 47, 44, 47, + 47, 44, 47, 47, 47, 47, 47, 47, 51, 51, 44, 44, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 47, 26, 26, 26, 26, 26, 26, 44, - 47, 44, 47, 81, 81, 81, 44, 47, 0, 0, 0, 0, 0, 138, 138, 138, 138, 155, - 138, 138, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 47, 44, 47, 47, 26, 26, 26, 26, 26, 26, 44, 47, 44, 47, 81, 81, 81, + 44, 47, 0, 0, 0, 0, 0, 138, 138, 138, 138, 155, 138, 138, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 0, 47, 0, 0, 0, 0, 0, 47, 0, 0, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 0, + 0, 0, 0, 0, 47, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, - 0, 51, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, - 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, - 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, - 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 81, 81, 81, 81, + 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 51, 83, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, + 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, + 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, + 0, 48, 48, 48, 48, 48, 48, 48, 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 138, 138, 28, 36, 28, 36, 138, - 138, 138, 28, 36, 138, 28, 36, 138, 138, 138, 138, 138, 138, 138, 138, - 138, 84, 138, 138, 84, 138, 28, 36, 138, 138, 28, 36, 165, 166, 165, 166, - 165, 166, 165, 166, 138, 138, 138, 138, 138, 52, 138, 138, 138, 138, 138, - 138, 138, 138, 138, 138, 84, 84, 138, 138, 138, 138, 84, 138, 199, 138, - 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 26, 26, 138, - 138, 138, 165, 166, 165, 166, 165, 166, 165, 166, 84, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, - 243, 243, 243, 243, 251, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 81, 81, 81, 81, 138, 138, 28, 36, 28, 36, 138, 138, 138, 28, 36, 138, 28, + 36, 138, 138, 138, 138, 138, 138, 138, 138, 138, 84, 138, 138, 84, 138, + 28, 36, 138, 138, 28, 36, 165, 166, 165, 166, 165, 166, 165, 166, 138, + 138, 138, 138, 138, 52, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 84, 84, 138, 138, 138, 138, 84, 138, 199, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 26, 26, 138, 138, 138, 165, 166, 165, + 166, 165, 166, 165, 166, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 243, 243, 243, 243, 251, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 251, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, @@ -2014,59 +2037,60 @@ static const unsigned short index2[] = { 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 252, 253, 253, 253, 243, 254, 172, 255, 256, - 257, 256, 257, 256, 257, 256, 257, 256, 257, 243, 243, 256, 257, 256, - 257, 256, 257, 256, 257, 258, 259, 260, 260, 243, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 261, 262, 263, 264, 265, 265, 258, 254, 254, - 254, 254, 254, 251, 243, 266, 266, 266, 254, 172, 253, 243, 26, 0, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, 267, 172, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 252, 253, + 253, 253, 243, 254, 172, 255, 256, 257, 256, 257, 256, 257, 256, 257, + 256, 257, 243, 243, 256, 257, 256, 257, 256, 257, 256, 257, 258, 259, + 260, 260, 243, 255, 255, 255, 255, 255, 255, 255, 255, 255, 261, 262, + 263, 264, 265, 265, 258, 254, 254, 254, 254, 254, 251, 243, 266, 266, + 266, 254, 172, 253, 243, 26, 0, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, + 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 172, + 267, 172, 267, 172, 267, 172, 172, 172, 172, 172, 172, 267, 267, 172, + 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 267, 172, 172, 0, 0, 268, 268, 269, 269, 254, + 270, 271, 258, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, - 267, 172, 267, 172, 267, 172, 172, 267, 172, 267, 172, 267, 172, 172, - 172, 172, 172, 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, - 267, 172, 267, 267, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, - 172, 0, 0, 268, 268, 269, 269, 254, 270, 271, 258, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 267, 172, 267, 172, 267, 172, 267, - 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, - 172, 267, 172, 172, 267, 172, 267, 172, 267, 172, 172, 172, 172, 172, - 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, - 267, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, 172, 267, 267, - 267, 267, 253, 254, 254, 270, 271, 0, 0, 0, 0, 0, 172, 172, 172, 172, + 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 172, 267, 172, 267, + 172, 267, 172, 172, 172, 172, 172, 172, 267, 267, 172, 267, 267, 172, + 267, 267, 172, 267, 267, 172, 267, 267, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 267, 172, 172, 267, 267, 267, 267, 253, 254, 254, 270, 271, 0, + 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 0, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 0, 272, 272, 273, 273, 273, 273, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 172, 172, 172, 172, 172, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 0, 272, + 272, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 243, + 172, 172, 172, 172, 172, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 251, 251, 0, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, 0, 273, 273, + 273, 273, 273, 273, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 275, 275, 275, 275, 275, 275, 275, 275, 251, 276, 276, - 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 275, 275, 275, 275, + 275, 275, 275, 275, 251, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 276, 276, 276, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, - 251, 251, 272, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 274, + 274, 274, 274, 274, 274, 274, 251, 251, 251, 272, 273, 273, 273, 273, + 273, 273, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 276, 276, 276, 276, - 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, 251, 251, 274, + 274, 274, 274, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 276, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 251, 251, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, @@ -2078,18 +2102,17 @@ static const unsigned short index2[] = { 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, - 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 251, 251, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 251, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 251, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -2102,12 +2125,14 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 254, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 254, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -2115,100 +2140,100 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, + 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 53, 53, 53, 53, 53, 53, 83, 83, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 138, 138, 138, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 48, 81, - 82, 82, 82, 138, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 138, 52, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 51, 51, 81, 81, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 53, 53, 53, 53, 53, 53, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 53, 138, 138, 138, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 48, 81, 82, 82, 82, 138, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 138, 52, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 51, 51, 81, 81, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 175, 175, 175, 175, 175, 175, - 175, 175, 175, 175, 81, 81, 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 52, 52, 52, 52, 52, 52, 52, 52, 52, 54, 54, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 47, 47, 44, 47, 44, + 48, 48, 48, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 81, 81, 83, + 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, + 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 54, 54, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 47, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 51, 47, 47, + 47, 47, 47, 47, 47, 47, 44, 47, 44, 47, 44, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 52, 277, 277, 44, 47, 44, 47, 48, 44, 47, 44, 47, 47, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 51, 47, 47, 47, 47, 47, 47, 47, 47, 44, 47, 44, 47, - 44, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 52, 277, 277, 44, 47, 44, 47, - 48, 44, 47, 44, 47, 47, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 44, 44, 44, 44, 47, 44, 44, 44, - 44, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 44, 44, 44, 47, 44, 47, 0, 0, 0, 0, 0, 44, 47, 0, 47, 0, 47, 44, 47, 44, - 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 51, 51, 51, 44, 47, 48, 51, 51, 47, 48, 48, 48, 48, 48, 48, 48, 135, - 48, 48, 48, 144, 48, 48, 48, 48, 135, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, - 135, 141, 26, 26, 26, 26, 144, 0, 0, 0, 150, 150, 150, 150, 150, 150, 80, - 80, 85, 229, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 138, 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 141, - 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 47, 44, 44, 44, 44, 44, 47, 44, 44, 44, 44, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 44, 44, 44, 47, 44, 47, 44, 44, + 47, 0, 0, 44, 47, 0, 47, 0, 47, 44, 47, 44, 47, 44, 47, 44, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 51, 51, 44, 47, + 48, 51, 51, 47, 48, 48, 48, 48, 48, 48, 48, 135, 48, 48, 48, 144, 48, 48, + 48, 48, 135, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, 135, 141, 26, 26, 26, 26, + 144, 0, 0, 0, 150, 150, 150, 150, 150, 150, 80, 80, 85, 229, 0, 0, 0, 0, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, - 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - 144, 135, 0, 0, 0, 0, 0, 0, 0, 0, 83, 83, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 81, 81, 48, 48, 48, 48, 48, 48, 83, 83, 83, - 48, 83, 48, 48, 135, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 86, 86, - 86, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 141, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 172, 172, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 138, + 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 141, 141, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 144, 135, 0, 0, 0, 0, 0, 0, 0, 0, + 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, + 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 48, 48, 48, 48, 48, 48, 83, 83, 83, 48, 83, 48, 48, 135, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 135, 135, 135, 135, 135, 86, 86, 86, 83, 83, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 141, 176, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 83, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, - 135, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, - 141, 135, 135, 135, 135, 141, 141, 135, 135, 141, 141, 176, 83, 83, 83, - 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 0, 53, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 0, 0, 0, 0, 83, 83, 48, 48, 48, 48, 48, 135, 53, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 146, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, - 135, 135, 135, 141, 141, 135, 135, 141, 141, 135, 135, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 48, 48, 48, 135, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, 0, 0, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 83, 83, 83, 83, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 48, - 48, 48, 48, 48, 48, 80, 80, 80, 48, 141, 135, 141, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 48, 81, 81, 86, 48, 48, 81, 81, - 48, 48, 48, 48, 48, 81, 81, 48, 81, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 53, 83, 83, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 141, 135, 135, 141, 141, 83, 83, 48, 53, - 53, 141, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, - 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 172, 172, 172, 172, 172, 172, 0, 0, 0, 135, 135, 135, 141, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, 141, 135, 135, 135, 135, 141, + 141, 135, 135, 141, 141, 176, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, + 83, 83, 0, 53, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, + 0, 83, 83, 48, 48, 48, 48, 48, 135, 53, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, 141, 141, 135, 135, + 141, 141, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 135, 48, 48, + 48, 48, 48, 48, 48, 48, 135, 141, 0, 0, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 0, 83, 83, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 53, 48, 48, 48, 48, 48, 48, 80, 80, 80, + 48, 141, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 81, 48, 81, 81, 86, 48, 48, 81, 81, 48, 48, 48, 48, 48, 81, 81, 48, + 81, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 48, 48, 53, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 141, 135, 135, 141, 141, 83, 83, 48, 53, 53, 141, 144, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, + 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 277, 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 51, 54, 54, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 277, + 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 51, 54, 54, 0, 0, 0, + 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, - 135, 141, 141, 135, 141, 141, 83, 141, 144, 0, 0, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 267, 267, 267, 267, 267, + 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, 141, 141, 135, 141, 141, + 83, 141, 144, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, + 0, 0, 0, 0, 0, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, @@ -2219,13 +2244,12 @@ static const unsigned short index2[] = { 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, - 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, - 267, 267, 267, 267, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 0, 0, 0, 0, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, @@ -2234,7 +2258,8 @@ static const unsigned short index2[] = { 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - 278, 278, 278, 278, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 279, 279, + 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, @@ -2243,7 +2268,6 @@ static const unsigned short index2[] = { 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, - 279, 279, 279, 279, 279, 279, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, @@ -2253,38 +2277,39 @@ static const unsigned short index2[] = { 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 280, 280, 172, 172, 280, 172, 280, 172, - 172, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 172, 280, 172, - 280, 172, 172, 280, 280, 172, 172, 172, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 172, 172, 280, 172, 280, 172, 172, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 172, 280, 172, 280, 172, 172, 280, 280, 172, + 172, 172, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 35, 35, 35, - 35, 35, 35, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 35, 35, 35, 35, - 0, 0, 0, 0, 0, 282, 283, 282, 284, 284, 284, 284, 284, 284, 284, 284, - 284, 219, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, - 282, 0, 282, 282, 282, 282, 282, 0, 282, 0, 282, 282, 0, 282, 282, 0, - 282, 282, 282, 282, 282, 282, 282, 282, 282, 284, 131, 131, 131, 131, + 281, 281, 281, 281, 281, 281, 35, 35, 35, 35, 35, 35, 35, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 35, 35, 35, 35, 35, 0, 0, 0, 0, 0, 282, 283, 282, + 284, 284, 284, 284, 284, 284, 284, 284, 284, 219, 282, 282, 282, 282, + 282, 282, 282, 282, 282, 282, 282, 282, 282, 0, 282, 282, 282, 282, 282, + 0, 282, 0, 282, 282, 0, 282, 282, 0, 282, 282, 282, 282, 282, 282, 282, + 282, 282, 284, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 140, 140, 140, 140, - 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, + 140, 140, 140, 140, 140, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, @@ -2300,27 +2325,27 @@ static const unsigned short index2[] = { 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 285, 199, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 285, 199, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 131, + 131, 131, 131, 131, 131, 131, 0, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, - 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 286, 26, 26, 26, 71, 71, 71, 71, 71, 71, 71, 71, 71, - 71, 71, 71, 71, 71, 71, 71, 287, 287, 287, 287, 287, 287, 287, 288, 289, - 287, 0, 0, 0, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 86, 86, 86, 86, 86, - 86, 86, 81, 81, 287, 290, 290, 291, 291, 288, 289, 288, 289, 288, 289, - 288, 289, 288, 289, 288, 289, 288, 289, 288, 289, 253, 253, 288, 289, - 287, 287, 287, 287, 291, 291, 291, 292, 287, 292, 0, 287, 292, 287, 287, - 290, 293, 294, 293, 294, 293, 294, 295, 287, 287, 296, 297, 298, 298, - 299, 0, 287, 300, 295, 287, 0, 0, 0, 0, 131, 131, 131, 118, 131, 0, 131, + 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 286, 26, 26, + 26, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 287, + 287, 287, 287, 287, 287, 287, 288, 289, 287, 0, 0, 0, 0, 0, 0, 81, 81, + 81, 81, 81, 81, 81, 86, 86, 86, 86, 86, 86, 86, 81, 81, 287, 290, 290, + 291, 291, 288, 289, 288, 289, 288, 289, 288, 289, 288, 289, 288, 289, + 288, 289, 288, 289, 253, 253, 288, 289, 287, 287, 287, 287, 291, 291, + 291, 292, 287, 292, 0, 287, 292, 287, 287, 290, 293, 294, 293, 294, 293, + 294, 295, 287, 287, 296, 297, 298, 298, 299, 0, 287, 300, 295, 287, 0, 0, + 0, 0, 131, 131, 131, 118, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, @@ -2330,133 +2355,133 @@ static const unsigned short index2[] = { 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 177, 0, 301, 301, 302, 303, - 302, 301, 301, 304, 305, 301, 306, 307, 308, 307, 307, 309, 309, 309, - 309, 309, 309, 309, 309, 309, 309, 307, 301, 310, 311, 310, 301, 301, + 131, 0, 0, 177, 0, 301, 301, 302, 303, 302, 301, 301, 304, 305, 301, 306, + 307, 308, 307, 307, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, + 307, 301, 310, 311, 310, 301, 301, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, - 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 304, 301, - 305, 313, 314, 313, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, + 312, 312, 312, 312, 312, 304, 301, 305, 313, 314, 313, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, - 315, 315, 304, 311, 305, 311, 304, 305, 316, 317, 318, 316, 316, 319, - 319, 319, 319, 319, 319, 319, 319, 319, 319, 320, 319, 319, 319, 319, + 315, 315, 315, 315, 315, 315, 315, 315, 315, 304, 311, 305, 311, 304, + 305, 316, 317, 318, 316, 316, 319, 319, 319, 319, 319, 319, 319, 319, + 319, 319, 320, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 320, - 320, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, + 319, 319, 319, 319, 319, 319, 320, 320, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - 319, 319, 319, 319, 0, 0, 0, 319, 319, 319, 319, 319, 319, 0, 0, 319, - 319, 319, 319, 319, 319, 0, 0, 319, 319, 319, 319, 319, 319, 0, 0, 319, - 319, 319, 0, 0, 0, 303, 303, 311, 313, 321, 303, 303, 0, 322, 323, 323, - 323, 323, 322, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 324, 324, 26, 30, - 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, + 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 0, 0, 0, 319, 319, + 319, 319, 319, 319, 0, 0, 319, 319, 319, 319, 319, 319, 0, 0, 319, 319, + 319, 319, 319, 319, 0, 0, 319, 319, 319, 0, 0, 0, 303, 303, 311, 313, + 321, 303, 303, 0, 322, 323, 323, 323, 323, 322, 322, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 324, 324, 324, 26, 30, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 83, 138, 83, 0, 0, 0, 0, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, + 0, 83, 138, 83, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 325, 325, 325, + 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - 325, 325, 325, 325, 325, 325, 325, 325, 155, 155, 155, 155, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 155, 155, 26, 80, - 80, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 26, + 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 155, + 155, 155, 155, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 155, 155, 26, 80, 80, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 86, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, - 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 0, 0, 0, - 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 150, 150, - 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 175, 48, 48, 48, 48, 48, 48, - 48, 48, 175, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 326, 326, 326, 326, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 83, 48, 48, 48, 48, + 175, 48, 48, 48, 48, 48, 48, 48, 48, 175, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 48, - 48, 48, 48, 48, 48, 48, 48, 83, 175, 175, 175, 175, 175, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 81, + 81, 81, 81, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 83, 175, 175, + 175, 175, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 146, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 0, 0, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, - 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 0, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 0, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, 47, 47, 0, 47, 47, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 44, 44, 44, 44, 44, 0, 44, + 44, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, 47, 47, 0, + 47, 47, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 53, 51, 51, 51, 51, 51, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, + 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 53, 51, 51, 51, 51, 51, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 107, 107, 107, 107, 107, 107, 0, 0, 107, 0, 107, 107, 107, 107, + 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 0, 0, 107, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 107, 107, - 0, 0, 0, 107, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 104, - 327, 327, 327, 327, 327, 327, 327, 327, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 107, + 107, 0, 0, 0, 107, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, + 104, 327, 327, 327, 327, 327, 327, 327, 327, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 328, 328, 327, 327, 327, 327, 327, 327, 327, 107, 107, + 107, 107, 107, 107, 328, 328, 327, 327, 327, 327, 327, 327, 327, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 0, 0, 0, 0, 0, 0, 0, 0, 327, 327, 327, 327, 327, 327, 327, 327, 327, + 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, 327, 327, 327, 327, 327, 327, 327, 327, + 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 107, 107, 0, 0, 0, 0, 0, 327, 327, 327, 327, 327, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, 327, 327, 327, @@ -2524,150 +2549,165 @@ static const unsigned short index2[] = { 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 81, 81, 81, 81, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 107, 107, 107, 107, 137, 107, 329, 329, 329, + 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, + 329, 329, 329, 329, 329, 0, 0, 0, 81, 81, 81, 81, 81, 84, 137, 330, 330, + 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + 330, 330, 330, 330, 330, 330, 0, 0, 0, 0, 0, 0, 0, 0, 331, 331, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, - 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, - 331, 331, 331, 331, 331, 331, 331, 0, 107, 107, 107, 107, 107, 107, 107, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 332, 332, 332, 332, 332, 332, + 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, + 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 0, 81, 81, 102, 0, 0, 107, 107, 0, 0, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 81, 81, 102, 0, 0, + 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 118, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 86, 86, 86, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 0, 0, 0, 0, 0, 0, 0, 135, 86, 86, 86, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 327, 327, 327, 327, 327, 327, 327, 327, 327, 327, - 107, 0, 0, 0, 0, 0, 0, 0, 0, 118, 118, 118, 118, 118, 118, 118, 118, 118, - 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 86, 86, - 81, 81, 81, 86, 81, 86, 86, 86, 86, 332, 332, 332, 332, 113, 113, 113, - 113, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 81, 86, 81, 86, 104, 104, 104, 104, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, 327, 327, 327, 327, + 327, 327, 327, 327, 107, 0, 0, 0, 0, 0, 0, 0, 0, 118, 118, 118, 118, 118, + 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, + 118, 118, 118, 86, 86, 81, 81, 81, 86, 81, 86, 86, 86, 86, 333, 333, 333, + 333, 113, 113, 113, 113, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, 327, - 327, 327, 327, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 141, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 144, 83, 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 155, 155, 155, - 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, - 155, 155, 155, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 144, 48, - 48, 135, 135, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 135, 135, 141, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 142, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 142, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 141, 141, - 144, 143, 83, 83, 192, 83, 83, 83, 83, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 192, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 81, 81, 81, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 151, - 135, 135, 135, 135, 141, 135, 152, 152, 135, 135, 135, 144, 144, 0, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 83, 83, 48, 141, - 141, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 147, 83, 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, - 141, 135, 135, 135, 135, 135, 135, 135, 135, 135, 141, 176, 48, 48, 48, - 48, 83, 83, 83, 83, 135, 147, 135, 135, 83, 141, 135, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 48, 83, 48, 83, 83, 83, 0, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 141, 141, 141, 135, 135, 135, 141, 141, 135, 176, 147, 135, 83, - 83, 83, 83, 83, 83, 135, 48, 48, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 0, 48, 48, 48, 48, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 83, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, + 107, 107, 107, 107, 107, 107, 107, 107, 81, 86, 81, 86, 104, 104, 104, + 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 327, 327, 327, 327, 327, 327, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 135, 141, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 144, 83, 83, 83, 83, 83, 83, 83, + 0, 0, 0, 0, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 144, 48, 48, 135, 135, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 144, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 142, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 48, 48, 141, 141, 141, + 135, 135, 135, 135, 141, 141, 144, 143, 83, 83, 192, 83, 83, 83, 83, 135, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, + 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, + 0, 0, 0, 81, 81, 81, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 151, 135, 135, 135, 135, 141, 135, 152, 152, 135, + 135, 135, 144, 144, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 83, 83, 83, 83, 48, 141, 141, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 135, 141, 141, 141, 135, 135, 135, 135, 135, 135, 147, - 144, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, - 0, 0, 0, 0, 0, 135, 135, 141, 141, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, - 0, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, - 0, 48, 48, 48, 48, 48, 0, 147, 147, 48, 148, 141, 135, 141, 141, 141, - 141, 0, 0, 141, 141, 0, 0, 149, 149, 176, 0, 0, 48, 0, 0, 0, 0, 0, 0, - 148, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 141, 141, 0, 0, 81, 81, 81, 81, - 81, 81, 81, 0, 0, 0, 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 83, 83, 48, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, - 141, 141, 135, 135, 135, 135, 135, 135, 135, 135, 141, 141, 144, 135, - 135, 141, 147, 48, 48, 48, 48, 83, 83, 83, 83, 83, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 83, 83, 0, 83, 81, 48, 48, 48, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 141, 176, 48, 48, 48, 48, 83, 83, 83, 83, 135, 147, 135, 135, 83, + 141, 135, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 83, 48, + 83, 83, 83, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, 141, 135, - 135, 135, 135, 135, 135, 141, 151, 149, 149, 148, 149, 135, 135, 141, - 144, 147, 48, 48, 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 141, + 141, 135, 176, 147, 135, 83, 83, 83, 83, 83, 83, 135, 48, 48, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 0, + 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 83, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, 141, 135, 135, 135, - 135, 0, 0, 141, 141, 149, 149, 135, 135, 141, 144, 147, 83, 83, 83, 83, - 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, - 83, 48, 48, 48, 48, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, 141, 141, 135, 135, + 135, 135, 135, 135, 147, 144, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 135, 135, 141, 141, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, + 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 0, 147, 147, 48, + 148, 141, 135, 141, 141, 141, 141, 0, 0, 141, 141, 0, 0, 149, 149, 176, + 0, 0, 48, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 141, + 141, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 0, 0, 81, 81, 81, 81, 81, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 142, 48, 142, 48, 48, 48, 48, 0, + 48, 0, 0, 142, 0, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 0, 48, 148, 141, 141, 151, 135, 135, 135, + 135, 135, 0, 148, 0, 0, 334, 0, 334, 334, 148, 141, 0, 141, 141, 144, + 176, 144, 48, 135, 48, 83, 83, 0, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 135, + 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, - 135, 135, 135, 135, 141, 141, 135, 141, 144, 135, 83, 83, 83, 48, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 0, 0, 0, 0, 0, 0, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, - 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 135, 135, 135, 135, 141, + 141, 144, 135, 135, 141, 147, 48, 48, 48, 48, 83, 83, 83, 83, 83, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 0, 83, 81, 48, 48, + 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, + 141, 141, 135, 135, 135, 135, 135, 135, 141, 151, 149, 149, 148, 149, + 135, 135, 141, 144, 147, 48, 48, 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 135, 141, 135, 141, 141, 135, 135, 135, 135, - 135, 135, 176, 147, 48, 83, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, + 141, 135, 135, 135, 135, 0, 0, 141, 141, 149, 149, 135, 135, 141, 144, + 147, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, + 83, 83, 83, 83, 83, 83, 48, 48, 48, 48, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, + 135, 135, 135, 135, 135, 135, 135, 141, 141, 135, 141, 144, 135, 83, 83, + 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 0, 135, 135, 135, 141, 141, 135, 135, 135, 135, 141, 135, 135, - 135, 135, 144, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 150, 150, 83, 83, 83, 80, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, 135, 141, 141, 135, + 135, 135, 135, 135, 135, 176, 147, 48, 83, 0, 0, 0, 0, 0, 0, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, + 135, 141, 135, 141, 141, 135, 135, 135, 135, 141, 135, 135, 135, 135, + 144, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 150, + 150, 83, 83, 83, 80, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 141, 144, 147, 83, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 141, 144, 147, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, 141, - 141, 141, 141, 0, 141, 149, 0, 0, 135, 135, 176, 144, 48, 141, 48, 141, - 147, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 47, 47, 47, 47, 47, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 0, 0, 48, 48, 48, 48, + 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, 141, 141, + 141, 141, 0, 141, 149, 0, 0, 135, 135, 176, 144, 48, 141, 48, 141, 147, + 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 0, 0, 135, 135, 141, 141, 141, 141, 144, 48, 83, @@ -2692,10 +2732,16 @@ static const unsigned short index2[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 83, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 135, - 135, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, 135, 135, 141, 333, + 135, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, 135, 135, 141, 335, 48, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 83, 83, @@ -2730,15 +2776,15 @@ static const unsigned short index2[] = { 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, 135, 135, 135, 135, 0, 0, 0, 141, 141, 135, 176, 144, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 146, 146, 146, 146, 146, 146, 146, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, 150, 150, + 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 26, 26, 26, 26, 26, 26, 26, 26, 85, 85, 85, 85, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 150, 150, 150, 150, 26, 26, 26, 26, 26, 26, 26, 26, 85, 85, 85, 85, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2771,14 +2817,21 @@ static const unsigned short index2[] = { 48, 48, 48, 48, 48, 48, 48, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 135, 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 151, 151, 151, 336, 336, 336, 336, 336, 336, 336, 336, 151, 141, 141, + 141, 135, 135, 144, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, @@ -2808,30 +2861,36 @@ static const unsigned short index2[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 0, 0, 0, 0, 0, 0, 53, 53, 53, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 173, 337, 142, 142, 53, 53, 83, 83, 83, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 47, 47, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 0, 0, 0, 0, 135, 48, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 48, 48, 48, 48, 48, 0, 0, 0, 0, 135, 48, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - 141, 141, 141, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 53, 53, 53, 53, - 53, 53, 53, 53, 53, 53, 53, 53, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 141, 141, 141, 141, 141, 141, 141, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, + 135, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 254, 254, 253, 254, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 335, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 254, 253, 254, 338, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 339, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -2840,16 +2899,16 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2858,14 +2917,14 @@ static const unsigned short index2[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 254, 254, 254, 0, - 254, 254, 254, 254, 254, 254, 254, 0, 254, 254, 0, 172, 172, 172, 172, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, + 254, 254, 254, 0, 254, 254, 254, 254, 254, 254, 254, 0, 254, 254, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 172, 172, 172, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -2875,34 +2934,55 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 0, 80, 135, 178, 83, 177, 177, 177, 177, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 0, 0, 80, 135, 178, 83, 177, 177, 177, 177, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 135, 135, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, + 135, 135, 135, 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 80, 80, 80, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, @@ -2914,13 +2994,13 @@ static const unsigned short index2[] = { 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 336, 336, - 336, 336, 336, 336, 336, 337, 337, 178, 178, 178, 80, 80, 80, 338, 337, - 337, 337, 337, 337, 177, 177, 177, 177, 177, 177, 177, 177, 86, 86, 86, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 342, 342, + 342, 342, 342, 342, 342, 343, 343, 178, 178, 178, 80, 80, 80, 344, 343, + 343, 343, 343, 343, 177, 177, 177, 177, 177, 177, 177, 177, 86, 86, 86, 86, 86, 86, 86, 86, 80, 80, 81, 81, 81, 81, 81, 86, 86, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 81, 81, 81, 81, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 336, 336, 336, 336, 336, 336, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 342, 342, 342, 342, 342, 342, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2936,129 +3016,137 @@ static const unsigned short index2[] = { 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, - 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, + 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 150, + 150, 0, 0, 0, 0, 0, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, + 35, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 49, 0, 49, 49, 0, 0, 49, 0, 0, 49, 49, 0, 0, 49, 49, 49, 49, 0, 49, 49, + 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 0, 35, 0, 35, 35, 35, 35, 35, 35, + 35, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, + 0, 49, 49, 49, 49, 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, 49, 49, 0, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, 49, 49, 49, 49, + 49, 0, 49, 0, 0, 0, 49, 49, 49, 49, 49, 49, 49, 0, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 0, 35, 35, 35, + 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 0, 49, 49, 0, 0, - 49, 0, 0, 49, 49, 0, 0, 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, 49, 49, - 49, 35, 35, 35, 35, 0, 35, 0, 35, 35, 35, 35, 35, 35, 35, 0, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, 0, 49, 49, 49, 49, - 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, 49, 49, 0, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, 0, 49, 0, 0, 0, - 49, 49, 49, 49, 49, 49, 49, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, + 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, + 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, + 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 339, 35, 35, 35, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 339, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 339, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, - 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 339, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, - 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 339, 35, 35, 35, 35, 35, + 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 35, 0, 0, 340, 340, 340, 340, - 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, - 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, - 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, - 340, 340, 340, 340, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 35, 0, 0, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 80, 80, 80, 80, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 80, 80, 80, 80, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 80, 80, 80, 80, 80, 80, 80, 80, 135, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 135, 80, 80, 83, 83, 83, 83, 83, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 80, - 80, 80, 80, 80, 80, 80, 80, 135, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 135, 80, 80, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, + 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 0, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 47, - 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 81, 81, - 81, 81, 81, 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 81, 81, 0, 81, 81, 81, - 81, 81, 0, 0, 0, 0, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 0, 0, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 0, 0, 81, 81, 81, 81, 81, 81, 81, + 0, 81, 81, 0, 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, 0, 0, 0, 0, + 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 53, 53, 53, - 53, 53, 53, 53, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 0, 0, 0, 0, 48, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 81, 81, 81, 81, + 81, 81, 81, 53, 53, 53, 53, 53, 53, 53, 0, 0, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 0, 0, 0, 0, 48, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 81, 81, 81, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 85, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 81, 81, 81, 81, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 53, 182, 182, 86, 81, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 182, 182, 86, 81, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 81, 86, 48, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 107, 107, 107, 107, 107, 107, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, + 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, @@ -3072,112 +3160,110 @@ static const unsigned short index2[] = { 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 0, 327, 327, 327, 327, - 327, 327, 327, 327, 327, 86, 86, 86, 86, 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, + 107, 107, 107, 107, 0, 0, 327, 327, 327, 327, 327, 327, 327, 327, 327, + 86, 86, 86, 86, 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 329, 329, 329, 329, 329, 329, 329, + 0, 0, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, - 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 330, 330, - 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + 329, 329, 329, 329, 329, 329, 329, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, - 330, 330, 330, 330, 81, 81, 81, 81, 81, 81, 147, 137, 0, 0, 0, 0, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 0, 0, 0, 0, 104, 104, 0, 0, + 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 81, 81, + 81, 81, 81, 81, 147, 137, 0, 0, 0, 0, 136, 136, 136, 136, 136, 136, 136, + 136, 136, 136, 0, 0, 0, 0, 104, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 133, 332, 332, 332, 111, 332, 332, 332, 332, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 133, 333, 333, 333, + 111, 333, 333, 333, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 133, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 133, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 0, 131, 131, 0, 131, 0, 0, 131, 0, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 0, 131, 131, 131, 131, 0, 131, 0, 131, 0, - 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 131, 0, 131, 0, 131, 0, 131, 131, 131, 0, - 131, 131, 0, 131, 0, 0, 131, 0, 131, 0, 131, 0, 131, 0, 131, 0, 131, 131, - 0, 131, 0, 0, 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, - 0, 131, 131, 131, 131, 0, 131, 131, 131, 131, 0, 131, 0, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 131, - 131, 131, 0, 131, 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 78, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 243, 26, 26, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 131, 131, + 0, 131, 0, 0, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 0, 131, 131, 131, 131, 0, 131, 0, 131, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, + 131, 0, 131, 0, 131, 0, 131, 131, 131, 0, 131, 131, 0, 131, 0, 0, 131, 0, + 131, 0, 131, 0, 131, 0, 131, 0, 131, 131, 0, 131, 0, 0, 131, 131, 131, + 131, 0, 131, 131, 131, 131, 131, 131, 131, 0, 131, 131, 131, 131, 0, 131, + 131, 131, 131, 0, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 0, 0, 0, 0, 0, 131, 131, 131, 0, 131, 131, 131, 131, + 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 0, 26, 26, + 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 243, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 155, - 155, 26, 26, 26, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 155, 155, 26, 26, 26, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 341, 26, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 340, 26, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 342, 342, 342, 342, - 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - 342, 342, 342, 342, 342, 342, 342, 342, 226, 226, 226, 26, 26, 26, 342, - 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - 342, 272, 342, 246, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, - 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - 342, 342, 342, 342, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 274, 274, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 274, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 226, 226, 226, 26, 26, 26, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 272, 346, 246, 272, 272, + 272, 272, 272, 272, 272, 272, 272, 272, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 274, 274, 274, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 0, - 0, 0, 0, 274, 274, 274, 274, 274, 274, 274, 274, 274, 0, 0, 0, 0, 0, 0, - 0, 274, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, - 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 274, 274, 274, 274, 274, 274, 274, 274, 0, 0, 0, 0, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 0, 0, 0, 0, 0, 0, 0, 274, 274, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 26, 26, 26, 26, 243, 243, 243, 243, 243, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 243, 26, 26, 26, - 243, 243, 243, 343, 343, 343, 343, 343, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 26, 26, 26, 243, 26, 26, 26, 243, 243, 243, 347, 347, 347, + 347, 347, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 26, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, @@ -3189,71 +3275,73 @@ static const unsigned short index2[] = { 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, - 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 26, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, - 243, 26, 26, 26, 243, 243, 243, 26, 26, 243, 243, 243, 0, 0, 0, 0, 243, - 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 0, - 0, 0, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, - 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 243, 243, + 243, 26, 26, 243, 243, 243, 0, 0, 0, 0, 243, 243, 243, 243, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 0, 0, 0, 26, 26, 26, 26, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, + 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, - 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, + 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, @@ -3266,58 +3354,50 @@ static const unsigned short index2[] = { 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 243, 243, - 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 340, 340, 340, 340, 340, 340, 340, - 340, 340, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, - 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -3325,9 +3405,9 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -3338,17 +3418,18 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, @@ -3359,9 +3440,9 @@ static const unsigned short index2[] = { 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 280, 281, 281, 281, 281, 281, 281, 281, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, @@ -3377,34 +3458,34 @@ static const unsigned short index2[] = { 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 0, 0, 172, 172, 172, 172, 172, 172, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, - 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 0, 177, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, - 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 71, - 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, + 177, 177, 177, 177, 177, 177, 177, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, @@ -3417,7 +3498,8 @@ static const unsigned short index2[] = { 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - 71, 71, 71, 71, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, @@ -3426,7 +3508,7 @@ static const unsigned short index2[] = { 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, - 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 0, 0, + 279, 279, 279, 279, 279, 279, 279, 0, 0, }; /* decomposition data */ @@ -4203,156 +4285,161 @@ static const unsigned int decomp_data[] = { 12638, 272, 12639, 272, 12640, 272, 12641, 272, 12642, 272, 12643, 264, 162, 264, 163, 264, 172, 264, 175, 264, 166, 264, 165, 264, 8361, 272, 9474, 272, 8592, 272, 8593, 272, 8594, 272, 8595, 272, 9632, 272, 9675, - 259, 720, 259, 721, 259, 230, 259, 665, 259, 595, 259, 675, 259, 43878, - 259, 677, 259, 676, 259, 598, 259, 599, 259, 7569, 259, 600, 259, 606, - 259, 681, 259, 612, 259, 610, 259, 608, 259, 667, 259, 295, 259, 668, - 259, 615, 259, 644, 259, 682, 259, 683, 259, 620, 259, 122628, 259, - 42894, 259, 622, 259, 122629, 259, 654, 259, 122630, 259, 248, 259, 630, - 259, 631, 259, 113, 259, 634, 259, 122632, 259, 637, 259, 638, 259, 640, - 259, 680, 259, 678, 259, 43879, 259, 679, 259, 648, 259, 11377, 259, 655, - 259, 673, 259, 674, 259, 664, 259, 448, 259, 449, 259, 450, 259, 122634, - 259, 122654, 512, 69785, 69818, 512, 69787, 69818, 512, 69797, 69818, - 512, 69937, 69927, 512, 69938, 69927, 512, 70471, 70462, 512, 70471, - 70487, 512, 70841, 70842, 512, 70841, 70832, 512, 70841, 70845, 512, - 71096, 71087, 512, 71097, 71087, 512, 71989, 71984, 512, 119127, 119141, - 512, 119128, 119141, 512, 119135, 119150, 512, 119135, 119151, 512, - 119135, 119152, 512, 119135, 119153, 512, 119135, 119154, 512, 119225, - 119141, 512, 119226, 119141, 512, 119227, 119150, 512, 119228, 119150, - 512, 119227, 119151, 512, 119228, 119151, 262, 65, 262, 71, 262, 74, 262, - 75, 262, 79, 262, 83, 262, 84, 262, 85, 262, 86, 262, 87, 262, 88, 262, - 89, 262, 97, 262, 98, 262, 99, 262, 102, 262, 107, 262, 109, 262, 110, - 262, 112, 262, 113, 262, 114, 262, 115, 262, 116, 262, 117, 262, 118, - 262, 119, 262, 120, 262, 121, 262, 122, 262, 305, 262, 567, 262, 913, - 262, 914, 262, 916, 262, 917, 262, 918, 262, 919, 262, 920, 262, 921, - 262, 922, 262, 923, 262, 924, 262, 925, 262, 926, 262, 927, 262, 929, - 262, 1012, 262, 931, 262, 932, 262, 933, 262, 934, 262, 935, 262, 936, - 262, 937, 262, 8711, 262, 945, 262, 946, 262, 948, 262, 949, 262, 950, - 262, 951, 262, 952, 262, 953, 262, 954, 262, 955, 262, 956, 262, 957, - 262, 958, 262, 959, 262, 961, 262, 962, 262, 963, 262, 964, 262, 965, - 262, 966, 262, 967, 262, 968, 262, 969, 262, 8706, 262, 1013, 262, 977, - 262, 1008, 262, 981, 262, 1009, 262, 982, 262, 988, 262, 989, 262, 48, - 262, 49, 262, 50, 262, 51, 262, 52, 262, 53, 262, 54, 262, 55, 262, 56, - 262, 57, 259, 1072, 259, 1073, 259, 1074, 259, 1075, 259, 1076, 259, - 1077, 259, 1078, 259, 1079, 259, 1080, 259, 1082, 259, 1083, 259, 1084, - 259, 1086, 259, 1087, 259, 1088, 259, 1089, 259, 1090, 259, 1091, 259, - 1092, 259, 1093, 259, 1094, 259, 1095, 259, 1096, 259, 1099, 259, 1101, - 259, 1102, 259, 42633, 259, 1241, 259, 1110, 259, 1112, 259, 1257, 259, - 1199, 259, 1231, 261, 1072, 261, 1073, 261, 1074, 261, 1075, 261, 1076, - 261, 1077, 261, 1078, 261, 1079, 261, 1080, 261, 1082, 261, 1083, 261, - 1086, 261, 1087, 261, 1089, 261, 1091, 261, 1092, 261, 1093, 261, 1094, - 261, 1095, 261, 1096, 261, 1098, 261, 1099, 261, 1169, 261, 1110, 261, - 1109, 261, 1119, 259, 1195, 259, 42577, 259, 1201, 262, 1575, 262, 1576, - 262, 1580, 262, 1583, 262, 1608, 262, 1586, 262, 1581, 262, 1591, 262, - 1610, 262, 1603, 262, 1604, 262, 1605, 262, 1606, 262, 1587, 262, 1593, - 262, 1601, 262, 1589, 262, 1602, 262, 1585, 262, 1588, 262, 1578, 262, - 1579, 262, 1582, 262, 1584, 262, 1590, 262, 1592, 262, 1594, 262, 1646, - 262, 1722, 262, 1697, 262, 1647, 262, 1607, 514, 48, 46, 514, 48, 44, - 514, 49, 44, 514, 50, 44, 514, 51, 44, 514, 52, 44, 514, 53, 44, 514, 54, - 44, 514, 55, 44, 514, 56, 44, 514, 57, 44, 770, 40, 65, 41, 770, 40, 66, - 41, 770, 40, 67, 41, 770, 40, 68, 41, 770, 40, 69, 41, 770, 40, 70, 41, - 770, 40, 71, 41, 770, 40, 72, 41, 770, 40, 73, 41, 770, 40, 74, 41, 770, - 40, 75, 41, 770, 40, 76, 41, 770, 40, 77, 41, 770, 40, 78, 41, 770, 40, - 79, 41, 770, 40, 80, 41, 770, 40, 81, 41, 770, 40, 82, 41, 770, 40, 83, - 41, 770, 40, 84, 41, 770, 40, 85, 41, 770, 40, 86, 41, 770, 40, 87, 41, - 770, 40, 88, 41, 770, 40, 89, 41, 770, 40, 90, 41, 770, 12308, 83, 12309, - 519, 67, 68, 519, 87, 90, 266, 65, 266, 66, 266, 67, 266, 68, 266, 69, - 266, 70, 266, 71, 266, 72, 266, 73, 266, 74, 266, 75, 266, 76, 266, 77, - 266, 78, 266, 79, 266, 80, 266, 81, 266, 82, 266, 83, 266, 84, 266, 85, - 266, 86, 266, 87, 266, 88, 266, 89, 266, 90, 522, 72, 86, 522, 83, 68, - 522, 83, 83, 778, 80, 80, 86, 522, 87, 67, 515, 77, 67, 515, 77, 68, 515, - 77, 82, 522, 68, 74, 522, 12411, 12363, 522, 12467, 12467, 266, 12469, - 266, 25163, 266, 23383, 266, 21452, 266, 12487, 266, 20108, 266, 22810, - 266, 35299, 266, 22825, 266, 20132, 266, 26144, 266, 28961, 266, 26009, - 266, 21069, 266, 24460, 266, 20877, 266, 26032, 266, 21021, 266, 32066, - 266, 29983, 266, 36009, 266, 22768, 266, 21561, 266, 28436, 266, 25237, - 266, 25429, 266, 19968, 266, 19977, 266, 36938, 266, 24038, 266, 20013, - 266, 21491, 266, 25351, 266, 36208, 266, 25171, 266, 31105, 266, 31354, - 266, 21512, 266, 28288, 266, 26377, 266, 26376, 266, 30003, 266, 21106, - 266, 21942, 266, 37197, 770, 12308, 26412, 12309, 770, 12308, 19977, - 12309, 770, 12308, 20108, 12309, 770, 12308, 23433, 12309, 770, 12308, - 28857, 12309, 770, 12308, 25171, 12309, 770, 12308, 30423, 12309, 770, - 12308, 21213, 12309, 770, 12308, 25943, 12309, 263, 24471, 263, 21487, - 256, 20029, 256, 20024, 256, 20033, 256, 131362, 256, 20320, 256, 20411, - 256, 20482, 256, 20602, 256, 20633, 256, 20687, 256, 13470, 256, 132666, - 256, 20820, 256, 20836, 256, 20855, 256, 132380, 256, 13497, 256, 20839, - 256, 20877, 256, 132427, 256, 20887, 256, 20900, 256, 20172, 256, 20908, - 256, 168415, 256, 20981, 256, 20995, 256, 13535, 256, 21051, 256, 21062, - 256, 21106, 256, 21111, 256, 13589, 256, 21253, 256, 21254, 256, 21321, - 256, 21338, 256, 21363, 256, 21373, 256, 21375, 256, 133676, 256, 28784, - 256, 21450, 256, 21471, 256, 133987, 256, 21483, 256, 21489, 256, 21510, - 256, 21662, 256, 21560, 256, 21576, 256, 21608, 256, 21666, 256, 21750, - 256, 21776, 256, 21843, 256, 21859, 256, 21892, 256, 21931, 256, 21939, - 256, 21954, 256, 22294, 256, 22295, 256, 22097, 256, 22132, 256, 22766, - 256, 22478, 256, 22516, 256, 22541, 256, 22411, 256, 22578, 256, 22577, - 256, 22700, 256, 136420, 256, 22770, 256, 22775, 256, 22790, 256, 22810, - 256, 22818, 256, 22882, 256, 136872, 256, 136938, 256, 23020, 256, 23067, - 256, 23079, 256, 23000, 256, 23142, 256, 14062, 256, 14076, 256, 23304, - 256, 23358, 256, 137672, 256, 23491, 256, 23512, 256, 23539, 256, 138008, - 256, 23551, 256, 23558, 256, 24403, 256, 23586, 256, 14209, 256, 23648, - 256, 23744, 256, 23693, 256, 138724, 256, 23875, 256, 138726, 256, 23918, - 256, 23915, 256, 23932, 256, 24033, 256, 24034, 256, 14383, 256, 24061, - 256, 24104, 256, 24125, 256, 24169, 256, 14434, 256, 139651, 256, 14460, - 256, 24240, 256, 24243, 256, 24246, 256, 172946, 256, 24318, 256, 140081, - 256, 33281, 256, 24354, 256, 14535, 256, 144056, 256, 156122, 256, 24418, - 256, 24427, 256, 14563, 256, 24474, 256, 24525, 256, 24535, 256, 24569, - 256, 24705, 256, 14650, 256, 14620, 256, 141012, 256, 24775, 256, 24904, - 256, 24908, 256, 24954, 256, 25010, 256, 24996, 256, 25007, 256, 25054, - 256, 25104, 256, 25115, 256, 25181, 256, 25265, 256, 25300, 256, 25424, - 256, 142092, 256, 25405, 256, 25340, 256, 25448, 256, 25475, 256, 25572, - 256, 142321, 256, 25634, 256, 25541, 256, 25513, 256, 14894, 256, 25705, - 256, 25726, 256, 25757, 256, 25719, 256, 14956, 256, 25964, 256, 143370, - 256, 26083, 256, 26360, 256, 26185, 256, 15129, 256, 15112, 256, 15076, - 256, 20882, 256, 20885, 256, 26368, 256, 26268, 256, 32941, 256, 17369, - 256, 26401, 256, 26462, 256, 26451, 256, 144323, 256, 15177, 256, 26618, - 256, 26501, 256, 26706, 256, 144493, 256, 26766, 256, 26655, 256, 26900, - 256, 26946, 256, 27043, 256, 27114, 256, 27304, 256, 145059, 256, 27355, - 256, 15384, 256, 27425, 256, 145575, 256, 27476, 256, 15438, 256, 27506, - 256, 27551, 256, 27579, 256, 146061, 256, 138507, 256, 146170, 256, - 27726, 256, 146620, 256, 27839, 256, 27853, 256, 27751, 256, 27926, 256, - 27966, 256, 28009, 256, 28024, 256, 28037, 256, 146718, 256, 27956, 256, - 28207, 256, 28270, 256, 15667, 256, 28359, 256, 147153, 256, 28153, 256, - 28526, 256, 147294, 256, 147342, 256, 28614, 256, 28729, 256, 28699, 256, - 15766, 256, 28746, 256, 28797, 256, 28791, 256, 28845, 256, 132389, 256, - 28997, 256, 148067, 256, 29084, 256, 148395, 256, 29224, 256, 29264, 256, - 149000, 256, 29312, 256, 29333, 256, 149301, 256, 149524, 256, 29562, - 256, 29579, 256, 16044, 256, 29605, 256, 16056, 256, 29767, 256, 29788, - 256, 29829, 256, 29898, 256, 16155, 256, 29988, 256, 150582, 256, 30014, - 256, 150674, 256, 139679, 256, 30224, 256, 151457, 256, 151480, 256, - 151620, 256, 16380, 256, 16392, 256, 151795, 256, 151794, 256, 151833, - 256, 151859, 256, 30494, 256, 30495, 256, 30603, 256, 16454, 256, 16534, - 256, 152605, 256, 30798, 256, 16611, 256, 153126, 256, 153242, 256, - 153285, 256, 31211, 256, 16687, 256, 31306, 256, 31311, 256, 153980, 256, - 154279, 256, 31470, 256, 16898, 256, 154539, 256, 31686, 256, 31689, 256, - 16935, 256, 154752, 256, 31954, 256, 17056, 256, 31976, 256, 31971, 256, - 32000, 256, 155526, 256, 32099, 256, 17153, 256, 32199, 256, 32258, 256, - 32325, 256, 17204, 256, 156200, 256, 156231, 256, 17241, 256, 156377, - 256, 32634, 256, 156478, 256, 32661, 256, 32762, 256, 156890, 256, - 156963, 256, 32864, 256, 157096, 256, 32880, 256, 144223, 256, 17365, - 256, 32946, 256, 33027, 256, 17419, 256, 33086, 256, 23221, 256, 157607, - 256, 157621, 256, 144275, 256, 144284, 256, 33284, 256, 36766, 256, - 17515, 256, 33425, 256, 33419, 256, 33437, 256, 21171, 256, 33457, 256, - 33459, 256, 33469, 256, 33510, 256, 158524, 256, 33565, 256, 33635, 256, - 33709, 256, 33571, 256, 33725, 256, 33767, 256, 33619, 256, 33738, 256, - 33740, 256, 33756, 256, 158774, 256, 159083, 256, 158933, 256, 17707, - 256, 34033, 256, 34035, 256, 34070, 256, 160714, 256, 34148, 256, 159532, - 256, 17757, 256, 17761, 256, 159665, 256, 159954, 256, 17771, 256, 34384, - 256, 34407, 256, 34409, 256, 34473, 256, 34440, 256, 34574, 256, 34530, - 256, 34600, 256, 34667, 256, 34694, 256, 17879, 256, 34785, 256, 34817, - 256, 17913, 256, 34912, 256, 34915, 256, 161383, 256, 35031, 256, 35038, - 256, 17973, 256, 35066, 256, 13499, 256, 161966, 256, 162150, 256, 18110, - 256, 18119, 256, 35488, 256, 35925, 256, 162984, 256, 36011, 256, 36033, - 256, 36123, 256, 36215, 256, 163631, 256, 133124, 256, 36299, 256, 36284, - 256, 36336, 256, 133342, 256, 36564, 256, 165330, 256, 165357, 256, - 37012, 256, 37105, 256, 37137, 256, 165678, 256, 37147, 256, 37432, 256, - 37591, 256, 37592, 256, 37500, 256, 37881, 256, 37909, 256, 166906, 256, - 38283, 256, 18837, 256, 38327, 256, 167287, 256, 18918, 256, 38595, 256, - 23986, 256, 38691, 256, 168261, 256, 168474, 256, 19054, 256, 19062, 256, - 38880, 256, 168970, 256, 19122, 256, 169110, 256, 38953, 256, 169398, - 256, 39138, 256, 19251, 256, 39209, 256, 39335, 256, 39362, 256, 39422, - 256, 19406, 256, 170800, 256, 40000, 256, 40189, 256, 19662, 256, 19693, - 256, 40295, 256, 172238, 256, 19704, 256, 172293, 256, 172558, 256, - 172689, 256, 40635, 256, 19798, 256, 40697, 256, 40702, 256, 40709, 256, - 40719, 256, 40726, 256, 40763, 256, 173568, + 512, 67026, 775, 512, 67034, 775, 259, 720, 259, 721, 259, 230, 259, 665, + 259, 595, 259, 675, 259, 43878, 259, 677, 259, 676, 259, 598, 259, 599, + 259, 7569, 259, 600, 259, 606, 259, 681, 259, 612, 259, 610, 259, 608, + 259, 667, 259, 295, 259, 668, 259, 615, 259, 644, 259, 682, 259, 683, + 259, 620, 259, 122628, 259, 42894, 259, 622, 259, 122629, 259, 654, 259, + 122630, 259, 248, 259, 630, 259, 631, 259, 113, 259, 634, 259, 122632, + 259, 637, 259, 638, 259, 640, 259, 680, 259, 678, 259, 43879, 259, 679, + 259, 648, 259, 11377, 259, 655, 259, 673, 259, 674, 259, 664, 259, 448, + 259, 449, 259, 450, 259, 122634, 259, 122654, 512, 69785, 69818, 512, + 69787, 69818, 512, 69797, 69818, 512, 69937, 69927, 512, 69938, 69927, + 512, 70471, 70462, 512, 70471, 70487, 512, 70530, 70601, 512, 70532, + 70587, 512, 70539, 70594, 512, 70544, 70601, 512, 70594, 70594, 512, + 70594, 70584, 512, 70594, 70601, 512, 70841, 70842, 512, 70841, 70832, + 512, 70841, 70845, 512, 71096, 71087, 512, 71097, 71087, 512, 71989, + 71984, 512, 90398, 90398, 512, 90398, 90409, 512, 90398, 90399, 512, + 90409, 90399, 512, 90398, 90400, 512, 90401, 90399, 512, 90402, 90399, + 512, 90401, 90400, 512, 93543, 93543, 512, 93539, 93543, 512, 93545, + 93543, 262, 65, 262, 71, 262, 74, 262, 75, 262, 79, 262, 83, 262, 84, + 262, 85, 262, 86, 262, 87, 262, 88, 262, 89, 262, 48, 262, 49, 262, 50, + 262, 51, 262, 52, 262, 53, 262, 54, 262, 55, 262, 56, 262, 57, 512, + 119127, 119141, 512, 119128, 119141, 512, 119135, 119150, 512, 119135, + 119151, 512, 119135, 119152, 512, 119135, 119153, 512, 119135, 119154, + 512, 119225, 119141, 512, 119226, 119141, 512, 119227, 119150, 512, + 119228, 119150, 512, 119227, 119151, 512, 119228, 119151, 262, 97, 262, + 98, 262, 99, 262, 102, 262, 107, 262, 109, 262, 110, 262, 112, 262, 113, + 262, 114, 262, 115, 262, 116, 262, 117, 262, 118, 262, 119, 262, 120, + 262, 121, 262, 122, 262, 305, 262, 567, 262, 913, 262, 914, 262, 916, + 262, 917, 262, 918, 262, 919, 262, 920, 262, 921, 262, 922, 262, 923, + 262, 924, 262, 925, 262, 926, 262, 927, 262, 929, 262, 1012, 262, 931, + 262, 932, 262, 933, 262, 934, 262, 935, 262, 936, 262, 937, 262, 8711, + 262, 945, 262, 946, 262, 948, 262, 949, 262, 950, 262, 951, 262, 952, + 262, 953, 262, 954, 262, 955, 262, 956, 262, 957, 262, 958, 262, 959, + 262, 961, 262, 962, 262, 963, 262, 964, 262, 965, 262, 966, 262, 967, + 262, 968, 262, 969, 262, 8706, 262, 1013, 262, 977, 262, 1008, 262, 981, + 262, 1009, 262, 982, 262, 988, 262, 989, 259, 1072, 259, 1073, 259, 1074, + 259, 1075, 259, 1076, 259, 1077, 259, 1078, 259, 1079, 259, 1080, 259, + 1082, 259, 1083, 259, 1084, 259, 1086, 259, 1087, 259, 1088, 259, 1089, + 259, 1090, 259, 1091, 259, 1092, 259, 1093, 259, 1094, 259, 1095, 259, + 1096, 259, 1099, 259, 1101, 259, 1102, 259, 42633, 259, 1241, 259, 1110, + 259, 1112, 259, 1257, 259, 1199, 259, 1231, 261, 1072, 261, 1073, 261, + 1074, 261, 1075, 261, 1076, 261, 1077, 261, 1078, 261, 1079, 261, 1080, + 261, 1082, 261, 1083, 261, 1086, 261, 1087, 261, 1089, 261, 1091, 261, + 1092, 261, 1093, 261, 1094, 261, 1095, 261, 1096, 261, 1098, 261, 1099, + 261, 1169, 261, 1110, 261, 1109, 261, 1119, 259, 1195, 259, 42577, 259, + 1201, 262, 1575, 262, 1576, 262, 1580, 262, 1583, 262, 1608, 262, 1586, + 262, 1581, 262, 1591, 262, 1610, 262, 1603, 262, 1604, 262, 1605, 262, + 1606, 262, 1587, 262, 1593, 262, 1601, 262, 1589, 262, 1602, 262, 1585, + 262, 1588, 262, 1578, 262, 1579, 262, 1582, 262, 1584, 262, 1590, 262, + 1592, 262, 1594, 262, 1646, 262, 1722, 262, 1697, 262, 1647, 262, 1607, + 514, 48, 46, 514, 48, 44, 514, 49, 44, 514, 50, 44, 514, 51, 44, 514, 52, + 44, 514, 53, 44, 514, 54, 44, 514, 55, 44, 514, 56, 44, 514, 57, 44, 770, + 40, 65, 41, 770, 40, 66, 41, 770, 40, 67, 41, 770, 40, 68, 41, 770, 40, + 69, 41, 770, 40, 70, 41, 770, 40, 71, 41, 770, 40, 72, 41, 770, 40, 73, + 41, 770, 40, 74, 41, 770, 40, 75, 41, 770, 40, 76, 41, 770, 40, 77, 41, + 770, 40, 78, 41, 770, 40, 79, 41, 770, 40, 80, 41, 770, 40, 81, 41, 770, + 40, 82, 41, 770, 40, 83, 41, 770, 40, 84, 41, 770, 40, 85, 41, 770, 40, + 86, 41, 770, 40, 87, 41, 770, 40, 88, 41, 770, 40, 89, 41, 770, 40, 90, + 41, 770, 12308, 83, 12309, 519, 67, 68, 519, 87, 90, 266, 65, 266, 66, + 266, 67, 266, 68, 266, 69, 266, 70, 266, 71, 266, 72, 266, 73, 266, 74, + 266, 75, 266, 76, 266, 77, 266, 78, 266, 79, 266, 80, 266, 81, 266, 82, + 266, 83, 266, 84, 266, 85, 266, 86, 266, 87, 266, 88, 266, 89, 266, 90, + 522, 72, 86, 522, 83, 68, 522, 83, 83, 778, 80, 80, 86, 522, 87, 67, 515, + 77, 67, 515, 77, 68, 515, 77, 82, 522, 68, 74, 522, 12411, 12363, 522, + 12467, 12467, 266, 12469, 266, 25163, 266, 23383, 266, 21452, 266, 12487, + 266, 20108, 266, 22810, 266, 35299, 266, 22825, 266, 20132, 266, 26144, + 266, 28961, 266, 26009, 266, 21069, 266, 24460, 266, 20877, 266, 26032, + 266, 21021, 266, 32066, 266, 29983, 266, 36009, 266, 22768, 266, 21561, + 266, 28436, 266, 25237, 266, 25429, 266, 19968, 266, 19977, 266, 36938, + 266, 24038, 266, 20013, 266, 21491, 266, 25351, 266, 36208, 266, 25171, + 266, 31105, 266, 31354, 266, 21512, 266, 28288, 266, 26377, 266, 26376, + 266, 30003, 266, 21106, 266, 21942, 266, 37197, 770, 12308, 26412, 12309, + 770, 12308, 19977, 12309, 770, 12308, 20108, 12309, 770, 12308, 23433, + 12309, 770, 12308, 28857, 12309, 770, 12308, 25171, 12309, 770, 12308, + 30423, 12309, 770, 12308, 21213, 12309, 770, 12308, 25943, 12309, 263, + 24471, 263, 21487, 256, 20029, 256, 20024, 256, 20033, 256, 131362, 256, + 20320, 256, 20411, 256, 20482, 256, 20602, 256, 20633, 256, 20687, 256, + 13470, 256, 132666, 256, 20820, 256, 20836, 256, 20855, 256, 132380, 256, + 13497, 256, 20839, 256, 20877, 256, 132427, 256, 20887, 256, 20900, 256, + 20172, 256, 20908, 256, 168415, 256, 20981, 256, 20995, 256, 13535, 256, + 21051, 256, 21062, 256, 21106, 256, 21111, 256, 13589, 256, 21253, 256, + 21254, 256, 21321, 256, 21338, 256, 21363, 256, 21373, 256, 21375, 256, + 133676, 256, 28784, 256, 21450, 256, 21471, 256, 133987, 256, 21483, 256, + 21489, 256, 21510, 256, 21662, 256, 21560, 256, 21576, 256, 21608, 256, + 21666, 256, 21750, 256, 21776, 256, 21843, 256, 21859, 256, 21892, 256, + 21931, 256, 21939, 256, 21954, 256, 22294, 256, 22295, 256, 22097, 256, + 22132, 256, 22766, 256, 22478, 256, 22516, 256, 22541, 256, 22411, 256, + 22578, 256, 22577, 256, 22700, 256, 136420, 256, 22770, 256, 22775, 256, + 22790, 256, 22810, 256, 22818, 256, 22882, 256, 136872, 256, 136938, 256, + 23020, 256, 23067, 256, 23079, 256, 23000, 256, 23142, 256, 14062, 256, + 14076, 256, 23304, 256, 23358, 256, 137672, 256, 23491, 256, 23512, 256, + 23539, 256, 138008, 256, 23551, 256, 23558, 256, 24403, 256, 23586, 256, + 14209, 256, 23648, 256, 23744, 256, 23693, 256, 138724, 256, 23875, 256, + 138726, 256, 23918, 256, 23915, 256, 23932, 256, 24033, 256, 24034, 256, + 14383, 256, 24061, 256, 24104, 256, 24125, 256, 24169, 256, 14434, 256, + 139651, 256, 14460, 256, 24240, 256, 24243, 256, 24246, 256, 172946, 256, + 24318, 256, 140081, 256, 33281, 256, 24354, 256, 14535, 256, 144056, 256, + 156122, 256, 24418, 256, 24427, 256, 14563, 256, 24474, 256, 24525, 256, + 24535, 256, 24569, 256, 24705, 256, 14650, 256, 14620, 256, 141012, 256, + 24775, 256, 24904, 256, 24908, 256, 24954, 256, 25010, 256, 24996, 256, + 25007, 256, 25054, 256, 25104, 256, 25115, 256, 25181, 256, 25265, 256, + 25300, 256, 25424, 256, 142092, 256, 25405, 256, 25340, 256, 25448, 256, + 25475, 256, 25572, 256, 142321, 256, 25634, 256, 25541, 256, 25513, 256, + 14894, 256, 25705, 256, 25726, 256, 25757, 256, 25719, 256, 14956, 256, + 25964, 256, 143370, 256, 26083, 256, 26360, 256, 26185, 256, 15129, 256, + 15112, 256, 15076, 256, 20882, 256, 20885, 256, 26368, 256, 26268, 256, + 32941, 256, 17369, 256, 26401, 256, 26462, 256, 26451, 256, 144323, 256, + 15177, 256, 26618, 256, 26501, 256, 26706, 256, 144493, 256, 26766, 256, + 26655, 256, 26900, 256, 26946, 256, 27043, 256, 27114, 256, 27304, 256, + 145059, 256, 27355, 256, 15384, 256, 27425, 256, 145575, 256, 27476, 256, + 15438, 256, 27506, 256, 27551, 256, 27579, 256, 146061, 256, 138507, 256, + 146170, 256, 27726, 256, 146620, 256, 27839, 256, 27853, 256, 27751, 256, + 27926, 256, 27966, 256, 28009, 256, 28024, 256, 28037, 256, 146718, 256, + 27956, 256, 28207, 256, 28270, 256, 15667, 256, 28359, 256, 147153, 256, + 28153, 256, 28526, 256, 147294, 256, 147342, 256, 28614, 256, 28729, 256, + 28699, 256, 15766, 256, 28746, 256, 28797, 256, 28791, 256, 28845, 256, + 132389, 256, 28997, 256, 148067, 256, 29084, 256, 148395, 256, 29224, + 256, 29264, 256, 149000, 256, 29312, 256, 29333, 256, 149301, 256, + 149524, 256, 29562, 256, 29579, 256, 16044, 256, 29605, 256, 16056, 256, + 29767, 256, 29788, 256, 29829, 256, 29898, 256, 16155, 256, 29988, 256, + 150582, 256, 30014, 256, 150674, 256, 139679, 256, 30224, 256, 151457, + 256, 151480, 256, 151620, 256, 16380, 256, 16392, 256, 151795, 256, + 151794, 256, 151833, 256, 151859, 256, 30494, 256, 30495, 256, 30603, + 256, 16454, 256, 16534, 256, 152605, 256, 30798, 256, 16611, 256, 153126, + 256, 153242, 256, 153285, 256, 31211, 256, 16687, 256, 31306, 256, 31311, + 256, 153980, 256, 154279, 256, 31470, 256, 16898, 256, 154539, 256, + 31686, 256, 31689, 256, 16935, 256, 154752, 256, 31954, 256, 17056, 256, + 31976, 256, 31971, 256, 32000, 256, 155526, 256, 32099, 256, 17153, 256, + 32199, 256, 32258, 256, 32325, 256, 17204, 256, 156200, 256, 156231, 256, + 17241, 256, 156377, 256, 32634, 256, 156478, 256, 32661, 256, 32762, 256, + 156890, 256, 156963, 256, 32864, 256, 157096, 256, 32880, 256, 144223, + 256, 17365, 256, 32946, 256, 33027, 256, 17419, 256, 33086, 256, 23221, + 256, 157607, 256, 157621, 256, 144275, 256, 144284, 256, 33284, 256, + 36766, 256, 17515, 256, 33425, 256, 33419, 256, 33437, 256, 21171, 256, + 33457, 256, 33459, 256, 33469, 256, 33510, 256, 158524, 256, 33565, 256, + 33635, 256, 33709, 256, 33571, 256, 33725, 256, 33767, 256, 33619, 256, + 33738, 256, 33740, 256, 33756, 256, 158774, 256, 159083, 256, 158933, + 256, 17707, 256, 34033, 256, 34035, 256, 34070, 256, 160714, 256, 34148, + 256, 159532, 256, 17757, 256, 17761, 256, 159665, 256, 159954, 256, + 17771, 256, 34384, 256, 34407, 256, 34409, 256, 34473, 256, 34440, 256, + 34574, 256, 34530, 256, 34600, 256, 34667, 256, 34694, 256, 17879, 256, + 34785, 256, 34817, 256, 17913, 256, 34912, 256, 34915, 256, 161383, 256, + 35031, 256, 35038, 256, 17973, 256, 35066, 256, 13499, 256, 161966, 256, + 162150, 256, 18110, 256, 18119, 256, 35488, 256, 35925, 256, 162984, 256, + 36011, 256, 36033, 256, 36123, 256, 36215, 256, 163631, 256, 133124, 256, + 36299, 256, 36284, 256, 36336, 256, 133342, 256, 36564, 256, 165330, 256, + 165357, 256, 37012, 256, 37105, 256, 37137, 256, 165678, 256, 37147, 256, + 37432, 256, 37591, 256, 37592, 256, 37500, 256, 37881, 256, 37909, 256, + 166906, 256, 38283, 256, 18837, 256, 38327, 256, 167287, 256, 18918, 256, + 38595, 256, 23986, 256, 38691, 256, 168261, 256, 168474, 256, 19054, 256, + 19062, 256, 38880, 256, 168970, 256, 19122, 256, 169110, 256, 38953, 256, + 169398, 256, 39138, 256, 19251, 256, 39209, 256, 39335, 256, 39362, 256, + 39422, 256, 19406, 256, 170800, 256, 40000, 256, 40189, 256, 19662, 256, + 19693, 256, 40295, 256, 172238, 256, 19704, 256, 172293, 256, 172558, + 256, 172689, 256, 40635, 256, 19798, 256, 40697, 256, 40702, 256, 40709, + 256, 40719, 256, 40726, 256, 40763, 256, 173568, }; /* index tables for the decomposition data */ @@ -4380,15 +4467,16 @@ static const unsigned char decomp_index1[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, - 74, 75, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 79, 0, 0, 0, 80, 0, 0, 81, 0, - 82, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 74, 75, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 78, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 80, 0, 0, 0, 81, 82, 0, 83, 0, + 84, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4396,12 +4484,12 @@ static const unsigned char decomp_index1[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 89, 90, 0, 0, 0, 0, 91, 92, + 93, 94, 95, 96, 97, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 100, 101, 0, 0, 0, 0, 102, 103, 104, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 85, 0, 0, 0, 0, 86, 87, - 88, 89, 90, 91, 92, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 95, 96, 0, 0, 0, 0, 97, 98, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4421,8 +4509,8 @@ static const unsigned char decomp_index1[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 107, 108, 109, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 102, 103, 104, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4720,8 +4808,7 @@ static const unsigned char decomp_index1[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; static const unsigned short decomp_index2[] = { @@ -5295,690 +5382,763 @@ static const unsigned short decomp_index2[] = { 10263, 10265, 10267, 10269, 10271, 0, 0, 10273, 10275, 10277, 10279, 10281, 10283, 0, 0, 10285, 10287, 10289, 0, 0, 0, 10291, 10293, 10295, 10297, 10299, 10301, 10303, 0, 10305, 10307, 10309, 10311, 10313, 10315, - 10317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10319, - 10321, 10323, 10325, 10327, 0, 10329, 10331, 10333, 10335, 10337, 10339, - 10341, 10343, 10345, 10347, 10349, 10351, 10353, 10355, 10357, 10359, - 10361, 10363, 10365, 10367, 10369, 10371, 10373, 10375, 10377, 10379, - 10381, 10383, 10385, 10387, 10389, 10391, 10393, 10395, 10397, 10399, - 10401, 10403, 10405, 10407, 10409, 10411, 0, 10413, 10415, 10417, 10419, - 10421, 10423, 10425, 10427, 10429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10431, 0, 10434, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 10437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 10325, 10327, 10329, 10331, 10333, 0, 10335, 10337, + 10339, 10341, 10343, 10345, 10347, 10349, 10351, 10353, 10355, 10357, + 10359, 10361, 10363, 10365, 10367, 10369, 10371, 10373, 10375, 10377, + 10379, 10381, 10383, 10385, 10387, 10389, 10391, 10393, 10395, 10397, + 10399, 10401, 10403, 10405, 10407, 10409, 10411, 10413, 10415, 10417, 0, + 10419, 10421, 10423, 10425, 10427, 10429, 10431, 10433, 10435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10437, 0, + 10440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10440, 10443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 10446, 10449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 10446, 10449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10452, 10455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 10452, 10455, 0, 10458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 10458, 0, 10461, 0, 0, 0, 0, 0, 0, 0, 0, 10464, 0, 0, + 10467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 10470, 0, 10473, 10476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10461, 10464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 10479, 10482, 0, 10485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10488, 10491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10470, 10473, - 10476, 10479, 10482, 10485, 10488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 10491, 10494, 10497, 10500, 10503, 10506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10497, 10500, 10503, 10506, 10509, + 10512, 10515, 10518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, - 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, - 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, - 10539, 3210, 3214, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, - 10549, 10551, 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, - 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, 10513, 10515, - 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, 10523, 10525, - 10527, 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, 10539, 3210, - 0, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, 10551, - 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, 10509, 3253, - 3192, 3289, 3257, 3259, 10511, 3212, 3218, 10513, 10515, 3220, 3261, - 3224, 10517, 3229, 3231, 3233, 10519, 10521, 10523, 10525, 10527, 10529, - 10531, 3245, 10533, 10535, 10537, 3291, 3255, 10539, 3210, 3214, 3273, - 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, 10551, 10553, 10555, - 10557, 10559, 10561, 10563, 10565, 10567, 10509, 0, 3192, 3289, 0, 0, - 10511, 0, 0, 10513, 10515, 0, 0, 3224, 10517, 3229, 3231, 0, 10519, - 10521, 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, 10537, - 3291, 0, 10539, 0, 3214, 3273, 3293, 10541, 3222, 10543, 10545, 0, 10547, - 10549, 10551, 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, - 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, 10513, 10515, - 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, 10523, 10525, - 10527, 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, 10539, 3210, - 3214, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, 10551, - 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, 10509, 3253, 0, - 3289, 3257, 3259, 10511, 0, 0, 10513, 10515, 3220, 3261, 3224, 10517, - 3229, 3231, 0, 10519, 10521, 10523, 10525, 10527, 10529, 10531, 0, 10533, - 10535, 10537, 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, 3222, - 10543, 10545, 3263, 10547, 10549, 10551, 10553, 10555, 10557, 10559, - 10561, 10563, 10565, 10567, 10509, 3253, 0, 3289, 3257, 3259, 10511, 0, - 3218, 10513, 10515, 3220, 3261, 0, 10517, 0, 0, 0, 10519, 10521, 10523, - 10525, 10527, 10529, 10531, 0, 10533, 10535, 10537, 3291, 3255, 10539, - 3210, 3214, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, - 10551, 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, 10509, - 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, 10513, 10515, 3220, - 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, 10523, 10525, 10527, - 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, 10539, 3210, 3214, - 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, 10551, 10553, - 10555, 10557, 10559, 10561, 10563, 10565, 10567, 10509, 3253, 3192, 3289, - 3257, 3259, 10511, 3212, 3218, 10513, 10515, 3220, 3261, 3224, 10517, - 3229, 3231, 3233, 10519, 10521, 10523, 10525, 10527, 10529, 10531, 3245, - 10533, 10535, 10537, 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, - 3222, 10543, 10545, 3263, 10547, 10549, 10551, 10553, 10555, 10557, - 10559, 10561, 10563, 10565, 10567, 10509, 3253, 3192, 3289, 3257, 3259, - 10511, 3212, 3218, 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, - 3233, 10519, 10521, 10523, 10525, 10527, 10529, 10531, 3245, 10533, - 10535, 10537, 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, 3222, - 10543, 10545, 3263, 10547, 10549, 10551, 10553, 10555, 10557, 10559, - 10561, 10563, 10565, 10567, 10509, 3253, 3192, 3289, 3257, 3259, 10511, - 3212, 3218, 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, 3233, - 10519, 10521, 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, - 10537, 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, 3222, 10543, - 10545, 3263, 10547, 10549, 10551, 10553, 10555, 10557, 10559, 10561, - 10563, 10565, 10567, 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, - 3218, 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, - 10521, 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, 10537, - 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, 3222, 10543, 10545, - 3263, 10547, 10549, 10551, 10553, 10555, 10557, 10559, 10561, 10563, - 10565, 10567, 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, - 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, - 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, - 10539, 3210, 3214, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, - 10549, 10551, 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, - 10569, 10571, 0, 0, 10573, 10575, 3283, 10577, 10579, 10581, 10583, - 10585, 10587, 10589, 10591, 10593, 10595, 10597, 10599, 3285, 10601, - 10603, 10605, 10607, 10609, 10611, 10613, 10615, 10617, 10619, 10621, - 10623, 3281, 10625, 10627, 10629, 10631, 10633, 10635, 10637, 10639, - 10641, 10643, 10645, 10647, 3279, 10649, 10651, 10653, 10655, 10657, - 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, - 10679, 10573, 10575, 3283, 10577, 10579, 10581, 10583, 10585, 10587, - 10589, 10591, 10593, 10595, 10597, 10599, 3285, 10601, 10603, 10605, - 10607, 10609, 10611, 10613, 10615, 10617, 10619, 10621, 10623, 3281, - 10625, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, - 10645, 10647, 3279, 10649, 10651, 10653, 10655, 10657, 10659, 10661, - 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, 10679, 10573, - 10575, 3283, 10577, 10579, 10581, 10583, 10585, 10587, 10589, 10591, - 10593, 10595, 10597, 10599, 3285, 10601, 10603, 10605, 10607, 10609, - 10611, 10613, 10615, 10617, 10619, 10621, 10623, 3281, 10625, 10627, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 10521, 10524, 10527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, + 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, + 10548, 10550, 10552, 3245, 10554, 10556, 10558, 10560, 10562, 10564, + 10566, 10568, 10570, 10572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10574, 10577, 10580, 10583, 10586, + 10589, 10592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10595, 10598, + 10601, 10604, 10607, 10610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, + 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, + 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, + 3214, 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, + 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, + 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, + 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, + 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, 0, 3273, 3293, + 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, + 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, + 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, + 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, + 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, + 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, + 10641, 10643, 10645, 10647, 10530, 0, 3192, 3289, 0, 0, 10532, 0, 0, + 10534, 10536, 0, 0, 3224, 10538, 3229, 3231, 0, 10540, 10542, 10544, + 10546, 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 0, 10619, 0, + 3214, 3273, 3293, 10621, 3222, 10623, 10625, 0, 10627, 10629, 10631, + 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, + 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, + 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, + 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, + 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, + 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, 0, 3289, 3257, + 3259, 10532, 0, 0, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 0, + 10540, 10542, 10544, 10546, 10548, 10550, 10552, 0, 10613, 10615, 10617, + 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, 10625, + 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, + 10645, 10647, 10530, 3253, 0, 3289, 3257, 3259, 10532, 0, 3218, 10534, + 10536, 3220, 3261, 0, 10538, 0, 0, 0, 10540, 10542, 10544, 10546, 10548, + 10550, 10552, 0, 10613, 10615, 10617, 3291, 3255, 10619, 3210, 3214, + 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, + 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, + 3257, 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, + 3229, 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, + 10613, 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, + 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, + 10639, 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, + 10532, 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, + 3233, 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, + 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, + 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, + 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, 10532, + 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 3233, + 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, 10615, + 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, + 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, + 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, + 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, + 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, 10615, 10617, + 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, 10625, + 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, + 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, + 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, + 10544, 10546, 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 3255, + 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, - 3279, 10649, 10651, 10653, 10655, 10657, 10659, 10661, 10663, 10665, - 10667, 10669, 10671, 10673, 10675, 10677, 10679, 10573, 10575, 3283, - 10577, 10579, 10581, 10583, 10585, 10587, 10589, 10591, 10593, 10595, - 10597, 10599, 3285, 10601, 10603, 10605, 10607, 10609, 10611, 10613, - 10615, 10617, 10619, 10621, 10623, 3281, 10625, 10627, 10629, 10631, - 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 3279, 10649, - 10651, 10653, 10655, 10657, 10659, 10661, 10663, 10665, 10667, 10669, - 10671, 10673, 10675, 10677, 10679, 10573, 10575, 3283, 10577, 10579, - 10581, 10583, 10585, 10587, 10589, 10591, 10593, 10595, 10597, 10599, - 3285, 10601, 10603, 10605, 10607, 10609, 10611, 10613, 10615, 10617, - 10619, 10621, 10623, 3281, 10625, 10627, 10629, 10631, 10633, 10635, - 10637, 10639, 10641, 10643, 10645, 10647, 3279, 10649, 10651, 10653, - 10655, 10657, 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, - 10675, 10677, 10679, 10681, 10683, 0, 0, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10705, 10707, 10709, 10711, - 10713, 10715, 10717, 10719, 10721, 10723, 10725, 10727, 10729, 10731, + 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, + 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, + 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, + 3214, 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, + 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10649, 10651, 0, + 0, 10653, 10655, 3283, 10657, 10659, 10661, 10663, 10665, 10667, 10669, + 10671, 10673, 10675, 10677, 10679, 3285, 10681, 10683, 10685, 10687, + 10689, 10691, 10693, 10695, 10697, 10699, 10701, 10703, 3281, 10705, + 10707, 10709, 10711, 10713, 10715, 10717, 10719, 10721, 10723, 10725, + 10727, 3279, 10729, 10731, 10733, 10735, 10737, 10739, 10741, 10743, + 10745, 10747, 10749, 10751, 10753, 10755, 10757, 10759, 10653, 10655, + 3283, 10657, 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, + 10675, 10677, 10679, 3285, 10681, 10683, 10685, 10687, 10689, 10691, + 10693, 10695, 10697, 10699, 10701, 10703, 3281, 10705, 10707, 10709, + 10711, 10713, 10715, 10717, 10719, 10721, 10723, 10725, 10727, 3279, + 10729, 10731, 10733, 10735, 10737, 10739, 10741, 10743, 10745, 10747, + 10749, 10751, 10753, 10755, 10757, 10759, 10653, 10655, 3283, 10657, + 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, + 10679, 3285, 10681, 10683, 10685, 10687, 10689, 10691, 10693, 10695, + 10697, 10699, 10701, 10703, 3281, 10705, 10707, 10709, 10711, 10713, + 10715, 10717, 10719, 10721, 10723, 10725, 10727, 3279, 10729, 10731, 10733, 10735, 10737, 10739, 10741, 10743, 10745, 10747, 10749, 10751, - 10753, 10755, 10757, 10759, 10761, 10763, 10765, 10767, 10769, 10771, - 10773, 10775, 10777, 10779, 10781, 10783, 10785, 10787, 10789, 10791, - 10793, 10795, 10797, 10799, 10801, 10803, 10805, 10807, 10809, 10811, - 10813, 10815, 10817, 10819, 10821, 10823, 10825, 10827, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10829, 10831, 10833, 10835, 0, 10837, + 10753, 10755, 10757, 10759, 10653, 10655, 3283, 10657, 10659, 10661, + 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, 10679, 3285, + 10681, 10683, 10685, 10687, 10689, 10691, 10693, 10695, 10697, 10699, + 10701, 10703, 3281, 10705, 10707, 10709, 10711, 10713, 10715, 10717, + 10719, 10721, 10723, 10725, 10727, 3279, 10729, 10731, 10733, 10735, + 10737, 10739, 10741, 10743, 10745, 10747, 10749, 10751, 10753, 10755, + 10757, 10759, 10653, 10655, 3283, 10657, 10659, 10661, 10663, 10665, + 10667, 10669, 10671, 10673, 10675, 10677, 10679, 3285, 10681, 10683, + 10685, 10687, 10689, 10691, 10693, 10695, 10697, 10699, 10701, 10703, + 3281, 10705, 10707, 10709, 10711, 10713, 10715, 10717, 10719, 10721, + 10723, 10725, 10727, 3279, 10729, 10731, 10733, 10735, 10737, 10739, + 10741, 10743, 10745, 10747, 10749, 10751, 10753, 10755, 10757, 10759, + 10761, 10763, 0, 0, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 10765, 10767, 10769, 10771, 10773, 10775, 10777, + 10779, 10781, 10783, 10785, 10787, 10789, 10791, 10793, 10795, 10797, + 10799, 10801, 10803, 10805, 10807, 10809, 10811, 10813, 10815, 10817, + 10819, 10821, 10823, 10825, 10827, 10829, 10831, 10833, 10835, 10837, 10839, 10841, 10843, 10845, 10847, 10849, 10851, 10853, 10855, 10857, 10859, 10861, 10863, 10865, 10867, 10869, 10871, 10873, 10875, 10877, - 10879, 10881, 10883, 10885, 10887, 10889, 0, 10831, 10833, 0, 10891, 0, - 0, 10841, 0, 10845, 10847, 10849, 10851, 10853, 10855, 10857, 10859, - 10861, 10863, 0, 10867, 10869, 10871, 10873, 0, 10877, 0, 10881, 0, 0, 0, - 0, 0, 0, 10833, 0, 0, 0, 0, 10841, 0, 10845, 0, 10849, 0, 10853, 10855, - 10857, 0, 10861, 10863, 0, 10867, 0, 0, 10873, 0, 10877, 0, 10881, 0, - 10885, 0, 10889, 0, 10831, 10833, 0, 10891, 0, 0, 10841, 10843, 10845, - 10847, 0, 10851, 10853, 10855, 10857, 10859, 10861, 10863, 0, 10867, - 10869, 10871, 10873, 0, 10877, 10879, 10881, 10883, 0, 10887, 0, 10829, - 10831, 10833, 10835, 10891, 10837, 10839, 10841, 10843, 10845, 0, 10849, - 10851, 10853, 10855, 10857, 10859, 10861, 10863, 10865, 10867, 10869, - 10871, 10873, 10875, 10877, 10879, 10881, 0, 0, 0, 0, 0, 10831, 10833, - 10835, 0, 10837, 10839, 10841, 10843, 10845, 0, 10849, 10851, 10853, - 10855, 10857, 10859, 10861, 10863, 10865, 10867, 10869, 10871, 10873, - 10875, 10877, 10879, 10881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 10893, 10896, 10899, 10902, 10905, 10908, 10911, 10914, - 10917, 10920, 10923, 0, 0, 0, 0, 0, 10926, 10930, 10934, 10938, 10942, - 10946, 10950, 10954, 10958, 10962, 10966, 10970, 10974, 10978, 10982, - 10986, 10990, 10994, 10998, 11002, 11006, 11010, 11014, 11018, 11022, - 11026, 11030, 3926, 3956, 11034, 11037, 0, 11040, 11042, 11044, 11046, - 11048, 11050, 11052, 11054, 11056, 11058, 11060, 11062, 11064, 11066, - 11068, 11070, 11072, 11074, 11076, 11078, 11080, 11082, 11084, 11086, - 11088, 11090, 11092, 6348, 11095, 11098, 11101, 11105, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11108, 11111, - 11114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11117, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 11120, 11123, 11126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 11128, 11130, 11132, 11134, 11136, 11138, 11140, 11142, 11144, - 11146, 11148, 11150, 11152, 11154, 11156, 11158, 11160, 11162, 11164, - 11166, 11168, 11170, 11172, 11174, 11176, 11178, 11180, 11182, 11184, - 11186, 11188, 11190, 11192, 11194, 11196, 11198, 11200, 11202, 11204, - 11206, 11208, 11210, 11212, 11214, 0, 0, 0, 0, 11216, 11220, 11224, - 11228, 11232, 11236, 11240, 11244, 11248, 0, 0, 0, 0, 0, 0, 0, 11252, - 11254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10685, 10687, 10689, - 10691, 10693, 10695, 10697, 10699, 10701, 10703, 0, 0, 0, 0, 0, 0, 11256, - 11258, 11260, 11262, 11264, 7195, 11266, 11268, 11270, 11272, 7197, - 11274, 11276, 11278, 7199, 11280, 11282, 11284, 11286, 11288, 11290, - 11292, 11294, 11296, 11298, 11300, 11302, 7315, 11304, 11306, 11308, - 11310, 11312, 11314, 11316, 11318, 11320, 7325, 7201, 7203, 7327, 11322, - 11324, 6817, 11326, 7205, 11328, 11330, 11332, 11334, 11334, 11334, - 11336, 11338, 11340, 11342, 11344, 11346, 11348, 11350, 11352, 11354, - 11356, 11358, 11360, 11362, 11364, 11366, 11368, 11370, 11370, 7331, - 11372, 11374, 11376, 11378, 7209, 11380, 11382, 11384, 7123, 11386, - 11388, 11390, 11392, 11394, 11396, 11398, 11400, 11402, 11404, 11406, - 11408, 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424, 11426, - 11428, 11430, 11432, 11434, 11436, 11436, 11438, 11440, 11442, 6809, - 11444, 11446, 11448, 11450, 11452, 11454, 11456, 11458, 7219, 11460, - 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476, 11478, 11480, - 11482, 11484, 11486, 11488, 11490, 11492, 11494, 11496, 11498, 11500, - 6701, 11502, 11504, 11506, 11506, 11508, 11510, 11510, 11512, 11514, - 11516, 11518, 11520, 11522, 11524, 11526, 11528, 11530, 11532, 11534, - 11536, 7221, 11538, 11540, 11542, 11544, 7355, 11544, 11546, 7225, 11548, - 11550, 11552, 11554, 7227, 6647, 11556, 11558, 11560, 11562, 11564, - 11566, 11568, 11570, 11572, 11574, 11576, 11578, 11580, 11582, 11584, - 11586, 11588, 11590, 11592, 11594, 11596, 11598, 7229, 11600, 11602, - 11604, 11606, 11608, 11610, 7233, 11612, 11614, 11616, 11618, 11620, - 11622, 11624, 11626, 6703, 7371, 11628, 11630, 11632, 11634, 11636, - 11638, 11640, 11642, 7235, 11644, 11646, 11648, 11650, 7457, 11652, - 11654, 11656, 11658, 11660, 11662, 11664, 11666, 11668, 11670, 11672, - 11674, 11676, 6843, 11678, 11680, 11682, 11684, 11686, 11688, 11690, - 11692, 11694, 11696, 11698, 7237, 7017, 11700, 11702, 11704, 11706, - 11708, 11710, 11712, 11714, 7379, 11716, 11718, 11720, 11722, 11724, - 11726, 11728, 11730, 7381, 11732, 11734, 11736, 11738, 11740, 11742, - 11744, 11746, 11748, 11750, 11752, 11754, 7385, 11756, 11758, 11760, - 11762, 11764, 11766, 11768, 11770, 11772, 11774, 11776, 11776, 11778, - 11780, 7389, 11782, 11784, 11786, 11788, 11790, 11792, 11794, 6815, - 11796, 11798, 11800, 11802, 11804, 11806, 11808, 7401, 11810, 11812, - 11814, 11816, 11818, 11820, 11820, 7403, 7461, 11822, 11824, 11826, - 11828, 11830, 6739, 7407, 11832, 11834, 7259, 11836, 11838, 7167, 11840, - 11842, 7267, 11844, 11846, 11848, 11850, 11850, 11852, 11854, 11856, - 11858, 11860, 11862, 11864, 11866, 11868, 11870, 11872, 11874, 11876, - 11878, 11880, 11882, 11884, 11886, 11888, 11890, 11892, 11894, 11896, - 11898, 11900, 11902, 11904, 7279, 11906, 11908, 11910, 11912, 11914, - 11916, 11918, 11920, 11922, 11924, 11926, 11928, 11930, 11932, 11934, - 11936, 11508, 11938, 11940, 11942, 11944, 11946, 11948, 11950, 11952, - 11954, 11956, 11958, 11960, 6851, 11962, 11964, 11966, 11968, 11970, - 11972, 7285, 11974, 11976, 11978, 11980, 11982, 11984, 11986, 11988, - 11990, 11992, 11994, 11996, 11998, 12000, 12002, 12004, 12006, 12008, - 12010, 12012, 6729, 12014, 12016, 12018, 12020, 12022, 12024, 7421, - 12026, 12028, 12030, 12032, 12034, 12036, 12038, 12040, 12042, 12044, - 12046, 12048, 12050, 12052, 12054, 12056, 12058, 12060, 12062, 12064, - 7431, 7433, 12066, 12068, 12070, 12072, 12074, 12076, 12078, 12080, - 12082, 12084, 12086, 12088, 12090, 7435, 12092, 12094, 12096, 12098, - 12100, 12102, 12104, 12106, 12108, 12110, 12112, 12114, 12116, 12118, - 12120, 12122, 12124, 12126, 12128, 12130, 12132, 12134, 12136, 12138, - 12140, 12142, 12144, 12146, 12148, 12150, 7447, 7447, 12152, 12154, - 12156, 12158, 12160, 12162, 12164, 12166, 12168, 12170, 7449, 12172, - 12174, 12176, 12178, 12180, 12182, 12184, 12186, 12188, 12190, 12192, - 12194, 12196, 12198, 12200, 12202, 12204, 12206, 12208, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10879, 10881, 10883, 10885, 10887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 10889, 10891, 10893, 10895, 0, 10897, 10899, 10901, 10903, + 10905, 10907, 10909, 10911, 10913, 10915, 10917, 10919, 10921, 10923, + 10925, 10927, 10929, 10931, 10933, 10935, 10937, 10939, 10941, 10943, + 10945, 10947, 10949, 0, 10891, 10893, 0, 10951, 0, 0, 10901, 0, 10905, + 10907, 10909, 10911, 10913, 10915, 10917, 10919, 10921, 10923, 0, 10927, + 10929, 10931, 10933, 0, 10937, 0, 10941, 0, 0, 0, 0, 0, 0, 10893, 0, 0, + 0, 0, 10901, 0, 10905, 0, 10909, 0, 10913, 10915, 10917, 0, 10921, 10923, + 0, 10927, 0, 0, 10933, 0, 10937, 0, 10941, 0, 10945, 0, 10949, 0, 10891, + 10893, 0, 10951, 0, 0, 10901, 10903, 10905, 10907, 0, 10911, 10913, + 10915, 10917, 10919, 10921, 10923, 0, 10927, 10929, 10931, 10933, 0, + 10937, 10939, 10941, 10943, 0, 10947, 0, 10889, 10891, 10893, 10895, + 10951, 10897, 10899, 10901, 10903, 10905, 0, 10909, 10911, 10913, 10915, + 10917, 10919, 10921, 10923, 10925, 10927, 10929, 10931, 10933, 10935, + 10937, 10939, 10941, 0, 0, 0, 0, 0, 10891, 10893, 10895, 0, 10897, 10899, + 10901, 10903, 10905, 0, 10909, 10911, 10913, 10915, 10917, 10919, 10921, + 10923, 10925, 10927, 10929, 10931, 10933, 10935, 10937, 10939, 10941, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10953, 10956, + 10959, 10962, 10965, 10968, 10971, 10974, 10977, 10980, 10983, 0, 0, 0, + 0, 0, 10986, 10990, 10994, 10998, 11002, 11006, 11010, 11014, 11018, + 11022, 11026, 11030, 11034, 11038, 11042, 11046, 11050, 11054, 11058, + 11062, 11066, 11070, 11074, 11078, 11082, 11086, 11090, 3926, 3956, + 11094, 11097, 0, 11100, 11102, 11104, 11106, 11108, 11110, 11112, 11114, + 11116, 11118, 11120, 11122, 11124, 11126, 11128, 11130, 11132, 11134, + 11136, 11138, 11140, 11142, 11144, 11146, 11148, 11150, 11152, 6348, + 11155, 11158, 11161, 11165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11168, 11171, 11174, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 11177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11180, + 11183, 11186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11188, 11190, 11192, + 11194, 11196, 11198, 11200, 11202, 11204, 11206, 11208, 11210, 11212, + 11214, 11216, 11218, 11220, 11222, 11224, 11226, 11228, 11230, 11232, + 11234, 11236, 11238, 11240, 11242, 11244, 11246, 11248, 11250, 11252, + 11254, 11256, 11258, 11260, 11262, 11264, 11266, 11268, 11270, 11272, + 11274, 0, 0, 0, 0, 11276, 11280, 11284, 11288, 11292, 11296, 11300, + 11304, 11308, 0, 0, 0, 0, 0, 0, 0, 11312, 11314, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 10554, 10556, 10558, 10560, 10562, 10564, 10566, 10568, + 10570, 10572, 0, 0, 0, 0, 0, 0, 11316, 11318, 11320, 11322, 11324, 7195, + 11326, 11328, 11330, 11332, 7197, 11334, 11336, 11338, 7199, 11340, + 11342, 11344, 11346, 11348, 11350, 11352, 11354, 11356, 11358, 11360, + 11362, 7315, 11364, 11366, 11368, 11370, 11372, 11374, 11376, 11378, + 11380, 7325, 7201, 7203, 7327, 11382, 11384, 6817, 11386, 7205, 11388, + 11390, 11392, 11394, 11394, 11394, 11396, 11398, 11400, 11402, 11404, + 11406, 11408, 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424, + 11426, 11428, 11430, 11430, 7331, 11432, 11434, 11436, 11438, 7209, + 11440, 11442, 11444, 7123, 11446, 11448, 11450, 11452, 11454, 11456, + 11458, 11460, 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476, + 11478, 11480, 11482, 11484, 11486, 11488, 11490, 11492, 11494, 11496, + 11496, 11498, 11500, 11502, 6809, 11504, 11506, 11508, 11510, 11512, + 11514, 11516, 11518, 7219, 11520, 11522, 11524, 11526, 11528, 11530, + 11532, 11534, 11536, 11538, 11540, 11542, 11544, 11546, 11548, 11550, + 11552, 11554, 11556, 11558, 11560, 6701, 11562, 11564, 11566, 11566, + 11568, 11570, 11570, 11572, 11574, 11576, 11578, 11580, 11582, 11584, + 11586, 11588, 11590, 11592, 11594, 11596, 7221, 11598, 11600, 11602, + 11604, 7355, 11604, 11606, 7225, 11608, 11610, 11612, 11614, 7227, 6647, + 11616, 11618, 11620, 11622, 11624, 11626, 11628, 11630, 11632, 11634, + 11636, 11638, 11640, 11642, 11644, 11646, 11648, 11650, 11652, 11654, + 11656, 11658, 7229, 11660, 11662, 11664, 11666, 11668, 11670, 7233, + 11672, 11674, 11676, 11678, 11680, 11682, 11684, 11686, 6703, 7371, + 11688, 11690, 11692, 11694, 11696, 11698, 11700, 11702, 7235, 11704, + 11706, 11708, 11710, 7457, 11712, 11714, 11716, 11718, 11720, 11722, + 11724, 11726, 11728, 11730, 11732, 11734, 11736, 6843, 11738, 11740, + 11742, 11744, 11746, 11748, 11750, 11752, 11754, 11756, 11758, 7237, + 7017, 11760, 11762, 11764, 11766, 11768, 11770, 11772, 11774, 7379, + 11776, 11778, 11780, 11782, 11784, 11786, 11788, 11790, 7381, 11792, + 11794, 11796, 11798, 11800, 11802, 11804, 11806, 11808, 11810, 11812, + 11814, 7385, 11816, 11818, 11820, 11822, 11824, 11826, 11828, 11830, + 11832, 11834, 11836, 11836, 11838, 11840, 7389, 11842, 11844, 11846, + 11848, 11850, 11852, 11854, 6815, 11856, 11858, 11860, 11862, 11864, + 11866, 11868, 7401, 11870, 11872, 11874, 11876, 11878, 11880, 11880, + 7403, 7461, 11882, 11884, 11886, 11888, 11890, 6739, 7407, 11892, 11894, + 7259, 11896, 11898, 7167, 11900, 11902, 7267, 11904, 11906, 11908, 11910, + 11910, 11912, 11914, 11916, 11918, 11920, 11922, 11924, 11926, 11928, + 11930, 11932, 11934, 11936, 11938, 11940, 11942, 11944, 11946, 11948, + 11950, 11952, 11954, 11956, 11958, 11960, 11962, 11964, 7279, 11966, + 11968, 11970, 11972, 11974, 11976, 11978, 11980, 11982, 11984, 11986, + 11988, 11990, 11992, 11994, 11996, 11568, 11998, 12000, 12002, 12004, + 12006, 12008, 12010, 12012, 12014, 12016, 12018, 12020, 6851, 12022, + 12024, 12026, 12028, 12030, 12032, 7285, 12034, 12036, 12038, 12040, + 12042, 12044, 12046, 12048, 12050, 12052, 12054, 12056, 12058, 12060, + 12062, 12064, 12066, 12068, 12070, 12072, 6729, 12074, 12076, 12078, + 12080, 12082, 12084, 7421, 12086, 12088, 12090, 12092, 12094, 12096, + 12098, 12100, 12102, 12104, 12106, 12108, 12110, 12112, 12114, 12116, + 12118, 12120, 12122, 12124, 7431, 7433, 12126, 12128, 12130, 12132, + 12134, 12136, 12138, 12140, 12142, 12144, 12146, 12148, 12150, 7435, + 12152, 12154, 12156, 12158, 12160, 12162, 12164, 12166, 12168, 12170, + 12172, 12174, 12176, 12178, 12180, 12182, 12184, 12186, 12188, 12190, + 12192, 12194, 12196, 12198, 12200, 12202, 12204, 12206, 12208, 12210, + 7447, 7447, 12212, 12214, 12216, 12218, 12220, 12222, 12224, 12226, + 12228, 12230, 7449, 12232, 12234, 12236, 12238, 12240, 12242, 12244, + 12246, 12248, 12250, 12252, 12254, 12256, 12258, 12260, 12262, 12264, + 12266, 12268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, }; /* NFC pairs */ #define COMP_SHIFT 2 static const unsigned short comp_index[] = { - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, - 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 11, 0, 12, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 15, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 17, 18, 19, 20, 21, 22, 0, 0, 0, 0, 0, 0, 0, 0, 23, 24, 25, 26, 27, - 28, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 31, 32, 33, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 36, - 37, 38, 39, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 42, 43, 44, 45, 46, 47, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 50, 0, 51, 52, 53, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 55, 56, 57, - 58, 59, 0, 0, 0, 0, 0, 0, 0, 0, 60, 61, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 63, 64, 65, 0, 66, 67, 68, 0, 0, 0, 0, 0, 0, 0, 0, 69, 70, 71, - 72, 73, 74, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 77, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 78, 79, 0, 80, 81, 82, 83, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 84, 85, 86, 0, 87, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 90, 0, 91, - 92, 93, 0, 0, 0, 0, 0, 0, 0, 0, 94, 95, 96, 97, 98, 99, 100, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 101, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, - 104, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 107, 108, 109, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 111, 112, 0, 113, 114, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 117, - 118, 119, 120, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 123, 0, - 124, 0, 0, 0, 0, 0, 0, 0, 0, 125, 126, 127, 128, 0, 129, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 130, 0, 131, 132, 133, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 135, 136, 137, 138, 139, 140, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 144, 145, 146, 0, 147, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 148, 149, 150, 151, 152, 153, 154, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 155, 156, 157, 158, 159, 160, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 162, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 165, 166, - 167, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 170, 171, 172, 173, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 174, 175, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 178, 179, 180, 0, 181, 182, 183, 0, 0, 0, 0, 0, 0, 0, 0, 184, 185, - 186, 187, 188, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 191, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 193, 194, 195, 196, 197, 198, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 199, 200, 201, 0, 202, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 204, 205, 206, 207, 208, 209, 0, 0, 0, 0, 0, 0, 0, 0, 210, 211, - 212, 213, 214, 215, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 218, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 220, 221, 222, 0, 223, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 226, - 227, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 230, 231, 0, 232, 0, - 233, 0, 0, 0, 0, 0, 0, 0, 0, 234, 235, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, - 0, 0, 0, 237, 238, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 245, 246, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, - 249, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 252, 253, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 257, 0, 258, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 260, 261, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, - 267, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 270, 271, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 273, 274, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 278, 279, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 282, - 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 285, 286, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 288, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 301, 302, 303, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 306, - 307, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 310, 311, 0, 312, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 314, 0, 315, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 327, - 0, 328, 329, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 332, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 333, 334, 0, 0, 335, 0, 0, 0, 336, 0, 0, 0, 0, 0, - 0, 0, 337, 338, 339, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, - 342, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 345, 346, 347, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 349, 0, 0, 0, 350, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, - 0, 0, 354, 355, 356, 0, 357, 0, 0, 358, 359, 0, 0, 0, 0, 0, 0, 0, 360, 0, - 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 363, 364, 0, 0, - 365, 0, 0, 0, 0, 0, 0, 0, 0, 366, 367, 0, 368, 0, 0, 0, 369, 0, 0, 0, 0, - 0, 0, 0, 370, 371, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 375, 376, 377, 378, 0, 0, - 379, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 381, 0, 0, 0, 382, 0, 0, 0, 0, 0, - 0, 0, 383, 384, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 0, 0, - 0, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, - 0, 0, 0, 0, 0, 389, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 393, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 395, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 399, 400, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 10, 0, 0, 11, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 15, 0, + 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 18, 0, 19, 20, 21, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 23, 24, 25, 26, 27, 28, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 30, 31, 32, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 35, 36, 0, + 37, 38, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 41, 42, 43, 44, 45, 46, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 48, 0, 49, 0, 50, 51, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 53, 0, 54, 0, 55, 56, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 59, 0, 0, + 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 62, 63, 0, 64, 65, 66, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 68, 69, 70, 71, 72, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 73, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 75, 76, 77, 78, 79, 80, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 83, 84, + 0, 85, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 88, 0, 89, 90, 91, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 93, 94, 95, 96, 97, 98, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 101, 102, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 106, 107, 0, 108, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 110, 111, 0, 112, 0, 113, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 115, 116, 117, 118, 119, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 121, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 123, 124, 125, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 127, 128, 0, 129, 130, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 133, + 134, 135, 136, 137, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 141, 142, 0, 0, 143, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 145, 146, 0, 147, 148, 149, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 150, 151, 152, 153, 154, 155, 156, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 157, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 159, 0, 160, 0, 161, 162, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, + 0, 165, 0, 166, 167, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 170, 0, + 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 173, 174, 0, 175, + 176, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 179, 180, 181, 182, 183, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 186, 187, 188, 189, 190, 191, 192, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 193, 194, 195, 0, 196, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 198, 199, 0, 200, 201, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 203, 204, 205, 206, 207, 208, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, + 0, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 213, 214, 0, + 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 218, 219, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 221, 222, 223, 0, 224, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 226, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, + 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 232, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 236, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 240, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 245, 246, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 252, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 254, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 258, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 260, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, + 264, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 267, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 269, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 283, 0, 284, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 286, 0, 287, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, + 290, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 293, 0, 294, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 305, 306, 0, 307, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 309, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 312, + 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 315, 0, 316, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 320, 321, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, + 324, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 328, 329, 0, 330, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 332, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, + 0, 335, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 338, 0, 339, 0, + 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 342, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 344, 345, 0, 346, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 348, 0, 0, 349, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, + 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 354, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 356, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 361, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 366, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, + 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 381, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 384, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 397, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 404, 405, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 411, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 413, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 417, 418, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 423, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 456, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 0, 470, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 477, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, - 0, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 0, 0, - 0, 0, 0, 0, 0, 493, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 505, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 509, 0, 0, 0, 0, 0, - 0, 0, 0, 510, 0, 0, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, - 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 0, 0, 0, 0, 521, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, - 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, - 530, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 532, 533, 0, 0, 0, 0, 0, - 534, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 0, 0, - 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, - 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 544, 0, 0, 0, 0, 0, 0, 545, 0, - 0, 0, 0, 0, 0, 0, 546, 547, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, - 549, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, - 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 558, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, - 0, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 0, 574, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 577, 578, 0, 0, 0, 0, 0, 579, + 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 409, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 414, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 419, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 440, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 442, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 0, 0, + 0, 0, 0, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, + 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 458, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, + 0, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 474, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 0, 0, 0, 0, 0, + 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 495, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 498, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, + 0, 0, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 0, 0, 0, + 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 509, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 0, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 0, 0, 0, 0, 521, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 0, + 0, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 538, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 645, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 661, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 666, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, + 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 631, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 643, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 648, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 655, }; static const unsigned int comp_data[] = { - 0, 0, 0, 0, 0, 0, 0, 8814, 0, 0, 8800, 0, 0, 8815, 0, 0, 0, 192, 193, - 194, 195, 256, 258, 550, 196, 7842, 197, 0, 461, 512, 514, 0, 0, 0, 7840, - 0, 7680, 0, 0, 260, 0, 0, 7682, 0, 0, 7684, 0, 0, 0, 0, 7686, 0, 262, - 264, 0, 0, 0, 266, 0, 0, 0, 0, 268, 0, 199, 0, 0, 0, 7690, 0, 0, 0, 0, - 270, 0, 0, 0, 0, 0, 7692, 0, 0, 0, 7696, 0, 7698, 0, 0, 7694, 0, 0, 0, 0, - 200, 201, 202, 7868, 274, 276, 278, 203, 7866, 0, 0, 282, 516, 518, 0, 0, - 0, 7864, 0, 0, 0, 552, 280, 7704, 0, 7706, 0, 0, 0, 7710, 0, 500, 284, 0, - 7712, 286, 288, 0, 0, 0, 0, 486, 0, 290, 0, 0, 0, 292, 0, 0, 0, 7714, - 7718, 0, 0, 0, 542, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 7720, 0, 0, 7722, 0, 0, - 204, 205, 206, 296, 298, 300, 304, 207, 7880, 0, 0, 463, 520, 522, 0, 0, - 0, 7882, 0, 0, 0, 0, 302, 0, 0, 7724, 0, 0, 0, 308, 0, 7728, 0, 0, 0, 0, - 0, 488, 0, 7730, 0, 0, 0, 310, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 313, 0, - 317, 0, 0, 0, 0, 0, 7734, 0, 0, 0, 315, 0, 7740, 0, 0, 7738, 0, 0, 0, 0, - 0, 7742, 0, 0, 0, 0, 7744, 0, 0, 7746, 0, 504, 323, 0, 209, 0, 0, 7748, - 0, 0, 0, 0, 327, 0, 7750, 0, 0, 0, 325, 0, 7754, 0, 0, 7752, 0, 0, 0, 0, - 210, 211, 212, 213, 332, 334, 558, 214, 7886, 0, 336, 465, 524, 526, 0, - 0, 416, 7884, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 7764, 7766, 0, 0, 0, 0, 0, - 340, 0, 0, 0, 0, 7768, 344, 528, 530, 0, 0, 0, 7770, 0, 0, 0, 342, 0, 0, - 0, 0, 7774, 0, 346, 348, 0, 0, 0, 7776, 0, 0, 0, 0, 352, 0, 7778, 0, 0, - 536, 350, 0, 0, 0, 7786, 0, 0, 0, 0, 356, 0, 7788, 0, 0, 538, 354, 0, - 7792, 0, 0, 7790, 0, 0, 0, 0, 217, 218, 219, 360, 362, 364, 0, 220, 7910, - 366, 368, 467, 532, 534, 0, 0, 431, 7908, 7794, 0, 0, 0, 370, 7798, 0, - 7796, 7804, 0, 0, 0, 0, 0, 7806, 0, 7808, 7810, 372, 0, 0, 0, 7814, 7812, - 0, 7816, 0, 0, 0, 7818, 7820, 0, 0, 0, 7922, 221, 374, 7928, 562, 0, - 7822, 376, 7926, 0, 0, 0, 0, 7924, 0, 0, 377, 7824, 0, 0, 0, 379, 381, 0, - 0, 0, 0, 0, 7826, 0, 0, 0, 0, 7828, 224, 225, 226, 227, 257, 259, 551, - 228, 7843, 229, 0, 462, 513, 515, 0, 0, 0, 7841, 0, 7681, 0, 0, 261, 0, - 0, 7683, 0, 0, 7685, 0, 0, 0, 0, 7687, 0, 0, 0, 0, 0, 263, 265, 0, 0, 0, - 267, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 231, 0, 0, 0, 7691, 271, 0, 0, 0, 0, - 0, 7693, 0, 0, 0, 7697, 0, 7699, 0, 0, 7695, 232, 233, 234, 7869, 275, - 277, 279, 235, 7867, 0, 0, 283, 517, 519, 0, 0, 0, 7865, 0, 0, 0, 553, - 281, 7705, 0, 7707, 0, 0, 0, 7711, 0, 0, 0, 0, 0, 501, 285, 0, 7713, 287, - 289, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 291, 0, 0, 0, 293, 0, 0, 0, 7715, - 7719, 0, 0, 0, 543, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 7721, 0, 0, 7723, 0, - 7830, 236, 237, 238, 297, 299, 301, 0, 239, 7881, 0, 0, 464, 521, 523, 0, - 0, 0, 7883, 0, 0, 0, 0, 303, 0, 0, 7725, 0, 0, 0, 309, 0, 0, 0, 0, 496, - 0, 0, 0, 0, 7729, 0, 489, 0, 0, 0, 0, 0, 7731, 0, 0, 0, 311, 7733, 0, 0, - 0, 0, 0, 314, 0, 318, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 316, 0, 7741, 0, 0, - 7739, 0, 7743, 0, 0, 0, 0, 7745, 0, 0, 7747, 0, 0, 0, 0, 0, 505, 324, 0, - 241, 0, 0, 7749, 0, 0, 0, 0, 328, 0, 7751, 0, 0, 0, 326, 0, 7755, 0, 0, - 7753, 0, 0, 0, 0, 242, 243, 244, 245, 333, 335, 559, 246, 7887, 0, 337, - 466, 525, 527, 0, 0, 417, 7885, 491, 0, 0, 0, 0, 0, 7765, 0, 0, 0, 0, - 7767, 0, 341, 0, 0, 0, 0, 7769, 0, 0, 0, 0, 345, 529, 531, 0, 0, 0, 7771, - 0, 0, 0, 343, 0, 0, 0, 0, 7775, 0, 347, 349, 0, 0, 0, 7777, 0, 0, 0, 0, - 353, 0, 7779, 0, 0, 537, 351, 0, 0, 0, 7787, 7831, 0, 0, 0, 357, 0, 0, 0, - 0, 0, 7789, 0, 0, 539, 355, 0, 7793, 0, 0, 7791, 0, 0, 0, 0, 249, 250, - 251, 361, 363, 365, 0, 252, 7911, 367, 369, 468, 533, 535, 0, 0, 432, - 7909, 7795, 0, 0, 0, 371, 7799, 0, 7797, 0, 0, 0, 0, 7805, 0, 7807, 0, 0, - 0, 0, 0, 7809, 7811, 373, 0, 0, 0, 7815, 7813, 0, 7832, 0, 0, 0, 7817, 0, - 0, 0, 7819, 7821, 0, 0, 0, 7923, 253, 375, 7929, 563, 0, 7823, 255, 7927, - 7833, 0, 0, 0, 7925, 0, 0, 378, 7825, 0, 0, 0, 380, 0, 0, 0, 0, 382, 0, - 7827, 0, 0, 0, 0, 7829, 0, 0, 0, 0, 8173, 901, 0, 0, 0, 0, 0, 0, 8129, 0, - 0, 7846, 7844, 0, 7850, 0, 0, 0, 0, 7848, 0, 0, 478, 0, 0, 0, 506, 0, 0, - 508, 0, 0, 482, 0, 0, 0, 7688, 0, 7872, 7870, 0, 7876, 0, 0, 0, 0, 7874, - 0, 0, 0, 7726, 0, 0, 0, 0, 0, 7890, 7888, 0, 7894, 0, 0, 0, 0, 7892, 0, - 0, 0, 7756, 0, 0, 556, 0, 0, 7758, 0, 0, 0, 554, 0, 0, 0, 510, 0, 0, 0, - 0, 0, 475, 471, 0, 0, 469, 0, 0, 473, 0, 0, 0, 7847, 7845, 0, 7851, 0, 0, - 0, 0, 7849, 0, 0, 479, 0, 0, 0, 507, 0, 0, 509, 0, 0, 483, 0, 0, 0, 7689, - 0, 7873, 7871, 0, 7877, 0, 0, 0, 0, 7875, 0, 0, 0, 7727, 0, 0, 0, 0, 0, - 7891, 7889, 0, 7895, 0, 0, 0, 0, 7893, 0, 0, 0, 7757, 0, 0, 557, 0, 0, - 7759, 0, 0, 0, 555, 0, 0, 0, 511, 0, 0, 0, 0, 0, 476, 472, 0, 0, 470, 0, - 0, 474, 0, 0, 0, 7856, 7854, 0, 7860, 0, 0, 0, 0, 7858, 0, 0, 7857, 7855, - 0, 7861, 0, 0, 0, 0, 7859, 0, 0, 7700, 7702, 0, 0, 0, 0, 0, 7701, 7703, - 0, 0, 0, 0, 0, 7760, 7762, 0, 7761, 7763, 0, 0, 0, 7780, 0, 0, 7781, 0, - 0, 7782, 0, 0, 0, 0, 0, 0, 7783, 0, 7800, 0, 0, 7801, 0, 0, 0, 0, 7802, - 0, 0, 7803, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 7900, 7898, 0, 7904, 0, 0, - 0, 0, 7902, 7906, 0, 0, 0, 0, 0, 7901, 7899, 0, 7905, 0, 0, 0, 0, 7903, - 0, 0, 0, 0, 7907, 0, 7914, 7912, 0, 7918, 0, 0, 0, 0, 7916, 0, 0, 0, 0, - 7920, 0, 7915, 7913, 0, 7919, 7917, 0, 0, 0, 0, 7921, 0, 0, 0, 0, 494, 0, - 0, 0, 492, 0, 0, 493, 0, 0, 480, 0, 0, 0, 0, 0, 0, 481, 0, 0, 0, 7708, 0, - 0, 7709, 0, 560, 0, 0, 0, 0, 0, 0, 561, 0, 495, 0, 0, 0, 8122, 902, 0, 0, - 8121, 8120, 0, 0, 0, 0, 7944, 7945, 0, 0, 0, 0, 0, 8124, 0, 8136, 904, 0, - 0, 0, 0, 7960, 7961, 0, 0, 0, 8138, 905, 0, 0, 0, 0, 7976, 7977, 0, 8140, - 0, 0, 0, 0, 0, 8154, 906, 0, 0, 8153, 8152, 0, 938, 0, 0, 7992, 7993, 0, - 0, 0, 8184, 908, 0, 0, 0, 0, 8008, 8009, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, - 8170, 910, 0, 0, 8169, 8168, 0, 939, 0, 0, 0, 8025, 0, 0, 0, 8186, 911, - 8040, 8041, 0, 0, 0, 0, 0, 8188, 0, 0, 8116, 0, 0, 8132, 0, 0, 0, 0, 0, - 8048, 940, 0, 0, 8113, 8112, 0, 0, 0, 0, 7936, 7937, 0, 0, 0, 0, 8118, - 8115, 0, 0, 0, 0, 0, 8050, 941, 7952, 7953, 0, 0, 0, 8052, 942, 0, 0, 0, - 0, 7968, 7969, 0, 0, 0, 0, 8134, 8131, 0, 8054, 943, 0, 0, 8145, 8144, 0, - 970, 0, 0, 7984, 7985, 8150, 0, 0, 0, 0, 0, 0, 8056, 972, 0, 0, 0, 0, - 8000, 8001, 0, 8164, 8165, 0, 0, 0, 8058, 973, 0, 0, 8161, 8160, 0, 971, - 0, 0, 0, 0, 0, 0, 8016, 8017, 0, 0, 0, 0, 8166, 0, 0, 8060, 974, 0, 0, 0, - 0, 8032, 8033, 8182, 8179, 0, 0, 0, 0, 0, 8146, 912, 0, 0, 0, 0, 0, 0, - 8151, 0, 0, 8162, 944, 0, 0, 8167, 0, 0, 0, 8180, 0, 0, 979, 0, 0, 0, 0, - 0, 980, 0, 0, 1031, 0, 0, 0, 0, 1232, 0, 1234, 0, 0, 0, 0, 1027, 0, 1024, - 0, 0, 0, 0, 1238, 0, 1025, 1217, 0, 1244, 0, 0, 1246, 0, 0, 0, 1037, 0, - 0, 0, 1250, 1049, 0, 1252, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 1254, 0, 0, 0, - 1262, 1038, 0, 1264, 0, 0, 1266, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 0, 1272, - 0, 0, 1260, 0, 0, 0, 0, 1233, 0, 1235, 0, 0, 0, 0, 1107, 0, 1104, 0, 0, - 0, 0, 1239, 0, 1105, 1218, 0, 1245, 0, 0, 1247, 0, 0, 0, 1117, 0, 0, 0, - 1251, 1081, 0, 1253, 0, 0, 0, 0, 1116, 0, 0, 0, 0, 1255, 0, 0, 0, 1263, - 1118, 0, 1265, 0, 0, 1267, 0, 0, 0, 1269, 0, 0, 0, 0, 0, 0, 1273, 0, 0, - 1261, 0, 0, 1111, 0, 0, 0, 1142, 0, 0, 1143, 0, 0, 0, 0, 0, 1242, 0, 0, - 1243, 0, 0, 1258, 0, 0, 0, 0, 0, 0, 1259, 0, 1570, 1571, 1573, 0, 1572, - 0, 0, 1574, 0, 0, 0, 0, 0, 0, 1730, 0, 0, 1747, 0, 0, 1728, 0, 0, 0, 0, - 2345, 0, 0, 2353, 0, 0, 2356, 0, 0, 0, 2507, 2508, 0, 0, 0, 2891, 2888, - 2892, 2964, 0, 0, 0, 0, 0, 3018, 3020, 0, 3019, 0, 0, 0, 0, 3144, 0, 0, - 0, 0, 3264, 0, 3274, 3271, 3272, 0, 3275, 0, 0, 0, 0, 3402, 3404, 0, - 3403, 0, 0, 0, 0, 3546, 3548, 3550, 0, 0, 0, 0, 3549, 0, 0, 0, 0, 0, - 4134, 0, 0, 0, 6918, 0, 0, 6920, 0, 0, 6922, 0, 0, 6924, 0, 0, 0, 0, 0, - 0, 6926, 0, 0, 6930, 0, 0, 6971, 0, 0, 6973, 0, 0, 0, 0, 0, 0, 6976, 0, - 0, 6977, 0, 0, 6979, 0, 0, 0, 7736, 0, 0, 7737, 0, 0, 0, 0, 0, 0, 7772, - 0, 0, 7773, 0, 0, 0, 0, 7784, 0, 0, 7785, 0, 0, 7852, 0, 0, 7862, 0, 0, - 0, 7853, 0, 0, 7863, 0, 0, 0, 7878, 0, 0, 7879, 0, 0, 7896, 0, 0, 7897, - 0, 0, 0, 0, 7938, 7940, 0, 0, 7942, 8064, 0, 7939, 7941, 0, 0, 7943, - 8065, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 8068, 0, 0, 8069, 0, 0, - 8070, 0, 0, 0, 0, 0, 0, 8071, 0, 7946, 7948, 0, 0, 7950, 8072, 0, 7947, - 7949, 0, 0, 7951, 8073, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 8076, - 0, 0, 8077, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 8079, 0, 7954, 7956, 0, 7955, - 7957, 0, 0, 0, 0, 0, 7962, 7964, 0, 0, 0, 0, 0, 7963, 7965, 0, 7970, - 7972, 0, 0, 7974, 8080, 0, 7971, 7973, 0, 0, 7975, 8081, 0, 0, 8082, 0, - 0, 0, 0, 0, 0, 8083, 0, 0, 8084, 0, 0, 8085, 0, 0, 8086, 0, 0, 0, 0, 0, - 0, 8087, 0, 7978, 7980, 0, 0, 7982, 8088, 0, 7979, 7981, 0, 0, 7983, - 8089, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 8092, 0, 0, 8093, 0, 0, - 8094, 0, 0, 0, 0, 0, 0, 8095, 0, 7986, 7988, 0, 0, 7990, 0, 0, 7987, - 7989, 0, 0, 7991, 0, 0, 0, 0, 0, 0, 7994, 7996, 0, 0, 0, 0, 0, 0, 7998, - 0, 0, 7995, 7997, 0, 0, 7999, 0, 0, 8002, 8004, 0, 8003, 8005, 0, 0, 0, - 0, 0, 8010, 8012, 0, 0, 0, 0, 0, 8011, 8013, 0, 8018, 8020, 0, 0, 8022, - 0, 0, 8019, 8021, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 8027, 8029, 0, 0, 0, 0, - 0, 0, 8031, 0, 0, 8034, 8036, 0, 0, 8038, 8096, 0, 8035, 8037, 0, 0, - 8039, 8097, 0, 0, 8098, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 8101, - 0, 0, 8102, 0, 0, 8103, 0, 0, 0, 0, 0, 8042, 8044, 0, 0, 8046, 8104, 0, - 8043, 8045, 0, 0, 8047, 8105, 0, 0, 8106, 0, 0, 8107, 0, 0, 0, 0, 0, 0, - 8108, 0, 0, 8109, 0, 0, 8110, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 8114, 0, 0, - 8130, 0, 0, 8178, 0, 0, 8119, 0, 0, 0, 0, 0, 8141, 8142, 0, 0, 8143, 0, - 0, 0, 8135, 0, 0, 8183, 0, 0, 0, 0, 0, 8157, 8158, 0, 0, 0, 0, 0, 0, - 8159, 0, 8602, 0, 0, 8603, 0, 0, 0, 0, 0, 0, 8622, 0, 0, 8653, 0, 0, - 8655, 0, 0, 8654, 0, 0, 0, 0, 0, 0, 8708, 0, 0, 8713, 0, 0, 8716, 0, 0, - 8740, 0, 0, 0, 0, 0, 0, 8742, 0, 0, 8769, 0, 0, 8772, 0, 0, 8775, 0, 0, - 0, 0, 0, 0, 8777, 0, 0, 8813, 0, 0, 8802, 0, 0, 8816, 0, 0, 0, 0, 0, 0, - 8817, 0, 0, 8820, 0, 0, 8821, 0, 0, 8824, 0, 0, 0, 0, 0, 0, 8825, 0, 0, - 8832, 0, 0, 8833, 0, 0, 8928, 0, 0, 0, 0, 0, 0, 8929, 0, 0, 8836, 0, 0, - 8837, 0, 0, 8840, 0, 0, 0, 0, 0, 0, 8841, 0, 0, 8930, 0, 0, 8931, 0, 0, - 8876, 0, 0, 0, 0, 0, 0, 8877, 0, 0, 8878, 0, 0, 8879, 0, 0, 8938, 0, 0, - 0, 0, 0, 0, 8939, 0, 0, 8940, 0, 0, 8941, 0, 0, 0, 12436, 0, 0, 12364, 0, - 0, 0, 0, 0, 0, 12366, 0, 0, 12368, 0, 0, 12370, 0, 0, 12372, 0, 0, 0, 0, - 0, 0, 12374, 0, 0, 12376, 0, 0, 12378, 0, 0, 12380, 0, 0, 0, 0, 0, 0, - 12382, 0, 0, 12384, 0, 0, 12386, 0, 0, 12389, 0, 0, 0, 0, 0, 0, 12391, 0, - 0, 12393, 0, 0, 12400, 12401, 0, 12403, 12404, 0, 0, 0, 0, 0, 12406, - 12407, 0, 0, 0, 0, 0, 12409, 12410, 0, 12412, 12413, 0, 12446, 0, 0, 0, - 0, 0, 0, 12532, 0, 0, 12460, 0, 0, 12462, 0, 0, 12464, 0, 0, 0, 0, 0, 0, - 12466, 0, 0, 12468, 0, 0, 12470, 0, 0, 12472, 0, 0, 0, 0, 0, 0, 12474, 0, - 0, 12476, 0, 0, 12478, 0, 0, 12480, 0, 0, 0, 0, 0, 0, 12482, 0, 0, 12485, - 0, 0, 12487, 0, 0, 12489, 0, 0, 0, 0, 0, 0, 12496, 12497, 0, 0, 0, 0, 0, - 12499, 12500, 0, 12502, 12503, 0, 12505, 12506, 0, 0, 0, 0, 0, 12508, - 12509, 0, 0, 0, 0, 0, 12535, 0, 0, 12536, 0, 0, 12537, 0, 0, 0, 0, 0, 0, - 12538, 0, 0, 12542, 0, 0, 0, 0, 69786, 0, 0, 69788, 0, 0, 69803, 0, 0, 0, - 69934, 0, 0, 69935, 0, 0, 0, 70475, 70476, 0, 0, 0, 70844, 70843, 70846, - 0, 0, 0, 71098, 0, 0, 71099, 0, 0, 0, 71992, + 0, 0, 0, 0, 0, 0, 0, 8814, 0, 0, 0, 8800, 0, 0, 0, 8815, 192, 193, 194, + 195, 256, 258, 550, 196, 7842, 197, 0, 461, 512, 514, 0, 0, 0, 7840, 0, + 7680, 0, 0, 260, 0, 0, 0, 7682, 0, 0, 7684, 0, 0, 0, 0, 7686, 0, 0, 262, + 264, 0, 0, 0, 266, 0, 0, 0, 0, 268, 0, 199, 0, 0, 0, 0, 7690, 0, 0, 0, 0, + 270, 0, 7692, 0, 0, 0, 7696, 0, 7698, 0, 0, 7694, 0, 200, 201, 202, 7868, + 274, 276, 278, 203, 7866, 0, 0, 282, 516, 518, 0, 0, 0, 7864, 0, 0, 0, + 552, 280, 7704, 0, 7706, 0, 0, 0, 0, 7710, 0, 0, 500, 284, 0, 7712, 286, + 288, 0, 0, 0, 0, 486, 0, 290, 0, 0, 0, 0, 292, 0, 0, 0, 7714, 7718, 0, 0, + 0, 542, 0, 7716, 0, 0, 0, 7720, 0, 0, 7722, 0, 0, 0, 204, 205, 206, 296, + 298, 300, 304, 207, 7880, 0, 0, 463, 520, 522, 0, 0, 0, 7882, 0, 0, 0, 0, + 302, 0, 0, 7724, 0, 0, 0, 0, 308, 0, 0, 7728, 0, 0, 0, 0, 0, 488, 0, + 7730, 0, 0, 0, 310, 0, 0, 0, 0, 7732, 0, 0, 313, 0, 0, 0, 0, 0, 317, 0, + 7734, 0, 0, 0, 315, 0, 7740, 0, 0, 7738, 0, 0, 7742, 0, 0, 0, 0, 7744, 0, + 0, 7746, 0, 0, 504, 323, 0, 209, 0, 0, 7748, 0, 0, 0, 0, 327, 0, 7750, 0, + 0, 0, 325, 0, 7754, 0, 0, 7752, 0, 210, 211, 212, 213, 332, 334, 558, + 214, 7886, 0, 336, 465, 524, 526, 0, 0, 416, 7884, 0, 0, 0, 0, 490, 0, 0, + 7764, 0, 0, 0, 0, 7766, 0, 0, 340, 0, 0, 0, 0, 7768, 0, 0, 0, 0, 344, + 528, 530, 0, 0, 0, 7770, 0, 0, 0, 342, 0, 0, 0, 0, 7774, 0, 0, 346, 348, + 0, 0, 0, 7776, 0, 0, 0, 0, 352, 0, 7778, 0, 0, 536, 350, 0, 0, 0, 0, + 7786, 0, 0, 0, 0, 356, 0, 7788, 0, 0, 538, 354, 0, 7792, 0, 0, 7790, 0, + 217, 218, 219, 360, 362, 364, 0, 220, 7910, 366, 368, 467, 532, 534, 0, + 0, 431, 7908, 7794, 0, 0, 0, 370, 7798, 0, 7796, 0, 0, 0, 0, 0, 7804, 0, + 7806, 0, 0, 7808, 7810, 372, 0, 0, 0, 7814, 7812, 0, 7816, 0, 0, 0, 0, + 7818, 7820, 7922, 221, 374, 7928, 562, 0, 7822, 376, 7926, 0, 0, 0, 0, + 7924, 0, 0, 0, 377, 7824, 0, 0, 0, 379, 0, 0, 0, 0, 381, 0, 7826, 0, 0, + 0, 0, 7828, 0, 224, 225, 226, 227, 257, 259, 551, 228, 7843, 229, 0, 462, + 513, 515, 0, 0, 0, 7841, 0, 7681, 0, 0, 261, 0, 0, 0, 7683, 0, 0, 7685, + 0, 0, 0, 0, 7687, 0, 0, 263, 265, 0, 0, 0, 267, 0, 0, 0, 0, 269, 0, 231, + 0, 0, 0, 0, 7691, 0, 0, 0, 0, 271, 0, 7693, 0, 0, 0, 7697, 0, 7699, 0, 0, + 7695, 0, 232, 233, 234, 7869, 275, 277, 279, 235, 7867, 0, 0, 283, 517, + 519, 0, 0, 0, 7865, 0, 0, 0, 553, 281, 7705, 0, 7707, 0, 0, 0, 0, 7711, + 0, 0, 501, 285, 0, 7713, 287, 289, 0, 0, 0, 0, 487, 0, 291, 0, 0, 0, 0, + 293, 0, 0, 0, 7715, 7719, 0, 0, 0, 543, 0, 7717, 0, 0, 0, 7721, 0, 0, + 7723, 0, 7830, 0, 236, 237, 238, 297, 299, 301, 0, 239, 7881, 0, 0, 464, + 521, 523, 0, 0, 0, 7883, 0, 0, 0, 0, 303, 0, 0, 7725, 0, 0, 0, 0, 309, 0, + 0, 0, 0, 496, 0, 7729, 0, 0, 0, 0, 0, 489, 0, 7731, 0, 0, 0, 311, 0, 0, + 0, 0, 7733, 0, 0, 314, 0, 0, 0, 0, 0, 318, 0, 7735, 0, 0, 0, 316, 0, + 7741, 0, 0, 7739, 0, 0, 7743, 0, 0, 0, 0, 7745, 0, 0, 7747, 0, 0, 505, + 324, 0, 241, 0, 0, 7749, 0, 0, 0, 0, 328, 0, 7751, 0, 0, 0, 326, 0, 7755, + 0, 0, 7753, 0, 242, 243, 244, 245, 333, 335, 559, 246, 7887, 0, 337, 466, + 525, 527, 0, 0, 417, 7885, 0, 0, 0, 0, 491, 0, 0, 7765, 0, 0, 0, 0, 7767, + 0, 0, 341, 0, 0, 0, 0, 7769, 0, 0, 0, 0, 345, 529, 531, 0, 0, 0, 7771, 0, + 0, 0, 343, 0, 0, 0, 0, 7775, 0, 0, 347, 349, 0, 0, 0, 7777, 0, 0, 0, 0, + 353, 0, 7779, 0, 0, 537, 351, 0, 0, 0, 0, 7787, 7831, 0, 0, 0, 357, 0, + 7789, 0, 0, 539, 355, 0, 7793, 0, 0, 7791, 0, 249, 250, 251, 361, 363, + 365, 0, 252, 7911, 367, 369, 468, 533, 535, 0, 0, 432, 7909, 7795, 0, 0, + 0, 371, 7799, 0, 7797, 0, 0, 0, 0, 0, 7805, 0, 7807, 0, 0, 7809, 7811, + 373, 0, 0, 0, 7815, 7813, 0, 7832, 0, 0, 0, 7817, 0, 0, 0, 0, 7819, 7821, + 7923, 253, 375, 7929, 563, 0, 7823, 255, 7927, 7833, 0, 0, 0, 7925, 0, 0, + 0, 378, 7825, 0, 0, 0, 380, 0, 0, 0, 0, 382, 0, 7827, 0, 0, 0, 0, 7829, + 0, 8173, 901, 0, 0, 8129, 0, 0, 0, 7846, 7844, 0, 7850, 7848, 0, 0, 0, + 478, 0, 0, 0, 0, 506, 0, 0, 0, 508, 0, 0, 482, 0, 0, 0, 0, 7688, 0, 0, + 7872, 7870, 0, 7876, 7874, 0, 0, 0, 0, 7726, 0, 0, 7890, 7888, 0, 7894, + 7892, 0, 0, 0, 0, 7756, 0, 0, 556, 0, 0, 7758, 554, 0, 0, 0, 0, 510, 0, + 0, 475, 471, 0, 0, 469, 0, 0, 0, 0, 0, 0, 473, 7847, 7845, 0, 7851, 7849, + 0, 0, 0, 479, 0, 0, 0, 0, 507, 0, 0, 0, 509, 0, 0, 483, 0, 0, 0, 0, 7689, + 0, 0, 7873, 7871, 0, 7877, 7875, 0, 0, 0, 0, 7727, 0, 0, 7891, 7889, 0, + 7895, 7893, 0, 0, 0, 0, 7757, 0, 0, 557, 0, 0, 7759, 555, 0, 0, 0, 0, + 511, 0, 0, 476, 472, 0, 0, 470, 0, 0, 0, 0, 0, 0, 474, 7856, 7854, 0, + 7860, 7858, 0, 0, 0, 7857, 7855, 0, 7861, 7859, 0, 0, 0, 7700, 7702, 0, + 0, 7701, 7703, 0, 0, 7760, 7762, 0, 0, 7761, 7763, 0, 0, 0, 0, 7780, 0, + 0, 0, 7781, 0, 0, 0, 7782, 0, 0, 0, 7783, 0, 0, 7800, 0, 0, 0, 7801, 0, + 0, 0, 0, 0, 7802, 0, 0, 0, 7803, 0, 0, 7835, 0, 7900, 7898, 0, 7904, + 7902, 0, 0, 0, 0, 7906, 0, 0, 7901, 7899, 0, 7905, 7903, 0, 0, 0, 0, + 7907, 0, 0, 7914, 7912, 0, 7918, 7916, 0, 0, 0, 0, 7920, 0, 0, 7915, + 7913, 0, 7919, 7917, 0, 0, 0, 0, 7921, 0, 0, 0, 0, 0, 494, 492, 0, 0, 0, + 493, 0, 0, 0, 480, 0, 0, 0, 481, 0, 0, 0, 0, 7708, 0, 0, 0, 7709, 0, 0, + 560, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 495, 8122, 902, 0, 0, 8121, 8120, 0, + 0, 0, 0, 7944, 7945, 0, 8124, 0, 0, 8136, 904, 0, 0, 0, 0, 7960, 7961, + 8138, 905, 0, 0, 0, 0, 7976, 7977, 0, 8140, 0, 0, 8154, 906, 0, 0, 8153, + 8152, 0, 938, 0, 0, 7992, 7993, 8184, 908, 0, 0, 0, 0, 8008, 8009, 0, 0, + 0, 8172, 8170, 910, 0, 0, 8169, 8168, 0, 939, 0, 0, 0, 8025, 8186, 911, + 0, 0, 0, 0, 8040, 8041, 0, 8188, 0, 0, 0, 8116, 0, 0, 0, 8132, 0, 0, + 8048, 940, 0, 0, 8113, 8112, 0, 0, 0, 0, 7936, 7937, 8118, 8115, 0, 0, + 8050, 941, 0, 0, 0, 0, 7952, 7953, 8052, 942, 0, 0, 0, 0, 7968, 7969, + 8134, 8131, 0, 0, 8054, 943, 0, 0, 8145, 8144, 0, 970, 0, 0, 7984, 7985, + 8150, 0, 0, 0, 8056, 972, 0, 0, 0, 0, 8000, 8001, 0, 0, 8164, 8165, 8058, + 973, 0, 0, 8161, 8160, 0, 971, 0, 0, 8016, 8017, 8166, 0, 0, 0, 8060, + 974, 0, 0, 0, 0, 8032, 8033, 8182, 8179, 0, 0, 8146, 912, 0, 0, 8151, 0, + 0, 0, 8162, 944, 0, 0, 8167, 0, 0, 0, 0, 8180, 0, 0, 0, 979, 0, 0, 0, 0, + 0, 980, 0, 0, 0, 1031, 0, 1232, 0, 1234, 0, 1027, 0, 0, 1024, 0, 0, 0, 0, + 1238, 0, 1025, 0, 1217, 0, 1244, 0, 0, 0, 1246, 1037, 0, 0, 0, 1250, + 1049, 0, 1252, 0, 1036, 0, 0, 0, 0, 0, 1254, 1262, 1038, 0, 1264, 0, 0, + 1266, 0, 0, 0, 0, 1268, 0, 0, 0, 1272, 0, 0, 0, 1260, 0, 1233, 0, 1235, + 0, 1107, 0, 0, 1104, 0, 0, 0, 0, 1239, 0, 1105, 0, 1218, 0, 1245, 0, 0, + 0, 1247, 1117, 0, 0, 0, 1251, 1081, 0, 1253, 0, 1116, 0, 0, 0, 0, 0, + 1255, 1263, 1118, 0, 1265, 0, 0, 1267, 0, 0, 0, 0, 1269, 0, 0, 0, 1273, + 0, 0, 0, 1261, 0, 0, 0, 1111, 1142, 0, 0, 0, 1143, 0, 0, 0, 0, 0, 0, + 1242, 0, 0, 0, 1243, 0, 0, 0, 1258, 0, 0, 0, 1259, 0, 0, 1570, 1571, + 1573, 0, 0, 0, 0, 0, 0, 1572, 0, 0, 0, 1574, 0, 0, 0, 1730, 0, 0, 0, + 1747, 0, 0, 0, 1728, 0, 2345, 0, 0, 0, 2353, 0, 0, 0, 2356, 0, 0, 0, 0, + 2507, 2508, 2891, 2888, 2892, 0, 2964, 0, 0, 0, 0, 0, 0, 3018, 3020, 0, + 0, 0, 0, 0, 0, 3019, 0, 3144, 0, 0, 0, 0, 0, 3264, 0, 0, 3274, 3271, + 3272, 0, 0, 0, 0, 0, 0, 3275, 0, 3402, 3404, 0, 0, 3403, 0, 0, 0, 0, 0, + 3546, 3548, 3550, 0, 0, 0, 0, 0, 3549, 0, 0, 4134, 0, 0, 0, 0, 6918, 0, + 0, 0, 6920, 0, 0, 0, 6922, 0, 0, 0, 6924, 0, 0, 0, 6926, 0, 0, 0, 6930, + 0, 0, 0, 6971, 0, 0, 0, 6973, 0, 0, 0, 6976, 0, 0, 0, 6977, 0, 0, 0, + 6979, 7736, 0, 0, 0, 7737, 0, 0, 0, 7772, 0, 0, 0, 7773, 0, 0, 0, 0, 0, + 7784, 0, 0, 0, 7785, 0, 0, 0, 7852, 0, 0, 7862, 0, 0, 0, 0, 7853, 0, 0, + 7863, 0, 0, 0, 0, 7878, 0, 0, 0, 7879, 0, 0, 0, 7896, 0, 0, 0, 7897, 0, + 7938, 7940, 0, 0, 7942, 8064, 0, 0, 7939, 7941, 0, 0, 7943, 8065, 0, 0, + 0, 8066, 0, 0, 0, 8067, 0, 0, 0, 8068, 0, 0, 0, 8069, 0, 0, 0, 8070, 0, + 0, 0, 8071, 0, 0, 7946, 7948, 0, 0, 7950, 8072, 0, 0, 7947, 7949, 0, 0, + 7951, 8073, 0, 0, 0, 8074, 0, 0, 0, 8075, 0, 0, 0, 8076, 0, 0, 0, 8077, + 0, 0, 0, 8078, 0, 0, 0, 8079, 0, 0, 7954, 7956, 0, 0, 7955, 7957, 0, 0, + 7962, 7964, 0, 0, 7963, 7965, 0, 0, 7970, 7972, 0, 0, 7974, 8080, 0, 0, + 7971, 7973, 0, 0, 7975, 8081, 0, 0, 0, 8082, 0, 0, 0, 8083, 0, 0, 0, + 8084, 0, 0, 0, 8085, 0, 0, 0, 8086, 0, 0, 0, 8087, 0, 0, 7978, 7980, 0, + 0, 7982, 8088, 0, 0, 7979, 7981, 0, 0, 7983, 8089, 0, 0, 0, 8090, 0, 0, + 0, 8091, 0, 0, 0, 8092, 0, 0, 0, 8093, 0, 0, 0, 8094, 0, 0, 0, 8095, 0, + 0, 7986, 7988, 0, 0, 7990, 0, 0, 0, 7987, 7989, 0, 0, 7991, 0, 0, 0, + 7994, 7996, 0, 0, 7998, 0, 0, 0, 7995, 7997, 0, 0, 7999, 0, 0, 0, 8002, + 8004, 0, 0, 8003, 8005, 0, 0, 8010, 8012, 0, 0, 8011, 8013, 0, 0, 8018, + 8020, 0, 0, 8022, 0, 0, 0, 8019, 8021, 0, 0, 8023, 0, 0, 0, 8027, 8029, + 0, 0, 8031, 0, 0, 0, 8034, 8036, 0, 0, 8038, 8096, 0, 0, 8035, 8037, 0, + 0, 8039, 8097, 0, 0, 0, 8098, 0, 0, 0, 8099, 0, 0, 0, 8100, 0, 0, 0, + 8101, 0, 0, 0, 8102, 0, 0, 0, 8103, 0, 0, 8042, 8044, 0, 0, 8046, 8104, + 0, 0, 8043, 8045, 0, 0, 8047, 8105, 0, 0, 0, 8106, 0, 0, 0, 8107, 0, 0, + 0, 8108, 0, 0, 0, 8109, 0, 0, 0, 8110, 0, 0, 0, 8111, 0, 0, 0, 8114, 0, + 0, 0, 8130, 0, 0, 0, 8178, 0, 0, 0, 8119, 0, 0, 8141, 8142, 0, 0, 8143, + 0, 0, 0, 0, 8135, 0, 0, 0, 8183, 0, 0, 8157, 8158, 0, 0, 8159, 0, 0, 0, + 0, 0, 0, 8602, 0, 0, 0, 8603, 0, 0, 0, 8622, 0, 0, 0, 8653, 0, 0, 0, + 8655, 0, 0, 0, 8654, 0, 0, 0, 8708, 0, 0, 0, 8713, 0, 0, 0, 8716, 0, 0, + 0, 8740, 0, 0, 0, 8742, 0, 0, 0, 8769, 0, 0, 0, 8772, 0, 0, 0, 8775, 0, + 0, 0, 8777, 0, 0, 0, 8813, 0, 0, 0, 8802, 0, 0, 0, 8816, 0, 0, 0, 8817, + 0, 0, 0, 8820, 0, 0, 0, 8821, 0, 0, 0, 8824, 0, 0, 0, 8825, 0, 0, 0, + 8832, 0, 0, 0, 8833, 0, 0, 0, 8928, 0, 0, 0, 8929, 0, 0, 0, 8836, 0, 0, + 0, 8837, 0, 0, 0, 8840, 0, 0, 0, 8841, 0, 0, 0, 8930, 0, 0, 0, 8931, 0, + 0, 0, 8876, 0, 0, 0, 8877, 0, 0, 0, 8878, 0, 0, 0, 8879, 0, 0, 0, 8938, + 0, 0, 0, 8939, 0, 0, 0, 8940, 0, 0, 0, 8941, 12436, 0, 0, 0, 12364, 0, 0, + 0, 12366, 0, 0, 0, 12368, 0, 0, 0, 12370, 0, 0, 0, 12372, 0, 0, 0, 12374, + 0, 0, 0, 12376, 0, 0, 0, 12378, 0, 0, 0, 12380, 0, 0, 0, 12382, 0, 0, 0, + 12384, 0, 0, 0, 12386, 0, 0, 0, 12389, 0, 0, 0, 12391, 0, 0, 0, 12393, 0, + 0, 0, 12400, 12401, 0, 0, 12403, 12404, 0, 0, 12406, 12407, 0, 0, 12409, + 12410, 0, 0, 12412, 12413, 0, 0, 12446, 0, 0, 0, 12532, 0, 0, 0, 12460, + 0, 0, 0, 12462, 0, 0, 0, 12464, 0, 0, 0, 12466, 0, 0, 0, 12468, 0, 0, 0, + 12470, 0, 0, 0, 12472, 0, 0, 0, 12474, 0, 0, 0, 12476, 0, 0, 0, 12478, 0, + 0, 0, 12480, 0, 0, 0, 12482, 0, 0, 0, 12485, 0, 0, 0, 12487, 0, 0, 0, + 12489, 0, 0, 0, 12496, 12497, 0, 0, 12499, 12500, 0, 0, 12502, 12503, 0, + 0, 12505, 12506, 0, 0, 12508, 12509, 0, 0, 12535, 0, 0, 0, 12536, 0, 0, + 0, 12537, 0, 0, 0, 12538, 0, 0, 0, 12542, 0, 0, 0, 0, 0, 67017, 0, 0, 0, + 67044, 0, 0, 0, 69786, 0, 0, 0, 69788, 0, 0, 0, 69803, 0, 0, 0, 0, 69934, + 0, 0, 0, 69935, 70475, 70476, 0, 0, 0, 70531, 0, 0, 0, 0, 0, 70533, + 70542, 0, 0, 0, 0, 70545, 0, 0, 0, 0, 70599, 0, 70597, 70600, 0, 0, 0, 0, + 70844, 70843, 70846, 0, 0, 0, 0, 71098, 0, 0, 0, 71099, 0, 0, 0, 0, + 71992, 0, 0, 0, 0, 90401, 90403, 90405, 90402, 0, 90406, 90408, 0, 0, + 90407, 0, 0, 0, 90404, 0, 0, 0, 0, 0, 0, 93545, 0, 0, 0, 93544, 0, 0, 0, + 93546, }; static const change_record change_records_3_2_0[] = { @@ -6039,6 +6199,7 @@ static const change_record change_records_3_2_0[] = { { 255, 255, 255, 255, 255, 1000000000000.0 }, { 255, 255, 255, 255, 255, 1e+20 }, { 255, 19, 255, 255, 255, -1 }, + { 1, 255, 255, 255, 255, 0 }, { 1, 255, 255, 0, 255, 0 }, }; static const unsigned char changes_3_2_0_index[] = { @@ -6056,31 +6217,31 @@ static const unsigned char changes_3_2_0_index[] = { 1, 1, 1, 1, 78, 79, 80, 1, 81, 82, 83, 84, 85, 86, 87, 88, 89, 28, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 28, 28, 28, 115, 116, 117, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 118, 28, 28, 28, 28, 119, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 28, 28, 120, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 121, - 122, 1, 1, 123, 124, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 125, 28, 28, 28, 28, 126, 127, 1, + 1, 1, 1, 1, 1, 1, 1, 118, 28, 28, 28, 28, 119, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 120, 28, 28, 121, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 122, 1, 1, 1, 1, 1, + 1, 28, 28, 123, 124, 1, 125, 126, 127, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 128, 28, 28, + 28, 28, 129, 130, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 131, 28, 132, 133, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 134, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 135, + 28, 136, 137, 1, 138, 139, 140, 141, 1, 142, 143, 28, 28, 144, 1, 1, 1, + 1, 145, 146, 147, 148, 1, 149, 150, 1, 151, 152, 153, 1, 1, 154, 155, + 156, 1, 157, 158, 159, 28, 28, 28, 160, 161, 162, 28, 163, 164, 1, 1, 1, + 1, 165, 67, 1, 1, 1, 1, 1, 1, 1, 166, 167, 168, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 169, 1, 1, 1, 1, 1, 170, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 171, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 128, 28, 129, 130, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 131, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 132, 1, 133, - 134, 135, 136, 1, 137, 138, 28, 28, 139, 1, 1, 1, 1, 140, 141, 142, 143, - 1, 144, 1, 1, 145, 146, 147, 1, 1, 148, 149, 150, 1, 151, 152, 153, 28, - 28, 28, 154, 155, 156, 28, 157, 158, 1, 1, 1, 1, 159, 67, 1, 1, 1, 1, 1, - 1, 1, 160, 161, 162, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 163, 1, 1, 1, 1, 1, 164, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 165, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 166, 1, + 1, 1, 1, 1, 1, 1, 1, 172, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 167, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 168, 169, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 170, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 171, 28, 28, 43, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 163, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 172, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 173, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 173, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 174, 175, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 176, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 177, + 28, 28, 43, 1, 1, 1, 1, 1, 1, 1, 1, 1, 169, 1, 1, 1, 1, 1, 1, 1, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 178, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 179, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -6196,7 +6357,7 @@ static const unsigned char changes_3_2_0_index[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 174, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 180, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -6228,7 +6389,7 @@ static const unsigned char changes_3_2_0_index[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }; static const unsigned char changes_3_2_0_data[] = { @@ -6323,7 +6484,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -6514,9 +6675,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -6527,7 +6688,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, @@ -6580,7 +6741,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6598,7 +6759,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, @@ -6620,27 +6781,27 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 0, 0, - 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 9, 0, - 0, 0, 0, 9, 9, 9, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, + 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, + 9, 0, 0, 0, 0, 9, 9, 9, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, @@ -6654,24 +6815,24 @@ static const unsigned char changes_3_2_0_data[] = { 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, - 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -6686,22 +6847,22 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 9, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 9, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6718,9 +6879,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6729,51 +6890,51 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6785,7 +6946,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6800,20 +6961,20 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6824,8 +6985,8 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6838,7 +6999,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6849,12 +7010,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6863,7 +7024,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6872,13 +7033,13 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6887,10 +7048,10 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6906,7 +7067,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6920,7 +7081,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6928,11 +7089,11 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6949,12 +7110,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6970,12 +7131,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6993,12 +7154,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7006,7 +7167,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7015,51 +7176,51 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 9, 9, 0, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 9, 9, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7067,208 +7228,208 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 19, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 49, 50, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 49, 50, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, - 21, 21, 21, 21, 21, 0, 0, 0, 1, 1, 21, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 21, 21, 21, 21, 21, 0, 0, 0, 1, 1, 21, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 14, 14, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 14, 14, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 9, 0, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 9, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, + 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7277,102 +7438,102 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 0, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 0, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 9, 9, 9, 9, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, - 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 9, 9, 9, 9, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, + 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 0, 9, + 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7380,41 +7541,39 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7422,25 +7581,27 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 0, 9, 9, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 0, 9, 9, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7453,35 +7614,35 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7489,7 +7650,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7503,25 +7664,46 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7529,50 +7711,61 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, - 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7582,7 +7775,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7591,9 +7784,9 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7614,12 +7807,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7635,60 +7828,77 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7697,43 +7907,48 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, - 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, - 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7742,21 +7957,21 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7769,12 +7984,22 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7783,9 +8008,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7794,9 +8019,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7804,13 +8029,13 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7822,15 +8047,15 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7838,50 +8063,50 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 0, 9, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 9, 0, 9, 0, 9, 0, 9, 9, - 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 9, 0, 9, 0, 0, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 0, 9, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 9, 0, 9, 0, 9, 0, 9, + 9, 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 9, 0, 9, 0, + 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7890,68 +8115,68 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, - 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, + 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7965,9 +8190,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7983,7 +8208,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7994,12 +8219,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8013,7 +8238,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8029,11 +8254,11 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8042,7 +8267,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8058,8 +8283,8 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8072,11 +8297,11 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8091,10 +8316,10 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8104,8 +8329,8 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; static const change_record* get_change_3_2_0(Py_UCS4 n) diff --git a/Modules/unicodename_db.h b/Modules/unicodename_db.h index 4fe60ad5ed4452..0697e259b39019 100644 --- a/Modules/unicodename_db.h +++ b/Modules/unicodename_db.h @@ -1,8936 +1,9227 @@ -/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ #define NAME_MAXLEN 256 /* name->code dictionary */ static const unsigned char packed_name_dawg[] = { - 228, 174, 4, 250, 2, 65, 206, 254, 2, 66, 222, 254, 2, 67, 198, 245, 4, - 68, 222, 134, 1, 69, 250, 151, 1, 70, 206, 52, 71, 186, 154, 1, 72, 242, - 223, 1, 73, 146, 65, 74, 138, 23, 75, 230, 134, 1, 76, 146, 130, 3, 77, - 162, 200, 3, 78, 162, 107, 79, 226, 124, 80, 128, 159, 1, 2, 81, 85, 194, - 7, 82, 182, 169, 1, 83, 166, 175, 4, 84, 170, 203, 2, 85, 222, 90, 86, - 182, 80, 87, 182, 111, 88, 254, 4, 89, 179, 48, 90, 182, 40, 222, 2, 67, - 238, 1, 68, 182, 10, 69, 172, 5, 4, 72, 79, 77, 32, 244, 6, 7, 73, 82, - 80, 76, 65, 78, 69, 82, 76, 210, 32, 77, 198, 1, 78, 142, 26, 80, 142, - 20, 82, 158, 136, 2, 83, 210, 2, 84, 70, 85, 214, 1, 86, 172, 213, 11, 4, - 70, 71, 72, 65, 206, 192, 9, 66, 220, 219, 2, 9, 75, 84, 73, 69, 83, 69, - 76, 83, 75, 164, 187, 2, 2, 81, 85, 199, 135, 10, 88, 18, 132, 1, 2, 67, - 79, 46, 75, 20, 5, 85, 84, 69, 32, 65, 140, 230, 16, 6, 84, 73, 86, 65, - 84, 69, 225, 150, 22, 5, 32, 67, 85, 82, 82, 4, 162, 185, 25, 85, 237, - 182, 13, 2, 82, 68, 5, 147, 175, 33, 78, 4, 178, 207, 34, 67, 167, 207, - 3, 78, 188, 1, 232, 1, 4, 76, 65, 77, 32, 242, 7, 77, 216, 141, 13, 7, - 72, 69, 83, 73, 86, 69, 32, 168, 151, 4, 19, 68, 82, 69, 83, 83, 69, 68, - 32, 84, 79, 32, 84, 72, 69, 32, 83, 85, 66, 74, 234, 178, 13, 85, 161, - 186, 1, 6, 73, 32, 83, 72, 65, 75, 176, 1, 218, 1, 67, 44, 4, 83, 77, 65, - 76, 168, 4, 7, 71, 69, 77, 73, 78, 65, 84, 116, 8, 73, 78, 73, 84, 73, - 65, 76, 32, 38, 78, 178, 209, 2, 72, 220, 244, 22, 4, 65, 76, 73, 70, 0, - 5, 86, 79, 87, 69, 76, 131, 130, 12, 68, 70, 40, 5, 65, 80, 73, 84, 65, - 215, 4, 79, 68, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 68, 142, 2, - 68, 38, 71, 34, 74, 2, 77, 22, 75, 46, 78, 42, 83, 210, 158, 29, 81, 190, - 205, 4, 76, 130, 40, 67, 202, 154, 4, 90, 212, 7, 2, 65, 76, 170, 2, 66, - 2, 89, 146, 1, 87, 182, 89, 84, 150, 14, 80, 158, 20, 70, 2, 72, 2, 82, - 2, 86, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 250, 186, 5, 65, 247, 251, 33, - 72, 4, 174, 162, 39, 66, 215, 22, 65, 2, 171, 182, 38, 73, 6, 194, 183, - 38, 65, 154, 98, 80, 191, 28, 72, 6, 206, 232, 38, 85, 158, 77, 72, 3, - 89, 4, 188, 181, 2, 6, 73, 78, 78, 89, 73, 73, 135, 128, 37, 72, 4, 40, - 4, 69, 32, 67, 79, 151, 206, 38, 73, 2, 173, 174, 38, 13, 78, 83, 79, 78, - 65, 78, 84, 32, 77, 79, 68, 73, 70, 4, 218, 222, 32, 69, 191, 141, 4, 81, - 4, 166, 151, 25, 65, 199, 185, 10, 85, 4, 182, 215, 26, 69, 185, 202, 11, - 12, 73, 83, 83, 73, 79, 78, 32, 84, 73, 67, 75, 69, 116, 80, 5, 71, 69, - 65, 78, 32, 249, 144, 33, 9, 82, 73, 65, 76, 32, 84, 82, 65, 77, 114, - 136, 1, 3, 68, 82, 89, 0, 6, 76, 73, 81, 85, 73, 68, 60, 8, 77, 69, 65, - 83, 85, 82, 69, 32, 26, 87, 130, 185, 26, 78, 147, 185, 11, 67, 2, 153, - 2, 11, 32, 77, 69, 65, 83, 85, 82, 69, 32, 70, 73, 4, 246, 1, 83, 31, 84, - 14, 100, 6, 69, 73, 71, 72, 84, 32, 241, 1, 14, 79, 82, 68, 32, 83, 69, - 80, 65, 82, 65, 84, 79, 82, 32, 10, 54, 70, 66, 83, 30, 84, 65, 5, 66, - 65, 83, 69, 32, 4, 38, 73, 89, 5, 79, 85, 82, 84, 72, 2, 85, 3, 82, 83, - 84, 2, 49, 4, 69, 67, 79, 78, 2, 21, 3, 72, 73, 82, 2, 11, 68, 2, 25, 4, - 32, 83, 85, 66, 2, 133, 148, 39, 2, 85, 78, 4, 130, 151, 38, 76, 247, 72, - 68, 130, 1, 140, 2, 22, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, - 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 88, 7, 76, 69, 84, 84, 69, 82, - 32, 242, 1, 83, 168, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 140, 234, 31, 8, 78, 85, 77, 66, 69, 82, 32, 84, 219, 209, 5, 68, 6, 26, - 76, 159, 171, 39, 82, 4, 132, 227, 38, 7, 73, 71, 65, 84, 73, 78, 71, - 207, 74, 65, 68, 154, 1, 65, 210, 195, 35, 68, 46, 84, 222, 225, 1, 76, - 246, 189, 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, - 2, 77, 2, 82, 3, 83, 9, 45, 9, 76, 84, 69, 82, 78, 65, 84, 69, 32, 6, - 146, 169, 39, 66, 2, 71, 3, 84, 10, 60, 4, 73, 71, 78, 32, 241, 193, 22, - 5, 89, 77, 66, 79, 76, 8, 50, 83, 182, 130, 25, 75, 145, 233, 9, 2, 82, - 85, 4, 148, 183, 36, 4, 77, 65, 76, 76, 207, 162, 2, 69, 22, 66, 65, 162, - 218, 35, 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 11, 186, 169, 39, - 65, 2, 73, 2, 77, 3, 87, 7, 11, 32, 4, 252, 190, 34, 3, 68, 69, 80, 141, - 153, 4, 5, 65, 82, 82, 73, 86, 130, 2, 232, 1, 20, 67, 72, 69, 77, 73, - 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 166, 28, 69, - 60, 2, 76, 32, 82, 77, 108, 6, 84, 69, 82, 78, 65, 84, 208, 179, 28, 7, - 73, 69, 78, 32, 77, 79, 78, 225, 230, 6, 3, 65, 82, 77, 232, 1, 158, 2, - 65, 246, 2, 66, 210, 1, 67, 138, 3, 68, 98, 71, 38, 72, 132, 1, 4, 73, - 82, 79, 78, 82, 76, 74, 77, 144, 1, 2, 78, 73, 34, 80, 176, 2, 3, 81, 85, - 73, 74, 82, 146, 2, 83, 210, 5, 84, 166, 1, 86, 200, 1, 2, 87, 65, 214, - 245, 33, 85, 190, 206, 1, 69, 202, 153, 1, 79, 195, 27, 70, 30, 194, 1, - 76, 72, 3, 81, 85, 65, 204, 8, 7, 78, 84, 73, 77, 79, 78, 89, 148, 137, - 3, 3, 77, 65, 76, 176, 222, 8, 5, 85, 82, 73, 80, 73, 188, 129, 12, 3, - 82, 83, 69, 234, 156, 14, 83, 159, 91, 73, 8, 224, 24, 2, 69, 77, 252, - 209, 25, 4, 75, 65, 76, 73, 179, 167, 13, 85, 10, 38, 32, 149, 139, 24, - 3, 70, 79, 82, 8, 216, 7, 4, 86, 73, 84, 65, 225, 194, 22, 4, 82, 69, 71, - 73, 16, 148, 1, 7, 65, 84, 72, 32, 79, 70, 32, 204, 6, 6, 73, 83, 77, 85, - 84, 72, 144, 1, 4, 79, 82, 65, 88, 164, 1, 4, 76, 65, 67, 75, 203, 232, - 37, 82, 4, 230, 182, 16, 77, 253, 218, 21, 5, 86, 65, 80, 79, 85, 28, 82, - 65, 92, 6, 79, 80, 80, 69, 82, 32, 34, 82, 217, 206, 36, 4, 73, 78, 78, - 65, 6, 216, 218, 5, 2, 68, 85, 196, 156, 32, 9, 80, 85, 84, 32, 77, 79, - 82, 84, 85, 235, 27, 76, 4, 230, 13, 65, 179, 240, 37, 79, 16, 72, 8, 79, - 67, 85, 83, 32, 79, 70, 32, 53, 6, 85, 67, 73, 66, 76, 69, 6, 228, 16, 5, - 67, 79, 80, 80, 69, 219, 232, 37, 73, 11, 11, 45, 8, 138, 157, 39, 50, 2, - 51, 2, 52, 3, 53, 8, 44, 2, 73, 83, 185, 133, 5, 3, 65, 89, 45, 6, 144, - 2, 4, 83, 79, 76, 86, 203, 169, 36, 84, 4, 234, 182, 29, 79, 191, 212, 9, - 85, 8, 32, 4, 65, 76, 70, 32, 47, 79, 4, 240, 191, 34, 2, 79, 85, 255, - 161, 3, 68, 4, 132, 245, 2, 4, 82, 83, 69, 32, 183, 237, 35, 85, 6, 56, - 3, 32, 79, 82, 73, 7, 45, 67, 79, 80, 80, 69, 82, 4, 147, 226, 25, 69, 4, - 48, 3, 69, 65, 68, 221, 158, 13, 3, 79, 68, 69, 2, 135, 225, 34, 32, 10, - 120, 16, 69, 82, 67, 85, 82, 89, 32, 83, 85, 66, 76, 73, 77, 65, 84, 69, - 176, 133, 16, 4, 65, 82, 67, 65, 135, 148, 21, 79, 7, 207, 151, 39, 45, - 4, 142, 248, 37, 84, 235, 117, 71, 14, 108, 11, 72, 73, 76, 79, 83, 79, - 80, 72, 69, 82, 83, 34, 79, 98, 85, 181, 155, 29, 6, 82, 69, 67, 73, 80, - 73, 2, 209, 9, 4, 32, 83, 85, 76, 6, 52, 4, 87, 68, 69, 82, 161, 145, 35, - 3, 84, 32, 65, 5, 237, 231, 37, 5, 69, 68, 32, 66, 82, 4, 180, 143, 33, - 4, 84, 82, 69, 70, 161, 245, 5, 3, 82, 73, 70, 4, 132, 185, 34, 5, 78, - 84, 69, 83, 83, 249, 196, 3, 4, 67, 75, 32, 76, 24, 58, 69, 213, 220, 25, - 8, 79, 67, 75, 32, 83, 65, 76, 84, 20, 68, 5, 71, 85, 76, 85, 83, 164, 7, - 3, 65, 76, 71, 179, 172, 25, 84, 15, 32, 4, 32, 79, 70, 32, 71, 45, 6, - 228, 219, 25, 8, 65, 78, 84, 73, 77, 79, 78, 89, 255, 147, 12, 73, 6, - 170, 147, 39, 50, 2, 51, 3, 52, 34, 164, 1, 2, 65, 76, 194, 1, 84, 142, - 1, 85, 184, 139, 17, 2, 80, 73, 200, 255, 11, 2, 73, 76, 194, 243, 8, 79, - 133, 20, 11, 67, 69, 80, 84, 69, 82, 32, 79, 70, 32, 74, 8, 58, 84, 197, - 234, 37, 8, 45, 65, 77, 77, 79, 78, 73, 65, 7, 25, 4, 32, 79, 70, 32, 4, - 52, 8, 67, 79, 80, 80, 69, 82, 32, 65, 231, 5, 65, 2, 237, 149, 29, 7, - 78, 84, 73, 77, 79, 78, 73, 6, 236, 3, 8, 65, 82, 82, 69, 68, 32, 84, 82, - 169, 212, 25, 19, 82, 65, 84, 85, 77, 32, 83, 85, 80, 69, 82, 32, 83, 84, + 238, 255, 4, 254, 2, 65, 218, 130, 3, 66, 130, 154, 3, 67, 174, 245, 4, + 68, 206, 135, 1, 69, 186, 157, 1, 70, 190, 54, 71, 210, 164, 1, 72, 182, + 225, 1, 73, 222, 65, 74, 138, 23, 75, 238, 138, 1, 76, 130, 158, 3, 77, + 250, 200, 3, 78, 242, 107, 79, 138, 131, 1, 80, 128, 159, 1, 2, 81, 85, + 226, 7, 82, 194, 180, 1, 83, 214, 193, 4, 84, 134, 225, 2, 85, 242, 104, + 86, 138, 84, 87, 170, 114, 88, 254, 4, 89, 187, 48, 90, 214, 40, 222, 2, + 67, 238, 1, 68, 182, 10, 69, 172, 5, 4, 72, 79, 77, 32, 244, 6, 7, 73, + 82, 80, 76, 65, 78, 69, 82, 76, 146, 36, 77, 198, 1, 78, 154, 26, 80, + 138, 20, 82, 226, 136, 2, 83, 210, 2, 84, 70, 85, 214, 1, 86, 144, 254, + 11, 4, 70, 71, 72, 65, 186, 231, 9, 66, 180, 227, 2, 9, 75, 84, 73, 69, + 83, 69, 76, 83, 75, 196, 198, 2, 2, 81, 85, 199, 196, 10, 88, 18, 132, 1, + 2, 67, 79, 46, 75, 20, 5, 85, 84, 69, 32, 65, 164, 153, 17, 6, 84, 73, + 86, 65, 84, 69, 145, 135, 23, 5, 32, 67, 85, 82, 82, 4, 170, 142, 26, 85, + 173, 133, 14, 2, 82, 68, 5, 151, 168, 34, 78, 4, 154, 199, 35, 67, 143, + 248, 3, 78, 188, 1, 232, 1, 4, 76, 65, 77, 32, 242, 7, 77, 212, 179, 13, + 7, 72, 69, 83, 73, 86, 69, 32, 144, 165, 4, 19, 68, 82, 69, 83, 83, 69, + 68, 32, 84, 79, 32, 84, 72, 69, 32, 83, 85, 66, 74, 146, 233, 13, 85, + 237, 185, 1, 6, 73, 32, 83, 72, 65, 75, 176, 1, 218, 1, 67, 44, 4, 83, + 77, 65, 76, 168, 4, 7, 71, 69, 77, 73, 78, 65, 84, 116, 8, 73, 78, 73, + 84, 73, 65, 76, 32, 38, 78, 198, 213, 2, 72, 200, 197, 23, 4, 65, 76, 73, + 70, 0, 5, 86, 79, 87, 69, 76, 211, 205, 12, 68, 70, 40, 5, 65, 80, 73, + 84, 65, 215, 4, 79, 68, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 68, + 142, 2, 68, 38, 71, 34, 74, 2, 77, 22, 75, 46, 78, 42, 83, 166, 133, 30, + 81, 238, 223, 4, 76, 194, 39, 67, 194, 197, 4, 90, 212, 7, 2, 65, 76, + 170, 2, 66, 2, 89, 154, 1, 87, 198, 89, 84, 150, 14, 80, 158, 20, 70, 2, + 72, 2, 82, 2, 86, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 170, 218, 5, 65, + 155, 128, 35, 72, 4, 130, 198, 40, 66, 215, 22, 65, 2, 231, 217, 39, 73, + 6, 254, 218, 39, 65, 178, 98, 80, 191, 28, 72, 6, 150, 140, 40, 85, 170, + 77, 72, 3, 89, 4, 212, 185, 2, 6, 73, 78, 78, 89, 73, 73, 195, 159, 38, + 72, 4, 40, 4, 69, 32, 67, 79, 219, 241, 39, 73, 2, 233, 209, 39, 13, 78, + 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 4, 214, 204, 33, 69, 183, + 188, 4, 81, 4, 174, 236, 25, 65, 251, 217, 10, 85, 4, 238, 178, 27, 69, + 189, 146, 12, 12, 73, 83, 83, 73, 79, 78, 32, 84, 73, 67, 75, 69, 116, + 80, 5, 71, 69, 65, 78, 32, 241, 135, 34, 9, 82, 73, 65, 76, 32, 84, 82, + 65, 77, 114, 136, 1, 3, 68, 82, 89, 0, 6, 76, 73, 81, 85, 73, 68, 60, 8, + 77, 69, 65, 83, 85, 82, 69, 32, 26, 87, 230, 147, 27, 78, 183, 255, 11, + 67, 2, 153, 2, 11, 32, 77, 69, 65, 83, 85, 82, 69, 32, 70, 73, 4, 246, 1, + 83, 31, 84, 14, 100, 6, 69, 73, 71, 72, 84, 32, 241, 1, 14, 79, 82, 68, + 32, 83, 69, 80, 65, 82, 65, 84, 79, 82, 32, 10, 54, 70, 66, 83, 30, 84, + 65, 5, 66, 65, 83, 69, 32, 4, 38, 73, 89, 5, 79, 85, 82, 84, 72, 2, 85, + 3, 82, 83, 84, 2, 49, 4, 69, 67, 79, 78, 2, 21, 3, 72, 73, 82, 2, 11, 68, + 2, 25, 4, 32, 83, 85, 66, 2, 217, 183, 40, 2, 85, 78, 4, 190, 186, 39, + 76, 131, 73, 68, 130, 1, 140, 2, 22, 67, 79, 78, 83, 79, 78, 65, 78, 84, + 32, 83, 73, 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 88, 7, 76, 69, 84, + 84, 69, 82, 32, 242, 1, 83, 168, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, + 71, 78, 32, 140, 212, 32, 8, 78, 85, 77, 66, 69, 82, 32, 84, 171, 136, 6, + 68, 6, 26, 76, 243, 206, 40, 82, 4, 204, 134, 40, 7, 73, 71, 65, 84, 73, + 78, 71, 219, 74, 65, 68, 154, 1, 65, 214, 245, 36, 68, 114, 84, 218, 207, + 1, 76, 250, 192, 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, + 69, 72, 2, 77, 2, 82, 3, 83, 9, 45, 9, 76, 84, 69, 82, 78, 65, 84, 69, + 32, 6, 230, 204, 40, 66, 2, 71, 3, 84, 10, 60, 4, 73, 71, 78, 32, 153, + 149, 23, 5, 89, 77, 66, 79, 76, 8, 50, 83, 170, 215, 25, 75, 229, 139, + 10, 2, 82, 85, 4, 180, 207, 37, 4, 77, 65, 76, 76, 247, 173, 2, 69, 22, + 66, 65, 234, 249, 36, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 11, + 142, 205, 40, 65, 2, 73, 2, 77, 3, 87, 7, 11, 32, 4, 192, 182, 35, 3, 68, + 69, 80, 145, 197, 4, 5, 65, 82, 82, 73, 86, 150, 2, 216, 1, 20, 67, 72, + 69, 77, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, + 166, 28, 69, 60, 4, 73, 69, 78, 32, 208, 3, 2, 76, 32, 82, 77, 108, 6, + 84, 69, 82, 78, 65, 84, 141, 142, 36, 3, 65, 82, 77, 232, 1, 158, 2, 65, + 246, 2, 66, 210, 1, 67, 138, 3, 68, 98, 71, 38, 72, 132, 1, 4, 73, 82, + 79, 78, 82, 76, 74, 77, 144, 1, 2, 78, 73, 34, 80, 176, 2, 3, 81, 85, 73, + 74, 82, 146, 2, 83, 210, 5, 84, 166, 1, 86, 200, 1, 2, 87, 65, 170, 238, + 34, 85, 154, 215, 1, 69, 158, 181, 1, 79, 139, 31, 70, 30, 194, 1, 76, + 72, 3, 81, 85, 65, 204, 8, 7, 78, 84, 73, 77, 79, 78, 89, 236, 142, 3, 3, + 77, 65, 76, 128, 174, 21, 3, 82, 83, 69, 248, 229, 13, 5, 85, 82, 73, 80, + 73, 210, 134, 1, 83, 171, 91, 73, 8, 224, 24, 2, 69, 77, 212, 171, 26, 4, + 75, 65, 76, 73, 191, 241, 13, 85, 10, 38, 32, 129, 223, 24, 3, 70, 79, + 82, 8, 216, 7, 4, 86, 73, 84, 65, 145, 150, 23, 4, 82, 69, 71, 73, 16, + 148, 1, 7, 65, 84, 72, 32, 79, 70, 32, 204, 6, 6, 73, 83, 77, 85, 84, 72, + 144, 1, 4, 79, 82, 65, 88, 164, 1, 4, 76, 65, 67, 75, 139, 139, 39, 82, + 4, 138, 234, 16, 77, 165, 203, 22, 5, 86, 65, 80, 79, 85, 28, 82, 65, 92, + 6, 79, 80, 80, 69, 82, 32, 34, 82, 185, 160, 39, 4, 73, 78, 78, 65, 6, + 152, 250, 5, 2, 68, 85, 204, 159, 33, 9, 80, 85, 84, 32, 77, 79, 82, 84, + 85, 239, 28, 76, 4, 230, 13, 65, 255, 147, 39, 79, 16, 72, 8, 79, 67, 85, + 83, 32, 79, 70, 32, 53, 6, 85, 67, 73, 66, 76, 69, 6, 228, 16, 5, 67, 79, + 80, 80, 69, 171, 139, 39, 73, 11, 11, 45, 8, 238, 192, 40, 50, 2, 51, 2, + 52, 3, 53, 8, 44, 2, 73, 83, 129, 198, 37, 3, 65, 89, 45, 6, 144, 2, 4, + 83, 79, 76, 86, 255, 197, 37, 84, 4, 218, 161, 30, 79, 179, 141, 10, 85, + 8, 32, 4, 65, 76, 70, 32, 47, 79, 4, 200, 183, 35, 2, 79, 85, 203, 204, + 3, 68, 4, 212, 250, 2, 4, 82, 83, 69, 32, 191, 139, 37, 85, 6, 56, 3, 32, + 79, 82, 73, 7, 45, 67, 79, 80, 80, 69, 82, 4, 235, 187, 26, 69, 4, 48, 3, + 69, 65, 68, 229, 204, 13, 3, 79, 68, 69, 2, 223, 216, 35, 32, 10, 120, + 16, 69, 82, 67, 85, 82, 89, 32, 83, 85, 66, 76, 73, 77, 65, 84, 69, 196, + 183, 16, 4, 65, 82, 67, 65, 203, 130, 22, 79, 7, 179, 187, 40, 45, 4, + 218, 155, 39, 84, 131, 118, 71, 14, 108, 11, 72, 73, 76, 79, 83, 79, 80, + 72, 69, 82, 83, 34, 79, 98, 85, 173, 134, 30, 6, 82, 69, 67, 73, 80, 73, + 2, 209, 9, 4, 32, 83, 85, 76, 6, 52, 4, 87, 68, 69, 82, 205, 136, 36, 3, + 84, 32, 65, 5, 173, 138, 39, 5, 69, 68, 32, 66, 82, 4, 200, 136, 34, 4, + 84, 82, 69, 70, 241, 159, 6, 3, 82, 73, 70, 4, 220, 176, 35, 5, 78, 84, + 69, 83, 83, 237, 240, 3, 4, 67, 75, 32, 76, 24, 58, 69, 173, 182, 26, 8, + 79, 67, 75, 32, 83, 65, 76, 84, 20, 68, 5, 71, 85, 76, 85, 83, 164, 7, 3, + 65, 76, 71, 195, 129, 26, 84, 15, 32, 4, 32, 79, 70, 32, 71, 45, 6, 188, + 181, 26, 8, 65, 78, 84, 73, 77, 79, 78, 89, 247, 220, 12, 73, 6, 142, + 183, 40, 50, 2, 51, 3, 52, 34, 164, 1, 2, 65, 76, 194, 1, 84, 142, 1, 85, + 192, 191, 17, 2, 80, 73, 164, 178, 12, 2, 73, 76, 170, 176, 9, 79, 141, + 20, 11, 67, 69, 80, 84, 69, 82, 32, 79, 70, 32, 74, 8, 58, 84, 141, 141, + 39, 8, 45, 65, 77, 77, 79, 78, 73, 65, 7, 25, 4, 32, 79, 70, 32, 4, 52, + 8, 67, 79, 80, 80, 69, 82, 32, 65, 231, 5, 65, 2, 229, 128, 30, 7, 78, + 84, 73, 77, 79, 78, 73, 6, 236, 3, 8, 65, 82, 82, 69, 68, 32, 84, 82, + 129, 174, 26, 19, 82, 65, 84, 85, 77, 32, 83, 85, 80, 69, 82, 32, 83, 84, 82, 65, 84, 85, 77, 12, 44, 6, 66, 76, 73, 77, 65, 84, 155, 1, 76, 10, - 44, 5, 69, 32, 79, 70, 32, 215, 190, 38, 73, 8, 68, 8, 83, 65, 76, 84, - 32, 79, 70, 32, 130, 3, 65, 175, 209, 26, 67, 4, 254, 2, 65, 175, 209, - 26, 67, 2, 135, 149, 38, 70, 12, 68, 3, 65, 82, 84, 32, 2, 73, 78, 34, - 82, 189, 162, 37, 2, 85, 84, 4, 11, 65, 4, 219, 212, 25, 82, 4, 222, 162, - 34, 67, 167, 49, 32, 2, 145, 201, 38, 2, 73, 68, 14, 50, 73, 245, 238, - 36, 6, 69, 82, 68, 73, 71, 82, 12, 64, 5, 78, 69, 71, 65, 82, 149, 211, - 25, 5, 84, 82, 73, 79, 76, 9, 44, 5, 32, 79, 70, 32, 65, 251, 137, 39, - 45, 2, 221, 188, 23, 3, 78, 84, 73, 4, 234, 204, 38, 84, 227, 61, 88, 6, - 38, 77, 246, 246, 37, 70, 231, 118, 82, 2, 151, 227, 37, 66, 4, 134, 227, - 32, 69, 233, 136, 4, 11, 65, 82, 79, 85, 78, 68, 45, 80, 82, 79, 70, 9, - 49, 10, 79, 83, 84, 32, 69, 81, 85, 65, 76, 32, 6, 32, 2, 84, 79, 235, - 176, 31, 79, 5, 147, 205, 6, 32, 4, 240, 192, 10, 3, 73, 86, 69, 253, - 159, 27, 5, 69, 32, 79, 78, 69, 10, 162, 1, 80, 180, 194, 28, 6, 69, 82, - 73, 67, 65, 78, 200, 232, 5, 3, 66, 85, 76, 209, 204, 3, 16, 65, 76, 71, - 65, 77, 65, 84, 73, 79, 78, 32, 79, 82, 32, 67, 79, 4, 158, 205, 35, 69, - 171, 78, 72, 192, 9, 92, 3, 65, 84, 79, 174, 20, 71, 174, 1, 84, 150, - 167, 24, 67, 182, 203, 11, 68, 155, 252, 2, 75, 144, 9, 104, 17, 76, 73, - 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 32, 65, 133, 241, 12, - 3, 77, 73, 67, 142, 9, 70, 48, 138, 4, 49, 198, 2, 50, 162, 3, 51, 174, - 5, 52, 163, 3, 53, 222, 1, 106, 50, 102, 52, 110, 54, 102, 57, 182, 7, - 51, 234, 190, 11, 49, 202, 174, 22, 48, 242, 1, 53, 2, 55, 3, 56, 22, - 238, 193, 33, 54, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 55, 2, 56, 3, 57, 28, 250, 135, 12, 54, 146, 185, 21, 49, 2, 53, 174, - 193, 5, 48, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 26, 210, 238, 11, - 54, 250, 146, 27, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, - 57, 24, 186, 191, 33, 55, 2, 56, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 3, 57, 232, 1, 98, 48, 114, 49, 254, 242, 11, 50, 2, - 51, 158, 129, 22, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 42, 162, 236, - 11, 52, 2, 55, 190, 24, 53, 146, 185, 21, 48, 2, 49, 2, 50, 174, 193, 5, - 51, 2, 54, 2, 56, 3, 57, 26, 238, 131, 12, 48, 146, 185, 21, 53, 174, - 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 222, 1, 102, - 48, 110, 49, 102, 57, 210, 1, 56, 170, 225, 11, 50, 2, 54, 250, 141, 22, - 51, 2, 52, 2, 53, 3, 55, 28, 150, 130, 12, 50, 146, 185, 21, 55, 2, 57, - 174, 193, 5, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 3, 56, 24, 186, 186, - 33, 53, 2, 54, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, - 3, 57, 24, 214, 185, 33, 52, 2, 57, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, - 2, 53, 2, 54, 2, 55, 3, 56, 228, 1, 102, 48, 2, 50, 2, 53, 102, 51, 110, - 54, 102, 56, 162, 1, 57, 174, 201, 11, 55, 162, 162, 22, 49, 3, 52, 22, - 134, 184, 33, 57, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 3, 56, 30, 214, 229, 11, 54, 242, 150, 8, 50, 138, 252, 18, - 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, 182, 182, 33, - 52, 2, 56, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 3, - 57, 26, 98, 51, 242, 180, 33, 49, 2, 54, 174, 193, 5, 48, 2, 50, 2, 52, - 2, 53, 2, 55, 2, 56, 3, 57, 4, 144, 173, 20, 6, 32, 82, 65, 32, 79, 82, - 139, 201, 18, 65, 20, 140, 203, 38, 3, 51, 32, 69, 210, 42, 48, 2, 49, 2, - 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 202, 1, 102, 49, 210, 1, - 53, 130, 132, 20, 57, 166, 228, 13, 48, 2, 50, 2, 51, 2, 52, 2, 54, 2, - 55, 3, 56, 22, 90, 48, 174, 243, 38, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 3, 57, 4, 60, 6, 32, 66, 69, 71, 73, 78, 1, 5, 65, 32, - 69, 78, 68, 2, 129, 248, 15, 8, 32, 76, 79, 71, 79, 71, 82, 65, 24, 138, - 177, 33, 48, 2, 55, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, - 2, 56, 3, 57, 60, 218, 142, 32, 51, 246, 214, 1, 48, 242, 1, 49, 3, 50, - 14, 96, 2, 76, 69, 250, 161, 2, 85, 208, 173, 31, 2, 83, 84, 230, 192, 3, - 69, 249, 162, 1, 2, 82, 89, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 254, - 133, 31, 83, 191, 161, 4, 85, 29, 104, 11, 73, 67, 76, 79, 67, 75, 87, - 73, 83, 69, 32, 233, 219, 35, 9, 69, 78, 78, 65, 32, 87, 73, 84, 72, 24, - 90, 84, 254, 220, 6, 67, 58, 68, 122, 71, 138, 4, 79, 253, 173, 27, 5, - 73, 78, 84, 69, 71, 12, 84, 15, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, - 65, 68, 69, 68, 32, 187, 225, 6, 79, 10, 112, 3, 76, 69, 70, 0, 4, 82, - 73, 71, 72, 12, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, 80, 223, 224, 6, - 79, 2, 11, 84, 2, 169, 152, 36, 7, 32, 85, 45, 83, 72, 65, 80, 158, 1, - 128, 1, 20, 76, 32, 70, 85, 78, 67, 84, 73, 79, 78, 65, 76, 32, 83, 89, - 77, 66, 79, 76, 32, 210, 15, 79, 38, 80, 167, 219, 38, 67, 140, 1, 222, - 2, 67, 186, 1, 68, 190, 2, 73, 44, 4, 65, 76, 80, 72, 0, 4, 79, 77, 69, - 71, 32, 4, 74, 79, 84, 32, 36, 4, 76, 69, 70, 84, 68, 2, 81, 85, 222, 3, - 82, 54, 83, 92, 4, 69, 80, 83, 73, 32, 6, 66, 65, 67, 75, 83, 76, 76, 2, - 85, 80, 208, 216, 15, 12, 71, 82, 69, 65, 84, 69, 82, 45, 84, 72, 65, 78, - 0, 5, 84, 73, 76, 68, 69, 191, 195, 19, 90, 14, 64, 6, 73, 82, 67, 76, - 69, 32, 165, 152, 36, 4, 79, 77, 77, 65, 12, 80, 2, 83, 84, 202, 138, 19, - 68, 170, 246, 5, 66, 214, 157, 10, 85, 175, 248, 2, 74, 4, 246, 197, 37, - 73, 215, 104, 65, 22, 76, 2, 69, 76, 120, 3, 79, 87, 78, 253, 140, 31, 6, - 73, 65, 77, 79, 78, 68, 10, 30, 32, 53, 3, 84, 65, 32, 6, 182, 137, 19, - 68, 174, 155, 16, 84, 159, 163, 1, 83, 4, 254, 156, 35, 85, 207, 170, 1, - 83, 10, 22, 32, 171, 9, 87, 8, 52, 5, 84, 65, 67, 75, 32, 182, 1, 83, - 231, 6, 67, 4, 138, 156, 35, 85, 175, 248, 2, 74, 6, 40, 2, 79, 84, 173, - 155, 31, 2, 45, 66, 4, 11, 65, 5, 219, 139, 31, 32, 4, 158, 135, 19, 68, - 255, 147, 16, 85, 4, 28, 2, 32, 83, 191, 7, 87, 2, 149, 197, 36, 4, 72, - 79, 69, 32, 46, 44, 2, 65, 68, 137, 3, 4, 79, 84, 69, 32, 43, 11, 32, 40, - 178, 1, 67, 34, 68, 92, 2, 85, 80, 44, 2, 76, 69, 186, 158, 6, 81, 200, - 170, 13, 3, 78, 79, 84, 22, 69, 182, 176, 5, 66, 198, 133, 7, 83, 234, - 209, 4, 71, 162, 111, 82, 183, 79, 74, 4, 158, 146, 37, 73, 147, 127, 79, - 12, 54, 73, 36, 3, 79, 87, 78, 197, 188, 35, 2, 69, 76, 4, 218, 246, 30, - 86, 219, 174, 6, 65, 4, 152, 167, 10, 2, 32, 67, 215, 153, 27, 87, 4, - 174, 209, 36, 83, 255, 110, 70, 4, 242, 248, 11, 81, 139, 158, 23, 85, 4, - 184, 3, 5, 73, 71, 72, 84, 87, 131, 189, 38, 72, 10, 88, 5, 69, 77, 73, - 67, 79, 34, 76, 34, 84, 129, 247, 11, 7, 81, 85, 73, 83, 72, 32, 81, 2, - 229, 133, 31, 3, 76, 79, 78, 2, 177, 142, 36, 3, 65, 83, 72, 4, 248, 216, - 15, 2, 65, 82, 239, 133, 7, 73, 12, 22, 32, 171, 1, 87, 10, 78, 67, 36, - 5, 84, 65, 67, 75, 32, 253, 139, 38, 6, 83, 72, 79, 69, 32, 74, 2, 229, - 221, 22, 4, 65, 82, 69, 84, 6, 210, 255, 18, 68, 162, 195, 16, 79, 139, - 201, 2, 74, 2, 169, 207, 35, 6, 65, 82, 68, 83, 32, 86, 4, 242, 237, 32, - 83, 235, 157, 5, 76, 12, 92, 2, 82, 79, 153, 200, 25, 15, 76, 73, 67, 65, - 84, 73, 79, 78, 32, 80, 82, 79, 71, 82, 65, 10, 112, 9, 88, 73, 77, 65, - 84, 69, 76, 89, 32, 153, 189, 38, 13, 65, 67, 72, 69, 83, 32, 84, 72, 69, - 32, 76, 73, 77, 8, 76, 6, 69, 81, 85, 65, 76, 32, 201, 161, 24, 7, 66, - 85, 84, 32, 78, 79, 84, 6, 32, 2, 84, 79, 255, 128, 31, 79, 5, 181, 201, - 25, 13, 32, 79, 82, 32, 84, 72, 69, 32, 73, 77, 65, 71, 69, 230, 22, 148, - 1, 4, 65, 66, 73, 67, 208, 245, 1, 7, 77, 69, 78, 73, 65, 78, 32, 224, - 12, 3, 82, 79, 87, 236, 3, 2, 84, 73, 254, 188, 35, 73, 227, 147, 1, 67, - 148, 21, 54, 32, 141, 244, 1, 7, 45, 73, 78, 68, 73, 67, 32, 248, 20, - 178, 3, 67, 158, 1, 68, 242, 2, 69, 198, 1, 70, 212, 1, 2, 72, 65, 104, - 5, 75, 65, 83, 82, 65, 86, 76, 248, 167, 1, 2, 77, 65, 220, 18, 7, 78, - 85, 77, 66, 69, 82, 32, 36, 5, 79, 80, 69, 78, 32, 82, 80, 214, 1, 82, - 156, 2, 8, 66, 65, 83, 69, 76, 73, 78, 69, 52, 6, 73, 78, 86, 69, 82, 84, - 98, 83, 154, 33, 84, 202, 4, 86, 188, 245, 16, 9, 87, 65, 86, 89, 32, 72, - 65, 77, 90, 138, 164, 17, 81, 201, 190, 1, 6, 90, 87, 65, 82, 65, 75, 14, - 44, 5, 85, 82, 76, 89, 32, 191, 135, 36, 79, 12, 72, 4, 68, 65, 77, 77, - 0, 4, 70, 65, 84, 72, 1, 4, 75, 65, 83, 82, 4, 11, 65, 5, 199, 138, 37, - 84, 24, 154, 1, 65, 108, 5, 79, 85, 66, 76, 69, 152, 2, 16, 73, 83, 80, - 85, 84, 69, 68, 32, 69, 78, 68, 32, 79, 70, 32, 65, 137, 253, 22, 5, 69, - 67, 73, 77, 65, 14, 36, 3, 77, 77, 65, 171, 189, 32, 84, 13, 22, 32, 211, - 5, 84, 6, 158, 226, 1, 73, 58, 77, 167, 202, 35, 87, 6, 202, 4, 68, 157, - 193, 1, 18, 32, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 72, 69, 65, - 68, 32, 65, 8, 88, 12, 77, 80, 84, 89, 32, 67, 69, 78, 84, 82, 69, 32, - 57, 6, 78, 68, 32, 79, 70, 32, 4, 184, 131, 36, 4, 72, 73, 71, 72, 1, 3, - 76, 79, 87, 4, 26, 65, 139, 249, 28, 84, 2, 255, 148, 31, 89, 22, 84, 4, - 65, 84, 72, 65, 230, 209, 1, 79, 164, 196, 4, 3, 73, 86, 69, 223, 235, - 29, 85, 17, 22, 32, 139, 2, 84, 10, 52, 5, 87, 73, 84, 72, 32, 162, 222, - 1, 73, 59, 77, 6, 170, 254, 36, 84, 174, 77, 68, 191, 47, 82, 6, 72, 13, - 76, 70, 32, 77, 65, 68, 68, 65, 32, 79, 86, 69, 82, 231, 16, 77, 2, 181, - 157, 17, 2, 32, 77, 13, 18, 32, 43, 84, 6, 230, 72, 87, 146, 148, 1, 73, - 59, 77, 4, 161, 82, 2, 65, 78, 224, 15, 84, 5, 65, 82, 71, 69, 32, 146, - 1, 69, 149, 90, 8, 73, 71, 65, 84, 85, 82, 69, 32, 8, 64, 10, 82, 79, 85, - 78, 68, 32, 68, 79, 84, 32, 251, 218, 10, 67, 6, 220, 218, 10, 6, 73, 78, - 83, 73, 68, 69, 238, 213, 25, 66, 223, 155, 1, 65, 252, 7, 80, 5, 84, 84, - 69, 82, 32, 157, 251, 6, 9, 70, 84, 32, 65, 82, 82, 79, 87, 72, 248, 7, - 206, 2, 65, 240, 10, 2, 66, 69, 150, 4, 68, 214, 5, 70, 246, 4, 71, 226, - 2, 72, 156, 9, 2, 74, 69, 122, 75, 246, 5, 76, 222, 2, 77, 154, 1, 78, - 140, 3, 3, 80, 69, 72, 176, 1, 3, 81, 65, 70, 214, 1, 82, 230, 3, 83, - 158, 9, 84, 186, 7, 85, 148, 2, 2, 86, 69, 28, 3, 87, 65, 87, 202, 1, 89, - 246, 3, 79, 248, 2, 2, 90, 65, 31, 69, 112, 92, 7, 70, 82, 73, 67, 65, - 78, 32, 92, 2, 73, 78, 144, 2, 3, 76, 69, 70, 195, 193, 38, 69, 8, 52, 3, - 81, 65, 70, 214, 144, 32, 70, 243, 161, 3, 78, 5, 221, 64, 5, 32, 87, 73, - 84, 72, 21, 11, 32, 18, 72, 6, 87, 73, 84, 72, 32, 84, 154, 160, 1, 70, - 230, 43, 73, 203, 9, 77, 10, 60, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, - 32, 131, 55, 87, 6, 216, 204, 14, 17, 80, 79, 73, 78, 84, 73, 78, 71, 32, - 68, 79, 87, 78, 87, 65, 82, 68, 166, 221, 21, 66, 223, 155, 1, 65, 83, - 11, 32, 80, 70, 87, 196, 45, 6, 77, 65, 75, 83, 85, 82, 202, 112, 70, - 247, 52, 73, 70, 48, 4, 73, 84, 72, 32, 145, 45, 3, 65, 83, 76, 64, 192, - 2, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, 220, 1, 19, 82, 73, 71, 72, 84, - 32, 77, 73, 68, 68, 76, 69, 32, 83, 84, 82, 79, 75, 69, 188, 1, 6, 72, - 65, 77, 90, 65, 32, 44, 8, 87, 65, 86, 89, 32, 72, 65, 77, 254, 69, 69, - 204, 1, 4, 77, 65, 68, 68, 196, 139, 35, 9, 76, 69, 70, 84, 32, 77, 73, - 68, 68, 243, 230, 1, 68, 28, 204, 1, 17, 66, 79, 84, 84, 79, 77, 32, 82, - 73, 71, 72, 84, 32, 75, 65, 83, 82, 0, 14, 84, 79, 80, 32, 82, 73, 71, - 72, 84, 32, 70, 65, 84, 72, 94, 82, 56, 3, 76, 69, 70, 186, 201, 1, 75, - 223, 160, 31, 70, 6, 11, 65, 7, 29, 5, 32, 65, 78, 68, 32, 4, 196, 231, - 18, 3, 76, 69, 70, 255, 211, 18, 68, 8, 52, 3, 73, 71, 72, 253, 214, 1, - 4, 79, 85, 78, 68, 4, 17, 2, 84, 32, 4, 162, 176, 1, 82, 227, 37, 72, 12, - 146, 72, 65, 37, 5, 66, 69, 76, 79, 87, 4, 211, 143, 36, 90, 50, 22, 72, - 219, 63, 69, 41, 22, 32, 195, 63, 69, 28, 68, 5, 87, 73, 84, 72, 32, 198, - 150, 1, 70, 230, 43, 73, 203, 9, 77, 20, 116, 6, 83, 77, 65, 76, 76, 32, - 34, 84, 154, 19, 73, 200, 33, 10, 68, 79, 84, 32, 66, 69, 76, 79, 87, 32, - 215, 19, 72, 6, 226, 39, 77, 175, 232, 22, 86, 8, 88, 10, 72, 82, 69, 69, - 32, 68, 79, 84, 83, 32, 177, 49, 7, 87, 79, 32, 68, 79, 84, 83, 6, 144, - 1, 22, 80, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, 87, 65, 82, 68, 83, - 32, 66, 69, 76, 79, 87, 133, 29, 8, 72, 79, 82, 73, 90, 79, 78, 84, 5, - 227, 191, 14, 32, 76, 90, 65, 164, 4, 2, 68, 65, 40, 7, 79, 84, 76, 69, - 83, 83, 32, 190, 28, 89, 255, 24, 85, 42, 34, 76, 154, 4, 72, 187, 46, - 68, 25, 11, 32, 22, 56, 5, 87, 73, 84, 72, 32, 226, 145, 1, 70, 247, 52, - 73, 18, 132, 1, 9, 68, 79, 84, 32, 66, 69, 76, 79, 87, 16, 9, 73, 78, 86, - 69, 82, 84, 69, 68, 32, 34, 84, 202, 215, 11, 70, 203, 138, 26, 82, 5, - 243, 1, 32, 4, 226, 14, 83, 199, 164, 38, 86, 6, 160, 1, 10, 72, 82, 69, - 69, 32, 68, 79, 84, 83, 32, 33, 25, 87, 79, 32, 68, 79, 84, 83, 32, 86, - 69, 82, 84, 73, 67, 65, 76, 76, 89, 32, 66, 69, 76, 79, 87, 32, 4, 234, - 53, 65, 195, 227, 35, 66, 2, 253, 235, 14, 10, 65, 78, 68, 32, 83, 77, - 65, 76, 76, 32, 12, 22, 72, 207, 62, 76, 6, 199, 53, 65, 6, 190, 253, 31, - 66, 2, 70, 139, 178, 5, 81, 44, 60, 8, 65, 82, 83, 73, 32, 89, 69, 72, - 169, 2, 2, 69, 72, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 194, 140, - 1, 70, 230, 43, 73, 203, 9, 77, 12, 144, 1, 28, 69, 88, 84, 69, 78, 68, - 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, - 71, 73, 84, 32, 30, 84, 163, 40, 73, 6, 250, 8, 70, 219, 49, 84, 4, 226, - 245, 6, 72, 131, 194, 7, 87, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, - 154, 138, 1, 70, 230, 43, 73, 203, 9, 77, 12, 32, 4, 68, 79, 84, 32, 51, - 84, 6, 254, 40, 66, 249, 142, 15, 4, 77, 79, 86, 69, 6, 64, 10, 72, 82, - 69, 69, 32, 68, 79, 84, 83, 32, 203, 132, 36, 87, 4, 198, 17, 80, 227, - 129, 36, 66, 44, 72, 2, 65, 70, 160, 1, 4, 72, 65, 73, 78, 178, 21, 85, - 143, 147, 37, 82, 19, 11, 32, 16, 68, 5, 87, 73, 84, 72, 32, 194, 135, 1, - 70, 230, 43, 73, 203, 9, 77, 8, 162, 24, 84, 134, 193, 37, 82, 229, 32, - 8, 73, 78, 86, 69, 82, 84, 69, 68, 15, 11, 32, 12, 68, 5, 87, 73, 84, 72, - 32, 162, 134, 1, 70, 230, 43, 73, 203, 9, 77, 4, 254, 37, 84, 171, 140, - 26, 68, 82, 78, 65, 236, 5, 2, 69, 72, 161, 2, 9, 73, 71, 72, 32, 72, 65, - 77, 90, 65, 34, 34, 72, 213, 47, 3, 77, 90, 65, 31, 11, 32, 28, 68, 5, - 87, 73, 84, 72, 32, 186, 132, 1, 70, 230, 43, 73, 203, 9, 77, 20, 132, 2, - 29, 69, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, - 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 70, 30, 73, 92, 24, 83, 77, - 65, 76, 76, 32, 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, - 84, 65, 72, 32, 62, 84, 135, 52, 72, 2, 145, 186, 1, 2, 79, 85, 2, 45, 9, - 78, 86, 69, 82, 84, 69, 68, 32, 83, 2, 221, 253, 35, 6, 77, 65, 76, 76, - 32, 86, 6, 26, 65, 199, 139, 36, 66, 4, 174, 31, 78, 255, 135, 37, 66, 8, - 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 33, 8, 87, 79, 32, 68, - 79, 84, 83, 32, 4, 230, 8, 80, 191, 157, 37, 65, 4, 232, 165, 37, 8, 86, - 69, 82, 84, 73, 67, 65, 76, 27, 65, 41, 11, 32, 38, 144, 1, 4, 71, 79, - 65, 76, 88, 5, 87, 73, 84, 72, 32, 220, 46, 10, 68, 79, 65, 67, 72, 65, - 83, 72, 77, 69, 242, 78, 70, 230, 43, 73, 203, 9, 77, 13, 11, 32, 10, - 156, 49, 6, 87, 73, 84, 72, 32, 72, 250, 76, 70, 230, 43, 73, 203, 9, 77, - 8, 218, 26, 73, 245, 18, 3, 89, 69, 72, 9, 11, 32, 6, 166, 254, 22, 65, - 178, 238, 8, 89, 135, 181, 5, 87, 22, 28, 2, 69, 77, 131, 45, 72, 17, 11, - 32, 14, 140, 31, 6, 87, 73, 84, 72, 32, 84, 170, 93, 70, 230, 43, 73, - 203, 9, 77, 68, 98, 65, 208, 1, 4, 69, 72, 69, 72, 160, 3, 7, 73, 82, 71, - 72, 73, 90, 32, 205, 30, 2, 72, 65, 22, 46, 70, 177, 162, 1, 5, 83, 72, - 77, 73, 82, 21, 11, 32, 18, 64, 5, 87, 73, 84, 72, 32, 138, 122, 70, 230, - 43, 73, 203, 9, 77, 10, 42, 84, 130, 174, 34, 68, 199, 157, 3, 82, 4, - 174, 166, 14, 87, 207, 206, 21, 72, 25, 11, 32, 22, 64, 5, 87, 73, 84, - 72, 32, 234, 120, 70, 230, 43, 73, 203, 9, 77, 14, 38, 84, 226, 41, 83, - 143, 241, 36, 68, 10, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, - 121, 8, 87, 79, 32, 68, 79, 84, 83, 32, 6, 42, 80, 226, 129, 36, 66, 223, - 155, 1, 65, 2, 165, 243, 35, 14, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, - 87, 65, 82, 68, 4, 48, 6, 86, 69, 82, 84, 73, 67, 191, 156, 37, 65, 2, - 145, 188, 11, 3, 65, 76, 76, 12, 198, 38, 79, 13, 2, 89, 85, 26, 40, 2, - 65, 77, 245, 169, 1, 2, 79, 87, 25, 11, 32, 22, 64, 5, 87, 73, 84, 72, - 32, 254, 116, 70, 230, 43, 73, 203, 9, 77, 14, 88, 2, 68, 79, 36, 6, 83, - 77, 65, 76, 76, 32, 216, 162, 32, 2, 84, 72, 223, 209, 4, 66, 4, 166, - 161, 17, 85, 231, 248, 19, 84, 4, 204, 26, 16, 65, 82, 65, 66, 73, 67, - 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 143, 252, 37, 86, 18, 36, 3, 69, - 69, 77, 195, 212, 37, 65, 17, 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, 166, - 114, 70, 230, 43, 73, 203, 9, 77, 6, 130, 18, 84, 199, 148, 34, 68, 60, - 38, 71, 26, 89, 17, 3, 79, 79, 78, 21, 22, 79, 255, 108, 32, 10, 247, 25, - 69, 31, 11, 32, 28, 92, 5, 71, 72, 85, 78, 78, 16, 5, 87, 73, 84, 72, 32, - 186, 112, 70, 230, 43, 73, 203, 9, 77, 6, 131, 33, 65, 14, 116, 6, 83, - 77, 65, 76, 76, 32, 38, 84, 184, 23, 8, 73, 78, 86, 69, 82, 84, 69, 68, - 238, 131, 26, 68, 227, 165, 11, 82, 4, 214, 217, 30, 84, 203, 184, 7, 86, - 4, 238, 217, 6, 72, 247, 144, 29, 87, 25, 22, 32, 151, 23, 69, 12, 88, - 11, 87, 73, 84, 72, 32, 83, 77, 65, 76, 76, 32, 134, 110, 70, 230, 43, - 73, 203, 9, 77, 4, 26, 77, 203, 144, 38, 86, 2, 209, 147, 37, 3, 69, 69, - 77, 19, 11, 32, 16, 64, 5, 87, 73, 84, 72, 32, 250, 108, 70, 230, 43, 73, - 203, 9, 77, 8, 36, 4, 68, 79, 84, 32, 179, 12, 84, 6, 44, 5, 66, 69, 76, - 79, 87, 167, 146, 37, 65, 5, 241, 152, 14, 6, 32, 65, 78, 68, 32, 78, 52, - 108, 2, 69, 72, 192, 27, 3, 82, 69, 72, 136, 3, 4, 78, 79, 79, 78, 197, - 118, 7, 79, 72, 73, 78, 71, 89, 65, 35, 11, 32, 32, 52, 5, 87, 73, 84, - 72, 32, 194, 106, 70, 247, 52, 73, 28, 134, 1, 83, 96, 2, 84, 87, 234, 5, - 73, 250, 21, 72, 154, 148, 11, 70, 148, 226, 6, 5, 68, 79, 84, 32, 66, - 234, 39, 76, 207, 128, 19, 82, 10, 44, 5, 77, 65, 76, 76, 32, 251, 219, - 37, 84, 8, 194, 6, 65, 182, 198, 6, 78, 215, 150, 16, 86, 4, 37, 7, 79, - 32, 68, 79, 84, 83, 32, 4, 162, 8, 86, 147, 134, 37, 65, 60, 184, 1, 2, - 65, 68, 120, 3, 69, 69, 78, 132, 6, 4, 72, 69, 69, 78, 220, 147, 1, 4, - 85, 80, 69, 82, 156, 145, 4, 7, 84, 82, 65, 73, 71, 72, 84, 169, 220, 31, - 6, 87, 65, 83, 72, 32, 75, 17, 11, 32, 14, 68, 6, 87, 73, 84, 72, 32, 84, - 130, 102, 70, 230, 43, 73, 203, 9, 77, 6, 218, 148, 32, 72, 239, 204, 3, - 87, 27, 11, 32, 24, 64, 5, 87, 73, 84, 72, 32, 142, 101, 70, 230, 43, 73, - 203, 9, 77, 16, 232, 1, 3, 68, 79, 84, 50, 73, 44, 7, 83, 77, 65, 76, 76, - 32, 65, 110, 84, 250, 168, 11, 70, 177, 137, 5, 31, 69, 88, 84, 69, 78, - 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, - 73, 71, 73, 84, 32, 70, 79, 85, 2, 249, 140, 18, 7, 32, 66, 69, 76, 79, - 87, 32, 2, 169, 20, 7, 78, 86, 69, 82, 84, 69, 68, 2, 85, 19, 82, 65, 66, - 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 72, 32, 65, 78, 2, 159, - 167, 22, 68, 6, 96, 11, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 66, 105, - 9, 87, 79, 32, 68, 79, 84, 83, 32, 86, 4, 25, 4, 69, 76, 79, 87, 5, 11, - 32, 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 84, 2, 243, 202, 6, 72, 2, 209, - 165, 11, 8, 69, 82, 84, 73, 67, 65, 76, 76, 13, 11, 32, 10, 46, 87, 158, - 95, 70, 230, 43, 73, 203, 9, 77, 2, 161, 139, 26, 5, 73, 84, 72, 32, 68, - 118, 92, 2, 65, 72, 144, 1, 4, 67, 72, 69, 72, 124, 2, 69, 72, 150, 3, - 72, 61, 3, 84, 69, 72, 19, 11, 32, 16, 64, 5, 87, 73, 84, 72, 32, 198, - 93, 70, 230, 43, 73, 203, 9, 77, 8, 26, 84, 179, 137, 26, 68, 6, 250, - 137, 14, 87, 139, 130, 18, 72, 25, 22, 32, 163, 5, 69, 12, 64, 5, 87, 73, - 84, 72, 32, 170, 92, 70, 230, 43, 73, 203, 9, 77, 4, 198, 13, 83, 143, - 241, 36, 68, 37, 22, 32, 167, 4, 69, 24, 62, 77, 116, 5, 87, 73, 84, 72, - 32, 190, 90, 70, 231, 43, 73, 10, 48, 6, 65, 82, 66, 85, 84, 65, 183, - 144, 1, 69, 9, 11, 32, 6, 238, 90, 70, 246, 52, 73, 197, 228, 35, 2, 71, - 79, 8, 104, 6, 83, 77, 65, 76, 76, 32, 56, 12, 84, 72, 82, 69, 69, 32, - 68, 79, 84, 83, 32, 65, 131, 171, 37, 82, 4, 32, 2, 84, 69, 143, 252, 37, - 86, 2, 151, 255, 36, 72, 2, 237, 135, 27, 4, 66, 79, 86, 69, 18, 42, 65, - 126, 69, 213, 129, 1, 2, 73, 78, 6, 131, 9, 76, 23, 18, 32, 91, 69, 10, - 60, 4, 87, 73, 84, 72, 230, 87, 70, 230, 43, 73, 203, 9, 77, 2, 129, 9, - 2, 32, 83, 10, 143, 11, 72, 15, 158, 1, 32, 181, 80, 33, 73, 71, 72, 85, - 82, 32, 75, 65, 90, 65, 75, 72, 32, 75, 73, 82, 71, 72, 73, 90, 32, 65, - 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 8, 96, 16, 87, 73, 84, 72, - 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 186, 85, 70, 247, 52, 73, - 5, 155, 75, 32, 17, 234, 8, 72, 167, 76, 32, 25, 11, 32, 22, 52, 5, 87, - 73, 84, 72, 32, 202, 84, 70, 247, 52, 73, 18, 80, 4, 68, 79, 84, 32, 162, - 2, 69, 182, 1, 72, 234, 252, 13, 84, 163, 165, 23, 82, 4, 192, 178, 30, - 3, 87, 73, 84, 183, 199, 6, 65, 54, 28, 2, 69, 72, 227, 3, 85, 49, 11, - 32, 46, 100, 6, 66, 65, 82, 82, 69, 69, 252, 2, 5, 87, 73, 84, 72, 32, - 182, 79, 70, 230, 43, 73, 203, 9, 77, 17, 11, 32, 14, 52, 5, 87, 73, 84, - 72, 32, 238, 81, 70, 247, 52, 73, 10, 22, 69, 183, 1, 72, 4, 121, 28, 88, - 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, - 67, 32, 68, 73, 71, 73, 84, 32, 84, 4, 160, 229, 34, 3, 72, 82, 69, 161, - 145, 2, 2, 87, 79, 6, 21, 3, 65, 77, 90, 6, 11, 65, 6, 17, 2, 32, 65, 6, - 21, 3, 66, 79, 86, 6, 11, 69, 7, 171, 79, 32, 22, 64, 10, 72, 65, 77, 90, - 65, 32, 65, 66, 79, 86, 18, 83, 51, 84, 10, 179, 2, 69, 2, 25, 4, 77, 65, - 76, 76, 2, 255, 150, 31, 32, 10, 108, 18, 87, 79, 32, 68, 79, 84, 83, 32, - 66, 69, 76, 79, 87, 32, 65, 78, 68, 32, 174, 195, 35, 65, 183, 5, 72, 6, - 70, 72, 232, 176, 6, 7, 83, 77, 65, 76, 76, 32, 78, 191, 190, 30, 68, 2, - 205, 248, 32, 3, 65, 77, 90, 18, 26, 72, 17, 2, 73, 78, 11, 243, 71, 32, - 9, 11, 32, 6, 158, 76, 70, 246, 52, 73, 161, 11, 13, 87, 73, 84, 72, 32, - 73, 78, 86, 69, 82, 84, 69, 68, 220, 7, 230, 5, 65, 158, 5, 66, 176, 2, - 9, 68, 65, 68, 32, 87, 73, 84, 72, 32, 80, 9, 70, 69, 72, 32, 87, 73, 84, - 72, 32, 72, 11, 71, 72, 65, 73, 78, 32, 87, 73, 84, 72, 32, 154, 1, 72, - 134, 3, 74, 190, 2, 75, 252, 1, 9, 76, 65, 77, 32, 87, 73, 84, 72, 32, - 226, 4, 77, 152, 4, 10, 78, 79, 79, 78, 32, 87, 73, 84, 72, 32, 182, 2, - 81, 190, 2, 82, 178, 2, 83, 146, 16, 84, 196, 9, 7, 87, 65, 83, 65, 76, - 76, 65, 40, 9, 89, 69, 72, 32, 87, 73, 84, 72, 32, 216, 3, 53, 85, 73, - 71, 72, 85, 82, 32, 75, 73, 82, 71, 72, 73, 90, 32, 89, 69, 72, 32, 87, - 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, - 84, 72, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 205, 6, 14, - 90, 65, 72, 32, 87, 73, 84, 72, 32, 77, 69, 69, 77, 32, 54, 128, 1, 8, - 73, 78, 32, 87, 73, 84, 72, 32, 70, 76, 232, 57, 4, 75, 66, 65, 82, 205, - 168, 35, 8, 90, 90, 65, 32, 87, 65, 32, 74, 28, 152, 19, 4, 77, 69, 69, - 77, 150, 16, 74, 238, 23, 65, 255, 8, 89, 22, 56, 3, 65, 89, 72, 216, 1, - 3, 69, 70, 32, 207, 13, 76, 12, 30, 73, 122, 65, 151, 56, 69, 8, 52, 9, - 32, 65, 83, 45, 83, 65, 76, 65, 65, 47, 77, 4, 80, 4, 84, 85, 32, 87, - 139, 229, 37, 77, 4, 18, 65, 23, 32, 2, 17, 2, 65, 32, 2, 149, 153, 32, - 6, 65, 83, 45, 83, 65, 76, 8, 212, 65, 29, 77, 65, 75, 83, 85, 82, 65, - 32, 87, 73, 84, 72, 32, 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, - 65, 76, 69, 70, 1, 13, 87, 73, 84, 72, 32, 70, 65, 84, 72, 65, 84, 65, - 78, 44, 160, 1, 8, 69, 72, 32, 87, 73, 84, 72, 32, 141, 135, 25, 25, 73, - 83, 77, 73, 76, 76, 65, 72, 32, 65, 82, 45, 82, 65, 72, 77, 65, 78, 32, - 65, 82, 45, 82, 65, 72, 42, 98, 72, 24, 3, 75, 72, 65, 230, 52, 65, 250, - 3, 74, 78, 77, 86, 78, 78, 90, 242, 2, 82, 43, 89, 10, 22, 65, 171, 56, - 69, 6, 131, 59, 72, 36, 222, 6, 72, 158, 7, 75, 218, 38, 65, 250, 3, 74, - 2, 77, 134, 5, 82, 3, 89, 30, 170, 13, 75, 218, 38, 65, 250, 3, 74, 146, - 2, 77, 110, 72, 139, 2, 89, 22, 64, 5, 77, 69, 69, 77, 32, 250, 50, 65, - 250, 3, 74, 135, 5, 89, 10, 40, 5, 87, 73, 84, 72, 32, 235, 103, 73, 6, - 174, 21, 77, 170, 25, 65, 203, 12, 89, 40, 80, 8, 65, 72, 32, 87, 73, 84, - 72, 32, 69, 8, 69, 72, 32, 87, 73, 84, 72, 32, 22, 236, 3, 4, 77, 69, 69, - 77, 226, 45, 65, 138, 6, 74, 247, 2, 89, 18, 164, 1, 5, 77, 69, 69, 77, - 32, 196, 11, 6, 65, 76, 69, 70, 32, 77, 150, 2, 89, 134, 38, 74, 53, 16, - 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 8, 40, 5, - 87, 73, 84, 72, 32, 215, 100, 73, 4, 254, 43, 74, 3, 77, 30, 104, 9, 69, - 69, 77, 32, 87, 73, 84, 72, 32, 173, 230, 3, 11, 65, 76, 76, 65, 74, 65, - 76, 65, 76, 79, 85, 28, 62, 72, 60, 5, 77, 69, 69, 77, 32, 186, 45, 65, - 255, 8, 89, 8, 17, 2, 65, 72, 8, 11, 32, 8, 178, 29, 87, 191, 69, 73, 12, - 40, 5, 87, 73, 84, 72, 32, 171, 98, 73, 8, 166, 37, 72, 242, 3, 65, 203, - 12, 89, 64, 84, 8, 65, 70, 32, 87, 73, 84, 72, 32, 93, 9, 72, 65, 72, 32, - 87, 73, 84, 72, 32, 46, 168, 46, 2, 65, 76, 218, 1, 74, 96, 2, 76, 65, - 146, 2, 75, 14, 72, 30, 77, 239, 1, 89, 18, 54, 72, 250, 42, 65, 250, 3, - 74, 2, 77, 135, 5, 89, 2, 247, 8, 65, 74, 116, 5, 65, 76, 69, 70, 32, - 210, 1, 72, 124, 5, 74, 69, 69, 77, 32, 78, 75, 28, 5, 77, 69, 69, 77, - 32, 187, 47, 89, 20, 64, 5, 87, 73, 84, 72, 32, 138, 44, 77, 222, 6, 70, - 247, 52, 73, 12, 68, 6, 72, 65, 77, 90, 65, 32, 41, 7, 77, 65, 68, 68, - 65, 32, 65, 8, 38, 65, 249, 45, 4, 66, 69, 76, 79, 4, 229, 45, 3, 66, 79, - 86, 14, 28, 2, 65, 72, 163, 44, 69, 12, 11, 32, 12, 40, 5, 87, 73, 84, - 72, 32, 231, 92, 73, 8, 210, 35, 65, 130, 12, 77, 75, 89, 14, 40, 5, 87, - 73, 84, 72, 32, 151, 92, 73, 10, 130, 47, 74, 2, 77, 75, 89, 8, 129, 15, - 3, 72, 65, 72, 14, 150, 14, 87, 222, 33, 70, 230, 43, 73, 203, 9, 77, 48, - 84, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 193, 36, 7, 79, 72, 65, 77, - 77, 65, 68, 46, 116, 5, 65, 76, 69, 70, 32, 66, 72, 0, 2, 75, 72, 104, 5, - 74, 69, 69, 77, 32, 92, 5, 77, 69, 69, 77, 32, 43, 89, 4, 22, 77, 219, - 45, 70, 2, 173, 35, 6, 65, 75, 83, 85, 82, 65, 10, 21, 3, 65, 72, 32, 10, - 40, 5, 87, 73, 84, 72, 32, 211, 88, 73, 6, 250, 31, 74, 2, 77, 143, 12, - 89, 12, 40, 5, 87, 73, 84, 72, 32, 131, 88, 73, 8, 170, 31, 77, 194, 7, - 75, 14, 72, 195, 4, 89, 8, 246, 40, 87, 246, 2, 70, 231, 43, 73, 2, 11, - 69, 2, 143, 33, 72, 60, 134, 1, 72, 28, 5, 74, 69, 69, 77, 32, 92, 5, 77, - 69, 69, 77, 32, 242, 31, 65, 154, 5, 78, 78, 90, 134, 1, 75, 238, 1, 82, - 43, 89, 14, 146, 27, 65, 155, 9, 69, 16, 40, 5, 87, 73, 84, 72, 32, 191, - 85, 73, 12, 186, 24, 72, 242, 3, 65, 130, 12, 77, 75, 89, 12, 206, 15, - 87, 218, 25, 70, 230, 43, 73, 203, 9, 77, 28, 70, 65, 197, 146, 30, 11, - 85, 68, 68, 73, 83, 65, 32, 83, 73, 82, 82, 26, 64, 7, 70, 32, 87, 73, - 84, 72, 32, 193, 8, 4, 76, 65, 32, 85, 24, 64, 5, 77, 69, 69, 77, 32, - 238, 29, 65, 246, 6, 72, 139, 2, 89, 12, 40, 5, 87, 73, 84, 72, 32, 223, - 82, 73, 8, 34, 77, 186, 21, 72, 187, 16, 89, 2, 129, 38, 3, 69, 69, 77, - 18, 30, 65, 129, 26, 2, 69, 72, 16, 128, 1, 14, 68, 73, 32, 65, 76, 76, - 65, 65, 72, 85, 32, 65, 78, 72, 76, 6, 72, 73, 77, 65, 72, 85, 161, 26, - 4, 83, 79, 85, 76, 11, 26, 85, 163, 197, 37, 65, 6, 26, 77, 251, 140, 36, - 78, 5, 131, 197, 37, 65, 4, 26, 32, 1, 2, 77, 32, 2, 193, 142, 30, 4, 65, - 76, 76, 65, 190, 1, 138, 1, 65, 192, 6, 9, 69, 69, 78, 32, 87, 73, 84, - 72, 32, 250, 3, 72, 233, 5, 13, 85, 66, 72, 65, 65, 78, 65, 72, 85, 32, - 87, 65, 32, 46, 48, 7, 68, 32, 87, 73, 84, 72, 32, 251, 1, 76, 32, 76, 4, - 72, 65, 72, 32, 82, 77, 238, 23, 65, 138, 4, 75, 246, 4, 82, 3, 89, 10, - 22, 87, 195, 77, 73, 6, 25, 4, 73, 84, 72, 32, 6, 162, 16, 72, 187, 16, - 89, 8, 21, 3, 69, 69, 77, 8, 11, 32, 8, 208, 31, 6, 87, 73, 84, 72, 32, - 77, 147, 45, 73, 14, 26, 65, 77, 2, 76, 65, 4, 134, 22, 77, 193, 189, 35, - 11, 65, 77, 85, 72, 85, 32, 65, 76, 65, 89, 78, 10, 34, 32, 133, 1, 3, - 76, 76, 65, 4, 22, 85, 159, 84, 73, 2, 217, 8, 23, 83, 69, 68, 32, 65, - 83, 32, 75, 79, 82, 65, 78, 73, 67, 32, 83, 84, 79, 80, 32, 83, 73, 71, - 6, 98, 72, 169, 191, 37, 18, 65, 72, 85, 32, 65, 76, 65, 89, 72, 73, 32, - 87, 65, 45, 65, 65, 76, 73, 4, 220, 63, 12, 79, 85, 32, 65, 76, 65, 89, - 72, 69, 32, 87, 65, 1, 22, 85, 32, 65, 76, 65, 89, 72, 73, 32, 87, 65, - 65, 65, 76, 73, 72, 69, 69, 32, 87, 65, 45, 60, 130, 1, 72, 100, 5, 74, - 69, 69, 77, 32, 84, 5, 75, 72, 65, 72, 32, 92, 5, 77, 69, 69, 77, 32, - 234, 15, 65, 254, 8, 82, 3, 89, 12, 32, 3, 65, 72, 32, 159, 21, 69, 8, - 156, 14, 6, 87, 73, 84, 72, 32, 74, 214, 56, 73, 203, 9, 77, 10, 52, 5, - 87, 73, 84, 72, 32, 254, 69, 73, 203, 9, 77, 4, 234, 12, 65, 139, 8, 72, - 10, 34, 87, 190, 69, 73, 203, 9, 77, 4, 25, 4, 73, 84, 72, 32, 4, 142, - 12, 65, 203, 12, 89, 16, 52, 5, 87, 73, 84, 72, 32, 206, 68, 73, 203, 9, - 77, 10, 202, 7, 72, 174, 4, 74, 199, 11, 77, 82, 96, 10, 65, 68, 68, 65, - 32, 87, 73, 84, 72, 32, 161, 1, 9, 69, 69, 78, 32, 87, 73, 84, 72, 32, - 18, 96, 4, 68, 65, 77, 77, 0, 4, 75, 65, 83, 82, 138, 11, 83, 237, 41, 6, - 70, 65, 84, 72, 65, 32, 6, 11, 65, 6, 28, 2, 84, 65, 203, 52, 32, 2, 163, - 12, 78, 64, 130, 1, 72, 36, 5, 74, 69, 69, 77, 32, 52, 5, 77, 69, 69, 77, - 32, 170, 11, 65, 228, 4, 4, 75, 72, 65, 72, 154, 4, 82, 3, 89, 18, 194, - 15, 69, 229, 3, 2, 65, 72, 10, 158, 18, 87, 246, 2, 70, 230, 43, 73, 203, - 9, 77, 16, 64, 5, 87, 73, 84, 72, 32, 158, 20, 70, 230, 43, 73, 203, 9, - 77, 8, 252, 2, 2, 75, 72, 243, 15, 77, 122, 66, 65, 176, 2, 8, 69, 72, - 32, 87, 73, 84, 72, 32, 179, 4, 72, 28, 88, 11, 66, 65, 65, 82, 65, 75, - 65, 32, 87, 65, 45, 29, 7, 72, 32, 87, 73, 84, 72, 32, 2, 177, 163, 27, - 2, 84, 65, 26, 64, 5, 77, 69, 69, 77, 32, 194, 8, 65, 246, 6, 72, 139, 2, - 89, 14, 52, 5, 87, 73, 84, 72, 32, 166, 61, 73, 203, 9, 77, 8, 34, 72, - 174, 4, 77, 143, 12, 89, 4, 133, 16, 2, 65, 72, 66, 138, 1, 72, 108, 3, - 75, 72, 65, 12, 4, 74, 69, 69, 77, 92, 5, 77, 69, 69, 77, 32, 238, 4, 65, - 154, 5, 78, 78, 90, 242, 2, 82, 43, 89, 14, 32, 3, 65, 72, 32, 227, 9, - 69, 10, 40, 5, 87, 73, 84, 72, 32, 143, 59, 73, 6, 182, 2, 77, 199, 11, - 74, 10, 11, 72, 10, 11, 32, 10, 40, 5, 87, 73, 84, 72, 32, 175, 58, 73, - 6, 154, 1, 65, 62, 77, 143, 12, 89, 18, 64, 5, 87, 73, 84, 72, 32, 226, - 13, 70, 230, 43, 73, 203, 9, 77, 10, 50, 65, 62, 74, 194, 7, 75, 14, 72, - 195, 4, 89, 2, 217, 12, 11, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, - 2, 225, 7, 3, 69, 69, 77, 28, 56, 2, 65, 76, 117, 8, 69, 72, 32, 87, 73, - 84, 72, 32, 2, 37, 7, 32, 87, 73, 84, 72, 32, 83, 2, 197, 1, 15, 85, 80, - 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 26, 108, 3, 74, 69, - 69, 126, 65, 198, 4, 77, 86, 78, 78, 90, 242, 2, 82, 42, 89, 233, 53, 5, - 72, 69, 72, 32, 77, 2, 11, 77, 2, 11, 32, 2, 143, 63, 73, 114, 82, 65, - 38, 72, 246, 4, 78, 78, 90, 38, 74, 98, 75, 42, 77, 198, 1, 82, 43, 89, - 4, 217, 2, 5, 76, 69, 70, 32, 77, 76, 22, 65, 139, 3, 69, 72, 80, 15, 77, - 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, 84, 72, 32, 147, 5, 72, 66, - 118, 65, 126, 69, 42, 72, 78, 74, 18, 75, 62, 77, 86, 78, 22, 79, 16, 2, - 87, 65, 18, 89, 26, 90, 242, 2, 82, 67, 85, 12, 22, 76, 247, 6, 69, 8, - 21, 3, 69, 70, 32, 8, 34, 77, 222, 6, 70, 247, 52, 73, 4, 181, 6, 6, 65, - 75, 83, 85, 82, 65, 6, 11, 32, 6, 166, 6, 70, 231, 43, 73, 8, 22, 69, - 191, 3, 65, 4, 11, 72, 4, 11, 32, 4, 230, 4, 73, 163, 54, 77, 4, 251, 48, - 69, 2, 11, 72, 2, 11, 65, 2, 11, 72, 2, 149, 4, 2, 32, 73, 8, 17, 2, 69, - 69, 8, 11, 77, 8, 11, 32, 8, 198, 4, 70, 230, 43, 73, 203, 9, 77, 2, 93, - 2, 79, 79, 4, 231, 3, 69, 4, 215, 3, 87, 8, 186, 3, 69, 15, 85, 2, 17, 2, - 65, 73, 2, 239, 2, 78, 6, 21, 3, 69, 69, 77, 6, 11, 32, 6, 22, 87, 219, - 46, 73, 2, 141, 2, 5, 73, 84, 72, 32, 89, 4, 11, 72, 4, 11, 65, 4, 135, - 46, 72, 14, 21, 3, 69, 69, 77, 14, 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, - 194, 1, 70, 230, 43, 73, 203, 9, 77, 6, 18, 77, 75, 89, 4, 21, 3, 69, 69, - 77, 4, 11, 32, 4, 18, 73, 119, 70, 2, 239, 44, 78, 2, 17, 2, 69, 72, 2, - 77, 2, 32, 70, 4, 11, 69, 4, 11, 72, 4, 11, 32, 4, 22, 70, 247, 52, 73, - 2, 205, 53, 2, 73, 78, 6, 202, 43, 73, 203, 9, 77, 166, 2, 92, 3, 68, 68, - 65, 52, 3, 82, 75, 32, 109, 11, 84, 72, 69, 77, 65, 84, 73, 67, 65, 76, - 32, 4, 148, 231, 35, 5, 32, 87, 65, 65, 74, 191, 61, 72, 4, 58, 78, 1, - 10, 83, 73, 68, 69, 87, 65, 89, 83, 32, 78, 2, 221, 254, 33, 7, 79, 79, - 78, 32, 71, 72, 85, 158, 2, 178, 2, 68, 144, 3, 8, 73, 78, 73, 84, 73, - 65, 76, 32, 222, 1, 76, 132, 2, 9, 79, 80, 69, 82, 65, 84, 79, 82, 32, - 166, 1, 83, 146, 3, 84, 158, 205, 23, 65, 102, 75, 110, 90, 246, 104, 74, - 2, 77, 166, 162, 5, 72, 170, 133, 1, 66, 2, 70, 2, 82, 2, 89, 174, 57, - 71, 198, 232, 2, 78, 154, 144, 2, 81, 255, 2, 87, 62, 26, 79, 187, 155, - 36, 65, 58, 88, 6, 84, 76, 69, 83, 83, 32, 61, 12, 85, 66, 76, 69, 45, - 83, 84, 82, 85, 67, 75, 32, 8, 214, 233, 30, 66, 2, 70, 242, 161, 3, 78, - 155, 144, 2, 81, 50, 190, 12, 83, 186, 20, 75, 158, 183, 23, 84, 74, 90, - 246, 104, 74, 2, 77, 166, 162, 5, 72, 170, 133, 1, 66, 2, 70, 2, 82, 2, - 89, 174, 57, 71, 198, 232, 2, 78, 246, 215, 1, 76, 150, 55, 68, 146, 1, - 81, 222, 1, 65, 163, 1, 87, 40, 182, 1, 84, 174, 9, 83, 150, 203, 23, 72, - 30, 75, 226, 105, 74, 2, 77, 206, 167, 6, 66, 2, 70, 2, 89, 174, 57, 71, - 198, 232, 2, 78, 246, 215, 1, 76, 166, 56, 81, 222, 1, 65, 159, 57, 68, - 4, 134, 230, 30, 72, 147, 178, 6, 69, 56, 48, 6, 79, 79, 80, 69, 68, 32, - 251, 135, 37, 65, 54, 210, 8, 83, 186, 20, 75, 150, 182, 23, 65, 74, 72, - 66, 84, 74, 90, 246, 104, 74, 2, 77, 206, 167, 6, 66, 2, 70, 2, 82, 2, - 89, 174, 57, 71, 198, 232, 2, 78, 246, 215, 1, 76, 150, 55, 68, 146, 1, - 81, 255, 2, 87, 4, 160, 154, 28, 23, 77, 69, 69, 77, 32, 87, 73, 84, 72, - 32, 72, 65, 72, 32, 87, 73, 84, 72, 32, 84, 65, 84, 87, 177, 233, 7, 9, - 72, 65, 72, 32, 87, 73, 84, 72, 32, 52, 84, 9, 84, 82, 69, 84, 67, 72, - 69, 68, 32, 134, 196, 35, 72, 14, 69, 227, 138, 1, 65, 46, 182, 1, 68, - 86, 84, 254, 2, 83, 150, 203, 23, 72, 30, 75, 226, 105, 74, 2, 77, 166, - 162, 5, 90, 170, 133, 1, 66, 2, 70, 2, 89, 174, 57, 71, 198, 232, 2, 78, - 154, 144, 2, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, 83, 83, 32, 131, - 205, 36, 65, 4, 242, 223, 30, 66, 3, 70, 6, 214, 223, 30, 72, 146, 178, - 6, 65, 3, 69, 38, 42, 65, 158, 206, 23, 72, 247, 194, 13, 69, 32, 44, 5, - 73, 76, 69, 68, 32, 243, 145, 37, 72, 30, 150, 1, 68, 94, 83, 186, 20, - 75, 218, 160, 24, 74, 166, 162, 5, 72, 170, 133, 1, 89, 174, 57, 71, 198, - 232, 2, 78, 246, 215, 1, 76, 166, 56, 81, 223, 1, 65, 6, 52, 7, 79, 84, - 76, 69, 83, 83, 32, 143, 202, 36, 65, 4, 238, 254, 33, 78, 155, 144, 2, - 81, 6, 250, 190, 35, 72, 14, 69, 227, 138, 1, 65, 4, 238, 252, 17, 77, - 147, 140, 18, 83, 6, 128, 149, 24, 3, 75, 65, 83, 12, 4, 68, 65, 77, 77, - 1, 4, 70, 65, 84, 72, 10, 130, 1, 79, 212, 156, 12, 11, 76, 65, 67, 69, - 32, 79, 70, 32, 83, 65, 74, 144, 222, 5, 6, 73, 65, 83, 84, 82, 69, 175, - 173, 13, 69, 4, 148, 141, 12, 8, 69, 84, 73, 67, 32, 86, 69, 82, 205, - 237, 5, 3, 85, 78, 68, 14, 238, 1, 65, 96, 5, 69, 86, 69, 82, 83, 36, 15, - 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 72, 69, 65, 68, 32, 201, 206, 22, - 28, 79, 85, 78, 68, 69, 68, 32, 72, 73, 71, 72, 32, 83, 84, 79, 80, 32, - 87, 73, 84, 72, 32, 70, 73, 76, 76, 69, 68, 4, 40, 4, 73, 83, 69, 68, - 239, 138, 37, 89, 2, 17, 2, 32, 82, 2, 165, 193, 35, 3, 79, 85, 78, 2, - 133, 135, 21, 4, 69, 68, 32, 68, 6, 26, 65, 195, 241, 34, 66, 4, 193, - 230, 35, 3, 66, 79, 86, 202, 1, 238, 1, 69, 244, 2, 5, 72, 65, 68, 68, - 65, 36, 4, 73, 71, 78, 32, 240, 4, 5, 77, 65, 76, 76, 32, 246, 15, 85, - 244, 2, 6, 89, 77, 66, 79, 76, 32, 209, 178, 35, 18, 84, 65, 82, 84, 32, - 79, 70, 32, 82, 85, 66, 32, 69, 76, 32, 72, 73, 90, 20, 52, 7, 81, 85, - 69, 78, 67, 69, 32, 223, 209, 33, 77, 18, 184, 1, 26, 89, 69, 72, 32, 87, - 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, - 84, 72, 32, 209, 210, 30, 13, 78, 79, 79, 78, 32, 87, 73, 84, 72, 32, 75, - 69, 72, 16, 70, 65, 142, 135, 36, 87, 182, 89, 89, 150, 14, 79, 214, 22, - 69, 3, 85, 6, 36, 3, 76, 69, 70, 131, 133, 37, 69, 5, 159, 13, 32, 7, 11, - 32, 4, 246, 22, 73, 59, 77, 22, 132, 1, 2, 82, 65, 158, 1, 83, 188, 1, 7, - 65, 76, 65, 89, 72, 69, 32, 160, 15, 2, 77, 73, 209, 148, 33, 6, 84, 65, - 75, 72, 65, 76, 4, 132, 1, 13, 72, 77, 65, 84, 85, 76, 76, 65, 72, 32, - 65, 76, 65, 213, 192, 32, 13, 68, 73, 32, 65, 76, 76, 65, 72, 79, 85, 32, - 65, 78, 2, 175, 213, 36, 89, 12, 46, 65, 193, 1, 6, 73, 78, 68, 72, 73, - 32, 8, 136, 1, 18, 76, 76, 65, 76, 76, 65, 72, 79, 85, 32, 65, 76, 65, - 89, 72, 69, 32, 87, 166, 200, 29, 78, 142, 193, 5, 70, 237, 99, 2, 77, - 86, 2, 17, 2, 65, 83, 2, 229, 173, 15, 3, 83, 65, 76, 4, 140, 249, 19, - 12, 80, 79, 83, 84, 80, 79, 83, 73, 84, 73, 79, 78, 233, 205, 13, 2, 65, - 77, 106, 132, 1, 2, 70, 65, 32, 5, 72, 73, 71, 72, 32, 240, 11, 4, 76, - 79, 87, 32, 110, 75, 170, 238, 20, 68, 146, 208, 9, 89, 135, 181, 5, 87, - 4, 190, 3, 82, 219, 131, 35, 84, 74, 236, 2, 17, 68, 79, 84, 76, 69, 83, - 83, 32, 72, 69, 65, 68, 32, 79, 70, 32, 75, 22, 70, 102, 76, 142, 3, 77, - 116, 4, 78, 79, 79, 78, 18, 83, 78, 84, 38, 87, 196, 2, 3, 89, 69, 72, - 228, 128, 14, 18, 85, 80, 82, 73, 71, 72, 84, 32, 82, 69, 67, 84, 65, 78, - 71, 85, 76, 65, 196, 180, 8, 7, 82, 79, 85, 78, 68, 69, 68, 130, 122, 90, - 246, 104, 74, 214, 217, 11, 81, 223, 1, 65, 2, 251, 194, 29, 72, 4, 24, - 2, 65, 82, 31, 79, 2, 11, 83, 2, 175, 2, 73, 2, 181, 220, 25, 6, 79, 84, - 78, 79, 84, 69, 10, 60, 8, 73, 71, 65, 84, 85, 82, 69, 32, 213, 11, 2, - 65, 77, 8, 88, 10, 65, 76, 69, 70, 32, 87, 73, 84, 72, 32, 116, 3, 81, - 65, 70, 1, 3, 83, 65, 68, 4, 84, 8, 76, 65, 77, 32, 87, 73, 84, 72, 153, - 140, 35, 7, 89, 69, 72, 32, 66, 65, 82, 2, 181, 197, 30, 2, 32, 89, 2, + 44, 5, 69, 32, 79, 70, 32, 175, 226, 39, 73, 8, 68, 8, 83, 65, 76, 84, + 32, 79, 70, 32, 130, 3, 65, 247, 172, 27, 67, 4, 254, 2, 65, 247, 172, + 27, 67, 2, 219, 184, 39, 70, 12, 68, 3, 65, 82, 84, 32, 2, 73, 78, 34, + 82, 157, 195, 38, 2, 85, 84, 4, 11, 65, 4, 179, 174, 26, 82, 4, 182, 154, + 35, 67, 167, 49, 32, 2, 233, 236, 39, 2, 73, 68, 14, 50, 73, 145, 140, + 38, 6, 69, 82, 68, 73, 71, 82, 12, 64, 5, 78, 69, 71, 65, 82, 237, 172, + 26, 5, 84, 82, 73, 79, 76, 9, 44, 5, 32, 79, 70, 32, 65, 223, 173, 40, + 45, 2, 181, 144, 24, 3, 78, 84, 73, 4, 194, 240, 39, 84, 239, 61, 88, 6, + 38, 77, 194, 154, 39, 70, 255, 118, 82, 2, 223, 133, 39, 66, 22, 104, 7, + 77, 79, 78, 83, 84, 69, 82, 138, 1, 83, 241, 204, 35, 10, 67, 82, 65, 66, + 32, 83, 84, 69, 80, 80, 11, 11, 32, 8, 88, 6, 67, 76, 79, 83, 69, 68, 0, + 4, 79, 80, 69, 78, 237, 170, 36, 4, 83, 84, 69, 80, 2, 205, 208, 37, 3, + 32, 74, 65, 8, 60, 6, 80, 73, 68, 69, 82, 32, 57, 5, 81, 85, 73, 68, 32, + 4, 140, 153, 29, 5, 67, 82, 79, 85, 67, 215, 245, 1, 83, 4, 56, 6, 67, + 76, 79, 83, 69, 68, 1, 4, 79, 80, 69, 78, 2, 217, 207, 27, 5, 32, 84, 69, + 78, 84, 4, 190, 214, 33, 69, 209, 174, 4, 11, 65, 82, 79, 85, 78, 68, 45, + 80, 82, 79, 70, 9, 49, 10, 79, 83, 84, 32, 69, 81, 85, 65, 76, 32, 6, 32, + 2, 84, 79, 251, 151, 32, 79, 5, 227, 232, 6, 32, 4, 232, 219, 10, 3, 73, + 86, 69, 253, 163, 28, 5, 69, 32, 79, 78, 69, 10, 162, 1, 80, 200, 165, + 29, 6, 69, 82, 73, 67, 65, 78, 188, 249, 5, 3, 66, 85, 76, 197, 248, 3, + 16, 65, 76, 71, 65, 77, 65, 84, 73, 79, 78, 32, 79, 82, 32, 67, 79, 4, + 210, 215, 36, 69, 139, 94, 72, 194, 9, 92, 3, 65, 84, 79, 174, 20, 71, + 174, 1, 84, 146, 248, 24, 67, 222, 139, 12, 68, 139, 139, 3, 75, 144, 9, + 104, 17, 76, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 32, + 65, 141, 148, 13, 3, 77, 73, 67, 142, 9, 70, 48, 138, 4, 49, 198, 2, 50, + 162, 3, 51, 174, 5, 52, 163, 3, 53, 222, 1, 106, 50, 102, 52, 110, 54, + 102, 57, 182, 7, 51, 210, 219, 11, 49, 206, 134, 23, 48, 242, 1, 53, 2, + 55, 3, 56, 22, 170, 183, 34, 54, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 55, 2, 56, 3, 57, 28, 158, 164, 12, 54, 170, 146, 22, 49, + 2, 53, 134, 236, 5, 48, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 26, + 246, 138, 12, 54, 234, 150, 28, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 55, 2, 56, 3, 57, 24, 246, 180, 34, 55, 2, 56, 134, 236, 5, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 57, 232, 1, 98, 48, 114, 49, 162, 143, + 12, 50, 2, 51, 230, 217, 22, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 42, + 198, 136, 12, 52, 2, 55, 190, 24, 53, 170, 146, 22, 48, 2, 49, 2, 50, + 134, 236, 5, 51, 2, 54, 2, 56, 3, 57, 26, 146, 160, 12, 48, 170, 146, 22, + 53, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, + 222, 1, 102, 48, 110, 49, 102, 57, 210, 1, 56, 206, 253, 11, 50, 2, 54, + 194, 230, 22, 51, 2, 52, 2, 53, 3, 55, 28, 186, 158, 12, 50, 170, 146, + 22, 55, 2, 57, 134, 236, 5, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 3, 56, + 24, 246, 175, 34, 53, 2, 54, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, + 2, 55, 2, 56, 3, 57, 24, 146, 175, 34, 52, 2, 57, 134, 236, 5, 48, 2, 49, + 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 3, 56, 228, 1, 102, 48, 2, 50, 2, 53, + 102, 51, 110, 54, 102, 56, 162, 1, 57, 130, 230, 11, 55, 186, 250, 22, + 49, 3, 52, 22, 194, 173, 34, 57, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 3, 56, 30, 250, 129, 12, 54, 150, 196, 8, 50, + 214, 210, 19, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, + 242, 171, 34, 52, 2, 56, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, + 54, 2, 55, 3, 57, 26, 98, 51, 174, 170, 34, 49, 2, 54, 134, 236, 5, 48, + 2, 50, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 4, 216, 246, 20, 6, 32, 82, 65, + 32, 79, 82, 215, 159, 19, 65, 20, 160, 235, 39, 3, 51, 32, 69, 210, 42, + 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 202, 1, 102, + 49, 210, 1, 53, 202, 205, 20, 57, 202, 143, 14, 48, 2, 50, 2, 51, 2, 52, + 2, 54, 2, 55, 3, 56, 22, 90, 48, 194, 147, 40, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 4, 60, 6, 32, 66, 69, 71, 73, 78, 1, + 5, 65, 32, 69, 78, 68, 2, 213, 167, 16, 8, 32, 76, 79, 71, 79, 71, 82, + 65, 24, 198, 166, 34, 48, 2, 55, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 2, 56, 3, 57, 60, 150, 249, 32, 51, 166, 225, 1, 48, 242, 1, + 49, 3, 50, 14, 96, 2, 76, 69, 198, 162, 2, 85, 136, 162, 32, 2, 83, 84, + 242, 232, 3, 69, 241, 165, 1, 2, 82, 89, 7, 33, 6, 32, 87, 73, 84, 72, + 32, 4, 182, 236, 31, 83, 231, 184, 4, 85, 31, 76, 4, 69, 78, 78, 65, 41, + 11, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 5, 241, 230, 36, 5, 32, + 87, 73, 84, 72, 24, 90, 84, 222, 248, 6, 67, 58, 68, 122, 71, 138, 4, 79, + 153, 134, 28, 5, 73, 78, 84, 69, 71, 12, 84, 15, 82, 73, 65, 78, 71, 76, + 69, 45, 72, 69, 65, 68, 69, 68, 32, 155, 253, 6, 79, 10, 112, 3, 76, 69, + 70, 0, 4, 82, 73, 71, 72, 12, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, + 80, 191, 252, 6, 79, 2, 11, 84, 2, 193, 178, 37, 7, 32, 85, 45, 83, 72, + 65, 80, 158, 1, 128, 1, 20, 76, 32, 70, 85, 78, 67, 84, 73, 79, 78, 65, + 76, 32, 83, 89, 77, 66, 79, 76, 32, 206, 15, 79, 38, 80, 179, 251, 39, + 67, 140, 1, 222, 2, 67, 186, 1, 68, 190, 2, 73, 44, 4, 65, 76, 80, 72, 0, + 4, 79, 77, 69, 71, 32, 4, 74, 79, 84, 32, 36, 4, 76, 69, 70, 84, 68, 2, + 81, 85, 218, 3, 82, 54, 83, 92, 4, 69, 80, 83, 73, 32, 6, 66, 65, 67, 75, + 83, 76, 76, 2, 85, 80, 156, 136, 16, 12, 71, 82, 69, 65, 84, 69, 82, 45, + 84, 72, 65, 78, 0, 5, 84, 73, 76, 68, 69, 203, 145, 20, 90, 14, 64, 6, + 73, 82, 67, 76, 69, 32, 169, 230, 38, 4, 79, 77, 77, 65, 12, 80, 2, 83, + 84, 190, 206, 19, 68, 234, 131, 6, 66, 246, 201, 10, 85, 215, 154, 3, 74, + 4, 250, 226, 38, 73, 207, 107, 65, 22, 76, 2, 69, 76, 120, 3, 79, 87, 78, + 129, 244, 31, 6, 73, 65, 77, 79, 78, 68, 10, 30, 32, 53, 3, 84, 65, 32, + 6, 170, 205, 19, 68, 142, 213, 16, 84, 223, 190, 1, 83, 4, 210, 154, 36, + 85, 143, 198, 1, 83, 10, 22, 32, 167, 9, 87, 8, 52, 5, 84, 65, 67, 75, + 32, 182, 1, 83, 227, 6, 67, 4, 222, 153, 36, 85, 215, 154, 3, 74, 6, 40, + 2, 79, 84, 225, 129, 32, 2, 45, 66, 4, 11, 65, 5, 223, 242, 31, 32, 4, + 146, 203, 19, 68, 223, 205, 16, 85, 4, 28, 2, 32, 83, 187, 7, 87, 2, 169, + 222, 37, 4, 72, 79, 69, 32, 46, 44, 2, 65, 68, 133, 3, 4, 79, 84, 69, 32, + 43, 11, 32, 40, 178, 1, 67, 38, 68, 92, 2, 85, 80, 36, 2, 76, 69, 142, + 186, 6, 81, 176, 216, 13, 3, 78, 79, 84, 22, 69, 178, 184, 5, 66, 194, + 158, 7, 83, 170, 132, 5, 71, 162, 114, 82, 195, 79, 74, 4, 178, 175, 38, + 73, 251, 129, 1, 79, 12, 54, 73, 36, 3, 79, 87, 78, 213, 199, 36, 2, 69, + 76, 4, 130, 221, 31, 86, 251, 230, 6, 65, 4, 222, 245, 3, 32, 255, 234, + 34, 87, 4, 162, 238, 37, 83, 255, 113, 70, 4, 238, 153, 12, 81, 231, 250, + 23, 85, 4, 184, 3, 5, 73, 71, 72, 84, 87, 143, 221, 39, 72, 10, 88, 5, + 69, 77, 73, 67, 79, 34, 76, 34, 84, 253, 151, 12, 7, 81, 85, 73, 83, 72, + 32, 81, 2, 237, 236, 31, 3, 76, 79, 78, 2, 185, 220, 38, 3, 65, 83, 72, + 4, 196, 136, 16, 2, 65, 82, 163, 166, 7, 73, 12, 22, 32, 171, 1, 87, 10, + 78, 67, 36, 5, 84, 65, 67, 75, 32, 253, 171, 39, 6, 83, 72, 79, 69, 32, + 74, 2, 229, 173, 23, 4, 65, 82, 69, 84, 6, 202, 195, 19, 68, 198, 138, + 17, 79, 239, 221, 2, 74, 2, 209, 222, 36, 6, 65, 82, 68, 83, 32, 86, 4, + 178, 227, 33, 83, 171, 200, 5, 76, 12, 92, 2, 82, 79, 177, 159, 26, 15, + 76, 73, 67, 65, 84, 73, 79, 78, 32, 80, 82, 79, 71, 82, 65, 10, 112, 9, + 88, 73, 77, 65, 84, 69, 76, 89, 32, 165, 221, 39, 13, 65, 67, 72, 69, 83, + 32, 84, 72, 69, 32, 76, 73, 77, 8, 76, 6, 69, 81, 85, 65, 76, 32, 245, + 242, 24, 7, 66, 85, 84, 32, 78, 79, 84, 6, 32, 2, 84, 79, 135, 232, 31, + 79, 5, 185, 160, 26, 13, 32, 79, 82, 32, 84, 72, 69, 32, 73, 77, 65, 71, + 69, 240, 22, 148, 1, 4, 65, 66, 73, 67, 156, 246, 1, 7, 77, 69, 78, 73, + 65, 78, 32, 224, 12, 3, 82, 79, 87, 228, 3, 2, 84, 73, 174, 220, 36, 73, + 251, 147, 1, 67, 158, 21, 54, 32, 217, 244, 1, 7, 45, 73, 78, 68, 73, 67, + 32, 130, 21, 178, 3, 67, 238, 1, 68, 246, 2, 69, 186, 1, 70, 212, 1, 2, + 72, 65, 104, 5, 75, 65, 83, 82, 65, 86, 76, 248, 167, 1, 2, 77, 65, 204, + 18, 7, 78, 85, 77, 66, 69, 82, 32, 36, 5, 79, 80, 69, 78, 32, 82, 80, + 238, 1, 82, 156, 2, 8, 66, 65, 83, 69, 76, 73, 78, 69, 52, 6, 73, 78, 86, + 69, 82, 84, 98, 83, 150, 33, 84, 202, 4, 86, 232, 184, 17, 9, 87, 65, 86, + 89, 32, 72, 65, 77, 90, 190, 249, 17, 81, 145, 197, 1, 6, 90, 87, 65, 82, + 65, 75, 16, 44, 2, 79, 77, 81, 5, 85, 82, 76, 89, 32, 4, 216, 158, 19, + 11, 66, 73, 78, 73, 78, 71, 32, 65, 76, 69, 70, 207, 209, 20, 77, 12, 72, + 4, 68, 65, 77, 77, 0, 4, 70, 65, 84, 72, 1, 4, 75, 65, 83, 82, 4, 11, 65, + 5, 183, 167, 38, 84, 24, 158, 1, 65, 108, 5, 79, 85, 66, 76, 69, 188, + 206, 23, 5, 69, 67, 73, 77, 65, 209, 192, 9, 16, 73, 83, 80, 85, 84, 69, + 68, 32, 69, 78, 68, 32, 79, 70, 32, 65, 14, 36, 3, 77, 77, 65, 147, 178, + 33, 84, 13, 22, 32, 199, 5, 84, 6, 154, 226, 1, 73, 54, 77, 135, 233, 36, + 87, 6, 190, 4, 68, 165, 193, 1, 18, 32, 82, 73, 71, 72, 84, 32, 65, 82, + 82, 79, 87, 72, 69, 65, 68, 32, 65, 8, 88, 12, 77, 80, 84, 89, 32, 67, + 69, 78, 84, 82, 69, 32, 57, 6, 78, 68, 32, 79, 70, 32, 4, 144, 156, 37, + 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 4, 146, 223, 29, 84, 255, 173, 3, + 65, 22, 84, 4, 65, 84, 72, 65, 238, 209, 1, 79, 156, 223, 4, 3, 73, 86, + 69, 195, 233, 30, 85, 17, 22, 32, 139, 2, 84, 10, 52, 5, 87, 73, 84, 72, + 32, 170, 222, 1, 73, 55, 77, 6, 250, 154, 38, 84, 138, 80, 68, 203, 47, + 82, 6, 72, 13, 76, 70, 32, 77, 65, 68, 68, 65, 32, 79, 86, 69, 82, 235, + 16, 77, 2, 165, 205, 17, 2, 32, 77, 13, 18, 32, 43, 84, 6, 254, 71, 87, + 130, 149, 1, 73, 55, 77, 4, 173, 82, 2, 65, 78, 230, 15, 84, 5, 65, 82, + 71, 69, 32, 146, 1, 69, 149, 90, 8, 73, 71, 65, 84, 85, 82, 69, 32, 8, + 64, 10, 82, 79, 85, 78, 68, 32, 68, 79, 84, 32, 179, 245, 10, 67, 6, 148, + 245, 10, 6, 73, 78, 83, 73, 68, 69, 162, 213, 26, 66, 167, 161, 1, 65, + 130, 8, 80, 5, 84, 84, 69, 82, 32, 185, 150, 7, 9, 70, 84, 32, 65, 82, + 82, 79, 87, 72, 254, 7, 210, 2, 65, 240, 10, 2, 66, 69, 150, 4, 68, 186, + 5, 70, 246, 4, 71, 226, 2, 72, 156, 9, 2, 74, 69, 154, 1, 75, 130, 5, 76, + 222, 2, 77, 154, 1, 78, 140, 3, 3, 80, 69, 72, 176, 1, 3, 81, 65, 70, + 214, 1, 82, 230, 3, 83, 162, 9, 84, 174, 8, 85, 148, 2, 2, 86, 69, 28, 3, + 87, 65, 87, 202, 1, 89, 246, 3, 79, 236, 2, 2, 90, 65, 31, 69, 112, 92, + 7, 70, 82, 73, 67, 65, 78, 32, 92, 2, 73, 78, 144, 2, 3, 76, 69, 70, 131, + 225, 39, 69, 8, 52, 3, 81, 65, 70, 218, 251, 32, 70, 139, 193, 3, 78, 5, + 241, 63, 5, 32, 87, 73, 84, 72, 21, 11, 32, 18, 72, 6, 87, 73, 84, 72, + 32, 84, 150, 160, 1, 70, 238, 43, 73, 199, 9, 77, 10, 60, 10, 72, 82, 69, + 69, 32, 68, 79, 84, 83, 32, 147, 54, 87, 6, 160, 250, 14, 17, 80, 79, 73, + 78, 84, 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 198, 201, 22, 66, + 167, 161, 1, 65, 83, 11, 32, 80, 70, 87, 212, 44, 6, 77, 65, 75, 83, 85, + 82, 182, 113, 70, 255, 52, 73, 70, 48, 4, 73, 84, 72, 32, 161, 44, 3, 65, + 83, 76, 64, 192, 2, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, 220, 1, 19, + 82, 73, 71, 72, 84, 32, 77, 73, 68, 68, 76, 69, 32, 83, 84, 82, 79, 75, + 69, 188, 1, 6, 72, 65, 77, 90, 65, 32, 44, 8, 87, 65, 86, 89, 32, 72, 65, + 77, 134, 70, 69, 204, 1, 4, 77, 65, 68, 68, 136, 165, 36, 9, 76, 69, 70, + 84, 32, 77, 73, 68, 68, 207, 236, 1, 68, 28, 204, 1, 17, 66, 79, 84, 84, + 79, 77, 32, 82, 73, 71, 72, 84, 32, 75, 65, 83, 82, 0, 14, 84, 79, 80, + 32, 82, 73, 71, 72, 84, 32, 70, 65, 84, 72, 94, 82, 56, 3, 76, 69, 70, + 190, 201, 1, 75, 207, 149, 32, 70, 6, 11, 65, 7, 29, 5, 32, 65, 78, 68, + 32, 4, 240, 170, 19, 3, 76, 69, 70, 251, 175, 19, 68, 8, 52, 3, 73, 71, + 72, 253, 214, 1, 4, 79, 85, 78, 68, 4, 17, 2, 84, 32, 4, 166, 176, 1, 82, + 223, 37, 72, 12, 154, 72, 65, 37, 5, 66, 69, 76, 79, 87, 4, 179, 168, 37, + 90, 50, 22, 72, 227, 63, 69, 41, 22, 32, 203, 63, 69, 28, 68, 5, 87, 73, + 84, 72, 32, 194, 150, 1, 70, 238, 43, 73, 199, 9, 77, 20, 116, 6, 83, 77, + 65, 76, 76, 32, 34, 84, 254, 18, 73, 248, 32, 10, 68, 79, 84, 32, 66, 69, + 76, 79, 87, 32, 191, 20, 72, 6, 242, 38, 77, 231, 184, 23, 86, 8, 88, 10, + 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 193, 48, 7, 87, 79, 32, 68, 79, + 84, 83, 6, 144, 1, 22, 80, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, 87, + 65, 82, 68, 83, 32, 66, 69, 76, 79, 87, 161, 54, 8, 72, 79, 82, 73, 90, + 79, 78, 84, 5, 171, 237, 14, 32, 78, 90, 65, 136, 4, 2, 68, 65, 40, 7, + 79, 84, 76, 69, 83, 83, 32, 234, 27, 89, 247, 25, 85, 44, 34, 76, 254, 3, + 72, 235, 45, 68, 27, 11, 32, 24, 56, 5, 87, 73, 84, 72, 32, 222, 145, 1, + 70, 255, 52, 73, 20, 108, 9, 73, 78, 86, 69, 82, 84, 69, 68, 32, 34, 84, + 168, 1, 3, 68, 79, 84, 250, 246, 11, 70, 207, 137, 27, 82, 4, 238, 14, + 83, 163, 196, 39, 86, 8, 132, 1, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, + 32, 33, 18, 87, 79, 32, 68, 79, 84, 83, 32, 86, 69, 82, 84, 73, 67, 65, + 76, 76, 89, 4, 182, 54, 65, 163, 253, 36, 66, 4, 33, 6, 32, 66, 69, 76, + 79, 87, 5, 157, 154, 15, 11, 32, 65, 78, 68, 32, 83, 77, 65, 76, 76, 32, + 12, 22, 72, 243, 62, 76, 6, 235, 53, 65, 6, 222, 232, 32, 66, 2, 70, 175, + 230, 5, 81, 44, 60, 8, 65, 82, 83, 73, 32, 89, 69, 72, 169, 2, 2, 69, 72, + 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 218, 140, 1, 70, 238, 43, 73, + 199, 9, 77, 12, 144, 1, 28, 69, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, + 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 30, + 84, 207, 39, 73, 6, 250, 8, 70, 255, 49, 84, 4, 150, 145, 7, 72, 179, + 212, 7, 87, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 178, 138, 1, 70, + 238, 43, 73, 199, 9, 77, 12, 32, 4, 68, 79, 84, 32, 51, 84, 6, 174, 40, + 66, 229, 190, 15, 4, 77, 79, 86, 69, 6, 64, 10, 72, 82, 69, 69, 32, 68, + 79, 84, 83, 32, 199, 157, 37, 87, 4, 198, 17, 80, 231, 155, 37, 66, 44, + 72, 2, 65, 70, 160, 1, 4, 72, 65, 73, 78, 222, 20, 85, 167, 179, 38, 82, + 19, 11, 32, 16, 68, 5, 87, 73, 84, 72, 32, 218, 135, 1, 70, 238, 43, 73, + 199, 9, 77, 8, 206, 23, 84, 170, 225, 38, 82, 241, 32, 8, 73, 78, 86, 69, + 82, 84, 69, 68, 15, 11, 32, 12, 68, 5, 87, 73, 84, 72, 32, 186, 134, 1, + 70, 238, 43, 73, 199, 9, 77, 4, 174, 37, 84, 131, 228, 26, 68, 82, 78, + 65, 236, 5, 2, 69, 72, 161, 2, 9, 73, 71, 72, 32, 72, 65, 77, 90, 65, 34, + 34, 72, 249, 47, 3, 77, 90, 65, 31, 11, 32, 28, 68, 5, 87, 73, 84, 72, + 32, 210, 132, 1, 70, 238, 43, 73, 199, 9, 77, 20, 132, 2, 29, 69, 88, 84, + 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, + 32, 68, 73, 71, 73, 84, 32, 70, 30, 73, 92, 24, 83, 77, 65, 76, 76, 32, + 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 72, 32, + 62, 84, 159, 52, 72, 2, 173, 186, 1, 2, 79, 85, 2, 45, 9, 78, 86, 69, 82, + 84, 69, 68, 32, 83, 2, 217, 150, 37, 6, 77, 65, 76, 76, 32, 86, 6, 26, + 65, 203, 165, 37, 66, 4, 222, 30, 78, 155, 168, 38, 66, 8, 88, 10, 72, + 82, 69, 69, 32, 68, 79, 84, 83, 32, 33, 8, 87, 79, 32, 68, 79, 84, 83, + 32, 4, 230, 8, 80, 139, 189, 38, 65, 4, 180, 197, 38, 8, 86, 69, 82, 84, + 73, 67, 65, 76, 27, 65, 41, 11, 32, 38, 144, 1, 4, 71, 79, 65, 76, 88, 5, + 87, 73, 84, 72, 32, 128, 47, 10, 68, 79, 65, 67, 72, 65, 83, 72, 77, 69, + 230, 78, 70, 238, 43, 73, 199, 9, 77, 13, 11, 32, 10, 180, 49, 6, 87, 73, + 84, 72, 32, 72, 250, 76, 70, 238, 43, 73, 199, 9, 77, 8, 134, 26, 73, + 237, 19, 3, 89, 69, 72, 9, 11, 32, 6, 138, 206, 23, 65, 238, 137, 9, 89, + 179, 233, 5, 87, 22, 28, 2, 69, 77, 167, 45, 72, 17, 11, 32, 14, 68, 6, + 87, 73, 84, 72, 32, 84, 138, 124, 70, 238, 43, 73, 199, 9, 77, 6, 166, + 214, 14, 87, 179, 201, 18, 72, 70, 98, 65, 204, 1, 4, 69, 72, 69, 72, + 176, 2, 7, 73, 82, 71, 72, 73, 90, 32, 197, 31, 2, 72, 65, 24, 46, 70, + 177, 162, 1, 5, 83, 72, 77, 73, 82, 23, 11, 32, 20, 64, 5, 87, 73, 84, + 72, 32, 130, 122, 70, 238, 43, 73, 199, 9, 77, 12, 42, 84, 134, 161, 35, + 68, 243, 201, 3, 82, 6, 222, 27, 87, 251, 241, 36, 72, 25, 11, 32, 22, + 64, 5, 87, 73, 84, 72, 32, 230, 120, 70, 238, 43, 73, 199, 9, 77, 14, 38, + 84, 234, 41, 83, 175, 144, 38, 68, 10, 60, 10, 72, 82, 69, 69, 32, 68, + 79, 84, 83, 32, 139, 26, 87, 6, 42, 80, 230, 155, 37, 66, 167, 161, 1, + 65, 2, 161, 140, 37, 14, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, 87, 65, + 82, 68, 12, 190, 39, 79, 13, 2, 89, 85, 26, 40, 2, 65, 77, 233, 170, 1, + 2, 79, 87, 25, 11, 32, 22, 64, 5, 87, 73, 84, 72, 32, 234, 117, 70, 238, + 43, 73, 199, 9, 77, 14, 88, 2, 68, 79, 36, 6, 83, 77, 65, 76, 76, 32, + 184, 152, 33, 2, 84, 72, 159, 253, 4, 66, 4, 146, 229, 17, 85, 155, 213, + 20, 84, 4, 196, 27, 16, 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, 69, + 82, 32, 84, 65, 199, 155, 39, 86, 18, 36, 3, 69, 69, 77, 231, 244, 38, + 65, 17, 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, 146, 115, 70, 238, 43, 73, + 199, 9, 77, 6, 134, 18, 84, 187, 136, 35, 68, 60, 38, 71, 26, 89, 17, 3, + 79, 79, 78, 21, 22, 79, 235, 109, 32, 10, 239, 26, 69, 31, 11, 32, 28, + 92, 5, 71, 72, 85, 78, 78, 16, 5, 87, 73, 84, 72, 32, 166, 113, 70, 238, + 43, 73, 199, 9, 77, 6, 251, 33, 65, 14, 116, 6, 83, 77, 65, 76, 76, 32, + 38, 84, 176, 24, 8, 73, 78, 86, 69, 82, 84, 69, 68, 210, 218, 26, 68, + 171, 238, 11, 82, 4, 206, 210, 32, 84, 131, 224, 6, 86, 4, 246, 245, 6, + 72, 191, 142, 30, 87, 25, 22, 32, 143, 24, 69, 12, 88, 11, 87, 73, 84, + 72, 32, 83, 77, 65, 76, 76, 32, 242, 110, 70, 238, 43, 73, 199, 9, 77, 4, + 26, 77, 251, 176, 39, 86, 2, 241, 179, 38, 3, 69, 69, 77, 19, 11, 32, 16, + 64, 5, 87, 73, 84, 72, 32, 230, 109, 70, 238, 43, 73, 199, 9, 77, 8, 36, + 4, 68, 79, 84, 32, 183, 12, 84, 6, 44, 5, 66, 69, 76, 79, 87, 199, 178, + 38, 65, 5, 169, 199, 14, 6, 32, 65, 78, 68, 32, 78, 52, 108, 2, 69, 72, + 184, 28, 3, 82, 69, 72, 252, 2, 4, 78, 79, 79, 78, 205, 118, 7, 79, 72, + 73, 78, 71, 89, 65, 35, 11, 32, 32, 52, 5, 87, 73, 84, 72, 32, 174, 107, + 70, 255, 52, 73, 28, 134, 1, 83, 96, 2, 84, 87, 234, 5, 73, 230, 22, 72, + 206, 180, 11, 70, 144, 133, 7, 5, 68, 79, 84, 32, 66, 234, 39, 76, 215, + 220, 19, 82, 10, 44, 5, 77, 65, 76, 76, 32, 171, 252, 38, 84, 8, 198, 6, + 65, 186, 226, 6, 78, 135, 203, 16, 86, 4, 37, 7, 79, 32, 68, 79, 84, 83, + 32, 4, 166, 8, 86, 175, 166, 38, 65, 60, 184, 1, 2, 65, 68, 120, 3, 69, + 69, 78, 136, 6, 4, 72, 69, 69, 78, 204, 148, 1, 4, 85, 80, 69, 82, 180, + 172, 4, 7, 84, 82, 65, 73, 71, 72, 84, 181, 224, 32, 6, 87, 65, 83, 72, + 32, 75, 17, 11, 32, 14, 68, 6, 87, 73, 84, 72, 32, 84, 238, 102, 70, 238, + 43, 73, 199, 9, 77, 6, 186, 138, 33, 72, 223, 240, 3, 87, 27, 11, 32, 24, + 64, 5, 87, 73, 84, 72, 32, 250, 101, 70, 238, 43, 73, 199, 9, 77, 16, + 232, 1, 3, 68, 79, 84, 50, 73, 48, 7, 83, 77, 65, 76, 76, 32, 65, 110, + 84, 150, 202, 11, 70, 221, 152, 5, 31, 69, 88, 84, 69, 78, 68, 69, 68, + 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, + 84, 32, 70, 79, 85, 2, 149, 209, 18, 7, 32, 66, 69, 76, 79, 87, 32, 2, + 173, 186, 17, 7, 78, 86, 69, 82, 84, 69, 68, 2, 85, 19, 82, 65, 66, 73, + 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 72, 32, 65, 78, 2, 179, 247, + 22, 68, 6, 96, 11, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 66, 105, 9, + 87, 79, 32, 68, 79, 84, 83, 32, 86, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, + 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 84, 2, 247, 230, 6, 72, 2, 237, 198, + 11, 8, 69, 82, 84, 73, 67, 65, 76, 76, 13, 11, 32, 10, 46, 87, 134, 96, + 70, 238, 43, 73, 199, 9, 77, 2, 249, 226, 26, 5, 73, 84, 72, 32, 68, 120, + 92, 2, 65, 72, 132, 2, 4, 67, 72, 69, 72, 124, 2, 69, 72, 150, 3, 72, 61, + 3, 84, 69, 72, 21, 11, 32, 18, 64, 5, 87, 73, 84, 72, 32, 174, 94, 70, + 238, 43, 73, 199, 9, 77, 10, 26, 84, 139, 225, 26, 68, 8, 26, 87, 199, + 129, 33, 72, 4, 37, 7, 79, 32, 68, 79, 84, 83, 32, 4, 48, 6, 86, 69, 82, + 84, 73, 67, 211, 162, 38, 65, 2, 165, 195, 11, 3, 65, 76, 76, 25, 22, 32, + 163, 5, 69, 12, 64, 5, 87, 73, 84, 72, 32, 158, 92, 70, 238, 43, 73, 199, + 9, 77, 4, 198, 13, 83, 175, 144, 38, 68, 37, 22, 32, 167, 4, 69, 24, 62, + 77, 116, 5, 87, 73, 84, 72, 32, 178, 90, 70, 239, 43, 73, 10, 48, 6, 65, + 82, 66, 85, 84, 65, 175, 144, 1, 69, 9, 11, 32, 6, 226, 90, 70, 254, 52, + 73, 233, 131, 37, 2, 71, 79, 8, 104, 6, 83, 77, 65, 76, 76, 32, 56, 12, + 84, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 65, 175, 202, 38, 82, 4, 32, + 2, 84, 69, 199, 155, 39, 86, 2, 191, 158, 38, 72, 2, 161, 233, 27, 4, 66, + 79, 86, 69, 18, 42, 65, 126, 69, 209, 129, 1, 2, 73, 78, 6, 131, 9, 76, + 23, 18, 32, 91, 69, 10, 60, 4, 87, 73, 84, 72, 218, 87, 70, 238, 43, 73, + 199, 9, 77, 2, 129, 9, 2, 32, 83, 10, 131, 11, 72, 15, 158, 1, 32, 169, + 80, 33, 73, 71, 72, 85, 82, 32, 75, 65, 90, 65, 75, 72, 32, 75, 73, 82, + 71, 72, 73, 90, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 8, + 96, 16, 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, + 174, 85, 70, 255, 52, 73, 5, 143, 75, 32, 17, 222, 8, 72, 167, 76, 32, + 25, 11, 32, 22, 52, 5, 87, 73, 84, 72, 32, 190, 84, 70, 255, 52, 73, 18, + 80, 4, 68, 79, 84, 32, 162, 2, 69, 182, 1, 72, 170, 170, 14, 84, 143, + 151, 24, 82, 4, 164, 152, 31, 3, 87, 73, 84, 251, 128, 7, 65, 54, 28, 2, + 69, 72, 227, 3, 85, 49, 11, 32, 46, 100, 6, 66, 65, 82, 82, 69, 69, 252, + 2, 5, 87, 73, 84, 72, 32, 170, 79, 70, 238, 43, 73, 199, 9, 77, 17, 11, + 32, 14, 52, 5, 87, 73, 84, 72, 32, 226, 81, 70, 255, 52, 73, 10, 22, 69, + 183, 1, 72, 4, 121, 28, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, + 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 84, 4, 184, + 239, 35, 3, 72, 82, 69, 177, 166, 2, 2, 87, 79, 6, 21, 3, 65, 77, 90, 6, + 11, 65, 6, 17, 2, 32, 65, 6, 21, 3, 66, 79, 86, 6, 11, 69, 7, 159, 79, + 32, 22, 64, 10, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 18, 83, 39, 84, + 10, 167, 2, 69, 2, 165, 165, 17, 4, 77, 65, 76, 76, 10, 108, 18, 87, 79, + 32, 68, 79, 84, 83, 32, 66, 69, 76, 79, 87, 32, 65, 78, 68, 32, 146, 220, + 36, 65, 183, 5, 72, 6, 70, 72, 132, 204, 6, 7, 83, 77, 65, 76, 76, 32, + 78, 207, 194, 31, 68, 2, 169, 236, 33, 3, 65, 77, 90, 18, 26, 72, 17, 2, + 73, 78, 11, 243, 71, 32, 9, 11, 32, 6, 158, 76, 70, 254, 52, 73, 157, 11, + 13, 87, 73, 84, 72, 32, 73, 78, 86, 69, 82, 84, 69, 68, 220, 7, 230, 5, + 65, 158, 5, 66, 176, 2, 9, 68, 65, 68, 32, 87, 73, 84, 72, 32, 80, 9, 70, + 69, 72, 32, 87, 73, 84, 72, 32, 72, 11, 71, 72, 65, 73, 78, 32, 87, 73, + 84, 72, 32, 154, 1, 72, 134, 3, 74, 190, 2, 75, 252, 1, 9, 76, 65, 77, + 32, 87, 73, 84, 72, 32, 226, 4, 77, 152, 4, 10, 78, 79, 79, 78, 32, 87, + 73, 84, 72, 32, 182, 2, 81, 190, 2, 82, 178, 2, 83, 146, 16, 84, 196, 9, + 7, 87, 65, 83, 65, 76, 76, 65, 40, 9, 89, 69, 72, 32, 87, 73, 84, 72, 32, + 216, 3, 53, 85, 73, 71, 72, 85, 82, 32, 75, 73, 82, 71, 72, 73, 90, 32, + 89, 69, 72, 32, 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, + 86, 69, 32, 87, 73, 84, 72, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, + 82, 65, 205, 6, 14, 90, 65, 72, 32, 87, 73, 84, 72, 32, 77, 69, 69, 77, + 32, 54, 128, 1, 8, 73, 78, 32, 87, 73, 84, 72, 32, 70, 76, 232, 57, 4, + 75, 66, 65, 82, 129, 197, 36, 8, 90, 90, 65, 32, 87, 65, 32, 74, 28, 152, + 19, 4, 77, 69, 69, 77, 150, 16, 74, 238, 23, 65, 255, 8, 89, 22, 56, 3, + 65, 89, 72, 216, 1, 3, 69, 70, 32, 207, 13, 76, 12, 30, 73, 122, 65, 151, + 56, 69, 8, 52, 9, 32, 65, 83, 45, 83, 65, 76, 65, 65, 47, 77, 4, 80, 4, + 84, 85, 32, 87, 207, 132, 39, 77, 4, 18, 65, 23, 32, 2, 17, 2, 65, 32, 2, + 137, 142, 33, 6, 65, 83, 45, 83, 65, 76, 8, 212, 65, 29, 77, 65, 75, 83, + 85, 82, 65, 32, 87, 73, 84, 72, 32, 83, 85, 80, 69, 82, 83, 67, 82, 73, + 80, 84, 32, 65, 76, 69, 70, 1, 13, 87, 73, 84, 72, 32, 70, 65, 84, 72, + 65, 84, 65, 78, 44, 160, 1, 8, 69, 72, 32, 87, 73, 84, 72, 32, 177, 222, + 25, 25, 73, 83, 77, 73, 76, 76, 65, 72, 32, 65, 82, 45, 82, 65, 72, 77, + 65, 78, 32, 65, 82, 45, 82, 65, 72, 42, 98, 72, 24, 3, 75, 72, 65, 230, + 52, 65, 250, 3, 74, 78, 77, 86, 78, 78, 90, 242, 2, 82, 43, 89, 10, 22, + 65, 171, 56, 69, 6, 131, 59, 72, 36, 222, 6, 72, 158, 7, 75, 218, 38, 65, + 250, 3, 74, 2, 77, 134, 5, 82, 3, 89, 30, 170, 13, 75, 218, 38, 65, 250, + 3, 74, 146, 2, 77, 110, 72, 139, 2, 89, 22, 64, 5, 77, 69, 69, 77, 32, + 250, 50, 65, 250, 3, 74, 135, 5, 89, 10, 40, 5, 87, 73, 84, 72, 32, 243, + 103, 73, 6, 174, 21, 77, 170, 25, 65, 203, 12, 89, 40, 80, 8, 65, 72, 32, + 87, 73, 84, 72, 32, 69, 8, 69, 72, 32, 87, 73, 84, 72, 32, 22, 236, 3, 4, + 77, 69, 69, 77, 226, 45, 65, 138, 6, 74, 247, 2, 89, 18, 164, 1, 5, 77, + 69, 69, 77, 32, 196, 11, 6, 65, 76, 69, 70, 32, 77, 150, 2, 89, 134, 38, + 74, 53, 16, 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, + 70, 8, 40, 5, 87, 73, 84, 72, 32, 223, 100, 73, 4, 254, 43, 74, 3, 77, + 30, 104, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 205, 129, 4, 11, 65, 76, + 76, 65, 74, 65, 76, 65, 76, 79, 85, 28, 62, 72, 60, 5, 77, 69, 69, 77, + 32, 186, 45, 65, 255, 8, 89, 8, 17, 2, 65, 72, 8, 11, 32, 8, 178, 29, 87, + 199, 69, 73, 12, 40, 5, 87, 73, 84, 72, 32, 179, 98, 73, 8, 166, 37, 72, + 242, 3, 65, 203, 12, 89, 64, 84, 8, 65, 70, 32, 87, 73, 84, 72, 32, 93, + 9, 72, 65, 72, 32, 87, 73, 84, 72, 32, 46, 168, 46, 2, 65, 76, 218, 1, + 74, 96, 2, 76, 65, 146, 2, 75, 14, 72, 30, 77, 239, 1, 89, 18, 54, 72, + 250, 42, 65, 250, 3, 74, 2, 77, 135, 5, 89, 2, 247, 8, 65, 74, 116, 5, + 65, 76, 69, 70, 32, 210, 1, 72, 124, 5, 74, 69, 69, 77, 32, 78, 75, 28, + 5, 77, 69, 69, 77, 32, 187, 47, 89, 20, 64, 5, 87, 73, 84, 72, 32, 138, + 44, 77, 222, 6, 70, 255, 52, 73, 12, 68, 6, 72, 65, 77, 90, 65, 32, 41, + 7, 77, 65, 68, 68, 65, 32, 65, 8, 38, 65, 249, 45, 4, 66, 69, 76, 79, 4, + 229, 45, 3, 66, 79, 86, 14, 28, 2, 65, 72, 163, 44, 69, 12, 11, 32, 12, + 40, 5, 87, 73, 84, 72, 32, 239, 92, 73, 8, 210, 35, 65, 130, 12, 77, 75, + 89, 14, 40, 5, 87, 73, 84, 72, 32, 159, 92, 73, 10, 130, 47, 74, 2, 77, + 75, 89, 8, 129, 15, 3, 72, 65, 72, 14, 150, 14, 87, 222, 33, 70, 238, 43, + 73, 199, 9, 77, 48, 84, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 193, 36, + 7, 79, 72, 65, 77, 77, 65, 68, 46, 116, 5, 65, 76, 69, 70, 32, 66, 72, 0, + 2, 75, 72, 104, 5, 74, 69, 69, 77, 32, 92, 5, 77, 69, 69, 77, 32, 43, 89, + 4, 22, 77, 219, 45, 70, 2, 173, 35, 6, 65, 75, 83, 85, 82, 65, 10, 21, 3, + 65, 72, 32, 10, 40, 5, 87, 73, 84, 72, 32, 219, 88, 73, 6, 250, 31, 74, + 2, 77, 143, 12, 89, 12, 40, 5, 87, 73, 84, 72, 32, 139, 88, 73, 8, 170, + 31, 77, 194, 7, 75, 14, 72, 195, 4, 89, 8, 246, 40, 87, 246, 2, 70, 239, + 43, 73, 2, 11, 69, 2, 143, 33, 72, 60, 134, 1, 72, 28, 5, 74, 69, 69, 77, + 32, 92, 5, 77, 69, 69, 77, 32, 242, 31, 65, 154, 5, 78, 78, 90, 134, 1, + 75, 238, 1, 82, 43, 89, 14, 146, 27, 65, 155, 9, 69, 16, 40, 5, 87, 73, + 84, 72, 32, 199, 85, 73, 12, 186, 24, 72, 242, 3, 65, 130, 12, 77, 75, + 89, 12, 206, 15, 87, 218, 25, 70, 238, 43, 73, 199, 9, 77, 28, 70, 65, + 209, 138, 32, 11, 85, 68, 68, 73, 83, 65, 32, 83, 73, 82, 82, 26, 64, 7, + 70, 32, 87, 73, 84, 72, 32, 193, 8, 4, 76, 65, 32, 85, 24, 64, 5, 77, 69, + 69, 77, 32, 238, 29, 65, 246, 6, 72, 139, 2, 89, 12, 40, 5, 87, 73, 84, + 72, 32, 231, 82, 73, 8, 34, 77, 186, 21, 72, 187, 16, 89, 2, 129, 38, 3, + 69, 69, 77, 18, 30, 65, 129, 26, 2, 69, 72, 16, 128, 1, 14, 68, 73, 32, + 65, 76, 76, 65, 65, 72, 85, 32, 65, 78, 72, 76, 6, 72, 73, 77, 65, 72, + 85, 161, 26, 4, 83, 79, 85, 76, 11, 26, 85, 231, 228, 38, 65, 6, 26, 77, + 255, 170, 37, 78, 5, 199, 228, 38, 65, 4, 26, 32, 1, 2, 77, 32, 2, 205, + 134, 32, 4, 65, 76, 76, 65, 190, 1, 138, 1, 65, 192, 6, 9, 69, 69, 78, + 32, 87, 73, 84, 72, 32, 250, 3, 72, 233, 5, 13, 85, 66, 72, 65, 65, 78, + 65, 72, 85, 32, 87, 65, 32, 46, 48, 7, 68, 32, 87, 73, 84, 72, 32, 251, + 1, 76, 32, 76, 4, 72, 65, 72, 32, 82, 77, 238, 23, 65, 138, 4, 75, 246, + 4, 82, 3, 89, 10, 22, 87, 203, 77, 73, 6, 25, 4, 73, 84, 72, 32, 6, 162, + 16, 72, 187, 16, 89, 8, 21, 3, 69, 69, 77, 8, 11, 32, 8, 208, 31, 6, 87, + 73, 84, 72, 32, 77, 155, 45, 73, 14, 26, 65, 77, 2, 76, 65, 4, 134, 22, + 77, 129, 218, 36, 11, 65, 77, 85, 72, 85, 32, 65, 76, 65, 89, 78, 10, 34, + 32, 133, 1, 3, 76, 76, 65, 4, 22, 85, 167, 84, 73, 2, 217, 8, 23, 83, 69, + 68, 32, 65, 83, 32, 75, 79, 82, 65, 78, 73, 67, 32, 83, 84, 79, 80, 32, + 83, 73, 71, 6, 98, 72, 237, 222, 38, 18, 65, 72, 85, 32, 65, 76, 65, 89, + 72, 73, 32, 87, 65, 45, 65, 65, 76, 73, 4, 228, 63, 12, 79, 85, 32, 65, + 76, 65, 89, 72, 69, 32, 87, 65, 1, 22, 85, 32, 65, 76, 65, 89, 72, 73, + 32, 87, 65, 65, 65, 76, 73, 72, 69, 69, 32, 87, 65, 45, 60, 130, 1, 72, + 100, 5, 74, 69, 69, 77, 32, 84, 5, 75, 72, 65, 72, 32, 92, 5, 77, 69, 69, + 77, 32, 234, 15, 65, 254, 8, 82, 3, 89, 12, 32, 3, 65, 72, 32, 159, 21, + 69, 8, 156, 14, 6, 87, 73, 84, 72, 32, 74, 222, 56, 73, 199, 9, 77, 10, + 52, 5, 87, 73, 84, 72, 32, 134, 70, 73, 199, 9, 77, 4, 234, 12, 65, 139, + 8, 72, 10, 34, 87, 198, 69, 73, 199, 9, 77, 4, 25, 4, 73, 84, 72, 32, 4, + 142, 12, 65, 203, 12, 89, 16, 52, 5, 87, 73, 84, 72, 32, 214, 68, 73, + 199, 9, 77, 10, 202, 7, 72, 174, 4, 74, 199, 11, 77, 82, 96, 10, 65, 68, + 68, 65, 32, 87, 73, 84, 72, 32, 161, 1, 9, 69, 69, 78, 32, 87, 73, 84, + 72, 32, 18, 96, 4, 68, 65, 77, 77, 0, 4, 75, 65, 83, 82, 138, 11, 83, + 245, 41, 6, 70, 65, 84, 72, 65, 32, 6, 11, 65, 6, 28, 2, 84, 65, 211, 52, + 32, 2, 163, 12, 78, 64, 130, 1, 72, 36, 5, 74, 69, 69, 77, 32, 52, 5, 77, + 69, 69, 77, 32, 170, 11, 65, 228, 4, 4, 75, 72, 65, 72, 154, 4, 82, 3, + 89, 18, 194, 15, 69, 229, 3, 2, 65, 72, 10, 158, 18, 87, 246, 2, 70, 238, + 43, 73, 199, 9, 77, 16, 64, 5, 87, 73, 84, 72, 32, 158, 20, 70, 238, 43, + 73, 199, 9, 77, 8, 252, 2, 2, 75, 72, 243, 15, 77, 122, 66, 65, 176, 2, + 8, 69, 72, 32, 87, 73, 84, 72, 32, 179, 4, 72, 28, 88, 11, 66, 65, 65, + 82, 65, 75, 65, 32, 87, 65, 45, 29, 7, 72, 32, 87, 73, 84, 72, 32, 2, + 245, 133, 28, 2, 84, 65, 26, 64, 5, 77, 69, 69, 77, 32, 194, 8, 65, 246, + 6, 72, 139, 2, 89, 14, 52, 5, 87, 73, 84, 72, 32, 174, 61, 73, 199, 9, + 77, 8, 34, 72, 174, 4, 77, 143, 12, 89, 4, 133, 16, 2, 65, 72, 66, 138, + 1, 72, 108, 3, 75, 72, 65, 12, 4, 74, 69, 69, 77, 92, 5, 77, 69, 69, 77, + 32, 238, 4, 65, 154, 5, 78, 78, 90, 242, 2, 82, 43, 89, 14, 32, 3, 65, + 72, 32, 227, 9, 69, 10, 40, 5, 87, 73, 84, 72, 32, 151, 59, 73, 6, 182, + 2, 77, 199, 11, 74, 10, 11, 72, 10, 11, 32, 10, 40, 5, 87, 73, 84, 72, + 32, 183, 58, 73, 6, 154, 1, 65, 62, 77, 143, 12, 89, 18, 64, 5, 87, 73, + 84, 72, 32, 226, 13, 70, 238, 43, 73, 199, 9, 77, 10, 50, 65, 62, 74, + 194, 7, 75, 14, 72, 195, 4, 89, 2, 217, 12, 11, 76, 69, 70, 32, 77, 65, + 75, 83, 85, 82, 65, 2, 225, 7, 3, 69, 69, 77, 28, 56, 2, 65, 76, 117, 8, + 69, 72, 32, 87, 73, 84, 72, 32, 2, 37, 7, 32, 87, 73, 84, 72, 32, 83, 2, + 197, 1, 15, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, + 26, 108, 3, 74, 69, 69, 126, 65, 198, 4, 77, 86, 78, 78, 90, 242, 2, 82, + 42, 89, 237, 53, 5, 72, 69, 72, 32, 77, 2, 11, 77, 2, 11, 32, 2, 151, 63, + 73, 114, 82, 65, 38, 72, 246, 4, 78, 78, 90, 38, 74, 98, 75, 42, 77, 198, + 1, 82, 43, 89, 4, 217, 2, 5, 76, 69, 70, 32, 77, 76, 22, 65, 139, 3, 69, + 72, 80, 15, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, 84, 72, 32, + 147, 5, 72, 66, 118, 65, 126, 69, 42, 72, 78, 74, 18, 75, 62, 77, 86, 78, + 22, 79, 16, 2, 87, 65, 18, 89, 26, 90, 242, 2, 82, 67, 85, 12, 22, 76, + 247, 6, 69, 8, 21, 3, 69, 70, 32, 8, 34, 77, 222, 6, 70, 255, 52, 73, 4, + 181, 6, 6, 65, 75, 83, 85, 82, 65, 6, 11, 32, 6, 166, 6, 70, 239, 43, 73, + 8, 22, 69, 191, 3, 65, 4, 11, 72, 4, 11, 32, 4, 230, 4, 73, 167, 54, 77, + 4, 131, 49, 69, 2, 11, 72, 2, 11, 65, 2, 11, 72, 2, 149, 4, 2, 32, 73, 8, + 17, 2, 69, 69, 8, 11, 77, 8, 11, 32, 8, 198, 4, 70, 238, 43, 73, 199, 9, + 77, 2, 93, 2, 79, 79, 4, 231, 3, 69, 4, 215, 3, 87, 8, 186, 3, 69, 15, + 85, 2, 17, 2, 65, 73, 2, 239, 2, 78, 6, 21, 3, 69, 69, 77, 6, 11, 32, 6, + 22, 87, 227, 46, 73, 2, 141, 2, 5, 73, 84, 72, 32, 89, 4, 11, 72, 4, 11, + 65, 4, 143, 46, 72, 14, 21, 3, 69, 69, 77, 14, 11, 32, 14, 64, 5, 87, 73, + 84, 72, 32, 194, 1, 70, 238, 43, 73, 199, 9, 77, 6, 18, 77, 75, 89, 4, + 21, 3, 69, 69, 77, 4, 11, 32, 4, 18, 73, 119, 70, 2, 247, 44, 78, 2, 17, + 2, 69, 72, 2, 77, 2, 32, 70, 4, 11, 69, 4, 11, 72, 4, 11, 32, 4, 22, 70, + 255, 52, 73, 2, 209, 53, 2, 73, 78, 6, 210, 43, 73, 199, 9, 77, 166, 2, + 92, 3, 68, 68, 65, 52, 3, 82, 75, 32, 109, 11, 84, 72, 69, 77, 65, 84, + 73, 67, 65, 76, 32, 4, 152, 133, 37, 5, 32, 87, 65, 65, 74, 239, 62, 72, + 4, 58, 78, 1, 10, 83, 73, 68, 69, 87, 65, 89, 83, 32, 78, 2, 173, 137, + 35, 7, 79, 79, 78, 32, 71, 72, 85, 158, 2, 174, 2, 68, 140, 3, 8, 73, 78, + 73, 84, 73, 65, 76, 32, 222, 1, 76, 132, 2, 9, 79, 80, 69, 82, 65, 84, + 79, 82, 32, 166, 1, 83, 142, 3, 84, 190, 162, 24, 65, 102, 75, 110, 90, + 134, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 82, 2, 89, 222, 7, 72, + 190, 59, 71, 242, 253, 2, 78, 166, 165, 2, 81, 135, 3, 87, 62, 26, 79, + 235, 186, 37, 65, 58, 88, 6, 84, 76, 69, 83, 83, 32, 61, 12, 85, 66, 76, + 69, 45, 83, 84, 82, 85, 67, 75, 32, 8, 226, 212, 31, 66, 2, 70, 138, 193, + 3, 78, 167, 165, 2, 81, 50, 178, 12, 83, 210, 20, 75, 170, 140, 24, 84, + 74, 90, 134, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 82, 2, 89, 222, + 7, 72, 190, 59, 71, 242, 253, 2, 78, 218, 235, 1, 76, 190, 56, 68, 146, + 1, 81, 222, 1, 65, 171, 1, 87, 40, 182, 1, 84, 166, 9, 83, 186, 160, 24, + 72, 30, 75, 242, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 89, 154, 67, + 71, 242, 253, 2, 78, 218, 235, 1, 76, 206, 57, 81, 222, 1, 65, 171, 57, + 68, 4, 150, 209, 31, 72, 207, 230, 6, 69, 56, 48, 6, 79, 79, 80, 69, 68, + 32, 199, 167, 38, 65, 54, 202, 8, 83, 210, 20, 75, 162, 139, 24, 65, 74, + 72, 66, 84, 74, 90, 134, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 82, + 2, 89, 154, 67, 71, 242, 253, 2, 78, 218, 235, 1, 76, 190, 56, 68, 146, + 1, 81, 135, 3, 87, 4, 168, 128, 29, 23, 77, 69, 69, 77, 32, 87, 73, 84, + 72, 32, 72, 65, 72, 32, 87, 73, 84, 72, 32, 84, 65, 84, 87, 221, 162, 8, + 9, 72, 65, 72, 32, 87, 73, 84, 72, 32, 52, 84, 9, 84, 82, 69, 84, 67, 72, + 69, 68, 32, 222, 224, 36, 72, 14, 69, 203, 141, 1, 65, 46, 178, 1, 68, + 86, 84, 250, 2, 83, 186, 160, 24, 72, 30, 75, 242, 107, 74, 2, 77, 178, + 187, 6, 66, 2, 70, 2, 89, 222, 7, 90, 190, 59, 71, 242, 253, 2, 78, 166, + 165, 2, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, 83, 83, 32, 199, 236, + 37, 65, 4, 134, 203, 31, 66, 3, 70, 6, 234, 202, 31, 72, 206, 230, 6, 65, + 3, 69, 38, 42, 65, 190, 163, 24, 72, 167, 141, 14, 69, 32, 44, 5, 73, 76, + 69, 68, 32, 195, 177, 38, 72, 30, 146, 1, 68, 94, 83, 210, 20, 75, 246, + 247, 24, 74, 178, 187, 6, 89, 222, 7, 72, 190, 59, 71, 242, 253, 2, 78, + 218, 235, 1, 76, 206, 57, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, 83, + 83, 32, 215, 233, 37, 65, 4, 158, 137, 35, 78, 167, 165, 2, 81, 6, 218, + 219, 36, 72, 14, 69, 203, 141, 1, 65, 4, 242, 197, 18, 77, 203, 226, 18, + 83, 6, 228, 235, 24, 3, 75, 65, 83, 12, 4, 68, 65, 77, 77, 1, 4, 70, 65, + 84, 72, 12, 118, 69, 38, 79, 172, 197, 12, 11, 76, 65, 67, 69, 32, 79, + 70, 32, 83, 65, 74, 165, 254, 5, 6, 73, 65, 83, 84, 82, 69, 4, 182, 157, + 32, 82, 235, 218, 5, 80, 4, 252, 181, 12, 8, 69, 84, 73, 67, 32, 86, 69, + 82, 209, 141, 6, 3, 85, 78, 68, 14, 238, 1, 65, 96, 5, 69, 86, 69, 82, + 83, 36, 15, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 72, 69, 65, 68, 32, + 225, 158, 23, 28, 79, 85, 78, 68, 69, 68, 32, 72, 73, 71, 72, 32, 83, 84, + 79, 80, 32, 87, 73, 84, 72, 32, 70, 73, 76, 76, 69, 68, 4, 40, 4, 73, 83, + 69, 68, 171, 170, 38, 89, 2, 17, 2, 32, 82, 2, 149, 222, 36, 3, 79, 85, + 78, 2, 181, 214, 21, 4, 69, 68, 32, 68, 6, 26, 65, 167, 139, 36, 66, 4, + 157, 133, 37, 3, 66, 79, 86, 202, 1, 238, 1, 69, 244, 2, 5, 72, 65, 68, + 68, 65, 36, 4, 73, 71, 78, 32, 240, 4, 5, 77, 65, 76, 76, 32, 246, 15, + 85, 240, 2, 6, 89, 77, 66, 79, 76, 32, 209, 208, 36, 18, 84, 65, 82, 84, + 32, 79, 70, 32, 82, 85, 66, 32, 69, 76, 32, 72, 73, 90, 20, 52, 7, 81, + 85, 69, 78, 67, 69, 32, 187, 219, 34, 77, 18, 184, 1, 26, 89, 69, 72, 32, + 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, + 73, 84, 72, 32, 209, 189, 31, 13, 78, 79, 79, 78, 32, 87, 73, 84, 72, 32, + 75, 69, 72, 16, 70, 65, 186, 166, 37, 87, 198, 89, 89, 150, 14, 79, 214, + 22, 69, 3, 85, 6, 36, 3, 76, 69, 70, 191, 164, 38, 69, 5, 159, 13, 32, 7, + 11, 32, 4, 246, 22, 73, 55, 77, 22, 132, 1, 2, 82, 65, 158, 1, 83, 188, + 1, 7, 65, 76, 65, 89, 72, 69, 32, 160, 15, 2, 77, 73, 209, 138, 34, 6, + 84, 65, 75, 72, 65, 76, 4, 132, 1, 13, 72, 77, 65, 84, 85, 76, 76, 65, + 72, 32, 65, 76, 65, 133, 180, 33, 13, 68, 73, 32, 65, 76, 76, 65, 72, 79, + 85, 32, 65, 78, 2, 235, 244, 37, 89, 12, 46, 65, 193, 1, 6, 73, 78, 68, + 72, 73, 32, 8, 136, 1, 18, 76, 76, 65, 76, 76, 65, 72, 79, 85, 32, 65, + 76, 65, 89, 72, 69, 32, 87, 170, 192, 31, 78, 194, 229, 4, 70, 217, 102, + 2, 77, 86, 2, 17, 2, 65, 83, 2, 189, 221, 15, 3, 83, 65, 76, 4, 152, 200, + 20, 12, 80, 79, 83, 84, 80, 79, 83, 73, 84, 73, 79, 78, 185, 136, 14, 2, + 65, 77, 106, 132, 1, 2, 70, 65, 32, 5, 72, 73, 71, 72, 32, 240, 11, 4, + 76, 79, 87, 32, 110, 75, 218, 189, 21, 68, 226, 235, 9, 89, 179, 233, 5, + 87, 4, 190, 3, 82, 147, 160, 36, 84, 74, 236, 2, 17, 68, 79, 84, 76, 69, + 83, 83, 32, 72, 69, 65, 68, 32, 79, 70, 32, 75, 22, 70, 102, 76, 142, 3, + 77, 116, 4, 78, 79, 79, 78, 18, 83, 78, 84, 38, 87, 196, 2, 3, 89, 69, + 72, 224, 175, 14, 18, 85, 80, 82, 73, 71, 72, 84, 32, 82, 69, 67, 84, 65, + 78, 71, 85, 76, 65, 236, 213, 8, 7, 82, 79, 85, 78, 68, 69, 68, 234, 126, + 90, 134, 107, 74, 222, 161, 12, 81, 223, 1, 65, 2, 255, 186, 31, 72, 4, + 24, 2, 65, 82, 31, 79, 2, 11, 83, 2, 175, 2, 73, 2, 253, 180, 26, 6, 79, + 84, 78, 79, 84, 69, 10, 60, 8, 73, 71, 65, 84, 85, 82, 69, 32, 213, 11, + 2, 65, 77, 8, 88, 10, 65, 76, 69, 70, 32, 87, 73, 84, 72, 32, 116, 3, 81, + 65, 70, 1, 3, 83, 65, 68, 4, 84, 8, 76, 65, 77, 32, 87, 73, 84, 72, 209, + 168, 36, 7, 89, 69, 72, 32, 66, 65, 82, 2, 181, 176, 31, 2, 32, 89, 2, 89, 20, 32, 87, 73, 84, 72, 32, 76, 65, 77, 32, 87, 73, 84, 72, 32, 65, - 76, 69, 70, 32, 2, 225, 193, 21, 3, 77, 65, 75, 6, 26, 69, 215, 201, 15, - 65, 4, 17, 2, 69, 77, 4, 17, 2, 32, 73, 4, 22, 78, 135, 9, 83, 2, 197, 9, - 2, 73, 84, 5, 207, 4, 32, 6, 252, 254, 34, 7, 73, 71, 78, 32, 83, 65, 70, - 202, 38, 69, 227, 138, 1, 65, 4, 194, 243, 22, 72, 159, 129, 14, 65, 20, - 40, 4, 79, 82, 68, 32, 147, 247, 35, 65, 18, 74, 65, 172, 1, 2, 83, 65, - 200, 214, 24, 3, 87, 65, 81, 143, 155, 11, 81, 10, 228, 225, 6, 3, 76, - 45, 74, 236, 253, 17, 3, 82, 45, 82, 132, 196, 6, 7, 84, 72, 45, 84, 72, - 65, 76, 240, 207, 4, 5, 78, 45, 78, 73, 83, 137, 92, 5, 83, 45, 83, 65, - 74, 4, 218, 206, 36, 75, 207, 36, 72, 5, 149, 150, 10, 12, 32, 66, 65, + 76, 69, 70, 32, 2, 165, 145, 22, 3, 77, 65, 75, 6, 26, 69, 191, 249, 15, + 65, 4, 17, 2, 69, 77, 4, 17, 2, 32, 73, 4, 22, 78, 135, 9, 83, 2, 193, 9, + 2, 73, 84, 5, 207, 4, 32, 6, 180, 155, 36, 7, 73, 71, 78, 32, 83, 65, 70, + 218, 38, 69, 203, 141, 1, 65, 4, 178, 196, 23, 72, 235, 207, 14, 65, 20, + 40, 4, 79, 82, 68, 32, 191, 150, 37, 65, 18, 74, 65, 172, 1, 2, 83, 65, + 172, 173, 25, 3, 87, 65, 81, 207, 227, 11, 81, 10, 192, 252, 6, 3, 76, + 45, 74, 244, 185, 18, 3, 82, 45, 82, 144, 226, 6, 7, 84, 72, 45, 84, 72, + 65, 76, 164, 250, 4, 5, 78, 45, 78, 73, 83, 161, 92, 5, 83, 45, 83, 65, + 74, 4, 150, 238, 37, 75, 207, 36, 72, 5, 193, 182, 10, 12, 32, 66, 65, 82, 82, 69, 69, 32, 87, 73, 84, 72, 20, 68, 5, 78, 79, 79, 78, 32, 70, - 87, 206, 150, 24, 77, 239, 137, 11, 83, 2, 33, 6, 87, 73, 84, 72, 32, 75, - 2, 11, 65, 2, 151, 173, 36, 83, 14, 40, 4, 79, 82, 68, 32, 255, 242, 35, - 65, 12, 106, 73, 198, 194, 15, 77, 204, 135, 9, 3, 84, 65, 83, 212, 192, - 8, 2, 83, 65, 237, 172, 3, 3, 81, 65, 83, 4, 252, 230, 12, 2, 77, 65, - 177, 189, 18, 3, 83, 72, 77, 12, 138, 1, 66, 64, 3, 75, 85, 78, 177, 152, + 87, 234, 237, 24, 77, 155, 207, 11, 83, 2, 33, 6, 87, 73, 84, 72, 32, 75, + 2, 11, 65, 2, 199, 204, 37, 83, 14, 40, 4, 79, 82, 68, 32, 171, 146, 37, + 65, 12, 106, 73, 174, 242, 15, 77, 148, 175, 9, 3, 84, 65, 83, 200, 218, + 8, 2, 83, 65, 249, 218, 3, 3, 81, 65, 83, 4, 156, 148, 13, 2, 77, 65, + 253, 132, 19, 3, 83, 72, 77, 12, 138, 1, 66, 64, 3, 75, 85, 78, 205, 239, 24, 22, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 32, 77, - 79, 75, 72, 65, 83, 83, 2, 33, 6, 83, 67, 82, 73, 80, 84, 2, 229, 204, - 21, 2, 32, 65, 9, 11, 32, 6, 34, 73, 58, 77, 255, 212, 34, 66, 2, 11, 83, - 2, 141, 239, 35, 6, 79, 76, 65, 84, 69, 68, 2, 11, 69, 2, 11, 68, 2, 11, - 73, 2, 189, 238, 35, 2, 65, 76, 34, 152, 1, 2, 68, 79, 114, 84, 224, 145, - 5, 8, 83, 77, 65, 76, 76, 32, 84, 65, 156, 229, 25, 4, 70, 79, 85, 82, - 228, 124, 4, 87, 65, 83, 76, 147, 166, 4, 82, 6, 88, 16, 85, 66, 76, 69, - 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 159, 210, 34, 84, 2, 151, - 196, 34, 82, 16, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 121, 8, - 87, 79, 32, 68, 79, 84, 83, 32, 8, 184, 246, 30, 17, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 154, 219, 3, 66, 223, - 155, 1, 65, 8, 176, 208, 34, 10, 86, 69, 82, 84, 73, 67, 65, 76, 76, 89, - 42, 66, 223, 155, 1, 65, 30, 202, 1, 65, 152, 2, 4, 79, 78, 69, 32, 244, - 209, 10, 12, 82, 73, 80, 76, 69, 32, 68, 79, 84, 32, 80, 85, 152, 164, 6, - 8, 85, 82, 78, 69, 68, 32, 68, 65, 149, 222, 13, 8, 72, 79, 85, 83, 65, - 78, 68, 83, 12, 68, 5, 84, 87, 69, 69, 76, 157, 180, 9, 6, 73, 76, 32, + 79, 75, 72, 65, 83, 83, 2, 33, 6, 83, 67, 82, 73, 80, 84, 2, 169, 156, + 22, 2, 32, 65, 9, 11, 32, 6, 34, 73, 54, 77, 231, 238, 35, 66, 2, 11, 83, + 2, 213, 204, 31, 5, 79, 76, 65, 84, 69, 2, 11, 69, 2, 11, 68, 2, 11, 73, + 2, 237, 141, 37, 2, 65, 76, 34, 152, 1, 2, 68, 79, 114, 84, 248, 172, 5, + 8, 83, 77, 65, 76, 76, 32, 84, 65, 244, 190, 26, 4, 70, 79, 85, 82, 204, + 123, 4, 87, 65, 83, 76, 239, 209, 4, 82, 6, 88, 16, 85, 66, 76, 69, 32, + 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 135, 236, 35, 84, 2, 247, + 220, 35, 82, 16, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 121, 8, + 87, 79, 32, 68, 79, 84, 83, 32, 8, 168, 235, 31, 17, 80, 79, 73, 78, 84, + 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 146, 128, 4, 66, 167, + 161, 1, 65, 8, 152, 234, 35, 10, 86, 69, 82, 84, 73, 67, 65, 76, 76, 89, + 42, 66, 167, 161, 1, 65, 30, 202, 1, 65, 152, 2, 4, 79, 78, 69, 32, 168, + 244, 10, 12, 82, 73, 80, 76, 69, 32, 68, 79, 84, 32, 80, 85, 144, 197, 6, + 8, 85, 82, 78, 69, 68, 32, 68, 65, 217, 143, 14, 8, 72, 79, 85, 83, 65, + 78, 68, 83, 12, 68, 5, 84, 87, 69, 69, 76, 153, 235, 35, 6, 73, 76, 32, 70, 82, 65, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 112, 11, 79, 86, 69, - 82, 83, 84, 82, 85, 67, 75, 32, 184, 166, 5, 7, 70, 65, 84, 72, 65, 84, - 65, 131, 226, 4, 84, 4, 26, 72, 223, 230, 35, 87, 2, 197, 254, 28, 2, 65, - 77, 12, 68, 3, 79, 78, 69, 234, 159, 29, 84, 205, 171, 5, 4, 76, 79, 79, - 80, 4, 213, 245, 32, 2, 32, 68, 8, 64, 10, 79, 87, 69, 76, 32, 83, 73, - 71, 78, 32, 243, 178, 23, 69, 6, 72, 10, 73, 78, 86, 69, 82, 84, 69, 68, - 32, 83, 2, 83, 147, 236, 24, 68, 2, 25, 4, 77, 65, 76, 76, 2, 213, 229, - 35, 2, 32, 86, 28, 104, 4, 80, 69, 82, 32, 136, 214, 5, 3, 67, 85, 66, - 208, 128, 5, 5, 70, 79, 85, 82, 84, 211, 156, 24, 68, 4, 250, 182, 24, + 82, 83, 84, 82, 85, 67, 75, 32, 208, 193, 5, 7, 70, 65, 84, 72, 65, 84, + 65, 155, 231, 4, 84, 4, 26, 72, 143, 134, 37, 87, 2, 169, 228, 29, 2, 65, + 77, 12, 68, 3, 79, 78, 69, 214, 133, 30, 84, 201, 223, 5, 4, 76, 79, 79, + 80, 4, 221, 232, 33, 2, 32, 68, 8, 64, 10, 79, 87, 69, 76, 32, 83, 73, + 71, 78, 32, 183, 137, 24, 69, 6, 72, 10, 73, 78, 86, 69, 82, 84, 69, 68, + 32, 83, 2, 83, 255, 194, 25, 68, 2, 25, 4, 77, 65, 76, 76, 2, 133, 133, + 37, 2, 32, 86, 28, 104, 4, 80, 69, 82, 32, 232, 240, 5, 3, 67, 85, 66, + 156, 137, 5, 5, 70, 79, 85, 82, 84, 227, 149, 25, 68, 4, 158, 142, 25, 77, 31, 84, 188, 1, 222, 1, 65, 34, 67, 138, 3, 69, 60, 7, 83, 77, 65, - 76, 76, 32, 76, 240, 176, 20, 17, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, - 69, 84, 84, 69, 82, 32, 76, 222, 225, 8, 72, 184, 168, 2, 3, 68, 82, 65, - 218, 213, 2, 70, 83, 81, 4, 214, 215, 30, 66, 135, 26, 80, 78, 80, 14, - 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 187, 147, 34, 79, + 76, 76, 32, 76, 152, 128, 21, 17, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, + 69, 84, 84, 69, 82, 32, 76, 162, 248, 8, 72, 252, 182, 2, 3, 68, 82, 65, + 138, 250, 2, 70, 83, 81, 4, 198, 204, 31, 66, 139, 26, 80, 78, 80, 14, + 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 155, 172, 35, 79, 76, 250, 1, 84, 36, 2, 89, 73, 150, 3, 67, 38, 82, 34, 69, 42, 71, 34, - 74, 38, 75, 22, 80, 38, 83, 106, 65, 22, 86, 158, 1, 76, 254, 161, 30, - 70, 2, 88, 170, 227, 1, 73, 134, 255, 2, 66, 2, 77, 234, 54, 78, 226, 26, - 90, 186, 96, 72, 190, 28, 68, 171, 1, 79, 4, 178, 139, 35, 73, 163, 209, - 1, 79, 5, 219, 140, 36, 87, 4, 216, 135, 25, 6, 77, 80, 72, 65, 83, 73, - 191, 233, 5, 88, 92, 76, 6, 69, 84, 84, 69, 82, 32, 157, 5, 8, 73, 71, - 65, 84, 85, 82, 69, 32, 80, 242, 1, 67, 38, 82, 34, 69, 42, 71, 34, 74, - 38, 75, 22, 80, 38, 83, 34, 84, 74, 65, 22, 86, 32, 2, 89, 73, 126, 76, - 254, 161, 30, 70, 2, 88, 170, 227, 1, 73, 134, 255, 2, 66, 2, 77, 234, - 54, 78, 226, 26, 90, 186, 96, 72, 190, 28, 68, 171, 1, 79, 8, 34, 72, - 250, 216, 36, 65, 3, 79, 4, 230, 215, 36, 69, 147, 1, 65, 6, 198, 215, - 36, 67, 146, 1, 72, 3, 84, 4, 142, 146, 36, 72, 203, 53, 73, 4, 238, 164, - 30, 72, 163, 179, 6, 65, 4, 207, 225, 32, 69, 4, 166, 217, 3, 73, 163, - 253, 32, 69, 4, 250, 212, 36, 72, 171, 1, 69, 6, 68, 7, 85, 82, 78, 69, - 68, 32, 65, 174, 133, 35, 73, 163, 209, 1, 79, 2, 167, 156, 35, 89, 4, - 166, 216, 35, 69, 147, 126, 79, 7, 222, 216, 20, 32, 231, 173, 15, 87, - 12, 84, 5, 69, 67, 72, 32, 89, 20, 4, 77, 69, 78, 32, 133, 136, 29, 4, - 86, 69, 87, 32, 2, 251, 131, 35, 73, 8, 230, 161, 30, 88, 170, 227, 1, - 73, 238, 181, 3, 78, 231, 119, 69, 14, 116, 10, 32, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 177, 151, 35, 13, 72, 69, 65, 68, 45, 83, 72, 65, 80, 69, - 68, 32, 80, 12, 156, 2, 24, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, - 84, 72, 69, 78, 32, 67, 85, 82, 86, 73, 78, 71, 32, 48, 16, 85, 80, 87, - 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 78, 79, 82, 201, 248, 33, 22, 68, - 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 67, 85, 82, 86, - 73, 78, 71, 6, 44, 3, 83, 79, 85, 210, 239, 25, 68, 35, 85, 2, 173, 173, - 28, 4, 84, 72, 32, 87, 4, 144, 135, 30, 10, 67, 85, 76, 65, 84, 69, 68, - 32, 76, 79, 137, 133, 5, 6, 83, 84, 32, 80, 65, 76, 20, 58, 67, 38, 84, - 146, 200, 22, 89, 241, 156, 9, 2, 83, 69, 4, 222, 141, 8, 69, 175, 128, - 11, 73, 12, 48, 4, 69, 82, 73, 83, 36, 2, 79, 78, 31, 82, 6, 238, 192, 2, - 75, 247, 141, 34, 77, 2, 209, 199, 13, 2, 73, 83, 4, 226, 254, 29, 65, - 189, 204, 4, 22, 79, 78, 79, 77, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, - 76, 32, 70, 79, 82, 32, 85, 82, 4, 216, 219, 18, 6, 72, 76, 69, 84, 73, - 67, 189, 222, 16, 2, 79, 77, 10, 68, 2, 84, 79, 240, 142, 21, 2, 83, 84, - 185, 162, 2, 3, 66, 69, 82, 6, 54, 77, 201, 205, 35, 7, 32, 82, 73, 67, - 75, 83, 72, 4, 218, 176, 23, 79, 153, 182, 5, 12, 65, 84, 69, 68, 32, 84, - 69, 76, 76, 69, 82, 32, 112, 56, 6, 69, 83, 84, 65, 78, 32, 133, 196, 4, - 2, 79, 67, 110, 52, 7, 76, 69, 84, 84, 69, 82, 32, 167, 194, 30, 65, 108, - 234, 1, 65, 58, 71, 42, 72, 34, 78, 50, 88, 42, 83, 42, 89, 34, 84, 246, - 246, 32, 85, 158, 144, 1, 79, 182, 56, 73, 198, 134, 1, 66, 2, 68, 2, 90, - 246, 63, 69, 194, 41, 67, 2, 70, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, - 82, 3, 86, 17, 254, 130, 34, 65, 158, 133, 2, 69, 150, 64, 78, 3, 79, 6, - 162, 177, 36, 71, 2, 72, 215, 22, 69, 4, 250, 176, 36, 77, 215, 22, 69, - 12, 46, 71, 174, 176, 36, 78, 2, 89, 215, 22, 69, 6, 170, 176, 36, 86, 2, - 89, 215, 22, 69, 8, 38, 72, 166, 153, 36, 83, 143, 45, 69, 4, 218, 175, - 36, 89, 215, 22, 69, 6, 186, 175, 36, 72, 2, 84, 215, 22, 69, 134, 37, - 178, 1, 65, 182, 169, 1, 69, 132, 16, 9, 72, 65, 73, 75, 83, 85, 75, 73, - 32, 198, 6, 73, 130, 3, 76, 186, 34, 79, 130, 55, 82, 138, 19, 85, 138, - 8, 89, 222, 139, 34, 80, 147, 1, 83, 182, 14, 132, 1, 2, 66, 89, 94, 67, - 206, 2, 68, 146, 1, 71, 106, 76, 140, 28, 4, 77, 85, 77, 32, 230, 114, - 78, 178, 1, 82, 98, 83, 223, 6, 84, 11, 11, 32, 8, 238, 188, 12, 67, 134, - 212, 5, 66, 204, 209, 13, 3, 65, 78, 71, 227, 205, 3, 83, 16, 62, 75, - 172, 223, 8, 5, 84, 82, 73, 65, 78, 139, 147, 27, 79, 12, 42, 32, 46, 83, - 229, 186, 10, 2, 45, 84, 4, 178, 220, 9, 87, 161, 145, 20, 2, 79, 70, 6, - 132, 1, 26, 76, 65, 78, 84, 69, 68, 32, 83, 79, 85, 84, 72, 32, 65, 82, - 82, 79, 87, 32, 87, 73, 84, 72, 32, 72, 79, 175, 131, 36, 80, 4, 150, - 215, 28, 82, 245, 130, 5, 2, 79, 75, 4, 188, 215, 33, 27, 77, 73, 78, 84, - 79, 78, 32, 82, 65, 67, 81, 85, 69, 84, 32, 65, 78, 68, 32, 83, 72, 85, - 84, 84, 76, 69, 67, 223, 170, 2, 71, 6, 228, 247, 29, 6, 85, 69, 84, 84, - 69, 32, 234, 189, 5, 69, 229, 6, 8, 71, 65, 71, 69, 32, 67, 76, 65, 150, - 2, 44, 6, 73, 78, 69, 83, 69, 32, 131, 24, 76, 248, 1, 188, 2, 6, 67, 65, - 82, 73, 75, 32, 104, 7, 76, 69, 84, 84, 69, 82, 32, 224, 8, 15, 77, 85, - 83, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 196, 6, 2, 80, 65, - 120, 5, 83, 73, 71, 78, 32, 180, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, - 71, 78, 32, 192, 246, 13, 5, 65, 68, 69, 71, 32, 162, 204, 18, 87, 219, - 247, 1, 68, 6, 32, 2, 80, 65, 255, 230, 2, 83, 4, 156, 130, 29, 5, 77, - 85, 78, 71, 75, 213, 231, 5, 3, 82, 69, 82, 110, 166, 2, 65, 112, 2, 66, - 65, 32, 2, 67, 65, 32, 2, 68, 65, 110, 69, 32, 2, 71, 65, 30, 73, 2, 79, - 2, 85, 36, 2, 74, 65, 30, 75, 68, 2, 76, 65, 18, 78, 72, 2, 80, 65, 36, - 2, 82, 65, 16, 2, 83, 65, 54, 84, 128, 1, 2, 86, 69, 0, 3, 90, 65, 76, - 170, 175, 36, 72, 2, 77, 2, 87, 3, 89, 10, 226, 2, 75, 184, 3, 5, 83, 89, - 85, 82, 65, 172, 243, 14, 8, 82, 67, 72, 65, 73, 67, 32, 74, 243, 217, 3, - 73, 5, 213, 165, 18, 3, 32, 75, 69, 5, 221, 192, 22, 3, 32, 76, 65, 9, - 17, 2, 32, 77, 6, 44, 5, 85, 82, 68, 65, 32, 175, 208, 32, 65, 4, 166, - 129, 14, 77, 25, 3, 65, 76, 80, 4, 254, 3, 70, 163, 237, 35, 75, 5, 185, - 202, 33, 2, 32, 71, 4, 11, 75, 4, 229, 14, 2, 65, 82, 5, 133, 169, 34, 2, - 32, 74, 8, 34, 65, 225, 2, 3, 72, 79, 84, 7, 222, 2, 70, 247, 252, 13, - 32, 7, 199, 12, 32, 8, 34, 65, 182, 177, 36, 71, 3, 89, 5, 221, 154, 26, - 4, 32, 82, 65, 77, 5, 157, 170, 35, 4, 32, 75, 65, 80, 7, 235, 11, 32, 7, - 21, 3, 32, 83, 65, 4, 194, 176, 36, 71, 3, 80, 10, 30, 65, 97, 3, 90, 73, - 82, 9, 11, 32, 6, 196, 253, 13, 6, 77, 85, 82, 68, 65, 32, 208, 7, 3, 76, - 65, 84, 151, 232, 17, 84, 2, 229, 249, 13, 2, 32, 83, 56, 168, 1, 10, 67, - 79, 77, 66, 73, 78, 73, 78, 71, 32, 246, 1, 68, 172, 1, 10, 76, 69, 70, - 84, 45, 72, 65, 78, 68, 32, 117, 11, 82, 73, 71, 72, 84, 45, 72, 65, 78, - 68, 32, 18, 132, 1, 4, 75, 69, 77, 80, 78, 74, 180, 181, 19, 2, 66, 69, - 136, 159, 7, 3, 69, 78, 68, 232, 166, 3, 3, 84, 69, 71, 195, 246, 3, 71, - 8, 32, 2, 76, 73, 1, 2, 85, 76, 5, 37, 7, 32, 87, 73, 84, 72, 32, 74, 2, - 225, 243, 13, 3, 69, 71, 79, 20, 50, 65, 90, 69, 174, 220, 35, 73, 2, 79, - 3, 85, 10, 40, 2, 78, 71, 218, 220, 35, 69, 3, 73, 7, 11, 32, 4, 158, 4, - 83, 227, 203, 10, 71, 4, 170, 220, 35, 85, 155, 80, 78, 10, 76, 6, 79, - 80, 69, 78, 32, 80, 113, 9, 67, 76, 79, 83, 69, 68, 32, 80, 76, 6, 186, - 219, 35, 65, 2, 73, 3, 85, 8, 72, 8, 67, 76, 79, 83, 69, 68, 32, 84, 29, - 6, 79, 80, 69, 78, 32, 68, 4, 158, 168, 36, 65, 3, 85, 4, 198, 170, 36, - 65, 3, 85, 10, 30, 77, 49, 3, 78, 84, 73, 6, 44, 3, 65, 68, 65, 129, 238, - 33, 2, 69, 78, 5, 233, 209, 35, 5, 32, 76, 65, 78, 84, 12, 110, 83, 20, - 4, 85, 76, 85, 32, 188, 240, 28, 3, 66, 73, 83, 150, 134, 1, 67, 153, - 248, 1, 4, 82, 69, 82, 69, 2, 247, 146, 24, 85, 4, 130, 245, 19, 67, 213, - 161, 16, 3, 82, 73, 67, 30, 120, 3, 76, 65, 32, 32, 3, 82, 65, 32, 12, 4, + 74, 38, 75, 22, 80, 38, 83, 106, 65, 22, 86, 158, 1, 76, 130, 141, 31, + 70, 2, 88, 218, 235, 1, 73, 158, 168, 3, 66, 2, 77, 198, 57, 78, 226, 26, + 90, 210, 96, 72, 190, 28, 68, 171, 1, 79, 4, 254, 167, 36, 73, 151, 212, + 1, 79, 5, 143, 172, 37, 87, 4, 138, 230, 31, 88, 169, 198, 4, 6, 77, 80, + 72, 65, 83, 73, 92, 76, 6, 69, 84, 84, 69, 82, 32, 157, 5, 8, 73, 71, 65, + 84, 85, 82, 69, 32, 80, 242, 1, 67, 38, 82, 34, 69, 42, 71, 34, 74, 38, + 75, 22, 80, 38, 83, 34, 84, 74, 65, 22, 86, 32, 2, 89, 73, 126, 76, 130, + 141, 31, 70, 2, 88, 218, 235, 1, 73, 158, 168, 3, 66, 2, 77, 198, 57, 78, + 226, 26, 90, 210, 96, 72, 190, 28, 68, 171, 1, 79, 8, 34, 72, 186, 248, + 37, 65, 3, 79, 4, 166, 247, 37, 69, 147, 1, 65, 6, 134, 247, 37, 67, 146, + 1, 72, 3, 84, 4, 194, 177, 37, 72, 215, 53, 73, 4, 242, 143, 31, 72, 223, + 231, 6, 65, 4, 211, 212, 33, 69, 4, 194, 244, 3, 73, 199, 129, 34, 69, 4, + 186, 244, 37, 72, 171, 1, 69, 6, 68, 7, 85, 82, 78, 69, 68, 32, 65, 250, + 161, 36, 73, 151, 212, 1, 79, 2, 167, 186, 36, 89, 4, 214, 247, 36, 69, + 163, 126, 79, 7, 146, 168, 21, 32, 231, 253, 15, 87, 12, 84, 5, 69, 67, + 72, 32, 89, 20, 4, 77, 69, 78, 32, 241, 237, 29, 4, 86, 69, 87, 32, 2, + 199, 160, 36, 73, 8, 234, 140, 31, 88, 218, 235, 1, 73, 226, 225, 3, 78, + 255, 119, 69, 14, 108, 10, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 169, + 181, 36, 11, 72, 69, 65, 68, 45, 83, 72, 65, 80, 69, 68, 12, 156, 2, 24, + 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 67, 85, + 82, 86, 73, 78, 71, 32, 48, 16, 85, 80, 87, 65, 82, 68, 83, 32, 84, 72, + 69, 78, 32, 78, 79, 82, 161, 146, 35, 22, 68, 79, 87, 78, 87, 65, 82, 68, + 83, 32, 84, 72, 69, 78, 32, 67, 85, 82, 86, 73, 78, 71, 6, 44, 3, 83, 79, + 85, 198, 209, 26, 68, 35, 85, 2, 149, 147, 29, 4, 84, 72, 32, 87, 4, 168, + 240, 30, 10, 67, 85, 76, 65, 84, 69, 68, 32, 76, 79, 237, 184, 5, 6, 83, + 84, 32, 80, 65, 76, 20, 58, 67, 38, 84, 142, 153, 23, 89, 173, 191, 9, 2, + 83, 69, 4, 206, 168, 8, 69, 231, 180, 11, 73, 12, 48, 4, 69, 82, 73, 83, + 36, 2, 79, 78, 31, 82, 6, 146, 220, 2, 75, 155, 146, 35, 77, 2, 217, 246, + 13, 2, 73, 83, 4, 250, 231, 30, 65, 221, 255, 4, 22, 79, 78, 79, 77, 73, + 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 85, 82, 4, + 132, 171, 19, 6, 72, 76, 69, 84, 73, 67, 193, 174, 17, 2, 79, 77, 10, 68, + 2, 84, 79, 200, 222, 21, 2, 83, 84, 181, 169, 2, 3, 66, 69, 82, 6, 54, + 77, 129, 237, 36, 7, 32, 82, 73, 67, 75, 83, 72, 4, 174, 135, 24, 79, + 189, 197, 5, 12, 65, 84, 69, 68, 32, 84, 69, 76, 76, 69, 82, 32, 112, 56, + 6, 69, 83, 84, 65, 78, 32, 165, 223, 4, 2, 79, 67, 110, 52, 7, 76, 69, + 84, 84, 69, 82, 32, 159, 183, 31, 65, 108, 234, 1, 65, 58, 71, 42, 72, + 34, 78, 50, 88, 42, 83, 42, 89, 34, 84, 178, 146, 34, 85, 202, 141, 1, + 79, 134, 60, 73, 190, 137, 1, 66, 2, 68, 2, 90, 130, 64, 69, 206, 41, 67, + 2, 70, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 3, 86, 17, 230, 155, 35, + 65, 242, 139, 2, 69, 162, 64, 78, 3, 79, 6, 234, 208, 37, 71, 2, 72, 215, + 22, 69, 4, 194, 208, 37, 77, 215, 22, 69, 12, 46, 71, 246, 207, 37, 78, + 2, 89, 215, 22, 69, 6, 242, 207, 37, 86, 2, 89, 215, 22, 69, 8, 38, 72, + 238, 184, 37, 83, 143, 45, 69, 4, 162, 207, 37, 89, 215, 22, 69, 6, 130, + 207, 37, 72, 2, 84, 215, 22, 69, 192, 41, 178, 1, 65, 242, 170, 1, 69, + 252, 15, 9, 72, 65, 73, 75, 83, 85, 75, 73, 32, 202, 6, 73, 130, 3, 76, + 182, 45, 79, 138, 70, 82, 242, 18, 85, 138, 8, 89, 130, 144, 35, 80, 147, + 1, 83, 190, 14, 132, 1, 2, 66, 89, 94, 67, 206, 2, 68, 146, 1, 71, 106, + 76, 192, 29, 4, 77, 85, 77, 32, 238, 114, 78, 178, 1, 82, 98, 83, 223, 6, + 84, 11, 11, 32, 8, 190, 234, 12, 67, 230, 241, 5, 66, 216, 249, 13, 3, + 65, 78, 71, 215, 249, 3, 83, 16, 62, 75, 160, 251, 8, 5, 84, 82, 73, 65, + 78, 211, 150, 28, 79, 12, 42, 32, 46, 83, 153, 222, 10, 2, 45, 84, 4, + 234, 252, 9, 87, 217, 218, 20, 2, 79, 70, 6, 132, 1, 26, 76, 65, 78, 84, + 69, 68, 32, 83, 79, 85, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, + 72, 32, 72, 79, 235, 162, 37, 80, 4, 130, 189, 29, 82, 221, 182, 5, 2, + 79, 75, 4, 144, 241, 34, 27, 77, 73, 78, 84, 79, 78, 32, 82, 65, 67, 81, + 85, 69, 84, 32, 65, 78, 68, 32, 83, 72, 85, 84, 84, 76, 69, 67, 199, 176, + 2, 71, 6, 192, 226, 30, 6, 85, 69, 84, 84, 69, 32, 190, 242, 5, 69, 229, + 6, 8, 71, 65, 71, 69, 32, 67, 76, 65, 156, 2, 44, 6, 73, 78, 69, 83, 69, + 32, 183, 25, 76, 254, 1, 132, 3, 6, 67, 65, 82, 73, 75, 32, 84, 15, 73, + 78, 86, 69, 82, 84, 69, 68, 32, 67, 65, 82, 73, 75, 32, 68, 7, 76, 69, + 84, 84, 69, 82, 32, 224, 8, 15, 77, 85, 83, 73, 67, 65, 76, 32, 83, 89, + 77, 66, 79, 76, 32, 196, 6, 2, 80, 65, 184, 1, 5, 83, 73, 71, 78, 32, + 176, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 224, 164, 14, 5, + 65, 68, 69, 71, 32, 254, 141, 19, 87, 239, 162, 2, 68, 6, 32, 2, 80, 65, + 219, 129, 3, 83, 4, 86, 82, 145, 249, 30, 5, 77, 85, 78, 71, 75, 4, 36, + 3, 80, 65, 82, 131, 129, 3, 83, 2, 165, 133, 36, 2, 69, 82, 110, 166, 2, + 65, 112, 2, 66, 65, 32, 2, 67, 65, 32, 2, 68, 65, 110, 69, 32, 2, 71, 65, + 30, 73, 2, 79, 2, 85, 36, 2, 74, 65, 30, 75, 68, 2, 76, 65, 18, 78, 72, + 2, 80, 65, 36, 2, 82, 65, 16, 2, 83, 65, 54, 84, 128, 1, 2, 86, 69, 0, 3, + 90, 65, 76, 250, 205, 37, 72, 2, 77, 2, 87, 3, 89, 10, 226, 2, 75, 184, + 3, 5, 83, 89, 85, 82, 65, 172, 162, 15, 8, 82, 67, 72, 65, 73, 67, 32, + 74, 167, 249, 3, 73, 5, 165, 244, 18, 3, 32, 75, 69, 5, 129, 248, 30, 3, + 32, 76, 65, 9, 17, 2, 32, 77, 6, 44, 5, 85, 82, 68, 65, 32, 231, 192, 33, + 65, 4, 130, 176, 14, 77, 25, 3, 65, 76, 80, 4, 254, 3, 70, 231, 139, 37, + 75, 5, 137, 227, 34, 2, 32, 71, 4, 11, 75, 4, 161, 15, 2, 65, 82, 5, 193, + 196, 35, 2, 32, 74, 8, 34, 65, 225, 2, 3, 72, 79, 84, 7, 222, 2, 70, 211, + 171, 14, 32, 7, 131, 13, 32, 8, 34, 65, 134, 208, 37, 71, 3, 89, 5, 173, + 252, 26, 4, 32, 82, 65, 77, 5, 213, 200, 36, 4, 32, 75, 65, 80, 7, 167, + 12, 32, 7, 21, 3, 32, 83, 65, 4, 146, 207, 37, 71, 3, 80, 10, 30, 65, 97, + 3, 90, 73, 82, 9, 11, 32, 6, 160, 172, 14, 6, 77, 85, 82, 68, 65, 32, + 208, 7, 3, 76, 65, 84, 255, 171, 18, 84, 2, 193, 168, 14, 2, 32, 83, 56, + 168, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 246, 1, 68, 172, 1, + 10, 76, 69, 70, 84, 45, 72, 65, 78, 68, 32, 117, 11, 82, 73, 71, 72, 84, + 45, 72, 65, 78, 68, 32, 18, 132, 1, 4, 75, 69, 77, 80, 78, 74, 240, 131, + 20, 2, 66, 69, 180, 182, 7, 3, 69, 78, 68, 148, 171, 3, 3, 84, 69, 71, + 159, 164, 4, 71, 8, 32, 2, 76, 73, 1, 2, 85, 76, 5, 37, 7, 32, 87, 73, + 84, 72, 32, 74, 2, 189, 162, 14, 3, 69, 71, 79, 20, 50, 65, 90, 69, 242, + 250, 36, 73, 2, 79, 3, 85, 10, 40, 2, 78, 71, 158, 251, 36, 69, 3, 73, 7, + 11, 32, 4, 218, 4, 83, 155, 244, 10, 71, 4, 238, 250, 36, 85, 167, 80, + 78, 10, 76, 6, 79, 80, 69, 78, 32, 80, 113, 9, 67, 76, 79, 83, 69, 68, + 32, 80, 76, 6, 254, 249, 36, 65, 2, 73, 3, 85, 8, 72, 8, 67, 76, 79, 83, + 69, 68, 32, 84, 29, 6, 79, 80, 69, 78, 32, 68, 4, 238, 198, 37, 65, 3, + 85, 4, 150, 201, 37, 65, 3, 85, 12, 30, 77, 69, 3, 78, 84, 73, 6, 44, 3, + 65, 68, 65, 241, 133, 35, 2, 69, 78, 5, 69, 2, 32, 76, 7, 11, 32, 4, 38, + 76, 245, 146, 35, 3, 66, 65, 87, 2, 225, 239, 36, 3, 65, 78, 84, 12, 106, + 83, 20, 4, 85, 76, 85, 32, 170, 224, 30, 67, 240, 6, 3, 66, 73, 83, 217, + 249, 1, 4, 82, 69, 82, 69, 2, 179, 232, 24, 85, 4, 254, 194, 20, 67, 237, + 241, 16, 3, 82, 73, 67, 30, 120, 3, 76, 65, 32, 32, 3, 82, 65, 32, 12, 4, 83, 85, 75, 85, 34, 84, 104, 5, 80, 69, 80, 69, 84, 53, 3, 85, 76, 85, 4, - 165, 1, 4, 76, 69, 78, 71, 4, 115, 82, 5, 233, 173, 35, 3, 32, 73, 76, + 165, 1, 4, 76, 69, 78, 71, 4, 115, 82, 5, 237, 203, 36, 3, 32, 73, 76, 10, 36, 5, 65, 76, 73, 78, 71, 99, 69, 9, 11, 32, 6, 18, 82, 55, 84, 4, - 17, 2, 69, 80, 4, 11, 65, 5, 17, 2, 32, 84, 2, 11, 69, 2, 131, 245, 29, - 68, 5, 133, 157, 29, 3, 32, 83, 65, 30, 86, 79, 164, 196, 22, 5, 32, 79, - 70, 32, 89, 245, 204, 12, 6, 69, 84, 32, 83, 72, 79, 26, 32, 2, 79, 78, - 21, 2, 84, 32, 5, 199, 147, 34, 45, 22, 44, 2, 66, 79, 246, 1, 83, 159, - 162, 36, 88, 18, 38, 88, 205, 1, 4, 76, 68, 32, 83, 17, 33, 6, 32, 87, - 73, 84, 72, 32, 14, 82, 66, 86, 83, 140, 167, 12, 4, 76, 73, 71, 72, 210, - 145, 10, 67, 195, 233, 13, 88, 6, 52, 4, 79, 76, 68, 32, 141, 210, 35, 3, - 65, 76, 76, 4, 26, 83, 223, 184, 22, 67, 2, 137, 167, 12, 4, 67, 82, 73, - 80, 162, 10, 120, 2, 67, 79, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 248, - 91, 3, 78, 74, 65, 186, 234, 27, 83, 146, 142, 5, 70, 83, 81, 8, 26, 77, - 255, 208, 35, 76, 6, 72, 12, 66, 73, 78, 73, 78, 71, 32, 77, 65, 82, 75, - 32, 219, 157, 36, 77, 4, 168, 137, 21, 6, 84, 85, 75, 87, 69, 78, 241, - 165, 3, 4, 75, 79, 81, 78, 146, 10, 170, 1, 70, 58, 75, 170, 1, 76, 66, - 77, 102, 78, 234, 1, 80, 206, 103, 82, 102, 83, 134, 1, 84, 54, 89, 202, - 193, 2, 87, 166, 215, 32, 69, 214, 22, 65, 2, 73, 2, 79, 3, 85, 8, 226, - 79, 65, 222, 183, 35, 69, 254, 5, 79, 219, 16, 85, 22, 78, 69, 46, 79, - 174, 176, 34, 89, 254, 233, 1, 80, 186, 2, 65, 2, 73, 3, 85, 6, 254, 144, - 35, 85, 142, 140, 1, 78, 3, 84, 7, 218, 173, 34, 86, 161, 222, 1, 2, 71, - 72, 10, 194, 102, 69, 190, 162, 25, 79, 182, 147, 10, 65, 2, 73, 3, 85, - 19, 62, 69, 166, 101, 66, 146, 182, 35, 65, 2, 73, 2, 79, 3, 85, 4, 254, - 174, 34, 69, 183, 236, 1, 78, 30, 102, 71, 50, 74, 50, 84, 150, 101, 85, - 250, 188, 33, 83, 246, 7, 68, 242, 219, 1, 89, 218, 19, 65, 3, 73, 6, - 178, 223, 31, 75, 190, 184, 4, 71, 187, 2, 65, 6, 222, 236, 25, 85, 198, - 150, 10, 69, 171, 4, 65, 4, 222, 244, 35, 85, 151, 14, 69, 254, 8, 76, 5, - 72, 65, 83, 69, 45, 210, 100, 69, 138, 2, 85, 254, 177, 35, 65, 3, 73, - 242, 8, 116, 2, 65, 32, 160, 18, 2, 66, 32, 136, 13, 2, 67, 32, 160, 20, - 2, 68, 32, 172, 18, 2, 69, 32, 137, 25, 2, 70, 32, 174, 1, 158, 1, 71, - 106, 75, 130, 1, 76, 102, 77, 190, 3, 78, 210, 4, 80, 146, 2, 83, 190, 2, - 84, 154, 1, 85, 144, 253, 28, 2, 70, 73, 182, 156, 5, 86, 211, 219, 1, - 82, 6, 60, 5, 72, 69, 85, 65, 69, 173, 46, 5, 66, 73, 69, 69, 32, 4, 168, - 27, 2, 71, 72, 179, 205, 25, 82, 12, 38, 65, 34, 69, 142, 67, 80, 3, 85, - 4, 234, 146, 36, 70, 187, 2, 81, 4, 132, 192, 26, 5, 85, 75, 69, 85, 84, - 255, 212, 9, 84, 10, 78, 85, 222, 65, 79, 224, 190, 25, 2, 65, 80, 129, - 154, 9, 4, 69, 84, 32, 75, 5, 243, 185, 26, 65, 34, 142, 1, 65, 168, 1, + 17, 2, 69, 80, 4, 11, 65, 5, 17, 2, 32, 84, 2, 11, 69, 2, 219, 222, 30, + 68, 5, 181, 230, 30, 3, 32, 83, 65, 30, 86, 79, 228, 147, 23, 5, 32, 79, + 70, 32, 89, 177, 155, 13, 6, 69, 84, 32, 83, 72, 79, 26, 32, 2, 79, 78, + 21, 2, 84, 32, 5, 195, 174, 35, 45, 22, 44, 2, 66, 79, 246, 1, 83, 179, + 192, 37, 88, 18, 38, 88, 205, 1, 4, 76, 68, 32, 83, 17, 33, 6, 32, 87, + 73, 84, 72, 32, 14, 82, 66, 86, 83, 168, 211, 12, 4, 76, 73, 71, 72, 246, + 180, 10, 67, 151, 184, 14, 88, 6, 52, 4, 79, 76, 68, 32, 149, 240, 36, 3, + 65, 76, 76, 4, 26, 83, 159, 136, 23, 67, 2, 165, 211, 12, 4, 67, 82, 73, + 80, 164, 10, 120, 2, 67, 79, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 128, + 92, 3, 78, 74, 65, 138, 207, 28, 83, 238, 192, 5, 70, 83, 81, 8, 26, 77, + 135, 239, 36, 76, 6, 72, 12, 66, 73, 78, 73, 78, 71, 32, 77, 65, 82, 75, + 32, 239, 187, 37, 77, 4, 196, 215, 21, 6, 84, 85, 75, 87, 69, 78, 201, + 173, 3, 4, 75, 79, 81, 78, 148, 10, 170, 1, 70, 58, 75, 170, 1, 76, 66, + 77, 102, 78, 234, 1, 80, 214, 103, 82, 102, 83, 134, 1, 84, 54, 89, 178, + 219, 2, 87, 202, 219, 33, 69, 214, 22, 65, 2, 73, 2, 79, 3, 85, 8, 234, + 79, 65, 234, 213, 36, 69, 254, 5, 79, 219, 16, 85, 22, 78, 69, 46, 79, + 190, 203, 35, 89, 130, 237, 1, 80, 186, 2, 65, 2, 73, 3, 85, 6, 250, 174, + 36, 85, 166, 140, 1, 78, 3, 84, 7, 234, 200, 35, 86, 165, 225, 1, 2, 71, + 72, 10, 202, 102, 69, 202, 131, 26, 79, 182, 208, 10, 65, 2, 73, 3, 85, + 19, 62, 69, 174, 101, 66, 158, 212, 36, 65, 2, 73, 2, 79, 3, 85, 4, 142, + 202, 35, 69, 187, 239, 1, 78, 30, 102, 71, 50, 74, 50, 84, 158, 101, 85, + 130, 216, 34, 83, 246, 7, 68, 246, 222, 1, 89, 218, 19, 65, 3, 73, 6, + 186, 209, 32, 75, 202, 228, 4, 71, 187, 2, 65, 6, 242, 205, 26, 85, 198, + 211, 10, 69, 171, 4, 65, 4, 242, 146, 37, 85, 151, 14, 69, 128, 9, 76, 5, + 72, 65, 83, 69, 45, 218, 100, 69, 138, 2, 85, 138, 208, 36, 65, 3, 73, + 244, 8, 116, 2, 65, 32, 168, 18, 2, 66, 32, 136, 13, 2, 67, 32, 160, 20, + 2, 68, 32, 172, 18, 2, 69, 32, 137, 25, 2, 70, 32, 176, 1, 158, 1, 71, + 106, 75, 130, 1, 76, 102, 77, 198, 3, 78, 210, 4, 80, 146, 2, 83, 190, 2, + 84, 154, 1, 85, 184, 198, 30, 2, 70, 73, 150, 238, 4, 86, 215, 222, 1, + 82, 6, 60, 5, 72, 69, 85, 65, 69, 181, 46, 5, 66, 73, 69, 69, 32, 4, 176, + 27, 2, 71, 72, 191, 174, 26, 82, 12, 38, 65, 34, 69, 150, 67, 80, 3, 85, + 4, 254, 176, 37, 70, 187, 2, 81, 4, 240, 214, 33, 5, 85, 75, 69, 85, 84, + 167, 220, 3, 84, 10, 78, 85, 230, 65, 79, 236, 159, 26, 2, 65, 80, 241, + 214, 9, 4, 69, 84, 32, 75, 5, 159, 159, 27, 65, 36, 142, 1, 65, 176, 1, 2, 66, 65, 38, 79, 76, 6, 86, 69, 85, 65, 69, 78, 244, 70, 8, 69, 85, 78, - 74, 79, 77, 78, 68, 157, 152, 33, 2, 71, 66, 18, 62, 69, 164, 49, 2, 78, - 83, 149, 253, 30, 4, 80, 32, 80, 73, 14, 58, 77, 242, 254, 11, 75, 246, - 134, 7, 78, 163, 248, 16, 83, 9, 178, 24, 66, 206, 44, 86, 183, 248, 26, - 75, 4, 182, 133, 19, 78, 251, 139, 17, 81, 6, 36, 2, 79, 77, 237, 1, 2, - 78, 32, 4, 250, 207, 12, 80, 171, 199, 22, 69, 2, 131, 215, 34, 71, 48, - 122, 65, 32, 2, 68, 65, 54, 71, 98, 75, 32, 2, 83, 72, 38, 84, 102, 89, - 74, 90, 210, 210, 34, 74, 234, 105, 69, 239, 70, 73, 4, 250, 8, 65, 183, - 134, 36, 81, 4, 22, 65, 159, 22, 32, 2, 153, 44, 3, 78, 71, 71, 8, 52, 3, - 75, 85, 69, 222, 202, 31, 65, 215, 242, 2, 71, 4, 250, 7, 32, 181, 239, - 11, 2, 78, 90, 4, 246, 25, 65, 147, 253, 18, 73, 4, 166, 161, 34, 73, - 183, 236, 1, 65, 8, 40, 2, 65, 80, 229, 184, 27, 2, 79, 81, 7, 11, 32, 4, - 180, 157, 34, 2, 77, 70, 1, 2, 78, 84, 6, 26, 73, 143, 240, 35, 69, 5, - 193, 13, 7, 84, 32, 77, 79, 78, 71, 75, 4, 214, 5, 65, 245, 243, 11, 4, - 85, 78, 32, 77, 22, 58, 65, 80, 3, 79, 78, 32, 166, 238, 35, 69, 163, 28, - 85, 10, 42, 65, 154, 18, 32, 250, 17, 77, 15, 83, 4, 218, 221, 25, 82, - 155, 173, 10, 77, 8, 56, 4, 77, 70, 79, 78, 1, 6, 80, 65, 32, 78, 74, 73, - 4, 37, 7, 32, 80, 73, 80, 65, 69, 77, 4, 206, 16, 71, 231, 246, 35, 66, - 22, 70, 72, 194, 1, 79, 192, 66, 2, 69, 85, 158, 160, 35, 85, 167, 34, - 73, 10, 74, 73, 70, 85, 129, 224, 34, 10, 79, 81, 32, 78, 83, 72, 85, 84, - 32, 89, 4, 158, 219, 25, 82, 233, 194, 6, 8, 78, 68, 65, 32, 80, 65, 32, - 78, 4, 188, 11, 4, 69, 78, 83, 72, 183, 252, 35, 77, 6, 132, 206, 34, 2, - 78, 74, 190, 185, 1, 81, 3, 84, 10, 44, 2, 69, 85, 44, 3, 73, 84, 65, 31, - 85, 4, 234, 181, 34, 65, 1, 4, 84, 69, 85, 87, 2, 11, 32, 2, 223, 30, 77, - 4, 186, 26, 32, 215, 145, 26, 65, 4, 228, 3, 5, 32, 89, 85, 81, 32, 145, - 130, 27, 3, 78, 75, 78, 112, 164, 1, 7, 71, 72, 69, 85, 71, 72, 69, 34, - 75, 126, 76, 122, 77, 154, 3, 78, 234, 2, 80, 90, 83, 178, 1, 84, 106, - 89, 190, 22, 87, 204, 46, 2, 70, 69, 147, 157, 11, 86, 4, 198, 56, 85, - 203, 203, 35, 78, 12, 40, 2, 69, 85, 50, 73, 191, 242, 35, 65, 6, 230, - 54, 89, 246, 139, 12, 80, 191, 174, 23, 65, 4, 230, 240, 35, 69, 175, 18, - 81, 8, 46, 65, 240, 69, 2, 79, 77, 171, 160, 35, 69, 4, 50, 65, 129, 61, - 7, 77, 32, 78, 83, 72, 85, 84, 2, 247, 212, 25, 78, 26, 70, 65, 74, 66, - 140, 1, 2, 69, 85, 58, 70, 145, 23, 3, 79, 78, 84, 7, 21, 3, 32, 78, 74, - 4, 194, 134, 18, 85, 137, 192, 16, 3, 69, 85, 65, 10, 90, 65, 154, 46, - 85, 144, 201, 28, 2, 69, 85, 173, 182, 6, 7, 73, 84, 32, 77, 66, 65, 65, - 4, 150, 12, 65, 245, 242, 18, 4, 32, 77, 65, 69, 4, 252, 155, 31, 5, 84, - 32, 78, 71, 71, 227, 227, 4, 81, 4, 48, 4, 79, 78, 32, 84, 181, 235, 25, - 2, 73, 89, 2, 163, 65, 69, 24, 110, 71, 166, 1, 83, 50, 89, 152, 23, 8, - 84, 73, 69, 69, 32, 83, 72, 69, 193, 245, 33, 5, 68, 85, 32, 78, 74, 12, - 70, 71, 168, 132, 35, 8, 75, 73, 78, 68, 73, 32, 77, 86, 175, 104, 79, 8, - 60, 3, 85, 79, 81, 216, 133, 19, 2, 69, 85, 219, 189, 15, 65, 5, 137, - 208, 27, 2, 32, 76, 4, 26, 72, 227, 172, 35, 69, 2, 167, 199, 35, 85, 4, - 248, 45, 2, 65, 69, 211, 17, 73, 8, 152, 155, 10, 2, 69, 69, 210, 130, - 12, 65, 160, 146, 8, 3, 85, 78, 71, 199, 175, 5, 73, 12, 88, 2, 65, 75, - 16, 2, 72, 69, 160, 236, 18, 2, 69, 84, 230, 161, 15, 73, 227, 213, 1, - 85, 2, 211, 25, 69, 4, 244, 230, 25, 4, 84, 32, 78, 74, 129, 239, 4, 4, - 85, 65, 69, 81, 6, 32, 2, 85, 32, 151, 192, 34, 65, 4, 36, 4, 77, 65, 69, - 77, 139, 7, 78, 2, 11, 71, 2, 135, 7, 66, 4, 44, 4, 65, 70, 85, 32, 229, - 4, 2, 69, 85, 2, 221, 179, 31, 6, 76, 69, 69, 82, 65, 69, 196, 1, 170, 1, - 71, 82, 75, 206, 2, 76, 50, 77, 250, 3, 78, 238, 6, 80, 78, 83, 118, 84, - 176, 1, 3, 86, 69, 85, 46, 87, 62, 89, 250, 179, 29, 66, 166, 208, 2, 70, - 247, 187, 3, 82, 6, 40, 2, 72, 65, 221, 234, 18, 2, 66, 65, 4, 186, 201, - 25, 82, 155, 173, 10, 80, 22, 66, 69, 182, 1, 85, 248, 224, 25, 3, 80, - 65, 82, 139, 145, 10, 65, 14, 40, 2, 78, 32, 54, 85, 143, 245, 35, 84, 4, - 212, 148, 32, 4, 70, 65, 84, 73, 207, 226, 2, 76, 8, 42, 83, 162, 225, - 25, 75, 195, 147, 10, 77, 4, 232, 14, 3, 72, 69, 85, 239, 53, 69, 4, 48, - 6, 79, 80, 32, 78, 75, 65, 135, 244, 35, 84, 2, 11, 65, 2, 223, 198, 25, - 82, 8, 202, 47, 65, 230, 176, 25, 73, 183, 147, 10, 85, 38, 82, 65, 130, - 1, 66, 202, 196, 25, 85, 216, 25, 4, 71, 66, 65, 83, 143, 167, 8, 73, 8, - 18, 32, 79, 69, 4, 42, 78, 209, 159, 17, 4, 75, 69, 85, 65, 2, 11, 83, 2, - 223, 133, 34, 73, 4, 214, 196, 35, 77, 211, 25, 83, 24, 34, 65, 114, 69, - 82, 73, 35, 85, 6, 32, 2, 65, 32, 163, 229, 18, 78, 4, 204, 219, 30, 8, - 67, 65, 66, 66, 65, 71, 69, 45, 129, 195, 4, 2, 80, 73, 8, 50, 85, 130, - 195, 25, 82, 185, 201, 5, 2, 69, 75, 4, 150, 240, 35, 77, 3, 88, 7, 226, - 7, 82, 155, 232, 35, 84, 4, 174, 221, 35, 65, 175, 18, 69, 72, 130, 1, - 65, 54, 68, 110, 71, 222, 1, 74, 102, 83, 130, 1, 84, 102, 90, 237, 7, - 12, 89, 73, 82, 32, 77, 75, 80, 65, 82, 65, 81, 32, 4, 244, 218, 25, 4, - 78, 83, 65, 78, 195, 147, 10, 81, 12, 60, 2, 69, 85, 174, 41, 65, 146, - 205, 18, 79, 135, 210, 16, 73, 4, 128, 129, 34, 2, 65, 69, 195, 236, 1, - 84, 20, 50, 71, 98, 75, 210, 216, 25, 65, 223, 130, 10, 79, 12, 26, 85, - 247, 156, 35, 69, 11, 180, 39, 3, 65, 69, 78, 190, 245, 34, 79, 202, 26, - 69, 143, 53, 77, 4, 36, 3, 85, 69, 32, 171, 216, 25, 65, 2, 145, 230, 18, - 3, 77, 65, 69, 10, 34, 65, 34, 69, 255, 169, 12, 85, 4, 190, 218, 35, 69, - 219, 16, 77, 4, 194, 254, 33, 69, 171, 96, 85, 12, 78, 85, 202, 214, 25, - 72, 216, 216, 5, 2, 69, 85, 146, 170, 4, 79, 219, 16, 65, 4, 212, 214, - 35, 4, 79, 84, 32, 78, 179, 19, 78, 8, 54, 69, 232, 204, 35, 4, 85, 32, - 77, 66, 131, 26, 65, 4, 204, 225, 18, 2, 85, 78, 211, 135, 17, 78, 4, - 254, 192, 34, 69, 247, 167, 1, 65, 6, 26, 73, 227, 152, 35, 69, 4, 26, - 82, 155, 232, 35, 78, 2, 151, 151, 34, 73, 10, 30, 69, 50, 72, 255, 6, - 85, 4, 26, 84, 191, 219, 34, 85, 2, 231, 151, 35, 70, 4, 182, 166, 12, - 85, 203, 173, 13, 73, 10, 40, 2, 65, 65, 34, 69, 41, 2, 73, 84, 2, 11, - 83, 2, 191, 185, 25, 72, 4, 228, 25, 2, 85, 84, 223, 204, 35, 84, 4, 38, - 85, 153, 133, 32, 3, 65, 32, 89, 2, 235, 139, 26, 65, 4, 180, 145, 27, 2, - 65, 69, 171, 212, 8, 88, 4, 40, 4, 65, 78, 71, 75, 255, 228, 35, 85, 2, - 143, 19, 85, 10, 42, 85, 230, 163, 12, 69, 179, 190, 23, 65, 6, 210, 18, - 87, 212, 3, 4, 32, 77, 85, 79, 167, 206, 35, 77, 234, 1, 134, 1, 70, 68, - 2, 71, 72, 34, 75, 254, 1, 76, 142, 1, 77, 130, 3, 78, 130, 5, 80, 122, - 82, 90, 83, 190, 1, 84, 158, 1, 87, 39, 89, 4, 36, 3, 69, 85, 70, 167, - 224, 35, 65, 2, 11, 69, 2, 151, 2, 85, 4, 202, 1, 69, 191, 222, 35, 65, - 22, 46, 69, 146, 1, 85, 42, 87, 135, 244, 33, 89, 10, 26, 85, 215, 225, - 35, 84, 8, 72, 3, 65, 69, 84, 20, 5, 79, 84, 32, 77, 66, 246, 224, 35, - 77, 3, 80, 2, 251, 205, 11, 77, 2, 219, 179, 25, 85, 9, 134, 208, 35, 79, - 218, 16, 78, 3, 81, 2, 171, 171, 35, 65, 14, 58, 69, 182, 204, 25, 79, - 130, 167, 8, 73, 223, 219, 1, 85, 8, 42, 85, 138, 243, 33, 69, 183, 236, - 1, 84, 4, 178, 133, 26, 65, 139, 218, 9, 77, 41, 94, 65, 58, 66, 66, 69, - 38, 70, 80, 2, 71, 66, 214, 140, 31, 79, 246, 214, 3, 86, 135, 121, 85, - 4, 228, 139, 17, 2, 76, 69, 197, 157, 18, 3, 69, 78, 74, 6, 32, 2, 65, - 65, 235, 190, 35, 85, 5, 237, 130, 28, 2, 32, 83, 6, 250, 226, 17, 85, - 147, 142, 16, 69, 10, 44, 2, 69, 85, 238, 228, 33, 79, 207, 11, 73, 4, - 150, 198, 35, 65, 215, 22, 84, 6, 150, 240, 33, 73, 192, 68, 2, 79, 70, - 223, 39, 69, 72, 78, 68, 46, 71, 226, 1, 74, 98, 83, 110, 84, 34, 89, - 210, 215, 35, 73, 3, 85, 8, 194, 39, 69, 178, 228, 34, 79, 255, 62, 65, - 24, 18, 71, 99, 75, 12, 54, 65, 202, 42, 69, 162, 245, 30, 87, 135, 170, - 4, 85, 6, 152, 38, 2, 65, 77, 183, 180, 35, 80, 12, 68, 2, 69, 85, 174, - 237, 33, 73, 2, 89, 226, 156, 1, 85, 203, 79, 65, 4, 226, 152, 12, 65, - 167, 173, 23, 82, 12, 60, 2, 69, 85, 230, 15, 73, 158, 136, 12, 85, 243, - 192, 23, 65, 4, 206, 198, 35, 65, 175, 18, 84, 10, 46, 72, 32, 3, 73, 69, - 69, 183, 199, 35, 85, 4, 254, 187, 35, 85, 219, 5, 69, 4, 138, 216, 35, - 80, 3, 84, 6, 130, 14, 69, 135, 165, 35, 85, 8, 162, 187, 35, 69, 218, 5, - 85, 254, 5, 65, 219, 16, 73, 12, 42, 69, 46, 85, 182, 214, 35, 65, 3, 73, - 4, 208, 169, 25, 2, 85, 84, 155, 173, 10, 69, 4, 146, 186, 35, 85, 175, - 28, 81, 8, 48, 3, 69, 78, 32, 150, 194, 35, 73, 175, 1, 65, 4, 214, 240, - 25, 79, 227, 194, 9, 77, 26, 58, 72, 90, 85, 186, 16, 65, 174, 6, 69, - 167, 161, 35, 79, 12, 54, 69, 162, 193, 25, 79, 222, 255, 9, 73, 219, 19, - 85, 6, 214, 7, 85, 255, 204, 35, 69, 6, 222, 189, 35, 65, 214, 22, 69, 3, - 85, 18, 62, 69, 74, 85, 210, 191, 25, 79, 226, 252, 9, 65, 215, 22, 73, - 8, 26, 85, 255, 230, 33, 69, 6, 186, 130, 34, 65, 246, 208, 1, 78, 3, 84, - 5, 215, 182, 35, 79, 4, 134, 152, 31, 85, 223, 186, 4, 65, 10, 40, 2, 65, - 69, 18, 85, 191, 130, 35, 69, 2, 251, 3, 77, 6, 22, 87, 227, 13, 79, 2, - 195, 190, 25, 79, 186, 2, 178, 1, 70, 154, 1, 71, 202, 1, 75, 170, 1, 76, - 158, 1, 77, 134, 2, 78, 190, 7, 80, 166, 2, 82, 78, 83, 166, 1, 84, 246, - 1, 86, 66, 87, 34, 89, 226, 186, 35, 65, 2, 73, 3, 79, 18, 54, 85, 218, - 172, 22, 65, 202, 140, 13, 69, 255, 5, 79, 10, 26, 32, 231, 154, 30, 69, - 6, 148, 143, 23, 4, 82, 69, 77, 69, 134, 188, 10, 67, 191, 132, 2, 73, - 16, 24, 2, 66, 69, 39, 72, 4, 234, 194, 34, 85, 143, 140, 1, 84, 12, 34, - 65, 34, 69, 187, 189, 35, 79, 2, 11, 65, 2, 139, 161, 25, 77, 8, 26, 85, - 247, 205, 35, 84, 6, 158, 183, 35, 65, 214, 22, 78, 3, 88, 18, 50, 69, - 62, 80, 18, 85, 206, 204, 35, 73, 3, 79, 6, 26, 85, 255, 204, 35, 84, 4, - 166, 182, 35, 65, 215, 22, 88, 2, 211, 28, 69, 6, 158, 176, 35, 69, 162, - 28, 79, 15, 84, 18, 50, 65, 40, 2, 69, 85, 22, 79, 183, 203, 35, 85, 6, - 150, 187, 35, 65, 218, 16, 80, 3, 81, 2, 155, 185, 35, 65, 8, 238, 208, - 17, 79, 198, 250, 17, 77, 3, 81, 32, 110, 65, 44, 2, 66, 69, 34, 70, 20, - 2, 71, 66, 34, 73, 130, 156, 25, 85, 198, 221, 9, 69, 2, 79, 255, 59, 86, - 11, 234, 164, 17, 69, 174, 165, 18, 80, 3, 81, 4, 146, 185, 35, 85, 219, - 16, 69, 2, 227, 136, 12, 69, 4, 226, 249, 34, 69, 215, 79, 65, 5, 195, - 178, 35, 69, 80, 114, 68, 170, 1, 71, 138, 3, 74, 112, 2, 83, 72, 58, 84, - 32, 3, 89, 73, 32, 54, 90, 210, 129, 35, 65, 179, 47, 75, 12, 34, 65, 98, - 73, 187, 247, 34, 85, 6, 32, 2, 65, 32, 203, 199, 35, 80, 4, 128, 215, - 17, 3, 77, 89, 32, 249, 140, 13, 3, 83, 79, 70, 4, 214, 179, 25, 65, 183, - 147, 10, 81, 36, 78, 71, 50, 85, 122, 75, 118, 79, 188, 177, 11, 3, 69, - 85, 82, 195, 146, 24, 65, 12, 18, 69, 51, 85, 6, 26, 85, 183, 217, 33, - 69, 4, 199, 187, 23, 65, 6, 64, 6, 65, 69, 83, 72, 65, 69, 250, 151, 25, - 82, 155, 173, 10, 80, 2, 11, 32, 2, 203, 231, 22, 78, 12, 34, 65, 20, 2, - 69, 85, 35, 85, 5, 231, 176, 35, 65, 4, 138, 178, 35, 65, 175, 18, 88, 4, - 150, 196, 35, 77, 3, 80, 4, 250, 195, 35, 80, 3, 81, 8, 18, 65, 31, 69, - 2, 201, 244, 30, 2, 69, 77, 6, 26, 69, 139, 183, 34, 85, 5, 233, 160, 35, - 4, 32, 69, 80, 79, 6, 26, 85, 163, 214, 33, 73, 4, 198, 194, 35, 79, 15, - 69, 4, 222, 177, 35, 85, 207, 16, 65, 4, 128, 190, 25, 4, 67, 76, 69, 65, - 247, 240, 1, 66, 4, 174, 174, 25, 65, 3, 85, 34, 42, 65, 90, 69, 58, 73, - 50, 79, 23, 85, 8, 32, 2, 32, 80, 207, 155, 17, 65, 4, 152, 176, 27, 2, - 69, 79, 169, 135, 7, 2, 76, 85, 6, 26, 85, 211, 175, 35, 69, 4, 166, 192, - 35, 84, 3, 88, 7, 11, 69, 4, 202, 172, 25, 69, 183, 147, 10, 84, 5, 135, - 240, 34, 79, 11, 82, 65, 246, 190, 35, 69, 3, 77, 8, 46, 65, 238, 14, 69, - 185, 168, 18, 2, 73, 77, 4, 242, 190, 35, 69, 3, 81, 18, 62, 69, 30, 72, - 150, 131, 31, 85, 146, 170, 4, 79, 163, 14, 65, 4, 150, 190, 35, 69, 3, - 84, 8, 42, 69, 182, 154, 22, 79, 235, 143, 3, 73, 2, 209, 252, 11, 2, 85, - 65, 28, 34, 65, 94, 69, 50, 79, 39, 85, 10, 56, 2, 69, 78, 186, 153, 22, - 65, 158, 163, 13, 77, 3, 81, 2, 221, 165, 11, 3, 32, 78, 84, 6, 26, 85, - 155, 188, 35, 78, 5, 155, 251, 11, 65, 6, 130, 196, 33, 79, 131, 248, 1, - 81, 6, 170, 7, 77, 227, 157, 35, 65, 6, 26, 69, 207, 170, 35, 79, 4, 138, - 142, 25, 85, 155, 173, 10, 69, 6, 246, 10, 69, 255, 203, 31, 85, 26, 68, - 2, 69, 85, 46, 73, 32, 3, 79, 81, 32, 54, 85, 143, 185, 35, 65, 8, 246, - 175, 23, 65, 162, 138, 12, 77, 3, 88, 4, 150, 163, 35, 69, 215, 22, 84, - 4, 248, 150, 12, 4, 83, 87, 73, 77, 223, 162, 9, 67, 8, 226, 165, 25, 69, - 194, 194, 8, 65, 246, 208, 1, 78, 3, 81, 108, 162, 1, 75, 82, 76, 46, 77, - 98, 78, 190, 1, 80, 66, 82, 50, 83, 110, 84, 38, 89, 170, 196, 2, 87, - 252, 172, 9, 2, 86, 85, 170, 170, 23, 69, 242, 5, 70, 231, 16, 85, 14, - 242, 188, 17, 69, 146, 142, 16, 89, 254, 233, 1, 80, 186, 2, 65, 2, 79, - 3, 85, 6, 178, 163, 25, 79, 182, 147, 10, 65, 3, 73, 13, 42, 66, 34, 69, - 242, 181, 35, 65, 3, 79, 4, 186, 230, 34, 69, 159, 77, 65, 2, 187, 201, - 33, 69, 22, 94, 71, 38, 74, 38, 85, 250, 188, 33, 83, 246, 7, 68, 150, 3, - 84, 222, 216, 1, 89, 219, 19, 73, 4, 134, 250, 30, 75, 191, 184, 4, 71, - 4, 190, 135, 25, 85, 239, 154, 10, 65, 5, 223, 157, 35, 65, 6, 26, 69, - 239, 134, 25, 85, 4, 246, 167, 34, 85, 143, 140, 1, 69, 12, 186, 2, 69, - 154, 231, 8, 73, 143, 202, 26, 85, 14, 66, 72, 230, 2, 69, 194, 170, 18, - 65, 226, 224, 16, 85, 235, 36, 73, 6, 146, 159, 35, 73, 218, 19, 79, 3, - 85, 6, 186, 190, 20, 65, 223, 215, 14, 69, 4, 234, 158, 25, 79, 183, 147, - 10, 65, 4, 166, 155, 35, 65, 215, 22, 69, 14, 54, 69, 154, 231, 8, 73, - 186, 179, 26, 65, 215, 22, 85, 6, 150, 165, 34, 85, 142, 140, 1, 69, 3, - 78, 16, 62, 72, 50, 69, 194, 170, 18, 65, 226, 224, 16, 85, 235, 36, 73, - 8, 46, 69, 178, 156, 35, 73, 218, 19, 79, 3, 85, 2, 251, 163, 34, 85, 10, - 174, 181, 17, 69, 190, 134, 3, 65, 139, 244, 14, 73, 6, 138, 156, 25, 79, - 2, 85, 183, 147, 10, 65, 14, 42, 75, 166, 244, 33, 65, 211, 155, 1, 74, - 11, 49, 10, 78, 79, 84, 69, 32, 87, 73, 84, 72, 32, 8, 224, 173, 9, 2, - 80, 79, 190, 1, 89, 188, 131, 12, 2, 69, 85, 147, 156, 6, 68, 6, 38, 32, - 237, 172, 16, 3, 66, 69, 82, 4, 170, 226, 28, 67, 189, 186, 5, 5, 79, 70, - 32, 83, 79, 78, 72, 3, 75, 69, 84, 60, 7, 83, 65, 32, 86, 65, 72, 32, - 183, 167, 33, 69, 5, 241, 218, 15, 10, 66, 65, 76, 76, 32, 65, 78, 68, - 32, 72, 72, 104, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 164, 1, 7, - 76, 69, 84, 84, 69, 82, 32, 235, 222, 32, 70, 10, 52, 4, 72, 73, 71, 72, - 44, 3, 76, 79, 87, 39, 77, 4, 172, 140, 21, 2, 45, 76, 251, 207, 10, 32, - 4, 32, 2, 45, 77, 219, 219, 31, 32, 2, 201, 219, 31, 2, 73, 68, 60, 238, - 1, 68, 38, 69, 38, 71, 34, 75, 38, 85, 20, 2, 87, 65, 22, 89, 128, 238, - 27, 2, 72, 87, 154, 196, 2, 77, 178, 181, 2, 79, 154, 154, 2, 86, 246, 5, - 74, 2, 84, 2, 90, 162, 8, 67, 2, 83, 158, 20, 66, 2, 70, 2, 80, 186, 2, - 65, 3, 73, 4, 222, 235, 32, 72, 159, 188, 2, 79, 7, 222, 215, 30, 78, - 251, 207, 4, 69, 4, 202, 130, 35, 66, 219, 35, 65, 4, 202, 238, 27, 80, - 203, 184, 7, 65, 5, 135, 130, 35, 87, 5, 215, 129, 35, 68, 4, 138, 167, - 34, 69, 235, 104, 73, 121, 48, 3, 65, 75, 32, 230, 10, 72, 179, 218, 21, - 84, 112, 196, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, - 78, 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 248, 4, 3, 80, 65, 78, 50, 83, - 141, 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 4, 214, 163, 35, - 78, 87, 72, 76, 194, 1, 77, 114, 78, 68, 2, 80, 65, 38, 83, 232, 144, 18, - 4, 75, 65, 82, 79, 214, 141, 17, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, - 2, 76, 2, 82, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 10, 26, 65, 251, - 159, 35, 66, 9, 45, 9, 78, 68, 65, 73, 76, 73, 78, 71, 32, 6, 198, 159, - 35, 72, 2, 78, 3, 83, 10, 152, 2, 2, 79, 82, 138, 157, 35, 68, 2, 71, 2, - 89, 187, 2, 65, 5, 189, 165, 20, 4, 75, 80, 65, 75, 24, 80, 10, 73, 77, - 65, 76, 85, 78, 71, 85, 78, 32, 96, 2, 79, 85, 195, 159, 35, 65, 20, 230, - 157, 35, 71, 2, 72, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 87, 2, 89, 187, - 2, 65, 2, 145, 145, 18, 5, 84, 72, 69, 82, 78, 4, 166, 2, 71, 161, 145, - 18, 4, 79, 78, 71, 79, 10, 96, 12, 89, 77, 66, 79, 76, 32, 66, 73, 78, - 68, 85, 32, 193, 140, 10, 6, 73, 71, 78, 32, 84, 79, 8, 78, 80, 152, 243, - 10, 3, 74, 85, 68, 249, 166, 24, 6, 78, 65, 32, 77, 69, 84, 4, 64, 3, 65, - 78, 71, 133, 199, 22, 7, 73, 78, 65, 82, 66, 79, 82, 2, 139, 131, 25, 79, - 18, 122, 85, 228, 173, 25, 6, 80, 65, 75, 80, 65, 75, 180, 248, 2, 5, 75, - 65, 82, 79, 32, 182, 181, 6, 69, 150, 64, 73, 3, 79, 5, 177, 200, 29, 15, - 32, 70, 79, 82, 32, 83, 73, 77, 65, 76, 85, 78, 71, 85, 78, 5, 143, 134, - 23, 84, 254, 1, 134, 1, 65, 230, 2, 69, 50, 76, 130, 1, 78, 198, 10, 84, - 144, 220, 31, 2, 67, 65, 204, 162, 2, 5, 86, 69, 82, 65, 71, 191, 140, 1, - 68, 20, 136, 1, 4, 77, 69, 68, 32, 170, 1, 82, 164, 143, 3, 10, 67, 72, - 32, 87, 73, 84, 72, 32, 85, 77, 210, 144, 7, 84, 234, 185, 24, 86, 19, - 78, 8, 86, 65, 0, 2, 68, 69, 52, 4, 69, 73, 71, 72, 1, 7, 83, 73, 88, 84, - 69, 69, 78, 2, 201, 214, 19, 8, 83, 67, 69, 78, 68, 73, 78, 71, 2, 193, - 214, 19, 2, 84, 72, 4, 128, 196, 28, 3, 68, 69, 68, 179, 150, 6, 32, 4, - 176, 166, 8, 3, 82, 32, 77, 183, 207, 25, 84, 13, 11, 76, 11, 38, 32, - 217, 187, 24, 3, 72, 79, 80, 6, 162, 140, 12, 80, 212, 232, 15, 6, 87, - 73, 84, 72, 32, 67, 187, 142, 6, 83, 208, 1, 84, 5, 71, 65, 76, 73, 32, - 162, 9, 84, 37, 9, 90, 69, 78, 69, 32, 82, 73, 78, 71, 200, 1, 210, 1, - 65, 44, 9, 67, 85, 82, 82, 69, 78, 67, 89, 32, 148, 2, 7, 76, 69, 84, 84, - 69, 82, 32, 204, 3, 6, 82, 85, 80, 69, 69, 32, 34, 83, 242, 128, 22, 73, - 130, 4, 86, 242, 153, 11, 68, 141, 105, 3, 71, 65, 78, 6, 190, 147, 30, - 85, 138, 149, 1, 66, 51, 78, 12, 120, 10, 78, 85, 77, 69, 82, 65, 84, 79, - 82, 32, 221, 212, 22, 14, 68, 69, 78, 79, 77, 73, 78, 65, 84, 79, 82, 32, - 83, 73, 10, 52, 3, 79, 78, 69, 214, 170, 30, 70, 155, 250, 2, 84, 5, 165, - 201, 28, 19, 32, 76, 69, 83, 83, 32, 84, 72, 65, 78, 32, 84, 72, 69, 32, - 68, 69, 78, 79, 108, 226, 1, 75, 90, 82, 226, 232, 20, 86, 234, 239, 7, - 89, 150, 205, 2, 65, 38, 68, 46, 84, 230, 24, 85, 210, 200, 1, 73, 158, - 190, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 80, 254, 68, 72, 2, - 76, 2, 77, 186, 2, 69, 3, 79, 8, 26, 72, 235, 142, 35, 65, 6, 26, 65, - 155, 208, 13, 73, 5, 157, 128, 18, 3, 78, 68, 65, 10, 34, 65, 210, 139, - 35, 72, 3, 82, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 244, 179, 24, 5, 76, - 79, 87, 69, 82, 1, 6, 77, 73, 68, 68, 76, 69, 4, 202, 134, 34, 83, 179, - 69, 77, 20, 116, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, - 69, 84, 84, 69, 82, 32, 154, 154, 25, 65, 187, 138, 6, 73, 6, 150, 131, - 22, 82, 147, 134, 13, 89, 4, 182, 173, 25, 32, 251, 209, 8, 79, 5, 137, - 247, 31, 4, 32, 87, 73, 84, 4, 150, 186, 33, 87, 255, 61, 32, 194, 1, - 180, 2, 7, 76, 69, 84, 84, 69, 82, 32, 244, 1, 7, 78, 85, 77, 66, 69, 82, - 32, 72, 5, 83, 73, 71, 78, 32, 44, 11, 86, 79, 87, 69, 76, 32, 83, 73, - 71, 78, 32, 250, 143, 25, 68, 130, 229, 3, 87, 136, 97, 10, 71, 65, 80, - 32, 70, 73, 76, 76, 69, 82, 153, 229, 4, 12, 72, 85, 78, 68, 82, 69, 68, - 83, 32, 85, 78, 73, 92, 210, 1, 86, 154, 158, 31, 65, 38, 68, 46, 84, - 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, 82, 66, 2, 67, 2, - 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, 2, - 69, 3, 79, 8, 234, 1, 79, 255, 132, 35, 65, 36, 138, 100, 69, 38, 70, 66, - 78, 26, 83, 250, 241, 20, 84, 183, 150, 12, 79, 10, 142, 159, 31, 65, 74, - 67, 171, 163, 3, 86, 24, 84, 2, 86, 79, 166, 160, 31, 65, 190, 21, 85, - 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 6, 33, 6, 67, 65, 76, 73, 67, - 32, 6, 178, 161, 31, 82, 167, 227, 3, 76, 26, 148, 1, 4, 67, 89, 67, 76, - 36, 2, 71, 32, 28, 2, 76, 76, 46, 82, 66, 84, 212, 216, 18, 4, 79, 72, - 65, 90, 164, 217, 11, 2, 75, 73, 155, 128, 4, 83, 4, 210, 187, 31, 73, - 203, 199, 3, 69, 4, 186, 240, 32, 82, 67, 83, 4, 164, 175, 9, 2, 69, 68, - 219, 251, 22, 73, 4, 176, 188, 31, 7, 84, 72, 68, 65, 89, 32, 67, 255, - 197, 3, 68, 4, 160, 165, 13, 2, 67, 79, 149, 219, 21, 4, 73, 78, 71, 32, - 230, 3, 42, 65, 222, 29, 79, 249, 3, 2, 85, 69, 228, 2, 32, 2, 67, 75, - 135, 159, 17, 78, 226, 2, 22, 32, 143, 28, 45, 210, 2, 214, 1, 67, 254, - 4, 68, 174, 2, 70, 102, 72, 82, 76, 170, 2, 77, 250, 2, 80, 46, 82, 202, - 3, 83, 210, 3, 84, 82, 85, 248, 2, 3, 86, 69, 82, 198, 215, 11, 79, 208, - 240, 19, 3, 66, 79, 87, 154, 227, 1, 78, 179, 1, 81, 98, 196, 1, 5, 73, - 82, 67, 76, 69, 200, 1, 6, 85, 82, 86, 69, 68, 32, 192, 150, 25, 12, 82, - 79, 83, 83, 32, 79, 78, 32, 83, 72, 73, 69, 240, 155, 3, 5, 69, 78, 84, - 82, 69, 178, 238, 4, 72, 203, 13, 76, 11, 11, 32, 8, 72, 5, 87, 73, 84, - 72, 32, 145, 164, 16, 7, 70, 79, 82, 32, 82, 69, 67, 6, 224, 51, 8, 87, - 72, 73, 84, 69, 32, 68, 79, 238, 181, 18, 68, 221, 144, 2, 8, 84, 87, 79, - 32, 87, 72, 73, 84, 16, 84, 4, 68, 79, 87, 78, 0, 2, 85, 80, 56, 3, 76, - 69, 70, 1, 4, 82, 73, 71, 72, 4, 141, 139, 31, 9, 87, 65, 82, 68, 83, 32, - 65, 78, 68, 4, 53, 11, 84, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, 4, - 178, 164, 10, 85, 179, 237, 21, 68, 30, 64, 6, 73, 65, 77, 79, 78, 68, - 152, 1, 3, 79, 87, 78, 43, 82, 13, 11, 32, 10, 210, 15, 67, 152, 208, 12, - 10, 77, 73, 78, 85, 83, 32, 87, 72, 73, 84, 184, 134, 6, 6, 87, 73, 84, - 72, 32, 68, 242, 212, 14, 79, 131, 11, 83, 12, 142, 16, 32, 62, 45, 175, - 185, 30, 87, 6, 196, 155, 33, 2, 79, 80, 175, 20, 65, 8, 18, 76, 39, 79, - 4, 150, 134, 27, 79, 191, 240, 7, 65, 4, 172, 192, 2, 2, 85, 82, 231, - 146, 30, 76, 12, 38, 69, 146, 179, 33, 65, 239, 2, 79, 6, 216, 180, 33, - 2, 65, 82, 215, 10, 88, 32, 60, 3, 69, 70, 84, 202, 1, 79, 161, 228, 32, - 3, 65, 82, 71, 22, 96, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 64, 6, - 87, 65, 82, 68, 83, 32, 155, 183, 33, 32, 12, 162, 6, 68, 246, 7, 73, - 174, 170, 33, 80, 166, 24, 83, 51, 84, 4, 226, 142, 32, 69, 143, 137, 1, - 66, 6, 230, 13, 87, 143, 178, 33, 90, 30, 76, 6, 69, 68, 73, 85, 77, 32, - 197, 145, 21, 7, 79, 79, 78, 32, 76, 73, 76, 28, 66, 68, 42, 76, 36, 4, - 82, 73, 71, 72, 12, 2, 85, 80, 111, 83, 6, 84, 3, 79, 87, 78, 219, 182, - 33, 73, 6, 32, 2, 69, 70, 235, 189, 33, 79, 4, 11, 84, 4, 81, 18, 45, 80, - 79, 73, 78, 84, 73, 78, 71, 32, 84, 82, 73, 65, 78, 71, 76, 69, 5, 217, - 7, 2, 32, 67, 8, 130, 12, 77, 207, 196, 33, 81, 8, 142, 169, 24, 85, 222, - 141, 9, 65, 87, 69, 32, 52, 4, 73, 71, 72, 84, 178, 228, 32, 79, 251, - 105, 69, 28, 100, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 244, 1, 6, - 87, 65, 82, 68, 83, 32, 159, 199, 33, 32, 16, 90, 68, 88, 8, 84, 82, 73, - 65, 78, 71, 76, 69, 158, 7, 73, 162, 175, 33, 80, 179, 19, 83, 4, 65, 14, - 79, 85, 66, 76, 69, 32, 84, 82, 73, 65, 78, 71, 76, 69, 5, 247, 146, 12, - 32, 5, 197, 153, 20, 11, 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 8, - 226, 211, 18, 65, 134, 180, 13, 69, 143, 137, 1, 66, 38, 150, 2, 77, 84, - 5, 81, 85, 65, 82, 69, 172, 156, 25, 3, 85, 78, 32, 196, 223, 1, 5, 75, - 85, 76, 76, 32, 208, 164, 3, 3, 78, 79, 87, 208, 195, 1, 5, 65, 70, 69, - 84, 89, 164, 118, 13, 76, 73, 71, 72, 84, 76, 89, 32, 83, 77, 65, 76, 76, - 146, 90, 67, 50, 72, 226, 1, 80, 143, 17, 84, 10, 40, 4, 65, 76, 76, 32, - 239, 182, 33, 73, 8, 134, 175, 33, 68, 134, 7, 76, 71, 83, 9, 11, 32, 6, - 54, 67, 228, 158, 32, 3, 70, 79, 82, 243, 152, 1, 66, 2, 245, 137, 30, 3, - 69, 78, 84, 14, 180, 4, 3, 73, 78, 89, 134, 221, 22, 82, 226, 221, 10, - 79, 54, 69, 135, 2, 87, 18, 38, 80, 133, 208, 31, 3, 78, 73, 86, 16, 46, - 32, 62, 45, 170, 1, 80, 135, 184, 30, 87, 2, 185, 194, 33, 10, 80, 79, - 73, 78, 84, 73, 78, 71, 32, 66, 8, 45, 9, 80, 79, 73, 78, 84, 73, 78, 71, - 32, 8, 66, 73, 236, 190, 33, 5, 68, 79, 85, 66, 76, 230, 3, 83, 51, 84, - 2, 129, 199, 32, 8, 83, 79, 83, 67, 69, 76, 69, 83, 4, 21, 3, 69, 82, 32, - 4, 190, 222, 23, 76, 143, 232, 8, 82, 10, 56, 6, 84, 73, 67, 65, 76, 32, - 29, 4, 89, 32, 83, 77, 4, 134, 195, 33, 69, 51, 82, 6, 21, 3, 65, 76, 76, - 6, 11, 32, 6, 182, 169, 33, 68, 134, 7, 76, 227, 19, 83, 16, 84, 15, 76, - 69, 84, 84, 69, 82, 32, 67, 65, 80, 73, 84, 65, 76, 32, 155, 234, 10, 70, - 10, 218, 227, 34, 67, 2, 72, 2, 73, 2, 82, 3, 90, 124, 84, 11, 67, 75, - 32, 83, 69, 88, 84, 65, 78, 84, 45, 238, 128, 2, 83, 135, 181, 10, 87, - 120, 62, 49, 214, 1, 50, 46, 51, 38, 52, 42, 53, 199, 223, 34, 54, 61, - 50, 50, 118, 51, 126, 52, 42, 53, 199, 223, 34, 54, 31, 46, 51, 194, 1, - 52, 42, 53, 199, 223, 34, 54, 15, 38, 52, 194, 1, 53, 199, 223, 34, 54, - 7, 130, 225, 34, 53, 3, 54, 15, 122, 52, 190, 248, 27, 53, 179, 231, 6, - 54, 31, 42, 51, 66, 52, 14, 53, 199, 223, 34, 54, 17, 34, 52, 42, 53, - 199, 223, 34, 54, 9, 38, 53, 199, 223, 34, 54, 7, 11, 53, 5, 195, 223, - 34, 54, 6, 178, 207, 21, 32, 241, 166, 8, 4, 66, 69, 82, 82, 144, 4, 200, - 1, 3, 76, 68, 32, 78, 79, 104, 7, 80, 79, 77, 79, 70, 79, 32, 188, 6, 2, - 84, 84, 140, 3, 5, 85, 81, 85, 69, 84, 54, 87, 198, 1, 88, 210, 204, 5, - 77, 194, 153, 3, 89, 218, 177, 25, 65, 255, 33, 78, 14, 246, 169, 8, 83, - 210, 164, 7, 69, 134, 252, 16, 70, 234, 2, 87, 207, 10, 71, 10, 26, 75, - 155, 198, 22, 77, 9, 40, 4, 77, 65, 82, 75, 155, 220, 34, 83, 5, 197, - 224, 22, 3, 32, 84, 65, 150, 1, 96, 13, 70, 73, 78, 65, 76, 32, 76, 69, - 84, 84, 69, 82, 32, 53, 7, 76, 69, 84, 84, 69, 82, 32, 10, 142, 219, 34, - 71, 2, 72, 2, 75, 2, 80, 3, 84, 140, 1, 234, 1, 65, 54, 85, 22, 69, 82, - 71, 46, 73, 70, 78, 38, 79, 98, 90, 226, 138, 7, 75, 162, 215, 23, 67, 2, - 76, 2, 83, 198, 36, 66, 210, 200, 1, 74, 206, 134, 2, 68, 2, 70, 2, 72, - 2, 77, 2, 80, 2, 81, 2, 82, 2, 84, 2, 86, 3, 88, 21, 50, 73, 2, 85, 74, - 78, 242, 215, 34, 72, 3, 77, 5, 227, 136, 34, 78, 17, 50, 78, 242, 215, - 34, 69, 2, 72, 2, 73, 3, 82, 7, 238, 215, 34, 71, 3, 78, 11, 210, 215, - 34, 72, 2, 78, 2, 85, 3, 87, 15, 164, 223, 32, 2, 78, 78, 130, 248, 1, - 72, 2, 77, 2, 82, 3, 85, 9, 206, 222, 32, 71, 151, 248, 1, 78, 17, 66, - 78, 146, 213, 33, 32, 238, 128, 1, 69, 2, 77, 2, 79, 3, 85, 4, 250, 213, - 34, 71, 3, 78, 9, 222, 213, 34, 72, 2, 73, 3, 89, 34, 104, 3, 79, 77, 32, - 197, 238, 20, 17, 76, 69, 32, 87, 73, 84, 72, 32, 80, 79, 80, 80, 73, 78, - 71, 32, 67, 32, 168, 1, 5, 72, 65, 76, 70, 32, 52, 14, 83, 81, 85, 65, - 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 134, 157, 15, 65, 166, 217, 15, - 67, 134, 2, 80, 126, 76, 22, 82, 175, 1, 84, 8, 226, 247, 30, 66, 46, 76, - 26, 82, 255, 214, 1, 73, 5, 193, 250, 30, 11, 32, 79, 86, 69, 82, 32, 84, - 79, 80, 32, 83, 5, 229, 195, 33, 8, 32, 79, 70, 32, 70, 76, 79, 87, 14, - 58, 76, 116, 3, 84, 73, 69, 181, 252, 31, 3, 32, 65, 78, 6, 26, 32, 187, - 128, 34, 73, 4, 160, 158, 9, 7, 79, 70, 32, 72, 89, 71, 73, 197, 161, 22, - 6, 87, 73, 84, 72, 32, 83, 7, 235, 196, 31, 32, 162, 2, 88, 10, 32, 68, - 82, 65, 87, 73, 78, 71, 83, 32, 137, 211, 33, 6, 73, 78, 71, 32, 71, 76, - 160, 2, 176, 1, 2, 68, 79, 148, 4, 6, 72, 69, 65, 86, 89, 32, 194, 2, 76, - 156, 17, 6, 82, 73, 71, 72, 84, 32, 144, 4, 3, 85, 80, 32, 245, 3, 9, 86, - 69, 82, 84, 73, 67, 65, 76, 32, 58, 48, 5, 85, 66, 76, 69, 32, 89, 3, 87, - 78, 32, 22, 68, 4, 68, 79, 87, 78, 0, 2, 85, 80, 250, 21, 86, 155, 177, - 28, 72, 6, 171, 22, 32, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, - 68, 32, 132, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 230, 29, 68, - 131, 4, 83, 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 248, 26, 2, 85, - 80, 151, 5, 72, 4, 17, 2, 84, 32, 4, 250, 23, 85, 227, 136, 34, 76, 12, - 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 176, 27, 2, 85, 80, 235, 4, 72, - 4, 17, 2, 84, 32, 4, 218, 23, 85, 135, 9, 72, 46, 120, 4, 76, 69, 70, 84, - 68, 2, 85, 80, 176, 14, 2, 68, 79, 106, 81, 34, 84, 234, 1, 86, 154, 177, - 28, 72, 199, 219, 5, 82, 5, 45, 9, 32, 65, 78, 68, 32, 76, 73, 71, 72, 2, - 207, 250, 32, 84, 11, 29, 5, 32, 65, 78, 68, 32, 8, 42, 76, 238, 193, 28, - 72, 199, 219, 5, 82, 4, 204, 223, 23, 4, 73, 71, 72, 84, 211, 189, 10, - 69, 108, 56, 4, 69, 70, 84, 32, 169, 2, 5, 73, 71, 72, 84, 32, 16, 244, - 18, 19, 68, 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, - 73, 71, 72, 24, 14, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, 73, 71, - 72, 100, 14, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 82, 73, 71, 72, 97, - 17, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, 73, 71, 72, - 92, 172, 1, 4, 65, 82, 67, 32, 30, 68, 172, 9, 4, 76, 69, 70, 84, 62, 81, - 34, 84, 112, 2, 85, 80, 122, 86, 176, 134, 34, 10, 72, 79, 82, 73, 90, - 79, 78, 84, 65, 76, 175, 6, 82, 8, 158, 143, 17, 68, 67, 85, 48, 52, 8, - 73, 65, 71, 79, 78, 65, 76, 32, 203, 8, 79, 36, 100, 7, 77, 73, 68, 68, - 76, 69, 32, 216, 2, 6, 85, 80, 80, 69, 82, 32, 194, 129, 33, 67, 191, 2, - 68, 12, 88, 8, 76, 69, 70, 84, 32, 84, 79, 32, 161, 1, 9, 82, 73, 71, 72, - 84, 32, 84, 79, 32, 8, 136, 1, 28, 85, 80, 80, 69, 82, 32, 67, 69, 78, - 84, 82, 69, 32, 84, 79, 32, 77, 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, - 84, 195, 3, 76, 5, 143, 132, 20, 32, 4, 232, 4, 15, 85, 80, 80, 69, 82, - 32, 67, 69, 78, 84, 82, 69, 32, 84, 79, 175, 255, 19, 76, 20, 196, 1, 17, - 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 77, 73, 68, 68, 76, 69, 32, 228, - 239, 22, 7, 76, 69, 70, 84, 32, 84, 79, 193, 155, 11, 14, 82, 73, 71, 72, - 84, 32, 84, 79, 32, 76, 79, 87, 69, 82, 16, 56, 4, 76, 69, 70, 84, 165, - 1, 5, 82, 73, 71, 72, 84, 9, 11, 32, 6, 60, 4, 84, 79, 32, 76, 137, 128, - 20, 5, 65, 78, 68, 32, 77, 4, 53, 11, 79, 87, 69, 82, 32, 67, 69, 78, 84, - 82, 69, 5, 169, 198, 31, 3, 32, 84, 79, 9, 11, 32, 6, 88, 3, 65, 78, 68, - 65, 15, 84, 79, 32, 76, 79, 87, 69, 82, 32, 67, 69, 78, 84, 82, 69, 2, - 249, 254, 19, 11, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 5, 165, - 187, 10, 9, 32, 84, 79, 32, 77, 73, 68, 68, 76, 12, 164, 1, 2, 85, 66, - 241, 1, 2, 87, 78, 5, 221, 235, 32, 10, 32, 65, 78, 68, 32, 72, 69, 65, - 86, 89, 4, 49, 5, 85, 65, 68, 82, 85, 4, 17, 2, 82, 73, 4, 11, 80, 4, 41, - 8, 76, 69, 32, 68, 65, 83, 72, 32, 4, 230, 130, 16, 86, 207, 175, 12, 72, - 11, 29, 5, 32, 65, 78, 68, 32, 8, 34, 72, 246, 140, 34, 76, 31, 82, 4, - 172, 207, 23, 4, 69, 65, 86, 89, 171, 226, 4, 79, 8, 37, 7, 69, 82, 84, - 73, 67, 65, 76, 9, 11, 32, 6, 25, 4, 65, 78, 68, 32, 6, 198, 176, 28, 72, - 170, 219, 5, 76, 31, 82, 16, 148, 2, 18, 68, 79, 87, 78, 32, 72, 69, 65, - 86, 89, 32, 65, 78, 68, 32, 76, 69, 70, 24, 13, 72, 69, 65, 86, 89, 32, - 65, 78, 68, 32, 76, 69, 70, 100, 13, 76, 73, 71, 72, 84, 32, 65, 78, 68, - 32, 76, 69, 70, 97, 16, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, - 32, 76, 69, 70, 2, 101, 3, 84, 32, 85, 6, 17, 2, 84, 32, 6, 58, 85, 178, - 3, 68, 245, 4, 6, 86, 69, 82, 84, 73, 67, 2, 227, 157, 32, 80, 6, 17, 2, - 84, 32, 6, 58, 85, 134, 4, 68, 205, 4, 6, 86, 69, 82, 84, 73, 67, 2, 239, - 8, 80, 2, 185, 2, 3, 84, 32, 68, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, - 65, 78, 68, 32, 188, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 186, - 2, 68, 131, 4, 83, 12, 80, 4, 68, 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, - 82, 73, 71, 72, 255, 4, 72, 2, 145, 5, 2, 32, 72, 4, 17, 2, 84, 32, 4, - 26, 68, 179, 133, 34, 76, 2, 177, 154, 32, 3, 79, 87, 78, 12, 80, 4, 68, - 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 211, 4, 72, 2, 229, - 4, 2, 32, 72, 4, 17, 2, 84, 32, 4, 22, 68, 131, 5, 72, 2, 233, 4, 3, 79, - 87, 78, 24, 130, 1, 68, 188, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, - 32, 144, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 183, 1, 83, 6, - 49, 10, 79, 85, 66, 76, 69, 32, 65, 78, 68, 32, 6, 92, 3, 76, 69, 70, 0, - 4, 82, 73, 71, 72, 13, 10, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 2, 11, - 84, 2, 189, 157, 26, 2, 32, 83, 6, 54, 72, 68, 3, 76, 69, 70, 1, 4, 82, - 73, 71, 72, 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 189, 149, 32, 2, 65, - 76, 2, 163, 149, 32, 84, 6, 54, 72, 60, 3, 76, 69, 70, 1, 4, 82, 73, 71, - 72, 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 29, 2, 65, 76, 2, 11, 84, 2, - 17, 2, 32, 72, 2, 253, 151, 34, 3, 69, 65, 86, 6, 49, 10, 73, 78, 71, 76, - 69, 32, 65, 78, 68, 32, 6, 96, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 165, - 225, 25, 9, 72, 79, 82, 73, 90, 79, 78, 84, 65, 2, 175, 225, 25, 84, 134, - 6, 46, 65, 186, 14, 69, 150, 1, 73, 191, 1, 79, 232, 5, 36, 4, 72, 77, - 73, 32, 255, 9, 73, 230, 1, 192, 1, 7, 76, 69, 84, 84, 69, 82, 32, 196, - 2, 7, 78, 85, 77, 66, 69, 82, 32, 144, 2, 12, 80, 85, 78, 67, 84, 85, 65, - 84, 73, 79, 78, 32, 84, 5, 83, 73, 71, 78, 32, 126, 86, 215, 169, 24, 68, - 108, 210, 1, 79, 178, 187, 30, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, - 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, - 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 77, 2, 82, 2, 89, 187, 2, 69, 15, - 45, 9, 76, 68, 32, 84, 65, 77, 73, 76, 32, 12, 226, 165, 28, 76, 186, - 153, 2, 83, 190, 66, 78, 199, 221, 2, 82, 42, 82, 69, 38, 70, 66, 78, 26, - 83, 250, 241, 20, 84, 174, 213, 5, 79, 155, 156, 7, 74, 4, 193, 175, 30, - 4, 73, 71, 72, 84, 8, 26, 79, 135, 218, 20, 73, 4, 134, 184, 32, 82, 167, - 177, 1, 85, 4, 65, 3, 73, 78, 69, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, - 5, 203, 143, 34, 84, 10, 46, 76, 170, 188, 12, 68, 169, 16, 2, 67, 82, 4, - 178, 138, 19, 79, 139, 253, 13, 73, 12, 204, 155, 16, 9, 79, 76, 68, 32, - 84, 65, 77, 73, 76, 182, 158, 14, 67, 198, 180, 1, 74, 158, 2, 86, 122, - 85, 231, 233, 1, 65, 34, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 247, 236, 31, 73, 32, 142, 1, 79, 140, 213, 28, 11, 66, 72, 65, 84, 84, - 73, 80, 82, 79, 76, 85, 250, 227, 1, 65, 106, 86, 214, 20, 85, 210, 200, - 1, 73, 207, 134, 2, 69, 7, 165, 185, 30, 10, 76, 68, 32, 84, 65, 77, 73, - 76, 32, 83, 130, 4, 72, 12, 76, 76, 69, 32, 80, 65, 84, 84, 69, 82, 78, - 32, 211, 156, 34, 78, 128, 4, 44, 5, 68, 79, 84, 83, 45, 215, 243, 6, 66, - 254, 3, 74, 49, 74, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 167, 153, - 34, 56, 129, 2, 66, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 167, 153, - 34, 56, 129, 1, 58, 51, 54, 52, 46, 53, 38, 54, 30, 55, 167, 153, 34, 56, - 65, 50, 52, 46, 53, 38, 54, 30, 55, 167, 153, 34, 56, 33, 42, 53, 38, 54, - 30, 55, 167, 153, 34, 56, 17, 34, 54, 30, 55, 167, 153, 34, 56, 9, 26, - 55, 167, 153, 34, 56, 5, 163, 153, 34, 56, 8, 26, 65, 163, 130, 34, 86, - 6, 212, 131, 20, 11, 75, 32, 80, 69, 82, 77, 73, 84, 84, 69, 68, 196, - 135, 8, 6, 83, 84, 45, 70, 69, 69, 219, 141, 6, 68, 10, 42, 68, 108, 2, - 69, 70, 207, 147, 34, 67, 4, 84, 6, 71, 69, 32, 65, 84, 32, 133, 234, 31, - 9, 69, 32, 87, 73, 84, 72, 32, 86, 69, 2, 167, 236, 33, 78, 4, 190, 134, - 31, 67, 167, 144, 3, 83, 12, 84, 4, 75, 69, 78, 32, 232, 177, 8, 2, 67, - 67, 220, 220, 24, 2, 87, 78, 207, 118, 79, 6, 140, 227, 26, 17, 67, 73, - 82, 67, 76, 69, 32, 87, 73, 84, 72, 32, 78, 79, 82, 84, 72, 178, 144, 6, - 66, 155, 27, 72, 134, 1, 208, 1, 4, 66, 66, 76, 69, 46, 71, 156, 4, 4, - 72, 73, 68, 32, 36, 2, 76, 76, 122, 83, 56, 4, 84, 84, 69, 82, 220, 230, - 10, 10, 73, 76, 68, 73, 78, 71, 32, 67, 79, 78, 208, 154, 16, 2, 82, 82, - 155, 185, 6, 67, 4, 216, 195, 27, 2, 32, 84, 195, 207, 6, 83, 63, 33, 6, - 73, 78, 69, 83, 69, 32, 60, 144, 1, 7, 76, 69, 84, 84, 69, 82, 32, 172, - 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 154, 252, 15, 69, 237, - 205, 13, 4, 80, 65, 76, 76, 46, 154, 1, 77, 34, 78, 198, 141, 34, 66, 2, - 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, 83, 2, 84, - 2, 86, 2, 89, 187, 2, 65, 4, 226, 141, 34, 80, 187, 2, 65, 12, 46, 71, - 34, 89, 246, 140, 34, 82, 187, 2, 65, 4, 146, 141, 34, 75, 187, 2, 65, 4, - 242, 140, 34, 67, 187, 2, 65, 10, 182, 248, 33, 65, 214, 22, 69, 2, 73, - 2, 79, 3, 85, 40, 134, 134, 10, 76, 183, 169, 18, 86, 10, 56, 2, 69, 84, - 20, 4, 72, 79, 82, 78, 151, 149, 9, 83, 5, 211, 227, 30, 32, 5, 221, 237, - 26, 5, 32, 87, 73, 84, 72, 9, 26, 84, 203, 194, 31, 32, 4, 214, 237, 26, - 83, 15, 32, 5, 207, 233, 33, 70, 240, 3, 140, 1, 23, 90, 65, 78, 84, 73, - 78, 69, 32, 77, 85, 83, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, - 173, 254, 19, 5, 84, 69, 32, 79, 82, 238, 3, 154, 2, 65, 128, 7, 2, 67, - 72, 170, 1, 68, 158, 5, 69, 206, 2, 70, 166, 8, 71, 202, 3, 73, 162, 2, - 75, 142, 5, 76, 190, 2, 77, 250, 4, 79, 114, 80, 174, 4, 82, 42, 83, 194, - 5, 84, 196, 5, 2, 86, 65, 250, 1, 89, 238, 162, 31, 78, 149, 136, 2, 9, - 88, 73, 82, 79, 78, 32, 75, 76, 65, 62, 60, 5, 71, 79, 71, 73, 32, 222, - 1, 78, 118, 80, 199, 2, 82, 16, 70, 65, 0, 2, 71, 79, 64, 2, 77, 69, 37, - 5, 80, 79, 76, 73, 32, 4, 17, 2, 82, 71, 4, 200, 181, 15, 2, 79, 84, 211, - 210, 18, 73, 4, 134, 138, 9, 84, 147, 234, 24, 83, 4, 26, 65, 1, 2, 71, - 79, 2, 223, 150, 17, 82, 6, 72, 6, 84, 73, 75, 69, 78, 79, 169, 229, 33, - 6, 65, 84, 82, 73, 67, 72, 4, 168, 32, 2, 75, 89, 239, 227, 33, 77, 22, - 52, 5, 69, 83, 79, 32, 69, 38, 79, 247, 241, 33, 76, 4, 204, 1, 2, 88, - 79, 147, 55, 75, 16, 80, 6, 83, 84, 82, 79, 70, 79, 188, 40, 3, 68, 69, - 82, 181, 128, 1, 2, 84, 72, 10, 24, 2, 73, 32, 79, 83, 4, 56, 9, 83, 89, - 78, 68, 69, 83, 77, 79, 83, 195, 24, 84, 2, 143, 44, 32, 7, 11, 32, 4, - 214, 60, 68, 251, 213, 21, 78, 18, 48, 2, 71, 79, 33, 6, 75, 84, 73, 75, - 79, 32, 4, 170, 21, 83, 135, 238, 33, 78, 14, 194, 242, 26, 86, 218, 241, - 6, 90, 162, 8, 75, 254, 2, 68, 2, 78, 162, 17, 71, 3, 80, 14, 80, 4, 82, - 79, 65, 32, 168, 37, 4, 79, 82, 69, 86, 221, 2, 4, 65, 77, 73, 76, 6, - 204, 161, 19, 2, 83, 80, 152, 184, 3, 3, 90, 89, 71, 161, 246, 9, 3, 75, - 76, 73, 42, 50, 73, 220, 205, 8, 2, 65, 83, 143, 148, 25, 89, 38, 122, - 65, 200, 1, 5, 69, 83, 73, 83, 32, 70, 71, 208, 1, 3, 80, 76, 73, 185, - 235, 26, 8, 70, 84, 79, 71, 71, 79, 83, 32, 10, 48, 6, 83, 84, 79, 76, - 73, 32, 187, 200, 32, 82, 8, 80, 6, 65, 80, 76, 73, 32, 77, 174, 55, 68, - 177, 159, 22, 5, 84, 72, 69, 83, 69, 4, 40, 2, 69, 71, 197, 245, 26, 2, - 73, 75, 2, 171, 175, 29, 65, 12, 38, 84, 246, 50, 65, 42, 68, 63, 77, 6, - 194, 10, 69, 239, 41, 82, 10, 72, 5, 79, 82, 71, 79, 78, 181, 252, 33, 7, - 82, 65, 77, 77, 65, 32, 71, 9, 69, 15, 32, 80, 65, 82, 69, 83, 84, 73, - 71, 77, 69, 78, 79, 78, 32, 6, 222, 38, 68, 137, 10, 8, 65, 82, 73, 83, - 84, 69, 82, 65, 5, 175, 47, 32, 16, 166, 1, 78, 116, 6, 84, 69, 82, 79, - 78, 32, 188, 44, 4, 88, 79, 32, 69, 252, 236, 31, 4, 80, 69, 71, 69, 196, - 46, 6, 75, 83, 84, 82, 69, 80, 209, 13, 3, 76, 65, 70, 4, 224, 22, 3, 68, - 79, 70, 157, 211, 26, 18, 65, 82, 88, 73, 83, 32, 75, 65, 73, 32, 70, 84, - 72, 79, 82, 65, 32, 86, 4, 208, 11, 5, 65, 82, 71, 79, 83, 151, 20, 80, - 44, 180, 1, 9, 65, 78, 69, 82, 79, 83, 73, 83, 32, 52, 6, 84, 72, 79, 82, - 65, 32, 165, 6, 22, 72, 84, 79, 82, 65, 32, 83, 75, 76, 73, 82, 79, 78, - 32, 67, 72, 82, 79, 77, 65, 32, 86, 6, 246, 4, 68, 14, 77, 25, 5, 84, 69, - 84, 82, 65, 36, 220, 2, 8, 65, 82, 67, 72, 65, 73, 79, 78, 80, 10, 68, - 73, 65, 84, 79, 78, 73, 75, 73, 32, 96, 11, 73, 32, 89, 70, 69, 83, 73, - 83, 32, 84, 69, 32, 15, 77, 65, 76, 65, 75, 79, 78, 32, 67, 72, 82, 79, - 77, 65, 32, 74, 78, 40, 8, 83, 75, 76, 73, 82, 79, 78, 32, 133, 157, 18, - 18, 69, 78, 65, 82, 77, 79, 78, 73, 79, 83, 32, 65, 78, 84, 73, 70, 79, - 78, 5, 57, 12, 32, 68, 69, 89, 84, 69, 82, 79, 85, 32, 73, 67, 2, 211, - 227, 26, 72, 14, 62, 78, 218, 212, 33, 90, 162, 8, 75, 254, 2, 68, 163, - 17, 80, 6, 242, 39, 73, 151, 145, 32, 65, 2, 237, 42, 4, 84, 65, 82, 84, - 4, 18, 68, 15, 77, 2, 35, 73, 2, 21, 3, 79, 78, 79, 2, 151, 19, 70, 4, - 166, 19, 65, 193, 218, 31, 2, 69, 78, 6, 84, 7, 67, 72, 82, 79, 77, 65, - 32, 205, 225, 16, 8, 68, 73, 65, 84, 79, 78, 79, 78, 4, 42, 86, 181, 229, - 16, 4, 83, 89, 78, 65, 2, 159, 212, 31, 65, 22, 80, 6, 69, 78, 73, 75, - 73, 32, 44, 2, 79, 82, 185, 26, 5, 82, 79, 78, 84, 72, 4, 140, 150, 30, - 2, 68, 73, 1, 2, 89, 70, 16, 68, 2, 71, 79, 225, 1, 10, 84, 72, 77, 73, - 75, 79, 78, 32, 78, 32, 12, 28, 2, 78, 32, 155, 1, 83, 10, 96, 14, 80, - 65, 82, 69, 83, 84, 73, 71, 77, 69, 78, 79, 78, 32, 242, 33, 65, 113, 3, - 78, 69, 79, 4, 214, 24, 68, 233, 201, 31, 5, 65, 82, 73, 83, 84, 2, 165, - 188, 32, 5, 89, 78, 84, 72, 69, 4, 142, 28, 65, 1, 2, 68, 73, 16, 56, 2, - 77, 73, 114, 83, 213, 236, 32, 4, 67, 72, 65, 68, 8, 34, 70, 169, 28, 3, - 68, 73, 65, 6, 40, 4, 84, 72, 79, 82, 183, 226, 32, 79, 4, 218, 156, 33, - 79, 215, 79, 65, 6, 44, 6, 65, 75, 73, 65, 32, 84, 231, 13, 79, 2, 145, - 141, 21, 12, 69, 76, 79, 85, 83, 32, 73, 67, 72, 73, 77, 65, 54, 108, 2, - 65, 84, 96, 6, 69, 78, 84, 73, 77, 65, 140, 1, 5, 76, 65, 83, 77, 65, 18, - 79, 98, 82, 175, 1, 89, 6, 40, 3, 65, 86, 65, 201, 3, 2, 72, 73, 4, 140, - 29, 5, 32, 84, 82, 79, 77, 227, 171, 33, 83, 18, 24, 2, 84, 65, 15, 32, - 11, 11, 32, 8, 36, 4, 78, 69, 79, 32, 183, 28, 65, 6, 220, 144, 25, 2, - 77, 69, 166, 247, 4, 75, 235, 183, 3, 65, 7, 243, 28, 32, 8, 64, 6, 78, - 84, 69, 86, 77, 65, 250, 198, 8, 82, 195, 255, 24, 85, 5, 165, 254, 13, - 2, 32, 65, 14, 44, 4, 65, 84, 73, 77, 105, 3, 69, 77, 65, 12, 18, 65, 35, - 79, 8, 182, 23, 32, 159, 205, 33, 84, 4, 134, 12, 75, 181, 201, 30, 5, - 89, 80, 79, 82, 82, 2, 187, 194, 26, 83, 2, 251, 196, 33, 76, 14, 22, 69, - 147, 22, 89, 12, 44, 5, 73, 77, 77, 65, 32, 167, 163, 29, 77, 10, 72, 2, - 69, 78, 0, 5, 73, 77, 73, 83, 69, 54, 84, 69, 3, 68, 89, 79, 2, 173, 212, - 26, 8, 79, 83, 32, 67, 72, 82, 79, 78, 4, 44, 5, 69, 83, 83, 65, 82, 1, - 2, 82, 73, 2, 17, 2, 79, 78, 2, 25, 4, 32, 67, 72, 82, 2, 235, 217, 32, - 79, 28, 84, 8, 65, 82, 84, 89, 82, 73, 65, 32, 241, 143, 27, 7, 73, 75, - 82, 79, 78, 32, 73, 26, 72, 5, 65, 76, 76, 73, 32, 38, 68, 38, 80, 134, - 1, 86, 30, 84, 87, 76, 4, 34, 68, 177, 2, 3, 80, 82, 79, 2, 237, 2, 5, - 69, 89, 84, 69, 82, 8, 60, 7, 76, 65, 71, 73, 79, 83, 32, 45, 4, 82, 79, - 84, 79, 4, 200, 1, 5, 84, 69, 84, 65, 82, 111, 73, 4, 22, 86, 227, 1, 83, - 2, 209, 1, 3, 65, 82, 89, 8, 56, 8, 69, 84, 65, 82, 84, 79, 83, 32, 61, - 2, 82, 73, 4, 22, 76, 135, 1, 73, 2, 21, 3, 69, 71, 69, 2, 63, 84, 4, 18, - 70, 35, 84, 2, 205, 137, 21, 3, 79, 78, 73, 2, 11, 79, 2, 11, 83, 2, 17, - 2, 32, 73, 2, 195, 178, 22, 67, 18, 92, 4, 76, 73, 71, 79, 180, 1, 5, 89, - 82, 65, 78, 73, 210, 14, 88, 141, 254, 32, 2, 77, 65, 4, 211, 1, 78, 42, + 74, 79, 77, 78, 68, 201, 175, 34, 2, 71, 66, 20, 62, 69, 172, 49, 2, 78, + 83, 149, 239, 31, 4, 80, 32, 80, 73, 16, 58, 77, 234, 170, 12, 75, 226, + 168, 7, 78, 211, 200, 17, 83, 11, 186, 24, 66, 14, 71, 194, 44, 86, 239, + 220, 27, 75, 4, 146, 211, 19, 78, 171, 220, 17, 81, 6, 36, 2, 79, 77, + 237, 1, 2, 78, 32, 4, 178, 252, 12, 80, 239, 184, 23, 69, 2, 207, 243, + 35, 71, 48, 122, 65, 32, 2, 68, 65, 54, 71, 98, 75, 32, 2, 83, 72, 38, + 84, 102, 89, 74, 90, 158, 239, 35, 74, 158, 107, 69, 251, 70, 73, 4, 250, + 8, 65, 195, 164, 37, 81, 4, 22, 65, 159, 22, 32, 2, 153, 44, 3, 78, 71, + 71, 8, 52, 3, 75, 85, 69, 222, 188, 32, 65, 239, 155, 3, 71, 4, 250, 7, + 32, 165, 155, 12, 2, 78, 90, 4, 246, 25, 65, 135, 203, 19, 73, 4, 174, + 188, 35, 73, 187, 239, 1, 65, 8, 40, 2, 65, 80, 157, 157, 28, 2, 79, 81, + 7, 11, 32, 4, 188, 184, 35, 2, 77, 70, 1, 2, 78, 84, 6, 26, 73, 155, 142, + 37, 69, 5, 193, 13, 7, 84, 32, 77, 79, 78, 71, 75, 4, 214, 5, 65, 229, + 159, 12, 4, 85, 78, 32, 77, 22, 58, 65, 80, 3, 79, 78, 32, 178, 140, 37, + 69, 163, 28, 85, 10, 42, 65, 154, 18, 32, 250, 17, 77, 15, 83, 4, 230, + 190, 26, 82, 155, 234, 10, 77, 8, 56, 4, 77, 70, 79, 78, 1, 6, 80, 65, + 32, 78, 74, 73, 4, 37, 7, 32, 80, 73, 80, 65, 69, 77, 4, 206, 16, 71, + 243, 148, 37, 66, 22, 70, 72, 194, 1, 79, 192, 66, 2, 69, 85, 170, 190, + 36, 85, 167, 34, 73, 10, 74, 73, 70, 85, 241, 252, 35, 10, 79, 81, 32, + 78, 83, 72, 85, 84, 32, 89, 4, 170, 188, 26, 82, 177, 211, 6, 8, 78, 68, + 65, 32, 80, 65, 32, 78, 4, 188, 11, 4, 69, 78, 83, 72, 195, 154, 37, 77, + 6, 208, 234, 35, 2, 78, 74, 254, 186, 1, 81, 3, 84, 10, 44, 2, 69, 85, + 44, 3, 73, 84, 65, 31, 85, 4, 130, 209, 35, 65, 1, 4, 84, 69, 85, 87, 2, + 11, 32, 2, 223, 30, 77, 4, 186, 26, 32, 251, 246, 26, 65, 4, 228, 3, 5, + 32, 89, 85, 81, 32, 217, 230, 27, 3, 78, 75, 78, 112, 164, 1, 7, 71, 72, + 69, 85, 71, 72, 69, 34, 75, 126, 76, 122, 77, 154, 3, 78, 234, 2, 80, 90, + 83, 178, 1, 84, 106, 89, 190, 22, 87, 204, 46, 2, 70, 69, 131, 201, 11, + 86, 4, 198, 56, 85, 215, 233, 36, 78, 12, 40, 2, 69, 85, 50, 73, 203, + 144, 37, 65, 6, 230, 54, 89, 174, 184, 12, 80, 147, 160, 24, 65, 4, 242, + 142, 37, 69, 175, 18, 81, 8, 46, 65, 240, 69, 2, 79, 77, 183, 190, 36, + 69, 4, 50, 65, 129, 61, 7, 77, 32, 78, 83, 72, 85, 84, 2, 131, 182, 26, + 78, 26, 70, 65, 74, 66, 140, 1, 2, 69, 85, 58, 70, 145, 23, 3, 79, 78, + 84, 7, 21, 3, 32, 78, 74, 4, 178, 212, 18, 85, 229, 142, 17, 3, 69, 85, + 65, 10, 90, 65, 154, 46, 85, 184, 146, 30, 2, 69, 85, 133, 139, 6, 7, 73, + 84, 32, 77, 66, 65, 65, 4, 150, 12, 65, 205, 192, 19, 4, 32, 77, 65, 69, + 4, 252, 141, 32, 5, 84, 32, 78, 71, 71, 239, 143, 5, 81, 4, 48, 4, 79, + 78, 32, 84, 193, 204, 26, 2, 73, 89, 2, 163, 65, 69, 24, 110, 71, 166, 1, + 83, 50, 89, 152, 23, 8, 84, 73, 69, 69, 32, 83, 72, 69, 201, 144, 35, 5, + 68, 85, 32, 78, 74, 12, 70, 71, 164, 162, 36, 8, 75, 73, 78, 68, 73, 32, + 77, 86, 191, 104, 79, 8, 60, 3, 85, 79, 81, 204, 211, 19, 2, 69, 85, 179, + 140, 16, 65, 5, 193, 180, 28, 2, 32, 76, 4, 26, 72, 227, 202, 36, 69, 2, + 167, 229, 36, 85, 4, 248, 45, 2, 65, 69, 211, 17, 73, 8, 200, 195, 10, 2, + 69, 69, 218, 169, 12, 65, 164, 182, 8, 3, 85, 78, 71, 151, 218, 5, 73, + 12, 88, 2, 65, 75, 16, 2, 72, 69, 252, 185, 19, 2, 69, 84, 146, 239, 15, + 73, 231, 216, 1, 85, 2, 211, 25, 69, 4, 128, 200, 26, 4, 84, 32, 78, 74, + 241, 128, 5, 4, 85, 65, 69, 81, 6, 32, 2, 85, 32, 227, 220, 35, 65, 4, + 36, 4, 77, 65, 69, 77, 139, 7, 78, 2, 11, 71, 2, 135, 7, 66, 4, 44, 4, + 65, 70, 85, 32, 229, 4, 2, 69, 85, 2, 221, 165, 32, 6, 76, 69, 69, 82, + 65, 69, 196, 1, 170, 1, 71, 82, 75, 206, 2, 76, 50, 77, 250, 3, 78, 238, + 6, 80, 78, 83, 118, 84, 176, 1, 3, 86, 69, 85, 46, 87, 62, 89, 202, 157, + 30, 66, 182, 218, 2, 70, 163, 230, 3, 82, 6, 40, 2, 72, 65, 185, 184, 19, + 2, 66, 65, 4, 198, 170, 26, 82, 155, 234, 10, 80, 22, 66, 69, 182, 1, 85, + 132, 194, 26, 3, 80, 65, 82, 139, 206, 10, 65, 14, 40, 2, 78, 32, 54, 85, + 155, 147, 37, 84, 4, 180, 136, 33, 4, 70, 65, 84, 73, 235, 140, 3, 76, 8, + 42, 83, 174, 194, 26, 75, 195, 208, 10, 77, 4, 232, 14, 3, 72, 69, 85, + 239, 53, 69, 4, 48, 6, 79, 80, 32, 78, 75, 65, 147, 146, 37, 84, 2, 11, + 65, 2, 235, 167, 26, 82, 8, 202, 47, 65, 242, 145, 26, 73, 183, 208, 10, + 85, 38, 82, 65, 130, 1, 66, 214, 165, 26, 85, 216, 25, 4, 71, 66, 65, 83, + 139, 225, 8, 73, 8, 18, 32, 79, 69, 4, 42, 78, 145, 231, 17, 4, 75, 69, + 85, 65, 2, 11, 83, 2, 231, 160, 35, 73, 4, 226, 226, 36, 77, 211, 25, 83, + 24, 34, 65, 114, 69, 82, 73, 35, 85, 6, 32, 2, 65, 32, 255, 178, 19, 78, + 4, 200, 206, 31, 8, 67, 65, 66, 66, 65, 71, 69, 45, 133, 238, 4, 2, 80, + 73, 8, 50, 85, 142, 164, 26, 82, 173, 218, 5, 2, 69, 75, 4, 162, 142, 37, + 77, 3, 88, 7, 226, 7, 82, 167, 134, 37, 84, 4, 186, 251, 36, 65, 175, 18, + 69, 72, 130, 1, 65, 54, 68, 110, 71, 222, 1, 74, 102, 83, 130, 1, 84, + 102, 90, 237, 7, 12, 89, 73, 82, 32, 77, 75, 80, 65, 82, 65, 81, 32, 4, + 128, 188, 26, 4, 78, 83, 65, 78, 195, 208, 10, 81, 12, 60, 2, 69, 85, + 174, 41, 65, 134, 155, 19, 79, 159, 162, 17, 73, 4, 136, 156, 35, 2, 65, + 69, 199, 239, 1, 84, 20, 50, 71, 98, 75, 222, 185, 26, 65, 223, 191, 10, + 79, 12, 26, 85, 247, 186, 36, 69, 11, 180, 39, 3, 65, 69, 78, 190, 147, + 36, 79, 202, 26, 69, 155, 53, 77, 4, 36, 3, 85, 69, 32, 183, 185, 26, 65, + 2, 241, 179, 19, 3, 77, 65, 69, 10, 34, 65, 34, 69, 183, 214, 12, 85, 4, + 202, 248, 36, 69, 219, 16, 77, 4, 202, 153, 35, 69, 151, 99, 85, 12, 78, + 85, 214, 183, 26, 72, 204, 233, 5, 2, 69, 85, 158, 214, 4, 79, 219, 16, + 65, 4, 224, 244, 36, 4, 79, 84, 32, 78, 179, 19, 78, 8, 54, 69, 244, 234, + 36, 4, 85, 32, 77, 66, 131, 26, 65, 4, 168, 175, 19, 2, 85, 78, 131, 216, + 17, 78, 4, 238, 221, 35, 69, 147, 169, 1, 65, 6, 26, 73, 227, 182, 36, + 69, 4, 26, 82, 167, 134, 37, 78, 2, 175, 178, 35, 73, 10, 30, 69, 50, 72, + 255, 6, 85, 4, 26, 84, 179, 249, 35, 85, 2, 231, 181, 36, 70, 4, 238, + 210, 12, 85, 159, 226, 13, 73, 10, 40, 2, 65, 65, 34, 69, 41, 2, 73, 84, + 2, 11, 83, 2, 203, 154, 26, 72, 4, 228, 25, 2, 85, 84, 235, 234, 36, 84, + 4, 38, 85, 249, 248, 32, 3, 65, 32, 89, 2, 143, 241, 26, 65, 4, 236, 245, + 27, 2, 65, 69, 255, 141, 9, 88, 4, 40, 4, 65, 78, 71, 75, 139, 131, 37, + 85, 2, 143, 19, 85, 10, 42, 85, 158, 208, 12, 69, 135, 176, 24, 65, 6, + 210, 18, 87, 212, 3, 4, 32, 77, 85, 79, 179, 236, 36, 77, 234, 1, 134, 1, + 70, 68, 2, 71, 72, 34, 75, 254, 1, 76, 142, 1, 77, 130, 3, 78, 130, 5, + 80, 122, 82, 90, 83, 190, 1, 84, 158, 1, 87, 39, 89, 4, 36, 3, 69, 85, + 70, 179, 254, 36, 65, 2, 11, 69, 2, 151, 2, 85, 4, 202, 1, 69, 203, 252, + 36, 65, 22, 46, 69, 146, 1, 85, 42, 87, 143, 143, 35, 89, 10, 26, 85, + 227, 255, 36, 84, 8, 72, 3, 65, 69, 84, 20, 5, 79, 84, 32, 77, 66, 130, + 255, 36, 77, 3, 80, 2, 235, 249, 11, 77, 2, 231, 148, 26, 85, 9, 146, + 238, 36, 79, 218, 16, 78, 3, 81, 2, 171, 201, 36, 65, 14, 58, 69, 194, + 173, 26, 79, 254, 224, 8, 73, 227, 222, 1, 85, 8, 42, 85, 146, 142, 35, + 69, 187, 239, 1, 84, 4, 214, 234, 26, 65, 243, 146, 10, 77, 41, 94, 65, + 58, 66, 66, 69, 38, 70, 80, 2, 71, 66, 214, 254, 31, 79, 242, 130, 4, 86, + 151, 121, 85, 4, 164, 211, 17, 2, 76, 69, 133, 244, 18, 3, 69, 78, 74, 6, + 32, 2, 65, 65, 247, 220, 36, 85, 5, 189, 231, 28, 2, 32, 83, 6, 234, 176, + 18, 85, 171, 219, 16, 69, 10, 44, 2, 69, 85, 246, 255, 34, 79, 207, 11, + 73, 4, 162, 228, 36, 65, 215, 22, 84, 6, 158, 139, 35, 73, 168, 70, 2, + 79, 70, 227, 40, 69, 72, 78, 68, 46, 71, 226, 1, 74, 98, 83, 110, 84, 34, + 89, 222, 245, 36, 73, 3, 85, 8, 194, 39, 69, 178, 130, 36, 79, 139, 63, + 65, 24, 18, 71, 99, 75, 12, 54, 65, 202, 42, 69, 162, 231, 31, 87, 147, + 214, 4, 85, 6, 152, 38, 2, 65, 77, 195, 210, 36, 80, 12, 68, 2, 69, 85, + 182, 136, 35, 73, 2, 89, 218, 159, 1, 85, 215, 79, 65, 4, 154, 197, 12, + 65, 251, 158, 24, 82, 12, 60, 2, 69, 85, 230, 15, 73, 214, 180, 12, 85, + 199, 178, 24, 65, 4, 218, 228, 36, 65, 175, 18, 84, 10, 46, 72, 32, 3, + 73, 69, 69, 195, 229, 36, 85, 4, 138, 218, 36, 85, 219, 5, 69, 4, 150, + 246, 36, 80, 3, 84, 6, 130, 14, 69, 147, 195, 36, 85, 8, 174, 217, 36, + 69, 218, 5, 85, 254, 5, 65, 219, 16, 73, 12, 42, 69, 46, 85, 194, 244, + 36, 65, 3, 73, 4, 220, 138, 26, 2, 85, 84, 155, 234, 10, 69, 4, 158, 216, + 36, 85, 175, 28, 81, 8, 48, 3, 69, 78, 32, 162, 224, 36, 73, 175, 1, 65, + 4, 238, 213, 26, 79, 215, 251, 9, 77, 26, 58, 72, 90, 85, 186, 16, 65, + 174, 6, 69, 179, 191, 36, 79, 12, 54, 69, 174, 162, 26, 79, 222, 188, 10, + 73, 219, 19, 85, 6, 214, 7, 85, 139, 235, 36, 69, 6, 234, 219, 36, 65, + 214, 22, 69, 3, 85, 18, 62, 69, 74, 85, 222, 160, 26, 79, 226, 185, 10, + 65, 215, 22, 73, 8, 26, 85, 135, 130, 35, 69, 6, 210, 157, 35, 65, 234, + 211, 1, 78, 3, 84, 5, 227, 212, 36, 79, 4, 134, 138, 32, 85, 235, 230, 4, + 65, 10, 40, 2, 65, 69, 18, 85, 191, 160, 36, 69, 2, 251, 3, 77, 6, 22, + 87, 227, 13, 79, 2, 207, 159, 26, 79, 186, 2, 178, 1, 70, 154, 1, 71, + 202, 1, 75, 170, 1, 76, 158, 1, 77, 134, 2, 78, 190, 7, 80, 166, 2, 82, + 78, 83, 166, 1, 84, 246, 1, 86, 66, 87, 34, 89, 238, 216, 36, 65, 2, 73, + 3, 79, 18, 54, 85, 234, 129, 23, 65, 198, 213, 13, 69, 255, 5, 79, 10, + 26, 32, 135, 142, 31, 69, 6, 132, 229, 23, 4, 82, 69, 77, 69, 146, 129, + 11, 67, 207, 135, 2, 73, 16, 24, 2, 66, 69, 39, 72, 4, 222, 224, 35, 85, + 167, 140, 1, 84, 12, 34, 65, 34, 69, 199, 219, 36, 79, 2, 11, 65, 2, 151, + 130, 26, 77, 8, 26, 85, 131, 236, 36, 84, 6, 170, 213, 36, 65, 214, 22, + 78, 3, 88, 18, 50, 69, 62, 80, 18, 85, 218, 234, 36, 73, 3, 79, 6, 26, + 85, 139, 235, 36, 84, 4, 178, 212, 36, 65, 215, 22, 88, 2, 211, 28, 69, + 6, 170, 206, 36, 69, 162, 28, 79, 15, 84, 18, 50, 65, 40, 2, 69, 85, 22, + 79, 195, 233, 36, 85, 6, 162, 217, 36, 65, 218, 16, 80, 3, 81, 2, 167, + 215, 36, 65, 8, 222, 158, 18, 79, 226, 202, 18, 77, 3, 81, 32, 110, 65, + 44, 2, 66, 69, 34, 70, 20, 2, 71, 66, 34, 73, 142, 253, 25, 85, 186, 154, + 10, 69, 2, 79, 139, 60, 86, 11, 250, 242, 17, 69, 170, 245, 18, 80, 3, + 81, 4, 158, 215, 36, 85, 219, 16, 69, 2, 155, 181, 12, 69, 4, 226, 151, + 36, 69, 227, 79, 65, 5, 207, 208, 36, 69, 80, 114, 68, 170, 1, 71, 138, + 3, 74, 112, 2, 83, 72, 58, 84, 32, 3, 89, 73, 32, 54, 90, 210, 159, 36, + 65, 191, 47, 75, 12, 34, 65, 98, 73, 187, 149, 36, 85, 6, 32, 2, 65, 32, + 215, 229, 36, 80, 4, 240, 164, 18, 3, 77, 89, 32, 137, 177, 13, 3, 83, + 79, 70, 4, 226, 148, 26, 65, 183, 208, 10, 81, 36, 78, 71, 50, 85, 122, + 75, 118, 79, 172, 221, 11, 3, 69, 85, 82, 223, 132, 25, 65, 12, 18, 69, + 51, 85, 6, 26, 85, 191, 244, 34, 69, 4, 255, 144, 24, 65, 6, 64, 6, 65, + 69, 83, 72, 65, 69, 134, 249, 25, 82, 155, 234, 10, 80, 2, 11, 32, 2, + 183, 189, 23, 78, 12, 34, 65, 20, 2, 69, 85, 35, 85, 5, 243, 206, 36, 65, + 4, 150, 208, 36, 65, 175, 18, 88, 4, 162, 226, 36, 77, 3, 80, 4, 134, + 226, 36, 80, 3, 81, 8, 18, 65, 31, 69, 2, 201, 230, 31, 2, 69, 77, 6, 26, + 69, 255, 212, 35, 85, 5, 245, 190, 36, 4, 32, 69, 80, 79, 6, 26, 85, 171, + 241, 34, 73, 4, 210, 224, 36, 79, 15, 69, 4, 234, 207, 36, 85, 207, 16, + 65, 4, 140, 159, 26, 4, 67, 76, 69, 65, 151, 244, 1, 66, 4, 186, 143, 26, + 65, 3, 85, 34, 42, 65, 90, 69, 58, 73, 50, 79, 23, 85, 8, 32, 2, 32, 80, + 223, 233, 17, 65, 4, 196, 148, 28, 2, 69, 79, 241, 192, 7, 2, 76, 85, 6, + 26, 85, 223, 205, 36, 69, 4, 178, 222, 36, 84, 3, 88, 7, 11, 69, 4, 214, + 141, 26, 69, 183, 208, 10, 84, 5, 135, 142, 36, 79, 11, 82, 65, 130, 221, + 36, 69, 3, 77, 8, 46, 65, 238, 14, 69, 149, 246, 18, 2, 73, 77, 4, 254, + 220, 36, 69, 3, 81, 18, 62, 69, 30, 72, 150, 245, 31, 85, 158, 214, 4, + 79, 163, 14, 65, 4, 162, 220, 36, 69, 3, 84, 8, 42, 69, 198, 239, 22, 79, + 231, 155, 3, 73, 2, 137, 169, 12, 2, 85, 65, 28, 34, 65, 94, 69, 50, 79, + 39, 85, 10, 56, 2, 69, 78, 202, 238, 22, 65, 154, 236, 13, 77, 3, 81, 2, + 205, 209, 11, 3, 32, 78, 84, 6, 26, 85, 167, 218, 36, 78, 5, 211, 167, + 12, 65, 6, 138, 223, 34, 79, 135, 251, 1, 81, 6, 170, 7, 77, 239, 187, + 36, 65, 6, 26, 69, 219, 200, 36, 79, 4, 150, 239, 25, 85, 155, 234, 10, + 69, 6, 246, 10, 69, 207, 187, 32, 85, 26, 68, 2, 69, 85, 46, 73, 32, 3, + 79, 81, 32, 54, 85, 155, 215, 36, 65, 8, 174, 133, 24, 65, 246, 210, 12, + 77, 3, 88, 4, 162, 193, 36, 69, 215, 22, 84, 4, 180, 195, 12, 4, 83, 87, + 73, 77, 227, 197, 9, 67, 8, 238, 134, 26, 69, 206, 252, 8, 65, 234, 211, + 1, 78, 3, 81, 108, 162, 1, 75, 82, 76, 46, 77, 98, 78, 190, 1, 80, 66, + 82, 50, 83, 110, 84, 38, 89, 146, 222, 2, 87, 204, 191, 9, 2, 86, 85, + 254, 155, 24, 69, 242, 5, 70, 231, 16, 85, 14, 226, 138, 18, 69, 170, + 219, 16, 89, 130, 237, 1, 80, 186, 2, 65, 2, 79, 3, 85, 6, 190, 132, 26, + 79, 182, 208, 10, 65, 3, 73, 13, 42, 66, 34, 69, 254, 211, 36, 65, 3, 79, + 4, 186, 132, 36, 69, 171, 77, 65, 2, 195, 228, 34, 69, 22, 94, 71, 38, + 74, 38, 85, 130, 216, 34, 83, 246, 7, 68, 150, 3, 84, 226, 219, 1, 89, + 219, 19, 73, 4, 134, 236, 31, 75, 203, 228, 4, 71, 4, 202, 232, 25, 85, + 239, 215, 10, 65, 5, 235, 187, 36, 65, 6, 26, 69, 251, 231, 25, 85, 4, + 234, 197, 35, 85, 167, 140, 1, 69, 12, 186, 2, 69, 150, 134, 9, 73, 159, + 201, 27, 85, 14, 66, 72, 230, 2, 69, 162, 248, 18, 65, 142, 177, 17, 85, + 235, 36, 73, 6, 158, 189, 36, 73, 218, 19, 79, 3, 85, 6, 182, 141, 21, + 65, 239, 166, 15, 69, 4, 246, 255, 25, 79, 183, 208, 10, 65, 4, 178, 185, + 36, 65, 215, 22, 69, 14, 54, 69, 150, 134, 9, 73, 202, 178, 27, 65, 215, + 22, 85, 6, 138, 195, 35, 85, 166, 140, 1, 69, 3, 78, 16, 62, 72, 50, 69, + 162, 248, 18, 65, 142, 177, 17, 85, 235, 36, 73, 8, 46, 69, 190, 186, 36, + 73, 218, 19, 79, 3, 85, 2, 239, 193, 35, 85, 10, 158, 131, 18, 69, 202, + 135, 3, 65, 155, 195, 15, 73, 6, 150, 253, 25, 79, 2, 85, 183, 208, 10, + 65, 14, 42, 75, 242, 144, 35, 65, 147, 157, 1, 74, 11, 49, 10, 78, 79, + 84, 69, 32, 87, 73, 84, 72, 32, 8, 184, 207, 9, 2, 80, 79, 198, 1, 89, + 156, 177, 12, 2, 69, 85, 163, 177, 6, 68, 6, 38, 32, 173, 244, 16, 3, 66, + 69, 82, 4, 134, 202, 29, 67, 213, 240, 5, 5, 79, 70, 32, 83, 79, 78, 72, + 3, 75, 69, 84, 60, 7, 83, 65, 32, 86, 65, 72, 32, 179, 194, 34, 69, 5, + 233, 156, 16, 10, 66, 65, 76, 76, 32, 65, 78, 68, 32, 72, 72, 104, 10, + 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 164, 1, 7, 76, 69, 84, 84, 69, + 82, 32, 151, 246, 33, 70, 10, 52, 4, 72, 73, 71, 72, 44, 3, 76, 79, 87, + 39, 77, 4, 236, 219, 21, 2, 45, 76, 159, 252, 10, 32, 4, 32, 2, 45, 77, + 191, 215, 32, 32, 2, 173, 215, 32, 2, 73, 68, 60, 238, 1, 68, 38, 69, 38, + 71, 34, 75, 38, 85, 20, 2, 87, 65, 22, 89, 212, 228, 29, 2, 72, 87, 230, + 191, 1, 77, 190, 218, 2, 79, 250, 160, 2, 86, 246, 5, 74, 2, 84, 2, 90, + 162, 8, 67, 2, 83, 158, 20, 66, 2, 70, 2, 80, 186, 2, 65, 3, 73, 4, 138, + 131, 34, 72, 255, 194, 2, 79, 7, 222, 201, 31, 78, 135, 252, 4, 69, 4, + 214, 160, 36, 66, 219, 35, 65, 4, 158, 229, 29, 80, 131, 224, 6, 65, 5, + 147, 160, 36, 87, 5, 227, 159, 36, 68, 4, 254, 196, 35, 69, 131, 105, 73, + 121, 48, 3, 65, 75, 32, 230, 10, 72, 143, 174, 22, 84, 112, 196, 1, 15, + 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 28, 7, 76, + 69, 84, 84, 69, 82, 32, 248, 4, 3, 80, 65, 78, 50, 83, 141, 2, 11, 86, + 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 4, 226, 193, 36, 78, 87, 72, 76, + 194, 1, 77, 114, 78, 68, 2, 80, 65, 38, 83, 196, 222, 18, 4, 75, 65, 82, + 79, 134, 222, 17, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 76, 2, 82, 2, + 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 10, 26, 65, 135, 190, 36, 66, 9, 45, + 9, 78, 68, 65, 73, 76, 73, 78, 71, 32, 6, 210, 189, 36, 72, 2, 78, 3, 83, + 10, 152, 2, 2, 79, 82, 150, 187, 36, 68, 2, 71, 2, 89, 187, 2, 65, 5, + 189, 244, 20, 4, 75, 80, 65, 75, 24, 80, 10, 73, 77, 65, 76, 85, 78, 71, + 85, 78, 32, 96, 2, 79, 85, 207, 189, 36, 65, 20, 242, 187, 36, 71, 2, 72, + 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 87, 2, 89, 187, 2, 65, 2, 237, 222, + 18, 5, 84, 72, 69, 82, 78, 4, 166, 2, 71, 253, 222, 18, 4, 79, 78, 71, + 79, 10, 96, 12, 89, 77, 66, 79, 76, 32, 66, 73, 78, 68, 85, 32, 249, 179, + 10, 6, 73, 71, 78, 32, 84, 79, 8, 78, 80, 136, 159, 11, 3, 74, 85, 68, + 149, 153, 25, 6, 78, 65, 32, 77, 69, 84, 4, 64, 3, 65, 78, 71, 241, 156, + 23, 7, 73, 78, 65, 82, 66, 79, 82, 2, 151, 228, 25, 79, 18, 122, 85, 252, + 146, 26, 6, 80, 65, 75, 80, 65, 75, 232, 250, 2, 5, 75, 65, 82, 79, 32, + 234, 235, 6, 69, 162, 64, 73, 3, 79, 5, 241, 187, 30, 15, 32, 70, 79, 82, + 32, 83, 73, 77, 65, 76, 85, 78, 71, 85, 78, 5, 195, 219, 23, 84, 254, 1, + 134, 1, 65, 230, 2, 69, 50, 76, 130, 1, 78, 190, 10, 84, 224, 228, 32, 2, + 67, 65, 248, 183, 2, 5, 86, 69, 82, 65, 71, 215, 140, 1, 68, 20, 136, 1, + 4, 77, 69, 68, 32, 170, 1, 82, 136, 169, 3, 10, 67, 72, 32, 87, 73, 84, + 72, 32, 85, 77, 178, 158, 7, 84, 166, 176, 25, 86, 19, 78, 8, 86, 65, 0, + 2, 68, 69, 52, 4, 69, 73, 71, 72, 1, 7, 83, 73, 88, 84, 69, 69, 78, 2, + 197, 164, 20, 8, 83, 67, 69, 78, 68, 73, 78, 71, 2, 189, 164, 20, 2, 84, + 72, 4, 220, 181, 32, 3, 68, 69, 68, 215, 194, 3, 32, 4, 172, 197, 8, 3, + 82, 32, 77, 195, 203, 26, 84, 13, 11, 76, 11, 38, 32, 149, 156, 25, 3, + 72, 79, 80, 6, 238, 185, 12, 80, 152, 160, 16, 6, 87, 73, 84, 72, 32, 67, + 159, 199, 6, 83, 208, 1, 84, 5, 71, 65, 76, 73, 32, 158, 9, 84, 37, 9, + 90, 69, 78, 69, 32, 82, 73, 78, 71, 200, 1, 210, 1, 65, 40, 9, 67, 85, + 82, 82, 69, 78, 67, 89, 32, 148, 2, 7, 76, 69, 84, 84, 69, 82, 32, 204, + 3, 6, 82, 85, 80, 69, 69, 32, 34, 83, 250, 213, 22, 73, 134, 4, 86, 242, + 223, 11, 68, 249, 107, 3, 71, 65, 78, 6, 154, 154, 32, 66, 50, 78, 187, + 57, 85, 12, 120, 10, 78, 85, 77, 69, 82, 65, 84, 79, 82, 32, 209, 170, + 23, 14, 68, 69, 78, 79, 77, 73, 78, 65, 84, 79, 82, 32, 83, 73, 10, 52, + 3, 79, 78, 69, 218, 156, 31, 70, 163, 163, 3, 84, 5, 157, 177, 29, 19, + 32, 76, 69, 83, 83, 32, 84, 72, 65, 78, 32, 84, 72, 69, 32, 68, 69, 78, + 79, 108, 226, 1, 75, 90, 82, 146, 184, 21, 86, 138, 138, 8, 89, 134, 144, + 3, 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, + 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 80, 138, 69, 72, 2, 76, 2, 77, 186, + 2, 69, 3, 79, 8, 26, 72, 251, 172, 36, 65, 6, 26, 65, 219, 254, 13, 73, + 5, 253, 205, 18, 3, 78, 68, 65, 10, 34, 65, 226, 169, 36, 72, 3, 82, 7, + 33, 6, 32, 87, 73, 84, 72, 32, 4, 180, 148, 25, 5, 76, 79, 87, 69, 82, 1, + 6, 77, 73, 68, 68, 76, 69, 4, 194, 164, 35, 83, 191, 69, 77, 20, 116, 19, + 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, + 32, 182, 255, 25, 65, 219, 149, 6, 73, 6, 158, 216, 22, 82, 155, 207, 13, + 89, 4, 226, 146, 26, 32, 199, 138, 9, 79, 5, 133, 179, 33, 3, 32, 87, 73, + 4, 182, 213, 34, 87, 219, 64, 32, 194, 1, 184, 2, 7, 76, 69, 84, 84, 69, + 82, 32, 244, 1, 7, 78, 85, 77, 66, 69, 82, 32, 72, 5, 83, 73, 71, 78, 32, + 48, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 142, 245, 25, 68, + 170, 243, 3, 87, 152, 186, 2, 10, 71, 65, 80, 32, 70, 73, 76, 76, 69, 82, + 205, 182, 3, 12, 72, 85, 78, 68, 82, 69, 68, 83, 32, 85, 78, 73, 92, 210, + 1, 86, 218, 202, 32, 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, + 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, + 69, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 8, 234, 1, 79, + 143, 163, 36, 65, 36, 142, 126, 69, 38, 70, 66, 78, 26, 83, 138, 173, 21, + 84, 155, 223, 12, 79, 10, 206, 143, 32, 67, 210, 61, 65, 231, 147, 3, 86, + 24, 80, 2, 86, 79, 194, 207, 32, 65, 38, 85, 206, 201, 1, 73, 222, 137, + 2, 69, 3, 79, 6, 33, 6, 67, 65, 76, 73, 67, 32, 6, 158, 208, 32, 82, 203, + 210, 3, 76, 26, 148, 1, 4, 67, 89, 67, 76, 36, 2, 71, 32, 28, 2, 76, 76, + 46, 82, 66, 84, 204, 166, 19, 4, 79, 72, 65, 90, 176, 253, 11, 2, 75, 73, + 155, 172, 4, 83, 4, 174, 183, 32, 73, 255, 233, 3, 69, 4, 178, 139, 34, + 82, 67, 83, 4, 216, 215, 9, 2, 69, 68, 199, 235, 23, 73, 4, 140, 184, 32, + 7, 84, 72, 68, 65, 89, 32, 67, 179, 232, 3, 68, 4, 188, 211, 13, 2, 67, + 79, 137, 203, 22, 4, 73, 78, 71, 32, 192, 7, 42, 65, 174, 33, 79, 165, + 11, 2, 85, 69, 242, 2, 32, 2, 67, 75, 247, 236, 17, 78, 240, 2, 22, 32, + 223, 31, 45, 224, 2, 210, 1, 67, 254, 4, 68, 174, 2, 70, 102, 72, 82, 76, + 186, 4, 77, 250, 2, 78, 38, 80, 46, 82, 150, 4, 83, 154, 4, 84, 82, 85, + 248, 2, 3, 86, 69, 82, 214, 128, 12, 79, 156, 205, 20, 3, 66, 79, 87, + 251, 247, 1, 81, 98, 196, 1, 5, 73, 82, 67, 76, 69, 200, 1, 6, 85, 82, + 86, 69, 68, 32, 224, 251, 25, 12, 82, 79, 83, 83, 32, 79, 78, 32, 83, 72, + 73, 69, 180, 158, 3, 5, 69, 78, 84, 82, 69, 230, 161, 5, 72, 211, 13, 76, + 11, 11, 32, 8, 72, 5, 87, 73, 84, 72, 32, 217, 235, 16, 7, 70, 79, 82, + 32, 82, 69, 67, 6, 140, 69, 8, 87, 72, 73, 84, 69, 32, 68, 79, 198, 242, + 18, 68, 161, 146, 2, 8, 84, 87, 79, 32, 87, 72, 73, 84, 16, 84, 4, 68, + 79, 87, 78, 0, 2, 85, 80, 56, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 4, + 229, 252, 31, 9, 87, 65, 82, 68, 83, 32, 65, 78, 68, 4, 53, 11, 84, 87, + 65, 82, 68, 83, 32, 65, 78, 68, 32, 4, 158, 143, 21, 85, 231, 154, 12, + 68, 30, 64, 6, 73, 65, 77, 79, 78, 68, 152, 1, 3, 79, 87, 78, 43, 82, 13, + 11, 32, 10, 154, 19, 67, 244, 250, 12, 10, 77, 73, 78, 85, 83, 32, 87, + 72, 73, 84, 152, 166, 6, 6, 87, 73, 84, 72, 32, 68, 206, 161, 15, 79, + 135, 13, 83, 12, 214, 19, 32, 62, 45, 247, 167, 31, 87, 6, 212, 182, 34, + 2, 79, 80, 231, 20, 65, 8, 18, 76, 39, 79, 4, 206, 234, 27, 79, 155, 170, + 8, 65, 4, 252, 217, 2, 2, 85, 82, 151, 155, 31, 76, 12, 38, 69, 218, 206, + 34, 65, 251, 2, 79, 6, 160, 208, 34, 2, 65, 82, 247, 11, 88, 40, 64, 5, + 65, 82, 71, 69, 32, 140, 2, 3, 69, 70, 84, 203, 1, 79, 12, 48, 6, 67, 73, + 82, 67, 76, 69, 203, 242, 34, 83, 11, 37, 7, 32, 77, 73, 78, 85, 83, 32, + 8, 54, 76, 36, 4, 82, 73, 71, 72, 13, 3, 85, 80, 80, 4, 32, 2, 69, 70, + 13, 2, 79, 87, 2, 31, 84, 2, 17, 2, 69, 82, 2, 173, 147, 33, 8, 32, 81, + 85, 65, 82, 84, 69, 82, 22, 96, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 64, 6, 87, 65, 82, 68, 83, 32, 235, 207, 34, 32, 12, 146, 7, 68, 190, + 8, 73, 190, 195, 34, 80, 206, 24, 83, 51, 84, 4, 242, 164, 33, 69, 255, + 139, 1, 66, 6, 158, 15, 87, 175, 203, 34, 90, 30, 76, 6, 69, 68, 73, 85, + 77, 32, 245, 222, 21, 7, 79, 79, 78, 32, 76, 73, 76, 28, 66, 68, 42, 76, + 36, 4, 82, 73, 71, 72, 12, 2, 85, 80, 111, 83, 6, 84, 3, 79, 87, 78, 163, + 209, 34, 73, 6, 32, 2, 69, 70, 195, 216, 34, 79, 4, 11, 84, 4, 81, 18, + 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 84, 82, 73, 65, 78, 71, 76, 69, + 5, 145, 9, 2, 32, 67, 8, 198, 13, 77, 247, 222, 34, 81, 4, 238, 145, 17, + 69, 155, 191, 17, 73, 8, 238, 135, 25, 85, 158, 201, 9, 65, 87, 69, 34, + 52, 4, 73, 71, 72, 84, 254, 252, 33, 79, 155, 85, 69, 30, 94, 32, 84, 10, + 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 245, 1, 6, 87, 65, 82, 68, 83, + 32, 6, 60, 9, 84, 82, 73, 65, 78, 71, 76, 69, 32, 171, 227, 34, 80, 2, + 231, 204, 6, 67, 16, 90, 68, 88, 8, 84, 82, 73, 65, 78, 71, 76, 69, 230, + 7, 73, 194, 200, 34, 80, 203, 19, 83, 4, 65, 14, 79, 85, 66, 76, 69, 32, + 84, 82, 73, 65, 78, 71, 76, 69, 5, 207, 189, 12, 32, 5, 205, 229, 20, 11, + 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 8, 230, 158, 19, 65, 162, + 254, 13, 69, 255, 139, 1, 66, 40, 158, 2, 77, 148, 1, 5, 81, 85, 65, 82, + 69, 180, 253, 25, 3, 85, 78, 32, 172, 223, 1, 5, 75, 85, 76, 76, 32, 160, + 178, 3, 3, 78, 79, 87, 168, 223, 1, 5, 65, 70, 69, 84, 89, 192, 131, 1, + 13, 76, 73, 71, 72, 84, 76, 89, 32, 83, 77, 65, 76, 76, 254, 91, 67, 50, + 72, 222, 1, 80, 175, 19, 84, 12, 40, 4, 65, 76, 76, 32, 203, 208, 34, 73, + 10, 214, 200, 34, 68, 150, 7, 76, 70, 83, 213, 15, 13, 85, 80, 45, 80, + 79, 73, 78, 84, 73, 78, 71, 32, 67, 9, 11, 32, 6, 54, 67, 208, 178, 33, + 3, 70, 79, 82, 163, 158, 1, 66, 2, 181, 248, 30, 3, 69, 78, 84, 14, 192, + 4, 3, 73, 78, 89, 242, 174, 23, 82, 154, 165, 11, 79, 54, 69, 135, 2, 87, + 18, 38, 80, 229, 213, 32, 3, 78, 73, 86, 16, 46, 32, 62, 45, 170, 1, 80, + 207, 166, 31, 87, 2, 233, 219, 34, 10, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 66, 8, 45, 9, 80, 79, 73, 78, 84, 73, 78, 71, 32, 8, 66, 73, 156, + 216, 34, 5, 68, 79, 85, 66, 76, 238, 3, 83, 51, 84, 2, 213, 218, 33, 8, + 83, 79, 83, 67, 69, 76, 69, 83, 4, 21, 3, 69, 82, 32, 4, 210, 182, 24, + 76, 207, 163, 9, 82, 10, 56, 6, 84, 73, 67, 65, 76, 32, 41, 4, 89, 32, + 83, 77, 4, 152, 198, 34, 2, 82, 69, 239, 22, 69, 6, 21, 3, 65, 76, 76, 6, + 11, 32, 6, 186, 194, 34, 68, 150, 7, 76, 247, 20, 83, 16, 84, 15, 76, 69, + 84, 84, 69, 82, 32, 67, 65, 80, 73, 84, 65, 76, 32, 227, 146, 11, 70, 10, + 154, 254, 35, 67, 2, 72, 2, 73, 2, 82, 3, 90, 200, 4, 52, 3, 67, 75, 32, + 166, 151, 2, 83, 183, 201, 10, 87, 196, 4, 80, 7, 79, 67, 84, 65, 78, 84, + 45, 149, 7, 8, 83, 69, 88, 84, 65, 78, 84, 45, 204, 3, 58, 49, 130, 3, + 50, 114, 53, 50, 54, 18, 51, 215, 1, 52, 234, 1, 74, 50, 246, 1, 51, 174, + 91, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 116, 62, 51, 226, 92, + 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 55, 54, 52, 214, 92, 53, + 38, 54, 30, 55, 187, 157, 35, 56, 22, 50, 53, 166, 2, 54, 190, 90, 55, + 187, 157, 35, 56, 11, 42, 54, 210, 220, 28, 55, 159, 157, 7, 56, 4, 234, + 249, 35, 55, 3, 56, 56, 170, 1, 53, 50, 54, 210, 89, 52, 110, 55, 187, + 157, 35, 56, 118, 62, 52, 254, 89, 51, 98, 53, 38, 54, 30, 55, 187, 157, + 35, 56, 24, 46, 53, 50, 54, 190, 90, 55, 187, 157, 35, 56, 13, 206, 1, + 54, 186, 217, 28, 55, 159, 157, 7, 56, 7, 187, 90, 55, 61, 54, 52, 118, + 53, 230, 88, 54, 30, 55, 187, 157, 35, 56, 31, 46, 53, 170, 89, 54, 30, + 55, 187, 157, 35, 56, 15, 38, 54, 158, 89, 55, 187, 157, 35, 56, 7, 210, + 246, 35, 55, 3, 56, 14, 226, 88, 54, 30, 55, 187, 157, 35, 56, 31, 46, + 54, 234, 87, 53, 66, 55, 187, 157, 35, 56, 6, 166, 88, 55, 187, 157, 35, + 56, 120, 70, 49, 238, 1, 50, 190, 188, 25, 51, 38, 52, 30, 53, 147, 182, + 10, 54, 61, 58, 50, 126, 51, 150, 189, 25, 52, 30, 53, 147, 182, 10, 54, + 31, 50, 51, 222, 189, 25, 52, 30, 53, 147, 182, 10, 54, 15, 42, 52, 206, + 189, 25, 53, 147, 182, 10, 54, 7, 218, 243, 35, 53, 3, 54, 15, 146, 189, + 25, 52, 166, 153, 3, 53, 139, 157, 7, 54, 31, 50, 52, 138, 188, 25, 51, + 66, 53, 147, 182, 10, 54, 7, 199, 188, 25, 53, 6, 190, 153, 22, 32, 237, + 195, 8, 4, 66, 69, 82, 82, 232, 4, 200, 1, 3, 76, 68, 32, 78, 79, 104, 7, + 80, 79, 77, 79, 70, 79, 32, 188, 6, 2, 84, 84, 216, 6, 5, 85, 81, 85, 69, + 84, 54, 87, 198, 1, 88, 206, 215, 5, 77, 178, 168, 3, 89, 170, 167, 26, + 65, 139, 34, 78, 14, 254, 190, 8, 83, 142, 204, 7, 69, 190, 207, 17, 70, + 234, 2, 87, 215, 10, 71, 10, 26, 75, 215, 144, 23, 77, 9, 40, 4, 77, 65, + 82, 75, 175, 239, 35, 83, 5, 133, 171, 23, 3, 32, 84, 65, 150, 1, 96, 13, + 70, 73, 78, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 53, 7, 76, 69, 84, + 84, 69, 82, 32, 10, 162, 238, 35, 71, 2, 72, 2, 75, 2, 80, 3, 84, 140, 1, + 234, 1, 65, 54, 85, 22, 69, 82, 71, 46, 73, 70, 78, 38, 79, 98, 90, 170, + 154, 7, 75, 178, 174, 24, 67, 2, 76, 2, 83, 246, 76, 66, 206, 201, 1, 74, + 222, 137, 2, 68, 2, 70, 2, 72, 2, 77, 2, 80, 2, 81, 2, 82, 2, 84, 2, 86, + 3, 88, 21, 50, 73, 2, 85, 74, 78, 134, 235, 35, 72, 3, 77, 5, 235, 155, + 35, 78, 17, 50, 78, 134, 235, 35, 69, 2, 72, 2, 73, 3, 82, 7, 130, 235, + 35, 71, 3, 78, 11, 230, 234, 35, 72, 2, 78, 2, 85, 3, 87, 15, 180, 239, + 33, 2, 78, 78, 134, 251, 1, 72, 2, 77, 2, 82, 3, 85, 9, 222, 238, 33, 71, + 155, 251, 1, 78, 17, 66, 78, 142, 232, 34, 32, 134, 129, 1, 69, 2, 77, 2, + 79, 3, 85, 4, 142, 233, 35, 71, 3, 78, 9, 242, 232, 35, 72, 2, 73, 3, 89, + 66, 104, 3, 79, 77, 32, 133, 179, 21, 17, 76, 69, 32, 87, 73, 84, 72, 32, + 80, 79, 80, 80, 73, 78, 71, 32, 67, 64, 152, 2, 5, 72, 65, 76, 70, 32, + 232, 1, 5, 76, 69, 70, 84, 32, 88, 6, 82, 73, 71, 72, 84, 32, 88, 14, 83, + 81, 85, 65, 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 206, 213, 15, 65, + 226, 134, 16, 67, 210, 3, 80, 140, 3, 13, 74, 85, 83, 84, 73, 70, 73, 69, + 68, 32, 85, 80, 80, 135, 5, 84, 32, 148, 1, 16, 70, 79, 82, 87, 65, 82, + 68, 45, 70, 65, 67, 73, 78, 71, 32, 82, 130, 225, 31, 76, 22, 82, 252, 2, + 2, 83, 84, 174, 5, 66, 231, 243, 1, 73, 10, 252, 153, 29, 11, 85, 78, 78, + 69, 82, 32, 70, 82, 65, 77, 69, 223, 200, 2, 79, 8, 192, 230, 31, 13, 74, + 85, 83, 84, 73, 70, 73, 69, 68, 32, 85, 80, 80, 126, 67, 51, 72, 8, 230, + 230, 31, 67, 50, 72, 33, 13, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, 85, + 80, 80, 5, 225, 214, 23, 9, 32, 79, 86, 69, 82, 32, 84, 79, 80, 5, 149, + 211, 34, 8, 32, 79, 70, 32, 70, 76, 79, 87, 14, 58, 76, 116, 3, 84, 73, + 69, 141, 134, 33, 3, 32, 65, 78, 6, 26, 32, 247, 143, 35, 73, 4, 148, + 183, 9, 7, 79, 70, 32, 72, 89, 71, 73, 245, 130, 23, 6, 87, 73, 84, 72, + 32, 83, 7, 207, 195, 32, 32, 218, 2, 88, 10, 32, 68, 82, 65, 87, 73, 78, + 71, 83, 32, 193, 226, 34, 6, 73, 78, 71, 32, 71, 76, 216, 2, 176, 1, 2, + 68, 79, 228, 6, 6, 72, 69, 65, 86, 89, 32, 254, 2, 76, 204, 25, 6, 82, + 73, 71, 72, 84, 32, 144, 4, 3, 85, 80, 32, 245, 3, 9, 86, 69, 82, 84, 73, + 67, 65, 76, 32, 66, 52, 5, 85, 66, 76, 69, 32, 165, 3, 3, 87, 78, 32, 30, + 58, 68, 216, 2, 2, 85, 80, 234, 5, 86, 211, 163, 29, 72, 14, 64, 8, 73, + 65, 71, 79, 78, 65, 76, 32, 149, 2, 3, 79, 87, 78, 8, 104, 6, 85, 80, 80, + 69, 82, 32, 205, 14, 15, 76, 79, 87, 69, 82, 32, 76, 69, 70, 84, 32, 84, + 79, 32, 77, 6, 76, 8, 76, 69, 70, 84, 32, 84, 79, 32, 237, 24, 6, 82, 73, + 71, 72, 84, 32, 4, 204, 23, 14, 77, 73, 68, 68, 76, 69, 32, 67, 69, 78, + 84, 82, 69, 32, 159, 173, 23, 76, 6, 199, 26, 32, 36, 128, 1, 10, 72, 69, + 65, 86, 89, 32, 65, 78, 68, 32, 132, 1, 10, 76, 73, 71, 72, 84, 32, 65, + 78, 68, 32, 210, 38, 68, 131, 4, 83, 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, + 71, 72, 228, 35, 2, 85, 80, 151, 5, 72, 4, 17, 2, 84, 32, 4, 230, 32, 85, + 239, 140, 35, 76, 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 156, 36, + 2, 85, 80, 235, 4, 72, 4, 17, 2, 84, 32, 4, 198, 32, 85, 135, 9, 72, 46, + 136, 1, 4, 76, 69, 70, 84, 68, 2, 85, 80, 130, 1, 86, 172, 20, 2, 68, 79, + 170, 2, 81, 100, 2, 84, 82, 154, 140, 29, 72, 151, 134, 6, 82, 5, 45, 9, + 32, 65, 78, 68, 32, 76, 73, 71, 72, 2, 195, 132, 34, 84, 11, 29, 5, 32, + 65, 78, 68, 32, 8, 42, 76, 134, 164, 29, 72, 151, 134, 6, 82, 4, 224, + 174, 24, 4, 73, 71, 72, 84, 167, 251, 10, 69, 8, 209, 20, 7, 69, 82, 84, + 73, 67, 65, 76, 156, 1, 56, 4, 69, 70, 84, 32, 169, 2, 5, 73, 71, 72, 84, + 32, 16, 160, 27, 19, 68, 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, + 68, 32, 82, 73, 71, 72, 24, 14, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, + 82, 73, 71, 72, 100, 14, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 82, 73, + 71, 72, 97, 17, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, + 73, 71, 72, 140, 1, 248, 1, 4, 65, 82, 67, 32, 30, 68, 248, 15, 10, 72, + 79, 82, 73, 90, 79, 78, 84, 65, 76, 112, 4, 76, 69, 70, 84, 62, 81, 34, + 84, 172, 1, 2, 85, 80, 120, 8, 86, 69, 82, 84, 73, 67, 65, 76, 192, 197, + 13, 7, 66, 79, 84, 84, 79, 77, 32, 143, 203, 21, 82, 8, 230, 202, 17, 68, + 67, 85, 80, 52, 8, 73, 65, 71, 79, 78, 65, 76, 32, 199, 14, 79, 68, 168, + 1, 14, 76, 79, 87, 69, 82, 32, 76, 69, 70, 84, 32, 84, 79, 32, 96, 7, 77, + 73, 68, 68, 76, 69, 32, 136, 3, 6, 85, 80, 80, 69, 82, 32, 162, 136, 34, + 67, 183, 4, 68, 4, 38, 77, 33, 5, 85, 80, 80, 69, 82, 2, 29, 5, 73, 68, + 68, 76, 69, 2, 129, 12, 2, 32, 67, 16, 88, 8, 76, 69, 70, 84, 32, 84, 79, + 32, 213, 1, 9, 82, 73, 71, 72, 84, 32, 84, 79, 32, 10, 156, 1, 6, 76, 79, + 87, 69, 82, 32, 33, 28, 85, 80, 80, 69, 82, 32, 67, 69, 78, 84, 82, 69, + 32, 84, 79, 32, 77, 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, 84, 6, 246, + 3, 67, 255, 155, 35, 82, 5, 135, 191, 20, 32, 6, 232, 4, 15, 85, 80, 80, + 69, 82, 32, 67, 69, 78, 84, 82, 69, 32, 84, 79, 235, 3, 76, 44, 144, 1, + 10, 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 248, 3, 8, 76, 69, 70, 84, + 32, 84, 79, 32, 193, 2, 9, 82, 73, 71, 72, 84, 32, 84, 79, 32, 20, 52, 7, + 77, 73, 68, 68, 76, 69, 32, 171, 176, 23, 76, 16, 56, 4, 76, 69, 70, 84, + 165, 1, 5, 82, 73, 71, 72, 84, 9, 11, 32, 6, 84, 10, 84, 79, 32, 76, 79, + 87, 69, 82, 32, 67, 233, 186, 20, 5, 65, 78, 68, 32, 77, 4, 29, 5, 69, + 78, 84, 82, 69, 5, 133, 196, 32, 3, 32, 84, 79, 9, 11, 32, 6, 88, 3, 65, + 78, 68, 65, 15, 84, 79, 32, 76, 79, 87, 69, 82, 32, 67, 69, 78, 84, 82, + 69, 2, 241, 185, 20, 11, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 5, + 189, 211, 10, 9, 32, 84, 79, 32, 77, 73, 68, 68, 76, 14, 68, 6, 76, 79, + 87, 69, 82, 32, 97, 7, 77, 73, 68, 68, 76, 69, 32, 6, 48, 6, 67, 69, 78, + 84, 82, 69, 227, 152, 35, 82, 5, 11, 32, 2, 233, 4, 4, 84, 79, 32, 85, 8, + 76, 10, 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 33, 5, 82, 73, 71, 72, + 84, 4, 250, 3, 85, 139, 201, 13, 76, 5, 11, 32, 2, 193, 204, 13, 2, 84, + 79, 10, 46, 76, 69, 7, 77, 73, 68, 68, 76, 69, 32, 4, 29, 5, 79, 87, 69, + 82, 32, 4, 198, 191, 32, 67, 235, 214, 2, 76, 6, 34, 67, 37, 4, 76, 69, + 70, 84, 2, 45, 6, 69, 78, 84, 82, 69, 32, 5, 11, 32, 2, 133, 170, 23, 2, + 84, 79, 12, 36, 2, 87, 78, 253, 2, 2, 85, 66, 9, 11, 32, 6, 25, 4, 65, + 78, 68, 32, 6, 210, 142, 29, 72, 250, 133, 6, 76, 31, 82, 9, 11, 32, 6, + 40, 4, 65, 78, 68, 32, 227, 141, 35, 87, 4, 26, 85, 179, 168, 23, 76, 2, + 193, 168, 23, 2, 80, 80, 5, 149, 237, 33, 10, 32, 65, 78, 68, 32, 72, 69, + 65, 86, 89, 4, 109, 5, 85, 65, 68, 82, 85, 6, 66, 82, 225, 199, 13, 10, + 79, 80, 32, 65, 78, 68, 32, 85, 80, 80, 4, 11, 73, 4, 11, 80, 4, 41, 8, + 76, 69, 32, 68, 65, 83, 72, 32, 4, 190, 178, 16, 86, 151, 217, 12, 72, + 11, 29, 5, 32, 65, 78, 68, 32, 8, 34, 72, 230, 144, 35, 76, 31, 82, 4, + 200, 149, 24, 4, 69, 65, 86, 89, 175, 245, 4, 79, 17, 29, 5, 32, 65, 78, + 68, 32, 14, 162, 143, 28, 66, 42, 84, 206, 122, 72, 250, 133, 6, 76, 31, + 82, 16, 148, 2, 18, 68, 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, + 68, 32, 76, 69, 70, 24, 13, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 76, + 69, 70, 100, 13, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 76, 69, 70, 97, + 16, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 76, 69, 70, 2, + 101, 3, 84, 32, 85, 6, 17, 2, 84, 32, 6, 58, 85, 178, 3, 68, 245, 4, 6, + 86, 69, 82, 84, 73, 67, 2, 183, 158, 33, 80, 6, 17, 2, 84, 32, 6, 58, 85, + 134, 4, 68, 205, 4, 6, 86, 69, 82, 84, 73, 67, 2, 239, 8, 80, 2, 185, 2, + 3, 84, 32, 68, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, + 188, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 186, 2, 68, 131, 4, + 83, 12, 80, 4, 68, 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, + 255, 4, 72, 2, 145, 5, 2, 32, 72, 4, 17, 2, 84, 32, 4, 26, 68, 191, 137, + 35, 76, 2, 133, 155, 33, 3, 79, 87, 78, 12, 80, 4, 68, 79, 87, 78, 24, 3, + 76, 69, 70, 0, 4, 82, 73, 71, 72, 211, 4, 72, 2, 229, 4, 2, 32, 72, 4, + 17, 2, 84, 32, 4, 22, 68, 131, 5, 72, 2, 233, 4, 3, 79, 87, 78, 24, 130, + 1, 68, 188, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 144, 1, 10, + 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 183, 1, 83, 6, 49, 10, 79, 85, + 66, 76, 69, 32, 65, 78, 68, 32, 6, 92, 3, 76, 69, 70, 0, 4, 82, 73, 71, + 72, 13, 10, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 2, 11, 84, 2, 233, + 231, 26, 2, 32, 83, 6, 54, 72, 68, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, + 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 145, 150, 33, 2, 65, 76, 2, 247, + 149, 33, 84, 6, 54, 72, 60, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 37, + 7, 79, 82, 73, 90, 79, 78, 84, 2, 29, 2, 65, 76, 2, 11, 84, 2, 17, 2, 32, + 72, 2, 137, 156, 35, 3, 69, 65, 86, 6, 49, 10, 73, 78, 71, 76, 69, 32, + 65, 78, 68, 32, 6, 96, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 221, 171, 26, + 9, 72, 79, 82, 73, 90, 79, 78, 84, 65, 2, 231, 171, 26, 84, 134, 6, 46, + 65, 178, 14, 69, 150, 1, 73, 175, 1, 79, 232, 5, 36, 4, 72, 77, 73, 32, + 247, 9, 73, 230, 1, 192, 1, 7, 76, 69, 84, 84, 69, 82, 32, 196, 2, 7, 78, + 85, 77, 66, 69, 82, 32, 144, 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, + 79, 78, 32, 84, 5, 83, 73, 71, 78, 32, 122, 86, 239, 244, 24, 68, 108, + 210, 1, 79, 238, 205, 31, 65, 38, 68, 114, 84, 46, 86, 186, 5, 85, 206, + 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, + 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, 187, 2, 69, 15, 45, 9, + 76, 68, 32, 84, 65, 77, 73, 76, 32, 12, 158, 255, 28, 76, 206, 149, 2, + 83, 134, 92, 78, 175, 242, 2, 82, 42, 82, 69, 38, 70, 66, 78, 26, 83, + 138, 173, 21, 84, 238, 228, 5, 79, 203, 213, 7, 74, 4, 145, 135, 31, 4, + 73, 71, 72, 84, 8, 26, 79, 227, 147, 21, 73, 4, 142, 185, 33, 82, 159, + 180, 1, 85, 4, 65, 3, 73, 78, 69, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, + 5, 215, 147, 35, 84, 10, 46, 76, 202, 208, 12, 68, 177, 16, 2, 67, 82, 4, + 198, 190, 19, 79, 235, 204, 14, 73, 12, 246, 143, 31, 67, 152, 62, 9, 79, + 76, 68, 32, 84, 65, 77, 73, 76, 230, 157, 1, 74, 158, 2, 86, 122, 85, + 187, 240, 1, 65, 34, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 167, + 234, 32, 73, 32, 138, 1, 79, 208, 174, 29, 11, 66, 72, 65, 84, 84, 73, + 80, 82, 79, 76, 85, 214, 159, 2, 65, 38, 85, 22, 86, 186, 201, 1, 73, + 223, 137, 2, 69, 7, 253, 142, 31, 10, 76, 68, 32, 84, 65, 77, 73, 76, 32, + 83, 130, 4, 72, 12, 76, 76, 69, 32, 80, 65, 84, 84, 69, 82, 78, 32, 231, + 160, 35, 78, 128, 4, 44, 5, 68, 79, 84, 83, 45, 171, 244, 6, 66, 254, 3, + 74, 49, 74, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, + 129, 2, 66, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, + 129, 1, 58, 51, 54, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 65, 50, + 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 33, 42, 53, 38, 54, 30, 55, + 187, 157, 35, 56, 17, 34, 54, 30, 55, 187, 157, 35, 56, 9, 26, 55, 187, + 157, 35, 56, 5, 183, 157, 35, 56, 8, 26, 65, 183, 134, 35, 86, 6, 156, + 185, 20, 11, 75, 32, 80, 69, 82, 77, 73, 84, 84, 69, 68, 196, 171, 8, 6, + 83, 84, 45, 70, 69, 69, 167, 184, 6, 68, 10, 42, 68, 92, 2, 69, 70, 243, + 151, 35, 67, 4, 204, 161, 32, 6, 71, 69, 32, 65, 84, 32, 193, 70, 9, 69, + 32, 87, 73, 84, 72, 32, 86, 69, 4, 190, 245, 31, 67, 203, 165, 3, 83, 12, + 84, 4, 75, 69, 78, 32, 176, 192, 8, 2, 67, 67, 160, 210, 25, 2, 87, 78, + 231, 118, 79, 6, 248, 173, 27, 17, 67, 73, 82, 67, 76, 69, 32, 87, 73, + 84, 72, 32, 78, 79, 82, 84, 72, 218, 202, 6, 66, 147, 26, 72, 134, 1, + 208, 1, 4, 66, 66, 76, 69, 46, 71, 156, 4, 4, 72, 73, 68, 32, 36, 2, 76, + 76, 122, 83, 56, 4, 84, 84, 69, 82, 176, 249, 10, 10, 73, 76, 68, 73, 78, + 71, 32, 67, 79, 78, 184, 213, 16, 2, 82, 82, 247, 239, 6, 67, 4, 204, + 145, 28, 2, 32, 84, 243, 133, 7, 83, 63, 33, 6, 73, 78, 69, 83, 69, 32, + 60, 144, 1, 7, 76, 69, 84, 84, 69, 82, 32, 172, 2, 11, 86, 79, 87, 69, + 76, 32, 83, 73, 71, 78, 32, 190, 176, 16, 69, 225, 241, 13, 4, 80, 65, + 76, 76, 46, 154, 1, 77, 34, 78, 234, 145, 35, 66, 2, 67, 2, 68, 2, 71, 2, + 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, + 2, 65, 4, 134, 146, 35, 80, 187, 2, 65, 12, 46, 71, 34, 89, 154, 145, 35, + 82, 187, 2, 65, 4, 182, 145, 35, 75, 187, 2, 65, 4, 150, 145, 35, 67, + 187, 2, 65, 10, 218, 252, 34, 65, 214, 22, 69, 2, 73, 2, 79, 3, 85, 40, + 142, 152, 10, 76, 135, 241, 18, 86, 10, 56, 2, 69, 84, 20, 4, 72, 79, 82, + 78, 231, 162, 9, 83, 5, 179, 210, 31, 32, 5, 241, 184, 27, 5, 32, 87, 73, + 84, 72, 9, 26, 84, 143, 192, 32, 32, 4, 234, 184, 27, 83, 15, 32, 5, 243, + 237, 34, 70, 240, 3, 140, 1, 23, 90, 65, 78, 84, 73, 78, 69, 32, 77, 85, + 83, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 245, 179, 20, 5, 84, + 69, 32, 79, 82, 238, 3, 154, 2, 65, 128, 7, 2, 67, 72, 170, 1, 68, 158, + 5, 69, 206, 2, 70, 166, 8, 71, 202, 3, 73, 162, 2, 75, 142, 5, 76, 190, + 2, 77, 250, 4, 79, 114, 80, 174, 4, 82, 42, 83, 194, 5, 84, 196, 5, 2, + 86, 65, 250, 1, 89, 178, 160, 32, 78, 245, 142, 2, 9, 88, 73, 82, 79, 78, + 32, 75, 76, 65, 62, 60, 5, 71, 79, 71, 73, 32, 222, 1, 78, 118, 80, 199, + 2, 82, 16, 70, 65, 0, 2, 71, 79, 64, 2, 77, 69, 37, 5, 80, 79, 76, 73, + 32, 4, 17, 2, 82, 71, 4, 160, 227, 15, 2, 79, 84, 159, 169, 19, 73, 4, + 214, 151, 9, 84, 231, 224, 25, 83, 4, 26, 65, 1, 2, 71, 79, 2, 131, 203, + 17, 82, 6, 72, 6, 84, 73, 75, 69, 78, 79, 205, 233, 34, 6, 65, 84, 82, + 73, 67, 72, 4, 168, 32, 2, 75, 89, 147, 232, 34, 77, 22, 52, 5, 69, 83, + 79, 32, 69, 38, 79, 155, 246, 34, 76, 4, 204, 1, 2, 88, 79, 147, 55, 75, + 16, 80, 6, 83, 84, 82, 79, 70, 79, 188, 40, 3, 68, 69, 82, 181, 128, 1, + 2, 84, 72, 10, 24, 2, 73, 32, 79, 83, 4, 56, 9, 83, 89, 78, 68, 69, 83, + 77, 79, 83, 195, 24, 84, 2, 143, 44, 32, 7, 11, 32, 4, 214, 60, 68, 255, + 145, 22, 78, 18, 48, 2, 71, 79, 33, 6, 75, 84, 73, 75, 79, 32, 4, 170, + 21, 83, 171, 242, 34, 78, 14, 182, 190, 27, 86, 138, 170, 7, 90, 162, 8, + 75, 254, 2, 68, 2, 78, 162, 17, 71, 3, 80, 14, 80, 4, 82, 79, 65, 32, + 168, 37, 4, 79, 82, 69, 86, 221, 2, 4, 65, 77, 73, 76, 6, 228, 214, 19, + 2, 83, 80, 220, 191, 3, 3, 90, 89, 71, 185, 188, 10, 3, 75, 76, 73, 42, + 50, 73, 172, 219, 8, 2, 65, 83, 227, 138, 26, 89, 38, 122, 65, 200, 1, 5, + 69, 83, 73, 83, 32, 70, 71, 208, 1, 3, 80, 76, 73, 173, 183, 27, 8, 70, + 84, 79, 71, 71, 79, 83, 32, 10, 48, 6, 83, 84, 79, 76, 73, 32, 179, 203, + 33, 82, 8, 80, 6, 65, 80, 76, 73, 32, 77, 174, 55, 68, 141, 220, 22, 5, + 84, 72, 69, 83, 69, 4, 40, 2, 69, 71, 133, 165, 28, 2, 73, 75, 2, 195, + 135, 30, 65, 12, 38, 84, 246, 50, 65, 42, 68, 63, 77, 6, 194, 10, 69, + 239, 41, 82, 10, 72, 5, 79, 82, 71, 79, 78, 217, 128, 35, 7, 82, 65, 77, + 77, 65, 32, 71, 9, 69, 15, 32, 80, 65, 82, 69, 83, 84, 73, 71, 77, 69, + 78, 79, 78, 32, 6, 222, 38, 68, 137, 10, 8, 65, 82, 73, 83, 84, 69, 82, + 65, 5, 175, 47, 32, 16, 166, 1, 78, 116, 6, 84, 69, 82, 79, 78, 32, 188, + 44, 4, 88, 79, 32, 69, 156, 238, 32, 4, 80, 69, 71, 69, 152, 48, 6, 75, + 83, 84, 82, 69, 80, 237, 13, 3, 76, 65, 70, 4, 224, 22, 3, 68, 79, 70, + 145, 159, 27, 18, 65, 82, 88, 73, 83, 32, 75, 65, 73, 32, 70, 84, 72, 79, + 82, 65, 32, 86, 4, 208, 11, 5, 65, 82, 71, 79, 83, 151, 20, 80, 44, 180, + 1, 9, 65, 78, 69, 82, 79, 83, 73, 83, 32, 52, 6, 84, 72, 79, 82, 65, 32, + 165, 6, 22, 72, 84, 79, 82, 65, 32, 83, 75, 76, 73, 82, 79, 78, 32, 67, + 72, 82, 79, 77, 65, 32, 86, 6, 246, 4, 68, 14, 77, 25, 5, 84, 69, 84, 82, + 65, 36, 220, 2, 8, 65, 82, 67, 72, 65, 73, 79, 78, 80, 10, 68, 73, 65, + 84, 79, 78, 73, 75, 73, 32, 96, 11, 73, 32, 89, 70, 69, 83, 73, 83, 32, + 84, 69, 32, 15, 77, 65, 76, 65, 75, 79, 78, 32, 67, 72, 82, 79, 77, 65, + 32, 74, 78, 40, 8, 83, 75, 76, 73, 82, 79, 78, 32, 141, 209, 18, 18, 69, + 78, 65, 82, 77, 79, 78, 73, 79, 83, 32, 65, 78, 84, 73, 70, 79, 78, 5, + 57, 12, 32, 68, 69, 89, 84, 69, 82, 79, 85, 32, 73, 67, 2, 199, 175, 27, + 72, 14, 62, 78, 254, 216, 34, 90, 162, 8, 75, 254, 2, 68, 163, 17, 80, 6, + 242, 39, 73, 251, 147, 33, 65, 2, 237, 42, 4, 84, 65, 82, 84, 4, 18, 68, + 15, 77, 2, 35, 73, 2, 21, 3, 79, 78, 79, 2, 151, 19, 70, 4, 166, 19, 65, + 213, 219, 32, 2, 69, 78, 6, 84, 7, 67, 72, 82, 79, 77, 65, 32, 193, 149, + 17, 8, 68, 73, 65, 84, 79, 78, 79, 78, 4, 42, 86, 169, 153, 17, 4, 83, + 89, 78, 65, 2, 251, 209, 32, 65, 22, 80, 6, 69, 78, 73, 75, 73, 32, 44, + 2, 79, 82, 185, 26, 5, 82, 79, 78, 84, 72, 4, 192, 242, 30, 2, 68, 73, 1, + 2, 89, 70, 16, 68, 2, 71, 79, 225, 1, 10, 84, 72, 77, 73, 75, 79, 78, 32, + 78, 32, 12, 28, 2, 78, 32, 155, 1, 83, 10, 96, 14, 80, 65, 82, 69, 83, + 84, 73, 71, 77, 69, 78, 79, 78, 32, 242, 33, 65, 113, 3, 78, 69, 79, 4, + 214, 24, 68, 249, 202, 32, 5, 65, 82, 73, 83, 84, 2, 153, 191, 33, 5, 89, + 78, 84, 72, 69, 4, 142, 28, 65, 1, 2, 68, 73, 16, 56, 2, 77, 73, 114, 83, + 225, 240, 33, 4, 67, 72, 65, 68, 8, 34, 70, 169, 28, 3, 68, 73, 65, 6, + 40, 4, 84, 72, 79, 82, 195, 230, 33, 79, 4, 242, 160, 34, 79, 227, 79, + 65, 6, 44, 6, 65, 75, 73, 65, 32, 84, 231, 13, 79, 2, 153, 201, 21, 12, + 69, 76, 79, 85, 83, 32, 73, 67, 72, 73, 77, 65, 54, 108, 2, 65, 84, 96, + 6, 69, 78, 84, 73, 77, 65, 140, 1, 5, 76, 65, 83, 77, 65, 18, 79, 98, 82, + 175, 1, 89, 6, 40, 3, 65, 86, 65, 201, 3, 2, 72, 73, 4, 140, 29, 5, 32, + 84, 82, 79, 77, 135, 176, 34, 83, 18, 24, 2, 84, 65, 15, 32, 11, 11, 32, + 8, 36, 4, 78, 69, 79, 32, 183, 28, 65, 6, 252, 140, 28, 2, 77, 69, 254, + 212, 2, 75, 151, 226, 3, 65, 7, 243, 28, 32, 8, 64, 6, 78, 84, 69, 86, + 77, 65, 202, 212, 8, 82, 151, 246, 25, 85, 5, 181, 166, 14, 2, 32, 65, + 14, 44, 4, 65, 84, 73, 77, 105, 3, 69, 77, 65, 12, 18, 65, 35, 79, 8, + 182, 23, 32, 195, 209, 34, 84, 4, 134, 12, 75, 181, 184, 31, 5, 89, 80, + 79, 82, 82, 2, 227, 140, 27, 83, 2, 159, 201, 34, 76, 14, 22, 69, 147, + 22, 89, 12, 44, 5, 73, 77, 77, 65, 32, 191, 251, 29, 77, 10, 72, 2, 69, + 78, 0, 5, 73, 77, 73, 83, 69, 54, 84, 69, 3, 68, 89, 79, 2, 161, 160, 27, + 8, 79, 83, 32, 67, 72, 82, 79, 78, 4, 44, 5, 69, 83, 83, 65, 82, 1, 2, + 82, 73, 2, 17, 2, 79, 78, 2, 25, 4, 32, 67, 72, 82, 2, 247, 221, 33, 79, + 28, 84, 8, 65, 82, 84, 89, 82, 73, 65, 32, 229, 231, 30, 7, 73, 75, 82, + 79, 78, 32, 73, 26, 72, 5, 65, 76, 76, 73, 32, 38, 68, 38, 80, 134, 1, + 86, 30, 84, 87, 76, 4, 34, 68, 177, 2, 3, 80, 82, 79, 2, 237, 2, 5, 69, + 89, 84, 69, 82, 8, 60, 7, 76, 65, 71, 73, 79, 83, 32, 45, 4, 82, 79, 84, + 79, 4, 200, 1, 5, 84, 69, 84, 65, 82, 111, 73, 4, 22, 86, 227, 1, 83, 2, + 209, 1, 3, 65, 82, 89, 8, 56, 8, 69, 84, 65, 82, 84, 79, 83, 32, 61, 2, + 82, 73, 4, 22, 76, 135, 1, 73, 2, 21, 3, 69, 71, 69, 2, 63, 84, 4, 18, + 70, 35, 84, 2, 209, 197, 21, 3, 79, 78, 73, 2, 11, 79, 2, 11, 83, 2, 17, + 2, 32, 73, 2, 155, 239, 22, 67, 18, 92, 4, 76, 73, 71, 79, 180, 1, 5, 89, + 82, 65, 78, 73, 210, 14, 88, 165, 130, 34, 2, 77, 65, 4, 211, 1, 78, 42, 60, 2, 65, 82, 100, 2, 73, 65, 90, 69, 169, 1, 2, 83, 73, 12, 40, 2, 65, - 75, 201, 246, 16, 2, 73, 67, 10, 52, 3, 65, 76, 69, 45, 6, 76, 73, 84, + 75, 181, 170, 17, 2, 73, 67, 10, 52, 3, 65, 76, 69, 45, 6, 76, 73, 84, 73, 75, 73, 4, 11, 83, 4, 17, 2, 77, 65, 4, 23, 32, 7, 11, 32, 4, 198, - 15, 65, 151, 219, 21, 78, 12, 72, 3, 84, 65, 83, 136, 3, 6, 76, 65, 83, - 84, 79, 78, 239, 202, 8, 82, 6, 26, 84, 211, 216, 33, 77, 4, 32, 2, 79, - 75, 231, 218, 33, 73, 2, 173, 185, 33, 2, 79, 85, 14, 36, 5, 70, 73, 83, + 15, 65, 155, 151, 22, 78, 12, 72, 3, 84, 65, 83, 136, 3, 6, 76, 65, 83, + 84, 79, 78, 191, 216, 8, 82, 6, 26, 84, 247, 220, 34, 77, 4, 32, 2, 79, + 75, 139, 223, 34, 73, 2, 209, 189, 34, 2, 79, 85, 14, 36, 5, 70, 73, 83, 84, 79, 67, 76, 10, 46, 80, 214, 1, 78, 170, 8, 76, 187, 1, 83, 2, 135, - 11, 65, 4, 138, 138, 33, 79, 215, 79, 73, 4, 178, 5, 69, 229, 178, 33, 2, + 11, 65, 4, 162, 142, 34, 79, 227, 79, 73, 4, 178, 5, 69, 137, 183, 34, 2, 65, 80, 42, 120, 5, 69, 73, 83, 77, 65, 32, 8, 73, 77, 65, 78, 83, 73, - 83, 32, 182, 1, 84, 154, 1, 89, 249, 240, 1, 3, 65, 88, 73, 5, 11, 32, 2, - 219, 230, 21, 78, 16, 36, 2, 65, 82, 1, 3, 84, 72, 69, 8, 25, 4, 83, 69, + 83, 32, 182, 1, 84, 154, 1, 89, 245, 240, 1, 3, 65, 88, 73, 5, 11, 32, 2, + 223, 162, 22, 78, 16, 36, 2, 65, 82, 1, 3, 84, 72, 69, 8, 25, 4, 83, 69, 79, 83, 9, 11, 32, 6, 18, 84, 39, 68, 4, 34, 82, 13, 4, 69, 84, 82, 65, - 2, 11, 73, 2, 153, 198, 26, 3, 83, 73, 77, 8, 68, 5, 65, 86, 82, 79, 83, - 52, 4, 82, 65, 71, 71, 175, 147, 16, 73, 5, 29, 5, 32, 65, 80, 79, 68, 2, - 175, 206, 8, 69, 2, 145, 241, 1, 2, 73, 83, 12, 34, 78, 149, 1, 3, 82, - 77, 65, 8, 36, 5, 65, 71, 77, 65, 32, 91, 69, 6, 154, 8, 65, 150, 219, - 21, 78, 233, 224, 4, 10, 77, 69, 84, 65, 32, 83, 84, 65, 86, 82, 2, 251, - 178, 33, 86, 5, 11, 84, 2, 255, 193, 16, 73, 40, 42, 69, 70, 72, 218, 1, - 82, 227, 2, 73, 6, 136, 12, 3, 84, 82, 65, 162, 148, 8, 76, 137, 167, 23, - 2, 83, 83, 12, 26, 69, 131, 174, 33, 73, 10, 64, 2, 77, 65, 133, 190, 32, + 2, 11, 73, 2, 141, 146, 27, 3, 83, 73, 77, 8, 68, 5, 65, 86, 82, 79, 83, + 52, 4, 82, 65, 71, 71, 179, 199, 16, 73, 5, 29, 5, 32, 65, 80, 79, 68, 2, + 255, 219, 8, 69, 2, 141, 241, 1, 2, 73, 83, 12, 34, 78, 149, 1, 3, 82, + 77, 65, 8, 36, 5, 65, 71, 77, 65, 32, 91, 69, 6, 154, 8, 65, 154, 151, + 22, 78, 217, 240, 4, 10, 77, 69, 84, 65, 32, 83, 84, 65, 86, 82, 2, 159, + 183, 34, 86, 5, 11, 84, 2, 243, 245, 16, 73, 40, 42, 69, 70, 72, 218, 1, + 82, 227, 2, 73, 6, 136, 12, 3, 84, 82, 65, 242, 161, 8, 76, 201, 154, 24, + 2, 83, 83, 12, 26, 69, 167, 178, 34, 73, 10, 64, 2, 77, 65, 145, 194, 33, 8, 83, 32, 75, 65, 73, 32, 65, 80, 9, 56, 2, 32, 65, 33, 8, 84, 73, 83, - 77, 79, 83, 32, 69, 2, 177, 209, 32, 3, 80, 76, 79, 4, 182, 178, 33, 83, - 3, 88, 20, 38, 73, 73, 5, 79, 77, 73, 75, 79, 6, 48, 2, 71, 79, 182, 129, - 29, 80, 131, 207, 4, 65, 2, 191, 153, 32, 82, 14, 42, 76, 32, 2, 78, 32, - 62, 80, 95, 83, 2, 11, 89, 2, 191, 174, 33, 71, 6, 26, 65, 135, 222, 21, - 78, 4, 250, 2, 82, 239, 169, 33, 76, 4, 46, 65, 141, 157, 32, 5, 83, 73, - 70, 73, 83, 2, 201, 173, 33, 6, 82, 65, 75, 65, 76, 69, 2, 11, 89, 2, - 141, 140, 16, 2, 78, 65, 10, 26, 82, 183, 172, 33, 84, 8, 21, 3, 69, 73, - 65, 8, 26, 32, 113, 2, 73, 32, 6, 38, 69, 242, 5, 68, 251, 213, 21, 78, - 2, 11, 75, 2, 29, 5, 70, 79, 78, 73, 84, 2, 141, 253, 32, 2, 73, 75, 2, - 11, 65, 2, 11, 82, 2, 129, 252, 32, 3, 67, 72, 65, 22, 28, 2, 70, 69, - 231, 3, 80, 14, 34, 78, 49, 4, 83, 73, 83, 32, 4, 11, 32, 4, 210, 234, - 29, 75, 235, 183, 3, 65, 10, 42, 65, 42, 68, 62, 77, 89, 2, 84, 82, 2, + 77, 79, 83, 32, 69, 2, 189, 213, 33, 3, 80, 76, 79, 4, 218, 182, 34, 83, + 3, 88, 20, 38, 73, 73, 5, 79, 77, 73, 75, 79, 6, 48, 2, 71, 79, 206, 217, + 29, 80, 143, 251, 4, 65, 2, 183, 156, 33, 82, 14, 42, 76, 32, 2, 78, 32, + 62, 80, 95, 83, 2, 11, 89, 2, 227, 178, 34, 71, 6, 26, 65, 139, 154, 22, + 78, 4, 250, 2, 82, 147, 174, 34, 76, 4, 46, 65, 129, 160, 33, 5, 83, 73, + 70, 73, 83, 2, 237, 177, 34, 6, 82, 65, 75, 65, 76, 69, 2, 11, 89, 2, + 145, 192, 16, 2, 78, 65, 10, 26, 82, 219, 176, 34, 84, 8, 21, 3, 69, 73, + 65, 8, 26, 32, 113, 2, 73, 32, 6, 38, 69, 242, 5, 68, 255, 145, 22, 78, + 2, 11, 75, 2, 29, 5, 70, 79, 78, 73, 84, 2, 165, 129, 34, 2, 73, 75, 2, + 11, 65, 2, 11, 82, 2, 153, 128, 34, 3, 67, 72, 65, 22, 28, 2, 70, 69, + 231, 3, 80, 14, 34, 78, 49, 4, 83, 73, 83, 32, 4, 11, 32, 4, 202, 196, + 30, 75, 151, 226, 3, 65, 10, 42, 65, 42, 68, 62, 77, 89, 2, 84, 82, 2, 133, 2, 6, 80, 76, 73, 32, 68, 89, 2, 233, 1, 11, 73, 71, 82, 65, 77, 77, 79, 83, 32, 69, 88, 2, 173, 1, 18, 79, 78, 79, 71, 82, 65, 77, 77, 79, 83, 32, 84, 69, 83, 83, 69, 82, 65, 4, 11, 73, 4, 60, 11, 71, 82, 65, 77, - 77, 79, 83, 32, 79, 75, 84, 59, 84, 2, 11, 79, 2, 189, 174, 2, 6, 32, 68, - 79, 68, 69, 75, 2, 129, 248, 32, 4, 73, 77, 79, 82, 8, 26, 79, 243, 248, - 28, 83, 6, 56, 6, 75, 82, 73, 83, 73, 83, 157, 133, 29, 2, 82, 82, 5, 17, - 2, 32, 68, 2, 11, 73, 2, 159, 248, 28, 80, 162, 82, 182, 1, 65, 134, 72, - 69, 230, 1, 72, 154, 31, 73, 240, 41, 3, 74, 75, 32, 238, 25, 76, 186, - 18, 79, 194, 113, 82, 162, 8, 85, 130, 128, 2, 89, 166, 221, 18, 71, 226, - 155, 10, 83, 203, 18, 67, 198, 13, 110, 68, 66, 76, 50, 77, 90, 78, 174, - 51, 80, 62, 82, 198, 7, 84, 138, 1, 85, 198, 240, 17, 67, 139, 181, 6, - 83, 4, 34, 85, 221, 189, 26, 2, 65, 32, 2, 201, 193, 31, 2, 67, 69, 4, - 144, 189, 9, 3, 76, 32, 77, 131, 130, 19, 69, 6, 36, 3, 69, 82, 65, 207, - 242, 32, 80, 5, 217, 226, 26, 7, 32, 87, 73, 84, 72, 32, 70, 193, 11, + 77, 79, 83, 32, 79, 75, 84, 59, 84, 2, 11, 79, 2, 181, 174, 2, 6, 32, 68, + 79, 68, 69, 75, 2, 153, 252, 33, 4, 73, 77, 79, 82, 8, 26, 79, 139, 209, + 29, 83, 6, 56, 6, 75, 82, 73, 83, 73, 83, 181, 221, 29, 2, 82, 82, 5, 17, + 2, 32, 68, 2, 11, 73, 2, 183, 208, 29, 80, 172, 82, 182, 1, 65, 134, 72, + 69, 230, 1, 72, 154, 31, 73, 208, 41, 3, 74, 75, 32, 138, 26, 76, 186, + 18, 79, 194, 113, 82, 234, 7, 85, 150, 128, 2, 89, 242, 164, 19, 71, 226, + 216, 10, 83, 203, 18, 67, 198, 13, 110, 68, 66, 76, 50, 77, 90, 78, 174, + 51, 80, 62, 82, 198, 7, 84, 138, 1, 85, 242, 164, 18, 67, 187, 203, 6, + 83, 4, 34, 85, 137, 139, 27, 2, 65, 32, 2, 221, 194, 32, 2, 67, 69, 4, + 152, 207, 9, 3, 76, 32, 77, 247, 200, 19, 69, 6, 36, 3, 69, 82, 65, 231, + 246, 33, 80, 5, 165, 177, 27, 7, 32, 87, 73, 84, 72, 32, 70, 193, 11, 148, 1, 16, 65, 68, 73, 65, 78, 32, 83, 89, 76, 76, 65, 66, 73, 67, 83, - 32, 148, 49, 2, 67, 69, 94, 68, 172, 152, 23, 3, 78, 69, 68, 139, 225, 9, - 79, 172, 11, 226, 1, 65, 190, 1, 66, 162, 2, 67, 242, 8, 69, 50, 70, 198, - 4, 72, 38, 73, 30, 75, 134, 1, 76, 82, 77, 174, 1, 78, 202, 4, 81, 178, - 1, 79, 194, 1, 80, 70, 82, 142, 1, 83, 194, 4, 84, 246, 3, 87, 254, 5, - 89, 223, 152, 17, 71, 21, 90, 65, 30, 73, 40, 10, 84, 72, 65, 80, 65, 83, - 67, 65, 78, 32, 242, 190, 33, 78, 3, 89, 7, 178, 191, 33, 73, 3, 89, 5, - 133, 150, 23, 5, 86, 73, 76, 73, 75, 4, 238, 190, 33, 77, 3, 83, 42, 156, - 1, 9, 76, 65, 67, 75, 70, 79, 79, 84, 32, 172, 226, 18, 9, 73, 66, 76, - 69, 45, 67, 82, 69, 69, 253, 180, 7, 10, 69, 65, 86, 69, 82, 32, 68, 69, - 78, 69, 36, 82, 87, 158, 208, 11, 75, 2, 78, 198, 236, 21, 65, 2, 69, 2, - 73, 2, 79, 3, 83, 11, 222, 188, 33, 65, 2, 69, 2, 73, 3, 79, 141, 3, 82, - 65, 186, 42, 87, 206, 155, 8, 72, 218, 182, 22, 79, 182, 56, 73, 207, - 134, 2, 69, 241, 2, 48, 6, 82, 82, 73, 69, 82, 32, 219, 180, 31, 65, 234, + 32, 148, 49, 2, 67, 69, 94, 68, 216, 227, 23, 3, 78, 69, 68, 131, 154, + 10, 79, 172, 11, 226, 1, 65, 190, 1, 66, 162, 2, 67, 242, 8, 69, 50, 70, + 198, 4, 72, 38, 73, 30, 75, 134, 1, 76, 82, 77, 174, 1, 78, 202, 4, 81, + 178, 1, 79, 194, 1, 80, 70, 82, 142, 1, 83, 194, 4, 84, 246, 3, 87, 254, + 5, 89, 247, 204, 17, 71, 21, 90, 65, 30, 73, 40, 10, 84, 72, 65, 80, 65, + 83, 67, 65, 78, 32, 150, 195, 34, 78, 3, 89, 7, 214, 195, 34, 73, 3, 89, + 5, 169, 221, 23, 5, 86, 73, 76, 73, 75, 4, 146, 195, 34, 77, 3, 83, 42, + 156, 1, 9, 76, 65, 67, 75, 70, 79, 79, 84, 32, 172, 151, 19, 9, 73, 66, + 76, 69, 45, 67, 82, 69, 69, 221, 202, 7, 10, 69, 65, 86, 69, 82, 32, 68, + 69, 78, 69, 36, 82, 87, 230, 228, 11, 75, 2, 78, 162, 220, 22, 65, 2, 69, + 2, 73, 2, 79, 3, 83, 11, 130, 193, 34, 65, 2, 69, 2, 73, 3, 79, 141, 3, + 82, 65, 186, 42, 87, 142, 169, 8, 72, 222, 166, 23, 79, 134, 60, 73, 223, + 137, 2, 69, 241, 2, 48, 6, 82, 82, 73, 69, 82, 32, 239, 181, 32, 65, 234, 2, 170, 1, 68, 122, 71, 94, 72, 46, 73, 46, 74, 82, 75, 30, 78, 66, 83, 66, 80, 2, 90, 58, 84, 38, 76, 132, 1, 2, 67, 72, 2, 77, 2, 82, 2, 87, 2, - 89, 171, 157, 33, 69, 32, 46, 69, 202, 5, 76, 2, 90, 255, 179, 33, 73, 6, - 26, 78, 171, 185, 33, 69, 4, 166, 210, 16, 69, 181, 191, 6, 2, 84, 65, - 30, 254, 4, 72, 174, 234, 20, 87, 166, 136, 7, 65, 154, 129, 5, 69, 150, - 64, 73, 2, 79, 3, 85, 19, 162, 4, 87, 170, 157, 33, 69, 215, 22, 73, 5, - 209, 223, 13, 6, 78, 73, 84, 73, 65, 76, 26, 202, 3, 74, 234, 243, 32, - 69, 222, 61, 87, 186, 2, 65, 2, 73, 2, 79, 3, 85, 26, 154, 1, 75, 227, 1, - 72, 14, 198, 246, 32, 69, 150, 64, 65, 2, 71, 2, 73, 2, 79, 3, 85, 26, - 62, 72, 202, 245, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 15, 198, 245, - 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 72, 34, 76, 70, 84, 66, 72, 3, - 83, 24, 130, 1, 72, 234, 243, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, - 24, 62, 83, 234, 243, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 12, 230, - 243, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 7, 236, 29, 4, 65, 83, 84, - 69, 215, 149, 33, 78, 51, 74, 65, 22, 73, 190, 231, 30, 85, 250, 11, 79, - 186, 79, 87, 203, 239, 1, 69, 7, 251, 171, 31, 65, 33, 40, 4, 78, 65, 76, - 32, 139, 178, 33, 73, 28, 160, 1, 2, 68, 79, 134, 1, 82, 78, 83, 150, - 221, 13, 71, 186, 2, 65, 132, 165, 3, 6, 66, 79, 84, 84, 79, 77, 0, 3, - 84, 79, 80, 202, 205, 12, 80, 211, 203, 2, 77, 6, 44, 5, 85, 66, 76, 69, - 32, 195, 212, 23, 87, 4, 232, 193, 6, 12, 83, 72, 79, 82, 84, 32, 86, 69, - 82, 84, 73, 67, 131, 159, 7, 65, 6, 38, 73, 253, 229, 31, 3, 65, 73, 83, - 4, 218, 132, 17, 71, 139, 170, 16, 78, 4, 184, 199, 24, 4, 77, 65, 76, - 76, 129, 202, 6, 4, 72, 79, 82, 84, 4, 198, 229, 25, 89, 139, 201, 7, 75, - 7, 170, 174, 33, 73, 3, 78, 39, 66, 87, 234, 27, 65, 230, 210, 30, 79, - 182, 56, 73, 207, 134, 2, 69, 19, 182, 250, 11, 65, 150, 244, 18, 79, - 182, 56, 73, 207, 134, 2, 69, 49, 142, 7, 72, 154, 20, 65, 66, 87, 166, - 210, 30, 79, 182, 56, 73, 207, 134, 2, 69, 43, 70, 69, 38, 79, 238, 25, - 65, 66, 87, 218, 138, 31, 73, 207, 134, 2, 72, 7, 229, 133, 26, 4, 68, - 73, 65, 76, 7, 11, 79, 5, 161, 233, 31, 8, 83, 69, 45, 67, 82, 69, 69, + 89, 207, 161, 34, 69, 32, 46, 69, 202, 5, 76, 2, 90, 163, 184, 34, 73, 6, + 26, 78, 207, 189, 34, 69, 4, 146, 134, 17, 69, 237, 210, 6, 2, 84, 65, + 30, 254, 4, 72, 182, 166, 21, 87, 234, 165, 7, 65, 230, 171, 5, 69, 162, + 64, 73, 2, 79, 3, 85, 19, 162, 4, 87, 206, 161, 34, 69, 215, 22, 73, 5, + 225, 135, 14, 6, 78, 73, 84, 73, 65, 76, 26, 202, 3, 74, 130, 248, 33, + 69, 234, 61, 87, 186, 2, 65, 2, 73, 2, 79, 3, 85, 26, 154, 1, 75, 227, 1, + 72, 14, 222, 250, 33, 69, 162, 64, 65, 2, 71, 2, 73, 2, 79, 3, 85, 26, + 62, 72, 226, 249, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 15, 222, 249, + 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 72, 34, 76, 70, 84, 66, 72, 3, + 83, 24, 130, 1, 72, 130, 248, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, + 24, 62, 83, 130, 248, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 12, 254, + 247, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 7, 236, 29, 4, 65, 83, 84, + 69, 251, 153, 34, 78, 51, 74, 65, 22, 73, 130, 229, 31, 85, 250, 11, 79, + 150, 83, 87, 207, 242, 1, 69, 7, 143, 173, 32, 65, 33, 40, 4, 78, 65, 76, + 32, 175, 182, 34, 73, 28, 160, 1, 2, 68, 79, 134, 1, 82, 78, 83, 166, + 133, 14, 71, 186, 2, 65, 128, 177, 3, 6, 66, 79, 84, 84, 79, 77, 0, 3, + 84, 79, 80, 166, 244, 12, 80, 247, 244, 2, 77, 6, 44, 5, 85, 66, 76, 69, + 32, 131, 160, 24, 87, 4, 252, 198, 6, 12, 83, 72, 79, 82, 84, 32, 86, 69, + 82, 84, 73, 67, 255, 193, 7, 65, 6, 38, 73, 213, 231, 32, 3, 65, 73, 83, + 4, 230, 184, 17, 71, 163, 250, 16, 78, 4, 152, 146, 25, 4, 77, 65, 76, + 76, 213, 252, 6, 4, 72, 79, 82, 84, 4, 150, 176, 26, 89, 223, 130, 8, 75, + 7, 206, 178, 34, 73, 3, 78, 39, 66, 87, 234, 27, 65, 170, 208, 31, 79, + 134, 60, 73, 223, 137, 2, 69, 19, 138, 143, 12, 65, 134, 221, 19, 79, + 134, 60, 73, 223, 137, 2, 69, 49, 142, 7, 72, 154, 20, 65, 66, 87, 234, + 207, 31, 79, 134, 60, 73, 223, 137, 2, 69, 43, 70, 69, 38, 79, 238, 25, + 65, 66, 87, 238, 139, 32, 73, 223, 137, 2, 72, 7, 197, 208, 26, 4, 68, + 73, 65, 76, 7, 11, 79, 5, 249, 234, 32, 8, 83, 69, 45, 67, 82, 69, 69, 32, 149, 1, 164, 1, 8, 45, 67, 82, 69, 69, 32, 84, 72, 38, 65, 250, 2, - 71, 76, 2, 78, 71, 48, 4, 85, 78, 65, 86, 142, 20, 79, 30, 87, 218, 138, - 31, 73, 206, 134, 2, 69, 3, 72, 6, 150, 163, 31, 73, 207, 134, 2, 69, 63, + 71, 76, 2, 78, 71, 48, 4, 85, 78, 65, 86, 142, 20, 79, 30, 87, 238, 139, + 32, 73, 222, 137, 2, 69, 3, 72, 6, 170, 164, 32, 73, 223, 137, 2, 69, 63, 104, 6, 83, 75, 65, 80, 73, 32, 188, 1, 7, 84, 84, 73, 76, 73, 75, 32, - 206, 160, 31, 65, 207, 134, 2, 89, 30, 70, 83, 86, 87, 210, 17, 67, 2, - 75, 2, 77, 2, 78, 2, 84, 3, 89, 14, 220, 230, 27, 2, 75, 87, 178, 124, - 67, 2, 80, 2, 84, 182, 213, 2, 87, 187, 113, 45, 4, 194, 136, 33, 79, - 191, 28, 65, 24, 30, 72, 1, 3, 83, 72, 82, 12, 202, 229, 27, 65, 174, - 130, 3, 79, 183, 56, 73, 19, 38, 65, 162, 231, 30, 79, 183, 56, 73, 9, - 210, 159, 31, 65, 207, 134, 2, 73, 15, 206, 228, 27, 65, 174, 130, 3, 79, - 183, 56, 73, 18, 224, 9, 3, 73, 75, 32, 149, 243, 22, 2, 85, 84, 33, 68, - 7, 74, 73, 66, 87, 65, 89, 32, 210, 164, 33, 78, 2, 79, 3, 89, 24, 74, - 78, 166, 176, 29, 83, 226, 243, 3, 67, 2, 75, 2, 77, 2, 80, 3, 84, 11, - 11, 87, 8, 246, 228, 30, 79, 183, 56, 73, 39, 206, 4, 87, 166, 13, 65, - 38, 79, 246, 138, 31, 73, 207, 134, 2, 69, 39, 104, 7, 45, 67, 82, 69, - 69, 32, 82, 146, 14, 87, 182, 2, 65, 230, 210, 30, 79, 182, 56, 73, 207, - 134, 2, 69, 4, 210, 139, 33, 87, 215, 22, 69, 125, 94, 65, 218, 1, 72, - 138, 1, 79, 238, 2, 87, 154, 175, 11, 80, 250, 229, 19, 73, 207, 134, 2, - 69, 43, 26, 89, 195, 154, 31, 65, 37, 25, 4, 73, 83, 73, 32, 34, 70, 72, - 54, 74, 218, 4, 83, 198, 135, 33, 89, 202, 18, 84, 147, 1, 77, 10, 166, - 225, 30, 79, 130, 191, 2, 65, 2, 69, 3, 73, 6, 214, 208, 29, 85, 159, - 207, 3, 73, 37, 70, 87, 202, 13, 79, 174, 222, 11, 65, 202, 172, 19, 73, - 207, 134, 2, 69, 16, 198, 13, 79, 150, 208, 27, 65, 226, 186, 3, 73, 207, - 134, 2, 69, 15, 80, 12, 85, 84, 72, 45, 83, 76, 65, 86, 69, 89, 32, 75, - 246, 157, 33, 79, 3, 89, 8, 226, 156, 33, 65, 2, 69, 2, 73, 3, 79, 117, - 110, 72, 190, 1, 76, 78, 84, 238, 8, 65, 66, 87, 226, 164, 11, 89, 198, - 173, 19, 79, 182, 56, 73, 207, 134, 2, 69, 39, 108, 7, 45, 67, 82, 69, - 69, 32, 84, 226, 4, 87, 222, 213, 27, 65, 174, 130, 3, 79, 182, 56, 73, - 207, 134, 2, 69, 16, 11, 72, 17, 174, 218, 27, 65, 174, 130, 3, 79, 182, - 56, 73, 207, 134, 2, 69, 12, 11, 72, 12, 142, 220, 30, 79, 174, 168, 2, - 87, 214, 22, 65, 2, 69, 3, 73, 24, 50, 72, 158, 154, 33, 65, 2, 69, 2, - 73, 3, 79, 17, 238, 216, 27, 65, 174, 130, 3, 79, 174, 168, 2, 87, 214, - 22, 69, 3, 73, 224, 1, 54, 69, 218, 3, 79, 174, 226, 11, 65, 203, 172, - 19, 73, 185, 1, 17, 2, 83, 84, 182, 1, 44, 6, 45, 67, 82, 69, 69, 32, + 226, 161, 32, 65, 223, 137, 2, 89, 30, 70, 83, 86, 87, 210, 17, 67, 2, + 75, 2, 77, 2, 78, 2, 84, 3, 89, 14, 168, 192, 28, 2, 75, 87, 254, 122, + 67, 2, 80, 2, 84, 190, 254, 2, 87, 175, 116, 45, 4, 230, 140, 34, 79, + 191, 28, 65, 24, 30, 72, 1, 3, 83, 72, 82, 12, 150, 191, 28, 65, 166, + 166, 3, 79, 135, 60, 73, 19, 38, 65, 230, 228, 31, 79, 135, 60, 73, 9, + 230, 160, 32, 65, 223, 137, 2, 73, 15, 154, 190, 28, 65, 166, 166, 3, 79, + 135, 60, 73, 18, 224, 9, 3, 73, 75, 32, 185, 186, 23, 2, 85, 84, 33, 68, + 7, 74, 73, 66, 87, 65, 89, 32, 246, 168, 34, 78, 2, 79, 3, 89, 24, 74, + 78, 142, 136, 30, 83, 158, 160, 4, 67, 2, 75, 2, 77, 2, 80, 3, 84, 11, + 11, 87, 8, 186, 226, 31, 79, 135, 60, 73, 39, 206, 4, 87, 166, 13, 65, + 38, 79, 138, 140, 32, 73, 223, 137, 2, 69, 39, 104, 7, 45, 67, 82, 69, + 69, 32, 82, 146, 14, 87, 182, 2, 65, 170, 208, 31, 79, 134, 60, 73, 223, + 137, 2, 69, 4, 246, 143, 34, 87, 215, 22, 69, 125, 94, 65, 218, 1, 72, + 138, 1, 79, 238, 2, 87, 226, 195, 11, 80, 198, 210, 20, 73, 223, 137, 2, + 69, 43, 26, 89, 215, 155, 32, 65, 37, 25, 4, 73, 83, 73, 32, 34, 70, 72, + 54, 74, 218, 4, 83, 234, 139, 34, 89, 202, 18, 84, 147, 1, 77, 10, 234, + 222, 31, 79, 226, 197, 2, 65, 2, 69, 3, 73, 6, 238, 208, 30, 85, 171, + 211, 3, 73, 37, 70, 87, 202, 13, 79, 130, 243, 11, 65, 138, 153, 20, 73, + 223, 137, 2, 69, 16, 198, 13, 79, 226, 169, 28, 65, 170, 226, 3, 73, 223, + 137, 2, 69, 15, 80, 12, 85, 84, 72, 45, 83, 76, 65, 86, 69, 89, 32, 75, + 154, 162, 34, 79, 3, 89, 8, 134, 161, 34, 65, 2, 69, 2, 73, 3, 79, 117, + 110, 72, 190, 1, 76, 78, 84, 238, 8, 65, 66, 87, 170, 185, 11, 89, 194, + 150, 20, 79, 134, 60, 73, 223, 137, 2, 69, 39, 108, 7, 45, 67, 82, 69, + 69, 32, 84, 226, 4, 87, 170, 175, 28, 65, 166, 166, 3, 79, 134, 60, 73, + 223, 137, 2, 69, 16, 11, 72, 17, 250, 179, 28, 65, 166, 166, 3, 79, 134, + 60, 73, 223, 137, 2, 69, 12, 11, 72, 12, 210, 217, 31, 79, 142, 175, 2, + 87, 214, 22, 65, 2, 69, 3, 73, 24, 50, 72, 194, 158, 34, 65, 2, 69, 2, + 73, 3, 79, 17, 186, 178, 28, 65, 166, 166, 3, 79, 142, 175, 2, 87, 214, + 22, 69, 3, 73, 224, 1, 54, 69, 218, 3, 79, 130, 247, 11, 65, 139, 153, + 20, 73, 185, 1, 17, 2, 83, 84, 182, 1, 44, 6, 45, 67, 82, 69, 69, 32, 251, 2, 69, 180, 1, 110, 76, 66, 77, 2, 80, 2, 89, 16, 2, 78, 87, 38, 82, - 62, 83, 26, 67, 2, 75, 18, 84, 26, 70, 199, 4, 87, 27, 178, 6, 87, 250, - 207, 27, 65, 174, 130, 3, 79, 131, 191, 2, 69, 17, 243, 5, 87, 6, 218, - 213, 27, 65, 175, 193, 5, 69, 13, 154, 167, 31, 87, 202, 239, 1, 65, 2, + 62, 83, 26, 67, 2, 75, 18, 84, 26, 70, 199, 4, 87, 27, 178, 6, 87, 198, + 169, 28, 65, 166, 166, 3, 79, 227, 197, 2, 69, 17, 243, 5, 87, 6, 166, + 175, 28, 65, 135, 236, 5, 69, 13, 186, 168, 32, 87, 206, 242, 1, 65, 2, 69, 2, 73, 3, 79, 28, 22, 72, 239, 4, 87, 14, 235, 4, 87, 16, 22, 72, - 199, 4, 87, 2, 159, 166, 31, 87, 2, 135, 240, 22, 82, 31, 11, 79, 29, 41, - 8, 68, 83, 45, 67, 82, 69, 69, 32, 26, 56, 2, 84, 72, 201, 149, 31, 6, - 70, 73, 78, 65, 76, 32, 25, 50, 87, 154, 148, 33, 65, 2, 69, 2, 73, 3, - 79, 14, 234, 210, 27, 65, 174, 130, 3, 79, 182, 56, 73, 251, 239, 1, 69, - 61, 92, 6, 45, 67, 82, 69, 69, 32, 150, 1, 65, 38, 79, 30, 87, 218, 138, - 31, 73, 207, 134, 2, 69, 24, 110, 80, 194, 209, 30, 67, 2, 75, 2, 76, 2, - 77, 2, 78, 2, 83, 2, 84, 2, 89, 214, 161, 2, 79, 247, 30, 87, 4, 190, - 162, 31, 87, 215, 208, 1, 79, 9, 150, 139, 31, 65, 207, 134, 2, 89, 7, - 190, 145, 33, 79, 3, 89, 14, 246, 207, 27, 65, 174, 130, 3, 79, 182, 56, - 73, 207, 134, 2, 69, 10, 26, 76, 203, 144, 33, 82, 9, 26, 32, 239, 162, - 9, 76, 4, 222, 217, 21, 67, 207, 208, 9, 84, 4, 178, 249, 32, 76, 215, - 22, 89, 4, 180, 247, 14, 3, 73, 84, 85, 253, 241, 7, 3, 82, 73, 67, 124, + 199, 4, 87, 2, 191, 167, 32, 87, 2, 171, 183, 23, 82, 31, 11, 79, 29, 41, + 8, 68, 83, 45, 67, 82, 69, 69, 32, 26, 56, 2, 84, 72, 225, 150, 32, 6, + 70, 73, 78, 65, 76, 32, 25, 50, 87, 190, 152, 34, 65, 2, 69, 2, 73, 3, + 79, 14, 182, 172, 28, 65, 166, 166, 3, 79, 134, 60, 73, 139, 243, 1, 69, + 61, 92, 6, 45, 67, 82, 69, 69, 32, 150, 1, 65, 38, 79, 30, 87, 238, 139, + 32, 73, 223, 137, 2, 69, 24, 110, 80, 134, 207, 31, 67, 2, 75, 2, 76, 2, + 77, 2, 78, 2, 83, 2, 84, 2, 89, 182, 168, 2, 79, 247, 30, 87, 4, 222, + 163, 32, 87, 219, 211, 1, 79, 9, 170, 140, 32, 65, 223, 137, 2, 89, 7, + 226, 149, 34, 79, 3, 89, 14, 194, 169, 28, 65, 166, 166, 3, 79, 134, 60, + 73, 223, 137, 2, 69, 10, 26, 76, 239, 148, 34, 82, 9, 26, 32, 155, 181, + 9, 76, 4, 182, 150, 22, 67, 151, 149, 10, 84, 4, 214, 253, 33, 76, 215, + 22, 89, 4, 216, 171, 15, 3, 73, 84, 85, 253, 132, 8, 3, 82, 73, 67, 124, 140, 1, 2, 68, 32, 102, 69, 72, 11, 73, 65, 78, 32, 76, 69, 84, 84, 69, - 82, 32, 210, 3, 79, 62, 80, 90, 82, 181, 250, 26, 4, 32, 83, 76, 73, 6, - 52, 5, 73, 78, 68, 69, 88, 157, 129, 32, 2, 70, 73, 5, 153, 255, 31, 6, - 32, 68, 73, 86, 73, 68, 6, 26, 84, 147, 131, 21, 32, 5, 209, 136, 10, 6, + 82, 32, 210, 3, 79, 62, 80, 90, 82, 141, 212, 27, 4, 32, 83, 76, 73, 6, + 52, 5, 73, 78, 68, 69, 88, 169, 133, 33, 2, 70, 73, 5, 165, 131, 33, 6, + 32, 68, 73, 86, 73, 68, 6, 26, 84, 227, 190, 21, 32, 5, 181, 156, 10, 6, 32, 73, 78, 83, 69, 82, 98, 196, 1, 2, 67, 45, 38, 76, 22, 77, 50, 78, - 38, 83, 46, 84, 22, 85, 230, 213, 3, 65, 2, 68, 2, 69, 2, 71, 2, 75, 2, - 80, 214, 208, 25, 82, 218, 220, 1, 73, 206, 134, 2, 66, 2, 79, 2, 81, 3, - 88, 4, 226, 163, 26, 49, 159, 148, 3, 51, 7, 139, 215, 3, 68, 11, 11, 66, - 9, 226, 138, 33, 50, 2, 51, 3, 52, 9, 190, 138, 33, 68, 2, 71, 3, 78, 13, - 162, 214, 3, 72, 2, 84, 251, 179, 29, 83, 7, 247, 213, 3, 84, 13, 11, 85, - 11, 11, 85, 9, 194, 137, 33, 50, 2, 51, 3, 85, 4, 148, 128, 32, 6, 85, - 83, 69, 76, 32, 72, 139, 137, 1, 78, 4, 152, 230, 26, 6, 32, 83, 84, 82, - 69, 65, 173, 164, 5, 7, 69, 78, 84, 82, 89, 32, 83, 4, 198, 240, 26, 73, - 155, 251, 5, 79, 9, 29, 5, 32, 70, 65, 67, 69, 7, 33, 6, 32, 87, 73, 84, - 72, 32, 4, 132, 229, 6, 2, 84, 69, 157, 132, 24, 6, 87, 82, 89, 32, 83, + 38, 83, 46, 84, 22, 85, 186, 213, 3, 65, 2, 68, 2, 69, 2, 71, 2, 75, 2, + 80, 130, 230, 26, 82, 238, 200, 1, 73, 222, 137, 2, 66, 2, 79, 2, 81, 3, + 88, 4, 174, 242, 26, 49, 219, 167, 3, 51, 7, 223, 214, 3, 68, 11, 11, 66, + 9, 134, 143, 34, 50, 2, 51, 3, 52, 9, 226, 142, 34, 68, 2, 71, 3, 78, 13, + 246, 213, 3, 72, 2, 84, 203, 184, 30, 83, 7, 203, 213, 3, 84, 13, 11, 85, + 11, 11, 85, 9, 230, 141, 34, 50, 2, 51, 3, 85, 4, 160, 132, 33, 6, 85, + 83, 69, 76, 32, 72, 163, 137, 1, 78, 4, 224, 189, 27, 6, 32, 83, 84, 82, + 69, 65, 249, 208, 5, 7, 69, 78, 84, 82, 89, 32, 83, 4, 142, 200, 27, 73, + 247, 167, 6, 79, 9, 29, 5, 32, 70, 65, 67, 69, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 208, 235, 6, 2, 84, 69, 129, 251, 24, 6, 87, 82, 89, 32, 83, 77, 108, 88, 16, 67, 65, 83, 73, 65, 78, 32, 65, 76, 66, 65, 78, 73, 65, - 78, 32, 159, 172, 29, 84, 106, 60, 7, 76, 69, 84, 84, 69, 82, 32, 189, - 160, 27, 2, 67, 73, 104, 174, 2, 65, 34, 67, 146, 1, 68, 78, 69, 34, 71, + 78, 32, 235, 137, 30, 84, 106, 60, 7, 76, 69, 84, 84, 69, 82, 32, 161, + 250, 27, 2, 67, 73, 104, 174, 2, 65, 34, 67, 146, 1, 68, 78, 69, 34, 71, 46, 73, 46, 74, 34, 75, 34, 76, 34, 80, 34, 83, 74, 84, 62, 89, 46, 90, - 140, 189, 16, 2, 81, 65, 166, 173, 6, 77, 208, 156, 6, 3, 86, 69, 89, - 182, 189, 2, 70, 220, 18, 3, 78, 79, 87, 146, 19, 82, 234, 21, 88, 150, - 46, 79, 202, 26, 66, 225, 24, 3, 72, 69, 89, 4, 194, 202, 32, 79, 167, - 28, 76, 16, 34, 65, 34, 72, 49, 2, 89, 65, 4, 250, 178, 32, 89, 215, 79, - 82, 8, 186, 205, 27, 65, 174, 163, 5, 79, 203, 17, 73, 4, 254, 129, 33, - 87, 3, 89, 8, 42, 90, 134, 186, 31, 89, 139, 171, 1, 65, 4, 202, 245, 31, - 89, 167, 122, 65, 4, 194, 177, 32, 89, 215, 79, 66, 4, 136, 214, 31, 2, - 72, 69, 151, 154, 1, 73, 6, 166, 198, 31, 82, 210, 106, 87, 251, 76, 78, - 4, 134, 244, 31, 72, 191, 14, 65, 4, 238, 129, 32, 73, 195, 69, 65, 4, - 138, 176, 32, 65, 159, 51, 89, 4, 142, 1, 73, 223, 174, 32, 69, 8, 34, - 72, 229, 251, 32, 2, 69, 89, 6, 142, 220, 19, 65, 167, 145, 13, 79, 6, - 38, 73, 198, 242, 31, 89, 159, 78, 65, 2, 223, 197, 32, 87, 4, 132, 184, - 32, 2, 65, 89, 1, 2, 79, 87, 6, 142, 223, 15, 72, 199, 177, 4, 65, 16, - 72, 2, 68, 73, 32, 2, 78, 84, 180, 155, 31, 3, 76, 84, 73, 207, 77, 82, - 4, 162, 246, 31, 32, 199, 100, 76, 8, 32, 2, 82, 69, 227, 245, 31, 32, 6, - 44, 5, 76, 73, 78, 69, 32, 207, 226, 4, 32, 4, 174, 154, 21, 76, 255, - 252, 9, 79, 252, 5, 102, 65, 170, 17, 69, 134, 8, 73, 150, 1, 79, 140, - 152, 4, 6, 82, 73, 83, 84, 77, 65, 195, 222, 18, 85, 198, 2, 66, 73, 32, - 4, 75, 77, 65, 32, 148, 6, 2, 77, 32, 131, 8, 82, 4, 226, 188, 32, 78, - 211, 61, 82, 142, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 142, 3, 83, - 98, 86, 170, 255, 22, 68, 170, 171, 7, 81, 236, 90, 5, 77, 65, 65, 89, - 89, 144, 174, 1, 2, 65, 85, 3, 79, 76, 202, 1, 68, 54, 78, 54, 84, 54, - 89, 142, 162, 28, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, 2, 80, 182, 228, - 2, 72, 2, 77, 2, 82, 2, 83, 2, 86, 2, 87, 146, 237, 1, 65, 186, 2, 69, 2, - 73, 3, 85, 8, 166, 163, 28, 68, 182, 228, 2, 72, 147, 237, 1, 65, 8, 166, - 135, 31, 71, 2, 78, 2, 89, 147, 237, 1, 65, 8, 190, 162, 28, 84, 182, - 228, 2, 72, 147, 237, 1, 65, 4, 190, 134, 31, 89, 147, 237, 1, 65, 8, 40, - 4, 73, 71, 78, 32, 195, 188, 21, 69, 6, 142, 143, 29, 67, 226, 182, 1, - 86, 223, 234, 1, 65, 26, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 203, 194, 30, 73, 24, 238, 139, 29, 65, 182, 25, 85, 210, 200, 1, 69, 2, + 128, 241, 16, 2, 81, 65, 214, 192, 6, 77, 148, 173, 6, 3, 86, 69, 89, + 178, 232, 2, 70, 128, 19, 3, 78, 79, 87, 150, 20, 82, 242, 21, 88, 154, + 46, 79, 202, 26, 66, 237, 24, 3, 72, 69, 89, 4, 218, 206, 33, 79, 179, + 28, 76, 16, 34, 65, 34, 72, 49, 2, 89, 65, 4, 146, 183, 33, 89, 227, 79, + 82, 8, 242, 166, 28, 65, 154, 206, 5, 79, 203, 17, 73, 4, 162, 134, 34, + 87, 3, 89, 8, 42, 90, 222, 187, 32, 89, 215, 173, 1, 65, 4, 214, 249, 32, + 89, 191, 122, 65, 4, 218, 181, 33, 89, 227, 79, 66, 4, 136, 217, 32, 2, + 72, 69, 187, 155, 1, 73, 6, 138, 201, 32, 82, 134, 108, 87, 135, 77, 78, + 4, 146, 248, 32, 72, 199, 14, 65, 4, 130, 134, 33, 73, 199, 69, 65, 4, + 162, 180, 33, 65, 171, 51, 89, 4, 142, 1, 73, 247, 178, 33, 69, 8, 34, + 72, 137, 128, 34, 2, 69, 89, 6, 182, 151, 20, 65, 163, 218, 13, 79, 6, + 38, 73, 210, 246, 32, 89, 171, 78, 65, 2, 247, 201, 33, 87, 4, 156, 188, + 33, 2, 65, 89, 1, 2, 79, 87, 6, 130, 147, 16, 72, 219, 185, 4, 65, 16, + 72, 2, 68, 73, 32, 2, 78, 84, 212, 156, 32, 3, 76, 84, 73, 187, 80, 82, + 4, 174, 250, 32, 32, 223, 100, 76, 8, 32, 2, 82, 69, 239, 249, 32, 32, 6, + 44, 5, 76, 73, 78, 69, 32, 147, 226, 4, 32, 4, 178, 214, 21, 76, 155, + 194, 10, 79, 252, 5, 102, 65, 170, 17, 69, 134, 8, 73, 150, 1, 79, 244, + 191, 13, 6, 82, 73, 83, 84, 77, 65, 139, 130, 10, 85, 198, 2, 66, 73, 32, + 4, 75, 77, 65, 32, 148, 6, 2, 77, 32, 131, 8, 82, 4, 250, 192, 33, 78, + 223, 61, 82, 142, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 142, 3, 83, + 98, 86, 214, 202, 23, 68, 194, 221, 7, 81, 200, 94, 5, 77, 65, 65, 89, + 89, 136, 177, 1, 2, 65, 85, 3, 79, 76, 202, 1, 68, 54, 78, 54, 84, 54, + 89, 166, 250, 28, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, 2, 80, 190, 141, + 3, 72, 2, 77, 2, 82, 2, 83, 2, 86, 2, 87, 150, 240, 1, 65, 186, 2, 69, 2, + 73, 3, 85, 8, 190, 251, 28, 68, 190, 141, 3, 72, 151, 240, 1, 65, 8, 198, + 136, 32, 71, 2, 78, 2, 89, 151, 240, 1, 65, 8, 214, 250, 28, 84, 190, + 141, 3, 72, 151, 240, 1, 65, 4, 222, 135, 32, 89, 151, 240, 1, 65, 8, 40, + 4, 73, 71, 78, 32, 155, 249, 21, 69, 6, 154, 229, 29, 67, 154, 222, 1, + 86, 179, 241, 1, 65, 26, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 143, 192, 31, 73, 24, 194, 158, 30, 65, 250, 6, 85, 206, 201, 1, 69, 2, 73, 3, 79, 166, 1, 236, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 112, 7, 76, 69, 84, 84, 69, 82, 32, 152, 4, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 60, 11, 86, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 175, 254, 30, 68, 14, 72, 6, 70, 73, 78, 65, 76, - 32, 138, 239, 32, 76, 2, 82, 2, 87, 3, 89, 6, 234, 240, 32, 78, 86, 72, + 32, 83, 73, 71, 78, 32, 207, 255, 31, 68, 14, 72, 6, 70, 73, 78, 65, 76, + 32, 174, 243, 33, 76, 2, 82, 2, 87, 3, 89, 6, 142, 245, 33, 78, 86, 72, 3, 77, 104, 132, 2, 6, 70, 73, 78, 65, 76, 32, 110, 78, 50, 77, 78, 80, - 146, 195, 11, 66, 198, 231, 5, 68, 234, 191, 2, 83, 166, 252, 10, 65, - 152, 191, 1, 2, 67, 72, 2, 71, 2, 74, 2, 75, 2, 84, 254, 68, 72, 2, 76, - 2, 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 22, 254, 246, 30, - 78, 198, 186, 1, 83, 194, 60, 67, 146, 1, 71, 2, 75, 2, 76, 2, 80, 2, 82, - 2, 84, 3, 89, 14, 46, 71, 34, 72, 134, 215, 32, 85, 215, 22, 65, 4, 162, - 215, 32, 85, 215, 22, 65, 6, 130, 215, 32, 85, 158, 20, 74, 187, 2, 65, - 6, 242, 234, 32, 72, 2, 80, 187, 2, 65, 8, 190, 136, 11, 83, 246, 142, - 12, 68, 45, 4, 84, 82, 73, 80, 20, 238, 135, 29, 65, 142, 222, 1, 73, - 186, 198, 1, 79, 2, 85, 191, 44, 69, 14, 72, 7, 65, 67, 84, 69, 82, 32, - 84, 49, 7, 84, 32, 87, 73, 84, 72, 32, 8, 140, 223, 9, 3, 65, 66, 85, - 211, 245, 22, 73, 6, 128, 1, 13, 85, 80, 87, 65, 82, 68, 83, 32, 84, 82, - 69, 78, 68, 149, 214, 24, 12, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, - 82, 5, 233, 234, 6, 6, 32, 65, 78, 68, 32, 89, 234, 2, 80, 2, 67, 75, 82, - 69, 90, 82, 224, 247, 4, 2, 83, 84, 241, 219, 14, 2, 81, 85, 6, 56, 8, - 69, 82, 32, 66, 79, 65, 82, 68, 151, 167, 32, 32, 5, 223, 247, 29, 32, 4, - 184, 215, 26, 4, 83, 69, 32, 87, 177, 231, 4, 9, 82, 73, 78, 71, 32, 77, + 226, 215, 11, 66, 138, 135, 6, 68, 206, 134, 12, 83, 194, 130, 2, 65, + 156, 194, 1, 2, 67, 72, 2, 71, 2, 74, 2, 75, 2, 84, 138, 69, 72, 2, 76, + 2, 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 22, 158, 248, 31, + 78, 190, 189, 1, 83, 206, 60, 67, 146, 1, 71, 2, 75, 2, 76, 2, 80, 2, 82, + 2, 84, 3, 89, 14, 46, 71, 34, 72, 170, 219, 33, 85, 215, 22, 65, 4, 198, + 219, 33, 85, 215, 22, 65, 6, 166, 219, 33, 85, 158, 20, 74, 187, 2, 65, + 6, 150, 239, 33, 72, 2, 80, 187, 2, 65, 8, 246, 156, 11, 83, 186, 247, + 18, 68, 45, 4, 84, 82, 73, 80, 20, 158, 157, 30, 65, 242, 201, 1, 73, + 190, 201, 1, 79, 2, 85, 203, 44, 69, 14, 72, 7, 65, 67, 84, 69, 82, 32, + 84, 49, 7, 84, 32, 87, 73, 84, 72, 32, 8, 240, 242, 9, 3, 65, 66, 85, + 147, 230, 23, 73, 6, 128, 1, 13, 85, 80, 87, 65, 82, 68, 83, 32, 84, 82, + 69, 78, 68, 217, 160, 25, 12, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, + 82, 5, 225, 242, 6, 6, 32, 65, 78, 68, 32, 89, 234, 2, 80, 2, 67, 75, 82, + 69, 90, 82, 156, 247, 4, 2, 83, 84, 229, 151, 15, 2, 81, 85, 6, 56, 8, + 69, 82, 32, 66, 79, 65, 82, 68, 175, 171, 33, 32, 5, 211, 244, 30, 32, 4, + 140, 177, 27, 4, 83, 69, 32, 87, 229, 144, 5, 9, 82, 73, 78, 71, 32, 77, 69, 71, 65, 220, 2, 40, 5, 79, 75, 69, 69, 32, 143, 5, 82, 216, 2, 46, 76, 1, 7, 83, 77, 65, 76, 76, 32, 76, 172, 1, 33, 6, 69, 84, 84, 69, 82, 32, 172, 1, 170, 1, 68, 74, 72, 74, 78, 70, 83, 62, 84, 54, 71, 2, 76, 2, - 77, 0, 2, 81, 85, 2, 87, 2, 89, 158, 224, 32, 75, 186, 2, 65, 2, 69, 2, - 73, 2, 79, 2, 85, 3, 86, 14, 222, 226, 32, 76, 186, 2, 65, 2, 69, 2, 73, - 2, 79, 2, 85, 3, 86, 14, 150, 226, 32, 78, 186, 2, 65, 2, 69, 2, 73, 2, - 79, 2, 85, 3, 86, 14, 166, 240, 28, 65, 226, 243, 3, 69, 2, 73, 2, 79, 2, - 85, 3, 86, 15, 194, 227, 32, 65, 2, 69, 2, 73, 2, 79, 2, 85, 3, 86, 30, - 50, 76, 2, 83, 214, 226, 32, 65, 2, 69, 3, 73, 12, 210, 226, 32, 65, 2, - 69, 2, 73, 2, 79, 2, 85, 3, 86, 4, 48, 6, 89, 32, 66, 76, 79, 83, 135, - 206, 31, 73, 2, 255, 208, 32, 83, 12, 96, 2, 76, 68, 174, 222, 7, 32, - 232, 239, 2, 2, 80, 77, 228, 193, 20, 2, 67, 75, 151, 129, 1, 82, 5, 213, - 128, 28, 7, 82, 69, 78, 32, 67, 82, 79, 60, 92, 8, 82, 65, 83, 77, 73, - 65, 78, 32, 222, 253, 4, 80, 249, 203, 11, 5, 67, 79, 76, 65, 84, 56, 52, - 7, 76, 69, 84, 84, 69, 82, 32, 243, 134, 21, 78, 42, 224, 1, 6, 67, 85, - 82, 76, 69, 68, 30, 83, 202, 131, 21, 68, 34, 76, 254, 194, 1, 82, 134, - 212, 2, 65, 50, 71, 90, 90, 98, 89, 154, 193, 1, 72, 234, 5, 75, 130, 76, - 66, 190, 173, 4, 78, 254, 1, 84, 2, 87, 202, 103, 80, 171, 4, 77, 2, 161, - 223, 31, 2, 32, 87, 6, 176, 131, 21, 6, 77, 65, 76, 76, 32, 65, 172, 210, - 5, 2, 65, 77, 139, 136, 5, 72, 232, 4, 66, 78, 20, 2, 82, 67, 233, 40, 7, - 84, 89, 83, 67, 65, 80, 69, 2, 247, 186, 32, 69, 226, 4, 28, 2, 76, 69, - 239, 39, 85, 220, 4, 30, 32, 181, 6, 2, 68, 32, 24, 244, 1, 5, 87, 73, - 84, 72, 32, 161, 133, 18, 49, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, + 77, 0, 2, 81, 85, 2, 87, 2, 89, 194, 228, 33, 75, 186, 2, 65, 2, 69, 2, + 73, 2, 79, 2, 85, 3, 86, 14, 130, 231, 33, 76, 186, 2, 65, 2, 69, 2, 73, + 2, 79, 2, 85, 3, 86, 14, 186, 230, 33, 78, 186, 2, 65, 2, 69, 2, 73, 2, + 79, 2, 85, 3, 86, 14, 142, 200, 29, 65, 158, 160, 4, 69, 2, 73, 2, 79, 2, + 85, 3, 86, 15, 230, 231, 33, 65, 2, 69, 2, 73, 2, 79, 2, 85, 3, 86, 30, + 50, 76, 2, 83, 250, 230, 33, 65, 2, 69, 3, 73, 12, 246, 230, 33, 65, 2, + 69, 2, 73, 2, 79, 2, 85, 3, 86, 4, 48, 6, 89, 32, 66, 76, 79, 83, 147, + 210, 32, 73, 2, 163, 213, 33, 83, 12, 96, 2, 76, 68, 254, 235, 7, 32, + 204, 246, 2, 2, 80, 77, 224, 174, 21, 2, 67, 75, 255, 131, 1, 82, 5, 237, + 216, 28, 7, 82, 69, 78, 32, 67, 82, 79, 60, 92, 8, 82, 65, 83, 77, 73, + 65, 78, 32, 174, 254, 4, 80, 189, 255, 11, 5, 67, 79, 76, 65, 84, 56, 52, + 7, 76, 69, 84, 84, 69, 82, 32, 247, 194, 21, 78, 42, 224, 1, 6, 67, 85, + 82, 76, 69, 68, 30, 83, 206, 191, 21, 68, 34, 76, 158, 206, 1, 82, 178, + 215, 2, 65, 50, 71, 90, 90, 98, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, + 66, 178, 216, 4, 78, 134, 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 2, 181, + 227, 32, 2, 32, 87, 6, 180, 191, 21, 6, 77, 65, 76, 76, 32, 65, 252, 239, + 5, 2, 65, 77, 195, 178, 5, 72, 232, 4, 66, 78, 20, 2, 82, 67, 201, 40, 7, + 84, 89, 83, 67, 65, 80, 69, 2, 155, 191, 33, 69, 226, 4, 28, 2, 76, 69, + 207, 39, 85, 220, 4, 30, 32, 185, 6, 2, 68, 32, 24, 244, 1, 5, 87, 73, + 84, 72, 32, 185, 186, 18, 49, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 65, 78, 68, 32, 84, 79, 80, 32, 72, 65, 76, 70, 32, 68, 73, 86, 73, 68, 69, 68, 32, - 66, 89, 22, 142, 2, 76, 46, 83, 108, 22, 84, 87, 79, 32, 72, 79, 82, 73, + 66, 89, 22, 146, 2, 76, 46, 83, 108, 22, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 83, 84, 82, 79, 75, 69, 83, 44, 6, 85, 80, 80, 69, 82, 32, 44, 17, 65, 76, 76, 32, 66, 85, 84, 32, 85, 80, 80, 69, - 82, 32, 76, 69, 70, 238, 230, 25, 86, 190, 227, 3, 82, 231, 58, 72, 4, - 138, 204, 29, 69, 49, 4, 79, 87, 69, 82, 4, 104, 11, 77, 65, 76, 76, 32, - 67, 73, 82, 67, 76, 69, 221, 5, 10, 85, 80, 69, 82, 73, 77, 80, 79, 83, - 69, 2, 173, 225, 29, 6, 32, 84, 79, 32, 84, 72, 4, 40, 4, 82, 73, 71, 72, - 227, 202, 29, 72, 2, 245, 202, 29, 10, 84, 32, 81, 85, 65, 68, 82, 65, - 78, 84, 196, 4, 230, 2, 65, 186, 1, 66, 58, 67, 230, 1, 68, 246, 1, 72, - 250, 2, 73, 210, 10, 75, 190, 2, 76, 38, 77, 136, 1, 7, 78, 85, 77, 66, - 69, 82, 32, 192, 4, 17, 79, 80, 69, 78, 32, 67, 69, 78, 84, 82, 69, 32, - 69, 73, 71, 72, 84, 54, 80, 114, 82, 50, 84, 78, 87, 224, 228, 9, 4, 90, - 69, 82, 79, 222, 253, 15, 69, 250, 129, 4, 86, 130, 65, 71, 190, 110, 83, - 199, 160, 1, 88, 6, 100, 12, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, - 69, 249, 166, 29, 7, 83, 84, 69, 82, 73, 83, 75, 4, 160, 220, 8, 11, 45, - 82, 79, 84, 65, 84, 69, 68, 32, 68, 73, 211, 218, 22, 32, 4, 32, 2, 79, - 76, 139, 244, 30, 85, 2, 175, 225, 13, 68, 16, 60, 4, 82, 79, 83, 83, - 210, 2, 32, 142, 205, 32, 67, 3, 68, 10, 26, 32, 219, 193, 1, 73, 8, 64, - 6, 70, 79, 82, 77, 69, 69, 201, 226, 30, 4, 80, 79, 77, 77, 7, 33, 6, 32, - 87, 73, 84, 72, 32, 4, 254, 177, 30, 70, 251, 78, 84, 30, 34, 73, 70, 79, - 183, 232, 30, 65, 24, 152, 188, 4, 8, 86, 73, 83, 73, 79, 78, 32, 83, - 247, 163, 26, 71, 4, 64, 10, 76, 76, 65, 82, 32, 83, 73, 71, 78, 32, 227, - 162, 29, 84, 2, 197, 230, 18, 13, 87, 73, 84, 72, 32, 79, 86, 69, 82, 76, - 65, 73, 68, 64, 212, 1, 6, 65, 78, 71, 85, 76, 32, 184, 151, 12, 20, 79, + 82, 32, 76, 69, 70, 182, 181, 26, 86, 178, 136, 4, 82, 231, 249, 1, 72, + 4, 198, 191, 30, 69, 49, 4, 79, 87, 69, 82, 4, 104, 11, 77, 65, 76, 76, + 32, 67, 73, 82, 67, 76, 69, 221, 5, 10, 85, 80, 69, 82, 73, 77, 80, 79, + 83, 69, 2, 153, 217, 30, 6, 32, 84, 79, 32, 84, 72, 4, 40, 4, 82, 73, 71, + 72, 159, 190, 30, 72, 2, 177, 190, 30, 10, 84, 32, 81, 85, 65, 68, 82, + 65, 78, 84, 196, 4, 230, 2, 65, 186, 1, 66, 58, 67, 230, 1, 68, 246, 1, + 72, 230, 2, 73, 194, 10, 75, 190, 2, 76, 38, 77, 136, 1, 7, 78, 85, 77, + 66, 69, 82, 32, 192, 4, 17, 79, 80, 69, 78, 32, 67, 69, 78, 84, 82, 69, + 32, 69, 73, 71, 72, 84, 54, 80, 114, 82, 50, 84, 78, 87, 232, 248, 9, 4, + 90, 69, 82, 79, 234, 183, 16, 69, 246, 177, 4, 86, 158, 68, 71, 190, 113, + 83, 223, 160, 1, 88, 6, 100, 12, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, + 83, 69, 213, 149, 30, 7, 83, 84, 69, 82, 73, 83, 75, 4, 200, 238, 8, 11, + 45, 82, 79, 84, 65, 84, 69, 68, 32, 68, 73, 179, 204, 23, 32, 4, 32, 2, + 79, 76, 167, 245, 31, 85, 2, 131, 143, 14, 68, 16, 60, 4, 82, 79, 83, 83, + 210, 2, 32, 174, 209, 33, 67, 3, 68, 10, 26, 32, 155, 193, 1, 73, 8, 64, + 6, 70, 79, 82, 77, 69, 69, 229, 227, 31, 4, 80, 79, 77, 77, 7, 33, 6, 32, + 87, 73, 84, 72, 32, 4, 214, 175, 31, 70, 207, 82, 84, 30, 34, 73, 70, 79, + 211, 233, 31, 65, 24, 216, 187, 4, 8, 86, 73, 83, 73, 79, 78, 32, 83, + 211, 165, 27, 71, 4, 64, 10, 76, 76, 65, 82, 32, 83, 73, 71, 78, 32, 191, + 145, 30, 84, 2, 145, 156, 19, 13, 87, 73, 84, 72, 32, 79, 86, 69, 82, 76, + 65, 73, 68, 64, 192, 1, 6, 65, 78, 71, 85, 76, 32, 204, 191, 12, 20, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 87, 73, 84, 72, 32, - 78, 152, 181, 9, 4, 69, 65, 86, 89, 189, 158, 6, 8, 85, 77, 65, 78, 32, - 70, 73, 71, 58, 110, 67, 160, 128, 20, 5, 73, 69, 85, 78, 71, 42, 72, 30, - 75, 66, 77, 34, 78, 34, 80, 62, 82, 30, 83, 27, 84, 8, 254, 255, 19, 72, - 157, 3, 4, 73, 69, 85, 67, 116, 220, 1, 9, 68, 69, 79, 71, 82, 65, 80, - 72, 32, 212, 8, 27, 84, 65, 76, 73, 67, 32, 76, 65, 84, 73, 78, 32, 67, - 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 165, 176, 26, 9, - 78, 70, 79, 82, 77, 65, 84, 73, 79, 110, 178, 1, 65, 110, 67, 90, 69, 86, - 70, 62, 72, 38, 75, 70, 76, 50, 77, 86, 78, 62, 81, 30, 82, 74, 83, 246, - 128, 20, 84, 50, 87, 146, 204, 1, 73, 172, 24, 2, 80, 82, 231, 193, 9, - 79, 8, 130, 129, 20, 76, 180, 132, 1, 3, 67, 67, 69, 196, 212, 6, 3, 84, - 84, 69, 245, 17, 5, 68, 86, 65, 78, 84, 8, 26, 79, 159, 208, 29, 69, 6, - 168, 180, 10, 2, 82, 82, 146, 204, 9, 78, 175, 179, 12, 80, 8, 206, 128, - 20, 78, 246, 253, 8, 65, 148, 131, 3, 5, 88, 67, 69, 76, 76, 219, 24, 73, - 10, 218, 128, 20, 73, 128, 169, 5, 2, 69, 77, 131, 162, 6, 79, 4, 214, - 203, 28, 73, 203, 251, 2, 65, 4, 240, 141, 7, 8, 73, 78, 68, 69, 82, 71, - 65, 82, 199, 233, 21, 79, 6, 234, 255, 19, 65, 130, 197, 11, 79, 179, 83, - 69, 8, 38, 69, 130, 161, 31, 65, 199, 81, 79, 4, 128, 169, 31, 3, 68, 73, - 67, 251, 15, 84, 6, 26, 73, 139, 170, 31, 65, 4, 162, 151, 32, 71, 231, - 19, 78, 2, 237, 218, 27, 2, 85, 69, 8, 26, 69, 195, 150, 32, 73, 6, 242, - 254, 19, 83, 205, 225, 7, 2, 76, 73, 22, 90, 69, 38, 85, 164, 230, 13, 2, - 67, 72, 166, 152, 6, 79, 22, 80, 34, 84, 211, 180, 11, 73, 4, 178, 132, - 10, 67, 215, 234, 20, 86, 6, 198, 255, 19, 80, 180, 177, 4, 3, 73, 84, - 65, 223, 142, 8, 78, 4, 150, 191, 32, 67, 3, 82, 98, 116, 8, 65, 84, 65, - 75, 65, 78, 65, 32, 133, 1, 16, 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, - 65, 67, 84, 69, 82, 32, 94, 194, 208, 10, 72, 2, 75, 2, 77, 2, 78, 2, 82, - 2, 83, 2, 84, 126, 87, 46, 89, 154, 236, 21, 65, 2, 69, 2, 73, 2, 79, 3, - 85, 4, 216, 214, 15, 3, 74, 85, 69, 165, 199, 16, 4, 67, 72, 65, 77, 106, - 166, 211, 26, 65, 203, 218, 3, 69, 4, 100, 19, 85, 76, 84, 73, 80, 76, - 73, 67, 65, 84, 73, 79, 78, 32, 83, 73, 71, 78, 32, 191, 184, 30, 73, 2, - 129, 84, 4, 87, 73, 84, 72, 98, 50, 69, 46, 70, 98, 83, 94, 84, 151, 251, - 19, 78, 6, 180, 1, 3, 73, 71, 72, 191, 222, 24, 76, 30, 28, 3, 73, 70, - 84, 35, 79, 6, 214, 1, 89, 203, 231, 30, 69, 24, 142, 2, 82, 255, 250, - 19, 85, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 11, 84, 4, 104, 2, 89, - 32, 191, 231, 30, 69, 52, 56, 2, 69, 78, 32, 4, 72, 73, 82, 84, 29, 2, - 87, 69, 5, 11, 32, 2, 135, 252, 24, 79, 24, 74, 89, 223, 230, 30, 69, 24, - 26, 78, 163, 187, 31, 76, 22, 17, 2, 84, 89, 23, 11, 32, 20, 72, 2, 79, - 78, 218, 201, 30, 70, 30, 83, 42, 84, 190, 83, 78, 211, 110, 69, 4, 226, - 250, 24, 32, 151, 188, 7, 69, 2, 213, 163, 30, 8, 32, 80, 79, 73, 78, 84, - 69, 68, 8, 184, 146, 17, 3, 79, 83, 84, 246, 223, 6, 65, 188, 189, 2, 8, - 69, 82, 80, 69, 78, 68, 73, 67, 203, 131, 4, 76, 4, 128, 139, 29, 3, 73, - 78, 71, 255, 151, 1, 69, 6, 52, 7, 82, 73, 65, 78, 71, 76, 69, 151, 224, - 25, 73, 5, 163, 204, 21, 32, 6, 44, 5, 72, 73, 84, 69, 32, 135, 180, 32, - 90, 4, 206, 215, 30, 66, 211, 24, 83, 6, 226, 198, 26, 77, 252, 138, 4, - 6, 76, 65, 84, 73, 79, 78, 233, 158, 1, 3, 83, 32, 84, 5, 233, 240, 30, - 6, 32, 65, 84, 32, 68, 85, 154, 18, 192, 1, 24, 67, 79, 77, 80, 65, 84, - 73, 66, 73, 76, 73, 84, 89, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, - 144, 3, 8, 82, 65, 68, 73, 67, 65, 76, 32, 245, 17, 7, 83, 84, 82, 79, - 75, 69, 32, 236, 15, 24, 2, 50, 70, 75, 70, 188, 8, 34, 65, 194, 225, 10, - 56, 3, 57, 60, 202, 1, 49, 143, 182, 18, 48, 176, 7, 26, 65, 131, 225, - 10, 57, 176, 3, 134, 1, 54, 142, 182, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, - 2, 53, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 231, 238, 8, 68, 28, - 130, 175, 32, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, - 2, 57, 2, 65, 2, 66, 2, 67, 3, 68, 230, 1, 210, 1, 66, 126, 67, 226, 4, - 68, 62, 69, 50, 70, 34, 71, 50, 72, 86, 74, 154, 1, 76, 62, 77, 130, 1, - 80, 46, 82, 78, 83, 182, 3, 84, 82, 87, 200, 197, 10, 2, 78, 69, 252, 16, - 4, 75, 78, 73, 70, 159, 225, 11, 79, 14, 74, 79, 192, 229, 10, 4, 82, 85, - 83, 72, 154, 218, 4, 65, 179, 139, 13, 76, 6, 230, 140, 10, 76, 170, 136, - 22, 78, 215, 22, 88, 56, 104, 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, - 68, 32, 162, 3, 73, 50, 76, 234, 168, 31, 79, 175, 78, 72, 44, 114, 69, - 38, 70, 50, 71, 38, 76, 34, 83, 94, 84, 214, 146, 14, 87, 42, 68, 246, - 149, 16, 66, 234, 117, 72, 135, 2, 67, 4, 222, 216, 30, 86, 203, 180, 1, - 65, 6, 146, 196, 30, 73, 214, 211, 1, 76, 235, 16, 82, 4, 234, 195, 22, - 79, 235, 160, 8, 65, 4, 194, 167, 31, 69, 175, 48, 79, 10, 174, 204, 6, - 73, 242, 253, 16, 65, 142, 194, 1, 72, 232, 249, 6, 3, 80, 69, 69, 215, - 11, 69, 6, 214, 10, 85, 176, 155, 10, 2, 65, 78, 143, 255, 3, 79, 4, 156, - 150, 20, 3, 86, 73, 76, 195, 255, 11, 84, 4, 214, 161, 17, 73, 171, 241, - 13, 79, 6, 220, 201, 23, 2, 73, 86, 194, 226, 2, 69, 227, 249, 5, 79, 10, - 196, 6, 2, 65, 84, 254, 136, 32, 87, 3, 89, 4, 198, 133, 31, 73, 247, 79, - 79, 8, 244, 5, 4, 82, 65, 83, 83, 255, 215, 28, 72, 10, 48, 2, 69, 65, - 218, 198, 27, 79, 175, 208, 3, 65, 6, 186, 8, 82, 171, 156, 32, 68, 10, - 72, 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, 68, 32, 135, 151, 30, 65, - 8, 42, 84, 234, 142, 14, 68, 251, 167, 16, 69, 4, 194, 6, 85, 191, 154, - 14, 79, 12, 132, 221, 10, 3, 79, 78, 71, 197, 176, 15, 3, 65, 77, 69, 14, - 18, 69, 35, 79, 4, 174, 134, 32, 65, 159, 27, 83, 10, 148, 220, 10, 3, - 85, 78, 68, 158, 236, 12, 84, 166, 150, 7, 82, 147, 116, 79, 6, 200, 219, - 10, 2, 65, 87, 143, 243, 14, 69, 8, 34, 65, 233, 252, 27, 2, 69, 80, 6, - 202, 209, 31, 73, 214, 79, 77, 3, 80, 38, 122, 69, 90, 73, 186, 1, 78, - 196, 1, 4, 80, 73, 82, 73, 132, 214, 10, 4, 77, 65, 76, 76, 250, 222, 11, - 72, 219, 151, 9, 85, 8, 40, 4, 67, 79, 78, 68, 183, 150, 31, 65, 6, 11, - 32, 6, 230, 178, 30, 84, 203, 83, 79, 12, 60, 9, 77, 80, 76, 73, 70, 73, - 69, 68, 32, 207, 155, 32, 76, 10, 34, 72, 50, 87, 159, 161, 10, 89, 4, - 184, 139, 10, 3, 65, 76, 70, 251, 180, 17, 79, 4, 130, 194, 6, 65, 215, - 183, 21, 72, 6, 192, 1, 2, 79, 85, 211, 236, 31, 65, 8, 58, 85, 174, 142, - 24, 65, 230, 199, 1, 72, 135, 143, 3, 73, 2, 243, 187, 23, 82, 12, 26, - 65, 49, 2, 69, 83, 8, 176, 214, 10, 2, 76, 75, 1, 3, 84, 69, 82, 4, 131, - 214, 10, 84, 72, 110, 72, 162, 1, 80, 38, 83, 226, 161, 30, 84, 172, 247, - 1, 2, 66, 88, 2, 87, 2, 88, 86, 68, 2, 78, 3, 81, 29, 50, 80, 22, 90, - 136, 154, 32, 2, 88, 87, 87, 71, 5, 155, 154, 32, 87, 19, 50, 90, 150, - 162, 30, 87, 150, 248, 1, 71, 3, 84, 9, 146, 162, 30, 90, 151, 248, 1, - 80, 9, 130, 154, 32, 68, 2, 71, 3, 90, 21, 50, 87, 30, 90, 146, 153, 32, - 71, 2, 80, 3, 84, 7, 170, 153, 32, 71, 3, 90, 7, 186, 152, 32, 87, 87, - 90, 124, 62, 65, 194, 1, 73, 114, 79, 173, 181, 20, 4, 69, 65, 82, 32, 8, - 132, 1, 2, 80, 80, 252, 177, 18, 6, 83, 83, 73, 67, 65, 76, 241, 187, 12, - 14, 77, 83, 72, 69, 76, 76, 32, 77, 79, 66, 73, 76, 69, 32, 4, 144, 130, - 19, 5, 73, 78, 71, 32, 72, 199, 226, 1, 69, 6, 48, 6, 78, 75, 73, 78, 71, - 32, 207, 255, 30, 80, 4, 234, 198, 24, 71, 229, 169, 3, 6, 66, 69, 69, - 82, 32, 77, 108, 72, 2, 67, 75, 140, 10, 2, 83, 69, 236, 3, 2, 85, 68, - 227, 154, 29, 87, 70, 64, 6, 32, 70, 65, 67, 69, 32, 237, 2, 5, 87, 73, - 83, 69, 32, 48, 58, 69, 46, 70, 36, 2, 78, 73, 2, 79, 18, 83, 51, 84, 8, - 120, 2, 76, 69, 125, 4, 73, 71, 72, 84, 8, 178, 1, 73, 25, 3, 79, 85, 82, - 4, 155, 1, 78, 8, 26, 69, 125, 2, 73, 88, 4, 57, 2, 86, 69, 16, 38, 69, - 14, 87, 41, 3, 72, 82, 69, 4, 63, 78, 8, 24, 2, 69, 76, 27, 79, 4, 11, - 86, 4, 11, 69, 4, 196, 164, 28, 2, 32, 79, 213, 131, 2, 3, 45, 84, 72, - 22, 90, 67, 58, 68, 122, 71, 48, 5, 82, 73, 71, 72, 84, 226, 2, 84, 122, - 79, 247, 135, 30, 73, 4, 196, 1, 3, 76, 79, 83, 141, 99, 4, 79, 78, 84, - 79, 2, 141, 3, 26, 79, 87, 78, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, - 85, 80, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 2, 21, 3, 65, 80, 80, - 2, 133, 4, 2, 69, 68, 6, 228, 1, 14, 32, 65, 78, 68, 32, 76, 69, 70, 84, - 32, 83, 69, 77, 73, 45, 38, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, 76, - 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 67, 73, 82, 67, - 76, 69, 32, 65, 82, 82, 79, 87, 83, 2, 129, 184, 29, 6, 67, 73, 82, 67, - 76, 69, 5, 161, 83, 15, 32, 87, 73, 84, 72, 32, 67, 73, 82, 67, 76, 69, - 68, 32, 79, 4, 82, 79, 37, 16, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, - 65, 68, 69, 68, 32, 79, 2, 69, 6, 80, 32, 83, 69, 77, 73, 2, 21, 3, 80, - 69, 78, 2, 11, 32, 2, 161, 182, 29, 4, 67, 73, 82, 67, 26, 32, 2, 68, 32, - 167, 149, 31, 32, 24, 216, 1, 2, 83, 85, 66, 85, 254, 243, 18, 77, 180, - 3, 9, 76, 79, 67, 75, 32, 87, 73, 84, 72, 222, 192, 11, 66, 145, 147, 1, - 23, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 79, 78, 32, 87, 73, 84, 72, - 32, 83, 69, 82, 73, 70, 8, 30, 66, 1, 3, 80, 69, 82, 4, 145, 177, 24, 3, - 83, 69, 84, 6, 82, 77, 33, 16, 78, 73, 79, 78, 32, 87, 73, 84, 72, 32, - 83, 69, 82, 73, 70, 83, 2, 197, 148, 24, 3, 66, 82, 69, 5, 221, 248, 30, - 9, 32, 65, 78, 68, 32, 83, 77, 65, 83, 11, 33, 6, 32, 87, 73, 84, 72, 32, - 8, 72, 4, 84, 79, 82, 78, 190, 162, 5, 76, 146, 151, 19, 83, 215, 161, 6, - 82, 2, 155, 140, 25, 65, 226, 9, 158, 1, 67, 82, 76, 98, 77, 162, 82, 78, - 220, 5, 2, 79, 75, 58, 80, 154, 19, 82, 138, 1, 85, 186, 165, 27, 87, - 160, 242, 2, 2, 70, 70, 234, 47, 73, 171, 51, 65, 6, 26, 75, 139, 149, 4, - 79, 4, 186, 214, 8, 82, 129, 151, 22, 4, 84, 65, 73, 76, 8, 44, 2, 79, - 78, 209, 187, 24, 3, 76, 73, 83, 7, 11, 32, 4, 146, 178, 25, 69, 171, - 203, 5, 83, 158, 6, 72, 7, 66, 73, 78, 73, 78, 71, 32, 166, 80, 77, 94, - 80, 239, 149, 31, 69, 142, 6, 214, 2, 65, 174, 2, 66, 134, 1, 67, 246, - 12, 68, 174, 9, 69, 146, 2, 70, 54, 71, 140, 9, 2, 72, 79, 82, 73, 216, - 1, 2, 75, 65, 162, 1, 76, 230, 13, 77, 250, 1, 78, 78, 79, 138, 2, 80, - 174, 1, 82, 244, 4, 5, 90, 73, 71, 90, 65, 166, 2, 83, 242, 3, 84, 238, - 2, 85, 148, 1, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 36, 2, 87, 73, 183, - 1, 88, 24, 148, 1, 4, 67, 85, 84, 69, 86, 78, 168, 185, 24, 6, 83, 84, - 69, 82, 73, 83, 201, 172, 5, 14, 76, 77, 79, 83, 84, 32, 69, 81, 85, 65, - 76, 32, 84, 79, 10, 22, 45, 167, 33, 32, 4, 180, 164, 12, 6, 71, 82, 65, - 86, 69, 45, 195, 193, 3, 77, 6, 244, 2, 4, 84, 73, 67, 76, 177, 180, 3, - 4, 78, 85, 73, 84, 12, 42, 82, 129, 215, 29, 4, 73, 78, 68, 85, 10, 48, - 3, 69, 86, 69, 197, 228, 29, 3, 73, 68, 71, 7, 162, 252, 11, 45, 183, - 218, 17, 32, 142, 1, 142, 1, 65, 34, 76, 98, 79, 116, 8, 89, 82, 73, 76, + 78, 136, 210, 9, 4, 85, 77, 65, 78, 145, 1, 4, 69, 65, 86, 89, 58, 110, + 67, 184, 188, 20, 5, 73, 69, 85, 78, 71, 42, 72, 30, 75, 66, 77, 34, 78, + 34, 80, 62, 82, 30, 83, 27, 84, 8, 150, 188, 20, 72, 157, 3, 4, 73, 69, + 85, 67, 116, 220, 1, 9, 68, 69, 79, 71, 82, 65, 80, 72, 32, 196, 8, 27, + 84, 65, 76, 73, 67, 32, 76, 65, 84, 73, 78, 32, 67, 65, 80, 73, 84, 65, + 76, 32, 76, 69, 84, 84, 69, 82, 32, 157, 138, 27, 9, 78, 70, 79, 82, 77, + 65, 84, 73, 79, 110, 174, 1, 65, 110, 67, 90, 69, 86, 70, 62, 72, 38, 75, + 70, 76, 50, 77, 86, 78, 62, 81, 30, 82, 74, 83, 162, 173, 6, 80, 242, + 143, 14, 84, 50, 87, 170, 214, 1, 73, 255, 151, 10, 79, 8, 158, 189, 20, + 76, 184, 133, 1, 3, 67, 67, 69, 204, 239, 6, 3, 84, 84, 69, 249, 17, 5, + 68, 86, 65, 78, 84, 8, 26, 79, 163, 200, 30, 69, 6, 240, 200, 10, 2, 82, + 82, 230, 243, 9, 78, 203, 251, 12, 80, 8, 234, 188, 20, 78, 222, 163, 9, + 65, 188, 165, 3, 5, 88, 67, 69, 76, 76, 231, 24, 73, 10, 246, 188, 20, + 73, 208, 184, 5, 2, 69, 77, 191, 218, 6, 79, 4, 210, 163, 29, 73, 247, + 167, 3, 65, 4, 212, 155, 7, 8, 73, 78, 68, 69, 82, 71, 65, 82, 231, 189, + 22, 79, 6, 134, 188, 20, 65, 142, 141, 12, 79, 195, 83, 69, 8, 38, 69, + 182, 162, 32, 65, 191, 84, 79, 4, 160, 173, 32, 3, 68, 73, 67, 251, 15, + 84, 6, 26, 73, 171, 174, 32, 65, 4, 218, 155, 33, 71, 231, 19, 78, 2, + 153, 179, 28, 2, 85, 69, 8, 26, 69, 251, 154, 33, 73, 6, 142, 187, 20, + 83, 221, 253, 7, 2, 76, 73, 22, 90, 69, 38, 85, 144, 148, 14, 2, 67, 72, + 214, 166, 6, 79, 22, 80, 34, 84, 215, 252, 11, 73, 4, 246, 152, 10, 67, + 215, 215, 21, 86, 6, 226, 187, 20, 80, 166, 255, 1, 73, 139, 137, 11, 78, + 4, 218, 195, 33, 67, 3, 82, 98, 116, 8, 65, 84, 65, 75, 65, 78, 65, 32, + 133, 1, 16, 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, 82, + 32, 94, 170, 229, 10, 72, 2, 75, 2, 77, 2, 78, 2, 82, 2, 83, 2, 84, 126, + 87, 46, 89, 246, 219, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 228, 138, + 16, 3, 74, 85, 69, 221, 151, 17, 4, 67, 72, 65, 77, 106, 158, 173, 27, + 65, 255, 129, 4, 69, 4, 100, 19, 85, 76, 84, 73, 80, 76, 73, 67, 65, 84, + 73, 79, 78, 32, 83, 73, 71, 78, 32, 243, 185, 31, 73, 2, 157, 84, 4, 87, + 73, 84, 72, 98, 50, 69, 46, 70, 98, 83, 94, 84, 191, 183, 20, 78, 6, 180, + 1, 3, 73, 71, 72, 199, 169, 25, 76, 30, 28, 3, 73, 70, 84, 35, 79, 6, + 214, 1, 89, 155, 233, 31, 69, 24, 142, 2, 82, 167, 183, 20, 85, 8, 40, 4, + 69, 86, 69, 78, 1, 2, 73, 88, 4, 11, 84, 4, 104, 2, 89, 32, 143, 233, 31, + 69, 52, 56, 2, 69, 78, 32, 4, 72, 73, 82, 84, 29, 2, 87, 69, 5, 11, 32, + 2, 247, 198, 25, 79, 24, 74, 89, 175, 232, 31, 69, 24, 26, 78, 215, 191, + 32, 76, 22, 17, 2, 84, 89, 23, 11, 32, 20, 72, 2, 79, 78, 154, 203, 31, + 70, 30, 83, 42, 84, 170, 86, 78, 235, 110, 69, 4, 210, 197, 25, 32, 235, + 245, 7, 69, 2, 129, 165, 31, 8, 32, 80, 79, 73, 78, 84, 69, 68, 8, 132, + 199, 17, 3, 79, 83, 84, 154, 246, 6, 65, 192, 204, 2, 8, 69, 82, 80, 69, + 78, 68, 73, 67, 139, 171, 4, 76, 4, 128, 250, 29, 3, 73, 78, 71, 171, + 170, 1, 69, 6, 52, 7, 82, 73, 65, 78, 71, 76, 69, 131, 175, 26, 73, 5, + 147, 147, 22, 32, 6, 44, 5, 72, 73, 84, 69, 32, 203, 184, 33, 90, 4, 142, + 217, 31, 66, 139, 25, 83, 6, 218, 160, 27, 77, 196, 178, 4, 6, 76, 65, + 84, 73, 79, 78, 225, 161, 1, 3, 83, 32, 84, 5, 225, 242, 31, 6, 32, 65, + 84, 32, 68, 85, 158, 18, 192, 1, 24, 67, 79, 77, 80, 65, 84, 73, 66, 73, + 76, 73, 84, 89, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 144, 3, 8, + 82, 65, 68, 73, 67, 65, 76, 32, 245, 17, 7, 83, 84, 82, 79, 75, 69, 32, + 236, 15, 24, 2, 50, 70, 75, 70, 188, 8, 34, 65, 162, 246, 10, 56, 3, 57, + 60, 202, 1, 49, 135, 236, 18, 48, 176, 7, 26, 65, 227, 245, 10, 57, 176, + 3, 134, 1, 54, 134, 236, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 139, 146, 9, 68, 28, 198, 179, 33, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, + 2, 66, 2, 67, 3, 68, 230, 1, 210, 1, 66, 126, 67, 226, 4, 68, 62, 69, 50, + 70, 34, 71, 50, 72, 86, 74, 154, 1, 76, 62, 77, 130, 1, 80, 46, 82, 78, + 83, 182, 3, 84, 82, 87, 168, 218, 10, 2, 78, 69, 144, 17, 4, 75, 78, 73, + 70, 251, 151, 12, 79, 14, 74, 79, 180, 250, 10, 4, 82, 85, 83, 72, 246, + 233, 4, 65, 251, 192, 13, 76, 6, 186, 161, 10, 76, 154, 248, 22, 78, 215, + 22, 88, 56, 104, 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, 68, 32, 162, + 3, 73, 50, 76, 158, 173, 32, 79, 191, 78, 72, 44, 114, 69, 38, 70, 50, + 71, 38, 76, 34, 83, 94, 84, 154, 199, 14, 87, 42, 68, 234, 226, 16, 66, + 222, 120, 72, 135, 2, 67, 4, 174, 218, 31, 86, 191, 183, 1, 65, 6, 210, + 197, 31, 73, 218, 214, 1, 76, 235, 16, 82, 4, 186, 143, 23, 79, 147, 215, + 8, 65, 4, 238, 171, 32, 69, 187, 48, 79, 10, 150, 219, 6, 73, 134, 186, + 17, 65, 198, 194, 1, 72, 248, 178, 7, 3, 80, 69, 69, 215, 11, 69, 6, 214, + 10, 85, 152, 176, 10, 2, 65, 78, 203, 158, 4, 79, 4, 140, 210, 20, 3, 86, + 73, 76, 151, 200, 12, 84, 4, 170, 214, 17, 73, 131, 193, 14, 79, 6, 216, + 148, 24, 2, 73, 86, 186, 241, 2, 69, 179, 164, 6, 79, 10, 196, 6, 2, 65, + 84, 194, 141, 33, 87, 3, 89, 4, 242, 137, 32, 73, 131, 80, 79, 8, 244, 5, + 4, 82, 65, 83, 83, 143, 186, 29, 72, 10, 48, 2, 69, 65, 146, 159, 28, 79, + 163, 252, 3, 65, 6, 186, 8, 82, 239, 160, 33, 68, 10, 72, 12, 45, 83, 73, + 77, 80, 76, 73, 70, 73, 69, 68, 32, 179, 152, 31, 65, 8, 42, 84, 174, + 195, 14, 68, 247, 244, 16, 69, 4, 194, 6, 85, 227, 206, 14, 79, 12, 248, + 241, 10, 3, 79, 78, 71, 197, 245, 15, 3, 65, 77, 69, 14, 18, 69, 35, 79, + 4, 242, 138, 33, 65, 159, 27, 83, 10, 136, 241, 10, 3, 85, 78, 68, 166, + 162, 13, 84, 162, 205, 7, 82, 211, 118, 79, 6, 188, 240, 10, 2, 65, 87, + 175, 182, 18, 69, 8, 34, 65, 169, 213, 28, 2, 69, 80, 6, 130, 214, 32, + 73, 226, 79, 77, 3, 80, 38, 122, 69, 90, 73, 186, 1, 78, 196, 1, 4, 80, + 73, 82, 73, 248, 234, 10, 4, 77, 65, 76, 76, 214, 149, 12, 72, 195, 208, + 9, 85, 8, 40, 4, 67, 79, 78, 68, 227, 154, 32, 65, 6, 11, 32, 6, 166, + 180, 31, 84, 183, 86, 79, 12, 60, 9, 77, 80, 76, 73, 70, 73, 69, 68, 32, + 147, 160, 33, 76, 10, 34, 72, 50, 87, 135, 182, 10, 89, 4, 140, 160, 10, + 3, 65, 76, 70, 223, 248, 17, 79, 4, 234, 208, 6, 65, 175, 129, 22, 72, 6, + 192, 1, 2, 79, 85, 151, 241, 32, 65, 8, 58, 85, 146, 217, 24, 65, 218, + 204, 1, 72, 147, 174, 3, 73, 2, 239, 134, 24, 82, 12, 26, 65, 49, 2, 69, + 83, 8, 164, 235, 10, 2, 76, 75, 1, 3, 84, 69, 82, 4, 247, 234, 10, 84, + 76, 110, 72, 182, 1, 80, 38, 83, 142, 163, 31, 84, 176, 250, 1, 2, 66, + 88, 2, 87, 2, 88, 86, 68, 2, 78, 3, 81, 31, 42, 80, 22, 88, 30, 90, 143, + 159, 33, 71, 5, 231, 158, 33, 87, 4, 210, 158, 33, 87, 87, 71, 19, 50, + 90, 194, 163, 31, 87, 154, 251, 1, 71, 3, 84, 9, 190, 163, 31, 90, 155, + 251, 1, 80, 9, 178, 158, 33, 68, 2, 71, 3, 90, 23, 50, 87, 30, 90, 194, + 157, 33, 71, 2, 80, 3, 84, 7, 218, 157, 33, 71, 3, 90, 9, 234, 156, 33, + 87, 86, 80, 3, 90, 124, 62, 65, 194, 1, 73, 114, 79, 181, 241, 20, 4, 69, + 65, 82, 32, 8, 132, 1, 2, 80, 80, 208, 231, 18, 6, 83, 83, 73, 67, 65, + 76, 169, 137, 13, 14, 77, 83, 72, 69, 76, 76, 32, 77, 79, 66, 73, 76, 69, + 32, 4, 196, 189, 19, 5, 73, 78, 71, 32, 72, 239, 227, 1, 69, 6, 48, 6, + 78, 75, 73, 78, 71, 32, 223, 131, 32, 80, 4, 190, 145, 25, 71, 181, 183, + 3, 6, 66, 69, 69, 82, 32, 77, 108, 72, 2, 67, 75, 140, 10, 2, 83, 69, + 236, 3, 2, 85, 68, 131, 153, 30, 87, 70, 64, 6, 32, 70, 65, 67, 69, 32, + 237, 2, 5, 87, 73, 83, 69, 32, 48, 58, 69, 46, 70, 36, 2, 78, 73, 2, 79, + 18, 83, 51, 84, 8, 120, 2, 76, 69, 125, 4, 73, 71, 72, 84, 8, 178, 1, 73, + 25, 3, 79, 85, 82, 4, 155, 1, 78, 8, 26, 69, 125, 2, 73, 88, 4, 57, 2, + 86, 69, 16, 38, 69, 14, 87, 41, 3, 72, 82, 69, 4, 63, 78, 8, 24, 2, 69, + 76, 27, 79, 4, 11, 86, 4, 11, 69, 4, 180, 252, 28, 2, 32, 79, 137, 173, + 2, 3, 45, 84, 72, 22, 90, 67, 58, 68, 122, 71, 48, 5, 82, 73, 71, 72, 84, + 226, 2, 84, 122, 79, 143, 137, 31, 73, 4, 196, 1, 3, 76, 79, 83, 141, 99, + 4, 79, 78, 84, 79, 2, 141, 3, 26, 79, 87, 78, 87, 65, 82, 68, 83, 32, 65, + 78, 68, 32, 85, 80, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 2, 21, 3, + 65, 80, 80, 2, 133, 4, 2, 69, 68, 6, 228, 1, 14, 32, 65, 78, 68, 32, 76, + 69, 70, 84, 32, 83, 69, 77, 73, 45, 38, 87, 65, 82, 68, 83, 32, 65, 78, + 68, 32, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 67, + 73, 82, 67, 76, 69, 32, 65, 82, 82, 79, 87, 83, 2, 185, 182, 30, 6, 67, + 73, 82, 67, 76, 69, 5, 161, 83, 15, 32, 87, 73, 84, 72, 32, 67, 73, 82, + 67, 76, 69, 68, 32, 79, 4, 82, 79, 37, 16, 82, 73, 65, 78, 71, 76, 69, + 45, 72, 69, 65, 68, 69, 68, 32, 79, 2, 69, 6, 80, 32, 83, 69, 77, 73, 2, + 21, 3, 80, 69, 78, 2, 11, 32, 2, 217, 180, 30, 4, 67, 73, 82, 67, 26, 32, + 2, 68, 32, 191, 153, 32, 32, 24, 216, 1, 2, 83, 85, 66, 85, 178, 175, 19, + 77, 180, 3, 9, 76, 79, 67, 75, 32, 87, 73, 84, 72, 222, 134, 12, 66, 249, + 149, 1, 23, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 79, 78, 32, 87, 73, + 84, 72, 32, 83, 69, 82, 73, 70, 8, 30, 66, 1, 3, 80, 69, 82, 4, 181, 252, + 24, 3, 83, 69, 84, 6, 82, 77, 33, 16, 78, 73, 79, 78, 32, 87, 73, 84, 72, + 32, 83, 69, 82, 73, 70, 83, 2, 145, 223, 24, 3, 66, 82, 69, 5, 237, 252, + 31, 9, 32, 65, 78, 68, 32, 83, 77, 65, 83, 11, 33, 6, 32, 87, 73, 84, 72, + 32, 8, 72, 4, 84, 79, 82, 78, 214, 167, 5, 76, 206, 220, 19, 83, 135, + 218, 6, 82, 2, 203, 217, 25, 65, 226, 9, 158, 1, 67, 82, 76, 98, 77, 162, + 82, 78, 220, 5, 2, 79, 75, 58, 80, 150, 19, 82, 138, 1, 85, 170, 253, 27, + 87, 196, 158, 3, 2, 70, 70, 246, 47, 73, 183, 51, 65, 6, 26, 75, 203, + 148, 4, 79, 4, 142, 233, 8, 82, 189, 136, 23, 4, 84, 65, 73, 76, 8, 44, + 2, 79, 78, 165, 134, 25, 3, 76, 73, 83, 7, 11, 32, 4, 138, 128, 26, 69, + 195, 129, 6, 83, 158, 6, 72, 7, 66, 73, 78, 73, 78, 71, 32, 166, 80, 77, + 94, 80, 151, 154, 32, 69, 142, 6, 214, 2, 65, 174, 2, 66, 134, 1, 67, + 246, 12, 68, 174, 9, 69, 146, 2, 70, 54, 71, 140, 9, 2, 72, 79, 82, 73, + 216, 1, 2, 75, 65, 162, 1, 76, 230, 13, 77, 250, 1, 78, 78, 79, 138, 2, + 80, 174, 1, 82, 244, 4, 5, 90, 73, 71, 90, 65, 166, 2, 83, 242, 3, 84, + 238, 2, 85, 148, 1, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 36, 2, 87, 73, + 183, 1, 88, 24, 148, 1, 4, 67, 85, 84, 69, 86, 78, 252, 131, 25, 6, 83, + 84, 69, 82, 73, 83, 197, 224, 5, 14, 76, 77, 79, 83, 84, 32, 69, 81, 85, + 65, 76, 32, 84, 79, 10, 22, 45, 167, 33, 32, 4, 200, 204, 12, 6, 71, 82, + 65, 86, 69, 45, 199, 205, 3, 77, 6, 244, 2, 4, 84, 73, 67, 76, 189, 179, + 3, 4, 78, 85, 73, 84, 12, 42, 82, 201, 212, 30, 4, 73, 78, 68, 85, 10, + 48, 3, 69, 86, 69, 149, 227, 30, 3, 73, 68, 71, 7, 166, 164, 12, 45, 251, + 175, 18, 32, 142, 1, 142, 1, 65, 34, 76, 98, 79, 116, 8, 89, 82, 73, 76, 76, 73, 67, 32, 184, 28, 11, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, - 175, 238, 11, 69, 6, 162, 18, 82, 199, 131, 28, 78, 4, 41, 8, 79, 67, 75, - 87, 73, 83, 69, 32, 4, 184, 131, 12, 4, 82, 73, 78, 71, 239, 160, 16, 65, + 195, 150, 12, 69, 6, 162, 18, 82, 215, 217, 28, 78, 4, 41, 8, 79, 67, 75, + 87, 73, 83, 69, 32, 4, 204, 171, 12, 4, 82, 73, 78, 71, 231, 218, 16, 65, 10, 76, 4, 77, 77, 65, 32, 189, 16, 10, 78, 74, 79, 73, 78, 73, 78, 71, - 32, 77, 6, 250, 229, 12, 65, 247, 251, 16, 66, 116, 252, 1, 8, 72, 85, + 32, 77, 6, 210, 147, 13, 65, 239, 204, 17, 66, 116, 252, 1, 8, 72, 85, 78, 68, 82, 69, 68, 32, 32, 7, 76, 69, 84, 84, 69, 82, 32, 166, 5, 80, - 116, 5, 68, 65, 83, 73, 65, 38, 84, 106, 77, 206, 243, 2, 75, 132, 214, - 12, 15, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 66, 89, 217, - 241, 15, 2, 86, 90, 4, 194, 7, 77, 215, 158, 3, 84, 84, 238, 1, 66, 38, - 68, 50, 69, 82, 73, 106, 79, 22, 83, 66, 85, 30, 89, 214, 137, 3, 76, - 140, 5, 5, 77, 79, 78, 79, 71, 142, 18, 72, 202, 174, 12, 84, 190, 163, - 15, 90, 254, 82, 67, 2, 71, 182, 8, 70, 134, 14, 80, 2, 86, 158, 20, 75, - 187, 2, 65, 4, 174, 147, 6, 73, 199, 226, 25, 69, 4, 168, 155, 25, 3, 74, - 69, 82, 167, 218, 6, 69, 14, 58, 83, 230, 244, 31, 70, 2, 76, 2, 77, 2, - 78, 3, 82, 5, 183, 176, 30, 45, 11, 56, 8, 79, 84, 73, 70, 73, 69, 68, - 32, 151, 244, 31, 69, 6, 194, 145, 6, 66, 210, 226, 25, 65, 3, 69, 5, - 239, 243, 24, 77, 6, 26, 72, 155, 209, 15, 79, 4, 250, 251, 29, 67, 191, - 247, 1, 65, 5, 193, 153, 3, 2, 75, 82, 8, 142, 230, 27, 69, 190, 240, 3, - 65, 174, 28, 73, 3, 85, 8, 66, 65, 48, 4, 83, 73, 76, 73, 133, 212, 28, - 4, 79, 75, 82, 89, 4, 170, 246, 2, 89, 241, 175, 12, 3, 76, 65, 84, 2, + 116, 5, 68, 65, 83, 73, 65, 38, 84, 106, 77, 178, 243, 2, 75, 176, 138, + 13, 15, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 66, 89, 229, + 193, 16, 2, 86, 90, 4, 194, 7, 77, 195, 158, 3, 84, 84, 238, 1, 66, 38, + 68, 50, 69, 82, 73, 106, 79, 22, 83, 66, 85, 30, 89, 186, 137, 3, 76, + 140, 5, 5, 77, 79, 78, 79, 71, 150, 18, 72, 238, 226, 12, 84, 190, 243, + 15, 90, 150, 83, 67, 2, 71, 182, 8, 70, 134, 14, 80, 2, 86, 158, 20, 75, + 187, 2, 65, 4, 250, 161, 6, 73, 163, 216, 26, 69, 4, 248, 233, 25, 3, 74, + 69, 82, 255, 143, 7, 69, 14, 58, 83, 142, 249, 32, 70, 2, 76, 2, 77, 2, + 78, 3, 82, 5, 147, 178, 31, 45, 11, 56, 8, 79, 84, 73, 70, 73, 69, 68, + 32, 191, 248, 32, 69, 6, 142, 160, 6, 66, 174, 216, 26, 65, 3, 69, 5, + 159, 193, 25, 77, 6, 26, 72, 171, 133, 16, 79, 4, 158, 253, 30, 67, 195, + 250, 1, 65, 5, 173, 153, 3, 2, 75, 82, 8, 254, 189, 28, 69, 246, 156, 4, + 65, 174, 28, 73, 3, 85, 8, 66, 65, 48, 4, 83, 73, 76, 73, 189, 195, 29, + 4, 79, 75, 82, 89, 4, 142, 246, 2, 89, 133, 228, 12, 3, 76, 65, 84, 2, 145, 13, 5, 32, 80, 78, 69, 85, 10, 80, 2, 69, 78, 0, 7, 72, 79, 85, 83, - 65, 78, 68, 177, 8, 4, 73, 84, 76, 79, 2, 17, 2, 32, 77, 2, 169, 164, 21, + 65, 78, 68, 177, 8, 4, 73, 84, 76, 79, 2, 17, 2, 32, 77, 2, 129, 235, 21, 5, 73, 76, 76, 73, 79, 110, 62, 69, 244, 1, 8, 73, 65, 69, 82, 69, 83, - 73, 83, 39, 79, 38, 68, 9, 86, 65, 78, 65, 71, 65, 82, 73, 32, 249, 133, - 31, 2, 76, 69, 36, 92, 7, 76, 69, 84, 84, 69, 82, 32, 240, 179, 25, 6, - 83, 73, 71, 78, 32, 65, 167, 204, 4, 68, 14, 214, 218, 31, 86, 162, 17, - 75, 2, 78, 2, 80, 2, 82, 186, 2, 65, 3, 85, 7, 246, 140, 27, 45, 151, - 186, 2, 32, 66, 58, 84, 128, 1, 4, 85, 66, 76, 69, 241, 4, 2, 87, 78, 14, - 38, 32, 149, 215, 15, 3, 84, 69, 68, 10, 64, 5, 65, 66, 79, 86, 69, 241, - 165, 6, 5, 66, 69, 76, 79, 87, 7, 151, 193, 31, 32, 48, 22, 32, 191, 4, + 73, 83, 39, 79, 38, 68, 9, 86, 65, 78, 65, 71, 65, 82, 73, 32, 145, 138, + 32, 2, 76, 69, 36, 92, 7, 76, 69, 84, 84, 69, 82, 32, 216, 131, 26, 6, + 83, 73, 71, 78, 32, 65, 227, 253, 4, 68, 14, 254, 222, 32, 86, 162, 17, + 75, 2, 78, 2, 80, 2, 82, 186, 2, 65, 3, 85, 7, 146, 229, 27, 45, 195, + 223, 2, 32, 66, 58, 84, 128, 1, 4, 85, 66, 76, 69, 241, 4, 2, 87, 78, 14, + 38, 32, 173, 139, 16, 3, 84, 69, 68, 10, 64, 5, 65, 66, 79, 86, 69, 197, + 179, 6, 5, 66, 69, 76, 79, 87, 7, 191, 197, 32, 32, 48, 22, 32, 191, 4, 68, 46, 230, 1, 66, 0, 10, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 26, 77, 54, 79, 34, 80, 68, 2, 82, 73, 48, 5, 84, 73, 76, 68, 69, 80, 9, 86, - 69, 82, 84, 73, 67, 65, 76, 32, 174, 210, 15, 65, 42, 71, 166, 179, 4, - 76, 187, 228, 10, 67, 4, 229, 16, 2, 82, 69, 4, 21, 3, 65, 67, 82, 4, - 181, 193, 16, 2, 79, 78, 4, 230, 35, 80, 255, 224, 29, 86, 6, 238, 36, - 76, 205, 206, 7, 9, 65, 82, 69, 78, 84, 72, 69, 83, 69, 4, 156, 51, 4, - 71, 72, 84, 87, 139, 228, 9, 78, 7, 11, 32, 4, 44, 3, 76, 69, 70, 1, 4, - 82, 73, 71, 72, 2, 199, 205, 29, 84, 6, 178, 239, 11, 83, 211, 255, 13, - 76, 2, 141, 205, 15, 2, 32, 67, 4, 128, 41, 2, 32, 84, 227, 158, 30, 87, - 16, 76, 9, 78, 67, 76, 79, 83, 73, 78, 71, 32, 129, 193, 25, 4, 81, 85, - 65, 76, 14, 132, 1, 6, 67, 73, 82, 67, 76, 69, 22, 83, 216, 145, 6, 3, - 75, 69, 89, 160, 241, 1, 7, 85, 80, 87, 65, 82, 68, 32, 227, 166, 22, 68, - 5, 219, 254, 17, 32, 4, 134, 138, 25, 67, 195, 186, 5, 81, 4, 32, 2, 69, - 82, 135, 137, 5, 79, 2, 143, 74, 77, 126, 88, 17, 76, 65, 71, 79, 76, 73, + 69, 82, 84, 73, 67, 65, 76, 32, 198, 134, 16, 65, 42, 71, 150, 187, 4, + 76, 203, 172, 11, 67, 4, 229, 16, 2, 82, 69, 4, 21, 3, 65, 67, 82, 4, + 229, 245, 16, 2, 79, 78, 4, 230, 35, 80, 163, 226, 30, 86, 6, 238, 36, + 76, 253, 224, 7, 9, 65, 82, 69, 78, 84, 72, 69, 83, 69, 4, 156, 51, 4, + 71, 72, 84, 87, 207, 248, 9, 78, 7, 11, 32, 4, 44, 3, 76, 69, 70, 1, 4, + 82, 73, 71, 72, 2, 199, 203, 30, 84, 6, 198, 151, 12, 83, 151, 177, 14, + 76, 2, 165, 129, 16, 2, 32, 67, 4, 128, 41, 2, 32, 84, 243, 162, 31, 87, + 16, 76, 9, 78, 67, 76, 79, 83, 73, 78, 71, 32, 205, 152, 26, 4, 81, 85, + 65, 76, 14, 132, 1, 6, 67, 73, 82, 67, 76, 69, 22, 83, 164, 160, 6, 3, + 75, 69, 89, 228, 203, 19, 7, 85, 80, 87, 65, 82, 68, 32, 191, 192, 5, 68, + 5, 175, 180, 18, 32, 4, 214, 216, 25, 67, 131, 240, 5, 81, 4, 32, 2, 69, + 82, 159, 142, 5, 79, 2, 139, 74, 77, 126, 88, 17, 76, 65, 71, 79, 76, 73, 84, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 143, 3, 82, 76, 238, 1, 68, - 30, 73, 46, 83, 50, 84, 182, 250, 5, 65, 22, 66, 94, 67, 134, 1, 70, 38, + 30, 73, 46, 83, 50, 84, 130, 137, 6, 65, 22, 66, 94, 67, 134, 1, 70, 38, 71, 238, 2, 77, 32, 2, 76, 74, 34, 78, 88, 2, 80, 79, 30, 82, 194, 2, 86, - 22, 89, 90, 90, 182, 199, 18, 72, 142, 243, 1, 85, 154, 253, 4, 79, 235, - 5, 75, 4, 238, 252, 5, 74, 31, 79, 11, 250, 253, 5, 78, 54, 79, 135, 182, - 25, 90, 8, 218, 254, 5, 77, 130, 4, 76, 243, 204, 20, 72, 4, 142, 131, 6, - 86, 255, 201, 25, 83, 50, 38, 65, 173, 3, 4, 69, 69, 75, 32, 36, 84, 5, - 78, 84, 72, 65, 32, 196, 1, 2, 86, 69, 153, 228, 27, 5, 80, 72, 69, 77, + 22, 89, 90, 90, 226, 132, 19, 72, 174, 128, 2, 85, 170, 168, 5, 79, 235, + 5, 75, 4, 186, 139, 6, 74, 31, 79, 11, 198, 140, 6, 78, 54, 79, 227, 171, + 26, 90, 8, 166, 141, 6, 77, 130, 4, 76, 211, 200, 22, 72, 4, 218, 145, 6, + 86, 219, 191, 26, 83, 50, 38, 65, 173, 3, 4, 69, 69, 75, 32, 36, 84, 5, + 78, 84, 72, 65, 32, 196, 1, 2, 86, 69, 133, 188, 28, 5, 80, 72, 69, 77, 69, 24, 68, 6, 68, 73, 71, 73, 84, 32, 65, 7, 76, 69, 84, 84, 69, 82, 32, - 14, 202, 177, 16, 83, 142, 192, 13, 70, 70, 84, 62, 90, 143, 83, 79, 10, - 222, 202, 31, 86, 162, 17, 75, 2, 78, 2, 80, 187, 2, 65, 10, 18, 32, 67, - 45, 6, 26, 65, 215, 242, 15, 84, 4, 245, 143, 4, 4, 67, 67, 69, 78, 4, - 208, 220, 11, 6, 65, 67, 85, 84, 69, 45, 195, 231, 3, 77, 14, 148, 1, 8, - 77, 85, 83, 73, 67, 65, 76, 32, 150, 151, 4, 75, 250, 182, 2, 80, 174, 4, - 89, 249, 227, 3, 11, 68, 73, 65, 76, 89, 84, 73, 75, 65, 32, 84, 6, 26, - 84, 131, 135, 15, 80, 4, 218, 135, 15, 69, 35, 82, 6, 206, 200, 12, 79, - 216, 149, 18, 8, 77, 79, 84, 72, 69, 84, 73, 67, 163, 45, 82, 14, 26, 78, - 207, 179, 29, 83, 12, 52, 7, 86, 69, 82, 84, 69, 68, 32, 143, 222, 27, + 14, 250, 229, 16, 83, 130, 141, 14, 70, 70, 84, 62, 90, 251, 85, 79, 10, + 134, 207, 32, 86, 162, 17, 75, 2, 78, 2, 80, 187, 2, 65, 10, 18, 32, 67, + 45, 6, 26, 65, 231, 166, 16, 84, 4, 209, 144, 4, 4, 67, 67, 69, 78, 4, + 212, 132, 12, 6, 65, 67, 85, 84, 69, 45, 215, 243, 3, 77, 14, 148, 1, 8, + 77, 85, 83, 73, 67, 65, 76, 32, 242, 151, 4, 75, 242, 195, 2, 80, 174, 4, + 89, 225, 238, 3, 11, 68, 73, 65, 76, 89, 84, 73, 75, 65, 32, 84, 6, 26, + 84, 247, 186, 15, 80, 4, 206, 187, 15, 69, 35, 82, 6, 166, 246, 12, 79, + 152, 236, 18, 8, 77, 79, 84, 72, 69, 84, 73, 67, 167, 45, 82, 14, 26, 78, + 151, 177, 30, 83, 12, 52, 7, 86, 69, 82, 84, 69, 68, 32, 251, 181, 28, 70, 10, 64, 2, 66, 82, 45, 10, 68, 79, 85, 66, 76, 69, 32, 65, 82, 67, 6, - 22, 69, 235, 37, 73, 4, 211, 181, 11, 86, 4, 167, 192, 29, 72, 8, 128, 1, - 16, 84, 65, 75, 65, 78, 65, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 153, - 180, 17, 9, 86, 89, 75, 65, 32, 65, 66, 79, 86, 4, 146, 237, 9, 83, 35, + 22, 69, 235, 37, 73, 4, 219, 221, 11, 86, 4, 247, 190, 30, 72, 8, 128, 1, + 16, 84, 65, 75, 65, 78, 65, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 229, + 233, 17, 9, 86, 89, 75, 65, 32, 65, 66, 79, 86, 4, 222, 129, 10, 83, 35, 86, 158, 1, 80, 5, 65, 84, 73, 78, 32, 156, 8, 3, 69, 70, 84, 184, 3, 2, 73, 71, 83, 79, 106, 156, 1, 21, 76, 69, 84, 84, 69, 82, 32, 83, 77, 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, 53, 13, 83, 77, 65, 76, 76, - 32, 76, 69, 84, 84, 69, 82, 32, 10, 218, 213, 31, 71, 2, 76, 2, 77, 2, + 32, 76, 69, 84, 84, 69, 82, 32, 10, 130, 218, 32, 71, 2, 76, 2, 77, 2, 78, 3, 82, 96, 226, 1, 65, 70, 67, 34, 69, 30, 70, 78, 73, 86, 76, 106, - 79, 2, 85, 134, 1, 82, 50, 84, 206, 131, 12, 83, 194, 134, 2, 66, 226, - 156, 2, 87, 158, 168, 15, 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 78, 2, 80, - 2, 86, 2, 88, 3, 90, 13, 142, 244, 2, 32, 146, 235, 12, 76, 166, 244, 15, - 69, 2, 79, 3, 86, 5, 221, 227, 11, 3, 32, 67, 69, 7, 206, 209, 31, 83, 3, - 84, 5, 185, 219, 26, 14, 76, 65, 84, 84, 69, 78, 69, 68, 32, 79, 80, 69, - 78, 32, 11, 37, 7, 78, 83, 85, 76, 65, 82, 32, 8, 206, 209, 31, 68, 2, - 71, 2, 82, 3, 84, 7, 136, 210, 15, 14, 32, 87, 73, 84, 72, 32, 68, 79, - 85, 66, 76, 69, 32, 77, 165, 172, 12, 3, 79, 78, 71, 7, 33, 6, 32, 87, - 73, 84, 72, 32, 4, 194, 243, 11, 68, 201, 187, 12, 15, 76, 73, 71, 72, - 84, 32, 67, 69, 78, 84, 82, 65, 76, 73, 90, 7, 11, 32, 4, 146, 210, 11, - 82, 227, 228, 17, 66, 5, 129, 26, 6, 85, 82, 78, 69, 68, 32, 32, 46, 32, + 79, 2, 85, 134, 1, 82, 50, 84, 226, 171, 12, 83, 182, 146, 2, 66, 138, + 157, 2, 87, 150, 248, 15, 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 78, 2, 80, + 2, 86, 2, 88, 3, 90, 13, 242, 243, 2, 32, 198, 139, 26, 76, 182, 216, 3, + 69, 2, 79, 3, 86, 5, 241, 139, 12, 3, 32, 67, 69, 7, 246, 213, 32, 83, 3, + 84, 5, 249, 179, 27, 14, 76, 65, 84, 84, 69, 78, 69, 68, 32, 79, 80, 69, + 78, 32, 11, 37, 7, 78, 83, 85, 76, 65, 82, 32, 8, 246, 213, 32, 68, 2, + 71, 2, 82, 3, 84, 7, 164, 134, 16, 14, 32, 87, 73, 84, 72, 32, 68, 79, + 85, 66, 76, 69, 32, 77, 149, 218, 12, 3, 79, 78, 71, 7, 33, 6, 32, 87, + 73, 84, 72, 32, 4, 214, 155, 12, 68, 225, 222, 12, 15, 76, 73, 71, 72, + 84, 32, 67, 69, 78, 84, 82, 65, 76, 73, 90, 7, 11, 32, 4, 150, 250, 11, + 82, 175, 187, 18, 66, 5, 129, 26, 6, 85, 82, 78, 69, 68, 32, 32, 46, 32, 213, 2, 6, 87, 65, 82, 68, 83, 32, 28, 154, 1, 65, 112, 12, 80, 65, 82, - 69, 78, 84, 72, 69, 83, 73, 83, 32, 226, 12, 72, 182, 1, 84, 177, 165, - 29, 11, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 12, 48, 4, 82, 82, - 79, 87, 233, 179, 29, 2, 78, 71, 8, 26, 72, 243, 179, 29, 32, 4, 225, - 179, 29, 3, 69, 65, 68, 4, 244, 204, 7, 4, 65, 66, 79, 86, 245, 204, 23, - 5, 66, 69, 76, 79, 87, 4, 230, 13, 72, 133, 198, 11, 5, 65, 82, 82, 79, + 69, 78, 84, 72, 69, 83, 73, 83, 32, 226, 12, 72, 182, 1, 84, 129, 164, + 30, 11, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 12, 48, 4, 82, 82, + 79, 87, 185, 178, 30, 2, 78, 71, 8, 26, 72, 195, 178, 30, 32, 4, 177, + 178, 30, 3, 69, 65, 68, 4, 164, 223, 7, 4, 65, 66, 79, 86, 225, 190, 24, + 5, 66, 69, 76, 79, 87, 4, 230, 13, 72, 153, 238, 11, 5, 65, 82, 82, 79, 87, 10, 52, 6, 65, 84, 85, 82, 69, 32, 169, 17, 2, 72, 84, 8, 238, 1, 76, - 23, 82, 10, 36, 3, 78, 71, 32, 251, 176, 30, 87, 8, 172, 7, 7, 68, 79, + 23, 82, 10, 36, 3, 78, 71, 32, 139, 181, 31, 87, 8, 172, 7, 7, 68, 79, 85, 66, 76, 69, 32, 246, 7, 83, 71, 86, 20, 52, 5, 65, 67, 82, 79, 78, - 145, 164, 25, 2, 73, 78, 19, 18, 32, 127, 45, 10, 34, 76, 22, 82, 183, - 176, 29, 66, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 181, 160, 16, 6, - 84, 32, 72, 65, 76, 70, 6, 166, 246, 11, 71, 186, 2, 65, 139, 157, 3, 66, - 4, 152, 9, 9, 85, 77, 66, 69, 82, 32, 83, 73, 71, 177, 173, 11, 2, 79, + 221, 251, 25, 2, 73, 78, 19, 18, 32, 127, 45, 10, 34, 76, 22, 82, 135, + 175, 30, 66, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 229, 212, 16, 6, + 84, 32, 72, 65, 76, 70, 6, 186, 158, 12, 71, 186, 2, 65, 135, 169, 3, 66, + 4, 152, 9, 9, 85, 77, 66, 69, 82, 32, 83, 73, 71, 181, 213, 11, 2, 79, 84, 18, 136, 1, 17, 76, 68, 32, 80, 69, 82, 77, 73, 67, 32, 76, 69, 84, - 84, 69, 82, 32, 82, 80, 248, 243, 11, 4, 71, 79, 78, 69, 135, 237, 17, - 86, 10, 246, 43, 90, 206, 218, 17, 78, 186, 253, 8, 68, 138, 134, 2, 83, - 203, 236, 1, 65, 2, 213, 128, 24, 6, 69, 78, 32, 77, 65, 82, 12, 18, 65, - 107, 76, 8, 236, 7, 9, 82, 69, 78, 84, 72, 69, 83, 69, 83, 141, 248, 23, - 9, 76, 65, 84, 65, 76, 73, 90, 69, 68, 4, 197, 171, 29, 7, 85, 83, 32, - 83, 73, 71, 78, 38, 18, 69, 127, 73, 6, 72, 5, 86, 69, 82, 83, 69, 133, - 254, 23, 7, 84, 82, 79, 70, 76, 69, 88, 4, 22, 32, 227, 12, 68, 2, 141, + 84, 69, 82, 32, 82, 80, 140, 156, 12, 4, 71, 79, 78, 69, 151, 198, 18, + 86, 10, 242, 43, 90, 202, 148, 18, 78, 222, 155, 9, 68, 182, 171, 2, 83, + 159, 243, 1, 65, 2, 169, 203, 24, 6, 69, 78, 32, 77, 65, 82, 12, 18, 65, + 107, 76, 8, 236, 7, 9, 82, 69, 78, 84, 72, 69, 83, 69, 83, 225, 194, 24, + 9, 76, 65, 84, 65, 76, 73, 90, 69, 68, 4, 149, 170, 30, 7, 85, 83, 32, + 83, 73, 71, 78, 38, 18, 69, 127, 73, 6, 72, 5, 86, 69, 82, 83, 69, 217, + 200, 24, 7, 84, 82, 79, 70, 76, 69, 88, 4, 22, 32, 227, 12, 68, 2, 141, 8, 2, 83, 79, 32, 40, 3, 71, 72, 84, 157, 5, 2, 78, 71, 26, 50, 32, 149, 4, 7, 87, 65, 82, 68, 83, 32, 72, 24, 104, 5, 65, 82, 82, 79, 87, 218, 1, 72, 124, 12, 80, 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 32, 59, 84, 12, - 44, 5, 72, 69, 65, 68, 32, 155, 168, 29, 32, 8, 26, 65, 155, 168, 29, 66, - 6, 36, 3, 78, 68, 32, 219, 195, 30, 66, 4, 40, 4, 68, 79, 87, 78, 1, 2, - 85, 80, 2, 157, 203, 8, 9, 32, 65, 82, 82, 79, 87, 72, 69, 65, 6, 11, 65, - 6, 48, 6, 76, 70, 32, 82, 73, 78, 21, 2, 82, 80, 4, 163, 166, 29, 71, 2, - 17, 2, 79, 79, 2, 231, 193, 30, 78, 4, 174, 198, 9, 65, 137, 170, 20, 5, - 66, 69, 76, 79, 87, 2, 149, 249, 23, 2, 65, 67, 2, 233, 201, 20, 16, 65, + 44, 5, 72, 69, 65, 68, 32, 235, 166, 30, 32, 8, 26, 65, 235, 166, 30, 66, + 6, 36, 3, 78, 68, 32, 243, 199, 31, 66, 4, 40, 4, 68, 79, 87, 78, 1, 2, + 85, 80, 2, 133, 223, 8, 9, 32, 65, 82, 82, 79, 87, 72, 69, 65, 6, 11, 65, + 6, 48, 6, 76, 70, 32, 82, 73, 78, 21, 2, 82, 80, 4, 243, 164, 30, 71, 2, + 17, 2, 79, 79, 2, 255, 197, 31, 78, 4, 250, 218, 9, 65, 241, 150, 21, 5, + 66, 69, 76, 79, 87, 2, 233, 195, 24, 2, 65, 67, 2, 141, 144, 21, 16, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 6, 11, 32, 6, - 182, 197, 11, 79, 166, 223, 17, 66, 223, 155, 1, 65, 18, 188, 1, 5, 72, + 202, 237, 11, 79, 226, 181, 18, 66, 167, 161, 1, 65, 18, 188, 1, 5, 72, 79, 82, 84, 32, 160, 1, 7, 81, 85, 65, 82, 69, 32, 66, 56, 5, 84, 82, 79, - 78, 71, 232, 193, 3, 2, 85, 83, 148, 179, 20, 2, 78, 65, 233, 157, 5, 6, - 69, 65, 71, 85, 76, 76, 6, 18, 83, 71, 86, 4, 26, 79, 223, 194, 11, 84, - 2, 253, 194, 11, 5, 76, 73, 68, 85, 83, 2, 49, 10, 69, 82, 84, 73, 67, - 65, 76, 32, 76, 73, 2, 151, 194, 11, 78, 4, 148, 196, 7, 5, 82, 65, 67, - 75, 69, 191, 221, 21, 69, 2, 225, 244, 23, 17, 32, 67, 69, 78, 84, 82, + 78, 71, 168, 193, 3, 2, 85, 83, 168, 254, 20, 2, 78, 65, 221, 208, 5, 6, + 69, 65, 71, 85, 76, 76, 6, 18, 83, 71, 86, 4, 26, 79, 243, 234, 11, 84, + 2, 145, 235, 11, 5, 76, 73, 68, 85, 83, 2, 49, 10, 69, 82, 84, 73, 67, + 65, 76, 32, 76, 73, 2, 171, 234, 11, 78, 4, 196, 214, 7, 5, 82, 65, 67, + 75, 69, 223, 201, 22, 69, 2, 181, 191, 24, 17, 32, 67, 69, 78, 84, 82, 65, 76, 73, 90, 65, 84, 73, 79, 78, 32, 83, 20, 98, 72, 32, 4, 73, 76, 68, 69, 136, 1, 6, 82, 73, 80, 76, 69, 32, 69, 5, 85, 82, 78, 69, 68, 2, - 137, 194, 7, 3, 82, 69, 69, 11, 11, 32, 8, 76, 3, 76, 69, 70, 0, 4, 82, - 73, 71, 72, 182, 191, 11, 79, 167, 223, 17, 66, 2, 169, 144, 29, 6, 84, - 32, 72, 65, 76, 70, 6, 186, 161, 15, 65, 184, 132, 15, 5, 85, 78, 68, 69, - 82, 199, 64, 68, 2, 145, 252, 14, 2, 32, 67, 10, 34, 80, 238, 184, 30, - 82, 3, 83, 6, 42, 87, 141, 240, 23, 4, 32, 84, 65, 67, 2, 45, 9, 65, 82, - 68, 83, 32, 65, 82, 82, 79, 2, 159, 142, 29, 87, 6, 158, 188, 25, 76, - 171, 183, 2, 84, 6, 52, 3, 68, 69, 32, 133, 189, 11, 4, 71, 71, 76, 89, - 4, 92, 12, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 82, 73, 161, 165, 28, - 5, 66, 82, 73, 68, 71, 2, 241, 238, 23, 2, 68, 71, 6, 178, 196, 11, 45, - 147, 214, 17, 32, 6, 52, 7, 69, 82, 67, 73, 65, 76, 32, 191, 178, 31, 65, - 4, 206, 152, 28, 77, 195, 253, 2, 65, 8, 178, 140, 8, 82, 144, 221, 15, - 3, 79, 83, 73, 246, 225, 2, 76, 171, 208, 3, 65, 38, 214, 1, 70, 84, 10, - 83, 84, 82, 85, 67, 84, 73, 79, 78, 32, 46, 84, 172, 236, 2, 8, 86, 69, - 78, 73, 69, 78, 67, 69, 172, 237, 17, 6, 73, 67, 65, 76, 32, 84, 170, - 243, 8, 74, 133, 97, 7, 71, 82, 85, 69, 78, 84, 32, 6, 168, 187, 26, 4, - 69, 84, 84, 73, 162, 214, 1, 85, 229, 141, 2, 4, 79, 85, 78, 68, 4, 240, - 144, 20, 2, 87, 79, 219, 151, 10, 83, 20, 80, 5, 65, 73, 78, 83, 32, 198, - 1, 79, 28, 4, 82, 79, 76, 32, 199, 146, 3, 73, 12, 48, 3, 65, 83, 32, 93, - 5, 87, 73, 84, 72, 32, 6, 242, 205, 23, 77, 253, 7, 15, 78, 79, 82, 77, - 65, 76, 32, 83, 85, 66, 71, 82, 79, 85, 80, 6, 162, 190, 4, 76, 218, 144, - 19, 86, 223, 196, 4, 79, 2, 193, 168, 29, 2, 85, 82, 4, 232, 176, 19, 3, - 75, 78, 79, 173, 242, 3, 8, 83, 69, 81, 85, 69, 78, 67, 69, 6, 26, 73, - 175, 158, 2, 69, 4, 250, 170, 31, 78, 87, 69, 206, 2, 36, 4, 84, 73, 67, - 32, 175, 18, 89, 202, 2, 186, 1, 67, 204, 1, 6, 69, 80, 65, 67, 84, 32, - 86, 70, 36, 11, 79, 76, 68, 32, 78, 85, 66, 73, 65, 78, 32, 110, 83, 129, - 133, 28, 13, 77, 79, 82, 80, 72, 79, 76, 79, 71, 73, 67, 65, 76, 126, 76, + 185, 212, 7, 3, 82, 69, 69, 11, 11, 32, 8, 76, 3, 76, 69, 70, 0, 4, 82, + 73, 71, 72, 202, 231, 11, 79, 227, 181, 18, 66, 2, 241, 141, 30, 6, 84, + 32, 72, 65, 76, 70, 6, 210, 213, 15, 65, 176, 212, 15, 5, 85, 78, 68, 69, + 82, 211, 64, 68, 2, 161, 176, 15, 2, 32, 67, 10, 34, 80, 134, 189, 31, + 82, 3, 83, 6, 42, 87, 225, 186, 24, 4, 32, 84, 65, 67, 2, 45, 9, 65, 82, + 68, 83, 32, 65, 82, 82, 79, 2, 231, 139, 30, 87, 6, 246, 149, 26, 76, + 199, 191, 2, 84, 6, 52, 3, 68, 69, 32, 153, 229, 11, 4, 71, 71, 76, 89, + 4, 92, 12, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 82, 73, 169, 148, 29, + 5, 66, 82, 73, 68, 71, 2, 197, 185, 24, 2, 68, 71, 6, 198, 236, 11, 45, + 207, 172, 18, 32, 6, 52, 7, 69, 82, 67, 73, 65, 76, 32, 231, 182, 32, 65, + 4, 134, 136, 29, 77, 179, 146, 3, 65, 8, 138, 159, 8, 82, 140, 149, 16, + 3, 79, 83, 73, 190, 239, 2, 76, 159, 252, 3, 65, 38, 214, 1, 70, 84, 10, + 83, 84, 82, 85, 67, 84, 73, 79, 78, 32, 46, 84, 252, 235, 2, 8, 86, 69, + 78, 73, 69, 78, 67, 69, 180, 180, 18, 6, 73, 67, 65, 76, 32, 84, 246, + 173, 9, 74, 241, 99, 7, 71, 82, 85, 69, 78, 84, 32, 6, 228, 147, 27, 4, + 69, 84, 84, 73, 158, 237, 1, 85, 189, 162, 2, 4, 79, 85, 78, 68, 4, 164, + 206, 20, 2, 87, 79, 183, 222, 10, 83, 20, 80, 5, 65, 73, 78, 83, 32, 198, + 1, 79, 28, 4, 82, 79, 76, 32, 143, 146, 3, 73, 12, 48, 3, 65, 83, 32, 93, + 5, 87, 73, 84, 72, 32, 6, 222, 152, 24, 77, 181, 8, 15, 78, 79, 82, 77, + 65, 76, 32, 83, 85, 66, 71, 82, 79, 85, 80, 6, 186, 195, 4, 76, 174, 214, + 19, 86, 171, 233, 4, 79, 2, 217, 169, 30, 2, 85, 82, 4, 188, 236, 19, 3, + 75, 78, 79, 165, 129, 4, 8, 83, 69, 81, 85, 69, 78, 67, 69, 6, 26, 73, + 143, 158, 2, 69, 4, 162, 175, 32, 78, 87, 69, 206, 2, 36, 4, 84, 73, 67, + 32, 171, 18, 89, 202, 2, 186, 1, 67, 204, 1, 6, 69, 80, 65, 67, 84, 32, + 86, 70, 36, 11, 79, 76, 68, 32, 78, 85, 66, 73, 65, 78, 32, 110, 83, 181, + 244, 28, 13, 77, 79, 82, 80, 72, 79, 76, 79, 71, 73, 67, 65, 76, 126, 76, 9, 79, 77, 66, 73, 78, 73, 78, 71, 32, 161, 3, 5, 65, 80, 73, 84, 65, 6, - 68, 9, 83, 80, 73, 82, 73, 84, 85, 83, 32, 189, 171, 30, 2, 78, 73, 4, - 196, 135, 6, 2, 76, 69, 161, 198, 16, 2, 65, 83, 56, 184, 211, 19, 8, 84, - 72, 79, 85, 83, 65, 78, 68, 166, 146, 1, 78, 195, 190, 2, 68, 4, 230, - 241, 17, 82, 203, 234, 10, 85, 8, 58, 68, 0, 3, 73, 78, 68, 146, 15, 86, - 183, 204, 28, 70, 2, 145, 220, 28, 7, 73, 82, 69, 67, 84, 32, 81, 134, 1, - 56, 3, 77, 65, 76, 197, 11, 6, 89, 77, 66, 79, 76, 32, 120, 45, 9, 76, + 68, 9, 83, 80, 73, 82, 73, 84, 85, 83, 32, 213, 175, 31, 2, 78, 73, 4, + 152, 149, 6, 2, 76, 69, 173, 131, 17, 2, 65, 83, 56, 134, 173, 21, 78, + 230, 193, 2, 68, 145, 238, 6, 8, 84, 72, 79, 85, 83, 65, 78, 68, 4, 146, + 173, 18, 82, 231, 172, 11, 85, 8, 58, 68, 0, 3, 73, 78, 68, 142, 15, 86, + 131, 202, 29, 70, 2, 217, 217, 29, 7, 73, 82, 69, 67, 84, 32, 81, 134, 1, + 56, 3, 77, 65, 76, 193, 11, 6, 89, 77, 66, 79, 76, 32, 120, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 120, 138, 2, 65, 48, 6, 66, 79, 72, 65, 73, 82, 32, 2, 67, 82, 170, 1, 68, 160, 1, 2, 71, 65, 34, 72, 38, 75, 46, - 70, 34, 76, 66, 79, 138, 3, 83, 90, 84, 46, 90, 146, 197, 5, 86, 210, 55, - 80, 206, 203, 3, 73, 138, 179, 21, 82, 198, 3, 69, 218, 7, 77, 2, 78, - 163, 17, 85, 4, 44, 5, 75, 72, 77, 73, 77, 143, 134, 19, 76, 2, 177, 1, + 70, 34, 76, 66, 79, 134, 3, 83, 90, 84, 46, 90, 230, 211, 5, 86, 214, 54, + 80, 190, 210, 3, 73, 238, 162, 22, 82, 198, 3, 69, 218, 7, 77, 2, 78, + 163, 17, 85, 4, 44, 5, 75, 72, 77, 73, 77, 223, 193, 19, 76, 2, 177, 1, 4, 73, 67, 32, 75, 10, 92, 12, 89, 80, 84, 79, 71, 82, 65, 77, 77, 73, - 67, 32, 53, 7, 79, 83, 83, 69, 68, 32, 83, 8, 50, 83, 226, 4, 71, 170, - 129, 31, 69, 219, 7, 78, 2, 215, 144, 24, 72, 12, 80, 9, 73, 65, 76, 69, - 67, 84, 45, 80, 32, 204, 251, 30, 2, 65, 76, 175, 17, 69, 8, 222, 137, 8, - 72, 162, 245, 7, 65, 240, 252, 11, 2, 75, 65, 139, 145, 3, 78, 4, 190, 3, - 78, 183, 251, 30, 77, 4, 250, 150, 24, 79, 191, 196, 5, 65, 8, 42, 72, - 182, 250, 27, 65, 139, 145, 3, 83, 4, 186, 139, 31, 69, 219, 19, 73, 4, - 224, 131, 19, 7, 45, 83, 72, 65, 80, 69, 68, 207, 212, 7, 65, 35, 36, 3, - 76, 68, 32, 167, 249, 30, 79, 30, 76, 7, 67, 79, 80, 84, 73, 67, 32, 193, - 1, 7, 78, 85, 66, 73, 65, 78, 32, 22, 98, 71, 42, 72, 188, 1, 2, 83, 72, - 238, 169, 14, 68, 190, 224, 9, 79, 178, 171, 5, 69, 227, 101, 65, 2, 17, - 2, 65, 78, 2, 171, 199, 15, 71, 8, 198, 147, 24, 79, 246, 211, 1, 65, - 159, 161, 5, 69, 8, 50, 78, 176, 253, 15, 2, 83, 72, 163, 139, 5, 87, 4, - 238, 135, 31, 71, 3, 89, 10, 54, 72, 178, 137, 6, 65, 222, 236, 24, 79, - 219, 3, 73, 4, 226, 249, 30, 73, 187, 13, 69, 4, 196, 237, 30, 3, 72, 69, - 84, 167, 8, 65, 2, 219, 245, 30, 65, 14, 62, 75, 30, 77, 2, 80, 22, 83, - 253, 146, 27, 3, 84, 65, 85, 4, 26, 72, 227, 133, 31, 65, 2, 143, 147, - 27, 73, 4, 136, 251, 15, 6, 72, 73, 77, 65, 32, 83, 145, 246, 3, 3, 84, - 65, 85, 4, 220, 252, 23, 3, 76, 69, 70, 157, 133, 2, 4, 82, 73, 71, 72, - 6, 96, 6, 78, 73, 83, 72, 32, 86, 252, 240, 24, 8, 82, 69, 83, 80, 79, - 78, 68, 83, 163, 157, 5, 65, 2, 157, 246, 27, 4, 69, 82, 83, 69, 44, 104, - 2, 78, 84, 208, 132, 3, 6, 67, 72, 32, 65, 78, 68, 181, 138, 27, 8, 80, - 76, 69, 32, 87, 73, 84, 72, 40, 56, 2, 69, 82, 33, 8, 73, 78, 71, 32, 82, - 79, 68, 32, 4, 158, 221, 26, 66, 235, 121, 83, 36, 48, 4, 84, 69, 78, 83, - 1, 4, 85, 78, 73, 84, 18, 185, 145, 23, 2, 32, 68, 51, 82, 69, 76, 5, 73, - 67, 75, 69, 84, 34, 79, 254, 5, 85, 50, 89, 239, 210, 29, 65, 4, 220, - 148, 8, 3, 68, 73, 84, 153, 238, 19, 7, 83, 67, 69, 78, 84, 32, 77, 5, - 237, 158, 26, 3, 32, 66, 65, 28, 84, 2, 83, 83, 228, 244, 28, 3, 67, 79, - 68, 188, 148, 1, 3, 73, 83, 83, 243, 57, 87, 22, 46, 32, 240, 2, 3, 69, - 68, 32, 223, 1, 73, 14, 44, 3, 79, 70, 32, 82, 80, 151, 208, 30, 77, 4, - 236, 202, 27, 6, 74, 69, 82, 85, 83, 65, 245, 173, 2, 5, 76, 79, 82, 82, - 65, 8, 76, 10, 65, 84, 84, 89, 32, 87, 73, 84, 72, 32, 97, 5, 79, 77, 77, - 69, 69, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 241, 192, 28, 7, - 84, 32, 67, 82, 79, 83, 83, 5, 233, 161, 3, 11, 32, 87, 73, 84, 72, 32, - 72, 65, 76, 70, 45, 6, 196, 181, 24, 37, 78, 69, 71, 65, 84, 73, 86, 69, - 32, 83, 81, 85, 65, 82, 69, 68, 32, 76, 65, 84, 73, 78, 32, 67, 65, 80, - 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 160, 212, 2, 2, 70, 76, 137, - 169, 2, 3, 83, 87, 79, 2, 141, 160, 23, 5, 78, 71, 32, 76, 65, 4, 228, - 145, 17, 3, 90, 69, 73, 179, 217, 13, 84, 6, 194, 159, 19, 73, 137, 249, - 6, 4, 83, 84, 65, 76, 204, 19, 154, 1, 66, 20, 8, 78, 69, 73, 70, 79, 82, - 77, 32, 138, 250, 1, 80, 114, 82, 172, 3, 2, 83, 84, 254, 165, 15, 67, - 209, 194, 9, 6, 84, 32, 79, 70, 32, 77, 2, 207, 160, 5, 69, 168, 19, 176, - 1, 13, 78, 85, 77, 69, 82, 73, 67, 32, 83, 73, 71, 78, 32, 184, 20, 17, - 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 83, 73, 71, 78, 32, 233, - 1, 5, 83, 73, 71, 78, 32, 222, 1, 78, 69, 166, 2, 70, 152, 3, 2, 78, 73, - 154, 2, 79, 130, 3, 83, 151, 4, 84, 24, 68, 5, 73, 71, 72, 84, 32, 153, - 1, 7, 76, 65, 77, 73, 84, 69, 32, 16, 142, 14, 83, 214, 113, 85, 238, 50, - 71, 136, 56, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 85, - 83, 83, 85, 186, 223, 25, 68, 171, 217, 1, 65, 8, 136, 197, 20, 5, 79, - 78, 69, 32, 84, 250, 214, 8, 70, 175, 14, 84, 58, 48, 4, 73, 86, 69, 32, - 125, 4, 79, 85, 82, 32, 26, 70, 83, 206, 1, 66, 250, 12, 65, 82, 71, 250, - 109, 85, 175, 202, 26, 68, 6, 182, 15, 72, 165, 212, 18, 5, 73, 88, 84, - 72, 83, 32, 150, 1, 66, 44, 18, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, - 82, 77, 32, 76, 73, 77, 77, 85, 206, 12, 65, 82, 71, 26, 83, 226, 109, - 85, 175, 202, 26, 68, 6, 188, 123, 3, 65, 78, 50, 139, 252, 25, 85, 9, - 194, 156, 12, 32, 195, 231, 18, 52, 24, 44, 4, 71, 73, 68, 65, 33, 3, 78, - 69, 32, 4, 146, 158, 29, 69, 227, 101, 77, 20, 156, 1, 19, 86, 65, 82, - 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 76, 73, 77, 77, 85, 174, 7, - 83, 214, 113, 85, 238, 50, 71, 194, 151, 26, 68, 171, 217, 1, 65, 9, 182, - 255, 30, 32, 186, 2, 51, 3, 52, 28, 92, 16, 76, 68, 32, 65, 83, 83, 89, - 82, 73, 65, 78, 32, 79, 78, 69, 32, 37, 3, 78, 69, 32, 4, 170, 134, 18, - 83, 203, 250, 10, 81, 24, 166, 1, 69, 52, 8, 81, 85, 65, 82, 84, 69, 82, - 32, 206, 7, 66, 54, 71, 84, 5, 84, 72, 73, 82, 68, 224, 244, 23, 2, 83, - 72, 241, 136, 6, 6, 72, 65, 76, 70, 32, 71, 4, 158, 8, 83, 189, 164, 1, - 5, 73, 71, 72, 84, 72, 4, 202, 153, 29, 65, 219, 108, 71, 38, 144, 1, 5, - 69, 86, 69, 78, 32, 188, 1, 20, 72, 65, 82, 50, 32, 84, 73, 77, 69, 83, - 32, 71, 65, 76, 32, 80, 76, 85, 83, 32, 37, 3, 73, 88, 32, 18, 148, 1, - 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 77, 73, 78, - 218, 1, 83, 214, 113, 85, 238, 50, 71, 194, 151, 26, 68, 171, 217, 1, 65, - 6, 218, 148, 12, 32, 195, 231, 18, 51, 4, 178, 189, 27, 68, 139, 191, 2, - 77, 16, 142, 1, 83, 142, 3, 65, 202, 110, 85, 238, 50, 71, 220, 130, 26, - 16, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 65, 83, 72, 231, - 20, 68, 2, 227, 61, 72, 50, 52, 5, 72, 82, 69, 69, 32, 241, 1, 3, 87, 79, - 32, 28, 142, 1, 66, 40, 4, 83, 72, 65, 82, 24, 16, 86, 65, 82, 73, 65, - 78, 84, 32, 70, 79, 82, 77, 32, 69, 83, 72, 118, 65, 82, 71, 167, 184, - 27, 68, 6, 248, 111, 3, 85, 82, 85, 199, 10, 65, 8, 210, 111, 50, 3, 85, - 4, 250, 144, 24, 49, 203, 3, 50, 22, 82, 65, 30, 66, 32, 2, 69, 83, 22, - 71, 26, 83, 61, 6, 84, 72, 73, 82, 68, 83, 4, 209, 184, 1, 2, 83, 72, 4, - 238, 120, 65, 199, 241, 25, 85, 2, 239, 147, 24, 72, 4, 53, 3, 69, 83, - 72, 4, 11, 72, 4, 17, 2, 65, 82, 4, 174, 246, 30, 50, 3, 85, 4, 11, 32, - 4, 128, 226, 26, 12, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 195, - 85, 68, 10, 160, 1, 9, 68, 73, 65, 71, 79, 78, 65, 76, 32, 188, 241, 24, - 6, 86, 69, 82, 84, 73, 67, 169, 225, 2, 14, 79, 76, 68, 32, 65, 83, 83, - 89, 82, 73, 65, 78, 32, 87, 6, 192, 190, 27, 2, 84, 82, 204, 229, 2, 4, - 81, 85, 65, 68, 15, 67, 192, 17, 202, 1, 65, 186, 15, 66, 170, 5, 68, - 190, 15, 69, 202, 10, 71, 182, 30, 72, 238, 3, 73, 238, 4, 75, 138, 20, + 67, 32, 53, 7, 79, 83, 83, 69, 68, 32, 83, 8, 50, 83, 226, 4, 71, 210, + 133, 32, 69, 219, 7, 78, 2, 207, 220, 24, 72, 12, 80, 9, 73, 65, 76, 69, + 67, 84, 45, 80, 32, 244, 255, 31, 2, 65, 76, 175, 17, 69, 8, 182, 156, 8, + 72, 250, 150, 8, 65, 244, 183, 12, 2, 75, 65, 255, 165, 3, 78, 4, 190, 3, + 78, 223, 255, 31, 77, 4, 190, 198, 25, 79, 215, 150, 5, 65, 8, 42, 72, + 234, 233, 28, 65, 255, 165, 3, 83, 4, 226, 143, 32, 69, 219, 19, 73, 4, + 176, 191, 19, 7, 45, 83, 72, 65, 80, 69, 68, 155, 241, 7, 65, 35, 36, 3, + 76, 68, 32, 207, 253, 31, 79, 30, 76, 7, 67, 79, 80, 84, 73, 67, 32, 189, + 1, 7, 78, 85, 66, 73, 65, 78, 32, 22, 98, 71, 42, 72, 184, 1, 2, 83, 72, + 154, 222, 14, 68, 142, 248, 9, 79, 222, 224, 5, 69, 207, 104, 65, 2, 17, + 2, 65, 78, 2, 183, 251, 15, 71, 8, 138, 195, 25, 79, 238, 125, 65, 139, + 204, 5, 69, 8, 50, 78, 228, 177, 16, 2, 83, 72, 155, 158, 5, 87, 4, 154, + 140, 32, 71, 3, 89, 10, 54, 72, 138, 151, 6, 65, 178, 227, 25, 79, 219, + 3, 73, 4, 142, 254, 31, 73, 187, 13, 69, 4, 240, 241, 31, 3, 72, 69, 84, + 167, 8, 65, 2, 135, 250, 31, 65, 14, 62, 75, 30, 77, 2, 80, 22, 83, 241, + 234, 27, 3, 84, 65, 85, 4, 26, 72, 143, 138, 32, 65, 2, 131, 235, 27, 73, + 4, 188, 175, 16, 6, 72, 73, 77, 65, 32, 83, 193, 254, 3, 3, 84, 65, 85, + 4, 188, 200, 24, 3, 76, 69, 70, 245, 145, 2, 4, 82, 73, 71, 72, 6, 96, 6, + 78, 73, 83, 72, 32, 86, 204, 200, 25, 8, 82, 69, 83, 80, 79, 78, 68, 83, + 231, 201, 5, 65, 2, 213, 229, 28, 4, 69, 82, 83, 69, 44, 104, 2, 78, 84, + 156, 132, 3, 6, 67, 72, 32, 65, 78, 68, 253, 142, 28, 8, 80, 76, 69, 32, + 87, 73, 84, 72, 40, 56, 2, 69, 82, 37, 8, 73, 78, 71, 32, 82, 79, 68, 32, + 4, 190, 181, 27, 66, 195, 131, 1, 83, 36, 48, 4, 84, 69, 78, 83, 1, 4, + 85, 78, 73, 84, 18, 133, 220, 23, 2, 32, 68, 51, 82, 69, 76, 5, 73, 67, + 75, 69, 84, 34, 79, 198, 5, 85, 50, 89, 143, 214, 30, 65, 4, 196, 168, 8, + 3, 68, 73, 84, 181, 201, 20, 7, 83, 67, 69, 78, 84, 32, 77, 5, 169, 247, + 26, 3, 32, 66, 65, 28, 84, 2, 83, 83, 152, 242, 29, 3, 67, 79, 68, 152, + 155, 1, 3, 73, 83, 83, 255, 57, 87, 22, 46, 32, 184, 2, 3, 69, 68, 32, + 223, 1, 73, 14, 44, 3, 79, 70, 32, 82, 80, 179, 212, 31, 77, 4, 224, 172, + 28, 6, 74, 69, 82, 85, 83, 65, 145, 208, 2, 5, 76, 79, 82, 82, 65, 8, 76, + 10, 65, 84, 84, 89, 32, 87, 73, 84, 72, 32, 41, 5, 79, 77, 77, 69, 69, 4, + 218, 165, 10, 82, 25, 3, 76, 69, 70, 5, 189, 161, 3, 11, 32, 87, 73, 84, + 72, 32, 72, 65, 76, 70, 45, 6, 204, 132, 25, 37, 78, 69, 71, 65, 84, 73, + 86, 69, 32, 83, 81, 85, 65, 82, 69, 68, 32, 76, 65, 84, 73, 78, 32, 67, + 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 192, 221, 2, 2, 70, + 76, 189, 210, 2, 3, 83, 87, 79, 2, 145, 235, 23, 5, 78, 71, 32, 76, 65, + 4, 248, 199, 17, 3, 90, 69, 73, 255, 167, 14, 84, 6, 130, 220, 19, 73, + 189, 149, 7, 4, 83, 84, 65, 76, 206, 19, 154, 1, 66, 20, 8, 78, 69, 73, + 70, 79, 82, 77, 32, 154, 250, 1, 80, 114, 82, 180, 3, 2, 83, 84, 242, + 219, 15, 67, 161, 229, 9, 6, 84, 32, 79, 70, 32, 77, 2, 211, 175, 5, 69, + 170, 19, 176, 1, 13, 78, 85, 77, 69, 82, 73, 67, 32, 83, 73, 71, 78, 32, + 184, 20, 17, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 83, 73, 71, + 78, 32, 233, 1, 5, 83, 73, 71, 78, 32, 222, 1, 78, 69, 166, 2, 70, 152, + 3, 2, 78, 73, 154, 2, 79, 130, 3, 83, 151, 4, 84, 24, 68, 5, 73, 71, 72, + 84, 32, 153, 1, 7, 76, 65, 77, 73, 84, 69, 32, 16, 142, 14, 83, 230, 113, + 85, 238, 50, 71, 136, 56, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, + 77, 32, 85, 83, 83, 85, 214, 193, 26, 68, 219, 248, 1, 65, 8, 232, 140, + 21, 5, 79, 78, 69, 32, 84, 246, 144, 9, 70, 175, 14, 84, 58, 48, 4, 73, + 86, 69, 32, 125, 4, 79, 85, 82, 32, 26, 70, 83, 206, 1, 66, 250, 12, 65, + 82, 71, 138, 110, 85, 203, 172, 27, 68, 6, 182, 15, 72, 173, 144, 19, 5, + 73, 88, 84, 72, 83, 32, 150, 1, 66, 44, 18, 86, 65, 82, 73, 65, 78, 84, + 32, 70, 79, 82, 77, 32, 76, 73, 77, 77, 85, 206, 12, 65, 82, 71, 26, 83, + 242, 109, 85, 203, 172, 27, 68, 6, 204, 123, 3, 65, 78, 50, 163, 212, 26, + 85, 9, 214, 202, 12, 32, 143, 190, 19, 52, 24, 44, 4, 71, 73, 68, 65, 33, + 3, 78, 69, 32, 4, 238, 159, 30, 69, 207, 104, 77, 20, 156, 1, 19, 86, 65, + 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 76, 73, 77, 77, 85, 174, + 7, 83, 230, 113, 85, 238, 50, 71, 222, 249, 26, 68, 219, 248, 1, 65, 9, + 150, 132, 32, 32, 186, 2, 51, 3, 52, 28, 92, 16, 76, 68, 32, 65, 83, 83, + 89, 82, 73, 65, 78, 32, 79, 78, 69, 32, 37, 3, 78, 69, 32, 4, 154, 194, + 18, 83, 175, 192, 11, 81, 24, 166, 1, 69, 52, 8, 81, 85, 65, 82, 84, 69, + 82, 32, 206, 7, 66, 54, 71, 84, 5, 84, 72, 73, 82, 68, 200, 194, 24, 2, + 83, 72, 217, 191, 6, 6, 72, 65, 76, 70, 32, 71, 4, 158, 8, 83, 205, 164, + 1, 5, 73, 71, 72, 84, 72, 4, 166, 155, 30, 65, 207, 111, 71, 38, 144, 1, + 5, 69, 86, 69, 78, 32, 188, 1, 20, 72, 65, 82, 50, 32, 84, 73, 77, 69, + 83, 32, 71, 65, 76, 32, 80, 76, 85, 83, 32, 37, 3, 73, 88, 32, 18, 148, + 1, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 77, 73, + 78, 218, 1, 83, 230, 113, 85, 238, 50, 71, 222, 249, 26, 68, 219, 248, 1, + 65, 6, 238, 194, 12, 32, 143, 190, 19, 51, 4, 222, 159, 28, 68, 167, 225, + 2, 77, 16, 142, 1, 83, 142, 3, 65, 218, 110, 85, 238, 50, 71, 144, 229, + 26, 16, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 65, 83, 72, + 207, 20, 68, 2, 227, 61, 72, 50, 52, 5, 72, 82, 69, 69, 32, 241, 1, 3, + 87, 79, 32, 28, 142, 1, 66, 40, 4, 83, 72, 65, 82, 24, 16, 86, 65, 82, + 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 69, 83, 72, 118, 65, 82, 71, 211, + 154, 28, 68, 6, 136, 112, 3, 85, 82, 85, 199, 10, 65, 8, 226, 111, 50, 3, + 85, 4, 130, 224, 24, 49, 203, 3, 50, 22, 82, 65, 30, 66, 32, 2, 69, 83, + 22, 71, 26, 83, 61, 6, 84, 72, 73, 82, 68, 83, 4, 225, 184, 1, 2, 83, 72, + 4, 254, 120, 65, 223, 201, 26, 85, 2, 247, 226, 24, 72, 4, 53, 3, 69, 83, + 72, 4, 11, 72, 4, 17, 2, 65, 82, 4, 142, 251, 31, 50, 3, 85, 4, 11, 32, + 4, 196, 186, 27, 12, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 171, + 95, 68, 10, 160, 1, 9, 68, 73, 65, 71, 79, 78, 65, 76, 32, 204, 203, 25, + 6, 86, 69, 82, 84, 73, 67, 133, 247, 2, 14, 79, 76, 68, 32, 65, 83, 83, + 89, 82, 73, 65, 78, 32, 87, 6, 192, 173, 28, 2, 84, 82, 160, 251, 2, 4, + 81, 85, 65, 68, 15, 67, 194, 17, 202, 1, 65, 186, 15, 66, 170, 5, 68, + 190, 15, 69, 202, 10, 71, 182, 30, 72, 238, 3, 73, 238, 4, 75, 154, 20, 76, 134, 37, 77, 134, 6, 78, 226, 17, 80, 210, 3, 82, 42, 83, 238, 19, 84, 186, 8, 85, 207, 20, 90, 141, 1, 160, 1, 7, 32, 84, 73, 77, 69, 83, 32, 142, 1, 66, 250, 3, 68, 38, 75, 90, 76, 212, 1, 3, 77, 65, 82, 78, - 78, 134, 2, 82, 42, 83, 210, 230, 29, 80, 191, 127, 50, 16, 128, 173, 1, - 5, 76, 65, 71, 65, 82, 226, 23, 71, 166, 187, 12, 73, 198, 144, 13, 77, - 142, 250, 1, 83, 226, 159, 1, 66, 234, 67, 72, 187, 2, 65, 45, 22, 32, - 219, 2, 50, 28, 48, 6, 84, 73, 77, 69, 83, 32, 251, 202, 1, 71, 26, 180, - 1, 2, 71, 65, 38, 73, 36, 2, 83, 72, 240, 112, 8, 85, 32, 80, 76, 85, 83, + 78, 134, 2, 82, 42, 83, 154, 235, 30, 80, 215, 127, 50, 16, 144, 173, 1, + 5, 76, 65, 71, 65, 82, 226, 23, 71, 246, 239, 12, 73, 154, 182, 13, 77, + 182, 161, 2, 83, 218, 162, 1, 66, 246, 67, 72, 187, 2, 65, 45, 22, 32, + 219, 2, 50, 28, 48, 6, 84, 73, 77, 69, 83, 32, 139, 203, 1, 71, 26, 180, + 1, 2, 71, 65, 38, 73, 36, 2, 83, 72, 128, 113, 8, 85, 32, 80, 76, 85, 83, 32, 85, 188, 56, 4, 68, 85, 78, 51, 166, 2, 65, 204, 2, 3, 78, 85, 78, - 242, 27, 76, 191, 160, 29, 72, 4, 178, 210, 1, 78, 151, 155, 29, 76, 4, - 194, 169, 1, 71, 167, 196, 28, 77, 4, 170, 137, 24, 85, 187, 254, 4, 69, - 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, 190, 133, 1, 77, 130, 59, 71, - 194, 9, 83, 234, 10, 84, 192, 177, 27, 2, 66, 65, 143, 230, 1, 65, 5, + 242, 27, 76, 143, 165, 30, 72, 4, 194, 210, 1, 78, 231, 159, 30, 76, 4, + 210, 169, 1, 71, 223, 200, 29, 77, 4, 178, 216, 24, 85, 143, 177, 5, 69, + 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, 206, 133, 1, 77, 130, 59, 71, + 194, 9, 83, 234, 10, 84, 140, 179, 28, 2, 66, 65, 147, 233, 1, 65, 5, 249, 51, 5, 32, 84, 73, 77, 69, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, - 252, 27, 5, 83, 72, 73, 84, 65, 199, 80, 69, 23, 68, 7, 32, 84, 73, 77, - 69, 83, 32, 202, 240, 24, 69, 247, 169, 5, 65, 16, 102, 75, 140, 173, 1, - 2, 68, 73, 218, 253, 25, 71, 170, 217, 1, 85, 130, 92, 65, 146, 92, 83, - 215, 42, 72, 4, 242, 157, 1, 65, 183, 203, 29, 73, 7, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 4, 130, 248, 3, 75, 207, 195, 26, 83, 13, 26, 32, 147, - 187, 30, 83, 8, 128, 1, 10, 80, 76, 85, 83, 32, 78, 65, 71, 65, 32, 148, - 164, 8, 7, 84, 72, 82, 69, 69, 32, 84, 245, 220, 5, 4, 79, 86, 69, 82, 4, - 128, 140, 1, 16, 79, 80, 80, 79, 83, 73, 78, 71, 32, 65, 78, 32, 80, 76, - 85, 83, 163, 226, 22, 83, 6, 200, 49, 2, 65, 68, 175, 219, 17, 75, 18, - 26, 72, 239, 245, 11, 65, 17, 42, 32, 138, 140, 18, 71, 191, 217, 12, 50, - 10, 68, 9, 79, 86, 69, 82, 32, 65, 83, 72, 32, 158, 97, 90, 135, 112, 75, - 6, 176, 1, 8, 79, 86, 69, 82, 32, 65, 83, 72, 141, 143, 1, 29, 84, 85, + 252, 27, 5, 83, 72, 73, 84, 65, 215, 80, 69, 23, 68, 7, 32, 84, 73, 77, + 69, 83, 32, 218, 202, 25, 69, 187, 212, 5, 65, 16, 102, 75, 156, 173, 1, + 2, 68, 73, 246, 223, 26, 71, 218, 248, 1, 85, 238, 94, 65, 170, 92, 83, + 215, 42, 72, 4, 130, 158, 1, 65, 135, 208, 30, 73, 7, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 4, 210, 253, 3, 75, 223, 194, 27, 83, 13, 26, 32, 243, + 191, 31, 83, 8, 128, 1, 10, 80, 76, 85, 83, 32, 78, 65, 71, 65, 32, 128, + 185, 8, 7, 84, 72, 82, 69, 69, 32, 84, 177, 252, 5, 4, 79, 86, 69, 82, 4, + 144, 140, 1, 16, 79, 80, 80, 79, 83, 73, 78, 71, 32, 65, 78, 32, 80, 76, + 85, 83, 251, 175, 23, 83, 6, 200, 49, 2, 65, 68, 239, 151, 18, 75, 18, + 26, 72, 131, 164, 12, 65, 17, 42, 32, 202, 200, 18, 71, 223, 161, 13, 50, + 10, 68, 9, 79, 86, 69, 82, 32, 65, 83, 72, 32, 174, 97, 90, 135, 112, 75, + 6, 176, 1, 8, 79, 86, 69, 82, 32, 65, 83, 72, 157, 143, 1, 29, 84, 85, 71, 50, 32, 79, 86, 69, 82, 32, 84, 85, 71, 50, 32, 84, 85, 71, 50, 32, - 79, 86, 69, 82, 32, 84, 85, 71, 50, 5, 133, 145, 1, 27, 32, 67, 82, 79, + 79, 86, 69, 82, 32, 84, 85, 71, 50, 5, 149, 145, 1, 27, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 65, 83, 72, 32, 79, 86, 69, 82, 32, 65, 83, 72, 32, 79, 86, 69, 82, 52, 30, 65, 158, 2, 73, 95, 85, 27, 66, 68, 44, 4, - 72, 65, 82, 50, 90, 76, 66, 82, 183, 140, 27, 71, 5, 129, 208, 1, 6, 32, - 84, 73, 77, 69, 83, 9, 37, 7, 32, 84, 73, 77, 69, 83, 32, 6, 222, 164, 1, - 65, 218, 168, 29, 78, 163, 17, 90, 7, 180, 215, 29, 7, 32, 79, 86, 69, - 82, 32, 66, 215, 136, 1, 65, 5, 203, 252, 23, 65, 9, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 6, 246, 155, 1, 73, 198, 161, 28, 71, 171, 162, 1, 65, - 19, 50, 32, 168, 1, 3, 76, 85, 71, 223, 169, 1, 82, 8, 88, 8, 79, 86, 69, - 82, 32, 66, 85, 32, 217, 222, 26, 8, 67, 82, 79, 83, 83, 73, 78, 71, 6, - 144, 152, 12, 7, 84, 73, 77, 69, 83, 32, 78, 234, 139, 17, 65, 211, 106, - 85, 5, 237, 236, 3, 8, 32, 79, 86, 69, 82, 32, 66, 85, 180, 1, 34, 65, - 222, 5, 73, 195, 2, 85, 67, 50, 71, 150, 5, 82, 146, 49, 32, 163, 166, - 30, 77, 55, 26, 32, 171, 220, 30, 51, 50, 76, 13, 75, 73, 83, 73, 77, 53, - 32, 84, 73, 77, 69, 83, 32, 211, 198, 1, 84, 48, 146, 1, 65, 30, 66, 38, - 71, 112, 2, 73, 82, 42, 76, 94, 85, 166, 131, 1, 80, 162, 61, 84, 234, - 232, 26, 75, 146, 152, 2, 78, 254, 2, 83, 163, 17, 72, 4, 110, 32, 175, - 183, 29, 77, 4, 154, 244, 28, 65, 143, 230, 1, 73, 10, 34, 65, 58, 73, - 151, 147, 30, 85, 5, 11, 32, 2, 169, 244, 28, 6, 80, 76, 85, 83, 32, 77, - 5, 211, 245, 23, 82, 5, 181, 203, 13, 5, 32, 80, 76, 85, 83, 8, 26, 85, - 211, 216, 30, 65, 7, 144, 151, 1, 7, 32, 80, 76, 85, 83, 32, 77, 191, - 193, 29, 77, 6, 52, 7, 50, 32, 80, 76, 85, 83, 32, 203, 214, 30, 83, 4, - 128, 27, 2, 71, 73, 175, 215, 28, 77, 7, 171, 147, 12, 65, 25, 54, 77, - 150, 1, 78, 76, 2, 83, 72, 135, 213, 30, 66, 13, 44, 7, 32, 84, 73, 77, - 69, 83, 32, 59, 50, 6, 220, 69, 2, 85, 32, 134, 160, 13, 73, 199, 195, - 16, 83, 5, 153, 228, 12, 6, 32, 84, 73, 77, 69, 83, 5, 173, 179, 18, 14, - 32, 75, 65, 83, 75, 65, 76, 32, 85, 32, 71, 85, 78, 85, 5, 229, 157, 1, - 5, 32, 80, 76, 85, 83, 91, 70, 32, 66, 66, 94, 71, 234, 4, 78, 190, 234, - 23, 82, 215, 227, 6, 72, 6, 138, 176, 1, 71, 170, 3, 83, 249, 144, 12, 4, - 79, 86, 69, 82, 9, 52, 7, 32, 84, 73, 77, 69, 83, 32, 159, 211, 30, 50, - 4, 234, 145, 1, 69, 167, 148, 29, 83, 61, 52, 7, 32, 84, 73, 77, 69, 83, - 32, 175, 140, 30, 85, 56, 122, 65, 58, 68, 30, 71, 74, 75, 90, 76, 110, - 77, 50, 83, 242, 79, 69, 218, 58, 73, 250, 139, 16, 72, 158, 164, 13, 78, - 3, 80, 6, 32, 2, 83, 72, 139, 164, 30, 78, 5, 143, 154, 14, 32, 4, 182, - 129, 30, 73, 3, 85, 8, 26, 73, 215, 208, 30, 65, 7, 252, 140, 1, 2, 82, - 50, 199, 194, 29, 83, 8, 26, 85, 195, 185, 1, 65, 6, 40, 4, 83, 72, 85, - 50, 227, 207, 30, 82, 5, 175, 26, 32, 8, 26, 65, 45, 2, 85, 72, 6, 202, - 26, 77, 197, 207, 23, 3, 75, 45, 48, 2, 225, 57, 5, 32, 80, 76, 85, 83, - 6, 198, 233, 28, 65, 206, 167, 1, 69, 211, 61, 73, 4, 222, 138, 1, 73, - 131, 173, 29, 72, 11, 26, 51, 247, 205, 30, 52, 7, 143, 9, 32, 117, 130, + 72, 65, 82, 50, 90, 76, 66, 82, 251, 238, 27, 71, 5, 145, 208, 1, 6, 32, + 84, 73, 77, 69, 83, 9, 37, 7, 32, 84, 73, 77, 69, 83, 32, 6, 238, 164, 1, + 65, 170, 173, 30, 78, 163, 17, 90, 7, 252, 219, 30, 7, 32, 79, 86, 69, + 82, 32, 66, 239, 136, 1, 65, 5, 211, 203, 24, 65, 9, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 6, 134, 156, 1, 73, 134, 167, 29, 71, 187, 161, 1, 65, + 19, 50, 32, 168, 1, 3, 76, 85, 71, 239, 169, 1, 82, 8, 88, 8, 79, 86, 69, + 82, 32, 66, 85, 32, 129, 183, 27, 8, 67, 82, 79, 83, 83, 73, 78, 71, 6, + 164, 198, 12, 7, 84, 73, 77, 69, 83, 32, 78, 246, 224, 17, 65, 135, 108, + 85, 5, 189, 242, 3, 8, 32, 79, 86, 69, 82, 32, 66, 85, 180, 1, 34, 65, + 222, 5, 73, 195, 2, 85, 67, 50, 71, 150, 5, 82, 146, 49, 32, 131, 171, + 31, 77, 55, 26, 32, 139, 225, 31, 51, 50, 76, 13, 75, 73, 83, 73, 77, 53, + 32, 84, 73, 77, 69, 83, 32, 227, 198, 1, 84, 48, 146, 1, 65, 30, 66, 38, + 71, 112, 2, 73, 82, 42, 76, 94, 85, 182, 131, 1, 80, 162, 61, 84, 218, + 230, 27, 75, 242, 158, 2, 78, 254, 2, 83, 163, 17, 72, 4, 110, 32, 255, + 188, 30, 77, 4, 246, 245, 29, 65, 147, 233, 1, 73, 10, 34, 65, 58, 73, + 235, 151, 31, 85, 5, 11, 32, 2, 133, 246, 29, 6, 80, 76, 85, 83, 32, 77, + 5, 219, 196, 24, 82, 5, 229, 255, 13, 5, 32, 80, 76, 85, 83, 8, 26, 85, + 179, 221, 31, 65, 7, 160, 151, 1, 7, 32, 80, 76, 85, 83, 32, 77, 143, + 198, 30, 77, 6, 52, 7, 50, 32, 80, 76, 85, 83, 32, 171, 219, 31, 83, 4, + 128, 27, 2, 71, 73, 139, 217, 29, 77, 7, 191, 193, 12, 65, 25, 54, 77, + 150, 1, 78, 76, 2, 83, 72, 231, 217, 31, 66, 13, 44, 7, 32, 84, 73, 77, + 69, 83, 32, 59, 50, 6, 220, 69, 2, 85, 32, 230, 212, 13, 73, 199, 147, + 17, 83, 5, 221, 152, 13, 6, 32, 84, 73, 77, 69, 83, 5, 181, 239, 18, 14, + 32, 75, 65, 83, 75, 65, 76, 32, 85, 32, 71, 85, 78, 85, 5, 245, 157, 1, + 5, 32, 80, 76, 85, 83, 91, 70, 32, 66, 66, 94, 71, 234, 4, 78, 198, 185, + 24, 82, 175, 153, 7, 72, 6, 154, 176, 1, 71, 170, 3, 83, 153, 197, 12, 4, + 79, 86, 69, 82, 9, 52, 7, 32, 84, 73, 77, 69, 83, 32, 255, 215, 31, 50, + 4, 250, 145, 1, 69, 247, 152, 30, 83, 61, 52, 7, 32, 84, 73, 77, 69, 83, + 32, 131, 145, 31, 85, 56, 122, 65, 58, 68, 30, 71, 74, 75, 90, 76, 110, + 77, 50, 83, 130, 80, 69, 218, 58, 73, 190, 198, 16, 72, 170, 238, 13, 78, + 3, 80, 6, 32, 2, 83, 72, 235, 168, 31, 78, 5, 215, 206, 14, 32, 4, 138, + 134, 31, 73, 3, 85, 8, 26, 73, 183, 213, 31, 65, 7, 140, 141, 1, 2, 82, + 50, 151, 199, 30, 83, 8, 26, 85, 211, 185, 1, 65, 6, 40, 4, 83, 72, 85, + 50, 195, 212, 31, 82, 5, 175, 26, 32, 8, 26, 65, 45, 2, 85, 72, 6, 202, + 26, 77, 205, 158, 24, 3, 75, 45, 48, 2, 225, 57, 5, 32, 80, 76, 85, 83, + 6, 162, 235, 29, 65, 198, 170, 1, 69, 223, 61, 73, 4, 238, 138, 1, 73, + 211, 177, 30, 72, 11, 26, 51, 215, 210, 31, 52, 7, 143, 9, 32, 117, 130, 1, 32, 90, 50, 210, 1, 78, 202, 1, 82, 60, 3, 83, 72, 50, 52, 3, 90, 69, - 78, 230, 164, 18, 71, 226, 163, 11, 68, 191, 127, 76, 4, 168, 25, 11, 79, - 86, 69, 82, 32, 69, 32, 78, 85, 78, 32, 237, 94, 4, 84, 73, 77, 69, 19, - 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 134, 1, 83, 128, 168, 1, 10, 65, - 32, 80, 76, 85, 83, 32, 72, 65, 32, 178, 128, 28, 71, 214, 17, 80, 146, - 24, 75, 238, 100, 77, 219, 19, 85, 4, 186, 193, 29, 65, 203, 114, 72, 15, + 78, 170, 225, 18, 71, 230, 235, 11, 68, 215, 127, 76, 4, 168, 25, 11, 79, + 86, 69, 82, 32, 69, 32, 78, 85, 78, 32, 253, 94, 4, 84, 73, 77, 69, 19, + 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 134, 1, 83, 144, 168, 1, 10, 65, + 32, 80, 76, 85, 83, 32, 72, 65, 32, 242, 133, 29, 71, 206, 16, 80, 154, + 24, 75, 254, 100, 77, 219, 19, 85, 4, 130, 198, 30, 65, 227, 114, 72, 15, 11, 32, 12, 96, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 36, 6, 84, 73, - 77, 69, 83, 32, 179, 208, 23, 83, 2, 189, 222, 13, 4, 83, 73, 78, 71, 6, - 188, 138, 1, 4, 71, 65, 78, 50, 147, 168, 29, 77, 6, 36, 3, 73, 78, 50, - 247, 248, 29, 69, 5, 187, 188, 29, 32, 5, 229, 16, 9, 32, 67, 82, 79, 83, + 77, 69, 83, 32, 155, 158, 24, 83, 2, 237, 146, 14, 4, 83, 73, 78, 71, 6, + 204, 138, 1, 4, 71, 65, 78, 50, 227, 172, 30, 77, 6, 36, 3, 73, 78, 50, + 203, 253, 30, 69, 5, 131, 193, 30, 32, 5, 229, 16, 9, 32, 67, 82, 79, 83, 83, 73, 78, 71, 63, 11, 32, 60, 96, 14, 83, 72, 69, 83, 72, 73, 71, 32, - 84, 73, 77, 69, 83, 32, 97, 6, 84, 73, 77, 69, 83, 32, 16, 162, 131, 1, - 73, 130, 219, 2, 77, 210, 131, 25, 65, 128, 105, 2, 76, 65, 182, 87, 83, + 84, 73, 77, 69, 83, 32, 97, 6, 84, 73, 77, 69, 83, 32, 16, 178, 131, 1, + 73, 194, 224, 2, 77, 222, 255, 25, 65, 244, 107, 2, 76, 65, 198, 87, 83, 147, 17, 72, 44, 134, 1, 65, 68, 5, 68, 85, 78, 51, 32, 26, 75, 58, 76, - 62, 83, 34, 85, 206, 127, 73, 204, 31, 2, 72, 65, 226, 249, 27, 71, 163, - 100, 66, 9, 228, 86, 9, 32, 80, 76, 85, 83, 32, 76, 65, 76, 195, 238, 29, - 78, 4, 217, 32, 2, 71, 85, 6, 192, 203, 23, 5, 65, 83, 75, 65, 76, 159, - 165, 3, 85, 8, 34, 65, 242, 195, 30, 73, 3, 85, 5, 185, 85, 2, 76, 32, 4, - 130, 173, 30, 72, 215, 22, 85, 4, 182, 195, 30, 50, 3, 68, 160, 2, 42, + 62, 83, 34, 85, 222, 127, 73, 204, 31, 2, 72, 65, 150, 253, 28, 71, 179, + 101, 66, 9, 244, 86, 9, 32, 80, 76, 85, 83, 32, 76, 65, 76, 147, 243, 30, + 78, 4, 217, 32, 2, 71, 85, 6, 168, 153, 24, 5, 65, 83, 75, 65, 76, 251, + 185, 3, 85, 8, 34, 65, 210, 200, 31, 73, 3, 85, 5, 201, 85, 2, 76, 32, 4, + 226, 177, 31, 72, 215, 22, 85, 4, 150, 200, 31, 50, 3, 68, 160, 2, 42, 65, 166, 19, 69, 122, 73, 135, 5, 85, 191, 1, 114, 50, 144, 16, 2, 66, - 65, 94, 68, 22, 76, 38, 78, 190, 140, 1, 32, 154, 6, 82, 218, 143, 27, - 83, 167, 142, 2, 77, 153, 1, 11, 32, 150, 1, 68, 6, 84, 73, 77, 69, 83, - 32, 137, 128, 1, 5, 79, 86, 69, 82, 32, 148, 1, 202, 1, 65, 162, 2, 66, + 65, 94, 68, 22, 76, 38, 78, 206, 140, 1, 32, 154, 6, 82, 146, 145, 28, + 83, 191, 145, 2, 77, 153, 1, 11, 32, 150, 1, 68, 6, 84, 73, 77, 69, 83, + 32, 153, 128, 1, 5, 79, 86, 69, 82, 32, 148, 1, 202, 1, 65, 162, 2, 66, 154, 1, 68, 178, 1, 69, 58, 71, 178, 1, 72, 230, 1, 73, 70, 75, 194, 1, - 76, 62, 77, 50, 78, 130, 1, 83, 142, 1, 85, 206, 154, 1, 84, 248, 179, - 22, 3, 90, 73, 90, 159, 225, 6, 80, 18, 132, 1, 6, 32, 80, 76, 85, 83, - 32, 50, 78, 52, 2, 83, 72, 161, 228, 17, 14, 66, 50, 32, 84, 69, 78, 85, - 32, 80, 76, 85, 83, 32, 84, 6, 186, 58, 68, 182, 147, 13, 73, 155, 238, - 16, 72, 5, 169, 61, 9, 32, 80, 76, 85, 83, 32, 75, 65, 75, 7, 11, 50, 5, - 245, 89, 6, 32, 80, 76, 85, 83, 32, 10, 26, 65, 77, 2, 85, 82, 6, 42, 72, - 44, 2, 82, 32, 183, 188, 30, 68, 2, 11, 65, 2, 255, 216, 23, 82, 5, 11, - 32, 2, 229, 241, 29, 4, 80, 76, 85, 83, 14, 34, 73, 54, 85, 187, 187, 30, - 65, 7, 17, 2, 77, 32, 4, 146, 22, 84, 191, 129, 1, 71, 6, 56, 8, 71, 32, - 84, 73, 77, 69, 83, 32, 255, 186, 30, 66, 4, 142, 119, 73, 151, 45, 75, - 10, 38, 78, 254, 2, 76, 203, 230, 28, 82, 5, 243, 28, 32, 18, 18, 65, 99, - 73, 11, 26, 82, 231, 158, 1, 78, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, - 254, 162, 30, 78, 255, 2, 68, 9, 158, 122, 52, 193, 176, 12, 7, 82, 50, - 32, 80, 76, 85, 83, 12, 62, 65, 138, 124, 85, 173, 174, 12, 6, 73, 32, + 76, 62, 77, 50, 78, 130, 1, 83, 142, 1, 85, 222, 154, 1, 84, 240, 130, + 23, 3, 90, 73, 90, 247, 150, 7, 80, 18, 132, 1, 6, 32, 80, 76, 85, 83, + 32, 50, 78, 52, 2, 83, 72, 225, 160, 18, 14, 66, 50, 32, 84, 69, 78, 85, + 32, 80, 76, 85, 83, 32, 84, 6, 202, 58, 68, 134, 200, 13, 73, 155, 190, + 17, 72, 5, 185, 61, 9, 32, 80, 76, 85, 83, 32, 75, 65, 75, 7, 11, 50, 5, + 133, 90, 6, 32, 80, 76, 85, 83, 32, 10, 26, 65, 77, 2, 85, 82, 6, 42, 72, + 44, 2, 82, 32, 151, 193, 31, 68, 2, 11, 65, 2, 135, 168, 24, 82, 5, 11, + 32, 2, 185, 246, 30, 4, 80, 76, 85, 83, 14, 34, 73, 54, 85, 155, 192, 31, + 65, 7, 17, 2, 77, 32, 4, 146, 22, 84, 207, 129, 1, 71, 6, 56, 8, 71, 32, + 84, 73, 77, 69, 83, 32, 223, 191, 31, 66, 4, 158, 119, 73, 151, 45, 75, + 10, 38, 78, 254, 2, 76, 183, 232, 29, 82, 5, 243, 28, 32, 18, 18, 65, 99, + 73, 11, 26, 82, 247, 158, 1, 78, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, + 222, 167, 31, 78, 255, 2, 68, 9, 174, 122, 52, 225, 228, 12, 7, 82, 50, + 32, 80, 76, 85, 83, 12, 62, 65, 154, 124, 85, 205, 226, 12, 6, 73, 32, 80, 76, 85, 83, 8, 40, 6, 32, 80, 76, 85, 83, 32, 83, 76, 4, 48, 6, 76, - 85, 32, 80, 76, 85, 219, 183, 30, 65, 2, 11, 83, 2, 199, 97, 32, 5, 201, - 169, 13, 5, 32, 80, 76, 85, 83, 4, 160, 100, 10, 83, 72, 32, 80, 76, 85, - 83, 32, 72, 85, 147, 15, 71, 12, 34, 65, 36, 2, 73, 68, 27, 85, 4, 218, - 252, 23, 83, 211, 185, 6, 75, 5, 209, 77, 2, 32, 80, 4, 60, 5, 83, 72, - 85, 50, 32, 181, 127, 5, 51, 32, 80, 76, 85, 2, 189, 158, 1, 3, 80, 76, - 85, 8, 26, 65, 231, 179, 30, 85, 7, 11, 77, 5, 211, 159, 1, 32, 6, 234, - 77, 69, 206, 129, 28, 85, 159, 229, 1, 73, 10, 26, 69, 73, 2, 85, 78, 7, - 33, 6, 32, 80, 76, 85, 83, 32, 4, 142, 208, 23, 69, 255, 207, 6, 71, 5, - 11, 32, 2, 171, 101, 79, 14, 42, 72, 250, 140, 23, 65, 171, 149, 7, 85, - 8, 18, 69, 51, 73, 5, 157, 186, 29, 7, 32, 80, 76, 85, 83, 32, 84, 4, - 178, 178, 30, 68, 3, 77, 7, 11, 68, 5, 161, 162, 13, 5, 32, 80, 76, 85, - 83, 7, 11, 32, 4, 170, 216, 28, 82, 133, 172, 1, 11, 67, 82, 79, 83, 83, - 73, 78, 71, 32, 71, 65, 5, 175, 131, 1, 32, 7, 246, 130, 1, 32, 167, 157, - 29, 65, 11, 11, 50, 9, 11, 32, 6, 80, 8, 67, 82, 79, 83, 83, 73, 78, 71, - 0, 4, 79, 86, 69, 82, 255, 221, 25, 84, 2, 181, 49, 3, 32, 71, 65, 8, 44, - 5, 83, 72, 84, 73, 78, 199, 203, 23, 50, 7, 37, 7, 32, 84, 73, 77, 69, - 83, 32, 4, 170, 182, 29, 75, 199, 120, 85, 49, 70, 32, 94, 52, 114, 82, - 190, 1, 83, 202, 168, 29, 68, 187, 130, 1, 71, 6, 172, 232, 8, 6, 84, 73, - 77, 69, 83, 32, 253, 212, 4, 8, 67, 82, 79, 83, 83, 73, 78, 71, 7, 11, + 85, 32, 80, 76, 85, 187, 188, 31, 65, 2, 11, 83, 2, 215, 97, 32, 5, 249, + 221, 13, 5, 32, 80, 76, 85, 83, 4, 176, 100, 10, 83, 72, 32, 80, 76, 85, + 83, 32, 72, 85, 147, 15, 71, 12, 34, 65, 36, 2, 73, 68, 27, 85, 4, 250, + 204, 24, 83, 147, 238, 6, 75, 5, 225, 77, 2, 32, 80, 4, 60, 5, 83, 72, + 85, 50, 32, 197, 127, 5, 51, 32, 80, 76, 85, 2, 205, 158, 1, 3, 80, 76, + 85, 8, 26, 65, 199, 184, 31, 85, 7, 11, 77, 5, 227, 159, 1, 32, 6, 250, + 77, 69, 154, 131, 29, 85, 163, 232, 1, 73, 10, 26, 69, 73, 2, 85, 78, 7, + 33, 6, 32, 80, 76, 85, 83, 32, 4, 150, 159, 24, 69, 215, 133, 7, 71, 5, + 11, 32, 2, 187, 101, 79, 14, 42, 72, 150, 216, 23, 65, 239, 206, 7, 85, + 8, 18, 69, 51, 73, 5, 237, 190, 30, 7, 32, 80, 76, 85, 83, 32, 84, 4, + 146, 183, 31, 68, 3, 77, 7, 11, 68, 5, 209, 214, 13, 5, 32, 80, 76, 85, + 83, 7, 11, 32, 4, 142, 218, 29, 82, 129, 175, 1, 11, 67, 82, 79, 83, 83, + 73, 78, 71, 32, 71, 65, 5, 191, 131, 1, 32, 7, 134, 131, 1, 32, 247, 161, + 30, 65, 11, 11, 50, 9, 11, 32, 6, 80, 8, 67, 82, 79, 83, 83, 73, 78, 71, + 0, 4, 79, 86, 69, 82, 211, 182, 26, 84, 2, 197, 49, 3, 32, 71, 65, 8, 44, + 5, 83, 72, 84, 73, 78, 207, 154, 24, 50, 7, 37, 7, 32, 84, 73, 77, 69, + 83, 32, 4, 250, 186, 30, 75, 215, 120, 85, 49, 70, 32, 94, 52, 114, 82, + 190, 1, 83, 146, 173, 30, 68, 211, 130, 1, 71, 6, 188, 253, 8, 6, 84, 73, + 77, 69, 83, 32, 205, 244, 4, 8, 67, 82, 79, 83, 83, 73, 78, 71, 7, 11, 32, 4, 64, 8, 67, 82, 79, 83, 83, 73, 78, 71, 1, 4, 79, 86, 69, 82, 2, - 225, 199, 23, 3, 32, 71, 73, 16, 26, 51, 131, 136, 1, 50, 13, 37, 7, 32, - 84, 73, 77, 69, 83, 32, 10, 70, 65, 0, 2, 76, 85, 190, 127, 71, 166, 187, - 12, 73, 155, 238, 16, 80, 2, 221, 186, 13, 7, 32, 80, 76, 85, 83, 32, 73, - 14, 26, 72, 203, 161, 29, 65, 13, 11, 32, 10, 22, 84, 247, 20, 67, 8, 44, - 5, 73, 77, 69, 83, 32, 159, 133, 30, 69, 6, 192, 20, 6, 71, 73, 83, 72, - 32, 67, 130, 126, 84, 175, 209, 28, 66, 43, 110, 50, 174, 1, 68, 218, 1, - 77, 54, 82, 176, 183, 13, 9, 32, 67, 82, 79, 83, 83, 73, 78, 71, 247, - 237, 16, 76, 15, 11, 32, 12, 48, 6, 84, 73, 77, 69, 83, 32, 151, 132, 1, - 71, 10, 252, 24, 3, 75, 65, 75, 178, 75, 73, 152, 20, 10, 83, 65, 76, 32, - 80, 76, 85, 83, 32, 84, 191, 175, 28, 78, 11, 11, 32, 8, 128, 1, 10, 80, - 76, 85, 83, 32, 71, 73, 83, 72, 32, 16, 6, 84, 73, 77, 69, 83, 32, 161, - 66, 8, 79, 86, 69, 82, 32, 71, 85, 68, 2, 247, 120, 84, 4, 156, 145, 1, - 5, 65, 32, 80, 76, 85, 175, 156, 28, 75, 5, 17, 2, 32, 84, 2, 225, 42, 4, - 73, 77, 69, 83, 9, 26, 85, 139, 165, 30, 55, 4, 246, 163, 30, 83, 147, 1, - 78, 50, 30, 65, 82, 73, 215, 1, 85, 11, 26, 32, 175, 164, 30, 76, 6, 32, - 2, 84, 69, 131, 128, 1, 71, 4, 219, 127, 78, 23, 37, 7, 32, 84, 73, 77, - 69, 83, 32, 20, 104, 3, 65, 83, 72, 162, 228, 26, 68, 174, 190, 2, 78, - 94, 75, 158, 57, 66, 2, 71, 150, 25, 83, 143, 45, 85, 7, 208, 55, 8, 32, - 79, 86, 69, 82, 32, 72, 73, 147, 235, 29, 50, 19, 48, 2, 66, 50, 158, - 190, 23, 76, 199, 226, 6, 83, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, - 238, 138, 1, 75, 150, 216, 25, 76, 158, 181, 2, 72, 214, 57, 65, 195, 9, - 85, 49, 104, 3, 68, 73, 77, 94, 71, 214, 1, 76, 62, 77, 238, 154, 30, 32, + 233, 150, 24, 3, 32, 71, 73, 16, 26, 51, 147, 136, 1, 50, 13, 37, 7, 32, + 84, 73, 77, 69, 83, 32, 10, 70, 65, 0, 2, 76, 85, 206, 127, 71, 246, 239, + 12, 73, 155, 190, 17, 80, 2, 189, 239, 13, 7, 32, 80, 76, 85, 83, 32, 73, + 14, 26, 72, 147, 166, 30, 65, 13, 11, 32, 10, 22, 84, 247, 20, 67, 8, 44, + 5, 73, 77, 69, 83, 32, 255, 137, 31, 69, 6, 192, 20, 6, 71, 73, 83, 72, + 32, 67, 146, 126, 84, 243, 213, 29, 66, 43, 110, 50, 174, 1, 68, 218, 1, + 77, 54, 82, 144, 236, 13, 9, 32, 67, 82, 79, 83, 83, 73, 78, 71, 247, + 189, 17, 76, 15, 11, 32, 12, 48, 6, 84, 73, 77, 69, 83, 32, 167, 132, 1, + 71, 10, 252, 24, 3, 75, 65, 75, 194, 75, 73, 152, 20, 10, 83, 65, 76, 32, + 80, 76, 85, 83, 32, 84, 247, 179, 29, 78, 11, 11, 32, 8, 128, 1, 10, 80, + 76, 85, 83, 32, 71, 73, 83, 72, 32, 16, 6, 84, 73, 77, 69, 83, 32, 177, + 66, 8, 79, 86, 69, 82, 32, 71, 85, 68, 2, 135, 121, 84, 4, 172, 145, 1, + 5, 65, 32, 80, 76, 85, 239, 160, 29, 75, 5, 17, 2, 32, 84, 2, 241, 42, 4, + 73, 77, 69, 83, 9, 26, 85, 235, 169, 31, 55, 4, 214, 168, 31, 83, 147, 1, + 78, 50, 30, 65, 82, 73, 215, 1, 85, 11, 26, 32, 143, 169, 31, 76, 6, 32, + 2, 84, 69, 147, 128, 1, 71, 4, 235, 127, 78, 23, 37, 7, 32, 84, 73, 77, + 69, 83, 32, 20, 104, 3, 65, 83, 72, 206, 198, 27, 68, 202, 224, 2, 78, + 94, 75, 170, 57, 66, 2, 71, 162, 25, 83, 143, 45, 85, 7, 224, 55, 8, 32, + 79, 86, 69, 82, 32, 72, 73, 227, 239, 30, 50, 19, 48, 2, 66, 50, 166, + 141, 24, 76, 159, 152, 7, 83, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, + 254, 138, 1, 75, 178, 186, 26, 76, 186, 215, 2, 72, 226, 57, 65, 195, 9, + 85, 49, 104, 3, 68, 73, 77, 94, 71, 214, 1, 76, 62, 77, 206, 159, 31, 32, 170, 1, 83, 146, 1, 66, 2, 78, 3, 82, 7, 53, 11, 32, 79, 86, 69, 82, 32, - 73, 68, 73, 77, 32, 4, 226, 166, 23, 83, 179, 128, 6, 66, 13, 11, 73, 11, - 11, 32, 8, 146, 123, 71, 168, 178, 11, 31, 79, 86, 69, 82, 32, 73, 71, + 73, 68, 73, 77, 32, 4, 202, 244, 23, 83, 155, 183, 6, 66, 13, 11, 73, 11, + 11, 32, 8, 162, 123, 71, 220, 230, 11, 31, 79, 86, 69, 82, 32, 73, 71, 73, 32, 83, 72, 73, 82, 32, 79, 86, 69, 82, 32, 83, 72, 73, 82, 32, 85, - 68, 32, 79, 86, 69, 82, 178, 183, 16, 68, 219, 166, 1, 82, 7, 26, 32, - 199, 157, 30, 50, 2, 153, 72, 4, 84, 73, 77, 69, 13, 26, 32, 183, 205, - 29, 73, 8, 76, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 146, 111, 84, - 183, 180, 22, 83, 2, 253, 153, 29, 5, 83, 73, 78, 71, 32, 222, 1, 78, 65, - 130, 15, 73, 242, 1, 85, 158, 73, 69, 237, 217, 22, 4, 87, 85, 51, 49, - 175, 1, 164, 1, 7, 32, 84, 73, 77, 69, 83, 32, 210, 9, 50, 66, 68, 102, - 75, 50, 76, 82, 77, 28, 4, 83, 75, 65, 76, 152, 140, 15, 6, 80, 32, 69, - 76, 65, 77, 235, 129, 15, 66, 134, 1, 170, 1, 65, 94, 66, 82, 69, 30, 71, - 182, 2, 73, 34, 75, 34, 76, 38, 77, 170, 1, 83, 118, 84, 34, 85, 198, 8, - 72, 246, 51, 78, 154, 157, 16, 80, 142, 147, 13, 82, 147, 17, 90, 13, 46, - 68, 154, 235, 29, 78, 165, 44, 2, 83, 72, 5, 213, 50, 7, 32, 80, 76, 85, - 83, 32, 75, 10, 34, 65, 226, 151, 30, 73, 3, 85, 6, 222, 177, 28, 76, - 130, 230, 1, 68, 3, 82, 4, 250, 24, 82, 151, 61, 83, 26, 30, 65, 98, 73, - 147, 1, 85, 11, 38, 82, 190, 123, 78, 151, 155, 29, 76, 5, 233, 21, 10, - 32, 80, 76, 85, 83, 32, 83, 72, 65, 51, 11, 32, 2, 83, 72, 163, 178, 23, - 82, 7, 11, 32, 4, 26, 67, 239, 130, 1, 80, 2, 37, 7, 82, 79, 83, 83, 73, - 78, 71, 2, 221, 214, 26, 2, 32, 71, 7, 242, 193, 26, 82, 151, 211, 3, 68, - 4, 138, 129, 30, 71, 219, 19, 77, 8, 234, 8, 73, 239, 232, 16, 65, 6, - 246, 238, 11, 85, 175, 165, 18, 73, 12, 18, 69, 83, 73, 9, 33, 6, 32, 80, - 76, 85, 83, 32, 6, 222, 238, 29, 68, 150, 14, 84, 255, 2, 71, 5, 45, 9, - 32, 80, 76, 85, 83, 32, 78, 85, 78, 2, 251, 155, 26, 85, 18, 62, 72, 246, - 174, 26, 65, 224, 234, 2, 2, 85, 72, 243, 119, 73, 10, 250, 149, 29, 85, - 138, 54, 73, 150, 70, 65, 3, 69, 4, 146, 217, 29, 65, 211, 56, 85, 17, - 110, 32, 222, 92, 82, 156, 182, 25, 9, 77, 85, 77, 32, 84, 73, 77, 69, - 83, 206, 252, 3, 83, 146, 1, 50, 3, 68, 2, 207, 190, 20, 85, 5, 177, 202, - 11, 11, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 75, 10, 42, 53, 214, 143, - 30, 50, 2, 51, 3, 52, 5, 249, 169, 23, 9, 32, 79, 86, 69, 82, 32, 75, 65, - 68, 5, 173, 75, 8, 32, 84, 73, 77, 69, 83, 32, 73, 7, 11, 32, 4, 166, - 105, 84, 233, 235, 22, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 4, 154, - 142, 30, 50, 3, 52, 7, 11, 32, 4, 70, 76, 1, 13, 79, 86, 69, 82, 32, 75, - 65, 83, 75, 65, 76, 32, 76, 2, 173, 116, 24, 65, 71, 65, 66, 32, 84, 73, - 77, 69, 83, 32, 85, 32, 79, 86, 69, 82, 32, 76, 65, 71, 65, 66, 32, 21, - 68, 7, 32, 84, 73, 77, 69, 83, 32, 50, 83, 198, 139, 30, 68, 3, 78, 6, - 26, 85, 187, 197, 29, 66, 5, 215, 139, 30, 68, 8, 52, 3, 73, 77, 53, 242, - 129, 29, 65, 159, 137, 1, 72, 5, 177, 165, 23, 11, 32, 79, 86, 69, 82, - 32, 75, 73, 83, 73, 77, 25, 200, 1, 29, 32, 79, 86, 69, 82, 32, 72, 73, - 32, 84, 73, 77, 69, 83, 32, 65, 83, 72, 50, 32, 75, 85, 32, 79, 86, 69, - 82, 32, 72, 18, 52, 54, 82, 186, 98, 83, 230, 1, 76, 162, 164, 29, 51, 2, - 55, 3, 78, 2, 155, 71, 73, 5, 157, 138, 29, 8, 32, 86, 65, 82, 73, 65, - 78, 84, 5, 213, 115, 9, 32, 79, 80, 80, 79, 83, 73, 78, 71, 240, 2, 30, - 65, 174, 26, 73, 59, 85, 141, 2, 68, 2, 71, 65, 252, 12, 2, 75, 45, 222, - 11, 76, 46, 77, 135, 55, 72, 118, 22, 66, 131, 11, 82, 109, 11, 32, 106, - 48, 6, 84, 73, 77, 69, 83, 32, 207, 141, 23, 83, 104, 190, 1, 65, 186, 1, - 66, 34, 71, 70, 72, 66, 73, 94, 75, 118, 76, 46, 77, 46, 83, 138, 2, 84, - 126, 85, 200, 150, 4, 8, 90, 85, 32, 79, 86, 69, 82, 32, 138, 196, 24, - 68, 214, 82, 69, 131, 57, 78, 15, 80, 6, 32, 80, 76, 85, 83, 32, 76, 4, - 83, 72, 32, 90, 226, 131, 30, 76, 3, 78, 6, 38, 68, 222, 225, 28, 71, - 131, 25, 76, 2, 201, 62, 5, 65, 32, 80, 76, 85, 2, 149, 112, 2, 73, 68, - 4, 178, 189, 29, 65, 151, 70, 73, 10, 48, 2, 85, 68, 210, 159, 26, 65, - 167, 227, 3, 73, 5, 191, 105, 32, 6, 244, 177, 17, 7, 73, 32, 84, 73, 77, - 69, 83, 211, 212, 11, 65, 8, 22, 77, 175, 62, 71, 7, 33, 6, 32, 80, 76, - 85, 83, 32, 4, 254, 220, 29, 76, 179, 34, 72, 10, 26, 85, 175, 137, 28, - 73, 6, 26, 76, 147, 129, 30, 51, 5, 41, 8, 32, 80, 76, 85, 83, 32, 72, - 73, 2, 219, 65, 32, 8, 198, 100, 65, 242, 182, 27, 73, 131, 105, 85, 6, - 26, 69, 243, 154, 28, 85, 5, 175, 25, 32, 12, 26, 72, 139, 239, 29, 85, - 10, 100, 14, 73, 84, 65, 32, 80, 76, 85, 83, 32, 71, 73, 83, 72, 32, 96, - 2, 85, 50, 217, 3, 2, 69, 32, 4, 48, 6, 80, 76, 85, 83, 32, 69, 223, 172, - 25, 84, 2, 11, 82, 2, 11, 73, 2, 215, 154, 23, 78, 5, 189, 73, 5, 32, 80, - 76, 85, 83, 6, 86, 65, 189, 30, 16, 69, 32, 80, 76, 85, 83, 32, 65, 32, - 80, 76, 85, 83, 32, 83, 85, 4, 162, 152, 23, 75, 251, 228, 6, 71, 13, 72, - 6, 32, 80, 76, 85, 83, 32, 190, 41, 50, 226, 209, 29, 83, 147, 1, 68, 4, - 26, 85, 147, 252, 29, 65, 2, 255, 40, 32, 11, 11, 32, 8, 68, 4, 71, 85, - 78, 85, 97, 9, 84, 73, 77, 69, 83, 32, 83, 72, 69, 5, 73, 16, 32, 79, 86, - 69, 82, 32, 76, 65, 71, 65, 82, 32, 71, 85, 78, 85, 2, 199, 205, 29, 32, - 5, 11, 32, 2, 229, 143, 14, 4, 80, 76, 85, 83, 136, 1, 82, 48, 146, 2, - 49, 30, 50, 114, 51, 82, 52, 222, 2, 54, 154, 4, 55, 243, 24, 53, 22, - 158, 1, 50, 30, 56, 180, 52, 15, 55, 57, 32, 79, 86, 69, 82, 32, 76, 65, - 75, 45, 48, 55, 57, 242, 199, 10, 53, 146, 152, 12, 54, 2, 57, 94, 51, - 143, 143, 3, 48, 4, 162, 248, 29, 49, 3, 53, 4, 144, 148, 23, 12, 49, 32, - 79, 86, 69, 82, 32, 76, 65, 75, 45, 48, 247, 227, 6, 48, 4, 222, 147, 23, - 52, 95, 51, 16, 46, 50, 38, 54, 230, 140, 23, 49, 159, 2, 51, 6, 230, - 246, 29, 48, 2, 53, 3, 56, 4, 194, 246, 29, 53, 3, 54, 12, 42, 52, 250, - 177, 11, 56, 143, 225, 11, 57, 6, 250, 245, 29, 51, 2, 55, 3, 56, 30, 62, - 52, 214, 1, 53, 30, 57, 178, 144, 23, 55, 143, 143, 3, 56, 14, 26, 57, - 255, 244, 29, 49, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, 116, 9, 80, - 65, 80, 32, 80, 76, 85, 83, 32, 208, 227, 12, 7, 85, 50, 32, 80, 76, 85, - 83, 190, 31, 73, 235, 203, 16, 71, 4, 210, 13, 80, 51, 76, 4, 194, 243, - 29, 48, 3, 55, 8, 166, 243, 29, 48, 2, 50, 2, 51, 3, 53, 46, 60, 2, 49, - 55, 240, 1, 2, 52, 56, 138, 137, 23, 48, 23, 51, 23, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 20, 122, 84, 34, 85, 162, 11, 75, 132, 34, 9, 68, 85, 78, - 51, 32, 71, 85, 78, 85, 186, 222, 27, 65, 254, 158, 1, 66, 235, 67, 76, - 4, 198, 184, 29, 65, 211, 56, 69, 6, 250, 146, 14, 82, 254, 221, 15, 50, - 3, 68, 21, 37, 7, 32, 84, 73, 77, 69, 83, 32, 18, 154, 1, 85, 220, 8, 4, - 80, 65, 80, 32, 146, 45, 73, 164, 167, 12, 10, 83, 72, 69, 83, 72, 32, - 80, 76, 85, 83, 250, 252, 4, 68, 170, 129, 12, 78, 163, 17, 71, 4, 138, - 145, 14, 82, 255, 221, 15, 68, 4, 234, 137, 23, 50, 211, 145, 3, 52, 5, - 11, 32, 2, 145, 6, 4, 84, 73, 77, 69, 7, 49, 10, 32, 84, 73, 77, 69, 83, - 32, 75, 85, 82, 5, 221, 160, 11, 5, 32, 80, 76, 85, 83, 9, 200, 160, 11, - 2, 77, 77, 222, 203, 18, 83, 147, 1, 76, 93, 90, 50, 212, 7, 3, 71, 65, - 76, 142, 1, 77, 130, 62, 32, 198, 165, 29, 51, 2, 72, 3, 76, 69, 11, 32, - 66, 88, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 44, 4, 71, 85, 78, 85, - 38, 83, 35, 84, 2, 205, 158, 11, 6, 83, 73, 78, 71, 32, 76, 2, 193, 24, - 5, 32, 84, 73, 77, 69, 6, 250, 68, 72, 139, 173, 22, 81, 54, 44, 5, 73, - 77, 69, 83, 32, 171, 197, 29, 69, 52, 188, 1, 4, 69, 83, 72, 50, 94, 72, - 42, 75, 68, 2, 76, 65, 34, 77, 60, 3, 80, 65, 80, 118, 83, 90, 84, 250, - 56, 71, 158, 230, 7, 78, 150, 135, 18, 68, 170, 181, 2, 65, 254, 66, 66, - 203, 53, 73, 7, 11, 32, 4, 26, 80, 207, 150, 25, 84, 2, 17, 2, 76, 85, 2, - 245, 222, 28, 3, 83, 32, 76, 4, 184, 66, 2, 73, 32, 171, 156, 28, 65, 8, - 32, 2, 65, 68, 183, 231, 29, 73, 6, 226, 19, 51, 211, 211, 29, 50, 4, - 174, 20, 32, 171, 249, 16, 71, 2, 11, 69, 2, 11, 32, 2, 209, 251, 12, 4, - 80, 76, 85, 83, 5, 11, 32, 2, 33, 6, 80, 76, 85, 83, 32, 80, 2, 45, 9, - 65, 80, 32, 80, 76, 85, 83, 32, 76, 2, 235, 145, 26, 85, 6, 26, 73, 195, - 192, 29, 72, 4, 194, 18, 32, 157, 211, 25, 7, 75, 50, 32, 80, 76, 85, 83, - 4, 162, 53, 85, 139, 24, 65, 9, 11, 32, 6, 22, 79, 207, 67, 83, 4, 56, 7, - 80, 80, 79, 83, 73, 78, 71, 1, 3, 86, 69, 82, 2, 21, 3, 32, 76, 85, 2, - 175, 218, 28, 71, 7, 45, 9, 32, 79, 86, 69, 82, 32, 76, 85, 77, 5, 187, - 59, 32, 70, 34, 65, 70, 69, 22, 73, 71, 85, 17, 170, 45, 32, 188, 1, 2, - 83, 72, 250, 179, 29, 50, 2, 72, 3, 82, 7, 187, 238, 25, 83, 7, 240, 142, - 26, 8, 32, 80, 76, 85, 83, 32, 90, 65, 151, 211, 3, 78, 43, 104, 2, 83, - 72, 186, 60, 71, 140, 215, 10, 5, 32, 79, 86, 69, 82, 40, 2, 82, 71, 237, - 183, 6, 2, 78, 83, 31, 22, 32, 183, 2, 51, 16, 136, 1, 9, 79, 86, 69, 82, - 32, 77, 85, 83, 72, 124, 6, 84, 73, 77, 69, 83, 32, 173, 254, 25, 10, 67, - 82, 79, 83, 83, 73, 78, 71, 32, 77, 9, 37, 7, 32, 84, 73, 77, 69, 83, 32, - 6, 42, 65, 254, 175, 27, 75, 175, 172, 2, 71, 2, 237, 142, 11, 5, 32, 80, - 76, 85, 83, 6, 242, 229, 28, 75, 142, 118, 90, 187, 2, 65, 13, 11, 32, - 10, 44, 6, 84, 73, 77, 69, 83, 32, 203, 57, 71, 8, 38, 65, 210, 201, 29, - 68, 163, 17, 90, 5, 173, 205, 12, 5, 32, 80, 76, 85, 83, 158, 1, 42, 65, - 134, 2, 69, 94, 73, 199, 7, 85, 23, 52, 2, 71, 65, 166, 1, 77, 246, 218, - 29, 50, 3, 52, 11, 26, 32, 255, 219, 29, 82, 6, 100, 8, 79, 80, 80, 79, - 83, 73, 78, 71, 146, 59, 73, 197, 14, 9, 84, 73, 77, 69, 83, 32, 83, 72, - 85, 2, 217, 152, 29, 3, 32, 78, 65, 7, 204, 21, 2, 32, 78, 167, 197, 29, - 50, 9, 11, 32, 6, 44, 6, 84, 73, 77, 69, 83, 32, 179, 57, 83, 4, 250, - 147, 29, 85, 151, 70, 65, 73, 90, 77, 78, 78, 204, 234, 19, 6, 32, 84, - 73, 77, 69, 83, 254, 135, 8, 83, 131, 230, 1, 50, 7, 45, 9, 32, 84, 73, - 77, 69, 83, 32, 71, 65, 4, 158, 3, 82, 179, 58, 78, 59, 36, 3, 68, 65, - 50, 163, 216, 29, 57, 55, 37, 7, 32, 84, 73, 77, 69, 83, 32, 52, 162, 1, - 65, 34, 71, 50, 75, 16, 5, 76, 65, 75, 45, 48, 22, 77, 86, 78, 34, 80, - 76, 3, 83, 72, 69, 94, 85, 240, 15, 3, 68, 73, 77, 174, 186, 28, 66, 211, - 117, 72, 6, 246, 2, 83, 223, 211, 29, 78, 8, 26, 73, 255, 211, 28, 85, 5, - 135, 213, 29, 83, 2, 211, 20, 69, 2, 251, 242, 22, 53, 4, 26, 69, 171, - 240, 27, 65, 2, 25, 4, 32, 80, 76, 85, 2, 177, 41, 3, 83, 32, 71, 4, 182, - 133, 29, 85, 215, 79, 69, 2, 33, 6, 65, 80, 32, 80, 76, 85, 2, 11, 83, 2, - 229, 195, 28, 2, 32, 80, 9, 37, 7, 32, 80, 76, 85, 83, 32, 65, 6, 26, 83, - 171, 140, 28, 32, 4, 11, 72, 5, 107, 32, 11, 50, 32, 34, 50, 246, 244, - 13, 82, 239, 220, 15, 83, 2, 233, 217, 13, 3, 80, 76, 85, 2, 11, 32, 2, - 21, 3, 80, 76, 85, 2, 11, 83, 2, 151, 237, 27, 32, 57, 24, 2, 49, 49, 99, - 78, 9, 11, 32, 6, 200, 25, 9, 79, 86, 69, 82, 32, 78, 85, 49, 49, 210, - 230, 24, 84, 191, 248, 2, 82, 47, 30, 32, 169, 3, 2, 85, 90, 18, 144, 1, - 12, 67, 82, 79, 83, 83, 73, 78, 71, 32, 78, 85, 78, 72, 12, 76, 65, 71, - 65, 82, 32, 84, 73, 77, 69, 83, 32, 174, 1, 79, 163, 252, 24, 84, 5, 209, - 40, 14, 32, 76, 65, 71, 65, 82, 32, 79, 86, 69, 82, 32, 76, 65, 10, 56, - 3, 83, 65, 76, 210, 233, 27, 77, 14, 85, 247, 66, 71, 5, 205, 197, 28, - 23, 32, 79, 86, 69, 82, 32, 78, 85, 78, 32, 76, 65, 71, 65, 82, 32, 84, - 73, 77, 69, 83, 32, 83, 2, 153, 253, 16, 3, 86, 69, 82, 27, 11, 32, 24, - 120, 10, 65, 66, 50, 32, 84, 73, 77, 69, 83, 32, 205, 37, 15, 75, 73, 83, - 73, 77, 53, 32, 84, 73, 77, 69, 83, 32, 66, 73, 20, 168, 1, 2, 75, 65, - 202, 7, 73, 212, 39, 2, 65, 83, 194, 171, 2, 68, 164, 166, 8, 3, 83, 73, - 76, 226, 230, 11, 85, 150, 185, 5, 71, 238, 147, 1, 78, 254, 2, 66, 163, - 17, 76, 2, 199, 247, 25, 68, 46, 42, 65, 36, 4, 69, 83, 72, 50, 23, 73, - 9, 242, 202, 29, 68, 2, 78, 3, 80, 5, 135, 186, 27, 32, 35, 22, 32, 151, - 1, 82, 20, 80, 6, 84, 73, 77, 69, 83, 32, 205, 199, 17, 8, 67, 82, 79, - 83, 83, 73, 78, 71, 18, 214, 21, 85, 146, 48, 65, 2, 73, 190, 195, 28, - 66, 175, 64, 69, 12, 32, 2, 73, 71, 239, 200, 29, 50, 11, 11, 32, 8, 96, - 6, 84, 73, 77, 69, 83, 32, 173, 193, 25, 12, 79, 80, 80, 79, 83, 73, 78, - 71, 32, 80, 73, 82, 6, 210, 190, 28, 75, 150, 67, 85, 223, 67, 90, 8, - 230, 67, 65, 234, 131, 29, 73, 3, 85, 202, 1, 46, 65, 250, 5, 72, 150, - 11, 73, 163, 1, 85, 63, 46, 71, 190, 4, 76, 122, 78, 147, 193, 29, 82, - 53, 11, 32, 50, 92, 4, 71, 85, 78, 85, 54, 78, 32, 6, 84, 73, 77, 69, 83, - 32, 241, 21, 4, 79, 86, 69, 82, 5, 29, 5, 32, 84, 73, 77, 69, 2, 167, - 170, 17, 83, 2, 177, 203, 24, 3, 85, 84, 73, 42, 150, 1, 73, 42, 75, 34, - 83, 64, 2, 84, 65, 38, 85, 216, 62, 2, 68, 85, 246, 219, 27, 76, 222, 39, - 78, 198, 48, 69, 254, 59, 77, 162, 17, 72, 187, 2, 65, 2, 11, 71, 2, 11, - 73, 2, 191, 31, 32, 4, 242, 138, 29, 85, 187, 53, 65, 6, 26, 72, 239, - 185, 28, 65, 4, 234, 139, 13, 69, 139, 241, 15, 73, 4, 234, 221, 22, 75, - 251, 228, 6, 66, 10, 174, 193, 29, 83, 146, 1, 50, 2, 66, 2, 77, 3, 82, - 5, 33, 6, 32, 76, 65, 71, 65, 66, 2, 37, 7, 32, 84, 73, 77, 69, 83, 32, - 2, 11, 65, 2, 11, 83, 2, 207, 221, 22, 72, 2, 131, 251, 10, 71, 106, 46, - 65, 250, 1, 69, 242, 2, 73, 239, 3, 85, 29, 50, 51, 182, 1, 54, 182, 215, - 22, 66, 223, 3, 82, 19, 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 90, 85, - 146, 25, 83, 194, 231, 25, 71, 130, 198, 2, 84, 166, 50, 66, 206, 47, 78, - 215, 22, 65, 5, 11, 32, 2, 129, 171, 25, 4, 80, 76, 85, 83, 5, 175, 45, - 32, 27, 62, 32, 140, 2, 2, 83, 72, 226, 232, 25, 71, 171, 211, 3, 78, 14, - 84, 8, 79, 86, 69, 82, 32, 83, 72, 69, 88, 5, 80, 76, 85, 83, 32, 255, - 151, 29, 72, 7, 11, 32, 4, 190, 15, 71, 141, 6, 12, 84, 65, 66, 32, 79, - 86, 69, 82, 32, 84, 65, 66, 6, 48, 2, 72, 85, 20, 2, 78, 65, 247, 153, - 28, 83, 2, 219, 216, 22, 66, 2, 199, 216, 22, 77, 7, 202, 130, 28, 76, - 191, 185, 1, 50, 40, 62, 68, 74, 77, 218, 1, 82, 182, 178, 25, 78, 155, - 132, 4, 84, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 162, 170, 29, 73, - 219, 16, 65, 25, 37, 7, 32, 84, 73, 77, 69, 83, 32, 22, 114, 66, 38, 73, - 130, 19, 75, 198, 181, 2, 77, 190, 198, 2, 76, 246, 135, 23, 71, 130, 25, - 83, 238, 9, 68, 191, 127, 65, 4, 218, 200, 2, 85, 203, 231, 25, 65, 4, - 249, 20, 2, 71, 73, 7, 11, 32, 4, 60, 9, 79, 86, 69, 82, 32, 83, 72, 73, - 82, 211, 230, 24, 84, 2, 241, 191, 28, 11, 32, 66, 85, 82, 32, 79, 86, - 69, 82, 32, 66, 13, 88, 14, 32, 79, 86, 69, 82, 32, 73, 78, 86, 69, 82, - 84, 69, 68, 34, 50, 187, 190, 28, 66, 2, 11, 32, 2, 167, 245, 24, 83, 7, - 33, 6, 32, 80, 76, 85, 83, 32, 4, 88, 7, 69, 50, 32, 84, 73, 77, 69, 173, - 234, 10, 9, 68, 85, 71, 32, 84, 73, 77, 69, 83, 2, 235, 207, 12, 83, 17, - 50, 32, 30, 71, 230, 234, 10, 76, 227, 230, 11, 75, 4, 138, 8, 84, 163, - 9, 71, 7, 11, 52, 5, 49, 10, 32, 79, 86, 69, 82, 32, 83, 73, 71, 52, 2, - 199, 14, 32, 19, 78, 68, 22, 77, 22, 82, 252, 164, 12, 5, 32, 79, 86, 69, - 82, 147, 150, 16, 72, 5, 247, 179, 29, 50, 5, 199, 206, 27, 65, 5, 207, - 179, 29, 57, 72, 46, 65, 150, 4, 73, 250, 1, 85, 227, 8, 69, 37, 66, 32, - 94, 66, 166, 1, 71, 148, 1, 2, 75, 52, 183, 175, 29, 82, 8, 60, 6, 84, - 73, 77, 69, 83, 32, 130, 14, 71, 207, 147, 27, 65, 4, 174, 158, 29, 72, - 3, 77, 7, 11, 32, 4, 188, 143, 17, 29, 79, 86, 69, 82, 32, 84, 65, 66, - 32, 78, 73, 32, 79, 86, 69, 82, 32, 78, 73, 32, 68, 73, 83, 72, 32, 79, - 86, 69, 82, 175, 169, 5, 83, 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, - 74, 84, 252, 233, 7, 2, 83, 72, 154, 155, 20, 71, 174, 100, 85, 191, 50, - 66, 2, 11, 85, 2, 243, 203, 22, 71, 5, 29, 5, 32, 80, 76, 85, 83, 2, 145, - 201, 27, 2, 32, 83, 17, 46, 82, 150, 29, 32, 182, 145, 29, 50, 3, 76, 9, - 11, 32, 6, 48, 8, 79, 86, 69, 82, 32, 84, 73, 82, 99, 84, 5, 11, 32, 2, - 11, 71, 2, 21, 3, 65, 68, 32, 2, 241, 5, 8, 79, 86, 69, 82, 32, 71, 65, - 68, 2, 217, 21, 6, 73, 77, 69, 83, 32, 84, 17, 50, 77, 114, 82, 138, 200, - 22, 71, 215, 227, 6, 75, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 46, 71, - 213, 137, 17, 5, 84, 72, 82, 69, 69, 2, 221, 16, 2, 65, 78, 5, 137, 221, - 10, 17, 32, 79, 86, 69, 82, 32, 84, 85, 82, 32, 90, 65, 32, 79, 86, 69, - 82, 179, 1, 138, 1, 32, 246, 2, 68, 226, 2, 78, 46, 77, 158, 2, 82, 128, - 9, 2, 83, 72, 174, 1, 90, 192, 133, 12, 2, 84, 85, 238, 145, 17, 50, 3, - 66, 12, 64, 7, 79, 86, 69, 82, 32, 85, 32, 158, 2, 85, 215, 252, 27, 71, - 6, 200, 1, 10, 80, 65, 32, 79, 86, 69, 82, 32, 80, 65, 168, 247, 8, 19, - 85, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, 86, 69, 82, 32, 85, 32, - 82, 249, 183, 19, 10, 83, 85, 82, 32, 79, 86, 69, 82, 32, 83, 2, 11, 32, - 2, 45, 9, 71, 65, 82, 32, 79, 86, 69, 82, 32, 2, 255, 132, 28, 71, 5, - 171, 130, 29, 32, 21, 26, 32, 147, 166, 29, 85, 16, 70, 75, 44, 2, 83, - 72, 100, 6, 84, 73, 77, 69, 83, 32, 135, 1, 71, 2, 11, 85, 2, 11, 83, 2, - 151, 217, 10, 72, 4, 29, 5, 69, 83, 72, 73, 71, 5, 11, 32, 2, 11, 84, 2, - 149, 223, 28, 6, 73, 77, 69, 83, 32, 66, 8, 92, 14, 85, 32, 80, 76, 85, - 83, 32, 85, 32, 80, 76, 85, 83, 32, 142, 222, 28, 66, 203, 50, 77, 4, 11, - 85, 5, 11, 32, 2, 11, 71, 2, 147, 255, 28, 85, 21, 72, 7, 32, 84, 73, 77, - 69, 83, 32, 136, 1, 2, 85, 77, 231, 162, 28, 66, 10, 50, 76, 16, 2, 77, - 69, 50, 83, 183, 162, 29, 85, 2, 231, 6, 65, 5, 11, 32, 2, 177, 153, 25, - 4, 80, 76, 85, 83, 2, 255, 215, 10, 72, 7, 37, 7, 32, 84, 73, 77, 69, 83, - 32, 4, 158, 11, 75, 163, 148, 29, 80, 93, 54, 32, 102, 50, 194, 2, 73, - 22, 85, 235, 157, 29, 52, 4, 62, 83, 157, 168, 28, 9, 67, 82, 79, 83, 83, - 73, 78, 71, 32, 2, 141, 154, 25, 4, 72, 69, 83, 72, 23, 11, 32, 20, 42, - 73, 37, 6, 84, 73, 77, 69, 83, 32, 2, 165, 168, 23, 4, 78, 86, 69, 82, - 18, 46, 65, 82, 85, 202, 158, 28, 78, 227, 125, 72, 6, 48, 6, 32, 80, 76, - 85, 83, 32, 255, 158, 29, 76, 4, 194, 156, 29, 72, 3, 78, 8, 26, 50, 199, - 158, 29, 68, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, 130, 185, 27, 65, 199, - 209, 1, 66, 5, 251, 157, 29, 51, 59, 56, 7, 32, 84, 73, 77, 69, 83, 32, - 165, 4, 2, 68, 65, 52, 134, 1, 65, 46, 68, 38, 71, 82, 73, 46, 76, 78, - 83, 46, 85, 186, 247, 27, 66, 238, 34, 77, 214, 90, 84, 146, 17, 75, 162, - 17, 72, 3, 80, 5, 11, 83, 2, 11, 72, 2, 207, 194, 16, 71, 4, 186, 208, - 10, 65, 235, 251, 17, 85, 10, 26, 65, 187, 155, 29, 85, 9, 34, 78, 150, - 155, 29, 76, 3, 82, 2, 211, 9, 50, 6, 170, 135, 29, 71, 202, 18, 83, 147, - 1, 77, 6, 46, 85, 229, 178, 22, 5, 65, 75, 45, 54, 54, 4, 166, 154, 29, - 51, 3, 77, 4, 144, 181, 22, 2, 73, 71, 167, 206, 6, 72, 6, 42, 32, 186, - 187, 13, 82, 255, 221, 15, 68, 2, 241, 238, 27, 6, 80, 76, 85, 83, 32, - 71, 5, 11, 32, 2, 213, 159, 13, 4, 84, 73, 77, 69, 17, 84, 7, 32, 84, 73, - 77, 69, 83, 32, 244, 139, 28, 2, 85, 77, 142, 140, 1, 50, 3, 88, 8, 50, - 84, 168, 180, 25, 2, 75, 85, 167, 227, 3, 65, 2, 11, 65, 2, 199, 178, 22, - 75, 6, 26, 51, 147, 151, 29, 85, 5, 29, 5, 32, 84, 73, 77, 69, 2, 21, 3, - 83, 32, 75, 2, 11, 65, 2, 251, 220, 22, 83, 42, 50, 65, 190, 1, 73, 146, - 1, 85, 231, 175, 22, 69, 13, 50, 32, 198, 137, 28, 77, 142, 140, 1, 55, - 3, 71, 4, 56, 8, 83, 81, 85, 65, 82, 69, 68, 32, 175, 195, 24, 84, 2, 11, - 84, 2, 21, 3, 73, 77, 69, 2, 11, 83, 2, 157, 156, 28, 2, 32, 75, 15, 86, - 66, 132, 152, 22, 6, 32, 79, 86, 69, 82, 32, 154, 24, 90, 214, 227, 6, - 51, 3, 71, 5, 17, 2, 32, 75, 2, 17, 2, 65, 66, 2, 135, 2, 65, 15, 84, 10, - 32, 79, 86, 69, 82, 32, 90, 85, 32, 80, 42, 53, 246, 153, 28, 66, 199, - 120, 77, 2, 181, 240, 27, 5, 76, 85, 83, 32, 83, 7, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 4, 44, 5, 84, 72, 82, 69, 69, 227, 145, 29, 65, 2, 29, 5, - 32, 68, 73, 83, 72, 2, 11, 32, 2, 203, 191, 24, 84, 8, 42, 32, 194, 150, - 22, 73, 187, 180, 3, 67, 4, 202, 212, 21, 79, 141, 190, 6, 8, 87, 73, 84, - 72, 32, 83, 84, 82, 18, 134, 1, 76, 154, 1, 82, 189, 202, 28, 23, 86, 69, - 68, 32, 83, 84, 69, 77, 32, 80, 65, 82, 65, 71, 82, 65, 80, 72, 32, 83, - 73, 71, 78, 10, 48, 2, 89, 32, 253, 147, 3, 4, 73, 78, 71, 32, 8, 60, 2, - 76, 79, 177, 128, 6, 7, 66, 82, 65, 67, 75, 69, 84, 6, 210, 226, 25, 71, - 215, 171, 3, 79, 6, 52, 5, 69, 78, 67, 89, 32, 53, 4, 89, 32, 65, 78, 4, - 248, 158, 24, 4, 69, 88, 67, 72, 143, 232, 3, 83, 2, 133, 160, 19, 3, 68, - 32, 82, 4, 174, 246, 27, 65, 133, 89, 2, 79, 77, 236, 8, 128, 1, 2, 80, - 82, 140, 9, 7, 82, 73, 76, 76, 73, 67, 32, 244, 134, 25, 7, 76, 73, 78, - 68, 82, 73, 67, 173, 210, 2, 2, 67, 76, 180, 2, 140, 1, 13, 73, 79, 84, - 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 169, 1, 16, 79, 45, 77, 73, 78, - 79, 65, 78, 32, 83, 73, 71, 78, 32, 67, 77, 110, 254, 156, 7, 75, 2, 76, - 2, 77, 2, 78, 2, 80, 2, 82, 2, 83, 2, 84, 126, 87, 254, 41, 74, 2, 90, - 194, 206, 6, 88, 138, 244, 14, 65, 2, 69, 2, 73, 2, 79, 3, 85, 198, 1, - 46, 48, 146, 5, 49, 141, 212, 23, 2, 51, 48, 170, 1, 102, 48, 78, 49, 74, - 50, 78, 51, 78, 52, 62, 53, 86, 55, 190, 140, 10, 57, 190, 3, 54, 179, - 234, 13, 56, 16, 246, 135, 29, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, - 56, 3, 57, 16, 194, 3, 50, 234, 131, 29, 48, 2, 49, 2, 51, 2, 53, 2, 55, - 3, 57, 16, 226, 134, 29, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, - 57, 16, 150, 134, 29, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, - 57, 12, 202, 133, 29, 48, 2, 49, 2, 52, 2, 54, 2, 55, 3, 57, 18, 142, - 133, 29, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 20, - 82, 53, 234, 131, 29, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 56, 3, - 57, 5, 231, 131, 29, 66, 24, 18, 48, 87, 49, 18, 190, 131, 29, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 6, 234, 130, 29, 48, 2, - 50, 3, 52, 180, 6, 140, 1, 9, 67, 65, 80, 73, 84, 65, 76, 32, 76, 194, 4, - 75, 32, 7, 76, 69, 84, 84, 69, 82, 32, 132, 1, 3, 80, 65, 89, 30, 83, - 203, 41, 84, 240, 2, 44, 6, 69, 84, 84, 69, 82, 32, 159, 43, 73, 234, 2, - 150, 2, 76, 46, 78, 34, 80, 34, 82, 58, 84, 54, 85, 190, 5, 65, 214, 1, - 66, 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, 72, 134, 1, - 73, 230, 3, 75, 226, 3, 77, 218, 1, 79, 226, 2, 83, 210, 7, 89, 166, 1, - 90, 230, 178, 28, 70, 134, 14, 74, 2, 86, 2, 87, 159, 20, 81, 6, 214, 25, - 73, 186, 206, 28, 74, 159, 20, 72, 4, 254, 26, 69, 231, 204, 28, 74, 8, - 246, 33, 69, 203, 200, 28, 83, 12, 212, 9, 3, 79, 85, 78, 226, 18, 69, - 139, 223, 28, 72, 18, 158, 33, 69, 106, 83, 174, 174, 28, 67, 187, 22, - 87, 13, 190, 34, 32, 115, 75, 2, 137, 244, 28, 3, 65, 86, 89, 6, 248, 10, - 5, 77, 85, 76, 84, 73, 156, 205, 12, 2, 80, 65, 229, 195, 11, 14, 83, 77, - 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, 2, 237, 209, 28, 2, 69, - 82, 184, 3, 136, 1, 6, 77, 65, 76, 76, 32, 76, 193, 37, 22, 85, 66, 83, - 67, 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, - 32, 132, 3, 44, 6, 69, 84, 84, 69, 82, 32, 143, 36, 73, 254, 2, 154, 2, - 65, 214, 1, 66, 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, - 72, 134, 1, 73, 230, 3, 75, 222, 2, 76, 134, 1, 77, 114, 78, 106, 79, - 110, 80, 50, 82, 198, 1, 83, 218, 2, 84, 210, 2, 85, 246, 1, 87, 54, 89, - 166, 1, 90, 230, 178, 28, 70, 134, 14, 74, 2, 86, 159, 20, 81, 17, 108, - 6, 32, 87, 73, 84, 72, 32, 36, 9, 66, 75, 72, 65, 83, 73, 65, 78, 32, - 229, 228, 11, 4, 76, 69, 85, 84, 4, 138, 154, 9, 68, 151, 170, 3, 66, 8, - 128, 209, 12, 3, 67, 72, 69, 250, 189, 6, 68, 139, 229, 9, 72, 18, 110, - 65, 78, 73, 32, 3, 82, 79, 65, 128, 146, 3, 6, 76, 69, 78, 68, 69, 68, - 178, 187, 9, 89, 243, 166, 16, 69, 6, 200, 21, 6, 82, 82, 69, 68, 32, 79, - 233, 205, 11, 5, 83, 72, 75, 73, 82, 4, 230, 2, 78, 187, 143, 3, 71, 2, - 179, 133, 10, 68, 14, 76, 2, 72, 69, 246, 9, 76, 164, 222, 12, 4, 82, 79, - 83, 83, 155, 222, 15, 67, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 142, 29, - 68, 195, 165, 28, 86, 26, 96, 2, 74, 69, 20, 6, 79, 85, 66, 76, 69, 32, - 66, 90, 182, 196, 28, 67, 186, 22, 87, 215, 22, 69, 5, 239, 151, 22, 82, - 4, 36, 3, 77, 79, 78, 219, 241, 28, 79, 2, 153, 13, 2, 79, 67, 12, 46, - 69, 138, 241, 27, 90, 182, 105, 72, 3, 87, 5, 155, 210, 28, 76, 41, 86, - 76, 98, 78, 110, 82, 166, 15, 32, 254, 185, 12, 83, 250, 247, 13, 77, - 195, 173, 2, 70, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 158, 20, 77, 222, - 182, 12, 68, 134, 248, 13, 84, 163, 92, 72, 13, 33, 6, 32, 87, 73, 84, - 72, 32, 10, 190, 19, 77, 222, 182, 12, 68, 202, 52, 76, 190, 195, 13, 84, - 163, 92, 72, 5, 229, 191, 27, 5, 32, 87, 73, 84, 72, 14, 32, 2, 72, 69, - 187, 215, 28, 74, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 134, 18, 77, - 146, 15, 85, 206, 167, 12, 68, 143, 31, 83, 12, 26, 65, 195, 214, 28, 87, - 11, 48, 6, 32, 87, 73, 84, 72, 32, 143, 135, 27, 82, 6, 218, 199, 12, 68, - 166, 212, 14, 72, 231, 160, 1, 83, 35, 84, 6, 32, 87, 73, 84, 72, 32, 50, - 69, 74, 79, 201, 1, 6, 90, 72, 73, 84, 83, 65, 6, 254, 142, 9, 68, 214, - 10, 71, 239, 184, 3, 77, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 254, 152, - 9, 71, 195, 159, 3, 66, 17, 11, 84, 14, 48, 6, 73, 70, 73, 69, 68, 32, - 159, 234, 28, 65, 12, 80, 2, 67, 76, 38, 76, 214, 134, 3, 66, 130, 207, - 24, 89, 210, 147, 1, 65, 3, 69, 2, 33, 6, 79, 83, 69, 68, 32, 76, 2, 151, - 4, 73, 5, 145, 212, 12, 14, 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, - 69, 32, 71, 34, 102, 65, 98, 79, 244, 185, 28, 11, 72, 65, 75, 65, 83, - 83, 73, 65, 78, 32, 67, 186, 22, 74, 255, 2, 83, 11, 33, 6, 32, 87, 73, - 84, 72, 32, 8, 182, 194, 12, 68, 166, 212, 14, 72, 154, 160, 1, 86, 79, - 83, 18, 36, 3, 77, 73, 32, 251, 193, 25, 80, 16, 58, 68, 198, 236, 11, - 76, 2, 78, 2, 83, 2, 84, 3, 90, 6, 194, 236, 11, 90, 134, 227, 16, 74, - 215, 22, 69, 8, 94, 73, 236, 154, 10, 10, 79, 78, 71, 45, 76, 69, 71, 71, - 69, 68, 206, 179, 18, 74, 159, 20, 72, 2, 233, 130, 3, 4, 84, 84, 76, 69, - 4, 21, 3, 79, 78, 79, 4, 18, 67, 39, 71, 2, 205, 188, 18, 4, 85, 76, 65, - 82, 2, 237, 10, 4, 82, 65, 80, 72, 6, 62, 69, 204, 187, 18, 5, 65, 82, - 82, 79, 87, 155, 145, 10, 74, 2, 249, 192, 12, 5, 85, 84, 82, 65, 76, 11, - 52, 4, 77, 69, 71, 65, 166, 3, 32, 183, 223, 28, 84, 5, 233, 185, 28, 8, - 32, 87, 73, 84, 72, 32, 84, 73, 10, 130, 6, 69, 230, 183, 12, 65, 231, - 144, 16, 83, 14, 50, 69, 100, 4, 79, 85, 78, 68, 167, 222, 28, 72, 8, 37, - 7, 86, 69, 82, 83, 69, 68, 32, 8, 214, 249, 18, 68, 206, 222, 8, 84, 142, - 100, 89, 151, 14, 90, 4, 250, 241, 9, 32, 241, 237, 1, 2, 69, 68, 34, - 108, 4, 67, 72, 87, 65, 34, 72, 132, 1, 4, 79, 70, 84, 32, 170, 186, 12, - 84, 213, 1, 5, 69, 77, 73, 83, 79, 5, 11, 32, 2, 183, 218, 5, 87, 16, 92, - 4, 79, 82, 84, 32, 136, 185, 12, 2, 72, 65, 254, 173, 14, 67, 234, 224, - 1, 87, 215, 22, 65, 6, 254, 176, 26, 73, 195, 173, 2, 85, 8, 38, 69, 166, - 215, 27, 83, 255, 111, 68, 4, 242, 221, 28, 76, 3, 77, 26, 132, 1, 4, 65, - 76, 76, 32, 50, 69, 106, 83, 212, 211, 11, 11, 72, 82, 69, 69, 45, 76, - 69, 71, 71, 69, 68, 218, 218, 16, 67, 187, 22, 87, 6, 178, 179, 12, 72, - 206, 149, 15, 89, 255, 124, 84, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, - 77, 223, 182, 12, 68, 2, 185, 169, 21, 5, 73, 68, 68, 76, 69, 8, 226, - 196, 28, 72, 2, 83, 2, 87, 215, 22, 69, 15, 58, 32, 114, 75, 53, 8, 78, - 66, 76, 69, 78, 68, 69, 68, 6, 29, 5, 87, 73, 84, 72, 32, 6, 26, 68, 255, - 192, 12, 77, 4, 172, 255, 8, 5, 79, 85, 66, 76, 69, 187, 8, 73, 5, 11, - 82, 2, 213, 4, 6, 65, 73, 78, 73, 65, 78, 2, 159, 144, 24, 32, 4, 176, - 197, 27, 4, 73, 68, 69, 32, 227, 147, 1, 69, 18, 62, 65, 28, 3, 69, 82, - 85, 134, 216, 28, 73, 2, 78, 3, 85, 7, 158, 216, 28, 69, 3, 84, 7, 33, 6, - 32, 87, 73, 84, 72, 32, 4, 138, 251, 8, 68, 131, 186, 3, 66, 18, 18, 69, - 71, 72, 9, 156, 1, 7, 32, 87, 73, 84, 72, 32, 68, 173, 211, 28, 2, 77, - 76, 10, 26, 69, 247, 191, 28, 87, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, - 26, 68, 203, 163, 12, 66, 4, 222, 131, 9, 73, 179, 173, 3, 69, 6, 37, 7, - 71, 65, 84, 85, 82, 69, 32, 6, 66, 65, 176, 140, 15, 2, 84, 69, 189, 155, - 13, 4, 69, 78, 32, 71, 2, 199, 185, 28, 32, 52, 198, 1, 66, 38, 68, 38, - 69, 36, 3, 71, 72, 69, 38, 72, 246, 188, 21, 89, 222, 157, 5, 83, 202, - 110, 84, 238, 8, 90, 246, 63, 73, 138, 19, 67, 186, 22, 80, 2, 86, 158, - 20, 75, 186, 2, 65, 2, 79, 3, 85, 4, 170, 172, 12, 89, 243, 166, 16, 69, - 6, 238, 210, 27, 90, 139, 128, 1, 69, 6, 210, 210, 28, 70, 2, 76, 3, 83, - 5, 201, 5, 5, 32, 87, 73, 84, 72, 4, 11, 65, 5, 171, 236, 26, 82, 2, 209, - 188, 15, 4, 72, 79, 85, 83, 184, 15, 178, 1, 65, 138, 4, 67, 54, 69, 198, - 35, 73, 250, 22, 79, 142, 44, 82, 174, 3, 85, 180, 180, 12, 13, 78, 65, - 32, 68, 79, 85, 66, 76, 69, 32, 72, 69, 76, 154, 231, 14, 86, 195, 47, - 76, 30, 116, 4, 71, 71, 69, 82, 146, 1, 78, 32, 4, 82, 75, 32, 83, 36, 2, - 83, 72, 200, 177, 21, 2, 76, 69, 155, 133, 1, 84, 9, 11, 32, 6, 44, 5, - 87, 73, 84, 72, 32, 143, 190, 6, 75, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, - 71, 72, 2, 201, 183, 27, 4, 84, 32, 71, 85, 4, 166, 144, 28, 67, 239, 30, - 71, 4, 254, 253, 20, 85, 175, 224, 4, 72, 10, 30, 32, 105, 3, 69, 68, 32, - 4, 60, 9, 87, 73, 84, 72, 32, 76, 69, 70, 84, 239, 185, 27, 83, 2, 17, 2, - 32, 85, 2, 175, 181, 22, 80, 6, 170, 215, 4, 84, 182, 147, 12, 76, 255, - 252, 9, 79, 10, 134, 204, 28, 49, 2, 50, 2, 51, 2, 52, 3, 83, 176, 4, - 182, 2, 67, 132, 2, 5, 71, 82, 69, 69, 32, 98, 76, 80, 21, 78, 84, 73, + 68, 32, 79, 86, 69, 82, 142, 134, 17, 68, 155, 168, 1, 82, 7, 26, 32, + 167, 162, 31, 50, 2, 169, 72, 4, 84, 73, 77, 69, 13, 26, 32, 139, 210, + 30, 73, 8, 76, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 162, 111, 84, + 143, 130, 23, 83, 2, 197, 158, 30, 5, 83, 73, 78, 71, 32, 224, 1, 78, 65, + 146, 15, 73, 242, 1, 85, 158, 73, 69, 229, 168, 23, 4, 87, 85, 51, 49, + 177, 1, 164, 1, 7, 32, 84, 73, 77, 69, 83, 32, 210, 9, 50, 66, 68, 102, + 75, 50, 76, 98, 77, 28, 4, 83, 75, 65, 76, 248, 192, 15, 6, 80, 32, 69, + 76, 65, 77, 219, 209, 15, 66, 134, 1, 170, 1, 65, 94, 66, 82, 69, 30, 71, + 182, 2, 73, 34, 75, 34, 76, 38, 77, 170, 1, 83, 118, 84, 34, 85, 214, 8, + 72, 246, 51, 78, 222, 215, 16, 80, 154, 221, 13, 82, 147, 17, 90, 13, 46, + 68, 250, 239, 30, 78, 165, 44, 2, 83, 72, 5, 229, 50, 7, 32, 80, 76, 85, + 83, 32, 75, 10, 34, 65, 194, 156, 31, 73, 3, 85, 6, 186, 179, 29, 76, + 134, 233, 1, 68, 3, 82, 4, 138, 25, 82, 151, 61, 83, 26, 30, 65, 98, 73, + 147, 1, 85, 11, 38, 82, 206, 123, 78, 231, 159, 30, 76, 5, 249, 21, 10, + 32, 80, 76, 85, 83, 32, 83, 72, 65, 51, 11, 32, 2, 83, 72, 171, 129, 24, + 82, 7, 11, 32, 4, 26, 67, 255, 130, 1, 80, 2, 37, 7, 82, 79, 83, 83, 73, + 78, 71, 2, 137, 185, 27, 2, 32, 71, 7, 182, 164, 27, 82, 179, 245, 3, 68, + 4, 234, 133, 31, 71, 219, 19, 77, 8, 250, 8, 73, 195, 164, 17, 65, 6, + 218, 163, 12, 85, 171, 245, 18, 73, 12, 18, 69, 83, 73, 9, 33, 6, 32, 80, + 76, 85, 83, 32, 6, 190, 243, 30, 68, 150, 14, 84, 255, 2, 71, 5, 45, 9, + 32, 80, 76, 85, 83, 32, 78, 85, 78, 2, 159, 244, 26, 85, 18, 62, 72, 178, + 196, 27, 65, 244, 217, 2, 2, 85, 72, 131, 120, 73, 10, 202, 154, 30, 85, + 142, 54, 73, 162, 70, 65, 3, 69, 4, 230, 221, 30, 65, 223, 56, 85, 17, + 110, 32, 238, 92, 82, 180, 142, 26, 9, 77, 85, 77, 32, 84, 73, 77, 69, + 83, 134, 169, 4, 83, 146, 1, 50, 3, 68, 2, 235, 137, 21, 85, 5, 197, 248, + 11, 11, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 75, 10, 42, 53, 182, 148, + 31, 50, 2, 51, 3, 52, 5, 129, 249, 23, 9, 32, 79, 86, 69, 82, 32, 75, 65, + 68, 5, 189, 75, 8, 32, 84, 73, 77, 69, 83, 32, 73, 9, 26, 32, 219, 130, + 31, 65, 4, 166, 105, 84, 249, 187, 23, 9, 67, 82, 79, 83, 83, 73, 78, 71, + 32, 4, 234, 146, 31, 50, 3, 52, 7, 11, 32, 4, 70, 76, 1, 13, 79, 86, 69, + 82, 32, 75, 65, 83, 75, 65, 76, 32, 76, 2, 173, 116, 24, 65, 71, 65, 66, + 32, 84, 73, 77, 69, 83, 32, 85, 32, 79, 86, 69, 82, 32, 76, 65, 71, 65, + 66, 32, 21, 68, 7, 32, 84, 73, 77, 69, 83, 32, 50, 83, 150, 144, 31, 68, + 3, 78, 6, 26, 85, 255, 201, 30, 66, 5, 167, 144, 31, 68, 8, 52, 3, 73, + 77, 53, 170, 134, 30, 65, 183, 137, 1, 72, 5, 169, 244, 23, 11, 32, 79, + 86, 69, 82, 32, 75, 73, 83, 73, 77, 25, 200, 1, 29, 32, 79, 86, 69, 82, + 32, 72, 73, 32, 84, 73, 77, 69, 83, 32, 65, 83, 72, 50, 32, 75, 85, 32, + 79, 86, 69, 82, 32, 72, 18, 52, 54, 82, 186, 98, 83, 230, 1, 76, 242, + 168, 30, 51, 2, 55, 3, 78, 2, 155, 71, 73, 5, 221, 142, 30, 8, 32, 86, + 65, 82, 73, 65, 78, 84, 5, 213, 115, 9, 32, 79, 80, 80, 79, 83, 73, 78, + 71, 240, 2, 30, 65, 174, 26, 73, 59, 85, 141, 2, 68, 2, 71, 65, 252, 12, + 2, 75, 45, 222, 11, 76, 46, 77, 135, 55, 72, 118, 22, 66, 131, 11, 82, + 109, 11, 32, 106, 48, 6, 84, 73, 77, 69, 83, 32, 167, 219, 23, 83, 104, + 190, 1, 65, 186, 1, 66, 34, 71, 70, 72, 66, 73, 94, 75, 118, 76, 46, 77, + 46, 83, 138, 2, 84, 126, 85, 188, 165, 4, 8, 90, 85, 32, 79, 86, 69, 82, + 32, 214, 186, 25, 68, 218, 81, 69, 143, 57, 78, 15, 80, 6, 32, 80, 76, + 85, 83, 32, 76, 4, 83, 72, 32, 90, 178, 136, 31, 76, 3, 78, 6, 38, 68, + 158, 231, 29, 71, 251, 23, 76, 2, 201, 62, 5, 65, 32, 80, 76, 85, 2, 149, + 112, 2, 73, 68, 4, 246, 193, 30, 65, 163, 70, 73, 10, 48, 2, 85, 68, 254, + 180, 27, 65, 203, 210, 3, 73, 5, 191, 105, 32, 6, 164, 238, 17, 7, 73, + 32, 84, 73, 77, 69, 83, 227, 156, 12, 65, 8, 22, 77, 175, 62, 71, 7, 33, + 6, 32, 80, 76, 85, 83, 32, 4, 206, 225, 30, 76, 179, 34, 72, 10, 26, 85, + 251, 138, 29, 73, 6, 26, 76, 227, 133, 31, 51, 5, 41, 8, 32, 80, 76, 85, + 83, 32, 72, 73, 2, 219, 65, 32, 8, 198, 100, 65, 190, 184, 28, 73, 247, + 107, 85, 6, 26, 69, 191, 156, 29, 85, 5, 175, 25, 32, 12, 26, 72, 219, + 243, 30, 85, 10, 100, 14, 73, 84, 65, 32, 80, 76, 85, 83, 32, 71, 73, 83, + 72, 32, 96, 2, 85, 50, 217, 3, 2, 69, 32, 4, 48, 6, 80, 76, 85, 83, 32, + 69, 163, 133, 26, 84, 2, 11, 82, 2, 11, 73, 2, 207, 233, 23, 78, 5, 189, + 73, 5, 32, 80, 76, 85, 83, 6, 86, 65, 189, 30, 16, 69, 32, 80, 76, 85, + 83, 32, 65, 32, 80, 76, 85, 83, 32, 83, 85, 4, 154, 231, 23, 75, 211, + 154, 7, 71, 13, 72, 6, 32, 80, 76, 85, 83, 32, 190, 41, 50, 178, 214, 30, + 83, 147, 1, 68, 4, 26, 85, 227, 128, 31, 65, 2, 255, 40, 32, 11, 11, 32, + 8, 68, 4, 71, 85, 78, 85, 97, 9, 84, 73, 77, 69, 83, 32, 83, 72, 69, 5, + 73, 16, 32, 79, 86, 69, 82, 32, 76, 65, 71, 65, 82, 32, 71, 85, 78, 85, + 2, 151, 210, 30, 32, 5, 11, 32, 2, 157, 196, 14, 4, 80, 76, 85, 83, 136, + 1, 82, 48, 146, 2, 49, 30, 50, 114, 51, 82, 52, 222, 2, 54, 154, 4, 55, + 243, 24, 53, 22, 158, 1, 50, 30, 56, 180, 52, 15, 55, 57, 32, 79, 86, 69, + 82, 32, 76, 65, 75, 45, 48, 55, 57, 246, 245, 10, 53, 134, 185, 12, 54, + 2, 57, 94, 51, 203, 162, 3, 48, 4, 242, 252, 30, 49, 3, 53, 4, 136, 227, + 23, 12, 49, 32, 79, 86, 69, 82, 32, 76, 65, 75, 45, 48, 207, 153, 7, 48, + 4, 214, 226, 23, 52, 95, 51, 16, 46, 50, 38, 54, 222, 219, 23, 49, 159, + 2, 51, 6, 182, 251, 30, 48, 2, 53, 3, 56, 4, 146, 251, 30, 53, 3, 54, 12, + 42, 52, 254, 223, 11, 56, 131, 130, 12, 57, 6, 202, 250, 30, 51, 2, 55, + 3, 56, 30, 62, 52, 214, 1, 53, 30, 57, 170, 223, 23, 55, 203, 162, 3, 56, + 14, 26, 57, 207, 249, 30, 49, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, + 116, 9, 80, 65, 80, 32, 80, 76, 85, 83, 32, 240, 151, 13, 7, 85, 50, 32, + 80, 76, 85, 83, 238, 31, 73, 235, 155, 17, 71, 4, 210, 13, 80, 51, 76, 4, + 146, 248, 30, 48, 3, 55, 8, 246, 247, 30, 48, 2, 50, 2, 51, 3, 53, 46, + 60, 2, 49, 55, 240, 1, 2, 52, 56, 130, 216, 23, 48, 23, 51, 23, 37, 7, + 32, 84, 73, 77, 69, 83, 32, 20, 122, 84, 34, 85, 162, 11, 75, 132, 34, 9, + 68, 85, 78, 51, 32, 71, 85, 78, 85, 134, 224, 28, 65, 246, 161, 1, 66, + 247, 67, 76, 4, 138, 189, 30, 65, 223, 56, 69, 6, 178, 199, 14, 82, 150, + 174, 16, 50, 3, 68, 21, 37, 7, 32, 84, 73, 77, 69, 83, 32, 18, 154, 1, + 85, 220, 8, 4, 80, 65, 80, 32, 146, 45, 73, 196, 219, 12, 10, 83, 72, 69, + 83, 72, 32, 80, 76, 85, 83, 210, 132, 5, 68, 130, 202, 12, 78, 163, 17, + 71, 4, 194, 197, 14, 82, 151, 174, 16, 68, 4, 226, 216, 23, 50, 143, 165, + 3, 52, 5, 11, 32, 2, 145, 6, 4, 84, 73, 77, 69, 7, 49, 10, 32, 84, 73, + 77, 69, 83, 32, 75, 85, 82, 5, 225, 206, 11, 5, 32, 80, 76, 85, 83, 9, + 204, 206, 11, 2, 77, 77, 170, 162, 19, 83, 147, 1, 76, 93, 90, 50, 212, + 7, 3, 71, 65, 76, 142, 1, 77, 130, 62, 32, 150, 170, 30, 51, 2, 72, 3, + 76, 69, 11, 32, 66, 88, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 44, 4, + 71, 85, 78, 85, 38, 83, 35, 84, 2, 209, 204, 11, 6, 83, 73, 78, 71, 32, + 76, 2, 193, 24, 5, 32, 84, 73, 77, 69, 6, 250, 68, 72, 227, 250, 22, 81, + 54, 44, 5, 73, 77, 69, 83, 32, 251, 201, 30, 69, 52, 188, 1, 4, 69, 83, + 72, 50, 94, 72, 42, 75, 68, 2, 76, 65, 34, 77, 60, 3, 80, 65, 80, 118, + 83, 90, 84, 250, 56, 71, 158, 251, 7, 78, 178, 212, 18, 68, 198, 215, 2, + 65, 138, 67, 66, 215, 53, 73, 7, 11, 32, 4, 26, 80, 147, 239, 25, 84, 2, + 17, 2, 76, 85, 2, 173, 227, 29, 3, 83, 32, 76, 4, 184, 66, 2, 73, 32, + 227, 160, 29, 65, 8, 32, 2, 65, 68, 135, 236, 30, 73, 6, 226, 19, 51, + 163, 216, 30, 50, 4, 174, 20, 32, 219, 181, 17, 71, 2, 11, 69, 2, 11, 32, + 2, 241, 175, 13, 4, 80, 76, 85, 83, 5, 11, 32, 2, 33, 6, 80, 76, 85, 83, + 32, 80, 2, 45, 9, 65, 80, 32, 80, 76, 85, 83, 32, 76, 2, 159, 244, 26, + 85, 6, 26, 73, 147, 197, 30, 72, 4, 194, 18, 32, 181, 171, 26, 7, 75, 50, + 32, 80, 76, 85, 83, 4, 162, 53, 85, 139, 24, 65, 9, 11, 32, 6, 22, 79, + 207, 67, 83, 4, 56, 7, 80, 80, 79, 83, 73, 78, 71, 1, 3, 86, 69, 82, 2, + 21, 3, 32, 76, 85, 2, 231, 222, 29, 71, 7, 45, 9, 32, 79, 86, 69, 82, 32, + 76, 85, 77, 5, 187, 59, 32, 70, 34, 65, 70, 69, 22, 73, 71, 85, 17, 170, + 45, 32, 188, 1, 2, 83, 72, 202, 184, 30, 50, 2, 72, 3, 82, 7, 207, 198, + 26, 83, 7, 164, 241, 26, 8, 32, 80, 76, 85, 83, 32, 90, 65, 179, 245, 3, + 78, 43, 104, 2, 83, 72, 186, 60, 71, 144, 133, 11, 5, 32, 79, 86, 69, 82, + 40, 2, 82, 71, 225, 197, 6, 2, 78, 83, 31, 22, 32, 183, 2, 51, 16, 136, + 1, 9, 79, 86, 69, 82, 32, 77, 85, 83, 72, 124, 6, 84, 73, 77, 69, 83, 32, + 209, 216, 26, 10, 67, 82, 79, 83, 83, 73, 78, 71, 32, 77, 9, 37, 7, 32, + 84, 73, 77, 69, 83, 32, 6, 42, 65, 238, 173, 28, 75, 143, 179, 2, 71, 2, + 241, 188, 11, 5, 32, 80, 76, 85, 83, 6, 178, 234, 29, 75, 158, 118, 90, + 187, 2, 65, 13, 11, 32, 10, 44, 6, 84, 73, 77, 69, 83, 32, 203, 57, 71, + 8, 38, 65, 162, 206, 30, 68, 163, 17, 90, 5, 205, 129, 13, 5, 32, 80, 76, + 85, 83, 158, 1, 42, 65, 134, 2, 69, 94, 73, 199, 7, 85, 23, 52, 2, 71, + 65, 166, 1, 77, 198, 223, 30, 50, 3, 52, 11, 26, 32, 207, 224, 30, 82, 6, + 100, 8, 79, 80, 80, 79, 83, 73, 78, 71, 146, 59, 73, 197, 14, 9, 84, 73, + 77, 69, 83, 32, 83, 72, 85, 2, 157, 157, 30, 3, 32, 78, 65, 7, 204, 21, + 2, 32, 78, 247, 201, 30, 50, 9, 11, 32, 6, 44, 6, 84, 73, 77, 69, 83, 32, + 179, 57, 83, 4, 190, 152, 30, 85, 163, 70, 65, 73, 90, 77, 78, 78, 168, + 182, 20, 6, 32, 84, 73, 77, 69, 83, 238, 189, 8, 83, 135, 233, 1, 50, 7, + 45, 9, 32, 84, 73, 77, 69, 83, 32, 71, 65, 4, 158, 3, 82, 179, 58, 78, + 59, 36, 3, 68, 65, 50, 243, 220, 30, 57, 55, 37, 7, 32, 84, 73, 77, 69, + 83, 32, 52, 162, 1, 65, 34, 71, 50, 75, 16, 5, 76, 65, 75, 45, 48, 22, + 77, 86, 78, 34, 80, 76, 3, 83, 72, 69, 94, 85, 240, 15, 3, 68, 73, 77, + 230, 190, 29, 66, 235, 117, 72, 6, 246, 2, 83, 175, 216, 30, 78, 8, 26, + 73, 183, 216, 29, 85, 5, 215, 217, 30, 83, 2, 211, 20, 69, 2, 243, 193, + 23, 53, 4, 26, 69, 247, 241, 28, 65, 2, 25, 4, 32, 80, 76, 85, 2, 177, + 41, 3, 83, 32, 71, 4, 250, 137, 30, 85, 227, 79, 69, 2, 33, 6, 65, 80, + 32, 80, 76, 85, 2, 11, 83, 2, 157, 200, 29, 2, 32, 80, 9, 37, 7, 32, 80, + 76, 85, 83, 32, 65, 6, 26, 83, 175, 142, 29, 32, 4, 11, 72, 5, 107, 32, + 11, 50, 32, 34, 50, 174, 169, 14, 82, 135, 173, 16, 83, 2, 173, 142, 14, + 3, 80, 76, 85, 2, 11, 32, 2, 21, 3, 80, 76, 85, 2, 11, 83, 2, 227, 238, + 28, 32, 57, 24, 2, 49, 49, 99, 78, 9, 11, 32, 6, 200, 25, 9, 79, 86, 69, + 82, 32, 78, 85, 49, 49, 150, 191, 25, 84, 207, 161, 3, 82, 47, 30, 32, + 169, 3, 2, 85, 90, 18, 144, 1, 12, 67, 82, 79, 83, 83, 73, 78, 71, 32, + 78, 85, 78, 72, 12, 76, 65, 71, 65, 82, 32, 84, 73, 77, 69, 83, 32, 174, + 1, 79, 231, 212, 25, 84, 5, 209, 40, 14, 32, 76, 65, 71, 65, 82, 32, 79, + 86, 69, 82, 32, 76, 65, 10, 56, 3, 83, 65, 76, 158, 235, 28, 77, 14, 85, + 235, 70, 71, 5, 133, 202, 29, 23, 32, 79, 86, 69, 82, 32, 78, 85, 78, 32, + 76, 65, 71, 65, 82, 32, 84, 73, 77, 69, 83, 32, 83, 2, 201, 185, 17, 3, + 86, 69, 82, 27, 11, 32, 24, 120, 10, 65, 66, 50, 32, 84, 73, 77, 69, 83, + 32, 205, 37, 15, 75, 73, 83, 73, 77, 53, 32, 84, 73, 77, 69, 83, 32, 66, + 73, 20, 168, 1, 2, 75, 65, 202, 7, 73, 212, 39, 2, 65, 83, 130, 177, 2, + 68, 232, 206, 8, 3, 83, 73, 76, 214, 135, 12, 85, 210, 237, 5, 71, 138, + 149, 1, 78, 254, 2, 66, 163, 17, 76, 2, 251, 217, 26, 68, 46, 42, 65, 36, + 4, 69, 83, 72, 50, 23, 73, 9, 194, 207, 30, 68, 2, 78, 3, 80, 5, 191, + 187, 28, 32, 35, 22, 32, 151, 1, 82, 20, 80, 6, 84, 73, 77, 69, 83, 32, + 201, 131, 18, 8, 67, 82, 79, 83, 83, 73, 78, 71, 18, 214, 21, 85, 150, + 48, 65, 2, 73, 254, 199, 29, 66, 187, 64, 69, 12, 32, 2, 73, 71, 191, + 205, 30, 50, 11, 11, 32, 8, 96, 6, 84, 73, 77, 69, 83, 32, 197, 153, 26, + 12, 79, 80, 80, 79, 83, 73, 78, 71, 32, 80, 73, 82, 6, 138, 195, 29, 75, + 162, 67, 85, 235, 67, 90, 8, 234, 67, 65, 182, 136, 30, 73, 3, 85, 202, + 1, 46, 65, 250, 5, 72, 150, 11, 73, 163, 1, 85, 63, 46, 71, 190, 4, 76, + 122, 78, 227, 197, 30, 82, 53, 11, 32, 50, 92, 4, 71, 85, 78, 85, 54, 78, + 32, 6, 84, 73, 77, 69, 83, 32, 241, 21, 4, 79, 86, 69, 82, 5, 29, 5, 32, + 84, 73, 77, 69, 2, 159, 230, 17, 83, 2, 149, 164, 25, 3, 85, 84, 73, 42, + 150, 1, 73, 42, 75, 34, 83, 64, 2, 84, 65, 38, 85, 220, 62, 2, 68, 85, + 166, 223, 28, 76, 226, 40, 78, 210, 48, 69, 138, 60, 77, 162, 17, 72, + 187, 2, 65, 2, 11, 71, 2, 11, 73, 2, 191, 31, 32, 4, 182, 143, 30, 85, + 199, 53, 65, 6, 26, 72, 167, 190, 29, 65, 4, 162, 192, 13, 69, 151, 193, + 16, 73, 4, 226, 172, 23, 75, 211, 154, 7, 66, 10, 254, 197, 30, 83, 146, + 1, 50, 2, 66, 2, 77, 3, 82, 5, 33, 6, 32, 76, 65, 71, 65, 66, 2, 37, 7, + 32, 84, 73, 77, 69, 83, 32, 2, 11, 65, 2, 11, 83, 2, 199, 172, 23, 72, 2, + 135, 169, 11, 71, 106, 46, 65, 250, 1, 69, 242, 2, 73, 239, 3, 85, 29, + 50, 51, 182, 1, 54, 174, 166, 23, 66, 223, 3, 82, 19, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 16, 90, 85, 146, 25, 83, 222, 201, 26, 71, 166, 232, 2, + 84, 170, 50, 66, 218, 47, 78, 215, 22, 65, 5, 11, 32, 2, 181, 131, 26, 4, + 80, 76, 85, 83, 5, 175, 45, 32, 27, 62, 32, 140, 2, 2, 83, 72, 150, 203, + 26, 71, 199, 245, 3, 78, 14, 84, 8, 79, 86, 69, 82, 32, 83, 72, 69, 88, + 5, 80, 76, 85, 83, 32, 207, 156, 30, 72, 7, 11, 32, 4, 190, 15, 71, 141, + 6, 12, 84, 65, 66, 32, 79, 86, 69, 82, 32, 84, 65, 66, 6, 48, 2, 72, 85, + 20, 2, 78, 65, 183, 159, 29, 83, 2, 211, 167, 23, 66, 2, 191, 167, 23, + 77, 7, 218, 133, 29, 76, 255, 186, 1, 50, 40, 62, 68, 74, 77, 218, 1, 82, + 206, 138, 26, 78, 211, 176, 4, 84, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, + 4, 242, 174, 30, 73, 219, 16, 65, 25, 37, 7, 32, 84, 73, 77, 69, 83, 32, + 22, 114, 66, 38, 73, 130, 19, 75, 134, 187, 2, 77, 178, 211, 2, 76, 130, + 251, 23, 71, 250, 23, 83, 238, 9, 68, 215, 127, 65, 4, 154, 206, 2, 85, + 195, 230, 26, 65, 4, 249, 20, 2, 71, 73, 7, 11, 32, 4, 60, 9, 79, 86, 69, + 82, 32, 83, 72, 73, 82, 151, 191, 25, 84, 2, 177, 196, 29, 11, 32, 66, + 85, 82, 32, 79, 86, 69, 82, 32, 66, 13, 88, 14, 32, 79, 86, 69, 82, 32, + 73, 78, 86, 69, 82, 84, 69, 68, 34, 50, 251, 194, 29, 66, 2, 11, 32, 2, + 235, 205, 25, 83, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, 88, 7, 69, 50, 32, + 84, 73, 77, 69, 177, 152, 11, 9, 68, 85, 71, 32, 84, 73, 77, 69, 83, 2, + 131, 132, 13, 83, 17, 50, 32, 30, 71, 234, 152, 11, 76, 215, 135, 12, 75, + 4, 138, 8, 84, 163, 9, 71, 7, 11, 52, 5, 49, 10, 32, 79, 86, 69, 82, 32, + 83, 73, 71, 52, 2, 199, 14, 32, 19, 78, 68, 22, 77, 22, 82, 156, 217, 12, + 5, 32, 79, 86, 69, 82, 179, 230, 16, 72, 5, 199, 184, 30, 50, 5, 147, + 208, 28, 65, 5, 159, 184, 30, 57, 72, 46, 65, 150, 4, 73, 250, 1, 85, + 227, 8, 69, 37, 66, 32, 94, 66, 166, 1, 71, 148, 1, 2, 75, 52, 135, 180, + 30, 82, 8, 60, 6, 84, 73, 77, 69, 83, 32, 130, 14, 71, 135, 149, 28, 65, + 4, 254, 162, 30, 72, 3, 77, 7, 11, 32, 4, 180, 203, 17, 29, 79, 86, 69, + 82, 32, 84, 65, 66, 32, 78, 73, 32, 79, 86, 69, 82, 32, 78, 73, 32, 68, + 73, 83, 72, 32, 79, 86, 69, 82, 143, 187, 5, 83, 15, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 12, 74, 84, 252, 254, 7, 2, 83, 72, 206, 137, 21, 71, + 190, 101, 85, 203, 50, 66, 2, 11, 85, 2, 235, 154, 23, 71, 5, 29, 5, 32, + 80, 76, 85, 83, 2, 221, 202, 28, 2, 32, 83, 17, 46, 82, 150, 29, 32, 134, + 150, 30, 50, 3, 76, 9, 11, 32, 6, 48, 8, 79, 86, 69, 82, 32, 84, 73, 82, + 99, 84, 5, 11, 32, 2, 11, 71, 2, 21, 3, 65, 68, 32, 2, 241, 5, 8, 79, 86, + 69, 82, 32, 71, 65, 68, 2, 217, 21, 6, 73, 77, 69, 83, 32, 84, 17, 50, + 77, 114, 82, 130, 151, 23, 71, 175, 153, 7, 75, 7, 37, 7, 32, 84, 73, 77, + 69, 83, 32, 4, 46, 71, 205, 197, 17, 5, 84, 72, 82, 69, 69, 2, 221, 16, + 2, 65, 78, 5, 141, 139, 11, 17, 32, 79, 86, 69, 82, 32, 84, 85, 82, 32, + 90, 65, 32, 79, 86, 69, 82, 179, 1, 138, 1, 32, 246, 2, 68, 226, 2, 78, + 46, 77, 158, 2, 82, 128, 9, 2, 83, 72, 174, 1, 90, 224, 185, 12, 2, 84, + 85, 158, 226, 17, 50, 3, 66, 12, 64, 7, 79, 86, 69, 82, 32, 85, 32, 158, + 2, 85, 139, 128, 29, 71, 6, 200, 1, 10, 80, 65, 32, 79, 86, 69, 82, 32, + 80, 65, 216, 159, 9, 19, 85, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, + 86, 69, 82, 32, 85, 32, 82, 137, 148, 20, 10, 83, 85, 82, 32, 79, 86, 69, + 82, 32, 83, 2, 11, 32, 2, 45, 9, 71, 65, 82, 32, 79, 86, 69, 82, 32, 2, + 191, 138, 29, 71, 5, 251, 134, 30, 32, 21, 26, 32, 227, 170, 30, 85, 16, + 70, 75, 44, 2, 83, 72, 100, 6, 84, 73, 77, 69, 83, 32, 135, 1, 71, 2, 11, + 85, 2, 11, 83, 2, 155, 135, 11, 72, 4, 29, 5, 69, 83, 72, 73, 71, 5, 11, + 32, 2, 11, 84, 2, 217, 227, 29, 6, 73, 77, 69, 83, 32, 66, 8, 92, 14, 85, + 32, 80, 76, 85, 83, 32, 85, 32, 80, 76, 85, 83, 32, 210, 226, 29, 66, + 215, 50, 77, 4, 11, 85, 5, 11, 32, 2, 11, 71, 2, 227, 131, 30, 85, 21, + 72, 7, 32, 84, 73, 77, 69, 83, 32, 136, 1, 2, 85, 77, 159, 167, 29, 66, + 10, 50, 76, 16, 2, 77, 69, 50, 83, 135, 167, 30, 85, 2, 231, 6, 65, 5, + 11, 32, 2, 201, 241, 25, 4, 80, 76, 85, 83, 2, 131, 134, 11, 72, 7, 37, + 7, 32, 84, 73, 77, 69, 83, 32, 4, 158, 11, 75, 243, 152, 30, 80, 93, 54, + 32, 102, 50, 194, 2, 73, 22, 85, 187, 162, 30, 52, 4, 62, 83, 221, 172, + 29, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 2, 165, 242, 25, 4, 72, 69, + 83, 72, 23, 11, 32, 20, 42, 73, 37, 6, 84, 73, 77, 69, 83, 32, 2, 165, + 130, 24, 4, 78, 86, 69, 82, 18, 46, 65, 82, 85, 130, 163, 29, 78, 251, + 125, 72, 6, 48, 6, 32, 80, 76, 85, 83, 32, 207, 163, 30, 76, 4, 146, 161, + 30, 72, 3, 78, 8, 26, 50, 151, 163, 30, 68, 7, 33, 6, 32, 80, 76, 85, 83, + 32, 4, 206, 186, 28, 65, 203, 212, 1, 66, 5, 203, 162, 30, 51, 59, 56, 7, + 32, 84, 73, 77, 69, 83, 32, 165, 4, 2, 68, 65, 52, 134, 1, 65, 46, 68, + 38, 71, 82, 73, 46, 76, 78, 83, 46, 85, 250, 252, 28, 66, 230, 33, 77, + 238, 90, 84, 146, 17, 75, 162, 17, 72, 3, 80, 5, 11, 83, 2, 11, 72, 2, + 255, 254, 16, 71, 4, 190, 254, 10, 65, 171, 210, 18, 85, 10, 26, 65, 139, + 160, 30, 85, 9, 34, 78, 230, 159, 30, 76, 3, 82, 2, 211, 9, 50, 6, 250, + 139, 30, 71, 202, 18, 83, 147, 1, 77, 6, 46, 85, 221, 129, 23, 5, 65, 75, + 45, 54, 54, 4, 246, 158, 30, 51, 3, 77, 4, 136, 132, 23, 2, 73, 71, 255, + 131, 7, 72, 6, 42, 32, 242, 239, 13, 82, 151, 174, 16, 68, 2, 165, 242, + 28, 6, 80, 76, 85, 83, 32, 71, 5, 11, 32, 2, 153, 212, 13, 4, 84, 73, 77, + 69, 17, 84, 7, 32, 84, 73, 77, 69, 83, 32, 172, 144, 29, 2, 85, 77, 166, + 140, 1, 50, 3, 88, 8, 50, 84, 212, 201, 26, 2, 75, 85, 203, 210, 3, 65, + 2, 11, 65, 2, 191, 129, 23, 75, 6, 26, 51, 227, 155, 30, 85, 5, 29, 5, + 32, 84, 73, 77, 69, 2, 21, 3, 83, 32, 75, 2, 11, 65, 2, 139, 173, 23, 83, + 42, 50, 65, 190, 1, 73, 146, 1, 85, 223, 254, 22, 69, 13, 50, 32, 254, + 141, 29, 77, 166, 140, 1, 55, 3, 71, 4, 56, 8, 83, 81, 85, 65, 82, 69, + 68, 32, 243, 155, 25, 84, 2, 11, 84, 2, 21, 3, 73, 77, 69, 2, 11, 83, 2, + 221, 160, 29, 2, 32, 75, 15, 86, 66, 220, 229, 22, 6, 32, 79, 86, 69, 82, + 32, 186, 25, 90, 174, 153, 7, 51, 3, 71, 5, 17, 2, 32, 75, 2, 17, 2, 65, + 66, 2, 135, 2, 65, 15, 84, 10, 32, 79, 86, 69, 82, 32, 90, 85, 32, 80, + 42, 53, 182, 158, 29, 66, 215, 120, 77, 2, 245, 245, 28, 5, 76, 85, 83, + 32, 83, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 44, 5, 84, 72, 82, 69, + 69, 179, 150, 30, 65, 2, 29, 5, 32, 68, 73, 83, 72, 2, 11, 32, 2, 143, + 152, 25, 84, 8, 42, 32, 154, 228, 22, 73, 255, 200, 3, 67, 4, 198, 159, + 22, 79, 209, 247, 6, 8, 87, 73, 84, 72, 32, 83, 84, 82, 18, 134, 1, 76, + 162, 1, 82, 249, 206, 29, 23, 86, 69, 68, 32, 83, 84, 69, 77, 32, 80, 65, + 82, 65, 71, 82, 65, 80, 72, 32, 83, 73, 71, 78, 10, 48, 2, 89, 32, 241, + 162, 3, 4, 73, 78, 71, 32, 8, 68, 2, 76, 79, 193, 243, 27, 9, 66, 82, 65, + 67, 75, 69, 84, 32, 69, 6, 214, 209, 26, 71, 155, 193, 3, 79, 6, 52, 5, + 69, 78, 67, 89, 32, 53, 4, 89, 32, 65, 78, 4, 212, 247, 24, 4, 69, 88, + 67, 72, 227, 147, 4, 83, 2, 217, 235, 19, 3, 68, 32, 82, 4, 218, 131, 3, + 79, 135, 247, 25, 65, 240, 8, 128, 1, 2, 80, 82, 140, 9, 7, 82, 73, 76, + 76, 73, 67, 32, 132, 223, 25, 7, 76, 73, 78, 68, 82, 73, 67, 205, 253, 2, + 2, 67, 76, 180, 2, 140, 1, 13, 73, 79, 84, 32, 83, 89, 76, 76, 65, 66, + 76, 69, 32, 169, 1, 16, 79, 45, 77, 73, 78, 79, 65, 78, 32, 83, 73, 71, + 78, 32, 67, 77, 110, 238, 177, 7, 75, 2, 76, 2, 77, 2, 78, 2, 80, 2, 82, + 2, 83, 2, 84, 126, 87, 138, 42, 74, 2, 90, 130, 239, 6, 88, 154, 195, 15, + 65, 2, 69, 2, 73, 2, 79, 3, 85, 198, 1, 46, 48, 146, 5, 49, 153, 135, 26, + 2, 51, 48, 170, 1, 102, 48, 78, 49, 74, 50, 78, 51, 78, 52, 62, 53, 86, + 55, 190, 186, 10, 57, 190, 3, 54, 215, 149, 14, 56, 16, 194, 140, 30, 49, + 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 16, 194, 3, 50, 182, + 136, 30, 48, 2, 49, 2, 51, 2, 53, 2, 55, 3, 57, 16, 174, 139, 30, 49, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 16, 226, 138, 30, 48, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 150, 138, 30, 48, 2, + 49, 2, 52, 2, 54, 2, 55, 3, 57, 18, 218, 137, 30, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 20, 82, 53, 182, 136, 30, 48, 2, + 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 56, 3, 57, 5, 179, 136, 30, 66, 24, + 18, 48, 87, 49, 18, 138, 136, 30, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 55, 2, 56, 3, 57, 6, 182, 135, 30, 48, 2, 50, 3, 52, 184, 6, 140, 1, + 9, 67, 65, 80, 73, 84, 65, 76, 32, 76, 202, 4, 75, 32, 7, 76, 69, 84, 84, + 69, 82, 32, 132, 1, 3, 80, 65, 89, 30, 83, 211, 41, 84, 242, 2, 44, 6, + 69, 84, 84, 69, 82, 32, 175, 43, 73, 236, 2, 150, 2, 76, 46, 78, 34, 80, + 34, 82, 58, 84, 62, 85, 190, 5, 65, 214, 1, 66, 242, 1, 67, 150, 1, 68, + 254, 1, 69, 206, 2, 71, 130, 1, 72, 134, 1, 73, 230, 3, 75, 226, 3, 77, + 218, 1, 79, 226, 2, 83, 218, 7, 89, 166, 1, 90, 162, 183, 29, 70, 134, + 14, 74, 2, 86, 2, 87, 159, 20, 81, 6, 222, 25, 73, 254, 210, 29, 74, 159, + 20, 72, 4, 134, 27, 69, 171, 209, 29, 74, 8, 134, 34, 69, 135, 205, 29, + 83, 12, 220, 9, 3, 79, 85, 78, 226, 18, 69, 207, 227, 29, 72, 20, 174, + 33, 69, 106, 83, 234, 178, 29, 67, 186, 22, 74, 3, 87, 13, 198, 34, 32, + 115, 75, 2, 205, 248, 29, 3, 65, 86, 89, 6, 248, 10, 5, 77, 85, 76, 84, + 73, 200, 129, 13, 2, 80, 65, 241, 231, 11, 14, 83, 77, 65, 76, 76, 32, + 67, 65, 80, 73, 84, 65, 76, 32, 2, 177, 214, 29, 2, 69, 82, 186, 3, 136, + 1, 6, 77, 65, 76, 76, 32, 76, 201, 37, 22, 85, 66, 83, 67, 82, 73, 80, + 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 134, 3, 44, + 6, 69, 84, 84, 69, 82, 32, 151, 36, 73, 128, 3, 154, 2, 65, 214, 1, 66, + 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, 72, 134, 1, 73, + 230, 3, 75, 222, 2, 76, 134, 1, 77, 114, 78, 106, 79, 110, 80, 50, 82, + 198, 1, 83, 218, 2, 84, 218, 2, 85, 246, 1, 87, 54, 89, 166, 1, 90, 162, + 183, 29, 70, 134, 14, 74, 2, 86, 159, 20, 81, 17, 108, 6, 32, 87, 73, 84, + 72, 32, 36, 9, 66, 75, 72, 65, 83, 73, 65, 78, 32, 249, 152, 12, 4, 76, + 69, 85, 84, 4, 186, 194, 9, 68, 147, 182, 3, 66, 8, 172, 133, 13, 3, 67, + 72, 69, 158, 213, 6, 68, 255, 157, 10, 72, 18, 110, 65, 78, 73, 32, 3, + 82, 79, 65, 232, 160, 3, 6, 76, 69, 78, 68, 69, 68, 246, 224, 9, 89, 139, + 247, 16, 69, 6, 200, 21, 6, 82, 82, 69, 68, 32, 79, 253, 129, 12, 5, 83, + 72, 75, 73, 82, 4, 230, 2, 78, 163, 158, 3, 71, 2, 171, 179, 10, 68, 14, + 76, 2, 72, 69, 246, 9, 76, 220, 146, 13, 4, 82, 79, 83, 83, 167, 174, 16, + 67, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 150, 29, 68, 255, 169, 29, 86, + 26, 96, 2, 74, 69, 20, 6, 79, 85, 66, 76, 69, 32, 66, 90, 250, 200, 29, + 67, 186, 22, 87, 215, 22, 69, 5, 219, 230, 22, 82, 4, 36, 3, 77, 79, 78, + 159, 246, 29, 79, 2, 153, 13, 2, 79, 67, 12, 46, 69, 182, 245, 28, 90, + 206, 105, 72, 3, 87, 5, 223, 214, 29, 76, 41, 86, 76, 98, 78, 110, 82, + 166, 15, 32, 170, 238, 12, 83, 178, 193, 14, 77, 163, 180, 2, 70, 11, 33, + 6, 32, 87, 73, 84, 72, 32, 8, 166, 20, 77, 130, 235, 12, 68, 190, 193, + 14, 84, 143, 96, 72, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 198, 19, 77, + 130, 235, 12, 68, 202, 52, 76, 246, 140, 14, 84, 143, 96, 72, 5, 133, + 195, 28, 5, 32, 87, 73, 84, 72, 14, 32, 2, 72, 69, 255, 219, 29, 74, 13, + 33, 6, 32, 87, 73, 84, 72, 32, 10, 142, 18, 77, 146, 15, 85, 242, 219, + 12, 68, 155, 31, 83, 12, 26, 65, 135, 219, 29, 87, 11, 48, 6, 32, 87, 73, + 84, 72, 32, 207, 136, 28, 82, 6, 134, 252, 12, 68, 202, 161, 15, 72, 219, + 163, 1, 83, 35, 84, 6, 32, 87, 73, 84, 72, 32, 50, 69, 74, 79, 201, 1, 6, + 90, 72, 73, 84, 83, 65, 6, 174, 183, 9, 68, 214, 10, 71, 243, 196, 3, 77, + 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 174, 193, 9, 71, 191, 171, 3, 66, + 17, 11, 84, 14, 48, 6, 73, 70, 73, 69, 68, 32, 227, 238, 29, 65, 12, 80, + 2, 67, 76, 38, 76, 190, 149, 3, 66, 198, 196, 25, 89, 234, 147, 1, 65, 3, + 69, 2, 33, 6, 79, 83, 69, 68, 32, 76, 2, 151, 4, 73, 5, 197, 136, 13, 14, + 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 69, 32, 71, 34, 102, 65, 98, + 79, 184, 190, 29, 11, 72, 65, 75, 65, 83, 83, 73, 65, 78, 32, 67, 186, + 22, 74, 255, 2, 83, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 226, 246, 12, + 68, 202, 161, 15, 72, 142, 163, 1, 86, 79, 83, 18, 36, 3, 77, 73, 32, + 203, 177, 26, 80, 16, 58, 68, 246, 160, 12, 76, 2, 78, 2, 83, 2, 84, 3, + 90, 6, 242, 160, 12, 90, 154, 179, 17, 74, 215, 22, 69, 8, 94, 73, 228, + 200, 10, 10, 79, 78, 71, 45, 76, 69, 71, 71, 69, 68, 154, 138, 19, 74, + 159, 20, 72, 2, 209, 145, 3, 4, 84, 84, 76, 69, 4, 21, 3, 79, 78, 79, 4, + 18, 67, 39, 71, 2, 145, 132, 19, 4, 85, 76, 65, 82, 2, 245, 10, 4, 82, + 65, 80, 72, 6, 62, 69, 144, 131, 19, 5, 65, 82, 82, 79, 87, 155, 206, 10, + 74, 2, 165, 245, 12, 5, 85, 84, 82, 65, 76, 11, 52, 4, 77, 69, 71, 65, + 166, 3, 32, 251, 227, 29, 84, 5, 173, 190, 29, 8, 32, 87, 73, 84, 72, 32, + 84, 73, 10, 138, 6, 69, 138, 236, 12, 65, 255, 224, 16, 83, 14, 50, 69, + 100, 4, 79, 85, 78, 68, 235, 226, 29, 72, 8, 37, 7, 86, 69, 82, 83, 69, + 68, 32, 8, 166, 197, 19, 68, 170, 151, 9, 84, 166, 100, 89, 151, 14, 90, + 4, 242, 159, 10, 32, 137, 244, 1, 2, 69, 68, 34, 108, 4, 67, 72, 87, 65, + 34, 72, 132, 1, 4, 79, 70, 84, 32, 214, 238, 12, 84, 213, 1, 5, 69, 77, + 73, 83, 79, 5, 11, 32, 2, 187, 238, 5, 87, 16, 92, 4, 79, 82, 84, 32, + 180, 237, 12, 2, 72, 65, 146, 251, 14, 67, 238, 227, 1, 87, 215, 22, 65, + 6, 226, 174, 27, 73, 163, 180, 2, 85, 8, 38, 69, 210, 219, 28, 83, 151, + 112, 68, 4, 182, 226, 29, 76, 3, 77, 28, 140, 1, 4, 65, 76, 76, 32, 50, + 69, 106, 83, 224, 135, 12, 11, 72, 82, 69, 69, 45, 76, 69, 71, 71, 69, + 68, 138, 171, 17, 67, 186, 22, 74, 3, 87, 6, 214, 231, 12, 72, 206, 229, + 15, 89, 151, 125, 84, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 77, 131, + 235, 12, 68, 2, 189, 244, 21, 5, 73, 68, 68, 76, 69, 8, 158, 201, 29, 72, + 2, 83, 2, 87, 215, 22, 69, 15, 58, 32, 114, 75, 53, 8, 78, 66, 76, 69, + 78, 68, 69, 68, 6, 29, 5, 87, 73, 84, 72, 32, 6, 26, 68, 171, 245, 12, + 77, 4, 212, 167, 9, 5, 79, 85, 66, 76, 69, 187, 8, 73, 5, 11, 82, 2, 213, + 4, 6, 65, 73, 78, 73, 65, 78, 2, 207, 232, 24, 32, 4, 212, 201, 28, 4, + 73, 68, 69, 32, 251, 147, 1, 69, 18, 62, 65, 28, 3, 69, 82, 85, 194, 220, + 29, 73, 2, 78, 3, 85, 7, 218, 220, 29, 69, 3, 84, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 178, 163, 9, 68, 255, 197, 3, 66, 18, 18, 69, 71, 72, 9, + 156, 1, 7, 32, 87, 73, 84, 72, 32, 68, 233, 215, 29, 2, 77, 76, 10, 26, + 69, 179, 196, 29, 87, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 26, 68, 239, + 215, 12, 66, 4, 134, 172, 9, 73, 175, 185, 3, 69, 6, 37, 7, 71, 65, 84, + 85, 82, 69, 32, 6, 66, 65, 188, 198, 15, 2, 84, 69, 237, 229, 13, 4, 69, + 78, 32, 71, 2, 131, 190, 29, 32, 52, 198, 1, 66, 38, 68, 38, 69, 36, 3, + 71, 72, 69, 38, 72, 130, 137, 22, 89, 138, 211, 5, 83, 182, 113, 84, 238, + 8, 90, 130, 64, 73, 150, 19, 67, 186, 22, 80, 2, 86, 158, 20, 75, 186, 2, + 65, 2, 79, 3, 85, 4, 206, 224, 12, 89, 139, 247, 16, 69, 6, 146, 215, 28, + 90, 163, 128, 1, 69, 6, 142, 215, 29, 70, 2, 76, 3, 83, 5, 201, 5, 5, 32, + 87, 73, 84, 72, 4, 11, 65, 5, 227, 237, 27, 82, 2, 153, 248, 15, 4, 72, + 79, 85, 83, 212, 15, 178, 1, 65, 138, 4, 67, 54, 69, 146, 35, 73, 138, + 23, 79, 154, 45, 82, 174, 3, 85, 144, 232, 12, 13, 78, 65, 32, 68, 79, + 85, 66, 76, 69, 32, 72, 69, 76, 134, 183, 15, 86, 207, 47, 76, 30, 116, + 4, 71, 71, 69, 82, 146, 1, 78, 32, 4, 82, 75, 32, 83, 36, 2, 83, 72, 184, + 253, 21, 2, 76, 69, 139, 145, 1, 84, 9, 11, 32, 6, 44, 5, 87, 73, 84, 72, + 32, 219, 210, 6, 75, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 237, + 187, 28, 4, 84, 32, 71, 85, 4, 214, 148, 29, 67, 251, 30, 71, 4, 230, + 200, 21, 85, 211, 146, 5, 72, 10, 30, 32, 105, 3, 69, 68, 32, 4, 60, 9, + 87, 73, 84, 72, 32, 76, 69, 70, 84, 147, 190, 28, 83, 2, 17, 2, 32, 85, + 2, 143, 141, 23, 80, 6, 238, 233, 4, 84, 142, 189, 12, 76, 155, 194, 10, + 79, 10, 194, 208, 29, 49, 2, 50, 2, 51, 2, 52, 3, 83, 180, 4, 226, 1, 67, + 248, 1, 5, 71, 82, 69, 69, 32, 98, 76, 82, 78, 228, 3, 8, 80, 65, 82, 84, + 77, 69, 78, 84, 34, 83, 174, 6, 86, 192, 140, 15, 11, 82, 69, 76, 73, 67, + 84, 32, 72, 79, 85, 83, 168, 180, 10, 2, 65, 70, 143, 199, 3, 69, 8, 50, + 73, 177, 250, 5, 6, 82, 69, 65, 83, 69, 32, 6, 56, 4, 77, 65, 76, 32, + 205, 168, 9, 4, 68, 85, 79, 85, 4, 88, 9, 83, 69, 80, 65, 82, 65, 84, 79, + 82, 149, 248, 21, 7, 69, 88, 80, 79, 78, 69, 78, 2, 21, 3, 32, 75, 69, 2, + 155, 185, 28, 89, 6, 140, 194, 22, 4, 67, 69, 76, 83, 186, 131, 6, 83, + 177, 106, 8, 70, 65, 72, 82, 69, 78, 72, 69, 9, 248, 250, 16, 5, 73, 86, + 69, 82, 89, 248, 252, 5, 2, 84, 65, 231, 140, 5, 69, 34, 104, 20, 84, 73, 83, 84, 82, 89, 32, 83, 89, 77, 66, 79, 76, 32, 76, 73, 71, 72, 84, 32, - 160, 3, 8, 80, 65, 82, 84, 77, 69, 78, 84, 34, 83, 174, 6, 86, 188, 214, - 14, 11, 82, 69, 76, 73, 67, 84, 32, 72, 79, 85, 83, 132, 146, 7, 2, 65, - 70, 235, 154, 6, 69, 8, 50, 73, 217, 229, 5, 6, 82, 69, 65, 83, 69, 32, - 6, 48, 4, 68, 85, 79, 85, 21, 4, 77, 65, 76, 32, 2, 219, 178, 23, 83, 4, - 88, 9, 83, 69, 80, 65, 82, 65, 84, 79, 82, 197, 171, 21, 7, 69, 88, 80, - 79, 78, 69, 78, 2, 21, 3, 32, 75, 69, 2, 151, 180, 27, 89, 6, 200, 242, - 21, 4, 67, 69, 76, 83, 250, 205, 5, 83, 153, 106, 8, 70, 65, 72, 82, 69, - 78, 72, 69, 9, 176, 190, 16, 5, 73, 86, 69, 82, 89, 128, 225, 5, 2, 84, - 65, 215, 226, 4, 69, 30, 88, 4, 68, 79, 87, 78, 0, 2, 85, 80, 153, 1, 9, - 86, 69, 82, 84, 73, 67, 65, 76, 32, 8, 69, 15, 32, 65, 78, 68, 32, 72, - 79, 82, 73, 90, 79, 78, 84, 65, 76, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, - 234, 219, 26, 87, 170, 25, 67, 191, 43, 84, 14, 52, 4, 65, 78, 68, 32, - 85, 5, 87, 73, 84, 72, 32, 10, 200, 158, 17, 2, 84, 79, 190, 188, 9, 87, - 165, 189, 1, 6, 66, 79, 84, 84, 79, 77, 4, 218, 243, 26, 67, 191, 43, 84, - 2, 245, 137, 24, 3, 32, 83, 84, 170, 1, 64, 2, 67, 69, 48, 2, 69, 82, - 129, 5, 5, 75, 84, 79, 80, 32, 2, 217, 180, 26, 7, 78, 68, 73, 78, 71, - 32, 78, 164, 1, 32, 3, 69, 84, 32, 183, 4, 84, 160, 1, 56, 6, 67, 65, 80, - 73, 84, 65, 1, 4, 83, 77, 65, 76, 80, 45, 9, 76, 32, 76, 69, 84, 84, 69, - 82, 32, 80, 218, 1, 69, 96, 4, 76, 79, 78, 71, 0, 5, 83, 72, 79, 82, 84, - 74, 79, 30, 84, 2, 90, 246, 178, 11, 67, 206, 157, 15, 66, 2, 68, 2, 74, - 2, 80, 2, 86, 2, 89, 202, 96, 71, 2, 75, 134, 103, 87, 162, 19, 65, 203, - 17, 72, 20, 214, 202, 24, 83, 206, 251, 1, 78, 134, 247, 1, 84, 146, 1, - 70, 2, 76, 2, 77, 2, 82, 3, 87, 12, 11, 32, 12, 234, 201, 24, 65, 226, - 180, 1, 79, 130, 191, 2, 69, 3, 73, 4, 142, 189, 28, 73, 3, 87, 4, 190, - 208, 26, 72, 227, 213, 1, 69, 5, 225, 174, 27, 4, 32, 73, 83, 76, 4, 186, - 153, 16, 67, 229, 136, 11, 4, 87, 73, 78, 68, 202, 2, 100, 8, 65, 78, 65, - 71, 65, 82, 73, 32, 185, 18, 12, 73, 67, 69, 32, 67, 79, 78, 84, 82, 79, - 76, 32, 192, 2, 222, 1, 65, 38, 67, 22, 71, 36, 4, 72, 69, 65, 68, 96, 7, - 76, 69, 84, 84, 69, 82, 32, 198, 5, 83, 176, 7, 11, 86, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 158, 181, 18, 68, 148, 245, 5, 4, 74, 65, 73, 78, - 243, 239, 3, 79, 4, 138, 164, 12, 67, 191, 170, 12, 66, 2, 187, 253, 5, - 65, 4, 250, 163, 12, 82, 215, 236, 1, 65, 6, 44, 5, 32, 77, 65, 82, 75, - 163, 136, 28, 83, 5, 209, 195, 18, 7, 32, 87, 73, 84, 72, 32, 72, 158, 1, - 158, 2, 65, 54, 67, 62, 68, 50, 71, 62, 72, 52, 2, 77, 65, 46, 83, 154, - 10, 79, 218, 254, 6, 66, 82, 74, 218, 220, 4, 75, 238, 194, 3, 82, 166, - 212, 6, 89, 170, 109, 85, 238, 30, 78, 230, 46, 76, 238, 146, 1, 84, 46, - 86, 138, 225, 1, 73, 154, 191, 1, 80, 2, 90, 254, 68, 70, 2, 81, 187, 2, - 69, 13, 198, 181, 28, 65, 2, 73, 2, 85, 2, 87, 3, 89, 10, 26, 65, 195, - 178, 28, 72, 9, 185, 2, 4, 78, 68, 82, 65, 12, 170, 204, 24, 68, 246, - 229, 3, 72, 187, 2, 65, 10, 182, 185, 12, 76, 190, 179, 15, 72, 254, 68, - 71, 187, 2, 65, 4, 232, 150, 16, 4, 69, 65, 86, 89, 131, 157, 12, 65, 5, - 229, 133, 7, 6, 82, 87, 65, 82, 73, 32, 12, 38, 72, 174, 176, 28, 83, - 187, 2, 65, 8, 36, 3, 79, 82, 84, 191, 178, 28, 65, 6, 163, 212, 11, 32, - 68, 168, 1, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, - 84, 84, 69, 82, 32, 104, 4, 73, 71, 78, 32, 141, 128, 26, 10, 84, 82, 69, - 83, 83, 32, 83, 73, 71, 78, 16, 226, 202, 3, 71, 2, 75, 220, 238, 22, 3, - 68, 68, 68, 2, 82, 226, 244, 1, 89, 38, 70, 2, 81, 3, 90, 48, 210, 3, 66, - 0, 10, 69, 88, 84, 69, 78, 68, 69, 68, 32, 66, 36, 11, 67, 65, 78, 68, - 82, 65, 66, 73, 78, 68, 85, 60, 8, 87, 69, 83, 84, 69, 82, 78, 32, 32, - 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 78, 148, 231, 4, 5, 80, 85, 83, - 72, 80, 182, 204, 1, 83, 156, 243, 3, 18, 68, 79, 85, 66, 76, 69, 32, 67, - 65, 78, 68, 82, 65, 66, 73, 78, 68, 85, 182, 148, 8, 73, 186, 137, 6, 65, - 126, 77, 46, 78, 204, 212, 2, 12, 72, 73, 71, 72, 32, 83, 80, 65, 67, 73, - 78, 71, 255, 77, 86, 4, 133, 170, 12, 4, 72, 65, 76, 69, 11, 11, 32, 8, - 206, 241, 21, 65, 210, 89, 86, 151, 244, 3, 84, 4, 30, 78, 21, 3, 70, 73, - 86, 2, 17, 2, 73, 78, 2, 217, 144, 21, 8, 69, 45, 76, 73, 75, 69, 32, 66, - 50, 146, 1, 65, 52, 7, 67, 65, 78, 68, 82, 65, 32, 62, 79, 190, 186, 18, - 80, 254, 165, 4, 85, 138, 228, 1, 83, 70, 86, 166, 221, 1, 73, 207, 134, - 2, 69, 10, 226, 169, 28, 65, 2, 73, 2, 85, 2, 87, 3, 89, 6, 176, 187, 18, - 4, 76, 79, 78, 71, 254, 237, 9, 69, 3, 79, 7, 158, 146, 28, 79, 215, 22, - 69, 10, 222, 193, 23, 70, 136, 6, 2, 83, 84, 146, 244, 2, 84, 203, 83, - 79, 228, 2, 182, 2, 65, 134, 2, 69, 74, 71, 224, 4, 6, 78, 71, 66, 65, - 84, 32, 248, 1, 5, 82, 69, 67, 84, 32, 98, 83, 178, 2, 86, 200, 7, 2, 89, - 65, 32, 4, 90, 90, 89, 32, 192, 152, 5, 2, 84, 84, 216, 229, 14, 10, 70, - 70, 69, 82, 69, 78, 67, 69, 32, 66, 229, 147, 7, 12, 77, 69, 78, 83, 73, - 79, 78, 32, 79, 82, 73, 71, 18, 26, 77, 167, 211, 8, 69, 16, 40, 4, 79, - 78, 68, 32, 195, 138, 3, 69, 14, 128, 1, 5, 87, 73, 84, 72, 32, 218, 220, - 17, 84, 128, 221, 2, 12, 83, 72, 65, 80, 69, 32, 87, 73, 84, 72, 32, 65, - 159, 192, 4, 79, 8, 202, 177, 21, 66, 162, 2, 84, 250, 228, 3, 76, 27, - 82, 14, 164, 240, 4, 5, 32, 70, 65, 67, 69, 225, 229, 16, 4, 83, 69, 76, - 32, 78, 64, 3, 73, 84, 32, 149, 2, 8, 82, 65, 77, 32, 70, 79, 82, 32, 60, - 98, 70, 44, 2, 78, 73, 2, 79, 14, 83, 46, 84, 44, 3, 90, 69, 82, 13, 5, - 69, 73, 71, 72, 84, 12, 128, 1, 2, 73, 86, 25, 3, 79, 85, 82, 6, 87, 78, - 12, 96, 4, 69, 86, 69, 78, 1, 2, 73, 88, 12, 28, 3, 72, 82, 69, 15, 87, - 6, 23, 69, 6, 11, 79, 7, 207, 186, 16, 32, 18, 88, 5, 69, 65, 82, 84, 72, - 64, 5, 71, 82, 69, 65, 84, 0, 4, 76, 69, 83, 83, 39, 72, 7, 25, 4, 76, - 89, 32, 72, 4, 238, 213, 23, 85, 187, 131, 1, 69, 4, 229, 207, 12, 4, 69, - 82, 32, 89, 4, 172, 129, 5, 7, 69, 65, 86, 69, 78, 76, 89, 1, 4, 85, 77, - 65, 78, 64, 120, 17, 78, 69, 71, 65, 84, 73, 86, 69, 32, 67, 73, 82, 67, - 76, 69, 68, 32, 41, 9, 67, 73, 82, 67, 76, 69, 68, 32, 83, 42, 38, 83, - 162, 32, 78, 139, 249, 19, 68, 22, 49, 10, 65, 78, 83, 45, 83, 69, 82, - 73, 70, 32, 22, 234, 31, 78, 203, 142, 26, 68, 4, 140, 143, 22, 15, 67, - 85, 82, 82, 69, 78, 84, 32, 83, 89, 77, 66, 79, 76, 32, 235, 240, 5, 72, - 10, 70, 65, 144, 1, 5, 67, 79, 78, 84, 73, 213, 251, 24, 3, 71, 85, 73, - 6, 76, 9, 80, 80, 79, 73, 78, 84, 69, 68, 32, 217, 255, 14, 4, 66, 76, - 69, 68, 4, 232, 226, 16, 7, 66, 85, 84, 32, 82, 69, 76, 211, 250, 10, 70, - 2, 53, 11, 78, 85, 79, 85, 83, 32, 85, 78, 68, 69, 82, 2, 213, 197, 16, - 3, 76, 73, 78, 156, 1, 80, 9, 69, 83, 32, 65, 75, 85, 82, 85, 32, 238, 5, - 73, 233, 190, 16, 2, 79, 82, 144, 1, 142, 2, 68, 36, 7, 76, 69, 84, 84, - 69, 82, 32, 236, 1, 5, 83, 73, 71, 78, 32, 58, 86, 252, 243, 8, 6, 77, - 69, 68, 73, 65, 76, 198, 247, 4, 71, 198, 211, 4, 69, 172, 189, 4, 12, - 80, 82, 69, 70, 73, 88, 69, 68, 32, 78, 65, 83, 241, 204, 4, 7, 73, 78, - 73, 84, 73, 65, 76, 22, 246, 192, 18, 79, 175, 231, 7, 73, 84, 186, 142, - 10, 84, 242, 209, 11, 89, 186, 116, 65, 130, 217, 1, 68, 146, 25, 85, - 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, - 2, 75, 2, 80, 254, 68, 72, 2, 74, 2, 77, 2, 82, 2, 86, 2, 90, 186, 2, 69, - 3, 79, 8, 234, 128, 24, 72, 146, 45, 67, 98, 78, 223, 160, 3, 65, 18, 64, - 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 179, 225, 25, 73, 16, 178, - 219, 14, 65, 218, 232, 9, 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, - 10, 68, 5, 83, 73, 79, 78, 32, 204, 234, 1, 2, 78, 71, 235, 147, 25, 68, - 6, 26, 83, 247, 206, 5, 84, 4, 206, 172, 26, 76, 231, 94, 73, 2, 173, - 184, 18, 3, 32, 76, 65, 4, 190, 254, 26, 83, 255, 85, 70, 190, 5, 188, 2, - 6, 67, 85, 77, 69, 78, 84, 124, 7, 69, 83, 32, 78, 79, 84, 32, 194, 3, - 71, 210, 3, 76, 36, 10, 77, 73, 78, 79, 32, 84, 73, 76, 69, 32, 226, 1, - 78, 38, 84, 246, 2, 85, 204, 17, 2, 87, 78, 220, 128, 16, 8, 32, 78, 79, - 84, 32, 76, 73, 84, 136, 178, 11, 8, 86, 69, 32, 79, 70, 32, 80, 69, 174, - 4, 79, 223, 25, 68, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 40, 4, 84, 69, - 88, 84, 191, 133, 2, 80, 5, 185, 133, 2, 6, 32, 65, 78, 68, 32, 80, 22, - 164, 1, 11, 67, 79, 78, 84, 65, 73, 78, 32, 65, 83, 32, 92, 6, 68, 73, - 86, 73, 68, 69, 112, 2, 80, 82, 48, 7, 83, 85, 67, 67, 69, 69, 68, 233, - 251, 21, 2, 70, 79, 6, 248, 1, 15, 78, 79, 82, 77, 65, 76, 32, 83, 85, - 66, 71, 82, 79, 85, 80, 195, 170, 20, 77, 5, 209, 170, 21, 23, 32, 87, - 73, 84, 72, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 78, 69, 71, 65, 84, - 73, 79, 78, 6, 44, 5, 69, 67, 69, 68, 69, 243, 141, 27, 79, 5, 217, 240, - 8, 2, 32, 79, 125, 36, 3, 82, 65, 32, 235, 204, 27, 32, 120, 120, 7, 76, - 69, 84, 84, 69, 82, 32, 212, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, - 78, 32, 158, 199, 22, 65, 191, 2, 83, 88, 214, 232, 23, 82, 206, 55, 65, - 38, 68, 46, 84, 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, - 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, - 86, 2, 89, 186, 2, 69, 3, 79, 22, 162, 193, 18, 86, 166, 225, 5, 65, 190, - 21, 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 4, 182, 194, 20, 80, - 243, 173, 2, 76, 200, 1, 72, 8, 72, 79, 82, 73, 90, 79, 78, 84, 1, 6, 86, - 69, 82, 84, 73, 67, 100, 17, 2, 65, 76, 100, 32, 2, 45, 48, 191, 162, 23, - 32, 98, 58, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 14, 197, 219, - 19, 2, 45, 48, 4, 254, 237, 25, 75, 223, 143, 1, 71, 26, 34, 32, 57, 4, - 84, 69, 68, 32, 8, 254, 166, 24, 80, 34, 77, 202, 50, 79, 187, 173, 2, - 65, 18, 214, 1, 67, 34, 83, 144, 213, 15, 3, 76, 73, 78, 192, 132, 1, 4, - 79, 66, 69, 76, 160, 9, 9, 84, 82, 65, 78, 83, 80, 79, 83, 73, 134, 194, - 6, 70, 141, 41, 14, 82, 73, 71, 72, 84, 45, 80, 79, 73, 78, 84, 73, 78, - 71, 4, 238, 178, 26, 73, 167, 17, 82, 4, 182, 241, 25, 79, 223, 111, 81, - 162, 1, 40, 3, 66, 76, 69, 137, 17, 2, 71, 72, 160, 1, 42, 32, 190, 10, - 45, 241, 5, 2, 68, 32, 106, 226, 2, 67, 174, 1, 68, 38, 72, 32, 4, 73, - 78, 84, 69, 38, 76, 144, 1, 7, 78, 69, 83, 84, 69, 68, 32, 74, 80, 66, - 83, 130, 2, 85, 36, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 132, 140, 6, - 6, 79, 66, 76, 73, 81, 85, 172, 167, 2, 9, 82, 73, 71, 72, 84, 32, 65, - 82, 67, 194, 174, 3, 65, 186, 189, 9, 69, 190, 141, 4, 81, 213, 101, 6, - 87, 65, 86, 89, 32, 79, 24, 100, 7, 73, 82, 67, 76, 69, 68, 32, 252, 166, - 7, 4, 85, 82, 76, 89, 145, 191, 1, 4, 79, 76, 79, 78, 20, 26, 78, 139, - 249, 19, 68, 2, 137, 199, 2, 5, 85, 77, 66, 69, 82, 4, 186, 160, 18, 79, - 131, 164, 6, 65, 4, 186, 243, 19, 73, 215, 63, 89, 4, 230, 136, 25, 82, - 223, 233, 1, 71, 12, 54, 79, 141, 222, 16, 7, 69, 70, 84, 32, 65, 82, 67, - 10, 26, 87, 163, 207, 24, 71, 6, 26, 45, 187, 225, 26, 32, 4, 162, 242, - 19, 82, 211, 1, 57, 6, 196, 204, 16, 9, 76, 69, 83, 83, 45, 84, 72, 65, - 78, 255, 158, 9, 71, 8, 26, 82, 247, 246, 25, 76, 6, 154, 152, 1, 69, - 139, 180, 15, 73, 18, 80, 6, 81, 85, 65, 82, 69, 32, 30, 84, 50, 85, 185, - 161, 13, 4, 79, 76, 73, 68, 4, 238, 221, 24, 73, 55, 85, 4, 252, 138, 12, - 3, 65, 67, 75, 187, 192, 4, 82, 8, 58, 83, 186, 150, 1, 67, 170, 182, 20, - 80, 191, 194, 4, 66, 2, 209, 142, 27, 4, 80, 69, 78, 83, 4, 210, 155, 18, - 80, 179, 139, 9, 78, 10, 36, 3, 66, 65, 82, 211, 221, 26, 76, 9, 11, 32, - 6, 52, 7, 68, 79, 85, 66, 76, 69, 32, 163, 215, 25, 76, 4, 158, 215, 25, - 76, 51, 82, 48, 112, 5, 76, 73, 78, 69, 32, 220, 1, 7, 83, 84, 82, 85, - 67, 75, 32, 189, 182, 8, 7, 69, 78, 68, 69, 68, 32, 77, 12, 48, 8, 83, - 76, 65, 78, 84, 69, 68, 32, 75, 69, 8, 70, 69, 56, 7, 71, 82, 69, 65, 84, - 69, 82, 1, 4, 76, 69, 83, 83, 4, 177, 232, 19, 9, 81, 85, 65, 76, 32, 84, - 79, 32, 79, 2, 193, 239, 13, 5, 45, 84, 72, 65, 78, 34, 160, 1, 8, 67, - 65, 80, 73, 84, 65, 76, 32, 92, 7, 73, 84, 65, 76, 73, 67, 32, 124, 6, - 83, 77, 65, 76, 76, 32, 153, 155, 13, 8, 78, 45, 65, 82, 89, 32, 83, 85, - 18, 234, 238, 11, 71, 230, 252, 13, 80, 206, 134, 2, 67, 2, 72, 2, 78, 2, - 81, 2, 82, 3, 90, 10, 76, 6, 83, 77, 65, 76, 76, 32, 185, 151, 21, 7, 67, - 65, 80, 73, 84, 65, 76, 8, 238, 240, 27, 68, 2, 69, 2, 73, 3, 74, 4, 146, - 237, 11, 71, 219, 239, 15, 80, 6, 146, 237, 9, 70, 26, 77, 255, 239, 16, - 83, 2, 179, 246, 26, 78, 144, 1, 86, 32, 156, 1, 10, 45, 80, 79, 73, 78, - 84, 73, 78, 71, 32, 65, 4, 87, 65, 82, 68, 10, 60, 4, 84, 65, 67, 75, - 250, 222, 24, 70, 30, 82, 251, 64, 65, 5, 29, 5, 32, 87, 73, 84, 72, 2, - 11, 32, 2, 11, 67, 2, 201, 169, 20, 4, 73, 82, 67, 76, 8, 178, 225, 24, - 82, 0, 7, 83, 77, 65, 76, 76, 32, 82, 35, 84, 126, 56, 8, 32, 70, 65, 67, - 73, 78, 71, 32, 89, 2, 83, 32, 8, 54, 72, 1, 9, 78, 79, 84, 67, 72, 69, - 68, 32, 72, 4, 137, 201, 26, 3, 79, 79, 75, 118, 158, 1, 65, 202, 2, 84, - 168, 192, 8, 2, 87, 72, 186, 138, 8, 90, 198, 184, 8, 66, 154, 1, 68, 50, - 70, 82, 72, 146, 4, 67, 46, 81, 42, 82, 22, 83, 243, 7, 80, 34, 40, 4, - 82, 82, 79, 87, 187, 255, 24, 78, 33, 11, 32, 30, 144, 1, 5, 87, 73, 84, - 72, 32, 208, 147, 3, 14, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, - 32, 85, 254, 235, 21, 65, 170, 10, 70, 175, 5, 84, 22, 180, 163, 5, 6, - 67, 79, 82, 78, 69, 82, 182, 222, 19, 68, 58, 76, 42, 77, 38, 78, 58, 83, - 66, 69, 246, 12, 84, 139, 59, 72, 36, 36, 2, 82, 73, 161, 2, 2, 87, 79, - 32, 44, 5, 65, 78, 71, 76, 69, 171, 146, 25, 80, 30, 56, 8, 45, 72, 69, - 65, 68, 69, 68, 32, 215, 152, 25, 32, 28, 68, 5, 65, 82, 82, 79, 87, 134, - 201, 16, 90, 162, 199, 8, 68, 39, 80, 23, 11, 32, 20, 216, 139, 25, 15, - 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 85, 80, 98, 84, 19, - 87, 4, 162, 145, 25, 32, 105, 15, 45, 72, 69, 65, 68, 69, 68, 32, 65, 82, - 82, 79, 87, 32, 87, 22, 138, 1, 65, 106, 79, 152, 1, 12, 85, 77, 32, 87, - 73, 84, 72, 32, 68, 82, 85, 77, 152, 182, 16, 6, 73, 86, 69, 32, 83, 76, - 179, 149, 10, 69, 8, 238, 189, 2, 67, 236, 243, 5, 10, 70, 84, 73, 78, - 71, 32, 80, 79, 73, 78, 201, 192, 15, 3, 71, 79, 78, 8, 56, 6, 77, 69, - 68, 65, 82, 89, 34, 80, 251, 174, 26, 79, 2, 217, 160, 20, 3, 32, 67, 65, - 4, 240, 234, 17, 6, 32, 79, 70, 32, 66, 76, 191, 194, 9, 76, 2, 217, 165, - 12, 3, 83, 84, 73, 162, 2, 76, 7, 80, 76, 79, 89, 65, 78, 32, 128, 187, - 19, 2, 77, 80, 247, 162, 8, 67, 158, 2, 212, 2, 6, 65, 70, 70, 73, 88, - 32, 152, 7, 7, 76, 69, 84, 84, 69, 82, 32, 208, 188, 1, 16, 84, 72, 73, - 67, 75, 32, 76, 69, 84, 84, 69, 82, 32, 83, 69, 76, 236, 160, 3, 4, 68, - 79, 85, 66, 204, 172, 16, 19, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, - 32, 67, 72, 73, 78, 79, 79, 75, 153, 143, 5, 11, 83, 73, 71, 78, 32, 79, - 32, 87, 73, 84, 72, 64, 140, 1, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, - 184, 1, 5, 72, 73, 71, 72, 32, 102, 76, 40, 4, 82, 73, 71, 72, 169, 2, 4, - 77, 73, 68, 32, 14, 112, 2, 84, 65, 224, 4, 13, 76, 69, 70, 84, 45, 84, - 79, 45, 82, 73, 71, 72, 84, 26, 83, 146, 165, 20, 69, 3, 73, 6, 44, 5, - 78, 71, 69, 78, 84, 227, 210, 26, 73, 5, 227, 169, 20, 32, 20, 166, 2, - 76, 50, 84, 38, 86, 238, 134, 8, 71, 186, 2, 65, 154, 231, 17, 87, 170, - 25, 67, 223, 126, 68, 24, 36, 2, 69, 70, 29, 3, 79, 87, 32, 2, 209, 2, 3, - 84, 32, 72, 22, 90, 65, 38, 76, 50, 84, 38, 86, 238, 134, 8, 71, 210, - 233, 17, 87, 170, 25, 67, 223, 126, 68, 4, 202, 150, 21, 67, 131, 169, 5, - 82, 4, 220, 216, 7, 3, 79, 78, 71, 207, 231, 18, 73, 2, 173, 254, 7, 4, - 73, 71, 72, 84, 4, 37, 7, 69, 82, 84, 73, 67, 65, 76, 5, 131, 1, 32, 4, - 42, 72, 41, 6, 86, 69, 82, 84, 73, 67, 2, 37, 7, 79, 82, 73, 90, 79, 78, - 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, 237, 205, 26, 2, 69, 67, - 214, 1, 222, 1, 65, 22, 68, 34, 69, 30, 70, 22, 71, 22, 74, 162, 1, 75, - 66, 76, 50, 77, 62, 78, 134, 1, 79, 50, 80, 86, 82, 74, 83, 210, 2, 84, - 66, 85, 42, 86, 38, 87, 70, 88, 230, 206, 26, 72, 138, 60, 73, 194, 41, - 89, 215, 22, 66, 5, 203, 176, 27, 79, 7, 206, 151, 27, 32, 211, 61, 72, - 7, 254, 212, 27, 69, 3, 85, 5, 143, 133, 27, 32, 5, 207, 129, 24, 32, 19, - 11, 32, 16, 76, 8, 87, 73, 84, 72, 32, 68, 79, 84, 230, 5, 77, 2, 78, - 215, 170, 26, 83, 5, 233, 214, 26, 12, 83, 32, 73, 78, 83, 73, 68, 69, - 32, 65, 78, 68, 9, 26, 32, 131, 211, 27, 75, 4, 138, 128, 24, 82, 247, - 210, 3, 77, 9, 248, 243, 21, 3, 79, 78, 71, 227, 222, 5, 72, 11, 11, 32, - 8, 162, 4, 78, 238, 170, 26, 87, 147, 163, 1, 83, 19, 38, 32, 49, 5, 65, - 83, 65, 76, 32, 8, 202, 3, 77, 238, 170, 26, 87, 147, 163, 1, 83, 8, 150, - 209, 27, 65, 2, 73, 2, 79, 3, 85, 11, 218, 207, 27, 79, 146, 1, 65, 2, - 85, 3, 87, 9, 52, 7, 69, 82, 78, 73, 78, 32, 65, 179, 128, 27, 32, 4, - 130, 208, 27, 77, 3, 78, 11, 248, 240, 21, 6, 79, 77, 65, 78, 73, 65, - 158, 161, 5, 32, 211, 61, 72, 49, 58, 32, 164, 1, 5, 76, 79, 65, 78, 32, - 195, 158, 6, 72, 26, 102, 74, 22, 75, 2, 80, 2, 84, 20, 7, 87, 73, 84, - 72, 32, 68, 79, 214, 205, 27, 77, 2, 78, 3, 83, 5, 171, 144, 27, 32, 5, - 151, 149, 27, 32, 4, 183, 165, 12, 84, 18, 74, 69, 138, 176, 5, 79, 154, - 205, 21, 65, 198, 78, 68, 146, 1, 74, 3, 85, 6, 242, 204, 27, 69, 2, 72, - 3, 78, 9, 26, 32, 183, 204, 27, 72, 4, 190, 249, 23, 82, 247, 210, 3, 83, - 9, 186, 252, 26, 32, 214, 79, 72, 3, 73, 5, 217, 237, 6, 4, 79, 67, 65, - 76, 17, 66, 79, 178, 146, 27, 32, 250, 36, 69, 218, 19, 65, 2, 72, 3, 73, - 5, 255, 202, 27, 87, 250, 28, 234, 2, 65, 188, 3, 10, 68, 73, 84, 79, 82, - 73, 65, 76, 32, 67, 22, 71, 240, 75, 4, 73, 71, 72, 84, 198, 1, 76, 194, - 9, 77, 214, 5, 78, 246, 3, 79, 36, 2, 81, 85, 182, 8, 82, 222, 1, 83, - 118, 84, 242, 31, 85, 198, 1, 88, 252, 3, 2, 89, 69, 152, 72, 4, 45, 77, - 65, 73, 132, 207, 18, 3, 74, 69, 67, 168, 134, 2, 8, 86, 69, 82, 71, 82, - 69, 69, 78, 159, 147, 5, 80, 22, 38, 82, 130, 230, 25, 83, 179, 64, 71, - 19, 30, 32, 137, 1, 2, 84, 72, 6, 88, 3, 79, 70, 32, 221, 210, 4, 13, 87, - 73, 84, 72, 32, 72, 69, 65, 82, 73, 78, 71, 32, 4, 216, 138, 11, 2, 77, - 65, 227, 206, 6, 82, 11, 17, 2, 32, 71, 8, 44, 5, 76, 79, 66, 69, 32, - 239, 165, 24, 82, 6, 70, 65, 149, 207, 13, 11, 69, 85, 82, 79, 80, 69, - 45, 65, 70, 82, 73, 4, 128, 214, 9, 4, 77, 69, 82, 73, 133, 154, 2, 11, - 83, 73, 65, 45, 65, 85, 83, 84, 82, 65, 76, 2, 191, 163, 2, 79, 174, 17, - 96, 18, 89, 80, 84, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, - 72, 32, 199, 195, 27, 71, 172, 17, 146, 3, 65, 178, 4, 66, 52, 2, 67, 48, - 224, 1, 2, 68, 48, 170, 4, 69, 198, 3, 70, 164, 4, 2, 71, 48, 210, 3, 72, - 222, 1, 73, 238, 2, 76, 122, 77, 134, 9, 78, 210, 5, 79, 140, 5, 2, 80, - 48, 120, 2, 82, 48, 232, 1, 2, 83, 48, 190, 3, 84, 220, 2, 2, 85, 48, - 250, 2, 86, 134, 5, 87, 236, 2, 3, 88, 48, 48, 88, 3, 89, 48, 48, 84, 2, - 90, 48, 224, 196, 3, 3, 75, 48, 48, 177, 218, 14, 3, 81, 48, 48, 228, 1, - 30, 48, 133, 3, 2, 65, 48, 160, 1, 86, 48, 98, 49, 102, 52, 234, 55, 51, - 138, 163, 20, 55, 230, 216, 1, 50, 2, 53, 3, 54, 24, 238, 68, 54, 146, - 185, 21, 53, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, - 24, 158, 253, 21, 52, 2, 55, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, - 2, 54, 2, 56, 3, 57, 28, 186, 252, 21, 48, 2, 50, 2, 51, 2, 53, 174, 193, - 5, 49, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 68, 46, 48, 186, 55, 51, 138, - 251, 21, 49, 3, 50, 22, 150, 66, 55, 190, 250, 26, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 56, 3, 57, 26, 148, 9, 4, 69, 71, 73, 78, 253, 25, - 2, 48, 48, 56, 34, 48, 90, 49, 207, 199, 8, 50, 24, 170, 40, 50, 250, - 146, 27, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 22, 158, - 249, 21, 48, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, - 2, 56, 3, 57, 184, 1, 70, 48, 94, 51, 102, 52, 102, 53, 106, 54, 134, 30, - 50, 251, 141, 22, 49, 20, 238, 247, 21, 56, 174, 193, 5, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 24, 146, 247, 21, 49, 2, 52, 174, - 193, 5, 48, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 174, - 246, 21, 54, 2, 56, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 55, 3, 57, 42, 154, 61, 48, 178, 184, 21, 50, 2, 52, 174, 193, 5, 49, - 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 32, 134, 61, 55, 138, 249, 26, - 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 94, 30, 48, 189, 2, 2, 78, - 68, 88, 38, 48, 94, 50, 102, 51, 235, 7, 49, 22, 202, 243, 21, 56, 2, 57, - 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 24, 238, 242, - 21, 48, 2, 56, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, - 3, 57, 18, 138, 242, 21, 52, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 54, - 2, 55, 3, 56, 6, 11, 32, 6, 68, 2, 83, 69, 136, 160, 5, 6, 87, 65, 76, - 76, 69, 68, 191, 19, 69, 2, 231, 238, 26, 71, 132, 1, 42, 48, 133, 9, 5, - 85, 76, 76, 32, 66, 130, 1, 54, 48, 94, 49, 102, 51, 102, 52, 102, 53, - 215, 1, 50, 20, 226, 239, 21, 49, 174, 193, 5, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 2, 55, 2, 56, 3, 57, 22, 134, 239, 21, 51, 174, 193, 5, 48, 2, 49, - 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 26, 162, 238, 21, 49, 2, - 55, 2, 56, 174, 193, 5, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 57, 26, - 190, 237, 21, 53, 2, 54, 2, 55, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 56, 3, 57, 14, 142, 27, 49, 250, 146, 27, 48, 2, 50, 3, 51, 128, - 1, 62, 48, 98, 49, 102, 51, 102, 52, 130, 28, 50, 239, 155, 8, 53, 24, - 214, 50, 55, 146, 185, 21, 54, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, - 53, 2, 56, 3, 57, 22, 134, 235, 21, 49, 174, 193, 5, 48, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 162, 234, 21, 54, 2, 55, 174, - 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 56, 3, 57, 24, 190, - 233, 21, 51, 2, 53, 174, 193, 5, 48, 2, 49, 2, 50, 2, 52, 2, 54, 2, 55, - 2, 56, 3, 57, 24, 80, 2, 48, 48, 84, 4, 65, 76, 70, 32, 209, 40, 7, 79, - 82, 73, 90, 79, 78, 84, 18, 134, 232, 21, 54, 174, 193, 5, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 55, 3, 56, 4, 22, 66, 175, 43, 76, 2, 237, 233, 23, - 2, 76, 65, 52, 58, 48, 181, 1, 9, 78, 83, 69, 82, 84, 32, 65, 84, 32, 38, - 18, 48, 95, 49, 22, 174, 230, 21, 53, 2, 57, 174, 193, 5, 49, 2, 50, 2, - 51, 2, 52, 2, 54, 2, 55, 3, 56, 16, 210, 229, 21, 48, 2, 49, 174, 193, 5, + 135, 237, 21, 83, 30, 88, 4, 68, 79, 87, 78, 0, 2, 85, 80, 153, 1, 9, 86, + 69, 82, 84, 73, 67, 65, 76, 32, 8, 69, 15, 32, 65, 78, 68, 32, 72, 79, + 82, 73, 90, 79, 78, 84, 65, 76, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 150, + 221, 27, 87, 186, 25, 67, 159, 45, 84, 14, 52, 4, 65, 78, 68, 32, 45, 5, + 87, 73, 84, 72, 32, 10, 142, 156, 22, 66, 42, 84, 255, 191, 5, 87, 4, + 190, 245, 27, 67, 159, 45, 84, 2, 193, 226, 24, 3, 32, 83, 84, 170, 1, + 64, 2, 67, 69, 48, 2, 69, 82, 129, 5, 5, 75, 84, 79, 80, 32, 2, 153, 182, + 27, 7, 78, 68, 73, 78, 71, 32, 78, 164, 1, 32, 3, 69, 84, 32, 183, 4, 84, + 160, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 80, 45, 9, + 76, 32, 76, 69, 84, 84, 69, 82, 32, 80, 218, 1, 69, 96, 4, 76, 79, 78, + 71, 0, 5, 83, 72, 79, 82, 84, 74, 79, 30, 84, 2, 90, 158, 231, 11, 67, + 250, 234, 15, 66, 2, 68, 2, 74, 2, 80, 2, 86, 2, 89, 182, 99, 71, 2, 75, + 158, 103, 87, 162, 19, 65, 203, 17, 72, 20, 242, 162, 25, 83, 134, 165, + 2, 78, 138, 250, 1, 84, 146, 1, 70, 2, 76, 2, 77, 2, 82, 3, 87, 12, 11, + 32, 12, 134, 162, 25, 65, 190, 218, 1, 79, 226, 197, 2, 69, 3, 73, 4, + 230, 193, 29, 73, 3, 87, 4, 146, 210, 27, 72, 231, 216, 1, 69, 5, 161, + 179, 28, 4, 32, 73, 83, 76, 4, 158, 198, 26, 67, 193, 224, 1, 4, 87, 73, + 78, 68, 202, 2, 100, 8, 65, 78, 65, 71, 65, 82, 73, 32, 161, 18, 12, 73, + 67, 69, 32, 67, 79, 78, 84, 82, 79, 76, 32, 192, 2, 222, 1, 65, 38, 67, + 22, 71, 36, 4, 72, 69, 65, 68, 96, 7, 76, 69, 84, 84, 69, 82, 32, 198, 5, + 83, 148, 7, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 154, 129, 19, + 68, 212, 129, 6, 4, 74, 65, 73, 78, 171, 156, 4, 79, 4, 210, 216, 12, 67, + 151, 206, 12, 66, 2, 159, 146, 6, 65, 4, 194, 216, 12, 82, 135, 238, 1, + 65, 6, 44, 5, 32, 77, 65, 82, 75, 251, 140, 29, 83, 5, 181, 143, 19, 7, + 32, 87, 73, 84, 72, 32, 72, 158, 1, 158, 2, 65, 54, 67, 62, 68, 50, 71, + 62, 72, 52, 2, 77, 65, 46, 83, 130, 10, 79, 246, 147, 7, 66, 206, 252, 4, + 75, 150, 202, 3, 82, 238, 232, 6, 89, 154, 119, 85, 174, 30, 78, 254, + 198, 1, 74, 150, 52, 76, 70, 84, 46, 86, 134, 207, 1, 73, 158, 194, 1, + 80, 2, 90, 138, 69, 70, 2, 81, 187, 2, 69, 13, 158, 186, 29, 65, 2, 73, + 2, 85, 2, 87, 3, 89, 10, 26, 65, 155, 183, 29, 72, 9, 185, 2, 4, 78, 68, + 82, 65, 12, 178, 223, 25, 68, 198, 215, 3, 72, 187, 2, 65, 10, 130, 238, + 12, 76, 190, 131, 16, 72, 138, 69, 71, 187, 2, 65, 4, 232, 210, 16, 4, + 69, 65, 86, 89, 219, 229, 12, 65, 5, 233, 154, 7, 6, 82, 87, 65, 82, 73, + 32, 12, 38, 72, 134, 181, 29, 83, 187, 2, 65, 8, 36, 3, 79, 82, 84, 151, + 183, 29, 65, 6, 199, 136, 12, 32, 68, 168, 1, 19, 69, 81, 85, 69, 78, 67, + 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 104, 4, 73, 71, 78, + 32, 133, 254, 26, 10, 84, 82, 69, 83, 83, 32, 83, 73, 71, 78, 16, 174, + 217, 3, 71, 2, 75, 228, 225, 23, 3, 68, 68, 68, 2, 82, 230, 247, 1, 89, + 38, 70, 2, 81, 3, 90, 48, 178, 3, 66, 0, 10, 69, 88, 84, 69, 78, 68, 69, + 68, 32, 66, 36, 11, 67, 65, 78, 68, 82, 65, 66, 73, 78, 68, 85, 64, 8, + 87, 69, 83, 84, 69, 82, 78, 32, 32, 10, 82, 69, 86, 69, 82, 83, 69, 68, + 32, 78, 226, 200, 6, 83, 190, 190, 12, 73, 142, 149, 6, 77, 46, 78, 242, + 60, 65, 72, 18, 68, 79, 85, 66, 76, 69, 32, 67, 65, 78, 68, 82, 65, 66, + 73, 78, 68, 85, 62, 80, 216, 196, 2, 12, 72, 73, 71, 72, 32, 83, 80, 65, + 67, 73, 78, 71, 139, 78, 86, 4, 241, 222, 12, 4, 72, 65, 76, 69, 11, 11, + 32, 8, 134, 194, 22, 65, 238, 152, 3, 86, 183, 230, 1, 84, 4, 30, 78, 21, + 3, 70, 73, 86, 2, 17, 2, 73, 78, 2, 129, 221, 21, 8, 69, 45, 76, 73, 75, + 69, 32, 66, 50, 150, 1, 65, 52, 7, 67, 65, 78, 68, 82, 65, 32, 62, 79, + 186, 134, 19, 80, 162, 180, 4, 85, 158, 224, 1, 83, 222, 63, 86, 186, + 201, 1, 73, 223, 137, 2, 69, 10, 210, 174, 29, 65, 2, 73, 2, 85, 2, 87, + 3, 89, 6, 172, 135, 19, 4, 76, 79, 78, 71, 242, 166, 10, 69, 3, 79, 7, + 142, 151, 29, 79, 215, 22, 69, 10, 194, 154, 24, 70, 136, 6, 2, 83, 84, + 154, 157, 3, 84, 183, 86, 79, 230, 2, 182, 2, 65, 150, 2, 69, 74, 71, + 224, 4, 6, 78, 71, 66, 65, 84, 32, 248, 1, 5, 82, 69, 67, 84, 32, 98, 83, + 178, 2, 86, 200, 7, 2, 89, 65, 32, 4, 90, 90, 89, 32, 224, 172, 5, 2, 84, + 84, 184, 156, 15, 10, 70, 70, 69, 82, 69, 78, 67, 69, 32, 66, 173, 205, + 7, 12, 77, 69, 78, 83, 73, 79, 78, 32, 79, 82, 73, 71, 20, 42, 77, 242, + 251, 8, 69, 183, 134, 13, 71, 16, 40, 4, 79, 78, 68, 32, 207, 152, 3, 69, + 14, 128, 1, 5, 87, 73, 84, 72, 32, 186, 164, 18, 84, 164, 224, 2, 12, 83, + 72, 65, 80, 69, 32, 87, 73, 84, 72, 32, 65, 183, 228, 4, 79, 8, 190, 128, + 22, 66, 182, 2, 84, 238, 137, 4, 76, 27, 82, 14, 176, 131, 5, 5, 32, 70, + 65, 67, 69, 133, 161, 17, 4, 83, 69, 76, 32, 78, 64, 3, 73, 84, 32, 149, + 2, 8, 82, 65, 77, 32, 70, 79, 82, 32, 60, 98, 70, 44, 2, 78, 73, 2, 79, + 14, 83, 46, 84, 44, 3, 90, 69, 82, 13, 5, 69, 73, 71, 72, 84, 12, 128, 1, + 2, 73, 86, 25, 3, 79, 85, 82, 6, 87, 78, 12, 96, 4, 69, 86, 69, 78, 1, 2, + 73, 88, 12, 28, 3, 72, 82, 69, 15, 87, 6, 23, 69, 6, 11, 79, 7, 143, 247, + 16, 32, 18, 88, 5, 69, 65, 82, 84, 72, 64, 5, 71, 82, 69, 65, 84, 0, 4, + 76, 69, 83, 83, 39, 72, 7, 25, 4, 76, 89, 32, 72, 4, 194, 174, 24, 85, + 147, 141, 1, 69, 4, 169, 132, 13, 4, 69, 82, 32, 89, 4, 188, 148, 5, 7, + 69, 65, 86, 69, 78, 76, 89, 1, 4, 85, 77, 65, 78, 64, 120, 17, 78, 69, + 71, 65, 84, 73, 86, 69, 32, 67, 73, 82, 67, 76, 69, 68, 32, 41, 9, 67, + 73, 82, 67, 76, 69, 68, 32, 83, 42, 38, 83, 154, 32, 78, 151, 196, 20, + 68, 22, 49, 10, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 22, 226, 31, 78, + 175, 144, 27, 68, 4, 148, 233, 22, 15, 67, 85, 82, 82, 69, 78, 84, 32, + 83, 89, 77, 66, 79, 76, 32, 195, 155, 6, 72, 10, 70, 65, 144, 1, 5, 67, + 79, 78, 84, 73, 197, 235, 25, 3, 71, 85, 73, 6, 76, 9, 80, 80, 79, 73, + 78, 84, 69, 68, 32, 133, 132, 27, 4, 66, 76, 69, 68, 4, 252, 159, 17, 7, + 66, 85, 84, 32, 82, 69, 76, 147, 194, 11, 70, 2, 53, 11, 78, 85, 79, 85, + 83, 32, 85, 78, 68, 69, 82, 2, 137, 130, 17, 3, 76, 73, 78, 156, 1, 80, + 9, 69, 83, 32, 65, 75, 85, 82, 85, 32, 238, 5, 73, 157, 251, 16, 2, 79, + 82, 144, 1, 142, 2, 68, 36, 7, 76, 69, 84, 84, 69, 82, 32, 236, 1, 5, 83, + 73, 71, 78, 32, 58, 86, 140, 162, 9, 6, 77, 69, 68, 73, 65, 76, 182, 255, + 4, 71, 222, 232, 4, 69, 128, 203, 4, 12, 80, 82, 69, 70, 73, 88, 69, 68, + 32, 78, 65, 83, 217, 248, 4, 7, 73, 78, 73, 84, 73, 65, 76, 22, 174, 190, + 25, 79, 211, 235, 1, 73, 84, 142, 195, 10, 84, 190, 237, 11, 89, 162, + 126, 65, 138, 146, 2, 68, 214, 6, 85, 206, 201, 1, 73, 42, 76, 250, 192, + 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 75, 2, 80, 138, 69, 72, 2, 74, 2, + 77, 2, 82, 2, 86, 2, 90, 186, 2, 69, 3, 79, 8, 174, 217, 24, 72, 150, 43, + 67, 98, 78, 235, 206, 3, 65, 18, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, + 78, 32, 179, 223, 26, 73, 16, 134, 150, 15, 65, 218, 174, 10, 85, 206, + 201, 1, 73, 222, 137, 2, 69, 3, 79, 10, 68, 5, 83, 73, 79, 78, 32, 212, + 241, 1, 2, 78, 71, 171, 145, 26, 68, 6, 26, 83, 227, 227, 5, 84, 4, 170, + 174, 27, 76, 211, 97, 73, 2, 165, 132, 19, 3, 32, 76, 65, 4, 134, 131, + 28, 83, 139, 86, 70, 212, 5, 188, 2, 6, 67, 85, 77, 69, 78, 84, 124, 7, + 69, 83, 32, 78, 79, 84, 32, 194, 3, 71, 202, 3, 76, 36, 10, 77, 73, 78, + 79, 32, 84, 73, 76, 69, 32, 226, 1, 78, 38, 84, 246, 2, 85, 204, 17, 2, + 87, 78, 164, 189, 16, 8, 32, 78, 79, 84, 32, 76, 73, 84, 156, 250, 11, 8, + 86, 69, 32, 79, 70, 32, 80, 69, 174, 4, 79, 235, 25, 68, 9, 33, 6, 32, + 87, 73, 84, 72, 32, 6, 40, 4, 84, 69, 88, 84, 139, 142, 2, 80, 5, 133, + 142, 2, 6, 32, 65, 78, 68, 32, 80, 22, 164, 1, 11, 67, 79, 78, 84, 65, + 73, 78, 32, 65, 83, 32, 92, 6, 68, 73, 86, 73, 68, 69, 112, 2, 80, 82, + 48, 7, 83, 85, 67, 67, 69, 69, 68, 253, 213, 22, 2, 70, 79, 6, 248, 1, + 15, 78, 79, 82, 77, 65, 76, 32, 83, 85, 66, 71, 82, 79, 85, 80, 231, 245, + 20, 77, 5, 217, 249, 21, 23, 32, 87, 73, 84, 72, 32, 82, 69, 86, 69, 82, + 83, 69, 68, 32, 78, 69, 71, 65, 84, 73, 79, 78, 6, 44, 5, 69, 67, 69, 68, + 69, 195, 146, 28, 79, 5, 233, 158, 9, 2, 32, 79, 125, 36, 3, 82, 65, 32, + 191, 209, 28, 32, 120, 120, 7, 76, 69, 84, 84, 69, 82, 32, 208, 1, 11, + 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 170, 161, 23, 65, 187, 2, 83, + 88, 178, 179, 25, 65, 38, 68, 82, 82, 34, 84, 230, 5, 85, 206, 201, 1, + 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, + 138, 69, 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 22, 194, 140, + 19, 86, 246, 171, 6, 65, 38, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, + 4, 202, 141, 21, 80, 183, 188, 2, 76, 200, 1, 72, 8, 72, 79, 82, 73, 90, + 79, 78, 84, 1, 6, 86, 69, 82, 84, 73, 67, 100, 17, 2, 65, 76, 100, 32, 2, + 45, 48, 155, 251, 23, 32, 98, 58, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 3, 54, 14, 205, 166, 20, 2, 45, 48, 4, 174, 239, 26, 75, 255, 146, 1, 71, + 26, 34, 32, 57, 4, 84, 69, 68, 32, 8, 170, 130, 25, 80, 34, 77, 194, 70, + 79, 239, 194, 2, 65, 18, 214, 1, 67, 34, 83, 220, 145, 16, 3, 76, 73, 78, + 148, 133, 1, 4, 79, 66, 69, 76, 244, 9, 9, 84, 82, 65, 78, 83, 80, 79, + 83, 73, 238, 220, 6, 70, 141, 41, 14, 82, 73, 71, 72, 84, 45, 80, 79, 73, + 78, 84, 73, 78, 71, 4, 226, 180, 27, 73, 231, 16, 82, 4, 134, 243, 26, + 79, 223, 114, 81, 162, 1, 40, 3, 66, 76, 69, 137, 17, 2, 71, 72, 160, 1, + 42, 32, 190, 10, 45, 241, 5, 2, 68, 32, 106, 226, 2, 67, 174, 1, 68, 38, + 72, 32, 4, 73, 78, 84, 69, 38, 76, 144, 1, 7, 78, 69, 83, 84, 69, 68, 32, + 74, 80, 66, 83, 130, 2, 85, 36, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, + 144, 161, 6, 6, 79, 66, 76, 73, 81, 85, 196, 189, 2, 9, 82, 73, 71, 72, + 84, 32, 65, 82, 67, 246, 183, 3, 65, 242, 215, 9, 69, 182, 188, 4, 81, + 177, 105, 6, 87, 65, 86, 89, 32, 79, 24, 100, 7, 73, 82, 67, 76, 69, 68, + 32, 180, 207, 7, 4, 85, 82, 76, 89, 241, 196, 1, 4, 79, 76, 79, 78, 20, + 26, 78, 151, 196, 20, 68, 2, 157, 213, 2, 5, 85, 77, 66, 69, 82, 4, 190, + 236, 18, 79, 135, 199, 6, 65, 4, 198, 190, 20, 73, 223, 63, 89, 4, 154, + 130, 26, 82, 251, 244, 1, 71, 12, 54, 79, 129, 156, 17, 7, 69, 70, 84, + 32, 65, 82, 67, 10, 26, 87, 199, 190, 25, 71, 6, 26, 45, 139, 230, 27, + 32, 4, 174, 189, 20, 82, 211, 1, 57, 6, 224, 137, 17, 9, 76, 69, 83, 83, + 45, 84, 72, 65, 78, 179, 227, 9, 71, 8, 26, 82, 207, 248, 26, 76, 6, 242, + 157, 1, 69, 207, 235, 15, 73, 18, 80, 6, 81, 85, 65, 82, 69, 32, 30, 84, + 50, 85, 149, 215, 13, 4, 79, 76, 73, 68, 4, 230, 205, 25, 73, 55, 85, 4, + 204, 191, 12, 3, 65, 67, 75, 135, 201, 4, 82, 8, 58, 83, 146, 156, 1, 67, + 222, 136, 21, 80, 151, 236, 4, 66, 2, 169, 147, 28, 4, 80, 69, 78, 83, 4, + 214, 231, 18, 80, 139, 196, 9, 78, 10, 36, 3, 66, 65, 82, 163, 226, 27, + 76, 9, 11, 32, 6, 52, 7, 68, 79, 85, 66, 76, 69, 32, 151, 213, 26, 76, 4, + 146, 213, 26, 76, 51, 82, 48, 112, 5, 76, 73, 78, 69, 32, 220, 1, 7, 83, + 84, 82, 85, 67, 75, 32, 141, 226, 8, 7, 69, 78, 68, 69, 68, 32, 77, 12, + 48, 8, 83, 76, 65, 78, 84, 69, 68, 32, 75, 69, 8, 70, 69, 56, 7, 71, 82, + 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 4, 185, 179, 20, 9, 81, 85, 65, + 76, 32, 84, 79, 32, 79, 2, 221, 165, 14, 5, 45, 84, 72, 65, 78, 34, 160, + 1, 8, 67, 65, 80, 73, 84, 65, 76, 32, 92, 7, 73, 84, 65, 76, 73, 67, 32, + 124, 6, 83, 77, 65, 76, 76, 32, 245, 208, 13, 8, 78, 45, 65, 82, 89, 32, + 83, 85, 18, 198, 163, 12, 71, 226, 201, 14, 80, 222, 137, 2, 67, 2, 72, + 2, 78, 2, 81, 2, 82, 3, 90, 10, 76, 6, 83, 77, 65, 76, 76, 32, 201, 230, + 21, 7, 67, 65, 80, 73, 84, 65, 76, 8, 214, 245, 28, 68, 2, 69, 2, 73, 3, + 74, 4, 238, 161, 12, 71, 231, 191, 16, 80, 6, 218, 161, 10, 70, 26, 77, + 135, 192, 17, 83, 2, 139, 251, 27, 78, 166, 1, 50, 32, 158, 1, 45, 189, + 1, 4, 87, 65, 82, 68, 10, 60, 4, 84, 65, 67, 75, 226, 206, 25, 70, 30, + 82, 191, 79, 65, 5, 29, 5, 32, 87, 73, 84, 72, 2, 11, 32, 2, 11, 67, 2, + 129, 245, 20, 4, 73, 82, 67, 76, 28, 60, 9, 80, 79, 73, 78, 84, 73, 78, + 71, 32, 211, 207, 25, 70, 24, 62, 83, 246, 209, 25, 65, 86, 69, 62, 70, + 46, 82, 207, 1, 84, 4, 142, 180, 17, 84, 205, 160, 8, 6, 77, 65, 76, 76, + 32, 82, 128, 1, 56, 8, 32, 70, 65, 67, 73, 78, 71, 32, 89, 2, 83, 32, 8, + 54, 72, 1, 9, 78, 79, 84, 67, 72, 69, 68, 32, 72, 4, 181, 204, 27, 3, 79, + 79, 75, 120, 178, 1, 65, 200, 2, 6, 66, 76, 65, 67, 75, 32, 38, 84, 172, + 237, 8, 2, 87, 72, 150, 154, 8, 90, 218, 250, 8, 68, 50, 70, 82, 72, 150, + 4, 67, 46, 81, 42, 82, 22, 83, 247, 7, 80, 34, 40, 4, 82, 82, 79, 87, + 179, 253, 25, 78, 33, 11, 32, 30, 144, 1, 5, 87, 73, 84, 72, 32, 160, + 228, 13, 14, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 85, 166, + 153, 12, 65, 178, 10, 70, 179, 5, 84, 22, 184, 183, 5, 6, 67, 79, 82, 78, + 69, 82, 182, 200, 20, 68, 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 250, + 12, 84, 139, 58, 72, 6, 178, 130, 21, 65, 175, 255, 4, 67, 36, 36, 2, 82, + 73, 161, 2, 2, 87, 79, 32, 44, 5, 65, 78, 71, 76, 69, 143, 144, 26, 80, + 30, 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 203, 149, 26, 32, 28, 68, 5, + 65, 82, 82, 79, 87, 230, 133, 17, 90, 166, 136, 9, 68, 39, 80, 23, 11, + 32, 20, 184, 137, 26, 15, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, + 32, 85, 80, 98, 84, 23, 87, 4, 134, 143, 26, 32, 105, 15, 45, 72, 69, 65, + 68, 69, 68, 32, 65, 82, 82, 79, 87, 32, 87, 22, 138, 1, 65, 106, 79, 152, + 1, 12, 85, 77, 32, 87, 73, 84, 72, 32, 68, 82, 85, 77, 160, 242, 16, 6, + 73, 86, 69, 32, 83, 76, 231, 220, 10, 69, 8, 238, 202, 2, 67, 200, 147, + 6, 10, 70, 84, 73, 78, 71, 32, 80, 79, 73, 78, 133, 235, 15, 3, 71, 79, + 78, 8, 56, 6, 77, 69, 68, 65, 82, 89, 34, 80, 159, 177, 27, 79, 2, 217, + 234, 20, 3, 32, 67, 65, 4, 204, 181, 18, 6, 32, 79, 70, 32, 66, 76, 171, + 251, 9, 76, 2, 189, 217, 12, 3, 83, 84, 73, 162, 2, 76, 7, 80, 76, 79, + 89, 65, 78, 32, 244, 132, 20, 2, 77, 80, 215, 220, 8, 67, 158, 2, 212, 2, + 6, 65, 70, 70, 73, 88, 32, 160, 7, 7, 76, 69, 84, 84, 69, 82, 32, 148, + 194, 1, 16, 84, 72, 73, 67, 75, 32, 76, 69, 84, 84, 69, 82, 32, 83, 69, + 76, 180, 174, 3, 4, 68, 79, 85, 66, 220, 230, 16, 19, 80, 85, 78, 67, 84, + 85, 65, 84, 73, 79, 78, 32, 67, 72, 73, 78, 79, 79, 75, 149, 194, 5, 11, + 83, 73, 71, 78, 32, 79, 32, 87, 73, 84, 72, 64, 140, 1, 9, 65, 84, 84, + 65, 67, 72, 69, 68, 32, 184, 1, 5, 72, 73, 71, 72, 32, 106, 76, 40, 4, + 82, 73, 71, 72, 173, 2, 4, 77, 73, 68, 32, 14, 112, 2, 84, 65, 232, 4, + 13, 76, 69, 70, 84, 45, 84, 79, 45, 82, 73, 71, 72, 84, 26, 83, 166, 239, + 20, 69, 3, 73, 6, 44, 5, 78, 71, 69, 78, 84, 159, 214, 27, 73, 5, 255, + 243, 20, 32, 20, 174, 2, 76, 50, 84, 38, 86, 166, 174, 8, 71, 186, 2, 65, + 170, 192, 18, 87, 186, 25, 67, 199, 129, 1, 68, 24, 36, 2, 69, 70, 29, 3, + 79, 87, 32, 2, 213, 2, 3, 84, 32, 72, 22, 94, 65, 38, 76, 50, 84, 38, 86, + 166, 174, 8, 71, 226, 194, 18, 87, 186, 25, 67, 199, 129, 1, 68, 4, 186, + 229, 21, 67, 199, 221, 5, 82, 4, 132, 128, 8, 3, 79, 78, 71, 219, 195, + 19, 73, 2, 229, 165, 8, 4, 73, 71, 72, 84, 4, 37, 7, 69, 82, 84, 73, 67, + 65, 76, 5, 131, 1, 32, 4, 42, 72, 41, 6, 86, 69, 82, 84, 73, 67, 2, 37, + 7, 79, 82, 73, 90, 79, 78, 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, + 161, 209, 27, 2, 69, 67, 214, 1, 222, 1, 65, 22, 68, 34, 69, 30, 70, 22, + 71, 22, 74, 162, 1, 75, 66, 76, 50, 77, 62, 78, 134, 1, 79, 50, 80, 86, + 82, 74, 83, 210, 2, 84, 66, 85, 42, 86, 38, 87, 70, 88, 162, 210, 27, 72, + 142, 60, 73, 206, 41, 89, 215, 22, 66, 5, 151, 180, 28, 79, 7, 142, 155, + 28, 32, 223, 61, 72, 7, 202, 216, 28, 69, 3, 85, 5, 207, 136, 28, 32, 5, + 255, 226, 24, 32, 19, 11, 32, 16, 76, 8, 87, 73, 84, 72, 32, 68, 79, 84, + 230, 5, 77, 2, 78, 195, 173, 27, 83, 5, 165, 218, 27, 12, 83, 32, 73, 78, + 83, 73, 68, 69, 32, 65, 78, 68, 9, 26, 32, 207, 214, 28, 75, 4, 186, 225, + 24, 82, 147, 245, 3, 77, 9, 248, 204, 22, 3, 79, 78, 71, 175, 137, 6, 72, + 11, 11, 32, 8, 162, 4, 78, 218, 173, 27, 87, 243, 163, 1, 83, 19, 38, 32, + 49, 5, 65, 83, 65, 76, 32, 8, 202, 3, 77, 218, 173, 27, 87, 243, 163, 1, + 83, 8, 226, 212, 28, 65, 2, 73, 2, 79, 3, 85, 11, 166, 211, 28, 79, 146, + 1, 65, 2, 85, 3, 87, 9, 52, 7, 69, 82, 78, 73, 78, 32, 65, 243, 131, 28, + 32, 4, 206, 211, 28, 77, 3, 78, 11, 248, 201, 22, 6, 79, 77, 65, 78, 73, + 65, 222, 203, 5, 32, 223, 61, 72, 49, 58, 32, 164, 1, 5, 76, 79, 65, 78, + 32, 191, 178, 6, 72, 26, 102, 74, 22, 75, 2, 80, 2, 84, 20, 7, 87, 73, + 84, 72, 32, 68, 79, 162, 209, 28, 77, 2, 78, 3, 83, 5, 235, 147, 28, 32, + 5, 215, 152, 28, 32, 4, 139, 217, 12, 84, 18, 74, 69, 230, 195, 5, 79, + 254, 188, 22, 65, 210, 78, 68, 146, 1, 74, 3, 85, 6, 190, 208, 28, 69, 2, + 72, 3, 78, 9, 26, 32, 131, 208, 28, 72, 4, 238, 218, 24, 82, 147, 245, 3, + 83, 9, 250, 255, 27, 32, 226, 79, 72, 3, 73, 5, 245, 148, 7, 4, 79, 67, + 65, 76, 17, 66, 79, 242, 149, 28, 32, 134, 37, 69, 218, 19, 65, 2, 72, 3, + 73, 5, 203, 206, 28, 87, 188, 91, 234, 2, 65, 188, 3, 10, 68, 73, 84, 79, + 82, 73, 65, 76, 32, 67, 22, 71, 224, 79, 4, 73, 71, 72, 84, 146, 2, 76, + 194, 9, 77, 214, 5, 78, 246, 3, 79, 36, 2, 81, 85, 182, 8, 82, 222, 1, + 83, 118, 84, 242, 31, 85, 198, 1, 88, 128, 5, 2, 89, 69, 200, 80, 4, 45, + 77, 65, 73, 148, 140, 19, 3, 74, 69, 67, 228, 147, 2, 8, 86, 69, 82, 71, + 82, 69, 69, 78, 175, 190, 5, 80, 22, 38, 82, 202, 230, 26, 83, 179, 64, + 71, 19, 30, 32, 137, 1, 2, 84, 72, 6, 88, 3, 79, 70, 32, 233, 229, 4, 13, + 87, 73, 84, 72, 32, 72, 69, 65, 82, 73, 78, 71, 32, 4, 140, 190, 11, 2, + 77, 65, 135, 230, 6, 82, 11, 17, 2, 32, 71, 8, 44, 5, 76, 79, 66, 69, 32, + 203, 148, 25, 82, 6, 70, 65, 181, 235, 21, 11, 69, 85, 82, 79, 80, 69, + 45, 65, 70, 82, 73, 4, 176, 137, 10, 4, 77, 69, 82, 73, 133, 154, 2, 11, + 83, 73, 65, 45, 65, 85, 83, 84, 82, 65, 76, 2, 183, 176, 2, 79, 228, 79, + 92, 17, 89, 80, 84, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, + 72, 151, 199, 28, 71, 226, 79, 30, 32, 197, 74, 2, 45, 49, 172, 17, 146, + 3, 65, 178, 4, 66, 52, 2, 67, 48, 224, 1, 2, 68, 48, 170, 4, 69, 178, 3, + 70, 164, 4, 2, 71, 48, 210, 3, 72, 214, 1, 73, 238, 2, 76, 122, 77, 222, + 8, 78, 210, 5, 79, 140, 5, 2, 80, 48, 120, 2, 82, 48, 232, 1, 2, 83, 48, + 190, 3, 84, 220, 2, 2, 85, 48, 250, 2, 86, 134, 5, 87, 236, 2, 3, 88, 48, + 48, 88, 3, 89, 48, 48, 84, 2, 90, 48, 252, 214, 3, 3, 75, 48, 48, 189, + 146, 15, 3, 81, 48, 48, 228, 1, 30, 48, 133, 3, 2, 65, 48, 160, 1, 86, + 48, 98, 49, 102, 52, 166, 55, 51, 162, 241, 20, 55, 150, 227, 1, 50, 2, + 53, 3, 54, 24, 170, 68, 54, 170, 146, 22, 53, 134, 236, 5, 49, 2, 50, 2, + 51, 2, 52, 2, 55, 2, 56, 3, 57, 24, 242, 213, 22, 52, 2, 55, 134, 236, 5, + 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 56, 3, 57, 28, 142, 213, 22, + 48, 2, 50, 2, 51, 2, 53, 134, 236, 5, 49, 2, 52, 2, 54, 2, 55, 2, 56, 3, + 57, 68, 46, 48, 246, 54, 51, 210, 211, 22, 49, 3, 50, 22, 210, 65, 55, + 174, 254, 27, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 26, + 148, 9, 4, 69, 71, 73, 78, 185, 25, 2, 48, 48, 56, 34, 48, 90, 49, 175, + 244, 8, 50, 24, 230, 39, 50, 234, 150, 28, 49, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 22, 242, 209, 22, 48, 134, 236, 5, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 184, 1, 70, 48, 94, 51, + 102, 52, 102, 53, 106, 54, 194, 29, 50, 195, 230, 22, 49, 20, 194, 208, + 22, 56, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, + 24, 230, 207, 22, 49, 2, 52, 134, 236, 5, 48, 2, 50, 2, 51, 2, 53, 2, 54, + 2, 55, 2, 56, 3, 57, 24, 130, 207, 22, 54, 2, 56, 134, 236, 5, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 57, 42, 214, 60, 48, 202, 145, 22, + 50, 2, 52, 134, 236, 5, 49, 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 32, + 194, 60, 55, 250, 252, 27, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, + 94, 30, 48, 189, 2, 2, 78, 68, 88, 38, 48, 94, 50, 102, 51, 215, 7, 49, + 22, 158, 204, 22, 56, 2, 57, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 3, 55, 24, 194, 203, 22, 48, 2, 56, 134, 236, 5, 49, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 18, 222, 202, 22, 52, 134, 236, 5, 48, + 2, 49, 2, 50, 2, 51, 2, 54, 2, 55, 3, 56, 6, 11, 32, 6, 136, 180, 5, 6, + 87, 65, 76, 76, 69, 68, 210, 19, 69, 203, 211, 20, 83, 132, 1, 42, 48, + 133, 9, 5, 85, 76, 76, 32, 66, 130, 1, 54, 48, 94, 49, 102, 51, 102, 52, + 102, 53, 215, 1, 50, 20, 202, 200, 22, 49, 134, 236, 5, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 22, 238, 199, 22, 51, 134, 236, 5, 48, + 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 26, 138, 199, 22, + 49, 2, 55, 2, 56, 134, 236, 5, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, + 57, 26, 166, 198, 22, 53, 2, 54, 2, 55, 134, 236, 5, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 56, 3, 57, 14, 222, 26, 49, 234, 150, 28, 48, 2, 50, 3, 51, + 128, 1, 62, 48, 98, 49, 102, 51, 102, 52, 210, 27, 50, 147, 201, 8, 53, + 24, 166, 50, 55, 170, 146, 22, 54, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 56, 3, 57, 22, 238, 195, 22, 49, 134, 236, 5, 48, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 138, 195, 22, 54, 2, 55, + 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 56, 3, 57, 24, + 166, 194, 22, 51, 2, 53, 134, 236, 5, 48, 2, 49, 2, 50, 2, 52, 2, 54, 2, + 55, 2, 56, 3, 57, 24, 80, 2, 48, 48, 84, 4, 65, 76, 70, 32, 161, 40, 7, + 79, 82, 73, 90, 79, 78, 84, 18, 238, 192, 22, 54, 134, 236, 5, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 4, 22, 66, 255, 42, 76, 2, 223, 238, + 16, 76, 52, 58, 48, 181, 1, 9, 78, 83, 69, 82, 84, 32, 65, 84, 32, 38, + 18, 48, 95, 49, 22, 158, 191, 22, 53, 2, 57, 134, 236, 5, 49, 2, 50, 2, + 51, 2, 52, 2, 54, 2, 55, 3, 56, 16, 194, 190, 22, 48, 2, 49, 134, 236, 5, 50, 2, 51, 2, 52, 3, 53, 14, 68, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, - 80, 239, 138, 19, 77, 7, 11, 32, 4, 134, 152, 26, 69, 221, 6, 2, 83, 84, - 22, 32, 2, 48, 48, 215, 162, 15, 79, 20, 238, 227, 21, 50, 2, 54, 174, - 193, 5, 49, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 164, 1, 158, 1, 48, 128, - 4, 15, 79, 68, 73, 70, 73, 69, 82, 32, 68, 65, 77, 65, 71, 69, 68, 189, - 239, 23, 14, 73, 82, 82, 79, 82, 32, 72, 79, 82, 73, 90, 79, 78, 84, 132, - 1, 42, 48, 98, 49, 106, 50, 102, 51, 107, 52, 24, 182, 40, 49, 146, 185, - 21, 51, 174, 193, 5, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 44, - 138, 41, 50, 222, 183, 21, 48, 2, 53, 2, 54, 2, 55, 174, 193, 5, 49, 2, - 51, 2, 52, 2, 56, 3, 57, 26, 254, 223, 21, 50, 2, 52, 2, 56, 174, 193, 5, - 48, 2, 49, 2, 51, 2, 53, 2, 54, 2, 55, 3, 57, 26, 138, 38, 51, 146, 185, - 21, 49, 174, 193, 5, 48, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, - 12, 178, 222, 21, 48, 174, 193, 5, 49, 2, 50, 2, 51, 3, 52, 31, 25, 4, - 32, 65, 84, 32, 28, 96, 6, 66, 79, 84, 84, 79, 77, 120, 5, 83, 84, 65, - 82, 84, 68, 3, 84, 79, 80, 139, 143, 26, 69, 11, 11, 32, 8, 56, 5, 83, - 84, 65, 82, 84, 186, 1, 65, 199, 142, 26, 69, 5, 129, 2, 8, 32, 65, 78, - 68, 32, 84, 79, 80, 7, 29, 5, 32, 65, 78, 68, 32, 4, 146, 212, 23, 66, - 247, 207, 2, 84, 11, 11, 32, 8, 54, 65, 20, 5, 83, 84, 65, 82, 84, 179, - 142, 26, 69, 2, 73, 2, 78, 68, 5, 53, 11, 32, 65, 78, 68, 32, 66, 79, 84, - 84, 79, 77, 2, 235, 135, 19, 32, 194, 1, 50, 48, 128, 2, 2, 76, 48, 229, - 1, 2, 85, 48, 98, 58, 49, 98, 51, 194, 14, 50, 150, 6, 52, 155, 249, 21, - 48, 24, 146, 32, 56, 190, 250, 26, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 2, 55, 3, 57, 28, 194, 216, 21, 51, 2, 52, 2, 53, 2, 55, 174, 193, - 5, 48, 2, 49, 2, 50, 2, 54, 2, 56, 3, 57, 44, 34, 48, 94, 49, 151, 181, - 20, 50, 20, 186, 215, 21, 53, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, - 54, 2, 55, 2, 56, 3, 57, 22, 222, 214, 21, 55, 174, 193, 5, 48, 2, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 52, 34, 49, 102, 50, 159, - 138, 22, 48, 26, 214, 213, 21, 48, 2, 49, 2, 56, 174, 193, 5, 50, 2, 51, - 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 8, 242, 212, 21, 50, 174, 193, 5, 48, - 3, 49, 152, 1, 50, 48, 229, 172, 18, 6, 86, 69, 82, 76, 65, 89, 150, 1, - 66, 48, 154, 1, 49, 138, 1, 50, 102, 51, 106, 53, 247, 134, 22, 52, 34, - 90, 54, 238, 210, 21, 49, 2, 53, 174, 193, 5, 50, 2, 51, 2, 52, 2, 55, 2, - 56, 3, 57, 15, 150, 148, 27, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 28, - 98, 48, 206, 209, 21, 57, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 3, 56, 9, 246, 146, 27, 65, 2, 66, 3, 67, 28, 166, 209, 21, - 48, 2, 52, 2, 53, 2, 57, 174, 193, 5, 49, 2, 50, 2, 51, 2, 54, 2, 55, 3, - 56, 32, 134, 23, 54, 190, 185, 21, 48, 2, 51, 174, 193, 5, 49, 2, 50, 2, - 52, 2, 53, 2, 55, 2, 56, 3, 57, 8, 202, 22, 48, 191, 250, 26, 49, 26, 26, - 48, 235, 148, 8, 49, 22, 158, 207, 21, 49, 2, 51, 174, 193, 5, 50, 2, 52, - 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 68, 34, 48, 98, 49, 219, 132, 22, 50, - 24, 142, 21, 51, 146, 185, 21, 50, 174, 193, 5, 49, 2, 52, 2, 53, 2, 54, - 2, 55, 2, 56, 3, 57, 24, 190, 205, 21, 48, 2, 54, 174, 193, 5, 49, 2, 50, - 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 108, 50, 48, 94, 49, 106, 50, - 98, 51, 155, 226, 18, 52, 22, 166, 204, 21, 50, 2, 54, 174, 193, 5, 49, - 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 26, 186, 18, 52, 146, 185, 21, - 55, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 56, 3, 57, 24, - 210, 17, 54, 190, 250, 26, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, - 2, 56, 3, 57, 22, 130, 202, 21, 53, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, - 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 90, 34, 48, 249, 12, 3, 65, 76, 76, - 88, 42, 48, 94, 49, 102, 51, 171, 254, 21, 50, 26, 206, 200, 21, 51, 2, - 55, 2, 56, 2, 57, 174, 193, 5, 49, 2, 50, 2, 52, 2, 53, 3, 54, 24, 242, - 199, 21, 49, 2, 54, 174, 193, 5, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, - 2, 56, 3, 57, 18, 142, 199, 21, 50, 2, 51, 174, 193, 5, 48, 2, 49, 2, 52, - 2, 53, 3, 54, 94, 50, 48, 90, 50, 102, 51, 102, 52, 139, 251, 21, 49, 22, - 254, 12, 54, 190, 250, 26, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, - 3, 57, 24, 182, 197, 21, 51, 2, 57, 174, 193, 5, 48, 2, 49, 2, 50, 2, 52, - 2, 53, 2, 54, 2, 55, 3, 56, 22, 210, 196, 21, 50, 174, 193, 5, 48, 2, 49, - 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 6, 154, 133, 27, 48, 2, - 49, 3, 50, 158, 1, 42, 48, 185, 4, 5, 69, 82, 84, 73, 67, 156, 1, 62, 48, - 98, 49, 98, 50, 210, 1, 51, 201, 191, 21, 2, 52, 48, 42, 198, 9, 55, 98, - 49, 178, 184, 21, 50, 174, 193, 5, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, - 32, 186, 8, 49, 46, 50, 190, 250, 26, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, - 55, 2, 56, 3, 57, 50, 98, 48, 182, 192, 21, 51, 2, 56, 2, 57, 174, 193, - 5, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 55, 27, 222, 129, 27, 65, 2, 66, 2, - 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 3, 76, 28, - 198, 191, 21, 48, 2, 49, 2, 51, 2, 55, 174, 193, 5, 50, 2, 52, 2, 53, 2, - 54, 2, 56, 3, 57, 2, 169, 134, 23, 2, 65, 76, 66, 34, 48, 161, 2, 3, 73, - 68, 69, 64, 26, 48, 94, 49, 103, 50, 22, 134, 190, 21, 51, 2, 57, 174, - 193, 5, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 28, 170, 189, 21, - 48, 2, 52, 2, 55, 2, 56, 174, 193, 5, 49, 2, 50, 2, 51, 2, 53, 2, 54, 3, - 57, 14, 198, 188, 21, 52, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 3, 53, 2, - 17, 2, 32, 76, 2, 211, 250, 14, 79, 24, 202, 2, 52, 146, 185, 21, 54, 2, - 56, 174, 193, 5, 49, 2, 50, 2, 51, 2, 53, 3, 55, 18, 130, 187, 21, 49, - 174, 193, 5, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 82, 22, 48, - 167, 1, 49, 34, 90, 50, 46, 51, 146, 185, 21, 52, 2, 53, 174, 193, 5, 49, - 2, 54, 2, 55, 2, 56, 3, 57, 11, 230, 250, 26, 65, 2, 66, 2, 67, 3, 68, 7, - 186, 250, 26, 65, 3, 66, 48, 66, 53, 86, 54, 138, 249, 26, 48, 2, 49, 2, - 50, 2, 51, 3, 52, 21, 218, 249, 26, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, - 70, 2, 71, 2, 72, 3, 73, 19, 134, 249, 26, 65, 2, 66, 2, 67, 2, 68, 2, - 69, 2, 70, 2, 71, 3, 72, 14, 26, 32, 159, 171, 15, 72, 12, 42, 80, 242, - 133, 3, 84, 135, 225, 21, 83, 8, 242, 132, 3, 79, 213, 129, 16, 22, 69, - 84, 65, 76, 76, 69, 68, 32, 79, 85, 84, 76, 73, 78, 69, 68, 32, 66, 76, - 65, 67, 75, 160, 1, 132, 1, 13, 66, 65, 83, 65, 78, 32, 76, 69, 84, 84, - 69, 82, 32, 166, 3, 69, 176, 4, 7, 89, 77, 65, 73, 67, 32, 76, 155, 238, - 26, 70, 80, 230, 1, 71, 78, 76, 34, 78, 50, 82, 158, 236, 24, 69, 198, - 134, 1, 67, 2, 68, 2, 75, 2, 83, 2, 84, 2, 90, 182, 105, 66, 2, 70, 2, - 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 86, 2, 88, 214, 22, 65, 2, 73, 2, 79, - 2, 85, 3, 89, 8, 38, 72, 138, 221, 26, 74, 215, 22, 69, 4, 194, 168, 24, - 65, 155, 203, 2, 69, 4, 226, 220, 26, 76, 215, 22, 69, 8, 194, 220, 26, - 68, 2, 74, 214, 22, 65, 3, 69, 4, 146, 220, 26, 82, 215, 22, 69, 32, 96, - 5, 67, 84, 82, 73, 67, 160, 1, 7, 77, 69, 78, 84, 32, 79, 70, 246, 230, - 25, 80, 223, 80, 86, 10, 26, 32, 215, 165, 23, 65, 8, 98, 80, 244, 134, - 17, 2, 84, 79, 204, 175, 8, 9, 76, 73, 71, 72, 84, 32, 66, 85, 76, 131, - 32, 65, 2, 11, 76, 2, 131, 240, 26, 85, 19, 11, 32, 16, 72, 5, 87, 73, - 84, 72, 32, 157, 142, 16, 7, 79, 80, 69, 78, 73, 78, 71, 12, 130, 1, 76, - 32, 12, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, 186, 144, 19, 86, - 186, 149, 4, 85, 166, 47, 79, 131, 153, 2, 68, 2, 141, 209, 24, 3, 79, - 78, 71, 2, 233, 158, 14, 7, 65, 76, 32, 83, 84, 82, 79, 46, 170, 154, 4, - 69, 153, 147, 15, 15, 73, 71, 65, 84, 85, 82, 69, 32, 90, 65, 89, 73, 78, - 45, 89, 53, 48, 4, 79, 74, 73, 32, 218, 2, 80, 163, 3, 32, 18, 164, 1, - 10, 67, 79, 77, 80, 79, 78, 69, 78, 84, 32, 109, 26, 77, 79, 68, 73, 70, - 73, 69, 82, 32, 70, 73, 84, 90, 80, 65, 84, 82, 73, 67, 75, 32, 84, 89, - 80, 69, 45, 8, 140, 200, 14, 2, 82, 69, 12, 5, 67, 85, 82, 76, 89, 0, 5, - 87, 72, 73, 84, 69, 173, 190, 2, 2, 66, 65, 10, 152, 135, 20, 2, 49, 45, - 214, 227, 6, 51, 2, 52, 2, 53, 3, 54, 26, 44, 3, 84, 89, 32, 225, 229, 3, - 2, 72, 65, 24, 82, 78, 60, 3, 80, 65, 71, 20, 3, 83, 69, 84, 177, 165, - 26, 4, 68, 79, 67, 85, 8, 36, 3, 79, 84, 69, 187, 161, 23, 69, 7, 131, - 232, 12, 32, 4, 139, 229, 24, 69, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, - 234, 203, 13, 82, 24, 3, 76, 69, 70, 176, 166, 1, 2, 83, 77, 207, 220, 8, - 79, 38, 86, 32, 64, 2, 68, 32, 214, 1, 81, 20, 6, 86, 69, 76, 79, 80, 69, - 235, 164, 15, 84, 6, 42, 81, 186, 129, 25, 68, 139, 167, 1, 83, 2, 191, - 160, 26, 85, 20, 32, 3, 79, 70, 32, 131, 1, 87, 18, 88, 3, 80, 82, 79, - 146, 149, 20, 71, 56, 2, 83, 69, 194, 59, 77, 30, 84, 139, 251, 4, 76, 4, - 210, 149, 20, 84, 239, 206, 5, 79, 2, 241, 169, 19, 7, 73, 84, 72, 32, - 76, 69, 70, 5, 167, 209, 20, 85, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 42, - 76, 221, 140, 23, 4, 68, 79, 87, 78, 2, 209, 248, 21, 4, 73, 71, 72, 84, - 6, 238, 227, 26, 76, 2, 77, 3, 84, 44, 28, 2, 65, 76, 231, 6, 73, 38, 30, - 32, 133, 2, 2, 83, 32, 12, 56, 3, 84, 79, 32, 237, 220, 12, 5, 65, 78, - 68, 32, 80, 10, 68, 3, 79, 82, 32, 217, 144, 26, 8, 66, 89, 32, 68, 69, - 70, 73, 78, 8, 64, 3, 80, 82, 69, 28, 3, 83, 85, 67, 170, 210, 24, 71, - 39, 76, 2, 133, 230, 14, 2, 67, 69, 2, 221, 132, 25, 3, 67, 69, 69, 26, - 72, 4, 83, 73, 71, 78, 232, 207, 24, 4, 87, 73, 84, 72, 223, 192, 1, 67, - 23, 11, 32, 20, 42, 65, 201, 1, 5, 87, 73, 84, 72, 32, 12, 112, 5, 66, - 79, 86, 69, 32, 65, 19, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 32, 80, - 65, 82, 65, 76, 76, 69, 76, 8, 158, 178, 20, 80, 154, 5, 84, 150, 184, 2, - 76, 231, 207, 2, 82, 5, 135, 205, 6, 32, 8, 160, 1, 4, 66, 85, 77, 80, - 20, 7, 73, 78, 70, 73, 78, 73, 84, 20, 18, 84, 87, 79, 32, 68, 79, 84, - 83, 32, 65, 66, 79, 86, 69, 32, 65, 78, 68, 155, 230, 14, 68, 2, 163, - 224, 25, 89, 2, 159, 182, 24, 89, 2, 17, 2, 32, 84, 2, 195, 181, 24, 87, - 6, 80, 7, 86, 65, 76, 69, 78, 84, 32, 185, 181, 20, 7, 65, 78, 71, 85, - 76, 65, 82, 4, 48, 6, 87, 73, 84, 72, 32, 70, 207, 188, 26, 84, 2, 11, - 79, 2, 177, 229, 2, 2, 85, 82, 20, 152, 1, 11, 82, 79, 82, 45, 66, 65, - 82, 82, 69, 68, 32, 216, 147, 5, 7, 73, 83, 32, 70, 79, 82, 77, 205, 162, - 7, 9, 65, 83, 69, 32, 84, 79, 32, 84, 72, 12, 240, 200, 5, 4, 87, 72, 73, - 84, 13, 5, 66, 76, 65, 67, 75, 10, 58, 67, 20, 6, 84, 73, 77, 65, 84, 69, - 239, 216, 26, 65, 5, 143, 148, 25, 65, 4, 218, 197, 25, 68, 147, 147, 1, - 83, 154, 8, 60, 7, 72, 73, 79, 80, 73, 67, 32, 134, 216, 26, 66, 3, 88, - 150, 8, 204, 1, 2, 67, 79, 232, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, - 80, 54, 83, 156, 24, 11, 84, 79, 78, 65, 76, 32, 77, 65, 82, 75, 32, 174, - 183, 18, 68, 218, 184, 5, 70, 82, 81, 137, 140, 2, 4, 87, 79, 82, 68, 10, - 26, 77, 183, 134, 26, 76, 8, 52, 7, 66, 73, 78, 73, 78, 71, 32, 167, 211, - 26, 77, 6, 60, 11, 71, 69, 77, 73, 78, 65, 84, 73, 79, 78, 32, 51, 86, 4, - 44, 5, 65, 78, 68, 32, 86, 207, 147, 26, 77, 2, 129, 213, 21, 4, 79, 87, - 69, 76, 22, 66, 84, 226, 171, 21, 72, 210, 188, 3, 69, 30, 70, 42, 78, - 39, 83, 8, 130, 134, 16, 69, 222, 227, 8, 72, 27, 87, 4, 242, 103, 65, - 201, 155, 25, 5, 82, 69, 70, 65, 67, 198, 7, 50, 69, 37, 8, 89, 76, 76, - 65, 66, 76, 69, 32, 4, 154, 157, 23, 77, 187, 204, 2, 67, 194, 7, 210, 1, - 66, 90, 67, 246, 1, 68, 186, 1, 70, 90, 71, 214, 2, 72, 162, 1, 75, 102, - 77, 90, 78, 90, 80, 138, 2, 81, 174, 1, 82, 86, 83, 210, 1, 84, 122, 74, - 2, 76, 138, 1, 87, 2, 89, 66, 88, 134, 1, 90, 95, 86, 38, 194, 12, 87, - 230, 8, 66, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, 85, - 78, 94, 67, 254, 15, 72, 146, 254, 20, 65, 2, 79, 154, 129, 5, 69, 222, - 61, 87, 186, 2, 73, 3, 85, 42, 70, 72, 198, 141, 21, 65, 154, 129, 5, 69, - 150, 64, 73, 2, 79, 3, 85, 28, 166, 19, 72, 158, 250, 20, 65, 154, 129, - 5, 69, 150, 64, 73, 2, 79, 3, 85, 60, 94, 68, 214, 14, 90, 198, 253, 20, - 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, 85, 30, 210, 14, - 72, 198, 253, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, - 85, 24, 190, 8, 87, 130, 131, 21, 65, 154, 129, 5, 69, 222, 61, 89, 186, - 2, 73, 2, 79, 3, 85, 118, 142, 1, 85, 226, 7, 71, 232, 3, 7, 76, 79, 84, - 84, 65, 76, 32, 158, 2, 87, 218, 1, 89, 158, 250, 20, 65, 2, 79, 154, - 129, 5, 69, 151, 64, 73, 39, 29, 5, 82, 65, 71, 69, 32, 36, 74, 66, 2, - 70, 2, 77, 2, 80, 46, 71, 2, 75, 2, 81, 191, 210, 11, 72, 4, 11, 87, 4, - 182, 179, 26, 69, 215, 22, 73, 6, 11, 87, 6, 202, 137, 26, 69, 151, 64, - 73, 52, 70, 72, 206, 135, 21, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, - 85, 36, 202, 4, 87, 230, 8, 89, 158, 250, 20, 65, 154, 129, 5, 69, 150, - 64, 73, 2, 79, 3, 85, 64, 250, 4, 88, 134, 6, 87, 218, 1, 89, 158, 250, - 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, 85, 26, 142, 3, 87, 130, - 131, 21, 65, 2, 79, 154, 129, 5, 69, 222, 61, 89, 186, 2, 73, 3, 85, 36, - 166, 7, 89, 146, 254, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, - 2, 73, 3, 85, 56, 82, 72, 142, 1, 87, 130, 131, 21, 65, 2, 79, 154, 129, - 5, 69, 150, 64, 73, 3, 85, 32, 74, 65, 194, 131, 21, 79, 154, 129, 5, 69, - 222, 61, 87, 186, 2, 73, 3, 85, 19, 160, 9, 8, 82, 89, 78, 71, 69, 65, - 76, 32, 203, 187, 26, 65, 8, 150, 132, 26, 69, 150, 64, 65, 3, 73, 64, - 94, 72, 134, 6, 87, 218, 1, 89, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, - 150, 64, 73, 3, 85, 24, 130, 6, 87, 246, 251, 20, 65, 154, 129, 5, 69, - 150, 64, 73, 2, 79, 3, 85, 20, 170, 129, 21, 65, 2, 79, 154, 129, 5, 69, - 222, 61, 87, 2, 89, 186, 2, 73, 3, 85, 74, 102, 69, 226, 1, 72, 170, 3, - 90, 78, 83, 158, 250, 20, 65, 2, 79, 246, 190, 5, 87, 186, 2, 73, 3, 85, - 13, 56, 8, 66, 65, 84, 66, 69, 73, 84, 32, 227, 192, 26, 69, 8, 226, 251, - 21, 66, 2, 70, 2, 77, 3, 80, 80, 118, 72, 76, 2, 84, 72, 62, 90, 162, 2, - 83, 234, 250, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, - 85, 18, 142, 254, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, - 73, 3, 85, 12, 218, 254, 25, 69, 222, 61, 65, 186, 2, 73, 2, 79, 3, 85, - 16, 134, 253, 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, 85, 40, 82, - 87, 218, 1, 89, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, - 85, 10, 242, 251, 20, 65, 154, 129, 5, 69, 151, 64, 73, 48, 90, 72, 78, - 90, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, - 85, 16, 230, 250, 20, 65, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 2, - 79, 3, 85, 14, 154, 250, 20, 65, 154, 129, 5, 69, 150, 64, 73, 2, 79, 3, - 85, 20, 130, 1, 68, 74, 72, 30, 75, 42, 82, 0, 7, 83, 72, 79, 82, 84, 32, - 82, 176, 253, 3, 3, 67, 72, 73, 189, 134, 22, 3, 89, 73, 90, 6, 48, 4, - 69, 82, 69, 84, 129, 166, 25, 2, 73, 70, 5, 17, 2, 45, 72, 2, 173, 132, - 26, 2, 73, 68, 4, 204, 165, 25, 2, 69, 78, 219, 12, 85, 2, 217, 139, 4, - 3, 73, 75, 82, 10, 68, 2, 82, 79, 205, 174, 25, 9, 76, 69, 82, 32, 67, - 79, 78, 83, 84, 8, 92, 5, 80, 69, 65, 78, 32, 172, 153, 23, 8, 45, 67, - 85, 82, 82, 69, 78, 67, 171, 151, 2, 32, 4, 186, 248, 3, 67, 19, 80, 42, - 82, 67, 102, 84, 201, 248, 25, 12, 80, 82, 69, 83, 83, 73, 79, 78, 76, - 69, 83, 83, 6, 60, 9, 76, 65, 77, 65, 84, 73, 79, 78, 32, 203, 159, 25, - 69, 4, 202, 235, 23, 81, 151, 137, 2, 77, 34, 98, 82, 217, 223, 20, 18, - 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, - 14, 140, 1, 12, 69, 77, 69, 76, 89, 32, 72, 69, 65, 86, 89, 32, 245, 226, - 24, 16, 65, 84, 69, 82, 82, 69, 83, 84, 82, 73, 65, 76, 32, 65, 76, 73, - 12, 50, 83, 214, 160, 24, 70, 234, 2, 87, 207, 10, 71, 4, 230, 161, 24, - 65, 43, 73, 7, 194, 227, 18, 71, 215, 207, 7, 83, 130, 4, 142, 1, 65, - 206, 18, 69, 194, 1, 73, 186, 7, 76, 214, 4, 79, 238, 6, 82, 142, 5, 85, - 136, 232, 19, 2, 86, 83, 254, 150, 4, 83, 207, 134, 2, 70, 90, 122, 67, - 254, 14, 76, 176, 2, 2, 88, 32, 230, 129, 3, 77, 252, 213, 5, 2, 82, 83, - 168, 250, 1, 2, 84, 72, 255, 192, 9, 73, 68, 72, 2, 69, 32, 236, 235, 15, - 4, 83, 73, 77, 73, 249, 183, 4, 2, 84, 79, 64, 226, 1, 83, 160, 1, 4, 87, - 73, 84, 72, 168, 203, 12, 2, 80, 65, 176, 130, 2, 2, 77, 65, 132, 201, - 10, 13, 84, 72, 82, 79, 87, 73, 78, 71, 32, 65, 32, 75, 73, 201, 1, 14, - 72, 79, 76, 68, 73, 78, 71, 32, 66, 65, 67, 75, 32, 84, 4, 212, 182, 16, - 18, 65, 86, 79, 85, 82, 73, 78, 71, 32, 68, 69, 76, 73, 67, 73, 79, 85, - 83, 161, 243, 4, 13, 67, 82, 69, 65, 77, 73, 78, 71, 32, 73, 78, 32, 70, - 52, 38, 32, 221, 190, 21, 3, 79, 85, 84, 50, 144, 4, 2, 67, 79, 44, 5, - 72, 69, 65, 68, 45, 38, 77, 98, 79, 92, 7, 78, 79, 32, 71, 79, 79, 68, - 238, 1, 80, 164, 1, 16, 83, 84, 85, 67, 75, 45, 79, 85, 84, 32, 84, 79, - 78, 71, 85, 69, 110, 84, 168, 140, 18, 22, 70, 73, 78, 71, 69, 82, 32, - 67, 79, 86, 69, 82, 73, 78, 71, 32, 67, 76, 79, 83, 69, 68, 172, 67, 3, - 82, 79, 76, 180, 217, 1, 13, 76, 79, 79, 75, 32, 79, 70, 32, 84, 82, 73, - 85, 77, 184, 139, 1, 8, 68, 73, 65, 71, 79, 78, 65, 76, 1, 20, 85, 78, - 69, 86, 69, 78, 32, 69, 89, 69, 83, 32, 65, 78, 68, 32, 87, 65, 86, 89, - 4, 252, 4, 3, 87, 66, 79, 163, 208, 18, 76, 2, 225, 205, 21, 4, 66, 65, - 78, 68, 4, 60, 6, 69, 68, 73, 67, 65, 76, 217, 216, 24, 3, 79, 78, 79, 2, - 165, 229, 24, 3, 32, 77, 65, 12, 90, 75, 36, 4, 80, 69, 78, 32, 165, 218, - 19, 10, 78, 69, 32, 69, 89, 69, 66, 82, 79, 87, 2, 217, 188, 21, 4, 32, - 71, 69, 83, 8, 116, 5, 77, 79, 85, 84, 72, 161, 183, 21, 18, 69, 89, 69, - 83, 32, 65, 78, 68, 32, 72, 65, 78, 68, 32, 79, 86, 69, 82, 7, 11, 32, 4, - 228, 206, 9, 3, 86, 79, 77, 241, 130, 9, 5, 65, 78, 68, 32, 67, 6, 132, - 1, 18, 65, 82, 84, 89, 32, 72, 79, 82, 78, 32, 65, 78, 68, 32, 80, 65, - 82, 84, 100, 2, 69, 69, 133, 208, 18, 4, 76, 69, 65, 68, 2, 213, 230, 21, - 2, 89, 32, 7, 29, 5, 32, 65, 78, 68, 32, 4, 36, 3, 87, 73, 78, 159, 208, - 18, 84, 2, 153, 170, 1, 4, 75, 73, 78, 71, 4, 50, 69, 149, 234, 21, 6, - 72, 69, 82, 77, 79, 77, 2, 197, 144, 26, 8, 65, 82, 83, 32, 79, 70, 32, - 74, 10, 34, 76, 237, 192, 21, 2, 65, 70, 8, 84, 13, 73, 78, 71, 32, 68, - 73, 65, 71, 79, 78, 65, 76, 32, 197, 236, 22, 2, 69, 78, 6, 128, 1, 9, - 67, 82, 79, 83, 83, 73, 78, 71, 32, 181, 227, 18, 16, 73, 78, 32, 87, 72, - 73, 84, 69, 32, 67, 73, 82, 67, 76, 69, 32, 4, 224, 197, 15, 3, 82, 73, - 83, 135, 165, 3, 78, 4, 166, 191, 14, 73, 175, 251, 3, 77, 14, 50, 65, - 54, 77, 44, 2, 82, 82, 147, 149, 18, 78, 4, 214, 196, 17, 84, 245, 156, - 8, 4, 82, 70, 85, 76, 4, 144, 188, 8, 2, 73, 78, 239, 157, 7, 65, 4, 168, - 199, 7, 2, 73, 83, 215, 214, 18, 89, 54, 200, 1, 5, 71, 85, 82, 69, 32, - 38, 76, 186, 1, 82, 182, 2, 83, 208, 12, 4, 86, 69, 32, 68, 172, 233, 4, - 10, 69, 76, 68, 32, 72, 79, 67, 75, 69, 89, 233, 157, 19, 9, 78, 73, 84, - 69, 32, 80, 65, 82, 84, 4, 222, 182, 24, 68, 139, 167, 1, 83, 10, 32, 2, - 69, 32, 65, 2, 77, 32, 6, 174, 133, 15, 70, 172, 59, 4, 67, 65, 66, 73, - 247, 193, 7, 83, 4, 52, 4, 80, 82, 79, 74, 149, 198, 19, 3, 70, 82, 65, - 2, 221, 225, 25, 2, 69, 67, 22, 34, 69, 189, 1, 3, 83, 84, 32, 13, 56, 2, - 32, 69, 68, 4, 87, 79, 82, 75, 207, 244, 14, 67, 4, 214, 254, 12, 78, - 237, 192, 4, 8, 88, 84, 73, 78, 71, 85, 73, 83, 4, 220, 209, 22, 6, 32, - 83, 80, 65, 82, 75, 171, 199, 3, 83, 10, 222, 137, 5, 81, 168, 146, 10, - 8, 83, 84, 82, 79, 78, 71, 32, 73, 219, 197, 6, 80, 10, 38, 72, 177, 224, - 22, 3, 84, 69, 68, 9, 236, 234, 3, 13, 73, 78, 71, 32, 80, 79, 76, 69, - 32, 65, 78, 68, 32, 182, 171, 5, 69, 209, 250, 15, 19, 32, 67, 65, 75, - 69, 32, 87, 73, 84, 72, 32, 83, 87, 73, 82, 76, 32, 68, 69, 36, 50, 65, - 134, 1, 69, 98, 79, 194, 1, 85, 39, 89, 10, 78, 84, 220, 148, 7, 6, 71, - 32, 73, 78, 32, 72, 229, 246, 16, 3, 77, 73, 78, 6, 214, 163, 8, 32, 202, - 170, 11, 66, 183, 176, 5, 78, 4, 132, 245, 22, 8, 88, 69, 68, 32, 66, 73, - 67, 69, 241, 130, 1, 7, 85, 82, 45, 68, 69, 45, 76, 10, 46, 82, 28, 3, - 87, 69, 82, 231, 208, 24, 80, 2, 173, 140, 25, 2, 65, 76, 7, 17, 2, 32, - 80, 4, 58, 85, 141, 187, 23, 8, 76, 65, 89, 73, 78, 71, 32, 67, 2, 193, - 169, 25, 4, 78, 67, 84, 85, 4, 214, 139, 3, 83, 155, 240, 22, 84, 9, 25, - 4, 73, 78, 71, 32, 6, 242, 213, 9, 68, 148, 179, 8, 2, 83, 65, 139, 181, - 1, 69, 54, 102, 71, 20, 2, 76, 68, 68, 2, 79, 84, 22, 82, 238, 1, 85, - 172, 173, 22, 2, 78, 68, 203, 163, 3, 88, 5, 151, 255, 25, 71, 4, 196, - 148, 9, 8, 73, 78, 71, 32, 72, 65, 78, 68, 183, 195, 16, 69, 5, 147, 224, - 13, 80, 16, 102, 75, 218, 252, 19, 77, 250, 141, 4, 32, 222, 112, 67, - 177, 120, 9, 84, 85, 78, 69, 32, 67, 79, 79, 75, 8, 80, 10, 32, 65, 78, - 68, 32, 75, 78, 73, 70, 69, 218, 225, 14, 69, 247, 219, 10, 73, 5, 133, - 146, 15, 7, 32, 87, 73, 84, 72, 32, 80, 22, 26, 82, 159, 200, 22, 78, 20, - 38, 32, 218, 2, 84, 255, 154, 18, 45, 16, 110, 67, 174, 1, 68, 234, 154, - 2, 66, 204, 240, 9, 7, 76, 69, 65, 70, 32, 67, 76, 242, 105, 84, 171, - 130, 11, 80, 4, 244, 246, 12, 3, 76, 85, 66, 229, 85, 32, 79, 82, 78, 69, - 82, 32, 65, 82, 82, 79, 87, 83, 32, 67, 73, 82, 67, 76, 73, 78, 71, 32, - 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 4, 21, 3, 79, 84, 32, 4, 186, - 222, 22, 80, 183, 235, 2, 77, 2, 255, 31, 72, 28, 78, 65, 226, 1, 69, 98, - 79, 165, 174, 16, 8, 73, 69, 68, 32, 83, 72, 82, 73, 10, 72, 6, 67, 84, - 73, 79, 78, 32, 69, 8, 77, 69, 32, 87, 73, 84, 72, 32, 4, 214, 168, 19, - 83, 177, 6, 9, 78, 85, 77, 69, 82, 65, 84, 79, 82, 6, 50, 80, 166, 155, - 2, 65, 213, 196, 20, 2, 84, 73, 2, 213, 158, 21, 2, 73, 67, 6, 48, 6, 78, - 67, 72, 32, 70, 82, 171, 185, 18, 69, 4, 150, 244, 24, 73, 249, 12, 3, - 65, 78, 67, 10, 52, 3, 78, 84, 45, 116, 2, 87, 78, 143, 201, 25, 71, 4, - 70, 84, 145, 128, 2, 11, 70, 65, 67, 73, 78, 71, 32, 66, 65, 66, 89, 2, - 253, 132, 12, 6, 73, 76, 84, 69, 68, 32, 5, 181, 176, 18, 15, 73, 78, 71, - 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 79, 224, 1, 80, 2, 76, 76, - 250, 5, 78, 216, 165, 16, 5, 69, 76, 32, 80, 85, 215, 194, 9, 83, 216, 1, - 42, 32, 73, 6, 87, 73, 68, 84, 72, 32, 10, 158, 210, 11, 77, 160, 159, 3, - 2, 79, 85, 194, 170, 8, 66, 131, 30, 83, 206, 1, 238, 1, 67, 42, 76, 78, - 78, 30, 80, 66, 82, 134, 1, 83, 38, 89, 130, 230, 9, 77, 130, 171, 10, - 65, 158, 2, 68, 58, 69, 98, 71, 118, 72, 190, 4, 81, 214, 139, 2, 87, - 182, 24, 84, 220, 113, 6, 66, 82, 79, 75, 69, 78, 202, 4, 70, 159, 177, - 1, 86, 10, 238, 148, 20, 73, 62, 79, 227, 75, 69, 116, 42, 69, 162, 152, - 20, 65, 171, 132, 4, 79, 10, 162, 1, 70, 139, 154, 20, 83, 4, 182, 234, - 20, 79, 35, 85, 6, 42, 79, 138, 155, 20, 69, 247, 203, 2, 76, 2, 239, - 154, 24, 85, 10, 36, 3, 73, 71, 72, 231, 237, 23, 69, 8, 17, 2, 84, 32, - 8, 182, 163, 22, 67, 240, 1, 5, 87, 72, 73, 84, 69, 22, 80, 155, 2, 83, - 4, 218, 201, 22, 69, 203, 165, 1, 79, 2, 247, 136, 24, 69, 4, 236, 146, - 11, 8, 67, 84, 73, 79, 78, 32, 65, 80, 253, 212, 8, 5, 69, 82, 65, 76, - 32, 228, 18, 110, 65, 62, 69, 198, 17, 73, 162, 1, 76, 134, 15, 79, 194, - 6, 82, 158, 93, 85, 226, 175, 21, 72, 215, 199, 3, 83, 4, 196, 214, 24, - 2, 82, 76, 189, 139, 1, 4, 77, 69, 32, 68, 242, 2, 112, 2, 65, 82, 110, - 77, 50, 79, 224, 220, 22, 9, 82, 77, 65, 78, 32, 80, 69, 78, 78, 130, - 153, 2, 84, 215, 105, 78, 7, 29, 5, 32, 87, 73, 84, 72, 4, 220, 230, 13, - 5, 79, 85, 84, 32, 72, 177, 236, 8, 5, 32, 72, 65, 78, 68, 4, 26, 32, - 131, 171, 21, 73, 2, 199, 172, 22, 83, 226, 2, 64, 6, 77, 69, 84, 82, 73, - 67, 73, 6, 82, 71, 73, 65, 78, 32, 6, 236, 185, 19, 4, 65, 76, 76, 89, - 137, 214, 1, 5, 32, 80, 82, 79, 80, 220, 2, 228, 1, 6, 67, 65, 80, 73, - 84, 65, 0, 4, 83, 77, 65, 76, 172, 3, 7, 76, 69, 84, 84, 69, 82, 32, 180, - 2, 24, 77, 84, 65, 86, 82, 85, 76, 73, 32, 67, 65, 80, 73, 84, 65, 76, - 32, 76, 69, 84, 84, 69, 82, 32, 165, 6, 2, 80, 65, 80, 45, 9, 76, 32, 76, - 69, 84, 84, 69, 82, 32, 80, 142, 2, 65, 34, 72, 166, 5, 67, 118, 71, 130, - 1, 74, 34, 75, 82, 80, 34, 83, 94, 90, 176, 155, 5, 2, 84, 65, 234, 250, - 7, 76, 214, 168, 2, 82, 230, 229, 8, 66, 2, 77, 2, 88, 142, 37, 78, 2, - 81, 238, 34, 86, 222, 47, 68, 14, 69, 2, 73, 2, 79, 2, 85, 2, 89, 131, - 57, 87, 4, 210, 165, 25, 69, 215, 79, 78, 10, 46, 65, 134, 222, 25, 73, - 2, 79, 215, 22, 69, 4, 214, 244, 25, 69, 3, 82, 94, 254, 1, 85, 178, 2, - 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, 75, 34, 76, 50, 80, 34, - 83, 34, 84, 62, 90, 238, 190, 15, 82, 230, 229, 8, 66, 2, 77, 2, 88, 142, - 37, 78, 2, 81, 238, 34, 86, 222, 47, 68, 14, 73, 2, 79, 2, 89, 130, 57, - 87, 255, 2, 70, 4, 252, 174, 21, 4, 45, 66, 82, 74, 191, 195, 4, 78, 92, - 250, 1, 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, 75, 34, 76, 50, - 80, 34, 83, 34, 84, 62, 90, 238, 190, 15, 82, 230, 229, 8, 66, 2, 77, 2, - 88, 142, 37, 78, 2, 81, 238, 34, 86, 222, 47, 68, 14, 73, 2, 79, 2, 85, - 2, 89, 130, 57, 87, 255, 2, 70, 6, 182, 160, 25, 69, 2, 73, 215, 79, 78, - 8, 38, 72, 162, 230, 24, 73, 203, 57, 65, 4, 230, 159, 25, 73, 135, 23, - 65, 4, 180, 227, 8, 2, 76, 73, 231, 139, 17, 78, 4, 186, 167, 24, 72, - 227, 119, 65, 12, 46, 65, 206, 215, 25, 73, 2, 79, 215, 22, 69, 6, 26, - 82, 135, 238, 25, 69, 5, 155, 231, 24, 68, 4, 186, 166, 24, 72, 155, 62, - 73, 4, 166, 203, 24, 72, 215, 82, 65, 4, 11, 65, 4, 134, 154, 15, 66, - 159, 211, 10, 83, 4, 214, 202, 24, 72, 219, 105, 65, 4, 162, 237, 24, 72, - 235, 47, 65, 6, 220, 177, 3, 6, 85, 82, 78, 69, 68, 32, 143, 234, 1, 65, - 4, 218, 201, 24, 72, 215, 82, 69, 2, 165, 219, 19, 7, 82, 65, 71, 82, 65, - 80, 72, 10, 48, 2, 77, 69, 20, 4, 78, 71, 69, 82, 31, 82, 2, 239, 215, - 24, 76, 2, 141, 252, 17, 2, 32, 82, 6, 38, 76, 237, 189, 13, 3, 65, 70, - 70, 5, 151, 215, 24, 83, 202, 1, 66, 65, 214, 13, 79, 217, 154, 25, 7, - 69, 73, 67, 72, 32, 83, 84, 194, 1, 84, 8, 71, 79, 76, 73, 84, 73, 67, - 32, 229, 12, 8, 83, 83, 32, 79, 70, 32, 77, 73, 192, 1, 56, 6, 67, 65, - 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 96, 45, 9, 76, 32, 76, 69, 84, 84, - 69, 82, 32, 96, 206, 1, 65, 22, 66, 42, 67, 94, 68, 94, 70, 38, 71, 46, - 73, 138, 2, 76, 58, 77, 66, 78, 34, 79, 30, 80, 58, 82, 30, 83, 186, 1, - 84, 110, 86, 22, 89, 90, 90, 182, 199, 18, 72, 142, 243, 1, 85, 131, 131, - 5, 75, 2, 199, 193, 25, 90, 4, 214, 3, 73, 253, 208, 25, 2, 85, 75, 4, - 48, 8, 65, 85, 68, 65, 84, 69, 32, 67, 15, 72, 2, 11, 72, 2, 197, 252, 8, - 2, 82, 73, 6, 42, 74, 30, 79, 129, 194, 25, 2, 90, 69, 2, 253, 251, 8, 2, - 69, 82, 2, 187, 248, 23, 66, 4, 238, 191, 20, 82, 131, 128, 5, 73, 2, 21, - 3, 76, 65, 71, 2, 255, 148, 21, 79, 13, 38, 78, 54, 79, 141, 1, 2, 90, - 72, 2, 181, 182, 25, 8, 73, 84, 73, 65, 76, 32, 73, 90, 4, 33, 6, 84, 65, - 84, 69, 68, 32, 4, 26, 66, 25, 2, 83, 77, 2, 11, 73, 2, 35, 71, 2, 21, 3, - 65, 76, 76, 2, 177, 223, 23, 2, 32, 89, 4, 178, 223, 25, 73, 211, 2, 69, - 4, 52, 9, 65, 84, 73, 78, 65, 84, 69, 32, 77, 35, 74, 2, 193, 223, 10, 3, - 89, 83, 76, 2, 177, 231, 8, 3, 85, 68, 73, 2, 11, 65, 2, 239, 188, 20, - 83, 4, 226, 187, 25, 78, 3, 84, 4, 26, 79, 151, 224, 25, 69, 2, 225, 245, - 21, 2, 75, 79, 2, 225, 161, 21, 2, 73, 84, 14, 106, 72, 58, 76, 168, 195, - 13, 6, 80, 73, 68, 69, 82, 89, 209, 237, 9, 8, 77, 65, 76, 76, 32, 89, - 85, 83, 6, 32, 2, 84, 65, 207, 222, 25, 65, 5, 223, 183, 24, 80, 2, 251, - 218, 8, 79, 6, 78, 86, 208, 201, 21, 9, 82, 79, 75, 85, 84, 65, 83, 84, - 73, 175, 128, 4, 83, 2, 157, 227, 18, 2, 82, 73, 2, 195, 208, 23, 69, 12, - 50, 69, 246, 184, 18, 65, 254, 163, 7, 79, 3, 85, 6, 174, 184, 20, 83, - 151, 228, 4, 82, 4, 192, 235, 8, 3, 69, 77, 76, 221, 172, 15, 4, 72, 73, - 86, 69, 2, 243, 216, 25, 76, 6, 212, 199, 11, 13, 66, 69, 32, 87, 73, 84, - 72, 32, 77, 69, 82, 73, 68, 146, 177, 4, 87, 179, 207, 8, 86, 68, 162, 1, - 65, 44, 12, 84, 72, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 178, 174, 17, - 71, 182, 55, 82, 252, 182, 5, 3, 78, 71, 71, 238, 179, 1, 79, 149, 75, 2, - 76, 70, 4, 132, 255, 14, 2, 76, 32, 231, 218, 10, 84, 54, 202, 2, 65, 50, - 72, 46, 73, 46, 78, 46, 80, 2, 81, 40, 2, 82, 65, 22, 84, 240, 227, 8, 2, - 87, 73, 142, 233, 11, 85, 160, 70, 3, 70, 65, 73, 204, 7, 4, 66, 65, 73, - 82, 248, 21, 2, 79, 84, 202, 30, 68, 218, 99, 77, 144, 117, 3, 83, 65, - 85, 138, 16, 69, 128, 26, 3, 76, 65, 71, 142, 197, 1, 74, 184, 16, 2, 71, - 73, 141, 12, 2, 75, 85, 4, 132, 212, 23, 3, 73, 72, 86, 171, 128, 2, 72, - 4, 154, 179, 13, 87, 141, 154, 11, 2, 65, 71, 4, 142, 229, 8, 85, 201, - 155, 14, 2, 71, 71, 6, 194, 149, 15, 73, 137, 195, 8, 2, 65, 85, 2, 137, - 145, 25, 5, 65, 73, 82, 84, 72, 2, 147, 176, 25, 73, 4, 136, 231, 20, 2, - 72, 73, 185, 152, 2, 2, 69, 73, 234, 9, 46, 65, 190, 6, 69, 206, 82, 73, - 139, 3, 79, 152, 1, 96, 7, 68, 85, 65, 84, 73, 79, 78, 32, 5, 78, 84, 72, - 65, 32, 226, 232, 19, 86, 215, 214, 4, 80, 2, 11, 32, 2, 211, 194, 24, - 67, 146, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 212, 2, 5, 83, 73, 71, - 78, 32, 104, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 234, 191, 7, - 65, 247, 253, 17, 79, 100, 214, 1, 86, 202, 231, 21, 65, 38, 68, 46, 84, - 230, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, - 67, 2, 71, 2, 74, 2, 75, 2, 80, 194, 40, 79, 162, 8, 69, 158, 20, 72, 2, - 77, 2, 82, 3, 89, 14, 60, 5, 69, 68, 73, 67, 32, 246, 235, 21, 79, 231, - 227, 3, 65, 4, 212, 160, 23, 6, 68, 79, 85, 66, 76, 69, 151, 234, 1, 65, - 16, 66, 67, 162, 191, 21, 80, 202, 40, 65, 170, 1, 78, 203, 162, 3, 86, - 4, 238, 200, 7, 79, 207, 159, 14, 65, 26, 218, 233, 21, 65, 106, 86, 214, - 20, 85, 210, 200, 1, 73, 218, 231, 1, 79, 163, 8, 69, 192, 8, 76, 10, 65, - 84, 69, 82, 45, 84, 72, 65, 78, 32, 206, 7, 69, 159, 190, 24, 89, 56, - 134, 1, 65, 150, 3, 66, 62, 79, 216, 2, 11, 69, 81, 85, 65, 76, 32, 84, - 79, 32, 79, 82, 218, 173, 6, 67, 138, 4, 87, 175, 141, 18, 83, 16, 44, 5, - 66, 79, 86, 69, 32, 183, 178, 6, 78, 12, 150, 1, 83, 180, 1, 19, 68, 79, - 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, 69, 81, 85, 65, 76, 32, 65, 220, - 172, 6, 4, 76, 69, 83, 83, 207, 252, 17, 82, 6, 148, 1, 7, 73, 77, 73, - 76, 65, 82, 32, 221, 175, 6, 23, 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, - 65, 76, 32, 65, 66, 79, 86, 69, 32, 76, 69, 83, 83, 4, 26, 65, 179, 175, - 6, 79, 2, 65, 3, 66, 79, 86, 6, 40, 4, 69, 83, 73, 68, 155, 176, 6, 85, - 2, 231, 2, 69, 20, 40, 2, 82, 32, 245, 1, 3, 86, 69, 82, 16, 120, 16, 83, - 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 84, 79, 226, 177, 6, - 65, 150, 213, 12, 69, 191, 177, 4, 76, 9, 49, 10, 32, 87, 73, 84, 72, 32, - 68, 79, 84, 32, 6, 44, 5, 65, 66, 79, 86, 69, 171, 220, 17, 73, 5, 251, - 147, 25, 32, 4, 52, 7, 76, 65, 80, 80, 73, 78, 71, 235, 158, 19, 32, 2, - 253, 182, 23, 2, 32, 76, 134, 8, 36, 2, 75, 32, 185, 73, 2, 78, 32, 254, - 7, 130, 3, 65, 216, 15, 8, 67, 65, 80, 73, 84, 65, 76, 32, 182, 11, 68, - 134, 1, 70, 68, 2, 73, 78, 222, 3, 75, 138, 1, 76, 174, 3, 78, 66, 77, - 84, 3, 88, 69, 83, 22, 79, 202, 1, 80, 90, 82, 182, 1, 83, 130, 22, 84, - 200, 2, 13, 85, 80, 83, 73, 76, 79, 78, 32, 87, 73, 84, 72, 32, 150, 1, - 86, 142, 2, 89, 254, 186, 7, 66, 164, 174, 3, 4, 71, 82, 65, 77, 204, 23, - 2, 90, 69, 167, 177, 11, 81, 112, 92, 10, 67, 82, 79, 80, 72, 79, 78, 73, - 67, 32, 172, 14, 6, 78, 79, 32, 84, 69, 76, 23, 82, 106, 188, 2, 6, 65, - 84, 84, 73, 67, 32, 222, 5, 67, 92, 3, 78, 65, 88, 32, 12, 68, 69, 76, - 80, 72, 73, 67, 32, 70, 73, 86, 69, 0, 14, 83, 84, 82, 65, 84, 73, 65, - 78, 32, 70, 73, 70, 84, 89, 40, 11, 69, 80, 73, 68, 65, 85, 82, 69, 65, - 78, 32, 112, 3, 72, 69, 82, 164, 1, 9, 77, 69, 83, 83, 69, 78, 73, 65, - 78, 35, 84, 48, 72, 2, 70, 73, 180, 2, 4, 79, 78, 69, 32, 205, 1, 4, 84, - 69, 78, 32, 26, 36, 3, 70, 84, 89, 105, 2, 86, 69, 11, 11, 32, 8, 22, 84, - 171, 4, 83, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 215, 3, 65, 5, 231, 3, - 32, 17, 11, 32, 14, 56, 7, 72, 85, 78, 68, 82, 69, 68, 18, 84, 143, 3, - 83, 7, 131, 2, 32, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 187, 2, 65, 5, - 213, 1, 2, 32, 84, 14, 98, 72, 48, 7, 84, 72, 79, 85, 83, 65, 78, 230, - 186, 23, 81, 229, 222, 1, 5, 68, 82, 65, 67, 72, 6, 44, 5, 85, 78, 68, - 82, 69, 203, 186, 23, 65, 4, 17, 2, 68, 32, 4, 22, 84, 131, 1, 83, 2, 95, - 65, 8, 30, 84, 86, 83, 175, 1, 77, 4, 50, 65, 21, 8, 72, 79, 85, 83, 65, - 78, 68, 32, 2, 171, 241, 15, 76, 2, 11, 83, 2, 201, 187, 23, 2, 84, 65, - 4, 88, 5, 65, 82, 89, 83, 84, 145, 1, 12, 89, 82, 69, 78, 65, 73, 67, 32, - 84, 87, 79, 32, 2, 101, 5, 73, 65, 78, 32, 70, 2, 17, 2, 32, 77, 2, 147, - 226, 12, 78, 6, 30, 70, 29, 3, 84, 87, 79, 2, 221, 248, 14, 2, 73, 86, 5, - 11, 32, 2, 161, 234, 9, 5, 68, 82, 65, 67, 72, 8, 112, 8, 77, 73, 79, 78, - 73, 65, 78, 32, 253, 145, 24, 14, 65, 69, 85, 77, 32, 79, 78, 69, 32, 80, - 76, 69, 84, 72, 6, 150, 140, 12, 70, 254, 216, 11, 84, 227, 55, 79, 2, - 11, 32, 2, 215, 228, 23, 84, 32, 92, 8, 72, 69, 83, 80, 73, 65, 78, 32, - 129, 1, 10, 82, 79, 69, 90, 69, 78, 73, 65, 78, 32, 20, 40, 2, 70, 73, - 38, 84, 239, 217, 17, 79, 6, 222, 139, 20, 86, 219, 190, 3, 70, 8, 246, - 244, 14, 72, 190, 239, 9, 69, 227, 48, 87, 12, 36, 2, 70, 73, 209, 24, 2, - 84, 69, 8, 142, 243, 17, 86, 145, 193, 6, 3, 70, 84, 89, 2, 231, 221, 9, - 69, 4, 252, 246, 21, 2, 79, 85, 149, 164, 1, 3, 84, 65, 66, 154, 2, 66, - 76, 174, 45, 82, 66, 68, 168, 188, 7, 2, 75, 65, 135, 6, 84, 144, 2, 44, - 6, 69, 84, 84, 69, 82, 32, 239, 45, 85, 142, 2, 198, 2, 65, 190, 1, 69, - 28, 4, 73, 79, 84, 65, 128, 1, 2, 79, 77, 156, 3, 3, 82, 72, 79, 46, 83, - 48, 7, 85, 80, 83, 73, 76, 79, 78, 146, 33, 80, 170, 2, 84, 138, 157, 5, - 68, 156, 164, 2, 2, 75, 65, 146, 192, 1, 71, 194, 223, 10, 67, 166, 255, - 1, 66, 2, 72, 2, 90, 166, 1, 76, 254, 210, 2, 89, 198, 43, 77, 2, 78, - 147, 17, 88, 48, 68, 4, 76, 80, 72, 65, 213, 28, 8, 82, 67, 72, 65, 73, - 67, 32, 83, 47, 33, 6, 32, 87, 73, 84, 72, 32, 44, 242, 2, 68, 30, 80, - 226, 29, 86, 226, 5, 79, 142, 226, 3, 84, 147, 140, 5, 77, 62, 186, 1, - 84, 131, 27, 80, 31, 33, 6, 32, 87, 73, 84, 72, 32, 28, 186, 5, 68, 136, - 25, 2, 80, 83, 158, 1, 86, 226, 5, 79, 142, 226, 3, 84, 147, 140, 5, 77, - 62, 28, 2, 69, 71, 235, 34, 73, 42, 11, 65, 43, 33, 6, 32, 87, 73, 84, - 72, 32, 40, 54, 68, 30, 80, 194, 35, 79, 22, 86, 251, 225, 3, 84, 16, 65, - 4, 65, 83, 73, 65, 18, 36, 4, 83, 73, 76, 73, 211, 16, 82, 17, 29, 5, 32, - 65, 78, 68, 32, 14, 44, 2, 79, 88, 0, 3, 86, 65, 82, 23, 80, 4, 81, 2, - 73, 65, 6, 60, 10, 69, 82, 73, 83, 80, 79, 77, 69, 78, 73, 175, 15, 82, - 5, 169, 15, 7, 32, 65, 78, 68, 32, 80, 82, 5, 161, 35, 7, 32, 87, 73, 84, - 72, 32, 68, 6, 170, 230, 7, 73, 206, 242, 16, 65, 227, 48, 72, 23, 33, 6, - 32, 87, 73, 84, 72, 32, 20, 66, 68, 166, 26, 86, 226, 5, 79, 142, 226, 3, - 84, 147, 140, 5, 77, 10, 130, 24, 65, 133, 203, 1, 5, 73, 65, 76, 89, 84, - 18, 76, 9, 73, 65, 76, 89, 84, 73, 75, 65, 32, 32, 3, 82, 65, 67, 227, - 22, 65, 8, 174, 24, 65, 223, 232, 3, 84, 2, 235, 146, 11, 72, 4, 40, 3, - 73, 86, 69, 1, 3, 79, 85, 82, 2, 205, 37, 2, 32, 79, 76, 144, 1, 27, 83, - 84, 82, 85, 77, 69, 78, 84, 65, 76, 32, 78, 79, 84, 65, 84, 73, 79, 78, - 32, 83, 89, 77, 66, 79, 76, 45, 165, 202, 21, 2, 68, 73, 74, 70, 49, 70, - 50, 62, 51, 62, 52, 170, 37, 53, 250, 252, 24, 55, 3, 56, 17, 218, 163, - 25, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, 150, 163, 25, 51, - 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 218, 162, 25, 48, 2, 50, 2, 54, 2, - 55, 2, 56, 3, 57, 17, 158, 162, 25, 48, 2, 50, 2, 51, 2, 53, 2, 55, 2, - 56, 3, 57, 8, 54, 65, 38, 79, 169, 32, 6, 89, 65, 84, 72, 79, 83, 4, 210, - 216, 7, 80, 195, 181, 16, 73, 2, 11, 82, 2, 11, 79, 2, 251, 131, 23, 78, - 32, 128, 1, 6, 69, 84, 84, 69, 82, 32, 168, 2, 6, 79, 87, 69, 82, 32, 78, - 32, 6, 85, 78, 65, 84, 69, 32, 157, 225, 21, 2, 73, 84, 24, 94, 83, 140, - 13, 9, 65, 82, 67, 72, 65, 73, 67, 32, 75, 2, 75, 182, 11, 68, 135, 182, - 24, 89, 16, 88, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, - 210, 12, 65, 195, 205, 7, 84, 12, 74, 80, 154, 154, 9, 71, 174, 131, 9, - 79, 238, 195, 2, 82, 243, 152, 1, 76, 4, 238, 137, 25, 83, 219, 19, 73, - 2, 249, 223, 9, 3, 85, 77, 69, 4, 222, 25, 83, 163, 186, 7, 69, 4, 80, 4, - 69, 84, 82, 69, 253, 208, 22, 10, 85, 83, 73, 67, 65, 76, 32, 76, 69, 73, - 2, 139, 206, 2, 84, 12, 88, 3, 78, 69, 32, 142, 198, 9, 88, 200, 136, 5, - 3, 85, 78, 75, 161, 181, 5, 2, 66, 79, 6, 64, 8, 72, 65, 76, 70, 32, 83, - 73, 71, 21, 4, 81, 85, 65, 82, 4, 207, 155, 24, 78, 2, 159, 132, 20, 84, - 16, 62, 82, 206, 11, 83, 114, 69, 230, 197, 7, 72, 203, 180, 16, 73, 2, - 217, 29, 2, 79, 83, 6, 100, 3, 72, 79, 32, 241, 207, 7, 16, 69, 86, 69, - 82, 83, 69, 68, 32, 76, 85, 78, 65, 84, 69, 32, 69, 4, 140, 197, 13, 10, - 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 243, 192, 10, 83, 226, 2, 220, 1, - 5, 77, 65, 76, 76, 32, 192, 19, 22, 85, 66, 83, 67, 82, 73, 80, 84, 32, - 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 72, 10, 89, 77, 66, - 79, 76, 32, 84, 65, 85, 32, 237, 156, 23, 6, 73, 78, 85, 83, 79, 73, 212, - 2, 56, 7, 76, 69, 84, 84, 69, 82, 32, 202, 17, 82, 67, 68, 206, 2, 178, - 2, 65, 162, 2, 68, 38, 69, 52, 4, 73, 79, 84, 65, 0, 7, 85, 80, 83, 73, - 76, 79, 78, 254, 3, 75, 28, 2, 79, 77, 182, 5, 80, 112, 3, 82, 72, 79, - 94, 83, 94, 84, 192, 192, 7, 2, 70, 73, 246, 192, 1, 71, 194, 223, 10, - 67, 166, 255, 1, 66, 2, 72, 2, 90, 166, 1, 76, 194, 254, 2, 77, 2, 78, - 147, 17, 88, 58, 64, 4, 76, 80, 72, 65, 149, 1, 7, 82, 67, 72, 65, 73, - 67, 32, 55, 33, 6, 32, 87, 73, 84, 72, 32, 52, 82, 86, 226, 6, 68, 30, - 80, 114, 79, 142, 1, 89, 250, 227, 3, 84, 147, 140, 5, 77, 6, 154, 5, 82, - 155, 3, 65, 4, 18, 75, 23, 83, 2, 163, 206, 7, 79, 2, 11, 65, 2, 203, - 143, 13, 77, 4, 214, 184, 7, 73, 191, 180, 14, 69, 70, 22, 80, 215, 4, - 84, 20, 249, 7, 3, 83, 73, 76, 41, 33, 6, 32, 87, 73, 84, 72, 32, 38, 78, - 68, 166, 1, 80, 178, 1, 86, 226, 5, 79, 142, 226, 3, 84, 147, 140, 5, 77, - 18, 50, 65, 29, 8, 73, 65, 76, 89, 84, 73, 75, 65, 8, 153, 1, 3, 83, 73, - 65, 11, 29, 5, 32, 65, 78, 68, 32, 8, 170, 1, 80, 154, 6, 79, 22, 86, - 251, 225, 3, 84, 10, 18, 83, 115, 69, 8, 21, 3, 73, 76, 73, 9, 17, 2, 32, - 65, 6, 21, 3, 78, 68, 32, 6, 30, 80, 154, 6, 79, 23, 86, 2, 11, 69, 2, - 11, 82, 2, 181, 17, 4, 73, 83, 80, 79, 4, 22, 82, 147, 15, 65, 2, 177, - 235, 24, 2, 65, 67, 4, 154, 201, 7, 65, 3, 79, 70, 28, 2, 69, 71, 151, 3, - 73, 50, 11, 65, 51, 33, 6, 32, 87, 73, 84, 72, 32, 48, 58, 68, 30, 80, - 114, 79, 62, 86, 82, 89, 251, 227, 3, 84, 16, 61, 4, 65, 83, 73, 65, 20, - 32, 4, 83, 73, 76, 73, 91, 69, 17, 29, 5, 32, 65, 78, 68, 32, 14, 42, 79, - 12, 2, 80, 69, 50, 86, 83, 89, 4, 83, 88, 4, 89, 9, 82, 73, 83, 80, 79, - 77, 69, 78, 73, 4, 11, 65, 4, 11, 82, 4, 17, 2, 73, 65, 5, 33, 6, 32, 65, - 78, 68, 32, 89, 2, 243, 12, 80, 20, 17, 2, 67, 82, 20, 17, 2, 79, 78, 21, - 33, 6, 32, 87, 73, 84, 72, 32, 18, 88, 5, 68, 65, 83, 73, 65, 0, 5, 80, - 83, 73, 76, 73, 54, 79, 22, 86, 251, 225, 3, 84, 7, 29, 5, 32, 65, 78, - 68, 32, 4, 18, 79, 23, 86, 2, 151, 178, 9, 88, 2, 179, 9, 65, 8, 88, 11, - 65, 77, 80, 72, 89, 76, 73, 65, 78, 32, 68, 218, 242, 24, 72, 2, 83, 219, - 19, 73, 2, 215, 173, 7, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 34, 68, - 201, 182, 20, 2, 80, 83, 2, 215, 159, 8, 65, 10, 54, 65, 134, 193, 7, 84, - 230, 1, 73, 175, 163, 17, 72, 4, 142, 131, 13, 77, 251, 129, 12, 78, 4, - 150, 224, 21, 72, 231, 255, 2, 65, 4, 41, 8, 69, 86, 69, 82, 83, 69, 68, - 32, 4, 18, 68, 43, 76, 2, 37, 7, 79, 84, 84, 69, 68, 32, 76, 2, 11, 85, - 2, 33, 6, 78, 65, 84, 69, 32, 83, 2, 245, 193, 7, 3, 73, 71, 77, 10, 214, - 255, 8, 71, 194, 223, 10, 67, 2, 80, 218, 103, 82, 207, 151, 1, 66, 2, - 167, 198, 20, 82, 16, 106, 72, 104, 7, 82, 89, 66, 76, 73, 79, 78, 44, 3, - 87, 79, 32, 150, 219, 3, 79, 141, 132, 16, 2, 65, 76, 6, 40, 4, 82, 69, - 69, 32, 219, 191, 7, 69, 4, 146, 1, 79, 185, 230, 21, 7, 81, 85, 65, 82, - 84, 69, 82, 2, 21, 3, 32, 66, 65, 2, 231, 232, 22, 83, 4, 42, 79, 253, - 234, 11, 4, 84, 72, 73, 82, 2, 229, 148, 19, 2, 66, 79, 6, 80, 5, 65, 67, - 85, 84, 69, 0, 9, 68, 73, 65, 69, 82, 69, 83, 73, 83, 39, 72, 2, 33, 6, - 32, 65, 78, 68, 32, 72, 2, 145, 157, 7, 2, 79, 79, 60, 102, 65, 21, 21, - 79, 67, 65, 76, 32, 78, 79, 84, 65, 84, 73, 79, 78, 32, 83, 89, 77, 66, - 79, 76, 45, 2, 207, 168, 9, 82, 58, 90, 50, 2, 53, 166, 195, 22, 49, 214, - 185, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 13, 246, 252, 24, 48, 2, - 49, 2, 50, 2, 51, 3, 52, 4, 26, 80, 139, 230, 19, 69, 2, 11, 79, 2, 33, - 6, 71, 69, 71, 82, 65, 77, 2, 253, 171, 20, 2, 77, 69, 8, 238, 189, 13, - 65, 142, 237, 9, 66, 210, 73, 72, 241, 64, 3, 83, 65, 76, 12, 60, 6, 78, - 78, 73, 78, 71, 32, 205, 242, 23, 3, 77, 65, 67, 10, 100, 4, 70, 65, 67, - 69, 217, 166, 17, 15, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, - 32, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 108, 23, 79, 78, 69, 32, 76, - 65, 82, 71, 69, 32, 65, 78, 68, 32, 79, 78, 69, 32, 83, 77, 65, 76, 76, - 35, 83, 2, 11, 32, 2, 159, 247, 7, 69, 4, 174, 165, 17, 77, 145, 1, 3, - 84, 65, 82, 6, 28, 3, 85, 80, 32, 39, 87, 4, 250, 222, 21, 83, 211, 215, - 2, 77, 2, 251, 211, 17, 73, 232, 3, 164, 1, 2, 65, 82, 70, 73, 52, 7, 74, - 65, 82, 65, 84, 73, 32, 212, 6, 12, 78, 74, 65, 76, 65, 32, 71, 79, 78, - 68, 73, 32, 137, 3, 7, 82, 77, 85, 75, 72, 73, 32, 4, 34, 65, 157, 172, - 20, 2, 68, 83, 2, 11, 78, 2, 239, 238, 23, 73, 4, 154, 211, 23, 84, 201, - 161, 1, 4, 68, 69, 32, 68, 182, 1, 168, 1, 7, 76, 69, 84, 84, 69, 82, 32, - 220, 1, 5, 83, 73, 71, 78, 32, 160, 2, 6, 86, 79, 87, 69, 76, 32, 226, - 175, 19, 65, 210, 24, 82, 154, 185, 3, 68, 199, 221, 1, 79, 98, 142, 139, - 21, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, 210, 200, 1, 73, 42, 76, - 234, 14, 90, 142, 175, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, - 2, 80, 254, 68, 72, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 24, 98, 67, - 28, 3, 77, 65, 68, 22, 84, 250, 194, 3, 83, 138, 199, 17, 65, 170, 1, 78, - 203, 162, 3, 86, 4, 118, 73, 143, 138, 21, 65, 2, 187, 184, 17, 68, 4, - 68, 5, 87, 79, 45, 67, 73, 29, 8, 72, 82, 69, 69, 45, 68, 79, 84, 2, 25, - 4, 82, 67, 76, 69, 2, 129, 249, 19, 5, 32, 78, 85, 75, 84, 34, 36, 5, 83, - 73, 71, 78, 32, 91, 67, 30, 86, 67, 142, 138, 21, 65, 106, 86, 214, 20, - 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 4, 201, 138, 21, 5, 65, 78, - 68, 82, 65, 126, 108, 7, 76, 69, 84, 84, 69, 82, 32, 212, 1, 5, 83, 73, - 71, 78, 32, 38, 86, 186, 253, 22, 68, 199, 221, 1, 79, 80, 250, 132, 21, - 65, 38, 68, 46, 84, 230, 24, 85, 22, 78, 190, 200, 1, 73, 42, 76, 242, - 190, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 194, 40, 79, 162, 8, 69, - 158, 20, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 89, 4, 174, 188, 22, 86, 223, - 234, 1, 65, 20, 230, 6, 79, 247, 178, 22, 73, 172, 1, 194, 1, 65, 44, 7, - 76, 69, 84, 84, 69, 82, 32, 242, 1, 83, 228, 2, 2, 86, 79, 244, 226, 12, - 3, 84, 73, 80, 146, 249, 4, 73, 172, 130, 3, 5, 69, 75, 32, 79, 78, 162, - 152, 2, 68, 235, 169, 1, 85, 4, 138, 255, 20, 66, 253, 187, 1, 2, 68, 68, - 96, 202, 156, 8, 71, 2, 75, 218, 172, 12, 82, 206, 55, 65, 38, 68, 46, - 84, 230, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 126, 66, 2, - 67, 2, 74, 2, 80, 2, 83, 194, 40, 79, 162, 8, 69, 158, 20, 70, 2, 72, 2, - 77, 2, 86, 2, 89, 3, 90, 26, 108, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, - 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 93, 4, 73, 71, 78, 32, 12, 70, - 71, 2, 75, 222, 238, 22, 83, 166, 213, 1, 76, 226, 31, 70, 3, 90, 2, 219, - 238, 22, 72, 14, 128, 1, 6, 65, 68, 65, 75, 32, 66, 2, 66, 164, 211, 14, - 2, 85, 68, 150, 172, 6, 78, 160, 128, 2, 3, 89, 65, 75, 171, 162, 1, 86, - 2, 139, 238, 7, 73, 18, 45, 9, 87, 69, 76, 32, 83, 73, 71, 78, 32, 18, - 226, 255, 20, 65, 190, 21, 85, 210, 200, 1, 73, 218, 231, 1, 79, 163, 8, - 69, 142, 23, 110, 65, 162, 95, 69, 206, 103, 73, 146, 9, 79, 142, 11, 84, - 30, 85, 130, 1, 89, 137, 186, 12, 4, 82, 89, 86, 78, 136, 11, 248, 1, 2, - 73, 82, 60, 8, 76, 70, 87, 73, 68, 84, 72, 32, 254, 10, 77, 210, 1, 78, - 236, 76, 21, 80, 80, 89, 32, 80, 69, 82, 83, 79, 78, 32, 82, 65, 73, 83, - 73, 78, 71, 32, 79, 78, 22, 84, 180, 183, 17, 2, 85, 77, 128, 141, 5, 2, - 82, 68, 227, 46, 68, 6, 26, 32, 163, 231, 23, 67, 4, 134, 178, 23, 80, - 187, 112, 83, 244, 1, 132, 2, 7, 72, 65, 78, 71, 85, 76, 32, 216, 4, 8, - 75, 65, 84, 65, 75, 65, 78, 65, 204, 3, 3, 76, 69, 70, 0, 4, 82, 73, 71, - 72, 58, 85, 228, 159, 6, 11, 70, 79, 82, 77, 83, 32, 76, 73, 71, 72, 84, - 194, 207, 6, 73, 174, 170, 4, 66, 190, 185, 4, 87, 167, 26, 68, 104, 52, - 7, 76, 69, 84, 84, 69, 82, 32, 255, 181, 10, 70, 102, 206, 1, 75, 28, 5, - 78, 73, 69, 85, 78, 42, 80, 24, 5, 82, 73, 69, 85, 76, 86, 83, 98, 89, - 222, 61, 67, 54, 69, 30, 73, 242, 4, 77, 138, 1, 84, 206, 3, 87, 198, 1, - 72, 214, 208, 23, 65, 2, 79, 151, 64, 85, 6, 242, 65, 72, 155, 3, 73, 7, - 11, 45, 4, 154, 72, 67, 131, 3, 72, 6, 166, 69, 72, 35, 73, 17, 11, 45, - 14, 226, 49, 84, 226, 14, 80, 130, 4, 77, 194, 3, 75, 218, 2, 72, 99, 83, - 12, 40, 4, 83, 65, 78, 71, 215, 178, 13, 73, 10, 230, 70, 67, 42, 75, 74, - 80, 34, 84, 211, 2, 83, 14, 154, 155, 22, 69, 238, 254, 1, 65, 150, 64, - 73, 2, 79, 3, 85, 118, 70, 32, 149, 238, 2, 11, 45, 72, 73, 82, 65, 71, - 65, 78, 65, 32, 80, 116, 76, 7, 76, 69, 84, 84, 69, 82, 32, 234, 237, 2, - 83, 34, 86, 147, 217, 20, 77, 110, 146, 1, 83, 222, 231, 2, 78, 150, 2, - 72, 2, 75, 2, 77, 2, 82, 2, 84, 170, 1, 89, 210, 41, 87, 202, 194, 21, - 65, 2, 69, 2, 73, 2, 79, 3, 85, 28, 76, 5, 77, 65, 76, 76, 32, 226, 214, - 24, 65, 2, 69, 2, 73, 2, 79, 3, 85, 18, 198, 234, 2, 89, 178, 199, 21, - 84, 234, 36, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 11, 84, 4, 236, 243, 12, - 2, 32, 67, 231, 194, 10, 87, 2, 155, 182, 23, 80, 14, 56, 3, 77, 69, 82, - 106, 83, 173, 156, 21, 3, 66, 85, 82, 9, 29, 5, 32, 65, 78, 68, 32, 6, - 172, 128, 12, 3, 87, 82, 69, 180, 213, 3, 2, 83, 73, 175, 208, 7, 80, 4, - 136, 151, 24, 3, 84, 69, 82, 151, 61, 65, 194, 8, 118, 68, 202, 2, 71, - 128, 66, 13, 73, 70, 73, 32, 82, 79, 72, 73, 78, 71, 89, 65, 32, 165, 5, - 5, 85, 78, 79, 79, 32, 10, 100, 12, 32, 87, 73, 84, 72, 32, 73, 78, 68, - 69, 88, 32, 188, 1, 2, 66, 65, 217, 138, 21, 2, 83, 72, 4, 156, 1, 18, - 65, 78, 68, 32, 77, 73, 68, 68, 76, 69, 32, 70, 73, 78, 71, 69, 82, 83, - 1, 16, 70, 73, 78, 71, 69, 82, 32, 65, 78, 68, 32, 84, 72, 85, 77, 66, 2, - 145, 206, 15, 2, 32, 67, 4, 182, 199, 23, 76, 159, 137, 1, 71, 170, 7, - 84, 3, 85, 76, 32, 217, 64, 13, 90, 72, 79, 85, 32, 78, 85, 77, 69, 82, - 65, 76, 32, 146, 7, 164, 1, 9, 67, 72, 79, 83, 69, 79, 78, 71, 32, 244, - 15, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 46, 74, 180, 31, 7, 76, 69, - 84, 84, 69, 82, 32, 215, 246, 9, 70, 250, 1, 246, 1, 67, 172, 2, 5, 73, - 69, 85, 78, 71, 146, 1, 75, 132, 1, 5, 77, 73, 69, 85, 77, 56, 5, 78, 73, - 69, 85, 78, 74, 80, 172, 2, 5, 82, 73, 69, 85, 76, 210, 1, 83, 166, 3, - 84, 124, 2, 89, 69, 200, 40, 5, 72, 73, 69, 85, 72, 139, 238, 9, 70, 30, - 76, 2, 72, 73, 84, 7, 69, 79, 78, 71, 67, 72, 73, 121, 4, 73, 69, 85, 67, - 16, 40, 4, 69, 85, 67, 72, 41, 2, 84, 85, 7, 11, 45, 4, 202, 31, 75, 243, - 26, 72, 10, 21, 3, 69, 85, 77, 10, 22, 83, 155, 46, 67, 6, 40, 4, 83, 65, - 78, 71, 179, 162, 13, 73, 4, 194, 54, 67, 227, 3, 83, 5, 215, 30, 45, 27, - 11, 45, 24, 90, 80, 234, 30, 82, 242, 13, 67, 194, 5, 77, 138, 1, 84, - 186, 2, 75, 218, 2, 72, 99, 83, 6, 214, 50, 72, 214, 3, 73, 207, 2, 65, - 16, 80, 7, 65, 80, 89, 69, 79, 85, 78, 228, 20, 5, 73, 89, 69, 79, 75, - 131, 25, 72, 10, 234, 29, 82, 178, 15, 80, 30, 83, 231, 3, 77, 11, 11, - 45, 8, 158, 52, 75, 74, 80, 34, 84, 211, 2, 83, 15, 11, 45, 12, 190, 51, - 67, 42, 75, 74, 80, 34, 84, 242, 1, 72, 99, 83, 42, 68, 6, 72, 73, 69, - 85, 80, 72, 40, 4, 73, 69, 85, 80, 223, 53, 65, 7, 11, 45, 4, 158, 51, - 80, 147, 2, 72, 35, 11, 45, 32, 82, 83, 234, 20, 80, 214, 7, 75, 162, 12, - 67, 202, 6, 84, 226, 2, 78, 179, 2, 72, 14, 32, 3, 73, 79, 83, 251, 3, - 83, 13, 11, 45, 10, 242, 46, 84, 146, 2, 67, 42, 75, 75, 80, 29, 11, 45, - 26, 78, 75, 42, 83, 190, 44, 77, 154, 3, 67, 82, 78, 34, 80, 34, 84, 243, - 1, 72, 6, 170, 22, 65, 130, 19, 72, 135, 7, 73, 8, 40, 4, 83, 65, 78, 71, - 151, 155, 13, 73, 6, 206, 47, 75, 74, 80, 35, 84, 58, 48, 3, 73, 79, 83, - 217, 1, 4, 83, 65, 78, 71, 33, 11, 45, 30, 130, 1, 80, 44, 2, 83, 83, - 210, 22, 82, 210, 1, 75, 162, 12, 67, 194, 5, 77, 138, 1, 84, 226, 2, 78, - 178, 2, 72, 223, 223, 17, 73, 6, 184, 23, 4, 73, 69, 85, 80, 179, 19, 72, - 2, 233, 48, 3, 65, 78, 71, 26, 236, 17, 5, 67, 73, 69, 85, 67, 172, 3, 4, - 83, 73, 79, 83, 154, 1, 82, 186, 20, 84, 54, 89, 134, 2, 75, 42, 78, 34, - 80, 146, 2, 72, 223, 223, 17, 73, 16, 40, 5, 73, 75, 69, 85, 84, 195, 41, - 72, 15, 11, 45, 12, 226, 20, 82, 178, 19, 77, 154, 3, 67, 42, 75, 74, 80, - 243, 2, 83, 4, 150, 19, 83, 139, 22, 79, 2, 165, 211, 8, 6, 76, 69, 32, - 68, 79, 84, 216, 3, 92, 9, 79, 78, 71, 83, 69, 79, 78, 71, 32, 165, 21, - 9, 85, 78, 71, 83, 69, 79, 78, 71, 32, 154, 2, 226, 1, 67, 80, 5, 72, 73, - 69, 85, 72, 60, 5, 73, 69, 85, 78, 71, 46, 75, 220, 1, 5, 77, 73, 69, 85, - 77, 188, 1, 5, 78, 73, 69, 85, 78, 94, 80, 240, 2, 5, 82, 73, 69, 85, 76, - 190, 4, 83, 194, 3, 84, 213, 1, 2, 89, 69, 8, 36, 4, 73, 69, 85, 67, 239, - 30, 72, 7, 11, 45, 4, 162, 32, 83, 239, 7, 80, 11, 11, 45, 8, 174, 16, - 82, 178, 19, 77, 234, 3, 78, 35, 80, 9, 11, 45, 6, 194, 17, 75, 57, 2, - 83, 83, 28, 76, 7, 65, 80, 89, 69, 79, 85, 78, 40, 5, 73, 89, 69, 79, 75, - 215, 30, 72, 8, 130, 15, 82, 178, 15, 80, 131, 4, 77, 19, 11, 45, 16, - 166, 13, 75, 170, 1, 82, 42, 83, 224, 13, 2, 67, 72, 146, 9, 78, 34, 80, - 147, 2, 72, 27, 11, 45, 24, 74, 80, 30, 83, 134, 13, 82, 242, 13, 67, - 130, 9, 75, 42, 78, 179, 2, 72, 6, 174, 33, 73, 131, 6, 65, 6, 40, 4, 83, - 65, 78, 71, 143, 143, 13, 73, 4, 238, 35, 78, 147, 3, 83, 21, 11, 45, 18, - 174, 12, 82, 242, 13, 67, 202, 6, 84, 186, 2, 75, 218, 2, 72, 62, 80, 39, - 83, 36, 88, 6, 65, 78, 83, 73, 79, 83, 48, 6, 72, 73, 69, 85, 80, 72, 53, - 4, 73, 69, 85, 80, 7, 11, 45, 4, 236, 7, 2, 75, 65, 195, 26, 80, 9, 11, - 45, 6, 150, 11, 84, 234, 22, 80, 243, 2, 83, 23, 11, 45, 20, 112, 5, 82, - 73, 69, 85, 76, 24, 4, 83, 73, 79, 83, 134, 3, 80, 246, 19, 67, 194, 5, - 77, 170, 4, 84, 243, 1, 72, 5, 153, 3, 2, 45, 80, 5, 221, 32, 2, 45, 84, - 57, 11, 45, 54, 102, 75, 92, 5, 77, 73, 69, 85, 77, 50, 80, 126, 83, 74, - 84, 44, 2, 89, 69, 154, 28, 78, 179, 2, 72, 10, 52, 5, 73, 89, 69, 79, - 75, 190, 4, 65, 131, 19, 72, 7, 11, 45, 4, 254, 32, 72, 99, 83, 9, 11, - 45, 6, 130, 30, 75, 218, 2, 72, 99, 83, 14, 48, 4, 73, 69, 85, 80, 174, - 26, 72, 163, 6, 65, 11, 11, 45, 8, 42, 80, 222, 29, 84, 242, 1, 72, 99, - 83, 2, 243, 25, 72, 6, 40, 4, 83, 65, 78, 71, 255, 135, 13, 73, 4, 182, - 28, 75, 187, 3, 83, 6, 100, 5, 73, 75, 69, 85, 84, 151, 25, 72, 6, 56, 9, - 79, 82, 73, 78, 72, 73, 69, 85, 72, 191, 3, 83, 5, 255, 29, 45, 52, 48, - 3, 73, 79, 83, 161, 1, 4, 83, 65, 78, 71, 25, 11, 45, 22, 82, 75, 162, 3, - 82, 242, 13, 67, 198, 2, 80, 254, 2, 77, 138, 1, 84, 147, 5, 72, 4, 22, - 65, 135, 26, 73, 2, 237, 18, 6, 80, 89, 69, 79, 85, 78, 28, 160, 1, 5, - 82, 73, 69, 85, 76, 36, 6, 84, 73, 75, 69, 85, 84, 16, 3, 89, 69, 83, - 138, 19, 83, 178, 1, 77, 154, 3, 67, 42, 75, 42, 78, 34, 80, 239, 225, - 17, 73, 5, 17, 2, 45, 75, 2, 159, 17, 72, 5, 255, 16, 45, 2, 171, 250, - 17, 73, 20, 40, 5, 73, 75, 69, 85, 84, 155, 21, 72, 19, 11, 45, 16, 58, - 82, 42, 83, 42, 84, 162, 13, 67, 130, 9, 75, 75, 80, 2, 17, 2, 73, 69, 2, - 255, 160, 23, 85, 4, 21, 3, 73, 79, 83, 5, 223, 1, 45, 2, 255, 19, 72, - 18, 44, 6, 83, 73, 69, 85, 78, 71, 243, 19, 79, 17, 11, 45, 14, 50, 75, - 30, 83, 198, 17, 77, 154, 6, 72, 63, 80, 4, 166, 14, 72, 135, 7, 73, 4, - 26, 83, 175, 128, 13, 73, 2, 21, 3, 65, 78, 71, 2, 207, 20, 75, 190, 1, - 122, 65, 118, 69, 134, 1, 73, 92, 7, 83, 83, 65, 78, 71, 65, 82, 106, 79, - 138, 1, 85, 102, 89, 174, 15, 87, 175, 234, 9, 70, 23, 48, 4, 82, 65, 69, - 65, 98, 45, 239, 165, 24, 69, 13, 11, 45, 10, 190, 231, 21, 69, 130, 191, - 2, 65, 2, 73, 3, 85, 25, 18, 79, 55, 85, 9, 11, 45, 6, 130, 129, 24, 69, - 234, 36, 79, 3, 85, 15, 11, 45, 12, 170, 9, 69, 142, 156, 24, 65, 2, 79, - 3, 85, 31, 11, 45, 28, 66, 65, 34, 89, 166, 1, 79, 142, 254, 23, 69, 234, - 36, 73, 3, 85, 5, 11, 82, 2, 235, 212, 17, 65, 14, 50, 65, 230, 228, 21, - 69, 130, 191, 2, 79, 3, 85, 7, 238, 132, 24, 45, 247, 30, 69, 23, 26, 45, - 171, 163, 24, 69, 18, 50, 79, 22, 89, 226, 227, 21, 69, 131, 191, 2, 85, - 5, 155, 143, 24, 45, 8, 222, 227, 21, 69, 239, 254, 1, 65, 17, 11, 45, - 14, 158, 19, 89, 234, 136, 5, 73, 192, 170, 4, 3, 69, 79, 45, 214, 155, - 14, 65, 151, 64, 85, 62, 42, 65, 70, 69, 66, 73, 22, 79, 107, 85, 11, 26, - 45, 147, 161, 24, 69, 6, 154, 130, 24, 89, 246, 30, 79, 3, 85, 11, 11, - 79, 9, 11, 45, 6, 150, 158, 24, 89, 186, 2, 79, 3, 85, 5, 191, 251, 23, - 45, 19, 11, 45, 16, 58, 89, 186, 223, 23, 65, 162, 33, 69, 246, 30, 73, - 3, 79, 6, 182, 223, 23, 65, 163, 33, 69, 21, 11, 45, 18, 142, 16, 89, - 146, 208, 21, 69, 238, 254, 1, 65, 150, 64, 73, 2, 79, 3, 85, 186, 1, - 226, 1, 65, 46, 67, 54, 69, 30, 73, 22, 75, 188, 1, 5, 77, 73, 69, 85, - 77, 64, 5, 78, 73, 69, 85, 78, 70, 80, 168, 1, 5, 82, 73, 69, 85, 76, - 254, 1, 84, 90, 83, 246, 2, 87, 50, 89, 150, 1, 72, 214, 208, 23, 79, - 151, 64, 85, 9, 156, 13, 3, 82, 65, 69, 207, 143, 24, 69, 4, 22, 72, 207, - 8, 73, 2, 241, 249, 23, 2, 73, 69, 7, 138, 156, 24, 79, 3, 85, 5, 239, - 234, 17, 69, 14, 56, 7, 65, 80, 89, 69, 79, 85, 78, 106, 72, 155, 3, 73, - 8, 30, 80, 30, 83, 231, 3, 77, 4, 190, 4, 72, 215, 3, 73, 2, 25, 4, 83, - 65, 78, 71, 2, 207, 7, 80, 2, 253, 59, 2, 73, 69, 9, 11, 45, 6, 22, 80, - 247, 9, 83, 4, 142, 7, 73, 207, 2, 65, 13, 11, 45, 10, 234, 5, 67, 146, - 1, 84, 242, 1, 72, 62, 80, 39, 83, 20, 48, 4, 73, 69, 85, 80, 170, 2, 72, - 163, 6, 65, 17, 11, 45, 14, 42, 83, 186, 2, 84, 146, 2, 67, 43, 75, 6, - 21, 3, 73, 79, 83, 7, 11, 45, 4, 202, 4, 75, 107, 84, 27, 11, 45, 24, 68, - 2, 75, 73, 34, 77, 34, 80, 106, 84, 54, 89, 222, 4, 72, 99, 83, 4, 149, - 1, 4, 89, 69, 79, 75, 2, 11, 73, 2, 255, 238, 22, 69, 8, 30, 72, 34, 73, - 131, 6, 65, 2, 245, 156, 18, 3, 73, 69, 85, 4, 21, 3, 69, 85, 80, 5, 243, - 5, 45, 4, 22, 72, 151, 3, 73, 2, 229, 167, 19, 2, 73, 69, 2, 17, 2, 69, - 79, 2, 167, 4, 82, 28, 44, 3, 73, 79, 83, 57, 4, 83, 65, 78, 71, 13, 11, - 45, 10, 122, 67, 42, 75, 42, 78, 34, 80, 35, 84, 16, 78, 67, 42, 75, 42, - 78, 34, 80, 34, 84, 242, 1, 72, 98, 83, 255, 222, 17, 73, 2, 11, 73, 2, - 249, 236, 22, 2, 69, 85, 2, 11, 73, 2, 209, 233, 23, 2, 89, 69, 2, 11, - 73, 2, 243, 146, 23, 69, 2, 11, 73, 2, 135, 157, 23, 69, 2, 11, 73, 2, - 11, 75, 2, 255, 152, 23, 69, 10, 170, 211, 21, 69, 238, 254, 1, 65, 151, - 64, 73, 34, 58, 69, 206, 1, 79, 62, 85, 166, 207, 23, 65, 151, 64, 73, - 13, 42, 79, 73, 6, 83, 73, 69, 85, 78, 71, 5, 11, 82, 2, 17, 2, 73, 78, - 2, 11, 72, 2, 157, 200, 9, 2, 73, 69, 7, 11, 45, 4, 18, 80, 39, 83, 2, - 11, 65, 2, 11, 78, 2, 11, 83, 2, 139, 232, 12, 73, 9, 11, 45, 6, 26, 89, - 207, 143, 24, 73, 4, 183, 207, 23, 65, 9, 11, 45, 6, 26, 89, 147, 143, - 24, 73, 4, 143, 208, 21, 69, 24, 178, 223, 10, 84, 170, 194, 11, 70, 30, - 83, 230, 83, 78, 14, 79, 199, 110, 69, 100, 156, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 196, 2, 5, 77, 65, 82, 75, 32, 72, 5, 83, 73, 71, 78, 32, - 196, 156, 2, 6, 86, 79, 87, 69, 76, 32, 167, 255, 19, 68, 58, 202, 1, 68, - 34, 75, 222, 249, 18, 84, 174, 113, 82, 246, 168, 2, 78, 246, 175, 1, 83, - 254, 68, 66, 2, 67, 2, 70, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, 2, 80, 2, - 86, 2, 87, 2, 89, 2, 90, 187, 2, 65, 4, 138, 137, 24, 68, 187, 2, 65, 8, - 56, 5, 73, 78, 78, 65, 32, 178, 136, 24, 72, 187, 2, 65, 4, 174, 136, 24, - 87, 3, 89, 4, 208, 232, 20, 6, 78, 65, 32, 75, 72, 79, 189, 162, 2, 3, - 83, 65, 75, 8, 52, 2, 84, 65, 209, 255, 21, 5, 72, 65, 82, 66, 65, 6, 42, - 72, 142, 203, 19, 83, 223, 187, 4, 78, 2, 135, 231, 23, 65, 42, 62, 76, - 172, 167, 18, 6, 83, 73, 71, 78, 32, 80, 139, 2, 86, 36, 33, 6, 69, 84, - 84, 69, 82, 32, 36, 158, 185, 20, 78, 210, 204, 3, 66, 2, 68, 2, 71, 2, - 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, - 2, 65, 2, 73, 3, 85, 2, 135, 249, 22, 69, 54, 52, 5, 67, 72, 73, 78, 71, - 41, 4, 82, 65, 78, 32, 2, 17, 2, 32, 67, 2, 207, 215, 22, 72, 52, 52, 7, - 76, 69, 84, 84, 69, 82, 32, 211, 145, 6, 78, 42, 218, 1, 65, 226, 132, - 11, 90, 210, 46, 84, 150, 119, 76, 50, 81, 60, 6, 68, 65, 76, 69, 84, 72, - 198, 150, 4, 71, 122, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, 130, 76, - 66, 190, 173, 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, 4, 222, 193, - 16, 76, 207, 242, 6, 89, 170, 9, 230, 1, 65, 184, 32, 5, 66, 82, 69, 87, - 32, 230, 32, 76, 200, 1, 16, 78, 84, 65, 73, 71, 65, 78, 65, 32, 76, 69, - 84, 84, 69, 82, 32, 174, 12, 82, 120, 11, 88, 65, 71, 82, 65, 77, 32, 70, - 79, 82, 32, 137, 177, 23, 4, 68, 71, 69, 72, 212, 1, 42, 68, 98, 82, 201, - 1, 3, 86, 89, 32, 6, 44, 5, 83, 84, 79, 78, 69, 223, 215, 22, 80, 5, 205, - 219, 2, 7, 32, 71, 82, 65, 86, 69, 89, 12, 32, 2, 84, 32, 143, 222, 16, - 45, 10, 60, 5, 87, 73, 84, 72, 32, 210, 250, 11, 68, 139, 220, 8, 72, 6, - 76, 9, 84, 73, 80, 32, 79, 78, 32, 84, 72, 130, 200, 12, 82, 139, 157, - 10, 65, 2, 243, 204, 23, 69, 194, 1, 134, 2, 65, 202, 1, 66, 230, 2, 67, - 154, 3, 68, 162, 1, 69, 186, 3, 70, 94, 72, 62, 76, 222, 1, 77, 110, 79, - 162, 1, 82, 142, 2, 83, 228, 1, 3, 78, 79, 82, 198, 1, 84, 128, 2, 2, 85, - 80, 174, 1, 87, 146, 215, 13, 73, 146, 223, 3, 80, 242, 220, 3, 86, 215, - 74, 71, 12, 108, 17, 82, 82, 79, 87, 32, 83, 72, 65, 70, 84, 32, 87, 73, - 68, 84, 72, 32, 154, 217, 17, 77, 215, 146, 4, 83, 8, 36, 3, 79, 78, 69, - 235, 158, 22, 84, 7, 11, 32, 4, 182, 185, 13, 84, 151, 194, 8, 72, 14, - 48, 4, 65, 76, 76, 79, 21, 4, 76, 65, 67, 75, 2, 143, 140, 5, 84, 12, 30, - 32, 153, 1, 2, 45, 70, 6, 52, 7, 67, 85, 82, 86, 69, 68, 32, 151, 243, - 22, 72, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 245, 217, 22, 8, 87, - 65, 82, 68, 83, 32, 65, 78, 6, 45, 9, 69, 65, 84, 72, 69, 82, 69, 68, 32, - 6, 194, 159, 13, 83, 210, 164, 3, 78, 163, 149, 6, 82, 14, 116, 2, 72, - 69, 52, 5, 73, 82, 67, 76, 69, 245, 159, 17, 14, 79, 78, 67, 65, 86, 69, - 45, 80, 79, 73, 78, 84, 69, 68, 4, 156, 134, 20, 4, 86, 82, 79, 78, 247, - 176, 2, 67, 9, 64, 6, 32, 87, 73, 84, 72, 32, 165, 229, 21, 4, 68, 32, - 83, 65, 4, 52, 7, 83, 84, 82, 79, 75, 69, 32, 159, 227, 4, 67, 2, 29, 5, - 65, 78, 68, 32, 84, 2, 11, 87, 2, 11, 79, 2, 21, 3, 32, 68, 79, 2, 11, - 84, 2, 231, 248, 22, 83, 14, 52, 7, 65, 83, 72, 69, 68, 32, 84, 18, 73, - 31, 79, 2, 175, 16, 82, 2, 157, 201, 18, 2, 86, 73, 10, 192, 16, 3, 87, - 78, 87, 206, 214, 13, 85, 135, 162, 4, 76, 16, 120, 5, 73, 71, 72, 84, - 32, 156, 2, 16, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, - 75, 32, 231, 133, 18, 81, 10, 40, 2, 80, 79, 126, 84, 135, 225, 21, 83, - 6, 33, 6, 73, 78, 84, 69, 68, 32, 6, 174, 130, 16, 80, 128, 221, 5, 11, - 82, 69, 67, 84, 73, 76, 73, 78, 69, 65, 82, 23, 66, 2, 173, 225, 21, 24, - 69, 65, 82, 68, 82, 79, 80, 45, 83, 80, 79, 75, 69, 68, 32, 80, 82, 79, - 80, 69, 76, 76, 69, 82, 4, 150, 222, 22, 83, 187, 79, 79, 6, 44, 5, 79, - 85, 82, 32, 66, 227, 226, 4, 73, 2, 225, 218, 10, 6, 65, 76, 76, 79, 79, - 78, 4, 210, 254, 16, 79, 157, 170, 5, 6, 69, 65, 82, 84, 32, 69, 20, 74, - 65, 44, 3, 69, 70, 84, 28, 2, 79, 87, 237, 221, 4, 3, 73, 71, 65, 4, 212, - 236, 20, 2, 82, 71, 167, 196, 1, 84, 8, 254, 3, 45, 195, 6, 87, 6, 26, - 32, 179, 236, 9, 69, 4, 162, 225, 13, 68, 25, 4, 83, 73, 78, 71, 4, 56, - 8, 85, 76, 84, 73, 80, 76, 73, 67, 227, 211, 20, 73, 2, 25, 4, 65, 84, - 73, 79, 2, 163, 254, 4, 78, 6, 172, 234, 5, 9, 80, 69, 78, 32, 67, 69, - 78, 84, 82, 204, 152, 10, 13, 86, 65, 76, 32, 87, 73, 84, 72, 32, 79, 86, - 65, 76, 213, 214, 5, 5, 85, 84, 76, 73, 78, 12, 76, 4, 73, 71, 72, 84, - 149, 203, 22, 9, 79, 85, 78, 68, 45, 84, 73, 80, 80, 10, 62, 45, 109, 11, - 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 4, 69, 15, 80, 79, 73, 78, - 84, 73, 78, 71, 32, 65, 78, 71, 76, 69, 32, 4, 130, 196, 6, 66, 247, 153, - 7, 81, 7, 139, 6, 32, 26, 106, 65, 78, 73, 44, 2, 79, 85, 168, 229, 13, - 7, 67, 82, 73, 80, 84, 32, 76, 225, 131, 1, 3, 80, 65, 82, 4, 244, 220, - 13, 10, 78, 83, 45, 83, 69, 82, 73, 70, 32, 73, 195, 250, 7, 76, 8, 224, - 218, 13, 2, 78, 71, 203, 249, 7, 88, 10, 21, 3, 84, 72, 32, 10, 60, 5, - 69, 65, 83, 84, 32, 29, 6, 87, 69, 83, 84, 32, 80, 6, 26, 80, 235, 204, - 22, 65, 4, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 4, 178, 181, 16, 86, - 239, 134, 6, 66, 12, 88, 9, 69, 65, 82, 68, 82, 79, 80, 45, 83, 130, 1, - 82, 169, 192, 6, 4, 87, 69, 76, 86, 6, 64, 6, 80, 79, 75, 69, 68, 32, - 145, 197, 22, 4, 72, 65, 78, 75, 4, 196, 212, 21, 8, 80, 73, 78, 87, 72, - 69, 69, 76, 27, 65, 2, 137, 3, 5, 73, 65, 78, 71, 76, 6, 26, 87, 159, - 226, 9, 80, 4, 53, 11, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 4, 29, - 5, 87, 73, 84, 72, 32, 4, 204, 181, 19, 5, 76, 65, 82, 71, 69, 187, 200, - 1, 69, 10, 72, 5, 72, 73, 84, 69, 32, 112, 2, 73, 68, 177, 254, 21, 3, - 69, 68, 71, 6, 56, 5, 68, 79, 87, 78, 45, 130, 147, 22, 67, 199, 46, 83, - 2, 165, 190, 22, 8, 80, 79, 73, 78, 84, 73, 78, 71, 2, 145, 153, 20, 2, - 69, 45, 140, 2, 112, 7, 65, 67, 67, 69, 78, 84, 32, 146, 8, 76, 164, 14, - 5, 77, 65, 82, 75, 32, 114, 80, 245, 188, 20, 2, 89, 79, 60, 132, 2, 9, - 65, 84, 78, 65, 72, 32, 72, 65, 70, 22, 68, 36, 3, 71, 69, 82, 74, 77, - 128, 1, 2, 80, 65, 28, 4, 69, 84, 78, 65, 20, 2, 81, 65, 58, 83, 58, 84, - 156, 1, 2, 89, 69, 78, 90, 152, 132, 8, 3, 82, 69, 86, 246, 179, 14, 79, - 221, 143, 1, 3, 73, 76, 85, 2, 255, 214, 17, 85, 4, 178, 186, 18, 69, - 175, 225, 4, 65, 6, 32, 3, 69, 83, 72, 195, 28, 83, 5, 237, 187, 6, 4, - 32, 77, 85, 81, 8, 88, 5, 69, 82, 75, 72, 65, 148, 164, 16, 2, 85, 78, - 145, 177, 1, 5, 65, 72, 65, 80, 65, 5, 149, 150, 19, 4, 32, 75, 69, 70, - 4, 26, 83, 191, 158, 23, 90, 2, 207, 183, 23, 72, 4, 196, 151, 23, 6, 82, - 78, 69, 89, 32, 80, 179, 35, 68, 4, 194, 24, 69, 241, 230, 21, 6, 72, 65, - 76, 83, 72, 69, 8, 38, 69, 181, 227, 21, 3, 73, 80, 69, 6, 48, 6, 76, 73, - 83, 72, 65, 32, 163, 183, 11, 86, 4, 136, 177, 21, 3, 81, 69, 84, 153, - 135, 2, 4, 71, 69, 68, 79, 4, 176, 171, 8, 10, 82, 65, 72, 32, 66, 69, - 78, 32, 89, 79, 167, 214, 2, 84, 8, 34, 65, 177, 160, 23, 2, 73, 78, 6, - 40, 4, 81, 69, 70, 32, 139, 222, 5, 82, 4, 194, 222, 10, 81, 197, 231, - 11, 3, 71, 65, 68, 150, 1, 76, 6, 69, 84, 84, 69, 82, 32, 201, 11, 8, 73, - 71, 65, 84, 85, 82, 69, 32, 140, 1, 134, 3, 65, 204, 1, 3, 66, 69, 84, 0, - 3, 75, 65, 70, 0, 2, 80, 69, 68, 6, 70, 73, 78, 65, 76, 32, 92, 2, 81, - 79, 16, 2, 72, 69, 52, 2, 78, 85, 0, 4, 90, 65, 89, 73, 18, 83, 48, 3, - 82, 69, 83, 170, 1, 84, 52, 4, 68, 65, 76, 69, 12, 5, 71, 73, 77, 69, 76, - 0, 5, 76, 65, 77, 69, 68, 0, 3, 77, 69, 77, 48, 3, 86, 65, 86, 80, 5, 87, - 73, 68, 69, 32, 153, 1, 3, 89, 79, 68, 14, 26, 76, 251, 212, 22, 89, 12, - 64, 2, 69, 70, 73, 10, 84, 69, 82, 78, 65, 84, 73, 86, 69, 32, 9, 33, 6, - 32, 87, 73, 84, 72, 32, 6, 250, 12, 77, 130, 1, 80, 35, 81, 4, 254, 145, - 16, 65, 187, 148, 1, 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 154, 15, - 82, 243, 172, 13, 68, 14, 88, 2, 75, 65, 236, 2, 2, 80, 69, 136, 152, 1, - 2, 84, 83, 218, 182, 21, 78, 239, 109, 77, 4, 235, 2, 70, 7, 236, 10, 5, - 32, 87, 73, 84, 72, 231, 198, 23, 84, 4, 167, 2, 78, 16, 44, 4, 65, 77, - 69, 75, 17, 3, 72, 73, 78, 4, 231, 1, 72, 13, 33, 6, 32, 87, 73, 84, 72, - 32, 10, 40, 6, 68, 65, 71, 69, 83, 72, 39, 83, 7, 33, 6, 32, 65, 78, 68, - 32, 83, 4, 210, 221, 5, 72, 167, 192, 2, 73, 12, 50, 69, 12, 2, 65, 86, - 1, 4, 83, 65, 68, 73, 4, 11, 84, 5, 161, 184, 13, 7, 32, 87, 73, 84, 72, - 32, 68, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 176, 251, 1, 2, 72, 79, 159, - 188, 11, 68, 16, 174, 2, 76, 250, 169, 8, 65, 246, 188, 2, 84, 158, 206, - 2, 82, 164, 186, 8, 2, 68, 65, 134, 91, 75, 198, 106, 72, 169, 4, 7, 70, - 73, 78, 65, 76, 32, 77, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 168, 7, 2, - 72, 73, 191, 174, 13, 68, 10, 72, 6, 65, 76, 69, 70, 32, 76, 29, 8, 89, - 73, 68, 68, 73, 83, 72, 32, 2, 133, 236, 18, 2, 65, 77, 8, 120, 7, 68, - 79, 85, 66, 76, 69, 32, 224, 4, 9, 89, 79, 68, 32, 89, 79, 68, 32, 80, - 237, 163, 20, 5, 86, 65, 86, 32, 89, 4, 210, 229, 10, 86, 251, 194, 9, - 89, 6, 80, 3, 76, 79, 87, 0, 3, 85, 80, 80, 153, 250, 21, 6, 77, 65, 83, - 79, 82, 65, 2, 185, 184, 22, 2, 69, 82, 50, 84, 5, 79, 73, 78, 84, 32, - 241, 5, 11, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 38, 220, 1, 9, - 68, 65, 71, 69, 83, 72, 32, 79, 82, 46, 72, 106, 80, 164, 1, 17, 74, 85, - 68, 69, 79, 45, 83, 80, 65, 78, 73, 83, 72, 32, 86, 65, 82, 22, 81, 86, - 82, 22, 83, 168, 174, 9, 2, 84, 83, 173, 207, 13, 3, 77, 69, 84, 2, 17, - 2, 32, 77, 2, 201, 1, 2, 65, 80, 12, 60, 5, 65, 84, 65, 70, 32, 106, 73, - 33, 4, 79, 76, 65, 77, 6, 38, 80, 34, 81, 165, 2, 2, 83, 69, 2, 11, 65, - 2, 251, 140, 16, 84, 2, 153, 177, 22, 3, 65, 77, 65, 2, 11, 82, 2, 207, - 177, 13, 73, 5, 145, 224, 10, 12, 32, 72, 65, 83, 69, 82, 32, 70, 79, 82, - 32, 86, 2, 195, 187, 23, 73, 6, 52, 5, 65, 77, 65, 84, 83, 133, 150, 11, - 2, 85, 66, 5, 193, 201, 10, 2, 32, 81, 2, 243, 191, 6, 65, 8, 34, 69, 22, - 72, 151, 145, 8, 73, 2, 179, 176, 22, 71, 4, 146, 145, 8, 73, 171, 187, - 7, 69, 12, 152, 1, 3, 71, 69, 82, 60, 3, 80, 65, 83, 20, 7, 83, 79, 70, - 32, 80, 65, 83, 224, 142, 21, 7, 78, 85, 78, 32, 72, 65, 70, 241, 176, 1, - 3, 77, 65, 81, 4, 26, 83, 131, 220, 21, 69, 2, 225, 190, 22, 3, 72, 65, - 89, 2, 199, 173, 13, 69, 2, 179, 173, 13, 85, 8, 114, 77, 248, 235, 11, - 15, 76, 83, 67, 72, 82, 69, 73, 66, 69, 82, 32, 80, 65, 85, 83, 221, 193, - 5, 3, 73, 67, 79, 4, 252, 188, 5, 12, 69, 84, 32, 87, 73, 84, 72, 32, 87, - 72, 73, 84, 223, 239, 16, 32, 188, 4, 164, 1, 2, 65, 45, 50, 72, 70, 75, - 230, 1, 77, 114, 78, 146, 2, 82, 66, 83, 154, 1, 84, 226, 1, 87, 62, 89, - 110, 69, 186, 206, 8, 85, 250, 148, 5, 79, 167, 178, 3, 73, 8, 234, 158, - 23, 87, 246, 30, 49, 2, 50, 3, 51, 72, 166, 6, 79, 182, 157, 8, 65, 222, - 201, 5, 85, 162, 115, 69, 3, 73, 74, 72, 2, 65, 45, 104, 2, 79, 45, 178, - 4, 73, 226, 3, 69, 211, 214, 14, 85, 24, 210, 193, 11, 49, 250, 227, 11, - 75, 214, 22, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 8, 222, - 167, 23, 75, 218, 19, 49, 2, 50, 3, 51, 54, 68, 2, 69, 45, 154, 7, 79, - 210, 214, 14, 65, 2, 73, 135, 191, 2, 85, 6, 134, 184, 23, 77, 186, 2, - 49, 3, 50, 68, 116, 2, 69, 45, 72, 2, 73, 45, 246, 2, 65, 246, 208, 8, - 79, 250, 148, 5, 85, 165, 160, 4, 6, 45, 77, 85, 45, 77, 79, 14, 170, - 154, 23, 75, 246, 30, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 16, 130, - 162, 23, 84, 214, 22, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 54, - 222, 4, 79, 2, 85, 210, 214, 14, 73, 134, 191, 2, 65, 3, 69, 68, 62, 65, - 2, 85, 226, 3, 73, 186, 206, 8, 69, 155, 136, 6, 79, 16, 11, 45, 16, 250, - 182, 23, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 64, 74, 69, - 20, 2, 79, 45, 72, 2, 85, 45, 178, 216, 14, 73, 135, 191, 2, 65, 18, 223, - 169, 18, 45, 14, 150, 179, 23, 82, 186, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 53, 3, 54, 10, 146, 150, 23, 84, 246, 30, 49, 2, 50, 2, 51, 3, 52, 42, - 222, 207, 8, 65, 2, 73, 154, 136, 6, 79, 135, 191, 2, 69, 32, 40, 2, 65, - 45, 66, 79, 215, 149, 17, 85, 12, 242, 148, 23, 89, 246, 30, 49, 2, 50, - 2, 51, 2, 52, 3, 53, 12, 11, 45, 12, 154, 179, 23, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 3, 54, 4, 176, 133, 8, 21, 77, 73, 84, 73, 65, 78, 32, 67, 79, - 78, 74, 85, 71, 65, 84, 69, 32, 77, 65, 84, 82, 175, 173, 15, 66, 128, 1, - 210, 2, 65, 110, 66, 144, 1, 2, 67, 79, 94, 68, 198, 2, 70, 66, 71, 40, - 4, 72, 79, 76, 68, 164, 1, 2, 73, 78, 112, 2, 77, 79, 58, 79, 122, 80, - 100, 2, 82, 69, 62, 83, 238, 1, 84, 210, 5, 87, 216, 227, 14, 4, 76, 73, - 77, 73, 184, 137, 5, 11, 89, 79, 85, 84, 72, 70, 85, 76, 32, 70, 79, 133, - 159, 3, 9, 69, 78, 84, 72, 85, 83, 73, 65, 83, 6, 76, 3, 80, 80, 82, 124, - 4, 70, 84, 69, 82, 245, 209, 18, 4, 66, 85, 78, 68, 2, 149, 140, 23, 2, - 79, 65, 6, 92, 5, 69, 70, 79, 82, 69, 192, 193, 3, 6, 73, 84, 73, 78, 71, - 32, 1, 4, 82, 69, 65, 75, 2, 253, 220, 22, 7, 32, 67, 79, 77, 80, 76, 69, - 6, 26, 78, 239, 200, 18, 77, 4, 196, 236, 19, 4, 84, 69, 77, 80, 217, - 177, 2, 3, 70, 76, 73, 14, 110, 69, 78, 73, 134, 205, 18, 85, 173, 200, - 2, 15, 65, 82, 75, 69, 78, 73, 78, 71, 32, 79, 70, 32, 84, 72, 69, 6, - 150, 201, 18, 67, 192, 6, 2, 76, 73, 201, 152, 4, 5, 86, 69, 76, 79, 80, - 4, 184, 191, 18, 21, 70, 70, 73, 67, 85, 76, 84, 89, 32, 65, 84, 32, 84, - 72, 69, 32, 66, 69, 71, 73, 78, 137, 210, 2, 4, 83, 80, 69, 82, 4, 148, - 196, 18, 2, 79, 76, 221, 144, 4, 5, 69, 76, 76, 79, 87, 12, 36, 5, 65, - 84, 72, 69, 82, 35, 82, 2, 213, 206, 14, 3, 73, 78, 71, 10, 40, 4, 69, - 65, 84, 32, 211, 235, 22, 65, 8, 22, 80, 207, 5, 84, 6, 22, 79, 143, 5, - 82, 4, 168, 2, 2, 83, 83, 135, 232, 22, 87, 8, 50, 78, 134, 197, 18, 67, - 217, 5, 3, 70, 76, 85, 4, 184, 185, 18, 5, 69, 82, 32, 84, 82, 161, 17, - 2, 79, 67, 4, 128, 189, 21, 3, 68, 69, 83, 221, 59, 3, 85, 84, 72, 6, 26, - 66, 37, 2, 80, 80, 2, 209, 213, 22, 4, 83, 84, 82, 85, 4, 26, 82, 171, - 212, 22, 79, 2, 169, 140, 21, 2, 69, 83, 6, 200, 211, 14, 9, 85, 83, 72, - 73, 78, 71, 32, 85, 80, 168, 134, 2, 3, 82, 79, 71, 219, 142, 6, 69, 6, - 26, 84, 235, 180, 18, 86, 4, 186, 198, 18, 85, 215, 57, 82, 8, 132, 1, 5, - 77, 65, 76, 76, 32, 204, 178, 20, 6, 84, 65, 78, 68, 83, 84, 209, 233, 1, - 11, 80, 76, 73, 84, 84, 73, 78, 71, 32, 65, 80, 4, 24, 2, 80, 82, 43, 84, - 2, 157, 199, 18, 5, 69, 80, 79, 78, 68, 2, 11, 65, 2, 215, 209, 22, 77, - 30, 36, 3, 72, 69, 32, 171, 148, 17, 82, 28, 186, 2, 65, 130, 1, 67, 132, - 1, 3, 70, 65, 77, 20, 9, 82, 69, 67, 69, 80, 84, 73, 86, 69, 30, 87, 156, - 21, 12, 77, 65, 82, 82, 89, 73, 78, 71, 32, 77, 65, 73, 232, 242, 4, 6, - 71, 69, 78, 84, 76, 69, 204, 199, 10, 13, 75, 69, 69, 80, 73, 78, 71, 32, - 83, 84, 73, 76, 76, 177, 220, 3, 7, 74, 79, 89, 79, 85, 83, 32, 6, 58, - 82, 141, 140, 11, 8, 66, 89, 83, 77, 65, 76, 32, 87, 4, 220, 217, 19, 8, - 79, 85, 83, 73, 78, 71, 32, 84, 247, 179, 3, 77, 6, 160, 129, 1, 2, 65, - 85, 152, 215, 18, 9, 82, 69, 65, 84, 73, 86, 69, 32, 72, 233, 180, 1, 9, - 76, 73, 78, 71, 73, 78, 71, 32, 70, 2, 163, 250, 22, 73, 2, 129, 215, 19, - 2, 32, 69, 4, 242, 152, 21, 69, 209, 198, 1, 5, 65, 78, 68, 69, 82, 4, - 174, 198, 6, 65, 217, 146, 6, 14, 79, 82, 75, 32, 79, 78, 32, 84, 72, 69, - 32, 68, 69, 67, 222, 1, 236, 1, 2, 71, 72, 180, 2, 7, 82, 65, 71, 65, 78, - 65, 32, 212, 4, 7, 83, 84, 79, 82, 73, 67, 32, 184, 130, 15, 7, 78, 68, - 85, 32, 84, 69, 77, 236, 33, 4, 75, 73, 78, 71, 150, 235, 1, 66, 241, - 128, 4, 8, 80, 80, 79, 80, 79, 84, 65, 77, 12, 18, 32, 119, 45, 6, 144, - 234, 4, 2, 66, 82, 156, 152, 16, 6, 86, 79, 76, 84, 65, 71, 149, 47, 9, - 79, 67, 84, 69, 84, 32, 80, 82, 69, 6, 92, 11, 83, 80, 69, 69, 68, 32, - 84, 82, 65, 73, 78, 233, 166, 5, 6, 72, 69, 69, 76, 69, 68, 5, 133, 145, - 11, 14, 32, 87, 73, 84, 72, 32, 66, 85, 76, 76, 69, 84, 32, 78, 200, 1, - 112, 9, 68, 73, 71, 82, 65, 80, 72, 32, 89, 20, 7, 76, 69, 84, 84, 69, - 82, 32, 234, 170, 1, 86, 215, 207, 19, 73, 2, 135, 142, 16, 79, 194, 1, - 194, 1, 65, 74, 83, 218, 161, 1, 66, 162, 3, 78, 150, 2, 68, 2, 71, 2, - 72, 2, 75, 2, 77, 2, 80, 2, 82, 2, 84, 2, 90, 126, 87, 46, 89, 178, 199, - 21, 86, 234, 36, 69, 2, 73, 2, 79, 3, 85, 7, 37, 7, 82, 67, 72, 65, 73, - 67, 32, 4, 130, 240, 22, 87, 151, 14, 89, 42, 76, 5, 77, 65, 76, 76, 32, - 254, 147, 23, 65, 2, 69, 2, 73, 2, 79, 3, 85, 32, 182, 167, 1, 87, 46, - 89, 130, 142, 5, 75, 178, 185, 16, 84, 234, 36, 65, 2, 69, 2, 73, 2, 79, - 3, 85, 2, 171, 145, 8, 83, 92, 166, 1, 76, 52, 3, 78, 69, 89, 46, 82, - 130, 6, 84, 138, 1, 85, 210, 130, 16, 83, 138, 203, 2, 67, 172, 149, 3, - 6, 77, 79, 84, 72, 69, 84, 214, 163, 1, 79, 155, 3, 80, 6, 180, 212, 15, - 4, 76, 79, 87, 32, 163, 189, 7, 69, 4, 238, 164, 21, 66, 137, 156, 1, 2, - 32, 80, 50, 60, 8, 73, 90, 79, 78, 84, 65, 76, 32, 137, 5, 2, 83, 69, 44, - 198, 1, 66, 108, 4, 76, 73, 78, 69, 28, 17, 79, 78, 69, 32, 69, 73, 71, + 80, 215, 212, 19, 77, 7, 11, 32, 4, 182, 155, 27, 69, 221, 6, 2, 83, 84, + 22, 32, 2, 48, 48, 203, 221, 15, 79, 20, 222, 188, 22, 50, 2, 54, 134, + 236, 5, 49, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 164, 1, 118, 48, 128, 4, + 15, 79, 68, 73, 70, 73, 69, 82, 32, 68, 65, 77, 65, 71, 69, 68, 205, 228, + 24, 5, 73, 82, 82, 79, 82, 132, 1, 42, 48, 98, 49, 106, 50, 102, 51, 107, + 52, 24, 182, 40, 49, 170, 146, 22, 51, 134, 236, 5, 50, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 44, 138, 41, 50, 246, 144, 22, 48, 2, 53, 2, 54, + 2, 55, 134, 236, 5, 49, 2, 51, 2, 52, 2, 56, 3, 57, 26, 150, 185, 22, 50, + 2, 52, 2, 56, 134, 236, 5, 48, 2, 49, 2, 51, 2, 53, 2, 54, 2, 55, 3, 57, + 26, 138, 38, 51, 170, 146, 22, 49, 134, 236, 5, 48, 2, 50, 2, 52, 2, 53, + 2, 54, 2, 55, 2, 56, 3, 57, 12, 202, 183, 22, 48, 134, 236, 5, 49, 2, 50, + 2, 51, 3, 52, 31, 25, 4, 32, 65, 84, 32, 28, 96, 6, 66, 79, 84, 84, 79, + 77, 120, 5, 83, 84, 65, 82, 84, 68, 3, 84, 79, 80, 227, 146, 27, 69, 11, + 11, 32, 8, 56, 5, 83, 84, 65, 82, 84, 186, 1, 65, 159, 146, 27, 69, 5, + 129, 2, 8, 32, 65, 78, 68, 32, 84, 79, 80, 7, 29, 5, 32, 65, 78, 68, 32, + 4, 206, 181, 24, 66, 155, 242, 2, 84, 11, 11, 32, 8, 54, 65, 20, 5, 83, + 84, 65, 82, 84, 139, 146, 27, 69, 2, 73, 2, 78, 68, 5, 53, 11, 32, 65, + 78, 68, 32, 66, 79, 84, 84, 79, 77, 2, 251, 209, 19, 32, 194, 1, 50, 48, + 128, 2, 2, 76, 48, 229, 1, 2, 85, 48, 98, 58, 49, 98, 51, 194, 14, 50, + 150, 6, 52, 227, 209, 22, 48, 24, 146, 32, 56, 174, 254, 27, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 28, 218, 177, 22, 51, 2, + 52, 2, 53, 2, 55, 134, 236, 5, 48, 2, 49, 2, 50, 2, 54, 2, 56, 3, 57, 44, + 34, 48, 94, 49, 175, 131, 21, 50, 20, 210, 176, 22, 53, 134, 236, 5, 49, + 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 22, 246, 175, 22, 55, + 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, + 52, 34, 49, 102, 50, 231, 226, 22, 48, 26, 238, 174, 22, 48, 2, 49, 2, + 56, 134, 236, 5, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 8, 138, + 174, 22, 50, 134, 236, 5, 48, 3, 49, 152, 1, 50, 48, 145, 247, 18, 6, 86, + 69, 82, 76, 65, 89, 150, 1, 66, 48, 154, 1, 49, 138, 1, 50, 102, 51, 106, + 53, 191, 223, 22, 52, 34, 90, 54, 134, 172, 22, 49, 2, 53, 134, 236, 5, + 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, 134, 152, 28, 65, 2, 66, 2, + 67, 2, 68, 2, 69, 3, 70, 28, 98, 48, 230, 170, 22, 57, 134, 236, 5, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 9, 230, 150, 28, 65, 2, + 66, 3, 67, 28, 190, 170, 22, 48, 2, 52, 2, 53, 2, 57, 134, 236, 5, 49, 2, + 50, 2, 51, 2, 54, 2, 55, 3, 56, 32, 134, 23, 54, 214, 146, 22, 48, 2, 51, + 134, 236, 5, 49, 2, 50, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 8, 202, 22, + 48, 175, 254, 27, 49, 26, 26, 48, 143, 194, 8, 49, 22, 182, 168, 22, 49, + 2, 51, 134, 236, 5, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 68, 34, + 48, 98, 49, 163, 221, 22, 50, 24, 142, 21, 51, 170, 146, 22, 50, 134, + 236, 5, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 214, 166, 22, + 48, 2, 54, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, + 57, 108, 50, 48, 94, 49, 106, 50, 98, 51, 171, 172, 19, 52, 22, 190, 165, + 22, 50, 2, 54, 134, 236, 5, 49, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, + 26, 186, 18, 52, 170, 146, 22, 55, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, + 2, 53, 2, 54, 2, 56, 3, 57, 24, 210, 17, 54, 174, 254, 27, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 22, 154, 163, 22, 53, 134, + 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 90, + 34, 48, 249, 12, 3, 65, 76, 76, 88, 42, 48, 94, 49, 102, 51, 243, 214, + 22, 50, 26, 230, 161, 22, 51, 2, 55, 2, 56, 2, 57, 134, 236, 5, 49, 2, + 50, 2, 52, 2, 53, 3, 54, 24, 138, 161, 22, 49, 2, 54, 134, 236, 5, 48, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 18, 166, 160, 22, 50, 2, + 51, 134, 236, 5, 48, 2, 49, 2, 52, 2, 53, 3, 54, 94, 50, 48, 90, 50, 102, + 51, 102, 52, 211, 211, 22, 49, 22, 254, 12, 54, 174, 254, 27, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, 206, 158, 22, 51, 2, 57, + 134, 236, 5, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 22, + 234, 157, 22, 50, 134, 236, 5, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 2, 56, 3, 57, 6, 138, 137, 28, 48, 2, 49, 3, 50, 158, 1, 42, 48, 185, + 4, 5, 69, 82, 84, 73, 67, 156, 1, 62, 48, 98, 49, 98, 50, 210, 1, 51, + 225, 152, 22, 2, 52, 48, 42, 198, 9, 55, 98, 49, 202, 145, 22, 50, 134, + 236, 5, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 32, 186, 8, 49, 46, 50, + 174, 254, 27, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 50, + 98, 48, 206, 153, 22, 51, 2, 56, 2, 57, 134, 236, 5, 49, 2, 50, 2, 52, 2, + 53, 2, 54, 3, 55, 27, 206, 133, 28, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, + 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 3, 76, 28, 222, 152, 22, 48, 2, + 49, 2, 51, 2, 55, 134, 236, 5, 50, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 2, + 221, 221, 23, 2, 65, 76, 66, 34, 48, 161, 2, 3, 73, 68, 69, 64, 26, 48, + 94, 49, 103, 50, 22, 158, 151, 22, 51, 2, 57, 134, 236, 5, 49, 2, 50, 2, + 52, 2, 53, 2, 54, 2, 55, 3, 56, 28, 194, 150, 22, 48, 2, 52, 2, 55, 2, + 56, 134, 236, 5, 49, 2, 50, 2, 51, 2, 53, 2, 54, 3, 57, 14, 222, 149, 22, + 52, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 3, 53, 2, 17, 2, 32, 76, 2, + 239, 181, 15, 79, 24, 202, 2, 52, 170, 146, 22, 54, 2, 56, 134, 236, 5, + 49, 2, 50, 2, 51, 2, 53, 3, 55, 18, 154, 148, 22, 49, 134, 236, 5, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 82, 22, 48, 167, 1, 49, 34, 90, + 50, 46, 51, 170, 146, 22, 52, 2, 53, 134, 236, 5, 49, 2, 54, 2, 55, 2, + 56, 3, 57, 11, 214, 254, 27, 65, 2, 66, 2, 67, 3, 68, 7, 170, 254, 27, + 65, 3, 66, 48, 66, 53, 86, 54, 250, 252, 27, 48, 2, 49, 2, 50, 2, 51, 3, + 52, 21, 202, 253, 27, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, + 72, 3, 73, 19, 246, 252, 27, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, + 71, 3, 72, 182, 62, 22, 51, 211, 1, 52, 192, 46, 106, 52, 174, 188, 5, + 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, + 192, 2, 214, 179, 13, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, + 68, 2, 69, 3, 70, 246, 15, 42, 51, 158, 187, 5, 48, 2, 49, 3, 50, 246, 3, + 142, 1, 70, 186, 177, 13, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 3, 69, 22, 250, 248, 27, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 3, 65, + 18, 26, 32, 239, 226, 15, 72, 16, 70, 80, 124, 5, 82, 65, 89, 83, 32, + 190, 146, 3, 84, 231, 207, 22, 83, 8, 186, 146, 3, 79, 241, 185, 16, 22, + 69, 84, 65, 76, 76, 69, 68, 32, 79, 85, 84, 76, 73, 78, 69, 68, 32, 66, + 76, 65, 67, 75, 4, 228, 234, 18, 2, 73, 78, 1, 3, 79, 85, 84, 160, 1, + 132, 1, 13, 66, 65, 83, 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 166, 3, + 69, 176, 4, 7, 89, 77, 65, 73, 67, 32, 76, 171, 237, 27, 70, 80, 230, 1, + 71, 78, 76, 34, 78, 50, 82, 158, 232, 25, 69, 190, 137, 1, 67, 2, 68, 2, + 75, 2, 83, 2, 84, 2, 90, 206, 105, 66, 2, 70, 2, 72, 2, 74, 2, 77, 2, 80, + 2, 81, 2, 86, 2, 88, 214, 22, 65, 2, 73, 2, 79, 2, 85, 3, 89, 8, 38, 72, + 154, 220, 27, 74, 215, 22, 69, 4, 242, 160, 25, 65, 251, 209, 2, 69, 4, + 242, 219, 27, 76, 215, 22, 69, 8, 210, 219, 27, 68, 2, 74, 214, 22, 65, + 3, 69, 4, 162, 219, 27, 82, 215, 22, 69, 32, 96, 5, 67, 84, 82, 73, 67, + 160, 1, 7, 77, 69, 78, 84, 32, 79, 70, 238, 229, 26, 80, 235, 80, 86, 10, + 26, 32, 179, 130, 24, 65, 8, 98, 80, 144, 205, 17, 2, 84, 79, 128, 231, + 8, 9, 76, 73, 71, 72, 84, 32, 66, 85, 76, 171, 33, 65, 2, 11, 76, 2, 147, + 239, 27, 85, 19, 11, 32, 16, 72, 5, 87, 73, 84, 72, 32, 217, 207, 16, 7, + 79, 80, 69, 78, 73, 78, 71, 12, 130, 1, 76, 32, 12, 84, 87, 79, 32, 72, + 79, 82, 73, 90, 79, 78, 84, 142, 214, 19, 86, 194, 172, 4, 85, 234, 60, + 79, 219, 173, 2, 68, 2, 173, 201, 25, 3, 79, 78, 71, 2, 217, 213, 14, 7, + 65, 76, 32, 83, 84, 82, 79, 46, 254, 168, 4, 69, 129, 202, 15, 15, 73, + 71, 65, 84, 85, 82, 69, 32, 90, 65, 89, 73, 78, 45, 89, 53, 48, 4, 79, + 74, 73, 32, 218, 2, 80, 163, 3, 32, 18, 164, 1, 10, 67, 79, 77, 80, 79, + 78, 69, 78, 84, 32, 109, 26, 77, 79, 68, 73, 70, 73, 69, 82, 32, 70, 73, + 84, 90, 80, 65, 84, 82, 73, 67, 75, 32, 84, 89, 80, 69, 45, 8, 128, 255, + 14, 2, 82, 69, 12, 5, 67, 85, 82, 76, 89, 0, 5, 87, 72, 73, 84, 69, 213, + 205, 2, 2, 66, 65, 10, 208, 208, 20, 2, 49, 45, 174, 153, 7, 51, 2, 52, + 2, 53, 3, 54, 26, 44, 3, 84, 89, 32, 177, 244, 3, 2, 72, 65, 24, 82, 78, + 60, 3, 80, 65, 71, 20, 3, 83, 69, 84, 181, 164, 27, 4, 68, 79, 67, 85, 8, + 36, 3, 79, 84, 69, 151, 254, 23, 69, 7, 199, 152, 13, 32, 4, 139, 225, + 25, 69, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 130, 130, 14, 82, 24, 3, + 76, 69, 70, 212, 166, 1, 2, 83, 77, 179, 144, 9, 79, 38, 86, 32, 64, 2, + 68, 32, 214, 1, 81, 20, 6, 86, 69, 76, 79, 80, 69, 227, 220, 15, 84, 6, + 42, 81, 198, 253, 25, 68, 131, 170, 1, 83, 2, 195, 159, 27, 85, 20, 32, + 3, 79, 70, 32, 131, 1, 87, 18, 88, 3, 80, 82, 79, 242, 221, 20, 71, 56, + 2, 83, 69, 162, 71, 77, 30, 84, 195, 165, 5, 76, 4, 178, 222, 20, 84, + 135, 133, 6, 79, 2, 173, 239, 19, 7, 73, 84, 72, 32, 76, 69, 70, 5, 231, + 165, 21, 85, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 42, 76, 209, 233, 23, + 4, 68, 79, 87, 78, 2, 209, 203, 22, 4, 73, 71, 72, 84, 6, 254, 226, 27, + 76, 2, 77, 3, 84, 44, 28, 2, 65, 76, 231, 6, 73, 38, 30, 32, 133, 2, 2, + 83, 32, 12, 56, 3, 84, 79, 32, 177, 141, 13, 5, 65, 78, 68, 32, 80, 10, + 68, 3, 79, 82, 32, 221, 143, 27, 8, 66, 89, 32, 68, 69, 70, 73, 78, 8, + 64, 3, 80, 82, 69, 28, 3, 83, 85, 67, 162, 206, 25, 71, 39, 76, 2, 193, + 156, 15, 2, 67, 69, 2, 233, 128, 26, 3, 67, 69, 69, 26, 72, 4, 83, 73, + 71, 78, 224, 203, 25, 4, 87, 73, 84, 72, 235, 195, 1, 67, 23, 11, 32, 20, + 42, 65, 201, 1, 5, 87, 73, 84, 72, 32, 12, 112, 5, 66, 79, 86, 69, 32, + 65, 19, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 32, 80, 65, 82, 65, 76, + 76, 69, 76, 8, 154, 132, 21, 80, 210, 5, 84, 182, 184, 2, 76, 139, 252, + 2, 82, 5, 243, 239, 6, 32, 8, 160, 1, 4, 66, 85, 77, 80, 20, 7, 73, 78, + 70, 73, 78, 73, 84, 20, 18, 84, 87, 79, 32, 68, 79, 84, 83, 32, 65, 66, + 79, 86, 69, 32, 65, 78, 68, 215, 156, 15, 68, 2, 163, 223, 26, 89, 2, + 207, 174, 25, 89, 2, 17, 2, 32, 84, 2, 243, 173, 25, 87, 6, 80, 7, 86, + 65, 76, 69, 78, 84, 32, 237, 135, 21, 7, 65, 78, 71, 85, 76, 65, 82, 4, + 48, 6, 87, 73, 84, 72, 32, 70, 223, 187, 27, 84, 2, 11, 79, 2, 201, 242, + 2, 2, 85, 82, 20, 152, 1, 11, 82, 79, 82, 45, 66, 65, 82, 82, 69, 68, 32, + 152, 163, 5, 7, 73, 83, 32, 70, 79, 82, 77, 193, 195, 7, 9, 65, 83, 69, + 32, 84, 79, 32, 84, 72, 12, 160, 220, 5, 4, 87, 72, 73, 84, 13, 5, 66, + 76, 65, 67, 75, 10, 58, 67, 20, 6, 84, 73, 77, 65, 84, 69, 255, 215, 27, + 65, 5, 211, 144, 26, 65, 4, 210, 196, 26, 68, 171, 147, 1, 83, 154, 8, + 60, 7, 72, 73, 79, 80, 73, 67, 32, 150, 215, 27, 66, 3, 88, 150, 8, 204, + 1, 2, 67, 79, 232, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, 80, 54, 83, + 156, 24, 11, 84, 79, 78, 65, 76, 32, 77, 65, 82, 75, 32, 226, 252, 18, + 68, 214, 235, 5, 70, 82, 81, 221, 146, 2, 4, 87, 79, 82, 68, 10, 26, 77, + 187, 133, 27, 76, 8, 52, 7, 66, 73, 78, 73, 78, 71, 32, 183, 210, 27, 77, + 6, 60, 11, 71, 69, 77, 73, 78, 65, 84, 73, 79, 78, 32, 51, 86, 4, 44, 5, + 65, 78, 68, 32, 86, 211, 146, 27, 77, 2, 201, 246, 23, 4, 79, 87, 69, 76, + 22, 66, 84, 226, 255, 21, 72, 222, 228, 3, 69, 30, 70, 42, 78, 39, 83, 8, + 194, 199, 16, 69, 170, 158, 9, 72, 27, 87, 4, 166, 113, 65, 153, 145, 26, + 5, 82, 69, 70, 65, 67, 198, 7, 50, 69, 37, 8, 89, 76, 76, 65, 66, 76, 69, + 32, 4, 202, 134, 24, 77, 139, 226, 2, 67, 194, 7, 210, 1, 66, 90, 67, + 246, 1, 68, 186, 1, 70, 90, 71, 214, 2, 72, 162, 1, 75, 102, 77, 90, 78, + 90, 80, 138, 2, 81, 174, 1, 82, 86, 83, 210, 1, 84, 122, 74, 2, 76, 138, + 1, 87, 2, 89, 66, 88, 134, 1, 90, 95, 86, 38, 194, 12, 87, 230, 8, 66, + 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 78, 94, 67, + 254, 15, 72, 202, 210, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, + 2, 73, 3, 85, 42, 70, 72, 254, 225, 21, 65, 230, 171, 5, 69, 162, 64, 73, + 2, 79, 3, 85, 28, 166, 19, 72, 214, 206, 21, 65, 230, 171, 5, 69, 162, + 64, 73, 2, 79, 3, 85, 60, 94, 68, 214, 14, 90, 254, 209, 21, 65, 2, 79, + 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 30, 210, 14, 72, 254, + 209, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 24, + 190, 8, 87, 186, 215, 21, 65, 230, 171, 5, 69, 234, 61, 89, 186, 2, 73, + 2, 79, 3, 85, 118, 142, 1, 85, 226, 7, 71, 232, 3, 7, 76, 79, 84, 84, 65, + 76, 32, 158, 2, 87, 218, 1, 89, 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, + 163, 64, 73, 39, 29, 5, 82, 65, 71, 69, 32, 36, 74, 66, 2, 70, 2, 77, 2, + 80, 46, 71, 2, 75, 2, 81, 191, 130, 12, 72, 4, 11, 87, 4, 198, 178, 27, + 69, 215, 22, 73, 6, 11, 87, 6, 206, 136, 27, 69, 163, 64, 73, 52, 70, 72, + 134, 220, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 36, 202, 4, + 87, 230, 8, 89, 214, 206, 21, 65, 230, 171, 5, 69, 162, 64, 73, 2, 79, 3, + 85, 64, 250, 4, 88, 134, 6, 87, 218, 1, 89, 214, 206, 21, 65, 2, 79, 230, + 171, 5, 69, 162, 64, 73, 3, 85, 26, 142, 3, 87, 186, 215, 21, 65, 2, 79, + 230, 171, 5, 69, 234, 61, 89, 186, 2, 73, 3, 85, 36, 166, 7, 89, 202, + 210, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 56, + 82, 72, 142, 1, 87, 186, 215, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, + 73, 3, 85, 32, 74, 65, 250, 215, 21, 79, 230, 171, 5, 69, 234, 61, 87, + 186, 2, 73, 3, 85, 19, 160, 9, 8, 82, 89, 78, 71, 69, 65, 76, 32, 219, + 186, 27, 65, 8, 154, 131, 27, 69, 162, 64, 65, 3, 73, 64, 94, 72, 134, 6, + 87, 218, 1, 89, 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, + 85, 24, 130, 6, 87, 174, 208, 21, 65, 230, 171, 5, 69, 162, 64, 73, 2, + 79, 3, 85, 20, 226, 213, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 2, + 89, 186, 2, 73, 3, 85, 74, 102, 69, 226, 1, 72, 170, 3, 90, 78, 83, 214, + 206, 21, 65, 2, 79, 206, 233, 5, 87, 186, 2, 73, 3, 85, 13, 56, 8, 66, + 65, 84, 66, 69, 73, 84, 32, 243, 191, 27, 69, 8, 230, 206, 22, 66, 2, 70, + 2, 77, 3, 80, 80, 118, 72, 76, 2, 84, 72, 62, 90, 162, 2, 83, 162, 207, + 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 18, 198, + 210, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 12, + 222, 253, 26, 69, 234, 61, 65, 186, 2, 73, 2, 79, 3, 85, 16, 190, 209, + 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 40, 82, 87, 218, 1, + 89, 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 10, + 170, 208, 21, 65, 230, 171, 5, 69, 163, 64, 73, 48, 90, 72, 78, 90, 214, + 206, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 16, + 158, 207, 21, 65, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 2, 79, 3, 85, + 14, 210, 206, 21, 65, 230, 171, 5, 69, 162, 64, 73, 2, 79, 3, 85, 20, + 130, 1, 68, 74, 72, 30, 75, 42, 82, 0, 7, 83, 72, 79, 82, 84, 32, 82, + 204, 140, 4, 3, 67, 72, 73, 165, 246, 22, 3, 89, 73, 90, 6, 48, 4, 69, + 82, 69, 84, 249, 164, 26, 2, 73, 70, 5, 17, 2, 45, 72, 2, 177, 131, 27, + 2, 73, 68, 4, 196, 164, 26, 2, 69, 78, 219, 12, 85, 2, 245, 154, 4, 3, + 73, 75, 82, 10, 68, 2, 82, 79, 197, 173, 26, 9, 76, 69, 82, 32, 67, 79, + 78, 83, 84, 8, 92, 5, 80, 69, 65, 78, 32, 204, 131, 24, 8, 45, 67, 85, + 82, 82, 69, 78, 67, 131, 172, 2, 32, 4, 214, 135, 4, 67, 19, 80, 50, 30, + 67, 102, 80, 187, 1, 84, 6, 60, 9, 76, 65, 77, 65, 84, 73, 79, 78, 32, + 247, 158, 26, 69, 4, 174, 228, 24, 81, 235, 143, 2, 77, 10, 96, 7, 76, + 79, 83, 73, 79, 78, 32, 157, 247, 26, 11, 82, 69, 83, 83, 73, 79, 78, 76, + 69, 83, 83, 8, 232, 209, 11, 4, 70, 82, 65, 77, 229, 146, 15, 8, 65, 84, + 32, 72, 79, 82, 73, 90, 34, 98, 82, 153, 179, 21, 18, 69, 78, 68, 69, 68, + 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 14, 140, 1, 12, 69, + 77, 69, 76, 89, 32, 72, 69, 65, 86, 89, 32, 141, 222, 25, 16, 65, 84, 69, + 82, 82, 69, 83, 84, 82, 73, 65, 76, 32, 65, 76, 73, 12, 50, 83, 202, 155, + 25, 70, 234, 2, 87, 215, 10, 71, 4, 218, 156, 25, 65, 43, 73, 7, 250, + 167, 19, 71, 171, 137, 8, 83, 142, 4, 142, 1, 65, 130, 19, 69, 190, 1, + 73, 230, 7, 76, 242, 5, 79, 238, 6, 82, 234, 4, 85, 228, 183, 20, 2, 86, + 83, 202, 192, 4, 83, 223, 137, 2, 70, 92, 122, 67, 178, 15, 76, 176, 2, + 2, 88, 32, 238, 141, 3, 77, 172, 247, 5, 2, 82, 83, 188, 250, 1, 2, 84, + 72, 187, 230, 9, 73, 70, 72, 2, 69, 32, 248, 172, 16, 4, 83, 73, 77, 73, + 169, 202, 4, 2, 84, 79, 66, 226, 1, 83, 160, 1, 4, 87, 73, 84, 72, 224, + 250, 12, 2, 80, 65, 228, 136, 2, 2, 77, 65, 140, 145, 11, 13, 84, 72, 82, + 79, 87, 73, 78, 71, 32, 65, 32, 75, 73, 201, 1, 14, 72, 79, 76, 68, 73, + 78, 71, 32, 66, 65, 67, 75, 32, 84, 4, 232, 251, 16, 18, 65, 86, 79, 85, + 82, 73, 78, 71, 32, 68, 69, 76, 73, 67, 73, 79, 85, 83, 241, 128, 5, 13, + 67, 82, 69, 65, 77, 73, 78, 71, 32, 73, 78, 32, 70, 54, 38, 32, 201, 137, + 24, 3, 79, 85, 84, 52, 196, 4, 2, 67, 79, 44, 5, 72, 69, 65, 68, 45, 38, + 77, 98, 79, 92, 7, 78, 79, 32, 71, 79, 79, 68, 238, 1, 80, 164, 1, 16, + 83, 84, 85, 67, 75, 45, 79, 85, 84, 32, 84, 79, 78, 71, 85, 69, 110, 84, + 192, 177, 1, 9, 66, 65, 71, 83, 32, 85, 78, 68, 69, 224, 158, 17, 22, 70, + 73, 78, 71, 69, 82, 32, 67, 79, 86, 69, 82, 73, 78, 71, 32, 67, 76, 79, + 83, 69, 68, 244, 67, 3, 82, 79, 76, 252, 231, 1, 13, 76, 79, 79, 75, 32, + 79, 70, 32, 84, 82, 73, 85, 77, 232, 130, 3, 8, 68, 73, 65, 71, 79, 78, + 65, 76, 1, 20, 85, 78, 69, 86, 69, 78, 32, 69, 89, 69, 83, 32, 65, 78, + 68, 32, 87, 65, 86, 89, 4, 252, 4, 3, 87, 66, 79, 227, 148, 19, 76, 2, + 173, 159, 22, 4, 66, 65, 78, 68, 4, 60, 6, 69, 68, 73, 67, 65, 76, 189, + 211, 25, 3, 79, 78, 79, 2, 177, 224, 25, 3, 32, 77, 65, 12, 90, 75, 36, + 4, 80, 69, 78, 32, 205, 161, 20, 10, 78, 69, 32, 69, 89, 69, 66, 82, 79, + 87, 2, 165, 142, 22, 4, 32, 71, 69, 83, 8, 116, 5, 77, 79, 85, 84, 72, + 217, 129, 24, 18, 69, 89, 69, 83, 32, 65, 78, 68, 32, 72, 65, 78, 68, 32, + 79, 86, 69, 82, 7, 11, 32, 4, 136, 252, 9, 3, 86, 79, 77, 141, 154, 9, 5, + 65, 78, 68, 32, 67, 6, 132, 1, 18, 65, 82, 84, 89, 32, 72, 79, 82, 78, + 32, 65, 78, 68, 32, 80, 65, 82, 84, 100, 2, 69, 69, 197, 148, 19, 4, 76, + 69, 65, 68, 2, 161, 184, 22, 2, 89, 32, 7, 29, 5, 32, 65, 78, 68, 32, 4, + 36, 3, 87, 73, 78, 231, 148, 19, 84, 2, 157, 177, 1, 4, 75, 73, 78, 71, + 4, 50, 69, 225, 187, 22, 6, 72, 69, 82, 77, 79, 77, 2, 157, 142, 27, 8, + 65, 82, 83, 32, 79, 70, 32, 74, 10, 34, 76, 185, 146, 22, 2, 65, 70, 8, + 84, 13, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 189, 212, 23, + 2, 69, 78, 6, 128, 1, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 185, 167, + 19, 16, 73, 78, 32, 87, 72, 73, 84, 69, 32, 67, 73, 82, 67, 76, 69, 32, + 4, 232, 133, 16, 3, 82, 73, 83, 143, 169, 3, 78, 4, 222, 244, 14, 73, + 255, 137, 4, 77, 14, 50, 65, 50, 77, 44, 2, 82, 82, 147, 217, 18, 78, 4, + 212, 161, 8, 3, 82, 70, 85, 147, 231, 9, 84, 4, 204, 233, 8, 2, 73, 78, + 143, 177, 7, 65, 4, 184, 238, 7, 2, 73, 83, 163, 173, 19, 89, 56, 156, 1, + 5, 71, 85, 82, 69, 32, 38, 76, 178, 1, 78, 98, 82, 182, 2, 83, 236, 13, + 4, 86, 69, 32, 68, 213, 249, 4, 10, 69, 76, 68, 32, 72, 79, 67, 75, 69, + 89, 4, 226, 177, 25, 68, 131, 170, 1, 83, 10, 32, 2, 69, 32, 65, 2, 77, + 32, 6, 166, 187, 13, 70, 136, 154, 2, 3, 67, 65, 66, 191, 150, 8, 83, 4, + 52, 4, 80, 82, 79, 74, 197, 142, 20, 3, 70, 82, 65, 2, 171, 203, 15, 69, + 4, 72, 4, 71, 69, 82, 80, 205, 144, 25, 8, 73, 84, 69, 32, 80, 65, 82, + 84, 2, 167, 255, 15, 82, 22, 34, 69, 189, 1, 3, 83, 84, 32, 13, 56, 2, + 32, 69, 68, 4, 87, 79, 82, 75, 171, 170, 15, 67, 4, 146, 179, 13, 78, + 153, 208, 4, 8, 88, 84, 73, 78, 71, 85, 73, 83, 4, 216, 172, 23, 6, 32, + 83, 80, 65, 82, 75, 223, 233, 3, 83, 10, 190, 170, 5, 81, 216, 176, 10, + 8, 83, 84, 82, 79, 78, 71, 32, 73, 239, 215, 6, 80, 10, 38, 72, 129, 200, + 23, 3, 84, 69, 68, 9, 168, 248, 3, 13, 73, 78, 71, 32, 80, 79, 76, 69, + 32, 65, 78, 68, 32, 246, 202, 5, 69, 237, 202, 16, 19, 32, 67, 65, 75, + 69, 32, 87, 73, 84, 72, 32, 83, 87, 73, 82, 76, 32, 68, 69, 42, 50, 65, + 170, 1, 69, 98, 79, 194, 1, 85, 39, 89, 12, 114, 84, 148, 157, 4, 5, 80, + 80, 73, 78, 71, 136, 158, 3, 6, 71, 32, 73, 78, 32, 72, 161, 202, 17, 3, + 77, 73, 78, 6, 198, 208, 8, 32, 250, 197, 11, 66, 139, 229, 5, 78, 4, + 160, 221, 23, 8, 88, 69, 68, 32, 66, 73, 67, 69, 153, 145, 1, 7, 85, 82, + 45, 68, 69, 45, 76, 10, 46, 82, 28, 3, 87, 69, 82, 167, 203, 25, 80, 2, + 161, 137, 26, 2, 65, 76, 7, 17, 2, 32, 80, 4, 58, 85, 169, 178, 24, 8, + 76, 65, 89, 73, 78, 71, 32, 67, 2, 189, 166, 26, 4, 78, 67, 84, 85, 4, + 162, 152, 3, 83, 219, 224, 23, 84, 13, 25, 4, 73, 78, 71, 32, 10, 64, 6, + 83, 65, 85, 67, 69, 82, 166, 130, 10, 68, 223, 130, 10, 69, 7, 29, 5, 32, + 87, 73, 84, 72, 4, 34, 32, 1, 4, 79, 85, 84, 32, 2, 21, 3, 66, 69, 65, 2, + 147, 208, 26, 77, 54, 102, 71, 20, 2, 76, 68, 68, 2, 79, 84, 22, 82, 238, + 1, 85, 148, 255, 22, 2, 78, 68, 235, 205, 3, 88, 5, 171, 251, 26, 71, 4, + 164, 192, 9, 8, 73, 78, 71, 32, 72, 65, 78, 68, 223, 147, 17, 69, 5, 139, + 148, 14, 80, 16, 102, 75, 158, 206, 20, 77, 186, 181, 4, 32, 214, 115, + 67, 201, 120, 9, 84, 85, 78, 69, 32, 67, 79, 79, 75, 8, 80, 10, 32, 65, + 78, 68, 32, 75, 78, 73, 70, 69, 162, 150, 15, 69, 183, 163, 11, 73, 5, + 249, 207, 15, 7, 32, 87, 73, 84, 72, 32, 80, 22, 26, 82, 255, 161, 23, + 78, 20, 38, 32, 218, 2, 84, 183, 221, 18, 45, 16, 110, 67, 174, 1, 68, + 134, 165, 2, 66, 248, 147, 10, 7, 76, 69, 65, 70, 32, 67, 76, 182, 111, + 84, 155, 200, 11, 80, 4, 128, 170, 13, 3, 76, 85, 66, 209, 86, 32, 79, + 82, 78, 69, 82, 32, 65, 82, 82, 79, 87, 83, 32, 67, 73, 82, 67, 76, 73, + 78, 71, 32, 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 4, 21, 3, 79, 84, 32, + 4, 138, 197, 23, 80, 239, 128, 3, 77, 2, 183, 38, 72, 28, 78, 65, 226, 1, + 69, 98, 79, 209, 241, 16, 8, 73, 69, 68, 32, 83, 72, 82, 73, 10, 72, 6, + 67, 84, 73, 79, 78, 32, 69, 8, 77, 69, 32, 87, 73, 84, 72, 32, 4, 146, + 239, 19, 83, 177, 6, 9, 78, 85, 77, 69, 82, 65, 84, 79, 82, 6, 50, 80, + 194, 165, 2, 65, 217, 161, 21, 2, 84, 73, 2, 221, 238, 21, 2, 73, 67, 6, + 48, 6, 78, 67, 72, 32, 70, 82, 235, 251, 18, 69, 4, 146, 240, 25, 73, + 249, 12, 3, 65, 78, 67, 10, 52, 3, 78, 84, 45, 116, 2, 87, 78, 151, 197, + 26, 71, 4, 70, 84, 173, 138, 2, 11, 70, 65, 67, 73, 78, 71, 32, 66, 65, + 66, 89, 2, 197, 178, 12, 6, 73, 76, 84, 69, 68, 32, 5, 149, 135, 8, 6, + 73, 78, 71, 32, 70, 65, 226, 1, 80, 2, 76, 76, 134, 6, 78, 156, 233, 16, + 5, 69, 76, 32, 80, 85, 191, 251, 9, 83, 216, 1, 42, 32, 73, 6, 87, 73, + 68, 84, 72, 32, 10, 246, 255, 11, 77, 252, 169, 3, 2, 79, 85, 210, 232, + 8, 66, 151, 29, 83, 206, 1, 242, 1, 67, 42, 76, 78, 78, 30, 80, 66, 82, + 142, 1, 83, 38, 89, 158, 146, 10, 77, 198, 208, 10, 65, 158, 2, 68, 58, + 69, 98, 71, 118, 72, 202, 4, 81, 190, 143, 2, 87, 218, 28, 84, 250, 145, + 1, 70, 148, 176, 1, 6, 66, 82, 79, 75, 69, 78, 211, 7, 86, 10, 214, 230, + 20, 73, 62, 79, 159, 75, 69, 116, 42, 69, 138, 234, 20, 65, 243, 171, 4, + 79, 10, 162, 1, 70, 251, 235, 20, 83, 4, 246, 186, 21, 79, 35, 85, 6, 42, + 79, 250, 236, 20, 69, 203, 225, 2, 76, 2, 159, 148, 25, 85, 10, 36, 3, + 73, 71, 72, 131, 231, 24, 69, 8, 17, 2, 84, 32, 8, 228, 162, 20, 5, 87, + 72, 73, 84, 69, 202, 211, 2, 67, 210, 3, 80, 239, 7, 83, 4, 166, 176, 23, + 69, 147, 184, 1, 79, 2, 159, 130, 25, 69, 6, 136, 192, 11, 8, 67, 84, 73, + 79, 78, 32, 65, 80, 176, 247, 8, 5, 69, 82, 65, 76, 32, 143, 183, 1, 78, + 226, 20, 114, 65, 250, 6, 69, 198, 17, 73, 162, 1, 76, 134, 15, 79, 194, + 6, 82, 178, 92, 85, 246, 131, 22, 72, 139, 234, 3, 83, 142, 1, 42, 82, + 237, 221, 26, 4, 77, 69, 32, 68, 140, 1, 36, 3, 65, 89, 32, 235, 208, 25, + 76, 138, 1, 166, 1, 67, 210, 1, 83, 192, 2, 6, 86, 79, 87, 69, 76, 32, + 132, 166, 8, 6, 82, 69, 68, 85, 80, 76, 202, 202, 10, 72, 230, 168, 1, + 80, 170, 171, 3, 77, 191, 189, 1, 68, 52, 42, 79, 205, 1, 5, 65, 80, 73, + 84, 65, 8, 88, 10, 77, 66, 73, 78, 73, 78, 71, 32, 68, 79, 37, 8, 78, 83, + 79, 78, 65, 78, 84, 32, 4, 246, 135, 12, 85, 195, 241, 13, 84, 4, 190, + 136, 12, 78, 203, 145, 11, 71, 46, 36, 3, 77, 65, 76, 215, 250, 21, 85, + 44, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 44, 200, 1, 4, 79, 76, 68, + 32, 174, 137, 20, 78, 238, 231, 6, 66, 2, 67, 2, 68, 2, 70, 2, 71, 2, 72, + 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 88, 2, + 89, 187, 2, 65, 4, 150, 241, 26, 75, 3, 78, 12, 44, 5, 83, 73, 71, 78, + 32, 139, 177, 26, 76, 10, 226, 178, 26, 69, 162, 64, 65, 2, 73, 3, 79, + 242, 2, 112, 2, 65, 82, 110, 77, 50, 79, 204, 189, 23, 9, 82, 77, 65, 78, + 32, 80, 69, 78, 78, 218, 173, 2, 84, 239, 105, 78, 7, 29, 5, 32, 87, 73, + 84, 72, 4, 224, 147, 14, 5, 79, 85, 84, 32, 72, 149, 160, 9, 5, 32, 72, + 65, 78, 68, 4, 26, 32, 211, 244, 21, 73, 2, 251, 255, 22, 83, 226, 2, 64, + 6, 77, 69, 84, 82, 73, 67, 73, 6, 82, 71, 73, 65, 78, 32, 6, 236, 250, + 19, 4, 65, 76, 76, 89, 213, 222, 1, 5, 32, 80, 82, 79, 80, 220, 2, 228, + 1, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 172, 3, 7, 76, 69, + 84, 84, 69, 82, 32, 180, 2, 24, 77, 84, 65, 86, 82, 85, 76, 73, 32, 67, + 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 165, 6, 2, 80, + 65, 80, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 80, 142, 2, 65, 34, + 72, 166, 5, 67, 118, 71, 130, 1, 74, 34, 75, 82, 80, 34, 83, 94, 90, 220, + 180, 5, 2, 84, 65, 250, 142, 8, 76, 246, 179, 2, 82, 154, 160, 9, 66, 2, + 77, 2, 88, 202, 40, 78, 2, 81, 230, 33, 86, 234, 47, 68, 14, 69, 2, 73, + 2, 79, 2, 85, 2, 89, 143, 57, 87, 4, 162, 155, 26, 69, 227, 79, 78, 10, + 46, 65, 226, 211, 26, 73, 2, 79, 215, 22, 69, 4, 178, 234, 26, 69, 3, 82, + 94, 254, 1, 85, 178, 2, 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, + 75, 34, 76, 50, 80, 34, 83, 34, 84, 62, 90, 202, 247, 15, 82, 154, 160, + 9, 66, 2, 77, 2, 88, 202, 40, 78, 2, 81, 230, 33, 86, 234, 47, 68, 14, + 73, 2, 79, 2, 89, 142, 57, 87, 255, 2, 70, 4, 204, 248, 21, 4, 45, 66, + 82, 74, 203, 239, 4, 78, 92, 250, 1, 65, 42, 67, 74, 69, 46, 71, 34, 72, + 98, 74, 34, 75, 34, 76, 50, 80, 34, 83, 34, 84, 62, 90, 202, 247, 15, 82, + 154, 160, 9, 66, 2, 77, 2, 88, 202, 40, 78, 2, 81, 230, 33, 86, 234, 47, + 68, 14, 73, 2, 79, 2, 85, 2, 89, 142, 57, 87, 255, 2, 70, 6, 134, 150, + 26, 69, 2, 73, 227, 79, 78, 8, 38, 72, 230, 219, 25, 73, 215, 57, 65, 4, + 182, 149, 26, 73, 135, 23, 65, 4, 224, 136, 9, 2, 76, 73, 151, 220, 17, + 78, 4, 202, 154, 25, 72, 163, 122, 65, 12, 46, 65, 170, 205, 26, 73, 2, + 79, 215, 22, 69, 6, 26, 82, 227, 227, 26, 69, 5, 223, 220, 25, 68, 4, + 202, 153, 25, 72, 207, 64, 73, 4, 242, 193, 25, 72, 219, 81, 65, 4, 11, + 65, 4, 146, 210, 15, 66, 239, 144, 11, 83, 4, 162, 193, 25, 72, 223, 104, + 65, 4, 230, 226, 25, 72, 247, 47, 65, 6, 196, 183, 3, 6, 85, 82, 78, 69, + 68, 32, 211, 253, 1, 65, 4, 166, 192, 25, 72, 219, 81, 69, 2, 177, 166, + 20, 7, 82, 65, 71, 82, 65, 80, 72, 10, 48, 2, 77, 69, 20, 4, 78, 71, 69, + 82, 31, 82, 2, 179, 205, 25, 76, 2, 141, 184, 18, 2, 32, 82, 6, 38, 76, + 173, 235, 13, 3, 65, 70, 70, 5, 219, 204, 25, 83, 202, 1, 66, 65, 214, + 13, 79, 169, 144, 26, 7, 69, 73, 67, 72, 32, 83, 84, 194, 1, 84, 8, 71, + 79, 76, 73, 84, 73, 67, 32, 229, 12, 8, 83, 83, 32, 79, 70, 32, 77, 73, + 192, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 96, 45, 9, + 76, 32, 76, 69, 84, 84, 69, 82, 32, 96, 206, 1, 65, 22, 66, 42, 67, 94, + 68, 94, 70, 38, 71, 46, 73, 138, 2, 76, 58, 77, 66, 78, 34, 79, 30, 80, + 58, 82, 30, 83, 186, 1, 84, 110, 86, 22, 89, 90, 90, 226, 132, 19, 72, + 174, 128, 2, 85, 147, 174, 5, 75, 2, 163, 183, 26, 90, 4, 214, 3, 73, + 217, 198, 26, 2, 85, 75, 4, 48, 8, 65, 85, 68, 65, 84, 69, 32, 67, 15, + 72, 2, 11, 72, 2, 217, 250, 19, 2, 82, 73, 6, 42, 74, 30, 79, 221, 183, + 26, 2, 90, 69, 2, 145, 250, 19, 2, 69, 82, 2, 147, 235, 24, 66, 4, 186, + 138, 21, 82, 147, 171, 5, 73, 2, 21, 3, 76, 65, 71, 2, 207, 222, 21, 79, + 13, 38, 78, 54, 79, 141, 1, 2, 90, 72, 2, 145, 172, 26, 8, 73, 84, 73, + 65, 76, 32, 73, 90, 4, 33, 6, 84, 65, 84, 69, 68, 32, 4, 26, 66, 25, 2, + 83, 77, 2, 11, 73, 2, 35, 71, 2, 21, 3, 65, 76, 76, 2, 253, 209, 24, 2, + 32, 89, 4, 142, 213, 26, 73, 211, 2, 69, 4, 52, 9, 65, 84, 73, 78, 65, + 84, 69, 32, 77, 35, 74, 2, 173, 133, 11, 3, 89, 83, 76, 2, 249, 140, 9, + 3, 85, 68, 73, 2, 11, 65, 2, 187, 135, 21, 83, 4, 190, 177, 26, 78, 3, + 84, 4, 26, 79, 243, 213, 26, 69, 2, 133, 191, 22, 2, 75, 79, 2, 177, 235, + 21, 2, 73, 84, 14, 106, 72, 58, 76, 172, 240, 13, 6, 80, 73, 68, 69, 82, + 89, 201, 175, 10, 8, 77, 65, 76, 76, 32, 89, 85, 83, 6, 32, 2, 84, 65, + 171, 212, 26, 65, 5, 159, 172, 25, 80, 2, 239, 246, 19, 79, 6, 78, 86, + 144, 147, 22, 9, 82, 79, 75, 85, 84, 65, 83, 84, 73, 203, 172, 4, 83, 2, + 129, 162, 19, 2, 82, 73, 2, 135, 195, 24, 69, 12, 50, 69, 214, 244, 18, + 65, 250, 221, 7, 79, 3, 85, 6, 250, 130, 21, 83, 155, 143, 5, 82, 4, 156, + 145, 9, 3, 69, 77, 76, 145, 250, 15, 4, 72, 73, 86, 69, 2, 207, 206, 26, + 76, 6, 228, 238, 11, 13, 66, 69, 32, 87, 73, 84, 72, 32, 77, 69, 82, 73, + 68, 250, 198, 4, 87, 255, 135, 9, 86, 68, 162, 1, 65, 44, 12, 84, 72, 73, + 67, 32, 76, 69, 84, 84, 69, 82, 32, 230, 161, 18, 82, 206, 102, 71, 172, + 131, 5, 3, 78, 71, 71, 182, 186, 1, 79, 161, 75, 2, 76, 70, 4, 144, 183, + 15, 2, 76, 32, 183, 152, 11, 84, 54, 206, 2, 65, 50, 72, 46, 73, 46, 78, + 46, 80, 2, 81, 40, 2, 82, 65, 22, 84, 200, 137, 9, 2, 87, 73, 230, 141, + 12, 85, 188, 69, 3, 70, 65, 73, 204, 7, 4, 66, 65, 73, 82, 128, 22, 2, + 79, 84, 150, 30, 68, 158, 123, 77, 164, 131, 1, 3, 83, 65, 85, 162, 16, + 69, 184, 29, 3, 76, 65, 71, 146, 200, 1, 74, 196, 16, 2, 71, 73, 141, 12, + 2, 75, 85, 4, 204, 198, 24, 3, 73, 72, 86, 187, 131, 2, 72, 4, 214, 224, + 13, 87, 145, 226, 11, 2, 65, 71, 4, 230, 138, 9, 85, 217, 229, 14, 2, 71, + 71, 6, 154, 206, 15, 73, 253, 252, 8, 2, 65, 85, 2, 213, 134, 26, 5, 65, + 73, 82, 84, 72, 2, 235, 165, 26, 73, 4, 192, 169, 23, 2, 72, 73, 233, 69, + 2, 69, 73, 234, 9, 46, 65, 198, 5, 69, 206, 82, 73, 151, 3, 79, 152, 1, + 96, 7, 68, 85, 65, 84, 73, 79, 78, 32, 5, 78, 84, 72, 65, 32, 242, 179, + 20, 86, 139, 129, 5, 80, 2, 11, 32, 2, 151, 184, 25, 67, 146, 1, 120, 7, + 76, 69, 84, 84, 69, 82, 32, 212, 2, 5, 83, 73, 71, 78, 32, 226, 231, 22, + 65, 248, 8, 2, 86, 79, 239, 195, 3, 79, 100, 214, 1, 86, 250, 235, 22, + 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, + 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, 79, 162, + 8, 69, 158, 20, 72, 2, 77, 2, 82, 3, 89, 14, 60, 5, 69, 68, 73, 67, 32, + 210, 242, 22, 79, 139, 211, 3, 65, 4, 244, 143, 24, 6, 68, 79, 85, 66, + 76, 69, 235, 240, 1, 65, 16, 66, 67, 250, 176, 22, 78, 242, 60, 65, 182, + 1, 80, 179, 146, 3, 86, 4, 226, 238, 7, 79, 195, 193, 14, 65, 192, 8, 76, + 10, 65, 84, 69, 82, 45, 84, 72, 65, 78, 32, 206, 7, 69, 219, 180, 25, 89, + 56, 134, 1, 65, 150, 3, 66, 62, 79, 216, 2, 11, 69, 81, 85, 65, 76, 32, + 84, 79, 32, 79, 82, 222, 205, 6, 67, 138, 4, 87, 231, 227, 18, 83, 16, + 44, 5, 66, 79, 86, 69, 32, 187, 210, 6, 78, 12, 150, 1, 83, 180, 1, 19, + 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, 69, 81, 85, 65, 76, 32, + 65, 224, 204, 6, 4, 76, 69, 83, 83, 135, 211, 18, 82, 6, 148, 1, 7, 73, + 77, 73, 76, 65, 82, 32, 225, 207, 6, 23, 76, 65, 78, 84, 69, 68, 32, 69, + 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, 32, 76, 69, 83, 83, 4, 26, 65, + 183, 207, 6, 79, 2, 65, 3, 66, 79, 86, 6, 40, 4, 69, 83, 73, 68, 159, + 208, 6, 85, 2, 231, 2, 69, 20, 40, 2, 82, 32, 245, 1, 3, 86, 69, 82, 16, + 120, 16, 83, 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 84, 79, + 230, 209, 6, 65, 138, 247, 12, 69, 131, 227, 4, 76, 9, 49, 10, 32, 87, + 73, 84, 72, 32, 68, 79, 84, 32, 6, 44, 5, 65, 66, 79, 86, 69, 163, 153, + 18, 73, 5, 195, 138, 26, 32, 4, 52, 7, 76, 65, 80, 80, 73, 78, 71, 227, + 232, 19, 32, 2, 185, 170, 24, 2, 32, 76, 134, 8, 36, 2, 75, 32, 185, 73, + 2, 78, 32, 254, 7, 130, 3, 65, 216, 15, 8, 67, 65, 80, 73, 84, 65, 76, + 32, 182, 11, 68, 134, 1, 70, 68, 2, 73, 78, 222, 3, 75, 138, 1, 76, 174, + 3, 78, 66, 77, 84, 3, 88, 69, 83, 22, 79, 202, 1, 80, 90, 82, 182, 1, 83, + 130, 22, 84, 200, 2, 13, 85, 80, 83, 73, 76, 79, 78, 32, 87, 73, 84, 72, + 32, 150, 1, 86, 142, 2, 89, 178, 225, 7, 66, 248, 175, 3, 4, 71, 82, 65, + 77, 204, 23, 2, 90, 69, 147, 249, 11, 81, 112, 92, 10, 67, 82, 79, 80, + 72, 79, 78, 73, 67, 32, 172, 14, 6, 78, 79, 32, 84, 69, 76, 23, 82, 106, + 188, 2, 6, 65, 84, 84, 73, 67, 32, 222, 5, 67, 92, 3, 78, 65, 88, 32, 12, + 68, 69, 76, 80, 72, 73, 67, 32, 70, 73, 86, 69, 0, 14, 83, 84, 82, 65, + 84, 73, 65, 78, 32, 70, 73, 70, 84, 89, 40, 11, 69, 80, 73, 68, 65, 85, + 82, 69, 65, 78, 32, 112, 3, 72, 69, 82, 164, 1, 9, 77, 69, 83, 83, 69, + 78, 73, 65, 78, 35, 84, 48, 72, 2, 70, 73, 180, 2, 4, 79, 78, 69, 32, + 205, 1, 4, 84, 69, 78, 32, 26, 36, 3, 70, 84, 89, 105, 2, 86, 69, 11, 11, + 32, 8, 22, 84, 171, 4, 83, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 215, 3, + 65, 5, 231, 3, 32, 17, 11, 32, 14, 56, 7, 72, 85, 78, 68, 82, 69, 68, 18, + 84, 143, 3, 83, 7, 131, 2, 32, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 187, + 2, 65, 5, 213, 1, 2, 32, 84, 14, 98, 72, 48, 7, 84, 72, 79, 85, 83, 65, + 78, 174, 174, 24, 81, 241, 225, 1, 5, 68, 82, 65, 67, 72, 6, 44, 5, 85, + 78, 68, 82, 69, 143, 174, 24, 65, 4, 17, 2, 68, 32, 4, 22, 84, 131, 1, + 83, 2, 95, 65, 8, 30, 84, 86, 83, 175, 1, 77, 4, 50, 65, 21, 8, 72, 79, + 85, 83, 65, 78, 68, 32, 2, 187, 174, 16, 76, 2, 11, 83, 2, 145, 175, 24, + 2, 84, 65, 4, 88, 5, 65, 82, 89, 83, 84, 145, 1, 12, 89, 82, 69, 78, 65, + 73, 67, 32, 84, 87, 79, 32, 2, 101, 5, 73, 65, 78, 32, 70, 2, 17, 2, 32, + 77, 2, 199, 144, 13, 78, 6, 30, 70, 29, 3, 84, 87, 79, 2, 177, 178, 15, + 2, 73, 86, 5, 11, 32, 2, 233, 144, 10, 5, 68, 82, 65, 67, 72, 8, 112, 8, + 77, 73, 79, 78, 73, 65, 78, 32, 189, 135, 25, 14, 65, 69, 85, 77, 32, 79, + 78, 69, 32, 80, 76, 69, 84, 72, 6, 238, 185, 12, 70, 134, 159, 12, 84, + 191, 58, 79, 2, 11, 32, 2, 183, 216, 24, 84, 32, 92, 8, 72, 69, 83, 80, + 73, 65, 78, 32, 129, 1, 10, 82, 79, 69, 90, 69, 78, 73, 65, 78, 32, 20, + 40, 2, 70, 73, 38, 84, 135, 151, 18, 79, 6, 162, 215, 20, 86, 231, 230, + 3, 70, 8, 202, 174, 15, 72, 178, 172, 10, 69, 239, 48, 87, 12, 36, 2, 70, + 73, 209, 24, 2, 84, 69, 8, 142, 176, 18, 86, 205, 250, 6, 3, 70, 84, 89, + 2, 159, 132, 10, 69, 4, 240, 215, 22, 2, 79, 85, 161, 181, 1, 3, 84, 65, + 66, 154, 2, 66, 76, 174, 45, 82, 66, 68, 220, 226, 7, 2, 75, 65, 135, 6, + 84, 144, 2, 44, 6, 69, 84, 84, 69, 82, 32, 239, 45, 85, 142, 2, 198, 2, + 65, 190, 1, 69, 28, 4, 73, 79, 84, 65, 128, 1, 2, 79, 77, 156, 3, 3, 82, + 72, 79, 46, 83, 48, 7, 85, 80, 83, 73, 76, 79, 78, 146, 33, 80, 170, 2, + 84, 202, 183, 5, 68, 144, 176, 2, 2, 75, 65, 166, 192, 1, 71, 190, 132, + 11, 67, 194, 149, 2, 66, 2, 72, 2, 90, 166, 1, 76, 230, 231, 2, 89, 210, + 43, 77, 2, 78, 147, 17, 88, 48, 68, 4, 76, 80, 72, 65, 213, 28, 8, 82, + 67, 72, 65, 73, 67, 32, 83, 47, 33, 6, 32, 87, 73, 84, 72, 32, 44, 242, + 2, 68, 30, 80, 226, 29, 86, 226, 5, 79, 246, 236, 3, 84, 239, 167, 5, 77, + 62, 186, 1, 84, 131, 27, 80, 31, 33, 6, 32, 87, 73, 84, 72, 32, 28, 186, + 5, 68, 136, 25, 2, 80, 83, 158, 1, 86, 226, 5, 79, 246, 236, 3, 84, 239, + 167, 5, 77, 62, 28, 2, 69, 71, 235, 34, 73, 42, 11, 65, 43, 33, 6, 32, + 87, 73, 84, 72, 32, 40, 54, 68, 30, 80, 194, 35, 79, 22, 86, 227, 236, 3, + 84, 16, 65, 4, 65, 83, 73, 65, 18, 36, 4, 83, 73, 76, 73, 211, 16, 82, + 17, 29, 5, 32, 65, 78, 68, 32, 14, 44, 2, 79, 88, 0, 3, 86, 65, 82, 23, + 80, 4, 81, 2, 73, 65, 6, 60, 10, 69, 82, 73, 83, 80, 79, 77, 69, 78, 73, + 175, 15, 82, 5, 169, 15, 7, 32, 65, 78, 68, 32, 80, 82, 5, 161, 35, 7, + 32, 87, 73, 84, 72, 32, 68, 6, 222, 140, 8, 73, 226, 194, 17, 65, 239, + 48, 72, 23, 33, 6, 32, 87, 73, 84, 72, 32, 20, 66, 68, 166, 26, 86, 226, + 5, 79, 246, 236, 3, 84, 239, 167, 5, 77, 10, 130, 24, 65, 181, 176, 22, + 5, 73, 65, 76, 89, 84, 18, 76, 9, 73, 65, 76, 89, 84, 73, 75, 65, 32, 32, + 3, 82, 65, 67, 227, 22, 65, 8, 174, 24, 65, 199, 243, 3, 84, 2, 243, 186, + 11, 72, 4, 40, 3, 73, 86, 69, 1, 3, 79, 85, 82, 2, 205, 37, 2, 32, 79, + 76, 144, 1, 27, 83, 84, 82, 85, 77, 69, 78, 84, 65, 76, 32, 78, 79, 84, + 65, 84, 73, 79, 78, 32, 83, 89, 77, 66, 79, 76, 45, 161, 154, 22, 2, 68, + 73, 74, 70, 49, 70, 50, 62, 51, 62, 52, 170, 37, 53, 206, 243, 25, 55, 3, + 56, 17, 174, 154, 26, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, + 234, 153, 26, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 174, 153, 26, + 48, 2, 50, 2, 54, 2, 55, 2, 56, 3, 57, 17, 242, 152, 26, 48, 2, 50, 2, + 51, 2, 53, 2, 55, 2, 56, 3, 57, 8, 54, 65, 38, 79, 169, 32, 6, 89, 65, + 84, 72, 79, 83, 4, 134, 255, 7, 80, 203, 133, 17, 73, 2, 11, 82, 2, 11, + 79, 2, 135, 244, 23, 78, 32, 128, 1, 6, 69, 84, 84, 69, 82, 32, 168, 2, + 6, 79, 87, 69, 82, 32, 78, 32, 6, 85, 78, 65, 84, 69, 32, 145, 194, 22, + 2, 73, 84, 24, 94, 83, 140, 13, 9, 65, 82, 67, 72, 65, 73, 67, 32, 75, 2, + 75, 182, 11, 68, 207, 172, 25, 89, 16, 88, 13, 77, 65, 76, 76, 32, 67, + 65, 80, 73, 84, 65, 76, 32, 210, 12, 65, 247, 243, 7, 84, 12, 74, 80, + 226, 192, 9, 71, 194, 156, 9, 79, 218, 206, 2, 82, 139, 176, 1, 76, 4, + 194, 128, 26, 83, 219, 19, 73, 2, 221, 134, 10, 3, 85, 77, 69, 4, 222, + 25, 83, 215, 224, 7, 69, 4, 80, 4, 69, 84, 82, 69, 241, 192, 23, 10, 85, + 83, 73, 67, 65, 76, 32, 76, 69, 73, 2, 163, 212, 2, 84, 12, 88, 3, 78, + 69, 32, 198, 236, 9, 88, 148, 155, 5, 3, 85, 78, 75, 253, 198, 5, 2, 66, + 79, 6, 64, 8, 72, 65, 76, 70, 32, 83, 73, 71, 21, 4, 81, 85, 65, 82, 4, + 139, 146, 25, 78, 2, 255, 206, 20, 84, 16, 62, 82, 206, 11, 83, 114, 69, + 154, 236, 7, 72, 211, 132, 17, 73, 2, 217, 29, 2, 79, 83, 6, 100, 3, 72, + 79, 32, 165, 246, 7, 16, 69, 86, 69, 82, 83, 69, 68, 32, 76, 85, 78, 65, + 84, 69, 32, 69, 4, 180, 243, 13, 10, 87, 73, 84, 72, 32, 83, 84, 82, 79, + 75, 135, 137, 11, 83, 226, 2, 220, 1, 5, 77, 65, 76, 76, 32, 192, 19, 22, + 85, 66, 83, 67, 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, + 84, 69, 82, 32, 72, 10, 89, 77, 66, 79, 76, 32, 84, 65, 85, 32, 189, 144, + 24, 6, 73, 78, 85, 83, 79, 73, 212, 2, 56, 7, 76, 69, 84, 84, 69, 82, 32, + 202, 17, 82, 67, 68, 206, 2, 178, 2, 65, 162, 2, 68, 38, 69, 52, 4, 73, + 79, 84, 65, 0, 7, 85, 80, 83, 73, 76, 79, 78, 254, 3, 75, 28, 2, 79, 77, + 182, 5, 80, 112, 3, 82, 72, 79, 94, 83, 94, 84, 244, 230, 7, 2, 70, 73, + 138, 193, 1, 71, 190, 132, 11, 67, 194, 149, 2, 66, 2, 72, 2, 90, 166, 1, + 76, 182, 147, 3, 77, 2, 78, 147, 17, 88, 58, 64, 4, 76, 80, 72, 65, 149, + 1, 7, 82, 67, 72, 65, 73, 67, 32, 55, 33, 6, 32, 87, 73, 84, 72, 32, 52, + 82, 86, 226, 6, 68, 30, 80, 114, 79, 142, 1, 89, 226, 238, 3, 84, 239, + 167, 5, 77, 6, 154, 5, 82, 155, 3, 65, 4, 18, 75, 23, 83, 2, 215, 244, 7, + 79, 2, 11, 65, 2, 203, 189, 13, 77, 4, 138, 223, 7, 73, 235, 239, 14, 69, + 70, 22, 80, 215, 4, 84, 20, 249, 7, 3, 83, 73, 76, 41, 33, 6, 32, 87, 73, + 84, 72, 32, 38, 78, 68, 166, 1, 80, 178, 1, 86, 226, 5, 79, 246, 236, 3, + 84, 239, 167, 5, 77, 18, 50, 65, 29, 8, 73, 65, 76, 89, 84, 73, 75, 65, + 8, 153, 1, 3, 83, 73, 65, 11, 29, 5, 32, 65, 78, 68, 32, 8, 170, 1, 80, + 154, 6, 79, 22, 86, 227, 236, 3, 84, 10, 18, 83, 115, 69, 8, 21, 3, 73, + 76, 73, 9, 17, 2, 32, 65, 6, 21, 3, 78, 68, 32, 6, 30, 80, 154, 6, 79, + 23, 86, 2, 11, 69, 2, 11, 82, 2, 181, 17, 4, 73, 83, 80, 79, 4, 22, 82, + 147, 15, 65, 2, 133, 226, 25, 2, 65, 67, 4, 206, 239, 7, 65, 3, 79, 70, + 28, 2, 69, 71, 151, 3, 73, 50, 11, 65, 51, 33, 6, 32, 87, 73, 84, 72, 32, + 48, 58, 68, 30, 80, 114, 79, 62, 86, 82, 89, 227, 238, 3, 84, 16, 61, 4, + 65, 83, 73, 65, 20, 32, 4, 83, 73, 76, 73, 91, 69, 17, 29, 5, 32, 65, 78, + 68, 32, 14, 42, 79, 12, 2, 80, 69, 50, 86, 83, 89, 4, 83, 88, 4, 89, 9, + 82, 73, 83, 80, 79, 77, 69, 78, 73, 4, 11, 65, 4, 11, 82, 4, 17, 2, 73, + 65, 5, 33, 6, 32, 65, 78, 68, 32, 89, 2, 243, 12, 80, 20, 17, 2, 67, 82, + 20, 17, 2, 79, 78, 21, 33, 6, 32, 87, 73, 84, 72, 32, 18, 88, 5, 68, 65, + 83, 73, 65, 0, 5, 80, 83, 73, 76, 73, 54, 79, 22, 86, 227, 236, 3, 84, 7, + 29, 5, 32, 65, 78, 68, 32, 4, 18, 79, 23, 86, 2, 207, 216, 9, 88, 2, 179, + 9, 65, 8, 88, 11, 65, 77, 80, 72, 89, 76, 73, 65, 78, 32, 68, 174, 233, + 25, 72, 2, 83, 219, 19, 73, 2, 139, 212, 7, 73, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 34, 68, 145, 129, 21, 2, 80, 83, 2, 243, 197, 8, 65, 10, 54, + 65, 186, 231, 7, 84, 230, 1, 73, 207, 243, 17, 72, 4, 142, 177, 13, 77, + 207, 202, 12, 78, 4, 246, 193, 22, 72, 219, 148, 3, 65, 4, 41, 8, 69, 86, + 69, 82, 83, 69, 68, 32, 4, 18, 68, 43, 76, 2, 37, 7, 79, 84, 84, 69, 68, + 32, 76, 2, 11, 85, 2, 33, 6, 78, 65, 84, 69, 32, 83, 2, 169, 232, 7, 3, + 73, 71, 77, 10, 158, 166, 9, 71, 190, 132, 11, 67, 2, 80, 222, 102, 82, + 231, 174, 1, 66, 2, 239, 144, 21, 82, 16, 106, 72, 104, 7, 82, 89, 66, + 76, 73, 79, 78, 44, 3, 87, 79, 32, 254, 229, 3, 79, 233, 196, 16, 2, 65, + 76, 6, 40, 4, 82, 69, 69, 32, 143, 230, 7, 69, 4, 146, 1, 79, 157, 200, + 22, 7, 81, 85, 65, 82, 84, 69, 82, 2, 21, 3, 32, 66, 65, 2, 231, 218, 23, + 83, 4, 42, 79, 221, 152, 12, 4, 84, 72, 73, 82, 2, 237, 224, 19, 2, 66, + 79, 6, 80, 5, 65, 67, 85, 84, 69, 0, 9, 68, 73, 65, 69, 82, 69, 83, 73, + 83, 39, 72, 2, 33, 6, 32, 65, 78, 68, 32, 72, 2, 197, 195, 7, 2, 79, 79, + 60, 102, 65, 21, 21, 79, 67, 65, 76, 32, 78, 79, 84, 65, 84, 73, 79, 78, + 32, 83, 89, 77, 66, 79, 76, 45, 2, 135, 207, 9, 82, 58, 90, 50, 2, 53, + 154, 179, 23, 49, 182, 192, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 13, + 202, 243, 25, 48, 2, 49, 2, 50, 2, 51, 3, 52, 4, 26, 80, 207, 177, 20, + 69, 2, 11, 79, 2, 33, 6, 71, 69, 71, 82, 65, 77, 2, 197, 246, 20, 2, 77, + 69, 8, 170, 237, 13, 65, 178, 177, 10, 66, 174, 76, 72, 253, 64, 3, 83, + 65, 76, 12, 60, 6, 78, 78, 73, 78, 71, 32, 137, 233, 24, 3, 77, 65, 67, + 10, 100, 4, 70, 65, 67, 69, 149, 228, 17, 15, 67, 65, 84, 32, 70, 65, 67, + 69, 32, 87, 73, 84, 72, 32, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 108, + 23, 79, 78, 69, 32, 76, 65, 82, 71, 69, 32, 65, 78, 68, 32, 79, 78, 69, + 32, 83, 77, 65, 76, 76, 35, 83, 2, 11, 32, 2, 191, 157, 8, 69, 4, 32, 2, + 84, 65, 203, 226, 17, 77, 2, 187, 243, 22, 82, 6, 28, 3, 85, 80, 32, 39, + 87, 4, 214, 192, 22, 83, 179, 236, 2, 77, 2, 227, 144, 18, 73, 220, 4, + 136, 1, 2, 65, 82, 70, 73, 52, 7, 74, 65, 82, 65, 84, 73, 32, 208, 6, 12, + 78, 74, 65, 76, 65, 32, 71, 79, 78, 68, 73, 32, 143, 3, 82, 4, 34, 65, + 245, 246, 20, 2, 68, 83, 2, 11, 78, 2, 187, 229, 24, 73, 4, 238, 202, 24, + 84, 217, 160, 1, 4, 68, 69, 32, 68, 182, 1, 168, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 220, 1, 5, 83, 73, 71, 78, 32, 160, 2, 6, 86, 79, 87, 69, 76, + 32, 238, 251, 19, 65, 154, 24, 82, 166, 225, 3, 68, 203, 224, 1, 79, 98, + 162, 144, 22, 65, 38, 68, 114, 84, 46, 86, 186, 5, 85, 206, 201, 1, 73, + 42, 76, 246, 14, 90, 134, 178, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, + 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, + 24, 98, 67, 28, 3, 77, 65, 68, 22, 84, 138, 202, 3, 83, 238, 137, 18, 78, + 242, 60, 65, 231, 147, 3, 86, 4, 118, 73, 219, 210, 21, 65, 2, 231, 135, + 19, 68, 4, 68, 5, 87, 79, 45, 67, 73, 29, 8, 72, 82, 69, 69, 45, 68, 79, + 84, 2, 25, 4, 82, 67, 76, 69, 2, 253, 195, 20, 5, 32, 78, 85, 75, 84, 34, + 36, 5, 83, 73, 71, 78, 32, 87, 67, 30, 82, 67, 130, 146, 22, 65, 38, 85, + 22, 86, 186, 201, 1, 73, 222, 137, 2, 69, 3, 79, 4, 245, 210, 21, 5, 65, + 78, 68, 82, 65, 126, 108, 7, 76, 69, 84, 84, 69, 82, 32, 216, 1, 5, 83, + 73, 71, 78, 32, 38, 86, 154, 241, 23, 68, 203, 224, 1, 79, 80, 130, 243, + 21, 78, 146, 23, 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 42, + 76, 246, 193, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, 79, 162, + 8, 69, 158, 20, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 89, 4, 178, 172, 23, + 86, 179, 241, 1, 65, 20, 190, 7, 79, 163, 162, 23, 73, 160, 2, 84, 6, 77, + 85, 75, 72, 73, 32, 189, 7, 10, 85, 78, 71, 32, 75, 72, 69, 77, 65, 32, + 172, 1, 194, 1, 65, 44, 7, 76, 69, 84, 84, 69, 82, 32, 238, 1, 83, 228, + 2, 2, 86, 79, 172, 144, 13, 3, 84, 73, 80, 130, 237, 5, 73, 216, 170, 2, + 5, 69, 75, 32, 79, 78, 214, 193, 2, 68, 227, 172, 1, 85, 4, 218, 200, 21, + 66, 213, 225, 1, 2, 68, 68, 96, 162, 194, 8, 71, 2, 75, 134, 195, 13, 65, + 38, 68, 82, 82, 34, 84, 230, 5, 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, + 78, 126, 66, 2, 67, 2, 74, 2, 80, 2, 83, 206, 40, 79, 162, 8, 69, 158, + 20, 70, 2, 72, 2, 77, 2, 86, 2, 89, 3, 90, 26, 108, 19, 69, 81, 85, 69, + 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 93, 4, 73, + 71, 78, 32, 12, 70, 71, 2, 75, 230, 225, 23, 83, 170, 216, 1, 76, 226, + 31, 70, 3, 90, 2, 227, 225, 23, 72, 14, 128, 1, 6, 65, 68, 65, 75, 32, + 66, 2, 66, 176, 140, 15, 2, 85, 68, 254, 186, 6, 78, 180, 171, 2, 3, 89, + 65, 75, 163, 165, 1, 86, 2, 255, 147, 8, 73, 18, 45, 9, 87, 69, 76, 32, + 83, 73, 71, 78, 32, 18, 250, 134, 22, 65, 38, 85, 206, 201, 1, 73, 234, + 234, 1, 79, 163, 8, 69, 116, 216, 1, 22, 67, 79, 78, 83, 79, 78, 65, 78, + 84, 32, 83, 73, 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 44, 7, 76, 69, + 84, 84, 69, 82, 32, 168, 1, 5, 83, 73, 71, 78, 32, 56, 6, 86, 79, 87, 69, + 76, 32, 251, 228, 23, 68, 8, 234, 213, 25, 72, 2, 82, 2, 86, 3, 89, 60, + 198, 230, 21, 78, 182, 23, 68, 114, 84, 206, 145, 3, 66, 2, 67, 2, 71, 2, + 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, 89, + 187, 2, 65, 4, 214, 145, 25, 65, 233, 35, 6, 84, 72, 79, 76, 72, 79, 24, + 130, 148, 20, 83, 155, 128, 5, 76, 162, 23, 110, 65, 174, 95, 69, 246, + 103, 73, 146, 9, 79, 158, 12, 84, 30, 85, 130, 1, 89, 249, 236, 12, 4, + 82, 89, 86, 78, 138, 11, 236, 1, 2, 73, 82, 60, 8, 76, 70, 87, 73, 68, + 84, 72, 32, 242, 10, 77, 210, 1, 78, 236, 76, 21, 80, 80, 89, 32, 80, 69, + 82, 83, 79, 78, 32, 82, 65, 73, 83, 73, 78, 71, 32, 79, 78, 22, 82, 38, + 84, 252, 242, 17, 2, 85, 77, 251, 241, 5, 68, 6, 26, 32, 163, 217, 24, + 67, 4, 242, 162, 24, 80, 211, 113, 83, 244, 1, 140, 2, 7, 72, 65, 78, 71, + 85, 76, 32, 216, 4, 8, 75, 65, 84, 65, 75, 65, 78, 65, 204, 3, 3, 76, 69, + 70, 0, 4, 82, 73, 71, 72, 140, 190, 6, 11, 70, 79, 82, 77, 83, 32, 76, + 73, 71, 72, 84, 142, 129, 4, 85, 174, 218, 2, 73, 250, 184, 4, 66, 242, + 227, 4, 87, 211, 35, 68, 104, 52, 7, 76, 69, 84, 84, 69, 82, 32, 167, + 217, 10, 70, 102, 206, 1, 75, 28, 5, 78, 73, 69, 85, 78, 42, 80, 24, 5, + 82, 73, 69, 85, 76, 86, 83, 98, 89, 202, 61, 67, 54, 69, 30, 73, 242, 4, + 77, 138, 1, 84, 206, 3, 87, 198, 1, 72, 230, 194, 24, 65, 2, 79, 163, 64, + 85, 6, 222, 65, 72, 155, 3, 73, 7, 11, 45, 4, 134, 72, 67, 131, 3, 72, 6, + 146, 69, 72, 35, 73, 17, 11, 45, 14, 206, 49, 84, 226, 14, 80, 130, 4, + 77, 194, 3, 75, 218, 2, 72, 99, 83, 12, 40, 4, 83, 65, 78, 71, 151, 221, + 13, 73, 10, 210, 70, 67, 42, 75, 74, 80, 34, 84, 211, 2, 83, 14, 194, + 134, 23, 69, 194, 133, 2, 65, 162, 64, 73, 2, 79, 3, 85, 118, 70, 32, + 193, 240, 2, 11, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 80, 116, 76, 7, + 76, 69, 84, 84, 69, 82, 32, 150, 240, 2, 83, 34, 86, 215, 200, 21, 77, + 110, 146, 1, 83, 138, 234, 2, 78, 150, 2, 72, 2, 75, 2, 77, 2, 82, 2, 84, + 170, 1, 89, 222, 41, 87, 154, 178, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, + 28, 76, 5, 77, 65, 76, 76, 32, 234, 200, 25, 65, 2, 69, 2, 73, 2, 79, 3, + 85, 18, 242, 236, 2, 89, 142, 183, 22, 84, 234, 36, 65, 2, 69, 2, 73, 2, + 79, 3, 85, 4, 11, 84, 4, 212, 157, 13, 2, 32, 67, 239, 138, 11, 87, 14, + 56, 3, 77, 69, 82, 106, 83, 233, 248, 21, 3, 66, 85, 82, 9, 29, 5, 32, + 65, 78, 68, 32, 6, 168, 170, 12, 3, 87, 82, 69, 144, 228, 3, 2, 83, 73, + 207, 136, 8, 80, 4, 152, 137, 25, 3, 84, 69, 82, 163, 61, 65, 194, 8, + 118, 68, 202, 2, 71, 128, 66, 13, 73, 70, 73, 32, 82, 79, 72, 73, 78, 71, + 89, 65, 32, 165, 5, 5, 85, 78, 79, 79, 32, 10, 100, 12, 32, 87, 73, 84, + 72, 32, 73, 78, 68, 69, 88, 32, 188, 1, 2, 66, 65, 193, 218, 21, 2, 83, + 72, 4, 156, 1, 18, 65, 78, 68, 32, 77, 73, 68, 68, 76, 69, 32, 70, 73, + 78, 71, 69, 82, 83, 1, 16, 70, 73, 78, 71, 69, 82, 32, 65, 78, 68, 32, + 84, 72, 85, 77, 66, 2, 233, 134, 16, 2, 32, 67, 4, 186, 185, 24, 76, 183, + 137, 1, 71, 170, 7, 84, 3, 85, 76, 32, 217, 64, 13, 90, 72, 79, 85, 32, + 78, 85, 77, 69, 82, 65, 76, 32, 146, 7, 164, 1, 9, 67, 72, 79, 83, 69, + 79, 78, 71, 32, 244, 15, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 46, 74, + 180, 31, 7, 76, 69, 84, 84, 69, 82, 32, 147, 154, 10, 70, 250, 1, 246, 1, + 67, 172, 2, 5, 73, 69, 85, 78, 71, 146, 1, 75, 132, 1, 5, 77, 73, 69, 85, + 77, 56, 5, 78, 73, 69, 85, 78, 74, 80, 172, 2, 5, 82, 73, 69, 85, 76, + 210, 1, 83, 166, 3, 84, 124, 2, 89, 69, 200, 40, 5, 72, 73, 69, 85, 72, + 199, 145, 10, 70, 30, 76, 2, 72, 73, 84, 7, 69, 79, 78, 71, 67, 72, 73, + 121, 4, 73, 69, 85, 67, 16, 40, 4, 69, 85, 67, 72, 41, 2, 84, 85, 7, 11, + 45, 4, 202, 31, 75, 243, 26, 72, 10, 21, 3, 69, 85, 77, 10, 22, 83, 155, + 46, 67, 6, 40, 4, 83, 65, 78, 71, 135, 205, 13, 73, 4, 194, 54, 67, 227, + 3, 83, 5, 215, 30, 45, 27, 11, 45, 24, 90, 80, 234, 30, 82, 242, 13, 67, + 194, 5, 77, 138, 1, 84, 186, 2, 75, 218, 2, 72, 99, 83, 6, 214, 50, 72, + 214, 3, 73, 207, 2, 65, 16, 80, 7, 65, 80, 89, 69, 79, 85, 78, 228, 20, + 5, 73, 89, 69, 79, 75, 131, 25, 72, 10, 234, 29, 82, 178, 15, 80, 30, 83, + 231, 3, 77, 11, 11, 45, 8, 158, 52, 75, 74, 80, 34, 84, 211, 2, 83, 15, + 11, 45, 12, 190, 51, 67, 42, 75, 74, 80, 34, 84, 242, 1, 72, 99, 83, 42, + 68, 6, 72, 73, 69, 85, 80, 72, 40, 4, 73, 69, 85, 80, 223, 53, 65, 7, 11, + 45, 4, 158, 51, 80, 147, 2, 72, 35, 11, 45, 32, 82, 83, 234, 20, 80, 214, + 7, 75, 162, 12, 67, 202, 6, 84, 226, 2, 78, 179, 2, 72, 14, 32, 3, 73, + 79, 83, 251, 3, 83, 13, 11, 45, 10, 242, 46, 84, 146, 2, 67, 42, 75, 75, + 80, 29, 11, 45, 26, 78, 75, 42, 83, 190, 44, 77, 154, 3, 67, 82, 78, 34, + 80, 34, 84, 243, 1, 72, 6, 170, 22, 65, 130, 19, 72, 135, 7, 73, 8, 40, + 4, 83, 65, 78, 71, 235, 197, 13, 73, 6, 206, 47, 75, 74, 80, 35, 84, 58, + 48, 3, 73, 79, 83, 217, 1, 4, 83, 65, 78, 71, 33, 11, 45, 30, 130, 1, 80, + 44, 2, 83, 83, 210, 22, 82, 210, 1, 75, 162, 12, 67, 194, 5, 77, 138, 1, + 84, 226, 2, 78, 178, 2, 72, 191, 157, 18, 73, 6, 184, 23, 4, 73, 69, 85, + 80, 179, 19, 72, 2, 233, 48, 3, 65, 78, 71, 26, 236, 17, 5, 67, 73, 69, + 85, 67, 172, 3, 4, 83, 73, 79, 83, 154, 1, 82, 186, 20, 84, 54, 89, 134, + 2, 75, 42, 78, 34, 80, 146, 2, 72, 191, 157, 18, 73, 16, 40, 5, 73, 75, + 69, 85, 84, 195, 41, 72, 15, 11, 45, 12, 226, 20, 82, 178, 19, 77, 154, + 3, 67, 42, 75, 74, 80, 243, 2, 83, 4, 150, 19, 83, 139, 22, 79, 2, 169, + 245, 8, 6, 76, 69, 32, 68, 79, 84, 216, 3, 92, 9, 79, 78, 71, 83, 69, 79, + 78, 71, 32, 165, 21, 9, 85, 78, 71, 83, 69, 79, 78, 71, 32, 154, 2, 226, + 1, 67, 80, 5, 72, 73, 69, 85, 72, 60, 5, 73, 69, 85, 78, 71, 46, 75, 220, + 1, 5, 77, 73, 69, 85, 77, 188, 1, 5, 78, 73, 69, 85, 78, 94, 80, 240, 2, + 5, 82, 73, 69, 85, 76, 190, 4, 83, 194, 3, 84, 213, 1, 2, 89, 69, 8, 36, + 4, 73, 69, 85, 67, 239, 30, 72, 7, 11, 45, 4, 162, 32, 83, 239, 7, 80, + 11, 11, 45, 8, 174, 16, 82, 178, 19, 77, 234, 3, 78, 35, 80, 9, 11, 45, + 6, 194, 17, 75, 57, 2, 83, 83, 28, 76, 7, 65, 80, 89, 69, 79, 85, 78, 40, + 5, 73, 89, 69, 79, 75, 215, 30, 72, 8, 130, 15, 82, 178, 15, 80, 131, 4, + 77, 19, 11, 45, 16, 166, 13, 75, 170, 1, 82, 42, 83, 224, 13, 2, 67, 72, + 146, 9, 78, 34, 80, 147, 2, 72, 27, 11, 45, 24, 74, 80, 30, 83, 134, 13, + 82, 242, 13, 67, 130, 9, 75, 42, 78, 179, 2, 72, 6, 174, 33, 73, 131, 6, + 65, 6, 40, 4, 83, 65, 78, 71, 227, 185, 13, 73, 4, 238, 35, 78, 147, 3, + 83, 21, 11, 45, 18, 174, 12, 82, 242, 13, 67, 202, 6, 84, 186, 2, 75, + 218, 2, 72, 62, 80, 39, 83, 36, 88, 6, 65, 78, 83, 73, 79, 83, 48, 6, 72, + 73, 69, 85, 80, 72, 53, 4, 73, 69, 85, 80, 7, 11, 45, 4, 236, 7, 2, 75, + 65, 195, 26, 80, 9, 11, 45, 6, 150, 11, 84, 234, 22, 80, 243, 2, 83, 23, + 11, 45, 20, 112, 5, 82, 73, 69, 85, 76, 24, 4, 83, 73, 79, 83, 134, 3, + 80, 246, 19, 67, 194, 5, 77, 170, 4, 84, 243, 1, 72, 5, 153, 3, 2, 45, + 80, 5, 221, 32, 2, 45, 84, 57, 11, 45, 54, 102, 75, 92, 5, 77, 73, 69, + 85, 77, 50, 80, 126, 83, 74, 84, 44, 2, 89, 69, 154, 28, 78, 179, 2, 72, + 10, 52, 5, 73, 89, 69, 79, 75, 190, 4, 65, 131, 19, 72, 7, 11, 45, 4, + 254, 32, 72, 99, 83, 9, 11, 45, 6, 130, 30, 75, 218, 2, 72, 99, 83, 14, + 48, 4, 73, 69, 85, 80, 174, 26, 72, 163, 6, 65, 11, 11, 45, 8, 42, 80, + 222, 29, 84, 242, 1, 72, 99, 83, 2, 243, 25, 72, 6, 40, 4, 83, 65, 78, + 71, 211, 178, 13, 73, 4, 182, 28, 75, 187, 3, 83, 6, 100, 5, 73, 75, 69, + 85, 84, 151, 25, 72, 6, 56, 9, 79, 82, 73, 78, 72, 73, 69, 85, 72, 191, + 3, 83, 5, 255, 29, 45, 52, 48, 3, 73, 79, 83, 161, 1, 4, 83, 65, 78, 71, + 25, 11, 45, 22, 82, 75, 162, 3, 82, 242, 13, 67, 198, 2, 80, 254, 2, 77, + 138, 1, 84, 147, 5, 72, 4, 22, 65, 135, 26, 73, 2, 237, 18, 6, 80, 89, + 69, 79, 85, 78, 28, 160, 1, 5, 82, 73, 69, 85, 76, 36, 6, 84, 73, 75, 69, + 85, 84, 16, 3, 89, 69, 83, 138, 19, 83, 178, 1, 77, 154, 3, 67, 42, 75, + 42, 78, 34, 80, 207, 159, 18, 73, 5, 17, 2, 45, 75, 2, 159, 17, 72, 5, + 255, 16, 45, 2, 139, 184, 18, 73, 20, 40, 5, 73, 75, 69, 85, 84, 155, 21, + 72, 19, 11, 45, 16, 58, 82, 42, 83, 42, 84, 162, 13, 67, 130, 9, 75, 75, + 80, 2, 17, 2, 73, 69, 2, 131, 147, 24, 85, 4, 21, 3, 73, 79, 83, 5, 223, + 1, 45, 2, 255, 19, 72, 18, 44, 6, 83, 73, 69, 85, 78, 71, 243, 19, 79, + 17, 11, 45, 14, 50, 75, 30, 83, 198, 17, 77, 154, 6, 72, 63, 80, 4, 166, + 14, 72, 135, 7, 73, 4, 26, 83, 131, 171, 13, 73, 2, 21, 3, 65, 78, 71, 2, + 207, 20, 75, 190, 1, 122, 65, 118, 69, 134, 1, 73, 92, 7, 83, 83, 65, 78, + 71, 65, 82, 106, 79, 138, 1, 85, 102, 89, 174, 15, 87, 235, 141, 10, 70, + 23, 48, 4, 82, 65, 69, 65, 98, 45, 139, 152, 25, 69, 13, 11, 45, 10, 250, + 210, 22, 69, 226, 197, 2, 65, 2, 73, 3, 85, 25, 18, 79, 55, 85, 9, 11, + 45, 6, 158, 243, 24, 69, 234, 36, 79, 3, 85, 15, 11, 45, 12, 170, 9, 69, + 170, 142, 25, 65, 2, 79, 3, 85, 31, 11, 45, 28, 66, 65, 34, 89, 166, 1, + 79, 170, 240, 24, 69, 234, 36, 73, 3, 85, 5, 11, 82, 2, 215, 144, 18, 65, + 14, 50, 65, 162, 208, 22, 69, 226, 197, 2, 79, 3, 85, 7, 138, 247, 24, + 45, 247, 30, 69, 23, 26, 45, 199, 149, 25, 69, 18, 50, 79, 22, 89, 158, + 207, 22, 69, 227, 197, 2, 85, 5, 183, 129, 25, 45, 8, 154, 207, 22, 69, + 195, 133, 2, 65, 17, 11, 45, 14, 158, 19, 89, 186, 164, 5, 73, 152, 254, + 12, 3, 69, 79, 45, 190, 158, 6, 65, 163, 64, 85, 62, 42, 65, 70, 69, 66, + 73, 22, 79, 107, 85, 11, 26, 45, 175, 147, 25, 69, 6, 182, 244, 24, 89, + 246, 30, 79, 3, 85, 11, 11, 79, 9, 11, 45, 6, 178, 144, 25, 89, 186, 2, + 79, 3, 85, 5, 219, 237, 24, 45, 19, 11, 45, 16, 58, 89, 202, 209, 24, 65, + 174, 33, 69, 246, 30, 73, 3, 79, 6, 198, 209, 24, 65, 175, 33, 69, 21, + 11, 45, 18, 142, 16, 89, 206, 187, 22, 69, 194, 133, 2, 65, 162, 64, 73, + 2, 79, 3, 85, 186, 1, 226, 1, 65, 46, 67, 54, 69, 30, 73, 22, 75, 188, 1, + 5, 77, 73, 69, 85, 77, 64, 5, 78, 73, 69, 85, 78, 70, 80, 168, 1, 5, 82, + 73, 69, 85, 76, 254, 1, 84, 90, 83, 246, 2, 87, 50, 89, 150, 1, 72, 230, + 194, 24, 79, 163, 64, 85, 9, 156, 13, 3, 82, 65, 69, 235, 129, 25, 69, 4, + 22, 72, 207, 8, 73, 2, 141, 236, 24, 2, 73, 69, 7, 166, 142, 25, 79, 3, + 85, 5, 207, 168, 18, 69, 14, 56, 7, 65, 80, 89, 69, 79, 85, 78, 106, 72, + 155, 3, 73, 8, 30, 80, 30, 83, 231, 3, 77, 4, 190, 4, 72, 215, 3, 73, 2, + 25, 4, 83, 65, 78, 71, 2, 207, 7, 80, 2, 213, 60, 2, 73, 69, 9, 11, 45, + 6, 22, 80, 247, 9, 83, 4, 142, 7, 73, 207, 2, 65, 13, 11, 45, 10, 234, 5, + 67, 146, 1, 84, 242, 1, 72, 62, 80, 39, 83, 20, 48, 4, 73, 69, 85, 80, + 170, 2, 72, 163, 6, 65, 17, 11, 45, 14, 42, 83, 186, 2, 84, 146, 2, 67, + 43, 75, 6, 21, 3, 73, 79, 83, 7, 11, 45, 4, 202, 4, 75, 107, 84, 27, 11, + 45, 24, 68, 2, 75, 73, 34, 77, 34, 80, 106, 84, 54, 89, 222, 4, 72, 99, + 83, 4, 149, 1, 4, 89, 69, 79, 75, 2, 11, 73, 2, 255, 223, 23, 69, 8, 30, + 72, 34, 73, 131, 6, 65, 2, 193, 228, 18, 3, 73, 69, 85, 4, 21, 3, 69, 85, + 80, 5, 243, 5, 45, 4, 22, 72, 151, 3, 73, 2, 225, 230, 21, 2, 73, 69, 2, + 17, 2, 69, 79, 2, 167, 4, 82, 28, 44, 3, 73, 79, 83, 57, 4, 83, 65, 78, + 71, 13, 11, 45, 10, 122, 67, 42, 75, 42, 78, 34, 80, 35, 84, 16, 78, 67, + 42, 75, 42, 78, 34, 80, 34, 84, 242, 1, 72, 98, 83, 223, 156, 18, 73, 2, + 11, 73, 2, 249, 221, 23, 2, 69, 85, 2, 11, 73, 2, 237, 219, 24, 2, 89, + 69, 2, 11, 73, 2, 247, 132, 24, 69, 2, 11, 73, 2, 147, 143, 24, 69, 2, + 11, 73, 2, 11, 75, 2, 139, 139, 24, 69, 10, 230, 190, 22, 69, 194, 133, + 2, 65, 163, 64, 73, 34, 58, 69, 206, 1, 79, 62, 85, 182, 193, 24, 65, + 163, 64, 73, 13, 42, 79, 73, 6, 83, 73, 69, 85, 78, 71, 5, 11, 82, 2, 17, + 2, 73, 78, 2, 11, 72, 2, 161, 235, 9, 2, 73, 69, 7, 11, 45, 4, 18, 80, + 39, 83, 2, 11, 65, 2, 11, 78, 2, 11, 83, 2, 223, 146, 13, 73, 9, 11, 45, + 6, 26, 89, 235, 129, 25, 73, 4, 199, 193, 24, 65, 9, 11, 45, 6, 26, 89, + 175, 129, 25, 73, 4, 203, 187, 22, 69, 24, 210, 136, 11, 84, 162, 136, + 12, 70, 30, 83, 210, 86, 78, 14, 79, 223, 110, 69, 100, 156, 1, 7, 76, + 69, 84, 84, 69, 82, 32, 196, 2, 5, 77, 65, 82, 75, 32, 72, 5, 83, 73, 71, + 78, 32, 132, 159, 2, 6, 86, 79, 87, 69, 76, 32, 255, 235, 20, 68, 58, + 202, 1, 68, 34, 75, 254, 241, 20, 84, 154, 50, 82, 130, 223, 1, 78, 238, + 178, 1, 83, 138, 69, 66, 2, 67, 2, 70, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, + 2, 80, 2, 86, 2, 87, 2, 89, 2, 90, 187, 2, 65, 4, 166, 251, 24, 68, 187, + 2, 65, 8, 56, 5, 73, 78, 78, 65, 32, 206, 250, 24, 72, 187, 2, 65, 4, + 202, 250, 24, 87, 3, 89, 4, 248, 197, 21, 6, 78, 65, 32, 75, 72, 79, 153, + 183, 2, 3, 83, 65, 75, 8, 52, 2, 84, 65, 217, 238, 22, 5, 72, 65, 82, 66, + 65, 6, 42, 72, 158, 145, 20, 83, 235, 231, 4, 78, 2, 163, 217, 24, 65, + 42, 62, 76, 144, 239, 18, 6, 83, 73, 71, 78, 32, 80, 247, 1, 86, 36, 33, + 6, 69, 84, 84, 69, 82, 32, 36, 146, 137, 21, 78, 250, 238, 3, 66, 2, 68, + 2, 71, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, + 89, 186, 2, 65, 2, 73, 3, 85, 2, 139, 235, 23, 69, 4, 130, 180, 23, 68, + 143, 197, 1, 80, 54, 52, 5, 67, 72, 73, 78, 71, 41, 4, 82, 65, 78, 32, 2, + 17, 2, 32, 67, 2, 163, 200, 23, 72, 52, 52, 7, 76, 69, 84, 84, 69, 82, + 32, 175, 179, 6, 78, 42, 218, 1, 65, 234, 173, 11, 90, 162, 47, 84, 150, + 119, 76, 50, 81, 60, 6, 68, 65, 76, 69, 84, 72, 146, 165, 4, 71, 122, 83, + 66, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, 178, 216, 4, 78, 134, + 2, 87, 218, 103, 80, 171, 4, 77, 4, 130, 250, 16, 76, 151, 172, 7, 89, + 172, 9, 230, 1, 65, 240, 32, 5, 66, 82, 69, 87, 32, 210, 32, 76, 200, 1, + 16, 78, 84, 65, 73, 71, 65, 78, 65, 32, 76, 69, 84, 84, 69, 82, 32, 174, + 12, 82, 120, 11, 88, 65, 71, 82, 65, 77, 32, 70, 79, 82, 32, 221, 162, + 24, 4, 68, 71, 69, 72, 214, 1, 42, 68, 98, 82, 201, 1, 3, 86, 89, 32, 6, + 44, 5, 83, 84, 79, 78, 69, 187, 200, 23, 80, 5, 217, 225, 2, 7, 32, 71, + 82, 65, 86, 69, 89, 12, 32, 2, 84, 32, 139, 151, 17, 45, 10, 60, 5, 87, + 73, 84, 72, 32, 246, 163, 12, 68, 155, 143, 9, 72, 6, 76, 9, 84, 73, 80, + 32, 79, 78, 32, 84, 72, 174, 242, 12, 82, 191, 228, 10, 65, 2, 223, 190, + 24, 69, 196, 1, 134, 2, 65, 202, 1, 66, 230, 2, 67, 154, 3, 68, 162, 1, + 69, 186, 3, 70, 94, 72, 62, 76, 222, 1, 77, 110, 79, 162, 1, 82, 142, 2, + 83, 228, 1, 3, 78, 79, 82, 198, 1, 84, 128, 2, 2, 85, 80, 174, 1, 87, + 138, 140, 14, 73, 254, 238, 3, 80, 210, 131, 4, 86, 251, 77, 71, 12, 108, + 17, 82, 82, 79, 87, 32, 83, 72, 65, 70, 84, 32, 87, 73, 68, 84, 72, 32, + 182, 158, 18, 77, 155, 188, 4, 83, 8, 36, 3, 79, 78, 69, 223, 141, 23, + 84, 7, 11, 32, 4, 174, 238, 13, 84, 135, 252, 8, 72, 14, 48, 4, 65, 76, + 76, 79, 21, 4, 76, 65, 67, 75, 2, 187, 167, 5, 84, 12, 30, 32, 153, 1, 2, + 45, 70, 6, 52, 7, 67, 85, 82, 86, 69, 68, 32, 247, 228, 23, 72, 4, 40, 4, + 68, 79, 87, 78, 1, 2, 85, 80, 2, 213, 203, 23, 8, 87, 65, 82, 68, 83, 32, + 65, 78, 6, 45, 9, 69, 65, 84, 72, 69, 82, 69, 68, 32, 6, 234, 211, 13, + 83, 218, 168, 3, 78, 211, 206, 6, 82, 14, 116, 2, 72, 69, 52, 5, 73, 82, + 67, 76, 69, 149, 220, 17, 14, 79, 78, 67, 65, 86, 69, 45, 80, 79, 73, 78, + 84, 69, 68, 4, 216, 203, 20, 4, 86, 82, 79, 78, 239, 218, 2, 67, 9, 64, + 6, 32, 87, 73, 84, 72, 32, 133, 212, 22, 4, 68, 32, 83, 65, 4, 52, 7, 83, + 84, 82, 79, 75, 69, 32, 199, 254, 4, 67, 2, 29, 5, 65, 78, 68, 32, 84, 2, + 11, 87, 2, 11, 79, 2, 21, 3, 32, 68, 79, 2, 11, 84, 2, 207, 234, 23, 83, + 14, 52, 7, 65, 83, 72, 69, 68, 32, 84, 18, 73, 31, 79, 2, 175, 16, 82, 2, + 137, 144, 19, 2, 86, 73, 10, 192, 16, 3, 87, 78, 87, 198, 139, 14, 85, + 187, 180, 4, 76, 16, 120, 5, 73, 71, 72, 84, 32, 156, 2, 16, 88, 67, 76, + 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, 147, 205, 18, 81, 10, + 40, 2, 80, 79, 126, 84, 231, 207, 22, 83, 6, 33, 6, 73, 78, 84, 69, 68, + 32, 6, 202, 186, 16, 80, 196, 147, 6, 11, 82, 69, 67, 84, 73, 76, 73, 78, + 69, 65, 82, 23, 66, 2, 141, 208, 22, 24, 69, 65, 82, 68, 82, 79, 80, 45, + 83, 80, 79, 75, 69, 68, 32, 80, 82, 79, 80, 69, 76, 76, 69, 82, 4, 246, + 207, 23, 83, 199, 79, 79, 6, 44, 5, 79, 85, 82, 32, 66, 139, 254, 4, 73, + 2, 209, 131, 11, 6, 65, 76, 76, 79, 79, 78, 4, 242, 186, 17, 79, 169, + 221, 5, 6, 69, 65, 82, 84, 32, 69, 20, 74, 65, 44, 3, 69, 70, 84, 28, 2, + 79, 87, 149, 249, 4, 3, 73, 71, 65, 4, 172, 207, 21, 2, 82, 71, 147, 208, + 1, 84, 8, 254, 3, 45, 195, 6, 87, 6, 26, 32, 223, 143, 10, 69, 4, 154, + 150, 14, 68, 25, 4, 83, 73, 78, 71, 4, 56, 8, 85, 76, 84, 73, 80, 76, 73, + 67, 235, 176, 21, 73, 2, 25, 4, 65, 84, 73, 79, 2, 207, 153, 5, 78, 6, + 132, 140, 6, 9, 80, 69, 78, 32, 67, 69, 78, 84, 82, 144, 175, 10, 13, 86, + 65, 76, 32, 87, 73, 84, 72, 32, 79, 86, 65, 76, 153, 141, 6, 5, 85, 84, + 76, 73, 78, 12, 76, 4, 73, 71, 72, 84, 245, 188, 23, 9, 79, 85, 78, 68, + 45, 84, 73, 80, 80, 10, 62, 45, 109, 11, 87, 65, 82, 68, 83, 32, 65, 82, + 82, 79, 87, 4, 69, 15, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 78, 71, + 76, 69, 32, 4, 222, 229, 6, 66, 147, 173, 7, 81, 7, 139, 6, 32, 26, 106, + 65, 78, 73, 44, 2, 79, 85, 160, 154, 14, 7, 67, 82, 73, 80, 84, 32, 76, + 157, 135, 1, 3, 80, 65, 82, 4, 236, 145, 14, 10, 78, 83, 45, 83, 69, 82, + 73, 70, 32, 73, 171, 180, 8, 76, 8, 216, 143, 14, 2, 78, 71, 179, 179, 8, + 88, 10, 21, 3, 84, 72, 32, 10, 60, 5, 69, 65, 83, 84, 32, 29, 6, 87, 69, + 83, 84, 32, 80, 6, 26, 80, 203, 190, 23, 65, 4, 41, 8, 79, 73, 78, 84, + 73, 78, 71, 32, 4, 242, 237, 16, 86, 139, 191, 6, 66, 12, 88, 9, 69, 65, + 82, 68, 82, 79, 80, 45, 83, 130, 1, 82, 241, 225, 6, 4, 87, 69, 76, 86, + 6, 64, 6, 80, 79, 75, 69, 68, 32, 241, 182, 23, 4, 72, 65, 78, 75, 4, + 164, 195, 22, 8, 80, 73, 78, 87, 72, 69, 69, 76, 27, 65, 2, 193, 3, 5, + 73, 65, 78, 71, 76, 6, 26, 87, 203, 133, 10, 80, 4, 53, 11, 65, 82, 68, + 83, 32, 65, 82, 82, 79, 87, 32, 4, 29, 5, 87, 73, 84, 72, 32, 4, 192, + 251, 19, 5, 76, 65, 82, 71, 69, 203, 238, 1, 69, 12, 76, 5, 72, 73, 84, + 69, 32, 164, 1, 2, 73, 68, 237, 236, 22, 3, 69, 68, 71, 8, 64, 6, 83, 81, + 85, 65, 82, 69, 166, 214, 17, 68, 215, 171, 5, 67, 5, 173, 161, 23, 19, + 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, + 86, 2, 157, 232, 20, 2, 69, 45, 140, 2, 112, 7, 65, 67, 67, 69, 78, 84, + 32, 138, 8, 76, 164, 14, 5, 77, 65, 82, 75, 32, 114, 80, 217, 157, 21, 2, + 89, 79, 60, 128, 2, 9, 65, 84, 78, 65, 72, 32, 72, 65, 70, 22, 68, 36, 3, + 71, 69, 82, 74, 77, 124, 2, 80, 65, 28, 4, 69, 84, 78, 65, 20, 2, 81, 65, + 58, 83, 58, 84, 156, 1, 2, 89, 69, 78, 90, 196, 165, 8, 3, 82, 69, 86, + 142, 129, 15, 79, 225, 146, 1, 3, 73, 76, 85, 2, 243, 157, 18, 85, 4, + 230, 128, 19, 69, 179, 140, 5, 65, 6, 32, 3, 69, 83, 72, 179, 28, 83, 5, + 129, 221, 6, 4, 32, 77, 85, 81, 8, 84, 5, 69, 82, 75, 72, 65, 164, 238, + 17, 2, 85, 78, 249, 45, 5, 65, 72, 65, 80, 65, 5, 209, 219, 19, 4, 32, + 75, 69, 70, 4, 26, 83, 251, 143, 24, 90, 2, 151, 169, 24, 72, 4, 128, + 137, 24, 6, 82, 78, 69, 89, 32, 80, 191, 35, 68, 4, 182, 24, 69, 193, + 213, 22, 6, 72, 65, 76, 83, 72, 69, 8, 38, 69, 249, 209, 22, 3, 73, 80, + 69, 6, 48, 6, 76, 73, 83, 72, 65, 32, 207, 224, 11, 86, 4, 240, 155, 22, + 3, 81, 69, 84, 249, 141, 2, 4, 71, 69, 68, 79, 4, 128, 205, 8, 10, 82, + 65, 72, 32, 66, 69, 78, 32, 89, 79, 255, 221, 2, 84, 8, 34, 65, 237, 145, + 24, 2, 73, 78, 6, 40, 4, 81, 69, 70, 32, 183, 255, 5, 82, 4, 154, 135, + 11, 81, 157, 176, 12, 3, 71, 65, 68, 150, 1, 76, 6, 69, 84, 84, 69, 82, + 32, 201, 11, 8, 73, 71, 65, 84, 85, 82, 69, 32, 140, 1, 134, 3, 65, 204, + 1, 3, 66, 69, 84, 0, 3, 75, 65, 70, 0, 2, 80, 69, 68, 6, 70, 73, 78, 65, + 76, 32, 92, 2, 81, 79, 16, 2, 72, 69, 52, 2, 78, 85, 0, 4, 90, 65, 89, + 73, 18, 83, 48, 3, 82, 69, 83, 170, 1, 84, 52, 4, 68, 65, 76, 69, 12, 5, + 71, 73, 77, 69, 76, 0, 5, 76, 65, 77, 69, 68, 0, 3, 77, 69, 77, 48, 3, + 86, 65, 86, 80, 5, 87, 73, 68, 69, 32, 153, 1, 3, 89, 79, 68, 14, 26, 76, + 171, 198, 23, 89, 12, 64, 2, 69, 70, 73, 10, 84, 69, 82, 78, 65, 84, 73, + 86, 69, 32, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 130, 13, 77, 130, 1, 80, + 35, 81, 4, 242, 201, 16, 65, 251, 160, 1, 80, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 142, 15, 82, 199, 225, 13, 68, 14, 88, 2, 75, 65, 236, 2, 2, + 80, 69, 220, 153, 1, 2, 84, 83, 182, 166, 22, 78, 135, 110, 77, 4, 235, + 2, 70, 7, 244, 10, 5, 32, 87, 73, 84, 72, 167, 184, 24, 84, 4, 167, 2, + 78, 16, 44, 4, 65, 77, 69, 75, 17, 3, 72, 73, 78, 4, 231, 1, 72, 13, 33, + 6, 32, 87, 73, 84, 72, 32, 10, 40, 6, 68, 65, 71, 69, 83, 72, 39, 83, 7, + 33, 6, 32, 65, 78, 68, 32, 83, 4, 254, 254, 5, 72, 203, 192, 2, 73, 12, + 50, 69, 12, 2, 65, 86, 1, 4, 83, 65, 68, 73, 4, 11, 84, 5, 233, 236, 13, + 7, 32, 87, 73, 84, 72, 32, 68, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 148, + 253, 1, 2, 72, 79, 131, 239, 11, 68, 16, 174, 2, 76, 202, 203, 8, 65, + 210, 196, 2, 84, 186, 217, 2, 82, 160, 244, 8, 2, 68, 65, 242, 93, 75, + 222, 106, 72, 169, 4, 7, 70, 73, 78, 65, 76, 32, 77, 7, 33, 6, 32, 87, + 73, 84, 72, 32, 4, 176, 7, 2, 72, 73, 255, 226, 13, 68, 10, 72, 6, 65, + 76, 69, 70, 32, 76, 29, 8, 89, 73, 68, 68, 73, 83, 72, 32, 2, 193, 177, + 19, 2, 65, 77, 8, 120, 7, 68, 79, 85, 66, 76, 69, 32, 232, 4, 9, 89, 79, + 68, 32, 89, 79, 68, 32, 80, 185, 128, 21, 5, 86, 65, 86, 32, 89, 4, 254, + 142, 11, 86, 163, 246, 9, 89, 6, 80, 3, 76, 79, 87, 0, 3, 85, 80, 80, + 237, 232, 22, 6, 77, 65, 83, 79, 82, 65, 2, 233, 169, 23, 2, 69, 82, 50, + 84, 5, 79, 73, 78, 84, 32, 229, 5, 11, 85, 78, 67, 84, 85, 65, 84, 73, + 79, 78, 32, 38, 228, 1, 9, 68, 65, 71, 69, 83, 72, 32, 79, 82, 46, 72, + 106, 80, 166, 1, 81, 86, 82, 22, 83, 176, 209, 9, 2, 84, 83, 132, 142, + 11, 17, 74, 85, 68, 69, 79, 45, 83, 80, 65, 78, 73, 83, 72, 32, 86, 65, + 82, 233, 143, 3, 3, 77, 69, 84, 2, 17, 2, 32, 77, 2, 201, 1, 2, 65, 80, + 12, 60, 5, 65, 84, 65, 70, 32, 106, 73, 33, 4, 79, 76, 65, 77, 6, 38, 80, + 34, 81, 145, 2, 2, 83, 69, 2, 11, 65, 2, 131, 215, 17, 84, 2, 193, 162, + 23, 3, 65, 77, 65, 2, 11, 82, 2, 143, 230, 13, 73, 5, 181, 137, 11, 12, + 32, 72, 65, 83, 69, 82, 32, 70, 79, 82, 32, 86, 6, 52, 5, 65, 77, 65, 84, + 83, 189, 191, 11, 2, 85, 66, 5, 165, 242, 10, 2, 32, 81, 2, 167, 225, 6, + 65, 8, 34, 69, 22, 72, 243, 178, 8, 73, 2, 239, 161, 23, 71, 4, 238, 178, + 8, 73, 199, 209, 7, 69, 12, 152, 1, 3, 71, 69, 82, 60, 3, 80, 65, 83, 20, + 7, 83, 79, 70, 32, 80, 65, 83, 212, 249, 21, 7, 78, 85, 78, 32, 72, 65, + 70, 185, 183, 1, 3, 77, 65, 81, 4, 26, 83, 211, 202, 22, 69, 2, 157, 176, + 23, 3, 72, 65, 89, 2, 155, 226, 13, 69, 2, 135, 226, 13, 85, 8, 114, 77, + 160, 149, 12, 15, 76, 83, 67, 72, 82, 69, 73, 66, 69, 82, 32, 80, 65, 85, + 83, 185, 223, 5, 3, 73, 67, 79, 4, 176, 222, 5, 12, 69, 84, 32, 87, 73, + 84, 72, 32, 87, 72, 73, 84, 231, 191, 17, 32, 188, 4, 164, 1, 2, 65, 45, + 50, 72, 70, 75, 230, 1, 77, 114, 78, 146, 2, 82, 66, 83, 154, 1, 84, 226, + 1, 87, 62, 89, 110, 69, 134, 241, 8, 85, 190, 170, 5, 79, 143, 191, 3, + 73, 8, 190, 144, 24, 87, 246, 30, 49, 2, 50, 3, 51, 72, 166, 6, 79, 146, + 191, 8, 65, 146, 224, 5, 85, 158, 115, 69, 3, 73, 74, 72, 2, 65, 45, 104, + 2, 79, 45, 178, 4, 73, 226, 3, 69, 223, 142, 15, 85, 24, 210, 234, 11, + 49, 206, 172, 12, 75, 214, 22, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, + 56, 3, 57, 8, 178, 153, 24, 75, 218, 19, 49, 2, 50, 3, 51, 54, 68, 2, 69, + 45, 154, 7, 79, 222, 142, 15, 65, 2, 73, 243, 203, 2, 85, 6, 218, 169, + 24, 77, 186, 2, 49, 3, 50, 68, 116, 2, 69, 45, 72, 2, 73, 45, 246, 2, 65, + 194, 243, 8, 79, 190, 170, 5, 85, 169, 136, 6, 6, 45, 77, 85, 45, 77, 79, + 14, 254, 139, 24, 75, 246, 30, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 16, + 214, 147, 24, 84, 214, 22, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, + 54, 222, 4, 79, 2, 85, 222, 142, 15, 73, 242, 203, 2, 65, 3, 69, 68, 62, + 65, 2, 85, 226, 3, 73, 134, 241, 8, 69, 219, 157, 6, 79, 16, 11, 45, 16, + 206, 168, 24, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 64, + 74, 69, 20, 2, 79, 45, 72, 2, 85, 45, 190, 144, 15, 73, 243, 203, 2, 65, + 18, 139, 240, 18, 45, 14, 234, 164, 24, 82, 186, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 3, 54, 10, 230, 135, 24, 84, 246, 30, 49, 2, 50, 2, 51, 3, 52, + 42, 170, 242, 8, 65, 2, 73, 218, 157, 6, 79, 243, 203, 2, 69, 32, 40, 2, + 65, 45, 66, 79, 207, 218, 17, 85, 12, 198, 134, 24, 89, 246, 30, 49, 2, + 50, 2, 51, 2, 52, 3, 53, 12, 11, 45, 12, 238, 164, 24, 49, 2, 50, 2, 51, + 2, 52, 2, 53, 3, 54, 4, 140, 167, 8, 21, 77, 73, 84, 73, 65, 78, 32, 67, + 79, 78, 74, 85, 71, 65, 84, 69, 32, 77, 65, 84, 82, 167, 253, 15, 66, + 128, 1, 210, 2, 65, 110, 66, 144, 1, 2, 67, 79, 94, 68, 198, 2, 70, 66, + 71, 40, 4, 72, 79, 76, 68, 164, 1, 2, 73, 78, 116, 2, 77, 79, 58, 79, + 122, 80, 100, 2, 82, 69, 62, 83, 238, 1, 84, 210, 5, 87, 212, 155, 15, 4, + 76, 73, 77, 73, 220, 180, 5, 11, 89, 79, 85, 84, 72, 70, 85, 76, 32, 70, + 79, 181, 173, 3, 9, 69, 78, 84, 72, 85, 83, 73, 65, 83, 6, 76, 3, 80, 80, + 82, 124, 4, 70, 84, 69, 82, 189, 151, 19, 4, 66, 85, 78, 68, 2, 233, 253, + 23, 2, 79, 65, 6, 92, 5, 69, 70, 79, 82, 69, 128, 215, 3, 6, 73, 84, 73, + 78, 71, 32, 1, 4, 82, 69, 65, 75, 2, 197, 206, 23, 7, 32, 67, 79, 77, 80, + 76, 69, 6, 26, 78, 183, 142, 19, 77, 4, 228, 187, 20, 4, 84, 69, 77, 80, + 245, 211, 2, 3, 70, 76, 73, 14, 110, 69, 78, 73, 206, 146, 19, 85, 129, + 241, 2, 15, 65, 82, 75, 69, 78, 73, 78, 71, 32, 79, 70, 32, 84, 72, 69, + 6, 222, 142, 19, 67, 192, 6, 2, 76, 73, 201, 196, 4, 5, 86, 69, 76, 79, + 80, 4, 252, 132, 19, 21, 70, 70, 73, 67, 85, 76, 84, 89, 32, 65, 84, 32, + 84, 72, 69, 32, 66, 69, 71, 73, 78, 177, 248, 2, 4, 83, 80, 69, 82, 4, + 220, 137, 19, 2, 79, 76, 217, 188, 4, 5, 69, 76, 76, 79, 87, 12, 36, 5, + 65, 84, 72, 69, 82, 35, 82, 2, 225, 134, 15, 3, 73, 78, 71, 10, 40, 4, + 69, 65, 84, 32, 155, 221, 23, 65, 8, 22, 80, 211, 5, 84, 6, 22, 79, 147, + 5, 82, 4, 172, 2, 2, 83, 83, 203, 217, 23, 87, 8, 50, 78, 206, 138, 19, + 67, 217, 5, 3, 70, 76, 85, 4, 160, 144, 19, 2, 79, 67, 205, 231, 1, 5, + 69, 82, 32, 84, 82, 4, 204, 171, 22, 3, 68, 69, 83, 189, 61, 3, 85, 84, + 72, 6, 26, 66, 37, 2, 80, 80, 2, 149, 199, 23, 4, 83, 84, 82, 85, 4, 26, + 82, 239, 197, 23, 79, 2, 145, 248, 21, 2, 69, 83, 6, 196, 139, 15, 9, 85, + 83, 72, 73, 78, 71, 32, 85, 80, 204, 137, 2, 3, 82, 79, 71, 255, 196, 6, + 69, 6, 26, 84, 207, 250, 18, 86, 4, 254, 139, 19, 85, 223, 57, 82, 8, + 132, 1, 5, 77, 65, 76, 76, 32, 236, 156, 21, 6, 84, 65, 78, 68, 83, 84, + 233, 240, 1, 11, 80, 76, 73, 84, 84, 73, 78, 71, 32, 65, 80, 4, 24, 2, + 80, 82, 43, 84, 2, 225, 140, 19, 5, 69, 80, 79, 78, 68, 2, 11, 65, 2, + 155, 195, 23, 77, 30, 36, 3, 72, 69, 32, 175, 219, 17, 82, 28, 186, 2, + 65, 130, 1, 67, 132, 1, 3, 70, 65, 77, 20, 9, 82, 69, 67, 69, 80, 84, 73, + 86, 69, 30, 87, 172, 22, 12, 77, 65, 82, 82, 89, 73, 78, 71, 32, 77, 65, + 73, 168, 147, 5, 6, 71, 69, 78, 84, 76, 69, 248, 221, 10, 13, 75, 69, 69, + 80, 73, 78, 71, 32, 83, 84, 73, 76, 76, 201, 233, 3, 7, 74, 79, 89, 79, + 85, 83, 32, 6, 58, 82, 133, 181, 11, 8, 66, 89, 83, 77, 65, 76, 32, 87, + 4, 248, 168, 20, 8, 79, 85, 83, 73, 78, 71, 32, 84, 171, 214, 3, 77, 6, + 128, 131, 1, 2, 65, 85, 212, 164, 19, 9, 82, 69, 65, 84, 73, 86, 69, 32, + 72, 133, 212, 1, 9, 76, 73, 78, 71, 73, 78, 71, 32, 70, 2, 243, 235, 23, + 73, 2, 157, 166, 20, 2, 32, 69, 4, 178, 135, 22, 69, 213, 201, 1, 5, 65, + 78, 68, 69, 82, 4, 202, 231, 6, 65, 141, 166, 6, 14, 79, 82, 75, 32, 79, + 78, 32, 84, 72, 69, 32, 68, 69, 67, 222, 1, 236, 1, 2, 71, 72, 180, 2, 7, + 82, 65, 71, 65, 78, 65, 32, 212, 4, 7, 83, 84, 79, 82, 73, 67, 32, 168, + 186, 15, 7, 78, 68, 85, 32, 84, 69, 77, 240, 33, 4, 75, 73, 78, 71, 162, + 250, 1, 66, 177, 168, 4, 8, 80, 80, 79, 80, 79, 84, 65, 77, 12, 18, 32, + 119, 45, 6, 204, 135, 5, 2, 66, 82, 220, 231, 16, 6, 86, 79, 76, 84, 65, + 71, 229, 48, 9, 79, 67, 84, 69, 84, 32, 80, 82, 69, 6, 92, 11, 83, 80, + 69, 69, 68, 32, 84, 82, 65, 73, 78, 157, 200, 5, 6, 72, 69, 69, 76, 69, + 68, 5, 129, 186, 11, 14, 32, 87, 73, 84, 72, 32, 66, 85, 76, 76, 69, 84, + 32, 78, 200, 1, 112, 9, 68, 73, 71, 82, 65, 80, 72, 32, 89, 20, 7, 76, + 69, 84, 84, 69, 82, 32, 222, 172, 1, 86, 139, 185, 20, 73, 2, 243, 170, + 17, 79, 194, 1, 194, 1, 65, 74, 83, 206, 163, 1, 66, 162, 3, 78, 150, 2, + 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 80, 2, 82, 2, 84, 2, 90, 126, 87, 46, + 89, 142, 183, 22, 86, 234, 36, 69, 2, 73, 2, 79, 3, 85, 7, 37, 7, 82, 67, + 72, 65, 73, 67, 32, 4, 210, 225, 23, 87, 151, 14, 89, 42, 76, 5, 77, 65, + 76, 76, 32, 206, 133, 24, 65, 2, 69, 2, 73, 2, 79, 3, 85, 32, 170, 169, + 1, 87, 46, 89, 170, 173, 5, 75, 230, 137, 17, 84, 234, 36, 65, 2, 69, 2, + 73, 2, 79, 3, 85, 2, 139, 179, 8, 83, 108, 166, 1, 76, 52, 3, 78, 69, 89, + 46, 82, 146, 7, 84, 138, 1, 85, 154, 188, 16, 83, 246, 213, 2, 67, 156, + 192, 3, 6, 77, 79, 84, 72, 69, 84, 242, 164, 1, 79, 155, 3, 80, 6, 176, + 140, 16, 4, 76, 79, 87, 32, 247, 246, 7, 69, 4, 186, 147, 22, 66, 129, + 159, 1, 2, 32, 80, 66, 60, 8, 73, 90, 79, 78, 84, 65, 76, 32, 153, 6, 2, + 83, 69, 60, 218, 1, 66, 110, 76, 152, 1, 17, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, 32, 66, 76, 79, 67, 75, 45, 88, 10, 83, 67, 65, 78, 32, 76, - 73, 78, 69, 45, 46, 84, 210, 239, 20, 69, 251, 4, 77, 6, 44, 5, 76, 65, - 67, 75, 32, 243, 213, 22, 65, 4, 38, 79, 153, 215, 21, 3, 72, 69, 88, 2, - 139, 215, 21, 67, 2, 173, 244, 20, 2, 32, 69, 14, 160, 166, 16, 3, 49, - 51, 53, 198, 231, 6, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 8, 142, 141, - 23, 49, 2, 51, 2, 55, 3, 57, 10, 32, 2, 65, 66, 215, 246, 20, 82, 8, 26, - 85, 215, 245, 20, 32, 6, 33, 6, 76, 65, 84, 73, 79, 78, 7, 11, 32, 4, - 208, 163, 9, 8, 87, 73, 84, 72, 32, 74, 85, 83, 151, 179, 13, 83, 7, 11, - 32, 4, 140, 159, 7, 2, 82, 65, 155, 175, 15, 70, 10, 26, 32, 195, 129, - 22, 69, 8, 86, 80, 188, 175, 18, 5, 66, 69, 86, 69, 82, 204, 80, 3, 83, - 80, 82, 159, 137, 4, 68, 2, 175, 175, 14, 69, 12, 48, 6, 82, 71, 76, 65, - 83, 83, 77, 2, 83, 69, 5, 153, 208, 19, 14, 32, 87, 73, 84, 72, 32, 70, - 76, 79, 87, 73, 78, 71, 32, 9, 11, 32, 6, 88, 8, 87, 73, 84, 72, 32, 71, - 65, 82, 169, 132, 21, 8, 66, 85, 73, 76, 68, 73, 78, 71, 2, 147, 183, 21, - 68, 7, 242, 135, 23, 74, 3, 83, 8, 106, 83, 208, 243, 21, 11, 78, 68, 82, - 69, 68, 32, 80, 79, 73, 78, 84, 148, 11, 2, 71, 71, 139, 136, 1, 84, 2, - 171, 246, 21, 72, 18, 90, 80, 224, 1, 5, 83, 84, 69, 82, 69, 220, 180, - 16, 2, 71, 73, 193, 207, 4, 2, 65, 67, 12, 60, 3, 72, 69, 78, 129, 133, - 4, 6, 79, 68, 73, 65, 83, 84, 11, 34, 32, 82, 65, 227, 167, 19, 45, 4, - 26, 87, 203, 168, 21, 66, 2, 21, 3, 73, 84, 72, 2, 141, 168, 3, 2, 32, - 68, 2, 193, 169, 12, 6, 84, 73, 79, 78, 32, 80, 2, 253, 240, 21, 2, 83, - 73, 202, 5, 160, 1, 3, 67, 69, 32, 152, 1, 2, 68, 69, 222, 24, 77, 214, - 2, 78, 132, 37, 7, 90, 65, 75, 65, 89, 65, 32, 237, 137, 19, 9, 32, 76, - 79, 86, 69, 32, 89, 79, 85, 8, 114, 67, 252, 249, 10, 18, 72, 79, 67, 75, - 69, 89, 32, 83, 84, 73, 67, 75, 32, 65, 78, 68, 32, 80, 155, 179, 1, 83, - 4, 250, 184, 15, 82, 187, 212, 2, 85, 246, 1, 68, 3, 78, 84, 73, 201, 1, + 73, 78, 69, 45, 46, 84, 162, 217, 21, 67, 42, 69, 238, 6, 77, 150, 1, 82, + 247, 2, 90, 6, 44, 5, 76, 65, 67, 75, 32, 163, 199, 23, 65, 4, 38, 79, + 169, 199, 22, 3, 72, 69, 88, 2, 155, 199, 22, 67, 10, 40, 4, 73, 78, 69, + 32, 135, 223, 21, 65, 8, 44, 5, 87, 73, 84, 72, 32, 171, 223, 21, 69, 6, + 26, 84, 219, 224, 21, 70, 4, 250, 224, 21, 72, 171, 90, 73, 14, 136, 225, + 16, 3, 49, 51, 53, 158, 157, 7, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 8, + 206, 253, 23, 49, 2, 51, 2, 55, 3, 57, 10, 32, 2, 65, 66, 223, 227, 21, + 82, 8, 26, 85, 223, 226, 21, 32, 6, 33, 6, 76, 65, 84, 73, 79, 78, 7, 11, + 32, 4, 188, 197, 9, 8, 87, 73, 84, 72, 32, 74, 85, 83, 223, 129, 14, 83, + 7, 11, 32, 4, 180, 191, 7, 2, 82, 65, 167, 255, 15, 70, 10, 26, 32, 235, + 241, 22, 69, 8, 86, 80, 240, 243, 18, 5, 66, 69, 86, 69, 82, 160, 80, 3, + 83, 80, 82, 215, 181, 4, 68, 2, 167, 230, 14, 69, 12, 48, 6, 82, 71, 76, + 65, 83, 83, 77, 2, 83, 69, 5, 249, 170, 20, 14, 32, 87, 73, 84, 72, 32, + 70, 76, 79, 87, 73, 78, 71, 32, 9, 11, 32, 6, 88, 8, 87, 73, 84, 72, 32, + 71, 65, 82, 217, 241, 21, 8, 66, 85, 73, 76, 68, 73, 78, 71, 2, 223, 164, + 22, 68, 7, 178, 248, 23, 74, 3, 83, 8, 106, 83, 248, 227, 22, 11, 78, 68, + 82, 69, 68, 32, 80, 79, 73, 78, 84, 148, 11, 2, 71, 71, 163, 136, 1, 84, + 2, 211, 230, 22, 72, 18, 90, 80, 224, 1, 5, 83, 84, 69, 82, 69, 236, 238, + 16, 2, 71, 73, 229, 130, 5, 2, 65, 67, 12, 60, 3, 72, 69, 78, 245, 158, + 4, 6, 79, 68, 73, 65, 83, 84, 11, 34, 32, 82, 65, 231, 238, 19, 45, 4, + 26, 87, 135, 150, 22, 66, 2, 21, 3, 73, 84, 72, 2, 185, 188, 3, 2, 32, + 68, 2, 177, 220, 12, 6, 84, 73, 79, 78, 32, 80, 2, 165, 225, 22, 2, 83, + 73, 206, 5, 160, 1, 3, 67, 69, 32, 152, 1, 2, 68, 69, 222, 24, 77, 222, + 2, 78, 200, 37, 7, 90, 65, 75, 65, 89, 65, 32, 129, 228, 19, 9, 32, 76, + 79, 86, 69, 32, 89, 79, 85, 8, 114, 67, 232, 161, 11, 18, 72, 79, 67, 75, + 69, 89, 32, 83, 84, 73, 67, 75, 32, 65, 78, 68, 32, 80, 159, 190, 1, 83, + 4, 230, 239, 15, 82, 163, 226, 2, 85, 246, 1, 68, 3, 78, 84, 73, 201, 1, 9, 79, 71, 82, 65, 80, 72, 73, 67, 32, 8, 64, 4, 67, 65, 76, 32, 105, 8, - 70, 73, 67, 65, 84, 73, 79, 78, 6, 32, 2, 84, 79, 175, 255, 21, 87, 5, - 129, 188, 14, 12, 32, 65, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 2, 229, - 232, 21, 2, 32, 67, 238, 1, 208, 2, 11, 65, 78, 78, 79, 84, 65, 84, 73, + 70, 73, 67, 65, 84, 73, 79, 78, 6, 32, 2, 84, 79, 215, 239, 22, 87, 5, + 237, 242, 14, 12, 32, 65, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 2, 141, + 217, 22, 2, 32, 67, 238, 1, 208, 2, 11, 65, 78, 78, 79, 84, 65, 84, 73, 79, 78, 32, 242, 3, 67, 72, 2, 68, 69, 248, 5, 5, 78, 85, 77, 66, 69, 22, - 84, 220, 204, 5, 8, 72, 65, 76, 70, 32, 70, 73, 76, 144, 186, 1, 5, 69, - 78, 84, 69, 82, 0, 3, 82, 73, 83, 24, 5, 76, 69, 86, 69, 76, 236, 240, 8, - 5, 86, 65, 82, 73, 65, 158, 175, 4, 70, 230, 46, 73, 131, 222, 1, 83, 32, + 84, 128, 237, 5, 8, 72, 65, 76, 70, 32, 70, 73, 76, 148, 186, 1, 5, 69, + 78, 84, 69, 82, 0, 3, 82, 73, 83, 24, 5, 76, 69, 86, 69, 76, 140, 138, 9, + 5, 86, 65, 82, 73, 65, 182, 223, 4, 70, 158, 47, 73, 159, 228, 1, 83, 32, 232, 1, 5, 66, 79, 84, 84, 79, 22, 70, 82, 77, 62, 84, 140, 1, 4, 76, 73, - 78, 75, 188, 142, 1, 4, 83, 69, 67, 79, 150, 255, 5, 79, 228, 238, 5, 6, - 82, 69, 86, 69, 82, 83, 148, 147, 9, 5, 72, 69, 65, 86, 69, 165, 39, 3, - 69, 65, 82, 2, 175, 185, 22, 77, 6, 48, 3, 79, 85, 82, 229, 175, 22, 3, - 73, 82, 83, 4, 218, 184, 22, 84, 27, 32, 4, 36, 3, 73, 68, 68, 235, 144, - 22, 65, 2, 211, 253, 12, 76, 8, 34, 72, 46, 87, 131, 226, 7, 79, 4, 222, - 236, 8, 82, 181, 192, 13, 2, 73, 82, 2, 191, 183, 22, 79, 4, 36, 3, 76, - 79, 83, 167, 173, 20, 79, 2, 129, 183, 22, 3, 73, 78, 71, 36, 104, 20, + 78, 75, 160, 143, 1, 4, 83, 69, 67, 79, 218, 158, 6, 79, 144, 134, 6, 6, + 82, 69, 86, 69, 82, 83, 240, 203, 9, 5, 72, 69, 65, 86, 69, 169, 39, 3, + 69, 65, 82, 2, 227, 169, 23, 77, 6, 48, 3, 79, 85, 82, 153, 160, 23, 3, + 73, 82, 83, 4, 142, 169, 23, 84, 27, 32, 4, 36, 3, 73, 68, 68, 155, 129, + 23, 65, 2, 167, 181, 13, 76, 8, 34, 72, 46, 87, 203, 130, 8, 79, 4, 194, + 142, 9, 82, 133, 143, 14, 2, 73, 82, 2, 243, 167, 23, 79, 4, 36, 3, 76, + 79, 83, 135, 151, 21, 79, 2, 181, 167, 23, 3, 73, 78, 71, 36, 104, 20, 83, 67, 82, 73, 80, 84, 73, 79, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, - 82, 32, 247, 139, 7, 80, 34, 144, 2, 9, 65, 66, 79, 86, 69, 32, 84, 79, + 82, 32, 159, 172, 7, 80, 34, 144, 2, 9, 65, 66, 79, 86, 69, 32, 84, 79, 32, 84, 8, 76, 69, 70, 84, 32, 84, 79, 32, 76, 5, 79, 86, 69, 82, 76, 20, - 2, 83, 85, 202, 186, 14, 82, 152, 153, 5, 8, 70, 85, 76, 76, 32, 83, 85, - 82, 173, 207, 2, 15, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 69, - 70, 76, 4, 60, 9, 77, 73, 68, 68, 76, 69, 32, 65, 78, 167, 220, 20, 66, - 2, 247, 205, 20, 68, 4, 180, 166, 21, 10, 77, 73, 68, 68, 76, 69, 32, 65, - 78, 68, 195, 163, 1, 82, 2, 231, 162, 16, 65, 18, 72, 12, 82, 82, 79, 85, - 78, 68, 32, 70, 82, 79, 77, 32, 163, 236, 16, 66, 16, 82, 76, 200, 163, - 11, 3, 85, 80, 80, 202, 182, 9, 66, 222, 155, 1, 65, 143, 82, 82, 6, 186, - 163, 11, 79, 175, 164, 11, 69, 2, 195, 180, 8, 82, 148, 1, 92, 10, 65, + 2, 83, 85, 182, 241, 14, 82, 252, 189, 5, 8, 70, 85, 76, 76, 32, 83, 85, + 82, 145, 228, 2, 15, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 69, + 70, 76, 4, 60, 9, 77, 73, 68, 68, 76, 69, 32, 65, 78, 143, 199, 21, 66, + 2, 215, 183, 21, 68, 4, 128, 148, 22, 10, 77, 73, 68, 68, 76, 69, 32, 65, + 78, 68, 183, 166, 1, 82, 2, 247, 220, 16, 65, 18, 72, 12, 82, 82, 79, 85, + 78, 68, 32, 70, 82, 79, 77, 32, 147, 178, 17, 66, 16, 82, 76, 220, 203, + 11, 3, 85, 80, 80, 158, 249, 9, 66, 166, 161, 1, 65, 159, 82, 82, 6, 206, + 203, 11, 79, 219, 236, 11, 69, 2, 235, 213, 8, 82, 148, 1, 92, 10, 65, 76, 76, 89, 32, 77, 65, 82, 75, 32, 41, 9, 69, 76, 69, 71, 82, 65, 80, - 72, 32, 10, 138, 132, 21, 70, 70, 84, 203, 83, 79, 138, 1, 140, 1, 11, - 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 205, 179, 12, 17, 76, 73, 78, + 72, 32, 10, 198, 241, 21, 70, 70, 84, 183, 86, 79, 138, 1, 140, 1, 11, + 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 141, 231, 12, 17, 76, 73, 78, 69, 32, 70, 69, 69, 68, 32, 83, 69, 80, 65, 82, 65, 84, 136, 1, 182, 1, 65, 58, 68, 200, 2, 5, 72, 79, 85, 82, 32, 214, 1, 74, 28, 4, 70, 69, 66, - 82, 64, 2, 77, 65, 144, 137, 15, 3, 78, 79, 86, 0, 4, 83, 69, 80, 84, 25, - 4, 79, 67, 84, 79, 4, 232, 166, 19, 3, 85, 71, 85, 213, 154, 1, 2, 80, - 82, 64, 44, 3, 65, 89, 32, 217, 141, 15, 2, 69, 67, 62, 66, 84, 130, 177, - 5, 69, 66, 70, 70, 78, 26, 83, 219, 161, 16, 79, 34, 34, 72, 90, 87, 187, - 156, 22, 69, 8, 36, 3, 73, 82, 84, 139, 128, 21, 82, 6, 26, 89, 175, 155, - 21, 69, 5, 247, 194, 21, 45, 24, 26, 69, 243, 235, 22, 79, 22, 36, 3, 78, - 84, 89, 135, 239, 21, 76, 21, 147, 232, 14, 45, 50, 78, 84, 222, 174, 5, - 69, 66, 70, 70, 78, 26, 83, 206, 206, 15, 90, 143, 83, 79, 20, 42, 87, - 186, 176, 5, 72, 175, 234, 16, 69, 14, 26, 69, 159, 234, 22, 79, 12, 36, - 3, 78, 84, 89, 179, 237, 21, 76, 11, 159, 212, 16, 45, 6, 24, 2, 65, 78, - 35, 85, 2, 11, 85, 2, 131, 221, 16, 65, 4, 206, 210, 22, 78, 143, 5, 76, - 4, 218, 198, 22, 82, 171, 34, 89, 68, 40, 6, 65, 71, 69, 32, 79, 70, 83, - 80, 5, 233, 225, 8, 15, 32, 79, 82, 32, 65, 80, 80, 82, 79, 88, 73, 77, + 82, 64, 2, 77, 65, 148, 192, 15, 3, 78, 79, 86, 0, 4, 83, 69, 80, 84, 25, + 4, 79, 67, 84, 79, 4, 244, 244, 19, 3, 85, 71, 85, 169, 182, 1, 2, 80, + 82, 64, 44, 3, 65, 89, 32, 221, 196, 15, 2, 69, 67, 62, 66, 84, 162, 209, + 5, 69, 66, 70, 70, 78, 26, 83, 227, 241, 16, 79, 34, 34, 72, 90, 87, 239, + 140, 23, 69, 8, 36, 3, 73, 82, 84, 199, 237, 21, 82, 6, 26, 89, 251, 136, + 22, 69, 5, 155, 178, 22, 45, 24, 26, 69, 179, 220, 23, 79, 22, 36, 3, 78, + 84, 89, 183, 223, 22, 76, 21, 247, 158, 15, 45, 50, 78, 84, 254, 206, 5, + 69, 66, 70, 70, 78, 26, 83, 234, 155, 16, 90, 251, 85, 79, 20, 42, 87, + 218, 208, 5, 72, 195, 186, 17, 69, 14, 26, 69, 223, 218, 23, 79, 12, 36, + 3, 78, 84, 89, 227, 221, 22, 76, 11, 143, 154, 17, 45, 6, 24, 2, 65, 78, + 35, 85, 2, 11, 85, 2, 243, 162, 17, 65, 4, 142, 195, 23, 78, 143, 5, 76, + 4, 154, 183, 23, 82, 171, 34, 89, 68, 40, 6, 65, 71, 69, 32, 79, 70, 83, + 80, 5, 221, 131, 9, 15, 32, 79, 82, 32, 65, 80, 80, 82, 79, 88, 73, 77, 65, 84, 69, 65, 65, 14, 69, 82, 73, 65, 76, 32, 65, 82, 65, 77, 65, 73, - 67, 32, 62, 64, 7, 78, 85, 77, 66, 69, 82, 32, 234, 18, 76, 239, 249, 18, - 83, 16, 26, 84, 171, 140, 15, 79, 10, 138, 143, 11, 87, 186, 137, 1, 69, - 195, 225, 8, 72, 132, 3, 202, 1, 67, 234, 1, 68, 218, 6, 70, 132, 2, 5, - 72, 73, 66, 73, 84, 156, 1, 15, 80, 85, 84, 32, 83, 89, 77, 66, 79, 76, - 32, 70, 79, 82, 32, 206, 1, 83, 236, 5, 2, 84, 69, 206, 8, 86, 175, 211, - 9, 66, 10, 32, 2, 79, 77, 69, 2, 82, 69, 4, 144, 144, 16, 3, 73, 78, 71, - 129, 216, 2, 5, 80, 76, 69, 84, 69, 6, 36, 3, 65, 83, 69, 151, 160, 22, - 77, 4, 34, 32, 161, 141, 10, 2, 83, 32, 2, 161, 143, 11, 8, 70, 79, 78, - 84, 32, 83, 73, 90, 145, 1, 24, 2, 69, 88, 103, 73, 5, 205, 201, 21, 20, - 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 84, 32, 84, 72, 69, 32, 86, - 73, 69, 138, 1, 72, 8, 67, 32, 83, 73, 89, 65, 81, 32, 209, 185, 17, 4, - 65, 78, 32, 82, 136, 1, 200, 1, 11, 65, 76, 84, 69, 82, 78, 65, 84, 69, - 32, 76, 2, 76, 28, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 100, 7, 78, 85, - 77, 66, 69, 82, 32, 222, 209, 8, 82, 201, 246, 2, 6, 80, 76, 65, 67, 69, - 72, 2, 237, 157, 22, 2, 65, 75, 6, 68, 4, 79, 78, 69, 32, 217, 224, 20, - 7, 84, 72, 82, 69, 69, 32, 81, 4, 178, 222, 20, 72, 43, 81, 122, 212, 1, + 67, 32, 62, 72, 7, 78, 85, 77, 66, 69, 82, 32, 230, 18, 76, 221, 195, 19, + 2, 83, 69, 16, 26, 84, 167, 195, 15, 79, 10, 162, 183, 11, 87, 138, 148, + 1, 69, 143, 156, 9, 72, 136, 3, 202, 1, 67, 234, 1, 68, 214, 6, 70, 132, + 2, 5, 72, 73, 66, 73, 84, 156, 1, 15, 80, 85, 84, 32, 83, 89, 77, 66, 79, + 76, 32, 70, 79, 82, 32, 206, 1, 83, 236, 5, 2, 84, 69, 206, 8, 86, 251, + 250, 9, 66, 10, 32, 2, 79, 77, 69, 2, 82, 69, 4, 240, 202, 16, 3, 73, 78, + 71, 157, 225, 2, 5, 80, 76, 69, 84, 69, 6, 36, 3, 65, 83, 69, 195, 144, + 23, 77, 4, 34, 32, 185, 181, 10, 2, 83, 32, 2, 173, 183, 11, 8, 70, 79, + 78, 84, 32, 83, 73, 90, 145, 1, 24, 2, 69, 88, 103, 73, 5, 237, 185, 22, + 20, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 84, 32, 84, 72, 69, 32, + 86, 73, 69, 138, 1, 72, 8, 67, 32, 83, 73, 89, 65, 81, 32, 249, 254, 17, + 4, 65, 78, 32, 82, 136, 1, 196, 1, 11, 65, 76, 84, 69, 82, 78, 65, 84, + 69, 32, 76, 2, 76, 28, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 100, 7, 78, + 85, 77, 66, 69, 82, 32, 190, 243, 8, 82, 149, 125, 6, 80, 76, 65, 67, 69, + 72, 2, 157, 142, 23, 2, 65, 75, 6, 68, 4, 79, 78, 69, 32, 137, 206, 21, + 7, 84, 72, 82, 69, 69, 32, 81, 4, 222, 203, 21, 72, 47, 81, 122, 212, 1, 10, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 5, 75, 65, 82, 79, 82, 0, - 4, 76, 65, 75, 72, 250, 226, 9, 69, 66, 70, 94, 78, 26, 83, 78, 84, 220, - 243, 4, 8, 80, 82, 69, 70, 73, 88, 69, 68, 167, 41, 79, 6, 26, 84, 175, - 195, 21, 79, 4, 236, 128, 6, 2, 69, 78, 223, 188, 16, 87, 5, 191, 140, + 4, 76, 65, 75, 72, 198, 138, 10, 69, 66, 70, 94, 78, 26, 83, 78, 84, 240, + 130, 5, 8, 80, 82, 69, 70, 73, 88, 69, 68, 199, 41, 79, 6, 26, 84, 211, + 179, 22, 79, 4, 244, 160, 6, 2, 69, 78, 147, 141, 17, 87, 5, 239, 252, 22, 65, 16, 72, 5, 73, 78, 73, 84, 89, 85, 9, 79, 82, 77, 65, 84, 73, 79, - 78, 32, 5, 45, 9, 32, 78, 69, 71, 65, 84, 69, 68, 32, 2, 237, 134, 8, 4, - 87, 73, 84, 72, 12, 42, 83, 253, 134, 16, 4, 68, 69, 83, 75, 10, 156, - 245, 6, 5, 69, 80, 65, 82, 65, 191, 214, 9, 79, 4, 11, 32, 4, 232, 136, + 78, 32, 5, 45, 9, 32, 78, 69, 71, 65, 84, 69, 68, 32, 2, 157, 168, 8, 4, + 87, 73, 84, 72, 12, 42, 83, 137, 203, 19, 4, 68, 69, 83, 75, 10, 192, + 149, 7, 5, 69, 80, 65, 82, 65, 139, 252, 9, 79, 4, 11, 32, 4, 152, 249, 22, 15, 65, 82, 65, 66, 73, 67, 32, 70, 79, 82, 77, 32, 83, 72, 65, 1, 14, 83, 89, 77, 77, 69, 84, 82, 73, 67, 32, 83, 87, 65, 80, 10, 80, 6, - 76, 65, 84, 73, 78, 32, 234, 185, 14, 83, 221, 143, 7, 4, 78, 85, 77, 66, + 76, 65, 84, 73, 78, 32, 198, 240, 14, 83, 165, 201, 7, 4, 78, 85, 77, 66, 6, 64, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 27, 76, 2, 21, 3, - 76, 32, 76, 2, 205, 217, 20, 2, 69, 84, 116, 84, 13, 67, 82, 73, 80, 84, - 73, 79, 78, 65, 76, 32, 80, 65, 233, 141, 15, 2, 69, 82, 114, 72, 6, 72, + 76, 32, 76, 2, 253, 198, 21, 2, 69, 84, 116, 84, 13, 67, 82, 73, 80, 84, + 73, 79, 78, 65, 76, 32, 80, 65, 209, 196, 15, 2, 69, 82, 114, 72, 6, 72, 76, 65, 86, 73, 32, 225, 1, 7, 82, 84, 72, 73, 65, 78, 32, 54, 48, 7, 76, - 69, 84, 84, 69, 82, 32, 191, 3, 78, 38, 234, 132, 10, 84, 226, 118, 65, - 22, 68, 34, 76, 22, 77, 50, 87, 238, 150, 4, 71, 90, 90, 34, 83, 66, 89, - 154, 193, 1, 72, 234, 5, 75, 130, 76, 66, 190, 173, 4, 78, 199, 105, 80, + 69, 84, 84, 69, 82, 32, 191, 3, 78, 38, 134, 173, 10, 84, 226, 118, 65, + 22, 68, 34, 76, 22, 77, 50, 87, 186, 165, 4, 71, 90, 90, 34, 83, 66, 89, + 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, 178, 216, 4, 78, 223, 105, 80, 60, 22, 76, 251, 1, 78, 44, 11, 69, 44, 29, 5, 84, 84, 69, 82, 32, 44, - 250, 130, 10, 84, 246, 118, 68, 34, 76, 50, 81, 206, 194, 1, 82, 134, - 212, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, - 130, 76, 66, 190, 173, 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, 16, - 33, 6, 85, 77, 66, 69, 82, 32, 16, 246, 249, 10, 84, 202, 253, 3, 79, - 171, 243, 2, 70, 50, 36, 4, 71, 82, 65, 76, 179, 3, 82, 23, 11, 32, 20, - 58, 65, 140, 1, 5, 87, 73, 84, 72, 32, 219, 181, 20, 69, 4, 96, 6, 86, - 69, 82, 65, 71, 69, 173, 237, 15, 12, 82, 79, 85, 78, 68, 32, 65, 32, 80, - 79, 73, 78, 2, 209, 238, 15, 5, 32, 87, 73, 84, 72, 14, 160, 1, 3, 84, - 73, 77, 20, 2, 85, 78, 200, 203, 6, 16, 76, 69, 70, 84, 87, 65, 82, 68, - 83, 32, 65, 82, 82, 79, 87, 32, 182, 231, 12, 73, 198, 1, 79, 251, 49, - 68, 2, 191, 180, 19, 69, 4, 202, 180, 19, 68, 203, 201, 2, 73, 28, 94, - 76, 232, 1, 7, 83, 69, 67, 84, 73, 79, 78, 202, 6, 82, 144, 200, 11, 2, + 150, 171, 10, 84, 246, 118, 68, 34, 76, 50, 81, 238, 205, 1, 82, 178, + 215, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, 75, + 198, 75, 66, 178, 216, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, 16, + 33, 6, 85, 77, 66, 69, 82, 32, 16, 146, 162, 11, 84, 174, 140, 4, 79, + 219, 128, 3, 70, 50, 36, 4, 71, 82, 65, 76, 179, 3, 82, 23, 11, 32, 20, + 58, 65, 140, 1, 5, 87, 73, 84, 72, 32, 175, 160, 21, 69, 4, 96, 6, 86, + 69, 82, 65, 71, 69, 145, 168, 16, 12, 82, 79, 85, 78, 68, 32, 65, 32, 80, + 79, 73, 78, 2, 181, 169, 16, 5, 32, 87, 73, 84, 72, 14, 160, 1, 3, 84, + 73, 77, 20, 2, 85, 78, 248, 235, 6, 16, 76, 69, 70, 84, 87, 65, 82, 68, + 83, 32, 65, 82, 82, 79, 87, 32, 210, 162, 13, 73, 198, 1, 79, 247, 64, + 68, 2, 139, 144, 20, 69, 4, 150, 144, 20, 68, 175, 222, 2, 73, 28, 94, + 76, 232, 1, 7, 83, 69, 67, 84, 73, 79, 78, 146, 7, 82, 228, 249, 11, 2, 67, 65, 43, 73, 8, 164, 1, 17, 73, 78, 69, 65, 82, 32, 65, 78, 78, 79, - 84, 65, 84, 73, 79, 78, 32, 201, 200, 4, 17, 79, 67, 75, 69, 68, 32, 70, - 69, 77, 65, 76, 69, 32, 65, 78, 68, 32, 6, 232, 130, 8, 3, 65, 78, 67, - 170, 128, 8, 84, 199, 175, 3, 83, 15, 11, 32, 12, 168, 1, 6, 65, 66, 79, - 86, 69, 32, 64, 5, 87, 73, 84, 72, 32, 133, 174, 19, 22, 66, 69, 83, 73, + 84, 65, 84, 73, 79, 78, 32, 229, 232, 4, 17, 79, 67, 75, 69, 68, 32, 70, + 69, 77, 65, 76, 69, 32, 65, 78, 68, 32, 6, 140, 164, 8, 3, 65, 78, 67, + 170, 153, 8, 84, 243, 208, 3, 83, 15, 11, 32, 12, 168, 1, 6, 65, 66, 79, + 86, 69, 32, 64, 5, 87, 73, 84, 72, 32, 209, 137, 20, 22, 66, 69, 83, 73, 68, 69, 32, 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 87, 73, 84, 72, - 4, 244, 174, 19, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 85, 6, 158, - 143, 4, 76, 190, 160, 15, 79, 139, 201, 2, 68, 36, 56, 2, 69, 82, 245, 4, - 7, 73, 83, 73, 66, 76, 69, 32, 30, 48, 3, 83, 69, 32, 153, 2, 4, 84, 69, - 68, 32, 12, 180, 1, 5, 67, 72, 69, 67, 75, 68, 24, 68, 79, 87, 78, 87, - 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, 84, 73, - 80, 166, 195, 19, 87, 230, 18, 77, 183, 147, 1, 66, 4, 196, 213, 19, 8, - 69, 82, 32, 66, 79, 65, 82, 68, 195, 175, 2, 32, 2, 229, 237, 19, 2, 32, - 76, 18, 144, 1, 6, 73, 78, 84, 69, 82, 82, 22, 76, 150, 195, 11, 80, 170, - 169, 4, 69, 148, 183, 1, 2, 79, 72, 136, 131, 2, 4, 85, 78, 68, 69, 163, - 83, 81, 2, 151, 179, 4, 79, 6, 60, 9, 79, 87, 32, 75, 65, 86, 89, 75, 65, - 187, 196, 6, 65, 5, 225, 204, 17, 11, 32, 87, 73, 84, 72, 32, 75, 65, 86, - 89, 75, 6, 38, 84, 138, 230, 18, 80, 143, 68, 83, 2, 207, 238, 15, 73, - 218, 1, 94, 65, 134, 21, 69, 62, 79, 42, 85, 165, 146, 11, 10, 73, 71, + 4, 192, 138, 20, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 85, 6, 202, + 171, 4, 76, 222, 223, 15, 79, 239, 221, 2, 68, 40, 56, 2, 69, 82, 189, 5, + 7, 73, 83, 73, 66, 76, 69, 32, 34, 48, 3, 83, 69, 32, 225, 2, 4, 84, 69, + 68, 32, 16, 174, 1, 66, 80, 5, 67, 72, 69, 67, 75, 68, 24, 68, 79, 87, + 78, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, + 84, 73, 80, 250, 163, 20, 87, 215, 26, 77, 6, 44, 5, 76, 65, 67, 75, 32, + 159, 215, 21, 85, 4, 226, 249, 21, 68, 211, 27, 83, 4, 136, 190, 20, 8, + 69, 82, 32, 66, 79, 65, 82, 68, 231, 182, 2, 32, 2, 233, 215, 20, 2, 32, + 76, 18, 144, 1, 6, 73, 78, 84, 69, 82, 82, 22, 76, 142, 242, 11, 80, 206, + 180, 4, 69, 220, 193, 1, 2, 79, 72, 168, 153, 2, 4, 85, 78, 68, 69, 179, + 97, 81, 2, 139, 211, 4, 79, 6, 60, 9, 79, 87, 32, 75, 65, 86, 89, 75, 65, + 163, 228, 6, 65, 5, 237, 144, 18, 11, 32, 87, 73, 84, 72, 32, 75, 65, 86, + 89, 75, 6, 38, 84, 194, 172, 19, 80, 223, 88, 83, 2, 235, 168, 16, 73, + 218, 1, 94, 65, 134, 21, 69, 62, 79, 42, 85, 165, 187, 11, 10, 73, 71, 83, 65, 87, 32, 80, 85, 90, 90, 202, 1, 120, 5, 67, 75, 45, 79, 45, 32, 7, 80, 65, 78, 69, 83, 69, 32, 184, 2, 7, 86, 65, 78, 69, 83, 69, 32, - 183, 190, 22, 82, 2, 233, 226, 17, 3, 76, 65, 78, 16, 246, 1, 67, 18, 80, - 128, 153, 1, 10, 73, 78, 68, 85, 83, 84, 82, 73, 65, 76, 140, 196, 3, 3, - 66, 65, 78, 250, 171, 3, 68, 244, 171, 12, 3, 71, 79, 66, 204, 105, 2, - 79, 71, 181, 98, 16, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 66, 69, - 71, 73, 78, 2, 199, 32, 65, 2, 197, 209, 12, 7, 79, 83, 84, 32, 79, 70, + 171, 174, 23, 82, 2, 209, 166, 18, 3, 76, 65, 78, 16, 246, 1, 67, 18, 80, + 136, 157, 1, 10, 73, 78, 68, 85, 83, 84, 82, 73, 65, 76, 216, 223, 3, 3, + 66, 65, 78, 186, 173, 3, 68, 252, 128, 11, 15, 83, 89, 77, 66, 79, 76, + 32, 70, 79, 82, 32, 66, 69, 71, 73, 200, 246, 1, 3, 71, 79, 66, 197, 108, + 2, 79, 71, 2, 203, 32, 65, 2, 197, 136, 13, 7, 79, 83, 84, 32, 79, 70, 70, 182, 1, 172, 2, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 76, 2, 76, 69, 40, 4, 82, 73, 71, 72, 224, 6, 2, 80, 65, 176, 3, 14, 84, 85, 82, 78, 69, 68, 32, 80, 65, 68, 65, 32, 80, 73, 88, 5, 83, 73, 71, 78, 32, 140, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 255, 192, 20, 68, 6, 52, 2, 75, 69, 228, 129, 16, 2, 80, 69, 175, 5, 67, - 2, 187, 134, 22, 82, 96, 38, 70, 65, 5, 84, 84, 69, 82, 32, 2, 41, 8, 84, - 32, 82, 69, 82, 69, 78, 71, 2, 171, 243, 20, 71, 94, 230, 1, 68, 26, 73, + 239, 173, 21, 68, 6, 52, 2, 75, 69, 152, 189, 16, 2, 80, 69, 175, 5, 67, + 2, 163, 246, 22, 82, 96, 38, 70, 65, 5, 84, 84, 69, 82, 32, 2, 41, 8, 84, + 32, 82, 69, 82, 69, 78, 71, 2, 211, 224, 21, 71, 94, 230, 1, 68, 26, 73, 48, 2, 75, 65, 66, 78, 130, 1, 66, 2, 67, 2, 71, 16, 2, 80, 65, 56, 2, - 82, 65, 32, 2, 83, 65, 42, 84, 74, 74, 186, 174, 20, 65, 150, 132, 2, 72, + 82, 65, 32, 2, 83, 65, 42, 84, 74, 74, 158, 155, 21, 65, 166, 135, 2, 72, 2, 76, 2, 77, 2, 87, 2, 89, 186, 2, 69, 2, 79, 3, 85, 8, 222, 3, 68, 15, - 65, 7, 164, 167, 5, 3, 32, 75, 65, 167, 145, 17, 73, 7, 11, 32, 4, 22, - 83, 215, 2, 77, 2, 145, 137, 20, 2, 65, 83, 14, 36, 2, 71, 65, 90, 89, + 65, 7, 232, 198, 5, 3, 32, 75, 65, 215, 225, 17, 73, 7, 11, 32, 4, 22, + 83, 215, 2, 77, 2, 165, 242, 20, 2, 65, 83, 14, 36, 2, 71, 65, 90, 89, 167, 1, 65, 7, 33, 6, 32, 76, 69, 76, 69, 84, 5, 29, 5, 32, 82, 65, 83, - 87, 2, 135, 192, 5, 65, 4, 163, 1, 65, 7, 11, 32, 4, 154, 1, 77, 129, - 178, 22, 3, 67, 69, 82, 5, 145, 133, 16, 3, 32, 65, 71, 7, 17, 2, 32, 77, + 87, 2, 227, 223, 5, 65, 4, 163, 1, 65, 7, 11, 32, 4, 154, 1, 77, 245, + 161, 23, 3, 67, 69, 82, 5, 201, 192, 16, 3, 32, 65, 71, 7, 17, 2, 32, 77, 4, 70, 85, 71, 65, 8, 18, 65, 55, 84, 5, 17, 2, 32, 77, 2, 11, 85, 2, - 147, 144, 22, 82, 4, 11, 65, 5, 11, 32, 2, 11, 77, 2, 11, 65, 2, 11, 72, - 2, 233, 232, 16, 2, 65, 80, 28, 40, 3, 68, 65, 32, 165, 3, 2, 78, 71, 24, - 174, 1, 65, 90, 76, 86, 80, 168, 240, 6, 10, 84, 73, 82, 84, 65, 32, 84, - 85, 77, 69, 138, 219, 11, 87, 128, 149, 2, 7, 73, 83, 69, 78, 45, 73, 83, - 153, 206, 1, 3, 77, 65, 68, 6, 44, 3, 68, 69, 71, 205, 161, 21, 2, 78, - 68, 5, 11, 32, 2, 181, 236, 21, 2, 65, 68, 6, 38, 85, 141, 175, 22, 3, - 73, 78, 71, 4, 184, 243, 17, 2, 78, 71, 211, 197, 3, 72, 4, 38, 73, 149, - 236, 17, 3, 65, 78, 71, 2, 217, 253, 15, 3, 83, 69, 76, 4, 208, 214, 12, - 5, 82, 65, 78, 71, 75, 171, 138, 9, 75, 10, 104, 5, 67, 69, 67, 65, 75, - 204, 165, 5, 5, 80, 65, 78, 89, 65, 176, 2, 3, 87, 73, 71, 239, 212, 10, - 76, 5, 233, 218, 17, 3, 32, 84, 69, 18, 116, 4, 83, 85, 75, 85, 42, 84, - 64, 4, 87, 85, 76, 85, 182, 251, 15, 80, 165, 210, 1, 7, 68, 73, 82, 71, - 65, 32, 77, 5, 221, 180, 21, 5, 32, 77, 69, 78, 68, 6, 26, 65, 243, 252, - 15, 79, 4, 218, 252, 15, 82, 139, 224, 5, 76, 5, 25, 4, 32, 77, 69, 76, - 2, 131, 170, 22, 73, 4, 36, 3, 76, 76, 89, 207, 254, 18, 65, 2, 155, 238, - 18, 70, 4, 228, 253, 20, 2, 89, 83, 147, 95, 73, 6, 148, 134, 14, 2, 71, - 71, 138, 230, 4, 80, 147, 161, 3, 78, 192, 24, 74, 65, 178, 78, 69, 202, - 1, 72, 178, 50, 73, 178, 2, 78, 50, 79, 111, 82, 198, 10, 200, 1, 5, 73, - 84, 72, 73, 32, 230, 4, 78, 168, 45, 6, 84, 65, 75, 65, 78, 65, 208, 13, - 3, 87, 73, 32, 240, 8, 7, 89, 65, 72, 32, 76, 73, 32, 252, 163, 4, 6, 75, - 84, 79, 86, 73, 75, 219, 143, 17, 65, 136, 1, 204, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 174, 2, 83, 234, 115, 68, 156, 206, 3, 2, 86, 79, 168, 136, - 3, 11, 78, 85, 77, 66, 69, 82, 32, 83, 73, 71, 78, 154, 155, 9, 65, 197, - 213, 1, 6, 69, 78, 85, 77, 69, 82, 90, 210, 1, 68, 210, 189, 18, 65, 82, - 84, 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, 82, 66, 2, 67, - 2, 71, 2, 74, 2, 75, 2, 80, 2, 82, 254, 68, 72, 2, 76, 2, 77, 2, 86, 2, - 89, 186, 2, 69, 3, 79, 10, 38, 68, 194, 163, 22, 72, 187, 2, 65, 6, 186, - 174, 20, 68, 134, 245, 1, 72, 187, 2, 65, 12, 40, 4, 73, 71, 78, 32, 167, - 236, 10, 69, 10, 242, 190, 18, 67, 98, 78, 222, 160, 3, 65, 239, 1, 86, - 230, 4, 42, 71, 221, 39, 5, 78, 65, 68, 65, 32, 174, 3, 76, 11, 88, 73, - 32, 82, 65, 68, 73, 67, 65, 76, 32, 153, 227, 19, 2, 65, 82, 172, 3, 130, - 2, 65, 94, 66, 230, 2, 67, 150, 2, 68, 158, 3, 69, 198, 1, 70, 138, 2, - 71, 146, 1, 72, 210, 2, 73, 76, 2, 74, 65, 34, 75, 30, 76, 230, 1, 77, - 254, 1, 78, 62, 79, 102, 80, 110, 82, 166, 1, 83, 210, 6, 84, 226, 2, 86, - 50, 87, 210, 2, 89, 131, 244, 20, 85, 10, 56, 2, 82, 82, 238, 245, 20, - 71, 158, 101, 78, 195, 47, 88, 4, 138, 163, 21, 79, 207, 1, 73, 34, 58, - 65, 38, 73, 46, 76, 54, 79, 102, 82, 167, 215, 20, 69, 4, 190, 180, 5, - 77, 143, 180, 13, 68, 6, 206, 233, 20, 84, 186, 112, 82, 151, 70, 71, 6, - 138, 254, 18, 79, 154, 243, 1, 65, 251, 151, 1, 85, 10, 62, 76, 210, 130, - 22, 65, 218, 5, 78, 142, 5, 68, 203, 17, 87, 2, 173, 193, 3, 4, 84, 32, - 79, 70, 6, 42, 73, 246, 201, 9, 65, 159, 239, 10, 85, 2, 151, 189, 13, - 83, 28, 58, 65, 70, 76, 74, 79, 190, 135, 10, 72, 163, 171, 10, 73, 6, - 38, 85, 138, 129, 22, 82, 219, 5, 86, 2, 165, 249, 20, 2, 76, 68, 8, 42, - 65, 206, 151, 7, 73, 171, 241, 13, 79, 4, 234, 156, 22, 78, 3, 87, 10, - 214, 250, 20, 82, 204, 1, 2, 77, 80, 206, 98, 86, 134, 5, 76, 223, 56, - 87, 34, 38, 69, 38, 73, 98, 79, 195, 1, 82, 4, 214, 156, 20, 65, 183, - 198, 1, 69, 8, 38, 83, 158, 190, 13, 86, 175, 2, 80, 4, 200, 220, 18, 5, - 84, 73, 78, 71, 85, 199, 190, 3, 72, 16, 94, 84, 76, 3, 85, 66, 76, 214, - 191, 12, 87, 172, 137, 9, 2, 32, 78, 222, 23, 79, 211, 56, 71, 7, 25, 4, - 84, 69, 68, 32, 4, 184, 148, 7, 3, 67, 76, 73, 135, 194, 14, 84, 2, 171, - 170, 3, 69, 6, 162, 226, 20, 65, 158, 166, 1, 85, 219, 16, 89, 20, 106, - 65, 38, 78, 32, 3, 86, 69, 78, 216, 214, 8, 6, 77, 66, 82, 79, 73, 68, - 158, 150, 13, 73, 243, 19, 89, 6, 202, 152, 16, 82, 179, 255, 5, 84, 4, - 158, 25, 67, 219, 192, 21, 84, 5, 203, 198, 21, 73, 28, 74, 65, 50, 73, - 62, 76, 38, 82, 150, 20, 69, 138, 177, 21, 79, 223, 23, 85, 6, 206, 188, - 13, 84, 182, 195, 8, 67, 131, 22, 78, 8, 146, 177, 12, 69, 222, 186, 9, - 71, 230, 19, 82, 199, 21, 83, 4, 190, 209, 20, 85, 175, 196, 1, 89, 4, - 220, 232, 20, 2, 65, 71, 151, 172, 1, 79, 14, 58, 79, 52, 2, 82, 65, 218, - 204, 18, 72, 171, 131, 2, 65, 7, 202, 206, 21, 76, 229, 34, 5, 32, 83, - 76, 79, 87, 4, 214, 196, 21, 73, 135, 18, 83, 22, 58, 65, 142, 1, 69, 60, - 5, 73, 68, 73, 78, 71, 19, 79, 8, 38, 76, 150, 205, 21, 78, 199, 13, 73, - 4, 34, 70, 177, 252, 20, 2, 66, 69, 2, 41, 8, 32, 84, 82, 69, 69, 32, 84, - 82, 2, 255, 211, 18, 85, 6, 26, 65, 159, 146, 22, 77, 4, 250, 245, 21, - 82, 175, 28, 68, 2, 175, 19, 32, 6, 26, 82, 199, 142, 22, 79, 4, 134, - 251, 21, 83, 215, 22, 78, 6, 26, 78, 207, 250, 21, 67, 4, 26, 83, 247, - 143, 22, 67, 2, 171, 130, 21, 69, 4, 154, 250, 21, 68, 215, 22, 82, 2, - 253, 140, 5, 2, 78, 73, 22, 46, 65, 34, 69, 78, 73, 41, 3, 79, 78, 71, 4, - 174, 249, 21, 77, 191, 19, 67, 8, 38, 65, 238, 209, 21, 71, 187, 58, 69, - 4, 178, 181, 13, 84, 139, 218, 8, 70, 6, 194, 248, 21, 70, 2, 78, 215, - 22, 68, 5, 209, 146, 11, 3, 32, 83, 84, 22, 42, 69, 34, 73, 46, 79, 135, - 190, 21, 65, 4, 194, 190, 21, 76, 183, 51, 65, 4, 156, 185, 11, 2, 78, - 73, 203, 248, 8, 76, 12, 34, 82, 34, 85, 195, 189, 21, 79, 4, 138, 235, - 20, 84, 179, 81, 78, 6, 26, 78, 235, 139, 22, 84, 4, 198, 225, 20, 84, - 179, 171, 1, 68, 6, 26, 79, 143, 240, 21, 69, 4, 226, 245, 21, 83, 215, - 22, 84, 10, 40, 2, 78, 69, 22, 80, 199, 197, 21, 76, 5, 139, 247, 5, 83, - 4, 170, 132, 10, 80, 215, 173, 2, 69, 10, 54, 82, 242, 240, 20, 76, 166, - 1, 79, 151, 152, 1, 73, 4, 152, 144, 12, 2, 73, 86, 229, 218, 6, 2, 79, - 70, 18, 62, 65, 42, 73, 214, 255, 9, 79, 186, 186, 11, 85, 195, 9, 69, 6, - 178, 186, 21, 73, 214, 79, 80, 3, 84, 6, 232, 175, 12, 3, 71, 72, 84, - 150, 156, 9, 86, 143, 39, 67, 74, 170, 1, 65, 86, 67, 46, 69, 62, 72, - 166, 1, 73, 46, 76, 62, 80, 106, 84, 234, 159, 17, 87, 182, 145, 1, 78, - 190, 50, 79, 182, 155, 1, 77, 134, 100, 81, 254, 32, 75, 235, 47, 85, 6, - 192, 168, 3, 9, 67, 82, 73, 70, 73, 67, 73, 65, 76, 138, 195, 18, 76, - 175, 28, 89, 4, 150, 198, 10, 82, 229, 186, 5, 2, 72, 79, 8, 242, 203, - 20, 67, 234, 49, 65, 146, 8, 76, 143, 129, 1, 69, 10, 18, 69, 39, 79, 4, - 138, 253, 20, 76, 147, 137, 1, 69, 6, 40, 4, 82, 84, 32, 84, 175, 233, - 21, 79, 4, 222, 190, 15, 72, 137, 200, 4, 7, 65, 73, 76, 69, 68, 32, 66, - 4, 148, 162, 17, 2, 67, 75, 235, 223, 4, 76, 6, 26, 65, 227, 199, 21, 73, - 4, 250, 237, 21, 86, 199, 21, 83, 10, 50, 69, 34, 73, 178, 177, 18, 82, - 207, 130, 3, 79, 4, 170, 203, 21, 65, 171, 22, 69, 2, 163, 231, 21, 82, - 12, 34, 69, 34, 79, 167, 245, 20, 65, 4, 202, 242, 21, 65, 219, 16, 80, - 6, 26, 80, 151, 236, 21, 78, 5, 239, 177, 21, 80, 30, 82, 65, 98, 73, 34, - 79, 38, 82, 52, 2, 85, 82, 32, 2, 87, 79, 183, 176, 21, 69, 6, 38, 78, - 182, 224, 20, 66, 139, 24, 76, 2, 33, 6, 78, 69, 68, 32, 76, 69, 2, 159, - 167, 13, 65, 4, 190, 195, 21, 71, 143, 39, 76, 4, 150, 160, 18, 78, 231, - 225, 1, 79, 6, 214, 159, 16, 73, 186, 178, 4, 65, 251, 151, 1, 69, 4, - 242, 184, 20, 66, 147, 38, 84, 5, 215, 214, 18, 32, 4, 170, 157, 12, 65, - 129, 136, 5, 3, 73, 76, 76, 26, 58, 65, 110, 69, 42, 72, 32, 2, 73, 78, - 30, 79, 39, 82, 6, 32, 2, 76, 75, 135, 193, 21, 84, 5, 11, 32, 2, 11, 69, - 2, 17, 2, 78, 67, 2, 201, 158, 17, 2, 76, 79, 4, 184, 174, 21, 2, 65, 80, - 183, 51, 83, 4, 198, 185, 20, 73, 223, 48, 69, 4, 210, 253, 21, 68, 3, - 69, 4, 130, 182, 20, 77, 159, 196, 1, 82, 4, 166, 172, 21, 79, 227, 80, - 65, 2, 197, 228, 19, 2, 69, 76, 184, 1, 92, 2, 76, 69, 148, 2, 5, 83, 73, - 71, 78, 32, 238, 249, 16, 65, 174, 9, 86, 175, 136, 3, 68, 110, 44, 5, - 84, 84, 69, 82, 32, 235, 185, 21, 78, 108, 214, 128, 17, 76, 38, 78, 174, - 90, 82, 206, 55, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, 158, 144, 1, - 79, 182, 56, 73, 202, 190, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, - 80, 162, 7, 69, 222, 61, 70, 2, 72, 2, 77, 3, 89, 22, 94, 67, 138, 1, 83, - 138, 145, 18, 65, 170, 1, 78, 230, 179, 1, 74, 150, 3, 85, 211, 235, 1, - 86, 4, 100, 19, 79, 77, 66, 73, 78, 73, 78, 71, 32, 65, 78, 85, 83, 86, - 65, 82, 65, 32, 65, 255, 145, 18, 65, 2, 185, 131, 19, 3, 66, 79, 86, 4, - 244, 135, 12, 6, 80, 65, 67, 73, 78, 71, 155, 249, 4, 73, 162, 2, 62, 32, - 173, 11, 10, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 154, 2, 140, 1, 7, - 76, 69, 84, 84, 69, 82, 32, 242, 9, 86, 236, 159, 18, 10, 68, 73, 71, 82, - 65, 80, 72, 32, 75, 79, 234, 175, 1, 73, 247, 138, 1, 77, 146, 2, 186, 1, - 65, 178, 1, 66, 106, 77, 186, 2, 78, 54, 83, 226, 1, 68, 2, 71, 2, 72, 2, - 75, 2, 80, 2, 82, 2, 84, 2, 86, 2, 90, 126, 87, 46, 89, 154, 236, 21, 69, - 2, 73, 2, 79, 3, 85, 19, 60, 4, 73, 78, 85, 32, 45, 7, 82, 67, 72, 65, - 73, 67, 32, 8, 130, 7, 84, 142, 214, 21, 67, 215, 22, 80, 8, 38, 89, 170, - 206, 21, 87, 235, 36, 69, 4, 142, 243, 21, 69, 3, 73, 20, 50, 73, 194, - 242, 21, 65, 2, 69, 2, 79, 3, 85, 13, 253, 4, 9, 68, 65, 75, 85, 79, 78, - 32, 78, 71, 36, 50, 73, 218, 241, 21, 65, 2, 69, 2, 79, 3, 85, 29, 29, 5, - 78, 78, 65, 78, 32, 26, 96, 15, 78, 65, 83, 65, 76, 73, 90, 69, 68, 32, - 84, 79, 78, 69, 45, 69, 5, 84, 79, 78, 69, 45, 14, 210, 240, 21, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 12, 142, 240, 21, 50, 2, 51, 2, - 52, 2, 53, 2, 55, 3, 56, 13, 210, 239, 21, 65, 2, 69, 2, 73, 2, 79, 3, - 85, 76, 76, 5, 77, 65, 76, 76, 32, 210, 238, 21, 65, 2, 69, 2, 73, 2, 79, - 3, 85, 66, 142, 1, 72, 2, 82, 54, 75, 46, 84, 30, 87, 46, 89, 254, 156, - 18, 78, 242, 142, 3, 83, 198, 27, 77, 234, 36, 65, 2, 69, 2, 73, 2, 79, - 3, 85, 10, 190, 237, 21, 65, 2, 69, 2, 73, 2, 79, 3, 85, 8, 138, 237, 21, - 65, 2, 69, 2, 79, 3, 85, 4, 222, 236, 21, 79, 3, 85, 8, 194, 236, 21, 65, - 2, 69, 2, 73, 3, 79, 6, 150, 236, 21, 65, 2, 79, 3, 85, 2, 189, 207, 19, - 5, 79, 73, 67, 69, 68, 8, 52, 5, 68, 79, 85, 66, 76, 22, 80, 38, 83, 35, - 86, 2, 251, 216, 7, 69, 2, 89, 6, 82, 79, 76, 79, 78, 71, 2, 29, 5, 69, - 77, 73, 45, 86, 2, 21, 3, 79, 73, 67, 2, 33, 6, 69, 68, 32, 83, 79, 85, - 2, 239, 157, 21, 78, 172, 1, 228, 1, 7, 76, 69, 84, 84, 69, 82, 32, 132, - 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 148, 3, 5, 83, 73, - 71, 78, 32, 92, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 178, 236, - 11, 68, 205, 183, 9, 3, 67, 79, 78, 94, 230, 1, 65, 214, 128, 4, 74, 154, - 253, 13, 68, 46, 84, 146, 4, 86, 214, 20, 85, 210, 200, 1, 73, 158, 190, - 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, - 77, 2, 82, 2, 87, 2, 89, 186, 2, 69, 3, 79, 7, 150, 230, 21, 65, 3, 73, - 22, 122, 67, 90, 68, 58, 70, 54, 83, 20, 12, 65, 76, 84, 69, 82, 78, 65, - 84, 69, 32, 83, 69, 149, 210, 20, 4, 84, 82, 73, 80, 4, 56, 8, 76, 79, - 83, 73, 78, 71, 32, 83, 203, 149, 20, 73, 2, 141, 219, 20, 2, 80, 73, 4, - 11, 79, 4, 132, 211, 20, 2, 85, 66, 151, 145, 1, 84, 4, 204, 148, 20, 5, - 73, 76, 76, 69, 68, 175, 54, 76, 6, 18, 69, 23, 80, 2, 203, 196, 10, 67, - 4, 156, 241, 3, 2, 65, 67, 195, 232, 16, 73, 10, 192, 238, 5, 2, 82, 69, - 166, 204, 1, 75, 218, 193, 10, 67, 226, 182, 1, 86, 223, 234, 1, 65, 20, - 66, 65, 134, 255, 3, 86, 170, 147, 14, 69, 2, 85, 211, 200, 1, 73, 6, - 244, 152, 16, 8, 76, 84, 69, 82, 78, 65, 84, 69, 210, 200, 5, 65, 3, 73, - 96, 148, 1, 7, 76, 69, 84, 84, 69, 82, 32, 204, 1, 5, 83, 73, 71, 78, 32, - 44, 5, 84, 79, 78, 69, 32, 92, 6, 86, 79, 87, 69, 76, 32, 247, 238, 19, - 68, 56, 154, 206, 16, 72, 190, 173, 1, 79, 230, 236, 1, 78, 246, 175, 1, - 75, 2, 80, 2, 83, 2, 84, 254, 68, 66, 2, 67, 2, 68, 2, 71, 2, 76, 2, 77, - 2, 82, 2, 86, 2, 87, 2, 89, 2, 90, 186, 2, 65, 3, 73, 4, 242, 204, 4, 67, - 201, 142, 17, 2, 83, 72, 6, 36, 5, 67, 65, 76, 89, 65, 23, 80, 5, 17, 2, - 32, 80, 2, 197, 155, 17, 3, 76, 79, 80, 10, 250, 156, 21, 69, 2, 85, 151, - 64, 79, 34, 24, 2, 76, 86, 23, 89, 2, 175, 230, 19, 73, 33, 52, 5, 66, - 79, 65, 82, 68, 37, 4, 67, 65, 80, 32, 5, 213, 172, 18, 4, 32, 65, 78, - 68, 26, 142, 197, 16, 78, 146, 134, 3, 65, 162, 34, 68, 171, 29, 84, 186, - 13, 202, 1, 65, 212, 9, 18, 73, 84, 65, 78, 32, 83, 77, 65, 76, 76, 32, - 83, 67, 82, 73, 80, 84, 32, 184, 3, 4, 77, 69, 82, 32, 204, 25, 5, 79, - 74, 75, 73, 32, 145, 6, 8, 85, 68, 65, 87, 65, 68, 73, 32, 138, 1, 56, 8, - 82, 79, 83, 72, 84, 72, 73, 32, 143, 180, 21, 78, 136, 1, 220, 1, 4, 68, - 73, 71, 73, 20, 7, 76, 69, 84, 84, 69, 82, 32, 144, 2, 7, 78, 85, 77, 66, - 69, 82, 32, 36, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 136, - 2, 5, 83, 73, 71, 78, 32, 198, 1, 86, 207, 155, 8, 70, 8, 203, 193, 15, - 84, 74, 182, 1, 75, 42, 84, 150, 229, 6, 78, 250, 135, 11, 68, 170, 160, - 3, 83, 82, 66, 2, 67, 2, 71, 2, 80, 2, 86, 254, 68, 72, 2, 74, 2, 76, 2, - 77, 2, 82, 2, 89, 2, 90, 187, 2, 65, 6, 166, 211, 21, 72, 2, 75, 187, 2, - 65, 12, 182, 237, 17, 84, 202, 229, 3, 72, 187, 2, 65, 8, 146, 149, 14, - 84, 159, 151, 2, 79, 18, 70, 67, 74, 68, 66, 76, 36, 5, 77, 65, 78, 71, - 65, 175, 131, 20, 83, 4, 26, 82, 135, 133, 20, 73, 2, 177, 132, 19, 6, - 69, 83, 67, 69, 78, 84, 6, 26, 79, 191, 254, 11, 65, 4, 134, 254, 11, 85, - 179, 213, 9, 84, 4, 246, 188, 6, 79, 199, 168, 7, 73, 2, 179, 153, 20, - 76, 12, 72, 2, 66, 65, 22, 67, 20, 2, 68, 79, 162, 162, 19, 86, 223, 234, - 1, 65, 2, 155, 213, 20, 82, 2, 255, 243, 5, 65, 4, 56, 8, 85, 66, 76, 69, - 32, 82, 73, 78, 203, 170, 19, 84, 2, 199, 170, 19, 71, 14, 44, 5, 79, 87, - 69, 76, 32, 243, 158, 19, 73, 12, 44, 5, 83, 73, 71, 78, 32, 203, 142, - 21, 76, 10, 242, 237, 3, 86, 198, 226, 17, 69, 2, 73, 2, 79, 3, 85, 174, - 7, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 175, 167, 7, - 70, 172, 7, 22, 66, 147, 1, 67, 128, 4, 146, 214, 7, 48, 2, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, - 2, 69, 3, 70, 172, 3, 122, 68, 138, 212, 7, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 3, 67, 12, 254, 204, - 21, 48, 2, 49, 2, 50, 2, 51, 2, 52, 3, 53, 246, 2, 202, 1, 67, 240, 2, - 18, 73, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 86, 79, 87, 69, 76, - 32, 220, 2, 7, 76, 69, 84, 84, 69, 82, 32, 254, 2, 83, 208, 12, 6, 86, - 79, 87, 69, 76, 32, 195, 199, 19, 68, 70, 176, 1, 20, 79, 78, 83, 79, 78, - 65, 78, 84, 32, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, 233, 232, 16, - 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, 32, 82, 73, - 68, 138, 1, 78, 154, 4, 67, 2, 75, 90, 80, 74, 84, 54, 68, 2, 76, 198, - 250, 20, 83, 146, 41, 77, 2, 82, 2, 86, 2, 89, 190, 28, 66, 3, 72, 8, - 190, 169, 21, 71, 2, 89, 246, 30, 65, 3, 79, 42, 82, 81, 196, 1, 11, 83, - 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, 50, 76, 3, 82, 26, 82, 65, 44, 6, - 79, 79, 32, 84, 89, 80, 34, 85, 198, 191, 19, 73, 207, 134, 2, 69, 8, - 218, 198, 21, 65, 2, 73, 2, 81, 3, 85, 4, 11, 69, 4, 159, 231, 9, 32, 9, - 214, 152, 7, 85, 187, 173, 14, 75, 8, 18, 81, 31, 82, 4, 214, 197, 21, - 69, 3, 85, 4, 195, 210, 17, 89, 70, 138, 1, 67, 2, 75, 42, 78, 50, 80, - 30, 83, 46, 84, 54, 68, 2, 76, 214, 163, 21, 77, 2, 82, 2, 86, 2, 89, - 190, 28, 66, 2, 72, 3, 81, 8, 210, 1, 72, 202, 194, 21, 65, 3, 79, 8, - 254, 164, 21, 71, 2, 78, 2, 89, 247, 30, 79, 6, 122, 72, 203, 194, 21, - 79, 6, 178, 164, 21, 83, 190, 28, 72, 187, 2, 65, 12, 50, 72, 0, 2, 84, - 72, 202, 194, 21, 65, 3, 79, 4, 198, 194, 21, 65, 3, 79, 130, 1, 60, 4, + 135, 128, 23, 82, 4, 11, 65, 5, 11, 32, 2, 11, 77, 2, 11, 65, 2, 11, 72, + 2, 141, 174, 17, 2, 65, 80, 28, 40, 3, 68, 65, 32, 165, 3, 2, 78, 71, 24, + 174, 1, 65, 90, 76, 86, 80, 140, 144, 7, 10, 84, 73, 82, 84, 65, 32, 84, + 85, 77, 69, 130, 253, 11, 87, 164, 192, 2, 7, 73, 83, 69, 78, 45, 73, 83, + 141, 209, 1, 3, 77, 65, 68, 6, 44, 3, 68, 69, 71, 169, 145, 22, 2, 78, + 68, 5, 11, 32, 2, 157, 220, 22, 2, 65, 68, 6, 38, 85, 129, 159, 23, 3, + 73, 78, 71, 4, 160, 183, 18, 2, 78, 71, 207, 241, 3, 72, 4, 38, 73, 253, + 175, 18, 3, 65, 78, 71, 2, 145, 185, 16, 3, 83, 69, 76, 4, 220, 141, 13, + 5, 82, 65, 78, 71, 75, 135, 195, 9, 75, 10, 104, 5, 67, 69, 67, 65, 75, + 144, 197, 5, 5, 80, 65, 78, 89, 65, 176, 2, 3, 87, 73, 71, 227, 240, 10, + 76, 5, 209, 158, 18, 3, 32, 84, 69, 18, 116, 4, 83, 85, 75, 85, 42, 84, + 64, 4, 87, 85, 76, 85, 238, 182, 16, 80, 213, 218, 1, 7, 68, 73, 82, 71, + 65, 32, 77, 5, 193, 164, 22, 5, 32, 77, 69, 78, 68, 6, 26, 65, 171, 184, + 16, 79, 4, 146, 184, 16, 82, 187, 148, 6, 76, 5, 25, 4, 32, 77, 69, 76, + 2, 247, 153, 23, 73, 4, 36, 3, 76, 76, 89, 255, 216, 19, 65, 2, 219, 187, + 19, 70, 4, 180, 236, 21, 2, 89, 83, 171, 96, 73, 6, 168, 188, 14, 2, 71, + 71, 182, 253, 4, 80, 199, 195, 3, 78, 186, 25, 74, 65, 178, 78, 69, 218, + 1, 72, 150, 50, 73, 198, 6, 78, 50, 79, 111, 82, 200, 10, 200, 1, 5, 73, + 84, 72, 73, 32, 234, 4, 78, 188, 45, 6, 84, 65, 75, 65, 78, 65, 208, 13, + 3, 87, 73, 32, 220, 8, 7, 89, 65, 72, 32, 76, 73, 32, 204, 195, 4, 6, 75, + 84, 79, 86, 73, 75, 251, 223, 17, 65, 136, 1, 204, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 178, 2, 83, 138, 116, 68, 208, 237, 3, 2, 86, 79, 160, 137, + 3, 11, 78, 85, 77, 66, 69, 82, 32, 83, 73, 71, 78, 230, 191, 9, 65, 229, + 211, 1, 6, 69, 78, 85, 77, 69, 82, 90, 214, 1, 68, 242, 187, 19, 65, 150, + 1, 84, 230, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, + 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, 82, 138, 69, 72, 2, 76, 2, 77, 2, 86, + 2, 89, 186, 2, 69, 3, 79, 10, 38, 68, 178, 147, 23, 72, 187, 2, 65, 6, + 166, 155, 21, 68, 138, 248, 1, 72, 187, 2, 65, 12, 40, 4, 73, 71, 78, 32, + 203, 148, 11, 69, 10, 202, 128, 19, 67, 98, 78, 234, 206, 3, 65, 239, 1, + 86, 230, 4, 42, 71, 241, 39, 5, 78, 65, 68, 65, 32, 174, 3, 76, 11, 88, + 73, 32, 82, 65, 68, 73, 67, 65, 76, 32, 169, 204, 20, 2, 65, 82, 172, 3, + 130, 2, 65, 94, 66, 230, 2, 67, 150, 2, 68, 158, 3, 69, 198, 1, 70, 138, + 2, 71, 146, 1, 72, 210, 2, 73, 76, 2, 74, 65, 34, 75, 30, 76, 230, 1, 77, + 254, 1, 78, 62, 79, 102, 80, 110, 82, 166, 1, 83, 230, 6, 84, 226, 2, 86, + 50, 87, 210, 2, 89, 199, 227, 21, 85, 10, 56, 2, 82, 82, 186, 228, 21, + 71, 182, 102, 78, 207, 47, 88, 4, 234, 146, 22, 79, 207, 1, 73, 34, 58, + 65, 38, 73, 46, 76, 54, 79, 102, 82, 203, 196, 21, 69, 4, 186, 196, 5, + 77, 163, 254, 13, 68, 6, 146, 216, 21, 84, 218, 113, 82, 163, 70, 71, 6, + 134, 217, 19, 79, 234, 134, 2, 65, 159, 153, 1, 85, 10, 62, 76, 194, 242, + 22, 65, 218, 5, 78, 142, 5, 68, 203, 17, 87, 2, 209, 218, 3, 4, 84, 32, + 79, 70, 6, 42, 73, 198, 241, 9, 65, 187, 180, 11, 85, 2, 191, 243, 13, + 83, 28, 58, 65, 70, 76, 74, 79, 214, 174, 10, 72, 247, 240, 10, 73, 6, + 38, 85, 250, 240, 22, 82, 219, 5, 86, 2, 129, 232, 21, 2, 76, 68, 8, 42, + 65, 206, 183, 7, 73, 131, 193, 14, 79, 4, 218, 140, 23, 78, 3, 87, 10, + 250, 233, 21, 82, 128, 2, 2, 77, 80, 218, 98, 86, 134, 5, 76, 235, 56, + 87, 34, 38, 69, 38, 73, 98, 79, 195, 1, 82, 4, 186, 137, 21, 65, 183, + 201, 1, 69, 8, 38, 83, 198, 244, 13, 86, 175, 2, 80, 4, 132, 170, 19, 5, + 84, 73, 78, 71, 85, 251, 224, 3, 72, 16, 94, 84, 76, 3, 85, 66, 76, 222, + 246, 12, 87, 136, 194, 9, 2, 32, 78, 222, 23, 79, 223, 56, 71, 7, 25, 4, + 84, 69, 68, 32, 4, 184, 180, 7, 3, 67, 76, 73, 235, 145, 15, 84, 2, 207, + 195, 3, 69, 6, 230, 208, 21, 65, 202, 167, 1, 85, 219, 16, 89, 20, 106, + 65, 38, 78, 32, 3, 86, 69, 78, 152, 252, 8, 6, 77, 66, 82, 79, 73, 68, + 206, 224, 13, 73, 243, 19, 89, 6, 234, 221, 16, 82, 131, 170, 6, 84, 4, + 178, 25, 67, 171, 176, 22, 84, 5, 175, 182, 22, 73, 28, 74, 65, 50, 73, + 62, 76, 38, 82, 170, 20, 69, 218, 160, 22, 79, 223, 23, 85, 6, 246, 242, + 13, 84, 254, 252, 8, 67, 131, 22, 78, 8, 142, 232, 12, 69, 210, 243, 9, + 71, 230, 19, 82, 199, 21, 83, 4, 226, 190, 21, 85, 251, 198, 1, 89, 4, + 168, 215, 21, 2, 65, 71, 187, 173, 1, 79, 14, 58, 79, 52, 2, 82, 65, 150, + 154, 19, 72, 147, 163, 2, 65, 7, 174, 190, 22, 76, 241, 34, 5, 32, 83, + 76, 79, 87, 4, 186, 180, 22, 73, 135, 18, 83, 22, 58, 65, 142, 1, 69, 60, + 5, 73, 68, 73, 78, 71, 19, 79, 8, 38, 76, 250, 188, 22, 78, 199, 13, 73, + 4, 34, 70, 137, 236, 21, 2, 66, 69, 2, 41, 8, 32, 84, 82, 69, 69, 32, 84, + 82, 2, 187, 161, 19, 85, 6, 26, 65, 143, 130, 23, 77, 4, 234, 229, 22, + 82, 175, 28, 68, 2, 195, 19, 32, 6, 26, 82, 183, 254, 22, 79, 4, 246, + 234, 22, 83, 215, 22, 78, 6, 26, 78, 191, 234, 22, 67, 4, 26, 83, 231, + 255, 22, 67, 2, 131, 242, 21, 69, 4, 138, 234, 22, 68, 215, 22, 82, 2, + 205, 172, 5, 2, 78, 73, 22, 46, 65, 34, 69, 78, 73, 41, 3, 79, 78, 71, 4, + 158, 233, 22, 77, 191, 19, 67, 8, 38, 65, 210, 193, 22, 71, 199, 58, 69, + 4, 218, 235, 13, 84, 211, 147, 9, 70, 6, 178, 232, 22, 70, 2, 78, 215, + 22, 68, 5, 161, 196, 11, 3, 32, 83, 84, 22, 42, 69, 34, 73, 46, 79, 235, + 173, 22, 65, 4, 166, 174, 22, 76, 195, 51, 65, 4, 188, 235, 11, 2, 78, + 73, 151, 179, 9, 76, 12, 34, 82, 34, 85, 167, 173, 22, 79, 4, 234, 219, + 21, 84, 183, 80, 78, 6, 26, 78, 219, 251, 22, 84, 4, 146, 208, 21, 84, + 215, 172, 1, 68, 6, 26, 79, 255, 223, 22, 69, 4, 210, 229, 22, 83, 215, + 22, 84, 10, 40, 2, 78, 69, 22, 80, 171, 181, 22, 76, 5, 235, 150, 6, 83, + 4, 198, 171, 10, 80, 195, 189, 2, 69, 10, 54, 82, 202, 224, 21, 76, 166, + 1, 79, 175, 152, 1, 73, 4, 156, 199, 12, 2, 73, 86, 225, 254, 6, 2, 79, + 70, 18, 62, 65, 42, 73, 242, 166, 10, 79, 130, 131, 12, 85, 195, 9, 69, + 6, 150, 170, 22, 73, 226, 79, 80, 3, 84, 6, 240, 230, 12, 3, 71, 72, 84, + 242, 212, 9, 86, 155, 39, 67, 74, 170, 1, 65, 86, 67, 54, 69, 62, 72, + 178, 1, 73, 46, 76, 62, 80, 106, 84, 186, 227, 17, 87, 166, 155, 1, 78, + 234, 63, 79, 150, 173, 1, 77, 254, 102, 81, 254, 32, 75, 247, 47, 85, 6, + 228, 193, 3, 9, 67, 82, 73, 70, 73, 67, 73, 65, 76, 214, 153, 19, 76, + 175, 28, 89, 4, 248, 238, 10, 2, 82, 73, 161, 215, 5, 2, 72, 79, 8, 158, + 186, 21, 67, 142, 51, 65, 146, 8, 76, 167, 129, 1, 69, 10, 18, 69, 39, + 79, 4, 218, 236, 21, 76, 171, 137, 1, 69, 6, 40, 4, 82, 84, 32, 84, 151, + 217, 22, 79, 4, 44, 5, 65, 73, 76, 69, 68, 175, 249, 15, 72, 2, 145, 243, + 20, 2, 32, 66, 4, 228, 229, 17, 2, 67, 75, 247, 139, 5, 76, 6, 26, 65, + 179, 183, 22, 73, 4, 214, 221, 22, 86, 199, 21, 83, 10, 50, 69, 34, 73, + 242, 254, 18, 82, 223, 164, 3, 79, 4, 250, 186, 22, 65, 183, 22, 69, 2, + 255, 214, 22, 82, 12, 34, 69, 34, 79, 235, 228, 21, 65, 4, 166, 226, 22, + 65, 219, 16, 80, 6, 26, 80, 243, 219, 22, 78, 5, 191, 161, 22, 80, 30, + 82, 65, 98, 73, 34, 79, 38, 82, 52, 2, 85, 82, 32, 2, 87, 79, 135, 160, + 22, 69, 6, 38, 78, 142, 205, 21, 66, 247, 26, 76, 2, 33, 6, 78, 69, 68, + 32, 76, 69, 2, 179, 221, 13, 65, 4, 142, 179, 22, 71, 155, 39, 76, 4, + 198, 229, 18, 78, 135, 137, 2, 79, 6, 250, 228, 16, 73, 206, 219, 4, 65, + 159, 153, 1, 69, 4, 130, 166, 21, 66, 219, 37, 84, 5, 239, 176, 19, 32, + 4, 162, 212, 12, 65, 217, 148, 5, 3, 73, 76, 76, 26, 58, 65, 110, 69, 42, + 72, 32, 2, 73, 78, 30, 79, 39, 82, 6, 32, 2, 76, 75, 215, 176, 22, 84, 5, + 11, 32, 2, 11, 69, 2, 17, 2, 78, 67, 2, 153, 226, 17, 2, 76, 79, 4, 136, + 158, 22, 2, 65, 80, 195, 51, 83, 4, 214, 166, 21, 73, 147, 51, 69, 4, + 174, 237, 22, 68, 3, 69, 4, 146, 163, 21, 77, 235, 198, 1, 82, 4, 246, + 155, 22, 79, 239, 80, 65, 2, 201, 206, 20, 2, 69, 76, 184, 1, 92, 2, 76, + 69, 148, 2, 5, 83, 73, 71, 78, 32, 150, 190, 17, 65, 250, 8, 86, 147, + 177, 3, 68, 110, 44, 5, 84, 84, 69, 82, 32, 187, 169, 22, 78, 108, 234, + 196, 17, 78, 150, 204, 1, 65, 38, 68, 46, 76, 38, 82, 34, 84, 46, 86, + 186, 5, 85, 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 82, 66, 2, 67, + 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 70, 2, 72, 2, 77, 3, 89, + 22, 94, 67, 138, 1, 83, 246, 211, 18, 78, 242, 60, 65, 174, 158, 1, 74, + 150, 3, 85, 167, 242, 1, 86, 4, 100, 19, 79, 77, 66, 73, 78, 73, 78, 71, + 32, 65, 78, 85, 83, 86, 65, 82, 65, 32, 65, 195, 211, 18, 65, 2, 225, + 230, 19, 3, 66, 79, 86, 4, 224, 190, 12, 6, 80, 65, 67, 73, 78, 71, 163, + 134, 5, 73, 162, 2, 62, 32, 173, 11, 10, 45, 72, 73, 82, 65, 71, 65, 78, + 65, 32, 154, 2, 140, 1, 7, 76, 69, 84, 84, 69, 82, 32, 242, 9, 86, 148, + 237, 18, 10, 68, 73, 71, 82, 65, 80, 72, 32, 75, 79, 246, 203, 1, 73, + 135, 145, 1, 77, 146, 2, 186, 1, 65, 178, 1, 66, 106, 77, 186, 2, 78, 54, + 83, 226, 1, 68, 2, 71, 2, 72, 2, 75, 2, 80, 2, 82, 2, 84, 2, 86, 2, 90, + 126, 87, 46, 89, 246, 219, 22, 69, 2, 73, 2, 79, 3, 85, 19, 60, 4, 73, + 78, 85, 32, 45, 7, 82, 67, 72, 65, 73, 67, 32, 8, 130, 7, 84, 234, 197, + 22, 67, 215, 22, 80, 8, 38, 89, 134, 190, 22, 87, 235, 36, 69, 4, 234, + 226, 22, 69, 3, 73, 20, 50, 73, 158, 226, 22, 65, 2, 69, 2, 79, 3, 85, + 13, 253, 4, 9, 68, 65, 75, 85, 79, 78, 32, 78, 71, 36, 50, 73, 182, 225, + 22, 65, 2, 69, 2, 79, 3, 85, 29, 29, 5, 78, 78, 65, 78, 32, 26, 96, 15, + 78, 65, 83, 65, 76, 73, 90, 69, 68, 32, 84, 79, 78, 69, 45, 69, 5, 84, + 79, 78, 69, 45, 14, 174, 224, 22, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, + 3, 56, 12, 234, 223, 22, 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 13, 174, + 223, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 76, 76, 5, 77, 65, 76, 76, 32, + 174, 222, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 66, 142, 1, 72, 2, 82, 54, + 75, 46, 84, 30, 87, 46, 89, 206, 136, 19, 78, 242, 146, 3, 83, 210, 27, + 77, 234, 36, 65, 2, 69, 2, 73, 2, 79, 3, 85, 10, 154, 221, 22, 65, 2, 69, + 2, 73, 2, 79, 3, 85, 8, 230, 220, 22, 65, 2, 69, 2, 79, 3, 85, 4, 186, + 220, 22, 79, 3, 85, 8, 158, 220, 22, 65, 2, 69, 2, 73, 3, 79, 6, 242, + 219, 22, 65, 2, 79, 3, 85, 2, 241, 184, 20, 5, 79, 73, 67, 69, 68, 8, 52, + 5, 68, 79, 85, 66, 76, 22, 80, 38, 83, 35, 86, 2, 139, 250, 7, 69, 2, 89, + 6, 82, 79, 76, 79, 78, 71, 2, 29, 5, 69, 77, 73, 45, 86, 2, 21, 3, 79, + 73, 67, 2, 33, 6, 69, 68, 32, 83, 79, 85, 2, 191, 141, 22, 78, 174, 1, + 216, 1, 7, 76, 69, 84, 84, 69, 82, 32, 128, 2, 12, 80, 85, 78, 67, 84, + 85, 65, 84, 73, 79, 78, 32, 148, 3, 5, 83, 73, 71, 78, 32, 88, 11, 86, + 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 170, 163, 12, 68, 195, 209, 6, + 67, 94, 226, 1, 65, 166, 160, 4, 74, 230, 219, 14, 68, 114, 84, 230, 5, + 85, 22, 86, 186, 201, 1, 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, + 71, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, 186, 2, + 69, 3, 79, 7, 130, 214, 22, 65, 3, 73, 22, 122, 67, 90, 68, 58, 70, 54, + 83, 20, 12, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 83, 69, 233, 193, 21, + 4, 84, 82, 73, 80, 4, 56, 8, 76, 79, 83, 73, 78, 71, 32, 83, 195, 130, + 21, 73, 2, 225, 202, 21, 2, 80, 73, 4, 11, 79, 4, 216, 194, 21, 2, 85, + 66, 175, 145, 1, 84, 4, 196, 129, 21, 5, 73, 76, 76, 69, 68, 139, 57, 76, + 6, 18, 69, 23, 80, 2, 195, 237, 10, 67, 4, 236, 144, 4, 2, 65, 67, 199, + 184, 17, 73, 12, 242, 219, 7, 75, 162, 226, 10, 67, 98, 78, 162, 59, 82, + 154, 162, 1, 86, 179, 241, 1, 65, 20, 66, 65, 214, 158, 4, 86, 190, 223, + 14, 69, 2, 85, 207, 201, 1, 73, 6, 148, 222, 16, 8, 76, 84, 69, 82, 78, + 65, 84, 69, 162, 243, 5, 65, 3, 73, 96, 148, 1, 7, 76, 69, 84, 84, 69, + 82, 32, 200, 1, 5, 83, 73, 71, 78, 32, 44, 5, 84, 79, 78, 69, 32, 92, 6, + 86, 79, 87, 69, 76, 32, 231, 219, 20, 68, 56, 138, 189, 18, 79, 134, 7, + 72, 154, 145, 2, 78, 238, 178, 1, 75, 2, 80, 2, 83, 2, 84, 138, 69, 66, + 2, 67, 2, 68, 2, 71, 2, 76, 2, 77, 2, 82, 2, 86, 2, 87, 2, 89, 2, 90, + 186, 2, 65, 3, 73, 4, 182, 236, 4, 67, 249, 222, 17, 2, 83, 72, 6, 36, 5, + 67, 65, 76, 89, 65, 23, 80, 5, 17, 2, 32, 80, 2, 173, 223, 17, 3, 76, 79, + 80, 10, 226, 140, 22, 69, 2, 85, 163, 64, 79, 36, 24, 2, 76, 86, 23, 89, + 2, 159, 211, 20, 73, 35, 68, 5, 66, 79, 65, 82, 68, 36, 4, 67, 65, 80, + 32, 187, 244, 2, 72, 5, 245, 134, 19, 4, 32, 65, 78, 68, 26, 254, 136, + 17, 78, 238, 174, 3, 65, 182, 34, 68, 187, 29, 84, 188, 13, 202, 1, 65, + 212, 9, 18, 73, 84, 65, 78, 32, 83, 77, 65, 76, 76, 32, 83, 67, 82, 73, + 80, 84, 32, 204, 3, 4, 77, 69, 82, 32, 204, 25, 5, 79, 74, 75, 73, 32, + 141, 6, 8, 85, 68, 65, 87, 65, 68, 73, 32, 138, 1, 56, 8, 82, 79, 83, 72, + 84, 72, 73, 32, 243, 163, 22, 78, 136, 1, 220, 1, 4, 68, 73, 71, 73, 20, + 7, 76, 69, 84, 84, 69, 82, 32, 144, 2, 7, 78, 85, 77, 66, 69, 82, 32, 36, + 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 136, 2, 5, 83, 73, + 71, 78, 32, 198, 1, 86, 183, 194, 8, 70, 8, 223, 134, 16, 84, 74, 182, 1, + 75, 42, 84, 234, 133, 7, 78, 186, 229, 11, 68, 238, 145, 3, 83, 82, 66, + 2, 67, 2, 71, 2, 80, 2, 86, 138, 69, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, + 89, 2, 90, 187, 2, 65, 6, 138, 195, 22, 72, 2, 75, 187, 2, 65, 12, 142, + 236, 18, 84, 214, 214, 3, 72, 187, 2, 65, 8, 162, 203, 14, 84, 227, 165, + 2, 79, 18, 70, 67, 74, 68, 66, 76, 36, 5, 77, 65, 78, 71, 65, 159, 240, + 20, 83, 4, 26, 82, 247, 241, 20, 73, 2, 145, 162, 21, 6, 69, 83, 67, 69, + 78, 84, 6, 26, 79, 251, 230, 18, 65, 4, 194, 230, 18, 85, 219, 220, 3, + 84, 4, 226, 220, 6, 79, 227, 190, 7, 73, 2, 215, 135, 21, 76, 12, 72, 2, + 66, 65, 22, 67, 20, 2, 68, 79, 166, 139, 20, 86, 179, 241, 1, 65, 2, 239, + 196, 21, 82, 2, 203, 147, 6, 65, 4, 56, 8, 85, 66, 76, 69, 32, 82, 73, + 78, 207, 147, 20, 84, 2, 203, 147, 20, 71, 14, 44, 5, 79, 87, 69, 76, 32, + 247, 135, 20, 73, 12, 44, 5, 83, 73, 71, 78, 32, 163, 254, 21, 76, 10, + 182, 141, 4, 86, 230, 178, 18, 69, 2, 73, 2, 79, 3, 85, 176, 7, 72, 12, + 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 179, 200, 7, 70, 174, 7, + 22, 66, 147, 1, 67, 128, 4, 170, 247, 7, 48, 2, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, + 70, 174, 3, 142, 1, 68, 142, 245, 7, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 163, 198, 13, 70, + 12, 206, 188, 22, 48, 2, 49, 2, 50, 2, 51, 2, 52, 3, 53, 246, 2, 202, 1, + 67, 240, 2, 18, 73, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 86, 79, + 87, 69, 76, 32, 220, 2, 7, 76, 69, 84, 84, 69, 82, 32, 254, 2, 83, 208, + 12, 6, 86, 79, 87, 69, 76, 32, 143, 180, 20, 68, 70, 176, 1, 20, 79, 78, + 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, + 173, 172, 17, 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, + 32, 82, 73, 68, 138, 1, 78, 154, 4, 67, 2, 75, 90, 80, 74, 84, 54, 68, 2, + 76, 138, 234, 21, 83, 158, 41, 77, 2, 82, 2, 86, 2, 89, 190, 28, 66, 3, + 72, 8, 142, 153, 22, 71, 2, 89, 246, 30, 65, 3, 79, 42, 82, 81, 196, 1, + 11, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, 50, 76, 3, 82, 26, 82, + 65, 44, 6, 79, 79, 32, 84, 89, 80, 34, 85, 134, 172, 20, 73, 223, 137, 2, + 69, 8, 170, 182, 22, 65, 2, 73, 2, 81, 3, 85, 4, 11, 69, 4, 207, 142, 10, + 32, 9, 198, 185, 7, 85, 155, 252, 14, 75, 8, 18, 81, 31, 82, 4, 166, 181, + 22, 69, 3, 85, 4, 215, 149, 18, 89, 70, 138, 1, 67, 2, 75, 42, 78, 50, + 80, 30, 83, 46, 84, 54, 68, 2, 76, 166, 147, 22, 77, 2, 82, 2, 86, 2, 89, + 190, 28, 66, 2, 72, 3, 81, 8, 210, 1, 72, 154, 178, 22, 65, 3, 79, 8, + 206, 148, 22, 71, 2, 78, 2, 89, 247, 30, 79, 6, 122, 72, 155, 178, 22, + 79, 6, 130, 148, 22, 83, 190, 28, 72, 187, 2, 65, 12, 50, 72, 0, 2, 84, + 72, 154, 178, 22, 65, 3, 79, 4, 150, 178, 22, 65, 3, 79, 130, 1, 60, 4, 73, 71, 78, 32, 221, 6, 6, 89, 77, 66, 79, 76, 32, 46, 202, 2, 65, 104, 10, 83, 65, 77, 89, 79, 75, 32, 83, 65, 78, 22, 66, 118, 67, 86, 75, 74, - 82, 54, 84, 244, 191, 4, 3, 76, 69, 75, 244, 10, 6, 80, 72, 78, 65, 69, - 75, 180, 204, 11, 11, 89, 85, 85, 75, 65, 76, 69, 65, 80, 73, 78, 152, - 97, 3, 78, 73, 75, 244, 251, 2, 9, 77, 85, 85, 83, 73, 75, 65, 84, 79, - 249, 13, 4, 86, 73, 82, 73, 6, 100, 9, 86, 65, 75, 82, 65, 72, 65, 83, - 65, 152, 189, 17, 4, 84, 84, 72, 65, 153, 220, 3, 2, 72, 83, 2, 215, 187, - 21, 78, 8, 38, 65, 209, 180, 20, 3, 69, 89, 89, 6, 174, 6, 84, 216, 1, 2, - 78, 84, 189, 238, 19, 6, 82, 73, 89, 79, 79, 83, 4, 224, 244, 6, 12, 65, - 77, 78, 85, 67, 32, 80, 73, 73, 32, 75, 85, 155, 140, 12, 79, 6, 188, - 163, 11, 2, 65, 75, 230, 209, 8, 72, 245, 77, 4, 79, 79, 77, 85, 4, 130, - 163, 11, 79, 133, 208, 5, 4, 69, 65, 72, 77, 4, 136, 168, 20, 8, 79, 65, + 82, 54, 84, 144, 223, 4, 3, 76, 69, 75, 168, 11, 6, 80, 72, 78, 65, 69, + 75, 248, 226, 10, 10, 89, 85, 85, 75, 65, 76, 69, 65, 80, 73, 200, 238, + 1, 3, 78, 73, 75, 180, 165, 3, 9, 77, 85, 85, 83, 73, 75, 65, 84, 79, + 133, 15, 4, 86, 73, 82, 73, 6, 100, 9, 86, 65, 75, 82, 65, 72, 65, 83, + 65, 176, 128, 18, 4, 84, 84, 72, 65, 209, 136, 4, 2, 72, 83, 2, 167, 171, + 22, 78, 8, 38, 65, 137, 164, 21, 3, 69, 89, 89, 6, 174, 6, 84, 216, 1, 2, + 78, 84, 193, 219, 20, 6, 82, 73, 89, 79, 79, 83, 4, 152, 149, 7, 12, 65, + 77, 78, 85, 67, 32, 80, 73, 73, 32, 75, 85, 211, 212, 12, 79, 6, 140, + 214, 11, 2, 65, 75, 154, 140, 9, 72, 177, 80, 4, 79, 79, 77, 85, 4, 210, + 213, 11, 79, 249, 224, 5, 4, 69, 65, 72, 77, 4, 192, 151, 21, 8, 79, 65, 78, 68, 65, 75, 72, 73, 253, 2, 4, 82, 73, 73, 83, 84, 128, 1, 3, 68, 65, 80, 92, 10, 76, 69, 75, 32, 65, 84, 84, 65, 75, 32, 182, 1, 80, 72, 5, 84, 85, 84, 69, 89, 78, 66, 47, 77, 24, 22, 45, 223, 3, 32, 20, 30, 80, 238, 2, 66, 47, 77, 8, 138, 3, 73, 37, 3, 82, 65, 77, 20, 50, 80, 98, 66, - 130, 221, 10, 77, 187, 139, 10, 83, 12, 36, 3, 82, 65, 77, 251, 164, 21, - 73, 11, 11, 45, 8, 42, 66, 130, 221, 10, 77, 255, 158, 8, 80, 4, 182, - 232, 20, 85, 139, 60, 69, 26, 44, 2, 65, 84, 44, 3, 82, 65, 77, 91, 73, - 2, 21, 3, 72, 65, 77, 2, 223, 237, 15, 65, 20, 18, 45, 119, 32, 16, 34, + 130, 143, 11, 77, 255, 200, 10, 83, 12, 36, 3, 82, 65, 77, 203, 148, 22, + 73, 11, 11, 45, 8, 42, 66, 130, 143, 11, 77, 239, 213, 8, 80, 4, 250, + 215, 21, 85, 151, 60, 69, 26, 44, 2, 65, 84, 44, 3, 82, 65, 77, 91, 73, + 2, 21, 3, 72, 65, 77, 2, 223, 178, 16, 65, 20, 18, 45, 119, 32, 16, 34, 66, 32, 2, 80, 73, 15, 77, 8, 30, 69, 37, 3, 85, 79, 78, 4, 35, 73, 4, - 21, 3, 85, 79, 89, 4, 11, 32, 4, 34, 82, 229, 128, 21, 2, 75, 79, 2, 143, - 143, 20, 79, 42, 76, 10, 73, 78, 72, 69, 82, 69, 78, 84, 32, 65, 29, 5, - 83, 73, 71, 78, 32, 4, 138, 181, 21, 65, 3, 81, 38, 102, 65, 54, 73, 30, - 79, 38, 89, 180, 228, 2, 5, 67, 79, 69, 78, 71, 250, 150, 7, 85, 235, - 183, 11, 69, 10, 218, 142, 3, 65, 174, 165, 18, 69, 2, 73, 3, 85, 7, 210, - 179, 21, 69, 3, 73, 6, 182, 179, 21, 69, 2, 77, 3, 79, 7, 146, 179, 21, + 21, 3, 85, 79, 89, 4, 11, 32, 4, 34, 82, 169, 240, 21, 2, 75, 79, 2, 195, + 253, 20, 79, 42, 76, 10, 73, 78, 72, 69, 82, 69, 78, 84, 32, 65, 29, 5, + 83, 73, 71, 78, 32, 4, 218, 164, 22, 65, 3, 81, 38, 102, 65, 54, 73, 30, + 79, 38, 89, 184, 253, 2, 5, 67, 79, 69, 78, 71, 250, 165, 7, 85, 183, + 255, 11, 69, 10, 174, 174, 3, 65, 170, 245, 18, 69, 2, 73, 3, 85, 7, 162, + 163, 22, 69, 3, 73, 6, 134, 163, 22, 69, 2, 77, 3, 79, 7, 226, 162, 22, 65, 3, 89, 130, 1, 146, 1, 68, 88, 7, 76, 69, 84, 84, 69, 82, 32, 170, 2, - 83, 164, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 226, 156, 15, - 87, 219, 80, 65, 6, 48, 6, 79, 85, 66, 76, 69, 32, 179, 220, 11, 65, 4, - 166, 248, 9, 83, 255, 227, 1, 68, 90, 234, 1, 83, 130, 5, 66, 42, 71, - 246, 230, 5, 68, 222, 137, 4, 74, 214, 208, 7, 65, 82, 84, 222, 225, 1, - 76, 246, 189, 1, 78, 126, 67, 2, 75, 2, 80, 254, 68, 72, 2, 77, 2, 81, 2, - 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 26, 72, 131, 175, - 21, 65, 2, 249, 238, 20, 3, 79, 82, 84, 12, 40, 4, 73, 71, 78, 32, 191, - 245, 9, 69, 10, 58, 83, 138, 205, 15, 86, 170, 251, 1, 78, 223, 160, 3, - 65, 4, 26, 72, 243, 222, 16, 85, 2, 11, 65, 2, 203, 136, 21, 68, 18, 246, - 202, 3, 86, 238, 253, 13, 65, 142, 222, 1, 73, 206, 134, 2, 69, 2, 79, 3, - 85, 138, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 216, 2, 5, 83, 73, 71, - 78, 32, 170, 236, 15, 86, 227, 206, 3, 68, 94, 222, 1, 66, 42, 71, 42, - 74, 206, 230, 5, 68, 230, 162, 11, 82, 206, 55, 65, 82, 84, 230, 24, 85, - 210, 200, 1, 73, 158, 190, 1, 78, 126, 67, 2, 75, 2, 80, 2, 83, 254, 68, - 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 226, 167, 21, 66, - 2, 72, 187, 2, 65, 6, 186, 167, 21, 71, 2, 72, 187, 2, 65, 6, 146, 167, - 21, 72, 2, 74, 187, 2, 65, 6, 178, 200, 15, 86, 170, 251, 1, 78, 223, - 160, 3, 65, 20, 92, 2, 83, 83, 140, 245, 19, 4, 87, 73, 70, 82, 158, 44, - 80, 216, 93, 2, 77, 79, 191, 18, 84, 13, 40, 4, 73, 78, 71, 32, 187, 230, - 20, 32, 8, 96, 4, 70, 65, 67, 69, 209, 212, 13, 14, 67, 65, 84, 32, 70, - 65, 67, 69, 32, 87, 73, 84, 72, 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, - 198, 211, 13, 83, 115, 67, 4, 236, 210, 14, 3, 69, 69, 76, 167, 183, 6, - 79, 4, 40, 4, 82, 69, 65, 78, 207, 131, 21, 65, 2, 33, 6, 32, 83, 84, 65, - 78, 68, 2, 129, 141, 18, 2, 65, 82, 2, 11, 79, 2, 155, 253, 9, 78, 128, - 41, 154, 1, 65, 190, 216, 1, 69, 242, 56, 73, 230, 83, 79, 146, 22, 85, - 94, 89, 182, 206, 7, 82, 168, 249, 8, 5, 32, 66, 32, 66, 65, 174, 157, 1, - 76, 227, 66, 70, 130, 22, 182, 1, 66, 44, 6, 67, 82, 79, 83, 83, 69, 46, - 68, 56, 2, 79, 32, 222, 11, 82, 208, 6, 4, 83, 84, 32, 81, 76, 4, 84, 73, - 78, 32, 161, 168, 19, 8, 78, 71, 85, 65, 71, 69, 32, 84, 4, 212, 131, 14, - 2, 32, 67, 183, 149, 6, 69, 2, 153, 173, 16, 6, 32, 83, 84, 73, 67, 75, - 4, 184, 192, 12, 5, 89, 32, 66, 69, 69, 183, 163, 8, 68, 174, 1, 200, 2, - 3, 72, 79, 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 214, 5, 83, 148, 1, 9, - 84, 79, 78, 69, 32, 77, 65, 73, 32, 84, 11, 86, 79, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 128, 137, 4, 2, 75, 79, 188, 211, 12, 2, 89, 65, 162, - 157, 2, 69, 250, 46, 68, 248, 132, 1, 7, 67, 65, 78, 67, 69, 76, 76, 205, - 68, 6, 78, 73, 71, 71, 65, 72, 4, 210, 255, 20, 77, 3, 78, 94, 176, 1, 3, - 70, 79, 32, 78, 75, 96, 2, 76, 79, 50, 80, 154, 1, 83, 86, 84, 30, 72, - 206, 212, 15, 78, 214, 165, 5, 66, 2, 67, 2, 68, 2, 77, 2, 82, 2, 87, 2, - 89, 247, 30, 79, 8, 42, 70, 206, 235, 14, 83, 199, 247, 4, 84, 4, 246, - 204, 20, 79, 143, 62, 65, 10, 26, 72, 147, 156, 21, 79, 8, 32, 3, 77, 85, - 32, 231, 2, 79, 4, 182, 195, 20, 78, 195, 57, 71, 7, 17, 2, 32, 76, 4, - 202, 202, 20, 73, 67, 79, 30, 52, 4, 65, 76, 73, 32, 210, 1, 72, 151, - 153, 21, 79, 24, 190, 165, 6, 68, 34, 84, 206, 6, 78, 254, 246, 12, 66, - 2, 67, 2, 71, 2, 74, 167, 213, 1, 76, 8, 52, 9, 65, 78, 83, 75, 82, 73, - 84, 32, 83, 71, 79, 4, 146, 151, 21, 72, 3, 83, 6, 26, 72, 151, 153, 21, - 79, 4, 11, 79, 4, 11, 32, 4, 250, 231, 14, 83, 199, 247, 4, 84, 6, 112, - 14, 69, 77, 73, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 137, 147, 3, - 8, 73, 71, 78, 32, 80, 65, 76, 73, 4, 174, 191, 20, 78, 195, 57, 76, 8, - 50, 84, 252, 209, 16, 2, 67, 65, 255, 193, 4, 69, 4, 154, 248, 20, 72, - 247, 30, 73, 32, 106, 65, 44, 5, 77, 65, 73, 32, 75, 226, 162, 17, 89, - 222, 35, 85, 210, 200, 1, 69, 2, 73, 207, 134, 2, 79, 11, 130, 150, 21, - 65, 2, 73, 2, 77, 3, 89, 4, 130, 198, 20, 65, 3, 79, 56, 32, 2, 71, 69, - 179, 142, 20, 73, 54, 26, 32, 231, 188, 13, 82, 50, 254, 1, 66, 44, 4, - 71, 82, 69, 69, 22, 79, 250, 1, 84, 186, 180, 11, 68, 36, 2, 85, 80, 168, - 180, 3, 12, 76, 69, 70, 84, 32, 84, 82, 73, 65, 78, 71, 76, 224, 151, 4, + 83, 160, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 230, 225, 15, + 87, 211, 80, 65, 6, 48, 6, 79, 85, 66, 76, 69, 32, 219, 196, 18, 65, 4, + 170, 160, 10, 83, 163, 164, 8, 68, 90, 234, 1, 83, 254, 4, 66, 42, 71, + 186, 134, 6, 68, 214, 134, 12, 74, 158, 50, 65, 150, 1, 84, 218, 207, 1, + 76, 250, 192, 1, 78, 126, 67, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 81, 2, + 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 26, 72, 211, 158, + 22, 65, 2, 189, 222, 21, 3, 79, 82, 84, 12, 40, 4, 73, 71, 78, 32, 195, + 157, 10, 69, 10, 54, 83, 238, 137, 18, 78, 206, 61, 86, 159, 145, 3, 65, + 4, 26, 72, 187, 162, 17, 85, 2, 11, 65, 2, 159, 248, 21, 68, 18, 170, + 234, 3, 86, 154, 223, 14, 65, 242, 201, 1, 73, 222, 137, 2, 69, 2, 79, 3, + 85, 138, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 176, 2, 5, 83, 73, 71, + 78, 32, 202, 177, 16, 86, 187, 246, 3, 68, 94, 222, 1, 66, 42, 71, 186, + 134, 6, 68, 174, 133, 12, 74, 198, 51, 65, 118, 82, 34, 84, 230, 5, 85, + 206, 201, 1, 73, 162, 193, 1, 78, 126, 67, 2, 75, 2, 80, 2, 83, 138, 69, + 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 182, 151, 22, 66, + 2, 72, 187, 2, 65, 6, 142, 151, 22, 71, 2, 72, 187, 2, 65, 6, 190, 133, + 18, 78, 206, 61, 86, 159, 145, 3, 65, 136, 1, 140, 1, 8, 82, 65, 84, 32, + 82, 65, 73, 32, 216, 3, 2, 83, 83, 208, 223, 20, 4, 87, 73, 70, 82, 182, + 45, 80, 240, 93, 2, 77, 79, 191, 18, 84, 116, 140, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 172, 1, 5, 83, 73, 71, 78, 32, 92, 11, 86, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 243, 229, 11, 68, 64, 206, 188, 18, 68, 114, 84, + 226, 222, 1, 78, 238, 178, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, + 83, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 12, 208, + 202, 9, 3, 84, 79, 78, 0, 2, 89, 85, 242, 251, 1, 83, 230, 137, 10, 65, + 239, 1, 86, 16, 246, 192, 18, 65, 174, 147, 3, 85, 162, 64, 69, 2, 73, 3, + 79, 13, 40, 4, 73, 78, 71, 32, 163, 210, 21, 32, 8, 96, 4, 70, 65, 67, + 69, 181, 135, 14, 14, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, + 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 162, 134, 14, 83, 215, 144, 5, + 67, 4, 168, 146, 18, 3, 69, 69, 76, 215, 227, 3, 79, 4, 40, 4, 82, 69, + 65, 78, 187, 239, 21, 65, 2, 33, 6, 32, 83, 84, 65, 78, 68, 2, 129, 228, + 18, 2, 65, 82, 2, 11, 79, 2, 191, 161, 10, 78, 174, 43, 154, 1, 65, 150, + 232, 1, 69, 182, 62, 73, 238, 83, 79, 222, 28, 85, 94, 89, 226, 224, 7, + 82, 244, 179, 9, 5, 32, 66, 32, 66, 65, 166, 160, 1, 76, 239, 66, 70, + 252, 22, 182, 1, 66, 44, 6, 67, 82, 79, 83, 83, 69, 46, 68, 56, 2, 79, + 32, 222, 11, 82, 236, 21, 4, 83, 84, 32, 81, 76, 4, 84, 73, 78, 32, 237, + 129, 20, 8, 78, 71, 85, 65, 71, 69, 32, 84, 4, 176, 182, 14, 2, 32, 67, + 175, 206, 6, 69, 2, 153, 237, 16, 6, 32, 83, 84, 73, 67, 75, 4, 220, 242, + 12, 5, 89, 32, 66, 69, 69, 243, 220, 8, 68, 174, 1, 200, 2, 3, 72, 79, + 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 214, 5, 83, 148, 1, 9, 84, 79, 78, + 69, 32, 77, 65, 73, 32, 84, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, + 32, 188, 164, 4, 2, 75, 79, 224, 247, 12, 2, 89, 65, 230, 194, 2, 69, + 190, 50, 68, 236, 135, 1, 7, 67, 65, 78, 67, 69, 76, 76, 221, 68, 6, 78, + 73, 71, 71, 65, 72, 4, 190, 235, 21, 77, 3, 78, 94, 176, 1, 3, 70, 79, + 32, 78, 75, 96, 2, 76, 79, 50, 80, 154, 1, 83, 86, 84, 30, 72, 234, 149, + 16, 78, 166, 208, 5, 66, 2, 67, 2, 68, 2, 77, 2, 82, 2, 87, 2, 89, 247, + 30, 79, 8, 42, 70, 254, 162, 15, 83, 195, 170, 5, 84, 4, 214, 184, 21, + 79, 155, 62, 65, 10, 26, 72, 255, 135, 22, 79, 8, 32, 3, 77, 85, 32, 231, + 2, 79, 4, 146, 175, 21, 78, 211, 57, 71, 7, 17, 2, 32, 76, 4, 170, 182, + 21, 73, 67, 79, 30, 52, 4, 65, 76, 73, 32, 210, 1, 72, 131, 133, 22, 79, + 24, 154, 194, 6, 68, 34, 84, 206, 6, 78, 138, 195, 13, 66, 2, 67, 2, 71, + 2, 74, 171, 216, 1, 76, 8, 52, 9, 65, 78, 83, 75, 82, 73, 84, 32, 83, 71, + 79, 4, 254, 130, 22, 72, 3, 83, 6, 26, 72, 131, 133, 22, 79, 4, 11, 79, + 4, 11, 32, 4, 170, 159, 15, 83, 195, 170, 5, 84, 6, 112, 14, 69, 77, 73, + 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 169, 173, 18, 8, 73, 71, 78, + 32, 80, 65, 76, 73, 4, 138, 171, 21, 78, 211, 57, 76, 8, 50, 84, 220, + 145, 17, 2, 67, 65, 139, 238, 4, 69, 4, 134, 228, 21, 72, 247, 30, 73, + 32, 106, 65, 44, 5, 77, 65, 73, 32, 75, 146, 226, 17, 89, 142, 76, 85, + 206, 201, 1, 69, 2, 73, 223, 137, 2, 79, 11, 238, 129, 22, 65, 2, 73, 2, + 77, 3, 89, 4, 226, 177, 21, 65, 3, 79, 166, 1, 32, 2, 71, 69, 131, 250, + 20, 73, 164, 1, 26, 32, 199, 239, 13, 82, 160, 1, 254, 1, 66, 44, 4, 71, + 82, 69, 69, 22, 79, 250, 1, 84, 182, 231, 11, 68, 36, 2, 85, 80, 176, + 192, 3, 12, 76, 69, 70, 84, 32, 84, 82, 73, 65, 78, 71, 76, 164, 193, 4, 5, 80, 85, 82, 80, 76, 12, 3, 82, 69, 68, 0, 6, 89, 69, 76, 76, 79, 87, - 207, 64, 67, 10, 40, 3, 82, 79, 87, 201, 1, 2, 76, 85, 4, 251, 130, 19, - 78, 10, 48, 3, 78, 69, 32, 129, 1, 4, 82, 65, 78, 71, 4, 200, 248, 7, 5, - 68, 79, 84, 32, 79, 173, 221, 2, 18, 82, 73, 78, 71, 32, 79, 86, 69, 82, - 32, 84, 87, 79, 32, 82, 73, 78, 71, 6, 11, 69, 6, 11, 32, 6, 230, 193, - 19, 67, 226, 19, 68, 231, 26, 83, 6, 104, 3, 87, 79, 32, 161, 229, 17, - 17, 82, 73, 80, 76, 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, - 4, 162, 162, 17, 68, 205, 64, 19, 82, 73, 78, 71, 83, 32, 79, 86, 69, 82, - 32, 79, 78, 69, 32, 82, 73, 78, 71, 6, 53, 11, 85, 65, 82, 84, 69, 82, - 32, 77, 79, 79, 78, 7, 167, 220, 6, 32, 138, 20, 150, 1, 67, 244, 45, 18, + 243, 64, 67, 10, 40, 3, 82, 79, 87, 201, 1, 2, 76, 85, 4, 195, 235, 19, + 78, 10, 48, 3, 78, 69, 32, 129, 1, 4, 82, 65, 78, 71, 4, 160, 155, 8, 5, + 68, 79, 84, 32, 79, 181, 233, 2, 18, 82, 73, 78, 71, 32, 79, 86, 69, 82, + 32, 84, 87, 79, 32, 82, 73, 78, 71, 6, 11, 69, 6, 11, 32, 6, 210, 170, + 20, 67, 154, 21, 68, 139, 28, 83, 116, 156, 1, 3, 87, 79, 32, 108, 10, + 89, 80, 69, 32, 80, 73, 69, 67, 69, 32, 157, 186, 18, 17, 82, 73, 80, 76, + 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 4, 150, 225, 17, 68, + 193, 87, 19, 82, 73, 78, 71, 83, 32, 79, 86, 69, 82, 32, 79, 78, 69, 32, + 82, 73, 78, 71, 110, 182, 1, 67, 136, 2, 9, 68, 73, 65, 71, 79, 78, 65, + 76, 32, 218, 1, 76, 186, 2, 82, 158, 1, 83, 204, 3, 6, 85, 80, 80, 69, + 82, 32, 145, 2, 9, 86, 69, 82, 84, 69, 88, 32, 79, 70, 14, 80, 9, 69, 78, + 84, 82, 69, 32, 79, 70, 32, 73, 7, 82, 79, 83, 83, 66, 65, 82, 8, 252, 8, + 6, 90, 32, 87, 73, 84, 72, 142, 239, 21, 75, 2, 88, 3, 89, 7, 33, 6, 32, + 87, 73, 84, 72, 32, 4, 40, 3, 76, 79, 87, 1, 3, 85, 80, 80, 2, 181, 197, + 10, 2, 69, 82, 12, 48, 6, 85, 80, 80, 69, 82, 32, 227, 222, 9, 76, 8, 56, + 4, 76, 69, 70, 84, 157, 223, 9, 4, 82, 73, 71, 72, 5, 11, 32, 2, 21, 3, + 65, 78, 68, 2, 17, 2, 32, 76, 2, 17, 2, 79, 87, 2, 213, 194, 21, 2, 69, + 82, 26, 48, 5, 79, 87, 69, 82, 32, 129, 3, 2, 69, 70, 24, 84, 5, 76, 69, + 70, 84, 32, 56, 6, 82, 73, 71, 72, 84, 32, 158, 6, 72, 179, 1, 84, 8, 22, + 65, 207, 7, 67, 4, 194, 1, 78, 159, 202, 20, 82, 10, 22, 65, 151, 7, 67, + 6, 192, 1, 13, 78, 68, 32, 85, 80, 80, 69, 82, 32, 82, 73, 71, 72, 193, + 189, 19, 2, 82, 67, 4, 44, 4, 65, 73, 83, 69, 77, 3, 73, 71, 72, 2, 53, + 11, 68, 32, 85, 80, 80, 69, 82, 32, 76, 69, 70, 2, 187, 245, 18, 84, 2, + 141, 243, 20, 3, 84, 32, 65, 34, 48, 5, 72, 79, 82, 84, 32, 77, 3, 84, + 69, 77, 4, 40, 3, 76, 79, 87, 1, 3, 85, 80, 80, 2, 217, 4, 4, 69, 82, 32, + 84, 31, 44, 6, 32, 87, 73, 84, 72, 32, 171, 1, 45, 8, 44, 5, 76, 69, 70, + 84, 32, 30, 82, 59, 67, 4, 82, 67, 167, 214, 10, 74, 2, 21, 3, 73, 71, + 72, 2, 11, 84, 2, 17, 2, 32, 67, 2, 177, 205, 20, 4, 82, 79, 83, 83, 20, + 48, 2, 49, 50, 22, 50, 26, 52, 247, 181, 11, 51, 5, 207, 211, 14, 51, 9, + 11, 51, 7, 11, 52, 5, 243, 237, 21, 53, 18, 62, 72, 96, 3, 76, 69, 70, 0, + 4, 82, 73, 71, 72, 83, 84, 4, 65, 14, 65, 76, 70, 32, 86, 69, 82, 84, 69, + 88, 32, 79, 70, 32, 4, 218, 236, 21, 77, 3, 87, 6, 17, 2, 84, 32, 6, 26, + 67, 131, 240, 18, 65, 4, 186, 92, 82, 211, 147, 17, 79, 2, 201, 243, 8, + 3, 69, 82, 77, 2, 211, 219, 14, 32, 6, 53, 11, 85, 65, 82, 84, 69, 82, + 32, 77, 79, 79, 78, 7, 151, 234, 6, 32, 150, 20, 150, 1, 67, 148, 46, 18, 69, 80, 73, 71, 82, 65, 80, 72, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, - 252, 1, 7, 76, 69, 84, 84, 69, 82, 32, 183, 13, 83, 192, 7, 56, 8, 65, - 80, 73, 84, 65, 76, 32, 76, 239, 206, 19, 82, 190, 7, 76, 6, 69, 84, 84, - 69, 82, 32, 189, 44, 8, 73, 71, 65, 84, 85, 82, 69, 32, 186, 7, 154, 2, + 252, 1, 7, 76, 69, 84, 84, 69, 82, 32, 183, 13, 83, 200, 7, 56, 8, 65, + 80, 73, 84, 65, 76, 32, 76, 139, 168, 20, 82, 198, 7, 76, 6, 69, 84, 84, + 69, 82, 32, 221, 44, 8, 73, 71, 65, 84, 85, 82, 69, 32, 194, 7, 154, 2, 65, 158, 3, 66, 154, 1, 67, 254, 1, 68, 250, 1, 69, 174, 2, 70, 82, 71, - 194, 1, 72, 146, 1, 73, 154, 2, 74, 118, 75, 126, 76, 222, 2, 77, 114, - 78, 134, 2, 79, 198, 2, 80, 102, 81, 62, 82, 174, 2, 83, 254, 2, 84, 142, + 194, 1, 72, 146, 1, 73, 154, 2, 74, 118, 75, 126, 76, 234, 2, 77, 114, + 78, 134, 2, 79, 198, 2, 80, 102, 81, 62, 82, 190, 2, 83, 130, 3, 84, 142, 3, 85, 234, 1, 86, 146, 1, 87, 118, 88, 50, 89, 167, 1, 90, 93, 172, 1, - 6, 32, 87, 73, 84, 72, 32, 166, 1, 69, 182, 65, 78, 166, 209, 4, 76, 236, - 154, 9, 6, 70, 82, 73, 67, 65, 78, 178, 136, 4, 86, 138, 209, 2, 65, 2, - 79, 2, 85, 3, 89, 66, 182, 64, 66, 34, 68, 108, 3, 82, 73, 78, 242, 33, - 77, 254, 21, 67, 146, 49, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 206, - 156, 3, 73, 214, 241, 12, 84, 175, 145, 3, 83, 7, 33, 6, 32, 87, 73, 84, - 72, 32, 4, 142, 183, 1, 65, 183, 182, 3, 77, 21, 60, 6, 32, 87, 73, 84, - 72, 32, 254, 67, 82, 175, 157, 20, 69, 14, 218, 66, 84, 146, 60, 70, 202, - 56, 76, 134, 224, 15, 68, 222, 157, 2, 72, 231, 160, 1, 83, 33, 108, 6, - 32, 87, 73, 84, 72, 32, 140, 70, 7, 76, 79, 83, 69, 68, 32, 73, 54, 85, - 154, 238, 19, 79, 255, 59, 72, 20, 94, 67, 134, 180, 1, 65, 222, 215, 3, - 80, 142, 167, 14, 72, 186, 46, 66, 246, 33, 68, 187, 80, 83, 8, 134, 68, - 69, 190, 112, 73, 255, 170, 18, 65, 31, 44, 6, 32, 87, 73, 84, 72, 32, - 179, 1, 90, 24, 78, 83, 182, 15, 67, 214, 47, 84, 218, 116, 76, 134, 224, - 15, 68, 223, 157, 2, 72, 8, 92, 13, 77, 65, 76, 76, 32, 76, 69, 84, 84, - 69, 82, 32, 90, 246, 136, 1, 72, 223, 200, 19, 84, 5, 177, 72, 2, 32, 87, - 91, 104, 6, 32, 87, 73, 84, 72, 32, 152, 1, 2, 90, 72, 134, 76, 71, 194, - 191, 16, 84, 210, 242, 3, 83, 63, 78, 70, 254, 73, 67, 158, 2, 68, 194, - 16, 84, 162, 23, 77, 250, 1, 86, 234, 44, 72, 158, 1, 79, 198, 10, 71, - 186, 2, 65, 206, 156, 3, 73, 62, 66, 199, 130, 16, 83, 7, 11, 32, 4, 138, - 70, 87, 211, 8, 82, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 246, 173, 19, - 72, 174, 80, 68, 187, 80, 83, 35, 76, 6, 32, 87, 73, 84, 72, 32, 190, 81, - 76, 138, 225, 17, 65, 227, 200, 2, 72, 20, 154, 84, 67, 250, 89, 65, 138, - 157, 3, 66, 174, 25, 77, 142, 133, 4, 79, 170, 195, 10, 72, 174, 80, 68, - 187, 80, 83, 29, 76, 6, 32, 87, 73, 84, 72, 32, 138, 126, 65, 238, 218, - 7, 87, 191, 210, 11, 69, 20, 186, 82, 66, 34, 67, 46, 68, 182, 216, 18, - 72, 231, 160, 1, 83, 59, 80, 6, 32, 87, 73, 84, 72, 32, 132, 88, 2, 78, - 83, 186, 254, 19, 79, 207, 36, 83, 40, 138, 1, 68, 162, 83, 67, 242, 1, - 77, 142, 1, 84, 130, 70, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 206, - 156, 3, 73, 62, 66, 199, 130, 16, 83, 10, 22, 79, 191, 83, 73, 6, 234, - 120, 85, 155, 232, 17, 84, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, 67, - 206, 183, 17, 84, 175, 145, 3, 83, 4, 234, 169, 1, 73, 199, 205, 3, 82, - 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 182, 89, 67, 34, 68, 50, 83, 222, - 78, 65, 138, 1, 76, 186, 186, 7, 79, 171, 195, 10, 72, 37, 48, 6, 32, 87, - 73, 84, 72, 32, 163, 247, 20, 74, 32, 138, 1, 66, 36, 2, 68, 79, 52, 7, - 77, 73, 68, 68, 76, 69, 32, 38, 83, 174, 2, 67, 198, 91, 72, 226, 71, 65, - 138, 1, 76, 155, 141, 16, 84, 4, 246, 151, 12, 69, 207, 165, 8, 65, 6, - 22, 85, 135, 92, 84, 2, 205, 223, 4, 2, 66, 76, 4, 146, 180, 17, 84, 255, - 240, 2, 68, 4, 214, 2, 77, 211, 194, 20, 84, 19, 44, 6, 32, 87, 73, 84, - 72, 32, 223, 94, 73, 10, 254, 164, 1, 65, 142, 225, 15, 68, 150, 45, 84, - 203, 240, 1, 72, 33, 48, 6, 32, 87, 73, 84, 72, 32, 215, 243, 20, 74, 28, - 102, 67, 44, 2, 83, 77, 194, 96, 76, 130, 64, 71, 186, 2, 65, 102, 68, - 222, 186, 7, 79, 227, 210, 8, 84, 6, 202, 131, 1, 69, 22, 73, 131, 203, - 18, 65, 2, 249, 194, 10, 10, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 101, - 108, 6, 32, 87, 73, 84, 72, 32, 194, 103, 76, 202, 140, 4, 80, 150, 253, - 8, 77, 250, 255, 6, 73, 2, 79, 3, 85, 84, 144, 1, 2, 76, 79, 34, 77, 242, - 96, 67, 70, 68, 154, 2, 79, 38, 83, 66, 84, 106, 86, 218, 43, 72, 242, - 12, 71, 186, 2, 65, 206, 156, 3, 73, 63, 66, 4, 174, 99, 78, 199, 140, - 20, 79, 8, 170, 99, 65, 171, 141, 4, 73, 17, 33, 6, 32, 87, 73, 84, 72, - 32, 14, 146, 104, 70, 34, 83, 162, 55, 65, 234, 254, 17, 72, 175, 80, 68, - 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 230, 104, 68, 39, 83, 41, 78, 32, - 156, 1, 8, 69, 86, 69, 82, 83, 69, 68, 32, 145, 111, 3, 85, 77, 32, 28, - 40, 5, 87, 73, 84, 72, 32, 143, 112, 82, 26, 174, 78, 67, 234, 28, 68, - 170, 2, 84, 158, 48, 65, 138, 1, 76, 198, 155, 3, 73, 246, 158, 4, 79, - 143, 228, 11, 83, 8, 146, 110, 72, 218, 138, 4, 79, 162, 246, 12, 67, - 171, 253, 2, 69, 47, 136, 1, 6, 32, 87, 73, 84, 72, 32, 132, 1, 5, 77, - 65, 76, 76, 32, 156, 115, 2, 65, 76, 234, 1, 72, 152, 2, 2, 73, 71, 235, - 248, 3, 67, 30, 82, 67, 194, 111, 65, 118, 68, 126, 83, 174, 1, 86, 178, - 227, 7, 79, 171, 195, 10, 72, 10, 154, 112, 65, 218, 10, 69, 82, 79, 203, - 31, 73, 4, 200, 188, 18, 11, 81, 32, 87, 73, 84, 72, 32, 72, 79, 79, 75, - 149, 237, 1, 7, 67, 65, 80, 73, 84, 65, 76, 59, 136, 1, 6, 32, 87, 73, - 84, 72, 32, 168, 1, 6, 85, 82, 78, 69, 68, 32, 232, 122, 2, 72, 79, 176, - 1, 2, 79, 78, 74, 82, 231, 233, 19, 90, 22, 82, 67, 50, 68, 170, 152, 1, - 76, 186, 220, 3, 82, 170, 161, 14, 72, 231, 160, 1, 83, 8, 246, 119, 69, - 22, 73, 62, 79, 199, 202, 18, 65, 6, 234, 141, 1, 73, 207, 234, 15, 79, - 18, 162, 40, 73, 146, 249, 2, 65, 146, 197, 17, 72, 2, 75, 2, 76, 2, 77, - 2, 84, 3, 86, 79, 26, 32, 179, 245, 4, 80, 74, 44, 5, 87, 73, 84, 72, 32, - 251, 194, 19, 66, 72, 210, 131, 1, 67, 74, 68, 150, 2, 72, 170, 1, 77, - 134, 1, 79, 142, 1, 84, 186, 9, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, - 186, 136, 14, 82, 143, 250, 1, 83, 23, 88, 6, 32, 87, 73, 84, 72, 32, - 222, 138, 1, 73, 66, 79, 174, 202, 18, 69, 227, 141, 1, 89, 8, 142, 138, - 1, 68, 242, 151, 16, 84, 203, 240, 1, 72, 19, 48, 6, 32, 87, 73, 84, 72, - 32, 159, 132, 7, 89, 14, 234, 143, 1, 67, 18, 68, 70, 71, 186, 2, 65, - 235, 254, 17, 72, 7, 245, 139, 1, 7, 32, 87, 73, 84, 72, 32, 68, 29, 48, - 6, 32, 87, 73, 84, 72, 32, 147, 233, 16, 79, 24, 198, 142, 1, 67, 18, 68, - 70, 71, 22, 72, 42, 76, 254, 1, 65, 182, 182, 3, 77, 238, 215, 12, 84, - 175, 145, 3, 83, 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 170, 57, 67, 150, - 87, 65, 102, 68, 38, 76, 34, 83, 182, 214, 3, 80, 143, 167, 14, 72, 4, - 250, 175, 10, 73, 231, 152, 10, 79, 12, 128, 1, 5, 65, 82, 67, 72, 65, - 30, 73, 64, 9, 82, 69, 86, 69, 82, 83, 69, 68, 32, 197, 173, 10, 7, 83, - 73, 68, 69, 87, 65, 89, 2, 213, 199, 13, 2, 73, 67, 4, 136, 107, 5, 78, - 86, 69, 82, 84, 149, 156, 14, 3, 32, 76, 79, 4, 174, 221, 20, 70, 3, 80, - 138, 1, 198, 3, 65, 36, 2, 66, 73, 152, 1, 21, 73, 78, 86, 69, 82, 84, - 69, 68, 32, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 72, 2, 82, - 69, 222, 1, 83, 162, 28, 87, 132, 33, 3, 84, 87, 79, 204, 246, 3, 2, 68, - 69, 140, 5, 2, 76, 65, 206, 11, 71, 248, 174, 1, 21, 80, 72, 65, 82, 89, - 78, 71, 69, 65, 76, 32, 86, 79, 73, 67, 69, 68, 32, 70, 82, 73, 192, 181, - 13, 20, 86, 79, 73, 67, 69, 68, 32, 76, 65, 82, 89, 78, 71, 69, 65, 76, - 32, 83, 80, 73, 155, 116, 89, 4, 150, 169, 4, 76, 223, 224, 15, 73, 6, - 76, 7, 76, 65, 66, 73, 65, 76, 32, 29, 8, 68, 69, 78, 84, 65, 76, 32, 80, - 4, 26, 80, 143, 188, 4, 67, 2, 153, 208, 15, 6, 69, 82, 67, 85, 83, 83, - 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 134, 225, 4, 67, 231, 198, 15, 83, - 8, 104, 7, 86, 69, 82, 83, 69, 68, 32, 149, 225, 4, 13, 84, 82, 79, 70, - 76, 69, 88, 32, 67, 76, 73, 67, 75, 4, 80, 3, 69, 83, 72, 193, 61, 12, - 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 2, 189, 132, 1, 2, 32, - 76, 96, 172, 1, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, - 220, 111, 10, 84, 82, 69, 84, 67, 72, 69, 68, 32, 67, 225, 211, 18, 10, - 73, 78, 79, 76, 79, 71, 73, 67, 65, 76, 90, 230, 1, 69, 30, 73, 22, 76, - 74, 79, 42, 82, 122, 84, 186, 164, 4, 66, 202, 41, 71, 166, 194, 15, 65, - 150, 64, 67, 2, 68, 2, 70, 2, 72, 2, 74, 2, 75, 2, 77, 2, 78, 2, 80, 2, - 81, 2, 83, 2, 85, 2, 86, 2, 87, 2, 89, 3, 90, 7, 190, 209, 20, 84, 3, 90, - 5, 191, 201, 4, 78, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 198, 199, 8, 66, - 187, 218, 11, 83, 9, 214, 94, 80, 130, 243, 19, 69, 3, 85, 11, 88, 8, 69, - 86, 69, 82, 83, 69, 68, 32, 216, 127, 5, 32, 87, 73, 84, 72, 167, 192, - 19, 85, 4, 210, 208, 20, 78, 3, 82, 13, 33, 6, 85, 82, 78, 69, 68, 32, - 10, 146, 208, 20, 69, 2, 71, 2, 75, 2, 77, 3, 82, 180, 11, 136, 1, 5, 77, - 65, 76, 76, 32, 253, 129, 1, 22, 85, 66, 83, 67, 82, 73, 80, 84, 32, 83, - 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 144, 11, 76, 15, 67, 65, - 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 27, 76, 4, 182, 53, - 73, 3, 85, 140, 11, 76, 6, 69, 84, 84, 69, 82, 32, 161, 127, 8, 73, 71, - 65, 84, 85, 82, 69, 32, 250, 10, 182, 2, 65, 190, 5, 66, 198, 3, 67, 186, - 4, 68, 182, 4, 69, 174, 9, 70, 214, 1, 71, 162, 2, 72, 230, 2, 73, 166, - 7, 74, 182, 1, 75, 178, 2, 76, 210, 6, 77, 178, 2, 78, 218, 3, 79, 242, - 8, 80, 230, 1, 81, 174, 1, 82, 142, 8, 83, 234, 10, 84, 246, 13, 85, 218, - 9, 86, 130, 3, 87, 110, 88, 226, 2, 89, 171, 3, 90, 101, 118, 32, 198, 3, - 69, 82, 78, 132, 208, 4, 4, 76, 80, 72, 65, 190, 164, 13, 86, 138, 209, - 2, 65, 2, 79, 2, 85, 3, 89, 72, 88, 5, 87, 73, 84, 72, 32, 209, 209, 5, - 11, 82, 69, 86, 69, 82, 83, 69, 68, 45, 83, 67, 70, 150, 1, 66, 34, 68, - 54, 82, 170, 34, 77, 254, 21, 67, 146, 49, 72, 158, 1, 79, 198, 10, 71, - 186, 2, 65, 206, 156, 3, 73, 214, 241, 12, 84, 175, 145, 3, 83, 12, 161, - 105, 4, 82, 69, 86, 69, 12, 22, 79, 155, 57, 73, 8, 218, 57, 84, 223, 12, - 85, 10, 26, 73, 255, 212, 4, 69, 8, 26, 78, 131, 156, 4, 71, 6, 17, 2, - 71, 32, 6, 240, 58, 4, 65, 66, 79, 86, 159, 243, 17, 66, 9, 33, 6, 32, - 87, 73, 84, 72, 32, 6, 242, 115, 71, 186, 2, 65, 183, 182, 3, 77, 2, 133, - 160, 10, 7, 71, 76, 73, 67, 65, 78, 65, 41, 140, 1, 6, 32, 87, 73, 84, - 72, 32, 130, 1, 65, 108, 11, 76, 65, 67, 75, 76, 69, 84, 84, 69, 82, 32, - 38, 82, 242, 182, 4, 79, 191, 230, 15, 69, 18, 110, 84, 146, 60, 70, 202, - 56, 76, 170, 207, 3, 77, 174, 7, 80, 178, 137, 12, 68, 222, 157, 2, 72, - 231, 160, 1, 83, 2, 255, 7, 79, 8, 64, 5, 82, 82, 69, 68, 32, 161, 80, 6, - 83, 69, 76, 73, 78, 69, 6, 182, 32, 65, 142, 162, 20, 69, 3, 79, 6, 186, - 177, 4, 79, 227, 144, 16, 69, 2, 133, 148, 10, 4, 79, 75, 69, 78, 47, - 108, 6, 32, 87, 73, 84, 72, 32, 196, 1, 2, 72, 73, 92, 6, 76, 79, 83, 69, - 68, 32, 90, 85, 155, 238, 19, 79, 24, 102, 67, 182, 112, 65, 222, 215, 3, - 80, 230, 5, 82, 170, 161, 14, 72, 186, 46, 66, 246, 33, 68, 187, 80, 83, - 10, 54, 69, 190, 112, 73, 238, 225, 6, 85, 147, 201, 11, 65, 4, 249, 51, - 5, 68, 73, 76, 76, 65, 7, 49, 10, 32, 87, 73, 84, 72, 32, 76, 79, 87, 32, - 4, 162, 106, 82, 45, 4, 76, 69, 70, 84, 8, 34, 73, 18, 79, 227, 181, 4, - 82, 2, 163, 87, 78, 4, 210, 202, 4, 80, 199, 243, 8, 77, 4, 37, 7, 65, - 84, 82, 73, 76, 76, 79, 5, 153, 224, 12, 5, 32, 87, 73, 84, 72, 73, 96, - 6, 32, 87, 73, 84, 72, 32, 182, 1, 69, 34, 79, 178, 1, 90, 246, 194, 4, - 66, 235, 229, 15, 85, 32, 98, 83, 34, 84, 130, 33, 67, 206, 44, 77, 170, - 31, 76, 206, 199, 3, 72, 138, 15, 80, 179, 137, 12, 68, 4, 134, 67, 72, - 223, 200, 19, 84, 4, 26, 79, 167, 142, 18, 65, 2, 215, 235, 17, 80, 8, - 246, 77, 90, 195, 200, 19, 76, 16, 60, 6, 84, 76, 69, 83, 83, 32, 49, 5, - 85, 66, 76, 69, 32, 8, 26, 74, 139, 186, 20, 73, 7, 235, 179, 4, 32, 8, - 42, 87, 210, 183, 4, 82, 187, 219, 5, 84, 2, 163, 219, 6, 89, 11, 11, 32, - 8, 26, 87, 219, 179, 4, 68, 2, 169, 89, 5, 73, 84, 72, 32, 67, 119, 112, - 6, 32, 87, 73, 84, 72, 32, 214, 4, 71, 72, 2, 78, 71, 68, 2, 83, 72, 164, - 1, 2, 90, 72, 147, 189, 16, 84, 76, 182, 1, 67, 158, 2, 68, 110, 78, 214, - 15, 84, 162, 23, 77, 250, 1, 86, 194, 3, 70, 170, 41, 72, 158, 1, 79, - 198, 10, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, 186, 64, 82, 143, 194, - 15, 83, 24, 92, 6, 69, 68, 73, 76, 76, 65, 32, 9, 73, 82, 67, 85, 77, 70, - 76, 69, 88, 179, 145, 19, 65, 5, 129, 131, 4, 3, 32, 65, 78, 19, 11, 32, - 16, 40, 4, 65, 78, 68, 32, 207, 156, 18, 66, 14, 162, 85, 67, 130, 2, 72, - 226, 11, 71, 186, 2, 65, 182, 182, 3, 77, 190, 162, 4, 68, 179, 181, 8, - 84, 12, 22, 79, 227, 97, 73, 10, 28, 2, 84, 32, 243, 50, 85, 8, 192, 87, - 5, 65, 66, 79, 86, 69, 239, 195, 17, 66, 2, 183, 232, 13, 79, 4, 237, - 237, 6, 13, 89, 80, 84, 79, 76, 79, 71, 73, 67, 65, 76, 32, 65, 7, 33, 6, - 32, 87, 73, 84, 72, 32, 4, 226, 176, 4, 67, 231, 9, 80, 13, 33, 6, 32, - 87, 73, 84, 72, 32, 10, 88, 10, 68, 79, 85, 66, 76, 69, 32, 66, 65, 82, - 170, 185, 4, 80, 142, 1, 67, 219, 4, 82, 5, 157, 186, 4, 4, 32, 65, 78, - 68, 15, 11, 32, 12, 38, 82, 37, 5, 87, 73, 84, 72, 32, 2, 209, 228, 13, - 4, 69, 86, 69, 82, 10, 54, 67, 246, 183, 4, 80, 230, 5, 82, 139, 197, 13, - 84, 4, 194, 194, 7, 85, 147, 201, 11, 65, 17, 84, 6, 32, 87, 73, 84, 72, - 32, 73, 11, 69, 78, 71, 32, 68, 73, 71, 82, 65, 80, 72, 10, 202, 175, 4, - 77, 174, 7, 80, 142, 167, 14, 72, 174, 80, 68, 187, 80, 83, 5, 197, 189, - 17, 8, 32, 87, 73, 84, 72, 32, 84, 82, 37, 72, 6, 32, 87, 73, 84, 72, 32, - 126, 76, 138, 225, 17, 65, 227, 200, 2, 72, 22, 218, 3, 67, 250, 89, 65, - 138, 157, 3, 66, 174, 25, 77, 170, 33, 80, 230, 227, 3, 79, 170, 195, 10, - 72, 174, 80, 68, 187, 80, 83, 8, 33, 6, 79, 84, 84, 65, 76, 32, 8, 178, - 225, 17, 83, 202, 202, 2, 65, 2, 73, 3, 85, 39, 140, 1, 6, 32, 87, 73, - 84, 72, 32, 166, 44, 65, 216, 25, 12, 79, 79, 75, 69, 68, 32, 83, 67, 72, - 87, 65, 32, 242, 225, 3, 69, 207, 130, 16, 86, 24, 86, 66, 34, 67, 46, - 68, 214, 90, 76, 214, 214, 3, 80, 142, 167, 14, 72, 231, 160, 1, 83, 2, - 157, 229, 12, 3, 82, 69, 86, 6, 158, 58, 69, 154, 32, 73, 255, 170, 18, - 65, 8, 234, 86, 73, 178, 173, 3, 69, 203, 182, 12, 79, 75, 80, 6, 32, 87, - 73, 84, 72, 32, 222, 4, 78, 188, 1, 2, 79, 84, 251, 161, 20, 83, 48, 178, - 1, 67, 34, 68, 210, 1, 77, 64, 6, 79, 71, 79, 78, 69, 75, 78, 84, 130, - 70, 72, 226, 11, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, 136, 64, 6, 83, - 84, 82, 79, 75, 69, 51, 82, 4, 210, 87, 73, 255, 170, 18, 65, 14, 18, 73, - 47, 79, 4, 221, 26, 7, 65, 69, 82, 69, 83, 73, 83, 10, 28, 2, 84, 32, - 231, 36, 85, 8, 64, 10, 65, 66, 79, 86, 69, 32, 65, 78, 68, 32, 227, 140, - 18, 66, 6, 150, 83, 71, 186, 2, 65, 163, 142, 16, 84, 4, 29, 5, 65, 67, - 82, 79, 78, 5, 233, 35, 4, 32, 65, 78, 68, 7, 145, 72, 15, 32, 65, 78, - 68, 32, 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, 4, 21, 3, 73, 76, 68, 4, - 171, 251, 4, 69, 16, 46, 83, 93, 7, 86, 69, 82, 84, 69, 68, 32, 12, 29, - 5, 85, 76, 65, 82, 32, 12, 226, 162, 20, 68, 2, 70, 2, 71, 2, 82, 2, 83, - 3, 84, 4, 26, 65, 187, 139, 20, 79, 2, 231, 173, 4, 76, 6, 150, 145, 4, - 65, 229, 162, 6, 5, 73, 70, 73, 69, 68, 13, 33, 6, 32, 87, 73, 84, 72, - 32, 10, 94, 67, 216, 161, 4, 13, 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, - 65, 78, 68, 235, 206, 15, 83, 6, 178, 81, 73, 198, 205, 3, 82, 187, 221, - 14, 65, 29, 48, 6, 32, 87, 73, 84, 72, 32, 163, 157, 20, 82, 24, 98, 67, - 34, 68, 50, 83, 222, 78, 65, 138, 1, 76, 214, 214, 3, 80, 230, 227, 3, - 79, 171, 195, 10, 72, 4, 210, 47, 69, 151, 203, 18, 65, 6, 214, 69, 73, - 134, 180, 3, 69, 175, 174, 4, 79, 4, 29, 5, 84, 82, 79, 75, 69, 5, 177, - 24, 6, 32, 65, 78, 68, 32, 68, 77, 156, 1, 6, 32, 87, 73, 84, 72, 32, - 248, 3, 5, 65, 77, 66, 68, 65, 22, 69, 48, 5, 79, 78, 71, 32, 83, 194, - 161, 4, 83, 2, 90, 234, 229, 15, 85, 219, 16, 74, 50, 178, 1, 66, 86, 67, - 56, 2, 68, 79, 100, 3, 77, 73, 68, 242, 1, 72, 226, 71, 65, 138, 1, 76, - 218, 206, 3, 73, 162, 1, 82, 222, 2, 70, 130, 4, 80, 198, 182, 12, 84, - 175, 145, 3, 83, 6, 36, 3, 69, 76, 84, 147, 226, 19, 65, 5, 241, 162, 4, - 6, 32, 65, 78, 68, 32, 80, 8, 146, 43, 69, 22, 73, 242, 129, 7, 85, 147, - 201, 11, 65, 8, 38, 84, 25, 5, 85, 66, 76, 69, 32, 4, 137, 24, 2, 32, 66, - 4, 162, 154, 4, 77, 243, 220, 14, 66, 8, 36, 4, 68, 76, 69, 32, 187, 43, - 45, 6, 198, 215, 16, 84, 178, 240, 2, 82, 79, 68, 2, 139, 155, 4, 32, 6, - 222, 162, 4, 90, 161, 192, 12, 3, 78, 73, 83, 9, 33, 6, 32, 87, 73, 84, - 72, 32, 6, 18, 68, 35, 72, 4, 202, 62, 73, 215, 216, 18, 79, 2, 133, 154, - 4, 2, 73, 71, 27, 56, 6, 32, 87, 73, 84, 72, 32, 102, 73, 151, 133, 20, - 85, 16, 134, 71, 65, 250, 205, 3, 67, 186, 2, 77, 174, 7, 80, 178, 137, - 12, 68, 150, 45, 84, 203, 240, 1, 72, 6, 25, 4, 68, 68, 76, 69, 6, 76, 7, - 45, 87, 69, 76, 83, 72, 32, 141, 194, 16, 6, 32, 83, 67, 79, 84, 83, 4, - 226, 139, 19, 76, 159, 137, 1, 86, 49, 58, 32, 216, 2, 2, 71, 32, 242, - 128, 20, 85, 219, 16, 74, 40, 88, 5, 87, 73, 84, 72, 32, 193, 250, 5, 11, - 80, 82, 69, 67, 69, 68, 69, 68, 32, 66, 89, 38, 122, 67, 74, 76, 154, 36, - 77, 234, 27, 71, 186, 2, 65, 102, 68, 250, 214, 3, 80, 230, 5, 82, 130, - 222, 3, 79, 227, 210, 8, 84, 10, 166, 35, 69, 22, 73, 202, 237, 3, 82, - 170, 148, 3, 85, 147, 201, 11, 65, 6, 128, 65, 3, 79, 78, 71, 202, 2, 73, - 135, 222, 3, 69, 2, 25, 4, 87, 73, 84, 72, 2, 149, 131, 17, 5, 32, 84, - 73, 76, 68, 115, 116, 6, 32, 87, 73, 84, 72, 32, 186, 6, 76, 52, 4, 80, - 69, 78, 32, 170, 137, 13, 77, 250, 255, 6, 73, 2, 79, 3, 85, 86, 154, 1, - 67, 70, 68, 152, 1, 2, 76, 79, 86, 77, 46, 79, 38, 83, 66, 84, 106, 86, - 218, 43, 72, 242, 12, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, 187, 64, - 82, 14, 168, 48, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 187, 186, 18, 65, - 14, 18, 73, 47, 79, 4, 233, 12, 7, 65, 69, 82, 69, 83, 73, 83, 10, 22, - 84, 167, 46, 85, 6, 11, 32, 6, 152, 12, 5, 65, 66, 79, 86, 69, 247, 232, - 17, 66, 6, 66, 78, 188, 162, 12, 6, 87, 32, 82, 73, 78, 71, 139, 234, 7, - 79, 2, 155, 20, 71, 6, 11, 65, 6, 189, 2, 4, 67, 82, 79, 78, 4, 229, 10, - 5, 71, 79, 78, 69, 75, 4, 25, 4, 84, 82, 79, 75, 4, 11, 69, 5, 209, 48, - 2, 32, 65, 8, 25, 4, 73, 76, 68, 69, 9, 29, 5, 32, 65, 78, 68, 32, 6, - 158, 46, 68, 142, 13, 65, 183, 182, 3, 77, 6, 81, 18, 69, 82, 84, 73, 67, - 65, 76, 32, 76, 73, 78, 69, 32, 66, 69, 76, 79, 87, 7, 217, 42, 4, 32, - 65, 78, 68, 2, 229, 225, 9, 8, 68, 32, 80, 79, 76, 73, 83, 72, 16, 26, - 79, 195, 147, 4, 69, 13, 48, 6, 32, 87, 73, 84, 72, 32, 211, 136, 20, 69, - 8, 206, 54, 71, 186, 2, 65, 194, 221, 3, 82, 143, 194, 15, 83, 23, 48, 6, - 32, 87, 73, 84, 72, 32, 139, 244, 19, 72, 18, 86, 70, 34, 83, 162, 55, - 65, 178, 208, 3, 77, 174, 7, 80, 142, 167, 14, 72, 175, 80, 68, 2, 181, - 135, 14, 3, 76, 79, 85, 6, 202, 27, 84, 185, 130, 12, 6, 81, 85, 73, 82, - 82, 69, 13, 48, 6, 32, 87, 73, 84, 72, 32, 191, 143, 4, 80, 8, 42, 68, - 16, 4, 72, 79, 79, 75, 23, 83, 2, 211, 44, 73, 5, 179, 216, 17, 32, 2, - 181, 26, 6, 84, 82, 79, 75, 69, 32, 77, 90, 32, 228, 4, 8, 69, 86, 69, - 82, 83, 69, 68, 32, 148, 2, 2, 85, 77, 231, 129, 4, 65, 46, 36, 4, 87, - 73, 84, 72, 235, 6, 82, 44, 26, 32, 219, 253, 14, 79, 42, 166, 1, 67, 50, - 68, 200, 1, 8, 70, 73, 83, 72, 72, 79, 79, 75, 66, 76, 34, 84, 254, 17, - 77, 162, 30, 65, 206, 156, 3, 73, 146, 59, 80, 230, 227, 3, 79, 143, 228, - 11, 83, 6, 154, 19, 69, 222, 237, 3, 82, 187, 221, 14, 65, 8, 11, 79, 8, - 24, 2, 84, 32, 111, 85, 6, 26, 66, 239, 132, 19, 65, 4, 25, 4, 69, 76, - 79, 87, 5, 29, 5, 32, 65, 78, 68, 32, 2, 247, 231, 3, 77, 2, 21, 3, 66, - 76, 69, 2, 11, 32, 2, 211, 46, 71, 7, 29, 5, 32, 65, 78, 68, 32, 4, 138, - 129, 4, 77, 175, 7, 80, 4, 206, 49, 73, 211, 218, 3, 79, 4, 238, 210, 17, - 65, 251, 22, 73, 22, 162, 1, 72, 40, 6, 79, 80, 69, 78, 32, 69, 140, 130, - 4, 8, 82, 32, 87, 73, 84, 72, 32, 70, 168, 2, 3, 83, 67, 82, 218, 129, - 14, 69, 198, 84, 67, 171, 163, 1, 75, 2, 11, 65, 2, 189, 213, 9, 2, 76, - 70, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 202, 139, 4, 82, 171, 161, 14, - 72, 5, 11, 32, 2, 11, 82, 2, 153, 168, 10, 3, 79, 84, 85, 83, 180, 1, 6, - 32, 87, 73, 84, 72, 32, 202, 4, 65, 66, 67, 218, 1, 72, 34, 73, 156, 2, - 13, 81, 85, 65, 84, 32, 82, 69, 86, 69, 82, 83, 69, 68, 173, 198, 9, 6, - 84, 73, 82, 82, 85, 80, 38, 106, 65, 34, 67, 86, 68, 126, 83, 174, 1, 86, - 210, 9, 77, 254, 245, 3, 80, 230, 227, 3, 79, 171, 195, 10, 72, 4, 193, - 1, 4, 67, 85, 84, 69, 12, 58, 65, 218, 10, 69, 82, 79, 202, 31, 73, 239, - 225, 6, 85, 4, 101, 3, 82, 79, 78, 6, 21, 3, 79, 84, 32, 6, 26, 66, 211, - 252, 18, 65, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, 2, 185, 248, 18, 3, 65, + 6, 32, 87, 73, 84, 72, 32, 166, 1, 69, 230, 65, 78, 216, 146, 14, 6, 70, + 82, 73, 67, 65, 78, 222, 182, 3, 76, 130, 182, 2, 86, 182, 162, 1, 65, 2, + 79, 2, 85, 3, 89, 66, 230, 64, 66, 34, 68, 108, 3, 82, 73, 78, 242, 33, + 77, 250, 21, 67, 162, 49, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 202, + 168, 3, 73, 186, 159, 13, 84, 227, 179, 3, 83, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 202, 183, 1, 65, 187, 194, 3, 77, 21, 60, 6, 32, 87, 73, 84, + 72, 32, 174, 68, 82, 207, 249, 20, 69, 14, 138, 67, 84, 142, 60, 70, 218, + 56, 76, 226, 143, 16, 68, 162, 199, 2, 72, 219, 163, 1, 83, 33, 108, 6, + 32, 87, 73, 84, 72, 32, 188, 70, 7, 76, 79, 83, 69, 68, 32, 73, 54, 85, + 174, 202, 20, 79, 139, 60, 72, 20, 94, 67, 194, 180, 1, 65, 230, 227, 3, + 80, 166, 244, 14, 72, 158, 50, 66, 238, 32, 68, 211, 80, 83, 8, 182, 68, + 69, 202, 112, 73, 255, 133, 19, 65, 31, 44, 6, 32, 87, 73, 84, 72, 32, + 179, 1, 90, 24, 78, 83, 194, 15, 67, 250, 47, 84, 230, 116, 76, 226, 143, + 16, 68, 163, 199, 2, 72, 8, 92, 13, 77, 65, 76, 76, 32, 76, 69, 84, 84, + 69, 82, 32, 90, 178, 137, 1, 72, 243, 164, 20, 84, 5, 225, 72, 2, 32, 87, + 91, 104, 6, 32, 87, 73, 84, 72, 32, 152, 1, 2, 90, 72, 182, 76, 71, 166, + 239, 16, 84, 142, 159, 4, 83, 63, 78, 70, 174, 74, 67, 158, 2, 68, 194, + 16, 84, 158, 23, 77, 250, 1, 86, 250, 44, 72, 158, 1, 79, 198, 10, 71, + 186, 2, 65, 202, 168, 3, 73, 62, 66, 223, 210, 16, 83, 7, 11, 32, 4, 186, + 70, 87, 211, 8, 82, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 210, 135, 20, + 72, 138, 83, 68, 211, 80, 83, 35, 76, 6, 32, 87, 73, 84, 72, 32, 238, 81, + 76, 202, 182, 18, 65, 195, 207, 2, 72, 20, 202, 84, 67, 134, 90, 65, 134, + 169, 3, 66, 182, 25, 77, 198, 140, 4, 79, 142, 137, 11, 72, 138, 83, 68, + 211, 80, 83, 29, 76, 6, 32, 87, 73, 84, 72, 32, 182, 126, 65, 246, 238, + 7, 87, 207, 154, 12, 69, 20, 234, 82, 66, 34, 67, 46, 68, 226, 177, 19, + 72, 219, 163, 1, 83, 59, 80, 6, 32, 87, 73, 84, 72, 32, 180, 88, 2, 78, + 83, 218, 218, 20, 79, 207, 36, 83, 40, 138, 1, 68, 210, 83, 67, 242, 1, + 77, 142, 1, 84, 142, 70, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 202, + 168, 3, 73, 62, 66, 223, 210, 16, 83, 10, 22, 79, 239, 83, 73, 6, 150, + 121, 85, 231, 190, 18, 84, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, 67, + 234, 241, 17, 84, 227, 179, 3, 83, 4, 166, 170, 1, 73, 207, 217, 3, 82, + 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 230, 89, 67, 34, 68, 50, 83, 234, + 78, 65, 138, 1, 76, 246, 205, 7, 79, 143, 137, 11, 72, 41, 60, 6, 32, 87, + 73, 84, 72, 32, 238, 94, 65, 251, 244, 20, 74, 32, 138, 1, 66, 36, 2, 68, + 79, 52, 7, 77, 73, 68, 68, 76, 69, 32, 38, 83, 174, 2, 67, 230, 91, 72, + 242, 71, 65, 138, 1, 76, 251, 198, 16, 84, 4, 242, 186, 12, 69, 139, 223, + 8, 65, 6, 22, 85, 151, 92, 84, 2, 129, 236, 4, 2, 66, 76, 4, 162, 238, + 17, 84, 167, 147, 3, 68, 4, 214, 2, 77, 151, 159, 21, 84, 19, 44, 6, 32, + 87, 73, 84, 72, 32, 255, 94, 73, 10, 174, 165, 1, 65, 234, 144, 16, 68, + 154, 55, 84, 139, 144, 2, 72, 33, 48, 6, 32, 87, 73, 84, 72, 32, 155, + 208, 21, 74, 28, 102, 67, 44, 2, 83, 77, 226, 96, 76, 146, 64, 71, 186, + 2, 65, 102, 68, 154, 206, 7, 79, 135, 249, 8, 84, 6, 250, 131, 1, 69, 22, + 73, 131, 166, 19, 65, 2, 189, 226, 10, 10, 65, 76, 76, 32, 76, 69, 84, + 84, 69, 82, 101, 108, 6, 32, 87, 73, 84, 72, 32, 226, 103, 76, 226, 152, + 4, 80, 170, 150, 9, 77, 242, 182, 7, 73, 2, 79, 3, 85, 84, 144, 1, 2, 76, + 79, 34, 77, 146, 97, 67, 70, 68, 154, 2, 79, 38, 83, 66, 84, 106, 86, + 234, 43, 72, 242, 12, 71, 186, 2, 65, 202, 168, 3, 73, 63, 66, 4, 206, + 99, 78, 235, 232, 20, 79, 8, 202, 99, 65, 195, 153, 4, 73, 17, 33, 6, 32, + 87, 73, 84, 72, 32, 14, 178, 104, 70, 34, 83, 178, 55, 65, 138, 216, 18, + 72, 139, 83, 68, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 134, 105, 68, 39, + 83, 43, 94, 32, 156, 1, 8, 69, 86, 69, 82, 83, 69, 68, 32, 160, 111, 3, + 85, 77, 32, 243, 141, 4, 65, 28, 40, 5, 87, 73, 84, 72, 32, 159, 112, 82, + 26, 194, 78, 67, 230, 28, 68, 170, 2, 84, 174, 48, 65, 138, 1, 76, 194, + 167, 3, 73, 182, 166, 4, 79, 231, 172, 12, 83, 8, 162, 110, 72, 230, 150, + 4, 79, 202, 183, 13, 67, 155, 140, 3, 69, 49, 136, 1, 6, 32, 87, 73, 84, + 72, 32, 136, 1, 5, 77, 65, 76, 76, 32, 184, 115, 2, 65, 76, 234, 1, 72, + 152, 2, 2, 73, 71, 243, 132, 4, 67, 32, 86, 67, 210, 111, 65, 118, 68, + 138, 1, 83, 174, 1, 86, 238, 246, 7, 79, 143, 137, 11, 72, 10, 170, 112, + 65, 230, 10, 69, 82, 79, 203, 31, 73, 4, 152, 146, 19, 11, 81, 32, 87, + 73, 84, 72, 32, 72, 79, 79, 75, 233, 243, 1, 7, 67, 65, 80, 73, 84, 65, + 76, 59, 136, 1, 6, 32, 87, 73, 84, 72, 32, 168, 1, 6, 85, 82, 78, 69, 68, + 32, 132, 123, 2, 72, 79, 176, 1, 2, 79, 78, 74, 82, 251, 197, 20, 90, 22, + 82, 67, 50, 68, 198, 152, 1, 76, 182, 232, 3, 82, 206, 238, 14, 72, 219, + 163, 1, 83, 8, 146, 120, 69, 22, 73, 62, 79, 199, 165, 19, 65, 6, 134, + 142, 1, 73, 171, 154, 16, 79, 18, 178, 40, 73, 146, 133, 3, 65, 178, 149, + 18, 72, 2, 75, 2, 76, 2, 77, 2, 84, 3, 86, 79, 26, 32, 203, 129, 5, 80, + 74, 44, 5, 87, 73, 84, 72, 32, 155, 160, 20, 66, 72, 238, 131, 1, 67, 74, + 68, 150, 2, 72, 170, 1, 77, 134, 1, 79, 142, 1, 84, 186, 9, 71, 186, 2, + 65, 202, 168, 3, 73, 62, 66, 242, 209, 14, 82, 239, 128, 2, 83, 23, 88, + 6, 32, 87, 73, 84, 72, 32, 250, 138, 1, 73, 66, 79, 170, 166, 19, 69, + 251, 141, 1, 89, 8, 170, 138, 1, 68, 210, 209, 16, 84, 139, 144, 2, 72, + 19, 48, 6, 32, 87, 73, 84, 72, 32, 251, 145, 7, 89, 14, 134, 144, 1, 67, + 18, 68, 70, 71, 186, 2, 65, 139, 216, 18, 72, 7, 145, 140, 1, 7, 32, 87, + 73, 84, 72, 32, 68, 29, 48, 6, 32, 87, 73, 84, 72, 32, 135, 153, 17, 79, + 24, 226, 142, 1, 67, 18, 68, 70, 71, 22, 72, 42, 76, 254, 1, 65, 186, + 194, 3, 77, 202, 133, 13, 84, 227, 179, 3, 83, 25, 33, 6, 32, 87, 73, 84, + 72, 32, 22, 186, 57, 67, 162, 87, 65, 102, 68, 38, 76, 34, 83, 190, 226, + 3, 80, 167, 244, 14, 72, 4, 170, 207, 10, 73, 231, 213, 10, 79, 12, 128, + 1, 5, 65, 82, 67, 72, 65, 30, 73, 64, 9, 82, 69, 86, 69, 82, 83, 69, 68, + 32, 245, 204, 10, 7, 83, 73, 68, 69, 87, 65, 89, 2, 177, 235, 13, 2, 73, + 67, 4, 164, 107, 5, 78, 86, 69, 82, 84, 221, 205, 14, 3, 32, 76, 79, 4, + 222, 185, 21, 70, 3, 80, 138, 1, 198, 3, 65, 36, 2, 66, 73, 152, 1, 21, + 73, 78, 86, 69, 82, 84, 69, 68, 32, 71, 76, 79, 84, 84, 65, 76, 32, 83, + 84, 79, 80, 72, 2, 82, 69, 222, 1, 83, 128, 8, 3, 84, 87, 79, 178, 20, + 87, 216, 163, 4, 2, 68, 69, 148, 5, 2, 76, 65, 206, 11, 71, 128, 176, 1, + 21, 80, 72, 65, 82, 89, 78, 71, 69, 65, 76, 32, 86, 79, 73, 67, 69, 68, + 32, 70, 82, 73, 164, 131, 14, 20, 86, 79, 73, 67, 69, 68, 32, 76, 65, 82, + 89, 78, 71, 69, 65, 76, 32, 83, 80, 73, 179, 117, 89, 4, 174, 181, 4, 76, + 235, 176, 16, 73, 6, 76, 7, 76, 65, 66, 73, 65, 76, 32, 29, 8, 68, 69, + 78, 84, 65, 76, 32, 80, 4, 26, 80, 175, 200, 4, 67, 2, 161, 129, 16, 6, + 69, 82, 67, 85, 83, 83, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 170, 237, 4, + 67, 243, 150, 16, 83, 8, 104, 7, 86, 69, 82, 83, 69, 68, 32, 185, 237, 4, + 13, 84, 82, 79, 70, 76, 69, 88, 32, 67, 76, 73, 67, 75, 4, 80, 3, 69, 83, + 72, 157, 8, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 2, 217, + 132, 1, 2, 32, 76, 96, 172, 1, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, + 84, 65, 76, 32, 248, 111, 10, 84, 82, 69, 84, 67, 72, 69, 68, 32, 67, + 221, 175, 19, 10, 73, 78, 79, 76, 79, 71, 73, 67, 65, 76, 90, 230, 1, 69, + 30, 73, 22, 76, 74, 79, 42, 82, 122, 84, 210, 176, 4, 66, 214, 41, 71, + 166, 146, 16, 65, 162, 64, 67, 2, 68, 2, 70, 2, 72, 2, 74, 2, 75, 2, 77, + 2, 78, 2, 80, 2, 81, 2, 83, 2, 85, 2, 86, 2, 87, 2, 89, 3, 90, 7, 238, + 173, 21, 84, 3, 90, 5, 227, 213, 4, 78, 7, 33, 6, 32, 87, 73, 84, 72, 32, + 4, 162, 219, 8, 66, 143, 163, 12, 83, 9, 242, 94, 80, 150, 207, 20, 69, + 3, 85, 11, 88, 8, 69, 86, 69, 82, 83, 69, 68, 32, 244, 127, 5, 32, 87, + 73, 84, 72, 187, 156, 20, 85, 4, 130, 173, 21, 78, 3, 82, 13, 33, 6, 85, + 82, 78, 69, 68, 32, 10, 194, 172, 21, 69, 2, 71, 2, 75, 2, 77, 3, 82, + 184, 11, 136, 1, 5, 77, 65, 76, 76, 32, 153, 130, 1, 22, 85, 66, 83, 67, + 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, + 148, 11, 76, 15, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, + 32, 43, 76, 4, 18, 73, 3, 85, 2, 211, 220, 4, 32, 144, 11, 76, 6, 69, 84, + 84, 69, 82, 32, 173, 127, 8, 73, 71, 65, 84, 85, 82, 69, 32, 254, 10, + 182, 2, 65, 190, 5, 66, 198, 3, 67, 186, 4, 68, 182, 4, 69, 174, 9, 70, + 214, 1, 71, 162, 2, 72, 230, 2, 73, 166, 7, 74, 182, 1, 75, 178, 2, 76, + 206, 6, 77, 178, 2, 78, 218, 3, 79, 242, 8, 80, 230, 1, 81, 174, 1, 82, + 142, 8, 83, 250, 10, 84, 246, 13, 85, 218, 9, 86, 130, 3, 87, 110, 88, + 226, 2, 89, 171, 3, 90, 101, 118, 32, 198, 3, 69, 82, 78, 152, 220, 4, 4, + 76, 80, 72, 65, 158, 163, 15, 86, 182, 162, 1, 65, 2, 79, 2, 85, 3, 89, + 72, 88, 5, 87, 73, 84, 72, 32, 225, 222, 5, 11, 82, 69, 86, 69, 82, 83, + 69, 68, 45, 83, 67, 70, 150, 1, 66, 34, 68, 54, 82, 170, 34, 77, 250, 21, + 67, 162, 49, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 202, 168, 3, 73, + 186, 159, 13, 84, 227, 179, 3, 83, 12, 173, 105, 4, 82, 69, 86, 69, 12, + 22, 79, 151, 57, 73, 8, 214, 57, 84, 223, 12, 85, 10, 26, 73, 135, 225, + 4, 69, 8, 26, 78, 139, 168, 4, 71, 6, 17, 2, 71, 32, 6, 236, 58, 4, 65, + 66, 79, 86, 235, 201, 18, 66, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 254, + 115, 71, 186, 2, 65, 187, 194, 3, 77, 2, 165, 191, 10, 7, 71, 76, 73, 67, + 65, 78, 65, 41, 140, 1, 6, 32, 87, 73, 84, 72, 32, 130, 1, 65, 108, 11, + 76, 65, 67, 75, 76, 69, 84, 84, 69, 82, 32, 38, 82, 134, 195, 4, 79, 203, + 182, 16, 69, 18, 110, 84, 142, 60, 70, 218, 56, 76, 178, 219, 3, 77, 174, + 7, 80, 134, 173, 12, 68, 162, 199, 2, 72, 219, 163, 1, 83, 2, 255, 7, 79, + 8, 64, 5, 82, 82, 69, 68, 32, 173, 80, 6, 83, 69, 76, 73, 78, 69, 6, 182, + 32, 65, 174, 254, 20, 69, 3, 79, 6, 202, 189, 4, 79, 243, 224, 16, 69, 2, + 165, 179, 10, 4, 79, 75, 69, 78, 47, 108, 6, 32, 87, 73, 84, 72, 32, 196, + 1, 2, 72, 73, 92, 6, 76, 79, 83, 69, 68, 32, 90, 85, 175, 202, 20, 79, + 24, 102, 67, 194, 112, 65, 230, 227, 3, 80, 218, 5, 82, 206, 238, 14, 72, + 158, 50, 66, 238, 32, 68, 211, 80, 83, 10, 54, 69, 202, 112, 73, 230, + 245, 6, 85, 155, 144, 12, 65, 4, 245, 51, 5, 68, 73, 76, 76, 65, 7, 49, + 10, 32, 87, 73, 84, 72, 32, 76, 79, 87, 32, 4, 174, 106, 82, 45, 4, 76, + 69, 70, 84, 8, 34, 73, 18, 79, 247, 193, 4, 82, 2, 175, 87, 78, 4, 218, + 214, 4, 80, 231, 140, 9, 77, 4, 37, 7, 65, 84, 82, 73, 76, 76, 79, 5, + 253, 130, 13, 5, 32, 87, 73, 84, 72, 73, 96, 6, 32, 87, 73, 84, 72, 32, + 182, 1, 69, 34, 79, 178, 1, 90, 138, 207, 4, 66, 247, 181, 16, 85, 32, + 98, 83, 34, 84, 238, 32, 67, 238, 44, 77, 170, 31, 76, 214, 211, 3, 72, + 138, 15, 80, 135, 173, 12, 68, 4, 146, 67, 72, 243, 164, 20, 84, 4, 26, + 79, 231, 227, 18, 65, 2, 243, 245, 19, 80, 8, 130, 78, 90, 215, 164, 20, + 76, 16, 60, 6, 84, 76, 69, 83, 83, 32, 49, 5, 85, 66, 76, 69, 32, 8, 26, + 74, 171, 150, 21, 73, 7, 255, 191, 4, 32, 8, 42, 87, 230, 195, 4, 82, + 199, 238, 5, 84, 2, 239, 232, 6, 89, 11, 11, 32, 8, 26, 87, 239, 191, 4, + 68, 2, 181, 89, 5, 73, 84, 72, 32, 67, 119, 112, 6, 32, 87, 73, 84, 72, + 32, 214, 4, 71, 72, 2, 78, 71, 68, 2, 83, 72, 164, 1, 2, 90, 72, 247, + 236, 16, 84, 76, 182, 1, 67, 158, 2, 68, 110, 78, 214, 15, 84, 158, 23, + 77, 250, 1, 86, 194, 3, 70, 186, 41, 72, 158, 1, 79, 198, 10, 71, 186, 2, + 65, 202, 168, 3, 73, 62, 66, 186, 64, 82, 167, 146, 16, 83, 24, 92, 6, + 69, 68, 73, 76, 76, 65, 32, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 191, + 236, 19, 65, 5, 137, 143, 4, 3, 32, 65, 78, 19, 11, 32, 16, 40, 4, 65, + 78, 68, 32, 151, 243, 18, 66, 14, 174, 85, 67, 130, 2, 72, 226, 11, 71, + 186, 2, 65, 186, 194, 3, 77, 250, 169, 4, 68, 211, 219, 8, 84, 12, 22, + 79, 239, 97, 73, 10, 28, 2, 84, 32, 239, 50, 85, 8, 204, 87, 5, 65, 66, + 79, 86, 69, 171, 154, 18, 66, 2, 167, 142, 14, 79, 4, 221, 255, 6, 13, + 89, 80, 84, 79, 76, 79, 71, 73, 67, 65, 76, 32, 65, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 246, 188, 4, 67, 231, 9, 80, 13, 33, 6, 32, 87, 73, 84, + 72, 32, 10, 88, 10, 68, 79, 85, 66, 76, 69, 32, 66, 65, 82, 190, 197, 4, + 80, 142, 1, 67, 207, 4, 82, 5, 177, 198, 4, 4, 32, 65, 78, 68, 15, 11, + 32, 12, 38, 82, 37, 5, 87, 73, 84, 72, 32, 2, 193, 138, 14, 4, 69, 86, + 69, 82, 10, 54, 67, 138, 196, 4, 80, 218, 5, 82, 195, 142, 14, 84, 4, + 198, 214, 7, 85, 155, 144, 12, 65, 17, 84, 6, 32, 87, 73, 84, 72, 32, 73, + 11, 69, 78, 71, 32, 68, 73, 71, 82, 65, 80, 72, 10, 222, 187, 4, 77, 174, + 7, 80, 166, 244, 14, 72, 138, 83, 68, 211, 80, 83, 5, 181, 146, 18, 8, + 32, 87, 73, 84, 72, 32, 84, 82, 37, 72, 6, 32, 87, 73, 84, 72, 32, 126, + 76, 202, 182, 18, 65, 195, 207, 2, 72, 22, 218, 3, 67, 134, 90, 65, 134, + 169, 3, 66, 182, 25, 77, 174, 33, 80, 154, 235, 3, 79, 142, 137, 11, 72, + 138, 83, 68, 211, 80, 83, 8, 33, 6, 79, 84, 84, 65, 76, 32, 8, 242, 182, + 18, 83, 170, 209, 2, 65, 2, 73, 3, 85, 39, 140, 1, 6, 32, 87, 73, 84, 72, + 32, 162, 44, 65, 232, 25, 12, 79, 79, 75, 69, 68, 32, 83, 67, 72, 87, 65, + 32, 250, 237, 3, 69, 219, 210, 16, 86, 24, 86, 66, 34, 67, 46, 68, 226, + 90, 76, 222, 226, 3, 80, 166, 244, 14, 72, 219, 163, 1, 83, 2, 233, 135, + 13, 3, 82, 69, 86, 6, 170, 58, 69, 154, 32, 73, 255, 133, 19, 65, 8, 246, + 86, 73, 174, 185, 3, 69, 171, 218, 12, 79, 75, 80, 6, 32, 87, 73, 84, 72, + 32, 222, 4, 78, 188, 1, 2, 79, 84, 155, 254, 20, 83, 48, 178, 1, 67, 34, + 68, 210, 1, 77, 64, 6, 79, 71, 79, 78, 69, 75, 78, 84, 142, 70, 72, 226, + 11, 71, 186, 2, 65, 202, 168, 3, 73, 62, 66, 136, 64, 6, 83, 84, 82, 79, + 75, 69, 51, 82, 4, 222, 87, 73, 255, 133, 19, 65, 14, 18, 73, 47, 79, 4, + 217, 26, 7, 65, 69, 82, 69, 83, 73, 83, 10, 28, 2, 84, 32, 227, 36, 85, + 8, 64, 10, 65, 66, 79, 86, 69, 32, 65, 78, 68, 32, 171, 227, 18, 66, 6, + 162, 83, 71, 186, 2, 65, 131, 200, 16, 84, 4, 29, 5, 65, 67, 82, 79, 78, + 5, 229, 35, 4, 32, 65, 78, 68, 7, 157, 72, 15, 32, 65, 78, 68, 32, 68, + 79, 84, 32, 65, 66, 79, 86, 69, 32, 4, 21, 3, 73, 76, 68, 4, 211, 135, 5, + 69, 16, 46, 83, 93, 7, 86, 69, 82, 84, 69, 68, 32, 12, 29, 5, 85, 76, 65, + 82, 32, 12, 130, 255, 20, 68, 2, 70, 2, 71, 2, 82, 2, 83, 3, 84, 4, 26, + 65, 219, 231, 20, 79, 2, 247, 165, 17, 76, 6, 166, 157, 4, 65, 129, 186, + 6, 5, 73, 70, 73, 69, 68, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 94, 67, + 236, 173, 4, 13, 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, 65, 78, 68, 247, + 158, 16, 83, 6, 190, 81, 73, 206, 217, 3, 82, 179, 172, 15, 65, 29, 48, + 6, 32, 87, 73, 84, 72, 32, 195, 249, 20, 82, 24, 98, 67, 34, 68, 50, 83, + 234, 78, 65, 138, 1, 76, 222, 226, 3, 80, 154, 235, 3, 79, 143, 137, 11, + 72, 4, 222, 47, 69, 151, 166, 19, 65, 6, 226, 69, 73, 130, 192, 3, 69, + 243, 181, 4, 79, 4, 29, 5, 84, 82, 79, 75, 69, 5, 173, 24, 6, 32, 65, 78, + 68, 32, 68, 79, 136, 1, 6, 32, 87, 73, 84, 72, 32, 250, 3, 65, 38, 69, + 48, 5, 79, 78, 71, 32, 83, 218, 173, 4, 83, 2, 90, 246, 181, 16, 85, 219, + 16, 74, 50, 178, 1, 66, 86, 67, 56, 2, 68, 79, 100, 3, 77, 73, 68, 130, + 2, 72, 242, 71, 65, 138, 1, 76, 226, 218, 3, 73, 162, 1, 82, 222, 2, 70, + 130, 4, 80, 158, 228, 12, 84, 227, 179, 3, 83, 6, 36, 3, 69, 76, 84, 187, + 190, 20, 65, 5, 153, 175, 4, 6, 32, 65, 78, 68, 32, 80, 8, 178, 43, 69, + 22, 73, 234, 149, 7, 85, 155, 144, 12, 65, 8, 38, 84, 25, 5, 85, 66, 76, + 69, 32, 4, 153, 24, 2, 32, 66, 4, 202, 166, 4, 77, 239, 173, 15, 66, 8, + 36, 4, 68, 76, 69, 32, 219, 43, 45, 6, 198, 145, 17, 84, 218, 146, 3, 82, + 79, 68, 4, 133, 148, 4, 4, 77, 66, 68, 65, 6, 246, 174, 4, 90, 205, 250, + 12, 3, 78, 73, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 18, 68, 35, 72, + 4, 218, 62, 73, 211, 180, 19, 79, 2, 157, 166, 4, 2, 73, 71, 27, 56, 6, + 32, 87, 73, 84, 72, 32, 102, 73, 187, 225, 20, 85, 16, 150, 71, 65, 130, + 218, 3, 67, 186, 2, 77, 174, 7, 80, 134, 173, 12, 68, 154, 55, 84, 139, + 144, 2, 72, 6, 25, 4, 68, 68, 76, 69, 6, 76, 7, 45, 87, 69, 76, 83, 72, + 32, 149, 252, 16, 6, 32, 83, 67, 79, 84, 83, 4, 238, 231, 19, 76, 183, + 137, 1, 86, 49, 58, 32, 216, 2, 2, 71, 32, 150, 221, 20, 85, 219, 16, 74, + 40, 88, 5, 87, 73, 84, 72, 32, 153, 136, 6, 11, 80, 82, 69, 67, 69, 68, + 69, 68, 32, 66, 89, 38, 122, 67, 74, 76, 170, 36, 77, 234, 27, 71, 186, + 2, 65, 102, 68, 130, 227, 3, 80, 218, 5, 82, 194, 229, 3, 79, 135, 249, + 8, 84, 10, 182, 35, 69, 22, 73, 210, 249, 3, 82, 154, 156, 3, 85, 155, + 144, 12, 65, 6, 144, 65, 3, 79, 78, 71, 202, 2, 73, 131, 234, 3, 69, 2, + 25, 4, 87, 73, 84, 72, 2, 153, 202, 17, 5, 32, 84, 73, 76, 68, 115, 116, + 6, 32, 87, 73, 84, 72, 32, 186, 6, 76, 52, 4, 80, 69, 78, 32, 214, 174, + 13, 77, 242, 182, 7, 73, 2, 79, 3, 85, 86, 154, 1, 67, 70, 68, 152, 1, 2, + 76, 79, 86, 77, 46, 79, 38, 83, 66, 84, 106, 86, 234, 43, 72, 242, 12, + 71, 186, 2, 65, 202, 168, 3, 73, 62, 66, 187, 64, 82, 14, 184, 48, 9, 73, + 82, 67, 85, 77, 70, 76, 69, 88, 187, 149, 19, 65, 14, 18, 73, 47, 79, 4, + 233, 12, 7, 65, 69, 82, 69, 83, 73, 83, 10, 22, 84, 183, 46, 85, 6, 11, + 32, 6, 152, 12, 5, 65, 66, 79, 86, 69, 195, 191, 18, 66, 6, 66, 78, 132, + 197, 12, 6, 87, 32, 82, 73, 78, 71, 231, 163, 8, 79, 2, 171, 20, 71, 6, + 11, 65, 6, 189, 2, 4, 67, 82, 79, 78, 4, 229, 10, 5, 71, 79, 78, 69, 75, + 4, 25, 4, 84, 82, 79, 75, 4, 11, 69, 5, 225, 48, 2, 32, 65, 8, 25, 4, 73, + 76, 68, 69, 9, 29, 5, 32, 65, 78, 68, 32, 6, 174, 46, 68, 142, 13, 65, + 187, 194, 3, 77, 6, 81, 18, 69, 82, 84, 73, 67, 65, 76, 32, 76, 73, 78, + 69, 32, 66, 69, 76, 79, 87, 7, 233, 42, 4, 32, 65, 78, 68, 2, 137, 129, + 10, 8, 68, 32, 80, 79, 76, 73, 83, 72, 16, 26, 79, 219, 159, 4, 69, 13, + 48, 6, 32, 87, 73, 84, 72, 32, 247, 228, 20, 69, 8, 222, 54, 71, 186, 2, + 65, 190, 233, 3, 82, 167, 146, 16, 83, 23, 48, 6, 32, 87, 73, 84, 72, 32, + 175, 208, 20, 72, 18, 86, 70, 34, 83, 178, 55, 65, 186, 220, 3, 77, 174, + 7, 80, 166, 244, 14, 72, 139, 83, 68, 2, 137, 185, 14, 3, 76, 79, 85, 6, + 218, 27, 84, 241, 164, 12, 6, 81, 85, 73, 82, 82, 69, 13, 48, 6, 32, 87, + 73, 84, 72, 32, 215, 155, 4, 80, 8, 42, 68, 16, 4, 72, 79, 79, 75, 23, + 83, 2, 227, 44, 73, 5, 247, 173, 18, 32, 2, 197, 26, 6, 84, 82, 79, 75, + 69, 32, 77, 90, 32, 228, 4, 8, 69, 86, 69, 82, 83, 69, 68, 32, 148, 2, 2, + 85, 77, 255, 141, 4, 65, 46, 36, 4, 87, 73, 84, 72, 235, 6, 82, 44, 26, + 32, 215, 174, 15, 79, 42, 166, 1, 67, 50, 68, 200, 1, 8, 70, 73, 83, 72, + 72, 79, 79, 75, 66, 76, 34, 84, 142, 18, 77, 162, 30, 65, 202, 168, 3, + 73, 158, 59, 80, 154, 235, 3, 79, 231, 172, 12, 83, 6, 170, 19, 69, 230, + 249, 3, 82, 179, 172, 15, 65, 8, 11, 79, 8, 24, 2, 84, 32, 111, 85, 6, + 26, 66, 131, 225, 19, 65, 4, 25, 4, 69, 76, 79, 87, 5, 29, 5, 32, 65, 78, + 68, 32, 2, 139, 244, 3, 77, 2, 21, 3, 66, 76, 69, 2, 11, 32, 2, 227, 46, + 71, 7, 29, 5, 32, 65, 78, 68, 32, 4, 162, 141, 4, 77, 175, 7, 80, 4, 222, + 49, 73, 207, 230, 3, 79, 4, 178, 168, 18, 65, 163, 26, 73, 22, 162, 1, + 72, 40, 6, 79, 80, 69, 78, 32, 69, 164, 142, 4, 8, 82, 32, 87, 73, 84, + 72, 32, 70, 168, 2, 3, 83, 67, 82, 226, 206, 14, 69, 234, 86, 67, 139, + 164, 1, 75, 2, 11, 65, 2, 225, 244, 9, 2, 76, 70, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 214, 151, 4, 82, 207, 238, 14, 72, 5, 11, 32, 2, 11, 82, + 2, 149, 253, 16, 3, 79, 84, 85, 85, 180, 1, 6, 32, 87, 73, 84, 72, 32, + 218, 4, 65, 66, 67, 218, 1, 72, 34, 73, 156, 2, 13, 81, 85, 65, 84, 32, + 82, 69, 86, 69, 82, 83, 69, 68, 193, 229, 9, 6, 84, 73, 82, 82, 85, 80, + 40, 110, 65, 34, 67, 86, 68, 138, 1, 83, 174, 1, 86, 210, 9, 77, 134, + 130, 4, 80, 154, 235, 3, 79, 143, 137, 11, 72, 4, 205, 1, 4, 67, 85, 84, + 69, 12, 58, 65, 230, 10, 69, 82, 79, 202, 31, 73, 231, 245, 6, 85, 4, + 113, 3, 82, 79, 78, 8, 32, 3, 79, 84, 32, 207, 32, 73, 6, 26, 66, 215, + 216, 19, 65, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, 2, 181, 212, 19, 3, 65, 78, 68, 4, 22, 72, 203, 42, 87, 2, 21, 3, 79, 82, 84, 2, 17, 2, 32, 83, - 2, 11, 84, 2, 21, 3, 82, 79, 75, 2, 11, 69, 2, 17, 2, 32, 79, 2, 237, - 235, 18, 4, 86, 69, 82, 76, 2, 37, 7, 69, 82, 84, 73, 67, 65, 76, 2, 205, - 40, 2, 32, 76, 4, 46, 76, 249, 226, 18, 5, 75, 72, 65, 32, 89, 2, 247, + 2, 11, 84, 2, 21, 3, 82, 79, 75, 2, 11, 69, 2, 17, 2, 32, 79, 2, 233, + 199, 19, 4, 86, 69, 82, 76, 2, 37, 7, 69, 82, 84, 73, 67, 65, 76, 2, 205, + 40, 2, 32, 76, 4, 46, 76, 245, 190, 19, 5, 75, 72, 65, 32, 89, 2, 247, 12, 84, 18, 48, 3, 72, 87, 65, 97, 5, 82, 73, 80, 84, 32, 11, 33, 6, 32, - 87, 73, 84, 72, 32, 8, 222, 35, 71, 186, 2, 65, 194, 221, 3, 82, 171, - 161, 14, 72, 8, 26, 82, 147, 243, 3, 71, 5, 173, 141, 11, 5, 32, 87, 73, - 84, 72, 2, 233, 161, 16, 3, 65, 82, 80, 14, 48, 7, 68, 69, 87, 65, 89, - 83, 32, 199, 1, 71, 12, 110, 79, 56, 4, 84, 85, 82, 78, 224, 160, 10, 11, - 68, 73, 65, 69, 82, 69, 83, 73, 90, 69, 68, 135, 210, 9, 85, 7, 26, 80, - 203, 245, 3, 32, 2, 145, 203, 9, 2, 69, 78, 2, 173, 220, 12, 2, 69, 68, - 2, 209, 159, 16, 4, 77, 79, 73, 68, 2, 227, 219, 9, 32, 147, 1, 188, 1, + 87, 73, 84, 72, 32, 8, 222, 35, 71, 186, 2, 65, 190, 233, 3, 82, 207, + 238, 14, 72, 8, 26, 82, 155, 255, 3, 71, 5, 253, 175, 11, 5, 32, 87, 73, + 84, 72, 2, 225, 219, 16, 3, 65, 82, 80, 14, 48, 7, 68, 69, 87, 65, 89, + 83, 32, 199, 1, 71, 12, 110, 79, 56, 4, 84, 85, 82, 78, 176, 195, 10, 11, + 68, 73, 65, 69, 82, 69, 83, 73, 90, 69, 68, 203, 139, 10, 85, 7, 26, 80, + 211, 129, 4, 32, 2, 165, 234, 9, 2, 69, 78, 2, 237, 255, 12, 2, 69, 68, + 2, 201, 217, 16, 4, 77, 79, 73, 68, 2, 247, 250, 9, 32, 147, 1, 188, 1, 6, 32, 87, 73, 84, 72, 32, 192, 3, 2, 69, 83, 70, 72, 130, 2, 79, 102, - 82, 54, 85, 242, 241, 3, 67, 202, 1, 83, 136, 210, 10, 9, 65, 73, 76, 76, - 69, 83, 83, 32, 80, 243, 163, 5, 90, 34, 110, 67, 182, 1, 68, 66, 77, - 170, 31, 76, 214, 214, 3, 80, 180, 5, 4, 72, 79, 79, 75, 50, 82, 143, - 194, 15, 83, 10, 58, 69, 22, 73, 62, 79, 182, 129, 7, 85, 147, 201, 11, - 65, 2, 195, 251, 11, 68, 2, 37, 7, 82, 67, 85, 77, 70, 76, 69, 2, 255, - 199, 17, 88, 2, 17, 2, 77, 77, 2, 215, 199, 17, 65, 8, 32, 2, 73, 65, - 215, 255, 15, 79, 4, 154, 21, 71, 215, 6, 69, 4, 17, 2, 73, 68, 4, 26, - 45, 195, 238, 3, 68, 2, 229, 252, 3, 6, 72, 69, 73, 71, 72, 84, 6, 45, 9, - 72, 32, 68, 73, 71, 82, 65, 80, 72, 7, 183, 244, 3, 32, 8, 64, 12, 32, - 87, 73, 84, 72, 32, 83, 84, 82, 73, 75, 69, 43, 79, 2, 245, 243, 15, 5, + 82, 54, 85, 250, 253, 3, 67, 202, 1, 83, 132, 247, 10, 9, 65, 73, 76, 76, + 69, 83, 83, 32, 80, 131, 207, 5, 90, 34, 110, 67, 182, 1, 68, 66, 77, + 170, 31, 76, 222, 226, 3, 80, 168, 5, 4, 72, 79, 79, 75, 50, 82, 167, + 146, 16, 83, 10, 58, 69, 22, 73, 62, 79, 174, 149, 7, 85, 155, 144, 12, + 65, 2, 251, 157, 12, 68, 2, 37, 7, 82, 67, 85, 77, 70, 76, 69, 2, 179, + 157, 18, 88, 2, 17, 2, 77, 77, 2, 139, 157, 18, 65, 8, 32, 2, 73, 65, + 179, 175, 16, 79, 4, 154, 21, 71, 215, 6, 69, 4, 17, 2, 73, 68, 4, 26, + 45, 203, 250, 3, 68, 2, 225, 136, 4, 6, 72, 69, 73, 71, 72, 84, 6, 45, 9, + 72, 32, 68, 73, 71, 82, 65, 80, 72, 7, 191, 128, 4, 32, 8, 64, 12, 32, + 87, 73, 84, 72, 32, 83, 84, 82, 73, 75, 69, 43, 79, 2, 205, 163, 16, 5, 84, 72, 82, 79, 85, 6, 17, 2, 82, 78, 7, 41, 8, 32, 87, 73, 84, 72, 32, - 83, 84, 4, 25, 4, 82, 79, 75, 69, 5, 11, 32, 2, 213, 197, 3, 6, 84, 72, - 82, 79, 85, 71, 8, 26, 78, 143, 244, 3, 80, 6, 17, 2, 69, 32, 6, 234, - 188, 4, 83, 190, 160, 9, 84, 215, 132, 1, 70, 2, 17, 2, 69, 83, 2, 11, - 73, 2, 243, 198, 19, 76, 76, 44, 5, 82, 78, 69, 68, 32, 131, 233, 19, 77, - 74, 162, 1, 68, 22, 72, 66, 73, 54, 79, 142, 1, 82, 110, 84, 22, 86, 138, - 240, 3, 65, 50, 77, 198, 2, 89, 246, 221, 15, 85, 218, 19, 69, 2, 71, 2, - 75, 2, 76, 3, 87, 2, 191, 195, 16, 69, 7, 209, 224, 3, 11, 32, 87, 73, - 84, 72, 32, 70, 73, 83, 72, 72, 5, 11, 78, 2, 149, 181, 9, 5, 83, 85, 76, - 65, 82, 12, 66, 69, 176, 213, 3, 7, 32, 79, 80, 69, 78, 45, 79, 167, 29, - 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 166, 200, 17, 72, 203, 237, 1, - 83, 15, 33, 6, 32, 87, 73, 84, 72, 32, 12, 138, 230, 3, 77, 174, 7, 80, - 142, 5, 76, 226, 197, 13, 84, 163, 92, 72, 5, 203, 237, 3, 32, 7, 11, 32, - 4, 161, 5, 4, 87, 73, 84, 72, 97, 90, 32, 224, 210, 3, 6, 80, 83, 73, 76, - 79, 78, 226, 144, 16, 69, 2, 73, 2, 77, 3, 79, 82, 48, 3, 66, 65, 82, 49, + 83, 84, 4, 25, 4, 82, 79, 75, 69, 5, 11, 32, 2, 209, 209, 3, 6, 84, 72, + 82, 79, 85, 71, 8, 26, 78, 151, 128, 4, 80, 6, 17, 2, 69, 32, 6, 134, + 201, 4, 83, 230, 197, 9, 84, 255, 131, 1, 70, 2, 17, 2, 69, 83, 2, 11, + 73, 2, 135, 163, 20, 76, 76, 44, 5, 82, 78, 69, 68, 32, 151, 197, 20, 77, + 74, 162, 1, 68, 22, 72, 66, 73, 54, 79, 142, 1, 82, 110, 84, 22, 86, 146, + 252, 3, 65, 38, 77, 198, 2, 89, 142, 174, 16, 85, 218, 19, 69, 2, 71, 2, + 75, 2, 76, 3, 87, 2, 223, 138, 17, 69, 7, 217, 236, 3, 11, 32, 87, 73, + 84, 72, 32, 70, 73, 83, 72, 72, 5, 11, 78, 2, 169, 212, 9, 5, 83, 85, 76, + 65, 82, 12, 66, 69, 180, 225, 3, 7, 32, 79, 80, 69, 78, 45, 79, 159, 29, + 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 202, 157, 18, 72, 187, 244, 1, + 83, 15, 33, 6, 32, 87, 73, 84, 72, 32, 12, 146, 242, 3, 77, 174, 7, 80, + 130, 5, 76, 154, 143, 14, 84, 143, 96, 72, 5, 211, 249, 3, 32, 7, 11, 32, + 4, 161, 5, 4, 87, 73, 84, 72, 97, 90, 32, 228, 222, 3, 6, 80, 83, 73, 76, + 79, 78, 242, 224, 16, 69, 2, 73, 2, 77, 3, 79, 82, 48, 3, 66, 65, 82, 49, 5, 87, 73, 84, 72, 32, 5, 245, 17, 8, 32, 87, 73, 84, 72, 32, 83, 72, 78, 142, 1, 67, 74, 68, 150, 2, 72, 170, 1, 77, 134, 1, 79, 106, 82, 38, 84, - 186, 9, 71, 82, 83, 234, 1, 65, 206, 156, 3, 73, 62, 66, 135, 66, 76, 6, - 172, 185, 4, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 147, 132, 14, 65, 18, + 186, 9, 71, 82, 83, 234, 1, 65, 202, 168, 3, 73, 62, 66, 135, 66, 76, 6, + 200, 197, 4, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 247, 210, 14, 65, 18, 52, 8, 73, 65, 69, 82, 69, 83, 73, 83, 131, 1, 79, 13, 11, 32, 10, 40, 4, - 65, 78, 68, 32, 223, 199, 17, 66, 8, 50, 67, 226, 13, 71, 186, 2, 65, - 183, 182, 3, 77, 2, 211, 187, 18, 65, 6, 26, 85, 207, 184, 17, 84, 4, 21, + 65, 78, 68, 32, 155, 158, 18, 66, 8, 50, 67, 226, 13, 71, 186, 2, 65, + 187, 194, 3, 77, 2, 211, 150, 19, 65, 6, 26, 85, 131, 142, 18, 84, 4, 21, 3, 66, 76, 69, 4, 11, 32, 4, 138, 13, 71, 187, 2, 65, 14, 11, 79, 14, 28, - 2, 82, 78, 251, 75, 79, 13, 29, 5, 32, 65, 78, 68, 32, 10, 66, 72, 226, - 11, 71, 186, 2, 65, 242, 216, 7, 68, 179, 181, 8, 84, 2, 145, 75, 2, 79, + 2, 82, 78, 191, 81, 79, 13, 29, 5, 32, 65, 78, 68, 32, 10, 66, 72, 226, + 11, 71, 186, 2, 65, 178, 236, 7, 68, 211, 219, 8, 84, 2, 213, 80, 2, 79, 79, 10, 29, 5, 65, 67, 82, 79, 78, 11, 29, 5, 32, 65, 78, 68, 32, 8, 50, - 68, 214, 10, 71, 186, 2, 65, 163, 142, 16, 84, 2, 171, 10, 73, 6, 29, 5, + 68, 214, 10, 71, 186, 2, 65, 131, 200, 16, 84, 2, 171, 10, 73, 6, 29, 5, 71, 79, 78, 69, 75, 7, 11, 32, 4, 25, 4, 65, 78, 68, 32, 4, 178, 12, 65, - 163, 142, 16, 84, 4, 222, 233, 3, 69, 131, 200, 13, 73, 6, 25, 4, 73, 76, - 68, 69, 7, 11, 32, 4, 26, 65, 191, 194, 17, 66, 2, 17, 2, 78, 68, 2, 11, + 131, 200, 16, 84, 4, 218, 245, 3, 69, 187, 145, 14, 73, 6, 25, 4, 73, 76, + 68, 69, 7, 11, 32, 4, 26, 65, 251, 152, 18, 66, 2, 17, 2, 78, 68, 2, 11, 32, 2, 139, 11, 65, 29, 84, 6, 32, 87, 73, 84, 72, 32, 162, 1, 73, 66, - 79, 174, 202, 18, 69, 227, 141, 1, 89, 14, 82, 68, 174, 225, 3, 80, 142, - 1, 67, 158, 7, 82, 158, 174, 12, 84, 203, 240, 1, 72, 4, 26, 73, 179, - 226, 7, 79, 2, 17, 2, 65, 71, 2, 189, 168, 19, 2, 79, 78, 2, 41, 8, 83, - 73, 71, 79, 84, 72, 73, 67, 2, 175, 225, 15, 32, 6, 33, 6, 76, 65, 80, - 85, 75, 32, 6, 146, 193, 19, 65, 2, 79, 3, 85, 19, 33, 6, 32, 87, 73, 84, - 72, 32, 16, 202, 4, 67, 18, 68, 70, 71, 186, 2, 65, 194, 165, 17, 82, - 171, 89, 72, 17, 33, 6, 32, 87, 73, 84, 72, 32, 14, 42, 68, 32, 2, 76, - 79, 255, 221, 3, 80, 4, 222, 3, 73, 131, 210, 18, 79, 8, 60, 11, 78, 71, + 79, 170, 166, 19, 69, 251, 141, 1, 89, 14, 82, 68, 182, 237, 3, 80, 142, + 1, 67, 146, 7, 82, 130, 220, 12, 84, 139, 144, 2, 72, 4, 26, 73, 243, + 245, 7, 79, 2, 17, 2, 65, 71, 2, 209, 132, 20, 2, 79, 78, 2, 41, 8, 83, + 73, 71, 79, 84, 72, 73, 67, 2, 135, 145, 16, 32, 6, 33, 6, 76, 65, 80, + 85, 75, 32, 6, 166, 157, 20, 65, 2, 79, 3, 85, 19, 33, 6, 32, 87, 73, 84, + 72, 32, 16, 202, 4, 67, 18, 68, 70, 71, 186, 2, 65, 246, 250, 17, 82, + 151, 93, 72, 17, 33, 6, 32, 87, 73, 84, 72, 32, 14, 42, 68, 32, 2, 76, + 79, 135, 234, 3, 80, 4, 222, 3, 73, 255, 173, 19, 79, 8, 60, 11, 78, 71, 32, 76, 69, 70, 84, 32, 76, 69, 71, 79, 87, 7, 11, 32, 4, 60, 7, 65, 78, 68, 32, 76, 79, 87, 65, 4, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 21, 3, - 73, 71, 72, 2, 215, 236, 10, 84, 2, 197, 210, 18, 4, 32, 83, 69, 82, 33, - 48, 6, 32, 87, 73, 84, 72, 32, 187, 219, 15, 79, 28, 110, 67, 18, 68, 70, - 71, 22, 72, 42, 76, 22, 83, 234, 1, 65, 182, 182, 3, 77, 238, 215, 12, - 84, 163, 151, 1, 82, 2, 203, 3, 73, 6, 26, 73, 251, 227, 15, 79, 2, 17, - 2, 65, 69, 2, 243, 247, 15, 82, 2, 207, 233, 17, 82, 4, 17, 2, 79, 79, 4, - 187, 246, 4, 75, 2, 191, 216, 18, 79, 4, 26, 72, 167, 161, 19, 84, 2, 21, - 3, 79, 82, 84, 2, 245, 226, 7, 6, 32, 82, 73, 71, 72, 84, 31, 33, 6, 32, - 87, 73, 84, 72, 32, 28, 98, 65, 22, 67, 82, 68, 38, 76, 34, 83, 138, 207, - 3, 77, 174, 7, 80, 230, 5, 82, 171, 161, 14, 72, 2, 175, 140, 13, 67, 6, - 42, 73, 238, 225, 6, 85, 147, 201, 11, 65, 2, 177, 158, 11, 4, 82, 67, - 85, 77, 6, 234, 169, 3, 69, 203, 182, 12, 79, 2, 11, 73, 2, 243, 137, 12, - 78, 4, 26, 87, 151, 158, 19, 84, 2, 195, 190, 16, 65, 18, 90, 70, 222, - 157, 9, 73, 172, 9, 6, 76, 79, 78, 71, 32, 83, 226, 137, 10, 83, 219, 5, - 79, 10, 34, 70, 242, 204, 19, 73, 3, 76, 7, 238, 204, 19, 73, 3, 76, 36, - 150, 1, 83, 190, 203, 19, 65, 2, 69, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, - 2, 77, 2, 78, 2, 79, 2, 80, 2, 82, 2, 84, 2, 85, 2, 86, 3, 88, 5, 139, - 212, 4, 67, 160, 5, 200, 1, 2, 65, 70, 104, 2, 70, 84, 186, 33, 79, 20, - 5, 80, 67, 72, 65, 32, 164, 8, 8, 83, 83, 45, 84, 72, 65, 78, 32, 142, - 231, 15, 68, 248, 22, 6, 86, 69, 76, 32, 83, 76, 158, 209, 2, 77, 227, - 79, 71, 4, 180, 180, 1, 14, 32, 70, 76, 85, 84, 84, 69, 82, 73, 78, 71, - 32, 73, 78, 169, 186, 11, 3, 89, 32, 71, 200, 3, 58, 32, 186, 17, 45, - 169, 1, 6, 87, 65, 82, 68, 83, 32, 218, 1, 190, 1, 65, 174, 3, 66, 202, - 1, 67, 96, 2, 68, 79, 112, 2, 72, 65, 250, 2, 76, 46, 77, 38, 79, 38, 82, - 146, 3, 83, 82, 84, 174, 1, 87, 190, 159, 8, 70, 210, 3, 78, 202, 1, 80, - 187, 11, 86, 26, 22, 78, 211, 2, 82, 20, 28, 2, 68, 32, 131, 2, 71, 14, - 72, 3, 76, 79, 87, 0, 3, 85, 80, 80, 93, 6, 82, 73, 71, 72, 84, 32, 4, - 21, 3, 69, 82, 32, 4, 132, 190, 16, 9, 65, 78, 68, 32, 82, 73, 71, 72, - 84, 175, 30, 79, 6, 50, 84, 161, 166, 17, 6, 68, 79, 85, 66, 76, 69, 4, - 138, 190, 16, 82, 131, 216, 1, 65, 6, 202, 163, 8, 69, 185, 1, 4, 76, 69, - 32, 66, 6, 26, 67, 167, 167, 8, 82, 2, 129, 167, 8, 5, 32, 76, 69, 83, - 83, 12, 40, 4, 65, 82, 66, 32, 191, 167, 8, 76, 8, 40, 4, 68, 79, 87, 78, - 1, 2, 85, 80, 4, 57, 12, 32, 82, 73, 71, 72, 84, 32, 66, 65, 82, 66, 32, - 4, 208, 176, 16, 4, 68, 79, 87, 78, 1, 2, 85, 80, 14, 130, 167, 8, 85, - 230, 21, 79, 250, 222, 2, 69, 217, 153, 2, 8, 76, 79, 83, 69, 68, 32, 69, - 78, 10, 44, 5, 85, 66, 76, 69, 32, 175, 167, 8, 84, 8, 226, 168, 8, 87, - 234, 24, 65, 198, 248, 2, 81, 151, 172, 4, 80, 26, 36, 3, 76, 70, 32, - 203, 170, 8, 78, 24, 216, 1, 6, 67, 73, 82, 67, 76, 69, 190, 167, 8, 66, - 90, 70, 34, 82, 128, 147, 8, 30, 73, 78, 86, 69, 82, 83, 69, 32, 77, 69, - 68, 73, 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, 68, 32, 82, 73, 71, - 72, 84, 139, 19, 77, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, 84, 254, - 160, 17, 70, 175, 204, 1, 68, 4, 166, 187, 5, 72, 219, 180, 12, 87, 4, - 252, 148, 1, 2, 85, 71, 207, 147, 7, 79, 2, 161, 172, 18, 4, 85, 76, 84, - 73, 6, 214, 169, 8, 85, 215, 150, 8, 78, 30, 44, 5, 73, 71, 72, 84, 32, - 219, 170, 8, 65, 28, 152, 1, 5, 65, 82, 82, 79, 87, 132, 1, 12, 68, 79, - 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 30, 87, 218, 209, 8, 79, 234, - 215, 7, 66, 54, 83, 159, 53, 84, 11, 11, 32, 8, 72, 5, 87, 73, 84, 72, - 32, 193, 234, 17, 7, 84, 72, 82, 79, 85, 71, 72, 6, 146, 248, 15, 68, - 190, 145, 3, 86, 79, 83, 7, 197, 201, 8, 2, 32, 87, 4, 202, 220, 8, 65, - 255, 248, 7, 72, 34, 238, 168, 8, 45, 70, 69, 78, 73, 168, 1, 3, 80, 69, - 69, 26, 81, 227, 2, 85, 18, 58, 82, 246, 173, 8, 72, 178, 178, 7, 79, - 179, 169, 2, 65, 6, 40, 4, 73, 65, 78, 71, 143, 175, 8, 65, 4, 208, 227, - 4, 8, 76, 69, 32, 66, 69, 83, 73, 68, 235, 230, 11, 85, 16, 174, 177, 8, - 72, 202, 1, 73, 141, 246, 9, 2, 82, 73, 30, 90, 83, 230, 178, 8, 70, 206, - 1, 72, 130, 1, 80, 189, 3, 7, 84, 79, 45, 82, 73, 71, 72, 4, 44, 5, 73, - 68, 69, 32, 65, 171, 183, 8, 72, 2, 209, 138, 1, 2, 82, 67, 208, 1, 172, - 1, 5, 65, 82, 82, 79, 87, 134, 5, 66, 70, 72, 134, 4, 84, 178, 2, 87, - 230, 182, 8, 68, 210, 1, 70, 222, 7, 76, 26, 79, 34, 80, 50, 82, 70, 83, - 134, 136, 8, 67, 47, 81, 71, 26, 32, 247, 134, 17, 45, 66, 102, 65, 138, - 1, 84, 132, 2, 5, 87, 73, 84, 72, 32, 238, 184, 8, 70, 173, 215, 9, 4, - 79, 86, 69, 82, 12, 44, 5, 66, 79, 86, 69, 32, 219, 186, 8, 78, 10, 64, - 4, 83, 72, 79, 82, 210, 185, 8, 82, 202, 208, 4, 65, 55, 84, 2, 175, 146, - 18, 84, 12, 56, 7, 72, 82, 79, 85, 71, 72, 32, 53, 3, 79, 32, 66, 6, 242, - 135, 13, 83, 134, 155, 4, 76, 223, 142, 2, 88, 6, 32, 2, 65, 82, 199, - 187, 8, 76, 5, 221, 128, 12, 23, 32, 79, 86, 69, 82, 32, 82, 73, 71, 72, - 84, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 36, 154, 188, 8, 68, - 98, 76, 38, 80, 30, 83, 38, 84, 210, 139, 8, 77, 38, 78, 122, 69, 226, - 148, 1, 72, 155, 160, 1, 86, 8, 194, 189, 8, 65, 224, 10, 5, 79, 84, 84, - 79, 77, 251, 129, 8, 76, 30, 26, 65, 163, 207, 16, 69, 26, 48, 6, 82, 80, - 79, 79, 78, 32, 243, 231, 18, 78, 24, 100, 10, 87, 73, 84, 72, 32, 66, - 65, 82, 66, 32, 213, 191, 8, 9, 79, 86, 69, 82, 32, 82, 73, 71, 72, 22, - 40, 4, 68, 79, 87, 78, 117, 2, 85, 80, 10, 26, 32, 251, 212, 16, 87, 8, - 226, 192, 8, 66, 188, 2, 7, 65, 66, 79, 86, 69, 32, 82, 130, 138, 8, 70, - 175, 5, 84, 12, 26, 32, 135, 212, 16, 87, 10, 60, 6, 65, 66, 79, 86, 69, - 32, 238, 203, 16, 70, 175, 5, 84, 6, 42, 76, 205, 192, 8, 4, 82, 73, 71, - 72, 4, 242, 190, 8, 69, 207, 201, 4, 79, 54, 44, 2, 82, 73, 194, 196, 8, - 79, 159, 5, 87, 34, 44, 5, 65, 78, 71, 76, 69, 247, 200, 8, 80, 30, 56, - 8, 45, 72, 69, 65, 68, 69, 68, 32, 243, 218, 16, 32, 28, 52, 5, 65, 82, - 82, 79, 87, 210, 210, 16, 68, 39, 80, 25, 11, 32, 22, 184, 197, 8, 9, 79, - 86, 69, 82, 32, 82, 73, 71, 72, 22, 87, 155, 137, 8, 84, 6, 26, 72, 131, - 203, 8, 65, 4, 45, 9, 73, 84, 69, 32, 65, 82, 82, 79, 87, 5, 165, 216, - 16, 2, 32, 87, 5, 215, 144, 18, 80, 148, 1, 252, 1, 15, 67, 79, 78, 83, - 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 124, 7, 76, 69, 84, 84, 69, - 82, 32, 236, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 198, - 1, 83, 172, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 175, 177, - 17, 68, 18, 66, 75, 22, 78, 202, 164, 19, 76, 2, 77, 2, 80, 2, 82, 3, 84, - 5, 239, 211, 18, 65, 5, 145, 170, 12, 4, 89, 73, 78, 45, 78, 194, 1, 75, - 2, 80, 198, 134, 7, 68, 250, 149, 10, 66, 2, 70, 2, 71, 2, 72, 2, 77, - 254, 14, 78, 202, 175, 1, 84, 46, 67, 2, 83, 254, 68, 74, 2, 76, 2, 82, - 2, 86, 2, 87, 2, 89, 187, 2, 65, 6, 166, 160, 19, 72, 2, 76, 187, 2, 65, - 10, 82, 84, 40, 14, 78, 89, 69, 84, 32, 84, 72, 89, 79, 79, 77, 32, 84, - 65, 43, 67, 6, 38, 65, 21, 5, 83, 72, 79, 79, 75, 2, 255, 233, 6, 45, 5, - 17, 2, 32, 67, 2, 149, 220, 14, 3, 69, 82, 45, 8, 92, 4, 73, 71, 78, 32, - 37, 15, 85, 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 4, - 202, 186, 15, 78, 151, 158, 2, 82, 4, 11, 32, 4, 170, 157, 19, 82, 3, 89, - 14, 170, 208, 15, 85, 158, 144, 1, 79, 202, 188, 2, 65, 186, 2, 69, 3, - 73, 52, 138, 1, 65, 128, 4, 11, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, - 82, 200, 1, 2, 66, 85, 42, 67, 186, 1, 79, 210, 2, 87, 175, 141, 18, 83, - 16, 40, 5, 66, 79, 86, 69, 32, 171, 4, 78, 12, 152, 1, 7, 71, 82, 69, 65, - 84, 69, 82, 110, 83, 176, 1, 19, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, - 69, 32, 69, 81, 85, 65, 76, 32, 65, 207, 170, 15, 76, 2, 181, 5, 23, 45, - 84, 72, 65, 78, 32, 65, 66, 79, 86, 69, 32, 68, 79, 85, 66, 76, 69, 45, - 76, 73, 78, 69, 6, 152, 1, 7, 73, 77, 73, 76, 65, 82, 32, 93, 26, 76, 65, - 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, 32, 71, - 82, 69, 65, 84, 69, 82, 4, 18, 65, 59, 79, 2, 25, 4, 66, 79, 86, 69, 2, - 241, 138, 17, 2, 32, 71, 2, 227, 2, 82, 2, 145, 2, 6, 45, 84, 72, 65, 78, - 32, 4, 17, 2, 68, 32, 4, 220, 3, 5, 78, 79, 84, 32, 65, 129, 238, 12, 11, - 83, 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 4, 253, 215, 12, 5, 84, 32, - 78, 79, 84, 4, 65, 14, 76, 79, 83, 69, 68, 32, 66, 89, 32, 67, 85, 82, - 86, 69, 5, 11, 32, 2, 61, 13, 65, 66, 79, 86, 69, 32, 83, 76, 65, 78, 84, - 69, 68, 2, 17, 2, 32, 69, 2, 183, 186, 14, 81, 18, 40, 2, 82, 32, 193, - 190, 11, 2, 86, 69, 16, 114, 65, 48, 16, 83, 76, 65, 78, 84, 69, 68, 32, - 69, 81, 85, 65, 76, 32, 84, 79, 230, 212, 12, 69, 155, 177, 4, 71, 2, - 201, 154, 9, 7, 80, 80, 82, 79, 88, 73, 77, 9, 49, 10, 32, 87, 73, 84, - 72, 32, 68, 79, 84, 32, 6, 26, 65, 175, 170, 11, 73, 4, 25, 4, 66, 79, - 86, 69, 5, 139, 198, 17, 32, 6, 25, 4, 73, 84, 72, 32, 6, 66, 67, 40, 8, - 81, 85, 69, 83, 84, 73, 79, 78, 203, 194, 18, 68, 2, 133, 169, 11, 5, 73, - 82, 67, 76, 69, 2, 17, 2, 32, 77, 2, 17, 2, 65, 82, 2, 215, 149, 18, 75, - 136, 11, 190, 1, 71, 186, 5, 77, 162, 7, 78, 168, 65, 2, 80, 83, 20, 3, - 83, 85, 32, 130, 135, 15, 82, 152, 34, 11, 86, 82, 69, 32, 84, 79, 85, - 82, 78, 79, 73, 234, 44, 79, 130, 214, 1, 90, 175, 82, 66, 44, 26, 65, - 57, 2, 72, 84, 2, 165, 142, 9, 9, 84, 85, 82, 69, 32, 79, 80, 69, 78, 42, - 38, 32, 137, 4, 4, 78, 73, 78, 71, 36, 146, 1, 69, 38, 70, 126, 82, 40, - 3, 76, 69, 70, 86, 83, 42, 84, 152, 184, 7, 3, 66, 76, 85, 134, 224, 8, - 87, 134, 36, 86, 214, 74, 71, 195, 68, 67, 2, 201, 253, 16, 4, 73, 71, - 72, 84, 8, 94, 73, 201, 201, 17, 17, 79, 85, 82, 32, 80, 79, 73, 78, 84, - 69, 68, 32, 66, 76, 65, 67, 75, 4, 221, 249, 16, 2, 86, 69, 4, 36, 3, 73, - 71, 72, 171, 224, 16, 65, 2, 241, 230, 1, 16, 84, 32, 84, 79, 82, 84, 79, - 73, 83, 69, 32, 83, 72, 69, 76, 76, 6, 138, 157, 16, 72, 242, 93, 65, 43, - 73, 4, 216, 216, 15, 2, 87, 69, 21, 3, 72, 82, 69, 7, 29, 5, 32, 77, 79, - 79, 68, 5, 163, 130, 8, 32, 138, 1, 80, 3, 66, 85, 32, 229, 136, 8, 11, - 73, 84, 69, 68, 32, 76, 73, 65, 66, 73, 76, 136, 1, 128, 1, 7, 76, 69, - 84, 84, 69, 82, 32, 218, 1, 83, 200, 2, 5, 86, 79, 87, 69, 76, 206, 173, - 12, 69, 190, 141, 4, 81, 155, 92, 68, 60, 166, 1, 71, 174, 254, 5, 84, - 174, 192, 8, 89, 214, 122, 78, 134, 135, 3, 83, 82, 66, 2, 67, 2, 68, 2, - 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, 82, 3, 87, 6, 254, 192, - 17, 89, 254, 196, 1, 72, 187, 2, 65, 32, 108, 4, 73, 71, 78, 32, 128, 1, - 12, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 193, 236, 6, 2, 85, - 66, 8, 72, 3, 75, 69, 77, 0, 3, 77, 85, 75, 32, 2, 83, 65, 227, 197, 16, - 76, 2, 149, 202, 16, 3, 80, 72, 82, 2, 207, 242, 18, 45, 18, 138, 183, - 15, 78, 158, 138, 3, 65, 182, 66, 75, 2, 76, 2, 77, 2, 80, 2, 82, 3, 84, - 20, 84, 6, 32, 83, 73, 71, 78, 32, 253, 82, 10, 45, 67, 65, 82, 82, 73, - 69, 82, 32, 76, 18, 142, 199, 5, 65, 214, 254, 10, 79, 238, 254, 1, 69, - 150, 64, 73, 3, 85, 226, 8, 22, 69, 199, 64, 75, 222, 8, 34, 32, 177, 3, - 3, 65, 82, 32, 14, 120, 12, 73, 78, 84, 69, 71, 82, 65, 84, 73, 79, 78, - 32, 190, 240, 12, 70, 178, 249, 2, 83, 193, 130, 1, 4, 84, 65, 66, 85, 6, - 108, 5, 87, 73, 84, 72, 32, 157, 1, 17, 78, 79, 84, 32, 73, 78, 67, 76, - 85, 68, 73, 78, 71, 32, 84, 72, 69, 4, 76, 7, 82, 69, 67, 84, 65, 78, 71, - 1, 8, 83, 69, 77, 73, 67, 73, 82, 67, 2, 73, 16, 85, 76, 65, 82, 32, 80, - 65, 84, 72, 32, 65, 82, 79, 85, 78, 68, 2, 17, 2, 32, 80, 2, 151, 223, - 17, 79, 208, 8, 60, 8, 65, 32, 83, 73, 71, 78, 32, 65, 221, 24, 2, 66, - 32, 170, 5, 122, 49, 86, 51, 202, 2, 52, 214, 1, 53, 158, 4, 54, 142, 3, - 55, 148, 4, 2, 56, 48, 78, 66, 201, 179, 17, 3, 48, 50, 56, 6, 140, 155, - 12, 5, 48, 48, 45, 49, 48, 176, 169, 5, 2, 50, 48, 197, 19, 2, 51, 49, - 150, 1, 78, 48, 90, 49, 130, 1, 55, 210, 241, 13, 50, 2, 51, 2, 52, 2, - 53, 3, 54, 22, 178, 1, 57, 138, 252, 18, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 2, 55, 3, 56, 24, 90, 51, 138, 252, 18, 48, 2, 49, 2, 50, 2, 52, - 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 6, 134, 252, 18, 65, 2, 66, 3, 67, 4, - 226, 251, 18, 48, 3, 49, 38, 18, 48, 91, 49, 20, 162, 1, 48, 2, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 74, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 2, 161, 164, 6, 2, 45, - 86, 160, 1, 102, 48, 78, 49, 62, 51, 86, 52, 70, 53, 86, 54, 206, 11, 50, - 250, 159, 2, 57, 190, 192, 11, 55, 3, 56, 16, 134, 249, 18, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 12, 186, 248, 18, 48, 2, 49, 2, - 50, 2, 51, 2, 53, 3, 54, 18, 254, 247, 18, 48, 2, 49, 2, 50, 2, 52, 2, - 53, 2, 54, 2, 55, 2, 56, 3, 57, 14, 170, 247, 18, 48, 2, 49, 2, 50, 2, - 53, 2, 55, 2, 56, 3, 57, 18, 230, 246, 18, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 146, 246, 18, 51, 2, 52, 2, 53, 2, - 54, 2, 56, 3, 57, 104, 70, 48, 78, 50, 86, 51, 38, 52, 78, 54, 254, 230, - 13, 53, 243, 1, 49, 16, 142, 245, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 54, 2, 56, 3, 57, 18, 194, 244, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 54, 2, 55, 2, 56, 3, 57, 6, 238, 243, 18, 52, 2, 55, 3, 56, 16, 202, 243, - 18, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 10, 254, 242, - 18, 48, 2, 49, 2, 50, 2, 51, 3, 52, 44, 86, 48, 134, 2, 49, 140, 246, 1, - 2, 51, 50, 157, 232, 16, 6, 50, 54, 32, 69, 89, 89, 26, 110, 57, 194, - 193, 8, 55, 182, 6, 50, 62, 54, 218, 58, 52, 198, 148, 3, 51, 110, 56, - 202, 196, 2, 49, 215, 94, 53, 10, 26, 45, 207, 231, 17, 32, 8, 96, 2, 50, - 32, 212, 136, 12, 3, 54, 32, 76, 184, 2, 3, 52, 32, 76, 141, 145, 3, 3, - 51, 32, 76, 2, 175, 140, 12, 76, 14, 110, 49, 22, 51, 32, 3, 52, 32, 65, - 0, 2, 53, 32, 198, 147, 4, 50, 202, 181, 4, 48, 141, 209, 6, 2, 55, 32, - 2, 243, 226, 17, 32, 2, 11, 32, 2, 219, 238, 11, 79, 2, 167, 180, 17, 66, - 16, 182, 238, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, + 73, 71, 72, 2, 167, 143, 11, 84, 2, 193, 174, 19, 4, 32, 83, 69, 82, 33, + 48, 6, 32, 87, 73, 84, 72, 32, 147, 139, 16, 79, 28, 110, 67, 18, 68, 70, + 71, 22, 72, 42, 76, 22, 83, 234, 1, 65, 186, 194, 3, 77, 202, 133, 13, + 84, 247, 178, 1, 82, 2, 203, 3, 73, 6, 26, 73, 215, 147, 16, 79, 2, 17, + 2, 65, 69, 2, 151, 172, 16, 82, 2, 223, 194, 18, 82, 4, 17, 2, 79, 79, 4, + 171, 131, 5, 75, 2, 195, 180, 19, 79, 4, 26, 72, 187, 253, 19, 84, 2, 21, + 3, 79, 82, 84, 2, 233, 246, 7, 6, 32, 82, 73, 71, 72, 84, 31, 33, 6, 32, + 87, 73, 84, 72, 32, 28, 98, 65, 22, 67, 82, 68, 38, 76, 34, 83, 146, 219, + 3, 77, 174, 7, 80, 218, 5, 82, 207, 238, 14, 72, 2, 231, 179, 13, 67, 6, + 42, 73, 230, 245, 6, 85, 155, 144, 12, 65, 2, 241, 192, 11, 4, 82, 67, + 85, 77, 6, 230, 181, 3, 69, 171, 218, 12, 79, 2, 11, 73, 2, 179, 172, 12, + 78, 4, 26, 87, 171, 250, 19, 84, 2, 179, 133, 17, 65, 18, 90, 70, 242, + 188, 9, 73, 172, 9, 6, 76, 79, 78, 71, 32, 83, 226, 198, 10, 83, 219, 5, + 79, 10, 34, 70, 134, 169, 20, 73, 3, 76, 7, 130, 169, 20, 73, 3, 76, 36, + 150, 1, 83, 210, 167, 20, 65, 2, 69, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, + 2, 77, 2, 78, 2, 79, 2, 80, 2, 82, 2, 84, 2, 85, 2, 86, 3, 88, 5, 143, + 225, 4, 67, 224, 5, 204, 1, 2, 65, 70, 144, 1, 2, 70, 84, 210, 38, 79, + 20, 5, 80, 67, 72, 65, 32, 164, 8, 8, 83, 83, 45, 84, 72, 65, 78, 32, + 254, 167, 16, 68, 228, 23, 6, 86, 69, 76, 32, 83, 76, 134, 230, 2, 77, + 239, 79, 71, 6, 120, 3, 76, 69, 83, 204, 191, 1, 14, 32, 70, 76, 85, 84, + 84, 69, 82, 73, 78, 71, 32, 73, 78, 209, 212, 11, 3, 89, 32, 71, 2, 135, + 228, 14, 83, 134, 4, 58, 32, 250, 20, 45, 217, 2, 6, 87, 65, 82, 68, 83, + 32, 246, 1, 166, 2, 65, 234, 4, 66, 202, 1, 67, 96, 2, 68, 79, 112, 2, + 72, 65, 166, 3, 76, 46, 77, 38, 79, 102, 82, 146, 3, 83, 82, 84, 222, 1, + 87, 242, 176, 8, 70, 236, 5, 14, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, + 82, 73, 71, 72, 86, 78, 202, 1, 80, 153, 13, 10, 86, 69, 82, 84, 73, 67, + 65, 76, 32, 66, 28, 22, 78, 143, 4, 82, 22, 28, 2, 68, 32, 191, 3, 71, + 16, 96, 6, 76, 79, 87, 69, 82, 32, 32, 6, 82, 73, 71, 72, 84, 32, 89, 6, + 85, 80, 80, 69, 82, 32, 4, 202, 1, 65, 235, 175, 17, 79, 6, 50, 84, 209, + 250, 17, 6, 68, 79, 85, 66, 76, 69, 4, 250, 136, 17, 82, 147, 231, 1, 65, + 6, 82, 65, 53, 16, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, 32, 66, 76, + 79, 67, 75, 2, 237, 134, 17, 8, 78, 68, 32, 82, 73, 71, 72, 84, 5, 165, + 253, 18, 17, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, + 67, 75, 6, 154, 182, 8, 69, 185, 1, 4, 76, 69, 32, 66, 6, 26, 67, 247, + 185, 8, 82, 2, 209, 185, 8, 5, 32, 76, 69, 83, 83, 12, 40, 4, 65, 82, 66, + 32, 143, 186, 8, 76, 8, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 4, 57, 12, + 32, 82, 73, 71, 72, 84, 32, 66, 65, 82, 66, 32, 4, 240, 244, 16, 4, 68, + 79, 87, 78, 1, 2, 85, 80, 14, 210, 185, 8, 85, 166, 26, 79, 206, 231, 2, + 69, 233, 168, 2, 8, 76, 79, 83, 69, 68, 32, 69, 78, 10, 44, 5, 85, 66, + 76, 69, 32, 255, 185, 8, 84, 8, 178, 187, 8, 87, 190, 30, 65, 138, 128, + 3, 81, 131, 190, 4, 80, 38, 36, 3, 76, 70, 32, 247, 191, 8, 78, 36, 132, + 2, 6, 67, 73, 82, 67, 76, 69, 246, 186, 8, 66, 90, 70, 82, 72, 50, 82, + 58, 84, 70, 87, 246, 13, 76, 22, 85, 220, 185, 8, 30, 73, 78, 86, 69, 82, + 83, 69, 32, 77, 69, 68, 73, 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, + 68, 32, 82, 73, 71, 72, 84, 251, 26, 77, 11, 33, 6, 32, 87, 73, 84, 72, + 32, 8, 42, 84, 206, 243, 17, 70, 235, 210, 1, 68, 4, 242, 197, 5, 72, + 179, 128, 13, 87, 4, 152, 158, 1, 2, 85, 71, 199, 159, 7, 79, 2, 161, + 133, 19, 4, 85, 76, 84, 73, 8, 36, 3, 78, 69, 32, 199, 190, 8, 85, 6, + 190, 162, 17, 81, 146, 4, 69, 45, 5, 84, 72, 73, 82, 68, 30, 44, 5, 73, + 71, 72, 84, 32, 155, 191, 8, 65, 28, 152, 1, 5, 65, 82, 82, 79, 87, 132, + 1, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 30, 87, 222, 236, + 8, 79, 154, 128, 8, 66, 54, 83, 207, 68, 84, 11, 11, 32, 8, 72, 5, 87, + 73, 84, 72, 32, 165, 192, 18, 7, 84, 72, 82, 79, 85, 71, 72, 6, 182, 174, + 16, 68, 242, 179, 3, 86, 79, 83, 7, 201, 228, 8, 2, 32, 87, 4, 210, 247, + 8, 65, 215, 176, 8, 72, 34, 174, 189, 8, 45, 70, 69, 78, 73, 168, 1, 3, + 80, 69, 69, 26, 81, 227, 2, 85, 26, 106, 82, 134, 194, 8, 72, 174, 212, + 7, 79, 208, 133, 1, 8, 87, 79, 32, 84, 72, 73, 82, 68, 219, 196, 1, 65, + 6, 40, 4, 73, 65, 78, 71, 255, 196, 8, 65, 4, 236, 236, 4, 8, 76, 69, 32, + 66, 69, 83, 73, 68, 207, 175, 12, 85, 16, 162, 199, 8, 72, 202, 1, 73, + 169, 184, 10, 2, 82, 73, 62, 128, 1, 9, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 138, 1, 83, 206, 199, 8, 70, 226, 2, 72, 153, 7, 7, 84, 79, 45, 82, + 73, 71, 72, 30, 86, 65, 214, 204, 8, 67, 94, 68, 58, 77, 58, 82, 182, 1, + 83, 74, 84, 255, 157, 8, 69, 6, 146, 205, 8, 78, 202, 160, 8, 84, 159, 2, + 73, 4, 44, 5, 73, 68, 69, 32, 65, 131, 208, 8, 72, 2, 253, 142, 1, 2, 82, + 67, 210, 1, 172, 1, 5, 65, 82, 82, 79, 87, 174, 5, 66, 70, 72, 134, 4, + 84, 178, 2, 87, 226, 207, 8, 68, 210, 1, 70, 222, 7, 76, 26, 79, 34, 80, + 50, 82, 70, 83, 230, 191, 8, 67, 47, 81, 73, 26, 32, 143, 215, 17, 45, + 68, 102, 65, 138, 1, 84, 132, 2, 5, 87, 73, 84, 72, 32, 198, 209, 8, 70, + 181, 149, 10, 4, 79, 86, 69, 82, 12, 44, 5, 66, 79, 86, 69, 32, 179, 211, + 8, 78, 10, 64, 4, 83, 72, 79, 82, 170, 210, 8, 82, 142, 226, 4, 65, 55, + 84, 2, 143, 233, 18, 84, 12, 56, 7, 72, 82, 79, 85, 71, 72, 32, 53, 3, + 79, 32, 66, 6, 142, 178, 13, 83, 202, 196, 4, 76, 247, 145, 2, 88, 6, 32, + 2, 65, 82, 187, 212, 8, 76, 5, 169, 158, 12, 23, 32, 79, 86, 69, 82, 32, + 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 36, + 118, 76, 154, 212, 8, 68, 182, 1, 80, 30, 83, 38, 84, 178, 195, 8, 77, + 38, 78, 122, 69, 226, 151, 1, 72, 143, 163, 1, 86, 4, 158, 153, 17, 65, + 219, 243, 1, 79, 8, 190, 214, 8, 65, 224, 10, 5, 79, 84, 84, 79, 77, 207, + 185, 8, 76, 30, 26, 65, 255, 159, 17, 69, 26, 48, 6, 82, 80, 79, 79, 78, + 32, 183, 190, 19, 78, 24, 100, 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, + 32, 209, 216, 8, 9, 79, 86, 69, 82, 32, 82, 73, 71, 72, 22, 40, 4, 68, + 79, 87, 78, 117, 2, 85, 80, 10, 26, 32, 219, 165, 17, 87, 8, 222, 217, 8, + 66, 188, 2, 7, 65, 66, 79, 86, 69, 32, 82, 222, 193, 8, 70, 179, 5, 84, + 12, 26, 32, 231, 164, 17, 87, 10, 60, 6, 65, 66, 79, 86, 69, 32, 198, + 156, 17, 70, 179, 5, 84, 6, 42, 76, 201, 217, 8, 4, 82, 73, 71, 72, 4, + 238, 215, 8, 69, 199, 218, 4, 79, 54, 44, 2, 82, 73, 190, 221, 8, 79, + 159, 5, 87, 34, 44, 5, 65, 78, 71, 76, 69, 243, 225, 8, 80, 30, 56, 8, + 45, 72, 69, 65, 68, 69, 68, 32, 227, 170, 17, 32, 28, 52, 5, 65, 82, 82, + 79, 87, 178, 163, 17, 68, 39, 80, 25, 11, 32, 22, 180, 222, 8, 9, 79, 86, + 69, 82, 32, 82, 73, 71, 72, 22, 87, 251, 192, 8, 84, 6, 26, 72, 131, 228, + 8, 65, 4, 45, 9, 73, 84, 69, 32, 65, 82, 82, 79, 87, 5, 149, 168, 17, 2, + 32, 87, 5, 143, 231, 18, 80, 148, 1, 252, 1, 15, 67, 79, 78, 83, 79, 78, + 65, 78, 84, 32, 83, 73, 71, 78, 32, 124, 7, 76, 69, 84, 84, 69, 82, 32, + 236, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 198, 1, 83, + 172, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 251, 132, 18, 68, + 18, 66, 75, 22, 78, 154, 251, 19, 76, 2, 77, 2, 80, 2, 82, 3, 84, 5, 179, + 170, 19, 65, 5, 233, 201, 12, 4, 89, 73, 78, 45, 78, 194, 1, 75, 2, 80, + 190, 148, 7, 68, 194, 219, 10, 66, 2, 70, 2, 71, 2, 72, 2, 77, 138, 15, + 78, 194, 178, 1, 84, 46, 67, 2, 83, 138, 69, 74, 2, 76, 2, 82, 2, 86, 2, + 87, 2, 89, 187, 2, 65, 6, 246, 246, 19, 72, 2, 76, 187, 2, 65, 10, 82, + 84, 40, 14, 78, 89, 69, 84, 32, 84, 72, 89, 79, 79, 77, 32, 84, 65, 43, + 67, 6, 38, 65, 21, 5, 83, 72, 79, 79, 75, 2, 179, 248, 6, 45, 5, 17, 2, + 32, 67, 2, 217, 134, 15, 3, 69, 82, 45, 8, 92, 4, 73, 71, 78, 32, 37, 15, + 85, 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 4, 130, 227, + 15, 78, 227, 201, 2, 82, 4, 11, 32, 4, 250, 243, 19, 82, 3, 89, 14, 238, + 162, 16, 85, 202, 141, 1, 79, 170, 195, 2, 65, 186, 2, 69, 3, 73, 52, + 138, 1, 65, 128, 4, 11, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, 200, + 1, 2, 66, 85, 42, 67, 186, 1, 79, 210, 2, 87, 231, 227, 18, 83, 16, 40, + 5, 66, 79, 86, 69, 32, 171, 4, 78, 12, 152, 1, 7, 71, 82, 69, 65, 84, 69, + 82, 110, 83, 176, 1, 19, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, + 69, 81, 85, 65, 76, 32, 65, 227, 212, 15, 76, 2, 181, 5, 23, 45, 84, 72, + 65, 78, 32, 65, 66, 79, 86, 69, 32, 68, 79, 85, 66, 76, 69, 45, 76, 73, + 78, 69, 6, 152, 1, 7, 73, 77, 73, 76, 65, 82, 32, 93, 26, 76, 65, 78, 84, + 69, 68, 32, 69, 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, 32, 71, 82, 69, + 65, 84, 69, 82, 4, 18, 65, 59, 79, 2, 25, 4, 66, 79, 86, 69, 2, 169, 222, + 17, 2, 32, 71, 2, 227, 2, 82, 2, 145, 2, 6, 45, 84, 72, 65, 78, 32, 4, + 17, 2, 68, 32, 4, 220, 3, 5, 78, 79, 84, 32, 65, 245, 151, 13, 11, 83, + 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 4, 241, 249, 12, 5, 84, 32, 78, + 79, 84, 4, 65, 14, 76, 79, 83, 69, 68, 32, 66, 89, 32, 67, 85, 82, 86, + 69, 5, 11, 32, 2, 61, 13, 65, 66, 79, 86, 69, 32, 83, 76, 65, 78, 84, 69, + 68, 2, 17, 2, 32, 69, 2, 251, 228, 14, 81, 18, 40, 2, 82, 32, 141, 220, + 11, 2, 86, 69, 16, 114, 65, 48, 16, 83, 76, 65, 78, 84, 69, 68, 32, 69, + 81, 85, 65, 76, 32, 84, 79, 218, 246, 12, 69, 223, 226, 4, 71, 2, 173, + 184, 9, 7, 80, 80, 82, 79, 88, 73, 77, 9, 49, 10, 32, 87, 73, 84, 72, 32, + 68, 79, 84, 32, 6, 26, 65, 163, 199, 11, 73, 4, 25, 4, 66, 79, 86, 69, 5, + 231, 153, 18, 32, 6, 25, 4, 73, 84, 72, 32, 6, 66, 67, 40, 8, 81, 85, 69, + 83, 84, 73, 79, 78, 143, 153, 19, 68, 2, 249, 197, 11, 5, 73, 82, 67, 76, + 69, 2, 17, 2, 32, 77, 2, 17, 2, 65, 82, 2, 151, 236, 18, 75, 136, 11, + 190, 1, 71, 186, 5, 77, 166, 7, 78, 168, 65, 2, 80, 83, 20, 3, 83, 85, + 32, 238, 199, 15, 82, 136, 35, 11, 86, 82, 69, 32, 84, 79, 85, 82, 78, + 79, 73, 210, 59, 79, 242, 219, 1, 90, 187, 82, 66, 44, 26, 65, 57, 2, 72, + 84, 2, 245, 167, 9, 9, 84, 85, 82, 69, 32, 79, 80, 69, 78, 42, 38, 32, + 137, 4, 4, 78, 73, 78, 71, 36, 146, 1, 69, 38, 70, 126, 82, 40, 3, 76, + 69, 70, 86, 83, 42, 84, 188, 198, 7, 3, 66, 76, 85, 138, 156, 9, 87, 250, + 41, 86, 250, 77, 71, 135, 69, 67, 2, 129, 209, 17, 4, 73, 71, 72, 84, 8, + 94, 73, 205, 157, 18, 17, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 32, + 66, 76, 65, 67, 75, 4, 149, 205, 17, 2, 86, 69, 4, 36, 3, 73, 71, 72, + 155, 176, 17, 65, 2, 165, 237, 1, 16, 84, 32, 84, 79, 82, 84, 79, 73, 83, + 69, 32, 83, 72, 69, 76, 76, 6, 170, 236, 16, 72, 138, 98, 65, 43, 73, 4, + 200, 153, 16, 2, 87, 69, 21, 3, 72, 82, 69, 7, 29, 5, 32, 77, 79, 79, 68, + 5, 155, 149, 8, 32, 138, 1, 80, 3, 66, 85, 32, 137, 157, 8, 11, 73, 84, + 69, 68, 32, 76, 73, 65, 66, 73, 76, 136, 1, 128, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 222, 1, 83, 200, 2, 5, 86, 79, 87, 69, 76, 194, 206, 12, 69, + 182, 188, 4, 81, 247, 95, 68, 60, 170, 1, 71, 242, 139, 6, 84, 170, 221, + 8, 89, 186, 132, 1, 78, 162, 169, 3, 83, 82, 66, 2, 67, 2, 68, 2, 74, 2, + 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 3, 87, 6, 254, 148, 18, 89, + 202, 199, 1, 72, 187, 2, 65, 32, 108, 4, 73, 71, 78, 32, 128, 1, 12, 77, + 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 181, 250, 6, 2, 85, 66, 8, + 72, 3, 75, 69, 77, 0, 3, 77, 85, 75, 32, 2, 83, 65, 207, 149, 17, 76, 2, + 129, 154, 17, 3, 80, 72, 82, 2, 155, 201, 19, 45, 18, 174, 235, 15, 78, + 186, 172, 3, 65, 194, 66, 75, 2, 76, 2, 77, 2, 80, 2, 82, 3, 84, 20, 84, + 6, 32, 83, 73, 71, 78, 32, 177, 85, 10, 45, 67, 65, 82, 82, 73, 69, 82, + 32, 76, 18, 170, 211, 5, 65, 166, 194, 11, 79, 194, 133, 2, 69, 162, 64, + 73, 3, 85, 226, 8, 22, 69, 199, 64, 75, 222, 8, 34, 32, 177, 3, 3, 65, + 82, 32, 14, 120, 12, 73, 78, 84, 69, 71, 82, 65, 84, 73, 79, 78, 32, 186, + 156, 13, 70, 150, 143, 3, 83, 245, 147, 1, 4, 84, 65, 66, 85, 6, 108, 5, + 87, 73, 84, 72, 32, 157, 1, 17, 78, 79, 84, 32, 73, 78, 67, 76, 85, 68, + 73, 78, 71, 32, 84, 72, 69, 4, 76, 7, 82, 69, 67, 84, 65, 78, 71, 1, 8, + 83, 69, 77, 73, 67, 73, 82, 67, 2, 73, 16, 85, 76, 65, 82, 32, 80, 65, + 84, 72, 32, 65, 82, 79, 85, 78, 68, 2, 17, 2, 32, 80, 2, 223, 178, 18, + 79, 208, 8, 60, 8, 65, 32, 83, 73, 71, 78, 32, 65, 221, 24, 2, 66, 32, + 170, 5, 122, 49, 86, 51, 202, 2, 52, 214, 1, 53, 158, 4, 54, 142, 3, 55, + 148, 4, 2, 56, 48, 78, 66, 213, 136, 18, 3, 48, 50, 56, 6, 128, 188, 12, + 5, 48, 48, 45, 49, 48, 200, 221, 5, 2, 50, 48, 233, 19, 2, 51, 49, 150, + 1, 78, 48, 90, 49, 130, 1, 55, 246, 156, 14, 50, 2, 51, 2, 52, 2, 53, 3, + 54, 22, 178, 1, 57, 214, 210, 19, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 2, 55, 3, 56, 24, 90, 51, 214, 210, 19, 48, 2, 49, 2, 50, 2, 52, 2, 53, + 2, 54, 2, 55, 2, 56, 3, 57, 6, 210, 210, 19, 65, 2, 66, 3, 67, 4, 174, + 210, 19, 48, 3, 49, 38, 18, 48, 91, 49, 20, 162, 1, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 74, 48, 2, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 2, 205, 178, 6, 2, 45, 86, 160, + 1, 102, 48, 78, 49, 62, 51, 86, 52, 70, 53, 86, 54, 206, 11, 50, 150, + 166, 2, 57, 198, 229, 11, 55, 3, 56, 16, 210, 207, 19, 49, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 12, 134, 207, 19, 48, 2, 49, 2, 50, 2, + 51, 2, 53, 3, 54, 18, 202, 206, 19, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 14, 246, 205, 19, 48, 2, 49, 2, 50, 2, 53, 2, + 55, 2, 56, 3, 57, 18, 178, 205, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 2, 55, 3, 57, 12, 222, 204, 19, 51, 2, 52, 2, 53, 2, 54, 2, + 56, 3, 57, 104, 70, 48, 78, 50, 86, 51, 38, 52, 78, 54, 162, 146, 14, 53, + 243, 1, 49, 16, 218, 203, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, + 56, 3, 57, 18, 142, 203, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, + 55, 2, 56, 3, 57, 6, 186, 202, 19, 52, 2, 55, 3, 56, 16, 150, 202, 19, + 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 10, 202, 201, 19, + 48, 2, 49, 2, 50, 2, 51, 3, 52, 44, 86, 48, 134, 2, 49, 196, 252, 1, 2, + 51, 50, 177, 184, 17, 6, 50, 54, 32, 69, 89, 89, 26, 110, 57, 142, 219, + 8, 55, 182, 6, 50, 62, 54, 230, 62, 52, 226, 151, 3, 51, 110, 56, 134, + 206, 2, 49, 147, 117, 53, 10, 26, 45, 131, 190, 18, 32, 8, 96, 2, 50, 32, + 200, 169, 12, 3, 54, 32, 76, 184, 2, 3, 52, 32, 76, 201, 164, 3, 3, 51, + 32, 76, 2, 163, 173, 12, 76, 14, 110, 49, 22, 51, 32, 3, 52, 32, 65, 0, + 2, 53, 32, 238, 154, 4, 50, 238, 199, 4, 48, 241, 235, 6, 2, 55, 32, 2, + 167, 185, 18, 32, 2, 11, 32, 2, 175, 142, 12, 79, 2, 179, 137, 18, 66, + 16, 130, 197, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 162, 1, 22, 48, 155, 5, 49, 140, 1, 82, 49, 54, 50, 118, 51, 62, 52, 78, - 53, 86, 54, 62, 55, 70, 56, 243, 220, 13, 48, 10, 246, 236, 18, 48, 2, - 49, 2, 51, 2, 54, 3, 55, 28, 86, 49, 2, 50, 194, 70, 51, 174, 165, 18, - 48, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 7, 234, 235, 18, 70, 3, 77, 12, - 206, 235, 18, 48, 2, 49, 2, 52, 2, 55, 2, 56, 3, 57, 16, 146, 235, 18, - 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 198, 234, 18, - 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 242, 233, - 18, 48, 2, 49, 2, 53, 2, 54, 2, 55, 3, 57, 14, 182, 233, 18, 48, 2, 51, - 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 12, 242, 232, 18, 48, 2, 49, 2, 50, 2, - 53, 2, 54, 3, 55, 22, 82, 50, 36, 2, 51, 49, 30, 56, 226, 255, 11, 49, - 206, 2, 54, 146, 1, 55, 3, 57, 6, 226, 231, 18, 48, 2, 50, 3, 51, 4, 190, - 231, 18, 65, 3, 66, 4, 162, 231, 18, 48, 3, 56, 166, 3, 116, 9, 73, 68, + 53, 86, 54, 62, 55, 70, 56, 151, 136, 14, 48, 10, 194, 195, 19, 48, 2, + 49, 2, 51, 2, 54, 3, 55, 28, 86, 49, 2, 50, 146, 77, 51, 170, 245, 18, + 48, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 7, 182, 194, 19, 70, 3, 77, 12, + 154, 194, 19, 48, 2, 49, 2, 52, 2, 55, 2, 56, 3, 57, 16, 222, 193, 19, + 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 146, 193, 19, + 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 190, 192, + 19, 48, 2, 49, 2, 53, 2, 54, 2, 55, 3, 57, 14, 130, 192, 19, 48, 2, 51, + 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 12, 190, 191, 19, 48, 2, 49, 2, 50, 2, + 53, 2, 54, 3, 55, 22, 82, 50, 36, 2, 51, 49, 30, 56, 214, 160, 12, 49, + 206, 2, 54, 146, 1, 55, 3, 57, 6, 174, 190, 19, 48, 2, 50, 3, 51, 4, 138, + 190, 19, 65, 3, 66, 4, 238, 189, 19, 48, 3, 56, 166, 3, 116, 9, 73, 68, 69, 79, 71, 82, 65, 77, 32, 216, 17, 10, 77, 79, 78, 79, 71, 82, 65, 77, 32, 66, 249, 1, 2, 83, 89, 234, 1, 54, 66, 249, 15, 8, 86, 69, 83, 83, 69, 76, 32, 66, 176, 1, 22, 49, 255, 10, 50, 126, 86, 48, 210, 3, 50, 162, 1, 51, 86, 52, 122, 53, 114, 54, 146, 1, 55, 118, 56, 71, 57, 28, - 114, 53, 98, 54, 58, 55, 78, 56, 62, 57, 148, 247, 6, 3, 50, 32, 87, 234, - 84, 48, 237, 215, 10, 4, 52, 32, 68, 69, 6, 252, 245, 6, 2, 32, 69, 188, - 205, 10, 3, 70, 32, 77, 245, 79, 7, 77, 32, 83, 84, 65, 76, 76, 4, 224, - 233, 1, 3, 70, 32, 69, 221, 191, 15, 2, 77, 32, 4, 36, 3, 70, 32, 83, 1, - 2, 77, 32, 2, 213, 195, 11, 4, 72, 69, 45, 71, 4, 228, 250, 8, 3, 77, 32, - 66, 145, 205, 8, 3, 70, 32, 83, 4, 148, 221, 16, 4, 77, 32, 66, 85, 165, - 106, 3, 70, 32, 67, 10, 236, 243, 8, 4, 51, 32, 83, 80, 224, 10, 5, 49, - 32, 66, 65, 82, 160, 218, 4, 4, 50, 32, 79, 76, 20, 6, 53, 32, 67, 89, - 80, 69, 209, 99, 4, 48, 32, 87, 72, 6, 54, 49, 192, 178, 16, 3, 48, 32, - 79, 247, 172, 2, 50, 2, 145, 198, 17, 2, 32, 87, 10, 224, 5, 3, 53, 32, - 87, 244, 241, 8, 6, 48, 32, 66, 82, 79, 78, 172, 2, 4, 49, 32, 71, 79, - 150, 229, 9, 50, 3, 54, 16, 82, 57, 150, 226, 2, 49, 186, 251, 15, 48, 2, - 50, 2, 51, 2, 52, 2, 55, 3, 56, 2, 253, 90, 3, 32, 67, 76, 20, 184, 246, - 13, 5, 51, 32, 65, 82, 77, 204, 163, 4, 5, 50, 32, 71, 65, 82, 170, 67, - 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 202, 198, 13, - 54, 196, 150, 3, 4, 51, 32, 77, 79, 146, 255, 1, 48, 2, 49, 2, 50, 2, 52, - 2, 55, 2, 56, 3, 57, 14, 170, 219, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 53, 3, 57, 4, 240, 157, 2, 3, 49, 32, 72, 247, 188, 16, 48, 50, 42, 50, - 110, 51, 130, 1, 52, 227, 1, 53, 4, 84, 8, 48, 32, 70, 79, 79, 84, 83, - 84, 181, 196, 6, 7, 53, 32, 66, 65, 84, 72, 84, 2, 219, 198, 17, 79, 12, - 104, 4, 51, 32, 83, 87, 248, 211, 13, 4, 48, 32, 83, 80, 130, 234, 3, 49, - 190, 154, 1, 50, 2, 52, 3, 54, 2, 223, 193, 17, 79, 16, 168, 1, 9, 48, - 32, 87, 72, 69, 69, 76, 69, 68, 2, 49, 34, 51, 240, 202, 16, 12, 50, 32, - 67, 72, 65, 82, 73, 79, 84, 32, 70, 82, 230, 139, 2, 53, 2, 54, 2, 56, 3, - 57, 2, 133, 205, 17, 3, 32, 67, 72, 2, 183, 176, 6, 32, 18, 184, 207, 17, - 3, 52, 32, 68, 134, 135, 1, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, - 3, 57, 58, 50, 50, 204, 239, 11, 2, 49, 53, 1, 2, 51, 48, 54, 50, 50, - 250, 241, 11, 53, 230, 216, 1, 48, 3, 49, 12, 238, 212, 18, 49, 2, 50, 2, - 54, 2, 55, 2, 56, 3, 57, 12, 46, 49, 153, 12, 6, 50, 52, 55, 32, 68, 73, - 10, 50, 50, 70, 51, 181, 11, 5, 53, 54, 32, 84, 85, 4, 252, 167, 3, 3, - 55, 32, 75, 197, 140, 15, 5, 56, 32, 75, 65, 78, 4, 56, 3, 53, 32, 77, - 241, 173, 15, 5, 51, 32, 65, 82, 69, 2, 231, 201, 11, 69, 176, 1, 84, 9, - 76, 76, 65, 66, 76, 69, 32, 66, 48, 229, 12, 7, 77, 66, 79, 76, 32, 66, - 48, 148, 1, 114, 48, 142, 1, 49, 162, 1, 50, 230, 1, 51, 174, 1, 52, 162, - 1, 53, 154, 1, 54, 186, 1, 55, 198, 1, 56, 87, 57, 18, 118, 54, 250, 191, - 1, 55, 150, 8, 52, 162, 14, 57, 138, 211, 10, 53, 186, 146, 2, 56, 226, - 10, 49, 254, 2, 50, 207, 8, 51, 2, 171, 149, 17, 32, 16, 122, 54, 18, 55, - 158, 201, 1, 49, 142, 2, 50, 158, 26, 52, 196, 186, 1, 2, 53, 32, 198, - 220, 5, 48, 177, 186, 8, 2, 51, 32, 2, 135, 84, 32, 2, 159, 232, 10, 32, - 18, 166, 1, 51, 22, 54, 20, 3, 57, 32, 80, 224, 6, 2, 53, 32, 136, 168, - 3, 2, 48, 32, 228, 253, 13, 2, 55, 32, 140, 7, 2, 52, 32, 146, 89, 56, - 173, 44, 3, 49, 32, 81, 2, 151, 250, 13, 32, 2, 151, 192, 14, 32, 2, 151, - 233, 11, 85, 16, 134, 1, 48, 20, 2, 51, 32, 254, 188, 1, 55, 254, 34, 54, - 194, 233, 4, 57, 138, 148, 2, 56, 210, 154, 4, 49, 185, 180, 5, 3, 50, - 32, 81, 2, 215, 251, 13, 32, 2, 135, 1, 82, 16, 116, 2, 51, 32, 22, 53, - 242, 184, 1, 48, 222, 1, 49, 134, 6, 50, 222, 10, 52, 146, 5, 54, 233, - 180, 12, 3, 56, 32, 78, 2, 199, 246, 14, 65, 2, 171, 189, 16, 32, 18, - 130, 1, 51, 162, 185, 1, 49, 150, 1, 56, 42, 57, 102, 55, 214, 5, 48, - 158, 140, 2, 52, 200, 210, 14, 2, 50, 32, 157, 4, 2, 53, 32, 2, 159, 192, - 11, 32, 16, 132, 1, 2, 50, 32, 20, 2, 56, 32, 204, 1, 3, 54, 32, 84, 150, - 180, 1, 55, 226, 3, 57, 146, 2, 53, 158, 228, 6, 49, 163, 221, 9, 48, 2, - 191, 131, 17, 80, 2, 129, 228, 11, 2, 82, 79, 18, 172, 1, 3, 54, 32, 82, - 182, 180, 1, 55, 158, 8, 48, 142, 16, 53, 12, 3, 49, 32, 68, 228, 145, 7, - 2, 52, 32, 214, 235, 3, 50, 152, 229, 5, 3, 56, 32, 81, 241, 2, 2, 51, - 32, 2, 183, 226, 11, 65, 8, 178, 180, 1, 49, 160, 24, 3, 55, 32, 84, 172, - 230, 6, 2, 53, 32, 211, 129, 6, 48, 4, 158, 184, 12, 49, 21, 3, 48, 32, - 68, 28, 90, 52, 30, 54, 30, 56, 186, 220, 11, 53, 158, 2, 49, 30, 51, - 166, 1, 50, 175, 144, 3, 55, 4, 158, 196, 18, 55, 3, 57, 4, 130, 196, 18, - 51, 3, 52, 8, 230, 195, 18, 50, 2, 51, 2, 54, 3, 57, 4, 156, 173, 10, 9, - 69, 68, 32, 80, 65, 80, 69, 82, 67, 155, 131, 7, 32, 5, 155, 148, 17, 84, - 98, 96, 7, 76, 69, 84, 84, 69, 82, 32, 253, 219, 6, 11, 80, 85, 78, 67, - 84, 85, 65, 84, 73, 79, 78, 94, 234, 1, 84, 186, 234, 2, 68, 226, 249, 9, - 85, 222, 140, 2, 69, 22, 78, 214, 135, 3, 67, 2, 71, 2, 72, 2, 75, 2, 80, - 2, 83, 2, 89, 2, 90, 162, 7, 65, 2, 79, 222, 61, 66, 2, 70, 2, 74, 2, 76, - 2, 77, 2, 87, 2, 88, 187, 2, 73, 20, 64, 4, 79, 78, 69, 32, 154, 248, 17, - 83, 254, 68, 72, 187, 2, 65, 12, 48, 4, 77, 89, 65, 32, 229, 185, 1, 2, - 78, 65, 10, 174, 227, 3, 74, 246, 188, 14, 66, 158, 11, 84, 254, 16, 67, - 39, 78, 208, 1, 244, 1, 2, 67, 75, 132, 1, 6, 71, 73, 67, 65, 76, 32, - 236, 3, 3, 78, 71, 32, 230, 4, 84, 116, 3, 86, 69, 32, 70, 87, 144, 8, 5, - 90, 69, 78, 71, 69, 218, 212, 7, 66, 220, 204, 9, 8, 85, 68, 76, 89, 32, - 67, 82, 89, 133, 15, 4, 76, 76, 73, 80, 9, 96, 10, 73, 78, 71, 45, 83, - 72, 73, 70, 84, 32, 149, 16, 9, 32, 87, 73, 84, 72, 32, 73, 78, 75, 4, - 210, 207, 16, 90, 143, 83, 79, 28, 36, 3, 65, 78, 68, 85, 2, 79, 82, 15, - 33, 6, 32, 87, 73, 84, 72, 32, 12, 226, 1, 68, 94, 72, 58, 77, 187, 239, - 14, 85, 15, 11, 32, 12, 88, 13, 79, 86, 69, 82, 76, 65, 80, 80, 73, 78, - 71, 32, 76, 49, 5, 87, 73, 84, 72, 32, 2, 129, 172, 17, 7, 79, 71, 73, - 67, 65, 76, 32, 10, 26, 68, 94, 72, 59, 77, 6, 11, 79, 6, 44, 5, 85, 66, - 76, 69, 32, 247, 187, 17, 84, 4, 142, 240, 14, 85, 167, 47, 79, 2, 149, - 144, 15, 9, 79, 82, 73, 90, 79, 78, 84, 65, 76, 2, 213, 196, 7, 5, 73, - 68, 68, 76, 69, 34, 66, 68, 236, 1, 4, 76, 69, 70, 84, 109, 5, 82, 73, - 71, 72, 84, 6, 172, 1, 30, 65, 83, 72, 32, 70, 82, 79, 77, 32, 76, 69, - 70, 84, 32, 77, 69, 77, 66, 69, 82, 32, 79, 70, 32, 68, 79, 85, 66, 76, - 69, 196, 180, 13, 2, 73, 86, 191, 217, 3, 82, 2, 17, 2, 32, 86, 2, 181, - 172, 17, 5, 69, 82, 84, 73, 67, 16, 18, 32, 119, 87, 6, 48, 6, 82, 73, - 71, 72, 84, 32, 215, 170, 15, 84, 4, 130, 165, 15, 68, 203, 245, 1, 65, - 12, 26, 87, 215, 216, 8, 32, 10, 29, 5, 65, 82, 68, 83, 32, 10, 82, 65, - 0, 8, 68, 79, 85, 66, 76, 69, 32, 65, 153, 205, 7, 4, 83, 81, 85, 73, 4, - 25, 4, 82, 82, 79, 87, 5, 233, 211, 15, 2, 32, 70, 6, 92, 5, 73, 79, 78, - 32, 66, 144, 153, 17, 9, 32, 79, 70, 32, 70, 79, 82, 84, 85, 199, 91, 85, - 2, 131, 209, 9, 79, 4, 38, 76, 145, 209, 13, 3, 72, 79, 84, 2, 153, 251, - 16, 2, 69, 84, 118, 34, 32, 229, 1, 3, 69, 82, 32, 14, 138, 1, 66, 140, - 169, 10, 11, 68, 79, 85, 66, 76, 69, 32, 80, 82, 73, 77, 214, 246, 5, 65, - 148, 109, 6, 75, 65, 86, 89, 75, 65, 139, 10, 76, 4, 38, 82, 149, 239, 4, - 3, 65, 84, 84, 2, 213, 156, 17, 7, 73, 71, 72, 84, 78, 69, 83, 104, 152, - 1, 5, 72, 65, 76, 70, 32, 92, 5, 76, 69, 70, 84, 32, 220, 2, 6, 82, 73, - 71, 72, 84, 32, 210, 164, 15, 66, 74, 70, 154, 9, 79, 150, 13, 83, 63, - 84, 8, 208, 137, 1, 7, 73, 78, 86, 69, 82, 83, 69, 238, 180, 14, 77, 162, - 7, 66, 195, 153, 1, 67, 40, 174, 1, 66, 60, 8, 70, 79, 85, 78, 84, 65, - 73, 78, 22, 80, 52, 12, 83, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, 82, - 202, 1, 84, 232, 166, 11, 3, 67, 82, 65, 147, 144, 4, 81, 24, 56, 8, 65, - 76, 76, 80, 79, 73, 78, 84, 227, 176, 15, 76, 2, 183, 215, 9, 32, 4, 176, - 203, 14, 5, 65, 73, 78, 84, 66, 171, 109, 69, 2, 177, 173, 7, 5, 32, 65, - 78, 84, 73, 40, 78, 83, 82, 84, 158, 168, 15, 66, 238, 3, 67, 130, 9, 68, - 206, 1, 80, 39, 81, 6, 172, 172, 7, 11, 69, 77, 73, 67, 73, 82, 67, 85, - 76, 65, 82, 131, 140, 8, 72, 4, 139, 185, 15, 82, 5, 149, 136, 17, 25, - 32, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, 72, 79, 82, 73, 90, 79, - 78, 84, 65, 76, 32, 82, 85, 6, 18, 71, 23, 78, 2, 231, 205, 13, 71, 4, - 140, 134, 17, 5, 65, 82, 32, 69, 67, 191, 100, 71, 114, 104, 12, 67, 73, - 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 228, 1, 5, 68, 73, 65, 78, 32, - 159, 157, 17, 73, 58, 206, 1, 77, 194, 130, 5, 75, 142, 175, 9, 66, 50, - 84, 178, 251, 1, 65, 2, 69, 2, 78, 130, 248, 1, 68, 2, 71, 2, 72, 2, 73, - 2, 74, 2, 76, 2, 80, 2, 81, 2, 82, 2, 83, 2, 85, 2, 87, 2, 88, 3, 90, 5, - 171, 165, 18, 77, 54, 88, 7, 76, 69, 84, 84, 69, 82, 32, 253, 177, 14, 9, - 84, 82, 73, 65, 78, 71, 85, 76, 65, 52, 198, 239, 12, 84, 254, 193, 1, - 76, 154, 239, 1, 83, 226, 11, 65, 2, 69, 2, 78, 130, 248, 1, 66, 2, 67, - 2, 68, 2, 70, 2, 71, 2, 73, 2, 75, 2, 77, 2, 79, 2, 81, 2, 82, 2, 85, 2, - 86, 3, 89, 194, 53, 162, 1, 65, 242, 103, 69, 138, 103, 73, 254, 24, 79, - 136, 112, 3, 82, 79, 32, 234, 3, 85, 164, 70, 7, 89, 65, 78, 77, 65, 82, - 32, 246, 193, 14, 86, 186, 61, 77, 27, 87, 204, 23, 186, 1, 71, 62, 72, - 130, 11, 75, 234, 3, 76, 156, 14, 2, 77, 77, 22, 78, 150, 17, 80, 118, - 82, 174, 7, 83, 194, 7, 84, 196, 36, 3, 89, 65, 78, 132, 129, 1, 3, 88, - 73, 77, 243, 151, 15, 67, 6, 156, 146, 17, 4, 73, 67, 32, 87, 226, 88, - 78, 143, 53, 69, 166, 1, 84, 6, 65, 74, 65, 78, 73, 32, 133, 3, 10, 74, - 79, 78, 71, 32, 84, 73, 76, 69, 32, 78, 38, 76, 166, 2, 83, 167, 220, 12, - 65, 72, 88, 6, 69, 84, 84, 69, 82, 32, 137, 149, 11, 10, 73, 71, 65, 84, - 85, 82, 69, 32, 83, 72, 70, 202, 173, 3, 78, 138, 208, 10, 82, 242, 55, - 68, 46, 84, 206, 160, 3, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, - 72, 2, 76, 2, 77, 2, 83, 2, 86, 186, 2, 65, 2, 69, 2, 73, 2, 79, 3, 85, - 4, 174, 227, 6, 69, 177, 211, 7, 5, 73, 71, 78, 32, 78, 88, 236, 1, 2, - 66, 65, 38, 69, 46, 70, 46, 78, 42, 79, 46, 80, 22, 83, 118, 84, 194, 1, - 87, 112, 5, 71, 82, 69, 69, 78, 0, 3, 82, 69, 68, 168, 128, 6, 3, 65, 85, - 84, 210, 21, 74, 141, 215, 10, 11, 67, 72, 82, 89, 83, 65, 78, 84, 72, - 69, 77, 4, 254, 173, 1, 77, 235, 232, 16, 67, 8, 228, 2, 4, 73, 71, 72, - 84, 195, 1, 65, 12, 172, 2, 2, 73, 86, 13, 3, 79, 85, 82, 8, 192, 1, 2, - 79, 82, 65, 2, 73, 78, 8, 218, 1, 78, 237, 197, 11, 3, 82, 67, 72, 2, - 187, 240, 16, 76, 18, 88, 2, 79, 85, 76, 4, 69, 86, 69, 78, 0, 2, 73, 88, - 162, 244, 11, 85, 235, 193, 1, 80, 2, 157, 2, 2, 84, 72, 12, 36, 3, 72, - 82, 69, 13, 2, 87, 79, 6, 11, 69, 6, 25, 4, 32, 79, 70, 32, 6, 46, 67, - 173, 238, 6, 5, 66, 65, 77, 66, 79, 4, 180, 132, 6, 2, 73, 82, 145, 148, - 10, 5, 72, 65, 82, 65, 67, 6, 50, 69, 60, 4, 72, 73, 84, 69, 163, 222, - 16, 73, 2, 17, 2, 83, 84, 2, 17, 2, 32, 87, 2, 171, 135, 17, 73, 2, 17, - 2, 32, 68, 2, 227, 221, 16, 82, 52, 52, 5, 65, 83, 65, 82, 32, 157, 206, - 14, 2, 69, 77, 50, 162, 1, 69, 40, 7, 76, 69, 84, 84, 69, 82, 32, 152, 1, - 5, 80, 65, 83, 83, 73, 32, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, - 32, 173, 136, 18, 3, 65, 78, 71, 2, 213, 159, 15, 5, 78, 68, 32, 79, 70, - 36, 162, 155, 16, 78, 242, 244, 1, 66, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, - 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, 2, 65, 2, - 11, 77, 2, 227, 184, 17, 66, 8, 146, 145, 18, 69, 2, 73, 2, 79, 3, 85, - 246, 1, 80, 7, 65, 89, 65, 76, 65, 77, 32, 208, 11, 2, 69, 32, 225, 1, 3, - 84, 69, 83, 236, 1, 190, 1, 65, 22, 68, 44, 9, 70, 82, 65, 67, 84, 73, - 79, 78, 32, 240, 1, 7, 76, 69, 84, 84, 69, 82, 32, 164, 5, 7, 78, 85, 77, - 66, 69, 82, 32, 36, 5, 83, 73, 71, 78, 32, 143, 144, 13, 86, 2, 227, 142, - 13, 85, 22, 172, 146, 8, 2, 65, 84, 251, 141, 8, 73, 26, 56, 4, 79, 78, - 69, 32, 121, 6, 84, 72, 82, 69, 69, 32, 18, 82, 84, 142, 128, 5, 83, 202, - 219, 7, 70, 58, 79, 170, 176, 3, 69, 46, 72, 43, 81, 4, 254, 222, 12, 87, - 239, 174, 3, 69, 8, 166, 128, 5, 83, 174, 223, 7, 69, 110, 84, 163, 174, - 3, 81, 132, 1, 226, 1, 65, 82, 67, 158, 1, 68, 78, 84, 82, 86, 222, 221, - 12, 78, 230, 46, 76, 210, 90, 82, 130, 81, 85, 158, 144, 1, 79, 182, 56, - 73, 202, 190, 1, 83, 82, 66, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 222, - 61, 72, 2, 77, 3, 89, 11, 148, 206, 15, 7, 82, 67, 72, 65, 73, 67, 32, - 158, 188, 2, 65, 2, 73, 3, 85, 22, 26, 72, 203, 137, 18, 65, 20, 44, 5, - 73, 76, 76, 85, 32, 155, 137, 18, 65, 18, 226, 211, 12, 76, 182, 189, 3, - 78, 178, 191, 1, 82, 210, 56, 75, 2, 77, 3, 89, 10, 164, 198, 10, 4, 79, - 84, 32, 82, 238, 250, 6, 68, 254, 68, 72, 187, 2, 65, 10, 38, 84, 158, - 133, 18, 72, 187, 2, 65, 6, 154, 133, 18, 72, 2, 84, 187, 2, 65, 12, 250, - 225, 3, 69, 206, 193, 10, 79, 231, 227, 3, 65, 6, 158, 222, 12, 79, 231, - 215, 3, 84, 18, 50, 67, 114, 86, 194, 158, 14, 65, 179, 162, 3, 80, 6, - 54, 79, 120, 5, 73, 82, 67, 85, 76, 215, 158, 14, 65, 2, 185, 142, 13, 9, - 77, 66, 73, 78, 73, 78, 71, 32, 65, 6, 60, 9, 69, 82, 84, 73, 67, 65, 76, - 32, 66, 255, 193, 17, 73, 2, 17, 2, 65, 82, 2, 233, 163, 12, 2, 32, 86, - 8, 80, 12, 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 69, 32, 70, 65, 219, - 252, 16, 83, 4, 64, 10, 65, 78, 68, 32, 77, 65, 76, 69, 32, 65, 219, 252, - 16, 83, 2, 25, 4, 78, 68, 32, 70, 2, 11, 69, 2, 11, 77, 2, 199, 190, 7, - 65, 2, 231, 196, 16, 69, 2, 199, 131, 16, 79, 185, 1, 210, 1, 32, 220, 2, - 5, 68, 65, 73, 67, 32, 220, 3, 8, 73, 67, 72, 65, 69, 65, 78, 32, 222, 8, - 83, 200, 165, 2, 3, 85, 65, 76, 166, 181, 10, 65, 220, 168, 1, 8, 84, 69, - 76, 80, 73, 69, 67, 69, 147, 207, 3, 71, 12, 132, 1, 3, 73, 78, 32, 128, - 1, 5, 87, 73, 84, 72, 32, 196, 146, 2, 3, 68, 65, 78, 169, 194, 12, 8, - 65, 78, 68, 32, 87, 79, 77, 65, 4, 208, 240, 9, 19, 66, 85, 83, 73, 78, - 69, 83, 83, 32, 83, 85, 73, 84, 32, 76, 69, 86, 73, 84, 209, 148, 1, 4, - 84, 85, 88, 69, 4, 204, 195, 13, 8, 71, 85, 65, 32, 80, 73, 32, 77, 213, - 243, 2, 4, 84, 85, 82, 66, 58, 136, 1, 7, 76, 69, 84, 84, 69, 82, 32, - 230, 209, 10, 71, 176, 184, 3, 3, 86, 79, 67, 146, 70, 80, 177, 195, 2, - 6, 65, 70, 70, 82, 73, 67, 50, 82, 65, 176, 1, 2, 68, 85, 2, 85, 28, 3, - 72, 65, 76, 22, 73, 167, 180, 17, 75, 38, 154, 1, 75, 150, 198, 12, 84, - 226, 193, 1, 83, 194, 163, 3, 73, 214, 79, 66, 2, 68, 2, 71, 2, 72, 2, - 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 82, 3, 90, 5, 207, 248, 17, 83, 2, - 253, 239, 7, 2, 83, 72, 2, 159, 248, 17, 81, 4, 194, 250, 17, 78, 3, 84, - 102, 216, 1, 7, 76, 69, 84, 84, 69, 82, 32, 194, 4, 78, 80, 12, 80, 85, - 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 220, 1, 4, 83, 73, 71, 78, 185, - 172, 10, 16, 65, 66, 66, 82, 69, 86, 73, 65, 84, 73, 79, 78, 32, 77, 65, - 82, 72, 218, 1, 65, 46, 66, 38, 68, 30, 71, 30, 74, 2, 90, 30, 75, 30, - 81, 38, 83, 50, 84, 54, 88, 238, 154, 6, 76, 254, 194, 1, 82, 238, 213, - 2, 89, 154, 193, 1, 72, 166, 255, 4, 78, 254, 1, 87, 202, 103, 70, 2, 80, - 171, 4, 77, 6, 206, 180, 10, 76, 122, 65, 239, 193, 6, 89, 4, 238, 200, - 12, 72, 211, 174, 3, 69, 4, 206, 156, 6, 65, 23, 72, 4, 246, 179, 10, 72, - 15, 73, 4, 178, 180, 10, 72, 15, 65, 4, 246, 251, 11, 72, 15, 65, 4, 162, - 156, 6, 72, 199, 223, 5, 79, 8, 250, 176, 10, 83, 154, 3, 65, 191, 193, - 6, 72, 6, 158, 155, 6, 72, 186, 218, 9, 69, 247, 128, 1, 65, 4, 222, 250, - 11, 65, 3, 79, 10, 33, 6, 85, 77, 66, 69, 82, 32, 10, 186, 179, 10, 79, - 58, 84, 219, 183, 2, 70, 14, 80, 2, 68, 79, 116, 3, 76, 73, 78, 138, 159, - 5, 70, 142, 133, 11, 84, 255, 9, 83, 6, 54, 84, 13, 9, 85, 66, 76, 69, - 32, 68, 79, 84, 32, 5, 11, 32, 2, 25, 4, 87, 73, 84, 72, 2, 163, 192, 2, - 73, 2, 223, 201, 3, 69, 2, 159, 199, 16, 32, 2, 11, 32, 2, 221, 218, 17, - 2, 83, 72, 4, 92, 17, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 76, - 73, 71, 72, 84, 175, 191, 10, 76, 2, 175, 193, 14, 72, 142, 1, 142, 1, - 65, 20, 5, 67, 72, 69, 78, 32, 160, 155, 6, 4, 82, 73, 65, 71, 165, 213, - 10, 13, 84, 73, 65, 76, 32, 65, 82, 84, 83, 32, 85, 78, 73, 2, 175, 153, - 5, 67, 136, 1, 152, 1, 7, 76, 69, 84, 84, 69, 82, 32, 194, 1, 83, 236, 2, - 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 154, 157, 17, 72, 225, 5, - 4, 77, 65, 82, 75, 60, 254, 3, 84, 146, 148, 2, 68, 178, 222, 13, 78, - 246, 175, 1, 67, 2, 75, 2, 80, 2, 83, 2, 90, 254, 68, 45, 2, 66, 2, 71, - 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, 187, 2, 65, 62, 96, 4, - 73, 71, 78, 32, 37, 16, 85, 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, - 84, 69, 82, 32, 4, 154, 133, 14, 67, 191, 161, 3, 65, 58, 182, 1, 84, - 146, 148, 2, 68, 178, 222, 13, 78, 246, 175, 1, 67, 2, 75, 2, 80, 2, 83, - 2, 90, 254, 68, 66, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, - 89, 187, 2, 65, 8, 178, 162, 17, 83, 254, 68, 72, 187, 2, 65, 10, 130, - 231, 17, 65, 186, 2, 69, 2, 73, 2, 79, 3, 85, 156, 1, 120, 11, 65, 82, - 65, 77, 32, 71, 79, 78, 68, 73, 32, 236, 5, 3, 67, 85, 76, 64, 5, 75, 32, - 87, 79, 82, 163, 160, 17, 85, 150, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, - 178, 2, 82, 56, 5, 83, 73, 71, 78, 32, 82, 86, 199, 245, 15, 68, 94, 210, - 1, 74, 42, 84, 206, 252, 13, 65, 38, 68, 146, 25, 85, 210, 200, 1, 73, - 42, 76, 210, 189, 1, 75, 38, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 80, - 254, 68, 72, 2, 77, 2, 82, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 230, 226, - 17, 78, 38, 72, 187, 2, 65, 10, 230, 157, 17, 84, 254, 68, 72, 2, 82, - 187, 2, 65, 4, 32, 2, 65, 45, 163, 240, 1, 69, 2, 131, 160, 17, 75, 10, - 174, 176, 1, 67, 202, 160, 12, 72, 242, 45, 78, 130, 182, 1, 86, 223, - 234, 1, 65, 22, 26, 79, 207, 177, 15, 73, 20, 45, 9, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 20, 78, 86, 238, 253, 13, 65, 190, 21, 85, 210, 200, 1, - 73, 206, 134, 2, 69, 3, 79, 2, 201, 182, 7, 6, 79, 67, 65, 76, 73, 67, 2, - 253, 168, 17, 11, 73, 78, 69, 32, 79, 82, 68, 73, 78, 65, 76, 2, 199, - 206, 16, 75, 226, 15, 76, 10, 72, 69, 77, 65, 84, 73, 67, 65, 76, 32, - 129, 162, 14, 3, 69, 32, 68, 224, 15, 252, 1, 5, 66, 79, 76, 68, 32, 168, - 6, 14, 68, 79, 85, 66, 76, 69, 45, 83, 84, 82, 85, 67, 75, 32, 238, 1, - 70, 164, 2, 7, 73, 84, 65, 76, 73, 67, 32, 180, 3, 10, 77, 79, 78, 79, - 83, 80, 65, 67, 69, 32, 40, 2, 82, 73, 36, 3, 76, 69, 70, 167, 1, 83, - 160, 5, 176, 1, 8, 67, 65, 80, 73, 84, 65, 76, 32, 130, 2, 83, 210, 14, - 73, 222, 3, 69, 38, 75, 38, 78, 30, 80, 98, 82, 242, 5, 84, 56, 7, 70, - 82, 65, 75, 84, 85, 82, 131, 211, 15, 68, 104, 190, 4, 68, 174, 15, 84, - 186, 4, 65, 22, 66, 2, 90, 42, 69, 98, 71, 22, 73, 22, 75, 34, 76, 22, - 79, 50, 80, 42, 82, 22, 83, 70, 85, 250, 196, 1, 67, 214, 173, 12, 77, 2, - 78, 210, 200, 1, 88, 206, 134, 2, 70, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, - 3, 89, 208, 1, 60, 5, 77, 65, 76, 76, 32, 213, 25, 5, 67, 82, 73, 80, 84, - 104, 250, 1, 68, 230, 19, 65, 22, 66, 2, 90, 42, 69, 38, 70, 62, 71, 22, - 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 250, - 196, 1, 67, 214, 173, 12, 77, 2, 78, 210, 200, 1, 88, 206, 134, 2, 72, 2, - 74, 2, 81, 2, 86, 2, 87, 3, 89, 7, 26, 73, 191, 180, 14, 69, 2, 167, 213, - 1, 71, 110, 68, 8, 67, 65, 80, 73, 84, 65, 76, 32, 162, 23, 83, 195, 210, - 15, 68, 38, 250, 215, 17, 65, 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, - 2, 74, 2, 75, 2, 76, 2, 77, 2, 79, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, - 88, 3, 89, 96, 52, 7, 82, 65, 75, 84, 85, 82, 32, 139, 252, 6, 65, 94, - 52, 8, 67, 65, 80, 73, 84, 65, 76, 32, 143, 21, 83, 42, 230, 213, 17, 65, - 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, - 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 3, 89, 222, - 1, 100, 6, 83, 77, 65, 76, 76, 32, 222, 7, 67, 230, 2, 69, 38, 75, 38, - 78, 30, 80, 98, 82, 243, 5, 84, 104, 242, 1, 68, 198, 12, 65, 22, 66, 2, + 114, 53, 98, 54, 58, 55, 78, 56, 62, 57, 192, 133, 7, 3, 50, 32, 87, 138, + 88, 48, 225, 156, 11, 4, 52, 32, 68, 69, 6, 168, 132, 7, 2, 32, 69, 196, + 149, 11, 3, 70, 32, 77, 129, 80, 7, 77, 32, 83, 84, 65, 76, 76, 4, 152, + 240, 1, 3, 70, 32, 69, 177, 142, 16, 2, 77, 32, 4, 36, 3, 70, 32, 83, 1, + 2, 77, 32, 2, 145, 225, 11, 4, 72, 69, 45, 71, 4, 188, 152, 9, 3, 77, 32, + 66, 237, 133, 9, 3, 70, 32, 83, 4, 208, 176, 17, 4, 77, 32, 66, 85, 157, + 109, 3, 70, 32, 67, 10, 196, 145, 9, 4, 51, 32, 83, 80, 240, 10, 5, 49, + 32, 66, 65, 82, 220, 231, 4, 4, 50, 32, 79, 76, 20, 6, 53, 32, 67, 89, + 80, 69, 245, 98, 4, 48, 32, 87, 72, 6, 54, 49, 172, 130, 17, 3, 48, 32, + 79, 215, 179, 2, 50, 2, 197, 156, 18, 2, 32, 87, 10, 224, 5, 3, 53, 32, + 87, 204, 143, 9, 6, 48, 32, 66, 82, 79, 78, 172, 2, 4, 49, 32, 71, 79, + 138, 158, 10, 50, 3, 54, 16, 82, 57, 214, 232, 2, 49, 198, 203, 16, 48, + 2, 50, 2, 51, 2, 52, 2, 55, 3, 56, 2, 169, 97, 3, 32, 67, 76, 20, 248, + 160, 14, 5, 51, 32, 65, 82, 77, 204, 207, 4, 5, 50, 32, 71, 65, 82, 182, + 67, 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 134, 242, + 13, 54, 200, 190, 3, 4, 51, 32, 77, 79, 158, 130, 2, 48, 2, 49, 2, 50, 2, + 52, 2, 55, 2, 56, 3, 57, 14, 246, 177, 19, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 3, 57, 4, 164, 164, 2, 3, 49, 32, 72, 143, 141, 17, 48, 50, + 42, 50, 110, 51, 130, 1, 52, 227, 1, 53, 4, 84, 8, 48, 32, 70, 79, 79, + 84, 83, 84, 169, 210, 6, 7, 53, 32, 66, 65, 84, 72, 84, 2, 143, 157, 18, + 79, 12, 104, 4, 51, 32, 83, 87, 156, 255, 13, 4, 48, 32, 83, 80, 146, + 149, 4, 49, 214, 154, 1, 50, 2, 52, 3, 54, 2, 147, 152, 18, 79, 16, 168, + 1, 9, 48, 32, 87, 72, 69, 69, 76, 69, 68, 2, 49, 34, 51, 168, 158, 17, + 12, 50, 32, 67, 72, 65, 82, 73, 79, 84, 32, 70, 82, 250, 142, 2, 53, 2, + 54, 2, 56, 3, 57, 2, 185, 163, 18, 3, 32, 67, 72, 2, 247, 190, 6, 32, 18, + 236, 165, 18, 3, 52, 32, 68, 158, 135, 1, 49, 2, 50, 2, 51, 2, 53, 2, 54, + 2, 55, 2, 56, 3, 57, 58, 50, 50, 192, 144, 12, 2, 49, 53, 1, 2, 51, 48, + 54, 50, 50, 238, 146, 12, 53, 150, 227, 1, 48, 3, 49, 12, 186, 171, 19, + 49, 2, 50, 2, 54, 2, 55, 2, 56, 3, 57, 12, 46, 49, 153, 12, 6, 50, 52, + 55, 32, 68, 73, 10, 50, 50, 70, 51, 181, 11, 5, 53, 54, 32, 84, 85, 4, + 208, 174, 3, 3, 55, 32, 75, 189, 220, 15, 5, 56, 32, 75, 65, 78, 4, 56, + 3, 53, 32, 77, 201, 239, 15, 5, 51, 32, 65, 82, 69, 2, 207, 203, 12, 69, + 176, 1, 84, 9, 76, 76, 65, 66, 76, 69, 32, 66, 48, 229, 12, 7, 77, 66, + 79, 76, 32, 66, 48, 148, 1, 114, 48, 142, 1, 49, 162, 1, 50, 230, 1, 51, + 174, 1, 52, 162, 1, 53, 154, 1, 54, 186, 1, 55, 198, 1, 56, 87, 57, 18, + 118, 54, 150, 198, 1, 55, 150, 8, 52, 190, 14, 57, 194, 246, 10, 53, 250, + 146, 2, 56, 198, 10, 49, 254, 2, 50, 207, 8, 51, 2, 183, 234, 17, 32, 16, + 122, 54, 18, 55, 190, 207, 1, 49, 134, 2, 50, 166, 26, 52, 248, 186, 1, + 2, 53, 32, 250, 242, 5, 48, 221, 243, 8, 2, 51, 32, 2, 183, 90, 32, 2, + 143, 133, 11, 32, 18, 166, 1, 51, 22, 54, 20, 3, 57, 32, 80, 224, 6, 2, + 53, 32, 220, 174, 3, 2, 48, 32, 196, 205, 14, 2, 55, 32, 140, 7, 2, 52, + 32, 158, 89, 56, 185, 44, 3, 49, 32, 81, 2, 215, 164, 14, 32, 2, 171, + 234, 14, 32, 2, 139, 138, 12, 85, 16, 134, 1, 48, 20, 2, 51, 32, 154, + 195, 1, 55, 254, 34, 54, 158, 241, 4, 57, 234, 163, 2, 56, 250, 168, 4, + 49, 133, 223, 5, 3, 50, 32, 81, 2, 151, 166, 14, 32, 2, 135, 1, 82, 16, + 116, 2, 51, 32, 22, 53, 142, 191, 1, 48, 222, 1, 49, 134, 6, 50, 218, 10, + 52, 178, 5, 54, 241, 216, 12, 3, 56, 32, 78, 2, 247, 170, 15, 65, 2, 223, + 144, 17, 32, 18, 130, 1, 51, 190, 191, 1, 49, 150, 1, 56, 42, 57, 102, + 55, 214, 5, 48, 194, 141, 2, 52, 212, 161, 15, 2, 50, 32, 157, 4, 2, 53, + 32, 2, 135, 194, 12, 32, 16, 132, 1, 2, 50, 32, 20, 2, 56, 32, 204, 1, 3, + 54, 32, 84, 178, 186, 1, 55, 226, 3, 57, 146, 2, 53, 206, 247, 6, 49, + 151, 154, 10, 48, 2, 191, 215, 17, 80, 2, 245, 132, 12, 2, 82, 79, 18, + 172, 1, 3, 54, 32, 82, 210, 186, 1, 55, 158, 8, 48, 170, 16, 53, 12, 3, + 49, 32, 68, 132, 169, 7, 2, 52, 32, 250, 249, 3, 50, 232, 143, 6, 3, 56, + 32, 81, 241, 2, 2, 51, 32, 2, 171, 131, 12, 65, 8, 206, 186, 1, 49, 188, + 24, 3, 55, 32, 84, 192, 249, 6, 2, 53, 32, 183, 146, 6, 48, 4, 154, 228, + 12, 49, 21, 3, 48, 32, 68, 28, 90, 52, 30, 54, 30, 56, 174, 253, 11, 53, + 158, 2, 49, 30, 51, 166, 1, 50, 235, 163, 3, 55, 4, 234, 154, 19, 55, 3, + 57, 4, 206, 154, 19, 51, 3, 52, 8, 178, 154, 19, 50, 2, 51, 2, 54, 3, 57, + 4, 136, 202, 10, 9, 69, 68, 32, 80, 65, 80, 69, 82, 67, 227, 188, 7, 32, + 5, 195, 233, 17, 84, 98, 96, 7, 76, 69, 84, 84, 69, 82, 32, 169, 234, 6, + 11, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 94, 238, 1, 84, 230, 240, + 2, 68, 178, 159, 10, 85, 150, 149, 2, 78, 138, 30, 69, 234, 139, 3, 67, + 2, 71, 2, 72, 2, 75, 2, 80, 2, 83, 2, 89, 2, 90, 162, 7, 65, 2, 79, 234, + 61, 66, 2, 70, 2, 74, 2, 76, 2, 77, 2, 87, 2, 88, 187, 2, 73, 20, 64, 4, + 79, 78, 69, 32, 214, 206, 18, 83, 138, 69, 72, 187, 2, 65, 12, 48, 4, 77, + 89, 65, 32, 129, 192, 1, 2, 78, 65, 10, 130, 183, 12, 74, 234, 191, 6, + 66, 158, 11, 84, 254, 16, 67, 39, 78, 196, 2, 232, 1, 2, 67, 75, 132, 1, + 3, 71, 73, 67, 176, 6, 3, 78, 71, 32, 222, 4, 84, 116, 3, 86, 69, 32, 70, + 87, 172, 12, 5, 90, 69, 78, 71, 69, 134, 231, 7, 66, 148, 138, 10, 8, 85, + 68, 76, 89, 32, 67, 82, 89, 141, 15, 4, 76, 76, 73, 80, 9, 96, 10, 73, + 78, 71, 45, 83, 72, 73, 70, 84, 32, 153, 20, 9, 32, 87, 73, 84, 72, 32, + 73, 78, 75, 4, 162, 163, 17, 90, 251, 85, 79, 40, 56, 6, 32, 71, 65, 84, + 69, 32, 137, 2, 3, 65, 76, 32, 12, 104, 6, 66, 85, 70, 70, 69, 82, 84, 9, + 73, 78, 86, 69, 82, 84, 69, 68, 32, 182, 130, 18, 65, 159, 85, 79, 5, + 133, 1, 17, 32, 87, 73, 84, 72, 32, 73, 78, 86, 69, 82, 84, 69, 68, 32, + 73, 78, 4, 48, 3, 79, 85, 84, 129, 154, 6, 3, 73, 78, 80, 2, 167, 150, + 18, 80, 28, 36, 3, 65, 78, 68, 85, 2, 79, 82, 15, 33, 6, 32, 87, 73, 84, + 72, 32, 12, 226, 1, 68, 94, 72, 58, 77, 151, 161, 15, 85, 15, 11, 32, 12, + 88, 13, 79, 86, 69, 82, 76, 65, 80, 80, 73, 78, 71, 32, 76, 49, 5, 87, + 73, 84, 72, 32, 2, 249, 255, 17, 7, 79, 71, 73, 67, 65, 76, 32, 10, 26, + 68, 94, 72, 59, 77, 6, 11, 79, 6, 44, 5, 85, 66, 76, 69, 32, 247, 143, + 18, 84, 4, 234, 161, 15, 85, 235, 60, 79, 2, 177, 207, 15, 9, 79, 82, 73, + 90, 79, 78, 84, 65, 76, 2, 225, 218, 7, 5, 73, 68, 68, 76, 69, 34, 66, + 68, 228, 1, 4, 76, 69, 70, 84, 109, 5, 82, 73, 71, 72, 84, 6, 164, 1, 30, + 65, 83, 72, 32, 70, 82, 79, 77, 32, 76, 69, 70, 84, 32, 77, 69, 77, 66, + 69, 82, 32, 79, 70, 32, 68, 79, 85, 66, 76, 69, 166, 144, 16, 73, 219, + 208, 1, 82, 2, 17, 2, 32, 86, 2, 181, 128, 18, 5, 69, 82, 84, 73, 67, 16, + 18, 32, 119, 87, 6, 48, 6, 82, 73, 71, 72, 84, 32, 139, 238, 15, 84, 4, + 246, 227, 15, 68, 215, 138, 2, 65, 12, 26, 87, 139, 244, 8, 32, 10, 29, + 5, 65, 82, 68, 83, 32, 10, 82, 65, 0, 8, 68, 79, 85, 66, 76, 69, 32, 65, + 221, 227, 7, 4, 83, 81, 85, 73, 4, 25, 4, 82, 82, 79, 87, 5, 137, 162, + 16, 2, 32, 70, 6, 92, 5, 73, 79, 78, 32, 66, 144, 237, 17, 9, 32, 79, 70, + 32, 70, 79, 82, 84, 85, 211, 91, 85, 2, 211, 235, 9, 79, 4, 38, 76, 157, + 249, 13, 3, 72, 79, 84, 2, 133, 206, 17, 2, 69, 84, 222, 1, 34, 32, 229, + 1, 3, 69, 82, 32, 14, 138, 1, 66, 196, 195, 10, 11, 68, 79, 85, 66, 76, + 69, 32, 80, 82, 73, 77, 154, 173, 6, 65, 204, 111, 6, 75, 65, 86, 89, 75, + 65, 211, 10, 76, 4, 38, 82, 249, 248, 4, 3, 65, 84, 84, 2, 209, 240, 17, + 7, 73, 71, 72, 84, 78, 69, 83, 208, 1, 158, 1, 72, 140, 2, 5, 76, 69, 70, + 84, 32, 236, 2, 6, 82, 73, 71, 72, 84, 32, 234, 230, 15, 66, 74, 67, 74, + 70, 234, 10, 77, 150, 2, 79, 166, 18, 83, 67, 84, 20, 84, 4, 65, 76, 70, + 32, 241, 184, 7, 11, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 18, 212, + 164, 7, 9, 65, 78, 68, 32, 85, 80, 80, 69, 82, 32, 7, 73, 78, 86, 69, 82, + 83, 69, 194, 201, 8, 72, 230, 1, 86, 170, 26, 77, 130, 3, 76, 22, 82, + 202, 5, 66, 195, 156, 1, 67, 72, 186, 1, 66, 60, 8, 70, 79, 85, 78, 84, + 65, 73, 78, 22, 80, 56, 12, 83, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, + 82, 154, 1, 81, 246, 2, 84, 248, 191, 11, 3, 67, 82, 65, 139, 177, 4, 79, + 24, 56, 8, 65, 76, 76, 80, 79, 73, 78, 84, 255, 246, 15, 76, 2, 179, 240, + 9, 32, 4, 212, 243, 14, 5, 65, 73, 78, 84, 66, 163, 140, 1, 69, 2, 221, + 193, 7, 5, 32, 65, 78, 84, 73, 74, 110, 81, 166, 2, 83, 82, 84, 238, 176, + 7, 82, 202, 184, 8, 66, 238, 3, 67, 226, 3, 79, 222, 7, 68, 207, 2, 80, + 30, 17, 2, 85, 65, 30, 48, 6, 68, 82, 65, 78, 84, 32, 187, 130, 16, 82, + 28, 74, 70, 60, 2, 78, 69, 50, 83, 226, 253, 15, 67, 226, 1, 77, 143, 1, + 84, 4, 26, 65, 255, 180, 17, 82, 2, 185, 183, 7, 3, 67, 69, 32, 2, 25, 4, + 85, 84, 82, 65, 2, 191, 189, 18, 76, 4, 254, 218, 10, 77, 143, 165, 5, + 84, 6, 148, 190, 7, 11, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, 82, 231, + 194, 8, 72, 6, 226, 129, 16, 82, 155, 1, 87, 5, 141, 213, 17, 25, 32, 68, + 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, 72, 79, 82, 73, 90, 79, 78, 84, + 65, 76, 32, 82, 85, 6, 18, 71, 23, 78, 2, 215, 241, 13, 71, 4, 188, 213, + 17, 5, 65, 82, 32, 69, 67, 255, 100, 71, 114, 104, 12, 67, 73, 65, 78, + 32, 76, 69, 84, 84, 69, 82, 32, 228, 1, 5, 68, 73, 65, 78, 32, 131, 237, + 17, 73, 58, 206, 1, 77, 194, 137, 5, 75, 206, 203, 9, 66, 50, 84, 234, + 164, 2, 65, 2, 69, 2, 78, 134, 251, 1, 68, 2, 71, 2, 72, 2, 73, 2, 74, 2, + 76, 2, 80, 2, 81, 2, 82, 2, 83, 2, 85, 2, 87, 2, 88, 3, 90, 5, 167, 245, + 18, 77, 54, 88, 7, 76, 69, 84, 84, 69, 82, 32, 189, 213, 14, 9, 84, 82, + 73, 65, 78, 71, 85, 76, 65, 52, 214, 148, 13, 84, 174, 192, 1, 76, 198, + 152, 2, 83, 238, 11, 65, 2, 69, 2, 78, 134, 251, 1, 66, 2, 67, 2, 68, 2, + 70, 2, 71, 2, 73, 2, 75, 2, 77, 2, 79, 2, 81, 2, 82, 2, 85, 2, 86, 3, 89, + 248, 53, 162, 1, 65, 206, 103, 69, 246, 102, 73, 154, 25, 79, 148, 112, + 3, 82, 79, 32, 234, 3, 85, 192, 70, 7, 89, 65, 78, 77, 65, 82, 32, 218, + 145, 15, 86, 198, 61, 77, 27, 87, 204, 23, 186, 1, 71, 62, 72, 254, 10, + 75, 234, 3, 76, 252, 13, 2, 77, 77, 22, 78, 154, 17, 80, 118, 82, 174, 7, + 83, 190, 7, 84, 196, 36, 3, 89, 65, 78, 140, 129, 1, 3, 88, 73, 77, 247, + 230, 15, 67, 6, 128, 226, 17, 4, 73, 67, 32, 87, 238, 88, 78, 155, 53, + 69, 166, 1, 84, 6, 65, 74, 65, 78, 73, 32, 129, 3, 10, 74, 79, 78, 71, + 32, 84, 73, 76, 69, 32, 78, 38, 76, 162, 2, 83, 207, 129, 13, 65, 72, 88, + 6, 69, 84, 84, 69, 82, 32, 161, 144, 12, 10, 73, 71, 65, 84, 85, 82, 69, + 32, 83, 72, 70, 182, 174, 3, 78, 186, 229, 11, 68, 82, 82, 34, 84, 206, + 145, 3, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, + 2, 83, 2, 86, 186, 2, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 226, 235, 6, 69, + 229, 236, 7, 5, 73, 71, 78, 32, 78, 88, 236, 1, 2, 66, 65, 38, 69, 46, + 70, 46, 78, 42, 79, 46, 80, 22, 83, 118, 84, 194, 1, 87, 112, 5, 71, 82, + 69, 69, 78, 0, 3, 82, 69, 68, 208, 135, 6, 3, 65, 85, 84, 214, 21, 74, + 197, 158, 11, 11, 67, 72, 82, 89, 83, 65, 78, 84, 72, 69, 77, 4, 138, + 158, 1, 77, 223, 200, 17, 67, 8, 228, 2, 4, 73, 71, 72, 84, 195, 1, 65, + 12, 172, 2, 2, 73, 86, 13, 3, 79, 85, 82, 8, 192, 1, 2, 79, 82, 65, 2, + 73, 78, 8, 218, 1, 78, 189, 223, 11, 3, 82, 67, 72, 2, 159, 191, 17, 76, + 18, 88, 2, 79, 85, 76, 4, 69, 86, 69, 78, 0, 2, 73, 88, 198, 151, 12, 85, + 187, 194, 1, 80, 2, 157, 2, 2, 84, 72, 12, 36, 3, 72, 82, 69, 13, 2, 87, + 79, 6, 11, 69, 6, 25, 4, 32, 79, 70, 32, 6, 46, 67, 229, 246, 6, 5, 66, + 65, 77, 66, 79, 4, 224, 139, 6, 2, 73, 82, 217, 217, 10, 5, 72, 65, 82, + 65, 67, 6, 50, 69, 60, 4, 72, 73, 84, 69, 247, 172, 17, 73, 2, 17, 2, 83, + 84, 2, 17, 2, 32, 87, 2, 147, 215, 17, 73, 2, 17, 2, 32, 68, 2, 183, 172, + 17, 82, 52, 52, 5, 65, 83, 65, 82, 32, 233, 251, 14, 2, 69, 77, 50, 162, + 1, 69, 40, 7, 76, 69, 84, 84, 69, 82, 32, 152, 1, 5, 80, 65, 83, 83, 73, + 32, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 173, 216, 18, 3, 65, + 78, 71, 2, 161, 228, 15, 5, 78, 68, 32, 79, 70, 36, 158, 232, 16, 78, + 246, 247, 1, 66, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, + 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, 2, 65, 2, 11, 77, 2, 211, 136, + 18, 66, 8, 146, 225, 18, 69, 2, 73, 2, 79, 3, 85, 246, 1, 80, 7, 65, 89, + 65, 76, 65, 77, 32, 176, 11, 2, 69, 32, 225, 1, 3, 84, 69, 83, 236, 1, + 198, 1, 68, 44, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 240, 1, 7, 76, 69, + 84, 84, 69, 82, 32, 164, 5, 7, 78, 85, 77, 66, 69, 82, 32, 36, 5, 83, 73, + 71, 78, 32, 178, 180, 13, 86, 247, 196, 1, 65, 22, 204, 169, 8, 2, 65, + 84, 227, 195, 8, 73, 26, 56, 4, 79, 78, 69, 32, 121, 6, 84, 72, 82, 69, + 69, 32, 18, 82, 84, 146, 135, 5, 83, 230, 249, 7, 70, 62, 79, 130, 216, + 3, 69, 46, 72, 47, 81, 4, 254, 131, 13, 87, 239, 214, 3, 69, 8, 170, 135, + 5, 83, 170, 253, 7, 69, 110, 84, 163, 214, 3, 81, 132, 1, 226, 1, 65, 82, + 67, 158, 1, 68, 78, 84, 82, 86, 218, 130, 13, 78, 146, 251, 1, 76, 38, + 82, 134, 6, 85, 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 82, 66, 2, + 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, 2, 77, 3, 89, 11, 192, + 151, 16, 7, 82, 67, 72, 65, 73, 67, 32, 254, 194, 2, 65, 2, 73, 3, 85, + 22, 26, 72, 215, 217, 18, 65, 20, 44, 5, 73, 76, 76, 85, 32, 167, 217, + 18, 65, 18, 130, 249, 12, 76, 158, 229, 3, 78, 170, 194, 1, 82, 222, 56, + 75, 2, 77, 3, 89, 10, 220, 220, 10, 4, 79, 84, 32, 82, 182, 180, 7, 68, + 138, 69, 72, 187, 2, 65, 10, 38, 84, 170, 213, 18, 72, 187, 2, 65, 6, + 166, 213, 18, 72, 2, 84, 187, 2, 65, 12, 166, 227, 3, 69, 138, 161, 11, + 79, 139, 211, 3, 65, 6, 154, 131, 13, 79, 131, 128, 4, 84, 18, 50, 67, + 114, 86, 206, 254, 14, 65, 167, 146, 3, 80, 6, 54, 79, 120, 5, 73, 82, + 67, 85, 76, 203, 192, 14, 65, 2, 221, 178, 13, 9, 77, 66, 73, 78, 73, 78, + 71, 32, 65, 6, 60, 9, 69, 82, 84, 73, 67, 65, 76, 32, 66, 255, 145, 18, + 73, 2, 209, 254, 14, 2, 65, 82, 8, 80, 12, 87, 73, 84, 72, 32, 83, 84, + 82, 79, 75, 69, 32, 70, 65, 227, 204, 17, 83, 4, 64, 10, 65, 78, 68, 32, + 77, 65, 76, 69, 32, 65, 227, 204, 17, 83, 2, 25, 4, 78, 68, 32, 70, 2, + 11, 69, 2, 11, 77, 2, 231, 209, 7, 65, 2, 211, 145, 17, 69, 2, 219, 208, + 16, 79, 185, 1, 210, 1, 32, 220, 2, 5, 68, 65, 73, 67, 32, 224, 3, 8, 73, + 67, 72, 65, 69, 65, 78, 32, 222, 8, 83, 216, 165, 2, 3, 85, 65, 76, 190, + 217, 10, 65, 152, 168, 1, 8, 84, 69, 76, 80, 73, 69, 67, 69, 203, 251, 3, + 71, 12, 132, 1, 3, 73, 78, 32, 128, 1, 5, 87, 73, 84, 72, 32, 204, 146, + 2, 3, 68, 65, 78, 253, 252, 12, 8, 65, 78, 68, 32, 87, 79, 77, 65, 4, + 144, 135, 10, 19, 66, 85, 83, 73, 78, 69, 83, 83, 32, 83, 85, 73, 84, 32, + 76, 69, 86, 73, 84, 185, 151, 1, 4, 84, 85, 88, 69, 4, 224, 231, 13, 8, + 71, 85, 65, 32, 80, 73, 32, 77, 149, 157, 3, 4, 84, 85, 82, 66, 58, 112, + 4, 65, 70, 70, 82, 28, 7, 76, 69, 84, 84, 69, 82, 32, 244, 173, 14, 3, + 86, 79, 67, 146, 67, 71, 251, 25, 80, 2, 249, 227, 17, 2, 73, 67, 50, 82, + 65, 176, 1, 2, 68, 85, 2, 85, 28, 3, 72, 65, 76, 22, 73, 183, 132, 18, + 75, 38, 154, 1, 75, 198, 235, 12, 84, 146, 192, 1, 83, 242, 207, 3, 73, + 226, 79, 66, 2, 68, 2, 71, 2, 72, 2, 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, + 82, 3, 90, 5, 235, 200, 18, 83, 2, 153, 131, 8, 2, 83, 72, 2, 187, 200, + 18, 81, 4, 222, 202, 18, 78, 3, 84, 102, 216, 1, 7, 76, 69, 84, 84, 69, + 82, 32, 194, 4, 78, 80, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, + 32, 220, 1, 4, 83, 73, 71, 78, 129, 195, 10, 16, 65, 66, 66, 82, 69, 86, + 73, 65, 84, 73, 79, 78, 32, 77, 65, 82, 72, 218, 1, 65, 46, 66, 38, 68, + 30, 71, 30, 74, 2, 90, 30, 75, 30, 81, 38, 83, 50, 84, 54, 88, 234, 162, + 6, 76, 158, 206, 1, 82, 154, 217, 2, 89, 158, 208, 1, 72, 222, 169, 5, + 78, 134, 2, 87, 218, 103, 70, 2, 80, 171, 4, 77, 6, 150, 203, 10, 76, + 122, 65, 171, 251, 6, 89, 4, 254, 237, 12, 72, 211, 214, 3, 69, 4, 202, + 164, 6, 65, 23, 72, 4, 190, 202, 10, 72, 15, 73, 4, 250, 202, 10, 72, 15, + 65, 4, 194, 161, 12, 72, 15, 65, 4, 158, 164, 6, 72, 151, 253, 5, 79, 8, + 194, 199, 10, 83, 154, 3, 65, 251, 250, 6, 72, 6, 154, 163, 6, 72, 206, + 159, 10, 69, 243, 131, 1, 65, 4, 170, 160, 12, 65, 3, 79, 10, 33, 6, 85, + 77, 66, 69, 82, 32, 10, 130, 202, 10, 79, 58, 84, 135, 198, 2, 70, 14, + 80, 2, 68, 79, 116, 3, 76, 73, 78, 134, 167, 5, 70, 186, 202, 11, 84, + 167, 10, 83, 6, 54, 84, 13, 9, 85, 66, 76, 69, 32, 68, 79, 84, 32, 5, 11, + 32, 2, 25, 4, 87, 73, 84, 72, 2, 199, 192, 2, 73, 2, 155, 203, 3, 69, 2, + 159, 150, 17, 32, 2, 11, 32, 2, 249, 170, 18, 2, 83, 72, 4, 92, 17, 32, + 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 76, 73, 71, 72, 84, 147, 214, + 10, 76, 2, 135, 252, 14, 72, 142, 1, 142, 1, 65, 20, 5, 67, 72, 69, 78, + 32, 144, 163, 6, 4, 82, 73, 65, 71, 193, 157, 11, 13, 84, 73, 65, 76, 32, + 65, 82, 84, 83, 32, 85, 78, 73, 2, 171, 161, 5, 67, 136, 1, 152, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 194, 1, 83, 236, 2, 11, 86, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 170, 237, 17, 72, 225, 5, 4, 77, 65, 82, 75, 60, + 254, 3, 84, 146, 148, 2, 68, 202, 171, 14, 78, 238, 178, 1, 67, 2, 75, 2, + 80, 2, 83, 2, 90, 138, 69, 45, 2, 66, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, + 2, 82, 2, 87, 2, 89, 187, 2, 65, 62, 96, 4, 73, 71, 78, 32, 37, 16, 85, + 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 32, 4, 158, 167, + 14, 67, 203, 207, 3, 65, 58, 182, 1, 84, 146, 148, 2, 68, 202, 171, 14, + 78, 238, 178, 1, 67, 2, 75, 2, 80, 2, 83, 2, 90, 138, 69, 66, 2, 71, 2, + 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, 187, 2, 65, 8, 194, 242, + 17, 83, 138, 69, 72, 187, 2, 65, 10, 158, 183, 18, 65, 186, 2, 69, 2, 73, + 2, 79, 3, 85, 156, 1, 120, 11, 65, 82, 65, 77, 32, 71, 79, 78, 68, 73, + 32, 232, 5, 3, 67, 85, 76, 64, 5, 75, 32, 87, 79, 82, 183, 240, 17, 85, + 150, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 178, 2, 82, 56, 5, 83, 73, + 71, 78, 32, 82, 86, 223, 194, 16, 68, 94, 210, 1, 74, 42, 84, 154, 219, + 14, 65, 38, 68, 214, 6, 85, 206, 201, 1, 73, 42, 76, 214, 192, 1, 75, 38, + 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, + 86, 2, 89, 186, 2, 69, 3, 79, 6, 130, 179, 18, 78, 38, 72, 187, 2, 65, + 10, 246, 237, 17, 84, 138, 69, 72, 2, 82, 187, 2, 65, 4, 32, 2, 65, 45, + 175, 220, 14, 69, 2, 147, 240, 17, 75, 10, 178, 176, 1, 67, 198, 196, 12, + 72, 246, 43, 78, 186, 221, 1, 86, 179, 241, 1, 65, 22, 26, 79, 139, 251, + 15, 73, 20, 45, 9, 87, 69, 76, 32, 83, 73, 71, 78, 32, 20, 74, 86, 154, + 223, 14, 65, 38, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 2, 233, + 201, 7, 6, 79, 67, 65, 76, 73, 67, 2, 145, 249, 17, 11, 73, 78, 69, 32, + 79, 82, 68, 73, 78, 65, 76, 2, 207, 158, 17, 75, 226, 15, 76, 10, 72, 69, + 77, 65, 84, 73, 67, 65, 76, 32, 237, 207, 14, 3, 69, 32, 68, 224, 15, + 252, 1, 5, 66, 79, 76, 68, 32, 168, 6, 14, 68, 79, 85, 66, 76, 69, 45, + 83, 84, 82, 85, 67, 75, 32, 238, 1, 70, 164, 2, 7, 73, 84, 65, 76, 73, + 67, 32, 180, 3, 10, 77, 79, 78, 79, 83, 80, 65, 67, 69, 32, 40, 2, 82, + 73, 36, 3, 76, 69, 70, 167, 1, 83, 160, 5, 176, 1, 8, 67, 65, 80, 73, 84, + 65, 76, 32, 130, 2, 83, 210, 14, 73, 222, 3, 69, 38, 75, 38, 78, 30, 80, + 98, 82, 242, 5, 84, 56, 7, 70, 82, 65, 75, 84, 85, 82, 159, 160, 16, 68, + 104, 190, 4, 68, 174, 15, 84, 186, 4, 65, 22, 66, 2, 90, 42, 69, 98, 71, + 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 70, 85, 142, 197, + 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, 222, 137, 2, 70, 2, 72, + 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 208, 1, 60, 5, 77, 65, 76, 76, 32, + 213, 25, 5, 67, 82, 73, 80, 84, 104, 250, 1, 68, 230, 19, 65, 22, 66, 2, 90, 42, 69, 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, - 82, 22, 83, 34, 84, 38, 85, 250, 196, 1, 67, 214, 173, 12, 77, 2, 78, - 210, 200, 1, 88, 206, 134, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 9, 52, 7, - 79, 84, 76, 69, 83, 83, 32, 131, 173, 14, 69, 4, 154, 209, 17, 73, 3, 74, - 124, 130, 16, 67, 34, 83, 195, 210, 15, 68, 12, 32, 2, 71, 72, 179, 246, - 6, 83, 10, 17, 2, 84, 32, 10, 112, 6, 87, 72, 73, 84, 69, 32, 170, 236, - 5, 68, 234, 99, 65, 197, 164, 7, 9, 70, 76, 65, 84, 84, 69, 78, 69, 68, - 4, 254, 246, 13, 83, 39, 84, 130, 6, 84, 10, 65, 78, 83, 45, 83, 69, 82, - 73, 70, 32, 133, 14, 6, 67, 82, 73, 80, 84, 32, 176, 5, 96, 5, 66, 79, - 76, 68, 32, 176, 12, 6, 73, 84, 65, 76, 73, 67, 34, 67, 34, 83, 195, 210, - 15, 68, 204, 3, 98, 73, 122, 67, 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, - 82, 30, 83, 214, 5, 84, 187, 211, 15, 68, 220, 1, 33, 6, 84, 65, 76, 73, - 67, 32, 220, 1, 74, 67, 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, 82, 30, - 83, 215, 5, 84, 102, 37, 7, 65, 80, 73, 84, 65, 76, 32, 102, 250, 1, 84, - 186, 4, 65, 22, 66, 2, 90, 22, 68, 22, 69, 98, 71, 22, 73, 22, 75, 34, - 76, 22, 79, 50, 80, 42, 82, 22, 83, 70, 85, 250, 196, 1, 67, 214, 173, - 12, 77, 2, 78, 210, 200, 1, 88, 206, 134, 2, 70, 2, 72, 2, 74, 2, 81, 2, - 86, 2, 87, 3, 89, 9, 40, 4, 72, 69, 84, 65, 183, 164, 17, 65, 5, 151, - 182, 16, 32, 2, 185, 212, 15, 4, 80, 83, 73, 76, 2, 17, 2, 65, 80, 2, - 159, 7, 80, 2, 165, 166, 17, 2, 65, 66, 6, 74, 72, 168, 141, 5, 8, 65, - 82, 84, 73, 65, 76, 32, 68, 163, 167, 11, 73, 2, 199, 180, 16, 73, 2, - 177, 180, 16, 2, 72, 79, 102, 29, 5, 77, 65, 76, 76, 32, 102, 246, 1, 65, - 22, 66, 2, 90, 22, 68, 22, 69, 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, - 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 250, 196, 1, 67, 214, - 173, 12, 77, 2, 78, 210, 200, 1, 88, 206, 134, 2, 72, 2, 74, 2, 81, 2, - 86, 2, 87, 3, 89, 5, 235, 208, 1, 76, 5, 175, 160, 17, 69, 5, 203, 160, - 14, 69, 7, 178, 212, 1, 80, 235, 237, 15, 84, 5, 11, 73, 2, 29, 5, 78, - 65, 76, 32, 83, 2, 227, 1, 73, 5, 219, 248, 14, 65, 5, 147, 159, 17, 79, - 5, 11, 65, 2, 223, 158, 14, 80, 5, 143, 160, 14, 65, 7, 11, 77, 4, 230, - 170, 1, 73, 239, 213, 15, 69, 9, 142, 175, 17, 72, 2, 83, 219, 19, 73, 5, - 203, 163, 17, 72, 5, 11, 73, 2, 143, 161, 17, 71, 7, 190, 157, 14, 72, - 231, 255, 2, 65, 5, 199, 209, 1, 80, 2, 11, 72, 2, 11, 69, 2, 11, 84, 2, - 159, 174, 16, 65, 104, 11, 32, 104, 18, 67, 35, 83, 52, 53, 5, 65, 80, - 73, 84, 65, 52, 21, 3, 77, 65, 76, 52, 183, 216, 11, 76, 82, 76, 8, 67, - 65, 80, 73, 84, 65, 76, 32, 157, 1, 6, 83, 77, 65, 76, 76, 32, 36, 222, - 191, 17, 65, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, 78, 2, 79, 2, 80, 2, - 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 46, 194, 190, - 17, 65, 2, 66, 2, 67, 2, 68, 2, 70, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, 2, - 77, 2, 78, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, - 2, 89, 3, 90, 40, 45, 9, 32, 78, 85, 77, 69, 82, 65, 76, 32, 40, 70, 69, - 66, 70, 70, 78, 26, 83, 66, 84, 142, 206, 15, 90, 143, 83, 79, 6, 40, 4, - 73, 71, 72, 84, 243, 224, 9, 76, 5, 231, 234, 15, 69, 8, 30, 73, 105, 3, - 79, 85, 82, 4, 202, 254, 4, 70, 139, 166, 12, 86, 4, 65, 3, 73, 78, 69, - 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 5, 191, 233, 15, 84, 10, 42, 72, - 190, 224, 9, 87, 243, 137, 7, 69, 4, 150, 253, 4, 73, 183, 208, 10, 82, - 248, 8, 234, 1, 65, 192, 4, 9, 67, 72, 65, 78, 73, 67, 65, 76, 32, 34, - 68, 180, 15, 11, 69, 84, 69, 73, 32, 77, 65, 89, 69, 75, 32, 178, 11, 76, - 34, 78, 162, 50, 82, 172, 15, 8, 83, 83, 65, 71, 69, 32, 87, 65, 20, 2, - 84, 82, 183, 183, 16, 77, 26, 72, 6, 83, 85, 82, 69, 68, 32, 249, 141, - 16, 6, 84, 32, 79, 78, 32, 66, 24, 96, 5, 65, 78, 71, 76, 69, 200, 247, - 9, 9, 82, 73, 71, 72, 84, 32, 65, 78, 71, 175, 173, 7, 66, 21, 11, 32, - 18, 212, 1, 39, 87, 73, 84, 72, 32, 79, 80, 69, 78, 32, 65, 82, 77, 32, - 69, 78, 68, 73, 78, 71, 32, 73, 78, 32, 65, 82, 82, 79, 87, 32, 80, 79, - 73, 78, 84, 73, 78, 71, 32, 161, 130, 17, 7, 79, 80, 69, 78, 73, 78, 71, - 16, 62, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 43, 85, 4, 73, 3, - 79, 87, 78, 4, 173, 249, 7, 5, 84, 32, 65, 78, 68, 4, 11, 80, 4, 145, - 190, 11, 3, 32, 65, 78, 4, 254, 180, 16, 65, 211, 56, 76, 250, 1, 56, 9, - 69, 70, 65, 73, 68, 82, 73, 78, 32, 159, 7, 73, 190, 1, 174, 1, 67, 52, - 6, 68, 73, 71, 73, 84, 32, 152, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, - 83, 132, 255, 6, 12, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 215, - 228, 7, 70, 70, 128, 3, 5, 65, 80, 73, 84, 65, 255, 226, 14, 79, 26, 82, - 84, 48, 2, 79, 78, 194, 194, 15, 70, 30, 83, 102, 90, 130, 83, 78, 211, - 110, 69, 8, 44, 3, 72, 82, 69, 209, 176, 16, 2, 87, 79, 4, 207, 176, 16, - 69, 20, 50, 84, 226, 240, 4, 69, 46, 70, 42, 78, 31, 83, 6, 162, 242, 4, - 72, 204, 227, 4, 2, 87, 69, 215, 137, 7, 69, 70, 68, 3, 77, 65, 76, 217, - 183, 9, 8, 89, 77, 66, 79, 76, 32, 65, 73, 68, 45, 9, 76, 32, 76, 69, 84, - 84, 69, 82, 32, 68, 242, 1, 65, 38, 78, 146, 221, 3, 72, 2, 75, 162, 255, - 9, 89, 138, 143, 3, 79, 150, 64, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, - 2, 73, 2, 74, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, - 86, 2, 87, 2, 88, 3, 90, 7, 162, 237, 3, 84, 207, 190, 13, 73, 7, 202, - 171, 17, 71, 3, 89, 60, 48, 5, 69, 86, 65, 76, 32, 45, 3, 85, 77, 32, 6, - 138, 211, 10, 69, 206, 140, 4, 67, 115, 81, 54, 210, 1, 66, 46, 70, 164, - 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 248, 1, 11, 77, 65, 84, 72, 69, - 77, 65, 84, 73, 67, 65, 22, 83, 104, 4, 84, 72, 82, 69, 186, 212, 14, 86, - 230, 59, 69, 166, 4, 87, 207, 10, 71, 4, 164, 4, 3, 79, 76, 68, 163, 201, - 13, 76, 10, 84, 9, 76, 65, 84, 84, 69, 78, 69, 68, 32, 204, 3, 3, 79, 85, - 82, 219, 145, 15, 73, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 213, - 1, 3, 84, 32, 80, 6, 11, 84, 6, 78, 32, 41, 15, 45, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 65, 78, 71, 76, 69, 4, 36, 5, 67, 85, 82, 76, 89, 59, 80, - 2, 17, 2, 32, 66, 2, 237, 163, 7, 4, 82, 65, 67, 75, 2, 153, 226, 16, 10, - 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 2, 251, 230, 16, 76, 10, 72, 4, - 77, 65, 76, 76, 154, 181, 14, 72, 188, 91, 2, 73, 88, 183, 2, 65, 2, 133, - 162, 14, 2, 32, 87, 4, 11, 69, 4, 45, 9, 32, 80, 79, 73, 78, 84, 69, 68, - 32, 4, 206, 179, 9, 80, 151, 221, 5, 66, 158, 1, 146, 1, 65, 104, 6, 67, - 72, 69, 73, 75, 72, 34, 76, 144, 6, 8, 83, 89, 76, 76, 65, 66, 76, 69, 0, - 4, 87, 79, 82, 68, 50, 86, 223, 172, 15, 68, 6, 80, 8, 72, 65, 78, 71, - 32, 75, 72, 85, 164, 6, 3, 80, 85, 78, 155, 177, 13, 78, 2, 175, 232, 15, - 68, 4, 130, 210, 16, 65, 255, 59, 69, 94, 52, 6, 69, 84, 84, 69, 82, 32, - 185, 5, 2, 85, 77, 92, 250, 1, 66, 36, 2, 67, 72, 38, 68, 50, 71, 38, 74, - 34, 75, 42, 78, 62, 80, 30, 83, 42, 84, 54, 73, 0, 3, 76, 65, 73, 0, 3, - 77, 73, 84, 230, 210, 12, 72, 166, 10, 82, 2, 87, 188, 144, 2, 2, 65, 84, - 230, 213, 1, 89, 242, 8, 85, 214, 79, 69, 3, 79, 4, 198, 229, 15, 72, - 191, 185, 1, 65, 4, 194, 149, 16, 73, 159, 137, 1, 65, 8, 238, 141, 10, - 72, 178, 135, 6, 73, 235, 65, 68, 4, 190, 141, 10, 72, 183, 141, 7, 79, - 4, 170, 228, 15, 72, 163, 48, 73, 6, 216, 1, 2, 79, 75, 163, 139, 10, 72, - 12, 178, 1, 65, 0, 3, 71, 79, 85, 182, 153, 17, 78, 3, 89, 6, 118, 65, - 179, 226, 15, 72, 6, 238, 139, 17, 65, 162, 14, 72, 3, 83, 10, 48, 2, 73, - 76, 162, 139, 10, 72, 155, 201, 6, 84, 5, 157, 177, 1, 4, 32, 76, 79, 78, - 2, 161, 152, 17, 3, 32, 73, 89, 2, 141, 178, 16, 7, 32, 82, 69, 80, 69, - 84, 73, 30, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 191, 232, 14, - 73, 28, 130, 1, 65, 44, 4, 67, 72, 69, 73, 2, 79, 0, 3, 83, 79, 85, 0, 2, - 89, 69, 22, 73, 38, 85, 206, 231, 10, 78, 187, 129, 4, 86, 8, 218, 136, - 16, 78, 170, 80, 65, 175, 64, 85, 2, 175, 136, 16, 78, 4, 154, 136, 16, - 78, 215, 144, 1, 73, 4, 246, 135, 16, 78, 215, 144, 1, 85, 4, 158, 144, - 16, 84, 183, 56, 79, 174, 3, 160, 1, 11, 68, 69, 32, 75, 73, 75, 65, 75, - 85, 73, 32, 244, 130, 16, 20, 79, 82, 65, 72, 32, 87, 73, 84, 72, 32, 78, - 73, 78, 69, 32, 66, 82, 65, 78, 67, 95, 83, 170, 3, 148, 1, 17, 67, 79, - 77, 66, 73, 78, 73, 78, 71, 32, 78, 85, 77, 66, 69, 82, 32, 164, 1, 10, - 83, 89, 76, 76, 65, 66, 76, 69, 32, 77, 199, 144, 9, 68, 14, 62, 84, 56, - 7, 72, 85, 78, 68, 82, 69, 68, 243, 177, 4, 77, 8, 26, 69, 187, 178, 4, - 72, 6, 26, 78, 187, 230, 13, 69, 4, 140, 178, 4, 2, 32, 84, 191, 226, 12, - 83, 138, 3, 22, 48, 167, 21, 49, 198, 1, 118, 48, 250, 1, 49, 210, 1, 50, - 138, 2, 51, 254, 1, 52, 150, 2, 53, 158, 2, 54, 238, 1, 55, 142, 2, 56, - 187, 2, 57, 18, 142, 1, 49, 34, 50, 22, 51, 22, 52, 218, 149, 2, 53, 142, - 230, 3, 56, 180, 149, 10, 3, 57, 32, 77, 92, 3, 55, 32, 77, 213, 90, 3, - 54, 32, 87, 2, 11, 32, 2, 147, 254, 16, 75, 2, 239, 136, 17, 32, 2, 183, - 239, 11, 32, 2, 11, 32, 2, 203, 253, 16, 87, 20, 130, 1, 49, 22, 54, 18, - 56, 22, 57, 200, 28, 2, 48, 32, 138, 161, 7, 53, 130, 190, 5, 52, 226, - 10, 55, 222, 9, 50, 199, 191, 3, 51, 2, 227, 226, 16, 32, 2, 143, 25, 32, - 2, 139, 170, 13, 32, 2, 191, 209, 12, 32, 20, 106, 49, 22, 50, 22, 51, - 22, 52, 22, 53, 22, 54, 22, 55, 22, 57, 158, 186, 11, 48, 253, 232, 1, 2, - 56, 32, 2, 155, 144, 10, 32, 2, 195, 234, 9, 32, 2, 223, 233, 16, 32, 2, - 215, 233, 11, 32, 2, 131, 191, 12, 32, 2, 215, 235, 16, 32, 2, 143, 185, - 12, 32, 2, 191, 28, 32, 20, 174, 1, 48, 16, 2, 49, 32, 20, 2, 52, 32, 20, - 2, 56, 32, 230, 186, 4, 57, 238, 51, 50, 22, 53, 188, 201, 7, 2, 54, 32, - 244, 211, 3, 3, 55, 32, 78, 213, 90, 3, 51, 32, 89, 2, 187, 37, 32, 2, - 159, 248, 16, 89, 2, 139, 248, 16, 70, 2, 155, 196, 15, 78, 20, 184, 1, - 2, 48, 32, 20, 2, 51, 32, 30, 53, 22, 56, 164, 8, 3, 52, 32, 75, 252, 21, - 3, 54, 32, 72, 230, 2, 55, 170, 205, 4, 49, 212, 164, 4, 3, 57, 32, 87, - 245, 181, 5, 3, 50, 32, 72, 2, 203, 157, 15, 72, 2, 177, 199, 16, 2, 78, - 71, 2, 223, 252, 10, 32, 2, 187, 234, 16, 32, 20, 178, 1, 48, 18, 53, 20, - 2, 54, 32, 20, 2, 56, 32, 22, 57, 172, 190, 9, 2, 50, 32, 236, 4, 2, 51, - 32, 166, 200, 1, 49, 164, 187, 3, 3, 52, 32, 76, 213, 137, 1, 3, 55, 32, - 78, 2, 143, 4, 32, 2, 183, 195, 15, 32, 2, 167, 218, 16, 71, 2, 227, 236, - 12, 78, 2, 177, 182, 15, 2, 32, 77, 20, 192, 1, 2, 50, 32, 22, 54, 242, - 25, 51, 164, 2, 3, 55, 32, 78, 52, 3, 49, 32, 87, 156, 215, 1, 2, 52, 32, - 206, 143, 3, 56, 156, 255, 4, 3, 48, 32, 78, 174, 92, 57, 137, 186, 4, 3, - 53, 32, 75, 2, 159, 216, 16, 77, 2, 195, 254, 9, 32, 20, 184, 1, 3, 52, - 32, 75, 20, 2, 53, 32, 20, 2, 55, 32, 22, 57, 178, 9, 51, 234, 11, 50, - 144, 2, 3, 49, 32, 71, 40, 3, 48, 32, 71, 144, 171, 14, 3, 54, 32, 75, - 149, 84, 3, 56, 32, 70, 2, 251, 222, 16, 80, 2, 251, 236, 16, 70, 2, 199, - 228, 16, 86, 2, 139, 171, 8, 32, 20, 228, 1, 2, 48, 32, 20, 2, 49, 32, - 20, 2, 52, 32, 22, 53, 248, 13, 6, 54, 32, 76, 79, 78, 71, 152, 12, 2, - 57, 32, 160, 167, 3, 3, 51, 32, 72, 132, 252, 8, 4, 50, 32, 78, 71, 164, - 195, 3, 3, 55, 32, 72, 165, 97, 3, 56, 32, 70, 2, 215, 234, 16, 89, 2, - 227, 148, 15, 80, 2, 207, 148, 15, 76, 2, 171, 132, 16, 32, 20, 230, 1, - 53, 164, 14, 4, 48, 32, 78, 71, 172, 8, 3, 51, 32, 71, 244, 243, 11, 2, - 55, 32, 214, 118, 57, 252, 88, 3, 50, 32, 75, 172, 184, 1, 3, 49, 32, 84, - 192, 27, 4, 56, 32, 78, 89, 148, 129, 1, 3, 52, 32, 77, 213, 34, 2, 54, - 32, 2, 155, 207, 16, 32, 196, 1, 118, 48, 154, 2, 49, 210, 2, 50, 170, 2, - 51, 222, 1, 52, 162, 2, 53, 214, 2, 54, 166, 2, 55, 238, 2, 56, 231, 2, - 57, 20, 222, 1, 49, 30, 52, 172, 11, 6, 54, 32, 76, 79, 78, 71, 164, 4, - 6, 53, 32, 76, 79, 78, 71, 24, 2, 51, 32, 162, 229, 4, 50, 230, 152, 2, - 48, 204, 249, 4, 3, 55, 32, 71, 252, 135, 3, 3, 57, 32, 89, 197, 39, 3, - 56, 32, 75, 2, 205, 180, 15, 2, 32, 70, 2, 157, 187, 14, 2, 32, 84, 20, + 82, 22, 83, 34, 84, 38, 85, 142, 197, 1, 67, 214, 249, 12, 77, 2, 78, + 206, 201, 1, 88, 222, 137, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 7, + 26, 73, 235, 239, 14, 69, 2, 187, 213, 1, 71, 110, 68, 8, 67, 65, 80, 73, + 84, 65, 76, 32, 162, 23, 83, 223, 159, 16, 68, 38, 154, 168, 18, 65, 2, + 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 79, + 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 3, 89, 96, 52, 7, 82, 65, 75, + 84, 85, 82, 32, 219, 142, 7, 65, 94, 52, 8, 67, 65, 80, 73, 84, 65, 76, + 32, 143, 21, 83, 42, 134, 166, 18, 65, 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, + 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, + 85, 2, 86, 2, 87, 2, 88, 3, 89, 222, 1, 100, 6, 83, 77, 65, 76, 76, 32, + 222, 7, 67, 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, 82, 243, 5, 84, 104, + 242, 1, 68, 198, 12, 65, 22, 66, 2, 90, 42, 69, 38, 70, 62, 71, 22, 73, + 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 142, 197, + 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, 222, 137, 2, 74, 2, 81, + 2, 86, 2, 87, 3, 89, 9, 52, 7, 79, 84, 76, 69, 83, 83, 32, 175, 232, 14, + 69, 4, 186, 161, 18, 73, 3, 74, 124, 130, 16, 67, 34, 83, 223, 159, 16, + 68, 12, 32, 2, 71, 72, 131, 137, 7, 83, 10, 17, 2, 84, 32, 10, 112, 6, + 87, 72, 73, 84, 69, 32, 170, 244, 5, 68, 234, 106, 65, 241, 230, 4, 9, + 70, 76, 65, 84, 84, 69, 78, 69, 68, 4, 130, 165, 14, 83, 39, 84, 130, 6, + 84, 10, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 133, 14, 6, 67, 82, 73, + 80, 84, 32, 176, 5, 96, 5, 66, 79, 76, 68, 32, 176, 12, 6, 73, 84, 65, + 76, 73, 67, 34, 67, 34, 83, 223, 159, 16, 68, 204, 3, 98, 73, 122, 67, + 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, 82, 30, 83, 214, 5, 84, 215, 160, + 16, 68, 220, 1, 33, 6, 84, 65, 76, 73, 67, 32, 220, 1, 74, 67, 230, 2, + 69, 38, 75, 38, 78, 30, 80, 98, 82, 30, 83, 215, 5, 84, 102, 37, 7, 65, + 80, 73, 84, 65, 76, 32, 102, 250, 1, 84, 186, 4, 65, 22, 66, 2, 90, 22, + 68, 22, 69, 98, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, + 83, 70, 85, 142, 197, 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, + 222, 137, 2, 70, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 9, 40, 4, 72, + 69, 84, 65, 215, 244, 17, 65, 5, 159, 134, 17, 32, 2, 213, 161, 16, 4, + 80, 83, 73, 76, 2, 17, 2, 65, 80, 2, 159, 7, 80, 2, 197, 246, 17, 2, 65, + 66, 6, 74, 72, 172, 149, 5, 8, 65, 82, 84, 73, 65, 76, 32, 68, 167, 239, + 11, 73, 2, 207, 132, 17, 73, 2, 185, 132, 17, 2, 72, 79, 102, 29, 5, 77, + 65, 76, 76, 32, 102, 246, 1, 65, 22, 66, 2, 90, 22, 68, 22, 69, 38, 70, + 62, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, + 38, 85, 142, 197, 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, 222, + 137, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 5, 251, 188, 14, 76, 5, + 207, 240, 17, 69, 5, 247, 219, 14, 69, 7, 186, 212, 1, 80, 131, 190, 16, + 84, 5, 11, 73, 2, 29, 5, 78, 65, 76, 32, 83, 2, 227, 1, 73, 5, 155, 194, + 15, 65, 5, 179, 239, 17, 79, 5, 11, 65, 2, 139, 218, 14, 80, 5, 187, 219, + 14, 65, 7, 11, 77, 4, 246, 170, 1, 73, 243, 165, 16, 69, 9, 174, 255, 17, + 72, 2, 83, 219, 19, 73, 5, 235, 243, 17, 72, 5, 11, 73, 2, 175, 241, 17, + 71, 7, 234, 216, 14, 72, 219, 148, 3, 65, 5, 207, 209, 1, 80, 2, 11, 72, + 2, 11, 69, 2, 11, 84, 2, 167, 254, 16, 65, 104, 11, 32, 104, 18, 67, 35, + 83, 52, 53, 5, 65, 80, 73, 84, 65, 52, 21, 3, 77, 65, 76, 52, 147, 254, + 11, 76, 82, 76, 8, 67, 65, 80, 73, 84, 65, 76, 32, 157, 1, 6, 83, 77, 65, + 76, 76, 32, 36, 254, 143, 18, 65, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, + 78, 2, 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, + 3, 90, 46, 226, 142, 18, 65, 2, 66, 2, 67, 2, 68, 2, 70, 2, 72, 2, 73, 2, + 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, + 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 40, 45, 9, 32, 78, 85, 77, 69, 82, 65, + 76, 32, 40, 70, 69, 66, 70, 70, 78, 26, 83, 66, 84, 170, 155, 16, 90, + 251, 85, 79, 6, 40, 4, 73, 71, 72, 84, 215, 247, 9, 76, 5, 147, 184, 16, + 69, 8, 30, 73, 105, 3, 79, 85, 82, 4, 206, 134, 5, 70, 167, 238, 12, 86, + 4, 65, 3, 73, 78, 69, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 5, 235, + 182, 16, 84, 10, 42, 72, 162, 247, 9, 87, 163, 195, 7, 69, 4, 154, 133, + 5, 73, 207, 149, 11, 82, 252, 8, 234, 1, 65, 192, 4, 9, 67, 72, 65, 78, + 73, 67, 65, 76, 32, 34, 68, 164, 15, 11, 69, 84, 69, 73, 32, 77, 65, 89, + 69, 75, 32, 178, 11, 76, 34, 78, 154, 50, 82, 176, 15, 8, 83, 83, 65, 71, + 69, 32, 87, 65, 20, 2, 84, 82, 235, 135, 17, 77, 26, 72, 6, 83, 85, 82, + 69, 68, 32, 253, 220, 16, 6, 84, 32, 79, 78, 32, 66, 24, 96, 5, 65, 78, + 71, 76, 69, 148, 142, 10, 9, 82, 73, 71, 72, 84, 32, 65, 78, 71, 131, + 231, 7, 66, 21, 11, 32, 18, 212, 1, 39, 87, 73, 84, 72, 32, 79, 80, 69, + 78, 32, 65, 82, 77, 32, 69, 78, 68, 73, 78, 71, 32, 73, 78, 32, 65, 82, + 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 181, 210, 17, 7, 79, + 80, 69, 78, 73, 78, 71, 16, 62, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, + 72, 43, 85, 4, 73, 3, 79, 87, 78, 4, 137, 144, 8, 5, 84, 32, 65, 78, 68, + 4, 11, 80, 4, 225, 227, 11, 3, 32, 65, 78, 4, 142, 133, 17, 65, 215, 56, + 76, 250, 1, 56, 9, 69, 70, 65, 73, 68, 82, 73, 78, 32, 159, 7, 73, 190, + 1, 174, 1, 67, 52, 6, 68, 73, 71, 73, 84, 32, 152, 1, 7, 78, 85, 77, 66, + 69, 82, 32, 114, 83, 164, 146, 7, 12, 69, 88, 67, 76, 65, 77, 65, 84, 73, + 79, 78, 32, 247, 154, 8, 70, 70, 128, 3, 5, 65, 80, 73, 84, 65, 191, 172, + 15, 79, 26, 82, 84, 48, 2, 79, 78, 222, 143, 16, 70, 30, 83, 102, 90, + 238, 85, 78, 235, 110, 69, 8, 44, 3, 72, 82, 69, 217, 128, 17, 2, 87, 79, + 4, 215, 128, 17, 69, 20, 50, 84, 230, 248, 4, 69, 46, 70, 42, 78, 31, 83, + 6, 166, 250, 4, 72, 172, 242, 4, 2, 87, 69, 135, 195, 7, 69, 70, 68, 3, + 77, 65, 76, 157, 206, 9, 8, 89, 77, 66, 79, 76, 32, 65, 73, 68, 45, 9, + 76, 32, 76, 69, 84, 84, 69, 82, 32, 68, 242, 1, 65, 38, 78, 130, 227, 3, + 72, 2, 75, 198, 197, 10, 89, 138, 147, 3, 79, 162, 64, 66, 2, 67, 2, 68, + 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, + 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 3, 90, 7, 146, 243, 3, 84, 255, + 136, 14, 73, 7, 234, 251, 17, 71, 3, 89, 60, 48, 5, 69, 86, 65, 76, 32, + 45, 3, 85, 77, 32, 6, 210, 237, 10, 69, 198, 187, 4, 67, 115, 81, 54, + 210, 1, 66, 50, 70, 164, 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 248, 1, + 11, 77, 65, 84, 72, 69, 77, 65, 84, 73, 67, 65, 22, 83, 84, 4, 84, 72, + 82, 69, 182, 158, 15, 86, 130, 63, 69, 166, 4, 87, 215, 10, 71, 4, 180, + 156, 6, 3, 79, 76, 68, 255, 225, 7, 76, 10, 84, 9, 76, 65, 84, 84, 69, + 78, 69, 68, 32, 184, 3, 3, 79, 85, 82, 243, 222, 15, 73, 4, 44, 3, 76, + 69, 70, 1, 4, 82, 73, 71, 72, 2, 213, 1, 3, 84, 32, 80, 6, 11, 84, 6, 78, + 32, 41, 15, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 78, 71, 76, 69, + 4, 36, 5, 67, 85, 82, 76, 89, 59, 80, 2, 17, 2, 32, 66, 2, 137, 183, 7, + 4, 82, 65, 67, 75, 2, 169, 178, 17, 10, 65, 82, 69, 78, 84, 72, 69, 83, + 73, 83, 2, 139, 183, 17, 76, 10, 212, 152, 6, 4, 77, 65, 76, 76, 250, + 229, 8, 72, 212, 95, 2, 73, 88, 183, 2, 65, 4, 11, 69, 4, 45, 9, 32, 80, + 79, 73, 78, 84, 69, 68, 32, 4, 162, 202, 9, 80, 219, 147, 6, 66, 158, 1, + 146, 1, 65, 104, 6, 67, 72, 69, 73, 75, 72, 34, 76, 144, 6, 8, 83, 89, + 76, 76, 65, 66, 76, 69, 0, 4, 87, 79, 82, 68, 50, 86, 139, 250, 15, 68, + 6, 80, 8, 72, 65, 78, 71, 32, 75, 72, 85, 164, 6, 3, 80, 85, 78, 147, + 213, 13, 78, 2, 159, 183, 16, 68, 4, 166, 162, 17, 65, 139, 60, 69, 94, + 52, 6, 69, 84, 84, 69, 82, 32, 185, 5, 2, 85, 77, 92, 250, 1, 66, 36, 2, + 67, 72, 38, 68, 50, 71, 38, 74, 34, 75, 42, 78, 62, 80, 30, 83, 42, 84, + 54, 73, 0, 3, 76, 65, 73, 0, 3, 77, 73, 84, 138, 247, 12, 72, 166, 10, + 82, 2, 87, 232, 181, 2, 2, 65, 84, 182, 220, 1, 89, 246, 8, 85, 226, 79, + 69, 3, 79, 4, 182, 180, 16, 72, 255, 186, 1, 65, 4, 218, 229, 16, 73, + 183, 137, 1, 65, 8, 238, 165, 10, 72, 202, 191, 6, 73, 247, 65, 68, 4, + 190, 165, 10, 72, 231, 197, 7, 79, 4, 154, 179, 16, 72, 203, 49, 73, 6, + 216, 1, 2, 79, 75, 163, 163, 10, 72, 12, 178, 1, 65, 0, 3, 71, 79, 85, + 230, 233, 17, 78, 3, 89, 6, 118, 65, 163, 177, 16, 72, 6, 158, 220, 17, + 65, 162, 14, 72, 3, 83, 10, 48, 2, 73, 76, 162, 163, 10, 72, 191, 129, 7, + 84, 5, 181, 177, 1, 4, 32, 76, 79, 78, 2, 209, 232, 17, 3, 32, 73, 89, 2, + 173, 130, 17, 7, 32, 82, 69, 80, 69, 84, 73, 30, 64, 10, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 143, 178, 15, 73, 28, 130, 1, 65, 44, 4, 67, 72, + 69, 73, 2, 79, 0, 3, 83, 79, 85, 0, 2, 89, 69, 22, 73, 38, 85, 194, 131, + 11, 78, 151, 175, 4, 86, 8, 242, 216, 16, 78, 182, 80, 65, 187, 64, 85, + 2, 199, 216, 16, 78, 4, 178, 216, 16, 78, 239, 144, 1, 73, 4, 142, 216, + 16, 78, 239, 144, 1, 85, 4, 182, 224, 16, 84, 195, 56, 79, 178, 3, 160, + 1, 11, 68, 69, 32, 75, 73, 75, 65, 75, 85, 73, 32, 140, 211, 16, 20, 79, + 82, 65, 72, 32, 87, 73, 84, 72, 32, 78, 73, 78, 69, 32, 66, 82, 65, 78, + 67, 95, 83, 174, 3, 148, 1, 17, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, + 78, 85, 77, 66, 69, 82, 32, 164, 1, 10, 83, 89, 76, 76, 65, 66, 76, 69, + 32, 77, 155, 167, 9, 68, 14, 62, 84, 56, 7, 72, 85, 78, 68, 82, 69, 68, + 135, 186, 4, 77, 8, 26, 69, 207, 186, 4, 72, 6, 26, 78, 167, 161, 14, 69, + 4, 160, 186, 4, 2, 32, 84, 219, 170, 13, 83, 142, 3, 22, 48, 163, 21, 49, + 198, 1, 118, 48, 250, 1, 49, 210, 1, 50, 138, 2, 51, 254, 1, 52, 150, 2, + 53, 158, 2, 54, 242, 1, 55, 134, 2, 56, 187, 2, 57, 18, 142, 1, 49, 34, + 50, 22, 51, 22, 52, 254, 150, 2, 53, 154, 240, 3, 56, 156, 218, 10, 3, + 57, 32, 77, 92, 3, 55, 32, 77, 237, 90, 3, 54, 32, 87, 2, 11, 32, 2, 195, + 206, 17, 75, 2, 159, 217, 17, 32, 2, 215, 148, 12, 32, 2, 11, 32, 2, 251, + 205, 17, 87, 20, 130, 1, 49, 22, 54, 18, 56, 22, 57, 248, 28, 2, 48, 32, + 198, 183, 7, 53, 170, 203, 5, 52, 198, 10, 55, 222, 9, 50, 243, 235, 3, + 51, 2, 147, 179, 17, 32, 2, 167, 25, 32, 2, 163, 204, 13, 32, 2, 227, + 245, 12, 32, 20, 106, 49, 22, 50, 22, 51, 22, 52, 22, 53, 22, 54, 22, 55, + 22, 57, 130, 224, 11, 48, 145, 231, 1, 2, 56, 32, 2, 211, 169, 10, 32, 2, + 247, 128, 10, 32, 2, 143, 186, 17, 32, 2, 247, 142, 12, 32, 2, 167, 227, + 12, 32, 2, 135, 188, 17, 32, 2, 179, 221, 12, 32, 2, 239, 28, 32, 20, + 174, 1, 48, 16, 2, 49, 32, 20, 2, 52, 32, 20, 2, 56, 32, 246, 194, 4, 57, + 182, 51, 50, 22, 53, 136, 230, 7, 2, 54, 32, 232, 255, 3, 3, 55, 32, 78, + 237, 90, 3, 51, 32, 89, 2, 179, 37, 32, 2, 207, 200, 17, 89, 2, 187, 200, + 17, 70, 2, 255, 145, 16, 78, 20, 184, 1, 2, 48, 32, 20, 2, 51, 32, 30, + 53, 22, 56, 160, 8, 3, 52, 32, 75, 128, 22, 3, 54, 32, 72, 222, 2, 55, + 138, 213, 4, 49, 208, 179, 4, 3, 57, 32, 87, 241, 232, 5, 3, 50, 32, 72, + 2, 247, 234, 15, 72, 2, 213, 151, 17, 2, 78, 71, 2, 191, 162, 11, 32, 2, + 235, 186, 17, 32, 20, 178, 1, 48, 18, 53, 20, 2, 54, 32, 20, 2, 56, 32, + 22, 57, 136, 213, 9, 2, 50, 32, 236, 4, 2, 51, 32, 170, 215, 1, 49, 148, + 223, 3, 3, 52, 32, 76, 137, 143, 1, 3, 55, 32, 78, 2, 159, 4, 32, 2, 155, + 145, 16, 32, 2, 215, 170, 17, 71, 2, 247, 144, 13, 78, 2, 237, 131, 16, + 2, 32, 77, 20, 196, 1, 2, 50, 32, 22, 54, 238, 25, 51, 168, 2, 3, 55, 32, + 78, 140, 216, 1, 2, 52, 32, 234, 150, 3, 56, 248, 144, 5, 3, 48, 32, 78, + 172, 89, 3, 49, 32, 87, 158, 13, 57, 225, 227, 4, 3, 53, 32, 75, 2, 203, + 168, 17, 77, 2, 247, 151, 10, 32, 20, 196, 1, 3, 52, 32, 75, 20, 2, 53, + 32, 22, 57, 232, 7, 3, 49, 32, 71, 234, 1, 51, 236, 6, 3, 48, 32, 71, + 146, 5, 50, 236, 224, 10, 2, 55, 32, 252, 149, 4, 3, 54, 32, 75, 241, 87, + 3, 56, 32, 70, 2, 155, 175, 17, 80, 2, 155, 189, 17, 70, 2, 187, 243, 10, + 32, 20, 228, 1, 2, 48, 32, 20, 2, 49, 32, 20, 2, 52, 32, 22, 53, 172, 14, + 6, 54, 32, 76, 79, 78, 71, 224, 11, 2, 57, 32, 168, 173, 3, 3, 51, 32, + 72, 168, 154, 9, 4, 50, 32, 78, 71, 152, 239, 3, 3, 55, 32, 72, 189, 97, + 3, 56, 32, 70, 2, 139, 187, 17, 89, 2, 147, 226, 15, 80, 2, 255, 225, 15, + 76, 2, 207, 212, 16, 32, 20, 230, 1, 53, 216, 14, 4, 48, 32, 78, 71, 244, + 7, 3, 51, 32, 71, 192, 152, 12, 2, 55, 32, 138, 118, 57, 192, 112, 3, 50, + 32, 75, 156, 202, 1, 3, 49, 32, 84, 208, 27, 4, 56, 32, 78, 89, 252, 131, + 1, 3, 52, 32, 77, 225, 34, 2, 54, 32, 2, 207, 159, 17, 32, 200, 1, 118, + 48, 186, 2, 49, 210, 2, 50, 166, 2, 51, 222, 1, 52, 186, 2, 53, 214, 2, + 54, 166, 2, 55, 190, 2, 56, 223, 2, 57, 20, 222, 1, 49, 30, 52, 28, 7, + 53, 32, 76, 79, 78, 71, 32, 12, 2, 51, 32, 184, 11, 6, 54, 32, 76, 79, + 78, 71, 138, 241, 4, 50, 198, 168, 2, 48, 212, 134, 5, 3, 55, 32, 71, + 228, 176, 3, 3, 57, 32, 89, 253, 39, 3, 56, 32, 75, 2, 181, 130, 16, 2, + 32, 70, 2, 241, 132, 15, 2, 32, 84, 2, 11, 77, 2, 203, 132, 15, 66, 20, 208, 1, 6, 48, 32, 76, 79, 78, 71, 22, 51, 34, 54, 22, 56, 32, 2, 57, 32, - 156, 18, 4, 53, 32, 78, 71, 224, 206, 2, 2, 55, 32, 192, 145, 2, 3, 50, - 32, 75, 140, 198, 9, 3, 52, 32, 87, 145, 254, 1, 2, 49, 32, 2, 211, 216, - 15, 32, 2, 11, 32, 2, 131, 227, 16, 74, 2, 207, 240, 15, 32, 2, 11, 32, - 2, 207, 226, 16, 87, 2, 155, 236, 14, 78, 20, 230, 1, 50, 32, 2, 51, 32, - 232, 3, 3, 52, 32, 71, 182, 10, 48, 220, 147, 7, 2, 57, 32, 176, 196, 2, - 4, 56, 32, 72, 79, 248, 207, 2, 5, 55, 32, 78, 71, 71, 236, 90, 2, 53, - 32, 180, 249, 1, 3, 54, 32, 87, 129, 148, 1, 2, 49, 32, 2, 11, 32, 2, - 203, 247, 12, 77, 2, 11, 78, 2, 143, 227, 16, 68, 20, 172, 8, 3, 52, 32, - 78, 204, 2, 2, 56, 32, 224, 233, 4, 3, 50, 32, 75, 194, 195, 2, 49, 2, - 53, 216, 173, 2, 2, 55, 32, 152, 208, 4, 3, 51, 32, 70, 0, 3, 54, 32, 83, - 164, 50, 2, 48, 32, 225, 145, 1, 3, 57, 32, 87, 20, 232, 1, 3, 53, 32, - 77, 22, 54, 224, 2, 3, 57, 32, 78, 172, 4, 6, 50, 32, 76, 79, 78, 71, - 130, 196, 6, 55, 140, 151, 3, 3, 51, 32, 87, 200, 212, 2, 2, 48, 32, 220, - 60, 3, 56, 32, 71, 244, 190, 1, 3, 49, 32, 89, 1, 3, 52, 32, 86, 2, 179, - 223, 16, 66, 2, 129, 191, 15, 3, 32, 78, 71, 20, 192, 1, 2, 50, 32, 34, - 52, 26, 53, 34, 54, 36, 2, 55, 32, 244, 7, 2, 48, 32, 180, 225, 9, 3, 56, - 32, 75, 148, 182, 2, 2, 49, 32, 148, 225, 2, 5, 57, 32, 78, 71, 71, 249, - 204, 1, 2, 51, 32, 2, 11, 78, 2, 207, 238, 16, 74, 2, 213, 5, 2, 32, 77, - 2, 11, 32, 2, 247, 220, 16, 71, 2, 241, 175, 14, 4, 32, 78, 71, 71, 2, - 227, 131, 15, 74, 20, 220, 1, 2, 48, 32, 20, 6, 49, 32, 76, 79, 78, 71, - 30, 56, 204, 2, 2, 55, 32, 128, 170, 9, 3, 52, 32, 78, 200, 48, 4, 54, - 32, 71, 85, 152, 208, 4, 2, 53, 32, 148, 84, 3, 50, 32, 83, 0, 2, 51, 32, - 185, 98, 2, 57, 32, 2, 219, 173, 14, 74, 2, 221, 249, 11, 2, 32, 77, 2, - 183, 170, 12, 32, 20, 198, 1, 50, 46, 52, 40, 6, 53, 32, 76, 79, 78, 71, - 28, 3, 55, 32, 78, 34, 56, 152, 199, 11, 2, 54, 32, 156, 99, 3, 57, 32, - 75, 160, 212, 2, 3, 51, 32, 86, 156, 108, 4, 48, 32, 78, 89, 207, 53, 49, - 2, 11, 32, 2, 11, 77, 2, 183, 171, 14, 66, 2, 17, 2, 32, 77, 2, 227, 204, - 16, 66, 2, 205, 204, 16, 2, 32, 74, 2, 11, 71, 2, 231, 163, 15, 85, 2, - 147, 154, 15, 32, 20, 208, 1, 2, 48, 32, 22, 49, 22, 50, 32, 6, 51, 32, - 76, 79, 78, 71, 34, 56, 252, 241, 8, 2, 53, 32, 152, 126, 3, 52, 32, 78, - 188, 182, 2, 2, 54, 32, 228, 210, 2, 4, 55, 32, 77, 66, 145, 30, 4, 57, - 32, 77, 85, 2, 215, 168, 14, 68, 2, 215, 238, 9, 32, 2, 11, 32, 2, 147, - 213, 16, 86, 2, 253, 222, 14, 3, 32, 78, 71, 2, 17, 2, 32, 77, 2, 239, - 251, 14, 66, 16, 142, 1, 48, 32, 3, 49, 32, 78, 20, 3, 50, 32, 78, 20, 2, - 51, 32, 20, 3, 52, 32, 87, 22, 53, 20, 2, 54, 32, 169, 165, 12, 3, 55, - 32, 70, 2, 11, 32, 2, 191, 250, 14, 71, 2, 171, 250, 14, 68, 2, 227, 193, - 16, 74, 2, 215, 150, 16, 72, 2, 203, 210, 16, 85, 2, 219, 158, 15, 32, 2, - 227, 144, 1, 83, 248, 1, 68, 6, 79, 73, 84, 73, 67, 32, 250, 145, 10, 80, - 241, 70, 2, 67, 85, 244, 1, 104, 8, 67, 85, 82, 83, 73, 86, 69, 32, 221, - 10, 13, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 73, 67, 32, 180, 1, 96, - 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 250, 2, 76, 133, 3, 7, 78, 85, 77, - 66, 69, 82, 32, 24, 78, 69, 50, 70, 44, 4, 79, 78, 69, 32, 46, 83, 50, - 84, 61, 3, 78, 73, 78, 4, 160, 1, 2, 76, 69, 93, 4, 73, 71, 72, 84, 4, - 192, 1, 2, 73, 86, 13, 3, 79, 85, 82, 4, 232, 225, 14, 4, 84, 87, 69, 76, - 19, 72, 4, 26, 69, 93, 2, 73, 88, 2, 65, 2, 86, 69, 6, 46, 69, 12, 3, 72, - 82, 69, 13, 2, 87, 79, 2, 23, 78, 2, 11, 69, 2, 197, 227, 14, 5, 32, 84, - 87, 69, 76, 52, 76, 6, 69, 84, 84, 69, 82, 32, 137, 2, 8, 79, 71, 79, 71, - 82, 65, 77, 32, 48, 182, 1, 65, 46, 84, 146, 234, 1, 78, 2, 83, 206, 252, - 12, 72, 138, 176, 1, 75, 254, 68, 66, 2, 68, 2, 76, 2, 77, 2, 80, 2, 81, - 2, 82, 2, 87, 2, 89, 186, 2, 69, 2, 73, 3, 79, 5, 149, 139, 11, 6, 82, - 67, 72, 65, 73, 67, 6, 150, 222, 16, 65, 2, 69, 3, 79, 4, 174, 201, 4, - 73, 153, 248, 11, 2, 82, 77, 104, 92, 5, 69, 73, 71, 72, 84, 30, 70, 92, - 4, 78, 73, 78, 69, 54, 83, 78, 84, 73, 2, 79, 78, 11, 150, 1, 89, 223, 1, - 32, 24, 18, 73, 35, 79, 12, 142, 2, 86, 139, 228, 3, 70, 12, 148, 2, 2, - 85, 82, 227, 227, 3, 82, 11, 28, 2, 84, 89, 223, 1, 32, 2, 231, 141, 6, - 32, 24, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 13, 154, 1, 32, 227, 227, 3, - 84, 28, 34, 72, 50, 87, 159, 140, 6, 69, 12, 32, 2, 82, 69, 215, 227, 3, - 73, 8, 39, 69, 12, 26, 79, 215, 227, 3, 69, 9, 11, 32, 6, 194, 138, 6, - 72, 171, 167, 5, 84, 64, 96, 7, 76, 69, 84, 84, 69, 82, 32, 229, 160, 3, - 11, 83, 89, 77, 66, 79, 76, 32, 86, 73, 68, 74, 60, 174, 1, 66, 2, 82, - 22, 78, 30, 83, 38, 84, 174, 223, 14, 72, 138, 176, 1, 75, 254, 68, 68, - 2, 76, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, - 79, 4, 179, 159, 3, 65, 8, 158, 159, 3, 65, 3, 69, 6, 130, 159, 3, 65, - 139, 184, 13, 69, 10, 222, 158, 3, 65, 2, 69, 139, 184, 13, 79, 2, 175, - 252, 11, 73, 20, 44, 5, 73, 67, 65, 76, 32, 251, 213, 16, 79, 18, 116, - 10, 76, 79, 78, 71, 32, 79, 86, 69, 82, 32, 74, 80, 26, 84, 168, 1, 7, - 83, 72, 79, 82, 84, 32, 79, 187, 32, 66, 4, 188, 244, 2, 2, 83, 72, 185, - 180, 10, 7, 84, 87, 79, 32, 83, 72, 79, 2, 109, 3, 69, 78, 84, 8, 66, 69, - 34, 82, 41, 10, 87, 79, 32, 83, 72, 79, 82, 84, 83, 32, 2, 17, 2, 84, 82, - 2, 23, 65, 2, 11, 73, 2, 217, 187, 15, 2, 83, 69, 4, 26, 79, 179, 231, 7, - 74, 2, 137, 162, 10, 4, 86, 69, 82, 32, 224, 2, 100, 3, 65, 79, 32, 136, - 18, 2, 67, 82, 142, 1, 68, 118, 76, 130, 1, 78, 245, 2, 4, 82, 82, 79, - 82, 170, 2, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 180, 10, 5, 83, 73, - 71, 78, 32, 212, 1, 5, 84, 79, 78, 69, 32, 69, 11, 86, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 178, 1, 214, 1, 65, 110, 66, 34, 68, 106, 71, 34, - 76, 50, 78, 106, 82, 170, 1, 83, 54, 84, 218, 1, 86, 32, 3, 89, 73, 32, - 118, 90, 238, 247, 12, 81, 230, 200, 1, 80, 242, 190, 1, 72, 2, 77, 254, - 68, 70, 2, 75, 2, 87, 3, 88, 10, 52, 7, 82, 67, 72, 65, 73, 67, 32, 179, - 206, 16, 72, 8, 254, 231, 8, 90, 170, 151, 4, 78, 211, 204, 3, 77, 4, - 162, 186, 16, 82, 219, 19, 65, 16, 50, 90, 154, 4, 76, 218, 198, 16, 68, - 187, 2, 65, 8, 206, 185, 16, 89, 162, 17, 72, 2, 90, 187, 2, 65, 6, 190, - 133, 16, 72, 183, 71, 65, 8, 238, 150, 10, 72, 174, 179, 6, 89, 187, 2, - 65, 18, 54, 65, 186, 132, 16, 71, 2, 78, 2, 89, 255, 68, 72, 5, 11, 83, - 2, 169, 159, 13, 4, 65, 76, 73, 90, 16, 60, 9, 69, 70, 79, 82, 77, 69, - 68, 32, 84, 171, 180, 16, 84, 14, 40, 4, 79, 78, 69, 45, 151, 211, 14, - 83, 12, 206, 202, 16, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 56, 8, 186, 182, - 16, 89, 162, 17, 72, 2, 83, 187, 2, 65, 32, 78, 76, 20, 4, 79, 78, 69, - 45, 70, 83, 130, 198, 16, 84, 186, 2, 65, 3, 69, 4, 171, 147, 10, 72, 14, - 250, 200, 16, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 8, 254, 197, - 16, 72, 2, 83, 186, 2, 65, 3, 69, 4, 206, 197, 16, 70, 187, 2, 65, 16, - 70, 84, 228, 207, 14, 2, 68, 90, 250, 60, 78, 142, 184, 1, 75, 3, 80, 8, - 234, 255, 15, 83, 254, 68, 84, 187, 2, 65, 16, 50, 90, 142, 255, 15, 83, - 254, 68, 72, 187, 2, 65, 8, 218, 144, 10, 83, 174, 179, 6, 89, 187, 2, - 65, 8, 144, 1, 9, 82, 69, 70, 79, 82, 77, 69, 68, 32, 34, 65, 145, 245, - 13, 18, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 73, - 69, 82, 4, 30, 65, 189, 88, 2, 86, 79, 2, 225, 230, 11, 3, 83, 80, 73, 8, - 130, 172, 14, 66, 168, 74, 3, 84, 79, 80, 182, 81, 65, 143, 82, 82, 104, - 146, 1, 65, 78, 69, 62, 73, 78, 79, 74, 85, 74, 89, 180, 250, 6, 9, 82, - 79, 85, 78, 68, 69, 68, 32, 69, 206, 134, 7, 87, 238, 70, 78, 135, 122, - 86, 19, 206, 202, 14, 78, 130, 184, 1, 69, 134, 63, 72, 146, 1, 65, 2, - 73, 3, 85, 15, 242, 222, 12, 82, 146, 235, 1, 78, 150, 248, 1, 65, 3, 73, - 23, 186, 201, 14, 65, 54, 79, 154, 247, 1, 78, 86, 69, 2, 71, 2, 73, 3, - 85, 13, 42, 69, 230, 192, 16, 71, 2, 79, 3, 85, 4, 226, 192, 16, 82, 3, - 89, 19, 166, 200, 14, 65, 202, 228, 1, 69, 134, 19, 78, 2, 79, 86, 73, 3, - 85, 7, 166, 172, 16, 85, 219, 19, 73, 12, 18, 32, 63, 79, 4, 172, 172, - 15, 4, 79, 78, 32, 85, 13, 4, 68, 65, 83, 72, 8, 190, 189, 11, 83, 158, - 216, 3, 80, 218, 34, 32, 139, 112, 66, 6, 64, 4, 68, 76, 69, 32, 173, - 215, 4, 6, 76, 73, 78, 69, 32, 72, 4, 152, 180, 5, 3, 84, 72, 73, 203, - 185, 10, 68, 8, 76, 6, 73, 84, 65, 82, 89, 32, 172, 153, 10, 3, 75, 89, - 32, 255, 156, 5, 76, 4, 26, 72, 187, 134, 12, 77, 2, 163, 170, 4, 69, 24, - 42, 73, 76, 2, 85, 83, 235, 187, 16, 89, 6, 34, 68, 22, 77, 147, 185, 14, - 66, 2, 195, 137, 6, 73, 2, 187, 212, 6, 73, 16, 46, 32, 201, 142, 10, 5, - 45, 79, 82, 45, 80, 14, 40, 4, 83, 73, 71, 78, 231, 249, 12, 84, 13, 11, - 32, 10, 44, 5, 87, 73, 84, 72, 32, 239, 153, 6, 73, 8, 66, 67, 182, 195, - 4, 68, 206, 135, 8, 82, 21, 4, 70, 65, 76, 76, 2, 129, 195, 11, 3, 79, - 77, 77, 5, 163, 181, 14, 32, 182, 9, 184, 1, 10, 66, 73, 76, 69, 32, 80, - 72, 79, 78, 69, 166, 1, 68, 190, 76, 78, 174, 27, 79, 156, 1, 3, 83, 81, - 85, 38, 84, 250, 1, 85, 242, 140, 11, 89, 217, 178, 2, 5, 86, 73, 69, 32, + 248, 17, 4, 53, 32, 78, 71, 204, 208, 2, 2, 55, 32, 184, 151, 2, 3, 50, + 32, 75, 128, 136, 10, 3, 52, 32, 87, 229, 132, 2, 2, 49, 32, 2, 227, 165, + 16, 32, 2, 11, 32, 2, 151, 179, 17, 74, 2, 203, 192, 16, 32, 2, 11, 32, + 2, 227, 178, 17, 87, 2, 151, 185, 15, 78, 20, 226, 1, 50, 32, 2, 51, 32, + 128, 4, 3, 52, 32, 71, 254, 9, 48, 148, 240, 9, 4, 56, 32, 72, 79, 156, + 220, 2, 5, 55, 32, 78, 71, 71, 224, 88, 2, 53, 32, 220, 55, 2, 57, 32, + 236, 236, 1, 3, 54, 32, 87, 245, 150, 1, 2, 49, 32, 2, 11, 32, 2, 171, + 155, 13, 77, 2, 11, 78, 2, 167, 179, 17, 68, 20, 200, 3, 2, 56, 32, 252, + 4, 3, 52, 32, 78, 216, 243, 4, 3, 50, 32, 75, 210, 210, 2, 49, 2, 53, + 236, 174, 2, 2, 55, 32, 232, 129, 5, 3, 51, 32, 70, 0, 3, 54, 32, 83, + 236, 53, 2, 48, 32, 233, 148, 1, 3, 57, 32, 87, 20, 236, 1, 8, 50, 32, + 76, 79, 78, 71, 32, 77, 20, 3, 53, 32, 77, 22, 54, 224, 2, 3, 57, 32, 78, + 174, 219, 6, 55, 220, 155, 3, 3, 51, 32, 87, 236, 224, 2, 2, 48, 32, 176, + 60, 3, 56, 32, 71, 204, 228, 1, 3, 49, 32, 89, 1, 3, 52, 32, 86, 2, 171, + 164, 17, 66, 2, 179, 175, 17, 66, 2, 213, 141, 16, 3, 32, 78, 71, 20, + 192, 1, 2, 50, 32, 34, 52, 26, 53, 34, 54, 36, 2, 55, 32, 188, 7, 2, 48, + 32, 244, 250, 9, 3, 56, 32, 75, 128, 193, 2, 2, 49, 32, 156, 138, 3, 5, + 57, 32, 78, 71, 71, 253, 207, 1, 2, 51, 32, 2, 11, 78, 2, 207, 190, 17, + 74, 2, 165, 5, 2, 32, 77, 2, 11, 32, 2, 247, 172, 17, 71, 2, 145, 249, + 14, 4, 32, 78, 71, 71, 2, 223, 208, 15, 74, 20, 220, 1, 2, 48, 32, 20, 6, + 49, 32, 76, 79, 78, 71, 30, 56, 156, 2, 2, 55, 32, 220, 192, 9, 3, 52, + 32, 78, 236, 49, 4, 54, 32, 71, 85, 232, 129, 5, 2, 53, 32, 240, 87, 3, + 50, 32, 83, 0, 2, 51, 32, 165, 101, 2, 57, 32, 2, 251, 246, 14, 74, 2, + 241, 157, 12, 2, 32, 77, 2, 171, 206, 12, 32, 24, 198, 1, 50, 2, 52, 36, + 6, 53, 32, 76, 79, 78, 71, 28, 3, 55, 32, 78, 34, 56, 184, 236, 11, 2, + 54, 32, 160, 98, 3, 57, 32, 75, 168, 253, 2, 3, 51, 32, 86, 136, 111, 4, + 48, 32, 78, 89, 219, 53, 49, 4, 181, 246, 14, 4, 32, 77, 66, 79, 2, 253, + 156, 17, 2, 32, 74, 2, 11, 71, 2, 203, 241, 15, 85, 2, 207, 231, 15, 32, + 20, 212, 1, 2, 48, 32, 22, 49, 22, 50, 20, 6, 51, 32, 76, 79, 78, 71, 34, + 56, 216, 136, 9, 2, 53, 32, 252, 128, 1, 3, 52, 32, 78, 168, 193, 2, 2, + 54, 32, 236, 251, 2, 4, 55, 32, 77, 66, 161, 30, 4, 57, 32, 77, 85, 2, + 163, 242, 14, 68, 2, 139, 136, 10, 32, 2, 239, 216, 10, 32, 2, 165, 172, + 15, 3, 32, 78, 71, 2, 17, 2, 32, 77, 2, 163, 201, 15, 66, 16, 142, 1, 48, + 32, 3, 49, 32, 78, 20, 3, 50, 32, 78, 20, 2, 51, 32, 20, 3, 52, 32, 87, + 22, 53, 20, 2, 54, 32, 213, 201, 12, 3, 55, 32, 70, 2, 11, 32, 2, 243, + 199, 15, 71, 2, 223, 199, 15, 68, 2, 155, 146, 17, 74, 2, 131, 231, 16, + 72, 2, 131, 163, 17, 85, 2, 199, 236, 15, 32, 2, 255, 144, 1, 83, 248, 1, + 72, 6, 79, 73, 84, 73, 67, 32, 204, 254, 10, 2, 67, 85, 179, 183, 2, 80, + 244, 1, 104, 8, 67, 85, 82, 83, 73, 86, 69, 32, 221, 10, 13, 72, 73, 69, + 82, 79, 71, 76, 89, 80, 72, 73, 67, 32, 180, 1, 96, 9, 70, 82, 65, 67, + 84, 73, 79, 78, 32, 250, 2, 76, 133, 3, 7, 78, 85, 77, 66, 69, 82, 32, + 24, 78, 69, 50, 70, 44, 4, 79, 78, 69, 32, 46, 83, 50, 84, 61, 3, 78, 73, + 78, 4, 160, 1, 2, 76, 69, 93, 4, 73, 71, 72, 84, 4, 192, 1, 2, 73, 86, + 13, 3, 79, 85, 82, 4, 140, 175, 15, 4, 84, 87, 69, 76, 19, 72, 4, 26, 69, + 93, 2, 73, 88, 2, 65, 2, 86, 69, 6, 46, 69, 12, 3, 72, 82, 69, 13, 2, 87, + 79, 2, 23, 78, 2, 11, 69, 2, 237, 176, 15, 5, 32, 84, 87, 69, 76, 52, 76, + 6, 69, 84, 84, 69, 82, 32, 137, 2, 8, 79, 71, 79, 71, 82, 65, 77, 32, 48, + 182, 1, 65, 46, 84, 182, 235, 1, 78, 2, 83, 218, 200, 13, 72, 130, 179, + 1, 75, 138, 69, 66, 2, 68, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, 87, 2, + 89, 186, 2, 69, 2, 73, 3, 79, 5, 253, 176, 11, 6, 82, 67, 72, 65, 73, 67, + 6, 202, 174, 17, 65, 2, 69, 3, 79, 4, 138, 209, 4, 73, 241, 192, 12, 2, + 82, 77, 104, 92, 5, 69, 73, 71, 72, 84, 30, 70, 92, 4, 78, 73, 78, 69, + 54, 83, 78, 84, 73, 2, 79, 78, 11, 150, 1, 89, 223, 1, 32, 24, 18, 73, + 35, 79, 12, 142, 2, 86, 207, 235, 3, 70, 12, 148, 2, 2, 85, 82, 167, 235, + 3, 82, 11, 28, 2, 84, 89, 223, 1, 32, 2, 203, 160, 6, 32, 24, 40, 4, 69, + 86, 69, 78, 1, 2, 73, 88, 13, 154, 1, 32, 167, 235, 3, 84, 28, 34, 72, + 50, 87, 131, 159, 6, 69, 12, 32, 2, 82, 69, 155, 235, 3, 73, 8, 39, 69, + 12, 26, 79, 155, 235, 3, 69, 9, 11, 32, 6, 166, 157, 6, 72, 235, 185, 5, + 84, 64, 96, 7, 76, 69, 84, 84, 69, 82, 32, 141, 167, 3, 11, 83, 89, 77, + 66, 79, 76, 32, 86, 73, 68, 74, 60, 174, 1, 66, 2, 82, 22, 78, 30, 83, + 38, 84, 222, 172, 15, 72, 130, 179, 1, 75, 138, 69, 68, 2, 76, 2, 77, 2, + 80, 2, 81, 2, 87, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 79, 4, 219, 165, 3, + 65, 8, 198, 165, 3, 65, 3, 69, 6, 170, 165, 3, 65, 151, 130, 14, 69, 10, + 134, 165, 3, 65, 2, 69, 151, 130, 14, 79, 2, 215, 160, 12, 73, 20, 44, 5, + 73, 67, 65, 76, 32, 175, 166, 17, 79, 18, 116, 10, 76, 79, 78, 71, 32, + 79, 86, 69, 82, 32, 74, 80, 26, 84, 168, 1, 7, 83, 72, 79, 82, 84, 32, + 79, 215, 32, 66, 4, 156, 246, 2, 2, 83, 72, 201, 237, 10, 7, 84, 87, 79, + 32, 83, 72, 79, 2, 109, 3, 69, 78, 84, 8, 66, 69, 34, 82, 41, 10, 87, 79, + 32, 83, 72, 79, 82, 84, 83, 32, 2, 17, 2, 84, 82, 2, 23, 65, 2, 11, 73, + 2, 245, 139, 16, 2, 83, 69, 4, 26, 79, 163, 254, 7, 74, 2, 129, 190, 10, + 4, 86, 69, 82, 32, 230, 2, 104, 3, 65, 79, 32, 136, 18, 2, 67, 82, 142, + 1, 68, 142, 1, 76, 130, 1, 78, 245, 2, 4, 82, 82, 79, 82, 170, 2, 156, 1, + 7, 76, 69, 84, 84, 69, 82, 32, 180, 10, 5, 83, 73, 71, 78, 32, 212, 1, 5, + 84, 79, 78, 69, 32, 69, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 178, 1, 214, 1, 65, 110, 66, 34, 68, 106, 71, 34, 76, 50, 78, 106, 82, + 170, 1, 83, 54, 84, 218, 1, 86, 32, 3, 89, 73, 32, 118, 90, 246, 165, 13, + 81, 254, 231, 1, 80, 246, 193, 1, 72, 2, 77, 138, 69, 70, 2, 75, 2, 87, + 3, 88, 10, 52, 7, 82, 67, 72, 65, 73, 67, 32, 227, 158, 17, 72, 8, 210, + 254, 8, 90, 222, 174, 4, 78, 251, 238, 3, 77, 4, 210, 138, 17, 82, 219, + 19, 65, 16, 50, 90, 154, 4, 76, 138, 151, 17, 68, 187, 2, 65, 8, 254, + 137, 17, 89, 162, 17, 72, 2, 90, 187, 2, 65, 6, 226, 213, 16, 72, 195, + 71, 65, 8, 222, 178, 10, 72, 238, 231, 6, 89, 187, 2, 65, 18, 54, 65, + 222, 212, 16, 71, 2, 78, 2, 89, 139, 69, 72, 5, 11, 83, 2, 149, 218, 13, + 4, 65, 76, 73, 90, 16, 60, 9, 69, 70, 79, 82, 77, 69, 68, 32, 84, 219, + 132, 17, 84, 14, 40, 4, 79, 78, 69, 45, 195, 160, 15, 83, 12, 254, 154, + 17, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 56, 8, 234, 134, 17, 89, 162, 17, + 72, 2, 83, 187, 2, 65, 32, 78, 76, 20, 4, 79, 78, 69, 45, 70, 83, 178, + 150, 17, 84, 186, 2, 65, 3, 69, 4, 155, 175, 10, 72, 14, 170, 153, 17, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 8, 174, 150, 17, 72, 2, 83, + 186, 2, 65, 3, 69, 4, 254, 149, 17, 70, 187, 2, 65, 16, 70, 84, 144, 157, + 15, 2, 68, 90, 190, 62, 78, 206, 185, 1, 75, 3, 80, 8, 142, 208, 16, 83, + 138, 69, 84, 187, 2, 65, 16, 50, 90, 178, 207, 16, 83, 138, 69, 72, 187, + 2, 65, 8, 202, 172, 10, 83, 238, 231, 6, 89, 187, 2, 65, 8, 144, 1, 9, + 82, 69, 70, 79, 82, 77, 69, 68, 32, 34, 65, 189, 243, 15, 18, 67, 79, 78, + 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 73, 69, 82, 4, 30, 65, + 213, 88, 2, 86, 79, 2, 133, 139, 12, 3, 83, 80, 73, 8, 218, 246, 14, 66, + 140, 77, 3, 84, 79, 80, 154, 84, 65, 159, 82, 82, 104, 146, 1, 65, 78, + 69, 62, 73, 78, 79, 74, 85, 74, 89, 160, 145, 7, 9, 82, 79, 85, 78, 68, + 69, 68, 32, 69, 178, 185, 7, 87, 202, 74, 78, 251, 124, 86, 19, 250, 151, + 15, 78, 250, 186, 1, 69, 146, 63, 72, 146, 1, 65, 2, 73, 3, 85, 15, 254, + 191, 13, 82, 178, 215, 1, 78, 154, 251, 1, 65, 3, 73, 23, 230, 150, 15, + 65, 54, 79, 158, 250, 1, 78, 86, 69, 2, 71, 2, 73, 3, 85, 13, 42, 69, + 150, 145, 17, 71, 2, 79, 3, 85, 4, 146, 145, 17, 82, 3, 89, 19, 210, 149, + 15, 65, 206, 231, 1, 69, 134, 19, 78, 2, 79, 86, 73, 3, 85, 7, 214, 252, + 16, 85, 219, 19, 73, 12, 18, 32, 63, 79, 4, 196, 252, 15, 4, 79, 78, 32, + 85, 13, 4, 68, 65, 83, 72, 8, 206, 226, 11, 83, 162, 130, 4, 80, 222, 35, + 32, 163, 112, 66, 12, 64, 4, 68, 76, 69, 32, 189, 223, 4, 6, 76, 73, 78, + 69, 32, 72, 10, 184, 192, 5, 3, 84, 72, 73, 238, 217, 8, 76, 22, 82, 207, + 163, 2, 68, 8, 76, 6, 73, 84, 65, 82, 89, 32, 232, 188, 10, 3, 75, 89, + 32, 195, 201, 5, 76, 4, 26, 72, 199, 170, 12, 77, 2, 227, 177, 4, 69, 24, + 42, 73, 76, 2, 85, 83, 131, 140, 17, 89, 6, 34, 68, 22, 77, 155, 134, 15, + 66, 2, 219, 156, 6, 73, 2, 223, 235, 6, 73, 16, 46, 32, 205, 177, 10, 5, + 45, 79, 82, 45, 80, 14, 40, 4, 83, 73, 71, 78, 203, 167, 13, 84, 13, 11, + 32, 10, 44, 5, 87, 73, 84, 72, 32, 135, 173, 6, 73, 8, 66, 67, 250, 202, + 4, 68, 234, 163, 8, 82, 21, 4, 70, 65, 76, 76, 2, 177, 231, 11, 3, 79, + 77, 77, 5, 171, 130, 15, 32, 184, 9, 184, 1, 10, 66, 73, 76, 69, 32, 80, + 72, 79, 78, 69, 166, 1, 68, 190, 76, 78, 170, 27, 79, 172, 1, 3, 83, 81, + 85, 38, 84, 250, 1, 85, 242, 176, 11, 89, 209, 219, 2, 5, 86, 73, 69, 32, 67, 7, 11, 32, 4, 108, 21, 87, 73, 84, 72, 32, 82, 73, 71, 72, 84, 87, - 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 255, 177, 1, 79, 2, 11, 32, 2, - 213, 132, 16, 2, 65, 84, 154, 6, 58, 69, 74, 73, 149, 75, 7, 85, 76, 79, - 32, 84, 87, 79, 4, 196, 230, 15, 10, 82, 78, 32, 80, 69, 78, 84, 65, 84, + 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 167, 178, 1, 79, 2, 11, 32, 2, + 225, 212, 16, 2, 65, 84, 154, 6, 58, 69, 74, 73, 149, 75, 7, 85, 76, 79, + 32, 84, 87, 79, 4, 208, 182, 16, 10, 82, 78, 32, 80, 69, 78, 84, 65, 84, 72, 159, 18, 76, 148, 6, 42, 32, 221, 1, 5, 70, 73, 69, 82, 32, 158, 1, - 88, 5, 83, 73, 71, 78, 32, 210, 172, 3, 86, 178, 146, 3, 68, 182, 1, 76, - 167, 180, 4, 65, 10, 42, 65, 234, 189, 8, 72, 203, 179, 7, 86, 4, 44, 5, - 82, 68, 72, 65, 67, 163, 239, 15, 78, 2, 201, 239, 15, 3, 65, 78, 68, + 88, 5, 83, 73, 71, 78, 32, 230, 179, 3, 86, 190, 162, 3, 68, 186, 1, 76, + 195, 194, 4, 65, 10, 42, 65, 166, 212, 8, 72, 155, 237, 7, 86, 4, 44, 5, + 82, 68, 72, 65, 67, 175, 191, 16, 78, 2, 213, 191, 16, 3, 65, 78, 68, 246, 4, 92, 12, 66, 82, 69, 86, 69, 32, 87, 73, 84, 72, 32, 73, 89, 7, 76, 69, 84, 84, 69, 82, 32, 2, 33, 6, 78, 86, 69, 82, 84, 69, 2, 21, 3, - 68, 32, 66, 2, 249, 181, 15, 2, 82, 69, 244, 4, 198, 1, 65, 82, 66, 66, - 67, 190, 13, 68, 194, 1, 69, 182, 2, 71, 82, 72, 46, 76, 230, 4, 77, 216, - 3, 7, 79, 80, 69, 78, 32, 83, 72, 22, 80, 38, 82, 182, 4, 83, 214, 33, - 84, 114, 85, 118, 86, 63, 89, 6, 38, 76, 150, 27, 67, 183, 167, 10, 80, - 2, 237, 19, 6, 86, 69, 79, 76, 65, 82, 6, 176, 19, 5, 73, 76, 65, 66, 73, - 201, 45, 4, 69, 71, 73, 78, 160, 1, 240, 1, 7, 65, 80, 73, 84, 65, 76, + 68, 32, 66, 2, 129, 134, 16, 2, 82, 69, 244, 4, 198, 1, 65, 82, 66, 66, + 67, 190, 13, 68, 202, 1, 69, 182, 2, 71, 82, 72, 46, 76, 230, 4, 77, 216, + 3, 7, 79, 80, 69, 78, 32, 83, 72, 22, 80, 38, 82, 182, 4, 83, 206, 33, + 84, 114, 85, 118, 86, 63, 89, 6, 38, 76, 158, 27, 67, 251, 204, 10, 80, + 2, 245, 19, 6, 86, 69, 79, 76, 65, 82, 6, 184, 19, 5, 73, 76, 65, 66, 73, + 193, 45, 4, 69, 71, 73, 78, 160, 1, 240, 1, 7, 65, 80, 73, 84, 65, 76, 32, 192, 2, 7, 69, 78, 84, 82, 69, 68, 32, 100, 13, 72, 73, 78, 69, 83, 69, 32, 84, 79, 78, 69, 32, 89, 108, 8, 89, 82, 73, 76, 76, 73, 67, 32, - 166, 188, 10, 73, 244, 2, 4, 82, 79, 83, 83, 191, 154, 5, 79, 56, 206, 1, - 66, 42, 82, 250, 26, 72, 198, 193, 12, 79, 138, 143, 3, 65, 150, 64, 67, + 242, 225, 10, 73, 244, 2, 4, 82, 79, 83, 83, 255, 196, 5, 79, 56, 206, 1, + 66, 42, 82, 130, 27, 72, 202, 141, 13, 79, 138, 147, 3, 65, 162, 64, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, - 80, 2, 81, 2, 84, 2, 85, 2, 86, 3, 87, 5, 237, 130, 6, 5, 65, 82, 82, 69, - 68, 7, 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 170, 171, 16, 69, 3, 78, + 80, 2, 81, 2, 84, 2, 85, 2, 86, 3, 87, 5, 133, 150, 6, 5, 65, 82, 82, 69, + 68, 7, 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 194, 251, 16, 69, 3, 78, 4, 30, 76, 21, 3, 82, 73, 71, 2, 29, 2, 69, 70, 2, 11, 72, 2, 11, 84, 2, - 173, 26, 2, 32, 72, 16, 36, 3, 65, 78, 71, 1, 2, 73, 78, 8, 11, 32, 8, - 170, 226, 11, 83, 214, 246, 3, 80, 146, 44, 81, 3, 82, 78, 34, 72, 30, - 83, 175, 217, 15, 69, 2, 197, 195, 14, 2, 65, 82, 74, 40, 5, 77, 65, 76, + 181, 26, 2, 32, 72, 16, 36, 3, 65, 78, 71, 1, 2, 73, 78, 8, 11, 32, 8, + 182, 134, 12, 83, 214, 162, 4, 80, 158, 44, 81, 3, 82, 78, 34, 72, 30, + 83, 187, 169, 16, 69, 2, 217, 144, 15, 2, 65, 82, 74, 40, 5, 77, 65, 76, 76, 32, 183, 6, 79, 72, 186, 1, 66, 138, 1, 68, 38, 69, 150, 1, 80, 58, - 83, 94, 84, 34, 89, 158, 163, 15, 90, 246, 63, 73, 138, 19, 67, 2, 71, + 83, 94, 84, 34, 89, 158, 243, 15, 90, 130, 64, 73, 150, 19, 67, 2, 71, 186, 22, 74, 2, 86, 158, 20, 72, 2, 75, 186, 2, 65, 2, 79, 3, 85, 6, 38, - 89, 186, 27, 65, 187, 139, 16, 69, 2, 221, 201, 1, 19, 69, 76, 79, 82, - 85, 83, 83, 73, 65, 78, 45, 85, 75, 82, 65, 73, 78, 73, 65, 4, 206, 190, - 6, 90, 195, 231, 9, 69, 15, 50, 83, 186, 165, 16, 70, 2, 76, 2, 77, 3, - 82, 5, 25, 4, 32, 87, 73, 84, 2, 21, 3, 72, 32, 68, 2, 11, 69, 2, 205, - 223, 12, 3, 83, 67, 69, 4, 26, 65, 191, 164, 16, 69, 2, 241, 247, 15, 2, - 76, 79, 8, 42, 84, 198, 172, 1, 67, 251, 244, 14, 72, 4, 145, 19, 8, 82, - 65, 73, 71, 72, 84, 32, 85, 4, 238, 140, 16, 83, 215, 22, 69, 6, 36, 3, - 69, 82, 85, 255, 162, 16, 85, 5, 37, 7, 32, 87, 73, 84, 72, 32, 66, 2, - 21, 3, 65, 67, 75, 2, 217, 228, 15, 2, 32, 89, 2, 191, 139, 11, 70, 16, - 18, 69, 27, 79, 2, 161, 5, 2, 78, 84, 14, 64, 2, 84, 32, 44, 5, 85, 66, - 76, 69, 32, 137, 52, 2, 87, 78, 6, 178, 192, 9, 83, 194, 144, 4, 72, 43, - 86, 4, 254, 135, 2, 65, 215, 240, 2, 80, 24, 40, 5, 88, 84, 82, 65, 45, - 139, 49, 78, 20, 52, 5, 72, 73, 71, 72, 32, 81, 4, 76, 79, 87, 32, 10, - 156, 1, 9, 69, 88, 84, 82, 65, 45, 76, 79, 87, 154, 7, 68, 70, 76, 79, - 84, 10, 76, 10, 69, 88, 84, 82, 65, 45, 72, 73, 71, 72, 154, 7, 68, 70, - 76, 79, 84, 2, 145, 8, 8, 32, 67, 79, 78, 84, 79, 85, 82, 8, 162, 9, 82, - 226, 3, 76, 237, 238, 14, 9, 69, 79, 82, 71, 73, 65, 78, 32, 78, 10, 248, - 5, 4, 73, 71, 72, 32, 135, 41, 65, 44, 46, 65, 84, 2, 79, 87, 201, 11, 2, - 69, 70, 2, 21, 3, 84, 69, 82, 2, 17, 2, 65, 76, 2, 17, 2, 32, 67, 2, 211, - 237, 14, 76, 36, 86, 32, 245, 231, 11, 15, 69, 82, 32, 82, 73, 71, 72, - 84, 32, 67, 79, 82, 78, 69, 82, 34, 158, 1, 67, 20, 2, 68, 79, 40, 4, 76, - 69, 70, 84, 82, 77, 12, 2, 82, 73, 50, 84, 178, 3, 65, 42, 71, 170, 2, - 73, 136, 196, 13, 2, 85, 80, 235, 180, 1, 86, 2, 131, 173, 10, 73, 6, - 238, 2, 84, 205, 200, 13, 2, 87, 78, 6, 28, 2, 32, 65, 247, 2, 45, 4, 25, - 4, 82, 82, 79, 87, 5, 151, 203, 13, 72, 2, 111, 65, 4, 180, 202, 13, 3, - 71, 72, 84, 255, 205, 2, 78, 4, 194, 2, 79, 155, 128, 14, 73, 18, 18, 65, - 23, 73, 2, 151, 244, 14, 67, 16, 26, 68, 163, 254, 12, 78, 14, 38, 32, - 217, 1, 4, 68, 76, 69, 32, 8, 26, 68, 70, 76, 79, 84, 4, 17, 2, 79, 84, - 4, 25, 4, 84, 69, 68, 32, 4, 18, 76, 79, 84, 2, 25, 4, 69, 70, 84, 45, 2, - 25, 4, 83, 84, 69, 77, 2, 17, 2, 32, 84, 2, 11, 79, 2, 11, 78, 2, 187, - 198, 13, 69, 6, 40, 6, 68, 79, 85, 66, 76, 69, 75, 71, 4, 11, 32, 4, 18, - 65, 43, 71, 2, 11, 67, 2, 229, 170, 10, 2, 85, 84, 2, 11, 82, 2, 179, - 170, 10, 65, 2, 207, 148, 14, 69, 4, 246, 250, 12, 76, 247, 129, 2, 82, - 26, 104, 6, 65, 73, 83, 69, 68, 32, 150, 1, 69, 216, 1, 3, 73, 71, 72, - 173, 222, 8, 5, 72, 79, 84, 73, 67, 10, 70, 68, 30, 73, 234, 186, 9, 69, - 180, 189, 5, 2, 85, 80, 199, 74, 67, 2, 129, 197, 14, 2, 79, 87, 2, 209, - 186, 9, 7, 78, 86, 69, 82, 84, 69, 68, 8, 104, 7, 86, 69, 82, 83, 69, 68, - 32, 133, 28, 14, 84, 82, 79, 70, 76, 69, 88, 32, 67, 76, 73, 67, 75, 32, - 6, 26, 71, 255, 197, 13, 67, 4, 11, 76, 4, 49, 10, 79, 84, 84, 65, 76, - 32, 83, 84, 79, 80, 5, 167, 19, 32, 6, 17, 2, 84, 32, 6, 38, 72, 250, - 133, 13, 84, 231, 59, 65, 2, 185, 168, 7, 3, 65, 76, 70, 156, 2, 138, 1, - 72, 48, 5, 77, 65, 76, 76, 32, 156, 31, 8, 84, 82, 69, 83, 83, 32, 65, - 78, 53, 11, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 4, 216, 130, 8, - 3, 79, 82, 84, 251, 139, 6, 69, 144, 2, 154, 2, 65, 46, 66, 130, 1, 67, - 238, 2, 68, 238, 2, 90, 66, 69, 46, 70, 30, 71, 110, 72, 102, 77, 34, 73, - 34, 74, 98, 76, 176, 3, 7, 78, 32, 87, 73, 84, 72, 32, 30, 79, 94, 80, - 22, 82, 146, 2, 83, 190, 1, 84, 130, 8, 85, 142, 1, 86, 134, 240, 15, 75, - 2, 81, 2, 87, 2, 88, 3, 89, 9, 238, 23, 76, 210, 164, 15, 73, 215, 79, - 69, 11, 46, 65, 50, 79, 222, 8, 32, 227, 221, 15, 69, 2, 17, 2, 82, 82, - 2, 181, 227, 5, 2, 69, 68, 2, 229, 20, 4, 84, 84, 79, 77, 41, 100, 7, 65, - 80, 73, 84, 65, 76, 32, 180, 1, 6, 76, 79, 83, 69, 68, 32, 190, 17, 32, - 187, 227, 15, 72, 30, 114, 73, 214, 6, 71, 238, 16, 76, 250, 218, 15, 79, - 158, 20, 65, 186, 2, 66, 2, 72, 2, 78, 2, 82, 2, 85, 3, 89, 7, 22, 78, - 191, 11, 32, 2, 237, 220, 5, 5, 86, 69, 82, 84, 69, 4, 26, 82, 175, 136, - 9, 79, 2, 229, 20, 9, 69, 86, 69, 82, 83, 69, 68, 32, 79, 23, 104, 6, 32, - 87, 73, 84, 72, 32, 86, 69, 32, 9, 79, 84, 76, 69, 83, 83, 32, 74, 32, - 105, 3, 90, 32, 68, 6, 26, 72, 247, 217, 13, 84, 4, 21, 3, 79, 79, 75, 5, - 201, 160, 13, 3, 32, 65, 78, 4, 238, 15, 90, 131, 210, 15, 76, 4, 33, 6, - 87, 73, 84, 72, 32, 83, 4, 29, 5, 84, 82, 79, 75, 69, 5, 177, 211, 8, 4, - 32, 65, 78, 68, 6, 33, 6, 73, 71, 82, 65, 80, 72, 7, 33, 6, 32, 87, 73, - 84, 72, 32, 4, 138, 14, 67, 219, 4, 82, 11, 190, 131, 16, 83, 2, 84, 2, - 90, 63, 78, 5, 225, 13, 3, 69, 78, 71, 11, 56, 5, 82, 69, 69, 75, 32, - 162, 1, 32, 151, 183, 13, 65, 4, 26, 71, 195, 223, 10, 80, 2, 151, 184, - 13, 65, 11, 40, 6, 32, 87, 73, 84, 72, 32, 39, 69, 4, 142, 178, 14, 72, - 231, 160, 1, 83, 4, 17, 2, 78, 71, 5, 11, 32, 2, 239, 207, 8, 87, 4, 222, - 4, 32, 239, 216, 15, 79, 5, 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, 11, 82, - 2, 181, 212, 13, 6, 79, 83, 83, 69, 68, 45, 25, 116, 6, 32, 87, 73, 84, - 72, 32, 226, 9, 83, 2, 90, 112, 2, 69, 90, 241, 222, 15, 8, 73, 71, 65, - 84, 85, 82, 69, 32, 12, 54, 73, 82, 77, 82, 82, 222, 6, 80, 187, 237, 3, - 66, 2, 49, 10, 78, 86, 69, 82, 84, 69, 68, 32, 76, 65, 2, 189, 172, 12, - 2, 90, 89, 2, 11, 73, 2, 17, 2, 68, 68, 2, 17, 2, 76, 69, 2, 185, 189, - 12, 2, 32, 84, 4, 61, 13, 69, 84, 82, 79, 70, 76, 69, 88, 32, 72, 79, 79, - 75, 5, 217, 12, 4, 32, 65, 78, 68, 4, 222, 11, 82, 207, 1, 76, 9, 18, 32, - 47, 80, 2, 21, 3, 87, 73, 84, 2, 139, 205, 15, 72, 4, 241, 152, 12, 2, - 69, 78, 5, 151, 233, 15, 72, 15, 80, 6, 32, 87, 73, 84, 72, 32, 62, 65, - 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 26, 70, 239, 206, 13, 84, 2, - 209, 201, 8, 3, 73, 83, 72, 2, 17, 2, 77, 83, 2, 247, 212, 5, 32, 6, 38, - 71, 182, 7, 79, 203, 243, 15, 69, 2, 11, 76, 2, 129, 176, 13, 4, 79, 84, - 84, 65, 13, 72, 6, 32, 87, 73, 84, 72, 32, 34, 67, 61, 6, 73, 68, 69, 87, - 65, 89, 4, 158, 3, 67, 131, 166, 14, 72, 4, 26, 82, 203, 180, 11, 72, 2, - 221, 199, 5, 3, 73, 80, 84, 2, 159, 167, 6, 83, 51, 106, 32, 102, 67, - 116, 2, 69, 83, 44, 2, 79, 80, 42, 83, 96, 6, 85, 82, 78, 69, 68, 32, - 207, 208, 12, 72, 4, 29, 5, 87, 73, 84, 72, 32, 4, 22, 80, 231, 5, 82, 2, - 197, 197, 8, 6, 65, 76, 65, 84, 65, 76, 2, 45, 9, 32, 68, 73, 71, 82, 65, - 80, 72, 32, 2, 25, 4, 87, 73, 84, 72, 2, 17, 2, 32, 67, 2, 195, 137, 3, - 85, 2, 11, 72, 2, 157, 241, 9, 3, 32, 68, 73, 2, 177, 206, 5, 5, 32, 72, - 65, 76, 70, 4, 37, 7, 32, 68, 73, 71, 82, 65, 80, 4, 11, 72, 5, 11, 32, - 2, 153, 3, 4, 87, 73, 84, 72, 32, 86, 65, 50, 77, 74, 79, 42, 82, 214, 1, - 89, 206, 241, 15, 72, 2, 73, 2, 86, 3, 87, 7, 26, 76, 167, 244, 15, 69, - 2, 231, 252, 13, 80, 5, 41, 8, 32, 87, 73, 84, 72, 32, 76, 79, 2, 221, - 133, 4, 2, 78, 71, 2, 11, 80, 2, 189, 133, 6, 2, 69, 78, 9, 33, 6, 32, - 87, 73, 84, 72, 32, 6, 26, 76, 131, 162, 14, 72, 4, 37, 7, 79, 78, 71, - 32, 76, 69, 71, 5, 25, 4, 32, 65, 78, 68, 2, 17, 2, 32, 82, 2, 11, 69, 2, - 213, 191, 8, 7, 84, 82, 79, 70, 76, 69, 88, 5, 29, 5, 32, 87, 73, 84, 72, - 2, 245, 230, 3, 2, 32, 66, 9, 18, 32, 95, 80, 4, 40, 4, 87, 73, 84, 72, - 171, 206, 14, 66, 2, 17, 2, 32, 76, 2, 11, 69, 2, 131, 1, 70, 2, 177, - 160, 15, 2, 83, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 82, 231, - 158, 14, 72, 2, 21, 3, 73, 71, 72, 2, 135, 189, 8, 84, 4, 11, 68, 4, 11, - 32, 4, 182, 208, 1, 72, 35, 76, 4, 24, 2, 72, 65, 31, 84, 2, 25, 4, 76, - 70, 32, 84, 2, 43, 82, 4, 30, 82, 53, 3, 85, 82, 78, 2, 237, 157, 15, 8, - 73, 65, 78, 71, 85, 76, 65, 82, 2, 161, 144, 8, 2, 69, 68, 8, 70, 80, - 140, 245, 9, 7, 78, 65, 83, 80, 73, 82, 65, 223, 247, 5, 83, 4, 11, 32, - 4, 190, 226, 12, 84, 231, 59, 65, 4, 26, 79, 199, 210, 14, 69, 2, 11, 73, - 2, 155, 155, 15, 67, 4, 36, 3, 65, 78, 71, 1, 2, 73, 78, 2, 25, 4, 32, + 89, 198, 27, 65, 199, 219, 16, 69, 2, 221, 202, 1, 19, 69, 76, 79, 82, + 85, 83, 83, 73, 65, 78, 45, 85, 75, 82, 65, 73, 78, 73, 65, 4, 242, 213, + 6, 90, 183, 160, 10, 69, 15, 50, 83, 210, 245, 16, 70, 2, 76, 2, 77, 3, + 82, 5, 25, 4, 32, 87, 73, 84, 2, 21, 3, 72, 32, 68, 2, 11, 69, 2, 177, + 141, 13, 3, 83, 67, 69, 4, 26, 65, 215, 244, 16, 69, 2, 137, 200, 16, 2, + 76, 79, 8, 42, 84, 206, 173, 1, 67, 139, 196, 15, 72, 4, 153, 19, 8, 82, + 65, 73, 71, 72, 84, 32, 85, 4, 134, 221, 16, 83, 215, 22, 69, 6, 36, 3, + 69, 82, 85, 151, 243, 16, 85, 5, 37, 7, 32, 87, 73, 84, 72, 32, 66, 2, + 21, 3, 65, 67, 75, 2, 229, 180, 16, 2, 32, 89, 2, 227, 175, 11, 70, 16, + 18, 69, 27, 79, 2, 169, 5, 2, 78, 84, 14, 64, 2, 84, 32, 52, 5, 85, 66, + 76, 69, 32, 129, 52, 2, 87, 78, 6, 242, 218, 9, 83, 142, 192, 4, 86, 171, + 180, 1, 72, 4, 194, 137, 2, 65, 219, 247, 2, 80, 24, 40, 5, 88, 84, 82, + 65, 45, 131, 49, 78, 20, 52, 5, 72, 73, 71, 72, 32, 81, 4, 76, 79, 87, + 32, 10, 156, 1, 9, 69, 88, 84, 82, 65, 45, 76, 79, 87, 154, 7, 68, 70, + 76, 79, 84, 10, 76, 10, 69, 88, 84, 82, 65, 45, 72, 73, 71, 72, 154, 7, + 68, 70, 76, 79, 84, 2, 145, 8, 8, 32, 67, 79, 78, 84, 79, 85, 82, 8, 162, + 9, 82, 226, 3, 76, 237, 191, 15, 9, 69, 79, 82, 71, 73, 65, 78, 32, 78, + 10, 248, 5, 4, 73, 71, 72, 32, 255, 40, 65, 44, 46, 65, 84, 2, 79, 87, + 201, 11, 2, 69, 70, 2, 21, 3, 84, 69, 82, 2, 17, 2, 65, 76, 2, 17, 2, 32, + 67, 2, 191, 188, 15, 76, 36, 86, 32, 249, 139, 12, 15, 69, 82, 32, 82, + 73, 71, 72, 84, 32, 67, 79, 82, 78, 69, 82, 34, 158, 1, 67, 20, 2, 68, + 79, 40, 4, 76, 69, 70, 84, 82, 77, 12, 2, 82, 73, 50, 84, 178, 3, 65, 42, + 71, 170, 2, 73, 184, 141, 14, 2, 85, 80, 179, 187, 1, 86, 2, 199, 210, + 10, 73, 6, 238, 2, 84, 253, 145, 14, 2, 87, 78, 6, 28, 2, 32, 65, 247, 2, + 45, 4, 25, 4, 82, 82, 79, 87, 5, 199, 148, 14, 72, 2, 111, 65, 4, 228, + 147, 14, 3, 71, 72, 84, 223, 212, 2, 78, 4, 194, 2, 79, 243, 204, 14, 73, + 18, 18, 65, 23, 73, 2, 147, 195, 15, 67, 16, 26, 68, 195, 185, 13, 78, + 14, 38, 32, 217, 1, 4, 68, 76, 69, 32, 8, 26, 68, 70, 76, 79, 84, 4, 17, + 2, 79, 84, 4, 25, 4, 84, 69, 68, 32, 4, 18, 76, 79, 84, 2, 25, 4, 69, 70, + 84, 45, 2, 25, 4, 83, 84, 69, 77, 2, 17, 2, 32, 84, 2, 11, 79, 2, 11, 78, + 2, 199, 196, 15, 69, 6, 40, 6, 68, 79, 85, 66, 76, 69, 75, 71, 4, 11, 32, + 4, 18, 65, 43, 71, 2, 11, 67, 2, 169, 208, 10, 2, 85, 84, 2, 11, 82, 2, + 247, 207, 10, 65, 2, 207, 225, 14, 69, 4, 150, 182, 13, 76, 207, 150, 2, + 82, 26, 104, 6, 65, 73, 83, 69, 68, 32, 150, 1, 69, 216, 1, 3, 73, 71, + 72, 133, 245, 8, 5, 72, 79, 84, 73, 67, 10, 70, 68, 30, 73, 162, 213, 9, + 69, 244, 242, 5, 2, 85, 80, 211, 74, 67, 2, 157, 146, 15, 2, 79, 87, 2, + 137, 213, 9, 7, 78, 86, 69, 82, 84, 69, 68, 8, 104, 7, 86, 69, 82, 83, + 69, 68, 32, 137, 28, 14, 84, 82, 79, 70, 76, 69, 88, 32, 67, 76, 73, 67, + 75, 32, 6, 26, 71, 175, 143, 14, 67, 4, 11, 76, 4, 49, 10, 79, 84, 84, + 65, 76, 32, 83, 84, 79, 80, 5, 171, 19, 32, 6, 17, 2, 84, 32, 6, 38, 72, + 166, 197, 13, 84, 235, 69, 65, 2, 133, 191, 7, 3, 65, 76, 70, 156, 2, + 138, 1, 72, 48, 5, 77, 65, 76, 76, 32, 148, 31, 8, 84, 82, 69, 83, 83, + 32, 65, 78, 53, 11, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 4, 136, + 153, 8, 3, 79, 82, 84, 203, 194, 6, 69, 144, 2, 154, 2, 65, 50, 66, 130, + 1, 67, 238, 2, 68, 238, 2, 90, 66, 69, 46, 70, 30, 71, 110, 72, 102, 77, + 34, 73, 34, 74, 98, 76, 176, 3, 7, 78, 32, 87, 73, 84, 72, 32, 30, 79, + 94, 80, 22, 82, 146, 2, 83, 190, 1, 84, 246, 7, 85, 142, 1, 86, 158, 192, + 16, 75, 2, 81, 2, 87, 2, 88, 3, 89, 9, 238, 131, 13, 76, 214, 136, 3, 73, + 227, 79, 69, 11, 46, 65, 50, 79, 222, 8, 32, 239, 173, 16, 69, 2, 17, 2, + 82, 82, 2, 193, 246, 5, 2, 69, 68, 2, 229, 20, 4, 84, 84, 79, 77, 41, + 100, 7, 65, 80, 73, 84, 65, 76, 32, 180, 1, 6, 76, 79, 83, 69, 68, 32, + 190, 17, 32, 199, 179, 16, 72, 30, 114, 73, 214, 6, 71, 226, 16, 76, 146, + 171, 16, 79, 158, 20, 65, 186, 2, 66, 2, 72, 2, 78, 2, 82, 2, 85, 3, 89, + 7, 22, 78, 191, 11, 32, 2, 249, 239, 5, 5, 86, 69, 82, 84, 69, 4, 26, 82, + 195, 161, 9, 79, 2, 217, 20, 9, 69, 86, 69, 82, 83, 69, 68, 32, 79, 23, + 104, 6, 32, 87, 73, 84, 72, 32, 86, 69, 32, 9, 79, 84, 76, 69, 83, 83, + 32, 74, 32, 105, 3, 90, 32, 68, 6, 26, 72, 163, 163, 14, 84, 4, 21, 3, + 79, 79, 75, 5, 225, 234, 13, 3, 32, 65, 78, 4, 238, 15, 90, 143, 162, 16, + 76, 4, 33, 6, 87, 73, 84, 72, 32, 83, 4, 29, 5, 84, 82, 79, 75, 69, 5, + 133, 234, 8, 4, 32, 65, 78, 68, 6, 33, 6, 73, 71, 82, 65, 80, 72, 7, 33, + 6, 32, 87, 73, 84, 72, 32, 4, 138, 14, 67, 207, 4, 82, 11, 202, 211, 16, + 83, 2, 84, 2, 90, 63, 78, 5, 225, 13, 3, 69, 78, 71, 11, 56, 5, 82, 69, + 69, 75, 32, 162, 1, 32, 195, 128, 14, 65, 4, 26, 71, 191, 132, 11, 80, 2, + 195, 129, 14, 65, 11, 40, 6, 32, 87, 73, 84, 72, 32, 39, 69, 4, 166, 255, + 14, 72, 219, 163, 1, 83, 4, 17, 2, 78, 71, 5, 11, 32, 2, 195, 230, 8, 87, + 4, 222, 4, 32, 251, 168, 16, 79, 5, 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, + 11, 82, 2, 225, 157, 14, 6, 79, 83, 83, 69, 68, 45, 25, 116, 6, 32, 87, + 73, 84, 72, 32, 226, 9, 83, 2, 90, 112, 2, 69, 90, 253, 174, 16, 8, 73, + 71, 65, 84, 85, 82, 69, 32, 12, 54, 73, 82, 77, 82, 82, 222, 6, 80, 243, + 244, 3, 66, 2, 49, 10, 78, 86, 69, 82, 84, 69, 68, 32, 76, 65, 2, 173, + 218, 12, 2, 90, 89, 2, 11, 73, 2, 17, 2, 68, 68, 2, 17, 2, 76, 69, 2, + 145, 235, 12, 2, 32, 84, 4, 61, 13, 69, 84, 82, 79, 70, 76, 69, 88, 32, + 72, 79, 79, 75, 5, 205, 12, 4, 32, 65, 78, 68, 4, 210, 11, 82, 207, 1, + 76, 9, 18, 32, 47, 80, 2, 21, 3, 87, 73, 84, 2, 151, 157, 16, 72, 4, 193, + 186, 12, 2, 69, 78, 5, 163, 185, 16, 72, 15, 80, 6, 32, 87, 73, 84, 72, + 32, 62, 65, 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 26, 70, 155, 152, + 14, 84, 2, 165, 224, 8, 3, 73, 83, 72, 2, 17, 2, 77, 83, 2, 131, 232, 5, + 32, 6, 38, 71, 170, 7, 79, 227, 195, 16, 69, 2, 11, 76, 2, 173, 249, 13, + 4, 79, 84, 84, 65, 13, 72, 6, 32, 87, 73, 84, 72, 32, 34, 67, 61, 6, 73, + 68, 69, 87, 65, 89, 4, 158, 3, 67, 155, 243, 14, 72, 4, 26, 82, 203, 216, + 11, 72, 2, 233, 218, 5, 3, 73, 80, 84, 2, 231, 189, 6, 83, 51, 106, 32, + 102, 67, 116, 2, 69, 83, 44, 2, 79, 80, 42, 83, 96, 6, 85, 82, 78, 69, + 68, 32, 231, 139, 13, 72, 4, 29, 5, 87, 73, 84, 72, 32, 4, 22, 80, 219, + 5, 82, 2, 153, 220, 8, 6, 65, 76, 65, 84, 65, 76, 2, 45, 9, 32, 68, 73, + 71, 82, 65, 80, 72, 32, 2, 25, 4, 87, 73, 84, 72, 2, 17, 2, 32, 67, 2, + 179, 145, 3, 85, 2, 11, 72, 2, 217, 150, 10, 3, 32, 68, 73, 2, 189, 225, + 5, 5, 32, 72, 65, 76, 70, 4, 37, 7, 32, 68, 73, 71, 82, 65, 80, 4, 11, + 72, 5, 11, 32, 2, 141, 3, 4, 87, 73, 84, 72, 32, 86, 65, 38, 77, 74, 79, + 42, 82, 214, 1, 89, 230, 193, 16, 72, 2, 73, 2, 86, 3, 87, 7, 150, 236, + 12, 76, 183, 216, 3, 69, 5, 41, 8, 32, 87, 73, 84, 72, 32, 76, 79, 2, + 213, 141, 4, 2, 78, 71, 2, 11, 80, 2, 225, 156, 6, 2, 69, 78, 9, 33, 6, + 32, 87, 73, 84, 72, 32, 6, 26, 76, 167, 239, 14, 72, 4, 37, 7, 79, 78, + 71, 32, 76, 69, 71, 5, 25, 4, 32, 65, 78, 68, 2, 17, 2, 32, 82, 2, 11, + 69, 2, 181, 214, 8, 7, 84, 82, 79, 70, 76, 69, 88, 5, 29, 5, 32, 87, 73, + 84, 72, 2, 185, 238, 3, 2, 32, 66, 9, 18, 32, 95, 80, 4, 40, 4, 87, 73, + 84, 72, 179, 159, 15, 66, 2, 17, 2, 32, 76, 2, 11, 69, 2, 131, 1, 70, 2, + 189, 240, 15, 2, 83, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 82, + 139, 236, 14, 72, 2, 21, 3, 73, 71, 72, 2, 231, 211, 8, 84, 4, 11, 68, 4, + 11, 32, 4, 130, 210, 1, 72, 35, 76, 4, 24, 2, 72, 65, 31, 84, 2, 25, 4, + 76, 70, 32, 84, 2, 43, 82, 4, 30, 82, 53, 3, 85, 82, 78, 2, 249, 237, 15, + 8, 73, 65, 78, 71, 85, 76, 65, 82, 2, 253, 166, 8, 2, 69, 68, 8, 70, 80, + 212, 154, 10, 7, 78, 65, 83, 80, 73, 82, 65, 175, 162, 6, 83, 4, 11, 32, + 4, 242, 161, 13, 84, 235, 69, 65, 4, 26, 79, 199, 162, 15, 69, 2, 11, 73, + 2, 167, 235, 15, 67, 4, 36, 3, 65, 78, 71, 1, 2, 73, 78, 2, 25, 4, 32, 68, 69, 80, 2, 21, 3, 65, 82, 84, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 84, - 2, 11, 79, 2, 227, 238, 5, 78, 2, 11, 32, 2, 215, 194, 14, 83, 234, 2, - 92, 2, 69, 89, 88, 7, 71, 79, 76, 73, 65, 78, 32, 210, 24, 79, 209, 222, - 11, 3, 75, 69, 89, 6, 26, 32, 251, 171, 15, 45, 4, 184, 223, 11, 6, 87, - 73, 84, 72, 32, 87, 255, 163, 2, 66, 214, 2, 194, 2, 68, 46, 70, 148, 1, + 2, 11, 79, 2, 143, 134, 6, 78, 2, 11, 32, 2, 211, 145, 15, 83, 234, 2, + 92, 2, 69, 89, 88, 7, 71, 79, 76, 73, 65, 78, 32, 206, 24, 79, 177, 130, + 12, 3, 75, 69, 89, 6, 26, 32, 135, 252, 15, 45, 4, 152, 131, 12, 6, 87, + 73, 84, 72, 32, 87, 179, 205, 2, 66, 214, 2, 194, 2, 68, 46, 70, 148, 1, 14, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 73, 82, 71, 65, 16, 7, 76, - 69, 84, 84, 69, 82, 32, 142, 16, 83, 132, 1, 7, 82, 79, 84, 65, 84, 69, - 68, 22, 66, 98, 84, 160, 2, 4, 86, 79, 87, 69, 158, 233, 3, 67, 164, 1, - 6, 77, 65, 78, 67, 72, 85, 196, 162, 7, 4, 78, 73, 82, 85, 171, 166, 2, - 69, 22, 236, 20, 3, 79, 85, 66, 183, 227, 13, 73, 12, 112, 19, 82, 69, - 69, 32, 86, 65, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 158, - 154, 13, 85, 175, 46, 79, 8, 177, 207, 9, 3, 84, 79, 82, 5, 231, 19, 32, - 136, 2, 130, 2, 65, 244, 4, 2, 67, 72, 88, 2, 77, 65, 174, 2, 83, 250, 1, - 84, 234, 3, 90, 190, 229, 13, 72, 150, 168, 1, 75, 2, 76, 162, 7, 69, 2, - 79, 2, 85, 222, 61, 66, 2, 68, 2, 70, 2, 71, 2, 74, 2, 78, 2, 80, 2, 81, + 69, 84, 84, 69, 82, 32, 138, 16, 83, 132, 1, 7, 82, 79, 84, 65, 84, 69, + 68, 22, 66, 98, 84, 160, 2, 4, 86, 79, 87, 69, 154, 241, 3, 67, 164, 1, + 6, 77, 65, 78, 67, 72, 85, 216, 190, 7, 4, 78, 73, 82, 85, 239, 203, 2, + 69, 22, 232, 20, 3, 79, 85, 66, 207, 176, 14, 73, 12, 112, 19, 82, 69, + 69, 32, 86, 65, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 214, + 227, 13, 85, 199, 46, 79, 8, 249, 244, 9, 3, 84, 79, 82, 5, 227, 19, 32, + 136, 2, 130, 2, 65, 244, 4, 2, 67, 72, 88, 2, 77, 65, 174, 2, 83, 246, 1, + 84, 234, 3, 90, 214, 178, 14, 72, 142, 171, 1, 75, 2, 76, 162, 7, 69, 2, + 79, 2, 85, 234, 61, 66, 2, 68, 2, 70, 2, 71, 2, 74, 2, 78, 2, 80, 2, 81, 2, 82, 2, 87, 2, 89, 187, 2, 73, 59, 56, 8, 76, 73, 32, 71, 65, 76, 73, - 32, 207, 225, 15, 78, 54, 174, 1, 65, 52, 6, 86, 73, 83, 65, 82, 71, 22, + 32, 231, 177, 16, 78, 54, 174, 1, 65, 52, 6, 86, 73, 83, 65, 82, 71, 22, 68, 76, 5, 72, 65, 76, 70, 32, 34, 73, 50, 85, 34, 78, 30, 84, 66, 66, - 222, 150, 15, 80, 2, 90, 242, 68, 83, 14, 67, 3, 75, 7, 48, 6, 78, 85, - 83, 86, 65, 82, 191, 224, 15, 72, 2, 131, 134, 9, 65, 8, 26, 65, 215, - 221, 15, 68, 7, 214, 222, 8, 77, 253, 190, 6, 3, 71, 65, 76, 4, 162, 221, - 15, 89, 187, 2, 85, 5, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 85, 2, 169, - 156, 15, 3, 66, 65, 68, 4, 178, 220, 15, 71, 3, 78, 8, 60, 6, 72, 82, 69, - 69, 32, 66, 222, 150, 15, 84, 183, 71, 65, 2, 17, 2, 65, 76, 2, 247, 184, - 15, 85, 6, 26, 65, 207, 221, 15, 73, 5, 29, 5, 32, 87, 73, 84, 72, 2, - 201, 143, 14, 2, 32, 84, 41, 29, 5, 78, 67, 72, 85, 32, 38, 104, 9, 65, - 76, 73, 32, 71, 65, 76, 73, 32, 202, 228, 13, 90, 134, 245, 1, 70, 2, 75, - 2, 82, 187, 2, 73, 28, 122, 68, 166, 165, 9, 67, 158, 164, 1, 84, 142, - 154, 3, 66, 2, 71, 2, 74, 2, 76, 138, 176, 1, 90, 254, 4, 78, 247, 63, - 83, 4, 202, 227, 13, 68, 135, 245, 1, 72, 48, 52, 4, 73, 66, 69, 32, 246, - 215, 15, 72, 187, 2, 65, 44, 186, 215, 2, 84, 162, 243, 2, 71, 2, 72, - 218, 156, 6, 73, 190, 251, 1, 67, 2, 83, 246, 7, 82, 222, 158, 1, 65, - 186, 9, 90, 162, 7, 85, 222, 61, 68, 2, 70, 2, 74, 2, 75, 2, 80, 187, 2, - 69, 60, 52, 4, 79, 68, 79, 32, 254, 213, 15, 83, 187, 2, 65, 56, 250, 1, - 65, 98, 68, 34, 74, 34, 78, 176, 163, 1, 8, 76, 79, 78, 71, 32, 86, 79, - 87, 250, 174, 1, 84, 162, 243, 2, 71, 150, 152, 8, 67, 246, 7, 72, 182, - 175, 1, 79, 2, 85, 222, 61, 66, 2, 75, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, - 186, 2, 69, 3, 73, 6, 56, 8, 76, 73, 32, 71, 65, 76, 73, 32, 171, 213, - 15, 78, 4, 190, 222, 13, 90, 135, 245, 1, 84, 4, 158, 211, 15, 90, 187, - 2, 65, 4, 254, 210, 15, 73, 187, 2, 65, 2, 223, 210, 15, 73, 6, 170, 193, - 15, 72, 162, 17, 82, 187, 2, 65, 8, 128, 1, 4, 87, 73, 82, 76, 149, 181, - 4, 21, 73, 66, 69, 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 66, 79, 85, - 78, 68, 65, 82, 89, 6, 17, 2, 32, 66, 6, 25, 4, 73, 82, 71, 65, 7, 33, 6, - 32, 87, 73, 84, 72, 32, 4, 150, 2, 68, 171, 141, 15, 79, 6, 152, 1, 3, - 82, 73, 80, 56, 18, 85, 82, 78, 69, 68, 32, 83, 87, 73, 82, 76, 32, 66, - 73, 82, 71, 65, 32, 245, 190, 1, 8, 79, 68, 79, 32, 83, 79, 70, 84, 2, - 209, 141, 15, 9, 76, 69, 32, 66, 73, 82, 71, 65, 32, 2, 33, 6, 87, 73, - 84, 72, 32, 68, 2, 145, 141, 15, 5, 79, 85, 66, 76, 69, 2, 131, 192, 9, - 76, 10, 96, 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 224, 251, 3, 5, 83, - 84, 65, 66, 76, 235, 166, 9, 82, 6, 26, 89, 243, 136, 12, 69, 4, 190, - 254, 14, 65, 155, 1, 73, 8, 48, 2, 78, 32, 162, 197, 4, 68, 231, 242, 10, - 83, 4, 72, 12, 86, 73, 69, 87, 73, 78, 71, 32, 67, 69, 82, 69, 147, 136, - 12, 67, 2, 177, 174, 15, 2, 77, 79, 4, 230, 129, 12, 73, 139, 204, 3, 69, - 10, 26, 72, 69, 2, 79, 82, 2, 45, 9, 69, 82, 32, 67, 72, 82, 73, 83, 84, - 2, 147, 247, 2, 77, 8, 50, 32, 52, 4, 73, 90, 69, 68, 155, 192, 14, 87, - 4, 230, 173, 8, 66, 133, 232, 5, 4, 83, 67, 79, 79, 2, 205, 168, 3, 7, - 32, 87, 72, 69, 69, 76, 67, 18, 52, 2, 78, 84, 152, 1, 2, 83, 69, 247, - 200, 15, 84, 10, 48, 3, 65, 73, 78, 185, 224, 11, 3, 32, 70, 85, 9, 11, - 32, 6, 222, 166, 9, 82, 24, 5, 67, 65, 66, 76, 69, 137, 235, 1, 6, 66, - 73, 67, 89, 67, 76, 7, 11, 32, 4, 160, 185, 14, 2, 84, 82, 219, 83, 70, - 86, 52, 7, 76, 69, 84, 84, 69, 82, 32, 191, 211, 5, 68, 62, 198, 1, 75, - 62, 77, 34, 78, 34, 79, 30, 80, 34, 84, 246, 104, 68, 220, 231, 7, 2, 72, - 65, 2, 82, 158, 180, 1, 69, 190, 29, 83, 150, 90, 76, 246, 7, 67, 246, - 162, 4, 89, 190, 28, 66, 2, 87, 187, 2, 65, 6, 180, 154, 5, 2, 69, 65, - 182, 238, 5, 72, 231, 190, 4, 79, 4, 230, 180, 15, 65, 215, 1, 73, 4, - 158, 247, 14, 73, 255, 59, 71, 7, 210, 198, 15, 76, 3, 79, 4, 222, 178, - 15, 72, 219, 19, 65, 6, 214, 246, 8, 72, 170, 204, 6, 69, 155, 3, 65, - 204, 4, 44, 2, 76, 84, 238, 6, 83, 159, 175, 13, 67, 102, 36, 4, 65, 78, - 73, 32, 223, 2, 73, 76, 52, 7, 76, 69, 84, 84, 69, 82, 32, 211, 139, 4, - 83, 74, 210, 1, 68, 222, 82, 78, 158, 230, 1, 82, 230, 208, 1, 74, 166, - 209, 7, 84, 206, 160, 3, 66, 2, 67, 2, 71, 2, 75, 2, 80, 254, 68, 72, 2, - 76, 2, 77, 2, 83, 2, 86, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 85, 10, 38, - 68, 162, 192, 15, 72, 187, 2, 65, 6, 158, 192, 15, 68, 2, 72, 187, 2, 65, - 26, 56, 2, 80, 76, 236, 2, 3, 83, 69, 84, 183, 174, 14, 77, 18, 50, 69, - 89, 8, 73, 67, 65, 84, 73, 79, 78, 32, 2, 41, 8, 32, 77, 85, 83, 73, 67, - 65, 76, 2, 21, 3, 32, 78, 79, 2, 155, 173, 14, 84, 16, 40, 4, 83, 73, 71, - 78, 191, 192, 15, 88, 15, 11, 32, 12, 48, 3, 73, 78, 32, 81, 5, 87, 73, - 84, 72, 32, 8, 166, 199, 3, 76, 22, 82, 140, 246, 8, 5, 68, 79, 85, 66, - 76, 235, 222, 1, 84, 4, 210, 246, 11, 85, 167, 200, 2, 68, 7, 11, 32, 4, - 232, 82, 5, 77, 85, 76, 84, 73, 207, 209, 11, 85, 226, 3, 32, 2, 73, 67, - 247, 152, 3, 72, 224, 3, 30, 32, 101, 3, 65, 76, 32, 6, 76, 4, 78, 65, - 84, 85, 200, 43, 2, 70, 76, 181, 229, 10, 4, 83, 72, 65, 82, 2, 239, 165, - 10, 82, 218, 3, 64, 8, 75, 69, 89, 66, 79, 65, 82, 68, 74, 83, 155, 239, - 3, 78, 5, 41, 8, 32, 87, 73, 84, 72, 32, 74, 65, 2, 209, 254, 14, 2, 67, - 75, 212, 3, 48, 6, 89, 77, 66, 79, 76, 32, 135, 131, 11, 67, 210, 3, 230, - 2, 66, 238, 1, 67, 226, 9, 68, 242, 2, 69, 162, 1, 70, 166, 2, 71, 112, - 9, 65, 82, 80, 69, 71, 71, 73, 65, 84, 168, 1, 2, 72, 65, 126, 75, 198, - 3, 76, 138, 1, 77, 190, 2, 78, 162, 1, 79, 218, 2, 80, 200, 2, 2, 81, 85, - 146, 2, 82, 190, 2, 83, 230, 5, 84, 182, 10, 86, 42, 88, 42, 87, 180, - 233, 8, 9, 73, 78, 86, 69, 82, 84, 69, 68, 32, 163, 151, 6, 90, 20, 36, - 5, 69, 71, 73, 78, 32, 59, 82, 8, 130, 15, 80, 30, 83, 182, 224, 7, 66, - 187, 173, 7, 84, 12, 24, 2, 65, 67, 35, 69, 4, 218, 130, 15, 75, 143, 53, - 69, 8, 26, 86, 223, 245, 14, 65, 6, 32, 2, 73, 83, 139, 183, 15, 69, 5, - 171, 26, 32, 84, 120, 2, 65, 69, 30, 76, 94, 79, 194, 7, 82, 234, 11, 32, - 224, 32, 7, 73, 82, 67, 76, 69, 32, 88, 145, 200, 5, 2, 85, 84, 2, 197, - 241, 14, 2, 83, 85, 8, 42, 73, 245, 23, 5, 85, 83, 84, 69, 82, 4, 26, 77, - 175, 152, 13, 86, 2, 207, 177, 9, 65, 64, 26, 77, 179, 178, 15, 68, 62, - 64, 7, 66, 73, 78, 73, 78, 71, 32, 185, 251, 5, 3, 77, 79, 78, 60, 202, - 1, 65, 80, 7, 77, 65, 82, 67, 65, 84, 79, 56, 2, 68, 79, 56, 2, 85, 80, - 28, 2, 70, 76, 40, 5, 72, 65, 82, 77, 79, 22, 83, 142, 2, 84, 202, 154, - 7, 66, 140, 180, 3, 2, 76, 79, 183, 222, 4, 82, 6, 76, 5, 67, 67, 69, 78, - 84, 37, 10, 85, 71, 77, 69, 78, 84, 65, 84, 73, 79, 5, 205, 2, 5, 45, 83, - 84, 65, 67, 2, 231, 160, 14, 78, 6, 52, 2, 87, 78, 168, 3, 2, 85, 66, - 199, 145, 15, 73, 2, 137, 151, 14, 2, 32, 66, 12, 136, 76, 2, 65, 71, - 231, 228, 14, 73, 2, 231, 137, 14, 78, 12, 132, 1, 9, 78, 65, 80, 32, 80, - 73, 90, 90, 73, 22, 84, 236, 187, 4, 11, 80, 82, 69, 67, 72, 71, 69, 83, - 65, 78, 71, 255, 238, 5, 77, 2, 191, 206, 11, 67, 6, 44, 5, 65, 67, 67, - 65, 84, 159, 158, 15, 69, 4, 40, 4, 73, 83, 83, 73, 203, 174, 15, 79, 2, - 211, 143, 15, 77, 10, 34, 82, 253, 225, 11, 2, 69, 78, 8, 28, 2, 73, 80, - 191, 6, 69, 2, 133, 205, 11, 6, 76, 69, 32, 84, 79, 78, 4, 22, 79, 191, - 2, 69, 2, 159, 161, 14, 73, 24, 98, 65, 142, 1, 69, 84, 6, 79, 85, 66, - 76, 69, 32, 213, 249, 9, 7, 82, 85, 77, 32, 67, 76, 69, 10, 96, 2, 32, - 67, 20, 2, 77, 80, 196, 29, 4, 83, 72, 69, 68, 233, 228, 14, 5, 76, 32, - 83, 69, 71, 2, 223, 164, 8, 65, 5, 255, 166, 13, 32, 4, 52, 3, 67, 82, - 69, 241, 201, 8, 4, 71, 82, 69, 69, 2, 157, 25, 3, 83, 67, 69, 6, 246, - 21, 83, 250, 6, 66, 131, 244, 4, 70, 14, 32, 3, 78, 68, 32, 155, 16, 73, - 10, 74, 80, 30, 83, 208, 13, 3, 79, 70, 32, 230, 210, 7, 66, 187, 173, 7, - 84, 2, 145, 153, 12, 2, 72, 82, 2, 215, 176, 14, 76, 34, 104, 6, 69, 82, - 77, 65, 84, 65, 22, 73, 122, 79, 102, 32, 144, 36, 3, 85, 83, 65, 157, - 221, 3, 2, 76, 65, 5, 195, 129, 13, 32, 10, 22, 78, 135, 34, 86, 8, 56, - 9, 71, 69, 82, 69, 68, 32, 84, 82, 69, 235, 20, 65, 6, 185, 215, 5, 4, - 77, 79, 76, 79, 6, 208, 25, 3, 85, 82, 45, 147, 201, 13, 82, 18, 54, 32, - 56, 7, 76, 73, 83, 83, 65, 78, 68, 23, 82, 6, 25, 4, 67, 76, 69, 70, 7, - 161, 13, 3, 32, 79, 84, 4, 163, 235, 5, 79, 8, 84, 9, 65, 67, 69, 32, 78, - 79, 84, 69, 32, 37, 8, 69, 71, 79, 82, 73, 65, 78, 32, 4, 132, 196, 8, 2, - 78, 79, 27, 83, 4, 250, 2, 67, 3, 70, 8, 44, 3, 76, 70, 32, 209, 8, 3, - 85, 80, 84, 6, 52, 3, 80, 69, 68, 246, 214, 3, 78, 163, 169, 3, 82, 2, - 135, 175, 5, 65, 24, 48, 6, 73, 69, 86, 65, 78, 32, 147, 255, 13, 79, 22, - 178, 1, 67, 46, 69, 68, 7, 81, 85, 65, 82, 84, 69, 82, 62, 70, 184, 211, - 3, 4, 72, 65, 76, 70, 0, 5, 87, 72, 79, 76, 69, 253, 208, 1, 9, 82, 69, - 67, 73, 84, 65, 84, 73, 86, 2, 11, 32, 2, 11, 67, 2, 171, 144, 5, 76, 6, - 64, 5, 73, 71, 72, 84, 72, 253, 235, 13, 5, 78, 68, 32, 79, 70, 4, 165, - 230, 5, 10, 32, 78, 79, 84, 69, 32, 83, 84, 69, 77, 4, 214, 14, 76, 225, - 196, 3, 4, 73, 78, 65, 76, 8, 44, 4, 79, 78, 71, 65, 213, 13, 2, 69, 70, - 7, 11, 32, 4, 28, 3, 73, 77, 80, 3, 80, 2, 197, 2, 7, 69, 82, 70, 69, 67, - 84, 65, 18, 104, 2, 65, 88, 20, 2, 69, 90, 20, 5, 73, 78, 73, 77, 65, 48, - 3, 79, 79, 78, 25, 4, 85, 76, 84, 73, 2, 135, 253, 14, 73, 2, 143, 255, - 14, 90, 7, 11, 32, 4, 146, 250, 6, 82, 171, 153, 5, 66, 4, 185, 17, 2, - 32, 78, 4, 60, 11, 80, 76, 69, 32, 77, 69, 65, 83, 85, 82, 69, 15, 32, 2, - 11, 32, 2, 139, 249, 6, 82, 10, 120, 4, 69, 66, 69, 78, 208, 26, 3, 85, - 76, 76, 148, 198, 5, 3, 65, 84, 85, 173, 176, 6, 7, 79, 84, 69, 72, 69, - 65, 68, 2, 245, 131, 14, 4, 83, 84, 73, 77, 32, 88, 2, 78, 69, 120, 14, - 82, 78, 65, 77, 69, 78, 84, 32, 83, 84, 82, 79, 75, 69, 107, 84, 6, 92, - 18, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 87, 69, 78, 84, 89, 45, 69, - 73, 155, 20, 45, 4, 177, 13, 2, 71, 72, 22, 11, 45, 22, 250, 158, 3, 49, - 206, 250, 11, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 4, - 173, 4, 3, 84, 65, 86, 18, 96, 9, 65, 82, 69, 78, 84, 72, 69, 83, 73, 0, - 2, 76, 85, 18, 69, 142, 1, 79, 247, 145, 13, 73, 2, 239, 22, 83, 6, 68, - 4, 68, 65, 76, 32, 49, 9, 83, 32, 83, 85, 66, 80, 85, 78, 67, 4, 26, 85, - 219, 213, 14, 77, 2, 191, 213, 14, 80, 2, 247, 249, 12, 84, 6, 48, 2, 68, - 65, 237, 5, 5, 82, 82, 69, 67, 84, 2, 175, 147, 13, 84, 12, 76, 6, 65, - 82, 84, 69, 82, 32, 125, 9, 73, 78, 68, 73, 67, 69, 83, 73, 77, 8, 60, 5, - 84, 79, 78, 69, 32, 138, 200, 3, 78, 163, 169, 3, 82, 4, 26, 83, 251, - 250, 4, 70, 2, 205, 148, 15, 3, 72, 65, 82, 4, 17, 2, 65, 32, 4, 138, - 240, 11, 65, 217, 161, 3, 3, 66, 65, 83, 14, 22, 69, 167, 1, 73, 10, 72, - 4, 80, 69, 65, 84, 73, 10, 86, 69, 82, 83, 69, 32, 70, 73, 78, 65, 8, - 152, 195, 5, 9, 69, 68, 32, 70, 73, 71, 85, 82, 69, 191, 130, 8, 32, 2, - 211, 4, 76, 4, 48, 2, 71, 72, 57, 6, 78, 70, 79, 82, 90, 65, 2, 33, 6, - 84, 32, 82, 69, 80, 69, 2, 131, 251, 9, 65, 2, 155, 151, 8, 78, 48, 136, - 1, 6, 67, 65, 78, 68, 73, 67, 62, 69, 166, 1, 72, 54, 73, 252, 1, 6, 81, - 85, 65, 82, 69, 32, 204, 128, 8, 2, 85, 66, 239, 2, 79, 4, 17, 2, 85, 83, - 5, 173, 141, 13, 5, 32, 70, 76, 69, 88, 14, 32, 2, 77, 73, 195, 230, 14, - 71, 12, 64, 6, 66, 82, 69, 86, 73, 83, 1, 6, 77, 73, 78, 73, 77, 65, 6, - 11, 32, 6, 138, 13, 87, 166, 222, 6, 82, 171, 153, 5, 66, 6, 84, 3, 79, - 82, 84, 169, 211, 5, 3, 65, 82, 80, 14, 32, 4, 78, 71, 76, 69, 43, 88, 2, - 17, 2, 32, 66, 2, 155, 169, 13, 65, 12, 18, 45, 79, 84, 4, 242, 7, 76, - 213, 238, 13, 11, 83, 84, 82, 73, 78, 71, 32, 70, 82, 69, 84, 8, 52, 3, - 69, 69, 78, 1, 6, 89, 45, 70, 79, 85, 82, 4, 193, 12, 2, 84, 72, 6, 26, - 78, 135, 140, 15, 66, 4, 229, 9, 7, 79, 84, 69, 72, 69, 65, 68, 60, 132, - 1, 6, 69, 77, 80, 85, 83, 32, 154, 4, 72, 88, 2, 87, 79, 84, 7, 79, 82, - 67, 85, 76, 85, 83, 46, 82, 141, 3, 3, 85, 82, 78, 16, 232, 1, 27, 73, - 77, 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, 67, 85, 77, 32, 80, 82, 79, - 76, 65, 84, 73, 79, 78, 69, 32, 129, 1, 25, 80, 69, 82, 70, 69, 67, 84, - 85, 77, 32, 67, 85, 77, 32, 80, 82, 79, 76, 65, 84, 73, 79, 78, 69, 32, - 10, 60, 10, 73, 77, 80, 69, 82, 70, 69, 67, 84, 65, 131, 1, 80, 9, 169, - 184, 5, 11, 32, 68, 73, 77, 73, 78, 85, 84, 73, 79, 78, 6, 60, 3, 73, 77, - 80, 41, 8, 80, 69, 82, 70, 69, 67, 84, 65, 2, 213, 226, 14, 5, 69, 82, - 70, 69, 67, 5, 145, 163, 8, 12, 32, 68, 73, 77, 73, 78, 85, 84, 73, 79, - 78, 45, 6, 72, 2, 82, 69, 249, 5, 11, 73, 82, 84, 89, 45, 83, 69, 67, 79, - 78, 68, 2, 11, 69, 2, 11, 45, 2, 11, 76, 2, 37, 7, 73, 78, 69, 32, 83, - 84, 65, 2, 147, 132, 14, 70, 5, 173, 168, 11, 6, 32, 82, 69, 83, 85, 80, - 27, 33, 6, 73, 65, 78, 71, 76, 69, 24, 128, 1, 10, 32, 78, 79, 84, 69, - 72, 69, 65, 68, 32, 61, 17, 45, 82, 79, 85, 78, 68, 32, 78, 79, 84, 69, - 72, 69, 65, 68, 32, 68, 20, 58, 68, 24, 3, 85, 80, 32, 38, 82, 25, 3, 76, - 69, 70, 4, 93, 3, 79, 87, 78, 8, 34, 82, 78, 87, 207, 247, 11, 66, 4, 21, - 3, 73, 71, 72, 4, 11, 84, 4, 11, 32, 4, 26, 87, 207, 247, 11, 66, 2, 11, - 72, 2, 187, 189, 13, 73, 7, 11, 32, 4, 222, 160, 8, 83, 223, 224, 6, 85, - 4, 36, 3, 79, 73, 68, 187, 190, 14, 73, 2, 221, 178, 12, 5, 32, 78, 79, - 84, 69, 6, 92, 4, 72, 79, 76, 69, 157, 174, 8, 13, 73, 84, 72, 32, 70, - 73, 78, 71, 69, 82, 78, 65, 73, 4, 11, 32, 4, 246, 178, 3, 78, 163, 169, - 3, 82, 192, 3, 252, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, - 73, 71, 78, 32, 254, 1, 76, 216, 14, 16, 77, 79, 68, 73, 70, 73, 69, 82, - 32, 76, 69, 84, 84, 69, 82, 32, 122, 83, 166, 10, 84, 216, 1, 11, 86, 79, - 87, 69, 76, 32, 83, 73, 71, 78, 32, 223, 241, 12, 68, 16, 66, 77, 165, 1, - 11, 83, 72, 65, 78, 32, 77, 69, 68, 73, 65, 76, 14, 80, 6, 69, 68, 73, - 65, 76, 32, 45, 10, 79, 78, 32, 77, 69, 68, 73, 65, 76, 32, 8, 238, 249, - 14, 72, 2, 82, 2, 87, 3, 89, 6, 194, 249, 14, 76, 2, 77, 3, 78, 2, 219, - 182, 10, 32, 238, 1, 104, 6, 69, 84, 84, 69, 82, 32, 189, 13, 15, 79, 71, - 79, 71, 82, 65, 77, 32, 75, 72, 65, 77, 84, 73, 32, 232, 1, 242, 1, 65, - 50, 69, 146, 1, 71, 50, 75, 254, 1, 77, 134, 1, 78, 58, 82, 86, 83, 130, - 3, 84, 198, 1, 87, 242, 133, 11, 68, 190, 4, 86, 214, 20, 85, 210, 200, - 1, 73, 42, 76, 242, 190, 1, 66, 2, 67, 2, 74, 2, 80, 254, 68, 72, 2, 89, - 187, 2, 79, 7, 140, 174, 14, 4, 73, 84, 79, 78, 207, 74, 85, 9, 77, 17, - 65, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 6, - 42, 71, 182, 178, 10, 89, 187, 138, 3, 78, 2, 179, 178, 10, 72, 6, 214, - 163, 9, 82, 142, 209, 5, 72, 187, 2, 65, 44, 32, 2, 72, 65, 203, 246, 14, - 65, 43, 25, 4, 77, 84, 73, 32, 40, 134, 1, 68, 34, 84, 162, 191, 8, 78, - 178, 238, 5, 67, 2, 72, 2, 74, 158, 37, 76, 226, 31, 70, 2, 71, 2, 82, 2, - 83, 2, 88, 3, 90, 6, 238, 173, 14, 68, 255, 68, 72, 4, 207, 173, 14, 84, - 12, 36, 3, 79, 78, 32, 203, 244, 14, 65, 10, 60, 2, 66, 66, 206, 252, 12, - 74, 134, 181, 1, 78, 187, 66, 69, 4, 134, 244, 14, 65, 3, 69, 10, 134, - 190, 8, 78, 174, 179, 6, 71, 2, 89, 187, 2, 65, 4, 196, 214, 2, 12, 85, - 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 239, 156, 12, 65, 50, 90, 72, - 136, 215, 2, 9, 71, 65, 87, 32, 75, 65, 82, 69, 78, 198, 152, 12, 83, - 187, 2, 65, 44, 66, 65, 197, 1, 11, 87, 69, 32, 80, 65, 76, 65, 85, 78, - 71, 32, 41, 17, 2, 78, 32, 38, 134, 1, 78, 234, 248, 12, 74, 2, 80, 2, - 84, 138, 176, 1, 66, 2, 67, 2, 71, 2, 75, 254, 68, 68, 2, 70, 2, 72, 2, - 90, 187, 2, 65, 6, 234, 237, 14, 78, 2, 89, 187, 2, 65, 4, 190, 248, 12, - 67, 3, 83, 36, 38, 65, 134, 168, 14, 84, 255, 68, 72, 31, 41, 8, 73, 32, - 76, 65, 73, 78, 71, 32, 28, 82, 78, 142, 134, 11, 68, 250, 160, 3, 66, 2, - 71, 2, 74, 158, 37, 76, 227, 31, 70, 4, 254, 235, 14, 78, 3, 89, 6, 92, - 17, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, - 191, 237, 14, 65, 4, 190, 168, 10, 80, 195, 205, 2, 84, 6, 170, 225, 13, - 79, 154, 60, 81, 255, 62, 72, 4, 56, 6, 75, 72, 65, 77, 84, 73, 1, 4, 83, - 72, 65, 78, 2, 29, 5, 32, 82, 69, 68, 85, 2, 133, 132, 1, 2, 80, 76, 90, - 88, 4, 73, 71, 78, 32, 232, 6, 6, 89, 77, 66, 79, 76, 32, 145, 143, 9, 3, - 72, 65, 78, 52, 202, 3, 65, 32, 12, 75, 72, 65, 77, 84, 73, 32, 84, 79, - 78, 69, 45, 30, 83, 132, 2, 15, 84, 65, 73, 32, 76, 65, 73, 78, 71, 32, - 84, 79, 78, 69, 45, 28, 22, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, - 32, 75, 65, 82, 69, 78, 32, 84, 79, 78, 69, 154, 238, 2, 68, 128, 145, 5, - 19, 82, 85, 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 32, 84, 79, 78, - 69, 45, 156, 151, 3, 9, 80, 65, 79, 32, 75, 65, 82, 69, 78, 160, 91, 6, - 76, 73, 84, 84, 76, 69, 159, 176, 2, 86, 4, 250, 211, 13, 83, 243, 78, - 78, 4, 170, 231, 14, 49, 3, 51, 18, 40, 4, 72, 65, 78, 32, 151, 150, 14, - 69, 16, 84, 8, 67, 79, 85, 78, 67, 73, 76, 32, 84, 5, 84, 79, 78, 69, 45, - 159, 231, 13, 83, 6, 204, 151, 11, 8, 69, 77, 80, 72, 65, 84, 73, 67, - 189, 205, 3, 4, 84, 79, 78, 69, 8, 182, 229, 14, 50, 2, 51, 2, 53, 3, 54, - 4, 138, 229, 14, 50, 3, 53, 10, 11, 45, 10, 226, 228, 14, 49, 2, 50, 2, - 51, 2, 52, 3, 53, 18, 130, 1, 65, 132, 1, 2, 76, 79, 28, 5, 83, 72, 65, - 78, 32, 232, 148, 8, 4, 71, 69, 78, 73, 201, 85, 6, 67, 79, 77, 80, 76, - 69, 8, 84, 5, 73, 84, 79, 78, 32, 217, 152, 6, 10, 70, 79, 82, 69, 77, - 69, 78, 84, 73, 79, 6, 98, 69, 130, 213, 8, 84, 223, 243, 4, 79, 2, 129, - 149, 8, 2, 67, 65, 4, 26, 69, 223, 200, 13, 79, 2, 205, 12, 4, 88, 67, - 76, 65, 24, 140, 1, 20, 79, 78, 69, 32, 77, 65, 82, 75, 32, 83, 71, 65, - 87, 32, 75, 65, 82, 69, 78, 32, 161, 139, 9, 8, 65, 73, 32, 76, 65, 73, - 78, 71, 4, 42, 72, 253, 163, 10, 4, 75, 69, 32, 80, 2, 161, 188, 9, 2, - 65, 84, 56, 154, 2, 65, 76, 9, 71, 69, 66, 65, 32, 75, 65, 82, 69, 20, 6, - 75, 65, 89, 65, 72, 32, 40, 4, 77, 79, 78, 32, 34, 83, 154, 1, 69, 40, - 18, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, - 32, 144, 146, 9, 2, 84, 65, 246, 228, 1, 86, 214, 20, 85, 211, 200, 1, - 73, 8, 26, 73, 195, 221, 14, 65, 7, 25, 4, 84, 79, 78, 32, 4, 215, 214, - 12, 65, 2, 139, 157, 14, 78, 6, 166, 198, 14, 69, 2, 79, 215, 22, 85, 4, - 250, 200, 14, 73, 219, 19, 79, 10, 72, 10, 71, 65, 87, 32, 75, 65, 82, - 69, 78, 32, 21, 4, 72, 65, 78, 32, 2, 255, 182, 14, 69, 8, 54, 69, 20, 5, - 70, 73, 78, 65, 76, 211, 216, 14, 65, 5, 179, 222, 13, 32, 2, 191, 201, - 14, 32, 4, 138, 182, 14, 69, 151, 14, 85, 216, 17, 128, 2, 5, 45, 65, 82, - 89, 32, 214, 4, 65, 242, 16, 66, 30, 69, 206, 31, 73, 136, 1, 3, 75, 79, - 32, 154, 10, 79, 162, 24, 85, 224, 8, 22, 89, 73, 65, 75, 69, 78, 71, 32, - 80, 85, 65, 67, 72, 85, 69, 32, 72, 77, 79, 78, 71, 32, 232, 246, 13, 2, - 78, 66, 27, 76, 32, 130, 1, 67, 114, 84, 46, 83, 160, 1, 5, 85, 78, 73, - 79, 78, 108, 4, 87, 72, 73, 84, 174, 168, 11, 76, 254, 16, 73, 135, 140, - 2, 80, 8, 52, 7, 73, 82, 67, 76, 69, 68, 32, 147, 200, 13, 79, 6, 40, 2, - 80, 76, 14, 84, 167, 244, 7, 68, 2, 35, 85, 2, 21, 3, 73, 77, 69, 2, 147, - 172, 11, 83, 6, 40, 6, 81, 85, 65, 82, 69, 32, 87, 85, 4, 60, 9, 73, 78, - 84, 69, 82, 83, 69, 67, 84, 1, 2, 85, 78, 2, 255, 146, 11, 73, 2, 11, 77, - 2, 235, 168, 11, 77, 7, 69, 15, 32, 79, 80, 69, 82, 65, 84, 79, 82, 32, - 87, 73, 84, 72, 32, 4, 198, 247, 10, 80, 211, 140, 3, 68, 2, 11, 69, 2, - 145, 132, 12, 2, 32, 86, 188, 2, 170, 2, 66, 252, 4, 10, 71, 32, 77, 85, - 78, 68, 65, 82, 73, 32, 154, 4, 73, 52, 2, 78, 68, 216, 4, 7, 84, 73, 79, - 78, 65, 76, 32, 240, 206, 1, 2, 85, 83, 136, 236, 7, 5, 77, 69, 32, 66, - 65, 148, 172, 3, 7, 90, 65, 82, 32, 65, 77, 85, 196, 157, 1, 8, 82, 82, - 79, 87, 32, 78, 79, 45, 219, 62, 75, 82, 52, 7, 65, 84, 65, 69, 65, 78, - 32, 219, 206, 14, 76, 80, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, 236, 2, - 7, 78, 85, 77, 66, 69, 82, 32, 193, 207, 9, 16, 67, 82, 85, 67, 73, 70, - 79, 82, 77, 32, 78, 85, 77, 66, 69, 82, 62, 236, 1, 6, 70, 73, 78, 65, - 76, 32, 174, 253, 1, 84, 246, 118, 68, 34, 76, 50, 81, 206, 194, 1, 82, - 134, 212, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, - 75, 130, 76, 66, 190, 173, 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, - 18, 138, 244, 2, 65, 54, 76, 174, 149, 4, 83, 190, 3, 89, 130, 199, 1, - 75, 130, 76, 66, 190, 173, 4, 78, 198, 105, 72, 171, 4, 77, 16, 154, 245, - 2, 84, 186, 151, 4, 79, 143, 211, 5, 70, 84, 84, 7, 76, 69, 84, 84, 69, - 82, 32, 164, 2, 5, 83, 73, 71, 78, 32, 183, 219, 12, 68, 54, 42, 65, 50, - 69, 66, 73, 50, 79, 47, 85, 11, 254, 185, 14, 78, 202, 17, 66, 2, 72, 3, - 74, 15, 150, 211, 12, 78, 230, 110, 76, 242, 108, 84, 174, 28, 71, 3, 77, - 11, 194, 132, 14, 68, 150, 70, 72, 2, 83, 3, 84, 11, 210, 201, 14, 78, - 86, 76, 2, 80, 3, 89, 11, 250, 201, 14, 67, 2, 68, 2, 75, 3, 82, 10, 112, - 2, 77, 85, 20, 3, 83, 85, 84, 172, 165, 2, 2, 73, 75, 184, 129, 9, 2, 79, - 74, 137, 233, 2, 3, 84, 79, 89, 2, 255, 143, 14, 72, 2, 183, 199, 14, 85, - 4, 236, 137, 11, 5, 76, 32, 80, 79, 76, 231, 2, 82, 133, 1, 41, 8, 73, - 78, 65, 71, 65, 82, 73, 32, 130, 1, 140, 1, 7, 76, 69, 84, 84, 69, 82, - 32, 252, 1, 5, 83, 73, 71, 78, 32, 52, 11, 86, 79, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 199, 207, 4, 72, 94, 214, 1, 86, 202, 164, 10, 82, 206, - 55, 65, 38, 68, 46, 84, 230, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, - 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, - 77, 2, 89, 186, 2, 69, 3, 79, 6, 242, 254, 4, 79, 243, 197, 9, 65, 10, - 202, 205, 9, 83, 134, 144, 1, 65, 243, 163, 3, 86, 24, 210, 213, 4, 80, - 190, 40, 86, 166, 225, 5, 65, 190, 21, 85, 210, 200, 1, 73, 206, 134, 2, - 69, 3, 79, 4, 202, 47, 68, 191, 210, 13, 80, 4, 250, 194, 14, 83, 15, 72, - 246, 4, 216, 1, 3, 71, 65, 84, 128, 7, 6, 73, 84, 72, 69, 82, 32, 194, 3, - 83, 136, 1, 2, 85, 84, 242, 1, 87, 144, 16, 3, 88, 84, 32, 140, 150, 8, - 4, 80, 84, 85, 78, 240, 239, 2, 2, 67, 75, 150, 141, 2, 82, 183, 144, 1, - 76, 154, 1, 152, 1, 4, 73, 86, 69, 32, 245, 160, 12, 27, 69, 68, 32, 68, - 79, 85, 66, 76, 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, - 32, 68, 79, 85, 66, 152, 1, 152, 1, 8, 67, 73, 82, 67, 76, 69, 68, 32, - 216, 1, 9, 68, 73, 65, 71, 79, 78, 65, 76, 32, 196, 1, 8, 83, 81, 85, 65, - 82, 69, 68, 32, 147, 170, 8, 65, 78, 104, 5, 68, 73, 71, 73, 84, 28, 7, - 78, 85, 77, 66, 69, 82, 32, 142, 3, 76, 238, 150, 13, 84, 139, 3, 83, 2, - 189, 209, 12, 2, 32, 90, 20, 50, 84, 218, 254, 1, 69, 46, 70, 42, 78, 31, - 83, 6, 154, 128, 2, 72, 47, 87, 6, 38, 77, 234, 254, 12, 67, 191, 2, 68, - 2, 49, 10, 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, 2, 17, 2, 84, 32, 2, - 25, 4, 84, 79, 32, 76, 2, 25, 4, 79, 87, 69, 82, 2, 169, 198, 11, 2, 32, - 67, 66, 118, 76, 248, 229, 2, 4, 67, 82, 79, 83, 142, 138, 9, 81, 146, - 144, 1, 65, 198, 19, 73, 2, 87, 170, 164, 1, 80, 3, 83, 52, 221, 209, 8, - 6, 65, 84, 73, 78, 32, 67, 18, 158, 1, 65, 216, 1, 17, 71, 82, 69, 65, - 84, 69, 82, 45, 84, 72, 65, 78, 32, 78, 79, 82, 32, 37, 14, 76, 69, 83, - 83, 45, 84, 72, 65, 78, 32, 78, 79, 82, 32, 6, 92, 3, 32, 83, 85, 85, 16, - 80, 80, 82, 79, 88, 73, 77, 65, 84, 69, 76, 89, 32, 78, 79, 82, 4, 30, - 66, 1, 3, 80, 69, 82, 2, 253, 223, 6, 8, 83, 69, 84, 32, 79, 70, 32, 78, - 2, 213, 48, 5, 32, 65, 67, 84, 85, 6, 250, 246, 7, 69, 191, 177, 4, 76, - 6, 214, 246, 7, 69, 155, 177, 4, 71, 6, 26, 84, 199, 179, 12, 83, 4, 76, - 5, 73, 78, 71, 32, 68, 145, 176, 10, 8, 32, 87, 73, 84, 72, 32, 69, 71, - 2, 133, 228, 7, 2, 79, 76, 64, 40, 4, 82, 65, 76, 32, 203, 252, 13, 69, - 62, 48, 6, 67, 72, 69, 83, 83, 32, 235, 247, 13, 70, 60, 74, 75, 134, - 218, 12, 66, 38, 69, 130, 5, 80, 22, 81, 38, 82, 131, 2, 84, 20, 44, 5, - 78, 73, 71, 72, 84, 151, 219, 12, 73, 15, 195, 219, 12, 32, 246, 2, 70, - 32, 184, 8, 2, 65, 32, 248, 6, 3, 76, 73, 78, 183, 206, 3, 83, 174, 1, - 90, 77, 64, 4, 83, 72, 69, 81, 20, 8, 84, 65, 73, 32, 76, 85, 69, 32, - 183, 152, 13, 76, 4, 25, 4, 79, 79, 78, 32, 4, 234, 244, 7, 87, 171, 170, - 5, 83, 2, 183, 154, 9, 69, 166, 1, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, - 244, 2, 8, 83, 73, 71, 78, 32, 76, 65, 69, 22, 84, 76, 11, 86, 79, 87, - 69, 76, 32, 83, 73, 71, 78, 32, 191, 190, 12, 68, 102, 76, 6, 70, 73, 78, - 65, 76, 32, 68, 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 14, 198, 183, 12, - 78, 150, 248, 1, 66, 2, 68, 2, 75, 2, 77, 3, 86, 44, 11, 32, 44, 146, 1, - 75, 2, 88, 34, 83, 210, 222, 10, 78, 134, 135, 3, 84, 82, 80, 254, 68, - 66, 2, 68, 2, 70, 2, 72, 2, 76, 2, 77, 2, 81, 2, 86, 3, 89, 4, 190, 171, - 14, 86, 187, 2, 65, 4, 158, 171, 14, 85, 187, 2, 65, 5, 183, 173, 14, 86, - 6, 224, 133, 7, 3, 79, 78, 69, 253, 76, 8, 72, 65, 77, 32, 68, 73, 71, - 73, 34, 110, 65, 46, 73, 30, 79, 38, 85, 216, 231, 11, 11, 86, 79, 87, - 69, 76, 32, 83, 72, 79, 82, 84, 167, 195, 2, 69, 8, 238, 184, 10, 65, - 250, 242, 3, 69, 3, 89, 4, 186, 171, 14, 73, 3, 89, 9, 166, 184, 10, 65, - 251, 242, 3, 89, 11, 130, 184, 10, 69, 250, 242, 3, 85, 3, 89, 194, 1, - 182, 1, 68, 106, 71, 72, 7, 76, 69, 84, 84, 69, 82, 32, 214, 2, 83, 190, - 23, 80, 202, 133, 1, 86, 138, 200, 7, 65, 144, 230, 1, 5, 73, 78, 83, 69, - 82, 210, 142, 1, 67, 207, 186, 2, 79, 26, 64, 6, 79, 85, 66, 76, 69, 32, - 214, 211, 4, 65, 239, 230, 7, 73, 4, 198, 211, 4, 68, 231, 137, 7, 67, 2, - 11, 65, 2, 11, 80, 2, 17, 2, 32, 70, 2, 213, 224, 10, 2, 73, 76, 108, - 218, 1, 78, 62, 86, 254, 188, 10, 65, 38, 68, 46, 84, 230, 24, 85, 210, - 200, 1, 73, 202, 190, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, - 2, 77, 2, 80, 2, 82, 254, 68, 72, 2, 87, 2, 89, 186, 2, 69, 3, 79, 14, - 210, 222, 13, 71, 2, 89, 254, 68, 72, 2, 78, 187, 2, 65, 10, 26, 69, 207, - 193, 10, 79, 2, 173, 246, 11, 3, 68, 73, 67, 22, 26, 73, 139, 179, 4, 65, - 20, 44, 3, 71, 78, 32, 213, 128, 9, 2, 68, 68, 18, 210, 189, 10, 65, 74, - 67, 98, 78, 230, 179, 1, 74, 228, 2, 5, 70, 73, 78, 65, 76, 50, 85, 211, - 235, 1, 86, 4, 203, 248, 13, 69, 4, 250, 200, 9, 80, 171, 193, 3, 76, 6, - 70, 78, 245, 141, 13, 11, 71, 72, 84, 32, 87, 73, 84, 72, 32, 83, 84, 4, - 156, 217, 7, 7, 69, 32, 80, 79, 73, 78, 84, 131, 199, 6, 74, 124, 152, 1, - 3, 67, 79, 77, 242, 2, 68, 78, 76, 156, 4, 4, 72, 73, 71, 72, 72, 7, 83, - 89, 77, 66, 79, 76, 32, 238, 192, 7, 69, 133, 178, 1, 3, 84, 65, 77, 20, - 52, 7, 66, 73, 78, 73, 78, 71, 32, 143, 158, 14, 77, 18, 116, 5, 76, 79, - 78, 71, 32, 76, 2, 78, 65, 20, 6, 83, 72, 79, 82, 84, 32, 165, 158, 13, - 6, 68, 79, 85, 66, 76, 69, 8, 130, 1, 72, 34, 76, 194, 207, 10, 82, 21, - 7, 68, 69, 83, 67, 69, 78, 68, 2, 155, 172, 10, 83, 6, 34, 72, 34, 76, - 195, 207, 10, 82, 2, 137, 208, 10, 3, 73, 71, 72, 2, 233, 207, 10, 2, 79, - 87, 24, 152, 1, 4, 65, 78, 84, 65, 152, 133, 12, 4, 79, 82, 79, 77, 203, - 41, 73, 70, 76, 4, 65, 74, 65, 78, 32, 6, 69, 84, 84, 69, 82, 32, 173, 3, - 2, 79, 87, 2, 185, 213, 12, 3, 89, 65, 76, 66, 228, 1, 2, 68, 65, 42, 74, - 90, 78, 206, 249, 9, 82, 158, 225, 1, 79, 198, 71, 67, 170, 183, 1, 69, - 238, 18, 71, 242, 42, 66, 2, 70, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, - 83, 2, 84, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 5, 237, 248, 10, 5, - 71, 66, 65, 83, 73, 8, 40, 4, 79, 78, 65, 32, 151, 154, 14, 65, 6, 214, - 162, 12, 67, 134, 245, 1, 74, 3, 82, 11, 26, 65, 1, 2, 89, 65, 5, 173, - 193, 5, 5, 32, 87, 79, 76, 79, 2, 29, 5, 32, 84, 79, 78, 69, 2, 17, 2, - 32, 65, 2, 135, 171, 8, 80, 4, 68, 7, 71, 66, 65, 75, 85, 82, 85, 1, 6, - 79, 79, 32, 68, 69, 78, 2, 159, 199, 12, 78, 180, 1, 90, 32, 156, 3, 2, - 77, 73, 118, 78, 150, 1, 82, 238, 7, 84, 186, 203, 13, 45, 135, 40, 83, - 18, 202, 1, 66, 96, 5, 69, 78, 84, 82, 89, 22, 80, 160, 180, 2, 15, 79, - 78, 69, 32, 85, 78, 68, 69, 82, 32, 69, 73, 71, 72, 84, 220, 242, 3, 9, - 77, 79, 66, 73, 76, 69, 32, 80, 72, 181, 30, 3, 83, 77, 79, 4, 52, 4, 82, - 69, 65, 75, 173, 131, 2, 3, 73, 67, 89, 2, 17, 2, 32, 72, 2, 231, 244, - 12, 69, 5, 151, 142, 13, 32, 4, 60, 6, 69, 68, 69, 83, 84, 82, 161, 167, - 9, 3, 73, 82, 65, 2, 137, 230, 10, 2, 73, 65, 4, 36, 5, 78, 65, 76, 32, - 68, 59, 83, 2, 249, 255, 12, 9, 73, 71, 73, 84, 32, 83, 72, 65, 80, 2, - 195, 215, 10, 77, 6, 38, 45, 185, 183, 9, 3, 70, 79, 82, 4, 76, 8, 66, - 82, 69, 65, 75, 73, 78, 71, 245, 162, 2, 5, 80, 79, 84, 65, 66, 2, 129, - 201, 6, 2, 32, 72, 93, 56, 2, 84, 72, 146, 11, 77, 173, 185, 10, 3, 68, - 73, 67, 84, 78, 32, 213, 131, 11, 13, 69, 65, 83, 84, 45, 80, 79, 73, 78, - 84, 73, 78, 71, 82, 96, 5, 69, 65, 83, 84, 32, 132, 2, 6, 73, 78, 68, 73, - 67, 32, 217, 1, 5, 87, 69, 83, 84, 32, 30, 82, 65, 182, 221, 6, 80, 130, - 1, 84, 246, 160, 4, 66, 38, 68, 18, 83, 203, 43, 87, 12, 40, 4, 82, 82, - 79, 87, 175, 217, 6, 78, 11, 11, 32, 8, 80, 9, 67, 82, 79, 83, 83, 73, - 78, 71, 32, 244, 2, 2, 65, 78, 255, 216, 6, 87, 4, 218, 180, 3, 83, 207, - 166, 3, 78, 20, 50, 80, 60, 3, 81, 85, 65, 66, 82, 215, 126, 70, 2, 37, - 7, 76, 65, 67, 69, 72, 79, 76, 2, 135, 158, 4, 68, 4, 240, 157, 4, 2, 82, - 84, 237, 173, 9, 5, 78, 84, 73, 84, 89, 2, 17, 2, 85, 80, 2, 187, 144, 4, - 69, 32, 82, 65, 218, 217, 6, 80, 130, 1, 84, 246, 160, 4, 66, 38, 68, 18, - 83, 203, 43, 87, 14, 34, 78, 33, 4, 82, 82, 79, 87, 2, 229, 177, 3, 3, - 68, 32, 83, 13, 11, 32, 10, 68, 3, 84, 79, 32, 138, 213, 6, 67, 40, 3, - 65, 78, 68, 219, 2, 87, 4, 146, 214, 6, 67, 229, 228, 4, 4, 76, 79, 78, - 71, 56, 54, 32, 236, 5, 5, 67, 72, 69, 68, 32, 207, 2, 69, 38, 162, 1, - 65, 132, 2, 4, 78, 79, 82, 77, 78, 80, 46, 83, 170, 1, 84, 250, 195, 7, - 69, 196, 25, 7, 73, 68, 69, 78, 84, 73, 67, 214, 151, 4, 71, 38, 76, 243, - 77, 67, 10, 88, 3, 32, 83, 85, 52, 7, 78, 32, 69, 76, 69, 77, 69, 28, 2, - 83, 89, 139, 223, 7, 76, 4, 30, 66, 1, 3, 80, 69, 82, 2, 29, 2, 83, 69, - 2, 11, 78, 2, 243, 119, 84, 2, 37, 7, 77, 80, 84, 79, 84, 73, 67, 2, 17, - 2, 65, 76, 2, 201, 223, 7, 2, 76, 89, 4, 181, 174, 6, 14, 65, 76, 32, 83, - 85, 66, 71, 82, 79, 85, 80, 32, 79, 70, 2, 153, 223, 7, 6, 65, 82, 65, - 76, 76, 69, 6, 48, 6, 81, 85, 65, 82, 69, 32, 195, 254, 12, 73, 4, 68, 5, - 73, 77, 65, 71, 69, 1, 8, 79, 82, 73, 71, 73, 78, 65, 76, 2, 21, 3, 32, - 79, 70, 2, 183, 172, 6, 32, 4, 178, 163, 10, 82, 227, 202, 1, 73, 8, 58, - 76, 40, 4, 82, 73, 71, 72, 133, 1, 3, 85, 80, 80, 4, 36, 2, 69, 70, 133, - 1, 2, 79, 87, 2, 89, 20, 84, 32, 83, 69, 77, 73, 67, 73, 82, 67, 76, 69, - 32, 87, 73, 84, 72, 32, 84, 72, 2, 17, 2, 82, 69, 2, 207, 180, 12, 69, 2, - 11, 69, 2, 41, 8, 82, 32, 82, 73, 71, 72, 84, 45, 2, 153, 131, 3, 6, 83, - 72, 65, 68, 79, 87, 11, 34, 32, 53, 4, 66, 79, 79, 75, 4, 17, 2, 80, 65, - 4, 146, 234, 13, 71, 215, 22, 68, 5, 81, 18, 32, 87, 73, 84, 72, 32, 68, - 69, 67, 79, 82, 65, 84, 73, 86, 69, 32, 67, 2, 223, 251, 3, 79, 186, 6, - 102, 77, 176, 3, 4, 83, 72, 85, 32, 168, 157, 5, 8, 84, 32, 65, 78, 68, - 32, 66, 79, 131, 226, 7, 76, 26, 36, 4, 66, 69, 82, 32, 247, 2, 69, 24, - 58, 69, 50, 70, 42, 83, 66, 84, 57, 4, 78, 73, 78, 69, 4, 204, 1, 3, 73, - 71, 72, 21, 3, 76, 69, 86, 4, 144, 1, 2, 79, 85, 13, 2, 73, 70, 6, 34, - 73, 85, 4, 69, 86, 69, 78, 4, 82, 88, 239, 172, 13, 71, 8, 40, 2, 72, 73, - 46, 69, 21, 2, 87, 69, 2, 11, 82, 2, 17, 2, 84, 69, 2, 11, 69, 2, 143, - 175, 7, 78, 4, 248, 174, 7, 3, 76, 86, 69, 1, 3, 78, 84, 89, 2, 155, 221, - 6, 82, 154, 6, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, - 215, 222, 11, 73, 152, 6, 18, 49, 87, 50, 160, 2, 222, 1, 55, 2, 56, 2, - 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 248, 3, 138, 1, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, - 67, 2, 68, 2, 69, 143, 1, 70, 32, 246, 248, 13, 48, 2, 49, 2, 50, 2, 51, - 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, - 69, 3, 70, 24, 234, 247, 13, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 2, 57, 2, 65, 3, 66, 142, 1, 108, 8, 67, 73, 82, 67, - 76, 69, 68, 32, 22, 76, 226, 3, 83, 160, 2, 5, 84, 79, 78, 69, 45, 223, - 129, 12, 68, 2, 211, 243, 13, 67, 92, 88, 6, 69, 84, 84, 69, 82, 32, 141, - 157, 1, 10, 79, 71, 79, 71, 82, 65, 77, 32, 78, 89, 90, 130, 2, 78, 90, - 84, 222, 188, 7, 88, 186, 116, 65, 146, 161, 1, 82, 188, 44, 2, 72, 65, - 226, 180, 1, 79, 222, 56, 68, 2, 77, 2, 80, 146, 198, 1, 69, 222, 61, 67, - 2, 70, 2, 71, 2, 75, 2, 76, 2, 81, 2, 83, 2, 86, 2, 89, 2, 90, 186, 2, - 73, 2, 85, 3, 87, 22, 86, 84, 158, 236, 11, 80, 238, 131, 2, 67, 2, 75, - 2, 81, 2, 82, 2, 89, 187, 2, 65, 6, 134, 240, 13, 83, 2, 88, 187, 2, 65, - 14, 64, 4, 73, 71, 78, 32, 185, 1, 7, 89, 76, 76, 65, 66, 76, 69, 12, 56, - 4, 70, 79, 82, 32, 137, 243, 12, 4, 88, 87, 32, 88, 10, 152, 9, 2, 76, - 79, 190, 147, 3, 84, 132, 89, 8, 73, 78, 86, 69, 82, 84, 69, 66, 154, - 168, 3, 80, 237, 86, 3, 65, 78, 73, 2, 221, 172, 11, 4, 32, 76, 69, 78, - 14, 246, 239, 13, 66, 2, 68, 2, 71, 2, 74, 2, 77, 2, 83, 3, 86, 214, 13, - 178, 2, 66, 226, 1, 67, 224, 5, 4, 70, 70, 73, 67, 54, 71, 248, 6, 4, 73, - 76, 32, 68, 22, 76, 222, 63, 78, 166, 5, 80, 214, 7, 82, 190, 10, 83, - 212, 8, 2, 84, 84, 154, 8, 85, 180, 1, 3, 86, 69, 82, 234, 159, 2, 89, - 138, 180, 5, 72, 190, 233, 1, 75, 174, 230, 1, 68, 230, 61, 77, 246, 9, - 87, 159, 137, 1, 88, 10, 132, 1, 6, 76, 73, 81, 85, 69, 32, 140, 152, 2, - 9, 83, 69, 82, 86, 69, 82, 32, 69, 89, 221, 28, 8, 74, 69, 67, 84, 32, - 82, 69, 80, 6, 160, 177, 4, 13, 65, 78, 71, 76, 69, 32, 79, 80, 69, 78, - 73, 78, 71, 191, 241, 1, 72, 28, 56, 2, 82, 32, 234, 4, 84, 229, 172, 5, - 3, 67, 85, 76, 22, 156, 1, 11, 65, 77, 79, 85, 78, 84, 32, 79, 70, 32, - 67, 22, 66, 198, 1, 67, 118, 68, 106, 70, 0, 10, 73, 78, 86, 69, 82, 84, - 69, 68, 32, 70, 175, 149, 12, 72, 2, 207, 158, 5, 72, 6, 136, 1, 15, 82, - 65, 78, 67, 72, 32, 66, 65, 78, 75, 32, 73, 68, 69, 78, 144, 233, 4, 5, - 69, 76, 84, 32, 66, 201, 225, 5, 3, 79, 87, 32, 2, 21, 3, 84, 73, 70, 2, - 11, 73, 2, 199, 187, 10, 67, 4, 92, 17, 85, 83, 84, 79, 77, 69, 82, 32, - 65, 67, 67, 79, 85, 78, 84, 32, 78, 227, 195, 1, 72, 2, 159, 135, 6, 85, - 4, 44, 5, 79, 85, 66, 76, 69, 171, 129, 12, 65, 2, 11, 32, 2, 11, 66, 2, - 181, 133, 7, 3, 65, 67, 75, 2, 247, 164, 13, 79, 4, 184, 206, 8, 4, 65, - 71, 79, 78, 209, 148, 3, 2, 79, 80, 2, 11, 69, 2, 213, 215, 7, 5, 32, 66, - 85, 73, 76, 60, 48, 4, 72, 65, 77, 32, 177, 225, 13, 2, 79, 78, 58, 122, - 70, 0, 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 70, 36, 7, 76, 69, 84, 84, - 69, 82, 32, 161, 231, 3, 3, 83, 80, 65, 2, 177, 244, 3, 4, 69, 65, 84, - 72, 52, 196, 1, 2, 65, 73, 22, 66, 2, 80, 34, 67, 36, 2, 69, 65, 64, 2, - 70, 69, 22, 71, 22, 73, 58, 76, 2, 82, 22, 78, 46, 79, 34, 83, 50, 85, - 150, 178, 1, 77, 170, 9, 68, 149, 138, 11, 3, 84, 73, 78, 2, 147, 209, - 13, 76, 2, 11, 69, 2, 199, 226, 11, 73, 4, 138, 239, 7, 69, 255, 237, 3, - 79, 6, 138, 1, 66, 2, 68, 201, 247, 3, 6, 77, 72, 65, 78, 67, 72, 2, 199, - 130, 9, 65, 2, 199, 217, 12, 79, 4, 32, 2, 79, 68, 207, 224, 12, 70, 2, - 235, 224, 7, 72, 2, 135, 195, 11, 85, 4, 176, 204, 12, 3, 71, 69, 65, - 207, 67, 73, 4, 222, 143, 13, 78, 215, 79, 82, 4, 158, 178, 11, 65, 161, - 171, 1, 3, 84, 82, 65, 6, 60, 5, 73, 76, 76, 69, 65, 162, 223, 11, 65, - 135, 255, 1, 82, 2, 207, 142, 13, 78, 2, 155, 182, 12, 82, 220, 7, 64, 7, - 32, 67, 72, 73, 75, 73, 32, 206, 6, 68, 167, 218, 12, 73, 96, 132, 1, 7, - 76, 69, 84, 84, 69, 82, 32, 148, 3, 2, 77, 85, 62, 71, 74, 80, 164, 198, - 3, 2, 82, 69, 214, 163, 8, 68, 243, 167, 1, 65, 60, 50, 65, 98, 69, 54, - 73, 50, 76, 62, 79, 51, 85, 16, 50, 65, 242, 218, 13, 78, 86, 71, 2, 76, - 3, 84, 8, 194, 219, 13, 74, 2, 75, 2, 77, 3, 87, 8, 130, 149, 13, 68, - 198, 13, 82, 210, 56, 78, 3, 80, 8, 154, 201, 13, 78, 202, 17, 72, 2, 82, - 3, 83, 12, 134, 153, 8, 65, 174, 193, 5, 69, 2, 73, 2, 79, 3, 85, 8, 202, - 189, 13, 84, 174, 28, 66, 2, 72, 3, 86, 8, 242, 137, 13, 78, 214, 79, 67, - 2, 68, 3, 89, 4, 56, 2, 45, 71, 221, 242, 11, 6, 32, 84, 84, 85, 68, 68, - 2, 217, 242, 11, 13, 65, 65, 72, 76, 65, 65, 32, 84, 84, 85, 68, 68, 65, - 6, 84, 11, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 137, 197, 6, 4, - 72, 65, 65, 82, 4, 48, 8, 68, 79, 85, 66, 76, 69, 32, 77, 3, 77, 2, 229, - 144, 13, 3, 85, 67, 65, 250, 6, 34, 32, 225, 55, 3, 69, 82, 32, 244, 6, - 240, 2, 8, 67, 72, 73, 78, 69, 83, 69, 32, 44, 10, 72, 85, 78, 71, 65, - 82, 73, 65, 78, 32, 244, 6, 7, 73, 84, 65, 76, 73, 67, 32, 212, 4, 14, - 78, 79, 82, 84, 72, 32, 65, 82, 65, 66, 73, 65, 78, 32, 196, 4, 3, 80, - 69, 82, 164, 12, 2, 83, 79, 144, 13, 14, 84, 85, 82, 75, 73, 67, 32, 76, - 69, 84, 84, 69, 82, 32, 132, 7, 7, 85, 89, 71, 72, 85, 82, 32, 151, 140, - 11, 75, 4, 166, 183, 11, 73, 205, 91, 3, 72, 79, 79, 216, 1, 96, 6, 67, - 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 221, 5, 7, 78, 85, 77, 66, 69, - 82, 32, 102, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 102, 226, 1, 65, - 54, 69, 164, 2, 10, 78, 73, 75, 79, 76, 83, 66, 85, 82, 71, 0, 9, 82, 85, - 68, 73, 77, 69, 78, 84, 65, 42, 79, 34, 85, 132, 223, 3, 5, 67, 76, 79, - 83, 69, 178, 231, 7, 73, 237, 200, 1, 6, 83, 72, 79, 82, 84, 32, 11, 250, - 149, 12, 77, 146, 116, 78, 150, 70, 65, 3, 75, 63, 174, 1, 77, 22, 78, - 78, 83, 146, 218, 9, 67, 86, 71, 2, 76, 2, 84, 154, 239, 1, 90, 226, 131, - 2, 66, 2, 68, 2, 69, 2, 70, 2, 72, 2, 74, 2, 75, 2, 80, 2, 82, 3, 86, 5, - 187, 206, 13, 80, 11, 34, 84, 134, 206, 13, 67, 3, 89, 5, 165, 145, 2, 5, - 45, 83, 72, 65, 80, 5, 219, 205, 13, 90, 4, 11, 32, 4, 230, 182, 13, 79, - 3, 85, 7, 202, 182, 13, 69, 215, 22, 79, 9, 230, 201, 13, 78, 154, 3, 83, - 3, 85, 12, 210, 4, 70, 234, 237, 5, 79, 171, 137, 6, 84, 78, 80, 7, 76, - 69, 84, 84, 69, 82, 32, 169, 3, 8, 78, 85, 77, 69, 82, 65, 76, 32, 70, - 190, 1, 69, 90, 75, 50, 83, 36, 3, 78, 79, 82, 206, 249, 9, 85, 210, 200, - 1, 73, 198, 134, 1, 80, 2, 84, 254, 82, 67, 186, 22, 66, 2, 68, 2, 72, 2, - 86, 2, 89, 2, 90, 214, 22, 65, 3, 79, 23, 198, 214, 9, 83, 238, 239, 1, - 82, 146, 198, 1, 75, 210, 61, 70, 2, 76, 2, 77, 3, 78, 8, 230, 178, 13, - 72, 214, 22, 65, 2, 69, 3, 85, 4, 32, 2, 79, 85, 151, 178, 13, 72, 2, 29, - 5, 84, 72, 69, 82, 78, 2, 209, 191, 12, 2, 32, 84, 8, 38, 70, 146, 247, - 11, 84, 227, 55, 79, 4, 11, 73, 4, 130, 222, 11, 70, 163, 211, 1, 86, 64, - 76, 7, 76, 69, 84, 84, 69, 82, 32, 209, 3, 7, 78, 85, 77, 66, 69, 82, 32, - 58, 210, 1, 65, 38, 71, 38, 72, 30, 75, 38, 84, 74, 90, 246, 104, 77, - 244, 138, 3, 2, 69, 83, 218, 156, 3, 66, 2, 70, 2, 82, 2, 89, 242, 161, - 3, 78, 246, 215, 1, 76, 150, 55, 68, 146, 1, 81, 254, 2, 87, 255, 55, 83, - 4, 138, 180, 3, 76, 211, 193, 9, 73, 4, 218, 153, 12, 72, 135, 153, 1, - 69, 4, 214, 195, 13, 65, 3, 69, 4, 130, 140, 6, 72, 191, 183, 6, 65, 8, - 34, 72, 246, 194, 13, 65, 3, 69, 4, 230, 186, 12, 65, 143, 136, 1, 69, 4, - 11, 65, 4, 254, 243, 12, 73, 215, 79, 72, 6, 190, 131, 6, 84, 227, 166, - 6, 79, 178, 1, 64, 11, 77, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 183, - 5, 83, 76, 228, 1, 2, 67, 72, 22, 68, 66, 69, 22, 73, 30, 77, 2, 78, 30, - 80, 22, 83, 70, 84, 50, 86, 38, 89, 94, 90, 218, 172, 6, 76, 2, 82, 254, - 209, 2, 71, 230, 255, 1, 79, 190, 198, 1, 66, 242, 40, 65, 242, 31, 75, - 174, 45, 72, 187, 2, 85, 2, 171, 180, 7, 69, 6, 26, 90, 219, 172, 13, 79, - 4, 138, 254, 8, 72, 219, 157, 4, 73, 5, 139, 192, 13, 70, 7, 246, 191, - 13, 65, 3, 69, 2, 249, 168, 13, 2, 69, 78, 2, 223, 174, 6, 69, 6, 26, 72, - 187, 171, 13, 73, 4, 232, 252, 8, 3, 67, 72, 79, 3, 79, 4, 26, 83, 247, - 170, 13, 65, 2, 227, 153, 13, 73, 4, 146, 252, 8, 79, 215, 137, 4, 69, - 14, 60, 2, 69, 82, 226, 136, 8, 65, 174, 163, 5, 82, 203, 17, 85, 7, 210, - 189, 13, 73, 3, 85, 4, 146, 251, 8, 72, 219, 157, 4, 65, 102, 52, 4, 73, - 65, 78, 32, 213, 153, 1, 3, 79, 78, 65, 100, 80, 7, 78, 85, 77, 66, 69, - 82, 32, 80, 5, 83, 73, 71, 78, 32, 139, 154, 10, 87, 10, 42, 84, 186, - 147, 8, 72, 147, 143, 4, 79, 6, 162, 228, 1, 87, 231, 135, 11, 69, 88, - 210, 1, 65, 86, 66, 62, 68, 98, 74, 2, 86, 30, 84, 42, 88, 226, 109, 71, - 2, 75, 2, 78, 2, 82, 250, 228, 8, 77, 214, 156, 3, 83, 206, 69, 67, 2, - 70, 2, 72, 2, 76, 2, 80, 2, 89, 2, 90, 186, 2, 73, 3, 85, 9, 45, 9, 85, - 82, 65, 77, 65, 90, 68, 65, 65, 7, 222, 213, 6, 45, 159, 225, 6, 72, 6, - 38, 65, 165, 250, 9, 3, 85, 85, 77, 5, 175, 182, 13, 71, 10, 34, 65, 178, - 184, 13, 73, 3, 85, 7, 37, 7, 72, 89, 65, 65, 85, 83, 72, 5, 179, 212, 6, - 45, 4, 242, 183, 13, 65, 3, 73, 6, 158, 181, 13, 72, 186, 2, 65, 3, 85, - 4, 132, 137, 11, 8, 83, 72, 65, 65, 89, 65, 84, 72, 171, 174, 2, 65, 144, - 1, 92, 6, 71, 68, 73, 65, 78, 32, 141, 7, 12, 85, 84, 72, 32, 65, 82, 65, - 66, 73, 65, 78, 32, 80, 58, 70, 74, 76, 145, 5, 7, 78, 85, 77, 66, 69, - 82, 32, 2, 11, 82, 2, 157, 155, 11, 10, 65, 67, 84, 73, 79, 78, 32, 79, - 78, 69, 60, 76, 6, 69, 84, 84, 69, 82, 32, 149, 4, 8, 73, 71, 65, 84, 85, - 82, 69, 32, 58, 234, 1, 65, 96, 6, 70, 73, 78, 65, 76, 32, 200, 1, 5, 82, - 69, 83, 72, 45, 230, 214, 1, 76, 178, 151, 4, 71, 90, 90, 34, 83, 66, 89, - 154, 193, 1, 72, 234, 5, 75, 130, 76, 66, 190, 173, 4, 78, 254, 1, 84, 2, - 87, 202, 103, 80, 171, 4, 77, 6, 26, 76, 243, 178, 12, 89, 4, 244, 240, - 5, 8, 84, 69, 82, 78, 65, 84, 69, 32, 239, 199, 1, 69, 18, 116, 3, 78, - 85, 78, 0, 5, 83, 65, 68, 72, 69, 0, 3, 84, 65, 87, 130, 215, 1, 65, 158, - 172, 6, 66, 131, 151, 5, 72, 5, 41, 8, 32, 87, 73, 84, 72, 32, 86, 69, 2, - 209, 199, 5, 4, 82, 84, 73, 67, 2, 193, 214, 1, 6, 65, 89, 73, 78, 45, - 68, 18, 42, 84, 158, 239, 5, 79, 143, 211, 5, 70, 10, 42, 72, 230, 215, - 1, 87, 231, 135, 11, 69, 4, 230, 194, 11, 82, 159, 2, 73, 64, 60, 7, 76, - 69, 84, 84, 69, 82, 32, 245, 3, 3, 78, 85, 77, 58, 202, 1, 65, 38, 68, - 74, 71, 34, 75, 34, 83, 78, 84, 218, 43, 90, 230, 165, 1, 76, 50, 81, - 206, 194, 1, 82, 238, 213, 2, 89, 154, 193, 1, 72, 234, 81, 66, 190, 173, - 4, 78, 254, 1, 87, 202, 103, 70, 171, 4, 77, 4, 202, 155, 3, 76, 211, - 193, 9, 89, 6, 32, 2, 72, 65, 219, 210, 1, 65, 4, 210, 254, 7, 76, 203, - 128, 5, 68, 4, 226, 44, 72, 163, 189, 5, 73, 4, 154, 178, 7, 65, 247, 75, - 72, 8, 26, 65, 239, 171, 12, 72, 6, 226, 163, 7, 77, 186, 218, 5, 68, - 143, 45, 84, 8, 166, 90, 72, 206, 209, 10, 69, 247, 128, 1, 65, 6, 56, 4, - 66, 69, 82, 32, 245, 240, 12, 4, 69, 82, 73, 67, 4, 26, 70, 223, 144, 12, - 79, 2, 207, 190, 11, 73, 146, 1, 80, 7, 79, 82, 75, 72, 79, 78, 32, 197, - 3, 8, 89, 69, 78, 73, 83, 69, 73, 32, 84, 54, 65, 202, 1, 69, 122, 73, - 30, 79, 203, 192, 11, 66, 45, 106, 69, 130, 180, 9, 83, 226, 243, 3, 66, - 2, 68, 2, 71, 2, 76, 2, 78, 2, 81, 2, 82, 2, 84, 3, 89, 20, 222, 167, 13, - 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 78, 2, 82, 2, 83, 2, 84, 3, 89, 20, - 74, 78, 142, 138, 13, 76, 158, 27, 83, 146, 1, 67, 2, 77, 2, 80, 3, 90, - 8, 182, 166, 13, 67, 2, 71, 2, 84, 3, 89, 7, 138, 166, 13, 67, 3, 81, 13, - 130, 3, 69, 238, 162, 13, 80, 2, 81, 3, 84, 62, 38, 65, 170, 1, 69, 86, - 73, 23, 79, 39, 98, 69, 166, 163, 13, 83, 62, 78, 86, 66, 2, 68, 2, 71, - 2, 76, 2, 81, 2, 82, 2, 84, 3, 89, 17, 158, 172, 11, 78, 150, 248, 1, 66, - 2, 71, 2, 75, 2, 84, 3, 89, 15, 46, 78, 178, 162, 13, 83, 146, 1, 67, 3, - 90, 6, 190, 163, 13, 67, 2, 84, 3, 89, 5, 155, 163, 13, 81, 6, 26, 69, - 239, 162, 13, 81, 5, 235, 162, 13, 75, 52, 148, 1, 10, 67, 79, 77, 66, - 73, 78, 73, 78, 71, 32, 36, 7, 76, 69, 84, 84, 69, 82, 32, 233, 1, 12, - 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 8, 146, 221, 5, 84, 243, - 213, 3, 68, 36, 170, 199, 1, 65, 178, 195, 1, 82, 214, 212, 2, 76, 58, - 90, 34, 83, 66, 89, 252, 195, 1, 6, 70, 73, 78, 65, 76, 32, 0, 6, 71, 73, - 77, 69, 76, 45, 134, 3, 75, 130, 76, 66, 190, 173, 4, 78, 254, 1, 84, 2, - 87, 202, 103, 80, 171, 4, 77, 8, 52, 4, 84, 87, 79, 32, 166, 130, 11, 70, - 179, 122, 66, 4, 150, 138, 12, 66, 71, 68, 6, 170, 180, 1, 87, 152, 140, - 3, 3, 65, 68, 85, 235, 150, 7, 77, 22, 212, 1, 34, 32, 87, 73, 84, 72, - 32, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, - 87, 73, 84, 72, 32, 76, 69, 70, 84, 32, 82, 44, 7, 67, 79, 77, 73, 78, - 71, 32, 210, 1, 69, 239, 202, 12, 73, 2, 21, 3, 73, 71, 72, 2, 151, 197, - 9, 84, 10, 144, 1, 6, 65, 85, 84, 79, 77, 79, 20, 7, 70, 73, 82, 69, 32, - 69, 78, 20, 6, 80, 79, 76, 73, 67, 69, 184, 212, 2, 2, 84, 65, 191, 195, - 8, 66, 2, 135, 253, 10, 66, 2, 211, 129, 12, 71, 2, 135, 206, 9, 32, 8, - 70, 32, 245, 230, 11, 11, 45, 80, 73, 69, 67, 69, 32, 83, 87, 73, 77, 6, - 40, 4, 68, 79, 84, 32, 147, 234, 9, 66, 4, 26, 79, 235, 235, 9, 76, 2, - 169, 221, 2, 11, 86, 69, 82, 32, 84, 87, 79, 32, 68, 79, 84, 46, 82, 69, - 168, 6, 2, 84, 73, 148, 143, 11, 5, 72, 73, 85, 67, 72, 167, 177, 1, 80, - 36, 70, 78, 181, 5, 12, 82, 65, 84, 73, 78, 71, 32, 83, 89, 83, 84, 69, - 34, 22, 32, 247, 3, 45, 28, 108, 2, 66, 79, 32, 7, 67, 69, 78, 84, 82, - 69, 32, 114, 70, 50, 72, 42, 77, 202, 233, 6, 83, 239, 194, 3, 76, 4, - 190, 147, 13, 79, 155, 3, 88, 8, 50, 84, 238, 130, 11, 66, 222, 2, 65, - 219, 82, 67, 2, 37, 7, 69, 65, 82, 68, 82, 79, 80, 2, 207, 132, 11, 45, - 4, 180, 255, 1, 5, 73, 76, 69, 32, 70, 15, 79, 2, 17, 2, 65, 78, 2, 187, - 251, 9, 68, 4, 57, 12, 65, 73, 76, 66, 79, 88, 32, 87, 73, 84, 72, 32, 4, - 44, 3, 76, 79, 87, 21, 4, 82, 65, 73, 83, 2, 17, 2, 69, 82, 2, 205, 173, - 11, 2, 69, 68, 6, 108, 15, 67, 73, 82, 67, 85, 73, 84, 45, 79, 85, 84, - 80, 85, 84, 32, 217, 242, 11, 6, 79, 85, 84, 76, 73, 78, 4, 18, 72, 3, - 76, 2, 229, 190, 1, 4, 45, 84, 89, 80, 2, 189, 132, 12, 6, 77, 32, 67, - 79, 77, 77, 6, 64, 2, 79, 78, 153, 177, 1, 8, 67, 65, 76, 32, 68, 73, 83, - 67, 2, 235, 250, 10, 32, 204, 1, 132, 1, 3, 65, 78, 71, 66, 73, 212, 8, - 5, 78, 65, 84, 69, 32, 192, 225, 3, 5, 84, 72, 79, 68, 79, 254, 186, 1, - 32, 227, 231, 5, 67, 6, 28, 2, 69, 32, 239, 21, 85, 4, 154, 191, 11, 66, - 211, 73, 72, 188, 1, 48, 5, 71, 73, 78, 65, 76, 21, 3, 89, 65, 32, 2, - 151, 133, 1, 32, 186, 1, 106, 65, 30, 70, 250, 1, 73, 32, 7, 76, 69, 84, - 84, 69, 82, 32, 142, 2, 83, 214, 1, 86, 243, 153, 11, 68, 4, 182, 142, 8, + 234, 230, 15, 80, 2, 90, 254, 68, 83, 14, 67, 3, 75, 7, 48, 6, 78, 85, + 83, 86, 65, 82, 215, 176, 16, 72, 2, 195, 160, 9, 65, 8, 26, 65, 239, + 173, 16, 68, 7, 246, 247, 8, 77, 233, 245, 6, 3, 71, 65, 76, 4, 186, 173, + 16, 89, 187, 2, 85, 5, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 85, 2, 181, + 236, 15, 3, 66, 65, 68, 4, 202, 172, 16, 71, 3, 78, 8, 60, 6, 72, 82, 69, + 69, 32, 66, 234, 230, 15, 84, 195, 71, 65, 2, 17, 2, 65, 76, 2, 143, 137, + 16, 85, 6, 26, 65, 231, 173, 16, 73, 5, 29, 5, 32, 87, 73, 84, 72, 2, + 237, 220, 14, 2, 32, 84, 41, 29, 5, 78, 67, 72, 85, 32, 38, 104, 9, 65, + 76, 73, 32, 71, 65, 76, 73, 32, 222, 177, 14, 90, 138, 248, 1, 70, 2, 75, + 2, 82, 187, 2, 73, 28, 122, 68, 254, 192, 9, 67, 226, 222, 2, 84, 134, + 145, 2, 66, 2, 71, 2, 74, 2, 76, 130, 179, 1, 90, 254, 4, 78, 131, 64, + 83, 4, 222, 176, 14, 68, 139, 248, 1, 72, 48, 52, 4, 73, 66, 69, 32, 142, + 168, 16, 72, 187, 2, 65, 44, 242, 221, 5, 71, 2, 72, 158, 173, 6, 73, + 154, 19, 84, 222, 145, 2, 67, 2, 83, 246, 7, 82, 214, 161, 1, 65, 186, 9, + 90, 162, 7, 85, 234, 61, 68, 2, 70, 2, 74, 2, 75, 2, 80, 187, 2, 69, 60, + 52, 4, 79, 68, 79, 32, 154, 166, 16, 83, 187, 2, 65, 56, 250, 1, 65, 98, + 68, 34, 74, 34, 78, 236, 164, 1, 8, 76, 79, 78, 71, 32, 86, 79, 87, 250, + 179, 4, 71, 182, 192, 6, 84, 222, 145, 2, 67, 246, 7, 72, 174, 178, 1, + 79, 2, 85, 234, 61, 66, 2, 75, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, 186, 2, + 69, 3, 73, 6, 56, 8, 76, 73, 32, 71, 65, 76, 73, 32, 199, 165, 16, 78, 4, + 214, 171, 14, 90, 139, 248, 1, 84, 4, 186, 163, 16, 90, 187, 2, 65, 4, + 154, 163, 16, 73, 187, 2, 65, 2, 251, 162, 16, 73, 6, 198, 145, 16, 72, + 162, 17, 82, 187, 2, 65, 8, 128, 1, 4, 87, 73, 82, 76, 189, 190, 4, 21, + 73, 66, 69, 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 66, 79, 85, 78, 68, + 65, 82, 89, 6, 17, 2, 32, 66, 6, 25, 4, 73, 82, 71, 65, 7, 33, 6, 32, 87, + 73, 84, 72, 32, 4, 150, 2, 68, 187, 221, 15, 79, 6, 152, 1, 3, 82, 73, + 80, 56, 18, 85, 82, 78, 69, 68, 32, 83, 87, 73, 82, 76, 32, 66, 73, 82, + 71, 65, 32, 197, 192, 1, 8, 79, 68, 79, 32, 83, 79, 70, 84, 2, 225, 221, + 15, 9, 76, 69, 32, 66, 73, 82, 71, 65, 32, 2, 33, 6, 87, 73, 84, 72, 32, + 68, 2, 161, 221, 15, 5, 79, 85, 66, 76, 69, 2, 207, 229, 9, 76, 10, 96, + 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 208, 131, 4, 5, 83, 84, 65, 66, + 76, 183, 232, 9, 82, 6, 26, 89, 219, 182, 12, 69, 4, 206, 206, 15, 65, + 155, 1, 73, 10, 48, 2, 78, 32, 230, 209, 4, 68, 191, 182, 11, 83, 6, 88, + 12, 86, 73, 69, 87, 73, 78, 71, 32, 67, 69, 82, 69, 234, 181, 12, 67, 85, + 2, 76, 65, 2, 189, 254, 15, 2, 77, 79, 4, 190, 175, 12, 73, 191, 238, 3, + 69, 10, 26, 72, 69, 2, 79, 82, 2, 45, 9, 69, 82, 32, 67, 72, 82, 73, 83, + 84, 2, 255, 254, 2, 77, 8, 50, 32, 52, 4, 73, 90, 69, 68, 143, 144, 15, + 87, 4, 226, 196, 8, 66, 233, 159, 6, 4, 83, 67, 79, 79, 2, 189, 176, 3, + 7, 32, 87, 72, 69, 69, 76, 67, 18, 52, 2, 78, 84, 152, 1, 2, 83, 69, 131, + 153, 16, 84, 10, 48, 3, 65, 73, 78, 141, 132, 12, 3, 32, 70, 85, 9, 11, + 32, 6, 142, 202, 9, 82, 24, 5, 67, 65, 66, 76, 69, 217, 235, 1, 6, 66, + 73, 67, 89, 67, 76, 7, 11, 32, 4, 148, 137, 15, 2, 84, 82, 231, 83, 70, + 86, 52, 7, 76, 69, 84, 84, 69, 82, 32, 211, 234, 5, 68, 62, 198, 1, 75, + 62, 77, 34, 78, 34, 79, 30, 80, 34, 84, 234, 105, 68, 180, 128, 8, 2, 72, + 65, 2, 82, 134, 192, 1, 69, 134, 29, 83, 154, 89, 76, 246, 7, 67, 130, + 207, 4, 89, 190, 28, 66, 2, 87, 187, 2, 65, 6, 192, 173, 5, 2, 69, 65, + 170, 255, 5, 72, 243, 234, 4, 79, 4, 242, 132, 16, 65, 215, 1, 73, 4, + 158, 199, 15, 73, 139, 60, 71, 7, 222, 150, 16, 76, 3, 79, 4, 234, 130, + 16, 72, 219, 19, 65, 6, 178, 144, 9, 72, 218, 130, 7, 69, 155, 3, 65, + 206, 4, 44, 2, 76, 84, 234, 6, 83, 151, 252, 13, 67, 102, 36, 4, 65, 78, + 73, 32, 219, 2, 73, 76, 52, 7, 76, 69, 84, 84, 69, 82, 32, 147, 148, 4, + 83, 74, 206, 1, 68, 222, 83, 78, 166, 236, 1, 82, 214, 198, 9, 74, 178, + 51, 84, 206, 145, 3, 66, 2, 67, 2, 71, 2, 75, 2, 80, 138, 69, 72, 2, 76, + 2, 77, 2, 83, 2, 86, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 85, 10, 38, 68, + 178, 144, 16, 72, 187, 2, 65, 6, 174, 144, 16, 68, 2, 72, 187, 2, 65, 26, + 56, 2, 80, 76, 236, 2, 3, 83, 69, 84, 175, 254, 14, 77, 18, 50, 69, 89, + 8, 73, 67, 65, 84, 73, 79, 78, 32, 2, 41, 8, 32, 77, 85, 83, 73, 67, 65, + 76, 2, 21, 3, 32, 78, 79, 2, 147, 253, 14, 84, 16, 40, 4, 83, 73, 71, 78, + 207, 144, 16, 88, 15, 11, 32, 12, 48, 3, 73, 78, 32, 81, 5, 87, 73, 84, + 72, 32, 8, 226, 206, 3, 76, 22, 82, 192, 175, 9, 5, 68, 79, 85, 66, 76, + 247, 236, 1, 84, 4, 174, 164, 12, 85, 195, 234, 2, 68, 7, 11, 32, 4, 232, + 83, 5, 77, 85, 76, 84, 73, 239, 139, 12, 85, 228, 3, 44, 5, 72, 82, 79, + 79, 77, 21, 2, 73, 67, 5, 175, 226, 14, 32, 224, 3, 30, 32, 101, 3, 65, + 76, 32, 6, 76, 4, 78, 65, 84, 85, 200, 43, 2, 70, 76, 161, 137, 11, 4, + 83, 72, 65, 82, 2, 235, 201, 10, 82, 218, 3, 64, 8, 75, 69, 89, 66, 79, + 65, 82, 68, 66, 83, 135, 247, 3, 78, 5, 41, 8, 32, 87, 73, 84, 72, 32, + 74, 65, 2, 143, 239, 13, 67, 212, 3, 48, 6, 89, 77, 66, 79, 76, 32, 243, + 166, 11, 67, 210, 3, 230, 2, 66, 238, 1, 67, 226, 9, 68, 242, 2, 69, 162, + 1, 70, 166, 2, 71, 112, 9, 65, 82, 80, 69, 71, 71, 73, 65, 84, 168, 1, 2, + 72, 65, 126, 75, 198, 3, 76, 138, 1, 77, 190, 2, 78, 162, 1, 79, 218, 2, + 80, 200, 2, 2, 81, 85, 146, 2, 82, 198, 2, 83, 230, 5, 84, 182, 10, 86, + 42, 88, 42, 87, 200, 140, 9, 9, 73, 78, 86, 69, 82, 84, 69, 68, 32, 255, + 195, 6, 90, 20, 36, 5, 69, 71, 73, 78, 32, 59, 82, 8, 130, 15, 80, 30, + 83, 218, 246, 7, 66, 143, 231, 7, 84, 12, 24, 2, 65, 67, 35, 69, 4, 198, + 210, 15, 75, 155, 53, 69, 8, 26, 86, 203, 197, 15, 65, 6, 32, 2, 73, 83, + 131, 135, 16, 69, 5, 171, 26, 32, 84, 120, 2, 65, 69, 30, 76, 94, 79, + 194, 7, 82, 234, 11, 32, 232, 32, 7, 73, 82, 67, 76, 69, 32, 88, 189, + 222, 5, 2, 85, 84, 2, 177, 193, 15, 2, 83, 85, 8, 42, 73, 245, 23, 5, 85, + 83, 84, 69, 82, 4, 26, 77, 223, 225, 13, 86, 2, 247, 214, 9, 65, 64, 26, + 77, 171, 130, 16, 68, 62, 64, 7, 66, 73, 78, 73, 78, 71, 32, 237, 145, 6, + 3, 77, 79, 78, 60, 202, 1, 65, 80, 7, 77, 65, 82, 67, 65, 84, 79, 56, 2, + 68, 79, 56, 2, 85, 80, 28, 2, 70, 76, 40, 5, 72, 65, 82, 77, 79, 22, 83, + 142, 2, 84, 226, 176, 7, 66, 224, 193, 3, 2, 76, 79, 195, 138, 5, 82, 6, + 76, 5, 67, 67, 69, 78, 84, 37, 10, 85, 71, 77, 69, 78, 84, 65, 84, 73, + 79, 5, 205, 2, 5, 45, 83, 84, 65, 67, 2, 199, 240, 14, 78, 6, 52, 2, 87, + 78, 168, 3, 2, 85, 66, 191, 225, 15, 73, 2, 233, 230, 14, 2, 32, 66, 12, + 248, 76, 2, 65, 71, 239, 179, 15, 73, 2, 195, 216, 14, 78, 12, 132, 1, 9, + 78, 65, 80, 32, 80, 73, 90, 90, 73, 22, 84, 224, 205, 4, 11, 80, 82, 69, + 67, 72, 71, 69, 83, 65, 78, 71, 219, 129, 6, 77, 2, 139, 244, 11, 67, 6, + 44, 5, 65, 67, 67, 65, 84, 151, 238, 15, 69, 4, 40, 4, 73, 83, 83, 73, + 195, 254, 15, 79, 2, 203, 223, 15, 77, 10, 34, 82, 193, 143, 12, 2, 69, + 78, 8, 28, 2, 73, 80, 191, 6, 69, 2, 209, 242, 11, 6, 76, 69, 32, 84, 79, + 78, 4, 22, 79, 191, 2, 69, 2, 255, 240, 14, 73, 24, 98, 65, 142, 1, 69, + 84, 6, 79, 85, 66, 76, 69, 32, 225, 158, 10, 7, 82, 85, 77, 32, 67, 76, + 69, 10, 96, 2, 32, 67, 20, 2, 77, 80, 204, 29, 4, 83, 72, 69, 68, 217, + 180, 15, 5, 76, 32, 83, 69, 71, 2, 223, 189, 8, 65, 5, 231, 243, 13, 32, + 4, 52, 3, 67, 82, 69, 145, 228, 8, 4, 71, 82, 69, 69, 2, 165, 25, 3, 83, + 67, 69, 6, 246, 21, 83, 130, 7, 66, 243, 134, 5, 70, 14, 32, 3, 78, 68, + 32, 155, 16, 73, 10, 74, 80, 30, 83, 208, 13, 3, 79, 70, 32, 138, 233, 7, + 66, 143, 231, 7, 84, 2, 229, 211, 12, 2, 72, 82, 2, 191, 128, 15, 76, 34, + 104, 6, 69, 82, 77, 65, 84, 65, 22, 73, 122, 79, 102, 32, 152, 36, 3, 85, + 83, 65, 205, 229, 3, 2, 76, 65, 5, 219, 202, 13, 32, 10, 22, 78, 143, 34, + 86, 8, 56, 9, 71, 69, 82, 69, 68, 32, 84, 82, 69, 243, 20, 65, 6, 237, + 237, 5, 4, 77, 79, 76, 79, 6, 216, 25, 3, 85, 82, 45, 183, 150, 14, 82, + 18, 54, 32, 56, 7, 76, 73, 83, 83, 65, 78, 68, 23, 82, 6, 25, 4, 67, 76, + 69, 70, 7, 161, 13, 3, 32, 79, 84, 4, 215, 129, 6, 79, 8, 84, 9, 65, 67, + 69, 32, 78, 79, 84, 69, 32, 37, 8, 69, 71, 79, 82, 73, 65, 78, 32, 4, + 164, 222, 8, 2, 78, 79, 27, 83, 4, 250, 2, 67, 3, 70, 8, 44, 3, 76, 70, + 32, 209, 8, 3, 85, 80, 84, 6, 52, 3, 80, 69, 68, 226, 222, 3, 78, 207, + 183, 3, 82, 2, 139, 198, 5, 65, 24, 48, 6, 73, 69, 86, 65, 78, 32, 247, + 205, 14, 79, 22, 178, 1, 67, 46, 69, 68, 7, 81, 85, 65, 82, 84, 69, 82, + 62, 70, 164, 219, 3, 4, 72, 65, 76, 70, 0, 5, 87, 72, 79, 76, 69, 157, + 224, 1, 9, 82, 69, 67, 73, 84, 65, 84, 73, 86, 2, 11, 32, 2, 11, 67, 2, + 163, 163, 5, 76, 6, 64, 5, 73, 71, 72, 84, 72, 201, 186, 14, 5, 78, 68, + 32, 79, 70, 4, 217, 252, 5, 10, 32, 78, 79, 84, 69, 32, 83, 84, 69, 77, + 4, 222, 14, 76, 197, 204, 3, 4, 73, 78, 65, 76, 8, 44, 4, 79, 78, 71, 65, + 221, 13, 2, 69, 70, 7, 11, 32, 4, 28, 3, 73, 77, 80, 3, 80, 2, 197, 2, 7, + 69, 82, 70, 69, 67, 84, 65, 18, 104, 2, 65, 88, 20, 2, 69, 90, 20, 5, 73, + 78, 73, 77, 65, 48, 3, 79, 79, 78, 25, 4, 85, 76, 84, 73, 2, 255, 204, + 15, 73, 2, 135, 207, 15, 90, 7, 11, 32, 4, 170, 144, 7, 82, 167, 194, 5, + 66, 4, 193, 17, 2, 32, 78, 4, 60, 11, 80, 76, 69, 32, 77, 69, 65, 83, 85, + 82, 69, 15, 32, 2, 11, 32, 2, 163, 143, 7, 82, 10, 120, 4, 69, 66, 69, + 78, 216, 26, 3, 85, 76, 76, 192, 220, 5, 3, 65, 84, 85, 141, 217, 6, 7, + 79, 84, 69, 72, 69, 65, 68, 2, 213, 211, 14, 4, 83, 84, 73, 77, 32, 88, + 2, 78, 69, 120, 14, 82, 78, 65, 77, 69, 78, 84, 32, 83, 84, 82, 79, 75, + 69, 107, 84, 6, 92, 18, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 87, 69, + 78, 84, 89, 45, 69, 73, 163, 20, 45, 4, 185, 13, 2, 71, 72, 22, 11, 45, + 22, 158, 166, 3, 49, 162, 195, 12, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, + 2, 56, 3, 57, 4, 173, 4, 3, 84, 65, 86, 18, 96, 9, 65, 82, 69, 78, 84, + 72, 69, 83, 73, 0, 2, 76, 85, 18, 69, 142, 1, 79, 223, 222, 13, 73, 2, + 247, 22, 83, 6, 68, 4, 68, 65, 76, 32, 49, 9, 83, 32, 83, 85, 66, 80, 85, + 78, 67, 4, 26, 85, 199, 165, 15, 77, 2, 171, 165, 15, 80, 2, 167, 195, + 13, 84, 6, 48, 2, 68, 65, 245, 5, 5, 82, 82, 69, 67, 84, 2, 151, 224, 13, + 84, 12, 76, 6, 65, 82, 84, 69, 82, 32, 125, 9, 73, 78, 68, 73, 67, 69, + 83, 73, 77, 8, 60, 5, 84, 79, 78, 69, 32, 246, 207, 3, 78, 207, 183, 3, + 82, 4, 26, 83, 243, 141, 5, 70, 2, 197, 228, 15, 3, 72, 65, 82, 4, 17, 2, + 65, 32, 4, 142, 171, 12, 65, 205, 182, 3, 3, 66, 65, 83, 14, 22, 69, 175, + 1, 73, 10, 72, 4, 80, 69, 65, 84, 81, 10, 86, 69, 82, 83, 69, 32, 70, 73, + 78, 65, 8, 56, 8, 69, 68, 32, 70, 73, 71, 85, 82, 163, 146, 14, 32, 6, + 147, 217, 5, 69, 2, 211, 4, 76, 4, 48, 2, 71, 72, 57, 6, 78, 70, 79, 82, + 90, 65, 2, 33, 6, 84, 32, 82, 69, 80, 69, 2, 255, 158, 10, 65, 2, 147, + 176, 8, 78, 48, 136, 1, 6, 67, 65, 78, 68, 73, 67, 62, 69, 166, 1, 72, + 54, 73, 252, 1, 6, 81, 85, 65, 82, 69, 32, 212, 153, 8, 2, 85, 66, 243, + 100, 79, 4, 17, 2, 85, 83, 5, 141, 218, 13, 5, 32, 70, 76, 69, 88, 14, + 32, 2, 77, 73, 179, 182, 15, 71, 12, 64, 6, 66, 82, 69, 86, 73, 83, 1, 6, + 77, 73, 78, 73, 77, 65, 6, 11, 32, 6, 138, 13, 87, 182, 244, 6, 82, 167, + 194, 5, 66, 6, 84, 3, 79, 82, 84, 213, 233, 5, 3, 65, 82, 80, 14, 32, 4, + 78, 71, 76, 69, 43, 88, 2, 17, 2, 32, 66, 2, 135, 246, 13, 65, 12, 18, + 45, 79, 84, 4, 242, 7, 76, 173, 190, 14, 11, 83, 84, 82, 73, 78, 71, 32, + 70, 82, 69, 84, 8, 52, 3, 69, 69, 78, 1, 6, 89, 45, 70, 79, 85, 82, 4, + 193, 12, 2, 84, 72, 6, 26, 78, 247, 219, 15, 66, 4, 229, 9, 7, 79, 84, + 69, 72, 69, 65, 68, 60, 132, 1, 6, 69, 77, 80, 85, 83, 32, 154, 4, 72, + 88, 2, 87, 79, 84, 7, 79, 82, 67, 85, 76, 85, 83, 46, 82, 141, 3, 3, 85, + 82, 78, 16, 232, 1, 27, 73, 77, 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, + 67, 85, 77, 32, 80, 82, 79, 76, 65, 84, 73, 79, 78, 69, 32, 129, 1, 25, + 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, 67, 85, 77, 32, 80, 82, 79, 76, + 65, 84, 73, 79, 78, 69, 32, 10, 60, 10, 73, 77, 80, 69, 82, 70, 69, 67, + 84, 65, 131, 1, 80, 9, 213, 206, 5, 11, 32, 68, 73, 77, 73, 78, 85, 84, + 73, 79, 78, 6, 60, 3, 73, 77, 80, 41, 8, 80, 69, 82, 70, 69, 67, 84, 65, + 2, 197, 178, 15, 5, 69, 82, 70, 69, 67, 5, 169, 189, 8, 12, 32, 68, 73, + 77, 73, 78, 85, 84, 73, 79, 78, 45, 6, 72, 2, 82, 69, 249, 5, 11, 73, 82, + 84, 89, 45, 83, 69, 67, 79, 78, 68, 2, 11, 69, 2, 11, 45, 2, 11, 76, 2, + 37, 7, 73, 78, 69, 32, 83, 84, 65, 2, 235, 211, 14, 70, 5, 225, 206, 11, + 6, 32, 82, 69, 83, 85, 80, 27, 33, 6, 73, 65, 78, 71, 76, 69, 24, 128, 1, + 10, 32, 78, 79, 84, 69, 72, 69, 65, 68, 32, 61, 17, 45, 82, 79, 85, 78, + 68, 32, 78, 79, 84, 69, 72, 69, 65, 68, 32, 68, 20, 58, 68, 24, 3, 85, + 80, 32, 38, 82, 25, 3, 76, 69, 70, 4, 93, 3, 79, 87, 78, 8, 34, 82, 78, + 87, 219, 182, 12, 66, 4, 21, 3, 73, 71, 72, 4, 11, 84, 4, 11, 32, 4, 26, + 87, 219, 182, 12, 66, 2, 11, 72, 2, 223, 138, 14, 73, 7, 11, 32, 4, 246, + 186, 8, 83, 183, 150, 7, 85, 4, 36, 3, 79, 73, 68, 159, 142, 15, 73, 2, + 237, 251, 12, 5, 32, 78, 79, 84, 69, 6, 92, 4, 72, 79, 76, 69, 221, 199, + 8, 13, 73, 84, 72, 32, 70, 73, 78, 71, 69, 82, 78, 65, 73, 4, 11, 32, 4, + 218, 186, 3, 78, 207, 183, 3, 82, 232, 3, 224, 2, 15, 67, 79, 78, 83, 79, + 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 254, 1, 76, 212, 14, 16, 77, 79, + 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, 122, 83, 80, 16, + 69, 65, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 222, 9, + 84, 204, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 160, 175, 9, + 3, 80, 65, 79, 207, 142, 4, 68, 16, 66, 77, 165, 1, 11, 83, 72, 65, 78, + 32, 77, 69, 68, 73, 65, 76, 14, 80, 6, 69, 68, 73, 65, 76, 32, 45, 10, + 79, 78, 32, 77, 69, 68, 73, 65, 76, 32, 8, 250, 200, 15, 72, 2, 82, 2, + 87, 3, 89, 6, 206, 200, 15, 76, 2, 77, 3, 78, 2, 219, 217, 10, 32, 238, + 1, 104, 6, 69, 84, 84, 69, 82, 32, 185, 13, 15, 79, 71, 79, 71, 82, 65, + 77, 32, 75, 72, 65, 77, 84, 73, 32, 232, 1, 238, 1, 65, 50, 69, 146, 1, + 71, 50, 75, 254, 1, 77, 134, 1, 78, 58, 82, 86, 83, 130, 3, 84, 198, 1, + 87, 178, 227, 11, 68, 214, 6, 85, 22, 86, 186, 201, 1, 73, 42, 76, 246, + 193, 1, 66, 2, 67, 2, 74, 2, 80, 138, 69, 72, 2, 89, 187, 2, 79, 7, 144, + 253, 14, 4, 73, 84, 79, 78, 219, 74, 85, 9, 77, 17, 65, 83, 84, 69, 82, + 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 6, 42, 71, 186, 213, 10, + 89, 135, 181, 3, 78, 2, 183, 213, 10, 72, 6, 154, 200, 9, 82, 218, 251, + 5, 72, 187, 2, 65, 44, 32, 2, 72, 65, 219, 197, 15, 65, 43, 25, 4, 77, + 84, 73, 32, 40, 134, 1, 68, 34, 84, 242, 217, 8, 78, 230, 162, 6, 67, 2, + 72, 2, 74, 170, 37, 76, 226, 31, 70, 2, 71, 2, 82, 2, 83, 2, 88, 3, 90, + 6, 242, 252, 14, 68, 139, 69, 72, 4, 211, 252, 14, 84, 12, 36, 3, 79, 78, + 32, 219, 195, 15, 65, 10, 60, 2, 66, 66, 218, 200, 13, 74, 254, 183, 1, + 78, 199, 66, 69, 4, 150, 195, 15, 65, 3, 69, 10, 214, 216, 8, 78, 238, + 231, 6, 71, 2, 89, 187, 2, 65, 4, 252, 220, 2, 12, 85, 77, 65, 73, 32, + 80, 65, 76, 65, 85, 78, 71, 199, 229, 12, 65, 50, 90, 72, 192, 221, 2, 9, + 71, 65, 87, 32, 75, 65, 82, 69, 78, 158, 225, 12, 83, 187, 2, 65, 44, 66, + 65, 197, 1, 11, 87, 69, 32, 80, 65, 76, 65, 85, 78, 71, 32, 41, 17, 2, + 78, 32, 38, 134, 1, 78, 246, 196, 13, 74, 2, 80, 2, 84, 130, 179, 1, 66, + 2, 67, 2, 71, 2, 75, 138, 69, 68, 2, 70, 2, 72, 2, 90, 187, 2, 65, 6, + 250, 188, 15, 78, 2, 89, 187, 2, 65, 4, 202, 196, 13, 67, 3, 83, 36, 38, + 65, 138, 247, 14, 84, 139, 69, 72, 31, 41, 8, 73, 32, 76, 65, 73, 78, 71, + 32, 28, 82, 78, 206, 227, 11, 68, 190, 146, 3, 66, 2, 71, 2, 74, 170, 37, + 76, 227, 31, 70, 4, 142, 187, 15, 78, 3, 89, 6, 92, 17, 69, 83, 84, 69, + 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 207, 188, 15, 65, 4, + 194, 203, 10, 80, 203, 246, 2, 84, 6, 162, 176, 14, 79, 166, 60, 81, 139, + 63, 72, 4, 56, 6, 75, 72, 65, 77, 84, 73, 1, 4, 83, 72, 65, 78, 2, 29, 5, + 32, 82, 69, 68, 85, 2, 193, 132, 1, 2, 80, 76, 90, 76, 2, 72, 65, 20, 4, + 73, 71, 78, 32, 233, 6, 6, 89, 77, 66, 79, 76, 32, 20, 199, 186, 9, 78, + 52, 202, 3, 65, 32, 12, 75, 72, 65, 77, 84, 73, 32, 84, 79, 78, 69, 45, + 30, 83, 132, 2, 15, 84, 65, 73, 32, 76, 65, 73, 78, 71, 32, 84, 79, 78, + 69, 45, 28, 22, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, + 82, 69, 78, 32, 84, 79, 78, 69, 206, 244, 2, 68, 252, 163, 5, 19, 82, 85, + 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 32, 84, 79, 78, 69, 45, 204, + 170, 3, 9, 80, 65, 79, 32, 75, 65, 82, 69, 78, 148, 114, 6, 76, 73, 84, + 84, 76, 69, 199, 187, 2, 86, 4, 234, 162, 14, 83, 255, 78, 78, 4, 178, + 182, 15, 49, 3, 51, 18, 40, 4, 72, 65, 78, 32, 147, 229, 14, 69, 16, 84, + 8, 67, 79, 85, 78, 67, 73, 76, 32, 84, 5, 84, 79, 78, 69, 45, 151, 182, + 14, 83, 6, 172, 196, 11, 8, 69, 77, 80, 72, 65, 84, 73, 67, 229, 239, 3, + 4, 84, 79, 78, 69, 8, 190, 180, 15, 50, 2, 51, 2, 53, 3, 54, 4, 146, 180, + 15, 50, 3, 53, 10, 11, 45, 10, 234, 179, 15, 49, 2, 50, 2, 51, 2, 52, 3, + 53, 18, 130, 1, 65, 132, 1, 2, 76, 79, 28, 5, 83, 72, 65, 78, 32, 192, + 173, 8, 4, 71, 69, 78, 73, 169, 97, 6, 67, 79, 77, 80, 76, 69, 8, 84, 5, + 73, 84, 79, 78, 32, 141, 174, 6, 10, 70, 79, 82, 69, 77, 69, 78, 84, 73, + 79, 6, 98, 69, 186, 249, 8, 84, 151, 158, 5, 79, 2, 217, 173, 8, 2, 67, + 65, 4, 26, 69, 207, 151, 14, 79, 2, 201, 12, 4, 88, 67, 76, 65, 24, 140, + 1, 20, 79, 78, 69, 32, 77, 65, 82, 75, 32, 83, 71, 65, 87, 32, 75, 65, + 82, 69, 78, 32, 221, 175, 9, 8, 65, 73, 32, 76, 65, 73, 78, 71, 4, 38, + 72, 129, 206, 8, 3, 75, 69, 32, 2, 139, 208, 8, 65, 56, 150, 2, 65, 76, + 9, 71, 69, 66, 65, 32, 75, 65, 82, 69, 20, 6, 75, 65, 89, 65, 72, 32, 40, + 4, 77, 79, 78, 32, 34, 83, 142, 1, 69, 40, 18, 87, 69, 83, 84, 69, 82, + 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 228, 182, 9, 2, 84, 65, + 142, 160, 2, 85, 22, 86, 187, 201, 1, 73, 8, 26, 73, 219, 172, 15, 65, 7, + 25, 4, 84, 79, 78, 32, 4, 223, 162, 13, 65, 2, 151, 236, 14, 78, 6, 190, + 149, 15, 69, 2, 79, 215, 22, 85, 4, 146, 152, 15, 73, 219, 19, 79, 10, + 80, 4, 72, 65, 78, 32, 157, 204, 8, 10, 71, 65, 87, 32, 75, 65, 82, 69, + 78, 32, 8, 54, 69, 20, 5, 70, 73, 78, 65, 76, 247, 167, 15, 65, 5, 199, + 173, 14, 32, 2, 227, 152, 15, 32, 4, 174, 133, 15, 69, 151, 14, 85, 230, + 17, 152, 2, 5, 45, 65, 82, 89, 32, 214, 4, 65, 222, 16, 66, 30, 69, 146, + 32, 73, 136, 1, 3, 75, 79, 32, 170, 10, 79, 162, 24, 85, 224, 8, 22, 89, + 73, 65, 75, 69, 78, 71, 32, 80, 85, 65, 67, 72, 85, 69, 32, 72, 77, 79, + 78, 71, 32, 216, 135, 2, 2, 80, 78, 220, 189, 12, 2, 78, 66, 27, 76, 32, + 130, 1, 67, 114, 84, 46, 83, 160, 1, 5, 85, 78, 73, 79, 78, 108, 4, 87, + 72, 73, 84, 246, 225, 11, 76, 210, 17, 73, 223, 160, 2, 80, 8, 52, 7, 73, + 82, 67, 76, 69, 68, 32, 135, 151, 14, 79, 6, 40, 2, 80, 76, 14, 84, 219, + 141, 8, 68, 2, 35, 85, 2, 21, 3, 73, 77, 69, 2, 219, 229, 11, 83, 6, 40, + 6, 81, 85, 65, 82, 69, 32, 87, 85, 4, 60, 9, 73, 78, 84, 69, 82, 83, 69, + 67, 84, 1, 2, 85, 78, 2, 215, 191, 11, 73, 2, 11, 77, 2, 179, 226, 11, + 77, 7, 69, 15, 32, 79, 80, 69, 82, 65, 84, 79, 82, 32, 87, 73, 84, 72, + 32, 4, 150, 157, 11, 80, 131, 182, 3, 68, 2, 11, 69, 2, 233, 204, 12, 2, + 32, 86, 188, 2, 170, 2, 66, 252, 4, 10, 71, 32, 77, 85, 78, 68, 65, 82, + 73, 32, 142, 4, 73, 52, 2, 78, 68, 208, 4, 7, 84, 73, 79, 78, 65, 76, 32, + 228, 213, 1, 2, 85, 83, 140, 137, 8, 5, 77, 69, 32, 66, 65, 184, 212, 3, + 7, 90, 65, 82, 32, 65, 77, 85, 188, 160, 1, 8, 82, 82, 79, 87, 32, 78, + 79, 45, 231, 62, 75, 82, 52, 7, 65, 84, 65, 69, 65, 78, 32, 231, 157, 15, + 76, 80, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, 236, 2, 7, 78, 85, 77, 66, + 69, 82, 32, 133, 243, 9, 16, 67, 82, 85, 67, 73, 70, 79, 82, 77, 32, 78, + 85, 77, 66, 69, 82, 62, 236, 1, 6, 70, 73, 78, 65, 76, 32, 154, 132, 2, + 84, 246, 118, 68, 34, 76, 50, 81, 238, 205, 1, 82, 178, 215, 2, 65, 50, + 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, + 178, 216, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, 18, 246, 250, 2, + 65, 54, 76, 250, 163, 4, 83, 190, 3, 89, 134, 214, 1, 75, 198, 75, 66, + 178, 216, 4, 78, 222, 105, 72, 171, 4, 77, 16, 134, 252, 2, 84, 134, 166, + 4, 79, 223, 137, 6, 70, 84, 84, 7, 76, 69, 84, 84, 69, 82, 32, 164, 2, 5, + 83, 73, 71, 78, 32, 191, 167, 13, 68, 54, 42, 65, 50, 69, 66, 73, 50, 79, + 47, 85, 11, 138, 137, 15, 78, 202, 17, 66, 2, 72, 3, 74, 15, 158, 159, + 13, 78, 210, 113, 76, 138, 109, 84, 174, 28, 71, 3, 77, 11, 194, 211, 14, + 68, 162, 70, 72, 2, 83, 3, 84, 11, 222, 152, 15, 78, 86, 76, 2, 80, 3, + 89, 11, 134, 153, 15, 67, 2, 68, 2, 75, 3, 82, 10, 100, 2, 77, 85, 20, 3, + 83, 85, 84, 182, 118, 73, 212, 234, 10, 2, 79, 74, 241, 253, 2, 3, 84, + 79, 89, 2, 139, 223, 14, 72, 2, 207, 150, 15, 85, 4, 208, 182, 11, 5, 76, + 32, 80, 79, 76, 187, 15, 82, 133, 1, 41, 8, 73, 78, 65, 71, 65, 82, 73, + 32, 130, 1, 140, 1, 7, 76, 69, 84, 84, 69, 82, 32, 248, 1, 5, 83, 73, 71, + 78, 32, 52, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 239, 229, 4, + 72, 94, 210, 1, 86, 226, 185, 11, 65, 38, 68, 82, 82, 34, 84, 230, 5, 85, + 206, 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, + 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 89, 186, 2, 69, 3, 79, 6, + 202, 148, 5, 79, 183, 255, 9, 65, 10, 254, 240, 9, 83, 238, 203, 1, 65, + 231, 147, 3, 86, 24, 250, 235, 4, 80, 238, 39, 86, 246, 171, 6, 65, 38, + 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 4, 158, 48, 68, 255, 160, + 14, 80, 4, 154, 146, 15, 83, 15, 72, 254, 4, 216, 1, 3, 71, 65, 84, 192, + 7, 6, 73, 84, 72, 69, 82, 32, 194, 3, 83, 136, 1, 2, 85, 84, 242, 1, 87, + 148, 16, 3, 88, 84, 32, 144, 186, 8, 4, 80, 84, 85, 78, 216, 133, 3, 2, + 67, 75, 238, 161, 2, 82, 207, 144, 1, 76, 162, 1, 152, 1, 4, 73, 86, 69, + 32, 161, 233, 12, 27, 69, 68, 32, 68, 79, 85, 66, 76, 69, 32, 86, 69, 82, + 84, 73, 67, 65, 76, 32, 66, 65, 82, 32, 68, 79, 85, 66, 160, 1, 152, 1, + 8, 67, 73, 82, 67, 76, 69, 68, 32, 224, 1, 9, 68, 73, 65, 71, 79, 78, 65, + 76, 32, 184, 1, 8, 83, 81, 85, 65, 82, 69, 68, 32, 231, 206, 8, 65, 78, + 112, 5, 68, 73, 71, 73, 84, 28, 7, 78, 85, 77, 66, 69, 82, 32, 180, 3, 2, + 76, 65, 202, 228, 13, 84, 135, 4, 83, 2, 209, 157, 13, 2, 32, 90, 20, 50, + 84, 214, 133, 2, 69, 46, 70, 42, 78, 31, 83, 6, 150, 135, 2, 72, 47, 87, + 6, 38, 77, 206, 202, 13, 67, 183, 4, 68, 2, 49, 10, 73, 68, 68, 76, 69, + 32, 82, 73, 71, 72, 2, 17, 2, 84, 32, 2, 41, 8, 84, 79, 32, 76, 79, 87, + 69, 82, 2, 153, 137, 12, 2, 32, 67, 74, 142, 1, 76, 70, 85, 230, 154, 12, + 68, 138, 29, 81, 216, 129, 1, 2, 67, 82, 218, 19, 65, 234, 19, 73, 2, 87, + 134, 8, 82, 194, 157, 1, 80, 3, 83, 54, 26, 65, 183, 237, 10, 69, 52, + 129, 246, 8, 5, 84, 73, 78, 32, 67, 2, 171, 233, 13, 80, 18, 158, 1, 65, + 216, 1, 17, 71, 82, 69, 65, 84, 69, 82, 45, 84, 72, 65, 78, 32, 78, 79, + 82, 32, 37, 14, 76, 69, 83, 83, 45, 84, 72, 65, 78, 32, 78, 79, 82, 32, + 6, 92, 3, 32, 83, 85, 85, 16, 80, 80, 82, 79, 88, 73, 77, 65, 84, 69, 76, + 89, 32, 78, 79, 82, 4, 30, 66, 1, 3, 80, 69, 82, 2, 217, 245, 6, 8, 83, + 69, 84, 32, 79, 70, 32, 78, 2, 233, 48, 5, 32, 65, 67, 84, 85, 6, 254, + 144, 8, 69, 131, 227, 4, 76, 6, 218, 144, 8, 69, 223, 226, 4, 71, 6, 26, + 84, 151, 255, 12, 83, 4, 76, 5, 73, 78, 71, 32, 68, 185, 210, 10, 8, 32, + 87, 73, 84, 72, 32, 69, 71, 2, 181, 252, 7, 2, 79, 76, 64, 40, 4, 82, 65, + 76, 32, 159, 203, 14, 69, 62, 48, 6, 67, 72, 69, 83, 83, 32, 191, 198, + 14, 70, 60, 74, 75, 234, 165, 13, 66, 38, 69, 130, 5, 80, 22, 81, 38, 82, + 131, 2, 84, 20, 44, 5, 78, 73, 71, 72, 84, 251, 166, 13, 73, 15, 167, + 167, 13, 32, 246, 2, 70, 32, 184, 8, 2, 65, 32, 252, 6, 3, 76, 73, 78, + 195, 223, 3, 83, 174, 1, 90, 77, 64, 4, 83, 72, 69, 81, 20, 8, 84, 65, + 73, 32, 76, 85, 69, 32, 255, 230, 13, 76, 4, 25, 4, 79, 79, 78, 32, 4, + 138, 143, 8, 87, 211, 222, 5, 83, 2, 163, 189, 9, 69, 166, 1, 160, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 244, 2, 8, 83, 73, 71, 78, 32, 76, 65, 69, + 22, 84, 76, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 155, 138, 13, + 68, 102, 76, 6, 70, 73, 78, 65, 76, 32, 68, 4, 72, 73, 71, 72, 1, 3, 76, + 79, 87, 14, 162, 131, 13, 78, 154, 251, 1, 66, 2, 68, 2, 75, 2, 77, 3, + 86, 44, 11, 32, 44, 146, 1, 75, 2, 88, 34, 83, 138, 139, 11, 78, 162, + 169, 3, 84, 82, 80, 138, 69, 66, 2, 68, 2, 70, 2, 72, 2, 76, 2, 77, 2, + 81, 2, 86, 3, 89, 4, 158, 250, 14, 86, 187, 2, 65, 4, 254, 249, 14, 85, + 187, 2, 65, 5, 151, 252, 14, 86, 6, 252, 154, 7, 3, 79, 78, 69, 233, 80, + 8, 72, 65, 77, 32, 68, 73, 71, 73, 34, 110, 65, 46, 73, 30, 79, 38, 85, + 216, 175, 12, 11, 86, 79, 87, 69, 76, 32, 83, 72, 79, 82, 84, 135, 202, + 2, 69, 8, 146, 219, 10, 65, 182, 159, 4, 69, 3, 89, 4, 154, 250, 14, 73, + 3, 89, 9, 202, 218, 10, 65, 183, 159, 4, 89, 11, 166, 218, 10, 69, 182, + 159, 4, 85, 3, 89, 194, 1, 182, 1, 68, 106, 71, 72, 7, 76, 69, 84, 84, + 69, 82, 32, 214, 2, 83, 194, 23, 80, 162, 139, 1, 86, 182, 230, 7, 65, + 144, 234, 1, 5, 73, 78, 83, 69, 82, 202, 174, 1, 67, 175, 193, 2, 79, 26, + 64, 6, 79, 85, 66, 76, 69, 32, 142, 155, 11, 65, 147, 235, 1, 73, 4, 254, + 154, 11, 68, 175, 138, 1, 67, 2, 11, 65, 2, 11, 80, 2, 17, 2, 32, 70, 2, + 129, 141, 11, 2, 73, 76, 108, 218, 1, 78, 62, 86, 142, 154, 11, 65, 38, + 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 206, 193, 1, 83, 82, 66, 2, 67, + 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 138, 69, 72, 2, 87, 2, + 89, 186, 2, 69, 3, 79, 14, 166, 173, 14, 71, 2, 89, 138, 69, 72, 2, 78, + 187, 2, 65, 10, 26, 69, 139, 161, 11, 79, 2, 173, 190, 12, 3, 68, 73, 67, + 22, 26, 73, 247, 200, 4, 65, 20, 44, 3, 71, 78, 32, 165, 164, 9, 2, 68, + 68, 18, 226, 222, 10, 67, 98, 78, 242, 60, 65, 174, 158, 1, 74, 228, 2, + 5, 70, 73, 78, 65, 76, 50, 85, 167, 242, 1, 86, 4, 167, 199, 14, 69, 4, + 202, 235, 9, 80, 159, 237, 3, 76, 6, 70, 78, 185, 220, 13, 11, 71, 72, + 84, 32, 87, 73, 84, 72, 32, 83, 84, 4, 200, 241, 7, 7, 69, 32, 80, 79, + 73, 78, 84, 179, 253, 6, 74, 124, 152, 1, 3, 67, 79, 77, 130, 3, 68, 78, + 76, 156, 4, 4, 72, 73, 71, 72, 72, 7, 83, 89, 77, 66, 79, 76, 32, 226, + 217, 7, 69, 205, 188, 1, 3, 84, 65, 77, 20, 52, 7, 66, 73, 78, 73, 78, + 71, 32, 235, 236, 14, 77, 18, 88, 3, 68, 79, 85, 32, 5, 76, 79, 78, 71, + 32, 78, 78, 33, 6, 83, 72, 79, 82, 84, 32, 2, 225, 237, 13, 3, 66, 76, + 69, 8, 142, 1, 72, 34, 76, 230, 251, 10, 82, 21, 7, 68, 69, 83, 67, 69, + 78, 68, 2, 11, 65, 2, 171, 206, 10, 83, 6, 34, 72, 34, 76, 231, 251, 10, + 82, 2, 173, 252, 10, 3, 73, 71, 72, 2, 141, 252, 10, 2, 79, 87, 24, 152, + 1, 4, 65, 78, 84, 65, 144, 207, 12, 4, 79, 82, 79, 77, 155, 43, 73, 70, + 76, 4, 65, 74, 65, 78, 32, 6, 69, 84, 84, 69, 82, 32, 173, 3, 2, 79, 87, + 2, 185, 161, 13, 3, 89, 65, 76, 66, 228, 1, 2, 68, 65, 42, 74, 90, 78, + 138, 143, 11, 82, 206, 147, 1, 79, 162, 75, 67, 162, 186, 1, 69, 250, 18, + 71, 242, 42, 66, 2, 70, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 83, 2, 84, + 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 5, 197, 178, 11, 5, 71, 66, 65, + 83, 73, 8, 40, 4, 79, 78, 65, 32, 227, 232, 14, 65, 6, 158, 238, 12, 67, + 138, 248, 1, 74, 3, 82, 11, 26, 65, 1, 2, 89, 65, 5, 245, 135, 8, 5, 32, + 87, 79, 76, 79, 2, 29, 5, 32, 84, 79, 78, 69, 2, 17, 2, 32, 65, 2, 135, + 207, 8, 80, 4, 68, 7, 71, 66, 65, 75, 85, 82, 85, 1, 6, 79, 79, 32, 68, + 69, 78, 2, 247, 146, 13, 78, 184, 1, 90, 32, 156, 3, 2, 77, 73, 118, 78, + 150, 1, 82, 238, 7, 84, 250, 153, 14, 45, 147, 40, 83, 18, 202, 1, 66, + 96, 5, 69, 78, 84, 82, 89, 22, 80, 204, 186, 2, 15, 79, 78, 69, 32, 85, + 78, 68, 69, 82, 32, 69, 73, 71, 72, 84, 160, 129, 4, 9, 77, 79, 66, 73, + 76, 69, 32, 80, 72, 189, 30, 3, 83, 77, 79, 4, 52, 4, 82, 69, 65, 75, + 165, 137, 2, 3, 73, 67, 89, 2, 17, 2, 32, 72, 2, 155, 195, 13, 69, 5, + 203, 220, 13, 32, 4, 60, 6, 69, 68, 69, 83, 84, 82, 221, 201, 9, 3, 73, + 82, 65, 2, 145, 159, 11, 2, 73, 65, 4, 36, 5, 78, 65, 76, 32, 68, 59, 83, + 2, 173, 206, 13, 9, 73, 71, 73, 84, 32, 83, 72, 65, 80, 2, 175, 144, 11, + 77, 6, 38, 45, 249, 217, 9, 3, 70, 79, 82, 4, 76, 8, 66, 82, 69, 65, 75, + 73, 78, 71, 161, 169, 2, 5, 80, 79, 84, 65, 66, 2, 249, 221, 6, 2, 32, + 72, 97, 56, 2, 84, 72, 146, 11, 77, 209, 229, 10, 3, 68, 73, 67, 88, 42, + 32, 201, 205, 6, 4, 69, 65, 83, 84, 86, 96, 5, 69, 65, 83, 84, 32, 148, + 2, 6, 73, 78, 68, 73, 67, 32, 221, 1, 5, 87, 69, 83, 84, 32, 32, 82, 65, + 238, 242, 6, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, 18, 83, 247, 58, + 87, 14, 40, 4, 82, 82, 79, 87, 215, 238, 6, 78, 13, 11, 32, 10, 96, 9, + 67, 82, 79, 83, 83, 73, 78, 71, 32, 248, 2, 2, 65, 78, 162, 238, 6, 87, + 203, 134, 5, 70, 4, 234, 197, 3, 83, 231, 170, 3, 78, 20, 54, 80, 60, 3, + 81, 85, 65, 66, 82, 171, 132, 1, 70, 2, 37, 7, 76, 65, 67, 69, 72, 79, + 76, 2, 243, 179, 4, 68, 4, 220, 179, 4, 2, 82, 84, 209, 230, 9, 5, 78, + 84, 73, 84, 89, 2, 17, 2, 85, 80, 2, 171, 166, 4, 69, 34, 82, 65, 254, + 238, 6, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, 18, 83, 247, 58, 87, 16, + 34, 78, 33, 4, 82, 82, 79, 87, 2, 241, 194, 3, 3, 68, 32, 83, 15, 11, 32, + 12, 84, 3, 84, 79, 32, 142, 234, 6, 67, 40, 3, 65, 78, 68, 234, 2, 87, + 203, 134, 5, 70, 4, 150, 235, 6, 67, 169, 204, 6, 4, 76, 79, 78, 71, 56, + 54, 32, 236, 5, 5, 67, 72, 69, 68, 32, 207, 2, 69, 38, 162, 1, 65, 132, + 2, 4, 78, 79, 82, 77, 78, 80, 46, 83, 170, 1, 84, 234, 221, 7, 69, 196, + 33, 7, 73, 68, 69, 78, 84, 73, 67, 154, 193, 4, 71, 38, 76, 191, 78, 67, + 10, 88, 3, 32, 83, 85, 52, 7, 78, 32, 69, 76, 69, 77, 69, 28, 2, 83, 89, + 251, 128, 8, 76, 4, 30, 66, 1, 3, 80, 69, 82, 2, 29, 2, 83, 69, 2, 11, + 78, 2, 183, 125, 84, 2, 37, 7, 77, 80, 84, 79, 84, 73, 67, 2, 17, 2, 65, + 76, 2, 185, 129, 8, 2, 76, 89, 4, 253, 195, 6, 14, 65, 76, 32, 83, 85, + 66, 71, 82, 79, 85, 80, 32, 79, 70, 2, 137, 129, 8, 6, 65, 82, 65, 76, + 76, 69, 6, 48, 6, 81, 85, 65, 82, 69, 32, 247, 204, 13, 73, 4, 68, 5, 73, + 77, 65, 71, 69, 1, 8, 79, 82, 73, 71, 73, 78, 65, 76, 2, 21, 3, 32, 79, + 70, 2, 255, 193, 6, 32, 4, 210, 199, 10, 82, 215, 241, 1, 73, 8, 58, 76, + 40, 4, 82, 73, 71, 72, 133, 1, 3, 85, 80, 80, 4, 36, 2, 69, 70, 133, 1, + 2, 79, 87, 2, 89, 20, 84, 32, 83, 69, 77, 73, 67, 73, 82, 67, 76, 69, 32, + 87, 73, 84, 72, 32, 84, 72, 2, 17, 2, 82, 69, 2, 167, 128, 13, 69, 2, 11, + 69, 2, 41, 8, 82, 32, 82, 73, 71, 72, 84, 45, 2, 197, 147, 3, 6, 83, 72, + 65, 68, 79, 87, 11, 34, 32, 53, 4, 66, 79, 79, 75, 4, 17, 2, 80, 65, 4, + 222, 184, 14, 71, 215, 22, 68, 5, 81, 18, 32, 87, 73, 84, 72, 32, 68, 69, + 67, 79, 82, 65, 84, 73, 86, 69, 32, 67, 2, 171, 141, 4, 79, 186, 6, 102, + 77, 176, 3, 4, 83, 72, 85, 32, 172, 178, 5, 8, 84, 32, 65, 78, 68, 32, + 66, 79, 187, 155, 8, 76, 26, 36, 4, 66, 69, 82, 32, 247, 2, 69, 24, 58, + 69, 50, 70, 42, 83, 66, 84, 57, 4, 78, 73, 78, 69, 4, 204, 1, 3, 73, 71, + 72, 21, 3, 76, 69, 86, 4, 144, 1, 2, 79, 85, 13, 2, 73, 70, 6, 34, 73, + 85, 4, 69, 86, 69, 78, 4, 82, 88, 175, 251, 13, 71, 8, 40, 2, 72, 73, 46, + 69, 21, 2, 87, 69, 2, 11, 82, 2, 17, 2, 84, 69, 2, 11, 69, 2, 171, 199, + 7, 78, 4, 148, 199, 7, 3, 76, 86, 69, 1, 3, 78, 84, 89, 2, 215, 242, 6, + 82, 154, 6, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 251, + 166, 12, 73, 152, 6, 18, 49, 87, 50, 160, 2, 222, 1, 55, 2, 56, 2, 57, 2, + 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 248, 3, 138, 1, 48, 2, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, + 68, 2, 69, 143, 1, 70, 32, 194, 199, 14, 48, 2, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, + 70, 24, 182, 198, 14, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 2, 56, 2, 57, 2, 65, 3, 66, 142, 1, 118, 76, 226, 3, 83, 164, 2, 5, + 84, 79, 78, 69, 45, 148, 225, 7, 8, 67, 73, 82, 67, 76, 69, 68, 32, 155, + 236, 4, 68, 92, 88, 6, 69, 84, 84, 69, 82, 32, 197, 163, 1, 10, 79, 71, + 79, 71, 82, 65, 77, 32, 78, 89, 90, 130, 2, 78, 90, 84, 246, 214, 7, 88, + 162, 126, 65, 232, 203, 1, 2, 72, 65, 242, 70, 82, 206, 147, 1, 79, 174, + 60, 68, 2, 77, 2, 80, 150, 201, 1, 69, 234, 61, 67, 2, 70, 2, 71, 2, 75, + 2, 76, 2, 81, 2, 83, 2, 86, 2, 89, 2, 90, 186, 2, 73, 2, 85, 3, 87, 22, + 86, 84, 230, 183, 12, 80, 254, 134, 2, 67, 2, 75, 2, 81, 2, 82, 2, 89, + 187, 2, 65, 6, 222, 190, 14, 83, 2, 88, 187, 2, 65, 14, 64, 4, 73, 71, + 78, 32, 189, 1, 7, 89, 76, 76, 65, 66, 76, 69, 12, 56, 4, 70, 79, 82, 32, + 209, 193, 13, 4, 88, 87, 32, 88, 10, 156, 9, 2, 76, 79, 194, 164, 3, 84, + 232, 93, 8, 73, 78, 86, 69, 82, 84, 69, 66, 160, 141, 4, 3, 65, 78, 73, + 183, 167, 2, 80, 2, 209, 244, 11, 4, 32, 76, 69, 78, 14, 202, 190, 14, + 66, 2, 68, 2, 71, 2, 74, 2, 77, 2, 83, 3, 86, 250, 14, 178, 2, 66, 226, + 1, 67, 224, 5, 4, 70, 70, 73, 67, 54, 71, 248, 6, 4, 73, 76, 32, 68, 22, + 76, 186, 69, 78, 150, 5, 80, 234, 7, 82, 218, 10, 83, 188, 8, 2, 84, 84, + 154, 8, 85, 248, 1, 3, 86, 69, 82, 198, 170, 2, 89, 202, 198, 5, 72, 238, + 254, 1, 75, 154, 249, 1, 68, 194, 64, 77, 246, 9, 87, 183, 137, 1, 88, + 10, 132, 1, 6, 76, 73, 81, 85, 69, 32, 180, 158, 2, 9, 83, 69, 82, 86, + 69, 82, 32, 69, 89, 189, 29, 8, 74, 69, 67, 84, 32, 82, 69, 80, 6, 176, + 198, 4, 13, 65, 78, 71, 76, 69, 32, 79, 80, 69, 78, 73, 78, 71, 175, 241, + 1, 72, 28, 56, 2, 82, 32, 234, 4, 84, 229, 193, 5, 3, 67, 85, 76, 22, + 156, 1, 11, 65, 77, 79, 85, 78, 84, 32, 79, 70, 32, 67, 22, 66, 198, 1, + 67, 118, 68, 106, 70, 0, 10, 73, 78, 86, 69, 82, 84, 69, 68, 32, 70, 143, + 225, 12, 72, 2, 207, 179, 5, 72, 6, 136, 1, 15, 82, 65, 78, 67, 72, 32, + 66, 65, 78, 75, 32, 73, 68, 69, 78, 160, 254, 4, 5, 69, 76, 84, 32, 66, + 157, 134, 6, 3, 79, 87, 32, 2, 21, 3, 84, 73, 70, 2, 11, 73, 2, 215, 244, + 10, 67, 4, 92, 17, 85, 83, 84, 79, 77, 69, 82, 32, 65, 67, 67, 79, 85, + 78, 84, 32, 78, 155, 202, 1, 72, 2, 183, 156, 6, 85, 4, 44, 5, 79, 85, + 66, 76, 69, 251, 204, 12, 65, 2, 11, 32, 2, 11, 66, 2, 177, 158, 7, 3, + 65, 67, 75, 2, 191, 243, 13, 79, 4, 152, 241, 8, 4, 65, 71, 79, 78, 181, + 189, 3, 2, 79, 80, 2, 11, 69, 2, 221, 251, 7, 5, 32, 66, 85, 73, 76, 60, + 48, 4, 72, 65, 77, 32, 133, 176, 14, 2, 79, 78, 58, 122, 70, 0, 10, 82, + 69, 86, 69, 82, 83, 69, 68, 32, 70, 36, 7, 76, 69, 84, 84, 69, 82, 32, + 137, 253, 3, 3, 83, 80, 65, 2, 149, 138, 4, 4, 69, 65, 84, 72, 52, 196, + 1, 2, 65, 73, 22, 66, 2, 80, 34, 67, 36, 2, 69, 65, 64, 2, 70, 69, 22, + 71, 22, 73, 58, 76, 2, 82, 22, 78, 46, 79, 34, 83, 50, 85, 206, 184, 1, + 77, 170, 9, 68, 153, 210, 11, 3, 84, 73, 78, 2, 231, 159, 14, 76, 2, 11, + 69, 2, 143, 174, 12, 73, 4, 142, 147, 8, 69, 191, 149, 4, 79, 6, 138, 1, + 66, 2, 68, 169, 141, 4, 6, 77, 72, 65, 78, 67, 72, 2, 143, 165, 9, 65, 2, + 131, 168, 13, 79, 4, 32, 2, 79, 68, 139, 175, 13, 70, 2, 239, 132, 8, 72, + 2, 147, 139, 12, 85, 4, 236, 154, 13, 3, 71, 69, 65, 219, 67, 73, 4, 166, + 222, 13, 78, 227, 79, 82, 4, 146, 250, 11, 65, 233, 177, 1, 3, 84, 82, + 65, 6, 60, 5, 73, 76, 76, 69, 65, 234, 170, 12, 65, 147, 130, 2, 82, 2, + 151, 221, 13, 78, 2, 211, 131, 13, 82, 182, 8, 38, 32, 142, 11, 68, 199, + 164, 13, 73, 184, 1, 64, 6, 67, 72, 73, 75, 73, 32, 205, 6, 5, 79, 78, + 65, 76, 32, 96, 132, 1, 7, 76, 69, 84, 84, 69, 82, 32, 148, 3, 2, 77, 85, + 62, 71, 74, 80, 204, 215, 3, 2, 82, 69, 210, 221, 8, 68, 235, 170, 1, 65, + 60, 50, 65, 98, 69, 54, 73, 50, 76, 62, 79, 51, 85, 16, 50, 65, 154, 169, + 14, 78, 86, 71, 2, 76, 3, 84, 8, 234, 169, 14, 74, 2, 75, 2, 77, 3, 87, + 8, 158, 227, 13, 68, 198, 13, 82, 222, 56, 78, 3, 80, 8, 194, 151, 14, + 78, 202, 17, 72, 2, 82, 3, 83, 12, 214, 188, 8, 65, 134, 236, 5, 69, 2, + 73, 2, 79, 3, 85, 8, 242, 139, 14, 84, 174, 28, 66, 2, 72, 3, 86, 8, 142, + 216, 13, 78, 226, 79, 67, 2, 68, 3, 89, 4, 56, 2, 45, 71, 129, 190, 12, + 6, 32, 84, 84, 85, 68, 68, 2, 253, 189, 12, 13, 65, 65, 72, 76, 65, 65, + 32, 84, 84, 85, 68, 68, 65, 6, 84, 11, 85, 78, 67, 84, 85, 65, 84, 73, + 79, 78, 32, 129, 219, 6, 4, 72, 65, 65, 82, 4, 48, 8, 68, 79, 85, 66, 76, + 69, 32, 77, 3, 77, 2, 129, 223, 13, 3, 85, 67, 65, 88, 100, 7, 76, 69, + 84, 84, 69, 82, 32, 192, 2, 5, 83, 73, 71, 78, 32, 142, 183, 8, 65, 191, + 249, 3, 68, 60, 42, 65, 54, 69, 58, 73, 54, 79, 59, 85, 13, 250, 163, 14, + 66, 2, 68, 2, 72, 2, 76, 3, 87, 13, 230, 211, 13, 78, 226, 79, 67, 2, 71, + 2, 72, 3, 83, 13, 174, 195, 8, 84, 226, 223, 5, 68, 2, 78, 3, 80, 13, + 254, 233, 13, 82, 138, 56, 78, 86, 77, 2, 79, 3, 89, 13, 130, 220, 13, + 68, 218, 52, 78, 202, 17, 74, 2, 75, 3, 82, 6, 58, 73, 236, 228, 12, 4, + 72, 79, 68, 68, 219, 151, 1, 77, 2, 243, 181, 1, 75, 252, 6, 34, 32, 153, + 57, 3, 69, 82, 32, 246, 6, 240, 2, 8, 67, 72, 73, 78, 69, 83, 69, 32, 44, + 10, 72, 85, 78, 71, 65, 82, 73, 65, 78, 32, 244, 6, 7, 73, 84, 65, 76, + 73, 67, 32, 212, 4, 14, 78, 79, 82, 84, 72, 32, 65, 82, 65, 66, 73, 65, + 78, 32, 196, 4, 3, 80, 69, 82, 216, 13, 2, 83, 79, 144, 13, 14, 84, 85, + 82, 75, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 132, 7, 7, 85, 89, 71, + 72, 85, 82, 32, 219, 209, 11, 75, 4, 174, 251, 11, 73, 177, 95, 3, 72, + 79, 79, 216, 1, 96, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 221, + 5, 7, 78, 85, 77, 66, 69, 82, 32, 102, 45, 9, 76, 32, 76, 69, 84, 84, 69, + 82, 32, 102, 226, 1, 65, 54, 69, 164, 2, 10, 78, 73, 75, 79, 76, 83, 66, + 85, 82, 71, 0, 9, 82, 85, 68, 73, 77, 69, 78, 84, 65, 42, 79, 34, 85, + 192, 240, 3, 5, 67, 76, 79, 83, 69, 150, 157, 8, 73, 241, 203, 1, 6, 83, + 72, 79, 82, 84, 32, 11, 234, 222, 12, 77, 198, 117, 78, 162, 70, 65, 3, + 75, 63, 174, 1, 77, 22, 78, 78, 83, 134, 248, 9, 67, 86, 71, 2, 76, 2, + 84, 198, 152, 2, 90, 242, 134, 2, 66, 2, 68, 2, 69, 2, 70, 2, 72, 2, 74, + 2, 75, 2, 80, 2, 82, 3, 86, 5, 235, 152, 14, 80, 11, 34, 84, 182, 152, + 14, 67, 3, 89, 5, 189, 148, 2, 5, 45, 83, 72, 65, 80, 5, 139, 152, 14, + 90, 4, 11, 32, 4, 150, 129, 14, 79, 3, 85, 7, 250, 128, 14, 69, 215, 22, + 79, 9, 150, 148, 14, 78, 154, 3, 83, 3, 85, 12, 210, 4, 70, 222, 254, 5, + 79, 243, 191, 6, 84, 78, 80, 7, 76, 69, 84, 84, 69, 82, 32, 169, 3, 8, + 78, 85, 77, 69, 82, 65, 76, 32, 70, 190, 1, 69, 90, 75, 50, 83, 36, 3, + 78, 79, 82, 242, 191, 10, 85, 206, 201, 1, 73, 190, 137, 1, 80, 2, 84, + 150, 83, 67, 186, 22, 66, 2, 68, 2, 72, 2, 86, 2, 89, 2, 90, 214, 22, 65, + 3, 79, 23, 186, 244, 9, 83, 154, 153, 2, 82, 150, 201, 1, 75, 222, 61, + 70, 2, 76, 2, 77, 3, 78, 8, 150, 253, 13, 72, 214, 22, 65, 2, 69, 3, 85, + 4, 32, 2, 79, 85, 199, 252, 13, 72, 2, 29, 5, 84, 72, 69, 82, 78, 2, 233, + 137, 13, 2, 32, 84, 8, 38, 70, 206, 190, 12, 84, 191, 58, 79, 4, 11, 73, + 4, 174, 165, 12, 70, 167, 214, 1, 86, 64, 76, 7, 76, 69, 84, 84, 69, 82, + 32, 209, 3, 7, 78, 85, 77, 66, 69, 82, 32, 58, 210, 1, 65, 38, 71, 38, + 72, 30, 75, 38, 84, 74, 90, 134, 107, 77, 208, 153, 3, 2, 69, 83, 226, + 161, 3, 66, 2, 70, 2, 82, 2, 89, 138, 193, 3, 78, 218, 235, 1, 76, 190, + 56, 68, 146, 1, 81, 134, 3, 87, 131, 56, 83, 4, 186, 193, 3, 76, 199, + 254, 9, 73, 4, 230, 226, 12, 72, 171, 154, 1, 69, 4, 134, 142, 14, 65, 3, + 69, 4, 250, 174, 7, 72, 223, 222, 5, 65, 8, 34, 72, 166, 141, 14, 65, 3, + 69, 4, 254, 132, 13, 65, 167, 136, 1, 69, 4, 11, 65, 4, 162, 190, 13, 73, + 227, 79, 72, 6, 154, 148, 6, 84, 159, 224, 6, 79, 180, 1, 64, 11, 77, 73, + 67, 32, 76, 69, 84, 84, 69, 82, 32, 183, 5, 83, 76, 228, 1, 2, 67, 72, + 22, 68, 66, 69, 22, 73, 30, 77, 2, 78, 30, 80, 22, 83, 70, 84, 50, 86, + 38, 89, 94, 90, 218, 190, 6, 76, 2, 82, 162, 222, 2, 71, 146, 165, 2, 79, + 142, 205, 1, 66, 246, 40, 65, 254, 31, 75, 174, 45, 72, 187, 2, 85, 2, + 139, 212, 7, 69, 6, 26, 90, 139, 247, 13, 79, 4, 174, 156, 9, 72, 231, + 201, 4, 73, 5, 187, 138, 14, 70, 7, 166, 138, 14, 65, 3, 69, 2, 169, 243, + 13, 2, 69, 78, 2, 223, 192, 6, 69, 6, 26, 72, 235, 245, 13, 73, 4, 140, + 155, 9, 3, 67, 72, 79, 3, 79, 4, 26, 83, 167, 245, 13, 65, 2, 147, 228, + 13, 73, 4, 182, 154, 9, 79, 215, 181, 4, 69, 14, 60, 2, 69, 82, 166, 168, + 8, 65, 154, 206, 5, 82, 203, 17, 85, 7, 130, 136, 14, 73, 3, 85, 4, 182, + 153, 9, 72, 231, 201, 4, 65, 104, 88, 4, 73, 65, 78, 32, 241, 5, 13, 79, + 78, 65, 76, 32, 67, 79, 77, 80, 85, 84, 69, 82, 100, 80, 7, 78, 85, 77, + 66, 69, 82, 32, 80, 5, 83, 73, 71, 78, 32, 163, 207, 10, 87, 10, 42, 84, + 182, 178, 8, 72, 139, 186, 4, 79, 6, 142, 230, 1, 87, 251, 207, 11, 69, + 88, 210, 1, 65, 86, 66, 62, 68, 98, 74, 2, 86, 30, 84, 42, 88, 210, 111, + 71, 2, 75, 2, 78, 2, 82, 162, 190, 9, 77, 190, 139, 3, 83, 218, 69, 67, + 2, 70, 2, 72, 2, 76, 2, 80, 2, 89, 2, 90, 186, 2, 73, 3, 85, 9, 45, 9, + 85, 82, 65, 77, 65, 90, 68, 65, 65, 7, 146, 234, 6, 45, 247, 150, 7, 72, + 6, 38, 65, 253, 161, 10, 3, 85, 85, 77, 5, 187, 128, 14, 71, 10, 34, 65, + 190, 130, 14, 73, 3, 85, 7, 37, 7, 72, 89, 65, 65, 85, 83, 72, 5, 231, + 232, 6, 45, 4, 254, 129, 14, 65, 3, 73, 6, 170, 255, 13, 72, 186, 2, 65, + 3, 85, 4, 176, 204, 11, 8, 83, 72, 65, 65, 89, 65, 84, 72, 139, 181, 2, + 65, 5, 141, 142, 5, 31, 32, 87, 73, 84, 72, 32, 77, 79, 78, 73, 84, 79, + 82, 32, 73, 78, 32, 80, 79, 82, 84, 82, 65, 73, 84, 32, 79, 82, 73, 69, + 78, 144, 1, 92, 6, 71, 68, 73, 65, 78, 32, 141, 7, 12, 85, 84, 72, 32, + 65, 82, 65, 66, 73, 65, 78, 32, 80, 58, 70, 74, 76, 145, 5, 7, 78, 85, + 77, 66, 69, 82, 32, 2, 11, 82, 2, 241, 221, 11, 10, 65, 67, 84, 73, 79, + 78, 32, 79, 78, 69, 60, 76, 6, 69, 84, 84, 69, 82, 32, 149, 4, 8, 73, 71, + 65, 84, 85, 82, 69, 32, 58, 234, 1, 65, 96, 6, 70, 73, 78, 65, 76, 32, + 200, 1, 5, 82, 69, 83, 72, 45, 194, 215, 1, 76, 254, 165, 4, 71, 90, 90, + 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, 178, 216, 4, + 78, 134, 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 6, 26, 76, 215, 251, 12, + 89, 4, 156, 128, 6, 8, 84, 69, 82, 78, 65, 84, 69, 32, 243, 214, 1, 69, + 18, 116, 3, 78, 85, 78, 0, 5, 83, 65, 68, 72, 69, 0, 3, 84, 65, 87, 222, + 215, 1, 65, 178, 201, 6, 66, 143, 194, 5, 72, 5, 41, 8, 32, 87, 73, 84, + 72, 32, 86, 69, 2, 241, 214, 5, 4, 82, 84, 73, 67, 2, 157, 215, 1, 6, 65, + 89, 73, 78, 45, 68, 18, 42, 84, 198, 254, 5, 79, 223, 137, 6, 70, 10, 42, + 72, 194, 216, 1, 87, 251, 207, 11, 69, 4, 222, 136, 12, 82, 159, 2, 73, + 64, 60, 7, 76, 69, 84, 84, 69, 82, 32, 245, 3, 3, 78, 85, 77, 58, 202, 1, + 65, 38, 68, 74, 71, 34, 75, 34, 83, 78, 84, 230, 43, 90, 182, 166, 1, 76, + 50, 81, 238, 205, 1, 82, 154, 217, 2, 89, 158, 208, 1, 72, 174, 81, 66, + 178, 216, 4, 78, 134, 2, 87, 218, 103, 70, 171, 4, 77, 4, 198, 167, 3, + 76, 199, 254, 9, 89, 6, 32, 2, 72, 65, 183, 211, 1, 65, 4, 194, 156, 8, + 76, 215, 171, 5, 68, 4, 238, 44, 72, 191, 204, 5, 73, 4, 198, 208, 7, 65, + 187, 75, 72, 8, 26, 65, 211, 244, 12, 72, 6, 142, 194, 7, 77, 138, 133, + 6, 68, 143, 45, 84, 8, 130, 91, 72, 226, 150, 11, 69, 243, 131, 1, 65, 6, + 56, 4, 66, 69, 82, 32, 229, 185, 13, 4, 69, 82, 73, 67, 4, 26, 70, 195, + 217, 12, 79, 2, 199, 132, 12, 73, 146, 1, 80, 7, 79, 82, 75, 72, 79, 78, + 32, 197, 3, 8, 89, 69, 78, 73, 83, 69, 73, 32, 84, 54, 65, 202, 1, 69, + 122, 73, 30, 79, 195, 134, 12, 66, 45, 106, 69, 194, 208, 9, 83, 158, + 160, 4, 66, 2, 68, 2, 71, 2, 76, 2, 78, 2, 81, 2, 82, 2, 84, 3, 89, 20, + 218, 240, 13, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 78, 2, 82, 2, 83, 2, 84, + 3, 89, 20, 74, 78, 138, 211, 13, 76, 158, 27, 83, 146, 1, 67, 2, 77, 2, + 80, 3, 90, 8, 178, 239, 13, 67, 2, 71, 2, 84, 3, 89, 7, 134, 239, 13, 67, + 3, 81, 13, 130, 3, 69, 234, 235, 13, 80, 2, 81, 3, 84, 62, 38, 65, 170, + 1, 69, 86, 73, 23, 79, 39, 98, 69, 162, 236, 13, 83, 62, 78, 86, 66, 2, + 68, 2, 71, 2, 76, 2, 81, 2, 82, 2, 84, 3, 89, 17, 150, 242, 11, 78, 154, + 251, 1, 66, 2, 71, 2, 75, 2, 84, 3, 89, 15, 46, 78, 174, 235, 13, 83, + 146, 1, 67, 3, 90, 6, 186, 236, 13, 67, 2, 84, 3, 89, 5, 151, 236, 13, + 81, 6, 26, 69, 235, 235, 13, 81, 5, 231, 235, 13, 75, 52, 148, 1, 10, 67, + 79, 77, 66, 73, 78, 73, 78, 71, 32, 36, 7, 76, 69, 84, 84, 69, 82, 32, + 233, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 8, 186, 236, + 5, 84, 143, 227, 3, 68, 36, 134, 200, 1, 65, 210, 206, 1, 82, 130, 216, + 2, 76, 58, 90, 34, 83, 66, 89, 128, 211, 1, 6, 70, 73, 78, 65, 76, 32, 0, + 6, 71, 73, 77, 69, 76, 45, 134, 3, 75, 198, 75, 66, 178, 216, 4, 78, 134, + 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 8, 56, 4, 84, 87, 79, 32, 214, + 196, 11, 70, 235, 129, 1, 66, 4, 246, 210, 12, 66, 71, 68, 6, 130, 181, + 1, 87, 240, 154, 3, 3, 65, 68, 85, 231, 205, 7, 77, 22, 212, 1, 34, 32, + 87, 73, 84, 72, 32, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 77, + 65, 82, 75, 32, 87, 73, 84, 72, 32, 76, 69, 70, 84, 32, 82, 44, 7, 67, + 79, 77, 73, 78, 71, 32, 194, 1, 69, 235, 147, 13, 73, 2, 21, 3, 73, 71, + 72, 2, 243, 235, 9, 84, 10, 148, 1, 6, 65, 85, 84, 79, 77, 79, 20, 7, 70, + 73, 82, 69, 32, 69, 78, 192, 224, 2, 2, 84, 65, 148, 232, 8, 6, 80, 79, + 76, 73, 67, 69, 155, 21, 66, 2, 135, 191, 11, 66, 2, 175, 202, 12, 71, 8, + 70, 32, 205, 174, 12, 11, 45, 80, 73, 69, 67, 69, 32, 83, 87, 73, 77, 6, + 40, 4, 68, 79, 84, 32, 215, 157, 10, 66, 4, 26, 79, 175, 159, 10, 76, 2, + 177, 233, 2, 11, 86, 69, 82, 32, 84, 87, 79, 32, 68, 79, 84, 46, 82, 69, + 188, 6, 2, 84, 73, 248, 212, 11, 5, 72, 73, 85, 67, 72, 171, 180, 1, 80, + 36, 70, 78, 201, 5, 12, 82, 65, 84, 73, 78, 71, 32, 83, 89, 83, 84, 69, + 34, 22, 32, 139, 4, 45, 28, 108, 2, 66, 79, 32, 7, 67, 69, 78, 84, 82, + 69, 32, 114, 70, 70, 72, 42, 77, 226, 133, 7, 83, 215, 233, 3, 76, 4, + 198, 220, 13, 79, 155, 3, 88, 8, 50, 84, 222, 200, 11, 66, 222, 2, 65, + 191, 82, 67, 2, 37, 7, 69, 65, 82, 68, 82, 79, 80, 2, 191, 202, 11, 45, + 4, 44, 5, 73, 76, 69, 32, 70, 239, 130, 2, 79, 2, 235, 130, 2, 79, 2, 17, + 2, 65, 78, 2, 191, 175, 10, 68, 4, 57, 12, 65, 73, 76, 66, 79, 88, 32, + 87, 73, 84, 72, 32, 4, 44, 3, 76, 79, 87, 21, 4, 82, 65, 73, 83, 2, 17, + 2, 69, 82, 2, 189, 243, 11, 2, 69, 68, 6, 108, 15, 67, 73, 82, 67, 85, + 73, 84, 45, 79, 85, 84, 80, 85, 84, 32, 181, 187, 12, 6, 79, 85, 84, 76, + 73, 78, 4, 18, 72, 3, 76, 2, 173, 191, 1, 4, 45, 84, 89, 80, 2, 153, 205, + 12, 6, 77, 32, 67, 79, 77, 77, 6, 64, 2, 79, 78, 237, 177, 1, 8, 67, 65, + 76, 32, 68, 73, 83, 67, 2, 167, 192, 11, 32, 206, 1, 112, 3, 65, 78, 71, + 66, 73, 212, 8, 5, 78, 65, 84, 69, 32, 32, 3, 84, 72, 79, 202, 171, 5, + 32, 175, 158, 6, 67, 6, 28, 2, 69, 32, 135, 22, 85, 4, 174, 133, 12, 66, + 175, 76, 72, 188, 1, 48, 5, 71, 73, 78, 65, 76, 21, 3, 89, 65, 32, 2, + 203, 133, 1, 32, 186, 1, 106, 65, 30, 70, 250, 1, 73, 32, 7, 76, 69, 84, + 84, 69, 82, 32, 142, 2, 83, 218, 1, 86, 243, 223, 11, 68, 4, 246, 249, 9, 73, 3, 85, 12, 41, 8, 82, 65, 67, 84, 73, 79, 78, 32, 12, 56, 4, 79, 78, - 69, 32, 81, 6, 84, 72, 82, 69, 69, 32, 8, 42, 83, 170, 140, 11, 69, 46, - 72, 43, 81, 2, 177, 141, 11, 4, 73, 88, 84, 69, 4, 26, 83, 187, 142, 11, - 81, 2, 249, 223, 7, 4, 73, 88, 84, 69, 2, 225, 233, 11, 3, 83, 83, 72, - 104, 226, 1, 82, 166, 212, 6, 89, 150, 205, 2, 65, 38, 68, 46, 84, 46, - 86, 186, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, - 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 77, 2, 87, 186, 2, - 69, 3, 79, 6, 206, 135, 13, 72, 2, 82, 187, 2, 65, 18, 112, 20, 69, 81, - 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 82, - 29, 4, 73, 71, 78, 32, 4, 178, 134, 13, 72, 3, 82, 14, 222, 161, 9, 65, - 74, 67, 98, 78, 234, 128, 2, 79, 227, 161, 1, 86, 26, 49, 10, 79, 87, 69, - 76, 32, 83, 73, 71, 78, 32, 26, 254, 162, 9, 65, 106, 86, 214, 20, 85, - 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 4, 242, 171, 9, 76, 27, 82, 226, - 1, 76, 4, 65, 71, 69, 32, 164, 4, 6, 77, 65, 78, 89, 65, 32, 247, 129, - 13, 67, 144, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 72, - 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 72, 194, 1, 65, 38, 69, 114, - 75, 42, 79, 22, 84, 178, 204, 6, 72, 170, 190, 4, 67, 2, 68, 2, 71, 138, - 176, 1, 83, 2, 90, 130, 3, 66, 254, 65, 76, 2, 77, 2, 78, 2, 80, 2, 87, - 186, 2, 73, 3, 85, 9, 226, 139, 11, 73, 131, 248, 1, 72, 15, 26, 72, 211, - 179, 12, 73, 10, 50, 84, 182, 139, 11, 67, 134, 245, 1, 75, 3, 80, 4, - 182, 128, 13, 83, 187, 2, 65, 6, 150, 128, 13, 72, 2, 89, 187, 2, 65, 5, - 211, 178, 12, 73, 6, 222, 186, 12, 83, 183, 71, 65, 80, 52, 7, 76, 69, - 84, 84, 69, 82, 32, 163, 147, 11, 68, 60, 246, 1, 65, 38, 67, 22, 68, 38, - 75, 34, 83, 30, 77, 130, 229, 2, 81, 174, 218, 7, 79, 204, 119, 2, 76, - 65, 156, 71, 2, 78, 85, 254, 1, 87, 138, 62, 69, 222, 61, 66, 2, 70, 2, - 71, 2, 72, 2, 74, 2, 82, 2, 84, 2, 88, 2, 89, 186, 2, 73, 3, 85, 7, 158, - 238, 2, 76, 167, 145, 10, 65, 2, 195, 129, 12, 65, 4, 178, 158, 8, 69, - 163, 222, 4, 72, 4, 206, 253, 11, 65, 227, 126, 72, 4, 26, 72, 175, 254, - 12, 65, 2, 239, 254, 11, 73, 124, 68, 11, 79, 77, 65, 78, 32, 83, 73, 89, - 65, 81, 32, 131, 197, 12, 69, 122, 172, 1, 17, 65, 76, 84, 69, 82, 78, - 65, 84, 69, 32, 78, 85, 77, 66, 69, 82, 32, 200, 1, 13, 70, 82, 65, 67, - 84, 73, 79, 78, 32, 79, 78, 69, 32, 48, 3, 77, 65, 82, 47, 78, 26, 54, - 70, 50, 83, 46, 84, 238, 225, 11, 78, 211, 110, 69, 6, 208, 187, 5, 3, - 79, 85, 82, 211, 195, 6, 73, 6, 160, 187, 5, 2, 73, 88, 171, 211, 5, 69, - 10, 138, 173, 2, 69, 12, 2, 87, 79, 183, 225, 8, 72, 4, 26, 83, 163, 250, - 10, 72, 2, 139, 251, 10, 73, 2, 11, 82, 2, 11, 65, 2, 219, 178, 11, 84, - 90, 33, 6, 85, 77, 66, 69, 82, 32, 90, 58, 69, 66, 70, 94, 78, 26, 83, - 78, 84, 131, 157, 5, 79, 10, 25, 4, 73, 71, 72, 84, 11, 138, 171, 2, 89, - 191, 165, 5, 32, 20, 18, 73, 35, 79, 10, 166, 2, 70, 147, 156, 5, 86, 10, - 134, 2, 82, 157, 156, 5, 2, 85, 82, 10, 65, 3, 73, 78, 69, 20, 40, 4, 69, - 86, 69, 78, 1, 2, 73, 88, 11, 166, 1, 84, 223, 205, 7, 32, 24, 34, 72, - 50, 87, 203, 168, 2, 69, 10, 34, 73, 197, 156, 5, 2, 82, 69, 4, 51, 82, - 10, 26, 69, 171, 156, 5, 79, 4, 11, 78, 4, 11, 84, 4, 159, 168, 2, 89, - 12, 34, 84, 133, 222, 10, 2, 78, 67, 10, 42, 66, 37, 6, 76, 73, 78, 69, - 68, 32, 2, 181, 233, 11, 4, 79, 88, 32, 84, 8, 254, 252, 2, 76, 254, 174, - 3, 87, 250, 181, 4, 66, 239, 13, 71, 12, 18, 72, 35, 76, 2, 245, 227, 11, - 3, 69, 65, 84, 10, 32, 2, 65, 80, 227, 218, 11, 73, 9, 29, 5, 80, 73, 78, - 71, 32, 6, 40, 6, 87, 72, 73, 84, 69, 32, 51, 66, 4, 44, 5, 65, 78, 68, - 32, 66, 247, 199, 9, 83, 2, 221, 199, 9, 4, 76, 65, 67, 75, 144, 13, 194, - 1, 65, 174, 65, 68, 30, 69, 194, 13, 72, 134, 25, 73, 206, 4, 76, 150, - 15, 79, 234, 8, 82, 208, 15, 15, 83, 65, 76, 84, 69, 82, 32, 80, 65, 72, - 76, 65, 86, 73, 32, 214, 5, 85, 151, 215, 11, 77, 154, 6, 134, 2, 68, 38, - 71, 212, 1, 11, 72, 65, 87, 72, 32, 72, 77, 79, 78, 71, 32, 138, 23, 76, - 130, 6, 78, 58, 82, 156, 22, 2, 83, 83, 132, 2, 10, 85, 32, 67, 73, 78, - 32, 72, 65, 85, 32, 180, 7, 3, 87, 32, 80, 172, 213, 7, 2, 67, 75, 145, - 217, 4, 4, 80, 69, 82, 67, 5, 157, 184, 1, 4, 68, 73, 78, 71, 14, 26, 69, - 191, 201, 12, 79, 13, 34, 32, 158, 238, 12, 82, 3, 83, 6, 72, 6, 87, 73, - 84, 72, 32, 67, 149, 139, 4, 6, 70, 65, 67, 73, 78, 71, 4, 50, 85, 181, - 197, 7, 6, 73, 82, 67, 76, 69, 68, 2, 255, 227, 11, 82, 254, 1, 190, 1, - 67, 160, 6, 9, 77, 65, 82, 75, 32, 67, 73, 77, 32, 160, 1, 7, 78, 85, 77, - 66, 69, 82, 32, 244, 1, 5, 83, 73, 71, 78, 32, 148, 10, 7, 86, 79, 87, - 69, 76, 32, 75, 227, 233, 10, 68, 78, 92, 9, 76, 65, 78, 32, 83, 73, 71, - 78, 32, 137, 3, 9, 79, 78, 83, 79, 78, 65, 78, 84, 32, 38, 132, 1, 2, 72, - 65, 38, 75, 46, 76, 34, 84, 82, 86, 30, 89, 148, 9, 2, 88, 89, 176, 5, 2, - 80, 72, 174, 1, 70, 165, 2, 2, 77, 85, 4, 162, 186, 2, 87, 187, 175, 10, - 77, 6, 250, 15, 72, 202, 186, 12, 79, 159, 14, 87, 4, 214, 12, 65, 231, - 158, 12, 73, 8, 22, 83, 247, 9, 72, 6, 152, 185, 2, 3, 72, 69, 69, 150, - 245, 8, 65, 3, 87, 4, 226, 184, 2, 65, 3, 87, 4, 198, 184, 2, 65, 223, - 233, 9, 69, 40, 122, 67, 38, 72, 46, 78, 60, 2, 80, 76, 2, 81, 222, 210, - 2, 76, 2, 77, 2, 82, 2, 86, 2, 88, 2, 89, 147, 238, 9, 65, 4, 230, 211, - 2, 72, 147, 238, 9, 65, 6, 194, 211, 2, 76, 2, 78, 147, 238, 9, 65, 12, - 58, 67, 22, 84, 202, 210, 2, 75, 2, 76, 147, 238, 9, 65, 2, 219, 210, 2, - 72, 4, 198, 210, 2, 72, 3, 83, 14, 42, 75, 50, 83, 38, 84, 195, 211, 12, - 72, 4, 26, 72, 143, 167, 12, 69, 2, 183, 138, 6, 65, 4, 138, 171, 11, 85, - 191, 185, 1, 79, 4, 254, 169, 11, 85, 179, 18, 65, 14, 52, 7, 72, 85, 78, - 68, 82, 69, 68, 38, 84, 79, 77, 4, 108, 2, 32, 77, 223, 226, 12, 83, 8, - 24, 2, 69, 78, 51, 82, 6, 26, 32, 243, 226, 12, 83, 4, 18, 66, 35, 84, 2, - 145, 194, 4, 3, 73, 76, 76, 2, 11, 72, 2, 137, 185, 9, 3, 79, 85, 83, 72, - 188, 1, 4, 67, 73, 77, 32, 250, 2, 72, 32, 3, 73, 66, 32, 22, 77, 86, 78, - 50, 84, 124, 4, 86, 79, 83, 32, 198, 1, 88, 208, 1, 6, 90, 87, 74, 32, - 84, 72, 150, 176, 1, 76, 219, 205, 4, 65, 16, 174, 1, 67, 84, 5, 78, 82, - 69, 83, 32, 22, 84, 148, 164, 11, 7, 80, 85, 66, 32, 68, 65, 87, 197, - 185, 1, 16, 72, 65, 73, 83, 32, 76, 85, 83, 32, 78, 84, 79, 71, 32, 78, - 84, 4, 48, 7, 85, 65, 77, 32, 84, 83, 72, 131, 4, 72, 2, 11, 79, 2, 167, - 175, 2, 79, 2, 207, 182, 1, 84, 6, 56, 3, 88, 87, 86, 157, 221, 12, 5, - 83, 79, 86, 32, 82, 5, 213, 131, 6, 4, 32, 67, 72, 87, 4, 214, 72, 78, - 171, 129, 12, 76, 2, 251, 163, 11, 89, 6, 40, 4, 69, 69, 74, 32, 171, - 159, 12, 85, 4, 128, 3, 2, 84, 83, 57, 2, 83, 85, 4, 26, 84, 231, 213, 8, - 81, 2, 251, 172, 2, 85, 6, 196, 1, 7, 88, 72, 69, 69, 74, 32, 67, 216, - 139, 8, 12, 72, 73, 82, 68, 45, 83, 84, 65, 71, 69, 32, 72, 155, 170, 4, - 65, 14, 54, 70, 22, 83, 30, 84, 190, 69, 76, 163, 212, 7, 78, 2, 191, - 200, 12, 69, 2, 177, 128, 6, 2, 69, 69, 6, 42, 72, 29, 6, 83, 72, 65, 66, - 32, 67, 4, 82, 73, 231, 200, 12, 79, 2, 223, 188, 5, 69, 14, 34, 73, 22, - 89, 191, 214, 10, 65, 2, 151, 159, 11, 65, 10, 40, 4, 69, 69, 77, 32, - 139, 186, 12, 79, 8, 84, 3, 78, 84, 88, 128, 254, 5, 2, 84, 79, 142, 158, - 2, 82, 233, 129, 3, 2, 70, 65, 2, 255, 253, 5, 73, 2, 215, 168, 2, 65, - 56, 50, 65, 66, 69, 38, 73, 2, 85, 38, 79, 39, 87, 20, 170, 1, 65, 2, 73, - 2, 85, 2, 87, 158, 214, 12, 66, 3, 86, 8, 106, 69, 158, 214, 12, 66, 3, - 86, 8, 70, 65, 158, 214, 12, 66, 3, 86, 8, 34, 79, 158, 214, 12, 66, 3, - 86, 4, 154, 214, 12, 66, 3, 86, 76, 18, 76, 23, 77, 2, 155, 152, 12, 65, - 74, 74, 32, 104, 6, 89, 82, 69, 78, 69, 32, 241, 249, 3, 4, 83, 32, 85, - 80, 8, 80, 3, 66, 82, 65, 250, 231, 10, 84, 192, 94, 4, 68, 79, 87, 78, - 1, 2, 85, 80, 2, 143, 178, 12, 78, 64, 80, 2, 76, 69, 40, 4, 82, 73, 71, - 72, 253, 2, 7, 78, 85, 77, 66, 69, 82, 32, 48, 38, 70, 89, 5, 84, 84, 69, - 82, 32, 2, 57, 12, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 70, 2, - 137, 209, 5, 2, 76, 69, 46, 224, 1, 5, 70, 73, 78, 65, 76, 30, 84, 246, - 118, 68, 34, 76, 50, 81, 206, 194, 1, 82, 134, 212, 2, 65, 50, 71, 90, - 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, 130, 76, 66, 190, 173, - 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, 2, 209, 208, 11, 2, 32, 78, - 4, 202, 209, 10, 69, 247, 128, 1, 65, 14, 198, 120, 84, 198, 234, 9, 70, - 143, 84, 79, 4, 32, 2, 67, 65, 183, 144, 12, 68, 2, 251, 187, 11, 75, - 184, 2, 94, 65, 216, 1, 11, 69, 78, 84, 72, 69, 83, 73, 90, 69, 68, 32, - 242, 16, 84, 243, 235, 11, 82, 14, 80, 5, 71, 82, 65, 80, 72, 48, 5, 76, - 76, 69, 76, 32, 201, 138, 6, 2, 67, 72, 6, 206, 121, 85, 198, 187, 8, 32, - 175, 219, 2, 79, 6, 44, 5, 87, 73, 84, 72, 32, 191, 174, 12, 84, 4, 214, - 165, 6, 84, 151, 138, 4, 72, 150, 2, 252, 1, 7, 72, 65, 78, 71, 85, 76, - 32, 188, 4, 10, 73, 68, 69, 79, 71, 82, 65, 80, 72, 32, 188, 7, 18, 75, - 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, 82, 32, 79, 44, - 7, 78, 85, 77, 66, 69, 82, 32, 138, 187, 4, 68, 197, 154, 2, 2, 76, 65, - 58, 102, 67, 110, 72, 30, 75, 66, 77, 34, 78, 34, 80, 62, 82, 30, 83, 26, - 84, 73, 5, 73, 69, 85, 78, 71, 10, 34, 72, 33, 4, 73, 69, 85, 67, 4, 141, - 3, 4, 73, 69, 85, 67, 7, 11, 32, 4, 206, 201, 12, 65, 3, 85, 4, 197, 2, - 3, 73, 69, 85, 8, 168, 2, 5, 72, 73, 69, 85, 75, 13, 5, 73, 89, 69, 79, - 75, 4, 245, 1, 4, 73, 69, 85, 77, 4, 213, 1, 4, 73, 69, 85, 78, 8, 168, - 1, 5, 72, 73, 69, 85, 80, 13, 4, 73, 69, 85, 80, 4, 121, 4, 73, 69, 85, - 76, 4, 93, 3, 73, 79, 83, 8, 56, 5, 72, 73, 69, 85, 84, 13, 5, 73, 75, - 69, 85, 84, 4, 11, 72, 5, 167, 196, 12, 32, 72, 148, 1, 2, 65, 76, 30, - 67, 74, 69, 82, 70, 112, 2, 76, 65, 22, 77, 38, 78, 32, 2, 82, 69, 78, - 83, 138, 2, 84, 50, 87, 134, 215, 10, 72, 159, 79, 79, 2, 225, 233, 7, 2, - 76, 73, 4, 32, 2, 79, 78, 199, 192, 10, 65, 2, 137, 212, 7, 4, 71, 82, - 65, 84, 6, 42, 78, 246, 253, 8, 65, 239, 155, 3, 73, 2, 169, 4, 5, 84, - 69, 82, 80, 82, 10, 58, 73, 156, 186, 11, 5, 69, 83, 84, 73, 86, 231, 16, - 79, 6, 208, 2, 3, 78, 65, 78, 158, 170, 12, 82, 3, 86, 2, 179, 138, 12, - 66, 4, 174, 245, 9, 69, 239, 253, 1, 79, 4, 194, 169, 11, 73, 195, 1, 65, - 8, 38, 83, 170, 178, 11, 80, 195, 109, 65, 4, 182, 179, 6, 79, 219, 142, - 6, 84, 18, 58, 69, 34, 79, 22, 80, 34, 84, 50, 85, 163, 180, 11, 73, 4, - 186, 240, 10, 86, 235, 79, 76, 2, 231, 212, 7, 67, 2, 11, 69, 2, 247, - 161, 4, 67, 4, 26, 85, 247, 145, 11, 79, 2, 247, 174, 12, 68, 4, 26, 80, - 147, 192, 12, 78, 2, 21, 3, 69, 82, 86, 2, 131, 183, 11, 73, 6, 162, 211, - 10, 72, 238, 156, 1, 69, 227, 48, 87, 4, 206, 157, 9, 79, 155, 235, 1, - 65, 4, 166, 253, 7, 32, 137, 242, 3, 2, 74, 69, 22, 42, 69, 46, 70, 42, - 78, 30, 83, 51, 84, 4, 216, 1, 3, 73, 71, 72, 243, 225, 4, 76, 4, 160, 1, - 2, 79, 85, 13, 2, 73, 70, 2, 133, 1, 3, 73, 78, 69, 4, 34, 73, 73, 4, 69, - 86, 69, 78, 2, 71, 88, 8, 34, 72, 46, 87, 247, 236, 11, 69, 2, 11, 73, 2, - 11, 82, 2, 219, 235, 10, 84, 4, 11, 69, 4, 198, 210, 10, 78, 179, 109, - 76, 18, 128, 1, 3, 73, 65, 76, 216, 1, 3, 89, 32, 80, 144, 141, 8, 6, 78, - 69, 82, 83, 72, 73, 189, 167, 3, 6, 32, 65, 76, 84, 69, 82, 12, 62, 32, - 189, 122, 10, 76, 89, 45, 82, 69, 67, 89, 67, 76, 69, 10, 38, 68, 41, 5, - 76, 73, 78, 69, 32, 2, 157, 155, 4, 5, 73, 70, 70, 69, 82, 8, 134, 186, - 3, 68, 242, 45, 70, 20, 4, 66, 65, 67, 75, 243, 209, 8, 85, 2, 235, 222, - 3, 79, 10, 98, 69, 52, 9, 73, 86, 69, 45, 80, 85, 76, 76, 45, 89, 9, 80, - 79, 82, 84, 32, 67, 79, 78, 84, 4, 182, 165, 11, 68, 145, 62, 5, 78, 71, - 69, 82, 32, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 233, 155, 5, 6, - 45, 79, 85, 84, 80, 85, 2, 227, 164, 11, 82, 114, 192, 1, 12, 71, 76, 79, - 84, 84, 65, 76, 32, 83, 84, 79, 80, 62, 76, 160, 3, 3, 82, 73, 83, 36, - 14, 77, 73, 68, 45, 76, 69, 86, 69, 76, 32, 84, 79, 78, 69, 57, 7, 83, - 65, 78, 68, 72, 73, 32, 7, 11, 32, 4, 206, 5, 70, 197, 166, 11, 4, 86, - 65, 82, 73, 82, 72, 6, 69, 84, 84, 69, 82, 32, 213, 2, 7, 79, 87, 45, 70, - 65, 76, 76, 74, 206, 1, 70, 226, 241, 6, 73, 2, 85, 166, 242, 1, 78, 214, - 135, 3, 67, 2, 75, 2, 80, 2, 84, 254, 68, 66, 2, 68, 2, 71, 2, 72, 2, 76, - 2, 77, 2, 82, 2, 83, 2, 86, 2, 90, 186, 2, 65, 2, 69, 3, 79, 20, 44, 5, - 73, 78, 65, 76, 32, 223, 178, 12, 65, 18, 198, 186, 10, 78, 150, 248, 1, - 75, 2, 76, 2, 77, 2, 80, 2, 84, 2, 87, 3, 89, 8, 157, 1, 5, 73, 78, 71, - 32, 84, 7, 11, 32, 4, 192, 1, 5, 76, 79, 78, 71, 32, 15, 70, 12, 66, 84, - 73, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 8, 21, 3, 79, 78, - 69, 9, 11, 32, 6, 32, 4, 76, 79, 78, 71, 27, 70, 5, 11, 32, 2, 11, 70, 2, - 135, 181, 3, 73, 2, 223, 161, 4, 82, 4, 222, 175, 12, 70, 3, 73, 80, 130, - 1, 65, 122, 78, 162, 1, 82, 222, 9, 83, 44, 3, 84, 82, 73, 254, 16, 68, - 173, 192, 11, 9, 79, 80, 76, 69, 32, 72, 85, 71, 71, 12, 50, 67, 50, 78, - 218, 204, 6, 32, 135, 225, 5, 82, 6, 190, 255, 10, 79, 190, 27, 69, 147, - 147, 1, 72, 2, 199, 153, 11, 85, 10, 118, 71, 20, 3, 83, 73, 86, 190, - 171, 1, 84, 224, 172, 4, 10, 32, 79, 86, 69, 82, 32, 83, 84, 65, 77, 175, - 167, 4, 67, 2, 147, 173, 11, 85, 2, 167, 239, 11, 69, 48, 186, 1, 32, - 116, 10, 80, 69, 78, 68, 73, 67, 85, 76, 65, 82, 42, 83, 198, 136, 7, 67, - 224, 10, 10, 77, 65, 78, 69, 78, 84, 32, 80, 65, 80, 153, 234, 1, 8, 70, - 79, 82, 77, 73, 78, 71, 32, 6, 34, 77, 30, 84, 223, 163, 11, 83, 2, 153, - 230, 1, 2, 73, 76, 2, 189, 196, 10, 8, 69, 78, 32, 84, 72, 79, 85, 83, 5, - 129, 215, 8, 5, 32, 87, 73, 84, 72, 32, 60, 2, 79, 78, 182, 75, 80, 213, - 213, 10, 4, 69, 86, 69, 82, 28, 22, 32, 243, 5, 65, 26, 216, 2, 10, 68, - 79, 73, 78, 71, 32, 67, 65, 82, 84, 32, 3, 73, 78, 32, 108, 5, 87, 73, - 84, 72, 32, 208, 185, 7, 4, 70, 82, 79, 87, 200, 13, 27, 82, 65, 73, 83, + 69, 32, 81, 6, 84, 72, 82, 69, 69, 32, 8, 42, 83, 162, 210, 11, 69, 46, + 72, 47, 81, 2, 173, 211, 11, 4, 73, 88, 84, 69, 4, 26, 83, 183, 212, 11, + 81, 2, 245, 253, 7, 4, 73, 88, 84, 69, 2, 217, 179, 12, 3, 83, 83, 72, + 104, 226, 1, 82, 238, 232, 6, 89, 134, 144, 3, 65, 38, 68, 114, 84, 46, + 86, 186, 5, 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, + 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 87, 186, 2, 69, + 3, 79, 6, 214, 208, 13, 72, 2, 82, 187, 2, 65, 18, 112, 20, 69, 81, 85, + 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 82, 29, + 4, 73, 71, 78, 32, 4, 186, 207, 13, 72, 3, 82, 14, 150, 189, 9, 67, 98, + 78, 242, 60, 65, 142, 239, 1, 79, 219, 164, 1, 86, 26, 49, 10, 79, 87, + 69, 76, 32, 83, 73, 71, 78, 32, 26, 142, 253, 9, 65, 38, 85, 22, 86, 186, + 201, 1, 73, 222, 137, 2, 69, 3, 79, 4, 174, 215, 6, 76, 243, 30, 82, 4, + 224, 240, 3, 2, 68, 79, 155, 183, 2, 71, 226, 1, 76, 4, 65, 71, 69, 32, + 140, 4, 6, 77, 65, 78, 89, 65, 32, 231, 202, 13, 67, 144, 1, 56, 6, 67, + 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 72, 45, 9, 76, 32, 76, 69, 84, + 84, 69, 82, 32, 72, 194, 1, 65, 38, 69, 90, 75, 42, 79, 22, 84, 226, 224, + 6, 72, 230, 239, 4, 67, 2, 68, 2, 71, 130, 179, 1, 83, 2, 90, 130, 3, 66, + 138, 66, 76, 2, 77, 2, 78, 2, 80, 2, 87, 186, 2, 73, 3, 85, 9, 182, 209, + 11, 73, 135, 251, 1, 72, 15, 26, 72, 159, 252, 12, 73, 10, 222, 191, 9, + 84, 222, 145, 2, 67, 138, 248, 1, 75, 3, 80, 6, 134, 201, 13, 72, 2, 89, + 187, 2, 65, 5, 183, 251, 12, 73, 6, 194, 131, 13, 83, 195, 71, 65, 80, + 52, 7, 76, 69, 84, 84, 69, 82, 32, 143, 217, 11, 68, 60, 246, 1, 65, 38, + 67, 22, 68, 38, 75, 34, 83, 30, 77, 242, 240, 2, 81, 206, 144, 8, 79, + 224, 123, 2, 76, 65, 208, 73, 2, 78, 85, 134, 2, 87, 142, 62, 69, 234, + 61, 66, 2, 70, 2, 71, 2, 72, 2, 74, 2, 82, 2, 84, 2, 88, 2, 89, 186, 2, + 73, 3, 85, 7, 142, 250, 2, 76, 167, 206, 10, 65, 2, 163, 202, 12, 65, 4, + 150, 187, 8, 69, 175, 138, 5, 72, 4, 166, 198, 12, 65, 251, 126, 72, 4, + 26, 72, 159, 199, 13, 65, 2, 199, 199, 12, 73, 124, 68, 11, 79, 77, 65, + 78, 32, 83, 73, 89, 65, 81, 32, 231, 141, 13, 69, 122, 172, 1, 17, 65, + 76, 84, 69, 82, 78, 65, 84, 69, 32, 78, 85, 77, 66, 69, 82, 32, 200, 1, + 13, 70, 82, 65, 67, 84, 73, 79, 78, 32, 79, 78, 69, 32, 48, 3, 77, 65, + 82, 47, 78, 26, 54, 70, 50, 83, 46, 84, 198, 170, 12, 78, 235, 110, 69, + 6, 236, 202, 5, 3, 79, 85, 82, 151, 253, 6, 73, 6, 188, 202, 5, 2, 73, + 88, 251, 137, 6, 69, 10, 170, 184, 2, 69, 12, 2, 87, 79, 131, 156, 9, 72, + 4, 26, 83, 131, 192, 11, 72, 2, 239, 192, 11, 73, 2, 11, 82, 2, 11, 65, + 2, 255, 248, 11, 84, 90, 33, 6, 85, 77, 66, 69, 82, 32, 90, 58, 69, 66, + 70, 94, 78, 26, 83, 78, 84, 183, 172, 5, 79, 10, 25, 4, 73, 71, 72, 84, + 11, 170, 182, 2, 89, 255, 183, 5, 32, 20, 18, 73, 35, 79, 10, 166, 2, 70, + 199, 171, 5, 86, 10, 134, 2, 82, 209, 171, 5, 2, 85, 82, 10, 65, 3, 73, + 78, 69, 20, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 11, 166, 1, 84, 191, + 235, 7, 32, 24, 34, 72, 50, 87, 235, 179, 2, 69, 10, 34, 73, 249, 171, 5, + 2, 82, 69, 4, 51, 82, 10, 26, 69, 223, 171, 5, 79, 4, 11, 78, 4, 11, 84, + 4, 191, 179, 2, 89, 84, 34, 84, 161, 162, 11, 2, 78, 67, 82, 42, 66, 37, + 6, 76, 73, 78, 69, 68, 32, 2, 141, 178, 12, 4, 79, 88, 32, 84, 80, 92, 7, + 76, 65, 84, 73, 78, 32, 67, 222, 189, 6, 87, 146, 233, 4, 66, 246, 13, + 71, 159, 23, 68, 54, 130, 170, 7, 65, 143, 210, 4, 82, 12, 18, 72, 35, + 76, 2, 137, 172, 12, 3, 69, 65, 84, 10, 32, 2, 65, 80, 247, 162, 12, 73, + 9, 29, 5, 80, 73, 78, 71, 32, 6, 40, 6, 87, 72, 73, 84, 69, 32, 51, 66, + 4, 44, 5, 65, 78, 68, 32, 66, 223, 250, 9, 83, 2, 197, 250, 9, 4, 76, 65, + 67, 75, 146, 13, 210, 1, 65, 174, 65, 68, 30, 69, 134, 13, 72, 138, 25, + 73, 206, 4, 76, 148, 15, 2, 78, 80, 54, 79, 234, 8, 82, 208, 15, 15, 83, + 65, 76, 84, 69, 82, 32, 80, 65, 72, 76, 65, 86, 73, 32, 214, 5, 85, 183, + 159, 12, 77, 154, 6, 134, 2, 68, 38, 71, 212, 1, 11, 72, 65, 87, 72, 32, + 72, 77, 79, 78, 71, 32, 134, 23, 76, 130, 6, 78, 58, 82, 160, 22, 2, 83, + 83, 132, 2, 10, 85, 32, 67, 73, 78, 32, 72, 65, 85, 32, 180, 7, 3, 87, + 32, 80, 188, 241, 7, 2, 67, 75, 157, 133, 5, 4, 80, 69, 82, 67, 5, 237, + 184, 1, 4, 68, 73, 78, 71, 14, 26, 69, 219, 145, 13, 79, 13, 34, 32, 186, + 182, 13, 82, 3, 83, 6, 72, 6, 87, 73, 84, 72, 32, 67, 241, 153, 4, 6, 70, + 65, 67, 73, 78, 71, 4, 50, 85, 193, 226, 7, 6, 73, 82, 67, 76, 69, 68, 2, + 131, 172, 12, 82, 254, 1, 190, 1, 67, 160, 6, 9, 77, 65, 82, 75, 32, 67, + 73, 77, 32, 160, 1, 7, 78, 85, 77, 66, 69, 82, 32, 244, 1, 5, 83, 73, 71, + 78, 32, 144, 10, 7, 86, 79, 87, 69, 76, 32, 75, 255, 174, 11, 68, 78, 92, + 9, 76, 65, 78, 32, 83, 73, 71, 78, 32, 137, 3, 9, 79, 78, 83, 79, 78, 65, + 78, 84, 32, 38, 132, 1, 2, 72, 65, 38, 75, 46, 76, 34, 84, 82, 86, 30, + 89, 148, 9, 2, 88, 89, 172, 5, 2, 80, 72, 174, 1, 70, 165, 2, 2, 77, 85, + 4, 190, 197, 2, 87, 187, 236, 10, 77, 6, 246, 15, 72, 234, 130, 13, 79, + 159, 14, 87, 4, 210, 12, 65, 251, 230, 12, 73, 8, 22, 83, 247, 9, 72, 6, + 180, 196, 2, 3, 72, 69, 69, 214, 176, 9, 65, 3, 87, 4, 254, 195, 2, 65, + 3, 87, 4, 226, 195, 2, 65, 211, 166, 10, 69, 40, 122, 67, 38, 72, 46, 78, + 60, 2, 80, 76, 2, 81, 250, 221, 2, 76, 2, 77, 2, 82, 2, 86, 2, 88, 2, 89, + 147, 171, 10, 65, 4, 130, 223, 2, 72, 147, 171, 10, 65, 6, 222, 222, 2, + 76, 2, 78, 147, 171, 10, 65, 12, 58, 67, 22, 84, 230, 221, 2, 75, 2, 76, + 147, 171, 10, 65, 2, 247, 221, 2, 72, 4, 226, 221, 2, 72, 3, 83, 14, 42, + 75, 50, 83, 38, 84, 223, 155, 13, 72, 4, 26, 72, 159, 239, 12, 69, 2, + 251, 156, 6, 65, 4, 230, 241, 11, 85, 255, 186, 1, 79, 4, 218, 240, 11, + 85, 215, 18, 65, 14, 52, 7, 72, 85, 78, 68, 82, 69, 68, 38, 84, 79, 77, + 4, 108, 2, 32, 77, 251, 170, 13, 83, 8, 24, 2, 69, 78, 51, 82, 6, 26, 32, + 143, 171, 13, 83, 4, 18, 66, 35, 84, 2, 205, 208, 4, 3, 73, 76, 76, 2, + 11, 72, 2, 225, 235, 9, 3, 79, 85, 83, 72, 188, 1, 4, 67, 73, 77, 32, + 246, 2, 72, 32, 3, 73, 66, 32, 22, 77, 86, 78, 50, 84, 124, 4, 86, 79, + 83, 32, 198, 1, 88, 208, 1, 6, 90, 87, 74, 32, 84, 72, 238, 176, 1, 76, + 203, 223, 4, 65, 16, 174, 1, 67, 84, 5, 78, 82, 69, 83, 32, 22, 84, 240, + 234, 11, 7, 80, 85, 66, 32, 68, 65, 87, 133, 187, 1, 16, 72, 65, 73, 83, + 32, 76, 85, 83, 32, 78, 84, 79, 71, 32, 78, 84, 4, 48, 7, 85, 65, 77, 32, + 84, 83, 72, 255, 3, 72, 2, 11, 79, 2, 195, 186, 2, 79, 2, 163, 183, 1, + 84, 6, 52, 3, 88, 87, 86, 173, 135, 10, 4, 83, 79, 86, 32, 5, 157, 150, + 6, 4, 32, 67, 72, 87, 4, 158, 72, 78, 131, 202, 12, 76, 2, 219, 234, 11, + 89, 6, 40, 4, 69, 69, 74, 32, 191, 231, 12, 85, 4, 128, 3, 2, 84, 83, 57, + 2, 83, 85, 4, 26, 84, 207, 241, 8, 81, 2, 155, 184, 2, 85, 6, 196, 1, 7, + 88, 72, 69, 69, 74, 32, 67, 236, 167, 8, 12, 72, 73, 82, 68, 45, 83, 84, + 65, 71, 69, 32, 72, 167, 214, 4, 65, 14, 54, 70, 22, 83, 30, 84, 134, 69, + 76, 239, 240, 7, 78, 2, 223, 144, 13, 69, 2, 249, 146, 6, 2, 69, 69, 6, + 42, 72, 29, 6, 83, 72, 65, 66, 32, 67, 4, 82, 73, 135, 145, 13, 79, 2, + 239, 203, 5, 69, 14, 34, 73, 22, 89, 207, 155, 11, 65, 2, 247, 229, 11, + 65, 10, 40, 4, 69, 69, 77, 32, 171, 130, 13, 79, 8, 84, 3, 78, 84, 88, + 200, 144, 6, 2, 84, 79, 218, 167, 2, 82, 181, 172, 3, 2, 70, 65, 2, 199, + 144, 6, 73, 2, 247, 179, 2, 65, 56, 50, 65, 66, 69, 38, 73, 2, 85, 38, + 79, 39, 87, 20, 170, 1, 65, 2, 73, 2, 85, 2, 87, 190, 158, 13, 66, 3, 86, + 8, 106, 69, 190, 158, 13, 66, 3, 86, 8, 70, 65, 190, 158, 13, 66, 3, 86, + 8, 34, 79, 190, 158, 13, 66, 3, 86, 4, 186, 158, 13, 66, 3, 86, 76, 18, + 76, 23, 77, 2, 175, 224, 12, 65, 74, 74, 32, 104, 6, 89, 82, 69, 78, 69, + 32, 201, 136, 4, 4, 83, 32, 85, 80, 8, 80, 3, 66, 82, 65, 150, 173, 11, + 84, 172, 97, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 175, 250, 12, 78, 64, + 80, 2, 76, 69, 40, 4, 82, 73, 71, 72, 253, 2, 7, 78, 85, 77, 66, 69, 82, + 32, 48, 38, 70, 89, 5, 84, 84, 69, 82, 32, 2, 57, 12, 84, 45, 80, 79, 73, + 78, 84, 73, 78, 71, 32, 70, 2, 177, 226, 5, 2, 76, 69, 46, 224, 1, 5, 70, + 73, 78, 65, 76, 30, 84, 246, 118, 68, 34, 76, 50, 81, 238, 205, 1, 82, + 178, 215, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, + 75, 198, 75, 66, 178, 216, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, + 2, 217, 152, 12, 2, 32, 78, 4, 222, 150, 11, 69, 243, 131, 1, 65, 14, + 198, 120, 84, 226, 175, 10, 70, 251, 86, 79, 4, 32, 2, 67, 65, 203, 216, + 12, 68, 2, 131, 132, 12, 75, 184, 2, 94, 65, 220, 1, 11, 69, 78, 84, 72, + 69, 83, 73, 90, 69, 68, 32, 242, 16, 84, 131, 180, 12, 82, 14, 80, 5, 71, + 82, 65, 80, 72, 52, 5, 76, 76, 69, 76, 32, 137, 158, 6, 2, 67, 72, 6, + 198, 232, 9, 32, 194, 222, 1, 85, 207, 145, 1, 79, 6, 44, 5, 87, 73, 84, + 72, 32, 219, 246, 12, 84, 4, 150, 193, 6, 84, 131, 176, 4, 72, 150, 2, + 252, 1, 7, 72, 65, 78, 71, 85, 76, 32, 188, 4, 10, 73, 68, 69, 79, 71, + 82, 65, 80, 72, 32, 188, 7, 18, 75, 79, 82, 69, 65, 78, 32, 67, 72, 65, + 82, 65, 67, 84, 69, 82, 32, 79, 44, 7, 78, 85, 77, 66, 69, 82, 32, 202, + 201, 4, 68, 213, 169, 2, 2, 76, 65, 58, 102, 67, 110, 72, 30, 75, 66, 77, + 34, 78, 34, 80, 62, 82, 30, 83, 26, 84, 73, 5, 73, 69, 85, 78, 71, 10, + 34, 72, 33, 4, 73, 69, 85, 67, 4, 141, 3, 4, 73, 69, 85, 67, 7, 11, 32, + 4, 234, 145, 13, 65, 3, 85, 4, 197, 2, 3, 73, 69, 85, 8, 168, 2, 5, 72, + 73, 69, 85, 75, 13, 5, 73, 89, 69, 79, 75, 4, 245, 1, 4, 73, 69, 85, 77, + 4, 213, 1, 4, 73, 69, 85, 78, 8, 168, 1, 5, 72, 73, 69, 85, 80, 13, 4, + 73, 69, 85, 80, 4, 121, 4, 73, 69, 85, 76, 4, 93, 3, 73, 79, 83, 8, 56, + 5, 72, 73, 69, 85, 84, 13, 5, 73, 75, 69, 85, 84, 4, 11, 72, 5, 195, 140, + 13, 32, 72, 148, 1, 2, 65, 76, 30, 67, 74, 69, 82, 70, 112, 2, 76, 65, + 22, 77, 38, 78, 32, 2, 82, 69, 78, 83, 138, 2, 84, 50, 87, 158, 156, 11, + 72, 139, 82, 79, 2, 241, 133, 8, 2, 76, 73, 4, 32, 2, 79, 78, 211, 133, + 11, 65, 2, 185, 240, 7, 4, 71, 82, 65, 84, 6, 42, 78, 222, 163, 9, 65, + 163, 190, 3, 73, 2, 169, 4, 5, 84, 69, 82, 80, 82, 10, 58, 73, 160, 130, + 12, 5, 69, 83, 84, 73, 86, 239, 16, 79, 6, 208, 2, 3, 78, 65, 78, 186, + 242, 12, 82, 3, 86, 2, 195, 210, 12, 66, 4, 234, 182, 10, 69, 195, 132, + 2, 79, 4, 198, 241, 11, 73, 195, 1, 65, 8, 38, 83, 174, 250, 11, 80, 219, + 109, 65, 4, 134, 209, 6, 79, 167, 185, 6, 84, 18, 58, 69, 34, 79, 22, 80, + 34, 84, 50, 85, 167, 252, 11, 73, 4, 226, 181, 11, 86, 199, 82, 76, 2, + 243, 240, 7, 67, 2, 11, 69, 2, 179, 176, 4, 67, 4, 26, 85, 239, 216, 11, + 79, 2, 147, 247, 12, 68, 4, 26, 80, 175, 136, 13, 78, 2, 21, 3, 69, 82, + 86, 2, 135, 255, 11, 73, 6, 186, 152, 11, 72, 230, 159, 1, 69, 239, 48, + 87, 4, 246, 208, 9, 79, 227, 254, 1, 65, 4, 182, 153, 8, 32, 137, 158, 4, + 2, 74, 69, 22, 42, 69, 46, 70, 42, 78, 30, 83, 51, 84, 4, 216, 1, 3, 73, + 71, 72, 211, 240, 4, 76, 4, 160, 1, 2, 79, 85, 13, 2, 73, 70, 2, 133, 1, + 3, 73, 78, 69, 4, 34, 73, 73, 4, 69, 86, 69, 78, 2, 71, 88, 8, 34, 72, + 46, 87, 135, 181, 12, 69, 2, 11, 73, 2, 11, 82, 2, 131, 177, 11, 84, 4, + 11, 69, 4, 222, 151, 11, 78, 167, 112, 76, 18, 128, 1, 3, 73, 65, 76, + 216, 1, 3, 89, 32, 80, 168, 169, 8, 6, 78, 69, 82, 83, 72, 73, 169, 211, + 3, 6, 32, 65, 76, 84, 69, 82, 12, 62, 32, 193, 123, 10, 76, 89, 45, 82, + 69, 67, 89, 67, 76, 69, 10, 38, 68, 41, 5, 76, 73, 78, 69, 32, 2, 217, + 169, 4, 5, 73, 70, 70, 69, 82, 8, 222, 200, 3, 68, 226, 45, 70, 20, 4, + 66, 65, 67, 75, 199, 139, 9, 85, 2, 191, 237, 3, 79, 10, 98, 69, 52, 9, + 73, 86, 69, 45, 80, 85, 76, 76, 45, 89, 9, 80, 79, 82, 84, 32, 67, 79, + 78, 84, 4, 224, 226, 9, 4, 78, 71, 69, 82, 219, 138, 2, 68, 4, 40, 4, 68, + 79, 87, 78, 1, 2, 85, 80, 2, 185, 171, 5, 6, 45, 79, 85, 84, 80, 85, 2, + 231, 236, 11, 82, 114, 192, 1, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, + 84, 79, 80, 62, 76, 160, 3, 3, 82, 73, 83, 36, 14, 77, 73, 68, 45, 76, + 69, 86, 69, 76, 32, 84, 79, 78, 69, 57, 7, 83, 65, 78, 68, 72, 73, 32, 7, + 11, 32, 4, 206, 5, 70, 201, 238, 11, 4, 86, 65, 82, 73, 82, 72, 6, 69, + 84, 84, 69, 82, 32, 213, 2, 7, 79, 87, 45, 70, 65, 76, 76, 74, 206, 1, + 70, 166, 143, 7, 73, 2, 85, 214, 250, 1, 78, 242, 169, 3, 67, 2, 75, 2, + 80, 2, 84, 138, 69, 66, 2, 68, 2, 71, 2, 72, 2, 76, 2, 77, 2, 82, 2, 83, + 2, 86, 2, 90, 186, 2, 65, 2, 69, 3, 79, 20, 44, 5, 73, 78, 65, 76, 32, + 251, 250, 12, 65, 18, 222, 255, 10, 78, 154, 251, 1, 75, 2, 76, 2, 77, 2, + 80, 2, 84, 2, 87, 3, 89, 8, 157, 1, 5, 73, 78, 71, 32, 84, 7, 11, 32, 4, + 192, 1, 5, 76, 79, 78, 71, 32, 15, 70, 12, 66, 84, 73, 12, 71, 76, 79, + 84, 84, 65, 76, 32, 83, 84, 79, 80, 8, 21, 3, 79, 78, 69, 9, 11, 32, 6, + 32, 4, 76, 79, 78, 71, 27, 70, 5, 11, 32, 2, 11, 70, 2, 223, 195, 3, 73, + 2, 155, 176, 4, 82, 4, 250, 247, 12, 70, 3, 73, 80, 130, 1, 65, 122, 78, + 162, 1, 82, 162, 9, 83, 44, 3, 84, 82, 73, 130, 17, 68, 245, 136, 12, 9, + 79, 80, 76, 69, 32, 72, 85, 71, 71, 12, 50, 67, 50, 78, 190, 234, 6, 32, + 191, 139, 6, 82, 6, 182, 198, 11, 79, 202, 28, 69, 171, 147, 1, 72, 2, + 203, 225, 11, 85, 10, 118, 71, 20, 3, 83, 73, 86, 222, 178, 1, 84, 132, + 184, 4, 10, 32, 79, 86, 69, 82, 32, 83, 84, 65, 77, 167, 214, 4, 67, 2, + 151, 245, 11, 85, 2, 183, 183, 12, 69, 48, 186, 1, 32, 116, 10, 80, 69, + 78, 68, 73, 67, 85, 76, 65, 82, 42, 83, 210, 165, 7, 67, 252, 9, 10, 77, + 65, 78, 69, 78, 84, 32, 80, 65, 80, 201, 128, 2, 8, 70, 79, 82, 77, 73, + 78, 71, 32, 6, 34, 77, 30, 84, 227, 235, 11, 83, 2, 181, 241, 1, 2, 73, + 76, 2, 213, 137, 11, 8, 69, 78, 32, 84, 72, 79, 85, 83, 5, 129, 253, 8, + 5, 32, 87, 73, 84, 72, 32, 60, 2, 79, 78, 178, 75, 80, 221, 157, 11, 4, + 69, 86, 69, 82, 28, 38, 32, 149, 246, 9, 3, 65, 76, 32, 26, 216, 2, 10, + 68, 79, 73, 78, 71, 32, 67, 65, 82, 84, 32, 3, 73, 78, 32, 92, 5, 87, 73, + 84, 72, 32, 220, 213, 7, 4, 70, 82, 79, 87, 204, 13, 27, 82, 65, 73, 83, 73, 78, 71, 32, 66, 79, 84, 72, 32, 72, 65, 78, 68, 83, 32, 73, 78, 32, - 67, 69, 76, 69, 66, 128, 141, 4, 5, 67, 76, 73, 77, 66, 201, 45, 11, 66, - 79, 87, 73, 78, 71, 32, 68, 69, 69, 80, 2, 11, 87, 2, 179, 169, 3, 72, 4, - 76, 7, 83, 84, 69, 65, 77, 89, 32, 217, 211, 11, 6, 76, 79, 84, 85, 83, - 32, 2, 197, 148, 12, 2, 82, 79, 12, 154, 1, 66, 212, 134, 2, 3, 80, 79, - 85, 160, 157, 1, 2, 67, 82, 196, 214, 5, 6, 70, 79, 76, 68, 69, 68, 177, - 168, 2, 8, 72, 69, 65, 68, 83, 67, 65, 82, 4, 36, 3, 76, 79, 78, 159, - 159, 10, 65, 2, 11, 68, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 203, 234, 11, - 73, 2, 21, 3, 76, 32, 67, 2, 161, 236, 10, 4, 79, 77, 80, 85, 4, 232, - 230, 8, 2, 69, 84, 251, 180, 2, 79, 2, 217, 227, 8, 2, 32, 68, 140, 2, - 66, 65, 180, 19, 9, 73, 76, 73, 80, 80, 73, 78, 69, 32, 47, 79, 204, 1, + 67, 69, 76, 69, 66, 128, 185, 4, 5, 67, 76, 73, 77, 66, 213, 45, 11, 66, + 79, 87, 73, 78, 71, 32, 68, 69, 69, 80, 2, 11, 87, 2, 251, 183, 3, 72, 4, + 164, 156, 12, 6, 76, 79, 84, 85, 83, 32, 253, 64, 9, 83, 84, 69, 65, 77, + 89, 32, 82, 79, 12, 154, 1, 66, 240, 145, 2, 3, 80, 79, 85, 220, 160, 1, + 2, 67, 82, 196, 250, 5, 6, 70, 79, 76, 68, 69, 68, 221, 189, 2, 8, 72, + 69, 65, 68, 83, 67, 65, 82, 4, 36, 3, 76, 79, 78, 171, 228, 10, 65, 2, + 11, 68, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 219, 178, 12, 73, 4, 224, + 153, 9, 2, 69, 84, 195, 202, 2, 79, 2, 253, 137, 9, 2, 32, 68, 140, 2, + 66, 65, 184, 19, 9, 73, 76, 73, 80, 80, 73, 78, 69, 32, 47, 79, 204, 1, 108, 6, 71, 83, 45, 80, 65, 32, 189, 7, 16, 73, 83, 84, 79, 83, 32, 68, 73, 83, 67, 32, 83, 73, 71, 78, 32, 112, 100, 7, 76, 69, 84, 84, 69, 82, 32, 248, 4, 5, 77, 65, 82, 75, 32, 30, 83, 33, 4, 68, 79, 85, 66, 96, - 138, 2, 65, 138, 1, 67, 50, 68, 42, 83, 64, 5, 86, 79, 73, 67, 69, 174, - 204, 8, 71, 218, 134, 3, 78, 82, 84, 46, 75, 2, 80, 2, 90, 162, 7, 69, - 222, 61, 66, 2, 70, 2, 72, 2, 74, 2, 76, 2, 77, 2, 81, 2, 82, 2, 87, 2, + 138, 2, 65, 138, 1, 67, 50, 68, 42, 83, 64, 5, 86, 79, 73, 67, 69, 222, + 242, 8, 71, 246, 168, 3, 78, 82, 84, 46, 75, 2, 80, 2, 90, 162, 7, 69, + 234, 61, 66, 2, 70, 2, 72, 2, 74, 2, 76, 2, 77, 2, 81, 2, 82, 2, 87, 2, 88, 2, 89, 186, 2, 73, 2, 79, 3, 85, 7, 80, 8, 76, 84, 69, 82, 78, 65, - 84, 69, 21, 8, 83, 80, 73, 82, 65, 84, 69, 68, 2, 163, 154, 12, 32, 2, - 11, 32, 2, 167, 154, 12, 70, 6, 26, 65, 251, 153, 12, 72, 5, 159, 182, 8, - 78, 6, 226, 153, 12, 68, 2, 90, 187, 2, 65, 6, 236, 135, 8, 4, 77, 65, - 76, 76, 206, 145, 4, 72, 187, 2, 65, 4, 34, 68, 21, 4, 76, 69, 83, 83, 2, - 211, 163, 10, 32, 2, 243, 247, 8, 32, 4, 130, 212, 11, 68, 59, 83, 10, - 28, 3, 73, 78, 71, 31, 85, 2, 241, 205, 11, 2, 76, 69, 8, 58, 66, 225, - 206, 11, 8, 80, 69, 82, 70, 73, 88, 69, 68, 6, 229, 150, 8, 13, 74, 79, + 84, 69, 21, 8, 83, 80, 73, 82, 65, 84, 69, 68, 2, 251, 226, 12, 32, 2, + 11, 32, 2, 255, 226, 12, 70, 6, 26, 65, 211, 226, 12, 72, 5, 223, 208, 8, + 78, 6, 186, 226, 12, 68, 2, 90, 187, 2, 65, 6, 168, 164, 8, 4, 77, 65, + 76, 76, 234, 189, 4, 72, 187, 2, 65, 4, 34, 68, 21, 4, 76, 69, 83, 83, 2, + 167, 233, 10, 32, 2, 215, 171, 9, 32, 4, 206, 156, 12, 68, 59, 83, 10, + 28, 3, 73, 78, 71, 31, 85, 2, 189, 150, 12, 2, 76, 69, 8, 58, 66, 173, + 151, 12, 8, 80, 69, 82, 70, 73, 88, 69, 68, 6, 133, 179, 8, 13, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 92, 238, 1, 66, 146, 1, 67, - 172, 2, 2, 68, 79, 38, 71, 66, 72, 64, 2, 76, 73, 32, 2, 77, 65, 66, 80, - 162, 1, 82, 38, 83, 150, 1, 84, 70, 87, 168, 203, 5, 2, 70, 76, 172, 224, - 1, 2, 79, 88, 188, 193, 3, 2, 69, 65, 142, 7, 65, 135, 1, 86, 10, 52, 2, - 69, 69, 22, 79, 205, 40, 4, 85, 76, 76, 83, 5, 199, 142, 7, 72, 4, 32, 2, - 79, 77, 175, 150, 12, 87, 2, 11, 69, 2, 219, 189, 11, 82, 16, 34, 65, 86, - 72, 22, 76, 23, 79, 6, 194, 200, 5, 80, 200, 192, 3, 8, 82, 80, 69, 78, - 84, 82, 89, 32, 223, 140, 3, 84, 2, 255, 175, 2, 73, 2, 219, 218, 10, 85, - 6, 26, 76, 33, 2, 77, 66, 2, 11, 85, 2, 239, 196, 11, 77, 5, 37, 7, 73, - 78, 73, 78, 71, 32, 79, 2, 237, 172, 9, 5, 66, 76, 73, 81, 85, 4, 222, - 235, 10, 76, 175, 145, 1, 86, 4, 42, 82, 245, 182, 10, 4, 65, 85, 78, 84, - 2, 195, 220, 10, 65, 6, 42, 69, 198, 180, 7, 79, 155, 209, 2, 73, 2, 183, - 221, 11, 76, 4, 242, 128, 12, 76, 203, 17, 68, 4, 40, 2, 78, 65, 181, - 169, 9, 2, 84, 84, 2, 159, 233, 8, 67, 8, 52, 2, 69, 68, 50, 76, 237, - 136, 7, 3, 65, 80, 89, 2, 25, 4, 69, 83, 84, 82, 2, 211, 201, 10, 73, 4, - 184, 167, 4, 2, 85, 77, 229, 211, 2, 3, 65, 78, 69, 4, 230, 132, 10, 79, - 255, 250, 1, 65, 12, 108, 2, 72, 73, 154, 145, 11, 65, 154, 45, 76, 128, - 19, 3, 84, 82, 65, 165, 39, 7, 77, 65, 76, 76, 32, 65, 88, 4, 146, 170, - 2, 69, 151, 229, 9, 80, 6, 200, 165, 4, 5, 65, 84, 84, 79, 79, 250, 164, - 7, 73, 195, 36, 85, 4, 246, 196, 7, 79, 221, 187, 3, 5, 65, 86, 89, 32, - 66, 4, 218, 232, 1, 83, 25, 4, 68, 79, 85, 66, 60, 56, 8, 69, 78, 73, 67, - 73, 65, 78, 32, 183, 138, 10, 76, 58, 92, 7, 76, 69, 84, 84, 69, 82, 32, - 160, 3, 7, 78, 85, 77, 66, 69, 82, 32, 199, 248, 5, 87, 44, 234, 1, 65, - 34, 68, 22, 72, 22, 81, 22, 83, 58, 84, 202, 246, 1, 87, 150, 212, 5, 90, - 130, 157, 1, 89, 200, 188, 1, 2, 82, 79, 128, 92, 3, 71, 65, 77, 134, 8, - 75, 130, 1, 78, 132, 58, 3, 76, 65, 77, 138, 17, 66, 186, 30, 80, 171, 4, - 77, 4, 198, 137, 11, 76, 187, 49, 73, 2, 151, 172, 3, 69, 4, 171, 213, 6, - 69, 2, 255, 136, 11, 79, 6, 130, 253, 9, 65, 186, 141, 1, 72, 165, 124, - 2, 69, 77, 4, 214, 228, 11, 65, 191, 8, 69, 12, 186, 49, 84, 187, 151, 4, + 172, 2, 2, 68, 79, 38, 71, 66, 72, 64, 2, 76, 73, 32, 2, 77, 65, 70, 80, + 162, 1, 82, 38, 83, 150, 1, 84, 70, 87, 160, 223, 5, 2, 70, 76, 252, 232, + 1, 2, 79, 88, 196, 234, 3, 2, 69, 65, 250, 9, 65, 135, 1, 86, 10, 52, 2, + 69, 69, 22, 79, 133, 41, 4, 85, 76, 76, 83, 5, 247, 171, 7, 72, 4, 32, 2, + 79, 77, 135, 223, 12, 87, 2, 11, 69, 2, 163, 134, 12, 82, 16, 34, 65, 86, + 72, 22, 76, 23, 79, 6, 234, 218, 5, 80, 148, 230, 3, 8, 82, 80, 69, 78, + 84, 82, 89, 32, 195, 157, 3, 84, 2, 227, 191, 2, 73, 2, 243, 161, 11, 85, + 6, 26, 76, 33, 2, 77, 66, 2, 11, 85, 2, 187, 141, 12, 77, 5, 37, 7, 73, + 78, 73, 78, 71, 32, 79, 2, 209, 239, 9, 5, 66, 76, 73, 81, 85, 4, 154, + 179, 11, 76, 203, 146, 1, 86, 4, 42, 82, 201, 252, 10, 4, 65, 85, 78, 84, + 2, 239, 163, 11, 65, 6, 42, 69, 146, 209, 7, 79, 143, 250, 2, 73, 2, 131, + 166, 12, 76, 4, 202, 201, 12, 76, 203, 17, 68, 4, 34, 78, 161, 236, 9, 2, + 84, 84, 2, 11, 65, 2, 255, 156, 9, 67, 8, 52, 2, 69, 68, 50, 76, 153, + 166, 7, 3, 65, 80, 89, 2, 25, 4, 69, 83, 84, 82, 2, 219, 143, 11, 73, 4, + 176, 182, 4, 2, 85, 77, 177, 226, 2, 3, 65, 78, 69, 4, 166, 202, 10, 79, + 147, 254, 1, 65, 12, 108, 2, 72, 73, 222, 217, 11, 65, 158, 45, 76, 128, + 19, 3, 84, 82, 65, 177, 39, 7, 77, 65, 76, 76, 32, 65, 88, 4, 242, 185, + 2, 69, 139, 158, 10, 80, 6, 192, 180, 4, 5, 65, 84, 84, 79, 79, 202, 222, + 7, 73, 207, 36, 85, 4, 190, 225, 7, 79, 209, 231, 3, 5, 65, 86, 89, 32, + 66, 4, 174, 244, 1, 83, 25, 4, 68, 79, 85, 66, 60, 56, 8, 69, 78, 73, 67, + 73, 65, 78, 32, 251, 207, 10, 76, 58, 92, 7, 76, 69, 84, 84, 69, 82, 32, + 160, 3, 7, 78, 85, 77, 66, 69, 82, 32, 203, 150, 6, 87, 44, 234, 1, 65, + 34, 68, 22, 72, 22, 81, 22, 83, 58, 84, 158, 130, 2, 87, 138, 229, 5, 90, + 154, 180, 1, 89, 184, 206, 1, 2, 82, 79, 236, 94, 3, 71, 65, 77, 134, 8, + 75, 130, 1, 78, 144, 58, 3, 76, 65, 77, 138, 17, 66, 198, 30, 80, 171, 4, + 77, 4, 130, 210, 11, 76, 199, 49, 73, 2, 163, 187, 3, 69, 4, 147, 243, 6, + 69, 2, 187, 209, 11, 79, 6, 190, 194, 10, 65, 186, 144, 1, 72, 189, 124, + 2, 69, 77, 4, 170, 173, 12, 65, 191, 8, 69, 12, 238, 49, 84, 135, 166, 4, 79, 40, 104, 2, 67, 75, 66, 71, 62, 76, 90, 78, 178, 1, 83, 32, 7, 84, - 67, 72, 70, 79, 82, 75, 243, 132, 12, 69, 5, 17, 2, 85, 80, 2, 21, 3, 32, - 84, 82, 2, 155, 217, 10, 85, 7, 11, 32, 4, 26, 78, 179, 202, 11, 70, 2, - 183, 254, 10, 79, 6, 52, 4, 69, 32, 79, 70, 238, 90, 67, 247, 171, 11, - 76, 2, 11, 32, 2, 139, 198, 9, 80, 14, 68, 2, 67, 72, 46, 69, 186, 149, - 4, 87, 242, 232, 6, 75, 219, 98, 65, 4, 144, 140, 3, 2, 69, 68, 211, 235, - 7, 73, 4, 28, 2, 32, 68, 195, 71, 65, 2, 185, 241, 4, 2, 69, 67, 4, 174, - 241, 10, 67, 139, 1, 84, 5, 165, 186, 9, 10, 32, 87, 73, 84, 72, 32, 84, - 69, 69, 32, 218, 1, 38, 65, 218, 9, 85, 179, 250, 11, 68, 176, 1, 78, 67, + 67, 72, 70, 79, 82, 75, 199, 205, 12, 69, 5, 17, 2, 85, 80, 2, 21, 3, 32, + 84, 82, 2, 203, 160, 11, 85, 7, 11, 32, 4, 26, 78, 251, 146, 12, 70, 2, + 243, 198, 11, 79, 6, 52, 4, 69, 32, 79, 70, 246, 91, 67, 195, 243, 11, + 76, 2, 11, 32, 2, 255, 135, 10, 80, 14, 68, 2, 67, 72, 46, 69, 178, 164, + 4, 87, 182, 162, 7, 75, 243, 98, 65, 4, 160, 155, 3, 2, 69, 68, 255, 164, + 8, 73, 4, 28, 2, 32, 68, 255, 72, 65, 2, 221, 129, 5, 2, 69, 67, 4, 234, + 185, 11, 67, 139, 1, 84, 5, 153, 252, 9, 10, 32, 87, 73, 84, 72, 32, 84, + 69, 69, 32, 218, 1, 38, 65, 218, 9, 85, 135, 195, 12, 68, 176, 1, 78, 67, 128, 1, 12, 78, 67, 75, 32, 67, 79, 78, 83, 84, 65, 78, 84, 83, 89, 6, - 44, 5, 69, 32, 79, 70, 32, 179, 236, 10, 65, 4, 56, 6, 73, 78, 84, 69, - 82, 69, 189, 173, 11, 2, 87, 79, 2, 231, 235, 6, 83, 5, 45, 9, 32, 79, - 86, 69, 82, 32, 84, 87, 79, 2, 11, 32, 2, 159, 238, 11, 80, 166, 1, 96, - 9, 73, 78, 71, 32, 67, 65, 82, 68, 32, 149, 151, 4, 9, 71, 82, 79, 85, + 44, 5, 69, 32, 79, 70, 32, 239, 180, 11, 65, 4, 56, 6, 73, 78, 84, 69, + 82, 69, 129, 246, 11, 2, 87, 79, 2, 199, 136, 7, 83, 5, 45, 9, 32, 79, + 86, 69, 82, 32, 84, 87, 79, 2, 11, 32, 2, 243, 182, 12, 80, 166, 1, 96, + 9, 73, 78, 71, 32, 67, 65, 82, 68, 32, 141, 166, 4, 9, 71, 82, 79, 85, 78, 68, 32, 83, 76, 164, 1, 182, 1, 66, 44, 3, 82, 69, 68, 0, 5, 87, 72, 73, 84, 69, 42, 70, 74, 75, 38, 69, 34, 83, 36, 3, 81, 85, 69, 14, 84, 92, 2, 65, 67, 0, 3, 78, 73, 78, 13, 4, 74, 65, 67, 75, 4, 40, 4, 76, 65, - 67, 75, 203, 208, 10, 65, 2, 17, 2, 32, 74, 2, 203, 225, 1, 79, 18, 30, - 79, 249, 1, 2, 73, 86, 10, 128, 2, 2, 85, 82, 175, 243, 10, 79, 16, 34, + 67, 75, 251, 151, 11, 65, 2, 17, 2, 32, 74, 2, 159, 237, 1, 79, 18, 30, + 79, 249, 1, 2, 73, 86, 10, 128, 2, 2, 85, 82, 235, 187, 11, 79, 16, 34, 78, 185, 1, 3, 73, 78, 71, 8, 181, 1, 4, 73, 71, 72, 84, 16, 32, 2, 69, 86, 117, 2, 73, 88, 8, 91, 69, 66, 78, 69, 12, 3, 72, 82, 69, 12, 2, 87, 79, 161, 1, 5, 82, 85, 77, 80, 45, 8, 23, 78, 8, 11, 69, 8, 25, 4, 32, - 79, 70, 32, 8, 88, 3, 67, 76, 85, 20, 3, 83, 80, 65, 158, 207, 8, 72, - 137, 3, 5, 68, 73, 65, 77, 79, 2, 255, 189, 11, 66, 2, 219, 231, 10, 68, - 42, 90, 50, 250, 192, 9, 49, 214, 185, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, - 2, 56, 3, 57, 7, 202, 250, 11, 48, 3, 49, 41, 46, 83, 160, 4, 2, 84, 79, - 239, 189, 8, 78, 26, 52, 5, 32, 83, 73, 71, 78, 221, 223, 8, 2, 45, 77, - 25, 11, 32, 22, 64, 3, 73, 78, 32, 124, 5, 87, 73, 84, 72, 32, 219, 235, - 3, 65, 6, 34, 76, 22, 82, 247, 212, 10, 84, 2, 41, 2, 69, 70, 2, 21, 3, - 73, 71, 72, 2, 133, 169, 10, 6, 84, 32, 72, 65, 76, 70, 14, 162, 1, 68, - 34, 83, 204, 221, 9, 4, 84, 73, 76, 68, 140, 117, 5, 66, 76, 65, 67, 75, - 185, 35, 16, 67, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, 67, 67, 69, - 78, 2, 11, 79, 2, 231, 207, 9, 84, 4, 54, 77, 165, 157, 5, 7, 85, 66, 83, - 67, 82, 73, 80, 2, 197, 231, 8, 3, 65, 76, 76, 11, 33, 6, 32, 70, 79, 82, - 77, 32, 8, 154, 136, 10, 70, 71, 84, 58, 232, 1, 5, 76, 73, 67, 69, 32, - 122, 80, 136, 1, 11, 82, 84, 65, 66, 76, 69, 32, 83, 84, 69, 82, 22, 83, - 158, 1, 84, 146, 1, 85, 196, 1, 4, 87, 69, 82, 32, 202, 231, 6, 79, 245, - 204, 4, 11, 67, 75, 69, 84, 32, 67, 65, 76, 67, 85, 76, 6, 52, 3, 67, 65, - 82, 141, 234, 3, 4, 79, 70, 70, 73, 5, 173, 221, 9, 11, 83, 32, 82, 69, - 86, 79, 76, 86, 73, 78, 71, 6, 76, 13, 32, 68, 73, 82, 69, 67, 84, 73, - 79, 78, 65, 76, 32, 187, 203, 1, 67, 4, 166, 117, 73, 185, 162, 6, 6, 70, - 79, 82, 77, 65, 84, 2, 191, 210, 11, 69, 12, 40, 2, 69, 73, 22, 84, 195, - 246, 4, 73, 2, 147, 161, 11, 68, 8, 36, 3, 65, 76, 32, 163, 228, 10, 66, - 6, 130, 202, 1, 72, 217, 180, 6, 4, 77, 65, 82, 75, 8, 66, 65, 210, 247, - 1, 32, 173, 238, 8, 6, 84, 69, 68, 32, 80, 76, 4, 26, 66, 179, 208, 11, - 84, 2, 29, 5, 76, 69, 32, 87, 65, 2, 135, 47, 84, 12, 108, 4, 76, 84, 82, - 89, 28, 7, 82, 73, 78, 71, 32, 76, 73, 28, 2, 84, 73, 250, 135, 10, 78, - 199, 228, 1, 67, 2, 165, 168, 11, 2, 32, 76, 2, 209, 156, 5, 2, 81, 85, - 4, 197, 137, 10, 2, 78, 71, 8, 24, 2, 79, 78, 47, 83, 4, 128, 218, 10, 4, - 45, 79, 70, 70, 15, 32, 4, 160, 213, 8, 3, 76, 69, 69, 219, 132, 2, 89, - 140, 1, 74, 69, 162, 10, 73, 234, 2, 79, 237, 139, 9, 6, 65, 89, 69, 82, - 32, 66, 102, 132, 1, 6, 71, 78, 65, 78, 84, 32, 66, 83, 156, 167, 5, 4, - 67, 69, 68, 69, 144, 193, 1, 5, 86, 73, 79, 85, 83, 157, 33, 2, 84, 90, - 6, 42, 87, 146, 151, 5, 80, 243, 139, 5, 77, 2, 235, 160, 7, 79, 70, 176, + 79, 70, 32, 8, 88, 3, 67, 76, 85, 20, 3, 83, 80, 65, 174, 130, 9, 72, + 137, 3, 5, 68, 73, 65, 77, 79, 2, 199, 134, 12, 66, 2, 151, 176, 11, 68, + 42, 90, 50, 238, 130, 10, 49, 182, 192, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 2, 56, 3, 57, 7, 158, 195, 12, 48, 3, 49, 41, 46, 83, 160, 4, 2, 84, + 79, 227, 240, 8, 78, 26, 52, 5, 32, 83, 73, 71, 78, 193, 147, 9, 2, 45, + 77, 25, 11, 32, 22, 64, 3, 73, 78, 32, 124, 5, 87, 73, 84, 72, 32, 207, + 250, 3, 65, 6, 34, 76, 22, 82, 183, 156, 11, 84, 2, 41, 2, 69, 70, 2, 21, + 3, 73, 71, 72, 2, 229, 238, 10, 6, 84, 32, 72, 65, 76, 70, 14, 162, 1, + 68, 34, 83, 200, 160, 10, 4, 84, 73, 76, 68, 208, 121, 5, 66, 76, 65, 67, + 75, 181, 36, 16, 67, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, 67, 67, + 69, 78, 2, 11, 79, 2, 219, 145, 10, 84, 4, 54, 77, 161, 176, 5, 7, 85, + 66, 83, 67, 82, 73, 80, 2, 249, 154, 9, 3, 65, 76, 76, 11, 33, 6, 32, 70, + 79, 82, 77, 32, 8, 234, 205, 10, 70, 71, 84, 2, 253, 132, 12, 8, 32, 84, + 82, 65, 78, 83, 73, 83, 58, 232, 1, 5, 76, 73, 67, 69, 32, 122, 80, 136, + 1, 11, 82, 84, 65, 66, 76, 69, 32, 83, 84, 69, 82, 22, 83, 158, 1, 84, + 146, 1, 85, 196, 1, 4, 87, 69, 82, 32, 194, 132, 7, 79, 145, 248, 4, 11, + 67, 75, 69, 84, 32, 67, 65, 76, 67, 85, 76, 6, 52, 3, 67, 65, 82, 209, + 248, 3, 4, 79, 70, 70, 73, 5, 149, 162, 10, 11, 83, 32, 82, 69, 86, 79, + 76, 86, 73, 78, 71, 6, 76, 13, 32, 68, 73, 82, 69, 67, 84, 73, 79, 78, + 65, 76, 32, 219, 214, 1, 67, 4, 166, 127, 73, 205, 180, 6, 6, 70, 79, 82, + 77, 65, 84, 2, 223, 154, 12, 69, 12, 40, 2, 69, 73, 22, 84, 235, 135, 5, + 73, 2, 167, 233, 11, 68, 8, 36, 3, 65, 76, 32, 171, 172, 11, 66, 6, 162, + 213, 1, 72, 157, 197, 6, 4, 77, 65, 82, 75, 8, 66, 65, 250, 134, 2, 32, + 141, 167, 9, 6, 84, 69, 68, 32, 80, 76, 4, 26, 66, 211, 152, 12, 84, 2, + 29, 5, 76, 69, 32, 87, 65, 2, 143, 48, 84, 12, 108, 4, 76, 84, 82, 89, + 28, 7, 82, 73, 78, 71, 32, 76, 73, 28, 2, 84, 73, 150, 205, 10, 78, 203, + 231, 1, 67, 2, 185, 240, 11, 2, 32, 76, 2, 193, 174, 5, 2, 81, 85, 4, + 225, 206, 10, 2, 78, 71, 8, 24, 2, 79, 78, 47, 83, 4, 136, 162, 11, 4, + 45, 79, 70, 70, 15, 32, 4, 212, 136, 9, 3, 76, 69, 69, 175, 153, 2, 89, + 140, 1, 74, 69, 162, 10, 73, 234, 2, 79, 157, 206, 9, 6, 65, 89, 69, 82, + 32, 66, 102, 132, 1, 6, 71, 78, 65, 78, 84, 32, 66, 83, 224, 186, 5, 4, + 67, 69, 68, 69, 204, 202, 1, 5, 86, 73, 79, 85, 83, 177, 32, 2, 84, 90, + 6, 42, 87, 130, 179, 8, 80, 215, 181, 2, 77, 2, 255, 188, 7, 79, 70, 176, 1, 27, 69, 78, 84, 65, 84, 73, 79, 78, 32, 70, 79, 82, 77, 32, 70, 79, - 82, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 241, 162, 8, 10, 67, 82, 73, + 82, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 221, 200, 8, 10, 67, 82, 73, 80, 84, 73, 79, 78, 32, 84, 68, 198, 1, 67, 22, 69, 46, 72, 50, 73, 94, 76, 188, 1, 6, 82, 73, 71, 72, 84, 32, 192, 2, 6, 87, 65, 86, 89, 32, 76, - 202, 136, 4, 83, 192, 171, 4, 9, 84, 87, 79, 32, 68, 79, 84, 32, 76, 163, - 99, 81, 4, 203, 170, 2, 79, 6, 138, 252, 5, 88, 214, 194, 2, 77, 3, 78, - 2, 233, 215, 8, 7, 79, 82, 73, 90, 79, 78, 84, 4, 49, 10, 68, 69, 79, 71, - 82, 65, 80, 72, 73, 67, 4, 11, 32, 4, 206, 154, 9, 67, 35, 70, 22, 40, 4, - 69, 70, 84, 32, 191, 128, 10, 79, 20, 112, 6, 87, 72, 73, 84, 69, 32, - 142, 1, 66, 42, 68, 218, 93, 67, 146, 6, 65, 218, 164, 7, 80, 154, 2, 83, - 39, 84, 4, 162, 2, 67, 159, 93, 76, 22, 110, 66, 42, 68, 36, 6, 87, 72, - 73, 84, 69, 32, 182, 93, 67, 146, 6, 65, 218, 164, 7, 80, 154, 2, 83, 39, - 84, 2, 177, 94, 6, 76, 65, 67, 75, 32, 76, 2, 209, 99, 5, 79, 85, 66, 76, - 69, 6, 74, 67, 17, 14, 76, 69, 78, 84, 73, 67, 85, 76, 65, 82, 32, 66, - 82, 65, 2, 131, 93, 79, 4, 198, 143, 11, 67, 177, 29, 2, 75, 67, 2, 235, - 252, 9, 79, 22, 46, 78, 156, 1, 2, 86, 65, 171, 201, 11, 77, 10, 34, 84, - 173, 171, 6, 2, 67, 69, 6, 26, 32, 53, 2, 69, 82, 2, 21, 3, 83, 67, 82, - 2, 253, 235, 9, 2, 69, 69, 5, 17, 2, 32, 73, 2, 175, 144, 11, 67, 10, 60, - 5, 67, 89, 32, 77, 69, 29, 6, 84, 69, 32, 85, 83, 69, 2, 241, 132, 7, 2, - 83, 83, 8, 26, 32, 191, 175, 6, 45, 4, 146, 210, 5, 84, 223, 243, 4, 79, - 14, 130, 1, 74, 30, 80, 228, 32, 5, 72, 73, 66, 73, 84, 148, 176, 8, 6, - 66, 73, 78, 71, 32, 67, 133, 210, 1, 5, 83, 69, 82, 80, 73, 2, 181, 144, - 5, 2, 69, 67, 6, 64, 6, 79, 82, 84, 73, 79, 78, 197, 195, 10, 4, 69, 82, - 84, 89, 5, 179, 182, 5, 65, 58, 172, 1, 15, 70, 79, 85, 82, 32, 68, 79, - 84, 83, 32, 87, 73, 84, 72, 32, 32, 7, 76, 69, 84, 84, 69, 82, 32, 230, - 2, 78, 178, 31, 83, 1, 8, 84, 85, 82, 78, 69, 68, 32, 83, 4, 210, 157, - 10, 67, 187, 109, 68, 36, 166, 1, 65, 22, 68, 34, 76, 22, 77, 50, 87, - 238, 150, 4, 71, 90, 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, - 130, 76, 66, 190, 173, 4, 78, 254, 1, 84, 203, 103, 80, 2, 207, 151, 4, - 76, 2, 11, 65, 2, 251, 171, 6, 76, 2, 235, 151, 4, 65, 2, 25, 4, 69, 77, - 45, 81, 2, 195, 223, 5, 79, 2, 37, 7, 65, 87, 45, 65, 89, 73, 78, 2, 141, - 194, 1, 2, 45, 82, 14, 33, 6, 85, 77, 66, 69, 82, 32, 14, 42, 84, 186, - 151, 4, 79, 187, 217, 2, 70, 8, 42, 87, 250, 234, 9, 72, 239, 156, 1, 69, - 4, 182, 237, 9, 69, 131, 234, 1, 79, 18, 252, 1, 4, 78, 67, 84, 85, 106, - 82, 56, 19, 84, 32, 76, 73, 84, 84, 69, 82, 32, 73, 78, 32, 73, 84, 83, - 32, 80, 76, 65, 170, 140, 1, 83, 140, 42, 20, 66, 76, 73, 67, 32, 65, 68, - 68, 82, 69, 83, 83, 32, 76, 79, 85, 68, 83, 80, 69, 194, 157, 10, 49, 3, - 50, 4, 80, 9, 83, 32, 69, 76, 69, 86, 65, 84, 85, 137, 150, 11, 5, 65, - 84, 73, 79, 78, 2, 135, 147, 11, 83, 4, 32, 2, 80, 76, 183, 189, 11, 83, - 2, 227, 204, 10, 69, 2, 11, 67, 2, 215, 192, 10, 69, 40, 98, 65, 148, 6, - 6, 69, 83, 84, 73, 79, 78, 214, 231, 5, 79, 253, 216, 4, 5, 73, 78, 67, - 85, 78, 30, 104, 2, 68, 82, 208, 4, 9, 84, 69, 82, 78, 73, 79, 78, 32, - 73, 48, 4, 82, 84, 69, 82, 223, 170, 10, 79, 24, 56, 4, 65, 78, 84, 32, - 253, 3, 5, 85, 80, 76, 69, 32, 20, 44, 6, 85, 80, 80, 69, 82, 32, 131, 2, - 76, 16, 56, 4, 76, 69, 70, 84, 249, 1, 5, 82, 73, 71, 72, 84, 11, 29, 5, - 32, 65, 78, 68, 32, 8, 108, 6, 76, 79, 87, 69, 82, 32, 53, 17, 85, 80, - 80, 69, 82, 32, 82, 73, 71, 72, 84, 32, 65, 78, 68, 32, 76, 4, 184, 1, 5, - 76, 69, 70, 84, 32, 195, 163, 11, 82, 4, 11, 79, 4, 11, 87, 4, 241, 163, - 11, 2, 69, 82, 7, 69, 15, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 76, - 69, 70, 84, 5, 11, 32, 2, 21, 3, 65, 78, 68, 2, 229, 255, 9, 6, 32, 76, - 79, 87, 69, 82, 4, 22, 73, 155, 37, 80, 2, 133, 163, 8, 7, 78, 84, 69, - 71, 82, 65, 76, 2, 17, 2, 32, 78, 2, 195, 136, 10, 79, 6, 34, 32, 185, - 165, 5, 2, 69, 68, 4, 198, 244, 4, 69, 211, 150, 6, 77, 152, 9, 114, 65, - 182, 7, 69, 224, 27, 6, 72, 73, 78, 79, 67, 69, 34, 73, 182, 78, 76, 46, - 79, 246, 18, 85, 151, 198, 10, 83, 60, 110, 67, 104, 2, 68, 73, 130, 1, - 73, 202, 4, 84, 160, 210, 7, 4, 66, 66, 73, 84, 130, 185, 3, 90, 223, 56, - 77, 6, 40, 4, 73, 78, 71, 32, 211, 184, 8, 67, 4, 140, 251, 9, 7, 77, 79, - 84, 79, 82, 67, 89, 203, 44, 67, 8, 66, 79, 245, 255, 7, 10, 67, 65, 76, - 32, 83, 89, 77, 66, 79, 76, 7, 200, 145, 6, 4, 65, 67, 84, 73, 197, 133, - 4, 2, 32, 66, 34, 60, 5, 76, 87, 65, 89, 32, 46, 78, 21, 4, 83, 69, 68, - 32, 4, 228, 189, 8, 2, 84, 82, 247, 231, 1, 67, 5, 203, 173, 10, 66, 26, - 156, 1, 3, 68, 79, 84, 34, 73, 48, 4, 72, 65, 78, 68, 182, 1, 77, 198, - 139, 7, 70, 186, 237, 1, 67, 226, 170, 1, 83, 197, 18, 7, 66, 65, 67, 75, - 32, 79, 70, 5, 29, 5, 84, 69, 68, 32, 73, 2, 165, 39, 8, 78, 84, 69, 82, - 80, 79, 76, 65, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 130, 27, 70, 201, - 176, 2, 28, 80, 65, 82, 84, 32, 66, 69, 84, 87, 69, 69, 78, 32, 77, 73, - 68, 68, 76, 69, 32, 65, 78, 68, 32, 82, 73, 78, 71, 6, 182, 189, 10, 67, - 2, 68, 3, 82, 5, 135, 165, 11, 73, 248, 1, 226, 1, 67, 232, 3, 2, 68, 32, - 64, 2, 71, 73, 144, 1, 5, 74, 65, 78, 71, 32, 254, 4, 76, 32, 8, 77, 73, - 78, 68, 69, 82, 32, 82, 34, 80, 106, 83, 136, 1, 5, 84, 85, 82, 78, 32, - 42, 86, 225, 184, 1, 5, 70, 69, 82, 69, 78, 24, 114, 69, 32, 3, 89, 67, - 76, 158, 176, 5, 79, 181, 237, 3, 13, 82, 69, 65, 84, 73, 79, 78, 65, 76, - 32, 86, 69, 72, 2, 11, 73, 2, 211, 164, 11, 80, 18, 78, 69, 53, 15, 73, - 78, 71, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 2, 29, 5, 68, 32, - 80, 65, 80, 2, 239, 223, 9, 69, 16, 100, 5, 84, 89, 80, 69, 45, 157, 237, - 4, 14, 71, 69, 78, 69, 82, 73, 67, 32, 77, 65, 84, 69, 82, 73, 14, 58, - 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 2, 229, 177, 5, 6, 32, 80, - 76, 65, 83, 84, 4, 42, 65, 229, 233, 4, 4, 71, 73, 70, 84, 2, 143, 173, - 3, 80, 54, 120, 4, 83, 84, 69, 82, 165, 212, 5, 20, 79, 78, 65, 76, 32, - 73, 78, 68, 73, 67, 65, 84, 79, 82, 32, 83, 89, 77, 66, 79, 2, 135, 215, - 9, 69, 74, 128, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, - 71, 78, 32, 44, 7, 76, 69, 84, 84, 69, 82, 32, 150, 2, 83, 35, 86, 8, - 174, 195, 9, 78, 150, 248, 1, 72, 3, 82, 46, 154, 1, 77, 34, 78, 166, - 183, 11, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, - 82, 2, 83, 2, 84, 2, 87, 2, 89, 187, 2, 65, 4, 194, 183, 11, 66, 187, 2, - 65, 12, 54, 89, 158, 234, 7, 71, 210, 204, 3, 68, 187, 2, 65, 4, 234, - 182, 11, 74, 187, 2, 65, 2, 11, 69, 2, 223, 207, 10, 67, 18, 64, 10, 79, - 87, 69, 76, 32, 83, 73, 71, 78, 32, 171, 134, 9, 73, 16, 54, 69, 190, - 247, 10, 65, 174, 64, 73, 2, 79, 3, 85, 7, 230, 183, 11, 65, 3, 85, 2, - 137, 167, 10, 3, 73, 69, 86, 2, 201, 231, 10, 3, 73, 66, 66, 2, 41, 8, - 76, 65, 67, 69, 77, 69, 78, 84, 2, 17, 2, 32, 67, 2, 253, 255, 9, 5, 72, - 65, 82, 65, 67, 8, 32, 2, 84, 82, 187, 217, 6, 80, 6, 168, 134, 6, 16, - 73, 67, 84, 69, 68, 32, 76, 69, 70, 84, 32, 69, 78, 84, 82, 89, 243, 158, - 5, 79, 6, 162, 162, 10, 83, 142, 104, 76, 31, 82, 70, 64, 4, 69, 82, 83, - 69, 173, 228, 3, 6, 79, 76, 86, 73, 78, 71, 68, 30, 32, 169, 3, 2, 68, - 32, 20, 184, 1, 6, 67, 72, 69, 67, 75, 69, 28, 2, 76, 73, 108, 7, 83, 79, - 76, 73, 68, 85, 83, 196, 193, 7, 16, 84, 73, 76, 68, 69, 32, 79, 80, 69, - 82, 65, 84, 79, 82, 32, 65, 147, 203, 2, 73, 2, 221, 155, 10, 2, 82, 32, - 4, 136, 194, 3, 18, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, - 84, 69, 68, 32, 80, 239, 221, 1, 78, 9, 11, 32, 6, 200, 135, 5, 9, 80, - 82, 69, 67, 69, 68, 73, 78, 71, 226, 255, 2, 79, 167, 140, 1, 87, 48, + 174, 151, 4, 83, 184, 207, 4, 9, 84, 87, 79, 32, 68, 79, 84, 32, 76, 135, + 114, 81, 4, 167, 185, 2, 79, 6, 222, 153, 6, 88, 174, 216, 2, 77, 3, 78, + 2, 229, 138, 9, 7, 79, 82, 73, 90, 79, 78, 84, 4, 49, 10, 68, 69, 79, 71, + 82, 65, 80, 72, 73, 67, 4, 11, 32, 4, 142, 220, 9, 67, 35, 70, 22, 40, 4, + 69, 70, 84, 32, 219, 197, 10, 79, 20, 112, 6, 87, 72, 73, 84, 69, 32, + 142, 1, 66, 42, 68, 198, 99, 67, 166, 7, 65, 138, 190, 7, 80, 238, 7, 83, + 39, 84, 4, 162, 2, 67, 139, 99, 76, 22, 110, 66, 42, 68, 36, 6, 87, 72, + 73, 84, 69, 32, 162, 99, 67, 166, 7, 65, 138, 190, 7, 80, 238, 7, 83, 39, + 84, 2, 157, 100, 6, 76, 65, 67, 75, 32, 76, 2, 209, 106, 5, 79, 85, 66, + 76, 69, 6, 74, 67, 17, 14, 76, 69, 78, 84, 73, 67, 85, 76, 65, 82, 32, + 66, 82, 65, 2, 239, 98, 79, 4, 218, 215, 11, 67, 177, 29, 2, 75, 67, 2, + 135, 194, 10, 79, 22, 46, 78, 156, 1, 2, 86, 65, 203, 145, 12, 77, 10, + 34, 84, 225, 200, 6, 2, 67, 69, 6, 26, 32, 53, 2, 69, 82, 2, 21, 3, 83, + 67, 82, 2, 153, 177, 10, 2, 69, 69, 5, 17, 2, 32, 73, 2, 195, 216, 11, + 67, 10, 60, 5, 67, 89, 32, 77, 69, 29, 6, 84, 69, 32, 85, 83, 69, 2, 133, + 161, 7, 2, 83, 83, 8, 26, 32, 159, 166, 8, 45, 4, 226, 239, 5, 84, 151, + 158, 5, 79, 14, 130, 1, 74, 30, 80, 236, 33, 5, 72, 73, 66, 73, 84, 200, + 230, 8, 6, 66, 73, 78, 71, 32, 67, 169, 225, 1, 5, 83, 69, 82, 80, 73, 2, + 165, 162, 5, 2, 69, 67, 6, 64, 6, 79, 82, 84, 73, 79, 78, 205, 139, 11, + 4, 69, 82, 84, 89, 5, 247, 209, 5, 65, 58, 172, 1, 15, 70, 79, 85, 82, + 32, 68, 79, 84, 83, 32, 87, 73, 84, 72, 32, 32, 7, 76, 69, 84, 84, 69, + 82, 32, 230, 2, 78, 134, 32, 83, 1, 8, 84, 85, 82, 78, 69, 68, 32, 83, 4, + 190, 226, 10, 67, 227, 112, 68, 36, 166, 1, 65, 22, 68, 34, 76, 22, 77, + 50, 87, 186, 165, 4, 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, + 75, 198, 75, 66, 178, 216, 4, 78, 134, 2, 84, 219, 103, 80, 2, 155, 166, + 4, 76, 2, 11, 65, 2, 143, 201, 6, 76, 2, 183, 166, 4, 65, 2, 25, 4, 69, + 77, 45, 81, 2, 147, 253, 5, 79, 2, 37, 7, 65, 87, 45, 65, 89, 73, 78, 2, + 173, 205, 1, 2, 45, 82, 14, 33, 6, 85, 77, 66, 69, 82, 32, 14, 42, 84, + 134, 166, 4, 79, 131, 231, 2, 70, 8, 42, 87, 150, 176, 10, 72, 231, 159, + 1, 69, 4, 210, 178, 10, 69, 135, 237, 1, 79, 18, 252, 1, 4, 78, 67, 84, + 85, 94, 82, 56, 19, 84, 32, 76, 73, 84, 84, 69, 82, 32, 73, 78, 32, 73, + 84, 83, 32, 80, 76, 65, 214, 151, 1, 83, 140, 42, 20, 66, 76, 73, 67, 32, + 65, 68, 68, 82, 69, 83, 83, 32, 76, 79, 85, 68, 83, 80, 69, 194, 218, 10, + 49, 3, 50, 4, 136, 206, 10, 9, 83, 32, 69, 76, 69, 86, 65, 84, 85, 229, + 144, 1, 5, 65, 84, 73, 79, 78, 4, 32, 2, 80, 76, 227, 133, 12, 83, 2, + 247, 148, 11, 69, 2, 11, 67, 2, 235, 136, 11, 69, 40, 98, 65, 180, 6, 6, + 69, 83, 84, 73, 79, 78, 162, 133, 6, 79, 165, 131, 5, 5, 73, 78, 67, 85, + 78, 30, 104, 2, 68, 82, 240, 4, 9, 84, 69, 82, 78, 73, 79, 78, 32, 73, + 48, 4, 82, 84, 69, 82, 219, 243, 10, 79, 24, 56, 4, 65, 78, 84, 32, 157, + 4, 5, 85, 80, 76, 69, 32, 20, 44, 6, 85, 80, 80, 69, 82, 32, 131, 2, 76, + 16, 56, 4, 76, 69, 70, 84, 249, 1, 5, 82, 73, 71, 72, 84, 11, 29, 5, 32, + 65, 78, 68, 32, 8, 108, 6, 76, 79, 87, 69, 82, 32, 53, 17, 85, 80, 80, + 69, 82, 32, 82, 73, 71, 72, 84, 32, 65, 78, 68, 32, 76, 4, 192, 1, 5, 76, + 69, 70, 84, 32, 231, 235, 11, 82, 4, 11, 79, 4, 11, 87, 4, 157, 236, 11, + 2, 69, 82, 7, 65, 14, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 76, 69, + 70, 4, 11, 84, 5, 11, 32, 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 76, 2, 17, + 2, 79, 87, 2, 237, 196, 10, 2, 69, 82, 4, 22, 73, 223, 37, 80, 2, 205, + 213, 8, 7, 78, 84, 69, 71, 82, 65, 76, 2, 17, 2, 32, 78, 2, 131, 206, 10, + 79, 6, 34, 32, 233, 192, 5, 2, 69, 68, 4, 250, 134, 5, 69, 159, 204, 6, + 77, 220, 9, 114, 65, 142, 8, 69, 204, 27, 6, 72, 73, 78, 79, 67, 69, 34, + 73, 174, 88, 76, 46, 79, 198, 19, 85, 151, 131, 11, 83, 62, 110, 67, 104, + 2, 68, 73, 130, 1, 73, 162, 5, 84, 152, 237, 7, 4, 66, 66, 73, 84, 178, + 229, 3, 90, 235, 56, 77, 6, 40, 4, 73, 78, 71, 32, 187, 235, 8, 67, 4, + 164, 192, 10, 7, 77, 79, 84, 79, 82, 67, 89, 175, 48, 67, 8, 66, 79, 205, + 165, 8, 10, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 7, 232, 174, 6, 4, + 65, 67, 84, 73, 129, 175, 4, 2, 32, 66, 36, 60, 5, 76, 87, 65, 89, 32, + 46, 78, 21, 4, 83, 69, 68, 32, 4, 140, 245, 8, 2, 84, 82, 203, 249, 1, + 67, 5, 191, 245, 10, 66, 28, 152, 1, 3, 68, 79, 84, 34, 73, 48, 4, 72, + 65, 78, 68, 182, 1, 77, 38, 83, 166, 167, 7, 70, 230, 146, 2, 67, 237, + 195, 1, 7, 66, 65, 67, 75, 32, 79, 70, 5, 29, 5, 84, 69, 68, 32, 73, 2, + 193, 40, 8, 78, 84, 69, 82, 80, 79, 76, 65, 7, 33, 6, 32, 87, 73, 84, 72, + 32, 4, 198, 27, 70, 209, 190, 2, 28, 80, 65, 82, 84, 32, 66, 69, 84, 87, + 69, 69, 78, 32, 77, 73, 68, 68, 76, 69, 32, 65, 78, 68, 32, 82, 73, 78, + 71, 6, 174, 133, 11, 67, 2, 68, 3, 82, 4, 60, 9, 77, 65, 76, 76, 32, 76, + 69, 70, 84, 255, 234, 10, 81, 2, 129, 145, 8, 2, 32, 83, 5, 187, 236, 11, + 73, 250, 1, 226, 1, 67, 132, 4, 2, 68, 32, 64, 2, 71, 73, 144, 1, 5, 74, + 65, 78, 71, 32, 202, 4, 76, 32, 8, 77, 73, 78, 68, 69, 82, 32, 82, 34, + 80, 106, 83, 136, 1, 5, 84, 85, 82, 78, 32, 42, 86, 193, 199, 1, 5, 70, + 69, 82, 69, 78, 26, 114, 69, 60, 3, 89, 67, 76, 230, 204, 5, 79, 149, + 145, 4, 13, 82, 69, 65, 84, 73, 79, 78, 65, 76, 32, 86, 69, 72, 4, 38, + 73, 181, 182, 10, 3, 80, 84, 65, 2, 235, 235, 11, 80, 18, 78, 69, 53, 15, + 73, 78, 71, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 2, 29, 5, 68, + 32, 80, 65, 80, 2, 131, 164, 10, 69, 16, 100, 5, 84, 89, 80, 69, 45, 133, + 254, 4, 14, 71, 69, 78, 69, 82, 73, 67, 32, 77, 65, 84, 69, 82, 73, 14, + 58, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 2, 173, 206, 5, 6, 32, + 80, 76, 65, 83, 84, 4, 42, 65, 165, 251, 4, 4, 71, 73, 70, 84, 2, 199, + 186, 3, 80, 54, 120, 4, 83, 84, 69, 82, 249, 240, 5, 20, 79, 78, 65, 76, + 32, 73, 78, 68, 73, 67, 65, 84, 79, 82, 32, 83, 89, 77, 66, 79, 2, 155, + 155, 10, 69, 74, 128, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, + 73, 71, 78, 32, 44, 7, 76, 69, 84, 84, 69, 82, 32, 226, 1, 83, 35, 86, 8, + 194, 135, 10, 78, 154, 251, 1, 72, 3, 82, 46, 162, 1, 78, 238, 243, 7, + 77, 234, 138, 4, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, + 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 187, 2, 65, 12, 206, 244, 7, 89, + 142, 27, 71, 250, 238, 3, 68, 187, 2, 65, 2, 11, 69, 2, 155, 151, 11, 67, + 18, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 151, 199, 9, 73, 16, + 54, 69, 254, 190, 11, 65, 186, 64, 73, 2, 79, 3, 85, 7, 178, 255, 11, 65, + 3, 85, 2, 189, 238, 10, 3, 73, 69, 86, 2, 137, 175, 11, 3, 73, 66, 66, 2, + 41, 8, 76, 65, 67, 69, 77, 69, 78, 84, 2, 17, 2, 32, 67, 2, 157, 198, 10, + 5, 72, 65, 82, 65, 67, 8, 32, 2, 84, 82, 251, 244, 6, 80, 6, 180, 252, 7, + 16, 73, 67, 84, 69, 68, 32, 76, 69, 70, 84, 32, 69, 78, 84, 82, 89, 179, + 240, 3, 79, 6, 214, 233, 10, 83, 166, 104, 76, 31, 82, 70, 64, 4, 69, 82, + 83, 69, 165, 242, 3, 6, 79, 76, 86, 73, 78, 71, 68, 30, 32, 169, 3, 2, + 68, 32, 20, 184, 1, 6, 67, 72, 69, 67, 75, 69, 28, 2, 76, 73, 108, 7, 83, + 79, 76, 73, 68, 85, 83, 212, 220, 7, 16, 84, 73, 76, 68, 69, 32, 79, 80, + 69, 82, 65, 84, 79, 82, 32, 65, 179, 246, 2, 73, 2, 145, 227, 10, 2, 82, + 32, 4, 248, 207, 3, 18, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, + 78, 84, 69, 68, 32, 80, 251, 236, 1, 78, 9, 11, 32, 6, 184, 162, 5, 9, + 80, 82, 69, 67, 69, 68, 73, 78, 71, 250, 150, 3, 79, 251, 154, 1, 87, 48, 232, 2, 5, 65, 78, 71, 76, 69, 20, 7, 68, 79, 85, 66, 76, 69, 32, 118, - 78, 20, 5, 70, 79, 82, 75, 69, 70, 80, 82, 82, 214, 1, 83, 102, 84, 212, - 201, 6, 30, 72, 65, 78, 68, 32, 87, 73, 84, 72, 32, 77, 73, 68, 68, 76, - 69, 32, 70, 73, 78, 71, 69, 82, 32, 69, 88, 84, 69, 78, 68, 158, 148, 2, - 67, 114, 81, 212, 61, 3, 86, 73, 67, 157, 36, 6, 69, 77, 80, 84, 89, 32, - 5, 159, 213, 3, 32, 6, 40, 3, 80, 82, 73, 41, 3, 83, 84, 82, 4, 17, 2, - 77, 69, 5, 195, 166, 3, 32, 2, 29, 5, 79, 75, 69, 32, 78, 2, 147, 150, 6, - 79, 2, 49, 10, 68, 32, 80, 65, 82, 65, 71, 82, 65, 80, 2, 175, 4, 72, 4, - 36, 3, 73, 76, 67, 151, 148, 10, 82, 2, 11, 82, 2, 253, 164, 10, 2, 79, + 78, 20, 5, 70, 79, 82, 75, 69, 70, 80, 82, 82, 214, 1, 83, 106, 84, 144, + 229, 6, 30, 72, 65, 78, 68, 32, 87, 73, 84, 72, 32, 77, 73, 68, 68, 76, + 69, 32, 70, 73, 78, 71, 69, 82, 32, 69, 88, 84, 69, 78, 68, 202, 185, 2, + 67, 114, 81, 156, 65, 3, 86, 73, 67, 177, 36, 6, 69, 77, 80, 84, 89, 32, + 5, 231, 227, 3, 32, 6, 40, 3, 80, 82, 73, 41, 3, 83, 84, 82, 4, 17, 2, + 77, 69, 5, 179, 180, 3, 32, 2, 29, 5, 79, 75, 69, 32, 78, 2, 235, 177, 6, + 79, 2, 49, 10, 68, 32, 80, 65, 82, 65, 71, 82, 65, 80, 2, 179, 4, 72, 4, + 36, 3, 73, 76, 67, 203, 219, 10, 82, 2, 11, 82, 2, 177, 236, 10, 2, 79, 87, 6, 156, 1, 17, 65, 73, 83, 69, 68, 32, 72, 65, 78, 68, 32, 87, 73, - 84, 72, 32, 70, 252, 96, 8, 79, 84, 65, 84, 69, 68, 32, 70, 229, 149, 6, - 4, 73, 71, 72, 84, 2, 185, 102, 9, 73, 78, 71, 69, 82, 83, 32, 83, 80, 4, - 132, 124, 17, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, 84, - 65, 76, 151, 248, 6, 69, 10, 88, 4, 73, 76, 68, 69, 28, 7, 82, 73, 80, - 76, 69, 32, 80, 173, 251, 6, 3, 72, 85, 77, 5, 189, 214, 4, 2, 32, 69, 2, - 187, 144, 10, 82, 2, 11, 82, 2, 171, 234, 10, 79, 213, 3, 228, 1, 4, 66, - 66, 79, 78, 164, 1, 3, 67, 69, 32, 60, 3, 71, 72, 84, 236, 71, 2, 78, 71, - 252, 1, 23, 83, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 67, - 82, 79, 83, 83, 73, 78, 71, 32, 134, 195, 5, 65, 143, 246, 3, 70, 19, 37, - 7, 32, 65, 82, 82, 79, 87, 32, 16, 80, 3, 76, 69, 70, 0, 4, 82, 73, 71, - 72, 22, 85, 213, 249, 10, 3, 68, 79, 87, 4, 175, 234, 1, 84, 4, 223, 249, - 10, 80, 4, 26, 67, 131, 160, 9, 66, 2, 133, 135, 1, 3, 82, 65, 67, 162, - 3, 110, 32, 158, 31, 45, 132, 8, 11, 72, 65, 78, 68, 32, 73, 78, 84, 69, - 82, 73, 29, 6, 87, 65, 82, 68, 83, 32, 176, 1, 198, 1, 65, 132, 6, 2, 66, - 76, 62, 67, 184, 1, 2, 68, 79, 242, 1, 70, 60, 2, 72, 65, 212, 2, 2, 76, - 79, 66, 78, 82, 79, 122, 80, 168, 1, 2, 82, 65, 58, 83, 154, 6, 84, 194, - 3, 86, 143, 1, 87, 28, 22, 78, 163, 4, 82, 22, 24, 2, 68, 32, 39, 71, 4, - 214, 41, 76, 21, 3, 85, 80, 80, 18, 26, 69, 17, 2, 76, 69, 2, 175, 23, - 82, 17, 11, 32, 14, 154, 1, 66, 44, 8, 68, 79, 84, 84, 69, 68, 32, 83, 2, - 83, 112, 5, 87, 73, 84, 72, 32, 189, 253, 9, 12, 86, 65, 82, 73, 65, 78, - 84, 32, 87, 73, 84, 72, 4, 133, 252, 9, 6, 82, 65, 67, 75, 69, 84, 2, 37, - 7, 85, 66, 83, 84, 73, 84, 85, 2, 25, 4, 84, 73, 79, 78, 2, 21, 3, 32, - 77, 65, 2, 243, 128, 1, 82, 4, 68, 11, 68, 79, 87, 78, 87, 65, 82, 68, - 83, 32, 90, 203, 171, 8, 65, 2, 141, 131, 10, 5, 73, 71, 90, 65, 71, 6, - 18, 67, 79, 82, 2, 41, 8, 32, 71, 82, 69, 65, 84, 69, 82, 2, 189, 20, 4, - 45, 84, 72, 65, 4, 41, 8, 79, 87, 32, 87, 73, 84, 72, 32, 4, 132, 191, 7, - 7, 67, 73, 82, 67, 76, 69, 68, 207, 141, 2, 83, 4, 25, 4, 65, 67, 75, 32, - 4, 194, 22, 76, 243, 172, 7, 84, 12, 38, 85, 230, 21, 79, 251, 222, 2, - 69, 8, 53, 11, 82, 76, 89, 32, 66, 82, 65, 67, 75, 69, 84, 9, 11, 32, 6, - 44, 4, 77, 73, 68, 68, 138, 8, 76, 23, 85, 2, 129, 229, 9, 2, 76, 69, 12, - 38, 84, 41, 5, 85, 66, 76, 69, 32, 2, 173, 14, 6, 84, 69, 68, 32, 83, 85, - 10, 50, 65, 94, 87, 174, 145, 3, 81, 151, 172, 4, 80, 4, 206, 25, 78, - 217, 149, 3, 15, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, 82, 79, 85, 78, - 68, 2, 203, 19, 73, 6, 26, 73, 227, 241, 2, 76, 4, 146, 146, 8, 86, 163, - 88, 83, 14, 32, 3, 76, 70, 32, 223, 1, 78, 12, 42, 66, 90, 70, 34, 82, - 139, 166, 8, 77, 6, 11, 76, 6, 40, 4, 65, 67, 75, 32, 231, 231, 9, 79, 4, - 166, 199, 9, 67, 167, 11, 83, 2, 11, 79, 2, 147, 242, 8, 76, 2, 33, 6, - 85, 78, 78, 73, 78, 71, 2, 131, 204, 6, 32, 2, 149, 145, 1, 16, 68, 32, - 84, 69, 76, 69, 80, 72, 79, 78, 69, 32, 82, 69, 67, 69, 2, 241, 193, 10, - 11, 87, 32, 80, 65, 82, 65, 80, 72, 82, 65, 83, 2, 197, 4, 16, 79, 82, - 77, 65, 76, 32, 70, 65, 67, 84, 79, 82, 32, 83, 69, 77, 8, 74, 85, 214, - 150, 8, 78, 237, 178, 1, 8, 80, 69, 78, 32, 83, 81, 85, 65, 2, 201, 147, - 10, 6, 84, 69, 82, 32, 74, 79, 8, 49, 10, 65, 82, 69, 78, 84, 72, 69, 83, - 73, 83, 9, 11, 32, 6, 34, 76, 22, 85, 147, 248, 8, 69, 2, 37, 2, 79, 87, - 2, 17, 2, 80, 80, 2, 169, 223, 3, 2, 69, 82, 2, 249, 8, 10, 73, 83, 69, - 68, 32, 79, 77, 73, 83, 83, 40, 62, 45, 70, 69, 78, 73, 68, 2, 80, 69, - 126, 81, 227, 2, 85, 2, 153, 208, 7, 12, 83, 72, 65, 80, 69, 68, 32, 66, - 65, 71, 32, 68, 4, 26, 77, 199, 160, 8, 86, 2, 205, 128, 10, 7, 73, 68, - 73, 82, 69, 67, 84, 4, 134, 136, 3, 78, 193, 180, 7, 8, 68, 69, 87, 65, - 89, 83, 32, 85, 8, 32, 4, 65, 75, 69, 82, 67, 69, 7, 33, 6, 32, 87, 73, - 84, 72, 32, 4, 186, 237, 3, 79, 75, 84, 2, 201, 4, 2, 67, 72, 20, 57, 12, - 85, 65, 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 21, 11, 32, 18, 84, 3, - 76, 79, 87, 0, 3, 85, 80, 80, 28, 5, 87, 73, 84, 72, 32, 223, 242, 8, 69, - 2, 133, 216, 3, 2, 69, 82, 12, 96, 8, 84, 73, 67, 75, 32, 73, 78, 32, - 162, 5, 81, 214, 189, 7, 85, 158, 97, 68, 195, 183, 2, 83, 4, 132, 215, - 3, 6, 66, 79, 84, 84, 79, 77, 1, 3, 84, 79, 80, 2, 197, 2, 6, 66, 83, 84, - 73, 84, 85, 20, 54, 72, 162, 1, 82, 146, 177, 7, 79, 179, 169, 2, 65, 8, - 54, 73, 60, 5, 79, 85, 71, 72, 84, 131, 155, 8, 82, 2, 201, 197, 1, 10, - 82, 68, 32, 87, 72, 73, 84, 69, 32, 82, 2, 11, 32, 2, 205, 250, 2, 3, 66, - 85, 66, 8, 34, 65, 89, 4, 73, 65, 78, 71, 2, 33, 6, 78, 83, 80, 79, 83, - 73, 2, 11, 84, 2, 17, 2, 73, 79, 2, 191, 181, 10, 78, 6, 32, 2, 76, 69, - 183, 154, 8, 85, 5, 41, 8, 32, 65, 66, 79, 86, 69, 32, 76, 2, 161, 232, - 8, 2, 69, 70, 4, 45, 9, 69, 82, 84, 73, 67, 65, 76, 32, 66, 4, 68, 9, 65, - 82, 32, 87, 73, 84, 72, 32, 81, 245, 236, 9, 2, 79, 88, 2, 215, 149, 8, - 85, 14, 22, 72, 203, 1, 73, 12, 25, 4, 73, 84, 69, 32, 12, 54, 67, 54, - 76, 182, 170, 7, 80, 154, 2, 83, 39, 84, 4, 26, 79, 215, 168, 7, 85, 2, - 65, 3, 82, 78, 69, 2, 41, 8, 69, 78, 84, 73, 67, 85, 76, 65, 2, 231, 177, - 10, 82, 2, 221, 167, 6, 6, 71, 71, 76, 89, 32, 70, 30, 82, 70, 206, 1, - 72, 130, 1, 80, 250, 1, 83, 197, 1, 6, 84, 79, 45, 76, 69, 70, 8, 33, 6, - 65, 67, 73, 78, 71, 32, 8, 144, 1, 14, 65, 82, 77, 69, 78, 73, 65, 78, - 32, 69, 84, 69, 82, 78, 144, 196, 3, 11, 83, 86, 65, 83, 84, 73, 32, 83, - 73, 71, 78, 151, 133, 3, 70, 2, 239, 227, 7, 73, 2, 81, 18, 65, 78, 68, - 69, 68, 32, 73, 78, 84, 69, 82, 76, 65, 67, 69, 68, 32, 80, 2, 21, 3, 69, - 78, 84, 2, 171, 199, 9, 65, 8, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 8, - 140, 1, 6, 67, 85, 82, 86, 69, 68, 22, 65, 224, 247, 2, 6, 68, 79, 85, - 66, 76, 69, 253, 239, 6, 10, 77, 65, 71, 78, 73, 70, 89, 73, 78, 71, 2, - 17, 2, 32, 65, 2, 11, 78, 2, 157, 172, 10, 2, 71, 76, 4, 46, 72, 85, 7, - 73, 68, 69, 32, 65, 82, 67, 2, 17, 2, 65, 68, 2, 17, 2, 69, 68, 2, 165, - 222, 9, 6, 32, 87, 72, 73, 84, 69, 2, 11, 32, 2, 193, 168, 8, 8, 67, 76, - 79, 67, 75, 87, 73, 83, 8, 17, 2, 84, 32, 8, 86, 73, 40, 4, 79, 86, 69, - 82, 196, 238, 4, 5, 69, 77, 66, 69, 68, 187, 204, 5, 77, 2, 17, 2, 83, - 79, 2, 207, 129, 1, 76, 2, 179, 146, 3, 82, 2, 249, 236, 9, 2, 79, 82, - 210, 1, 160, 1, 5, 65, 82, 82, 79, 87, 182, 8, 66, 86, 68, 210, 1, 70, - 122, 72, 230, 6, 76, 26, 79, 34, 80, 50, 82, 70, 83, 94, 84, 202, 8, 87, - 226, 254, 7, 67, 47, 81, 71, 26, 32, 159, 205, 8, 45, 66, 94, 65, 170, 2, - 70, 82, 84, 184, 1, 5, 87, 73, 84, 72, 32, 149, 152, 1, 4, 79, 86, 69, - 82, 12, 40, 5, 66, 79, 86, 69, 32, 143, 1, 78, 10, 70, 82, 208, 155, 1, - 5, 83, 72, 79, 82, 84, 250, 180, 3, 65, 55, 84, 4, 37, 7, 69, 86, 69, 82, - 83, 69, 32, 4, 158, 208, 4, 65, 55, 84, 2, 61, 13, 68, 32, 85, 80, 80, - 69, 82, 32, 65, 78, 68, 32, 76, 2, 17, 2, 79, 87, 2, 137, 142, 8, 2, 69, - 82, 4, 37, 7, 82, 79, 77, 32, 66, 65, 82, 5, 189, 1, 6, 32, 84, 79, 32, - 66, 76, 10, 56, 7, 72, 82, 79, 85, 71, 72, 32, 65, 3, 79, 32, 66, 6, 144, - 203, 4, 3, 83, 85, 80, 130, 156, 4, 71, 131, 143, 2, 88, 4, 26, 76, 235, - 188, 10, 65, 2, 213, 171, 9, 3, 65, 67, 75, 38, 140, 1, 6, 67, 79, 82, - 78, 69, 82, 26, 68, 98, 76, 38, 80, 30, 83, 38, 84, 210, 139, 8, 77, 38, - 78, 122, 69, 226, 148, 1, 72, 155, 160, 1, 86, 2, 213, 18, 2, 32, 68, 4, - 11, 79, 4, 40, 4, 84, 84, 69, 68, 203, 177, 7, 85, 2, 17, 2, 32, 83, 2, - 247, 224, 10, 84, 4, 150, 140, 8, 65, 247, 237, 1, 79, 2, 249, 203, 8, 2, - 76, 85, 6, 218, 140, 8, 77, 135, 182, 2, 84, 10, 206, 16, 73, 171, 3, 65, - 8, 58, 65, 128, 12, 5, 79, 84, 84, 79, 77, 219, 128, 8, 76, 2, 145, 2, 2, - 67, 75, 14, 48, 6, 79, 85, 66, 76, 69, 32, 235, 154, 8, 65, 12, 40, 5, - 65, 82, 82, 79, 87, 135, 19, 68, 11, 26, 32, 183, 195, 8, 45, 6, 26, 87, - 243, 144, 8, 70, 4, 25, 4, 73, 84, 72, 32, 4, 190, 191, 10, 86, 79, 83, - 4, 40, 4, 82, 79, 78, 84, 143, 140, 8, 73, 2, 141, 139, 8, 14, 45, 84, - 73, 76, 84, 69, 68, 32, 83, 72, 65, 68, 79, 87, 30, 26, 65, 195, 143, 8, - 69, 26, 48, 6, 82, 80, 79, 79, 78, 32, 147, 168, 10, 78, 24, 88, 8, 79, - 86, 69, 82, 32, 76, 69, 70, 45, 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, - 32, 2, 237, 219, 7, 6, 84, 87, 65, 82, 68, 83, 22, 44, 4, 68, 79, 87, 78, - 173, 1, 2, 85, 80, 10, 26, 32, 247, 148, 8, 87, 8, 76, 8, 65, 66, 79, 86, - 69, 32, 76, 69, 18, 66, 190, 140, 8, 70, 175, 5, 84, 2, 227, 2, 70, 2, - 185, 201, 4, 7, 69, 76, 79, 87, 32, 76, 79, 12, 26, 32, 203, 147, 8, 87, - 10, 60, 6, 65, 66, 79, 86, 69, 32, 178, 139, 8, 70, 175, 5, 84, 6, 22, - 76, 155, 1, 82, 4, 32, 2, 69, 70, 247, 199, 4, 79, 2, 233, 135, 2, 24, - 84, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, - 72, 32, 66, 65, 82, 66, 2, 21, 3, 73, 71, 72, 2, 105, 24, 84, 87, 65, 82, - 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, - 82, 66, 2, 11, 32, 2, 163, 232, 1, 68, 2, 141, 1, 2, 69, 70, 2, 221, 141, - 8, 3, 80, 69, 78, 4, 218, 145, 8, 65, 221, 199, 1, 3, 85, 83, 72, 4, 36, - 3, 73, 71, 72, 187, 148, 10, 79, 2, 11, 84, 2, 171, 1, 45, 6, 32, 2, 81, - 85, 235, 136, 8, 65, 4, 26, 73, 131, 137, 8, 65, 2, 245, 144, 8, 2, 71, - 71, 54, 38, 79, 60, 2, 82, 73, 227, 4, 87, 2, 11, 80, 2, 11, 32, 2, 149, - 129, 8, 4, 83, 72, 65, 68, 34, 40, 5, 65, 78, 71, 76, 69, 255, 3, 80, 30, - 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 251, 149, 8, 32, 28, 52, 5, 65, - 82, 82, 79, 87, 218, 141, 8, 68, 39, 80, 25, 11, 32, 22, 64, 8, 79, 86, - 69, 82, 32, 76, 69, 70, 22, 87, 155, 137, 8, 84, 2, 203, 136, 8, 84, 18, - 25, 4, 73, 84, 72, 32, 18, 128, 1, 7, 68, 79, 85, 66, 76, 69, 32, 36, 7, - 76, 79, 78, 71, 32, 84, 73, 246, 136, 8, 66, 158, 1, 77, 34, 78, 34, 86, - 35, 72, 4, 186, 196, 8, 72, 255, 236, 1, 86, 4, 11, 80, 4, 11, 32, 4, 18, - 68, 35, 85, 2, 197, 137, 8, 3, 79, 87, 78, 2, 167, 137, 8, 80, 4, 21, 3, - 76, 69, 32, 4, 138, 3, 68, 183, 195, 9, 65, 18, 11, 79, 18, 56, 8, 45, - 72, 69, 65, 68, 69, 68, 32, 191, 139, 8, 32, 16, 76, 6, 65, 82, 82, 79, - 87, 32, 213, 1, 8, 84, 82, 73, 80, 76, 69, 32, 68, 14, 44, 5, 87, 73, 84, - 72, 32, 203, 255, 7, 70, 12, 42, 84, 206, 156, 7, 68, 191, 145, 3, 86, 8, - 26, 65, 131, 139, 8, 82, 6, 17, 2, 73, 76, 7, 33, 6, 32, 87, 73, 84, 72, - 32, 4, 246, 155, 7, 68, 191, 145, 3, 86, 2, 153, 127, 2, 65, 83, 8, 58, - 65, 21, 10, 72, 73, 84, 69, 32, 65, 82, 82, 79, 87, 2, 227, 135, 8, 86, - 7, 11, 32, 4, 164, 185, 2, 4, 70, 82, 79, 77, 147, 212, 5, 87, 19, 66, - 32, 136, 1, 6, 69, 68, 32, 80, 76, 65, 21, 3, 73, 78, 71, 12, 82, 66, 22, - 80, 132, 180, 4, 2, 73, 78, 26, 69, 214, 252, 2, 79, 187, 173, 2, 65, 2, - 195, 201, 10, 85, 2, 11, 79, 2, 231, 151, 10, 73, 2, 215, 165, 10, 78, 2, - 189, 190, 3, 2, 32, 66, 4, 24, 2, 70, 65, 75, 83, 2, 17, 2, 76, 76, 2, - 21, 3, 73, 78, 71, 2, 205, 222, 1, 2, 32, 68, 2, 225, 164, 3, 2, 79, 85, - 8, 178, 217, 10, 69, 2, 73, 2, 77, 3, 79, 124, 136, 2, 2, 67, 75, 20, 2, - 76, 76, 134, 2, 79, 56, 4, 77, 65, 78, 32, 192, 8, 2, 83, 69, 20, 6, 84, - 65, 84, 69, 68, 32, 152, 3, 3, 85, 78, 68, 230, 169, 3, 87, 228, 189, 3, - 15, 65, 83, 84, 69, 68, 32, 83, 87, 69, 69, 84, 32, 80, 79, 84, 165, 253, - 1, 2, 66, 79, 5, 207, 186, 10, 69, 10, 130, 1, 69, 64, 4, 32, 79, 70, 32, - 101, 21, 73, 78, 71, 32, 79, 78, 32, 84, 72, 69, 32, 70, 76, 79, 79, 82, - 32, 76, 65, 85, 71, 6, 60, 9, 68, 45, 85, 80, 32, 78, 69, 87, 83, 33, 2, - 82, 32, 2, 11, 80, 2, 207, 250, 1, 65, 4, 28, 3, 67, 79, 65, 23, 83, 2, - 151, 158, 9, 83, 2, 239, 89, 75, 2, 199, 131, 10, 72, 72, 140, 1, 6, 67, - 69, 78, 84, 85, 82, 22, 68, 100, 3, 81, 85, 73, 28, 8, 78, 85, 77, 69, - 82, 65, 76, 32, 182, 4, 83, 114, 85, 159, 179, 7, 65, 2, 223, 187, 5, 73, - 6, 98, 69, 232, 5, 5, 85, 80, 79, 78, 68, 61, 12, 73, 77, 73, 68, 73, 65, - 32, 83, 69, 88, 84, 85, 2, 229, 5, 3, 78, 65, 82, 48, 142, 1, 70, 136, 1, - 3, 79, 78, 69, 134, 1, 83, 66, 84, 152, 14, 10, 82, 69, 86, 69, 82, 83, - 69, 68, 32, 79, 142, 229, 2, 69, 235, 193, 6, 78, 14, 26, 73, 155, 216, - 9, 79, 12, 36, 3, 70, 84, 89, 167, 246, 2, 86, 7, 11, 32, 4, 158, 168, 5, - 84, 185, 169, 4, 5, 69, 65, 82, 76, 89, 11, 11, 32, 8, 50, 72, 41, 8, 84, - 72, 79, 85, 83, 65, 78, 68, 4, 185, 1, 6, 85, 78, 68, 82, 69, 68, 5, 205, - 245, 3, 2, 32, 67, 6, 32, 2, 73, 88, 223, 225, 8, 69, 5, 221, 207, 9, 2, - 32, 76, 10, 42, 69, 190, 244, 2, 87, 135, 237, 5, 72, 4, 11, 78, 5, 11, - 32, 2, 223, 165, 5, 84, 10, 46, 69, 197, 145, 7, 5, 73, 76, 73, 81, 85, - 8, 60, 2, 77, 85, 40, 5, 83, 84, 69, 82, 84, 21, 2, 88, 84, 2, 17, 2, 78, - 67, 2, 239, 144, 7, 73, 2, 231, 178, 7, 73, 4, 18, 65, 23, 85, 2, 203, - 178, 7, 78, 2, 159, 144, 7, 76, 5, 191, 135, 9, 84, 10, 166, 2, 70, 32, + 84, 72, 32, 70, 200, 107, 8, 79, 84, 65, 84, 69, 68, 32, 70, 217, 166, 6, + 4, 73, 71, 72, 84, 2, 133, 113, 9, 73, 78, 71, 69, 82, 83, 32, 83, 80, 4, + 208, 134, 1, 17, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, + 84, 65, 76, 183, 159, 7, 69, 10, 88, 4, 73, 76, 68, 69, 28, 7, 82, 73, + 80, 76, 69, 32, 80, 241, 150, 7, 3, 72, 85, 77, 5, 213, 231, 4, 2, 32, + 69, 2, 235, 215, 10, 82, 2, 11, 82, 2, 231, 177, 11, 79, 147, 4, 228, 1, + 4, 66, 66, 79, 78, 152, 1, 3, 67, 69, 32, 60, 3, 71, 72, 84, 240, 81, 2, + 78, 71, 252, 1, 23, 83, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, + 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 226, 212, 5, 65, 255, 158, 4, 70, + 19, 37, 7, 32, 65, 82, 82, 79, 87, 32, 16, 88, 3, 76, 69, 70, 0, 4, 82, + 73, 71, 72, 158, 193, 4, 85, 141, 128, 7, 3, 68, 79, 87, 4, 171, 248, 1, + 84, 4, 26, 67, 199, 228, 9, 66, 2, 217, 145, 1, 3, 82, 65, 67, 224, 3, + 110, 32, 190, 36, 45, 152, 12, 11, 72, 65, 78, 68, 32, 73, 78, 84, 69, + 82, 73, 29, 6, 87, 65, 82, 68, 83, 32, 202, 1, 166, 2, 65, 132, 6, 2, 66, + 76, 62, 67, 184, 1, 2, 68, 79, 242, 1, 70, 60, 2, 72, 65, 176, 5, 13, 74, + 85, 83, 84, 73, 70, 73, 69, 68, 32, 76, 69, 70, 20, 2, 76, 79, 66, 78, + 82, 79, 122, 80, 148, 1, 2, 82, 65, 58, 83, 154, 6, 84, 160, 5, 9, 86, + 69, 82, 84, 73, 67, 65, 76, 32, 147, 1, 87, 28, 22, 78, 163, 4, 82, 22, + 24, 2, 68, 32, 39, 71, 4, 170, 50, 76, 21, 3, 85, 80, 80, 18, 26, 69, 17, + 2, 76, 69, 2, 203, 26, 82, 17, 11, 32, 14, 154, 1, 66, 44, 8, 68, 79, 84, + 84, 69, 68, 32, 83, 2, 83, 112, 5, 87, 73, 84, 72, 32, 153, 196, 10, 12, + 86, 65, 82, 73, 65, 78, 84, 32, 87, 73, 84, 72, 4, 153, 194, 10, 6, 82, + 65, 67, 75, 69, 84, 2, 37, 7, 85, 66, 83, 84, 73, 84, 85, 2, 25, 4, 84, + 73, 79, 78, 2, 21, 3, 32, 77, 65, 2, 231, 138, 1, 82, 4, 68, 11, 68, 79, + 87, 78, 87, 65, 82, 68, 83, 32, 90, 179, 232, 8, 65, 2, 233, 201, 10, 5, + 73, 71, 90, 65, 71, 6, 18, 67, 79, 82, 2, 41, 8, 32, 71, 82, 69, 65, 84, + 69, 82, 2, 249, 24, 4, 45, 84, 72, 65, 4, 41, 8, 79, 87, 32, 87, 73, 84, + 72, 32, 4, 188, 220, 7, 7, 67, 73, 82, 67, 76, 69, 68, 151, 180, 2, 83, + 4, 25, 4, 65, 67, 75, 32, 4, 130, 27, 76, 139, 205, 7, 84, 12, 38, 85, + 166, 26, 79, 207, 231, 2, 69, 8, 53, 11, 82, 76, 89, 32, 66, 82, 65, 67, + 75, 69, 84, 9, 11, 32, 6, 44, 4, 77, 73, 68, 68, 250, 10, 76, 27, 85, 2, + 201, 170, 10, 2, 76, 69, 12, 38, 84, 41, 5, 85, 66, 76, 69, 32, 2, 137, + 17, 6, 84, 69, 68, 32, 83, 85, 10, 50, 65, 94, 87, 198, 158, 3, 81, 131, + 190, 4, 80, 4, 162, 31, 78, 157, 157, 3, 15, 82, 82, 79, 87, 32, 87, 73, + 84, 72, 32, 82, 79, 85, 78, 68, 2, 139, 24, 73, 6, 26, 73, 247, 254, 2, + 76, 4, 154, 201, 8, 86, 175, 97, 83, 28, 32, 3, 76, 70, 32, 187, 4, 78, + 26, 128, 1, 8, 65, 78, 68, 32, 76, 69, 70, 84, 62, 66, 90, 70, 82, 72, + 50, 82, 58, 84, 70, 87, 246, 13, 76, 22, 85, 215, 212, 8, 77, 2, 29, 5, + 32, 72, 65, 76, 70, 2, 245, 202, 8, 2, 32, 87, 6, 11, 76, 6, 40, 4, 65, + 67, 75, 32, 163, 172, 10, 79, 4, 146, 138, 10, 67, 207, 11, 83, 4, 58, + 79, 221, 152, 3, 8, 76, 89, 73, 78, 71, 32, 83, 65, 2, 175, 187, 9, 76, + 2, 253, 201, 8, 7, 79, 82, 73, 90, 79, 78, 84, 2, 33, 6, 85, 78, 78, 73, + 78, 71, 2, 247, 228, 6, 32, 2, 209, 169, 5, 12, 82, 73, 80, 76, 69, 32, + 68, 65, 83, 72, 32, 72, 2, 189, 151, 10, 4, 72, 73, 84, 69, 2, 173, 152, + 1, 16, 68, 32, 84, 69, 76, 69, 80, 72, 79, 78, 69, 32, 82, 69, 67, 69, 4, + 231, 217, 7, 84, 2, 233, 133, 11, 11, 87, 32, 80, 65, 82, 65, 80, 72, 82, + 65, 83, 2, 177, 4, 16, 79, 82, 77, 65, 76, 32, 70, 65, 67, 84, 79, 82, + 32, 83, 69, 77, 8, 74, 85, 210, 206, 8, 78, 169, 188, 1, 8, 80, 69, 78, + 32, 83, 81, 85, 65, 2, 181, 215, 10, 6, 84, 69, 82, 32, 74, 79, 8, 49, + 10, 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 9, 11, 32, 6, 34, 76, 26, 85, + 171, 182, 9, 69, 2, 157, 37, 2, 79, 87, 2, 133, 37, 2, 80, 80, 2, 217, + 10, 10, 73, 83, 69, 68, 32, 79, 77, 73, 83, 83, 40, 62, 45, 70, 69, 78, + 73, 68, 2, 80, 69, 126, 81, 227, 2, 85, 2, 253, 241, 7, 12, 83, 72, 65, + 80, 69, 68, 32, 66, 65, 71, 32, 68, 4, 26, 77, 231, 221, 8, 86, 2, 205, + 196, 10, 7, 73, 68, 73, 82, 69, 67, 84, 4, 194, 146, 3, 78, 145, 238, 7, + 8, 68, 69, 87, 65, 89, 83, 32, 85, 8, 32, 4, 65, 75, 69, 82, 67, 69, 7, + 33, 6, 32, 87, 73, 84, 72, 32, 4, 214, 248, 3, 79, 55, 84, 2, 137, 5, 2, + 67, 72, 20, 57, 12, 85, 65, 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 21, + 11, 32, 18, 84, 3, 76, 79, 87, 0, 3, 85, 80, 80, 28, 5, 87, 73, 84, 72, + 32, 143, 177, 9, 69, 2, 213, 226, 3, 2, 69, 82, 12, 96, 8, 84, 73, 67, + 75, 32, 73, 78, 32, 230, 6, 81, 246, 221, 7, 85, 222, 125, 68, 183, 189, + 2, 83, 4, 212, 225, 3, 6, 66, 79, 84, 84, 79, 77, 1, 3, 84, 79, 80, 2, + 165, 4, 6, 66, 83, 84, 73, 84, 85, 26, 54, 72, 130, 3, 82, 174, 209, 7, + 79, 171, 202, 2, 65, 14, 62, 73, 116, 5, 79, 85, 71, 72, 84, 45, 4, 82, + 69, 69, 32, 4, 21, 3, 82, 68, 32, 4, 68, 4, 73, 78, 68, 85, 185, 207, 1, + 7, 87, 72, 73, 84, 69, 32, 82, 2, 175, 148, 11, 67, 2, 11, 32, 2, 197, + 132, 3, 3, 66, 85, 66, 8, 60, 9, 81, 85, 65, 82, 84, 69, 82, 83, 32, 191, + 215, 8, 69, 6, 34, 76, 22, 85, 179, 221, 8, 66, 2, 37, 2, 79, 87, 2, 17, + 2, 80, 80, 2, 139, 216, 8, 69, 8, 34, 65, 89, 4, 73, 65, 78, 71, 2, 33, + 6, 78, 83, 80, 79, 83, 73, 2, 11, 84, 2, 17, 2, 73, 79, 2, 235, 247, 10, + 78, 6, 32, 2, 76, 69, 199, 214, 8, 85, 5, 41, 8, 32, 65, 66, 79, 86, 69, + 32, 76, 2, 225, 163, 9, 2, 69, 70, 6, 18, 66, 95, 82, 4, 68, 9, 65, 82, + 32, 87, 73, 84, 72, 32, 81, 177, 175, 10, 2, 79, 88, 2, 251, 208, 8, 85, + 2, 241, 173, 9, 3, 85, 76, 69, 14, 22, 72, 203, 1, 73, 12, 25, 4, 73, 84, + 69, 32, 12, 54, 67, 54, 76, 250, 196, 7, 80, 238, 7, 83, 39, 84, 4, 26, + 79, 207, 193, 7, 85, 2, 65, 3, 82, 78, 69, 2, 41, 8, 69, 78, 84, 73, 67, + 85, 76, 65, 2, 143, 244, 10, 82, 2, 133, 190, 6, 6, 71, 71, 76, 89, 32, + 70, 62, 118, 70, 226, 2, 72, 128, 1, 9, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 214, 4, 83, 197, 1, 6, 84, 79, 45, 76, 69, 70, 18, 33, 6, 65, 67, 73, + 78, 71, 32, 18, 116, 14, 65, 82, 77, 69, 78, 73, 65, 78, 32, 69, 84, 69, + 82, 78, 20, 4, 66, 65, 83, 83, 16, 3, 70, 73, 83, 87, 83, 2, 171, 145, 8, + 73, 2, 227, 43, 73, 6, 26, 72, 239, 195, 11, 84, 5, 11, 32, 2, 149, 162, + 8, 6, 87, 73, 84, 72, 32, 79, 8, 236, 203, 3, 10, 86, 65, 83, 84, 73, 32, + 83, 73, 71, 78, 143, 213, 4, 78, 2, 81, 18, 65, 78, 68, 69, 68, 32, 73, + 78, 84, 69, 82, 76, 65, 67, 69, 68, 32, 80, 2, 21, 3, 69, 78, 84, 2, 231, + 134, 10, 65, 30, 90, 65, 30, 67, 94, 68, 58, 77, 58, 82, 182, 1, 83, 74, + 84, 254, 157, 8, 69, 179, 124, 71, 4, 90, 78, 203, 160, 8, 84, 2, 29, 5, + 85, 82, 86, 69, 68, 2, 17, 2, 32, 65, 2, 11, 78, 2, 177, 237, 10, 2, 71, + 76, 4, 248, 254, 2, 5, 79, 85, 66, 76, 69, 131, 177, 6, 73, 2, 129, 168, + 10, 9, 65, 71, 78, 73, 70, 89, 73, 78, 71, 10, 38, 79, 206, 163, 9, 65, + 247, 28, 73, 6, 92, 5, 67, 75, 69, 84, 32, 137, 163, 9, 12, 76, 76, 69, + 82, 32, 67, 79, 65, 83, 84, 69, 82, 4, 228, 43, 3, 66, 79, 79, 179, 189, + 10, 83, 2, 11, 84, 2, 21, 3, 73, 67, 75, 2, 221, 177, 6, 4, 32, 70, 73, + 71, 2, 155, 220, 7, 65, 4, 46, 72, 85, 7, 73, 68, 69, 32, 65, 82, 67, 2, + 17, 2, 65, 68, 2, 17, 2, 69, 68, 2, 173, 156, 10, 6, 32, 87, 72, 73, 84, + 69, 2, 11, 32, 2, 241, 224, 8, 8, 67, 76, 79, 67, 75, 87, 73, 83, 8, 17, + 2, 84, 32, 8, 86, 73, 40, 4, 79, 86, 69, 82, 152, 130, 5, 5, 69, 77, 66, + 69, 68, 251, 246, 5, 77, 2, 17, 2, 83, 79, 2, 131, 135, 1, 76, 2, 247, + 150, 3, 82, 2, 129, 171, 10, 2, 79, 82, 214, 1, 160, 1, 5, 65, 82, 82, + 79, 87, 130, 9, 66, 86, 68, 210, 1, 70, 122, 72, 230, 6, 76, 26, 79, 34, + 80, 50, 82, 70, 83, 94, 84, 206, 8, 87, 190, 182, 8, 67, 47, 81, 75, 26, + 32, 223, 132, 9, 45, 70, 94, 65, 170, 2, 70, 110, 84, 184, 1, 5, 87, 73, + 84, 72, 32, 181, 157, 1, 4, 79, 86, 69, 82, 12, 40, 5, 66, 79, 86, 69, + 32, 143, 1, 78, 10, 70, 82, 140, 161, 1, 5, 83, 72, 79, 82, 84, 130, 193, + 3, 65, 55, 84, 4, 37, 7, 69, 86, 69, 82, 83, 69, 32, 4, 226, 225, 4, 65, + 55, 84, 2, 61, 13, 68, 32, 85, 80, 80, 69, 82, 32, 65, 78, 68, 32, 76, 2, + 17, 2, 79, 87, 2, 169, 198, 8, 2, 69, 82, 6, 25, 4, 82, 79, 77, 32, 6, + 36, 3, 66, 65, 82, 227, 198, 8, 68, 5, 189, 1, 6, 32, 84, 79, 32, 66, 76, + 10, 56, 7, 72, 82, 79, 85, 71, 72, 32, 65, 3, 79, 32, 66, 6, 184, 220, 4, + 3, 83, 85, 80, 198, 197, 4, 71, 155, 146, 2, 88, 4, 26, 76, 227, 250, 10, + 65, 2, 141, 231, 9, 3, 65, 67, 75, 40, 140, 1, 6, 67, 79, 82, 78, 69, 82, + 26, 68, 98, 76, 86, 80, 30, 83, 38, 84, 178, 195, 8, 77, 38, 78, 122, 69, + 226, 151, 1, 72, 143, 163, 1, 86, 2, 133, 19, 2, 32, 68, 4, 11, 79, 4, + 40, 4, 84, 84, 69, 68, 155, 205, 7, 85, 2, 17, 2, 32, 83, 2, 251, 158, + 11, 84, 6, 26, 79, 143, 196, 8, 65, 4, 26, 87, 211, 176, 11, 79, 2, 253, + 196, 3, 2, 69, 82, 2, 237, 130, 9, 2, 76, 85, 6, 186, 196, 8, 77, 251, + 187, 2, 84, 10, 206, 16, 73, 171, 3, 65, 8, 58, 65, 128, 12, 5, 79, 84, + 84, 79, 77, 175, 184, 8, 76, 2, 145, 2, 2, 67, 75, 14, 48, 6, 79, 85, 66, + 76, 69, 32, 207, 210, 8, 65, 12, 40, 5, 65, 82, 82, 79, 87, 135, 19, 68, + 11, 26, 32, 171, 250, 8, 45, 6, 26, 87, 207, 200, 8, 70, 4, 25, 4, 73, + 84, 72, 32, 4, 146, 253, 10, 86, 79, 83, 4, 40, 4, 82, 79, 78, 84, 235, + 195, 8, 73, 2, 233, 194, 8, 14, 45, 84, 73, 76, 84, 69, 68, 32, 83, 72, + 65, 68, 79, 87, 30, 26, 65, 163, 199, 8, 69, 26, 48, 6, 82, 80, 79, 79, + 78, 32, 219, 229, 10, 78, 24, 88, 8, 79, 86, 69, 82, 32, 76, 69, 70, 45, + 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, 32, 2, 157, 132, 8, 6, 84, 87, + 65, 82, 68, 83, 22, 44, 4, 68, 79, 87, 78, 173, 1, 2, 85, 80, 10, 26, 32, + 219, 204, 8, 87, 8, 76, 8, 65, 66, 79, 86, 69, 32, 76, 69, 18, 66, 154, + 196, 8, 70, 179, 5, 84, 2, 227, 2, 70, 2, 177, 218, 4, 7, 69, 76, 79, 87, + 32, 76, 79, 12, 26, 32, 175, 203, 8, 87, 10, 60, 6, 65, 66, 79, 86, 69, + 32, 142, 195, 8, 70, 179, 5, 84, 6, 22, 76, 155, 1, 82, 4, 32, 2, 69, 70, + 239, 216, 4, 79, 2, 253, 139, 2, 24, 84, 87, 65, 82, 68, 83, 32, 72, 65, + 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 2, 21, 3, 73, + 71, 72, 2, 105, 24, 84, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, + 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 2, 11, 32, 2, 179, 236, 1, + 68, 2, 141, 1, 2, 69, 70, 2, 189, 197, 8, 3, 80, 69, 78, 4, 190, 201, 8, + 65, 181, 205, 1, 3, 85, 83, 72, 4, 36, 3, 73, 71, 72, 131, 210, 10, 79, + 2, 11, 84, 2, 171, 1, 45, 6, 32, 2, 81, 85, 203, 192, 8, 65, 4, 26, 73, + 227, 192, 8, 65, 2, 217, 200, 8, 2, 71, 71, 54, 38, 79, 60, 2, 82, 73, + 227, 4, 87, 2, 11, 80, 2, 11, 32, 2, 241, 184, 8, 4, 83, 72, 65, 68, 34, + 40, 5, 65, 78, 71, 76, 69, 255, 3, 80, 30, 56, 8, 45, 72, 69, 65, 68, 69, + 68, 32, 239, 204, 8, 32, 28, 52, 5, 65, 82, 82, 79, 87, 190, 197, 8, 68, + 39, 80, 25, 11, 32, 22, 64, 8, 79, 86, 69, 82, 32, 76, 69, 70, 22, 87, + 251, 192, 8, 84, 2, 171, 192, 8, 84, 18, 25, 4, 73, 84, 72, 32, 18, 128, + 1, 7, 68, 79, 85, 66, 76, 69, 32, 36, 7, 76, 79, 78, 71, 32, 84, 73, 218, + 192, 8, 66, 158, 1, 77, 34, 78, 34, 86, 35, 72, 4, 158, 251, 8, 72, 239, + 243, 1, 86, 4, 11, 80, 4, 11, 32, 4, 18, 68, 35, 85, 2, 169, 193, 8, 3, + 79, 87, 78, 2, 139, 193, 8, 80, 4, 21, 3, 76, 69, 32, 4, 138, 3, 68, 243, + 128, 10, 65, 18, 11, 79, 18, 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 163, + 195, 8, 32, 16, 76, 6, 65, 82, 82, 79, 87, 32, 213, 1, 8, 84, 82, 73, 80, + 76, 69, 32, 68, 14, 44, 5, 87, 73, 84, 72, 32, 167, 183, 8, 70, 12, 42, + 84, 238, 183, 7, 68, 243, 179, 3, 86, 8, 26, 65, 231, 194, 8, 82, 6, 17, + 2, 73, 76, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 150, 183, 7, 68, 243, + 179, 3, 86, 2, 137, 132, 1, 2, 65, 83, 8, 58, 65, 21, 10, 72, 73, 84, 69, + 32, 65, 82, 82, 79, 87, 2, 195, 191, 8, 86, 7, 11, 32, 4, 148, 189, 2, 4, + 70, 82, 79, 77, 147, 135, 6, 87, 19, 66, 32, 136, 1, 6, 69, 68, 32, 80, + 76, 65, 21, 3, 73, 78, 71, 12, 82, 66, 22, 80, 248, 196, 4, 2, 73, 78, + 26, 69, 238, 147, 3, 79, 239, 194, 2, 65, 2, 147, 135, 11, 85, 2, 11, 79, + 2, 171, 213, 10, 73, 2, 155, 227, 10, 78, 2, 253, 194, 3, 2, 32, 66, 4, + 24, 2, 70, 65, 75, 83, 2, 17, 2, 76, 76, 2, 21, 3, 73, 78, 71, 2, 217, + 226, 1, 2, 32, 68, 2, 233, 168, 3, 2, 79, 85, 8, 130, 151, 11, 69, 2, 73, + 2, 77, 3, 79, 126, 136, 2, 2, 67, 75, 20, 2, 76, 76, 188, 2, 4, 77, 65, + 78, 32, 194, 8, 79, 80, 2, 83, 69, 20, 6, 84, 65, 84, 69, 68, 32, 152, 3, + 3, 85, 78, 68, 214, 173, 3, 87, 200, 204, 3, 15, 65, 83, 84, 69, 68, 32, + 83, 87, 69, 69, 84, 32, 80, 79, 84, 205, 164, 2, 2, 66, 79, 5, 159, 248, + 10, 69, 10, 130, 1, 69, 64, 4, 32, 79, 70, 32, 101, 21, 73, 78, 71, 32, + 79, 78, 32, 84, 72, 69, 32, 70, 76, 79, 79, 82, 32, 76, 65, 85, 71, 6, + 60, 9, 68, 45, 85, 80, 32, 78, 69, 87, 83, 33, 2, 82, 32, 2, 11, 80, 2, + 215, 254, 1, 65, 4, 28, 3, 67, 79, 65, 23, 83, 2, 187, 218, 9, 83, 2, + 211, 94, 75, 2, 139, 193, 10, 72, 72, 140, 1, 6, 67, 69, 78, 84, 85, 82, + 22, 68, 100, 3, 81, 85, 73, 28, 8, 78, 85, 77, 69, 82, 65, 76, 32, 182, + 4, 83, 114, 85, 255, 219, 7, 65, 2, 187, 205, 5, 73, 6, 98, 69, 232, 5, + 5, 85, 80, 79, 78, 68, 61, 12, 73, 77, 73, 68, 73, 65, 32, 83, 69, 88, + 84, 85, 2, 229, 5, 3, 78, 65, 82, 48, 142, 1, 70, 136, 1, 3, 79, 78, 69, + 134, 1, 83, 66, 84, 232, 14, 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, + 210, 232, 2, 69, 143, 251, 6, 78, 14, 26, 73, 219, 149, 10, 79, 12, 36, + 3, 70, 84, 89, 187, 250, 2, 86, 7, 11, 32, 4, 222, 186, 5, 84, 185, 212, + 4, 5, 69, 65, 82, 76, 89, 11, 11, 32, 8, 50, 72, 41, 8, 84, 72, 79, 85, + 83, 65, 78, 68, 4, 185, 1, 6, 85, 78, 68, 82, 69, 68, 5, 197, 253, 3, 2, + 32, 67, 6, 32, 2, 73, 88, 171, 156, 9, 69, 5, 149, 141, 10, 2, 32, 76, + 10, 42, 69, 210, 248, 2, 87, 191, 163, 6, 72, 4, 11, 78, 5, 11, 32, 2, + 159, 184, 5, 84, 10, 46, 69, 181, 185, 7, 5, 73, 76, 73, 81, 85, 8, 60, + 2, 77, 85, 40, 5, 83, 84, 69, 82, 84, 21, 2, 88, 84, 2, 17, 2, 78, 67, 2, + 223, 184, 7, 73, 2, 199, 219, 7, 73, 4, 18, 65, 23, 85, 2, 171, 219, 7, + 78, 2, 143, 184, 7, 76, 4, 48, 6, 84, 32, 86, 69, 71, 69, 147, 209, 9, + 83, 2, 201, 192, 2, 2, 84, 65, 5, 243, 193, 9, 84, 10, 166, 2, 70, 32, 11, 72, 69, 65, 86, 89, 32, 66, 76, 65, 67, 75, 52, 24, 76, 73, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 32, 66, 76, 65, 67, 75, 0, 18, 87, 72, 73, 84, 69, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 169, 53, 8, 67, 65, 80, 73, 84, 65, 76, 32, 2, 29, 5, 76, - 79, 82, 65, 76, 2, 217, 236, 8, 8, 32, 72, 69, 65, 82, 84, 32, 66, 2, - 225, 132, 9, 2, 32, 67, 16, 74, 32, 89, 14, 69, 68, 32, 83, 89, 77, 66, - 79, 76, 32, 70, 79, 82, 32, 4, 24, 2, 80, 85, 43, 84, 2, 11, 83, 2, 141, - 200, 9, 2, 72, 80, 2, 175, 210, 3, 65, 12, 68, 2, 83, 72, 242, 135, 6, - 67, 254, 153, 4, 70, 2, 76, 147, 17, 88, 4, 40, 4, 85, 65, 78, 71, 195, - 161, 10, 79, 2, 207, 178, 10, 88, 136, 2, 226, 1, 66, 20, 3, 71, 66, 89, + 79, 82, 65, 76, 2, 213, 166, 9, 8, 32, 72, 69, 65, 82, 84, 32, 66, 2, + 149, 191, 9, 2, 32, 67, 16, 74, 32, 89, 14, 69, 68, 32, 83, 89, 77, 66, + 79, 76, 32, 70, 79, 82, 32, 4, 24, 2, 80, 85, 43, 84, 2, 11, 83, 2, 245, + 132, 10, 2, 72, 80, 2, 203, 217, 3, 65, 12, 68, 2, 83, 72, 230, 152, 6, + 67, 138, 198, 4, 70, 2, 76, 147, 17, 88, 4, 40, 4, 85, 65, 78, 71, 195, + 222, 10, 79, 2, 207, 239, 10, 88, 136, 2, 226, 1, 66, 20, 3, 71, 66, 89, 40, 5, 76, 69, 45, 68, 69, 40, 3, 77, 73, 32, 154, 5, 78, 156, 26, 26, 83, 83, 73, 65, 78, 32, 65, 83, 84, 82, 79, 76, 79, 71, 73, 67, 65, 76, - 32, 83, 89, 77, 66, 79, 76, 32, 247, 252, 4, 80, 2, 211, 172, 8, 76, 2, - 205, 191, 8, 5, 32, 70, 79, 79, 84, 2, 11, 76, 2, 161, 228, 5, 2, 65, 89, - 62, 68, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 102, 78, 195, 190, 2, 68, - 8, 40, 4, 79, 78, 69, 32, 203, 229, 8, 84, 6, 34, 84, 150, 194, 8, 72, - 43, 81, 2, 191, 195, 8, 72, 36, 33, 6, 85, 77, 66, 69, 82, 32, 36, 76, 5, + 32, 83, 89, 77, 66, 79, 76, 32, 231, 142, 5, 80, 2, 255, 228, 8, 76, 2, + 189, 249, 8, 5, 32, 70, 79, 79, 84, 2, 11, 76, 2, 149, 245, 5, 2, 65, 89, + 62, 68, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 102, 78, 231, 193, 2, 68, + 8, 40, 4, 79, 78, 69, 32, 199, 159, 9, 84, 6, 34, 84, 134, 252, 8, 72, + 47, 81, 2, 179, 253, 8, 72, 36, 33, 6, 85, 77, 66, 69, 82, 32, 36, 76, 5, 69, 73, 71, 72, 84, 38, 70, 92, 2, 78, 73, 22, 79, 18, 83, 83, 84, 4, - 158, 129, 3, 32, 139, 192, 7, 89, 8, 18, 73, 35, 79, 4, 130, 2, 86, 251, - 212, 8, 70, 4, 136, 2, 2, 85, 82, 211, 212, 8, 82, 4, 77, 2, 78, 69, 2, - 167, 1, 78, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 206, 255, 2, 32, - 195, 174, 7, 84, 10, 34, 72, 50, 87, 143, 239, 9, 69, 4, 32, 2, 82, 69, - 215, 212, 8, 73, 2, 39, 69, 4, 26, 79, 199, 212, 8, 69, 2, 187, 254, 2, + 202, 132, 3, 32, 223, 249, 7, 89, 8, 18, 73, 35, 79, 4, 130, 2, 86, 247, + 142, 9, 70, 4, 136, 2, 2, 85, 82, 207, 142, 9, 82, 4, 77, 2, 78, 69, 2, + 167, 1, 78, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 250, 130, 3, 32, + 151, 232, 7, 84, 10, 34, 72, 50, 87, 131, 172, 10, 69, 4, 32, 2, 82, 69, + 211, 142, 9, 73, 2, 39, 69, 4, 26, 79, 195, 142, 9, 69, 2, 231, 129, 3, 32, 182, 1, 32, 3, 73, 67, 32, 147, 25, 78, 178, 1, 220, 1, 6, 66, 69, 76, 71, 84, 72, 20, 4, 67, 82, 79, 83, 20, 7, 76, 69, 84, 84, 69, 82, 32, - 210, 22, 83, 24, 6, 77, 85, 76, 84, 73, 80, 188, 141, 7, 5, 65, 82, 76, - 65, 85, 205, 183, 1, 7, 84, 86, 73, 77, 65, 68, 85, 2, 151, 220, 8, 79, - 2, 167, 143, 7, 83, 166, 1, 202, 4, 65, 110, 67, 98, 68, 126, 69, 82, 70, + 210, 22, 83, 24, 6, 77, 85, 76, 84, 73, 80, 208, 181, 7, 5, 65, 82, 76, + 65, 85, 181, 201, 1, 7, 84, 86, 73, 77, 65, 68, 85, 2, 147, 150, 9, 79, + 2, 227, 182, 7, 83, 166, 1, 202, 4, 65, 110, 67, 98, 68, 126, 69, 82, 70, 222, 1, 71, 104, 2, 72, 65, 50, 73, 220, 1, 5, 74, 69, 82, 65, 78, 34, 75, 58, 76, 234, 1, 79, 128, 1, 13, 82, 65, 73, 68, 79, 32, 82, 65, 68, 32, 82, 69, 73, 34, 83, 176, 2, 16, 66, 69, 82, 75, 65, 78, 65, 78, 32, 66, 69, 79, 82, 67, 32, 66, 144, 1, 12, 78, 65, 85, 68, 73, 90, 32, 78, - 89, 68, 32, 78, 110, 84, 194, 1, 87, 244, 83, 7, 85, 82, 85, 90, 32, 85, - 82, 208, 187, 2, 10, 77, 65, 78, 78, 65, 90, 32, 77, 65, 78, 184, 60, 13, - 80, 69, 82, 84, 72, 79, 32, 80, 69, 79, 82, 84, 72, 218, 246, 2, 89, 166, - 227, 3, 81, 2, 86, 2, 88, 3, 90, 8, 222, 4, 69, 202, 158, 6, 67, 0, 4, - 78, 83, 85, 90, 249, 135, 3, 9, 76, 71, 73, 90, 32, 69, 79, 76, 72, 11, - 46, 69, 30, 65, 197, 239, 6, 3, 87, 69, 79, 4, 26, 65, 247, 181, 10, 78, - 2, 147, 143, 9, 76, 11, 84, 6, 79, 84, 84, 69, 68, 45, 209, 219, 3, 9, - 65, 71, 65, 90, 32, 68, 65, 69, 71, 6, 134, 181, 10, 76, 2, 78, 3, 80, - 11, 228, 70, 7, 72, 87, 65, 90, 32, 69, 72, 174, 181, 9, 65, 194, 55, 84, + 89, 68, 32, 78, 110, 84, 194, 1, 87, 176, 87, 7, 85, 82, 85, 90, 32, 85, + 82, 192, 188, 2, 10, 77, 65, 78, 78, 65, 90, 32, 77, 65, 78, 180, 63, 13, + 80, 69, 82, 84, 72, 79, 32, 80, 69, 79, 82, 84, 72, 142, 189, 3, 89, 202, + 210, 3, 81, 2, 86, 2, 88, 3, 90, 8, 222, 4, 69, 174, 175, 6, 67, 0, 4, + 78, 83, 85, 90, 253, 179, 3, 9, 76, 71, 73, 90, 32, 69, 79, 76, 72, 11, + 46, 69, 30, 65, 145, 138, 7, 3, 87, 69, 79, 4, 26, 65, 247, 242, 10, 78, + 2, 247, 202, 9, 76, 11, 84, 6, 79, 84, 84, 69, 68, 45, 249, 226, 3, 9, + 65, 71, 65, 90, 32, 68, 65, 69, 71, 6, 134, 242, 10, 76, 2, 78, 3, 80, + 11, 240, 74, 7, 72, 87, 65, 90, 32, 69, 72, 150, 238, 9, 65, 206, 55, 84, 63, 78, 12, 120, 13, 82, 65, 78, 75, 83, 32, 67, 65, 83, 75, 69, 84, 32, - 233, 253, 6, 11, 69, 72, 85, 32, 70, 69, 79, 72, 32, 70, 69, 10, 46, 65, - 154, 245, 9, 73, 2, 79, 195, 60, 69, 4, 26, 69, 207, 178, 10, 67, 2, 235, - 139, 9, 83, 9, 26, 69, 207, 249, 9, 65, 4, 52, 7, 66, 79, 32, 71, 89, 70, - 85, 231, 177, 10, 82, 2, 143, 177, 10, 32, 4, 236, 8, 2, 69, 71, 13, 4, - 71, 76, 65, 90, 12, 156, 1, 2, 78, 71, 20, 9, 83, 65, 90, 32, 73, 83, 32, - 73, 83, 20, 5, 87, 65, 90, 32, 69, 136, 247, 9, 10, 67, 69, 76, 65, 78, - 68, 73, 67, 45, 89, 3, 79, 5, 131, 185, 6, 87, 2, 231, 239, 9, 83, 2, - 199, 174, 10, 79, 2, 11, 32, 2, 183, 175, 10, 74, 7, 21, 3, 65, 85, 78, - 4, 242, 171, 10, 32, 155, 3, 65, 12, 120, 15, 65, 85, 75, 65, 90, 32, 76, - 65, 71, 85, 32, 76, 79, 71, 82, 21, 11, 79, 78, 71, 45, 66, 82, 65, 78, - 67, 72, 45, 2, 211, 164, 9, 32, 10, 64, 3, 65, 82, 32, 158, 4, 72, 62, - 77, 66, 79, 179, 239, 9, 89, 2, 195, 150, 10, 65, 15, 150, 5, 83, 0, 12, - 84, 72, 65, 76, 65, 78, 32, 69, 84, 72, 69, 76, 224, 167, 10, 4, 80, 69, - 78, 45, 14, 69, 2, 78, 3, 79, 2, 11, 68, 2, 167, 243, 9, 32, 26, 150, 1, - 72, 244, 2, 18, 73, 71, 69, 76, 32, 76, 79, 78, 71, 45, 66, 82, 65, 78, - 67, 72, 45, 83, 228, 212, 6, 5, 79, 87, 73, 76, 79, 195, 139, 2, 84, 21, - 45, 9, 79, 82, 84, 45, 84, 87, 73, 71, 45, 18, 102, 66, 58, 72, 62, 77, - 30, 78, 38, 79, 42, 83, 186, 1, 84, 156, 146, 6, 2, 65, 82, 183, 219, 3, - 89, 2, 33, 6, 74, 65, 82, 75, 65, 78, 2, 235, 238, 8, 32, 2, 25, 4, 65, - 71, 65, 76, 2, 11, 76, 2, 195, 167, 10, 32, 2, 253, 145, 3, 2, 65, 68, 2, - 205, 216, 9, 4, 65, 85, 68, 32, 2, 17, 2, 83, 83, 2, 247, 136, 10, 32, 2, - 11, 79, 2, 215, 212, 6, 76, 4, 116, 15, 72, 85, 82, 73, 83, 65, 90, 32, - 84, 72, 85, 82, 83, 32, 84, 33, 10, 73, 87, 65, 90, 32, 84, 73, 82, 32, - 84, 2, 11, 72, 2, 167, 200, 5, 79, 2, 17, 2, 89, 82, 2, 223, 137, 10, 32, - 5, 41, 8, 85, 78, 74, 79, 32, 87, 89, 78, 2, 11, 78, 2, 175, 167, 9, 32, - 2, 21, 3, 73, 78, 71, 2, 169, 248, 6, 2, 76, 69, 4, 204, 191, 8, 16, 73, - 78, 71, 32, 83, 72, 73, 82, 84, 32, 87, 73, 84, 72, 32, 83, 219, 172, 1, - 69, 12, 120, 3, 66, 73, 78, 2, 78, 28, 2, 81, 85, 0, 3, 86, 73, 71, 134, - 234, 6, 83, 249, 154, 1, 6, 84, 82, 69, 68, 69, 67, 2, 149, 133, 8, 2, - 79, 86, 2, 237, 132, 8, 2, 73, 78, 214, 37, 244, 1, 2, 32, 73, 22, 65, - 238, 25, 67, 138, 5, 69, 146, 7, 72, 186, 33, 73, 186, 233, 1, 75, 154, - 2, 76, 130, 9, 77, 170, 21, 78, 174, 2, 79, 142, 39, 80, 180, 11, 2, 81, - 85, 158, 67, 83, 38, 84, 142, 16, 85, 150, 35, 87, 186, 1, 89, 187, 251, - 4, 71, 2, 131, 253, 8, 78, 196, 2, 140, 2, 5, 70, 69, 84, 89, 32, 36, 4, - 71, 73, 84, 84, 30, 76, 112, 8, 77, 65, 82, 73, 84, 65, 78, 32, 142, 14, - 78, 190, 3, 84, 100, 2, 85, 82, 132, 236, 2, 2, 73, 76, 198, 87, 88, 180, - 209, 5, 15, 75, 69, 32, 66, 79, 84, 84, 76, 69, 32, 65, 78, 68, 32, 67, - 143, 98, 82, 4, 234, 250, 1, 86, 151, 164, 7, 80, 2, 205, 201, 3, 2, 65, - 82, 6, 18, 84, 75, 85, 4, 36, 3, 32, 83, 72, 163, 253, 8, 73, 2, 11, 65, - 2, 211, 223, 9, 75, 2, 151, 149, 9, 84, 122, 184, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 198, 3, 77, 248, 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, - 79, 78, 32, 132, 4, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 155, - 137, 4, 65, 44, 202, 1, 66, 32, 2, 68, 65, 22, 73, 38, 75, 22, 76, 34, - 83, 46, 84, 182, 2, 65, 208, 204, 5, 2, 71, 65, 134, 139, 1, 82, 210, - 237, 1, 90, 190, 78, 77, 172, 1, 2, 81, 85, 118, 78, 218, 6, 89, 235, - 101, 70, 4, 214, 253, 9, 73, 247, 25, 65, 2, 147, 134, 9, 76, 6, 206, - 153, 10, 78, 2, 84, 3, 89, 2, 147, 152, 9, 65, 2, 11, 65, 2, 187, 133, 9, - 66, 4, 152, 7, 3, 73, 78, 71, 171, 202, 8, 72, 6, 178, 151, 9, 65, 238, - 100, 73, 229, 10, 5, 83, 65, 65, 68, 73, 18, 96, 4, 65, 82, 75, 32, 165, - 1, 15, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, 12, - 82, 68, 40, 2, 73, 78, 90, 69, 242, 2, 78, 173, 249, 7, 5, 79, 67, 67, - 76, 85, 2, 17, 2, 65, 71, 2, 163, 177, 8, 69, 5, 17, 2, 45, 65, 2, 255, - 148, 9, 76, 6, 46, 69, 180, 6, 2, 83, 72, 163, 143, 10, 73, 2, 145, 156, - 9, 11, 80, 69, 78, 84, 72, 69, 84, 73, 67, 32, 89, 28, 130, 1, 65, 154, - 1, 66, 22, 78, 30, 83, 130, 1, 90, 238, 155, 3, 84, 200, 208, 6, 9, 77, - 69, 76, 79, 68, 73, 67, 32, 81, 3, 81, 10, 76, 3, 70, 83, 65, 34, 78, 28, - 2, 84, 77, 145, 238, 9, 4, 82, 75, 65, 65, 2, 11, 65, 2, 179, 147, 10, - 81, 4, 26, 78, 199, 179, 5, 71, 2, 11, 65, 2, 143, 238, 9, 65, 2, 177, - 132, 3, 2, 69, 81, 4, 64, 4, 72, 73, 89, 89, 41, 8, 79, 70, 32, 77, 65, - 83, 72, 70, 2, 17, 2, 65, 65, 2, 167, 162, 8, 76, 2, 139, 254, 8, 65, 4, - 34, 65, 221, 161, 8, 2, 73, 81, 2, 151, 144, 9, 69, 30, 92, 5, 76, 79, - 78, 71, 32, 54, 79, 66, 83, 146, 206, 4, 65, 174, 193, 5, 69, 2, 73, 3, - 85, 10, 130, 207, 4, 65, 174, 193, 5, 69, 2, 73, 3, 85, 7, 41, 8, 86, 69, - 82, 76, 79, 78, 71, 32, 4, 163, 206, 4, 65, 4, 26, 72, 183, 192, 5, 85, - 2, 153, 251, 5, 3, 79, 82, 84, 10, 68, 8, 83, 45, 83, 69, 82, 73, 70, 32, - 145, 236, 9, 3, 68, 87, 73, 8, 44, 6, 72, 69, 65, 86, 89, 32, 139, 2, 73, - 6, 48, 3, 68, 79, 85, 81, 5, 76, 79, 87, 32, 68, 4, 11, 66, 4, 21, 3, 76, - 69, 32, 4, 84, 6, 84, 85, 82, 78, 69, 68, 23, 67, 2, 21, 3, 79, 85, 66, - 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 33, 6, 79, 77, 77, 65, 32, 81, 2, - 153, 197, 8, 3, 85, 79, 84, 2, 169, 200, 9, 10, 78, 84, 69, 82, 82, 79, - 66, 65, 78, 71, 6, 48, 6, 69, 76, 76, 73, 84, 69, 135, 173, 5, 85, 5, 25, - 4, 32, 65, 78, 84, 2, 251, 232, 6, 69, 166, 1, 52, 7, 65, 83, 72, 84, 82, - 65, 32, 163, 169, 4, 79, 164, 1, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, - 212, 1, 5, 83, 73, 71, 78, 32, 186, 17, 68, 156, 244, 2, 16, 67, 79, 78, - 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 72, 251, 138, 2, 86, 100, - 150, 160, 6, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, 158, 144, 1, 79, - 182, 56, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, - 74, 2, 75, 2, 80, 162, 7, 69, 222, 61, 72, 2, 77, 2, 82, 3, 89, 8, 234, - 160, 6, 67, 190, 161, 3, 65, 239, 1, 86, 52, 66, 65, 32, 4, 72, 79, 79, - 76, 46, 79, 74, 82, 143, 133, 10, 73, 4, 190, 242, 8, 76, 215, 18, 82, 5, - 165, 165, 5, 6, 32, 83, 65, 84, 67, 72, 6, 36, 3, 82, 80, 73, 227, 206, - 8, 79, 4, 214, 181, 9, 79, 135, 18, 85, 36, 66, 69, 72, 4, 73, 80, 84, - 32, 166, 243, 1, 85, 175, 140, 6, 79, 4, 36, 3, 87, 68, 82, 207, 180, 9, - 69, 2, 11, 73, 2, 179, 198, 9, 86, 28, 80, 8, 67, 65, 80, 73, 84, 65, 76, - 32, 86, 76, 81, 6, 83, 77, 65, 76, 76, 32, 18, 170, 131, 10, 66, 2, 69, - 2, 70, 2, 72, 2, 73, 2, 76, 2, 77, 2, 80, 3, 82, 2, 37, 7, 73, 71, 65, - 84, 85, 82, 69, 2, 11, 32, 2, 205, 190, 9, 2, 69, 84, 8, 134, 130, 10, - 69, 2, 71, 2, 76, 3, 79, 64, 250, 1, 65, 30, 67, 74, 69, 36, 5, 71, 77, - 69, 78, 84, 28, 2, 77, 73, 174, 1, 82, 158, 1, 83, 68, 2, 84, 32, 200, - 245, 4, 8, 87, 73, 78, 71, 32, 78, 69, 69, 172, 237, 1, 6, 80, 65, 82, - 65, 84, 69, 178, 122, 88, 202, 99, 68, 149, 159, 1, 2, 76, 70, 4, 222, - 255, 9, 76, 3, 84, 6, 34, 84, 157, 200, 5, 2, 79, 78, 4, 194, 165, 6, 73, - 143, 161, 3, 79, 4, 238, 216, 1, 68, 223, 131, 1, 45, 23, 181, 169, 4, 2, - 69, 68, 6, 128, 77, 28, 68, 73, 82, 69, 67, 84, 32, 80, 82, 79, 68, 85, - 67, 84, 32, 87, 73, 84, 72, 32, 66, 79, 84, 84, 79, 77, 32, 67, 172, 147, - 7, 3, 83, 69, 88, 151, 206, 1, 67, 4, 120, 2, 86, 73, 217, 171, 2, 22, - 73, 79, 85, 83, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 83, 89, 77, - 66, 79, 76, 83, 2, 11, 67, 2, 207, 186, 9, 69, 4, 52, 7, 81, 85, 73, 81, - 85, 65, 68, 155, 198, 8, 65, 2, 91, 82, 4, 64, 10, 84, 82, 65, 78, 83, - 77, 73, 84, 32, 83, 147, 158, 6, 77, 2, 11, 84, 2, 179, 182, 8, 65, 240, - 2, 102, 65, 254, 19, 73, 102, 79, 250, 10, 69, 70, 82, 184, 202, 8, 5, - 85, 70, 70, 76, 69, 143, 143, 1, 89, 176, 2, 164, 1, 12, 68, 79, 87, 69, - 68, 32, 87, 72, 73, 84, 69, 32, 68, 9, 76, 76, 79, 87, 32, 80, 65, 78, - 32, 62, 82, 206, 9, 86, 172, 133, 7, 2, 77, 82, 183, 224, 1, 75, 6, 38, - 76, 134, 169, 8, 67, 167, 11, 83, 2, 249, 185, 8, 2, 65, 84, 2, 17, 2, - 79, 70, 2, 17, 2, 32, 70, 2, 227, 213, 6, 79, 194, 1, 40, 4, 65, 68, 65, - 32, 135, 247, 9, 75, 192, 1, 162, 1, 68, 42, 69, 110, 72, 34, 76, 246, 1, - 83, 208, 2, 6, 86, 79, 87, 69, 76, 32, 226, 175, 4, 65, 196, 213, 1, 7, - 67, 79, 78, 84, 73, 78, 85, 235, 217, 3, 79, 24, 158, 32, 79, 66, 65, - 239, 230, 7, 73, 4, 84, 15, 88, 84, 82, 65, 32, 83, 72, 79, 82, 84, 32, - 86, 79, 87, 69, 163, 187, 8, 75, 2, 163, 179, 9, 76, 2, 189, 196, 9, 3, - 69, 65, 68, 96, 33, 6, 69, 84, 84, 69, 82, 32, 96, 178, 139, 6, 65, 38, - 68, 46, 84, 46, 86, 186, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, - 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, - 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 30, 70, 65, 38, 69, 56, 4, 73, 71, - 78, 32, 141, 236, 8, 3, 85, 84, 82, 2, 177, 176, 9, 4, 78, 68, 72, 73, 6, - 132, 202, 2, 5, 67, 84, 73, 79, 78, 231, 142, 4, 80, 20, 102, 73, 182, - 249, 4, 83, 134, 144, 1, 65, 74, 67, 98, 78, 230, 179, 1, 74, 150, 3, 85, - 211, 235, 1, 86, 2, 37, 7, 78, 86, 69, 82, 84, 69, 68, 2, 213, 137, 6, 2, - 32, 67, 30, 60, 6, 77, 79, 68, 73, 70, 73, 21, 5, 83, 73, 71, 78, 32, 2, - 227, 252, 5, 69, 28, 86, 80, 226, 137, 6, 65, 106, 86, 214, 20, 85, 210, - 200, 1, 73, 206, 134, 2, 69, 3, 79, 2, 57, 12, 82, 73, 83, 72, 84, 72, - 65, 77, 65, 84, 82, 65, 2, 167, 215, 9, 32, 98, 72, 3, 69, 68, 32, 21, - 11, 73, 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 2, 171, 176, 9, 73, 96, - 158, 2, 65, 120, 3, 67, 72, 85, 22, 69, 70, 72, 46, 73, 46, 76, 22, 77, - 38, 79, 94, 80, 18, 82, 22, 83, 38, 84, 64, 2, 87, 79, 36, 2, 89, 69, - 208, 221, 4, 2, 74, 85, 162, 186, 2, 68, 202, 13, 90, 150, 84, 70, 182, - 6, 71, 210, 43, 66, 218, 11, 75, 206, 20, 86, 246, 25, 78, 143, 128, 1, - 85, 16, 82, 82, 198, 177, 9, 73, 222, 25, 68, 162, 8, 71, 2, 87, 198, 21, - 83, 147, 1, 72, 4, 166, 222, 8, 82, 239, 139, 1, 69, 2, 199, 199, 9, 82, - 8, 38, 65, 230, 176, 9, 82, 255, 55, 71, 4, 178, 233, 9, 82, 3, 84, 4, - 216, 241, 7, 2, 65, 45, 211, 166, 1, 85, 6, 150, 153, 9, 65, 130, 57, 67, - 215, 22, 70, 2, 143, 228, 7, 79, 4, 178, 136, 5, 69, 167, 200, 3, 73, 12, - 70, 79, 166, 222, 8, 73, 242, 108, 85, 150, 25, 65, 154, 3, 78, 3, 82, 2, - 235, 208, 9, 90, 2, 163, 13, 69, 2, 243, 196, 8, 79, 4, 230, 198, 8, 85, - 163, 160, 1, 79, 6, 26, 72, 159, 202, 9, 79, 4, 186, 238, 5, 73, 199, - 230, 3, 69, 4, 134, 221, 8, 79, 159, 137, 1, 69, 4, 254, 229, 9, 65, 3, - 87, 10, 78, 69, 222, 214, 3, 70, 192, 121, 6, 78, 84, 79, 32, 83, 72, - 251, 148, 5, 80, 2, 255, 158, 9, 76, 44, 236, 1, 2, 79, 84, 32, 6, 80, - 80, 73, 78, 71, 32, 72, 2, 82, 84, 180, 8, 7, 85, 76, 68, 69, 82, 69, 68, - 208, 240, 1, 24, 67, 75, 69, 68, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, - 32, 69, 88, 80, 76, 79, 68, 73, 78, 71, 199, 171, 7, 87, 2, 145, 208, 7, - 3, 73, 78, 71, 4, 36, 3, 84, 82, 79, 239, 220, 5, 66, 2, 11, 76, 2, 247, - 203, 7, 76, 32, 102, 32, 248, 5, 12, 72, 65, 78, 68, 32, 70, 79, 82, 77, - 65, 84, 32, 206, 149, 6, 67, 255, 197, 3, 83, 20, 154, 2, 66, 92, 11, 83, - 76, 65, 78, 84, 69, 68, 32, 78, 79, 82, 196, 1, 22, 82, 73, 71, 72, 84, - 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 65, 66, 79, 86, 69, 28, - 7, 85, 80, 32, 84, 65, 67, 75, 128, 1, 4, 76, 69, 70, 84, 133, 160, 2, 9, - 68, 79, 87, 78, 32, 84, 65, 67, 75, 4, 88, 14, 65, 67, 75, 83, 76, 65, - 78, 84, 69, 68, 32, 83, 79, 85, 33, 4, 69, 78, 84, 32, 2, 11, 84, 2, 255, - 195, 8, 72, 2, 213, 37, 37, 65, 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, + 137, 165, 7, 11, 69, 72, 85, 32, 70, 69, 79, 72, 32, 70, 69, 10, 46, 65, + 142, 178, 10, 73, 2, 79, 207, 60, 69, 4, 26, 69, 207, 239, 10, 67, 2, + 207, 199, 9, 83, 9, 26, 69, 195, 182, 10, 65, 4, 52, 7, 66, 79, 32, 71, + 89, 70, 85, 231, 238, 10, 82, 2, 143, 238, 10, 32, 4, 236, 8, 2, 69, 71, + 13, 4, 71, 76, 65, 90, 12, 156, 1, 2, 78, 71, 20, 9, 83, 65, 90, 32, 73, + 83, 32, 73, 83, 20, 5, 87, 65, 90, 32, 69, 252, 179, 10, 10, 67, 69, 76, + 65, 78, 68, 73, 67, 45, 89, 3, 79, 5, 199, 201, 6, 87, 2, 219, 172, 10, + 83, 2, 199, 235, 10, 79, 2, 11, 32, 2, 183, 236, 10, 74, 7, 21, 3, 65, + 85, 78, 4, 242, 232, 10, 32, 155, 3, 65, 12, 120, 15, 65, 85, 75, 65, 90, + 32, 76, 65, 71, 85, 32, 76, 79, 71, 82, 21, 11, 79, 78, 71, 45, 66, 82, + 65, 78, 67, 72, 45, 2, 187, 225, 9, 32, 10, 64, 3, 65, 82, 32, 158, 4, + 72, 62, 77, 66, 79, 167, 172, 10, 89, 2, 195, 211, 10, 65, 15, 150, 5, + 83, 0, 12, 84, 72, 65, 76, 65, 78, 32, 69, 84, 72, 69, 76, 224, 228, 10, + 4, 80, 69, 78, 45, 14, 69, 2, 78, 3, 79, 2, 11, 68, 2, 155, 176, 10, 32, + 26, 150, 1, 72, 244, 2, 18, 73, 71, 69, 76, 32, 76, 79, 78, 71, 45, 66, + 82, 65, 78, 67, 72, 45, 83, 200, 239, 6, 5, 79, 87, 73, 76, 79, 147, 171, + 2, 84, 21, 45, 9, 79, 82, 84, 45, 84, 87, 73, 71, 45, 18, 102, 66, 58, + 72, 62, 77, 30, 78, 38, 79, 42, 83, 186, 1, 84, 128, 163, 6, 2, 65, 82, + 199, 135, 4, 89, 2, 33, 6, 74, 65, 82, 75, 65, 78, 2, 171, 170, 9, 32, 2, + 25, 4, 65, 71, 65, 76, 2, 11, 76, 2, 195, 228, 10, 32, 2, 169, 150, 3, 2, + 65, 68, 2, 193, 149, 10, 4, 65, 85, 68, 32, 2, 17, 2, 83, 83, 2, 247, + 197, 10, 32, 2, 11, 79, 2, 187, 239, 6, 76, 4, 116, 15, 72, 85, 82, 73, + 83, 65, 90, 32, 84, 72, 85, 82, 83, 32, 84, 33, 10, 73, 87, 65, 90, 32, + 84, 73, 82, 32, 84, 2, 11, 72, 2, 155, 217, 5, 79, 2, 17, 2, 89, 82, 2, + 223, 198, 10, 32, 5, 41, 8, 85, 78, 74, 79, 32, 87, 89, 78, 2, 11, 78, 2, + 159, 228, 9, 32, 2, 21, 3, 73, 78, 71, 2, 229, 159, 7, 2, 76, 69, 4, 200, + 249, 8, 16, 73, 78, 71, 32, 83, 72, 73, 82, 84, 32, 87, 73, 84, 72, 32, + 83, 211, 175, 1, 69, 12, 120, 3, 66, 73, 78, 2, 78, 28, 2, 81, 85, 0, 3, + 86, 73, 71, 166, 145, 7, 83, 229, 169, 1, 6, 84, 82, 69, 68, 69, 67, 2, + 161, 187, 8, 2, 79, 86, 2, 249, 186, 8, 2, 73, 78, 250, 39, 244, 1, 2, + 32, 73, 22, 65, 238, 25, 67, 138, 5, 69, 166, 11, 72, 190, 33, 73, 198, + 232, 1, 75, 154, 2, 76, 142, 9, 77, 166, 21, 78, 174, 2, 79, 158, 39, 80, + 172, 12, 2, 81, 85, 146, 70, 83, 38, 84, 250, 16, 85, 166, 43, 87, 186, + 1, 89, 251, 166, 5, 71, 2, 239, 184, 9, 78, 196, 2, 140, 2, 5, 70, 69, + 84, 89, 32, 36, 4, 71, 73, 84, 84, 30, 76, 112, 8, 77, 65, 82, 73, 84, + 65, 78, 32, 142, 14, 78, 190, 3, 84, 100, 2, 85, 82, 244, 239, 2, 2, 73, + 76, 234, 91, 88, 144, 134, 6, 15, 75, 69, 32, 66, 79, 84, 84, 76, 69, 32, + 65, 78, 68, 32, 67, 159, 98, 82, 4, 138, 254, 1, 86, 223, 221, 7, 80, 2, + 245, 208, 3, 2, 65, 82, 6, 18, 84, 75, 85, 4, 36, 3, 32, 83, 72, 139, + 186, 9, 73, 2, 11, 65, 2, 199, 156, 10, 75, 2, 255, 209, 9, 84, 122, 184, + 1, 7, 76, 69, 84, 84, 69, 82, 32, 198, 3, 77, 248, 2, 12, 80, 85, 78, 67, + 84, 85, 65, 84, 73, 79, 78, 32, 132, 4, 11, 86, 79, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 203, 155, 4, 65, 44, 202, 1, 66, 32, 2, 68, 65, 22, 73, + 38, 75, 22, 76, 34, 83, 46, 84, 182, 2, 65, 196, 221, 5, 2, 71, 65, 222, + 148, 1, 82, 146, 141, 2, 90, 154, 81, 77, 172, 1, 2, 81, 85, 118, 78, + 226, 6, 89, 251, 101, 70, 4, 214, 186, 10, 73, 247, 25, 65, 2, 251, 194, + 9, 76, 6, 206, 214, 10, 78, 2, 84, 3, 89, 2, 251, 212, 9, 65, 2, 11, 65, + 2, 163, 194, 9, 66, 4, 152, 7, 3, 73, 78, 71, 223, 132, 9, 72, 6, 154, + 212, 9, 65, 134, 101, 73, 229, 10, 5, 83, 65, 65, 68, 73, 18, 96, 4, 65, + 82, 75, 32, 165, 1, 15, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, + 69, 82, 32, 12, 82, 68, 40, 2, 73, 78, 90, 69, 242, 2, 78, 197, 176, 8, + 5, 79, 67, 67, 76, 85, 2, 17, 2, 65, 71, 2, 159, 235, 8, 69, 5, 17, 2, + 45, 65, 2, 231, 209, 9, 76, 6, 46, 69, 180, 6, 2, 83, 72, 163, 204, 10, + 73, 2, 129, 217, 9, 11, 80, 69, 78, 84, 72, 69, 84, 73, 67, 32, 89, 28, + 130, 1, 65, 154, 1, 66, 22, 78, 30, 83, 130, 1, 90, 174, 162, 3, 84, 136, + 135, 7, 9, 77, 69, 76, 79, 68, 73, 67, 32, 81, 3, 81, 10, 76, 3, 70, 83, + 65, 34, 78, 28, 2, 84, 77, 145, 171, 10, 4, 82, 75, 65, 65, 2, 11, 65, 2, + 179, 208, 10, 81, 4, 26, 78, 187, 196, 5, 71, 2, 11, 65, 2, 143, 171, 10, + 65, 2, 201, 138, 3, 2, 69, 81, 4, 64, 4, 72, 73, 89, 89, 41, 8, 79, 70, + 32, 77, 65, 83, 72, 70, 2, 17, 2, 65, 65, 2, 163, 220, 8, 76, 2, 243, + 186, 9, 65, 4, 34, 65, 217, 219, 8, 2, 73, 81, 2, 255, 204, 9, 69, 30, + 92, 5, 76, 79, 78, 71, 32, 54, 79, 66, 83, 186, 224, 4, 65, 134, 236, 5, + 69, 2, 73, 3, 85, 10, 170, 225, 4, 65, 134, 236, 5, 69, 2, 73, 3, 85, 7, + 41, 8, 86, 69, 82, 76, 79, 78, 71, 32, 4, 203, 224, 4, 65, 4, 26, 72, + 171, 209, 5, 85, 2, 253, 139, 6, 3, 79, 82, 84, 10, 68, 8, 83, 45, 83, + 69, 82, 73, 70, 32, 145, 169, 10, 3, 68, 87, 73, 8, 44, 6, 72, 69, 65, + 86, 89, 32, 139, 2, 73, 6, 48, 3, 68, 79, 85, 81, 5, 76, 79, 87, 32, 68, + 4, 11, 66, 4, 21, 3, 76, 69, 32, 4, 84, 6, 84, 85, 82, 78, 69, 68, 23, + 67, 2, 21, 3, 79, 85, 66, 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 33, 6, + 79, 77, 77, 65, 32, 81, 2, 205, 255, 8, 3, 85, 79, 84, 2, 157, 133, 10, + 10, 78, 84, 69, 82, 82, 79, 66, 65, 78, 71, 6, 48, 6, 69, 76, 76, 73, 84, + 69, 251, 189, 5, 85, 5, 25, 4, 32, 65, 78, 84, 2, 135, 145, 7, 69, 166, + 1, 52, 7, 65, 83, 72, 84, 82, 65, 32, 235, 187, 4, 79, 164, 1, 180, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 212, 1, 5, 83, 73, 71, 78, 32, 194, 21, 68, + 156, 246, 2, 16, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, + 32, 72, 139, 150, 2, 86, 100, 198, 235, 6, 65, 38, 68, 114, 84, 46, 86, + 186, 5, 85, 202, 141, 1, 79, 134, 60, 73, 42, 76, 250, 192, 1, 78, 46, + 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, + 2, 77, 2, 82, 3, 89, 8, 210, 175, 6, 67, 202, 207, 3, 65, 239, 1, 86, 52, + 66, 65, 32, 4, 72, 79, 79, 76, 46, 79, 74, 82, 143, 194, 10, 73, 4, 166, + 175, 9, 76, 215, 18, 82, 5, 153, 182, 5, 6, 32, 83, 65, 84, 67, 72, 6, + 36, 3, 82, 80, 73, 183, 138, 9, 79, 4, 202, 242, 9, 79, 135, 18, 85, 36, + 66, 69, 72, 4, 73, 80, 84, 32, 198, 246, 1, 85, 255, 194, 6, 79, 4, 36, + 3, 87, 68, 82, 195, 241, 9, 69, 2, 11, 73, 2, 167, 131, 10, 86, 28, 80, + 8, 67, 65, 80, 73, 84, 65, 76, 32, 86, 76, 81, 6, 83, 77, 65, 76, 76, 32, + 18, 170, 192, 10, 66, 2, 69, 2, 70, 2, 72, 2, 73, 2, 76, 2, 77, 2, 80, 3, + 82, 2, 37, 7, 73, 71, 65, 84, 85, 82, 69, 2, 11, 32, 2, 193, 251, 9, 2, + 69, 84, 8, 134, 191, 10, 69, 2, 71, 2, 76, 3, 79, 220, 1, 130, 2, 65, 30, + 67, 74, 69, 36, 5, 71, 77, 69, 78, 84, 28, 2, 77, 73, 172, 1, 8, 80, 65, + 82, 65, 84, 69, 68, 32, 138, 4, 82, 158, 1, 83, 68, 2, 84, 32, 140, 131, + 5, 8, 87, 73, 78, 71, 32, 78, 69, 69, 146, 140, 3, 88, 254, 104, 68, 213, + 160, 1, 2, 76, 70, 4, 210, 188, 10, 76, 3, 84, 6, 34, 84, 133, 217, 5, 2, + 79, 78, 4, 222, 187, 6, 73, 219, 199, 3, 79, 4, 130, 220, 1, 68, 195, + 132, 1, 45, 23, 221, 187, 4, 2, 69, 68, 6, 176, 80, 28, 68, 73, 82, 69, + 67, 84, 32, 80, 82, 79, 68, 85, 67, 84, 32, 87, 73, 84, 72, 32, 66, 79, + 84, 84, 79, 77, 32, 67, 252, 197, 7, 3, 83, 69, 88, 255, 212, 1, 67, 158, + 1, 48, 6, 66, 76, 79, 67, 75, 32, 191, 166, 9, 83, 156, 1, 88, 9, 81, 85, + 65, 68, 82, 65, 78, 84, 45, 129, 1, 8, 83, 69, 88, 84, 65, 78, 84, 45, + 30, 42, 49, 38, 50, 30, 51, 131, 184, 10, 52, 17, 34, 50, 30, 51, 131, + 184, 10, 52, 9, 26, 51, 131, 184, 10, 52, 5, 255, 183, 10, 52, 126, 58, + 49, 54, 50, 46, 51, 38, 52, 30, 53, 147, 182, 10, 54, 65, 50, 50, 46, 51, + 38, 52, 30, 53, 147, 182, 10, 54, 33, 42, 51, 38, 52, 30, 53, 147, 182, + 10, 54, 17, 34, 52, 30, 53, 147, 182, 10, 54, 9, 26, 53, 147, 182, 10, + 54, 5, 143, 182, 10, 54, 4, 120, 2, 86, 73, 241, 170, 2, 22, 73, 79, 85, + 83, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 83, 89, 77, 66, 79, 76, + 83, 2, 11, 67, 2, 175, 243, 9, 69, 4, 52, 7, 81, 85, 73, 81, 85, 65, 68, + 219, 253, 8, 65, 2, 91, 82, 4, 64, 10, 84, 82, 65, 78, 83, 77, 73, 84, + 32, 83, 195, 173, 6, 77, 2, 11, 84, 2, 211, 236, 8, 65, 242, 2, 102, 65, + 246, 19, 73, 102, 79, 134, 11, 69, 70, 82, 136, 131, 9, 5, 85, 70, 70, + 76, 69, 167, 143, 1, 89, 176, 2, 164, 1, 12, 68, 79, 87, 69, 68, 32, 87, + 72, 73, 84, 69, 32, 56, 9, 76, 76, 79, 87, 32, 80, 65, 78, 32, 62, 82, + 210, 9, 86, 172, 184, 7, 2, 77, 82, 147, 230, 1, 75, 6, 162, 223, 8, 67, + 206, 11, 83, 237, 4, 3, 76, 65, 84, 2, 17, 2, 79, 70, 2, 17, 2, 32, 70, + 2, 231, 249, 6, 79, 194, 1, 40, 4, 65, 68, 65, 32, 255, 175, 10, 75, 192, + 1, 162, 1, 68, 46, 69, 110, 72, 34, 76, 246, 1, 83, 212, 2, 6, 86, 79, + 87, 69, 76, 32, 250, 189, 4, 65, 228, 211, 1, 7, 67, 79, 78, 84, 73, 78, + 85, 163, 134, 4, 79, 24, 238, 209, 6, 79, 66, 65, 147, 235, 1, 73, 4, 84, + 15, 88, 84, 82, 65, 32, 83, 72, 79, 82, 84, 32, 86, 79, 87, 69, 215, 242, + 8, 75, 2, 139, 236, 9, 76, 2, 177, 253, 9, 3, 69, 65, 68, 96, 33, 6, 69, + 84, 84, 69, 82, 32, 96, 214, 210, 6, 65, 38, 68, 114, 84, 46, 86, 186, 5, + 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, 2, 67, 2, + 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, + 79, 30, 70, 65, 38, 69, 56, 4, 73, 71, 78, 32, 233, 164, 9, 3, 85, 84, + 82, 2, 153, 233, 9, 4, 78, 68, 72, 73, 6, 180, 201, 2, 5, 67, 84, 73, 79, + 78, 191, 179, 4, 80, 20, 106, 73, 190, 134, 5, 83, 158, 142, 1, 67, 98, + 78, 242, 60, 65, 174, 158, 1, 74, 150, 3, 85, 167, 242, 1, 86, 2, 37, 7, + 78, 86, 69, 82, 84, 69, 68, 2, 173, 148, 6, 2, 32, 67, 30, 60, 6, 77, 79, + 68, 73, 70, 73, 21, 5, 83, 73, 71, 78, 32, 2, 151, 137, 6, 69, 28, 82, + 80, 226, 211, 6, 65, 38, 85, 22, 86, 186, 201, 1, 73, 222, 137, 2, 69, 3, + 79, 2, 57, 12, 82, 73, 83, 72, 84, 72, 65, 77, 65, 84, 82, 65, 2, 155, + 144, 10, 32, 98, 72, 3, 69, 68, 32, 21, 11, 73, 65, 78, 32, 76, 69, 84, + 84, 69, 82, 32, 2, 147, 233, 9, 73, 96, 158, 2, 65, 120, 3, 67, 72, 85, + 22, 69, 70, 72, 46, 73, 46, 76, 22, 77, 38, 79, 94, 80, 18, 82, 22, 83, + 38, 84, 64, 2, 87, 79, 36, 2, 89, 69, 156, 235, 4, 2, 74, 85, 234, 222, + 2, 68, 202, 13, 90, 242, 87, 70, 182, 6, 71, 150, 45, 66, 246, 11, 75, + 218, 21, 86, 246, 25, 78, 167, 128, 1, 85, 16, 82, 82, 174, 234, 9, 73, + 234, 25, 68, 162, 8, 71, 2, 87, 198, 21, 83, 147, 1, 72, 4, 130, 151, 9, + 82, 135, 140, 1, 69, 2, 187, 128, 10, 82, 8, 38, 65, 206, 233, 9, 82, + 139, 56, 71, 4, 166, 162, 10, 82, 3, 84, 4, 200, 167, 8, 2, 65, 45, 203, + 169, 1, 85, 6, 254, 209, 9, 65, 142, 57, 67, 215, 22, 70, 2, 243, 153, 8, + 79, 4, 154, 149, 5, 69, 155, 244, 3, 73, 12, 70, 79, 130, 151, 9, 73, + 138, 109, 85, 150, 25, 65, 154, 3, 78, 3, 82, 2, 223, 137, 10, 90, 2, + 175, 13, 69, 2, 215, 254, 8, 79, 4, 194, 255, 8, 85, 187, 160, 1, 79, 6, + 26, 72, 147, 131, 10, 79, 4, 242, 250, 5, 73, 131, 147, 4, 69, 4, 226, + 149, 9, 79, 183, 137, 1, 69, 4, 242, 158, 10, 65, 3, 87, 10, 78, 69, 134, + 229, 3, 70, 252, 120, 6, 78, 84, 79, 32, 83, 72, 139, 192, 5, 80, 2, 231, + 215, 9, 76, 46, 252, 1, 2, 79, 84, 32, 6, 80, 80, 73, 78, 71, 32, 72, 2, + 82, 84, 176, 8, 7, 85, 76, 68, 69, 82, 69, 68, 220, 239, 1, 24, 67, 75, + 69, 68, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 69, 88, 80, 76, 79, + 68, 73, 78, 71, 158, 150, 3, 86, 251, 206, 4, 87, 2, 221, 133, 8, 3, 73, + 78, 71, 4, 36, 3, 84, 82, 79, 155, 233, 5, 66, 2, 11, 76, 2, 163, 129, 8, + 76, 32, 102, 32, 248, 5, 12, 72, 65, 78, 68, 32, 70, 79, 82, 77, 65, 84, + 32, 254, 171, 6, 67, 179, 232, 3, 83, 20, 154, 2, 66, 92, 11, 83, 76, 65, + 78, 84, 69, 68, 32, 78, 79, 82, 196, 1, 22, 82, 73, 71, 72, 84, 87, 65, + 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 65, 66, 79, 86, 69, 28, 7, 85, + 80, 32, 84, 65, 67, 75, 128, 1, 4, 76, 69, 70, 84, 149, 159, 2, 9, 68, + 79, 87, 78, 32, 84, 65, 67, 75, 4, 88, 14, 65, 67, 75, 83, 76, 65, 78, + 84, 69, 68, 32, 83, 79, 85, 33, 4, 69, 78, 84, 32, 2, 11, 84, 2, 203, + 252, 8, 72, 2, 245, 36, 37, 65, 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, - 32, 78, 79, 82, 84, 72, 32, 69, 2, 189, 237, 5, 2, 32, 76, 7, 11, 32, 4, - 76, 13, 65, 66, 79, 86, 69, 32, 83, 72, 79, 82, 84, 32, 68, 191, 131, 2, - 87, 2, 11, 79, 2, 11, 87, 2, 11, 78, 2, 11, 32, 2, 191, 209, 6, 84, 8, + 32, 78, 79, 82, 84, 72, 32, 69, 2, 229, 249, 5, 2, 32, 76, 7, 11, 32, 4, + 76, 13, 65, 66, 79, 86, 69, 32, 83, 72, 79, 82, 84, 32, 68, 159, 131, 2, + 87, 2, 11, 79, 2, 11, 87, 2, 11, 78, 2, 11, 32, 2, 191, 249, 6, 84, 8, 120, 10, 67, 79, 78, 84, 73, 78, 85, 73, 78, 71, 0, 6, 76, 69, 84, 84, - 69, 82, 44, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 245, 201, 8, 6, 32, 79, - 86, 69, 82, 76, 2, 21, 3, 32, 83, 84, 2, 251, 217, 9, 69, 2, 25, 4, 32, - 79, 80, 69, 2, 167, 205, 8, 78, 4, 26, 73, 215, 216, 9, 85, 2, 155, 217, - 9, 77, 159, 14, 174, 1, 68, 132, 20, 2, 71, 78, 232, 181, 1, 6, 77, 73, - 76, 65, 82, 32, 158, 2, 78, 198, 24, 88, 225, 171, 6, 15, 76, 72, 79, 85, - 69, 84, 84, 69, 32, 79, 70, 32, 74, 65, 80, 200, 1, 64, 5, 68, 72, 65, - 77, 32, 225, 17, 6, 69, 87, 65, 89, 83, 32, 184, 1, 194, 1, 68, 106, 69, - 68, 7, 76, 69, 84, 84, 69, 82, 32, 172, 4, 15, 82, 69, 80, 69, 84, 73, - 84, 73, 79, 78, 32, 77, 65, 82, 75, 50, 83, 165, 8, 11, 86, 79, 87, 69, - 76, 32, 83, 73, 71, 78, 32, 4, 18, 79, 67, 65, 2, 11, 85, 2, 11, 66, 2, - 25, 4, 76, 69, 32, 68, 2, 11, 65, 2, 247, 175, 9, 78, 2, 37, 7, 78, 68, - 32, 79, 70, 32, 84, 2, 129, 138, 9, 2, 69, 88, 102, 210, 1, 65, 98, 84, - 198, 233, 5, 68, 90, 86, 186, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, - 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, - 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 11, 72, 8, 76, 84, 69, 82, 78, 65, - 84, 69, 134, 210, 9, 65, 2, 73, 3, 85, 2, 155, 173, 9, 32, 14, 134, 1, - 72, 196, 212, 5, 19, 87, 79, 45, 67, 73, 82, 67, 76, 69, 32, 65, 76, 84, - 69, 82, 78, 65, 84, 69, 242, 180, 3, 84, 183, 71, 65, 4, 224, 144, 9, 20, - 82, 69, 69, 45, 67, 73, 82, 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, - 84, 69, 135, 64, 65, 6, 11, 45, 6, 234, 207, 9, 49, 2, 50, 3, 51, 44, 38, - 69, 181, 7, 4, 73, 71, 78, 32, 32, 96, 11, 67, 84, 73, 79, 78, 32, 77, - 65, 82, 75, 32, 177, 6, 8, 80, 65, 82, 65, 84, 79, 82, 32, 28, 80, 11, - 68, 79, 85, 66, 76, 69, 32, 82, 73, 78, 71, 57, 5, 87, 73, 84, 72, 32, 5, - 11, 32, 2, 225, 162, 8, 6, 87, 73, 84, 72, 32, 82, 24, 212, 1, 12, 67, - 73, 82, 67, 76, 69, 83, 32, 65, 78, 68, 32, 116, 5, 81, 85, 65, 68, 82, - 0, 4, 83, 69, 80, 84, 12, 16, 82, 65, 89, 83, 32, 65, 78, 68, 32, 68, 79, - 84, 84, 69, 68, 32, 46, 68, 45, 3, 84, 82, 73, 6, 60, 4, 70, 79, 85, 82, - 0, 3, 84, 87, 79, 163, 160, 8, 82, 2, 169, 160, 6, 8, 32, 69, 78, 67, 76, - 79, 83, 85, 2, 83, 85, 6, 42, 68, 28, 3, 84, 82, 73, 215, 1, 67, 2, 197, - 1, 3, 79, 85, 66, 2, 171, 1, 80, 6, 52, 9, 68, 69, 78, 84, 32, 65, 78, - 68, 32, 103, 80, 4, 116, 6, 68, 79, 84, 84, 69, 68, 49, 14, 85, 45, 83, - 72, 65, 80, 69, 68, 32, 79, 82, 78, 65, 77, 2, 17, 2, 76, 69, 2, 17, 2, - 32, 67, 2, 25, 4, 82, 69, 83, 67, 2, 151, 186, 1, 69, 4, 226, 165, 8, 66, - 255, 81, 68, 12, 246, 208, 4, 83, 206, 144, 1, 67, 98, 78, 222, 160, 3, - 65, 239, 1, 86, 26, 74, 65, 94, 86, 226, 246, 5, 85, 210, 200, 1, 73, - 206, 134, 2, 69, 3, 79, 10, 184, 247, 5, 10, 76, 84, 69, 82, 78, 65, 84, - 69, 32, 85, 158, 207, 3, 65, 2, 73, 3, 85, 4, 11, 79, 4, 33, 6, 67, 65, - 76, 73, 67, 32, 4, 167, 226, 5, 82, 16, 56, 5, 66, 76, 65, 67, 75, 1, 5, - 87, 72, 73, 84, 69, 8, 11, 32, 8, 70, 82, 24, 3, 76, 69, 70, 12, 4, 68, - 79, 87, 78, 1, 2, 85, 80, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 225, 92, 6, - 32, 80, 79, 73, 78, 84, 194, 10, 96, 8, 87, 82, 73, 84, 73, 78, 71, 32, - 217, 238, 1, 10, 32, 79, 70, 32, 84, 72, 69, 32, 72, 79, 192, 10, 136, 3, - 4, 65, 73, 82, 32, 192, 1, 2, 66, 82, 102, 67, 138, 1, 68, 218, 4, 69, - 242, 6, 70, 244, 3, 4, 87, 65, 76, 76, 138, 1, 72, 238, 77, 76, 224, 6, - 2, 77, 79, 222, 42, 78, 230, 1, 82, 202, 7, 83, 162, 5, 84, 180, 10, 5, - 71, 82, 65, 83, 80, 184, 5, 30, 85, 80, 80, 69, 82, 32, 66, 79, 68, 89, - 32, 84, 73, 76, 84, 73, 78, 71, 32, 70, 82, 79, 77, 32, 72, 73, 80, 32, - 74, 79, 211, 179, 4, 80, 8, 48, 4, 66, 76, 79, 87, 29, 4, 83, 85, 67, 75, - 4, 58, 32, 195, 207, 4, 73, 4, 30, 32, 61, 3, 73, 78, 71, 2, 157, 158, 1, - 10, 83, 77, 65, 76, 76, 32, 82, 79, 84, 65, 2, 251, 190, 8, 32, 10, 52, - 5, 69, 65, 84, 72, 32, 149, 170, 1, 2, 85, 83, 4, 152, 163, 2, 2, 69, 88, - 1, 2, 73, 78, 10, 40, 6, 72, 69, 69, 75, 83, 32, 63, 79, 6, 170, 107, 83, - 146, 38, 78, 153, 204, 3, 4, 80, 85, 70, 70, 4, 246, 236, 8, 76, 211, 46, - 77, 28, 108, 15, 82, 69, 65, 77, 89, 32, 69, 89, 69, 66, 82, 79, 87, 83, - 32, 165, 1, 7, 89, 78, 65, 77, 73, 67, 32, 8, 64, 4, 68, 79, 87, 78, 0, - 2, 85, 80, 29, 4, 78, 69, 85, 84, 2, 169, 143, 1, 2, 32, 78, 4, 21, 3, - 82, 65, 76, 4, 11, 32, 4, 194, 58, 68, 247, 255, 8, 85, 20, 180, 1, 11, - 69, 86, 69, 82, 89, 32, 79, 84, 72, 69, 82, 30, 70, 22, 83, 144, 122, 9, - 65, 82, 82, 79, 87, 72, 69, 65, 68, 138, 39, 82, 232, 186, 3, 2, 84, 69, - 21, 4, 71, 82, 65, 68, 2, 137, 161, 8, 2, 32, 84, 2, 135, 241, 5, 65, 6, - 68, 11, 73, 77, 85, 76, 84, 65, 78, 69, 79, 85, 83, 235, 157, 8, 76, 5, - 251, 151, 1, 32, 54, 64, 2, 89, 69, 188, 208, 4, 4, 88, 67, 73, 84, 167, - 216, 3, 65, 50, 166, 1, 32, 56, 15, 66, 82, 79, 87, 83, 32, 83, 84, 82, - 65, 73, 71, 72, 84, 32, 44, 5, 71, 65, 90, 69, 45, 140, 2, 7, 76, 65, 83, - 72, 69, 83, 32, 65, 2, 83, 32, 6, 140, 151, 1, 5, 66, 76, 73, 78, 75, - 131, 224, 4, 87, 6, 186, 53, 68, 170, 84, 78, 207, 171, 8, 85, 18, 100, - 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 25, 10, 87, 65, 76, 76, - 80, 76, 65, 78, 69, 32, 8, 82, 83, 207, 45, 67, 10, 18, 67, 43, 83, 4, - 254, 45, 85, 241, 95, 3, 73, 82, 67, 6, 37, 7, 84, 82, 65, 73, 71, 72, - 84, 7, 11, 32, 4, 250, 163, 1, 65, 55, 68, 6, 130, 51, 68, 196, 154, 4, - 4, 70, 76, 85, 84, 179, 229, 4, 85, 14, 112, 5, 72, 65, 76, 70, 32, 26, - 67, 28, 4, 87, 73, 68, 69, 246, 92, 79, 233, 244, 3, 6, 83, 81, 85, 69, - 69, 90, 4, 22, 67, 147, 93, 79, 2, 205, 229, 2, 2, 76, 79, 4, 162, 67, - 32, 233, 61, 4, 78, 73, 78, 71, 38, 204, 1, 28, 65, 67, 69, 32, 68, 73, - 82, 69, 67, 84, 73, 79, 78, 32, 80, 79, 83, 73, 84, 73, 79, 78, 32, 78, - 79, 83, 69, 32, 138, 1, 73, 82, 76, 176, 1, 8, 79, 82, 69, 72, 69, 65, - 68, 32, 163, 225, 6, 85, 6, 88, 10, 85, 80, 32, 79, 82, 32, 68, 79, 87, - 78, 13, 8, 70, 79, 82, 87, 65, 82, 68, 32, 5, 11, 32, 2, 169, 212, 4, 3, - 84, 73, 76, 12, 248, 210, 3, 11, 76, 76, 32, 77, 79, 68, 73, 70, 73, 69, - 82, 171, 163, 2, 78, 12, 44, 4, 73, 67, 75, 32, 29, 3, 79, 79, 82, 10, - 174, 141, 1, 76, 35, 83, 2, 169, 176, 8, 20, 80, 76, 65, 78, 69, 32, 83, - 72, 79, 85, 76, 68, 69, 82, 32, 72, 73, 80, 32, 77, 6, 182, 89, 87, 222, - 38, 67, 47, 78, 156, 4, 34, 65, 137, 75, 3, 69, 65, 68, 140, 4, 36, 3, - 78, 68, 45, 211, 242, 8, 73, 138, 4, 92, 5, 65, 78, 71, 76, 69, 138, 5, - 67, 150, 10, 70, 186, 42, 72, 241, 12, 4, 79, 86, 65, 76, 37, 11, 32, 34, - 188, 1, 5, 73, 78, 68, 69, 88, 176, 1, 7, 76, 73, 84, 84, 76, 69, 32, - 136, 1, 5, 82, 73, 78, 71, 32, 173, 66, 18, 77, 73, 68, 68, 76, 69, 32, - 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 17, 11, 32, 14, 128, 1, 7, - 77, 73, 68, 68, 76, 69, 32, 228, 24, 11, 82, 73, 78, 71, 32, 76, 73, 84, - 84, 76, 69, 241, 42, 5, 84, 72, 85, 77, 66, 4, 178, 70, 76, 183, 144, 8, - 82, 8, 44, 5, 73, 78, 68, 69, 88, 135, 167, 9, 85, 7, 145, 24, 18, 32, - 84, 72, 85, 77, 66, 32, 73, 78, 68, 69, 88, 32, 84, 72, 85, 77, 66, 4, - 108, 22, 68, 79, 87, 78, 32, 77, 73, 68, 68, 76, 69, 32, 84, 72, 85, 77, - 66, 32, 73, 78, 68, 69, 159, 68, 76, 2, 207, 231, 7, 88, 88, 64, 5, 73, - 82, 67, 76, 69, 184, 3, 3, 76, 65, 87, 183, 2, 85, 35, 11, 32, 32, 116, - 5, 73, 78, 68, 69, 88, 200, 1, 7, 76, 73, 84, 84, 76, 69, 32, 36, 7, 77, - 73, 68, 68, 76, 69, 32, 167, 64, 82, 21, 11, 32, 18, 72, 6, 77, 73, 68, - 68, 76, 69, 198, 26, 72, 246, 38, 82, 195, 158, 8, 66, 13, 11, 32, 10, - 64, 5, 67, 82, 79, 83, 83, 194, 64, 84, 90, 76, 183, 144, 8, 82, 4, 138, - 65, 32, 167, 155, 8, 69, 4, 170, 253, 7, 73, 239, 164, 1, 85, 6, 252, 47, - 10, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 247, 241, 8, 85, 17, 11, 32, - 14, 144, 2, 28, 77, 73, 68, 68, 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, - 84, 84, 76, 69, 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 176, 49, 2, 70, - 79, 194, 11, 78, 170, 1, 84, 237, 118, 23, 73, 78, 68, 69, 88, 32, 84, - 72, 85, 77, 66, 32, 67, 85, 82, 86, 69, 32, 84, 72, 85, 77, 66, 5, 163, - 181, 1, 32, 38, 46, 80, 149, 2, 6, 82, 76, 73, 67, 85, 69, 31, 11, 32, - 28, 188, 1, 5, 73, 78, 68, 69, 88, 56, 19, 70, 73, 86, 69, 32, 70, 73, - 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 196, 50, 7, 77, 73, 68, - 68, 76, 69, 32, 18, 79, 214, 7, 78, 171, 1, 84, 9, 11, 32, 6, 40, 5, 84, - 72, 85, 77, 66, 247, 58, 82, 5, 215, 46, 32, 9, 11, 32, 6, 72, 5, 73, 78, - 68, 69, 88, 0, 6, 77, 73, 68, 68, 76, 69, 195, 71, 79, 2, 133, 204, 8, - 13, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 172, 2, 44, 3, - 73, 83, 84, 177, 33, 3, 76, 65, 84, 247, 1, 11, 32, 244, 1, 160, 2, 5, - 73, 78, 68, 69, 88, 192, 16, 7, 76, 73, 84, 84, 76, 69, 32, 196, 2, 7, - 77, 73, 68, 68, 76, 69, 32, 200, 4, 5, 82, 73, 78, 71, 32, 132, 2, 5, 84, - 72, 85, 77, 66, 138, 2, 72, 205, 9, 22, 70, 79, 85, 82, 32, 70, 73, 78, - 71, 69, 82, 83, 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 137, 1, 11, 32, - 134, 1, 232, 1, 4, 66, 69, 78, 84, 36, 6, 72, 73, 78, 71, 69, 68, 76, 6, - 77, 73, 68, 68, 76, 69, 168, 7, 2, 67, 85, 64, 6, 84, 72, 85, 77, 66, 32, - 160, 5, 16, 85, 80, 32, 77, 73, 68, 68, 76, 69, 32, 72, 73, 78, 71, 69, - 68, 151, 4, 82, 5, 217, 45, 5, 32, 79, 86, 69, 82, 9, 11, 32, 6, 252, 20, - 8, 77, 73, 68, 68, 76, 69, 32, 85, 251, 230, 7, 76, 71, 11, 32, 68, 236, - 1, 4, 66, 69, 78, 84, 42, 67, 244, 1, 10, 85, 80, 32, 83, 80, 82, 69, 65, - 68, 32, 100, 6, 72, 73, 78, 71, 69, 68, 46, 82, 80, 5, 84, 72, 85, 77, - 66, 188, 28, 14, 83, 84, 82, 65, 73, 71, 72, 84, 32, 84, 72, 85, 77, 66, - 231, 17, 76, 5, 229, 92, 6, 32, 84, 72, 85, 77, 66, 28, 68, 8, 79, 78, - 74, 79, 73, 78, 69, 68, 233, 1, 4, 82, 79, 83, 83, 23, 11, 32, 20, 144, - 1, 6, 67, 85, 80, 80, 69, 68, 28, 6, 84, 72, 85, 77, 66, 32, 68, 5, 72, - 73, 78, 71, 69, 166, 22, 73, 165, 7, 6, 77, 73, 68, 68, 76, 69, 5, 11, - 32, 2, 203, 27, 84, 8, 160, 1, 4, 83, 73, 68, 69, 235, 61, 70, 6, 22, 69, - 163, 47, 32, 4, 223, 15, 68, 5, 241, 30, 7, 32, 83, 80, 82, 69, 65, 68, - 8, 32, 3, 73, 78, 71, 247, 19, 65, 7, 11, 32, 4, 138, 36, 67, 199, 168, - 8, 66, 19, 11, 32, 16, 64, 6, 65, 78, 71, 76, 69, 68, 22, 67, 106, 72, - 231, 202, 8, 66, 5, 207, 188, 5, 32, 6, 74, 85, 230, 7, 73, 241, 25, 10, - 79, 78, 74, 79, 73, 78, 69, 68, 32, 72, 2, 217, 174, 4, 2, 80, 80, 4, - 194, 33, 73, 233, 26, 2, 79, 79, 40, 164, 1, 7, 65, 78, 71, 76, 69, 68, - 32, 46, 67, 220, 1, 14, 70, 79, 82, 87, 65, 82, 68, 32, 73, 78, 68, 69, - 88, 32, 32, 4, 72, 79, 79, 75, 53, 4, 83, 73, 68, 69, 4, 128, 1, 2, 73, - 78, 1, 3, 79, 85, 84, 12, 36, 5, 73, 82, 67, 76, 69, 15, 85, 5, 47, 68, - 8, 32, 4, 80, 80, 69, 68, 39, 82, 2, 221, 40, 5, 32, 77, 73, 68, 68, 6, - 56, 9, 86, 69, 32, 84, 72, 85, 77, 66, 32, 143, 37, 76, 4, 206, 160, 1, - 73, 219, 164, 4, 85, 4, 218, 84, 83, 183, 242, 7, 66, 7, 157, 8, 9, 69, - 68, 32, 77, 73, 68, 68, 76, 69, 15, 11, 32, 12, 92, 6, 73, 78, 68, 69, - 88, 32, 156, 13, 5, 84, 72, 85, 77, 66, 193, 8, 4, 66, 79, 84, 72, 4, 26, - 72, 183, 197, 8, 66, 2, 207, 212, 7, 73, 7, 37, 7, 32, 84, 72, 85, 77, - 66, 32, 4, 178, 28, 67, 251, 129, 1, 83, 22, 88, 4, 68, 79, 87, 78, 186, - 1, 84, 158, 6, 82, 130, 21, 73, 170, 167, 8, 66, 147, 67, 85, 9, 11, 32, - 6, 80, 9, 79, 84, 72, 69, 82, 83, 32, 67, 73, 25, 7, 82, 73, 80, 80, 76, - 69, 32, 2, 241, 51, 2, 82, 67, 4, 22, 67, 187, 80, 83, 2, 11, 85, 2, 185, - 166, 4, 2, 82, 86, 4, 192, 35, 6, 79, 85, 67, 72, 69, 83, 39, 72, 30, - 134, 1, 82, 28, 6, 84, 72, 85, 77, 66, 32, 138, 3, 85, 134, 1, 68, 214, - 30, 76, 161, 188, 7, 9, 66, 69, 78, 84, 32, 79, 86, 69, 82, 4, 238, 4, - 65, 235, 29, 73, 16, 80, 7, 65, 78, 71, 76, 69, 68, 32, 126, 67, 124, 4, - 72, 79, 79, 75, 151, 32, 76, 6, 60, 10, 79, 85, 84, 32, 73, 78, 68, 69, - 88, 32, 215, 1, 73, 4, 26, 67, 211, 130, 9, 85, 2, 241, 182, 2, 3, 82, - 79, 83, 6, 76, 12, 73, 82, 67, 76, 69, 68, 32, 73, 78, 68, 69, 88, 45, 3, - 85, 80, 80, 4, 11, 32, 4, 150, 21, 72, 191, 236, 8, 85, 2, 25, 4, 69, 68, - 32, 73, 2, 229, 30, 4, 78, 68, 69, 88, 4, 11, 80, 5, 175, 15, 32, 18, - 102, 68, 20, 6, 77, 73, 68, 68, 76, 69, 42, 82, 194, 29, 84, 90, 76, 182, - 188, 7, 73, 239, 164, 1, 85, 2, 223, 174, 7, 79, 7, 11, 32, 4, 206, 3, - 82, 179, 16, 67, 2, 11, 65, 2, 37, 7, 73, 83, 69, 68, 32, 75, 78, 2, 11, - 85, 2, 11, 67, 2, 211, 221, 7, 75, 35, 11, 32, 32, 148, 1, 8, 66, 69, 84, - 87, 69, 69, 78, 32, 102, 72, 20, 5, 79, 86, 69, 82, 32, 120, 4, 83, 73, - 68, 69, 100, 6, 85, 78, 68, 69, 82, 32, 223, 40, 70, 8, 80, 12, 73, 78, - 68, 69, 88, 32, 77, 73, 68, 68, 76, 69, 246, 20, 77, 159, 6, 82, 5, 151, - 70, 32, 2, 131, 156, 4, 69, 4, 52, 6, 70, 79, 85, 82, 32, 82, 161, 2, 2, - 84, 87, 2, 11, 65, 2, 133, 81, 9, 73, 83, 69, 68, 32, 75, 78, 85, 67, 6, - 11, 32, 6, 38, 68, 190, 15, 67, 199, 168, 8, 66, 2, 25, 4, 73, 65, 71, - 79, 2, 215, 241, 7, 78, 10, 54, 73, 34, 84, 48, 4, 70, 79, 85, 82, 255, - 22, 76, 2, 161, 7, 4, 78, 68, 69, 88, 4, 34, 87, 13, 4, 72, 82, 69, 69, - 2, 11, 79, 2, 153, 235, 7, 5, 32, 70, 73, 78, 71, 55, 11, 32, 52, 194, 1, - 70, 172, 3, 4, 72, 69, 69, 76, 192, 1, 6, 83, 80, 76, 73, 84, 32, 236, 1, - 6, 84, 72, 85, 77, 66, 32, 153, 231, 4, 16, 66, 69, 84, 87, 69, 69, 78, - 32, 80, 65, 76, 77, 32, 70, 65, 67, 24, 140, 1, 18, 73, 86, 69, 32, 70, - 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 165, 1, 11, 79, 85, - 82, 32, 70, 73, 78, 71, 69, 82, 83, 15, 11, 32, 12, 68, 6, 72, 73, 78, - 71, 69, 68, 42, 84, 186, 2, 70, 143, 176, 8, 66, 7, 11, 32, 4, 190, 4, - 84, 151, 15, 78, 2, 205, 35, 6, 72, 85, 77, 66, 32, 70, 11, 11, 32, 8, - 72, 9, 67, 79, 78, 74, 79, 73, 78, 69, 68, 154, 8, 72, 175, 169, 8, 66, - 5, 153, 216, 8, 3, 32, 83, 80, 11, 11, 32, 8, 96, 19, 70, 73, 86, 69, 32, - 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 151, 2, 84, 7, - 11, 32, 4, 26, 70, 143, 176, 8, 66, 2, 21, 3, 79, 85, 82, 2, 167, 1, 32, - 10, 72, 6, 67, 69, 78, 84, 82, 69, 96, 5, 73, 78, 68, 69, 88, 171, 16, - 76, 7, 49, 10, 32, 84, 72, 85, 77, 66, 32, 83, 73, 68, 4, 11, 69, 5, 11, - 32, 2, 203, 174, 8, 66, 2, 11, 32, 2, 11, 84, 2, 197, 135, 1, 5, 72, 85, - 77, 66, 32, 6, 130, 31, 70, 170, 104, 83, 207, 166, 7, 66, 82, 48, 4, 73, - 78, 71, 69, 181, 9, 3, 79, 79, 75, 63, 11, 32, 60, 206, 1, 70, 172, 1, 5, - 73, 78, 68, 69, 88, 196, 2, 6, 76, 73, 84, 84, 76, 69, 80, 6, 77, 73, 68, - 68, 76, 69, 30, 79, 64, 4, 82, 73, 78, 71, 124, 6, 84, 72, 85, 77, 66, - 32, 154, 6, 78, 215, 128, 4, 83, 4, 92, 19, 73, 86, 69, 32, 70, 73, 78, - 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 32, 19, 79, 2, 207, 25, 79, - 2, 249, 1, 11, 85, 82, 32, 70, 73, 78, 71, 69, 82, 83, 32, 23, 11, 32, - 20, 86, 72, 40, 7, 77, 73, 68, 68, 76, 69, 32, 104, 5, 84, 72, 85, 77, - 66, 223, 9, 82, 2, 11, 73, 2, 249, 140, 4, 2, 78, 71, 6, 36, 4, 82, 73, - 78, 71, 207, 10, 76, 5, 11, 32, 2, 11, 67, 2, 21, 3, 79, 78, 74, 2, 191, - 33, 79, 11, 11, 32, 8, 34, 83, 226, 22, 79, 203, 39, 76, 4, 146, 222, 6, - 73, 187, 8, 77, 9, 11, 32, 6, 22, 73, 195, 8, 84, 4, 25, 4, 78, 68, 69, - 88, 5, 151, 8, 32, 5, 11, 32, 2, 175, 8, 82, 8, 21, 3, 80, 69, 78, 9, 11, - 32, 6, 174, 7, 78, 171, 1, 84, 5, 97, 22, 32, 68, 79, 87, 78, 32, 73, 78, - 68, 69, 88, 32, 84, 72, 85, 77, 66, 32, 72, 79, 79, 75, 2, 205, 77, 2, - 32, 77, 6, 68, 9, 66, 69, 84, 87, 69, 69, 78, 32, 77, 53, 4, 83, 73, 68, - 69, 2, 29, 5, 73, 68, 68, 76, 69, 2, 235, 134, 4, 32, 5, 33, 6, 32, 84, - 79, 85, 67, 72, 2, 145, 194, 7, 3, 73, 78, 71, 21, 11, 32, 18, 172, 1, 4, - 67, 85, 82, 76, 28, 18, 73, 78, 68, 69, 88, 32, 82, 73, 78, 71, 32, 76, - 73, 84, 84, 76, 69, 32, 48, 7, 77, 73, 68, 68, 76, 69, 32, 221, 2, 4, 82, - 73, 78, 71, 2, 229, 132, 5, 2, 73, 67, 6, 254, 159, 5, 85, 234, 203, 2, - 79, 239, 41, 73, 8, 104, 21, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, - 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 139, 2, 84, 7, 231, 213, 2, 32, - 17, 11, 32, 14, 130, 1, 76, 66, 78, 78, 82, 94, 84, 136, 154, 2, 15, 70, - 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 131, 162, 5, 73, - 2, 21, 3, 73, 84, 84, 2, 17, 2, 76, 69, 2, 215, 236, 7, 32, 2, 11, 79, 2, - 11, 32, 2, 11, 84, 2, 11, 72, 2, 209, 167, 7, 2, 85, 77, 2, 11, 73, 2, - 21, 3, 78, 71, 32, 2, 11, 76, 2, 11, 73, 2, 11, 84, 2, 239, 191, 7, 84, - 4, 29, 5, 72, 85, 77, 66, 32, 4, 206, 14, 70, 171, 104, 83, 17, 11, 32, - 14, 56, 8, 77, 79, 86, 69, 77, 69, 78, 84, 199, 221, 7, 82, 12, 26, 45, - 207, 144, 7, 32, 10, 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, - 25, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 4, 62, 67, 235, 40, 83, - 6, 38, 67, 28, 2, 84, 73, 207, 40, 83, 2, 241, 225, 7, 2, 85, 82, 2, 239, - 193, 8, 76, 38, 50, 73, 225, 3, 7, 79, 67, 65, 84, 73, 79, 78, 22, 32, 3, - 77, 66, 32, 171, 1, 80, 16, 56, 4, 67, 79, 77, 66, 29, 6, 76, 69, 78, 71, - 84, 72, 2, 189, 176, 5, 2, 73, 78, 14, 11, 45, 14, 202, 220, 8, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 6, 58, 32, 153, 1, 9, 83, 32, 80, - 82, 69, 83, 83, 69, 68, 4, 116, 12, 76, 79, 87, 69, 82, 32, 79, 86, 69, - 82, 32, 85, 213, 193, 7, 11, 85, 80, 80, 69, 82, 32, 79, 86, 69, 82, 32, - 2, 11, 80, 2, 231, 156, 8, 80, 2, 29, 5, 32, 84, 79, 71, 69, 2, 11, 84, - 2, 167, 156, 8, 72, 16, 22, 32, 215, 1, 45, 12, 140, 1, 2, 72, 69, 48, 3, - 84, 79, 82, 152, 217, 6, 3, 68, 69, 80, 0, 3, 87, 73, 68, 241, 106, 10, - 76, 73, 77, 66, 83, 32, 68, 73, 71, 73, 4, 216, 13, 4, 65, 68, 32, 78, - 175, 160, 8, 73, 2, 167, 185, 8, 83, 4, 52, 5, 70, 76, 79, 79, 82, 1, 4, - 87, 65, 76, 76, 2, 149, 153, 8, 5, 80, 76, 65, 78, 69, 156, 3, 64, 4, 85, - 84, 72, 32, 161, 5, 7, 86, 69, 77, 69, 78, 84, 45, 54, 186, 1, 67, 88, 5, - 70, 82, 79, 87, 78, 0, 5, 83, 77, 73, 76, 69, 56, 4, 75, 73, 83, 83, 36, - 5, 79, 80, 69, 78, 32, 192, 1, 5, 84, 69, 78, 83, 69, 165, 29, 5, 87, 82, - 73, 78, 75, 8, 48, 6, 76, 79, 83, 69, 68, 32, 243, 166, 7, 79, 6, 222, 2, - 70, 142, 38, 67, 47, 78, 7, 11, 32, 4, 22, 79, 203, 1, 87, 2, 179, 131, - 7, 80, 7, 11, 32, 4, 166, 1, 87, 83, 70, 18, 100, 4, 79, 86, 65, 76, 0, - 9, 82, 69, 67, 84, 65, 78, 71, 76, 69, 42, 87, 82, 70, 139, 131, 7, 67, - 7, 11, 32, 4, 26, 87, 183, 129, 7, 89, 2, 25, 4, 82, 73, 78, 75, 2, 131, - 243, 3, 76, 7, 11, 32, 4, 18, 70, 43, 83, 2, 17, 2, 79, 82, 2, 155, 187, - 7, 87, 2, 17, 2, 85, 67, 2, 147, 242, 3, 75, 230, 2, 192, 1, 9, 68, 73, - 65, 71, 79, 78, 65, 76, 32, 148, 1, 11, 70, 76, 79, 79, 82, 80, 76, 65, - 78, 69, 32, 184, 14, 6, 72, 73, 78, 71, 69, 32, 189, 2, 10, 87, 65, 76, - 76, 80, 76, 65, 78, 69, 32, 32, 56, 8, 66, 69, 84, 87, 69, 69, 78, 32, - 22, 65, 31, 84, 16, 18, 65, 31, 84, 8, 229, 28, 3, 87, 65, 89, 8, 201, - 28, 6, 79, 87, 65, 82, 68, 83, 150, 1, 208, 2, 24, 65, 82, 77, 32, 67, - 73, 82, 67, 76, 69, 32, 72, 73, 84, 84, 73, 78, 71, 32, 87, 65, 76, 76, - 32, 38, 66, 34, 67, 224, 1, 8, 70, 73, 78, 71, 69, 82, 32, 67, 52, 5, 72, - 85, 77, 80, 32, 184, 3, 5, 76, 79, 79, 80, 32, 198, 1, 83, 108, 7, 84, - 82, 73, 80, 76, 69, 32, 110, 87, 206, 12, 68, 198, 2, 80, 154, 6, 90, - 159, 175, 7, 74, 12, 178, 7, 76, 154, 9, 77, 39, 83, 8, 150, 17, 79, 243, - 172, 7, 69, 28, 86, 72, 20, 5, 85, 82, 86, 69, 32, 196, 29, 5, 79, 82, - 78, 69, 82, 195, 239, 6, 82, 2, 163, 156, 7, 69, 18, 80, 4, 67, 79, 77, - 66, 158, 8, 72, 230, 15, 76, 198, 157, 2, 77, 243, 178, 1, 83, 2, 11, 73, - 2, 187, 234, 3, 78, 6, 128, 9, 6, 73, 82, 67, 76, 69, 83, 239, 20, 79, - 18, 56, 8, 72, 73, 84, 84, 73, 78, 71, 32, 167, 231, 3, 83, 16, 72, 8, - 67, 69, 73, 76, 73, 78, 71, 32, 101, 6, 70, 76, 79, 79, 82, 32, 8, 56, 5, - 76, 65, 82, 71, 69, 1, 5, 83, 77, 65, 76, 76, 4, 11, 32, 4, 238, 54, 84, - 135, 2, 68, 8, 88, 4, 83, 77, 65, 76, 12, 5, 76, 65, 82, 71, 69, 17, 7, - 84, 82, 73, 80, 76, 69, 32, 2, 11, 76, 2, 255, 18, 32, 4, 56, 5, 76, 65, - 82, 71, 69, 1, 5, 83, 77, 65, 76, 76, 2, 157, 53, 2, 32, 84, 18, 56, 8, - 72, 73, 84, 84, 73, 78, 71, 32, 239, 227, 3, 83, 16, 64, 7, 67, 69, 73, - 76, 73, 78, 71, 1, 5, 70, 76, 79, 79, 82, 8, 11, 32, 8, 22, 76, 191, 9, - 83, 4, 249, 22, 4, 65, 82, 71, 69, 12, 44, 6, 72, 65, 75, 73, 78, 71, - 243, 16, 73, 2, 21, 3, 32, 80, 65, 2, 149, 227, 3, 4, 82, 65, 76, 76, 8, - 76, 12, 65, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 32, 138, 18, 87, 63, - 83, 4, 134, 18, 87, 159, 21, 77, 18, 80, 4, 65, 86, 69, 32, 169, 1, 11, - 82, 73, 83, 84, 32, 67, 73, 82, 67, 76, 69, 14, 30, 72, 102, 83, 171, 20, - 76, 8, 37, 7, 73, 84, 84, 73, 78, 71, 32, 8, 252, 2, 4, 67, 69, 73, 76, - 25, 5, 70, 76, 79, 79, 82, 4, 138, 251, 4, 78, 195, 193, 1, 77, 4, 209, - 5, 10, 32, 72, 73, 84, 84, 73, 78, 71, 32, 87, 14, 112, 3, 85, 80, 32, - 184, 1, 6, 68, 79, 87, 78, 32, 83, 245, 183, 5, 10, 83, 73, 68, 69, 32, - 84, 79, 32, 83, 73, 10, 40, 5, 68, 79, 87, 78, 32, 143, 1, 83, 8, 68, 8, - 65, 76, 84, 69, 82, 78, 65, 84, 230, 17, 76, 143, 203, 3, 83, 4, 21, 3, - 73, 78, 71, 4, 11, 32, 4, 190, 17, 76, 143, 203, 3, 83, 2, 219, 30, 69, - 162, 1, 148, 2, 11, 65, 82, 77, 32, 67, 73, 82, 67, 76, 69, 32, 98, 66, - 54, 67, 174, 4, 68, 100, 8, 70, 73, 78, 71, 69, 82, 32, 67, 64, 5, 72, - 85, 77, 80, 32, 60, 5, 76, 79, 79, 80, 32, 102, 80, 34, 83, 216, 1, 7, - 84, 82, 73, 80, 76, 69, 32, 218, 1, 87, 202, 2, 90, 159, 175, 7, 74, 8, - 18, 77, 39, 83, 4, 225, 13, 5, 69, 68, 73, 85, 77, 4, 11, 77, 4, 177, 13, - 3, 65, 76, 76, 12, 34, 79, 185, 13, 3, 69, 78, 68, 6, 183, 13, 88, 46, - 100, 6, 79, 82, 78, 69, 82, 32, 88, 4, 85, 82, 86, 69, 232, 11, 4, 72, - 69, 67, 75, 195, 239, 6, 82, 8, 54, 82, 194, 12, 76, 158, 152, 2, 77, - 243, 178, 1, 83, 2, 11, 79, 2, 223, 140, 5, 84, 30, 50, 32, 137, 2, 7, - 68, 32, 67, 82, 79, 83, 83, 26, 66, 72, 68, 2, 84, 72, 133, 5, 7, 81, 85, - 65, 82, 84, 69, 82, 12, 196, 5, 10, 65, 76, 70, 45, 67, 73, 82, 67, 76, - 69, 159, 14, 73, 6, 96, 2, 69, 78, 29, 18, 82, 69, 69, 45, 81, 85, 65, - 82, 84, 69, 82, 32, 67, 73, 82, 67, 76, 69, 2, 11, 32, 2, 131, 1, 83, 4, - 11, 32, 4, 242, 161, 2, 77, 243, 178, 1, 83, 8, 33, 6, 79, 85, 66, 76, - 69, 32, 8, 30, 83, 150, 4, 65, 75, 87, 2, 253, 138, 8, 3, 84, 82, 65, 6, - 32, 3, 73, 82, 67, 151, 9, 79, 4, 173, 7, 3, 76, 69, 83, 10, 142, 8, 76, - 178, 8, 72, 238, 143, 2, 77, 243, 178, 1, 83, 12, 68, 5, 83, 77, 65, 76, - 76, 142, 7, 76, 178, 8, 72, 239, 143, 2, 77, 5, 11, 32, 2, 239, 36, 68, - 6, 181, 6, 4, 69, 65, 75, 83, 12, 22, 73, 243, 36, 72, 10, 29, 5, 78, 71, - 76, 69, 32, 10, 52, 8, 83, 84, 82, 65, 73, 71, 72, 84, 207, 1, 87, 8, 11, - 32, 8, 42, 76, 198, 157, 2, 77, 243, 178, 1, 83, 4, 25, 4, 65, 82, 71, - 69, 5, 191, 149, 8, 83, 8, 26, 65, 74, 87, 63, 83, 4, 49, 10, 76, 84, 69, - 82, 78, 65, 84, 73, 78, 71, 5, 17, 2, 32, 87, 2, 29, 5, 82, 73, 83, 84, - 32, 2, 237, 139, 7, 2, 70, 76, 2, 37, 7, 84, 82, 65, 73, 71, 72, 84, 2, - 171, 20, 32, 26, 104, 4, 65, 86, 69, 32, 185, 1, 17, 82, 73, 83, 84, 32, - 67, 73, 82, 67, 76, 69, 32, 70, 82, 79, 78, 84, 22, 108, 6, 67, 85, 82, - 86, 69, 32, 140, 1, 13, 68, 73, 65, 71, 79, 78, 65, 76, 32, 80, 65, 84, - 72, 235, 8, 72, 12, 48, 4, 68, 79, 85, 66, 1, 4, 84, 82, 73, 80, 6, 85, - 2, 76, 69, 4, 11, 32, 4, 222, 30, 68, 43, 83, 6, 29, 5, 73, 71, 90, 65, - 71, 6, 11, 32, 6, 42, 76, 158, 152, 2, 77, 243, 178, 1, 83, 2, 143, 240, - 6, 65, 10, 40, 4, 79, 83, 69, 32, 215, 253, 6, 69, 8, 26, 67, 46, 78, 35, - 87, 2, 11, 79, 2, 157, 157, 7, 3, 78, 84, 65, 2, 161, 162, 7, 3, 69, 85, - 84, 4, 36, 2, 73, 71, 21, 3, 82, 73, 78, 2, 191, 130, 5, 71, 2, 171, 130, - 5, 75, 72, 56, 7, 79, 84, 65, 84, 73, 79, 78, 225, 22, 2, 85, 66, 66, 60, - 10, 32, 77, 79, 68, 73, 70, 73, 69, 82, 45, 155, 1, 45, 30, 82, 49, 154, - 169, 8, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 14, 150, - 169, 8, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 36, 104, 11, 70, - 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 133, 2, 10, 87, 65, 76, 76, 80, - 76, 65, 78, 69, 32, 18, 100, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 21, - 11, 65, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 6, 17, 2, 76, 69, 7, 45, - 9, 32, 72, 73, 84, 84, 73, 78, 71, 32, 4, 32, 2, 67, 69, 33, 2, 70, 76, - 2, 11, 73, 2, 147, 213, 7, 76, 2, 155, 237, 7, 79, 18, 88, 8, 65, 76, 84, - 69, 82, 78, 65, 84, 44, 4, 68, 79, 85, 66, 1, 4, 83, 73, 78, 71, 6, 72, - 4, 73, 78, 71, 32, 191, 164, 8, 69, 6, 17, 2, 76, 69, 7, 11, 32, 4, 11, - 72, 4, 11, 73, 4, 33, 6, 84, 84, 73, 78, 71, 32, 4, 44, 2, 67, 72, 21, 5, - 70, 82, 79, 78, 84, 2, 135, 220, 4, 69, 2, 253, 158, 6, 2, 32, 87, 30, - 172, 1, 8, 72, 79, 85, 76, 68, 69, 82, 32, 196, 1, 7, 81, 85, 69, 69, 90, - 69, 32, 240, 1, 7, 85, 82, 70, 65, 67, 69, 32, 240, 10, 5, 84, 82, 73, - 75, 69, 243, 221, 4, 69, 6, 100, 4, 72, 73, 80, 32, 173, 232, 3, 15, 84, - 73, 76, 84, 73, 78, 71, 32, 70, 82, 79, 77, 32, 87, 65, 4, 48, 4, 80, 79, - 83, 73, 181, 135, 7, 2, 83, 80, 2, 11, 84, 2, 177, 242, 4, 2, 73, 79, 12, - 48, 6, 70, 76, 73, 67, 75, 32, 18, 76, 35, 83, 2, 211, 16, 65, 4, 129, 1, - 4, 65, 82, 71, 69, 6, 34, 69, 65, 4, 77, 65, 76, 76, 2, 17, 2, 81, 85, 2, - 21, 3, 69, 78, 84, 2, 207, 149, 7, 73, 4, 11, 32, 4, 214, 11, 77, 187, 4, - 83, 4, 22, 83, 135, 11, 66, 2, 201, 204, 1, 4, 89, 77, 66, 79, 82, 58, - 69, 218, 2, 79, 137, 8, 6, 82, 65, 86, 69, 76, 45, 20, 76, 3, 69, 84, 72, - 185, 1, 11, 78, 83, 69, 32, 67, 72, 69, 69, 75, 83, 32, 15, 11, 32, 12, - 56, 3, 79, 78, 32, 86, 77, 177, 5, 4, 66, 73, 84, 69, 8, 56, 4, 76, 73, - 80, 83, 1, 6, 84, 79, 78, 71, 85, 69, 5, 11, 32, 2, 11, 77, 2, 245, 180, - 3, 2, 79, 86, 6, 50, 77, 252, 162, 4, 2, 72, 73, 131, 222, 2, 76, 2, 157, - 149, 3, 2, 73, 68, 28, 76, 5, 78, 71, 85, 69, 32, 196, 4, 4, 82, 83, 79, - 45, 129, 2, 2, 85, 67, 16, 192, 2, 7, 67, 69, 78, 84, 82, 69, 32, 52, 14, - 73, 78, 83, 73, 68, 69, 32, 77, 79, 85, 84, 72, 32, 82, 36, 4, 84, 73, - 80, 32, 88, 7, 76, 73, 67, 75, 73, 78, 71, 128, 244, 6, 14, 83, 84, 73, - 67, 75, 73, 78, 71, 32, 79, 85, 84, 32, 70, 133, 159, 1, 17, 77, 79, 86, - 69, 83, 32, 65, 71, 65, 73, 78, 83, 84, 32, 67, 72, 69, 4, 214, 1, 73, - 193, 166, 3, 5, 83, 84, 73, 67, 75, 2, 225, 183, 3, 4, 69, 76, 65, 88, 4, - 84, 7, 66, 69, 84, 87, 69, 69, 78, 41, 10, 84, 79, 85, 67, 72, 73, 78, - 71, 32, 73, 2, 11, 32, 2, 189, 246, 4, 2, 76, 73, 2, 237, 167, 3, 5, 78, - 83, 73, 68, 69, 6, 120, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 169, - 186, 3, 14, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 84, 87, 73, 4, - 108, 8, 67, 85, 82, 86, 69, 68, 32, 66, 241, 200, 1, 13, 83, 84, 82, 65, - 73, 71, 72, 84, 32, 83, 84, 82, 69, 2, 139, 134, 7, 69, 6, 11, 72, 6, 11, - 32, 6, 30, 66, 34, 77, 187, 4, 83, 2, 157, 194, 6, 3, 69, 84, 87, 2, 149, - 2, 3, 85, 76, 84, 34, 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, - 32, 29, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 14, 178, 1, 82, 151, - 2, 83, 20, 72, 11, 65, 82, 77, 32, 83, 80, 73, 82, 65, 76, 32, 78, 82, - 151, 2, 83, 6, 30, 84, 134, 2, 68, 43, 83, 2, 11, 82, 2, 11, 73, 2, 183, - 239, 6, 80, 12, 41, 8, 79, 84, 65, 84, 73, 79, 78, 45, 12, 52, 5, 70, 76, - 79, 79, 82, 1, 4, 87, 65, 76, 76, 6, 33, 6, 80, 76, 65, 78, 69, 32, 6, - 26, 65, 54, 68, 43, 83, 2, 29, 5, 76, 84, 69, 82, 78, 2, 247, 180, 3, 65, - 2, 17, 2, 79, 85, 2, 183, 237, 6, 66, 2, 139, 237, 6, 73, 2, 11, 72, 2, - 247, 178, 3, 65, 2, 11, 73, 2, 243, 249, 6, 78, 10, 100, 6, 65, 66, 79, - 86, 69, 32, 162, 1, 79, 197, 133, 2, 10, 77, 73, 78, 85, 83, 32, 83, 73, - 77, 73, 4, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 2, - 37, 7, 45, 84, 72, 65, 78, 32, 65, 2, 25, 4, 66, 79, 86, 69, 2, 157, 228, - 1, 2, 32, 69, 4, 203, 252, 5, 82, 250, 1, 64, 3, 71, 76, 69, 212, 4, 5, - 72, 65, 76, 65, 32, 135, 102, 69, 20, 50, 32, 221, 137, 8, 6, 45, 83, 72, - 73, 70, 84, 16, 138, 1, 67, 0, 9, 71, 82, 65, 80, 72, 73, 67, 32, 67, - 116, 2, 72, 73, 74, 76, 36, 4, 82, 73, 71, 72, 145, 208, 4, 4, 83, 72, - 73, 70, 2, 41, 8, 72, 65, 82, 65, 67, 84, 69, 82, 2, 37, 7, 32, 73, 78, - 84, 82, 79, 68, 2, 11, 85, 2, 143, 203, 7, 67, 2, 25, 4, 71, 72, 45, 82, - 2, 185, 1, 7, 69, 86, 69, 82, 83, 69, 68, 4, 32, 2, 69, 70, 109, 2, 79, - 87, 2, 49, 10, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 21, 3, 32, 65, - 78, 2, 17, 2, 71, 76, 2, 31, 69, 2, 17, 2, 45, 57, 2, 11, 32, 2, 11, 81, - 2, 205, 161, 2, 2, 85, 79, 228, 1, 168, 2, 8, 65, 82, 67, 72, 65, 73, 67, - 32, 224, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, - 32, 114, 76, 188, 8, 5, 83, 73, 71, 78, 32, 144, 1, 11, 86, 79, 87, 69, - 76, 32, 83, 73, 71, 78, 32, 229, 212, 2, 17, 80, 85, 78, 67, 84, 85, 65, - 84, 73, 79, 78, 32, 75, 85, 78, 68, 68, 40, 46, 68, 109, 7, 78, 85, 77, - 66, 69, 82, 32, 18, 25, 4, 73, 71, 73, 84, 18, 11, 32, 18, 146, 150, 6, - 70, 30, 83, 42, 84, 190, 83, 78, 14, 79, 199, 110, 69, 22, 150, 218, 2, - 79, 134, 189, 3, 69, 30, 70, 42, 78, 38, 83, 39, 84, 6, 18, 82, 63, 89, - 4, 56, 6, 65, 75, 65, 65, 82, 65, 245, 254, 7, 2, 69, 80, 2, 241, 254, 7, - 3, 65, 78, 83, 138, 1, 60, 6, 69, 84, 84, 69, 82, 32, 217, 171, 2, 3, 73, - 84, 72, 118, 246, 1, 65, 106, 69, 34, 73, 62, 85, 34, 77, 220, 1, 4, 68, - 65, 78, 84, 58, 79, 28, 8, 83, 65, 78, 89, 65, 75, 65, 32, 50, 70, 2, 72, - 2, 82, 2, 86, 2, 89, 32, 8, 84, 65, 65, 76, 85, 74, 65, 32, 29, 9, 75, - 65, 78, 84, 65, 74, 65, 32, 78, 34, 102, 69, 204, 1, 2, 76, 80, 144, 2, - 5, 77, 66, 65, 32, 66, 14, 65, 2, 73, 2, 85, 175, 216, 4, 89, 4, 230, 3, - 69, 175, 216, 4, 89, 12, 46, 76, 2, 82, 154, 3, 73, 175, 216, 4, 89, 4, - 11, 85, 4, 138, 3, 85, 175, 216, 4, 89, 28, 42, 65, 177, 1, 5, 85, 85, - 82, 68, 72, 22, 32, 2, 72, 65, 203, 218, 4, 89, 20, 41, 8, 65, 80, 82, - 65, 65, 78, 65, 32, 20, 70, 84, 138, 1, 68, 22, 66, 2, 67, 2, 71, 2, 74, - 2, 75, 3, 80, 4, 154, 1, 84, 15, 65, 6, 25, 4, 65, 74, 65, 32, 6, 102, - 76, 2, 78, 3, 83, 4, 86, 79, 175, 216, 4, 89, 8, 26, 68, 22, 71, 3, 74, - 4, 18, 68, 15, 65, 2, 11, 65, 2, 171, 216, 4, 89, 6, 26, 78, 21, 2, 83, - 65, 2, 89, 2, 65, 65, 4, 68, 11, 78, 89, 79, 79, 71, 65, 32, 78, 65, 65, - 75, 163, 215, 4, 89, 2, 185, 246, 7, 4, 83, 73, 75, 89, 8, 66, 65, 150, - 146, 4, 67, 193, 227, 3, 6, 86, 73, 83, 65, 82, 71, 4, 220, 113, 5, 76, - 45, 76, 65, 75, 245, 131, 7, 6, 78, 85, 83, 86, 65, 82, 34, 56, 5, 68, - 73, 71, 65, 32, 62, 71, 82, 75, 155, 2, 65, 12, 58, 71, 48, 4, 75, 79, - 77, 66, 118, 65, 26, 73, 19, 80, 4, 11, 65, 4, 236, 2, 3, 69, 84, 84, 67, - 89, 2, 11, 85, 2, 131, 244, 7, 86, 16, 52, 5, 69, 84, 84, 73, 32, 85, 4, - 79, 77, 66, 85, 6, 26, 65, 26, 73, 19, 80, 2, 213, 1, 2, 69, 68, 2, 203, - 1, 83, 2, 175, 1, 65, 10, 40, 2, 86, 65, 173, 198, 7, 2, 32, 68, 9, 29, - 5, 32, 72, 65, 65, 32, 6, 62, 65, 0, 6, 68, 73, 71, 65, 32, 65, 85, 3, - 71, 65, 89, 2, 17, 2, 69, 76, 2, 11, 65, 2, 17, 2, 45, 80, 2, 11, 73, 2, - 183, 209, 7, 76, 2, 193, 202, 5, 5, 65, 78, 85, 75, 73, 12, 84, 2, 32, - 80, 206, 2, 45, 137, 223, 5, 10, 84, 69, 69, 78, 32, 80, 79, 73, 78, 84, - 8, 140, 1, 23, 69, 84, 65, 76, 76, 69, 68, 32, 66, 76, 65, 67, 75, 32, - 65, 78, 68, 32, 87, 72, 73, 84, 69, 49, 7, 79, 73, 78, 84, 69, 68, 32, 2, - 25, 4, 32, 70, 76, 79, 2, 207, 172, 6, 82, 6, 78, 80, 150, 221, 5, 66, - 217, 129, 1, 9, 83, 84, 65, 82, 32, 87, 73, 84, 72, 2, 21, 3, 73, 78, 87, - 2, 149, 221, 5, 4, 72, 69, 69, 76, 2, 179, 194, 3, 80, 12, 46, 73, 102, - 85, 213, 215, 6, 3, 65, 84, 69, 4, 56, 8, 32, 65, 78, 68, 32, 83, 75, 73, - 163, 182, 7, 69, 2, 17, 2, 32, 66, 2, 175, 158, 7, 79, 6, 32, 2, 76, 76, - 143, 235, 7, 78, 5, 11, 32, 2, 49, 10, 65, 78, 68, 32, 67, 82, 79, 83, - 83, 66, 2, 11, 79, 2, 247, 217, 6, 78, 42, 46, 65, 198, 4, 69, 210, 1, - 73, 155, 1, 79, 14, 64, 5, 78, 84, 69, 68, 32, 253, 219, 5, 5, 86, 79, - 78, 73, 67, 12, 148, 1, 12, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, - 32, 229, 1, 19, 78, 79, 82, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, - 84, 72, 32, 72, 79, 8, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, - 83, 83, 4, 29, 5, 45, 84, 72, 65, 78, 5, 11, 32, 2, 25, 4, 87, 73, 84, - 72, 2, 25, 4, 32, 68, 79, 84, 2, 17, 2, 32, 73, 2, 11, 78, 2, 11, 83, 2, - 243, 220, 5, 73, 4, 24, 2, 79, 75, 43, 82, 2, 17, 2, 69, 68, 2, 135, 155, - 5, 32, 2, 29, 5, 73, 90, 79, 78, 84, 2, 11, 65, 2, 203, 187, 5, 76, 12, - 80, 2, 69, 80, 180, 214, 7, 9, 85, 84, 72, 32, 79, 82, 32, 83, 80, 203, - 17, 68, 8, 40, 4, 73, 78, 71, 32, 191, 170, 7, 89, 6, 152, 187, 4, 8, 65, - 67, 67, 79, 77, 77, 79, 68, 190, 153, 2, 83, 255, 85, 70, 6, 76, 9, 67, - 69, 32, 79, 70, 32, 80, 73, 90, 21, 6, 71, 72, 84, 76, 89, 32, 2, 247, - 227, 7, 90, 4, 196, 162, 6, 2, 70, 82, 145, 16, 3, 83, 77, 73, 10, 18, - 80, 75, 84, 6, 140, 186, 4, 9, 73, 78, 71, 32, 76, 65, 82, 71, 69, 199, - 171, 3, 69, 4, 26, 32, 243, 228, 7, 72, 2, 11, 77, 2, 193, 203, 6, 3, 65, - 67, 72, 148, 1, 34, 65, 210, 12, 73, 159, 8, 79, 114, 32, 2, 76, 76, 227, - 212, 6, 83, 112, 30, 32, 193, 11, 2, 69, 82, 108, 178, 2, 65, 44, 3, 66, + 69, 82, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 205, 178, 3, 5, 32, 79, + 86, 69, 82, 2, 21, 3, 32, 83, 84, 2, 227, 146, 10, 69, 2, 25, 4, 32, 79, + 80, 69, 2, 247, 133, 9, 78, 4, 26, 73, 191, 145, 10, 85, 2, 131, 146, 10, + 77, 159, 14, 174, 1, 68, 168, 19, 2, 71, 78, 204, 181, 1, 6, 77, 73, 76, + 65, 82, 32, 158, 2, 78, 202, 24, 88, 241, 226, 6, 15, 76, 72, 79, 85, 69, + 84, 84, 69, 32, 79, 70, 32, 74, 65, 80, 200, 1, 64, 5, 68, 72, 65, 77, + 32, 133, 17, 6, 69, 87, 65, 89, 83, 32, 184, 1, 202, 1, 69, 68, 7, 76, + 69, 84, 84, 69, 82, 32, 176, 4, 15, 82, 69, 80, 69, 84, 73, 84, 73, 79, + 78, 32, 77, 65, 82, 75, 50, 83, 164, 8, 11, 86, 79, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 243, 163, 6, 68, 2, 37, 7, 78, 68, 32, 79, 70, 32, 84, 2, + 189, 195, 9, 2, 69, 88, 102, 214, 1, 65, 98, 84, 186, 177, 6, 68, 158, 1, + 86, 186, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, + 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, + 2, 69, 3, 79, 11, 72, 8, 76, 84, 69, 82, 78, 65, 84, 69, 202, 139, 10, + 65, 2, 73, 3, 85, 2, 223, 230, 9, 32, 14, 134, 1, 72, 204, 225, 5, 19, + 87, 79, 45, 67, 73, 82, 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, + 69, 162, 225, 3, 84, 195, 71, 65, 4, 152, 202, 9, 20, 82, 69, 69, 45, 67, + 73, 82, 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, 69, 147, 64, 65, + 6, 11, 45, 6, 174, 137, 10, 49, 2, 50, 3, 51, 44, 38, 69, 181, 7, 4, 73, + 71, 78, 32, 32, 96, 11, 67, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, 177, + 6, 8, 80, 65, 82, 65, 84, 79, 82, 32, 28, 80, 11, 68, 79, 85, 66, 76, 69, + 32, 82, 73, 78, 71, 57, 5, 87, 73, 84, 72, 32, 5, 11, 32, 2, 129, 219, 8, + 6, 87, 73, 84, 72, 32, 82, 24, 212, 1, 12, 67, 73, 82, 67, 76, 69, 83, + 32, 65, 78, 68, 32, 116, 5, 81, 85, 65, 68, 82, 0, 4, 83, 69, 80, 84, 12, + 16, 82, 65, 89, 83, 32, 65, 78, 68, 32, 68, 79, 84, 84, 69, 68, 32, 46, + 68, 45, 3, 84, 82, 73, 6, 60, 4, 70, 79, 85, 82, 0, 3, 84, 87, 79, 195, + 216, 8, 82, 2, 169, 196, 6, 8, 32, 69, 78, 67, 76, 79, 83, 85, 2, 83, 85, + 6, 42, 68, 28, 3, 84, 82, 73, 215, 1, 67, 2, 197, 1, 3, 79, 85, 66, 2, + 171, 1, 80, 6, 52, 9, 68, 69, 78, 84, 32, 65, 78, 68, 32, 103, 80, 4, + 116, 6, 68, 79, 84, 84, 69, 68, 49, 14, 85, 45, 83, 72, 65, 80, 69, 68, + 32, 79, 82, 78, 65, 77, 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 25, 4, 82, + 69, 83, 67, 2, 251, 185, 1, 69, 4, 150, 224, 8, 66, 131, 81, 68, 12, 210, + 222, 4, 83, 158, 142, 1, 67, 98, 78, 234, 206, 3, 65, 239, 1, 86, 26, 74, + 65, 94, 86, 154, 172, 6, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 10, + 240, 172, 6, 10, 76, 84, 69, 82, 78, 65, 84, 69, 32, 85, 170, 211, 3, 65, + 2, 73, 3, 85, 4, 11, 79, 4, 33, 6, 67, 65, 76, 73, 67, 32, 4, 199, 172, + 6, 82, 16, 56, 5, 66, 76, 65, 67, 75, 1, 5, 87, 72, 73, 84, 69, 8, 11, + 32, 8, 70, 82, 24, 3, 76, 69, 70, 12, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, + 21, 3, 73, 71, 72, 2, 11, 84, 2, 225, 92, 6, 32, 80, 79, 73, 78, 84, 194, + 10, 96, 8, 87, 82, 73, 84, 73, 78, 71, 32, 133, 239, 1, 10, 32, 79, 70, + 32, 84, 72, 69, 32, 72, 79, 192, 10, 136, 3, 4, 65, 73, 82, 32, 192, 1, + 2, 66, 82, 102, 67, 138, 1, 68, 218, 4, 69, 242, 6, 70, 244, 3, 4, 87, + 65, 76, 76, 138, 1, 72, 234, 77, 76, 212, 6, 2, 77, 79, 222, 42, 78, 218, + 1, 82, 202, 7, 83, 162, 5, 84, 180, 10, 5, 71, 82, 65, 83, 80, 184, 5, + 30, 85, 80, 80, 69, 82, 32, 66, 79, 68, 89, 32, 84, 73, 76, 84, 73, 78, + 71, 32, 70, 82, 79, 77, 32, 72, 73, 80, 32, 74, 79, 195, 197, 4, 80, 8, + 48, 4, 66, 76, 79, 87, 29, 4, 83, 85, 67, 75, 4, 58, 32, 155, 221, 4, 73, + 4, 30, 32, 61, 3, 73, 78, 71, 2, 129, 158, 1, 10, 83, 77, 65, 76, 76, 32, + 82, 79, 84, 65, 2, 167, 248, 8, 32, 10, 52, 5, 69, 65, 84, 72, 32, 249, + 169, 1, 2, 85, 83, 4, 144, 164, 2, 2, 69, 88, 1, 2, 73, 78, 10, 40, 6, + 72, 69, 69, 75, 83, 32, 63, 79, 6, 154, 107, 83, 146, 38, 78, 225, 217, + 3, 4, 80, 85, 70, 70, 4, 174, 166, 9, 76, 223, 46, 77, 28, 108, 15, 82, + 69, 65, 77, 89, 32, 69, 89, 69, 66, 82, 79, 87, 83, 32, 165, 1, 7, 89, + 78, 65, 77, 73, 67, 32, 8, 64, 4, 68, 79, 87, 78, 0, 2, 85, 80, 29, 4, + 78, 69, 85, 84, 2, 153, 143, 1, 2, 32, 78, 4, 21, 3, 82, 65, 76, 4, 11, + 32, 4, 194, 58, 68, 187, 185, 9, 85, 20, 180, 1, 11, 69, 86, 69, 82, 89, + 32, 79, 84, 72, 69, 82, 30, 70, 22, 83, 128, 122, 9, 65, 82, 82, 79, 87, + 72, 69, 65, 68, 254, 38, 82, 188, 200, 3, 2, 84, 69, 21, 4, 71, 82, 65, + 68, 2, 181, 218, 8, 2, 32, 84, 2, 151, 136, 6, 65, 6, 68, 11, 73, 77, 85, + 76, 84, 65, 78, 69, 79, 85, 83, 151, 215, 8, 76, 5, 223, 151, 1, 32, 54, + 64, 2, 89, 69, 244, 221, 4, 4, 88, 67, 73, 84, 155, 132, 4, 65, 50, 166, + 1, 32, 56, 15, 66, 82, 79, 87, 83, 32, 83, 84, 82, 65, 73, 71, 72, 84, + 32, 44, 5, 71, 65, 90, 69, 45, 140, 2, 7, 76, 65, 83, 72, 69, 83, 32, 65, + 2, 83, 32, 6, 240, 150, 1, 5, 66, 76, 73, 78, 75, 175, 247, 4, 87, 6, + 186, 53, 68, 154, 84, 78, 163, 229, 8, 85, 18, 100, 11, 70, 76, 79, 79, + 82, 80, 76, 65, 78, 69, 32, 25, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, + 32, 8, 82, 83, 207, 45, 67, 10, 18, 67, 43, 83, 4, 254, 45, 85, 213, 95, + 3, 73, 82, 67, 6, 37, 7, 84, 82, 65, 73, 71, 72, 84, 7, 11, 32, 4, 222, + 163, 1, 65, 55, 68, 6, 130, 51, 68, 252, 167, 4, 4, 70, 76, 85, 84, 191, + 145, 5, 85, 14, 112, 5, 72, 65, 76, 70, 32, 26, 67, 28, 4, 87, 73, 68, + 69, 230, 92, 79, 177, 130, 4, 6, 83, 81, 85, 69, 69, 90, 4, 22, 67, 131, + 93, 79, 2, 225, 232, 2, 2, 76, 79, 4, 162, 67, 32, 217, 61, 4, 78, 73, + 78, 71, 38, 204, 1, 28, 65, 67, 69, 32, 68, 73, 82, 69, 67, 84, 73, 79, + 78, 32, 80, 79, 83, 73, 84, 73, 79, 78, 32, 78, 79, 83, 69, 32, 138, 1, + 73, 82, 76, 176, 1, 8, 79, 82, 69, 72, 69, 65, 68, 32, 135, 148, 7, 85, + 6, 88, 10, 85, 80, 32, 79, 82, 32, 68, 79, 87, 78, 13, 8, 70, 79, 82, 87, + 65, 82, 68, 32, 5, 11, 32, 2, 225, 225, 4, 3, 84, 73, 76, 12, 240, 225, + 3, 11, 76, 76, 32, 77, 79, 68, 73, 70, 73, 69, 82, 151, 184, 2, 78, 12, + 44, 4, 73, 67, 75, 32, 29, 3, 79, 79, 82, 10, 146, 141, 1, 76, 35, 83, 2, + 221, 233, 8, 20, 80, 76, 65, 78, 69, 32, 83, 72, 79, 85, 76, 68, 69, 82, + 32, 72, 73, 80, 32, 77, 6, 166, 89, 87, 222, 38, 67, 47, 78, 156, 4, 34, + 65, 133, 75, 3, 69, 65, 68, 140, 4, 36, 3, 78, 68, 45, 139, 172, 9, 73, + 138, 4, 92, 5, 65, 78, 71, 76, 69, 138, 5, 67, 150, 10, 70, 186, 42, 72, + 241, 12, 4, 79, 86, 65, 76, 37, 11, 32, 34, 188, 1, 5, 73, 78, 68, 69, + 88, 176, 1, 7, 76, 73, 84, 84, 76, 69, 32, 136, 1, 5, 82, 73, 78, 71, 32, + 173, 66, 18, 77, 73, 68, 68, 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, 84, + 84, 76, 69, 17, 11, 32, 14, 128, 1, 7, 77, 73, 68, 68, 76, 69, 32, 228, + 24, 11, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 241, 42, 5, 84, 72, + 85, 77, 66, 4, 174, 70, 76, 243, 201, 8, 82, 8, 44, 5, 73, 78, 68, 69, + 88, 203, 224, 9, 85, 7, 145, 24, 18, 32, 84, 72, 85, 77, 66, 32, 73, 78, + 68, 69, 88, 32, 84, 72, 85, 77, 66, 4, 108, 22, 68, 79, 87, 78, 32, 77, + 73, 68, 68, 76, 69, 32, 84, 72, 85, 77, 66, 32, 73, 78, 68, 69, 155, 68, + 76, 2, 223, 157, 8, 88, 88, 64, 5, 73, 82, 67, 76, 69, 184, 3, 3, 76, 65, + 87, 183, 2, 85, 35, 11, 32, 32, 116, 5, 73, 78, 68, 69, 88, 200, 1, 7, + 76, 73, 84, 84, 76, 69, 32, 36, 7, 77, 73, 68, 68, 76, 69, 32, 163, 64, + 82, 21, 11, 32, 18, 72, 6, 77, 73, 68, 68, 76, 69, 198, 26, 72, 242, 38, + 82, 255, 215, 8, 66, 13, 11, 32, 10, 64, 5, 67, 82, 79, 83, 83, 198, 64, + 84, 82, 76, 243, 201, 8, 82, 4, 134, 65, 32, 227, 212, 8, 69, 4, 210, + 181, 8, 73, 139, 166, 1, 85, 6, 252, 47, 10, 82, 73, 78, 71, 32, 76, 73, + 84, 84, 76, 187, 171, 9, 85, 17, 11, 32, 14, 144, 2, 28, 77, 73, 68, 68, + 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 67, 79, 78, + 74, 79, 73, 78, 69, 68, 176, 49, 2, 70, 79, 198, 11, 78, 162, 1, 84, 217, + 118, 23, 73, 78, 68, 69, 88, 32, 84, 72, 85, 77, 66, 32, 67, 85, 82, 86, + 69, 32, 84, 72, 85, 77, 66, 5, 139, 181, 1, 32, 38, 46, 80, 149, 2, 6, + 82, 76, 73, 67, 85, 69, 31, 11, 32, 28, 188, 1, 5, 73, 78, 68, 69, 88, + 56, 19, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, + 69, 65, 68, 196, 50, 7, 77, 73, 68, 68, 76, 69, 32, 18, 79, 218, 7, 78, + 163, 1, 84, 9, 11, 32, 6, 40, 5, 84, 72, 85, 77, 66, 243, 58, 82, 5, 215, + 46, 32, 9, 11, 32, 6, 72, 5, 73, 78, 68, 69, 88, 0, 6, 77, 73, 68, 68, + 76, 69, 179, 71, 79, 2, 189, 133, 9, 13, 32, 82, 73, 78, 71, 32, 76, 73, + 84, 84, 76, 69, 32, 172, 2, 44, 3, 73, 83, 84, 177, 33, 3, 76, 65, 84, + 247, 1, 11, 32, 244, 1, 160, 2, 5, 73, 78, 68, 69, 88, 192, 16, 7, 76, + 73, 84, 84, 76, 69, 32, 196, 2, 7, 77, 73, 68, 68, 76, 69, 32, 200, 4, 5, + 82, 73, 78, 71, 32, 132, 2, 5, 84, 72, 85, 77, 66, 138, 2, 72, 205, 9, + 22, 70, 79, 85, 82, 32, 70, 73, 78, 71, 69, 82, 83, 32, 67, 79, 78, 74, + 79, 73, 78, 69, 68, 137, 1, 11, 32, 134, 1, 232, 1, 4, 66, 69, 78, 84, + 36, 6, 72, 73, 78, 71, 69, 68, 76, 6, 77, 73, 68, 68, 76, 69, 168, 7, 2, + 67, 85, 64, 6, 84, 72, 85, 77, 66, 32, 160, 5, 16, 85, 80, 32, 77, 73, + 68, 68, 76, 69, 32, 72, 73, 78, 71, 69, 68, 151, 4, 82, 5, 217, 45, 5, + 32, 79, 86, 69, 82, 9, 11, 32, 6, 252, 20, 8, 77, 73, 68, 68, 76, 69, 32, + 85, 167, 160, 8, 76, 71, 11, 32, 68, 236, 1, 4, 66, 69, 78, 84, 42, 67, + 244, 1, 10, 85, 80, 32, 83, 80, 82, 69, 65, 68, 32, 100, 6, 72, 73, 78, + 71, 69, 68, 46, 82, 80, 5, 84, 72, 85, 77, 66, 188, 28, 14, 83, 84, 82, + 65, 73, 71, 72, 84, 32, 84, 72, 85, 77, 66, 227, 17, 76, 5, 213, 92, 6, + 32, 84, 72, 85, 77, 66, 28, 68, 8, 79, 78, 74, 79, 73, 78, 69, 68, 233, + 1, 4, 82, 79, 83, 83, 23, 11, 32, 20, 144, 1, 6, 67, 85, 80, 80, 69, 68, + 28, 6, 84, 72, 85, 77, 66, 32, 68, 5, 72, 73, 78, 71, 69, 166, 22, 73, + 165, 7, 6, 77, 73, 68, 68, 76, 69, 5, 11, 32, 2, 203, 27, 84, 8, 160, 1, + 4, 83, 73, 68, 69, 219, 61, 70, 6, 22, 69, 159, 47, 32, 4, 223, 15, 68, + 5, 241, 30, 7, 32, 83, 80, 82, 69, 65, 68, 8, 32, 3, 73, 78, 71, 247, 19, + 65, 7, 11, 32, 4, 138, 36, 67, 255, 225, 8, 66, 19, 11, 32, 16, 64, 6, + 65, 78, 71, 76, 69, 68, 22, 67, 106, 72, 159, 132, 9, 66, 5, 247, 211, 5, + 32, 6, 74, 85, 230, 7, 73, 241, 25, 10, 79, 78, 74, 79, 73, 78, 69, 68, + 32, 72, 2, 145, 188, 4, 2, 80, 80, 4, 194, 33, 73, 217, 26, 2, 79, 79, + 40, 164, 1, 7, 65, 78, 71, 76, 69, 68, 32, 46, 67, 220, 1, 14, 70, 79, + 82, 87, 65, 82, 68, 32, 73, 78, 68, 69, 88, 32, 32, 4, 72, 79, 79, 75, + 53, 4, 83, 73, 68, 69, 4, 128, 1, 2, 73, 78, 1, 3, 79, 85, 84, 12, 36, 5, + 73, 82, 67, 76, 69, 15, 85, 5, 47, 68, 8, 32, 4, 80, 80, 69, 68, 39, 82, + 2, 225, 40, 5, 32, 77, 73, 68, 68, 6, 56, 9, 86, 69, 32, 84, 72, 85, 77, + 66, 32, 143, 37, 76, 4, 182, 160, 1, 73, 131, 188, 4, 85, 4, 202, 84, 83, + 255, 171, 8, 66, 7, 157, 8, 9, 69, 68, 32, 77, 73, 68, 68, 76, 69, 15, + 11, 32, 12, 92, 6, 73, 78, 68, 69, 88, 32, 156, 13, 5, 84, 72, 85, 77, + 66, 193, 8, 4, 66, 79, 84, 72, 4, 26, 72, 239, 254, 8, 66, 2, 231, 140, + 8, 73, 7, 37, 7, 32, 84, 72, 85, 77, 66, 32, 4, 178, 28, 67, 227, 129, 1, + 83, 22, 88, 4, 68, 79, 87, 78, 186, 1, 84, 158, 6, 82, 130, 21, 73, 226, + 224, 8, 66, 159, 67, 85, 9, 11, 32, 6, 80, 9, 79, 84, 72, 69, 82, 83, 32, + 67, 73, 25, 7, 82, 73, 80, 80, 76, 69, 32, 2, 225, 51, 2, 82, 67, 4, 22, + 67, 171, 80, 83, 2, 11, 85, 2, 241, 179, 4, 2, 82, 86, 4, 196, 35, 6, 79, + 85, 67, 72, 69, 83, 39, 72, 30, 134, 1, 82, 28, 6, 84, 72, 85, 77, 66, + 32, 138, 3, 85, 134, 1, 68, 210, 30, 76, 205, 244, 7, 9, 66, 69, 78, 84, + 32, 79, 86, 69, 82, 4, 238, 4, 65, 231, 29, 73, 16, 80, 7, 65, 78, 71, + 76, 69, 68, 32, 126, 67, 124, 4, 72, 79, 79, 75, 147, 32, 76, 6, 60, 10, + 79, 85, 84, 32, 73, 78, 68, 69, 88, 32, 215, 1, 73, 4, 26, 67, 151, 188, + 9, 85, 2, 133, 186, 2, 3, 82, 79, 83, 6, 76, 12, 73, 82, 67, 76, 69, 68, + 32, 73, 78, 68, 69, 88, 45, 3, 85, 80, 80, 4, 11, 32, 4, 150, 21, 72, + 131, 166, 9, 85, 2, 25, 4, 69, 68, 32, 73, 2, 233, 30, 4, 78, 68, 69, 88, + 4, 11, 80, 5, 175, 15, 32, 18, 102, 68, 20, 6, 77, 73, 68, 68, 76, 69, + 42, 82, 198, 29, 84, 82, 76, 226, 244, 7, 73, 139, 166, 1, 85, 2, 175, + 229, 7, 79, 7, 11, 32, 4, 206, 3, 82, 179, 16, 67, 2, 11, 65, 2, 37, 7, + 73, 83, 69, 68, 32, 75, 78, 2, 11, 85, 2, 11, 67, 2, 147, 148, 8, 75, 35, + 11, 32, 32, 148, 1, 8, 66, 69, 84, 87, 69, 69, 78, 32, 102, 72, 20, 5, + 79, 86, 69, 82, 32, 120, 4, 83, 73, 68, 69, 100, 6, 85, 78, 68, 69, 82, + 32, 207, 40, 70, 8, 80, 12, 73, 78, 68, 69, 88, 32, 77, 73, 68, 68, 76, + 69, 246, 20, 77, 155, 6, 82, 5, 135, 70, 32, 2, 187, 169, 4, 69, 4, 52, + 6, 70, 79, 85, 82, 32, 82, 161, 2, 2, 84, 87, 2, 11, 65, 2, 233, 80, 9, + 73, 83, 69, 68, 32, 75, 78, 85, 67, 6, 11, 32, 6, 38, 68, 190, 15, 67, + 255, 225, 8, 66, 2, 25, 4, 73, 65, 71, 79, 2, 131, 171, 8, 78, 10, 54, + 73, 34, 84, 48, 4, 70, 79, 85, 82, 131, 23, 76, 2, 161, 7, 4, 78, 68, 69, + 88, 4, 34, 87, 13, 4, 72, 82, 69, 69, 2, 11, 79, 2, 197, 164, 8, 5, 32, + 70, 73, 78, 71, 55, 11, 32, 52, 194, 1, 70, 172, 3, 4, 72, 69, 69, 76, + 192, 1, 6, 83, 80, 76, 73, 84, 32, 236, 1, 6, 84, 72, 85, 77, 66, 32, + 165, 244, 4, 16, 66, 69, 84, 87, 69, 69, 78, 32, 80, 65, 76, 77, 32, 70, + 65, 67, 24, 140, 1, 18, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, + 83, 80, 82, 69, 65, 68, 165, 1, 11, 79, 85, 82, 32, 70, 73, 78, 71, 69, + 82, 83, 15, 11, 32, 12, 68, 6, 72, 73, 78, 71, 69, 68, 42, 84, 186, 2, + 70, 199, 233, 8, 66, 7, 11, 32, 4, 190, 4, 84, 155, 15, 78, 2, 189, 35, + 6, 72, 85, 77, 66, 32, 70, 11, 11, 32, 8, 72, 9, 67, 79, 78, 74, 79, 73, + 78, 69, 68, 154, 8, 72, 231, 226, 8, 66, 5, 221, 145, 9, 3, 32, 83, 80, + 11, 11, 32, 8, 96, 19, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, + 32, 83, 80, 82, 69, 65, 68, 151, 2, 84, 7, 11, 32, 4, 26, 70, 199, 233, + 8, 66, 2, 21, 3, 79, 85, 82, 2, 167, 1, 32, 10, 72, 6, 67, 69, 78, 84, + 82, 69, 96, 5, 73, 78, 68, 69, 88, 167, 16, 76, 7, 49, 10, 32, 84, 72, + 85, 77, 66, 32, 83, 73, 68, 4, 11, 69, 5, 11, 32, 2, 131, 232, 8, 66, 2, + 11, 32, 2, 11, 84, 2, 173, 135, 1, 5, 72, 85, 77, 66, 32, 6, 242, 30, 70, + 162, 104, 83, 159, 224, 7, 66, 82, 48, 4, 73, 78, 71, 69, 181, 9, 3, 79, + 79, 75, 63, 11, 32, 60, 206, 1, 70, 172, 1, 5, 73, 78, 68, 69, 88, 196, + 2, 6, 76, 73, 84, 84, 76, 69, 80, 6, 77, 73, 68, 68, 76, 69, 30, 79, 64, + 4, 82, 73, 78, 71, 124, 6, 84, 72, 85, 77, 66, 32, 158, 6, 78, 139, 142, + 4, 83, 4, 92, 19, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, + 82, 69, 65, 68, 32, 19, 79, 2, 191, 25, 79, 2, 249, 1, 11, 85, 82, 32, + 70, 73, 78, 71, 69, 82, 83, 32, 23, 11, 32, 20, 86, 72, 40, 7, 77, 73, + 68, 68, 76, 69, 32, 104, 5, 84, 72, 85, 77, 66, 219, 9, 82, 2, 11, 73, 2, + 177, 154, 4, 2, 78, 71, 6, 36, 4, 82, 73, 78, 71, 203, 10, 76, 5, 11, 32, + 2, 11, 67, 2, 21, 3, 79, 78, 74, 2, 175, 33, 79, 11, 11, 32, 8, 34, 83, + 210, 22, 79, 203, 39, 76, 4, 190, 148, 7, 73, 195, 8, 77, 9, 11, 32, 6, + 22, 73, 199, 8, 84, 4, 25, 4, 78, 68, 69, 88, 5, 155, 8, 32, 5, 11, 32, + 2, 171, 8, 82, 8, 21, 3, 80, 69, 78, 9, 11, 32, 6, 178, 7, 78, 163, 1, + 84, 5, 97, 22, 32, 68, 79, 87, 78, 32, 73, 78, 68, 69, 88, 32, 84, 72, + 85, 77, 66, 32, 72, 79, 79, 75, 2, 177, 77, 2, 32, 77, 6, 68, 9, 66, 69, + 84, 87, 69, 69, 78, 32, 77, 53, 4, 83, 73, 68, 69, 2, 29, 5, 73, 68, 68, + 76, 69, 2, 163, 148, 4, 32, 5, 33, 6, 32, 84, 79, 85, 67, 72, 2, 185, + 250, 7, 3, 73, 78, 71, 21, 11, 32, 18, 172, 1, 4, 67, 85, 82, 76, 28, 18, + 73, 78, 68, 69, 88, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, + 48, 7, 77, 73, 68, 68, 76, 69, 32, 225, 2, 4, 82, 73, 78, 71, 2, 253, + 147, 5, 2, 73, 67, 6, 142, 183, 5, 85, 142, 238, 2, 79, 243, 41, 73, 8, + 104, 21, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 67, 79, 78, 74, + 79, 73, 78, 69, 68, 143, 2, 84, 7, 223, 228, 2, 32, 17, 11, 32, 14, 112, + 14, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 22, 76, 66, + 78, 70, 82, 94, 84, 183, 244, 7, 73, 2, 215, 160, 2, 80, 2, 21, 3, 73, + 84, 84, 2, 17, 2, 76, 69, 2, 135, 166, 8, 32, 2, 11, 79, 2, 11, 32, 2, + 11, 84, 2, 11, 72, 2, 247, 251, 5, 85, 2, 11, 73, 2, 21, 3, 78, 71, 32, + 2, 11, 76, 2, 11, 73, 2, 11, 84, 2, 179, 246, 7, 84, 4, 29, 5, 72, 85, + 77, 66, 32, 4, 194, 14, 70, 163, 104, 83, 17, 11, 32, 14, 56, 8, 77, 79, + 86, 69, 77, 69, 78, 84, 247, 150, 8, 82, 12, 26, 45, 163, 199, 7, 32, 10, + 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 25, 10, 87, 65, 76, + 76, 80, 76, 65, 78, 69, 32, 4, 62, 67, 223, 40, 83, 6, 38, 67, 28, 2, 84, + 73, 195, 40, 83, 2, 169, 155, 8, 2, 85, 82, 2, 183, 251, 8, 76, 38, 50, + 73, 225, 3, 7, 79, 67, 65, 84, 73, 79, 78, 22, 32, 3, 77, 66, 32, 171, 1, + 80, 16, 56, 4, 67, 79, 77, 66, 29, 6, 76, 69, 78, 71, 84, 72, 2, 193, + 212, 5, 2, 73, 78, 14, 11, 45, 14, 146, 150, 9, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 3, 55, 6, 58, 32, 153, 1, 9, 83, 32, 80, 82, 69, 83, 83, + 69, 68, 4, 116, 12, 76, 79, 87, 69, 82, 32, 79, 86, 69, 82, 32, 85, 133, + 251, 7, 11, 85, 80, 80, 69, 82, 32, 79, 86, 69, 82, 32, 2, 11, 80, 2, + 163, 214, 8, 80, 2, 29, 5, 32, 84, 79, 71, 69, 2, 11, 84, 2, 227, 213, 8, + 72, 16, 22, 32, 203, 1, 45, 12, 148, 1, 2, 72, 69, 236, 177, 2, 3, 84, + 79, 82, 144, 222, 4, 3, 68, 69, 80, 0, 3, 87, 73, 68, 229, 109, 10, 76, + 73, 77, 66, 83, 32, 68, 73, 71, 73, 4, 196, 13, 4, 65, 68, 32, 78, 131, + 218, 8, 73, 4, 52, 5, 70, 76, 79, 79, 82, 1, 4, 87, 65, 76, 76, 2, 221, + 210, 8, 5, 80, 76, 65, 78, 69, 156, 3, 64, 4, 85, 84, 72, 32, 161, 5, 7, + 86, 69, 77, 69, 78, 84, 45, 54, 186, 1, 67, 88, 5, 70, 82, 79, 87, 78, 0, + 5, 83, 77, 73, 76, 69, 56, 4, 75, 73, 83, 83, 36, 5, 79, 80, 69, 78, 32, + 192, 1, 5, 84, 69, 78, 83, 69, 165, 29, 5, 87, 82, 73, 78, 75, 8, 48, 6, + 76, 79, 83, 69, 68, 32, 163, 223, 7, 79, 6, 222, 2, 70, 142, 38, 67, 47, + 78, 7, 11, 32, 4, 22, 79, 203, 1, 87, 2, 147, 186, 7, 80, 7, 11, 32, 4, + 166, 1, 87, 83, 70, 18, 100, 4, 79, 86, 65, 76, 0, 9, 82, 69, 67, 84, 65, + 78, 71, 76, 69, 42, 87, 82, 70, 235, 185, 7, 67, 7, 11, 32, 4, 26, 87, + 151, 184, 7, 89, 2, 25, 4, 82, 73, 78, 75, 2, 203, 128, 4, 76, 7, 11, 32, + 4, 18, 70, 43, 83, 2, 17, 2, 79, 82, 2, 215, 244, 7, 87, 2, 17, 2, 85, + 67, 2, 219, 255, 3, 75, 230, 2, 192, 1, 9, 68, 73, 65, 71, 79, 78, 65, + 76, 32, 148, 1, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 184, 14, + 6, 72, 73, 78, 71, 69, 32, 189, 2, 10, 87, 65, 76, 76, 80, 76, 65, 78, + 69, 32, 32, 56, 8, 66, 69, 84, 87, 69, 69, 78, 32, 22, 65, 31, 84, 16, + 18, 65, 31, 84, 8, 229, 28, 3, 87, 65, 89, 8, 201, 28, 6, 79, 87, 65, 82, + 68, 83, 150, 1, 208, 2, 24, 65, 82, 77, 32, 67, 73, 82, 67, 76, 69, 32, + 72, 73, 84, 84, 73, 78, 71, 32, 87, 65, 76, 76, 32, 38, 66, 34, 67, 224, + 1, 8, 70, 73, 78, 71, 69, 82, 32, 67, 52, 5, 72, 85, 77, 80, 32, 184, 3, + 5, 76, 79, 79, 80, 32, 198, 1, 83, 108, 7, 84, 82, 73, 80, 76, 69, 32, + 110, 87, 206, 12, 68, 198, 2, 80, 154, 6, 90, 227, 232, 7, 74, 12, 178, + 7, 76, 154, 9, 77, 39, 83, 8, 150, 17, 79, 175, 230, 7, 69, 28, 86, 72, + 20, 5, 85, 82, 86, 69, 32, 196, 29, 5, 79, 82, 78, 69, 82, 227, 165, 7, + 82, 2, 211, 212, 7, 69, 18, 80, 4, 67, 79, 77, 66, 158, 8, 72, 230, 15, + 76, 202, 172, 2, 77, 183, 177, 1, 83, 2, 11, 73, 2, 131, 248, 3, 78, 6, + 128, 9, 6, 73, 82, 67, 76, 69, 83, 239, 20, 79, 18, 56, 8, 72, 73, 84, + 84, 73, 78, 71, 32, 239, 244, 3, 83, 16, 72, 8, 67, 69, 73, 76, 73, 78, + 71, 32, 101, 6, 70, 76, 79, 79, 82, 32, 8, 56, 5, 76, 65, 82, 71, 69, 1, + 5, 83, 77, 65, 76, 76, 4, 11, 32, 4, 226, 54, 84, 135, 2, 68, 8, 88, 4, + 83, 77, 65, 76, 12, 5, 76, 65, 82, 71, 69, 17, 7, 84, 82, 73, 80, 76, 69, + 32, 2, 11, 76, 2, 255, 18, 32, 4, 56, 5, 76, 65, 82, 71, 69, 1, 5, 83, + 77, 65, 76, 76, 2, 145, 53, 2, 32, 84, 18, 56, 8, 72, 73, 84, 84, 73, 78, + 71, 32, 183, 241, 3, 83, 16, 64, 7, 67, 69, 73, 76, 73, 78, 71, 1, 5, 70, + 76, 79, 79, 82, 8, 11, 32, 8, 22, 76, 191, 9, 83, 4, 249, 22, 4, 65, 82, + 71, 69, 12, 44, 6, 72, 65, 75, 73, 78, 71, 243, 16, 73, 2, 21, 3, 32, 80, + 65, 2, 221, 240, 3, 4, 82, 65, 76, 76, 8, 76, 12, 65, 76, 84, 69, 82, 78, + 65, 84, 73, 78, 71, 32, 138, 18, 87, 63, 83, 4, 134, 18, 87, 147, 21, 77, + 18, 80, 4, 65, 86, 69, 32, 169, 1, 11, 82, 73, 83, 84, 32, 67, 73, 82, + 67, 76, 69, 14, 30, 72, 102, 83, 171, 20, 76, 8, 37, 7, 73, 84, 84, 73, + 78, 71, 32, 8, 252, 2, 4, 67, 69, 73, 76, 25, 5, 70, 76, 79, 79, 82, 4, + 170, 146, 5, 78, 231, 224, 1, 77, 4, 209, 5, 10, 32, 72, 73, 84, 84, 73, + 78, 71, 32, 87, 14, 112, 3, 85, 80, 32, 184, 1, 6, 68, 79, 87, 78, 32, + 83, 149, 225, 5, 10, 83, 73, 68, 69, 32, 84, 79, 32, 83, 73, 10, 40, 5, + 68, 79, 87, 78, 32, 143, 1, 83, 8, 68, 8, 65, 76, 84, 69, 82, 78, 65, 84, + 230, 17, 76, 215, 216, 3, 83, 4, 21, 3, 73, 78, 71, 4, 11, 32, 4, 190, + 17, 76, 215, 216, 3, 83, 2, 207, 30, 69, 162, 1, 148, 2, 11, 65, 82, 77, + 32, 67, 73, 82, 67, 76, 69, 32, 98, 66, 54, 67, 174, 4, 68, 100, 8, 70, + 73, 78, 71, 69, 82, 32, 67, 64, 5, 72, 85, 77, 80, 32, 60, 5, 76, 79, 79, + 80, 32, 102, 80, 34, 83, 216, 1, 7, 84, 82, 73, 80, 76, 69, 32, 218, 1, + 87, 202, 2, 90, 227, 232, 7, 74, 8, 18, 77, 39, 83, 4, 225, 13, 5, 69, + 68, 73, 85, 77, 4, 11, 77, 4, 177, 13, 3, 65, 76, 76, 12, 34, 79, 185, + 13, 3, 69, 78, 68, 6, 183, 13, 88, 46, 100, 6, 79, 82, 78, 69, 82, 32, + 88, 4, 85, 82, 86, 69, 232, 11, 4, 72, 69, 67, 75, 227, 165, 7, 82, 8, + 54, 82, 194, 12, 76, 162, 167, 2, 77, 183, 177, 1, 83, 2, 11, 79, 2, 239, + 176, 5, 84, 30, 50, 32, 137, 2, 7, 68, 32, 67, 82, 79, 83, 83, 26, 66, + 72, 68, 2, 84, 72, 133, 5, 7, 81, 85, 65, 82, 84, 69, 82, 12, 196, 5, 10, + 65, 76, 70, 45, 67, 73, 82, 67, 76, 69, 147, 14, 73, 6, 96, 2, 69, 78, + 29, 18, 82, 69, 69, 45, 81, 85, 65, 82, 84, 69, 82, 32, 67, 73, 82, 67, + 76, 69, 2, 11, 32, 2, 131, 1, 83, 4, 11, 32, 4, 246, 176, 2, 77, 183, + 177, 1, 83, 8, 33, 6, 79, 85, 66, 76, 69, 32, 8, 30, 83, 150, 4, 65, 75, + 87, 2, 209, 196, 8, 3, 84, 82, 65, 6, 32, 3, 73, 82, 67, 151, 9, 79, 4, + 173, 7, 3, 76, 69, 83, 10, 142, 8, 76, 166, 8, 72, 254, 158, 2, 77, 183, + 177, 1, 83, 12, 68, 5, 83, 77, 65, 76, 76, 142, 7, 76, 166, 8, 72, 255, + 158, 2, 77, 5, 11, 32, 2, 227, 36, 68, 6, 181, 6, 4, 69, 65, 75, 83, 12, + 22, 73, 231, 36, 72, 10, 29, 5, 78, 71, 76, 69, 32, 10, 52, 8, 83, 84, + 82, 65, 73, 71, 72, 84, 207, 1, 87, 8, 11, 32, 8, 42, 76, 202, 172, 2, + 77, 183, 177, 1, 83, 4, 25, 4, 65, 82, 71, 69, 5, 147, 207, 8, 83, 8, 26, + 65, 74, 87, 63, 83, 4, 49, 10, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 5, + 17, 2, 32, 87, 2, 29, 5, 82, 73, 83, 84, 32, 2, 165, 196, 7, 2, 70, 76, + 2, 37, 7, 84, 82, 65, 73, 71, 72, 84, 2, 159, 20, 32, 26, 104, 4, 65, 86, + 69, 32, 185, 1, 17, 82, 73, 83, 84, 32, 67, 73, 82, 67, 76, 69, 32, 70, + 82, 79, 78, 84, 22, 108, 6, 67, 85, 82, 86, 69, 32, 140, 1, 13, 68, 73, + 65, 71, 79, 78, 65, 76, 32, 80, 65, 84, 72, 223, 8, 72, 12, 48, 4, 68, + 79, 85, 66, 1, 4, 84, 82, 73, 80, 6, 85, 2, 76, 69, 4, 11, 32, 4, 210, + 30, 68, 43, 83, 6, 29, 5, 73, 71, 90, 65, 71, 6, 11, 32, 6, 42, 76, 162, + 167, 2, 77, 183, 177, 1, 83, 2, 175, 166, 7, 65, 10, 40, 4, 79, 83, 69, + 32, 135, 182, 7, 69, 8, 26, 67, 46, 78, 35, 87, 2, 11, 79, 2, 217, 214, + 7, 3, 78, 84, 65, 2, 221, 219, 7, 3, 69, 85, 84, 4, 44, 3, 82, 73, 78, + 241, 157, 1, 2, 73, 71, 2, 151, 167, 5, 75, 72, 56, 7, 79, 84, 65, 84, + 73, 79, 78, 225, 22, 2, 85, 66, 66, 60, 10, 32, 77, 79, 68, 73, 70, 73, + 69, 82, 45, 155, 1, 45, 30, 82, 49, 250, 226, 8, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 2, 55, 2, 56, 3, 57, 14, 246, 226, 8, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 3, 54, 36, 104, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, + 32, 133, 2, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 18, 100, 4, 68, + 79, 85, 66, 0, 4, 83, 73, 78, 71, 21, 11, 65, 76, 84, 69, 82, 78, 65, 84, + 73, 78, 71, 6, 17, 2, 76, 69, 7, 45, 9, 32, 72, 73, 84, 84, 73, 78, 71, + 32, 4, 32, 2, 67, 69, 33, 2, 70, 76, 2, 11, 73, 2, 231, 142, 8, 76, 2, + 239, 166, 8, 79, 18, 88, 8, 65, 76, 84, 69, 82, 78, 65, 84, 44, 4, 68, + 79, 85, 66, 1, 4, 83, 73, 78, 71, 6, 72, 4, 73, 78, 71, 32, 159, 222, 8, + 69, 6, 17, 2, 76, 69, 7, 11, 32, 4, 11, 72, 4, 11, 73, 4, 33, 6, 84, 84, + 73, 78, 71, 32, 4, 44, 2, 67, 72, 21, 5, 70, 82, 79, 78, 84, 2, 179, 243, + 4, 69, 2, 205, 213, 6, 2, 32, 87, 30, 172, 1, 8, 72, 79, 85, 76, 68, 69, + 82, 32, 196, 1, 7, 81, 85, 69, 69, 90, 69, 32, 240, 1, 7, 85, 82, 70, 65, + 67, 69, 32, 240, 10, 5, 84, 82, 73, 75, 69, 243, 129, 5, 69, 6, 100, 4, + 72, 73, 80, 32, 129, 246, 3, 15, 84, 73, 76, 84, 73, 78, 71, 32, 70, 82, + 79, 77, 32, 87, 65, 4, 48, 4, 80, 79, 83, 73, 253, 192, 7, 2, 83, 80, 2, + 11, 84, 2, 205, 150, 5, 2, 73, 79, 12, 48, 6, 70, 76, 73, 67, 75, 32, 18, + 76, 35, 83, 2, 211, 16, 65, 4, 129, 1, 4, 65, 82, 71, 69, 6, 34, 69, 65, + 4, 77, 65, 76, 76, 2, 17, 2, 81, 85, 2, 21, 3, 69, 78, 84, 2, 151, 207, + 7, 73, 4, 11, 32, 4, 214, 11, 77, 187, 4, 83, 4, 22, 83, 135, 11, 66, 2, + 249, 207, 1, 4, 89, 77, 66, 79, 82, 58, 69, 218, 2, 79, 137, 8, 6, 82, + 65, 86, 69, 76, 45, 20, 76, 3, 69, 84, 72, 185, 1, 11, 78, 83, 69, 32, + 67, 72, 69, 69, 75, 83, 32, 15, 11, 32, 12, 56, 3, 79, 78, 32, 86, 77, + 177, 5, 4, 66, 73, 84, 69, 8, 56, 4, 76, 73, 80, 83, 1, 6, 84, 79, 78, + 71, 85, 69, 5, 11, 32, 2, 11, 77, 2, 201, 194, 3, 2, 79, 86, 6, 50, 77, + 160, 176, 4, 2, 72, 73, 167, 138, 3, 76, 2, 213, 163, 3, 2, 73, 68, 28, + 76, 5, 78, 71, 85, 69, 32, 196, 4, 4, 82, 83, 79, 45, 129, 2, 2, 85, 67, + 16, 192, 2, 7, 67, 69, 78, 84, 82, 69, 32, 52, 14, 73, 78, 83, 73, 68, + 69, 32, 77, 79, 85, 84, 72, 32, 82, 36, 4, 84, 73, 80, 32, 88, 7, 76, 73, + 67, 75, 73, 78, 71, 208, 174, 7, 14, 83, 84, 73, 67, 75, 73, 78, 71, 32, + 79, 85, 84, 32, 70, 149, 158, 1, 17, 77, 79, 86, 69, 83, 32, 65, 71, 65, + 73, 78, 83, 84, 32, 67, 72, 69, 4, 214, 1, 73, 181, 180, 3, 5, 83, 84, + 73, 67, 75, 2, 181, 197, 3, 4, 69, 76, 65, 88, 4, 84, 7, 66, 69, 84, 87, + 69, 69, 78, 41, 10, 84, 79, 85, 67, 72, 73, 78, 71, 32, 73, 2, 11, 32, 2, + 173, 155, 5, 2, 76, 73, 2, 173, 174, 5, 5, 78, 83, 73, 68, 69, 6, 120, + 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 253, 199, 3, 14, 70, 76, 79, + 79, 82, 80, 76, 65, 78, 69, 32, 84, 87, 73, 4, 108, 8, 67, 85, 82, 86, + 69, 68, 32, 66, 161, 204, 1, 13, 83, 84, 82, 65, 73, 71, 72, 84, 32, 83, + 84, 82, 69, 2, 211, 191, 7, 69, 6, 11, 72, 6, 11, 32, 6, 30, 66, 34, 77, + 187, 4, 83, 2, 137, 249, 6, 3, 69, 84, 87, 2, 149, 2, 3, 85, 76, 84, 34, + 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 29, 10, 87, 65, 76, + 76, 80, 76, 65, 78, 69, 32, 14, 178, 1, 82, 151, 2, 83, 20, 72, 11, 65, + 82, 77, 32, 83, 80, 73, 82, 65, 76, 32, 78, 82, 151, 2, 83, 6, 30, 84, + 134, 2, 68, 43, 83, 2, 11, 82, 2, 11, 73, 2, 147, 166, 7, 80, 12, 41, 8, + 79, 84, 65, 84, 73, 79, 78, 45, 12, 52, 5, 70, 76, 79, 79, 82, 1, 4, 87, + 65, 76, 76, 6, 33, 6, 80, 76, 65, 78, 69, 32, 6, 26, 65, 54, 68, 43, 83, + 2, 29, 5, 76, 84, 69, 82, 78, 2, 203, 194, 3, 65, 2, 17, 2, 79, 85, 2, + 147, 164, 7, 66, 2, 231, 163, 7, 73, 2, 11, 72, 2, 203, 192, 3, 65, 2, + 11, 73, 2, 187, 179, 7, 78, 10, 100, 6, 65, 66, 79, 86, 69, 32, 162, 1, + 79, 213, 148, 2, 10, 77, 73, 78, 85, 83, 32, 83, 73, 77, 73, 4, 60, 7, + 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 2, 37, 7, 45, 84, 72, + 65, 78, 32, 65, 2, 25, 4, 66, 79, 86, 69, 2, 161, 241, 1, 2, 32, 69, 4, + 147, 179, 6, 82, 250, 1, 68, 3, 71, 76, 69, 212, 4, 5, 72, 65, 76, 65, + 32, 143, 161, 5, 69, 20, 50, 32, 185, 195, 8, 6, 45, 83, 72, 73, 70, 84, + 16, 138, 1, 67, 0, 9, 71, 82, 65, 80, 72, 73, 67, 32, 67, 116, 2, 72, 73, + 74, 76, 36, 4, 82, 73, 71, 72, 141, 244, 4, 4, 83, 72, 73, 70, 2, 41, 8, + 72, 65, 82, 65, 67, 84, 69, 82, 2, 37, 7, 32, 73, 78, 84, 82, 79, 68, 2, + 11, 85, 2, 223, 132, 8, 67, 2, 25, 4, 71, 72, 45, 82, 2, 185, 1, 7, 69, + 86, 69, 82, 83, 69, 68, 4, 32, 2, 69, 70, 109, 2, 79, 87, 2, 49, 10, 84, + 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 21, 3, 32, 65, 78, 2, 17, 2, 71, + 76, 2, 31, 69, 2, 17, 2, 45, 57, 2, 11, 32, 2, 11, 81, 2, 233, 176, 2, 2, + 85, 79, 228, 1, 168, 2, 8, 65, 82, 67, 72, 65, 73, 67, 32, 224, 1, 15, + 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 114, 76, 188, + 8, 5, 83, 73, 71, 78, 32, 144, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, + 78, 32, 177, 227, 2, 17, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, + 75, 85, 78, 68, 68, 40, 46, 68, 109, 7, 78, 85, 77, 66, 69, 82, 32, 18, + 25, 4, 73, 71, 73, 84, 18, 11, 32, 18, 234, 204, 6, 70, 30, 83, 42, 84, + 170, 86, 78, 14, 79, 223, 110, 69, 22, 226, 232, 2, 79, 146, 229, 3, 69, + 30, 70, 42, 78, 38, 83, 39, 84, 6, 18, 82, 63, 89, 4, 56, 6, 65, 75, 65, + 65, 82, 65, 209, 184, 8, 2, 69, 80, 2, 205, 184, 8, 3, 65, 78, 83, 138, + 1, 60, 6, 69, 84, 84, 69, 82, 32, 233, 186, 2, 3, 73, 84, 72, 118, 246, + 1, 65, 106, 69, 34, 73, 62, 85, 34, 77, 220, 1, 4, 68, 65, 78, 84, 58, + 79, 28, 8, 83, 65, 78, 89, 65, 75, 65, 32, 50, 70, 2, 72, 2, 82, 2, 86, + 2, 89, 32, 8, 84, 65, 65, 76, 85, 74, 65, 32, 29, 9, 75, 65, 78, 84, 65, + 74, 65, 32, 78, 34, 102, 69, 204, 1, 2, 76, 80, 144, 2, 5, 77, 66, 65, + 32, 66, 14, 65, 2, 73, 2, 85, 151, 253, 4, 89, 4, 230, 3, 69, 151, 253, + 4, 89, 12, 46, 76, 2, 82, 154, 3, 73, 151, 253, 4, 89, 4, 11, 85, 4, 138, + 3, 85, 151, 253, 4, 89, 28, 42, 65, 177, 1, 5, 85, 85, 82, 68, 72, 22, + 32, 2, 72, 65, 179, 255, 4, 89, 20, 41, 8, 65, 80, 82, 65, 65, 78, 65, + 32, 20, 70, 84, 138, 1, 68, 22, 66, 2, 67, 2, 71, 2, 74, 2, 75, 3, 80, 4, + 154, 1, 84, 15, 65, 6, 25, 4, 65, 74, 65, 32, 6, 102, 76, 2, 78, 3, 83, + 4, 86, 79, 151, 253, 4, 89, 8, 26, 68, 22, 71, 3, 74, 4, 18, 68, 15, 65, + 2, 11, 65, 2, 147, 253, 4, 89, 6, 26, 78, 21, 2, 83, 65, 2, 89, 2, 65, + 65, 4, 68, 11, 78, 89, 79, 79, 71, 65, 32, 78, 65, 65, 75, 139, 252, 4, + 89, 2, 149, 176, 8, 4, 83, 73, 75, 89, 8, 66, 65, 218, 157, 4, 67, 217, + 145, 4, 6, 86, 73, 83, 65, 82, 71, 4, 192, 116, 5, 76, 45, 76, 65, 75, + 237, 186, 7, 6, 78, 85, 83, 86, 65, 82, 34, 56, 5, 68, 73, 71, 65, 32, + 62, 71, 82, 75, 155, 2, 65, 12, 58, 71, 48, 4, 75, 79, 77, 66, 118, 65, + 26, 73, 19, 80, 4, 11, 65, 4, 236, 2, 3, 69, 84, 84, 67, 89, 2, 11, 85, + 2, 223, 173, 8, 86, 16, 52, 5, 69, 84, 84, 73, 32, 85, 4, 79, 77, 66, 85, + 6, 26, 65, 26, 73, 19, 80, 2, 213, 1, 2, 69, 68, 2, 203, 1, 83, 2, 175, + 1, 65, 10, 40, 2, 86, 65, 137, 128, 8, 2, 32, 68, 9, 29, 5, 32, 72, 65, + 65, 32, 6, 62, 65, 0, 6, 68, 73, 71, 65, 32, 65, 85, 3, 71, 65, 89, 2, + 17, 2, 69, 76, 2, 11, 65, 2, 17, 2, 45, 80, 2, 11, 73, 2, 147, 139, 8, + 76, 2, 189, 253, 5, 5, 65, 78, 85, 75, 73, 12, 84, 2, 32, 80, 206, 2, 45, + 205, 149, 6, 10, 84, 69, 69, 78, 32, 80, 79, 73, 78, 84, 8, 140, 1, 23, + 69, 84, 65, 76, 76, 69, 68, 32, 66, 76, 65, 67, 75, 32, 65, 78, 68, 32, + 87, 72, 73, 84, 69, 49, 7, 79, 73, 78, 84, 69, 68, 32, 2, 25, 4, 32, 70, + 76, 79, 2, 223, 227, 6, 82, 6, 78, 80, 218, 147, 6, 66, 217, 132, 1, 9, + 83, 84, 65, 82, 32, 87, 73, 84, 72, 2, 21, 3, 73, 78, 87, 2, 217, 147, 6, + 4, 72, 69, 69, 76, 2, 139, 208, 3, 80, 12, 46, 73, 102, 85, 153, 145, 7, + 3, 65, 84, 69, 4, 56, 8, 32, 65, 78, 68, 32, 83, 75, 73, 243, 239, 7, 69, + 2, 17, 2, 32, 66, 2, 255, 215, 7, 79, 6, 32, 2, 76, 76, 235, 164, 8, 78, + 5, 11, 32, 2, 49, 10, 65, 78, 68, 32, 67, 82, 79, 83, 83, 66, 2, 11, 79, + 2, 187, 147, 7, 78, 42, 46, 65, 198, 4, 69, 210, 1, 73, 167, 1, 79, 14, + 64, 5, 78, 84, 69, 68, 32, 193, 146, 6, 5, 86, 79, 78, 73, 67, 12, 148, + 1, 12, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, 32, 229, 1, 19, 78, + 79, 82, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, 72, 79, + 8, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 4, 29, 5, 45, + 84, 72, 65, 78, 5, 11, 32, 2, 25, 4, 87, 73, 84, 72, 2, 25, 4, 32, 68, + 79, 84, 2, 17, 2, 32, 73, 2, 11, 78, 2, 11, 83, 2, 183, 147, 6, 73, 4, + 24, 2, 79, 75, 43, 82, 2, 17, 2, 69, 68, 2, 131, 206, 5, 32, 2, 29, 5, + 73, 90, 79, 78, 84, 2, 11, 65, 2, 199, 238, 5, 76, 12, 80, 2, 69, 80, + 144, 144, 8, 9, 85, 84, 72, 32, 79, 82, 32, 83, 80, 203, 17, 68, 8, 40, + 4, 73, 78, 71, 32, 143, 228, 7, 89, 6, 176, 223, 4, 8, 65, 67, 67, 79, + 77, 77, 79, 68, 234, 174, 2, 83, 139, 86, 70, 6, 76, 9, 67, 69, 32, 79, + 70, 32, 80, 73, 90, 21, 6, 71, 72, 84, 76, 89, 32, 2, 211, 157, 8, 90, 4, + 40, 2, 83, 77, 173, 217, 6, 2, 70, 82, 2, 215, 234, 6, 73, 10, 18, 80, + 75, 84, 6, 152, 222, 4, 9, 73, 78, 71, 32, 76, 65, 82, 71, 69, 139, 193, + 3, 69, 4, 26, 32, 195, 158, 8, 72, 2, 11, 77, 2, 249, 132, 7, 3, 65, 67, + 72, 150, 1, 34, 65, 158, 13, 73, 207, 7, 79, 116, 32, 2, 76, 76, 155, + 142, 7, 83, 114, 30, 32, 141, 12, 2, 69, 82, 110, 194, 2, 65, 48, 3, 66, 76, 85, 0, 5, 79, 82, 65, 78, 71, 20, 2, 67, 79, 158, 1, 68, 22, 69, 140, 2, 11, 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 67, 28, 2, 76, 69, 34, 82, - 218, 3, 83, 22, 84, 36, 4, 86, 69, 69, 32, 208, 238, 1, 3, 71, 82, 69, - 34, 72, 138, 4, 80, 154, 79, 78, 202, 203, 2, 70, 83, 81, 6, 226, 167, 4, - 77, 218, 44, 73, 187, 124, 83, 2, 151, 151, 6, 69, 12, 68, 7, 78, 84, 65, - 73, 78, 83, 32, 218, 243, 1, 77, 231, 156, 5, 76, 6, 36, 4, 65, 83, 32, - 77, 175, 1, 87, 2, 11, 69, 2, 11, 77, 2, 251, 161, 7, 66, 2, 235, 243, 1, - 79, 12, 84, 9, 76, 69, 77, 69, 78, 84, 32, 79, 70, 178, 243, 1, 81, 42, - 88, 215, 194, 2, 77, 7, 17, 2, 32, 87, 4, 25, 4, 73, 84, 72, 32, 4, 26, - 86, 223, 196, 4, 79, 2, 181, 192, 5, 21, 69, 82, 84, 73, 67, 65, 76, 32, - 66, 65, 82, 32, 65, 84, 32, 69, 78, 68, 32, 79, 70, 2, 129, 146, 5, 2, - 32, 67, 8, 130, 1, 70, 255, 245, 1, 83, 40, 96, 3, 73, 71, 72, 64, 13, - 79, 77, 65, 78, 32, 78, 85, 77, 69, 82, 65, 76, 32, 155, 201, 5, 69, 6, - 17, 2, 84, 32, 6, 170, 255, 3, 67, 134, 2, 80, 191, 2, 84, 32, 62, 69, - 50, 70, 62, 79, 46, 84, 238, 236, 5, 83, 231, 83, 78, 4, 26, 76, 171, - 176, 7, 73, 2, 255, 237, 5, 69, 8, 26, 73, 251, 225, 6, 79, 6, 154, 26, - 86, 175, 214, 5, 70, 6, 11, 78, 6, 11, 69, 7, 203, 177, 2, 32, 8, 42, 87, - 134, 237, 5, 72, 239, 156, 1, 69, 4, 26, 69, 171, 217, 7, 79, 2, 227, - 220, 6, 76, 2, 183, 163, 4, 69, 4, 178, 235, 3, 87, 211, 215, 1, 73, 2, - 11, 87, 2, 21, 3, 73, 84, 72, 2, 11, 32, 2, 211, 143, 4, 85, 4, 29, 5, - 32, 84, 72, 65, 78, 5, 11, 32, 2, 11, 79, 2, 223, 176, 1, 82, 32, 34, 76, - 161, 207, 6, 2, 82, 75, 30, 40, 4, 73, 78, 71, 32, 255, 214, 7, 69, 28, - 112, 14, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 65, 10, - 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 4, 30, 79, 157, 1, 2, 72, 69, 2, - 213, 231, 2, 3, 80, 69, 78, 24, 86, 72, 104, 10, 79, 80, 69, 78, 32, 77, - 79, 85, 84, 72, 178, 2, 83, 159, 205, 2, 84, 6, 34, 69, 50, 79, 207, 177, - 7, 65, 2, 189, 2, 8, 65, 82, 84, 45, 83, 72, 65, 80, 2, 135, 166, 4, 82, - 9, 29, 5, 32, 65, 78, 68, 32, 6, 26, 67, 58, 83, 63, 84, 2, 17, 2, 79, - 76, 2, 253, 174, 3, 4, 68, 32, 83, 87, 2, 11, 77, 2, 11, 73, 2, 11, 76, - 2, 117, 3, 73, 78, 71, 2, 37, 7, 73, 71, 72, 84, 76, 89, 45, 2, 11, 67, - 2, 21, 3, 76, 79, 83, 2, 17, 2, 69, 68, 2, 149, 190, 6, 3, 32, 69, 89, 8, - 64, 11, 77, 73, 76, 73, 78, 71, 32, 69, 89, 69, 83, 175, 1, 85, 7, 29, 5, - 32, 65, 78, 68, 32, 4, 52, 4, 72, 65, 78, 68, 57, 5, 84, 72, 82, 69, 69, - 2, 169, 226, 2, 9, 32, 67, 79, 86, 69, 82, 73, 78, 71, 2, 225, 163, 4, 2, - 32, 72, 2, 17, 2, 78, 71, 2, 241, 187, 6, 4, 76, 65, 83, 83, 2, 195, 183, - 4, 75, 16, 42, 65, 32, 2, 69, 69, 21, 2, 79, 87, 4, 210, 197, 6, 73, 203, - 114, 75, 2, 199, 198, 6, 90, 10, 100, 7, 32, 67, 65, 80, 80, 69, 68, 28, - 3, 77, 65, 78, 212, 203, 2, 3, 66, 79, 65, 195, 144, 1, 70, 2, 233, 135, - 4, 2, 32, 77, 5, 49, 10, 32, 87, 73, 84, 72, 79, 85, 84, 32, 83, 2, 247, - 178, 6, 78, 151, 3, 162, 2, 67, 48, 2, 70, 84, 228, 1, 6, 71, 68, 73, 65, + 218, 3, 83, 22, 84, 36, 2, 85, 80, 56, 4, 86, 69, 69, 32, 136, 253, 1, 3, + 71, 82, 69, 34, 72, 146, 4, 80, 234, 77, 78, 222, 240, 2, 70, 83, 81, 6, + 194, 203, 4, 77, 214, 49, 73, 135, 138, 1, 83, 2, 135, 206, 6, 69, 12, + 68, 7, 78, 84, 65, 73, 78, 83, 32, 202, 130, 2, 77, 167, 199, 5, 76, 6, + 36, 4, 65, 83, 32, 77, 175, 1, 87, 2, 11, 69, 2, 11, 77, 2, 171, 219, 7, + 66, 2, 219, 130, 2, 79, 12, 84, 9, 76, 69, 77, 69, 78, 84, 32, 79, 70, + 162, 130, 2, 81, 42, 88, 175, 216, 2, 77, 7, 17, 2, 32, 87, 4, 25, 4, 73, + 84, 72, 32, 4, 26, 86, 171, 233, 4, 79, 2, 129, 243, 5, 21, 69, 82, 84, + 73, 67, 65, 76, 32, 66, 65, 82, 32, 65, 84, 32, 69, 78, 68, 32, 79, 70, + 2, 221, 196, 5, 2, 32, 67, 8, 130, 1, 70, 247, 132, 2, 83, 40, 96, 3, 73, + 71, 72, 64, 13, 79, 77, 65, 78, 32, 78, 85, 77, 69, 82, 65, 76, 32, 191, + 255, 5, 69, 6, 17, 2, 84, 32, 6, 170, 143, 4, 67, 210, 3, 80, 147, 8, 84, + 32, 62, 69, 50, 70, 62, 79, 46, 84, 166, 163, 6, 83, 211, 86, 78, 4, 26, + 76, 231, 233, 7, 73, 2, 183, 164, 6, 69, 8, 26, 73, 167, 155, 7, 79, 6, + 130, 26, 86, 255, 140, 6, 70, 6, 11, 78, 6, 11, 69, 7, 247, 191, 2, 32, + 8, 42, 87, 190, 163, 6, 72, 231, 159, 1, 69, 4, 26, 69, 231, 146, 8, 79, + 2, 143, 150, 7, 76, 2, 147, 199, 4, 69, 4, 182, 248, 3, 87, 211, 128, 2, + 73, 2, 185, 243, 4, 9, 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 11, 87, 2, + 21, 3, 73, 84, 72, 2, 11, 32, 2, 163, 166, 4, 85, 4, 29, 5, 32, 84, 72, + 65, 78, 5, 11, 32, 2, 11, 79, 2, 135, 189, 1, 82, 32, 34, 76, 141, 136, + 7, 2, 82, 75, 30, 40, 4, 73, 78, 71, 32, 131, 144, 8, 69, 28, 112, 14, + 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 41, 10, 70, 65, + 67, 69, 32, 87, 73, 84, 72, 32, 4, 160, 1, 2, 72, 69, 163, 236, 4, 79, + 24, 86, 72, 108, 10, 79, 80, 69, 78, 32, 77, 79, 85, 84, 72, 246, 1, 83, + 203, 219, 2, 84, 6, 34, 69, 54, 79, 231, 234, 7, 65, 2, 165, 146, 5, 8, + 65, 82, 84, 45, 83, 72, 65, 80, 2, 219, 201, 4, 82, 9, 29, 5, 32, 65, 78, + 68, 32, 6, 26, 67, 58, 83, 71, 84, 2, 17, 2, 79, 76, 2, 145, 188, 3, 4, + 68, 32, 83, 87, 2, 11, 77, 2, 11, 73, 2, 11, 76, 2, 217, 144, 5, 3, 73, + 78, 71, 2, 37, 7, 73, 71, 72, 84, 76, 89, 45, 2, 231, 143, 5, 67, 8, 64, + 11, 77, 73, 76, 73, 78, 71, 32, 69, 89, 69, 83, 175, 1, 85, 7, 29, 5, 32, + 65, 78, 68, 32, 4, 52, 4, 72, 65, 78, 68, 57, 5, 84, 72, 82, 69, 69, 2, + 221, 232, 4, 9, 32, 67, 79, 86, 69, 82, 73, 78, 71, 2, 241, 199, 4, 2, + 32, 72, 2, 17, 2, 78, 71, 2, 173, 245, 6, 4, 76, 65, 83, 83, 2, 171, 220, + 4, 75, 16, 42, 65, 32, 2, 69, 69, 21, 2, 79, 87, 4, 142, 255, 6, 73, 227, + 114, 75, 2, 131, 128, 7, 90, 10, 100, 7, 32, 67, 65, 80, 80, 69, 68, 28, + 3, 77, 65, 78, 136, 218, 2, 3, 66, 79, 65, 167, 143, 1, 70, 2, 137, 159, + 4, 2, 32, 77, 5, 49, 10, 32, 87, 73, 84, 72, 79, 85, 84, 32, 83, 2, 179, + 236, 6, 78, 155, 3, 162, 2, 67, 48, 2, 70, 84, 228, 1, 6, 71, 68, 73, 65, 78, 32, 244, 10, 3, 76, 73, 68, 140, 2, 12, 79, 78, 32, 87, 73, 84, 72, 32, 82, 73, 71, 72, 20, 11, 82, 65, 32, 83, 79, 77, 80, 69, 78, 71, 32, - 238, 2, 85, 172, 8, 6, 89, 79, 77, 66, 79, 32, 162, 169, 6, 77, 234, 134, - 1, 72, 3, 83, 4, 128, 214, 2, 3, 67, 69, 82, 247, 182, 4, 75, 10, 70, 32, - 124, 9, 87, 65, 82, 69, 45, 70, 85, 78, 67, 155, 196, 5, 66, 6, 58, 72, - 44, 6, 73, 67, 69, 32, 67, 82, 231, 133, 6, 83, 2, 11, 89, 2, 11, 80, 2, - 135, 248, 5, 72, 2, 171, 143, 6, 69, 2, 11, 84, 2, 251, 211, 5, 73, 84, + 234, 2, 85, 216, 8, 6, 89, 79, 77, 66, 79, 32, 182, 226, 6, 77, 130, 135, + 1, 72, 3, 83, 4, 232, 227, 2, 3, 67, 69, 82, 215, 226, 4, 75, 10, 70, 32, + 124, 9, 87, 65, 82, 69, 45, 70, 85, 78, 67, 223, 250, 5, 66, 6, 58, 72, + 44, 6, 73, 67, 69, 32, 67, 82, 239, 188, 6, 83, 2, 11, 89, 2, 11, 80, 2, + 231, 174, 6, 72, 2, 191, 199, 6, 69, 2, 11, 84, 2, 203, 138, 6, 73, 84, 248, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 168, 2, 13, 73, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 83, 20, 7, 76, 69, 84, 84, 69, 82, 32, 188, 3, 7, 78, 85, 77, 66, 69, 82, 32, 113, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 22, 140, 1, 3, 72, 79, 79, 20, 4, 76, 79, - 78, 71, 50, 83, 58, 84, 242, 213, 3, 68, 216, 199, 1, 4, 82, 69, 83, 72, - 249, 13, 4, 67, 85, 82, 86, 4, 199, 172, 5, 75, 2, 25, 4, 32, 72, 79, 79, - 2, 147, 158, 5, 75, 2, 21, 3, 84, 82, 79, 2, 11, 75, 2, 219, 157, 5, 69, - 4, 189, 208, 1, 2, 87, 79, 2, 211, 196, 6, 72, 42, 190, 1, 65, 50, 71, - 34, 76, 56, 5, 82, 69, 83, 72, 45, 2, 90, 34, 83, 66, 89, 154, 193, 1, - 72, 234, 5, 75, 130, 76, 66, 2, 70, 190, 173, 4, 78, 254, 1, 84, 2, 87, - 202, 103, 80, 171, 4, 77, 4, 26, 76, 231, 194, 6, 89, 2, 215, 200, 1, 69, - 2, 11, 73, 2, 167, 225, 2, 77, 4, 26, 65, 183, 220, 5, 69, 2, 189, 194, - 1, 2, 77, 69, 2, 11, 65, 2, 235, 193, 6, 89, 6, 26, 65, 191, 193, 6, 72, - 4, 178, 185, 1, 77, 187, 218, 5, 68, 2, 195, 193, 1, 79, 8, 18, 79, 59, - 84, 4, 11, 78, 4, 11, 69, 5, 11, 32, 2, 227, 151, 2, 72, 4, 230, 213, 5, - 87, 187, 154, 1, 69, 10, 66, 67, 0, 6, 72, 65, 76, 70, 32, 67, 53, 4, 84, - 87, 79, 32, 2, 21, 3, 73, 82, 67, 2, 201, 193, 4, 2, 76, 69, 6, 100, 13, + 78, 71, 50, 83, 58, 84, 142, 227, 3, 68, 176, 237, 1, 4, 82, 69, 83, 72, + 129, 15, 4, 67, 85, 82, 86, 4, 195, 224, 5, 75, 2, 25, 4, 32, 72, 79, 79, + 2, 135, 209, 5, 75, 2, 21, 3, 84, 82, 79, 2, 11, 75, 2, 207, 208, 5, 69, + 4, 193, 223, 1, 2, 87, 79, 2, 143, 254, 6, 72, 42, 190, 1, 65, 50, 71, + 34, 76, 56, 5, 82, 69, 83, 72, 45, 2, 90, 34, 83, 66, 89, 158, 208, 1, + 72, 234, 5, 75, 198, 75, 66, 2, 70, 178, 216, 4, 78, 134, 2, 84, 2, 87, + 218, 103, 80, 171, 4, 77, 4, 26, 76, 163, 252, 6, 89, 2, 219, 215, 1, 69, + 2, 11, 73, 2, 239, 238, 2, 77, 4, 26, 65, 135, 147, 6, 69, 2, 193, 209, + 1, 2, 77, 69, 2, 11, 65, 2, 167, 251, 6, 89, 6, 26, 65, 251, 250, 6, 72, + 4, 182, 200, 1, 77, 139, 133, 6, 68, 2, 199, 208, 1, 79, 8, 18, 79, 59, + 84, 4, 11, 78, 4, 11, 69, 5, 11, 32, 2, 167, 166, 2, 72, 4, 182, 140, 6, + 87, 179, 157, 1, 69, 10, 66, 67, 0, 6, 72, 65, 76, 70, 32, 67, 53, 4, 84, + 87, 79, 32, 2, 21, 3, 73, 82, 67, 2, 173, 236, 4, 2, 76, 69, 6, 100, 13, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 83, 13, 8, 67, 73, 82, - 67, 76, 69, 83, 32, 5, 11, 32, 2, 153, 240, 5, 4, 87, 73, 84, 72, 8, 30, + 67, 76, 69, 83, 32, 5, 11, 32, 2, 249, 166, 6, 4, 87, 73, 84, 72, 8, 30, 32, 189, 1, 2, 85, 83, 4, 93, 21, 81, 85, 73, 76, 84, 32, 83, 81, 85, 65, 82, 69, 32, 79, 82, 78, 65, 77, 69, 78, 84, 5, 11, 32, 2, 11, 73, 2, 17, - 2, 78, 32, 2, 11, 66, 2, 173, 155, 6, 4, 76, 65, 67, 75, 5, 173, 162, 4, - 7, 32, 87, 73, 84, 72, 32, 79, 2, 251, 227, 3, 84, 70, 52, 7, 76, 69, 84, - 84, 69, 82, 32, 207, 204, 5, 68, 50, 190, 1, 69, 32, 2, 77, 65, 30, 78, - 38, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, - 83, 2, 84, 2, 86, 2, 89, 186, 183, 7, 65, 2, 73, 2, 79, 3, 85, 4, 150, - 184, 7, 69, 147, 1, 72, 4, 134, 185, 7, 69, 3, 72, 6, 34, 71, 2, 89, 187, - 183, 7, 65, 2, 183, 183, 7, 65, 54, 104, 3, 84, 72, 32, 197, 233, 5, 17, - 78, 68, 32, 82, 69, 67, 79, 82, 68, 73, 78, 71, 32, 67, 79, 80, 89, 52, - 60, 5, 69, 65, 83, 84, 32, 237, 2, 5, 87, 69, 83, 84, 32, 28, 96, 5, 65, - 82, 82, 79, 87, 162, 4, 80, 130, 1, 84, 246, 160, 4, 66, 38, 68, 18, 83, - 203, 43, 87, 11, 11, 32, 8, 52, 2, 65, 78, 38, 67, 120, 2, 84, 79, 139, - 2, 87, 2, 237, 2, 5, 68, 32, 83, 79, 85, 2, 37, 7, 82, 79, 83, 83, 73, - 78, 71, 2, 17, 2, 32, 78, 2, 17, 2, 79, 82, 2, 221, 209, 4, 5, 84, 72, - 32, 69, 65, 2, 17, 2, 32, 67, 2, 239, 219, 3, 79, 24, 96, 5, 65, 82, 82, - 79, 87, 182, 1, 80, 130, 1, 84, 246, 160, 4, 66, 38, 68, 18, 83, 203, 43, - 87, 7, 11, 32, 4, 36, 5, 65, 78, 68, 32, 78, 75, 87, 2, 17, 2, 79, 82, 2, - 21, 3, 84, 72, 32, 2, 201, 207, 4, 2, 87, 69, 2, 21, 3, 73, 84, 72, 2, - 11, 32, 2, 183, 225, 5, 72, 6, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 6, - 42, 86, 154, 253, 3, 76, 215, 137, 2, 66, 2, 17, 2, 73, 78, 2, 239, 252, - 3, 69, 4, 89, 20, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, 65, 68, 69, 68, - 32, 65, 82, 82, 79, 87, 5, 11, 32, 2, 147, 214, 4, 84, 166, 1, 212, 3, - 23, 67, 76, 85, 83, 84, 69, 82, 45, 73, 78, 73, 84, 73, 65, 76, 32, 76, - 69, 84, 84, 69, 82, 32, 40, 21, 70, 73, 78, 65, 76, 32, 67, 79, 78, 83, - 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 114, 71, 32, 27, 72, 69, 65, - 68, 32, 77, 65, 82, 75, 32, 87, 73, 84, 72, 32, 77, 79, 79, 78, 32, 65, - 78, 68, 32, 83, 85, 78, 104, 7, 76, 69, 84, 84, 69, 82, 32, 180, 1, 5, - 77, 65, 82, 75, 32, 54, 83, 100, 8, 84, 69, 82, 77, 73, 78, 65, 76, 41, - 6, 86, 79, 87, 69, 76, 32, 8, 246, 228, 6, 83, 254, 68, 76, 3, 82, 24, - 162, 184, 3, 83, 206, 251, 1, 78, 222, 245, 1, 45, 186, 2, 66, 2, 68, 2, - 71, 2, 75, 2, 76, 2, 77, 3, 82, 2, 157, 166, 6, 3, 69, 77, 73, 7, 29, 5, - 32, 65, 78, 68, 32, 4, 50, 70, 1, 8, 84, 82, 73, 80, 76, 69, 32, 70, 2, - 187, 158, 5, 76, 82, 234, 174, 2, 68, 206, 178, 4, 75, 38, 78, 46, 83, - 38, 84, 46, 66, 2, 67, 2, 71, 2, 74, 2, 80, 2, 90, 254, 68, 45, 2, 72, 2, - 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 8, 158, 153, 3, 80, 214, 200, - 3, 68, 58, 83, 63, 84, 10, 40, 4, 73, 71, 78, 32, 211, 229, 6, 85, 8, - 146, 246, 4, 74, 158, 2, 86, 122, 85, 231, 233, 1, 65, 4, 233, 247, 1, 5, - 32, 77, 65, 82, 75, 22, 44, 5, 83, 73, 71, 78, 32, 251, 228, 6, 76, 20, - 88, 7, 86, 79, 67, 65, 76, 73, 67, 226, 229, 6, 65, 26, 79, 2, 85, 150, - 64, 69, 3, 73, 4, 11, 32, 4, 254, 165, 7, 76, 3, 82, 61, 166, 1, 65, 142, - 1, 69, 136, 5, 13, 72, 69, 82, 73, 67, 65, 76, 32, 65, 78, 71, 76, 69, - 82, 73, 220, 1, 12, 76, 65, 83, 72, 73, 78, 71, 32, 83, 87, 69, 65, 23, - 79, 13, 60, 4, 71, 72, 69, 84, 20, 3, 82, 75, 76, 151, 141, 7, 67, 2, - 163, 144, 7, 84, 6, 26, 73, 239, 159, 5, 69, 2, 165, 156, 6, 2, 78, 71, - 18, 52, 2, 65, 75, 250, 3, 69, 69, 4, 83, 77, 73, 76, 12, 70, 45, 64, 2, - 69, 82, 173, 2, 8, 73, 78, 71, 32, 72, 69, 65, 68, 2, 221, 139, 5, 11, - 78, 79, 45, 69, 86, 73, 76, 32, 77, 79, 78, 9, 33, 6, 32, 87, 73, 84, 72, - 32, 6, 26, 67, 78, 79, 75, 84, 2, 33, 6, 65, 78, 67, 69, 76, 76, 2, 245, - 240, 6, 5, 65, 84, 73, 79, 78, 2, 41, 8, 78, 69, 32, 83, 79, 85, 78, 68, - 2, 133, 184, 5, 2, 32, 87, 2, 25, 4, 72, 82, 69, 69, 2, 157, 140, 6, 10, - 32, 83, 79, 85, 78, 68, 32, 87, 65, 86, 2, 11, 32, 2, 245, 218, 5, 9, 73, - 78, 32, 83, 73, 76, 72, 79, 85, 4, 34, 68, 241, 232, 2, 2, 67, 72, 2, 11, - 66, 2, 255, 138, 6, 79, 2, 211, 151, 6, 79, 7, 45, 9, 32, 79, 80, 69, 78, - 73, 78, 71, 32, 4, 138, 243, 6, 76, 227, 42, 85, 10, 44, 3, 68, 69, 82, - 41, 4, 82, 65, 76, 32, 5, 17, 2, 32, 87, 2, 239, 226, 5, 69, 6, 80, 8, - 67, 65, 76, 69, 78, 68, 65, 82, 0, 4, 78, 79, 84, 69, 29, 2, 83, 72, 2, - 137, 214, 6, 2, 32, 80, 2, 223, 151, 5, 69, 2, 235, 136, 6, 84, 10, 90, - 79, 60, 7, 85, 84, 73, 78, 71, 32, 87, 128, 228, 2, 3, 82, 84, 83, 223, - 130, 3, 78, 4, 144, 84, 7, 76, 32, 79, 70, 32, 84, 72, 251, 198, 6, 78, - 2, 11, 72, 2, 159, 249, 5, 65, 206, 5, 26, 65, 255, 211, 6, 73, 204, 5, - 28, 2, 82, 69, 175, 66, 84, 202, 5, 30, 32, 129, 44, 2, 68, 32, 218, 3, - 246, 1, 65, 116, 2, 86, 32, 58, 66, 158, 1, 67, 170, 1, 68, 102, 69, 194, - 2, 70, 74, 71, 222, 1, 72, 214, 2, 73, 94, 75, 206, 4, 76, 102, 77, 210, - 5, 78, 94, 79, 178, 1, 80, 222, 3, 82, 154, 3, 83, 186, 1, 84, 74, 87, - 218, 6, 89, 175, 158, 6, 85, 14, 114, 32, 152, 71, 2, 78, 80, 236, 142, - 2, 3, 82, 85, 72, 238, 51, 65, 240, 43, 2, 80, 65, 250, 224, 3, 77, 3, - 85, 2, 21, 3, 79, 86, 69, 2, 11, 82, 2, 219, 133, 7, 32, 14, 98, 65, 36, - 4, 85, 83, 83, 89, 178, 136, 3, 73, 212, 64, 2, 79, 82, 202, 39, 69, 207, - 164, 3, 81, 4, 32, 2, 65, 82, 155, 149, 7, 82, 2, 175, 136, 3, 69, 22, - 98, 65, 30, 79, 230, 18, 77, 204, 128, 7, 7, 32, 79, 86, 69, 82, 32, 75, - 74, 85, 14, 67, 3, 68, 4, 158, 148, 7, 76, 3, 80, 5, 17, 2, 82, 80, 2, - 247, 181, 2, 79, 20, 82, 65, 234, 17, 77, 142, 195, 2, 69, 174, 49, 79, - 234, 140, 4, 66, 2, 74, 3, 76, 5, 207, 20, 65, 18, 66, 69, 22, 82, 236, - 23, 5, 83, 85, 75, 85, 85, 183, 250, 6, 86, 2, 235, 162, 5, 75, 12, 52, - 7, 65, 32, 78, 65, 77, 69, 32, 235, 145, 7, 71, 10, 132, 1, 4, 72, 69, - 73, 83, 20, 5, 84, 65, 73, 83, 89, 224, 20, 3, 77, 69, 73, 240, 182, 2, - 3, 82, 69, 73, 1, 4, 83, 89, 79, 85, 2, 135, 253, 6, 69, 2, 227, 235, 6, - 79, 6, 26, 79, 159, 144, 7, 77, 4, 128, 226, 5, 2, 85, 82, 239, 145, 1, - 79, 24, 102, 65, 46, 73, 188, 12, 5, 85, 82, 65, 77, 85, 178, 139, 3, 72, - 186, 244, 3, 80, 186, 2, 66, 3, 89, 6, 166, 191, 6, 82, 210, 46, 78, 147, - 33, 76, 6, 42, 82, 150, 210, 4, 78, 231, 185, 2, 71, 2, 229, 158, 5, 2, - 85, 68, 30, 130, 1, 65, 22, 69, 54, 79, 62, 85, 226, 202, 1, 80, 196, - 184, 5, 10, 73, 82, 65, 71, 65, 78, 65, 32, 72, 79, 234, 8, 71, 3, 90, 5, - 207, 232, 1, 73, 4, 184, 11, 3, 75, 85, 84, 129, 221, 1, 2, 82, 85, 6, - 26, 79, 175, 140, 7, 78, 4, 194, 231, 6, 82, 235, 36, 78, 6, 54, 73, 160, - 17, 4, 65, 82, 65, 68, 131, 179, 5, 82, 2, 203, 191, 3, 73, 14, 54, 78, - 228, 12, 4, 77, 65, 71, 69, 167, 254, 6, 85, 7, 190, 199, 2, 73, 243, - 175, 4, 84, 58, 230, 1, 65, 64, 2, 89, 85, 20, 3, 73, 82, 79, 78, 77, 90, - 79, 60, 2, 85, 82, 144, 8, 2, 69, 69, 202, 135, 3, 72, 202, 237, 2, 67, - 208, 120, 3, 32, 79, 72, 162, 14, 80, 186, 2, 66, 2, 71, 2, 75, 2, 76, 2, - 84, 2, 86, 3, 87, 9, 18, 73, 23, 82, 2, 143, 245, 6, 82, 4, 22, 79, 143, - 20, 65, 2, 159, 204, 4, 82, 9, 252, 16, 3, 77, 69, 69, 248, 2, 2, 87, 65, - 169, 161, 2, 3, 71, 85, 82, 9, 11, 32, 6, 22, 67, 199, 14, 83, 4, 22, 65, - 179, 6, 85, 2, 209, 185, 4, 2, 80, 73, 4, 18, 79, 23, 82, 2, 251, 231, 6, - 80, 2, 151, 204, 5, 85, 4, 176, 154, 5, 4, 85, 90, 69, 73, 239, 66, 79, - 12, 62, 79, 244, 13, 2, 69, 70, 230, 247, 6, 77, 2, 78, 3, 88, 4, 202, - 209, 5, 90, 143, 180, 1, 71, 82, 162, 1, 32, 70, 65, 122, 66, 22, 69, 54, - 73, 110, 77, 68, 2, 85, 32, 78, 86, 2, 87, 162, 137, 3, 72, 214, 237, 2, - 79, 230, 134, 1, 80, 186, 2, 71, 2, 76, 3, 83, 10, 34, 79, 242, 2, 67, - 139, 8, 83, 6, 198, 10, 86, 207, 232, 6, 72, 13, 54, 73, 44, 2, 78, 83, - 146, 82, 82, 159, 185, 4, 72, 4, 252, 150, 5, 2, 75, 85, 175, 199, 1, 82, - 2, 135, 179, 6, 89, 5, 243, 160, 2, 32, 6, 28, 2, 71, 65, 251, 10, 69, 5, - 191, 178, 6, 84, 8, 42, 75, 20, 2, 82, 73, 207, 129, 7, 76, 2, 211, 221, - 5, 85, 5, 11, 66, 2, 11, 65, 2, 203, 244, 2, 65, 7, 11, 32, 4, 22, 67, - 139, 8, 83, 2, 11, 85, 2, 175, 161, 2, 66, 16, 218, 128, 7, 65, 2, 70, 2, - 71, 2, 76, 2, 77, 2, 83, 2, 86, 3, 87, 5, 11, 32, 2, 11, 77, 2, 191, 189, - 6, 69, 16, 70, 65, 238, 10, 79, 178, 244, 6, 70, 2, 77, 2, 83, 2, 86, 3, - 87, 5, 167, 224, 6, 78, 12, 78, 78, 20, 7, 82, 73, 71, 73, 78, 65, 76, - 170, 171, 2, 79, 255, 210, 4, 86, 2, 207, 217, 6, 83, 6, 21, 3, 32, 79, - 70, 7, 25, 4, 32, 79, 82, 32, 4, 214, 86, 78, 51, 69, 46, 118, 65, 82, - 69, 94, 73, 74, 79, 150, 234, 6, 80, 218, 16, 67, 2, 70, 2, 72, 2, 77, 2, - 82, 2, 83, 2, 86, 3, 87, 9, 38, 65, 221, 220, 3, 3, 32, 65, 77, 4, 236, - 1, 2, 83, 69, 227, 213, 6, 84, 8, 34, 69, 22, 78, 231, 220, 6, 83, 2, - 147, 232, 6, 90, 4, 230, 215, 1, 73, 139, 255, 4, 83, 6, 34, 75, 233, 3, - 3, 65, 83, 85, 4, 166, 238, 2, 85, 235, 140, 4, 79, 6, 34, 73, 22, 78, - 21, 2, 83, 73, 2, 191, 174, 3, 78, 2, 191, 219, 6, 68, 2, 133, 193, 6, 4, - 84, 73, 79, 78, 22, 60, 2, 65, 68, 114, 69, 62, 73, 134, 1, 85, 227, 166, - 6, 79, 7, 21, 3, 32, 79, 86, 4, 25, 4, 69, 82, 32, 83, 5, 17, 2, 32, 83, - 2, 11, 81, 2, 153, 153, 2, 2, 85, 65, 4, 36, 3, 78, 84, 79, 191, 211, 6, - 77, 2, 175, 167, 5, 71, 6, 40, 2, 71, 72, 62, 84, 243, 244, 6, 82, 2, - 181, 205, 3, 10, 84, 32, 79, 80, 69, 78, 32, 66, 79, 88, 2, 189, 234, 2, - 2, 84, 79, 4, 172, 1, 2, 85, 66, 195, 185, 4, 80, 14, 82, 65, 72, 3, 69, - 78, 84, 128, 178, 2, 3, 73, 82, 73, 202, 195, 4, 82, 3, 86, 4, 48, 2, 73, - 75, 225, 162, 2, 4, 78, 84, 73, 73, 2, 243, 232, 2, 85, 4, 198, 245, 6, - 73, 3, 79, 6, 42, 65, 146, 254, 2, 72, 159, 167, 3, 79, 2, 241, 191, 6, - 2, 82, 71, 34, 50, 65, 20, 4, 73, 84, 72, 32, 159, 244, 6, 66, 2, 167, - 168, 3, 84, 30, 222, 1, 66, 40, 3, 68, 73, 65, 0, 5, 79, 82, 84, 72, 79, - 84, 2, 72, 79, 42, 76, 126, 84, 28, 6, 85, 80, 80, 69, 82, 32, 186, 1, - 86, 190, 227, 3, 82, 225, 241, 1, 13, 67, 79, 78, 84, 79, 85, 82, 69, 68, - 32, 79, 85, 84, 2, 209, 231, 3, 5, 79, 84, 84, 79, 77, 2, 165, 129, 4, - 16, 71, 79, 78, 65, 76, 32, 67, 82, 79, 83, 83, 72, 65, 84, 67, 72, 2, - 149, 3, 6, 82, 73, 90, 79, 78, 84, 6, 44, 5, 79, 87, 69, 82, 32, 211, - 229, 3, 69, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 189, 1, 3, 84, - 32, 68, 2, 177, 229, 3, 2, 79, 80, 8, 60, 5, 76, 69, 70, 84, 32, 37, 6, - 82, 73, 71, 72, 84, 32, 4, 70, 68, 253, 241, 3, 2, 84, 79, 4, 34, 68, - 205, 253, 3, 2, 84, 79, 2, 141, 228, 3, 7, 73, 65, 71, 79, 78, 65, 76, 2, - 29, 5, 69, 82, 84, 73, 67, 2, 185, 253, 3, 2, 65, 76, 6, 32, 2, 65, 65, - 183, 166, 5, 85, 4, 254, 200, 6, 82, 247, 5, 68, 240, 1, 226, 1, 67, 174, - 9, 68, 34, 70, 88, 3, 82, 73, 83, 142, 1, 72, 66, 75, 106, 76, 166, 1, - 77, 38, 78, 34, 79, 94, 80, 34, 83, 190, 2, 84, 156, 1, 5, 69, 73, 71, - 72, 84, 22, 85, 90, 86, 166, 199, 4, 65, 222, 105, 87, 207, 96, 73, 90, - 128, 1, 21, 74, 75, 32, 85, 78, 73, 70, 73, 69, 68, 32, 73, 68, 69, 79, - 71, 82, 65, 80, 72, 45, 134, 216, 5, 79, 219, 146, 1, 76, 86, 68, 2, 52, - 69, 82, 53, 206, 2, 54, 150, 2, 55, 166, 1, 56, 95, 57, 10, 50, 48, 234, - 4, 65, 154, 190, 5, 56, 207, 96, 50, 4, 222, 233, 6, 48, 3, 57, 30, 150, - 1, 50, 42, 51, 38, 52, 48, 2, 53, 66, 0, 2, 68, 69, 22, 57, 164, 4, 2, - 56, 70, 162, 143, 3, 66, 248, 172, 2, 2, 70, 56, 205, 96, 2, 49, 56, 6, - 214, 4, 55, 194, 157, 6, 49, 3, 52, 4, 150, 148, 3, 70, 143, 173, 2, 67, - 4, 26, 48, 159, 148, 3, 51, 2, 195, 231, 6, 56, 2, 175, 231, 6, 54, 4, - 242, 147, 3, 50, 243, 208, 3, 49, 24, 86, 50, 46, 51, 50, 53, 34, 54, 16, - 2, 55, 48, 28, 2, 70, 49, 129, 2, 2, 69, 56, 6, 70, 57, 238, 145, 3, 53, - 203, 153, 2, 52, 4, 26, 53, 199, 146, 3, 48, 2, 215, 229, 6, 53, 4, 202, - 2, 66, 211, 143, 3, 57, 2, 171, 2, 50, 4, 146, 229, 6, 56, 3, 57, 2, 247, - 228, 6, 52, 12, 74, 53, 38, 57, 12, 2, 49, 50, 20, 2, 68, 52, 157, 225, - 6, 2, 65, 55, 4, 170, 144, 3, 51, 227, 210, 2, 49, 2, 11, 56, 2, 231, - 227, 6, 49, 2, 211, 227, 6, 50, 6, 50, 57, 20, 2, 68, 55, 209, 143, 3, 2, - 67, 65, 2, 159, 143, 3, 69, 2, 247, 226, 6, 48, 4, 204, 156, 6, 2, 49, - 52, 221, 67, 2, 48, 52, 2, 11, 79, 2, 251, 183, 3, 84, 10, 84, 3, 65, 76, - 76, 104, 4, 79, 85, 82, 32, 240, 5, 3, 73, 86, 69, 179, 238, 4, 82, 2, - 57, 12, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, 2, 11, 32, 2, 11, - 83, 2, 191, 251, 4, 76, 4, 210, 179, 3, 68, 131, 173, 3, 75, 8, 192, 181, - 3, 2, 73, 45, 158, 242, 2, 68, 210, 56, 67, 3, 86, 8, 56, 8, 65, 84, 65, - 75, 65, 78, 65, 32, 239, 205, 6, 69, 6, 150, 172, 6, 75, 202, 28, 68, - 159, 20, 83, 60, 36, 5, 65, 84, 73, 78, 32, 79, 79, 54, 164, 5, 12, 83, - 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 227, 112, 67, 6, 178, 178, 3, - 71, 157, 149, 2, 3, 83, 83, 76, 4, 234, 218, 4, 73, 251, 130, 2, 86, 4, - 174, 223, 5, 69, 147, 126, 71, 4, 140, 3, 15, 78, 69, 32, 72, 85, 78, 68, - 82, 69, 68, 32, 84, 87, 69, 78, 147, 218, 6, 75, 4, 158, 2, 80, 187, 215, - 4, 76, 20, 110, 69, 146, 1, 72, 20, 2, 73, 88, 198, 200, 4, 65, 150, 66, - 77, 130, 47, 81, 234, 98, 79, 210, 61, 68, 3, 83, 4, 56, 7, 67, 79, 78, - 68, 32, 83, 67, 21, 3, 86, 69, 78, 2, 247, 137, 5, 82, 2, 29, 5, 32, 80, - 79, 73, 78, 2, 11, 84, 2, 143, 177, 5, 32, 2, 163, 218, 6, 86, 2, 17, 2, - 84, 89, 2, 239, 217, 6, 32, 8, 44, 4, 72, 82, 69, 69, 22, 87, 243, 4, 73, - 2, 163, 147, 6, 32, 4, 68, 13, 69, 78, 84, 89, 45, 84, 87, 79, 32, 80, - 79, 73, 78, 19, 79, 2, 215, 75, 84, 2, 179, 213, 6, 32, 4, 48, 6, 80, 32, - 87, 73, 84, 72, 243, 145, 6, 72, 2, 17, 2, 32, 69, 2, 235, 108, 88, 4, - 202, 145, 6, 79, 151, 70, 83, 2, 201, 183, 5, 6, 32, 66, 76, 65, 67, 75, - 6, 146, 215, 6, 50, 2, 51, 3, 65, 85, 134, 1, 65, 138, 7, 69, 236, 1, 10, - 73, 67, 75, 32, 70, 73, 71, 85, 82, 69, 182, 1, 79, 90, 82, 234, 2, 85, - 130, 200, 6, 83, 3, 88, 38, 92, 6, 70, 70, 32, 79, 70, 32, 106, 77, 80, - 2, 78, 68, 90, 82, 206, 3, 84, 175, 167, 5, 68, 4, 60, 8, 65, 69, 83, 67, - 85, 76, 65, 80, 21, 3, 72, 69, 82, 2, 219, 209, 4, 73, 2, 211, 192, 5, - 77, 2, 21, 3, 80, 69, 68, 2, 17, 2, 32, 69, 2, 173, 210, 1, 4, 78, 86, - 69, 76, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 80, 2, 11, 69, 2, 11, 82, 2, - 171, 131, 6, 83, 24, 42, 32, 205, 1, 5, 84, 32, 79, 70, 32, 12, 82, 69, - 54, 79, 240, 224, 1, 8, 65, 78, 68, 32, 67, 82, 69, 83, 131, 229, 1, 87, - 2, 17, 2, 81, 85, 2, 11, 65, 2, 135, 148, 6, 76, 4, 44, 5, 70, 32, 68, - 65, 86, 143, 167, 3, 80, 2, 255, 138, 6, 73, 12, 66, 71, 30, 83, 40, 4, - 80, 82, 79, 84, 142, 66, 72, 155, 102, 84, 2, 89, 4, 85, 65, 82, 68, 4, - 26, 69, 155, 239, 1, 84, 2, 11, 76, 2, 21, 3, 69, 67, 84, 2, 29, 5, 69, - 68, 32, 65, 82, 2, 135, 205, 6, 69, 4, 144, 229, 4, 10, 85, 69, 32, 79, - 70, 32, 76, 73, 66, 69, 187, 154, 1, 73, 8, 92, 2, 65, 77, 112, 9, 78, - 79, 71, 82, 65, 80, 72, 73, 67, 193, 203, 1, 4, 84, 72, 79, 83, 4, 54, - 32, 173, 196, 5, 7, 73, 78, 71, 32, 66, 79, 87, 2, 33, 6, 76, 79, 67, 79, - 77, 79, 2, 135, 197, 1, 84, 2, 137, 130, 4, 2, 32, 70, 11, 11, 32, 8, 60, - 5, 87, 73, 84, 72, 32, 193, 153, 2, 4, 76, 69, 65, 78, 4, 32, 4, 65, 82, - 77, 83, 51, 68, 2, 25, 4, 32, 82, 65, 73, 2, 167, 236, 1, 83, 2, 143, - 181, 5, 82, 4, 44, 4, 67, 75, 32, 67, 21, 3, 80, 87, 65, 2, 139, 196, 5, - 72, 2, 211, 168, 6, 84, 12, 66, 65, 108, 10, 69, 83, 83, 32, 79, 85, 84, - 76, 73, 78, 79, 73, 6, 44, 4, 73, 71, 72, 84, 45, 3, 87, 66, 69, 4, 188, - 130, 3, 2, 32, 82, 243, 176, 2, 78, 2, 147, 61, 82, 2, 17, 2, 69, 68, 2, - 17, 2, 32, 87, 2, 145, 182, 4, 4, 72, 73, 84, 69, 4, 52, 4, 78, 71, 32, - 84, 233, 7, 4, 67, 84, 76, 89, 2, 17, 2, 69, 82, 2, 169, 143, 6, 3, 77, - 73, 78, 6, 100, 8, 68, 73, 79, 32, 77, 73, 67, 82, 20, 9, 70, 70, 69, 68, - 32, 70, 76, 65, 84, 207, 196, 6, 80, 2, 215, 157, 5, 79, 2, 11, 66, 2, - 231, 248, 3, 82, 202, 2, 150, 1, 66, 196, 2, 5, 67, 67, 69, 69, 68, 192, - 3, 8, 77, 77, 65, 84, 73, 79, 78, 32, 70, 78, 132, 15, 3, 80, 69, 82, - 148, 11, 2, 82, 70, 47, 83, 63, 11, 83, 60, 72, 6, 67, 82, 73, 80, 84, - 32, 124, 3, 69, 84, 32, 93, 3, 84, 73, 84, 30, 222, 22, 69, 162, 1, 80, - 150, 208, 2, 77, 202, 1, 76, 26, 82, 222, 237, 1, 70, 30, 83, 42, 84, 62, - 90, 130, 83, 78, 15, 79, 28, 56, 6, 65, 66, 79, 86, 69, 32, 246, 25, 79, - 159, 3, 87, 6, 142, 24, 83, 203, 139, 5, 82, 2, 235, 254, 4, 85, 22, 11, - 83, 23, 11, 32, 20, 128, 1, 6, 65, 66, 79, 86, 69, 32, 156, 1, 7, 66, 85, - 84, 32, 78, 79, 84, 32, 2, 79, 82, 229, 255, 2, 5, 85, 78, 68, 69, 82, - 12, 100, 4, 78, 79, 84, 32, 28, 12, 83, 73, 78, 71, 76, 69, 45, 76, 73, - 78, 69, 32, 218, 24, 65, 39, 69, 4, 242, 24, 65, 167, 1, 69, 4, 250, 24, - 69, 83, 78, 2, 93, 5, 32, 69, 81, 85, 73, 4, 17, 2, 32, 69, 4, 17, 2, 81, - 85, 4, 22, 73, 167, 25, 65, 2, 173, 25, 6, 86, 65, 76, 69, 78, 84, 6, - 202, 246, 2, 66, 228, 196, 1, 4, 87, 73, 84, 72, 147, 139, 1, 84, 161, 1, - 166, 1, 32, 72, 7, 68, 65, 78, 69, 83, 69, 32, 200, 12, 4, 82, 73, 83, - 69, 200, 166, 2, 14, 83, 69, 84, 32, 79, 86, 69, 82, 32, 66, 85, 73, 76, - 68, 255, 240, 2, 70, 4, 50, 87, 197, 146, 5, 6, 66, 69, 72, 73, 78, 68, - 2, 247, 255, 5, 73, 146, 1, 198, 2, 65, 20, 17, 67, 79, 78, 83, 79, 78, - 65, 78, 84, 32, 83, 73, 71, 78, 32, 80, 65, 172, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 188, 3, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, - 66, 73, 78, 68, 85, 32, 228, 1, 5, 83, 73, 71, 78, 32, 200, 1, 13, 86, - 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 80, 65, 255, 195, 4, 68, 2, 235, - 211, 2, 86, 10, 68, 2, 77, 73, 40, 2, 78, 89, 33, 7, 83, 65, 78, 71, 65, - 78, 32, 2, 17, 2, 78, 71, 2, 167, 176, 5, 75, 4, 142, 5, 65, 163, 146, 1, - 73, 4, 226, 182, 6, 77, 3, 87, 76, 246, 1, 65, 58, 70, 78, 76, 2, 82, 34, - 83, 202, 230, 2, 69, 246, 215, 1, 78, 246, 175, 1, 66, 2, 75, 254, 68, - 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 84, 2, 86, 2, 87, - 2, 88, 2, 89, 2, 90, 186, 2, 73, 2, 79, 3, 85, 7, 128, 247, 5, 6, 82, 67, - 72, 65, 73, 67, 135, 64, 69, 6, 44, 5, 73, 78, 65, 76, 32, 163, 182, 6, - 65, 4, 158, 182, 6, 75, 3, 77, 4, 154, 145, 6, 69, 235, 36, 65, 4, 170, - 179, 6, 89, 187, 2, 65, 16, 90, 66, 2, 68, 2, 75, 12, 3, 76, 69, 85, 38, - 67, 34, 80, 189, 177, 6, 3, 83, 85, 82, 2, 11, 65, 2, 137, 94, 5, 32, 83, - 65, 84, 65, 2, 11, 65, 2, 247, 239, 5, 75, 4, 168, 3, 3, 65, 78, 71, 141, - 238, 5, 3, 85, 82, 78, 10, 28, 2, 80, 65, 211, 82, 86, 8, 28, 3, 77, 65, - 65, 23, 78, 2, 143, 178, 6, 69, 6, 18, 71, 71, 89, 4, 38, 76, 177, 236, - 5, 3, 87, 73, 83, 2, 165, 144, 5, 2, 65, 89, 2, 141, 175, 6, 2, 69, 67, - 12, 18, 77, 23, 78, 2, 179, 244, 2, 69, 10, 96, 3, 69, 85, 76, 34, 79, - 22, 89, 140, 220, 1, 3, 71, 72, 85, 157, 152, 2, 4, 65, 69, 76, 65, 2, - 11, 69, 2, 147, 224, 5, 85, 2, 139, 244, 3, 76, 2, 203, 142, 1, 85, 5, - 173, 130, 3, 13, 32, 79, 86, 69, 82, 32, 77, 79, 85, 78, 84, 65, 73, 72, - 54, 83, 166, 195, 4, 72, 241, 64, 4, 86, 73, 76, 76, 68, 56, 6, 67, 82, - 73, 80, 84, 32, 173, 2, 3, 69, 84, 32, 34, 118, 69, 34, 76, 130, 1, 80, - 150, 208, 2, 77, 226, 1, 82, 222, 237, 1, 70, 30, 83, 42, 84, 62, 90, - 130, 83, 78, 15, 79, 4, 226, 66, 81, 235, 192, 5, 73, 6, 96, 18, 65, 84, - 73, 78, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 135, 210, - 2, 69, 4, 138, 173, 6, 73, 3, 78, 2, 155, 147, 3, 76, 34, 148, 1, 6, 65, - 66, 79, 86, 69, 32, 96, 7, 66, 69, 83, 73, 68, 69, 32, 162, 1, 79, 158, - 3, 87, 153, 148, 4, 9, 80, 82, 69, 67, 69, 68, 73, 78, 71, 6, 26, 83, - 231, 187, 2, 76, 4, 11, 85, 4, 26, 80, 191, 194, 4, 66, 2, 185, 194, 4, - 2, 69, 82, 4, 108, 23, 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 66, - 89, 32, 68, 65, 83, 72, 32, 87, 73, 84, 72, 23, 83, 2, 17, 2, 32, 83, 2, - 153, 193, 4, 2, 85, 66, 16, 11, 70, 17, 11, 32, 14, 120, 6, 65, 66, 79, - 86, 69, 32, 132, 1, 4, 87, 73, 84, 72, 181, 166, 5, 11, 79, 82, 32, 69, - 81, 85, 65, 76, 32, 84, 79, 8, 34, 65, 38, 69, 18, 84, 67, 78, 2, 11, 76, - 2, 113, 3, 77, 79, 83, 2, 187, 60, 81, 2, 21, 3, 73, 76, 68, 2, 147, 253, - 2, 69, 2, 17, 2, 32, 78, 2, 11, 79, 2, 11, 84, 2, 11, 32, 2, 11, 69, 2, - 17, 2, 81, 85, 2, 11, 65, 2, 11, 76, 2, 179, 218, 2, 32, 6, 25, 4, 73, - 84, 72, 32, 6, 96, 2, 80, 76, 24, 14, 77, 85, 76, 84, 73, 80, 76, 73, 67, - 65, 84, 73, 79, 78, 231, 212, 5, 68, 2, 11, 85, 2, 11, 83, 2, 169, 254, - 3, 5, 32, 83, 73, 71, 78, 4, 196, 160, 4, 2, 65, 67, 211, 203, 1, 69, 4, - 60, 9, 80, 69, 78, 83, 73, 79, 78, 32, 82, 167, 144, 6, 72, 2, 21, 3, 65, - 73, 76, 2, 247, 151, 5, 87, 8, 26, 65, 98, 73, 35, 85, 4, 44, 5, 83, 72, - 32, 65, 77, 135, 163, 6, 78, 2, 173, 223, 5, 7, 80, 69, 82, 83, 65, 78, - 68, 2, 11, 77, 2, 231, 228, 5, 77, 2, 129, 250, 2, 2, 78, 71, 230, 2, 92, - 11, 76, 79, 84, 73, 32, 78, 65, 71, 82, 73, 32, 226, 5, 77, 182, 15, 78, - 97, 2, 82, 73, 90, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 168, 2, 11, 80, - 79, 69, 84, 82, 89, 32, 77, 65, 82, 75, 56, 5, 83, 73, 71, 78, 32, 145, - 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 64, 174, 1, 68, 46, - 82, 34, 84, 154, 89, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 246, 165, 5, - 72, 2, 76, 2, 77, 2, 78, 2, 83, 246, 30, 65, 2, 69, 2, 73, 2, 79, 3, 85, - 8, 226, 89, 68, 246, 165, 5, 72, 247, 30, 79, 4, 170, 255, 5, 82, 247, - 30, 79, 8, 150, 89, 84, 246, 165, 5, 72, 247, 30, 79, 8, 11, 45, 8, 198, - 157, 6, 49, 2, 50, 2, 51, 3, 52, 8, 46, 65, 70, 72, 213, 215, 5, 3, 68, - 86, 73, 4, 64, 10, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 203, 215, 5, - 78, 2, 129, 137, 2, 2, 65, 83, 10, 150, 253, 5, 79, 246, 30, 65, 2, 69, - 2, 73, 3, 85, 82, 60, 8, 66, 79, 76, 32, 70, 79, 82, 32, 213, 14, 2, 77, - 69, 80, 238, 2, 66, 48, 2, 67, 65, 118, 68, 222, 1, 69, 130, 2, 70, 56, - 8, 72, 79, 82, 73, 90, 79, 78, 84, 0, 6, 86, 69, 82, 84, 73, 67, 44, 3, - 76, 73, 78, 52, 9, 77, 65, 82, 75, 83, 32, 67, 72, 65, 22, 78, 90, 65, - 68, 3, 82, 69, 67, 32, 5, 71, 82, 79, 85, 80, 0, 4, 85, 78, 73, 84, 22, - 83, 205, 3, 15, 84, 89, 80, 69, 32, 65, 32, 69, 76, 69, 67, 84, 82, 79, - 78, 4, 242, 147, 4, 69, 133, 198, 1, 3, 65, 67, 75, 4, 44, 3, 82, 82, 73, - 237, 182, 1, 2, 78, 67, 2, 33, 6, 65, 71, 69, 32, 82, 69, 2, 11, 84, 2, - 135, 185, 1, 85, 14, 24, 2, 65, 84, 55, 69, 2, 161, 3, 9, 65, 32, 76, 73, - 78, 75, 32, 69, 83, 12, 76, 12, 86, 73, 67, 69, 32, 67, 79, 78, 84, 82, - 79, 76, 157, 8, 2, 76, 69, 8, 11, 32, 8, 222, 174, 1, 70, 154, 250, 2, - 84, 203, 83, 79, 12, 22, 78, 207, 1, 83, 10, 48, 5, 68, 32, 79, 70, 32, - 137, 1, 2, 81, 85, 8, 18, 77, 31, 84, 2, 193, 236, 4, 2, 69, 68, 6, 64, - 11, 82, 65, 78, 83, 77, 73, 83, 83, 73, 79, 78, 159, 108, 69, 5, 143, - 171, 3, 32, 2, 155, 7, 73, 2, 205, 206, 4, 2, 67, 65, 4, 36, 2, 73, 76, - 73, 3, 79, 82, 77, 2, 191, 2, 69, 2, 129, 162, 1, 6, 65, 76, 32, 84, 65, - 66, 2, 11, 69, 2, 17, 2, 32, 70, 2, 223, 178, 1, 69, 2, 191, 219, 4, 80, - 6, 26, 69, 175, 141, 4, 85, 4, 56, 8, 71, 65, 84, 73, 86, 69, 32, 65, - 247, 247, 4, 87, 2, 21, 3, 67, 75, 78, 2, 21, 3, 79, 87, 76, 2, 191, 136, - 1, 69, 2, 11, 79, 2, 17, 2, 82, 68, 2, 191, 247, 2, 32, 18, 176, 1, 7, - 65, 77, 65, 82, 73, 84, 65, 60, 3, 72, 73, 70, 52, 8, 84, 65, 82, 84, 32, - 79, 70, 32, 64, 7, 85, 66, 83, 84, 73, 84, 85, 204, 1, 3, 89, 78, 67, - 255, 206, 5, 80, 2, 25, 4, 78, 32, 83, 79, 2, 11, 85, 2, 219, 209, 5, 82, - 4, 17, 2, 84, 32, 4, 226, 148, 5, 79, 239, 41, 73, 4, 22, 72, 155, 102, - 84, 2, 17, 2, 69, 65, 2, 223, 188, 5, 68, 4, 11, 84, 4, 11, 69, 5, 11, - 32, 2, 25, 4, 70, 79, 82, 77, 2, 17, 2, 32, 84, 2, 251, 237, 5, 87, 2, - 137, 207, 5, 2, 73, 67, 2, 11, 84, 2, 235, 250, 5, 82, 7, 38, 67, 145, - 171, 2, 3, 65, 71, 79, 2, 141, 134, 1, 9, 72, 82, 79, 78, 79, 85, 83, 32, - 73, 180, 1, 36, 3, 65, 67, 32, 163, 215, 4, 78, 178, 1, 150, 3, 65, 52, - 4, 66, 65, 82, 82, 32, 2, 67, 79, 80, 13, 68, 79, 84, 84, 69, 68, 32, 90, - 76, 65, 77, 65, 32, 118, 69, 90, 72, 204, 2, 7, 76, 69, 84, 84, 69, 82, - 32, 236, 9, 9, 79, 66, 76, 73, 81, 85, 69, 32, 76, 28, 4, 80, 84, 72, 65, - 0, 4, 90, 81, 65, 80, 110, 82, 88, 2, 83, 85, 174, 2, 84, 216, 205, 4, 3, - 77, 85, 83, 228, 21, 7, 70, 69, 77, 73, 78, 73, 78, 229, 136, 1, 5, 81, - 85, 83, 72, 83, 2, 11, 66, 2, 197, 158, 5, 5, 66, 82, 69, 86, 73, 2, 11, - 69, 2, 167, 134, 6, 75, 6, 38, 78, 165, 17, 4, 76, 79, 78, 32, 2, 17, 2, - 84, 82, 2, 151, 182, 5, 65, 4, 28, 3, 65, 78, 71, 35, 72, 2, 11, 85, 2, - 255, 227, 4, 76, 2, 11, 79, 2, 185, 184, 3, 5, 82, 73, 90, 79, 78, 6, 60, - 10, 78, 68, 32, 79, 70, 32, 80, 65, 82, 65, 151, 14, 83, 2, 201, 11, 2, - 71, 82, 14, 104, 8, 65, 82, 75, 76, 69, 65, 78, 32, 132, 1, 4, 66, 65, - 83, 65, 65, 7, 79, 82, 73, 90, 79, 78, 84, 6, 60, 5, 65, 83, 84, 69, 82, - 40, 4, 77, 69, 84, 79, 3, 79, 2, 17, 2, 73, 83, 2, 219, 128, 4, 67, 2, - 181, 166, 2, 2, 66, 69, 6, 164, 11, 8, 45, 69, 83, 65, 83, 65, 32, 68, - 159, 223, 3, 32, 2, 197, 178, 5, 2, 65, 76, 92, 154, 2, 68, 102, 72, 32, - 3, 76, 65, 77, 34, 77, 204, 1, 2, 80, 69, 142, 1, 82, 78, 83, 144, 1, 8, - 70, 73, 78, 65, 76, 32, 83, 69, 106, 65, 14, 75, 2, 81, 34, 84, 40, 5, - 71, 65, 77, 65, 76, 40, 4, 89, 85, 68, 72, 146, 75, 66, 166, 130, 4, 90, - 154, 43, 78, 254, 1, 87, 159, 126, 69, 4, 76, 14, 79, 84, 76, 69, 83, 83, - 32, 68, 65, 76, 65, 84, 72, 32, 139, 3, 65, 2, 139, 193, 2, 82, 4, 11, - 69, 5, 159, 254, 5, 84, 2, 11, 65, 2, 255, 253, 5, 68, 24, 60, 9, 65, 76, - 65, 89, 65, 76, 65, 77, 32, 231, 237, 5, 73, 22, 78, 76, 22, 78, 158, - 134, 4, 66, 242, 210, 1, 84, 138, 34, 83, 14, 74, 3, 82, 4, 199, 247, 3, - 76, 8, 182, 84, 78, 234, 166, 5, 71, 3, 89, 9, 33, 6, 82, 83, 73, 65, 78, - 32, 6, 50, 66, 16, 3, 68, 72, 65, 17, 3, 71, 72, 65, 2, 131, 79, 72, 2, - 147, 2, 76, 2, 143, 243, 4, 77, 4, 52, 7, 69, 86, 69, 82, 83, 69, 68, - 211, 150, 4, 73, 2, 251, 182, 4, 32, 14, 142, 1, 69, 40, 7, 79, 71, 68, - 73, 65, 78, 32, 64, 12, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, - 166, 250, 4, 72, 177, 82, 2, 65, 68, 2, 17, 2, 77, 75, 2, 175, 251, 3, - 65, 6, 42, 90, 32, 2, 75, 72, 131, 227, 5, 70, 2, 195, 206, 4, 72, 2, 11, - 76, 2, 11, 65, 2, 183, 248, 5, 80, 6, 36, 3, 69, 84, 72, 255, 250, 4, 65, - 5, 237, 85, 6, 32, 71, 65, 82, 83, 72, 5, 215, 203, 5, 32, 4, 225, 223, - 3, 2, 73, 78, 6, 21, 3, 72, 65, 32, 6, 42, 68, 186, 223, 3, 66, 223, 155, - 1, 65, 2, 17, 2, 79, 84, 2, 155, 152, 1, 84, 8, 58, 66, 210, 144, 2, 87, - 217, 189, 1, 4, 85, 75, 75, 65, 4, 173, 204, 3, 2, 65, 83, 14, 88, 8, 66, - 76, 73, 78, 69, 65, 82, 32, 113, 10, 80, 82, 65, 76, 73, 78, 69, 65, 82, - 32, 8, 44, 5, 67, 79, 76, 79, 78, 227, 170, 3, 70, 7, 11, 32, 4, 29, 5, - 83, 75, 69, 87, 69, 4, 171, 202, 5, 68, 6, 44, 5, 67, 79, 76, 79, 78, - 243, 169, 3, 70, 5, 185, 194, 5, 7, 32, 83, 75, 69, 87, 69, 68, 8, 62, - 72, 25, 11, 87, 79, 32, 86, 69, 82, 84, 73, 67, 65, 76, 4, 21, 3, 82, 69, - 69, 4, 25, 4, 32, 68, 79, 84, 4, 239, 218, 3, 83, 148, 41, 102, 45, 58, - 65, 166, 106, 69, 254, 45, 72, 174, 46, 73, 254, 72, 79, 182, 20, 82, - 226, 16, 85, 191, 9, 87, 4, 32, 2, 83, 72, 203, 205, 4, 82, 2, 179, 235, - 4, 73, 244, 26, 182, 1, 66, 82, 71, 192, 17, 2, 73, 32, 182, 29, 75, 152, - 5, 9, 76, 76, 89, 32, 77, 65, 82, 75, 32, 34, 77, 226, 37, 78, 156, 13, - 3, 80, 69, 32, 94, 85, 150, 233, 4, 67, 159, 11, 88, 5, 249, 123, 16, 76, - 69, 32, 84, 69, 78, 78, 73, 83, 32, 80, 65, 68, 68, 76, 69, 144, 2, 78, - 32, 224, 11, 5, 65, 76, 79, 71, 32, 253, 2, 6, 66, 65, 78, 87, 65, 32, - 190, 1, 158, 1, 65, 102, 67, 186, 1, 68, 58, 69, 98, 71, 118, 72, 46, 76, - 222, 3, 80, 54, 81, 62, 82, 106, 83, 194, 77, 78, 166, 213, 1, 84, 166, - 118, 70, 159, 177, 1, 86, 6, 42, 80, 170, 180, 2, 77, 147, 169, 1, 83, 2, - 17, 2, 79, 83, 2, 197, 192, 5, 4, 84, 82, 79, 80, 8, 18, 73, 63, 79, 2, - 25, 4, 82, 67, 85, 77, 2, 213, 2, 4, 70, 76, 69, 88, 6, 26, 77, 231, 156, - 5, 76, 4, 11, 77, 4, 228, 216, 4, 7, 69, 82, 67, 73, 65, 76, 32, 211, - 147, 1, 65, 22, 26, 79, 207, 253, 3, 73, 2, 11, 76, 2, 175, 85, 76, 4, - 18, 81, 43, 88, 2, 17, 2, 85, 65, 2, 207, 209, 2, 76, 2, 221, 129, 5, 4, - 67, 76, 65, 77, 4, 11, 82, 4, 18, 65, 55, 69, 2, 11, 86, 2, 11, 69, 2, - 209, 121, 3, 32, 65, 67, 2, 137, 4, 4, 65, 84, 69, 82, 2, 129, 141, 2, 6, - 89, 80, 72, 69, 78, 45, 114, 38, 65, 246, 2, 69, 183, 129, 4, 79, 104, - 25, 4, 84, 73, 78, 32, 104, 34, 67, 33, 4, 83, 77, 65, 76, 52, 29, 5, 65, - 80, 73, 84, 65, 52, 41, 8, 76, 32, 76, 69, 84, 84, 69, 82, 52, 11, 32, - 52, 254, 231, 5, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 72, 2, - 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 79, 2, 80, 2, 81, 2, 82, 2, 83, - 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 8, 22, 83, 195, 1, 70, - 2, 11, 83, 2, 145, 64, 3, 45, 84, 72, 4, 26, 69, 247, 203, 2, 76, 2, 185, - 68, 2, 82, 67, 4, 11, 85, 4, 26, 79, 235, 154, 3, 69, 2, 227, 251, 4, 84, - 8, 36, 3, 73, 71, 72, 159, 210, 3, 69, 6, 17, 2, 84, 32, 6, 238, 135, 2, - 67, 134, 2, 80, 155, 2, 83, 6, 174, 174, 2, 69, 202, 165, 1, 79, 155, - 211, 1, 80, 46, 80, 7, 76, 69, 84, 84, 69, 82, 32, 208, 1, 5, 83, 73, 71, - 78, 32, 159, 2, 86, 38, 162, 1, 65, 218, 146, 2, 78, 210, 204, 3, 66, 2, - 68, 2, 71, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, - 2, 89, 186, 2, 73, 3, 85, 5, 145, 151, 5, 6, 82, 67, 72, 65, 73, 67, 4, - 18, 80, 51, 86, 2, 25, 4, 65, 77, 85, 68, 2, 159, 191, 2, 80, 2, 247, - 174, 3, 73, 36, 48, 7, 76, 69, 84, 84, 69, 82, 32, 147, 1, 86, 32, 158, - 145, 2, 78, 210, 204, 3, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 77, 2, 80, 2, - 83, 2, 84, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 4, 41, 8, 79, 87, 69, - 76, 32, 83, 73, 71, 4, 11, 78, 4, 179, 158, 5, 32, 212, 3, 112, 10, 76, - 69, 32, 76, 69, 84, 84, 69, 82, 32, 248, 2, 5, 84, 72, 65, 77, 32, 193, - 18, 5, 86, 73, 69, 84, 32, 70, 186, 1, 65, 34, 69, 30, 84, 214, 140, 2, - 78, 138, 144, 1, 79, 206, 247, 1, 75, 2, 80, 162, 7, 85, 222, 61, 70, 2, - 72, 2, 76, 2, 77, 2, 81, 2, 83, 2, 86, 2, 88, 2, 89, 187, 2, 73, 7, 194, - 197, 5, 85, 215, 22, 73, 7, 246, 219, 5, 69, 3, 72, 18, 60, 3, 79, 78, - 69, 234, 147, 5, 83, 254, 68, 72, 187, 2, 65, 10, 11, 45, 10, 142, 219, - 5, 50, 2, 51, 2, 52, 2, 53, 3, 54, 254, 1, 196, 1, 2, 67, 79, 240, 3, 4, - 72, 79, 82, 65, 0, 4, 84, 72, 65, 77, 28, 7, 76, 69, 84, 84, 69, 82, 32, - 220, 4, 5, 83, 73, 71, 78, 32, 217, 5, 11, 86, 79, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 20, 164, 1, 13, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, - 71, 78, 32, 237, 198, 4, 21, 77, 66, 73, 78, 73, 78, 71, 32, 67, 82, 89, - 80, 84, 79, 71, 82, 65, 77, 77, 73, 67, 18, 148, 1, 6, 70, 73, 78, 65, - 76, 32, 22, 76, 56, 16, 72, 73, 71, 72, 32, 82, 65, 84, 72, 65, 32, 79, - 82, 32, 76, 79, 22, 77, 186, 211, 5, 66, 3, 83, 2, 151, 148, 5, 78, 4, - 54, 79, 149, 151, 1, 7, 65, 32, 84, 65, 78, 71, 32, 2, 167, 216, 1, 87, - 6, 48, 6, 69, 68, 73, 65, 76, 32, 191, 213, 5, 65, 4, 130, 211, 5, 76, 3, - 82, 20, 129, 231, 3, 2, 32, 68, 106, 188, 1, 2, 71, 82, 44, 5, 72, 73, - 71, 72, 32, 94, 76, 222, 1, 82, 198, 129, 2, 85, 210, 200, 1, 73, 166, - 15, 78, 182, 216, 1, 79, 162, 8, 69, 158, 20, 66, 2, 68, 2, 77, 2, 87, - 187, 2, 65, 2, 21, 3, 69, 65, 84, 2, 231, 208, 5, 32, 32, 242, 1, 75, 42, - 82, 250, 136, 5, 83, 82, 67, 2, 80, 2, 84, 254, 68, 70, 2, 72, 3, 89, 36, - 60, 3, 79, 87, 32, 234, 145, 5, 65, 194, 41, 85, 159, 20, 76, 28, 86, 75, - 42, 82, 202, 137, 5, 67, 2, 80, 2, 84, 254, 68, 70, 2, 72, 2, 83, 3, 89, - 6, 234, 206, 5, 72, 2, 88, 187, 2, 65, 2, 189, 217, 3, 2, 65, 84, 8, 26, - 65, 243, 185, 5, 85, 7, 138, 206, 5, 78, 3, 84, 50, 182, 1, 72, 34, 75, - 176, 1, 4, 77, 65, 73, 32, 82, 82, 136, 1, 2, 83, 65, 92, 5, 87, 73, 65, - 78, 71, 152, 27, 4, 84, 79, 78, 69, 224, 100, 3, 68, 79, 75, 177, 242, 3, - 2, 67, 65, 4, 130, 254, 4, 65, 167, 63, 79, 14, 52, 4, 72, 85, 69, 78, - 134, 3, 65, 139, 177, 4, 69, 8, 80, 6, 32, 84, 79, 78, 69, 45, 145, 134, - 4, 8, 45, 76, 85, 69, 32, 75, 65, 82, 6, 194, 205, 5, 51, 2, 52, 3, 53, - 8, 56, 4, 75, 65, 78, 71, 194, 136, 1, 89, 235, 138, 3, 83, 5, 243, 141, - 1, 32, 4, 92, 3, 65, 32, 72, 21, 16, 69, 86, 69, 82, 83, 69, 68, 32, 82, - 79, 84, 65, 84, 69, 68, 32, 2, 179, 146, 4, 65, 2, 171, 162, 3, 82, 8, - 48, 3, 84, 75, 65, 226, 131, 4, 87, 139, 119, 75, 4, 17, 2, 65, 78, 5, - 251, 219, 3, 75, 5, 133, 156, 3, 2, 87, 65, 38, 90, 65, 36, 4, 77, 65, - 73, 32, 22, 79, 46, 84, 86, 85, 242, 193, 3, 73, 207, 134, 2, 69, 9, 242, - 201, 5, 65, 2, 69, 3, 73, 2, 255, 181, 4, 83, 11, 218, 176, 3, 65, 226, - 152, 2, 79, 3, 89, 4, 42, 65, 129, 138, 1, 4, 72, 65, 77, 32, 2, 17, 2, - 76, 76, 2, 135, 217, 3, 32, 9, 166, 136, 5, 85, 151, 64, 69, 144, 1, 184, - 1, 7, 76, 69, 84, 84, 69, 82, 32, 188, 2, 5, 77, 65, 73, 32, 75, 32, 7, - 83, 89, 77, 66, 79, 76, 32, 124, 9, 84, 79, 78, 69, 32, 77, 65, 73, 32, - 81, 6, 86, 79, 87, 69, 76, 32, 96, 44, 4, 72, 73, 71, 72, 1, 3, 76, 79, - 87, 48, 11, 32, 48, 154, 1, 75, 30, 67, 2, 80, 2, 84, 34, 78, 214, 165, - 5, 66, 2, 68, 2, 70, 2, 71, 2, 72, 2, 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, - 89, 247, 30, 79, 6, 26, 72, 235, 196, 5, 79, 4, 242, 165, 5, 72, 247, 30, - 79, 6, 210, 165, 5, 71, 2, 89, 247, 30, 79, 4, 178, 243, 4, 65, 183, 52, - 72, 10, 72, 2, 75, 79, 110, 78, 164, 128, 1, 4, 72, 79, 32, 72, 235, 136, - 3, 83, 4, 140, 129, 1, 3, 73, 32, 75, 167, 194, 4, 78, 8, 58, 78, 178, - 134, 1, 84, 186, 255, 1, 83, 203, 185, 2, 69, 2, 151, 134, 3, 85, 26, 50, - 65, 58, 85, 30, 73, 174, 193, 5, 69, 3, 79, 10, 170, 171, 5, 85, 214, 22, - 65, 2, 77, 2, 78, 3, 89, 9, 26, 69, 175, 193, 5, 65, 5, 171, 193, 5, 65, - 138, 1, 52, 3, 82, 73, 32, 173, 180, 4, 4, 69, 79, 85, 84, 136, 1, 82, - 65, 20, 7, 76, 69, 84, 84, 69, 82, 32, 170, 2, 83, 78, 86, 227, 206, 3, - 68, 2, 163, 213, 1, 66, 88, 214, 1, 65, 254, 157, 1, 82, 242, 55, 68, 46, - 84, 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 126, 66, 2, 67, 2, 71, - 2, 74, 2, 75, 2, 80, 2, 83, 254, 68, 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, - 2, 69, 3, 79, 11, 176, 149, 3, 7, 82, 67, 72, 65, 73, 67, 32, 230, 168, - 2, 65, 2, 73, 3, 85, 8, 25, 4, 73, 71, 78, 32, 8, 226, 215, 1, 78, 222, - 160, 3, 65, 239, 1, 86, 18, 49, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, - 32, 18, 238, 215, 1, 65, 190, 21, 85, 210, 200, 1, 73, 206, 134, 2, 69, - 3, 79, 4, 226, 51, 70, 139, 239, 3, 79, 188, 6, 36, 3, 73, 76, 32, 147, - 154, 4, 65, 186, 6, 170, 3, 65, 102, 67, 194, 2, 68, 72, 3, 87, 69, 84, - 48, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 136, 8, 9, 73, 78, 32, 80, 79, - 83, 83, 69, 83, 22, 76, 204, 2, 7, 78, 85, 77, 66, 69, 82, 32, 152, 1, - 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 69, 78, 68, 32, 79, - 70, 54, 82, 42, 83, 188, 14, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, - 32, 78, 84, 228, 1, 2, 89, 69, 184, 143, 4, 5, 77, 79, 78, 84, 72, 147, - 118, 79, 6, 76, 5, 83, 32, 65, 66, 79, 206, 55, 85, 149, 154, 3, 5, 78, - 68, 32, 79, 68, 2, 219, 159, 3, 86, 52, 80, 9, 79, 78, 83, 79, 78, 65, - 78, 84, 32, 192, 21, 3, 85, 82, 82, 143, 10, 82, 48, 130, 1, 75, 22, 76, - 22, 78, 46, 84, 170, 193, 1, 83, 170, 16, 82, 166, 227, 3, 67, 2, 72, 2, - 74, 2, 77, 2, 80, 2, 86, 3, 89, 5, 247, 247, 4, 83, 7, 151, 185, 4, 76, - 11, 158, 189, 3, 78, 130, 248, 1, 71, 3, 89, 5, 243, 180, 5, 84, 26, 68, - 2, 82, 89, 172, 29, 2, 69, 66, 234, 248, 1, 65, 247, 175, 1, 73, 2, 213, - 201, 1, 7, 32, 67, 85, 76, 84, 73, 86, 42, 168, 1, 4, 79, 78, 69, 32, - 252, 4, 6, 84, 72, 82, 69, 69, 32, 165, 172, 5, 22, 68, 79, 87, 78, 83, - 67, 65, 76, 73, 78, 71, 32, 70, 65, 67, 84, 79, 82, 32, 75, 73, 73, 30, - 112, 5, 69, 73, 71, 72, 84, 34, 70, 40, 3, 72, 65, 76, 18, 79, 88, 4, 83, - 73, 88, 84, 110, 84, 187, 175, 3, 81, 4, 242, 3, 73, 215, 173, 5, 72, 4, - 188, 3, 2, 79, 82, 191, 173, 3, 73, 4, 171, 1, 70, 2, 133, 3, 18, 78, 69, - 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 83, 73, 88, 6, 64, 5, - 69, 69, 78, 84, 72, 165, 233, 1, 5, 89, 45, 70, 79, 85, 4, 11, 45, 4, - 198, 175, 5, 49, 3, 50, 8, 38, 72, 138, 1, 87, 239, 174, 3, 69, 4, 132, - 1, 18, 82, 69, 69, 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, - 84, 87, 249, 242, 3, 8, 73, 82, 84, 89, 45, 83, 69, 67, 2, 17, 2, 69, 78, - 2, 17, 2, 84, 73, 2, 207, 174, 3, 69, 10, 58, 69, 28, 4, 83, 73, 88, 84, - 82, 84, 163, 174, 3, 81, 2, 129, 1, 3, 73, 71, 72, 4, 50, 69, 149, 175, - 3, 6, 89, 45, 70, 79, 85, 82, 2, 145, 175, 3, 2, 69, 78, 2, 21, 3, 87, - 69, 78, 2, 221, 174, 3, 3, 84, 73, 69, 2, 247, 209, 1, 83, 72, 48, 6, 69, - 84, 84, 69, 82, 32, 187, 197, 3, 65, 70, 198, 1, 78, 138, 24, 84, 222, - 22, 76, 210, 90, 82, 206, 55, 65, 182, 25, 85, 158, 144, 1, 79, 182, 56, - 73, 202, 190, 1, 83, 242, 7, 69, 222, 61, 67, 2, 72, 2, 74, 2, 75, 2, 77, - 2, 80, 2, 86, 3, 89, 10, 46, 78, 234, 166, 5, 71, 2, 89, 187, 2, 65, 4, - 230, 166, 5, 78, 187, 2, 65, 8, 38, 79, 230, 215, 3, 84, 167, 74, 83, 4, - 11, 78, 4, 17, 2, 69, 32, 4, 18, 72, 31, 84, 2, 213, 72, 3, 85, 78, 68, - 2, 229, 238, 1, 3, 72, 79, 85, 2, 17, 2, 32, 84, 2, 11, 69, 2, 155, 139, - 5, 88, 2, 17, 2, 85, 80, 2, 199, 143, 3, 69, 194, 4, 152, 1, 5, 65, 76, - 84, 32, 80, 20, 4, 73, 71, 78, 32, 206, 4, 80, 28, 9, 84, 65, 82, 84, 73, - 78, 71, 32, 70, 41, 8, 89, 76, 76, 65, 66, 76, 69, 32, 2, 207, 175, 3, - 65, 40, 90, 65, 48, 3, 67, 69, 86, 34, 75, 80, 2, 77, 85, 102, 85, 14, - 80, 118, 86, 163, 65, 78, 4, 216, 2, 4, 65, 90, 72, 65, 195, 221, 4, 78, - 2, 11, 73, 2, 215, 255, 4, 84, 6, 38, 85, 157, 255, 4, 3, 65, 65, 67, 4, - 18, 90, 87, 82, 2, 151, 144, 5, 72, 6, 60, 4, 75, 75, 85, 82, 16, 2, 84, - 72, 21, 3, 85, 86, 85, 2, 163, 83, 85, 2, 203, 241, 2, 65, 2, 75, 90, 8, - 26, 65, 255, 210, 4, 79, 6, 34, 84, 230, 232, 3, 65, 15, 78, 2, 11, 72, - 2, 17, 2, 65, 75, 2, 163, 253, 4, 75, 10, 38, 65, 210, 82, 69, 143, 140, - 4, 73, 4, 176, 103, 3, 82, 65, 65, 213, 202, 2, 6, 75, 65, 73, 89, 65, - 82, 2, 11, 69, 2, 159, 10, 78, 2, 17, 2, 82, 79, 2, 243, 153, 4, 77, 148, - 4, 122, 75, 170, 1, 76, 174, 1, 78, 194, 1, 82, 90, 83, 186, 1, 84, 86, - 67, 2, 72, 2, 74, 2, 77, 2, 80, 2, 86, 3, 89, 46, 88, 2, 83, 83, 150, - 186, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, - 24, 154, 182, 1, 65, 182, 25, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, - 1, 69, 66, 82, 76, 246, 184, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, - 56, 73, 187, 198, 1, 69, 44, 250, 4, 76, 250, 179, 1, 65, 190, 21, 85, - 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 110, 102, 78, 190, 3, 71, - 2, 89, 250, 179, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, - 198, 1, 69, 44, 186, 3, 78, 250, 179, 1, 65, 190, 21, 85, 158, 144, 1, - 79, 182, 56, 73, 187, 198, 1, 69, 44, 226, 2, 82, 250, 179, 1, 65, 190, - 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 68, 94, 72, 174, - 1, 83, 250, 179, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, - 198, 1, 69, 24, 162, 181, 1, 65, 190, 21, 85, 158, 144, 1, 79, 230, 2, - 82, 210, 53, 73, 187, 198, 1, 69, 44, 82, 84, 250, 179, 1, 65, 190, 21, - 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 22, 246, 179, 1, 65, - 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 6, 68, 2, 79, - 84, 33, 11, 82, 65, 68, 73, 84, 73, 79, 78, 65, 76, 32, 2, 11, 65, 2, - 199, 144, 4, 76, 4, 24, 2, 67, 82, 55, 78, 2, 17, 2, 69, 68, 2, 11, 73, - 2, 247, 143, 4, 84, 2, 11, 85, 2, 17, 2, 77, 66, 2, 23, 69, 2, 11, 65, 2, - 171, 143, 4, 82, 184, 13, 36, 5, 65, 66, 65, 84, 65, 35, 71, 2, 11, 32, - 2, 135, 169, 3, 84, 182, 13, 54, 69, 20, 3, 83, 65, 32, 241, 6, 3, 85, - 84, 32, 2, 215, 251, 3, 82, 178, 1, 52, 7, 76, 69, 84, 84, 69, 82, 32, - 143, 166, 3, 68, 158, 1, 210, 1, 65, 58, 70, 54, 72, 62, 76, 58, 77, 54, - 78, 50, 83, 126, 85, 114, 69, 2, 73, 2, 79, 2, 86, 158, 198, 4, 84, 82, - 67, 2, 68, 2, 71, 2, 75, 2, 80, 254, 68, 66, 2, 82, 2, 87, 2, 88, 2, 89, - 3, 90, 16, 174, 4, 87, 162, 142, 5, 67, 2, 81, 2, 88, 3, 90, 4, 168, 193, - 4, 5, 73, 78, 65, 76, 32, 239, 80, 65, 6, 26, 84, 203, 145, 5, 65, 4, - 142, 143, 5, 84, 187, 2, 65, 4, 192, 236, 3, 5, 79, 78, 71, 32, 85, 231, - 164, 1, 65, 10, 238, 144, 5, 65, 2, 67, 2, 81, 2, 88, 3, 90, 8, 130, 142, - 5, 71, 2, 72, 2, 89, 187, 2, 65, 8, 26, 72, 243, 143, 5, 65, 6, 40, 4, - 79, 82, 84, 32, 199, 143, 5, 65, 4, 208, 152, 1, 2, 85, 69, 229, 234, 2, - 2, 65, 87, 32, 58, 73, 54, 69, 162, 142, 5, 67, 2, 81, 2, 88, 3, 90, 16, - 50, 85, 162, 142, 5, 67, 2, 81, 2, 88, 3, 90, 8, 158, 142, 5, 67, 2, 81, + 2, 78, 32, 2, 11, 66, 2, 233, 212, 6, 4, 76, 65, 67, 75, 5, 145, 199, 4, + 7, 32, 87, 73, 84, 72, 32, 79, 2, 179, 251, 3, 84, 70, 52, 7, 76, 69, 84, + 84, 69, 82, 32, 159, 131, 6, 68, 50, 198, 1, 69, 32, 2, 77, 65, 30, 78, + 186, 146, 1, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, + 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 242, 222, 6, 65, 2, 73, 2, 79, 3, 85, + 4, 226, 241, 7, 69, 147, 1, 72, 4, 210, 242, 7, 69, 3, 72, 6, 182, 146, + 1, 71, 2, 89, 243, 222, 6, 65, 58, 104, 3, 84, 72, 32, 169, 160, 6, 17, + 78, 68, 32, 82, 69, 67, 79, 82, 68, 73, 78, 71, 32, 67, 79, 80, 89, 56, + 60, 5, 69, 65, 83, 84, 32, 253, 2, 5, 87, 69, 83, 84, 32, 30, 96, 5, 65, + 82, 82, 79, 87, 194, 4, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, 18, 83, + 247, 58, 87, 13, 11, 32, 10, 68, 2, 65, 78, 38, 67, 120, 2, 84, 79, 154, + 2, 87, 203, 134, 5, 70, 2, 253, 2, 5, 68, 32, 83, 79, 85, 2, 37, 7, 82, + 79, 83, 83, 73, 78, 71, 2, 17, 2, 32, 78, 2, 17, 2, 79, 82, 2, 173, 133, + 5, 5, 84, 72, 32, 69, 65, 2, 17, 2, 32, 67, 2, 243, 241, 3, 79, 26, 96, + 5, 65, 82, 82, 79, 87, 198, 1, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, + 18, 83, 247, 58, 87, 9, 11, 32, 6, 52, 5, 65, 78, 68, 32, 78, 74, 87, + 203, 134, 5, 70, 2, 17, 2, 79, 82, 2, 21, 3, 84, 72, 32, 2, 137, 131, 5, + 2, 87, 69, 2, 21, 3, 73, 84, 72, 2, 11, 32, 2, 251, 151, 6, 72, 6, 41, 8, + 79, 73, 78, 84, 73, 78, 71, 32, 6, 42, 86, 242, 160, 4, 76, 155, 158, 2, + 66, 2, 17, 2, 73, 78, 2, 199, 160, 4, 69, 4, 73, 16, 82, 73, 65, 78, 71, + 76, 69, 45, 72, 69, 65, 68, 69, 68, 32, 65, 4, 25, 4, 82, 82, 79, 87, 5, + 11, 32, 2, 203, 137, 5, 84, 166, 1, 220, 3, 23, 67, 76, 85, 83, 84, 69, + 82, 45, 73, 78, 73, 84, 73, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 40, + 21, 70, 73, 78, 65, 76, 32, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, + 73, 71, 78, 32, 112, 27, 72, 69, 65, 68, 32, 77, 65, 82, 75, 32, 87, 73, + 84, 72, 32, 77, 79, 79, 78, 32, 65, 78, 68, 32, 83, 85, 78, 104, 7, 76, + 69, 84, 84, 69, 82, 32, 180, 1, 5, 77, 65, 82, 75, 32, 54, 83, 100, 8, + 84, 69, 82, 77, 73, 78, 65, 76, 40, 6, 86, 79, 87, 69, 76, 32, 207, 132, + 4, 71, 8, 142, 158, 7, 83, 138, 69, 76, 3, 82, 24, 138, 197, 3, 83, 134, + 165, 2, 78, 226, 248, 1, 45, 186, 2, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, + 77, 3, 82, 7, 29, 5, 32, 65, 78, 68, 32, 4, 50, 70, 1, 8, 84, 82, 73, 80, + 76, 69, 32, 70, 2, 235, 212, 5, 76, 82, 230, 188, 2, 68, 138, 222, 4, 75, + 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 74, 2, 80, 2, 90, 138, + 69, 45, 2, 72, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 8, 222, + 140, 4, 80, 206, 142, 3, 68, 58, 83, 63, 84, 10, 40, 4, 73, 71, 78, 32, + 139, 159, 7, 85, 8, 246, 168, 5, 74, 158, 2, 86, 122, 85, 187, 240, 1, + 65, 4, 237, 223, 3, 5, 32, 77, 65, 82, 75, 22, 44, 5, 83, 73, 71, 78, 32, + 179, 158, 7, 76, 20, 88, 7, 86, 79, 67, 65, 76, 73, 67, 154, 159, 7, 65, + 26, 79, 2, 85, 162, 64, 69, 3, 73, 4, 11, 32, 4, 194, 223, 7, 76, 3, 82, + 71, 122, 65, 206, 1, 69, 168, 5, 13, 72, 69, 82, 73, 67, 65, 76, 32, 65, + 78, 71, 76, 69, 82, 73, 220, 1, 2, 76, 65, 91, 79, 17, 48, 4, 71, 72, 69, + 84, 22, 82, 147, 199, 7, 67, 2, 159, 202, 7, 84, 10, 24, 2, 75, 76, 59, + 83, 6, 26, 73, 191, 214, 5, 69, 2, 237, 213, 6, 2, 78, 71, 4, 17, 2, 69, + 32, 4, 214, 53, 72, 135, 3, 86, 22, 104, 2, 65, 75, 230, 3, 69, 68, 4, + 83, 77, 73, 76, 141, 214, 3, 9, 67, 75, 76, 69, 32, 70, 73, 76, 76, 12, + 70, 45, 64, 2, 69, 82, 153, 2, 8, 73, 78, 71, 32, 72, 69, 65, 68, 2, 161, + 193, 5, 11, 78, 79, 45, 69, 86, 73, 76, 32, 77, 79, 78, 9, 33, 6, 32, 87, + 73, 84, 72, 32, 6, 26, 67, 78, 79, 55, 84, 2, 33, 6, 65, 78, 67, 69, 76, + 76, 2, 241, 169, 7, 5, 65, 84, 73, 79, 78, 2, 209, 186, 4, 8, 78, 69, 32, + 83, 79, 85, 78, 68, 2, 25, 4, 72, 82, 69, 69, 2, 149, 197, 6, 10, 32, 83, + 79, 85, 78, 68, 32, 87, 65, 86, 2, 11, 32, 2, 185, 145, 6, 9, 73, 78, 32, + 83, 73, 76, 72, 79, 85, 4, 34, 68, 245, 245, 2, 2, 67, 72, 2, 11, 66, 2, + 247, 195, 6, 79, 2, 203, 208, 6, 79, 7, 45, 9, 32, 79, 80, 69, 78, 73, + 78, 71, 32, 4, 154, 172, 7, 76, 227, 42, 85, 10, 44, 3, 68, 69, 82, 41, + 4, 82, 65, 76, 32, 5, 17, 2, 32, 87, 2, 191, 154, 6, 69, 6, 80, 8, 67, + 65, 76, 69, 78, 68, 65, 82, 0, 4, 78, 79, 84, 69, 29, 2, 83, 72, 2, 141, + 143, 7, 2, 32, 80, 2, 223, 205, 5, 69, 4, 64, 10, 83, 72, 73, 78, 71, 32, + 83, 87, 69, 65, 211, 156, 6, 84, 2, 159, 193, 6, 84, 10, 90, 79, 60, 7, + 85, 84, 73, 78, 71, 32, 87, 192, 240, 2, 3, 82, 84, 83, 191, 173, 3, 78, + 4, 240, 87, 7, 76, 32, 79, 70, 32, 84, 72, 231, 251, 6, 78, 2, 11, 72, 2, + 231, 174, 6, 65, 222, 5, 26, 65, 191, 140, 7, 73, 220, 5, 28, 2, 82, 69, + 163, 69, 84, 218, 5, 30, 32, 245, 46, 2, 68, 32, 234, 3, 250, 1, 65, 104, + 2, 86, 32, 94, 66, 158, 1, 67, 170, 1, 68, 102, 69, 194, 2, 70, 146, 2, + 71, 222, 1, 72, 214, 2, 73, 94, 75, 190, 4, 76, 102, 77, 210, 5, 78, 94, + 79, 178, 1, 80, 222, 3, 82, 154, 3, 83, 206, 2, 84, 74, 87, 230, 6, 89, + 251, 211, 6, 85, 16, 102, 32, 58, 80, 132, 73, 2, 78, 80, 144, 153, 2, 3, + 82, 85, 72, 194, 51, 65, 162, 185, 4, 77, 3, 85, 2, 21, 3, 79, 86, 69, 2, + 11, 82, 2, 175, 190, 7, 32, 4, 234, 184, 2, 69, 231, 138, 1, 65, 14, 98, + 65, 36, 4, 85, 83, 83, 89, 170, 148, 3, 73, 216, 74, 2, 79, 82, 138, 53, + 69, 195, 185, 3, 81, 4, 32, 2, 65, 82, 203, 205, 7, 82, 2, 167, 148, 3, + 69, 22, 98, 65, 30, 79, 158, 20, 77, 196, 183, 7, 7, 32, 79, 86, 69, 82, + 32, 75, 74, 85, 14, 67, 3, 68, 4, 206, 204, 7, 76, 3, 80, 5, 17, 2, 82, + 80, 2, 155, 194, 2, 79, 20, 82, 65, 162, 19, 77, 250, 205, 2, 69, 130, + 49, 79, 162, 185, 4, 66, 2, 74, 3, 76, 5, 135, 22, 65, 18, 66, 69, 22, + 82, 164, 25, 5, 83, 85, 75, 85, 85, 175, 177, 7, 86, 2, 151, 216, 5, 75, + 12, 52, 7, 65, 32, 78, 65, 77, 69, 32, 155, 202, 7, 71, 10, 132, 1, 4, + 72, 69, 73, 83, 20, 5, 84, 65, 73, 83, 89, 152, 22, 3, 77, 69, 73, 220, + 193, 2, 3, 82, 69, 73, 1, 4, 83, 89, 79, 85, 2, 183, 181, 7, 69, 2, 147, + 164, 7, 79, 12, 26, 79, 207, 200, 7, 77, 10, 60, 9, 85, 82, 32, 67, 79, + 82, 78, 69, 82, 227, 171, 7, 79, 8, 26, 32, 243, 199, 7, 83, 6, 128, 1, + 11, 66, 76, 65, 67, 75, 32, 84, 82, 73, 65, 78, 216, 62, 6, 68, 73, 65, + 71, 79, 78, 229, 199, 3, 5, 83, 65, 76, 84, 73, 2, 167, 137, 4, 71, 24, + 102, 65, 46, 73, 172, 12, 5, 85, 82, 65, 77, 85, 238, 149, 3, 72, 246, + 160, 4, 80, 186, 2, 66, 3, 89, 6, 130, 246, 6, 82, 222, 46, 78, 147, 33, + 76, 6, 42, 82, 158, 130, 5, 78, 199, 192, 2, 71, 2, 201, 210, 5, 2, 85, + 68, 30, 130, 1, 65, 22, 69, 54, 79, 62, 85, 242, 214, 1, 80, 156, 227, 5, + 10, 73, 82, 65, 71, 65, 78, 65, 32, 72, 79, 234, 8, 71, 3, 90, 5, 167, + 244, 1, 73, 4, 168, 11, 3, 75, 85, 84, 233, 232, 1, 2, 82, 85, 6, 26, 79, + 151, 195, 7, 78, 4, 170, 158, 7, 82, 235, 36, 78, 6, 54, 73, 144, 17, 4, + 65, 82, 65, 68, 175, 231, 5, 82, 2, 255, 211, 3, 73, 14, 54, 78, 212, 12, + 4, 77, 65, 71, 69, 159, 181, 7, 85, 7, 154, 210, 2, 73, 255, 219, 4, 84, + 58, 230, 1, 65, 48, 2, 89, 85, 20, 3, 73, 82, 79, 78, 77, 90, 79, 60, 2, + 85, 82, 144, 8, 2, 69, 69, 134, 146, 3, 72, 238, 153, 3, 67, 232, 120, 3, + 32, 79, 72, 162, 14, 80, 186, 2, 66, 2, 71, 2, 75, 2, 76, 2, 84, 2, 86, + 3, 87, 9, 22, 82, 131, 98, 73, 4, 22, 79, 163, 21, 65, 2, 183, 252, 4, + 82, 9, 252, 16, 3, 77, 69, 69, 140, 4, 2, 87, 65, 129, 171, 2, 3, 71, 85, + 82, 9, 11, 32, 6, 22, 67, 199, 14, 83, 4, 22, 65, 179, 6, 85, 2, 233, + 233, 4, 2, 80, 73, 4, 18, 79, 23, 82, 2, 243, 158, 7, 80, 2, 207, 129, 6, + 85, 4, 164, 206, 5, 4, 85, 90, 69, 73, 215, 68, 79, 12, 62, 79, 244, 13, + 2, 69, 70, 222, 174, 7, 77, 2, 78, 3, 88, 4, 150, 135, 6, 90, 187, 181, + 1, 71, 82, 162, 1, 32, 70, 65, 122, 66, 22, 69, 54, 73, 110, 77, 68, 2, + 85, 32, 78, 86, 2, 87, 222, 147, 3, 72, 250, 153, 3, 79, 254, 134, 1, 80, + 186, 2, 71, 2, 76, 3, 83, 10, 34, 79, 242, 2, 67, 139, 8, 83, 6, 198, 10, + 86, 199, 159, 7, 72, 13, 54, 73, 44, 2, 78, 83, 206, 84, 82, 215, 234, 4, + 72, 4, 240, 202, 5, 2, 75, 85, 179, 202, 1, 82, 2, 243, 233, 6, 89, 5, + 223, 171, 2, 32, 6, 28, 2, 71, 65, 251, 10, 69, 5, 171, 233, 6, 84, 8, + 42, 75, 20, 2, 82, 73, 199, 184, 7, 76, 2, 183, 147, 6, 85, 5, 11, 66, 2, + 11, 65, 2, 139, 255, 2, 65, 7, 11, 32, 4, 22, 67, 139, 8, 83, 2, 11, 85, + 2, 155, 172, 2, 66, 16, 210, 183, 7, 65, 2, 70, 2, 71, 2, 76, 2, 77, 2, + 83, 2, 86, 3, 87, 5, 11, 32, 2, 11, 77, 2, 171, 244, 6, 69, 16, 70, 65, + 130, 12, 79, 150, 170, 7, 70, 2, 77, 2, 83, 2, 86, 3, 87, 5, 159, 151, 7, + 78, 12, 78, 78, 20, 7, 82, 73, 71, 73, 78, 65, 76, 150, 182, 2, 79, 139, + 255, 4, 86, 2, 199, 144, 7, 83, 6, 21, 3, 32, 79, 70, 7, 25, 4, 32, 79, + 82, 32, 4, 242, 96, 78, 51, 69, 46, 118, 65, 82, 69, 94, 73, 74, 79, 142, + 161, 7, 80, 218, 16, 67, 2, 70, 2, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 87, + 9, 38, 65, 229, 254, 3, 3, 32, 65, 77, 4, 236, 1, 2, 83, 69, 219, 140, 7, + 84, 8, 34, 69, 22, 78, 223, 147, 7, 83, 2, 139, 159, 7, 90, 4, 206, 227, + 1, 73, 155, 170, 5, 83, 6, 34, 75, 233, 3, 3, 65, 83, 85, 4, 230, 248, 2, + 85, 163, 185, 4, 79, 6, 34, 73, 22, 78, 21, 2, 83, 73, 2, 131, 195, 3, + 78, 2, 183, 146, 7, 68, 2, 241, 247, 6, 4, 84, 73, 79, 78, 22, 60, 2, 65, + 68, 114, 69, 62, 73, 134, 1, 85, 207, 221, 6, 79, 7, 21, 3, 32, 79, 86, + 4, 25, 4, 69, 82, 32, 83, 5, 17, 2, 32, 83, 2, 11, 81, 2, 133, 164, 2, 2, + 85, 65, 4, 36, 3, 78, 84, 79, 183, 138, 7, 77, 2, 179, 219, 5, 71, 6, 40, + 2, 71, 72, 62, 84, 235, 171, 7, 82, 2, 233, 238, 3, 10, 84, 32, 79, 80, + 69, 78, 32, 66, 79, 88, 2, 253, 244, 2, 2, 84, 79, 4, 228, 1, 2, 85, 66, + 163, 233, 4, 80, 22, 138, 1, 65, 72, 3, 69, 78, 84, 28, 11, 80, 73, 82, + 65, 76, 32, 70, 82, 79, 77, 32, 152, 188, 2, 3, 73, 82, 73, 214, 239, 4, + 82, 3, 86, 4, 48, 2, 73, 75, 149, 173, 2, 4, 78, 84, 73, 73, 2, 251, 242, + 2, 85, 4, 134, 172, 7, 73, 3, 79, 8, 18, 66, 43, 84, 4, 201, 128, 7, 5, + 79, 84, 84, 79, 77, 4, 11, 79, 4, 151, 128, 7, 80, 6, 42, 65, 186, 135, + 3, 72, 207, 211, 3, 79, 2, 201, 245, 6, 2, 82, 71, 34, 50, 65, 20, 4, 73, + 84, 72, 32, 131, 170, 7, 66, 2, 215, 187, 3, 84, 30, 214, 1, 66, 40, 3, + 68, 73, 65, 0, 5, 79, 82, 84, 72, 79, 94, 72, 54, 76, 126, 84, 28, 6, 85, + 80, 80, 69, 82, 32, 186, 1, 86, 178, 136, 4, 82, 173, 130, 2, 13, 67, 79, + 78, 84, 79, 85, 82, 69, 68, 32, 79, 85, 84, 2, 217, 140, 4, 5, 79, 84, + 84, 79, 77, 2, 11, 71, 2, 213, 175, 4, 15, 79, 78, 65, 76, 32, 67, 82, + 79, 83, 83, 72, 65, 84, 67, 72, 2, 11, 79, 2, 149, 3, 6, 82, 73, 90, 79, + 78, 84, 6, 44, 5, 79, 87, 69, 82, 32, 199, 138, 4, 69, 4, 44, 3, 76, 69, + 70, 1, 4, 82, 73, 71, 72, 2, 189, 1, 3, 84, 32, 68, 2, 165, 138, 4, 2, + 79, 80, 8, 60, 5, 76, 69, 70, 84, 32, 37, 6, 82, 73, 71, 72, 84, 32, 4, + 70, 68, 185, 153, 4, 2, 84, 79, 4, 34, 68, 245, 171, 4, 2, 84, 79, 2, + 129, 137, 4, 7, 73, 65, 71, 79, 78, 65, 76, 2, 29, 5, 69, 82, 84, 73, 67, + 2, 225, 171, 4, 2, 65, 76, 6, 32, 2, 65, 65, 195, 217, 5, 85, 4, 214, + 254, 6, 82, 247, 5, 68, 240, 1, 226, 1, 67, 174, 9, 68, 34, 70, 88, 3, + 82, 73, 83, 142, 1, 72, 66, 75, 106, 76, 166, 1, 77, 38, 78, 34, 79, 94, + 80, 34, 83, 190, 2, 84, 156, 1, 5, 69, 73, 71, 72, 84, 22, 85, 90, 86, + 230, 249, 4, 65, 218, 107, 87, 223, 97, 73, 90, 128, 1, 21, 74, 75, 32, + 85, 78, 73, 70, 73, 69, 68, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, + 198, 141, 6, 79, 243, 146, 1, 76, 86, 68, 2, 52, 69, 82, 53, 206, 2, 54, + 150, 2, 55, 166, 1, 56, 95, 57, 10, 50, 48, 234, 4, 65, 214, 242, 5, 56, + 223, 97, 50, 4, 182, 159, 7, 48, 3, 57, 30, 150, 1, 50, 42, 51, 38, 52, + 48, 2, 53, 66, 0, 2, 68, 69, 22, 57, 164, 4, 2, 56, 70, 222, 162, 3, 66, + 248, 205, 2, 2, 70, 56, 221, 97, 2, 49, 56, 6, 214, 4, 55, 142, 211, 6, + 49, 3, 52, 4, 210, 167, 3, 70, 143, 206, 2, 67, 4, 26, 48, 219, 167, 3, + 51, 2, 155, 157, 7, 56, 2, 135, 157, 7, 54, 4, 174, 167, 3, 50, 143, 243, + 3, 49, 24, 86, 50, 46, 51, 50, 53, 34, 54, 16, 2, 55, 48, 28, 2, 70, 49, + 129, 2, 2, 69, 56, 6, 70, 57, 170, 165, 3, 53, 167, 186, 2, 52, 4, 26, + 53, 131, 166, 3, 48, 2, 175, 155, 7, 53, 4, 202, 2, 66, 143, 163, 3, 57, + 2, 171, 2, 50, 4, 234, 154, 7, 56, 3, 57, 2, 207, 154, 7, 52, 12, 74, 53, + 38, 57, 12, 2, 49, 50, 20, 2, 68, 52, 245, 150, 7, 2, 65, 55, 4, 230, + 163, 3, 51, 231, 244, 2, 49, 2, 11, 56, 2, 191, 153, 7, 49, 2, 171, 153, + 7, 50, 6, 50, 57, 20, 2, 68, 55, 141, 163, 3, 2, 67, 65, 2, 219, 162, 3, + 69, 2, 207, 152, 7, 48, 4, 152, 210, 6, 2, 49, 52, 233, 67, 2, 48, 52, 2, + 11, 79, 2, 143, 216, 3, 84, 10, 84, 3, 65, 76, 76, 104, 4, 79, 85, 82, + 32, 240, 5, 3, 73, 86, 69, 135, 161, 5, 82, 2, 57, 12, 73, 78, 71, 32, + 68, 73, 65, 71, 79, 78, 65, 76, 2, 11, 32, 2, 11, 83, 2, 147, 174, 5, 76, + 4, 230, 211, 3, 68, 199, 194, 3, 75, 8, 212, 213, 3, 2, 73, 45, 214, 135, + 3, 68, 222, 56, 67, 3, 86, 8, 56, 8, 65, 84, 65, 75, 65, 78, 65, 32, 199, + 131, 7, 69, 6, 226, 225, 6, 75, 214, 28, 68, 159, 20, 83, 60, 36, 5, 65, + 84, 73, 78, 32, 79, 79, 54, 164, 5, 12, 83, 77, 65, 76, 76, 32, 76, 69, + 84, 84, 69, 82, 239, 123, 67, 6, 198, 210, 3, 71, 201, 170, 2, 3, 83, 83, + 76, 4, 178, 141, 5, 73, 139, 134, 2, 86, 4, 246, 148, 6, 69, 163, 126, + 71, 4, 140, 3, 15, 78, 69, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 87, + 69, 78, 235, 143, 7, 75, 4, 158, 2, 80, 131, 138, 5, 76, 20, 110, 69, + 146, 1, 72, 20, 2, 73, 88, 134, 251, 4, 65, 186, 66, 77, 222, 49, 81, + 246, 98, 79, 222, 61, 68, 3, 83, 4, 56, 7, 67, 79, 78, 68, 32, 83, 67, + 21, 3, 86, 69, 78, 2, 219, 188, 5, 82, 2, 29, 5, 32, 80, 79, 73, 78, 2, + 11, 84, 2, 203, 229, 5, 32, 2, 251, 143, 7, 86, 2, 17, 2, 84, 89, 2, 199, + 143, 7, 32, 8, 44, 4, 72, 82, 69, 69, 22, 87, 243, 4, 73, 2, 239, 200, 6, + 32, 4, 68, 13, 69, 78, 84, 89, 45, 84, 87, 79, 32, 80, 79, 73, 78, 19, + 79, 2, 223, 86, 84, 2, 139, 139, 7, 32, 4, 48, 6, 80, 32, 87, 73, 84, 72, + 191, 199, 6, 72, 2, 17, 2, 32, 69, 2, 247, 119, 88, 4, 150, 199, 6, 79, + 163, 70, 83, 2, 137, 237, 5, 6, 32, 66, 76, 65, 67, 75, 6, 234, 140, 7, + 50, 2, 51, 3, 65, 93, 134, 1, 65, 178, 6, 69, 236, 1, 10, 73, 67, 75, 32, + 70, 73, 71, 85, 82, 69, 182, 1, 79, 90, 82, 174, 4, 85, 238, 252, 6, 83, + 3, 88, 38, 92, 6, 70, 70, 32, 79, 70, 32, 106, 77, 82, 82, 206, 3, 84, + 210, 133, 3, 78, 243, 214, 2, 68, 4, 60, 8, 65, 69, 83, 67, 85, 76, 65, + 80, 21, 3, 72, 69, 82, 2, 163, 132, 5, 73, 2, 147, 246, 5, 77, 2, 21, 3, + 80, 69, 68, 2, 17, 2, 32, 69, 2, 229, 220, 1, 4, 78, 86, 69, 76, 24, 42, + 32, 205, 1, 5, 84, 32, 79, 70, 32, 12, 82, 69, 54, 79, 180, 235, 1, 8, + 65, 78, 68, 32, 67, 82, 69, 83, 139, 128, 2, 87, 2, 17, 2, 81, 85, 2, 11, + 65, 2, 171, 202, 6, 76, 4, 44, 5, 70, 32, 68, 65, 86, 251, 199, 3, 80, 2, + 163, 193, 6, 73, 12, 66, 71, 30, 83, 40, 4, 80, 82, 79, 84, 242, 77, 72, + 215, 101, 84, 2, 89, 4, 85, 65, 82, 68, 4, 26, 69, 191, 249, 1, 84, 2, + 11, 76, 2, 21, 3, 69, 67, 84, 2, 29, 5, 69, 68, 32, 65, 82, 2, 183, 131, + 7, 69, 4, 188, 152, 5, 10, 85, 69, 32, 79, 70, 32, 76, 73, 66, 69, 179, + 157, 1, 73, 8, 92, 2, 65, 77, 112, 9, 78, 79, 71, 82, 65, 80, 72, 73, 67, + 209, 214, 1, 4, 84, 72, 79, 83, 4, 54, 32, 197, 250, 5, 7, 73, 78, 71, + 32, 66, 79, 87, 2, 33, 6, 76, 79, 67, 79, 77, 79, 2, 143, 208, 1, 84, 2, + 217, 177, 4, 2, 32, 70, 11, 11, 32, 8, 60, 5, 87, 73, 84, 72, 32, 233, + 163, 2, 4, 76, 69, 65, 78, 4, 32, 4, 65, 82, 77, 83, 51, 68, 2, 25, 4, + 32, 82, 65, 73, 2, 203, 246, 1, 83, 2, 167, 235, 5, 82, 4, 44, 4, 67, 75, + 32, 67, 21, 3, 80, 87, 65, 2, 163, 250, 5, 72, 2, 131, 223, 6, 84, 20, + 66, 65, 108, 10, 69, 83, 83, 32, 79, 85, 84, 76, 73, 78, 79, 73, 6, 44, + 4, 73, 71, 72, 84, 45, 3, 87, 66, 69, 4, 184, 150, 3, 2, 32, 82, 143, + 211, 2, 78, 2, 243, 72, 82, 2, 17, 2, 69, 68, 2, 17, 2, 32, 87, 2, 169, + 233, 4, 4, 72, 73, 84, 69, 12, 76, 4, 78, 71, 32, 84, 52, 4, 80, 69, 68, + 32, 241, 8, 4, 67, 84, 76, 89, 2, 17, 2, 69, 82, 2, 181, 197, 6, 3, 77, + 73, 78, 8, 70, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 13, 2, 85, + 80, 2, 33, 3, 79, 87, 78, 2, 11, 84, 2, 11, 45, 2, 169, 216, 5, 8, 80, + 79, 73, 78, 84, 73, 78, 71, 6, 100, 8, 68, 73, 79, 32, 77, 73, 67, 82, + 20, 9, 70, 70, 69, 68, 32, 70, 76, 65, 84, 187, 249, 6, 80, 2, 167, 209, + 5, 79, 2, 11, 66, 2, 243, 166, 4, 82, 162, 3, 150, 1, 66, 212, 2, 5, 67, + 67, 69, 69, 68, 192, 3, 8, 77, 77, 65, 84, 73, 79, 78, 32, 70, 78, 204, + 22, 3, 80, 69, 82, 204, 11, 2, 82, 70, 47, 83, 63, 11, 83, 60, 76, 6, 67, + 82, 73, 80, 84, 32, 136, 1, 3, 69, 84, 32, 93, 3, 84, 73, 84, 30, 118, + 76, 186, 29, 69, 166, 1, 80, 22, 82, 218, 211, 2, 77, 246, 149, 2, 70, + 30, 83, 42, 84, 62, 90, 238, 85, 78, 15, 79, 2, 207, 30, 69, 28, 56, 6, + 65, 66, 79, 86, 69, 32, 246, 33, 79, 159, 3, 87, 6, 142, 32, 83, 143, + 184, 5, 82, 2, 251, 176, 5, 85, 22, 11, 83, 23, 11, 32, 20, 128, 1, 6, + 65, 66, 79, 86, 69, 32, 156, 1, 7, 66, 85, 84, 32, 78, 79, 84, 32, 2, 79, + 82, 141, 146, 3, 5, 85, 78, 68, 69, 82, 12, 100, 4, 78, 79, 84, 32, 28, + 12, 83, 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 32, 218, 32, 65, 39, 69, + 4, 242, 32, 65, 167, 1, 69, 4, 250, 32, 69, 83, 78, 2, 93, 5, 32, 69, 81, + 85, 73, 4, 17, 2, 32, 69, 4, 17, 2, 81, 85, 4, 22, 73, 167, 33, 65, 2, + 173, 33, 6, 86, 65, 76, 69, 78, 84, 6, 242, 136, 3, 66, 136, 228, 1, 4, + 87, 73, 84, 72, 147, 142, 1, 84, 249, 1, 194, 1, 32, 72, 7, 68, 65, 78, + 69, 83, 69, 32, 204, 12, 4, 82, 73, 83, 69, 72, 5, 85, 87, 65, 82, 32, + 132, 174, 2, 14, 83, 69, 84, 32, 79, 86, 69, 82, 32, 66, 85, 73, 76, 68, + 159, 157, 3, 70, 4, 50, 87, 225, 197, 5, 6, 66, 69, 72, 73, 78, 68, 2, + 171, 180, 6, 73, 146, 1, 198, 2, 65, 20, 17, 67, 79, 78, 83, 79, 78, 65, + 78, 84, 32, 83, 73, 71, 78, 32, 80, 65, 172, 1, 7, 76, 69, 84, 84, 69, + 82, 32, 188, 3, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 66, + 73, 78, 68, 85, 32, 228, 1, 5, 83, 73, 71, 78, 32, 204, 1, 13, 86, 79, + 87, 69, 76, 32, 83, 73, 71, 78, 32, 80, 65, 183, 245, 4, 68, 2, 171, 152, + 3, 86, 10, 68, 2, 77, 73, 40, 2, 78, 89, 33, 7, 83, 65, 78, 71, 65, 78, + 32, 2, 17, 2, 78, 71, 2, 207, 228, 5, 75, 4, 142, 5, 65, 211, 155, 1, 73, + 4, 162, 235, 6, 77, 3, 87, 76, 246, 1, 65, 58, 70, 78, 76, 2, 82, 34, 83, + 254, 150, 3, 69, 254, 216, 1, 78, 238, 178, 1, 66, 2, 75, 138, 69, 67, 2, + 68, 2, 71, 2, 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 84, 2, 86, 2, 87, 2, 88, + 2, 89, 2, 90, 186, 2, 73, 2, 79, 3, 85, 7, 180, 171, 6, 6, 82, 67, 72, + 65, 73, 67, 147, 64, 69, 6, 44, 5, 73, 78, 65, 76, 32, 227, 234, 6, 65, + 4, 222, 234, 6, 75, 3, 77, 4, 218, 197, 6, 69, 235, 36, 65, 4, 234, 231, + 6, 89, 187, 2, 65, 16, 90, 66, 2, 68, 2, 75, 12, 3, 76, 69, 85, 38, 67, + 34, 80, 253, 229, 6, 3, 83, 85, 82, 2, 11, 65, 2, 253, 103, 5, 32, 83, + 65, 84, 65, 2, 11, 65, 2, 171, 164, 6, 75, 4, 172, 3, 3, 65, 78, 71, 189, + 162, 6, 3, 85, 82, 78, 10, 32, 2, 80, 65, 235, 145, 3, 86, 8, 28, 3, 77, + 65, 65, 23, 78, 2, 203, 230, 6, 69, 6, 18, 71, 71, 89, 4, 38, 76, 225, + 160, 6, 3, 87, 73, 83, 2, 209, 197, 5, 2, 65, 89, 2, 201, 227, 6, 2, 69, + 67, 12, 18, 77, 23, 78, 2, 187, 134, 3, 69, 10, 96, 3, 69, 85, 76, 34, + 79, 22, 89, 188, 228, 1, 3, 71, 72, 85, 201, 189, 2, 4, 65, 69, 76, 65, + 2, 11, 69, 2, 195, 148, 6, 85, 2, 231, 161, 4, 76, 2, 247, 151, 1, 85, 5, + 165, 161, 3, 13, 32, 79, 86, 69, 82, 32, 77, 79, 85, 78, 84, 65, 73, 88, + 88, 7, 76, 69, 84, 84, 69, 82, 32, 184, 6, 6, 83, 73, 71, 78, 32, 80, + 135, 236, 4, 68, 66, 142, 2, 65, 66, 67, 66, 68, 44, 3, 72, 65, 77, 22, + 74, 34, 75, 42, 78, 38, 79, 2, 85, 34, 80, 38, 82, 20, 4, 83, 72, 89, 69, + 34, 84, 112, 3, 86, 65, 82, 230, 169, 4, 71, 148, 57, 4, 76, 79, 65, 67, + 136, 89, 2, 73, 77, 210, 13, 89, 150, 27, 66, 214, 89, 69, 203, 28, 77, + 6, 42, 80, 238, 215, 5, 65, 255, 134, 1, 86, 2, 251, 248, 1, 80, 4, 40, + 2, 72, 69, 253, 204, 6, 2, 65, 82, 2, 243, 207, 5, 76, 4, 22, 69, 155, + 95, 79, 2, 219, 204, 6, 86, 2, 171, 193, 6, 83, 2, 11, 89, 2, 239, 222, + 6, 65, 6, 194, 172, 6, 76, 146, 48, 73, 99, 72, 4, 138, 190, 5, 71, 171, + 160, 1, 65, 2, 11, 84, 2, 147, 144, 1, 84, 4, 198, 189, 5, 72, 175, 161, + 1, 73, 2, 243, 185, 6, 69, 4, 242, 199, 6, 76, 215, 22, 82, 8, 42, 69, + 22, 72, 209, 187, 6, 2, 65, 83, 2, 171, 142, 1, 78, 4, 26, 65, 167, 185, + 5, 69, 2, 243, 201, 6, 82, 2, 255, 218, 6, 67, 2, 175, 190, 6, 86, 72, + 54, 83, 182, 237, 4, 72, 209, 66, 4, 86, 73, 76, 76, 68, 56, 6, 67, 82, + 73, 80, 84, 32, 229, 2, 3, 69, 84, 32, 34, 114, 69, 34, 76, 134, 1, 80, + 22, 82, 218, 211, 2, 77, 246, 149, 2, 70, 30, 83, 42, 84, 62, 90, 238, + 85, 78, 15, 79, 4, 174, 69, 81, 183, 235, 5, 73, 6, 88, 18, 65, 84, 73, + 78, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 31, 69, 4, + 170, 218, 6, 73, 3, 78, 2, 55, 70, 2, 191, 171, 3, 76, 2, 21, 3, 73, 71, + 72, 2, 11, 84, 2, 139, 215, 2, 32, 34, 148, 1, 6, 65, 66, 79, 86, 69, 32, + 96, 7, 66, 69, 83, 73, 68, 69, 32, 162, 1, 79, 158, 3, 87, 221, 189, 4, + 9, 80, 82, 69, 67, 69, 68, 73, 78, 71, 6, 26, 83, 135, 188, 2, 76, 4, 11, + 85, 4, 26, 80, 151, 236, 4, 66, 2, 145, 236, 4, 2, 69, 82, 4, 108, 23, + 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 66, 89, 32, 68, 65, 83, 72, + 32, 87, 73, 84, 72, 23, 83, 2, 17, 2, 32, 83, 2, 241, 234, 4, 2, 85, 66, + 16, 11, 70, 17, 11, 32, 14, 120, 6, 65, 66, 79, 86, 69, 32, 132, 1, 4, + 87, 73, 84, 72, 249, 210, 5, 11, 79, 82, 32, 69, 81, 85, 65, 76, 32, 84, + 79, 8, 34, 65, 38, 69, 18, 84, 67, 78, 2, 11, 76, 2, 113, 3, 77, 79, 83, + 2, 203, 62, 81, 2, 21, 3, 73, 76, 68, 2, 171, 148, 3, 69, 2, 17, 2, 32, + 78, 2, 11, 79, 2, 11, 84, 2, 11, 32, 2, 11, 69, 2, 17, 2, 81, 85, 2, 11, + 65, 2, 11, 76, 2, 219, 228, 2, 32, 6, 25, 4, 73, 84, 72, 32, 6, 96, 2, + 80, 76, 24, 14, 77, 85, 76, 84, 73, 80, 76, 73, 67, 65, 84, 73, 79, 78, + 183, 129, 6, 68, 2, 11, 85, 2, 11, 83, 2, 165, 164, 4, 5, 32, 83, 73, 71, + 78, 4, 144, 202, 4, 2, 65, 67, 215, 206, 1, 69, 4, 60, 9, 80, 69, 78, 83, + 73, 79, 78, 32, 82, 131, 189, 6, 72, 2, 21, 3, 65, 73, 76, 2, 187, 196, + 5, 87, 8, 26, 65, 98, 73, 35, 85, 4, 44, 5, 83, 72, 32, 65, 77, 227, 207, + 6, 78, 2, 253, 139, 6, 7, 80, 69, 82, 83, 65, 78, 68, 2, 11, 77, 2, 183, + 145, 6, 77, 2, 233, 145, 3, 2, 78, 71, 236, 2, 92, 11, 76, 79, 84, 73, + 32, 78, 65, 71, 82, 73, 32, 226, 5, 77, 194, 17, 78, 97, 2, 82, 73, 90, + 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 168, 2, 11, 80, 79, 69, 84, 82, + 89, 32, 77, 65, 82, 75, 56, 5, 83, 73, 71, 78, 32, 145, 1, 11, 86, 79, + 87, 69, 76, 32, 83, 73, 71, 78, 32, 64, 174, 1, 68, 46, 82, 34, 84, 166, + 91, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 198, 208, 5, 72, 2, 76, 2, 77, + 2, 78, 2, 83, 246, 30, 65, 2, 69, 2, 73, 2, 79, 3, 85, 8, 238, 91, 68, + 198, 208, 5, 72, 247, 30, 79, 4, 134, 172, 6, 82, 247, 30, 79, 8, 162, + 91, 84, 198, 208, 5, 72, 247, 30, 79, 8, 11, 45, 8, 162, 202, 6, 49, 2, + 50, 2, 51, 3, 52, 8, 46, 65, 70, 72, 165, 132, 6, 3, 68, 86, 73, 4, 64, + 10, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 155, 132, 6, 78, 2, 193, 137, + 2, 2, 65, 83, 10, 242, 169, 6, 79, 246, 30, 65, 2, 69, 2, 73, 3, 85, 88, + 60, 8, 66, 79, 76, 32, 70, 79, 82, 32, 225, 16, 2, 77, 69, 86, 238, 2, + 66, 48, 2, 67, 65, 118, 68, 234, 3, 69, 130, 2, 70, 56, 8, 72, 79, 82, + 73, 90, 79, 78, 84, 0, 6, 86, 69, 82, 84, 73, 67, 44, 3, 76, 73, 78, 52, + 9, 77, 65, 82, 75, 83, 32, 67, 72, 65, 22, 78, 90, 65, 68, 3, 82, 69, 67, + 32, 5, 71, 82, 79, 85, 80, 0, 4, 85, 78, 73, 84, 22, 83, 205, 3, 15, 84, + 89, 80, 69, 32, 65, 32, 69, 76, 69, 67, 84, 82, 79, 78, 4, 190, 189, 4, + 69, 137, 201, 1, 3, 65, 67, 75, 4, 44, 3, 82, 82, 73, 189, 183, 1, 2, 78, + 67, 2, 33, 6, 65, 71, 69, 32, 82, 69, 2, 11, 84, 2, 215, 185, 1, 85, 20, + 24, 2, 65, 84, 55, 69, 2, 173, 5, 9, 65, 32, 76, 73, 78, 75, 32, 69, 83, + 18, 84, 4, 76, 69, 84, 69, 133, 2, 12, 86, 73, 67, 69, 32, 67, 79, 78, + 84, 82, 79, 76, 11, 11, 32, 8, 160, 1, 11, 82, 69, 67, 84, 65, 78, 71, + 85, 76, 65, 82, 0, 6, 83, 81, 85, 65, 82, 69, 154, 9, 70, 129, 185, 5, + 11, 77, 69, 68, 73, 85, 77, 32, 83, 72, 65, 68, 2, 61, 13, 32, 67, 72, + 69, 67, 75, 69, 82, 32, 66, 79, 65, 82, 2, 227, 193, 5, 68, 8, 11, 32, 8, + 162, 173, 1, 70, 162, 163, 3, 84, 183, 86, 79, 12, 22, 78, 207, 1, 83, + 10, 48, 5, 68, 32, 79, 70, 32, 137, 1, 2, 81, 85, 8, 18, 77, 31, 84, 2, + 245, 149, 5, 2, 69, 68, 6, 64, 11, 82, 65, 78, 83, 77, 73, 83, 83, 73, + 79, 78, 223, 107, 69, 5, 235, 207, 3, 32, 2, 155, 7, 73, 2, 209, 246, 4, + 2, 67, 65, 4, 36, 2, 73, 76, 73, 3, 79, 82, 77, 2, 191, 2, 69, 2, 229, + 160, 1, 6, 65, 76, 32, 84, 65, 66, 2, 11, 69, 2, 17, 2, 32, 70, 2, 163, + 177, 1, 69, 2, 227, 132, 5, 80, 6, 26, 69, 239, 180, 4, 85, 4, 56, 8, 71, + 65, 84, 73, 86, 69, 32, 65, 175, 162, 5, 87, 2, 21, 3, 67, 75, 78, 2, 21, + 3, 79, 87, 76, 2, 231, 135, 1, 69, 2, 11, 79, 2, 17, 2, 82, 68, 2, 163, + 141, 3, 32, 18, 180, 1, 7, 65, 77, 65, 82, 73, 84, 65, 60, 3, 72, 73, 70, + 52, 8, 84, 65, 82, 84, 32, 79, 70, 32, 64, 8, 85, 66, 83, 84, 73, 84, 85, + 84, 200, 1, 3, 89, 78, 67, 195, 249, 5, 80, 2, 25, 4, 78, 32, 83, 79, 2, + 11, 85, 2, 155, 252, 5, 82, 4, 17, 2, 84, 32, 4, 158, 191, 5, 79, 243, + 41, 73, 4, 22, 72, 215, 101, 84, 2, 17, 2, 69, 65, 2, 159, 231, 5, 68, 4, + 11, 69, 5, 11, 32, 2, 11, 70, 2, 21, 3, 79, 82, 77, 2, 17, 2, 32, 84, 2, + 203, 152, 6, 87, 2, 205, 249, 5, 2, 73, 67, 2, 11, 84, 2, 187, 165, 6, + 82, 7, 38, 67, 181, 171, 2, 3, 65, 71, 79, 2, 181, 133, 1, 9, 72, 82, 79, + 78, 79, 85, 83, 32, 73, 180, 1, 36, 3, 65, 67, 32, 199, 128, 5, 78, 178, + 1, 150, 3, 65, 52, 4, 66, 65, 82, 82, 32, 2, 67, 79, 80, 13, 68, 79, 84, + 84, 69, 68, 32, 90, 76, 65, 77, 65, 32, 118, 69, 90, 72, 204, 2, 7, 76, + 69, 84, 84, 69, 82, 32, 236, 9, 9, 79, 66, 76, 73, 81, 85, 69, 32, 76, + 28, 4, 80, 84, 72, 65, 0, 4, 90, 81, 65, 80, 110, 82, 88, 2, 83, 85, 174, + 2, 84, 140, 247, 4, 3, 77, 85, 83, 232, 22, 7, 70, 69, 77, 73, 78, 73, + 78, 253, 136, 1, 5, 81, 85, 83, 72, 83, 2, 11, 66, 2, 133, 201, 5, 5, 66, + 82, 69, 86, 73, 2, 11, 69, 2, 247, 176, 6, 75, 6, 38, 78, 165, 17, 4, 76, + 79, 78, 32, 2, 17, 2, 84, 82, 2, 219, 224, 5, 65, 4, 28, 3, 65, 78, 71, + 35, 72, 2, 11, 85, 2, 191, 143, 5, 76, 2, 11, 79, 2, 169, 220, 3, 5, 82, + 73, 90, 79, 78, 6, 60, 10, 78, 68, 32, 79, 70, 32, 80, 65, 82, 65, 151, + 14, 83, 2, 201, 11, 2, 71, 82, 14, 104, 8, 65, 82, 75, 76, 69, 65, 78, + 32, 132, 1, 4, 66, 65, 83, 65, 65, 7, 79, 82, 73, 90, 79, 78, 84, 6, 60, + 5, 65, 83, 84, 69, 82, 40, 4, 77, 69, 84, 79, 3, 79, 2, 17, 2, 73, 83, 2, + 155, 168, 4, 67, 2, 201, 167, 2, 2, 66, 69, 6, 164, 11, 8, 45, 69, 83, + 65, 83, 65, 32, 68, 151, 132, 4, 32, 2, 137, 221, 5, 2, 65, 76, 92, 154, + 2, 68, 102, 72, 32, 3, 76, 65, 77, 34, 77, 204, 1, 2, 80, 69, 142, 1, 82, + 78, 83, 144, 1, 8, 70, 73, 78, 65, 76, 32, 83, 69, 106, 65, 14, 75, 2, + 81, 34, 84, 40, 5, 71, 65, 77, 65, 76, 40, 4, 89, 85, 68, 72, 214, 74, + 66, 142, 172, 4, 90, 166, 44, 78, 134, 2, 87, 175, 126, 69, 4, 76, 14, + 79, 84, 76, 69, 83, 83, 32, 68, 65, 76, 65, 84, 72, 32, 139, 3, 65, 2, + 167, 201, 2, 82, 4, 11, 69, 5, 239, 168, 6, 84, 2, 11, 65, 2, 207, 168, + 6, 68, 24, 60, 9, 65, 76, 65, 89, 65, 76, 65, 77, 32, 183, 152, 6, 73, + 22, 78, 76, 22, 78, 234, 173, 4, 66, 246, 213, 1, 84, 138, 34, 83, 14, + 74, 3, 82, 4, 135, 159, 4, 76, 8, 246, 83, 78, 250, 209, 5, 71, 3, 89, 9, + 33, 6, 82, 83, 73, 65, 78, 32, 6, 50, 66, 16, 3, 68, 72, 65, 17, 3, 71, + 72, 65, 2, 199, 78, 72, 2, 147, 2, 76, 2, 199, 157, 5, 77, 4, 52, 7, 69, + 86, 69, 82, 83, 69, 68, 159, 190, 4, 73, 2, 255, 222, 4, 32, 14, 142, 1, + 69, 40, 7, 79, 71, 68, 73, 65, 78, 32, 64, 12, 85, 80, 69, 82, 83, 67, + 82, 73, 80, 84, 32, 65, 222, 164, 5, 72, 201, 82, 2, 65, 68, 2, 17, 2, + 77, 75, 2, 243, 162, 4, 65, 6, 42, 90, 32, 2, 75, 72, 211, 141, 6, 70, 2, + 239, 247, 4, 72, 2, 11, 76, 2, 11, 65, 2, 135, 163, 6, 80, 6, 36, 3, 69, + 84, 72, 191, 165, 5, 65, 5, 173, 85, 6, 32, 71, 65, 82, 83, 72, 5, 167, + 246, 5, 32, 4, 217, 132, 4, 2, 73, 78, 6, 21, 3, 72, 65, 32, 6, 42, 68, + 178, 132, 4, 66, 167, 161, 1, 65, 2, 17, 2, 79, 84, 2, 223, 150, 1, 84, + 8, 58, 66, 162, 203, 2, 87, 249, 166, 1, 4, 85, 75, 75, 65, 4, 157, 240, + 3, 2, 65, 83, 14, 88, 8, 66, 76, 73, 78, 69, 65, 82, 32, 113, 10, 80, 82, + 65, 76, 73, 78, 69, 65, 82, 32, 8, 44, 5, 67, 79, 76, 79, 78, 211, 206, + 3, 70, 7, 11, 32, 4, 29, 5, 83, 75, 69, 87, 69, 4, 251, 244, 5, 68, 6, + 44, 5, 67, 79, 76, 79, 78, 227, 205, 3, 70, 5, 253, 236, 5, 7, 32, 83, + 75, 69, 87, 69, 68, 8, 62, 72, 25, 11, 87, 79, 32, 86, 69, 82, 84, 73, + 67, 65, 76, 4, 21, 3, 82, 69, 69, 4, 25, 4, 32, 68, 79, 84, 4, 231, 255, + 3, 83, 190, 43, 102, 45, 58, 65, 206, 105, 69, 154, 45, 72, 182, 46, 73, + 230, 69, 79, 170, 33, 82, 214, 16, 85, 147, 22, 87, 4, 32, 2, 83, 72, + 255, 246, 4, 82, 2, 235, 149, 5, 73, 244, 26, 182, 1, 66, 82, 71, 196, + 17, 2, 73, 32, 170, 29, 75, 144, 5, 9, 76, 76, 89, 32, 77, 65, 82, 75, + 32, 34, 77, 178, 37, 78, 132, 13, 3, 80, 69, 32, 94, 85, 190, 148, 5, 67, + 159, 11, 88, 5, 221, 122, 16, 76, 69, 32, 84, 69, 78, 78, 73, 83, 32, 80, + 65, 68, 68, 76, 69, 144, 2, 78, 32, 240, 11, 5, 65, 76, 79, 71, 32, 241, + 2, 6, 66, 65, 78, 87, 65, 32, 190, 1, 162, 1, 65, 102, 67, 186, 1, 68, + 58, 69, 98, 71, 118, 72, 46, 76, 230, 3, 80, 58, 81, 62, 82, 106, 83, + 142, 76, 78, 230, 222, 1, 84, 250, 145, 1, 70, 231, 183, 1, 86, 6, 42, + 80, 150, 201, 2, 77, 219, 187, 1, 83, 2, 17, 2, 79, 83, 2, 145, 235, 5, + 4, 84, 82, 79, 80, 8, 18, 73, 63, 79, 2, 25, 4, 82, 67, 85, 77, 2, 213, + 2, 4, 70, 76, 69, 88, 6, 26, 77, 167, 199, 5, 76, 4, 11, 77, 4, 152, 131, + 5, 7, 69, 82, 67, 73, 65, 76, 32, 235, 147, 1, 65, 22, 26, 79, 151, 165, + 4, 73, 2, 11, 76, 2, 235, 84, 76, 4, 18, 81, 43, 88, 2, 17, 2, 85, 65, 2, + 171, 231, 2, 76, 2, 153, 172, 5, 4, 67, 76, 65, 77, 4, 11, 82, 4, 18, 65, + 55, 69, 2, 11, 86, 2, 11, 69, 2, 177, 120, 3, 32, 65, 67, 2, 145, 4, 4, + 65, 84, 69, 82, 2, 145, 142, 2, 6, 89, 80, 72, 69, 78, 45, 114, 38, 65, + 254, 2, 69, 247, 168, 4, 79, 104, 25, 4, 84, 73, 78, 32, 104, 34, 67, 41, + 4, 83, 77, 65, 76, 52, 11, 65, 52, 25, 4, 80, 73, 84, 65, 52, 41, 8, 76, + 32, 76, 69, 84, 84, 69, 82, 52, 11, 32, 52, 194, 146, 6, 65, 2, 66, 2, + 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, + 2, 78, 2, 79, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, + 88, 2, 89, 3, 90, 8, 22, 83, 199, 1, 70, 2, 11, 83, 2, 197, 63, 3, 45, + 84, 72, 4, 26, 69, 203, 225, 2, 76, 2, 11, 82, 2, 227, 67, 67, 4, 11, 85, + 4, 26, 79, 203, 190, 3, 69, 2, 147, 166, 5, 84, 8, 36, 3, 73, 71, 72, + 199, 249, 3, 69, 6, 17, 2, 84, 32, 6, 242, 136, 2, 67, 210, 3, 80, 239, + 7, 83, 6, 142, 195, 2, 69, 146, 184, 1, 79, 167, 214, 1, 80, 46, 80, 7, + 76, 69, 84, 84, 69, 82, 32, 208, 1, 5, 83, 73, 71, 78, 32, 147, 2, 86, + 38, 162, 1, 65, 242, 154, 2, 78, 250, 238, 3, 66, 2, 68, 2, 71, 2, 72, 2, + 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 73, + 3, 85, 5, 197, 193, 5, 6, 82, 67, 72, 65, 73, 67, 4, 26, 80, 199, 181, 2, + 86, 2, 25, 4, 65, 77, 85, 68, 2, 227, 212, 2, 80, 36, 48, 7, 76, 69, 84, + 84, 69, 82, 32, 147, 1, 86, 32, 194, 153, 2, 78, 250, 238, 3, 66, 2, 68, + 2, 71, 2, 75, 2, 76, 2, 77, 2, 80, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, + 65, 2, 73, 3, 85, 4, 41, 8, 79, 87, 69, 76, 32, 83, 73, 71, 4, 11, 78, 4, + 243, 200, 5, 32, 212, 3, 112, 10, 76, 69, 32, 76, 69, 84, 84, 69, 82, 32, + 248, 2, 5, 84, 72, 65, 77, 32, 189, 18, 5, 86, 73, 69, 84, 32, 70, 186, + 1, 65, 34, 69, 30, 84, 250, 148, 2, 78, 210, 171, 1, 79, 162, 254, 1, 75, + 2, 80, 162, 7, 85, 234, 61, 70, 2, 72, 2, 76, 2, 77, 2, 81, 2, 83, 2, 86, + 2, 88, 2, 89, 187, 2, 73, 7, 142, 240, 5, 85, 215, 22, 73, 7, 194, 134, + 6, 69, 3, 72, 18, 60, 3, 79, 78, 69, 170, 190, 5, 83, 138, 69, 72, 187, + 2, 65, 10, 11, 45, 10, 218, 133, 6, 50, 2, 51, 2, 52, 2, 53, 3, 54, 254, + 1, 196, 1, 2, 67, 79, 240, 3, 4, 72, 79, 82, 65, 0, 4, 84, 72, 65, 77, + 28, 7, 76, 69, 84, 84, 69, 82, 32, 220, 4, 5, 83, 73, 71, 78, 32, 213, 5, + 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 20, 164, 1, 13, 78, 83, + 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 161, 241, 4, 21, 77, 66, 73, + 78, 73, 78, 71, 32, 67, 82, 89, 80, 84, 79, 71, 82, 65, 77, 77, 73, 67, + 18, 148, 1, 6, 70, 73, 78, 65, 76, 32, 22, 76, 56, 16, 72, 73, 71, 72, + 32, 82, 65, 84, 72, 65, 32, 79, 82, 32, 76, 79, 22, 77, 134, 254, 5, 66, + 3, 83, 2, 215, 190, 5, 78, 4, 54, 79, 213, 149, 1, 7, 65, 32, 84, 65, 78, + 71, 32, 2, 187, 214, 1, 87, 6, 48, 6, 69, 68, 73, 65, 76, 32, 139, 128, + 6, 65, 4, 206, 253, 5, 76, 3, 82, 20, 201, 142, 4, 2, 32, 68, 106, 188, + 1, 2, 71, 82, 44, 5, 72, 73, 71, 72, 32, 94, 76, 222, 1, 82, 134, 168, 2, + 85, 206, 201, 1, 73, 178, 15, 78, 186, 219, 1, 79, 162, 8, 69, 158, 20, + 66, 2, 68, 2, 77, 2, 87, 187, 2, 65, 2, 21, 3, 69, 65, 84, 2, 179, 251, + 5, 32, 32, 242, 1, 75, 42, 82, 186, 179, 5, 83, 82, 67, 2, 80, 2, 84, + 138, 69, 70, 2, 72, 3, 89, 36, 60, 3, 79, 87, 32, 170, 188, 5, 65, 206, + 41, 85, 159, 20, 76, 28, 86, 75, 42, 82, 138, 180, 5, 67, 2, 80, 2, 84, + 138, 69, 70, 2, 72, 2, 83, 3, 89, 6, 182, 249, 5, 72, 2, 88, 187, 2, 65, + 2, 133, 129, 4, 2, 65, 84, 8, 26, 65, 191, 228, 5, 85, 7, 214, 248, 5, + 78, 3, 84, 50, 178, 1, 72, 34, 75, 176, 1, 4, 77, 65, 73, 32, 82, 82, + 136, 1, 2, 83, 65, 92, 5, 87, 73, 65, 78, 71, 188, 126, 3, 68, 79, 75, + 224, 117, 3, 84, 79, 78, 205, 168, 3, 2, 67, 65, 4, 198, 168, 5, 65, 179, + 63, 79, 14, 52, 4, 72, 85, 69, 78, 134, 3, 65, 195, 219, 4, 69, 8, 80, 6, + 32, 84, 79, 78, 69, 45, 149, 174, 4, 8, 45, 76, 85, 69, 32, 75, 65, 82, + 6, 146, 248, 5, 51, 2, 52, 3, 53, 8, 56, 4, 75, 65, 78, 71, 134, 135, 1, + 89, 183, 181, 3, 83, 5, 183, 140, 1, 32, 4, 92, 3, 65, 32, 72, 21, 16, + 69, 86, 69, 82, 83, 69, 68, 32, 82, 79, 84, 65, 84, 69, 68, 32, 2, 195, + 187, 4, 65, 2, 155, 198, 3, 82, 8, 48, 3, 84, 75, 65, 230, 171, 4, 87, + 203, 121, 75, 4, 17, 2, 65, 78, 5, 199, 131, 4, 75, 5, 245, 191, 3, 2, + 87, 65, 38, 90, 65, 36, 4, 77, 65, 73, 32, 22, 79, 46, 84, 86, 85, 178, + 233, 3, 73, 223, 137, 2, 69, 9, 194, 244, 5, 65, 2, 69, 3, 73, 2, 183, + 224, 4, 83, 11, 210, 213, 3, 65, 186, 158, 2, 79, 3, 89, 4, 42, 65, 197, + 136, 1, 4, 72, 65, 77, 32, 2, 17, 2, 76, 76, 2, 211, 128, 4, 32, 9, 234, + 178, 5, 85, 163, 64, 69, 144, 1, 184, 1, 7, 76, 69, 84, 84, 69, 82, 32, + 188, 2, 5, 77, 65, 73, 32, 75, 32, 7, 83, 89, 77, 66, 79, 76, 32, 116, 9, + 84, 79, 78, 69, 32, 77, 65, 73, 32, 81, 6, 86, 79, 87, 69, 76, 32, 96, + 44, 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 48, 11, 32, 48, 154, 1, 75, 30, + 67, 2, 80, 2, 84, 34, 78, 166, 208, 5, 66, 2, 68, 2, 70, 2, 71, 2, 72, 2, + 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, 89, 247, 30, 79, 6, 26, 72, 187, 239, + 5, 79, 4, 194, 208, 5, 72, 247, 30, 79, 6, 162, 208, 5, 71, 2, 89, 247, + 30, 79, 4, 246, 157, 5, 65, 195, 52, 72, 10, 68, 2, 75, 79, 106, 78, 240, + 126, 4, 72, 79, 32, 72, 183, 179, 3, 83, 4, 212, 127, 3, 73, 32, 75, 179, + 238, 4, 78, 8, 58, 78, 254, 132, 1, 84, 230, 164, 2, 83, 171, 192, 2, 69, + 2, 143, 170, 3, 85, 26, 50, 65, 58, 85, 30, 73, 134, 236, 5, 69, 3, 79, + 10, 130, 214, 5, 85, 214, 22, 65, 2, 77, 2, 78, 3, 89, 9, 26, 69, 135, + 236, 5, 65, 5, 131, 236, 5, 65, 138, 1, 52, 3, 82, 73, 32, 237, 222, 4, + 4, 69, 79, 85, 84, 136, 1, 82, 65, 20, 7, 76, 69, 84, 84, 69, 82, 32, + 166, 2, 83, 78, 86, 187, 246, 3, 68, 2, 195, 211, 1, 66, 88, 210, 1, 65, + 250, 142, 2, 68, 82, 82, 34, 84, 230, 5, 85, 206, 201, 1, 73, 162, 193, + 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, 83, 138, 69, 72, 2, + 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 11, 172, 185, 3, 7, 82, 67, + 72, 65, 73, 67, 32, 198, 175, 2, 65, 2, 73, 3, 85, 8, 25, 4, 73, 71, 78, + 32, 8, 166, 212, 1, 78, 234, 206, 3, 65, 239, 1, 86, 18, 49, 10, 79, 87, + 69, 76, 32, 83, 73, 71, 78, 32, 18, 214, 147, 2, 65, 38, 85, 206, 201, 1, + 73, 222, 137, 2, 69, 3, 79, 4, 154, 51, 70, 155, 154, 4, 79, 188, 6, 36, + 3, 73, 76, 32, 239, 193, 4, 65, 186, 6, 154, 3, 65, 106, 67, 194, 2, 68, + 72, 3, 87, 69, 84, 48, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 232, 7, 9, + 73, 78, 32, 80, 79, 83, 83, 69, 83, 22, 76, 200, 2, 7, 78, 85, 77, 66, + 69, 82, 32, 152, 1, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, + 69, 78, 68, 32, 79, 70, 54, 82, 42, 83, 166, 14, 84, 228, 1, 7, 86, 79, + 87, 69, 76, 32, 83, 100, 2, 89, 69, 176, 186, 4, 5, 77, 79, 78, 84, 72, + 171, 118, 79, 6, 80, 5, 83, 32, 65, 66, 79, 242, 132, 2, 85, 217, 244, 1, + 5, 78, 68, 32, 79, 68, 2, 243, 197, 3, 86, 52, 80, 9, 79, 78, 83, 79, 78, + 65, 78, 84, 32, 156, 21, 3, 85, 82, 82, 171, 9, 82, 48, 130, 1, 75, 22, + 76, 22, 78, 46, 84, 218, 191, 1, 83, 194, 77, 82, 202, 210, 3, 67, 2, 72, + 2, 74, 2, 77, 2, 80, 2, 86, 3, 89, 5, 215, 162, 5, 83, 7, 243, 227, 4, + 76, 11, 134, 229, 3, 78, 134, 251, 1, 71, 3, 89, 5, 223, 223, 5, 84, 26, + 68, 2, 82, 89, 164, 28, 2, 69, 66, 238, 143, 2, 65, 227, 193, 1, 73, 2, + 137, 200, 1, 7, 32, 67, 85, 76, 84, 73, 86, 42, 168, 1, 4, 79, 78, 69, + 32, 220, 4, 6, 84, 72, 82, 69, 69, 32, 177, 215, 5, 22, 68, 79, 87, 78, + 83, 67, 65, 76, 73, 78, 71, 32, 70, 65, 67, 84, 79, 82, 32, 75, 73, 73, + 30, 112, 5, 69, 73, 71, 72, 84, 34, 70, 40, 3, 72, 65, 76, 22, 79, 88, 4, + 83, 73, 88, 84, 74, 84, 187, 215, 3, 81, 4, 210, 3, 73, 227, 216, 5, 72, + 4, 156, 3, 2, 79, 82, 187, 213, 3, 73, 4, 215, 218, 1, 70, 2, 225, 2, 18, + 78, 69, 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 83, 73, 88, + 6, 232, 217, 1, 5, 69, 69, 78, 84, 72, 177, 24, 5, 89, 45, 70, 79, 85, 8, + 38, 72, 138, 1, 87, 239, 214, 3, 69, 4, 132, 1, 18, 82, 69, 69, 45, 72, + 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 84, 87, 201, 156, 4, 8, 73, + 82, 84, 89, 45, 83, 69, 67, 2, 17, 2, 69, 78, 2, 17, 2, 84, 73, 2, 207, + 214, 3, 69, 10, 58, 69, 28, 4, 83, 73, 88, 84, 82, 84, 163, 214, 3, 81, + 2, 129, 1, 3, 73, 71, 72, 4, 50, 69, 149, 215, 3, 6, 89, 45, 70, 79, 85, + 82, 2, 145, 215, 3, 2, 69, 78, 2, 21, 3, 87, 69, 78, 2, 221, 214, 3, 3, + 84, 73, 69, 2, 171, 214, 1, 83, 72, 48, 6, 69, 84, 84, 69, 82, 32, 195, + 237, 3, 65, 70, 194, 1, 78, 158, 201, 1, 84, 166, 49, 65, 82, 76, 38, 82, + 134, 6, 85, 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 242, 7, 69, + 234, 61, 67, 2, 72, 2, 74, 2, 75, 2, 77, 2, 80, 2, 86, 3, 89, 10, 46, 78, + 250, 209, 5, 71, 2, 89, 187, 2, 65, 4, 246, 209, 5, 78, 187, 2, 65, 8, + 38, 79, 130, 128, 4, 84, 131, 77, 83, 4, 11, 78, 4, 17, 2, 69, 32, 4, 18, + 72, 31, 84, 2, 217, 71, 3, 85, 78, 68, 2, 149, 132, 2, 3, 72, 79, 85, 2, + 17, 2, 32, 84, 2, 11, 69, 2, 171, 182, 5, 88, 2, 17, 2, 85, 80, 2, 131, + 182, 3, 69, 194, 4, 152, 1, 5, 65, 76, 84, 32, 80, 20, 4, 73, 71, 78, 32, + 206, 4, 80, 28, 9, 84, 65, 82, 84, 73, 78, 71, 32, 70, 41, 8, 89, 76, 76, + 65, 66, 76, 69, 32, 2, 219, 215, 3, 65, 40, 90, 65, 48, 3, 67, 69, 86, + 34, 75, 80, 2, 77, 85, 102, 85, 14, 80, 118, 86, 167, 64, 78, 4, 216, 2, + 4, 65, 90, 72, 65, 199, 136, 5, 78, 2, 11, 73, 2, 231, 170, 5, 84, 6, 38, + 85, 173, 170, 5, 3, 65, 65, 67, 4, 18, 90, 87, 82, 2, 167, 187, 5, 72, 6, + 60, 4, 75, 75, 85, 82, 16, 2, 84, 72, 21, 3, 85, 86, 85, 2, 167, 82, 85, + 2, 251, 149, 3, 65, 2, 75, 90, 8, 26, 65, 131, 254, 4, 79, 6, 34, 84, + 182, 146, 4, 65, 15, 78, 2, 11, 72, 2, 17, 2, 65, 75, 2, 179, 168, 5, 75, + 10, 38, 65, 214, 81, 69, 143, 184, 4, 73, 4, 180, 102, 3, 82, 65, 65, + 221, 243, 2, 6, 75, 65, 73, 89, 65, 82, 2, 11, 69, 2, 187, 9, 78, 2, 17, + 2, 82, 79, 2, 235, 196, 4, 77, 148, 4, 130, 1, 75, 166, 1, 76, 166, 1, + 78, 186, 1, 82, 86, 83, 178, 1, 84, 214, 2, 67, 2, 72, 2, 74, 2, 77, 2, + 80, 2, 86, 3, 89, 46, 84, 2, 83, 83, 174, 246, 1, 65, 38, 85, 202, 141, + 1, 79, 134, 60, 73, 191, 201, 1, 69, 24, 214, 239, 1, 65, 250, 6, 85, + 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 66, 78, 76, 146, 245, 1, + 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 44, 226, 6, + 76, 174, 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, + 69, 110, 98, 78, 174, 5, 71, 2, 89, 174, 238, 1, 65, 38, 85, 202, 141, 1, + 79, 134, 60, 73, 191, 201, 1, 69, 44, 170, 5, 78, 174, 238, 1, 65, 38, + 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 44, 214, 4, 82, 174, + 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 68, + 90, 72, 170, 3, 83, 174, 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, + 73, 191, 201, 1, 69, 24, 210, 241, 1, 65, 38, 85, 202, 141, 1, 79, 230, + 2, 82, 162, 57, 73, 191, 201, 1, 69, 44, 210, 2, 84, 174, 238, 1, 65, 38, + 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 6, 68, 2, 79, 84, 33, + 11, 82, 65, 68, 73, 84, 73, 79, 78, 65, 76, 32, 2, 11, 65, 2, 163, 188, + 4, 76, 4, 24, 2, 67, 82, 55, 78, 2, 17, 2, 69, 68, 2, 11, 73, 2, 211, + 187, 4, 84, 2, 11, 85, 2, 17, 2, 77, 66, 2, 123, 69, 22, 25, 4, 73, 71, + 78, 32, 22, 170, 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, + 201, 1, 69, 2, 11, 65, 2, 163, 186, 4, 82, 184, 13, 36, 5, 65, 66, 65, + 84, 65, 35, 71, 2, 11, 32, 2, 147, 209, 3, 84, 182, 13, 54, 69, 20, 3, + 83, 65, 32, 217, 6, 3, 85, 84, 32, 2, 207, 166, 4, 82, 178, 1, 52, 7, 76, + 69, 84, 84, 69, 82, 32, 155, 206, 3, 68, 158, 1, 210, 1, 65, 58, 70, 54, + 72, 38, 76, 58, 77, 54, 78, 50, 83, 126, 85, 114, 69, 2, 73, 2, 79, 2, + 86, 186, 241, 4, 84, 82, 67, 2, 68, 2, 71, 2, 75, 2, 80, 138, 69, 66, 2, + 82, 2, 87, 2, 88, 2, 89, 3, 90, 16, 150, 4, 87, 202, 185, 5, 67, 2, 81, + 2, 88, 3, 90, 4, 172, 236, 4, 5, 73, 78, 65, 76, 32, 251, 80, 65, 6, 174, + 177, 1, 84, 199, 139, 4, 65, 4, 204, 150, 4, 5, 79, 78, 71, 32, 85, 131, + 166, 1, 65, 10, 150, 188, 5, 65, 2, 67, 2, 81, 2, 88, 3, 90, 8, 170, 185, + 5, 71, 2, 72, 2, 89, 187, 2, 65, 8, 26, 72, 155, 187, 5, 65, 6, 40, 4, + 79, 82, 84, 32, 239, 186, 5, 65, 4, 188, 151, 1, 2, 85, 69, 137, 151, 3, + 2, 65, 87, 32, 58, 73, 54, 69, 202, 185, 5, 67, 2, 81, 2, 88, 3, 90, 16, + 50, 85, 202, 185, 5, 67, 2, 81, 2, 88, 3, 90, 8, 198, 185, 5, 67, 2, 81, 2, 88, 3, 90, 130, 12, 64, 10, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, - 143, 241, 2, 73, 128, 12, 70, 48, 178, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 143, 150, 3, 73, 128, 12, 70, 48, 178, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 95, 55, 198, 1, 86, 48, 242, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 3, 57, 18, 130, 140, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 54, 2, 55, 2, 56, 3, 57, 18, 170, 183, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 200, 1, 150, 1, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 138, 1, 58, 48, 2, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 95, 54, 20, 146, 138, 5, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 182, 137, 5, 48, 2, 49, 2, 50, + 51, 2, 52, 2, 53, 95, 54, 20, 186, 181, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 222, 180, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 4, 48, 6, 67, 65, 82, 84, 82, - 73, 21, 2, 68, 82, 2, 187, 212, 3, 68, 2, 215, 139, 4, 73, 2, 155, 133, - 3, 82, 142, 3, 154, 1, 65, 152, 2, 5, 68, 68, 89, 32, 66, 22, 76, 234, + 73, 21, 2, 68, 82, 2, 183, 254, 3, 68, 2, 239, 182, 4, 73, 2, 179, 173, + 3, 82, 142, 3, 154, 1, 65, 152, 2, 5, 68, 68, 89, 32, 66, 22, 76, 166, 14, 78, 144, 1, 5, 83, 84, 32, 84, 85, 21, 12, 84, 82, 65, 71, 82, 65, 77, 32, 70, 79, 82, 32, 10, 68, 9, 67, 85, 80, 32, 87, 73, 84, 72, 79, - 58, 82, 171, 181, 4, 80, 2, 33, 6, 85, 84, 32, 72, 65, 78, 2, 195, 228, - 3, 68, 6, 72, 9, 45, 79, 70, 70, 32, 67, 65, 76, 69, 29, 5, 68, 82, 79, - 80, 45, 2, 221, 226, 3, 2, 78, 68, 4, 234, 243, 2, 83, 245, 112, 4, 66, - 65, 82, 66, 2, 147, 226, 3, 69, 216, 1, 38, 69, 225, 2, 4, 85, 71, 85, - 32, 16, 52, 6, 80, 72, 79, 78, 69, 32, 246, 1, 83, 35, 86, 12, 132, 1, 3, - 82, 69, 67, 252, 151, 1, 3, 76, 79, 67, 242, 227, 2, 83, 165, 116, 13, - 79, 78, 32, 84, 79, 80, 32, 79, 70, 32, 77, 79, 68, 6, 36, 5, 69, 73, 86, - 69, 82, 51, 79, 5, 29, 5, 32, 87, 73, 84, 72, 2, 159, 39, 32, 2, 243, - 221, 2, 82, 2, 11, 67, 2, 211, 188, 3, 79, 2, 247, 231, 2, 73, 200, 1, - 162, 1, 65, 40, 15, 70, 82, 65, 67, 84, 73, 79, 78, 32, 68, 73, 71, 73, - 84, 32, 164, 2, 2, 76, 69, 156, 4, 5, 83, 73, 71, 78, 32, 198, 2, 86, - 175, 136, 3, 68, 2, 11, 73, 2, 245, 189, 4, 2, 32, 76, 14, 74, 84, 40, 2, - 79, 78, 81, 10, 90, 69, 82, 79, 32, 70, 79, 82, 32, 79, 8, 36, 3, 72, 82, - 69, 13, 2, 87, 79, 4, 11, 69, 4, 29, 5, 32, 70, 79, 82, 32, 4, 34, 79, - 21, 4, 69, 86, 69, 78, 2, 17, 2, 68, 68, 2, 49, 10, 32, 80, 79, 87, 69, - 82, 83, 32, 79, 70, 2, 201, 22, 2, 32, 70, 114, 44, 5, 84, 84, 69, 82, - 32, 159, 187, 4, 78, 112, 214, 1, 68, 54, 76, 38, 78, 106, 82, 34, 84, - 242, 144, 1, 65, 126, 86, 186, 24, 85, 158, 144, 1, 79, 182, 56, 73, 202, - 190, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 222, - 61, 72, 2, 77, 3, 89, 10, 234, 179, 4, 68, 254, 68, 72, 2, 90, 187, 2, - 65, 6, 198, 244, 2, 76, 167, 134, 2, 65, 10, 42, 65, 230, 247, 4, 71, 2, - 78, 3, 89, 5, 41, 8, 75, 65, 65, 82, 65, 32, 80, 79, 2, 163, 37, 76, 6, - 194, 89, 82, 159, 160, 4, 65, 10, 138, 178, 4, 84, 254, 68, 72, 2, 83, - 187, 2, 65, 20, 86, 67, 194, 1, 83, 248, 35, 3, 84, 85, 85, 142, 108, 65, - 170, 1, 78, 203, 162, 3, 86, 6, 60, 9, 79, 77, 66, 73, 78, 73, 78, 71, - 32, 219, 145, 1, 65, 4, 70, 65, 185, 250, 3, 11, 67, 65, 78, 68, 82, 65, - 66, 73, 78, 68, 85, 2, 33, 6, 78, 85, 83, 86, 65, 82, 2, 147, 250, 3, 65, - 2, 11, 73, 2, 169, 189, 3, 3, 68, 68, 72, 30, 49, 10, 79, 87, 69, 76, 32, - 83, 73, 71, 78, 32, 30, 186, 145, 1, 65, 106, 86, 214, 20, 85, 158, 144, - 1, 79, 182, 56, 73, 187, 198, 1, 69, 6, 80, 10, 78, 73, 83, 32, 82, 65, - 67, 81, 85, 69, 150, 221, 2, 71, 219, 151, 2, 84, 2, 11, 84, 2, 25, 4, - 32, 65, 78, 68, 2, 251, 239, 2, 32, 2, 219, 221, 4, 66, 162, 1, 210, 2, - 65, 134, 1, 66, 174, 1, 67, 130, 3, 68, 198, 3, 69, 154, 2, 70, 182, 1, - 71, 198, 1, 72, 64, 8, 89, 79, 85, 84, 72, 70, 85, 76, 52, 2, 73, 78, 46, - 75, 98, 76, 146, 1, 77, 134, 1, 79, 62, 80, 142, 1, 82, 210, 1, 83, 224, - 1, 14, 86, 65, 83, 84, 78, 69, 83, 83, 32, 79, 82, 32, 87, 65, 12, 2, 87, - 65, 202, 94, 85, 143, 234, 3, 74, 8, 88, 4, 67, 67, 85, 77, 22, 83, 132, - 21, 2, 68, 86, 157, 175, 1, 5, 71, 71, 82, 65, 86, 2, 183, 176, 1, 85, 2, - 183, 173, 4, 67, 6, 92, 7, 79, 76, 68, 32, 82, 69, 83, 32, 5, 82, 65, 78, - 67, 72, 221, 230, 3, 3, 65, 82, 82, 2, 153, 159, 4, 3, 79, 76, 85, 2, 11, - 73, 2, 133, 157, 1, 3, 78, 71, 32, 22, 54, 72, 20, 3, 76, 79, 83, 102, - 79, 211, 248, 1, 69, 2, 219, 186, 3, 65, 6, 26, 69, 143, 206, 3, 85, 4, - 26, 68, 207, 215, 3, 78, 2, 21, 3, 32, 77, 79, 2, 243, 238, 2, 85, 12, - 28, 3, 77, 80, 76, 35, 78, 4, 242, 17, 73, 143, 139, 4, 69, 8, 32, 4, 83, - 84, 65, 78, 23, 84, 2, 183, 219, 4, 67, 6, 46, 69, 20, 3, 82, 65, 82, - 211, 221, 3, 65, 2, 247, 155, 4, 78, 2, 171, 129, 3, 73, 20, 80, 4, 65, - 82, 75, 69, 22, 69, 206, 1, 73, 74, 85, 237, 204, 4, 3, 79, 85, 66, 2, - 199, 154, 4, 78, 6, 132, 1, 19, 70, 69, 67, 84, 73, 86, 69, 78, 69, 83, - 83, 32, 79, 82, 32, 68, 73, 83, 84, 34, 80, 133, 7, 6, 67, 73, 83, 73, - 86, 69, 2, 11, 79, 2, 211, 153, 4, 82, 2, 17, 2, 65, 82, 2, 143, 10, 84, - 8, 68, 6, 70, 70, 73, 67, 85, 76, 34, 77, 161, 12, 4, 86, 69, 82, 71, 2, - 11, 84, 2, 171, 213, 3, 73, 4, 140, 1, 2, 73, 78, 243, 150, 4, 77, 14, - 100, 5, 77, 66, 69, 76, 76, 34, 78, 124, 4, 88, 72, 65, 85, 236, 106, 3, - 84, 69, 82, 239, 242, 2, 65, 2, 189, 164, 4, 3, 73, 83, 72, 6, 80, 4, 68, - 69, 65, 86, 20, 4, 76, 65, 82, 71, 129, 233, 2, 4, 67, 79, 85, 78, 2, - 175, 238, 3, 79, 2, 183, 163, 4, 69, 2, 135, 150, 4, 83, 12, 70, 79, 76, - 3, 85, 76, 76, 196, 5, 3, 65, 73, 76, 147, 181, 4, 76, 4, 18, 76, 35, 83, - 2, 225, 148, 4, 3, 76, 79, 87, 2, 205, 4, 2, 84, 69, 4, 142, 150, 3, 32, - 211, 56, 78, 10, 96, 8, 65, 84, 72, 69, 82, 73, 78, 71, 22, 79, 64, 3, - 82, 69, 65, 65, 5, 85, 65, 82, 68, 69, 5, 227, 228, 3, 32, 2, 41, 8, 73, - 78, 71, 32, 84, 79, 32, 77, 2, 211, 174, 4, 69, 2, 75, 84, 4, 48, 2, 65, - 82, 33, 6, 79, 76, 68, 73, 78, 71, 2, 11, 68, 2, 187, 204, 3, 78, 2, 11, - 32, 2, 195, 216, 1, 66, 4, 26, 67, 211, 164, 4, 78, 2, 143, 5, 82, 4, 60, - 7, 69, 69, 80, 73, 78, 71, 32, 225, 140, 4, 2, 73, 78, 2, 11, 83, 2, 151, - 221, 2, 77, 6, 18, 65, 107, 69, 4, 60, 3, 66, 79, 85, 21, 8, 87, 32, 79, - 82, 32, 77, 79, 68, 2, 243, 143, 4, 82, 2, 187, 215, 3, 69, 2, 139, 144, - 4, 71, 6, 26, 65, 30, 69, 47, 73, 2, 153, 143, 4, 2, 83, 83, 2, 11, 65, - 2, 11, 83, 2, 191, 191, 3, 85, 2, 11, 82, 2, 171, 153, 4, 69, 4, 204, - 162, 3, 7, 78, 32, 84, 72, 69, 32, 86, 139, 107, 80, 8, 54, 65, 64, 4, - 69, 78, 69, 84, 165, 98, 2, 85, 82, 4, 22, 84, 243, 2, 67, 2, 17, 2, 84, - 69, 2, 183, 142, 4, 82, 2, 195, 177, 1, 82, 12, 30, 69, 157, 1, 2, 73, - 84, 10, 34, 76, 22, 83, 231, 186, 4, 65, 2, 251, 204, 1, 69, 6, 18, 73, - 51, 80, 4, 30, 68, 137, 1, 2, 83, 84, 2, 147, 1, 69, 2, 11, 79, 2, 199, - 211, 3, 78, 2, 255, 210, 3, 85, 10, 34, 69, 64, 2, 73, 78, 23, 84, 2, 11, - 86, 2, 17, 2, 69, 82, 2, 11, 65, 2, 211, 158, 4, 78, 2, 183, 138, 4, 75, - 6, 42, 79, 237, 219, 2, 4, 82, 69, 78, 71, 4, 26, 80, 131, 196, 4, 86, 2, - 11, 80, 2, 211, 166, 3, 65, 2, 39, 83, 4, 26, 73, 231, 183, 4, 84, 2, - 147, 137, 4, 84, 224, 2, 86, 65, 180, 31, 2, 69, 82, 246, 1, 73, 206, 1, - 79, 100, 3, 82, 69, 69, 179, 8, 85, 146, 2, 44, 4, 65, 78, 65, 32, 145, - 10, 2, 73, 32, 100, 122, 65, 38, 69, 76, 7, 76, 69, 84, 84, 69, 82, 32, - 154, 7, 79, 76, 3, 73, 66, 73, 0, 3, 85, 66, 85, 45, 2, 83, 85, 4, 184, - 8, 3, 65, 66, 65, 3, 66, 6, 58, 66, 0, 3, 69, 66, 69, 245, 7, 4, 89, 66, - 69, 89, 2, 243, 7, 69, 78, 202, 1, 65, 42, 68, 82, 70, 2, 81, 14, 71, 50, - 72, 38, 75, 62, 76, 32, 3, 77, 69, 69, 20, 2, 67, 72, 2, 74, 2, 80, 18, - 78, 42, 83, 94, 84, 102, 86, 2, 87, 42, 90, 130, 225, 2, 66, 2, 82, 3, - 89, 4, 252, 1, 2, 76, 73, 191, 174, 4, 73, 6, 30, 65, 29, 3, 72, 65, 65, - 4, 218, 2, 65, 255, 1, 86, 2, 227, 175, 4, 76, 2, 123, 65, 6, 110, 65, - 86, 78, 245, 173, 4, 3, 72, 65, 73, 4, 178, 228, 2, 72, 147, 237, 1, 65, - 4, 26, 65, 247, 227, 2, 72, 2, 11, 65, 2, 199, 174, 4, 70, 4, 18, 65, 35, - 72, 2, 11, 65, 2, 147, 174, 4, 77, 2, 211, 1, 65, 4, 224, 173, 4, 2, 79, - 79, 191, 34, 65, 8, 32, 2, 65, 65, 18, 72, 23, 69, 2, 163, 16, 68, 4, 18, - 69, 87, 65, 2, 243, 172, 4, 69, 10, 62, 65, 16, 3, 72, 65, 65, 190, 225, - 2, 84, 203, 239, 1, 79, 2, 131, 1, 86, 5, 155, 172, 4, 76, 2, 17, 2, 65, - 65, 2, 243, 171, 4, 86, 6, 26, 65, 175, 208, 4, 79, 4, 26, 86, 147, 208, - 4, 65, 2, 21, 3, 73, 89, 65, 2, 159, 188, 4, 78, 6, 48, 3, 65, 66, 79, - 14, 66, 1, 3, 79, 66, 79, 2, 23, 65, 2, 11, 79, 2, 11, 70, 2, 11, 73, 2, - 167, 187, 4, 76, 2, 211, 206, 3, 75, 174, 1, 26, 67, 159, 224, 2, 68, - 154, 1, 132, 1, 9, 72, 65, 82, 65, 67, 84, 69, 82, 32, 245, 162, 4, 17, - 85, 82, 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, 32, 66, 65, 152, - 1, 176, 2, 6, 66, 79, 32, 66, 65, 73, 16, 6, 67, 72, 79, 32, 67, 72, 56, - 3, 68, 79, 32, 44, 2, 70, 79, 76, 3, 72, 79, 32, 66, 75, 190, 1, 76, 138, - 1, 77, 252, 1, 7, 65, 78, 71, 75, 72, 65, 78, 46, 78, 158, 1, 80, 234, 1, - 82, 70, 83, 238, 2, 84, 224, 2, 3, 87, 79, 32, 34, 89, 145, 226, 3, 2, - 79, 32, 2, 143, 12, 77, 8, 204, 210, 2, 2, 65, 78, 170, 167, 1, 73, 155, - 58, 79, 4, 164, 165, 4, 3, 67, 72, 65, 227, 33, 68, 6, 32, 2, 32, 70, 21, - 2, 78, 71, 4, 219, 209, 2, 65, 2, 147, 130, 3, 77, 4, 40, 4, 78, 79, 75, - 72, 223, 199, 4, 72, 2, 239, 197, 4, 85, 14, 40, 2, 72, 79, 229, 9, 3, - 79, 32, 75, 12, 26, 32, 239, 206, 3, 77, 10, 36, 2, 75, 72, 57, 3, 82, - 65, 75, 8, 158, 9, 87, 150, 171, 3, 85, 254, 67, 79, 255, 59, 65, 2, 135, - 239, 3, 72, 8, 76, 2, 79, 32, 224, 11, 8, 65, 75, 75, 72, 65, 78, 71, 89, - 139, 187, 4, 85, 4, 32, 2, 67, 72, 207, 245, 3, 76, 2, 171, 164, 4, 85, - 16, 28, 2, 65, 73, 235, 52, 79, 14, 42, 32, 176, 1, 3, 84, 65, 73, 19, - 89, 10, 76, 4, 67, 72, 65, 84, 44, 5, 72, 65, 78, 45, 65, 22, 84, 191, - 193, 4, 69, 2, 11, 84, 2, 11, 65, 2, 195, 194, 4, 87, 2, 151, 177, 3, 75, - 4, 222, 165, 4, 72, 159, 11, 82, 2, 203, 2, 75, 2, 193, 154, 4, 2, 65, - 77, 8, 60, 3, 71, 79, 32, 32, 3, 73, 75, 72, 29, 3, 79, 32, 78, 2, 11, - 78, 2, 211, 158, 4, 71, 2, 237, 166, 4, 2, 65, 72, 4, 182, 243, 3, 69, - 215, 79, 85, 12, 68, 6, 65, 73, 89, 65, 78, 78, 22, 72, 249, 159, 4, 3, - 79, 32, 80, 2, 203, 174, 4, 79, 8, 36, 3, 73, 78, 84, 21, 2, 79, 32, 2, - 255, 156, 4, 72, 6, 44, 2, 80, 72, 161, 5, 4, 83, 65, 77, 80, 4, 182, - 240, 3, 85, 155, 1, 65, 4, 32, 2, 79, 32, 227, 192, 4, 85, 2, 11, 82, 2, - 155, 190, 4, 85, 36, 44, 4, 65, 82, 65, 32, 225, 1, 2, 79, 32, 28, 62, - 65, 130, 1, 85, 134, 184, 2, 73, 206, 134, 2, 69, 3, 79, 13, 64, 6, 73, - 32, 77, 65, 73, 77, 142, 191, 4, 65, 2, 69, 3, 77, 4, 26, 65, 191, 247, - 2, 85, 2, 11, 76, 2, 139, 171, 4, 65, 9, 186, 254, 3, 69, 151, 64, 85, 8, - 24, 2, 82, 85, 23, 83, 2, 187, 170, 4, 83, 6, 230, 155, 4, 65, 226, 31, - 85, 187, 2, 79, 18, 30, 72, 133, 2, 2, 79, 32, 14, 48, 6, 65, 78, 84, 72, - 65, 75, 21, 2, 79, 32, 2, 175, 169, 3, 72, 12, 80, 8, 78, 65, 78, 71, 77, - 79, 78, 84, 20, 4, 80, 72, 85, 84, 13, 2, 84, 72, 2, 163, 157, 4, 72, 2, - 123, 72, 8, 34, 65, 234, 234, 3, 79, 3, 85, 4, 158, 244, 2, 72, 183, 199, - 1, 78, 4, 38, 84, 165, 140, 2, 3, 80, 65, 84, 2, 147, 156, 4, 65, 2, 11, - 87, 2, 243, 233, 2, 65, 6, 30, 65, 45, 3, 79, 32, 89, 2, 21, 3, 77, 65, - 75, 2, 231, 242, 2, 75, 4, 154, 233, 3, 73, 215, 77, 65, 10, 30, 69, 145, - 1, 2, 77, 79, 6, 18, 32, 107, 70, 4, 84, 11, 68, 79, 69, 83, 32, 78, 79, - 84, 32, 69, 88, 201, 164, 3, 4, 69, 88, 73, 83, 2, 147, 113, 73, 2, 171, - 152, 3, 79, 4, 46, 77, 157, 145, 3, 5, 68, 89, 78, 65, 77, 2, 187, 129, - 3, 69, 10, 18, 78, 95, 82, 8, 26, 32, 191, 175, 3, 75, 6, 26, 83, 255, - 177, 2, 71, 4, 218, 165, 2, 65, 211, 212, 1, 80, 2, 21, 3, 68, 32, 80, 2, - 25, 4, 76, 65, 67, 69, 2, 17, 2, 32, 77, 2, 143, 163, 3, 69, 4, 56, 4, - 85, 71, 72, 84, 173, 162, 3, 4, 78, 71, 32, 83, 2, 185, 164, 1, 5, 32, - 66, 65, 76, 76, 46, 22, 32, 199, 3, 45, 30, 154, 1, 68, 90, 76, 114, 82, - 130, 131, 1, 66, 86, 67, 242, 5, 83, 170, 149, 1, 80, 253, 21, 15, 78, - 69, 84, 87, 79, 82, 75, 69, 68, 32, 67, 79, 77, 80, 85, 4, 64, 10, 73, - 77, 69, 78, 83, 73, 79, 78, 65, 76, 219, 134, 1, 79, 2, 131, 146, 3, 32, - 6, 76, 12, 73, 78, 69, 83, 32, 67, 79, 78, 86, 69, 82, 71, 237, 127, 2, - 69, 70, 4, 217, 135, 4, 3, 73, 78, 71, 10, 36, 4, 65, 89, 83, 32, 147, - 127, 73, 8, 230, 153, 2, 66, 222, 155, 1, 65, 242, 81, 76, 31, 82, 16, - 44, 2, 68, 32, 254, 3, 80, 131, 133, 1, 69, 12, 196, 2, 12, 84, 79, 80, - 45, 76, 73, 71, 72, 84, 69, 68, 32, 80, 17, 76, 69, 70, 84, 45, 76, 73, - 71, 72, 84, 69, 68, 32, 68, 79, 87, 78, 0, 16, 82, 73, 71, 72, 84, 45, - 76, 73, 71, 72, 84, 69, 68, 32, 85, 80, 209, 223, 1, 25, 66, 79, 84, 84, - 79, 77, 45, 76, 73, 71, 72, 84, 69, 68, 32, 82, 73, 71, 72, 84, 87, 65, - 82, 68, 83, 6, 76, 4, 76, 69, 70, 84, 69, 11, 82, 73, 71, 72, 84, 87, 65, - 82, 68, 83, 32, 2, 11, 87, 2, 25, 4, 65, 82, 68, 83, 2, 181, 200, 1, 2, - 32, 69, 4, 154, 200, 1, 69, 139, 23, 65, 2, 129, 239, 3, 5, 69, 82, 45, - 69, 77, 8, 34, 77, 85, 4, 78, 68, 69, 82, 4, 21, 3, 66, 83, 32, 4, 38, - 85, 149, 182, 2, 3, 68, 79, 87, 2, 199, 165, 3, 80, 4, 212, 128, 3, 10, - 32, 67, 76, 79, 85, 68, 32, 65, 78, 68, 141, 45, 2, 83, 84, 230, 5, 204, - 1, 6, 66, 69, 84, 65, 78, 32, 240, 45, 6, 69, 32, 79, 86, 69, 82, 72, 7, - 70, 73, 78, 65, 71, 72, 32, 150, 9, 71, 152, 1, 3, 76, 68, 69, 248, 2, 2, - 77, 69, 56, 2, 78, 89, 78, 82, 223, 154, 3, 67, 160, 3, 228, 2, 18, 65, - 83, 84, 82, 79, 76, 79, 71, 73, 67, 65, 76, 32, 83, 73, 71, 78, 32, 172, - 1, 18, 67, 65, 78, 84, 73, 76, 76, 65, 84, 73, 79, 78, 32, 83, 73, 71, - 78, 32, 208, 1, 6, 68, 73, 71, 73, 84, 32, 100, 9, 75, 85, 32, 82, 85, - 32, 75, 72, 65, 26, 76, 204, 3, 5, 77, 65, 82, 75, 32, 246, 18, 83, 181, - 15, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 6, 46, 83, 129, 41, - 6, 45, 75, 72, 89, 85, 68, 4, 104, 8, 68, 79, 78, 71, 32, 84, 83, 72, - 229, 21, 13, 71, 82, 65, 32, 71, 67, 65, 78, 32, 45, 67, 72, 65, 2, 159, - 32, 85, 8, 144, 1, 5, 72, 69, 65, 86, 89, 0, 5, 76, 73, 71, 72, 84, 40, - 7, 83, 66, 85, 66, 32, 45, 67, 189, 255, 3, 8, 67, 65, 78, 71, 32, 84, - 69, 45, 2, 17, 2, 32, 66, 2, 231, 144, 3, 69, 2, 251, 154, 3, 72, 40, - 156, 182, 2, 4, 72, 65, 76, 70, 82, 70, 30, 83, 42, 84, 62, 90, 130, 83, - 78, 14, 79, 199, 110, 69, 5, 245, 10, 2, 32, 66, 92, 96, 6, 69, 84, 84, - 69, 82, 32, 181, 2, 13, 79, 71, 79, 84, 89, 80, 69, 32, 83, 73, 71, 78, - 32, 88, 226, 1, 75, 50, 82, 208, 213, 3, 10, 70, 73, 88, 69, 68, 45, 70, - 79, 82, 77, 202, 1, 68, 86, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, - 80, 2, 90, 254, 68, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 87, 2, 89, 187, 2, - 65, 8, 134, 158, 4, 83, 14, 72, 2, 75, 187, 2, 65, 4, 226, 157, 4, 82, - 187, 2, 65, 4, 248, 25, 3, 76, 72, 65, 13, 4, 67, 72, 65, 68, 80, 250, 2, - 66, 176, 1, 8, 77, 78, 89, 65, 77, 32, 89, 73, 114, 67, 172, 2, 13, 89, - 73, 71, 32, 77, 71, 79, 32, 84, 83, 72, 69, 71, 190, 1, 71, 212, 2, 9, - 65, 78, 71, 32, 75, 72, 65, 78, 71, 58, 72, 48, 2, 73, 78, 246, 1, 78, - 220, 1, 2, 80, 65, 30, 82, 118, 83, 46, 84, 36, 4, 76, 69, 65, 68, 233, - 250, 1, 17, 68, 69, 76, 73, 77, 73, 84, 69, 82, 32, 84, 83, 72, 69, 71, - 32, 66, 10, 52, 9, 75, 65, 45, 32, 83, 72, 79, 71, 32, 27, 83, 4, 142, 1, - 71, 67, 89, 6, 34, 75, 229, 21, 3, 68, 85, 83, 4, 56, 6, 65, 45, 32, 83, - 72, 79, 89, 4, 85, 82, 32, 89, 2, 21, 3, 71, 32, 71, 2, 41, 8, 73, 32, - 77, 71, 79, 32, 82, 71, 2, 163, 211, 2, 89, 2, 221, 2, 2, 73, 71, 12, 84, - 5, 65, 82, 69, 84, 32, 240, 1, 2, 72, 69, 29, 7, 76, 79, 83, 73, 78, 71, - 32, 6, 112, 12, 45, 68, 90, 85, 68, 32, 82, 84, 65, 71, 83, 32, 97, 12, - 89, 73, 71, 32, 77, 71, 79, 32, 80, 72, 85, 82, 4, 42, 66, 37, 6, 77, 69, - 32, 76, 79, 78, 2, 33, 6, 90, 72, 73, 32, 77, 73, 2, 231, 22, 71, 2, 225, - 3, 3, 32, 83, 72, 2, 137, 142, 2, 2, 32, 77, 4, 68, 13, 66, 82, 68, 65, - 32, 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, 2, 213, 5, 11, 73, 71, 32, 77, - 71, 79, 32, 83, 71, 65, 66, 12, 68, 4, 84, 69, 82, 32, 141, 2, 8, 85, 71, - 32, 82, 84, 65, 71, 83, 8, 56, 8, 89, 73, 71, 32, 77, 71, 79, 32, 211, - 207, 3, 84, 6, 68, 4, 45, 85, 77, 32, 117, 9, 84, 82, 85, 78, 67, 65, 84, - 69, 68, 4, 88, 7, 82, 78, 65, 77, 32, 66, 67, 245, 2, 10, 71, 84, 69, 82, - 32, 84, 83, 72, 69, 71, 2, 241, 2, 2, 65, 68, 2, 203, 145, 4, 32, 4, 21, - 3, 32, 71, 89, 4, 130, 196, 3, 79, 135, 18, 65, 2, 17, 2, 65, 76, 2, 213, - 238, 3, 2, 65, 78, 6, 92, 6, 73, 84, 73, 65, 76, 32, 165, 204, 3, 11, 84, - 69, 82, 83, 89, 76, 76, 65, 66, 73, 67, 4, 68, 13, 66, 82, 68, 65, 32, - 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, 2, 53, 11, 73, 71, 32, 77, 71, 79, - 32, 77, 68, 85, 78, 2, 151, 240, 3, 32, 10, 72, 10, 71, 65, 83, 32, 66, - 90, 85, 78, 71, 32, 77, 4, 89, 73, 83, 32, 4, 56, 3, 83, 71, 79, 245, - 237, 3, 5, 78, 89, 73, 32, 90, 2, 151, 10, 82, 6, 44, 5, 84, 83, 72, 69, - 71, 163, 201, 3, 83, 5, 147, 201, 3, 32, 2, 233, 234, 3, 2, 76, 85, 4, - 212, 200, 3, 8, 71, 89, 65, 32, 71, 82, 65, 77, 1, 14, 73, 78, 32, 67, - 72, 69, 78, 32, 83, 80, 85, 78, 71, 83, 4, 224, 199, 3, 4, 66, 82, 85, + 58, 82, 199, 224, 4, 80, 2, 33, 6, 85, 84, 32, 72, 65, 78, 2, 231, 140, + 4, 68, 6, 72, 9, 45, 79, 70, 70, 32, 67, 65, 76, 69, 29, 5, 68, 82, 79, + 80, 45, 2, 245, 142, 4, 2, 78, 68, 4, 250, 155, 3, 83, 245, 115, 4, 66, + 65, 82, 66, 2, 171, 142, 4, 69, 216, 1, 38, 69, 213, 2, 4, 85, 71, 85, + 32, 16, 60, 6, 80, 72, 79, 78, 69, 32, 246, 1, 83, 151, 179, 2, 86, 12, + 132, 1, 3, 82, 69, 67, 228, 150, 1, 3, 76, 79, 67, 146, 144, 3, 83, 189, + 116, 13, 79, 78, 32, 84, 79, 80, 32, 79, 70, 32, 77, 79, 68, 6, 36, 5, + 69, 73, 86, 69, 82, 51, 79, 5, 29, 5, 32, 87, 73, 84, 72, 2, 179, 38, 32, + 2, 255, 140, 3, 82, 2, 11, 67, 2, 167, 229, 3, 79, 200, 1, 162, 1, 65, + 20, 15, 70, 82, 65, 67, 84, 73, 79, 78, 32, 68, 73, 71, 73, 84, 32, 164, + 2, 2, 76, 69, 252, 3, 5, 83, 73, 71, 78, 32, 198, 2, 86, 147, 177, 3, 68, + 2, 247, 205, 1, 73, 14, 74, 84, 40, 2, 79, 78, 81, 10, 90, 69, 82, 79, + 32, 70, 79, 82, 32, 79, 8, 36, 3, 72, 82, 69, 13, 2, 87, 79, 4, 11, 69, + 4, 29, 5, 32, 70, 79, 82, 32, 4, 34, 79, 21, 4, 69, 86, 69, 78, 2, 17, 2, + 68, 68, 2, 49, 10, 32, 80, 79, 87, 69, 82, 83, 32, 79, 70, 2, 133, 22, 2, + 32, 70, 114, 44, 5, 84, 84, 69, 82, 32, 219, 230, 4, 78, 112, 214, 1, 68, + 54, 78, 106, 82, 38, 84, 138, 203, 1, 65, 82, 76, 114, 86, 186, 5, 85, + 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 82, 66, 2, 67, 2, 71, 2, + 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, 2, 77, 3, 89, 10, 166, 223, 4, + 68, 138, 69, 72, 2, 90, 187, 2, 65, 10, 42, 65, 210, 163, 5, 71, 2, 78, + 3, 89, 5, 41, 8, 75, 65, 65, 82, 65, 32, 80, 79, 2, 131, 37, 76, 6, 158, + 204, 1, 82, 175, 217, 3, 65, 10, 230, 221, 4, 84, 138, 69, 72, 2, 83, + 187, 2, 65, 20, 86, 67, 194, 1, 83, 212, 35, 3, 84, 85, 85, 170, 107, 78, + 242, 60, 65, 231, 147, 3, 86, 6, 60, 9, 79, 77, 66, 73, 78, 73, 78, 71, + 32, 171, 143, 1, 65, 4, 70, 65, 145, 166, 4, 11, 67, 65, 78, 68, 82, 65, + 66, 73, 78, 68, 85, 2, 33, 6, 78, 85, 83, 86, 65, 82, 2, 235, 165, 4, 65, + 2, 11, 73, 2, 209, 231, 3, 3, 68, 68, 72, 30, 49, 10, 79, 87, 69, 76, 32, + 83, 73, 71, 78, 32, 30, 174, 206, 1, 65, 38, 85, 22, 86, 182, 141, 1, 79, + 134, 60, 73, 191, 201, 1, 69, 6, 80, 10, 78, 73, 83, 32, 82, 65, 67, 81, + 85, 69, 174, 132, 3, 71, 175, 156, 2, 84, 2, 11, 84, 2, 25, 4, 32, 65, + 78, 68, 2, 215, 152, 3, 32, 2, 199, 137, 5, 66, 162, 1, 210, 2, 65, 134, + 1, 66, 174, 1, 67, 222, 2, 68, 202, 3, 69, 154, 2, 70, 182, 1, 71, 198, + 1, 72, 64, 8, 89, 79, 85, 84, 72, 70, 85, 76, 52, 2, 73, 78, 46, 75, 98, + 76, 146, 1, 77, 134, 1, 79, 62, 80, 142, 1, 82, 210, 1, 83, 224, 1, 14, + 86, 65, 83, 84, 78, 69, 83, 83, 32, 79, 82, 32, 87, 65, 12, 2, 87, 65, + 154, 94, 85, 203, 150, 4, 74, 8, 88, 4, 67, 67, 85, 77, 22, 83, 228, 20, + 2, 68, 86, 229, 197, 1, 5, 71, 71, 82, 65, 86, 2, 239, 185, 1, 85, 2, + 151, 217, 4, 67, 6, 92, 7, 79, 76, 68, 32, 82, 69, 83, 32, 5, 82, 65, 78, + 67, 72, 177, 146, 4, 3, 65, 82, 82, 2, 249, 202, 4, 3, 79, 76, 85, 2, 11, + 73, 2, 213, 166, 1, 3, 78, 71, 32, 22, 54, 72, 20, 3, 76, 79, 83, 66, 79, + 175, 152, 2, 69, 2, 155, 229, 3, 65, 6, 26, 69, 227, 249, 3, 85, 4, 230, + 248, 1, 68, 215, 138, 2, 78, 12, 28, 3, 77, 80, 76, 35, 78, 4, 246, 17, + 73, 143, 183, 4, 69, 8, 32, 4, 83, 84, 65, 78, 23, 84, 2, 199, 135, 5, + 67, 6, 46, 69, 20, 3, 82, 65, 82, 203, 137, 4, 65, 2, 251, 199, 4, 78, 2, + 183, 170, 3, 73, 20, 80, 4, 65, 82, 75, 69, 22, 69, 210, 1, 73, 74, 85, + 249, 248, 4, 3, 79, 85, 66, 2, 203, 198, 4, 78, 6, 132, 1, 19, 70, 69, + 67, 84, 73, 86, 69, 78, 69, 83, 83, 32, 79, 82, 32, 68, 73, 83, 84, 34, + 80, 137, 7, 6, 67, 73, 83, 73, 86, 69, 2, 11, 79, 2, 215, 197, 4, 82, 2, + 11, 65, 2, 11, 82, 2, 143, 10, 84, 8, 68, 6, 70, 70, 73, 67, 85, 76, 34, + 77, 161, 12, 4, 86, 69, 82, 71, 2, 11, 84, 2, 159, 129, 4, 73, 4, 140, 1, + 2, 73, 78, 243, 194, 4, 77, 14, 100, 5, 77, 66, 69, 76, 76, 34, 78, 124, + 4, 88, 72, 65, 85, 188, 106, 3, 84, 69, 82, 147, 159, 3, 65, 2, 189, 208, + 4, 3, 73, 83, 72, 6, 80, 4, 68, 69, 65, 86, 20, 4, 76, 65, 82, 71, 129, + 146, 3, 4, 67, 79, 85, 78, 2, 171, 154, 4, 79, 2, 183, 207, 4, 69, 2, + 135, 194, 4, 83, 12, 70, 79, 76, 3, 85, 76, 76, 196, 5, 3, 65, 73, 76, + 159, 225, 4, 76, 4, 18, 76, 35, 83, 2, 225, 192, 4, 3, 76, 79, 87, 2, + 205, 4, 2, 84, 69, 4, 166, 191, 3, 32, 175, 59, 78, 10, 96, 8, 65, 84, + 72, 69, 82, 73, 78, 71, 22, 79, 64, 3, 82, 69, 65, 65, 5, 85, 65, 82, 68, + 69, 5, 215, 144, 4, 32, 2, 41, 8, 73, 78, 71, 32, 84, 79, 32, 77, 2, 211, + 218, 4, 69, 2, 75, 84, 4, 48, 2, 65, 82, 33, 6, 79, 76, 68, 73, 78, 71, + 2, 11, 68, 2, 175, 248, 3, 78, 2, 11, 32, 2, 235, 243, 1, 66, 4, 26, 67, + 211, 208, 4, 78, 2, 143, 5, 82, 4, 60, 7, 69, 69, 80, 73, 78, 71, 32, + 221, 184, 4, 2, 73, 78, 2, 11, 83, 2, 147, 134, 3, 77, 6, 18, 65, 107, + 69, 4, 60, 3, 66, 79, 85, 21, 8, 87, 32, 79, 82, 32, 77, 79, 68, 2, 243, + 187, 4, 82, 2, 175, 131, 4, 69, 2, 139, 188, 4, 71, 6, 26, 65, 30, 69, + 47, 73, 2, 153, 187, 4, 2, 83, 83, 2, 11, 65, 2, 11, 83, 2, 179, 235, 3, + 85, 2, 11, 82, 2, 171, 197, 4, 69, 4, 164, 203, 3, 7, 78, 32, 84, 72, 69, + 32, 86, 179, 110, 80, 8, 54, 65, 64, 4, 69, 78, 69, 84, 245, 97, 2, 85, + 82, 4, 22, 84, 243, 2, 67, 2, 17, 2, 84, 69, 2, 183, 186, 4, 82, 2, 139, + 200, 1, 82, 12, 30, 69, 157, 1, 2, 73, 84, 10, 34, 76, 22, 83, 243, 230, + 4, 65, 2, 227, 227, 1, 69, 6, 18, 73, 51, 80, 4, 30, 68, 137, 1, 2, 83, + 84, 2, 147, 1, 69, 2, 11, 79, 2, 187, 255, 3, 78, 2, 243, 254, 3, 85, 10, + 34, 69, 64, 2, 73, 78, 23, 84, 2, 11, 86, 2, 17, 2, 69, 82, 2, 11, 65, 2, + 211, 202, 4, 78, 2, 183, 182, 4, 75, 6, 42, 79, 237, 132, 3, 4, 82, 69, + 78, 71, 4, 26, 80, 143, 240, 4, 86, 2, 11, 80, 2, 179, 209, 3, 65, 2, 39, + 83, 4, 26, 73, 243, 227, 4, 84, 2, 147, 181, 4, 84, 224, 2, 86, 65, 180, + 31, 2, 69, 82, 246, 1, 73, 206, 1, 79, 100, 3, 82, 69, 69, 187, 8, 85, + 146, 2, 44, 4, 65, 78, 65, 32, 145, 10, 2, 73, 32, 100, 122, 65, 38, 69, + 76, 7, 76, 69, 84, 84, 69, 82, 32, 154, 7, 79, 76, 3, 73, 66, 73, 0, 3, + 85, 66, 85, 45, 2, 83, 85, 4, 184, 8, 3, 65, 66, 65, 3, 66, 6, 58, 66, 0, + 3, 69, 66, 69, 245, 7, 4, 89, 66, 69, 89, 2, 243, 7, 69, 78, 202, 1, 65, + 42, 68, 82, 70, 2, 81, 14, 71, 50, 72, 38, 75, 62, 76, 32, 3, 77, 69, 69, + 20, 2, 67, 72, 2, 74, 2, 80, 18, 78, 42, 83, 94, 84, 102, 86, 2, 87, 42, + 90, 138, 138, 3, 66, 2, 82, 3, 89, 4, 252, 1, 2, 76, 73, 203, 218, 4, 73, + 6, 30, 65, 29, 3, 72, 65, 65, 4, 218, 2, 65, 255, 1, 86, 2, 239, 219, 4, + 76, 2, 123, 65, 6, 110, 65, 86, 78, 129, 218, 4, 3, 72, 65, 73, 4, 186, + 141, 3, 72, 151, 240, 1, 65, 4, 26, 65, 255, 140, 3, 72, 2, 11, 65, 2, + 211, 218, 4, 70, 4, 18, 65, 35, 72, 2, 11, 65, 2, 159, 218, 4, 77, 2, + 211, 1, 65, 4, 236, 217, 4, 2, 79, 79, 191, 34, 65, 8, 32, 2, 65, 65, 18, + 72, 23, 69, 2, 163, 16, 68, 4, 18, 69, 87, 65, 2, 255, 216, 4, 69, 10, + 62, 65, 16, 3, 72, 65, 65, 198, 138, 3, 84, 207, 242, 1, 79, 2, 131, 1, + 86, 5, 167, 216, 4, 76, 2, 17, 2, 65, 65, 2, 255, 215, 4, 86, 6, 26, 65, + 187, 252, 4, 79, 4, 26, 86, 159, 252, 4, 65, 2, 21, 3, 73, 89, 65, 2, + 171, 232, 4, 78, 6, 48, 3, 65, 66, 79, 14, 66, 1, 3, 79, 66, 79, 2, 23, + 65, 2, 11, 79, 2, 11, 70, 2, 11, 73, 2, 179, 231, 4, 76, 2, 199, 250, 3, + 75, 174, 1, 26, 67, 167, 137, 3, 68, 154, 1, 132, 1, 9, 72, 65, 82, 65, + 67, 84, 69, 82, 32, 129, 207, 4, 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, + 89, 77, 66, 79, 76, 32, 66, 65, 152, 1, 176, 2, 6, 66, 79, 32, 66, 65, + 73, 16, 6, 67, 72, 79, 32, 67, 72, 56, 3, 68, 79, 32, 44, 2, 70, 79, 76, + 3, 72, 79, 32, 66, 75, 190, 1, 76, 138, 1, 77, 252, 1, 7, 65, 78, 71, 75, + 72, 65, 78, 46, 78, 158, 1, 80, 234, 1, 82, 70, 83, 238, 2, 84, 224, 2, + 3, 87, 79, 32, 34, 89, 141, 142, 4, 2, 79, 32, 2, 143, 12, 77, 8, 212, + 251, 2, 2, 65, 78, 162, 170, 1, 73, 167, 58, 79, 4, 176, 209, 4, 3, 67, + 72, 65, 227, 33, 68, 6, 32, 2, 32, 70, 21, 2, 78, 71, 4, 227, 250, 2, 65, + 2, 211, 171, 3, 77, 4, 40, 4, 78, 79, 75, 72, 235, 243, 4, 72, 2, 251, + 241, 4, 85, 14, 40, 2, 72, 79, 229, 9, 3, 79, 32, 75, 12, 26, 32, 235, + 250, 3, 77, 10, 36, 2, 75, 72, 57, 3, 82, 65, 75, 8, 158, 9, 87, 138, + 215, 3, 85, 138, 68, 79, 139, 60, 65, 2, 131, 155, 4, 72, 8, 76, 2, 79, + 32, 224, 11, 8, 65, 75, 75, 72, 65, 78, 71, 89, 151, 231, 4, 85, 4, 32, + 2, 67, 72, 207, 161, 4, 76, 2, 183, 208, 4, 85, 16, 28, 2, 65, 73, 219, + 52, 79, 14, 42, 32, 176, 1, 3, 84, 65, 73, 19, 89, 10, 76, 4, 67, 72, 65, + 84, 44, 5, 72, 65, 78, 45, 65, 22, 84, 203, 237, 4, 69, 2, 11, 84, 2, 11, + 65, 2, 207, 238, 4, 87, 2, 139, 221, 3, 75, 4, 234, 209, 4, 72, 159, 11, + 82, 2, 203, 2, 75, 2, 205, 198, 4, 2, 65, 77, 8, 60, 3, 71, 79, 32, 32, + 3, 73, 75, 72, 29, 3, 79, 32, 78, 2, 11, 78, 2, 223, 202, 4, 71, 2, 249, + 210, 4, 2, 65, 72, 4, 182, 159, 4, 69, 227, 79, 85, 12, 68, 6, 65, 73, + 89, 65, 78, 78, 22, 72, 133, 204, 4, 3, 79, 32, 80, 2, 215, 218, 4, 79, + 8, 36, 3, 73, 78, 84, 21, 2, 79, 32, 2, 139, 201, 4, 72, 6, 44, 2, 80, + 72, 161, 5, 4, 83, 65, 77, 80, 4, 182, 156, 4, 85, 155, 1, 65, 4, 32, 2, + 79, 32, 239, 236, 4, 85, 2, 11, 82, 2, 167, 234, 4, 85, 36, 44, 4, 65, + 82, 65, 32, 225, 1, 2, 79, 32, 28, 62, 65, 130, 1, 85, 130, 225, 2, 73, + 222, 137, 2, 69, 3, 79, 13, 64, 6, 73, 32, 77, 65, 73, 77, 154, 235, 4, + 65, 2, 69, 3, 77, 4, 26, 65, 255, 160, 3, 85, 2, 11, 76, 2, 151, 215, 4, + 65, 9, 186, 170, 4, 69, 163, 64, 85, 8, 24, 2, 82, 85, 23, 83, 2, 199, + 214, 4, 83, 6, 242, 199, 4, 65, 226, 31, 85, 187, 2, 79, 18, 30, 72, 133, + 2, 2, 79, 32, 14, 48, 6, 65, 78, 84, 72, 65, 75, 21, 2, 79, 32, 2, 163, + 213, 3, 72, 12, 80, 8, 78, 65, 78, 71, 77, 79, 78, 84, 20, 4, 80, 72, 85, + 84, 13, 2, 84, 72, 2, 175, 201, 4, 72, 2, 123, 72, 8, 34, 65, 234, 150, + 4, 79, 3, 85, 4, 222, 157, 3, 72, 131, 202, 1, 78, 4, 38, 84, 209, 177, + 2, 3, 80, 65, 84, 2, 159, 200, 4, 65, 2, 11, 87, 2, 139, 147, 3, 65, 6, + 30, 65, 45, 3, 79, 32, 89, 2, 21, 3, 77, 65, 75, 2, 167, 156, 3, 75, 4, + 154, 149, 4, 73, 227, 77, 65, 10, 30, 69, 145, 1, 2, 77, 79, 6, 18, 32, + 107, 70, 4, 84, 11, 68, 79, 69, 83, 32, 78, 79, 84, 32, 69, 88, 189, 208, + 3, 4, 69, 88, 73, 83, 2, 235, 122, 73, 2, 159, 196, 3, 79, 4, 46, 77, + 141, 188, 3, 5, 68, 89, 78, 65, 77, 2, 155, 172, 3, 69, 10, 18, 78, 95, + 82, 8, 26, 32, 179, 219, 3, 75, 6, 26, 83, 251, 218, 2, 71, 4, 206, 206, + 2, 65, 223, 215, 1, 80, 2, 21, 3, 68, 32, 80, 2, 25, 4, 76, 65, 67, 69, + 2, 17, 2, 32, 77, 2, 131, 207, 3, 69, 4, 56, 4, 85, 71, 72, 84, 161, 206, + 3, 4, 78, 71, 32, 83, 2, 161, 187, 1, 5, 32, 66, 65, 76, 76, 46, 22, 32, + 207, 3, 45, 30, 154, 1, 68, 90, 76, 118, 82, 198, 153, 1, 66, 86, 67, + 194, 6, 83, 134, 167, 1, 80, 137, 22, 15, 78, 69, 84, 87, 79, 82, 75, 69, + 68, 32, 67, 79, 77, 80, 85, 4, 64, 10, 73, 77, 69, 78, 83, 73, 79, 78, + 65, 76, 163, 157, 1, 79, 2, 139, 187, 3, 32, 6, 80, 12, 73, 78, 69, 83, + 32, 67, 79, 78, 86, 69, 82, 71, 149, 150, 1, 2, 69, 70, 4, 225, 179, 4, + 3, 73, 78, 71, 10, 40, 4, 65, 89, 83, 32, 183, 149, 1, 73, 8, 146, 192, + 2, 66, 166, 161, 1, 65, 130, 82, 76, 31, 82, 16, 44, 2, 68, 32, 254, 3, + 80, 147, 156, 1, 69, 12, 196, 2, 12, 84, 79, 80, 45, 76, 73, 71, 72, 84, + 69, 68, 32, 80, 17, 76, 69, 70, 84, 45, 76, 73, 71, 72, 84, 69, 68, 32, + 68, 79, 87, 78, 0, 16, 82, 73, 71, 72, 84, 45, 76, 73, 71, 72, 84, 69, + 68, 32, 85, 80, 245, 132, 2, 25, 66, 79, 84, 84, 79, 77, 45, 76, 73, 71, + 72, 84, 69, 68, 32, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 6, 76, 4, 76, + 69, 70, 84, 69, 11, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 2, 11, + 87, 2, 25, 4, 65, 82, 68, 83, 2, 197, 238, 1, 2, 32, 69, 4, 170, 238, 1, + 69, 159, 22, 65, 2, 249, 154, 4, 5, 69, 82, 45, 69, 77, 8, 34, 77, 85, 4, + 78, 68, 69, 82, 4, 21, 3, 66, 83, 32, 4, 38, 85, 149, 223, 2, 3, 68, 79, + 87, 2, 179, 209, 3, 80, 4, 180, 171, 3, 10, 32, 67, 76, 79, 85, 68, 32, + 65, 78, 68, 161, 46, 2, 83, 84, 230, 5, 204, 1, 6, 66, 69, 84, 65, 78, + 32, 184, 45, 6, 69, 32, 79, 86, 69, 82, 72, 7, 70, 73, 78, 65, 71, 72, + 32, 150, 9, 71, 152, 1, 3, 76, 68, 69, 252, 2, 2, 77, 69, 56, 2, 78, 89, + 78, 82, 139, 199, 3, 67, 160, 3, 228, 2, 18, 65, 83, 84, 82, 79, 76, 79, + 71, 73, 67, 65, 76, 32, 83, 73, 71, 78, 32, 172, 1, 18, 67, 65, 78, 84, + 73, 76, 76, 65, 84, 73, 79, 78, 32, 83, 73, 71, 78, 32, 208, 1, 6, 68, + 73, 71, 73, 84, 32, 100, 9, 75, 85, 32, 82, 85, 32, 75, 72, 65, 26, 76, + 176, 3, 5, 77, 65, 82, 75, 32, 222, 18, 83, 181, 15, 11, 86, 79, 87, 69, + 76, 32, 83, 73, 71, 78, 32, 6, 46, 83, 205, 40, 6, 45, 75, 72, 89, 85, + 68, 4, 104, 8, 68, 79, 78, 71, 32, 84, 83, 72, 205, 21, 13, 71, 82, 65, + 32, 71, 67, 65, 78, 32, 45, 67, 72, 65, 2, 235, 31, 85, 8, 144, 1, 5, 72, + 69, 65, 86, 89, 0, 5, 76, 73, 71, 72, 84, 40, 7, 83, 66, 85, 66, 32, 45, + 67, 193, 171, 4, 8, 67, 65, 78, 71, 32, 84, 69, 45, 2, 17, 2, 32, 66, 2, + 211, 188, 3, 69, 2, 231, 198, 3, 72, 40, 156, 223, 2, 4, 72, 65, 76, 70, + 82, 70, 30, 83, 42, 84, 62, 90, 238, 85, 78, 14, 79, 223, 110, 69, 5, + 221, 10, 2, 32, 66, 92, 96, 6, 69, 84, 84, 69, 82, 32, 153, 2, 13, 79, + 71, 79, 84, 89, 80, 69, 32, 83, 73, 71, 78, 32, 88, 230, 1, 75, 162, 115, + 82, 212, 142, 3, 10, 70, 73, 88, 69, 68, 45, 70, 79, 82, 77, 202, 1, 68, + 86, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 80, 2, 90, 138, 69, 45, + 2, 72, 2, 74, 2, 76, 2, 77, 2, 87, 2, 89, 187, 2, 65, 8, 134, 202, 4, 83, + 14, 72, 2, 75, 187, 2, 65, 4, 224, 25, 3, 76, 72, 65, 13, 4, 67, 72, 65, + 68, 80, 254, 2, 66, 176, 1, 8, 77, 78, 89, 65, 77, 32, 89, 73, 114, 67, + 172, 2, 13, 89, 73, 71, 32, 77, 71, 79, 32, 84, 83, 72, 69, 71, 190, 1, + 71, 212, 2, 9, 65, 78, 71, 32, 75, 72, 65, 78, 71, 58, 72, 48, 2, 73, 78, + 246, 1, 78, 222, 1, 82, 118, 83, 46, 84, 36, 4, 76, 69, 65, 68, 212, 100, + 2, 80, 65, 181, 191, 1, 17, 68, 69, 76, 73, 77, 73, 84, 69, 82, 32, 84, + 83, 72, 69, 71, 32, 66, 10, 52, 9, 75, 65, 45, 32, 83, 72, 79, 71, 32, + 27, 83, 4, 142, 1, 71, 67, 89, 6, 34, 75, 201, 21, 3, 68, 85, 83, 4, 56, + 6, 65, 45, 32, 83, 72, 79, 89, 4, 85, 82, 32, 89, 2, 21, 3, 71, 32, 71, + 2, 41, 8, 73, 32, 77, 71, 79, 32, 82, 71, 2, 243, 252, 2, 89, 2, 221, 2, + 2, 73, 71, 12, 84, 5, 65, 82, 69, 84, 32, 240, 1, 2, 72, 69, 29, 7, 76, + 79, 83, 73, 78, 71, 32, 6, 112, 12, 45, 68, 90, 85, 68, 32, 82, 84, 65, + 71, 83, 32, 97, 12, 89, 73, 71, 32, 77, 71, 79, 32, 80, 72, 85, 82, 4, + 42, 66, 37, 6, 77, 69, 32, 76, 79, 78, 2, 33, 6, 90, 72, 73, 32, 77, 73, + 2, 203, 22, 71, 2, 225, 3, 3, 32, 83, 72, 2, 149, 183, 2, 2, 32, 77, 4, + 68, 13, 66, 82, 68, 65, 32, 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, 2, + 213, 5, 11, 73, 71, 32, 77, 71, 79, 32, 83, 71, 65, 66, 12, 68, 4, 84, + 69, 82, 32, 141, 2, 8, 85, 71, 32, 82, 84, 65, 71, 83, 8, 56, 8, 89, 73, + 71, 32, 77, 71, 79, 32, 227, 251, 3, 84, 6, 68, 4, 45, 85, 77, 32, 117, + 9, 84, 82, 85, 78, 67, 65, 84, 69, 68, 4, 88, 7, 82, 78, 65, 77, 32, 66, + 67, 245, 2, 10, 71, 84, 69, 82, 32, 84, 83, 72, 69, 71, 2, 241, 2, 2, 65, + 68, 2, 231, 189, 4, 32, 4, 21, 3, 32, 71, 89, 4, 146, 240, 3, 79, 135, + 18, 65, 2, 17, 2, 65, 76, 2, 241, 154, 4, 2, 65, 78, 6, 92, 6, 73, 84, + 73, 65, 76, 32, 181, 248, 3, 11, 84, 69, 82, 83, 89, 76, 76, 65, 66, 73, + 67, 4, 68, 13, 66, 82, 68, 65, 32, 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, + 2, 53, 11, 73, 71, 32, 77, 71, 79, 32, 77, 68, 85, 78, 2, 179, 156, 4, + 32, 10, 72, 10, 71, 65, 83, 32, 66, 90, 85, 78, 71, 32, 77, 4, 89, 73, + 83, 32, 4, 56, 3, 83, 71, 79, 145, 154, 4, 5, 78, 89, 73, 32, 90, 2, 251, + 9, 82, 6, 44, 5, 84, 83, 72, 69, 71, 179, 245, 3, 83, 5, 163, 245, 3, 32, + 4, 128, 245, 3, 8, 71, 89, 65, 32, 71, 82, 65, 77, 1, 14, 73, 78, 32, 67, + 72, 69, 78, 32, 83, 80, 85, 78, 71, 83, 4, 140, 244, 3, 4, 66, 82, 85, 76, 39, 72, 6, 32, 4, 82, 65, 73, 76, 55, 83, 2, 225, 7, 9, 73, 78, 71, - 32, 77, 67, 72, 65, 78, 4, 56, 5, 65, 32, 45, 80, 72, 165, 198, 3, 3, 72, - 69, 71, 2, 255, 231, 3, 82, 156, 1, 84, 4, 73, 71, 78, 32, 228, 6, 9, 85, + 32, 77, 67, 72, 65, 78, 4, 56, 5, 65, 32, 45, 80, 72, 209, 242, 3, 3, 72, + 69, 71, 2, 183, 148, 4, 82, 156, 1, 84, 4, 73, 71, 78, 32, 228, 6, 9, 85, 66, 74, 79, 73, 78, 69, 68, 32, 143, 4, 89, 42, 176, 1, 4, 71, 82, 85, 32, 96, 2, 76, 67, 30, 77, 36, 11, 78, 89, 73, 32, 90, 76, 65, 32, 78, 65, 65, 22, 82, 252, 2, 2, 89, 65, 130, 4, 73, 165, 4, 5, 83, 78, 65, 32, 76, 4, 40, 3, 67, 65, 78, 1, 3, 77, 69, 68, 2, 25, 4, 32, 82, 71, 89, 2, 169, 4, 2, 73, 78, 4, 238, 3, 73, 179, 4, 69, 4, 136, 4, 2, 65, 82, 231, - 3, 67, 2, 159, 228, 3, 32, 20, 116, 4, 68, 69, 76, 32, 240, 1, 10, 74, - 69, 83, 32, 83, 85, 32, 78, 71, 65, 137, 192, 3, 6, 78, 65, 77, 32, 66, - 67, 16, 52, 5, 68, 75, 65, 82, 32, 53, 4, 78, 65, 71, 32, 8, 94, 71, 129, - 161, 2, 6, 82, 68, 69, 76, 32, 78, 8, 42, 71, 69, 6, 82, 68, 69, 76, 32, - 68, 6, 46, 67, 228, 233, 1, 2, 78, 89, 251, 116, 83, 2, 251, 133, 4, 73, - 2, 147, 228, 2, 75, 2, 147, 154, 2, 32, 4, 18, 78, 71, 82, 2, 11, 71, 2, - 21, 3, 32, 82, 84, 2, 11, 65, 2, 255, 199, 3, 71, 2, 17, 2, 32, 84, 2, - 187, 241, 2, 83, 94, 68, 7, 76, 69, 84, 84, 69, 82, 32, 145, 2, 5, 83, + 3, 67, 2, 215, 144, 4, 32, 20, 116, 4, 68, 69, 76, 32, 240, 1, 10, 74, + 69, 83, 32, 83, 85, 32, 78, 71, 65, 181, 236, 3, 6, 78, 65, 77, 32, 66, + 67, 16, 52, 5, 68, 75, 65, 82, 32, 53, 4, 78, 65, 71, 32, 8, 94, 71, 181, + 202, 2, 6, 82, 68, 69, 76, 32, 78, 8, 42, 71, 69, 6, 82, 68, 69, 76, 32, + 68, 6, 46, 67, 212, 143, 2, 2, 78, 89, 167, 122, 83, 2, 179, 178, 4, 73, + 2, 187, 145, 3, 75, 2, 199, 195, 2, 32, 4, 18, 78, 71, 82, 2, 11, 71, 2, + 21, 3, 32, 82, 84, 2, 11, 65, 2, 171, 244, 3, 71, 2, 17, 2, 32, 84, 2, + 219, 157, 3, 83, 94, 68, 7, 76, 69, 84, 84, 69, 82, 32, 145, 2, 5, 83, 73, 71, 78, 32, 88, 220, 1, 10, 70, 73, 88, 69, 68, 45, 70, 79, 82, 77, - 234, 185, 3, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, - 80, 2, 90, 254, 68, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, - 187, 2, 65, 6, 11, 32, 6, 166, 128, 4, 82, 2, 87, 3, 89, 6, 38, 73, 50, + 150, 230, 3, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, + 80, 2, 90, 138, 69, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, + 187, 2, 65, 6, 11, 32, 6, 222, 172, 4, 82, 2, 87, 3, 89, 6, 38, 73, 50, 77, 33, 3, 76, 67, 69, 2, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 77, 2, - 11, 67, 2, 45, 2, 72, 85, 2, 25, 4, 32, 84, 83, 65, 2, 11, 32, 2, 231, - 185, 2, 67, 20, 60, 6, 76, 76, 65, 66, 76, 69, 21, 5, 77, 66, 79, 76, 32, - 2, 227, 239, 3, 32, 18, 104, 4, 68, 82, 73, 76, 32, 6, 78, 79, 82, 32, - 66, 85, 130, 1, 80, 93, 7, 82, 68, 79, 32, 82, 74, 69, 2, 11, 32, 2, 211, - 218, 3, 66, 9, 11, 32, 6, 72, 4, 66, 90, 72, 73, 0, 4, 71, 83, 85, 77, 1, - 4, 78, 89, 73, 83, 2, 217, 209, 1, 5, 32, 45, 75, 72, 89, 4, 52, 6, 65, - 68, 77, 65, 32, 71, 21, 3, 72, 85, 82, 2, 187, 182, 2, 68, 2, 251, 88, - 32, 5, 245, 195, 2, 6, 32, 82, 71, 89, 65, 32, 30, 120, 8, 82, 69, 86, - 69, 82, 83, 69, 68, 182, 24, 86, 214, 20, 85, 158, 144, 1, 79, 182, 56, - 73, 186, 198, 1, 69, 223, 61, 65, 4, 213, 245, 1, 2, 32, 73, 2, 25, 4, - 32, 73, 78, 70, 2, 11, 73, 2, 11, 78, 2, 223, 145, 2, 73, 118, 216, 1, 9, - 67, 79, 78, 83, 79, 78, 65, 78, 84, 20, 7, 76, 69, 84, 84, 69, 82, 32, - 236, 6, 20, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, - 32, 76, 65, 66, 73, 37, 8, 83, 69, 80, 65, 82, 65, 84, 79, 2, 211, 187, - 3, 32, 112, 106, 65, 108, 17, 66, 69, 82, 66, 69, 82, 32, 65, 67, 65, 68, + 11, 67, 2, 45, 2, 72, 85, 2, 25, 4, 32, 84, 83, 65, 2, 11, 32, 2, 211, + 227, 2, 67, 20, 60, 6, 76, 76, 65, 66, 76, 69, 21, 5, 77, 66, 79, 76, 32, + 2, 155, 156, 4, 32, 18, 104, 4, 68, 82, 73, 76, 32, 6, 78, 79, 82, 32, + 66, 85, 130, 1, 80, 93, 7, 82, 68, 79, 32, 82, 74, 69, 2, 11, 32, 2, 139, + 135, 4, 66, 9, 11, 32, 6, 72, 4, 66, 90, 72, 73, 0, 4, 71, 83, 85, 77, 1, + 4, 78, 89, 73, 83, 2, 177, 247, 1, 5, 32, 45, 75, 72, 89, 4, 52, 6, 65, + 68, 77, 65, 32, 71, 21, 3, 72, 85, 82, 2, 167, 224, 2, 68, 2, 191, 112, + 32, 5, 237, 238, 2, 6, 32, 82, 71, 89, 65, 32, 30, 116, 8, 82, 69, 86, + 69, 82, 83, 69, 68, 186, 85, 85, 22, 86, 182, 141, 1, 79, 134, 60, 73, + 190, 201, 1, 69, 235, 61, 65, 4, 129, 159, 2, 2, 32, 73, 2, 25, 4, 32, + 73, 78, 70, 2, 11, 73, 2, 11, 78, 2, 151, 187, 2, 73, 118, 216, 1, 9, 67, + 79, 78, 83, 79, 78, 65, 78, 84, 20, 7, 76, 69, 84, 84, 69, 82, 32, 236, + 6, 20, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, + 76, 65, 66, 73, 37, 8, 83, 69, 80, 65, 82, 65, 84, 79, 2, 131, 232, 3, + 32, 112, 106, 65, 108, 17, 66, 69, 82, 66, 69, 82, 32, 65, 67, 65, 68, 69, 77, 89, 32, 89, 65, 30, 84, 223, 1, 89, 4, 84, 6, 89, 69, 82, 32, 89, - 65, 241, 246, 3, 9, 72, 65, 71, 71, 65, 82, 32, 89, 65, 2, 251, 246, 3, - 71, 4, 246, 247, 3, 72, 3, 74, 18, 92, 11, 65, 87, 69, 76, 76, 69, 77, - 69, 84, 32, 89, 33, 8, 85, 65, 82, 69, 71, 32, 89, 65, 2, 11, 65, 2, 239, - 246, 3, 90, 16, 62, 71, 190, 2, 75, 210, 242, 3, 90, 62, 78, 86, 72, 3, - 81, 4, 154, 246, 3, 72, 3, 78, 86, 54, 65, 210, 2, 69, 250, 242, 3, 73, + 65, 173, 163, 4, 9, 72, 65, 71, 71, 65, 82, 32, 89, 65, 2, 183, 163, 4, + 71, 4, 178, 164, 4, 72, 3, 74, 18, 92, 11, 65, 87, 69, 76, 76, 69, 77, + 69, 84, 32, 89, 33, 8, 85, 65, 82, 69, 71, 32, 89, 65, 2, 11, 65, 2, 171, + 163, 4, 90, 16, 62, 71, 190, 2, 75, 142, 159, 4, 90, 62, 78, 86, 72, 3, + 81, 4, 214, 162, 4, 72, 3, 78, 86, 54, 65, 210, 2, 69, 182, 159, 4, 73, 2, 79, 3, 85, 77, 190, 1, 68, 30, 71, 2, 75, 14, 66, 2, 72, 22, 83, 30, - 84, 30, 90, 242, 15, 82, 150, 226, 3, 67, 146, 1, 65, 2, 70, 2, 74, 2, - 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 86, 2, 87, 3, 89, 9, 38, 68, 227, 243, - 3, 72, 7, 11, 72, 5, 223, 243, 3, 72, 7, 202, 243, 3, 72, 3, 83, 7, 174, - 243, 3, 72, 3, 84, 7, 146, 243, 3, 72, 3, 90, 5, 247, 242, 3, 89, 2, 189, - 137, 3, 4, 65, 76, 73, 90, 2, 135, 177, 3, 82, 6, 76, 2, 69, 82, 21, 13, - 72, 84, 32, 84, 82, 73, 70, 79, 76, 73, 65, 84, 69, 5, 211, 180, 3, 32, - 2, 33, 6, 32, 83, 78, 79, 87, 70, 2, 167, 43, 76, 19, 11, 32, 16, 72, 8, - 79, 80, 69, 82, 65, 84, 79, 82, 229, 1, 5, 87, 73, 84, 72, 32, 11, 11, - 32, 8, 38, 65, 97, 5, 87, 73, 84, 72, 32, 4, 25, 4, 66, 79, 86, 69, 4, - 11, 32, 4, 26, 76, 231, 207, 2, 82, 2, 249, 207, 2, 2, 69, 70, 4, 26, 82, - 219, 238, 2, 68, 2, 17, 2, 73, 83, 2, 173, 161, 2, 3, 73, 78, 71, 6, 26, - 68, 183, 196, 1, 82, 4, 11, 79, 4, 207, 213, 1, 84, 6, 22, 82, 155, 98, - 83, 2, 11, 32, 2, 175, 133, 1, 67, 5, 21, 3, 32, 84, 87, 2, 21, 3, 79, - 32, 68, 2, 249, 63, 3, 79, 84, 83, 170, 1, 100, 5, 72, 85, 84, 65, 32, - 156, 9, 11, 79, 78, 73, 65, 78, 32, 83, 73, 71, 78, 32, 223, 210, 2, 69, - 164, 1, 178, 1, 65, 88, 7, 76, 69, 84, 84, 69, 82, 32, 232, 2, 2, 83, 73, - 140, 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 142, 247, 1, 68, - 216, 149, 1, 2, 71, 86, 239, 71, 79, 4, 18, 66, 51, 78, 2, 29, 5, 66, 82, - 69, 86, 73, 2, 203, 16, 65, 2, 215, 214, 3, 74, 94, 202, 1, 65, 38, 68, - 46, 84, 46, 86, 186, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, - 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, - 82, 2, 89, 186, 2, 69, 3, 79, 9, 206, 232, 3, 65, 2, 73, 3, 85, 8, 246, - 160, 3, 68, 254, 68, 72, 187, 2, 65, 8, 202, 160, 3, 84, 254, 68, 72, - 187, 2, 65, 10, 238, 3, 79, 231, 227, 3, 65, 12, 21, 3, 71, 78, 32, 12, - 42, 65, 74, 67, 98, 78, 203, 162, 3, 86, 4, 26, 86, 247, 161, 3, 78, 2, - 21, 3, 65, 71, 82, 2, 255, 238, 1, 65, 2, 11, 65, 2, 11, 78, 2, 25, 4, - 68, 82, 65, 66, 2, 11, 73, 2, 11, 78, 2, 243, 192, 3, 68, 2, 11, 85, 2, - 239, 192, 3, 75, 30, 78, 65, 38, 83, 70, 86, 214, 20, 85, 210, 200, 1, - 73, 206, 134, 2, 69, 3, 79, 6, 214, 228, 3, 65, 2, 73, 3, 85, 4, 25, 4, - 72, 79, 82, 84, 4, 11, 32, 4, 138, 228, 3, 69, 3, 79, 8, 11, 79, 8, 33, - 6, 67, 65, 76, 73, 67, 32, 8, 26, 82, 139, 231, 2, 76, 5, 163, 227, 3, - 82, 4, 128, 174, 3, 8, 67, 65, 80, 73, 84, 65, 76, 32, 227, 24, 69, 138, - 1, 210, 1, 77, 18, 78, 34, 79, 80, 2, 80, 32, 152, 9, 23, 82, 84, 79, 73, + 84, 30, 90, 138, 77, 82, 186, 209, 3, 67, 146, 1, 65, 2, 70, 2, 74, 2, + 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 86, 2, 87, 3, 89, 9, 38, 68, 159, 160, + 4, 72, 7, 11, 72, 5, 155, 160, 4, 72, 7, 134, 160, 4, 72, 3, 83, 7, 234, + 159, 4, 72, 3, 84, 7, 206, 159, 4, 72, 3, 90, 5, 179, 159, 4, 89, 2, 233, + 181, 3, 4, 65, 76, 73, 90, 2, 183, 221, 3, 82, 6, 76, 2, 69, 82, 21, 13, + 72, 84, 32, 84, 82, 73, 70, 79, 76, 73, 65, 84, 69, 5, 131, 225, 3, 32, + 2, 33, 6, 32, 83, 78, 79, 87, 70, 2, 175, 53, 76, 19, 11, 32, 16, 72, 8, + 79, 80, 69, 82, 65, 84, 79, 82, 233, 1, 5, 87, 73, 84, 72, 32, 11, 11, + 32, 8, 38, 65, 101, 5, 87, 73, 84, 72, 32, 4, 25, 4, 66, 79, 86, 69, 4, + 11, 32, 4, 26, 76, 139, 252, 2, 82, 2, 11, 69, 2, 147, 252, 2, 70, 4, 26, + 82, 251, 154, 3, 68, 2, 17, 2, 73, 83, 2, 241, 202, 2, 3, 73, 78, 71, 6, + 26, 68, 143, 234, 1, 82, 4, 11, 79, 4, 175, 252, 1, 84, 6, 22, 82, 239, + 125, 83, 2, 11, 32, 2, 243, 171, 1, 67, 5, 21, 3, 32, 84, 87, 2, 21, 3, + 79, 32, 68, 2, 237, 86, 3, 79, 84, 83, 170, 1, 100, 5, 72, 85, 84, 65, + 32, 184, 6, 11, 79, 78, 73, 65, 78, 32, 83, 73, 71, 78, 32, 227, 129, 3, + 69, 164, 1, 178, 1, 65, 88, 7, 76, 69, 84, 84, 69, 82, 32, 204, 1, 2, 83, + 73, 200, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 162, 162, 2, + 68, 204, 152, 1, 2, 71, 86, 255, 71, 79, 4, 18, 66, 51, 78, 2, 29, 5, 66, + 82, 69, 86, 73, 2, 171, 22, 65, 2, 143, 131, 4, 74, 94, 178, 60, 65, 38, + 68, 114, 84, 46, 86, 186, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, + 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, + 2, 82, 2, 89, 186, 2, 69, 3, 79, 12, 21, 3, 71, 78, 32, 12, 46, 67, 98, + 78, 242, 60, 65, 231, 147, 3, 86, 2, 11, 65, 2, 11, 78, 2, 25, 4, 68, 82, + 65, 66, 2, 11, 73, 2, 11, 78, 2, 139, 239, 3, 68, 2, 11, 85, 2, 135, 239, + 3, 75, 30, 78, 83, 166, 63, 65, 38, 85, 22, 86, 186, 201, 1, 73, 222, + 137, 2, 69, 3, 79, 4, 25, 4, 72, 79, 82, 84, 4, 11, 32, 4, 198, 146, 4, + 69, 3, 79, 4, 144, 221, 3, 8, 67, 65, 80, 73, 84, 65, 76, 32, 239, 24, + 69, 146, 2, 140, 2, 12, 68, 72, 82, 73, 32, 76, 69, 84, 84, 69, 82, 32, + 182, 4, 77, 18, 78, 34, 79, 80, 2, 80, 32, 168, 17, 23, 82, 84, 79, 73, 83, 69, 32, 83, 72, 69, 76, 76, 32, 66, 82, 65, 67, 75, 69, 84, 69, 68, - 32, 130, 4, 84, 198, 246, 1, 73, 169, 67, 5, 75, 89, 79, 32, 84, 2, 239, - 20, 65, 2, 11, 71, 2, 135, 202, 3, 85, 6, 32, 2, 84, 72, 131, 212, 2, 76, - 5, 11, 66, 2, 11, 82, 2, 239, 250, 1, 85, 40, 140, 1, 4, 65, 82, 67, 32, - 162, 2, 67, 44, 2, 72, 65, 218, 1, 80, 126, 76, 22, 82, 138, 1, 83, 38, - 84, 81, 7, 87, 73, 84, 72, 32, 85, 80, 6, 180, 1, 19, 65, 78, 84, 73, 67, - 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, 87, 73, 21, 67, 76, - 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, - 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 80, 2, 143, 218, 1, 76, - 2, 11, 77, 2, 227, 217, 1, 73, 2, 11, 85, 2, 213, 137, 3, 3, 82, 76, 89, - 12, 36, 3, 76, 70, 32, 247, 219, 3, 84, 10, 50, 66, 46, 76, 26, 82, 114, - 83, 143, 214, 1, 73, 2, 11, 76, 2, 145, 140, 2, 3, 65, 67, 75, 2, 11, 69, - 2, 35, 70, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, 80, 2, 33, 6, - 65, 82, 69, 78, 84, 72, 2, 167, 189, 1, 69, 2, 11, 69, 2, 11, 67, 2, 11, - 84, 2, 11, 73, 2, 191, 227, 1, 79, 6, 41, 2, 69, 70, 6, 21, 3, 73, 71, - 72, 6, 17, 2, 84, 32, 6, 42, 67, 249, 133, 3, 4, 72, 65, 76, 70, 4, 26, - 79, 191, 223, 2, 82, 2, 211, 154, 3, 82, 2, 185, 133, 3, 4, 81, 85, 65, - 82, 2, 11, 79, 2, 149, 133, 3, 12, 82, 84, 79, 73, 83, 69, 32, 83, 72, - 69, 76, 76, 2, 29, 5, 87, 65, 82, 68, 83, 2, 17, 2, 32, 65, 2, 141, 218, - 2, 4, 82, 82, 79, 87, 20, 188, 1, 22, 67, 74, 75, 32, 85, 78, 73, 70, 73, - 69, 68, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 161, 2, 19, 76, 65, - 84, 73, 78, 32, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 18, - 40, 2, 52, 69, 34, 53, 54, 54, 87, 55, 4, 202, 1, 48, 203, 172, 2, 56, 4, - 30, 50, 141, 1, 2, 66, 56, 2, 159, 142, 3, 68, 6, 48, 2, 50, 53, 22, 53, - 249, 172, 2, 2, 55, 50, 2, 235, 211, 3, 51, 2, 67, 53, 4, 32, 2, 48, 66, - 21, 2, 54, 68, 2, 167, 211, 3, 57, 2, 147, 211, 3, 55, 2, 11, 82, 2, 163, - 149, 3, 32, 64, 48, 6, 65, 76, 32, 82, 85, 78, 21, 2, 79, 32, 2, 235, - 216, 2, 79, 62, 72, 7, 76, 69, 84, 84, 69, 82, 32, 229, 2, 6, 83, 73, 71, - 78, 32, 82, 60, 202, 1, 66, 102, 73, 22, 78, 138, 144, 1, 69, 198, 71, - 67, 170, 183, 1, 65, 222, 61, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, - 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 79, 3, 85, 12, 52, - 7, 82, 69, 65, 84, 72, 89, 32, 203, 207, 3, 65, 10, 42, 73, 158, 144, 1, - 69, 175, 168, 2, 65, 5, 155, 207, 3, 85, 4, 206, 204, 3, 71, 187, 2, 65, - 2, 17, 2, 73, 83, 2, 21, 3, 73, 78, 71, 2, 11, 32, 2, 135, 165, 2, 84, - 98, 46, 65, 138, 2, 73, 246, 11, 79, 195, 1, 85, 16, 66, 67, 36, 2, 68, - 69, 34, 77, 32, 2, 78, 83, 243, 252, 2, 73, 4, 222, 200, 1, 75, 239, 203, - 1, 84, 2, 153, 16, 4, 32, 77, 65, 82, 5, 11, 32, 2, 175, 170, 2, 67, 4, - 44, 2, 80, 76, 33, 5, 86, 69, 82, 83, 65, 2, 11, 85, 2, 147, 173, 3, 84, - 2, 231, 48, 76, 66, 156, 1, 3, 65, 78, 71, 176, 3, 8, 68, 69, 78, 84, 32, - 69, 77, 66, 20, 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 140, 2, 4, 80, 76, - 69, 32, 255, 244, 2, 67, 16, 48, 2, 76, 69, 245, 1, 5, 85, 76, 65, 82, - 32, 10, 44, 6, 32, 87, 73, 84, 72, 32, 163, 1, 45, 8, 80, 9, 83, 69, 82, - 73, 70, 83, 32, 65, 84, 54, 85, 150, 136, 2, 82, 147, 64, 68, 2, 17, 2, - 32, 66, 2, 145, 184, 3, 3, 79, 84, 84, 2, 173, 47, 2, 78, 68, 2, 177, - 168, 2, 4, 72, 69, 65, 68, 6, 68, 9, 70, 76, 65, 71, 32, 79, 78, 32, 80, - 34, 82, 131, 235, 1, 66, 2, 11, 79, 2, 155, 171, 3, 83, 2, 11, 85, 2, - 199, 137, 3, 76, 2, 231, 180, 3, 76, 16, 66, 69, 34, 72, 34, 76, 22, 77, - 46, 84, 42, 87, 227, 179, 1, 70, 2, 11, 65, 2, 171, 199, 1, 82, 2, 11, - 69, 2, 163, 217, 1, 65, 2, 163, 150, 3, 65, 2, 21, 3, 79, 85, 78, 2, 159, - 154, 2, 84, 2, 17, 2, 72, 85, 2, 183, 161, 1, 78, 4, 198, 142, 2, 65, - 239, 40, 73, 30, 176, 2, 3, 67, 79, 76, 24, 20, 72, 79, 82, 73, 90, 79, - 78, 84, 65, 76, 32, 66, 65, 82, 32, 87, 73, 84, 72, 32, 98, 80, 34, 84, - 20, 13, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 32, 44, 9, 83, - 79, 76, 73, 68, 85, 83, 32, 66, 232, 8, 2, 68, 65, 224, 168, 1, 6, 78, - 69, 83, 84, 69, 68, 163, 11, 73, 2, 145, 24, 2, 79, 78, 4, 34, 68, 33, 4, - 84, 82, 73, 80, 2, 17, 2, 79, 85, 2, 11, 66, 2, 133, 145, 3, 2, 76, 69, - 4, 218, 190, 1, 76, 143, 107, 82, 2, 171, 171, 1, 73, 8, 42, 66, 74, 68, - 198, 161, 1, 82, 111, 87, 2, 29, 5, 73, 78, 65, 82, 89, 2, 21, 3, 32, 82, - 69, 2, 251, 19, 76, 2, 25, 4, 69, 76, 73, 77, 2, 183, 137, 2, 73, 10, 24, - 2, 76, 76, 35, 80, 5, 229, 188, 1, 3, 69, 89, 66, 6, 44, 5, 73, 67, 65, - 76, 32, 191, 173, 3, 72, 4, 18, 68, 47, 70, 2, 11, 82, 2, 11, 73, 2, 191, - 187, 3, 78, 2, 167, 217, 1, 73, 6, 18, 69, 79, 77, 5, 205, 153, 3, 14, - 32, 76, 73, 71, 72, 84, 32, 77, 79, 79, 78, 32, 65, 82, 2, 195, 136, 3, - 80, 56, 80, 3, 71, 82, 73, 22, 82, 148, 165, 2, 5, 77, 66, 76, 69, 82, - 151, 150, 1, 76, 2, 255, 181, 2, 75, 50, 50, 75, 84, 4, 78, 69, 68, 32, - 171, 154, 2, 84, 4, 48, 6, 73, 83, 72, 32, 76, 73, 167, 170, 3, 69, 2, - 11, 82, 2, 247, 180, 2, 65, 44, 238, 1, 65, 80, 6, 66, 76, 65, 67, 75, - 32, 40, 7, 87, 72, 73, 84, 69, 32, 83, 22, 67, 50, 68, 96, 2, 78, 79, 32, - 2, 79, 75, 30, 83, 185, 146, 3, 21, 71, 82, 69, 69, 75, 32, 83, 77, 65, - 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 73, 79, 4, 26, 77, 147, 152, 2, - 78, 2, 17, 2, 80, 69, 2, 11, 82, 2, 179, 171, 2, 83, 4, 18, 80, 23, 83, - 2, 223, 129, 2, 69, 2, 207, 131, 2, 72, 4, 144, 3, 5, 65, 80, 73, 84, 65, - 167, 106, 79, 6, 30, 65, 33, 3, 73, 71, 73, 2, 11, 71, 2, 147, 250, 2, - 71, 4, 129, 203, 1, 3, 84, 32, 84, 6, 162, 2, 82, 183, 174, 2, 84, 2, - 177, 209, 1, 2, 32, 72, 14, 128, 1, 18, 65, 78, 83, 45, 83, 69, 82, 73, - 70, 32, 67, 65, 80, 73, 84, 65, 76, 32, 38, 69, 32, 3, 77, 65, 76, 33, 2, - 79, 85, 6, 254, 181, 3, 71, 2, 76, 3, 89, 2, 11, 77, 2, 203, 229, 2, 73, - 2, 11, 76, 2, 163, 180, 2, 32, 4, 21, 3, 84, 72, 32, 4, 32, 2, 69, 65, 1, - 2, 87, 69, 2, 53, 11, 83, 84, 32, 80, 79, 73, 78, 84, 73, 78, 71, 2, 17, - 2, 32, 76, 2, 251, 178, 2, 69, 36, 58, 69, 52, 8, 73, 83, 84, 69, 68, 32, - 82, 73, 63, 79, 2, 17, 2, 76, 86, 2, 209, 159, 1, 3, 69, 32, 80, 2, 17, - 2, 71, 72, 2, 153, 93, 6, 84, 87, 65, 82, 68, 83, 32, 30, 32, 249, 9, 2, - 45, 69, 30, 200, 2, 24, 65, 83, 84, 69, 82, 73, 83, 75, 83, 32, 65, 76, - 73, 71, 78, 69, 68, 32, 86, 69, 82, 84, 73, 67, 34, 66, 86, 67, 116, 3, - 68, 79, 84, 226, 1, 72, 44, 14, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, - 78, 71, 32, 76, 92, 6, 74, 79, 73, 78, 69, 68, 64, 8, 76, 79, 71, 73, 67, - 65, 76, 32, 98, 77, 0, 3, 87, 79, 77, 119, 83, 2, 11, 65, 2, 171, 140, 3, - 76, 2, 29, 5, 85, 84, 84, 79, 78, 2, 17, 2, 32, 77, 2, 11, 79, 2, 147, - 166, 2, 85, 2, 93, 21, 79, 78, 83, 69, 67, 85, 84, 73, 86, 69, 32, 69, - 81, 85, 65, 76, 83, 32, 83, 73, 71, 2, 195, 240, 2, 78, 6, 18, 32, 55, - 83, 4, 22, 76, 131, 1, 80, 2, 225, 137, 1, 2, 69, 65, 2, 53, 11, 32, 79, - 86, 69, 82, 32, 79, 78, 69, 32, 68, 2, 11, 79, 2, 11, 84, 2, 17, 2, 32, - 80, 2, 29, 5, 85, 78, 67, 84, 85, 2, 239, 219, 2, 65, 2, 11, 69, 2, 11, - 65, 2, 243, 151, 2, 82, 4, 17, 2, 79, 71, 4, 25, 4, 73, 67, 65, 76, 4, - 11, 32, 4, 214, 157, 2, 65, 147, 85, 79, 2, 17, 2, 32, 83, 2, 21, 3, 81, - 85, 65, 2, 139, 151, 2, 82, 4, 30, 79, 13, 3, 65, 78, 68, 2, 11, 82, 2, - 11, 32, 2, 11, 79, 2, 11, 80, 2, 183, 17, 69, 2, 11, 69, 2, 45, 9, 78, - 32, 72, 79, 76, 68, 73, 78, 71, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 207, - 204, 1, 78, 2, 49, 10, 80, 69, 69, 67, 72, 32, 66, 85, 66, 66, 2, 239, - 148, 2, 76, 2, 11, 77, 2, 133, 195, 1, 2, 32, 68, 140, 3, 140, 1, 8, 71, - 65, 82, 73, 84, 73, 67, 32, 148, 6, 7, 77, 66, 82, 69, 76, 76, 65, 166, - 1, 78, 154, 8, 80, 174, 148, 1, 82, 135, 131, 2, 83, 62, 48, 7, 76, 69, - 84, 84, 69, 82, 32, 159, 5, 87, 60, 206, 1, 65, 28, 2, 81, 79, 22, 66, - 22, 68, 50, 71, 44, 2, 72, 79, 22, 75, 34, 76, 32, 2, 82, 65, 22, 83, 74, - 84, 74, 89, 22, 90, 186, 161, 2, 78, 178, 91, 80, 246, 5, 87, 202, 12, - 77, 174, 18, 73, 3, 85, 4, 26, 76, 143, 213, 2, 73, 2, 167, 162, 3, 80, - 2, 255, 255, 2, 69, 4, 26, 69, 247, 154, 2, 72, 2, 207, 255, 2, 76, 4, - 214, 248, 1, 72, 153, 137, 1, 2, 65, 77, 5, 163, 161, 3, 84, 4, 186, 162, - 2, 65, 215, 126, 72, 2, 11, 65, 2, 151, 254, 2, 77, 2, 203, 171, 1, 83, - 8, 38, 65, 146, 163, 2, 72, 215, 90, 83, 4, 246, 139, 3, 68, 239, 13, 77, - 6, 38, 72, 218, 133, 3, 69, 175, 28, 79, 2, 11, 65, 2, 179, 231, 1, 78, - 2, 207, 219, 2, 79, 4, 130, 253, 2, 69, 207, 36, 85, 2, 21, 3, 79, 82, - 68, 2, 25, 4, 32, 68, 73, 86, 2, 139, 125, 73, 7, 11, 32, 4, 84, 4, 79, - 78, 32, 71, 45, 13, 87, 73, 84, 72, 32, 82, 65, 73, 78, 32, 68, 82, 79, - 2, 11, 82, 2, 11, 79, 2, 143, 146, 2, 85, 2, 139, 226, 2, 80, 32, 160, 1, - 3, 65, 77, 85, 20, 7, 67, 69, 82, 84, 65, 73, 78, 34, 68, 62, 73, 241, 5, - 18, 77, 65, 82, 82, 73, 69, 68, 32, 80, 65, 82, 84, 78, 69, 82, 83, 72, - 73, 2, 227, 141, 2, 83, 2, 11, 84, 2, 155, 151, 2, 89, 4, 26, 69, 199, - 138, 2, 79, 2, 11, 82, 2, 151, 130, 3, 84, 22, 60, 2, 79, 78, 230, 3, 84, - 106, 86, 201, 43, 3, 67, 79, 82, 15, 11, 32, 12, 160, 1, 6, 65, 66, 79, - 86, 69, 32, 108, 22, 66, 69, 83, 73, 68, 69, 32, 65, 78, 68, 32, 74, 79, - 73, 78, 69, 68, 32, 87, 73, 84, 72, 41, 5, 87, 73, 84, 72, 32, 4, 52, 9, - 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 73, 2, 17, 2, 32, 73, 2, 229, 39, - 4, 78, 84, 69, 82, 2, 17, 2, 32, 85, 2, 139, 202, 2, 78, 6, 66, 77, 58, - 79, 217, 224, 2, 8, 76, 79, 71, 73, 67, 65, 76, 32, 2, 11, 73, 2, 11, 78, - 2, 11, 85, 2, 223, 146, 2, 83, 2, 11, 86, 2, 229, 73, 2, 69, 82, 4, 18, - 32, 67, 69, 2, 11, 83, 2, 17, 2, 69, 80, 2, 11, 65, 2, 219, 223, 2, 82, - 2, 171, 133, 2, 68, 2, 57, 12, 69, 82, 83, 65, 76, 32, 82, 69, 67, 89, - 67, 76, 2, 17, 2, 73, 78, 2, 199, 132, 2, 71, 2, 179, 132, 2, 80, 164, 2, - 170, 1, 32, 128, 8, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 248, 3, - 4, 80, 69, 82, 32, 168, 29, 8, 83, 73, 68, 69, 45, 68, 79, 87, 21, 6, 87, - 65, 82, 68, 83, 32, 38, 140, 1, 5, 65, 82, 82, 79, 87, 240, 1, 5, 66, 65, - 82, 66, 32, 228, 1, 5, 68, 79, 87, 78, 32, 210, 1, 70, 30, 82, 89, 4, 84, - 65, 67, 75, 8, 60, 7, 32, 84, 72, 82, 79, 85, 71, 21, 4, 72, 69, 65, 68, - 2, 143, 197, 1, 72, 7, 11, 32, 4, 112, 22, 66, 69, 84, 87, 69, 69, 78, - 32, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 159, 134, 2, - 73, 2, 181, 254, 1, 2, 32, 66, 8, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, - 72, 4, 57, 12, 84, 32, 68, 79, 87, 78, 32, 66, 65, 82, 66, 32, 4, 44, 3, - 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 11, 84, 2, 25, 4, 32, 72, 65, 82, 2, - 11, 80, 2, 207, 193, 2, 79, 14, 76, 5, 65, 82, 82, 79, 87, 62, 66, 38, - 68, 18, 83, 202, 43, 87, 215, 9, 84, 5, 37, 7, 32, 87, 73, 84, 72, 32, - 66, 2, 175, 135, 2, 65, 2, 209, 245, 1, 4, 76, 65, 67, 75, 2, 147, 44, - 79, 2, 131, 50, 65, 2, 11, 73, 2, 155, 98, 83, 2, 53, 11, 73, 71, 72, 84, - 32, 68, 73, 65, 71, 79, 78, 2, 225, 113, 4, 65, 76, 32, 69, 5, 29, 5, 32, - 87, 73, 84, 72, 2, 33, 6, 32, 67, 73, 82, 67, 76, 2, 159, 145, 2, 69, 14, - 96, 8, 77, 73, 76, 73, 84, 65, 82, 89, 20, 6, 83, 77, 65, 76, 76, 32, 22, - 65, 66, 82, 35, 84, 2, 37, 2, 32, 65, 4, 18, 65, 67, 82, 2, 11, 73, 2, - 11, 82, 2, 17, 2, 80, 76, 2, 179, 243, 1, 65, 2, 11, 69, 2, 199, 232, 1, - 68, 4, 37, 7, 82, 73, 65, 78, 71, 76, 69, 4, 11, 32, 4, 11, 87, 4, 25, 4, - 73, 84, 72, 32, 4, 18, 76, 27, 82, 2, 11, 69, 2, 35, 70, 2, 21, 3, 73, - 71, 72, 2, 11, 84, 2, 17, 2, 32, 72, 2, 21, 3, 65, 76, 70, 2, 17, 2, 32, - 66, 2, 11, 76, 2, 207, 219, 1, 65, 110, 172, 1, 4, 65, 78, 68, 32, 186, - 2, 66, 74, 70, 36, 5, 72, 65, 76, 70, 32, 148, 3, 5, 76, 69, 70, 84, 32, - 226, 5, 79, 60, 6, 82, 73, 71, 72, 84, 32, 218, 12, 83, 63, 84, 8, 34, - 76, 53, 4, 82, 73, 71, 72, 6, 48, 2, 69, 70, 149, 1, 5, 79, 87, 69, 82, - 32, 2, 53, 11, 84, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 2, 217, 25, - 19, 32, 84, 82, 73, 65, 78, 71, 85, 76, 65, 82, 32, 84, 72, 82, 69, 69, - 32, 81, 4, 28, 2, 84, 82, 179, 29, 79, 2, 205, 3, 7, 73, 65, 78, 71, 85, - 76, 65, 2, 17, 2, 76, 65, 2, 17, 2, 68, 69, 2, 177, 208, 1, 3, 32, 83, - 67, 2, 17, 2, 73, 86, 2, 191, 22, 69, 12, 96, 5, 66, 76, 79, 67, 75, 108, - 8, 73, 78, 86, 69, 82, 83, 69, 32, 254, 19, 77, 227, 160, 1, 67, 5, 209, - 20, 23, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 72, 65, 76, 70, 32, - 73, 78, 86, 69, 82, 83, 69, 4, 100, 21, 77, 69, 68, 73, 85, 77, 32, 83, - 72, 65, 68, 69, 32, 65, 78, 68, 32, 76, 79, 87, 69, 51, 87, 2, 11, 82, 2, - 141, 26, 5, 32, 72, 65, 76, 70, 2, 21, 3, 72, 73, 84, 2, 147, 179, 1, 69, - 34, 106, 66, 238, 3, 67, 54, 84, 176, 9, 13, 79, 82, 32, 76, 79, 87, 69, - 82, 32, 82, 73, 71, 72, 143, 1, 81, 22, 65, 14, 76, 79, 67, 75, 32, 68, - 73, 65, 71, 79, 78, 65, 76, 32, 22, 144, 1, 6, 76, 79, 87, 69, 82, 32, - 173, 6, 24, 85, 80, 80, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, - 70, 84, 32, 84, 79, 32, 85, 80, 80, 18, 176, 1, 10, 67, 69, 78, 84, 82, - 69, 32, 84, 79, 32, 36, 8, 76, 69, 70, 84, 32, 84, 79, 32, 169, 8, 18, - 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 85, 80, 80, - 6, 70, 76, 193, 6, 3, 85, 80, 80, 6, 34, 76, 153, 7, 3, 85, 80, 80, 2, - 217, 7, 2, 79, 87, 2, 29, 5, 79, 82, 78, 69, 82, 2, 139, 218, 1, 32, 6, - 22, 79, 187, 12, 82, 2, 129, 12, 11, 32, 76, 79, 87, 69, 82, 32, 82, 73, - 71, 72, 4, 11, 78, 4, 17, 2, 69, 32, 4, 150, 15, 81, 139, 4, 69, 40, 128, - 1, 2, 66, 76, 186, 6, 68, 100, 12, 79, 82, 32, 76, 79, 87, 69, 82, 32, - 76, 69, 70, 106, 80, 38, 81, 104, 2, 83, 72, 51, 84, 22, 61, 13, 79, 67, - 75, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 22, 140, 1, 24, 76, 79, 87, - 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, - 76, 79, 87, 57, 6, 85, 80, 80, 69, 82, 32, 4, 21, 3, 69, 82, 32, 4, 246, - 3, 67, 211, 202, 2, 82, 18, 176, 1, 10, 67, 69, 78, 84, 82, 69, 32, 84, - 79, 32, 92, 8, 76, 69, 70, 84, 32, 84, 79, 32, 141, 1, 18, 77, 73, 68, - 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 76, 79, 87, 6, 32, 3, 76, - 79, 87, 139, 1, 85, 4, 21, 3, 69, 82, 32, 4, 142, 2, 77, 171, 202, 2, 82, - 6, 28, 3, 76, 79, 87, 51, 85, 4, 21, 3, 69, 82, 32, 4, 142, 1, 67, 43, - 77, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 117, 2, 32, 77, 6, 21, 3, 69, - 82, 32, 6, 34, 67, 42, 77, 171, 202, 2, 82, 2, 11, 69, 2, 241, 212, 1, 2, - 78, 84, 2, 25, 4, 73, 68, 68, 76, 2, 203, 166, 1, 69, 2, 65, 14, 82, 79, - 80, 45, 83, 72, 65, 68, 79, 87, 69, 68, 32, 87, 2, 225, 181, 1, 3, 72, - 73, 84, 2, 69, 15, 84, 32, 67, 85, 82, 76, 89, 32, 66, 82, 65, 67, 75, - 69, 84, 2, 11, 32, 2, 179, 162, 2, 83, 2, 11, 69, 2, 249, 69, 2, 78, 67, - 2, 81, 18, 85, 65, 68, 82, 65, 78, 84, 32, 67, 73, 82, 67, 85, 76, 65, - 82, 32, 65, 2, 163, 203, 1, 82, 4, 245, 97, 8, 65, 68, 79, 87, 69, 68, - 32, 87, 6, 18, 79, 107, 82, 2, 49, 10, 32, 76, 79, 87, 69, 82, 32, 76, - 69, 70, 2, 11, 84, 2, 11, 32, 2, 11, 70, 2, 163, 108, 73, 4, 25, 4, 73, - 65, 78, 71, 4, 40, 4, 85, 76, 65, 82, 171, 217, 2, 76, 2, 17, 2, 32, 77, - 2, 41, 8, 69, 68, 73, 85, 77, 32, 83, 72, 2, 199, 98, 65, 2, 17, 2, 69, - 86, 2, 17, 2, 69, 78, 2, 117, 2, 32, 69, 6, 52, 2, 72, 82, 129, 1, 6, 82, - 73, 65, 78, 71, 85, 4, 21, 3, 69, 69, 32, 4, 18, 69, 35, 81, 2, 65, 5, - 73, 71, 72, 84, 72, 2, 33, 6, 85, 65, 82, 84, 69, 82, 2, 231, 4, 83, 2, - 45, 9, 76, 65, 82, 32, 79, 78, 69, 32, 81, 2, 165, 4, 6, 85, 65, 82, 84, - 69, 82, 2, 203, 175, 2, 78, 128, 1, 146, 1, 65, 174, 6, 66, 154, 1, 68, - 50, 70, 82, 72, 146, 4, 67, 46, 81, 42, 82, 22, 83, 102, 84, 142, 7, 80, - 173, 3, 6, 87, 72, 73, 84, 69, 32, 32, 34, 78, 33, 4, 82, 82, 79, 87, 2, - 11, 67, 2, 207, 166, 2, 79, 31, 11, 32, 28, 134, 1, 65, 180, 1, 14, 76, - 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 68, 32, 5, 87, 73, 84, - 72, 32, 214, 8, 70, 175, 5, 84, 2, 41, 8, 78, 68, 32, 82, 73, 71, 72, 84, - 2, 17, 2, 32, 79, 2, 25, 4, 78, 69, 32, 69, 2, 29, 5, 73, 71, 72, 84, 72, - 2, 11, 32, 2, 11, 66, 2, 11, 76, 2, 239, 185, 1, 79, 2, 233, 200, 1, 3, - 79, 87, 78, 20, 74, 68, 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 246, 12, - 84, 139, 59, 72, 2, 21, 3, 79, 85, 66, 2, 11, 76, 2, 131, 183, 2, 69, 2, - 11, 65, 2, 233, 23, 3, 82, 71, 69, 2, 205, 23, 5, 69, 68, 73, 85, 77, 2, - 25, 4, 79, 84, 67, 72, 2, 11, 69, 2, 247, 56, 68, 4, 11, 77, 4, 25, 4, - 65, 76, 76, 32, 4, 22, 69, 203, 22, 84, 2, 237, 22, 10, 81, 85, 73, 76, - 65, 84, 69, 82, 65, 76, 4, 11, 76, 4, 25, 4, 65, 67, 75, 32, 4, 44, 5, - 67, 73, 82, 67, 76, 143, 202, 1, 65, 2, 25, 4, 69, 68, 32, 87, 2, 11, 72, - 2, 229, 14, 2, 73, 84, 4, 22, 79, 187, 13, 65, 2, 169, 14, 2, 85, 66, 2, - 11, 73, 2, 37, 7, 78, 71, 69, 82, 45, 80, 79, 2, 205, 200, 1, 2, 83, 84, - 20, 88, 17, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, - 66, 32, 255, 2, 69, 16, 56, 4, 76, 69, 70, 84, 245, 1, 5, 82, 73, 71, 72, - 84, 10, 22, 32, 231, 9, 87, 8, 60, 7, 66, 69, 83, 73, 68, 69, 32, 206, 1, - 70, 175, 5, 84, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 193, 146, 1, - 23, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, - 72, 32, 66, 65, 82, 66, 6, 22, 32, 243, 7, 87, 4, 22, 70, 175, 5, 84, 2, - 229, 15, 3, 82, 79, 77, 4, 25, 4, 65, 86, 89, 32, 4, 26, 67, 191, 196, 1, - 65, 2, 185, 10, 7, 79, 77, 80, 82, 69, 83, 83, 2, 133, 9, 6, 85, 65, 68, - 82, 85, 80, 2, 195, 139, 2, 79, 4, 30, 65, 53, 3, 81, 85, 65, 2, 153, - 195, 1, 8, 78, 83, 45, 83, 69, 82, 73, 70, 2, 255, 8, 82, 36, 36, 2, 82, - 73, 225, 7, 2, 87, 79, 30, 40, 5, 65, 78, 71, 76, 69, 151, 7, 80, 28, 52, - 8, 45, 72, 69, 65, 68, 69, 68, 32, 199, 13, 32, 26, 48, 5, 65, 82, 82, - 79, 87, 170, 5, 68, 39, 80, 23, 11, 32, 20, 92, 17, 76, 69, 70, 84, 87, - 65, 82, 68, 83, 32, 79, 70, 32, 68, 79, 87, 78, 98, 84, 19, 87, 2, 37, 7, - 87, 65, 82, 68, 83, 32, 84, 2, 37, 7, 82, 73, 65, 78, 71, 76, 69, 2, 215, - 5, 45, 2, 187, 10, 79, 16, 25, 4, 73, 84, 72, 32, 16, 114, 66, 28, 6, 76, - 79, 78, 71, 32, 84, 130, 1, 77, 34, 78, 34, 86, 34, 72, 149, 57, 6, 68, - 79, 85, 66, 76, 69, 2, 149, 2, 3, 79, 76, 68, 4, 17, 2, 73, 80, 4, 11, - 32, 4, 34, 76, 21, 4, 82, 73, 71, 72, 2, 17, 2, 69, 70, 2, 11, 84, 2, 11, - 87, 2, 135, 123, 65, 2, 121, 5, 69, 68, 73, 85, 77, 2, 89, 5, 65, 82, 82, - 79, 87, 2, 29, 5, 69, 82, 89, 32, 72, 2, 25, 4, 69, 65, 86, 89, 2, 237, - 171, 2, 4, 32, 83, 72, 65, 2, 11, 65, 2, 249, 1, 2, 83, 72, 2, 11, 65, 2, - 25, 4, 73, 82, 69, 68, 2, 29, 5, 32, 65, 82, 82, 79, 2, 239, 151, 2, 87, - 2, 11, 76, 2, 227, 186, 1, 69, 6, 74, 32, 61, 14, 45, 72, 69, 65, 68, 69, - 68, 32, 65, 82, 82, 79, 87, 32, 2, 25, 4, 72, 69, 65, 68, 2, 11, 69, 2, - 219, 185, 1, 68, 4, 42, 87, 129, 132, 1, 4, 70, 82, 79, 77, 2, 33, 6, 73, - 84, 72, 32, 84, 82, 2, 61, 13, 73, 65, 78, 71, 76, 69, 32, 65, 82, 82, - 79, 87, 72, 2, 169, 118, 2, 69, 65, 18, 88, 5, 65, 82, 82, 79, 87, 133, - 4, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 15, 11, 32, 12, - 104, 8, 70, 82, 79, 77, 32, 66, 65, 82, 40, 11, 79, 78, 32, 80, 69, 68, - 69, 83, 84, 65, 76, 183, 1, 87, 5, 93, 7, 32, 87, 73, 84, 72, 32, 72, 7, - 33, 6, 32, 87, 73, 84, 72, 32, 4, 18, 72, 43, 86, 2, 69, 7, 79, 82, 73, - 90, 79, 78, 84, 2, 29, 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, 76, 2, 11, - 32, 2, 151, 173, 1, 66, 2, 29, 5, 73, 84, 72, 73, 78, 2, 17, 2, 32, 84, - 2, 37, 7, 82, 73, 65, 78, 71, 76, 69, 2, 11, 32, 2, 11, 65, 2, 25, 4, 82, - 82, 79, 87, 2, 11, 72, 2, 239, 135, 2, 69, 5, 45, 9, 32, 79, 78, 32, 80, - 69, 68, 69, 83, 2, 163, 196, 1, 84, 226, 15, 86, 65, 254, 16, 83, 174, 3, - 69, 162, 42, 73, 178, 8, 79, 134, 2, 84, 21, 2, 85, 76, 218, 8, 116, 2, - 73, 32, 128, 16, 17, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 173, 42, 2, 77, 80, 216, 4, 54, 67, 34, 70, 82, 81, 40, - 2, 83, 89, 243, 91, 68, 2, 11, 79, 2, 135, 170, 2, 77, 2, 11, 85, 2, 11, - 76, 2, 11, 76, 2, 11, 32, 2, 11, 83, 2, 195, 209, 1, 84, 2, 17, 2, 85, - 69, 2, 135, 225, 1, 83, 190, 4, 68, 7, 76, 76, 65, 66, 76, 69, 32, 197, - 11, 5, 77, 66, 79, 76, 32, 164, 4, 214, 1, 68, 70, 66, 2, 83, 2, 84, 2, - 90, 70, 71, 122, 72, 82, 75, 134, 1, 76, 130, 1, 77, 86, 78, 134, 3, 67, - 2, 70, 2, 74, 2, 80, 2, 82, 2, 86, 2, 89, 78, 87, 50, 69, 34, 79, 158, - 70, 65, 2, 73, 3, 85, 42, 66, 72, 162, 8, 79, 238, 254, 1, 69, 150, 64, - 65, 2, 73, 3, 85, 28, 230, 7, 72, 58, 79, 238, 254, 1, 69, 150, 64, 65, - 2, 73, 3, 85, 34, 62, 66, 202, 6, 69, 86, 79, 130, 191, 2, 65, 2, 73, 3, - 85, 18, 106, 79, 222, 5, 69, 214, 191, 2, 65, 2, 73, 3, 85, 24, 50, 79, - 222, 5, 69, 214, 71, 65, 2, 73, 3, 85, 7, 174, 197, 2, 78, 3, 79, 34, 70, - 80, 206, 5, 79, 130, 71, 65, 238, 183, 1, 69, 150, 64, 73, 3, 85, 18, - 246, 4, 69, 86, 79, 130, 71, 65, 130, 248, 1, 73, 3, 85, 16, 54, 69, 218, - 4, 79, 130, 191, 2, 65, 2, 73, 3, 85, 7, 26, 78, 191, 195, 2, 69, 2, 21, - 3, 71, 84, 72, 2, 183, 133, 2, 69, 42, 214, 3, 66, 0, 2, 71, 66, 58, 79, - 238, 254, 1, 69, 150, 64, 65, 2, 73, 3, 85, 90, 90, 68, 174, 1, 71, 126, - 74, 2, 89, 58, 79, 238, 254, 1, 69, 150, 64, 65, 2, 73, 3, 85, 24, 54, - 79, 150, 129, 2, 69, 150, 64, 65, 2, 73, 3, 85, 15, 36, 3, 76, 69, 32, - 131, 193, 2, 79, 10, 54, 83, 214, 161, 2, 68, 190, 28, 70, 2, 75, 3, 77, - 2, 211, 161, 2, 79, 25, 42, 71, 182, 240, 1, 65, 2, 69, 3, 79, 16, 50, - 69, 86, 79, 130, 191, 2, 65, 2, 73, 3, 85, 7, 210, 191, 2, 69, 3, 78, 14, - 54, 79, 238, 254, 1, 69, 150, 64, 65, 2, 73, 3, 85, 5, 255, 190, 2, 79, - 28, 46, 69, 34, 79, 158, 70, 65, 2, 73, 3, 85, 9, 186, 70, 69, 131, 248, - 1, 78, 9, 154, 70, 79, 131, 248, 1, 78, 26, 66, 68, 62, 70, 30, 74, 22, - 75, 50, 78, 22, 84, 199, 227, 1, 66, 6, 26, 79, 179, 236, 1, 65, 4, 174, - 236, 1, 79, 251, 49, 45, 4, 74, 69, 251, 185, 2, 65, 2, 243, 235, 1, 79, - 4, 26, 69, 199, 235, 1, 85, 2, 195, 235, 1, 69, 2, 195, 168, 2, 73, 6, - 154, 235, 1, 73, 2, 79, 183, 78, 65, 128, 4, 74, 49, 94, 50, 98, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 223, 1, 182, 1, 48, 2, 49, 2, 50, - 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 137, 1, 90, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 94, 53, 250, 184, 2, 54, 2, 55, 2, 56, 3, 57, 23, - 210, 185, 2, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, - 3, 57, 17, 246, 184, 2, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, - 180, 1, 116, 4, 68, 73, 67, 32, 222, 19, 82, 208, 148, 1, 13, 67, 84, 79, - 82, 32, 79, 82, 32, 67, 82, 79, 83, 83, 195, 106, 83, 86, 60, 5, 83, 73, - 71, 78, 32, 153, 10, 5, 84, 79, 78, 69, 32, 48, 218, 2, 65, 216, 1, 17, - 68, 79, 85, 66, 76, 69, 32, 65, 78, 85, 83, 86, 65, 82, 65, 32, 65, 98, - 74, 52, 6, 78, 73, 72, 83, 72, 86, 22, 82, 240, 1, 8, 72, 69, 88, 73, 70, - 79, 82, 77, 22, 76, 52, 4, 84, 73, 82, 89, 22, 85, 60, 8, 86, 73, 83, 65, - 82, 71, 65, 32, 237, 6, 17, 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, - 77, 73, 68, 76, 73, 78, 14, 76, 8, 78, 85, 83, 86, 65, 82, 65, 32, 212, - 1, 3, 84, 73, 75, 151, 2, 82, 10, 134, 1, 65, 24, 4, 66, 65, 72, 73, 24, - 9, 85, 66, 72, 65, 89, 65, 84, 79, 32, 201, 4, 10, 86, 65, 77, 65, 71, - 79, 77, 85, 75, 72, 2, 21, 3, 78, 84, 65, 2, 21, 3, 82, 71, 79, 2, 11, - 77, 2, 163, 9, 85, 2, 151, 239, 1, 82, 2, 33, 6, 73, 72, 86, 65, 77, 85, - 2, 151, 3, 76, 2, 235, 174, 2, 65, 8, 144, 1, 15, 69, 86, 69, 82, 83, 69, - 68, 32, 86, 73, 83, 65, 82, 71, 65, 36, 9, 79, 84, 65, 84, 69, 68, 32, - 65, 82, 57, 5, 84, 72, 65, 78, 71, 4, 11, 32, 4, 218, 6, 65, 23, 85, 2, - 25, 4, 68, 72, 65, 86, 2, 245, 236, 1, 2, 73, 83, 2, 17, 2, 32, 76, 2, - 21, 3, 79, 78, 71, 2, 145, 234, 1, 2, 32, 65, 2, 203, 171, 2, 65, 2, 37, - 7, 80, 65, 68, 72, 77, 65, 78, 2, 203, 171, 2, 73, 10, 40, 3, 65, 78, 85, - 2, 85, 179, 9, 83, 4, 25, 4, 68, 65, 84, 84, 4, 11, 65, 5, 25, 4, 32, 87, - 73, 84, 2, 11, 72, 2, 11, 32, 2, 11, 84, 2, 11, 65, 2, 215, 163, 1, 73, - 38, 128, 2, 5, 67, 65, 78, 68, 82, 16, 2, 68, 79, 96, 2, 75, 65, 136, 1, - 4, 80, 82, 69, 78, 16, 2, 82, 73, 106, 84, 164, 1, 11, 89, 65, 74, 85, - 82, 86, 69, 68, 73, 67, 32, 180, 1, 12, 65, 84, 72, 65, 82, 86, 65, 86, - 69, 68, 73, 67, 213, 224, 1, 2, 83, 72, 4, 251, 17, 65, 6, 40, 5, 85, 66, - 76, 69, 32, 191, 3, 84, 4, 22, 82, 211, 5, 83, 2, 11, 73, 2, 251, 1, 78, - 4, 56, 3, 82, 83, 72, 17, 7, 84, 72, 65, 75, 65, 32, 65, 2, 235, 110, 65, - 2, 17, 2, 78, 85, 2, 17, 2, 68, 65, 2, 171, 132, 2, 84, 2, 167, 49, 75, - 4, 82, 78, 237, 2, 15, 71, 86, 69, 68, 73, 67, 32, 75, 65, 83, 72, 77, - 73, 82, 73, 2, 139, 171, 1, 71, 6, 42, 72, 24, 4, 82, 73, 80, 76, 19, 87, - 2, 49, 3, 82, 69, 69, 2, 219, 2, 69, 2, 11, 79, 2, 25, 4, 32, 68, 79, 84, - 2, 11, 83, 2, 11, 32, 2, 159, 14, 66, 8, 176, 1, 10, 65, 71, 71, 82, 65, - 86, 65, 84, 69, 68, 22, 73, 105, 27, 75, 65, 84, 72, 65, 75, 65, 32, 73, - 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 83, 86, 65, 82, 73, 84, 65, - 2, 17, 2, 32, 73, 2, 49, 10, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 2, - 17, 2, 32, 83, 2, 221, 255, 1, 3, 86, 65, 82, 5, 37, 7, 32, 83, 67, 72, - 82, 79, 69, 2, 143, 230, 1, 68, 90, 62, 83, 16, 6, 84, 73, 67, 65, 76, - 32, 221, 13, 2, 89, 32, 2, 171, 84, 73, 62, 208, 2, 4, 66, 65, 82, 32, - 166, 3, 69, 62, 70, 36, 11, 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 67, - 48, 12, 75, 65, 78, 65, 32, 82, 69, 80, 69, 65, 84, 32, 252, 1, 4, 76, - 73, 78, 69, 238, 1, 77, 88, 17, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, - 32, 66, 76, 79, 67, 75, 45, 62, 84, 192, 73, 3, 83, 73, 88, 241, 32, 5, - 90, 73, 71, 90, 65, 8, 108, 6, 66, 69, 83, 73, 68, 69, 60, 8, 68, 79, 85, - 66, 76, 69, 32, 76, 20, 4, 84, 82, 73, 80, 139, 1, 87, 2, 17, 2, 32, 82, - 2, 21, 3, 73, 71, 72, 2, 187, 123, 84, 2, 69, 2, 69, 70, 2, 25, 4, 76, - 69, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 29, 5, 32, 84, 85, 82, - 78, 2, 11, 83, 2, 11, 84, 2, 219, 124, 73, 2, 21, 3, 73, 84, 72, 2, 17, - 2, 32, 72, 2, 153, 237, 1, 7, 79, 82, 73, 90, 79, 78, 84, 2, 25, 4, 76, - 76, 73, 80, 2, 11, 83, 2, 155, 223, 1, 73, 2, 11, 79, 2, 129, 79, 2, 85, - 82, 2, 17, 2, 32, 73, 2, 237, 178, 1, 2, 84, 69, 10, 120, 4, 77, 65, 82, - 75, 45, 22, 87, 73, 84, 72, 32, 86, 79, 73, 67, 69, 68, 32, 83, 79, 85, - 78, 68, 32, 77, 65, 82, 75, 7, 11, 32, 4, 50, 85, 21, 3, 76, 79, 87, 5, - 17, 2, 32, 85, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 129, 26, 2, 32, 72, - 11, 11, 32, 8, 38, 69, 49, 5, 87, 73, 84, 72, 32, 2, 25, 4, 88, 84, 69, - 78, 2, 255, 200, 1, 83, 6, 40, 4, 67, 73, 82, 67, 207, 135, 1, 77, 4, 11, - 76, 4, 11, 69, 4, 11, 32, 4, 26, 66, 223, 155, 1, 65, 2, 11, 69, 2, 159, - 126, 76, 2, 65, 14, 65, 76, 69, 32, 87, 73, 84, 72, 32, 83, 84, 82, 79, - 75, 2, 239, 144, 1, 69, 12, 194, 151, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, - 3, 55, 10, 32, 2, 65, 66, 106, 73, 19, 82, 6, 18, 32, 35, 85, 2, 11, 75, - 2, 251, 132, 2, 69, 4, 33, 6, 76, 65, 84, 73, 79, 78, 5, 199, 45, 32, 2, - 135, 9, 76, 2, 29, 5, 65, 70, 70, 73, 67, 2, 249, 234, 1, 2, 32, 76, 26, - 80, 6, 72, 69, 65, 86, 89, 32, 176, 5, 3, 77, 85, 67, 233, 9, 3, 66, 79, - 76, 20, 62, 69, 190, 1, 70, 38, 82, 82, 83, 246, 1, 87, 207, 10, 71, 4, - 25, 4, 73, 71, 72, 84, 4, 11, 32, 4, 22, 80, 223, 2, 83, 2, 25, 4, 79, - 73, 78, 84, 2, 17, 2, 69, 68, 2, 17, 2, 32, 66, 2, 25, 4, 76, 65, 67, 75, - 2, 11, 32, 2, 139, 79, 83, 2, 11, 73, 2, 185, 1, 2, 86, 69, 2, 11, 69, 2, - 29, 5, 86, 69, 82, 83, 69, 2, 17, 2, 32, 83, 2, 231, 1, 79, 6, 30, 65, - 42, 73, 147, 1, 79, 2, 11, 76, 2, 11, 84, 2, 147, 113, 73, 2, 11, 88, 2, - 11, 32, 2, 11, 83, 2, 21, 3, 80, 79, 75, 2, 17, 2, 69, 68, 2, 11, 32, 2, - 11, 65, 2, 11, 83, 2, 129, 78, 3, 84, 69, 82, 2, 165, 13, 3, 76, 73, 68, - 4, 21, 3, 72, 73, 84, 4, 11, 69, 4, 11, 32, 4, 190, 64, 67, 199, 46, 83, - 4, 11, 72, 4, 11, 32, 4, 18, 71, 39, 76, 2, 69, 6, 82, 69, 65, 84, 69, - 82, 2, 11, 69, 2, 11, 83, 2, 11, 83, 2, 17, 2, 45, 84, 2, 239, 70, 72, - 160, 1, 140, 1, 9, 66, 82, 65, 84, 73, 79, 78, 32, 77, 34, 67, 32, 3, 68, - 69, 79, 126, 69, 218, 1, 79, 22, 82, 21, 7, 84, 72, 75, 85, 81, 73, 32, - 2, 11, 79, 2, 159, 246, 1, 68, 2, 221, 126, 4, 84, 79, 82, 89, 6, 30, 32, - 77, 3, 67, 65, 83, 4, 18, 67, 43, 71, 2, 17, 2, 65, 77, 2, 195, 199, 1, - 69, 2, 143, 116, 65, 2, 147, 71, 83, 6, 168, 1, 31, 84, 78, 65, 77, 69, - 83, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 82, 69, 65, 68, 73, - 78, 71, 32, 77, 65, 82, 75, 32, 193, 105, 5, 87, 68, 65, 84, 65, 4, 26, - 78, 199, 135, 2, 67, 2, 143, 126, 72, 2, 175, 138, 1, 76, 2, 227, 234, 1, - 71, 140, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 70, 45, - 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 70, 230, 1, 66, 34, 69, 22, 73, - 22, 76, 34, 78, 254, 133, 1, 67, 2, 68, 2, 83, 2, 84, 214, 56, 72, 226, - 48, 70, 2, 74, 2, 77, 2, 80, 2, 82, 2, 86, 2, 88, 2, 90, 158, 20, 71, 2, - 75, 2, 81, 186, 2, 65, 2, 79, 2, 85, 3, 89, 4, 150, 240, 1, 66, 215, 22, - 69, 5, 203, 134, 2, 73, 5, 227, 239, 1, 74, 4, 234, 131, 2, 76, 187, 2, - 65, 4, 174, 239, 1, 74, 215, 22, 69, 8, 28, 3, 73, 68, 69, 59, 76, 2, 21, - 3, 68, 32, 71, 2, 177, 71, 4, 82, 69, 69, 75, 6, 46, 67, 20, 3, 76, 69, - 89, 41, 2, 85, 77, 2, 203, 219, 1, 65, 2, 11, 66, 2, 11, 65, 2, 147, 123, - 76, 2, 11, 69, 2, 17, 2, 32, 73, 2, 201, 122, 4, 78, 84, 69, 71, 5, 223, - 131, 2, 83, 40, 70, 67, 45, 13, 71, 65, 82, 32, 70, 82, 65, 67, 84, 73, - 79, 78, 32, 2, 11, 65, 2, 11, 78, 2, 155, 197, 1, 85, 38, 106, 70, 96, 4, - 79, 78, 69, 32, 174, 2, 84, 80, 7, 83, 69, 86, 69, 78, 32, 69, 145, 33, - 3, 90, 69, 82, 6, 56, 4, 73, 86, 69, 32, 249, 3, 5, 79, 85, 82, 32, 70, - 4, 158, 3, 69, 113, 3, 83, 73, 88, 18, 66, 69, 28, 2, 70, 73, 18, 72, 30, - 78, 14, 81, 30, 83, 55, 84, 2, 193, 1, 3, 73, 71, 72, 2, 167, 1, 70, 2, - 11, 65, 2, 131, 127, 76, 2, 111, 73, 2, 165, 73, 3, 85, 65, 82, 4, 24, 2, - 69, 86, 15, 73, 2, 43, 69, 2, 43, 88, 4, 18, 69, 35, 72, 2, 11, 78, 2, - 243, 253, 1, 84, 2, 155, 104, 73, 10, 48, 5, 72, 82, 69, 69, 32, 93, 3, - 87, 79, 32, 6, 26, 69, 26, 81, 67, 70, 2, 109, 3, 73, 71, 72, 2, 21, 3, - 85, 65, 82, 2, 151, 111, 84, 4, 22, 70, 211, 32, 84, 2, 11, 73, 2, 11, - 70, 2, 205, 191, 1, 2, 84, 72, 156, 6, 86, 65, 242, 23, 69, 158, 3, 72, - 142, 66, 73, 234, 8, 79, 134, 6, 82, 167, 144, 1, 74, 204, 2, 122, 78, - 220, 5, 2, 88, 73, 158, 1, 82, 210, 10, 84, 194, 1, 86, 252, 70, 2, 70, - 70, 233, 78, 6, 83, 84, 69, 66, 65, 83, 122, 36, 4, 67, 72, 79, 32, 183, - 5, 73, 118, 100, 7, 76, 69, 84, 84, 69, 82, 32, 252, 3, 3, 78, 71, 85, - 16, 5, 84, 79, 78, 69, 32, 243, 7, 68, 88, 210, 1, 65, 38, 79, 34, 69, - 22, 73, 22, 76, 50, 78, 42, 84, 50, 85, 22, 89, 138, 175, 1, 75, 2, 80, - 2, 83, 254, 68, 66, 2, 67, 2, 68, 2, 70, 2, 71, 2, 72, 2, 74, 2, 77, 2, - 82, 2, 86, 2, 87, 3, 90, 13, 34, 65, 206, 247, 1, 78, 87, 85, 7, 11, 78, - 5, 147, 248, 1, 71, 5, 255, 247, 1, 78, 5, 151, 247, 1, 78, 4, 26, 76, - 191, 247, 1, 65, 2, 131, 245, 1, 72, 6, 238, 244, 1, 71, 2, 89, 187, 2, - 65, 8, 198, 244, 1, 72, 2, 82, 2, 83, 187, 2, 65, 5, 251, 166, 1, 69, 4, - 170, 245, 1, 73, 147, 1, 65, 2, 179, 111, 78, 8, 40, 3, 75, 79, 73, 1, 3, - 84, 85, 80, 5, 135, 226, 1, 78, 4, 21, 3, 78, 71, 32, 4, 76, 8, 67, 82, - 69, 83, 67, 69, 78, 84, 1, 7, 71, 73, 66, 66, 79, 85, 83, 2, 21, 3, 32, - 77, 79, 2, 11, 79, 2, 175, 97, 78, 170, 1, 72, 9, 65, 78, 71, 32, 67, 73, - 84, 73, 32, 249, 108, 4, 78, 73, 78, 71, 168, 1, 128, 1, 6, 67, 65, 80, - 73, 84, 65, 0, 4, 83, 77, 65, 76, 190, 4, 68, 156, 2, 7, 78, 85, 77, 66, - 69, 82, 32, 171, 219, 1, 79, 64, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, - 32, 64, 174, 1, 65, 38, 69, 42, 72, 74, 78, 50, 79, 22, 83, 50, 85, 30, - 89, 170, 42, 84, 248, 107, 2, 86, 73, 206, 51, 66, 2, 80, 246, 5, 75, - 158, 11, 73, 2, 87, 162, 17, 68, 3, 71, 9, 162, 240, 1, 78, 86, 77, 3, - 84, 7, 11, 78, 4, 198, 240, 1, 78, 3, 89, 8, 38, 79, 210, 151, 1, 73, - 231, 31, 65, 4, 178, 183, 1, 82, 223, 25, 76, 4, 26, 71, 223, 158, 1, 85, - 2, 143, 237, 1, 65, 5, 159, 169, 1, 68, 4, 26, 83, 175, 219, 1, 73, 2, - 155, 202, 1, 85, 4, 238, 238, 1, 67, 3, 85, 8, 34, 85, 178, 238, 1, 65, - 3, 79, 5, 175, 238, 1, 74, 20, 11, 73, 20, 11, 71, 20, 17, 2, 73, 84, 20, - 11, 32, 20, 66, 70, 30, 83, 42, 84, 62, 90, 130, 83, 78, 14, 79, 199, - 110, 69, 4, 218, 112, 73, 131, 4, 79, 4, 22, 69, 227, 96, 73, 2, 247, 27, - 86, 4, 26, 72, 207, 205, 1, 87, 2, 11, 82, 2, 223, 213, 1, 69, 2, 11, 69, - 2, 159, 205, 1, 82, 18, 42, 69, 30, 70, 42, 78, 38, 83, 39, 84, 2, 221, - 1, 3, 73, 71, 72, 4, 22, 73, 139, 1, 79, 2, 171, 1, 70, 2, 17, 2, 73, 78, - 2, 135, 1, 69, 4, 92, 2, 69, 86, 25, 2, 73, 88, 6, 34, 72, 26, 87, 187, - 154, 1, 69, 2, 11, 73, 2, 35, 82, 2, 11, 69, 2, 11, 78, 2, 171, 216, 1, - 84, 12, 32, 2, 69, 82, 175, 232, 1, 67, 10, 34, 32, 173, 153, 1, 2, 77, - 69, 8, 80, 3, 67, 76, 79, 22, 87, 224, 197, 1, 5, 66, 85, 70, 70, 65, 1, - 2, 80, 79, 2, 183, 179, 1, 83, 2, 235, 107, 65, 20, 60, 2, 69, 32, 124, - 4, 73, 78, 71, 32, 161, 1, 2, 89, 32, 6, 182, 2, 68, 157, 186, 1, 23, 65, - 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 68, 73, 82, 69, - 67, 84, 76, 89, 6, 64, 5, 66, 76, 65, 67, 75, 0, 5, 87, 72, 73, 84, 69, - 55, 72, 2, 17, 2, 32, 70, 2, 11, 76, 2, 171, 229, 1, 65, 2, 11, 65, 2, - 11, 78, 2, 235, 94, 68, 8, 26, 68, 34, 76, 43, 79, 2, 11, 65, 2, 139, - 228, 1, 83, 4, 22, 79, 223, 75, 73, 2, 183, 75, 87, 2, 11, 86, 2, 11, 69, - 2, 155, 75, 82, 14, 48, 3, 65, 82, 89, 70, 68, 70, 73, 171, 1, 83, 4, 11, - 32, 4, 32, 2, 67, 65, 219, 166, 1, 70, 2, 191, 166, 1, 84, 4, 26, 71, - 175, 146, 1, 68, 2, 149, 67, 6, 69, 45, 84, 65, 73, 76, 4, 116, 17, 69, - 82, 83, 84, 82, 65, 83, 83, 32, 69, 76, 76, 73, 80, 84, 73, 67, 185, 43, - 7, 71, 72, 84, 32, 76, 73, 70, 2, 17, 2, 32, 70, 2, 157, 145, 1, 2, 85, - 78, 2, 37, 7, 84, 32, 83, 89, 82, 73, 65, 2, 179, 35, 67, 180, 2, 52, 3, - 69, 69, 76, 100, 3, 73, 84, 69, 219, 62, 65, 7, 60, 7, 32, 79, 70, 32, - 68, 72, 65, 21, 4, 67, 72, 65, 73, 2, 143, 191, 1, 82, 2, 251, 76, 82, - 172, 2, 54, 32, 181, 63, 8, 45, 70, 69, 65, 84, 72, 69, 82, 170, 2, 148, - 2, 18, 65, 82, 82, 79, 87, 32, 83, 72, 65, 70, 84, 32, 87, 73, 68, 84, - 72, 32, 114, 66, 46, 67, 194, 15, 68, 166, 5, 69, 50, 70, 178, 3, 72, - 222, 3, 76, 218, 3, 77, 226, 1, 78, 34, 80, 146, 1, 81, 78, 82, 250, 1, - 83, 142, 12, 84, 228, 3, 2, 85, 80, 217, 3, 3, 86, 69, 82, 4, 22, 84, - 251, 67, 79, 2, 11, 87, 2, 21, 3, 79, 32, 84, 2, 17, 2, 72, 73, 2, 11, - 82, 2, 247, 158, 1, 68, 2, 11, 85, 2, 11, 76, 2, 143, 167, 1, 76, 92, - 146, 1, 72, 168, 10, 5, 73, 82, 67, 76, 69, 162, 3, 76, 25, 20, 79, 78, - 67, 65, 86, 69, 45, 83, 73, 68, 69, 68, 32, 68, 73, 65, 77, 79, 78, 68, - 66, 25, 4, 69, 83, 83, 32, 66, 66, 66, 38, 69, 118, 75, 142, 4, 80, 22, - 81, 38, 82, 131, 2, 84, 6, 241, 5, 5, 73, 83, 72, 79, 80, 4, 45, 9, 81, - 85, 73, 72, 79, 80, 80, 69, 82, 5, 17, 2, 32, 82, 2, 129, 6, 8, 79, 84, - 65, 84, 69, 68, 32, 78, 26, 38, 73, 25, 5, 78, 73, 71, 72, 84, 6, 177, 4, - 2, 78, 71, 21, 22, 32, 151, 3, 45, 12, 41, 8, 82, 79, 84, 65, 84, 69, 68, - 32, 12, 104, 2, 70, 79, 0, 15, 79, 78, 69, 32, 72, 85, 78, 68, 82, 69, - 68, 32, 84, 72, 73, 18, 84, 215, 3, 78, 2, 207, 1, 82, 6, 148, 1, 11, 87, - 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, 133, 3, 20, 72, 82, 69, 69, 32, - 72, 85, 78, 68, 82, 69, 68, 32, 70, 73, 70, 84, 69, 69, 78, 4, 36, 4, 84, - 87, 69, 78, 175, 2, 83, 2, 217, 2, 7, 84, 89, 45, 70, 73, 86, 69, 6, 166, - 3, 66, 94, 81, 47, 82, 6, 41, 2, 65, 87, 6, 21, 3, 85, 69, 69, 6, 35, 78, - 6, 21, 3, 79, 79, 75, 7, 45, 9, 32, 82, 79, 84, 65, 84, 69, 68, 32, 4, - 70, 78, 25, 13, 84, 87, 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, 83, 2, - 49, 3, 73, 78, 69, 2, 25, 4, 69, 86, 69, 78, 2, 17, 2, 84, 89, 2, 253, - 62, 6, 32, 68, 69, 71, 82, 69, 12, 33, 6, 85, 82, 78, 69, 68, 32, 12, 42, - 66, 30, 75, 34, 80, 34, 81, 47, 82, 2, 225, 88, 3, 73, 83, 72, 4, 222, - 128, 1, 73, 159, 38, 78, 2, 11, 65, 2, 203, 129, 1, 87, 2, 11, 85, 2, 11, - 69, 2, 159, 129, 1, 69, 2, 255, 166, 1, 79, 19, 11, 32, 16, 100, 16, 67, - 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 121, 5, 87, - 73, 84, 72, 32, 2, 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 11, - 32, 2, 11, 67, 2, 11, 73, 2, 11, 82, 2, 203, 45, 67, 14, 56, 2, 68, 79, - 70, 84, 148, 30, 2, 76, 79, 231, 1, 85, 4, 18, 84, 35, 87, 2, 11, 32, 2, - 179, 163, 1, 82, 2, 179, 51, 78, 2, 11, 87, 2, 11, 79, 2, 11, 32, 2, 155, - 57, 68, 2, 141, 26, 2, 85, 66, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 148, - 29, 2, 76, 69, 49, 2, 82, 73, 28, 76, 6, 73, 65, 77, 79, 78, 68, 216, 2, - 3, 79, 87, 78, 177, 1, 2, 82, 65, 15, 11, 32, 12, 160, 1, 17, 67, 79, 78, - 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, 128, 1, 9, 87, - 73, 84, 72, 32, 67, 69, 78, 84, 198, 22, 83, 173, 18, 2, 73, 78, 6, 74, - 83, 0, 6, 86, 69, 82, 89, 32, 83, 29, 6, 77, 69, 68, 73, 85, 77, 2, 25, - 4, 77, 65, 76, 76, 2, 181, 14, 2, 32, 68, 2, 11, 82, 2, 11, 69, 2, 179, - 56, 68, 10, 96, 10, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 53, 10, 45, - 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 250, 34, 66, 24, 5, 76, 69, 70, - 84, 32, 51, 73, 4, 138, 36, 83, 51, 84, 4, 33, 6, 85, 71, 72, 84, 83, 32, - 4, 22, 77, 191, 118, 75, 2, 223, 119, 65, 2, 29, 5, 88, 67, 76, 65, 77, - 2, 155, 14, 65, 14, 74, 76, 144, 2, 11, 79, 85, 82, 32, 80, 79, 73, 78, - 84, 69, 68, 71, 82, 8, 28, 2, 65, 71, 175, 1, 79, 5, 149, 1, 34, 32, 87, - 73, 84, 72, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 77, 73, 68, - 68, 76, 69, 32, 66, 76, 65, 67, 75, 32, 83, 84, 82, 73, 2, 155, 174, 1, - 80, 4, 26, 82, 227, 134, 1, 87, 2, 17, 2, 69, 84, 2, 215, 173, 1, 84, 4, - 11, 32, 4, 18, 67, 23, 83, 2, 223, 195, 1, 85, 2, 187, 33, 84, 2, 183, - 59, 79, 16, 34, 65, 150, 1, 69, 219, 1, 79, 2, 25, 4, 82, 68, 32, 83, 2, - 45, 9, 72, 69, 76, 76, 32, 70, 76, 79, 80, 2, 17, 2, 80, 89, 2, 17, 2, - 32, 68, 2, 11, 73, 2, 135, 191, 1, 83, 10, 22, 65, 247, 10, 88, 8, 30, - 82, 29, 3, 86, 89, 32, 4, 11, 84, 5, 183, 14, 32, 4, 74, 67, 41, 14, 83, - 65, 76, 84, 73, 82, 69, 32, 87, 73, 84, 72, 32, 82, 2, 21, 3, 72, 69, 67, - 2, 155, 127, 75, 2, 139, 18, 79, 4, 152, 30, 10, 82, 73, 90, 79, 78, 84, - 65, 76, 32, 69, 229, 10, 2, 85, 82, 18, 170, 1, 65, 116, 3, 69, 70, 84, - 165, 9, 31, 79, 90, 69, 78, 71, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, - 78, 71, 32, 66, 76, 65, 67, 75, 32, 83, 77, 65, 76, 76, 32, 76, 4, 24, 2, - 82, 71, 19, 84, 2, 199, 29, 69, 2, 11, 73, 2, 11, 78, 2, 17, 2, 32, 67, - 2, 11, 82, 2, 159, 39, 79, 12, 58, 32, 77, 10, 45, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 6, 44, 6, 76, 65, 78, 69, 32, 77, 223, 22, 80, 2, 11, 69, - 2, 219, 8, 82, 6, 30, 80, 166, 24, 83, 51, 84, 2, 187, 5, 79, 12, 68, 6, - 69, 68, 73, 85, 77, 32, 121, 7, 79, 79, 78, 32, 83, 69, 76, 10, 30, 68, - 54, 83, 211, 6, 76, 2, 11, 73, 2, 11, 65, 2, 11, 77, 2, 163, 45, 79, 6, - 202, 24, 84, 186, 1, 77, 59, 81, 2, 11, 69, 2, 139, 184, 1, 78, 2, 11, - 73, 2, 163, 186, 1, 66, 6, 18, 65, 87, 69, 2, 37, 7, 82, 65, 76, 76, 69, - 76, 79, 2, 11, 71, 2, 11, 82, 2, 227, 168, 1, 65, 4, 11, 78, 4, 154, 2, - 84, 191, 45, 78, 2, 21, 3, 85, 69, 83, 2, 145, 117, 9, 84, 73, 79, 78, - 32, 77, 65, 82, 75, 14, 36, 4, 73, 71, 72, 84, 179, 22, 69, 12, 60, 10, - 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 195, 17, 32, 8, 30, 80, 178, 19, - 83, 51, 84, 4, 18, 69, 55, 79, 2, 11, 78, 2, 11, 84, 2, 11, 65, 2, 147, - 103, 71, 2, 11, 73, 2, 11, 78, 2, 235, 120, 84, 56, 118, 67, 32, 3, 69, - 83, 65, 18, 72, 58, 77, 170, 1, 80, 68, 5, 81, 85, 65, 82, 69, 168, 6, 2, - 85, 78, 163, 10, 84, 2, 133, 39, 4, 73, 83, 83, 79, 2, 155, 36, 77, 2, - 21, 3, 79, 71, 73, 2, 137, 120, 4, 32, 80, 73, 69, 8, 32, 4, 65, 76, 76, - 32, 119, 73, 6, 18, 76, 71, 83, 2, 11, 79, 2, 11, 90, 2, 11, 69, 2, 11, - 78, 2, 159, 157, 1, 71, 4, 182, 17, 84, 243, 1, 81, 2, 187, 43, 76, 2, - 21, 3, 65, 68, 69, 2, 11, 32, 2, 11, 83, 2, 203, 150, 1, 85, 29, 11, 32, + 32, 130, 4, 84, 222, 149, 2, 73, 149, 70, 5, 75, 89, 79, 32, 84, 104, + 242, 1, 71, 42, 74, 30, 77, 34, 78, 70, 72, 34, 80, 34, 83, 210, 50, 82, + 206, 147, 1, 79, 134, 60, 69, 42, 76, 198, 2, 65, 178, 191, 1, 67, 2, 68, + 2, 75, 2, 84, 2, 88, 2, 90, 138, 69, 66, 2, 70, 2, 81, 2, 86, 186, 2, 73, + 2, 85, 3, 89, 6, 170, 139, 4, 72, 2, 74, 187, 2, 65, 4, 186, 141, 4, 65, + 3, 89, 4, 230, 138, 4, 66, 187, 2, 65, 14, 66, 71, 190, 194, 2, 74, 194, + 130, 1, 88, 138, 69, 68, 187, 2, 65, 4, 130, 138, 4, 74, 187, 2, 65, 4, + 226, 137, 4, 83, 187, 2, 65, 10, 54, 72, 198, 193, 2, 75, 202, 199, 1, + 84, 187, 2, 65, 4, 138, 137, 4, 84, 187, 2, 65, 2, 231, 28, 65, 2, 11, + 71, 2, 179, 244, 3, 85, 6, 32, 2, 84, 72, 151, 254, 2, 76, 5, 11, 66, 2, + 11, 82, 2, 151, 162, 2, 85, 72, 252, 1, 4, 65, 82, 67, 32, 162, 2, 67, + 44, 2, 72, 65, 166, 3, 80, 140, 3, 13, 74, 85, 83, 84, 73, 70, 73, 69, + 68, 32, 76, 79, 87, 84, 5, 76, 69, 70, 84, 32, 228, 1, 6, 82, 73, 71, 72, + 84, 32, 170, 2, 83, 38, 84, 81, 7, 87, 73, 84, 72, 32, 85, 80, 6, 180, 1, + 19, 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, + 79, 87, 73, 21, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, + 87, 32, 87, 73, 84, 72, 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, + 80, 2, 187, 128, 2, 76, 2, 11, 77, 2, 143, 128, 2, 73, 2, 11, 85, 2, 133, + 179, 3, 3, 82, 76, 89, 34, 36, 3, 76, 70, 32, 179, 133, 4, 84, 32, 70, + 70, 186, 1, 76, 22, 82, 178, 2, 83, 250, 5, 66, 231, 243, 1, 73, 8, 136, + 1, 15, 79, 82, 87, 65, 82, 68, 45, 70, 65, 67, 73, 78, 71, 32, 82, 157, + 2, 13, 76, 65, 73, 76, 73, 78, 71, 32, 82, 79, 66, 79, 84, 4, 22, 85, + 243, 1, 79, 2, 203, 197, 3, 78, 8, 41, 2, 69, 70, 8, 21, 3, 73, 71, 72, + 8, 11, 84, 8, 54, 32, 69, 9, 45, 70, 65, 67, 73, 78, 71, 32, 82, 2, 11, + 80, 2, 33, 6, 65, 82, 69, 78, 84, 72, 2, 207, 222, 1, 69, 6, 38, 79, 21, + 5, 85, 78, 78, 69, 82, 2, 155, 177, 3, 66, 4, 29, 5, 32, 70, 82, 65, 77, + 4, 11, 69, 4, 11, 45, 4, 134, 129, 4, 49, 3, 50, 4, 18, 69, 59, 84, 2, + 11, 67, 2, 11, 84, 2, 11, 73, 2, 147, 135, 2, 79, 2, 17, 2, 65, 78, 2, + 11, 68, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 80, 2, 11, 69, 2, 11, 82, 2, + 207, 175, 3, 83, 4, 17, 2, 69, 82, 4, 33, 6, 32, 72, 65, 76, 70, 32, 4, + 250, 3, 66, 139, 105, 87, 10, 180, 1, 13, 74, 85, 83, 84, 73, 70, 73, 69, + 68, 32, 76, 79, 87, 126, 67, 50, 72, 225, 214, 2, 21, 66, 76, 65, 67, 75, + 32, 76, 69, 70, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 83, 2, 237, + 1, 7, 69, 82, 32, 82, 73, 71, 72, 8, 78, 67, 50, 72, 33, 13, 74, 85, 83, + 84, 73, 70, 73, 69, 68, 32, 76, 79, 87, 4, 26, 79, 243, 130, 3, 82, 2, + 139, 190, 3, 82, 2, 253, 168, 3, 3, 65, 76, 70, 2, 33, 6, 69, 82, 32, 76, + 69, 70, 2, 53, 11, 84, 32, 81, 85, 65, 82, 84, 69, 82, 32, 66, 2, 11, 76, + 2, 209, 168, 2, 3, 65, 67, 75, 2, 201, 167, 3, 4, 81, 85, 65, 82, 2, 11, + 79, 2, 165, 167, 3, 12, 82, 84, 79, 73, 83, 69, 32, 83, 72, 69, 76, 76, + 2, 29, 5, 87, 65, 82, 68, 83, 2, 17, 2, 32, 65, 2, 153, 252, 2, 4, 82, + 82, 79, 87, 20, 188, 1, 22, 67, 74, 75, 32, 85, 78, 73, 70, 73, 69, 68, + 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 161, 2, 19, 76, 65, 84, 73, + 78, 32, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 18, 40, 2, + 52, 69, 34, 53, 54, 54, 87, 55, 4, 202, 1, 48, 203, 205, 2, 56, 4, 30, + 50, 141, 1, 2, 66, 56, 2, 175, 176, 3, 68, 6, 48, 2, 50, 53, 22, 53, 249, + 205, 2, 2, 55, 50, 2, 135, 246, 3, 51, 2, 67, 53, 4, 32, 2, 48, 66, 21, + 2, 54, 68, 2, 195, 245, 3, 57, 2, 175, 245, 3, 55, 2, 11, 82, 2, 179, + 183, 3, 32, 64, 48, 6, 65, 76, 32, 82, 85, 78, 21, 2, 79, 32, 2, 247, + 250, 2, 79, 62, 72, 7, 76, 69, 84, 84, 69, 82, 32, 217, 2, 6, 83, 73, 71, + 78, 32, 82, 60, 206, 1, 66, 106, 78, 138, 30, 73, 202, 141, 1, 69, 162, + 75, 67, 162, 186, 1, 65, 234, 61, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, + 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 79, 3, 85, 12, + 52, 7, 82, 69, 65, 84, 72, 89, 32, 227, 241, 3, 65, 10, 182, 30, 73, 202, + 141, 1, 69, 143, 175, 2, 65, 4, 246, 238, 3, 71, 187, 2, 65, 2, 17, 2, + 73, 83, 2, 21, 3, 73, 78, 71, 2, 11, 32, 2, 147, 198, 2, 84, 98, 46, 65, + 254, 1, 73, 246, 11, 79, 195, 1, 85, 16, 66, 67, 36, 2, 68, 69, 34, 77, + 20, 2, 78, 83, 155, 159, 3, 73, 4, 246, 231, 1, 75, 243, 206, 1, 84, 2, + 141, 16, 4, 32, 77, 65, 82, 5, 131, 212, 1, 32, 4, 44, 2, 80, 76, 33, 5, + 86, 69, 82, 83, 65, 2, 11, 85, 2, 199, 207, 3, 84, 2, 171, 62, 76, 66, + 156, 1, 3, 65, 78, 71, 176, 3, 8, 68, 69, 78, 84, 32, 69, 77, 66, 20, 9, + 71, 82, 65, 77, 32, 70, 79, 82, 32, 140, 2, 4, 80, 76, 69, 32, 167, 151, + 3, 67, 16, 48, 2, 76, 69, 245, 1, 5, 85, 76, 65, 82, 32, 10, 44, 6, 32, + 87, 73, 84, 72, 32, 163, 1, 45, 8, 80, 9, 83, 69, 82, 73, 70, 83, 32, 65, + 84, 54, 85, 138, 168, 2, 82, 187, 66, 68, 2, 17, 2, 32, 66, 2, 197, 218, + 3, 3, 79, 84, 84, 2, 241, 60, 2, 78, 68, 2, 205, 202, 2, 4, 72, 69, 65, + 68, 6, 68, 9, 70, 76, 65, 71, 32, 79, 78, 32, 80, 34, 82, 179, 138, 2, + 66, 2, 11, 79, 2, 207, 205, 3, 83, 2, 11, 85, 2, 239, 171, 3, 76, 2, 155, + 215, 3, 76, 16, 66, 69, 34, 72, 34, 76, 22, 77, 46, 84, 42, 87, 255, 210, + 1, 70, 2, 11, 65, 2, 211, 230, 1, 82, 2, 11, 69, 2, 211, 248, 1, 65, 2, + 215, 184, 3, 65, 2, 21, 3, 79, 85, 78, 2, 175, 187, 2, 84, 2, 17, 2, 72, + 85, 2, 215, 199, 1, 78, 4, 206, 175, 2, 65, 131, 42, 73, 30, 176, 2, 3, + 67, 79, 76, 24, 20, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, + 82, 32, 87, 73, 84, 72, 32, 98, 80, 34, 84, 20, 13, 86, 69, 82, 84, 73, + 67, 65, 76, 32, 66, 65, 82, 32, 44, 9, 83, 79, 76, 73, 68, 85, 83, 32, + 66, 188, 21, 2, 68, 65, 168, 187, 1, 6, 78, 69, 83, 84, 69, 68, 171, 11, + 73, 2, 129, 37, 2, 79, 78, 4, 34, 68, 33, 4, 84, 82, 73, 80, 2, 17, 2, + 79, 85, 2, 11, 66, 2, 185, 179, 3, 2, 76, 69, 4, 254, 221, 1, 76, 135, + 110, 82, 2, 167, 202, 1, 73, 8, 42, 66, 74, 68, 134, 189, 1, 82, 115, 87, + 2, 29, 5, 73, 78, 65, 82, 89, 2, 21, 3, 32, 82, 69, 2, 235, 32, 76, 2, + 25, 4, 69, 76, 73, 77, 2, 191, 170, 2, 73, 10, 24, 2, 76, 76, 35, 80, 5, + 137, 220, 1, 3, 69, 89, 66, 6, 44, 5, 73, 67, 65, 76, 32, 243, 207, 3, + 72, 4, 18, 68, 47, 70, 2, 11, 82, 2, 11, 73, 2, 243, 221, 3, 78, 2, 215, + 248, 1, 73, 6, 18, 69, 79, 77, 5, 129, 188, 3, 14, 32, 76, 73, 71, 72, + 84, 32, 77, 79, 79, 78, 32, 65, 82, 2, 235, 170, 3, 80, 216, 1, 76, 3, + 71, 82, 73, 22, 76, 214, 12, 82, 221, 186, 2, 5, 77, 66, 76, 69, 82, 2, + 155, 216, 2, 75, 162, 1, 68, 11, 85, 45, 84, 73, 71, 65, 76, 65, 82, 73, + 32, 179, 222, 3, 73, 160, 1, 122, 65, 42, 67, 30, 68, 106, 71, 32, 7, 76, + 69, 84, 84, 69, 82, 32, 178, 3, 82, 32, 5, 83, 73, 71, 78, 32, 131, 3, + 86, 2, 11, 85, 2, 177, 155, 3, 2, 32, 76, 2, 245, 158, 3, 2, 79, 78, 4, + 18, 79, 67, 65, 2, 11, 85, 2, 11, 66, 2, 25, 4, 76, 69, 32, 68, 2, 11, + 65, 2, 159, 183, 3, 78, 2, 129, 215, 2, 3, 69, 77, 73, 100, 206, 1, 65, + 38, 68, 46, 76, 38, 82, 34, 84, 46, 86, 186, 5, 85, 206, 201, 1, 73, 162, + 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, + 79, 162, 8, 69, 158, 20, 72, 2, 77, 3, 89, 9, 158, 218, 3, 65, 2, 73, 3, + 85, 8, 186, 146, 3, 68, 138, 69, 72, 187, 2, 65, 6, 154, 208, 1, 76, 183, + 137, 2, 65, 4, 242, 214, 3, 82, 187, 2, 65, 8, 202, 145, 3, 84, 138, 69, + 72, 187, 2, 65, 10, 214, 5, 79, 139, 211, 3, 65, 2, 11, 69, 2, 243, 221, + 1, 80, 18, 174, 1, 65, 72, 6, 76, 79, 79, 80, 69, 68, 40, 2, 79, 77, 0, + 5, 83, 72, 82, 73, 73, 48, 13, 67, 65, 78, 68, 82, 65, 32, 65, 78, 85, + 78, 65, 83, 22, 80, 179, 146, 3, 86, 4, 26, 86, 235, 145, 3, 78, 2, 21, + 3, 65, 71, 82, 2, 251, 219, 1, 65, 2, 17, 2, 32, 86, 2, 187, 157, 1, 73, + 2, 17, 2, 32, 80, 2, 25, 4, 85, 83, 72, 80, 2, 231, 204, 3, 73, 2, 237, + 176, 3, 2, 76, 85, 30, 56, 10, 69, 68, 73, 67, 32, 84, 79, 78, 69, 32, + 35, 79, 4, 226, 164, 1, 65, 235, 4, 83, 26, 45, 9, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 26, 70, 65, 38, 85, 22, 86, 186, 201, 1, 73, 234, 234, 1, + 79, 163, 8, 69, 6, 202, 211, 3, 65, 2, 73, 3, 85, 5, 167, 211, 3, 85, 8, + 11, 79, 8, 33, 6, 67, 65, 76, 73, 67, 32, 8, 26, 82, 159, 214, 2, 76, 5, + 199, 210, 3, 82, 50, 50, 75, 84, 4, 78, 69, 68, 32, 135, 173, 2, 84, 4, + 48, 6, 73, 83, 72, 32, 76, 73, 135, 192, 3, 69, 2, 11, 82, 2, 191, 202, + 2, 65, 44, 238, 1, 65, 80, 6, 66, 76, 65, 67, 75, 32, 40, 7, 87, 72, 73, + 84, 69, 32, 83, 22, 67, 50, 68, 96, 2, 78, 79, 32, 2, 79, 75, 30, 83, + 153, 168, 3, 21, 71, 82, 69, 69, 75, 32, 83, 77, 65, 76, 76, 32, 76, 69, + 84, 84, 69, 82, 32, 73, 79, 4, 26, 77, 239, 170, 2, 78, 2, 17, 2, 80, 69, + 2, 11, 82, 2, 251, 192, 2, 83, 4, 18, 80, 23, 83, 2, 147, 150, 2, 69, 2, + 131, 152, 2, 72, 4, 144, 3, 5, 65, 80, 73, 84, 65, 167, 121, 79, 6, 30, + 65, 33, 3, 73, 71, 73, 2, 11, 71, 2, 231, 143, 3, 71, 4, 221, 221, 1, 3, + 84, 32, 84, 6, 162, 2, 82, 255, 195, 2, 84, 2, 141, 228, 1, 2, 32, 72, + 14, 128, 1, 18, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, + 84, 65, 76, 32, 38, 69, 32, 3, 77, 65, 76, 33, 2, 79, 85, 6, 222, 203, 3, + 71, 2, 76, 3, 89, 2, 11, 77, 2, 159, 251, 2, 73, 2, 11, 76, 2, 235, 201, + 2, 32, 4, 21, 3, 84, 72, 32, 4, 32, 2, 69, 65, 1, 2, 87, 69, 2, 53, 11, + 83, 84, 32, 80, 79, 73, 78, 84, 73, 78, 71, 2, 17, 2, 32, 76, 2, 195, + 200, 2, 69, 38, 58, 69, 52, 8, 73, 83, 84, 69, 68, 32, 82, 73, 63, 79, 2, + 17, 2, 76, 86, 2, 153, 178, 1, 3, 69, 32, 80, 2, 17, 2, 71, 72, 2, 137, + 109, 6, 84, 87, 65, 82, 68, 83, 34, 30, 32, 229, 10, 2, 45, 69, 32, 134, + 3, 66, 86, 67, 116, 3, 68, 79, 84, 226, 1, 72, 44, 14, 73, 78, 84, 69, + 82, 83, 69, 67, 84, 73, 78, 71, 32, 76, 92, 6, 74, 79, 73, 78, 69, 68, + 64, 8, 76, 79, 71, 73, 67, 65, 76, 32, 98, 77, 0, 3, 87, 79, 77, 116, 13, + 82, 73, 78, 71, 83, 32, 65, 76, 73, 71, 78, 69, 68, 48, 24, 65, 83, 84, + 69, 82, 73, 83, 75, 83, 32, 65, 76, 73, 71, 78, 69, 68, 32, 86, 69, 82, + 84, 73, 67, 35, 83, 2, 29, 5, 85, 84, 84, 79, 78, 2, 17, 2, 32, 77, 2, + 11, 79, 2, 191, 187, 2, 85, 2, 93, 21, 79, 78, 83, 69, 67, 85, 84, 73, + 86, 69, 32, 69, 81, 85, 65, 76, 83, 32, 83, 73, 71, 2, 251, 133, 3, 78, + 6, 18, 32, 55, 83, 4, 22, 76, 131, 1, 80, 2, 145, 163, 1, 2, 69, 65, 2, + 53, 11, 32, 79, 86, 69, 82, 32, 79, 78, 69, 32, 68, 2, 11, 79, 2, 11, 84, + 2, 17, 2, 32, 80, 2, 29, 5, 85, 78, 67, 84, 85, 2, 167, 241, 2, 65, 2, + 11, 69, 2, 11, 65, 2, 159, 173, 2, 82, 4, 17, 2, 79, 71, 4, 25, 4, 73, + 67, 65, 76, 4, 11, 32, 4, 130, 179, 2, 65, 159, 85, 79, 2, 17, 2, 32, 83, + 2, 21, 3, 81, 85, 65, 2, 183, 172, 2, 82, 4, 30, 79, 13, 3, 65, 78, 68, + 2, 11, 82, 2, 11, 32, 2, 11, 79, 2, 11, 80, 2, 143, 18, 69, 2, 11, 69, 2, + 45, 9, 78, 32, 72, 79, 76, 68, 73, 78, 71, 2, 11, 32, 2, 11, 72, 2, 11, + 65, 2, 143, 223, 1, 78, 2, 45, 9, 32, 72, 79, 82, 73, 90, 79, 78, 84, 2, + 11, 65, 2, 219, 154, 3, 76, 2, 49, 10, 80, 69, 69, 67, 72, 32, 66, 85, + 66, 66, 2, 203, 169, 2, 76, 2, 11, 77, 2, 245, 212, 1, 2, 32, 68, 248, 3, + 140, 1, 8, 71, 65, 82, 73, 84, 73, 67, 32, 152, 6, 7, 77, 66, 82, 69, 76, + 76, 65, 166, 1, 78, 158, 8, 80, 138, 166, 1, 82, 151, 134, 2, 83, 62, 48, + 7, 76, 69, 84, 84, 69, 82, 32, 159, 5, 87, 60, 206, 1, 65, 28, 2, 81, 79, + 22, 66, 22, 68, 50, 71, 44, 2, 72, 79, 22, 75, 34, 76, 32, 2, 82, 65, 22, + 83, 74, 84, 74, 89, 22, 90, 150, 182, 2, 78, 202, 91, 80, 246, 5, 87, + 202, 12, 77, 174, 18, 73, 3, 85, 4, 26, 76, 247, 233, 2, 73, 2, 155, 183, + 3, 80, 2, 243, 148, 3, 69, 4, 26, 69, 211, 175, 2, 72, 2, 195, 148, 3, + 76, 4, 166, 140, 2, 72, 189, 138, 1, 2, 65, 77, 5, 151, 182, 3, 84, 4, + 150, 183, 2, 65, 239, 126, 72, 2, 11, 65, 2, 139, 147, 3, 77, 2, 187, + 189, 1, 83, 8, 38, 65, 238, 183, 2, 72, 239, 90, 83, 4, 234, 160, 3, 68, + 239, 13, 77, 6, 38, 72, 206, 154, 3, 69, 175, 28, 79, 2, 11, 65, 2, 231, + 250, 1, 78, 2, 183, 240, 2, 79, 4, 246, 145, 3, 69, 207, 36, 85, 2, 21, + 3, 79, 82, 68, 2, 25, 4, 32, 68, 73, 86, 2, 235, 149, 1, 73, 7, 11, 32, + 4, 84, 4, 79, 78, 32, 71, 45, 13, 87, 73, 84, 72, 32, 82, 65, 73, 78, 32, + 68, 82, 79, 2, 11, 82, 2, 11, 79, 2, 231, 166, 2, 85, 2, 239, 246, 2, 80, + 32, 160, 1, 3, 65, 77, 85, 20, 7, 67, 69, 82, 84, 65, 73, 78, 34, 68, 62, + 73, 245, 5, 18, 77, 65, 82, 82, 73, 69, 68, 32, 80, 65, 82, 84, 78, 69, + 82, 83, 72, 73, 2, 187, 162, 2, 83, 2, 11, 84, 2, 243, 171, 2, 89, 4, 26, + 69, 159, 159, 2, 79, 2, 11, 82, 2, 135, 151, 3, 84, 22, 60, 2, 79, 78, + 234, 3, 84, 106, 86, 173, 58, 3, 67, 79, 82, 15, 11, 32, 12, 160, 1, 6, + 65, 66, 79, 86, 69, 32, 108, 22, 66, 69, 83, 73, 68, 69, 32, 65, 78, 68, + 32, 74, 79, 73, 78, 69, 68, 32, 87, 73, 84, 72, 41, 5, 87, 73, 84, 72, + 32, 4, 52, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 73, 2, 17, 2, 32, + 73, 2, 161, 49, 4, 78, 84, 69, 82, 2, 17, 2, 32, 85, 2, 239, 222, 2, 78, + 6, 66, 77, 58, 79, 189, 245, 2, 8, 76, 79, 71, 73, 67, 65, 76, 32, 2, 11, + 73, 2, 11, 78, 2, 11, 85, 2, 183, 167, 2, 83, 2, 11, 86, 2, 209, 140, 2, + 2, 69, 82, 4, 18, 32, 67, 69, 2, 11, 83, 2, 17, 2, 69, 80, 2, 11, 65, 2, + 187, 244, 2, 82, 2, 255, 153, 2, 68, 2, 57, 12, 69, 82, 83, 65, 76, 32, + 82, 69, 67, 89, 67, 76, 2, 17, 2, 73, 78, 2, 155, 153, 2, 71, 2, 135, + 153, 2, 80, 144, 3, 130, 1, 32, 246, 7, 45, 180, 8, 4, 80, 69, 82, 32, + 132, 40, 8, 83, 73, 68, 69, 45, 68, 79, 87, 21, 6, 87, 65, 82, 68, 83, + 32, 38, 140, 1, 5, 65, 82, 82, 79, 87, 224, 1, 5, 66, 65, 82, 66, 32, + 228, 1, 5, 68, 79, 87, 78, 32, 210, 1, 70, 30, 82, 93, 4, 84, 65, 67, 75, + 8, 64, 4, 72, 69, 65, 68, 137, 51, 7, 32, 84, 72, 82, 79, 85, 71, 7, 11, + 32, 4, 112, 22, 66, 69, 84, 87, 69, 69, 78, 32, 84, 87, 79, 32, 72, 79, + 82, 73, 90, 79, 78, 84, 65, 76, 171, 155, 2, 73, 2, 193, 147, 2, 2, 32, + 66, 8, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 4, 57, 12, 84, 32, 68, + 79, 87, 78, 32, 66, 65, 82, 66, 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, + 71, 72, 2, 11, 84, 2, 25, 4, 32, 72, 65, 82, 2, 11, 80, 2, 231, 214, 2, + 79, 14, 76, 5, 65, 82, 82, 79, 87, 62, 66, 38, 68, 18, 83, 246, 58, 87, + 219, 9, 84, 5, 37, 7, 32, 87, 73, 84, 72, 32, 66, 2, 187, 156, 2, 65, 2, + 221, 138, 2, 4, 76, 65, 67, 75, 2, 191, 59, 79, 2, 179, 65, 65, 2, 11, + 73, 2, 223, 112, 83, 2, 53, 11, 73, 71, 72, 84, 32, 68, 73, 65, 71, 79, + 78, 2, 189, 128, 1, 4, 65, 76, 32, 69, 5, 29, 5, 32, 87, 73, 84, 72, 2, + 33, 6, 32, 67, 73, 82, 67, 76, 2, 175, 166, 2, 69, 32, 58, 70, 225, 1, 9, + 80, 79, 73, 78, 84, 73, 78, 71, 32, 4, 41, 8, 65, 67, 73, 78, 71, 32, 83, + 78, 4, 65, 14, 65, 75, 69, 32, 72, 69, 65, 68, 32, 87, 73, 84, 72, 32, 4, + 42, 79, 25, 6, 67, 76, 79, 83, 69, 68, 2, 21, 3, 80, 69, 78, 2, 21, 3, + 32, 77, 79, 2, 135, 159, 1, 85, 28, 130, 1, 65, 86, 69, 62, 70, 20, 8, + 77, 73, 76, 73, 84, 65, 82, 89, 26, 82, 88, 6, 83, 77, 65, 76, 76, 32, + 118, 84, 255, 121, 71, 4, 22, 84, 159, 2, 73, 2, 37, 7, 79, 77, 73, 67, + 32, 66, 79, 2, 219, 227, 1, 77, 2, 29, 5, 78, 69, 82, 71, 89, 2, 213, + 179, 1, 2, 32, 87, 2, 143, 158, 3, 82, 2, 129, 1, 2, 32, 65, 8, 64, 5, + 79, 67, 75, 69, 84, 118, 69, 222, 129, 1, 65, 247, 28, 73, 2, 239, 200, + 2, 32, 4, 18, 65, 67, 82, 2, 11, 73, 2, 11, 82, 2, 17, 2, 80, 76, 2, 255, + 131, 2, 65, 2, 11, 69, 2, 151, 248, 1, 68, 4, 37, 7, 82, 73, 65, 78, 71, + 76, 69, 4, 11, 32, 4, 11, 87, 4, 25, 4, 73, 84, 72, 32, 4, 18, 76, 27, + 82, 2, 11, 69, 2, 35, 70, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, + 72, 2, 21, 3, 65, 76, 70, 2, 17, 2, 32, 66, 2, 11, 76, 2, 143, 235, 1, + 65, 200, 1, 192, 1, 4, 65, 78, 68, 32, 210, 2, 66, 74, 67, 74, 70, 36, 5, + 72, 65, 76, 70, 32, 200, 4, 5, 76, 69, 70, 84, 32, 254, 5, 77, 150, 2, + 79, 60, 6, 82, 73, 71, 72, 84, 32, 234, 17, 83, 67, 84, 8, 34, 76, 53, 4, + 82, 73, 71, 72, 6, 48, 2, 69, 70, 173, 1, 5, 79, 87, 69, 82, 32, 2, 53, + 11, 84, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 2, 201, 35, 25, 32, 84, + 82, 73, 65, 78, 71, 85, 76, 65, 82, 32, 84, 72, 82, 69, 69, 32, 81, 85, + 65, 82, 84, 69, 82, 4, 28, 2, 84, 82, 231, 39, 79, 2, 225, 4, 7, 73, 65, + 78, 71, 85, 76, 65, 2, 17, 2, 76, 65, 2, 17, 2, 68, 69, 2, 185, 223, 1, + 3, 32, 83, 67, 10, 33, 6, 69, 78, 84, 82, 69, 32, 10, 174, 12, 76, 22, + 82, 251, 21, 79, 2, 17, 2, 73, 86, 2, 179, 31, 69, 20, 140, 1, 5, 66, 76, + 79, 67, 75, 110, 72, 32, 8, 73, 78, 86, 69, 82, 83, 69, 32, 198, 1, 86, + 170, 26, 77, 130, 3, 76, 22, 82, 139, 162, 1, 67, 5, 225, 28, 23, 32, 65, + 78, 68, 32, 76, 79, 87, 69, 82, 32, 72, 65, 76, 70, 32, 73, 78, 86, 69, + 82, 83, 69, 2, 225, 18, 4, 69, 65, 86, 89, 4, 100, 21, 77, 69, 68, 73, + 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, 68, 32, 76, 79, 87, 69, 51, + 87, 2, 11, 82, 2, 185, 35, 5, 32, 72, 65, 76, 70, 2, 21, 3, 72, 73, 84, + 2, 191, 191, 1, 69, 2, 29, 5, 69, 82, 84, 73, 67, 2, 197, 115, 14, 65, + 76, 32, 76, 73, 78, 69, 32, 87, 73, 84, 72, 32, 84, 62, 50, 66, 238, 3, + 67, 54, 79, 74, 84, 179, 13, 81, 22, 65, 14, 76, 79, 67, 75, 32, 68, 73, + 65, 71, 79, 78, 65, 76, 32, 22, 144, 1, 6, 76, 79, 87, 69, 82, 32, 233, + 8, 24, 85, 80, 80, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, + 84, 32, 84, 79, 32, 85, 80, 80, 18, 176, 1, 10, 67, 69, 78, 84, 82, 69, + 32, 84, 79, 32, 36, 8, 76, 69, 70, 84, 32, 84, 79, 32, 229, 10, 18, 77, + 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 85, 80, 80, 6, + 70, 76, 253, 8, 3, 85, 80, 80, 6, 34, 76, 213, 9, 3, 85, 80, 80, 2, 149, + 10, 2, 79, 87, 2, 29, 5, 79, 82, 78, 69, 82, 2, 155, 232, 1, 32, 4, 182, + 12, 78, 53, 12, 82, 32, 76, 79, 87, 69, 82, 32, 82, 73, 71, 72, 8, 34, + 79, 166, 19, 82, 155, 1, 87, 2, 237, 18, 11, 32, 76, 79, 87, 69, 82, 32, + 82, 73, 71, 72, 12, 33, 6, 73, 68, 68, 76, 69, 32, 12, 52, 7, 67, 69, 78, + 84, 82, 69, 32, 74, 76, 23, 82, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, + 72, 2, 117, 3, 84, 32, 79, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 17, + 2, 84, 32, 4, 30, 79, 253, 17, 2, 84, 87, 2, 139, 9, 78, 4, 11, 78, 4, + 17, 2, 69, 32, 4, 150, 21, 81, 147, 4, 69, 68, 84, 2, 66, 76, 186, 6, 68, + 114, 79, 222, 1, 80, 38, 81, 224, 4, 2, 83, 72, 51, 84, 22, 61, 13, 79, + 67, 75, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 22, 140, 1, 24, 76, 79, + 87, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, + 32, 76, 79, 87, 57, 6, 85, 80, 80, 69, 82, 32, 4, 21, 3, 69, 82, 32, 4, + 246, 3, 67, 135, 215, 2, 82, 18, 176, 1, 10, 67, 69, 78, 84, 82, 69, 32, + 84, 79, 32, 92, 8, 76, 69, 70, 84, 32, 84, 79, 32, 141, 1, 18, 77, 73, + 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 76, 79, 87, 6, 32, 3, + 76, 79, 87, 139, 1, 85, 4, 21, 3, 69, 82, 32, 4, 142, 2, 77, 223, 214, 2, + 82, 6, 28, 3, 76, 79, 87, 51, 85, 4, 21, 3, 69, 82, 32, 4, 142, 1, 67, + 43, 77, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 117, 2, 32, 77, 6, 21, 3, + 69, 82, 32, 6, 34, 67, 42, 77, 223, 214, 2, 82, 2, 11, 69, 2, 141, 225, + 1, 2, 78, 84, 2, 25, 4, 73, 68, 68, 76, 2, 139, 176, 1, 69, 2, 57, 12, + 82, 79, 80, 45, 83, 72, 65, 68, 79, 87, 69, 68, 2, 17, 2, 32, 87, 2, 213, + 190, 1, 3, 72, 73, 84, 4, 62, 78, 53, 11, 82, 32, 76, 79, 87, 69, 82, 32, + 76, 69, 70, 2, 225, 16, 9, 69, 32, 83, 73, 88, 84, 69, 69, 78, 2, 69, 15, + 84, 32, 67, 85, 82, 76, 89, 32, 66, 82, 65, 67, 75, 69, 84, 2, 11, 32, 2, + 219, 173, 2, 83, 2, 11, 69, 2, 205, 74, 2, 78, 67, 26, 17, 2, 85, 65, 26, + 44, 6, 68, 82, 65, 78, 84, 32, 255, 3, 82, 24, 90, 67, 112, 10, 70, 65, + 67, 69, 32, 87, 73, 84, 72, 32, 114, 77, 76, 2, 83, 84, 67, 84, 14, 68, + 7, 73, 82, 67, 85, 76, 65, 82, 145, 167, 1, 4, 72, 69, 83, 83, 2, 17, 2, + 32, 65, 2, 147, 212, 1, 82, 4, 34, 67, 45, 4, 79, 80, 69, 78, 2, 21, 3, + 76, 79, 83, 2, 17, 2, 69, 68, 2, 177, 231, 1, 3, 32, 69, 89, 2, 25, 4, + 73, 67, 82, 79, 2, 11, 67, 2, 233, 194, 1, 4, 79, 77, 80, 85, 2, 41, 8, + 65, 78, 68, 73, 78, 71, 32, 75, 2, 195, 207, 2, 78, 2, 21, 3, 69, 76, 69, + 2, 11, 86, 2, 187, 90, 73, 2, 181, 167, 1, 3, 84, 69, 82, 4, 153, 102, 8, + 65, 68, 79, 87, 69, 68, 32, 87, 8, 30, 79, 106, 82, 155, 1, 87, 2, 49, + 10, 32, 76, 79, 87, 69, 82, 32, 76, 69, 70, 2, 11, 84, 2, 11, 32, 2, 11, + 70, 2, 195, 112, 73, 4, 25, 4, 73, 65, 78, 71, 4, 40, 4, 85, 76, 65, 82, + 219, 224, 2, 76, 2, 17, 2, 32, 77, 2, 41, 8, 69, 68, 73, 85, 77, 32, 83, + 72, 2, 223, 102, 65, 2, 21, 3, 69, 76, 70, 2, 11, 84, 2, 163, 164, 1, 72, + 2, 17, 2, 69, 86, 2, 17, 2, 69, 78, 2, 145, 1, 2, 32, 69, 10, 64, 5, 72, + 82, 69, 69, 32, 193, 1, 6, 82, 73, 65, 78, 71, 85, 8, 54, 69, 49, 9, 81, + 85, 65, 82, 84, 69, 82, 83, 32, 2, 29, 5, 73, 71, 72, 84, 72, 2, 243, 5, + 83, 6, 30, 76, 22, 82, 203, 5, 66, 2, 41, 2, 69, 70, 2, 21, 3, 73, 71, + 72, 2, 43, 84, 2, 17, 2, 76, 65, 2, 11, 82, 2, 17, 2, 32, 79, 2, 25, 4, + 78, 69, 32, 81, 2, 185, 4, 6, 85, 65, 82, 84, 69, 82, 2, 199, 181, 2, 78, + 128, 1, 154, 1, 65, 184, 6, 2, 66, 76, 150, 1, 68, 50, 70, 82, 72, 150, + 4, 67, 46, 81, 42, 82, 22, 83, 102, 84, 146, 7, 80, 173, 3, 6, 87, 72, + 73, 84, 69, 32, 32, 34, 78, 33, 4, 82, 82, 79, 87, 2, 11, 67, 2, 195, + 172, 2, 79, 31, 11, 32, 28, 134, 1, 65, 192, 1, 14, 76, 69, 70, 84, 87, + 65, 82, 68, 83, 32, 79, 70, 32, 68, 32, 5, 87, 73, 84, 72, 32, 210, 8, + 70, 179, 5, 84, 2, 41, 8, 78, 68, 32, 82, 73, 71, 72, 84, 2, 17, 2, 32, + 79, 2, 25, 4, 78, 69, 32, 69, 2, 21, 3, 73, 71, 72, 2, 17, 2, 84, 72, 2, + 11, 32, 2, 11, 66, 2, 11, 76, 2, 191, 190, 1, 79, 2, 197, 206, 1, 3, 79, + 87, 78, 20, 74, 68, 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 250, 12, 84, + 139, 58, 72, 2, 21, 3, 79, 85, 66, 2, 11, 76, 2, 247, 188, 2, 69, 2, 11, + 65, 2, 253, 22, 3, 82, 71, 69, 2, 225, 22, 5, 69, 68, 73, 85, 77, 2, 25, + 4, 79, 84, 67, 72, 2, 11, 69, 2, 139, 56, 68, 4, 11, 77, 4, 25, 4, 65, + 76, 76, 32, 4, 22, 69, 223, 21, 84, 2, 129, 22, 10, 81, 85, 73, 76, 65, + 84, 69, 82, 65, 76, 4, 25, 4, 65, 67, 75, 32, 4, 26, 67, 139, 208, 1, 65, + 2, 25, 4, 73, 82, 67, 76, 2, 25, 4, 69, 68, 32, 87, 2, 11, 72, 2, 237, + 14, 2, 73, 84, 4, 22, 79, 195, 13, 65, 2, 177, 14, 2, 85, 66, 2, 11, 73, + 2, 37, 7, 78, 71, 69, 82, 45, 80, 79, 2, 173, 206, 1, 2, 83, 84, 20, 88, + 17, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 32, + 131, 3, 69, 16, 56, 4, 76, 69, 70, 84, 245, 1, 5, 82, 73, 71, 72, 84, 10, + 22, 32, 239, 9, 87, 8, 60, 7, 66, 69, 83, 73, 68, 69, 32, 206, 1, 70, + 179, 5, 84, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 197, 149, 1, 23, + 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, + 32, 66, 65, 82, 66, 6, 22, 32, 251, 7, 87, 4, 22, 70, 179, 5, 84, 2, 217, + 195, 1, 3, 82, 79, 77, 4, 25, 4, 65, 86, 89, 32, 4, 26, 67, 155, 202, 1, + 65, 2, 189, 10, 7, 79, 77, 80, 82, 69, 83, 83, 2, 137, 9, 6, 85, 65, 68, + 82, 85, 80, 2, 171, 145, 2, 79, 4, 30, 65, 53, 3, 81, 85, 65, 2, 245, + 200, 1, 8, 78, 83, 45, 83, 69, 82, 73, 70, 2, 131, 9, 82, 36, 36, 2, 82, + 73, 229, 7, 2, 87, 79, 30, 40, 5, 65, 78, 71, 76, 69, 155, 7, 80, 28, 52, + 8, 45, 72, 69, 65, 68, 69, 68, 32, 219, 12, 32, 26, 48, 5, 65, 82, 82, + 79, 87, 174, 5, 68, 39, 80, 23, 11, 32, 20, 92, 17, 76, 69, 70, 84, 87, + 65, 82, 68, 83, 32, 79, 70, 32, 68, 79, 87, 78, 98, 84, 23, 87, 2, 37, 7, + 87, 65, 82, 68, 83, 32, 84, 2, 37, 7, 82, 73, 65, 78, 71, 76, 69, 2, 219, + 5, 45, 2, 171, 190, 1, 79, 16, 25, 4, 73, 84, 72, 32, 16, 114, 66, 28, 6, + 76, 79, 78, 71, 32, 84, 130, 1, 77, 34, 78, 34, 86, 34, 72, 149, 56, 6, + 68, 79, 85, 66, 76, 69, 2, 149, 2, 3, 79, 76, 68, 4, 17, 2, 73, 80, 4, + 11, 32, 4, 34, 76, 21, 4, 82, 73, 71, 72, 2, 17, 2, 69, 70, 2, 11, 84, 2, + 11, 87, 2, 243, 125, 65, 2, 121, 5, 69, 68, 73, 85, 77, 2, 89, 5, 65, 82, + 82, 79, 87, 2, 29, 5, 69, 82, 89, 32, 72, 2, 25, 4, 69, 65, 86, 89, 2, + 221, 177, 2, 4, 32, 83, 72, 65, 2, 11, 65, 2, 249, 1, 2, 83, 72, 2, 11, + 65, 2, 25, 4, 73, 82, 69, 68, 2, 29, 5, 32, 65, 82, 82, 79, 2, 211, 157, + 2, 87, 2, 11, 76, 2, 187, 192, 1, 69, 6, 74, 32, 61, 14, 45, 72, 69, 65, + 68, 69, 68, 32, 65, 82, 82, 79, 87, 32, 2, 25, 4, 72, 69, 65, 68, 2, 11, + 69, 2, 179, 191, 1, 68, 4, 42, 87, 253, 134, 1, 4, 70, 82, 79, 77, 2, 33, + 6, 73, 84, 72, 32, 84, 82, 2, 61, 13, 73, 65, 78, 71, 76, 69, 32, 65, 82, + 82, 79, 87, 72, 2, 149, 121, 2, 69, 65, 18, 88, 5, 65, 82, 82, 79, 87, + 149, 3, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 15, 11, 32, + 12, 108, 11, 79, 78, 32, 80, 69, 68, 69, 83, 84, 65, 76, 106, 87, 217, + 179, 1, 8, 70, 82, 79, 77, 32, 66, 65, 82, 7, 33, 6, 32, 87, 73, 84, 72, + 32, 4, 26, 86, 171, 180, 1, 72, 2, 205, 180, 1, 5, 69, 82, 84, 73, 67, 2, + 29, 5, 73, 84, 72, 73, 78, 2, 17, 2, 32, 84, 2, 37, 7, 82, 73, 65, 78, + 71, 76, 69, 2, 11, 32, 2, 11, 65, 2, 25, 4, 82, 82, 79, 87, 2, 11, 72, 2, + 195, 142, 2, 69, 5, 45, 9, 32, 79, 78, 32, 80, 69, 68, 69, 83, 2, 235, + 202, 1, 84, 240, 15, 86, 65, 254, 16, 83, 174, 3, 69, 234, 45, 73, 186, + 8, 79, 134, 2, 84, 21, 2, 85, 76, 218, 8, 116, 2, 73, 32, 128, 16, 17, + 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 84, 79, 82, 45, 245, + 45, 2, 77, 80, 216, 4, 54, 67, 34, 70, 82, 81, 40, 2, 83, 89, 207, 95, + 68, 2, 11, 79, 2, 231, 176, 2, 77, 2, 11, 85, 2, 11, 76, 2, 11, 76, 2, + 11, 32, 2, 11, 83, 2, 147, 216, 1, 84, 2, 17, 2, 85, 69, 2, 215, 231, 1, + 83, 190, 4, 68, 7, 76, 76, 65, 66, 76, 69, 32, 197, 11, 5, 77, 66, 79, + 76, 32, 164, 4, 214, 1, 68, 70, 66, 2, 83, 2, 84, 2, 90, 70, 71, 122, 72, + 82, 75, 134, 1, 76, 130, 1, 77, 86, 78, 134, 3, 67, 2, 70, 2, 74, 2, 80, + 2, 82, 2, 86, 2, 89, 78, 87, 50, 69, 34, 79, 250, 73, 65, 2, 73, 3, 85, + 42, 66, 72, 162, 8, 79, 194, 133, 2, 69, 162, 64, 65, 2, 73, 3, 85, 28, + 230, 7, 72, 58, 79, 194, 133, 2, 69, 162, 64, 65, 2, 73, 3, 85, 34, 62, + 66, 202, 6, 69, 86, 79, 226, 197, 2, 65, 2, 73, 3, 85, 18, 106, 79, 222, + 5, 69, 182, 198, 2, 65, 2, 73, 3, 85, 24, 50, 79, 222, 5, 69, 178, 75, + 65, 2, 73, 3, 85, 7, 142, 204, 2, 78, 3, 79, 34, 70, 80, 206, 5, 79, 222, + 74, 65, 230, 186, 1, 69, 162, 64, 73, 3, 85, 18, 246, 4, 69, 86, 79, 222, + 74, 65, 134, 251, 1, 73, 3, 85, 16, 54, 69, 218, 4, 79, 226, 197, 2, 65, + 2, 73, 3, 85, 7, 26, 78, 159, 202, 2, 69, 2, 21, 3, 71, 84, 72, 2, 139, + 140, 2, 69, 42, 214, 3, 66, 0, 2, 71, 66, 58, 79, 194, 133, 2, 69, 162, + 64, 65, 2, 73, 3, 85, 90, 90, 68, 174, 1, 71, 126, 74, 2, 89, 58, 79, + 194, 133, 2, 69, 162, 64, 65, 2, 73, 3, 85, 24, 54, 79, 234, 135, 2, 69, + 162, 64, 65, 2, 73, 3, 85, 15, 36, 3, 76, 69, 32, 227, 199, 2, 79, 10, + 54, 83, 182, 168, 2, 68, 190, 28, 70, 2, 75, 3, 77, 2, 179, 168, 2, 79, + 25, 42, 71, 138, 247, 1, 65, 2, 69, 3, 79, 16, 50, 69, 86, 79, 226, 197, + 2, 65, 2, 73, 3, 85, 7, 178, 198, 2, 69, 3, 78, 14, 54, 79, 194, 133, 2, + 69, 162, 64, 65, 2, 73, 3, 85, 5, 223, 197, 2, 79, 28, 46, 69, 34, 79, + 250, 73, 65, 2, 73, 3, 85, 9, 150, 74, 69, 135, 251, 1, 78, 9, 246, 73, + 79, 135, 251, 1, 78, 26, 66, 68, 62, 70, 30, 74, 22, 75, 50, 78, 22, 84, + 151, 234, 1, 66, 6, 26, 79, 135, 243, 1, 65, 4, 130, 243, 1, 79, 135, 50, + 45, 4, 74, 69, 219, 192, 2, 65, 2, 199, 242, 1, 79, 4, 26, 69, 155, 242, + 1, 85, 2, 151, 242, 1, 69, 2, 163, 175, 2, 73, 6, 238, 241, 1, 73, 2, 79, + 195, 78, 65, 128, 4, 74, 49, 94, 50, 98, 51, 2, 52, 2, 53, 2, 54, 2, 55, + 2, 56, 3, 57, 223, 1, 182, 1, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 137, 1, 90, 48, 2, 49, 2, 50, 2, 51, 2, 52, 94, + 53, 218, 191, 2, 54, 2, 55, 2, 56, 3, 57, 23, 178, 192, 2, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 17, 214, 191, 2, 48, + 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 194, 1, 116, 4, 68, 73, 67, 32, + 206, 19, 82, 168, 155, 1, 13, 67, 84, 79, 82, 32, 79, 82, 32, 67, 82, 79, + 83, 83, 219, 106, 83, 86, 60, 5, 83, 73, 71, 78, 32, 153, 10, 5, 84, 79, + 78, 69, 32, 48, 218, 2, 65, 216, 1, 17, 68, 79, 85, 66, 76, 69, 32, 65, + 78, 85, 83, 86, 65, 82, 65, 32, 65, 98, 74, 52, 6, 78, 73, 72, 83, 72, + 86, 22, 82, 240, 1, 8, 72, 69, 88, 73, 70, 79, 82, 77, 22, 76, 52, 4, 84, + 73, 82, 89, 22, 85, 60, 8, 86, 73, 83, 65, 82, 71, 65, 32, 237, 6, 17, + 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, 77, 73, 68, 76, 73, 78, 14, + 76, 8, 78, 85, 83, 86, 65, 82, 65, 32, 212, 1, 3, 84, 73, 75, 151, 2, 82, + 10, 134, 1, 65, 24, 4, 66, 65, 72, 73, 24, 9, 85, 66, 72, 65, 89, 65, 84, + 79, 32, 201, 4, 10, 86, 65, 77, 65, 71, 79, 77, 85, 75, 72, 2, 21, 3, 78, + 84, 65, 2, 21, 3, 82, 71, 79, 2, 11, 77, 2, 163, 9, 85, 2, 235, 245, 1, + 82, 2, 33, 6, 73, 72, 86, 65, 77, 85, 2, 151, 3, 76, 2, 203, 181, 2, 65, + 8, 144, 1, 15, 69, 86, 69, 82, 83, 69, 68, 32, 86, 73, 83, 65, 82, 71, + 65, 36, 9, 79, 84, 65, 84, 69, 68, 32, 65, 82, 57, 5, 84, 72, 65, 78, 71, + 4, 11, 32, 4, 218, 6, 65, 23, 85, 2, 25, 4, 68, 72, 65, 86, 2, 201, 243, + 1, 2, 73, 83, 2, 17, 2, 32, 76, 2, 21, 3, 79, 78, 71, 2, 229, 240, 1, 2, + 32, 65, 2, 171, 178, 2, 65, 2, 37, 7, 80, 65, 68, 72, 77, 65, 78, 2, 171, + 178, 2, 73, 10, 40, 3, 65, 78, 85, 2, 85, 179, 9, 83, 4, 25, 4, 68, 65, + 84, 84, 4, 11, 65, 5, 25, 4, 32, 87, 73, 84, 2, 11, 72, 2, 11, 32, 2, 11, + 84, 2, 11, 65, 2, 159, 170, 1, 73, 38, 128, 2, 5, 67, 65, 78, 68, 82, 16, + 2, 68, 79, 96, 2, 75, 65, 136, 1, 4, 80, 82, 69, 78, 16, 2, 82, 73, 106, + 84, 164, 1, 11, 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, 180, 1, 12, + 65, 84, 72, 65, 82, 86, 65, 86, 69, 68, 73, 67, 169, 231, 1, 2, 83, 72, + 4, 131, 19, 65, 6, 40, 5, 85, 66, 76, 69, 32, 191, 3, 84, 4, 22, 82, 211, + 5, 83, 2, 11, 73, 2, 251, 1, 78, 4, 56, 3, 82, 83, 72, 17, 7, 84, 72, 65, + 75, 65, 32, 65, 2, 139, 116, 65, 2, 17, 2, 78, 85, 2, 17, 2, 68, 65, 2, + 139, 139, 2, 84, 2, 131, 53, 75, 4, 82, 78, 237, 2, 15, 71, 86, 69, 68, + 73, 67, 32, 75, 65, 83, 72, 77, 73, 82, 73, 2, 219, 177, 1, 71, 6, 42, + 72, 24, 4, 82, 73, 80, 76, 19, 87, 2, 49, 3, 82, 69, 69, 2, 219, 2, 69, + 2, 11, 79, 2, 25, 4, 32, 68, 79, 84, 2, 11, 83, 2, 11, 32, 2, 167, 15, + 66, 8, 176, 1, 10, 65, 71, 71, 82, 65, 86, 65, 84, 69, 68, 22, 73, 105, + 27, 75, 65, 84, 72, 65, 75, 65, 32, 73, 78, 68, 69, 80, 69, 78, 68, 69, + 78, 84, 32, 83, 86, 65, 82, 73, 84, 65, 2, 17, 2, 32, 73, 2, 49, 10, 78, + 68, 69, 80, 69, 78, 68, 69, 78, 84, 2, 17, 2, 32, 83, 2, 189, 134, 2, 3, + 86, 65, 82, 5, 241, 10, 7, 32, 83, 67, 72, 82, 79, 69, 104, 62, 83, 16, + 6, 84, 73, 67, 65, 76, 32, 181, 17, 2, 89, 32, 2, 167, 88, 73, 76, 200, + 2, 4, 66, 65, 82, 32, 174, 3, 67, 42, 69, 62, 70, 38, 71, 32, 11, 73, 68, + 69, 79, 71, 82, 65, 80, 72, 73, 67, 48, 12, 75, 65, 78, 65, 32, 82, 69, + 80, 69, 65, 84, 32, 254, 1, 76, 194, 3, 77, 88, 17, 79, 78, 69, 32, 69, + 73, 71, 72, 84, 72, 32, 66, 76, 79, 67, 75, 45, 62, 82, 158, 1, 84, 218, + 1, 90, 173, 72, 3, 83, 73, 88, 8, 108, 6, 66, 69, 83, 73, 68, 69, 64, 8, + 68, 79, 85, 66, 76, 69, 32, 76, 20, 4, 84, 82, 73, 80, 143, 1, 87, 2, 17, + 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 159, 129, 1, 84, 2, 69, 2, 69, 70, 2, + 25, 4, 76, 69, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 29, 5, 32, 84, + 85, 82, 78, 2, 11, 83, 2, 11, 84, 2, 203, 128, 1, 73, 2, 21, 3, 73, 84, + 72, 2, 17, 2, 32, 72, 2, 137, 244, 1, 7, 79, 82, 73, 90, 79, 78, 84, 2, + 141, 235, 1, 5, 65, 80, 65, 67, 73, 2, 25, 4, 76, 76, 73, 80, 2, 11, 83, + 2, 215, 229, 1, 73, 2, 11, 79, 2, 213, 82, 2, 85, 82, 2, 221, 155, 1, 3, + 79, 45, 75, 2, 17, 2, 32, 73, 2, 133, 185, 1, 2, 84, 69, 10, 120, 4, 77, + 65, 82, 75, 45, 22, 87, 73, 84, 72, 32, 86, 79, 73, 67, 69, 68, 32, 83, + 79, 85, 78, 68, 32, 77, 65, 82, 75, 7, 11, 32, 4, 50, 85, 21, 3, 76, 79, + 87, 5, 17, 2, 32, 85, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 153, 29, 2, + 32, 72, 16, 30, 65, 33, 3, 73, 78, 69, 2, 11, 68, 2, 151, 226, 1, 68, 15, + 11, 32, 12, 38, 69, 49, 5, 87, 73, 84, 72, 32, 2, 25, 4, 88, 84, 69, 78, + 2, 219, 206, 1, 83, 10, 56, 4, 67, 73, 82, 67, 98, 70, 26, 84, 151, 140, + 1, 77, 4, 11, 76, 4, 11, 69, 4, 11, 32, 4, 26, 66, 167, 161, 1, 65, 2, + 11, 69, 2, 223, 131, 1, 76, 2, 57, 3, 79, 85, 82, 2, 11, 72, 2, 21, 3, + 82, 69, 69, 2, 45, 9, 32, 84, 73, 67, 75, 32, 77, 65, 82, 2, 167, 223, 1, + 75, 2, 65, 14, 65, 76, 69, 32, 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 2, + 171, 149, 1, 69, 12, 150, 156, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, + 4, 42, 65, 57, 6, 69, 83, 73, 83, 84, 79, 2, 25, 4, 67, 73, 78, 71, 2, + 11, 32, 2, 207, 121, 67, 2, 21, 3, 82, 32, 83, 2, 11, 69, 2, 159, 215, 1, + 71, 10, 32, 2, 65, 66, 106, 73, 19, 82, 6, 18, 32, 35, 85, 2, 11, 75, 2, + 179, 136, 2, 69, 4, 33, 6, 76, 65, 84, 73, 79, 78, 5, 251, 45, 32, 2, + 167, 9, 76, 2, 29, 5, 65, 70, 70, 73, 67, 2, 177, 238, 1, 2, 32, 76, 2, + 233, 106, 4, 73, 71, 90, 65, 26, 80, 6, 72, 69, 65, 86, 89, 32, 176, 5, + 3, 77, 85, 67, 241, 9, 3, 66, 79, 76, 20, 62, 69, 190, 1, 70, 38, 82, 82, + 83, 246, 1, 87, 215, 10, 71, 4, 25, 4, 73, 71, 72, 84, 4, 11, 32, 4, 22, + 80, 223, 2, 83, 2, 25, 4, 79, 73, 78, 84, 2, 17, 2, 69, 68, 2, 17, 2, 32, + 66, 2, 25, 4, 76, 65, 67, 75, 2, 11, 32, 2, 215, 79, 83, 2, 11, 73, 2, + 185, 1, 2, 86, 69, 2, 11, 69, 2, 29, 5, 86, 69, 82, 83, 69, 2, 17, 2, 32, + 83, 2, 231, 1, 79, 6, 30, 65, 42, 73, 147, 1, 79, 2, 11, 76, 2, 11, 84, + 2, 147, 116, 73, 2, 11, 88, 2, 11, 32, 2, 11, 83, 2, 21, 3, 80, 79, 75, + 2, 17, 2, 69, 68, 2, 11, 32, 2, 11, 65, 2, 11, 83, 2, 205, 78, 3, 84, 69, + 82, 2, 173, 13, 3, 76, 73, 68, 4, 21, 3, 72, 73, 84, 4, 11, 69, 4, 11, + 32, 4, 226, 64, 67, 163, 49, 83, 4, 11, 72, 4, 11, 32, 4, 18, 71, 39, 76, + 2, 69, 6, 82, 69, 65, 84, 69, 82, 2, 11, 69, 2, 11, 83, 2, 11, 83, 2, 17, + 2, 45, 84, 2, 187, 71, 72, 160, 1, 140, 1, 9, 66, 82, 65, 84, 73, 79, 78, + 32, 77, 34, 67, 36, 3, 68, 69, 79, 126, 69, 222, 1, 79, 22, 82, 21, 7, + 84, 72, 75, 85, 81, 73, 32, 2, 11, 79, 2, 183, 249, 1, 68, 2, 221, 129, + 1, 4, 84, 79, 82, 89, 6, 30, 32, 77, 3, 67, 65, 83, 4, 18, 67, 43, 71, 2, + 17, 2, 65, 77, 2, 203, 202, 1, 69, 2, 139, 119, 65, 2, 219, 71, 83, 6, + 168, 1, 31, 84, 78, 65, 77, 69, 83, 69, 32, 65, 76, 84, 69, 82, 78, 65, + 84, 69, 32, 82, 69, 65, 68, 73, 78, 71, 32, 77, 65, 82, 75, 32, 189, 108, + 5, 87, 68, 65, 84, 65, 4, 26, 78, 219, 138, 2, 67, 2, 139, 129, 1, 72, 2, + 167, 141, 1, 76, 2, 243, 237, 1, 71, 140, 1, 56, 6, 67, 65, 80, 73, 84, + 65, 1, 4, 83, 77, 65, 76, 70, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, + 70, 230, 1, 66, 34, 69, 22, 73, 22, 76, 34, 78, 246, 136, 1, 67, 2, 68, + 2, 83, 2, 84, 226, 56, 72, 238, 48, 70, 2, 74, 2, 77, 2, 80, 2, 82, 2, + 86, 2, 88, 2, 90, 158, 20, 71, 2, 75, 2, 81, 186, 2, 65, 2, 79, 2, 85, 3, + 89, 4, 166, 243, 1, 66, 215, 22, 69, 5, 219, 137, 2, 73, 5, 243, 242, 1, + 74, 4, 250, 134, 2, 76, 187, 2, 65, 4, 190, 242, 1, 74, 215, 22, 69, 8, + 28, 3, 73, 68, 69, 59, 76, 2, 21, 3, 68, 32, 71, 2, 141, 71, 4, 82, 69, + 69, 75, 6, 46, 67, 20, 3, 76, 69, 89, 41, 2, 85, 77, 2, 219, 222, 1, 65, + 2, 11, 66, 2, 11, 65, 2, 139, 126, 76, 2, 11, 69, 2, 17, 2, 32, 73, 2, + 193, 125, 4, 78, 84, 69, 71, 5, 239, 134, 2, 83, 40, 70, 67, 45, 13, 71, + 65, 82, 32, 70, 82, 65, 67, 84, 73, 79, 78, 32, 2, 11, 65, 2, 11, 78, 2, + 159, 200, 1, 85, 38, 106, 70, 96, 4, 79, 78, 69, 32, 178, 2, 84, 80, 7, + 83, 69, 86, 69, 78, 32, 69, 153, 33, 3, 90, 69, 82, 6, 56, 4, 73, 86, 69, + 32, 253, 3, 5, 79, 85, 82, 32, 70, 4, 162, 3, 69, 113, 3, 83, 73, 88, 18, + 66, 69, 28, 2, 70, 73, 18, 72, 34, 78, 14, 81, 30, 83, 55, 84, 2, 197, 1, + 3, 73, 71, 72, 2, 171, 1, 70, 2, 11, 65, 2, 251, 129, 1, 76, 2, 111, 73, + 2, 133, 75, 3, 85, 65, 82, 4, 24, 2, 69, 86, 15, 73, 2, 43, 69, 2, 43, + 88, 4, 18, 69, 35, 72, 2, 11, 78, 2, 255, 128, 2, 84, 2, 143, 107, 73, + 10, 48, 5, 72, 82, 69, 69, 32, 93, 3, 87, 79, 32, 6, 26, 69, 26, 81, 67, + 70, 2, 109, 3, 73, 71, 72, 2, 21, 3, 85, 65, 82, 2, 139, 114, 84, 4, 22, + 70, 219, 32, 84, 2, 11, 73, 2, 11, 70, 2, 205, 194, 1, 2, 84, 72, 164, 6, + 86, 65, 250, 23, 69, 158, 3, 72, 250, 68, 73, 234, 8, 79, 134, 6, 82, + 191, 144, 1, 74, 204, 2, 114, 70, 18, 78, 220, 5, 2, 88, 73, 158, 1, 82, + 210, 10, 84, 194, 1, 86, 221, 152, 1, 6, 83, 84, 69, 66, 65, 83, 2, 147, + 90, 70, 122, 36, 4, 67, 72, 79, 32, 183, 5, 73, 118, 100, 7, 76, 69, 84, + 84, 69, 82, 32, 252, 3, 3, 78, 71, 85, 16, 5, 84, 79, 78, 69, 32, 243, 7, + 68, 88, 210, 1, 65, 38, 79, 34, 69, 22, 73, 22, 76, 50, 78, 42, 84, 50, + 85, 22, 89, 130, 178, 1, 75, 2, 80, 2, 83, 138, 69, 66, 2, 67, 2, 68, 2, + 70, 2, 71, 2, 72, 2, 74, 2, 77, 2, 82, 2, 86, 2, 87, 3, 90, 13, 34, 65, + 210, 250, 1, 78, 87, 85, 7, 11, 78, 5, 151, 251, 1, 71, 5, 131, 251, 1, + 78, 5, 155, 250, 1, 78, 4, 26, 76, 195, 250, 1, 65, 2, 135, 248, 1, 72, + 6, 242, 247, 1, 71, 2, 89, 187, 2, 65, 8, 202, 247, 1, 72, 2, 82, 2, 83, + 187, 2, 65, 5, 243, 169, 1, 69, 4, 174, 248, 1, 73, 147, 1, 65, 2, 159, + 114, 78, 8, 40, 3, 75, 79, 73, 1, 3, 84, 85, 80, 5, 139, 229, 1, 78, 4, + 21, 3, 78, 71, 32, 4, 76, 8, 67, 82, 69, 83, 67, 69, 78, 84, 1, 7, 71, + 73, 66, 66, 79, 85, 83, 2, 21, 3, 32, 77, 79, 2, 11, 79, 2, 155, 100, 78, + 170, 1, 72, 9, 65, 78, 71, 32, 67, 73, 84, 73, 32, 229, 111, 4, 78, 73, + 78, 71, 168, 1, 128, 1, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, + 190, 4, 68, 156, 2, 7, 78, 85, 77, 66, 69, 82, 32, 175, 222, 1, 79, 64, + 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 64, 174, 1, 65, 38, 69, 42, + 72, 74, 78, 50, 79, 22, 83, 50, 85, 30, 89, 226, 42, 84, 180, 110, 2, 86, + 73, 222, 51, 66, 2, 80, 246, 5, 75, 158, 11, 73, 2, 87, 162, 17, 68, 3, + 71, 9, 166, 243, 1, 78, 86, 77, 3, 84, 7, 11, 78, 4, 202, 243, 1, 78, 3, + 89, 8, 38, 79, 198, 154, 1, 73, 235, 31, 65, 4, 170, 186, 1, 82, 235, 25, + 76, 4, 26, 71, 215, 161, 1, 85, 2, 147, 240, 1, 65, 5, 151, 172, 1, 68, + 4, 26, 83, 179, 222, 1, 73, 2, 159, 205, 1, 85, 4, 242, 241, 1, 67, 3, + 85, 8, 34, 85, 182, 241, 1, 65, 3, 79, 5, 179, 241, 1, 74, 20, 11, 73, + 20, 11, 71, 20, 17, 2, 73, 84, 20, 11, 32, 20, 66, 70, 30, 83, 42, 84, + 62, 90, 238, 85, 78, 14, 79, 223, 110, 69, 4, 206, 115, 73, 131, 4, 79, + 4, 22, 69, 207, 99, 73, 2, 135, 28, 86, 4, 26, 72, 211, 208, 1, 87, 2, + 11, 82, 2, 227, 216, 1, 69, 2, 11, 69, 2, 163, 208, 1, 82, 18, 42, 69, + 30, 70, 42, 78, 38, 83, 39, 84, 2, 221, 1, 3, 73, 71, 72, 4, 22, 73, 139, + 1, 79, 2, 171, 1, 70, 2, 17, 2, 73, 78, 2, 135, 1, 69, 4, 92, 2, 69, 86, + 25, 2, 73, 88, 6, 34, 72, 26, 87, 179, 157, 1, 69, 2, 11, 73, 2, 35, 82, + 2, 11, 69, 2, 11, 78, 2, 175, 219, 1, 84, 12, 32, 2, 69, 82, 179, 235, 1, + 67, 10, 34, 32, 165, 156, 1, 2, 77, 69, 8, 80, 3, 67, 76, 79, 22, 87, + 228, 200, 1, 5, 66, 85, 70, 70, 65, 1, 2, 80, 79, 2, 175, 182, 1, 83, 2, + 223, 110, 65, 20, 60, 2, 69, 32, 124, 4, 73, 78, 71, 32, 161, 1, 2, 89, + 32, 6, 182, 2, 68, 161, 189, 1, 23, 65, 82, 82, 79, 87, 32, 80, 79, 73, + 78, 84, 73, 78, 71, 32, 68, 73, 82, 69, 67, 84, 76, 89, 6, 64, 5, 66, 76, + 65, 67, 75, 0, 5, 87, 72, 73, 84, 69, 55, 72, 2, 17, 2, 32, 70, 2, 11, + 76, 2, 175, 232, 1, 65, 2, 11, 65, 2, 11, 78, 2, 215, 97, 68, 8, 26, 68, + 34, 76, 43, 79, 2, 11, 65, 2, 143, 231, 1, 83, 4, 22, 79, 203, 78, 73, 2, + 163, 78, 87, 2, 11, 86, 2, 11, 69, 2, 135, 78, 82, 14, 48, 3, 65, 82, 89, + 70, 68, 70, 73, 171, 1, 83, 4, 11, 32, 4, 32, 2, 67, 65, 211, 169, 1, 70, + 2, 183, 169, 1, 84, 4, 26, 71, 167, 149, 1, 68, 2, 129, 70, 6, 69, 45, + 84, 65, 73, 76, 4, 116, 17, 69, 82, 83, 84, 82, 65, 83, 83, 32, 69, 76, + 76, 73, 80, 84, 73, 67, 145, 45, 7, 71, 72, 84, 32, 76, 73, 70, 2, 17, 2, + 32, 70, 2, 149, 148, 1, 2, 85, 78, 2, 37, 7, 84, 32, 83, 89, 82, 73, 65, + 2, 131, 35, 67, 188, 2, 52, 3, 69, 69, 76, 100, 3, 73, 84, 69, 219, 62, + 65, 7, 60, 7, 32, 79, 70, 32, 68, 72, 65, 21, 4, 67, 72, 65, 73, 2, 147, + 194, 1, 82, 2, 231, 79, 82, 180, 2, 54, 32, 161, 66, 8, 45, 70, 69, 65, + 84, 72, 69, 82, 178, 2, 148, 2, 18, 65, 82, 82, 79, 87, 32, 83, 72, 65, + 70, 84, 32, 87, 73, 68, 84, 72, 32, 114, 66, 46, 67, 250, 15, 68, 166, 5, + 69, 50, 70, 178, 3, 72, 246, 3, 76, 210, 4, 77, 222, 1, 78, 34, 80, 146, + 1, 81, 78, 82, 142, 2, 83, 158, 12, 84, 228, 3, 2, 85, 80, 149, 4, 3, 86, + 69, 82, 4, 22, 84, 231, 70, 79, 2, 11, 87, 2, 21, 3, 79, 32, 84, 2, 17, + 2, 72, 73, 2, 11, 82, 2, 239, 161, 1, 68, 2, 11, 85, 2, 11, 76, 2, 135, + 170, 1, 76, 94, 154, 1, 72, 176, 10, 5, 73, 82, 67, 76, 69, 162, 3, 76, + 24, 20, 79, 78, 67, 65, 86, 69, 45, 83, 73, 68, 69, 68, 32, 68, 73, 65, + 77, 79, 78, 68, 79, 82, 66, 25, 4, 69, 83, 83, 32, 66, 66, 66, 38, 69, + 118, 75, 142, 4, 80, 22, 81, 38, 82, 131, 2, 84, 6, 241, 5, 5, 73, 83, + 72, 79, 80, 4, 45, 9, 81, 85, 73, 72, 79, 80, 80, 69, 82, 5, 17, 2, 32, + 82, 2, 129, 6, 8, 79, 84, 65, 84, 69, 68, 32, 78, 26, 38, 73, 25, 5, 78, + 73, 71, 72, 84, 6, 177, 4, 2, 78, 71, 21, 22, 32, 151, 3, 45, 12, 41, 8, + 82, 79, 84, 65, 84, 69, 68, 32, 12, 104, 2, 70, 79, 0, 15, 79, 78, 69, + 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 72, 73, 18, 84, 215, 3, 78, 2, + 207, 1, 82, 6, 148, 1, 11, 87, 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, + 133, 3, 20, 72, 82, 69, 69, 32, 72, 85, 78, 68, 82, 69, 68, 32, 70, 73, + 70, 84, 69, 69, 78, 4, 36, 4, 84, 87, 69, 78, 175, 2, 83, 2, 217, 2, 7, + 84, 89, 45, 70, 73, 86, 69, 6, 174, 3, 66, 94, 81, 47, 82, 6, 41, 2, 65, + 87, 6, 21, 3, 85, 69, 69, 6, 35, 78, 6, 21, 3, 79, 79, 75, 7, 45, 9, 32, + 82, 79, 84, 65, 84, 69, 68, 32, 4, 70, 78, 25, 13, 84, 87, 79, 32, 72, + 85, 78, 68, 82, 69, 68, 32, 83, 2, 49, 3, 73, 78, 69, 2, 25, 4, 69, 86, + 69, 78, 2, 17, 2, 84, 89, 2, 225, 65, 6, 32, 68, 69, 71, 82, 69, 12, 29, + 5, 85, 82, 78, 69, 68, 12, 11, 32, 12, 42, 66, 30, 75, 34, 80, 34, 81, + 47, 82, 2, 197, 91, 3, 73, 83, 72, 4, 198, 131, 1, 73, 171, 38, 78, 2, + 11, 65, 2, 179, 132, 1, 87, 2, 11, 85, 2, 11, 69, 2, 135, 132, 1, 69, 2, + 243, 169, 1, 79, 19, 11, 32, 16, 100, 16, 67, 79, 78, 84, 65, 73, 78, 73, + 78, 71, 32, 66, 76, 65, 67, 75, 121, 5, 87, 73, 84, 72, 32, 2, 17, 2, 32, + 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 11, 32, 2, 11, 67, 2, 11, 73, 2, + 11, 82, 2, 187, 45, 67, 14, 56, 2, 68, 79, 70, 84, 228, 31, 2, 76, 79, + 231, 1, 85, 4, 18, 84, 35, 87, 2, 11, 32, 2, 167, 166, 1, 82, 2, 143, 54, + 78, 2, 11, 87, 2, 11, 79, 2, 11, 32, 2, 247, 59, 68, 2, 209, 27, 2, 85, + 66, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 228, 30, 2, 76, 69, 49, 2, 82, + 73, 2, 17, 2, 79, 83, 2, 227, 141, 1, 83, 28, 76, 6, 73, 65, 77, 79, 78, + 68, 216, 2, 3, 79, 87, 78, 177, 1, 2, 82, 65, 15, 11, 32, 12, 160, 1, 17, + 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, 128, + 1, 9, 87, 73, 84, 72, 32, 67, 69, 78, 84, 226, 23, 83, 197, 19, 2, 73, + 78, 6, 74, 83, 0, 6, 86, 69, 82, 89, 32, 83, 29, 6, 77, 69, 68, 73, 85, + 77, 2, 25, 4, 77, 65, 76, 76, 2, 197, 15, 2, 32, 68, 2, 11, 82, 2, 11, + 69, 2, 231, 58, 68, 10, 96, 10, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, + 53, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 170, 36, 66, 24, 5, + 76, 69, 70, 84, 32, 51, 73, 4, 194, 37, 83, 51, 84, 4, 33, 6, 85, 71, 72, + 84, 83, 32, 4, 22, 77, 255, 120, 75, 2, 159, 122, 65, 2, 29, 5, 88, 67, + 76, 65, 77, 2, 167, 15, 65, 14, 74, 76, 144, 2, 11, 79, 85, 82, 32, 80, + 79, 73, 78, 84, 69, 68, 71, 82, 8, 28, 2, 65, 71, 175, 1, 79, 5, 149, 1, + 34, 32, 87, 73, 84, 72, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, + 77, 73, 68, 68, 76, 69, 32, 66, 76, 65, 67, 75, 32, 83, 84, 82, 73, 2, + 231, 176, 1, 80, 4, 26, 82, 163, 137, 1, 87, 2, 17, 2, 69, 84, 2, 163, + 176, 1, 84, 4, 11, 32, 4, 18, 67, 23, 83, 2, 171, 198, 1, 85, 2, 247, 36, + 84, 2, 235, 61, 79, 16, 34, 65, 150, 1, 69, 231, 1, 79, 2, 25, 4, 82, 68, + 32, 83, 2, 45, 9, 72, 69, 76, 76, 32, 70, 76, 79, 80, 2, 17, 2, 80, 89, + 2, 17, 2, 32, 68, 2, 11, 73, 2, 211, 193, 1, 83, 10, 22, 65, 151, 12, 88, + 8, 30, 82, 29, 3, 86, 89, 32, 4, 11, 84, 5, 211, 15, 32, 4, 74, 67, 53, + 14, 83, 65, 76, 84, 73, 82, 69, 32, 87, 73, 84, 72, 32, 82, 2, 17, 2, 72, + 69, 2, 11, 67, 2, 211, 129, 1, 75, 2, 167, 19, 79, 4, 60, 8, 82, 73, 90, + 79, 78, 84, 65, 76, 233, 42, 2, 85, 82, 2, 205, 31, 2, 32, 69, 22, 42, + 65, 116, 3, 69, 70, 84, 175, 1, 79, 4, 24, 2, 82, 71, 19, 84, 2, 227, 32, + 69, 2, 11, 73, 2, 11, 78, 2, 17, 2, 32, 67, 2, 11, 82, 2, 187, 42, 79, + 12, 58, 32, 77, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 44, 6, 76, + 65, 78, 69, 32, 77, 247, 24, 80, 2, 11, 69, 2, 227, 10, 82, 6, 150, 2, + 80, 206, 24, 83, 51, 84, 6, 160, 1, 4, 87, 69, 82, 32, 221, 8, 30, 90, + 69, 78, 71, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, + 65, 67, 75, 32, 83, 77, 65, 76, 76, 32, 76, 4, 44, 3, 76, 69, 70, 1, 4, + 82, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, 80, 2, 203, 5, 79, 12, 68, 6, + 69, 68, 73, 85, 77, 32, 117, 7, 79, 79, 78, 32, 83, 69, 76, 10, 30, 68, + 54, 83, 227, 6, 76, 2, 11, 73, 2, 11, 65, 2, 11, 77, 2, 199, 46, 79, 6, + 246, 26, 84, 50, 77, 59, 81, 2, 11, 69, 2, 203, 185, 1, 78, 2, 11, 73, 2, + 227, 187, 1, 66, 6, 18, 65, 87, 69, 2, 37, 7, 82, 65, 76, 76, 69, 76, 79, + 2, 11, 71, 2, 11, 82, 2, 163, 170, 1, 65, 4, 11, 78, 4, 174, 2, 84, 211, + 46, 78, 2, 21, 3, 85, 69, 83, 2, 197, 118, 9, 84, 73, 79, 78, 32, 77, 65, + 82, 75, 14, 34, 69, 25, 4, 73, 71, 72, 84, 2, 129, 21, 2, 67, 84, 12, 60, + 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 211, 17, 32, 8, 30, 80, 202, + 19, 83, 51, 84, 4, 18, 69, 55, 79, 2, 11, 78, 2, 11, 84, 2, 11, 65, 2, + 179, 104, 71, 2, 11, 73, 2, 11, 78, 2, 139, 122, 84, 56, 118, 67, 32, 3, + 69, 83, 65, 18, 72, 58, 77, 166, 1, 80, 68, 5, 81, 85, 65, 82, 69, 180, + 6, 2, 85, 78, 183, 12, 84, 2, 153, 40, 4, 73, 83, 83, 79, 2, 175, 37, 77, + 2, 21, 3, 79, 71, 73, 2, 169, 121, 4, 32, 80, 73, 69, 8, 32, 4, 65, 76, + 76, 32, 115, 73, 6, 18, 76, 71, 83, 2, 11, 79, 2, 11, 90, 2, 11, 69, 2, + 11, 78, 2, 203, 158, 1, 71, 4, 210, 19, 84, 107, 81, 2, 211, 44, 76, 2, + 21, 3, 65, 68, 69, 2, 11, 32, 2, 11, 83, 2, 251, 151, 1, 85, 29, 11, 32, 26, 114, 66, 36, 17, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, - 65, 67, 75, 32, 73, 5, 87, 73, 84, 72, 32, 2, 17, 2, 85, 84, 2, 251, 97, - 84, 6, 184, 16, 4, 86, 69, 82, 89, 22, 83, 37, 6, 77, 69, 68, 73, 85, 77, - 18, 150, 1, 76, 50, 82, 214, 1, 85, 144, 1, 17, 86, 69, 82, 84, 73, 67, - 65, 76, 32, 66, 73, 83, 69, 67, 84, 73, 78, 221, 18, 6, 67, 69, 78, 84, - 82, 69, 6, 18, 69, 15, 79, 2, 67, 70, 4, 247, 1, 87, 4, 18, 73, 115, 79, - 2, 17, 2, 71, 72, 2, 33, 6, 84, 87, 65, 82, 68, 83, 2, 11, 32, 2, 11, 84, - 2, 11, 73, 2, 179, 171, 1, 67, 2, 29, 5, 85, 78, 68, 69, 68, 2, 21, 3, - 32, 67, 79, 2, 181, 31, 2, 82, 78, 4, 17, 2, 80, 80, 4, 21, 3, 69, 82, - 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 33, 6, 84, 32, 81, 85, - 65, 68, 2, 163, 35, 82, 2, 159, 19, 71, 11, 11, 32, 8, 84, 12, 66, 69, - 72, 73, 78, 68, 32, 67, 76, 79, 85, 68, 69, 5, 87, 73, 84, 72, 32, 5, 29, - 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 243, 43, 65, 4, 38, 82, 29, - 5, 83, 77, 65, 76, 76, 2, 11, 65, 2, 155, 109, 89, 2, 25, 4, 32, 67, 76, - 79, 2, 171, 100, 85, 10, 38, 79, 54, 69, 62, 82, 203, 1, 87, 2, 49, 10, - 85, 67, 72, 84, 79, 78, 69, 32, 84, 69, 2, 17, 2, 76, 69, 2, 11, 80, 2, - 11, 72, 2, 147, 16, 79, 4, 148, 1, 4, 65, 80, 69, 90, 33, 28, 73, 65, 78, - 71, 76, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 83, 77, 65, - 76, 76, 32, 87, 72, 73, 84, 2, 11, 73, 2, 155, 151, 1, 85, 2, 255, 3, 69, - 2, 11, 79, 2, 85, 19, 45, 87, 65, 89, 32, 76, 69, 70, 84, 32, 87, 65, 89, - 32, 84, 82, 65, 70, 70, 2, 11, 73, 2, 223, 166, 1, 67, 12, 62, 32, 185, - 1, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 4, 11, 80, 4, 41, 8, 79, - 73, 78, 84, 73, 78, 71, 32, 4, 18, 66, 75, 73, 2, 21, 3, 65, 67, 75, 2, - 25, 4, 72, 65, 78, 68, 2, 17, 2, 32, 73, 2, 17, 2, 78, 68, 2, 215, 24, - 69, 8, 68, 4, 67, 72, 69, 86, 18, 83, 73, 7, 84, 82, 73, 65, 78, 71, 76, - 2, 171, 84, 82, 2, 25, 4, 77, 65, 76, 76, 2, 17, 2, 32, 84, 2, 129, 2, 2, - 82, 73, 4, 11, 69, 5, 11, 32, 2, 11, 87, 2, 145, 18, 3, 73, 84, 72, 8, - 44, 6, 84, 73, 67, 65, 76, 32, 183, 1, 89, 6, 26, 66, 18, 69, 51, 82, 2, - 215, 105, 65, 2, 11, 76, 2, 17, 2, 76, 73, 2, 131, 25, 80, 2, 11, 69, 2, - 17, 2, 67, 84, 2, 11, 65, 2, 11, 78, 2, 11, 71, 2, 207, 138, 1, 76, 2, - 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 17, 2, 32, 83, 2, 11, - 81, 2, 11, 85, 2, 11, 65, 2, 203, 137, 1, 82, 2, 11, 69, 2, 11, 68, 2, - 17, 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 11, 87, 2, 241, 4, 4, - 65, 82, 68, 83, 100, 140, 1, 10, 68, 69, 45, 72, 69, 65, 68, 69, 68, 32, - 132, 4, 4, 71, 71, 76, 89, 124, 6, 76, 84, 69, 68, 32, 70, 42, 78, 189, - 1, 2, 82, 69, 80, 128, 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 12, 4, 68, - 79, 87, 78, 0, 2, 85, 80, 32, 3, 78, 79, 82, 1, 3, 83, 79, 85, 10, 11, - 84, 10, 109, 5, 87, 65, 82, 68, 83, 20, 21, 3, 84, 72, 32, 20, 32, 2, 69, - 65, 1, 2, 87, 69, 10, 17, 2, 83, 84, 10, 11, 32, 10, 110, 72, 0, 6, 86, - 69, 82, 89, 32, 72, 28, 5, 76, 73, 71, 72, 84, 0, 6, 77, 69, 68, 73, 85, - 77, 23, 66, 2, 25, 4, 69, 65, 86, 89, 2, 17, 2, 32, 66, 2, 21, 3, 65, 82, - 66, 2, 11, 32, 2, 11, 65, 2, 11, 82, 2, 11, 82, 2, 251, 27, 79, 2, 17, 2, - 32, 86, 2, 11, 69, 2, 33, 6, 82, 84, 73, 67, 65, 76, 2, 11, 32, 2, 11, - 76, 2, 11, 73, 2, 191, 130, 1, 78, 2, 11, 76, 2, 11, 79, 2, 135, 91, 87, - 12, 46, 68, 106, 69, 186, 15, 75, 139, 136, 1, 71, 6, 22, 32, 131, 26, - 79, 4, 44, 2, 67, 72, 217, 15, 4, 66, 76, 79, 87, 2, 11, 73, 2, 255, 128, - 1, 77, 2, 11, 32, 2, 121, 3, 71, 76, 65, 4, 36, 5, 68, 32, 75, 69, 89, - 51, 76, 2, 17, 2, 66, 79, 2, 11, 65, 2, 191, 80, 82, 2, 11, 69, 2, 231, - 88, 83, 30, 66, 77, 158, 3, 82, 226, 11, 78, 214, 64, 79, 129, 9, 2, 76, - 70, 14, 36, 2, 65, 78, 161, 2, 2, 69, 78, 13, 72, 12, 32, 87, 73, 84, 72, - 32, 66, 85, 78, 78, 89, 32, 29, 2, 83, 32, 2, 11, 69, 2, 167, 6, 65, 8, - 48, 2, 66, 79, 28, 2, 67, 76, 54, 72, 19, 83, 2, 11, 79, 2, 195, 86, 84, - 2, 11, 79, 2, 11, 84, 2, 11, 72, 2, 143, 86, 69, 2, 163, 119, 65, 2, 17, - 2, 65, 78, 2, 131, 10, 68, 2, 11, 83, 2, 11, 32, 2, 11, 83, 2, 11, 89, 2, - 17, 2, 77, 66, 2, 187, 9, 79, 10, 72, 2, 68, 32, 156, 1, 3, 76, 68, 32, - 32, 2, 82, 73, 195, 144, 1, 77, 4, 56, 9, 83, 69, 80, 65, 82, 65, 84, 79, - 82, 191, 83, 74, 2, 17, 2, 32, 77, 2, 21, 3, 73, 68, 68, 2, 11, 76, 2, - 11, 69, 2, 11, 32, 2, 195, 64, 68, 2, 11, 77, 2, 199, 144, 1, 65, 2, 11, - 69, 2, 159, 83, 68, 10, 56, 7, 65, 80, 80, 69, 68, 32, 80, 30, 69, 163, - 1, 73, 2, 201, 76, 3, 82, 69, 83, 6, 48, 2, 65, 84, 80, 3, 83, 84, 76, - 163, 108, 78, 2, 11, 72, 2, 11, 32, 2, 11, 80, 2, 25, 4, 82, 79, 68, 85, - 2, 171, 114, 67, 2, 11, 69, 2, 235, 80, 82, 2, 11, 84, 2, 11, 73, 2, 17, - 2, 78, 71, 2, 17, 2, 32, 72, 2, 11, 65, 2, 203, 71, 78, 34, 76, 2, 32, - 73, 138, 1, 45, 28, 7, 73, 65, 78, 71, 81, 73, 32, 143, 83, 79, 2, 53, - 11, 78, 32, 65, 32, 82, 69, 67, 84, 65, 78, 71, 2, 11, 76, 2, 11, 69, 2, - 11, 32, 2, 11, 66, 2, 11, 79, 2, 139, 140, 1, 88, 2, 11, 82, 2, 163, 122, - 65, 28, 48, 5, 66, 76, 65, 67, 75, 1, 3, 82, 69, 68, 14, 11, 32, 14, 130, - 1, 67, 72, 4, 69, 76, 69, 80, 28, 4, 71, 69, 78, 69, 46, 72, 36, 4, 83, - 79, 76, 68, 169, 9, 6, 77, 65, 78, 68, 65, 82, 4, 24, 2, 65, 78, 19, 72, - 2, 155, 58, 78, 2, 189, 57, 3, 65, 82, 73, 2, 11, 72, 2, 179, 70, 65, 2, - 11, 82, 2, 11, 65, 2, 155, 137, 1, 76, 2, 17, 2, 79, 82, 2, 159, 114, 83, - 2, 131, 75, 73, 242, 19, 50, 65, 58, 69, 218, 11, 73, 217, 35, 2, 79, 45, - 2, 17, 2, 87, 78, 2, 11, 73, 2, 229, 74, 2, 78, 71, 98, 60, 4, 76, 76, - 79, 87, 62, 78, 53, 5, 90, 73, 68, 73, 32, 2, 17, 2, 32, 72, 2, 11, 69, - 2, 11, 65, 2, 203, 106, 82, 2, 11, 32, 2, 11, 83, 2, 11, 73, 2, 239, 54, - 71, 94, 112, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 76, 5, 72, 89, - 80, 72, 69, 17, 7, 76, 69, 84, 84, 69, 82, 32, 4, 44, 3, 77, 65, 68, 13, - 4, 72, 65, 77, 90, 2, 11, 68, 2, 203, 67, 65, 2, 207, 27, 78, 88, 250, 1, - 67, 50, 77, 18, 68, 42, 69, 58, 72, 30, 75, 22, 71, 2, 81, 32, 3, 76, 65, - 77, 98, 78, 18, 80, 30, 83, 66, 84, 28, 2, 86, 65, 118, 87, 14, 79, 18, - 88, 52, 3, 89, 79, 84, 154, 1, 90, 130, 53, 82, 226, 48, 66, 254, 5, 85, - 162, 14, 70, 3, 74, 6, 22, 72, 251, 113, 73, 4, 22, 72, 227, 113, 73, 2, - 223, 113, 73, 4, 11, 65, 4, 154, 130, 1, 68, 3, 76, 8, 42, 76, 130, 50, - 89, 214, 79, 84, 3, 87, 2, 71, 73, 4, 254, 111, 65, 147, 15, 72, 4, 18, - 72, 15, 65, 2, 11, 65, 2, 139, 129, 1, 70, 5, 11, 32, 2, 11, 87, 2, 21, - 3, 73, 84, 72, 2, 17, 2, 32, 68, 2, 11, 79, 2, 179, 3, 84, 2, 195, 48, - 85, 4, 178, 105, 72, 215, 22, 69, 8, 46, 72, 234, 47, 73, 194, 9, 65, - 151, 70, 69, 2, 231, 47, 73, 4, 214, 104, 72, 215, 22, 65, 5, 37, 7, 32, - 65, 76, 84, 69, 82, 78, 2, 21, 3, 65, 84, 69, 2, 11, 32, 2, 11, 70, 2, - 11, 79, 2, 211, 109, 82, 2, 11, 65, 2, 143, 126, 87, 4, 22, 72, 235, 125, - 65, 2, 11, 69, 2, 135, 46, 89, 5, 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, - 45, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 2, 11, 32, 2, 11, 65, 2, 11, - 66, 2, 11, 79, 2, 239, 101, 86, 6, 22, 65, 159, 124, 69, 5, 155, 124, 76, - 140, 19, 34, 32, 161, 35, 3, 78, 32, 89, 138, 19, 88, 8, 82, 65, 68, 73, - 67, 65, 76, 32, 181, 7, 9, 83, 89, 76, 76, 65, 66, 76, 69, 32, 110, 170, - 1, 66, 42, 67, 86, 68, 42, 71, 74, 72, 66, 74, 66, 75, 30, 76, 30, 77, - 26, 78, 74, 80, 26, 83, 74, 84, 30, 86, 30, 89, 30, 90, 246, 35, 81, 186, - 49, 87, 235, 30, 79, 4, 22, 66, 243, 64, 85, 2, 147, 93, 85, 12, 42, 85, - 18, 89, 162, 98, 72, 203, 22, 73, 2, 247, 120, 79, 7, 242, 120, 80, 3, - 84, 4, 22, 68, 199, 120, 85, 2, 243, 63, 85, 10, 42, 71, 222, 91, 79, - 162, 28, 69, 15, 65, 4, 146, 89, 85, 235, 30, 79, 8, 22, 88, 227, 88, 77, - 6, 222, 88, 85, 202, 2, 73, 163, 28, 79, 8, 22, 74, 151, 119, 79, 6, 230, - 90, 85, 218, 5, 73, 215, 22, 89, 4, 190, 90, 73, 175, 28, 69, 6, 186, 54, - 73, 199, 7, 89, 4, 166, 118, 79, 15, 73, 8, 30, 89, 26, 90, 183, 90, 66, - 4, 238, 117, 73, 3, 79, 2, 215, 117, 85, 4, 166, 89, 85, 3, 89, 10, 22, - 72, 207, 97, 83, 8, 210, 60, 85, 166, 28, 65, 162, 28, 79, 15, 89, 4, - 198, 88, 65, 175, 28, 85, 4, 134, 60, 85, 199, 56, 69, 4, 142, 88, 73, - 175, 28, 79, 10, 54, 85, 220, 62, 2, 90, 73, 226, 24, 79, 175, 28, 65, 4, - 230, 115, 80, 3, 82, 156, 18, 134, 2, 66, 134, 1, 67, 162, 1, 68, 110, - 70, 50, 71, 150, 1, 72, 138, 3, 73, 134, 1, 74, 98, 75, 54, 76, 62, 77, - 134, 1, 78, 234, 4, 80, 54, 81, 2, 89, 46, 82, 162, 1, 83, 134, 1, 84, - 102, 86, 82, 87, 58, 88, 50, 90, 140, 2, 2, 85, 79, 66, 65, 2, 79, 107, - 69, 132, 1, 66, 66, 238, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, - 89, 64, 234, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 3, 89, 122, 66, - 72, 238, 20, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, 89, 54, 46, 85, - 146, 22, 65, 2, 69, 2, 79, 67, 89, 19, 142, 22, 79, 106, 82, 214, 88, 80, - 3, 88, 106, 58, 68, 138, 15, 85, 134, 5, 73, 94, 69, 66, 65, 3, 79, 54, - 210, 19, 85, 58, 73, 94, 69, 66, 65, 3, 79, 42, 182, 20, 79, 66, 65, 2, - 73, 2, 89, 67, 85, 116, 58, 71, 214, 15, 85, 146, 4, 73, 42, 65, 2, 69, - 3, 79, 56, 50, 73, 162, 15, 85, 186, 4, 65, 2, 69, 3, 79, 13, 150, 19, - 69, 254, 89, 84, 3, 88, 246, 1, 78, 73, 30, 76, 58, 77, 54, 78, 110, 88, - 50, 85, 254, 15, 69, 66, 65, 3, 79, 6, 198, 19, 69, 215, 88, 84, 64, 238, - 16, 85, 58, 73, 94, 69, 2, 79, 66, 65, 67, 89, 58, 182, 16, 85, 58, 73, - 158, 1, 65, 2, 79, 35, 89, 42, 46, 79, 34, 85, 202, 16, 69, 26, 73, 43, - 65, 6, 226, 106, 80, 2, 84, 3, 88, 6, 238, 17, 79, 215, 88, 84, 46, 46, - 85, 254, 15, 69, 26, 73, 42, 65, 3, 79, 8, 187, 16, 79, 19, 42, 84, 130, - 16, 69, 190, 89, 80, 3, 88, 5, 11, 69, 2, 11, 82, 2, 11, 65, 2, 11, 84, - 2, 11, 73, 2, 11, 79, 2, 187, 39, 78, 106, 50, 74, 190, 10, 85, 146, 4, - 73, 42, 79, 67, 89, 50, 158, 13, 85, 174, 1, 73, 42, 79, 3, 89, 56, 242, - 12, 85, 58, 73, 158, 1, 65, 2, 69, 3, 79, 70, 218, 9, 85, 250, 3, 69, 26, - 73, 42, 65, 2, 79, 67, 89, 106, 70, 71, 218, 8, 85, 158, 3, 73, 158, 1, - 65, 2, 79, 2, 89, 107, 69, 44, 186, 11, 85, 150, 1, 69, 66, 65, 2, 79, - 105, 2, 73, 69, 248, 2, 102, 66, 54, 68, 94, 71, 90, 74, 46, 82, 50, 89, - 78, 90, 134, 7, 85, 58, 73, 94, 65, 2, 69, 67, 79, 54, 202, 10, 73, 158, - 1, 65, 2, 79, 66, 85, 3, 89, 46, 46, 73, 198, 10, 69, 66, 65, 2, 79, 67, - 85, 13, 234, 11, 69, 214, 88, 80, 2, 84, 3, 88, 34, 60, 2, 85, 79, 218, - 9, 69, 0, 2, 73, 69, 66, 65, 3, 79, 7, 210, 99, 84, 3, 88, 50, 230, 1, - 85, 242, 7, 73, 42, 79, 67, 89, 46, 146, 9, 79, 66, 65, 2, 69, 66, 85, 3, - 89, 38, 30, 85, 222, 8, 73, 43, 79, 15, 194, 8, 79, 254, 89, 80, 2, 84, - 3, 88, 56, 62, 85, 206, 4, 79, 178, 2, 73, 158, 1, 65, 66, 89, 43, 69, - 15, 254, 8, 79, 2, 82, 214, 88, 80, 3, 88, 60, 150, 6, 85, 58, 73, 158, - 1, 65, 2, 79, 67, 89, 56, 254, 2, 85, 146, 4, 73, 42, 79, 67, 89, 100, - 58, 82, 254, 4, 85, 150, 1, 69, 66, 65, 2, 79, 67, 89, 48, 46, 85, 162, - 6, 69, 2, 79, 66, 89, 43, 65, 17, 134, 7, 79, 2, 82, 214, 88, 80, 2, 84, - 3, 88, 176, 1, 70, 83, 158, 3, 72, 50, 85, 58, 73, 94, 69, 66, 65, 2, 79, - 67, 89, 56, 130, 4, 73, 94, 69, 66, 65, 2, 79, 2, 85, 67, 89, 56, 46, 85, - 158, 3, 73, 94, 69, 66, 65, 3, 79, 21, 182, 4, 79, 106, 82, 214, 88, 80, - 2, 84, 3, 88, 60, 54, 69, 166, 3, 73, 42, 65, 2, 79, 66, 85, 3, 89, 4, - 134, 93, 80, 3, 88, 28, 38, 85, 206, 2, 69, 2, 79, 67, 65, 9, 203, 2, 79, - 40, 210, 2, 73, 42, 79, 66, 89, 41, 2, 85, 79, 178, 1, 66, 72, 50, 85, - 58, 73, 42, 90, 54, 69, 66, 65, 2, 79, 67, 89, 54, 46, 85, 214, 1, 65, 2, - 69, 2, 79, 67, 89, 19, 146, 1, 79, 170, 1, 82, 214, 88, 80, 2, 84, 3, 88, - 15, 90, 69, 254, 89, 80, 2, 84, 3, 88, 58, 50, 69, 2, 79, 26, 73, 42, 65, - 34, 85, 35, 89, 7, 250, 89, 80, 3, 88, 17, 38, 69, 190, 89, 80, 2, 84, 3, - 88, 9, 186, 89, 80, 2, 84, 3, 88, 11, 70, 82, 214, 88, 80, 3, 88, 13, 38, - 82, 214, 88, 80, 2, 84, 3, 88, 5, 211, 88, 88, 2, 215, 7, 65, 2, 191, 57, - 89, 180, 4, 252, 1, 10, 32, 78, 79, 84, 65, 84, 73, 79, 78, 32, 220, 6, - 16, 65, 78, 65, 66, 65, 90, 65, 82, 32, 83, 81, 85, 65, 82, 69, 32, 210, - 15, 69, 180, 2, 6, 73, 80, 80, 69, 82, 45, 96, 8, 78, 65, 77, 69, 78, 78, - 89, 32, 176, 33, 3, 79, 77, 66, 235, 26, 87, 26, 144, 1, 9, 66, 65, 71, - 32, 77, 69, 77, 66, 69, 38, 82, 104, 6, 68, 79, 77, 65, 73, 78, 60, 3, - 76, 69, 70, 154, 1, 83, 145, 2, 3, 84, 89, 80, 2, 11, 82, 2, 209, 83, 2, - 83, 72, 8, 100, 4, 65, 78, 71, 69, 60, 3, 73, 71, 72, 221, 1, 11, 69, 76, - 65, 84, 73, 79, 78, 65, 76, 32, 67, 2, 173, 3, 11, 32, 65, 78, 84, 73, - 82, 69, 83, 84, 82, 73, 4, 17, 2, 84, 32, 4, 60, 4, 73, 77, 65, 71, 13, - 7, 66, 73, 78, 68, 73, 78, 71, 2, 11, 69, 2, 25, 4, 32, 66, 82, 65, 2, - 11, 67, 2, 175, 29, 75, 8, 44, 6, 67, 72, 69, 77, 65, 32, 211, 1, 80, 6, - 18, 67, 71, 80, 2, 17, 2, 79, 77, 2, 11, 80, 2, 11, 79, 2, 11, 83, 2, - 107, 73, 4, 30, 73, 41, 3, 82, 79, 74, 2, 11, 80, 2, 11, 73, 2, 151, 80, - 78, 2, 11, 69, 2, 11, 67, 2, 11, 84, 2, 87, 73, 2, 255, 51, 79, 2, 11, - 69, 2, 11, 32, 2, 11, 67, 2, 11, 79, 2, 11, 76, 2, 11, 79, 2, 211, 79, - 78, 144, 1, 240, 1, 2, 67, 76, 68, 7, 73, 78, 73, 84, 73, 65, 76, 204, 2, - 14, 70, 73, 78, 65, 76, 32, 67, 79, 78, 83, 79, 78, 65, 78, 16, 7, 76, - 69, 84, 84, 69, 82, 32, 148, 3, 5, 77, 65, 82, 75, 32, 206, 1, 83, 181, - 3, 6, 86, 79, 87, 69, 76, 32, 14, 64, 5, 79, 83, 73, 78, 71, 137, 1, 6, - 85, 83, 84, 69, 82, 45, 4, 11, 32, 4, 64, 12, 68, 79, 85, 66, 76, 69, 45, - 76, 73, 78, 69, 68, 23, 72, 2, 17, 2, 32, 72, 2, 17, 2, 69, 65, 2, 215, - 10, 68, 10, 96, 13, 70, 73, 78, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, - 41, 7, 73, 78, 73, 84, 73, 65, 76, 8, 226, 72, 76, 2, 82, 2, 86, 3, 89, - 2, 37, 7, 32, 76, 69, 84, 84, 69, 82, 2, 151, 6, 32, 2, 131, 9, 84, 82, - 166, 1, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 80, - 2, 90, 254, 68, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, - 2, 65, 12, 206, 1, 68, 2, 90, 254, 68, 72, 187, 2, 65, 6, 142, 70, 83, - 14, 72, 187, 2, 65, 8, 246, 69, 71, 2, 78, 2, 89, 187, 2, 65, 6, 202, 69, - 72, 2, 83, 187, 2, 65, 12, 42, 83, 2, 84, 254, 68, 72, 187, 2, 65, 4, - 250, 68, 72, 187, 2, 65, 8, 50, 68, 58, 83, 40, 4, 76, 79, 78, 71, 23, - 84, 2, 29, 5, 79, 85, 66, 76, 69, 2, 11, 32, 2, 11, 83, 2, 11, 72, 2, 11, - 65, 2, 147, 70, 68, 2, 17, 2, 32, 84, 2, 17, 2, 83, 72, 2, 135, 69, 69, - 14, 36, 4, 73, 71, 78, 32, 255, 2, 85, 12, 54, 65, 72, 6, 67, 65, 78, 68, - 82, 65, 167, 1, 86, 2, 11, 78, 2, 11, 85, 2, 17, 2, 83, 86, 2, 11, 65, 2, - 251, 65, 82, 6, 36, 5, 66, 73, 78, 68, 85, 15, 32, 5, 11, 32, 2, 25, 4, - 87, 73, 84, 72, 2, 17, 2, 32, 79, 2, 21, 3, 82, 78, 65, 2, 11, 77, 2, 11, - 69, 2, 227, 38, 78, 4, 11, 73, 4, 18, 82, 19, 83, 2, 207, 33, 65, 2, 11, - 65, 2, 11, 82, 2, 255, 63, 71, 2, 151, 4, 66, 20, 38, 76, 109, 5, 83, 73, - 71, 78, 32, 2, 17, 2, 69, 78, 2, 11, 71, 2, 11, 84, 2, 11, 72, 2, 11, 32, - 2, 11, 77, 2, 11, 65, 2, 251, 61, 82, 18, 86, 65, 26, 79, 2, 85, 16, 8, - 82, 69, 86, 69, 82, 83, 69, 68, 134, 64, 69, 3, 73, 4, 170, 64, 73, 3, - 85, 5, 147, 64, 69, 2, 171, 44, 32, 12, 72, 2, 66, 82, 16, 9, 82, 79, 32, - 87, 73, 68, 84, 72, 32, 203, 1, 85, 2, 147, 2, 65, 8, 32, 2, 78, 79, 86, - 83, 31, 74, 4, 26, 45, 73, 2, 78, 45, 2, 29, 5, 66, 82, 69, 65, 75, 2, - 11, 32, 2, 11, 83, 2, 163, 1, 80, 2, 11, 74, 2, 11, 79, 2, 11, 73, 2, 11, - 78, 2, 143, 5, 69, 2, 207, 61, 83, 2, 21, 3, 77, 79, 85, 2, 17, 2, 84, - 72, 2, 11, 32, 2, 11, 70, 2, 11, 65, 2, 155, 38, 67, 242, 2, 168, 1, 10, - 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 212, 18, 6, 78, 69, 85, 77, 69, - 32, 181, 38, 17, 80, 82, 73, 90, 78, 65, 75, 32, 77, 79, 68, 73, 70, 73, - 69, 82, 32, 128, 1, 144, 2, 17, 76, 79, 87, 69, 82, 32, 84, 79, 78, 65, - 76, 32, 82, 65, 78, 71, 69, 88, 5, 77, 65, 82, 75, 32, 148, 3, 18, 65, - 84, 84, 65, 67, 72, 73, 78, 71, 32, 86, 69, 82, 84, 73, 67, 65, 76, 221, - 11, 17, 84, 79, 78, 65, 76, 32, 82, 65, 78, 71, 69, 32, 77, 65, 82, 75, - 32, 2, 33, 6, 32, 73, 78, 68, 73, 67, 2, 11, 65, 2, 11, 84, 2, 11, 79, 2, - 207, 56, 82, 118, 182, 2, 67, 142, 1, 68, 104, 8, 71, 79, 82, 65, 90, 68, - 79, 32, 34, 78, 106, 75, 114, 79, 72, 2, 80, 79, 140, 2, 8, 77, 65, 76, - 79, 32, 80, 79, 86, 100, 2, 82, 65, 50, 83, 186, 1, 84, 108, 7, 86, 89, - 83, 79, 75, 79, 32, 234, 1, 90, 224, 10, 2, 76, 79, 224, 22, 4, 85, 68, - 65, 82, 129, 6, 4, 66, 79, 82, 90, 6, 60, 6, 72, 65, 83, 72, 75, 65, 29, - 5, 85, 82, 86, 69, 68, 5, 185, 47, 3, 32, 80, 79, 2, 17, 2, 32, 79, 2, - 11, 77, 2, 223, 24, 69, 4, 228, 10, 13, 69, 77, 69, 83, 84, 86, 69, 78, - 78, 89, 32, 90, 65, 217, 14, 6, 86, 79, 69, 84, 79, 67, 10, 30, 78, 89, - 3, 86, 89, 83, 8, 29, 5, 73, 90, 75, 79, 32, 8, 152, 8, 8, 83, 32, 75, - 82, 89, 90, 72, 69, 35, 79, 2, 155, 20, 79, 8, 56, 4, 82, 89, 90, 72, - 138, 6, 65, 233, 39, 2, 85, 80, 5, 21, 3, 32, 79, 78, 2, 11, 32, 2, 191, - 7, 76, 6, 208, 5, 3, 84, 83, 69, 200, 13, 5, 66, 76, 65, 67, 72, 131, 31, - 78, 18, 22, 68, 247, 1, 86, 6, 60, 7, 67, 72, 65, 83, 72, 73, 69, 189, - 22, 3, 86, 69, 82, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 86, 2, 29, - 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, 11, 84, - 2, 11, 82, 2, 11, 79, 2, 131, 25, 75, 12, 29, 5, 89, 83, 72, 69, 32, 12, - 22, 83, 251, 3, 79, 8, 154, 3, 32, 229, 2, 4, 84, 82, 65, 78, 4, 28, 2, - 90, 83, 183, 5, 86, 2, 219, 37, 69, 10, 136, 1, 2, 75, 79, 16, 16, 84, - 82, 65, 78, 78, 79, 32, 77, 65, 76, 79, 32, 80, 79, 86, 89, 236, 1, 5, - 82, 69, 68, 78, 69, 231, 26, 79, 2, 239, 42, 66, 2, 11, 83, 2, 183, 22, - 72, 10, 50, 79, 16, 5, 83, 65, 84, 65, 32, 139, 36, 73, 2, 187, 28, 67, - 6, 158, 1, 79, 201, 24, 3, 83, 32, 75, 10, 24, 2, 83, 32, 95, 79, 6, 11, - 75, 6, 44, 6, 72, 79, 75, 72, 76, 79, 247, 24, 82, 4, 11, 77, 4, 17, 2, - 32, 79, 4, 11, 78, 4, 11, 32, 4, 18, 76, 31, 82, 2, 11, 69, 2, 179, 14, - 70, 2, 11, 73, 2, 11, 71, 2, 139, 14, 72, 6, 18, 65, 31, 69, 2, 249, 25, - 3, 68, 69, 82, 4, 22, 86, 239, 10, 76, 2, 199, 38, 79, 6, 60, 5, 77, 82, - 65, 67, 72, 18, 83, 1, 4, 84, 82, 69, 83, 2, 155, 10, 78, 2, 17, 2, 86, - 69, 2, 155, 6, 84, 232, 1, 128, 2, 2, 67, 72, 38, 68, 218, 1, 70, 28, 10, - 71, 79, 76, 85, 66, 67, 72, 73, 75, 32, 158, 1, 75, 160, 2, 6, 77, 69, - 67, 72, 73, 75, 184, 1, 2, 78, 69, 18, 79, 138, 2, 80, 146, 2, 82, 114, - 83, 180, 20, 9, 86, 82, 65, 75, 72, 73, 89, 65, 32, 151, 2, 90, 4, 246, - 11, 69, 209, 10, 2, 65, 83, 10, 74, 69, 98, 85, 16, 9, 86, 65, 32, 86, - 32, 67, 72, 69, 76, 187, 27, 79, 4, 76, 2, 82, 66, 213, 2, 12, 77, 69, - 83, 84, 86, 69, 78, 78, 89, 32, 75, 76, 2, 195, 34, 73, 2, 203, 34, 68, - 2, 11, 78, 2, 231, 36, 85, 2, 11, 73, 2, 147, 34, 84, 10, 78, 84, 38, 83, - 240, 3, 5, 77, 82, 65, 67, 72, 145, 14, 4, 66, 79, 82, 90, 4, 32, 3, 82, - 69, 83, 251, 1, 73, 2, 21, 3, 86, 69, 84, 2, 231, 17, 76, 14, 76, 4, 72, - 65, 77, 73, 18, 76, 40, 3, 79, 66, 89, 16, 2, 82, 89, 87, 85, 2, 219, 3, - 76, 2, 11, 89, 2, 11, 85, 2, 151, 33, 67, 2, 223, 31, 76, 6, 28, 2, 85, - 75, 219, 32, 90, 5, 21, 3, 32, 84, 73, 2, 11, 75, 2, 251, 15, 72, 2, 11, - 70, 2, 11, 73, 2, 11, 83, 2, 215, 30, 77, 11, 11, 32, 8, 44, 7, 75, 76, - 89, 85, 67, 72, 69, 83, 80, 6, 64, 9, 78, 69, 80, 79, 83, 84, 79, 89, 65, - 14, 80, 163, 14, 86, 2, 39, 78, 2, 25, 4, 79, 86, 79, 68, 2, 143, 14, 78, - 2, 223, 22, 77, 14, 40, 2, 66, 76, 41, 4, 83, 79, 75, 65, 2, 11, 65, 2, - 11, 75, 2, 243, 30, 79, 13, 11, 32, 10, 30, 75, 142, 26, 84, 39, 83, 6, - 104, 11, 76, 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, 185, 25, 10, 82, 89, - 85, 75, 79, 86, 65, 89, 65, 32, 4, 202, 16, 78, 251, 8, 83, 14, 58, 65, - 88, 8, 69, 82, 69, 86, 79, 68, 75, 65, 27, 79, 6, 44, 3, 82, 65, 75, 222, - 19, 76, 211, 5, 85, 2, 11, 76, 2, 11, 73, 2, 171, 28, 84, 5, 153, 15, 2, - 32, 78, 4, 68, 5, 68, 67, 72, 65, 83, 245, 9, 7, 76, 75, 85, 76, 73, 90, - 77, 2, 11, 72, 2, 219, 4, 73, 4, 64, 11, 69, 86, 69, 82, 83, 69, 68, 32, - 67, 72, 69, 139, 26, 79, 2, 25, 4, 76, 89, 85, 83, 2, 215, 17, 84, 126, - 96, 9, 75, 65, 77, 69, 89, 84, 83, 65, 32, 148, 2, 8, 76, 79, 90, 72, 73, - 84, 73, 69, 119, 84, 22, 128, 1, 13, 68, 86, 79, 69, 67, 72, 69, 76, 78, - 65, 89, 65, 32, 44, 7, 75, 76, 89, 85, 67, 72, 69, 74, 84, 218, 18, 77, - 119, 83, 8, 218, 10, 75, 110, 78, 178, 8, 80, 75, 83, 6, 40, 5, 86, 65, - 89, 65, 32, 243, 10, 78, 4, 178, 15, 84, 183, 4, 83, 4, 162, 15, 73, 147, - 4, 82, 9, 11, 32, 6, 48, 2, 83, 32, 25, 6, 90, 65, 75, 82, 89, 84, 4, - 166, 4, 75, 95, 90, 2, 11, 79, 2, 211, 22, 69, 96, 92, 4, 65, 84, 89, 65, - 172, 4, 6, 79, 80, 73, 84, 83, 65, 189, 1, 5, 82, 69, 76, 65, 32, 23, 11, - 32, 20, 76, 2, 83, 32, 236, 2, 9, 90, 65, 75, 82, 89, 84, 65, 89, 65, - 159, 5, 78, 14, 160, 1, 14, 68, 86, 85, 77, 89, 65, 32, 90, 65, 80, 89, - 65, 84, 89, 28, 7, 75, 82, 89, 90, 72, 69, 77, 24, 2, 82, 79, 17, 8, 90, - 65, 80, 89, 65, 84, 79, 89, 2, 11, 77, 2, 215, 19, 73, 5, 189, 1, 2, 32, - 73, 2, 203, 2, 71, 7, 21, 3, 32, 73, 32, 4, 54, 75, 37, 9, 80, 79, 68, - 67, 72, 65, 83, 72, 73, 2, 11, 82, 2, 21, 3, 89, 90, 72, 2, 211, 1, 69, - 5, 17, 2, 32, 83, 2, 17, 2, 32, 90, 2, 29, 5, 65, 80, 89, 65, 84, 2, 11, - 79, 2, 199, 17, 89, 7, 11, 32, 4, 68, 6, 83, 32, 79, 67, 72, 75, 29, 7, - 87, 73, 84, 72, 32, 83, 79, 2, 11, 79, 2, 215, 16, 77, 2, 45, 9, 82, 79, - 67, 72, 89, 65, 32, 78, 79, 2, 11, 90, 2, 163, 7, 72, 68, 140, 1, 9, 68, - 86, 79, 69, 67, 72, 69, 76, 78, 162, 1, 75, 78, 78, 174, 1, 71, 152, 3, - 8, 77, 82, 65, 67, 72, 78, 79, 84, 42, 80, 75, 84, 10, 18, 79, 71, 65, 6, - 64, 7, 80, 79, 86, 79, 68, 78, 65, 189, 5, 4, 75, 82, 89, 90, 4, 17, 2, - 89, 65, 5, 17, 2, 32, 75, 2, 145, 5, 4, 76, 89, 85, 67, 26, 48, 6, 76, - 89, 85, 67, 72, 69, 77, 2, 82, 89, 4, 22, 78, 203, 6, 80, 2, 157, 8, 9, - 69, 80, 79, 83, 84, 79, 89, 65, 78, 22, 48, 7, 85, 75, 79, 86, 65, 89, - 65, 195, 3, 90, 21, 11, 32, 18, 54, 71, 236, 2, 5, 84, 82, 89, 65, 83, - 179, 2, 80, 14, 21, 3, 82, 79, 77, 14, 32, 4, 78, 65, 89, 65, 47, 79, 5, - 237, 1, 7, 32, 87, 73, 84, 72, 32, 83, 10, 92, 10, 75, 82, 89, 90, 72, - 69, 86, 65, 89, 65, 17, 9, 80, 79, 86, 79, 68, 78, 65, 89, 65, 5, 175, 2, - 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 24, 2, 68, 79, 23, 83, 2, 41, 2, - 85, 66, 2, 21, 3, 73, 78, 71, 2, 133, 6, 6, 76, 69, 32, 90, 65, 80, 2, - 175, 6, 75, 2, 237, 5, 3, 72, 69, 86, 2, 11, 73, 2, 11, 75, 2, 187, 5, - 72, 6, 22, 79, 187, 3, 82, 4, 28, 2, 76, 85, 187, 1, 86, 2, 163, 1, 80, - 8, 80, 5, 82, 89, 65, 83, 79, 153, 2, 10, 73, 75, 72, 65, 89, 65, 32, 80, - 85, 84, 6, 62, 80, 44, 4, 83, 84, 82, 69, 173, 1, 4, 71, 76, 65, 83, 2, - 17, 2, 79, 86, 2, 193, 1, 2, 79, 68, 2, 171, 1, 76, 16, 88, 12, 75, 76, - 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, 38, 77, 46, 80, 38, 84, 39, 83, - 8, 34, 77, 46, 80, 38, 84, 39, 83, 2, 25, 4, 82, 65, 67, 72, 2, 247, 1, - 78, 2, 11, 82, 2, 205, 1, 2, 79, 83, 2, 11, 82, 2, 11, 69, 2, 11, 83, 2, - 157, 1, 4, 86, 69, 84, 76, 6, 30, 65, 121, 3, 77, 69, 89, 4, 32, 4, 78, - 79, 90, 72, 31, 80, 2, 11, 69, 2, 151, 3, 75, 2, 17, 2, 89, 65, 2, 11, - 84, 2, 11, 65, 2, 35, 89, 2, 11, 84, 2, 11, 83, 2, 183, 2, 65, 10, 108, - 11, 68, 73, 82, 69, 67, 84, 73, 79, 78, 32, 70, 28, 3, 75, 82, 89, 28, 5, - 76, 69, 86, 69, 76, 35, 82, 2, 11, 76, 2, 159, 1, 73, 2, 11, 90, 2, 143, - 1, 72, 4, 11, 45, 4, 114, 50, 3, 51, 2, 11, 79, 2, 83, 71, 8, 26, 78, 34, - 83, 15, 74, 4, 18, 66, 27, 74, 2, 11, 83, 2, 11, 80, 3, 0, + 65, 67, 75, 32, 85, 5, 87, 73, 84, 72, 32, 2, 17, 2, 85, 84, 2, 159, 99, + 84, 6, 54, 86, 178, 17, 83, 37, 6, 77, 69, 68, 73, 85, 77, 2, 153, 17, 3, + 69, 82, 89, 18, 150, 1, 76, 50, 82, 214, 1, 85, 144, 1, 17, 86, 69, 82, + 84, 73, 67, 65, 76, 32, 66, 73, 83, 69, 67, 84, 73, 78, 233, 19, 6, 67, + 69, 78, 84, 82, 69, 6, 18, 69, 15, 79, 2, 67, 70, 4, 247, 1, 87, 4, 18, + 73, 115, 79, 2, 17, 2, 71, 72, 2, 33, 6, 84, 87, 65, 82, 68, 83, 2, 11, + 32, 2, 11, 84, 2, 11, 73, 2, 215, 172, 1, 67, 2, 29, 5, 85, 78, 68, 69, + 68, 2, 21, 3, 32, 67, 79, 2, 193, 32, 2, 82, 78, 4, 17, 2, 80, 80, 4, 21, + 3, 69, 82, 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 33, 6, 84, + 32, 81, 85, 65, 68, 2, 175, 36, 82, 2, 171, 20, 71, 11, 11, 32, 8, 84, + 12, 66, 69, 72, 73, 78, 68, 32, 67, 76, 79, 85, 68, 69, 5, 87, 73, 84, + 72, 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 255, 44, 65, + 4, 38, 82, 29, 5, 83, 77, 65, 76, 76, 2, 11, 65, 2, 179, 110, 89, 2, 11, + 32, 2, 21, 3, 67, 76, 79, 2, 187, 101, 85, 10, 38, 79, 54, 69, 62, 82, + 203, 1, 87, 2, 49, 10, 85, 67, 72, 84, 79, 78, 69, 32, 84, 69, 2, 17, 2, + 76, 69, 2, 11, 80, 2, 11, 72, 2, 151, 17, 79, 4, 148, 1, 4, 65, 80, 69, + 90, 33, 28, 73, 65, 78, 71, 76, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, + 78, 71, 32, 83, 77, 65, 76, 76, 32, 87, 72, 73, 84, 2, 11, 73, 2, 183, + 152, 1, 85, 2, 135, 4, 69, 2, 11, 79, 2, 85, 19, 45, 87, 65, 89, 32, 76, + 69, 70, 84, 32, 87, 65, 89, 32, 84, 82, 65, 70, 70, 2, 11, 73, 2, 251, + 167, 1, 67, 12, 62, 32, 185, 1, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 4, 11, 80, 4, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 4, 18, 66, 75, + 73, 2, 21, 3, 65, 67, 75, 2, 25, 4, 72, 65, 78, 68, 2, 17, 2, 32, 73, 2, + 17, 2, 78, 68, 2, 219, 25, 69, 8, 54, 67, 42, 83, 125, 7, 84, 82, 73, 65, + 78, 71, 76, 2, 21, 3, 72, 69, 86, 2, 179, 85, 82, 2, 25, 4, 77, 65, 76, + 76, 2, 17, 2, 32, 84, 2, 17, 2, 82, 73, 2, 11, 65, 2, 11, 78, 2, 11, 71, + 2, 211, 141, 1, 76, 4, 11, 69, 5, 11, 32, 2, 11, 87, 2, 217, 18, 3, 73, + 84, 72, 10, 44, 6, 84, 73, 67, 65, 76, 32, 255, 1, 89, 8, 62, 69, 48, 9, + 82, 69, 67, 84, 65, 78, 71, 76, 69, 127, 66, 2, 11, 76, 2, 17, 2, 76, 73, + 2, 183, 25, 80, 5, 37, 7, 32, 87, 73, 84, 72, 32, 72, 2, 37, 7, 79, 82, + 73, 90, 79, 78, 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 66, 2, 219, 104, + 65, 2, 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 17, 2, 32, 83, + 2, 11, 81, 2, 11, 85, 2, 11, 65, 2, 227, 137, 1, 82, 2, 11, 69, 2, 11, + 68, 2, 17, 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 11, 87, 2, 241, + 4, 4, 65, 82, 68, 83, 100, 140, 1, 10, 68, 69, 45, 72, 69, 65, 68, 69, + 68, 32, 132, 4, 4, 71, 71, 76, 89, 124, 6, 76, 84, 69, 68, 32, 70, 42, + 78, 189, 1, 2, 82, 69, 80, 128, 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, + 12, 4, 68, 79, 87, 78, 0, 2, 85, 80, 32, 3, 78, 79, 82, 1, 3, 83, 79, 85, + 10, 11, 84, 10, 109, 5, 87, 65, 82, 68, 83, 20, 21, 3, 84, 72, 32, 20, + 32, 2, 69, 65, 1, 2, 87, 69, 10, 17, 2, 83, 84, 10, 11, 32, 10, 110, 72, + 0, 6, 86, 69, 82, 89, 32, 72, 28, 5, 76, 73, 71, 72, 84, 0, 6, 77, 69, + 68, 73, 85, 77, 23, 66, 2, 25, 4, 69, 65, 86, 89, 2, 17, 2, 32, 66, 2, + 21, 3, 65, 82, 66, 2, 11, 32, 2, 11, 65, 2, 11, 82, 2, 11, 82, 2, 131, + 28, 79, 2, 17, 2, 32, 86, 2, 11, 69, 2, 33, 6, 82, 84, 73, 67, 65, 76, 2, + 11, 32, 2, 11, 76, 2, 11, 73, 2, 215, 130, 1, 78, 2, 11, 76, 2, 11, 79, + 2, 147, 91, 87, 12, 46, 68, 106, 69, 186, 15, 75, 163, 136, 1, 71, 6, 22, + 32, 139, 26, 79, 4, 44, 2, 67, 72, 217, 15, 4, 66, 76, 79, 87, 2, 11, 73, + 2, 151, 129, 1, 77, 2, 11, 32, 2, 121, 3, 71, 76, 65, 4, 36, 5, 68, 32, + 75, 69, 89, 51, 76, 2, 17, 2, 66, 79, 2, 11, 65, 2, 203, 80, 82, 2, 11, + 69, 2, 243, 88, 83, 30, 66, 77, 158, 3, 82, 226, 11, 78, 226, 64, 79, + 129, 9, 2, 76, 70, 14, 36, 2, 65, 78, 161, 2, 2, 69, 78, 13, 72, 12, 32, + 87, 73, 84, 72, 32, 66, 85, 78, 78, 89, 32, 29, 2, 83, 32, 2, 11, 69, 2, + 167, 6, 65, 8, 48, 2, 66, 79, 28, 2, 67, 76, 54, 72, 19, 83, 2, 11, 79, + 2, 207, 86, 84, 2, 11, 79, 2, 11, 84, 2, 11, 72, 2, 155, 86, 69, 2, 187, + 119, 65, 2, 17, 2, 65, 78, 2, 131, 10, 68, 2, 11, 83, 2, 11, 32, 2, 11, + 83, 2, 11, 89, 2, 17, 2, 77, 66, 2, 187, 9, 79, 10, 72, 2, 68, 32, 156, + 1, 3, 76, 68, 32, 32, 2, 82, 73, 219, 144, 1, 77, 4, 56, 9, 83, 69, 80, + 65, 82, 65, 84, 79, 82, 203, 83, 74, 2, 17, 2, 32, 77, 2, 21, 3, 73, 68, + 68, 2, 11, 76, 2, 11, 69, 2, 11, 32, 2, 207, 64, 68, 2, 11, 77, 2, 223, + 144, 1, 65, 2, 11, 69, 2, 171, 83, 68, 10, 56, 7, 65, 80, 80, 69, 68, 32, + 80, 30, 69, 163, 1, 73, 2, 213, 76, 3, 82, 69, 83, 6, 48, 2, 65, 84, 80, + 3, 83, 84, 76, 187, 108, 78, 2, 11, 72, 2, 11, 32, 2, 11, 80, 2, 25, 4, + 82, 79, 68, 85, 2, 195, 114, 67, 2, 11, 69, 2, 247, 80, 82, 2, 11, 84, 2, + 11, 73, 2, 17, 2, 78, 71, 2, 17, 2, 32, 72, 2, 11, 65, 2, 215, 71, 78, + 34, 76, 2, 32, 73, 138, 1, 45, 28, 7, 73, 65, 78, 71, 81, 73, 32, 155, + 83, 79, 2, 53, 11, 78, 32, 65, 32, 82, 69, 67, 84, 65, 78, 71, 2, 11, 76, + 2, 11, 69, 2, 11, 32, 2, 11, 66, 2, 11, 79, 2, 163, 140, 1, 88, 2, 11, + 82, 2, 187, 122, 65, 28, 48, 5, 66, 76, 65, 67, 75, 1, 3, 82, 69, 68, 14, + 11, 32, 14, 130, 1, 67, 72, 4, 69, 76, 69, 80, 28, 4, 71, 69, 78, 69, 46, + 72, 36, 4, 83, 79, 76, 68, 169, 9, 6, 77, 65, 78, 68, 65, 82, 4, 24, 2, + 65, 78, 19, 72, 2, 167, 58, 78, 2, 201, 57, 3, 65, 82, 73, 2, 11, 72, 2, + 191, 70, 65, 2, 11, 82, 2, 11, 65, 2, 179, 137, 1, 76, 2, 17, 2, 79, 82, + 2, 183, 114, 83, 2, 143, 75, 73, 242, 19, 50, 65, 58, 69, 226, 11, 73, + 217, 35, 2, 79, 45, 2, 17, 2, 87, 78, 2, 11, 73, 2, 241, 74, 2, 78, 71, + 98, 60, 4, 76, 76, 79, 87, 62, 78, 53, 5, 90, 73, 68, 73, 32, 2, 17, 2, + 32, 72, 2, 11, 69, 2, 11, 65, 2, 227, 106, 82, 2, 11, 32, 2, 11, 83, 2, + 11, 73, 2, 251, 54, 71, 94, 112, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, + 32, 76, 5, 72, 89, 80, 72, 69, 17, 7, 76, 69, 84, 84, 69, 82, 32, 4, 44, + 3, 77, 65, 68, 13, 4, 72, 65, 77, 90, 2, 11, 68, 2, 215, 67, 65, 2, 215, + 27, 78, 88, 250, 1, 67, 50, 77, 18, 68, 42, 69, 58, 72, 30, 75, 22, 71, + 2, 81, 32, 3, 76, 65, 77, 98, 78, 18, 80, 30, 83, 66, 84, 28, 2, 86, 65, + 126, 87, 14, 79, 18, 88, 52, 3, 89, 79, 84, 154, 1, 90, 134, 53, 82, 238, + 48, 66, 254, 5, 85, 162, 14, 70, 3, 74, 6, 22, 72, 147, 114, 73, 4, 22, + 72, 251, 113, 73, 2, 247, 113, 73, 4, 11, 65, 4, 178, 130, 1, 68, 3, 76, + 8, 42, 76, 142, 50, 89, 226, 79, 84, 3, 87, 2, 71, 73, 4, 150, 112, 65, + 147, 15, 72, 4, 18, 72, 15, 65, 2, 11, 65, 2, 163, 129, 1, 70, 5, 11, 32, + 2, 11, 87, 2, 21, 3, 73, 84, 72, 2, 17, 2, 32, 68, 2, 11, 79, 2, 187, 3, + 84, 2, 207, 48, 85, 4, 202, 105, 72, 215, 22, 69, 8, 46, 72, 246, 47, 73, + 194, 9, 65, 163, 70, 69, 2, 243, 47, 73, 4, 238, 104, 72, 215, 22, 65, 5, + 37, 7, 32, 65, 76, 84, 69, 82, 78, 2, 17, 2, 65, 84, 2, 11, 69, 2, 11, + 32, 2, 11, 70, 2, 11, 79, 2, 227, 109, 82, 2, 11, 65, 2, 159, 126, 87, 4, + 22, 72, 251, 125, 65, 2, 11, 69, 2, 139, 46, 89, 5, 37, 7, 32, 87, 73, + 84, 72, 32, 67, 2, 45, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 2, 11, 32, + 2, 11, 65, 2, 11, 66, 2, 11, 79, 2, 255, 101, 86, 6, 22, 65, 175, 124, + 69, 5, 171, 124, 76, 140, 19, 34, 32, 161, 35, 3, 78, 32, 89, 138, 19, + 88, 8, 82, 65, 68, 73, 67, 65, 76, 32, 181, 7, 9, 83, 89, 76, 76, 65, 66, + 76, 69, 32, 110, 170, 1, 66, 42, 67, 86, 68, 42, 71, 74, 72, 66, 74, 66, + 75, 30, 76, 30, 77, 26, 78, 74, 80, 26, 83, 74, 84, 30, 86, 30, 89, 30, + 90, 250, 35, 81, 198, 49, 87, 235, 30, 79, 4, 22, 66, 247, 64, 85, 2, + 163, 93, 85, 12, 42, 85, 18, 89, 178, 98, 72, 203, 22, 73, 2, 135, 121, + 79, 7, 130, 121, 80, 3, 84, 4, 22, 68, 215, 120, 85, 2, 247, 63, 85, 10, + 42, 71, 238, 91, 79, 162, 28, 69, 15, 65, 4, 162, 89, 85, 235, 30, 79, 8, + 22, 88, 243, 88, 77, 6, 238, 88, 85, 202, 2, 73, 163, 28, 79, 8, 22, 74, + 167, 119, 79, 6, 246, 90, 85, 218, 5, 73, 215, 22, 89, 4, 206, 90, 73, + 175, 28, 69, 6, 190, 54, 73, 199, 7, 89, 4, 182, 118, 79, 15, 73, 8, 30, + 89, 26, 90, 199, 90, 66, 4, 254, 117, 73, 3, 79, 2, 231, 117, 85, 4, 182, + 89, 85, 3, 89, 10, 22, 72, 223, 97, 83, 8, 214, 60, 85, 178, 28, 65, 162, + 28, 79, 15, 89, 4, 214, 88, 65, 175, 28, 85, 4, 138, 60, 85, 211, 56, 69, + 4, 158, 88, 73, 175, 28, 79, 10, 54, 85, 224, 62, 2, 90, 73, 238, 24, 79, + 175, 28, 65, 4, 246, 115, 80, 3, 82, 156, 18, 134, 2, 66, 134, 1, 67, + 162, 1, 68, 110, 70, 50, 71, 150, 1, 72, 138, 3, 73, 134, 1, 74, 98, 75, + 54, 76, 62, 77, 134, 1, 78, 234, 4, 80, 54, 81, 2, 89, 46, 82, 162, 1, + 83, 134, 1, 84, 102, 86, 82, 87, 58, 88, 50, 90, 140, 2, 2, 85, 79, 66, + 65, 2, 79, 107, 69, 132, 1, 66, 66, 238, 21, 85, 150, 1, 69, 26, 73, 42, + 65, 2, 79, 67, 89, 64, 234, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 3, + 89, 122, 66, 72, 238, 20, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, 89, + 54, 46, 85, 146, 22, 65, 2, 69, 2, 79, 67, 89, 19, 142, 22, 79, 106, 82, + 230, 88, 80, 3, 88, 106, 58, 68, 138, 15, 85, 134, 5, 73, 94, 69, 66, 65, + 3, 79, 54, 210, 19, 85, 58, 73, 94, 69, 66, 65, 3, 79, 42, 182, 20, 79, + 66, 65, 2, 73, 2, 89, 67, 85, 116, 58, 71, 214, 15, 85, 146, 4, 73, 42, + 65, 2, 69, 3, 79, 56, 50, 73, 162, 15, 85, 186, 4, 65, 2, 69, 3, 79, 13, + 150, 19, 69, 142, 90, 84, 3, 88, 246, 1, 78, 73, 30, 76, 58, 77, 54, 78, + 110, 88, 50, 85, 254, 15, 69, 66, 65, 3, 79, 6, 198, 19, 69, 231, 88, 84, + 64, 238, 16, 85, 58, 73, 94, 69, 2, 79, 66, 65, 67, 89, 58, 182, 16, 85, + 58, 73, 158, 1, 65, 2, 79, 35, 89, 42, 46, 79, 34, 85, 202, 16, 69, 26, + 73, 43, 65, 6, 242, 106, 80, 2, 84, 3, 88, 6, 238, 17, 79, 231, 88, 84, + 46, 46, 85, 254, 15, 69, 26, 73, 42, 65, 3, 79, 8, 187, 16, 79, 19, 42, + 84, 130, 16, 69, 206, 89, 80, 3, 88, 5, 11, 69, 2, 11, 82, 2, 11, 65, 2, + 11, 84, 2, 11, 73, 2, 11, 79, 2, 191, 39, 78, 106, 50, 74, 190, 10, 85, + 146, 4, 73, 42, 79, 67, 89, 50, 158, 13, 85, 174, 1, 73, 42, 79, 3, 89, + 56, 242, 12, 85, 58, 73, 158, 1, 65, 2, 69, 3, 79, 70, 218, 9, 85, 250, + 3, 69, 26, 73, 42, 65, 2, 79, 67, 89, 106, 70, 71, 218, 8, 85, 158, 3, + 73, 158, 1, 65, 2, 79, 2, 89, 107, 69, 44, 186, 11, 85, 150, 1, 69, 66, + 65, 2, 79, 105, 2, 73, 69, 248, 2, 102, 66, 54, 68, 94, 71, 90, 74, 46, + 82, 50, 89, 78, 90, 134, 7, 85, 58, 73, 94, 65, 2, 69, 67, 79, 54, 202, + 10, 73, 158, 1, 65, 2, 79, 66, 85, 3, 89, 46, 46, 73, 198, 10, 69, 66, + 65, 2, 79, 67, 85, 13, 234, 11, 69, 230, 88, 80, 2, 84, 3, 88, 34, 60, 2, + 85, 79, 218, 9, 69, 0, 2, 73, 69, 66, 65, 3, 79, 7, 226, 99, 84, 3, 88, + 50, 230, 1, 85, 242, 7, 73, 42, 79, 67, 89, 46, 146, 9, 79, 66, 65, 2, + 69, 66, 85, 3, 89, 38, 30, 85, 222, 8, 73, 43, 79, 15, 194, 8, 79, 142, + 90, 80, 2, 84, 3, 88, 56, 62, 85, 206, 4, 79, 178, 2, 73, 158, 1, 65, 66, + 89, 43, 69, 15, 254, 8, 79, 2, 82, 230, 88, 80, 3, 88, 60, 150, 6, 85, + 58, 73, 158, 1, 65, 2, 79, 67, 89, 56, 254, 2, 85, 146, 4, 73, 42, 79, + 67, 89, 100, 58, 82, 254, 4, 85, 150, 1, 69, 66, 65, 2, 79, 67, 89, 48, + 46, 85, 162, 6, 69, 2, 79, 66, 89, 43, 65, 17, 134, 7, 79, 2, 82, 230, + 88, 80, 2, 84, 3, 88, 176, 1, 70, 83, 158, 3, 72, 50, 85, 58, 73, 94, 69, + 66, 65, 2, 79, 67, 89, 56, 130, 4, 73, 94, 69, 66, 65, 2, 79, 2, 85, 67, + 89, 56, 46, 85, 158, 3, 73, 94, 69, 66, 65, 3, 79, 21, 182, 4, 79, 106, + 82, 230, 88, 80, 2, 84, 3, 88, 60, 54, 69, 166, 3, 73, 42, 65, 2, 79, 66, + 85, 3, 89, 4, 150, 93, 80, 3, 88, 28, 38, 85, 206, 2, 69, 2, 79, 67, 65, + 9, 203, 2, 79, 40, 210, 2, 73, 42, 79, 66, 89, 41, 2, 85, 79, 178, 1, 66, + 72, 50, 85, 58, 73, 42, 90, 54, 69, 66, 65, 2, 79, 67, 89, 54, 46, 85, + 214, 1, 65, 2, 69, 2, 79, 67, 89, 19, 146, 1, 79, 170, 1, 82, 230, 88, + 80, 2, 84, 3, 88, 15, 90, 69, 142, 90, 80, 2, 84, 3, 88, 58, 50, 69, 2, + 79, 26, 73, 42, 65, 34, 85, 35, 89, 7, 138, 90, 80, 3, 88, 17, 38, 69, + 206, 89, 80, 2, 84, 3, 88, 9, 202, 89, 80, 2, 84, 3, 88, 11, 70, 82, 230, + 88, 80, 3, 88, 13, 38, 82, 230, 88, 80, 2, 84, 3, 88, 5, 227, 88, 88, 2, + 219, 7, 65, 2, 207, 57, 89, 180, 4, 252, 1, 10, 32, 78, 79, 84, 65, 84, + 73, 79, 78, 32, 224, 6, 16, 65, 78, 65, 66, 65, 90, 65, 82, 32, 83, 81, + 85, 65, 82, 69, 32, 210, 15, 69, 180, 2, 6, 73, 80, 80, 69, 82, 45, 96, + 8, 78, 65, 77, 69, 78, 78, 89, 32, 188, 33, 3, 79, 77, 66, 235, 26, 87, + 26, 144, 1, 9, 66, 65, 71, 32, 77, 69, 77, 66, 69, 42, 82, 104, 6, 68, + 79, 77, 65, 73, 78, 60, 3, 76, 69, 70, 154, 1, 83, 145, 2, 3, 84, 89, 80, + 2, 11, 82, 2, 11, 83, 2, 215, 83, 72, 8, 100, 4, 65, 78, 71, 69, 60, 3, + 73, 71, 72, 221, 1, 11, 69, 76, 65, 84, 73, 79, 78, 65, 76, 32, 67, 2, + 173, 3, 11, 32, 65, 78, 84, 73, 82, 69, 83, 84, 82, 73, 4, 17, 2, 84, 32, + 4, 60, 4, 73, 77, 65, 71, 13, 7, 66, 73, 78, 68, 73, 78, 71, 2, 11, 69, + 2, 25, 4, 32, 66, 82, 65, 2, 11, 67, 2, 175, 29, 75, 8, 44, 6, 67, 72, + 69, 77, 65, 32, 211, 1, 80, 6, 18, 67, 71, 80, 2, 17, 2, 79, 77, 2, 11, + 80, 2, 11, 79, 2, 11, 83, 2, 107, 73, 4, 30, 73, 41, 3, 82, 79, 74, 2, + 11, 80, 2, 11, 73, 2, 163, 80, 78, 2, 11, 69, 2, 11, 67, 2, 11, 84, 2, + 87, 73, 2, 139, 52, 79, 2, 11, 69, 2, 11, 32, 2, 11, 67, 2, 11, 79, 2, + 11, 76, 2, 11, 79, 2, 223, 79, 78, 144, 1, 240, 1, 2, 67, 76, 68, 7, 73, + 78, 73, 84, 73, 65, 76, 204, 2, 14, 70, 73, 78, 65, 76, 32, 67, 79, 78, + 83, 79, 78, 65, 78, 16, 7, 76, 69, 84, 84, 69, 82, 32, 148, 3, 5, 77, 65, + 82, 75, 32, 206, 1, 83, 181, 3, 6, 86, 79, 87, 69, 76, 32, 14, 64, 5, 79, + 83, 73, 78, 71, 137, 1, 6, 85, 83, 84, 69, 82, 45, 4, 11, 32, 4, 64, 12, + 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 68, 23, 72, 2, 17, 2, 32, 72, + 2, 17, 2, 69, 65, 2, 215, 10, 68, 10, 96, 13, 70, 73, 78, 65, 76, 32, 76, + 69, 84, 84, 69, 82, 32, 41, 7, 73, 78, 73, 84, 73, 65, 76, 8, 238, 72, + 76, 2, 82, 2, 86, 3, 89, 2, 37, 7, 32, 76, 69, 84, 84, 69, 82, 2, 151, 6, + 32, 2, 131, 9, 84, 82, 166, 1, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, + 66, 2, 67, 2, 71, 2, 80, 2, 90, 138, 69, 45, 2, 72, 2, 74, 2, 76, 2, 77, + 2, 82, 2, 86, 2, 89, 187, 2, 65, 12, 206, 1, 68, 2, 90, 138, 69, 72, 187, + 2, 65, 6, 154, 70, 83, 14, 72, 187, 2, 65, 8, 130, 70, 71, 2, 78, 2, 89, + 187, 2, 65, 6, 214, 69, 72, 2, 83, 187, 2, 65, 12, 42, 83, 2, 84, 138, + 69, 72, 187, 2, 65, 4, 134, 69, 72, 187, 2, 65, 8, 50, 68, 58, 83, 40, 4, + 76, 79, 78, 71, 23, 84, 2, 29, 5, 79, 85, 66, 76, 69, 2, 11, 32, 2, 11, + 83, 2, 11, 72, 2, 11, 65, 2, 159, 70, 68, 2, 17, 2, 32, 84, 2, 17, 2, 83, + 72, 2, 147, 69, 69, 14, 36, 4, 73, 71, 78, 32, 255, 2, 85, 12, 54, 65, + 72, 6, 67, 65, 78, 68, 82, 65, 167, 1, 86, 2, 11, 78, 2, 11, 85, 2, 17, + 2, 83, 86, 2, 11, 65, 2, 135, 66, 82, 6, 36, 5, 66, 73, 78, 68, 85, 15, + 32, 5, 11, 32, 2, 25, 4, 87, 73, 84, 72, 2, 17, 2, 32, 79, 2, 21, 3, 82, + 78, 65, 2, 11, 77, 2, 11, 69, 2, 239, 38, 78, 4, 11, 73, 4, 18, 82, 19, + 83, 2, 219, 33, 65, 2, 11, 65, 2, 11, 82, 2, 139, 64, 71, 2, 151, 4, 66, + 20, 38, 76, 109, 5, 83, 73, 71, 78, 32, 2, 17, 2, 69, 78, 2, 11, 71, 2, + 11, 84, 2, 11, 72, 2, 11, 32, 2, 11, 77, 2, 11, 65, 2, 135, 62, 82, 18, + 86, 65, 26, 79, 2, 85, 16, 8, 82, 69, 86, 69, 82, 83, 69, 68, 146, 64, + 69, 3, 73, 4, 182, 64, 73, 3, 85, 5, 159, 64, 69, 2, 183, 44, 32, 12, 72, + 2, 66, 82, 16, 9, 82, 79, 32, 87, 73, 68, 84, 72, 32, 203, 1, 85, 2, 147, + 2, 65, 8, 32, 2, 78, 79, 86, 83, 31, 74, 4, 26, 45, 73, 2, 78, 45, 2, 29, + 5, 66, 82, 69, 65, 75, 2, 11, 32, 2, 11, 83, 2, 163, 1, 80, 2, 11, 74, 2, + 11, 79, 2, 11, 73, 2, 11, 78, 2, 143, 5, 69, 2, 219, 61, 83, 2, 21, 3, + 77, 79, 85, 2, 17, 2, 84, 72, 2, 11, 32, 2, 11, 70, 2, 11, 65, 2, 167, + 38, 67, 242, 2, 168, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 224, + 18, 6, 78, 69, 85, 77, 69, 32, 181, 38, 17, 80, 82, 73, 90, 78, 65, 75, + 32, 77, 79, 68, 73, 70, 73, 69, 82, 32, 128, 1, 144, 2, 17, 76, 79, 87, + 69, 82, 32, 84, 79, 78, 65, 76, 32, 82, 65, 78, 71, 69, 88, 5, 77, 65, + 82, 75, 32, 148, 3, 18, 65, 84, 84, 65, 67, 72, 73, 78, 71, 32, 86, 69, + 82, 84, 73, 67, 65, 76, 233, 11, 17, 84, 79, 78, 65, 76, 32, 82, 65, 78, + 71, 69, 32, 77, 65, 82, 75, 32, 2, 33, 6, 32, 73, 78, 68, 73, 67, 2, 11, + 65, 2, 11, 84, 2, 11, 79, 2, 219, 56, 82, 118, 182, 2, 67, 142, 1, 68, + 104, 8, 71, 79, 82, 65, 90, 68, 79, 32, 34, 78, 106, 75, 114, 79, 72, 2, + 80, 79, 152, 2, 8, 77, 65, 76, 79, 32, 80, 79, 86, 100, 2, 82, 65, 50, + 83, 186, 1, 84, 108, 7, 86, 89, 83, 79, 75, 79, 32, 234, 1, 90, 224, 10, + 2, 76, 79, 224, 22, 4, 85, 68, 65, 82, 129, 6, 4, 66, 79, 82, 90, 6, 60, + 6, 72, 65, 83, 72, 75, 65, 29, 5, 85, 82, 86, 69, 68, 5, 197, 47, 3, 32, + 80, 79, 2, 17, 2, 32, 79, 2, 11, 77, 2, 235, 24, 69, 4, 240, 10, 13, 69, + 77, 69, 83, 84, 86, 69, 78, 78, 89, 32, 90, 65, 217, 14, 6, 86, 79, 69, + 84, 79, 67, 10, 30, 78, 89, 3, 86, 89, 83, 8, 29, 5, 73, 90, 75, 79, 32, + 8, 164, 8, 8, 83, 32, 75, 82, 89, 90, 72, 69, 35, 79, 2, 167, 20, 79, 8, + 56, 4, 82, 89, 90, 72, 150, 6, 65, 233, 39, 2, 85, 80, 5, 21, 3, 32, 79, + 78, 2, 11, 32, 2, 203, 7, 76, 6, 220, 5, 3, 84, 83, 69, 200, 13, 5, 66, + 76, 65, 67, 72, 131, 31, 78, 18, 22, 68, 131, 2, 86, 6, 60, 7, 67, 72, + 65, 83, 72, 73, 69, 201, 22, 3, 86, 69, 82, 5, 17, 2, 32, 87, 2, 21, 3, + 73, 84, 72, 2, 17, 2, 32, 86, 2, 29, 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, + 76, 2, 11, 32, 2, 11, 83, 2, 11, 84, 2, 11, 82, 2, 11, 79, 2, 131, 25, + 75, 12, 29, 5, 89, 83, 72, 69, 32, 12, 22, 83, 251, 3, 79, 8, 154, 3, 32, + 229, 2, 4, 84, 82, 65, 78, 4, 28, 2, 90, 83, 183, 5, 86, 2, 219, 37, 69, + 10, 136, 1, 2, 75, 79, 16, 16, 84, 82, 65, 78, 78, 79, 32, 77, 65, 76, + 79, 32, 80, 79, 86, 89, 236, 1, 5, 82, 69, 68, 78, 69, 231, 26, 79, 2, + 239, 42, 66, 2, 11, 83, 2, 183, 22, 72, 10, 50, 79, 16, 5, 83, 65, 84, + 65, 32, 139, 36, 73, 2, 187, 28, 67, 6, 158, 1, 79, 201, 24, 3, 83, 32, + 75, 10, 24, 2, 83, 32, 95, 79, 6, 11, 75, 6, 44, 6, 72, 79, 75, 72, 76, + 79, 247, 24, 82, 4, 11, 77, 4, 17, 2, 32, 79, 4, 11, 78, 4, 11, 32, 4, + 18, 76, 31, 82, 2, 11, 69, 2, 179, 14, 70, 2, 11, 73, 2, 11, 71, 2, 139, + 14, 72, 6, 18, 65, 31, 69, 2, 249, 25, 3, 68, 69, 82, 4, 22, 86, 239, 10, + 76, 2, 199, 38, 79, 6, 60, 5, 77, 82, 65, 67, 72, 18, 83, 1, 4, 84, 82, + 69, 83, 2, 155, 10, 78, 2, 17, 2, 86, 69, 2, 155, 6, 84, 232, 1, 128, 2, + 2, 67, 72, 38, 68, 218, 1, 70, 28, 10, 71, 79, 76, 85, 66, 67, 72, 73, + 75, 32, 158, 1, 75, 160, 2, 6, 77, 69, 67, 72, 73, 75, 184, 1, 2, 78, 69, + 18, 79, 138, 2, 80, 146, 2, 82, 114, 83, 180, 20, 9, 86, 82, 65, 75, 72, + 73, 89, 65, 32, 151, 2, 90, 4, 246, 11, 69, 209, 10, 2, 65, 83, 10, 74, + 69, 98, 85, 16, 9, 86, 65, 32, 86, 32, 67, 72, 69, 76, 187, 27, 79, 4, + 76, 2, 82, 66, 213, 2, 12, 77, 69, 83, 84, 86, 69, 78, 78, 89, 32, 75, + 76, 2, 195, 34, 73, 2, 203, 34, 68, 2, 11, 78, 2, 231, 36, 85, 2, 11, 73, + 2, 147, 34, 84, 10, 78, 84, 38, 83, 240, 3, 5, 77, 82, 65, 67, 72, 145, + 14, 4, 66, 79, 82, 90, 4, 32, 3, 82, 69, 83, 251, 1, 73, 2, 21, 3, 86, + 69, 84, 2, 231, 17, 76, 14, 76, 4, 72, 65, 77, 73, 18, 76, 40, 3, 79, 66, + 89, 16, 2, 82, 89, 87, 85, 2, 219, 3, 76, 2, 11, 89, 2, 11, 85, 2, 151, + 33, 67, 2, 223, 31, 76, 6, 28, 2, 85, 75, 219, 32, 90, 5, 21, 3, 32, 84, + 73, 2, 11, 75, 2, 251, 15, 72, 2, 11, 70, 2, 11, 73, 2, 11, 83, 2, 215, + 30, 77, 11, 11, 32, 8, 44, 7, 75, 76, 89, 85, 67, 72, 69, 83, 80, 6, 64, + 9, 78, 69, 80, 79, 83, 84, 79, 89, 65, 14, 80, 163, 14, 86, 2, 39, 78, 2, + 25, 4, 79, 86, 79, 68, 2, 143, 14, 78, 2, 223, 22, 77, 14, 40, 2, 66, 76, + 41, 4, 83, 79, 75, 65, 2, 11, 65, 2, 11, 75, 2, 243, 30, 79, 13, 11, 32, + 10, 30, 75, 142, 26, 84, 39, 83, 6, 104, 11, 76, 89, 85, 67, 72, 69, 86, + 65, 89, 65, 32, 185, 25, 10, 82, 89, 85, 75, 79, 86, 65, 89, 65, 32, 4, + 202, 16, 78, 251, 8, 83, 14, 58, 65, 88, 8, 69, 82, 69, 86, 79, 68, 75, + 65, 27, 79, 6, 44, 3, 82, 65, 75, 222, 19, 76, 211, 5, 85, 2, 11, 76, 2, + 11, 73, 2, 171, 28, 84, 5, 153, 15, 2, 32, 78, 4, 68, 5, 68, 67, 72, 65, + 83, 245, 9, 7, 76, 75, 85, 76, 73, 90, 77, 2, 11, 72, 2, 219, 4, 73, 4, + 64, 11, 69, 86, 69, 82, 83, 69, 68, 32, 67, 72, 69, 139, 26, 79, 2, 25, + 4, 76, 89, 85, 83, 2, 215, 17, 84, 126, 96, 9, 75, 65, 77, 69, 89, 84, + 83, 65, 32, 148, 2, 8, 76, 79, 90, 72, 73, 84, 73, 69, 119, 84, 22, 128, + 1, 13, 68, 86, 79, 69, 67, 72, 69, 76, 78, 65, 89, 65, 32, 44, 7, 75, 76, + 89, 85, 67, 72, 69, 74, 84, 218, 18, 77, 119, 83, 8, 218, 10, 75, 110, + 78, 178, 8, 80, 75, 83, 6, 40, 5, 86, 65, 89, 65, 32, 243, 10, 78, 4, + 178, 15, 84, 183, 4, 83, 4, 162, 15, 73, 147, 4, 82, 9, 11, 32, 6, 48, 2, + 83, 32, 25, 6, 90, 65, 75, 82, 89, 84, 4, 166, 4, 75, 95, 90, 2, 11, 79, + 2, 211, 22, 69, 96, 92, 4, 65, 84, 89, 65, 172, 4, 6, 79, 80, 73, 84, 83, + 65, 189, 1, 5, 82, 69, 76, 65, 32, 23, 11, 32, 20, 76, 2, 83, 32, 236, 2, + 9, 90, 65, 75, 82, 89, 84, 65, 89, 65, 159, 5, 78, 14, 160, 1, 14, 68, + 86, 85, 77, 89, 65, 32, 90, 65, 80, 89, 65, 84, 89, 28, 7, 75, 82, 89, + 90, 72, 69, 77, 24, 2, 82, 79, 17, 8, 90, 65, 80, 89, 65, 84, 79, 89, 2, + 11, 77, 2, 215, 19, 73, 5, 189, 1, 2, 32, 73, 2, 203, 2, 71, 7, 21, 3, + 32, 73, 32, 4, 54, 75, 37, 9, 80, 79, 68, 67, 72, 65, 83, 72, 73, 2, 11, + 82, 2, 21, 3, 89, 90, 72, 2, 211, 1, 69, 5, 17, 2, 32, 83, 2, 17, 2, 32, + 90, 2, 29, 5, 65, 80, 89, 65, 84, 2, 11, 79, 2, 199, 17, 89, 7, 11, 32, + 4, 68, 6, 83, 32, 79, 67, 72, 75, 29, 7, 87, 73, 84, 72, 32, 83, 79, 2, + 11, 79, 2, 215, 16, 77, 2, 45, 9, 82, 79, 67, 72, 89, 65, 32, 78, 79, 2, + 11, 90, 2, 163, 7, 72, 68, 140, 1, 9, 68, 86, 79, 69, 67, 72, 69, 76, 78, + 162, 1, 75, 78, 78, 174, 1, 71, 152, 3, 8, 77, 82, 65, 67, 72, 78, 79, + 84, 42, 80, 75, 84, 10, 18, 79, 71, 65, 6, 64, 7, 80, 79, 86, 79, 68, 78, + 65, 189, 5, 4, 75, 82, 89, 90, 4, 17, 2, 89, 65, 5, 17, 2, 32, 75, 2, + 145, 5, 4, 76, 89, 85, 67, 26, 48, 6, 76, 89, 85, 67, 72, 69, 77, 2, 82, + 89, 4, 22, 78, 203, 6, 80, 2, 157, 8, 9, 69, 80, 79, 83, 84, 79, 89, 65, + 78, 22, 48, 7, 85, 75, 79, 86, 65, 89, 65, 195, 3, 90, 21, 11, 32, 18, + 54, 71, 236, 2, 5, 84, 82, 89, 65, 83, 179, 2, 80, 14, 21, 3, 82, 79, 77, + 14, 32, 4, 78, 65, 89, 65, 47, 79, 5, 237, 1, 7, 32, 87, 73, 84, 72, 32, + 83, 10, 92, 10, 75, 82, 89, 90, 72, 69, 86, 65, 89, 65, 17, 9, 80, 79, + 86, 79, 68, 78, 65, 89, 65, 5, 175, 2, 32, 7, 33, 6, 32, 87, 73, 84, 72, + 32, 4, 24, 2, 68, 79, 23, 83, 2, 41, 2, 85, 66, 2, 21, 3, 73, 78, 71, 2, + 133, 6, 6, 76, 69, 32, 90, 65, 80, 2, 175, 6, 75, 2, 237, 5, 3, 72, 69, + 86, 2, 11, 73, 2, 11, 75, 2, 187, 5, 72, 6, 22, 79, 187, 3, 82, 4, 28, 2, + 76, 85, 187, 1, 86, 2, 163, 1, 80, 8, 80, 5, 82, 89, 65, 83, 79, 153, 2, + 10, 73, 75, 72, 65, 89, 65, 32, 80, 85, 84, 6, 62, 80, 44, 4, 83, 84, 82, + 69, 173, 1, 4, 71, 76, 65, 83, 2, 17, 2, 79, 86, 2, 193, 1, 2, 79, 68, 2, + 171, 1, 76, 16, 88, 12, 75, 76, 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, + 38, 77, 46, 80, 38, 84, 39, 83, 8, 34, 77, 46, 80, 38, 84, 39, 83, 2, 25, + 4, 82, 65, 67, 72, 2, 247, 1, 78, 2, 11, 82, 2, 205, 1, 2, 79, 83, 2, 11, + 82, 2, 11, 69, 2, 11, 83, 2, 157, 1, 4, 86, 69, 84, 76, 6, 30, 65, 121, + 3, 77, 69, 89, 4, 32, 4, 78, 79, 90, 72, 31, 80, 2, 11, 69, 2, 151, 3, + 75, 2, 17, 2, 89, 65, 2, 11, 84, 2, 11, 65, 2, 35, 89, 2, 11, 84, 2, 11, + 83, 2, 183, 2, 65, 10, 108, 11, 68, 73, 82, 69, 67, 84, 73, 79, 78, 32, + 70, 28, 3, 75, 82, 89, 28, 5, 76, 69, 86, 69, 76, 35, 82, 2, 11, 76, 2, + 159, 1, 73, 2, 11, 90, 2, 143, 1, 72, 4, 11, 45, 4, 114, 50, 3, 51, 2, + 11, 79, 2, 83, 71, 8, 26, 78, 34, 83, 15, 74, 4, 18, 66, 27, 74, 2, 11, + 83, 2, 11, 80, 3, 0, }; static const unsigned int dawg_pos_to_codepoint[] = { @@ -8959,7 +9250,7 @@ static const unsigned int dawg_pos_to_codepoint[] = { 71458, 71459, 71462, 71464, 71483, 71482, 71477, 71476, 71479, 71478, 71475, 71474, 71472, 71481, 71473, 71480, 9992, 128747, 128748, 128874, 128822, 128823, 128837, 128776, 128777, 128774, 128775, 128773, 128811, - 128859, 128829, 128826, 128855, 128769, 128875, 128876, 128830, 128834, + 128859, 128826, 128829, 128855, 128769, 128875, 128876, 128830, 128834, 128835, 128836, 128783, 128857, 128848, 128846, 128844, 128805, 128800, 128803, 128804, 128798, 128869, 128870, 128871, 128872, 128873, 128787, 128865, 128866, 128864, 128880, 128794, 128841, 128883, 128882, 128854, @@ -8971,525 +9262,558 @@ static const unsigned int dawg_pos_to_codepoint[] = { 128862, 128781, 128839, 128795, 128852, 128847, 128831, 128832, 128840, 128809, 128849, 128845, 128778, 128815, 128779, 128780, 128790, 128791, 128808, 128772, 128842, 128853, 128771, 128838, 128770, 9879, 8501, - 983054, 8780, 9006, 983203, 8776, 10863, 8778, 9095, 9941, 128126, 9200, - 38, 127994, 127944, 128657, 10815, 82970, 82971, 82964, 82965, 82966, - 82967, 82968, 82969, 82972, 82973, 82974, 82994, 82995, 82996, 82987, - 82988, 82992, 82993, 82986, 82989, 82990, 82991, 82997, 82998, 82999, - 83016, 83017, 83018, 83019, 83010, 83011, 83012, 83013, 83014, 83015, - 83020, 83021, 83022, 83050, 83051, 83052, 83053, 83043, 83044, 83045, - 83046, 83047, 83048, 83049, 83054, 82984, 82985, 82975, 82976, 82977, - 82978, 82979, 82980, 82981, 82982, 82983, 82953, 82954, 82955, 82956, - 82957, 82958, 82959, 82960, 82961, 82962, 82963, 82944, 82945, 82946, - 82947, 82948, 82949, 82950, 82951, 82952, 83000, 83001, 83002, 83003, - 83004, 83005, 83006, 83007, 83008, 83009, 83023, 83024, 83025, 83026, - 83027, 83028, 83029, 83030, 83031, 83032, 83033, 83034, 83035, 83036, - 83037, 83038, 83039, 83040, 83041, 83042, 83062, 83063, 83064, 83065, - 83070, 83071, 83072, 83073, 83066, 83067, 83068, 83055, 83056, 83057, - 83058, 83059, 83060, 83061, 83069, 83074, 83075, 83076, 83077, 83078, - 83083, 83084, 83079, 83080, 83081, 83082, 83085, 83086, 83087, 83088, - 83094, 83095, 83089, 83090, 83091, 83092, 83093, 83096, 83097, 83098, - 83099, 83105, 83106, 83100, 83101, 83102, 83103, 83104, 83107, 83108, - 83109, 83110, 83111, 83112, 83113, 83114, 83115, 83116, 83117, 83118, - 83119, 83120, 83121, 83122, 83123, 83124, 83125, 83126, 83127, 83128, - 83129, 83130, 83131, 83132, 83133, 83134, 83135, 83136, 83137, 83138, - 83139, 83140, 83141, 83142, 83143, 83144, 83145, 83146, 83147, 83148, - 83149, 83150, 83151, 83152, 83153, 83154, 83155, 83156, 83157, 83158, - 83159, 83160, 83161, 83162, 83163, 83164, 83165, 83166, 83167, 83168, - 83169, 83170, 83173, 83174, 83175, 83180, 83181, 83183, 83184, 83171, - 83172, 83176, 83177, 83178, 83179, 83182, 83190, 83191, 83192, 83193, - 83185, 83186, 83187, 83188, 83189, 83194, 83195, 83196, 83274, 83275, - 83280, 83281, 83270, 83271, 83272, 83273, 83276, 83277, 83278, 83279, - 83268, 83269, 83259, 83260, 83261, 83262, 83263, 83264, 83265, 83266, - 83267, 83204, 83205, 83197, 83198, 83199, 83200, 83201, 83202, 83203, - 83206, 83207, 83245, 83246, 83238, 83239, 83240, 83241, 83242, 83243, - 83244, 83247, 83248, 83208, 83209, 83210, 83211, 83212, 83213, 83214, - 83215, 83216, 83217, 83218, 83219, 83220, 83221, 83222, 83223, 83224, - 83225, 83226, 83227, 83228, 83229, 83230, 83231, 83232, 83233, 83234, - 83235, 83236, 83237, 83249, 83250, 83251, 83252, 83253, 83254, 83255, - 83256, 83257, 83258, 83291, 83292, 83282, 83283, 83284, 83285, 83286, - 83287, 83288, 83289, 83290, 83312, 83313, 83303, 83304, 83305, 83306, - 83307, 83308, 83309, 83310, 83311, 83348, 83349, 83339, 83340, 83341, - 83342, 83343, 83344, 83345, 83346, 83347, 83322, 83323, 83324, 83325, - 83316, 83317, 83318, 83314, 83315, 83319, 83320, 83321, 83326, 83327, - 83328, 83354, 83355, 83359, 83360, 83350, 83351, 83352, 83353, 83356, - 83357, 83358, 83361, 83377, 83378, 83374, 83375, 83381, 83382, 83373, - 83376, 83379, 83380, 83383, 83384, 83385, 83389, 83386, 83387, 83388, - 83390, 83391, 83392, 83393, 83394, 83395, 83363, 83364, 83362, 83365, - 83366, 83367, 83368, 83369, 83370, 83371, 83372, 83293, 83294, 83295, - 83296, 83297, 83298, 83299, 83300, 83301, 83302, 83329, 83330, 83331, - 83332, 83333, 83334, 83335, 83336, 83337, 83338, 83406, 83407, 83408, - 83409, 83410, 83411, 83412, 83413, 83414, 83415, 83416, 83447, 83448, - 83455, 83456, 83449, 83450, 83451, 83452, 83453, 83454, 83457, 83458, - 83489, 83490, 83491, 83492, 83493, 83494, 83495, 83496, 83396, 83397, - 83398, 83399, 83400, 83401, 83402, 83403, 83404, 83405, 83417, 83418, - 83419, 83420, 83421, 83422, 83423, 83424, 83425, 83426, 83427, 83428, - 83429, 83430, 83431, 83432, 83433, 83434, 83435, 83436, 83437, 83438, - 83439, 83440, 83441, 83442, 83443, 83444, 83445, 83446, 83459, 83460, - 83461, 83462, 83463, 83464, 83465, 83466, 83467, 83468, 83469, 83470, - 83471, 83472, 83473, 83474, 83475, 83476, 83477, 83478, 83479, 83480, - 83481, 83482, 83483, 83484, 83485, 83486, 83487, 83488, 83526, 83497, - 83498, 83499, 83500, 83501, 83502, 83503, 83504, 83505, 83506, 83507, - 83508, 83509, 83510, 83511, 83512, 83513, 83514, 83515, 83516, 83517, - 83518, 83519, 83520, 83521, 83522, 83523, 83524, 83525, 129728, 8736, - 10654, 10660, 128551, 8491, 128162, 128544, 128028, 11150, 11148, 11149, - 11151, 11119, 8630, 10560, 8755, 128260, 10226, 8634, 10769, 128246, - 9875, 10193, 9765, 9021, 9055, 9061, 9033, 9052, 9022, 9066, 9058, 9067, - 9042, 9049, 9035, 9034, 9038, 9062, 9073, 9046, 9050, 9075, 9080, 9014, - 9082, 9078, 9077, 9081, 9060, 9051, 9063, 9029, 9109, 9020, 9056, 9017, - 9018, 9036, 9047, 9044, 9037, 9043, 9040, 9027, 9031, 9072, 9071, 9016, - 9026, 9025, 9028, 9032, 9019, 9054, 9048, 9030, 9076, 9070, 9023, 9059, - 9069, 9015, 9079, 9024, 9074, 9057, 9041, 9045, 9053, 9039, 9065, 9064, - 9068, 39, 11236, 8773, 8786, 10864, 8774, 8784, 983195, 983196, 2277, - 2280, 2276, 2279, 2278, 2281, 1548, 1615, 65144, 65145, 2302, 1612, - 65138, 1549, 2206, 2299, 2300, 2274, 1643, 1771, 1770, 1757, 1565, 1614, - 1630, 2293, 2292, 65142, 65143, 1611, 65136, 1538, 1645, 1748, 2207, - 1621, 1620, 1616, 2294, 65146, 65147, 1613, 65140, 2258, 2255, 2254, - 2257, 2236, 2244, 2235, 2237, 1593, 1886, 2227, 1696, 1887, 1885, 65226, - 65227, 65225, 65228, 1575, 2165, 2176, 2173, 2161, 2174, 2171, 2167, - 2177, 2169, 2166, 2168, 2178, 2164, 2160, 2162, 2175, 2172, 1571, 65156, - 65155, 1573, 65160, 65159, 1651, 1650, 1908, 1907, 1570, 65154, 65153, - 2163, 2170, 1649, 64337, 64336, 1609, 65264, 65263, 65166, 65165, 1749, - 1576, 2230, 1878, 2208, 1874, 1875, 1872, 1876, 1877, 1873, 2209, 65168, - 65169, 65167, 65170, 1664, 64347, 64348, 64346, 64349, 1659, 64339, - 64340, 64338, 64341, 1583, 1674, 1675, 1882, 1774, 1679, 2222, 1881, - 1680, 1673, 65194, 65193, 1676, 64389, 64388, 1590, 1787, 65214, 65215, - 65213, 65216, 1677, 64387, 64386, 1672, 64393, 64392, 1646, 1697, 1647, - 1668, 64371, 64372, 64370, 64373, 1678, 64391, 64390, 1740, 1911, 1910, - 1909, 1599, 1598, 1597, 64509, 64510, 64508, 64511, 1601, 1699, 2212, - 1698, 1889, 1701, 1888, 65234, 65235, 65233, 65236, 1711, 1716, 1714, - 1712, 2224, 64403, 64404, 64402, 64405, 1594, 2243, 1788, 65230, 65231, - 65229, 65232, 1715, 64407, 64408, 64406, 64409, 2248, 1581, 1916, 2186, - 1903, 1906, 1902, 1880, 1669, 1666, 1879, 1665, 65186, 65187, 65185, - 65188, 1569, 65152, 1607, 1729, 1730, 64423, 64424, 64422, 64425, 1791, - 1728, 64421, 64420, 1726, 64427, 64428, 64426, 64429, 65258, 65259, - 65257, 65260, 1652, 1653, 1656, 1654, 1580, 2210, 2246, 2245, 65182, - 65183, 65181, 65184, 1688, 64395, 64394, 1603, 1919, 1710, 2228, 1708, - 1707, 65242, 65243, 65241, 65244, 1568, 1705, 1892, 1596, 1891, 2189, - 1595, 2242, 1890, 64399, 64400, 64398, 64401, 1733, 64481, 64480, 1737, - 64483, 64482, 1582, 65190, 65191, 65189, 65192, 1604, 2214, 1718, 2247, - 1717, 1720, 1719, 1898, 65246, 65247, 65245, 65248, 2221, 1605, 2215, - 1894, 1893, 65250, 65251, 65249, 65252, 1564, 1709, 1713, 64411, 64412, - 64410, 64413, 64468, 64469, 64467, 64470, 1667, 64375, 64376, 64374, - 64377, 1606, 1722, 64415, 64414, 1896, 1897, 1725, 1895, 2185, 1721, - 1724, 65254, 65255, 65253, 65256, 1662, 2231, 2238, 64343, 64344, 64342, - 64345, 1702, 64367, 64368, 64366, 64369, 1602, 2213, 2229, 1703, 1704, - 65238, 65239, 65237, 65240, 1585, 1905, 2233, 1682, 1685, 1883, 1899, - 1687, 1775, 1900, 1689, 1684, 1686, 2218, 1683, 65198, 65197, 1681, - 64397, 64396, 1723, 64417, 64418, 64416, 64419, 2220, 1589, 2223, 1694, - 1693, 65210, 65211, 65209, 65212, 1587, 1690, 1918, 1904, 1691, 1692, - 1901, 1884, 1917, 65202, 65203, 65201, 65204, 1588, 1786, 65206, 65207, - 65205, 65208, 1648, 2225, 1706, 1591, 2211, 2188, 1695, 2187, 65218, - 65219, 65217, 65220, 1670, 2241, 1727, 64379, 64380, 64378, 64381, 1671, - 64383, 64384, 64382, 64385, 1578, 1577, 65172, 65171, 1731, 65176, 2232, - 2239, 1661, 1660, 65174, 65175, 65173, 1663, 64355, 64356, 64354, 64357, - 1584, 65196, 65195, 1579, 65178, 65179, 65177, 65180, 2182, 1657, 2240, - 64359, 64360, 64358, 64361, 1658, 64351, 64352, 64350, 64353, 1735, 1655, - 64477, 64472, 64471, 64488, 64489, 1739, 1700, 64363, 64364, 64362, - 64365, 64479, 64478, 1608, 2219, 1743, 1913, 1912, 1572, 65158, 65157, - 1738, 1732, 65262, 65261, 1610, 1746, 1915, 1914, 1747, 64433, 64432, - 64431, 64430, 1574, 65162, 65163, 65161, 65164, 1742, 2216, 2234, 2217, - 1741, 1745, 65266, 65267, 65265, 65268, 1736, 64476, 64475, 1734, 64474, - 64473, 1592, 65222, 65223, 65221, 65224, 1586, 65200, 65199, 2226, 1744, - 64485, 64486, 64484, 64487, 2297, 2295, 64888, 64887, 64886, 64950, - 64699, 64554, 64964, 64885, 64698, 64553, 64787, 64759, 64788, 64760, - 64842, 64839, 64841, 64840, 64845, 65015, 64656, 64605, 64828, 64829, - 65010, 65011, 65023, 64962, 64669, 64518, 64672, 64738, 64926, 64670, - 64519, 64622, 64521, 64668, 64517, 64620, 64671, 64520, 64737, 64621, - 64619, 64618, 64623, 64522, 65021, 64878, 64939, 64693, 64547, 64880, - 64879, 64694, 64548, 64803, 64775, 64692, 64546, 64695, 64549, 64812, - 64784, 64804, 64776, 64893, 64892, 64704, 64559, 64636, 64561, 64702, - 64557, 64961, 64705, 64560, 64703, 64558, 64637, 64562, 64889, 64891, - 64890, 64701, 64556, 64789, 64761, 64700, 64555, 64790, 64762, 64859, - 64858, 64682, 64536, 64795, 64767, 64959, 64681, 64535, 64796, 64768, - 64915, 64916, 64728, 64594, 64595, 64596, 64727, 64593, 64729, 64934, - 64958, 64679, 64533, 64857, 64856, 64935, 64933, 64680, 64534, 64797, - 64769, 64798, 64770, 65019, 64643, 64573, 64640, 64567, 64708, 64568, - 64641, 64711, 64571, 64747, 64710, 64570, 64709, 64569, 64963, 64955, - 64951, 64642, 64712, 64572, 64748, 64644, 64574, 64538, 64799, 64771, - 64683, 64537, 64684, 64539, 64800, 64772, 65272, 65271, 65274, 65273, - 65270, 65269, 64646, 64579, 65276, 65275, 64898, 64949, 64896, 64897, - 64714, 64576, 64717, 64899, 64900, 64954, 64956, 64940, 64713, 64575, - 64902, 64901, 64715, 64577, 64904, 64903, 64941, 64645, 64716, 64578, - 64749, 64647, 64580, 64585, 64648, 64905, 64906, 64907, 64719, 64582, - 64910, 64911, 64953, 64720, 64583, 64909, 64914, 64908, 64960, 64718, - 64581, 64945, 64649, 64721, 64584, 64586, 65012, 64918, 64917, 64947, - 64723, 64588, 64726, 64751, 64952, 64957, 64921, 64920, 64919, 64967, - 64722, 64587, 64923, 64922, 64652, 64725, 64590, 64750, 64654, 64591, - 64653, 64651, 64724, 64589, 64650, 64655, 64592, 64895, 64948, 64894, - 64946, 64707, 64564, 64638, 64565, 64706, 64563, 64639, 64566, 65009, - 64843, 64833, 64835, 64836, 64837, 64834, 64832, 64847, 65014, 64604, - 64869, 64868, 64937, 64689, 64544, 64965, 64870, 64691, 64545, 64801, - 64773, 64690, 64811, 64783, 64802, 64774, 65013, 64975, 65008, 65017, - 65018, 64844, 64838, 64860, 64686, 64541, 64821, 64817, 64744, 64862, - 64861, 64685, 64540, 64820, 64936, 64966, 64687, 64542, 64822, 64864, - 64863, 64865, 64867, 64866, 64688, 64543, 64743, 64791, 64763, 64810, - 64782, 64792, 64764, 64606, 64609, 64755, 64607, 64610, 64756, 64611, - 64608, 64754, 64818, 64746, 64872, 64871, 64938, 64806, 64814, 64778, - 64824, 64873, 64805, 64813, 64777, 64823, 64875, 64874, 64877, 64876, - 64808, 64816, 64780, 64745, 64793, 64765, 64807, 64815, 64779, 64825, - 64809, 64781, 64794, 64766, 65022, 64846, 64882, 64881, 64883, 64884, - 64819, 64551, 64826, 64785, 64757, 64696, 64550, 64786, 64758, 64851, - 64850, 64849, 64674, 64524, 64677, 64740, 64930, 64852, 64929, 64675, - 64525, 64928, 64848, 64927, 64673, 64523, 64932, 64853, 64855, 64854, - 64931, 64626, 64676, 64526, 64739, 64628, 64527, 64627, 64625, 64624, - 64629, 64528, 64603, 64529, 64634, 64531, 64632, 64678, 64530, 64741, - 64633, 64631, 64630, 64635, 64532, 64742, 65016, 64661, 64601, 64616, - 64515, 64491, 64490, 64493, 64492, 64503, 64504, 64502, 64667, 64736, - 64664, 64513, 64663, 64512, 64665, 64614, 64666, 64514, 64735, 64615, - 64499, 64498, 64495, 64494, 64617, 64516, 64501, 64500, 64613, 64612, - 64497, 64496, 64942, 64731, 64598, 64734, 64753, 64660, 64658, 64943, - 64730, 64597, 64732, 64599, 64925, 64924, 64944, 64659, 64733, 64600, - 64752, 64657, 64662, 64602, 64506, 64507, 64505, 64697, 64552, 64827, - 2204, 1619, 1624, 2303, 126492, 126494, 126493, 126495, 126644, 126638, - 126641, 126647, 126648, 126646, 126632, 126645, 126630, 126650, 126626, - 126636, 126631, 126625, 126640, 126643, 126633, 126651, 126637, 126635, - 126649, 126627, 126642, 126639, 126629, 126489, 126467, 126518, 126517, - 126516, 126510, 126513, 126503, 126500, 126519, 126506, 126498, 126508, - 126497, 126512, 126505, 126523, 126509, 126507, 126514, 126511, 126521, - 126612, 126606, 126609, 126615, 126592, 126607, 126599, 126596, 126616, - 126614, 126600, 126613, 126598, 126618, 126594, 126604, 126593, 126608, - 126611, 126601, 126619, 126605, 126603, 126617, 126595, 126610, 126597, - 126475, 126704, 126705, 126588, 126590, 126585, 126582, 126568, 126581, - 126580, 126574, 126577, 126567, 126564, 126583, 126570, 126562, 126572, - 126586, 126561, 126576, 126569, 126587, 126573, 126578, 126575, 126484, - 126478, 126481, 126557, 126559, 126553, 126548, 126542, 126545, 126551, - 126530, 126535, 126537, 126555, 126541, 126539, 126546, 126543, 126472, - 126488, 126486, 126485, 126464, 126479, 126487, 126474, 126470, 126490, - 126466, 126476, 126471, 126465, 126480, 126483, 126473, 126491, 126477, - 126482, 126469, 1541, 1536, 2290, 2289, 2288, 1550, 2192, 1769, 2193, - 1642, 2184, 1544, 1629, 2296, 2301, 2298, 1772, 2183, 1623, 983633, - 983635, 983640, 983634, 983638, 983636, 983639, 983637, 983641, 1563, - 1617, 65148, 65149, 1554, 1555, 1552, 1537, 1539, 1540, 1790, 1789, 1553, - 1551, 1556, 2249, 1560, 1761, 2250, 2272, 1558, 983202, 1751, 1750, 1753, - 1752, 1762, 1764, 1768, 2264, 2273, 1756, 2261, 1755, 1557, 2200, 2260, - 2266, 2268, 2267, 2269, 2252, 2271, 2270, 2291, 1767, 2251, 1760, 1759, - 1559, 2253, 1754, 2263, 2262, 2265, 2202, 2201, 69375, 2203, 69373, - 69374, 2259, 1773, 1763, 1562, 1561, 1766, 1765, 1622, 1618, 65150, - 65151, 2256, 2205, 64444, 64435, 64434, 64441, 64440, 64439, 64438, - 64446, 64445, 64437, 64436, 64449, 64448, 64443, 64442, 64450, 64447, - 1758, 1600, 2179, 2180, 65137, 2181, 65139, 2285, 2282, 2286, 2283, 2287, - 2284, 1566, 2275, 1644, 1627, 1626, 1628, 2190, 1631, 1567, 1625, 1545, - 1546, 1542, 1543, 1637, 1636, 1639, 1638, 1635, 1634, 1632, 1641, 1633, - 1640, 1375, 1370, 1359, 1337, 1349, 1362, 1347, 1353, 1342, 1361, 1360, - 1356, 1333, 1335, 1336, 1346, 1331, 1355, 1345, 1364, 1343, 1363, 1354, - 1351, 1357, 1329, 1358, 1352, 1340, 1366, 1341, 1339, 1330, 1348, 1350, - 1338, 1334, 1344, 1332, 1365, 1373, 1371, 1372, 1395, 1401, 1390, 1409, - 1408, 1404, 1381, 1383, 1384, 1394, 1379, 1403, 1393, 1412, 1391, 1411, - 1402, 1399, 1405, 1376, 1407, 1385, 1377, 1406, 1400, 1397, 1416, 1410, - 1388, 1414, 1389, 1387, 1378, 1396, 1398, 1386, 1382, 1392, 1380, 1413, - 1415, 64279, 64277, 64275, 64276, 64278, 1369, 1418, 1423, 1417, 1374, - 129201, 10549, 10548, 129200, 10550, 10551, 129968, 128667, 127912, 9800, - 8978, 9738, 65948, 42, 8727, 8258, 128562, 11225, 9954, 8771, 8870, - 128095, 9883, 128663, 127975, 128762, 8371, 127814, 68352, 68353, 68357, - 68355, 68358, 68359, 68356, 68354, 68373, 68374, 68372, 68393, 68405, - 68388, 68387, 68386, 68391, 68390, 68389, 68371, 68370, 68369, 68403, - 68401, 68404, 68399, 68394, 68395, 68378, 68381, 68377, 68366, 68367, - 68362, 68363, 68364, 68365, 68385, 68384, 68380, 68379, 68402, 68400, - 68360, 68361, 68375, 68383, 68376, 68368, 68398, 68392, 68382, 68397, - 68396, 68409, 129361, 1547, 129518, 8525, 9810, 129683, 128118, 128036, - 127868, 128124, 128700, 128281, 128386, 11101, 11099, 983056, 10155, - 128043, 129363, 127992, 129441, 129366, 129391, 128708, 7005, 7007, 7006, - 6917, 6918, 6987, 6988, 6928, 6953, 6954, 6936, 6937, 6948, 6944, 6943, - 6949, 6984, 6927, 6933, 6934, 6919, 6920, 6929, 6930, 6921, 6922, 6938, - 6939, 6931, 6981, 6932, 6982, 6958, 6925, 6926, 6950, 6945, 6935, 6940, - 6951, 6952, 6957, 6923, 6924, 6962, 6960, 6961, 6946, 6942, 6941, 6947, - 6983, 6985, 6986, 6963, 6955, 6959, 6956, 7022, 7025, 7021, 7024, 7023, - 7026, 7020, 7019, 7027, 7012, 7013, 7018, 7015, 7017, 7016, 7010, 7014, - 7009, 7011, 7032, 7036, 7033, 7034, 7035, 7031, 7030, 7029, 7028, 7003, - 7038, 7008, 7002, 7037, 6915, 6913, 6912, 6916, 6914, 6964, 6972, 6973, - 6970, 6971, 6968, 6969, 6974, 6975, 6977, 6976, 6965, 6978, 6979, 6966, - 6967, 6980, 7004, 6997, 6996, 6999, 6998, 6995, 6994, 6992, 7001, 6993, - 7000, 127880, 10057, 9744, 128503, 128505, 128499, 128501, 11197, 9745, - 9746, 128502, 128500, 10007, 129526, 129648, 42737, 42736, 42741, 42740, - 42733, 42699, 42713, 42712, 42692, 42706, 42683, 42719, 42734, 42735, - 42682, 42729, 42657, 42725, 42659, 42670, 42718, 42666, 42716, 42701, - 42675, 42671, 42722, 42720, 42727, 42723, 42702, 42726, 42677, 42707, - 42708, 42709, 42686, 42694, 42674, 42731, 42695, 42685, 42684, 42691, - 42673, 42664, 42715, 42703, 92193, 92188, 92161, 92199, 92240, 92179, - 92211, 92219, 92213, 92243, 92207, 92216, 92183, 92184, 92238, 92171, - 92174, 92203, 92186, 92210, 92230, 92175, 92221, 92232, 92246, 92198, - 92214, 92190, 92176, 92197, 92196, 92164, 92245, 92212, 92201, 92160, - 92182, 92173, 92229, 92227, 92180, 92237, 92241, 92223, 92185, 92194, - 92178, 92239, 92225, 92233, 92167, 92191, 92231, 92244, 92204, 92226, - 92236, 92200, 92189, 92187, 92162, 92163, 92169, 92170, 92202, 92205, - 92222, 92168, 92165, 92215, 92224, 92208, 92220, 92235, 92177, 92181, - 92195, 92234, 92209, 92166, 92172, 92206, 92192, 92228, 92217, 92242, - 92218, 92262, 92271, 92272, 92270, 92294, 92253, 92301, 92256, 92273, - 92259, 92251, 92297, 92300, 92296, 92295, 92255, 92252, 92292, 92286, - 92268, 92290, 92281, 92267, 92284, 92287, 92282, 92283, 92277, 92298, - 92276, 92302, 92247, 92299, 92288, 92269, 92260, 92261, 92257, 92274, - 92289, 92263, 92279, 92265, 92266, 92250, 92249, 92285, 92248, 92264, - 92280, 92258, 92254, 92278, 92291, 92293, 92275, 92310, 92320, 92312, - 92394, 92393, 92319, 92384, 92321, 92386, 92361, 92373, 92366, 92328, - 92329, 92357, 92342, 92397, 92365, 92376, 92339, 92382, 92363, 92318, - 92387, 92383, 92315, 92385, 92354, 92311, 92381, 92343, 92326, 92364, - 92324, 92391, 92344, 92390, 92338, 92396, 92308, 92349, 92375, 92378, - 92317, 92331, 92362, 92336, 92341, 92370, 92347, 92307, 92303, 92309, - 92395, 92368, 92356, 92367, 92360, 92389, 92333, 92359, 92374, 92351, - 92325, 92371, 92350, 92345, 92372, 92314, 92340, 92323, 92304, 92313, - 92316, 92398, 92399, 92380, 92330, 92379, 92392, 92335, 92332, 92346, - 92400, 92358, 92334, 92353, 92337, 92306, 92369, 92388, 92322, 92305, - 92327, 92377, 92352, 92348, 92355, 92436, 92517, 92488, 92434, 92415, - 92431, 92454, 92460, 92420, 92489, 92422, 92474, 92479, 92449, 92502, - 92439, 92484, 92495, 92464, 92511, 92406, 92446, 92497, 92426, 92482, - 92448, 92515, 92401, 92478, 92427, 92496, 92458, 92424, 92445, 92404, - 92466, 92444, 92438, 92442, 92441, 92510, 92499, 92425, 92437, 92440, - 92471, 92465, 92409, 92483, 92475, 92467, 92485, 92457, 92432, 92476, - 92435, 92412, 92414, 92430, 92407, 92403, 92405, 92487, 92418, 92486, - 92408, 92447, 92459, 92480, 92472, 92505, 92514, 92450, 92463, 92410, - 92493, 92507, 92503, 92462, 92506, 92443, 92509, 92469, 92461, 92490, - 92512, 92494, 92455, 92417, 92501, 92413, 92508, 92500, 92504, 92473, - 92419, 92498, 92423, 92516, 92428, 92452, 92451, 92481, 92416, 92456, - 92433, 92477, 92492, 92491, 92513, 92411, 92402, 92421, 92453, 92468, - 92470, 92429, 92661, 92626, 92616, 92596, 92612, 92603, 92673, 92651, - 92627, 92597, 92585, 92578, 92615, 92565, 92552, 92602, 92674, 92570, - 92646, 92599, 92532, 92587, 92538, 92620, 92670, 92665, 92575, 92521, - 92633, 92595, 92523, 92556, 92539, 92664, 92653, 92667, 92600, 92542, - 92555, 92668, 92520, 92582, 92591, 92581, 92551, 92654, 92611, 92614, - 92666, 92671, 92617, 92637, 92562, 92518, 92592, 92558, 92528, 92607, - 92535, 92557, 92563, 92550, 92624, 92640, 92657, 92531, 92601, 92553, - 92543, 92619, 92598, 92541, 92544, 92579, 92658, 92554, 92628, 92648, - 92547, 92527, 92545, 92540, 92604, 92622, 92589, 92608, 92583, 92609, - 92662, 92613, 92584, 92625, 92634, 92524, 92536, 92605, 92647, 92548, - 92663, 92593, 92621, 92568, 92610, 92576, 92529, 92618, 92649, 92561, - 92656, 92526, 92655, 92546, 92534, 92560, 92580, 92659, 92660, 92638, - 92571, 92525, 92549, 92559, 92635, 92577, 92530, 92636, 92669, 92572, - 92672, 92537, 92519, 92630, 92586, 92569, 92566, 92573, 92652, 92522, - 92574, 92650, 92533, 92567, 92623, 92606, 92639, 92564, 92590, 92642, - 92643, 92594, 92641, 92645, 92644, 92588, 92629, 92632, 92631, 92710, - 92695, 92694, 92726, 92675, 92719, 92677, 92718, 92682, 92717, 92689, - 92720, 92724, 92685, 92722, 92723, 92711, 92712, 92698, 92688, 92697, - 92696, 92702, 92687, 92704, 92681, 92708, 92703, 92706, 92714, 92709, - 92679, 92721, 92684, 92683, 92707, 92691, 92713, 92700, 92693, 92727, - 92690, 92692, 92686, 92680, 92725, 92699, 92701, 92705, 92716, 92728, - 92678, 92715, 92676, 42693, 42698, 42711, 42696, 42667, 42717, 42704, - 42661, 42721, 42669, 42668, 42705, 42700, 42680, 42678, 42710, 42688, - 42681, 42732, 42676, 42679, 42730, 42728, 42672, 42662, 42724, 42687, - 42689, 42690, 42697, 42714, 42660, 42656, 42665, 42663, 42658, 42738, - 42742, 42739, 42743, 127974, 128183, 128180, 128182, 128181, 127820, - 129685, 128202, 129532, 128136, 129530, 127936, 92916, 92912, 92915, - 92913, 92914, 92887, 92894, 92908, 92880, 92907, 92893, 92886, 92888, - 92881, 92906, 92896, 92891, 92902, 92900, 92885, 92890, 92884, 92904, - 92905, 92899, 92889, 92897, 92892, 92895, 92882, 92898, 92883, 92901, - 92903, 92909, 92917, 9918, 129415, 7152, 7153, 7124, 7108, 7114, 7130, - 7139, 7127, 7138, 7133, 7136, 7113, 7111, 7117, 7119, 7107, 7135, 7125, - 7112, 7123, 7129, 7116, 7132, 7105, 7126, 7128, 7110, 7109, 7137, 7121, - 7118, 7106, 7120, 7134, 7122, 7115, 7131, 7104, 7140, 7141, 7154, 7155, - 7167, 7165, 7166, 7164, 7142, 7150, 7151, 7144, 7147, 7149, 7143, 7145, - 7146, 7148, 128704, 128705, 128267, 127900, 127901, 9835, 9836, 129492, - 128059, 127958, 128147, 129451, 129752, 127866, 129714, 983055, 128276, - 129745, 128277, 9086, 128718, 2519, 2557, 2432, 2548, 2552, 2551, 2550, - 2549, 2553, 2454, 2510, 983661, 2453, 2480, 2545, 2544, 2525, 2524, 2556, - 2443, 2528, 2444, 2529, 2527, 2479, 2437, 2438, 2448, 2452, 2466, 2465, - 2471, 2470, 2464, 2463, 2469, 2468, 2441, 2442, 2439, 2440, 2457, 2467, - 2462, 2472, 2486, 2487, 2488, 2477, 2476, 2459, 2458, 2456, 2455, 2461, - 2460, 2475, 2474, 2489, 2482, 2478, 2447, 2451, 2547, 2546, 983651, - 983650, 983652, 2558, 2493, 2434, 2433, 2492, 2509, 2435, 2554, 2494, - 2504, 2508, 2499, 2500, 2530, 2531, 2497, 2498, 2495, 2496, 2503, 2507, - 2539, 2538, 2541, 2540, 2537, 2536, 2534, 2543, 2535, 2542, 2555, 11102, - 127857, 9004, 9187, 8812, 8502, 8757, 129475, 128719, 72710, 72711, - 72712, 72746, 72704, 72705, 72715, 72717, 72731, 72730, 72736, 72735, - 72729, 72728, 72734, 72733, 72708, 72709, 72706, 72707, 72722, 72732, - 72727, 72737, 72747, 72748, 72749, 72741, 72740, 72724, 72723, 72721, - 72720, 72726, 72725, 72719, 72718, 72739, 72738, 72750, 72745, 72742, - 72744, 72743, 72714, 72716, 72801, 72810, 72806, 72797, 72807, 72798, - 72802, 72811, 72800, 72809, 72799, 72808, 72796, 72805, 72804, 72795, - 72803, 72794, 72768, 72765, 72764, 72767, 72766, 72756, 72757, 72758, - 72751, 72761, 72763, 72754, 72755, 72752, 72753, 72760, 72762, 72770, - 72769, 72789, 72788, 72791, 72790, 72787, 72786, 72784, 72793, 72785, - 72792, 72771, 72772, 72773, 72812, 128692, 128690, 10745, 10744, 129506, - 127921, 127874, 128038, 8383, 129766, 9763, 128089, 129452, 9679, 9864, - 10733, 9865, 9210, 11176, 11177, 11178, 11179, 11180, 11182, 11181, - 11183, 9960, 10028, 9821, 129554, 129596, 129609, 129612, 9818, 129551, - 129593, 9822, 129543, 129564, 129585, 129597, 129606, 129555, 129619, - 129617, 129618, 9823, 129556, 129598, 9819, 129552, 129594, 9820, 129553, - 129595, 129575, 129572, 129576, 129577, 129573, 129574, 9827, 9670, - 11201, 10070, 10730, 11230, 9830, 128419, 128899, 9196, 9662, 9660, - 11167, 127778, 9922, 9923, 10047, 9873, 10022, 128447, 128420, 9829, - 11042, 128426, 11052, 10711, 9194, 9198, 128896, 9668, 9666, 9664, 11164, - 8268, 9944, 128412, 9754, 9699, 9698, 10731, 11044, 11035, 9207, 11206, - 11045, 9204, 11207, 11047, 9205, 11208, 9206, 11205, 128921, 128927, - 9726, 9724, 9912, 128392, 9648, 11039, 127986, 9193, 9197, 9654, 9199, - 128898, 11091, 9658, 9656, 10145, 10148, 11166, 8269, 128413, 9755, - 127990, 9644, 11049, 11050, 11089, 9642, 9787, 9632, 11200, 9209, 128306, - 9728, 128369, 9927, 9984, 128900, 128909, 9986, 9751, 9824, 9733, 128919, - 128925, 128908, 9951, 128383, 9742, 9942, 128418, 128897, 9195, 9652, - 9650, 9700, 9701, 11165, 9851, 11054, 9646, 128920, 128926, 11037, 11204, - 10707, 10002, 10067, 8493, 8460, 8465, 8476, 8488, 10164, 10166, 10165, - 9250, 129792, 129794, 129798, 129806, 129821, 129836, 129813, 129844, - 129829, 129802, 129817, 129848, 129832, 129810, 129840, 129825, 129796, - 129804, 129819, 129850, 129834, 129842, 129827, 129800, 129815, 129846, - 129831, 129808, 129838, 129823, 129793, 129797, 129805, 129820, 129851, - 129835, 129812, 129843, 129828, 129801, 129816, 129847, 129809, 129839, - 129824, 129795, 129803, 129818, 129849, 129833, 129811, 129841, 129826, - 129799, 129814, 129845, 129830, 129807, 129837, 129822, 127804, 128033, - 128216, 128153, 129744, 128939, 128951, 128957, 128945, 128902, 128912, - 128932, 128366, 128278, 128209, 128218, 129667, 12731, 12727, 12726, - 12724, 12725, 12570, 12574, 12718, 12576, 12719, 12578, 12580, 12713, - 12735, 12720, 12584, 12715, 12572, 12579, 12581, 12709, 12708, 12573, - 12575, 12582, 12557, 12728, 12588, 12707, 12732, 12583, 12714, 12723, - 12589, 12716, 12712, 12585, 12555, 12587, 12717, 12591, 12571, 12722, - 12711, 12590, 12734, 12721, 12710, 12577, 12567, 12563, 12705, 12730, - 12558, 12733, 12568, 12564, 12556, 12729, 12569, 12565, 12549, 12704, - 12560, 12706, 12553, 12552, 12559, 12551, 12550, 12561, 12566, 12554, - 12586, 12562, 11211, 11867, 11868, 8993, 9141, 9142, 10555, 9183, 9181, - 8990, 8973, 11812, 8991, 8972, 11813, 9185, 127870, 128144, 127893, - 128335, 129379, 127923, 8904, 10705, 10706, 127993, 9574, 9559, 9556, - 9577, 9565, 9562, 9553, 9580, 9571, 9568, 9552, 9511, 9490, 9503, 9486, - 9537, 9520, 9513, 9489, 9505, 9485, 9543, 9519, 9558, 9555, 9573, 9557, - 9554, 9572, 9592, 9598, 9593, 9599, 9499, 9531, 9495, 9551, 9549, 9595, - 9523, 9491, 9487, 9483, 9481, 9479, 9477, 9475, 9547, 9515, 9507, 9473, - 9594, 9541, 9525, 9517, 9533, 9530, 9522, 9546, 9539, 9582, 9581, 9583, - 9584, 129959, 129964, 129954, 129958, 129965, 129955, 129952, 129956, - 129963, 129960, 129953, 129961, 129957, 129962, 9586, 9585, 9587, 129966, - 9550, 9548, 9591, 9516, 9488, 9484, 9588, 9596, 9482, 9480, 9478, 9476, - 9589, 9597, 9524, 9496, 9492, 9474, 9532, 9508, 9500, 9472, 129967, 9590, - 9542, 9526, 9518, 9534, 9529, 9521, 9545, 9540, 9536, 9510, 9498, 9502, - 9494, 9528, 9544, 9514, 9497, 9506, 9493, 9527, 9564, 9561, 9576, 9563, - 9560, 9575, 9570, 9567, 9579, 9538, 9512, 9504, 9535, 9509, 9501, 9569, - 9566, 9578, 129354, 983263, 128163, 128102, 128713, 128023, 129460, - 69649, 69685, 69749, 69745, 69746, 69687, 69686, 69637, 69638, 69648, - 69650, 69664, 69663, 69669, 69668, 69662, 69661, 69667, 69666, 69643, - 69644, 69645, 69646, 69679, 69641, 69642, 69639, 69640, 69684, 69678, - 69655, 69665, 69660, 69670, 69680, 69681, 69682, 69674, 69673, 69657, - 69656, 69654, 69653, 69659, 69658, 69652, 69651, 69672, 69671, 69683, - 69675, 69677, 69676, 69647, 69721, 69730, 69726, 69717, 69727, 69718, - 69722, 69731, 69720, 69729, 69719, 69728, 69716, 69725, 69724, 69715, - 69723, 69714, 69732, 69733, 69759, 69709, 69707, 69706, 69705, 69708, - 69744, 69632, 69635, 69634, 69636, 69633, 69700, 69747, 69748, 69689, - 69688, 69699, 69701, 69694, 69695, 69696, 69697, 69692, 69693, 69690, - 69691, 69698, 69702, 69704, 69703, 69739, 69738, 69741, 69740, 69737, - 69736, 69734, 69743, 69735, 69742, 10241, 10243, 10247, 10255, 10271, - 10303, 10367, 10495, 10431, 10335, 10463, 10399, 10287, 10351, 10479, - 10415, 10319, 10447, 10383, 10263, 10295, 10359, 10487, 10423, 10327, - 10455, 10391, 10279, 10343, 10471, 10407, 10311, 10439, 10375, 10251, - 10267, 10299, 10363, 10491, 10427, 10331, 10459, 10395, 10283, 10347, - 10475, 10411, 10315, 10443, 10379, 10259, 10291, 10355, 10483, 10419, - 10323, 10451, 10387, 10275, 10339, 10467, 10403, 10307, 10435, 10371, - 10245, 10253, 10269, 10301, 10365, 10493, 10429, 10333, 10461, 10397, - 10285, 10349, 10477, 10413, 10317, 10445, 10381, 10261, 10293, 10357, - 10485, 10421, 10325, 10453, 10389, 10277, 10341, 10469, 10405, 10309, - 10437, 10373, 10249, 10265, 10297, 10361, 10489, 10425, 10329, 10457, - 10393, 10281, 10345, 10473, 10409, 10313, 10441, 10377, 10257, 10289, - 10353, 10481, 10417, 10321, 10449, 10385, 10273, 10337, 10465, 10401, - 10305, 10433, 10369, 10242, 10246, 10254, 10270, 10302, 10366, 10494, - 10430, 10334, 10462, 10398, 10286, 10350, 10478, 10414, 10318, 10446, - 10382, 10262, 10294, 10358, 10486, 10422, 10326, 10454, 10390, 10278, - 10342, 10470, 10406, 10310, 10438, 10374, 10250, 10266, 10298, 10362, - 10490, 10426, 10330, 10458, 10394, 10282, 10346, 10474, 10410, 10314, - 10442, 10378, 10258, 10290, 10354, 10482, 10418, 10322, 10450, 10386, - 10274, 10338, 10466, 10402, 10306, 10434, 10370, 10244, 10252, 10268, - 10300, 10364, 10492, 10428, 10332, 10460, 10396, 10284, 10348, 10476, - 10412, 10316, 10444, 10380, 10260, 10292, 10356, 10484, 10420, 10324, - 10452, 10388, 10276, 10340, 10468, 10404, 10308, 10436, 10372, 10248, - 10264, 10296, 10360, 10488, 10424, 10328, 10456, 10392, 10280, 10344, - 10472, 10408, 10312, 10440, 10376, 10256, 10288, 10352, 10480, 10416, - 10320, 10448, 10384, 10272, 10336, 10464, 10400, 10304, 10432, 10368, - 10240, 129504, 983125, 129329, 127838, 728, 127753, 128112, 128188, - 129650, 129521, 9099, 166, 128148, 129382, 129294, 129529, 129483, - 129767, 128027, 6663, 6662, 6659, 6658, 6671, 6670, 6667, 6666, 6661, - 6668, 6665, 6657, 6678, 6669, 6656, 6674, 6660, 6673, 6676, 6664, 6675, - 6672, 6677, 6683, 6681, 6679, 6682, 6680, 6687, 6686, 5957, 5960, 5962, - 5959, 5956, 5969, 5955, 5966, 5963, 5961, 5965, 5968, 5958, 5967, 5964, - 5952, 5953, 5954, 5970, 5971, 8226, 8729, 128363, 128364, 9678, 128652, - 128101, 128100, 128655, 129480, 129419, 127959, 127791, 129699, 118939, - 118940, 118944, 118943, 118941, 118942, 118938, 118945, 118882, 118876, - 118824, 118835, 118797, 118866, 118957, 118801, 118802, 118865, 118818, - 118916, 118819, 118917, 118899, 118935, 119018, 119022, 119021, 119020, - 119023, 119019, 119017, 118986, 118984, 118985, 118843, 118887, 118808, - 118870, 119003, 119002, 119004, 119005, 118937, 118991, 118995, 118990, - 118992, 118994, 118993, 118930, 118933, 118931, 118932, 119014, 118918, - 118912, 119015, 118785, 118831, 118907, 118966, 118900, 118880, 118798, - 118888, 118884, 118869, 118960, 118959, 118958, 118836, 118969, 118977, - 118978, 118971, 118976, 118975, 118973, 118970, 118987, 118979, 118980, - 118972, 118983, 983272, 118982, 118974, 118988, 118981, 119000, 119001, - 118929, 118928, 118806, 119029, 118927, 118898, 118964, 118965, 118853, - 118968, 118837, 118967, 118936, 118956, 118810, 118854, 118847, 118839, - 118906, 118791, 118800, 119024, 119026, 118862, 118812, 119025, 119027, - 118863, 118811, 118820, 119028, 118911, 118842, 118850, 118921, 118858, - 118913, 118914, 118915, 118834, 118860, 118868, 118796, 118881, 118922, - 118926, 118925, 118924, 118923, 118830, 118877, 118949, 118947, 118948, - 118963, 118955, 118962, 118946, 118961, 118953, 118952, 118951, 118950, - 118954, 118871, 118805, 118855, 118828, 118901, 118787, 118788, 118856, - 118816, 118875, 118845, 118879, 118793, 118846, 118878, 118814, 118822, - 118873, 118859, 118857, 118849, 118840, 118861, 118786, 118895, 118841, - 118874, 118892, 118897, 118807, 118784, 118821, 118844, 118825, 118889, - 119010, 119012, 119013, 119011, 119006, 119008, 119009, 119007, 118910, - 118815, 118852, 119016, 118826, 118885, 118827, 118803, 118886, 118792, - 118813, 118920, 118799, 118833, 118829, 118904, 118902, 118903, 118905, - 118804, 118934, 118919, 118832, 118893, 118838, 118851, 118883, 118894, - 118891, 118896, 118823, 118789, 118790, 118872, 118817, 118809, 118908, - 118909, 118996, 118998, 118997, 118999, 118989, 118794, 118795, 118867, - 118864, 118848, 118890, 983262, 983126, 983057, 9764, 8454, 129305, - 128197, 128247, 128248, 127957, 983098, 5130, 5131, 5122, 6322, 5148, - 5551, 5310, 5382, 5166, 6321, 5759, 5559, 5556, 5557, 5558, 5567, 5564, - 5565, 5566, 5563, 5560, 5561, 5562, 5555, 5552, 5553, 5554, 5384, 5439, - 6387, 6388, 5281, 5264, 6382, 6389, 5203, 5674, 5675, 5677, 5676, 5673, - 5672, 5706, 5707, 5709, 5708, 5705, 5704, 5204, 5574, 5575, 5577, 5576, - 5573, 5572, 6384, 6381, 5620, 6383, 5617, 5618, 5619, 5616, 5615, 5195, - 5592, 5593, 5595, 5594, 5591, 5590, 5174, 5175, 5129, 5703, 5662, 5663, - 5665, 5664, 5661, 5660, 5655, 5656, 6386, 5659, 5657, 5654, 5652, 5633, - 5629, 5630, 5632, 5631, 5628, 5627, 5623, 5624, 5626, 5625, 5622, 5621, - 5636, 5637, 5639, 5329, 5638, 5635, 5634, 5722, 5718, 5719, 5721, 5720, - 5717, 5716, 5712, 5713, 5715, 5714, 5711, 5710, 5614, 5610, 5611, 5613, - 5612, 5609, 5608, 5702, 5698, 5699, 5701, 5700, 5697, 5696, 5686, 5687, - 5689, 5688, 5685, 5684, 5692, 5693, 5695, 5694, 5691, 5690, 5737, 5738, - 5740, 5739, 5736, 5735, 5604, 5605, 5607, 5606, 5603, 5602, 5598, 5599, - 5601, 5600, 5597, 5596, 5725, 5726, 5728, 5727, 5724, 5723, 5680, 5681, - 5683, 5682, 5679, 5678, 5668, 5669, 5671, 5670, 5667, 5666, 5731, 5732, - 5734, 5733, 5730, 5729, 5642, 5643, 5645, 5644, 5641, 5640, 5580, 5581, - 5583, 5582, 5579, 5578, 5586, 5587, 5589, 5588, 5585, 5584, 5648, 5649, - 5651, 5650, 5647, 5646, 5128, 5265, 5258, 5276, 5278, 5272, 5274, 5268, - 5270, 5266, 5741, 5261, 5262, 5259, 5260, 5257, 5121, 6364, 5163, 5469, - 5465, 5466, 5460, 5461, 5158, 5157, 5162, 5155, 5156, 6367, 6366, 5160, - 5152, 5151, 5153, 5154, 5161, 5159, 5462, 5742, 5463, 5464, 5467, 5459, - 5120, 5501, 5123, 5124, 5164, 5251, 5252, 5246, 5248, 6329, 5242, 5244, - 5238, 5240, 5236, 5234, 5235, 5228, 6328, 5231, 5232, 5229, 5230, 5227, - 5354, 5542, 5540, 5541, 5538, 5539, 5536, 5537, 5338, 5339, 5332, 6333, - 5350, 5352, 5346, 5348, 5342, 5344, 5340, 5335, 5336, 5333, 5334, 5331, - 5307, 5283, 5356, 5458, 5287, 5288, 5385, 5290, 5291, 5284, 6330, 5302, - 5304, 5298, 5300, 5294, 5296, 5292, 5285, 5286, 5309, 5328, 5473, 5475, - 5471, 5319, 5386, 5390, 5391, 5388, 5389, 5380, 5387, 5142, 5147, 5280, - 5250, 5306, 5327, 5221, 5437, 72372, 72373, 72370, 72371, 72368, 72369, - 72378, 72379, 72376, 72377, 72374, 72375, 5320, 5313, 6332, 5525, 5523, - 5524, 5518, 5744, 5521, 5522, 5519, 5520, 5526, 5749, 5750, 5747, 5748, - 5745, 5746, 5499, 5497, 5498, 5495, 5496, 5493, 5494, 5492, 5500, 5316, - 5317, 6331, 5323, 5325, 6346, 6348, 6342, 6344, 5321, 5314, 5315, 5312, - 5330, 5509, 5507, 5508, 5502, 5743, 5505, 5506, 5503, 5504, 5125, 6361, - 6347, 6349, 6343, 6345, 6362, 6363, 6359, 6358, 6360, 6356, 6357, 5165, - 5126, 6320, 5193, 5184, 5186, 6326, 5188, 5190, 5180, 5182, 5178, 5176, - 5177, 5168, 6325, 5171, 5172, 6324, 5169, 5170, 5167, 5456, 6368, 5443, - 6355, 5454, 6353, 6354, 6351, 6352, 6350, 5451, 5452, 5445, 6341, 5448, - 5449, 5446, 5447, 5442, 5381, 5364, 6335, 5570, 6380, 5571, 5568, 5569, - 5653, 6385, 5658, 5529, 6379, 6378, 5530, 5527, 5528, 5441, 5282, 5311, - 5365, 5358, 5413, 5405, 5407, 6338, 5409, 5411, 5401, 5403, 5399, 5395, - 5396, 6336, 5397, 5398, 6337, 5393, 5394, 5392, 5361, 5256, 5253, 5254, - 5255, 5362, 6334, 5383, 5376, 5378, 5372, 5374, 5368, 5370, 5366, 72383, - 72380, 72381, 72382, 5359, 5360, 5357, 5222, 5482, 5550, 5548, 5549, - 5546, 5547, 5544, 5545, 5543, 6372, 5480, 6371, 5478, 5479, 5476, 5477, - 5472, 5474, 5470, 5512, 6377, 6376, 5513, 5510, 5511, 5487, 5486, 6375, - 5485, 6374, 6373, 5483, 5484, 5226, 5223, 5224, 5225, 5205, 5206, 5197, - 6327, 5217, 5219, 5213, 5215, 5209, 5211, 5207, 5491, 5488, 5489, 5490, - 5200, 5201, 5198, 5199, 5196, 5132, 5355, 5351, 5353, 5347, 5349, 5343, - 5345, 5341, 5453, 6370, 5450, 6369, 5444, 5308, 5303, 5305, 5299, 5301, - 5295, 5297, 5293, 5194, 5189, 5191, 5185, 5187, 5181, 5183, 5179, 5440, - 5434, 5436, 5430, 5432, 5426, 5428, 5424, 5324, 5326, 5322, 5457, 5455, - 5517, 5514, 5515, 5516, 5410, 5412, 5406, 5408, 5402, 5404, 5400, 5377, - 5379, 5373, 5375, 5369, 5371, 5367, 5277, 5279, 5273, 5275, 5269, 5271, - 5267, 5247, 5249, 5243, 5245, 5239, 5241, 5237, 5481, 5218, 5220, 5214, - 5216, 5210, 5212, 5208, 5468, 5144, 5146, 5139, 5141, 5135, 5137, 5133, - 6365, 5138, 5140, 5535, 5756, 5757, 5754, 5755, 5752, 5753, 5751, 5534, - 5531, 5532, 5533, 5758, 5143, 5145, 6323, 5134, 5136, 5438, 5192, 5173, - 5263, 5233, 5337, 5289, 5318, 5363, 5202, 5420, 5127, 5149, 5421, 5422, - 5415, 6340, 5418, 5419, 6339, 5433, 5435, 5429, 5431, 5425, 5427, 5423, - 5416, 5417, 5414, 5150, 983097, 983171, 917631, 128473, 9803, 128367, - 127852, 129387, 128758, 11839, 9809, 128199, 128450, 128451, 8248, 8257, - 8453, 66225, 66246, 66211, 66214, 66254, 66218, 66250, 66251, 66252, - 66253, 66229, 66238, 66244, 66227, 66224, 66222, 66223, 66242, 66243, - 66232, 66221, 66247, 66230, 66226, 66239, 66212, 66248, 66256, 66235, - 66208, 66215, 66210, 66220, 66234, 66255, 66240, 66241, 66236, 66237, - 66231, 66209, 66213, 66249, 66233, 66245, 66217, 66219, 66216, 66228, - 127904, 711, 127887, 129690, 983073, 129365, 9936, 128008, 128049, + 983054, 128126, 117837, 117836, 117844, 117845, 117842, 117843, 117839, + 117838, 117841, 117840, 8780, 9006, 983203, 8776, 10863, 8778, 9095, + 9941, 9200, 38, 127994, 127944, 128657, 10815, 82970, 82971, 82964, + 82965, 82966, 82967, 82968, 82969, 82972, 82973, 82974, 82994, 82995, + 82996, 82987, 82988, 82992, 82993, 82986, 82989, 82990, 82991, 82997, + 82998, 82999, 83016, 83017, 83018, 83019, 83010, 83011, 83012, 83013, + 83014, 83015, 83020, 83021, 83022, 83050, 83051, 83052, 83053, 83043, + 83044, 83045, 83046, 83047, 83048, 83049, 83054, 82984, 82985, 82975, + 82976, 82977, 82978, 82979, 82980, 82981, 82982, 82983, 82953, 82954, + 82955, 82956, 82957, 82958, 82959, 82960, 82961, 82962, 82963, 82944, + 82945, 82946, 82947, 82948, 82949, 82950, 82951, 82952, 83000, 83001, + 83002, 83003, 83004, 83005, 83006, 83007, 83008, 83009, 83023, 83024, + 83025, 83026, 83027, 83028, 83029, 83030, 83031, 83032, 83033, 83034, + 83035, 83036, 83037, 83038, 83039, 83040, 83041, 83042, 83062, 83063, + 83064, 83065, 83070, 83071, 83072, 83073, 83066, 83067, 83068, 83055, + 83056, 83057, 83058, 83059, 83060, 83061, 83069, 83074, 83075, 83076, + 83077, 83078, 83083, 83084, 83079, 83080, 83081, 83082, 83085, 83086, + 83087, 83088, 83094, 83095, 83089, 83090, 83091, 83092, 83093, 83096, + 83097, 83098, 83099, 83105, 83106, 83100, 83101, 83102, 83103, 83104, + 83107, 83108, 83109, 83110, 83111, 83112, 83113, 83114, 83115, 83116, + 83117, 83118, 83119, 83120, 83121, 83122, 83123, 83124, 83125, 83126, + 83127, 83128, 83129, 83130, 83131, 83132, 83133, 83134, 83135, 83136, + 83137, 83138, 83139, 83140, 83141, 83142, 83143, 83144, 83145, 83146, + 83147, 83148, 83149, 83150, 83151, 83152, 83153, 83154, 83155, 83156, + 83157, 83158, 83159, 83160, 83161, 83162, 83163, 83164, 83165, 83166, + 83167, 83168, 83169, 83170, 83173, 83174, 83175, 83180, 83181, 83183, + 83184, 83171, 83172, 83176, 83177, 83178, 83179, 83182, 83190, 83191, + 83192, 83193, 83185, 83186, 83187, 83188, 83189, 83194, 83195, 83196, + 83274, 83275, 83280, 83281, 83270, 83271, 83272, 83273, 83276, 83277, + 83278, 83279, 83268, 83269, 83259, 83260, 83261, 83262, 83263, 83264, + 83265, 83266, 83267, 83204, 83205, 83197, 83198, 83199, 83200, 83201, + 83202, 83203, 83206, 83207, 83245, 83246, 83238, 83239, 83240, 83241, + 83242, 83243, 83244, 83247, 83248, 83208, 83209, 83210, 83211, 83212, + 83213, 83214, 83215, 83216, 83217, 83218, 83219, 83220, 83221, 83222, + 83223, 83224, 83225, 83226, 83227, 83228, 83229, 83230, 83231, 83232, + 83233, 83234, 83235, 83236, 83237, 83249, 83250, 83251, 83252, 83253, + 83254, 83255, 83256, 83257, 83258, 83291, 83292, 83282, 83283, 83284, + 83285, 83286, 83287, 83288, 83289, 83290, 83312, 83313, 83303, 83304, + 83305, 83306, 83307, 83308, 83309, 83310, 83311, 83348, 83349, 83339, + 83340, 83341, 83342, 83343, 83344, 83345, 83346, 83347, 83322, 83323, + 83324, 83325, 83316, 83317, 83318, 83314, 83315, 83319, 83320, 83321, + 83326, 83327, 83328, 83354, 83355, 83359, 83360, 83350, 83351, 83352, + 83353, 83356, 83357, 83358, 83361, 83377, 83378, 83374, 83375, 83381, + 83382, 83373, 83376, 83379, 83380, 83383, 83384, 83385, 83389, 83386, + 83387, 83388, 83390, 83391, 83392, 83393, 83394, 83395, 83363, 83364, + 83362, 83365, 83366, 83367, 83368, 83369, 83370, 83371, 83372, 83293, + 83294, 83295, 83296, 83297, 83298, 83299, 83300, 83301, 83302, 83329, + 83330, 83331, 83332, 83333, 83334, 83335, 83336, 83337, 83338, 83406, + 83407, 83408, 83409, 83410, 83411, 83412, 83413, 83414, 83415, 83416, + 83447, 83448, 83455, 83456, 83449, 83450, 83451, 83452, 83453, 83454, + 83457, 83458, 83489, 83490, 83491, 83492, 83493, 83494, 83495, 83496, + 83396, 83397, 83398, 83399, 83400, 83401, 83402, 83403, 83404, 83405, + 83417, 83418, 83419, 83420, 83421, 83422, 83423, 83424, 83425, 83426, + 83427, 83428, 83429, 83430, 83431, 83432, 83433, 83434, 83435, 83436, + 83437, 83438, 83439, 83440, 83441, 83442, 83443, 83444, 83445, 83446, + 83459, 83460, 83461, 83462, 83463, 83464, 83465, 83466, 83467, 83468, + 83469, 83470, 83471, 83472, 83473, 83474, 83475, 83476, 83477, 83478, + 83479, 83480, 83481, 83482, 83483, 83484, 83485, 83486, 83487, 83488, + 83526, 83497, 83498, 83499, 83500, 83501, 83502, 83503, 83504, 83505, + 83506, 83507, 83508, 83509, 83510, 83511, 83512, 83513, 83514, 83515, + 83516, 83517, 83518, 83519, 83520, 83521, 83522, 83523, 83524, 83525, + 129728, 8736, 10654, 10660, 128551, 8491, 128162, 128544, 128028, 117768, + 128246, 11150, 11148, 11149, 11151, 11119, 8630, 10560, 8755, 128260, + 10226, 8634, 10769, 9875, 10193, 9765, 9021, 9055, 9061, 9033, 9052, + 9022, 9066, 9058, 9067, 9042, 9049, 9035, 9034, 9038, 9062, 9073, 9046, + 9050, 9075, 9080, 9014, 9082, 9078, 9077, 9081, 9060, 9051, 9063, 9029, + 9109, 9020, 9056, 9017, 9018, 9036, 9047, 9044, 9037, 9043, 9040, 9027, + 9031, 9072, 9071, 9016, 9026, 9025, 9028, 9032, 9019, 9054, 9048, 9030, + 9076, 9070, 9023, 9059, 9069, 9015, 9079, 9024, 9074, 9057, 9041, 9045, + 9053, 9039, 9065, 9064, 9068, 39, 11236, 8773, 8786, 10864, 8774, 8784, + 983195, 983196, 69372, 1548, 2277, 2280, 2276, 2279, 2278, 2281, 1615, + 65144, 65145, 2302, 1612, 65138, 1549, 2206, 2299, 2300, 1643, 2274, + 1771, 1770, 1565, 1757, 1614, 1630, 2293, 2292, 65142, 65143, 1611, + 65136, 1538, 1645, 1748, 2207, 1621, 1620, 1616, 2294, 65146, 65147, + 1613, 65140, 2258, 2255, 2254, 2257, 2236, 2244, 2235, 2237, 1593, 1886, + 2227, 1696, 1887, 1885, 65226, 65227, 65225, 65228, 1575, 2165, 2176, + 2173, 2161, 2174, 2171, 2167, 2177, 2169, 2166, 2168, 2178, 2164, 2160, + 2162, 2175, 2172, 1571, 65156, 65155, 1573, 65160, 65159, 1651, 1650, + 1908, 1907, 1570, 65154, 65153, 2163, 2170, 1649, 64337, 64336, 1609, + 65264, 65263, 65166, 65165, 1749, 1576, 2230, 1878, 2208, 1874, 1875, + 1872, 1876, 1877, 1873, 2209, 65168, 65169, 65167, 65170, 1664, 64347, + 64348, 64346, 64349, 1659, 64339, 64340, 64338, 64341, 1583, 1882, 1774, + 1679, 2222, 69314, 1881, 1674, 1675, 1680, 1673, 65194, 65193, 1676, + 64389, 64388, 1590, 1787, 65214, 65215, 65213, 65216, 1677, 64387, 64386, + 1672, 64393, 64392, 1646, 1697, 1647, 1668, 64371, 64372, 64370, 64373, + 1678, 64391, 64390, 1740, 1911, 1910, 1909, 1599, 1598, 1597, 64509, + 64510, 64508, 64511, 1601, 1699, 2212, 1698, 1889, 1701, 1888, 65234, + 65235, 65233, 65236, 1711, 1716, 1714, 1712, 2224, 64403, 64404, 64402, + 64405, 1594, 2243, 1788, 65230, 65231, 65229, 65232, 1715, 64407, 64408, + 64406, 64409, 2248, 1581, 1916, 2186, 1903, 1906, 1902, 1880, 1669, 1666, + 1879, 1665, 65186, 65187, 65185, 65188, 1569, 65152, 1607, 1729, 1730, + 64423, 64424, 64422, 64425, 1791, 1728, 64421, 64420, 1726, 64427, 64428, + 64426, 64429, 65258, 65259, 65257, 65260, 1652, 1653, 1656, 1654, 1580, + 2210, 2246, 2245, 65182, 65183, 65181, 65184, 1688, 64395, 64394, 1603, + 69316, 1919, 1710, 2228, 1708, 1707, 65242, 65243, 65241, 65244, 1568, + 1705, 1892, 1596, 1891, 2189, 1595, 2242, 1890, 64399, 64400, 64398, + 64401, 1733, 64481, 64480, 1737, 64483, 64482, 1582, 65190, 65191, 65189, + 65192, 1604, 2214, 1718, 2247, 1717, 1720, 1719, 1898, 65246, 65247, + 65245, 65248, 2221, 1605, 2215, 1894, 1893, 65250, 65251, 65249, 65252, + 1564, 1709, 1713, 64411, 64412, 64410, 64413, 64468, 64469, 64467, 64470, + 1667, 64375, 64376, 64374, 64377, 1606, 1722, 64415, 64414, 1896, 1897, + 1725, 1895, 2185, 1721, 1724, 65254, 65255, 65253, 65256, 1662, 2231, + 2238, 64343, 64344, 64342, 64345, 1702, 64367, 64368, 64366, 64369, 1602, + 2213, 2229, 1703, 1704, 65238, 65239, 65237, 65240, 1585, 1905, 2233, + 1682, 1685, 1883, 1899, 1687, 1775, 1900, 1689, 1684, 1686, 2218, 1683, + 65198, 65197, 1681, 64397, 64396, 1723, 64417, 64418, 64416, 64419, 2220, + 1589, 2223, 1694, 1693, 65210, 65211, 65209, 65212, 1587, 1690, 1918, + 1904, 1691, 1692, 1901, 1884, 1917, 65202, 65203, 65201, 65204, 1588, + 1786, 65206, 65207, 65205, 65208, 1648, 2225, 1706, 1591, 69315, 2211, + 2188, 1695, 2187, 65218, 65219, 65217, 65220, 1670, 2241, 1727, 64379, + 64380, 64378, 64381, 1671, 64383, 64384, 64382, 64385, 1578, 1577, 65172, + 65171, 1731, 65176, 2232, 2239, 1661, 1660, 65174, 65175, 65173, 1663, + 64355, 64356, 64354, 64357, 1584, 65196, 65195, 1579, 65178, 65179, + 65177, 65180, 2182, 1657, 2240, 64359, 64360, 64358, 64361, 1658, 64351, + 64352, 64350, 64353, 1735, 1655, 64477, 64472, 64471, 64488, 64489, 1739, + 1700, 64363, 64364, 64362, 64365, 64479, 64478, 1608, 2219, 1743, 1913, + 1912, 1572, 65158, 65157, 1738, 1732, 65262, 65261, 1610, 1746, 1915, + 1914, 1747, 64433, 64432, 64431, 64430, 1574, 65162, 65163, 65161, 65164, + 1742, 2216, 2234, 2217, 1741, 1745, 65266, 65267, 65265, 65268, 1736, + 64476, 64475, 1734, 64474, 64473, 1592, 65222, 65223, 65221, 65224, 1586, + 65200, 65199, 2226, 1744, 64485, 64486, 64484, 64487, 2297, 2295, 64888, + 64887, 64886, 64950, 64699, 64554, 64964, 64885, 64698, 64553, 64787, + 64759, 64788, 64760, 64842, 64839, 64841, 64840, 64845, 65015, 64656, + 64605, 64828, 64829, 65010, 65011, 65023, 64962, 64669, 64518, 64672, + 64738, 64926, 64670, 64519, 64622, 64521, 64668, 64517, 64620, 64671, + 64520, 64737, 64621, 64619, 64618, 64623, 64522, 65021, 64878, 64939, + 64693, 64547, 64880, 64879, 64694, 64548, 64803, 64775, 64692, 64546, + 64695, 64549, 64812, 64784, 64804, 64776, 64893, 64892, 64704, 64559, + 64636, 64561, 64702, 64557, 64961, 64705, 64560, 64703, 64558, 64637, + 64562, 64889, 64891, 64890, 64701, 64556, 64789, 64761, 64700, 64555, + 64790, 64762, 64859, 64858, 64682, 64536, 64795, 64767, 64959, 64681, + 64535, 64796, 64768, 64915, 64916, 64728, 64594, 64595, 64596, 64727, + 64593, 64729, 64934, 64958, 64679, 64533, 64857, 64856, 64935, 64933, + 64680, 64534, 64797, 64769, 64798, 64770, 65019, 64643, 64573, 64640, + 64567, 64708, 64568, 64641, 64711, 64571, 64747, 64710, 64570, 64709, + 64569, 64963, 64955, 64951, 64642, 64712, 64572, 64748, 64644, 64574, + 64538, 64799, 64771, 64683, 64537, 64684, 64539, 64800, 64772, 65272, + 65271, 65274, 65273, 65270, 65269, 64646, 64579, 65276, 65275, 64898, + 64949, 64896, 64897, 64714, 64576, 64717, 64899, 64900, 64954, 64956, + 64940, 64713, 64575, 64902, 64901, 64715, 64577, 64904, 64903, 64941, + 64645, 64716, 64578, 64749, 64647, 64580, 64585, 64648, 64905, 64906, + 64907, 64719, 64582, 64910, 64911, 64953, 64720, 64583, 64909, 64914, + 64908, 64960, 64718, 64581, 64945, 64649, 64721, 64584, 64586, 65012, + 64918, 64917, 64947, 64723, 64588, 64726, 64751, 64952, 64957, 64921, + 64920, 64919, 64967, 64722, 64587, 64923, 64922, 64652, 64725, 64590, + 64750, 64654, 64591, 64653, 64651, 64724, 64589, 64650, 64655, 64592, + 64895, 64948, 64894, 64946, 64707, 64564, 64638, 64565, 64706, 64563, + 64639, 64566, 65009, 64843, 64833, 64835, 64836, 64837, 64834, 64832, + 64847, 65014, 64604, 64869, 64868, 64937, 64689, 64544, 64965, 64870, + 64691, 64545, 64801, 64773, 64690, 64811, 64783, 64802, 64774, 65013, + 64975, 65008, 65017, 65018, 64844, 64838, 64860, 64686, 64541, 64821, + 64817, 64744, 64862, 64861, 64685, 64540, 64820, 64936, 64966, 64687, + 64542, 64822, 64864, 64863, 64865, 64867, 64866, 64688, 64543, 64743, + 64791, 64763, 64810, 64782, 64792, 64764, 64606, 64609, 64755, 64607, + 64610, 64756, 64611, 64608, 64754, 64818, 64746, 64872, 64871, 64938, + 64806, 64814, 64778, 64824, 64873, 64805, 64813, 64777, 64823, 64875, + 64874, 64877, 64876, 64808, 64816, 64780, 64745, 64793, 64765, 64807, + 64815, 64779, 64825, 64809, 64781, 64794, 64766, 65022, 64846, 64882, + 64881, 64883, 64884, 64819, 64551, 64826, 64785, 64757, 64696, 64550, + 64786, 64758, 64851, 64850, 64849, 64674, 64524, 64677, 64740, 64930, + 64852, 64929, 64675, 64525, 64928, 64848, 64927, 64673, 64523, 64932, + 64853, 64855, 64854, 64931, 64626, 64676, 64526, 64739, 64628, 64527, + 64627, 64625, 64624, 64629, 64528, 64603, 64529, 64634, 64531, 64632, + 64678, 64530, 64741, 64633, 64631, 64630, 64635, 64532, 64742, 65016, + 64661, 64601, 64616, 64515, 64491, 64490, 64493, 64492, 64503, 64504, + 64502, 64667, 64736, 64664, 64513, 64663, 64512, 64665, 64614, 64666, + 64514, 64735, 64615, 64499, 64498, 64495, 64494, 64617, 64516, 64501, + 64500, 64613, 64612, 64497, 64496, 64942, 64731, 64598, 64734, 64753, + 64660, 64658, 64943, 64730, 64597, 64732, 64599, 64925, 64924, 64944, + 64659, 64733, 64600, 64752, 64657, 64662, 64602, 64506, 64507, 64505, + 64697, 64552, 64827, 2204, 1619, 1624, 2303, 126492, 126494, 126493, + 126495, 126644, 126638, 126641, 126647, 126648, 126646, 126632, 126645, + 126630, 126650, 126626, 126636, 126625, 126640, 126643, 126633, 126631, + 126651, 126637, 126635, 126649, 126627, 126642, 126639, 126629, 126489, + 126467, 126518, 126517, 126516, 126510, 126513, 126503, 126500, 126519, + 126506, 126498, 126508, 126497, 126512, 126505, 126523, 126509, 126507, + 126514, 126511, 126521, 126612, 126606, 126609, 126615, 126592, 126607, + 126599, 126596, 126616, 126614, 126600, 126613, 126598, 126618, 126594, + 126604, 126593, 126608, 126611, 126601, 126619, 126605, 126603, 126617, + 126595, 126610, 126597, 126475, 126704, 126705, 126588, 126590, 126585, + 126582, 126568, 126581, 126580, 126574, 126577, 126567, 126564, 126583, + 126570, 126562, 126572, 126561, 126576, 126569, 126586, 126587, 126573, + 126578, 126575, 126484, 126478, 126481, 126557, 126559, 126553, 126548, + 126542, 126545, 126551, 126530, 126537, 126535, 126555, 126541, 126539, + 126546, 126543, 126472, 126488, 126486, 126485, 126464, 126479, 126487, + 126474, 126470, 126490, 126466, 126476, 126465, 126480, 126483, 126473, + 126471, 126491, 126477, 126482, 126469, 1541, 1536, 2290, 2289, 2288, + 1642, 2199, 1550, 2192, 1769, 2193, 2184, 1544, 1629, 2296, 2301, 2298, + 1772, 2183, 1623, 983633, 983635, 983640, 983634, 983638, 983636, 983639, + 983637, 983641, 1563, 1617, 65148, 65149, 1554, 1555, 1552, 1537, 1539, + 1540, 1790, 1789, 1553, 1551, 1556, 2249, 1560, 1761, 2250, 2272, 1558, + 983202, 1751, 1750, 1753, 1752, 1762, 1764, 1768, 2264, 2273, 1756, 2261, + 1755, 1557, 2200, 2260, 2266, 2268, 2267, 2269, 2252, 2271, 2270, 2291, + 1767, 2251, 1760, 1759, 1559, 2253, 1754, 2263, 2262, 2265, 2202, 2201, + 69375, 2203, 69373, 69374, 2259, 1773, 1763, 1562, 1561, 1766, 1765, + 1622, 1618, 65150, 65151, 2256, 2205, 64444, 64435, 64434, 64441, 64440, + 64439, 64438, 64446, 64445, 64437, 64436, 64449, 64448, 64443, 64442, + 64450, 64447, 1758, 1600, 2179, 2180, 65137, 2181, 65139, 2285, 2282, + 2286, 2283, 2287, 2284, 1566, 2275, 1644, 1627, 1626, 1628, 2190, 1631, + 1567, 1625, 1545, 1546, 1542, 1543, 1637, 1636, 1639, 1638, 1635, 1634, + 1632, 1641, 1633, 1640, 1375, 1370, 1359, 1337, 1349, 1362, 1347, 1353, + 1342, 1361, 1360, 1356, 1333, 1335, 1336, 1346, 1331, 1355, 1345, 1364, + 1343, 1363, 1354, 1351, 1357, 1329, 1358, 1352, 1340, 1366, 1341, 1339, + 1330, 1348, 1350, 1338, 1334, 1344, 1332, 1365, 1373, 1372, 1371, 1395, + 1401, 1390, 1409, 1408, 1404, 1381, 1383, 1384, 1394, 1379, 1403, 1393, + 1412, 1391, 1411, 1402, 1399, 1405, 1376, 1407, 1385, 1377, 1406, 1400, + 1397, 1416, 1410, 1388, 1414, 1389, 1387, 1378, 1396, 1398, 1386, 1382, + 1392, 1380, 1413, 1415, 64279, 64277, 64275, 64276, 64278, 1369, 1418, + 1423, 1417, 1374, 129201, 10549, 10548, 129200, 10550, 10551, 129968, + 128667, 127912, 9800, 8978, 9738, 65948, 42, 8727, 8258, 128562, 11225, + 9954, 8771, 8870, 128095, 9883, 128663, 127975, 128762, 8371, 127814, + 68352, 68353, 68357, 68355, 68358, 68359, 68356, 68354, 68373, 68374, + 68372, 68393, 68405, 68388, 68387, 68386, 68391, 68390, 68389, 68371, + 68370, 68369, 68403, 68401, 68404, 68399, 68394, 68395, 68378, 68381, + 68377, 68366, 68367, 68362, 68363, 68364, 68365, 68385, 68384, 68380, + 68379, 68402, 68400, 68360, 68361, 68375, 68383, 68376, 68368, 68398, + 68392, 68382, 68397, 68396, 68409, 129361, 1547, 129518, 8525, 9810, + 129683, 128118, 128036, 127868, 128124, 128700, 128281, 128386, 11101, + 11099, 983056, 10155, 128043, 129363, 127992, 129441, 129366, 129391, + 128708, 7007, 7005, 7006, 6991, 6990, 6917, 6918, 6987, 6988, 6928, 6953, + 6954, 6936, 6937, 6948, 6944, 6943, 6949, 6984, 6927, 6933, 6934, 6919, + 6920, 6929, 6930, 6921, 6922, 6938, 6939, 6931, 6981, 6932, 6982, 6958, + 6925, 6926, 6950, 6945, 6935, 6940, 6951, 6952, 6957, 6923, 6924, 6962, + 6960, 6961, 6946, 6942, 6941, 6947, 6983, 6985, 6986, 6963, 6955, 6959, + 6956, 7022, 7025, 7021, 7024, 7023, 7026, 7020, 7019, 7027, 7012, 7013, + 7018, 7015, 7017, 7016, 7010, 7014, 7009, 7011, 7032, 7036, 7033, 7034, + 7035, 7031, 7030, 7029, 7028, 7003, 7038, 7008, 7002, 7037, 7039, 6915, + 6913, 6912, 6914, 6916, 6964, 6972, 6973, 6970, 6971, 6968, 6969, 6974, + 6975, 6977, 6976, 6965, 6978, 6979, 6966, 6967, 6980, 7004, 6997, 6996, + 6999, 6998, 6995, 6994, 6992, 7001, 6993, 7000, 127880, 10057, 9744, + 128503, 128505, 128499, 128501, 11197, 9745, 9746, 128502, 128500, 10007, + 129526, 129648, 42737, 42736, 42741, 42740, 42733, 42699, 42713, 42712, + 42692, 42706, 42683, 42719, 42734, 42735, 42682, 42729, 42657, 42725, + 42659, 42670, 42718, 42666, 42716, 42701, 42675, 42671, 42722, 42720, + 42727, 42723, 42702, 42726, 42677, 42707, 42708, 42709, 42686, 42694, + 42674, 42731, 42695, 42685, 42684, 42691, 42673, 42664, 42715, 42703, + 92193, 92188, 92161, 92199, 92240, 92179, 92211, 92219, 92213, 92243, + 92207, 92216, 92183, 92184, 92238, 92171, 983268, 92174, 92203, 92186, + 92210, 92230, 92175, 92221, 92232, 92246, 92198, 92214, 92190, 92176, + 92197, 92196, 92164, 92245, 92212, 92201, 92160, 92182, 92173, 92229, + 92227, 92180, 92237, 92241, 92223, 92185, 92194, 92178, 92239, 92225, + 92233, 92167, 92191, 92231, 92244, 92204, 92226, 92236, 92200, 92189, + 92187, 92162, 92163, 92169, 92170, 92202, 92205, 92222, 92168, 92165, + 92215, 92224, 92208, 92220, 92235, 92177, 92181, 92195, 92234, 92209, + 92166, 92172, 92206, 92192, 92228, 92217, 92242, 92218, 92262, 92271, + 92272, 92270, 92294, 92253, 92301, 92256, 92273, 92259, 92251, 92297, + 92300, 92296, 92295, 92255, 92252, 92292, 92286, 92268, 92290, 92281, + 92267, 92284, 92287, 92282, 92283, 92277, 92298, 92276, 92302, 92247, + 92299, 92288, 92269, 92260, 92261, 92257, 92274, 92289, 92263, 92279, + 92265, 92266, 92250, 92249, 92285, 92248, 92264, 92280, 92258, 92254, + 92278, 92291, 92293, 92275, 92310, 92320, 92312, 92394, 92393, 92319, + 92384, 92321, 92386, 92361, 92373, 92366, 92328, 92329, 92357, 92342, + 92397, 92365, 92376, 92339, 92382, 92363, 92318, 92387, 92383, 92315, + 92385, 92354, 92311, 92381, 92343, 92326, 92364, 92324, 92391, 92344, + 92390, 92338, 92396, 92308, 92349, 92375, 92378, 92317, 92331, 92362, + 92336, 92341, 92370, 92347, 92307, 92303, 92309, 92395, 92368, 92356, + 92367, 92360, 92389, 92333, 92359, 92374, 92351, 92325, 92371, 92350, + 92345, 92372, 92314, 92340, 92323, 92304, 92313, 92316, 92398, 92399, + 92380, 92330, 92379, 92392, 92335, 92332, 92346, 92400, 92358, 92334, + 92353, 92337, 92306, 92369, 92388, 92322, 92305, 92327, 92377, 92352, + 92348, 92355, 92436, 92517, 92488, 92434, 92415, 92431, 92454, 92460, + 92420, 92489, 92422, 92474, 92479, 92449, 92502, 92439, 92484, 92495, + 92464, 92511, 92406, 92446, 92497, 92426, 92482, 92448, 92515, 92401, + 92478, 92427, 92496, 92458, 92424, 92445, 92404, 92466, 92444, 92438, + 92442, 92441, 92510, 92499, 92425, 92437, 92440, 92471, 92465, 92409, + 92483, 92475, 92467, 92485, 92457, 92432, 92476, 92435, 92412, 92414, + 92430, 92407, 92403, 92405, 92487, 92418, 92486, 92408, 92447, 92459, + 92480, 92472, 92505, 92514, 92450, 92463, 92410, 92493, 92507, 92503, + 92462, 92506, 92443, 92509, 92469, 92461, 92490, 92512, 92494, 92455, + 92417, 92501, 92413, 92508, 92500, 92504, 92473, 92419, 92498, 92423, + 92516, 92428, 92452, 92451, 92481, 92416, 92456, 92433, 92477, 92492, + 92491, 92513, 92411, 92402, 92421, 92453, 92468, 92470, 92429, 92661, + 92626, 92616, 92596, 92612, 92603, 92673, 92651, 92627, 92597, 92585, + 92578, 92615, 92565, 92552, 92602, 92674, 92570, 92646, 92599, 92532, + 92587, 92538, 92620, 92670, 92665, 92575, 92521, 92633, 92595, 92523, + 92556, 92539, 92664, 92653, 92667, 92600, 92542, 92555, 92668, 92520, + 92582, 92591, 92581, 92551, 92654, 92611, 92614, 92666, 92671, 92617, + 92637, 92562, 92518, 92592, 92558, 92528, 92607, 92535, 92557, 92563, + 92550, 92624, 92640, 92657, 92531, 92601, 92553, 92543, 92619, 92598, + 92541, 92544, 92579, 92658, 92554, 92628, 92648, 92547, 92527, 92545, + 92540, 92604, 92622, 92589, 92608, 92583, 92609, 92662, 92613, 92584, + 92625, 92634, 92524, 92536, 92605, 92647, 92548, 92663, 92593, 92621, + 92568, 92610, 92576, 92529, 92618, 92649, 92561, 92656, 92526, 92655, + 92546, 92534, 92560, 92580, 92659, 92660, 92638, 92571, 92525, 92549, + 92559, 92635, 92577, 92530, 92636, 92669, 92572, 92672, 92537, 92519, + 92630, 92586, 92569, 92566, 92573, 92652, 92522, 92574, 92650, 92533, + 92567, 92623, 92606, 92639, 92564, 92590, 92642, 92643, 92594, 92641, + 92645, 92644, 92588, 92629, 92632, 92631, 92710, 92695, 92694, 92726, + 92675, 92719, 92677, 92718, 92682, 92717, 92689, 92720, 92724, 92685, + 92722, 92723, 92711, 92712, 92698, 92688, 92697, 92696, 92702, 92687, + 92704, 92681, 92708, 92703, 92706, 92714, 92709, 92679, 92721, 92684, + 92683, 92707, 92691, 92713, 92700, 92693, 92727, 92690, 92692, 92686, + 92680, 92725, 92699, 92701, 92705, 92716, 92728, 92678, 92715, 92676, + 42693, 42698, 42711, 42696, 42667, 42717, 42704, 42661, 42721, 42669, + 42668, 42705, 42700, 42680, 42678, 42710, 42688, 42681, 42732, 42676, + 42679, 42730, 42728, 42672, 42662, 42724, 42687, 42689, 42690, 42697, + 42714, 42660, 42656, 42665, 42663, 42658, 42738, 42742, 42739, 42743, + 127974, 128183, 128180, 128182, 128181, 127820, 129685, 128202, 129532, + 128136, 129530, 127936, 92916, 92912, 92915, 92913, 92914, 92887, 92894, + 92908, 92880, 92907, 92893, 92886, 92888, 92881, 92906, 92896, 92891, + 92902, 92900, 92885, 92890, 92884, 92904, 92905, 92899, 92889, 92897, + 92892, 92895, 92882, 92898, 92883, 92901, 92903, 92909, 92917, 9918, + 129415, 7152, 7153, 7124, 7108, 7114, 7130, 7139, 7127, 7138, 7133, 7136, + 7113, 7111, 7117, 7119, 7107, 7135, 7125, 7112, 7123, 7129, 7116, 7132, + 7105, 7126, 7128, 7110, 7109, 7137, 7121, 7118, 7106, 7120, 7134, 7122, + 7115, 7131, 7104, 7140, 7141, 7154, 7155, 7167, 7165, 7166, 7164, 7142, + 7150, 7151, 7144, 7147, 7149, 7143, 7145, 7146, 7148, 128704, 128705, + 128267, 127900, 127901, 9835, 9836, 129492, 128059, 127958, 128147, + 129451, 129752, 127866, 129714, 983055, 128276, 129745, 128277, 9086, + 128718, 2557, 2432, 2519, 2548, 2552, 2551, 2550, 2549, 2553, 2454, 2510, + 983661, 2453, 2480, 2545, 2544, 2525, 2524, 2556, 2443, 2528, 2444, 2529, + 2527, 2479, 2437, 2438, 2448, 2452, 2466, 2465, 2471, 2470, 2464, 2463, + 2469, 2468, 2441, 2442, 2439, 2440, 2457, 2467, 2462, 2472, 2486, 2487, + 2488, 2477, 2476, 2459, 2458, 2456, 2455, 2461, 2460, 2475, 2474, 2489, + 2482, 2478, 2447, 2451, 2547, 2546, 983651, 983650, 983652, 2558, 2433, + 2492, 2493, 2434, 2509, 2435, 2554, 2494, 2504, 2508, 2497, 2498, 2499, + 2500, 2530, 2531, 2495, 2496, 2503, 2507, 2539, 2538, 2541, 2540, 2537, + 2536, 2534, 2543, 2535, 2542, 2555, 11102, 127857, 9004, 9187, 8812, + 8502, 8757, 129475, 128719, 72710, 72711, 72712, 72746, 72704, 72705, + 72715, 72717, 72731, 72730, 72736, 72735, 72729, 72728, 72734, 72733, + 72708, 72709, 72706, 72707, 72722, 72732, 72727, 72737, 72747, 72748, + 72749, 72741, 72740, 72724, 72723, 72721, 72720, 72726, 72725, 72719, + 72718, 72739, 72738, 72750, 72745, 72742, 72744, 72743, 72714, 72716, + 72801, 72810, 72806, 72797, 72807, 72798, 72802, 72811, 72800, 72809, + 72799, 72808, 72796, 72805, 72804, 72795, 72803, 72794, 72764, 72768, + 72765, 72767, 72766, 72756, 72757, 72758, 72751, 72761, 72763, 72754, + 72755, 72752, 72753, 72760, 72762, 72770, 72769, 72789, 72788, 72791, + 72790, 72787, 72786, 72784, 72793, 72785, 72792, 72771, 72772, 72773, + 72812, 128692, 128690, 10745, 10744, 129506, 127921, 127874, 128038, + 8383, 129766, 9763, 128089, 129452, 9679, 9864, 10733, 9865, 9210, 11176, + 11177, 11178, 11179, 11180, 11182, 11181, 11183, 9960, 10028, 9821, + 129554, 129596, 129609, 129612, 9818, 129551, 129593, 9822, 129543, + 129564, 129585, 129597, 129606, 129555, 129619, 129617, 129618, 9823, + 129556, 129598, 9819, 129552, 129594, 9820, 129553, 129595, 129575, + 129572, 129576, 129577, 129573, 129574, 9827, 9670, 11201, 10070, 10730, + 11230, 9830, 128419, 128899, 9196, 9662, 9660, 11167, 127778, 9922, 9923, + 10047, 9873, 10022, 128447, 128420, 9829, 11042, 128426, 11052, 10711, + 11044, 117867, 117870, 117869, 117868, 11035, 9194, 9198, 128896, 9668, + 9666, 9664, 11164, 8268, 9944, 128412, 9754, 9699, 9698, 10731, 9207, + 11206, 11045, 9204, 11207, 11047, 9205, 11208, 9206, 11205, 128921, + 128927, 9726, 9724, 9912, 117871, 10002, 128392, 9648, 11039, 127986, + 118451, 128413, 9755, 9193, 9197, 9654, 9199, 128898, 11091, 9658, 9656, + 10145, 10148, 11166, 8269, 127990, 9644, 11049, 11050, 11089, 9642, + 129997, 9787, 9632, 11200, 9209, 128306, 9728, 128369, 9927, 9984, + 128900, 128909, 9986, 9751, 9824, 9733, 128919, 128925, 128908, 9951, + 128383, 9742, 9942, 128418, 128897, 9195, 9652, 9650, 9700, 9701, 11165, + 9851, 9646, 11054, 128920, 128926, 11037, 11204, 10707, 10067, 8493, + 8460, 8465, 8476, 8488, 10164, 10166, 10165, 9250, 118018, 118039, + 118128, 118187, 118218, 118159, 118054, 118112, 118234, 118084, 118202, + 118143, 118031, 118062, 118120, 118240, 118179, 118090, 118210, 118151, + 118047, 118104, 118226, 118165, 118076, 118194, 118135, 118021, 118035, + 118066, 118124, 118243, 118183, 118093, 118214, 118155, 118050, 118108, + 118230, 118168, 118080, 118198, 118139, 118028, 118058, 118116, 118237, + 118175, 118087, 118206, 118147, 118043, 118100, 118222, 118162, 118072, + 118190, 118131, 118029, 118060, 118118, 118177, 118208, 118149, 118045, + 118102, 118224, 118023, 118037, 118068, 118126, 118244, 118185, 118095, + 118216, 118157, 118052, 118110, 118232, 118170, 118082, 118200, 118141, + 118074, 118192, 118133, 118020, 118033, 118064, 118122, 118242, 118181, + 118092, 118212, 118153, 118048, 118106, 118228, 118167, 118078, 118196, + 118137, 118026, 118056, 118114, 118235, 118173, 118085, 118204, 118145, + 118041, 118098, 118220, 118160, 118070, 118188, 118129, 118034, 118065, + 118123, 118182, 118213, 118154, 118049, 118107, 118229, 118079, 118197, + 118138, 118017, 118024, 118038, 118069, 118127, 118245, 118186, 118096, + 118217, 118158, 118053, 118111, 118233, 118171, 118083, 118201, 118142, + 118030, 118061, 118119, 118239, 118178, 118089, 118209, 118150, 118046, + 118103, 118225, 118164, 118075, 118193, 118134, 118027, 118057, 118115, + 118236, 118174, 118086, 118205, 118146, 118042, 118099, 118221, 118161, + 118071, 118189, 118130, 118025, 118055, 118113, 118172, 118203, 118144, + 118040, 118097, 118219, 118016, 118022, 118036, 118067, 118125, 118184, + 118094, 118215, 118156, 118051, 118109, 118231, 118169, 118081, 118199, + 118140, 118059, 118117, 118238, 118176, 118088, 118207, 118148, 118044, + 118101, 118223, 118163, 118073, 118191, 118132, 118019, 118105, 118227, + 118166, 118032, 118063, 118121, 118241, 118180, 118091, 118211, 118152, + 118077, 118195, 118136, 129792, 129794, 129798, 129806, 129821, 129836, + 129813, 129844, 129829, 129802, 129817, 129848, 129832, 129810, 129840, + 129825, 129796, 129804, 129819, 129850, 129834, 129842, 129827, 129800, + 129815, 129846, 129831, 129808, 129838, 129823, 129793, 129801, 129816, + 129847, 129797, 129805, 129820, 129851, 129835, 129812, 129843, 129828, + 129809, 129839, 129824, 129795, 129803, 129818, 129849, 129833, 129811, + 129841, 129826, 129799, 129814, 129845, 129830, 129807, 129837, 129822, + 127804, 128033, 128216, 128153, 129744, 128939, 128951, 128957, 128945, + 128902, 128912, 128932, 128366, 128278, 128209, 128218, 129667, 12731, + 12727, 12726, 12724, 12725, 12570, 12574, 12718, 12576, 12719, 12578, + 12580, 12713, 12735, 12720, 12584, 12715, 12572, 12579, 12581, 12709, + 12708, 12573, 12575, 12582, 12557, 12728, 12588, 12707, 12732, 12583, + 12714, 12723, 12589, 12716, 12712, 12585, 12555, 12587, 12717, 12591, + 12571, 12722, 12711, 12590, 12734, 12721, 12710, 12577, 12567, 12563, + 12705, 12730, 12558, 12733, 12568, 12564, 12556, 12729, 12569, 12565, + 12549, 12704, 12560, 12706, 12553, 12552, 12559, 12551, 12550, 12561, + 12566, 12554, 12586, 12562, 118257, 118258, 118259, 118260, 117854, + 11867, 117853, 118253, 118255, 11868, 117855, 118249, 118251, 118247, + 11211, 8993, 130029, 8990, 8973, 11812, 8991, 8972, 11813, 130030, 9141, + 9142, 10555, 9183, 9181, 130026, 130018, 9185, 127870, 128144, 127893, + 128335, 129379, 127923, 8904, 10705, 10706, 127993, 118282, 117792, + 117791, 118281, 9574, 9559, 9556, 9577, 9565, 9562, 9553, 9580, 9571, + 9568, 9552, 9511, 9490, 9503, 9486, 9537, 9520, 9513, 9489, 9505, 9485, + 9543, 9519, 9558, 9555, 9573, 9557, 9554, 9572, 9592, 9598, 9593, 9599, + 9499, 9531, 9495, 9475, 9547, 9515, 9507, 9595, 9523, 9491, 9487, 9551, + 9549, 9483, 9481, 9479, 9477, 9473, 9594, 9541, 9525, 9517, 9533, 9530, + 9522, 9546, 9539, 9582, 9581, 9583, 9584, 130010, 130014, 129954, 129958, + 130003, 129959, 129964, 129965, 129955, 130000, 129952, 129956, 129963, + 129960, 129953, 129961, 129957, 129962, 130007, 130005, 130004, 130012, + 9586, 130008, 130011, 130002, 130015, 130006, 9585, 130009, 130001, + 130013, 9587, 129966, 9591, 9516, 9488, 9484, 9550, 9548, 9472, 117787, + 117788, 129967, 9588, 9596, 9482, 9480, 9478, 9476, 117789, 9589, 9597, + 9524, 9496, 9492, 9474, 118297, 118295, 118296, 118294, 9532, 9508, 9500, + 117790, 9590, 9542, 9526, 9518, 9534, 9529, 9521, 9545, 9540, 9536, 9510, + 9498, 9502, 9494, 9528, 9544, 9514, 9497, 9506, 9493, 9527, 9564, 9561, + 9576, 9563, 9560, 9575, 9570, 9567, 9579, 9538, 9512, 9504, 9535, 9509, + 9501, 9569, 9566, 9578, 129354, 983263, 128163, 128102, 128713, 128023, + 129460, 69649, 69685, 69749, 69745, 69746, 69687, 69686, 69637, 69638, + 69648, 69650, 69664, 69663, 69669, 69668, 69662, 69661, 69667, 69666, + 69643, 69644, 69645, 69646, 69679, 69641, 69642, 69639, 69640, 69684, + 69678, 69655, 69665, 69660, 69670, 69680, 69681, 69682, 69674, 69673, + 69657, 69656, 69654, 69653, 69659, 69658, 69652, 69651, 69672, 69671, + 69683, 69675, 69677, 69676, 69647, 69721, 69730, 69726, 69717, 69727, + 69718, 69722, 69731, 69720, 69729, 69719, 69728, 69716, 69725, 69724, + 69715, 69723, 69714, 69732, 69733, 69759, 69709, 69707, 69706, 69705, + 69708, 69632, 69744, 69635, 69634, 69636, 69633, 69700, 69747, 69748, + 69689, 69688, 69699, 69701, 69692, 69693, 69694, 69695, 69696, 69697, + 69690, 69691, 69698, 69702, 69704, 69703, 69739, 69738, 69741, 69740, + 69737, 69736, 69734, 69743, 69735, 69742, 10241, 10243, 10247, 10255, + 10271, 10303, 10367, 10495, 10431, 10335, 10463, 10399, 10287, 10351, + 10479, 10415, 10319, 10447, 10383, 10263, 10295, 10359, 10487, 10423, + 10327, 10455, 10391, 10279, 10343, 10471, 10407, 10311, 10439, 10375, + 10251, 10267, 10299, 10363, 10491, 10427, 10331, 10459, 10395, 10283, + 10347, 10475, 10411, 10315, 10443, 10379, 10259, 10291, 10355, 10483, + 10419, 10323, 10451, 10387, 10275, 10339, 10467, 10403, 10307, 10435, + 10371, 10245, 10253, 10269, 10301, 10365, 10493, 10429, 10333, 10461, + 10397, 10285, 10349, 10477, 10413, 10317, 10445, 10381, 10261, 10293, + 10357, 10485, 10421, 10325, 10453, 10389, 10277, 10341, 10469, 10405, + 10309, 10437, 10373, 10249, 10265, 10297, 10361, 10489, 10425, 10329, + 10457, 10393, 10281, 10345, 10473, 10409, 10313, 10441, 10377, 10257, + 10289, 10353, 10481, 10417, 10321, 10449, 10385, 10273, 10337, 10465, + 10401, 10305, 10433, 10369, 10242, 10246, 10254, 10270, 10302, 10366, + 10494, 10430, 10334, 10462, 10398, 10286, 10350, 10478, 10414, 10318, + 10446, 10382, 10262, 10294, 10358, 10486, 10422, 10326, 10454, 10390, + 10278, 10342, 10470, 10406, 10310, 10438, 10374, 10250, 10266, 10298, + 10362, 10490, 10426, 10330, 10458, 10394, 10282, 10346, 10474, 10410, + 10314, 10442, 10378, 10258, 10290, 10354, 10482, 10418, 10322, 10450, + 10386, 10274, 10338, 10466, 10402, 10306, 10434, 10370, 10244, 10252, + 10268, 10300, 10364, 10492, 10428, 10332, 10460, 10396, 10284, 10348, + 10476, 10412, 10316, 10444, 10380, 10260, 10292, 10356, 10484, 10420, + 10324, 10452, 10388, 10276, 10340, 10468, 10404, 10308, 10436, 10372, + 10248, 10264, 10296, 10360, 10488, 10424, 10328, 10456, 10392, 10280, + 10344, 10472, 10408, 10312, 10440, 10376, 10256, 10288, 10352, 10480, + 10416, 10320, 10448, 10384, 10272, 10336, 10464, 10400, 10304, 10432, + 10368, 10240, 129504, 983125, 129329, 127838, 728, 127753, 128112, + 128188, 129650, 129521, 9099, 166, 128148, 129382, 129294, 129529, + 129483, 129767, 128027, 6663, 6662, 6659, 6658, 6671, 6670, 6667, 6666, + 6661, 6668, 6665, 6657, 6678, 6669, 6656, 6674, 6660, 6673, 6676, 6664, + 6675, 6672, 6677, 6683, 6681, 6679, 6682, 6680, 6687, 6686, 5957, 5960, + 5962, 5959, 5956, 5969, 5955, 5966, 5963, 5961, 5965, 5968, 5958, 5967, + 5964, 5952, 5953, 5954, 5970, 5971, 8226, 8729, 128363, 128364, 9678, + 128652, 128101, 128100, 128655, 129480, 129419, 127959, 127791, 129699, + 118939, 118940, 118944, 118943, 118941, 118942, 118938, 118945, 118882, + 118876, 118824, 118835, 118797, 118866, 118957, 118801, 118802, 118865, + 118818, 118916, 118819, 118917, 118899, 118935, 119018, 119022, 119021, + 119020, 119023, 119019, 119017, 118986, 118984, 118985, 118843, 118887, + 118808, 118870, 119003, 119002, 119004, 119005, 118937, 118991, 118995, + 118990, 118992, 118994, 118993, 118930, 118933, 118931, 118932, 119014, + 118918, 118912, 119015, 118785, 118831, 118907, 118966, 118900, 118880, + 118798, 118888, 118884, 118869, 118960, 118959, 118958, 118836, 118969, + 118977, 118978, 118971, 118976, 118975, 118973, 118970, 118987, 118979, + 118980, 118972, 118983, 983274, 118982, 118974, 118988, 118981, 119000, + 119001, 118929, 118928, 118806, 119029, 118927, 118898, 118964, 118965, + 118853, 118968, 118837, 118967, 118936, 118956, 118810, 118854, 118847, + 118839, 118906, 118791, 118800, 119024, 119026, 118862, 118812, 119025, + 119027, 118863, 118811, 118820, 119028, 118911, 118842, 118850, 118921, + 118858, 118913, 118914, 118915, 118834, 118860, 118868, 118796, 118881, + 118922, 118926, 118925, 118924, 118923, 118830, 118877, 118949, 118947, + 118948, 118963, 118955, 118962, 118946, 118961, 118953, 118952, 118951, + 118950, 118954, 118871, 118805, 118855, 118828, 118901, 118787, 118788, + 118856, 118816, 118875, 118845, 118879, 118793, 118846, 118878, 118814, + 118822, 118873, 118859, 118857, 118849, 118840, 118861, 118786, 118895, + 118841, 118874, 118892, 118897, 118807, 118784, 118821, 118844, 118825, + 118889, 119010, 119012, 119013, 119011, 119006, 119008, 119009, 119007, + 118910, 118815, 118852, 119016, 118826, 118885, 118827, 118803, 118886, + 118792, 118813, 118920, 118799, 118833, 118829, 118904, 118902, 118903, + 118905, 118804, 118934, 118919, 118832, 118893, 118838, 118851, 118883, + 118894, 118891, 118896, 118823, 118789, 118790, 118872, 118817, 118809, + 118908, 118909, 118996, 118998, 118997, 118999, 118989, 118794, 118795, + 118867, 118864, 118848, 118890, 983262, 983126, 983057, 9764, 8454, + 129305, 128197, 128247, 128248, 127957, 983098, 5130, 5131, 5122, 6322, + 5148, 5551, 5310, 5382, 5166, 6321, 5759, 5559, 5556, 5557, 5558, 5567, + 5564, 5565, 5566, 5563, 5560, 5561, 5562, 5555, 5552, 5553, 5554, 5384, + 5439, 6387, 6388, 5281, 5264, 6382, 6389, 5203, 5674, 5675, 5677, 5676, + 5673, 5672, 5706, 5707, 5709, 5708, 5705, 5704, 5204, 5574, 5575, 5577, + 5576, 5573, 5572, 6384, 6381, 5620, 6383, 5617, 5618, 5619, 5616, 5615, + 5195, 5592, 5593, 5595, 5594, 5591, 5590, 5174, 5175, 5129, 5703, 5662, + 5663, 5665, 5664, 5661, 5660, 5655, 5656, 6386, 5659, 5657, 5654, 5652, + 5633, 5629, 5630, 5632, 5631, 5628, 5627, 5623, 5624, 5626, 5625, 5622, + 5621, 5636, 5637, 5639, 5329, 5638, 5635, 5634, 5722, 5718, 5719, 5721, + 5720, 5717, 5716, 5712, 5713, 5715, 5714, 5711, 5710, 5614, 5610, 5611, + 5613, 5612, 5609, 5608, 5702, 5698, 5699, 5701, 5700, 5697, 5696, 5686, + 5687, 5689, 5688, 5685, 5684, 5692, 5693, 5695, 5694, 5691, 5690, 5737, + 5738, 5740, 5739, 5736, 5735, 5604, 5605, 5607, 5606, 5603, 5602, 5598, + 5599, 5601, 5600, 5597, 5596, 5725, 5726, 5728, 5727, 5724, 5723, 5680, + 5681, 5683, 5682, 5679, 5678, 5668, 5669, 5671, 5670, 5667, 5666, 5731, + 5732, 5734, 5733, 5730, 5729, 5642, 5643, 5645, 5644, 5641, 5640, 5580, + 5581, 5583, 5582, 5579, 5578, 5586, 5587, 5589, 5588, 5585, 5584, 5648, + 5649, 5651, 5650, 5647, 5646, 5128, 5265, 5258, 5276, 5278, 5272, 5274, + 5268, 5270, 5266, 5741, 5261, 5262, 5259, 5260, 5257, 5121, 6364, 5163, + 5469, 5465, 5466, 5460, 5461, 5158, 5157, 5162, 5155, 5156, 6367, 6366, + 5160, 5152, 5151, 5153, 5154, 5161, 5159, 5462, 5742, 5463, 5464, 5467, + 5459, 5120, 5501, 5123, 5124, 5164, 5251, 5252, 5246, 5248, 6329, 5242, + 5244, 5238, 5240, 5236, 5234, 5235, 5228, 6328, 5231, 5232, 5229, 5230, + 5227, 5354, 5542, 5540, 5541, 5538, 5539, 5536, 5537, 5338, 5339, 5332, + 6333, 5350, 5352, 5346, 5348, 5342, 5344, 5340, 5335, 5336, 5333, 5334, + 5331, 5307, 5283, 5356, 5458, 5287, 5288, 5385, 5290, 5291, 5284, 6330, + 5302, 5304, 5298, 5300, 5294, 5296, 5292, 5285, 5286, 5309, 5328, 5473, + 5475, 5471, 5319, 5386, 5390, 5391, 5388, 5389, 5380, 5387, 5142, 5147, + 5280, 5250, 5306, 5327, 5221, 5437, 72372, 72373, 72370, 72371, 72368, + 72369, 72378, 72379, 72376, 72377, 72374, 72375, 5320, 5313, 6332, 5525, + 5523, 5524, 5518, 5744, 5521, 5522, 5519, 5520, 5526, 5749, 5750, 5747, + 5748, 5745, 5746, 5499, 5497, 5498, 5495, 5496, 5493, 5494, 5492, 5500, + 5316, 5317, 6331, 5323, 5325, 6346, 6348, 6342, 6344, 5321, 5314, 5315, + 5312, 5330, 5509, 5507, 5508, 5502, 5743, 5505, 5506, 5503, 5504, 5125, + 6361, 6347, 6349, 6343, 6345, 6362, 6363, 6359, 6358, 6360, 6356, 6357, + 5165, 5126, 6320, 5193, 5184, 5186, 6326, 5188, 5190, 5180, 5182, 5178, + 5176, 5177, 5168, 6325, 5171, 5172, 6324, 5169, 5170, 5167, 5456, 6368, + 5443, 6355, 5454, 6353, 6354, 6351, 6352, 6350, 5451, 5452, 5445, 6341, + 5448, 5449, 5446, 5447, 5442, 5381, 5364, 6335, 5570, 6380, 5571, 5568, + 5569, 5653, 6385, 5658, 5529, 6379, 6378, 5530, 5527, 5528, 5441, 5282, + 5311, 5365, 5358, 5413, 5405, 5407, 6338, 5409, 5411, 5401, 5403, 5399, + 5395, 5396, 6336, 5397, 5398, 6337, 5393, 5394, 5392, 5361, 5256, 5253, + 5254, 5255, 5362, 6334, 5383, 5376, 5378, 5372, 5374, 5368, 5370, 5366, + 72383, 72380, 72381, 72382, 5359, 5360, 5357, 5222, 5482, 5550, 5548, + 5549, 5546, 5547, 5544, 5545, 5543, 6372, 5480, 6371, 5478, 5479, 5476, + 5477, 5472, 5474, 5470, 5512, 6377, 6376, 5513, 5510, 5511, 5487, 5486, + 6375, 5485, 6374, 6373, 5483, 5484, 5226, 5223, 5224, 5225, 5205, 5206, + 5197, 6327, 5217, 5219, 5213, 5215, 5209, 5211, 5207, 5491, 5488, 5489, + 5490, 5200, 5201, 5198, 5199, 5196, 5132, 5355, 5351, 5353, 5347, 5349, + 5343, 5345, 5341, 5453, 6370, 5450, 6369, 5444, 5308, 5303, 5305, 5299, + 5301, 5295, 5297, 5293, 5194, 5189, 5191, 5185, 5187, 5181, 5183, 5179, + 5440, 5434, 5436, 5430, 5432, 5426, 5428, 5424, 5324, 5326, 5322, 5457, + 5455, 5517, 5514, 5515, 5516, 5410, 5412, 5406, 5408, 5402, 5404, 5400, + 5377, 5379, 5373, 5375, 5369, 5371, 5367, 5277, 5279, 5273, 5275, 5269, + 5271, 5267, 5247, 5249, 5243, 5245, 5239, 5241, 5237, 5481, 5218, 5220, + 5214, 5216, 5210, 5212, 5208, 5468, 5144, 5146, 5139, 5141, 5135, 5137, + 5133, 6365, 5138, 5140, 5535, 5756, 5757, 5754, 5755, 5752, 5753, 5751, + 5534, 5531, 5532, 5533, 5758, 5143, 5145, 6323, 5134, 5136, 5438, 5192, + 5173, 5263, 5233, 5337, 5289, 5318, 5363, 5202, 5420, 5127, 5149, 5421, + 5422, 5415, 6340, 5418, 5419, 6339, 5433, 5435, 5429, 5431, 5425, 5427, + 5423, 5416, 5417, 5414, 5150, 983097, 983171, 917631, 128473, 9803, + 128367, 127852, 129387, 128758, 11839, 9809, 128199, 128450, 128451, + 8248, 8257, 8453, 66225, 66246, 66211, 66214, 66254, 66218, 66250, 66251, + 66252, 66253, 66229, 66238, 66244, 66227, 66224, 66222, 66223, 66242, + 66243, 66232, 66221, 66247, 66230, 66226, 66239, 66212, 66248, 66256, + 66235, 66208, 66215, 66210, 66220, 66234, 66255, 66240, 66241, 66236, + 66237, 66231, 66209, 66213, 66249, 66233, 66245, 66217, 66219, 66216, + 66228, 127904, 711, 127887, 129690, 983073, 129365, 9936, 128008, 128049, 128569, 128572, 66888, 66864, 66912, 66882, 66873, 66902, 66889, 66890, 66911, 66891, 66895, 66901, 66881, 66867, 66870, 66868, 66904, 66866, 66876, 66910, 66879, 66883, 66897, 66915, 66884, 66878, 66885, 66914, @@ -9540,12 +9864,12 @@ static const unsigned int dawg_pos_to_codepoint[] = { 127247, 8857, 8861, 12905, 12919, 12904, 12918, 12903, 12917, 12926, 12909, 12923, 12906, 12920, 12896, 12910, 12900, 12914, 12897, 12911, 12908, 12922, 12901, 12915, 12899, 12913, 12902, 12916, 12907, 12921, - 12898, 12912, 9097, 10162, 127343, 12975, 127569, 12959, 127568, 12963, + 12898, 12912, 9097, 127343, 10162, 12975, 127569, 12959, 127568, 12963, 12951, 12962, 12965, 12973, 12943, 12957, 12935, 12950, 12939, 12932, 12955, 12931, 12964, 12946, 12869, 12871, 12952, 12966, 12967, 12969, 12942, 12954, 12938, 12976, 12936, 12948, 12868, 12974, 12961, 12970, 12968, 12953, 12934, 12972, 12956, 12944, 12870, 12947, 12949, 12971, - 12945, 12933, 12930, 12937, 12929, 12941, 12940, 12960, 12958, 12928, + 12945, 12933, 12958, 12930, 12937, 12929, 12941, 12940, 12960, 12928, 127275, 127276, 128712, 13033, 13036, 13034, 13037, 13035, 13013, 13016, 13014, 13017, 13015, 13038, 13041, 13039, 13042, 13040, 13028, 13031, 13029, 13032, 13030, 13046, 13049, 13047, 13050, 13048, 13018, 13021, @@ -9682,30 +10006,30 @@ static const unsigned int dawg_pos_to_codepoint[] = { 12009, 11951, 11925, 11924, 11922, 11949, 11948, 11917, 11916, 11958, 11932, 12017, 11911, 11923, 11969, 11982, 11981, 11938, 11937, 11972, 11971, 11956, 11955, 11954, 11953, 11913, 11912, 11961, 12752, 12743, - 12748, 12757, 12741, 12750, 12769, 12747, 12749, 12744, 12742, 12746, - 12768, 12758, 12754, 12763, 12770, 12764, 12753, 12740, 12767, 12760, - 12759, 12745, 12766, 12762, 12755, 12761, 12736, 12765, 12739, 12737, - 12738, 12756, 12751, 12771, 128079, 127916, 127963, 128385, 129346, - 127867, 128203, 128346, 128358, 128343, 128355, 128340, 128352, 128339, - 128351, 128344, 128356, 128336, 128348, 128342, 128354, 128341, 128353, - 128345, 128357, 128347, 128359, 128337, 128349, 128338, 128350, 10561, - 8754, 128259, 10227, 128472, 128257, 128258, 8631, 11118, 8635, 8753, - 10959, 10961, 10960, 10962, 127746, 10828, 10832, 128234, 128235, 128272, - 128213, 10829, 8272, 9729, 127786, 127785, 127784, 127783, 129313, 9114, - 129715, 127864, 129381, 58, 8788, 8353, 128165, 7625, 7623, 769, 791, - 833, 8410, 8404, 8423, 857, 8432, 7677, 844, 774, 7627, 814, 810, 838, - 70459, 780, 812, 784, 8409, 8405, 787, 789, 806, 65062, 65069, 42609, - 1160, 11774, 11744, 11768, 11747, 11757, 11765, 42654, 11751, 11752, - 11753, 11756, 42613, 11775, 11772, 42655, 11767, 11754, 42619, 11763, - 11762, 42618, 42615, 42612, 42617, 11770, 42614, 11771, 11773, 11769, - 11759, 42616, 11760, 11758, 11748, 11749, 11761, 11746, 11764, 11755, - 11745, 11750, 11766, 42621, 1156, 1158, 1159, 1157, 42608, 42610, 1155, - 65070, 65071, 1161, 42620, 123023, 42607, 770, 813, 807, 43248, 43244, - 43245, 43246, 43247, 43242, 43243, 43249, 43237, 43236, 43239, 43238, - 43235, 43234, 43232, 43241, 43233, 43240, 7675, 776, 6833, 804, 775, - 7672, 856, 803, 7674, 7617, 7616, 861, 860, 865, 7676, 862, 863, 6840, - 831, 6858, 6857, 6844, 866, 858, 864, 65058, 65059, 8422, 840, 782, 779, - 783, 819, 7629, 6832, 798, 6835, 8413, 8416, 8418, 8414, 8419, 8420, + 12748, 12768, 12772, 12757, 12741, 12750, 12769, 12747, 12749, 12744, + 12742, 12746, 12758, 12754, 12763, 12770, 12764, 12753, 12740, 12767, + 12760, 12759, 12745, 12773, 12766, 12762, 12755, 12761, 12736, 12765, + 12739, 12737, 12738, 12756, 12751, 12771, 128079, 127916, 127963, 128385, + 129346, 127867, 128203, 128346, 128358, 128343, 128355, 128340, 128352, + 128339, 128351, 128344, 128356, 128336, 128348, 128342, 128354, 128341, + 128353, 128345, 128357, 128347, 128359, 128337, 128349, 128338, 128350, + 10561, 8754, 128259, 10227, 128472, 128257, 128258, 8631, 11118, 8635, + 8753, 10959, 10961, 10960, 10962, 127746, 10828, 10832, 128234, 128235, + 128272, 128213, 10829, 8272, 9729, 127786, 127785, 127784, 127783, + 129313, 9114, 129715, 127864, 129381, 58, 8788, 8353, 128165, 7625, 7623, + 769, 791, 833, 8410, 8404, 8423, 857, 8432, 7677, 844, 774, 7627, 814, + 810, 838, 70459, 780, 812, 784, 8409, 8405, 787, 789, 806, 65062, 65069, + 42609, 1160, 11774, 11744, 11768, 11747, 11757, 11765, 42654, 11751, + 11752, 11753, 11756, 42613, 11775, 11772, 42655, 11767, 11754, 42619, + 11763, 11762, 42618, 42615, 42612, 42617, 11770, 42614, 11771, 11773, + 11769, 11759, 42616, 11760, 11758, 11748, 11749, 11761, 11746, 11764, + 11755, 11745, 11750, 11766, 42621, 1156, 1158, 1159, 1157, 42608, 42610, + 1155, 65070, 65071, 1161, 42620, 123023, 42607, 770, 813, 807, 43248, + 43244, 43245, 43246, 43247, 43242, 43243, 43249, 43237, 43236, 43239, + 43238, 43235, 43234, 43232, 43241, 43233, 43240, 7675, 776, 6833, 804, + 775, 7672, 856, 803, 7674, 7617, 7616, 861, 860, 865, 7676, 862, 863, + 6840, 831, 6858, 6857, 6844, 866, 858, 864, 65058, 65059, 8422, 840, 782, + 779, 783, 819, 7629, 6832, 798, 6835, 8413, 8416, 8418, 8414, 8419, 8420, 8415, 839, 850, 8412, 122892, 122884, 122891, 122890, 122921, 122919, 122889, 122916, 122900, 122907, 122910, 122901, 122908, 122880, 122920, 122881, 122909, 122903, 122922, 122883, 122895, 122894, 122896, 122898, @@ -9735,10 +10059,10 @@ static const unsigned int dawg_pos_to_codepoint[] = { 996, 11434, 11472, 11414, 11422, 11478, 11468, 11470, 11476, 11474, 11482, 11460, 11480, 11454, 11462, 11444, 11486, 11488, 11484, 11490, 11440, 1004, 994, 11456, 11402, 11428, 11408, 11430, 11404, 11394, 11438, - 11424, 11410, 11426, 11400, 11416, 11418, 11432, 66272, 66298, 66289, - 66295, 66286, 66294, 66285, 66299, 66290, 66291, 66297, 66288, 66296, - 66287, 66293, 66284, 66292, 66283, 66282, 66277, 66276, 66279, 66278, - 66275, 66274, 66281, 66273, 66280, 11517, 11518, 11514, 11515, 11516, + 11424, 11410, 11426, 11400, 11416, 11418, 11432, 66298, 66289, 66295, + 66286, 66294, 66285, 66299, 66290, 66291, 66297, 66288, 66296, 66287, + 66293, 66284, 66292, 66283, 66282, 66277, 66276, 66279, 66278, 66275, + 66274, 66281, 66273, 66280, 66272, 11517, 11518, 11514, 11515, 11516, 11513, 11465, 11393, 11507, 11500, 11502, 11447, 11453, 11459, 11467, 11443, 11449, 11451, 11399, 1007, 1003, 11397, 1001, 11407, 999, 11437, 11413, 11421, 997, 11435, 11473, 11415, 11423, 11479, 11469, 11471, @@ -9749,7 +10073,7 @@ static const unsigned int dawg_pos_to_codepoint[] = { 8792, 129720, 9012, 9013, 119661, 119660, 119663, 119662, 119659, 119658, 119665, 119657, 119664, 119652, 119651, 119654, 119653, 119650, 119649, 119656, 119648, 119655, 128715, 128145, 128004, 128046, 9904, 129689, - 129509, 983074, 128179, 127769, 129431, 127951, 9769, 9768, 11857, 11856, + 129509, 983074, 128179, 127769, 129431, 127951, 9769, 9768, 11856, 11857, 128322, 128321, 10060, 127370, 127884, 9876, 9932, 128010, 129360, 128081, 8354, 129660, 128575, 128546, 128302, 129408, 8731, 74794, 74771, 74861, 74780, 74820, 74821, 74765, 74758, 74853, 74856, 74855, 74854, @@ -9815,462 +10139,871 @@ static const unsigned int dawg_pos_to_codepoint[] = { 74117, 74116, 74118, 74630, 74119, 74962, 74963, 74631, 74122, 74123, 74120, 74121, 74626, 74107, 74106, 74628, 74108, 74109, 74124, 74125, 74126, 74131, 74132, 74128, 74129, 74130, 74133, 74134, 74135, 74136, - 74137, 74138, 74139, 74140, 74141, 74142, 74607, 74127, 74144, 74146, - 74147, 74145, 74152, 74153, 74150, 74151, 74148, 74149, 74154, 74155, - 74157, 74158, 74163, 74164, 74165, 74160, 74161, 74156, 74159, 74162, - 74143, 74166, 74167, 74168, 74169, 74170, 74171, 74172, 74175, 74173, - 74174, 74176, 74177, 74182, 74183, 74180, 74181, 74632, 74186, 74184, - 74185, 74188, 74190, 74189, 74187, 74194, 74195, 74193, 74191, 74192, - 74196, 74197, 74198, 74199, 74200, 74201, 74202, 74205, 74206, 74207, - 74208, 74204, 74209, 74211, 74210, 74212, 74213, 74215, 74214, 74216, - 74218, 74217, 74964, 74178, 74179, 74203, 74219, 74220, 74223, 74224, - 74221, 74222, 74966, 74967, 74974, 74973, 74972, 74969, 74970, 74971, - 74975, 74968, 74965, 74977, 74976, 74980, 74981, 74982, 74984, 74985, - 74978, 74979, 74983, 74986, 74987, 74988, 74989, 74990, 74991, 74993, - 74997, 74996, 74998, 74995, 74994, 74992, 74999, 75000, 75003, 75004, - 75005, 75006, 75001, 75002, 75009, 75015, 75016, 75019, 75017, 75018, - 75013, 75012, 75010, 75011, 75014, 75021, 75030, 75029, 75027, 75024, - 75025, 75028, 75022, 75026, 75023, 75008, 75020, 75031, 75032, 75007, - 74226, 74227, 74228, 74229, 74230, 74225, 74231, 74233, 74234, 74232, - 74235, 74237, 74258, 74259, 75033, 74261, 74633, 74260, 74240, 74634, - 74241, 74243, 75035, 74246, 74247, 74245, 74248, 74249, 74250, 74251, - 75036, 75037, 74254, 74255, 74635, 74256, 75038, 74242, 74252, 74253, - 75034, 74238, 74239, 74244, 74257, 74263, 74265, 74264, 74266, 74269, - 74270, 74271, 74236, 74262, 74267, 74268, 74272, 74273, 74274, 74278, - 74279, 74275, 74276, 74277, 74280, 74281, 74636, 74282, 75039, 74283, - 74284, 74290, 74294, 74295, 75041, 75040, 74292, 74293, 74291, 74296, - 74297, 74298, 74299, 74300, 74637, 74301, 74286, 74287, 74285, 74289, - 74288, 74302, 74304, 74307, 74305, 74306, 74308, 74310, 74309, 74311, - 74303, 74638, 74312, 74314, 74313, 74315, 74316, 74319, 74321, 74320, - 74639, 74322, 74324, 74325, 74323, 74642, 75043, 74326, 75044, 75046, - 75047, 74327, 75048, 74329, 74328, 75049, 74330, 74332, 74333, 74331, - 75050, 75051, 74334, 75052, 74335, 75042, 74641, 75045, 74640, 74317, - 74336, 74318, 74337, 74338, 983266, 983265, 74643, 74339, 74347, 74348, - 74342, 74343, 74341, 74344, 74340, 74346, 74345, 74349, 74355, 74354, - 74350, 74352, 74358, 74359, 74353, 74357, 74351, 74356, 74360, 74361, - 74362, 74363, 74364, 74365, 74366, 74644, 74367, 74375, 74376, 74368, - 74369, 74373, 74374, 74370, 74371, 74372, 74377, 74378, 74379, 74380, - 74381, 74382, 74645, 74383, 74384, 74385, 74386, 74387, 74389, 74408, - 75053, 74388, 74646, 74395, 74394, 75055, 74400, 74399, 75056, 74401, - 74406, 74402, 74403, 74404, 74405, 74391, 74392, 74396, 74398, 75054, - 74397, 74393, 74390, 74407, 74409, 74410, 74411, 74412, 74413, 74414, - 74421, 74422, 74419, 74417, 74420, 74416, 74418, 74415, 74423, 75057, - 74424, 74425, 74426, 75058, 74428, 74429, 75059, 75060, 75061, 74427, - 74432, 74434, 74433, 74430, 74431, 74435, 74437, 74436, 74438, 74441, - 74440, 74444, 74445, 74446, 74448, 74447, 74443, 74449, 74442, 74439, - 74451, 74453, 74452, 74450, 74454, 74455, 74456, 74457, 75063, 75062, - 74458, 74459, 75064, 74460, 74461, 74462, 74463, 74465, 74464, 74466, - 74468, 74469, 74471, 74472, 74473, 74474, 74467, 74470, 74475, 74477, - 74478, 74479, 74476, 74480, 74481, 74482, 74483, 74488, 74486, 74487, - 74485, 74489, 74484, 74490, 75065, 74491, 74494, 74497, 74499, 74500, - 74498, 74495, 74647, 74496, 74501, 74504, 75066, 75067, 74505, 74506, - 74502, 74503, 74492, 74493, 74507, 74510, 74512, 74511, 74649, 74509, - 74508, 74515, 74516, 74522, 74523, 74519, 74520, 74517, 74518, 74521, - 74524, 74534, 74535, 74525, 74526, 74648, 74527, 74528, 74529, 74531, - 74532, 74533, 74530, 74536, 74538, 74537, 74539, 75068, 74540, 74541, - 74542, 74545, 74546, 74547, 75069, 74544, 74543, 74549, 74550, 74551, - 74552, 74553, 75070, 74555, 74556, 74558, 74557, 74559, 74560, 74562, - 74564, 74563, 75072, 74566, 75071, 74570, 74569, 74572, 74574, 74573, - 74554, 74567, 74571, 74565, 74561, 74568, 74575, 74576, 74548, 74577, - 74581, 74579, 74580, 74578, 74584, 74583, 74582, 74586, 74587, 74588, - 74585, 74513, 74514, 74589, 74591, 74590, 74593, 75073, 74592, 74595, - 74598, 74599, 74596, 74601, 74597, 74600, 74602, 75074, 74603, 75075, - 74604, 74605, 74606, 74594, 9982, 129380, 11232, 129473, 8911, 8910, - 10160, 9130, 129356, 128177, 164, 127835, 10081, 127854, 128707, 129362, - 129385, 67594, 67595, 67596, 67597, 67598, 67599, 67600, 67601, 67602, - 67603, 67604, 67605, 67606, 67607, 67608, 67609, 67610, 67611, 67612, - 67613, 67614, 67615, 67616, 67617, 67618, 67619, 67620, 67621, 67622, - 67623, 67624, 67625, 67626, 67627, 67628, 67629, 67630, 67631, 67632, - 67633, 67634, 67635, 67636, 67637, 67589, 67592, 67644, 67647, 67639, - 67640, 67584, 67585, 67586, 67587, 67588, 77712, 77713, 77714, 77715, - 77716, 77717, 77718, 77719, 77722, 77723, 77720, 77721, 77724, 77725, - 77726, 77727, 77728, 77729, 77730, 77731, 77732, 77733, 77734, 77735, - 77736, 77737, 77738, 77739, 77740, 77741, 77742, 77743, 77744, 77745, - 77746, 77747, 77748, 77749, 77750, 77751, 77752, 77753, 77754, 77755, - 77756, 77757, 77758, 77773, 77774, 77768, 77769, 77770, 77771, 77772, - 77775, 77776, 77777, 77788, 77789, 77790, 77791, 77792, 77793, 77794, - 77795, 77796, 77759, 77760, 77761, 77762, 77763, 77764, 77765, 77766, - 77767, 77778, 77779, 77780, 77781, 77782, 77783, 77784, 77785, 77786, - 77787, 77797, 77798, 77799, 77800, 77801, 77802, 77803, 77804, 77805, - 77806, 77807, 77808, 77809, 77810, 1126, 1033, 1300, 42574, 1034, 1055, - 1190, 1316, 1136, 1146, 42564, 42592, 42580, 1296, 1302, 1058, 42634, - 1196, 1035, 42640, 42638, 1062, 42642, 42636, 1059, 1266, 1264, 1262, - 1144, 1028, 1040, 1234, 1232, 1212, 1214, 1248, 1192, 1310, 1256, 1258, - 1184, 42602, 1130, 42572, 42586, 1030, 1041, 1063, 1268, 1206, 1208, - 42584, 42650, 42630, 1026, 42568, 42604, 42648, 1029, 42562, 1322, 42632, - 1039, 42626, 1324, 42624, 1044, 1069, 1051, 1312, 1326, 1221, 1298, 1053, - 1314, 1186, 1320, 1225, 1223, 1056, 1166, 1260, 1057, 1194, 1052, 1229, - 1060, 1043, 1172, 1168, 1270, 1170, 1274, 1027, 1061, 1202, 1276, 1278, - 1066, 42644, 1048, 1252, 1037, 1250, 1045, 1024, 1238, 1025, 42588, 1128, - 1132, 42578, 42582, 1124, 42566, 1140, 1142, 1050, 1178, 1219, 1180, - 1182, 1286, 1282, 1280, 1288, 1290, 1292, 1294, 1284, 1152, 1227, 1036, - 1134, 42600, 42570, 1054, 1120, 1148, 1254, 1150, 1240, 1242, 1049, 1162, - 1038, 1210, 1318, 1065, 42646, 1064, 42596, 42598, 1068, 42594, 1198, - 1200, 1164, 1071, 1304, 1122, 1067, 1272, 42576, 1031, 42590, 1070, 1047, - 1246, 1176, 42560, 1046, 1244, 1174, 1217, 42628, 1138, 1032, 1042, 1308, - 1306, 1236, 1204, 1188, 42622, 42606, 1216, 7467, 42623, 1072, 1235, - 1233, 1213, 1215, 1249, 1193, 1311, 1257, 1259, 1185, 42603, 1131, 42573, - 42587, 1110, 1073, 1095, 1269, 1207, 1209, 42585, 42651, 42631, 1106, - 42569, 42605, 42649, 1109, 42563, 1323, 42633, 1119, 42627, 1325, 42625, - 1076, 1101, 1083, 1313, 1327, 1222, 1299, 1085, 1315, 1187, 1321, 1226, - 1224, 1088, 1167, 1261, 1089, 1195, 1084, 1230, 1092, 1075, 1173, 1169, - 1271, 1171, 1275, 1107, 1093, 1203, 1277, 1279, 1098, 42645, 1080, 1253, - 1117, 1251, 1077, 1104, 1239, 1105, 42589, 1129, 1133, 42579, 42583, - 1125, 42567, 1141, 1143, 1082, 1179, 1220, 1181, 1183, 1287, 1283, 1281, - 1289, 1291, 1293, 1295, 1285, 1153, 1228, 1116, 1135, 1127, 7297, 1113, - 1301, 42601, 42571, 42575, 7298, 1114, 1086, 1121, 1149, 1255, 1151, - 1087, 1191, 1317, 1231, 1137, 42565, 42593, 42581, 1297, 1147, 7296, - 1303, 1241, 1243, 1081, 1163, 1118, 1211, 1319, 1097, 42647, 1096, 42597, - 42599, 1100, 42595, 1199, 1201, 1165, 7302, 7303, 7300, 1090, 42635, - 1197, 1115, 42641, 42639, 1094, 7301, 42643, 42637, 1091, 1267, 1265, - 1263, 1145, 1108, 7304, 7299, 1309, 1103, 1305, 1123, 1099, 1273, 42577, - 1111, 42591, 1102, 1079, 1247, 1177, 42561, 1078, 1245, 1175, 1218, - 42629, 1139, 1112, 1074, 1307, 1237, 1205, 1189, 122984, 122962, 122986, - 122985, 122965, 122976, 122971, 122974, 122964, 122983, 122977, 122981, - 122982, 122980, 122978, 122967, 122968, 122969, 122966, 122979, 122973, - 122963, 122970, 122961, 122972, 122975, 1154, 9005, 127744, 983201, - 983188, 983172, 8224, 11830, 11831, 128481, 128131, 127841, 128374, 9619, - 11843, 128168, 10143, 65101, 65097, 8504, 983081, 983084, 983086, 983088, - 983090, 983162, 127795, 9110, 9192, 128475, 8451, 176, 8457, 983120, - 128666, 8796, 983119, 9161, 9159, 9153, 9156, 9162, 9160, 9154, 9157, - 9163, 9150, 9158, 9164, 9151, 9152, 9155, 127980, 9739, 66589, 66591, - 66596, 66597, 66587, 66585, 66594, 66595, 66593, 66599, 66562, 66563, - 66564, 66565, 66561, 66560, 66568, 66569, 66570, 66571, 66567, 66566, - 66598, 66573, 66588, 66579, 66592, 66590, 66581, 66578, 66580, 66582, - 66577, 66586, 66575, 66584, 66583, 66574, 66572, 66576, 66629, 66631, - 66636, 66637, 66627, 66625, 66634, 66635, 66633, 66639, 66602, 66603, - 66604, 66605, 66601, 66600, 66608, 66609, 66610, 66611, 66607, 66606, - 66638, 66613, 66628, 66619, 66632, 66630, 66621, 66618, 66620, 66622, - 66617, 66626, 66615, 66624, 66623, 66614, 66612, 66616, 127964, 127965, - 128421, 128468, 2388, 2416, 43258, 2387, 43257, 72448, 72449, 43259, - 2309, 2310, 2320, 2324, 2421, 43262, 2330, 2418, 2317, 2321, 2331, 2396, - 2430, 2338, 2337, 2343, 2342, 2429, 2394, 2328, 2427, 2327, 2426, 2361, - 2350, 2424, 2308, 2318, 2322, 2358, 2359, 2360, 2323, 2420, 2419, 2431, - 2349, 2348, 2333, 2428, 2332, 2393, 2326, 2325, 2397, 2353, 2352, 2399, - 2351, 2313, 2314, 2423, 2422, 2345, 2339, 2329, 2334, 2344, 2356, 2355, - 2354, 2336, 2335, 2341, 2340, 2315, 2400, 2316, 2401, 2357, 2311, 2312, - 2347, 2346, 2425, 2395, 2398, 2392, 2319, 983644, 983643, 983646, 983647, - 983649, 983648, 983642, 983645, 72450, 72451, 72452, 72453, 2305, 43255, - 43251, 43254, 43253, 72455, 72454, 72456, 43256, 43250, 43260, 43252, - 2304, 2365, 2306, 72457, 2364, 2417, 2381, 2307, 2386, 2385, 2366, 2376, - 2380, 2383, 43263, 2389, 2373, 2377, 2379, 2363, 2362, 2382, 2369, 2370, - 2391, 2390, 2374, 2378, 2371, 2372, 2402, 2403, 2367, 2368, 2375, 2405, - 2404, 2411, 2410, 2413, 2412, 2409, 2408, 2406, 2415, 2407, 2414, 43261, - 2384, 983089, 983161, 983087, 983085, 983083, 127962, 129487, 129420, - 11033, 11032, 11030, 11031, 128924, 128160, 8900, 8960, 168, 9856, 9857, - 9858, 9859, 9860, 9861, 128754, 53, 127238, 9356, 52, 127237, 9355, 57, - 127242, 9360, 49, 127234, 9352, 55, 127240, 9358, 54, 127239, 9357, 51, - 127236, 9354, 50, 127235, 9353, 48, 127233, 127232, 56, 127241, 9359, - 119557, 119556, 119555, 9868, 9871, 9870, 9869, 119553, 119554, 10131, - 10126, 10125, 10128, 10127, 10124, 10123, 127244, 10130, 10122, 10129, - 10111, 10106, 10105, 10108, 10107, 10104, 10103, 10110, 10102, 10109, - 10121, 10116, 10115, 10118, 10117, 10114, 10113, 127243, 10120, 10112, - 10119, 9107, 127919, 128549, 128542, 9933, 9090, 129400, 72004, 72021, - 72020, 72023, 72022, 72019, 72018, 72016, 72025, 72017, 72024, 71964, - 71958, 71963, 71974, 71973, 71936, 71937, 71961, 71960, 71966, 71965, - 71940, 71941, 71938, 71939, 71982, 71976, 71952, 71962, 71957, 71967, - 71978, 71979, 71980, 71971, 71970, 71954, 71953, 71951, 71950, 71949, - 71948, 71969, 71968, 71981, 71955, 71972, 71975, 71977, 71983, 71942, - 71945, 71997, 71996, 72003, 71995, 71984, 71991, 71987, 71988, 71985, - 71986, 71989, 71992, 71998, 72002, 72000, 72005, 72006, 71999, 72001, - 8725, 247, 8903, 129343, 8739, 9902, 129684, 128171, 128565, 12291, 8783, - 9009, 128462, 128441, 128442, 128443, 8939, 8941, 8716, 8740, 10990, - 8832, 8928, 8876, 8833, 8929, 8878, 128021, 71723, 71716, 71680, 71681, - 71687, 71689, 71703, 71702, 71708, 71707, 71701, 71700, 71706, 71705, - 71684, 71685, 71682, 71683, 71694, 71704, 71699, 71709, 71719, 71720, - 71721, 71713, 71712, 71696, 71695, 71693, 71692, 71698, 71697, 71691, - 71690, 71711, 71710, 71722, 71717, 71714, 71718, 71715, 71686, 71688, - 71729, 71730, 71724, 71732, 71734, 71727, 71728, 71725, 71726, 71731, - 71733, 71739, 71738, 71735, 71737, 71736, 128054, 128044, 36, 127025, - 127026, 127027, 127028, 127029, 127030, 127031, 127032, 127033, 127034, - 127035, 127036, 127037, 127038, 127039, 127040, 127041, 127042, 127043, - 127044, 127045, 127046, 127047, 127048, 127049, 127050, 127051, 127052, - 127053, 127054, 127055, 127056, 127057, 127058, 127059, 127060, 127061, - 127062, 127063, 127064, 127065, 127066, 127067, 127068, 127069, 127070, - 127071, 127072, 127073, 127024, 127075, 127076, 127077, 127078, 127079, - 127080, 127081, 127082, 127083, 127084, 127085, 127086, 127087, 127088, - 127089, 127090, 127091, 127092, 127093, 127094, 127095, 127096, 127097, - 127098, 127099, 127100, 127101, 127102, 127103, 127104, 127105, 127106, - 127107, 127108, 127109, 127110, 127111, 127112, 127113, 127114, 127115, - 127116, 127117, 127118, 127119, 127120, 127121, 127122, 127123, 127074, - 129743, 8363, 8724, 8760, 8901, 729, 9676, 8284, 11850, 11034, 129765, - 11795, 11784, 10649, 11798, 9470, 9465, 9464, 9467, 9466, 9463, 9462, - 9469, 9461, 9468, 10175, 10868, 10986, 8225, 8223, 11840, 8914, 8748, - 11842, 8222, 8215, 10835, 10836, 10645, 10913, 10915, 10914, 10939, 8243, - 12318, 10746, 10830, 10831, 11849, 10988, 11844, 10940, 8913, 8912, - 11005, 10987, 8915, 9208, 10981, 8875, 10979, 8214, 11799, 10646, 733, - 8252, 8263, 65100, 10908, 10907, 11002, 11001, 10906, 10905, 8510, 8473, - 8511, 8450, 8461, 8469, 8474, 8477, 8484, 8518, 8519, 8520, 8521, 8517, - 8509, 8508, 8512, 10719, 9890, 9891, 11260, 127849, 8868, 10993, 10623, - 8945, 8964, 128315, 128317, 10728, 10729, 129288, 129290, 129289, 129291, - 8595, 8629, 8671, 129035, 129031, 129179, 129043, 129027, 129047, 8626, - 8627, 10504, 8693, 129975, 8615, 10515, 11796, 11107, 11139, 11123, - 129067, 11168, 11169, 129063, 129059, 129075, 129071, 11133, 11085, - 11117, 11143, 129171, 10507, 8609, 11247, 8681, 129175, 8623, 11147, - 11015, 8659, 8675, 129079, 10597, 10607, 10593, 10585, 8643, 10589, - 10581, 8642, 129091, 129095, 129087, 10225, 128623, 129107, 129083, 8650, - 128687, 128330, 128682, 129444, 8367, 10139, 128009, 128050, 128042, - 129656, 128167, 129316, 129345, 9946, 128087, 113784, 113788, 113785, - 113783, 113782, 113786, 113787, 113795, 113798, 113793, 113800, 113781, - 113794, 113792, 113799, 113797, 113796, 113776, 113808, 113817, 113811, - 113814, 113809, 113816, 113779, 113810, 113815, 113813, 113812, 113778, - 113777, 113780, 113729, 113733, 113672, 113677, 113683, 113735, 113739, - 113746, 113668, 113678, 113674, 113726, 113691, 113699, 113700, 113695, - 113709, 113712, 113713, 113705, 113711, 113669, 113725, 113679, 113684, - 113670, 113743, 113749, 113687, 113689, 113693, 113707, 113697, 113703, - 113690, 113694, 113708, 113698, 113704, 113764, 113763, 113762, 113761, - 113732, 113753, 113731, 113755, 113754, 113666, 113766, 113765, 113676, - 113675, 113741, 113750, 113680, 113688, 113692, 113696, 113710, 113727, - 113728, 113716, 113717, 113714, 113715, 113701, 113702, 113724, 113723, - 113706, 113742, 113740, 113767, 113769, 113730, 113768, 113682, 113685, - 113752, 113737, 113667, 113719, 113718, 113681, 113745, 113748, 113751, - 113738, 113673, 113770, 113720, 113757, 113760, 113722, 113759, 113756, - 113721, 113758, 113665, 113747, 113664, 113686, 113734, 113736, 113744, - 113671, 113821, 113822, 113823, 113820, 129375, 129414, 129516, 128192, - 983082, 128066, 127805, 127806, 129467, 9793, 127758, 127759, 127757, - 9178, 9841, 129413, 11790, 77830, 77831, 77832, 77828, 77829, 77824, - 77825, 77826, 77827, 77833, 77834, 77835, 77840, 77841, 77844, 77845, - 77836, 77837, 77838, 77839, 77842, 77843, 77846, 77847, 77869, 77870, - 77872, 77873, 77874, 77875, 77877, 77878, 77871, 77876, 77879, 77880, - 77881, 77882, 77860, 77861, 77858, 77859, 77862, 77863, 77864, 77865, - 77866, 77867, 77868, 77903, 77848, 77849, 77850, 77851, 77852, 77853, - 77854, 77855, 77856, 77857, 77883, 77884, 77885, 77886, 77887, 77888, - 77889, 77890, 77891, 77892, 77893, 77894, 77895, 77896, 77897, 77898, - 77899, 77900, 77901, 77902, 78867, 78868, 78869, 78861, 78862, 78863, - 78864, 78865, 78866, 78870, 78871, 78892, 78893, 78894, 78872, 78873, - 78874, 78875, 78876, 78877, 78878, 78879, 78880, 78881, 78882, 78883, - 78884, 78885, 78886, 78887, 78888, 78889, 78890, 78891, 78903, 78910, - 78908, 77908, 77909, 77904, 77905, 77906, 77907, 77910, 77911, 77912, - 77913, 77915, 77916, 77917, 77918, 77914, 77919, 77920, 77921, 77922, - 77923, 77924, 77925, 77926, 77927, 77928, 77929, 77930, 77931, 77932, - 77933, 77934, 77935, 77936, 77937, 77938, 77939, 77940, 77941, 77949, - 77950, 77942, 77943, 77944, 77945, 77946, 77947, 77948, 77951, 77974, - 77975, 77978, 77979, 77973, 77976, 77977, 77980, 77981, 77982, 77983, - 77984, 77991, 77992, 77994, 77995, 77985, 77986, 77987, 77988, 77989, - 77990, 77993, 77996, 77997, 77998, 77999, 78000, 78001, 78002, 78003, - 78004, 78005, 78006, 78008, 78009, 78011, 78012, 78007, 78010, 78013, - 78014, 78015, 78016, 78017, 78025, 78026, 78027, 78028, 78029, 78030, - 78031, 78032, 78033, 78018, 78019, 78020, 78021, 78022, 78023, 78024, - 77969, 77970, 77962, 77963, 77964, 77965, 77966, 77967, 77968, 77971, - 77972, 77952, 77953, 77954, 77955, 77956, 77957, 77958, 77959, 77960, - 77961, 78041, 78042, 78043, 78044, 78034, 78035, 78036, 78037, 78038, - 78039, 78040, 78057, 78058, 78066, 78067, 78059, 78060, 78061, 78062, - 78063, 78064, 78065, 78068, 78073, 78074, 78069, 78070, 78071, 78072, - 78075, 78076, 78077, 78051, 78052, 78053, 78054, 78045, 78046, 78047, - 78048, 78049, 78050, 78055, 78056, 78904, 78911, 78909, 78078, 78079, - 78080, 78081, 78082, 78083, 78084, 78085, 78086, 78087, 78091, 78092, - 78088, 78089, 78090, 78093, 78094, 78095, 78096, 78097, 78098, 78111, - 78112, 78118, 78119, 78120, 78121, 78110, 78113, 78114, 78115, 78116, - 78117, 78122, 78128, 78129, 78130, 78131, 78132, 78133, 78123, 78124, - 78125, 78126, 78127, 78134, 78135, 78137, 78138, 78139, 78140, 78136, - 78141, 78142, 78100, 78101, 78099, 78102, 78103, 78104, 78105, 78106, - 78107, 78108, 78109, 78913, 78150, 78151, 78152, 78148, 78149, 78143, - 78144, 78145, 78146, 78147, 78153, 78154, 78156, 78157, 78155, 78158, - 78159, 78160, 78161, 78162, 78163, 78164, 78165, 78184, 78185, 78186, - 78187, 78178, 78179, 78180, 78181, 78182, 78183, 78188, 78189, 78193, - 78194, 78196, 78197, 78190, 78191, 78192, 78195, 78198, 78199, 78200, - 78201, 78166, 78167, 78173, 78174, 78168, 78169, 78170, 78171, 78172, - 78175, 78176, 78177, 78202, 78203, 78204, 78205, 78206, 78212, 78213, - 78207, 78208, 78209, 78210, 78211, 78214, 78215, 78914, 78916, 78897, - 78220, 78221, 78225, 78226, 78216, 78217, 78218, 78219, 78222, 78223, - 78224, 78227, 78228, 78229, 78230, 78231, 78232, 78233, 78234, 78907, - 78901, 78899, 78906, 78900, 78898, 78905, 78244, 78245, 78249, 78250, - 78243, 78246, 78247, 78248, 78251, 78252, 78915, 78253, 78254, 78255, - 78257, 78258, 78256, 78259, 78260, 78261, 78262, 78263, 78264, 78268, - 78269, 78270, 78271, 78272, 78273, 78274, 78275, 78276, 78265, 78266, - 78279, 78280, 78281, 78282, 78283, 78284, 78267, 78277, 78278, 78285, - 78286, 78289, 78290, 78292, 78293, 78297, 78298, 78287, 78288, 78291, - 78294, 78295, 78296, 78299, 78304, 78305, 78306, 78301, 78302, 78300, - 78303, 78307, 78308, 78309, 78310, 78311, 78312, 78313, 78314, 78315, - 78316, 78317, 78318, 78933, 78928, 78920, 78924, 78932, 78926, 78921, - 78929, 78925, 78923, 78931, 78919, 78927, 78922, 78930, 78912, 78336, - 78337, 78338, 78328, 78329, 78330, 78331, 78332, 78333, 78334, 78335, - 78339, 78354, 78355, 78356, 78357, 78358, 78359, 78361, 78362, 78351, - 78352, 78353, 78360, 78363, 78364, 78345, 78346, 78340, 78341, 78342, - 78343, 78344, 78347, 78348, 78349, 78350, 78365, 78366, 78367, 78319, - 78320, 78321, 78322, 78323, 78324, 78325, 78326, 78327, 78372, 78373, - 78368, 78369, 78370, 78371, 78374, 78375, 78376, 78377, 78385, 78386, - 78378, 78379, 78380, 78381, 78382, 78383, 78384, 78387, 78388, 78389, - 78399, 78400, 78401, 78402, 78409, 78410, 78403, 78404, 78405, 78406, - 78407, 78408, 78411, 78414, 78415, 78412, 78413, 78390, 78391, 78392, - 78393, 78394, 78395, 78396, 78397, 78398, 78423, 78424, 78425, 78426, - 78427, 78428, 78429, 78416, 78417, 78421, 78422, 78418, 78419, 78420, - 78430, 78431, 78432, 78433, 78434, 78435, 78436, 78445, 78446, 78437, - 78438, 78439, 78440, 78441, 78442, 78443, 78444, 78447, 78448, 78452, - 78453, 78454, 78455, 78459, 78460, 78449, 78450, 78451, 78456, 78457, - 78458, 78469, 78470, 78471, 78472, 78473, 78461, 78462, 78465, 78466, - 78463, 78464, 78467, 78468, 78474, 78475, 78476, 78487, 78488, 78489, - 78490, 78477, 78478, 78479, 78480, 78481, 78482, 78483, 78484, 78485, - 78486, 78902, 78491, 78492, 78494, 78495, 78493, 78496, 78497, 78498, - 78499, 78500, 78501, 78502, 78503, 78514, 78515, 78516, 78512, 78513, - 78511, 78517, 78518, 78519, 78520, 78521, 78522, 78523, 78524, 78530, - 78531, 78525, 78526, 78527, 78528, 78529, 78532, 78533, 78534, 78535, - 78536, 78537, 78538, 78539, 78540, 78541, 78542, 78543, 78544, 78546, - 78547, 78551, 78552, 78545, 78548, 78549, 78550, 78553, 78554, 78555, - 78560, 78561, 78562, 78565, 78566, 78556, 78557, 78558, 78559, 78563, - 78564, 78567, 78568, 78575, 78576, 78577, 78569, 78570, 78571, 78572, - 78573, 78574, 78578, 78579, 78580, 78586, 78587, 78581, 78582, 78583, - 78584, 78585, 78588, 78589, 78590, 78591, 78592, 78593, 78594, 78595, - 78596, 78597, 78598, 78601, 78602, 78606, 78607, 78608, 78609, 78610, - 78611, 78599, 78600, 78603, 78604, 78605, 78613, 78614, 78619, 78620, - 78612, 78615, 78616, 78617, 78618, 78621, 78622, 78623, 78636, 78637, - 78638, 78639, 78634, 78635, 78640, 78641, 78642, 78624, 78625, 78626, - 78627, 78628, 78629, 78630, 78631, 78632, 78633, 78917, 78648, 78649, - 78650, 78643, 78644, 78645, 78646, 78647, 78651, 78652, 78653, 78667, - 78668, 78674, 78675, 78664, 78665, 78666, 78669, 78670, 78671, 78672, - 78673, 78678, 78679, 78676, 78677, 78680, 78681, 78682, 78683, 78684, - 78685, 78686, 78687, 78688, 78689, 78654, 78655, 78656, 78657, 78658, - 78659, 78660, 78661, 78662, 78663, 78706, 78707, 78708, 78690, 78691, - 78692, 78693, 78694, 78695, 78696, 78697, 78698, 78699, 78700, 78701, - 78702, 78703, 78704, 78705, 78709, 78710, 78712, 78713, 78714, 78715, - 78895, 78716, 78717, 78718, 78711, 78719, 78720, 78721, 78722, 78723, - 78724, 78725, 78726, 78727, 78728, 78729, 78730, 78731, 78732, 78733, - 78734, 78735, 78736, 78737, 78738, 78741, 78742, 78747, 78748, 78749, - 78750, 78739, 78740, 78743, 78744, 78745, 78746, 78751, 78752, 78753, - 78754, 78756, 78757, 78761, 78762, 78755, 78758, 78759, 78760, 78763, - 78764, 78765, 78766, 78896, 78769, 78770, 78776, 78777, 78767, 78768, - 78771, 78772, 78773, 78774, 78775, 78778, 78779, 78783, 78784, 78787, - 78788, 78789, 78790, 78780, 78781, 78782, 78785, 78786, 78791, 78796, - 78797, 78792, 78793, 78794, 78795, 78798, 78918, 78802, 78803, 78804, - 78806, 78807, 78809, 78810, 78799, 78800, 78801, 78805, 78808, 78811, - 78812, 78813, 78814, 78815, 78816, 78817, 78818, 78819, 78821, 78822, - 78823, 78824, 78825, 78826, 78827, 78828, 78829, 78830, 78831, 78832, - 78820, 78833, 78834, 78835, 78836, 78842, 78843, 78844, 78845, 78846, - 78847, 78848, 78849, 78850, 78851, 78852, 78853, 78854, 78855, 78856, - 78857, 78858, 78859, 78860, 78837, 78838, 78839, 78840, 78841, 78235, - 78236, 78237, 78238, 78239, 78240, 78241, 78242, 78504, 78505, 78506, - 78507, 78508, 78509, 78510, 129370, 10037, 10039, 10036, 10049, 10058, - 10035, 9834, 66854, 66853, 66827, 66826, 66833, 66832, 66821, 66837, - 66836, 66835, 66842, 66841, 66824, 66823, 66819, 66818, 66822, 66820, - 66855, 66831, 66844, 66843, 66846, 66845, 66852, 66851, 66817, 66825, - 66828, 66830, 66834, 66839, 66840, 66848, 66849, 66816, 66829, 66838, - 66847, 66850, 128268, 128294, 128161, 8961, 9191, 8712, 8946, 8953, 8947, - 8952, 8950, 8949, 10969, 10194, 128024, 128727, 69608, 69621, 69603, - 69611, 69618, 69619, 69600, 69615, 69602, 69606, 69614, 69617, 69620, - 69609, 69604, 69607, 69610, 69601, 69613, 69605, 69616, 69612, 69622, - 129501, 983101, 129456, 129457, 129459, 129458, 127995, 127996, 127997, - 127998, 127999, 128453, 128454, 128455, 129721, 128460, 128461, 8709, - 10675, 10676, 10674, 10673, 128459, 9091, 8193, 8212, 8195, 8192, 8211, - 8194, 983179, 8718, 983178, 983135, 983099, 983048, 983095, 983046, - 983064, 128282, 983051, 983050, 9993, 128388, 128233, 9094, 983067, - 983100, 983049, 8926, 8927, 8925, 8924, 8797, 8917, 61, 10865, 10867, - 11072, 10609, 10723, 10724, 10926, 11257, 10871, 10854, 10862, 8789, - 10872, 8781, 8794, 10738, 10736, 10734, 10739, 10737, 10735, 11249, - 11248, 9003, 8998, 983105, 983104, 8494, 8793, 983136, 4957, 4959, 4958, - 4963, 4965, 4978, 4988, 4980, 4979, 4987, 4985, 4982, 4981, 4986, 4984, - 4983, 4968, 4966, 4964, 4960, 4999, 4998, 4711, 4997, 43816, 43819, - 43821, 43820, 43818, 43822, 43817, 4704, 4707, 4710, 11653, 4709, 4708, - 4706, 4705, 43808, 43811, 43813, 43812, 43810, 43814, 43809, 11704, - 11707, 11709, 11708, 11706, 11710, 11705, 11688, 11691, 11693, 11692, - 11690, 11694, 11689, 4904, 4907, 4910, 11664, 4909, 4908, 4911, 4906, - 4905, 4728, 4731, 4734, 11655, 4733, 4732, 4735, 4730, 4729, 43789, - 43788, 43787, 43786, 43790, 43785, 4856, 4859, 4862, 11661, 4861, 4860, - 4863, 4858, 4857, 43797, 43796, 43795, 43794, 43798, 43793, 4848, 4851, - 4854, 11660, 4853, 4852, 4855, 4850, 4849, 5003, 5002, 4943, 5001, 4936, - 4939, 4941, 4940, 4954, 4938, 4942, 4937, 4873, 124916, 124915, 124924, - 124923, 124910, 124909, 124926, 124925, 124922, 124921, 124920, 124919, - 124918, 124917, 124914, 124913, 124912, 124904, 11667, 4895, 11670, - 11669, 11668, 4888, 4891, 4893, 4892, 4890, 4894, 4889, 4768, 4771, 4774, - 11658, 4773, 4772, 4775, 4770, 4769, 4880, 4883, 4885, 4884, 4882, 11736, - 11739, 11741, 11740, 11738, 11742, 11737, 4872, 4875, 4878, 4879, 4877, - 4876, 4874, 124907, 124906, 4631, 124905, 124896, 124899, 124901, 124900, - 124898, 124902, 124897, 4624, 4627, 4629, 4628, 4626, 4630, 4625, 4608, - 4611, 4614, 4615, 4613, 4612, 4610, 4609, 4800, 4803, 4805, 4804, 4802, - 4792, 4795, 4797, 4796, 4794, 4798, 4793, 4784, 4787, 4789, 4788, 4786, - 11720, 11723, 11725, 11724, 11722, 11726, 11721, 4776, 4779, 4782, 4783, - 4781, 4780, 4778, 4777, 4995, 4994, 4639, 4993, 4632, 4635, 4638, 11649, - 4637, 4636, 4953, 4634, 4633, 4760, 4763, 4766, 11657, 4765, 4764, 4767, - 4762, 4761, 4752, 4755, 4758, 11656, 4757, 4756, 4759, 4754, 4753, 4912, - 4816, 4819, 4821, 4820, 4818, 4822, 4817, 4915, 4918, 11665, 4917, 4916, - 4919, 4914, 4913, 5007, 5006, 4951, 5005, 4944, 4947, 4950, 11666, 4949, - 4948, 4946, 4945, 4696, 4699, 4701, 4700, 4698, 4688, 4691, 4693, 4692, - 4690, 4694, 4689, 4680, 4683, 4685, 4684, 4682, 11712, 11715, 11717, - 11716, 11714, 11718, 11713, 4672, 4675, 4678, 4679, 4677, 4676, 4674, - 4673, 4648, 4651, 4654, 11650, 4653, 4652, 4655, 4952, 4650, 4649, 4661, - 4996, 5000, 4992, 5004, 4660, 4664, 4667, 4670, 11652, 4669, 4668, 4671, - 4666, 4665, 4640, 4643, 4645, 4644, 4647, 4642, 4646, 4641, 11680, 11683, - 11685, 11684, 11682, 11686, 11681, 4656, 4659, 4662, 11651, 4663, 4658, - 4657, 4896, 4899, 4902, 11663, 4901, 4900, 4903, 4898, 4897, 43781, - 43780, 43779, 43778, 43782, 43777, 4928, 4931, 4934, 4935, 4933, 4932, - 4930, 4929, 4920, 4923, 4925, 4924, 4927, 4922, 4926, 4921, 4720, 4723, - 4726, 11654, 4725, 4724, 4727, 4722, 4721, 4864, 4867, 4870, 11662, 4869, - 4868, 4871, 4866, 4865, 4616, 4619, 4622, 11648, 4621, 4620, 4623, 4618, - 4617, 4808, 4811, 4814, 4815, 4813, 4812, 4810, 4809, 4840, 4843, 4846, - 4847, 4845, 4844, 4842, 4841, 4744, 4747, 4749, 4748, 4746, 11728, 11731, - 11733, 11732, 11730, 11734, 11729, 4736, 4739, 4742, 4743, 4741, 4740, - 4738, 4737, 4832, 4835, 4837, 4836, 4839, 4834, 4838, 4833, 11696, 11699, - 11701, 11700, 11698, 11702, 11697, 4824, 4827, 4830, 11659, 4829, 4828, - 4831, 4826, 4825, 4712, 4715, 4717, 4716, 4719, 4714, 4718, 4713, 5009, - 5016, 5012, 5015, 5013, 5017, 5010, 5011, 5014, 5008, 4973, 4972, 4975, - 4974, 4971, 4970, 4977, 4969, 4976, 4962, 4967, 4961, 983096, 983047, - 127984, 127972, 8352, 8364, 8455, 8265, 33, 8761, 128942, 128954, 128948, - 128905, 128915, 128935, 128125, 1781, 1780, 1783, 1782, 1779, 1778, 1776, - 1785, 1777, 1784, 128529, 128065, 128083, 128064, 128231, 9167, 127794, - 983180, 128523, 128561, 129312, 128531, 129301, 128567, 129488, 128582, - 128558, 129326, 128560, 129762, 129320, 128581, 129395, 129763, 129402, - 128539, 128540, 128541, 128514, 129298, 129323, 128580, 128548, 129764, - 129396, 128566, 129318, 128134, 128536, 129401, 8507, 127981, 10540, - 10543, 9950, 127810, 129478, 128439, 128224, 128106, 9771, 127877, - 129498, 129718, 128552, 170, 9792, 127905, 9972, 129338, 8210, 8199, - 128193, 128452, 983107, 128253, 127902, 128293, 128658, 129519, 127879, - 127878, 129512, 9789, 127771, 127763, 8296, 129351, 128031, 127907, 9673, - 127845, 128074, 8281, 11821, 127953, 10765, 129407, 129747, 9189, 9971, - 129449, 128170, 9884, 10086, 9880, 8277, 127924, 128190, 128563, 129672, - 129712, 129359, 128760, 128389, 127787, 127745, 129709, 128448, 129462, - 128099, 127860, 127869, 11792, 10972, 983071, 8704, 8873, 129376, 10021, - 11156, 8280, 8283, 10019, 127808, 10018, 128966, 8732, 8197, 9970, + 74137, 983267, 74138, 74139, 74140, 74141, 74142, 74607, 74127, 74144, + 74146, 74147, 74145, 74152, 74153, 74150, 74151, 74148, 74149, 74154, + 74155, 74157, 74158, 74163, 74164, 74165, 74160, 74161, 74156, 74159, + 74162, 74143, 74166, 74167, 74168, 74169, 74170, 74171, 74172, 74175, + 74173, 74174, 74176, 74177, 74182, 74183, 74180, 74181, 74632, 74186, + 74184, 74185, 74188, 74190, 74189, 74187, 74194, 74195, 74193, 74191, + 74192, 74196, 74197, 74198, 74199, 74200, 74201, 74202, 74205, 74206, + 74207, 74208, 74204, 74209, 74211, 74210, 74212, 74213, 74215, 74214, + 74216, 74218, 74217, 74964, 74178, 74179, 74203, 74219, 74220, 74223, + 74224, 74221, 74222, 74966, 74967, 74974, 74973, 74972, 74969, 74970, + 74971, 74975, 74968, 74965, 74977, 74976, 74980, 74981, 74982, 74984, + 74985, 74978, 74979, 74983, 74986, 74987, 74988, 74989, 74990, 74991, + 74993, 74997, 74996, 74998, 74995, 74994, 74992, 74999, 75000, 75003, + 75004, 75005, 75006, 75001, 75002, 75009, 75015, 75016, 75019, 75017, + 75018, 75013, 75012, 75010, 75011, 75014, 75021, 75030, 75029, 75027, + 75024, 75025, 75028, 75022, 75026, 75023, 75008, 75020, 75031, 75032, + 75007, 74226, 74227, 74228, 74229, 74230, 74225, 74231, 74233, 74234, + 74232, 74235, 74237, 74258, 74259, 75033, 74261, 74633, 74260, 74240, + 74634, 74241, 74243, 75035, 74246, 74247, 74245, 74248, 74249, 74250, + 74251, 75036, 75037, 74254, 74255, 74635, 74256, 75038, 74242, 74252, + 74253, 75034, 74238, 74239, 74244, 74257, 74263, 74265, 74264, 74266, + 74269, 74270, 74271, 74236, 74262, 74267, 74268, 74272, 74273, 74274, + 74278, 74279, 74275, 74276, 74277, 74280, 74281, 74636, 74282, 75039, + 74283, 74284, 74290, 74294, 74295, 75041, 75040, 74292, 74293, 74291, + 74296, 74297, 74298, 74299, 74300, 74637, 74301, 74286, 74287, 74285, + 74289, 74288, 74302, 74304, 74307, 74305, 74306, 74308, 74310, 74309, + 74311, 74303, 74638, 74312, 74314, 74313, 74315, 74316, 74319, 74321, + 74320, 74639, 74322, 74324, 74325, 74323, 74642, 75043, 74326, 75044, + 75046, 75047, 74327, 75048, 74329, 74328, 75049, 74330, 74332, 74333, + 74331, 75050, 75051, 74334, 75052, 74335, 75042, 74641, 75045, 74640, + 74317, 74336, 74318, 74337, 74338, 983266, 983265, 74643, 74339, 74347, + 74348, 74342, 74343, 74341, 74344, 74340, 74346, 74345, 74349, 74355, + 74354, 74350, 74352, 74358, 74359, 74353, 74357, 74351, 74356, 74360, + 74361, 74362, 74363, 74364, 74365, 74366, 74644, 74367, 74375, 74376, + 74368, 74369, 74373, 74374, 74370, 74371, 74372, 74377, 74378, 74379, + 74380, 74381, 74382, 74645, 74383, 74384, 74385, 74386, 74387, 74389, + 74408, 75053, 74388, 74646, 74395, 74394, 75055, 74400, 74399, 75056, + 74401, 74406, 74402, 74403, 74404, 74405, 74391, 74392, 74396, 74398, + 75054, 74397, 74393, 74390, 74407, 74409, 74410, 74411, 74412, 74413, + 74414, 74421, 74422, 74419, 74417, 74420, 74416, 74418, 74415, 74423, + 75057, 74424, 74425, 74426, 75058, 74428, 74429, 75059, 75060, 75061, + 74427, 74432, 74434, 74433, 74430, 74431, 74435, 74437, 74436, 74438, + 74441, 74440, 74444, 74445, 74446, 74448, 74447, 74443, 74449, 74442, + 74439, 74451, 74453, 74452, 74450, 74454, 74455, 74456, 74457, 75063, + 75062, 74458, 74459, 75064, 74460, 74461, 74462, 74463, 74465, 74464, + 74466, 74468, 74469, 74471, 74472, 74473, 74474, 74467, 74470, 74475, + 74477, 74478, 74479, 74476, 74480, 74481, 74482, 74483, 74488, 74486, + 74487, 74485, 74489, 74484, 74490, 75065, 74491, 74494, 74497, 74499, + 74500, 74498, 74495, 74647, 74496, 74501, 74504, 75066, 75067, 74505, + 74506, 74502, 74503, 74492, 74493, 74507, 74510, 74512, 74511, 74649, + 74509, 74508, 74515, 74516, 74522, 74523, 74519, 74520, 74517, 74518, + 74521, 74524, 74534, 74535, 74525, 74526, 74648, 74527, 74528, 74529, + 74531, 74532, 74533, 74530, 74536, 74538, 74537, 74539, 75068, 74540, + 74541, 74542, 74545, 74546, 74547, 75069, 74544, 74543, 74549, 74550, + 74551, 74552, 74553, 75070, 74555, 74556, 74558, 74557, 74559, 74560, + 74562, 74564, 74563, 75072, 74566, 75071, 74570, 74569, 74572, 74574, + 74573, 74554, 74567, 74571, 74565, 74561, 74568, 74575, 74576, 74548, + 74577, 74581, 74579, 74580, 74578, 74584, 74583, 74582, 74586, 74587, + 74588, 74585, 74513, 74514, 74589, 74591, 74590, 74593, 75073, 74592, + 74595, 74598, 74599, 74596, 74601, 74597, 74600, 74602, 75074, 74603, + 75075, 74604, 74605, 74606, 74594, 9982, 129380, 11232, 129473, 8911, + 8910, 10160, 9130, 129356, 128177, 164, 127835, 10081, 128707, 127854, + 129362, 129385, 67594, 67595, 67596, 67597, 67598, 67599, 67600, 67601, + 67602, 67603, 67604, 67605, 67606, 67607, 67608, 67609, 67610, 67611, + 67612, 67613, 67614, 67615, 67616, 67617, 67618, 67619, 67620, 67621, + 67622, 67623, 67624, 67625, 67626, 67627, 67628, 67629, 67630, 67631, + 67632, 67633, 67634, 67635, 67636, 67637, 67589, 67592, 67644, 67647, + 67639, 67640, 67584, 67585, 67586, 67587, 67588, 77712, 77713, 77714, + 77715, 77716, 77717, 77718, 77719, 77722, 77723, 77720, 77721, 77724, + 77725, 77726, 77727, 77728, 77729, 77730, 77731, 77732, 77733, 77734, + 77735, 77736, 77737, 77738, 77739, 77740, 77741, 77742, 77743, 77744, + 77745, 77746, 77747, 77748, 77749, 77750, 77751, 77752, 77753, 77754, + 77755, 77756, 77757, 77758, 77773, 77774, 77768, 77769, 77770, 77771, + 77772, 77775, 77776, 77777, 77788, 77789, 77790, 77791, 77792, 77793, + 77794, 77795, 77796, 77759, 77760, 77761, 77762, 77763, 77764, 77765, + 77766, 77767, 77778, 77779, 77780, 77781, 77782, 77783, 77784, 77785, + 77786, 77787, 77797, 77798, 77799, 77800, 77801, 77802, 77803, 77804, + 77805, 77806, 77807, 77808, 77809, 77810, 1126, 1033, 1300, 42574, 1034, + 1055, 1190, 1316, 1136, 1146, 42564, 42592, 42580, 1296, 1302, 1058, + 42634, 1196, 1035, 42640, 42638, 1062, 42642, 7305, 42636, 1059, 1266, + 1264, 1262, 1144, 1028, 1040, 1234, 1232, 1212, 1214, 1248, 1192, 1310, + 1256, 1258, 1184, 42602, 1130, 42572, 42586, 1030, 1041, 1063, 1268, + 1206, 1208, 42584, 42650, 42630, 1026, 42568, 42604, 42648, 1029, 42562, + 1322, 42632, 1039, 42626, 1324, 42624, 1044, 1069, 1051, 1312, 1326, + 1221, 1298, 1053, 1314, 1186, 1320, 1225, 1223, 1056, 1166, 1260, 1057, + 1194, 1052, 1229, 1060, 1043, 1172, 1168, 1270, 1170, 1274, 1027, 1061, + 1202, 1276, 1278, 1066, 42644, 1048, 1252, 1037, 1250, 1045, 1024, 1238, + 1025, 42588, 1128, 1132, 42578, 42582, 1124, 42566, 1140, 1142, 1050, + 1178, 1219, 1180, 1182, 1286, 1282, 1280, 1288, 1290, 1292, 1294, 1284, + 1152, 1227, 1036, 1134, 42600, 42570, 1054, 1120, 1148, 1254, 1150, 1240, + 1242, 1049, 1162, 1038, 1210, 1318, 1065, 42646, 1064, 42596, 42598, + 1068, 42594, 1198, 1200, 1164, 1071, 1304, 1122, 1067, 1272, 42576, 1031, + 42590, 1070, 1047, 1246, 1176, 42560, 1046, 1244, 1174, 1217, 42628, + 1138, 1032, 1042, 1308, 1306, 1236, 1204, 1188, 42622, 42606, 1216, 7467, + 42623, 1072, 1235, 1233, 1213, 1215, 1249, 1193, 1311, 1257, 1259, 1185, + 42603, 1131, 42573, 42587, 1110, 1073, 1095, 1269, 1207, 1209, 42585, + 42651, 42631, 1106, 42569, 42605, 42649, 1109, 42563, 1323, 42633, 1119, + 42627, 1325, 42625, 1076, 1101, 1083, 1313, 1327, 1222, 1299, 1085, 1315, + 1187, 1321, 1226, 1224, 1088, 1167, 1261, 1089, 1195, 1084, 1230, 1092, + 1075, 1173, 1169, 1271, 1171, 1275, 1107, 1093, 1203, 1277, 1279, 1098, + 42645, 1080, 1253, 1117, 1251, 1077, 1104, 1239, 1105, 42589, 1129, 1133, + 42579, 42583, 1125, 42567, 1141, 1143, 1082, 1179, 1220, 1181, 1183, + 1287, 1283, 1281, 1289, 1291, 1293, 1295, 1285, 1153, 1228, 1116, 1135, + 1127, 7297, 1113, 1301, 42601, 42571, 42575, 7298, 1114, 1086, 1121, + 1149, 1255, 1151, 1087, 1191, 1317, 1231, 1137, 42565, 42593, 42581, + 1297, 1147, 7296, 1303, 1241, 1243, 1081, 1163, 1118, 1211, 1319, 1097, + 42647, 1096, 42597, 42599, 1100, 42595, 1199, 1201, 1165, 7302, 7303, + 7300, 1090, 42635, 1197, 1115, 42641, 42639, 1094, 7301, 42643, 7306, + 42637, 1091, 1267, 1265, 1263, 1145, 1108, 7304, 7299, 1309, 1103, 1305, + 1123, 1099, 1273, 42577, 1111, 42591, 1102, 1079, 1247, 1177, 42561, + 1078, 1245, 1175, 1218, 42629, 1139, 1112, 1074, 1307, 1237, 1205, 1189, + 122984, 122962, 122986, 122985, 122965, 122976, 122971, 122974, 122964, + 122983, 122977, 122981, 122982, 122980, 122978, 122967, 122968, 122969, + 122966, 122979, 122973, 122963, 122970, 122961, 122972, 122975, 1154, + 9005, 127744, 983201, 983188, 983172, 8224, 11830, 11831, 128481, 128131, + 127841, 128374, 9619, 11843, 128168, 10143, 65101, 65097, 8504, 983081, + 983084, 983086, 983088, 983090, 983162, 9110, 9192, 127795, 128475, 8451, + 176, 8457, 983120, 128666, 8796, 983119, 9161, 9159, 9153, 9156, 9162, + 9160, 9154, 9157, 9164, 9151, 9163, 9150, 9158, 9152, 9155, 117829, + 117828, 127980, 9739, 66589, 66591, 66596, 66597, 66587, 66585, 66594, + 66595, 66593, 66599, 66562, 66563, 66564, 66565, 66561, 66560, 66568, + 66569, 66570, 66571, 66567, 66566, 66598, 66573, 66588, 66579, 66592, + 66590, 66581, 66578, 66580, 66582, 66577, 66586, 66575, 66584, 66583, + 66574, 66572, 66576, 66629, 66631, 66636, 66637, 66627, 66625, 66634, + 66635, 66633, 66639, 66602, 66603, 66604, 66605, 66601, 66600, 66608, + 66609, 66610, 66611, 66607, 66606, 66638, 66613, 66628, 66619, 66632, + 66630, 66621, 66618, 66620, 66622, 66617, 66626, 66615, 66624, 66623, + 66614, 66612, 66616, 127964, 127965, 128421, 128468, 2388, 2416, 43258, + 2387, 43257, 72448, 72449, 43259, 2309, 2310, 2320, 2324, 2421, 43262, + 2330, 2418, 2317, 2321, 2331, 2396, 2430, 2338, 2337, 2343, 2342, 2429, + 2394, 2328, 2427, 2327, 2426, 2361, 2350, 2424, 2308, 2318, 2322, 2358, + 2359, 2360, 2323, 2420, 2419, 2431, 2349, 2348, 2393, 2326, 2325, 2397, + 2353, 2352, 2399, 2351, 2313, 2314, 2423, 2422, 2345, 2339, 2329, 2334, + 2344, 2333, 2428, 2332, 2356, 2355, 2354, 2336, 2335, 2341, 2340, 2315, + 2400, 2316, 2401, 2357, 2311, 2312, 2347, 2346, 2425, 2395, 2398, 2392, + 2319, 983644, 983643, 983646, 983647, 983649, 983648, 983642, 983645, + 72450, 72451, 72452, 72453, 2305, 43255, 43251, 43254, 43253, 72455, + 72454, 72456, 43250, 43260, 2304, 72457, 2364, 2365, 2306, 43252, 43256, + 2417, 2381, 2307, 2386, 2385, 2366, 2376, 2380, 2383, 43263, 2389, 2373, + 2377, 2379, 2363, 2362, 2382, 2369, 2370, 2391, 2390, 2374, 2378, 2371, + 2372, 2402, 2403, 2367, 2368, 2375, 2405, 2404, 2411, 2410, 2413, 2412, + 2409, 2408, 2406, 2415, 2407, 2414, 43261, 2384, 983089, 983161, 983087, + 983085, 983083, 127962, 129487, 129420, 11033, 11032, 11030, 11031, + 128924, 128160, 8900, 8960, 168, 117827, 9856, 9857, 9858, 9859, 9860, + 9861, 128754, 53, 127238, 9356, 52, 127237, 9355, 57, 127242, 9360, 49, + 127234, 9352, 55, 127240, 9358, 54, 127239, 9357, 51, 127236, 9354, 50, + 127235, 9353, 48, 127233, 127232, 56, 127241, 9359, 119557, 119556, + 119555, 9868, 9871, 9870, 9869, 119553, 119554, 10131, 10126, 10125, + 10128, 10127, 10124, 10123, 127244, 10130, 10122, 10129, 10111, 10106, + 10105, 10108, 10107, 10104, 10103, 10110, 10102, 10109, 10121, 10116, + 10115, 10118, 10117, 10114, 10113, 127243, 10120, 10112, 10119, 9107, + 127919, 128549, 128542, 9933, 9090, 129400, 72004, 72021, 72020, 72023, + 72022, 72019, 72018, 72016, 72025, 72017, 72024, 71964, 71958, 71963, + 71974, 71973, 71936, 71937, 71961, 71960, 71966, 71965, 71940, 71941, + 71938, 71939, 71982, 71976, 71952, 71962, 71957, 71967, 71978, 71979, + 71980, 71971, 71970, 71954, 71953, 71951, 71950, 71949, 71948, 71969, + 71968, 71981, 71955, 71972, 71975, 71977, 71983, 71942, 71945, 71997, + 71996, 72003, 71995, 71984, 71991, 71987, 71988, 71985, 71986, 71989, + 71992, 71998, 72002, 72000, 72005, 72006, 71999, 72001, 8725, 247, 8903, + 129343, 8739, 9902, 129684, 128171, 128565, 12291, 8783, 9009, 128462, + 128441, 128442, 128443, 8939, 8941, 8716, 8740, 10990, 8832, 8928, 8876, + 8833, 8929, 8878, 128021, 71680, 71681, 71687, 71689, 71703, 71702, + 71708, 71707, 71723, 71716, 71701, 71700, 71706, 71705, 71684, 71685, + 71682, 71683, 71694, 71704, 71699, 71709, 71719, 71720, 71721, 71713, + 71712, 71696, 71695, 71693, 71692, 71698, 71697, 71691, 71690, 71711, + 71710, 71722, 71717, 71714, 71718, 71715, 71686, 71688, 71729, 71730, + 71724, 71732, 71734, 71727, 71728, 71725, 71726, 71731, 71733, 71739, + 71738, 71735, 71737, 71736, 128054, 128044, 36, 127025, 127026, 127027, + 127028, 127029, 127030, 127031, 127032, 127033, 127034, 127035, 127036, + 127037, 127038, 127039, 127040, 127041, 127042, 127043, 127044, 127045, + 127046, 127047, 127048, 127049, 127050, 127051, 127052, 127053, 127054, + 127055, 127056, 127057, 127058, 127059, 127060, 127061, 127062, 127063, + 127064, 127065, 127066, 127067, 127068, 127069, 127070, 127071, 127072, + 127073, 127024, 127075, 127076, 127077, 127078, 127079, 127080, 127081, + 127082, 127083, 127084, 127085, 127086, 127087, 127088, 127089, 127090, + 127091, 127092, 127093, 127094, 127095, 127096, 127097, 127098, 127099, + 127100, 127101, 127102, 127103, 127104, 127105, 127106, 127107, 127108, + 127109, 127110, 127111, 127112, 127113, 127114, 127115, 127116, 127117, + 127118, 127119, 127120, 127121, 127122, 127123, 127074, 129743, 8363, + 8724, 8760, 8901, 729, 9676, 8284, 11850, 11034, 129765, 11795, 11784, + 10649, 11798, 9470, 9465, 9464, 9467, 9466, 9463, 9462, 9469, 9461, 9468, + 10175, 10868, 10986, 8225, 8223, 11840, 8914, 8748, 11842, 8222, 8215, + 10835, 10836, 10645, 10913, 10915, 10914, 10939, 8243, 12318, 10746, + 10830, 10831, 11849, 10988, 11844, 10940, 8913, 8912, 11005, 10987, 8915, + 9208, 10981, 8875, 10979, 8214, 11799, 10646, 733, 8252, 8263, 65100, + 10908, 10907, 11002, 11001, 10906, 10905, 8510, 8473, 8511, 8450, 8461, + 8469, 8474, 8477, 8484, 8518, 8519, 8520, 8521, 8517, 8509, 8508, 8512, + 10719, 9890, 9891, 11260, 127849, 8868, 10993, 10623, 8945, 8964, 117764, + 128317, 117859, 118264, 117883, 118268, 117849, 128315, 117914, 117864, + 10728, 10729, 117875, 117879, 129288, 129290, 129289, 129291, 8595, 8629, + 8671, 129035, 129031, 129179, 129043, 129027, 129047, 8626, 8627, 10504, + 8693, 129975, 8615, 10515, 11796, 11015, 129203, 11147, 11107, 11139, + 11123, 129067, 11168, 11169, 129063, 129059, 129075, 129071, 11133, + 11085, 11117, 11143, 129171, 10507, 8609, 11247, 8681, 129175, 8623, + 8659, 8675, 129079, 10597, 10607, 10593, 10585, 8643, 10589, 10581, 8642, + 129091, 129095, 129087, 10225, 128623, 129107, 129083, 8650, 128687, + 128330, 128682, 129444, 8367, 10139, 128009, 128050, 128042, 129656, + 128167, 129316, 129345, 9946, 128087, 113784, 113788, 113785, 113783, + 113782, 113786, 113787, 113795, 113798, 113793, 113800, 113781, 113794, + 113792, 113799, 113797, 113796, 113776, 113808, 113817, 113811, 113814, + 113809, 113816, 113779, 113810, 113815, 113813, 113812, 113778, 113777, + 113780, 113729, 113733, 113672, 113677, 113683, 113735, 113739, 113746, + 113668, 113678, 113674, 113726, 113691, 113699, 113700, 113695, 113709, + 113712, 113713, 113705, 113711, 113669, 113725, 113679, 113684, 113670, + 113743, 113749, 113687, 113689, 113693, 113707, 113697, 113703, 113690, + 113694, 113708, 113698, 113704, 113764, 113763, 113762, 113761, 113732, + 113753, 113731, 113755, 113754, 113666, 113766, 113765, 113676, 113675, + 113741, 113750, 113680, 113688, 113692, 113696, 113710, 113727, 113728, + 113716, 113717, 113714, 113715, 113701, 113702, 113724, 113723, 113706, + 113742, 113740, 113767, 113769, 113730, 113768, 113682, 113685, 113752, + 113737, 113667, 113719, 113718, 113681, 113745, 113748, 113751, 113738, + 113673, 113770, 113720, 113757, 113760, 113722, 113759, 113756, 113721, + 113758, 113665, 113747, 113664, 113686, 113734, 113736, 113744, 113671, + 113821, 113822, 113823, 113820, 129375, 129414, 129516, 128192, 983082, + 128066, 127805, 127806, 129467, 9793, 127758, 127759, 127757, 9178, 9841, + 129413, 11790, 77830, 77831, 77832, 77828, 77829, 77824, 77825, 77826, + 77827, 77833, 77834, 77835, 77840, 77841, 77844, 77845, 77836, 77837, + 77838, 77839, 77842, 77843, 77846, 77847, 77869, 77870, 77872, 77873, + 77874, 77875, 77877, 77878, 77871, 77876, 77879, 77880, 77881, 77882, + 77860, 77861, 77858, 77859, 77862, 77863, 77864, 77865, 77866, 77867, + 77868, 77903, 77848, 77849, 77850, 77851, 77852, 77853, 77854, 77855, + 77856, 77857, 77883, 77884, 77885, 77886, 77887, 77888, 77889, 77890, + 77891, 77892, 77893, 77894, 77895, 77896, 77897, 77898, 77899, 77900, + 77901, 77902, 78867, 78868, 78869, 78861, 78862, 78863, 78864, 78865, + 78866, 78870, 78871, 78892, 78893, 78894, 78872, 78873, 78874, 78875, + 78876, 78877, 78878, 78879, 78880, 78881, 78882, 78883, 78884, 78885, + 78886, 78887, 78888, 78889, 78890, 78891, 78910, 78908, 78903, 77908, + 77909, 77904, 77905, 77906, 77907, 77910, 77911, 77912, 77913, 77915, + 77916, 77917, 77918, 77914, 77919, 77920, 77921, 77922, 77923, 77924, + 77925, 77926, 77927, 77928, 77929, 77930, 77931, 77932, 77933, 77934, + 77935, 77936, 77937, 77938, 77939, 77940, 77941, 77949, 77950, 77942, + 77943, 77944, 77945, 77946, 77947, 77948, 77951, 77974, 77975, 77978, + 77979, 77973, 77976, 77977, 77980, 77981, 77982, 77983, 77984, 77991, + 77992, 77994, 77995, 77985, 77986, 77987, 77988, 77989, 77990, 77993, + 77996, 77997, 77998, 77999, 78000, 78001, 78002, 78003, 78004, 78005, + 78006, 78008, 78009, 78011, 78012, 78007, 78010, 78013, 78014, 78015, + 78016, 78017, 78025, 78026, 78027, 78028, 78029, 78030, 78031, 78032, + 78033, 78018, 78019, 78020, 78021, 78022, 78023, 78024, 77969, 77970, + 77962, 77963, 77964, 77965, 77966, 77967, 77968, 77971, 77972, 77952, + 77953, 77954, 77955, 77956, 77957, 77958, 77959, 77960, 77961, 78041, + 78042, 78043, 78044, 78034, 78035, 78036, 78037, 78038, 78039, 78040, + 78057, 78058, 78066, 78067, 78059, 78060, 78061, 78062, 78063, 78064, + 78065, 78068, 78073, 78074, 78069, 78070, 78071, 78072, 78075, 78076, + 78077, 78051, 78052, 78053, 78054, 78045, 78046, 78047, 78048, 78049, + 78050, 78055, 78056, 78911, 78909, 78904, 78078, 78079, 78080, 78081, + 78082, 78083, 78084, 78085, 78086, 78087, 78091, 78092, 78088, 78089, + 78090, 78093, 78094, 78095, 78096, 78097, 78098, 78111, 78112, 78118, + 78119, 78120, 78121, 78110, 78113, 78114, 78115, 78116, 78117, 78122, + 78128, 78129, 78130, 78131, 78132, 78133, 78123, 78124, 78125, 78126, + 78127, 78134, 78135, 78137, 78138, 78139, 78140, 78136, 78141, 78142, + 78100, 78101, 78099, 78102, 78103, 78104, 78105, 78106, 78107, 78108, + 78109, 78913, 78150, 78151, 78152, 78148, 78149, 78143, 78144, 78145, + 78146, 78147, 78153, 78154, 78156, 78157, 78155, 78158, 78159, 78160, + 78161, 78162, 78163, 78164, 78165, 78184, 78185, 78186, 78187, 78178, + 78179, 78180, 78181, 78182, 78183, 78188, 78189, 78193, 78194, 78196, + 78197, 78190, 78191, 78192, 78195, 78198, 78199, 78200, 78201, 78166, + 78167, 78173, 78174, 78168, 78169, 78170, 78171, 78172, 78175, 78176, + 78177, 78202, 78203, 78204, 78205, 78206, 78212, 78213, 78207, 78208, + 78209, 78210, 78211, 78214, 78215, 78914, 78916, 78897, 78220, 78221, + 78225, 78226, 78216, 78217, 78218, 78219, 78222, 78223, 78224, 78227, + 78228, 78229, 78230, 78231, 78232, 78233, 78234, 78907, 78901, 78899, + 78906, 78900, 78898, 78905, 78244, 78245, 78249, 78250, 78243, 78246, + 78247, 78248, 78251, 78252, 78915, 78253, 78254, 78255, 78257, 78258, + 78256, 78259, 78260, 78261, 78262, 78263, 78264, 78268, 78269, 78270, + 78271, 78272, 78273, 78274, 78275, 78276, 78265, 78266, 78279, 78280, + 78281, 78282, 78283, 78284, 78267, 78277, 78278, 78285, 78286, 78289, + 78290, 78292, 78293, 78297, 78298, 78287, 78288, 78291, 78294, 78295, + 78296, 78299, 78304, 78305, 78306, 78301, 78302, 78300, 78303, 78307, + 78308, 78309, 78310, 78311, 78312, 78313, 78314, 78315, 78316, 78317, + 78318, 78933, 78928, 78920, 78924, 78932, 78926, 78921, 78929, 78925, + 78923, 78931, 78919, 78927, 78922, 78930, 78912, 78336, 78337, 78338, + 78328, 78329, 78330, 78331, 78332, 78333, 78334, 78335, 78339, 78354, + 78355, 78356, 78357, 78358, 78359, 78361, 78362, 78351, 78352, 78353, + 78360, 78363, 78364, 78345, 78346, 78340, 78341, 78342, 78343, 78344, + 78347, 78348, 78349, 78350, 78365, 78366, 78367, 78319, 78320, 78321, + 78322, 78323, 78324, 78325, 78326, 78327, 78372, 78373, 78368, 78369, + 78370, 78371, 78374, 78375, 78376, 78377, 78385, 78386, 78378, 78379, + 78380, 78381, 78382, 78383, 78384, 78387, 78388, 78389, 78399, 78400, + 78401, 78402, 78409, 78410, 78403, 78404, 78405, 78406, 78407, 78408, + 78411, 78414, 78415, 78412, 78413, 78390, 78391, 78392, 78393, 78394, + 78395, 78396, 78397, 78398, 78423, 78424, 78425, 78426, 78427, 78428, + 78429, 78416, 78417, 78421, 78422, 78418, 78419, 78420, 78430, 78431, + 78432, 78433, 78434, 78435, 78436, 78445, 78446, 78437, 78438, 78439, + 78440, 78441, 78442, 78443, 78444, 78447, 78448, 78452, 78453, 78454, + 78455, 78459, 78460, 78449, 78450, 78451, 78456, 78457, 78458, 78469, + 78470, 78471, 78472, 78473, 78461, 78462, 78465, 78466, 78463, 78464, + 78467, 78468, 78474, 78475, 78476, 78487, 78488, 78489, 78490, 78477, + 78478, 78479, 78480, 78481, 78482, 78483, 78484, 78485, 78486, 78902, + 78491, 78492, 78494, 78495, 78493, 78496, 78497, 78498, 78499, 78500, + 78501, 78502, 78503, 78514, 78515, 78516, 78512, 78513, 78511, 78517, + 78518, 78519, 78520, 78521, 78522, 78523, 78524, 78530, 78531, 78525, + 78526, 78527, 78528, 78529, 78532, 78533, 78534, 78535, 78536, 78537, + 78538, 78539, 78540, 78541, 78542, 78543, 78544, 78546, 78547, 78551, + 78552, 78545, 78548, 78549, 78550, 78553, 78554, 78555, 78560, 78561, + 78562, 78565, 78566, 78556, 78557, 78558, 78559, 78563, 78564, 78567, + 78568, 78575, 78576, 78577, 78569, 78570, 78571, 78572, 78573, 78574, + 78578, 78579, 78580, 78586, 78587, 78581, 78582, 78583, 78584, 78585, + 78588, 78589, 78590, 78591, 78592, 78593, 78594, 78595, 78596, 78597, + 78598, 78601, 78602, 78606, 78607, 78608, 78609, 78610, 78611, 78599, + 78600, 78603, 78604, 78605, 78613, 78614, 78619, 78620, 78612, 78615, + 78616, 78617, 78618, 78621, 78622, 78623, 78636, 78637, 78638, 78639, + 78634, 78635, 78640, 78641, 78642, 78624, 78625, 78626, 78627, 78628, + 78629, 78630, 78631, 78632, 78633, 78917, 78648, 78649, 78650, 78643, + 78644, 78645, 78646, 78647, 78651, 78652, 78653, 78667, 78668, 78674, + 78675, 78664, 78665, 78666, 78669, 78670, 78671, 78672, 78673, 78678, + 78679, 78676, 78677, 78680, 78681, 78682, 78683, 78684, 78685, 78686, + 78687, 78688, 78689, 78654, 78655, 78656, 78657, 78658, 78659, 78660, + 78661, 78662, 78663, 78706, 78707, 78708, 78690, 78691, 78692, 78693, + 78694, 78695, 78696, 78697, 78698, 78699, 78700, 78701, 78702, 78703, + 78704, 78705, 78709, 78710, 78712, 78713, 78714, 78715, 78895, 78716, + 78717, 78718, 78711, 78719, 78720, 78721, 78722, 78723, 78724, 78725, + 78726, 78727, 78728, 78729, 78730, 78731, 78732, 78733, 78734, 78735, + 78736, 78737, 78738, 78741, 78742, 78747, 78748, 78749, 78750, 78739, + 78740, 78743, 78744, 78745, 78746, 78751, 78752, 78753, 78754, 78756, + 78757, 78761, 78762, 78755, 78758, 78759, 78760, 78763, 78764, 78765, + 78766, 78896, 78769, 78770, 78776, 78777, 78767, 78768, 78771, 78772, + 78773, 78774, 78775, 78778, 78779, 78783, 78784, 78787, 78788, 78789, + 78790, 78780, 78781, 78782, 78785, 78786, 78791, 78796, 78797, 78792, + 78793, 78794, 78795, 78798, 78918, 78802, 78803, 78804, 78806, 78807, + 78809, 78810, 78799, 78800, 78801, 78805, 78808, 78811, 78812, 78813, + 78814, 78815, 78816, 78817, 78818, 78819, 78821, 78822, 78823, 78824, + 78825, 78826, 78827, 78828, 78829, 78830, 78831, 78832, 78820, 78833, + 78834, 78835, 78836, 78842, 78843, 78844, 78845, 78846, 78847, 78848, + 78849, 78850, 78851, 78852, 78853, 78854, 78855, 78856, 78857, 78858, + 78859, 78860, 78837, 78838, 78839, 78840, 78841, 78235, 78236, 78237, + 78238, 78239, 78240, 78241, 78242, 78504, 78505, 78506, 78507, 78508, + 78509, 78510, 78944, 78945, 78946, 78947, 78948, 78949, 78950, 78951, + 78952, 78953, 78954, 78955, 78956, 78957, 78958, 78959, 78960, 78961, + 78962, 78963, 78964, 78965, 78966, 78967, 78968, 78969, 78970, 78971, + 78972, 78973, 78974, 78975, 78976, 78977, 78978, 78979, 78980, 78981, + 78982, 78983, 78984, 78985, 78986, 78987, 78988, 78989, 78990, 78991, + 78992, 78993, 78994, 78995, 78996, 78997, 78998, 78999, 79000, 79001, + 79002, 79003, 79004, 79005, 79006, 79007, 79008, 79009, 79010, 79011, + 79012, 79013, 79014, 79015, 79016, 79017, 79018, 79019, 79020, 79021, + 79022, 79023, 79024, 79025, 79026, 79027, 79028, 79029, 79030, 79031, + 79032, 79033, 79034, 79035, 79036, 79037, 79038, 79039, 79040, 79041, + 79042, 79043, 79044, 79045, 79046, 79047, 79048, 79049, 79050, 79051, + 79052, 79053, 79054, 79055, 79056, 79057, 79058, 79059, 79060, 79061, + 79062, 79063, 79064, 79065, 79066, 79067, 79068, 79069, 79070, 79071, + 79072, 79073, 79074, 79075, 79076, 79077, 79078, 79079, 79080, 79081, + 79082, 79083, 79084, 79085, 79086, 79087, 79088, 79089, 79090, 79091, + 79092, 79093, 79094, 79095, 79096, 79097, 79098, 79099, 79100, 79101, + 79102, 79103, 79104, 79105, 79106, 79107, 79108, 79109, 79110, 79111, + 79112, 79113, 79114, 79115, 79116, 79117, 79118, 79119, 79120, 79121, + 79122, 79123, 79124, 79125, 79126, 79127, 79128, 79129, 79130, 79131, + 79132, 79133, 79134, 79135, 79136, 79137, 79138, 79139, 79140, 79141, + 79142, 79143, 79144, 79145, 79146, 79147, 79148, 79149, 79150, 79151, + 79152, 79153, 79154, 79155, 79156, 79157, 79158, 79159, 79160, 79161, + 79162, 79163, 79164, 79165, 79166, 79167, 79168, 79169, 79170, 79171, + 79172, 79173, 79174, 79175, 79176, 79177, 79178, 79179, 79180, 79181, + 79182, 79183, 79184, 79185, 79186, 79187, 79188, 79189, 79190, 79191, + 79192, 79193, 79194, 79195, 79196, 79197, 79198, 79199, 79200, 79201, + 79202, 79203, 79204, 79205, 79206, 79207, 79208, 79209, 79210, 79211, + 79212, 79213, 79214, 79215, 79216, 79217, 79218, 79219, 79220, 79221, + 79222, 79223, 79224, 79225, 79226, 79227, 79228, 79229, 79230, 79231, + 79232, 79233, 79234, 79235, 79236, 79237, 79238, 79239, 79240, 79241, + 79242, 79243, 79244, 79245, 79246, 79247, 79248, 79249, 79250, 79251, + 79252, 79253, 79254, 79255, 79256, 79257, 79258, 79259, 79260, 79261, + 79262, 79263, 79264, 79265, 79266, 79267, 79268, 79269, 79270, 79271, + 79272, 79273, 79274, 79275, 79276, 79277, 79278, 79279, 79280, 79281, + 79282, 79283, 79284, 79285, 79286, 79287, 79288, 79289, 79290, 79291, + 79292, 79293, 79294, 79295, 79296, 79297, 79298, 79299, 79300, 79301, + 79302, 79303, 79304, 79305, 79306, 79307, 79308, 79309, 79310, 79311, + 79312, 79313, 79314, 79315, 79316, 79317, 79318, 79319, 79320, 79321, + 79322, 79323, 79324, 79325, 79326, 79327, 79328, 79329, 79330, 79331, + 79332, 79333, 79334, 79335, 79336, 79337, 79338, 79339, 79340, 79341, + 79342, 79343, 79344, 79345, 79346, 79347, 79348, 79349, 79350, 79351, + 79352, 79353, 79354, 79355, 79356, 79357, 79358, 79359, 79360, 79361, + 79362, 79363, 79364, 79365, 79366, 79367, 79368, 79369, 79370, 79371, + 79372, 79373, 79374, 79375, 79376, 79377, 79378, 79379, 79380, 79381, + 79382, 79383, 79384, 79385, 79386, 79387, 79388, 79389, 79390, 79391, + 79392, 79393, 79394, 79395, 79396, 79397, 79398, 79399, 79400, 79401, + 79402, 79403, 79404, 79405, 79406, 79407, 79408, 79409, 79410, 79411, + 79412, 79413, 79414, 79415, 79416, 79417, 79418, 79419, 79420, 79421, + 79422, 79423, 79424, 79425, 79426, 79427, 79428, 79429, 79430, 79431, + 79432, 79433, 79434, 79435, 79436, 79437, 79438, 79439, 79440, 79441, + 79442, 79443, 79444, 79445, 79446, 79447, 79448, 79449, 79450, 79451, + 79452, 79453, 79454, 79455, 79456, 79457, 79458, 79459, 79460, 79461, + 79462, 79463, 79464, 79465, 79466, 79467, 79468, 79469, 79470, 79471, + 79472, 79473, 79474, 79475, 79476, 79477, 79478, 79479, 79480, 79481, + 79482, 79483, 79484, 79485, 79486, 79487, 79488, 79489, 79490, 79491, + 79492, 79493, 79494, 79495, 79496, 79497, 79498, 79499, 79500, 79501, + 79502, 79503, 79504, 79505, 79506, 79507, 79508, 79509, 79510, 79511, + 79512, 79513, 79514, 79515, 79516, 79517, 79518, 79519, 79520, 79521, + 79522, 79523, 79524, 79525, 79526, 79527, 79528, 79529, 79530, 79531, + 79532, 79533, 79534, 79535, 79536, 79537, 79538, 79539, 79540, 79541, + 79542, 79543, 79544, 79545, 79546, 79547, 79548, 79549, 79550, 79551, + 79552, 79553, 79554, 79555, 79556, 79557, 79558, 79559, 79560, 79561, + 79562, 79563, 79564, 79565, 79566, 79567, 79568, 79569, 79570, 79571, + 79572, 79573, 79574, 79575, 79576, 79577, 79578, 79579, 79580, 79581, + 79582, 79583, 79584, 79585, 79586, 79587, 79588, 79589, 79590, 79591, + 79592, 79593, 79594, 79595, 79596, 79597, 79598, 79599, 79600, 79601, + 79602, 79603, 79604, 79605, 79606, 79607, 79608, 79609, 79610, 79611, + 79612, 79613, 79614, 79615, 79616, 79617, 79618, 79619, 79620, 79621, + 79622, 79623, 79624, 79625, 79626, 79627, 79628, 79629, 79630, 79631, + 79632, 79633, 79634, 79635, 79636, 79637, 79638, 79639, 79640, 79641, + 79642, 79643, 79644, 79645, 79646, 79647, 79648, 79649, 79650, 79651, + 79652, 79653, 79654, 79655, 79656, 79657, 79658, 79659, 79660, 79661, + 79662, 79663, 79664, 79665, 79666, 79667, 79668, 79669, 79670, 79671, + 79672, 79673, 79674, 79675, 79676, 79677, 79678, 79679, 79680, 79681, + 79682, 79683, 79684, 79685, 79686, 79687, 79688, 79689, 79690, 79691, + 79692, 79693, 79694, 79695, 79696, 79697, 79698, 79699, 79700, 79701, + 79702, 79703, 79704, 79705, 79706, 79707, 79708, 79709, 79710, 79711, + 79712, 79713, 79714, 79715, 79716, 79717, 79718, 79719, 79720, 79721, + 79722, 79723, 79724, 79725, 79726, 79727, 79728, 79729, 79730, 79731, + 79732, 79733, 79734, 79735, 79736, 79737, 79738, 79739, 79740, 79741, + 79742, 79743, 79744, 79745, 79746, 79747, 79748, 79749, 79750, 79751, + 79752, 79753, 79754, 79755, 79756, 79757, 79758, 79759, 79760, 79761, + 79762, 79763, 79764, 79765, 79766, 79767, 79768, 79769, 79770, 79771, + 79772, 79773, 79774, 79775, 79776, 79777, 79778, 79779, 79780, 79781, + 79782, 79783, 79784, 79785, 79786, 79787, 79788, 79789, 79790, 79791, + 79792, 79793, 79794, 79795, 79796, 79797, 79798, 79799, 79800, 79801, + 79802, 79803, 79804, 79805, 79806, 79807, 79808, 79809, 79810, 79811, + 79812, 79813, 79814, 79815, 79816, 79817, 79818, 79819, 79820, 79821, + 79822, 79823, 79824, 79825, 79826, 79827, 79828, 79829, 79830, 79831, + 79832, 79833, 79834, 79835, 79836, 79837, 79838, 79839, 79840, 79841, + 79842, 79843, 79844, 79845, 79846, 79847, 79848, 79849, 79850, 79851, + 79852, 79853, 79854, 79855, 79856, 79857, 79858, 79859, 79860, 79861, + 79862, 79863, 79864, 79865, 79866, 79867, 79868, 79869, 79870, 79871, + 79872, 79873, 79874, 79875, 79876, 79877, 79878, 79879, 79880, 79881, + 79882, 79883, 79884, 79885, 79886, 79887, 79888, 79889, 79890, 79891, + 79892, 79893, 79894, 79895, 79896, 79897, 79898, 79899, 79900, 79901, + 79902, 79903, 79904, 79905, 79906, 79907, 79908, 79909, 79910, 79911, + 79912, 79913, 79914, 79915, 79916, 79917, 79918, 79919, 79920, 79921, + 79922, 79923, 79924, 79925, 79926, 79927, 79928, 79929, 79930, 79931, + 79932, 79933, 79934, 79935, 79936, 79937, 79938, 79939, 79940, 79941, + 79942, 79943, 79944, 79945, 79946, 79947, 79948, 79949, 79950, 79951, + 79952, 79953, 79954, 79955, 79956, 79957, 79958, 79959, 79960, 79961, + 79962, 79963, 79964, 79965, 79966, 79967, 79968, 79969, 79970, 79971, + 79972, 79973, 79974, 79975, 79976, 79977, 79978, 79979, 79980, 79981, + 79982, 79983, 79984, 79985, 79986, 79987, 79988, 79989, 79990, 79991, + 79992, 79993, 79994, 79995, 79996, 79997, 79998, 79999, 80000, 80001, + 80002, 80003, 80004, 80005, 80006, 80007, 80008, 80009, 80010, 80011, + 80012, 80013, 80014, 80015, 80016, 80017, 80018, 80019, 80020, 80021, + 80022, 80023, 80024, 80025, 80026, 80027, 80028, 80029, 80030, 80031, + 80032, 80033, 80034, 80035, 80036, 80037, 80038, 80039, 80040, 80041, + 80042, 80043, 80044, 80045, 80046, 80047, 80048, 80049, 80050, 80051, + 80052, 80053, 80054, 80055, 80056, 80057, 80058, 80059, 80060, 80061, + 80062, 80063, 80064, 80065, 80066, 80067, 80068, 80069, 80070, 80071, + 80072, 80073, 80074, 80075, 80076, 80077, 80078, 80079, 80080, 80081, + 80082, 80083, 80084, 80085, 80086, 80087, 80088, 80089, 80090, 80091, + 80092, 80093, 80094, 80095, 80096, 80097, 80098, 80099, 80100, 80101, + 80102, 80103, 80104, 80105, 80106, 80107, 80108, 80109, 80110, 80111, + 80112, 80113, 80114, 80115, 80116, 80117, 80118, 80119, 80120, 80121, + 80122, 80123, 80124, 80125, 80126, 80127, 80128, 80129, 80130, 80131, + 80132, 80133, 80134, 80135, 80136, 80137, 80138, 80139, 80140, 80141, + 80142, 80143, 80144, 80145, 80146, 80147, 80148, 80149, 80150, 80151, + 80152, 80153, 80154, 80155, 80156, 80157, 80158, 80159, 80160, 80161, + 80162, 80163, 80164, 80165, 80166, 80167, 80168, 80169, 80170, 80171, + 80172, 80173, 80174, 80175, 80176, 80177, 80178, 80179, 80180, 80181, + 80182, 80183, 80184, 80185, 80186, 80187, 80188, 80189, 80190, 80191, + 80192, 80193, 80194, 80195, 80196, 80197, 80198, 80199, 80200, 80201, + 80202, 80203, 80204, 80205, 80206, 80207, 80208, 80209, 80210, 80211, + 80212, 80213, 80214, 80215, 80216, 80217, 80218, 80219, 80220, 80221, + 80222, 80223, 80224, 80225, 80226, 80227, 80228, 80229, 80230, 80231, + 80232, 80233, 80234, 80235, 80236, 80237, 80238, 80239, 80240, 80241, + 80242, 80243, 80244, 80245, 80246, 80247, 80248, 80249, 80250, 80251, + 80252, 80253, 80254, 80255, 80256, 80257, 80258, 80259, 80260, 80261, + 80262, 80263, 80264, 80265, 80266, 80267, 80268, 80269, 80270, 80271, + 80272, 80273, 80274, 80275, 80276, 80277, 80278, 80279, 80280, 80281, + 80282, 80283, 80284, 80285, 80286, 80287, 80288, 80289, 80290, 80291, + 80292, 80293, 80294, 80295, 80296, 80297, 80298, 80299, 80300, 80301, + 80302, 80303, 80304, 80305, 80306, 80307, 80308, 80309, 80310, 80311, + 80312, 80313, 80314, 80315, 80316, 80317, 80318, 80319, 80320, 80321, + 80322, 80323, 80324, 80325, 80326, 80327, 80328, 80329, 80330, 80331, + 80332, 80333, 80334, 80335, 80336, 80337, 80338, 80339, 80340, 80341, + 80342, 80343, 80344, 80345, 80346, 80347, 80348, 80349, 80350, 80351, + 80352, 80353, 80354, 80355, 80356, 80357, 80358, 80359, 80360, 80361, + 80362, 80363, 80364, 80365, 80366, 80367, 80368, 80369, 80370, 80371, + 80372, 80373, 80374, 80375, 80376, 80377, 80378, 80379, 80380, 80381, + 80382, 80383, 80384, 80385, 80386, 80387, 80388, 80389, 80390, 80391, + 80392, 80393, 80394, 80395, 80396, 80397, 80398, 80399, 80400, 80401, + 80402, 80403, 80404, 80405, 80406, 80407, 80408, 80409, 80410, 80411, + 80412, 80413, 80414, 80415, 80416, 80417, 80418, 80419, 80420, 80421, + 80422, 80423, 80424, 80425, 80426, 80427, 80428, 80429, 80430, 80431, + 80432, 80433, 80434, 80435, 80436, 80437, 80438, 80439, 80440, 80441, + 80442, 80443, 80444, 80445, 80446, 80447, 80448, 80449, 80450, 80451, + 80452, 80453, 80454, 80455, 80456, 80457, 80458, 80459, 80460, 80461, + 80462, 80463, 80464, 80465, 80466, 80467, 80468, 80469, 80470, 80471, + 80472, 80473, 80474, 80475, 80476, 80477, 80478, 80479, 80480, 80481, + 80482, 80483, 80484, 80485, 80486, 80487, 80488, 80489, 80490, 80491, + 80492, 80493, 80494, 80495, 80496, 80497, 80498, 80499, 80500, 80501, + 80502, 80503, 80504, 80505, 80506, 80507, 80508, 80509, 80510, 80511, + 80512, 80513, 80514, 80515, 80516, 80517, 80518, 80519, 80520, 80521, + 80522, 80523, 80524, 80525, 80526, 80527, 80528, 80529, 80530, 80531, + 80532, 80533, 80534, 80535, 80536, 80537, 80538, 80539, 80540, 80541, + 80542, 80543, 80544, 80545, 80546, 80547, 80548, 80549, 80550, 80551, + 80552, 80553, 80554, 80555, 80556, 80557, 80558, 80559, 80560, 80561, + 80562, 80563, 80564, 80565, 80566, 80567, 80568, 80569, 80570, 80571, + 80572, 80573, 80574, 80575, 80576, 80577, 80578, 80579, 80580, 80581, + 80582, 80583, 80584, 80585, 80586, 80587, 80588, 80589, 80590, 80591, + 80592, 80593, 80594, 80595, 80596, 80597, 80598, 80599, 80600, 80601, + 80602, 80603, 80604, 80605, 80606, 80607, 80608, 80609, 80610, 80611, + 80612, 80613, 80614, 80615, 80616, 80617, 80618, 80619, 80620, 80621, + 80622, 80623, 80624, 80625, 80626, 80627, 80628, 80629, 80630, 80631, + 80632, 80633, 80634, 80635, 80636, 80637, 80638, 80639, 80640, 80641, + 80642, 80643, 80644, 80645, 80646, 80647, 80648, 80649, 80650, 80651, + 80652, 80653, 80654, 80655, 80656, 80657, 80658, 80659, 80660, 80661, + 80662, 80663, 80664, 80665, 80666, 80667, 80668, 80669, 80670, 80671, + 80672, 80673, 80674, 80675, 80676, 80677, 80678, 80679, 80680, 80681, + 80682, 80683, 80684, 80685, 80686, 80687, 80688, 80689, 80690, 80691, + 80692, 80693, 80694, 80695, 80696, 80697, 80698, 80699, 80700, 80701, + 80702, 80703, 80704, 80705, 80706, 80707, 80708, 80709, 80710, 80711, + 80712, 80713, 80714, 80715, 80716, 80717, 80718, 80719, 80720, 80721, + 80722, 80723, 80724, 80725, 80726, 80727, 80728, 80729, 80730, 80731, + 80732, 80733, 80734, 80735, 80736, 80737, 80738, 80739, 80740, 80741, + 80742, 80743, 80744, 80745, 80746, 80747, 80748, 80749, 80750, 80751, + 80752, 80753, 80754, 80755, 80756, 80757, 80758, 80759, 80760, 80761, + 80762, 80763, 80764, 80765, 80766, 80767, 80768, 80769, 80770, 80771, + 80772, 80773, 80774, 80775, 80776, 80777, 80778, 80779, 80780, 80781, + 80782, 80783, 80784, 80785, 80786, 80787, 80788, 80789, 80790, 80791, + 80792, 80793, 80794, 80795, 80796, 80797, 80798, 80799, 80800, 80801, + 80802, 80803, 80804, 80805, 80806, 80807, 80808, 80809, 80810, 80811, + 80812, 80813, 80814, 80815, 80816, 80817, 80818, 80819, 80820, 80821, + 80822, 80823, 80824, 80825, 80826, 80827, 80828, 80829, 80830, 80831, + 80832, 80833, 80834, 80835, 80836, 80837, 80838, 80839, 80840, 80841, + 80842, 80843, 80844, 80845, 80846, 80847, 80848, 80849, 80850, 80851, + 80852, 80853, 80854, 80855, 80856, 80857, 80858, 80859, 80860, 80861, + 80862, 80863, 80864, 80865, 80866, 80867, 80868, 80869, 80870, 80871, + 80872, 80873, 80874, 80875, 80876, 80877, 80878, 80879, 80880, 80881, + 80882, 80883, 80884, 80885, 80886, 80887, 80888, 80889, 80890, 80891, + 80892, 80893, 80894, 80895, 80896, 80897, 80898, 80899, 80900, 80901, + 80902, 80903, 80904, 80905, 80906, 80907, 80908, 80909, 80910, 80911, + 80912, 80913, 80914, 80915, 80916, 80917, 80918, 80919, 80920, 80921, + 80922, 80923, 80924, 80925, 80926, 80927, 80928, 80929, 80930, 80931, + 80932, 80933, 80934, 80935, 80936, 80937, 80938, 80939, 80940, 80941, + 80942, 80943, 80944, 80945, 80946, 80947, 80948, 80949, 80950, 80951, + 80952, 80953, 80954, 80955, 80956, 80957, 80958, 80959, 80960, 80961, + 80962, 80963, 80964, 80965, 80966, 80967, 80968, 80969, 80970, 80971, + 80972, 80973, 80974, 80975, 80976, 80977, 80978, 80979, 80980, 80981, + 80982, 80983, 80984, 80985, 80986, 80987, 80988, 80989, 80990, 80991, + 80992, 80993, 80994, 80995, 80996, 80997, 80998, 80999, 81000, 81001, + 81002, 81003, 81004, 81005, 81006, 81007, 81008, 81009, 81010, 81011, + 81012, 81013, 81014, 81015, 81016, 81017, 81018, 81019, 81020, 81021, + 81022, 81023, 81024, 81025, 81026, 81027, 81028, 81029, 81030, 81031, + 81032, 81033, 81034, 81035, 81036, 81037, 81038, 81039, 81040, 81041, + 81042, 81043, 81044, 81045, 81046, 81047, 81048, 81049, 81050, 81051, + 81052, 81053, 81054, 81055, 81056, 81057, 81058, 81059, 81060, 81061, + 81062, 81063, 81064, 81065, 81066, 81067, 81068, 81069, 81070, 81071, + 81072, 81073, 81074, 81075, 81076, 81077, 81078, 81079, 81080, 81081, + 81082, 81083, 81084, 81085, 81086, 81087, 81088, 81089, 81090, 81091, + 81092, 81093, 81094, 81095, 81096, 81097, 81098, 81099, 81100, 81101, + 81102, 81103, 81104, 81105, 81106, 81107, 81108, 81109, 81110, 81111, + 81112, 81113, 81114, 81115, 81116, 81117, 81118, 81119, 81120, 81121, + 81122, 81123, 81124, 81125, 81126, 81127, 81128, 81129, 81130, 81131, + 81132, 81133, 81134, 81135, 81136, 81137, 81138, 81139, 81140, 81141, + 81142, 81143, 81144, 81145, 81146, 81147, 81148, 81149, 81150, 81151, + 81152, 81153, 81154, 81155, 81156, 81157, 81158, 81159, 81160, 81161, + 81162, 81163, 81164, 81165, 81166, 81167, 81168, 81169, 81170, 81171, + 81172, 81173, 81174, 81175, 81176, 81177, 81178, 81179, 81180, 81181, + 81182, 81183, 81184, 81185, 81186, 81187, 81188, 81189, 81190, 81191, + 81192, 81193, 81194, 81195, 81196, 81197, 81198, 81199, 81200, 81201, + 81202, 81203, 81204, 81205, 81206, 81207, 81208, 81209, 81210, 81211, + 81212, 81213, 81214, 81215, 81216, 81217, 81218, 81219, 81220, 81221, + 81222, 81223, 81224, 81225, 81226, 81227, 81228, 81229, 81230, 81231, + 81232, 81233, 81234, 81235, 81236, 81237, 81238, 81239, 81240, 81241, + 81242, 81243, 81244, 81245, 81246, 81247, 81248, 81249, 81250, 81251, + 81252, 81253, 81254, 81255, 81256, 81257, 81258, 81259, 81260, 81261, + 81262, 81263, 81264, 81265, 81266, 81267, 81268, 81269, 81270, 81271, + 81272, 81273, 81274, 81275, 81276, 81277, 81278, 81279, 81280, 81281, + 81282, 81283, 81284, 81285, 81286, 81287, 81288, 81289, 81290, 81291, + 81292, 81293, 81294, 81295, 81296, 81297, 81298, 81299, 81300, 81301, + 81302, 81303, 81304, 81305, 81306, 81307, 81308, 81309, 81310, 81311, + 81312, 81313, 81314, 81315, 81316, 81317, 81318, 81319, 81320, 81321, + 81322, 81323, 81324, 81325, 81326, 81327, 81328, 81329, 81330, 81331, + 81332, 81333, 81334, 81335, 81336, 81337, 81338, 81339, 81340, 81341, + 81342, 81343, 81344, 81345, 81346, 81347, 81348, 81349, 81350, 81351, + 81352, 81353, 81354, 81355, 81356, 81357, 81358, 81359, 81360, 81361, + 81362, 81363, 81364, 81365, 81366, 81367, 81368, 81369, 81370, 81371, + 81372, 81373, 81374, 81375, 81376, 81377, 81378, 81379, 81380, 81381, + 81382, 81383, 81384, 81385, 81386, 81387, 81388, 81389, 81390, 81391, + 81392, 81393, 81394, 81395, 81396, 81397, 81398, 81399, 81400, 81401, + 81402, 81403, 81404, 81405, 81406, 81407, 81408, 81409, 81410, 81411, + 81412, 81413, 81414, 81415, 81416, 81417, 81418, 81419, 81420, 81421, + 81422, 81423, 81424, 81425, 81426, 81427, 81428, 81429, 81430, 81431, + 81432, 81433, 81434, 81435, 81436, 81437, 81438, 81439, 81440, 81441, + 81442, 81443, 81444, 81445, 81446, 81447, 81448, 81449, 81450, 81451, + 81452, 81453, 81454, 81455, 81456, 81457, 81458, 81459, 81460, 81461, + 81462, 81463, 81464, 81465, 81466, 81467, 81468, 81469, 81470, 81471, + 81472, 81473, 81474, 81475, 81476, 81477, 81478, 81479, 81480, 81481, + 81482, 81483, 81484, 81485, 81486, 81487, 81488, 81489, 81490, 81491, + 81492, 81493, 81494, 81495, 81496, 81497, 81498, 81499, 81500, 81501, + 81502, 81503, 81504, 81505, 81506, 81507, 81508, 81509, 81510, 81511, + 81512, 81513, 81514, 81515, 81516, 81517, 81518, 81519, 81520, 81521, + 81522, 81523, 81524, 81525, 81526, 81527, 81528, 81529, 81530, 81531, + 81532, 81533, 81534, 81535, 81536, 81537, 81538, 81539, 81540, 81541, + 81542, 81543, 81544, 81545, 81546, 81547, 81548, 81549, 81550, 81551, + 81552, 81553, 81554, 81555, 81556, 81557, 81558, 81559, 81560, 81561, + 81562, 81563, 81564, 81565, 81566, 81567, 81568, 81569, 81570, 81571, + 81572, 81573, 81574, 81575, 81576, 81577, 81578, 81579, 81580, 81581, + 81582, 81583, 81584, 81585, 81586, 81587, 81588, 81589, 81590, 81591, + 81592, 81593, 81594, 81595, 81596, 81597, 81598, 81599, 81600, 81601, + 81602, 81603, 81604, 81605, 81606, 81607, 81608, 81609, 81610, 81611, + 81612, 81613, 81614, 81615, 81616, 81617, 81618, 81619, 81620, 81621, + 81622, 81623, 81624, 81625, 81626, 81627, 81628, 81629, 81630, 81631, + 81632, 81633, 81634, 81635, 81636, 81637, 81638, 81639, 81640, 81641, + 81642, 81643, 81644, 81645, 81646, 81647, 81648, 81649, 81650, 81651, + 81652, 81653, 81654, 81655, 81656, 81657, 81658, 81659, 81660, 81661, + 81662, 81663, 81664, 81665, 81666, 81667, 81668, 81669, 81670, 81671, + 81672, 81673, 81674, 81675, 81676, 81677, 81678, 81679, 81680, 81681, + 81682, 81683, 81684, 81685, 81686, 81687, 81688, 81689, 81690, 81691, + 81692, 81693, 81694, 81695, 81696, 81697, 81698, 81699, 81700, 81701, + 81702, 81703, 81704, 81705, 81706, 81707, 81708, 81709, 81710, 81711, + 81712, 81713, 81714, 81715, 81716, 81717, 81718, 81719, 81720, 81721, + 81722, 81723, 81724, 81725, 81726, 81727, 81728, 81729, 81730, 81731, + 81732, 81733, 81734, 81735, 81736, 81737, 81738, 81739, 81740, 81741, + 81742, 81743, 81744, 81745, 81746, 81747, 81748, 81749, 81750, 81751, + 81752, 81753, 81754, 81755, 81756, 81757, 81758, 81759, 81760, 81761, + 81762, 81763, 81764, 81765, 81766, 81767, 81768, 81769, 81770, 81771, + 81772, 81773, 81774, 81775, 81776, 81777, 81778, 81779, 81780, 81781, + 81782, 81783, 81784, 81785, 81786, 81787, 81788, 81789, 81790, 81791, + 81792, 81793, 81794, 81795, 81796, 81797, 81798, 81799, 81800, 81801, + 81802, 81803, 81804, 81805, 81806, 81807, 81808, 81809, 81810, 81811, + 81812, 81813, 81814, 81815, 81816, 81817, 81818, 81819, 81820, 81821, + 81822, 81823, 81824, 81825, 81826, 81827, 81828, 81829, 81830, 81831, + 81832, 81833, 81834, 81835, 81836, 81837, 81838, 81839, 81840, 81841, + 81842, 81843, 81844, 81845, 81846, 81847, 81848, 81849, 81850, 81851, + 81852, 81853, 81854, 81855, 81856, 81857, 81858, 81859, 81860, 81861, + 81862, 81863, 81864, 81865, 81866, 81867, 81868, 81869, 81870, 81871, + 81872, 81873, 81874, 81875, 81876, 81877, 81878, 81879, 81880, 81881, + 81882, 81883, 81884, 81885, 81886, 81887, 81888, 81889, 81890, 81891, + 81892, 81893, 81894, 81895, 81896, 81897, 81898, 81899, 81900, 81901, + 81902, 81903, 81904, 81905, 81906, 81907, 81908, 81909, 81910, 81911, + 81912, 81913, 81914, 81915, 81916, 81917, 81918, 81919, 82928, 82929, + 82930, 82931, 82932, 82933, 82934, 82935, 82936, 82937, 82938, 82688, + 82689, 82690, 82691, 82692, 82693, 82694, 82695, 82696, 82697, 82698, + 82699, 82700, 82701, 82702, 82703, 82704, 82705, 82706, 82707, 82708, + 82709, 82710, 82711, 82712, 82713, 82714, 82715, 82716, 82717, 82718, + 82719, 82720, 82721, 82722, 82723, 82724, 82725, 82726, 82727, 82728, + 82729, 82730, 82731, 82732, 82733, 82734, 82735, 82736, 82737, 82738, + 82739, 82740, 82741, 82742, 82743, 82744, 82745, 82746, 82747, 82748, + 82749, 82750, 82751, 82752, 82753, 82754, 82755, 82756, 82757, 82758, + 82759, 82760, 82761, 82762, 82763, 82764, 82765, 82766, 82767, 82768, + 82769, 82770, 82771, 82772, 82773, 82774, 82775, 82776, 82777, 82778, + 82779, 82780, 82781, 82782, 82783, 82784, 82785, 82786, 82787, 82788, + 82789, 82790, 82791, 82792, 82793, 82794, 82795, 82796, 82797, 82798, + 82799, 82800, 82801, 82802, 82803, 82804, 82805, 82806, 82807, 82808, + 82809, 82810, 82811, 82812, 82813, 82814, 82815, 82816, 82817, 82818, + 82819, 82820, 82821, 82822, 82823, 82824, 82825, 82826, 82827, 82828, + 82829, 82830, 82831, 82832, 82833, 82834, 82835, 82836, 82837, 82838, + 82839, 82840, 82841, 82842, 82843, 82844, 82845, 82846, 82847, 82848, + 82849, 82850, 82851, 82852, 82853, 82854, 82855, 82856, 82857, 82858, + 82859, 82860, 82861, 82862, 82863, 82864, 82865, 82866, 82867, 82868, + 82869, 82870, 82871, 82872, 82873, 82874, 82875, 82876, 82877, 82878, + 82879, 82880, 82881, 82882, 82883, 82884, 82885, 82886, 82887, 82888, + 82889, 82890, 82891, 82892, 82893, 82894, 82895, 82896, 82897, 82898, + 82899, 82900, 82901, 82902, 82903, 82904, 82905, 82906, 82907, 82908, + 82909, 82910, 82911, 82912, 82913, 82914, 82915, 82916, 82917, 82918, + 82919, 82920, 82921, 82922, 82923, 82924, 82925, 82926, 82927, 81920, + 81921, 81922, 81923, 81924, 81925, 81926, 81927, 81928, 81929, 81930, + 81931, 81932, 81933, 81934, 81935, 81936, 81937, 81938, 81939, 81940, + 81941, 81942, 81943, 81944, 81945, 81946, 81947, 81948, 81949, 81950, + 81951, 81952, 81953, 81954, 81955, 81956, 81957, 81958, 81959, 81960, + 81961, 81962, 81963, 81964, 81965, 81966, 81967, 81968, 81969, 81970, + 81971, 81972, 81973, 81974, 81975, 81976, 81977, 81978, 81979, 81980, + 81981, 81982, 81983, 81984, 81985, 81986, 81987, 81988, 81989, 81990, + 81991, 81992, 81993, 81994, 81995, 81996, 81997, 81998, 81999, 82000, + 82001, 82002, 82003, 82004, 82005, 82006, 82007, 82008, 82009, 82010, + 82011, 82012, 82013, 82014, 82015, 82016, 82017, 82018, 82019, 82020, + 82021, 82022, 82023, 82024, 82025, 82026, 82027, 82028, 82029, 82030, + 82031, 82032, 82033, 82034, 82035, 82036, 82037, 82038, 82039, 82040, + 82041, 82042, 82043, 82044, 82045, 82046, 82047, 82048, 82049, 82050, + 82051, 82052, 82053, 82054, 82055, 82056, 82057, 82058, 82059, 82060, + 82061, 82062, 82063, 82064, 82065, 82066, 82067, 82068, 82069, 82070, + 82071, 82072, 82073, 82074, 82075, 82076, 82077, 82078, 82079, 82080, + 82081, 82082, 82083, 82084, 82085, 82086, 82087, 82088, 82089, 82090, + 82091, 82092, 82093, 82094, 82095, 82096, 82097, 82098, 82099, 82100, + 82101, 82102, 82103, 82104, 82105, 82106, 82107, 82108, 82109, 82110, + 82111, 82112, 82113, 82114, 82115, 82116, 82117, 82118, 82119, 82120, + 82121, 82122, 82123, 82124, 82125, 82126, 82127, 82128, 82129, 82130, + 82131, 82132, 82133, 82134, 82135, 82136, 82137, 82138, 82139, 82140, + 82141, 82142, 82143, 82144, 82145, 82146, 82147, 82148, 82149, 82150, + 82151, 82152, 82153, 82154, 82155, 82156, 82157, 82158, 82159, 82160, + 82161, 82162, 82163, 82164, 82165, 82166, 82167, 82168, 82169, 82170, + 82171, 82172, 82173, 82174, 82175, 82176, 82177, 82178, 82179, 82180, + 82181, 82182, 82183, 82184, 82185, 82186, 82187, 82188, 82189, 82190, + 82191, 82192, 82193, 82194, 82195, 82196, 82197, 82198, 82199, 82200, + 82201, 82202, 82203, 82204, 82205, 82206, 82207, 82208, 82209, 82210, + 82211, 82212, 82213, 82214, 82215, 82216, 82217, 82218, 82219, 82220, + 82221, 82222, 82223, 82224, 82225, 82226, 82227, 82228, 82229, 82230, + 82231, 82232, 82233, 82234, 82235, 82236, 82237, 82238, 82239, 82240, + 82241, 82242, 82243, 82244, 82245, 82246, 82247, 82248, 82249, 82250, + 82251, 82252, 82253, 82254, 82255, 82256, 82257, 82258, 82259, 82260, + 82261, 82262, 82263, 82264, 82265, 82266, 82267, 82268, 82269, 82270, + 82271, 82272, 82273, 82274, 82275, 82276, 82277, 82278, 82279, 82280, + 82281, 82282, 82283, 82284, 82285, 82286, 82287, 82288, 82289, 82290, + 82291, 82292, 82293, 82294, 82295, 82296, 82297, 82298, 82299, 82300, + 82301, 82302, 82303, 82304, 82305, 82306, 82307, 82308, 82309, 82310, + 82311, 82312, 82313, 82314, 82315, 82316, 82317, 82318, 82319, 82320, + 82321, 82322, 82323, 82324, 82325, 82326, 82327, 82328, 82329, 82330, + 82331, 82332, 82333, 82334, 82335, 82336, 82337, 82338, 82339, 82340, + 82341, 82342, 82343, 82344, 82345, 82346, 82347, 82348, 82349, 82350, + 82351, 82352, 82353, 82354, 82355, 82356, 82357, 82358, 82359, 82360, + 82361, 82362, 82363, 82364, 82365, 82366, 82367, 82368, 82369, 82370, + 82371, 82372, 82373, 82374, 82375, 82376, 82377, 82378, 82379, 82380, + 82381, 82382, 82383, 82384, 82385, 82386, 82387, 82388, 82389, 82390, + 82391, 82392, 82393, 82394, 82395, 82396, 82397, 82398, 82399, 82400, + 82401, 82402, 82403, 82404, 82405, 82406, 82407, 82408, 82409, 82410, + 82411, 82412, 82413, 82414, 82415, 82416, 82417, 82418, 82419, 82420, + 82421, 82422, 82423, 82424, 82425, 82426, 82427, 82428, 82429, 82430, + 82431, 82432, 82433, 82434, 82435, 82436, 82437, 82438, 82439, 82440, + 82441, 82442, 82443, 82444, 82445, 82446, 82447, 82448, 82449, 82450, + 82451, 82452, 82453, 82454, 82455, 82456, 82457, 82458, 82459, 82460, + 82461, 82462, 82463, 82464, 82465, 82466, 82467, 82468, 82469, 82470, + 82471, 82472, 82473, 82474, 82475, 82476, 82477, 82478, 82479, 82480, + 82481, 82482, 82483, 82484, 82485, 82486, 82487, 82488, 82489, 82490, + 82491, 82492, 82493, 82494, 82495, 82496, 82497, 82498, 82499, 82500, + 82501, 82502, 82503, 82504, 82505, 82506, 82507, 82508, 82509, 82510, + 82511, 82512, 82513, 82514, 82515, 82516, 82517, 82518, 82519, 82520, + 82521, 82522, 82523, 82524, 82525, 82526, 82527, 82528, 82529, 82530, + 82531, 82532, 82533, 82534, 82535, 82536, 82537, 82538, 82539, 82540, + 82541, 82542, 82543, 82544, 82545, 82546, 82547, 82548, 82549, 82550, + 82551, 82552, 82553, 82554, 82555, 82556, 82557, 82558, 82559, 82560, + 82561, 82562, 82563, 82564, 82565, 82566, 82567, 82568, 82569, 82570, + 82571, 82572, 82573, 82574, 82575, 82576, 82577, 82578, 82579, 82580, + 82581, 82582, 82583, 82584, 82585, 82586, 82587, 82588, 82589, 82590, + 82591, 82592, 82593, 82594, 82595, 82596, 82597, 82598, 82599, 82600, + 82601, 82602, 82603, 82604, 82605, 82606, 82607, 82608, 82609, 82610, + 82611, 82612, 82613, 82614, 82615, 82616, 82617, 82618, 82619, 82620, + 82621, 82622, 82623, 82624, 82625, 82626, 82627, 82628, 82629, 82630, + 82631, 82632, 82633, 82634, 82635, 82636, 82637, 82638, 82639, 82640, + 82641, 82642, 82643, 82644, 82645, 82646, 82647, 82648, 82649, 82650, + 82651, 82652, 82653, 82654, 82655, 82656, 82657, 82658, 82659, 82660, + 82661, 82662, 82663, 82664, 82665, 82666, 82667, 82668, 82669, 82670, + 82671, 82672, 82673, 82674, 82675, 82676, 82677, 82678, 82679, 82680, + 82681, 82682, 82683, 82684, 82685, 82686, 82687, 129370, 10037, 10039, + 10036, 10049, 117865, 117866, 10058, 10035, 9834, 66854, 66853, 66827, + 66826, 66833, 66832, 66821, 66837, 66836, 66835, 66842, 66841, 66824, + 66823, 66819, 66818, 66822, 66820, 66855, 66831, 66844, 66843, 66846, + 66845, 66852, 66851, 66817, 66825, 66828, 66830, 66834, 66839, 66840, + 66848, 66849, 66816, 66829, 66838, 66847, 66850, 128268, 128294, 128161, + 8961, 9191, 8712, 8946, 8953, 8947, 8952, 8950, 8949, 10969, 10194, + 128024, 128727, 69608, 69621, 69603, 69611, 69618, 69619, 69600, 69615, + 69602, 69606, 69614, 69617, 69620, 69609, 69604, 69607, 69610, 69601, + 69613, 69605, 69616, 69612, 69622, 129501, 983101, 129456, 129457, + 129459, 129458, 127995, 127996, 127997, 127998, 127999, 128453, 128454, + 128455, 129721, 128460, 128461, 8709, 10675, 10676, 10674, 10673, 128459, + 9091, 8193, 8212, 8195, 8192, 8211, 8194, 983179, 8718, 983178, 983135, + 983099, 983048, 983095, 983046, 983064, 128282, 983051, 983050, 9993, + 128388, 128233, 9094, 983067, 983100, 983049, 8926, 8927, 8925, 8924, + 8797, 8917, 61, 10865, 10867, 11072, 10609, 10723, 10724, 10926, 11257, + 10871, 10854, 10862, 8789, 10872, 8781, 8794, 10738, 10736, 10734, 10739, + 10737, 10735, 11249, 11248, 9003, 8998, 983105, 983104, 8494, 8793, + 983136, 4957, 4959, 4958, 4963, 4965, 4978, 4988, 4980, 4979, 4987, 4985, + 4982, 4981, 4986, 4984, 4983, 4968, 4966, 4964, 4960, 4999, 4998, 4711, + 4997, 43816, 43819, 43821, 43820, 43818, 43822, 43817, 4704, 4707, 4710, + 11653, 4709, 4708, 4706, 4705, 43808, 43811, 43813, 43812, 43810, 43814, + 43809, 11704, 11707, 11709, 11708, 11706, 11710, 11705, 11688, 11691, + 11693, 11692, 11690, 11694, 11689, 4904, 4907, 4910, 11664, 4909, 4908, + 4911, 4906, 4905, 4728, 4731, 4734, 11655, 4733, 4732, 4735, 4730, 4729, + 43789, 43788, 43787, 43786, 43790, 43785, 4856, 4859, 4862, 11661, 4861, + 4860, 4863, 4858, 4857, 43797, 43796, 43795, 43794, 43798, 43793, 4848, + 4851, 4854, 11660, 4853, 4852, 4855, 4850, 4849, 5003, 5002, 4943, 5001, + 4936, 4939, 4941, 4940, 4954, 4938, 4942, 4937, 4873, 124916, 124915, + 124924, 124923, 124910, 124909, 124926, 124925, 124922, 124921, 124920, + 124919, 124918, 124917, 124914, 124913, 124912, 124904, 11667, 4895, + 11670, 11669, 11668, 4888, 4891, 4893, 4892, 4890, 4894, 4889, 4768, + 4771, 4774, 11658, 4773, 4772, 4775, 4770, 4769, 4880, 4883, 4885, 4884, + 4882, 11736, 11739, 11741, 11740, 11738, 11742, 11737, 4872, 4875, 4878, + 4879, 4877, 4876, 4874, 124907, 124906, 4631, 124905, 124896, 124899, + 124901, 124900, 124898, 124902, 124897, 4624, 4627, 4629, 4628, 4626, + 4630, 4625, 4608, 4611, 4614, 4615, 4613, 4612, 4610, 4609, 4800, 4803, + 4805, 4804, 4802, 4792, 4795, 4797, 4796, 4794, 4798, 4793, 4784, 4787, + 4789, 4788, 4786, 11720, 11723, 11725, 11724, 11722, 11726, 11721, 4776, + 4779, 4782, 4783, 4781, 4780, 4778, 4777, 4995, 4994, 4639, 4993, 4632, + 4635, 4638, 11649, 4637, 4636, 4953, 4634, 4633, 4760, 4763, 4766, 11657, + 4765, 4764, 4767, 4762, 4761, 4752, 4755, 4758, 11656, 4757, 4756, 4759, + 4754, 4753, 4912, 4816, 4819, 4821, 4820, 4818, 4822, 4817, 4915, 4918, + 11665, 4917, 4916, 4919, 4914, 4913, 5007, 5006, 4951, 5005, 4944, 4947, + 4950, 11666, 4949, 4948, 4946, 4945, 4696, 4699, 4701, 4700, 4698, 4688, + 4691, 4693, 4692, 4690, 4694, 4689, 4680, 4683, 4685, 4684, 4682, 11712, + 11715, 11717, 11716, 11714, 11718, 11713, 4672, 4675, 4678, 4679, 4677, + 4676, 4674, 4673, 4648, 4651, 4654, 11650, 4653, 4652, 4655, 4952, 4650, + 4649, 4661, 4996, 5000, 4992, 5004, 4660, 4664, 4667, 4670, 11652, 4669, + 4668, 4671, 4666, 4665, 4640, 4643, 4645, 4644, 4647, 4642, 4646, 4641, + 11680, 11683, 11685, 11684, 11682, 11686, 11681, 4656, 4659, 4662, 11651, + 4663, 4658, 4657, 4896, 4899, 4902, 11663, 4901, 4900, 4903, 4898, 4897, + 43781, 43780, 43779, 43778, 43782, 43777, 4928, 4931, 4934, 4935, 4933, + 4932, 4930, 4929, 4920, 4923, 4925, 4924, 4927, 4922, 4926, 4921, 4720, + 4723, 4726, 11654, 4725, 4724, 4727, 4722, 4721, 4864, 4867, 4870, 11662, + 4869, 4868, 4871, 4866, 4865, 4616, 4619, 4622, 11648, 4621, 4620, 4623, + 4618, 4617, 4808, 4811, 4814, 4815, 4813, 4812, 4810, 4809, 4840, 4843, + 4846, 4847, 4845, 4844, 4842, 4841, 4744, 4747, 4749, 4748, 4746, 11728, + 11731, 11733, 11732, 11730, 11734, 11729, 4736, 4739, 4742, 4743, 4741, + 4740, 4738, 4737, 4832, 4835, 4837, 4836, 4839, 4834, 4838, 4833, 11696, + 11699, 11701, 11700, 11698, 11702, 11697, 4824, 4827, 4830, 11659, 4829, + 4828, 4831, 4826, 4825, 4712, 4715, 4717, 4716, 4719, 4714, 4718, 4713, + 5009, 5016, 5012, 5015, 5013, 5017, 5010, 5011, 5014, 5008, 4973, 4972, + 4975, 4974, 4971, 4970, 4977, 4969, 4976, 4962, 4967, 4961, 983096, + 983047, 127984, 127972, 8352, 8364, 8455, 8265, 33, 8761, 118269, 118270, + 118271, 118274, 128529, 128942, 128954, 128948, 128905, 128915, 128935, + 128125, 1781, 1780, 1783, 1782, 1779, 1778, 1776, 1785, 1777, 1784, + 128065, 128083, 128064, 128231, 9167, 127794, 983180, 128523, 128561, + 129312, 128531, 129301, 128567, 129488, 128582, 128558, 129326, 128560, + 129762, 129320, 128581, 129395, 129763, 129402, 128539, 128540, 128541, + 128514, 129298, 129769, 129323, 128580, 128548, 129764, 129396, 128566, + 129318, 128134, 128536, 129401, 8507, 127981, 10540, 10543, 9950, 127810, + 129478, 128439, 128224, 128106, 9771, 127877, 129498, 128552, 129718, + 170, 9792, 127905, 9972, 129338, 8210, 8199, 128193, 128452, 983107, + 128253, 127902, 129734, 10765, 128293, 128658, 129519, 127879, 127878, + 129512, 9789, 127771, 127763, 8296, 129351, 128031, 127907, 9673, 127845, + 128074, 8281, 11821, 127953, 129407, 129747, 9189, 117910, 9971, 129449, + 128170, 9884, 10086, 9880, 8277, 127924, 128190, 128563, 129672, 129712, + 128760, 117834, 117835, 129359, 128389, 127787, 127745, 129709, 128448, + 129462, 128099, 127860, 127869, 11792, 10972, 983071, 8704, 8873, 129376, + 10021, 11156, 8280, 8283, 10019, 127808, 10018, 128966, 8732, 8197, 9970, 129749, 129418, 8260, 8543, 128444, 128446, 128445, 127839, 8355, 129398, 10156, 128037, 8994, 128550, 128056, 127844, 127773, 127765, 10199, 9608, - 46, 65342, 65312, 65292, 65306, 65504, 65371, 65375, 65288, 65339, 65308, + 46, 65342, 65312, 65292, 65306, 65504, 65375, 65371, 65288, 65339, 65308, 65313, 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321, 65322, 65323, 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, 65333, 65334, 65335, 65336, 65337, 65338, 65345, 65346, 65347, 65348, 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, 65359, 65360, 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, - 65369, 65370, 65343, 65506, 65283, 65505, 65285, 65291, 65373, 65376, + 65369, 65370, 65343, 65506, 65283, 65505, 65285, 65291, 65376, 65373, 65289, 65341, 65340, 65307, 65295, 65509, 65507, 65287, 65286, 65290, 65284, 65301, 65300, 65303, 65302, 65299, 65298, 65296, 65305, 65297, 65304, 65309, 65281, 65344, 65310, 65293, 65282, 65311, 65510, 65374, - 65508, 65294, 65372, 8289, 9905, 9981, 9179, 983215, 983216, 983217, - 983219, 983108, 983236, 983072, 129476, 127922, 9881, 9965, 9966, 128142, - 9802, 8782, 8785, 8762, 4301, 4256, 4288, 4292, 4290, 4293, 4289, 4281, - 4285, 4284, 4282, 4278, 4258, 4287, 4283, 4277, 4265, 4276, 4270, 4280, - 4273, 4271, 4262, 4263, 4274, 4266, 4272, 4257, 4267, 4286, 4268, 4279, - 4261, 4259, 4260, 4264, 4269, 4275, 4295, 4291, 11565, 11520, 11552, - 11556, 11554, 11557, 11553, 11545, 11549, 11548, 11546, 11542, 11522, - 11551, 11547, 11541, 11529, 11540, 11534, 11544, 11537, 11535, 11526, - 11527, 11538, 11530, 11536, 11521, 11531, 11550, 11532, 11543, 11525, - 11523, 11524, 11528, 11533, 11539, 11559, 11555, 983955, 4323, 4349, - 4346, 4304, 4329, 4333, 4332, 4330, 4344, 4308, 4326, 4306, 4340, 4350, - 4336, 4338, 4341, 4337, 4335, 4331, 4325, 4313, 4351, 4314, 4324, 4318, - 4328, 4321, 4345, 4311, 4322, 4319, 4310, 4320, 4305, 4315, 4334, 4316, - 4327, 4309, 4307, 4312, 4317, 4343, 4339, 4342, 7357, 7354, 7312, 7337, - 7341, 7340, 7338, 7352, 7316, 7334, 7314, 7348, 7358, 7344, 7346, 7349, - 7345, 7343, 7339, 7333, 7321, 7359, 7322, 7332, 7326, 7336, 7329, 7353, - 7319, 7330, 7327, 7318, 7328, 7313, 7323, 7342, 7324, 7335, 7317, 7315, - 7320, 7325, 7331, 7351, 7347, 7350, 4347, 8368, 12307, 129502, 8503, - 129754, 128103, 128714, 129426, 11264, 11304, 11265, 11311, 11293, 11276, - 11268, 11271, 11287, 11306, 11267, 11275, 11274, 11305, 11303, 11307, - 11273, 11310, 11278, 11279, 11280, 11281, 11289, 11282, 11290, 11283, - 11291, 11308, 11294, 11284, 11298, 11300, 11301, 11285, 11309, 11292, - 11266, 11269, 11296, 11295, 11297, 11302, 11299, 11272, 11270, 11288, - 11286, 11277, 11312, 11352, 11313, 11359, 11341, 11324, 11316, 11319, - 11335, 11354, 11315, 11323, 11322, 11353, 11351, 11355, 11321, 11358, - 11326, 11327, 11328, 11329, 11337, 11330, 11338, 11331, 11339, 11356, - 11342, 11332, 11346, 11348, 11349, 11333, 11357, 11340, 11314, 11317, - 11344, 11343, 11345, 11350, 11347, 11320, 11318, 11336, 11334, 11325, - 129371, 127760, 127775, 129508, 10726, 129349, 128016, 66356, 66352, - 66376, 66359, 66358, 66375, 66378, 66369, 66365, 66368, 66357, 66370, - 66360, 66372, 66373, 66367, 66374, 66353, 66377, 66355, 66364, 66371, - 66361, 66363, 66366, 66354, 66362, 129405, 129421, 128893, 129727, + 65294, 65508, 65372, 8289, 9905, 118280, 9981, 9179, 983215, 983216, + 983217, 983219, 983108, 983236, 983072, 68972, 68971, 68973, 68970, + 68964, 68965, 68959, 68961, 68948, 68945, 68954, 68960, 68953, 68963, + 68949, 68947, 68952, 68946, 68962, 68958, 68950, 68957, 68951, 68955, + 68956, 68944, 68996, 68997, 68991, 68993, 68980, 68977, 68986, 68992, + 68985, 68995, 68981, 68979, 68984, 68978, 68994, 68990, 68982, 68989, + 68983, 68987, 68988, 68976, 68943, 68969, 68941, 68938, 68939, 68940, + 68942, 68975, 68974, 69006, 69007, 68933, 68932, 68935, 68934, 68931, + 68930, 68928, 68937, 68929, 68936, 129476, 127922, 9881, 9965, 9966, + 128142, 9802, 8782, 8785, 8762, 4301, 4256, 4288, 4292, 4290, 4293, 4289, + 4281, 4285, 4284, 4282, 4278, 4258, 4287, 4283, 4277, 4265, 4276, 4270, + 4280, 4273, 4271, 4262, 4263, 4274, 4266, 4272, 4257, 4267, 4286, 4268, + 4279, 4261, 4259, 4260, 4264, 4269, 4275, 4295, 4291, 11565, 11520, + 11552, 11556, 11554, 11557, 11553, 11545, 11549, 11548, 11546, 11542, + 11522, 11551, 11547, 11541, 11529, 11540, 11534, 11544, 11537, 11535, + 11526, 11527, 11538, 11530, 11536, 11521, 11531, 11550, 11532, 11543, + 11525, 11523, 11524, 11528, 11533, 11539, 11559, 11555, 983955, 4323, + 4349, 4346, 4304, 4329, 4333, 4332, 4330, 4344, 4308, 4326, 4306, 4340, + 4350, 4336, 4338, 4341, 4337, 4335, 4331, 4325, 4313, 4351, 4314, 4324, + 4318, 4328, 4321, 4345, 4311, 4322, 4319, 4310, 4320, 4305, 4315, 4334, + 4316, 4327, 4309, 4307, 4312, 4317, 4343, 4339, 4342, 7357, 7354, 7312, + 7337, 7341, 7340, 7338, 7352, 7316, 7334, 7314, 7348, 7358, 7344, 7346, + 7349, 7345, 7343, 7339, 7333, 7321, 7359, 7322, 7332, 7326, 7336, 7329, + 7353, 7319, 7330, 7327, 7318, 7328, 7313, 7323, 7342, 7324, 7335, 7317, + 7315, 7320, 7325, 7331, 7351, 7347, 7350, 4347, 8368, 12307, 129502, + 8503, 129754, 128103, 128714, 129426, 11264, 11304, 11265, 11311, 11293, + 11276, 11268, 11271, 11287, 11306, 11267, 11275, 11274, 11305, 11303, + 11307, 11273, 11310, 11278, 11279, 11280, 11281, 11289, 11282, 11290, + 11283, 11291, 11308, 11294, 11284, 11298, 11300, 11301, 11285, 11309, + 11292, 11266, 11269, 11296, 11295, 11297, 11302, 11299, 11272, 11270, + 11288, 11286, 11277, 11312, 11352, 11313, 11359, 11341, 11324, 11316, + 11319, 11335, 11354, 11315, 11323, 11322, 11353, 11351, 11355, 11321, + 11358, 11326, 11327, 11328, 11329, 11337, 11330, 11338, 11331, 11339, + 11356, 11342, 11332, 11346, 11348, 11349, 11333, 11357, 11340, 11314, + 11317, 11344, 11343, 11345, 11350, 11347, 11320, 11318, 11336, 11334, + 11325, 129371, 127760, 127775, 129508, 10726, 129349, 128016, 66356, + 66352, 66376, 66359, 66358, 66375, 66378, 66369, 66365, 66368, 66357, + 66370, 66360, 66372, 66373, 66367, 66374, 66353, 66377, 66355, 66364, + 66371, 66361, 66363, 66366, 66354, 66362, 129421, 129405, 128893, 129727, 127948, 127891, 70495, 70494, 70411, 70496, 70412, 70497, 70453, 70405, 70406, 70416, 70420, 70434, 70433, 70439, 70438, 70432, 70431, 70437, 70436, 70409, 70410, 70407, 70408, 70451, 70450, 70425, 70435, 70430, 70440, 70454, 70455, 70456, 70445, 70444, 70427, 70426, 70424, 70423, 70429, 70428, 70422, 70421, 70443, 70442, 70419, 70415, 70457, 70446, - 70448, 70447, 70400, 70401, 70493, 70461, 70402, 70460, 70477, 70403, - 70462, 70472, 70476, 70467, 70468, 70498, 70499, 70465, 70466, 70463, - 70464, 70475, 70471, 70487, 70480, 96, 127815, 10896, 10894, 10900, + 70448, 70447, 70400, 70401, 70460, 70461, 70402, 70493, 70477, 70403, + 70487, 70462, 70472, 70476, 70465, 70466, 70467, 70468, 70498, 70499, + 70463, 70464, 70475, 70471, 70480, 96, 127815, 10896, 10894, 10900, 10892, 10898, 10616, 10890, 10888, 10917, 8935, 8809, 10878, 10882, 10884, 10880, 10886, 8819, 8805, 8823, 10916, 8807, 8923, 10919, 10921, 10874, 10876, 8919, 62, 65860, 65863, 65878, 65866, 65873, 65859, 65861, @@ -10318,192 +11051,199 @@ static const unsigned int dawg_pos_to_codepoint[] = { 119322, 119323, 119324, 119296, 119305, 119306, 119307, 119308, 119309, 119310, 119311, 119312, 119313, 119314, 119298, 119299, 119301, 119302, 119303, 119304, 890, 65913, 976, 65928, 65930, 894, 127823, 128215, - 128154, 129367, 129654, 128512, 129322, 128513, 129321, 128568, 128556, + 128154, 129367, 129654, 128512, 129322, 129321, 128513, 128568, 128556, 983110, 11218, 128151, 8370, 128130, 127928, 129454, 2693, 2694, 2704, 2708, 2722, 2721, 2727, 2726, 2720, 2719, 2725, 2724, 2699, 2784, 2700, 2785, 2741, 2697, 2698, 2695, 2696, 2739, 2738, 2809, 2713, 2723, 2718, 2728, 2742, 2743, 2744, 2733, 2732, 2715, 2714, 2712, 2711, 2717, 2716, 2710, 2709, 2731, 2730, 2745, 2734, 2736, 2735, 2703, 2707, 2814, 2689, - 2812, 2815, 2813, 2811, 2810, 2749, 2690, 2748, 2765, 2691, 2757, 2761, - 2750, 2760, 2764, 2755, 2756, 2786, 2787, 2753, 2754, 2751, 2752, 2759, + 2812, 2815, 2813, 2811, 2810, 2748, 2749, 2690, 2765, 2691, 2757, 2761, + 2750, 2760, 2764, 2753, 2754, 2755, 2756, 2786, 2787, 2751, 2752, 2759, 2763, 2701, 2705, 2800, 2801, 2795, 2794, 2797, 2796, 2793, 2792, 2790, - 2799, 2791, 2798, 2768, 73056, 73057, 73064, 73067, 73091, 73090, 73081, - 73080, 73086, 73085, 73076, 73075, 73060, 73061, 73092, 73082, 73058, + 2799, 2791, 2798, 2768, 73092, 73082, 73056, 73057, 73064, 73067, 73091, + 73090, 73081, 73080, 73086, 73085, 73076, 73075, 73060, 73061, 73058, 73059, 73087, 73077, 73071, 73070, 73084, 73083, 73079, 73078, 73089, 73088, 73074, 73073, 73094, 73093, 73066, 73063, 73095, 73072, 73096, 73097, 73069, 73068, 73110, 73109, 73098, 73105, 73108, 73101, 73102, 73099, 73100, 73107, 73104, 73111, 73125, 73124, 73127, 73126, 73123, 73122, 73120, 73129, 73121, 73128, 73112, 2678, 2673, 2650, 2584, 2583, - 2649, 2582, 2581, 2652, 2608, 2565, 2566, 2576, 2580, 2594, 2593, 2599, - 2598, 2592, 2591, 2597, 2596, 2569, 2570, 2567, 2568, 2611, 2610, 2585, + 2649, 2582, 2581, 2565, 2566, 2576, 2580, 2594, 2593, 2599, 2598, 2652, + 2608, 2592, 2591, 2597, 2596, 2569, 2570, 2567, 2568, 2611, 2610, 2585, 2595, 2590, 2600, 2605, 2604, 2587, 2586, 2589, 2588, 2603, 2602, 2614, 2616, 2579, 2575, 2654, 2617, 2606, 2613, 2607, 2651, 983656, 983655, 983654, 983653, 983658, 983657, 2561, 2562, 2641, 2620, 2677, 2637, 2563, 2622, 2632, 2636, 2625, 2626, 2623, 2624, 2635, 2631, 2672, 2674, 2676, - 2667, 2666, 2669, 2668, 2665, 2664, 2662, 2671, 2663, 2670, 2675, 128123, - 983111, 129710, 8202, 128135, 65467, 65441, 65443, 65444, 65445, 65446, - 65469, 65458, 65460, 65449, 65454, 65455, 65452, 65451, 65450, 65456, - 65453, 65465, 65442, 65459, 65448, 65462, 65461, 65483, 65482, 65476, - 65477, 65499, 65490, 65495, 65466, 65464, 65479, 65478, 65498, 65500, - 65463, 65457, 65468, 65447, 65493, 65492, 65485, 65486, 65494, 65470, - 65474, 65475, 65484, 65487, 65491, 65440, 65388, 65390, 65389, 65391, - 65383, 65386, 65384, 65387, 65385, 65403, 65406, 65404, 65407, 65405, - 65437, 65413, 65416, 65414, 65417, 65415, 65418, 65421, 65419, 65422, - 65420, 65398, 65401, 65399, 65402, 65400, 65423, 65426, 65424, 65427, - 65425, 65431, 65434, 65432, 65435, 65433, 65408, 65411, 65409, 65412, - 65410, 65428, 65430, 65429, 65436, 65382, 65393, 65396, 65394, 65397, - 65395, 65439, 65438, 65381, 65392, 65378, 65513, 65379, 65515, 65514, - 65512, 65380, 65377, 65517, 65518, 65516, 128296, 128736, 9773, 9874, - 128057, 129708, 127828, 129310, 129776, 129342, 128092, 129309, 4366, - 4434, 4435, 4431, 4413, 4412, 4436, 4430, 4433, 4415, 4414, 4437, 4432, - 4364, 4429, 4363, 4427, 4420, 4422, 43382, 4425, 4424, 4419, 4426, 4418, - 4417, 43383, 4421, 4379, 4439, 4395, 4396, 4381, 4352, 4442, 4367, 4358, - 43375, 4380, 43376, 43377, 4354, 4444, 4371, 4374, 4373, 4445, 4443, - 4369, 4438, 43386, 4359, 4385, 43378, 4387, 4390, 4386, 4388, 4389, 4394, - 43379, 4382, 4392, 4391, 4393, 4384, 4383, 43380, 4416, 4357, 43371, - 43374, 43364, 43365, 43370, 43367, 43372, 43368, 43373, 4376, 43369, - 43366, 4378, 4361, 4402, 4403, 4410, 4404, 4400, 4408, 4397, 4407, 4406, - 4401, 4409, 4399, 4398, 4411, 4405, 4365, 43384, 4362, 43381, 4377, - 43385, 4356, 43388, 4353, 4372, 4360, 4440, 4423, 4355, 4446, 43360, - 43363, 4375, 43361, 43362, 4368, 4428, 4441, 4370, 43387, 4447, 12335, - 12334, 4541, 55288, 55287, 4542, 4546, 4598, 4599, 4597, 4600, 4540, - 4591, 4588, 4589, 55261, 4596, 4582, 4578, 4520, 4605, 4547, 4522, 4548, - 4604, 4602, 4603, 4606, 4543, 4535, 4572, 55265, 4575, 55263, 4574, 4573, - 4571, 4576, 55266, 4570, 55262, 4577, 4523, 55243, 55244, 4524, 4553, - 4550, 4549, 4525, 4552, 4551, 4587, 55284, 55283, 4545, 55291, 4595, - 55290, 4536, 4579, 55268, 4537, 55271, 4580, 55273, 55272, 55269, 55267, - 4581, 4527, 4528, 55254, 4556, 4565, 4568, 4529, 4561, 55256, 4562, 4530, - 55258, 55257, 4564, 4563, 4533, 4567, 55253, 4566, 4531, 4558, 4559, - 4532, 4569, 55260, 55259, 4557, 4534, 4538, 55275, 4583, 4585, 55280, - 55279, 4586, 55278, 55274, 55281, 4584, 55282, 4560, 55255, 55245, 55246, - 983213, 4539, 55276, 55277, 55264, 55289, 4521, 4607, 55270, 4590, 4526, - 4555, 55248, 55249, 55252, 55251, 55250, 4554, 55247, 4544, 4592, 983214, - 983211, 983212, 4593, 55285, 55286, 4594, 4601, 4449, 4510, 55238, 4511, - 55237, 4513, 4512, 4515, 4470, 4471, 4450, 4454, 4453, 4476, 4474, 4475, - 4467, 55227, 55226, 4502, 55225, 55228, 4501, 4469, 4504, 4509, 4505, - 55229, 55230, 55232, 55231, 55234, 55235, 4506, 55233, 4508, 55236, 4507, - 4514, 4457, 4482, 55217, 4481, 55216, 4518, 4519, 4480, 4479, 4483, 4460, - 4462, 4492, 55221, 55222, 4491, 4489, 4490, 4493, 4451, 4473, 4472, 4516, - 4452, 4456, 4455, 4517, 4477, 4478, 4468, 4503, 4461, 4484, 4485, 4486, - 55218, 55219, 55220, 4488, 4487, 4466, 4498, 4497, 4496, 4495, 4494, - 55223, 4500, 55224, 4499, 4464, 4463, 4458, 4459, 4465, 4448, 12623, - 12685, 12686, 12624, 12618, 12616, 12628, 12627, 12641, 12643, 12615, - 12676, 12664, 12665, 12657, 12619, 12593, 12595, 12609, 12654, 12656, - 12655, 12596, 12597, 12646, 12598, 12648, 12647, 12610, 12612, 12660, - 12661, 12663, 12659, 12662, 12658, 12621, 12671, 12601, 12602, 12649, - 12603, 12607, 12604, 12651, 12652, 12606, 12650, 12653, 12608, 12605, - 12620, 12599, 12613, 12670, 12666, 12667, 12669, 12668, 12617, 12594, - 12645, 12611, 12600, 12677, 12614, 12672, 12638, 12637, 12632, 12633, - 12639, 12630, 12629, 12678, 12673, 12675, 12674, 12635, 12679, 12680, - 12681, 12640, 12683, 12682, 12684, 12625, 12626, 12642, 12622, 12631, - 12634, 12636, 12644, 12323, 12346, 12345, 12322, 12344, 12325, 12324, - 12327, 12326, 12329, 12321, 12328, 68875, 68874, 68887, 68889, 68872, - 68881, 68868, 68867, 68877, 68876, 68890, 68891, 68885, 68880, 68879, - 68865, 68870, 68873, 68882, 68871, 68869, 68883, 68884, 68866, 68892, - 68886, 68888, 68878, 68864, 68899, 68898, 68901, 68903, 68902, 68900, - 68893, 68896, 68894, 68897, 68895, 68917, 68916, 68919, 68918, 68915, - 68914, 68912, 68921, 68913, 68920, 5925, 5928, 5930, 5927, 5924, 5937, - 5923, 5934, 5931, 5929, 5933, 5936, 5926, 5935, 5932, 5920, 5921, 5922, - 5940, 5938, 5939, 128587, 128035, 67808, 67823, 67814, 67816, 67829, + 2667, 2666, 2669, 2668, 2665, 2664, 2662, 2671, 2663, 2670, 2675, 90412, + 90414, 90411, 90410, 90373, 90388, 90382, 90381, 90387, 90386, 90380, + 90379, 90385, 90384, 90392, 90391, 90375, 90374, 90372, 90371, 90377, + 90376, 90370, 90369, 90390, 90389, 90378, 90396, 90393, 90395, 90397, + 90383, 90394, 90368, 90413, 90415, 90398, 90405, 90408, 90401, 90402, + 90406, 90407, 90399, 90400, 90403, 90404, 90409, 90421, 90420, 90423, + 90422, 90419, 90418, 90416, 90425, 90417, 90424, 128123, 983111, 129710, + 8202, 128135, 65467, 65441, 65443, 65444, 65445, 65446, 65469, 65458, + 65460, 65449, 65454, 65455, 65452, 65451, 65450, 65456, 65453, 65465, + 65442, 65459, 65448, 65462, 65461, 65483, 65482, 65476, 65477, 65499, + 65490, 65495, 65466, 65464, 65479, 65478, 65498, 65500, 65463, 65457, + 65468, 65447, 65493, 65492, 65485, 65486, 65494, 65470, 65474, 65475, + 65484, 65487, 65491, 65440, 65388, 65390, 65389, 65391, 65383, 65386, + 65384, 65387, 65385, 65403, 65406, 65404, 65407, 65405, 65437, 65413, + 65416, 65414, 65417, 65415, 65418, 65421, 65419, 65422, 65420, 65398, + 65401, 65399, 65402, 65400, 65423, 65426, 65424, 65427, 65425, 65431, + 65434, 65432, 65435, 65433, 65408, 65411, 65409, 65412, 65410, 65428, + 65430, 65429, 65436, 65382, 65393, 65396, 65394, 65397, 65395, 65439, + 65438, 65381, 65392, 65378, 65513, 65379, 65515, 65512, 65514, 65380, + 65377, 65517, 65518, 65516, 128296, 128736, 9773, 9874, 128057, 129708, + 127828, 129310, 129776, 129342, 128092, 129309, 4366, 4434, 4435, 4431, + 4413, 4412, 4436, 4430, 4433, 4415, 4414, 4437, 4432, 4364, 4429, 4363, + 4427, 4420, 4422, 43382, 4425, 4424, 4419, 4426, 4418, 4417, 43383, 4421, + 4379, 4439, 4395, 4396, 4381, 4352, 4442, 4367, 4358, 43375, 4380, 43376, + 43377, 4354, 4444, 4371, 4374, 4373, 4445, 4443, 4369, 4438, 43386, 4359, + 4385, 43378, 4387, 4390, 4386, 4388, 4389, 4394, 43379, 4382, 4392, 4391, + 4393, 4384, 4383, 43380, 4416, 4357, 43371, 43374, 43364, 43365, 43370, + 43367, 43372, 43368, 43373, 4376, 43369, 43366, 4378, 4361, 4402, 4403, + 4410, 4404, 4400, 4408, 4397, 4407, 4406, 4401, 4409, 4399, 4398, 4411, + 4405, 4365, 43384, 4362, 43381, 4377, 43385, 4356, 43388, 4353, 4372, + 4360, 4440, 4423, 4355, 4446, 43360, 43363, 4375, 43361, 43362, 4368, + 4428, 4441, 4370, 43387, 4447, 12335, 12334, 4541, 55288, 55287, 4542, + 4546, 4598, 4599, 4597, 4600, 4540, 4591, 4588, 4589, 55261, 4596, 4582, + 4578, 4520, 4605, 4547, 4522, 4548, 4604, 4602, 4603, 4606, 4543, 4535, + 4572, 55265, 4575, 55263, 4574, 4573, 4571, 4576, 55266, 4570, 55262, + 4577, 4523, 55243, 55244, 4524, 4553, 4550, 4549, 4525, 4552, 4551, 4587, + 55284, 55283, 4545, 55291, 4595, 55290, 4536, 4579, 55268, 4537, 55271, + 4580, 55273, 55272, 55269, 55267, 4581, 4527, 4528, 55254, 4556, 4565, + 4568, 4529, 4561, 55256, 4562, 4530, 55258, 55257, 4564, 4563, 4533, + 4567, 55253, 4566, 4531, 4558, 4559, 4532, 4569, 55260, 55259, 4557, + 4534, 4538, 55275, 4583, 4585, 55280, 55279, 4586, 55278, 55274, 55281, + 4584, 55282, 4560, 55255, 55245, 55246, 983213, 4539, 55276, 55277, + 55264, 55289, 4521, 4607, 55270, 4590, 4526, 4555, 55248, 55249, 55252, + 55251, 55250, 4554, 55247, 4544, 4592, 983214, 983211, 983212, 4593, + 55285, 55286, 4594, 4601, 4449, 4510, 55238, 4511, 55237, 4513, 4512, + 4515, 4470, 4471, 4450, 4454, 4453, 4476, 4474, 4475, 4467, 55227, 55226, + 4502, 55225, 55228, 4501, 4469, 4504, 4509, 4505, 55229, 55230, 55232, + 55231, 55234, 55235, 4506, 55233, 4508, 55236, 4507, 4514, 4457, 4482, + 55217, 4481, 55216, 4518, 4519, 4480, 4479, 4483, 4460, 4462, 4492, + 55221, 55222, 4491, 4489, 4490, 4493, 4451, 4473, 4472, 4516, 4452, 4456, + 4455, 4517, 4477, 4478, 4468, 4503, 4461, 4484, 4485, 4486, 55218, 55219, + 55220, 4488, 4487, 4466, 4498, 4497, 4496, 4495, 4494, 55223, 4500, + 55224, 4499, 4464, 4463, 4458, 4459, 4465, 4448, 12623, 12685, 12686, + 12624, 12618, 12616, 12628, 12627, 12641, 12643, 12615, 12676, 12664, + 12665, 12657, 12619, 12593, 12595, 12609, 12654, 12656, 12655, 12596, + 12597, 12646, 12598, 12648, 12647, 12610, 12612, 12660, 12661, 12663, + 12659, 12662, 12658, 12621, 12671, 12601, 12602, 12649, 12603, 12607, + 12604, 12651, 12652, 12606, 12650, 12653, 12608, 12605, 12620, 12599, + 12613, 12670, 12666, 12667, 12669, 12668, 12617, 12594, 12645, 12611, + 12600, 12677, 12614, 12672, 12638, 12637, 12632, 12633, 12639, 12630, + 12629, 12678, 12673, 12675, 12674, 12635, 12679, 12680, 12681, 12640, + 12683, 12682, 12684, 12625, 12626, 12642, 12622, 12631, 12634, 12636, + 12644, 12323, 12346, 12345, 12322, 12344, 12325, 12324, 12327, 12326, + 12329, 12321, 12328, 68875, 68874, 68887, 68889, 68872, 68881, 68868, + 68867, 68877, 68876, 68890, 68891, 68885, 68880, 68879, 68865, 68870, + 68873, 68882, 68871, 68869, 68883, 68884, 68866, 68892, 68886, 68888, + 68878, 68864, 68899, 68898, 68901, 68903, 68902, 68900, 68893, 68896, + 68894, 68897, 68895, 68917, 68916, 68919, 68918, 68915, 68914, 68912, + 68921, 68913, 68920, 5925, 5928, 5930, 5927, 5924, 5937, 5923, 5934, + 5931, 5929, 5933, 5936, 5926, 5935, 5932, 5920, 5921, 5922, 5940, 5938, + 5939, 128587, 128436, 129673, 128035, 67808, 67823, 67814, 67816, 67829, 67819, 67826, 67811, 67810, 67822, 67825, 67828, 67817, 67812, 67815, 67818, 67809, 67821, 67813, 67824, 67820, 67835, 67839, 67838, 67837, - 67836, 128891, 128436, 11233, 129702, 9980, 127911, 127892, 128157, - 128152, 128159, 129782, 128585, 129180, 129183, 129182, 129181, 128628, - 10033, 10008, 10149, 10150, 10084, 10167, 10169, 10168, 10054, 10004, - 11096, 9955, 11095, 11097, 10152, 10144, 10135, 129055, 129051, 10077, - 10078, 128178, 128977, 10040, 128975, 10059, 128958, 10071, 10082, - 129008, 10020, 128970, 128946, 129943, 10083, 11093, 128327, 10096, - 10094, 129052, 129048, 10080, 10079, 10157, 128627, 10006, 10134, 10012, - 11094, 10030, 10097, 10095, 10137, 129054, 129050, 10140, 128635, 128940, - 10075, 10076, 128972, 128952, 128607, 128615, 10136, 128605, 128613, - 128625, 10056, 128606, 128614, 10138, 128604, 128612, 10051, 10045, - 10171, 10142, 128980, 128979, 129053, 129049, 10158, 9947, 128903, - 128913, 10132, 10173, 128633, 10133, 10074, 10010, 1442, 1453, 1447, - 1436, 1437, 1438, 1445, 1446, 1443, 1444, 1433, 1441, 1425, 1439, 1448, - 1426, 1427, 1449, 1440, 1435, 1430, 1450, 1434, 1428, 1429, 1432, 1454, - 1431, 1451, 1452, 1488, 64304, 64302, 64303, 64288, 64297, 1506, 1489, - 64332, 64305, 1499, 64333, 64315, 1508, 64334, 64324, 1498, 64314, 1507, - 64323, 1509, 1503, 1501, 1511, 64327, 1492, 64308, 1495, 1504, 64320, - 1494, 64310, 1505, 64321, 1513, 64329, 64300, 64301, 64298, 64299, 1512, - 64328, 1496, 64312, 1514, 64330, 1510, 64326, 1491, 64307, 1490, 64306, - 1500, 64316, 1502, 64318, 1493, 64331, 64309, 64293, 64289, 64296, 64295, - 64290, 64292, 64291, 64294, 1497, 64285, 64313, 64335, 1520, 1522, 64287, - 1521, 1477, 1476, 1455, 1468, 1458, 1459, 1457, 1460, 1465, 1466, 1463, - 64286, 1464, 1479, 1467, 1471, 1462, 1473, 1456, 1474, 1461, 1469, 1524, - 1523, 1472, 1475, 1478, 1470, 1519, 9937, 9096, 11263, 128641, 110597, - 110594, 110595, 110596, 110778, 110779, 110780, 110781, 110782, 110783, - 110784, 110785, 110750, 110759, 110760, 110751, 110752, 110753, 110754, - 110755, 110756, 110757, 110758, 110768, 110769, 110770, 110771, 110772, - 110773, 110774, 110775, 110776, 110777, 110761, 110762, 110763, 110764, - 110765, 110766, 110767, 110615, 110624, 110625, 110626, 110616, 110617, - 110618, 110619, 110620, 110621, 110622, 110623, 110651, 110648, 110649, - 110650, 110627, 110628, 110629, 110630, 110631, 110632, 110633, 110634, - 110642, 110643, 110644, 110645, 110646, 110647, 110635, 110636, 110637, - 110638, 110639, 110640, 110641, 110806, 110804, 110805, 110807, 110808, - 110809, 110810, 110811, 110812, 110786, 110787, 110788, 110789, 110790, - 110791, 110792, 110793, 110794, 110795, 110796, 110797, 110798, 110799, - 110800, 110801, 110802, 110803, 110744, 110738, 110739, 110740, 110741, - 110742, 110743, 110734, 110727, 110728, 110729, 110730, 110731, 110732, - 110733, 110718, 110719, 110720, 110721, 110722, 110723, 110724, 110725, - 110726, 110745, 110746, 110747, 110748, 110749, 110735, 110736, 110737, - 110877, 110878, 110850, 110851, 110852, 110853, 110854, 110855, 110840, - 110841, 110842, 110843, 110844, 110845, 110833, 110834, 110835, 110836, - 110837, 110838, 110839, 110829, 110830, 110831, 110832, 110846, 110847, - 110848, 110849, 110652, 110653, 110654, 110655, 110656, 110657, 110658, - 110659, 110666, 110667, 110668, 110669, 110670, 110671, 110672, 110673, - 110660, 110661, 110662, 110663, 110664, 110665, 110674, 110675, 110676, - 110677, 110678, 110679, 110680, 110681, 110682, 110683, 110684, 110685, - 110702, 110703, 110704, 110705, 110706, 110707, 110708, 110709, 110710, - 110717, 110711, 110712, 110713, 110714, 110715, 110716, 110701, 110697, - 110698, 110699, 110700, 110690, 110691, 110692, 110693, 110694, 110695, - 110696, 110686, 110687, 110688, 110689, 110856, 110857, 110858, 110859, - 110860, 110861, 110862, 110863, 110864, 110865, 110870, 110871, 110872, - 110873, 110874, 110875, 110876, 110866, 110867, 110868, 110869, 110818, - 110813, 110814, 110815, 110816, 110817, 110823, 110824, 110825, 110826, - 110827, 110828, 110819, 110820, 110821, 110822, 983271, 110607, 110608, - 110609, 110610, 110611, 110602, 110603, 110604, 110605, 110606, 110612, - 110613, 110614, 110598, 110599, 110600, 110601, 8889, 127807, 19922, - 19966, 19958, 19967, 19924, 19946, 19923, 19909, 19947, 19944, 19943, - 19956, 19906, 19962, 19935, 19939, 19920, 19916, 19948, 19917, 19937, - 19931, 19929, 19925, 19911, 19964, 19928, 19945, 19934, 19918, 19930, - 19942, 19950, 19941, 19949, 19938, 19914, 19927, 19936, 19952, 19965, - 19912, 19915, 19926, 19954, 19910, 19932, 19953, 19904, 19933, 19940, - 19905, 19951, 19959, 19957, 19960, 19955, 19961, 19913, 19908, 19921, - 19963, 19907, 19919, 129428, 128262, 9889, 983123, 128644, 128645, - 128096, 12447, 12354, 110879, 110593, 12430, 110929, 110928, 110930, - 12419, 12423, 12421, 12437, 12438, 110898, 12387, 12353, 12359, 12355, - 12361, 12357, 12373, 12379, 12375, 12381, 12377, 12403, 983997, 984000, - 983998, 984001, 983999, 12400, 12409, 12412, 12406, 12435, 12394, 12397, - 12395, 12398, 12396, 12384, 12391, 12386, 12393, 12389, 12364, 12370, - 12366, 12372, 12368, 12399, 12408, 12402, 12411, 12405, 12363, 12369, - 12365, 12371, 12367, 12414, 12417, 12415, 12418, 12416, 12401, 12410, - 12404, 12413, 12407, 12425, 12428, 12426, 12429, 12427, 12383, 12390, - 12385, 12392, 12388, 12374, 12380, 12376, 12382, 12378, 12431, 12433, - 12432, 12434, 12420, 12424, 12422, 12436, 12360, 12356, 12362, 12358, - 12446, 12445, 9964, 128725, 129406, 127802, 129435, 128616, 128617, - 128371, 128029, 127855, 11203, 11043, 8213, 9135, 129921, 129910, 129911, - 129912, 129913, 129914, 129915, 9146, 9147, 9148, 9149, 983059, 983141, - 983138, 11134, 128677, 8230, 9897, 128014, 127943, 128052, 127798, 9749, - 9832, 127789, 127976, 8987, 9203, 8962, 127969, 127968, 127960, 127973, - 128298, 8763, 129693, 983124, 983060, 983142, 983139, 128559, 128175, - 129303, 128726, 8208, 11802, 8259, 8231, 45, 11794, 9102, 11226, 129723, - 8372, 127848, 129482, 127954, 9976, 8801, 10725, 10855, 129706, 12696, - 12700, 12693, 12697, 12695, 12703, 12692, 12699, 12691, 12694, 12688, - 12698, 12690, 12689, 12701, 12702, 12294, 12289, 12275, 12273, 12274, - 12272, 12283, 12282, 12285, 12279, 12280, 12281, 12278, 12277, 12284, - 12783, 12287, 12276, 12286, 12332, 12295, 119670, 119669, 119668, 119667, - 119666, 12999, 12995, 13309, 13310, 13292, 13282, 13299, 13304, 13303, - 13306, 13305, 13302, 13301, 13308, 13300, 13307, 13291, 13281, 13289, - 13287, 13297, 13290, 13294, 13284, 13283, 13293, 13288, 13298, 13286, - 13296, 13285, 13295, 13280, 13003, 13164, 13168, 13167, 13166, 13165, - 13156, 13146, 13157, 13147, 13154, 13152, 13162, 13155, 13159, 13149, - 13148, 13158, 13153, 13163, 13151, 13161, 13150, 13160, 13144, 13145, - 12992, 12997, 12998, 12993, 12994, 12996, 13002, 13000, 13001, 12343, - 12351, 12333, 12331, 12330, 12350, 12290, 12293, 12288, 8887, 8787, - 128127, 67676, 67673, 67675, 67679, 67674, 67672, 67677, 67678, 67656, - 67669, 67651, 67659, 67666, 67667, 67648, 67663, 67650, 67654, 67662, - 67665, 67668, 67657, 67652, 67655, 67658, 67649, 67661, 67653, 67664, - 67660, 67671, 128232, 10716, 128474, 10721, 8710, 983130, 983129, 129781, + 67836, 128891, 11233, 129702, 9980, 127911, 127892, 128157, 128152, + 128159, 129782, 128585, 129180, 129183, 129182, 129181, 128628, 10033, + 10008, 10149, 10150, 10084, 10167, 10169, 10168, 10054, 10004, 11096, + 9955, 11095, 11097, 10152, 10144, 10135, 129055, 129051, 10077, 10078, + 128178, 128977, 10040, 128975, 10059, 128958, 10071, 10082, 129008, + 10020, 128970, 128946, 129943, 10083, 11093, 128327, 10096, 10094, + 129052, 129048, 10080, 10079, 10157, 128627, 10006, 10134, 10012, 11094, + 10030, 10097, 10095, 10137, 129054, 129050, 10140, 128635, 128940, 10075, + 10076, 128972, 128952, 128607, 128615, 10136, 128605, 128613, 128625, + 10056, 128606, 128614, 10138, 128604, 128612, 10051, 10045, 10171, 10142, + 128980, 128979, 129053, 129049, 10158, 128913, 118277, 9947, 128903, + 10132, 10173, 128633, 10133, 10074, 10010, 1442, 1453, 1447, 1436, 1437, + 1438, 1445, 1446, 1443, 1444, 1433, 1441, 1425, 1439, 1448, 1426, 1427, + 1449, 1440, 1435, 1430, 1450, 1434, 1428, 1429, 1432, 1454, 1431, 1451, + 1452, 1488, 64304, 64302, 64303, 64288, 64297, 1506, 1489, 64332, 64305, + 1499, 64333, 64315, 1508, 64334, 64324, 1498, 64314, 1507, 64323, 1509, + 1503, 1501, 1511, 64327, 1492, 64308, 1495, 1504, 64320, 1494, 64310, + 1505, 64321, 1513, 64329, 64300, 64301, 64298, 64299, 1512, 64328, 1496, + 64312, 1514, 64330, 1510, 64326, 1491, 64307, 1490, 64306, 1500, 64316, + 1502, 64318, 1493, 64331, 64309, 64293, 64289, 64296, 64295, 64290, + 64292, 64291, 64294, 1497, 64285, 64313, 64335, 1520, 1522, 64287, 1521, + 1477, 1476, 1455, 1468, 1458, 1459, 1457, 1460, 1465, 1466, 1463, 1464, + 1479, 1467, 1471, 1462, 1473, 1456, 1474, 1461, 64286, 1469, 1524, 1523, + 1472, 1475, 1478, 1470, 1519, 9937, 9096, 11263, 128641, 110597, 110594, + 110595, 110596, 110778, 110779, 110780, 110781, 110782, 110783, 110784, + 110785, 110750, 110759, 110760, 110751, 110752, 110753, 110754, 110755, + 110756, 110757, 110758, 110768, 110769, 110770, 110771, 110772, 110773, + 110774, 110775, 110776, 110777, 110761, 110762, 110763, 110764, 110765, + 110766, 110767, 110615, 110624, 110625, 110626, 110616, 110617, 110618, + 110619, 110620, 110621, 110622, 110623, 110651, 110648, 110649, 110650, + 110627, 110628, 110629, 110630, 110631, 110632, 110633, 110634, 110642, + 110643, 110644, 110645, 110646, 110647, 110635, 110636, 110637, 110638, + 110639, 110640, 110641, 110806, 110804, 110805, 110807, 110808, 110809, + 110810, 110811, 110812, 110786, 110787, 110788, 110789, 110790, 110791, + 110792, 110793, 110794, 110795, 110796, 110797, 110798, 110799, 110800, + 110801, 110802, 110803, 110744, 110738, 110739, 110740, 110741, 110742, + 110743, 110734, 110727, 110728, 110729, 110730, 110731, 110732, 110733, + 110718, 110719, 110720, 110721, 110722, 110723, 110724, 110725, 110726, + 110745, 110746, 110747, 110748, 110749, 110735, 110736, 110737, 110877, + 110878, 110850, 110851, 110852, 110853, 110854, 110855, 110840, 110841, + 110842, 110843, 110844, 110845, 110833, 110834, 110835, 110836, 110837, + 110838, 110839, 110829, 110830, 110831, 110832, 110846, 110847, 110848, + 110849, 110652, 110653, 110654, 110655, 110656, 110657, 110658, 110659, + 110666, 110667, 110668, 110669, 110670, 110671, 110672, 110673, 110660, + 110661, 110662, 110663, 110664, 110665, 110674, 110675, 110676, 110677, + 110678, 110679, 110680, 110681, 110682, 110683, 110684, 110685, 110702, + 110703, 110704, 110705, 110706, 110707, 110708, 110709, 110710, 110717, + 110711, 110712, 110713, 110714, 110715, 110716, 110701, 110697, 110698, + 110699, 110700, 110690, 110691, 110692, 110693, 110694, 110695, 110696, + 110686, 110687, 110688, 110689, 110856, 110857, 110858, 110859, 110860, + 110861, 110862, 110863, 110864, 110865, 110870, 110871, 110872, 110873, + 110874, 110875, 110876, 110866, 110867, 110868, 110869, 110818, 110813, + 110814, 110815, 110816, 110817, 110823, 110824, 110825, 110826, 110827, + 110828, 110819, 110820, 110821, 110822, 983273, 110607, 110608, 110609, + 110610, 110611, 110602, 110603, 110604, 110605, 110606, 110612, 110613, + 110614, 110598, 110599, 110600, 110601, 8889, 127807, 19922, 19966, + 19958, 19967, 19924, 19946, 19923, 19909, 19947, 19944, 19943, 19956, + 19906, 19962, 19935, 19939, 19920, 19916, 19948, 19917, 19937, 19931, + 19929, 19925, 19911, 19928, 19964, 19945, 19934, 19918, 19930, 19942, + 19950, 19941, 19949, 19938, 19914, 19927, 19936, 19952, 19965, 19912, + 19915, 19926, 19954, 19910, 19932, 19953, 19904, 19933, 19940, 19905, + 19951, 19959, 19957, 19960, 19955, 19961, 19913, 19908, 19921, 19963, + 19907, 19919, 129428, 128262, 9889, 983123, 128644, 128645, 128096, + 12447, 12354, 110879, 110593, 12430, 110929, 110928, 110930, 12419, + 12423, 12421, 12437, 12438, 110898, 12387, 12353, 12359, 12355, 12361, + 12357, 12373, 12379, 12375, 12381, 12377, 12403, 983997, 984000, 983998, + 984001, 983999, 12400, 12409, 12412, 12406, 12435, 12394, 12397, 12395, + 12398, 12396, 12384, 12391, 12386, 12393, 12389, 12364, 12370, 12366, + 12372, 12368, 12399, 12408, 12402, 12411, 12405, 12363, 12369, 12365, + 12371, 12367, 12414, 12417, 12415, 12418, 12416, 12401, 12410, 12404, + 12413, 12407, 12425, 12428, 12426, 12429, 12427, 12383, 12390, 12385, + 12392, 12388, 12374, 12380, 12376, 12382, 12378, 12431, 12433, 12432, + 12434, 12420, 12424, 12422, 12436, 12360, 12356, 12362, 12358, 12446, + 12445, 9964, 128725, 129406, 127802, 129435, 128616, 128617, 128371, + 128029, 127855, 11203, 11043, 8213, 118290, 118287, 117905, 9135, 117893, + 129921, 129910, 129911, 129912, 129913, 129914, 129915, 9146, 9147, 9148, + 9149, 983059, 983141, 983138, 11134, 128677, 117779, 8230, 9897, 117915, + 117769, 118448, 128014, 127943, 128052, 127798, 9749, 9832, 127789, + 127976, 8987, 9203, 8962, 127969, 127968, 127960, 127973, 128298, 8763, + 129693, 983124, 983060, 983142, 983139, 128559, 128175, 129303, 128726, + 8208, 11802, 8259, 8231, 45, 11794, 9102, 11226, 129723, 8372, 127848, + 129482, 127954, 9976, 8801, 10725, 10855, 129706, 12696, 12700, 12693, + 12697, 12695, 12703, 12692, 12699, 12691, 12694, 12688, 12698, 12690, + 12689, 12701, 12702, 12294, 12289, 12275, 12273, 12274, 12272, 12283, + 12282, 12285, 12279, 12280, 12281, 12278, 12277, 12284, 12783, 12287, + 12276, 12286, 12332, 12295, 119670, 119669, 119668, 119667, 119666, + 12999, 12995, 13309, 13310, 13292, 13282, 13299, 13304, 13303, 13306, + 13305, 13302, 13301, 13308, 13300, 13307, 13291, 13281, 13289, 13287, + 13297, 13290, 13294, 13284, 13283, 13293, 13288, 13298, 13286, 13296, + 13285, 13295, 13280, 13003, 13164, 13168, 13167, 13166, 13165, 13156, + 13146, 13157, 13147, 13154, 13152, 13162, 13155, 13159, 13149, 13148, + 13158, 13153, 13163, 13151, 13161, 13150, 13160, 13144, 13145, 12992, + 12997, 12998, 12993, 12994, 12996, 13002, 13000, 13001, 12343, 12351, + 12333, 12331, 12330, 12350, 12290, 12293, 12288, 8887, 8787, 128127, + 67676, 67673, 67675, 67679, 67674, 67672, 67677, 67678, 67656, 67669, + 67651, 67659, 67666, 67667, 67648, 67663, 67650, 67654, 67662, 67665, + 67668, 67657, 67652, 67655, 67658, 67649, 67661, 67653, 67664, 67660, + 67671, 128232, 10716, 128474, 10721, 8710, 983130, 983129, 129781, 126132, 126112, 126126, 126125, 126127, 126131, 126130, 126129, 126113, 126114, 126110, 126111, 126072, 126081, 126108, 126090, 126099, 126078, 126105, 126069, 126087, 126096, 126077, 126104, 126068, 126086, 126095, @@ -10521,235 +11261,248 @@ static const unsigned int dawg_pos_to_codepoint[] = { 68428, 68445, 68441, 68442, 68444, 68440, 68446, 68447, 68443, 9088, 8747, 10767, 10773, 10776, 10780, 10778, 10775, 10777, 10779, 10766, 9134, 65529, 65531, 65530, 9892, 8745, 10825, 10823, 10820, 10819, 10816, - 10827, 8253, 8890, 10812, 129942, 129969, 129972, 9689, 129936, 9688, - 11800, 11845, 11846, 8766, 9959, 161, 8487, 8276, 191, 8290, 8292, 8291, - 128229, 127982, 129311, 127875, 127983, 127971, 12292, 9979, 127886, - 128122, 128121, 128304, 43453, 43454, 43455, 43457, 43421, 43422, 43426, - 43427, 43398, 43397, 43399, 43407, 43408, 43409, 43412, 43402, 43403, - 43418, 43416, 43428, 43423, 43431, 43432, 43413, 43414, 43410, 43411, - 43429, 43430, 43401, 43435, 43436, 43441, 43439, 43440, 43424, 43425, - 43419, 43420, 43415, 43417, 43396, 43405, 43442, 43437, 43433, 43438, - 43434, 43404, 43406, 43400, 43458, 43466, 43467, 43459, 43465, 43461, - 43464, 43468, 43463, 43486, 43462, 43487, 43460, 43471, 43456, 43469, - 43393, 43443, 43392, 43395, 43394, 43448, 43449, 43444, 43450, 43445, - 43446, 43447, 43452, 43451, 43477, 43476, 43479, 43478, 43475, 43474, - 43472, 43481, 43473, 43480, 129753, 129724, 128086, 128377, 10781, - 129337, 9795, 9909, 129513, 69786, 69787, 69785, 69793, 69792, 69763, - 69764, 69770, 69772, 69784, 69783, 69791, 69790, 69767, 69768, 69765, - 69766, 69777, 69789, 69782, 69794, 69804, 69805, 69806, 69798, 69797, - 69779, 69778, 69776, 69775, 69781, 69780, 69774, 69773, 69796, 69795, - 69788, 69801, 69807, 69802, 69799, 69803, 69800, 69769, 69771, 69760, - 69818, 69761, 69817, 69762, 69822, 69823, 69825, 69824, 69826, 69808, - 69814, 69816, 69811, 69812, 69809, 69810, 69813, 69815, 69821, 69837, - 69819, 69820, 12142, 12164, 12060, 12157, 12100, 12149, 12184, 12191, - 12227, 12068, 12174, 12234, 12205, 12134, 12168, 12219, 12189, 12088, - 12090, 12096, 12160, 12182, 12224, 12190, 12147, 12114, 12118, 12058, - 12176, 12075, 12112, 12045, 12170, 12124, 12070, 12194, 12109, 12229, - 12196, 12139, 12056, 12099, 12034, 12084, 12136, 12120, 12044, 12111, - 12094, 12125, 12243, 12238, 12082, 12159, 12063, 12215, 12062, 12042, - 12241, 12067, 12235, 12043, 12140, 12119, 12207, 12123, 12133, 12222, - 12117, 12226, 12245, 12214, 12217, 12236, 12155, 12188, 12113, 12065, - 12198, 12066, 12146, 12171, 12225, 12200, 12121, 12093, 12095, 12221, - 12092, 12216, 12231, 12054, 12218, 12179, 12037, 12173, 12072, 12046, - 12127, 12152, 12049, 12074, 12107, 12208, 12212, 12041, 12210, 12131, - 12033, 12039, 12199, 12085, 12128, 12161, 12162, 12233, 12165, 12192, - 12077, 12201, 12061, 12105, 12040, 12240, 12102, 12153, 12032, 12080, - 12167, 12048, 12156, 12059, 12126, 12158, 12050, 12183, 12204, 12097, - 12239, 12053, 12078, 12150, 12071, 12187, 12186, 12223, 12228, 12104, - 12098, 12064, 12036, 12057, 12163, 12178, 12185, 12154, 12083, 12203, - 12087, 12135, 12151, 12202, 12035, 12122, 12141, 12180, 12144, 12076, - 12052, 12115, 12091, 12108, 12169, 12143, 12148, 12130, 12089, 12211, - 12073, 12101, 12138, 12103, 12209, 12047, 12220, 12172, 12129, 12166, - 12242, 12237, 12145, 12106, 12081, 12244, 12038, 12086, 12055, 12181, - 12197, 12193, 12175, 12116, 12110, 12177, 12137, 12230, 12213, 12195, - 12069, 12079, 12206, 12051, 12232, 12132, 129432, 983205, 3251, 3250, - 3240, 3293, 3225, 3235, 3230, 3249, 3248, 3205, 3206, 3216, 3220, 3234, - 3233, 3239, 3238, 3232, 3231, 3237, 3236, 3211, 3296, 3212, 3297, 3253, - 3209, 3210, 3218, 3219, 3207, 3208, 3254, 3255, 3256, 3245, 3244, 3227, - 3226, 3224, 3223, 3229, 3228, 3222, 3221, 3243, 3242, 3214, 3215, 3294, - 3257, 3246, 3247, 3285, 3315, 3201, 3200, 3204, 3261, 3202, 3260, 3313, - 3314, 3277, 3203, 3286, 3262, 3272, 3276, 3267, 3268, 3298, 3299, 3265, - 3266, 3274, 3275, 3263, 3264, 3270, 3271, 3307, 3306, 3309, 3308, 3305, - 3304, 3302, 3311, 3303, 3310, 12450, 984009, 984008, 984007, 984010, - 110881, 110880, 110882, 110592, 12499, 984002, 984005, 984003, 984006, - 984004, 12496, 12505, 12508, 12502, 12511, 110583, 110584, 110585, - 110586, 110587, 110589, 110590, 110576, 110577, 110578, 110579, 110581, - 110582, 12510, 12513, 12514, 12512, 12531, 12490, 12493, 12491, 12494, - 12492, 12789, 12792, 12790, 12793, 12791, 12795, 12798, 12796, 12799, - 12797, 12533, 12534, 110933, 12784, 12787, 12483, 12526, 110949, 110948, - 110950, 12515, 12519, 12517, 110951, 12788, 12785, 12786, 12794, 12449, - 12455, 12451, 12457, 12453, 12469, 12475, 12471, 12477, 12473, 12480, - 12487, 12482, 12489, 12485, 12460, 12466, 12462, 12468, 12464, 12495, - 12504, 12498, 12507, 12501, 12459, 12465, 12461, 12467, 12463, 12497, - 12506, 12500, 12509, 12503, 12521, 12524, 12522, 12525, 12523, 12479, - 12486, 12481, 12488, 12484, 12535, 12537, 12536, 12538, 12532, 12470, - 12476, 12472, 12478, 12474, 12527, 12529, 12528, 12530, 12516, 12520, - 12518, 12456, 12452, 12458, 12454, 12542, 12543, 12541, 12539, 12448, - 12540, 12444, 12443, 73476, 73477, 73487, 73523, 73498, 73497, 73503, - 73502, 73508, 73507, 73501, 73500, 73506, 73505, 73482, 73483, 73484, - 73485, 73480, 73481, 73478, 73479, 73494, 73504, 73499, 73509, 73519, - 73520, 73521, 73513, 73512, 73496, 73495, 73493, 73492, 73491, 73490, - 73511, 73510, 73522, 73517, 73514, 73516, 73518, 73515, 73486, 73488, - 73551, 73548, 73546, 73545, 73549, 73543, 73541, 73544, 73550, 73542, - 73547, 73474, 73537, 73472, 73475, 73473, 73525, 73524, 73535, 73530, - 73534, 73536, 73528, 73529, 73526, 73527, 73540, 73539, 73557, 73556, - 73559, 73558, 73555, 73554, 73552, 73561, 73553, 73560, 73538, 43283, - 43295, 43299, 43301, 43277, 43281, 43284, 43275, 43274, 43286, 43285, - 43279, 43278, 43294, 43282, 43289, 43297, 43288, 43276, 43292, 43287, - 43290, 43296, 43293, 43291, 43280, 43298, 43300, 43310, 43311, 43308, - 43309, 43307, 43303, 43305, 43304, 43302, 43306, 43269, 43268, 43271, - 43270, 43267, 43266, 43264, 43273, 43265, 43272, 119496, 119506, 119499, - 119503, 119493, 119492, 119502, 119497, 119507, 119495, 119505, 119494, - 119504, 119501, 119491, 119500, 119490, 119498, 119488, 119489, 128331, - 8490, 128273, 9000, 128422, 983552, 983553, 983559, 983558, 983561, - 983560, 983557, 983556, 983554, 983563, 983555, 983562, 128287, 68163, - 68162, 68161, 68160, 68113, 68146, 68112, 68147, 68148, 68123, 68122, - 68128, 68127, 68126, 68121, 68131, 68125, 68124, 68130, 68129, 68141, - 68142, 68143, 68135, 68134, 68118, 68117, 68115, 68114, 68133, 68132, - 68149, 68140, 68145, 68119, 68139, 68136, 68138, 68137, 68144, 68096, - 68165, 68164, 68166, 68167, 68179, 68178, 68183, 68176, 68182, 68181, - 68184, 68180, 68177, 68152, 68153, 68109, 68154, 68111, 68110, 68099, - 68101, 68097, 68102, 68098, 68108, 68159, 68168, 129711, 101120, 101121, - 101122, 101123, 101124, 101125, 101126, 101127, 101128, 101129, 101130, - 101131, 101132, 101133, 101134, 101135, 101136, 101137, 101138, 101139, - 101140, 101141, 101142, 101143, 101144, 101145, 101146, 101147, 101148, - 101149, 101150, 101151, 101152, 101153, 101154, 101155, 101156, 101157, - 101158, 101159, 101160, 101161, 101162, 101163, 101164, 101165, 101166, - 101167, 101168, 101169, 101170, 101171, 101172, 101173, 101174, 101175, - 101176, 101177, 101178, 101179, 101180, 101181, 101182, 101183, 101184, - 101185, 101186, 101187, 101188, 101189, 101190, 101191, 101192, 101193, - 101194, 101195, 101196, 101197, 101198, 101199, 101200, 101201, 101202, - 101203, 101204, 101205, 101206, 101207, 101208, 101209, 101210, 101211, - 101212, 101213, 101214, 101215, 101216, 101217, 101218, 101219, 101220, - 101221, 101222, 101223, 101224, 101225, 101226, 101227, 101228, 101229, - 101230, 101231, 101232, 101233, 101234, 101235, 101236, 101237, 101238, - 101239, 101240, 101241, 101242, 101243, 101244, 101245, 101246, 101247, - 101248, 101249, 101250, 101251, 101252, 101253, 101254, 101255, 101256, - 101257, 101258, 101259, 101260, 101261, 101262, 101263, 101264, 101265, - 101266, 101267, 101268, 101269, 101270, 101271, 101272, 101273, 101274, - 101275, 101276, 101277, 101278, 101279, 101280, 101281, 101282, 101283, - 101284, 101285, 101286, 101287, 101288, 101289, 101290, 101291, 101292, - 101293, 101294, 101295, 101296, 101297, 101298, 101299, 101300, 101301, - 101302, 101303, 101304, 101305, 101306, 101307, 101308, 101309, 101310, - 101311, 101312, 101313, 101314, 101315, 101316, 101317, 101318, 101319, - 101320, 101321, 101322, 101323, 101324, 101325, 101326, 101327, 101328, - 101329, 101330, 101331, 101332, 101333, 101334, 101335, 101336, 101337, - 101338, 101339, 101340, 101341, 101342, 101343, 101344, 101345, 101346, - 101347, 101348, 101349, 101350, 101351, 101352, 101353, 101354, 101355, - 101356, 101357, 101358, 101359, 101360, 101361, 101362, 101363, 101364, - 101365, 101366, 101367, 101368, 101369, 101370, 101371, 101372, 101373, - 101374, 101375, 101584, 101585, 101586, 101587, 101588, 101589, 101376, - 101377, 101378, 101379, 101380, 101381, 101382, 101383, 101384, 101385, - 101386, 101387, 101388, 101389, 101390, 101391, 101392, 101393, 101394, - 101395, 101396, 101397, 101398, 101399, 101400, 101401, 101402, 101403, - 101404, 101405, 101406, 101407, 101408, 101409, 101410, 101411, 101412, - 101413, 101414, 101415, 101416, 101417, 101418, 101419, 101420, 101421, - 101422, 101423, 101424, 101425, 101426, 101427, 101428, 101429, 101430, - 101431, 101432, 101433, 101434, 101435, 101436, 101437, 101438, 101439, - 101440, 101441, 101442, 101443, 101444, 101445, 101446, 101447, 101448, - 101449, 101450, 101451, 101452, 101453, 101454, 101455, 101456, 101457, - 101458, 101459, 101460, 101461, 101462, 101463, 101464, 101465, 101466, - 101467, 101468, 101469, 101470, 101471, 101472, 101473, 101474, 101475, - 101476, 101477, 101478, 101479, 101480, 101481, 101482, 101483, 101484, - 101485, 101486, 101487, 101488, 101489, 101490, 101491, 101492, 101493, - 101494, 101495, 101496, 101497, 101498, 101499, 101500, 101501, 101502, - 101503, 101504, 101505, 101506, 101507, 101508, 101509, 101510, 101511, - 101512, 101513, 101514, 101515, 101516, 101517, 101518, 101519, 101520, - 101521, 101522, 101523, 101524, 101525, 101526, 101527, 101528, 101529, - 101530, 101531, 101532, 101533, 101534, 101535, 101536, 101537, 101538, - 101539, 101540, 101541, 101542, 101543, 101544, 101545, 101546, 101547, - 101548, 101549, 101550, 101551, 101552, 101553, 101554, 101555, 101556, - 101557, 101558, 101559, 101560, 101561, 101562, 101563, 101564, 101565, - 101566, 101567, 101568, 101569, 101570, 101571, 101572, 101573, 101574, - 101575, 101576, 101577, 101578, 101579, 101580, 101581, 101582, 101583, - 94180, 983960, 983965, 983970, 983975, 983962, 983964, 983961, 983963, - 983957, 983959, 983956, 983958, 983977, 983979, 983978, 983972, 983974, - 983967, 983969, 983971, 983973, 983966, 983968, 983989, 983983, 983985, - 983986, 983987, 983980, 983982, 983984, 983981, 983976, 983988, 6107, - 6052, 6064, 6051, 6067, 6066, 6065, 6055, 6057, 6058, 6056, 6053, 6054, - 6063, 983994, 983991, 983992, 983993, 6061, 6062, 6059, 6060, 6022, 6024, - 6021, 6023, 6017, 6019, 6016, 6018, 6020, 6030, 6025, 6035, 6037, 6039, - 6038, 6046, 6045, 6047, 6032, 6034, 6027, 6029, 6031, 6033, 6026, 6028, - 6049, 6043, 6040, 6042, 6044, 6041, 6036, 6048, 6050, 6108, 6109, 6095, - 6096, 6099, 6091, 6101, 6104, 6102, 6098, 6094, 6100, 6106, 6092, 6087, - 6093, 6090, 6103, 6105, 6088, 6086, 6089, 6097, 6652, 6636, 6655, 6639, - 6653, 6637, 6654, 6638, 6651, 6635, 6650, 6634, 6133, 6137, 6136, 6134, - 6135, 6130, 6132, 6131, 6129, 6128, 6624, 6648, 6632, 6649, 6633, 6647, - 6631, 6646, 6630, 6645, 6629, 6642, 6626, 6640, 6643, 6627, 6644, 6628, - 6641, 6625, 6069, 6068, 6070, 983996, 6082, 6083, 6085, 6071, 6080, 6072, - 6078, 983995, 6084, 6073, 6079, 6074, 983990, 6075, 6077, 6076, 6081, - 6117, 6116, 6119, 6118, 6115, 6114, 6112, 6121, 6113, 6120, 70204, 70201, - 70200, 70208, 70185, 70178, 70179, 70177, 70155, 70156, 70154, 70172, - 70167, 70166, 70173, 70171, 70161, 70160, 70144, 70145, 70149, 70151, - 70165, 70164, 70170, 70169, 70187, 70183, 70157, 70168, 70163, 70174, - 70159, 70158, 70153, 70152, 70176, 70175, 70186, 70180, 70207, 70182, - 70184, 70181, 70148, 70146, 70150, 70147, 70199, 70206, 70197, 70198, - 70196, 70203, 70209, 70188, 70193, 70195, 70189, 70190, 70192, 70194, - 70191, 70202, 70205, 70357, 70358, 70356, 70333, 70334, 70332, 70340, - 70339, 70338, 70345, 70347, 70344, 70352, 70351, 70346, 70361, 70320, - 70321, 70327, 70329, 70343, 70342, 70350, 70349, 70324, 70325, 70322, - 70323, 70335, 70348, 70341, 70353, 70337, 70336, 70331, 70330, 70355, - 70354, 70364, 70365, 70366, 70362, 70359, 70363, 70360, 70326, 70328, - 70378, 70377, 70367, 70368, 70374, 70376, 70371, 70372, 70369, 70370, - 70373, 70375, 70389, 70388, 70391, 70390, 70387, 70386, 70384, 70393, - 70385, 70392, 128143, 128535, 128537, 128538, 128573, 128139, 129373, - 8365, 128088, 129665, 129486, 129698, 12927, 128040, 11235, 129404, - 127991, 129357, 128030, 129692, 3805, 3804, 983206, 983207, 3743, 3741, - 3807, 3806, 3714, 3716, 3713, 983209, 3747, 3749, 3730, 3729, 3736, 3728, - 3727, 3731, 3726, 3744, 3721, 3718, 3724, 3756, 3740, 3742, 3739, 3752, - 3753, 3754, 3722, 3734, 3735, 3733, 3755, 3758, 3719, 3725, 3737, 3738, - 3720, 3732, 3745, 983208, 3751, 3746, 3757, 3773, 3772, 3770, 3785, 3786, - 3787, 3784, 3760, 3762, 3780, 3763, 3779, 3761, 3771, 3766, 3767, 3768, - 3769, 3776, 3777, 3764, 3765, 3778, 3782, 3790, 3759, 3797, 3796, 3799, - 3798, 3795, 3794, 3792, 3801, 3793, 3800, 3788, 3789, 128996, 129003, - 128309, 128311, 128998, 128994, 129001, 68413, 68415, 128992, 128310, - 128999, 68412, 68414, 11004, 10201, 10200, 10782, 128995, 129002, 128308, - 128997, 128993, 129000, 9711, 10923, 10925, 8382, 9790, 127772, 127767, - 65, 258, 7858, 7856, 7854, 7862, 7860, 550, 480, 7840, 512, 196, 478, - 197, 506, 7680, 256, 983564, 194, 7848, 7846, 7844, 7852, 7850, 461, - 7842, 260, 983590, 983592, 192, 193, 514, 195, 570, 198, 508, 482, 42946, - 11373, 393, 42808, 42810, 42802, 42804, 42806, 42812, 66, 386, 42902, - 7686, 7684, 7682, 385, 579, 42822, 42932, 67, 199, 7688, 264, 268, 262, - 42948, 391, 42898, 266, 571, 42960, 42796, 42798, 42862, 42931, 68, 498, - 453, 42951, 272, 7696, 7698, 270, 395, 7694, 7692, 7690, 394, 497, 452, - 69, 552, 7708, 202, 983586, 7874, 7872, 7870, 983584, 7878, 7876, 7704, - 282, 278, 983598, 983600, 7864, 516, 203, 7868, 7706, 274, 7700, 7702, - 983570, 983572, 983574, 7866, 280, 983594, 983596, 200, 201, 518, 276, - 582, 439, 494, 440, 42786, 42788, 42858, 208, 425, 330, 70, 401, 7710, - 42904, 71, 290, 284, 486, 500, 286, 7712, 42912, 403, 288, 484, 577, - 42938, 42940, 42942, 404, 983199, 72, 7722, 7720, 292, 542, 7718, 11367, - 7716, 7714, 42922, 294, 11381, 502, 42790, 73, 520, 7882, 304, 207, 7726, - 206, 463, 298, 983566, 296, 7724, 7880, 302, 983604, 983606, 204, 205, - 522, 300, 407, 42873, 42875, 42877, 42882, 42884, 42886, 406, 42860, 74, - 308, 42930, 983608, 584, 75, 310, 488, 42818, 11369, 7730, 42816, 42820, - 7728, 7732, 42914, 408, 76, 42925, 573, 11360, 7734, 7736, 11362, 319, - 456, 321, 315, 7740, 317, 42824, 313, 7738, 983610, 455, 77, 7742, 7746, - 7744, 983612, 11374, 7930, 7932, 42966, 78, 325, 7754, 327, 459, 544, - 7752, 413, 504, 323, 42896, 7750, 7748, 42916, 209, 458, 79, 42826, - 42828, 332, 7760, 7762, 415, 212, 7892, 7890, 7888, 7896, 7894, 465, 214, - 554, 558, 560, 7884, 524, 336, 490, 492, 216, 510, 213, 7758, 7756, 556, - 983576, 983578, 983580, 416, 7902, 7900, 7898, 7906, 7904, 7886, 210, - 211, 526, 334, 42944, 400, 390, 42934, 418, 42830, 546, 80, 42834, 11363, - 42832, 42836, 7764, 420, 7766, 81, 42840, 42838, 82, 342, 344, 7770, - 7772, 7768, 528, 11364, 983614, 340, 7774, 530, 42918, 588, 42842, 42997, - 42923, 42814, 398, 42844, 83, 352, 7782, 350, 536, 348, 346, 7780, 7778, - 7784, 7776, 42953, 11390, 983582, 42920, 42949, 586, 42926, 42891, 7838, - 42968, 42924, 399, 84, 354, 7792, 538, 356, 574, 7788, 7786, 7790, 430, - 428, 358, 42878, 11375, 11376, 42893, 42928, 42880, 412, 42929, 581, 222, - 42852, 42854, 388, 423, 444, 42794, 42792, 85, 219, 7798, 467, 220, 473, - 475, 471, 469, 7794, 532, 368, 7908, 431, 7916, 7914, 7912, 7920, 7918, - 7910, 362, 7802, 983568, 983620, 983622, 370, 983616, 983618, 360, 7800, - 7796, 217, 218, 534, 364, 366, 42936, 580, 433, 86, 42846, 7806, 7804, - 434, 42850, 42906, 42908, 42910, 42856, 42848, 87, 372, 7812, 7816, 7814, - 7808, 7810, 11378, 503, 88, 7820, 7818, 89, 374, 376, 7924, 7822, 7922, - 435, 7926, 7934, 221, 562, 7928, 590, 540, 90, 7824, 381, 377, 11371, - 7826, 379, 7828, 11391, 437, 42950, 548, 306, 338, 10013, 43007, 43005, - 43006, 43003, 43004, 42999, 450, 7461, 684, 664, 685, 662, 122638, 446, - 426, 674, 451, 122634, 7431, 7430, 7459, 618, 641, 671, 122628, 7436, - 7439, 7440, 630, 7445, 640, 7438, 7449, 43846, 42870, 7451, 11387, + 10827, 8253, 8890, 10812, 117901, 117903, 9688, 129942, 129969, 129972, + 9689, 129936, 11800, 11845, 11846, 8766, 9959, 161, 8487, 8276, 191, + 8290, 8292, 8291, 128229, 127982, 129311, 127875, 127983, 127971, 12292, + 9979, 127886, 128304, 128122, 128121, 43453, 43454, 43455, 43457, 43421, + 43422, 43426, 43427, 43398, 43397, 43399, 43407, 43408, 43409, 43412, + 43402, 43403, 43418, 43416, 43428, 43423, 43431, 43432, 43413, 43414, + 43410, 43411, 43429, 43430, 43401, 43435, 43436, 43441, 43439, 43440, + 43424, 43425, 43419, 43420, 43415, 43417, 43396, 43405, 43442, 43437, + 43433, 43438, 43434, 43404, 43406, 43400, 43458, 43466, 43467, 43459, + 43465, 43461, 43464, 43468, 43463, 43486, 43462, 43487, 43460, 43471, + 43456, 43469, 43393, 43443, 43392, 43395, 43394, 43448, 43449, 43444, + 43450, 43445, 43446, 43447, 43452, 43451, 43477, 43476, 43479, 43478, + 43475, 43474, 43472, 43481, 43473, 43480, 129753, 129724, 128086, 128377, + 10781, 129337, 9795, 9909, 129513, 69786, 69787, 69785, 69793, 69792, + 69763, 69764, 69770, 69772, 69784, 69783, 69791, 69790, 69767, 69768, + 69765, 69766, 69777, 69789, 69782, 69794, 69804, 69805, 69806, 69798, + 69797, 69779, 69778, 69776, 69775, 69781, 69780, 69774, 69773, 69796, + 69795, 69788, 69801, 69807, 69802, 69799, 69803, 69800, 69769, 69771, + 69760, 69818, 69761, 69817, 69762, 69822, 69823, 69825, 69824, 69826, + 69808, 69814, 69816, 69811, 69812, 69809, 69810, 69813, 69815, 69821, + 69837, 69819, 69820, 12142, 12164, 12060, 12157, 12100, 12149, 12184, + 12191, 12227, 12068, 12174, 12234, 12205, 12134, 12168, 12219, 12189, + 12088, 12090, 12096, 12160, 12182, 12224, 12190, 12147, 12114, 12118, + 12058, 12176, 12075, 12112, 12045, 12170, 12124, 12070, 12194, 12109, + 12229, 12196, 12139, 12056, 12099, 12034, 12084, 12136, 12120, 12044, + 12111, 12094, 12125, 12243, 12238, 12082, 12159, 12063, 12215, 12062, + 12042, 12241, 12067, 12235, 12043, 12140, 12119, 12207, 12123, 12133, + 12222, 12117, 12226, 12245, 12214, 12217, 12236, 12155, 12188, 12113, + 12065, 12198, 12066, 12146, 12171, 12225, 12200, 12121, 12093, 12095, + 12221, 12092, 12216, 12231, 12054, 12218, 12179, 12037, 12173, 12072, + 12046, 12127, 12152, 12049, 12074, 12107, 12208, 12212, 12041, 12210, + 12131, 12033, 12039, 12199, 12085, 12128, 12161, 12162, 12233, 12165, + 12192, 12077, 12201, 12061, 12105, 12040, 12240, 12102, 12153, 12032, + 12080, 12167, 12048, 12156, 12059, 12126, 12158, 12050, 12183, 12204, + 12097, 12239, 12053, 12078, 12150, 12071, 12187, 12186, 12223, 12228, + 12104, 12098, 12064, 12036, 12057, 12163, 12178, 12185, 12154, 12203, + 12083, 12087, 12135, 12151, 12202, 12035, 12122, 12141, 12180, 12144, + 12076, 12052, 12115, 12091, 12108, 12169, 12143, 12148, 12130, 12089, + 12211, 12073, 12101, 12138, 12103, 12209, 12047, 12220, 12172, 12129, + 12166, 12242, 12237, 12145, 12106, 12081, 12244, 12038, 12086, 12055, + 12181, 12197, 12193, 12175, 12116, 12110, 12177, 12137, 12230, 12213, + 12195, 12069, 12079, 12206, 12051, 12232, 12132, 129432, 3240, 3293, + 3225, 3235, 3230, 3205, 3206, 3216, 3220, 3234, 3233, 3239, 3238, 983205, + 3251, 3250, 3249, 3248, 3232, 3231, 3237, 3236, 3211, 3296, 3212, 3297, + 3253, 3209, 3210, 3218, 3219, 3207, 3208, 3254, 3255, 3256, 3245, 3244, + 3227, 3226, 3224, 3223, 3229, 3228, 3222, 3221, 3243, 3242, 3214, 3215, + 3294, 3257, 3246, 3247, 3285, 3315, 3201, 3200, 3204, 3260, 3261, 3202, + 3313, 3314, 3277, 3203, 3286, 3262, 3272, 3276, 3265, 3266, 3267, 3268, + 3298, 3299, 3274, 3275, 3263, 3264, 3270, 3271, 3307, 3306, 3309, 3308, + 3305, 3304, 3302, 3311, 3303, 3310, 12450, 984009, 984008, 984007, + 984010, 110881, 110880, 110882, 110592, 12499, 984002, 984005, 984003, + 984006, 984004, 12496, 12505, 12508, 12502, 12511, 110583, 110584, + 110585, 110586, 110587, 110589, 110590, 110576, 110577, 110578, 110579, + 110581, 110582, 12510, 12513, 12514, 12512, 12531, 12490, 12493, 12491, + 12494, 12492, 12789, 12792, 12790, 12793, 12791, 12795, 12798, 12796, + 12799, 12797, 12533, 12534, 110933, 12784, 12787, 12483, 12526, 110949, + 110948, 110950, 12515, 12519, 12517, 110951, 12788, 12785, 12786, 12794, + 12449, 12455, 12451, 12457, 12453, 12469, 12475, 12471, 12477, 12473, + 12480, 12487, 12482, 12489, 12485, 12460, 12466, 12462, 12468, 12464, + 12495, 12504, 12498, 12507, 12501, 12459, 12465, 12461, 12467, 12463, + 12497, 12506, 12500, 12509, 12503, 12521, 12524, 12522, 12525, 12523, + 12479, 12486, 12481, 12488, 12484, 12535, 12537, 12536, 12538, 12532, + 12470, 12476, 12472, 12478, 12474, 12527, 12529, 12528, 12530, 12516, + 12520, 12518, 12456, 12452, 12458, 12454, 12542, 12543, 12541, 12539, + 12448, 12540, 12444, 12443, 73476, 73477, 73487, 73523, 73498, 73497, + 73503, 73502, 73508, 73507, 73501, 73500, 73506, 73505, 73480, 73481, + 73482, 73483, 73484, 73485, 73478, 73479, 73494, 73504, 73499, 73509, + 73519, 73520, 73521, 73513, 73512, 73496, 73495, 73493, 73492, 73491, + 73490, 73511, 73510, 73522, 73517, 73514, 73516, 73518, 73515, 73486, + 73488, 73551, 73548, 73546, 73545, 73549, 73543, 73541, 73544, 73550, + 73542, 73547, 73537, 73472, 73562, 73474, 73475, 73473, 73525, 73524, + 73535, 73530, 73534, 73536, 73528, 73529, 73526, 73527, 73540, 73539, + 73557, 73556, 73559, 73558, 73555, 73554, 73552, 73561, 73553, 73560, + 73538, 43299, 43301, 43283, 43295, 43277, 43281, 43284, 43275, 43274, + 43286, 43285, 43279, 43278, 43294, 43282, 43289, 43297, 43288, 43276, + 43292, 43287, 43290, 43296, 43293, 43291, 43280, 43298, 43300, 43310, + 43311, 43308, 43309, 43307, 43303, 43305, 43304, 43302, 43306, 43269, + 43268, 43271, 43270, 43267, 43266, 43264, 43273, 43265, 43272, 119496, + 119506, 119499, 119503, 119493, 119492, 119502, 119497, 119507, 119495, + 119505, 119494, 119504, 119501, 119491, 119500, 119490, 119498, 119488, + 119489, 128331, 8490, 128273, 9000, 128422, 983552, 983553, 983559, + 983558, 983561, 983560, 983557, 983556, 983554, 983563, 983555, 983562, + 128287, 118449, 68163, 68162, 68161, 68160, 68113, 68146, 68112, 68147, + 68148, 68123, 68122, 68128, 68127, 68126, 68121, 68131, 68125, 68124, + 68130, 68129, 68141, 68142, 68143, 68135, 68134, 68118, 68117, 68115, + 68114, 68133, 68132, 68149, 68140, 68145, 68119, 68139, 68136, 68138, + 68137, 68144, 68096, 68165, 68164, 68166, 68167, 68179, 68178, 68183, + 68176, 68182, 68181, 68184, 68180, 68177, 68152, 68153, 68109, 68154, + 68111, 68110, 68099, 68101, 68097, 68102, 68098, 68108, 68159, 68168, + 129711, 101120, 101121, 101122, 101123, 101124, 101125, 101126, 101127, + 101128, 101129, 101130, 101131, 101132, 101133, 101134, 101135, 101136, + 101137, 101138, 101139, 101140, 101141, 101142, 101143, 101144, 101145, + 101146, 101147, 101148, 101149, 101150, 101151, 101152, 101153, 101154, + 101155, 101156, 101157, 101158, 101159, 101160, 101161, 101162, 101163, + 101164, 101165, 101166, 101167, 101168, 101169, 101170, 101171, 101172, + 101173, 101174, 101175, 101176, 101177, 101178, 101179, 101180, 101181, + 101182, 101183, 101184, 101185, 101186, 101187, 101188, 101189, 101190, + 101191, 101192, 101193, 101194, 101195, 101196, 101197, 101198, 101199, + 101200, 101201, 101202, 101203, 101204, 101205, 101206, 101207, 101208, + 101209, 101210, 101211, 101212, 101213, 101214, 101215, 101216, 101217, + 101218, 101219, 101220, 101221, 101222, 101223, 101224, 101225, 101226, + 101227, 101228, 101229, 101230, 101231, 101232, 101233, 101234, 101235, + 101236, 101237, 101238, 101239, 101240, 101241, 101242, 101243, 101244, + 101245, 101246, 101247, 101248, 101249, 101250, 101251, 101252, 101253, + 101254, 101255, 101256, 101257, 101258, 101259, 101260, 101261, 101262, + 101263, 101264, 101265, 101266, 101267, 101268, 101269, 101270, 101271, + 101272, 101273, 101274, 101275, 101276, 101277, 101278, 101279, 101280, + 101281, 101282, 101283, 101284, 101285, 101286, 101287, 101288, 101289, + 101290, 101291, 101292, 101293, 101294, 101295, 101296, 101297, 101298, + 101299, 101300, 101301, 101302, 101303, 101304, 101305, 101306, 101307, + 101308, 101309, 101310, 101311, 101312, 101313, 101314, 101315, 101316, + 101317, 101318, 101319, 101320, 101321, 101322, 101323, 101324, 101325, + 101326, 101327, 101328, 101329, 101330, 101331, 101332, 101333, 101334, + 101335, 101336, 101337, 101338, 101339, 101340, 101341, 101342, 101343, + 101344, 101345, 101346, 101347, 101348, 101349, 101350, 101351, 101352, + 101353, 101354, 101355, 101356, 101357, 101358, 101359, 101360, 101361, + 101362, 101363, 101364, 101365, 101366, 101367, 101368, 101369, 101370, + 101371, 101372, 101373, 101374, 101375, 101584, 101585, 101586, 101587, + 101588, 101589, 101376, 101377, 101378, 101379, 101380, 101381, 101382, + 101383, 101384, 101385, 101386, 101387, 101388, 101389, 101390, 101391, + 101392, 101393, 101394, 101395, 101396, 101397, 101398, 101399, 101400, + 101401, 101402, 101403, 101404, 101405, 101406, 101407, 101408, 101409, + 101410, 101411, 101412, 101413, 101414, 101415, 101416, 101417, 101418, + 101419, 101420, 101421, 101422, 101423, 101424, 101425, 101426, 101427, + 101428, 101429, 101430, 101431, 101432, 101433, 101434, 101435, 101436, + 101437, 101438, 101439, 101440, 101441, 101442, 101443, 101444, 101445, + 101446, 101447, 101448, 101449, 101450, 101451, 101452, 101453, 101454, + 101455, 101456, 101457, 101458, 101459, 101460, 101461, 101462, 101463, + 101464, 101465, 101466, 101467, 101468, 101469, 101470, 101471, 101472, + 101473, 101474, 101475, 101476, 101477, 101478, 101479, 101480, 101481, + 101482, 101483, 101484, 101485, 101486, 101487, 101488, 101489, 101490, + 101491, 101492, 101493, 101494, 101495, 101496, 101497, 101498, 101499, + 101500, 101501, 101502, 101503, 101504, 101505, 101506, 101507, 101508, + 101509, 101510, 101511, 101512, 101513, 101514, 101515, 101516, 101517, + 101518, 101519, 101520, 101521, 101522, 101523, 101524, 101525, 101526, + 101527, 101528, 101529, 101530, 101531, 101532, 101533, 101534, 101535, + 101536, 101537, 101538, 101539, 101540, 101541, 101542, 101543, 101544, + 101545, 101546, 101547, 101548, 101549, 101550, 101551, 101552, 101553, + 101554, 101555, 101556, 101557, 101558, 101559, 101560, 101561, 101562, + 101563, 101564, 101565, 101566, 101567, 101568, 101569, 101570, 101571, + 101572, 101573, 101574, 101575, 101576, 101577, 101578, 101579, 101580, + 101581, 101582, 101583, 101631, 94180, 983960, 983965, 983970, 983975, + 983962, 983964, 983961, 983963, 983957, 983959, 983956, 983958, 983977, + 983979, 983978, 983972, 983974, 983967, 983969, 983971, 983973, 983966, + 983968, 983989, 983983, 983985, 983986, 983987, 983980, 983982, 983984, + 983981, 983976, 983988, 6107, 6052, 6064, 6051, 6067, 6066, 6065, 6055, + 6057, 6058, 6056, 6053, 6054, 6063, 983994, 983991, 983992, 983993, 6061, + 6062, 6059, 6060, 6022, 6024, 6021, 6023, 6017, 6019, 6016, 6018, 6020, + 6030, 6025, 6035, 6037, 6039, 6038, 6046, 6045, 6047, 6032, 6034, 6027, + 6029, 6031, 6033, 6026, 6028, 6049, 6043, 6040, 6042, 6044, 6041, 6036, + 6048, 6050, 6108, 6109, 6095, 6096, 6099, 6091, 6101, 6104, 6102, 6098, + 6094, 6100, 6106, 6092, 6087, 6093, 6090, 6103, 6105, 6088, 6086, 6089, + 6097, 6652, 6636, 6655, 6639, 6653, 6637, 6654, 6638, 6651, 6635, 6650, + 6634, 6133, 6137, 6136, 6134, 6135, 6130, 6132, 6131, 6129, 6128, 6624, + 6648, 6632, 6649, 6633, 6647, 6631, 6646, 6630, 6645, 6629, 6642, 6626, + 6640, 6643, 6627, 6644, 6628, 6641, 6625, 6069, 6068, 6070, 983996, 6082, + 6083, 6085, 6071, 6080, 6072, 6078, 983995, 6084, 6073, 6079, 6074, + 983990, 6075, 6077, 6076, 6081, 6117, 6116, 6119, 6118, 6115, 6114, 6112, + 6121, 6113, 6120, 70204, 70201, 70200, 70208, 70185, 70178, 70179, 70177, + 70155, 70156, 70154, 70172, 70167, 70166, 70173, 70171, 70161, 70160, + 70144, 70145, 70149, 70151, 70165, 70164, 70170, 70169, 70187, 70183, + 70157, 70168, 70163, 70174, 70159, 70158, 70153, 70152, 70176, 70175, + 70186, 70180, 70207, 70182, 70184, 70181, 70148, 70146, 70150, 70147, + 70199, 70206, 70198, 70197, 70196, 70203, 70209, 70188, 70193, 70195, + 70189, 70190, 70192, 70194, 70191, 70202, 70205, 70357, 70358, 70356, + 70333, 70334, 70332, 70345, 70347, 70344, 70352, 70351, 70340, 70339, + 70338, 70320, 70321, 70327, 70329, 70346, 70361, 70343, 70342, 70350, + 70349, 70324, 70325, 70322, 70323, 70335, 70348, 70341, 70353, 70337, + 70336, 70331, 70330, 70355, 70354, 70364, 70365, 70366, 70362, 70359, + 70363, 70360, 70326, 70328, 70377, 70378, 70367, 70368, 70374, 70376, + 70371, 70372, 70369, 70370, 70373, 70375, 70389, 70388, 70391, 70390, + 70387, 70386, 70384, 70393, 70385, 70392, 93521, 93520, 93525, 93524, + 93519, 93518, 93523, 93522, 93512, 93517, 93526, 93530, 93529, 93514, + 93513, 93511, 93510, 93516, 93515, 93509, 93508, 93528, 93527, 93537, + 93536, 93538, 93534, 93531, 93533, 93535, 93532, 93507, 93505, 93549, + 93548, 93504, 93547, 93506, 93539, 93544, 93546, 93541, 93542, 93543, + 93540, 93545, 93551, 93550, 93557, 93556, 93559, 93558, 93555, 93554, + 93552, 93561, 93553, 93560, 128143, 128535, 128537, 128538, 128573, + 128139, 129373, 8365, 128088, 129665, 129486, 129698, 12927, 128040, + 11235, 129404, 127991, 129357, 128030, 129692, 3805, 3804, 983206, + 983207, 3743, 3741, 3807, 3806, 3714, 3716, 3713, 983209, 3747, 3749, + 3730, 3729, 3736, 3728, 3727, 3731, 3726, 3744, 3721, 3718, 3724, 3756, + 3740, 3742, 3739, 3752, 3753, 3754, 3722, 3734, 3735, 3733, 3755, 3758, + 3719, 3725, 3737, 3738, 3720, 3732, 3745, 983208, 3751, 3746, 3757, 3773, + 3772, 3770, 3785, 3786, 3787, 3784, 3760, 3762, 3780, 3763, 3779, 3761, + 3771, 3766, 3767, 3768, 3769, 3776, 3777, 3764, 3765, 3778, 3782, 3790, + 3759, 3797, 3796, 3799, 3798, 3795, 3794, 3792, 3801, 3793, 3800, 3788, + 3789, 128996, 129003, 128309, 128311, 128998, 128994, 129001, 68413, + 68415, 128992, 128310, 128999, 68412, 68414, 118324, 118319, 118322, + 118323, 118303, 118304, 118336, 118331, 118328, 118315, 118314, 118306, + 118316, 118318, 118334, 118335, 118333, 118327, 118339, 118341, 118342, + 118340, 118320, 118338, 118332, 118302, 118325, 118309, 118317, 118307, + 118313, 118326, 118329, 118312, 118330, 118352, 118348, 118351, 118350, + 118347, 118344, 118345, 118343, 118349, 118346, 118305, 118321, 118301, + 118299, 118298, 118310, 118311, 118308, 118300, 118337, 11004, 10201, + 10200, 10782, 128995, 129002, 128308, 128997, 128993, 129000, 9711, + 10923, 10925, 8382, 9790, 127772, 127767, 65, 258, 7858, 7856, 7854, + 7862, 7860, 550, 480, 7840, 512, 196, 478, 197, 506, 7680, 256, 983564, + 194, 7848, 7846, 7844, 7852, 7850, 461, 7842, 260, 983590, 983592, 192, + 193, 514, 195, 570, 198, 508, 482, 42946, 393, 11373, 42808, 42810, + 42802, 42804, 42806, 42812, 66, 386, 42902, 7686, 7684, 7682, 385, 579, + 42822, 42932, 67, 199, 7688, 264, 268, 262, 42948, 391, 42898, 266, 571, + 42960, 42796, 42798, 42862, 42931, 68, 498, 453, 42951, 272, 7696, 7698, + 270, 395, 7694, 7692, 7690, 394, 497, 452, 69, 552, 7708, 202, 983586, + 7874, 7872, 7870, 983584, 7878, 7876, 7704, 282, 278, 983598, 983600, + 7864, 516, 203, 7868, 7706, 274, 7700, 7702, 983570, 983572, 983574, + 7866, 280, 983594, 983596, 200, 201, 518, 276, 582, 439, 494, 440, 42786, + 42788, 42858, 208, 425, 330, 70, 401, 7710, 42904, 71, 290, 284, 486, + 500, 286, 7712, 42912, 403, 288, 484, 577, 42938, 42940, 42942, 404, + 983199, 72, 7722, 7720, 292, 542, 7718, 11367, 7716, 7714, 42922, 294, + 11381, 502, 42790, 73, 520, 7882, 304, 207, 7726, 206, 463, 298, 983566, + 296, 7724, 7880, 302, 983604, 983606, 204, 205, 522, 300, 407, 42873, + 42875, 42877, 42882, 42884, 42886, 406, 42860, 74, 308, 42930, 983608, + 584, 75, 310, 488, 42818, 11369, 7730, 42816, 42820, 7728, 7732, 42914, + 408, 76, 42925, 573, 11360, 7734, 7736, 11362, 319, 456, 321, 315, 7740, + 317, 42824, 313, 7738, 983610, 42970, 42972, 455, 77, 7742, 7746, 7744, + 983612, 11374, 7930, 7932, 42966, 78, 325, 7754, 327, 459, 544, 7752, + 413, 504, 323, 42896, 7750, 7748, 42916, 209, 458, 79, 42826, 42828, 332, + 7760, 7762, 415, 212, 7892, 7890, 7888, 7896, 7894, 465, 214, 554, 558, + 560, 7884, 524, 336, 490, 492, 216, 510, 213, 7758, 7756, 556, 983576, + 983578, 983580, 416, 7902, 7900, 7898, 7906, 7904, 7886, 210, 211, 526, + 334, 42944, 400, 390, 42934, 418, 42830, 546, 80, 42834, 11363, 42832, + 42836, 7764, 420, 7766, 81, 42840, 42838, 82, 342, 344, 7770, 7772, 7768, + 528, 11364, 983614, 340, 7774, 530, 42918, 588, 42842, 42997, 42923, + 42814, 398, 42844, 42955, 83, 352, 7782, 350, 536, 348, 346, 7780, 7778, + 7784, 7776, 42956, 42953, 11390, 983582, 42920, 42949, 586, 42926, 42891, + 7838, 42968, 42924, 399, 84, 354, 7792, 538, 356, 574, 7788, 7786, 7790, + 430, 428, 358, 42878, 11375, 11376, 42893, 42928, 42880, 412, 42929, 581, + 222, 42852, 42854, 388, 423, 444, 42794, 42792, 85, 219, 7798, 467, 220, + 473, 475, 471, 469, 7794, 532, 368, 7908, 431, 7916, 7914, 7912, 7920, + 7918, 7910, 362, 7802, 983568, 983620, 983622, 370, 983616, 983618, 360, + 7800, 7796, 217, 218, 534, 364, 366, 42936, 580, 433, 86, 42846, 7806, + 7804, 434, 42850, 42906, 42908, 42910, 42856, 42848, 87, 372, 7812, 7816, + 7814, 7808, 7810, 11378, 503, 88, 7820, 7818, 89, 374, 376, 7924, 7822, + 7922, 435, 7926, 7934, 221, 562, 7928, 590, 540, 90, 7824, 381, 377, + 11371, 7826, 379, 7828, 11391, 437, 42950, 548, 306, 338, 10013, 43007, + 43005, 43006, 43003, 43004, 42999, 450, 7461, 684, 664, 685, 662, 122638, + 446, 426, 674, 451, 122634, 7431, 7430, 7459, 618, 641, 671, 122628, + 7436, 7439, 7440, 630, 7445, 640, 7438, 7449, 43846, 42870, 7451, 11387, 122626, 122640, 43002, 7450, 665, 7427, 610, 667, 7424, 7425, 7428, 7429, 42800, 668, 7434, 7435, 7437, 628, 7448, 42927, 42801, 7452, 7456, 7457, - 655, 7458, 663, 122639, 42895, 447, 443, 448, 449, 660, 673, 661, 7460, + 655, 7458, 663, 122639, 42895, 443, 447, 448, 449, 660, 673, 661, 7460, 422, 7547, 7550, 97, 259, 7859, 7857, 7855, 7863, 7861, 551, 481, 7841, 513, 228, 479, 229, 507, 7681, 7834, 7567, 257, 983565, 226, 7849, 7847, 7845, 7853, 7851, 462, 7843, 261, 983591, 983593, 224, 225, 515, 227, @@ -10776,400 +11529,411 @@ static const unsigned int dawg_pos_to_codepoint[] = { 669, 496, 983609, 585, 107, 311, 489, 42819, 11370, 7731, 42817, 42821, 7729, 7733, 7556, 42915, 409, 312, 108, 620, 122643, 410, 316, 7741, 564, 318, 7735, 7737, 43832, 11361, 619, 43833, 320, 122662, 42825, 314, 7739, - 43831, 621, 42894, 122641, 7557, 983611, 322, 411, 622, 122629, 43829, - 383, 7836, 7835, 7837, 682, 683, 42866, 457, 109, 7743, 43834, 7535, - 7558, 7747, 7745, 983613, 625, 7931, 7933, 42967, 42867, 110, 326, 7755, - 43835, 565, 328, 414, 7753, 626, 122663, 7536, 505, 324, 42897, 7751, - 7749, 7559, 627, 42917, 241, 329, 983589, 42868, 460, 111, 244, 7893, - 7891, 7889, 7897, 7895, 466, 246, 555, 559, 561, 7885, 525, 337, 42827, - 11386, 42829, 333, 7761, 7763, 491, 493, 248, 511, 245, 7759, 7757, 557, - 983577, 983579, 983581, 417, 7903, 7901, 7899, 7907, 7905, 7887, 242, - 243, 527, 335, 122651, 42945, 596, 983625, 983626, 7575, 43839, 43874, - 603, 7571, 42935, 419, 42831, 547, 112, 42835, 7549, 42833, 42837, 7765, - 7537, 7560, 421, 7767, 632, 113, 42841, 672, 587, 42839, 569, 114, 343, - 43849, 345, 7771, 7773, 7769, 529, 638, 7539, 122646, 7775, 636, 637, - 983615, 122664, 7538, 341, 531, 7561, 42919, 589, 43847, 42843, 42998, - 604, 7572, 605, 639, 122625, 600, 122631, 8580, 42815, 122627, 42869, - 42845, 612, 115, 347, 7781, 353, 7783, 351, 537, 349, 122654, 7779, 7785, - 7777, 42954, 575, 983583, 122665, 7540, 7562, 42921, 642, 42892, 43872, - 601, 983629, 983630, 7573, 602, 43851, 43852, 609, 43830, 223, 7441, - 7442, 7443, 7455, 7454, 7453, 42969, 645, 43845, 116, 355, 7793, 539, - 566, 357, 11366, 7831, 7789, 7787, 122666, 7541, 7791, 427, 429, 122633, - 648, 359, 679, 122647, 122652, 7546, 254, 42853, 42855, 389, 424, 445, - 7446, 42795, 397, 613, 686, 687, 7433, 42879, 7444, 43842, 43841, 43843, - 43844, 7432, 633, 43880, 122645, 634, 122632, 11385, 635, 647, 122637, - 652, 983627, 983628, 592, 594, 7426, 623, 624, 654, 122630, 43857, 477, - 7543, 670, 42881, 653, 42871, 680, 678, 43879, 11383, 42793, 117, 649, - 43855, 251, 7799, 468, 252, 474, 476, 472, 470, 7795, 533, 369, 7909, - 432, 7917, 7915, 7913, 7921, 7919, 7911, 363, 7803, 983569, 983621, - 983623, 371, 983617, 983619, 7577, 367, 361, 7801, 7797, 249, 43854, - 42937, 250, 535, 365, 43858, 650, 7551, 7531, 43856, 42872, 43875, 118, - 42847, 7807, 7564, 11380, 11377, 7805, 651, 42851, 42907, 42909, 42911, - 42857, 42849, 119, 373, 7813, 7817, 7815, 7809, 7811, 7832, 11379, 120, - 7821, 7819, 43863, 43864, 43865, 43862, 7565, 121, 375, 255, 7925, 7823, - 7923, 436, 7927, 7935, 43866, 591, 253, 563, 7929, 7833, 541, 122, 378, - 7825, 657, 382, 11372, 7827, 380, 7829, 576, 438, 7542, 7566, 656, 549, - 64256, 64259, 64260, 64257, 64258, 307, 64261, 64262, 339, 8347, 8340, - 8336, 8337, 8341, 7522, 11388, 8342, 8343, 8344, 8345, 8338, 8346, 7523, - 8348, 7524, 7525, 8339, 917505, 127811, 129388, 129897, 129916, 129899, - 129917, 129947, 10203, 10202, 128494, 12296, 10641, 10643, 11058, 11056, - 10576, 10571, 10570, 10574, 12304, 10647, 123, 9128, 9129, 9127, 12300, - 8968, 9948, 10714, 12298, 8220, 11816, 11780, 129287, 129284, 129285, - 129283, 129286, 9686, 11240, 9612, 129977, 129970, 129940, 129932, - 128379, 128709, 11804, 10204, 10197, 9614, 9615, 8596, 8700, 8697, 8622, - 10568, 8660, 10500, 8654, 8621, 11012, 8703, 11020, 129112, 11108, 11788, - 10181, 8907, 9609, 8216, 11814, 128488, 91, 9123, 9121, 10639, 10637, - 8261, 10635, 11863, 11861, 9122, 11778, 10703, 129900, 11785, 129985, - 128492, 9613, 9610, 12308, 8867, 12302, 10627, 12310, 10629, 12314, - 12312, 10712, 128398, 9611, 10620, 8970, 8905, 40, 9117, 9115, 9116, - 11808, 9144, 10553, 10154, 1422, 4054, 4056, 129307, 9958, 10748, 9001, - 171, 128269, 8294, 8237, 8234, 8206, 8592, 10563, 11074, 11083, 11082, - 10611, 129973, 10618, 10615, 11070, 8676, 8633, 10525, 11064, 8698, - 129032, 8619, 10566, 129040, 129024, 8602, 11024, 11025, 8610, 11066, - 11065, 129028, 129176, 129044, 8617, 8695, 8612, 10527, 8646, 10521, - 129192, 129184, 11144, 11013, 10603, 10599, 10590, 10582, 8637, 10594, - 10602, 10598, 10586, 10578, 8636, 8651, 129778, 129088, 129092, 11104, - 11136, 11130, 983241, 11174, 11172, 129064, 129060, 129056, 129072, - 129068, 11120, 11114, 11140, 129168, 10510, 8666, 129186, 11067, 11069, - 11068, 11244, 11061, 11060, 11062, 11063, 8606, 8678, 129172, 8604, 8656, - 10498, 8653, 10502, 10523, 10508, 8672, 129194, 129076, 129188, 8701, - 8647, 129783, 129190, 128620, 8668, 129080, 129104, 129084, 11077, 9804, - 128006, 7213, 7221, 7216, 7220, 7215, 7214, 7217, 7218, 7219, 7170, 7169, - 7168, 7184, 7183, 7182, 7247, 7193, 7180, 7188, 7187, 7186, 7185, 7172, - 7171, 7198, 7197, 7190, 7189, 7173, 7177, 7181, 7192, 7191, 7246, 7245, - 7179, 7178, 7175, 7174, 7201, 7200, 7176, 7196, 7195, 7199, 7202, 7194, - 7203, 7227, 7231, 7230, 7228, 7229, 7223, 7222, 7205, 7204, 7210, 7211, - 7208, 7209, 7206, 7212, 7207, 7237, 7236, 7239, 7238, 7235, 7234, 7232, - 7241, 7233, 7240, 10897, 10895, 10893, 10899, 10891, 10614, 10889, 10887, - 8922, 8934, 8808, 10918, 10920, 10885, 10877, 10881, 10883, 10879, 8818, - 8804, 8822, 8806, 10873, 10875, 8918, 60, 128210, 127898, 127819, 129461, - 128626, 128955, 128969, 128943, 11212, 128964, 10099, 128648, 10098, - 9617, 128937, 128949, 128978, 128960, 129653, 128910, 10072, 128930, - 128504, 9735, 128498, 128497, 6429, 6404, 6403, 6412, 6430, 6411, 6421, - 6410, 6405, 6415, 6425, 6426, 6427, 6419, 6418, 6407, 6406, 6414, 6413, - 6409, 6408, 6402, 6401, 6417, 6416, 6428, 6423, 6420, 6422, 6424, 6458, - 6457, 6459, 6464, 6449, 6452, 6450, 6448, 6456, 6454, 6453, 6455, 6451, - 6442, 6443, 6441, 6432, 6436, 6438, 6440, 6437, 6439, 6435, 6433, 6434, - 6400, 6468, 6469, 6475, 6474, 6477, 6476, 6473, 6472, 6470, 6479, 6471, - 6478, 13007, 10770, 10771, 10772, 983062, 8232, 983068, 983143, 67143, - 67146, 67151, 67165, 67166, 67167, 67157, 67158, 67159, 67160, 67161, - 67162, 67163, 67164, 67171, 67172, 67173, 67168, 67169, 67170, 67174, - 67175, 67176, 67177, 67178, 67179, 67230, 67231, 67180, 67181, 67182, - 67183, 67184, 67185, 67186, 67187, 67188, 67189, 67190, 67191, 67192, - 67193, 67194, 67195, 67196, 67197, 67198, 67199, 67200, 67201, 67202, - 67203, 67204, 67205, 67206, 67207, 67208, 67209, 67210, 67211, 67212, - 67213, 67214, 67215, 67216, 67217, 67218, 67219, 67220, 67221, 67222, - 67223, 67224, 67225, 67226, 67227, 67228, 67229, 67232, 67233, 67234, - 67235, 67236, 67237, 67238, 67239, 67240, 67241, 67242, 67243, 67244, - 67245, 67246, 67247, 67248, 67249, 67250, 67251, 67252, 67253, 67254, - 67255, 67256, 67257, 67258, 67259, 67260, 67261, 67262, 67263, 67264, - 67274, 67275, 67276, 67277, 67278, 67279, 67280, 67281, 67282, 67283, - 67284, 67285, 67286, 67287, 67288, 67289, 67290, 67291, 67292, 67293, - 67294, 67295, 67296, 67297, 67298, 67299, 67300, 67301, 67302, 67303, - 67304, 67265, 67266, 67267, 67268, 67269, 67270, 67271, 67272, 67273, - 67325, 67326, 67327, 67328, 67329, 67330, 67305, 67306, 67307, 67308, - 67309, 67310, 67311, 67312, 67313, 67314, 67315, 67316, 67317, 67318, - 67319, 67320, 67321, 67322, 67323, 67324, 67331, 67332, 67333, 67334, - 67335, 67336, 67337, 67338, 67349, 67350, 67351, 67352, 67353, 67354, - 67355, 67356, 67357, 67358, 67359, 67360, 67361, 67362, 67363, 67364, - 67365, 67366, 67367, 67368, 67378, 67379, 67380, 67381, 67382, 67369, - 67370, 67371, 67372, 67373, 67374, 67375, 67376, 67377, 67339, 67340, - 67341, 67342, 67343, 67344, 67345, 67346, 67347, 67348, 67401, 67404, - 67403, 67402, 67400, 67398, 67393, 67397, 67395, 67394, 67399, 67392, - 67396, 67406, 67408, 67409, 67410, 67407, 67405, 67411, 67413, 67412, - 67424, 67425, 67426, 67427, 67428, 67429, 67430, 67431, 67081, 67082, - 67083, 67084, 67085, 67087, 67088, 67089, 67090, 67091, 67092, 67093, - 67094, 67086, 67095, 67096, 67097, 67098, 67100, 67101, 67102, 67103, - 67104, 67105, 67106, 67107, 67108, 67109, 67110, 67111, 67112, 67113, - 67114, 67115, 67116, 67117, 67118, 67119, 67120, 67121, 67122, 67123, - 67124, 67125, 67126, 67127, 67128, 67129, 67130, 67131, 67132, 67133, - 67134, 67135, 67136, 67137, 67138, 67139, 67140, 67141, 67142, 67072, - 67073, 67074, 67075, 67076, 67077, 67078, 67079, 67080, 67145, 67147, - 67148, 67149, 67150, 67154, 67155, 67144, 67152, 67153, 67156, 67099, - 65667, 65668, 65669, 65670, 65671, 65672, 65673, 65675, 65674, 65677, - 65676, 65665, 65664, 65666, 65681, 65679, 65680, 65682, 65678, 65686, - 65685, 65687, 65693, 65690, 65691, 65692, 65694, 65703, 65696, 65695, - 65697, 65698, 65699, 65701, 65702, 65707, 65706, 65704, 65705, 65708, - 65709, 65710, 65711, 65712, 65713, 65719, 65717, 65714, 65715, 65716, - 65718, 65720, 65721, 65722, 65723, 65724, 65725, 65726, 65727, 65728, - 65729, 65731, 65730, 65732, 65733, 65737, 65734, 65735, 65736, 65738, - 65739, 65740, 65741, 65743, 65742, 65744, 65745, 65747, 65748, 65752, - 65749, 65750, 65751, 65753, 65754, 65755, 65756, 65757, 65779, 65780, - 65781, 65782, 65783, 65784, 65785, 65759, 65760, 65761, 65762, 65763, - 65764, 65765, 65766, 65767, 65768, 65769, 65770, 65771, 65772, 65773, - 65774, 65775, 65776, 65777, 65778, 65758, 65786, 65683, 65684, 65689, - 65688, 65700, 65746, 65561, 65543, 65587, 65582, 65589, 65536, 65541, - 65579, 65566, 65571, 65596, 65569, 65584, 65544, 65559, 65540, 65557, - 65560, 65580, 65606, 65600, 65599, 65577, 65562, 65538, 65573, 65563, - 65609, 65588, 65549, 65568, 65537, 65581, 65574, 65601, 65542, 65593, - 65583, 65594, 65552, 65547, 65605, 65578, 65545, 65585, 65586, 65546, - 65570, 65591, 65564, 65565, 65607, 65610, 65611, 65553, 65590, 65550, - 65539, 65576, 65608, 65551, 65554, 65592, 65603, 65597, 65567, 65572, - 65558, 65555, 65612, 65602, 65556, 65613, 65604, 65620, 65621, 65623, - 65624, 65626, 65627, 65628, 65629, 65622, 65616, 65617, 65619, 65618, - 65625, 128391, 128279, 128482, 128132, 42237, 42235, 42232, 42234, 42236, - 42233, 42206, 42205, 42197, 42196, 42204, 42195, 42228, 42229, 42230, - 42224, 42225, 42213, 42208, 42203, 42202, 42221, 42198, 42216, 42214, - 42200, 42199, 42194, 42193, 42219, 42210, 73648, 42220, 42211, 42212, - 42222, 42223, 42227, 42231, 42192, 42217, 42201, 42209, 42207, 42218, - 42215, 42226, 42238, 42239, 8356, 8374, 129409, 129422, 9806, 128274, - 983079, 983076, 128271, 8743, 10848, 10846, 10833, 10844, 10842, 10847, - 8744, 10841, 10851, 10850, 10834, 10845, 10843, 10982, 10188, 129688, - 10234, 10231, 10206, 10229, 10235, 10232, 10237, 11059, 10230, 10236, - 10233, 10238, 10239, 10205, 129524, 128884, 129719, 128140, 127977, - 128261, 129707, 12319, 8270, 11847, 11848, 95, 9691, 129935, 9604, 9697, - 128394, 129852, 129853, 129870, 129872, 129868, 129856, 129871, 129869, - 129854, 129873, 129855, 128395, 128396, 128393, 10559, 129951, 9722, - 128397, 9695, 10558, 128318, 10065, 129950, 9727, 129863, 129865, 129867, - 129864, 129861, 129866, 129859, 129862, 129860, 129857, 129858, 10195, - 10063, 9998, 9694, 9987, 9605, 9602, 9601, 9607, 9603, 9606, 129903, - 9674, 10208, 129438, 128557, 127853, 129523, 128886, 129729, 66190, - 66192, 66187, 66196, 66178, 66179, 66199, 66185, 66200, 66176, 66201, - 66177, 66202, 66191, 66193, 66181, 66180, 66203, 66182, 66186, 66189, - 66195, 66188, 66197, 66198, 66194, 66183, 66204, 66184, 67887, 67892, - 67881, 67895, 67891, 67886, 67872, 67893, 67876, 67894, 67883, 67896, - 67873, 67897, 67875, 67889, 67874, 67878, 67880, 67882, 67884, 67890, - 67885, 67888, 67877, 67879, 67903, 129317, 983226, 983234, 983224, - 983229, 8468, 129433, 983065, 129668, 129522, 129497, 69986, 69981, - 69991, 70002, 69997, 69985, 69984, 69990, 69989, 69983, 69982, 69988, - 69987, 69995, 69994, 69978, 69977, 69976, 69975, 69980, 69979, 69974, - 69973, 69993, 69992, 70001, 69998, 69996, 70000, 69999, 69968, 69971, - 69969, 69972, 69970, 70006, 70005, 70003, 70004, 127012, 127019, 127008, - 126990, 126999, 126976, 127005, 126987, 126996, 127004, 126986, 126995, - 126979, 127009, 126991, 127000, 127001, 126983, 126992, 127011, 127010, - 126977, 127007, 126989, 126998, 127006, 126988, 126997, 127015, 127014, - 127003, 126985, 126994, 127002, 126984, 126993, 126978, 126982, 127017, - 126981, 126980, 127016, 127018, 127013, 73464, 73442, 73451, 73448, - 73444, 73449, 73447, 73441, 73450, 73440, 73454, 73445, 73443, 73453, - 73456, 73446, 73455, 73452, 73457, 73463, 73461, 73459, 73462, 73460, - 73458, 128892, 3415, 3449, 3435, 3434, 3437, 3436, 3433, 3432, 3430, - 3439, 3431, 3438, 3419, 3420, 3446, 3417, 3422, 3416, 3447, 3444, 3443, - 3448, 3418, 3421, 3445, 3333, 3423, 3334, 3344, 3348, 3453, 3454, 3414, - 3451, 3450, 3452, 3455, 3412, 3413, 3355, 3354, 3406, 3362, 3361, 3367, - 3366, 3360, 3386, 3359, 3365, 3364, 3332, 3339, 3424, 3340, 3425, 3381, - 3369, 3363, 3353, 3358, 3368, 3380, 3379, 3378, 3377, 3376, 3337, 3338, - 3346, 3347, 3335, 3336, 3382, 3383, 3384, 3373, 3372, 3352, 3351, 3357, - 3356, 3350, 3349, 3371, 3370, 3342, 3343, 3385, 3374, 3375, 3441, 3442, - 3440, 3328, 3388, 3329, 3387, 3405, 3331, 3389, 3330, 3407, 3390, 3400, - 3404, 3395, 3396, 3426, 3427, 3393, 3394, 3402, 3403, 3391, 3392, 3398, - 3399, 9895, 9894, 9893, 9794, 10016, 129443, 128104, 128372, 129333, - 128114, 128115, 128378, 128107, 2122, 2121, 2133, 2120, 2126, 2132, 2129, - 2136, 2113, 2115, 2114, 2116, 2123, 2124, 2125, 2128, 2130, 2131, 2118, - 2134, 2117, 2112, 2127, 2119, 2135, 2139, 2138, 2142, 2137, 68288, 68314, - 68313, 68290, 68289, 68293, 68308, 68292, 68291, 68300, 68299, 68298, - 68297, 68306, 68304, 68320, 68318, 68323, 68312, 68317, 68322, 68309, - 68302, 68324, 68305, 68319, 68307, 68321, 68303, 68294, 68301, 68311, - 68295, 68316, 68315, 68310, 68331, 68335, 68334, 68333, 68332, 68340, - 68339, 68338, 68342, 68337, 68341, 68336, 68296, 68326, 68325, 128094, - 129469, 8380, 128368, 129389, 9967, 127809, 129671, 72835, 72834, 72827, - 72826, 72836, 72828, 72821, 72825, 72829, 72823, 72822, 72819, 72818, - 72831, 72830, 72844, 72845, 72838, 72839, 72840, 72832, 72820, 72846, - 72824, 72843, 72833, 72842, 72837, 72841, 72847, 72886, 72885, 72867, - 72866, 72859, 72858, 72868, 72860, 72853, 72857, 72861, 72855, 72854, - 72851, 72850, 72863, 72862, 72876, 72877, 72870, 72871, 72864, 72852, - 72878, 72856, 72875, 72865, 72874, 72869, 72873, 72879, 72880, 72883, - 72881, 72884, 72882, 72816, 72817, 9901, 129355, 73007, 72980, 72979, - 72983, 72982, 72988, 73008, 72987, 72960, 72961, 72968, 72971, 72985, - 72984, 72990, 72989, 72964, 72965, 72962, 72963, 73005, 72999, 73006, - 72973, 72972, 72976, 72986, 72981, 72991, 73001, 73002, 73003, 72995, - 72994, 72978, 72977, 72975, 72974, 72993, 72992, 73004, 72996, 72998, - 73000, 72997, 72966, 72969, 73031, 73030, 73027, 73028, 73026, 73025, - 73024, 73014, 73009, 73020, 73023, 73012, 73013, 73010, 73011, 73018, - 73021, 73029, 73045, 73044, 73047, 73046, 73043, 73042, 73040, 73049, - 73041, 73048, 186, 127405, 12348, 119811, 120778, 120491, 119827, 120495, - 120505, 120507, 119808, 120488, 119809, 120489, 119833, 120493, 119812, - 120492, 120494, 119814, 120490, 119816, 120496, 119818, 120497, 119819, - 120498, 119822, 120502, 120512, 119823, 120509, 120511, 120503, 119825, - 120504, 119826, 120506, 119828, 120508, 119810, 120510, 119820, 120499, - 119821, 120500, 119831, 120501, 119813, 119815, 119817, 119824, 119829, - 119830, 119832, 119837, 120779, 120517, 119834, 120514, 119835, 120515, - 119859, 120519, 119838, 120518, 120520, 119839, 120531, 119840, 120516, - 119842, 120522, 119844, 120523, 119845, 120524, 119848, 120528, 120538, - 119849, 120535, 120537, 120529, 119851, 120530, 119852, 120532, 119853, - 120521, 120533, 119854, 120534, 119836, 120536, 119846, 120525, 119847, - 120526, 119857, 120527, 119841, 119843, 119850, 119855, 119856, 119858, - 120016, 120017, 120018, 120019, 120020, 120021, 120022, 120023, 120024, - 120025, 120026, 120027, 120028, 120029, 120030, 120031, 120032, 120033, - 120034, 120035, 120036, 120037, 120038, 120039, 120040, 120041, 120042, - 120043, 120044, 120045, 120046, 120047, 120048, 120049, 120050, 120051, - 120052, 120053, 120054, 120055, 120056, 120057, 120058, 120059, 120060, - 120061, 120062, 120063, 120064, 120065, 120066, 120067, 119931, 120611, - 120621, 120623, 119912, 120604, 119913, 120605, 119937, 120609, 119915, - 120607, 119916, 120608, 120610, 119918, 120606, 119920, 120612, 119922, - 120613, 119923, 120614, 119926, 120618, 120628, 119927, 120625, 120627, - 120619, 119929, 120620, 119930, 120622, 119932, 120624, 119914, 120626, - 119924, 120615, 119925, 120616, 119935, 120617, 119917, 119919, 119921, - 119928, 119933, 119934, 119936, 120656, 120658, 120629, 120659, 120655, - 120661, 120660, 119938, 120630, 119939, 120631, 119963, 120635, 119941, - 120633, 119942, 120634, 120636, 119943, 120647, 119944, 120632, 119946, - 120638, 119948, 120639, 119949, 120640, 119952, 120644, 120654, 119953, - 120651, 120653, 120645, 119955, 120646, 119956, 120648, 119957, 120637, - 120649, 119958, 120650, 119940, 120652, 119950, 120641, 119951, 120642, - 119961, 120643, 119945, 119947, 119954, 119959, 119960, 119962, 120657, - 120540, 120542, 120513, 120543, 120539, 120545, 120544, 120541, 120172, - 120173, 120174, 120175, 120176, 120177, 120178, 120179, 120180, 120181, - 120182, 120183, 120184, 120185, 120186, 120187, 120188, 120189, 120190, - 120191, 120192, 120193, 120194, 120195, 120196, 120197, 120198, 120199, - 120200, 120201, 120202, 120203, 120204, 120205, 120206, 120207, 120208, - 120209, 120210, 120211, 120212, 120213, 120214, 120215, 120216, 120217, - 120218, 120219, 120220, 120221, 120222, 120223, 120787, 120786, 120789, - 120788, 120785, 120784, 120782, 120791, 120783, 120790, 120120, 120121, - 120123, 120124, 120125, 120126, 120128, 120129, 120130, 120131, 120132, - 120134, 120138, 120139, 120140, 120141, 120142, 120143, 120144, 120146, - 120147, 120148, 120149, 120150, 120151, 120152, 120153, 120154, 120155, - 120156, 120157, 120158, 120159, 120160, 120161, 120162, 120163, 120164, - 120165, 120166, 120167, 120168, 120169, 120170, 120171, 120797, 120796, - 120799, 120798, 120795, 120794, 120792, 120801, 120793, 120800, 120068, - 120069, 120071, 120072, 120073, 120074, 120077, 120078, 120079, 120080, - 120081, 120082, 120083, 120084, 120086, 120087, 120088, 120089, 120090, - 120091, 120092, 120094, 120095, 120096, 120097, 120098, 120099, 120100, - 120101, 120102, 120103, 120104, 120105, 120106, 120107, 120108, 120109, - 120110, 120111, 120112, 120113, 120114, 120115, 120116, 120117, 120118, - 120119, 10189, 119889, 120484, 120485, 120575, 119886, 120572, 119887, - 120573, 119911, 120577, 119890, 120576, 120578, 119891, 120589, 119892, - 120574, 119894, 120580, 119896, 120581, 119897, 120582, 119900, 120586, - 120596, 119901, 120593, 120595, 120587, 119903, 120588, 119904, 120590, - 119905, 120579, 120591, 119906, 120592, 119888, 120594, 119898, 120583, - 119899, 120584, 119909, 120585, 119895, 119902, 119907, 119908, 119910, - 119879, 120553, 120563, 120565, 119860, 120546, 119861, 120547, 119885, - 120551, 119863, 120549, 119864, 120550, 120552, 119866, 120548, 119868, - 120554, 119870, 120555, 119871, 120556, 119874, 120560, 120570, 119875, - 120567, 120569, 120561, 119877, 120562, 119878, 120564, 119880, 120566, - 119862, 120568, 119872, 120557, 119873, 120558, 119883, 120559, 119865, - 119867, 119869, 119876, 119881, 119882, 119884, 120598, 120600, 120571, - 120601, 120597, 120603, 120602, 120599, 120432, 120433, 120434, 120435, - 120436, 120437, 120438, 120439, 120440, 120441, 120442, 120443, 120444, - 120445, 120446, 120447, 120448, 120449, 120450, 120451, 120452, 120453, - 120454, 120455, 120456, 120457, 120458, 120459, 120460, 120461, 120462, - 120463, 120464, 120465, 120466, 120467, 120468, 120469, 120470, 120471, - 120472, 120473, 120474, 120475, 120476, 120477, 120478, 120479, 120480, - 120481, 120482, 120483, 120827, 120826, 120829, 120828, 120825, 120824, - 120822, 120831, 120823, 120830, 10215, 10221, 10219, 10217, 10223, 10187, - 10214, 10220, 10218, 10216, 10222, 120399, 120727, 120737, 120739, - 120380, 120720, 120381, 120721, 120405, 120725, 120383, 120723, 120384, - 120724, 120726, 120386, 120722, 120388, 120728, 120390, 120729, 120391, - 120730, 120394, 120734, 120744, 120395, 120741, 120743, 120735, 120397, - 120736, 120398, 120738, 120400, 120740, 120382, 120742, 120392, 120731, - 120393, 120732, 120403, 120733, 120385, 120387, 120389, 120396, 120401, - 120402, 120404, 120772, 120774, 120745, 120775, 120771, 120777, 120776, - 120406, 120746, 120407, 120747, 120431, 120751, 120409, 120749, 120410, - 120750, 120752, 120411, 120763, 120412, 120748, 120414, 120754, 120416, - 120755, 120417, 120756, 120420, 120760, 120770, 120421, 120767, 120769, - 120761, 120423, 120762, 120424, 120764, 120425, 120753, 120765, 120426, - 120766, 120408, 120768, 120418, 120757, 120419, 120758, 120429, 120759, - 120413, 120415, 120422, 120427, 120428, 120430, 120773, 120295, 120669, - 120679, 120681, 120276, 120662, 120277, 120663, 120301, 120667, 120279, - 120665, 120280, 120666, 120668, 120282, 120664, 120284, 120670, 120286, - 120671, 120287, 120672, 120290, 120676, 120686, 120291, 120683, 120685, - 120677, 120293, 120678, 120294, 120680, 120296, 120682, 120278, 120684, - 120288, 120673, 120289, 120674, 120299, 120675, 120281, 120283, 120285, - 120292, 120297, 120298, 120300, 120714, 120716, 120687, 120717, 120713, - 120719, 120718, 120302, 120688, 120303, 120689, 120327, 120693, 120305, - 120691, 120306, 120692, 120694, 120307, 120705, 120308, 120690, 120310, - 120696, 120312, 120697, 120313, 120698, 120316, 120702, 120712, 120317, - 120709, 120711, 120703, 120319, 120704, 120320, 120706, 120321, 120695, - 120707, 120322, 120708, 120304, 120710, 120314, 120699, 120315, 120700, - 120325, 120701, 120309, 120311, 120318, 120323, 120324, 120326, 120715, - 120817, 120816, 120819, 120818, 120815, 120814, 120812, 120821, 120813, - 120820, 120328, 120329, 120330, 120331, 120332, 120333, 120334, 120335, - 120336, 120337, 120338, 120339, 120340, 120341, 120342, 120343, 120344, - 120345, 120346, 120347, 120348, 120349, 120350, 120351, 120352, 120353, - 120354, 120355, 120356, 120357, 120358, 120359, 120360, 120361, 120362, - 120363, 120364, 120365, 120366, 120367, 120368, 120369, 120370, 120371, - 120372, 120373, 120374, 120375, 120376, 120377, 120378, 120379, 120224, - 120225, 120226, 120227, 120228, 120229, 120230, 120231, 120232, 120233, - 120234, 120235, 120236, 120237, 120238, 120239, 120240, 120241, 120242, - 120243, 120244, 120245, 120246, 120247, 120248, 120249, 120250, 120251, - 120252, 120253, 120254, 120255, 120256, 120257, 120258, 120259, 120260, - 120261, 120262, 120263, 120264, 120265, 120266, 120267, 120268, 120269, - 120270, 120271, 120272, 120273, 120274, 120275, 120807, 120806, 120809, - 120808, 120805, 120804, 120802, 120811, 120803, 120810, 119964, 119966, - 119967, 119970, 119973, 119974, 119977, 119978, 119979, 119980, 119982, - 119983, 119984, 119985, 119986, 119987, 119988, 119989, 119990, 119991, - 119992, 119993, 119995, 119997, 119998, 119999, 120000, 120001, 120002, - 120003, 120005, 120006, 120007, 120008, 120009, 120010, 120011, 120012, - 120013, 120014, 120015, 129481, 119528, 119538, 119531, 119535, 119525, - 119524, 119534, 119529, 119539, 119527, 119537, 119526, 119536, 119533, - 119523, 119532, 119522, 119530, 119520, 119521, 128470, 175, 8737, 10667, - 10666, 10671, 10669, 10670, 10668, 10665, 10664, 10651, 10653, 8798, - 127830, 129470, 129471, 93773, 93764, 93790, 93787, 983268, 93783, - 983267, 93782, 93772, 93766, 93791, 93779, 93789, 93786, 93776, 93777, - 93785, 93775, 93770, 93769, 93771, 93774, 93780, 93760, 93767, 93781, - 93788, 93761, 93768, 93778, 93762, 93763, 93784, 93765, 93847, 93827, - 93846, 93826, 93845, 93825, 93844, 93829, 93828, 93831, 93830, 93824, - 93833, 93832, 93837, 93836, 93834, 93842, 93835, 93838, 93839, 93843, - 93840, 93841, 93805, 93796, 93822, 93819, 983270, 93815, 983269, 93814, - 93804, 93798, 93823, 93811, 93821, 93818, 93808, 93809, 93817, 93807, - 93802, 93801, 93803, 93806, 93812, 93792, 93799, 93813, 93820, 93793, - 93800, 93810, 93794, 93795, 93816, 93797, 93849, 93850, 93848, 11859, - 11852, 11860, 128901, 9899, 10090, 10091, 128967, 128965, 128944, 10100, - 10088, 10092, 10101, 10089, 10093, 8287, 9900, 9618, 128971, 128950, - 128938, 128963, 128961, 10073, 128974, 128956, 9898, 128911, 128931, - 43761, 44013, 43762, 43760, 44011, 43994, 43989, 43974, 43746, 43993, - 43991, 43751, 43750, 43992, 43986, 43987, 43990, 43968, 43995, 43976, - 43973, 43999, 43977, 44001, 43752, 43747, 43972, 43998, 43984, 43969, - 43753, 43754, 43975, 44000, 43978, 43749, 43748, 43983, 44002, 43970, - 43996, 43971, 43997, 43981, 43988, 43979, 43985, 43980, 43982, 43744, - 43745, 44012, 43763, 43764, 44005, 43757, 43759, 43758, 44009, 44003, - 44007, 44006, 44004, 43755, 44008, 43756, 44010, 43765, 43766, 44021, - 44020, 44023, 44022, 44019, 44018, 44016, 44025, 44017, 44024, 129760, - 127816, 125140, 125137, 125136, 125139, 125141, 125138, 125142, 124928, - 124929, 124930, 124936, 124937, 124949, 124950, 124948, 124938, 124956, - 124990, 124992, 124974, 124955, 124964, 124963, 124991, 124957, 124962, - 124976, 124996, 124997, 124998, 124982, 124983, 124984, 125004, 124975, - 125003, 125005, 125011, 125018, 125028, 125029, 125012, 125019, 125020, - 125027, 125013, 125035, 125048, 124942, 124934, 125090, 125046, 125078, - 125033, 124946, 125037, 125070, 125000, 125095, 125121, 124951, 125044, - 125041, 125072, 124987, 125066, 125076, 125074, 125009, 125107, 124945, - 125108, 125068, 125124, 125002, 124931, 125089, 125022, 125119, 124980, - 124986, 125080, 125099, 125100, 124933, 125021, 125015, 125071, 124985, - 125117, 125056, 124993, 125039, 125049, 125043, 125024, 124932, 125047, - 125097, 124959, 125069, 125088, 124999, 125123, 124952, 125036, 125026, - 125001, 125057, 125085, 124960, 125073, 124966, 125098, 125014, 125091, - 124989, 125007, 124978, 124940, 125106, 125050, 125030, 125092, 124941, - 125060, 125077, 125102, 125094, 125053, 125103, 125040, 125055, 125104, - 124939, 125017, 125112, 124961, 125087, 124970, 124971, 124969, 125023, - 124979, 125042, 124947, 125075, 125051, 125111, 125086, 124968, 124944, - 125038, 125096, 125016, 125118, 125109, 124953, 125059, 125052, 125006, - 124958, 125093, 125115, 125054, 124988, 125008, 125084, 125061, 125064, - 125120, 125063, 124967, 124977, 124965, 125031, 125081, 125082, 125010, - 125067, 124973, 125032, 124935, 125116, 125122, 125101, 124994, 124995, - 125113, 125058, 125079, 125114, 125065, 125034, 125083, 124954, 125062, - 125105, 125110, 125045, 124943, 124972, 124981, 125025, 125131, 125130, - 125133, 125132, 125129, 125128, 125135, 125127, 125134, 128334, 128697, - 68028, 68093, 68090, 68089, 68086, 68029, 68092, 68091, 68095, 68088, - 68087, 68094, 68000, 68016, 68020, 68021, 68022, 68009, 68010, 68015, - 68017, 68014, 68013, 68018, 68006, 68023, 68012, 68008, 68007, 68019, - 68011, 68005, 68004, 68001, 68002, 68003, 68031, 68030, 68039, 68075, - 68057, 68084, 68066, 68036, 68054, 68081, 68063, 68045, 68072, 68035, - 68053, 68080, 68062, 68044, 68071, 68040, 68076, 68058, 68085, 68067, - 68038, 68056, 68083, 68065, 68047, 68074, 68037, 68055, 68082, 68064, - 68046, 68073, 68034, 68052, 68079, 68061, 68043, 68070, 68033, 68051, - 68078, 68060, 68042, 68069, 68041, 68068, 68032, 68050, 68077, 68059, - 67974, 67975, 67982, 67983, 67978, 67979, 67980, 67981, 67987, 67988, - 67989, 67992, 67993, 67994, 67995, 67996, 67986, 67985, 67990, 67997, - 67984, 67977, 67976, 67991, 67973, 67972, 67968, 67969, 67970, 67971, - 67998, 67999, 129500, 9791, 983173, 9170, 9172, 9177, 9176, 9175, 9173, - 9174, 9171, 9169, 128647, 128221, 94015, 93989, 93971, 93958, 94019, - 94021, 93953, 94023, 93999, 93995, 94008, 93981, 93979, 93967, 93963, - 93993, 93992, 93983, 93977, 93976, 93975, 93974, 93968, 94032, 93988, - 93987, 93973, 93972, 93997, 93996, 93969, 94106, 94107, 94108, 94109, - 94110, 94111, 94002, 94026, 94022, 94003, 94004, 94010, 93980, 93978, - 94099, 94100, 94101, 94102, 94103, 94104, 94105, 93998, 93994, 94007, - 94025, 93966, 93962, 94024, 93961, 93960, 94000, 94009, 93964, 93965, - 94001, 93970, 93984, 93954, 94017, 94014, 94016, 94013, 94006, 94012, - 94005, 94011, 93986, 93985, 93955, 93952, 94020, 93990, 93957, 93956, - 93959, 93982, 94018, 93991, 94035, 94034, 94033, 94031, 94098, 94096, - 94097, 94095, 94036, 94039, 94040, 94067, 94068, 94038, 94037, 94073, - 94075, 94045, 94071, 94069, 94046, 94047, 94085, 94074, 94049, 94050, - 94051, 94052, 94053, 94086, 94057, 94054, 94084, 94055, 94056, 94041, - 94082, 94048, 94081, 94042, 94076, 94058, 94059, 94060, 94061, 94063, - 94064, 94079, 94087, 94062, 94065, 94080, 94066, 94072, 94070, 94044, - 94043, 94077, 94078, 94083, 983239, 983240, 128300, 127908, 181, 129440, - 129986, 183, 8943, 129686, 127894, 127756, 8357, 128189, 128469, 128656, + 43831, 621, 42894, 122641, 7557, 983611, 322, 42971, 411, 622, 122629, + 43829, 383, 7836, 7835, 7837, 682, 683, 42866, 457, 109, 7743, 43834, + 7535, 7558, 7747, 7745, 983613, 625, 7931, 7933, 42967, 42867, 110, 326, + 7755, 43835, 565, 328, 414, 7753, 626, 122663, 7536, 505, 324, 42897, + 7751, 7749, 7559, 627, 42917, 241, 329, 983589, 42868, 460, 111, 244, + 7893, 7891, 7889, 7897, 7895, 466, 246, 555, 559, 561, 7885, 525, 337, + 42827, 11386, 42829, 333, 7761, 7763, 491, 493, 248, 511, 245, 7759, + 7757, 557, 983577, 983579, 983581, 417, 7903, 7901, 7899, 7907, 7905, + 7887, 242, 243, 527, 335, 122651, 42945, 596, 983625, 983626, 7575, + 43839, 43874, 603, 7571, 42935, 419, 42831, 547, 112, 42835, 7549, 42833, + 42837, 7765, 7537, 7560, 421, 7767, 632, 113, 42841, 672, 587, 42839, + 569, 114, 343, 43849, 345, 7771, 7773, 7769, 529, 638, 7539, 122646, + 7775, 636, 637, 983615, 122664, 7538, 341, 531, 7561, 42919, 589, 43847, + 42843, 42998, 604, 7572, 605, 639, 122625, 600, 122631, 8580, 42815, + 122627, 42869, 42845, 612, 115, 347, 7781, 353, 7783, 351, 537, 349, + 122654, 7779, 7785, 7777, 42957, 42954, 575, 983583, 122665, 7540, 7562, + 42921, 642, 42892, 43872, 601, 983629, 983630, 7573, 602, 43851, 43852, + 609, 43830, 223, 7441, 7442, 7443, 7455, 7454, 7453, 42969, 645, 43845, + 116, 355, 7793, 539, 566, 357, 11366, 7831, 7789, 7787, 122666, 7541, + 7791, 427, 429, 122633, 648, 359, 679, 122647, 122652, 7546, 254, 42853, + 42855, 389, 424, 445, 7446, 42795, 397, 613, 686, 687, 7433, 42879, 7444, + 43842, 43841, 43843, 43844, 7432, 633, 43880, 122645, 634, 122632, 11385, + 635, 647, 122637, 652, 983627, 983628, 592, 594, 7426, 623, 624, 654, + 122630, 43857, 477, 7543, 670, 42881, 653, 42871, 680, 678, 43879, 11383, + 42793, 117, 649, 43855, 251, 7799, 468, 252, 474, 476, 472, 470, 7795, + 533, 369, 7909, 432, 7917, 7915, 7913, 7921, 7919, 7911, 363, 7803, + 983569, 983621, 983623, 371, 983617, 983619, 7577, 367, 361, 7801, 7797, + 249, 43854, 42937, 250, 535, 365, 43858, 650, 7551, 7531, 43856, 42872, + 43875, 118, 42847, 7807, 7564, 11380, 11377, 7805, 651, 42851, 42907, + 42909, 42911, 42857, 42849, 119, 373, 7813, 7817, 7815, 7809, 7811, 7832, + 11379, 120, 7821, 7819, 43863, 43864, 43865, 43862, 7565, 121, 375, 255, + 7925, 7823, 7923, 436, 7927, 7935, 43866, 591, 253, 563, 7929, 7833, 541, + 122, 378, 7825, 657, 382, 11372, 7827, 380, 7829, 576, 438, 7542, 7566, + 656, 549, 64256, 64259, 64260, 64257, 64258, 307, 64261, 64262, 339, + 8347, 8340, 8336, 8337, 8341, 7522, 11388, 8342, 8343, 8344, 8345, 8338, + 8346, 7523, 8348, 7524, 7525, 8339, 917505, 129726, 127811, 129388, + 129897, 129916, 129947, 10203, 10202, 129899, 129917, 117902, 128494, + 12296, 10641, 10643, 11058, 11056, 10576, 10571, 10570, 10574, 12304, + 10647, 123, 9128, 9129, 9127, 12300, 8968, 9948, 10714, 12298, 8220, + 11816, 11780, 129287, 129284, 129285, 129283, 129286, 9686, 11240, 9612, + 129977, 117924, 118288, 129970, 118285, 118283, 118435, 118440, 129940, + 129932, 128379, 128709, 11804, 10204, 9614, 9615, 129999, 10197, 8596, + 8700, 8697, 8622, 10568, 8660, 10500, 8654, 8621, 11012, 8703, 11020, + 129112, 11108, 11788, 10181, 8907, 9609, 8216, 11814, 128488, 91, 9123, + 9121, 10639, 10637, 8261, 10635, 11863, 11861, 9122, 11778, 10703, + 129900, 11785, 117771, 129985, 128492, 118434, 118441, 9610, 9613, 12308, + 129998, 8867, 12302, 10627, 12310, 10629, 12314, 12312, 10712, 128398, + 9611, 10620, 8970, 130027, 130019, 8905, 40, 9117, 9115, 9116, 11808, + 9144, 9001, 117856, 118265, 10748, 171, 117774, 128269, 117920, 117846, + 117922, 117911, 117861, 117762, 117918, 117880, 10553, 10154, 1422, + 117832, 117906, 117908, 129307, 4054, 4056, 117872, 117876, 9958, 8294, + 8237, 8234, 8206, 8592, 10563, 11074, 11083, 11082, 10611, 129973, 10618, + 10615, 11070, 8676, 8633, 10525, 129032, 8619, 11064, 8698, 10566, + 129040, 129024, 8602, 11024, 11025, 8610, 11066, 11065, 129028, 129176, + 129044, 8617, 8695, 8612, 10527, 129216, 8646, 10521, 129192, 129184, + 11144, 11013, 10603, 10599, 10590, 10582, 8637, 10594, 10602, 10598, + 10586, 10578, 8636, 8651, 129778, 129088, 129092, 11104, 11136, 11130, + 983241, 11174, 11172, 129064, 129060, 129056, 129072, 129068, 11120, + 11114, 11140, 129168, 10510, 8666, 129186, 11067, 11069, 11068, 11244, + 11061, 11060, 11062, 11063, 8606, 8678, 129172, 8604, 8656, 10498, 8653, + 10502, 10523, 10508, 8672, 129194, 129076, 129188, 8701, 8647, 129783, + 129190, 128620, 8668, 129080, 129104, 129084, 11077, 9804, 128006, 7213, + 7221, 7216, 7220, 7215, 7214, 7217, 7218, 7219, 7170, 7169, 7168, 7184, + 7183, 7182, 7247, 7193, 7180, 7188, 7187, 7186, 7185, 7172, 7171, 7198, + 7197, 7190, 7189, 7173, 7177, 7181, 7192, 7191, 7246, 7245, 7179, 7178, + 7175, 7174, 7201, 7200, 7176, 7196, 7195, 7199, 7202, 7194, 7203, 7227, + 7231, 7230, 7228, 7229, 7223, 7222, 7205, 7204, 7210, 7211, 7208, 7209, + 7206, 7212, 7207, 7237, 7236, 7239, 7238, 7235, 7234, 7232, 7241, 7233, + 7240, 10897, 10895, 10893, 10899, 10891, 10614, 10889, 10887, 8922, 8934, + 8808, 10918, 10920, 10885, 10877, 10881, 10883, 10879, 8818, 8804, 8822, + 8806, 10873, 10875, 8918, 60, 128210, 127898, 127819, 129461, 128626, + 128955, 128969, 128943, 11212, 128964, 10099, 128648, 10098, 9617, + 128937, 128949, 128978, 128960, 129653, 128910, 10072, 128930, 128504, + 9735, 128498, 128497, 6429, 6404, 6403, 6412, 6430, 6411, 6421, 6410, + 6405, 6415, 6425, 6426, 6427, 6419, 6418, 6407, 6406, 6414, 6413, 6409, + 6408, 6402, 6401, 6417, 6416, 6428, 6423, 6420, 6422, 6424, 6458, 6457, + 6459, 6464, 6449, 6452, 6450, 6448, 6456, 6454, 6453, 6455, 6451, 6442, + 6443, 6441, 6432, 6436, 6438, 6440, 6437, 6439, 6435, 6433, 6434, 6400, + 6468, 6469, 6475, 6474, 6477, 6476, 6473, 6472, 6470, 6479, 6471, 6478, + 13007, 10770, 10771, 10772, 983062, 8232, 983068, 983143, 67143, 67146, + 67151, 67165, 67166, 67167, 67157, 67158, 67159, 67160, 67161, 67162, + 67163, 67164, 67171, 67172, 67173, 67168, 67169, 67170, 67174, 67175, + 67176, 67177, 67178, 67179, 67230, 67231, 67180, 67181, 67182, 67183, + 67184, 67185, 67186, 67187, 67188, 67189, 67190, 67191, 67192, 67193, + 67194, 67195, 67196, 67197, 67198, 67199, 67200, 67201, 67202, 67203, + 67204, 67205, 67206, 67207, 67208, 67209, 67210, 67211, 67212, 67213, + 67214, 67215, 67216, 67217, 67218, 67219, 67220, 67221, 67222, 67223, + 67224, 67225, 67226, 67227, 67228, 67229, 67232, 67233, 67234, 67235, + 67236, 67237, 67238, 67239, 67240, 67241, 67242, 67243, 67244, 67245, + 67246, 67247, 67248, 67249, 67250, 67251, 67252, 67253, 67254, 67255, + 67256, 67257, 67258, 67259, 67260, 67261, 67262, 67263, 67264, 67274, + 67275, 67276, 67277, 67278, 67279, 67280, 67281, 67282, 67283, 67284, + 67285, 67286, 67287, 67288, 67289, 67290, 67291, 67292, 67293, 67294, + 67295, 67296, 67297, 67298, 67299, 67300, 67301, 67302, 67303, 67304, + 67265, 67266, 67267, 67268, 67269, 67270, 67271, 67272, 67273, 67325, + 67326, 67327, 67328, 67329, 67330, 67305, 67306, 67307, 67308, 67309, + 67310, 67311, 67312, 67313, 67314, 67315, 67316, 67317, 67318, 67319, + 67320, 67321, 67322, 67323, 67324, 67331, 67332, 67333, 67334, 67335, + 67336, 67337, 67338, 67349, 67350, 67351, 67352, 67353, 67354, 67355, + 67356, 67357, 67358, 67359, 67360, 67361, 67362, 67363, 67364, 67365, + 67366, 67367, 67368, 67378, 67379, 67380, 67381, 67382, 67369, 67370, + 67371, 67372, 67373, 67374, 67375, 67376, 67377, 67339, 67340, 67341, + 67342, 67343, 67344, 67345, 67346, 67347, 67348, 67401, 67404, 67403, + 67402, 67400, 67398, 67393, 67397, 67395, 67394, 67399, 67392, 67396, + 67406, 67408, 67409, 67410, 67407, 67405, 67411, 67413, 67412, 67424, + 67425, 67426, 67427, 67428, 67429, 67430, 67431, 67081, 67082, 67083, + 67084, 67085, 67087, 67088, 67089, 67090, 67091, 67092, 67093, 67094, + 67086, 67095, 67096, 67097, 67098, 67100, 67101, 67102, 67103, 67104, + 67105, 67106, 67107, 67108, 67109, 67110, 67111, 67112, 67113, 67114, + 67115, 67116, 67117, 67118, 67119, 67120, 67121, 67122, 67123, 67124, + 67125, 67126, 67127, 67128, 67129, 67130, 67131, 67132, 67133, 67134, + 67135, 67136, 67137, 67138, 67139, 67140, 67141, 67142, 67072, 67073, + 67074, 67075, 67076, 67077, 67078, 67079, 67080, 67145, 67147, 67148, + 67149, 67150, 67154, 67155, 67144, 67152, 67153, 67156, 67099, 65667, + 65668, 65669, 65670, 65671, 65672, 65673, 65675, 65674, 65677, 65676, + 65665, 65664, 65666, 65681, 65679, 65680, 65682, 65678, 65686, 65685, + 65687, 65693, 65690, 65691, 65692, 65694, 65703, 65696, 65695, 65697, + 65698, 65699, 65701, 65702, 65707, 65706, 65704, 65705, 65708, 65709, + 65710, 65711, 65712, 65713, 65719, 65717, 65714, 65715, 65716, 65718, + 65720, 65721, 65722, 65723, 65724, 65725, 65726, 65727, 65728, 65729, + 65731, 65730, 65732, 65733, 65737, 65734, 65735, 65736, 65738, 65739, + 65740, 65741, 65743, 65742, 65744, 65745, 65747, 65748, 65752, 65749, + 65750, 65751, 65753, 65754, 65755, 65756, 65757, 65779, 65780, 65781, + 65782, 65783, 65784, 65785, 65759, 65760, 65761, 65762, 65763, 65764, + 65765, 65766, 65767, 65768, 65769, 65770, 65771, 65772, 65773, 65774, + 65775, 65776, 65777, 65778, 65758, 65786, 65683, 65684, 65689, 65688, + 65700, 65746, 65561, 65543, 65587, 65582, 65589, 65536, 65541, 65579, + 65566, 65571, 65596, 65569, 65584, 65544, 65559, 65540, 65557, 65560, + 65580, 65606, 65600, 65599, 65577, 65562, 65538, 65573, 65563, 65609, + 65588, 65549, 65568, 65537, 65581, 65574, 65601, 65542, 65593, 65583, + 65594, 65552, 65547, 65605, 65578, 65545, 65585, 65586, 65546, 65570, + 65591, 65564, 65565, 65607, 65610, 65611, 65553, 65590, 65550, 65539, + 65576, 65608, 65551, 65554, 65592, 65603, 65597, 65567, 65572, 65558, + 65555, 65612, 65602, 65556, 65613, 65604, 65620, 65621, 65623, 65624, + 65626, 65627, 65628, 65629, 65622, 65616, 65617, 65619, 65618, 65625, + 128391, 128279, 128482, 128132, 42237, 42235, 42232, 42234, 42236, 42233, + 42206, 42205, 42197, 42196, 42204, 42195, 42228, 42229, 42230, 42213, + 42208, 42224, 42225, 42203, 42202, 42221, 42198, 42216, 42214, 42200, + 42199, 42194, 42193, 42219, 42210, 73648, 42220, 42211, 42212, 42222, + 42223, 42227, 42231, 42192, 42217, 42201, 42209, 42207, 42218, 42215, + 42226, 42238, 42239, 8356, 8374, 129409, 129422, 9806, 128274, 983079, + 983076, 128271, 117785, 117786, 117784, 117783, 117782, 117781, 8743, + 10848, 10846, 10833, 10844, 10842, 10847, 8744, 10841, 10851, 10850, + 10834, 10845, 10843, 10982, 10188, 129688, 10234, 10231, 10206, 10229, + 10235, 10232, 10237, 11059, 10230, 10236, 10233, 10238, 10239, 10205, + 129524, 128884, 129719, 128140, 127977, 128261, 129707, 12319, 8270, + 11847, 11848, 95, 118273, 9691, 118276, 118291, 129935, 118436, 118447, + 9604, 9697, 117765, 128394, 129852, 129853, 129870, 129872, 129868, + 129856, 129871, 129869, 129854, 129873, 129855, 128395, 128396, 128393, + 10559, 117934, 117936, 117932, 117930, 117972, 9695, 117960, 117948, + 117964, 117968, 117952, 117956, 117944, 117940, 117817, 129951, 9722, + 117820, 128397, 118428, 117935, 117937, 117933, 117931, 117973, 9694, + 117961, 117949, 117965, 117969, 117953, 117957, 117945, 117941, 117818, + 10558, 128318, 10065, 129950, 9727, 117823, 117767, 129863, 129865, + 129867, 129864, 129861, 129866, 129859, 129862, 129860, 129857, 129858, + 10195, 118431, 10063, 9998, 9987, 118429, 117821, 118430, 117822, 130021, + 9605, 118425, 118426, 118424, 117816, 118427, 117819, 9602, 9601, 9607, + 9603, 118437, 118446, 9606, 129903, 9674, 10208, 129438, 128557, 127853, + 129523, 128886, 129729, 66190, 66192, 66187, 66196, 66178, 66179, 66199, + 66185, 66200, 66176, 66201, 66177, 66202, 66191, 66193, 66181, 66180, + 66203, 66182, 66186, 66189, 66195, 66188, 66197, 66198, 66194, 66183, + 66204, 66184, 67887, 67892, 67881, 67895, 67891, 67886, 67872, 67893, + 67876, 67894, 67883, 67896, 67873, 67897, 67875, 67889, 67874, 67878, + 67880, 67882, 67884, 67890, 67885, 67888, 67877, 67879, 67903, 129317, + 983226, 983234, 983224, 983229, 8468, 129433, 983065, 129668, 129522, + 129497, 69986, 69981, 69991, 69985, 69984, 69990, 69989, 70002, 69997, + 69983, 69982, 69988, 69987, 69995, 69994, 69978, 69977, 69976, 69975, + 69980, 69979, 69974, 69973, 69993, 69992, 70001, 69998, 69996, 70000, + 69999, 69968, 69971, 69969, 69972, 69970, 70006, 70005, 70003, 70004, + 127012, 127019, 127008, 126990, 126999, 126976, 127005, 126987, 126996, + 127004, 126986, 126995, 126979, 127009, 126991, 127000, 127001, 126983, + 126992, 127011, 127010, 126977, 127007, 126989, 126998, 127006, 126988, + 126997, 127015, 127014, 127003, 126985, 126994, 127002, 126984, 126993, + 126978, 126982, 127017, 126981, 126980, 127016, 127018, 127013, 73464, + 73442, 73451, 73448, 73444, 73449, 73447, 73441, 73450, 73440, 73454, + 73445, 73443, 73453, 73456, 73446, 73455, 73452, 73457, 73463, 73461, + 73459, 73462, 73460, 73458, 128892, 3449, 3435, 3434, 3437, 3436, 3433, + 3432, 3430, 3439, 3431, 3438, 3419, 3420, 3446, 3417, 3422, 3416, 3447, + 3444, 3443, 3448, 3418, 3421, 3445, 3333, 3423, 3334, 3344, 3348, 3453, + 3454, 3414, 3451, 3450, 3452, 3455, 3412, 3413, 3355, 3354, 3406, 3362, + 3361, 3367, 3366, 3360, 3386, 3359, 3365, 3364, 3332, 3339, 3424, 3340, + 3425, 3381, 3369, 3363, 3353, 3358, 3368, 3380, 3379, 3378, 3377, 3376, + 3337, 3338, 3346, 3347, 3335, 3336, 3382, 3383, 3384, 3373, 3372, 3352, + 3351, 3357, 3356, 3350, 3349, 3371, 3370, 3342, 3343, 3385, 3374, 3375, + 3441, 3442, 3440, 3328, 3388, 3329, 3387, 3405, 3331, 3389, 3330, 3407, + 3390, 3400, 3404, 3393, 3394, 3395, 3396, 3426, 3427, 3402, 3403, 3391, + 3392, 3398, 3399, 3415, 9895, 9894, 9893, 9794, 10016, 129443, 128104, + 128372, 129333, 128114, 128115, 128378, 128107, 2137, 2122, 2121, 2133, + 2120, 2126, 2132, 2129, 2136, 2113, 2115, 2114, 2116, 2123, 2124, 2125, + 2128, 2130, 2131, 2118, 2134, 2117, 2112, 2127, 2119, 2135, 2138, 2139, + 2142, 68288, 68314, 68313, 68290, 68289, 68293, 68308, 68292, 68291, + 68300, 68299, 68298, 68297, 68306, 68304, 68320, 68318, 68323, 68312, + 68317, 68322, 68309, 68302, 68324, 68305, 68319, 68307, 68321, 68303, + 68294, 68301, 68311, 68295, 68316, 68315, 68310, 68331, 68335, 68334, + 68333, 68332, 68340, 68339, 68338, 68342, 68337, 68341, 68336, 68296, + 68326, 68325, 128094, 129469, 8380, 128368, 129389, 9967, 127809, 129671, + 72835, 72834, 72827, 72826, 72836, 72828, 72821, 72825, 72829, 72823, + 72822, 72819, 72818, 72831, 72830, 72844, 72845, 72838, 72839, 72840, + 72832, 72820, 72846, 72824, 72843, 72833, 72842, 72837, 72841, 72847, + 72886, 72885, 72867, 72866, 72859, 72858, 72868, 72860, 72853, 72857, + 72861, 72855, 72854, 72851, 72850, 72863, 72862, 72876, 72877, 72870, + 72871, 72864, 72852, 72878, 72856, 72875, 72865, 72874, 72869, 72873, + 72879, 72880, 72883, 72881, 72884, 72882, 72816, 72817, 9901, 129355, + 73007, 72980, 72979, 72983, 72982, 72988, 73008, 72987, 72960, 72961, + 72968, 72971, 72985, 72984, 72990, 72989, 72964, 72965, 72962, 72963, + 73005, 72999, 73006, 72973, 72972, 72976, 72986, 72981, 72991, 73001, + 73002, 73003, 72995, 72994, 72978, 72977, 72975, 72974, 72993, 72992, + 73004, 72996, 72998, 73000, 72997, 72966, 72969, 73031, 73030, 73027, + 73028, 73026, 73025, 73024, 73014, 73009, 73020, 73023, 73012, 73013, + 73010, 73011, 73018, 73021, 73029, 73045, 73044, 73047, 73046, 73043, + 73042, 73040, 73049, 73041, 73048, 186, 127405, 12348, 119811, 120778, + 120491, 119827, 120495, 120505, 120507, 119808, 120488, 119809, 120489, + 119833, 120493, 119812, 120492, 120494, 119814, 120490, 119816, 120496, + 119818, 120497, 119819, 120498, 119822, 120502, 120512, 119823, 120509, + 120511, 120503, 119825, 120504, 119826, 120506, 119828, 120508, 119810, + 120510, 119820, 120499, 119821, 120500, 119831, 120501, 119813, 119815, + 119817, 119824, 119829, 119830, 119832, 119837, 120779, 120517, 119834, + 120514, 119835, 120515, 119859, 120519, 119838, 120518, 120520, 119839, + 120531, 119840, 120516, 119842, 120522, 119844, 120523, 119845, 120524, + 119848, 120528, 120538, 119849, 120535, 120537, 120529, 119851, 120530, + 119852, 120532, 119853, 120521, 120533, 119854, 120534, 119836, 120536, + 119846, 120525, 119847, 120526, 119857, 120527, 119841, 119843, 119850, + 119855, 119856, 119858, 120016, 120017, 120018, 120019, 120020, 120021, + 120022, 120023, 120024, 120025, 120026, 120027, 120028, 120029, 120030, + 120031, 120032, 120033, 120034, 120035, 120036, 120037, 120038, 120039, + 120040, 120041, 120042, 120043, 120044, 120045, 120046, 120047, 120048, + 120049, 120050, 120051, 120052, 120053, 120054, 120055, 120056, 120057, + 120058, 120059, 120060, 120061, 120062, 120063, 120064, 120065, 120066, + 120067, 119931, 120611, 120621, 120623, 119912, 120604, 119913, 120605, + 119937, 120609, 119915, 120607, 119916, 120608, 120610, 119918, 120606, + 119920, 120612, 119922, 120613, 119923, 120614, 119926, 120618, 120628, + 119927, 120625, 120627, 120619, 119929, 120620, 119930, 120622, 119932, + 120624, 119914, 120626, 119924, 120615, 119925, 120616, 119935, 120617, + 119917, 119919, 119921, 119928, 119933, 119934, 119936, 120656, 120658, + 120629, 120659, 120655, 120661, 120660, 119938, 120630, 119939, 120631, + 119963, 120635, 119941, 120633, 119942, 120634, 120636, 119943, 120647, + 119944, 120632, 119946, 120638, 119948, 120639, 119949, 120640, 119952, + 120644, 120654, 119953, 120651, 120653, 120645, 119955, 120646, 119956, + 120648, 119957, 120637, 120649, 119958, 120650, 119940, 120652, 119950, + 120641, 119951, 120642, 119961, 120643, 119945, 119947, 119954, 119959, + 119960, 119962, 120657, 120540, 120542, 120513, 120543, 120539, 120545, + 120544, 120541, 120172, 120173, 120174, 120175, 120176, 120177, 120178, + 120179, 120180, 120181, 120182, 120183, 120184, 120185, 120186, 120187, + 120188, 120189, 120190, 120191, 120192, 120193, 120194, 120195, 120196, + 120197, 120198, 120199, 120200, 120201, 120202, 120203, 120204, 120205, + 120206, 120207, 120208, 120209, 120210, 120211, 120212, 120213, 120214, + 120215, 120216, 120217, 120218, 120219, 120220, 120221, 120222, 120223, + 120787, 120786, 120789, 120788, 120785, 120784, 120782, 120791, 120783, + 120790, 120120, 120121, 120123, 120124, 120125, 120126, 120128, 120129, + 120130, 120131, 120132, 120134, 120138, 120139, 120140, 120141, 120142, + 120143, 120144, 120146, 120147, 120148, 120149, 120150, 120151, 120152, + 120153, 120154, 120155, 120156, 120157, 120158, 120159, 120160, 120161, + 120162, 120163, 120164, 120165, 120166, 120167, 120168, 120169, 120170, + 120171, 120797, 120796, 120799, 120798, 120795, 120794, 120792, 120801, + 120793, 120800, 120068, 120069, 120071, 120072, 120073, 120074, 120077, + 120078, 120079, 120080, 120081, 120082, 120083, 120084, 120086, 120087, + 120088, 120089, 120090, 120091, 120092, 120094, 120095, 120096, 120097, + 120098, 120099, 120100, 120101, 120102, 120103, 120104, 120105, 120106, + 120107, 120108, 120109, 120110, 120111, 120112, 120113, 120114, 120115, + 120116, 120117, 120118, 120119, 10189, 119889, 120484, 120485, 120575, + 119886, 120572, 119887, 120573, 119911, 120577, 119890, 120576, 120578, + 119891, 120589, 119892, 120574, 119894, 120580, 119896, 120581, 119897, + 120582, 119900, 120586, 120596, 119901, 120593, 120595, 120587, 119903, + 120588, 119904, 120590, 119905, 120579, 120591, 119906, 120592, 119888, + 120594, 119898, 120583, 119899, 120584, 119909, 120585, 119895, 119902, + 119907, 119908, 119910, 119879, 120553, 120563, 120565, 119860, 120546, + 119861, 120547, 119885, 120551, 119863, 120549, 119864, 120550, 120552, + 119866, 120548, 119868, 120554, 119870, 120555, 119871, 120556, 119874, + 120560, 120570, 119875, 120567, 120569, 120561, 119877, 120562, 119878, + 120564, 119880, 120566, 119862, 120568, 119872, 120557, 119873, 120558, + 119883, 120559, 119865, 119867, 119869, 119876, 119881, 119882, 119884, + 120598, 120600, 120571, 120601, 120597, 120603, 120602, 120599, 120432, + 120433, 120434, 120435, 120436, 120437, 120438, 120439, 120440, 120441, + 120442, 120443, 120444, 120445, 120446, 120447, 120448, 120449, 120450, + 120451, 120452, 120453, 120454, 120455, 120456, 120457, 120458, 120459, + 120460, 120461, 120462, 120463, 120464, 120465, 120466, 120467, 120468, + 120469, 120470, 120471, 120472, 120473, 120474, 120475, 120476, 120477, + 120478, 120479, 120480, 120481, 120482, 120483, 120827, 120826, 120829, + 120828, 120825, 120824, 120822, 120831, 120823, 120830, 10215, 10221, + 10219, 10217, 10223, 10187, 10214, 10220, 10218, 10216, 10222, 120399, + 120727, 120737, 120739, 120380, 120720, 120381, 120721, 120405, 120725, + 120383, 120723, 120384, 120724, 120726, 120386, 120722, 120388, 120728, + 120390, 120729, 120391, 120730, 120394, 120734, 120744, 120395, 120741, + 120743, 120735, 120397, 120736, 120398, 120738, 120400, 120740, 120382, + 120742, 120392, 120731, 120393, 120732, 120403, 120733, 120385, 120387, + 120389, 120396, 120401, 120402, 120404, 120772, 120774, 120745, 120775, + 120771, 120777, 120776, 120406, 120746, 120407, 120747, 120431, 120751, + 120409, 120749, 120410, 120750, 120752, 120411, 120763, 120412, 120748, + 120414, 120754, 120416, 120755, 120417, 120756, 120420, 120760, 120770, + 120421, 120767, 120769, 120761, 120423, 120762, 120424, 120764, 120425, + 120753, 120765, 120426, 120766, 120408, 120768, 120418, 120757, 120419, + 120758, 120429, 120759, 120413, 120415, 120422, 120427, 120428, 120430, + 120773, 120295, 120669, 120679, 120681, 120276, 120662, 120277, 120663, + 120301, 120667, 120279, 120665, 120280, 120666, 120668, 120282, 120664, + 120284, 120670, 120286, 120671, 120287, 120672, 120290, 120676, 120686, + 120291, 120683, 120685, 120677, 120293, 120678, 120294, 120680, 120296, + 120682, 120278, 120684, 120288, 120673, 120289, 120674, 120299, 120675, + 120281, 120283, 120285, 120292, 120297, 120298, 120300, 120714, 120716, + 120687, 120717, 120713, 120719, 120718, 120302, 120688, 120303, 120689, + 120327, 120693, 120305, 120691, 120306, 120692, 120694, 120307, 120705, + 120308, 120690, 120310, 120696, 120312, 120697, 120313, 120698, 120316, + 120702, 120712, 120317, 120709, 120711, 120703, 120319, 120704, 120320, + 120706, 120321, 120695, 120707, 120322, 120708, 120304, 120710, 120314, + 120699, 120315, 120700, 120325, 120701, 120309, 120311, 120318, 120323, + 120324, 120326, 120715, 120817, 120816, 120819, 120818, 120815, 120814, + 120812, 120821, 120813, 120820, 120328, 120329, 120330, 120331, 120332, + 120333, 120334, 120335, 120336, 120337, 120338, 120339, 120340, 120341, + 120342, 120343, 120344, 120345, 120346, 120347, 120348, 120349, 120350, + 120351, 120352, 120353, 120354, 120355, 120356, 120357, 120358, 120359, + 120360, 120361, 120362, 120363, 120364, 120365, 120366, 120367, 120368, + 120369, 120370, 120371, 120372, 120373, 120374, 120375, 120376, 120377, + 120378, 120379, 120224, 120225, 120226, 120227, 120228, 120229, 120230, + 120231, 120232, 120233, 120234, 120235, 120236, 120237, 120238, 120239, + 120240, 120241, 120242, 120243, 120244, 120245, 120246, 120247, 120248, + 120249, 120250, 120251, 120252, 120253, 120254, 120255, 120256, 120257, + 120258, 120259, 120260, 120261, 120262, 120263, 120264, 120265, 120266, + 120267, 120268, 120269, 120270, 120271, 120272, 120273, 120274, 120275, + 120807, 120806, 120809, 120808, 120805, 120804, 120802, 120811, 120803, + 120810, 119964, 119966, 119967, 119970, 119973, 119974, 119977, 119978, + 119979, 119980, 119982, 119983, 119984, 119985, 119986, 119987, 119988, + 119989, 119990, 119991, 119992, 119993, 119995, 119997, 119998, 119999, + 120000, 120001, 120002, 120003, 120005, 120006, 120007, 120008, 120009, + 120010, 120011, 120012, 120013, 120014, 120015, 129481, 119528, 119538, + 119531, 119535, 119525, 119524, 119534, 119529, 119539, 119527, 119537, + 119526, 119536, 119533, 119523, 119532, 119522, 119530, 119520, 119521, + 128470, 175, 8737, 10667, 10666, 10671, 10669, 10670, 10668, 10665, + 10664, 10651, 10653, 8798, 127830, 129470, 129471, 93773, 93764, 93790, + 93787, 983270, 93783, 983269, 93782, 93772, 93766, 93791, 93779, 93789, + 93786, 93776, 93777, 93785, 93775, 93770, 93769, 93771, 93774, 93780, + 93760, 93767, 93781, 93788, 93761, 93768, 93778, 93762, 93763, 93784, + 93765, 93847, 93827, 93846, 93826, 93845, 93825, 93844, 93829, 93828, + 93831, 93830, 93824, 93833, 93832, 93837, 93836, 93834, 93842, 93835, + 93838, 93839, 93843, 93840, 93841, 93805, 93796, 93822, 93819, 983272, + 93815, 983271, 93814, 93804, 93798, 93823, 93811, 93821, 93818, 93808, + 93809, 93817, 93807, 93802, 93801, 93803, 93806, 93812, 93792, 93799, + 93813, 93820, 93793, 93800, 93810, 93794, 93795, 93816, 93797, 93849, + 93850, 93848, 11859, 11852, 11860, 128901, 9899, 10090, 10091, 128967, + 128965, 128944, 10100, 10088, 10092, 10101, 10089, 10093, 8287, 9900, + 9618, 128971, 128950, 128938, 128963, 128961, 10073, 128974, 128956, + 9898, 128911, 128931, 43761, 44013, 43762, 43760, 44011, 43994, 43989, + 43974, 43746, 43993, 43991, 43751, 43750, 43992, 43986, 43987, 43990, + 43968, 43995, 43976, 43973, 43999, 43977, 44001, 43752, 43747, 43972, + 43998, 43984, 43969, 43753, 43754, 43975, 44000, 43978, 43749, 43748, + 43983, 44002, 43970, 43996, 43971, 43997, 43981, 43988, 43979, 43985, + 43980, 43982, 43744, 43745, 44012, 43763, 43764, 44005, 43757, 43759, + 43758, 44009, 44003, 44007, 44006, 44004, 43755, 44008, 43756, 44010, + 43765, 43766, 44021, 44020, 44023, 44022, 44019, 44018, 44016, 44025, + 44017, 44024, 129760, 127816, 125140, 125137, 125136, 125139, 125141, + 125138, 125142, 124928, 124929, 124930, 124936, 124937, 124949, 124950, + 124948, 124938, 124956, 124990, 124992, 124974, 124955, 124964, 124963, + 124991, 124957, 124962, 124976, 124996, 124997, 124998, 124982, 124983, + 124984, 125004, 124975, 125003, 125005, 125011, 125018, 125028, 125029, + 125012, 125019, 125020, 125027, 125013, 125035, 125048, 124942, 124934, + 125090, 125046, 125078, 125033, 124946, 125037, 125070, 125000, 125095, + 125121, 124951, 125044, 125041, 125072, 124987, 125066, 125076, 125074, + 125009, 125107, 125108, 125068, 125124, 124945, 125002, 124931, 125089, + 125022, 124980, 125099, 124986, 125100, 125080, 125119, 124933, 125021, + 125015, 125071, 124985, 125117, 125056, 124993, 125039, 125049, 125043, + 125024, 124932, 125047, 125097, 124959, 125069, 125088, 124999, 125123, + 124952, 125036, 125026, 125001, 125085, 124960, 125057, 125073, 124966, + 125098, 125014, 125091, 124989, 125007, 124978, 124940, 125106, 125050, + 125030, 125092, 124941, 125060, 125077, 125102, 125094, 125053, 125040, + 125055, 125104, 125103, 124939, 125017, 124961, 125112, 125087, 124970, + 124971, 124969, 125023, 124979, 125042, 124947, 125086, 125075, 125051, + 125111, 124968, 124944, 125038, 125096, 125016, 125118, 125109, 124953, + 125059, 125052, 125006, 124958, 125093, 125115, 125054, 124988, 125008, + 125084, 125061, 125064, 125120, 125063, 124967, 124977, 124965, 125031, + 983275, 125081, 125082, 983276, 125010, 125067, 124973, 125032, 124935, + 125116, 125122, 125101, 124994, 124995, 125113, 125058, 125079, 125114, + 125065, 125034, 125083, 124954, 125062, 125105, 125110, 125045, 124943, + 124972, 124981, 125025, 125131, 125130, 125133, 125132, 125129, 125128, + 125135, 125127, 125134, 128334, 128697, 68028, 68093, 68090, 68089, + 68086, 68029, 68092, 68091, 68095, 68088, 68087, 68094, 68000, 68016, + 68020, 68021, 68022, 68009, 68010, 68015, 68017, 68014, 68013, 68018, + 68006, 68023, 68012, 68008, 68007, 68019, 68011, 68005, 68004, 68001, + 68002, 68003, 68031, 68030, 68039, 68075, 68057, 68084, 68066, 68036, + 68054, 68081, 68063, 68045, 68072, 68035, 68053, 68080, 68062, 68044, + 68071, 68040, 68076, 68058, 68085, 68067, 68038, 68056, 68083, 68065, + 68047, 68074, 68037, 68055, 68082, 68064, 68046, 68073, 68034, 68052, + 68079, 68061, 68043, 68070, 68033, 68051, 68078, 68060, 68042, 68069, + 68041, 68068, 68032, 68050, 68077, 68059, 67974, 67975, 67982, 67983, + 67978, 67979, 67980, 67981, 67987, 67988, 67989, 67992, 67993, 67994, + 67995, 67996, 67986, 67985, 67990, 67997, 67984, 67977, 67976, 67991, + 67973, 67972, 67968, 67969, 67970, 67971, 67998, 67999, 9791, 129500, + 983173, 9170, 9172, 9177, 9176, 9175, 9173, 9174, 9171, 9169, 128647, + 128221, 94015, 93989, 93971, 93958, 94019, 94021, 93953, 94023, 93999, + 93995, 94008, 93981, 93979, 93967, 93963, 93993, 93992, 93983, 93977, + 93976, 93975, 93974, 93968, 94032, 93988, 93987, 93973, 93972, 93997, + 93996, 93969, 94106, 94107, 94108, 94109, 94110, 94111, 94002, 94026, + 94022, 94003, 94004, 94010, 93980, 93978, 94099, 94100, 94101, 94102, + 94103, 94104, 94105, 93998, 93994, 94007, 94025, 93966, 93962, 94024, + 93961, 93960, 94000, 94009, 93964, 93965, 94001, 93970, 93984, 93954, + 94017, 94014, 94016, 94013, 94006, 94012, 94005, 94011, 93986, 93985, + 93955, 93952, 94020, 93990, 93957, 93956, 93959, 93982, 94018, 93991, + 94035, 94034, 94033, 94031, 94098, 94096, 94097, 94095, 94036, 94039, + 94040, 94067, 94068, 94038, 94037, 94073, 94075, 94045, 94071, 94069, + 94046, 94047, 94085, 94074, 94049, 94050, 94051, 94052, 94053, 94086, + 94057, 94054, 94084, 94055, 94056, 94041, 94082, 94048, 94081, 94042, + 94076, 94058, 94059, 94060, 94061, 94063, 94064, 94079, 94087, 94062, + 94065, 94080, 94066, 94072, 94070, 94044, 94043, 94077, 94078, 94083, + 983239, 983240, 128300, 127908, 181, 129440, 117772, 129986, 130022, + 130023, 183, 8943, 129686, 127894, 127756, 8357, 128189, 128469, 128656, 8722, 10793, 10794, 10796, 10795, 10810, 8770, 8723, 10751, 129694, 129705, 128241, 128242, 128244, 129339, 8871, 71232, 71229, 71236, 71231, - 71230, 71216, 71226, 71228, 71221, 71222, 71223, 71224, 71219, 71220, + 71230, 71216, 71226, 71228, 71219, 71220, 71221, 71222, 71223, 71224, 71217, 71218, 71225, 71227, 71234, 71233, 71253, 71252, 71255, 71254, 71251, 71250, 71248, 71257, 71249, 71256, 71168, 71169, 71179, 71181, 71195, 71194, 71200, 71199, 71193, 71192, 71198, 71197, 71174, 71175, @@ -11185,7 +11949,7 @@ static const unsigned int dawg_pos_to_codepoint[] = { 122941, 122959, 122989, 122955, 122950, 122948, 122944, 122951, 122988, 122953, 122934, 122935, 122936, 122933, 122949, 122931, 122957, 122930, 122947, 122937, 122928, 122940, 122945, 42653, 7544, 710, 735, 42889, - 67510, 42776, 42777, 42775, 750, 698, 725, 709, 984011, 42765, 42760, + 67510, 42776, 42775, 42777, 750, 698, 725, 709, 984011, 42765, 42760, 42770, 741, 984012, 42769, 42764, 42774, 745, 762, 764, 715, 704, 67507, 4348, 42766, 42761, 42771, 742, 721, 67511, 42888, 42768, 42763, 751, 767, 753, 42773, 717, 754, 755, 744, 759, 719, 718, 42783, 752, 716, @@ -11211,191 +11975,194 @@ static const unsigned int dawg_pos_to_codepoint[] = { 6287, 6278, 6285, 6284, 6288, 6277, 6291, 6290, 6293, 6294, 6292, 6283, 6281, 6185, 6196, 6264, 6210, 6190, 6303, 6305, 6307, 6300, 6302, 6304, 6312, 6298, 6301, 6314, 6308, 6309, 6299, 6306, 6263, 6262, 6260, 6261, - 6259, 6254, 6248, 6244, 6252, 6245, 6253, 6238, 6239, 6257, 6247, 6256, + 6259, 6244, 6252, 6245, 6253, 6238, 6239, 6254, 6248, 6257, 6247, 6256, 6242, 6258, 6255, 6241, 6240, 6249, 6251, 6250, 6243, 6246, 6237, 6193, - 6192, 6297, 6296, 6218, 6236, 6225, 6234, 6227, 6235, 6211, 6228, 6224, - 6222, 6232, 6226, 6233, 6214, 6216, 6215, 6217, 6219, 6231, 6223, 6220, + 6192, 6297, 6296, 6218, 6236, 6225, 6234, 6227, 6235, 6211, 6222, 6232, + 6228, 6224, 6226, 6233, 6214, 6216, 6215, 6217, 6219, 6231, 6223, 6220, 6221, 6230, 6229, 6212, 6213, 6204, 6194, 6209, 6207, 6205, 6206, 6203, 6202, 6208, 6191, 6177, 6183, 6179, 6181, 6180, 6182, 6186, 6195, 6201, 6189, 6197, 6184, 6187, 6188, 6199, 6200, 6198, 6178, 71273, 71275, 71274, 6151, 71265, 71270, 71269, 6144, 71268, 71264, 71267, 71276, 6150, 6158, 6148, 6146, 6152, 6153, 6154, 6145, 9866, 9867, 119552, 9101, - 128669, 128018, 128053, 127889, 129390, 128496, 129742, 129439, 128332, - 129334, 128741, 128757, 129468, 128739, 9968, 128670, 128672, 128693, - 128507, 128001, 129700, 128045, 128068, 128511, 127909, 92748, 92744, - 92761, 92750, 92739, 92751, 92737, 92754, 92749, 92753, 92743, 92752, - 92757, 92766, 92736, 92741, 92746, 92764, 92745, 92765, 92755, 92760, - 92758, 92756, 92763, 92762, 92747, 92738, 92740, 92759, 92742, 92783, - 92782, 92773, 92772, 92775, 92774, 92771, 92770, 92768, 92777, 92769, - 92776, 70291, 70292, 70290, 70297, 70296, 70293, 70287, 70298, 70312, - 70311, 70306, 70285, 70284, 70289, 70288, 70295, 70294, 70303, 70301, - 70283, 70282, 70280, 70278, 70277, 70276, 70300, 70299, 70310, 70307, - 70304, 70309, 70308, 70305, 70272, 70275, 70273, 70274, 70313, 127926, - 215, 10804, 10805, 10807, 10811, 10801, 10800, 10005, 8844, 8845, 8846, - 8888, 9838, 9837, 9839, 127929, 127896, 119161, 119159, 119155, 119157, - 119061, 119060, 119224, 119235, 119132, 119058, 119059, 119255, 119253, - 119130, 119131, 119163, 119169, 119149, 119167, 119168, 119210, 119178, - 119173, 119211, 119150, 119151, 119152, 119153, 119154, 119175, 119212, - 119213, 119166, 119164, 119141, 119142, 119176, 119179, 119143, 119144, - 119145, 119165, 119177, 119170, 119174, 119092, 119052, 119247, 119186, - 119073, 119109, 119093, 119050, 119220, 119221, 119044, 119049, 119187, - 119209, 119082, 119041, 119083, 119077, 119078, 119162, 119160, 119208, - 119156, 119158, 119136, 119102, 119056, 119057, 119146, 119147, 119148, - 119042, 119066, 119065, 119069, 119185, 119074, 119075, 119076, 119231, - 119232, 119085, 119084, 119070, 119071, 119072, 119218, 119217, 119189, - 119188, 119248, 119249, 119172, 119171, 119216, 119134, 119100, 119206, - 119262, 119270, 119271, 119263, 119268, 119269, 119272, 119264, 119267, - 119266, 119265, 119274, 119223, 119234, 119233, 119046, 119222, 119184, - 119227, 119237, 119228, 119122, 119123, 119081, 119098, 119207, 119129, - 119087, 119086, 119128, 119140, 119106, 119062, 119195, 119204, 119205, - 119196, 119197, 119198, 119199, 119200, 119201, 119202, 119203, 119094, - 119095, 119126, 119108, 119215, 119214, 119261, 119252, 119257, 119258, - 119183, 119090, 119091, 119135, 119101, 119096, 119097, 119053, 119054, - 119055, 119048, 119043, 119047, 119180, 119254, 119259, 119225, 119236, - 119226, 119229, 119238, 119230, 119051, 119045, 119089, 119088, 119040, - 119067, 119068, 119137, 119103, 119139, 119105, 119110, 119111, 119250, - 119181, 119273, 119243, 119244, 119245, 119246, 119242, 119240, 119239, - 119241, 119064, 119138, 119104, 119063, 119256, 119260, 119190, 119118, - 119119, 119120, 119121, 119112, 119113, 119116, 119117, 119114, 119115, - 119124, 119125, 119191, 119193, 119194, 119127, 119251, 119107, 119133, - 119099, 119219, 119192, 119182, 127932, 127925, 127812, 8811, 8810, 4158, - 4156, 4157, 4155, 4192, 4191, 4190, 4226, 4129, 43642, 4138, 4135, 4208, - 4207, 4206, 4159, 4099, 4098, 4097, 43625, 43624, 43626, 43623, 43622, - 43621, 43627, 43618, 43617, 43630, 43629, 43620, 43619, 983244, 43631, - 43616, 43635, 43628, 43633, 43634, 4096, 4188, 4189, 4187, 4186, 4136, - 4121, 4106, 4111, 4100, 4105, 4116, 4238, 4123, 4176, 43491, 4218, 4220, - 43490, 4221, 4224, 43492, 4223, 43489, 4216, 43488, 4215, 4214, 4213, - 4219, 4222, 4225, 4217, 4130, 43646, 43647, 4193, 4177, 4126, 4112, - 43503, 43495, 43502, 43501, 43516, 43515, 43518, 43517, 43498, 43497, - 43500, 43499, 43514, 43496, 4108, 4107, 4113, 4198, 4197, 4125, 4110, - 4109, 4115, 4114, 4178, 4179, 4180, 4181, 4133, 4134, 4131, 4132, 4128, - 4124, 4120, 4119, 4102, 4101, 4104, 4103, 4118, 4117, 4127, 4122, 4137, - 43636, 43637, 43638, 43632, 43494, 4154, 4150, 4250, 4251, 4237, 4235, - 4236, 4231, 4232, 4233, 4234, 43493, 4171, 43644, 43645, 4201, 4202, - 4203, 4204, 4205, 4151, 4239, 43643, 4170, 4153, 4152, 43639, 43641, - 43640, 4174, 4172, 4255, 4254, 4175, 4173, 4245, 4244, 4247, 4246, 4243, - 4242, 4240, 4249, 4241, 4248, 4195, 4196, 43509, 43508, 43511, 43510, - 43507, 43506, 43504, 43513, 43505, 43512, 4146, 4252, 4253, 4140, 4209, - 4212, 4210, 4211, 4147, 4148, 4194, 4228, 4229, 4230, 4227, 4145, 4149, - 4199, 4200, 4139, 4182, 4183, 4184, 4185, 4143, 4144, 4141, 4142, 4165, - 4164, 4167, 4166, 4163, 4162, 4160, 4169, 4161, 4168, 983218, 983232, - 983174, 10753, 10754, 10752, 8720, 10761, 10757, 10758, 8721, 8899, - 10756, 10755, 11007, 8896, 8897, 8898, 8719, 67712, 67728, 67740, 67724, - 67726, 67714, 67732, 67718, 67730, 67723, 67742, 67717, 67729, 67738, - 67739, 67713, 67735, 67716, 67721, 67734, 67737, 67741, 67725, 67719, - 67722, 67727, 67715, 67733, 67720, 67736, 67731, 67758, 67752, 67753, - 67757, 67751, 67759, 67756, 67754, 67755, 8711, 124117, 124120, 124119, - 124121, 124118, 124132, 124136, 124133, 124138, 124137, 124134, 124135, - 124122, 124124, 124126, 124123, 124125, 124112, 124116, 124114, 124113, - 124115, 124127, 124128, 124129, 124130, 124131, 124140, 124143, 124142, - 124139, 124141, 124149, 124148, 124151, 124150, 124147, 124146, 124144, - 124153, 124145, 124152, 128133, 8358, 8892, 72102, 72103, 72138, 72144, - 72136, 72096, 72097, 72107, 72109, 72123, 72122, 72128, 72127, 72121, - 72120, 72126, 72125, 72100, 72101, 72098, 72099, 72143, 72137, 72114, - 72124, 72119, 72129, 72139, 72140, 72141, 72133, 72132, 72116, 72115, - 72113, 72112, 72118, 72117, 72111, 72110, 72131, 72130, 72142, 72134, - 72135, 72106, 72108, 72162, 72161, 72158, 72160, 72159, 72164, 72150, - 72151, 72145, 72155, 72157, 72148, 72149, 72146, 72147, 72154, 72156, - 72163, 8302, 127966, 129314, 128219, 129535, 8239, 983092, 983197, - 983128, 9471, 9453, 9460, 9452, 9458, 9451, 9454, 9455, 9459, 9456, 9457, - 127312, 127313, 127314, 127315, 127316, 127317, 127318, 127319, 127320, - 127321, 127322, 127323, 127324, 127325, 127326, 127327, 127328, 127329, - 127330, 127331, 127332, 127333, 127334, 127335, 127336, 127337, 128982, - 128984, 129982, 129981, 129983, 127344, 127345, 127346, 127347, 127348, - 127349, 127350, 127351, 127352, 127353, 127354, 127355, 127356, 127357, - 127358, 127359, 127360, 127361, 127362, 127363, 127364, 127365, 127366, - 127367, 127368, 127369, 10062, 129988, 127374, 127371, 127375, 127372, - 127373, 983091, 8879, 8840, 8841, 8775, 8821, 8817, 8825, 8820, 8816, - 8824, 129670, 129722, 11228, 129540, 129544, 129565, 129586, 129603, - 129607, 129561, 129536, 129557, 129599, 129539, 129560, 129602, 129610, - 129613, 129541, 129562, 129604, 129537, 129558, 129600, 129538, 129559, - 129601, 129581, 129578, 129582, 129583, 129579, 129580, 128528, 9906, - 127770, 127761, 8362, 6595, 6594, 6599, 6598, 6597, 6596, 6593, 6566, - 6530, 6567, 6531, 6570, 6537, 6532, 6544, 6543, 6536, 6542, 6549, 6548, - 6562, 6561, 6554, 6560, 6556, 6550, 6528, 6555, 6538, 6568, 6533, 6569, - 6534, 6571, 6540, 6535, 6547, 6546, 6539, 6545, 6552, 6551, 6565, 6564, - 6557, 6563, 6559, 6553, 6529, 6558, 6541, 6622, 6623, 6600, 6601, 6618, - 6577, 6587, 6582, 6586, 6578, 6592, 6583, 6584, 6590, 6589, 6579, 6585, - 6591, 6580, 6588, 6576, 6581, 6613, 6612, 6615, 6614, 6611, 6610, 6608, - 6617, 6609, 6616, 983063, 70732, 70746, 70731, 70741, 70740, 70743, - 70742, 70739, 70738, 70736, 70745, 70737, 70744, 70734, 70675, 70674, - 70681, 70680, 70692, 70686, 70691, 70751, 70662, 70663, 70664, 70665, - 70656, 70657, 70667, 70669, 70685, 70684, 70690, 70689, 70683, 70682, - 70688, 70687, 70660, 70661, 70658, 70659, 70705, 70706, 70707, 70696, - 70695, 70677, 70676, 70673, 70672, 70679, 70678, 70671, 70670, 70703, - 70702, 70698, 70697, 70694, 70693, 70701, 70700, 70708, 70704, 70699, - 70666, 70668, 70727, 70724, 70723, 70726, 70752, 70728, 70753, 70722, - 70725, 70730, 70750, 70747, 70709, 70719, 70721, 70714, 70715, 70716, - 70717, 70712, 70713, 70710, 70711, 70718, 70720, 70735, 70749, 70733, - 70729, 11154, 11155, 128240, 9112, 983131, 9798, 11209, 128084, 129299, - 983132, 128985, 129399, 127747, 2031, 2032, 2033, 2030, 2034, 2027, 2028, - 2029, 2035, 2040, 2045, 2046, 1989, 1988, 1991, 1990, 1987, 1986, 1984, - 1993, 1985, 1992, 2042, 2008, 2001, 2025, 2024, 2026, 2006, 2002, 2019, - 2016, 2018, 2023, 2010, 2009, 2000, 1999, 2007, 1997, 1995, 2012, 2003, - 2013, 2020, 2014, 2015, 2017, 2004, 2011, 2005, 2021, 2022, 1994, 1996, - 1998, 2037, 2036, 2039, 2038, 2041, 2047, 983127, 128691, 9940, 128683, - 128695, 128370, 128286, 128245, 128685, 8303, 65934, 8209, 128689, 10973, - 8893, 8599, 10542, 10545, 10536, 10532, 10530, 128602, 128594, 128610, - 11111, 11127, 11016, 8663, 129109, 11008, 43063, 43062, 43065, 43064, - 43059, 43060, 43057, 43056, 43061, 43058, 10529, 8598, 8689, 8632, 10546, - 10535, 10531, 128600, 128592, 128608, 11110, 11126, 11017, 8662, 129108, - 11009, 128746, 8882, 8884, 8379, 8836, 8837, 8713, 8772, 8777, 8938, - 8940, 8742, 8930, 8931, 172, 8877, 8769, 8813, 8800, 8802, 8815, 8814, - 9083, 128323, 10159, 128324, 10161, 128456, 128457, 128458, 128211, - 128212, 160, 128067, 9369, 9362, 9365, 9366, 9367, 35, 9368, 9364, 9361, - 9363, 9371, 9370, 8470, 110960, 110961, 110962, 110963, 110964, 110965, - 110966, 110967, 110968, 110969, 110970, 110971, 110972, 110973, 110974, - 110975, 110976, 110977, 110978, 110979, 110980, 110981, 110982, 110983, - 110984, 110985, 110986, 110987, 110988, 110989, 110990, 110991, 110992, - 110993, 110994, 110995, 110996, 110997, 110998, 110999, 111000, 111001, - 111002, 111003, 111004, 111005, 111006, 111007, 111008, 111009, 111010, - 111011, 111012, 111013, 111014, 111015, 111016, 111017, 111018, 111019, - 111020, 111021, 111022, 111023, 111024, 111025, 111026, 111027, 111028, - 111029, 111030, 111031, 111032, 111033, 111034, 111035, 111036, 111037, - 111038, 111039, 111040, 111041, 111042, 111043, 111044, 111045, 111046, - 111047, 111048, 111049, 111050, 111051, 111052, 111053, 111054, 111055, - 111056, 111057, 111058, 111059, 111060, 111061, 111062, 111063, 111064, - 111065, 111066, 111067, 111068, 111069, 111070, 111071, 111072, 111073, - 111074, 111075, 111076, 111077, 111078, 111079, 111080, 111081, 111082, - 111083, 111084, 111085, 111086, 111087, 111088, 111089, 111090, 111091, - 111092, 111093, 111094, 111095, 111096, 111097, 111098, 111099, 111100, - 111101, 111102, 111103, 111104, 111105, 111106, 111107, 111108, 111109, - 111110, 111111, 111112, 111113, 111114, 111115, 111116, 111117, 111118, - 111119, 111120, 111121, 111122, 111123, 111124, 111125, 111126, 111127, - 111128, 111129, 111130, 111131, 111132, 111133, 111134, 111135, 111136, - 111137, 111138, 111139, 111140, 111141, 111142, 111143, 111144, 111145, - 111146, 111147, 111148, 111149, 111150, 111151, 111152, 111153, 111154, - 111155, 111156, 111157, 111158, 111159, 111160, 111161, 111162, 111163, - 111164, 111165, 111166, 111167, 111168, 111169, 111170, 111171, 111172, - 111173, 111174, 111175, 111176, 111177, 111178, 111179, 111180, 111181, - 111182, 111183, 111184, 111185, 111186, 111187, 111188, 111189, 111190, - 111191, 111192, 111193, 111194, 111195, 111196, 111197, 111198, 111199, - 111200, 111201, 111202, 111203, 111204, 111205, 111206, 111207, 111208, - 111209, 111210, 111211, 111212, 111213, 111214, 111215, 111216, 111217, - 111218, 111219, 111220, 111221, 111222, 111223, 111224, 111225, 111226, - 111227, 111228, 111229, 111230, 111231, 111232, 111233, 111234, 111235, - 111236, 111237, 111238, 111239, 111240, 111241, 111242, 111243, 111244, - 111245, 111246, 111247, 111248, 111249, 111250, 111251, 111252, 111253, - 111254, 111255, 111256, 111257, 111258, 111259, 111260, 111261, 111262, - 111263, 111264, 111265, 111266, 111267, 111268, 111269, 111270, 111271, - 111272, 111273, 111274, 111275, 111276, 111277, 111278, 111279, 111280, - 111281, 111282, 111283, 111284, 111285, 111286, 111287, 111288, 111289, - 111290, 111291, 111292, 111293, 111294, 111295, 111296, 111297, 111298, - 111299, 111300, 111301, 111302, 111303, 111304, 111305, 111306, 111307, - 111308, 111309, 111310, 111311, 111312, 111313, 111314, 111315, 111316, - 111317, 111318, 111319, 111320, 111321, 111322, 111323, 111324, 111325, - 111326, 111327, 111328, 111329, 111330, 111331, 111332, 111333, 111334, - 111335, 111336, 111337, 111338, 111339, 111340, 111341, 111342, 111343, - 111344, 111345, 111346, 111347, 111348, 111349, 111350, 111351, 111352, - 111353, 111354, 111355, 94177, 128297, 983041, 983040, 123215, 123149, - 123155, 123138, 123166, 123164, 123148, 123143, 123161, 123153, 123152, - 123141, 123137, 123156, 123139, 123163, 123142, 123172, 123173, 123171, - 123158, 123140, 123167, 123176, 123177, 123165, 123151, 123168, 123136, - 123169, 123162, 123178, 123179, 123144, 123157, 123170, 123150, 123145, - 123159, 123146, 123154, 123160, 123147, 123174, 123175, 123180, 123214, - 123193, 123192, 123195, 123191, 123194, 123196, 123197, 123184, 123190, - 123189, 123186, 123185, 123188, 123187, 123205, 123204, 123207, 123206, - 123203, 123202, 123200, 123209, 123201, 123208, 983231, 983066, 10663, + 128669, 128018, 128053, 127889, 129390, 118261, 128496, 129742, 129439, + 128332, 129334, 128741, 128757, 129468, 128739, 9968, 128670, 128672, + 128693, 128507, 128001, 129700, 128045, 128068, 128511, 127909, 92748, + 92744, 92761, 92750, 92739, 92751, 92737, 92754, 92749, 92753, 92743, + 92752, 92757, 92766, 92736, 92741, 92746, 92764, 92745, 92765, 92755, + 92760, 92758, 92756, 92763, 92762, 92747, 92738, 92740, 92759, 92742, + 92783, 92782, 92773, 92772, 92775, 92774, 92771, 92770, 92768, 92777, + 92769, 92776, 70291, 70292, 70290, 70297, 70296, 70293, 70287, 70298, + 70312, 70311, 70306, 70285, 70284, 70289, 70288, 70295, 70294, 70303, + 70301, 70283, 70282, 70280, 70278, 70277, 70276, 70300, 70299, 70310, + 70307, 70304, 70309, 70308, 70305, 70272, 70275, 70273, 70274, 70313, + 127926, 215, 10804, 10805, 10807, 10811, 10801, 10800, 10005, 8844, 8845, + 8846, 8888, 127812, 117860, 9838, 9837, 9839, 127929, 127896, 119161, + 119159, 119155, 119157, 119061, 119060, 119224, 119235, 119132, 119058, + 119059, 119255, 119253, 119130, 119131, 119163, 119169, 119149, 119167, + 119168, 119210, 119178, 119173, 119211, 119150, 119151, 119152, 119153, + 119154, 119175, 119212, 119213, 119166, 119164, 119141, 119142, 119176, + 119179, 119143, 119144, 119145, 119165, 119177, 119170, 119174, 119092, + 119052, 119247, 119186, 119073, 119109, 119093, 119050, 119220, 119221, + 119044, 119049, 119187, 119209, 119082, 119041, 119083, 119077, 119078, + 119162, 119160, 119208, 119156, 119158, 119136, 119102, 119056, 119057, + 119146, 119147, 119148, 119042, 119066, 119065, 119069, 119185, 119074, + 119075, 119076, 119231, 119232, 119085, 119084, 119070, 119071, 119072, + 119218, 119217, 119189, 119188, 119248, 119249, 119172, 119171, 119216, + 119134, 119100, 119206, 119262, 119270, 119271, 119263, 119268, 119269, + 119272, 119264, 119267, 119266, 119265, 119274, 119223, 119234, 119233, + 119046, 119222, 119184, 119227, 119237, 119228, 119122, 119123, 119081, + 119098, 119207, 119129, 119087, 119086, 119128, 119140, 119106, 119062, + 119195, 119204, 119205, 119196, 119197, 119198, 119199, 119200, 119201, + 119202, 119203, 119094, 119095, 119126, 119108, 119215, 119214, 119261, + 119252, 119257, 119258, 119183, 119090, 119091, 119135, 119101, 119096, + 119097, 119053, 119054, 119055, 119048, 119043, 119047, 119180, 119254, + 119259, 119225, 119236, 119226, 119229, 119238, 119230, 119051, 119045, + 119089, 119088, 119040, 119067, 119068, 119137, 119103, 119139, 119105, + 119110, 119111, 119250, 119181, 119273, 119243, 119244, 119245, 119246, + 119242, 119240, 119239, 119241, 119064, 119138, 119104, 119063, 119256, + 119260, 119190, 119118, 119119, 119120, 119121, 119112, 119113, 119116, + 119117, 119114, 119115, 119124, 119125, 119191, 119193, 119194, 119127, + 119251, 119107, 119133, 119099, 119219, 119192, 119182, 127932, 127925, + 8811, 8810, 4158, 4156, 4157, 4155, 4192, 4191, 4190, 4226, 4129, 43642, + 4138, 4135, 4208, 4207, 4206, 4159, 4099, 4098, 4097, 43625, 43624, + 43626, 43623, 43622, 43621, 43627, 43618, 43617, 43630, 43629, 43620, + 43619, 983244, 43631, 43616, 43635, 43628, 43633, 43634, 4096, 4188, + 4189, 4187, 4186, 4136, 4121, 4106, 4111, 4100, 4105, 4116, 4238, 4123, + 4176, 43491, 4218, 4220, 43490, 4221, 4224, 43492, 4223, 43489, 4216, + 43488, 4215, 4214, 4213, 4219, 4222, 4225, 4217, 4130, 43646, 43647, + 4193, 4177, 4126, 4112, 43503, 43495, 43502, 43501, 43516, 43515, 43518, + 43517, 43498, 43497, 43500, 43499, 43514, 43496, 4108, 4107, 4113, 4198, + 4197, 4125, 4110, 4109, 4115, 4114, 4133, 4134, 4178, 4179, 4180, 4181, + 4131, 4132, 4128, 4124, 4120, 4119, 4102, 4101, 4104, 4103, 4118, 4117, + 4127, 4122, 4137, 43636, 43637, 43638, 43632, 43494, 4245, 4244, 4247, + 4246, 4243, 4242, 4240, 4249, 4241, 4248, 4154, 4150, 4250, 4251, 4237, + 4235, 4236, 4231, 4232, 4233, 4234, 43493, 4171, 43644, 43645, 4201, + 4202, 4203, 4204, 4205, 4151, 4239, 43643, 4170, 4153, 4152, 43639, + 43641, 43640, 4174, 4172, 4255, 4254, 4175, 4173, 71391, 71390, 71393, + 71392, 71389, 71388, 71386, 71395, 71387, 71394, 4195, 4196, 43509, + 43508, 43511, 43510, 43507, 43506, 43504, 43513, 43505, 43512, 4146, + 4252, 4253, 4140, 4209, 4212, 4210, 4211, 4147, 4148, 4228, 4229, 4230, + 4227, 4194, 4145, 4149, 4199, 4200, 4139, 4143, 4144, 4182, 4183, 4184, + 4185, 4141, 4142, 71381, 71380, 71383, 71382, 71379, 71378, 71376, 71385, + 71377, 71384, 4165, 4164, 4167, 4166, 4163, 4162, 4160, 4169, 4161, 4168, + 983218, 983232, 983174, 10753, 10754, 10752, 8720, 10761, 10757, 10758, + 8721, 8899, 10756, 10755, 11007, 8896, 8897, 8898, 8719, 67712, 67728, + 67740, 67724, 67726, 67714, 67732, 67718, 67730, 67723, 67742, 67717, + 67729, 67738, 67739, 67713, 67735, 67716, 67721, 67734, 67737, 67741, + 67725, 67719, 67722, 67727, 67715, 67733, 67720, 67736, 67731, 67758, + 67752, 67753, 67757, 67751, 67759, 67756, 67754, 67755, 8711, 124117, + 124120, 124119, 124121, 124118, 124132, 124136, 124133, 124138, 124137, + 124134, 124135, 124122, 124124, 124126, 124123, 124125, 124112, 124116, + 124114, 124113, 124115, 124127, 124128, 124129, 124130, 124131, 124140, + 124143, 124142, 124139, 124141, 124149, 124148, 124151, 124150, 124147, + 124146, 124144, 124153, 124145, 124152, 128133, 8358, 8892, 72102, 72103, + 72138, 72096, 72097, 72107, 72109, 72123, 72122, 72128, 72127, 72144, + 72136, 72121, 72120, 72126, 72125, 72100, 72101, 72098, 72099, 72143, + 72137, 72114, 72124, 72119, 72129, 72139, 72140, 72141, 72133, 72132, + 72116, 72115, 72113, 72112, 72118, 72117, 72111, 72110, 72131, 72130, + 72142, 72134, 72135, 72106, 72108, 72162, 72161, 72158, 72160, 72159, + 72164, 72150, 72151, 72145, 72155, 72157, 72148, 72149, 72146, 72147, + 72154, 72156, 72163, 8302, 127966, 129314, 128219, 129535, 8239, 983092, + 983197, 983128, 9471, 9453, 9460, 9452, 9458, 9451, 9454, 9455, 9459, + 9456, 9457, 127312, 127313, 127314, 127315, 127316, 127317, 127318, + 127319, 127320, 127321, 127322, 127323, 127324, 127325, 127326, 127327, + 127328, 127329, 127330, 127331, 127332, 127333, 127334, 127335, 127336, + 127337, 128982, 128984, 129982, 129981, 129983, 127344, 127345, 127346, + 127347, 127348, 127349, 127350, 127351, 127352, 127353, 127354, 127355, + 127356, 127357, 127358, 127359, 127360, 127361, 127362, 127363, 127364, + 127365, 127366, 127367, 127368, 127369, 129204, 129205, 129207, 129988, + 10062, 127374, 127371, 127375, 129206, 127372, 127373, 983091, 8879, + 8840, 8841, 8775, 8821, 8817, 8825, 8820, 8816, 8824, 129670, 129722, + 11228, 129540, 129544, 129565, 129586, 129603, 129607, 129561, 129536, + 129557, 129599, 129539, 129560, 129602, 129610, 129613, 129541, 129562, + 129604, 129537, 129558, 129600, 129538, 129559, 129601, 129581, 129578, + 129582, 129583, 129579, 129580, 128528, 9906, 127770, 127761, 8362, 6595, + 6594, 6599, 6598, 6597, 6596, 6593, 6566, 6530, 6567, 6531, 6570, 6537, + 6532, 6544, 6543, 6536, 6542, 6549, 6548, 6562, 6561, 6554, 6560, 6556, + 6550, 6528, 6555, 6538, 6568, 6533, 6569, 6534, 6571, 6540, 6535, 6547, + 6546, 6539, 6545, 6552, 6551, 6565, 6564, 6557, 6563, 6559, 6553, 6529, + 6558, 6541, 6622, 6623, 6600, 6601, 6618, 6577, 6587, 6582, 6586, 6578, + 6592, 6583, 6584, 6590, 6589, 6579, 6585, 6591, 6580, 6588, 6576, 6581, + 6613, 6612, 6615, 6614, 6611, 6610, 6608, 6617, 6609, 6616, 983063, + 70732, 70746, 70731, 70741, 70740, 70743, 70742, 70739, 70738, 70736, + 70745, 70737, 70744, 70734, 70675, 70674, 70681, 70680, 70692, 70686, + 70691, 70751, 70662, 70663, 70664, 70665, 70656, 70657, 70667, 70669, + 70685, 70684, 70690, 70689, 70683, 70682, 70688, 70687, 70660, 70661, + 70658, 70659, 70705, 70706, 70707, 70696, 70695, 70677, 70676, 70673, + 70672, 70679, 70678, 70671, 70670, 70703, 70702, 70698, 70697, 70694, + 70693, 70701, 70700, 70708, 70704, 70699, 70666, 70668, 70723, 70726, + 70727, 70724, 70752, 70728, 70753, 70722, 70725, 70730, 70750, 70747, + 70709, 70719, 70721, 70712, 70713, 70714, 70715, 70716, 70717, 70710, + 70711, 70718, 70720, 70735, 70749, 70733, 70729, 11154, 11155, 128240, + 9112, 983131, 9798, 11209, 128084, 129299, 983132, 128985, 129399, + 127747, 2035, 2031, 2032, 2033, 2030, 2034, 2027, 2028, 2029, 2040, 2045, + 2046, 1989, 1988, 1991, 1990, 1987, 1986, 1984, 1993, 1985, 1992, 2042, + 2008, 2001, 2025, 2024, 2026, 2006, 2002, 2019, 2016, 2018, 2023, 2010, + 2009, 2000, 1999, 2007, 1997, 1995, 2012, 2003, 2013, 2020, 2014, 2015, + 2017, 2004, 2011, 2005, 2021, 2022, 1994, 1996, 1998, 2037, 2036, 2039, + 2038, 2041, 2047, 983127, 128691, 9940, 128683, 128695, 128370, 128286, + 128245, 128685, 8303, 65934, 8209, 128689, 10973, 8893, 8599, 10542, + 10545, 10536, 10532, 129209, 10530, 128602, 128594, 128610, 11111, 11127, + 11016, 8663, 129109, 11008, 43063, 43062, 43065, 43064, 43059, 43060, + 43057, 43056, 43061, 43058, 10529, 8598, 8689, 8632, 10546, 10535, 10531, + 129208, 128600, 128592, 128608, 11110, 11126, 11017, 8662, 129108, 11009, + 128746, 8882, 8884, 8379, 8836, 8837, 8713, 8772, 8777, 8938, 8940, 8742, + 8930, 8931, 172, 8877, 8769, 8813, 8800, 8802, 8815, 8814, 9083, 128323, + 10159, 128324, 10161, 128456, 128457, 128458, 128211, 128212, 160, + 128067, 9369, 9362, 9365, 9366, 9367, 35, 9368, 9364, 9361, 9363, 9371, + 9370, 8470, 110960, 110961, 110962, 110963, 110964, 110965, 110966, + 110967, 110968, 110969, 110970, 110971, 110972, 110973, 110974, 110975, + 110976, 110977, 110978, 110979, 110980, 110981, 110982, 110983, 110984, + 110985, 110986, 110987, 110988, 110989, 110990, 110991, 110992, 110993, + 110994, 110995, 110996, 110997, 110998, 110999, 111000, 111001, 111002, + 111003, 111004, 111005, 111006, 111007, 111008, 111009, 111010, 111011, + 111012, 111013, 111014, 111015, 111016, 111017, 111018, 111019, 111020, + 111021, 111022, 111023, 111024, 111025, 111026, 111027, 111028, 111029, + 111030, 111031, 111032, 111033, 111034, 111035, 111036, 111037, 111038, + 111039, 111040, 111041, 111042, 111043, 111044, 111045, 111046, 111047, + 111048, 111049, 111050, 111051, 111052, 111053, 111054, 111055, 111056, + 111057, 111058, 111059, 111060, 111061, 111062, 111063, 111064, 111065, + 111066, 111067, 111068, 111069, 111070, 111071, 111072, 111073, 111074, + 111075, 111076, 111077, 111078, 111079, 111080, 111081, 111082, 111083, + 111084, 111085, 111086, 111087, 111088, 111089, 111090, 111091, 111092, + 111093, 111094, 111095, 111096, 111097, 111098, 111099, 111100, 111101, + 111102, 111103, 111104, 111105, 111106, 111107, 111108, 111109, 111110, + 111111, 111112, 111113, 111114, 111115, 111116, 111117, 111118, 111119, + 111120, 111121, 111122, 111123, 111124, 111125, 111126, 111127, 111128, + 111129, 111130, 111131, 111132, 111133, 111134, 111135, 111136, 111137, + 111138, 111139, 111140, 111141, 111142, 111143, 111144, 111145, 111146, + 111147, 111148, 111149, 111150, 111151, 111152, 111153, 111154, 111155, + 111156, 111157, 111158, 111159, 111160, 111161, 111162, 111163, 111164, + 111165, 111166, 111167, 111168, 111169, 111170, 111171, 111172, 111173, + 111174, 111175, 111176, 111177, 111178, 111179, 111180, 111181, 111182, + 111183, 111184, 111185, 111186, 111187, 111188, 111189, 111190, 111191, + 111192, 111193, 111194, 111195, 111196, 111197, 111198, 111199, 111200, + 111201, 111202, 111203, 111204, 111205, 111206, 111207, 111208, 111209, + 111210, 111211, 111212, 111213, 111214, 111215, 111216, 111217, 111218, + 111219, 111220, 111221, 111222, 111223, 111224, 111225, 111226, 111227, + 111228, 111229, 111230, 111231, 111232, 111233, 111234, 111235, 111236, + 111237, 111238, 111239, 111240, 111241, 111242, 111243, 111244, 111245, + 111246, 111247, 111248, 111249, 111250, 111251, 111252, 111253, 111254, + 111255, 111256, 111257, 111258, 111259, 111260, 111261, 111262, 111263, + 111264, 111265, 111266, 111267, 111268, 111269, 111270, 111271, 111272, + 111273, 111274, 111275, 111276, 111277, 111278, 111279, 111280, 111281, + 111282, 111283, 111284, 111285, 111286, 111287, 111288, 111289, 111290, + 111291, 111292, 111293, 111294, 111295, 111296, 111297, 111298, 111299, + 111300, 111301, 111302, 111303, 111304, 111305, 111306, 111307, 111308, + 111309, 111310, 111311, 111312, 111313, 111314, 111315, 111316, 111317, + 111318, 111319, 111320, 111321, 111322, 111323, 111324, 111325, 111326, + 111327, 111328, 111329, 111330, 111331, 111332, 111333, 111334, 111335, + 111336, 111337, 111338, 111339, 111340, 111341, 111342, 111343, 111344, + 111345, 111346, 111347, 111348, 111349, 111350, 111351, 111352, 111353, + 111354, 111355, 94177, 128297, 983041, 983040, 123149, 123155, 123138, + 123166, 123164, 123148, 123143, 123161, 123153, 123152, 123141, 123137, + 123156, 123139, 123163, 123142, 123172, 123173, 123140, 123167, 123171, + 123158, 123176, 123177, 123165, 123151, 123168, 123136, 123169, 123162, + 123178, 123179, 123144, 123157, 123170, 123150, 123145, 123159, 123146, + 123154, 123160, 123147, 123174, 123175, 123180, 123214, 123193, 123192, + 123195, 123194, 123191, 123196, 123197, 123184, 123190, 123189, 123186, + 123185, 123188, 123187, 123215, 123205, 123204, 123207, 123206, 123203, + 123202, 123200, 123209, 123201, 123208, 117776, 983231, 983066, 10663, 10662, 11869, 9215, 65532, 9287, 9286, 9284, 9285, 9289, 9281, 9290, 9288, 9282, 9283, 9280, 128721, 128025, 128885, 127970, 5787, 5788, 5776, 5761, 5786, 5770, 5769, 5781, 5779, 5785, 5763, 5772, 5780, 5784, 5762, @@ -11404,34 +12171,39 @@ static const unsigned int dawg_pos_to_codepoint[] = { 7280, 7282, 7281, 7279, 7271, 7270, 7272, 7269, 7258, 7263, 7278, 7268, 7283, 7273, 7284, 7285, 7287, 7286, 7276, 7274, 7275, 7277, 7290, 7288, 7289, 7295, 7294, 7292, 7291, 7253, 7252, 7255, 7254, 7251, 7250, 7248, - 7257, 7249, 7256, 7293, 94179, 94178, 68736, 68739, 68744, 68737, 68756, - 68745, 68760, 68769, 68761, 68775, 68785, 68741, 68762, 68772, 68773, - 68740, 68777, 68742, 68749, 68750, 68758, 68759, 68774, 68776, 68783, - 68784, 68738, 68743, 68747, 68748, 68751, 68754, 68755, 68768, 68770, - 68782, 68765, 68780, 68766, 68781, 68763, 68767, 68764, 68778, 68757, - 68786, 68779, 68746, 68752, 68753, 68771, 68800, 68803, 68808, 68801, - 68820, 68809, 68824, 68833, 68825, 68839, 68849, 68805, 68826, 68836, - 68837, 68804, 68841, 68806, 68813, 68814, 68822, 68823, 68838, 68840, - 68847, 68848, 68802, 68807, 68811, 68812, 68815, 68818, 68819, 68832, - 68834, 68846, 68829, 68844, 68830, 68845, 68827, 68831, 68828, 68842, - 68821, 68850, 68843, 68810, 68816, 68817, 68835, 68861, 68859, 68858, - 68862, 68863, 68860, 66308, 66324, 66318, 66335, 66323, 66331, 66327, - 66330, 66315, 66316, 66317, 66329, 66314, 66306, 66322, 66351, 66321, - 66350, 66326, 66334, 66313, 66333, 66328, 66320, 66312, 66325, 66332, - 66305, 66307, 66311, 66309, 66349, 66310, 66304, 66319, 66339, 66337, - 66338, 66336, 68241, 68242, 68246, 68244, 68226, 68224, 68237, 68235, - 68249, 68251, 68247, 68233, 68248, 68252, 68227, 68234, 68230, 68239, - 68232, 68240, 68231, 68250, 68236, 68225, 68243, 68245, 68228, 68229, - 68238, 68255, 68254, 68253, 66404, 66390, 66392, 66387, 66388, 66411, - 66393, 66421, 66418, 66396, 66397, 66399, 66406, 66405, 66401, 66413, - 66402, 66398, 66414, 66415, 66416, 66408, 66420, 66417, 66407, 66419, - 66389, 66391, 66395, 66400, 66386, 66409, 66410, 66385, 66384, 66394, - 66412, 66403, 66516, 66514, 66515, 66517, 66513, 66464, 66504, 66505, - 66506, 66482, 66510, 66511, 66477, 66508, 66509, 66478, 66479, 66473, - 66474, 66490, 66491, 66480, 66475, 66476, 66507, 66471, 66469, 66470, - 66467, 66468, 66484, 66485, 66492, 66493, 66486, 66487, 66488, 66497, - 66498, 66495, 66472, 66483, 66499, 66494, 66481, 66489, 66496, 66465, - 66466, 66512, 128435, 69414, 69395, 69376, 69394, 69391, 69392, 69398, + 7257, 7249, 7256, 7293, 124374, 124376, 124375, 124377, 124378, 124379, + 124392, 124396, 124395, 124397, 124394, 124393, 124380, 124381, 124383, + 124384, 124385, 124382, 124368, 124371, 124370, 124369, 124372, 124373, + 124386, 124388, 124390, 124389, 124387, 124391, 124399, 124400, 124398, + 124415, 124406, 124405, 124408, 124407, 124404, 124403, 124401, 124410, + 124402, 124409, 94179, 94178, 68736, 68739, 68744, 68737, 68756, 68745, + 68760, 68769, 68761, 68775, 68785, 68741, 68762, 68772, 68773, 68740, + 68777, 68742, 68749, 68750, 68758, 68759, 68774, 68776, 68783, 68784, + 68738, 68743, 68747, 68748, 68751, 68754, 68755, 68768, 68770, 68782, + 68765, 68780, 68766, 68781, 68763, 68767, 68764, 68778, 68757, 68786, + 68779, 68746, 68752, 68753, 68771, 68800, 68803, 68808, 68801, 68820, + 68809, 68824, 68833, 68825, 68839, 68849, 68805, 68826, 68836, 68837, + 68804, 68841, 68806, 68813, 68814, 68822, 68823, 68838, 68840, 68847, + 68848, 68802, 68807, 68811, 68812, 68815, 68818, 68819, 68832, 68834, + 68846, 68829, 68844, 68830, 68845, 68827, 68831, 68828, 68842, 68821, + 68850, 68843, 68810, 68816, 68817, 68835, 68861, 68859, 68858, 68862, + 68863, 68860, 66308, 66324, 66318, 66335, 66323, 66331, 66327, 66330, + 66315, 66316, 66317, 66329, 66314, 66306, 66322, 66351, 66321, 66350, + 66326, 66334, 66313, 66333, 66328, 66320, 66312, 66325, 66332, 66305, + 66307, 66311, 66309, 66349, 66310, 66304, 66319, 66339, 66337, 66338, + 66336, 68241, 68242, 68246, 68244, 68226, 68224, 68237, 68235, 68249, + 68251, 68247, 68233, 68248, 68252, 68227, 68234, 68230, 68239, 68232, + 68240, 68231, 68250, 68236, 68225, 68243, 68245, 68228, 68229, 68238, + 68255, 68254, 68253, 66404, 66390, 66392, 66387, 66388, 66411, 66393, + 66421, 66418, 66396, 66397, 66399, 66406, 66405, 66401, 66413, 66402, + 66398, 66414, 66415, 66416, 66408, 66420, 66417, 66407, 66419, 66389, + 66391, 66395, 66400, 66386, 66409, 66410, 66385, 66384, 66394, 66412, + 66403, 66516, 66514, 66515, 66517, 66513, 66464, 66504, 66505, 66506, + 66482, 66510, 66511, 66477, 66508, 66509, 66478, 66479, 66473, 66474, + 66490, 66491, 66480, 66475, 66476, 66507, 66471, 66469, 66470, 66467, + 66468, 66484, 66485, 66492, 66493, 66486, 66487, 66488, 66497, 66498, + 66495, 66472, 66483, 66499, 66494, 66481, 66489, 66496, 66465, 66466, + 66512, 128435, 118450, 69414, 69395, 69376, 69394, 69391, 69392, 69398, 69399, 69403, 69404, 69377, 69379, 69382, 69400, 69388, 69380, 69384, 69393, 69397, 69401, 69386, 69381, 69385, 69387, 69378, 69390, 69402, 69383, 69396, 69389, 69415, 69407, 69412, 69411, 69406, 69410, 69405, @@ -11449,7 +12221,7 @@ static const unsigned int dawg_pos_to_codepoint[] = { 69507, 69506, 69488, 69502, 69496, 69505, 69492, 69499, 69501, 69503, 69494, 69493, 69490, 69495, 69489, 69498, 69504, 69491, 69500, 69497, 69511, 69512, 69513, 69510, 128477, 128117, 129491, 128116, 129746, - 128283, 128664, 128753, 128660, 128662, 128653, 11819, 8228, 128431, + 128283, 128664, 128753, 128662, 128660, 128653, 11819, 8228, 128431, 129649, 129477, 128214, 9251, 10044, 10027, 10034, 10011, 128194, 128449, 128080, 128237, 128236, 10180, 10179, 128275, 9103, 9104, 10174, 983191, 8997, 128191, 128440, 9934, 9741, 128217, 129505, 129447, 8886, 2902, @@ -11458,214 +12230,230 @@ static const unsigned int dawg_pos_to_codepoint[] = { 2852, 2827, 2912, 2828, 2913, 2869, 2825, 2826, 2823, 2824, 2867, 2866, 2841, 2851, 2846, 2856, 2870, 2871, 2872, 2861, 2860, 2843, 2842, 2840, 2839, 2845, 2844, 2838, 2837, 2859, 2858, 2873, 2862, 2929, 2831, 2835, - 983660, 983659, 2877, 2818, 2817, 2876, 2901, 2893, 2819, 2878, 2888, - 2892, 2883, 2884, 2914, 2915, 2881, 2882, 2879, 2880, 2887, 2891, 2923, + 983660, 983659, 2817, 2876, 2877, 2818, 2901, 2893, 2819, 2878, 2888, + 2892, 2881, 2882, 2883, 2884, 2914, 2915, 2879, 2880, 2887, 2891, 2923, 2922, 2925, 2924, 2921, 2920, 2918, 2927, 2919, 2926, 64830, 64831, 9766, - 10183, 128895, 66736, 66737, 66738, 66739, 66743, 66763, 66761, 66742, - 66749, 66757, 66744, 66768, 66750, 66748, 66754, 66755, 66764, 66762, - 66760, 66746, 66745, 66741, 66765, 66769, 66759, 66758, 66771, 66770, - 66740, 66751, 66752, 66753, 66756, 66767, 66747, 66766, 66776, 66777, - 66778, 66779, 66783, 66803, 66801, 66782, 66789, 66797, 66784, 66808, - 66790, 66788, 66794, 66795, 66804, 66802, 66800, 66786, 66785, 66781, - 66805, 66809, 66799, 66798, 66811, 66810, 66780, 66791, 66792, 66793, - 66796, 66807, 66787, 66806, 66710, 66688, 66715, 66699, 66694, 66698, - 66703, 66693, 66697, 66696, 66705, 66702, 66713, 66717, 66704, 66706, - 66707, 66711, 66716, 66689, 66701, 66700, 66708, 66691, 66695, 66690, - 66692, 66709, 66712, 66714, 66725, 66724, 66727, 66726, 66723, 66722, - 66720, 66729, 66721, 66728, 983192, 126257, 126264, 126258, 126259, - 126265, 126260, 126263, 126267, 126255, 126266, 126256, 126262, 126261, - 126269, 126268, 126254, 126216, 126225, 126252, 126234, 126243, 126222, - 126249, 126213, 126231, 126240, 126221, 126248, 126212, 126230, 126239, - 126217, 126226, 126253, 126235, 126244, 126215, 126224, 126251, 126233, - 126242, 126214, 126223, 126250, 126232, 126241, 126220, 126247, 126211, - 126229, 126238, 126219, 126246, 126210, 126228, 126237, 126218, 126245, - 126209, 126227, 126236, 129446, 128228, 10015, 9885, 10029, 10009, 8485, - 129397, 128471, 11195, 11194, 11196, 8254, 129450, 8486, 128076, 127842, - 128329, 129417, 128002, 983122, 983121, 128463, 128195, 128479, 128196, - 128223, 128464, 128724, 93059, 93065, 93064, 93058, 93070, 93056, 93055, - 93053, 93062, 93069, 93061, 93066, 93071, 93068, 93054, 93057, 93063, - 93067, 93060, 92967, 92975, 92965, 92969, 92959, 92968, 92971, 92957, - 92962, 92960, 92972, 92970, 92963, 92958, 92966, 92961, 92956, 92974, - 92964, 92973, 92979, 92978, 92980, 92977, 92976, 92982, 92981, 93023, - 93020, 93024, 93021, 93019, 93025, 93022, 93043, 92985, 93047, 93044, - 93045, 92997, 93046, 93042, 93032, 93029, 92995, 93038, 92993, 93041, - 93035, 93033, 93037, 93030, 93039, 92987, 92992, 92986, 92983, 92984, - 93027, 92994, 93034, 92988, 92990, 92989, 92991, 93028, 92996, 93031, - 93040, 93036, 92954, 92955, 92938, 92939, 92932, 92933, 92942, 92943, - 92950, 92951, 92928, 92929, 92936, 92937, 92948, 92949, 92930, 92931, - 92944, 92945, 92934, 92935, 92940, 92941, 92946, 92947, 92952, 92953, - 93013, 93012, 93015, 93014, 93011, 93010, 93008, 93017, 93009, 93016, - 9908, 11801, 127796, 129779, 129780, 67703, 67693, 67688, 67702, 67683, - 67691, 67699, 67700, 67680, 67696, 67682, 67686, 67695, 67698, 67701, - 67689, 67684, 67687, 67690, 67681, 67694, 67685, 67697, 67692, 67704, - 67711, 67706, 67707, 67710, 67709, 67708, 67705, 129330, 129374, 128060, - 11853, 8233, 11791, 10995, 10994, 8741, 129666, 12809, 12823, 12808, - 12822, 12828, 12813, 12827, 12810, 12824, 12800, 12814, 12804, 12818, - 12801, 12815, 12812, 12826, 12805, 12819, 12803, 12817, 12806, 12820, - 12811, 12825, 12802, 12816, 12807, 12821, 12863, 12855, 12858, 12861, - 12847, 12839, 12854, 12843, 12836, 12864, 12835, 12856, 12846, 12842, - 12840, 12852, 12862, 12865, 12857, 12867, 12838, 12866, 12851, 12853, - 12859, 12849, 12860, 12848, 12837, 12834, 12841, 12833, 12845, 12844, - 12850, 12832, 12830, 12829, 9349, 9342, 9345, 9346, 9350, 9347, 9348, - 9344, 9351, 9343, 9341, 9336, 9335, 9338, 9337, 9334, 9333, 9340, 9332, - 9339, 127248, 127249, 127250, 127251, 127252, 127253, 127254, 127255, - 127256, 127257, 127258, 127259, 127260, 127261, 127262, 127263, 127264, - 127265, 127266, 127267, 127268, 127269, 127270, 127271, 127272, 127273, - 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380, 9381, 9382, 9383, - 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, 9393, 9394, 9395, - 9396, 9397, 8706, 983147, 983146, 983149, 983150, 9853, 127881, 12880, - 12349, 129436, 11261, 128755, 9105, 9106, 128706, 72437, 72440, 72432, - 72416, 72419, 72413, 72417, 72415, 72412, 72414, 72418, 72420, 72403, - 72407, 72411, 72409, 72410, 72391, 72400, 72404, 72397, 72394, 72385, - 72401, 72384, 72399, 72398, 72396, 72388, 72393, 72392, 72386, 72387, - 72402, 72395, 72390, 72389, 72405, 72406, 72408, 72436, 72435, 72438, - 72439, 72422, 72421, 72424, 72425, 72431, 72433, 72434, 72428, 72427, - 72429, 72430, 72423, 72426, 128062, 128230, 128206, 983228, 983237, - 129434, 9774, 127825, 129372, 129755, 127824, 128039, 128532, 9956, - 128390, 9999, 8240, 8241, 8524, 10178, 10977, 129336, 129494, 129496, - 128113, 9977, 128590, 129733, 128591, 129493, 128589, 128588, 129495, - 128583, 128187, 8966, 128547, 37, 9854, 127917, 8359, 8369, 129515, - 128694, 129730, 43101, 43117, 43120, 43076, 43123, 43077, 43115, 43090, - 43082, 43094, 43098, 43099, 43119, 43118, 43109, 43074, 43075, 43116, - 43079, 43083, 43089, 43088, 43114, 43113, 43081, 43080, 43073, 43072, - 43085, 43084, 43092, 43093, 43104, 43110, 43086, 43108, 43100, 43078, - 43097, 43087, 43106, 43096, 43091, 43107, 43095, 43102, 43105, 43103, - 43127, 43126, 43124, 43121, 43111, 43112, 43122, 43125, 66033, 66023, - 66017, 66010, 66027, 66003, 66018, 66028, 66004, 66012, 66022, 66020, - 66045, 66019, 66031, 66041, 66007, 66006, 66025, 66026, 66038, 66016, - 66013, 66014, 66000, 66001, 66034, 66036, 66037, 66029, 66011, 66024, - 66015, 66021, 66042, 66043, 66002, 66008, 66032, 66005, 66044, 66040, - 66039, 66030, 66009, 66035, 5941, 5942, 67840, 67855, 67843, 67844, - 67847, 67858, 67857, 67860, 67854, 67861, 67848, 67845, 67846, 67849, - 67859, 67842, 67850, 67853, 67851, 67841, 67856, 67852, 67864, 67866, - 67867, 67863, 67862, 67865, 67871, 11227, 9935, 128763, 128022, 128061, - 128055, 128169, 182, 128138, 129292, 129295, 127885, 127821, 10031, - 129655, 129669, 9811, 128299, 8916, 10970, 129383, 8984, 128720, 129703, - 8462, 8463, 127183, 127136, 127167, 127199, 127188, 127140, 127156, - 127172, 127200, 127189, 127141, 127157, 127173, 127196, 127148, 127164, - 127180, 127198, 127150, 127166, 127182, 127192, 127144, 127160, 127176, - 127191, 127143, 127159, 127175, 127190, 127142, 127158, 127174, 127197, - 127149, 127165, 127181, 127194, 127146, 127162, 127178, 127187, 127139, - 127155, 127171, 127186, 127138, 127154, 127170, 127202, 127220, 127221, - 127201, 127210, 127211, 127212, 127213, 127214, 127215, 127216, 127217, - 127218, 127219, 127203, 127204, 127205, 127206, 127207, 127208, 127209, - 127185, 127137, 127153, 127169, 127193, 127145, 127161, 127177, 127195, - 127147, 127163, 127179, 128733, 983151, 43, 10797, 10798, 10809, 10789, - 10786, 10791, 10790, 10788, 10792, 10787, 10866, 177, 9799, 11222, 11221, - 11220, 11219, 129696, 983148, 128659, 128680, 128110, 8297, 8236, 127871, - 128254, 11239, 128239, 12306, 12320, 128238, 8982, 128688, 129364, - 127858, 129716, 127831, 129751, 128574, 128545, 163, 128093, 9212, 9213, - 9214, 9211, 128041, 128425, 129328, 129732, 129731, 65043, 65040, 65045, - 65073, 65074, 65049, 65041, 65042, 65091, 65047, 65083, 65085, 65089, - 65079, 65087, 65077, 65095, 65081, 65075, 65084, 65086, 65092, 983261, - 65048, 65090, 65080, 65088, 65078, 65096, 65082, 65076, 65044, 65072, - 65046, 8478, 8826, 10937, 10933, 10927, 10929, 10935, 10931, 8936, 8830, - 8828, 8880, 9111, 129384, 9113, 128424, 128438, 129332, 128120, 983193, - 983166, 983163, 983164, 983167, 8242, 8965, 8759, 8733, 8522, 128711, - 129455, 11224, 128255, 68507, 68508, 68480, 68483, 68490, 68491, 68485, - 68482, 68486, 68493, 68495, 68496, 68488, 68484, 68487, 68489, 68481, - 68492, 68497, 68494, 68526, 68522, 68523, 68525, 68521, 68527, 68524, - 68505, 68506, 11854, 8200, 128156, 128091, 128686, 128204, 128226, - 983165, 983168, 983194, 9624, 9625, 9626, 9627, 9628, 9629, 9630, 9631, - 9622, 9623, 10764, 8279, 10774, 9833, 128894, 8264, 63, 8799, 34, 9915, - 127949, 127950, 129437, 128251, 9762, 128280, 9143, 128740, 128643, 9926, - 127752, 11827, 11783, 11782, 9995, 128400, 128406, 127338, 127339, - 127340, 9994, 11828, 11787, 129306, 128000, 8758, 128007, 128048, 129682, - 128015, 129534, 9852, 9843, 9844, 9845, 9846, 9847, 9848, 9849, 9850, - 983113, 128665, 127822, 129511, 174, 127462, 127463, 127464, 127465, - 127466, 127467, 127468, 127469, 127470, 127471, 127472, 127473, 127474, - 127475, 127476, 127477, 127478, 127479, 127480, 127481, 127482, 127483, - 127484, 127485, 127486, 127487, 43344, 43343, 43346, 43345, 43330, 43320, - 43333, 43323, 43331, 43314, 43332, 43317, 43319, 43321, 43316, 43313, - 43329, 43322, 43312, 43326, 43318, 43325, 43324, 43315, 43328, 43327, - 43334, 43359, 43337, 43342, 43341, 43338, 43340, 43335, 43339, 43336, - 43347, 128524, 127895, 65533, 9952, 9953, 128699, 8479, 9166, 11152, - 11153, 128639, 128968, 983152, 92, 10184, 10741, 10743, 11073, 11079, - 983153, 10659, 10661, 8246, 12317, 10989, 8976, 11793, 8267, 8245, - 128401, 9753, 11262, 8515, 8271, 8765, 8909, 8247, 128402, 128403, - 128405, 11841, 11822, 128404, 10672, 128158, 8251, 129423, 983154, - 127872, 11190, 11188, 11191, 11189, 11186, 11187, 11184, 11185, 127832, - 127833, 129919, 129918, 128495, 8735, 12297, 10642, 11777, 11776, 9084, - 8894, 10652, 10644, 10228, 8692, 12305, 10648, 125, 9132, 9133, 9131, - 12301, 8969, 11781, 12299, 10608, 10715, 8221, 11817, 129929, 10621, - 8971, 9687, 11241, 9616, 129978, 129971, 129933, 128381, 11805, 8906, - 10198, 129927, 9621, 129980, 41, 9120, 9118, 9119, 11789, 10182, 8908, - 129931, 8217, 11815, 128360, 128361, 128362, 128489, 93, 9126, 9124, - 10638, 10640, 8262, 10636, 11864, 11862, 9125, 11779, 129987, 128493, - 129928, 129930, 11786, 8895, 10702, 129902, 12309, 8866, 11809, 9145, - 12303, 10628, 12311, 10630, 12315, 12313, 10713, 1421, 4053, 4055, - 129308, 9957, 10749, 9002, 187, 128270, 10153, 10552, 8295, 8238, 8235, - 8207, 10813, 8594, 11080, 11084, 10562, 10613, 10612, 129974, 8614, - 10528, 11076, 11075, 10567, 10526, 8677, 8628, 10513, 8699, 129034, 8620, - 10565, 129042, 129026, 8603, 11022, 11023, 8611, 10517, 10516, 129030, - 129178, 129046, 8618, 8696, 8644, 10522, 129193, 129185, 11146, 11157, - 8658, 10499, 8655, 10503, 10524, 10509, 8674, 129195, 129078, 8652, - 10601, 10605, 10591, 10583, 8641, 10600, 10604, 10596, 10587, 10579, - 8640, 129777, 129090, 129094, 129191, 8702, 8649, 129784, 129189, 128622, - 8669, 129082, 129106, 129187, 11106, 11138, 11132, 983242, 11175, 11173, - 129066, 129062, 129058, 129074, 129070, 11122, 11116, 11142, 129170, - 10511, 8667, 10518, 10520, 10519, 11246, 10497, 10496, 10501, 10512, - 8608, 8605, 8680, 8688, 129174, 129086, 11078, 128141, 128735, 11824, - 8790, 8791, 8728, 730, 129680, 128365, 10539, 10544, 65020, 129350, - 983227, 983235, 983225, 983230, 129704, 128640, 128478, 127906, 128764, - 129531, 129315, 128019, 65947, 65942, 65945, 65940, 65943, 8556, 8583, - 8582, 8548, 8558, 8577, 8547, 8544, 8557, 8584, 8559, 8576, 8549, 8581, - 8550, 8553, 8578, 8555, 8545, 8546, 8579, 8554, 8551, 8552, 65938, 65944, - 65936, 65939, 65941, 65937, 65946, 127801, 127989, 10087, 10085, 11213, - 11215, 8506, 128205, 128907, 127588, 127586, 127589, 127584, 127585, - 127587, 128675, 127840, 129302, 8381, 127945, 10740, 69245, 69243, 69244, - 69246, 69241, 69232, 69238, 69229, 69237, 69228, 69242, 69233, 69234, - 69240, 69231, 69239, 69230, 69236, 69227, 69235, 69226, 69225, 69220, - 69219, 69222, 69221, 69218, 69217, 69224, 69216, 69223, 5872, 5869, 5803, - 5802, 5800, 5833, 5837, 5860, 5811, 5859, 5858, 5841, 5851, 5824, 5844, - 5854, 5826, 5846, 5856, 5799, 5814, 5880, 5879, 5877, 5876, 5878, 5792, - 5813, 5815, 5828, 5816, 5819, 5818, 5853, 5852, 5825, 5831, 5864, 5857, - 5827, 5873, 5812, 5810, 5850, 5829, 5820, 5848, 5804, 5862, 5806, 5801, - 5855, 5845, 5807, 5808, 5875, 5809, 5874, 5843, 5821, 5849, 5823, 5805, - 5836, 5840, 5830, 5863, 5835, 5834, 5861, 5842, 5822, 5798, 5839, 5797, - 5817, 5794, 5847, 5832, 5796, 5795, 5865, 5793, 5866, 5838, 5867, 5868, - 5870, 5871, 127933, 127939, 11254, 11252, 11253, 11251, 11255, 11256, - 8360, 983114, 10700, 129466, 129527, 9808, 129474, 9747, 129761, 2049, - 2053, 2051, 2063, 2055, 2052, 2058, 2059, 2062, 2068, 2069, 2056, 2065, - 2048, 2050, 2067, 2054, 2060, 2066, 2061, 2057, 2064, 2073, 2070, 2071, - 2075, 2093, 2072, 2074, 2084, 2088, 2097, 2110, 2098, 2100, 2108, 2099, - 2096, 2101, 2109, 2106, 2104, 2107, 2103, 2105, 2082, 2079, 2076, 2089, - 2086, 2091, 2081, 2078, 2085, 2092, 2083, 2080, 2077, 2090, 2087, 2102, - 128630, 128631, 128632, 128634, 129386, 128752, 128225, 9796, 43138, - 43139, 43150, 43153, 43167, 43166, 43172, 43171, 43165, 43164, 43170, - 43169, 43144, 43145, 43146, 43147, 43182, 43142, 43143, 43151, 43152, - 43140, 43141, 43187, 43181, 43158, 43168, 43163, 43173, 43183, 43184, - 43185, 43177, 43176, 43160, 43159, 43157, 43156, 43162, 43161, 43155, - 43154, 43175, 43174, 43148, 43149, 43186, 43178, 43180, 43179, 43205, - 43136, 43204, 43137, 43215, 43214, 43221, 43220, 43223, 43222, 43219, - 43218, 43216, 43225, 43217, 43224, 43188, 43189, 43200, 43203, 43194, - 43195, 43196, 43197, 43192, 43193, 43201, 43202, 43190, 43191, 43198, - 43199, 129429, 9973, 127927, 127862, 129403, 9878, 129507, 127979, - 127890, 129410, 9807, 128756, 129691, 128437, 8492, 8496, 8497, 8459, - 8464, 8466, 8499, 8472, 8475, 128624, 8495, 8458, 8467, 8500, 8456, - 128220, 983186, 129453, 128186, 167, 8980, 129352, 127793, 128584, 8979, - 130037, 130036, 130039, 130038, 130035, 130034, 130032, 130041, 130033, - 130040, 10802, 9914, 59, 8480, 129324, 9916, 65093, 983169, 8726, 129697, - 11259, 9913, 11250, 129331, 10014, 10061, 10032, 129368, 70086, 70085, - 70101, 70100, 70103, 70102, 70099, 70098, 70096, 70105, 70097, 70104, - 70092, 70106, 70108, 70019, 70020, 70030, 70032, 70046, 70045, 70051, - 70050, 70044, 70043, 70049, 70048, 70025, 70026, 70027, 70028, 70062, - 70023, 70024, 70021, 70022, 70061, 70060, 70037, 70047, 70042, 70052, - 70063, 70064, 70065, 70056, 70055, 70039, 70038, 70036, 70035, 70041, - 70040, 70034, 70033, 70054, 70053, 70066, 70057, 70059, 70058, 70029, - 70031, 70089, 70110, 70111, 70088, 70095, 70107, 70081, 70017, 70016, - 70090, 70082, 70083, 70080, 70018, 70093, 70091, 70094, 70067, 70077, - 70079, 70072, 70073, 70074, 70075, 70070, 70071, 70068, 70069, 70076, - 70078, 70087, 70109, 70084, 129416, 127847, 66684, 66680, 66682, 66665, - 66673, 66679, 66664, 66669, 66647, 66685, 66672, 66683, 66663, 66659, - 66649, 66686, 66674, 66662, 66660, 66656, 66661, 66677, 66678, 66668, - 66676, 66666, 66681, 66640, 66670, 66646, 66645, 66644, 66654, 66641, - 66667, 66658, 66648, 66687, 66657, 66651, 66655, 66643, 66652, 66650, - 66642, 66653, 66671, 66675, 9752, 129768, 128737, 983075, 983078, 9961, - 128674, 127776, 128722, 128717, 11087, 11103, 11086, 10564, 10976, 10985, - 10984, 10974, 10975, 10983, 113825, 113824, 113826, 113827, 127856, - 129651, 9085, 129327, 128703, 128017, 129424, 129335, 10722, 983198, - 983080, 71107, 71106, 71113, 71040, 71131, 71041, 71051, 71053, 71128, + 117826, 10183, 128895, 66736, 66737, 66738, 66739, 66743, 66763, 66761, + 66742, 66749, 66757, 66744, 66768, 66750, 66748, 66754, 66755, 66764, + 66762, 66760, 66746, 66745, 66741, 66765, 66769, 66759, 66758, 66771, + 66770, 66740, 66751, 66752, 66753, 66756, 66767, 66747, 66766, 66776, + 66777, 66778, 66779, 66783, 66803, 66801, 66782, 66789, 66797, 66784, + 66808, 66790, 66788, 66794, 66795, 66804, 66802, 66800, 66786, 66785, + 66781, 66805, 66809, 66799, 66798, 66811, 66810, 66780, 66791, 66792, + 66793, 66796, 66807, 66787, 66806, 66710, 66688, 66715, 66699, 66694, + 66698, 66703, 66693, 66697, 66696, 66705, 66702, 66713, 66717, 66704, + 66706, 66707, 66711, 66716, 66689, 66701, 66700, 66708, 66691, 66695, + 66690, 66692, 66709, 66712, 66714, 66725, 66724, 66727, 66726, 66723, + 66722, 66720, 66729, 66721, 66728, 983192, 126257, 126264, 126258, + 126259, 126265, 126260, 126263, 126267, 126255, 126266, 126256, 126262, + 126261, 126269, 126268, 126254, 126216, 126225, 126252, 126234, 126243, + 126222, 126249, 126213, 126231, 126240, 126221, 126248, 126212, 126230, + 126239, 126217, 126226, 126253, 126235, 126244, 126215, 126224, 126251, + 126233, 126242, 126214, 126223, 126250, 126232, 126241, 126220, 126247, + 126211, 126229, 126238, 126219, 126246, 126210, 126228, 126237, 126218, + 126245, 126209, 126227, 126236, 129446, 128228, 117974, 117975, 117976, + 117977, 117978, 117979, 117980, 117981, 117982, 117983, 117984, 117985, + 117986, 117987, 117988, 117989, 117990, 117991, 117992, 117993, 117994, + 117995, 117996, 117997, 117998, 117999, 10015, 9885, 10029, 10009, + 118005, 118004, 118007, 118006, 118003, 118002, 118000, 118009, 118001, + 118008, 8485, 129397, 128471, 11195, 11194, 11196, 8254, 129450, 8486, + 128076, 127842, 128329, 129417, 128002, 983122, 983121, 128463, 128195, + 128479, 128196, 128223, 128464, 128724, 93059, 93065, 93064, 93058, + 93070, 93056, 93055, 93053, 93062, 93069, 93061, 93066, 93071, 93068, + 93054, 93057, 93063, 93067, 93060, 92967, 92975, 92965, 92969, 92959, + 92968, 92971, 92957, 92962, 92960, 92972, 92970, 92963, 92958, 92966, + 92961, 92956, 92974, 92964, 92973, 92979, 92978, 92980, 92977, 92976, + 92982, 92981, 93023, 93020, 93024, 93021, 93019, 93025, 93022, 93043, + 92985, 93047, 93044, 93045, 92997, 93046, 93042, 93032, 93029, 92995, + 93038, 92993, 93041, 93035, 93033, 93037, 93030, 93039, 92987, 92992, + 92986, 92983, 92984, 93027, 92994, 93034, 92988, 92990, 92989, 92991, + 93028, 92996, 93031, 93040, 93036, 92954, 92955, 92938, 92939, 92932, + 92933, 92942, 92943, 92950, 92951, 92928, 92929, 92936, 92937, 92948, + 92949, 92930, 92931, 92944, 92945, 92934, 92935, 92940, 92941, 92946, + 92947, 92952, 92953, 93013, 93012, 93015, 93014, 93011, 93010, 93008, + 93017, 93009, 93016, 9908, 11801, 127796, 129779, 129780, 67703, 67693, + 67688, 67702, 67683, 67691, 67699, 67700, 67680, 67696, 67682, 67686, + 67695, 67698, 67701, 67689, 67684, 67687, 67690, 67681, 67694, 67685, + 67697, 67692, 67704, 67711, 67706, 67707, 67710, 67709, 67708, 67705, + 129330, 129374, 128060, 8233, 11853, 11791, 10995, 10994, 8741, 129666, + 12809, 12823, 12808, 12822, 12828, 12813, 12827, 12810, 12824, 12800, + 12814, 12804, 12818, 12801, 12815, 12812, 12826, 12805, 12819, 12803, + 12817, 12806, 12820, 12811, 12825, 12802, 12816, 12807, 12821, 12863, + 12855, 12858, 12861, 12847, 12839, 12854, 12843, 12836, 12864, 12835, + 12856, 12846, 12842, 12840, 12852, 12862, 12865, 12857, 12867, 12838, + 12866, 12851, 12853, 12859, 12849, 12860, 12848, 12837, 12834, 12841, + 12833, 12845, 12844, 12850, 12832, 12830, 12829, 9349, 9342, 9345, 9346, + 9350, 9347, 9348, 9344, 9351, 9343, 9341, 9336, 9335, 9338, 9337, 9334, + 9333, 9340, 9332, 9339, 127248, 127249, 127250, 127251, 127252, 127253, + 127254, 127255, 127256, 127257, 127258, 127259, 127260, 127261, 127262, + 127263, 127264, 127265, 127266, 127267, 127268, 127269, 127270, 127271, + 127272, 127273, 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380, + 9381, 9382, 9383, 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, + 9393, 9394, 9395, 9396, 9397, 8706, 983147, 983146, 983149, 983150, 9853, + 127881, 12880, 12349, 129436, 128755, 11261, 9105, 9106, 128706, 72437, + 72440, 72432, 72416, 72419, 72413, 72417, 72415, 72412, 72414, 72418, + 72420, 72403, 72407, 72411, 72409, 72410, 72391, 72400, 72404, 72397, + 72394, 72385, 72401, 72384, 72399, 72398, 72396, 72388, 72393, 72392, + 72386, 72387, 72402, 72395, 72390, 72389, 72405, 72406, 72408, 72436, + 72435, 72438, 72439, 72422, 72421, 72424, 72425, 72431, 72433, 72434, + 72428, 72427, 72429, 72430, 72423, 72426, 128062, 128230, 128206, 983228, + 983237, 129434, 9774, 127825, 129372, 129755, 127824, 128039, 128532, + 9956, 128390, 9999, 8240, 8241, 8524, 10178, 10977, 129336, 129496, + 129494, 128113, 9977, 128590, 129733, 128591, 129493, 128589, 128588, + 129495, 128583, 128187, 8966, 128547, 37, 9854, 127917, 8359, 8369, + 129515, 128694, 129730, 43101, 43117, 43120, 43076, 43123, 43077, 43115, + 43090, 43082, 43094, 43098, 43099, 43119, 43118, 43109, 43074, 43075, + 43116, 43079, 43083, 43089, 43088, 43114, 43113, 43081, 43080, 43073, + 43072, 43085, 43084, 43092, 43093, 43104, 43110, 43086, 43108, 43100, + 43078, 43097, 43087, 43106, 43096, 43091, 43107, 43095, 43102, 43105, + 43103, 43127, 43126, 43124, 43121, 43111, 43112, 43122, 43125, 66033, + 66023, 66017, 66010, 66027, 66003, 66018, 66028, 66004, 66012, 66022, + 66020, 66045, 66019, 66031, 66041, 66007, 66006, 66025, 66026, 66038, + 66016, 66013, 66014, 66000, 66001, 66034, 66036, 66037, 66029, 66011, + 66024, 66015, 66021, 66042, 66043, 66002, 66008, 66032, 66005, 66044, + 66040, 66039, 66030, 66009, 66035, 5941, 5942, 67840, 67855, 67843, + 67844, 67847, 67858, 67857, 67860, 67854, 67861, 67848, 67845, 67846, + 67849, 67859, 67842, 67850, 67853, 67851, 67841, 67856, 67852, 67864, + 67866, 67867, 67863, 67862, 67865, 67871, 11227, 9935, 128763, 128022, + 128061, 128055, 128169, 182, 128138, 129292, 129295, 127885, 127821, + 10031, 129655, 129669, 9811, 128299, 8916, 10970, 129383, 8984, 128720, + 129703, 8462, 8463, 127183, 127136, 127167, 127199, 127188, 127140, + 127156, 127172, 127200, 127189, 127141, 127157, 127173, 127196, 127148, + 127164, 127180, 127198, 127150, 127166, 127182, 127192, 127144, 127160, + 127176, 127191, 127143, 127159, 127175, 127190, 127142, 127158, 127174, + 127197, 127149, 127165, 127181, 127194, 127146, 127162, 127178, 127187, + 127139, 127155, 127171, 127186, 127138, 127154, 127170, 127202, 127220, + 127221, 127201, 127210, 127211, 127212, 127213, 127214, 127215, 127216, + 127217, 127218, 127219, 127203, 127204, 127205, 127206, 127207, 127208, + 127209, 127185, 127137, 127153, 127169, 127193, 127145, 127161, 127177, + 127195, 127147, 127163, 127179, 128733, 983151, 43, 10797, 10798, 10809, + 10789, 10786, 10791, 10790, 10788, 10792, 10787, 10866, 177, 9799, 11222, + 11221, 11220, 11219, 129696, 983148, 117777, 128659, 128680, 128110, + 8297, 8236, 127871, 128254, 11239, 128239, 12306, 12320, 128238, 8982, + 128688, 129364, 127858, 129716, 127831, 129751, 128574, 128545, 163, + 128093, 9212, 9213, 9214, 9211, 128041, 128425, 129328, 129732, 129731, + 65043, 65040, 65045, 65073, 65074, 65049, 65041, 65042, 65091, 65047, + 65083, 65085, 65089, 65079, 65087, 65077, 65095, 65081, 65075, 65084, + 65086, 65092, 983261, 65048, 65090, 65080, 65088, 65078, 65096, 65082, + 65076, 65044, 65072, 65046, 8478, 8826, 10937, 10933, 10927, 10929, + 10935, 10931, 8936, 8830, 8828, 8880, 9111, 129384, 9113, 128424, 128438, + 129332, 128120, 983193, 983166, 983163, 983164, 983167, 8242, 8965, 8759, + 8733, 8522, 128711, 129455, 11224, 128255, 68507, 68508, 68480, 68483, + 68490, 68491, 68485, 68482, 68486, 68493, 68495, 68496, 68488, 68484, + 68487, 68489, 68481, 68492, 68497, 68494, 68526, 68522, 68523, 68525, + 68521, 68527, 68524, 68505, 68506, 11854, 8200, 128156, 128091, 128686, + 128204, 128226, 983165, 983168, 983194, 9624, 9625, 9626, 9627, 9628, + 9629, 9630, 9631, 9622, 9623, 10764, 8279, 10774, 9833, 128894, 8264, 63, + 8799, 34, 9915, 127949, 127950, 129437, 128251, 9762, 128280, 9143, + 128740, 128643, 9926, 127752, 11827, 11783, 11782, 9995, 128400, 128406, + 127338, 127339, 127340, 129996, 11787, 9994, 11828, 129306, 128000, 8758, + 128007, 128048, 129682, 128015, 129534, 117778, 9852, 9843, 9844, 9845, + 9846, 9847, 9848, 9849, 9850, 983113, 128665, 127822, 129511, 174, + 127462, 127463, 127464, 127465, 127466, 127467, 127468, 127469, 127470, + 127471, 127472, 127473, 127474, 127475, 127476, 127477, 127478, 127479, + 127480, 127481, 127482, 127483, 127484, 127485, 127486, 127487, 43344, + 43343, 43346, 43345, 43333, 43323, 43331, 43314, 43332, 43317, 43330, + 43320, 43319, 43321, 43316, 43313, 43329, 43322, 43312, 43326, 43318, + 43325, 43324, 43315, 43328, 43327, 43334, 43359, 43337, 43342, 43341, + 43338, 43340, 43335, 43339, 43336, 43347, 128524, 127895, 65533, 9952, + 9953, 128699, 8479, 9166, 11152, 11153, 128639, 128968, 983152, 92, + 10184, 10741, 10743, 11073, 11079, 983153, 10659, 10661, 8246, 12317, + 10989, 8976, 11793, 8267, 8245, 128401, 9753, 11262, 8515, 8271, 8765, + 8909, 8247, 128402, 128403, 128405, 11841, 11822, 128404, 10672, 128158, + 8251, 129423, 983154, 127872, 11190, 11188, 11191, 11189, 11186, 11187, + 11184, 11185, 127832, 127833, 129919, 129918, 128495, 8735, 12297, 10642, + 11777, 11776, 9084, 8894, 10652, 10644, 10228, 8692, 12305, 10648, 125, + 9132, 9133, 9131, 12301, 8969, 11781, 12299, 10608, 10715, 8221, 11817, + 129929, 10621, 8971, 118272, 9687, 11241, 9616, 129978, 117925, 118289, + 129971, 118286, 118284, 118432, 118443, 129933, 128381, 130025, 130017, + 11805, 8906, 10198, 129927, 9621, 129980, 41, 9120, 9118, 9119, 11789, + 10182, 8908, 129931, 8217, 11815, 128360, 128361, 128362, 128489, 93, + 9126, 9124, 10638, 10640, 8262, 10636, 11864, 11862, 9125, 11779, 117773, + 129987, 128493, 118433, 118442, 129930, 129928, 11786, 8895, 10702, + 129902, 12309, 8866, 11809, 9145, 117766, 12303, 10628, 12311, 10630, + 12315, 12313, 10713, 1421, 117833, 117907, 117909, 129308, 4053, 4055, + 117874, 117878, 9957, 9002, 117858, 10749, 187, 117775, 128270, 117921, + 117848, 117923, 117913, 117863, 117763, 117919, 117882, 117761, 10153, + 10552, 8295, 8238, 8235, 8207, 10813, 8594, 11080, 11084, 10562, 10613, + 10612, 129974, 8614, 10528, 129217, 11076, 11075, 10567, 10526, 8677, + 8628, 10513, 8699, 129202, 8620, 129034, 10565, 129042, 129026, 8603, + 11022, 11023, 8611, 10517, 10516, 129030, 129178, 129046, 8618, 8696, + 8644, 10522, 129193, 129185, 11146, 11157, 8658, 10499, 8655, 10503, + 10524, 10509, 8674, 129195, 129078, 8652, 10601, 10605, 10591, 10583, + 8641, 10600, 10604, 10596, 10587, 10579, 8640, 129777, 129090, 129094, + 129191, 8702, 8649, 129784, 129189, 128622, 8669, 129082, 129106, 129187, + 11106, 11138, 11132, 983242, 11175, 11173, 129066, 129062, 129058, + 129074, 129070, 11122, 11116, 11142, 129170, 10511, 8667, 10518, 10520, + 10519, 11246, 10497, 10496, 10501, 10512, 8608, 8605, 8680, 8688, 129174, + 129086, 11078, 128141, 128735, 11824, 8790, 8791, 8728, 730, 129680, + 128365, 10539, 10544, 65020, 129350, 983227, 983235, 983225, 983230, + 129704, 128640, 128478, 127906, 128764, 129531, 129315, 65947, 65942, + 65945, 65940, 65943, 8556, 8583, 8582, 8548, 8558, 8577, 8547, 8544, + 8557, 8584, 8559, 8576, 8549, 8581, 8550, 8553, 8578, 8555, 8545, 8546, + 8579, 8554, 8551, 8552, 65938, 65944, 65936, 65939, 65941, 65937, 65946, + 129756, 128019, 127801, 127989, 10087, 10085, 11213, 11215, 8506, 128205, + 128907, 127588, 127586, 127589, 127584, 127585, 127587, 128675, 127840, + 129302, 8381, 127945, 10740, 69245, 69243, 69244, 69246, 69241, 69232, + 69238, 69229, 69237, 69228, 69242, 69233, 69234, 69240, 69231, 69239, + 69230, 69236, 69227, 69235, 69226, 69225, 69220, 69219, 69222, 69221, + 69218, 69217, 69224, 69216, 69223, 5872, 5869, 5803, 5802, 5800, 5833, + 5837, 5860, 5811, 5859, 5858, 5841, 5851, 5824, 5844, 5854, 5826, 5846, + 5856, 5799, 5814, 5880, 5879, 5877, 5876, 5878, 5792, 5813, 5815, 5828, + 5816, 5819, 5818, 5853, 5852, 5825, 5831, 5864, 5857, 5827, 5873, 5812, + 5810, 5850, 5829, 5820, 5848, 5804, 5862, 5806, 5801, 5855, 5845, 5807, + 5808, 5875, 5809, 5874, 5843, 5821, 5849, 5823, 5805, 5836, 5840, 5830, + 5863, 5835, 5834, 5861, 5842, 5822, 5798, 5839, 5797, 5817, 5794, 5847, + 5832, 5796, 5795, 5865, 5793, 5866, 5838, 5867, 5868, 5870, 5871, 127933, + 127939, 11254, 11252, 11253, 11251, 11255, 11256, 8360, 983114, 10700, + 129466, 129527, 9808, 129474, 9747, 129761, 2049, 2053, 2051, 2063, 2055, + 2052, 2058, 2059, 2062, 2068, 2069, 2056, 2065, 2048, 2050, 2067, 2054, + 2060, 2066, 2061, 2057, 2064, 2073, 2070, 2071, 2075, 2093, 2072, 2074, + 2084, 2088, 2097, 2110, 2098, 2100, 2108, 2099, 2096, 2101, 2109, 2106, + 2104, 2107, 2103, 2105, 2082, 2079, 2076, 2089, 2086, 2091, 2081, 2078, + 2085, 2092, 2083, 2080, 2077, 2090, 2087, 2102, 128630, 128631, 128632, + 128634, 129386, 128752, 128225, 9796, 43138, 43139, 43150, 43153, 43167, + 43166, 43172, 43171, 43165, 43164, 43170, 43169, 43144, 43145, 43146, + 43147, 43182, 43142, 43143, 43151, 43152, 43140, 43141, 43187, 43181, + 43158, 43168, 43163, 43173, 43183, 43184, 43185, 43177, 43176, 43160, + 43159, 43157, 43156, 43162, 43161, 43155, 43154, 43175, 43174, 43148, + 43149, 43186, 43178, 43180, 43179, 43205, 43136, 43204, 43137, 43215, + 43214, 43221, 43220, 43223, 43222, 43219, 43218, 43216, 43225, 43217, + 43224, 43188, 43189, 43200, 43203, 43192, 43193, 43194, 43195, 43196, + 43197, 43201, 43202, 43190, 43191, 43198, 43199, 129429, 9973, 127927, + 127862, 129403, 9878, 129507, 127979, 127890, 129410, 9807, 128756, + 129691, 128437, 8492, 8496, 8497, 8459, 8464, 8466, 8499, 8472, 8475, + 128624, 8495, 8458, 8467, 8500, 8456, 128220, 983186, 129453, 128186, + 167, 8980, 129352, 127793, 128584, 8979, 130037, 130036, 130039, 130038, + 130035, 130034, 130032, 130041, 130033, 130040, 10802, 9914, 59, 117793, + 117795, 117799, 117807, 117803, 117797, 117805, 117801, 117794, 117798, + 117806, 117802, 117796, 117804, 117800, 118353, 118355, 118359, 118367, + 118383, 118415, 118399, 118375, 118407, 118391, 118363, 118379, 118411, + 118395, 118371, 118403, 118387, 118357, 118365, 118381, 118413, 118397, + 118373, 118405, 118389, 118361, 118377, 118409, 118393, 118369, 118401, + 118385, 118354, 118358, 118366, 118382, 118414, 118398, 118374, 118406, + 118390, 118362, 118378, 118410, 118394, 118370, 118402, 118386, 118356, + 118364, 118380, 118412, 118396, 118372, 118404, 118388, 118360, 118376, + 118408, 118392, 118368, 118400, 118384, 11259, 8480, 129324, 9916, 65093, + 983169, 8726, 129697, 9913, 11250, 129331, 10061, 10032, 10014, 129368, + 70086, 70085, 70101, 70100, 70103, 70102, 70099, 70098, 70096, 70105, + 70097, 70104, 70092, 70106, 70108, 70019, 70020, 70030, 70032, 70046, + 70045, 70051, 70050, 70044, 70043, 70049, 70048, 70025, 70026, 70027, + 70028, 70062, 70023, 70024, 70021, 70022, 70061, 70060, 70037, 70047, + 70042, 70052, 70063, 70064, 70065, 70056, 70055, 70039, 70038, 70036, + 70035, 70041, 70040, 70034, 70033, 70054, 70053, 70066, 70057, 70059, + 70058, 70029, 70031, 70089, 70110, 70111, 70088, 70095, 70107, 70016, + 70090, 70081, 70017, 70082, 70083, 70080, 70018, 70093, 70091, 70094, + 70067, 70077, 70079, 70070, 70071, 70072, 70073, 70074, 70075, 70068, + 70069, 70076, 70078, 70087, 70109, 70084, 129416, 127847, 66684, 66680, + 66682, 66665, 66673, 66679, 66664, 66669, 66647, 66685, 66672, 66683, + 66663, 66659, 66649, 66686, 66674, 66662, 66660, 66656, 66661, 66677, + 66678, 66668, 66676, 66666, 66681, 66640, 66670, 66646, 66645, 66644, + 66654, 66641, 66667, 66658, 66648, 66687, 66657, 66651, 66655, 66643, + 66652, 66650, 66642, 66653, 66671, 66675, 9752, 129768, 128737, 983075, + 983078, 9961, 128674, 127776, 128722, 128717, 11087, 11103, 11086, 10564, + 10976, 10985, 10984, 10974, 10975, 10983, 113825, 113824, 113826, 113827, + 127856, 129651, 9085, 129327, 129679, 128703, 128017, 129424, 129335, + 10722, 983198, 983080, 71113, 71040, 71131, 71041, 71051, 71053, 71128, 71070, 71129, 71130, 71065, 71064, 71069, 71067, 71066, 71072, 71071, 71046, 71047, 71048, 71049, 71082, 71044, 71045, 71042, 71043, 71058, 71068, 71063, 71073, 71083, 71084, 71085, 71077, 71076, 71060, 71059, @@ -11674,138 +12462,139 @@ static const unsigned int dawg_pos_to_codepoint[] = { 71127, 71126, 71125, 71123, 71124, 71117, 71118, 71116, 71121, 71115, 71114, 71122, 71109, 71108, 71105, 71100, 71104, 71101, 71103, 71102, 71132, 71133, 71087, 71097, 71099, 71092, 71093, 71090, 71091, 71088, - 71089, 71096, 71098, 128411, 128410, 128417, 128416, 128409, 128408, - 128415, 128414, 121399, 121397, 121400, 121398, 121402, 121401, 121104, - 121103, 121102, 121388, 121387, 121386, 121482, 121479, 121358, 121359, - 121357, 121360, 121341, 121335, 121339, 121340, 121336, 121334, 121333, - 121338, 121337, 121342, 121368, 121367, 121373, 121356, 121355, 121354, - 121380, 121382, 121381, 121384, 121383, 121385, 121377, 121379, 121378, - 121375, 121376, 121374, 121371, 121369, 121366, 121370, 121372, 121364, - 121365, 121452, 121392, 121352, 121353, 121351, 121499, 121500, 121501, - 121502, 121503, 121470, 121117, 121115, 121119, 121118, 121116, 121456, - 121363, 121362, 121361, 121480, 121455, 120965, 120837, 121000, 120969, - 121027, 121026, 121076, 121074, 121075, 120995, 120991, 120990, 120982, - 121005, 121011, 121044, 121042, 121043, 120950, 120833, 120847, 121002, - 120859, 120863, 120997, 120967, 120845, 121019, 120839, 120993, 120980, - 121038, 121045, 121031, 121009, 120934, 121046, 121047, 120905, 120936, - 120937, 120935, 121065, 120941, 120834, 121068, 121069, 121024, 120915, - 120916, 121040, 120940, 120944, 120946, 120942, 120945, 120947, 120943, - 120949, 121020, 121039, 120948, 121091, 120832, 120838, 120861, 120843, - 120852, 120870, 120844, 120846, 120848, 120865, 120853, 120856, 120885, - 120877, 120878, 120879, 120884, 120857, 120882, 120854, 120855, 120858, - 120883, 121001, 120871, 120850, 120867, 120966, 120972, 120971, 120849, - 120862, 120895, 120894, 120872, 120873, 120893, 120875, 120874, 120866, - 120864, 120996, 120842, 120891, 120890, 121067, 120892, 120887, 121064, - 121066, 121063, 121060, 121061, 121062, 120899, 120881, 121052, 121059, - 121057, 121054, 121055, 121056, 121058, 120851, 120869, 120868, 120841, - 121018, 120974, 120977, 120976, 120975, 120985, 120986, 120983, 120992, - 120988, 120984, 120978, 121032, 121037, 120897, 120896, 120898, 120889, - 120888, 120886, 120880, 121035, 121030, 121033, 121028, 121036, 120860, - 121003, 121012, 121014, 121013, 121007, 121016, 121008, 121015, 121006, - 121077, 121083, 120876, 121084, 121085, 121078, 121090, 121087, 121079, - 121080, 121081, 120840, 121086, 121088, 121089, 120979, 121082, 121092, - 120906, 120922, 120908, 120919, 120920, 120921, 120914, 120910, 120912, - 120900, 120903, 120904, 120902, 120901, 120924, 120909, 120911, 120913, - 120926, 120930, 120931, 120932, 120929, 120933, 120928, 120925, 120927, - 120923, 120957, 120917, 120907, 120836, 121051, 120968, 120973, 120998, - 121072, 121053, 121073, 121070, 121071, 121025, 120970, 120994, 120989, - 120987, 121029, 121041, 120955, 120961, 120956, 120959, 120999, 121004, - 120964, 120960, 120963, 120962, 120958, 120939, 120938, 121023, 121021, - 121022, 121050, 121048, 121049, 121034, 121017, 120951, 120981, 120953, - 121010, 120954, 120952, 120918, 120835, 121451, 121343, 121349, 121347, - 121348, 121346, 121345, 121350, 121344, 121462, 121463, 121464, 121465, - 121466, 121467, 121468, 121469, 121428, 121429, 121427, 121476, 121473, - 121477, 121475, 121474, 121478, 121472, 121471, 121404, 121405, 121403, - 121430, 121409, 121411, 121410, 121406, 121408, 121407, 121421, 121423, - 121422, 121415, 121416, 121417, 121418, 121419, 121420, 121414, 121413, - 121412, 121424, 121425, 121426, 121432, 121431, 121183, 121184, 121182, - 121181, 121187, 121188, 121186, 121185, 121175, 121176, 121174, 121173, - 121179, 121180, 121178, 121177, 121324, 121321, 121323, 121320, 121322, - 121319, 121210, 121209, 121208, 121203, 121207, 121305, 121272, 121271, - 121287, 121286, 121303, 121304, 121302, 121301, 121206, 121205, 121204, - 121198, 121332, 121331, 121129, 121276, 121274, 121275, 121273, 121288, - 121289, 121291, 121290, 121306, 121280, 121278, 121279, 121277, 121295, - 121293, 121294, 121292, 121307, 121314, 121191, 121192, 121190, 121189, - 121193, 121202, 121201, 121200, 121199, 121282, 121281, 121297, 121296, - 121308, 121309, 121310, 121328, 121327, 121194, 121196, 121197, 121195, - 121216, 121215, 121214, 121213, 121212, 121211, 121449, 121125, 121126, - 121121, 121122, 121123, 121124, 121127, 121318, 121316, 121317, 121315, - 121156, 121155, 121154, 121146, 121145, 121144, 121150, 121149, 121148, - 121147, 121230, 121231, 121229, 121228, 121261, 121254, 121247, 121233, - 121232, 121226, 121227, 121225, 121224, 121249, 121248, 121153, 121152, - 121151, 121139, 121135, 121137, 121138, 121136, 121330, 121329, 121128, - 121236, 121262, 121255, 121235, 121234, 121237, 121240, 121239, 121263, - 121256, 121238, 121162, 121161, 121160, 121132, 121133, 121131, 121130, - 121134, 121253, 121142, 121143, 121141, 121140, 121243, 121242, 121241, - 121246, 121245, 121244, 121270, 121269, 121268, 121264, 121257, 121326, - 121325, 121159, 121158, 121157, 121448, 121394, 121393, 121396, 121395, - 121450, 121513, 121514, 121515, 121516, 121517, 121518, 121519, 121505, - 121506, 121507, 121508, 121509, 121510, 121511, 121512, 121312, 121284, - 121299, 121311, 121283, 121298, 121313, 121285, 121300, 121267, 121260, - 121252, 121251, 121266, 121259, 121250, 121265, 121258, 121107, 121106, - 121105, 121454, 121453, 121457, 121120, 121112, 121110, 121114, 121113, - 121111, 121108, 121109, 121101, 121100, 121099, 121481, 121441, 121445, - 121446, 121443, 121444, 121442, 121447, 121390, 121389, 121391, 121440, - 121439, 121437, 121435, 121436, 121434, 121433, 121438, 121459, 121458, - 121460, 121095, 121094, 121093, 121219, 121218, 121217, 121222, 121221, - 121220, 121223, 121172, 121171, 121170, 121168, 121167, 121166, 121165, - 121164, 121163, 121169, 121098, 121097, 121096, 121461, 121483, 129304, - 10912, 10911, 10910, 10909, 10860, 983185, 983183, 8219, 8249, 8218, - 8250, 983158, 983155, 983156, 983159, 70117, 70116, 70119, 70118, 70115, - 70114, 70121, 70113, 70120, 70131, 70132, 70129, 70126, 70125, 70130, - 70128, 70127, 70124, 70123, 70122, 983953, 983954, 983952, 3464, 3463, - 3495, 3501, 3497, 3503, 3510, 3488, 3484, 3490, 3482, 3508, 3513, 3462, - 3475, 3478, 3461, 3474, 3473, 3472, 3471, 3470, 3469, 3466, 3465, 3468, - 3467, 3496, 3502, 3498, 3504, 3511, 3489, 3485, 3491, 3483, 3509, 3512, - 3525, 3499, 3522, 3517, 3505, 3523, 3477, 3476, 3500, 3507, 3487, 3494, - 3526, 3524, 3515, 3520, 3514, 3492, 3493, 3521, 3486, 3563, 3562, 3565, - 3564, 3561, 3560, 3558, 3567, 3559, 3566, 3530, 3458, 3457, 3459, 3570, - 3571, 3546, 3537, 3539, 3542, 3544, 3551, 3536, 3538, 3540, 3545, 3548, - 3549, 3550, 3547, 3535, 3572, 8767, 10046, 128973, 10038, 128303, 8198, - 10042, 128510, 127935, 9975, 128128, 9760, 129448, 128761, 10902, 10904, - 10901, 10903, 11098, 11100, 42611, 128716, 128164, 128564, 128554, - 128373, 128759, 127829, 128577, 128578, 10840, 10839, 9011, 127920, - 129445, 65120, 128745, 65121, 128313, 128312, 8717, 8956, 8958, 65131, - 65104, 65109, 65129, 8714, 8948, 8951, 65126, 65111, 65112, 65105, 65115, - 65113, 65117, 65124, 65116, 65114, 65118, 8570, 8567, 8564, 8574, 8572, - 8563, 8560, 8573, 8575, 8571, 8561, 8562, 8569, 8566, 8565, 8568, 65128, - 65108, 68411, 732, 10849, 65125, 65123, 65130, 65122, 65119, 65106, - 65110, 10922, 10924, 10803, 128570, 128571, 128525, 128520, 128519, - 128515, 128517, 128516, 128518, 128522, 129325, 129392, 128526, 129394, - 8995, 128527, 128684, 128012, 128013, 129319, 127956, 9731, 9924, 127938, - 10052, 983077, 9917, 129510, 173, 127846, 128428, 9108, 129358, 69453, - 69452, 69454, 69456, 69447, 69449, 69446, 69448, 69455, 69451, 69450, - 69445, 69424, 69437, 69426, 69433, 69444, 69440, 69429, 69436, 69439, - 69441, 69431, 69427, 69430, 69432, 69425, 69443, 69435, 69442, 69428, - 69438, 69434, 69457, 69460, 69459, 69458, 69463, 69465, 69461, 69462, - 69464, 128618, 128619, 47, 10742, 128284, 69859, 69863, 69864, 69846, - 69847, 69857, 69849, 69842, 69843, 69844, 69845, 69854, 69856, 69855, - 69848, 69851, 69853, 69840, 69841, 69850, 69852, 69858, 69860, 69862, - 69861, 69877, 69876, 69879, 69878, 69875, 69874, 69872, 69881, 69873, - 69880, 8600, 10537, 10541, 8690, 10533, 128603, 128595, 128611, 11112, - 11128, 11018, 8664, 129110, 11010, 8601, 10538, 10534, 128601, 128593, - 128609, 11113, 11129, 11019, 8665, 129111, 11011, 8471, 72328, 72329, - 72327, 72326, 72340, 72339, 72334, 72332, 72341, 72335, 72333, 72330, - 72331, 72338, 72336, 72337, 72344, 72352, 72351, 72350, 72297, 72296, - 72302, 72311, 72301, 72323, 72285, 72284, 72288, 72298, 72293, 72303, - 72319, 72320, 72321, 72310, 72309, 72295, 72294, 72300, 72299, 72307, - 72306, 72290, 72289, 72287, 72286, 72292, 72291, 72305, 72304, 72312, - 72313, 72314, 72322, 72317, 72308, 72316, 72318, 72315, 72272, 72349, - 72348, 72347, 72346, 72324, 72343, 72325, 72342, 72345, 72353, 72354, - 72282, 72281, 72279, 72280, 72277, 72278, 72275, 72274, 72276, 72273, - 72283, 8384, 983043, 983182, 983118, 983177, 127837, 128150, 10055, - 10024, 32, 128586, 128264, 128263, 128265, 128266, 128483, 128676, - 128172, 8375, 8738, 10656, 10657, 128375, 128376, 128467, 128466, 128026, - 128166, 129525, 129348, 128051, 127941, 129533, 13279, 13058, 13057, - 13059, 13056, 13250, 13171, 13278, 13101, 13172, 13108, 13105, 13118, - 13116, 13251, 13192, 8851, 13255, 13183, 13213, 13220, 13216, 13254, - 8852, 13252, 13253, 13170, 13092, 13175, 13177, 13176, 13093, 13094, - 13256, 127376, 13207, 13064, 13179, 13181, 13182, 13055, 13180, 13005, - 13063, 13006, 9974, 9165, 13209, 13070, 13071, 13311, 13075, 13073, + 71089, 71096, 71098, 71107, 71106, 128411, 128410, 128417, 128416, + 128409, 128408, 128415, 128414, 121399, 121397, 121400, 121398, 121402, + 121401, 121104, 121103, 121102, 121388, 121387, 121386, 121482, 121479, + 121358, 121359, 121357, 121360, 121341, 121335, 121339, 121340, 121336, + 121334, 121333, 121338, 121337, 121342, 121368, 121367, 121373, 121356, + 121355, 121354, 121380, 121382, 121381, 121384, 121383, 121385, 121377, + 121379, 121378, 121375, 121376, 121374, 121371, 121369, 121366, 121370, + 121372, 121364, 121365, 121452, 121392, 121352, 121353, 121351, 121499, + 121500, 121501, 121502, 121503, 121470, 121117, 121115, 121119, 121118, + 121116, 121456, 121363, 121362, 121361, 121480, 121455, 120965, 120837, + 121000, 120969, 121027, 121026, 121076, 121074, 121075, 120995, 120991, + 120990, 120982, 121005, 121011, 121044, 121042, 121043, 120950, 120833, + 120847, 121002, 120859, 120863, 120997, 120967, 120845, 121019, 120839, + 120993, 120980, 121038, 121045, 121031, 121009, 120934, 121046, 121047, + 120905, 120936, 120937, 120935, 121065, 120941, 120834, 121068, 121069, + 121024, 120915, 120916, 121040, 120940, 120944, 120946, 120942, 120945, + 120947, 120943, 120949, 121020, 121039, 120948, 121091, 120832, 120838, + 120861, 120843, 120852, 120870, 120844, 120846, 120848, 120865, 120853, + 120856, 120885, 120877, 120878, 120879, 120884, 120857, 120882, 120854, + 120855, 120858, 120883, 121001, 120871, 120850, 120867, 120966, 120972, + 120971, 120849, 120862, 120895, 120894, 120872, 120873, 120893, 120875, + 120874, 120866, 120864, 120996, 120842, 120891, 120890, 121067, 120892, + 120887, 121064, 121066, 121063, 121060, 121061, 121062, 120899, 120881, + 121052, 121059, 121057, 121054, 121055, 121056, 121058, 120851, 120869, + 120868, 120841, 121018, 120974, 120977, 120976, 120975, 120985, 120986, + 120983, 120992, 120988, 120984, 120978, 121032, 121037, 120897, 120896, + 120898, 120889, 120888, 120886, 120880, 121035, 121030, 121033, 121028, + 121036, 120860, 121003, 121012, 121014, 121013, 121007, 121016, 121008, + 121015, 121006, 121077, 121083, 120876, 121084, 121085, 121078, 121090, + 121087, 121079, 121080, 121081, 120840, 121086, 121088, 121089, 120979, + 121082, 121092, 120906, 120922, 120908, 120919, 120920, 120921, 120914, + 120910, 120912, 120900, 120903, 120904, 120902, 120901, 120924, 120909, + 120911, 120913, 120926, 120930, 120931, 120932, 120929, 120933, 120928, + 120925, 120927, 120923, 120957, 120917, 120907, 120836, 121051, 120968, + 120973, 120998, 121072, 121053, 121073, 121070, 121071, 121025, 120970, + 120994, 120989, 120987, 121029, 121041, 120955, 120961, 120956, 120959, + 120999, 121004, 120964, 120960, 120963, 120962, 120958, 120939, 120938, + 121023, 121021, 121022, 121050, 121048, 121049, 121034, 121017, 120951, + 120918, 120981, 120953, 121010, 120954, 120952, 120835, 121451, 121343, + 121349, 121347, 121348, 121346, 121345, 121350, 121344, 121462, 121463, + 121464, 121465, 121466, 121467, 121468, 121469, 121428, 121429, 121427, + 121476, 121473, 121477, 121475, 121474, 121478, 121472, 121471, 121404, + 121405, 121403, 121430, 121409, 121411, 121410, 121406, 121408, 121407, + 121421, 121423, 121422, 121415, 121416, 121417, 121418, 121419, 121420, + 121414, 121413, 121412, 121424, 121425, 121426, 121432, 121431, 121183, + 121184, 121182, 121181, 121187, 121188, 121186, 121185, 121175, 121176, + 121174, 121173, 121179, 121180, 121178, 121177, 121324, 121321, 121323, + 121320, 121322, 121319, 121210, 121209, 121208, 121203, 121207, 121305, + 121272, 121271, 121287, 121286, 121303, 121304, 121302, 121301, 121206, + 121205, 121204, 121198, 121332, 121331, 121129, 121276, 121274, 121275, + 121273, 121288, 121289, 121291, 121290, 121306, 121280, 121278, 121279, + 121277, 121295, 121293, 121294, 121292, 121307, 121314, 121191, 121192, + 121190, 121189, 121193, 121202, 121201, 121200, 121199, 121282, 121281, + 121297, 121296, 121308, 121309, 121310, 121328, 121327, 121194, 121196, + 121197, 121195, 121216, 121215, 121214, 121213, 121212, 121211, 121449, + 121125, 121126, 121121, 121122, 121123, 121124, 121127, 121318, 121316, + 121317, 121315, 121156, 121155, 121154, 121146, 121145, 121144, 121150, + 121149, 121148, 121147, 121230, 121231, 121229, 121228, 121261, 121254, + 121247, 121233, 121232, 121226, 121227, 121225, 121224, 121249, 121248, + 121153, 121152, 121151, 121139, 121135, 121137, 121138, 121136, 121330, + 121329, 121128, 121236, 121262, 121255, 121235, 121234, 121237, 121240, + 121239, 121263, 121256, 121238, 121162, 121161, 121160, 121132, 121133, + 121131, 121130, 121134, 121253, 121142, 121143, 121141, 121140, 121243, + 121242, 121241, 121246, 121245, 121244, 121270, 121269, 121268, 121264, + 121257, 121326, 121325, 121159, 121158, 121157, 121448, 121394, 121393, + 121395, 121396, 121450, 121513, 121514, 121515, 121516, 121517, 121518, + 121519, 121505, 121506, 121507, 121508, 121509, 121510, 121511, 121512, + 121312, 121284, 121299, 121311, 121283, 121298, 121313, 121285, 121300, + 121267, 121260, 121252, 121251, 121266, 121259, 121250, 121265, 121258, + 121107, 121106, 121105, 121454, 121453, 121457, 121120, 121112, 121110, + 121114, 121113, 121111, 121108, 121109, 121101, 121100, 121099, 121481, + 121441, 121445, 121446, 121443, 121444, 121442, 121447, 121390, 121389, + 121391, 121440, 121439, 121437, 121435, 121436, 121434, 121433, 121438, + 121459, 121458, 121460, 121095, 121094, 121093, 121219, 121218, 121217, + 121222, 121221, 121220, 121223, 121172, 121171, 121170, 121168, 121167, + 121166, 121165, 121164, 121163, 121169, 121098, 121097, 121096, 121461, + 121483, 129304, 10912, 10911, 10910, 10909, 10860, 983185, 983183, 8219, + 8249, 8218, 8250, 983158, 983155, 983156, 983159, 70117, 70116, 70119, + 70118, 70115, 70114, 70121, 70113, 70120, 70131, 70132, 70129, 70126, + 70125, 70130, 70128, 70127, 70124, 70123, 70122, 983953, 983954, 983952, + 3464, 3463, 3495, 3501, 3497, 3503, 3510, 3488, 3484, 3490, 3482, 3508, + 3513, 3462, 3475, 3478, 3461, 3474, 3473, 3472, 3471, 3470, 3469, 3466, + 3465, 3468, 3467, 3496, 3502, 3498, 3504, 3511, 3489, 3485, 3491, 3483, + 3509, 3512, 3525, 3499, 3522, 3517, 3505, 3523, 3477, 3476, 3500, 3507, + 3487, 3494, 3526, 3524, 3515, 3520, 3514, 3492, 3493, 3521, 3486, 3563, + 3562, 3565, 3564, 3561, 3560, 3558, 3567, 3559, 3566, 3530, 3458, 3457, + 3459, 3570, 3571, 3546, 3537, 3539, 3542, 3544, 3551, 3536, 3538, 3540, + 3545, 3548, 3549, 3550, 3547, 3535, 3572, 8767, 10046, 128973, 10038, + 128303, 8198, 10042, 128510, 127935, 9975, 128128, 9760, 129448, 128761, + 10902, 10904, 10901, 10903, 11098, 11100, 42611, 128716, 128164, 128564, + 128554, 128373, 128759, 127829, 128578, 128577, 10840, 10839, 9011, + 127920, 129445, 65120, 128745, 65121, 128313, 128312, 8717, 8956, 8958, + 65131, 65104, 65109, 65129, 8714, 8948, 8951, 65126, 65111, 65112, 65105, + 65115, 65113, 65117, 65124, 65116, 65114, 65118, 8570, 8567, 8564, 8574, + 8572, 8563, 8560, 8573, 8575, 8571, 8561, 8562, 8569, 8566, 8565, 8568, + 65128, 65108, 68411, 732, 118266, 10849, 65125, 65123, 65130, 65122, + 65119, 65106, 65110, 10922, 10924, 10803, 128571, 128570, 128525, 128520, + 128519, 128515, 128517, 128516, 128518, 128522, 129325, 129392, 128526, + 129394, 8995, 128527, 128684, 128012, 128013, 129319, 127956, 9731, 9924, + 127938, 10052, 983077, 9917, 129510, 173, 127846, 128428, 9108, 129358, + 69453, 69452, 69454, 69456, 69447, 69449, 69446, 69448, 69455, 69451, + 69450, 69445, 69424, 69437, 69426, 69433, 69444, 69440, 69429, 69436, + 69439, 69441, 69431, 69427, 69430, 69432, 69425, 69443, 69435, 69442, + 69428, 69438, 69434, 69457, 69460, 69459, 69458, 69463, 69465, 69461, + 69462, 69464, 128618, 128619, 47, 10742, 128284, 69859, 69863, 69864, + 69846, 69847, 69857, 69849, 69842, 69843, 69844, 69845, 69854, 69856, + 69855, 69848, 69851, 69853, 69840, 69841, 69850, 69852, 69858, 69860, + 69862, 69861, 69877, 69876, 69879, 69878, 69875, 69874, 69872, 69881, + 69873, 69880, 8600, 10537, 10541, 8690, 10533, 129210, 128603, 128595, + 128611, 11112, 11128, 11018, 8664, 129110, 11010, 8601, 10538, 10534, + 129211, 128601, 128593, 128609, 11113, 11129, 11019, 8665, 129111, 11011, + 8471, 72328, 72329, 72327, 72326, 72340, 72339, 72334, 72332, 72341, + 72335, 72333, 72330, 72331, 72338, 72336, 72337, 72352, 72351, 72350, + 72297, 72296, 72302, 72311, 72301, 72323, 72285, 72284, 72288, 72298, + 72293, 72303, 72319, 72320, 72321, 72310, 72309, 72295, 72294, 72300, + 72299, 72307, 72306, 72290, 72289, 72287, 72286, 72292, 72291, 72305, + 72304, 72312, 72313, 72314, 72322, 72317, 72308, 72316, 72318, 72315, + 72272, 72349, 72348, 72347, 72346, 72324, 72343, 72325, 72342, 72345, + 72353, 72354, 72282, 72281, 72279, 72280, 72277, 72278, 72275, 72274, + 72276, 72273, 72283, 72344, 8384, 983043, 983182, 983118, 983177, 127837, + 128150, 10055, 10024, 117824, 117825, 32, 128586, 128264, 128263, 128265, + 128266, 128483, 128676, 128172, 8375, 117830, 117831, 8738, 10656, 10657, + 128375, 128376, 128467, 128466, 128026, 128166, 129759, 129525, 129348, + 128051, 127941, 129533, 13279, 117900, 13056, 13058, 13057, 13059, 13250, + 13171, 13278, 13101, 13172, 13108, 13105, 13118, 13116, 13251, 13192, + 8851, 13255, 13183, 13213, 13220, 13216, 13254, 8852, 13252, 13253, + 13170, 13092, 13175, 13177, 13176, 13093, 13094, 13256, 127376, 13207, + 13064, 13179, 13181, 13182, 13055, 13180, 13005, 13063, 13006, 117899, + 117898, 117897, 9974, 9165, 13209, 13070, 13071, 13311, 13075, 13073, 13072, 13080, 13081, 13203, 13228, 13191, 13257, 13258, 13098, 13110, 13113, 13121, 13122, 13119, 13107, 13106, 13109, 13259, 13169, 127488, - 13004, 13200, 13260, 13060, 13061, 8847, 8932, 8849, 13178, 13188, 13067, - 13069, 13068, 13074, 13076, 13078, 13079, 13077, 13214, 13262, 13222, + 13004, 13200, 13260, 13060, 13061, 8847, 8932, 8849, 13178, 13188, 13069, + 13068, 13067, 13074, 13076, 13078, 13079, 13077, 13214, 13262, 13222, 13218, 13086, 13085, 13082, 13083, 13084, 13201, 13193, 13248, 13226, 13189, 13199, 13261, 13208, 13263, 13240, 13246, 8977, 13266, 10957, 13264, 13265, 13267, 13223, 13224, 13249, 13221, 13217, 13187, 13123, @@ -11817,28 +12606,29 @@ static const unsigned int dawg_pos_to_codepoint[] = { 13184, 13115, 13112, 13114, 13111, 13103, 13104, 13102, 13117, 13120, 11216, 13273, 13174, 13194, 13271, 13272, 13274, 13232, 13236, 13242, 13229, 13230, 13231, 13142, 13141, 10958, 13137, 13138, 13140, 13139, - 8730, 13087, 13088, 13090, 13091, 13089, 13275, 13276, 128918, 13204, - 13095, 13143, 11027, 9641, 9638, 9636, 11029, 9706, 9703, 11026, 9705, - 9639, 11028, 9640, 9637, 9704, 10720, 13277, 13135, 13134, 13136, 13062, - 127529, 127530, 127512, 127508, 127533, 127545, 127520, 127516, 127534, - 127506, 127540, 127525, 127546, 127532, 127511, 127509, 127524, 127505, - 127517, 127518, 127527, 127537, 127504, 127528, 127535, 127519, 127515, - 127513, 127543, 127542, 127526, 127541, 127544, 127522, 127538, 127514, - 127521, 127539, 127510, 127536, 127523, 127547, 127531, 127378, 127377, - 8865, 10693, 11820, 127390, 127392, 127379, 10692, 127400, 127399, - 127398, 127306, 127489, 127507, 127490, 9919, 127397, 127280, 127281, - 127282, 127283, 127284, 127285, 127286, 127287, 127288, 127289, 127290, - 127291, 127292, 127293, 127294, 127295, 127296, 127297, 127298, 127299, - 127300, 127301, 127302, 127303, 127304, 127305, 10190, 10191, 127401, - 8863, 127307, 127381, 127382, 127396, 127383, 127310, 8862, 127388, - 127393, 127402, 127395, 9949, 10695, 10696, 127384, 127308, 127309, - 127387, 127394, 127389, 8864, 127391, 127385, 127403, 127404, 127386, - 10694, 127311, 127380, 10151, 129425, 983157, 983160, 983134, 983190, - 9877, 9882, 128387, 129485, 8795, 10017, 8902, 9770, 11242, 11243, - 983175, 983133, 983181, 983176, 983042, 983044, 128509, 128649, 127967, - 128642, 127836, 11836, 129658, 129989, 129990, 129993, 129991, 129992, - 128480, 9201, 128207, 9188, 127827, 10025, 983189, 8803, 127897, 129369, - 128723, 983170, 983045, 983103, 8332, 8328, 8330, 8331, 8333, 8334, 8325, + 8730, 13087, 13088, 13090, 13091, 117887, 117886, 117884, 117885, 13089, + 13275, 13276, 128918, 13204, 13095, 13143, 11027, 9641, 9638, 9636, + 11029, 9706, 9703, 11026, 9705, 9639, 11028, 9640, 9637, 9704, 10720, + 13277, 13135, 13134, 13136, 13062, 127529, 127530, 127512, 127508, + 127533, 127545, 127520, 127516, 127534, 127506, 127540, 127525, 127546, + 127532, 127511, 127509, 127524, 127505, 127517, 127518, 127527, 127537, + 127504, 127528, 127535, 127519, 127515, 127513, 127543, 127542, 127526, + 127541, 127544, 127522, 127538, 127514, 127521, 127539, 127510, 127536, + 127523, 127547, 127531, 127378, 127377, 8865, 10693, 11820, 127390, + 127392, 127379, 10692, 127400, 127399, 127398, 127306, 127489, 127507, + 127490, 9919, 127397, 127280, 127281, 127282, 127283, 127284, 127285, + 127286, 127287, 127288, 127289, 127290, 127291, 127292, 127293, 127294, + 127295, 127296, 127297, 127298, 127299, 127300, 127301, 127302, 127303, + 127304, 127305, 10190, 10191, 127401, 8863, 127307, 127381, 127382, + 127396, 127383, 127310, 8862, 127388, 127393, 127402, 127395, 9949, + 10695, 10696, 127384, 127308, 127309, 127387, 127394, 127389, 8864, + 127391, 127385, 127403, 127404, 127386, 10694, 127311, 127380, 10151, + 129425, 983157, 983160, 983134, 983190, 9877, 9882, 128387, 8795, 10017, + 8902, 9770, 11242, 11243, 983175, 983133, 983181, 983176, 983042, 983044, + 128509, 128649, 129485, 127967, 128642, 127836, 11836, 129658, 129989, + 129990, 129993, 129991, 129992, 128480, 9201, 128207, 9188, 127827, + 10025, 983189, 117891, 117888, 117890, 117889, 8803, 127897, 129369, + 128723, 983170, 983045, 983103, 8333, 8332, 8328, 8330, 8334, 8331, 8325, 8324, 8327, 8326, 8323, 8322, 8320, 8329, 8321, 10963, 10965, 10617, 8834, 10953, 10949, 10951, 10955, 8842, 8838, 10947, 10943, 10945, 10941, 983102, 8827, 10938, 10934, 10928, 10930, 10936, 10932, 8937, 8831, 8829, @@ -11849,406 +12639,433 @@ static const unsigned int dawg_pos_to_codepoint[] = { 7056, 7044, 7047, 7045, 7367, 7366, 7365, 7364, 7363, 7361, 7362, 7360, 7082, 7041, 7042, 7040, 7083, 7080, 7081, 7079, 7077, 7076, 7078, 7093, 7092, 7095, 7094, 7091, 7090, 7088, 7097, 7089, 7096, 127749, 127748, - 127751, 127803, 8316, 8312, 8305, 8319, 8317, 8314, 8315, 8318, 8309, - 8308, 8311, 8310, 179, 178, 8304, 8313, 185, 10966, 10964, 10619, 10968, - 10967, 8835, 10954, 10950, 10952, 10956, 8843, 8839, 10948, 10944, 10946, - 10942, 10185, 129464, 129465, 8751, 127940, 128671, 127843, 128629, - 129442, 127946, 8275, 43027, 43026, 43031, 43030, 43040, 43038, 43025, - 43024, 43029, 43028, 43036, 43035, 43021, 43020, 43018, 43017, 43023, - 43022, 43016, 43015, 43034, 43033, 43042, 43039, 43037, 43032, 43041, - 43008, 43012, 43009, 43013, 43011, 43048, 43049, 43050, 43051, 43052, - 43019, 43014, 43010, 43047, 43043, 43046, 43044, 43045, 9223, 9224, 9229, - 9240, 9232, 9236, 9235, 9234, 9233, 9249, 9253, 9241, 9220, 9239, 9219, - 9221, 9243, 9244, 9228, 9225, 9227, 9226, 128325, 9237, 9252, 9216, 9222, - 9246, 9245, 9247, 8527, 9230, 9231, 9217, 9218, 9242, 9254, 9238, 9248, - 11159, 9007, 983094, 983093, 128333, 1807, 1866, 1802, 1798, 1799, 1849, - 1848, 1792, 1854, 1853, 1805, 1804, 1803, 1852, 1851, 1850, 1797, 1814, - 1813, 1815, 1818, 1824, 2153, 2152, 2149, 2148, 2144, 2146, 2150, 2147, - 2154, 2145, 2151, 1825, 1830, 1837, 1839, 1838, 1831, 1834, 1827, 1869, - 1870, 1871, 1809, 1835, 1832, 1828, 1808, 1823, 1833, 1819, 1820, 1836, - 1811, 1812, 1821, 1822, 1810, 1817, 1826, 1816, 1829, 1864, 1863, 1842, - 1841, 1840, 1845, 1844, 1843, 1847, 1846, 1855, 1858, 1796, 983204, 1801, - 1794, 1795, 1800, 1793, 1862, 1861, 1860, 1859, 1865, 1856, 1857, 128137, - 983184, 128085, 129430, 983061, 127955, 917543, 917542, 917546, 917598, - 917568, 917548, 917562, 917540, 917557, 917556, 917559, 917558, 917555, - 917554, 917552, 917561, 917553, 917560, 917565, 917537, 917600, 917566, - 917549, 917569, 917570, 917571, 917572, 917573, 917574, 917575, 917576, - 917577, 917578, 917579, 917580, 917581, 917582, 917583, 917584, 917585, - 917586, 917587, 917588, 917589, 917590, 917591, 917592, 917593, 917594, - 917601, 917602, 917603, 917604, 917605, 917606, 917607, 917608, 917609, - 917610, 917611, 917612, 917613, 917614, 917615, 917616, 917617, 917618, - 917619, 917620, 917621, 917622, 917623, 917624, 917625, 917626, 917564, - 917627, 917544, 917595, 917599, 917541, 917547, 917538, 917567, 917629, - 917545, 917597, 917596, 917563, 917551, 917536, 917539, 917630, 917550, - 917628, 5888, 5919, 5893, 5896, 5898, 5895, 5892, 5905, 5891, 5902, 5899, - 5897, 5901, 5904, 5894, 5903, 5900, 5889, 5890, 5909, 5908, 5906, 5907, - 5989, 5992, 5994, 5991, 5988, 5987, 5998, 5995, 5993, 6000, 5990, 5999, - 5996, 5984, 5985, 5986, 6002, 6003, 6499, 6508, 6509, 6507, 6501, 6502, - 6512, 6513, 6514, 6515, 6516, 6497, 6483, 6487, 6486, 6482, 6498, 6505, - 6504, 6496, 6480, 6490, 6489, 6503, 6506, 6492, 6494, 6488, 6491, 6495, - 6484, 6493, 6481, 6485, 6500, 6745, 6746, 6743, 6747, 6742, 6741, 6748, - 6749, 6750, 6783, 6789, 6788, 6791, 6790, 6787, 6786, 6784, 6793, 6785, - 6792, 6805, 6804, 6807, 6806, 6803, 6802, 6800, 6809, 6801, 6808, 6740, - 6689, 6690, 6688, 6702, 6726, 6727, 6728, 6696, 6695, 6713, 6712, 6707, - 6706, 6714, 6729, 6720, 6693, 6692, 6691, 6704, 6699, 6697, 6717, 6715, - 6709, 6708, 6716, 6732, 6698, 6719, 6723, 6739, 6724, 6730, 6721, 6705, - 6701, 6722, 6735, 6736, 6733, 6734, 6694, 6700, 6710, 6738, 6737, 6711, - 6703, 6718, 6725, 6731, 6828, 6820, 6775, 6776, 6777, 6780, 6824, 6825, - 6819, 6772, 6744, 6823, 6779, 6778, 6822, 6826, 6827, 6818, 6752, 6816, - 6817, 6773, 6774, 6821, 6829, 6753, 6755, 6767, 6769, 6754, 6763, 6764, - 6771, 6768, 6765, 6756, 6770, 6761, 6762, 6760, 6759, 6757, 6758, 6766, - 43653, 43651, 43649, 43661, 43659, 43679, 43677, 43671, 43669, 43657, - 43665, 43673, 43675, 43667, 43681, 43655, 43693, 43689, 43683, 43687, - 43663, 43691, 43685, 43695, 43652, 43650, 43648, 43660, 43658, 43678, - 43676, 43670, 43668, 43656, 43664, 43672, 43674, 43666, 43680, 43654, - 43692, 43688, 43682, 43686, 43662, 43690, 43684, 43694, 43696, 43703, - 43743, 43739, 43740, 43742, 43741, 43712, 43713, 43714, 43711, 43707, - 43697, 43710, 43709, 43708, 43700, 43699, 43705, 43706, 43698, 43704, - 43701, 43702, 71353, 71296, 71352, 71297, 71303, 71305, 71338, 71332, - 71319, 71318, 71324, 71323, 71317, 71316, 71322, 71321, 71300, 71301, - 71298, 71299, 71310, 71320, 71315, 71325, 71329, 71328, 71312, 71311, - 71309, 71308, 71314, 71313, 71307, 71306, 71327, 71326, 71335, 71336, - 71337, 71333, 71330, 71334, 71331, 71302, 71304, 71351, 71339, 71350, - 71340, 71341, 71347, 71349, 71344, 71345, 71342, 71343, 71346, 71348, - 71365, 71364, 71367, 71366, 71363, 71362, 71360, 71369, 71361, 71368, - 129377, 119672, 119671, 3064, 3031, 73707, 983662, 983685, 983674, - 983677, 983676, 983669, 983667, 983679, 983663, 983665, 983668, 983666, - 983683, 983681, 983682, 983673, 983678, 983664, 983684, 983680, 983671, - 983670, 983675, 983672, 73706, 3063, 73701, 3062, 3059, 3051, 3050, 3053, - 3052, 3049, 3048, 3046, 3055, 3047, 3054, 73700, 73666, 73676, 73668, - 73679, 73681, 73682, 73665, 73673, 73674, 73667, 73664, 73669, 73672, - 73675, 73680, 73670, 73678, 73671, 73677, 73683, 73684, 73710, 2985, - 2979, 2969, 2974, 2984, 2975, 2980, 2996, 2995, 2994, 2993, 2992, 2949, - 2950, 2960, 2964, 2953, 2954, 2962, 2963, 2951, 2952, 2998, 2999, 3000, - 2958, 2959, 2970, 3001, 2972, 2965, 2990, 2986, 2997, 2991, 73702, 3057, - 3058, 3056, 3066, 73727, 3065, 73703, 73687, 2946, 73686, 73698, 73690, - 73693, 73692, 73712, 73689, 73688, 73691, 73697, 73694, 73695, 73696, - 73713, 73699, 3021, 2947, 73685, 73708, 73711, 983939, 983940, 983947, - 983950, 983943, 983944, 983948, 983949, 983941, 983942, 983945, 983946, - 983686, 983693, 983696, 983689, 983690, 983694, 983695, 983687, 983688, - 983691, 983692, 983840, 983847, 983850, 983843, 983844, 983848, 983849, - 983841, 983842, 983845, 983846, 983851, 983858, 983861, 983854, 983855, - 983859, 983860, 983852, 983853, 983856, 983857, 983818, 983825, 983828, - 983821, 983822, 983826, 983827, 983819, 983820, 983823, 983824, 983873, - 983880, 983883, 983876, 983877, 983881, 983882, 983874, 983875, 983878, - 983879, 983741, 983748, 983751, 983744, 983745, 983749, 983750, 983742, - 983743, 983746, 983747, 983697, 983704, 983707, 983700, 983701, 983705, - 983706, 983698, 983699, 983702, 983703, 983719, 983726, 983729, 983722, - 983723, 983727, 983728, 983720, 983721, 983724, 983725, 983763, 983770, - 983773, 983766, 983767, 983771, 983772, 983764, 983765, 983768, 983769, - 983862, 983869, 983872, 983865, 983866, 983870, 983871, 983863, 983864, - 983867, 983868, 983807, 983814, 983817, 983810, 983811, 983815, 983816, - 983808, 983809, 983812, 983813, 983895, 983902, 983905, 983898, 983899, - 983903, 983904, 983951, 983896, 983897, 983900, 983901, 983906, 983913, - 983916, 983909, 983910, 983914, 983915, 983907, 983908, 983911, 983912, - 983917, 983924, 983927, 983920, 983921, 983925, 983926, 983918, 983919, - 983922, 983923, 983730, 983737, 983740, 983733, 983734, 983738, 983739, - 983731, 983732, 983735, 983736, 983752, 983759, 983762, 983755, 983756, - 983760, 983761, 983753, 983754, 983757, 983758, 983708, 983715, 983718, - 983711, 983712, 983716, 983717, 983709, 983710, 983713, 983714, 983928, - 983935, 983938, 983931, 983932, 983936, 983937, 983929, 983930, 983933, - 983934, 983884, 983891, 983894, 983887, 983888, 983892, 983893, 983885, - 983886, 983889, 983890, 983785, 983792, 983795, 983788, 983789, 983793, - 983794, 983786, 983787, 983790, 983791, 983774, 983781, 983784, 983777, - 983778, 983782, 983783, 983775, 983776, 983779, 983780, 983829, 983836, - 983839, 983832, 983833, 983837, 983838, 983830, 983831, 983834, 983835, - 983796, 983803, 983806, 983799, 983800, 983804, 983805, 983797, 983798, - 983801, 983802, 3006, 3016, 3020, 3009, 3010, 3018, 3019, 3007, 3008, - 3014, 3015, 73709, 73704, 73705, 3061, 3060, 3024, 129748, 127883, - 127818, 92809, 92810, 92811, 92808, 92789, 92790, 92791, 92788, 92816, - 92859, 92856, 92847, 92845, 92817, 92846, 92843, 92829, 92830, 92831, - 92828, 92835, 92851, 92840, 92844, 92818, 92819, 92852, 92836, 92825, - 92826, 92827, 92824, 92813, 92814, 92815, 92812, 92820, 92822, 92823, - 92821, 92805, 92806, 92807, 92804, 92797, 92798, 92799, 92796, 92801, - 92802, 92803, 92800, 92785, 92786, 92787, 92784, 92793, 92794, 92795, - 92792, 92857, 92854, 92848, 92861, 92853, 92860, 92849, 92855, 92834, - 92833, 92832, 92841, 92839, 92842, 92850, 92838, 92858, 92837, 92862, - 92869, 92868, 92871, 92870, 92867, 92866, 92864, 92873, 92865, 92872, - 100352, 100353, 100354, 100355, 100356, 100357, 100358, 100359, 100360, - 100361, 100362, 100363, 100364, 100365, 100366, 100367, 100368, 100369, - 100370, 100371, 100372, 100373, 100374, 100375, 100376, 100377, 100378, - 100379, 100380, 100381, 100382, 100383, 100384, 100385, 100386, 100387, - 100388, 100389, 100390, 100391, 100392, 100393, 100394, 100395, 100396, - 100397, 100398, 100399, 100400, 100401, 100402, 100403, 100404, 100405, - 100406, 100407, 100408, 100409, 100410, 100411, 100412, 100413, 100414, - 100415, 100416, 100417, 100418, 100419, 100420, 100421, 100422, 100423, - 100424, 100425, 100426, 100427, 100428, 100429, 100430, 100431, 100432, - 100433, 100434, 100435, 100436, 100437, 100438, 100439, 100440, 100441, - 100442, 100443, 100444, 100445, 100446, 100447, 100448, 100449, 100450, - 100451, 100452, 100453, 100454, 100455, 100456, 100457, 100458, 100459, - 100460, 100461, 100462, 100463, 100464, 100465, 100466, 100467, 100468, - 100469, 100470, 100471, 100472, 100473, 100474, 100475, 100476, 100477, - 100478, 100479, 100480, 100481, 100482, 100483, 100484, 100485, 100486, - 100487, 100488, 100489, 100490, 100491, 100492, 100493, 100494, 100495, - 100496, 100497, 100498, 100499, 100500, 100501, 100502, 100503, 100504, - 100505, 100506, 100507, 100508, 100509, 100510, 100511, 100512, 100513, - 100514, 100515, 100516, 100517, 100518, 100519, 100520, 100521, 100522, - 100523, 100524, 100525, 100526, 100527, 100528, 100529, 100530, 100531, - 100532, 100533, 100534, 100535, 100536, 100537, 100538, 100539, 100540, - 100541, 100542, 100543, 100544, 100545, 100546, 100547, 100548, 100549, - 100550, 100551, 100552, 100553, 100554, 100555, 100556, 100557, 100558, - 100559, 100560, 100561, 100562, 100563, 100564, 100565, 100566, 100567, - 100568, 100569, 100570, 100571, 100572, 100573, 100574, 100575, 100576, - 100577, 100578, 100579, 100580, 100581, 100582, 100583, 100584, 100585, - 100586, 100587, 100588, 100589, 100590, 100591, 100592, 100593, 100594, - 100595, 100596, 100597, 100598, 100599, 100600, 100601, 100602, 100603, - 100604, 100605, 100606, 100607, 100608, 100609, 100610, 100611, 100612, - 100613, 100614, 100615, 100616, 100617, 100618, 100619, 100620, 100621, - 100622, 100623, 100624, 100625, 100626, 100627, 100628, 100629, 100630, - 100631, 100632, 100633, 100634, 100635, 100636, 100637, 100638, 100639, - 100640, 100641, 100642, 100643, 100644, 100645, 100646, 100647, 100648, - 100649, 100650, 100651, 100652, 100653, 100654, 100655, 100656, 100657, - 100658, 100659, 100660, 100661, 100662, 100663, 100664, 100665, 100666, - 100667, 100668, 100669, 100670, 100671, 100672, 100673, 100674, 100675, - 100676, 100677, 100678, 100679, 100680, 100681, 100682, 100683, 100684, - 100685, 100686, 100687, 100688, 100689, 100690, 100691, 100692, 100693, - 100694, 100695, 100696, 100697, 100698, 100699, 100700, 100701, 100702, - 100703, 100704, 100705, 100706, 100707, 100708, 100709, 100710, 100711, - 100712, 100713, 100714, 100715, 100716, 100717, 100718, 100719, 100720, - 100721, 100722, 100723, 100724, 100725, 100726, 100727, 100728, 100729, - 100730, 100731, 100732, 100733, 100734, 100735, 100736, 100737, 100738, - 100739, 100740, 100741, 100742, 100743, 100744, 100745, 100746, 100747, - 100748, 100749, 100750, 100751, 100752, 100753, 100754, 100755, 100756, - 100757, 100758, 100759, 100760, 100761, 100762, 100763, 100764, 100765, - 100766, 100767, 100768, 100769, 100770, 100771, 100772, 100773, 100774, - 100775, 100776, 100777, 100778, 100779, 100780, 100781, 100782, 100783, - 100784, 100785, 100786, 100787, 100788, 100789, 100790, 100791, 100792, - 100793, 100794, 100795, 100796, 100797, 100798, 100799, 100800, 100801, - 100802, 100803, 100804, 100805, 100806, 100807, 100808, 100809, 100810, - 100811, 100812, 100813, 100814, 100815, 100816, 100817, 100818, 100819, - 100820, 100821, 100822, 100823, 100824, 100825, 100826, 100827, 100828, - 100829, 100830, 100831, 100832, 100833, 100834, 100835, 100836, 100837, - 100838, 100839, 100840, 100841, 100842, 100843, 100844, 100845, 100846, - 100847, 100848, 100849, 100850, 100851, 100852, 100853, 100854, 100855, - 100856, 100857, 100858, 100859, 100860, 100861, 100862, 100863, 100864, - 100865, 100866, 100867, 100868, 100869, 100870, 100871, 100872, 100873, - 100874, 100875, 100876, 100877, 100878, 100879, 100880, 100881, 100882, - 100883, 100884, 100885, 100886, 100887, 100888, 100889, 100890, 100891, - 100892, 100893, 100894, 100895, 100896, 100897, 100898, 100899, 100900, - 100901, 100902, 100903, 100904, 100905, 100906, 100907, 100908, 100909, - 100910, 100911, 100912, 100913, 100914, 100915, 100916, 100917, 100918, - 100919, 100920, 100921, 100922, 100923, 100924, 100925, 100926, 100927, - 100928, 100929, 100930, 100931, 100932, 100933, 100934, 100935, 100936, - 100937, 100938, 100939, 100940, 100941, 100942, 100943, 100944, 100945, - 100946, 100947, 100948, 100949, 100950, 100951, 100952, 100953, 100954, - 100955, 100956, 100957, 100958, 100959, 100960, 100961, 100962, 100963, - 100964, 100965, 100966, 100967, 100968, 100969, 100970, 100971, 100972, - 100973, 100974, 100975, 100976, 100977, 100978, 100979, 100980, 100981, - 100982, 100983, 100984, 100985, 100986, 100987, 100988, 100989, 100990, - 100991, 100992, 100993, 100994, 100995, 100996, 100997, 100998, 100999, - 101000, 101001, 101002, 101003, 101004, 101005, 101006, 101007, 101008, - 101009, 101010, 101011, 101012, 101013, 101014, 101015, 101016, 101017, - 101018, 101019, 101020, 101021, 101022, 101023, 101024, 101025, 101026, - 101027, 101028, 101029, 101030, 101031, 101032, 101033, 101034, 101035, - 101036, 101037, 101038, 101039, 101040, 101041, 101042, 101043, 101044, - 101045, 101046, 101047, 101048, 101049, 101050, 101051, 101052, 101053, - 101054, 101055, 101056, 101057, 101058, 101059, 101060, 101061, 101062, - 101063, 101064, 101065, 101066, 101067, 101068, 101069, 101070, 101071, - 101072, 101073, 101074, 101075, 101076, 101077, 101078, 101079, 101080, - 101081, 101082, 101083, 101084, 101085, 101086, 101087, 101088, 101089, - 101090, 101091, 101092, 101093, 101094, 101095, 101096, 101097, 101098, - 101099, 101100, 101101, 101102, 101103, 101104, 101105, 101106, 101107, - 101108, 101109, 101110, 101111, 101112, 101113, 101114, 101115, 101116, - 101117, 101118, 101119, 94176, 128429, 9991, 9801, 127790, 128661, - 127861, 128198, 10043, 10170, 129750, 129528, 128222, 128380, 8981, 9990, - 8481, 128384, 128301, 128250, 3158, 3195, 3198, 3194, 3197, 3193, 3196, - 3192, 3106, 3105, 3111, 3161, 3110, 3124, 3123, 3122, 3112, 3165, 3097, - 3107, 3102, 3162, 3121, 3120, 3104, 3103, 3109, 3160, 3108, 3077, 3078, - 3088, 3092, 3083, 3168, 3084, 3169, 3125, 3081, 3082, 3090, 3091, 3079, - 3080, 3126, 3127, 3128, 3117, 3116, 3099, 3098, 3096, 3095, 3101, 3100, - 3094, 3093, 3115, 3114, 3086, 3087, 3129, 3118, 3119, 3157, 3076, 3072, - 3073, 3191, 3199, 3133, 3074, 3132, 3149, 3075, 3134, 3144, 3148, 3139, - 3140, 3170, 3171, 3137, 3138, 3146, 3147, 3135, 3136, 3142, 3143, 3179, - 3178, 3181, 3180, 3177, 3176, 3174, 3183, 3175, 3182, 127934, 8376, 9978, - 129514, 119617, 119564, 119577, 119633, 119587, 119566, 119561, 119585, - 119613, 119590, 119631, 119634, 119630, 119608, 119582, 119563, 119573, - 119558, 119624, 119567, 119623, 119586, 119636, 119612, 119625, 119568, - 119584, 119619, 119618, 119583, 119603, 119600, 119626, 119610, 119580, - 119576, 119638, 119559, 119595, 119632, 119606, 119592, 119615, 119599, - 119602, 119614, 119629, 119574, 119569, 119570, 119622, 119562, 119591, - 119637, 119597, 119589, 119616, 119609, 119560, 119635, 119565, 119604, - 119588, 119571, 119594, 119578, 119596, 119579, 119598, 119572, 119605, - 119627, 119621, 119628, 119601, 119593, 119607, 119575, 119620, 119611, - 119581, 1959, 1958, 1964, 1961, 1965, 1927, 1954, 1951, 1937, 1931, 1930, - 1956, 1934, 1935, 1955, 1945, 1920, 1926, 1946, 1933, 1925, 1929, 1943, - 1942, 1941, 1922, 1969, 1950, 1949, 1921, 1936, 1939, 1932, 1947, 1944, - 1952, 1928, 1957, 1938, 1948, 1953, 1924, 1923, 1940, 1967, 1966, 1963, - 1960, 1962, 1968, 3610, 3592, 3594, 3593, 3596, 3598, 3604, 3613, 3615, - 3663, 3630, 3627, 3588, 3587, 3589, 3586, 3590, 3675, 3585, 3628, 3621, - 3653, 3622, 3659, 3633, 3657, 3658, 3656, 3655, 3654, 3617, 3674, 3591, - 3661, 3603, 3609, 3631, 3642, 3612, 3614, 3616, 3611, 3619, 3620, 3632, - 3652, 3651, 3634, 3649, 3635, 3640, 3638, 3639, 3641, 3636, 3637, 3648, - 3650, 3625, 3624, 3626, 3595, 3660, 3601, 3602, 3607, 3600, 3608, 3606, - 3605, 3599, 3623, 3662, 3597, 3618, 3629, 3647, 3669, 3668, 3671, 3670, - 3667, 3666, 3664, 3673, 3665, 3672, 8708, 8707, 8756, 127777, 10727, - 128936, 8201, 128929, 129300, 129353, 128173, 129652, 10176, 8278, 9887, - 9886, 11057, 128485, 128484, 128486, 128487, 8694, 128433, 10870, 128491, - 128962, 128423, 11160, 11162, 10146, 11163, 11161, 10147, 8196, 11835, - 128077, 128078, 9928, 9736, 3865, 3863, 3864, 4032, 4033, 4035, 4034, - 3886, 3885, 3888, 3887, 3884, 3883, 3891, 3890, 3882, 3889, 3877, 3876, - 3879, 3878, 3875, 3874, 3872, 3881, 3873, 3880, 4030, 4031, 3945, 3905, - 3947, 3904, 3948, 3938, 3946, 3917, 3916, 3932, 3931, 3922, 3921, 3908, - 3918, 3913, 3923, 3940, 3941, 3942, 3930, 3929, 3915, 3914, 3920, 3919, - 3927, 3926, 3910, 3909, 3907, 3906, 3925, 3924, 3934, 3935, 3936, 3943, - 3911, 3939, 3928, 3933, 3937, 3944, 3862, 3861, 983210, 3850, 4048, 3849, - 3892, 4049, 3894, 3859, 3846, 3896, 4052, 3845, 3847, 3842, 3843, 3841, - 3860, 3898, 3899, 3900, 3901, 3972, 4051, 3844, 3851, 3895, 3893, 4050, - 3856, 3854, 3973, 3858, 3857, 3848, 3853, 4058, 3897, 3855, 4057, 3852, - 3978, 3979, 3974, 3976, 3903, 3977, 3970, 3866, 3867, 3868, 3871, 3869, - 3870, 4047, 4046, 3966, 3967, 3975, 3902, 3980, 3971, 4028, 4026, 4027, - 3997, 3996, 4012, 4011, 4002, 4001, 4025, 3985, 3984, 3988, 3998, 3993, - 4003, 4020, 4021, 4022, 4010, 4009, 3995, 3994, 4000, 3999, 4007, 4006, - 3990, 3989, 3987, 3986, 4005, 4004, 4014, 4015, 4016, 4023, 3991, 4019, - 4008, 4018, 4013, 4017, 4024, 3983, 3982, 3981, 3840, 4036, 4041, 4044, - 4043, 4042, 4038, 4040, 4037, 4039, 3968, 3969, 3958, 3959, 3960, 3961, - 3956, 3957, 3964, 3965, 3954, 3955, 3962, 3963, 3953, 10717, 11647, - 11608, 11595, 11585, 11573, 11620, 11607, 11600, 11582, 11590, 11596, - 11601, 11586, 11592, 11568, 11575, 11577, 11578, 11576, 11571, 11606, - 11572, 11581, 11589, 11583, 11569, 11570, 11584, 11587, 11609, 11611, - 11610, 11612, 11613, 11615, 11619, 11594, 11621, 11604, 11605, 11614, - 11588, 11580, 11574, 11597, 11598, 11599, 11602, 11591, 11616, 11617, - 11618, 11622, 11579, 11593, 11623, 11603, 11631, 11632, 128005, 128047, - 10053, 126, 8764, 11081, 10610, 10859, 10858, 11807, 11806, 11803, 9202, - 10708, 10709, 10750, 68410, 70854, 70784, 70785, 70786, 70796, 70798, - 70812, 70811, 70817, 70816, 70810, 70809, 70815, 70814, 70791, 70792, - 70793, 70794, 70827, 70789, 70790, 70787, 70788, 70803, 70813, 70808, - 70818, 70828, 70829, 70830, 70822, 70821, 70805, 70804, 70802, 70801, - 70807, 70806, 70800, 70799, 70820, 70819, 70831, 70826, 70823, 70825, - 70824, 70795, 70797, 70852, 70848, 70847, 70851, 70850, 70849, 70832, - 70843, 70846, 70842, 70845, 70837, 70838, 70839, 70840, 70835, 70836, - 70833, 70834, 70841, 70844, 70869, 70868, 70871, 70870, 70867, 70866, - 70864, 70873, 70865, 70872, 70853, 70855, 11858, 8266, 128555, 127915, - 127813, 128069, 129463, 129701, 129520, 10554, 10557, 10556, 9182, 11210, - 11865, 11866, 11833, 8992, 127913, 9180, 8988, 8975, 11810, 8989, 8974, - 11811, 9140, 9184, 128285, 127553, 127554, 127559, 127555, 127557, - 127560, 127552, 127556, 127558, 127274, 9008, 123554, 123556, 123559, - 123561, 123564, 123537, 123553, 123555, 123544, 123543, 123558, 123560, - 123546, 123565, 123563, 123539, 123541, 123550, 123549, 123540, 123552, - 123542, 123536, 123551, 123545, 123538, 123548, 123547, 123562, 123557, - 123566, 128701, 128508, 128434, 128668, 8482, 128650, 128651, 11223, - 10971, 128646, 10701, 10699, 128710, 10698, 10141, 128681, 128208, 8227, - 128305, 9783, 9776, 9777, 9782, 9779, 9781, 9780, 9778, 10998, 10856, - 10857, 10747, 8244, 8779, 10996, 10624, 8874, 10997, 11003, 11851, 11000, - 10999, 8749, 8285, 129484, 128654, 127865, 128032, 127942, 8872, 11231, - 127930, 8366, 8378, 129411, 8523, 10658, 11202, 9930, 9929, 8498, 11826, + 72648, 72662, 72661, 72669, 72652, 72640, 72663, 72651, 72655, 72672, + 72646, 72667, 72666, 72653, 72657, 72645, 72665, 72649, 72644, 72658, + 72668, 72670, 72664, 72671, 72641, 72659, 72650, 72656, 72643, 72660, + 72654, 72642, 72647, 72673, 72693, 72692, 72695, 72694, 72691, 72690, + 72688, 72697, 72689, 72696, 127751, 127803, 8316, 8312, 8305, 8319, 8317, + 8314, 8318, 8315, 8309, 8308, 8311, 8310, 179, 178, 8304, 8313, 185, + 10966, 10964, 10619, 10968, 10967, 8835, 10954, 10950, 10952, 10956, + 8843, 8839, 10948, 10944, 10946, 10942, 10185, 129464, 129465, 8751, + 127940, 128671, 127843, 128629, 129442, 127946, 8275, 43027, 43026, + 43031, 43030, 43040, 43038, 43025, 43024, 43029, 43028, 43036, 43035, + 43021, 43020, 43018, 43017, 43023, 43022, 43016, 43015, 43034, 43033, + 43042, 43039, 43037, 43032, 43041, 43008, 43012, 43009, 43013, 43011, + 43048, 43049, 43050, 43051, 43052, 43019, 43014, 43010, 43047, 43043, + 43046, 43044, 43045, 9223, 9224, 9229, 9240, 9232, 9249, 9256, 9255, + 9253, 9257, 9236, 9235, 9234, 9233, 9241, 9220, 9239, 9219, 9221, 9243, + 9244, 9228, 9225, 9227, 9226, 128325, 9237, 9252, 9216, 9222, 9246, 9245, + 9247, 8527, 9230, 9231, 9217, 9218, 9242, 9254, 9238, 9248, 11159, 9007, + 983094, 983093, 128333, 1807, 1866, 1802, 1798, 1799, 1849, 1848, 1792, + 1854, 1853, 1805, 1804, 1803, 1852, 1851, 1850, 1797, 1814, 1813, 1815, + 1818, 1824, 2153, 2152, 2149, 2148, 2144, 2146, 2150, 2147, 2154, 2145, + 2151, 1825, 1830, 1837, 1839, 1838, 1831, 1834, 1827, 1869, 1870, 1871, + 1809, 1835, 1832, 1828, 1808, 1823, 1833, 1819, 1820, 1836, 1811, 1812, + 1821, 1822, 1810, 1817, 1826, 1816, 1829, 1864, 1863, 1842, 1841, 1840, + 1845, 1844, 1843, 1847, 1846, 1855, 1858, 1796, 983204, 1801, 1794, 1795, + 1800, 1793, 1862, 1861, 1860, 1859, 1865, 1856, 1857, 128137, 983184, + 128085, 129430, 983061, 127955, 917543, 917542, 917546, 917598, 917568, + 917548, 917562, 917540, 917557, 917556, 917559, 917558, 917555, 917554, + 917552, 917561, 917553, 917560, 917565, 917537, 917600, 917566, 917549, + 917569, 917570, 917571, 917572, 917573, 917574, 917575, 917576, 917577, + 917578, 917579, 917580, 917581, 917582, 917583, 917584, 917585, 917586, + 917587, 917588, 917589, 917590, 917591, 917592, 917593, 917594, 917601, + 917602, 917603, 917604, 917605, 917606, 917607, 917608, 917609, 917610, + 917611, 917612, 917613, 917614, 917615, 917616, 917617, 917618, 917619, + 917620, 917621, 917622, 917623, 917624, 917625, 917626, 917564, 917627, + 917544, 917595, 917599, 917541, 917547, 917538, 917567, 917629, 917545, + 917597, 917596, 917563, 917551, 917536, 917539, 917630, 917550, 917628, + 5888, 5919, 5893, 5896, 5898, 5895, 5892, 5905, 5891, 5902, 5899, 5897, + 5901, 5904, 5894, 5903, 5900, 5889, 5890, 5909, 5908, 5906, 5907, 5989, + 5992, 5994, 5991, 5988, 5987, 5998, 5995, 5993, 6000, 5990, 5999, 5996, + 5984, 5985, 5986, 6002, 6003, 6499, 6508, 6509, 6507, 6501, 6502, 6512, + 6513, 6514, 6515, 6516, 6497, 6483, 6487, 6486, 6482, 6498, 6505, 6504, + 6496, 6480, 6490, 6489, 6503, 6506, 6492, 6494, 6488, 6491, 6495, 6484, + 6493, 6481, 6485, 6500, 6745, 6746, 6743, 6747, 6742, 6741, 6748, 6749, + 6750, 6783, 6789, 6788, 6791, 6790, 6787, 6786, 6784, 6793, 6785, 6792, + 6805, 6804, 6807, 6806, 6803, 6802, 6800, 6809, 6801, 6808, 6740, 6689, + 6690, 6688, 6702, 6726, 6727, 6728, 6696, 6695, 6713, 6712, 6707, 6706, + 6714, 6729, 6720, 6693, 6692, 6691, 6704, 6699, 6697, 6717, 6715, 6709, + 6708, 6716, 6732, 6698, 6719, 6723, 6739, 6724, 6730, 6721, 6705, 6701, + 6722, 6735, 6736, 6733, 6734, 6694, 6700, 6710, 6738, 6737, 6711, 6703, + 6718, 6725, 6731, 6828, 6820, 6775, 6776, 6777, 6780, 6824, 6825, 6819, + 6772, 6744, 6823, 6779, 6778, 6822, 6826, 6827, 6818, 6752, 6816, 6817, + 6821, 6773, 6774, 6829, 6753, 6755, 6767, 6769, 6754, 6763, 6764, 6771, + 6768, 6765, 6756, 6770, 6761, 6762, 6760, 6759, 6757, 6758, 6766, 43653, + 43651, 43649, 43661, 43659, 43679, 43677, 43671, 43669, 43657, 43665, + 43673, 43675, 43667, 43681, 43655, 43693, 43689, 43683, 43687, 43663, + 43691, 43685, 43695, 43652, 43650, 43648, 43660, 43658, 43678, 43676, + 43670, 43668, 43656, 43664, 43672, 43674, 43666, 43680, 43654, 43692, + 43688, 43682, 43686, 43662, 43690, 43684, 43694, 43696, 43703, 43743, + 43739, 43740, 43742, 43741, 43712, 43713, 43714, 43711, 43707, 43697, + 43710, 43709, 43708, 43700, 43699, 43705, 43706, 43698, 43704, 43701, + 43702, 71353, 71296, 71352, 71297, 71303, 71305, 71319, 71318, 71324, + 71323, 71338, 71332, 71317, 71316, 71322, 71321, 71300, 71301, 71298, + 71299, 71310, 71320, 71315, 71325, 71329, 71328, 71312, 71311, 71309, + 71308, 71314, 71313, 71307, 71306, 71327, 71326, 71335, 71336, 71337, + 71333, 71330, 71334, 71331, 71302, 71304, 71351, 71339, 71350, 71340, + 71341, 71347, 71349, 71344, 71345, 71342, 71343, 71346, 71348, 71365, + 71364, 71367, 71366, 71363, 71362, 71360, 71369, 71361, 71368, 129377, + 119672, 119671, 3064, 3031, 73707, 983662, 983685, 983674, 983677, + 983676, 983669, 983667, 983679, 983663, 983665, 983668, 983666, 983683, + 983681, 983682, 983673, 983678, 983664, 983684, 983680, 983671, 983670, + 983675, 983672, 73706, 3063, 73701, 3062, 3059, 3051, 3050, 3053, 3052, + 3049, 3048, 3046, 3055, 3047, 3054, 73700, 73666, 73676, 73668, 73679, + 73681, 73682, 73665, 73673, 73674, 73667, 73664, 73669, 73672, 73675, + 73680, 73670, 73678, 73671, 73677, 73683, 73684, 73710, 2985, 2979, 2969, + 2974, 2984, 2975, 2980, 2949, 2950, 2960, 2964, 2996, 2995, 2994, 2993, + 2992, 2953, 2954, 2962, 2963, 2951, 2952, 2998, 2999, 3000, 2958, 2959, + 2970, 3001, 2972, 2965, 2990, 2986, 2997, 2991, 73702, 3057, 3058, 3056, + 3066, 73727, 3065, 73703, 73687, 2946, 73686, 73698, 73690, 73693, 73692, + 73712, 73689, 73688, 73691, 73697, 73694, 73695, 73696, 73713, 73699, + 3021, 2947, 73685, 73708, 73711, 983939, 983940, 983947, 983950, 983943, + 983944, 983948, 983949, 983941, 983942, 983945, 983946, 983686, 983693, + 983696, 983689, 983690, 983694, 983695, 983687, 983688, 983691, 983692, + 983840, 983847, 983850, 983843, 983844, 983848, 983849, 983841, 983842, + 983845, 983846, 983851, 983858, 983861, 983854, 983855, 983859, 983860, + 983852, 983853, 983856, 983857, 983818, 983825, 983828, 983821, 983822, + 983826, 983827, 983819, 983820, 983823, 983824, 983873, 983880, 983883, + 983876, 983877, 983881, 983882, 983874, 983875, 983878, 983879, 983741, + 983748, 983751, 983744, 983745, 983749, 983750, 983742, 983743, 983746, + 983747, 983697, 983704, 983707, 983700, 983701, 983705, 983706, 983698, + 983699, 983702, 983703, 983719, 983726, 983729, 983722, 983723, 983727, + 983728, 983720, 983721, 983724, 983725, 983763, 983770, 983773, 983766, + 983767, 983771, 983772, 983764, 983765, 983768, 983769, 983862, 983869, + 983872, 983865, 983866, 983870, 983871, 983863, 983864, 983867, 983868, + 983807, 983814, 983817, 983810, 983811, 983815, 983816, 983808, 983809, + 983812, 983813, 983895, 983902, 983905, 983898, 983899, 983903, 983904, + 983951, 983896, 983897, 983900, 983901, 983906, 983913, 983916, 983909, + 983910, 983914, 983915, 983907, 983908, 983911, 983912, 983917, 983924, + 983927, 983920, 983921, 983925, 983926, 983918, 983919, 983922, 983923, + 983730, 983737, 983740, 983733, 983734, 983738, 983739, 983731, 983732, + 983735, 983736, 983752, 983759, 983762, 983755, 983756, 983760, 983761, + 983753, 983754, 983757, 983758, 983708, 983715, 983718, 983711, 983712, + 983716, 983717, 983709, 983710, 983713, 983714, 983928, 983935, 983938, + 983931, 983932, 983936, 983937, 983929, 983930, 983933, 983934, 983884, + 983891, 983894, 983887, 983888, 983892, 983893, 983885, 983886, 983889, + 983890, 983785, 983792, 983795, 983788, 983789, 983793, 983794, 983786, + 983787, 983790, 983791, 983774, 983781, 983784, 983777, 983778, 983782, + 983783, 983775, 983776, 983779, 983780, 983829, 983836, 983839, 983832, + 983833, 983837, 983838, 983830, 983831, 983834, 983835, 983796, 983803, + 983806, 983799, 983800, 983804, 983805, 983797, 983798, 983801, 983802, + 73709, 73704, 73705, 3006, 3016, 3020, 3009, 3010, 3018, 3019, 3007, + 3008, 3014, 3015, 3061, 3060, 3024, 129748, 127883, 127818, 92809, 92810, + 92811, 92808, 92789, 92790, 92791, 92788, 92816, 92859, 92856, 92847, + 92845, 92817, 92846, 92843, 92829, 92830, 92831, 92828, 92835, 92851, + 92840, 92844, 92818, 92819, 92852, 92836, 92825, 92826, 92827, 92824, + 92813, 92814, 92815, 92812, 92820, 92822, 92823, 92821, 92805, 92806, + 92807, 92804, 92797, 92798, 92799, 92796, 92801, 92802, 92803, 92800, + 92785, 92786, 92787, 92784, 92793, 92794, 92795, 92792, 92857, 92854, + 92848, 92861, 92853, 92860, 92849, 92855, 92834, 92833, 92832, 92841, + 92839, 92842, 92850, 92838, 92858, 92837, 92862, 92869, 92868, 92871, + 92870, 92867, 92866, 92864, 92873, 92865, 92872, 100352, 100353, 100354, + 100355, 100356, 100357, 100358, 100359, 100360, 100361, 100362, 100363, + 100364, 100365, 100366, 100367, 100368, 100369, 100370, 100371, 100372, + 100373, 100374, 100375, 100376, 100377, 100378, 100379, 100380, 100381, + 100382, 100383, 100384, 100385, 100386, 100387, 100388, 100389, 100390, + 100391, 100392, 100393, 100394, 100395, 100396, 100397, 100398, 100399, + 100400, 100401, 100402, 100403, 100404, 100405, 100406, 100407, 100408, + 100409, 100410, 100411, 100412, 100413, 100414, 100415, 100416, 100417, + 100418, 100419, 100420, 100421, 100422, 100423, 100424, 100425, 100426, + 100427, 100428, 100429, 100430, 100431, 100432, 100433, 100434, 100435, + 100436, 100437, 100438, 100439, 100440, 100441, 100442, 100443, 100444, + 100445, 100446, 100447, 100448, 100449, 100450, 100451, 100452, 100453, + 100454, 100455, 100456, 100457, 100458, 100459, 100460, 100461, 100462, + 100463, 100464, 100465, 100466, 100467, 100468, 100469, 100470, 100471, + 100472, 100473, 100474, 100475, 100476, 100477, 100478, 100479, 100480, + 100481, 100482, 100483, 100484, 100485, 100486, 100487, 100488, 100489, + 100490, 100491, 100492, 100493, 100494, 100495, 100496, 100497, 100498, + 100499, 100500, 100501, 100502, 100503, 100504, 100505, 100506, 100507, + 100508, 100509, 100510, 100511, 100512, 100513, 100514, 100515, 100516, + 100517, 100518, 100519, 100520, 100521, 100522, 100523, 100524, 100525, + 100526, 100527, 100528, 100529, 100530, 100531, 100532, 100533, 100534, + 100535, 100536, 100537, 100538, 100539, 100540, 100541, 100542, 100543, + 100544, 100545, 100546, 100547, 100548, 100549, 100550, 100551, 100552, + 100553, 100554, 100555, 100556, 100557, 100558, 100559, 100560, 100561, + 100562, 100563, 100564, 100565, 100566, 100567, 100568, 100569, 100570, + 100571, 100572, 100573, 100574, 100575, 100576, 100577, 100578, 100579, + 100580, 100581, 100582, 100583, 100584, 100585, 100586, 100587, 100588, + 100589, 100590, 100591, 100592, 100593, 100594, 100595, 100596, 100597, + 100598, 100599, 100600, 100601, 100602, 100603, 100604, 100605, 100606, + 100607, 100608, 100609, 100610, 100611, 100612, 100613, 100614, 100615, + 100616, 100617, 100618, 100619, 100620, 100621, 100622, 100623, 100624, + 100625, 100626, 100627, 100628, 100629, 100630, 100631, 100632, 100633, + 100634, 100635, 100636, 100637, 100638, 100639, 100640, 100641, 100642, + 100643, 100644, 100645, 100646, 100647, 100648, 100649, 100650, 100651, + 100652, 100653, 100654, 100655, 100656, 100657, 100658, 100659, 100660, + 100661, 100662, 100663, 100664, 100665, 100666, 100667, 100668, 100669, + 100670, 100671, 100672, 100673, 100674, 100675, 100676, 100677, 100678, + 100679, 100680, 100681, 100682, 100683, 100684, 100685, 100686, 100687, + 100688, 100689, 100690, 100691, 100692, 100693, 100694, 100695, 100696, + 100697, 100698, 100699, 100700, 100701, 100702, 100703, 100704, 100705, + 100706, 100707, 100708, 100709, 100710, 100711, 100712, 100713, 100714, + 100715, 100716, 100717, 100718, 100719, 100720, 100721, 100722, 100723, + 100724, 100725, 100726, 100727, 100728, 100729, 100730, 100731, 100732, + 100733, 100734, 100735, 100736, 100737, 100738, 100739, 100740, 100741, + 100742, 100743, 100744, 100745, 100746, 100747, 100748, 100749, 100750, + 100751, 100752, 100753, 100754, 100755, 100756, 100757, 100758, 100759, + 100760, 100761, 100762, 100763, 100764, 100765, 100766, 100767, 100768, + 100769, 100770, 100771, 100772, 100773, 100774, 100775, 100776, 100777, + 100778, 100779, 100780, 100781, 100782, 100783, 100784, 100785, 100786, + 100787, 100788, 100789, 100790, 100791, 100792, 100793, 100794, 100795, + 100796, 100797, 100798, 100799, 100800, 100801, 100802, 100803, 100804, + 100805, 100806, 100807, 100808, 100809, 100810, 100811, 100812, 100813, + 100814, 100815, 100816, 100817, 100818, 100819, 100820, 100821, 100822, + 100823, 100824, 100825, 100826, 100827, 100828, 100829, 100830, 100831, + 100832, 100833, 100834, 100835, 100836, 100837, 100838, 100839, 100840, + 100841, 100842, 100843, 100844, 100845, 100846, 100847, 100848, 100849, + 100850, 100851, 100852, 100853, 100854, 100855, 100856, 100857, 100858, + 100859, 100860, 100861, 100862, 100863, 100864, 100865, 100866, 100867, + 100868, 100869, 100870, 100871, 100872, 100873, 100874, 100875, 100876, + 100877, 100878, 100879, 100880, 100881, 100882, 100883, 100884, 100885, + 100886, 100887, 100888, 100889, 100890, 100891, 100892, 100893, 100894, + 100895, 100896, 100897, 100898, 100899, 100900, 100901, 100902, 100903, + 100904, 100905, 100906, 100907, 100908, 100909, 100910, 100911, 100912, + 100913, 100914, 100915, 100916, 100917, 100918, 100919, 100920, 100921, + 100922, 100923, 100924, 100925, 100926, 100927, 100928, 100929, 100930, + 100931, 100932, 100933, 100934, 100935, 100936, 100937, 100938, 100939, + 100940, 100941, 100942, 100943, 100944, 100945, 100946, 100947, 100948, + 100949, 100950, 100951, 100952, 100953, 100954, 100955, 100956, 100957, + 100958, 100959, 100960, 100961, 100962, 100963, 100964, 100965, 100966, + 100967, 100968, 100969, 100970, 100971, 100972, 100973, 100974, 100975, + 100976, 100977, 100978, 100979, 100980, 100981, 100982, 100983, 100984, + 100985, 100986, 100987, 100988, 100989, 100990, 100991, 100992, 100993, + 100994, 100995, 100996, 100997, 100998, 100999, 101000, 101001, 101002, + 101003, 101004, 101005, 101006, 101007, 101008, 101009, 101010, 101011, + 101012, 101013, 101014, 101015, 101016, 101017, 101018, 101019, 101020, + 101021, 101022, 101023, 101024, 101025, 101026, 101027, 101028, 101029, + 101030, 101031, 101032, 101033, 101034, 101035, 101036, 101037, 101038, + 101039, 101040, 101041, 101042, 101043, 101044, 101045, 101046, 101047, + 101048, 101049, 101050, 101051, 101052, 101053, 101054, 101055, 101056, + 101057, 101058, 101059, 101060, 101061, 101062, 101063, 101064, 101065, + 101066, 101067, 101068, 101069, 101070, 101071, 101072, 101073, 101074, + 101075, 101076, 101077, 101078, 101079, 101080, 101081, 101082, 101083, + 101084, 101085, 101086, 101087, 101088, 101089, 101090, 101091, 101092, + 101093, 101094, 101095, 101096, 101097, 101098, 101099, 101100, 101101, + 101102, 101103, 101104, 101105, 101106, 101107, 101108, 101109, 101110, + 101111, 101112, 101113, 101114, 101115, 101116, 101117, 101118, 101119, + 94176, 128429, 9991, 9801, 127790, 128661, 127861, 128198, 10043, 10170, + 129750, 129528, 128222, 128380, 8981, 9990, 8481, 128384, 128301, 128250, + 3158, 3195, 3198, 3194, 3197, 3193, 3196, 3192, 3106, 3105, 3111, 3161, + 3110, 3112, 3165, 3097, 3107, 3102, 3162, 3121, 3120, 3104, 3103, 3109, + 3160, 3108, 3077, 3078, 3088, 3092, 3124, 3123, 3122, 3083, 3168, 3084, + 3169, 3125, 3081, 3082, 3090, 3091, 3079, 3080, 3126, 3127, 3128, 3117, + 3116, 3099, 3098, 3096, 3095, 3101, 3100, 3094, 3093, 3115, 3114, 3086, + 3087, 3129, 3118, 3119, 3157, 3076, 3072, 3073, 3191, 3199, 3132, 3133, + 3074, 3149, 3075, 3134, 3144, 3148, 3137, 3138, 3139, 3140, 3170, 3171, + 3146, 3147, 3135, 3136, 3142, 3143, 3179, 3178, 3181, 3180, 3177, 3176, + 3174, 3183, 3175, 3182, 127934, 8376, 9978, 129514, 119617, 119564, + 119577, 119633, 119587, 119566, 119561, 119585, 119613, 119590, 119631, + 119634, 119630, 119608, 119582, 119563, 119573, 119558, 119624, 119567, + 119623, 119586, 119636, 119612, 119625, 119568, 119584, 119619, 119618, + 119583, 119603, 119600, 119626, 119610, 119580, 119576, 119638, 119559, + 119595, 119632, 119606, 119592, 119615, 119599, 119602, 119614, 119629, + 119574, 119569, 119570, 119622, 119562, 119591, 119637, 119597, 119589, + 119616, 119609, 119560, 119635, 119565, 119604, 119588, 119571, 119594, + 119578, 119596, 119579, 119598, 119572, 119605, 119627, 119621, 119628, + 119601, 119593, 119607, 119575, 119620, 119611, 119581, 1959, 1958, 1964, + 1961, 1965, 1927, 1954, 1951, 1937, 1931, 1930, 1956, 1934, 1935, 1955, + 1945, 1920, 1926, 1946, 1933, 1925, 1929, 1943, 1942, 1941, 1922, 1969, + 1950, 1949, 1921, 1936, 1939, 1932, 1947, 1944, 1952, 1928, 1957, 1938, + 1948, 1953, 1924, 1923, 1940, 1967, 1966, 1963, 1960, 1962, 1968, 3610, + 3592, 3594, 3593, 3596, 3598, 3604, 3613, 3615, 3663, 3630, 3627, 3588, + 3587, 3589, 3586, 3590, 3675, 3585, 3628, 3621, 3653, 3622, 3659, 3633, + 3657, 3658, 3656, 3655, 3654, 3617, 3674, 3591, 3661, 3603, 3609, 3631, + 3642, 3612, 3614, 3616, 3611, 3619, 3620, 3632, 3652, 3651, 3634, 3649, + 3635, 3640, 3638, 3639, 3641, 3636, 3637, 3648, 3650, 3625, 3624, 3626, + 3595, 3660, 3601, 3602, 3607, 3600, 3608, 3606, 3605, 3599, 3623, 3662, + 3597, 3618, 3629, 3647, 3669, 3668, 3671, 3670, 3667, 3666, 3664, 3673, + 3665, 3672, 8708, 8707, 8756, 127777, 10727, 128936, 8201, 128929, + 129300, 129353, 128173, 129652, 10176, 8278, 9887, 9886, 11057, 128485, + 128484, 128486, 128487, 8694, 128433, 10870, 128491, 128962, 128423, + 11160, 11162, 10146, 11163, 11161, 10147, 8196, 11835, 128077, 128078, + 9928, 9736, 3865, 3863, 3864, 4032, 4033, 4035, 4034, 3886, 3885, 3888, + 3887, 3884, 3883, 3891, 3890, 3882, 3889, 3877, 3876, 3879, 3878, 3875, + 3874, 3872, 3881, 3873, 3880, 4030, 4031, 3945, 3905, 3947, 3904, 3948, + 3938, 3946, 3917, 3916, 3932, 3931, 3922, 3921, 3908, 3918, 3913, 3923, + 3940, 3941, 3942, 3930, 3929, 3915, 3914, 3920, 3919, 3927, 3926, 3910, + 3909, 3907, 3906, 3925, 3924, 3934, 3935, 3936, 3943, 3911, 3939, 3928, + 3933, 3937, 3944, 3862, 3861, 983210, 3850, 4048, 3849, 3892, 4049, 3894, + 3859, 3846, 3896, 4052, 3845, 3847, 3842, 3843, 3841, 3860, 3898, 3899, + 3900, 3901, 3972, 4051, 3844, 3851, 3895, 3893, 4050, 3856, 3854, 3858, + 3857, 3848, 3853, 4058, 3897, 3855, 4057, 3973, 3852, 3978, 3979, 3974, + 3976, 3903, 3977, 3970, 3866, 3867, 3868, 3871, 3869, 3870, 4047, 4046, + 3966, 3967, 3975, 3902, 3980, 3971, 4028, 4026, 4027, 3997, 3996, 4012, + 4011, 4002, 4001, 4025, 3985, 3984, 3988, 3998, 3993, 4003, 4020, 4021, + 4022, 4010, 4009, 3995, 3994, 4000, 3999, 4007, 4006, 3990, 3989, 3987, + 3986, 4005, 4004, 4014, 4015, 4016, 4023, 3991, 4019, 4008, 4018, 4013, + 4017, 4024, 3983, 3982, 3981, 3840, 4036, 4041, 4044, 4043, 4042, 4038, + 4040, 4037, 4039, 3968, 3969, 3956, 3957, 3958, 3959, 3960, 3961, 3964, + 3965, 3954, 3955, 3962, 3963, 3953, 10717, 11647, 11608, 11595, 11585, + 11573, 11620, 11607, 11600, 11582, 11590, 11596, 11601, 11586, 11592, + 11568, 11575, 11577, 11578, 11576, 11571, 11606, 11572, 11581, 11589, + 11583, 11569, 11570, 11584, 11587, 11609, 11611, 11610, 11612, 11613, + 11615, 11619, 11594, 11621, 11604, 11605, 11614, 11588, 11580, 11574, + 11597, 11598, 11599, 11602, 11591, 11616, 11617, 11618, 11622, 11579, + 11593, 11623, 11603, 11631, 11632, 128005, 128047, 10053, 126, 8764, + 11081, 10610, 10859, 10858, 11807, 11806, 11803, 9202, 10708, 10709, + 10750, 68410, 70854, 70784, 70785, 70786, 70796, 70798, 70812, 70811, + 70817, 70816, 70810, 70809, 70815, 70814, 70791, 70792, 70793, 70794, + 70827, 70789, 70790, 70787, 70788, 70803, 70813, 70808, 70818, 70828, + 70829, 70830, 70822, 70821, 70805, 70804, 70802, 70801, 70807, 70806, + 70800, 70799, 70820, 70819, 70831, 70826, 70823, 70825, 70824, 70795, + 70797, 70847, 70851, 70852, 70848, 70850, 70849, 70842, 70845, 70832, + 70843, 70846, 70835, 70836, 70837, 70838, 70839, 70840, 70833, 70834, + 70841, 70844, 70869, 70868, 70871, 70870, 70867, 70866, 70864, 70873, + 70865, 70872, 70853, 70855, 11858, 8266, 128555, 127915, 67054, 67022, + 67020, 67027, 67051, 67011, 67031, 67023, 67021, 67033, 67049, 67047, + 67015, 67032, 67025, 67024, 67058, 67035, 67041, 67040, 67056, 67055, + 67039, 67038, 67037, 67034, 67059, 67018, 67017, 67030, 67029, 67008, + 67009, 67013, 67012, 67016, 67014, 67057, 67028, 67043, 67042, 67048, + 67046, 67053, 67052, 67010, 67019, 67036, 67045, 67026, 67044, 67050, + 127813, 128069, 129463, 129701, 129520, 10554, 10557, 10556, 9182, + 118256, 117851, 118262, 118263, 11865, 117850, 118252, 118254, 11866, + 117852, 118248, 118250, 11833, 118246, 11210, 8992, 127913, 9180, 130024, + 130016, 130031, 8988, 8975, 11810, 118279, 8989, 8974, 11811, 130028, + 9140, 9184, 128285, 127553, 127554, 127559, 127555, 127557, 127560, + 127552, 127556, 127558, 127274, 9008, 123554, 123556, 123559, 123561, + 123564, 123537, 123544, 123543, 123553, 123555, 123558, 123560, 123546, + 123565, 123563, 123539, 123541, 123550, 123549, 123540, 123552, 123542, + 123536, 123551, 123545, 123538, 123548, 123547, 123562, 123557, 123566, + 128701, 128508, 128434, 128668, 8482, 128650, 128651, 11223, 10971, + 128646, 10701, 10699, 128710, 10698, 10141, 128681, 128208, 8227, 128305, + 9783, 9776, 9777, 9782, 9779, 9781, 9780, 9778, 10998, 10856, 10857, + 10747, 8244, 8779, 10996, 10624, 8874, 10997, 11003, 11851, 11000, 10999, + 8749, 8285, 129484, 128654, 127865, 128032, 127942, 8872, 11231, 127930, + 8366, 70601, 70608, 70613, 70612, 70610, 70528, 70529, 70542, 70545, + 70559, 70558, 70564, 70563, 70581, 70579, 70573, 70580, 70572, 70557, + 70556, 70562, 70561, 70534, 70535, 70536, 70537, 70574, 70532, 70533, + 70530, 70531, 70550, 70560, 70555, 70565, 70575, 70576, 70577, 70569, + 70568, 70552, 70551, 70549, 70548, 70554, 70553, 70547, 70546, 70567, + 70566, 70544, 70539, 70578, 70570, 70571, 70609, 70583, 70604, 70607, + 70615, 70616, 70602, 70611, 70606, 70605, 70626, 70625, 70584, 70597, + 70600, 70587, 70588, 70589, 70590, 70591, 70592, 70585, 70586, 70599, + 70594, 127799, 8378, 129411, 8523, 10658, 11202, 9930, 9929, 8498, 11826, 11832, 8587, 8586, 128598, 128596, 8985, 128399, 8513, 8514, 8516, 11829, - 8526, 128599, 128597, 8489, 128034, 129347, 127799, 10041, 128256, 8273, - 128432, 10869, 8229, 8282, 11818, 128149, 10837, 10838, 10697, 10760, - 10759, 128108, 128109, 128490, 11834, 66432, 66451, 66454, 66433, 66436, + 8526, 128599, 128597, 8489, 128034, 129347, 10041, 128256, 128432, 10869, + 8229, 8282, 11818, 128149, 10837, 10838, 10697, 10760, 10759, 128108, + 128109, 117896, 8273, 128490, 11834, 66432, 66451, 66454, 66433, 66436, 66447, 66457, 66434, 66437, 66440, 66443, 66435, 66445, 66455, 66453, 66450, 66444, 66461, 66456, 66441, 66458, 66442, 66439, 66449, 66448, 66452, 66438, 66446, 66459, 66460, 66463, 9730, 9969, 9748, 128530, 11217, 8255, 9100, 8746, 10824, 10822, 10826, 10817, 10818, 10821, - 983116, 11258, 9842, 129412, 9903, 10685, 8963, 8996, 11193, 10577, + 983116, 11258, 9842, 129412, 9903, 8963, 8996, 11193, 10685, 10577, 10573, 10572, 10575, 8597, 8616, 11021, 8661, 129113, 8691, 11109, 10622, - 8944, 8869, 10207, 128742, 128744, 128316, 128743, 128314, 9709, 9710, - 129898, 129946, 129920, 129896, 9985, 129924, 9600, 129937, 129938, 9690, - 129934, 9696, 129885, 129887, 129889, 129886, 129883, 129888, 129881, - 129884, 129882, 129879, 129880, 10196, 129944, 129948, 9720, 9136, 9692, - 129922, 9620, 129874, 129875, 129892, 129894, 129890, 129878, 129893, - 129891, 129876, 129895, 129877, 10064, 9137, 10000, 9693, 128319, 10066, - 129945, 129949, 9721, 129926, 129923, 129925, 129901, 128579, 11797, - 8593, 129976, 8645, 8670, 129033, 129029, 129177, 129041, 129025, 129045, - 8624, 8625, 10505, 8613, 10514, 11145, 11014, 8657, 8673, 129077, 10606, - 10595, 10592, 10584, 8639, 10588, 10580, 8638, 129089, 129093, 129085, - 10224, 128621, 129105, 129081, 11105, 11137, 11121, 129065, 11170, 11171, - 129061, 129057, 129073, 129069, 11131, 11115, 11141, 129169, 10506, 8607, - 11245, 10569, 8648, 8679, 8682, 11192, 8683, 8684, 8685, 129173, 8686, - 8687, 9797, 983117, 42509, 42510, 42511, 42446, 42370, 42486, 42257, - 42333, 42294, 42407, 42445, 42369, 42485, 42256, 42332, 42293, 42406, - 42449, 42373, 42489, 42260, 42336, 42297, 42410, 42434, 42359, 42473, - 42246, 42321, 42283, 42396, 42435, 42360, 42474, 42247, 42322, 42284, - 42397, 42452, 42376, 42492, 42263, 42339, 42300, 42413, 42451, 42375, - 42491, 42262, 42338, 42299, 42412, 42444, 42368, 42484, 42255, 42331, - 42292, 42405, 42443, 42367, 42483, 42254, 42330, 42291, 42404, 42454, - 42378, 42494, 42265, 42341, 42302, 42415, 42453, 42377, 42493, 42264, - 42340, 42301, 42414, 42439, 42440, 42364, 42479, 42251, 42480, 42327, - 42288, 42401, 42502, 42272, 42503, 42461, 42385, 42349, 42309, 42422, - 42429, 42430, 42355, 42468, 42242, 42469, 42316, 42317, 42278, 42279, - 42391, 42392, 42476, 42249, 42477, 42437, 42362, 42324, 42325, 42286, - 42399, 42459, 42383, 42346, 42347, 42499, 42270, 42307, 42420, 42487, - 42508, 42258, 42447, 42371, 42334, 42295, 42408, 42436, 42361, 42475, - 42248, 42323, 42285, 42398, 42438, 42363, 42478, 42250, 42326, 42287, - 42400, 42462, 42386, 42504, 42273, 42350, 42310, 42423, 42450, 42514, - 42539, 42512, 42513, 42538, 42374, 42490, 42261, 42337, 42298, 42411, - 42507, 42500, 42271, 42501, 42460, 42384, 42348, 42308, 42421, 42315, - 42467, 42428, 42457, 42381, 42497, 42268, 42344, 42305, 42418, 42464, - 42388, 42506, 42275, 42352, 42312, 42425, 42463, 42387, 42505, 42274, - 42351, 42311, 42424, 42455, 42379, 42495, 42266, 42342, 42303, 42416, - 42441, 42365, 42481, 42252, 42328, 42289, 42402, 42456, 42380, 42496, - 42267, 42343, 42304, 42417, 42433, 42358, 42472, 42245, 42320, 42282, - 42395, 42448, 42372, 42488, 42259, 42335, 42296, 42409, 42442, 42366, - 42482, 42253, 42329, 42290, 42403, 42458, 42382, 42498, 42269, 42345, - 42306, 42419, 42470, 42243, 42244, 42471, 42431, 42356, 42357, 42432, - 42318, 42319, 42280, 42281, 42393, 42394, 42465, 42240, 42241, 42466, - 42426, 42353, 42354, 42427, 42313, 42314, 42276, 42277, 42389, 42390, - 42523, 42526, 42522, 42515, 42520, 42527, 42516, 42524, 42518, 42517, - 42525, 42521, 42519, 42533, 42532, 42535, 42534, 42531, 42530, 42528, - 42537, 42529, 42536, 65024, 65033, 917843, 917844, 917845, 917846, - 917847, 917848, 917849, 917850, 917851, 917852, 65034, 917853, 917854, - 917855, 917856, 917857, 917858, 917859, 917860, 917861, 917862, 65035, - 917863, 917864, 917865, 917866, 917867, 917868, 917869, 917870, 917871, - 917872, 65036, 917873, 917874, 917875, 917876, 917877, 917878, 917879, - 917880, 917881, 917882, 65037, 917883, 917884, 917885, 917886, 917887, - 917888, 917889, 917890, 917891, 917892, 65038, 917893, 917894, 917895, - 917896, 917897, 917898, 917899, 917900, 917901, 917902, 65039, 917903, - 917904, 917905, 917906, 917907, 917908, 917909, 917910, 917911, 917912, - 917760, 917913, 917914, 917915, 917916, 917917, 917918, 917919, 917920, - 917921, 917922, 917761, 917923, 917924, 917925, 917926, 917927, 917928, - 917929, 917930, 917931, 917932, 917762, 917933, 917934, 917935, 917936, - 917937, 917938, 917939, 917940, 917941, 917942, 65025, 917763, 917943, - 917944, 917945, 917946, 917947, 917948, 917949, 917950, 917951, 917952, - 917764, 917953, 917954, 917955, 917956, 917957, 917958, 917959, 917960, - 917961, 917962, 917765, 917963, 917964, 917965, 917966, 917967, 917968, - 917969, 917970, 917971, 917972, 917766, 917973, 917974, 917975, 917976, - 917977, 917978, 917979, 917980, 917981, 917982, 917767, 917983, 917984, - 917985, 917986, 917987, 917988, 917989, 917990, 917991, 917992, 917768, - 917993, 917994, 917995, 917996, 917997, 917998, 917999, 917769, 917770, - 917771, 917772, 65026, 917773, 917774, 917775, 917776, 917777, 917778, - 917779, 917780, 917781, 917782, 65027, 917783, 917784, 917785, 917786, - 917787, 917788, 917789, 917790, 917791, 917792, 65028, 917793, 917794, - 917795, 917796, 917797, 917798, 917799, 917800, 917801, 917802, 65029, - 917803, 917804, 917805, 917806, 917807, 917808, 917809, 917810, 917811, - 917812, 65030, 917813, 917814, 917815, 917816, 917817, 917818, 917819, - 917820, 917821, 917822, 65031, 917823, 917824, 917825, 917826, 917827, - 917828, 917829, 917830, 917831, 917832, 65032, 917833, 917834, 917835, - 917836, 917837, 917838, 917839, 917840, 917841, 917842, 129499, 983245, - 983254, 983356, 983357, 983358, 983359, 983360, 983361, 983362, 983363, - 983364, 983365, 983255, 983366, 983367, 983368, 983369, 983370, 983371, - 983372, 983373, 983374, 983375, 983256, 983376, 983377, 983378, 983379, - 983380, 983381, 983382, 983383, 983384, 983385, 983257, 983386, 983387, - 983388, 983389, 983390, 983391, 983392, 983393, 983394, 983395, 983258, - 983396, 983397, 983398, 983399, 983400, 983401, 983402, 983403, 983404, - 983405, 983259, 983406, 983407, 983408, 983409, 983410, 983411, 983412, - 983413, 983414, 983415, 983260, 983416, 983417, 983418, 983419, 983420, - 983421, 983422, 983423, 983424, 983425, 983273, 983426, 983427, 983428, - 983429, 983430, 983431, 983432, 983433, 983434, 983435, 983274, 983436, - 983437, 983438, 983439, 983440, 983441, 983442, 983443, 983444, 983445, - 983275, 983446, 983447, 983448, 983449, 983450, 983451, 983452, 983453, - 983454, 983455, 983246, 983276, 983456, 983457, 983458, 983459, 983460, - 983461, 983462, 983463, 983464, 983465, 983277, 983466, 983467, 983468, - 983469, 983470, 983471, 983472, 983473, 983474, 983475, 983278, 983476, - 983477, 983478, 983479, 983480, 983481, 983482, 983483, 983484, 983485, - 983279, 983486, 983487, 983488, 983489, 983490, 983491, 983492, 983493, - 983494, 983495, 983280, 983496, 983497, 983498, 983499, 983500, 983501, - 983502, 983503, 983504, 983505, 983281, 983506, 983507, 983508, 983509, - 983510, 983511, 983512, 983282, 983283, 983284, 983285, 983247, 983286, - 983287, 983288, 983289, 983290, 983291, 983292, 983293, 983294, 983295, - 983248, 983296, 983297, 983298, 983299, 983300, 983301, 983302, 983303, - 983304, 983305, 983249, 983306, 983307, 983308, 983309, 983310, 983311, - 983312, 983313, 983314, 983315, 983250, 983316, 983317, 983318, 983319, - 983320, 983321, 983322, 983323, 983324, 983325, 983251, 983326, 983327, - 983328, 983329, 983330, 983331, 983332, 983333, 983334, 983335, 983252, - 983336, 983337, 983338, 983339, 983340, 983341, 983342, 983343, 983344, - 983345, 983253, 983346, 983347, 983348, 983349, 983350, 983351, 983352, - 983353, 983354, 983355, 7401, 7402, 7409, 7403, 7404, 7415, 7410, 7418, - 7413, 7379, 7398, 7396, 7411, 7408, 7406, 7407, 7405, 7414, 7397, 7400, - 7395, 7399, 7394, 7380, 7384, 7412, 7417, 7386, 7389, 7376, 7388, 7378, - 7416, 7392, 7391, 7387, 7390, 7381, 7382, 7383, 7385, 7393, 7377, 8483, - 10704, 10980, 10978, 10186, 8942, 8286, 12347, 12337, 12339, 12341, - 12338, 12340, 124, 9168, 10992, 10991, 9087, 9896, 129904, 129905, - 129906, 129907, 129908, 129909, 11135, 983069, 983144, 11823, 128678, - 11837, 10650, 128976, 128959, 128947, 128637, 128941, 128953, 128636, + 8944, 8869, 10207, 117873, 117877, 117857, 128743, 117881, 118267, + 128742, 117847, 128314, 117912, 117862, 128744, 128316, 9709, 9710, + 117760, 129898, 129946, 129920, 129896, 9985, 118417, 117809, 118418, + 117810, 130020, 129924, 9600, 129937, 118275, 129938, 9690, 118292, + 129934, 118439, 118444, 9696, 129885, 129887, 129889, 129886, 129883, + 129888, 129881, 129884, 129882, 129879, 129880, 10196, 118416, 9136, + 129944, 129948, 9720, 117808, 9692, 117958, 117946, 117962, 117966, + 117950, 117954, 117928, 117926, 117942, 117970, 117938, 117813, 118421, + 118422, 118420, 117812, 118423, 117815, 129922, 9620, 129874, 129875, + 129892, 129894, 129890, 129878, 129893, 129891, 129876, 129895, 129877, + 10064, 118419, 9137, 10000, 9693, 117959, 117947, 117963, 117967, 117951, + 117955, 117929, 117927, 117943, 117971, 117939, 117814, 128319, 10066, + 129945, 129949, 9721, 117811, 129926, 129923, 118438, 118445, 129925, + 129901, 128579, 11797, 8593, 129976, 8645, 8670, 129033, 129029, 129177, + 129041, 129025, 129045, 8624, 8625, 10505, 8613, 10514, 11145, 11014, + 8657, 8673, 129077, 10606, 10595, 10592, 10584, 8639, 10588, 10580, 8638, + 129089, 129093, 129085, 10224, 128621, 129105, 129081, 11105, 11137, + 11121, 129065, 11170, 11171, 129061, 129057, 129073, 129069, 11131, + 11115, 11141, 129169, 10506, 8607, 11245, 10569, 8648, 8679, 8683, 8685, + 8684, 129173, 8682, 11192, 8686, 8687, 9797, 983117, 42509, 42510, 42511, + 42446, 42370, 42486, 42257, 42333, 42294, 42407, 42445, 42369, 42485, + 42256, 42332, 42293, 42406, 42449, 42373, 42489, 42260, 42336, 42297, + 42410, 42434, 42359, 42473, 42246, 42321, 42283, 42396, 42435, 42360, + 42474, 42247, 42322, 42284, 42397, 42452, 42376, 42492, 42263, 42339, + 42300, 42413, 42451, 42375, 42491, 42262, 42338, 42299, 42412, 42444, + 42368, 42484, 42255, 42331, 42292, 42405, 42443, 42367, 42483, 42254, + 42330, 42291, 42404, 42454, 42378, 42494, 42265, 42341, 42302, 42415, + 42453, 42377, 42493, 42264, 42340, 42301, 42414, 42439, 42440, 42364, + 42479, 42251, 42480, 42327, 42288, 42401, 42502, 42272, 42503, 42461, + 42385, 42349, 42309, 42422, 42429, 42430, 42355, 42468, 42242, 42469, + 42316, 42317, 42278, 42279, 42391, 42392, 42476, 42249, 42477, 42437, + 42362, 42324, 42325, 42286, 42399, 42459, 42383, 42346, 42347, 42499, + 42270, 42307, 42420, 42487, 42508, 42258, 42447, 42371, 42334, 42295, + 42408, 42436, 42361, 42475, 42248, 42323, 42285, 42398, 42438, 42363, + 42478, 42250, 42326, 42287, 42400, 42462, 42386, 42504, 42273, 42350, + 42310, 42423, 42450, 42514, 42539, 42512, 42513, 42538, 42374, 42490, + 42261, 42337, 42298, 42411, 42507, 42500, 42271, 42501, 42460, 42384, + 42348, 42308, 42421, 42315, 42467, 42428, 42457, 42381, 42497, 42268, + 42344, 42305, 42418, 42464, 42388, 42506, 42275, 42352, 42312, 42425, + 42463, 42387, 42505, 42274, 42351, 42311, 42424, 42455, 42379, 42495, + 42266, 42342, 42303, 42416, 42441, 42365, 42481, 42252, 42328, 42289, + 42402, 42456, 42380, 42496, 42267, 42343, 42304, 42417, 42433, 42358, + 42472, 42245, 42320, 42282, 42395, 42448, 42372, 42488, 42259, 42335, + 42296, 42409, 42442, 42366, 42482, 42253, 42329, 42290, 42403, 42458, + 42382, 42498, 42269, 42345, 42306, 42419, 42470, 42243, 42244, 42471, + 42431, 42356, 42357, 42432, 42318, 42319, 42280, 42281, 42393, 42394, + 42465, 42240, 42241, 42466, 42426, 42353, 42354, 42427, 42313, 42314, + 42276, 42277, 42389, 42390, 42523, 42526, 42522, 42515, 42520, 42527, + 42516, 42524, 42518, 42517, 42525, 42521, 42519, 42533, 42532, 42535, + 42534, 42531, 42530, 42528, 42537, 42529, 42536, 65024, 65033, 917843, + 917844, 917845, 917846, 917847, 917848, 917849, 917850, 917851, 917852, + 65034, 917853, 917854, 917855, 917856, 917857, 917858, 917859, 917860, + 917861, 917862, 65035, 917863, 917864, 917865, 917866, 917867, 917868, + 917869, 917870, 917871, 917872, 65036, 917873, 917874, 917875, 917876, + 917877, 917878, 917879, 917880, 917881, 917882, 65037, 917883, 917884, + 917885, 917886, 917887, 917888, 917889, 917890, 917891, 917892, 65038, + 917893, 917894, 917895, 917896, 917897, 917898, 917899, 917900, 917901, + 917902, 65039, 917903, 917904, 917905, 917906, 917907, 917908, 917909, + 917910, 917911, 917912, 917760, 917913, 917914, 917915, 917916, 917917, + 917918, 917919, 917920, 917921, 917922, 917761, 917923, 917924, 917925, + 917926, 917927, 917928, 917929, 917930, 917931, 917932, 917762, 917933, + 917934, 917935, 917936, 917937, 917938, 917939, 917940, 917941, 917942, + 65025, 917763, 917943, 917944, 917945, 917946, 917947, 917948, 917949, + 917950, 917951, 917952, 917764, 917953, 917954, 917955, 917956, 917957, + 917958, 917959, 917960, 917961, 917962, 917765, 917963, 917964, 917965, + 917966, 917967, 917968, 917969, 917970, 917971, 917972, 917766, 917973, + 917974, 917975, 917976, 917977, 917978, 917979, 917980, 917981, 917982, + 917767, 917983, 917984, 917985, 917986, 917987, 917988, 917989, 917990, + 917991, 917992, 917768, 917993, 917994, 917995, 917996, 917997, 917998, + 917999, 917769, 917770, 917771, 917772, 65026, 917773, 917774, 917775, + 917776, 917777, 917778, 917779, 917780, 917781, 917782, 65027, 917783, + 917784, 917785, 917786, 917787, 917788, 917789, 917790, 917791, 917792, + 65028, 917793, 917794, 917795, 917796, 917797, 917798, 917799, 917800, + 917801, 917802, 65029, 917803, 917804, 917805, 917806, 917807, 917808, + 917809, 917810, 917811, 917812, 65030, 917813, 917814, 917815, 917816, + 917817, 917818, 917819, 917820, 917821, 917822, 65031, 917823, 917824, + 917825, 917826, 917827, 917828, 917829, 917830, 917831, 917832, 65032, + 917833, 917834, 917835, 917836, 917837, 917838, 917839, 917840, 917841, + 917842, 129499, 983245, 983254, 983360, 983361, 983362, 983363, 983364, + 983365, 983366, 983367, 983368, 983369, 983255, 983370, 983371, 983372, + 983373, 983374, 983375, 983376, 983377, 983378, 983379, 983256, 983380, + 983381, 983382, 983383, 983384, 983385, 983386, 983387, 983388, 983389, + 983257, 983390, 983391, 983392, 983393, 983394, 983395, 983396, 983397, + 983398, 983399, 983258, 983400, 983401, 983402, 983403, 983404, 983405, + 983406, 983407, 983408, 983409, 983259, 983410, 983411, 983412, 983413, + 983414, 983415, 983416, 983417, 983418, 983419, 983260, 983420, 983421, + 983422, 983423, 983424, 983425, 983426, 983427, 983428, 983429, 983277, + 983430, 983431, 983432, 983433, 983434, 983435, 983436, 983437, 983438, + 983439, 983278, 983440, 983441, 983442, 983443, 983444, 983445, 983446, + 983447, 983448, 983449, 983279, 983450, 983451, 983452, 983453, 983454, + 983455, 983456, 983457, 983458, 983459, 983246, 983280, 983460, 983461, + 983462, 983463, 983464, 983465, 983466, 983467, 983468, 983469, 983281, + 983470, 983471, 983472, 983473, 983474, 983475, 983476, 983477, 983478, + 983479, 983282, 983480, 983481, 983482, 983483, 983484, 983485, 983486, + 983487, 983488, 983489, 983283, 983490, 983491, 983492, 983493, 983494, + 983495, 983496, 983497, 983498, 983499, 983284, 983500, 983501, 983502, + 983503, 983504, 983505, 983506, 983507, 983508, 983509, 983285, 983510, + 983511, 983512, 983513, 983514, 983515, 983516, 983286, 983287, 983288, + 983289, 983247, 983290, 983291, 983292, 983293, 983294, 983295, 983296, + 983297, 983298, 983299, 983248, 983300, 983301, 983302, 983303, 983304, + 983305, 983306, 983307, 983308, 983309, 983249, 983310, 983311, 983312, + 983313, 983314, 983315, 983316, 983317, 983318, 983319, 983250, 983320, + 983321, 983322, 983323, 983324, 983325, 983326, 983327, 983328, 983329, + 983251, 983330, 983331, 983332, 983333, 983334, 983335, 983336, 983337, + 983338, 983339, 983252, 983340, 983341, 983342, 983343, 983344, 983345, + 983346, 983347, 983348, 983349, 983253, 983350, 983351, 983352, 983353, + 983354, 983355, 983356, 983357, 983358, 983359, 7401, 7402, 7409, 7403, + 7404, 7415, 7410, 7418, 7413, 7379, 7398, 7396, 7411, 7408, 7406, 7407, + 7405, 7414, 7397, 7400, 7395, 7399, 7394, 7380, 7384, 7412, 7417, 7386, + 7389, 7376, 7388, 7378, 7416, 7392, 7391, 7387, 7390, 7381, 7382, 7383, + 7385, 7393, 7377, 8483, 10704, 10980, 10978, 10186, 117780, 8942, 8286, + 117917, 12347, 12337, 12339, 12341, 12338, 12340, 117892, 124, 9168, + 10992, 10991, 117904, 118293, 9087, 9896, 129904, 129905, 129906, 129907, + 129908, 129909, 117916, 117770, 11135, 983069, 983144, 11823, 128678, + 10650, 11837, 128976, 128959, 128947, 128637, 128941, 128953, 128636, 128904, 128914, 128934, 8921, 8920, 128933, 10799, 9910, 128243, 9996, 128249, 127918, 128252, 94193, 94192, 8983, 127931, 9805, 66929, 66930, 66936, 66935, 66942, 66943, 66947, 66946, 66950, 66949, 66932, 66931, @@ -12260,211 +13077,211 @@ static const unsigned int dawg_pos_to_codepoint[] = { 67004, 66977, 66984, 66992, 66967, 66990, 66999, 67003, 129979, 127755, 127952, 8752, 983070, 983145, 11238, 8541, 8538, 8536, 8539, 8533, 189, 8529, 188, 8528, 8537, 8530, 8531, 8540, 190, 8535, 8534, 8532, 8542, - 8585, 123585, 123584, 123624, 123619, 123620, 123606, 123605, 123622, - 123618, 123611, 123623, 123612, 123621, 123615, 123592, 123613, 123625, - 123593, 123596, 123617, 123616, 123595, 123614, 123626, 123627, 123590, - 123609, 123604, 123591, 123594, 123599, 123598, 123586, 123587, 123588, - 123597, 123589, 123610, 123600, 123608, 123607, 123603, 123602, 123601, - 123647, 123630, 123631, 123628, 123629, 123637, 123636, 123639, 123638, - 123635, 123634, 123632, 123641, 123633, 123640, 127768, 127766, 127762, - 127764, 71841, 71850, 71862, 71861, 71848, 71856, 71853, 71867, 71866, - 71865, 71868, 71840, 71859, 71849, 71857, 71869, 71870, 71855, 71847, - 71843, 71854, 71844, 71845, 71858, 71871, 71863, 71864, 71852, 71846, - 71842, 71860, 71851, 71873, 71882, 71894, 71893, 71880, 71888, 71885, - 71899, 71898, 71897, 71900, 71872, 71891, 71881, 71889, 71901, 71902, - 71887, 71879, 71875, 71886, 71876, 71877, 71890, 71903, 71895, 71896, - 71884, 71878, 71874, 71892, 71883, 71909, 71908, 71911, 71910, 71907, - 71906, 71904, 71913, 71905, 71912, 71921, 71918, 71917, 71922, 71920, - 71919, 71916, 71915, 71914, 71935, 9888, 128702, 127754, 128003, 129341, - 127817, 8986, 12316, 11071, 10547, 127988, 127987, 128075, 12336, 65103, - 8967, 65099, 129479, 128465, 128576, 128553, 10172, 128146, 983238, + 8585, 129479, 123585, 123584, 123624, 123619, 123620, 123606, 123605, + 123622, 123618, 123611, 123623, 123612, 123621, 123615, 123592, 123613, + 123625, 123593, 123596, 123617, 123616, 123595, 123614, 123626, 123627, + 123590, 123609, 123604, 123591, 123594, 123599, 123598, 123586, 123587, + 123588, 123597, 123589, 123610, 123600, 123608, 123607, 123603, 123602, + 123601, 123647, 123630, 123631, 123628, 123629, 123637, 123636, 123639, + 123638, 123635, 123634, 123632, 123641, 123633, 123640, 127768, 127766, + 127762, 127764, 71841, 71850, 71862, 71861, 71848, 71856, 71853, 71867, + 71866, 71865, 71868, 71840, 71859, 71849, 71857, 71869, 71870, 71855, + 71847, 71843, 71854, 71844, 71845, 71858, 71871, 71863, 71864, 71852, + 71846, 71842, 71860, 71851, 71873, 71882, 71894, 71893, 71880, 71888, + 71885, 71899, 71898, 71897, 71900, 71872, 71891, 71881, 71889, 71901, + 71902, 71887, 71879, 71875, 71886, 71876, 71877, 71890, 71903, 71895, + 71896, 71884, 71878, 71874, 71892, 71883, 71909, 71908, 71911, 71910, + 71907, 71906, 71904, 71913, 71905, 71912, 71921, 71918, 71917, 71922, + 71920, 71919, 71916, 71915, 71914, 71935, 9888, 128702, 127754, 128003, + 129341, 127817, 8986, 12316, 11071, 10547, 127988, 127987, 128075, 12336, + 65103, 8967, 65099, 128465, 128576, 128553, 10172, 128146, 983238, 127947, 9840, 128734, 9784, 9855, 129197, 129196, 9702, 9815, 129548, 129590, 129608, 129611, 9812, 129545, 129587, 9816, 129542, 129563, 129584, 129591, 129605, 129549, 129616, 129614, 129615, 9817, 129550, 129592, 9813, 129546, 129588, 9814, 129547, 129589, 129569, 129566, 129570, 129571, 129567, 129568, 9675, 128906, 9862, 10732, 9863, 9717, - 9718, 9716, 9719, 9831, 10209, 10210, 10211, 9671, 9672, 128922, 128923, - 10192, 9826, 9931, 128071, 128407, 9759, 9663, 9661, 9920, 9921, 10069, - 9872, 9983, 10048, 128174, 11214, 10023, 9785, 128427, 129293, 9825, - 9989, 129984, 11041, 11053, 10710, 11036, 128326, 9945, 128072, 9756, - 9669, 9667, 9665, 128928, 11046, 11088, 9725, 9723, 11048, 11229, 10001, - 9649, 11040, 127985, 10068, 11092, 9659, 9657, 9655, 128073, 9758, 9645, - 9988, 65094, 9750, 11051, 11090, 9643, 9786, 9828, 9633, 128307, 128916, - 9635, 128917, 10212, 9713, 9714, 10213, 9634, 9712, 9715, 9707, 9093, - 127779, 127781, 127782, 9788, 127780, 9734, 128382, 9743, 9186, 10177, - 9943, 128070, 9757, 129994, 9653, 9651, 9708, 11006, 11055, 9647, 11038, - 10163, 128011, 129144, 129152, 129120, 129136, 129128, 129146, 129154, - 129122, 129138, 129130, 129147, 129155, 129123, 129139, 129131, 129145, - 129153, 129121, 129137, 129129, 129149, 129157, 129125, 129141, 129133, - 129148, 129156, 129124, 129140, 129132, 129150, 129158, 129126, 129142, - 129134, 129151, 129159, 129127, 129143, 129135, 11838, 129344, 127888, - 127788, 129695, 127863, 128521, 129725, 128430, 128732, 128105, 128111, - 128098, 128090, 128082, 128097, 128698, 11825, 8288, 128506, 128543, - 129713, 8361, 129717, 128058, 127873, 8768, 129340, 128295, 9997, 983233, - 8999, 129659, 129644, 129643, 129641, 129639, 129642, 129645, 129640, - 129637, 129636, 129634, 129632, 129635, 129638, 129633, 8891, 129393, - 128155, 165, 69292, 69291, 69293, 69256, 69255, 69254, 69281, 69268, - 69259, 69248, 69271, 69289, 69286, 69287, 69257, 69278, 69277, 69279, - 69276, 69280, 69296, 69282, 69251, 69250, 69266, 69265, 69267, 69253, - 69252, 69269, 69274, 69275, 69284, 69285, 69272, 69258, 69288, 69297, - 69263, 69260, 69270, 69262, 69261, 69249, 69283, 69273, 69264, 42139, - 42149, 42173, 42172, 42132, 42147, 42179, 42174, 42148, 42169, 42150, - 42134, 42166, 42135, 42145, 42143, 42137, 42175, 42157, 42154, 42167, - 42165, 42163, 42130, 42182, 42129, 42171, 42138, 42140, 42136, 42131, - 42151, 42164, 42181, 42142, 42156, 42170, 42176, 42178, 42160, 42133, - 42144, 42152, 42159, 42161, 42158, 42141, 42146, 42177, 42180, 42155, - 42162, 42128, 42168, 42153, 41070, 41059, 41060, 41058, 41073, 41072, - 41071, 41068, 41069, 41066, 41067, 41065, 41048, 41052, 41053, 41050, - 41051, 41049, 41046, 41047, 41056, 41057, 41054, 41055, 41063, 41064, - 41061, 41062, 41076, 41077, 41074, 41075, 41006, 40995, 40996, 40994, - 41009, 41008, 41007, 41004, 41005, 41002, 41003, 41001, 40984, 40988, - 40989, 40986, 40987, 40985, 40982, 40983, 40992, 40993, 40990, 40991, - 40999, 41000, 40997, 40998, 41012, 41015, 41014, 41013, 41010, 41011, - 41842, 41831, 41832, 41829, 41830, 41845, 41844, 41843, 41841, 41827, - 41828, 41825, 41826, 41839, 41840, 41837, 41838, 41835, 41836, 41833, - 41834, 41848, 41851, 41850, 41849, 41846, 41847, 41670, 41659, 41660, - 41658, 41673, 41672, 41671, 41668, 41669, 41666, 41667, 41665, 41648, - 41652, 41653, 41650, 41651, 41649, 41646, 41647, 41656, 41657, 41654, - 41655, 41663, 41664, 41661, 41662, 41676, 41679, 41678, 41677, 41674, - 41675, 41293, 41282, 41283, 41281, 41296, 41295, 41294, 41291, 41292, - 41272, 41275, 41276, 41274, 41273, 41270, 41271, 41289, 41290, 41288, - 41279, 41280, 41277, 41278, 41286, 41287, 41284, 41285, 41238, 41228, - 41227, 41241, 41240, 41239, 41236, 41237, 41218, 41221, 41222, 41220, - 41219, 41216, 41217, 41234, 41235, 41233, 41225, 41226, 41223, 41224, - 41231, 41232, 41229, 41230, 41174, 41175, 41173, 41171, 41172, 41169, - 41170, 41167, 41168, 41165, 41166, 41184, 41185, 41182, 41183, 41178, - 41181, 41180, 41179, 41176, 41177, 41494, 41496, 41497, 41495, 41492, - 41493, 41516, 41504, 41505, 41502, 41503, 41519, 41518, 41517, 41514, - 41515, 41500, 41501, 41498, 41499, 41512, 41513, 41510, 41511, 41508, - 41509, 41506, 41507, 41460, 41448, 41449, 41446, 41447, 41463, 41462, - 41461, 41458, 41459, 41436, 41440, 41441, 41438, 41439, 41437, 41434, - 41435, 41444, 41445, 41442, 41443, 41456, 41457, 41454, 41455, 41452, - 41453, 41450, 41451, 41584, 41583, 41582, 41389, 41379, 41380, 41378, - 41392, 41391, 41390, 41387, 41388, 41369, 41372, 41373, 41371, 41370, - 41367, 41368, 41385, 41386, 41384, 41382, 41383, 41381, 41376, 41377, - 41374, 41375, 41395, 41398, 41397, 41396, 41393, 41394, 41125, 41117, - 41118, 41116, 41128, 41127, 41126, 41123, 41124, 41107, 41110, 41111, - 41109, 41108, 41105, 41106, 41114, 41115, 41112, 41113, 41121, 41122, - 41119, 41120, 41130, 41133, 41132, 41131, 41129, 41336, 41334, 41335, - 41333, 41332, 41340, 41338, 41339, 41337, 41322, 41326, 41327, 41324, - 41325, 41323, 41320, 41321, 41330, 41331, 41328, 41329, 41556, 41557, - 41554, 41555, 41563, 41564, 41562, 41544, 41548, 41549, 41546, 41547, - 41545, 41542, 41543, 41552, 41553, 41550, 41551, 41560, 41561, 41558, - 41559, 41591, 41592, 41589, 41590, 41598, 41599, 41597, 41587, 41588, - 41585, 41586, 41595, 41596, 41593, 41594, 40962, 40960, 983243, 40966, - 40967, 40964, 40965, 40963, 40961, 42025, 42017, 42018, 42016, 42028, - 42027, 42026, 42023, 42024, 42010, 42014, 42015, 42012, 42013, 42011, - 42008, 42009, 42021, 42022, 42019, 42020, 42031, 42032, 42029, 42030, - 41970, 41962, 41963, 41960, 41961, 41973, 41972, 41971, 41968, 41969, - 41954, 41958, 41959, 41956, 41957, 41955, 41952, 41953, 41966, 41967, - 41964, 41965, 41976, 41979, 41978, 41977, 41974, 41975, 41488, 41476, - 41477, 41475, 41491, 41490, 41489, 41486, 41487, 41466, 41469, 41470, - 41468, 41467, 41464, 41465, 41473, 41474, 41471, 41472, 41484, 41485, - 41482, 41483, 41480, 41481, 41478, 41479, 41424, 41413, 41414, 41411, - 41412, 41427, 41426, 41425, 41422, 41423, 41420, 41421, 41419, 41401, - 41405, 41406, 41403, 41404, 41402, 41399, 41400, 41409, 41410, 41407, - 41408, 41417, 41418, 41415, 41416, 41430, 41433, 41432, 41431, 41428, - 41429, 41538, 41527, 41528, 41526, 41541, 41540, 41539, 41536, 41537, - 41534, 41535, 41533, 41524, 41525, 41522, 41523, 41531, 41532, 41529, - 41530, 41521, 41520, 41157, 41147, 41148, 41145, 41146, 41160, 41159, - 41158, 41155, 41156, 41136, 41139, 41140, 41138, 41137, 41134, 41135, - 41143, 41144, 41141, 41142, 41151, 41152, 41149, 41150, 41163, 41164, - 41161, 41162, 41154, 41153, 41080, 41083, 41084, 41082, 41081, 41078, - 41079, 41087, 41088, 41085, 41086, 41091, 41092, 41089, 41090, 41095, - 41098, 41097, 41096, 41093, 41094, 41101, 41104, 41103, 41102, 41099, - 41100, 41299, 41302, 41301, 41300, 41297, 41298, 41312, 41313, 41311, - 41305, 41306, 41303, 41304, 41309, 41310, 41307, 41308, 41316, 41319, - 41318, 41317, 41314, 41315, 41574, 41572, 41573, 41580, 41581, 41579, - 41566, 41567, 41565, 41570, 41571, 41568, 41569, 41577, 41578, 41575, - 41576, 42048, 42042, 42041, 42051, 42050, 42049, 42047, 42035, 42039, - 42040, 42037, 42038, 42036, 42033, 42034, 42045, 42046, 42043, 42044, - 42054, 42057, 42056, 42055, 42052, 42053, 41881, 41882, 41880, 41878, - 41879, 41876, 41877, 41885, 41886, 41883, 41884, 41889, 41892, 41891, - 41890, 41887, 41888, 41895, 41898, 41897, 41896, 41893, 41894, 42075, - 42067, 42068, 42066, 42076, 42073, 42074, 42060, 42064, 42065, 42062, - 42063, 42061, 42058, 42059, 42071, 42072, 42069, 42070, 41727, 41721, - 41720, 41730, 41729, 41728, 41726, 41723, 41722, 41711, 41714, 41715, - 41713, 41712, 41709, 41710, 41718, 41719, 41716, 41717, 41733, 41736, - 41735, 41734, 41731, 41732, 41725, 41724, 41363, 41352, 41353, 41351, - 41366, 41365, 41364, 41361, 41362, 41343, 41346, 41347, 41345, 41344, - 41341, 41342, 41349, 41350, 41348, 41359, 41360, 41358, 41356, 41357, - 41354, 41355, 41036, 41028, 41029, 41027, 41039, 41038, 41037, 41034, - 41035, 41018, 41021, 41022, 41020, 41019, 41016, 41017, 41025, 41026, - 41023, 41024, 41032, 41033, 41030, 41031, 41042, 41045, 41044, 41043, - 41040, 41041, 41998, 41990, 41991, 41988, 41989, 42001, 42000, 41999, - 41996, 41997, 41982, 41986, 41987, 41984, 41985, 41983, 41980, 41981, - 41994, 41995, 41992, 41993, 42004, 42007, 42006, 42005, 42002, 42003, - 42115, 42107, 42108, 42105, 42106, 42118, 42117, 42116, 42113, 42114, - 42099, 42103, 42104, 42101, 42102, 42100, 42097, 42098, 42111, 42112, - 42109, 42110, 42121, 42124, 42123, 42122, 42119, 42120, 41866, 41855, - 41854, 41869, 41868, 41867, 41864, 41865, 41862, 41863, 41860, 41861, - 41858, 41859, 41856, 41857, 41872, 41875, 41874, 41873, 41870, 41871, - 41853, 41852, 41942, 41931, 41932, 41930, 41945, 41944, 41943, 41940, - 41941, 41938, 41939, 41937, 41928, 41929, 41926, 41927, 41935, 41936, - 41933, 41934, 41948, 41951, 41950, 41949, 41946, 41947, 41772, 41775, - 41776, 41774, 41773, 41770, 41771, 41786, 41787, 41785, 41779, 41780, - 41777, 41778, 41783, 41784, 41781, 41782, 41790, 41791, 41788, 41789, - 41794, 41797, 41796, 41795, 41792, 41793, 41916, 41904, 41905, 41903, - 41919, 41918, 41917, 41914, 41915, 41901, 41902, 41899, 41900, 41912, - 41913, 41910, 41911, 41908, 41909, 41906, 41907, 41922, 41925, 41924, - 41923, 41920, 41921, 41760, 41749, 41750, 41748, 41763, 41762, 41761, - 41758, 41759, 41739, 41742, 41743, 41741, 41740, 41737, 41738, 41756, - 41757, 41755, 41746, 41747, 41744, 41745, 41753, 41754, 41751, 41752, - 41766, 41769, 41768, 41767, 41764, 41765, 41266, 41255, 41256, 41253, - 41254, 41269, 41268, 41267, 41264, 41265, 41244, 41247, 41248, 41246, - 41245, 41242, 41243, 41262, 41263, 41261, 41251, 41252, 41249, 41250, - 41259, 41260, 41257, 41258, 41203, 41202, 41188, 41192, 41193, 41190, - 41191, 41189, 41186, 41187, 41196, 41197, 41194, 41195, 41200, 41201, - 41198, 41199, 41206, 41209, 41208, 41207, 41204, 41205, 41212, 41215, - 41214, 41213, 41210, 41211, 40981, 41605, 41606, 41604, 41611, 41612, - 41610, 41608, 41609, 41607, 41602, 41603, 41600, 41601, 42079, 42083, - 42084, 42081, 42082, 42080, 42077, 42078, 42089, 42090, 42087, 42088, - 42093, 42096, 42095, 42094, 42091, 42092, 42086, 42085, 41815, 41803, - 41804, 41802, 41818, 41817, 41816, 41813, 41814, 41800, 41801, 41798, - 41799, 41811, 41812, 41809, 41810, 41807, 41808, 41805, 41806, 41821, - 41824, 41823, 41822, 41819, 41820, 41636, 41625, 41626, 41624, 41639, - 41638, 41637, 41634, 41635, 41615, 41618, 41619, 41617, 41616, 41613, - 41614, 41696, 41697, 41695, 41693, 41694, 41692, 41682, 41686, 41687, - 41684, 41685, 41683, 41680, 41681, 41690, 41691, 41688, 41689, 41699, - 41702, 41701, 41700, 41698, 41705, 41708, 41707, 41706, 41703, 41704, - 41632, 41633, 41631, 41622, 41623, 41620, 41621, 41629, 41630, 41627, - 41628, 41642, 41645, 41644, 41643, 41640, 41641, 40973, 40974, 40972, - 40970, 40971, 40968, 40969, 40977, 40978, 40975, 40976, 40980, 40979, - 9775, 129664, 8959, 10853, 10632, 10634, 10814, 10852, 10631, 10633, - 10783, 10784, 10785, 10625, 10626, 72262, 72256, 72253, 72252, 72254, - 72251, 72250, 72261, 72255, 72243, 72215, 72214, 72230, 72229, 72220, - 72219, 72242, 72204, 72203, 72207, 72216, 72211, 72221, 72238, 72239, - 72240, 72228, 72227, 72213, 72212, 72218, 72217, 72225, 72224, 72209, - 72208, 72206, 72205, 72223, 72222, 72231, 72232, 72233, 72241, 72210, - 72236, 72226, 72235, 72237, 72234, 72192, 72259, 72258, 72260, 72257, - 72248, 72245, 72246, 72247, 72244, 72249, 72263, 72202, 72199, 72200, - 72198, 72197, 72195, 72194, 72201, 72196, 72193, 129427, 65279, 8204, - 8203, 8205, 11234, 129296, 118593, 118584, 118585, 118591, 118580, - 118589, 118528, 118540, 118531, 118543, 118559, 118529, 118541, 118532, - 118544, 118592, 118573, 118569, 118568, 118581, 118586, 118561, 118582, - 118583, 118566, 118538, 118550, 118555, 118556, 118535, 118547, 118537, - 118549, 118553, 118558, 118534, 118546, 118572, 118562, 118571, 118554, - 118533, 118545, 118587, 118588, 118530, 118542, 118552, 118563, 118539, - 118551, 118557, 118536, 118548, 118579, 118570, 118560, 118567, 118565, - 118564, 118590, 118576, 118577, 118578, 118619, 118639, 118637, 118645, - 118721, 118611, 118635, 118659, 118626, 118623, 118625, 118624, 118622, - 118638, 118612, 118660, 118608, 118609, 118657, 118719, 118695, 118699, - 118698, 118697, 118696, 118722, 118720, 118703, 118708, 118707, 118706, - 118705, 118704, 118610, 118620, 118723, 118616, 118617, 118640, 118672, - 118636, 118658, 118652, 118651, 118649, 118650, 118648, 118646, 118647, - 118644, 118643, 118641, 118642, 118653, 118656, 118654, 118655, 118662, - 118670, 118664, 118666, 118669, 118663, 118665, 118671, 118667, 118668, - 118673, 118614, 118615, 118618, 118685, 118687, 118684, 118683, 118686, - 118681, 118680, 118709, 118713, 118711, 118716, 118717, 118714, 118715, - 118712, 118718, 118710, 118676, 118679, 118690, 118688, 118693, 118694, - 118691, 118692, 118689, 118675, 118677, 118678, 118674, 118701, 118702, - 118700, 118682, 118632, 118631, 118634, 118633, 118628, 118627, 118630, - 118629, 118613, 118621, 118661, 118596, 118597, 118594, 118595, 118598, - 129503, 983264, 983222, 983221, 983223, + 9718, 9716, 9719, 9831, 10209, 10210, 10211, 129995, 9671, 9672, 128922, + 128923, 10192, 9826, 9931, 128071, 128407, 9759, 9663, 9661, 9920, 9921, + 10069, 9872, 9983, 10048, 128174, 11214, 10023, 9785, 128427, 129293, + 9825, 9989, 129984, 11041, 11053, 10710, 11036, 128326, 9945, 128072, + 9756, 9669, 9667, 9665, 117894, 117895, 128928, 11046, 11088, 9725, 9723, + 11048, 11229, 10001, 9649, 11040, 127985, 10068, 9645, 11092, 9659, 9657, + 9655, 128073, 9758, 9988, 65094, 9750, 11051, 11090, 9643, 9786, 9828, + 9633, 128307, 128916, 9635, 128917, 10212, 9713, 9714, 10213, 9634, 9712, + 9715, 9707, 9093, 127779, 127781, 127782, 9788, 127780, 9734, 128382, + 9743, 9186, 10177, 9943, 128070, 9757, 129994, 9653, 9651, 9708, 11055, + 9647, 118278, 11006, 11038, 10163, 128011, 129144, 129152, 129120, + 129136, 129128, 129146, 129154, 129122, 129138, 129130, 129147, 129155, + 129123, 129139, 129131, 129145, 129153, 129121, 129137, 129129, 129149, + 129157, 129125, 129141, 129133, 129148, 129156, 129124, 129140, 129132, + 129150, 129158, 129126, 129142, 129134, 129151, 129159, 129127, 129143, + 129135, 11838, 129344, 127888, 127788, 129695, 127863, 128521, 129725, + 128430, 128732, 128105, 128111, 128098, 128090, 128082, 128097, 128698, + 11825, 8288, 128506, 128543, 129713, 8361, 129717, 128058, 127873, 8768, + 129340, 128295, 9997, 983233, 8999, 129659, 129644, 129643, 129641, + 129639, 129642, 129645, 129640, 129637, 129636, 129634, 129632, 129635, + 129638, 129633, 8891, 129393, 128155, 165, 69292, 69291, 69293, 69256, + 69255, 69254, 69281, 69268, 69259, 69248, 69271, 69289, 69286, 69287, + 69257, 69278, 69277, 69279, 69276, 69280, 69296, 69282, 69251, 69250, + 69266, 69265, 69267, 69253, 69252, 69269, 69274, 69275, 69284, 69285, + 69272, 69258, 69288, 69297, 69263, 69260, 69270, 69262, 69261, 69249, + 69283, 69273, 69264, 42139, 42149, 42173, 42172, 42132, 42147, 42179, + 42174, 42148, 42169, 42150, 42134, 42166, 42135, 42145, 42143, 42137, + 42175, 42157, 42154, 42167, 42165, 42163, 42130, 42182, 42129, 42171, + 42138, 42140, 42136, 42131, 42151, 42164, 42181, 42142, 42156, 42170, + 42176, 42178, 42160, 42133, 42144, 42152, 42159, 42161, 42158, 42141, + 42146, 42177, 42180, 42155, 42162, 42128, 42168, 42153, 41070, 41059, + 41060, 41058, 41073, 41072, 41071, 41068, 41069, 41066, 41067, 41065, + 41048, 41052, 41053, 41050, 41051, 41049, 41046, 41047, 41056, 41057, + 41054, 41055, 41063, 41064, 41061, 41062, 41076, 41077, 41074, 41075, + 41006, 40995, 40996, 40994, 41009, 41008, 41007, 41004, 41005, 41002, + 41003, 41001, 40984, 40988, 40989, 40986, 40987, 40985, 40982, 40983, + 40992, 40993, 40990, 40991, 40999, 41000, 40997, 40998, 41012, 41015, + 41014, 41013, 41010, 41011, 41842, 41831, 41832, 41829, 41830, 41845, + 41844, 41843, 41841, 41827, 41828, 41825, 41826, 41839, 41840, 41837, + 41838, 41835, 41836, 41833, 41834, 41848, 41851, 41850, 41849, 41846, + 41847, 41670, 41659, 41660, 41658, 41673, 41672, 41671, 41668, 41669, + 41666, 41667, 41665, 41648, 41652, 41653, 41650, 41651, 41649, 41646, + 41647, 41656, 41657, 41654, 41655, 41663, 41664, 41661, 41662, 41676, + 41679, 41678, 41677, 41674, 41675, 41293, 41282, 41283, 41281, 41296, + 41295, 41294, 41291, 41292, 41272, 41275, 41276, 41274, 41273, 41270, + 41271, 41289, 41290, 41288, 41279, 41280, 41277, 41278, 41286, 41287, + 41284, 41285, 41238, 41228, 41227, 41241, 41240, 41239, 41236, 41237, + 41218, 41221, 41222, 41220, 41219, 41216, 41217, 41234, 41235, 41233, + 41225, 41226, 41223, 41224, 41231, 41232, 41229, 41230, 41174, 41175, + 41173, 41171, 41172, 41169, 41170, 41167, 41168, 41165, 41166, 41184, + 41185, 41182, 41183, 41178, 41181, 41180, 41179, 41176, 41177, 41494, + 41496, 41497, 41495, 41492, 41493, 41516, 41504, 41505, 41502, 41503, + 41519, 41518, 41517, 41514, 41515, 41500, 41501, 41498, 41499, 41512, + 41513, 41510, 41511, 41508, 41509, 41506, 41507, 41460, 41448, 41449, + 41446, 41447, 41463, 41462, 41461, 41458, 41459, 41436, 41440, 41441, + 41438, 41439, 41437, 41434, 41435, 41444, 41445, 41442, 41443, 41456, + 41457, 41454, 41455, 41452, 41453, 41450, 41451, 41584, 41583, 41582, + 41389, 41379, 41380, 41378, 41392, 41391, 41390, 41387, 41388, 41369, + 41372, 41373, 41371, 41370, 41367, 41368, 41385, 41386, 41384, 41382, + 41383, 41381, 41376, 41377, 41374, 41375, 41395, 41398, 41397, 41396, + 41393, 41394, 41125, 41117, 41118, 41116, 41128, 41127, 41126, 41123, + 41124, 41107, 41110, 41111, 41109, 41108, 41105, 41106, 41114, 41115, + 41112, 41113, 41121, 41122, 41119, 41120, 41130, 41133, 41132, 41131, + 41129, 41336, 41334, 41335, 41333, 41332, 41340, 41338, 41339, 41337, + 41322, 41326, 41327, 41324, 41325, 41323, 41320, 41321, 41330, 41331, + 41328, 41329, 41556, 41557, 41554, 41555, 41563, 41564, 41562, 41544, + 41548, 41549, 41546, 41547, 41545, 41542, 41543, 41552, 41553, 41550, + 41551, 41560, 41561, 41558, 41559, 41591, 41592, 41589, 41590, 41598, + 41599, 41597, 41587, 41588, 41585, 41586, 41595, 41596, 41593, 41594, + 40962, 40960, 983243, 40966, 40967, 40964, 40965, 40963, 40961, 42025, + 42017, 42018, 42016, 42028, 42027, 42026, 42023, 42024, 42010, 42014, + 42015, 42012, 42013, 42011, 42008, 42009, 42021, 42022, 42019, 42020, + 42031, 42032, 42029, 42030, 41970, 41962, 41963, 41960, 41961, 41973, + 41972, 41971, 41968, 41969, 41954, 41958, 41959, 41956, 41957, 41955, + 41952, 41953, 41966, 41967, 41964, 41965, 41976, 41979, 41978, 41977, + 41974, 41975, 41488, 41476, 41477, 41475, 41491, 41490, 41489, 41486, + 41487, 41466, 41469, 41470, 41468, 41467, 41464, 41465, 41473, 41474, + 41471, 41472, 41484, 41485, 41482, 41483, 41480, 41481, 41478, 41479, + 41424, 41413, 41414, 41411, 41412, 41427, 41426, 41425, 41422, 41423, + 41420, 41421, 41419, 41401, 41405, 41406, 41403, 41404, 41402, 41399, + 41400, 41409, 41410, 41407, 41408, 41417, 41418, 41415, 41416, 41430, + 41433, 41432, 41431, 41428, 41429, 41538, 41527, 41528, 41526, 41541, + 41540, 41539, 41536, 41537, 41534, 41535, 41533, 41524, 41525, 41522, + 41523, 41531, 41532, 41529, 41530, 41521, 41520, 41157, 41147, 41148, + 41145, 41146, 41160, 41159, 41158, 41155, 41156, 41136, 41139, 41140, + 41138, 41137, 41134, 41135, 41143, 41144, 41141, 41142, 41151, 41152, + 41149, 41150, 41163, 41164, 41161, 41162, 41154, 41153, 41080, 41083, + 41084, 41082, 41081, 41078, 41079, 41087, 41088, 41085, 41086, 41091, + 41092, 41089, 41090, 41095, 41098, 41097, 41096, 41093, 41094, 41101, + 41104, 41103, 41102, 41099, 41100, 41299, 41302, 41301, 41300, 41297, + 41298, 41312, 41313, 41311, 41305, 41306, 41303, 41304, 41309, 41310, + 41307, 41308, 41316, 41319, 41318, 41317, 41314, 41315, 41574, 41572, + 41573, 41580, 41581, 41579, 41566, 41567, 41565, 41570, 41571, 41568, + 41569, 41577, 41578, 41575, 41576, 42048, 42042, 42041, 42051, 42050, + 42049, 42047, 42035, 42039, 42040, 42037, 42038, 42036, 42033, 42034, + 42045, 42046, 42043, 42044, 42054, 42057, 42056, 42055, 42052, 42053, + 41881, 41882, 41880, 41878, 41879, 41876, 41877, 41885, 41886, 41883, + 41884, 41889, 41892, 41891, 41890, 41887, 41888, 41895, 41898, 41897, + 41896, 41893, 41894, 42075, 42067, 42068, 42066, 42076, 42073, 42074, + 42060, 42064, 42065, 42062, 42063, 42061, 42058, 42059, 42071, 42072, + 42069, 42070, 41727, 41721, 41720, 41730, 41729, 41728, 41726, 41723, + 41722, 41711, 41714, 41715, 41713, 41712, 41709, 41710, 41718, 41719, + 41716, 41717, 41733, 41736, 41735, 41734, 41731, 41732, 41725, 41724, + 41363, 41352, 41353, 41351, 41366, 41365, 41364, 41361, 41362, 41343, + 41346, 41347, 41345, 41344, 41341, 41342, 41349, 41350, 41348, 41359, + 41360, 41358, 41356, 41357, 41354, 41355, 41036, 41028, 41029, 41027, + 41039, 41038, 41037, 41034, 41035, 41018, 41021, 41022, 41020, 41019, + 41016, 41017, 41025, 41026, 41023, 41024, 41032, 41033, 41030, 41031, + 41042, 41045, 41044, 41043, 41040, 41041, 41998, 41990, 41991, 41988, + 41989, 42001, 42000, 41999, 41996, 41997, 41982, 41986, 41987, 41984, + 41985, 41983, 41980, 41981, 41994, 41995, 41992, 41993, 42004, 42007, + 42006, 42005, 42002, 42003, 42115, 42107, 42108, 42105, 42106, 42118, + 42117, 42116, 42113, 42114, 42099, 42103, 42104, 42101, 42102, 42100, + 42097, 42098, 42111, 42112, 42109, 42110, 42121, 42124, 42123, 42122, + 42119, 42120, 41866, 41855, 41854, 41869, 41868, 41867, 41864, 41865, + 41862, 41863, 41860, 41861, 41858, 41859, 41856, 41857, 41872, 41875, + 41874, 41873, 41870, 41871, 41853, 41852, 41942, 41931, 41932, 41930, + 41945, 41944, 41943, 41940, 41941, 41938, 41939, 41937, 41928, 41929, + 41926, 41927, 41935, 41936, 41933, 41934, 41948, 41951, 41950, 41949, + 41946, 41947, 41772, 41775, 41776, 41774, 41773, 41770, 41771, 41786, + 41787, 41785, 41779, 41780, 41777, 41778, 41783, 41784, 41781, 41782, + 41790, 41791, 41788, 41789, 41794, 41797, 41796, 41795, 41792, 41793, + 41916, 41904, 41905, 41903, 41919, 41918, 41917, 41914, 41915, 41901, + 41902, 41899, 41900, 41912, 41913, 41910, 41911, 41908, 41909, 41906, + 41907, 41922, 41925, 41924, 41923, 41920, 41921, 41760, 41749, 41750, + 41748, 41763, 41762, 41761, 41758, 41759, 41739, 41742, 41743, 41741, + 41740, 41737, 41738, 41756, 41757, 41755, 41746, 41747, 41744, 41745, + 41753, 41754, 41751, 41752, 41766, 41769, 41768, 41767, 41764, 41765, + 41266, 41255, 41256, 41253, 41254, 41269, 41268, 41267, 41264, 41265, + 41244, 41247, 41248, 41246, 41245, 41242, 41243, 41262, 41263, 41261, + 41251, 41252, 41249, 41250, 41259, 41260, 41257, 41258, 41203, 41202, + 41188, 41192, 41193, 41190, 41191, 41189, 41186, 41187, 41196, 41197, + 41194, 41195, 41200, 41201, 41198, 41199, 41206, 41209, 41208, 41207, + 41204, 41205, 41212, 41215, 41214, 41213, 41210, 41211, 40981, 41605, + 41606, 41604, 41611, 41612, 41610, 41608, 41609, 41607, 41602, 41603, + 41600, 41601, 42079, 42083, 42084, 42081, 42082, 42080, 42077, 42078, + 42089, 42090, 42087, 42088, 42093, 42096, 42095, 42094, 42091, 42092, + 42086, 42085, 41815, 41803, 41804, 41802, 41818, 41817, 41816, 41813, + 41814, 41800, 41801, 41798, 41799, 41811, 41812, 41809, 41810, 41807, + 41808, 41805, 41806, 41821, 41824, 41823, 41822, 41819, 41820, 41636, + 41625, 41626, 41624, 41639, 41638, 41637, 41634, 41635, 41615, 41618, + 41619, 41617, 41616, 41613, 41614, 41696, 41697, 41695, 41693, 41694, + 41692, 41682, 41686, 41687, 41684, 41685, 41683, 41680, 41681, 41690, + 41691, 41688, 41689, 41699, 41702, 41701, 41700, 41698, 41705, 41708, + 41707, 41706, 41703, 41704, 41632, 41633, 41631, 41622, 41623, 41620, + 41621, 41629, 41630, 41627, 41628, 41642, 41645, 41644, 41643, 41640, + 41641, 40973, 40974, 40972, 40970, 40971, 40968, 40969, 40977, 40978, + 40975, 40976, 40980, 40979, 9775, 129664, 8959, 10853, 10632, 10634, + 10814, 10852, 10631, 10633, 10783, 10784, 10785, 10625, 10626, 72262, + 72256, 72253, 72252, 72254, 72251, 72250, 72261, 72255, 72243, 72215, + 72214, 72230, 72229, 72220, 72219, 72242, 72204, 72203, 72207, 72216, + 72211, 72221, 72238, 72239, 72240, 72228, 72227, 72213, 72212, 72218, + 72217, 72225, 72224, 72209, 72208, 72206, 72205, 72223, 72222, 72231, + 72232, 72233, 72241, 72210, 72236, 72226, 72235, 72237, 72234, 72192, + 72259, 72258, 72260, 72257, 72248, 72245, 72246, 72247, 72244, 72249, + 72263, 72202, 72199, 72200, 72198, 72197, 72195, 72194, 72201, 72196, + 72193, 129427, 65279, 8204, 8203, 8205, 11234, 129296, 118593, 118584, + 118585, 118591, 118580, 118589, 118528, 118540, 118531, 118543, 118559, + 118529, 118541, 118532, 118544, 118592, 118573, 118569, 118568, 118581, + 118586, 118561, 118582, 118583, 118566, 118538, 118550, 118555, 118556, + 118535, 118547, 118537, 118549, 118553, 118558, 118534, 118546, 118572, + 118562, 118571, 118554, 118533, 118545, 118587, 118588, 118530, 118542, + 118552, 118563, 118539, 118551, 118557, 118536, 118548, 118579, 118570, + 118560, 118567, 118565, 118564, 118590, 118576, 118577, 118578, 118619, + 118639, 118637, 118645, 118721, 118611, 118635, 118659, 118626, 118623, + 118625, 118624, 118622, 118638, 118612, 118660, 118608, 118609, 118657, + 118719, 118695, 118699, 118698, 118697, 118696, 118722, 118720, 118703, + 118708, 118707, 118706, 118705, 118704, 118610, 118620, 118723, 118616, + 118617, 118640, 118672, 118636, 118658, 118652, 118651, 118649, 118650, + 118648, 118646, 118647, 118644, 118643, 118641, 118642, 118653, 118656, + 118654, 118655, 118662, 118670, 118664, 118666, 118669, 118663, 118665, + 118671, 118667, 118668, 118673, 118614, 118615, 118618, 118685, 118687, + 118684, 118683, 118686, 118681, 118680, 118709, 118713, 118711, 118716, + 118717, 118714, 118715, 118712, 118718, 118710, 118676, 118679, 118690, + 118688, 118693, 118694, 118691, 118692, 118689, 118675, 118677, 118678, + 118674, 118701, 118702, 118700, 118682, 118632, 118631, 118634, 118633, + 118628, 118627, 118630, 118629, 118613, 118621, 118661, 118596, 118597, + 118594, 118595, 118598, 129503, 983264, 983222, 983221, 983223, }; #define DAWG_CODEPOINT_TO_POS_SHIFT 8 -#define DAWG_CODEPOINT_TO_POS_NOTFOUND 35762 +#define DAWG_CODEPOINT_TO_POS_NOTFOUND 40951 static const unsigned char dawg_codepoint_to_pos_index1[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, @@ -12483,18 +13300,20 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 52, 52, 52, 52, 52, 52, 52, 52, 52, 112, 113, - 114, 115, 116, 117, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 118, 119, 120, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 136, 52, 52, 52, 52, 52, 52, 137, 138, 139, 140, 52, 141, 142, 143, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 121, 122, 123, 124, 52, 52, 125, 126, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 144, 145, 146, 147, 148, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 127, 128, 129, 130, 131, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 149, 150, 151, 152, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 153, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, + 166, 167, 168, 52, 52, 52, 52, 169, 170, 171, 172, 52, 173, 174, 52, 175, + 176, 177, 52, 52, 178, 179, 180, 52, 181, 182, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 132, 133, 134, 135, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 136, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 52, 52, - 52, 52, 149, 150, 151, 152, 52, 153, 52, 52, 154, 155, 156, 52, 52, 157, - 158, 159, 52, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -12507,9 +13326,9 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 193, 194, 195, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 172, 173, 174, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -12664,9 +13483,8 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 196, 197, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 175, - 176, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -12679,8 +13497,8 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 198, 199, 200, 201, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 177, 178, 179, 180, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -12708,4467 +13526,5003 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, }; static const unsigned short dawg_codepoint_to_pos_index2[] = { - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 29174, 13037, 26950, 24736, 10779, 26523, 358, 1041, 19550, - 27163, 2511, 26774, 8150, 15996, 13206, 29017, 10563, 10548, 10560, - 10557, 10542, 10539, 10554, 10551, 10566, 10545, 7753, 27749, 19776, - 12475, 13751, 26948, 8149, 18159, 18205, 18215, 18231, 18246, 18291, - 18295, 18312, 18326, 18355, 18360, 18372, 18390, 18399, 18415, 18465, - 18473, 18476, 18496, 18519, 18548, 18587, 18598, 18607, 18610, 18624, - 19518, 27073, 27177, 6518, 20535, 13722, 18716, 18766, 18786, 18809, - 18845, 18904, 18912, 18930, 18949, 18986, 18992, 19006, 19044, 19057, - 19081, 19138, 19149, 19155, 19193, 19234, 19307, 19355, 19369, 19378, - 19386, 19402, 19466, 33652, 27135, 32321, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 24729, - 16349, 5830, 26815, 9647, 34209, 4636, 27731, 10531, 8344, 13111, 19565, - 24711, 28968, 26996, 22169, 10248, 26786, 29740, 29739, 4, 22895, 26671, - 22898, 5826, 29743, 21136, 27214, 33780, 33778, 33786, 16352, 18188, - 18189, 18177, 18191, 18170, 18172, 18193, 18216, 18277, 18278, 18249, - 18264, 18342, 18343, 18332, 18330, 18288, 18413, 18454, 18455, 18422, - 18440, 18429, 23603, 18438, 18579, 18580, 18549, 18552, 18619, 18540, - 19224, 18747, 18748, 18736, 18750, 18727, 18729, 18753, 18787, 18878, - 18879, 18848, 18863, 18967, 18968, 18950, 18952, 18903, 19076, 19120, - 19121, 19082, 19106, 19089, 10690, 19104, 19342, 19345, 19310, 19313, - 19397, 19256, 19388, 18175, 18734, 18160, 18717, 18185, 18744, 18220, - 18792, 18218, 18789, 18224, 18797, 18219, 18791, 18238, 18817, 18235, - 18811, 18267, 18867, 18280, 18881, 18259, 18858, 18274, 18875, 18258, - 18857, 18297, 18914, 18300, 18917, 18304, 18922, 18296, 18913, 18315, - 18933, 18322, 18942, 18336, 18964, 18334, 18959, 18345, 18970, 18339, - 18961, 18329, 18833, 18636, 19422, 18356, 18987, 18361, 18993, 19005, - 18386, 19023, 18382, 19010, 18384, 19013, 18379, 19020, 18381, 19031, - 18408, 19069, 18400, 19058, 18402, 19062, 19077, 18290, 18886, 18418, - 19099, 18457, 19123, 18435, 19095, 18637, 19425, 18485, 19172, 18477, - 19156, 18478, 19158, 18502, 19194, 18501, 19200, 18499, 19198, 18497, - 19196, 18520, 19235, 18523, 19239, 18530, 19251, 18576, 19339, 18568, - 19329, 18582, 19347, 18583, 19338, 18559, 19320, 18573, 19334, 18599, - 19370, 18611, 19387, 18612, 18627, 19403, 18630, 19409, 18626, 19406, - 19036, 18775, 18211, 18206, 18767, 18543, 19259, 18460, 18222, 18795, - 18198, 18243, 18239, 18812, 19264, 18494, 18518, 18459, 18292, 18907, - 18303, 18310, 18948, 18353, 18346, 18371, 19004, 19009, 19032, 18537, - 18406, 19063, 18421, 18447, 19113, 18462, 19135, 18471, 19146, 18713, - 18544, 19260, 18289, 18653, 19247, 18529, 19248, 18528, 18561, 19322, - 18586, 18591, 18616, 19392, 18633, 19412, 18282, 18284, 18896, 18901, - 18706, 18545, 19261, 18652, 18705, 18707, 18708, 18645, 18655, 18245, - 18233, 18839, 18389, 18380, 19043, 18414, 18403, 19080, 18183, 18742, - 18333, 18951, 18428, 19088, 18551, 19312, 18556, 19317, 18555, 19316, - 18553, 19314, 18554, 19315, 19296, 18171, 18728, 18167, 18724, 18195, - 18756, 18305, 18923, 18298, 18915, 18362, 18994, 18436, 19102, 18437, - 19103, 18283, 18898, 18989, 18244, 18232, 18838, 18299, 18916, 18324, - 18606, 18407, 19068, 18173, 18730, 18194, 18755, 18439, 19105, 18169, - 18726, 18190, 18749, 18263, 18862, 18279, 18880, 18327, 18958, 18344, - 18969, 18434, 19094, 18456, 19122, 18482, 19162, 18487, 19173, 18558, - 19319, 18581, 19346, 18500, 19199, 18522, 19237, 18623, 19401, 18316, - 18934, 18404, 18816, 18464, 19137, 18635, 19416, 18166, 18723, 18247, - 18846, 18430, 19090, 18443, 19109, 18431, 19091, 18432, 19092, 18620, - 19398, 19012, 19061, 19238, 18830, 18843, 19154, 18192, 18225, 18798, - 18374, 18524, 19206, 19411, 18306, 18924, 18212, 18585, 18539, 18281, - 18883, 18359, 18991, 18512, 19152, 18489, 19176, 18622, 19396, 19288, - 18758, 19289, 18774, 19126, 18790, 18813, 18821, 19185, 19215, 19219, - 19132, 19180, 19182, 18805, 18831, 18921, 19222, 18682, 18928, 19192, - 19265, 18941, 18947, 18971, 18982, 18660, 19018, 19007, 19026, 19033, - 19291, 19292, 19052, 19065, 19074, 18693, 18778, 18667, 18804, 19148, - 19276, 19279, 19282, 19167, 19168, 19163, 19183, 18669, 18661, 19212, - 18889, 18832, 19232, 18893, 19283, 19250, 19308, 19349, 19362, 19285, - 19300, 19293, 18700, 19415, 19405, 18895, 18897, 18709, 18711, 18650, - 18702, 18648, 18680, 18803, 18683, 18689, 18988, 19298, 18662, 19151, - 18710, 18654, 18840, 18826, 18841, 19303, 19252, 19302, 18910, 19040, - 19041, 18647, 18649, 19266, 19267, 23222, 23223, 23231, 23253, 23280, - 23283, 23178, 23300, 23302, 23151, 23094, 23308, 23005, 23159, 23161, - 23137, 23110, 23157, 23139, 23163, 23310, 23096, 23086, 5759, 23314, - 23140, 23004, 23109, 23135, 23126, 23132, 23131, 23307, 23117, 23038, - 23037, 23309, 23095, 23150, 23148, 4629, 10885, 27334, 25266, 28928, - 10944, 23164, 23087, 23221, 23233, 23260, 23301, 23257, 23101, 23116, - 23144, 23129, 23106, 23316, 23315, 23313, 23311, 23093, 23122, 23134, - 23124, 23127, 23128, 23147, 23146, 23145, 23130, 23156, 23007, 23107, - 23008, 23108, 23166, 23149, 23123, 7961, 7759, 7843, 8125, 8067, 8087, - 7769, 7868, 7865, 7974, 8112, 7894, 7775, 8140, 7893, 7895, 7777, 7977, - 8133, 7780, 8095, 7781, 7962, 7760, 8052, 8108, 8046, 7976, 8049, 8135, - 7899, 8092, 8076, 8091, 8096, 7871, 7867, 8111, 7782, 7845, 8085, 8139, - 7772, 7980, 7776, 7844, 7771, 7978, 8129, 8072, 8066, 7896, 8128, 8116, - 8064, 8115, 8063, 8103, 7979, 8119, 8123, 8147, 8141, 7882, 7963, 7761, - 7971, 7970, 7973, 7972, 7773, 7908, 7892, 8045, 8078, 7975, 7768, 8053, - 8134, 7966, 8099, 8050, 7909, 8146, 8041, 8100, 8098, 8104, 7870, 7765, - 7887, 8114, 7876, 7875, 7879, 7880, 7888, 7877, 7886, 7993, 8001, 8006, - 8014, 8017, 7999, 8031, 8033, 8035, 8020, 8023, 8038, 8039, 13936, 14200, - 13831, 14067, 14018, 14014, 13925, 14182, 35762, 35762, 14257, 14207, - 14208, 14206, 14262, 13939, 35762, 35762, 35762, 35762, 14222, 13952, - 13829, 13805, 13862, 13852, 13876, 35762, 13908, 35762, 13923, 13898, - 14114, 13808, 13935, 13933, 13931, 13853, 13937, 13832, 13929, 13863, - 13932, 13938, 13940, 13941, 13942, 13899, 13928, 13909, 35762, 13911, - 13930, 13914, 13926, 13934, 13927, 13878, 13868, 13919, 14064, 14079, - 14104, 14123, 14134, 14039, 14199, 14197, 14069, 14070, 14201, 14080, - 14194, 14105, 14145, 14202, 14203, 14204, 14205, 14172, 14185, 14186, - 14196, 14192, 14195, 14125, 14183, 14198, 14184, 14147, 14110, 14130, - 14181, 14143, 14171, 13947, 14259, 14218, 14226, 14224, 14225, 14034, - 14035, 13999, 14010, 14066, 14009, 14191, 14012, 14068, 14011, 14146, - 14008, 14189, 8225, 8319, 8203, 8297, 8199, 8293, 8197, 8291, 8195, 8289, - 8224, 8318, 8194, 8288, 13998, 14037, 14015, 14013, 13948, 14016, 14038, - 13913, 14193, 13943, 13912, 14190, 14036, 13945, 13946, 13944, 9913, - 9915, 9862, 9901, 9837, 9866, 9853, 9972, 9985, 9809, 9812, 9826, 9940, - 9910, 9953, 9870, 9838, 9854, 9986, 9895, 9874, 9912, 9979, 9975, 9908, - 9951, 9925, 9876, 9892, 9881, 9944, 9813, 9887, 9890, 9823, 9832, 9894, - 9902, 9829, 9855, 9958, 9956, 9906, 9969, 9961, 9875, 9974, 9966, 9997, - 10013, 10186, 10054, 10033, 10071, 10179, 10175, 10067, 10129, 10084, - 10035, 10051, 10040, 10110, 10115, 10046, 10049, 10147, 10157, 10053, - 10061, 10153, 10014, 10136, 10134, 10065, 10169, 10139, 10034, 10174, - 10166, 10072, 10074, 10021, 10060, 10162, 10025, 10012, 10172, 10185, - 10103, 10109, 10150, 10099, 10069, 10131, 10029, 9945, 10111, 9968, - 10168, 9921, 10080, 9808, 10101, 9917, 10076, 9850, 10009, 9918, 10077, - 9941, 10100, 9816, 10119, 9984, 10184, 9923, 10082, 9924, 10083, 9836, - 10161, 9817, 10124, 9946, 10112, 9948, 10114, 9938, 10097, 10217, 7836, - 7830, 7833, 7831, 7832, 7786, 7839, 9952, 10130, 9965, 10143, 9888, - 10047, 9897, 10056, 9899, 10058, 9896, 10055, 9981, 10181, 9977, 10177, - 9926, 10085, 9928, 10087, 9929, 10088, 9848, 10007, 9883, 10042, 9991, - 10190, 9814, 10116, 9844, 10003, 9891, 10050, 9825, 10149, 9963, 10141, - 9964, 10142, 9903, 10062, 9990, 10189, 9857, 10016, 9858, 10017, 9954, - 10132, 9841, 10000, 9842, 10001, 9994, 9982, 10182, 9927, 10086, 9879, - 10038, 9886, 10045, 9885, 10044, 9939, 10098, 9893, 10052, 10118, 9840, - 9999, 9839, 9998, 9989, 10188, 9914, 10073, 9949, 10127, 9950, 10128, - 9980, 10180, 9976, 10176, 9843, 10002, 9911, 10070, 9909, 10068, 9947, - 10113, 9846, 10005, 9847, 10006, 9889, 10048, 9835, 10160, 9834, 10159, - 9833, 10158, 9856, 10015, 9898, 10057, 9970, 10170, 9900, 10059, 9904, - 10063, 9905, 10064, 9932, 10091, 9931, 10090, 9937, 10096, 9930, 10089, - 9933, 10092, 9934, 10093, 9935, 10094, 9936, 10095, 9821, 10123, 9880, - 10039, 9810, 10104, 9822, 10126, 9967, 10167, 9988, 10187, 9987, 10165, - 9845, 10004, 9877, 10036, 9882, 10041, 9815, 10117, 9955, 10133, 9884, - 10043, 9868, 10027, 9872, 10031, 9878, 10037, 35762, 2429, 2436, 2420, - 2442, 2416, 2440, 2417, 2418, 2407, 2439, 2435, 2432, 2434, 2412, 2424, - 2441, 2422, 2419, 2410, 2437, 2408, 2438, 2427, 2431, 2411, 2426, 2421, - 2415, 2428, 2430, 2406, 2414, 2413, 2409, 2425, 2423, 2443, 2433, 35762, - 35762, 2493, 2405, 2445, 2446, 2444, 2497, 2404, 2466, 2469, 2479, 2457, - 2485, 2453, 2483, 2454, 2455, 2468, 2482, 2478, 2475, 2477, 2449, 2461, - 2484, 2459, 2456, 2447, 2480, 2472, 2481, 2464, 2471, 2448, 2463, 2458, - 2452, 2465, 2470, 2467, 2451, 2450, 2474, 2462, 2460, 2486, 2476, 2487, - 2473, 2496, 2494, 35762, 35762, 27207, 19558, 2495, 35762, 15349, 15352, - 15353, 15360, 15361, 15357, 15364, 15362, 15347, 15359, 15356, 15340, - 15341, 15342, 15350, 15355, 15348, 15337, 15345, 15346, 15343, 15344, - 15339, 15351, 15354, 15358, 15365, 15366, 15338, 15363, 15444, 15460, - 15448, 15446, 15447, 15449, 15462, 15458, 15452, 15454, 15450, 15451, - 15456, 15445, 15463, 15469, 15457, 15466, 15459, 15461, 15467, 15443, - 15442, 15468, 15455, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 15367, 15374, 15417, 15415, 15392, 15423, 15397, 15394, 15409, - 15434, 15383, 15377, 15419, 15389, 15421, 15388, 15395, 15399, 15373, - 15385, 15380, 15387, 15413, 15390, 15407, 15401, 15411, 35762, 35762, - 35762, 35762, 15470, 15438, 15441, 15439, 15465, 15464, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 2249, - 2283, 1081, 2284, 2285, 2248, 2392, 2393, 2259, 2390, 2391, 2582, 1056, - 1063, 2253, 2289, 2282, 2288, 2280, 2281, 2290, 2310, 2296, 2325, 2292, - 2341, 2340, 2276, 1367, 1072, 2380, 2388, 1322, 1275, 1139, 1129, 1554, - 1132, 1570, 1111, 1153, 1497, 1496, 1517, 1301, 1260, 1341, 1178, 1514, - 1419, 1596, 1453, 1466, 1445, 1193, 1475, 1591, 1101, 1247, 1328, 1325, - 1222, 1221, 1220, 2368, 1227, 1410, 1312, 1346, 1359, 1383, 1277, 1549, - 1147, 1561, 1079, 1061, 1091, 1073, 1057, 1087, 2277, 2345, 2102, 1086, - 1085, 2344, 2266, 2103, 2389, 2384, 2383, 2385, 2260, 1074, 2387, 2400, - 2402, 2399, 2398, 2395, 2394, 2397, 2396, 2403, 2401, 2257, 1068, 2382, - 1082, 1205, 1207, 1472, 1144, 1136, 1135, 1297, 1298, 1300, 1535, 1299, - 1523, 1529, 1173, 1505, 1504, 1398, 1509, 1168, 1270, 1268, 1378, 1208, - 1267, 1484, 1491, 1202, 1187, 1179, 1180, 1190, 1199, 1213, 1183, 1186, - 1436, 1422, 1433, 1430, 1423, 1431, 1426, 1309, 1429, 1454, 1457, 1458, - 1448, 1447, 1478, 1104, 1206, 1230, 1228, 1542, 1232, 1405, 1413, 1414, - 1323, 1474, 1317, 1316, 1368, 1314, 1238, 1241, 1369, 1240, 1254, 1239, - 1350, 1348, 1352, 1351, 1392, 1384, 1439, 1393, 1389, 1288, 1486, 1285, - 1278, 1279, 1500, 1558, 1335, 1588, 1534, 1585, 1338, 1557, 1541, 1216, - 1579, 1575, 1551, 1600, 1580, 1562, 1565, 1083, 1152, 2299, 2298, 2301, - 2300, 2327, 2309, 2307, 1071, 2367, 2324, 2323, 2293, 2302, 2339, 2303, - 2343, 2342, 2321, 2304, 2255, 1070, 1069, 2264, 2338, 1182, 1427, 13052, - 13054, 13051, 13050, 13047, 13046, 13049, 13048, 13055, 13053, 1467, - 1194, 1249, 2287, 2286, 1284, 29867, 29941, 29938, 29939, 29935, 29876, - 29863, 29864, 29940, 29937, 29862, 29872, 29871, 29870, 35762, 29860, - 29908, 29904, 29918, 29914, 29915, 29878, 29877, 29879, 29921, 29919, - 29880, 29911, 29912, 29916, 29917, 29909, 29881, 29893, 29920, 29900, - 29907, 29922, 29894, 29898, 29906, 29910, 29899, 29905, 29913, 29895, - 29897, 29896, 29927, 29926, 29925, 29930, 29929, 29928, 29932, 29931, - 29866, 29865, 29875, 29874, 29873, 29869, 29868, 29933, 29947, 29948, - 29934, 29945, 29944, 29943, 29942, 29924, 29923, 29946, 29861, 35762, - 35762, 29901, 29902, 29903, 1159, 1162, 1157, 1158, 1160, 1161, 1155, - 1269, 1266, 1185, 1181, 1424, 1460, 1106, 1102, 1105, 1233, 1231, 1330, - 1326, 1324, 1362, 1361, 1390, 1387, 1388, 1353, 1425, 1428, 1459, 1265, - 1263, 1456, 1420, 1264, 1138, 1137, 1219, 1218, 1217, 1553, 1552, 1564, - 1563, 1261, 1461, 1455, 1313, 31890, 31903, 31899, 31916, 31915, 31894, - 31891, 31879, 31910, 31895, 31884, 31883, 31906, 31893, 31886, 31887, - 31904, 31882, 31912, 31905, 31917, 31898, 31897, 31896, 31908, 31889, - 31892, 31907, 31913, 31902, 31901, 31881, 31909, 31914, 31880, 31888, - 31885, 31911, 31875, 31874, 31921, 31877, 31922, 31920, 31876, 31878, - 31919, 31918, 31923, 31900, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 24597, 24599, - 24596, 24595, 24592, 24591, 24594, 24593, 24600, 24598, 24632, 24619, - 24633, 24618, 24634, 24616, 24615, 24603, 24608, 24621, 24627, 24629, - 24607, 24617, 24602, 24614, 24613, 24628, 24620, 24622, 24624, 24625, - 24610, 24626, 24611, 24609, 24623, 24630, 24631, 24612, 24605, 24604, - 24606, 24584, 24585, 24586, 24582, 24579, 24580, 24581, 24583, 24587, - 24636, 24635, 24638, 24637, 24588, 24639, 24601, 35762, 35762, 24589, - 24590, 24640, 27560, 27547, 27561, 27549, 27552, 27548, 27563, 27551, - 27558, 27567, 27553, 27554, 27564, 27566, 27555, 27550, 27568, 27559, - 27565, 27562, 27556, 27557, 27570, 27571, 27574, 27569, 27575, 27572, - 27594, 27604, 27599, 27593, 27603, 27598, 27592, 27602, 27576, 27600, - 27596, 27606, 27577, 27595, 27605, 27597, 27601, 27573, 35762, 35762, - 27584, 27578, 27580, 27583, 27581, 27585, 27607, 27590, 27588, 27591, - 27587, 27589, 27582, 27586, 27579, 35762, 20924, 20911, 20913, 20912, - 20914, 20923, 20921, 20926, 20906, 20904, 20903, 20915, 20916, 20917, - 20907, 20925, 20918, 20909, 20919, 20920, 20908, 20905, 20922, 20927, - 20910, 20931, 20929, 20928, 35762, 35762, 20930, 35762, 29886, 29891, - 29887, 29889, 29885, 29884, 29888, 29892, 29883, 29882, 29890, 35762, - 35762, 35762, 35762, 35762, 1125, 1115, 1126, 1142, 1124, 1112, 1121, - 1118, 1122, 1120, 1143, 1117, 1128, 1114, 1116, 1127, 1113, 1119, 1123, - 2369, 2370, 2372, 1522, 2265, 2258, 1391, 1262, 1479, 1477, 1327, 2386, - 35762, 2254, 2256, 35762, 35762, 35762, 35762, 35762, 35762, 2311, 2332, - 2331, 2334, 2101, 2349, 1064, 1084, 1156, 1163, 1302, 1476, 1229, 1411, - 1347, 1360, 1576, 1578, 1432, 1550, 1444, 1358, 1184, 1446, 1242, 1473, - 1599, 1103, 1315, 1412, 1154, 1399, 1502, 1421, 1577, 1099, 1097, 1100, - 1400, 1503, 1524, 1485, 1329, 1248, 1098, 1304, 1303, 1349, 1259, 2291, - 2294, 2322, 2317, 2326, 1095, 1094, 2348, 1096, 1093, 2337, 2312, 2308, - 2329, 2328, 2305, 2330, 2313, 2315, 2314, 2316, 2319, 2318, 2295, 2306, - 1067, 2381, 1052, 1050, 1054, 1053, 1051, 1055, 2375, 2377, 2379, 2374, - 2376, 2378, 2252, 2251, 2250, 2320, 1076, 1075, 1088, 1606, 2261, 1605, - 2263, 1065, 1066, 2262, 1060, 2104, 10466, 10454, 10468, 10473, 10389, - 10363, 10364, 10433, 10434, 10412, 10413, 10428, 10430, 10371, 10390, - 10441, 10365, 10372, 10391, 10395, 10366, 10406, 10405, 10384, 10382, - 10418, 10369, 10373, 10403, 10401, 10419, 10425, 10424, 10377, 10376, - 10417, 10427, 10426, 10379, 10378, 10420, 10416, 10436, 10435, 10400, - 10399, 10387, 10411, 10409, 10408, 10423, 10422, 10421, 10432, 10392, - 10393, 10394, 10386, 10486, 10485, 10470, 10467, 10476, 10498, 10499, - 10488, 10489, 10494, 10495, 10482, 10492, 10500, 10477, 10483, 10493, - 10484, 10478, 10472, 10487, 10479, 10514, 10475, 10474, 10358, 10355, - 10481, 10491, 10490, 10440, 10404, 10381, 10438, 10374, 10407, 10439, - 10410, 10429, 10431, 10496, 10497, 10502, 10501, 10509, 10511, 10508, - 10507, 10504, 10503, 10506, 10505, 10512, 10510, 10356, 10471, 10370, - 10397, 10396, 10367, 10415, 10414, 10388, 10437, 10385, 10383, 10402, - 10380, 10375, 10398, 3530, 3599, 3598, 3602, 35762, 3553, 3554, 3567, - 3568, 3565, 3566, 3547, 3549, 35762, 35762, 3589, 3555, 35762, 35762, - 3590, 3556, 3540, 3537, 3581, 3580, 3569, 3579, 3578, 3583, 3582, 3571, - 3562, 3561, 3558, 3557, 3570, 3564, 3563, 3560, 3559, 3572, 35762, 3585, - 3584, 3577, 3576, 3588, 3552, 3541, 35762, 3587, 35762, 35762, 35762, - 3573, 3574, 3575, 3586, 35762, 35762, 3600, 3597, 3604, 3613, 3614, 3611, - 3612, 3607, 3608, 35762, 35762, 3615, 3605, 35762, 35762, 3616, 3606, - 3601, 3538, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 3528, - 35762, 35762, 35762, 35762, 3545, 3544, 35762, 3551, 3548, 3550, 3609, - 3610, 35762, 35762, 3623, 3625, 3622, 3621, 3618, 3617, 3620, 3619, 3626, - 3624, 3543, 3542, 3592, 3591, 3531, 3535, 3534, 3533, 3532, 3536, 3603, - 3627, 3546, 3529, 3596, 35762, 35762, 14491, 14492, 14497, 35762, 14445, - 14446, 14459, 14460, 14457, 14458, 35762, 35762, 35762, 35762, 14478, - 14447, 35762, 35762, 14477, 14448, 14442, 14441, 14439, 14438, 14463, - 14470, 14469, 14472, 14471, 14465, 14454, 14453, 14450, 14449, 14464, - 14456, 14455, 14452, 14451, 14466, 35762, 14474, 14473, 14468, 14467, - 14481, 14483, 14444, 35762, 14462, 14461, 35762, 14482, 14475, 35762, - 14476, 14480, 35762, 35762, 14494, 35762, 14498, 14503, 14504, 14501, - 14502, 35762, 35762, 35762, 35762, 14506, 14499, 35762, 35762, 14505, - 14500, 14496, 35762, 35762, 35762, 14493, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 14440, 14437, 14484, 14443, 35762, 14479, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 14516, 14518, 14515, 14514, - 14511, 14510, 14513, 14512, 14519, 14517, 14507, 14436, 14508, 14520, - 14509, 14495, 14435, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 14331, 14338, 14341, 35762, 14281, 14282, 14300, - 14301, 14298, 14299, 14293, 14295, 14357, 35762, 14328, 14283, 14358, - 35762, 14329, 14284, 14321, 14320, 14317, 14316, 14305, 14315, 14314, - 14319, 14318, 14307, 14290, 14289, 14286, 14285, 14306, 14292, 14291, - 14288, 14287, 14308, 35762, 14323, 14322, 14313, 14312, 14325, 14327, - 14326, 35762, 14303, 14302, 35762, 14297, 14309, 14310, 14311, 14324, - 35762, 35762, 14339, 14337, 14344, 14353, 14354, 14351, 14352, 14347, - 14348, 14342, 35762, 14355, 14345, 14343, 35762, 14356, 14346, 14340, - 35762, 35762, 14371, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 14294, 14296, - 14349, 14350, 35762, 35762, 14367, 14369, 14366, 14365, 14362, 14361, - 14364, 14363, 14370, 14368, 14359, 14360, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 14304, 14336, 14335, 14332, 14334, 14330, 14333, - 35762, 25865, 25864, 25869, 35762, 25814, 25815, 25833, 25834, 25831, - 25832, 25826, 25828, 35762, 35762, 25859, 25816, 35762, 35762, 25860, - 25817, 25853, 25852, 25849, 25848, 25837, 25847, 25846, 25851, 25850, - 25839, 25823, 25822, 25819, 25818, 25838, 25825, 25824, 25821, 25820, - 25840, 35762, 25855, 25854, 25845, 25844, 25857, 25813, 25811, 35762, - 25836, 25835, 35762, 25830, 25841, 25842, 25843, 25856, 35762, 35762, - 25866, 25863, 25870, 25879, 25880, 25877, 25878, 25873, 25874, 35762, - 35762, 25881, 25871, 35762, 35762, 25882, 25872, 25868, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 25867, 25800, 25801, 35762, 35762, - 35762, 35762, 25810, 25809, 35762, 25812, 25827, 25829, 25875, 25876, - 35762, 35762, 25889, 25891, 25888, 25887, 25884, 25883, 25886, 25885, - 25892, 25890, 25808, 25858, 25805, 25804, 25807, 25802, 25803, 25806, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30505, 30522, 35762, 30473, 30474, 30481, 30482, 30477, 30478, 35762, - 35762, 35762, 30486, 30487, 30475, 35762, 30479, 30480, 30476, 30491, - 35762, 35762, 35762, 30463, 30488, 35762, 30490, 35762, 30464, 30466, - 35762, 35762, 35762, 30462, 30467, 35762, 35762, 35762, 30465, 30461, - 30493, 35762, 35762, 35762, 30492, 30495, 30472, 30471, 30470, 30469, - 30468, 30494, 30483, 30484, 30485, 30489, 35762, 35762, 35762, 35762, - 30792, 30799, 30800, 30795, 30796, 35762, 35762, 35762, 30801, 30802, - 30793, 35762, 30797, 30798, 30794, 30521, 35762, 35762, 30808, 35762, - 35762, 35762, 35762, 35762, 35762, 30397, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30434, 30436, 30433, 30432, 30429, 30428, 30431, 30430, 30437, 30435, - 30499, 30497, 30498, 30427, 30807, 30806, 30426, 30424, 30396, 30502, - 30500, 35762, 35762, 35762, 35762, 35762, 31755, 31756, 31760, 31763, - 31754, 31718, 31719, 31731, 31732, 31727, 31728, 31722, 31724, 35762, - 31748, 31749, 31720, 35762, 31729, 31730, 31721, 31745, 31744, 31741, - 31740, 31707, 31739, 31738, 31743, 31742, 31709, 31714, 31713, 31698, - 31697, 31708, 31717, 31715, 31701, 31699, 31705, 35762, 31747, 31746, - 31737, 31736, 31751, 31752, 31712, 31711, 31704, 31703, 31702, 31726, - 31733, 31734, 31735, 31750, 35762, 35762, 31761, 31759, 31764, 31775, - 31776, 31771, 31772, 31767, 31768, 35762, 31777, 31778, 31765, 35762, - 31773, 31774, 31766, 31762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 31753, 31689, 35762, 31716, 31700, 31710, 35762, 35762, 31706, - 35762, 35762, 31723, 31725, 31769, 31770, 35762, 35762, 31785, 31787, - 31784, 31783, 31780, 31779, 31782, 31781, 31788, 31786, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 31757, 31696, 31694, 31692, 31690, - 31695, 31693, 31691, 31758, 16808, 16807, 16811, 16816, 16809, 16761, - 16762, 16782, 16783, 16778, 16779, 16773, 16775, 35762, 16799, 16800, - 16763, 35762, 16780, 16781, 16764, 16796, 16795, 16792, 16791, 16756, - 16790, 16789, 16794, 16793, 16758, 16770, 16769, 16766, 16765, 16757, - 16772, 16771, 16768, 16767, 16754, 35762, 16798, 16797, 16788, 16787, - 16803, 16804, 16760, 16759, 16753, 16752, 35762, 16777, 16784, 16785, - 16786, 16802, 35762, 35762, 16812, 16810, 16818, 16829, 16830, 16825, - 16826, 16821, 16822, 35762, 16831, 16832, 16819, 35762, 16827, 16828, - 16820, 16815, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16805, - 16817, 35762, 35762, 35762, 35762, 35762, 35762, 16755, 16801, 35762, - 16774, 16776, 16823, 16824, 35762, 35762, 16839, 16841, 16838, 16837, - 16834, 16833, 16836, 16835, 16842, 16840, 35762, 16813, 16814, 16806, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 20866, 20868, 20873, 20871, 20823, 20797, 20799, 20843, - 20844, 20839, 20840, 20824, 20826, 35762, 20858, 20859, 20800, 35762, - 20841, 20842, 20801, 20855, 20854, 20851, 20850, 20831, 20812, 20811, - 20853, 20852, 20832, 20820, 20818, 20815, 20814, 20830, 20822, 20821, - 20817, 20816, 20833, 20829, 20857, 20856, 20849, 20848, 20861, 20862, - 20838, 20837, 20836, 20835, 20834, 20828, 20845, 20846, 20847, 20860, - 20819, 20869, 20867, 20872, 20875, 20886, 20887, 20882, 20883, 20878, - 20879, 35762, 20888, 20889, 20876, 35762, 20884, 20885, 20877, 20870, - 20813, 20874, 35762, 35762, 35762, 35762, 20809, 20810, 20804, 20772, - 20789, 20787, 20794, 20784, 20785, 20795, 20788, 20798, 20825, 20827, - 20880, 20881, 35762, 35762, 20780, 20782, 20779, 20778, 20775, 20774, - 20777, 20776, 20783, 20781, 20865, 20863, 20864, 20792, 20791, 20796, - 20786, 20790, 20793, 20773, 20806, 20805, 20807, 20802, 20803, 20808, - 35762, 28828, 28827, 28829, 35762, 28773, 28770, 28758, 28757, 28781, - 28780, 28783, 28782, 28779, 28778, 28777, 28776, 28775, 28774, 28771, - 28802, 28801, 28772, 35762, 35762, 35762, 28767, 28792, 28765, 28790, - 28815, 28805, 28764, 28789, 28766, 28791, 28812, 28813, 28806, 28759, - 28784, 28761, 28786, 28796, 28803, 28760, 28785, 28762, 28787, 28799, - 35762, 28804, 28768, 28793, 28763, 28788, 28794, 28769, 28811, 28809, - 35762, 28798, 35762, 35762, 28810, 28814, 28797, 28800, 28808, 28795, - 28807, 35762, 35762, 35762, 28826, 35762, 35762, 35762, 35762, 28846, - 28838, 28833, 28839, 28834, 28840, 35762, 28835, 35762, 28836, 28841, - 28832, 28845, 28842, 28843, 28844, 28837, 35762, 35762, 35762, 35762, - 35762, 35762, 28822, 28824, 28821, 28820, 28817, 28816, 28819, 28818, - 28825, 28823, 35762, 35762, 28830, 28831, 28847, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 31942, - 31939, 31937, 31936, 31938, 31940, 31956, 31925, 31927, 31926, 31985, - 31928, 31997, 31929, 31994, 31990, 31987, 31988, 31958, 31930, 31993, - 31992, 31989, 31991, 31959, 31924, 31965, 31962, 31931, 31963, 31932, - 31964, 31954, 31998, 31966, 31967, 31944, 31946, 31995, 31983, 31982, - 31984, 31935, 31943, 31999, 31934, 31960, 31968, 31948, 31971, 31973, - 31978, 31979, 31975, 31976, 31974, 31977, 31961, 35762, 35762, 35762, - 35762, 32000, 31980, 31972, 31981, 31970, 31969, 31945, 31953, 31952, - 31951, 31949, 31950, 31947, 31986, 31957, 31996, 31933, 32007, 32009, - 32006, 32005, 32002, 32001, 32004, 32003, 32010, 32008, 31955, 31941, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 18051, 18049, 35762, - 18050, 35762, 18064, 18079, 18083, 18063, 18073, 35762, 18065, 18080, - 18061, 18059, 18058, 18056, 18055, 18060, 18084, 18076, 18074, 18075, - 18057, 18081, 18082, 18069, 18067, 18046, 18068, 18045, 18062, 18085, - 18088, 18053, 35762, 18054, 35762, 18087, 18070, 18071, 18072, 18077, - 18066, 18089, 18078, 18115, 18097, 18102, 18098, 18100, 18110, 18111, - 18104, 18105, 18106, 18107, 18092, 18103, 18091, 18090, 35762, 35762, - 18108, 18109, 18112, 18101, 18099, 35762, 18113, 35762, 18096, 18093, - 18094, 18095, 18126, 18127, 18114, 35762, 18122, 18124, 18121, 18120, - 18117, 18116, 18119, 18118, 18125, 18123, 35762, 35762, 18042, 18041, - 18048, 18047, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 32233, 32140, 32138, 32139, 32148, 32136, - 32133, 32137, 32158, 32128, 32126, 32149, 32164, 32159, 32154, 32162, - 32153, 32157, 32156, 32132, 32141, 32124, 32123, 32051, 32052, 32050, - 32172, 32173, 32174, 32176, 32177, 32175, 32073, 32075, 32072, 32071, - 32068, 32067, 32070, 32069, 32076, 32074, 32065, 32062, 32061, 32058, - 32057, 32060, 32059, 32066, 32064, 32063, 32129, 32151, 32131, 32150, - 32134, 32161, 32142, 32143, 32144, 32145, 32183, 32169, 32082, 32080, - 32110, 32109, 32092, 32108, 32107, 32117, 35762, 32094, 32102, 32101, - 32087, 32086, 32093, 32104, 32103, 32091, 32090, 32095, 32112, 32111, - 32106, 32105, 32119, 32100, 32099, 32089, 32088, 32120, 32113, 32114, - 32115, 32121, 32084, 32118, 32096, 32097, 32098, 32116, 32122, 32079, - 32085, 32081, 32083, 35762, 35762, 35762, 35762, 32257, 32253, 32254, - 32249, 32250, 32245, 32246, 32247, 32248, 32255, 32256, 32251, 32252, - 32180, 32181, 32243, 32244, 32171, 32185, 32146, 32155, 32167, 32182, - 32168, 32170, 32165, 32166, 32184, 32232, 32231, 32230, 32197, 32196, - 32216, 32215, 32198, 32214, 32213, 32223, 35762, 32200, 32208, 32207, - 32190, 32189, 32199, 32210, 32209, 32194, 32193, 32201, 32218, 32217, - 32212, 32211, 32225, 32206, 32205, 32192, 32191, 32227, 32219, 32220, - 32221, 32228, 32226, 32224, 32202, 32203, 32204, 32222, 32229, 32195, - 32187, 32188, 32186, 35762, 32077, 32078, 32053, 32054, 32056, 32055, - 32234, 32241, 32239, 32242, 32240, 32235, 32238, 32237, 32236, 35762, - 32179, 32178, 32127, 32130, 32152, 32147, 32135, 27208, 19559, 27209, - 19560, 32163, 32160, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 23897, 23876, 23875, 23874, 23906, 23974, 23973, 23976, 23975, 23907, - 23904, 23952, 23951, 23958, 23957, 23905, 23936, 23953, 23960, 23959, - 23908, 23978, 23977, 23972, 23971, 23903, 23980, 23910, 23970, 23956, - 23935, 23979, 23969, 23866, 23930, 23967, 23968, 23965, 23966, 23869, - 23902, 23981, 23868, 24063, 24047, 24070, 24071, 24068, 24069, 24059, - 24044, 24052, 24053, 24060, 23988, 24007, 24012, 24011, 23987, 23861, - 23859, 23860, 23858, 23873, 24078, 24080, 24077, 24076, 24073, 24072, - 24075, 24074, 24081, 24079, 24010, 23999, 24017, 24021, 24016, 24020, - 23911, 23934, 23961, 23962, 23963, 23964, 24064, 24065, 24066, 24067, - 23901, 23900, 23898, 23899, 23864, 23863, 23862, 23933, 24054, 24032, - 24033, 23955, 23954, 24061, 24062, 24002, 24003, 24004, 24005, 24006, - 23872, 23871, 23870, 24048, 24050, 24051, 24049, 23925, 23924, 23923, - 23921, 23929, 23913, 23926, 23914, 23916, 23927, 23919, 23917, 23928, - 23865, 24058, 24055, 24056, 24057, 23994, 23995, 23996, 23997, 23992, - 23993, 23991, 23909, 24008, 24028, 24030, 24027, 24026, 24023, 24022, - 24025, 24024, 24031, 24029, 23989, 23990, 24045, 24046, 24019, 24018, - 13332, 13358, 13343, 13364, 13365, 13363, 13353, 13354, 13366, 13347, - 13356, 13359, 13361, 13367, 13349, 13352, 13357, 13351, 13355, 13368, - 13348, 13346, 13342, 13362, 13350, 13338, 13341, 13345, 13340, 13339, - 13360, 13344, 13333, 13337, 13335, 13370, 13334, 13336, 35762, 13369, - 35762, 35762, 35762, 35762, 35762, 13331, 35762, 35762, 13415, 13446, - 13423, 13452, 13421, 13451, 13444, 13441, 13453, 13433, 13435, 13447, - 13449, 13454, 13437, 13443, 13445, 13439, 13442, 13412, 13436, 13432, - 13422, 13450, 13438, 13416, 13419, 13431, 13418, 13417, 13448, 13430, - 13426, 13429, 13427, 13456, 13424, 13428, 13457, 13455, 13420, 13440, - 13414, 13504, 23112, 13413, 13425, 13434, 14693, 14767, 14701, 14772, - 14765, 14729, 14696, 14711, 14769, 14743, 14761, 14675, 14673, 14759, - 14660, 14695, 14779, 14708, 14782, 14703, 14768, 14705, 14704, 14776, - 14739, 14763, 14742, 14688, 14698, 14692, 14721, 14726, 14725, 14712, - 14716, 14714, 14717, 14718, 14715, 14723, 14722, 14724, 14719, 14690, - 14691, 14750, 14756, 14755, 14748, 14753, 14744, 14745, 14747, 14758, - 14752, 14751, 14749, 14754, 14746, 14757, 14665, 14664, 14670, 14669, - 14728, 14685, 14684, 14682, 14677, 14687, 14678, 14771, 14681, 14680, - 14683, 14676, 14780, 14674, 14667, 14663, 14672, 14668, 14661, 14662, - 14666, 14671, 14709, 14689, 14770, 14781, 14694, 14707, 14702, 14706, - 14773, 14784, 15022, 14928, 14938, 14986, 14990, 14940, 14939, 14992, - 14991, 14967, 15019, 15020, 14977, 14998, 14978, 15018, 15017, 15021, - 15007, 14944, 14996, 14951, 14936, 14937, 14988, 14987, 14942, 14943, - 14941, 14994, 14995, 14975, 14974, 14970, 14968, 14976, 14999, 15000, - 15001, 15006, 15005, 14983, 14984, 14982, 14979, 14985, 15012, 15011, - 15010, 15009, 15008, 15016, 15014, 14950, 14947, 14997, 14952, 14954, - 14961, 14965, 14963, 14953, 14929, 14931, 14934, 14933, 14966, 14935, - 14989, 14993, 14972, 14973, 14804, 14905, 14807, 14827, 14830, 14834, - 14909, 14855, 14856, 14861, 14865, 14874, 14877, 14870, 14882, 14814, - 14844, 14847, 14883, 14900, 14796, 14787, 14790, 14813, 14918, 14840, - 14791, 14806, 14808, 14833, 14832, 14836, 14835, 14831, 14916, 14910, - 14858, 14881, 14875, 14876, 14895, 14862, 14864, 14869, 14868, 14859, - 14873, 14871, 14860, 14878, 14824, 14821, 14815, 14820, 14819, 14817, - 14822, 14826, 14803, 14845, 14849, 14854, 14802, 14885, 14893, 14886, - 14889, 14837, 14798, 14799, 14908, 14797, 14919, 14923, 14926, 14842, - 14801, 14794, 14792, 14793, 14795, 14927, 14810, 14811, 14809, 14805, - 14812, 14906, 12703, 12710, 12709, 12704, 12708, 12707, 12705, 12706, - 12930, 12938, 12937, 12931, 12935, 12934, 12932, 12936, 12696, 12702, - 12700, 12697, 12699, 12698, 12701, 12687, 12747, 12755, 12754, 12748, - 12752, 12751, 12749, 12745, 12859, 12866, 12864, 12860, 12862, 12861, - 12865, 12863, 12834, 12843, 12842, 12835, 12839, 12838, 12836, 12840, - 12874, 12880, 12879, 12875, 12849, 12844, 12876, 12878, 12850, 12858, - 12857, 12851, 12855, 12854, 12852, 12856, 12826, 12833, 12832, 12827, - 12831, 12830, 12828, 12829, 12814, 35762, 12818, 12815, 12817, 12816, - 35762, 35762, 12807, 12813, 12811, 12808, 12810, 12809, 12812, 35762, - 12802, 35762, 12806, 12803, 12805, 12804, 35762, 35762, 12537, 12544, - 12543, 12538, 12542, 12541, 12539, 12528, 12999, 13006, 13004, 13000, - 13002, 13001, 13005, 13003, 12912, 12920, 12919, 12913, 12917, 12916, - 12914, 12918, 12575, 12583, 12582, 12576, 12580, 12579, 12577, 12581, - 12967, 12974, 12973, 12968, 12972, 12971, 12969, 12970, 12955, 35762, - 12959, 12956, 12958, 12957, 35762, 35762, 12765, 12773, 12772, 12766, - 12770, 12769, 12767, 12771, 12756, 12764, 12763, 12757, 12761, 12760, - 12758, 12762, 12657, 12665, 12664, 12658, 12662, 12661, 12659, 12663, - 12735, 12742, 12741, 12736, 12740, 12739, 12737, 12738, 12723, 35762, - 12727, 12724, 12726, 12725, 35762, 35762, 12716, 12722, 12720, 12717, - 12719, 12718, 12721, 35762, 12711, 35762, 12715, 12712, 12714, 12713, - 35762, 35762, 12939, 12946, 12945, 12940, 12944, 12943, 12941, 12942, - 12775, 12781, 12779, 12776, 12778, 12777, 12780, 35762, 12990, 12998, - 12997, 12991, 12995, 12994, 12992, 12996, 12975, 12982, 12980, 12976, - 12978, 12977, 12981, 12979, 12947, 12954, 12953, 12948, 12952, 12951, - 12949, 12950, 12605, 12613, 12612, 12606, 12610, 12609, 12607, 12611, - 12590, 12598, 12597, 12591, 12595, 12594, 12592, 12596, 12921, 12929, - 12928, 12922, 12926, 12925, 12923, 12927, 12678, 12626, 12684, 12679, - 12683, 12682, 12680, 12681, 12666, 35762, 12670, 12667, 12669, 12668, - 35762, 35762, 12650, 12656, 12654, 12651, 12653, 12652, 12655, 12646, - 12881, 12889, 12888, 12882, 12886, 12885, 12883, 12887, 12566, 12574, - 12573, 12567, 12571, 12570, 12568, 12572, 12774, 12789, 12788, 12782, - 12786, 12785, 12783, 12787, 12904, 12911, 12909, 12905, 12907, 12906, - 12910, 12908, 12896, 12903, 12902, 12897, 12901, 12900, 12898, 12899, - 12618, 12625, 12623, 12619, 12621, 12620, 12624, 12616, 12794, 12801, - 12800, 12795, 12799, 12798, 12796, 12792, 12841, 12753, 12622, 35762, - 35762, 12506, 12508, 12507, 12525, 13028, 13026, 12509, 12524, 12510, - 12523, 13027, 12522, 13024, 13022, 13021, 13018, 13017, 13020, 13019, - 13025, 13023, 12511, 12514, 12513, 12518, 12517, 12521, 12520, 12516, - 12519, 12515, 12512, 35762, 35762, 35762, 12847, 12746, 12744, 12743, - 12845, 12529, 12527, 12526, 12846, 12617, 12615, 12614, 12848, 12793, - 12791, 12790, 13016, 13007, 13013, 13014, 13009, 13011, 13015, 13010, - 13008, 13012, 35762, 35762, 35762, 35762, 35762, 35762, 6081, 6082, 6083, - 6084, 6085, 6086, 6044, 6080, 6045, 6046, 6047, 6048, 6049, 6009, 6010, - 6011, 6012, 6013, 6014, 6050, 6051, 6052, 6053, 6054, 6055, 6056, 6057, - 6058, 6059, 6060, 6015, 6008, 6016, 6017, 6018, 6019, 6020, 6021, 6062, - 6063, 6064, 6065, 6066, 6067, 6023, 6022, 6024, 6025, 6026, 6027, 6028, - 6002, 6041, 6003, 6042, 6004, 6043, 6005, 6006, 6007, 6001, 6029, 6030, - 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039, 6040, 6068, 6069, - 6070, 6071, 6072, 6073, 6074, 6075, 6076, 6077, 6078, 6079, 6061, 35762, - 35762, 6161, 6162, 6163, 6164, 6165, 6147, 35762, 35762, 5223, 5195, - 4968, 5225, 5226, 5375, 5389, 5672, 5179, 5039, 4966, 4967, 5549, 5639, - 5659, 5637, 5660, 5638, 5641, 5635, 5642, 5636, 5304, 5656, 5633, 5657, - 5634, 5305, 4970, 5673, 5691, 5212, 5211, 5213, 5214, 5206, 5207, 5204, - 5203, 5216, 5210, 5215, 5205, 5197, 5227, 5388, 4974, 5409, 5402, 5407, - 5408, 5404, 5405, 5663, 5037, 5038, 5400, 5401, 5399, 5578, 5397, 5576, - 5398, 5577, 5392, 5574, 5393, 5575, 5395, 5572, 5396, 5573, 5662, 5391, - 5571, 5030, 5548, 5531, 5546, 5547, 5544, 5545, 5670, 5001, 5014, 5529, - 5530, 5539, 5631, 5537, 5629, 5538, 5630, 5535, 5627, 5536, 5628, 5533, - 5625, 5534, 5626, 5310, 5491, 5526, 5527, 5528, 5525, 5246, 5240, 5244, - 5245, 5242, 5243, 5665, 5238, 5239, 5237, 5623, 5235, 5621, 5236, 5622, - 5233, 5619, 5234, 5620, 5230, 5617, 5231, 5618, 5307, 5228, 5229, 5471, - 5472, 5473, 5470, 5194, 5181, 5192, 5193, 5190, 5191, 5664, 4998, 5180, - 5188, 5616, 5186, 5614, 5187, 5615, 5184, 5612, 5185, 5613, 5182, 5610, - 5183, 5611, 5306, 4997, 5447, 5272, 5280, 5289, 5290, 5275, 5276, 5667, - 5278, 5279, 5288, 5570, 5286, 5568, 5287, 5569, 5284, 5566, 5285, 5567, - 5282, 5564, 5283, 5565, 5308, 5271, 5563, 5291, 4972, 5448, 5364, 5325, - 5362, 5363, 5352, 5353, 5668, 5296, 5324, 5361, 5589, 5355, 5587, 5356, - 5588, 5309, 5292, 5070, 5365, 5270, 5257, 5268, 5269, 5266, 5267, 5666, - 5255, 5256, 5265, 5557, 5263, 5555, 5264, 5556, 5261, 5553, 5262, 5554, - 5259, 5551, 5260, 5552, 5247, 5550, 5273, 5490, 5450, 5488, 5489, 5469, - 5474, 5669, 5430, 5449, 5483, 5609, 5481, 5607, 5482, 5608, 5479, 5605, - 5480, 5606, 5477, 5603, 5478, 5604, 5302, 5429, 4973, 5476, 4993, 5277, - 5297, 5303, 5300, 5301, 5298, 5299, 5468, 5466, 5467, 5460, 5461, 5463, - 5464, 5459, 5602, 5457, 5600, 5458, 5601, 5452, 5598, 5453, 5599, 5455, - 5596, 5456, 5597, 5451, 5690, 5676, 5688, 5689, 5678, 5679, 5671, 5674, - 5675, 5687, 5586, 5685, 5584, 5686, 5585, 5683, 5582, 5684, 5583, 5681, - 5580, 5682, 5581, 5311, 5661, 4994, 5579, 5446, 5428, 5412, 5562, 5422, - 5426, 5427, 5424, 5425, 5560, 5420, 5421, 5558, 5414, 5591, 5410, 5590, - 5274, 5222, 5201, 5202, 5217, 5219, 5220, 5199, 5200, 5221, 5632, 5198, - 5510, 5295, 5508, 5293, 5509, 5294, 5506, 5507, 5504, 5505, 5502, 5624, - 5492, 5523, 5524, 5520, 5518, 5517, 5541, 5542, 5543, 5540, 5350, 5348, - 5349, 5346, 5347, 5344, 5345, 5343, 5351, 5224, 5369, 5373, 5374, 5371, - 5372, 5367, 5368, 5366, 5515, 5516, 5511, 5514, 5593, 5594, 5595, 5592, - 5330, 5334, 5335, 5332, 5333, 5328, 5329, 5327, 5336, 5444, 5445, 5440, - 5443, 5652, 5653, 5654, 5651, 5643, 5253, 5254, 5251, 5252, 5249, 5250, - 5248, 5500, 5498, 5499, 5496, 5497, 5494, 5495, 5493, 4971, 4990, 4991, - 4992, 4989, 4978, 4979, 4980, 4977, 4986, 4987, 4988, 4985, 4982, 4983, - 4984, 4981, 5435, 5436, 5432, 5434, 5020, 5019, 5015, 5016, 5018, 5017, - 5166, 5165, 5161, 5162, 5164, 5163, 5172, 5171, 5167, 5168, 5170, 5169, - 5036, 5035, 5031, 5032, 5034, 5033, 5130, 5129, 5125, 5126, 5128, 5127, - 5124, 5123, 5119, 5120, 5122, 5121, 5093, 5092, 5088, 5089, 5091, 5090, - 5087, 5029, 5028, 5025, 5026, 5027, 5023, 5066, 5065, 5061, 5062, 5064, - 5063, 5060, 5059, 5055, 5056, 5058, 5057, 5054, 5073, 5072, 5067, 5068, - 5071, 5069, 5160, 5159, 5155, 5156, 5158, 5157, 5178, 5177, 5173, 5174, - 5176, 5175, 5053, 5437, 5052, 5047, 5048, 5051, 5439, 5050, 5046, 5045, - 5041, 5042, 5044, 5043, 5148, 5147, 5143, 5144, 5146, 5145, 5007, 5006, - 5002, 5003, 5005, 5004, 5142, 5141, 5137, 5138, 5140, 5139, 5106, 5105, - 5101, 5102, 5104, 5103, 5112, 5111, 5107, 5108, 5110, 5109, 5100, 5099, - 5095, 5096, 5098, 5097, 5094, 5040, 5013, 5012, 5008, 5009, 5011, 5010, - 5086, 5085, 5081, 5082, 5084, 5083, 5080, 5079, 5075, 5076, 5078, 5077, - 5074, 5136, 5135, 5131, 5132, 5134, 5133, 5154, 5153, 5149, 5150, 5152, - 5151, 5118, 5117, 5113, 5114, 5116, 5115, 5189, 5218, 5370, 5331, 5341, - 5342, 5339, 5340, 5337, 5338, 5650, 5648, 5649, 5646, 5647, 5644, 5645, - 5655, 4976, 25265, 25240, 25251, 25247, 25257, 25254, 25260, 25263, - 25264, 25243, 25242, 25262, 25248, 25253, 25258, 25252, 25239, 25255, - 25261, 25245, 25249, 25244, 25256, 25259, 25250, 25246, 25241, 25237, - 25238, 35762, 35762, 35762, 27467, 27523, 27517, 27521, 27520, 27515, - 27513, 27460, 27445, 27491, 27444, 27443, 27488, 27503, 27490, 27494, - 27495, 27497, 27483, 27449, 27482, 27468, 27461, 27469, 27471, 27516, - 27473, 27472, 27486, 27500, 27512, 27502, 27454, 27476, 27457, 27480, - 27470, 27485, 27506, 27477, 27519, 27446, 27509, 27508, 27504, 27447, - 27525, 27514, 27505, 27452, 27511, 27499, 27455, 27493, 27458, 27518, - 27487, 27501, 27484, 27453, 27475, 27474, 27456, 27492, 27459, 27479, - 27451, 27450, 27448, 27510, 27489, 27507, 27478, 27522, 27524, 27526, - 27527, 27442, 27528, 27529, 27441, 27481, 27498, 27496, 27465, 27464, - 27466, 27463, 27462, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30050, 30067, 30068, 30058, 30056, 30052, 30064, 30055, 30053, 30061, - 30054, 30060, 30066, 30062, 30059, 30065, 30063, 30057, 30071, 30072, - 30070, 30069, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 30051, 15194, 15195, 15196, 15185, 15183, 15179, 15191, 15182, - 15180, 15188, 15181, 15187, 15193, 15189, 15186, 15192, 15190, 15184, - 15198, 15199, 15197, 26633, 26634, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 4689, 4690, 4691, 4680, 4678, 4674, 4686, - 4677, 4675, 4683, 4676, 4682, 4688, 4684, 4681, 4687, 4685, 4679, 4692, - 4693, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 30086, 30087, 30088, 30078, 30077, 30073, 30083, - 30076, 30074, 30081, 30075, 30080, 30085, 35762, 30079, 30084, 30082, - 35762, 30089, 30090, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 17762, 17760, 17763, 17761, 17764, - 17758, 17756, 17759, 17757, 17766, 17780, 17776, 17781, 17777, 17765, - 17778, 17774, 17779, 17775, 17767, 17788, 17768, 17770, 17769, 17784, - 17787, 17785, 17783, 17786, 17772, 17771, 17773, 17789, 17782, 17790, - 17737, 17735, 17745, 17746, 17741, 17744, 17742, 17743, 17754, 17755, - 17752, 17753, 17747, 17736, 17740, 17739, 17738, 17857, 17856, 17858, - 17863, 17865, 17869, 17871, 17873, 17875, 17874, 17866, 17870, 17864, - 17876, 17860, 17861, 17868, 17862, 17811, 17805, 17810, 17812, 17807, - 17796, 17804, 17806, 17801, 17793, 17794, 17813, 17800, 17795, 17802, - 17797, 17799, 17808, 17798, 17809, 17803, 17734, 17791, 17792, 35762, - 35762, 17883, 17885, 17882, 17881, 17878, 17877, 17880, 17879, 17886, - 17884, 35762, 35762, 35762, 35762, 35762, 35762, 17835, 17834, 17831, - 17833, 17832, 17826, 17829, 17830, 17828, 17827, 35762, 35762, 35762, - 35762, 35762, 35762, 23479, 23491, 23487, 23336, 23486, 23337, 23484, - 23475, 23488, 23489, 23490, 23335, 23334, 23333, 23485, 23332, 23328, - 23330, 23327, 23326, 23323, 23322, 23325, 23324, 23331, 23329, 35762, - 35762, 35762, 35762, 35762, 35762, 23340, 23454, 23471, 23456, 23458, - 23457, 23459, 23455, 23465, 23368, 23460, 23466, 23467, 23463, 23372, - 23453, 23415, 23414, 23445, 23461, 23369, 23464, 23470, 23468, 23469, - 23462, 23451, 23450, 23444, 23448, 23449, 23447, 23452, 23446, 23371, - 23424, 23442, 23443, 23431, 23433, 23432, 23434, 23418, 23435, 23438, - 23439, 23427, 23437, 23426, 23420, 23429, 23422, 23425, 23441, 23440, - 23436, 23428, 23430, 23421, 23423, 23419, 23413, 23398, 23399, 23407, - 23406, 23403, 23411, 23394, 23396, 23412, 23401, 23393, 23408, 23410, - 23409, 23395, 23397, 23392, 23405, 23402, 23400, 23404, 23391, 23389, - 23390, 23388, 23387, 23370, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 23342, 23344, 23346, 23353, 23352, 23360, 23356, 23341, 23351, - 23367, 23354, 23366, 23358, 23357, 23348, 23355, 23359, 23345, 23362, - 23361, 23365, 23363, 23364, 23343, 23417, 23416, 23380, 23385, 23376, - 23381, 23377, 23373, 23378, 23374, 23386, 23375, 23383, 23384, 23350, - 23349, 23379, 23347, 23382, 35762, 35762, 35762, 35762, 35762, 5390, - 4975, 4969, 5658, 5406, 5403, 5394, 5532, 5241, 5232, 5281, 5354, 5326, - 5258, 5475, 5431, 5462, 5465, 5454, 5680, 5677, 5423, 5359, 5379, 5360, - 5380, 5357, 5377, 5358, 5378, 5419, 5417, 5418, 5415, 5416, 5413, 5386, - 5387, 5384, 5383, 5385, 5376, 5381, 5382, 5196, 5640, 5209, 5208, 5411, - 5561, 5559, 5503, 5501, 5522, 5521, 5519, 5513, 5512, 5442, 5441, 5433, - 5022, 4999, 5024, 5021, 5438, 5049, 4995, 4996, 5000, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 19858, 19825, - 19824, 19805, 19804, 19811, 19819, 19818, 19823, 19822, 19810, 19808, - 19806, 19821, 19820, 19812, 19827, 19826, 19817, 19816, 19830, 19809, - 19831, 19829, 19832, 19813, 19814, 19815, 19828, 19803, 19807, 35762, - 19849, 19856, 19857, 19855, 19850, 19853, 19851, 19854, 19852, 19848, - 19846, 19847, 35762, 35762, 35762, 35762, 19840, 19837, 19839, 19845, - 19838, 19843, 19842, 19844, 19841, 19834, 19833, 19835, 35762, 35762, - 35762, 35762, 19836, 35762, 35762, 35762, 19859, 19860, 19867, 19869, - 19866, 19865, 19862, 19861, 19864, 19863, 19870, 19868, 30111, 30123, - 30106, 30103, 30121, 30124, 30105, 30104, 30118, 30113, 30112, 30119, - 30116, 30122, 30117, 30120, 30110, 30102, 30107, 30091, 30125, 30095, - 30096, 30114, 30109, 30108, 30115, 30094, 30092, 30093, 35762, 35762, - 30097, 30098, 30099, 30100, 30101, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 24411, 24433, 24393, 24395, - 24398, 24415, 24417, 24420, 24401, 24397, 24413, 24423, 24419, 24435, - 24402, 24400, 24399, 24424, 24422, 24421, 24404, 24403, 24410, 24426, - 24425, 24432, 24407, 24412, 24409, 24429, 24434, 24431, 24408, 24406, - 24405, 24430, 24428, 24427, 24392, 24394, 24414, 24416, 24396, 24418, - 35762, 35762, 35762, 35762, 24456, 24441, 24445, 24451, 24454, 24457, - 24443, 24447, 24448, 24452, 24444, 24442, 24455, 24450, 24449, 24453, - 24446, 24391, 24386, 24385, 24390, 24389, 24388, 24387, 24438, 24439, - 35762, 35762, 35762, 35762, 35762, 35762, 24464, 24466, 24463, 24462, - 24459, 24458, 24461, 24460, 24467, 24465, 24440, 35762, 35762, 35762, - 24436, 24437, 17836, 17855, 17848, 17851, 17853, 17846, 17844, 17842, - 17838, 17840, 17825, 17823, 17815, 17819, 17821, 17817, 17849, 17854, - 17847, 17850, 17852, 17845, 17843, 17841, 17837, 17839, 17824, 17822, - 17814, 17818, 17820, 17816, 4658, 4655, 4647, 4646, 4660, 4652, 4645, - 4644, 4663, 4654, 4651, 4650, 4653, 4657, 4649, 4648, 4665, 4661, 4659, - 4664, 4662, 4666, 4656, 4669, 4671, 4668, 4670, 4667, 35762, 35762, 4673, - 4672, 30159, 30157, 30158, 30175, 30174, 30173, 30199, 30165, 30164, - 30178, 30185, 30177, 30200, 30193, 30160, 30205, 30176, 30192, 30169, - 30168, 30182, 30181, 30201, 30204, 30167, 30166, 30170, 30180, 30183, - 30179, 30206, 30186, 30172, 30191, 30194, 30187, 30189, 30207, 30161, - 30162, 30163, 30171, 30190, 30208, 30184, 30197, 30198, 30195, 30196, - 30203, 30202, 30188, 30156, 30131, 30130, 30128, 30219, 30126, 30127, - 30129, 30132, 30133, 30134, 35762, 30227, 30234, 30238, 30235, 30244, - 30250, 30251, 30249, 30248, 30246, 30247, 30239, 30240, 30243, 30252, - 30236, 30242, 30237, 30245, 30241, 30218, 30230, 30231, 30211, 30212, - 30213, 30222, 30221, 30214, 35762, 35762, 30135, 30142, 30144, 30141, - 30140, 30137, 30136, 30139, 30138, 30145, 30143, 35762, 35762, 35762, - 35762, 35762, 35762, 30152, 30154, 30151, 30150, 30147, 30146, 30149, - 30148, 30155, 30153, 35762, 35762, 35762, 35762, 35762, 35762, 30228, - 30229, 30226, 30217, 30210, 30232, 30223, 30220, 30215, 30216, 30224, - 30225, 30209, 30233, 35762, 35762, 7898, 7866, 7982, 7900, 8132, 8145, - 8144, 8084, 7881, 8061, 8120, 8090, 7885, 8089, 8088, 8030, 8024, 8047, - 8106, 8048, 8107, 8118, 8077, 7981, 8093, 7884, 7883, 8130, 8008, 8009, - 8010, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 2698, 2697, 2700, 2696, 2699, 2608, 2609, 2625, 2626, 2629, 2630, 2647, - 2648, 2638, 2639, 2622, 2612, 2627, 2628, 2633, 2635, 2623, 2624, 2642, - 2615, 2616, 2631, 2632, 2643, 2654, 2653, 2619, 2618, 2641, 2652, 2655, - 2617, 2620, 2640, 2644, 2645, 2613, 2614, 2660, 2662, 2646, 2637, 2661, - 2650, 2651, 2649, 2659, 2701, 2712, 2715, 2716, 2706, 2707, 2704, 2705, - 2702, 2703, 2708, 2709, 2711, 2710, 2713, 2714, 2717, 2634, 2636, 2656, - 2621, 2657, 2658, 2610, 2611, 35762, 35762, 35762, 2725, 2727, 2724, - 2723, 2720, 2719, 2722, 2721, 2728, 2726, 2694, 2691, 2718, 2605, 2607, - 2606, 2693, 2680, 2678, 2681, 2672, 2673, 2679, 2675, 2677, 2676, 2674, - 2670, 2669, 2665, 2663, 2667, 2666, 2664, 2668, 2671, 2690, 2689, 2688, - 2687, 2682, 2684, 2685, 2686, 2683, 2695, 2692, 35762, 29705, 29703, - 29704, 29656, 29691, 29693, 29658, 29692, 29668, 29669, 29676, 29684, - 29679, 29670, 29677, 29681, 29690, 29671, 29685, 29678, 29672, 29683, - 29661, 29686, 29674, 29682, 29689, 29665, 29663, 29687, 29667, 29688, - 29680, 29651, 29652, 29653, 29711, 29710, 29712, 29709, 29707, 29708, - 29702, 29706, 29654, 29655, 29675, 29666, 29719, 29721, 29718, 29717, - 29714, 29713, 29716, 29715, 29722, 29720, 29650, 29664, 29662, 29673, - 29659, 29660, 3488, 3474, 3482, 3466, 3454, 3478, 3477, 3463, 3469, 3462, - 3455, 3486, 3472, 3464, 3481, 3465, 3483, 3480, 3485, 3470, 3453, 3468, - 3475, 3458, 3476, 3471, 3456, 3487, 3473, 3460, 3484, 3467, 3461, 3479, - 3459, 3457, 3489, 3490, 3497, 3503, 3500, 3504, 3505, 3501, 3506, 3502, - 3498, 3499, 3451, 3452, 3491, 3492, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 3496, 3494, 3495, 3493, 19688, 19687, 19686, 19700, - 19699, 19705, 19715, 19714, 19718, 19706, 19713, 19712, 19694, 19707, - 19691, 19690, 19689, 19698, 19697, 19696, 19695, 19704, 19703, 19709, - 19708, 19693, 19723, 19720, 19719, 19702, 19701, 19721, 19717, 19716, - 19722, 19724, 19733, 19732, 19738, 19740, 19736, 19737, 19734, 19735, - 19739, 19677, 19682, 19681, 19679, 19683, 19684, 19685, 19680, 19678, - 19731, 19730, 35762, 35762, 35762, 19725, 19728, 19729, 19727, 19726, - 19747, 19749, 19746, 19745, 19742, 19741, 19744, 19743, 19750, 19748, - 35762, 35762, 35762, 19711, 19710, 19692, 25311, 25313, 25310, 25309, - 25306, 25305, 25308, 25307, 25314, 25312, 25284, 25275, 25273, 25272, - 25274, 25285, 25269, 25268, 25270, 25271, 25287, 25283, 25281, 25280, - 25282, 25289, 25295, 25296, 25294, 25297, 25286, 25279, 25276, 25278, - 25277, 25288, 25290, 25291, 25293, 25292, 25299, 25300, 25298, 25304, - 25303, 25315, 25302, 25301, 10125, 10102, 10108, 10164, 10146, 10154, - 10144, 10145, 10163, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 13460, 13491, 13468, 13497, 13466, 13496, 13489, 13486, 13498, 13478, - 13480, 13492, 13494, 13499, 13482, 13488, 13490, 13484, 13487, 13500, - 13481, 13477, 13467, 13495, 13483, 13461, 13464, 13476, 13463, 13462, - 13493, 13475, 13471, 13474, 13472, 13502, 13469, 13473, 13503, 13501, - 13465, 13485, 13459, 35762, 35762, 13458, 13470, 13479, 29701, 29699, - 29700, 29698, 29697, 29696, 29695, 29694, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 33625, 33638, 33627, 33605, 33619, 33633, - 33634, 33635, 33620, 33636, 33623, 33631, 33626, 33624, 33632, 33630, - 33629, 33637, 33618, 33616, 33607, 33614, 33606, 33617, 33615, 33596, - 33597, 33599, 33600, 33612, 33610, 33611, 33609, 33598, 33602, 33608, - 33621, 33604, 33613, 33601, 33628, 33622, 33603, 35762, 35762, 35762, - 35762, 35762, 18684, 18685, 19290, 18681, 18686, 18687, 18658, 18657, - 19275, 19268, 18690, 18691, 18664, 18692, 18670, 18665, 18666, 19225, - 19226, 19227, 19270, 18668, 19262, 18784, 18694, 18671, 18679, 18674, - 18697, 19230, 19229, 19228, 18698, 18699, 18701, 18659, 18712, 18646, - 14004, 14007, 14003, 14006, 14002, 9995, 23018, 23019, 23009, 23010, - 23021, 23022, 23012, 23024, 23014, 23025, 23026, 23027, 23028, 23029, - 23030, 23013, 23016, 23017, 23031, 23011, 23033, 23034, 23036, 23167, - 23274, 23168, 23276, 23171, 23196, 23210, 23264, 23249, 23279, 23217, - 23287, 23298, 23227, 23214, 23247, 23250, 23271, 23173, 23251, 23266, - 23291, 23265, 23277, 23295, 23169, 23175, 23218, 23201, 23219, 23195, - 19431, 19439, 19441, 19442, 14213, 14209, 14212, 14211, 14210, 19351, - 18770, 18819, 18905, 19047, 19067, 19144, 19171, 19164, 19209, 19245, - 19413, 19297, 23085, 18976, 19255, 18714, 18983, 19140, 18715, 19350, - 18771, 18823, 18906, 18919, 19002, 19029, 19048, 19073, 19145, 19174, - 19210, 18892, 19358, 19385, 19414, 18733, 18759, 18822, 18882, 19133, - 19181, 19218, 18973, 19129, 18894, 19337, 18900, 23275, 23176, 23194, - 23212, 23258, 23215, 23202, 23263, 23286, 23229, 23230, 23177, 23179, - 23232, 23236, 23238, 23182, 23228, 23278, 23246, 23245, 23188, 23172, - 23252, 23262, 23211, 23267, 23293, 23294, 23190, 23297, 23288, 23207, - 23209, 23208, 23213, 23290, 7874, 7873, 8122, 8121, 8074, 7965, 8073, - 7758, 7964, 7757, 8022, 7770, 8075, 7897, 8086, 8113, 7983, 8137, 8138, - 8005, 7996, 7997, 7998, 8000, 8007, 8003, 8032, 7988, 8034, 8011, 7989, - 7990, 8036, 7991, 7992, 8021, 8025, 8013, 8040, 7995, 8027, 8028, 8026, - 8004, 8012, 8016, 8037, 8002, 8019, 8029, 7994, 8015, 8018, 8136, 7987, - 7986, 7869, 8142, 7872, 7864, 7878, 7767, 8042, 8097, 18174, 18731, - 18210, 18773, 18209, 18772, 18208, 18769, 18217, 18788, 18242, 18825, - 18241, 18824, 18240, 18820, 18236, 18814, 18237, 18815, 18268, 18868, - 18269, 18869, 18257, 18856, 18266, 18866, 18248, 18847, 18293, 18908, - 18301, 18918, 18320, 18938, 18319, 18937, 18317, 18935, 18314, 18932, - 18313, 18931, 18337, 18965, 18331, 18953, 18368, 19000, 18365, 18997, - 18369, 19001, 18376, 19014, 18377, 19015, 18387, 19024, 18383, 19011, - 18391, 19045, 18393, 19050, 18392, 19049, 18411, 19072, 18410, 19071, - 18405, 19064, 18401, 19059, 18442, 19108, 18441, 19107, 18419, 19100, - 18420, 19101, 18470, 19143, 18472, 19147, 18481, 19161, 18479, 19159, - 18480, 19160, 18486, 19166, 18506, 19204, 18504, 19202, 18503, 19195, - 18498, 19197, 18505, 19203, 18526, 19243, 18525, 19242, 18527, 19246, - 18521, 19236, 18557, 19318, 18578, 19341, 18550, 19311, 18577, 19340, - 18569, 19330, 18590, 19361, 18589, 19357, 18603, 19374, 18604, 19375, - 18600, 19371, 18602, 19373, 18601, 19372, 18609, 19380, 18608, 19379, - 18614, 19390, 18625, 19404, 18629, 19408, 18631, 19410, 18939, 19241, - 19376, 19400, 18732, 19038, 19037, 19039, 18515, 18829, 18168, 18725, - 18184, 18743, 18180, 18739, 18179, 18738, 18178, 18737, 18182, 18741, - 18181, 18740, 18163, 18720, 18162, 18719, 18161, 18718, 18165, 18722, - 18164, 18721, 18262, 18861, 18273, 18874, 18265, 18865, 18253, 18852, - 18252, 18851, 18251, 18850, 18256, 18855, 18255, 18854, 18338, 18966, - 18328, 18957, 18433, 19093, 18453, 19119, 18425, 19085, 18424, 19084, - 18423, 19083, 18427, 19087, 18426, 19086, 18450, 19116, 18449, 19115, - 18448, 19114, 18452, 19118, 18451, 19117, 18560, 19321, 18567, 19328, - 18564, 19325, 18563, 19324, 18562, 19323, 18566, 19327, 18565, 19326, - 18615, 19391, 18613, 19389, 18617, 19393, 18621, 19399, 18396, 19053, - 18397, 19054, 18618, 19394, 14051, 14043, 14056, 14048, 14052, 14044, - 14054, 14046, 13817, 13809, 13820, 13812, 13818, 13810, 13822, 13814, - 14074, 14071, 14076, 14073, 14075, 14072, 35762, 35762, 13857, 13854, - 13859, 13856, 13858, 13855, 35762, 35762, 14089, 14081, 14094, 14086, - 14090, 14082, 14092, 14084, 13841, 13833, 13844, 13836, 13842, 13834, - 13846, 13838, 14115, 14106, 14118, 14109, 14117, 14108, 14116, 14107, - 13869, 13864, 13872, 13867, 13871, 13866, 13870, 13865, 14176, 14173, - 14178, 14175, 14177, 14174, 35762, 35762, 13903, 13900, 13905, 13902, - 13904, 13901, 35762, 35762, 14135, 14126, 14138, 14129, 14137, 14128, - 14136, 14127, 35762, 13915, 35762, 13918, 35762, 13917, 35762, 13916, - 14156, 14148, 14161, 14153, 14157, 14149, 14159, 14151, 13887, 13879, - 13890, 13882, 13888, 13880, 13892, 13884, 14041, 14061, 14078, 14077, - 14101, 14099, 14121, 14122, 14180, 14179, 14141, 14142, 14168, 14166, - 35762, 35762, 14058, 14050, 14057, 14049, 14053, 14045, 14055, 14047, - 13824, 13816, 13821, 13813, 13819, 13811, 13823, 13815, 14096, 14088, - 14095, 14087, 14091, 14083, 14093, 14085, 13848, 13840, 13845, 13837, - 13843, 13835, 13847, 13839, 14163, 14155, 14162, 14154, 14158, 14150, - 14160, 14152, 13894, 13886, 13891, 13883, 13889, 13881, 13893, 13885, - 14040, 14065, 14042, 14063, 14062, 35762, 14059, 14060, 13826, 13830, - 13827, 13828, 13825, 14000, 14028, 14029, 14033, 13949, 14102, 14103, - 14100, 35762, 14097, 14098, 13861, 13860, 13851, 13850, 13849, 14032, - 14031, 14030, 14120, 14124, 14113, 14112, 35762, 35762, 14119, 14111, - 13873, 13877, 13874, 13875, 35762, 13957, 13956, 13955, 14140, 14144, - 14133, 14132, 14188, 14187, 14139, 14131, 13920, 13924, 13921, 13922, - 13910, 13951, 13950, 14227, 35762, 35762, 14169, 14170, 14167, 35762, - 14164, 14165, 13907, 13906, 13897, 13896, 13895, 14025, 13954, 35762, - 12447, 12444, 12449, 12446, 32044, 13184, 28853, 13117, 26923, 32017, - 14524, 35568, 35567, 35569, 19570, 27221, 15992, 24652, 13116, 12448, - 12445, 15946, 10941, 10915, 19515, 27171, 28728, 28726, 19475, 27145, - 10914, 10909, 10223, 10908, 4694, 32505, 25769, 32572, 15964, 15995, - 19876, 26269, 19569, 27220, 26798, 19568, 27219, 24257, 26502, 26503, - 26884, 10923, 32519, 27088, 27082, 27096, 5706, 28727, 28729, 27105, - 10945, 16335, 26084, 32621, 5992, 5707, 2513, 15994, 13188, 19523, 27182, - 10946, 26947, 13036, 32418, 27087, 3828, 3867, 20532, 27093, 7742, 32569, - 8148, 29770, 16351, 13152, 32024, 26943, 13177, 13139, 32573, 13178, - 10887, 32530, 33645, 22296, 34177, 13310, 16353, 16355, 16354, 35762, - 19567, 27218, 13132, 26797, 16249, 7, 16248, 6, 24252, 24650, 29741, - 29729, 35762, 35762, 29736, 29735, 29738, 29737, 29728, 29742, 29732, - 29733, 29727, 29731, 29734, 29730, 29615, 29617, 29614, 29613, 29610, - 29609, 29612, 29611, 29604, 29616, 29605, 29606, 29603, 29607, 29608, - 35762, 19428, 19429, 19437, 19443, 19427, 19430, 19433, 19434, 19435, - 19436, 19438, 19426, 19440, 35762, 35762, 35762, 13033, 7755, 8394, - 13194, 20484, 22903, 24185, 26526, 27538, 34181, 24384, 10881, 13034, - 18028, 32539, 11052, 13505, 26527, 14277, 2524, 16001, 5825, 20485, - 29183, 31790, 16239, 32540, 24700, 20985, 27407, 18155, 3742, 29165, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 8051, 8105, 8065, 8117, 7763, 7779, - 8044, 8102, 8110, 7778, 7762, 8124, 7910, 7901, 7904, 7907, 7902, 8054, - 7903, 7905, 7906, 8094, 7891, 7764, 8131, 8143, 8056, 8062, 8109, 8055, - 8043, 8101, 7766, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 0, 100, 10957, 10247, - 5829, 5708, 4959, 13035, 27726, 10249, 27723, 27715, 3917, 10958, 26688, - 26689, 27716, 3918, 27717, 27724, 20657, 10959, 24743, 29081, 27719, - 10955, 10960, 27720, 3919, 10961, 26860, 27066, 27750, 31685, 32492, - 33639, 10962, 26078, 26086, 16350, 3920, 32563, 17143, 951, 27712, 3916, - 12503, 27722, 27713, 27714, 32547, 27718, 27725, 346, 3633, 13508, 10236, - 16246, 27395, 13096, 10969, 10968, 10954, 10956, 10970, 32556, 32557, - 27092, 32558, 10967, 10963, 10964, 10965, 10966, 26888, 32542, 26504, - 2584, 32560, 29846, 33781, 33779, 33783, 33784, 33789, 33777, 33788, - 33787, 33775, 33782, 33774, 33776, 33785, 33773, 33790, 13189, 27365, - 27376, 27377, 27364, 27361, 27370, 27372, 27380, 27381, 27373, 27379, - 27375, 27358, 27366, 27362, 27368, 28915, 28919, 28920, 28914, 28911, - 28923, 28922, 28910, 28924, 28921, 28909, 28918, 28913, 28916, 28912, - 28917, 27369, 27363, 27374, 27378, 19187, 27371, 27360, 27359, 27367, - 33791, 32551, 32550, 35762, 35762, 35762, 35762, 19571, 32718, 27223, - 10989, 19497, 32643, 24682, 24656, 29055, 29069, 19591, 27245, 19654, - 27322, 19651, 32768, 27321, 11022, 19594, 27248, 19602, 32731, 27230, - 11003, 32644, 19600, 27254, 19587, 27241, 19505, 19500, 11026, 32728, - 32729, 10998, 10999, 27237, 10990, 960, 7726, 24684, 19582, 965, 7728, - 19620, 19614, 32745, 32742, 27282, 27276, 11039, 11036, 27256, 32720, - 19604, 19666, 32771, 27288, 11047, 19621, 27271, 19657, 19504, 27264, - 19655, 32735, 27262, 11029, 19502, 32646, 24694, 24668, 29066, 29078, - 19641, 27312, 19670, 27292, 32721, 10991, 19661, 32736, 27268, 11030, - 19581, 27236, 19652, 32772, 27323, 11024, 32773, 32775, 32776, 32777, - 32779, 32780, 27324, 24683, 29058, 32648, 27132, 11001, 32032, 19601, - 27255, 19499, 19585, 27239, 19498, 19665, 27287, 19507, 13172, 8153, - 26414, 32012, 32011, 12437, 16167, 24141, 12386, 24703, 28895, 8161, - 10707, 28888, 12451, 24100, 24088, 24092, 22907, 22914, 10882, 10689, - 27755, 2512, 27333, 4695, 29403, 8400, 13183, 26887, 16240, 27122, 947, - 22170, 29184, 10693, 10708, 26273, 24708, 20493, 20500, 16328, 32623, - 16313, 10912, 32529, 8167, 29763, 33769, 7729, 7720, 962, 32013, 3634, - 26977, 26886, 10883, 13038, 13330, 15982, 32322, 27094, 16347, 28848, - 34185, 24713, 22913, 2517, 24704, 1043, 1046, 24340, 351, 24705, 353, - 32520, 348, 12489, 13328, 10699, 1047, 13329, 1044, 16130, 7754, 12487, - 27331, 27332, 8346, 12504, 12490, 29565, 10252, 12473, 22181, 26949, - 24715, 16006, 24716, 29596, 19770, 13741, 19772, 13744, 19761, 13734, - 23857, 23856, 3632, 24714, 24718, 24717, 24345, 24342, 19769, 13740, - 24344, 24341, 19771, 13742, 24346, 24343, 26861, 29633, 26870, 29642, - 26869, 29641, 10710, 10713, 29621, 29749, 24701, 24702, 29627, 29755, - 24338, 24339, 29626, 29754, 23611, 23612, 23613, 29276, 29365, 29278, - 29367, 29214, 29221, 6503, 6449, 6508, 6241, 6254, 6504, 6230, 6513, - 6255, 29532, 29525, 29546, 29480, 27197, 19537, 10976, 32652, 2518, - 22922, 32536, 13173, 32523, 10939, 10712, 24712, 10715, 24337, 26871, - 29643, 24698, 8162, 24699, 8163, 25799, 16129, 23614, 15761, 16336, - 34206, 24186, 24655, 27128, 27193, 24097, 24098, 24099, 24093, 10529, - 10884, 29567, 10691, 4099, 19549, 27158, 19513, 27169, 27095, 9642, 9641, - 10933, 10932, 10911, 10936, 26682, 12474, 19775, 13750, 33682, 33681, - 19759, 13745, 12472, 12471, 12469, 12470, 10711, 10714, 24709, 24710, - 29277, 29366, 19760, 13733, 26868, 29640, 24706, 10705, 24707, 10706, - 33644, 22899, 32651, 10979, 12387, 12389, 28896, 12392, 12391, 28897, - 12390, 12388, 8164, 8165, 28889, 8166, 28890, 35480, 10530, 12384, 15976, - 32636, 10980, 26885, 26521, 33954, 19471, 27140, 19548, 27149, 4090, - 4087, 32441, 32438, 27085, 29309, 2508, 27736, 27732, 31683, 26806, - 33693, 26685, 32554, 33945, 15974, 32437, 32440, 4086, 4089, 32434, 4080, - 13198, 28954, 32637, 25791, 12500, 34190, 17145, 19564, 27213, 12499, - 3630, 10218, 349, 29856, 32456, 10700, 8172, 28880, 8348, 8349, 991, - 1029, 1015, 1003, 1004, 1020, 1001, 971, 976, 1026, 1031, 1017, 1016, - 1011, 1018, 999, 1023, 1012, 1019, 974, 983, 982, 1005, 1008, 984, 1037, - 1010, 1034, 980, 1009, 1007, 1035, 987, 1006, 1022, 981, 988, 997, 975, - 1036, 1021, 972, 1002, 1033, 978, 1027, 996, 973, 985, 998, 1039, 1038, - 977, 979, 1040, 1028, 1025, 1014, 1013, 986, 1032, 989, 1024, 994, 993, - 1030, 990, 995, 992, 24719, 27127, 27937, 3526, 33656, 16312, 8170, - 10615, 12443, 8152, 34095, 12465, 354, 15472, 6285, 6507, 4635, 32622, - 23495, 15998, 25787, 25788, 26426, 26427, 10610, 28971, 1000, 10244, - 26872, 24569, 26874, 7749, 19552, 19553, 19551, 27165, 27166, 27164, - 19520, 19527, 19519, 27179, 27186, 27178, 19469, 19467, 19468, 9644, - 27138, 27136, 27137, 16323, 15947, 32688, 32704, 29646, 29644, 32443, - 4081, 4082, 26958, 19555, 27199, 15955, 15956, 15957, 15958, 10263, - 10266, 10267, 10256, 10260, 10268, 10257, 10261, 10264, 10255, 10259, - 10254, 10258, 10262, 10265, 29244, 27067, 13061, 33653, 22739, 22731, - 22738, 22732, 22736, 22737, 22735, 22734, 22733, 11219, 13313, 32436, - 4085, 32429, 4084, 32444, 4092, 34104, 3631, 29592, 13145, 8, 12385, - 10245, 3856, 3821, 3900, 3804, 3857, 3822, 3859, 357, 29590, 32330, - 15975, 3840, 3843, 3845, 3837, 10937, 3879, 3751, 26820, 26817, 26818, - 26819, 25220, 29841, 29849, 29850, 29830, 29828, 29831, 29842, 29816, - 29817, 29835, 29837, 29836, 29834, 29818, 29847, 29848, 29820, 29824, - 29823, 29822, 29821, 29839, 29853, 29829, 29819, 29827, 29851, 29832, - 29833, 29844, 29843, 29845, 29854, 29825, 3924, 25774, 29840, 29826, - 29852, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 25232, 25227, 25230, 25231, - 25224, 25225, 25223, 25222, 25229, 25226, 25228, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 6251, 6248, 6247, - 6244, 6243, 6246, 6245, 6252, 6250, 6472, 6452, 6497, 6485, 6467, 6455, - 6471, 6469, 6451, 6498, 6486, 26360, 26358, 26357, 26354, 26353, 26356, - 26355, 26361, 26359, 26352, 26343, 26351, 26349, 26344, 26345, 26347, - 26348, 26342, 26346, 26350, 10550, 10562, 10559, 10544, 10541, 10556, - 10553, 10568, 10547, 24739, 24732, 24740, 24738, 24733, 24734, 24735, - 24737, 24731, 24742, 24741, 26388, 26389, 26390, 26391, 26392, 26393, - 26394, 26395, 26396, 26397, 26398, 26399, 26400, 26401, 26402, 26403, - 26404, 26405, 26406, 26407, 26408, 26409, 26410, 26411, 26412, 26413, - 6395, 6396, 6397, 6398, 6399, 6400, 6401, 6402, 6403, 6404, 6405, 6406, - 6407, 6408, 6409, 6410, 6411, 6412, 6413, 6414, 6415, 6416, 6417, 6418, - 6419, 6420, 6421, 6422, 6423, 6424, 6425, 6426, 6427, 6428, 6429, 6430, - 6431, 6432, 6433, 6434, 6435, 6436, 6437, 6438, 6439, 6440, 6441, 6442, - 6443, 6444, 6445, 6446, 6249, 24266, 24264, 24262, 24267, 24268, 24270, - 24271, 24265, 24269, 24263, 10903, 10901, 10900, 10897, 10896, 10899, - 10898, 10904, 10902, 10895, 24261, 4206, 4153, 4202, 4149, 4196, 4148, - 4195, 4147, 4194, 4146, 4193, 4145, 4190, 4123, 4117, 4144, 4189, 4121, - 4115, 4143, 4201, 4227, 4221, 4138, 4200, 4225, 4219, 4136, 4205, 4243, - 4220, 4116, 4240, 4122, 4226, 4152, 4204, 4242, 4218, 4114, 4239, 4120, - 4224, 4151, 4188, 4157, 4211, 4125, 4119, 4214, 4160, 4142, 4199, 4156, - 4210, 4228, 4222, 4213, 4159, 4137, 4203, 4158, 4212, 4241, 4217, 4118, - 4238, 4162, 4216, 4155, 4209, 4124, 4223, 4215, 4161, 4150, 4186, 4140, - 4185, 4139, 4113, 4109, 4130, 4127, 4105, 4129, 4126, 4104, 4233, 4230, - 4108, 4232, 4229, 4107, 4245, 4236, 4112, 4244, 4235, 4111, 4131, 4128, - 4103, 4234, 4231, 4106, 4246, 4237, 4110, 4164, 4163, 4165, 4166, 4182, - 4181, 4183, 4191, 4197, 4208, 4187, 4132, 4134, 4154, 4141, 4192, 4198, - 4133, 4135, 32667, 20583, 20582, 20585, 20538, 20581, 20586, 20584, - 13205, 19514, 19535, 19546, 19485, 19534, 19495, 19496, 27152, 19790, - 22298, 10230, 32691, 27161, 26940, 26941, 26932, 26933, 26934, 26935, - 26936, 26937, 26938, 26939, 3877, 34082, 34091, 34085, 29418, 29427, - 29417, 29424, 29426, 29416, 3875, 34079, 3871, 34073, 3908, 34115, 3853, - 34063, 3902, 34111, 3901, 34110, 3858, 34070, 3863, 34069, 3862, 34068, - 3806, 34028, 3805, 34027, 3826, 34054, 3825, 34053, 3824, 34052, 3796, - 34017, 34018, 13136, 20588, 34004, 10886, 6224, 4698, 3747, 6216, 6225, - 6217, 6222, 6221, 6223, 19483, 27150, 16343, 16341, 32670, 20536, 32689, - 32706, 20579, 20559, 32672, 20539, 3833, 3832, 3903, 3904, 33970, 29421, - 29428, 29423, 29420, 34094, 34112, 32659, 32660, 18152, 34092, 34088, - 34089, 34093, 34011, 34009, 34010, 34012, 32687, 32711, 20557, 34059, - 3850, 34058, 3849, 20564, 3881, 7743, 32616, 28961, 8155, 3890, 34101, - 19800, 32049, 29647, 2509, 10270, 8173, 25795, 3896, 34103, 2731, 2737, - 2738, 27545, 32618, 15970, 34076, 3888, 27911, 27090, 3831, 3869, 34051, - 34108, 34072, 34026, 28859, 5822, 26956, 3744, 4958, 970, 25895, 6179, - 8381, 8380, 29568, 13106, 102, 14650, 26492, 35478, 32508, 32509, 32514, - 32511, 32513, 32512, 32510, 32507, 33966, 34038, 34080, 3876, 34099, - 13129, 18156, 22729, 13112, 11215, 20893, 16465, 27615, 32781, 24571, - 26787, 2507, 31672, 13327, 5696, 19675, 33695, 20488, 27708, 27543, 5702, - 2585, 26680, 33976, 33992, 33995, 33971, 33979, 33989, 3767, 3783, 3786, - 3762, 3770, 3780, 3889, 34041, 34022, 3795, 34081, 3816, 3801, 34013, - 15971, 26945, 12340, 3512, 3513, 23616, 23615, 23617, 33964, 11220, - 32632, 26984, 26985, 26986, 26987, 26988, 26989, 26990, 26991, 3906, - 26983, 26419, 26524, 33967, 10532, 10533, 10534, 10535, 10536, 10537, - 34006, 34008, 3748, 3750, 23492, 23493, 10572, 10575, 10574, 10573, - 34032, 3812, 14651, 968, 8389, 29561, 27703, 345, 13151, 13323, 29562, - 2520, 13149, 26075, 32026, 32025, 33939, 15829, 10972, 10973, 16327, - 20892, 20891, 20890, 33657, 15965, 22307, 22284, 22297, 21059, 10694, - 32634, 8372, 13311, 24381, 5832, 26228, 16466, 33685, 6182, 3851, 27758, - 27748, 26951, 27752, 28966, 3449, 29494, 34029, 34030, 3809, 3810, 28962, - 29649, 26961, 3883, 32048, 32546, 32545, 34023, 8390, 10614, 25794, - 26665, 5764, 15471, 6237, 5833, 24643, 355, 3897, 34106, 3829, 34049, - 11061, 15328, 19472, 29537, 13100, 3894, 27063, 27064, 2516, 15256, - 26499, 27211, 19562, 16348, 3760, 27916, 6214, 5824, 15934, 13324, 13325, - 20988, 23510, 32617, 13185, 13146, 13114, 27699, 29243, 28857, 16005, - 26511, 31791, 16363, 15232, 13312, 9637, 34033, 3884, 32665, 3887, 20580, - 34074, 34042, 31684, 31671, 226, 12462, 26972, 26966, 33687, 34188, - 20578, 26501, 32705, 34062, 3914, 5998, 15254, 23610, 15290, 2741, 15246, - 26077, 15336, 25778, 15292, 18638, 27761, 26074, 20894, 29566, 13181, - 13179, 15275, 13175, 3813, 34037, 29173, 29594, 6510, 25776, 3761, 26076, - 15294, 26677, 27763, 15245, 25777, 12339, 12336, 12334, 28851, 12335, - 15268, 32567, 28854, 31677, 25775, 15320, 28849, 3811, 34034, 12337, - 6499, 15319, 28964, 32320, 15253, 29172, 15313, 2730, 12338, 15270, 8386, - 27762, 24329, 20577, 32703, 20562, 32708, 3915, 34066, 34031, 3798, - 15272, 19797, 22304, 15335, 15303, 15304, 15264, 15265, 15287, 15286, - 9649, 15273, 15279, 15249, 27392, 13150, 27391, 22291, 22294, 22285, - 22286, 22292, 22295, 15283, 15296, 15282, 15295, 19789, 19787, 22290, - 22293, 10597, 10595, 10594, 10591, 10590, 10593, 10592, 10598, 10596, - 10589, 10608, 10605, 10604, 10601, 10600, 10603, 10602, 10609, 10607, - 10599, 10587, 10584, 10583, 10580, 10579, 10582, 10581, 10588, 10586, - 10578, 15331, 15334, 15291, 15261, 15309, 15297, 15316, 11053, 15300, - 32502, 15322, 10233, 15260, 3864, 32040, 32043, 3865, 15247, 15248, - 29555, 15259, 27216, 19557, 2597, 13196, 15288, 15327, 24721, 9643, - 24723, 6286, 34117, 3921, 3923, 3922, 15250, 15252, 15251, 31678, 15321, - 33960, 15332, 25789, 10905, 32023, 34105, 26505, 25785, 25784, 19512, - 27168, 25896, 27074, 29760, 33643, 21756, 20508, 21577, 29522, 29523, - 34021, 969, 12394, 20576, 32684, 19494, 27159, 13204, 18144, 18143, - 19453, 19452, 19493, 20523, 20512, 32653, 20589, 34014, 34015, 34016, - 34087, 34090, 21757, 21751, 21760, 21754, 21759, 21753, 21758, 21752, - 21761, 21755, 32749, 11043, 964, 7722, 27131, 20513, 20518, 20511, 20515, - 20520, 20510, 20514, 20519, 20516, 20521, 20522, 4624, 4369, 4497, 4370, - 4561, 4434, 4498, 4371, 4593, 4466, 4530, 4403, 4562, 4435, 4499, 4372, - 4609, 4482, 4546, 4419, 4578, 4451, 4515, 4388, 4594, 4467, 4531, 4404, - 4563, 4436, 4500, 4373, 4617, 4490, 4554, 4427, 4586, 4459, 4523, 4396, - 4602, 4475, 4539, 4412, 4571, 4444, 4508, 4381, 4610, 4483, 4547, 4420, - 4579, 4452, 4516, 4389, 4595, 4468, 4532, 4405, 4564, 4437, 4501, 4374, - 4621, 4494, 4558, 4431, 4590, 4463, 4527, 4400, 4606, 4479, 4543, 4416, - 4575, 4448, 4512, 4385, 4614, 4487, 4551, 4424, 4583, 4456, 4520, 4393, - 4599, 4472, 4536, 4409, 4568, 4441, 4505, 4378, 4618, 4491, 4555, 4428, - 4587, 4460, 4524, 4397, 4603, 4476, 4540, 4413, 4572, 4445, 4509, 4382, - 4611, 4484, 4548, 4421, 4580, 4453, 4517, 4390, 4596, 4469, 4533, 4406, - 4565, 4438, 4502, 4375, 4623, 4496, 4560, 4433, 4592, 4465, 4529, 4402, - 4608, 4481, 4545, 4418, 4577, 4450, 4514, 4387, 4616, 4489, 4553, 4426, - 4585, 4458, 4522, 4395, 4601, 4474, 4538, 4411, 4570, 4443, 4507, 4380, - 4620, 4493, 4557, 4430, 4589, 4462, 4526, 4399, 4605, 4478, 4542, 4415, - 4574, 4447, 4511, 4384, 4613, 4486, 4550, 4423, 4582, 4455, 4519, 4392, - 4598, 4471, 4535, 4408, 4567, 4440, 4504, 4377, 4622, 4495, 4559, 4432, - 4591, 4464, 4528, 4401, 4607, 4480, 4544, 4417, 4576, 4449, 4513, 4386, - 4615, 4488, 4552, 4425, 4584, 4457, 4521, 4394, 4600, 4473, 4537, 4410, - 4569, 4442, 4506, 4379, 4619, 4492, 4556, 4429, 4588, 4461, 4525, 4398, - 4604, 4477, 4541, 4414, 4573, 4446, 4510, 4383, 4612, 4485, 4549, 4422, - 4581, 4454, 4518, 4391, 4597, 4470, 4534, 4407, 4566, 4439, 4503, 4376, - 27318, 27317, 19656, 27263, 19503, 27319, 19658, 27265, 11000, 32730, - 32767, 11021, 19660, 27267, 19640, 27311, 27320, 27238, 32732, 11004, - 27250, 27249, 27313, 27315, 27314, 19605, 27257, 19659, 27266, 19583, - 27235, 19603, 27231, 24681, 24661, 24687, 24660, 29059, 29071, 24686, - 24659, 29056, 29070, 27337, 13098, 29057, 24657, 13099, 27338, 24658, - 24685, 33948, 2500, 2499, 2502, 2503, 27217, 19556, 32426, 4083, 32428, - 32427, 20560, 20555, 961, 7719, 27226, 19572, 27924, 27242, 19588, 27234, - 19501, 32770, 19462, 19461, 32641, 32640, 19463, 32642, 19460, 32639, - 19619, 27281, 32744, 11038, 19613, 27275, 32741, 11035, 19618, 27280, - 32743, 11037, 19612, 27274, 32740, 11034, 19615, 32739, 27279, 11032, - 19617, 19611, 27277, 27272, 19616, 19610, 27278, 27273, 32738, 11033, - 27143, 12479, 32324, 19576, 27228, 27227, 19756, 19579, 13729, 29620, - 19578, 29746, 19547, 27148, 32650, 10978, 32522, 35491, 35492, 19539, - 27201, 19541, 27203, 35486, 35482, 35487, 35483, 19524, 27183, 19522, - 27180, 19521, 27181, 19456, 27124, 19457, 27130, 10918, 10943, 19465, - 27134, 10893, 33670, 22179, 27129, 22180, 948, 5, 29185, 29186, 32543, - 27080, 949, 27081, 25218, 25217, 22178, 22177, 22172, 22171, 22176, - 22174, 22175, 22173, 27103, 12441, 12440, 12438, 12439, 6226, 6514, 6501, - 6505, 6502, 6227, 6219, 6228, 32635, 6509, 6232, 6447, 6515, 6218, 6220, - 29486, 29481, 29552, 29538, 29539, 32578, 32501, 32499, 27540, 32498, - 27194, 19529, 33640, 4100, 4101, 3913, 32331, 32332, 34046, 3820, 19544, - 27206, 19473, 27144, 16164, 32258, 16241, 10971, 29429, 16166, 27943, - 12480, 12481, 16007, 13613, 32015, 10983, 10984, 3799, 3834, 34007, 3749, - 12493, 12496, 12492, 12495, 12491, 12494, 27409, 27075, 29018, 27076, - 3737, 3736, 10925, 32518, 19563, 27212, 32333, 22915, 24087, 24085, - 24086, 24095, 24094, 24090, 24091, 32580, 32579, 24089, 23317, 29645, - 26942, 13142, 16322, 16314, 6519, 966, 19872, 19873, 19874, 16315, 26944, - 16319, 16316, 16320, 16318, 16321, 16317, 16463, 18145, 35488, 35489, - 35490, 26779, 26784, 26782, 26778, 26781, 26780, 26783, 22908, 22909, - 22911, 22910, 26775, 26776, 33684, 23609, 23608, 27747, 28939, 23604, - 23605, 6448, 23606, 6242, 26777, 22912, 23607, 16337, 27222, 35484, 362, - 16333, 32627, 32628, 16332, 16331, 32629, 32625, 16330, 32624, 16329, - 32626, 16334, 7735, 7741, 10926, 10927, 7736, 20496, 20504, 10916, 10917, - 32576, 32577, 28879, 28878, 20501, 20498, 20506, 20497, 20505, 20495, - 20499, 20494, 28929, 20503, 20502, 35485, 35481, 12485, 16008, 32516, - 32517, 32326, 32325, 28723, 8174, 12486, 352, 1045, 12476, 26785, 12477, - 10906, 32571, 32034, 12484, 12488, 19773, 13748, 19774, 13749, 19765, - 13735, 19768, 13738, 19766, 13736, 19767, 13737, 19764, 13739, 19758, - 13731, 19757, 13730, 19755, 13727, 19753, 13725, 19752, 13724, 19751, - 13728, 19754, 13726, 28864, 28862, 28865, 28863, 10953, 10952, 10949, - 10948, 28722, 28721, 28720, 28719, 10919, 10921, 10920, 13743, 13732, - 19762, 13746, 19763, 13747, 28937, 18153, 28938, 18154, 12482, 26864, - 29636, 26865, 29637, 26867, 29639, 26863, 29635, 26866, 29638, 26862, - 29634, 10922, 10931, 29631, 29759, 29629, 29757, 29630, 29758, 29628, - 29756, 29623, 29751, 29624, 29752, 29622, 29750, 29625, 29753, 29311, - 29398, 7730, 7732, 7731, 7733, 29618, 29745, 29619, 29744, 29748, 29747, - 12393, 26683, 32496, 13170, 24654, 27928, 27929, 27925, 26506, 33642, - 10940, 33641, 10938, 20507, 27930, 27927, 27926, 10907, 10935, 10929, - 27084, 10709, 33655, 33654, 10977, 26272, 26271, 32521, 32524, 32515, - 32528, 32527, 10951, 10950, 32525, 18142, 10934, 34113, 24096, 24670, - 24696, 29068, 29080, 19506, 19609, 32734, 11028, 24667, 24693, 29065, - 29077, 19508, 32645, 27246, 27247, 19592, 19593, 29422, 29415, 29425, - 29419, 10525, 10526, 10524, 10523, 10889, 3836, 34047, 3911, 34116, 3854, - 34064, 34044, 3817, 15945, 3835, 3839, 34056, 3842, 34060, 3872, 3873, - 34077, 3819, 34045, 3907, 34114, 19459, 32027, 19458, 20517, 19648, - 19647, 19649, 19650, 19584, 19596, 19595, 19643, 19645, 19644, 19580, - 33947, 12478, 27077, 19573, 27233, 27232, 19674, 27327, 27078, 27224, - 32323, 19575, 19574, 27225, 11017, 27923, 27921, 34057, 3874, 34078, - 3861, 34067, 15280, 15293, 15257, 15255, 15258, 28866, 2595, 28867, 2594, - 3628, 27922, 19625, 32753, 27296, 11006, 19510, 32649, 24691, 24665, - 29063, 29075, 19637, 32764, 27308, 11018, 7727, 959, 19636, 32755, 27307, - 11008, 35762, 35762, 24692, 24666, 29064, 29076, 19627, 32763, 27298, - 11016, 15962, 33664, 19626, 32754, 27297, 11007, 19638, 32765, 27309, - 11019, 19608, 32733, 27260, 11027, 956, 957, 955, 958, 27068, 27069, - 24566, 24567, 13176, 27261, 35762, 29855, 32038, 32042, 32039, 32041, - 3827, 3905, 3866, 3807, 11010, 11011, 32757, 32758, 19630, 27301, 19629, - 27300, 3752, 3753, 3754, 3755, 3756, 3758, 3757, 3759, 27115, 27116, - 27113, 27114, 27110, 27112, 27109, 27111, 32774, 32638, 26082, 26081, - 26083, 2736, 6517, 6231, 3878, 3797, 32544, 15944, 3912, 3846, 3838, - 3841, 3844, 24572, 32430, 4077, 19785, 27393, 34036, 27394, 29383, 32620, - 14275, 26791, 26790, 26789, 26788, 32495, 26891, 2515, 15999, 26664, - 24349, 34061, 3800, 32537, 9639, 15230, 35570, 18035, 1042, 97, 33772, - 26801, 19484, 27151, 29569, 29570, 19646, 32769, 27316, 11023, 12498, - 12497, 27759, 27535, 27533, 27534, 27532, 27536, 27537, 12483, 32631, - 27757, 10974, 26424, 27091, 15473, 13513, 13515, 13549, 13523, 13519, - 13550, 13557, 13520, 13556, 13529, 13525, 13524, 13518, 13560, 13531, - 13532, 13533, 13534, 13536, 13538, 13542, 13546, 13559, 13521, 13558, - 13535, 13537, 13539, 13548, 13517, 13541, 13552, 13551, 13553, 13543, - 13555, 13544, 13545, 13554, 13527, 13514, 13526, 13522, 13528, 13540, - 13547, 13530, 13516, 13561, 13563, 13597, 13571, 13567, 13598, 13605, - 13568, 13604, 13577, 13573, 13572, 13566, 13608, 13579, 13580, 13581, - 13582, 13584, 13586, 13590, 13594, 13607, 13569, 13606, 13583, 13585, - 13587, 13596, 13565, 13589, 13600, 13599, 13601, 13591, 13603, 13592, - 13593, 13602, 13575, 13562, 13574, 13570, 13576, 13588, 13595, 13578, - 13564, 18375, 19017, 18378, 18467, 18483, 18751, 19240, 18318, 18936, - 18364, 18996, 18628, 19407, 18197, 18395, 18532, 18533, 19360, 18605, - 19377, 19359, 18323, 18943, 19305, 18864, 19281, 19097, 18675, 19432, - 23035, 18508, 18632, 8182, 8276, 8232, 8326, 8196, 8290, 8193, 8287, - 8237, 8331, 8227, 8321, 8231, 8325, 8198, 8292, 8229, 8323, 8235, 8329, - 8201, 8295, 8206, 8300, 8238, 8332, 8239, 8333, 8202, 8296, 8207, 8301, - 8234, 8328, 8236, 8330, 8228, 8322, 8230, 8324, 8240, 8334, 8204, 8298, - 8200, 8294, 8233, 8327, 8223, 8317, 8190, 8284, 8218, 8312, 8186, 8280, - 8191, 8285, 8192, 8286, 8187, 8281, 8216, 8310, 8226, 8320, 8188, 8282, - 8214, 8308, 8217, 8311, 8181, 8275, 8189, 8283, 8209, 8303, 8210, 8304, - 8205, 8299, 8212, 8306, 8211, 8305, 8208, 8302, 8215, 8309, 8213, 8307, - 8221, 8315, 8219, 8313, 8220, 8314, 8222, 8316, 8336, 8337, 8338, 8340, - 8341, 8335, 8339, 8184, 8278, 8185, 8279, 8180, 8179, 8178, 8183, 8277, - 35762, 35762, 35762, 35762, 35762, 8274, 8271, 8272, 8273, 8269, 8270, - 8342, 13372, 13398, 13383, 13404, 13405, 13403, 13393, 13394, 13406, - 13387, 13396, 13399, 13401, 13407, 13389, 13392, 13397, 13391, 13395, - 13408, 13388, 13386, 13382, 13402, 13390, 13378, 13381, 13385, 13380, - 13379, 13400, 13384, 13373, 13377, 13375, 13410, 13374, 13376, 35762, - 13409, 35762, 35762, 35762, 35762, 35762, 13371, 35762, 35762, 32273, - 32284, 32285, 32278, 32280, 32263, 32302, 32274, 32277, 32275, 32276, - 32312, 32301, 32281, 32267, 32283, 32286, 32262, 32271, 32287, 32300, - 32282, 32268, 32307, 32272, 32313, 32295, 32261, 32269, 32303, 32304, - 32305, 32266, 32270, 32306, 32315, 32297, 32298, 32279, 32265, 32260, - 32288, 32290, 32289, 32291, 32292, 32299, 32293, 32308, 32309, 32310, - 32294, 32264, 32296, 32311, 32314, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 32316, 32317, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 32259, 12933, - 12750, 12837, 12877, 12853, 12540, 12915, 12578, 12768, 12759, 12660, - 12993, 12608, 12593, 12924, 12884, 12569, 12784, 12797, 12645, 12649, - 12648, 12647, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 12867, 12873, 12871, 12868, 12870, 12869, 12872, 35762, 12559, - 12565, 12563, 12560, 12562, 12561, 12564, 35762, 12983, 12989, 12987, - 12984, 12986, 12985, 12988, 35762, 12552, 12558, 12556, 12553, 12555, - 12554, 12557, 35762, 12819, 12825, 12823, 12820, 12822, 12821, 12824, - 35762, 12728, 12734, 12732, 12729, 12731, 12730, 12733, 35762, 12960, - 12966, 12964, 12961, 12963, 12962, 12965, 35762, 12671, 12677, 12675, - 12672, 12674, 12673, 12676, 35762, 7788, 7826, 7823, 7790, 7820, 7821, - 7827, 7794, 7795, 7796, 7803, 7825, 7797, 7791, 7819, 7816, 7818, 7822, - 7806, 7805, 7824, 7792, 7828, 7802, 7789, 7815, 7811, 7813, 7800, 7814, - 7787, 7799, 27126, 27125, 19528, 27187, 19477, 27141, 26965, 26964, - 10892, 19531, 27192, 26974, 19511, 27167, 11222, 26270, 13169, 27086, - 15997, 10891, 11005, 32717, 10894, 10942, 16344, 26229, 15993, 32329, - 19492, 27157, 32328, 32327, 19554, 27198, 32439, 32442, 4088, 4091, - 19516, 27172, 19476, 27146, 32574, 25768, 29482, 13140, 27101, 33667, - 27330, 34176, 32548, 26963, 26973, 32559, 10224, 10225, 32549, 32433, - 32584, 32045, 29582, 33669, 34159, 5701, 10910, 27100, 10913, 10231, - 10930, 16345, 16346, 20533, 20534, 10928, 10888, 32526, 22281, 26268, - 26922, 8345, 8383, 8382, 32417, 22280, 22282, 19526, 27185, 19525, 27184, - 32431, 32432, 4078, 4079, 25219, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 7615, - 7568, 7618, 7617, 7616, 7611, 7539, 7636, 7650, 7649, 7572, 7619, 7632, - 7631, 7601, 7600, 7599, 7598, 7628, 7637, 7627, 7626, 7587, 7586, 7590, - 7614, 35762, 7571, 7634, 7608, 7573, 7606, 7566, 7642, 7641, 7580, 7610, - 7609, 7620, 7570, 7574, 7595, 7537, 7579, 7630, 7629, 7542, 7625, 7553, - 7648, 7647, 7646, 7645, 7603, 7633, 7613, 7578, 7651, 7541, 7540, 7602, - 7607, 7584, 7583, 7582, 7638, 7569, 7644, 7643, 7557, 7621, 7589, 7556, - 7555, 7581, 7565, 7622, 7640, 7639, 7567, 7549, 7597, 7596, 7552, 7550, - 7605, 7604, 7612, 7543, 7559, 7551, 7561, 7547, 7577, 7576, 7575, 7545, - 7588, 7564, 7538, 7585, 7546, 7563, 7554, 7623, 7624, 7548, 7594, 7544, - 7592, 7560, 7593, 7562, 7635, 7591, 7558, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16662, 16644, - 16578, 16698, 16686, 16630, 16730, 16645, 16658, 16641, 16593, 16597, - 16582, 16567, 16633, 16719, 16665, 16636, 16670, 16747, 16704, 16675, - 16627, 16732, 16576, 16687, 16563, 16667, 16538, 16656, 16592, 16590, - 16685, 16613, 16615, 16595, 16545, 16744, 16570, 16678, 16632, 16714, - 16637, 16565, 16703, 16654, 16676, 16745, 16663, 16728, 16588, 16692, - 16579, 16647, 16731, 16694, 16553, 16712, 16554, 16706, 16624, 16621, - 16584, 16622, 16555, 16673, 16684, 16577, 16540, 16715, 16660, 16717, - 16683, 16657, 16727, 16638, 16707, 16572, 16738, 16583, 16566, 16612, - 16561, 16705, 16737, 16604, 16562, 16599, 16581, 16620, 16699, 16601, - 16569, 16585, 16668, 16634, 16648, 16722, 16711, 16643, 16749, 16602, - 16549, 16695, 16580, 16740, 16716, 16575, 16598, 16700, 16536, 16709, - 16702, 16726, 16616, 16560, 16710, 16541, 16677, 16696, 16635, 16661, - 16691, 16610, 16666, 16539, 16669, 16589, 16556, 16649, 16650, 16688, - 16537, 16652, 16723, 16664, 16550, 16708, 16568, 16617, 16721, 16631, - 16546, 16736, 16564, 16739, 16689, 16629, 16701, 16733, 16557, 16671, - 16542, 16690, 16680, 16679, 16611, 16552, 16559, 16543, 16653, 16735, - 16571, 16743, 16574, 16734, 16614, 16646, 16619, 16655, 16697, 16693, - 16672, 16548, 16746, 16600, 16639, 16718, 16642, 16713, 16640, 16742, - 16607, 16591, 16625, 16608, 16628, 16551, 16720, 16623, 16603, 16681, - 16558, 16618, 16605, 16544, 16682, 16573, 16741, 16626, 16748, 16651, - 16547, 16596, 16609, 16725, 16587, 16674, 16659, 16594, 16724, 16586, - 16729, 16606, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16031, 16029, - 16030, 16028, 16043, 16039, 16038, 16035, 16036, 16037, 16033, 16032, - 16040, 16034, 16044, 16042, 16128, 16027, 16126, 10698, 16362, 16127, - 16026, 16046, 19455, 27123, 19474, 27142, 19470, 27139, 19538, 27200, - 19464, 27133, 26803, 13506, 19536, 27196, 19540, 27202, 19543, 27205, - 19542, 27204, 33946, 27083, 10924, 20531, 26804, 15127, 15120, 15117, - 15123, 15122, 15125, 15124, 15128, 15126, 16124, 16123, 16045, 16122, - 14786, 14785, 33952, 33647, 33650, 33648, 33651, 33649, 6500, 16120, - 15121, 15119, 15118, 33646, 21138, 26422, 16125, 16121, 35762, 15849, - 15835, 15851, 15929, 15853, 15931, 15850, 15928, 15852, 15930, 15890, - 15880, 15892, 15882, 15894, 15884, 15891, 15881, 15893, 15883, 15854, - 15915, 15856, 15917, 15858, 15919, 15855, 15916, 15857, 15918, 15910, - 15875, 15912, 15877, 15848, 15914, 15879, 15911, 15876, 15913, 15878, - 15870, 15872, 15874, 15871, 15873, 15885, 15865, 15900, 15887, 15859, - 15902, 15889, 15868, 15904, 15886, 15866, 15901, 15888, 15867, 15903, - 15895, 15897, 15899, 15896, 15898, 15842, 15924, 15844, 15926, 15843, - 15925, 15905, 15907, 15909, 15906, 15908, 15838, 15920, 15922, 15921, - 15923, 15869, 15927, 15845, 15846, 35762, 35762, 7985, 7984, 16987, - 16986, 15933, 15932, 15834, 16984, 16914, 16843, 16916, 16977, 16918, - 16979, 16915, 16976, 16917, 16978, 16939, 16929, 16941, 16931, 16943, - 16933, 16940, 16930, 16942, 16932, 16919, 16964, 16921, 16966, 16923, - 16968, 16920, 16965, 16922, 16967, 16954, 16924, 16956, 16926, 16901, - 16958, 16928, 16955, 16925, 16957, 16927, 16881, 16883, 16885, 16882, - 16884, 16934, 16858, 16944, 16936, 16852, 16946, 16938, 16861, 16948, - 16935, 16859, 16945, 16937, 16860, 16947, 16876, 16862, 16879, 16877, - 16878, 16906, 16973, 16908, 16975, 16907, 16974, 16949, 16951, 16953, - 16950, 16952, 16902, 16969, 16971, 16970, 16972, 16880, 16963, 16896, - 16897, 16959, 16961, 16960, 16962, 16983, 16985, 16982, 16980, 16981, - 35762, 35762, 35762, 35762, 35762, 4063, 4071, 4070, 4068, 4067, 4074, - 4039, 4059, 4027, 4055, 4069, 4065, 4072, 4076, 4052, 4058, 4062, 4073, - 4051, 4057, 4061, 4007, 4043, 4019, 4024, 4008, 4025, 4010, 4050, 4012, - 4020, 4013, 4021, 4026, 4032, 4017, 4038, 4075, 4040, 4029, 4035, 4046, - 4042, 35762, 15039, 15083, 15040, 15045, 15046, 15048, 15075, 15086, - 15061, 15062, 15064, 15066, 15073, 15069, 15065, 15072, 15041, 15051, - 15085, 15052, 15076, 15088, 15033, 15028, 15082, 15027, 15038, 15074, - 15059, 15112, 15023, 15026, 15109, 15110, 15030, 15029, 15096, 15095, - 15113, 15092, 15093, 15114, 15101, 15115, 15091, 15090, 15094, 15105, - 15031, 15111, 15032, 15116, 15084, 15047, 15050, 15049, 15063, 15070, - 15067, 15068, 15071, 15042, 15044, 15043, 15037, 15058, 15056, 15053, - 15054, 15057, 15055, 15035, 15036, 15078, 15079, 15081, 15080, 15077, - 15060, 15089, 15098, 15100, 15099, 15034, 15087, 15097, 15102, 15103, - 15104, 15107, 15106, 15108, 15024, 15025, 35762, 16020, 16023, 16022, - 16018, 16016, 16012, 16019, 16014, 16010, 16013, 16021, 16017, 16011, - 16024, 16025, 16015, 4064, 4053, 4066, 4030, 4023, 4022, 4049, 4045, - 4037, 4014, 4033, 4018, 4036, 4041, 4009, 4011, 4016, 4048, 4044, 4034, - 4005, 4006, 4004, 4003, 4028, 4060, 4054, 4002, 4031, 4056, 4047, 4015, - 7680, 7683, 7684, 7682, 7671, 7656, 7662, 7653, 7661, 7675, 7663, 7659, - 7654, 7660, 7657, 7686, 7652, 7670, 7666, 7678, 7685, 7655, 7665, 7674, - 7673, 7679, 7677, 7667, 7669, 7681, 7676, 7672, 7664, 7658, 7668, 7687, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 16041, 16899, 16911, 16912, 16900, 16910, 16886, 16888, 16890, - 16887, 16889, 16913, 16891, 16893, 16895, 16892, 16894, 26284, 26288, - 26300, 26294, 26286, 26292, 26296, 26302, 26277, 26275, 26282, 26298, - 26290, 26280, 26285, 26289, 26301, 26295, 26287, 26293, 26297, 26303, - 26278, 26276, 26283, 26299, 26291, 26281, 26279, 26341, 26340, 35762, - 26339, 26335, 26333, 26314, 26312, 26332, 26324, 26309, 26318, 26334, - 26317, 26311, 26337, 26336, 26316, 26308, 26331, 26329, 26338, 26326, - 26319, 26327, 26310, 26305, 26315, 26322, 26306, 26328, 26330, 26307, - 26320, 26304, 26313, 26321, 26325, 26323, 6319, 6307, 6329, 6308, 6473, - 6487, 6475, 6457, 6454, 6470, 6468, 6450, 26421, 6488, 6494, 6493, 6490, - 6489, 6492, 6491, 6496, 6495, 6474, 6476, 6482, 6481, 6478, 6477, 6267, - 6271, 6283, 6277, 6269, 6275, 6279, 6260, 6258, 6256, 6265, 6281, 6273, - 6263, 6268, 6272, 6284, 6278, 6270, 6276, 6280, 6261, 6259, 6257, 6266, - 6282, 6274, 6264, 6394, 6393, 6262, 18033, 6342, 6337, 6335, 6304, 6302, - 6334, 6325, 6299, 6317, 6336, 6315, 6301, 6339, 6338, 6313, 6297, 6328, - 6333, 6306, 6330, 6318, 6331, 6300, 6293, 6309, 6324, 6314, 6303, 6327, - 6298, 6341, 6290, 6340, 6321, 6294, 6292, 6305, 6295, 6310, 6311, 6323, - 6312, 6322, 6332, 6326, 6296, 6320, 6288, 6316, 6480, 6479, 6484, 6483, - 6456, 6458, 6464, 6463, 6460, 6459, 6462, 6461, 6466, 6465, 6453, 16111, - 16114, 16115, 16053, 16116, 16112, 16113, 16052, 16118, 16119, 16117, - 16085, 29271, 29240, 29242, 19871, 6388, 6390, 6392, 6389, 6391, 6351, - 6353, 6355, 6352, 6354, 6371, 6373, 6375, 6372, 6374, 6376, 6378, 6380, - 6377, 6379, 6361, 6363, 6365, 6362, 6364, 6346, 6348, 6350, 6347, 6349, - 6356, 6358, 6360, 6357, 6359, 6385, 6387, 6386, 6366, 6368, 6370, 6367, - 6369, 6381, 6383, 6382, 6384, 29238, 29202, 29200, 29199, 29201, 29274, - 29275, 29434, 29241, 29234, 29364, 29368, 29281, 29283, 29282, 29246, - 29247, 29251, 29250, 29284, 29249, 29285, 29288, 29286, 29287, 29252, - 29253, 29295, 29296, 29297, 29294, 29293, 29404, 29405, 29408, 29406, - 29407, 29225, 29229, 29230, 29413, 29357, 29358, 29259, 29371, 29372, - 29206, 29380, 29378, 29379, 29209, 29266, 29265, 29208, 29267, 29260, - 29377, 29375, 29261, 29376, 29374, 29211, 29381, 29210, 29264, 29382, - 29262, 29263, 29321, 29322, 29325, 29324, 29323, 29331, 29332, 29333, - 29328, 29329, 29330, 29432, 29431, 29433, 29399, 29400, 29402, 29401, - 29397, 29396, 29414, 16109, 16110, 16092, 16094, 16101, 16100, 16107, - 16105, 16096, 16103, 16095, 16098, 16091, 16093, 16102, 16099, 16108, - 16106, 16097, 16104, 16086, 16090, 16089, 16088, 16087, 29269, 29224, - 29204, 29207, 29369, 29385, 29226, 29228, 29227, 29279, 29235, 29239, - 29236, 29237, 29216, 29373, 29356, 29338, 29320, 29280, 29302, 29326, - 29256, 29213, 29299, 29386, 29359, 29339, 29340, 29353, 29303, 29272, - 29298, 29350, 29254, 29412, 29341, 29354, 29233, 29305, 29245, 29360, - 29342, 29335, 29217, 29289, 29337, 29219, 29319, 29292, 29336, 29218, - 29318, 29291, 29315, 29316, 29370, 29301, 29352, 29255, 29393, 29394, - 29395, 29390, 29361, 29343, 29355, 29391, 29362, 29344, 29346, 29307, - 29347, 29392, 29363, 29345, 29348, 29308, 29349, 29300, 29317, 29203, - 29212, 29222, 29223, 29220, 29215, 29231, 29257, 29258, 29268, 29273, - 29304, 29290, 29306, 29312, 29313, 29310, 29314, 29327, 29334, 29351, - 29387, 29388, 29384, 29389, 29409, 29410, 29430, 29205, 29198, 16084, - 16069, 16057, 16076, 16075, 16082, 16080, 16071, 16078, 16070, 16073, - 16068, 16056, 16077, 16074, 16083, 16081, 16072, 16079, 16058, 16066, - 16064, 16063, 16060, 16059, 16062, 16061, 16067, 16065, 16054, 16055, - 29248, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 15811, - 15814, 15775, 15825, 15822, 15770, 15808, 15787, 15804, 15821, 15799, - 15805, 15780, 15782, 15792, 15826, 15779, 15823, 15763, 15769, 15767, - 15786, 15806, 15800, 15789, 15785, 15793, 15784, 15809, 15812, 15791, - 15777, 15801, 15783, 15798, 15778, 15813, 15796, 15794, 15773, 15772, - 15790, 15768, 15771, 15781, 15797, 15795, 15815, 15802, 15810, 15807, - 15819, 15774, 15817, 15765, 15816, 15818, 15820, 15776, 15824, 15788, - 15803, 15764, 15766, 34695, 34702, 34694, 34701, 34699, 34700, 34697, - 34698, 35470, 35471, 35468, 35469, 35467, 35465, 35466, 35474, 35475, - 35472, 35473, 35477, 35476, 35342, 34362, 34363, 34356, 34361, 34359, - 34360, 34357, 34358, 34366, 34367, 34364, 34365, 34347, 34345, 34346, - 34370, 34371, 34368, 34369, 34355, 34353, 34354, 34351, 34352, 34344, - 34350, 34349, 34348, 34376, 34377, 34372, 34375, 34374, 34373, 35074, - 35075, 35069, 35073, 35072, 35070, 35071, 35078, 35079, 35076, 35077, - 35063, 35061, 35062, 35082, 35083, 35080, 35081, 35067, 35068, 35060, - 35066, 35065, 35064, 35088, 35089, 35084, 35087, 35086, 35085, 34330, - 34331, 34324, 34329, 34327, 34328, 34325, 34326, 34334, 34335, 34332, - 34333, 34315, 34313, 34314, 34338, 34339, 34336, 34337, 34323, 34321, - 34322, 34319, 34320, 34312, 34318, 34317, 34316, 34342, 34343, 34340, - 34341, 34877, 34878, 34872, 34876, 34875, 34873, 34874, 34881, 34882, - 34879, 34880, 34885, 34886, 34883, 34884, 34891, 34892, 34887, 34890, - 34889, 34888, 34897, 34898, 34893, 34896, 34895, 34894, 34620, 34621, - 34615, 34619, 34618, 34616, 34617, 34624, 34625, 34622, 34623, 34609, - 34607, 34608, 34628, 34629, 34626, 34627, 34613, 34614, 34606, 34612, - 34611, 34610, 34634, 34630, 34633, 34632, 34631, 34856, 34857, 34851, - 34855, 34854, 34852, 34853, 34860, 34861, 34858, 34859, 34844, 34845, - 34842, 34843, 34864, 34865, 34862, 34863, 34871, 34870, 34849, 34850, - 34841, 34848, 34847, 34846, 34868, 34869, 34866, 34867, 34501, 34502, - 34499, 34500, 34497, 34498, 34495, 34496, 34494, 34492, 34493, 34511, - 34512, 34507, 34510, 34509, 34508, 34505, 34506, 34503, 34504, 35320, - 35321, 35314, 35319, 35317, 35318, 35315, 35316, 35324, 35325, 35322, - 35323, 35328, 35329, 35326, 35327, 35313, 35312, 35334, 35335, 35330, - 35333, 35332, 35331, 35340, 35341, 35336, 35339, 35338, 35337, 34479, - 34480, 34474, 34478, 34477, 34475, 34476, 34486, 34487, 34484, 34485, - 34468, 34467, 34490, 34491, 34488, 34489, 34483, 34481, 34482, 34472, - 34473, 34466, 34471, 34470, 34469, 35299, 35300, 35294, 35298, 35297, - 35295, 35296, 35306, 35307, 35304, 35305, 35287, 35288, 35285, 35286, - 35310, 35311, 35308, 35309, 35303, 35301, 35302, 35292, 35293, 35284, - 35291, 35290, 35289, 34453, 34454, 34448, 34452, 34451, 34449, 34450, - 34460, 34461, 34458, 34459, 34442, 34440, 34441, 34464, 34465, 34462, - 34463, 34457, 34455, 34456, 34446, 34447, 34439, 34445, 34444, 34443, - 34903, 34904, 34899, 34902, 34901, 34900, 34910, 34911, 34908, 34909, - 34914, 34915, 34912, 34913, 34907, 34905, 34906, 34920, 34921, 34916, - 34919, 34918, 34917, 34650, 34651, 34644, 34649, 34647, 34648, 34645, - 34646, 34654, 34655, 34652, 34653, 34639, 34638, 34636, 34637, 34635, - 34643, 34641, 34642, 34640, 35048, 35049, 35043, 35047, 35046, 35044, - 35045, 35052, 35050, 35051, 35037, 35035, 35036, 35058, 35059, 35056, - 35057, 35055, 35053, 35054, 35041, 35042, 35034, 35040, 35039, 35038, - 34588, 34589, 34583, 34587, 34586, 34584, 34585, 34598, 34599, 34596, - 34597, 34577, 34575, 34576, 34595, 34593, 34594, 34592, 34590, 34591, - 34581, 34582, 34574, 34580, 34579, 34578, 34604, 34605, 34600, 34603, - 34602, 34601, 34803, 34804, 34797, 34802, 34800, 34801, 34798, 34799, - 34807, 34808, 34805, 34806, 34787, 34788, 34785, 34786, 34811, 34812, - 34809, 34810, 34796, 34794, 34795, 34792, 34793, 34784, 34791, 34790, - 34789, 34817, 34818, 34813, 34816, 34815, 34814, 34557, 34558, 34551, - 34556, 34554, 34555, 34552, 34553, 34561, 34562, 34559, 34560, 34544, - 34545, 34542, 34543, 34569, 34570, 34567, 34568, 34565, 34566, 34563, - 34564, 34549, 34550, 34541, 34548, 34547, 34546, 34770, 34771, 34765, - 34769, 34768, 34766, 34767, 34774, 34775, 34772, 34773, 34759, 34757, - 34758, 34782, 34783, 34780, 34781, 34778, 34779, 34776, 34777, 34763, - 34764, 34756, 34762, 34761, 34760, 34517, 34518, 34513, 34516, 34514, - 34515, 34531, 34532, 34529, 34530, 34522, 34523, 34520, 34521, 34539, - 34540, 34537, 34538, 34535, 34536, 34533, 34534, 34527, 34528, 34519, - 34526, 34525, 34524, 34840, 34839, 34833, 34834, 34831, 34832, 34822, - 34820, 34821, 34837, 34838, 34835, 34836, 34830, 34828, 34829, 34826, - 34827, 34819, 34825, 34824, 34823, 34669, 34670, 34663, 34668, 34666, - 34667, 34664, 34665, 34673, 34674, 34671, 34672, 34658, 34659, 34656, - 34657, 34677, 34678, 34675, 34676, 34662, 34660, 34661, 34930, 34928, - 34929, 34933, 34934, 34931, 34932, 34923, 34924, 34922, 34937, 34938, - 34935, 34936, 34927, 34925, 34926, 34573, 34572, 34571, 34688, 34689, - 34686, 34687, 34681, 34682, 34679, 34680, 34692, 34693, 34690, 34691, - 34685, 34683, 34684, 35354, 35355, 35352, 35353, 35345, 35343, 35344, - 35351, 35349, 35350, 35348, 35346, 35347, 35417, 35418, 35412, 35416, - 35415, 35413, 35414, 35453, 35454, 35451, 35452, 35406, 35404, 35405, - 35457, 35458, 35455, 35456, 35450, 35448, 35449, 35410, 35411, 35403, - 35409, 35408, 35407, 35463, 35464, 35459, 35462, 35461, 35460, 34423, - 34424, 34417, 34422, 34420, 34421, 34418, 34419, 34427, 34428, 34425, - 34426, 34408, 34406, 34407, 34431, 34432, 34429, 34430, 34416, 34414, - 34415, 34412, 34413, 34405, 34411, 34410, 34409, 34437, 34438, 34433, - 34436, 34435, 34434, 35431, 35432, 35425, 35430, 35428, 35429, 35426, - 35427, 35435, 35436, 35433, 35434, 35424, 35422, 35423, 35421, 35419, - 35420, 35441, 35437, 35440, 35439, 35438, 35446, 35447, 35442, 35445, - 35444, 35443, 35020, 35021, 35015, 35019, 35018, 35016, 35017, 35024, - 35025, 35022, 35023, 35008, 35007, 35014, 35013, 35033, 35032, 35012, - 35006, 35011, 35010, 35009, 35030, 35031, 35026, 35029, 35028, 35027, - 35265, 35266, 35260, 35264, 35263, 35261, 35262, 35272, 35273, 35270, - 35271, 35254, 35252, 35253, 35276, 35277, 35274, 35275, 35269, 35267, - 35268, 35258, 35259, 35251, 35257, 35256, 35255, 35282, 35283, 35278, - 35281, 35280, 35279, 35201, 35202, 35196, 35200, 35199, 35197, 35198, - 35208, 35209, 35206, 35207, 35212, 35213, 35210, 35211, 35205, 35203, - 35204, 35216, 35217, 35214, 35215, 35222, 35223, 35218, 35221, 35220, - 35219, 35387, 35388, 35385, 35386, 35379, 35377, 35378, 35395, 35396, - 35393, 35394, 35391, 35392, 35389, 35390, 35383, 35384, 35376, 35382, - 35381, 35380, 35401, 35402, 35397, 35400, 35399, 35398, 34389, 34390, - 34387, 34388, 34381, 34382, 34379, 34380, 34397, 34398, 34395, 34396, - 34393, 34394, 34391, 34392, 34386, 34378, 34385, 34384, 34383, 34403, - 34404, 34399, 34402, 34401, 34400, 35169, 35168, 35148, 35147, 35160, - 35161, 35158, 35159, 35156, 35157, 35154, 35155, 35152, 35153, 35146, - 35151, 35150, 35149, 35166, 35167, 35162, 35165, 35164, 35163, 34969, - 34970, 34967, 34968, 34966, 34964, 34965, 34973, 34974, 34971, 34972, - 34979, 34980, 34975, 34978, 34977, 34976, 34985, 34986, 34981, 34984, - 34983, 34982, 35235, 35236, 35233, 35234, 35227, 35225, 35226, 35243, - 35244, 35241, 35242, 35239, 35240, 35237, 35238, 35231, 35232, 35224, - 35230, 35229, 35228, 35249, 35250, 35245, 35248, 35247, 35246, 35184, - 35185, 35182, 35183, 35173, 35171, 35172, 35188, 35189, 35186, 35187, - 35181, 35179, 35180, 35177, 35178, 35170, 35176, 35175, 35174, 35194, - 35195, 35190, 35193, 35192, 35191, 34744, 34745, 34738, 34743, 34741, - 34742, 34739, 34740, 34731, 34732, 34729, 34730, 34748, 34749, 34746, - 34747, 34736, 34737, 34728, 34735, 34734, 34733, 34754, 34755, 34750, - 34753, 34752, 34751, 35106, 35107, 35100, 35105, 35103, 35104, 35101, - 35102, 35093, 35094, 35091, 35092, 35110, 35111, 35108, 35109, 35098, - 35099, 35090, 35097, 35096, 35095, 35116, 35117, 35112, 35115, 35114, - 35113, 34718, 34719, 34712, 34717, 34715, 34716, 34713, 34714, 34706, - 34704, 34705, 34722, 34723, 34720, 34721, 34710, 34711, 34703, 34709, - 34708, 34707, 34726, 34727, 34724, 34725, 34952, 34953, 34946, 34951, - 34949, 34950, 34947, 34948, 34941, 34940, 34956, 34957, 34954, 34955, - 34945, 34939, 34944, 34943, 34942, 34962, 34963, 34958, 34961, 34960, - 34959, 35000, 35001, 34994, 34999, 34997, 34998, 34995, 34996, 34990, - 34988, 34989, 35004, 35005, 35002, 35003, 34992, 34993, 34987, 34991, - 35362, 35363, 35356, 35361, 35359, 35360, 35357, 35358, 35375, 35374, - 35366, 35367, 35364, 35365, 35372, 35373, 35368, 35371, 35370, 35369, - 35134, 35135, 35128, 35133, 35131, 35132, 35129, 35130, 35121, 35122, - 35119, 35120, 35138, 35139, 35136, 35137, 35126, 35127, 35118, 35125, - 35124, 35123, 35144, 35145, 35140, 35143, 35142, 35141, 35762, 35762, - 35762, 34309, 34282, 34280, 34287, 34261, 34297, 34268, 34270, 34286, - 34273, 34284, 34257, 34285, 34303, 34291, 34272, 34298, 34271, 34304, - 34262, 34265, 34258, 34267, 34288, 34299, 34311, 34276, 34307, 34292, - 34275, 34302, 34300, 34296, 34301, 34308, 34279, 34289, 34278, 34269, - 34277, 34310, 34266, 34293, 34283, 34260, 34259, 34264, 34274, 34294, - 34305, 34295, 34263, 34306, 34290, 34281, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 20474, 20463, 20462, 20446, 20444, - 20443, 20457, 20461, 20460, 20476, 20455, 20454, 20445, 20442, 20441, - 20478, 20453, 20477, 20465, 20468, 20469, 20452, 20459, 20480, 20458, - 20475, 20479, 20464, 20467, 20456, 20470, 20471, 20450, 20451, 20481, - 20472, 20447, 20448, 20449, 20473, 20437, 20440, 20438, 20436, 20439, - 20435, 20482, 20483, 33047, 33048, 32884, 33033, 33034, 33007, 32810, - 32817, 32920, 32893, 32927, 32867, 32993, 33021, 32845, 32838, 32796, - 32789, 32911, 33014, 32803, 32946, 32831, 32824, 32859, 32852, 32986, - 33000, 32965, 33028, 32906, 32952, 32873, 32934, 32979, 32972, 33056, - 33057, 32888, 32889, 33042, 33043, 33009, 32812, 32819, 32922, 32899, - 32929, 32870, 32995, 33023, 32847, 32840, 32798, 32791, 32915, 33016, - 32805, 32948, 32833, 32826, 32861, 32854, 32988, 33002, 32967, 33030, - 32907, 32957, 32878, 32936, 32981, 32974, 33054, 33055, 32959, 32886, - 32887, 33040, 33041, 33008, 32811, 32818, 32921, 32897, 32898, 32928, - 32869, 32994, 33022, 32846, 32839, 32797, 32790, 32914, 33015, 32804, - 32947, 32832, 32825, 32860, 32853, 32987, 33001, 32966, 33029, 32903, - 32904, 32956, 32877, 32935, 32980, 32973, 33051, 33052, 32882, 33037, - 33038, 33005, 32808, 32815, 32918, 32896, 32925, 32865, 32991, 33019, - 32843, 32836, 32794, 32787, 32913, 33012, 32801, 32944, 32829, 32822, - 32857, 32850, 32984, 32998, 32963, 33026, 32902, 32955, 32876, 32932, - 32977, 32970, 33058, 33059, 32890, 32891, 33044, 33045, 33010, 32813, - 32820, 32923, 32900, 32930, 32871, 32996, 33024, 32848, 32841, 32799, - 32792, 32916, 33017, 32806, 32949, 32834, 32827, 32862, 32855, 32989, - 33003, 32968, 33031, 32908, 32958, 32879, 32937, 32982, 32975, 33050, - 33053, 32961, 32880, 32881, 33036, 33039, 33004, 32807, 32814, 32917, - 32895, 32924, 32863, 32864, 32990, 33018, 32842, 32835, 32793, 32786, - 32912, 33011, 32800, 32938, 32828, 32821, 32856, 32849, 32983, 32997, - 32962, 33025, 32901, 32954, 32875, 32931, 32976, 32969, 33046, 33049, - 32960, 32883, 32885, 33032, 33035, 33006, 32809, 32816, 32919, 32892, - 32894, 32926, 32866, 32868, 32992, 33020, 32844, 32837, 32795, 32788, - 32909, 33013, 32802, 32945, 32830, 32823, 32858, 32851, 32985, 32999, - 32964, 33027, 32905, 32951, 32953, 32872, 32874, 32933, 32978, 32971, - 32950, 32910, 32783, 32784, 32785, 32941, 32942, 32939, 33063, 33066, - 33069, 33068, 33072, 33064, 33071, 33062, 33060, 33067, 33070, 33061, - 33065, 33079, 33081, 33078, 33077, 33074, 33073, 33076, 33075, 33082, - 33080, 32943, 32940, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 9978, 10178, 9867, 10026, 9818, 10120, 9922, 10081, - 9863, 10022, 9943, 10106, 9851, 10010, 9811, 10107, 9971, 10171, 9919, - 10078, 9820, 10122, 9920, 10079, 9859, 10018, 9852, 10011, 9916, 10075, - 9973, 10173, 9819, 10121, 9962, 10140, 9959, 10137, 9960, 10138, 9942, - 10105, 9849, 10008, 9864, 10023, 9993, 7842, 7834, 7785, 7835, 28868, - 7809, 7798, 7812, 7808, 7817, 7810, 7807, 7804, 7840, 7829, 9992, 9996, - 9873, 10032, 9871, 10030, 9983, 10183, 9861, 10020, 9869, 10028, 9824, - 10148, 9831, 10156, 9828, 10152, 9827, 10151, 9830, 10155, 9907, 10066, - 9957, 10135, 9865, 10024, 9860, 10019, 23047, 23084, 7793, 7801, 3393, - 2760, 3396, 2762, 3392, 3368, 3385, 3395, 2789, 3394, 2765, 3365, 3371, - 3370, 2763, 2769, 3384, 2788, 2782, 2768, 3380, 2776, 3375, 3381, 3374, - 3378, 2758, 2754, 2786, 2785, 2780, 3387, 3377, 3388, 3389, 2787, 2752, - 3361, 2781, 2784, 3364, 3390, 3362, 2749, 3373, 2767, 2774, 2791, 3367, - 3372, 2753, 2777, 2778, 2779, 3376, 3363, 2751, 2750, 3391, 2790, 2766, - 3366, 2764, 2755, 2771, 3369, 2770, 2773, 3386, 2761, 2775, 2772, 3383, - 2759, 3382, 2783, 3379, 2748, 2756, 2757, 2745, 2744, 3397, 3399, 2747, - 2746, 3398, 3400, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 23044, 23040, 23043, 23039, 23045, 23041, 23046, 23042, 23099, 23114, - 23142, 23121, 23104, 23098, 23113, 23141, 23120, 23103, 23100, 23115, - 23143, 23125, 23105, 23092, 23090, 23091, 23136, 23155, 23152, 23154, - 23153, 23133, 23303, 23304, 18285, 18884, 18286, 18885, 18325, 18946, - 18547, 19306, 18546, 19263, 18227, 18806, 18228, 18807, 18688, 18696, - 18201, 18762, 18202, 18763, 18203, 18764, 18199, 18760, 18200, 18761, - 18204, 18765, 18493, 19188, 18366, 18998, 18363, 18995, 18367, 18999, - 18213, 18783, 18385, 19022, 18416, 19096, 18417, 19098, 18463, 19136, - 18468, 19141, 18466, 19139, 18469, 19142, 18475, 19153, 18474, 19150, - 18490, 19178, 18495, 19191, 18588, 19356, 18597, 19368, 18592, 19363, - 18541, 19257, 18542, 19258, 18596, 19367, 18287, 18902, 18354, 18985, - 18229, 18808, 23312, 18844, 19042, 19056, 19079, 19190, 18673, 19301, - 19353, 18347, 18974, 18348, 18975, 18349, 18531, 19269, 18536, 19299, - 18350, 18977, 18351, 18978, 18352, 18979, 23119, 23088, 23165, 18514, - 19213, 18534, 19027, 18704, 18409, 19070, 18223, 18796, 18793, 18940, - 18207, 18768, 18294, 18909, 18593, 19364, 18594, 19365, 18595, 19366, - 18302, 18920, 18370, 19003, 18412, 19075, 18488, 19175, 18510, 19211, - 18321, 18492, 18517, 18373, 18513, 18695, 18535, 18538, 18357, 18230, - 18214, 18785, 18461, 19134, 18584, 19344, 18307, 18925, 18308, 18926, - 18309, 18927, 18458, 19125, 18196, 18757, 18221, 18511, 18634, 18234, - 18810, 18507, 19205, 35762, 35762, 35762, 35762, 35762, 18226, 18802, - 35762, 18837, 35762, 18834, 18398, 19055, 18516, 19231, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 23020, 23023, 23032, 18491, 19179, 18644, 23015, 23244, - 18678, 18642, 18643, 18640, 18641, 18639, 29798, 29800, 29810, 29802, - 29799, 29801, 29809, 29790, 29789, 29786, 29785, 29808, 29784, 29783, - 29788, 29787, 29778, 29777, 29772, 29771, 29780, 29779, 29774, 29773, - 29796, 29792, 29791, 29782, 29781, 29795, 29776, 29794, 29775, 29797, - 29793, 29812, 29814, 29815, 29813, 29811, 29803, 29804, 29805, 29806, - 29807, 35762, 35762, 35762, 24678, 24677, 24680, 24675, 24676, 24679, - 24672, 24671, 24674, 24673, 35762, 35762, 35762, 35762, 35762, 35762, - 26558, 26557, 26546, 26547, 26534, 26536, 26568, 26549, 26556, 26555, - 26539, 26550, 26560, 26559, 26565, 26570, 26552, 26551, 26538, 26573, - 26561, 26562, 26540, 26575, 26572, 26569, 26541, 26542, 26567, 26531, - 26576, 26578, 26563, 26577, 26571, 26574, 26566, 26545, 26564, 26583, - 26584, 26554, 26553, 26537, 26548, 26532, 26544, 26543, 26533, 26582, - 26585, 26535, 26581, 26586, 26580, 26579, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 27667, 27669, 27616, 27617, 27637, 27638, - 27633, 27634, 27628, 27629, 27630, 27631, 27660, 27661, 27618, 27635, - 27636, 27619, 27657, 27656, 27653, 27652, 27641, 27651, 27650, 27655, - 27654, 27643, 27625, 27624, 27621, 27620, 27642, 27627, 27626, 27623, - 27622, 27644, 27659, 27658, 27649, 27648, 27663, 27665, 27664, 27640, - 27632, 27645, 27646, 27647, 27662, 27639, 27682, 27683, 27694, 27695, - 27690, 27691, 27686, 27687, 27688, 27689, 27696, 27697, 27684, 27692, - 27693, 27685, 27668, 27666, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 27671, 27670, 27678, 27680, 27677, 27676, 27673, 27672, - 27675, 27674, 27681, 27679, 35762, 35762, 35762, 35762, 35762, 35762, - 7860, 7862, 7859, 7858, 7855, 7854, 7857, 7856, 7863, 7861, 7851, 7852, - 7847, 7848, 7849, 7850, 7846, 7853, 10463, 10456, 10465, 10458, 10457, - 10455, 10462, 10359, 10357, 10362, 10464, 10513, 10368, 10480, 17118, - 17120, 17117, 17116, 17113, 17112, 17115, 17114, 17121, 17119, 17082, - 17081, 17092, 17078, 17086, 17085, 17099, 17079, 17088, 17074, 17080, - 17084, 17083, 17094, 17091, 17089, 17095, 17098, 17093, 17097, 17087, - 17075, 17096, 17090, 17100, 17076, 17101, 17077, 17110, 17107, 17109, - 17108, 17111, 17106, 17104, 17105, 17102, 17103, 27041, 27038, 27032, - 27046, 27037, 27034, 27043, 27035, 27028, 27036, 27040, 27030, 27045, - 27044, 27042, 27048, 27047, 27039, 27027, 27031, 27033, 27029, 27049, - 27056, 27058, 27051, 27054, 27057, 27055, 27053, 27052, 27024, 27023, - 27026, 27025, 27059, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 27050, 14774, 14777, 14778, 14775, 14732, - 14733, 14741, 14735, 14737, 14740, 14734, 14730, 14736, 14738, 14731, - 14697, 14699, 14700, 14713, 14720, 14727, 14762, 14679, 14686, 14760, - 14764, 14710, 14783, 14766, 35762, 35762, 35762, 16437, 16435, 16439, - 16438, 16409, 16377, 16376, 16378, 16418, 16397, 16383, 16384, 16416, - 16410, 16417, 16379, 16380, 16381, 16393, 16394, 16382, 16391, 16392, - 16407, 16386, 16408, 16385, 16405, 16406, 16372, 16373, 16388, 16403, - 16404, 16374, 16375, 16387, 16395, 16396, 16389, 16390, 16413, 16415, - 16398, 16399, 16412, 16414, 16401, 16402, 16400, 16411, 16436, 16442, - 16444, 16445, 16446, 16440, 16441, 16443, 16448, 16447, 16368, 16369, - 16370, 16433, 16371, 16419, 16422, 16431, 16424, 16429, 16427, 16425, - 16423, 16420, 16421, 16426, 16434, 35762, 16432, 16455, 16457, 16454, - 16453, 16450, 16449, 16452, 16451, 16458, 16456, 35762, 35762, 35762, - 35762, 16428, 16430, 23922, 23920, 23915, 23912, 23918, 23998, 23986, - 23938, 23950, 23946, 23945, 23948, 23947, 23940, 23939, 23937, 24040, - 24042, 24039, 24038, 24035, 24034, 24037, 24036, 24043, 24041, 23949, - 23942, 23941, 23944, 23943, 35762, 5944, 5962, 5964, 5961, 5945, 5963, - 5953, 5952, 5949, 5948, 5924, 5925, 5947, 5946, 5951, 5950, 5926, 5928, - 5927, 5955, 5954, 5941, 5940, 5929, 5930, 5939, 5935, 5934, 5933, 5938, - 5937, 5931, 5932, 5936, 5960, 5958, 5957, 5959, 5942, 5943, 5956, 5969, - 5972, 5973, 5978, 5976, 5975, 5974, 5970, 5971, 5977, 5912, 5910, 5909, - 5911, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 5918, 5917, 5914, 5906, 5916, 5922, 5913, 5920, 5923, 5921, 5919, 5915, - 5908, 5907, 35762, 35762, 5985, 5987, 5984, 5983, 5980, 5979, 5982, 5981, - 5988, 5986, 35762, 35762, 5965, 5967, 5966, 5968, 23892, 23885, 23884, - 23889, 23888, 23882, 23881, 23880, 23878, 23877, 23879, 23883, 23894, - 23887, 23886, 23891, 23985, 23895, 23896, 23893, 23982, 23983, 23984, - 24013, 24015, 24014, 23867, 24009, 24000, 24001, 23931, 23932, 30279, - 30255, 30278, 30254, 30277, 30253, 30292, 30268, 30286, 30262, 30281, - 30257, 30280, 30256, 30297, 30273, 30287, 30263, 30290, 30266, 30285, - 30261, 30284, 30260, 30288, 30264, 30289, 30265, 30283, 30259, 30282, - 30258, 30291, 30267, 30295, 30271, 30299, 30275, 30296, 30272, 30294, - 30270, 30298, 30274, 30293, 30269, 30300, 30276, 30301, 30313, 30321, - 30318, 30317, 30323, 30324, 30302, 30322, 30319, 30320, 30312, 30316, - 30315, 30314, 30311, 30308, 30309, 30310, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30304, 30305, 30307, 30306, 30303, 22359, 22360, 22318, 22335, 22346, - 22345, 22322, 22321, 22334, 22340, 22341, 22373, 22375, 22365, 22367, - 22366, 22313, 22310, 22312, 22362, 22363, 22377, 22378, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 12895, 12893, - 12892, 12891, 12890, 12894, 35762, 35762, 12589, 12587, 12586, 12585, - 12584, 12588, 35762, 35762, 12604, 12602, 12601, 12600, 12599, 12603, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 12545, - 12551, 12549, 12546, 12548, 12547, 12550, 35762, 12530, 12536, 12534, - 12531, 12533, 12532, 12535, 35762, 18776, 18752, 18782, 18777, 18873, - 19035, 19223, 19025, 19016, 19019, 19046, 19060, 18887, 18780, 18781, - 19130, 18981, 19272, 19271, 19273, 19274, 19233, 18672, 19177, 18835, - 19157, 18836, 19220, 19221, 18779, 19343, 19309, 19352, 19295, 19348, - 18799, 18800, 18801, 19384, 19381, 19382, 19383, 19395, 23002, 23225, - 23234, 23235, 23292, 19214, 18984, 19131, 19354, 18980, 14005, 18842, - 19304, 19277, 23289, 23138, 23162, 35762, 35762, 35762, 35762, 6167, - 6168, 6169, 6170, 6171, 6172, 6130, 6166, 6131, 6132, 6133, 6134, 6135, - 6095, 6096, 6097, 6098, 6099, 6100, 6136, 6137, 6138, 6139, 6140, 6141, - 6142, 6143, 6144, 6145, 6146, 6101, 6094, 6102, 6103, 6104, 6105, 6106, - 6107, 6148, 6149, 6150, 6151, 6152, 6153, 6109, 6108, 6110, 6111, 6112, - 6113, 6114, 6088, 6127, 6089, 6128, 6090, 6129, 6091, 6092, 6093, 6087, - 6115, 6116, 6117, 6118, 6119, 6120, 6121, 6122, 6123, 6124, 6125, 6126, - 6154, 6155, 6156, 6157, 6158, 6159, 6160, 22327, 22339, 22349, 22351, - 22336, 22330, 22317, 22342, 22329, 22332, 22344, 22355, 22357, 22353, - 22358, 22347, 22338, 22356, 22324, 22325, 22354, 22316, 22326, 22320, - 22323, 22319, 22315, 22328, 22350, 22352, 22337, 22331, 22343, 22333, - 22348, 22369, 22372, 22364, 22371, 22370, 22374, 22368, 22376, 22314, - 22361, 22311, 35762, 35762, 22385, 22387, 22384, 22383, 22380, 22379, - 22382, 22381, 22388, 22386, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 14971, 14969, 15002, 15003, - 15004, 14980, 14981, 15013, 15015, 14948, 14946, 14945, 14949, 14955, - 14956, 14958, 14957, 14962, 14959, 14960, 14964, 14932, 14930, 35762, - 35762, 35762, 35762, 14828, 14829, 14897, 14898, 14917, 14911, 14912, - 14915, 14914, 14913, 14872, 14857, 14896, 14863, 14867, 14866, 14880, - 14879, 14800, 14825, 14818, 14903, 14816, 14823, 14853, 14846, 14852, - 14907, 14848, 14851, 14850, 14891, 14884, 14901, 14902, 14890, 14888, - 14887, 14892, 14894, 14839, 14838, 14924, 14925, 14789, 14788, 14904, - 14843, 14841, 35762, 35762, 35762, 35762, 7281, 7282, 7283, 7284, 7285, - 7286, 7287, 7288, 7289, 7290, 7291, 7292, 7293, 7294, 7295, 7296, 7297, - 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305, 7306, 7307, 7308, 7309, - 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, - 7322, 7323, 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, - 7334, 7335, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, 7344, 7345, - 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7355, 7356, 7357, - 7358, 7359, 7360, 7361, 7362, 7363, 7364, 7365, 7366, 7367, 7368, 7369, - 7370, 7371, 7372, 7373, 7374, 7375, 7376, 7377, 7378, 7379, 7380, 7381, - 7382, 7383, 7384, 7385, 7386, 7387, 7388, 7389, 7390, 7391, 7392, 7393, - 7394, 7395, 7396, 7397, 7398, 7399, 7400, 7401, 7402, 7403, 7404, 7405, - 7406, 7407, 7408, 7409, 7410, 7411, 7412, 7413, 7414, 7415, 7416, 7417, - 7418, 7419, 7420, 7421, 7422, 7423, 7424, 7425, 7426, 7427, 7428, 7429, - 7430, 7431, 7432, 7433, 7434, 7435, 7436, 7437, 7438, 7439, 7440, 7441, - 7442, 7443, 7444, 7445, 7446, 7447, 7448, 7449, 7450, 7451, 7452, 7453, - 7454, 7455, 7456, 7457, 7458, 7459, 7460, 7461, 7462, 7463, 7464, 7465, - 7466, 7467, 7468, 7469, 7470, 7471, 7472, 7473, 7474, 7475, 7476, 7477, - 7478, 7479, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, - 7490, 7491, 7492, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, - 7502, 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510, 7511, 7512, 7513, - 7514, 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, - 7526, 7527, 7528, 7529, 7530, 7531, 7532, 7533, 7534, 7535, 7536, 7079, - 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, - 7092, 7093, 7094, 7095, 7096, 7097, 7098, 7099, 7100, 7101, 7102, 7103, - 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7113, 7114, 7115, - 7116, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, - 7128, 7129, 7130, 7131, 7132, 7133, 7134, 7135, 7136, 7137, 7138, 7139, - 7140, 7141, 7142, 7143, 7144, 7145, 7146, 7147, 7148, 7149, 7150, 7151, - 7152, 7153, 7154, 7155, 7156, 7157, 7158, 7159, 7160, 7161, 7162, 7163, - 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7065, - 7066, 7067, 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7076, 7077, - 7078, 35762, 35762, 7175, 7176, 7177, 7178, 7179, 7180, 7181, 7182, 7183, - 7184, 7185, 7186, 7187, 7188, 7189, 7190, 7191, 7192, 7193, 7194, 7195, - 7196, 7197, 7198, 7199, 7200, 7201, 7202, 7203, 7204, 7205, 7206, 7207, - 7208, 7209, 7210, 7211, 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, - 7220, 7221, 7222, 7223, 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, - 7232, 7233, 7234, 7235, 7236, 7237, 7238, 7239, 7240, 7241, 7242, 7243, - 7244, 7245, 7246, 7247, 7248, 7249, 7250, 7251, 7252, 7253, 7254, 7255, - 7256, 7257, 7258, 7259, 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, - 7268, 7269, 7270, 7271, 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, - 7280, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 19417, - 19420, 19421, 19418, 19419, 19423, 19424, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 2490, 2491, 2489, - 2492, 2488, 35762, 35762, 35762, 35762, 35762, 15435, 15453, 15440, - 15371, 15427, 15430, 15432, 15431, 15426, 15433, 15429, 15428, 15372, - 15405, 15406, 15403, 15404, 15369, 15370, 15368, 15376, 15418, 15416, - 15393, 15425, 15398, 35762, 15410, 15436, 15384, 15379, 15420, 35762, - 15422, 35762, 15396, 15400, 35762, 15386, 15382, 35762, 15414, 15391, - 15408, 15402, 15412, 15424, 15375, 15378, 15381, 15437, 1146, 1145, 1176, - 1174, 1175, 1177, 1403, 1401, 1402, 1404, 1171, 1169, 1170, 1172, 1532, - 1530, 1531, 1533, 1512, 1510, 1511, 1513, 1527, 1525, 1526, 1528, 1545, - 1543, 1544, 1546, 1408, 1406, 1407, 1409, 1211, 1209, 1210, 1212, 1381, - 1379, 1380, 1382, 1489, 1487, 1488, 1490, 1494, 1492, 1493, 1495, 1201, - 1200, 1192, 1191, 1215, 1214, 1204, 1203, 1311, 1310, 1438, 1437, 1333, - 1331, 1332, 1334, 1245, 1243, 1244, 1246, 1257, 1255, 1256, 1258, 1372, - 1370, 1371, 1373, 1386, 1385, 1442, 1440, 1441, 1443, 1287, 1286, 1282, - 1280, 1281, 1283, 1291, 1289, 1290, 1292, 1569, 1568, 1567, 1566, 2352, - 2351, 2360, 2359, 2356, 2355, 2354, 2353, 2364, 2363, 2350, 2358, 2357, - 2366, 2362, 2361, 2365, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 1376, - 1374, 1375, 1377, 1538, 1537, 1590, 1589, 1587, 1586, 1536, 1548, 1547, - 1337, 1336, 1340, 1339, 1603, 1601, 1602, 1604, 1539, 1540, 2043, 2042, - 2045, 2044, 2064, 2063, 2072, 2071, 2062, 2061, 2068, 2067, 2048, 2046, - 2047, 2097, 2095, 2096, 1225, 1223, 1224, 1226, 2054, 2052, 2058, 2041, - 2066, 1645, 1636, 1641, 1648, 1643, 1654, 2006, 1994, 2001, 2014, 2017, - 2022, 2024, 2029, 2026, 2035, 1723, 1729, 1708, 1703, 1762, 1758, 1764, - 1913, 1906, 1918, 1926, 1885, 1889, 1667, 1659, 1663, 1669, 1987, 1982, - 2099, 1616, 1612, 1697, 1693, 1681, 1686, 1677, 1684, 1679, 1688, 1867, - 1863, 1865, 1869, 1738, 1740, 1748, 1746, 1743, 1754, 1736, 1757, 1790, - 1782, 1794, 1800, 1774, 1803, 1821, 1810, 1815, 1825, 1804, 1826, 1842, - 1832, 1854, 1847, 1850, 1857, 1718, 1714, 1715, 1716, 2082, 2075, 2084, - 2090, 2039, 2094, 2023, 1880, 1628, 1934, 1937, 1941, 1935, 1938, 1940, - 2070, 2069, 2056, 2060, 2040, 2065, 1652, 1651, 1646, 1650, 1642, 1653, - 2020, 2019, 2012, 2018, 2016, 2021, 2033, 2032, 2027, 2031, 2025, 2034, - 1678, 1687, 1864, 1868, 1737, 1741, 1752, 1735, 1756, 1798, 1773, 1802, - 1805, 1823, 1855, 1852, 1845, 1851, 1849, 1856, 1627, 2092, 2079, 2088, - 2078, 2038, 2093, 2053, 2051, 2055, 2057, 2049, 1644, 1635, 1640, 1647, - 1637, 2005, 1993, 2000, 2013, 1995, 2028, 1722, 1728, 1707, 1702, 1761, - 1763, 1912, 1905, 1917, 1925, 1884, 1892, 1888, 1666, 1658, 1662, 1668, - 1986, 2098, 1615, 1611, 1696, 1692, 1680, 1685, 1676, 1683, 1866, 1862, - 1739, 1747, 1745, 1742, 1753, 1789, 1781, 1793, 1799, 1783, 1820, 1809, - 1814, 1824, 1841, 1831, 1853, 1846, 1833, 1717, 1713, 1719, 2081, 2074, - 2083, 2089, 2076, 2059, 2050, 1649, 1638, 2015, 1996, 2030, 2036, 1927, - 1909, 1964, 1944, 1744, 1755, 1801, 1848, 1834, 2091, 2077, 1942, 1936, - 1939, 1985, 1989, 1618, 1620, 1695, 1699, 1929, 1933, 1966, 1974, 1705, - 1710, 1731, 1733, 1760, 1766, 1891, 1896, 1665, 1673, 1955, 1950, 1969, - 1963, 1972, 1931, 1894, 1671, 1984, 1988, 1617, 1619, 1694, 1698, 1928, - 1932, 1965, 1973, 1704, 1709, 1730, 1732, 1759, 1765, 1890, 1895, 1664, - 1672, 1953, 1948, 1967, 1961, 1971, 1930, 1893, 1670, 1954, 1949, 1968, - 1962, 1908, 1943, 1981, 1914, 1907, 1919, 1956, 1951, 1970, 1983, 2100, - 1629, 1630, 25893, 25894, 1877, 1872, 1876, 1873, 1874, 1875, 1903, 1622, - 1624, 1623, 1621, 1871, 1902, 1625, 1976, 1878, 2003, 1992, 1991, 1990, - 1998, 2008, 2010, 2009, 1725, 1724, 1701, 1700, 1904, 1911, 1910, 1921, - 1920, 1922, 1924, 1923, 1882, 1881, 1887, 1946, 1945, 1952, 1958, 1957, - 1960, 1959, 1656, 1661, 1660, 1978, 1977, 1979, 1980, 1614, 1609, 1608, - 1607, 1689, 1691, 1690, 1675, 1674, 1860, 1858, 1779, 1780, 1777, 1784, - 1785, 1792, 1791, 1796, 1795, 1806, 1807, 1808, 1818, 1816, 1811, 1812, - 35762, 35762, 1817, 1711, 1712, 1829, 1828, 1839, 1838, 1837, 1844, 1843, - 2086, 2085, 1639, 2004, 2002, 1999, 1997, 2011, 2007, 1727, 1720, 1726, - 1915, 1883, 1947, 1657, 1788, 1797, 2073, 2080, 2087, 1822, 1861, 1830, - 1859, 1778, 1610, 1751, 1835, 1813, 1786, 1750, 1787, 1836, 1721, 1706, - 1819, 1682, 1634, 1749, 1613, 1886, 1916, 1840, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 1898, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 1899, 1870, 1631, 1632, - 1827, 1897, 1879, 1626, 2037, 1900, 1901, 1734, 27339, 1655, 1975, 1633, - 33083, 33194, 33262, 33273, 33284, 33295, 33306, 33317, 33328, 33084, - 33095, 33106, 33117, 33128, 33139, 33150, 26827, 26832, 26833, 26826, - 26857, 26828, 26859, 26835, 26849, 26831, 35762, 35762, 35762, 35762, - 35762, 35762, 8057, 8059, 7889, 7890, 8068, 8070, 7783, 8058, 8060, 8126, - 8127, 8069, 8071, 7784, 7837, 7838, 26858, 26829, 26830, 26844, 26856, - 26841, 26853, 26839, 26851, 26843, 26855, 26836, 26845, 26837, 26846, - 26840, 26852, 26838, 26850, 26834, 26847, 27753, 34075, 26842, 26854, - 10235, 5828, 33955, 10947, 10234, 5827, 33953, 28892, 28901, 28935, - 35762, 28926, 28893, 28936, 28899, 28900, 28903, 28907, 28902, 28906, - 28904, 28908, 28934, 28883, 28885, 28933, 28931, 28905, 28930, 28898, - 35762, 28925, 28894, 28932, 28891, 35762, 35762, 35762, 35762, 1080, - 2371, 1062, 2373, 1092, 35762, 1077, 1078, 1058, 1059, 1089, 1090, 2278, - 2279, 2346, 2347, 1276, 1141, 1140, 1131, 1130, 1556, 1555, 1134, 1133, - 1573, 1571, 1572, 1574, 1151, 1150, 1166, 1164, 1165, 1167, 1499, 1498, - 1508, 1506, 1507, 1501, 1520, 1518, 1519, 1521, 1307, 1305, 1306, 1308, - 1273, 1271, 1272, 1274, 1344, 1342, 1343, 1345, 1189, 1188, 1516, 1515, - 1435, 1434, 1598, 1597, 1464, 1462, 1463, 1465, 1470, 1468, 1469, 1471, - 1451, 1449, 1450, 1452, 1197, 1195, 1196, 1198, 1482, 1480, 1481, 1483, - 1594, 1592, 1593, 1595, 1109, 1107, 1108, 1110, 1252, 1250, 1251, 1253, - 1236, 1234, 1235, 1237, 1417, 1415, 1416, 1418, 1320, 1318, 1319, 1321, - 1356, 1354, 1355, 1357, 1365, 1363, 1364, 1366, 1396, 1394, 1395, 1397, - 1295, 1293, 1294, 1296, 1560, 1559, 1149, 1148, 1583, 1581, 1582, 1584, - 1772, 1771, 1768, 1767, 1770, 1769, 1776, 1775, 35762, 35762, 35566, - 35762, 13299, 13303, 13271, 13287, 13273, 13285, 13284, 13214, 13277, - 13286, 13274, 13209, 13302, 13308, 13281, 13294, 13296, 13293, 13292, - 13289, 13288, 13291, 13290, 13297, 13295, 13210, 13280, 13216, 13298, - 13301, 13304, 13208, 13217, 13218, 13219, 13220, 13221, 13222, 13223, - 13224, 13225, 13226, 13227, 13228, 13229, 13230, 13231, 13232, 13233, - 13234, 13235, 13236, 13237, 13238, 13239, 13240, 13241, 13242, 13215, - 13279, 13278, 13207, 13269, 13300, 13243, 13244, 13245, 13246, 13247, - 13248, 13249, 13250, 13251, 13252, 13253, 13254, 13255, 13256, 13257, - 13258, 13259, 13260, 13261, 13262, 13263, 13264, 13265, 13266, 13267, - 13268, 13212, 13309, 13275, 13306, 13213, 13276, 14644, 14637, 14639, - 14643, 14635, 14627, 14582, 14584, 14586, 14583, 14585, 14578, 14580, - 14579, 14581, 14636, 14628, 14630, 14632, 14629, 14631, 14603, 14605, - 14607, 14604, 14606, 14587, 14589, 14591, 14588, 14590, 14618, 14620, - 14622, 14619, 14621, 14593, 14595, 14597, 14594, 14596, 14598, 14600, - 14602, 14599, 14601, 14608, 14610, 14612, 14609, 14611, 14623, 14625, - 14624, 14613, 14615, 14617, 14614, 14616, 14626, 14592, 14634, 14633, - 14577, 14527, 14544, 14528, 14529, 14530, 14531, 14565, 14546, 14535, - 14540, 14539, 14538, 14542, 14536, 14537, 14541, 14563, 14533, 14545, - 14534, 14548, 14547, 14562, 14557, 14543, 14556, 14526, 14564, 14532, - 14571, 35762, 35762, 35762, 14572, 14573, 14551, 14552, 14559, 14558, - 35762, 35762, 14550, 14549, 14574, 14568, 14569, 14575, 35762, 35762, - 14554, 14576, 14567, 14566, 14570, 14555, 35762, 35762, 14560, 14553, - 14561, 35762, 35762, 35762, 13211, 13272, 13270, 13283, 13307, 13282, - 13305, 35762, 14642, 14638, 14641, 14640, 14647, 14645, 14646, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16324, - 16326, 16325, 25221, 27062, 35762, 35762, 20348, 20374, 20367, 20400, - 20358, 20349, 20378, 20344, 20356, 20386, 20389, 20383, 35762, 20372, - 20399, 20403, 20382, 20397, 20404, 20411, 20414, 20359, 20410, 20357, - 20360, 20343, 20366, 20369, 20392, 20393, 20351, 20408, 20373, 20354, - 20390, 20352, 20409, 20368, 20376, 35762, 20401, 20365, 20385, 20350, - 20361, 20375, 20346, 20380, 20355, 20387, 20388, 20345, 20371, 20347, - 20398, 20391, 20405, 20379, 20381, 35762, 20353, 20407, 35762, 20364, - 20363, 20377, 20413, 20406, 20416, 20384, 20362, 20394, 20402, 20370, - 20395, 20396, 20412, 20415, 35762, 35762, 20426, 20427, 20429, 20428, - 20417, 20418, 20425, 20419, 20420, 20430, 20421, 20422, 20423, 20424, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 20232, 20231, 20233, 20220, 20221, 20222, - 20223, 20224, 20225, 20226, 20228, 20227, 20230, 20229, 20238, 20235, - 20236, 20234, 20237, 20337, 20338, 20240, 20239, 20241, 20340, 20339, - 20243, 20244, 20245, 20242, 20246, 20249, 20248, 20250, 20251, 20252, - 20341, 20253, 20254, 20247, 20257, 20258, 20256, 20255, 20259, 20260, - 20261, 20262, 20263, 20264, 20267, 20268, 20269, 20266, 20270, 20265, - 20271, 20272, 20273, 20274, 20275, 20276, 20277, 20278, 20279, 20280, - 20282, 20281, 20283, 20284, 20286, 20287, 20288, 20285, 20289, 20290, - 20291, 20292, 20294, 20293, 20295, 20296, 20342, 20297, 20298, 20300, - 20301, 20302, 20299, 20303, 20304, 20305, 20306, 20307, 20335, 20315, - 20316, 20317, 20318, 20319, 20320, 20321, 20322, 20323, 20324, 20325, - 20326, 20327, 20328, 20329, 20330, 20331, 20332, 20333, 20334, 20308, - 20309, 20310, 20311, 20312, 20313, 20314, 20336, 35762, 35762, 35762, - 35762, 35762, 112, 113, 159, 35762, 35762, 35762, 35762, 156, 151, 146, - 126, 121, 139, 134, 114, 129, 154, 149, 144, 124, 119, 140, 135, 115, - 130, 157, 152, 147, 127, 122, 142, 137, 117, 132, 158, 153, 148, 128, - 123, 143, 138, 118, 133, 155, 150, 145, 125, 120, 141, 136, 116, 131, - 35762, 35762, 35762, 111, 107, 109, 110, 108, 103, 104, 105, 106, 13770, - 13767, 13771, 13757, 13752, 13758, 13761, 13753, 13763, 13772, 13755, - 13765, 13759, 13768, 13762, 13764, 13774, 13756, 13766, 13760, 13769, - 13773, 13754, 13775, 13787, 13796, 13786, 13782, 13795, 13777, 13783, - 13799, 13803, 13804, 13785, 13788, 13794, 13793, 13801, 13802, 13784, - 13791, 13797, 13792, 13781, 13800, 13789, 13776, 13778, 13798, 13790, - 13779, 13780, 14022, 14023, 14221, 14217, 14258, 14223, 13953, 14027, - 14220, 14216, 13959, 13958, 14019, 14001, 14017, 14026, 14021, 13807, - 13806, 14260, 14219, 14261, 14024, 14215, 13997, 24651, 35762, 27384, - 27387, 27382, 27385, 27356, 27386, 27354, 27357, 27383, 27355, 27388, - 27353, 2510, 35762, 35762, 35762, 14214, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 26611, 26612, 26623, 26592, 26595, 26626, 26604, - 26603, 26624, 26631, 26590, 26617, 26596, 26609, 26610, 26619, 26608, - 26589, 26593, 26600, 26598, 26620, 26597, 26588, 26618, 26605, 26606, - 26591, 26594, 26616, 26630, 26601, 26625, 26587, 26613, 26632, 26614, - 26615, 26607, 26629, 26628, 26602, 26621, 26622, 26627, 26599, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 20605, - 20607, 20600, 20601, 20612, 20611, 20614, 20622, 20624, 20603, 20615, - 20598, 20618, 20616, 20596, 20609, 20597, 20610, 20621, 20617, 20599, - 20619, 20620, 20602, 20604, 20606, 20608, 20613, 20623, 35762, 35762, - 35762, 5738, 5749, 5740, 5711, 5734, 5750, 5712, 5739, 5756, 5754, 5714, - 5755, 5741, 5729, 5724, 5725, 5723, 5709, 5732, 5722, 5757, 5719, 5731, - 5748, 5728, 5752, 5742, 5737, 5746, 5747, 5720, 5733, 5744, 5745, 5726, - 5727, 5721, 5753, 5710, 5730, 5735, 5751, 5715, 5716, 5717, 5718, 5713, - 5743, 5736, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 8241, 8267, 8265, 8264, - 8261, 8260, 8263, 8262, 8268, 8266, 8259, 8258, 8256, 8247, 8245, 8254, - 8252, 8243, 8249, 8250, 8257, 8255, 8246, 8244, 8253, 8251, 8242, 8248, - 35762, 35762, 35762, 35762, 25459, 25453, 25439, 25454, 25426, 25456, - 25458, 25455, 25450, 25446, 25438, 25434, 25435, 25436, 25428, 25460, - 25449, 25442, 25440, 25430, 25427, 25451, 25444, 25432, 25448, 25437, - 25433, 25431, 25452, 25447, 25445, 25429, 25464, 25462, 25463, 25461, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 25457, - 25443, 25441, 13617, 13633, 13641, 13635, 13616, 13626, 13620, 13619, - 13628, 13638, 13642, 13639, 13636, 13624, 13640, 13631, 13625, 13623, - 13627, 13637, 13629, 13630, 13632, 13621, 13618, 13634, 13622, 35762, - 35762, 35762, 35762, 35762, 25531, 25530, 25527, 25500, 25501, 25523, - 25498, 25524, 25499, 25503, 25532, 25525, 25506, 25507, 25514, 25508, - 25526, 25511, 25513, 25534, 25497, 25510, 25509, 25521, 25518, 25528, - 25529, 25502, 25533, 25512, 25515, 25516, 25517, 25520, 25505, 25522, - 25519, 25504, 8083, 8081, 8079, 8080, 8082, 35762, 35762, 35762, 35762, - 35762, 32585, 32588, 32592, 32596, 32589, 32593, 32611, 32607, 32594, - 32604, 32606, 32595, 32601, 32597, 32612, 32590, 32609, 32608, 32600, - 32586, 32610, 32599, 32587, 32598, 32603, 32591, 32605, 32613, 32614, - 32602, 35762, 32615, 25540, 25582, 25583, 25563, 25564, 25561, 25562, - 25560, 25575, 25552, 25553, 25557, 25558, 25547, 25550, 25551, 25556, - 25579, 25544, 25576, 25565, 25566, 25569, 25570, 25571, 25580, 25554, - 25555, 25567, 25568, 25578, 25574, 25581, 25572, 25573, 25577, 35762, - 35762, 35762, 35762, 25541, 25542, 25543, 25559, 25548, 25549, 25545, - 25546, 25584, 25539, 25536, 25537, 25535, 25538, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 10286, - 10285, 10281, 10282, 10283, 10284, 10292, 10291, 10287, 10288, 10289, - 10290, 10309, 10294, 10308, 10305, 10310, 10303, 10300, 10296, 10301, - 10299, 10302, 10307, 10306, 10276, 10304, 10275, 10295, 10271, 10298, - 10272, 10297, 10279, 10277, 10278, 10273, 10274, 10293, 10280, 10326, - 10325, 10321, 10322, 10323, 10324, 10332, 10331, 10327, 10328, 10329, - 10330, 10349, 10334, 10348, 10345, 10350, 10343, 10340, 10336, 10341, - 10339, 10342, 10347, 10346, 10316, 10344, 10315, 10335, 10311, 10338, - 10312, 10337, 10319, 10317, 10318, 10313, 10314, 10333, 10320, 27890, - 27896, 27907, 27904, 27894, 27893, 27892, 27871, 27899, 27877, 27906, - 27902, 27905, 27908, 27895, 27903, 27882, 27901, 27898, 27876, 27881, - 27883, 27880, 27875, 27869, 27866, 27888, 27897, 27886, 27870, 27891, - 27909, 27873, 27867, 27879, 27910, 27887, 27884, 27885, 27868, 27864, - 27889, 27865, 27874, 27863, 27872, 27878, 27900, 25971, 25989, 25995, - 25993, 25996, 25977, 25974, 25994, 25979, 25978, 25975, 25973, 25991, - 25990, 25981, 25976, 25984, 25980, 25985, 25986, 25992, 25997, 25970, - 25987, 25998, 25982, 25999, 25972, 25988, 25983, 35762, 35762, 26006, - 26008, 26005, 26004, 26001, 26000, 26003, 26002, 26009, 26007, 35762, - 35762, 35762, 35762, 35762, 35762, 25898, 25899, 25900, 25901, 25926, - 25919, 25905, 25902, 25908, 25918, 25917, 25932, 25911, 25906, 25910, - 25927, 25928, 25929, 25912, 25913, 25930, 25907, 25923, 25922, 25916, - 25904, 25915, 25903, 25914, 25920, 25933, 25931, 25909, 25921, 25925, - 25924, 35762, 35762, 35762, 35762, 25934, 25935, 25936, 25937, 25962, - 25955, 25941, 25938, 25944, 25954, 25953, 25968, 25947, 25942, 25946, - 25963, 25964, 25965, 25948, 25949, 25966, 25943, 25959, 25958, 25952, - 25940, 25951, 25939, 25950, 25956, 25969, 25967, 25945, 25957, 25961, - 25960, 35762, 35762, 35762, 35762, 12376, 12367, 12356, 12355, 12358, - 12347, 12357, 12354, 12353, 12368, 12344, 12343, 12369, 12377, 12370, - 12360, 12346, 12345, 12371, 12350, 12349, 12348, 12378, 12372, 12373, - 12352, 12351, 12362, 12361, 12364, 12363, 12379, 12374, 12375, 12380, - 12366, 12365, 12342, 12341, 12359, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 5770, 5819, 5786, 5782, 5784, 5809, 5783, 5807, - 5804, 5773, 5806, 5808, 5787, 5798, 5794, 5789, 5817, 5781, 5772, 5790, - 5793, 5795, 5820, 5811, 5769, 5775, 5776, 5778, 5812, 5810, 5815, 5779, - 5799, 5791, 5818, 5803, 5814, 5780, 5774, 5797, 5785, 5816, 5801, 5813, - 5802, 5800, 5788, 5777, 5771, 5805, 5796, 5792, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 5821, 33727, - 33696, 33697, 33707, 33706, 33709, 33708, 33699, 33698, 33716, 33724, - 35762, 33715, 33714, 33700, 33701, 33717, 33725, 33703, 33702, 33718, - 33705, 33704, 33728, 33719, 33726, 33720, 35762, 33711, 33710, 33713, - 33712, 33729, 33721, 33722, 35762, 33730, 33723, 35762, 33762, 33731, - 33732, 33742, 33741, 33744, 33743, 33734, 33733, 33751, 33759, 35762, - 33750, 33749, 33735, 33736, 33752, 33760, 33738, 33737, 33753, 33740, - 33739, 33763, 33754, 33761, 33755, 35762, 33746, 33745, 33748, 33747, - 33764, 33756, 33757, 35762, 33765, 33758, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 20199, 20200, 20201, 20202, 20203, 20204, 20205, - 20206, 20207, 20138, 20139, 20140, 20141, 20142, 20151, 20143, 20144, - 20145, 20146, 20147, 20148, 20149, 20150, 20152, 20153, 20154, 20155, - 20219, 20156, 20157, 20158, 20159, 20160, 20161, 20162, 20163, 20164, - 20165, 20166, 20167, 20168, 20169, 20170, 20171, 20172, 20173, 20174, - 20175, 20176, 20177, 20178, 20179, 20180, 20181, 20182, 20183, 20184, - 20185, 20186, 20187, 20188, 20189, 20190, 20191, 20192, 20193, 20194, - 20195, 20196, 20197, 20198, 19879, 20215, 20208, 19880, 20209, 20210, - 20211, 20212, 19881, 20216, 20217, 20213, 20214, 20218, 19885, 19886, - 19887, 19888, 19889, 19890, 19891, 19892, 19882, 19883, 19884, 19896, - 19897, 19898, 19893, 19894, 19895, 19899, 19900, 19901, 19902, 19903, - 19904, 19907, 19908, 19909, 19910, 19911, 19912, 19913, 19914, 19915, - 19916, 19917, 19918, 19919, 19920, 19921, 19922, 19923, 19924, 19925, - 19926, 19927, 19928, 19929, 19930, 19931, 19932, 19933, 19934, 19935, - 19936, 19937, 19938, 19939, 19940, 19941, 19942, 19943, 19944, 19945, - 19946, 19947, 19948, 19949, 19950, 19951, 19952, 19953, 19954, 19955, - 19956, 19905, 19906, 19957, 19958, 19959, 19960, 19961, 19962, 19963, - 19964, 19965, 19966, 19967, 19968, 19969, 19970, 19971, 19972, 19973, - 19974, 19975, 19976, 19977, 19978, 19979, 19980, 19981, 19982, 19983, - 19984, 19985, 19986, 19987, 19988, 19989, 20021, 20022, 20023, 20024, - 20025, 20026, 20027, 20028, 20029, 19990, 19991, 19992, 19993, 19994, - 19995, 19996, 19997, 19998, 19999, 20000, 20001, 20002, 20003, 20004, - 20005, 20006, 20007, 20008, 20009, 20010, 20011, 20012, 20013, 20014, - 20015, 20016, 20017, 20018, 20019, 20020, 20036, 20037, 20038, 20039, - 20040, 20041, 20042, 20043, 20044, 20045, 20046, 20047, 20048, 20049, - 20050, 20051, 20052, 20053, 20054, 20055, 20030, 20031, 20032, 20033, - 20034, 20035, 20056, 20057, 20058, 20059, 20060, 20061, 20062, 20063, - 20098, 20099, 20100, 20101, 20102, 20103, 20104, 20105, 20106, 20107, - 20064, 20065, 20066, 20067, 20068, 20069, 20070, 20071, 20072, 20073, - 20074, 20075, 20076, 20077, 20078, 20079, 20080, 20081, 20082, 20083, - 20089, 20090, 20091, 20092, 20093, 20094, 20095, 20096, 20097, 20084, - 20085, 20086, 20087, 20088, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 20119, 20114, 20117, 20116, 20120, 20115, 20113, - 20118, 20112, 20108, 20111, 20110, 20109, 20126, 20121, 20125, 20122, - 20123, 20124, 20127, 20129, 20128, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 20130, 20131, 20132, 20133, 20134, - 20135, 20136, 20137, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 23185, 23306, 23305, - 23170, 23186, 23174, 35762, 23204, 23206, 23205, 23200, 23199, 23197, - 23198, 23259, 23192, 23216, 23256, 23180, 23220, 23181, 23224, 23187, - 23226, 23203, 23240, 23241, 23239, 23183, 23237, 23242, 23243, 23284, - 23285, 23248, 23184, 23193, 23299, 23281, 23282, 23255, 23254, 23189, - 23269, 23272, 23273, 23270, 23268, 23296, 35762, 23191, 23111, 23158, - 23006, 23089, 23118, 23003, 23160, 23261, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 9704, 9705, 9706, 9707, 9708, 9698, - 35762, 35762, 9699, 35762, 9654, 9655, 9656, 9657, 9658, 9659, 9660, - 9661, 9662, 9663, 9664, 9665, 9666, 9667, 9668, 9669, 9670, 9671, 9672, - 9673, 9674, 9675, 9676, 9677, 9678, 9679, 9680, 9681, 9682, 9683, 9684, - 9685, 9686, 9687, 9688, 9689, 9690, 9691, 9692, 9693, 9694, 9695, 9696, - 9697, 35762, 9702, 9703, 35762, 35762, 35762, 9700, 35762, 35762, 9701, - 16146, 16157, 16148, 16142, 16154, 16159, 16149, 16155, 16140, 16153, - 16156, 16143, 16161, 16158, 16150, 16147, 16160, 16151, 16144, 16145, - 16152, 16141, 35762, 16162, 16137, 16133, 16136, 16134, 16132, 16138, - 16139, 16135, 26241, 26252, 26243, 26237, 26249, 26254, 26244, 26250, - 26235, 26248, 26251, 26238, 26256, 26234, 26253, 26245, 26242, 26255, - 26246, 26239, 26240, 26247, 26236, 26233, 26257, 26264, 26259, 26260, - 26263, 26262, 26261, 26258, 24101, 24116, 24106, 24127, 24118, 24112, - 24108, 24124, 24129, 24119, 24125, 24110, 24104, 24123, 24105, 24126, - 24102, 24113, 24109, 24131, 24107, 24128, 24120, 24117, 24130, 24121, - 24114, 24115, 24103, 24122, 24111, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 24136, 24133, 24134, 24139, 24140, 24138, 24135, - 24132, 24137, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 15202, 15218, 15210, 15209, 15215, 15220, 15204, 15216, 15205, 15214, - 15217, 15207, 15222, 15219, 15211, 15203, 15221, 15212, 15208, 35762, - 15213, 15206, 35762, 35762, 35762, 35762, 35762, 15223, 15227, 15226, - 15225, 15224, 26635, 26654, 26650, 26637, 26638, 26646, 26647, 26639, - 26645, 26648, 26651, 26653, 26656, 26652, 26643, 26636, 26655, 26641, - 26640, 26649, 26642, 26644, 26661, 26660, 26657, 26662, 26658, 26659, - 35762, 35762, 35762, 26663, 20631, 20637, 20641, 20639, 20633, 20649, - 20642, 20650, 20643, 20627, 20644, 20635, 20645, 20647, 20630, 20625, - 20648, 20640, 20646, 20629, 20626, 20632, 20634, 20628, 20636, 20638, - 35762, 35762, 35762, 35762, 35762, 20651, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 22722, 22723, 22724, 22725, 22721, 22720, 22696, 22697, 22718, 22717, - 22700, 22701, 22702, 22703, 22698, 22699, 22716, 22713, 22712, 22704, - 22705, 22706, 22714, 22719, 22707, 22708, 22709, 22710, 22711, 22715, - 22726, 22727, 22618, 22639, 22640, 22641, 22638, 22637, 22630, 22634, - 22633, 22623, 22624, 22636, 22632, 22628, 22627, 22625, 22619, 22626, - 22629, 22635, 22620, 22621, 22622, 22631, 35762, 35762, 35762, 35762, - 22606, 22611, 22643, 22642, 22692, 22684, 22678, 22655, 22649, 22672, - 22666, 22644, 22661, 22690, 22688, 22682, 22659, 22653, 22676, 22670, - 35762, 35762, 22693, 22685, 22679, 22656, 22650, 22673, 22667, 22646, - 22663, 22695, 22687, 22681, 22658, 22652, 22675, 22669, 22648, 22665, - 22691, 22689, 22683, 22660, 22654, 22677, 22671, 22645, 22662, 22694, - 22686, 22680, 22657, 22651, 22674, 22668, 22647, 22664, 22610, 22616, - 22615, 22609, 22608, 22613, 22612, 22607, 22617, 22614, 17200, 17222, - 17224, 17220, 35762, 17221, 17223, 35762, 35762, 35762, 35762, 35762, - 17225, 17216, 17219, 17218, 17166, 17164, 17188, 17187, 35762, 17186, - 17185, 17194, 35762, 17174, 17170, 17169, 17177, 17176, 17173, 17172, - 17171, 17179, 17178, 17175, 17190, 17189, 17184, 17183, 17196, 17198, - 17197, 17195, 17192, 17180, 17181, 17182, 17199, 17193, 17165, 17167, - 17168, 17191, 35762, 35762, 17214, 17215, 17217, 35762, 35762, 35762, - 35762, 17226, 17163, 17162, 17161, 17160, 17202, 17201, 17203, 17204, - 17227, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 17208, 17213, - 17206, 17205, 17212, 17210, 17209, 17207, 17211, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 25648, 25644, 25649, 25654, 25645, 25652, - 25638, 25646, 25650, 25642, 25637, 25633, 25651, 25634, 25636, 25635, - 25653, 25626, 25627, 25629, 25632, 25630, 25631, 25641, 25643, 25628, - 25647, 25640, 25639, 25656, 25655, 25657, 25470, 25488, 25469, 25479, - 25491, 25492, 25481, 25485, 25483, 25476, 25480, 25472, 25487, 25471, - 25493, 25482, 25484, 25465, 25466, 25489, 25468, 25490, 25467, 25475, - 25477, 25473, 25486, 25474, 25478, 25496, 25495, 25494, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 20932, 20936, 20935, 20940, 20939, 20937, 20961, 20964, 20980, 20944, - 20943, 20942, 20941, 20962, 20954, 20960, 20946, 20956, 20945, 20958, - 20938, 20953, 20967, 20963, 20950, 20934, 20933, 20966, 20965, 20951, - 20948, 20957, 20947, 20959, 20952, 20949, 20955, 20982, 20981, 35762, - 35762, 35762, 35762, 20968, 20972, 20971, 20970, 20969, 20979, 20977, - 20975, 20974, 20973, 20978, 20976, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 2526, 2527, 2533, 2529, 2532, 2528, 2530, - 2531, 2569, 2570, 2559, 2560, 2561, 2562, 2557, 2558, 2574, 2547, 2546, - 2545, 2536, 2534, 2535, 2571, 2573, 2556, 2554, 2566, 2565, 2555, 2577, - 2572, 2564, 2563, 2541, 2540, 2539, 2544, 2543, 2542, 2576, 2537, 2552, - 2553, 2579, 2578, 2575, 2551, 2568, 2549, 2567, 2548, 2550, 2538, 35762, - 35762, 35762, 2580, 32334, 28927, 18140, 18135, 18141, 18136, 16288, - 16299, 16290, 16284, 16296, 16301, 16291, 16297, 16282, 16295, 16298, - 16285, 16303, 16300, 16292, 16289, 16302, 16293, 16286, 16287, 16294, - 16283, 35762, 35762, 16308, 16305, 16306, 16311, 16307, 16304, 16309, - 16310, 16257, 16271, 16262, 16258, 16268, 16261, 16263, 16269, 16255, - 16267, 16270, 16259, 16260, 16272, 16264, 16273, 16265, 16266, 16256, - 35762, 35762, 35762, 35762, 35762, 16278, 16275, 16276, 16281, 16277, - 16274, 16279, 16280, 26895, 26909, 26900, 26896, 26906, 26899, 26901, - 26907, 26905, 26908, 26897, 26898, 26910, 26902, 26912, 26903, 26904, - 26911, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 26920, 26921, - 26893, 26894, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 26917, 26914, 26915, 26919, 26916, 26913, - 26918, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 25658, 25700, 25701, 25690, 25726, 25719, 25693, 25694, 25728, - 25671, 25711, 25659, 25704, 25673, 25713, 25661, 25705, 25672, 25712, - 25660, 25689, 25725, 25679, 25718, 25668, 25708, 25662, 25706, 25695, - 25729, 25674, 25714, 25663, 25684, 25687, 25675, 25664, 25702, 25682, - 25721, 25680, 25720, 25683, 25722, 25710, 25681, 25703, 25688, 25696, - 25691, 25686, 25724, 25676, 25715, 25692, 25727, 25697, 25730, 25677, - 25716, 25665, 25669, 25666, 25670, 25709, 25685, 25723, 25678, 25717, - 25667, 25707, 25698, 25699, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 25318, - 25321, 25344, 25319, 25333, 25329, 25335, 25345, 25320, 25323, 25365, - 25346, 25347, 25336, 25337, 25348, 25366, 25367, 25349, 25350, 25322, - 25362, 25338, 25339, 25324, 25326, 25330, 25358, 25360, 25354, 25356, - 25359, 25351, 25325, 25352, 25368, 25331, 25332, 25340, 25327, 25341, - 25334, 25361, 25364, 25355, 25357, 25353, 25342, 25343, 25328, 25363, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 25369, 25372, 25395, 25370, 25384, 25380, 25386, - 25396, 25371, 25374, 25416, 25397, 25398, 25387, 25388, 25399, 25417, - 25418, 25400, 25401, 25373, 25413, 25389, 25390, 25375, 25377, 25381, - 25409, 25411, 25405, 25407, 25410, 25402, 25376, 25403, 25419, 25382, - 25383, 25391, 25378, 25392, 25385, 25412, 25415, 25406, 25408, 25404, - 25393, 25394, 25379, 25414, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 25422, 25421, 25425, 25420, 25423, 25424, 15157, 15144, 15152, - 15136, 15135, 15149, 15145, 15148, 15133, 15146, 15130, 15129, 15138, - 15137, 15156, 15143, 15142, 15134, 15147, 15150, 15151, 15141, 15154, - 15131, 15155, 15132, 15139, 15140, 15153, 15164, 15166, 15168, 15165, - 15167, 15159, 15158, 15163, 15160, 15162, 15161, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 15175, 15177, 15174, 15173, 15170, - 15169, 15172, 15171, 15178, 15176, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 27439, - 27437, 27436, 27433, 27432, 27435, 27434, 27440, 27438, 27431, 27430, - 27428, 27419, 27417, 27426, 27424, 27415, 27421, 27422, 27429, 27427, - 27418, 27416, 27425, 27423, 27414, 27420, 27411, 27412, 27410, 27413, - 35762, 34219, 34253, 34233, 34232, 34238, 34237, 34215, 34214, 34213, - 34224, 34245, 34218, 34249, 34252, 34251, 34248, 34256, 34235, 34234, - 34236, 34217, 34239, 34250, 34220, 34244, 34255, 34240, 34241, 34228, - 34226, 34225, 34227, 34229, 34216, 34231, 34254, 34242, 34243, 34222, - 34223, 34246, 34221, 35762, 34211, 34210, 34212, 35762, 35762, 34230, - 34247, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 2335, 2336, 2333, 25588, 25596, - 25610, 25597, 25601, 25607, 25598, 25613, 25602, 25608, 25606, 25609, - 25600, 25615, 25611, 25590, 25591, 25603, 25589, 25587, 25614, 25604, - 25592, 25593, 25599, 25605, 25612, 25594, 25595, 25622, 25620, 25617, - 25625, 25624, 25621, 25619, 25618, 25623, 25586, 25616, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 28985, 28999, 28987, 28996, - 29003, 28991, 28997, 28995, 28998, 28988, 29005, 29001, 28992, 28986, - 29004, 28993, 28990, 28994, 29002, 29000, 28989, 28984, 28979, 28977, - 28980, 28978, 28983, 28982, 28974, 28973, 28975, 28981, 28976, 29006, - 29009, 29008, 29007, 29012, 29013, 29010, 29014, 29011, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 25735, 25747, 25745, 25750, 25739, 25744, 25743, 25746, 25737, 25752, - 25748, 25740, 25751, 25741, 25736, 25742, 25749, 25738, 25734, 25733, - 25732, 25731, 25756, 25753, 25754, 25755, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 6190, 6184, 6198, 6192, 6187, 6195, 6201, - 6183, 6193, 6196, 6194, 6197, 6188, 6203, 6199, 6185, 6191, 6202, 6189, - 6186, 6200, 6208, 6205, 6206, 6210, 6207, 6204, 6209, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 12403, 12414, - 12405, 12399, 12411, 12416, 12406, 12412, 12397, 12410, 12413, 12400, - 12418, 12415, 12407, 12404, 12417, 12408, 12401, 12402, 12409, 12398, - 12419, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 4335, 4339, 4337, 4336, 4338, 4261, 4262, 4280, 4281, 4278, 4279, 4273, - 4274, 4275, 4276, 4307, 4263, 4254, 4264, 4300, 4299, 4296, 4295, 4284, - 4294, 4293, 4298, 4297, 4286, 4270, 4269, 4266, 4265, 4285, 4272, 4271, - 4268, 4267, 4287, 4302, 4301, 4292, 4291, 4304, 4306, 4305, 4283, 4277, - 4288, 4289, 4290, 4303, 4282, 4255, 4260, 4259, 4344, 4343, 4353, 4354, - 4351, 4352, 4347, 4348, 4349, 4350, 4355, 4345, 4340, 4346, 4356, 4358, - 4357, 4332, 4331, 4330, 4333, 4329, 35762, 35762, 35762, 35762, 4325, - 4323, 4320, 4311, 4313, 4318, 4316, 4308, 4314, 4324, 4322, 4321, 4310, - 4312, 4319, 4317, 4309, 4315, 4326, 4327, 4365, 4367, 4364, 4363, 4360, - 4359, 4362, 4361, 4368, 4366, 4334, 4257, 4258, 4341, 4342, 4256, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 4328, 16513, - 16515, 16517, 16473, 16474, 16483, 16484, 16481, 16482, 16511, 16475, - 16512, 16476, 16501, 16500, 16497, 16496, 16485, 16495, 16494, 16499, - 16498, 16487, 16478, 16477, 16470, 16468, 16469, 16504, 16486, 16480, - 16479, 16472, 16471, 16488, 16503, 16502, 16493, 16492, 16508, 16510, - 16505, 16507, 16509, 16489, 16490, 16491, 16506, 16523, 16528, 16529, - 16526, 16527, 16530, 16524, 16531, 16525, 16516, 16514, 16534, 16535, - 16532, 16518, 16519, 16521, 16520, 16522, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 16533, 35762, 35762, 29037, - 29038, 29027, 29028, 29029, 29030, 29023, 29024, 29034, 29026, 29039, - 29035, 29040, 29036, 29031, 29033, 29032, 29025, 29041, 29020, 29042, - 29044, 29043, 29021, 29022, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 29051, 29053, 29050, 29049, 29046, 29045, 29048, 29047, 29054, - 29052, 35762, 35762, 35762, 35762, 35762, 35762, 5873, 5875, 5874, 5869, - 5871, 5872, 5870, 5858, 5857, 5854, 5853, 5839, 5852, 5851, 5856, 5855, - 5841, 5844, 5843, 5836, 5835, 5840, 5846, 5845, 5838, 5837, 5842, 5862, - 5861, 5850, 5849, 5864, 5847, 5848, 5865, 5860, 5868, 5866, 5863, 5877, - 5885, 5886, 5881, 5882, 5883, 5879, 5887, 5880, 5888, 5905, 5904, 5889, - 5903, 35762, 5898, 5900, 5897, 5896, 5893, 5892, 5895, 5894, 5901, 5899, - 5876, 5891, 5890, 5902, 5859, 5878, 5884, 5867, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 20693, 20695, 20697, 20694, 20696, - 20685, 20684, 20681, 20680, 20679, 20678, 20683, 20682, 20664, 20673, - 20672, 20669, 20668, 20663, 20675, 20674, 20671, 20670, 20665, 20687, - 20686, 20677, 20676, 20690, 20667, 20689, 20692, 20691, 20688, 20666, - 20700, 20701, 20699, 20698, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 27836, 27835, 27841, 27780, 27781, 27799, 27800, - 27797, 27798, 27792, 27793, 27794, 27795, 27826, 27782, 27827, 27783, - 27819, 27818, 27815, 27814, 27803, 27813, 27812, 27817, 27816, 27805, - 27789, 27788, 27785, 27784, 27804, 27791, 27790, 27787, 27786, 27806, - 27821, 27820, 27811, 27810, 27823, 27825, 27824, 27802, 27801, 27796, - 27807, 27808, 27809, 27822, 27845, 27854, 27855, 27852, 27853, 27848, - 27849, 27850, 27851, 27856, 27846, 27857, 27847, 27840, 27834, 27838, - 27839, 27860, 27766, 27765, 27858, 27831, 27828, 27837, 27843, 27777, - 27842, 27844, 27832, 27773, 27775, 27772, 27771, 27768, 27767, 27770, - 27769, 27776, 27774, 27778, 27833, 27779, 27859, 27829, 27830, 35762, - 28741, 28739, 28738, 28735, 28734, 28737, 28736, 28742, 28740, 28753, - 28752, 28751, 28747, 28746, 28750, 28749, 28745, 28748, 28743, 28744, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 17905, 17906, 17932, 17934, 17931, 17907, 17933, 17908, 17922, - 17921, 17897, 17895, 17896, 17915, 17920, 17919, 17904, 17903, 35762, - 17917, 17910, 17909, 17900, 17899, 17916, 17912, 17911, 17902, 17898, - 17901, 17918, 17924, 17923, 17894, 17892, 17893, 17926, 17930, 17928, - 17914, 17929, 17891, 17925, 17913, 17942, 17945, 17946, 17949, 17947, - 17943, 17948, 17944, 17939, 17937, 17938, 17935, 17889, 17888, 17950, - 17940, 17887, 17951, 17936, 17927, 17890, 17941, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 23597, - 23599, 23600, 23598, 23588, 23587, 23586, 35762, 23585, 35762, 23584, - 23583, 23576, 23575, 35762, 23570, 23578, 23577, 23566, 23564, 23565, - 23569, 23580, 23579, 23568, 23567, 23571, 23590, 23589, 23582, 35762, - 23581, 23593, 23596, 23574, 23592, 23595, 23594, 23591, 23573, 23572, - 23601, 35762, 35762, 35762, 35762, 35762, 35762, 17968, 17969, 17978, - 17979, 17976, 17977, 17997, 17970, 17998, 17971, 17987, 17986, 17957, - 17955, 17956, 17980, 17985, 17984, 17960, 17959, 17958, 17982, 17973, - 17972, 17963, 17961, 17966, 17962, 17981, 17975, 17974, 17965, 17964, - 17983, 17989, 17988, 17954, 17952, 17953, 17994, 17996, 17967, 17993, - 17995, 17990, 17991, 17992, 18001, 18002, 18007, 18008, 18005, 18006, - 18009, 18003, 18010, 18004, 18000, 17999, 35762, 35762, 35762, 35762, - 35762, 18017, 18019, 18016, 18015, 18012, 18011, 18014, 18013, 18020, - 18018, 35762, 35762, 35762, 35762, 35762, 35762, 13699, 13700, 13703, - 13706, 35762, 13656, 13657, 13670, 13671, 13668, 13669, 13651, 13653, - 35762, 35762, 13694, 13658, 35762, 35762, 13693, 13659, 13690, 13689, - 13686, 13685, 13674, 13684, 13683, 13688, 13687, 13676, 13665, 13664, - 13661, 13660, 13675, 13667, 13666, 13663, 13662, 13677, 35762, 13692, - 13691, 13682, 13681, 13696, 13698, 13697, 35762, 13673, 13672, 35762, - 13655, 13678, 13679, 13680, 13695, 35762, 7774, 13704, 13702, 13707, - 13716, 13717, 13714, 13715, 13710, 13711, 35762, 35762, 13719, 13708, - 35762, 35762, 13718, 13709, 13705, 35762, 35762, 13721, 35762, 35762, - 35762, 35762, 35762, 35762, 13720, 35762, 35762, 35762, 35762, 35762, - 13701, 13650, 13649, 13652, 13654, 13712, 13713, 35762, 35762, 7954, - 7955, 7953, 7952, 7951, 7950, 7949, 35762, 35762, 35762, 7960, 7957, - 7958, 7956, 7959, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 24495, 24496, 24509, 24510, 24507, 24508, 24491, 24492, 24493, - 24494, 24535, 24497, 24536, 24498, 24523, 24522, 24519, 24518, 24484, - 24483, 24517, 24516, 24521, 24520, 24486, 24485, 24504, 24503, 24500, - 24499, 24488, 24506, 24505, 24502, 24501, 24489, 24487, 24529, 24528, - 24515, 24514, 24527, 24526, 24534, 24531, 24530, 24525, 24524, 24533, - 24511, 24512, 24513, 24532, 24549, 24558, 24559, 24556, 24557, 24552, - 24553, 24554, 24555, 24560, 24550, 24561, 24551, 24544, 24539, 24538, - 24545, 24540, 24537, 24542, 24565, 24546, 24471, 24469, 24564, 24482, - 24562, 24478, 24480, 24477, 24476, 24473, 24472, 24475, 24474, 24481, - 24479, 24470, 24548, 35762, 24563, 24547, 24490, 24541, 24543, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 32336, - 32337, 32338, 32356, 32357, 32354, 32355, 32349, 32350, 32351, 32352, - 32382, 32339, 32383, 32340, 32374, 32373, 32370, 32369, 32358, 32368, - 32367, 32372, 32371, 32360, 32346, 32345, 32342, 32341, 32359, 32348, - 32347, 32344, 32343, 32361, 32376, 32375, 32366, 32365, 32379, 32381, - 32380, 32378, 32353, 32362, 32363, 32364, 32377, 32390, 32401, 32402, - 32399, 32400, 32395, 32396, 32397, 32398, 32403, 32393, 32391, 32404, - 32394, 32392, 32386, 32385, 32389, 32388, 32387, 32384, 32415, 32335, - 32416, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 32411, - 32413, 32410, 32409, 32406, 32405, 32408, 32407, 32414, 32412, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 27949, 27951, 27972, 27973, 27970, - 27971, 27965, 27966, 27967, 27968, 27998, 27952, 27999, 27953, 27990, - 27989, 27986, 27985, 27974, 27984, 27983, 27988, 27987, 27976, 27959, - 27958, 27962, 27961, 27975, 27960, 27955, 27964, 27963, 27977, 27992, - 27991, 27982, 27981, 27995, 27997, 27996, 27994, 27969, 27978, 27979, - 27980, 27993, 28027, 28034, 28035, 28032, 28033, 28030, 28031, 35762, - 35762, 28036, 28028, 28037, 28029, 28020, 28022, 28024, 28023, 28021, - 28019, 27947, 27946, 28018, 28017, 28000, 28001, 28002, 27948, 28015, - 28014, 28012, 28010, 28011, 28003, 28004, 28013, 28016, 28008, 28009, - 28007, 28006, 28005, 27954, 27956, 27957, 27950, 28025, 28026, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 22953, 22954, 22972, 22973, 22970, 22971, 22965, - 22966, 22967, 22968, 22999, 22955, 23000, 22956, 22992, 22991, 22988, - 22987, 22976, 22986, 22985, 22990, 22989, 22978, 22962, 22961, 22958, - 22957, 22977, 22964, 22963, 22960, 22959, 22979, 22994, 22993, 22984, - 22983, 22996, 22998, 22997, 22975, 22969, 22980, 22981, 22982, 22995, - 22974, 22928, 22937, 22938, 22935, 22936, 22931, 22932, 22933, 22934, - 22939, 22929, 22940, 22930, 22924, 22927, 22926, 22923, 22942, 22941, - 23001, 22925, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 22949, 22951, 22948, 22947, 22944, 22943, 22946, - 22945, 22952, 22950, 35762, 35762, 35762, 35762, 35762, 35762, 23481, - 23476, 23321, 23482, 23480, 23478, 23477, 23338, 23339, 23472, 23474, - 23473, 23483, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 30326, 30328, 30343, 30344, 30341, 30342, 30368, 30329, 30369, - 30330, 30358, 30357, 30354, 30353, 30345, 30352, 30351, 30356, 30355, - 30347, 30338, 30337, 30334, 30333, 30346, 30340, 30339, 30336, 30335, - 30348, 30360, 30359, 30350, 30349, 30365, 30367, 30332, 30364, 30366, - 30361, 30362, 30363, 30331, 30371, 30373, 30374, 30379, 30380, 30377, - 30378, 30381, 30375, 30382, 30376, 30372, 30370, 30327, 30325, 35762, - 35762, 35762, 35762, 35762, 35762, 30389, 30391, 30388, 30387, 30384, - 30383, 30386, 30385, 30392, 30390, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 191, 190, - 178, 181, 175, 167, 193, 192, 183, 195, 189, 184, 174, 196, 177, 197, - 180, 194, 164, 171, 170, 187, 166, 186, 182, 188, 165, 35762, 35762, 162, - 163, 161, 203, 204, 210, 211, 208, 209, 212, 207, 213, 205, 206, 200, - 35762, 35762, 35762, 35762, 222, 224, 221, 220, 217, 216, 219, 218, 225, - 223, 215, 214, 198, 199, 201, 202, 185, 173, 172, 169, 168, 179, 176, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 10719, 10720, 10733, 10734, 10731, - 10732, 10759, 10721, 10760, 10722, 10751, 10750, 10747, 10746, 10735, - 10745, 10744, 10749, 10748, 10737, 10728, 10727, 10724, 10723, 10736, - 10730, 10729, 10726, 10725, 10738, 10753, 10752, 10743, 10742, 10756, - 10758, 10718, 10755, 10757, 10739, 10740, 10741, 10754, 10717, 10763, - 10768, 10769, 10766, 10767, 10761, 10762, 10770, 10764, 10771, 10765, - 10774, 10776, 10775, 10773, 10772, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 33866, 33855, 33884, 33874, 33876, - 33877, 33883, 33873, 33859, 33868, 33856, 33886, 33882, 33861, 33875, - 33872, 33860, 33869, 33878, 33867, 33885, 33858, 33857, 33880, 33881, - 33864, 33863, 33862, 33865, 33870, 33871, 33879, 33898, 33887, 33916, - 33906, 33908, 33909, 33915, 33905, 33891, 33900, 33888, 33918, 33914, - 33893, 33907, 33904, 33892, 33901, 33910, 33899, 33917, 33890, 33889, - 33912, 33913, 33896, 33895, 33894, 33897, 33902, 33903, 33911, 33925, - 33927, 33924, 33923, 33920, 33919, 33922, 33921, 33928, 33926, 33937, - 33936, 33935, 33931, 33930, 33934, 33933, 33929, 33932, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 33938, 10633, 10634, 10641, 10642, 10639, 10640, 10668, 35762, 35762, - 10669, 35762, 35762, 10659, 10658, 10657, 10656, 10645, 10655, 10654, - 10663, 35762, 10647, 10629, 35762, 10636, 10635, 10646, 10630, 10628, - 10638, 10637, 10648, 10661, 10660, 10653, 10652, 10664, 10632, 10631, - 10665, 10644, 10666, 10649, 10650, 10651, 10662, 10643, 10667, 10674, - 10678, 10679, 10676, 10677, 10680, 35762, 10675, 10681, 35762, 35762, - 10673, 10671, 10670, 10682, 10687, 10684, 10688, 10683, 10672, 10617, - 10685, 10686, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 10624, 10626, 10623, 10622, 10619, 10618, 10621, 10620, 10627, - 10625, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 24192, 24193, 24206, 24207, 24204, 24205, 24187, 24188, 35762, - 35762, 24232, 24194, 24233, 24195, 24226, 24225, 24222, 24221, 24210, - 24220, 24219, 24224, 24223, 24212, 24201, 24200, 24197, 24196, 24211, - 24203, 24202, 24199, 24198, 24213, 24228, 24227, 24218, 24217, 24230, - 24231, 24191, 24209, 24189, 24214, 24215, 24216, 24229, 24208, 24190, - 24242, 24247, 24248, 24245, 24246, 24240, 24241, 35762, 35762, 24249, - 24243, 24250, 24244, 24236, 24238, 24237, 24235, 24234, 24251, 24239, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35543, 35564, 35561, - 35560, 35563, 35559, 35558, 35556, 35557, 35562, 35555, 35511, 35510, - 35530, 35529, 35512, 35528, 35527, 35537, 35514, 35522, 35521, 35504, - 35503, 35513, 35524, 35523, 35508, 35507, 35515, 35532, 35531, 35526, - 35525, 35539, 35520, 35519, 35506, 35505, 35533, 35534, 35535, 35542, - 35540, 35538, 35541, 35516, 35517, 35518, 35536, 35509, 35502, 35552, - 35549, 35550, 35551, 35548, 35553, 35499, 35498, 35496, 35495, 35497, - 35501, 35494, 35547, 35545, 35544, 35546, 35500, 35493, 35554, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 29142, 29163, 29161, - 29160, 29162, 29158, 29159, 29156, 29157, 29155, 29154, 29164, 29109, - 29108, 29128, 29127, 29110, 29126, 29125, 29130, 29129, 29112, 29120, - 29119, 29103, 29102, 29111, 29122, 29121, 29106, 29104, 29113, 29132, - 29131, 29124, 29123, 29138, 29118, 29117, 29105, 29133, 29134, 29135, - 29141, 29139, 29137, 29140, 29114, 29115, 29116, 29136, 29107, 29147, - 29149, 29085, 29084, 29082, 29083, 29093, 29094, 29089, 29092, 29088, - 29091, 29096, 29097, 29095, 29087, 29086, 29090, 29150, 29148, 29098, - 29151, 29146, 29145, 29144, 29143, 29101, 29100, 29099, 29152, 29153, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 5316, 5317, 5314, 5315, 5312, 5313, 5322, 5323, - 5320, 5321, 5318, 5319, 5485, 5486, 5487, 5484, 26453, 26451, 26460, - 26461, 26457, 26465, 26464, 26446, 26459, 26458, 26450, 26463, 26456, - 26449, 26455, 26454, 26447, 26452, 26462, 26441, 26448, 26466, 26467, - 26442, 26468, 26444, 26445, 26443, 26437, 26434, 26438, 26436, 26432, - 26435, 26439, 26433, 26440, 26474, 26473, 26484, 26475, 26476, 26485, - 26481, 26480, 26482, 26483, 26477, 26431, 26478, 26479, 26470, 26469, - 26429, 26471, 26472, 26430, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 10360, 10361, 10450, 10451, 10452, 10453, 10460, 10459, 10461, - 10469, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 3641, 3642, 3655, 3656, - 3653, 3654, 3637, 3638, 3639, 35762, 3681, 3643, 3682, 3644, 3673, 3672, - 3669, 3668, 3657, 3667, 3666, 3671, 3670, 3659, 3650, 3649, 3646, 3645, - 3658, 3652, 3651, 3648, 3647, 3660, 3675, 3674, 3665, 3664, 3678, 3680, - 3679, 3677, 3640, 3661, 3662, 3663, 3676, 3709, 3714, 3715, 3712, 3713, - 3706, 3707, 3708, 35762, 3716, 3710, 3717, 3711, 3703, 3702, 3705, 3704, - 3701, 3719, 3718, 3730, 3731, 3732, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 3726, 3728, 3725, 3724, 3721, 3720, - 3723, 3722, 3729, 3727, 3700, 3698, 3695, 3686, 3688, 3693, 3691, 3683, - 3689, 3699, 3697, 3696, 3685, 3687, 3694, 3692, 3684, 3690, 3733, 35762, - 35762, 35762, 21057, 21058, 21003, 21002, 21012, 20997, 21001, 21000, - 21014, 20998, 20994, 20993, 20996, 20999, 21005, 21004, 21011, 21016, - 20992, 20991, 20995, 21018, 21008, 21009, 21010, 21019, 21017, 21015, - 21006, 21007, 21013, 21020, 35762, 35762, 21035, 21034, 21043, 21029, - 21033, 21032, 21045, 21030, 21026, 21025, 21028, 21031, 21037, 21036, - 21042, 21047, 21024, 21023, 21027, 21049, 21040, 21041, 35762, 21050, - 21048, 21046, 21038, 21039, 21044, 21051, 21052, 21054, 21056, 21053, - 21055, 21022, 21021, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 21069, 21070, 21079, 21080, - 21077, 21078, 21106, 35762, 21071, 21107, 35762, 21072, 21085, 21084, - 21098, 21097, 21086, 21096, 21095, 21063, 21062, 21088, 21065, 21064, - 21074, 21073, 21087, 21068, 21066, 21076, 21075, 21089, 21100, 21099, - 21094, 21093, 21102, 21105, 21103, 21082, 21104, 21090, 21091, 21092, - 21101, 21081, 21083, 21061, 21067, 21116, 21121, 21122, 21119, 21120, - 21115, 35762, 35762, 35762, 21123, 35762, 21117, 21124, 35762, 21118, - 21114, 21113, 21112, 21110, 21111, 21125, 21109, 21108, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 21132, 21134, 21131, 21130, - 21127, 21126, 21129, 21128, 21135, 21133, 35762, 35762, 35762, 35762, - 35762, 35762, 14372, 14373, 14388, 14389, 14384, 14385, 35762, 14405, - 14374, 35762, 14404, 14375, 14411, 14410, 14393, 14392, 14407, 14401, - 14400, 14383, 14382, 14391, 14397, 14396, 14379, 14378, 14387, 14395, - 14394, 14381, 14380, 14390, 14399, 14398, 14377, 14376, 14386, 14403, - 14402, 14406, 14408, 14409, 14414, 14419, 14420, 14417, 14418, 35762, - 14422, 14415, 35762, 14421, 14416, 14413, 14412, 14423, 14434, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 14430, 14432, 14429, 14428, - 14425, 14424, 14427, 14426, 14433, 14431, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 20755, 20753, 20747, 20758, - 20750, 20757, 20761, 20752, 20749, 20751, 20754, 20748, 20763, 20759, - 20756, 20762, 20760, 20764, 20770, 20767, 20769, 20766, 20768, 20765, - 20746, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 17048, 17050, - 17046, 17049, 16988, 16989, 17008, 17009, 17006, 17007, 17002, 17003, - 17004, 17005, 17033, 16990, 17034, 35762, 17024, 17023, 17022, 17021, - 17010, 17020, 17019, 16993, 16992, 17012, 16999, 16998, 16995, 16994, - 17011, 17001, 17000, 16997, 16996, 17013, 17026, 17025, 17018, 17017, - 17029, 17032, 17030, 17028, 17031, 17014, 17015, 17016, 17027, 16991, - 17052, 17051, 17059, 17060, 17057, 17058, 17054, 35762, 35762, 35762, - 17055, 17053, 17056, 17047, 17073, 17062, 17061, 17041, 17044, 17040, - 17042, 17038, 17037, 17045, 17036, 17039, 17043, 17035, 17069, 17071, - 17068, 17067, 17064, 17063, 17066, 17065, 17072, 17070, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 20466, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30449, 30445, 30439, 30448, 30441, 30450, 30454, 30456, 30451, 30446, - 30447, 30452, 30440, 30457, 30455, 30442, 30453, 30443, 30444, 30458, - 30459, 30523, 30506, 30504, 30513, 30512, 30508, 30514, 30510, 30509, - 30516, 30517, 30518, 30515, 30507, 30520, 30438, 30425, 30496, 30503, - 30804, 30805, 30423, 30398, 30524, 30803, 30460, 30525, 30511, 30519, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 30501, 8517, 8525, 8523, 8519, 8524, 8520, 8518, - 8521, 8522, 8586, 8526, 8535, 8534, 8528, 8527, 8539, 8529, 8530, 8538, - 8532, 8533, 8540, 8541, 8546, 8543, 8542, 8544, 8545, 8548, 8550, 8552, - 8551, 8553, 8559, 8556, 8557, 8561, 8554, 8555, 8560, 8558, 8563, 8562, - 8564, 8566, 8567, 8571, 8570, 8568, 8569, 8572, 8585, 8573, 8574, 8575, - 8584, 8576, 8580, 8581, 8579, 8577, 8578, 8583, 8582, 8587, 8588, 8599, - 8590, 8594, 8595, 8596, 8597, 8598, 8600, 8603, 8602, 8601, 8604, 8606, - 8607, 8608, 8609, 8610, 8611, 8612, 8613, 8614, 8615, 8616, 8617, 8618, - 8619, 8620, 8621, 8622, 8623, 8638, 8624, 8625, 8635, 8629, 8626, 8627, - 8628, 8636, 8633, 8637, 8634, 8630, 8632, 8645, 8641, 8642, 8643, 8646, - 8657, 8647, 8650, 8651, 8653, 8654, 8655, 8658, 8661, 8659, 8660, 8662, - 8663, 8665, 8666, 8695, 8702, 8696, 8697, 8698, 8699, 8700, 8701, 8703, - 8705, 8704, 8706, 8709, 8710, 8713, 8707, 8708, 8714, 8759, 8758, 8760, - 8715, 8718, 8719, 8720, 8716, 8717, 8721, 8724, 8722, 8725, 8727, 8736, - 8737, 8738, 8739, 8757, 8740, 8741, 8754, 8755, 8753, 8742, 8743, 8744, - 8745, 8746, 8747, 8748, 8751, 8752, 8761, 8851, 8762, 8763, 8765, 8764, - 8771, 8766, 8768, 8770, 8774, 8773, 8775, 8776, 8783, 8777, 8778, 8782, - 8786, 8787, 8784, 8785, 8793, 8790, 8794, 8795, 8796, 8797, 8798, 8800, - 8801, 8803, 8802, 8805, 8804, 8807, 8806, 8808, 8809, 8811, 8812, 8816, - 8818, 8822, 8823, 8836, 8828, 8829, 8824, 8825, 8826, 8830, 8834, 8831, - 8832, 8833, 8837, 8838, 8840, 8841, 8842, 8843, 8844, 8845, 8855, 8846, - 8847, 8850, 8849, 8848, 8852, 8853, 8854, 8856, 8857, 8860, 8861, 8862, - 8863, 8864, 8866, 8865, 8882, 8873, 8874, 8867, 8868, 8870, 8871, 8869, - 8872, 8881, 8875, 8880, 8878, 8877, 8879, 8884, 8903, 8885, 8886, 8887, - 8890, 8889, 8891, 8892, 8894, 8895, 8896, 8904, 8897, 8898, 8899, 8902, - 8901, 8900, 8905, 8906, 8908, 8909, 8910, 8911, 8913, 8917, 8914, 8918, - 8916, 8915, 8919, 8920, 8921, 8922, 8926, 8925, 8923, 8924, 8927, 8928, - 8930, 8949, 8951, 8931, 8933, 8932, 8934, 8935, 8938, 8939, 8937, 8936, - 8940, 8941, 8942, 8943, 8946, 8944, 8945, 8947, 8948, 8952, 8953, 8950, - 8954, 8955, 8956, 8957, 8959, 8962, 8961, 8963, 8964, 8966, 8967, 8968, - 8972, 8971, 8969, 8970, 8973, 8977, 8976, 8975, 8978, 8979, 8981, 8982, - 8986, 8983, 8984, 8989, 8987, 8990, 8991, 8993, 8992, 8994, 8995, 9017, - 9016, 9019, 9020, 9001, 9002, 8999, 9000, 8998, 8996, 9004, 9003, 9005, - 9007, 9013, 9014, 9011, 9012, 9021, 9022, 9023, 9040, 9026, 9027, 9028, - 9024, 9025, 9029, 9030, 9031, 9032, 9033, 9034, 9035, 9036, 9037, 9038, - 9063, 9041, 9044, 9042, 9043, 9049, 9050, 9047, 9048, 9045, 9046, 9051, - 9052, 9060, 9053, 9054, 9061, 9058, 9059, 9062, 9055, 9056, 9057, 9064, - 9065, 9066, 9067, 9068, 9069, 9070, 9072, 9073, 9071, 9074, 9075, 9116, - 9117, 9078, 9079, 9076, 9077, 9082, 9083, 9081, 9087, 9084, 9086, 9085, - 9091, 9092, 9090, 9088, 9089, 9093, 9094, 9095, 9096, 9097, 9098, 9099, - 9118, 9104, 9100, 9101, 9102, 9103, 9105, 9107, 9106, 9108, 9109, 9111, - 9110, 9112, 9114, 9113, 9119, 9120, 9123, 9124, 9121, 9122, 9198, 9193, - 9194, 9195, 9196, 9197, 9199, 9202, 9200, 9201, 9203, 9245, 9204, 9234, - 9235, 9211, 9213, 9230, 9214, 9236, 9218, 9216, 9217, 9219, 9220, 9221, - 9222, 9231, 9232, 9225, 9226, 9228, 9237, 9205, 9206, 9210, 9208, 9246, - 9238, 9240, 9239, 9241, 9247, 9248, 9242, 9243, 9244, 9249, 9250, 9251, - 9254, 9255, 9256, 9252, 9253, 9257, 9258, 9260, 9262, 9263, 9281, 9279, - 9280, 9283, 9282, 9264, 9271, 9269, 9270, 9265, 9266, 9272, 9273, 9274, - 9275, 9276, 9278, 9284, 9293, 9285, 9287, 9288, 9286, 9289, 9291, 9290, - 9292, 9295, 9297, 9296, 9298, 9299, 9332, 9334, 9300, 9302, 9301, 9304, - 9307, 9305, 9306, 9310, 9314, 9317, 9316, 9319, 9322, 9320, 9321, 9325, - 9327, 9333, 9335, 9336, 9340, 9347, 9345, 9343, 9344, 9346, 9349, 9348, - 9341, 9342, 9350, 9353, 9359, 9354, 9357, 9352, 9351, 9360, 9358, 9355, - 9356, 9361, 9362, 9363, 9364, 9365, 9366, 9367, 9369, 9372, 9373, 9376, - 9377, 9378, 9374, 9375, 9370, 9371, 9379, 9380, 9381, 9382, 9383, 9384, - 9386, 9387, 9388, 9389, 9390, 9394, 9391, 9415, 9408, 9409, 9414, 9397, - 9396, 9410, 9413, 9411, 9400, 9399, 9402, 9404, 9405, 9406, 9407, 9403, - 9416, 9392, 9417, 9418, 9419, 9420, 9421, 9422, 9430, 9428, 9426, 9429, - 9425, 9427, 9423, 9424, 9431, 9433, 9434, 9435, 9442, 9437, 9438, 9446, - 9447, 9443, 9445, 9444, 9448, 9450, 9449, 9451, 9462, 9453, 9452, 9461, - 9459, 9454, 9455, 9456, 9458, 9457, 9460, 9466, 9463, 9465, 9464, 9467, - 9468, 9469, 9470, 9473, 9474, 9476, 9477, 9478, 9479, 9481, 9480, 9482, - 9489, 9483, 9484, 9490, 9485, 9486, 9487, 9488, 9491, 9495, 9492, 9493, - 9494, 9496, 9497, 9498, 9499, 9505, 9503, 9501, 9502, 9500, 9504, 9506, - 9508, 9525, 9526, 9509, 9514, 9516, 9510, 9513, 9511, 9512, 9517, 9523, - 9524, 9518, 9521, 9522, 9527, 9533, 9532, 9528, 9530, 9529, 9614, 9615, - 9534, 9535, 9540, 9541, 9538, 9539, 9542, 9536, 9537, 9543, 9546, 9547, - 9549, 9550, 9551, 9555, 9552, 9553, 9554, 9544, 9545, 9556, 9558, 9557, - 9559, 9561, 9562, 9563, 9569, 9568, 9564, 9565, 9566, 9601, 9570, 9571, - 9572, 9573, 9574, 9593, 9576, 9577, 9579, 9578, 9580, 9581, 9597, 9582, - 9584, 9583, 9596, 9586, 9594, 9598, 9589, 9588, 9595, 9590, 9592, 9591, - 9599, 9600, 9602, 9606, 9604, 9605, 9603, 9609, 9608, 9607, 9613, 9610, - 9611, 9612, 9616, 9618, 9617, 9621, 9619, 9636, 9622, 9625, 9627, 9623, - 9624, 9628, 9626, 9629, 9631, 9633, 9634, 9635, 9039, 8536, 8547, 8565, - 8631, 8640, 8656, 8664, 8756, 8749, 8767, 8769, 8859, 8883, 8929, 8958, - 8960, 8974, 8980, 9015, 8988, 9018, 8997, 9006, 9010, 9080, 9209, 9212, - 9227, 9259, 9277, 9294, 9303, 9331, 9329, 9308, 9339, 9368, 9385, 9395, - 9515, 9548, 9531, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 8501, 8496, 8433, 8419, 8480, 8476, 8408, - 8453, 8500, 8441, 8425, 8486, 8475, 8407, 8452, 8439, 8423, 8482, 8472, - 8402, 8449, 8462, 8506, 8498, 8435, 8421, 8484, 8474, 8404, 8451, 8463, - 8507, 8499, 8436, 8422, 8508, 8490, 8491, 8437, 8413, 8479, 8471, 8401, - 8448, 8466, 8509, 8492, 8493, 8438, 8414, 8477, 8478, 8461, 8504, 8487, - 8488, 8428, 8418, 8494, 8495, 8429, 8432, 8430, 8431, 8485, 8470, 8468, - 8469, 8405, 8406, 8444, 8446, 8447, 8445, 8502, 8497, 8434, 8420, 8481, - 8460, 8503, 8489, 8426, 8427, 8416, 8417, 8443, 8442, 8456, 8505, 8465, - 8511, 8415, 8464, 8510, 8457, 8458, 8454, 8455, 8459, 8467, 8409, 8412, - 8411, 8410, 8440, 8424, 8483, 8473, 8403, 8450, 35762, 8516, 8515, 8514, - 8512, 8513, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 8537, 8531, 8549, 8589, 8591, 8592, 8593, 8605, - 8644, 8639, 8649, 8648, 8652, 8669, 8667, 8668, 8670, 8671, 8689, 8675, - 8672, 8673, 8674, 8691, 8692, 8690, 8679, 8678, 8676, 8677, 8682, 8680, - 8681, 8683, 8684, 8685, 8686, 8693, 8694, 8688, 8687, 8712, 8711, 8723, - 8726, 8731, 8735, 8728, 8732, 8733, 8729, 8730, 8734, 8750, 8772, 8779, - 8780, 8781, 8788, 8789, 8792, 8791, 8799, 8810, 8813, 8814, 8815, 8817, - 8819, 8820, 8821, 8827, 8835, 8839, 8858, 8876, 8888, 8893, 8907, 8912, - 8965, 8985, 9008, 9009, 9115, 9135, 9125, 9126, 9134, 9130, 9131, 9132, - 9129, 9128, 9127, 9133, 9137, 9136, 9143, 9144, 9138, 9139, 9140, 9145, - 9141, 9142, 9146, 9147, 9148, 9149, 9150, 9151, 9158, 9152, 9157, 9156, - 9154, 9153, 9155, 9159, 9160, 9165, 9166, 9161, 9162, 9163, 9164, 9192, - 9188, 9167, 9175, 9176, 9174, 9173, 9177, 9168, 9169, 9171, 9172, 9170, - 9189, 9178, 9185, 9187, 9182, 9183, 9186, 9181, 9184, 9180, 9179, 9190, - 9191, 9207, 9233, 9215, 9223, 9224, 9229, 9261, 9268, 9267, 9328, 9309, - 9311, 9330, 9312, 9313, 9315, 9318, 9323, 9324, 9326, 9393, 9412, 9398, - 9401, 9432, 9436, 9439, 9440, 9441, 9472, 9471, 9475, 9507, 9519, 9520, - 9560, 9567, 9575, 9587, 9585, 9620, 9630, 9632, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 9709, - 9710, 9711, 9712, 9713, 9714, 9715, 9716, 9719, 9720, 9717, 9718, 9721, - 9722, 9723, 9724, 9725, 9726, 9727, 9728, 9729, 9730, 9731, 9732, 9733, - 9734, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9742, 9743, 9744, 9745, - 9746, 9747, 9748, 9749, 9750, 9751, 9752, 9753, 9754, 9755, 9775, 9776, - 9777, 9778, 9779, 9780, 9781, 9782, 9783, 9758, 9759, 9760, 9761, 9762, - 9756, 9757, 9763, 9764, 9765, 9784, 9785, 9786, 9787, 9788, 9789, 9790, - 9791, 9792, 9793, 9766, 9767, 9768, 9769, 9770, 9771, 9772, 9773, 9774, - 9794, 9795, 9796, 9797, 9798, 9799, 9800, 9801, 9802, 9803, 9804, 9805, - 9806, 9807, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 11228, 11229, 11230, 11231, 11226, - 11227, 11223, 11224, 11225, 11232, 11233, 11234, 11239, 11240, 11241, - 11242, 11235, 11236, 11243, 11244, 11237, 11238, 11245, 11246, 11273, - 11274, 11275, 11276, 11277, 11278, 11279, 11280, 11281, 11282, 11263, - 11264, 11261, 11262, 11265, 11266, 11267, 11268, 11269, 11270, 11271, - 11247, 11248, 11255, 11249, 11250, 11251, 11252, 11256, 11253, 11254, - 11257, 11258, 11259, 11260, 11283, 11284, 11285, 11286, 11287, 11288, - 11289, 11290, 11291, 11292, 11293, 11294, 11295, 11296, 11297, 11298, - 11299, 11300, 11301, 11302, 11272, 11342, 11343, 11344, 11345, 11340, - 11341, 11346, 11347, 11348, 11349, 11354, 11350, 11351, 11352, 11353, - 11355, 11356, 11357, 11358, 11359, 11360, 11361, 11362, 11363, 11364, - 11365, 11366, 11367, 11368, 11369, 11370, 11371, 11372, 11373, 11374, - 11375, 11376, 11377, 11380, 11381, 11382, 11383, 11384, 11385, 11386, - 11378, 11379, 11387, 11460, 11461, 11462, 11463, 11464, 11465, 11466, - 11467, 11468, 11469, 11451, 11452, 11453, 11454, 11455, 11456, 11457, - 11449, 11450, 11458, 11459, 11392, 11388, 11389, 11393, 11394, 11390, - 11391, 11395, 11396, 11397, 11398, 11399, 11404, 11405, 11406, 11407, - 11408, 11409, 11400, 11401, 11410, 11402, 11403, 11411, 11412, 11413, - 11414, 11415, 11416, 11417, 11418, 11419, 11420, 11421, 11426, 11422, - 11423, 11427, 11424, 11425, 11428, 11429, 11430, 11431, 11432, 11442, - 11443, 11444, 11445, 11446, 11447, 11448, 11433, 11434, 11435, 11436, - 11437, 11438, 11439, 11440, 11441, 11474, 11475, 11476, 11477, 11478, - 11479, 11480, 11470, 11471, 11472, 11473, 11506, 11507, 11508, 11509, - 11510, 11511, 11502, 11503, 11504, 11505, 11512, 11513, 11481, 11482, - 11485, 11486, 11487, 11488, 11489, 11490, 11491, 11483, 11484, 11492, - 11495, 11496, 11497, 11498, 11493, 11494, 11499, 11500, 11501, 11517, - 11518, 11519, 11520, 11521, 11522, 11523, 11524, 11525, 11526, 11529, - 11530, 11531, 11527, 11528, 11532, 11533, 11534, 11535, 11536, 11537, - 11573, 11571, 11572, 11574, 11575, 11576, 11577, 11578, 11579, 11580, - 11581, 11544, 11538, 11539, 11545, 11546, 11547, 11548, 11549, 11540, - 11541, 11542, 11543, 11550, 11557, 11558, 11559, 11560, 11561, 11551, - 11552, 11553, 11554, 11555, 11556, 11562, 11563, 11568, 11564, 11565, - 11566, 11567, 11569, 11570, 11588, 11589, 11590, 11591, 11592, 11586, - 11587, 11583, 11584, 11585, 11593, 11594, 11597, 11595, 11596, 11598, - 11599, 11600, 11601, 11602, 11603, 11604, 11605, 11630, 11631, 11634, - 11635, 11636, 11637, 11638, 11632, 11633, 11639, 11640, 11641, 11610, - 11611, 11612, 11613, 11614, 11615, 11606, 11607, 11608, 11609, 11616, - 11617, 11622, 11623, 11624, 11618, 11619, 11625, 11620, 11621, 11626, - 11627, 11628, 11629, 11642, 11643, 11644, 11645, 11646, 11649, 11650, - 11651, 11652, 11653, 11647, 11648, 11654, 11655, 11663, 11664, 11665, - 11666, 11659, 11660, 11667, 11668, 11669, 11661, 11662, 11670, 11671, - 11672, 11673, 11674, 11675, 11676, 11677, 12318, 12319, 12320, 12321, - 12322, 12323, 12324, 12325, 11689, 11685, 11686, 11690, 11691, 11692, - 11687, 11688, 11693, 11694, 11696, 11697, 11698, 11701, 11699, 11700, - 11702, 11703, 11704, 11705, 11706, 11707, 11717, 11718, 11725, 11708, - 11709, 11710, 11711, 11712, 11713, 11714, 11715, 11716, 11726, 11727, - 11719, 11720, 11721, 11722, 11723, 11724, 11728, 11729, 11736, 11737, - 11730, 11731, 11738, 11732, 11733, 11739, 11740, 11741, 11734, 11735, - 11742, 11748, 11746, 11747, 11749, 11743, 11744, 11745, 11750, 11751, - 11752, 11753, 11754, 11755, 11756, 11757, 11758, 11759, 11760, 11761, - 11818, 11819, 11820, 11821, 11822, 11823, 11824, 11825, 11826, 11781, - 11782, 11783, 11784, 11785, 11786, 11787, 11788, 11778, 11779, 11780, - 11789, 11806, 11807, 11808, 11809, 11810, 11804, 11805, 11811, 11812, - 11813, 11814, 11798, 11799, 11800, 11790, 11791, 11792, 11793, 11794, - 11795, 11801, 11796, 11797, 11802, 11803, 11815, 11816, 11817, 11829, - 11830, 11831, 11832, 11827, 11828, 11833, 11834, 11835, 11836, 11839, - 11840, 11841, 11842, 11843, 11844, 11845, 11837, 11838, 11846, 11847, - 11848, 11866, 11867, 11868, 11869, 11870, 11871, 11872, 11873, 11874, - 11849, 11850, 11851, 11852, 11855, 11856, 11857, 11858, 11859, 11860, - 11853, 11854, 11861, 11864, 11865, 11862, 11863, 11882, 11883, 11886, - 11887, 11888, 11884, 11885, 11875, 11876, 11877, 11878, 11879, 11880, - 11881, 11889, 11890, 11891, 11892, 11893, 11894, 11895, 11898, 11899, - 11900, 11901, 11902, 11903, 11904, 11905, 11896, 11897, 11906, 11907, - 11914, 11915, 11916, 11908, 11909, 11910, 11911, 11917, 11918, 11919, - 11912, 11913, 11925, 11926, 11929, 11930, 11927, 11928, 11931, 11932, - 11920, 11921, 11922, 11923, 11924, 11933, 11934, 11935, 11940, 11941, - 11942, 11943, 11944, 11945, 11946, 11947, 11948, 11949, 11936, 11937, - 11938, 11939, 11951, 11952, 11955, 11953, 11954, 11956, 11957, 11958, - 11959, 11960, 11961, 11962, 11963, 12326, 12327, 12328, 12329, 12330, - 12331, 12332, 11969, 11967, 11968, 11964, 11965, 11966, 11970, 11971, - 11972, 11973, 11974, 11975, 11976, 11977, 11980, 11981, 11982, 11983, - 11984, 11978, 11979, 11985, 11986, 11987, 11988, 11989, 11990, 11991, - 11992, 11993, 11994, 11995, 11996, 11997, 12002, 11998, 11999, 12003, - 12004, 12005, 12000, 12001, 12006, 12007, 12008, 12014, 12015, 12016, - 12017, 12009, 12010, 12011, 12018, 12019, 12012, 12013, 12020, 12021, - 12025, 12026, 12027, 12028, 12029, 12030, 12022, 12023, 12024, 12031, - 12032, 12033, 12036, 12037, 12038, 12039, 12040, 12034, 12035, 12041, - 12042, 12043, 12044, 12045, 12046, 12047, 12048, 12049, 12050, 12051, - 12060, 12061, 12052, 12053, 12062, 12063, 12064, 12054, 12055, 12056, - 12057, 12058, 12059, 12069, 12065, 12066, 12070, 12071, 12072, 12073, - 12067, 12068, 12074, 12075, 12076, 12086, 12087, 12088, 12089, 12090, - 12091, 12092, 12093, 12094, 12095, 12081, 12082, 12077, 12078, 12079, - 12080, 12083, 12084, 12085, 12100, 12101, 12102, 12103, 12104, 12097, - 12098, 12099, 12105, 12106, 12107, 12134, 12135, 12136, 12137, 12138, - 12139, 12140, 12141, 12142, 12143, 12112, 12113, 12114, 12108, 12109, - 12115, 12116, 12117, 12118, 12119, 12110, 12111, 12122, 12123, 12120, - 12121, 12124, 12125, 12126, 12127, 12128, 12129, 12130, 12131, 12132, - 12133, 12147, 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155, - 12156, 12157, 12158, 12159, 12160, 12161, 12162, 12144, 12145, 12146, - 12163, 12164, 12173, 12165, 12166, 12167, 12168, 12170, 12171, 12172, - 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, - 12184, 12185, 12186, 12187, 12188, 12189, 12190, 12191, 12192, 12193, - 12200, 12201, 12194, 12195, 12202, 12203, 12204, 12205, 12196, 12197, - 12198, 12199, 12206, 12207, 12208, 12209, 12214, 12210, 12211, 12215, - 12216, 12217, 12212, 12213, 12218, 12219, 12220, 12221, 12227, 12228, - 12223, 12224, 12229, 12230, 12231, 12232, 12233, 12225, 12226, 12234, - 12235, 12242, 12243, 12244, 12236, 12237, 12245, 12246, 12238, 12239, - 12240, 12241, 12247, 12250, 12251, 12252, 12253, 12248, 12249, 12254, - 12263, 12264, 12265, 12256, 12257, 12258, 12266, 12259, 12260, 12267, - 12261, 12262, 12268, 12269, 12270, 12271, 12272, 12273, 12274, 12275, - 12276, 12289, 12277, 12278, 12279, 12280, 12281, 12282, 12283, 12284, - 12285, 12286, 12287, 12288, 12290, 12291, 12292, 12293, 12313, 12314, - 12315, 12316, 12317, 12294, 12295, 12296, 12297, 12298, 12299, 12300, - 12301, 12302, 12303, 12304, 12305, 12306, 12307, 12308, 12309, 12310, - 12311, 12312, 11306, 11307, 11308, 11309, 11310, 11311, 11303, 11304, - 11305, 11312, 11313, 11317, 11318, 11319, 11320, 11321, 11322, 11323, - 11324, 11325, 11326, 11327, 11328, 11329, 11330, 11331, 11332, 11333, - 11334, 11335, 11336, 11314, 11315, 11316, 12169, 12222, 11658, 11683, - 11680, 11682, 11679, 11950, 11337, 11514, 11684, 11681, 11678, 11339, - 11516, 11338, 11515, 11777, 11582, 11656, 11695, 11657, 12096, 12255, - 11773, 11764, 11768, 11775, 11771, 11765, 11770, 11767, 11774, 11763, - 11769, 11776, 11772, 11766, 11762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 435, 436, 437, 438, 439, 440, 441, - 442, 443, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 365, - 366, 367, 368, 369, 370, 363, 364, 371, 372, 373, 415, 416, 417, 418, - 419, 420, 421, 422, 423, 413, 414, 381, 377, 378, 382, 383, 384, 379, - 380, 374, 375, 376, 385, 386, 387, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 392, 393, 394, 395, 396, 397, 388, 389, 390, 391, 398, - 399, 400, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, - 466, 467, 468, 469, 470, 471, 472, 473, 405, 406, 407, 408, 409, 410, - 411, 401, 402, 403, 404, 412, 485, 486, 487, 488, 489, 490, 491, 474, - 475, 476, 477, 482, 483, 484, 492, 478, 479, 480, 481, 493, 494, 495, - 496, 497, 500, 501, 502, 503, 498, 499, 504, 505, 506, 507, 510, 511, - 512, 513, 514, 508, 509, 515, 516, 517, 518, 521, 522, 523, 524, 525, - 519, 520, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, - 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, - 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, - 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, - 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 597, 598, 590, 591, - 592, 599, 600, 601, 602, 593, 594, 603, 595, 596, 608, 609, 610, 611, - 612, 604, 605, 606, 607, 613, 614, 615, 641, 642, 643, 644, 645, 646, - 647, 639, 640, 648, 649, 661, 662, 663, 664, 665, 666, 667, 668, 669, - 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, - 684, 685, 686, 687, 688, 689, 690, 652, 653, 654, 655, 656, 657, 658, - 650, 651, 659, 660, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, - 630, 631, 632, 633, 634, 635, 636, 637, 638, 628, 629, 620, 621, 622, - 623, 616, 617, 624, 625, 626, 627, 618, 619, 703, 704, 705, 706, 707, - 708, 709, 710, 711, 701, 702, 795, 796, 797, 798, 799, 800, 801, 802, - 803, 804, 714, 715, 716, 717, 718, 719, 720, 721, 722, 712, 713, 741, - 742, 738, 739, 740, 743, 744, 745, 734, 735, 736, 737, 746, 747, 748, - 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 725, 726, 727, 728, - 729, 730, 731, 732, 733, 723, 724, 753, 754, 755, 756, 749, 750, 757, - 758, 759, 751, 752, 760, 786, 784, 785, 787, 788, 789, 790, 791, 792, - 793, 794, 767, 763, 764, 768, 761, 762, 769, 770, 765, 766, 771, 772, - 773, 775, 776, 777, 774, 778, 779, 780, 781, 782, 783, 846, 847, 848, - 849, 850, 851, 852, 853, 854, 855, 815, 816, 817, 818, 819, 820, 821, - 822, 823, 824, 825, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, - 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, - 880, 881, 882, 883, 884, 885, 826, 827, 830, 831, 832, 833, 834, 835, - 828, 829, 836, 837, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, - 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, - 910, 911, 912, 913, 914, 915, 838, 839, 840, 841, 842, 843, 844, 845, - 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, - 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, - 945, 916, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 2827, 2794, 2852, 2853, 2823, - 2860, 2871, 2842, 2859, 2854, 2855, 2807, 2872, 2829, 2808, 2813, 2820, - 2866, 2838, 2797, 2832, 2867, 2828, 2804, 2805, 2836, 2810, 2851, 2793, - 2850, 2819, 2843, 2874, 2792, 2837, 2868, 2822, 2821, 2817, 2795, 2849, - 2826, 2856, 2809, 2846, 2857, 2873, 2802, 2863, 2870, 2811, 2798, 2825, - 2800, 2818, 2861, 2803, 2876, 2878, 2799, 2864, 2814, 2858, 2835, 2862, - 2840, 2847, 2831, 2875, 2830, 2812, 2844, 2815, 2841, 2869, 2865, 2848, - 2833, 2806, 2839, 2796, 2834, 2877, 2801, 2845, 2824, 2816, 2910, 2926, - 2924, 2923, 2889, 2895, 2884, 2930, 2894, 2886, 2916, 2929, 2888, 2914, - 2915, 2879, 2919, 2927, 2921, 2922, 2901, 2898, 2913, 2882, 2880, 2881, - 2887, 2917, 2934, 2908, 2906, 2931, 2920, 2928, 2900, 2904, 2905, 2902, - 2925, 2897, 2903, 2912, 2918, 2899, 2932, 2896, 2933, 2883, 2893, 2892, - 2890, 2907, 2911, 2891, 2885, 2909, 2986, 3006, 3027, 3023, 2985, 2974, - 2987, 2935, 2963, 2937, 3007, 3003, 2960, 3008, 2978, 2957, 2940, 2936, - 2942, 3026, 3005, 2968, 2998, 2966, 3028, 2947, 2948, 3012, 2979, 3016, - 2994, 3020, 3015, 2981, 3022, 2972, 2954, 3004, 2982, 2950, 2965, 2970, - 3001, 3017, 2984, 3031, 2975, 3000, 2997, 3030, 3021, 2962, 3032, 2990, - 2949, 3019, 2995, 2992, 2944, 2980, 2956, 2967, 2952, 2946, 2991, 2989, - 3024, 2983, 2999, 3002, 2945, 2996, 2976, 2953, 3029, 2977, 3013, 3011, - 2964, 2955, 2959, 2941, 2961, 2943, 2958, 3025, 2993, 2971, 2969, 3014, - 2939, 2938, 2988, 2973, 2951, 3009, 3010, 3018, 3060, 3144, 3093, 3067, - 3094, 3053, 3092, 3098, 3080, 3107, 3143, 3089, 3123, 3090, 3037, 3136, - 3121, 3096, 3128, 3041, 3145, 3043, 3130, 3065, 3075, 3056, 3062, 3132, - 3149, 3091, 3038, 3086, 3138, 3036, 3088, 3033, 3076, 3070, 3048, 3077, - 3072, 3071, 3113, 3069, 3066, 3054, 3099, 3058, 3046, 3105, 3134, 3133, - 3146, 3039, 3120, 3137, 3085, 3064, 3100, 3040, 3116, 3111, 3106, 3051, - 3079, 3068, 3083, 3147, 3115, 3148, 3078, 3102, 3127, 3044, 3082, 3087, - 3139, 3061, 3045, 3101, 3135, 3057, 3081, 3049, 3084, 3097, 3095, 3035, - 3042, 3117, 3141, 3140, 3108, 3119, 3050, 3063, 3055, 3129, 3074, 3125, - 3122, 3047, 3110, 3126, 3103, 3112, 3109, 3124, 3114, 3073, 3052, 3118, - 3142, 3104, 3059, 3131, 3034, 3203, 3280, 3190, 3177, 3287, 3180, 3243, - 3269, 3259, 3229, 3206, 3254, 3274, 3215, 3170, 3290, 3262, 3208, 3244, - 3279, 3172, 3182, 3231, 3221, 3187, 3218, 3222, 3230, 3261, 3228, 3247, - 3270, 3211, 3194, 3164, 3217, 3225, 3188, 3181, 3209, 3205, 3271, 3263, - 3257, 3202, 3210, 3295, 3163, 3284, 3291, 3251, 3283, 3167, 3268, 3277, - 3285, 3288, 3176, 3253, 3273, 3161, 3223, 3264, 3193, 3191, 3236, 3240, - 3160, 3282, 3171, 3303, 3234, 3296, 3192, 3204, 3249, 3299, 3179, 3153, - 3159, 3220, 3169, 3186, 3216, 3165, 3155, 3232, 3245, 3293, 3207, 3235, - 3237, 3252, 3196, 3154, 3239, 3197, 3162, 3152, 3200, 3255, 3219, 3173, - 3250, 3233, 3292, 3212, 3241, 3151, 3158, 3226, 3304, 3281, 3306, 3305, - 3178, 3242, 3272, 3275, 3201, 3267, 3294, 3213, 3300, 3297, 3298, 3302, - 3301, 3168, 3246, 3227, 3256, 3289, 3157, 3286, 3184, 3195, 3260, 3258, - 3214, 3224, 3265, 3266, 3150, 3238, 3248, 3183, 3175, 3198, 3185, 3189, - 3276, 3174, 3199, 3278, 3156, 3166, 3311, 3360, 3313, 3358, 3338, 3351, - 3332, 3315, 3341, 3340, 3320, 3350, 3330, 3326, 3317, 3348, 3343, 3349, - 3346, 3309, 3308, 3328, 3327, 3325, 3353, 3345, 3354, 3329, 3334, 3331, - 3355, 3335, 3342, 3333, 3337, 3307, 3323, 3324, 3344, 3336, 3359, 3356, - 3316, 3314, 3312, 3318, 3339, 3321, 3322, 3319, 3352, 3310, 3347, 3357, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 23535, 23527, 23548, - 23525, 23549, 23536, 23551, 23531, 23522, 23539, 23537, 23547, 23521, - 23529, 23524, 23526, 23532, 23530, 23528, 23541, 23544, 23533, 23543, - 23550, 23542, 23523, 23546, 23545, 23538, 23540, 23534, 35762, 23560, - 23562, 23559, 23558, 23555, 23554, 23557, 23556, 23563, 23561, 35762, - 35762, 35762, 35762, 23553, 23552, 30867, 30864, 30865, 30866, 30819, - 30816, 30817, 30818, 30871, 30868, 30869, 30870, 30859, 30856, 30857, - 30858, 30863, 30860, 30861, 30862, 30855, 30852, 30853, 30854, 30815, - 30812, 30813, 30814, 30847, 30844, 30845, 30846, 30820, 30825, 30836, - 30837, 30848, 30851, 30849, 30850, 30843, 30840, 30841, 30842, 30831, - 30828, 30829, 30830, 30882, 30881, 30880, 30832, 30839, 30889, 30887, - 30884, 30834, 30883, 30885, 30827, 30835, 30824, 30826, 30823, 30874, - 30878, 30886, 30833, 30838, 30876, 30873, 30879, 30822, 30872, 30888, - 30821, 30877, 30875, 30890, 35762, 30897, 30899, 30896, 30895, 30892, - 30891, 30894, 30893, 30900, 30898, 35762, 35762, 35762, 35762, 35762, - 35762, 3421, 3426, 3442, 3444, 3434, 3432, 3424, 3418, 3425, 3438, 3433, - 3429, 3440, 3423, 3419, 3441, 3428, 3439, 3443, 3437, 3431, 3445, 3430, - 3446, 3435, 3436, 3427, 3422, 3420, 3447, 35762, 35762, 3414, 3416, 3417, - 3415, 3413, 3448, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 26200, 26201, 26206, 26207, 26194, 26195, 26210, 26211, - 26202, 26203, 26192, 26193, 26212, 26213, 26196, 26197, 26208, 26209, - 26214, 26215, 26204, 26205, 26198, 26199, 26216, 26217, 26190, 26191, - 26136, 26127, 26133, 26124, 26129, 26135, 26128, 26132, 26138, 26122, - 26134, 26120, 26125, 26123, 26131, 26126, 26130, 26139, 26137, 26121, - 26144, 26143, 26141, 26140, 26142, 26146, 26145, 26176, 26177, 26155, - 26175, 26173, 26181, 26183, 26182, 26184, 26174, 26166, 26179, 26164, - 26186, 26159, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 26224, 26226, 26223, 26222, 26219, 26218, 26221, 26220, - 26227, 26225, 35762, 26151, 26148, 26150, 26153, 26147, 26149, 26152, - 35762, 26178, 26185, 26163, 26171, 26187, 26162, 26169, 26180, 26168, - 26189, 26170, 26165, 26172, 26188, 26167, 26161, 26154, 26157, 26158, - 26160, 26156, 35762, 35762, 35762, 35762, 35762, 26108, 26115, 26107, - 26106, 26116, 26104, 26101, 26119, 26111, 26109, 26117, 26103, 26102, - 26112, 26118, 26114, 26110, 26105, 26113, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 22208, 22212, 22215, 22216, 22186, 22218, 22194, 22209, - 22213, 22204, 22203, 22205, 22193, 22185, 22206, 22202, 22199, 22200, - 22214, 22196, 22207, 22210, 22192, 22190, 22217, 22201, 22198, 22188, - 22211, 22197, 22187, 22195, 22266, 22270, 22273, 22274, 22244, 22276, - 22252, 22267, 22271, 22262, 22261, 22263, 22251, 22243, 22264, 22260, - 22257, 22258, 22272, 22254, 22265, 22268, 22250, 22248, 22275, 22259, - 22256, 22246, 22269, 22255, 22245, 22253, 22230, 22224, 22222, 22220, - 22227, 22226, 22229, 22228, 22232, 22231, 22235, 22237, 22234, 22233, - 22238, 22239, 22241, 22242, 22236, 22240, 22225, 22223, 22221, 22219, - 22279, 22277, 22278, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 22822, 22748, 22810, 22821, 22826, 22825, - 22745, 22827, 22802, 22801, 22799, 22756, 22805, 22806, 22798, 22755, - 22764, 22772, 22808, 22744, 22769, 22768, 22763, 22762, 22761, 22760, - 22786, 22754, 22785, 22753, 22828, 22759, 22809, 22820, 22819, 22767, - 22766, 22743, 22824, 22830, 22758, 22757, 22795, 22751, 22771, 22770, - 22794, 22750, 22803, 22807, 22779, 22782, 22783, 22817, 22815, 22796, - 22752, 22804, 22784, 22818, 22816, 22814, 22812, 22742, 22813, 22811, - 22829, 22746, 22823, 22747, 22781, 22749, 22800, 22797, 22780, 35762, - 35762, 35762, 35762, 22834, 22765, 22833, 22832, 22831, 22839, 22845, - 22844, 22840, 22841, 22866, 22870, 22887, 22886, 22848, 22851, 22852, - 22868, 22855, 22856, 22857, 22858, 22859, 22862, 22864, 22865, 22861, - 22872, 22873, 22874, 22875, 22880, 22876, 22877, 22881, 22883, 22842, - 22843, 22850, 22885, 22849, 22884, 22846, 22854, 22847, 22871, 22888, - 22889, 22878, 22882, 22869, 22867, 22890, 22863, 22853, 22860, 22879, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 22838, 22836, 22837, - 22835, 22787, 22788, 22789, 22790, 22791, 22792, 22793, 22773, 22774, - 22775, 22776, 22777, 22778, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 31669, 25140, - 25317, 25316, 17699, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 33692, 33691, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30901, 30902, 30903, 30904, 30905, 30906, 30907, 30908, 30909, 30910, - 30911, 30912, 30913, 30914, 30915, 30916, 30917, 30918, 30919, 30920, - 30921, 30922, 30923, 30924, 30925, 30926, 30927, 30928, 30929, 30930, - 30931, 30932, 30933, 30934, 30935, 30936, 30937, 30938, 30939, 30940, - 30941, 30942, 30943, 30944, 30945, 30946, 30947, 30948, 30949, 30950, - 30951, 30952, 30953, 30954, 30955, 30956, 30957, 30958, 30959, 30960, - 30961, 30962, 30963, 30964, 30965, 30966, 30967, 30968, 30969, 30970, - 30971, 30972, 30973, 30974, 30975, 30976, 30977, 30978, 30979, 30980, - 30981, 30982, 30983, 30984, 30985, 30986, 30987, 30988, 30989, 30990, - 30991, 30992, 30993, 30994, 30995, 30996, 30997, 30998, 30999, 31000, - 31001, 31002, 31003, 31004, 31005, 31006, 31007, 31008, 31009, 31010, - 31011, 31012, 31013, 31014, 31015, 31016, 31017, 31018, 31019, 31020, - 31021, 31022, 31023, 31024, 31025, 31026, 31027, 31028, 31029, 31030, - 31031, 31032, 31033, 31034, 31035, 31036, 31037, 31038, 31039, 31040, - 31041, 31042, 31043, 31044, 31045, 31046, 31047, 31048, 31049, 31050, - 31051, 31052, 31053, 31054, 31055, 31056, 31057, 31058, 31059, 31060, - 31061, 31062, 31063, 31064, 31065, 31066, 31067, 31068, 31069, 31070, - 31071, 31072, 31073, 31074, 31075, 31076, 31077, 31078, 31079, 31080, - 31081, 31082, 31083, 31084, 31085, 31086, 31087, 31088, 31089, 31090, - 31091, 31092, 31093, 31094, 31095, 31096, 31097, 31098, 31099, 31100, - 31101, 31102, 31103, 31104, 31105, 31106, 31107, 31108, 31109, 31110, - 31111, 31112, 31113, 31114, 31115, 31116, 31117, 31118, 31119, 31120, - 31121, 31122, 31123, 31124, 31125, 31126, 31127, 31128, 31129, 31130, - 31131, 31132, 31133, 31134, 31135, 31136, 31137, 31138, 31139, 31140, - 31141, 31142, 31143, 31144, 31145, 31146, 31147, 31148, 31149, 31150, - 31151, 31152, 31153, 31154, 31155, 31156, 31157, 31158, 31159, 31160, - 31161, 31162, 31163, 31164, 31165, 31166, 31167, 31168, 31169, 31170, - 31171, 31172, 31173, 31174, 31175, 31176, 31177, 31178, 31179, 31180, - 31181, 31182, 31183, 31184, 31185, 31186, 31187, 31188, 31189, 31190, - 31191, 31192, 31193, 31194, 31195, 31196, 31197, 31198, 31199, 31200, - 31201, 31202, 31203, 31204, 31205, 31206, 31207, 31208, 31209, 31210, - 31211, 31212, 31213, 31214, 31215, 31216, 31217, 31218, 31219, 31220, - 31221, 31222, 31223, 31224, 31225, 31226, 31227, 31228, 31229, 31230, - 31231, 31232, 31233, 31234, 31235, 31236, 31237, 31238, 31239, 31240, - 31241, 31242, 31243, 31244, 31245, 31246, 31247, 31248, 31249, 31250, - 31251, 31252, 31253, 31254, 31255, 31256, 31257, 31258, 31259, 31260, - 31261, 31262, 31263, 31264, 31265, 31266, 31267, 31268, 31269, 31270, - 31271, 31272, 31273, 31274, 31275, 31276, 31277, 31278, 31279, 31280, - 31281, 31282, 31283, 31284, 31285, 31286, 31287, 31288, 31289, 31290, - 31291, 31292, 31293, 31294, 31295, 31296, 31297, 31298, 31299, 31300, - 31301, 31302, 31303, 31304, 31305, 31306, 31307, 31308, 31309, 31310, - 31311, 31312, 31313, 31314, 31315, 31316, 31317, 31318, 31319, 31320, - 31321, 31322, 31323, 31324, 31325, 31326, 31327, 31328, 31329, 31330, - 31331, 31332, 31333, 31334, 31335, 31336, 31337, 31338, 31339, 31340, - 31341, 31342, 31343, 31344, 31345, 31346, 31347, 31348, 31349, 31350, - 31351, 31352, 31353, 31354, 31355, 31356, 31357, 31358, 31359, 31360, - 31361, 31362, 31363, 31364, 31365, 31366, 31367, 31368, 31369, 31370, - 31371, 31372, 31373, 31374, 31375, 31376, 31377, 31378, 31379, 31380, - 31381, 31382, 31383, 31384, 31385, 31386, 31387, 31388, 31389, 31390, - 31391, 31392, 31393, 31394, 31395, 31396, 31397, 31398, 31399, 31400, - 31401, 31402, 31403, 31404, 31405, 31406, 31407, 31408, 31409, 31410, - 31411, 31412, 31413, 31414, 31415, 31416, 31417, 31418, 31419, 31420, - 31421, 31422, 31423, 31424, 31425, 31426, 31427, 31428, 31429, 31430, - 31431, 31432, 31433, 31434, 31435, 31436, 31437, 31438, 31439, 31440, - 31441, 31442, 31443, 31444, 31445, 31446, 31447, 31448, 31449, 31450, - 31451, 31452, 31453, 31454, 31455, 31456, 31457, 31458, 31459, 31460, - 31461, 31462, 31463, 31464, 31465, 31466, 31467, 31468, 31469, 31470, - 31471, 31472, 31473, 31474, 31475, 31476, 31477, 31478, 31479, 31480, - 31481, 31482, 31483, 31484, 31485, 31486, 31487, 31488, 31489, 31490, - 31491, 31492, 31493, 31494, 31495, 31496, 31497, 31498, 31499, 31500, - 31501, 31502, 31503, 31504, 31505, 31506, 31507, 31508, 31509, 31510, - 31511, 31512, 31513, 31514, 31515, 31516, 31517, 31518, 31519, 31520, - 31521, 31522, 31523, 31524, 31525, 31526, 31527, 31528, 31529, 31530, - 31531, 31532, 31533, 31534, 31535, 31536, 31537, 31538, 31539, 31540, - 31541, 31542, 31543, 31544, 31545, 31546, 31547, 31548, 31549, 31550, - 31551, 31552, 31553, 31554, 31555, 31556, 31557, 31558, 31559, 31560, - 31561, 31562, 31563, 31564, 31565, 31566, 31567, 31568, 31569, 31570, - 31571, 31572, 31573, 31574, 31575, 31576, 31577, 31578, 31579, 31580, - 31581, 31582, 31583, 31584, 31585, 31586, 31587, 31588, 31589, 31590, - 31591, 31592, 31593, 31594, 31595, 31596, 31597, 31598, 31599, 31600, - 31601, 31602, 31603, 31604, 31605, 31606, 31607, 31608, 31609, 31610, - 31611, 31612, 31613, 31614, 31615, 31616, 31617, 31618, 31619, 31620, - 31621, 31622, 31623, 31624, 31625, 31626, 31627, 31628, 31629, 31630, - 31631, 31632, 31633, 31634, 31635, 31636, 31637, 31638, 31639, 31640, - 31641, 31642, 31643, 31644, 31645, 31646, 31647, 31648, 31649, 31650, - 31651, 31652, 31653, 31654, 31655, 31656, 31657, 31658, 31659, 31660, - 31661, 31662, 31663, 31664, 31665, 31666, 31667, 31668, 17229, 17230, - 17231, 17232, 17233, 17234, 17235, 17236, 17237, 17238, 17239, 17240, - 17241, 17242, 17243, 17244, 17245, 17246, 17247, 17248, 17249, 17250, - 17251, 17252, 17253, 17254, 17255, 17256, 17257, 17258, 17259, 17260, - 17261, 17262, 17263, 17264, 17265, 17266, 17267, 17268, 17269, 17270, - 17271, 17272, 17273, 17274, 17275, 17276, 17277, 17278, 17279, 17280, - 17281, 17282, 17283, 17284, 17285, 17286, 17287, 17288, 17289, 17290, - 17291, 17292, 17293, 17294, 17295, 17296, 17297, 17298, 17299, 17300, - 17301, 17302, 17303, 17304, 17305, 17306, 17307, 17308, 17309, 17310, - 17311, 17312, 17313, 17314, 17315, 17316, 17317, 17318, 17319, 17320, - 17321, 17322, 17323, 17324, 17325, 17326, 17327, 17328, 17329, 17330, - 17331, 17332, 17333, 17334, 17335, 17336, 17337, 17338, 17339, 17340, - 17341, 17342, 17343, 17344, 17345, 17346, 17347, 17348, 17349, 17350, - 17351, 17352, 17353, 17354, 17355, 17356, 17357, 17358, 17359, 17360, - 17361, 17362, 17363, 17364, 17365, 17366, 17367, 17368, 17369, 17370, - 17371, 17372, 17373, 17374, 17375, 17376, 17377, 17378, 17379, 17380, - 17381, 17382, 17383, 17384, 17385, 17386, 17387, 17388, 17389, 17390, - 17391, 17392, 17393, 17394, 17395, 17396, 17397, 17398, 17399, 17400, - 17401, 17402, 17403, 17404, 17405, 17406, 17407, 17408, 17409, 17410, - 17411, 17412, 17413, 17414, 17415, 17416, 17417, 17418, 17419, 17420, - 17421, 17422, 17423, 17424, 17425, 17426, 17427, 17428, 17429, 17430, - 17431, 17432, 17433, 17434, 17435, 17436, 17437, 17438, 17439, 17440, - 17441, 17442, 17443, 17444, 17445, 17446, 17447, 17448, 17449, 17450, - 17451, 17452, 17453, 17454, 17455, 17456, 17457, 17458, 17459, 17460, - 17461, 17462, 17463, 17464, 17465, 17466, 17467, 17468, 17469, 17470, - 17471, 17472, 17473, 17474, 17475, 17476, 17477, 17478, 17479, 17480, - 17481, 17482, 17483, 17484, 17491, 17492, 17493, 17494, 17495, 17496, - 17497, 17498, 17499, 17500, 17501, 17502, 17503, 17504, 17505, 17506, - 17507, 17508, 17509, 17510, 17511, 17512, 17513, 17514, 17515, 17516, - 17517, 17518, 17519, 17520, 17521, 17522, 17523, 17524, 17525, 17526, - 17527, 17528, 17529, 17530, 17531, 17532, 17533, 17534, 17535, 17536, - 17537, 17538, 17539, 17540, 17541, 17542, 17543, 17544, 17545, 17546, - 17547, 17548, 17549, 17550, 17551, 17552, 17553, 17554, 17555, 17556, - 17557, 17558, 17559, 17560, 17561, 17562, 17563, 17564, 17565, 17566, - 17567, 17568, 17569, 17570, 17571, 17572, 17573, 17574, 17575, 17576, - 17577, 17578, 17579, 17580, 17581, 17582, 17583, 17584, 17585, 17586, - 17587, 17588, 17589, 17590, 17591, 17592, 17593, 17594, 17595, 17596, - 17597, 17598, 17599, 17600, 17601, 17602, 17603, 17604, 17605, 17606, - 17607, 17608, 17609, 17610, 17611, 17612, 17613, 17614, 17615, 17616, - 17617, 17618, 17619, 17620, 17621, 17622, 17623, 17624, 17625, 17626, - 17627, 17628, 17629, 17630, 17631, 17632, 17633, 17634, 17635, 17636, - 17637, 17638, 17639, 17640, 17641, 17642, 17643, 17644, 17645, 17646, - 17647, 17648, 17649, 17650, 17651, 17652, 17653, 17654, 17655, 17656, - 17657, 17658, 17659, 17660, 17661, 17662, 17663, 17664, 17665, 17666, - 17667, 17668, 17669, 17670, 17671, 17672, 17673, 17674, 17675, 17676, - 17677, 17678, 17679, 17680, 17681, 17682, 17683, 17684, 17685, 17686, - 17687, 17688, 17689, 17690, 17691, 17692, 17693, 17694, 17695, 17696, - 17697, 17698, 17485, 17486, 17487, 17488, 17489, 17490, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 16870, 16871, 16872, 16873, 35762, 16874, 16875, 16863, 16864, 16865, - 16866, 16867, 35762, 16868, 16869, 35762, 16851, 15837, 15476, 15477, - 15478, 15475, 15757, 15758, 15759, 15760, 15749, 15750, 15751, 15752, - 15753, 15744, 15745, 15746, 15747, 15748, 15754, 15755, 15756, 15515, - 15519, 15520, 15521, 15522, 15523, 15524, 15525, 15526, 15516, 15517, - 15518, 15531, 15532, 15533, 15534, 15535, 15536, 15537, 15538, 15545, - 15546, 15547, 15548, 15549, 15550, 15551, 15539, 15540, 15541, 15542, - 15543, 15544, 15528, 15529, 15530, 15527, 15640, 15641, 15642, 15643, - 15644, 15645, 15646, 15647, 15656, 15657, 15658, 15659, 15660, 15661, - 15648, 15649, 15650, 15651, 15652, 15653, 15654, 15655, 15662, 15663, - 15664, 15665, 15666, 15667, 15668, 15669, 15670, 15671, 15672, 15673, - 15702, 15703, 15704, 15705, 15695, 15696, 15697, 15698, 15699, 15700, - 15701, 15691, 15692, 15693, 15694, 15690, 15674, 15675, 15676, 15677, - 15678, 15679, 15680, 15681, 15682, 15684, 15685, 15686, 15687, 15688, - 15689, 15683, 15594, 15595, 15596, 15597, 15598, 15599, 15600, 15601, - 15602, 15587, 15588, 15589, 15590, 15591, 15592, 15593, 15586, 15608, - 15609, 15610, 15580, 15581, 15582, 15583, 15584, 15585, 15579, 15603, - 15604, 15605, 15606, 15607, 15487, 15490, 15491, 15492, 15493, 15494, - 15495, 15496, 15497, 15488, 15489, 15508, 15509, 15510, 15511, 15512, - 15513, 15514, 15498, 15499, 15500, 15501, 15502, 15503, 15504, 15505, - 15506, 15507, 15479, 15480, 15481, 15482, 15483, 15484, 15485, 15486, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 34087, 17354, 31745, 29447, 11088, 31317, 368, 1052, 24153, + 31969, 2527, 31568, 8453, 20460, 17532, 33926, 10872, 10857, 10869, + 10866, 10851, 10848, 10863, 10860, 10875, 10854, 8056, 32578, 24396, + 16792, 18147, 31743, 8452, 22741, 22787, 22797, 22813, 22828, 22873, + 22877, 22894, 22908, 22937, 22942, 22954, 22974, 22983, 22999, 23049, + 23057, 23060, 23081, 23105, 23134, 23173, 23184, 23193, 23196, 23210, + 24115, 31870, 31983, 6819, 25161, 18118, 23302, 23352, 23372, 23395, + 23431, 23490, 23498, 23516, 23535, 23572, 23578, 23592, 23631, 23644, + 23668, 23725, 23736, 23742, 23780, 23822, 23895, 23943, 23957, 23966, + 23974, 23990, 24056, 38833, 31932, 37296, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 29440, + 20815, 6131, 31610, 9951, 39398, 4937, 32560, 10839, 8647, 17433, 24163, + 29422, 33877, 31793, 26847, 10554, 31580, 34712, 34711, 4, 27575, 31465, + 27581, 6127, 34715, 25814, 32030, 38965, 38963, 38971, 20818, 22770, + 22771, 22759, 22773, 22752, 22754, 22775, 22798, 22859, 22860, 22831, + 22846, 22924, 22925, 22914, 22912, 22870, 22997, 23038, 23039, 23006, + 23024, 23013, 28287, 23022, 23165, 23166, 23135, 23138, 23205, 23126, + 23812, 23333, 23334, 23322, 23336, 23313, 23315, 23339, 23373, 23464, + 23465, 23434, 23449, 23553, 23554, 23536, 23538, 23489, 23663, 23707, + 23708, 23669, 23693, 23676, 10999, 23691, 23930, 23933, 23898, 23901, + 23985, 23844, 23976, 22757, 23320, 22742, 23303, 22767, 23330, 22802, + 23378, 22800, 23375, 22806, 23383, 22801, 23377, 22820, 23403, 22817, + 23397, 22849, 23453, 22862, 23467, 22841, 23444, 22856, 23461, 22840, + 23443, 22879, 23500, 22882, 23503, 22886, 23508, 22878, 23499, 22897, + 23519, 22904, 23528, 22918, 23550, 22916, 23545, 22927, 23556, 22921, + 23547, 22911, 23419, 23222, 24010, 22938, 23573, 22943, 23579, 23591, + 22968, 23609, 22964, 23596, 22966, 23599, 22961, 23606, 22963, 23617, + 22992, 23656, 22984, 23645, 22986, 23649, 23664, 22872, 23472, 23002, + 23686, 23041, 23710, 23019, 23682, 23223, 24013, 23069, 23759, 23061, + 23743, 23062, 23745, 23087, 23781, 23086, 23787, 23084, 23785, 23082, + 23783, 23106, 23823, 23109, 23827, 23116, 23839, 23162, 23927, 23154, + 23917, 23168, 23935, 23169, 23926, 23145, 23908, 23159, 23922, 23185, + 23958, 23197, 23975, 23198, 23213, 23991, 23216, 23997, 23212, 23994, + 23623, 23361, 22793, 22788, 23353, 23129, 23847, 23044, 22804, 23381, + 22779, 22825, 22821, 23398, 23852, 23078, 23104, 23043, 22874, 23493, + 22885, 22892, 23534, 22935, 22928, 22953, 23590, 23595, 23619, 23123, + 22990, 23650, 23005, 23031, 23700, 23046, 23722, 23055, 23733, 23299, + 23130, 23848, 22871, 23239, 23835, 23115, 23836, 23114, 23147, 23910, + 23172, 23177, 23202, 23980, 23219, 24000, 22864, 22866, 23482, 23487, + 23291, 23131, 23849, 23238, 23292, 23293, 23294, 23231, 23241, 22827, + 22815, 23425, 22973, 22962, 23630, 22998, 22987, 23667, 22765, 23328, + 22915, 23537, 23012, 23675, 23137, 23900, 23142, 23905, 23141, 23904, + 23139, 23902, 23140, 23903, 23884, 22753, 23314, 22749, 23310, 22777, + 23342, 22887, 23509, 22880, 23501, 22944, 23580, 23020, 23689, 23021, + 23690, 22865, 23484, 23575, 22826, 22814, 23424, 22881, 23502, 22906, + 23192, 22991, 23655, 22755, 23316, 22776, 23341, 23023, 23692, 22751, + 23312, 22772, 23335, 22845, 23448, 22861, 23466, 22909, 23544, 22926, + 23555, 23018, 23681, 23040, 23709, 23066, 23749, 23071, 23760, 23144, + 23907, 23167, 23934, 23085, 23786, 23108, 23825, 23209, 23989, 22898, + 23520, 22988, 23402, 23048, 23724, 23221, 24004, 22748, 23309, 22829, + 23432, 23014, 23677, 23027, 23696, 23015, 23678, 23016, 23679, 23206, + 23986, 23598, 23648, 23826, 23416, 23429, 23741, 22774, 22807, 23384, + 22956, 23110, 23794, 23999, 22888, 23510, 22794, 23171, 23125, 22863, + 23469, 22941, 23577, 23098, 23739, 23073, 23763, 23208, 23984, 23876, + 23344, 23877, 23360, 23713, 23376, 23399, 23407, 23772, 23803, 23807, + 23719, 23767, 23769, 23391, 23417, 23507, 23810, 23268, 23514, 23779, + 23853, 23527, 23533, 23557, 23568, 23246, 23604, 23593, 23612, 23620, + 23879, 23880, 23639, 23652, 23661, 23279, 23364, 23253, 23390, 23735, + 23864, 23867, 23870, 23754, 23755, 23750, 23770, 23255, 23247, 23800, + 23475, 23418, 23820, 23479, 23871, 23838, 23896, 23937, 23950, 23873, + 23888, 23881, 23286, 24003, 23993, 23481, 23483, 23295, 23297, 23236, + 23288, 23234, 23266, 23389, 23269, 23275, 23574, 23886, 23248, 23738, + 23296, 23240, 23426, 23412, 23427, 23891, 23840, 23890, 23496, 23627, + 23628, 23233, 23235, 23854, 23855, 27905, 27906, 27914, 27936, 27963, + 27966, 27861, 27983, 27985, 27834, 27777, 27991, 27688, 27842, 27844, + 27820, 27793, 27840, 27822, 27846, 27993, 27779, 27769, 6060, 27997, + 27823, 27687, 27792, 27818, 27809, 27815, 27814, 27990, 27800, 27721, + 27720, 27992, 27778, 27833, 27831, 4930, 11194, 32162, 29978, 33836, + 11253, 27847, 27770, 27904, 27916, 27943, 27984, 27940, 27784, 27799, + 27827, 27812, 27789, 27999, 27998, 27996, 27994, 27776, 27805, 27817, + 27807, 27810, 27811, 27830, 27829, 27828, 27813, 27839, 27690, 27790, + 27691, 27791, 27849, 27832, 27806, 8264, 8062, 8146, 8428, 8370, 8390, + 8072, 8171, 8168, 8277, 8415, 8197, 8078, 8443, 8196, 8198, 8080, 8280, + 8436, 8083, 8398, 8084, 8265, 8063, 8355, 8411, 8349, 8279, 8352, 8438, + 8202, 8395, 8379, 8394, 8399, 8174, 8170, 8414, 8085, 8148, 8388, 8442, + 8075, 8283, 8079, 8147, 8074, 8281, 8432, 8375, 8369, 8199, 8431, 8419, + 8367, 8418, 8366, 8406, 8282, 8422, 8426, 8450, 8444, 8185, 8266, 8064, + 8274, 8273, 8276, 8275, 8076, 8211, 8195, 8348, 8381, 8278, 8071, 8356, + 8437, 8269, 8402, 8353, 8212, 8449, 8344, 8403, 8401, 8407, 8173, 8068, + 8190, 8417, 8179, 8178, 8182, 8183, 8191, 8180, 8189, 8296, 8304, 8309, + 8317, 8320, 8302, 8334, 8336, 8338, 8323, 8326, 8341, 8342, 18332, 18596, + 18227, 18463, 18414, 18410, 18321, 18578, 40951, 40951, 18653, 18603, + 18604, 18602, 18658, 18335, 40951, 40951, 40951, 40951, 18618, 18348, + 18225, 18201, 18258, 18248, 18272, 40951, 18304, 40951, 18319, 18294, + 18510, 18204, 18331, 18329, 18327, 18249, 18333, 18228, 18325, 18259, + 18328, 18334, 18336, 18337, 18338, 18295, 18324, 18305, 40951, 18307, + 18326, 18310, 18322, 18330, 18323, 18274, 18264, 18315, 18460, 18475, + 18500, 18519, 18530, 18435, 18595, 18593, 18465, 18466, 18597, 18476, + 18590, 18501, 18541, 18598, 18599, 18600, 18601, 18568, 18581, 18582, + 18592, 18588, 18591, 18521, 18579, 18594, 18580, 18543, 18506, 18526, + 18577, 18539, 18567, 18343, 18655, 18614, 18622, 18620, 18621, 18430, + 18431, 18395, 18406, 18462, 18405, 18587, 18408, 18464, 18407, 18542, + 18404, 18585, 8528, 8622, 8506, 8600, 8502, 8596, 8500, 8594, 8498, 8592, + 8527, 8621, 8497, 8591, 18394, 18433, 18411, 18409, 18344, 18412, 18434, + 18309, 18589, 18339, 18308, 18586, 18432, 18341, 18342, 18340, 10218, + 10220, 10167, 10206, 10142, 10171, 10158, 10277, 10290, 10113, 10116, + 10130, 10245, 10215, 10258, 10175, 10143, 10159, 10291, 10200, 10179, + 10217, 10284, 10280, 10213, 10256, 10230, 10181, 10197, 10186, 10249, + 10117, 10192, 10195, 10127, 10137, 10199, 10207, 10133, 10160, 10263, + 10261, 10211, 10274, 10266, 10180, 10279, 10271, 10302, 10318, 10492, + 10359, 10338, 10376, 10485, 10481, 10372, 10434, 10389, 10340, 10356, + 10345, 10415, 10420, 10351, 10354, 10452, 10463, 10358, 10366, 10458, + 10319, 10441, 10439, 10370, 10475, 10444, 10339, 10480, 10472, 10377, + 10379, 10326, 10365, 10468, 10330, 10317, 10478, 10491, 10408, 10414, + 10455, 10404, 10374, 10436, 10334, 10250, 10416, 10273, 10474, 10226, + 10385, 10112, 10406, 10222, 10381, 10155, 10314, 10223, 10382, 10246, + 10405, 10120, 10424, 10289, 10490, 10228, 10387, 10229, 10388, 10141, + 10467, 10121, 10429, 10251, 10417, 10253, 10419, 10243, 10402, 10523, + 8139, 8133, 8136, 8134, 8135, 8089, 8142, 10257, 10435, 10270, 10448, + 10193, 10352, 10202, 10361, 10204, 10363, 10201, 10360, 10286, 10487, + 10282, 10483, 10231, 10390, 10233, 10392, 10234, 10393, 10153, 10312, + 10188, 10347, 10296, 10496, 10118, 10421, 10149, 10308, 10196, 10355, + 10129, 10454, 10268, 10446, 10269, 10447, 10208, 10367, 10295, 10495, + 10162, 10321, 10163, 10322, 10259, 10437, 10146, 10305, 10147, 10306, + 10299, 10287, 10488, 10232, 10391, 10184, 10343, 10191, 10350, 10190, + 10349, 10244, 10403, 10198, 10357, 10423, 10145, 10304, 10144, 10303, + 10294, 10494, 10219, 10378, 10254, 10432, 10255, 10433, 10285, 10486, + 10281, 10482, 10148, 10307, 10216, 10375, 10214, 10373, 10252, 10418, + 10151, 10310, 10152, 10311, 10194, 10353, 10140, 10466, 10139, 10465, + 10138, 10464, 10161, 10320, 10203, 10362, 10275, 10476, 10205, 10364, + 10209, 10368, 10210, 10369, 10237, 10396, 10236, 10395, 10242, 10401, + 10235, 10394, 10238, 10397, 10239, 10398, 10240, 10399, 10241, 10400, + 10125, 10428, 10185, 10344, 10114, 10409, 10126, 10431, 10272, 10473, + 10293, 10493, 10292, 10471, 10150, 10309, 10182, 10341, 10187, 10346, + 10119, 10422, 10260, 10438, 10189, 10348, 10173, 10332, 10177, 10336, + 10183, 10342, 40951, 2445, 2452, 2436, 2458, 2432, 2456, 2433, 2434, + 2423, 2455, 2451, 2448, 2450, 2428, 2440, 2457, 2438, 2435, 2426, 2453, + 2424, 2454, 2443, 2447, 2427, 2442, 2437, 2431, 2444, 2446, 2422, 2430, + 2429, 2425, 2441, 2439, 2459, 2449, 40951, 40951, 2509, 2421, 2462, 2461, + 2460, 2513, 2420, 2482, 2485, 2495, 2473, 2501, 2469, 2499, 2470, 2471, + 2484, 2498, 2494, 2491, 2493, 2465, 2477, 2500, 2475, 2472, 2463, 2496, + 2488, 2497, 2480, 2487, 2464, 2479, 2474, 2468, 2481, 2486, 2483, 2467, + 2466, 2490, 2478, 2476, 2502, 2492, 2503, 2489, 2512, 2510, 40951, 40951, + 32017, 24176, 2511, 40951, 19805, 19808, 19809, 19816, 19817, 19813, + 19820, 19818, 19803, 19815, 19812, 19796, 19797, 19798, 19806, 19811, + 19804, 19793, 19801, 19802, 19799, 19800, 19795, 19807, 19810, 19814, + 19821, 19822, 19794, 19819, 19900, 19915, 19904, 19902, 19903, 19905, + 19917, 19913, 19908, 19909, 19906, 19907, 19911, 19901, 19919, 19925, + 19912, 19922, 19914, 19916, 19923, 19899, 19898, 19924, 19910, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 19823, 19830, 19873, + 19871, 19848, 19879, 19853, 19850, 19865, 19890, 19839, 19833, 19875, + 19845, 19877, 19844, 19851, 19855, 19829, 19841, 19836, 19843, 19869, + 19846, 19863, 19857, 19867, 40951, 40951, 40951, 40951, 19926, 19894, + 19897, 19895, 19921, 19920, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 2264, 2299, 1093, 2300, 2301, 2263, + 2408, 2409, 2275, 2406, 2407, 2598, 1062, 1075, 2270, 2305, 2298, 2304, + 2296, 2297, 2306, 2326, 2312, 2341, 2308, 2357, 2356, 2292, 1381, 1083, + 2396, 2404, 1336, 1288, 1151, 1141, 1569, 1144, 1585, 1123, 1165, 1512, + 1511, 1532, 1314, 1273, 1355, 1190, 1529, 1433, 1611, 1467, 1480, 1459, + 1206, 1489, 1606, 1113, 1260, 1342, 1339, 1235, 1234, 1233, 2384, 1240, + 1424, 1325, 1360, 1373, 1397, 1290, 1564, 1159, 1576, 1091, 1073, 1103, + 1085, 1069, 1099, 2293, 2361, 2117, 1098, 1097, 2360, 2282, 2118, 2405, + 2400, 2399, 2401, 2276, 1086, 2403, 2416, 2418, 2415, 2414, 2411, 2410, + 2413, 2412, 2419, 2417, 2268, 1079, 2398, 1094, 1218, 1220, 1486, 1156, + 1148, 1147, 1310, 1311, 1313, 1550, 1312, 1538, 1544, 1185, 1520, 1519, + 1412, 1524, 1180, 1283, 1281, 1392, 1221, 1280, 1499, 1506, 1215, 1200, + 1197, 1198, 1203, 1212, 1226, 1193, 1199, 1450, 1436, 1447, 1444, 1437, + 1445, 1440, 1322, 1443, 1468, 1471, 1472, 1462, 1461, 1493, 1116, 1219, + 1243, 1241, 1557, 1245, 1419, 1427, 1428, 1337, 1488, 1331, 1330, 1382, + 1328, 1251, 1254, 1383, 1253, 1267, 1252, 1364, 1362, 1366, 1365, 1406, + 1398, 1453, 1407, 1403, 1301, 1501, 1298, 1291, 1292, 1515, 1573, 1349, + 1603, 1549, 1600, 1352, 1572, 1556, 1229, 1594, 1590, 1566, 1615, 1595, + 1577, 1580, 1095, 1164, 2315, 2314, 2317, 2316, 2343, 2325, 2323, 1084, + 2383, 2340, 2339, 2309, 2318, 2355, 2319, 2359, 2358, 2337, 2320, 2272, + 1082, 1081, 2280, 2354, 1192, 1441, 17374, 17376, 17373, 17372, 17369, + 17368, 17371, 17370, 17377, 17375, 1481, 1207, 1262, 2303, 2302, 1297, + 34842, 34916, 34913, 34914, 34910, 34851, 34838, 34839, 34915, 34912, + 34837, 34847, 34846, 34845, 40951, 34835, 34883, 34879, 34893, 34889, + 34890, 34853, 34852, 34854, 34896, 34894, 34855, 34886, 34887, 34891, + 34892, 34884, 34856, 34868, 34895, 34875, 34882, 34897, 34869, 34873, + 34881, 34885, 34874, 34880, 34888, 34870, 34872, 34871, 34902, 34901, + 34900, 34905, 34904, 34903, 34907, 34906, 34841, 34840, 34850, 34849, + 34848, 34844, 34843, 34908, 34922, 34923, 34909, 34920, 34919, 34918, + 34917, 34899, 34898, 34921, 34836, 40951, 40951, 34876, 34877, 34878, + 1171, 1174, 1169, 1170, 1172, 1173, 1167, 1282, 1279, 1196, 1191, 1438, + 1474, 1118, 1114, 1117, 1246, 1244, 1344, 1340, 1338, 1376, 1375, 1404, + 1401, 1402, 1367, 1439, 1442, 1473, 1278, 1276, 1470, 1434, 1277, 1150, + 1149, 1232, 1231, 1230, 1568, 1567, 1579, 1578, 1274, 1475, 1469, 1327, + 36865, 36878, 36874, 36891, 36890, 36869, 36866, 36854, 36885, 36870, + 36859, 36858, 36881, 36868, 36861, 36862, 36879, 36857, 36887, 36880, + 36892, 36873, 36872, 36871, 36883, 36864, 36867, 36882, 36888, 36877, + 36876, 36856, 36884, 36889, 36855, 36863, 36860, 36886, 36850, 36849, + 36896, 36852, 36897, 36895, 36851, 36853, 36894, 36893, 36898, 36875, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 29306, 29308, 29305, 29304, 29301, 29300, + 29303, 29302, 29309, 29307, 29341, 29328, 29342, 29327, 29343, 29325, + 29324, 29312, 29317, 29330, 29336, 29338, 29316, 29326, 29311, 29323, + 29322, 29337, 29329, 29331, 29333, 29334, 29319, 29335, 29320, 29318, + 29332, 29339, 29340, 29321, 29314, 29313, 29315, 29294, 29295, 29296, + 29292, 29289, 29290, 29291, 29293, 29288, 29345, 29344, 29347, 29346, + 29297, 29348, 29310, 40951, 40951, 29298, 29299, 29349, 32389, 32376, + 32390, 32378, 32381, 32377, 32392, 32380, 32387, 32396, 32382, 32383, + 32393, 32395, 32384, 32379, 32397, 32388, 32394, 32391, 32385, 32386, + 32399, 32400, 32403, 32398, 32404, 32401, 32423, 32433, 32428, 32422, + 32432, 32427, 32421, 32431, 32405, 32429, 32425, 32435, 32406, 32424, + 32434, 32426, 32430, 32402, 40951, 40951, 32413, 32407, 32409, 32412, + 32410, 32414, 32436, 32419, 32417, 32420, 32416, 32418, 32411, 32415, + 32408, 40951, 25603, 25590, 25592, 25591, 25593, 25602, 25600, 25605, + 25585, 25583, 25582, 25594, 25595, 25596, 25586, 25604, 25597, 25588, + 25598, 25599, 25587, 25584, 25601, 25606, 25589, 25581, 25607, 25608, + 40951, 40951, 25609, 40951, 34861, 34866, 34862, 34864, 34860, 34859, + 34863, 34867, 34858, 34857, 34865, 40951, 40951, 40951, 40951, 40951, + 1137, 1127, 1138, 1154, 1136, 1124, 1133, 1130, 1134, 1132, 1155, 1129, + 1140, 1126, 1128, 1139, 1125, 1131, 1135, 2385, 2386, 2388, 1537, 2281, + 2274, 1405, 1275, 1494, 1492, 1341, 2402, 40951, 2271, 2273, 40951, + 40951, 40951, 40951, 40951, 2269, 2327, 2348, 2347, 2350, 2116, 2365, + 1076, 1096, 1168, 1175, 1315, 1491, 1242, 1425, 1361, 1374, 1591, 1593, + 1446, 1565, 1458, 1372, 1194, 1460, 1255, 1487, 1614, 1115, 1329, 1426, + 1166, 1413, 1517, 1435, 1592, 1111, 1109, 1112, 1414, 1518, 1539, 1500, + 1343, 1261, 1110, 1317, 1316, 1363, 1272, 2307, 2310, 2338, 2333, 2342, + 1107, 1106, 2364, 1108, 1105, 2353, 2328, 2324, 2345, 2344, 2321, 2346, + 2329, 2331, 2330, 2332, 2335, 2334, 2311, 2322, 1080, 2397, 1065, 1063, + 1067, 1066, 1064, 1068, 2391, 2393, 2395, 2390, 2392, 2394, 2267, 2266, + 2265, 2336, 1088, 1087, 1100, 1621, 2277, 1620, 2279, 1077, 1078, 2278, + 1072, 2119, 10772, 10762, 10776, 10781, 10697, 10671, 10672, 10741, + 10742, 10717, 10718, 10736, 10738, 10679, 10698, 10749, 10673, 10680, + 10699, 10703, 10674, 10711, 10710, 10692, 10690, 10723, 10677, 10681, + 10728, 10726, 10724, 10733, 10732, 10685, 10684, 10722, 10735, 10734, + 10687, 10686, 10725, 10721, 10744, 10743, 10708, 10707, 10695, 10716, + 10714, 10713, 10731, 10730, 10729, 10740, 10700, 10701, 10702, 10694, + 10794, 10793, 10774, 10775, 10784, 10806, 10807, 10796, 10797, 10802, + 10803, 10790, 10800, 10808, 10785, 10791, 10801, 10792, 10786, 10780, + 10795, 10787, 10822, 10783, 10782, 10666, 10663, 10789, 10799, 10798, + 10748, 10709, 10689, 10746, 10682, 10712, 10747, 10715, 10737, 10739, + 10804, 10805, 10810, 10809, 10817, 10819, 10816, 10815, 10812, 10811, + 10814, 10813, 10820, 10818, 10664, 10779, 10678, 10705, 10704, 10675, + 10720, 10719, 10696, 10745, 10693, 10691, 10727, 10688, 10683, 10706, + 3549, 3617, 3620, 3622, 40951, 3573, 3574, 3587, 3588, 3585, 3586, 3567, + 3569, 40951, 40951, 3609, 3575, 40951, 40951, 3610, 3576, 3560, 3557, + 3601, 3600, 3589, 3599, 3598, 3603, 3602, 3591, 3582, 3581, 3578, 3577, + 3590, 3584, 3583, 3580, 3579, 3592, 40951, 3605, 3604, 3597, 3596, 3608, + 3572, 3561, 40951, 3607, 40951, 40951, 40951, 3593, 3594, 3595, 3606, + 40951, 40951, 3618, 3619, 3624, 3633, 3634, 3627, 3628, 3629, 3630, + 40951, 40951, 3635, 3625, 40951, 40951, 3636, 3626, 3621, 3558, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 3550, 40951, 40951, + 40951, 40951, 3565, 3564, 40951, 3571, 3568, 3570, 3631, 3632, 40951, + 40951, 3643, 3645, 3642, 3641, 3638, 3637, 3640, 3639, 3646, 3644, 3563, + 3562, 3612, 3611, 3551, 3555, 3554, 3553, 3552, 3556, 3623, 3647, 3566, + 3548, 3616, 40951, 40951, 18887, 18888, 18893, 40951, 18839, 18840, + 18855, 18856, 18853, 18854, 40951, 40951, 40951, 40951, 18874, 18841, + 40951, 40951, 18873, 18842, 18838, 18837, 18835, 18834, 18859, 18866, + 18865, 18868, 18867, 18861, 18850, 18849, 18844, 18843, 18860, 18852, + 18851, 18846, 18845, 18862, 40951, 18870, 18869, 18864, 18863, 18877, + 18879, 18848, 40951, 18858, 18857, 40951, 18878, 18871, 40951, 18872, + 18876, 40951, 40951, 18890, 40951, 18894, 18899, 18900, 18897, 18898, + 40951, 40951, 40951, 40951, 18902, 18895, 40951, 40951, 18901, 18896, + 18892, 40951, 40951, 40951, 18889, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 18836, 18833, 18880, 18847, 40951, 18875, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 18912, 18914, 18911, 18910, 18907, + 18906, 18909, 18908, 18915, 18913, 18903, 18832, 18904, 18916, 18905, + 18891, 18831, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 18727, 18735, 18737, 40951, 18677, 18678, 18696, 18697, + 18694, 18695, 18689, 18691, 18753, 40951, 18724, 18679, 18754, 40951, + 18725, 18680, 18717, 18716, 18713, 18712, 18701, 18711, 18710, 18715, + 18714, 18703, 18686, 18685, 18682, 18681, 18702, 18688, 18687, 18684, + 18683, 18704, 40951, 18719, 18718, 18709, 18708, 18721, 18723, 18722, + 40951, 18699, 18698, 40951, 18693, 18705, 18706, 18707, 18720, 40951, + 40951, 18733, 18734, 18740, 18749, 18750, 18743, 18744, 18745, 18746, + 18738, 40951, 18751, 18741, 18739, 40951, 18752, 18742, 18736, 40951, + 40951, 18767, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 18690, 18692, 18747, + 18748, 40951, 40951, 18763, 18765, 18762, 18761, 18758, 18757, 18760, + 18759, 18766, 18764, 18755, 18756, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 18700, 18732, 18731, 18728, 18730, 18726, 18729, 40951, + 30620, 30623, 30626, 40951, 30571, 30572, 30590, 30591, 30588, 30589, + 30583, 30585, 40951, 40951, 30616, 30573, 40951, 40951, 30617, 30574, + 30610, 30609, 30606, 30605, 30594, 30604, 30603, 30608, 30607, 30596, + 30580, 30579, 30576, 30575, 30595, 30582, 30581, 30578, 30577, 30597, + 40951, 30612, 30611, 30602, 30601, 30614, 30570, 30568, 40951, 30593, + 30592, 40951, 30587, 30598, 30599, 30600, 30613, 40951, 40951, 30621, + 30622, 30627, 30636, 30637, 30630, 30631, 30632, 30633, 40951, 40951, + 30638, 30628, 40951, 40951, 30639, 30629, 30625, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 30624, 30557, 30558, 40951, 40951, 40951, + 40951, 30567, 30566, 40951, 30569, 30584, 30586, 30634, 30635, 40951, + 40951, 30646, 30648, 30645, 30644, 30641, 30640, 30643, 30642, 30649, + 30647, 30565, 30615, 30562, 30561, 30564, 30559, 30560, 30563, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35480, + 35497, 40951, 35443, 35444, 35456, 35457, 35452, 35453, 40951, 40951, + 40951, 35461, 35462, 35445, 40951, 35454, 35455, 35446, 35466, 40951, + 40951, 40951, 35438, 35463, 40951, 35465, 40951, 35439, 35441, 40951, + 40951, 40951, 35437, 35442, 40951, 40951, 40951, 35440, 35436, 35468, + 40951, 40951, 40951, 35467, 35470, 35451, 35450, 35449, 35448, 35447, + 35469, 35458, 35459, 35460, 35464, 40951, 40951, 40951, 40951, 35770, + 35777, 35778, 35773, 35774, 40951, 40951, 40951, 35779, 35780, 35771, + 40951, 35775, 35776, 35772, 35496, 40951, 40951, 35783, 40951, 40951, + 40951, 40951, 40951, 40951, 35372, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35409, + 35411, 35408, 35407, 35404, 35403, 35406, 35405, 35412, 35410, 35474, + 35472, 35473, 35402, 35782, 35781, 35401, 35399, 35371, 35477, 35475, + 40951, 40951, 40951, 40951, 40951, 36730, 36731, 36736, 36738, 36729, + 36690, 36691, 36706, 36707, 36702, 36703, 36697, 36699, 40951, 36723, + 36724, 36692, 40951, 36704, 36705, 36693, 36720, 36719, 36716, 36715, + 36679, 36714, 36713, 36718, 36717, 36681, 36686, 36685, 36673, 36672, + 36680, 36689, 36687, 36676, 36674, 36677, 40951, 36722, 36721, 36712, + 36711, 36726, 36727, 36684, 36683, 36696, 36695, 36694, 36701, 36708, + 36709, 36710, 36725, 40951, 40951, 36734, 36735, 36739, 36750, 36751, + 36742, 36743, 36744, 36745, 40951, 36752, 36753, 36740, 40951, 36748, + 36749, 36741, 36737, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 36728, 36664, 40951, 36688, 36675, 36682, 40951, 40951, 36678, 40951, + 40951, 36698, 36700, 36746, 36747, 40951, 40951, 36760, 36762, 36759, + 36758, 36755, 36754, 36757, 36756, 36763, 36761, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 36732, 36671, 36669, 36667, 36665, 36670, + 36668, 36666, 36733, 21274, 21273, 21278, 21282, 21275, 21222, 21223, + 21248, 21249, 21244, 21245, 21239, 21241, 40951, 21265, 21266, 21224, + 40951, 21246, 21247, 21225, 21262, 21261, 21258, 21257, 21219, 21256, + 21255, 21260, 21259, 21221, 21236, 21235, 21227, 21226, 21220, 21238, + 21237, 21229, 21228, 21217, 40951, 21264, 21263, 21254, 21253, 21269, + 21270, 21234, 21233, 21232, 21231, 40951, 21243, 21250, 21251, 21252, + 21268, 40951, 40951, 21276, 21277, 21284, 21295, 21296, 21287, 21288, + 21289, 21290, 40951, 21297, 21298, 21285, 40951, 21293, 21294, 21286, + 21281, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 21271, 21283, + 40951, 40951, 40951, 40951, 40951, 40951, 21218, 21267, 40951, 21240, + 21242, 21291, 21292, 40951, 40951, 21305, 21307, 21304, 21303, 21300, + 21299, 21302, 21301, 21308, 21306, 40951, 21279, 21280, 21272, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 25543, 25545, 25550, 25548, 25500, 25474, 25476, 25520, 25521, + 25516, 25517, 25501, 25503, 40951, 25535, 25536, 25477, 40951, 25518, + 25519, 25478, 25532, 25531, 25528, 25527, 25508, 25489, 25488, 25530, + 25529, 25509, 25497, 25495, 25492, 25491, 25507, 25499, 25498, 25494, + 25493, 25510, 25506, 25534, 25533, 25526, 25525, 25538, 25539, 25515, + 25514, 25513, 25512, 25511, 25505, 25522, 25523, 25524, 25537, 25496, + 25546, 25544, 25549, 25552, 25563, 25564, 25555, 25556, 25557, 25558, + 40951, 25565, 25566, 25553, 40951, 25561, 25562, 25554, 25547, 25490, + 25551, 40951, 40951, 40951, 40951, 25486, 25487, 25481, 25567, 25466, + 25464, 25471, 25461, 25462, 25472, 25465, 25475, 25502, 25504, 25559, + 25560, 40951, 40951, 25457, 25459, 25456, 25455, 25452, 25451, 25454, + 25453, 25460, 25458, 25542, 25540, 25541, 25469, 25468, 25473, 25463, + 25467, 25470, 25450, 25483, 25482, 25484, 25479, 25480, 25485, 40951, + 33736, 33735, 33737, 40951, 33681, 33678, 33666, 33665, 33689, 33688, + 33691, 33690, 33687, 33686, 33685, 33684, 33683, 33682, 33679, 33710, + 33709, 33680, 40951, 40951, 40951, 33675, 33700, 33673, 33698, 33723, + 33713, 33672, 33697, 33674, 33699, 33720, 33721, 33714, 33667, 33692, + 33669, 33694, 33704, 33711, 33668, 33693, 33670, 33695, 33707, 40951, + 33712, 33676, 33701, 33671, 33696, 33702, 33677, 33719, 33717, 40951, + 33706, 40951, 40951, 33718, 33722, 33705, 33708, 33716, 33703, 33715, + 40951, 40951, 40951, 33734, 40951, 40951, 40951, 40951, 33754, 33746, + 33741, 33747, 33742, 33748, 40951, 33743, 40951, 33744, 33749, 33740, + 33753, 33750, 33751, 33752, 33745, 40951, 40951, 40951, 40951, 40951, + 40951, 33730, 33732, 33729, 33728, 33725, 33724, 33727, 33726, 33733, + 33731, 40951, 40951, 33738, 33739, 33755, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 36917, 36914, + 36912, 36911, 36913, 36915, 36931, 36900, 36902, 36901, 36960, 36903, + 36972, 36904, 36969, 36965, 36962, 36963, 36933, 36905, 36968, 36967, + 36964, 36966, 36934, 36899, 36940, 36937, 36906, 36938, 36907, 36939, + 36929, 36973, 36941, 36942, 36919, 36921, 36970, 36958, 36957, 36959, + 36910, 36918, 36974, 36909, 36935, 36943, 36923, 36946, 36948, 36953, + 36954, 36950, 36951, 36949, 36952, 36936, 40951, 40951, 40951, 40951, + 36975, 36955, 36947, 36956, 36945, 36944, 36920, 36928, 36927, 36926, + 36924, 36925, 36922, 36961, 36932, 36971, 36908, 36982, 36984, 36981, + 36980, 36977, 36976, 36979, 36978, 36985, 36983, 36930, 36916, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 22578, 22576, 40951, 22577, + 40951, 22591, 22606, 22610, 22590, 22600, 40951, 22592, 22607, 22588, + 22586, 22585, 22583, 22582, 22587, 22611, 22603, 22601, 22602, 22584, + 22608, 22609, 22596, 22594, 22573, 22595, 22572, 22589, 22612, 22615, + 22580, 40951, 22581, 40951, 22614, 22597, 22598, 22599, 22604, 22593, + 22616, 22605, 22642, 22624, 22629, 22625, 22627, 22637, 22638, 22631, + 22632, 22633, 22634, 22619, 22630, 22618, 22617, 40951, 40951, 22635, + 22636, 22639, 22628, 22626, 40951, 22640, 40951, 22623, 22620, 22621, + 22622, 22653, 22654, 22641, 40951, 22649, 22651, 22648, 22647, 22644, + 22643, 22646, 22645, 22652, 22650, 40951, 40951, 22569, 22568, 22575, + 22574, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 37208, 37115, 37113, 37114, 37123, 37111, 37108, + 37112, 37132, 37103, 37101, 37124, 37139, 37133, 37129, 37136, 37128, + 37131, 37130, 37107, 37116, 37099, 37098, 37026, 37027, 37025, 37147, + 37148, 37149, 37151, 37152, 37150, 37048, 37050, 37047, 37046, 37043, + 37042, 37045, 37044, 37051, 37049, 37040, 37037, 37036, 37033, 37032, + 37035, 37034, 37041, 37039, 37038, 37104, 37126, 37106, 37125, 37109, + 37135, 37117, 37118, 37119, 37120, 37158, 37144, 37057, 37055, 37085, + 37084, 37067, 37083, 37082, 37092, 40951, 37069, 37077, 37076, 37062, + 37061, 37068, 37079, 37078, 37066, 37065, 37070, 37087, 37086, 37081, + 37080, 37094, 37075, 37074, 37064, 37063, 37095, 37088, 37089, 37090, + 37096, 37059, 37093, 37071, 37072, 37073, 37091, 37097, 37054, 37060, + 37056, 37058, 40951, 40951, 40951, 40951, 37232, 37228, 37229, 37220, + 37221, 37222, 37223, 37224, 37225, 37230, 37231, 37226, 37227, 37155, + 37156, 37218, 37219, 37146, 37160, 37121, 37138, 37142, 37157, 37143, + 37145, 37140, 37141, 37159, 37207, 37206, 37205, 37172, 37171, 37191, + 37190, 37173, 37189, 37188, 37198, 40951, 37175, 37183, 37182, 37165, + 37164, 37174, 37185, 37184, 37169, 37168, 37176, 37193, 37192, 37187, + 37186, 37200, 37181, 37180, 37167, 37166, 37202, 37194, 37195, 37196, + 37203, 37201, 37199, 37177, 37178, 37179, 37197, 37204, 37170, 37162, + 37163, 37161, 40951, 37052, 37053, 37028, 37029, 37031, 37030, 37209, + 37216, 37214, 37217, 37215, 37210, 37213, 37212, 37211, 40951, 37154, + 37153, 37102, 37105, 37127, 37122, 37110, 32022, 24181, 32023, 24182, + 37137, 37134, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 28582, + 28561, 28560, 28559, 28591, 28659, 28658, 28661, 28660, 28592, 28589, + 28637, 28636, 28643, 28642, 28590, 28621, 28638, 28645, 28644, 28593, + 28663, 28662, 28657, 28656, 28588, 28665, 28595, 28655, 28641, 28620, + 28664, 28654, 28551, 28615, 28652, 28653, 28646, 28647, 28554, 28587, + 28666, 28553, 28758, 28742, 28765, 28766, 28759, 28760, 28754, 28739, + 28747, 28748, 28755, 28683, 28702, 28707, 28706, 28682, 28546, 28544, + 28545, 28543, 28558, 28783, 28785, 28782, 28781, 28778, 28777, 28780, + 28779, 28786, 28784, 28705, 28694, 28712, 28716, 28711, 28715, 28596, + 28619, 28648, 28649, 28650, 28651, 28761, 28762, 28763, 28764, 28586, + 28585, 28583, 28584, 28549, 28548, 28547, 28618, 28753, 28727, 28728, + 28640, 28639, 28756, 28757, 28697, 28698, 28699, 28700, 28701, 28557, + 28556, 28555, 28743, 28745, 28746, 28744, 28610, 28609, 28608, 28606, + 28614, 28598, 28611, 28599, 28601, 28612, 28604, 28602, 28613, 28550, + 28752, 28749, 28750, 28751, 28689, 28690, 28691, 28692, 28687, 28688, + 28686, 28594, 28703, 28678, 28680, 28677, 28676, 28673, 28672, 28675, + 28674, 28681, 28679, 28684, 28685, 28740, 28741, 28714, 28713, 17728, + 17754, 17739, 17760, 17761, 17759, 17749, 17750, 17762, 17743, 17752, + 17755, 17757, 17763, 17745, 17748, 17753, 17747, 17751, 17764, 17744, + 17742, 17738, 17758, 17746, 17734, 17737, 17741, 17736, 17735, 17756, + 17740, 17729, 17733, 17731, 17766, 17730, 17732, 40951, 17765, 40951, + 40951, 40951, 40951, 40951, 17727, 40951, 40951, 17811, 17842, 17819, + 17848, 17817, 17847, 17840, 17837, 17849, 17829, 17831, 17843, 17845, + 17850, 17833, 17839, 17841, 17835, 17838, 17808, 17832, 17828, 17818, + 17846, 17834, 17812, 17815, 17827, 17814, 17813, 17844, 17826, 17822, + 17825, 17823, 17852, 17820, 17824, 17853, 17851, 17816, 17836, 17810, + 17900, 27795, 17809, 17821, 17830, 19147, 19221, 19155, 19226, 19219, + 19183, 19150, 19165, 19223, 19197, 19215, 19129, 19127, 19213, 19114, + 19149, 19233, 19162, 19236, 19157, 19222, 19159, 19158, 19230, 19193, + 19217, 19196, 19142, 19152, 19146, 19175, 19180, 19179, 19166, 19170, + 19168, 19171, 19172, 19169, 19177, 19176, 19178, 19173, 19144, 19145, + 19204, 19210, 19209, 19202, 19207, 19198, 19199, 19201, 19212, 19206, + 19205, 19203, 19208, 19200, 19211, 19119, 19118, 19124, 19123, 19182, + 19139, 19138, 19136, 19131, 19141, 19132, 19225, 19135, 19134, 19137, + 19130, 19234, 19128, 19121, 19117, 19126, 19122, 19115, 19116, 19120, + 19125, 19163, 19143, 19224, 19235, 19148, 19161, 19156, 19160, 19227, + 19238, 19476, 19382, 19392, 19440, 19444, 19394, 19393, 19446, 19445, + 19421, 19473, 19474, 19431, 19452, 19432, 19472, 19471, 19475, 19461, + 19398, 19450, 19405, 19390, 19391, 19442, 19441, 19396, 19397, 19395, + 19448, 19449, 19429, 19428, 19424, 19422, 19430, 19453, 19454, 19455, + 19460, 19459, 19437, 19438, 19436, 19433, 19439, 19466, 19465, 19464, + 19463, 19462, 19470, 19468, 19404, 19401, 19451, 19406, 19408, 19415, + 19419, 19417, 19407, 19383, 19385, 19388, 19387, 19420, 19389, 19443, + 19447, 19426, 19427, 19258, 19359, 19261, 19281, 19284, 19288, 19363, + 19309, 19310, 19315, 19319, 19328, 19331, 19324, 19336, 19268, 19298, + 19301, 19337, 19354, 19250, 19241, 19244, 19267, 19372, 19294, 19245, + 19260, 19262, 19287, 19286, 19290, 19289, 19285, 19370, 19364, 19312, + 19335, 19329, 19330, 19349, 19316, 19318, 19323, 19322, 19313, 19327, + 19325, 19314, 19332, 19278, 19275, 19269, 19274, 19273, 19271, 19276, + 19280, 19257, 19299, 19303, 19308, 19256, 19339, 19347, 19340, 19343, + 19291, 19252, 19253, 19362, 19251, 19373, 19377, 19380, 19296, 19255, + 19248, 19246, 19247, 19249, 19381, 19264, 19265, 19263, 19259, 19266, + 19360, 17020, 17027, 17026, 17021, 17025, 17024, 17022, 17023, 17247, + 17255, 17254, 17248, 17252, 17251, 17249, 17253, 17013, 17019, 17017, + 17014, 17016, 17015, 17018, 17004, 17064, 17072, 17071, 17065, 17069, + 17068, 17066, 17062, 17176, 17183, 17181, 17177, 17179, 17178, 17182, + 17180, 17151, 17160, 17159, 17152, 17156, 17155, 17153, 17157, 17191, + 17197, 17196, 17192, 17166, 17161, 17193, 17195, 17167, 17175, 17174, + 17168, 17172, 17171, 17169, 17173, 17143, 17150, 17149, 17144, 17148, + 17147, 17145, 17146, 17131, 40951, 17135, 17132, 17134, 17133, 40951, + 40951, 17124, 17130, 17128, 17125, 17127, 17126, 17129, 40951, 17119, + 40951, 17123, 17120, 17122, 17121, 40951, 40951, 16854, 16861, 16860, + 16855, 16859, 16858, 16856, 16845, 17316, 17323, 17321, 17317, 17319, + 17318, 17322, 17320, 17229, 17237, 17236, 17230, 17234, 17233, 17231, + 17235, 16892, 16900, 16899, 16893, 16897, 16896, 16894, 16898, 17284, + 17291, 17290, 17285, 17289, 17288, 17286, 17287, 17272, 40951, 17276, + 17273, 17275, 17274, 40951, 40951, 17082, 17090, 17089, 17083, 17087, + 17086, 17084, 17088, 17073, 17081, 17080, 17074, 17078, 17077, 17075, + 17079, 16974, 16982, 16981, 16975, 16979, 16978, 16976, 16980, 17052, + 17059, 17058, 17053, 17057, 17056, 17054, 17055, 17040, 40951, 17044, + 17041, 17043, 17042, 40951, 40951, 17033, 17039, 17037, 17034, 17036, + 17035, 17038, 40951, 17028, 40951, 17032, 17029, 17031, 17030, 40951, + 40951, 17256, 17263, 17262, 17257, 17261, 17260, 17258, 17259, 17092, + 17098, 17096, 17093, 17095, 17094, 17097, 40951, 17307, 17315, 17314, + 17308, 17312, 17311, 17309, 17313, 17292, 17299, 17297, 17293, 17295, + 17294, 17298, 17296, 17264, 17271, 17270, 17265, 17269, 17268, 17266, + 17267, 16922, 16930, 16929, 16923, 16927, 16926, 16924, 16928, 16907, + 16915, 16914, 16908, 16912, 16911, 16909, 16913, 17238, 17246, 17245, + 17239, 17243, 17242, 17240, 17244, 16995, 16943, 17001, 16996, 17000, + 16999, 16997, 16998, 16983, 40951, 16987, 16984, 16986, 16985, 40951, + 40951, 16967, 16973, 16971, 16968, 16970, 16969, 16972, 16963, 17198, + 17206, 17205, 17199, 17203, 17202, 17200, 17204, 16883, 16891, 16890, + 16884, 16888, 16887, 16885, 16889, 17091, 17106, 17105, 17099, 17103, + 17102, 17100, 17104, 17221, 17228, 17226, 17222, 17224, 17223, 17227, + 17225, 17213, 17220, 17219, 17214, 17218, 17217, 17215, 17216, 16935, + 16942, 16940, 16936, 16938, 16937, 16941, 16933, 17111, 17118, 17117, + 17112, 17116, 17115, 17113, 17109, 17158, 17070, 16939, 40951, 40951, + 16823, 16825, 16824, 16842, 17345, 17343, 16826, 16841, 16827, 16840, + 17344, 16839, 17341, 17339, 17338, 17335, 17334, 17337, 17336, 17342, + 17340, 16828, 16831, 16830, 16835, 16834, 16838, 16837, 16833, 16836, + 16832, 16829, 40951, 40951, 40951, 17164, 17063, 17061, 17060, 17162, + 16846, 16844, 16843, 17163, 16934, 16932, 16931, 17165, 17110, 17108, + 17107, 17333, 17324, 17330, 17331, 17326, 17328, 17332, 17327, 17325, + 17329, 40951, 40951, 40951, 40951, 40951, 40951, 6382, 6383, 6384, 6385, + 6386, 6387, 6345, 6381, 6346, 6347, 6348, 6349, 6350, 6310, 6311, 6312, + 6313, 6314, 6315, 6351, 6352, 6353, 6354, 6355, 6356, 6357, 6358, 6359, + 6360, 6361, 6316, 6309, 6317, 6318, 6319, 6320, 6321, 6322, 6363, 6364, + 6365, 6366, 6367, 6368, 6324, 6323, 6325, 6326, 6327, 6328, 6329, 6303, + 6342, 6304, 6343, 6305, 6344, 6306, 6307, 6308, 6302, 6330, 6331, 6332, + 6333, 6334, 6335, 6336, 6337, 6338, 6339, 6340, 6341, 6369, 6370, 6371, + 6372, 6373, 6374, 6375, 6376, 6377, 6378, 6379, 6380, 6362, 40951, 40951, + 6462, 6463, 6464, 6465, 6466, 6448, 40951, 40951, 5524, 5496, 5269, 5526, + 5527, 5676, 5690, 5973, 5480, 5340, 5267, 5268, 5850, 5940, 5960, 5938, + 5961, 5939, 5942, 5936, 5943, 5937, 5605, 5957, 5934, 5958, 5935, 5606, + 5271, 5974, 5992, 5513, 5512, 5514, 5515, 5507, 5508, 5505, 5504, 5517, + 5511, 5516, 5506, 5498, 5528, 5689, 5275, 5710, 5703, 5708, 5709, 5705, + 5706, 5964, 5338, 5339, 5701, 5702, 5700, 5879, 5698, 5877, 5699, 5878, + 5693, 5875, 5694, 5876, 5696, 5873, 5697, 5874, 5963, 5692, 5872, 5331, + 5849, 5832, 5847, 5848, 5845, 5846, 5971, 5302, 5315, 5830, 5831, 5840, + 5932, 5838, 5930, 5839, 5931, 5836, 5928, 5837, 5929, 5834, 5926, 5835, + 5927, 5611, 5792, 5827, 5828, 5829, 5826, 5547, 5541, 5545, 5546, 5543, + 5544, 5966, 5539, 5540, 5538, 5924, 5536, 5922, 5537, 5923, 5534, 5920, + 5535, 5921, 5531, 5918, 5532, 5919, 5608, 5529, 5530, 5772, 5773, 5774, + 5771, 5495, 5482, 5493, 5494, 5491, 5492, 5965, 5299, 5481, 5489, 5917, + 5487, 5915, 5488, 5916, 5485, 5913, 5486, 5914, 5483, 5911, 5484, 5912, + 5607, 5298, 5748, 5573, 5581, 5590, 5591, 5576, 5577, 5968, 5579, 5580, + 5589, 5871, 5587, 5869, 5588, 5870, 5585, 5867, 5586, 5868, 5583, 5865, + 5584, 5866, 5609, 5572, 5864, 5592, 5273, 5749, 5665, 5626, 5663, 5664, + 5653, 5654, 5969, 5597, 5625, 5662, 5890, 5656, 5888, 5657, 5889, 5610, + 5593, 5371, 5666, 5571, 5558, 5569, 5570, 5567, 5568, 5967, 5556, 5557, + 5566, 5858, 5564, 5856, 5565, 5857, 5562, 5854, 5563, 5855, 5560, 5852, + 5561, 5853, 5548, 5851, 5574, 5791, 5751, 5789, 5790, 5770, 5775, 5970, + 5731, 5750, 5784, 5910, 5782, 5908, 5783, 5909, 5780, 5906, 5781, 5907, + 5778, 5904, 5779, 5905, 5603, 5730, 5274, 5777, 5294, 5578, 5598, 5604, + 5601, 5602, 5599, 5600, 5769, 5767, 5768, 5761, 5762, 5764, 5765, 5760, + 5903, 5758, 5901, 5759, 5902, 5753, 5899, 5754, 5900, 5756, 5897, 5757, + 5898, 5752, 5991, 5977, 5989, 5990, 5979, 5980, 5972, 5975, 5976, 5988, + 5887, 5986, 5885, 5987, 5886, 5984, 5883, 5985, 5884, 5982, 5881, 5983, + 5882, 5612, 5962, 5295, 5880, 5747, 5729, 5713, 5863, 5723, 5727, 5728, + 5725, 5726, 5861, 5721, 5722, 5859, 5715, 5892, 5711, 5891, 5575, 5523, + 5502, 5503, 5518, 5520, 5521, 5500, 5501, 5522, 5933, 5499, 5811, 5596, + 5809, 5594, 5810, 5595, 5807, 5808, 5805, 5806, 5803, 5925, 5793, 5824, + 5825, 5821, 5819, 5818, 5842, 5843, 5844, 5841, 5651, 5649, 5650, 5647, + 5648, 5645, 5646, 5644, 5652, 5525, 5670, 5674, 5675, 5672, 5673, 5668, + 5669, 5667, 5816, 5817, 5812, 5815, 5894, 5895, 5896, 5893, 5631, 5635, + 5636, 5633, 5634, 5629, 5630, 5628, 5637, 5745, 5746, 5741, 5744, 5953, + 5954, 5955, 5952, 5944, 5554, 5555, 5552, 5553, 5550, 5551, 5549, 5801, + 5799, 5800, 5797, 5798, 5795, 5796, 5794, 5272, 5291, 5292, 5293, 5290, + 5279, 5280, 5281, 5278, 5287, 5288, 5289, 5286, 5283, 5284, 5285, 5282, + 5736, 5737, 5733, 5735, 5321, 5320, 5316, 5317, 5319, 5318, 5467, 5466, + 5462, 5463, 5465, 5464, 5473, 5472, 5468, 5469, 5471, 5470, 5337, 5336, + 5332, 5333, 5335, 5334, 5431, 5430, 5426, 5427, 5429, 5428, 5425, 5424, + 5420, 5421, 5423, 5422, 5394, 5393, 5389, 5390, 5392, 5391, 5388, 5330, + 5329, 5326, 5327, 5328, 5324, 5367, 5366, 5362, 5363, 5365, 5364, 5361, + 5360, 5356, 5357, 5359, 5358, 5355, 5374, 5373, 5368, 5369, 5372, 5370, + 5461, 5460, 5456, 5457, 5459, 5458, 5479, 5478, 5474, 5475, 5477, 5476, + 5354, 5738, 5353, 5348, 5349, 5352, 5740, 5351, 5347, 5346, 5342, 5343, + 5345, 5344, 5449, 5448, 5444, 5445, 5447, 5446, 5308, 5307, 5303, 5304, + 5306, 5305, 5443, 5442, 5438, 5439, 5441, 5440, 5407, 5406, 5402, 5403, + 5405, 5404, 5413, 5412, 5408, 5409, 5411, 5410, 5401, 5400, 5396, 5397, + 5399, 5398, 5395, 5341, 5314, 5313, 5309, 5310, 5312, 5311, 5387, 5386, + 5382, 5383, 5385, 5384, 5381, 5380, 5376, 5377, 5379, 5378, 5375, 5437, + 5436, 5432, 5433, 5435, 5434, 5455, 5454, 5450, 5451, 5453, 5452, 5419, + 5418, 5414, 5415, 5417, 5416, 5490, 5519, 5671, 5632, 5642, 5643, 5640, + 5641, 5638, 5639, 5951, 5949, 5950, 5947, 5948, 5945, 5946, 5956, 5277, + 29977, 29952, 29963, 29959, 29969, 29966, 29972, 29975, 29976, 29955, + 29954, 29974, 29960, 29965, 29970, 29964, 29951, 29967, 29973, 29957, + 29961, 29956, 29968, 29971, 29962, 29958, 29953, 29949, 29950, 40951, + 40951, 40951, 32296, 32352, 32346, 32350, 32349, 32344, 32342, 32289, + 32274, 32320, 32273, 32272, 32317, 32332, 32319, 32323, 32324, 32326, + 32312, 32278, 32311, 32297, 32290, 32298, 32300, 32345, 32302, 32301, + 32315, 32329, 32341, 32331, 32283, 32305, 32286, 32309, 32299, 32314, + 32335, 32306, 32348, 32275, 32338, 32337, 32333, 32276, 32354, 32343, + 32334, 32281, 32340, 32328, 32284, 32322, 32287, 32347, 32316, 32330, + 32313, 32282, 32304, 32303, 32285, 32321, 32288, 32308, 32280, 32279, + 32277, 32339, 32318, 32336, 32307, 32351, 32353, 32355, 32356, 32271, + 32357, 32358, 32270, 32310, 32327, 32325, 32294, 32293, 32295, 32292, + 32291, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35025, 35042, + 35043, 35033, 35031, 35027, 35039, 35030, 35028, 35036, 35029, 35035, + 35041, 35037, 35034, 35040, 35038, 35032, 35046, 35047, 35045, 35044, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35026, + 19648, 19649, 19650, 19639, 19637, 19633, 19645, 19636, 19634, 19642, + 19635, 19641, 19647, 19643, 19640, 19646, 19644, 19638, 19652, 19653, + 19651, 31427, 31428, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 4990, 4991, 4992, 4981, 4979, 4975, 4987, 4978, 4976, 4984, + 4977, 4983, 4989, 4985, 4982, 4988, 4986, 4980, 4993, 4994, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 35061, 35062, 35063, 35053, 35052, 35048, 35058, 35051, 35049, 35056, + 35050, 35055, 35060, 40951, 35054, 35059, 35057, 40951, 35064, 35065, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 22231, 22229, 22232, 22230, 22233, 22227, 22225, 22228, + 22226, 22235, 22249, 22245, 22250, 22246, 22234, 22247, 22243, 22248, + 22244, 22236, 22257, 22237, 22239, 22238, 22253, 22256, 22254, 22252, + 22255, 22241, 22240, 22242, 22258, 22251, 22259, 22206, 22204, 22214, + 22215, 22210, 22213, 22211, 22212, 22223, 22224, 22221, 22222, 22216, + 22205, 22209, 22208, 22207, 22326, 22325, 22327, 22332, 22334, 22338, + 22340, 22342, 22344, 22343, 22335, 22339, 22333, 22345, 22329, 22330, + 22337, 22331, 22280, 22274, 22279, 22281, 22276, 22265, 22273, 22275, + 22270, 22262, 22263, 22282, 22269, 22264, 22271, 22266, 22268, 22277, + 22267, 22278, 22272, 22203, 22260, 22261, 40951, 40951, 22352, 22354, + 22351, 22350, 22347, 22346, 22349, 22348, 22355, 22353, 40951, 40951, + 40951, 40951, 40951, 40951, 22304, 22303, 22300, 22302, 22301, 22295, + 22298, 22299, 22297, 22296, 40951, 40951, 40951, 40951, 40951, 40951, + 28162, 28174, 28170, 28019, 28169, 28020, 28167, 28158, 28171, 28172, + 28173, 28018, 28017, 28016, 28168, 28015, 28011, 28013, 28010, 28009, + 28006, 28005, 28008, 28007, 28014, 28012, 40951, 40951, 40951, 40951, + 40951, 40951, 28023, 28137, 28154, 28139, 28141, 28140, 28142, 28138, + 28148, 28051, 28143, 28149, 28150, 28146, 28055, 28136, 28098, 28097, + 28128, 28144, 28052, 28147, 28153, 28151, 28152, 28145, 28134, 28133, + 28127, 28131, 28132, 28130, 28135, 28129, 28054, 28107, 28125, 28126, + 28114, 28116, 28115, 28117, 28101, 28118, 28121, 28122, 28108, 28120, + 28111, 28103, 28112, 28105, 28110, 28124, 28123, 28119, 28109, 28113, + 28104, 28106, 28102, 28096, 28079, 28080, 28090, 28089, 28086, 28094, + 28075, 28077, 28095, 28084, 28082, 28091, 28093, 28092, 28076, 28078, + 28081, 28088, 28085, 28083, 28087, 28074, 28072, 28073, 28071, 28070, + 28053, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 28025, 28027, + 28029, 28036, 28035, 28043, 28039, 28024, 28034, 28050, 28037, 28049, + 28041, 28040, 28031, 28038, 28042, 28028, 28045, 28044, 28048, 28046, + 28047, 28026, 28100, 28099, 28063, 28068, 28059, 28064, 28060, 28056, + 28061, 28057, 28069, 28058, 28066, 28067, 28033, 28032, 28062, 28030, + 28065, 40951, 40951, 40951, 40951, 40951, 5691, 5276, 5270, 5959, 5707, + 5704, 5695, 5833, 5542, 5533, 5582, 5655, 5627, 5559, 5776, 5732, 5763, + 5766, 5755, 5981, 5978, 5724, 5660, 5680, 5661, 5681, 5658, 5678, 5659, + 5679, 5720, 5718, 5719, 5716, 5717, 5714, 5687, 5688, 5685, 5684, 5686, + 5677, 5682, 5683, 5497, 5941, 5510, 5509, 5712, 5862, 5860, 5804, 5802, + 5823, 5822, 5820, 5814, 5813, 5743, 5742, 5734, 5323, 5300, 5325, 5322, + 5739, 5350, 5296, 5297, 5301, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 24478, 24445, 24444, 24425, 24424, 24431, + 24439, 24438, 24443, 24442, 24430, 24428, 24426, 24441, 24440, 24432, + 24447, 24446, 24437, 24436, 24450, 24429, 24451, 24449, 24452, 24433, + 24434, 24435, 24448, 24423, 24427, 40951, 24469, 24476, 24477, 24475, + 24470, 24473, 24471, 24474, 24472, 24468, 24466, 24467, 40951, 40951, + 40951, 40951, 24460, 24457, 24459, 24465, 24458, 24463, 24462, 24464, + 24461, 24454, 24453, 24455, 40951, 40951, 40951, 40951, 24456, 40951, + 40951, 40951, 24479, 24480, 24487, 24489, 24486, 24485, 24482, 24481, + 24484, 24483, 24490, 24488, 35086, 35098, 35081, 35078, 35096, 35099, + 35080, 35079, 35093, 35088, 35087, 35094, 35091, 35097, 35092, 35095, + 35085, 35077, 35082, 35066, 35100, 35070, 35071, 35089, 35084, 35083, + 35090, 35069, 35067, 35068, 40951, 40951, 35072, 35073, 35074, 35075, + 35076, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 29120, 29142, 29102, 29104, 29107, 29124, 29126, 29129, + 29110, 29106, 29122, 29132, 29128, 29144, 29111, 29109, 29108, 29133, + 29131, 29130, 29113, 29112, 29119, 29135, 29134, 29141, 29116, 29121, + 29118, 29138, 29143, 29140, 29117, 29115, 29114, 29139, 29137, 29136, + 29101, 29103, 29123, 29125, 29105, 29127, 40951, 40951, 40951, 40951, + 29165, 29150, 29154, 29160, 29163, 29166, 29152, 29156, 29157, 29161, + 29153, 29151, 29164, 29159, 29158, 29162, 29155, 29100, 29095, 29094, + 29099, 29098, 29097, 29096, 29147, 29148, 40951, 40951, 40951, 40951, + 40951, 40951, 29173, 29175, 29172, 29171, 29168, 29167, 29170, 29169, + 29176, 29174, 29149, 40951, 40951, 40951, 29145, 29146, 22305, 22324, + 22317, 22320, 22322, 22315, 22313, 22311, 22307, 22309, 22294, 22292, + 22284, 22288, 22290, 22286, 22318, 22323, 22316, 22319, 22321, 22314, + 22312, 22310, 22306, 22308, 22293, 22291, 22283, 22287, 22289, 22285, + 4959, 4956, 4948, 4947, 4961, 4953, 4946, 4945, 4964, 4955, 4952, 4951, + 4954, 4958, 4950, 4949, 4966, 4962, 4960, 4965, 4963, 4967, 4957, 4970, + 4972, 4969, 4971, 4968, 40951, 40951, 4974, 4973, 35134, 35132, 35133, + 35150, 35149, 35148, 35174, 35140, 35139, 35153, 35160, 35152, 35175, + 35168, 35135, 35180, 35151, 35167, 35144, 35143, 35157, 35156, 35176, + 35179, 35142, 35141, 35145, 35155, 35158, 35154, 35181, 35161, 35147, + 35166, 35169, 35162, 35164, 35182, 35136, 35137, 35138, 35146, 35165, + 35183, 35159, 35172, 35173, 35170, 35171, 35178, 35177, 35163, 35131, + 35106, 35105, 35103, 35194, 35101, 35102, 35104, 35107, 35108, 35109, + 40951, 35202, 35209, 35213, 35210, 35219, 35225, 35226, 35224, 35223, + 35221, 35222, 35214, 35215, 35218, 35227, 35211, 35217, 35212, 35220, + 35216, 35193, 35206, 35207, 35186, 35187, 35188, 35197, 35196, 35189, + 40951, 40951, 35110, 35117, 35119, 35116, 35115, 35112, 35111, 35114, + 35113, 35120, 35118, 40951, 40951, 40951, 40951, 40951, 40951, 35127, + 35129, 35126, 35125, 35122, 35121, 35124, 35123, 35130, 35128, 40951, + 40951, 40951, 40951, 40951, 40951, 35203, 35204, 35201, 35192, 35185, + 35205, 35198, 35195, 35190, 35191, 35199, 35200, 35184, 35208, 40951, + 40951, 8201, 8169, 8285, 8203, 8435, 8448, 8447, 8387, 8184, 8364, 8423, + 8393, 8188, 8392, 8391, 8333, 8327, 8350, 8409, 8351, 8410, 8421, 8380, + 8284, 8396, 8187, 8186, 8433, 8311, 8312, 8313, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 2717, 2716, 2718, 2715, 2719, + 2626, 2627, 2643, 2644, 2647, 2648, 2665, 2666, 2656, 2657, 2640, 2630, + 2645, 2646, 2651, 2653, 2641, 2642, 2660, 2633, 2634, 2649, 2650, 2661, + 2672, 2671, 2637, 2636, 2659, 2670, 2673, 2635, 2638, 2658, 2662, 2663, + 2631, 2632, 2678, 2680, 2664, 2655, 2679, 2668, 2669, 2667, 2677, 2720, + 2731, 2734, 2735, 2725, 2726, 2723, 2724, 2721, 2722, 2727, 2728, 2730, + 2729, 2732, 2733, 2736, 2652, 2654, 2674, 2639, 2675, 2676, 2628, 2629, + 40951, 2625, 2624, 2744, 2746, 2743, 2742, 2739, 2738, 2741, 2740, 2747, + 2745, 2712, 2709, 2737, 2622, 2623, 2621, 2711, 2698, 2696, 2699, 2690, + 2691, 2697, 2693, 2695, 2694, 2692, 2688, 2687, 2683, 2681, 2685, 2684, + 2682, 2686, 2689, 2708, 2707, 2706, 2705, 2700, 2702, 2703, 2704, 2701, + 2713, 2710, 2714, 34633, 34631, 34632, 34584, 34619, 34621, 34586, 34620, + 34596, 34597, 34604, 34612, 34607, 34598, 34605, 34609, 34618, 34599, + 34613, 34606, 34600, 34611, 34589, 34614, 34602, 34610, 34617, 34593, + 34591, 34615, 34595, 34616, 34608, 34579, 34580, 34581, 34639, 34638, + 34640, 34637, 34635, 34636, 34630, 34634, 34582, 34583, 34603, 34594, + 34647, 34649, 34646, 34645, 34642, 34641, 34644, 34643, 34650, 34648, + 34578, 34592, 34590, 34601, 34587, 34588, 3508, 3494, 3502, 3486, 3474, + 3498, 3497, 3483, 3489, 3482, 3475, 3506, 3492, 3484, 3501, 3485, 3503, + 3500, 3505, 3490, 3473, 3488, 3495, 3478, 3496, 3491, 3476, 3507, 3493, + 3480, 3504, 3487, 3481, 3499, 3479, 3477, 3509, 3510, 3517, 3523, 3520, + 3524, 3525, 3521, 3526, 3522, 3518, 3519, 3471, 3472, 3511, 3512, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 3516, 3514, 3515, 3513, + 24308, 24307, 24306, 24320, 24319, 24325, 24335, 24334, 24338, 24326, + 24333, 24332, 24314, 24327, 24311, 24310, 24309, 24318, 24317, 24316, + 24315, 24324, 24323, 24329, 24328, 24313, 24343, 24340, 24339, 24322, + 24321, 24341, 24337, 24336, 24342, 24344, 24353, 24352, 24358, 24360, + 24356, 24357, 24354, 24355, 24359, 24297, 24302, 24301, 24299, 24303, + 24304, 24305, 24300, 24298, 24351, 24350, 40951, 40951, 40951, 24345, + 24348, 24349, 24347, 24346, 24367, 24369, 24366, 24365, 24362, 24361, + 24364, 24363, 24370, 24368, 40951, 40951, 40951, 24331, 24330, 24312, + 30023, 30025, 30022, 30021, 30018, 30017, 30020, 30019, 30026, 30024, + 29996, 29987, 29985, 29984, 29986, 29997, 29981, 29980, 29982, 29983, + 29999, 29995, 29993, 29992, 29994, 30001, 30007, 30008, 30006, 30009, + 29998, 29991, 29988, 29990, 29989, 30000, 30002, 30003, 30005, 30004, + 30011, 30012, 30010, 30016, 30015, 30027, 30014, 30013, 10430, 10407, + 10413, 10470, 10451, 10459, 10449, 10450, 10469, 10135, 10461, 40951, + 40951, 40951, 40951, 40951, 17856, 17887, 17864, 17893, 17862, 17892, + 17885, 17882, 17894, 17874, 17876, 17888, 17890, 17895, 17878, 17884, + 17886, 17880, 17883, 17896, 17877, 17873, 17863, 17891, 17879, 17857, + 17860, 17872, 17859, 17858, 17889, 17871, 17867, 17870, 17868, 17898, + 17865, 17869, 17899, 17897, 17861, 17881, 17855, 40951, 40951, 17854, + 17866, 17875, 34629, 34627, 34628, 34626, 34625, 34624, 34623, 34622, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 38803, 38816, + 38805, 38783, 38797, 38811, 38812, 38813, 38798, 38814, 38801, 38809, + 38804, 38802, 38810, 38808, 38807, 38815, 38796, 38794, 38785, 38792, + 38784, 38795, 38793, 38774, 38775, 38777, 38778, 38790, 38788, 38789, + 38787, 38776, 38780, 38786, 38799, 38782, 38791, 38779, 38806, 38800, + 38781, 40951, 40951, 40951, 40951, 40951, 23270, 23271, 23878, 23267, + 23272, 23273, 23244, 23243, 23863, 23856, 23276, 23277, 23250, 23278, + 23256, 23251, 23252, 23813, 23814, 23815, 23858, 23254, 23850, 23370, + 23280, 23257, 23265, 23260, 23283, 23818, 23817, 23816, 23284, 23285, + 23287, 23245, 23298, 23232, 18400, 18403, 18399, 18402, 18398, 10300, + 27701, 27702, 27692, 27693, 27704, 27705, 27695, 27707, 27697, 27708, + 27709, 27710, 27711, 27712, 27713, 27696, 27699, 27700, 27714, 27694, + 27716, 27717, 27719, 27850, 27957, 27851, 27959, 27854, 27879, 27893, + 27947, 27932, 27962, 27900, 27970, 27981, 27910, 27897, 27930, 27933, + 27954, 27856, 27934, 27949, 27974, 27948, 27960, 27978, 27852, 27858, + 27901, 27884, 27902, 27878, 24019, 24027, 24029, 24030, 18609, 18605, + 18608, 18607, 18606, 23939, 23356, 23405, 23491, 23634, 23654, 23731, + 23758, 23751, 23797, 23833, 24001, 23885, 27768, 23562, 23843, 23300, + 23569, 23727, 23301, 23938, 23357, 23409, 23492, 23505, 23588, 23615, + 23635, 23660, 23732, 23761, 23798, 23478, 23946, 23973, 24002, 23319, + 23345, 23408, 23468, 23720, 23768, 23806, 23559, 23716, 23480, 23925, + 23486, 27958, 27859, 27877, 27895, 27941, 27898, 27885, 27946, 27969, + 27912, 27913, 27860, 27862, 27915, 27919, 27921, 27865, 27911, 27961, + 27929, 27928, 27871, 27855, 27935, 27945, 27894, 27950, 27976, 27977, + 27873, 27980, 27971, 27890, 27892, 27891, 27896, 27973, 8177, 8176, 8425, + 8424, 8377, 8268, 8376, 8061, 8267, 8060, 8325, 8073, 8378, 8200, 8389, + 8416, 8286, 8440, 8441, 8308, 8299, 8300, 8301, 8303, 8310, 8306, 8335, + 8291, 8337, 8314, 8292, 8293, 8339, 8294, 8295, 8324, 8328, 8316, 8343, + 8298, 8330, 8331, 8329, 8307, 8315, 8319, 8340, 8305, 8322, 8332, 8297, + 8318, 8321, 8439, 8290, 8289, 8172, 8445, 8175, 8167, 8181, 8070, 8345, + 8400, 22756, 23317, 22792, 23359, 22791, 23358, 22790, 23355, 22799, + 23374, 22824, 23411, 22823, 23410, 22822, 23406, 22818, 23400, 22819, + 23401, 22850, 23454, 22851, 23455, 22839, 23442, 22848, 23452, 22830, + 23433, 22875, 23494, 22883, 23504, 22902, 23524, 22901, 23523, 22899, + 23521, 22896, 23518, 22895, 23517, 22919, 23551, 22913, 23539, 22950, + 23586, 22947, 23583, 22951, 23587, 22958, 23600, 22959, 23601, 22969, + 23610, 22965, 23597, 22975, 23632, 22977, 23637, 22976, 23636, 22995, + 23659, 22994, 23658, 22989, 23651, 22985, 23646, 23026, 23695, 23025, + 23694, 23003, 23687, 23004, 23688, 23054, 23730, 23056, 23734, 23065, + 23748, 23063, 23746, 23064, 23747, 23070, 23753, 23091, 23791, 23089, + 23789, 23088, 23782, 23083, 23784, 23090, 23790, 23112, 23831, 23111, + 23830, 23113, 23834, 23107, 23824, 23143, 23906, 23164, 23929, 23136, + 23899, 23163, 23928, 23155, 23918, 23176, 23949, 23175, 23945, 23189, + 23962, 23190, 23963, 23186, 23959, 23188, 23961, 23187, 23960, 23195, + 23968, 23194, 23967, 23200, 23978, 23211, 23992, 23215, 23996, 23217, + 23998, 23525, 23829, 23964, 23988, 23318, 23625, 23624, 23626, 23101, + 23415, 22750, 23311, 22766, 23329, 22762, 23325, 22761, 23324, 22760, + 23323, 22764, 23327, 22763, 23326, 22745, 23306, 22744, 23305, 22743, + 23304, 22747, 23308, 22746, 23307, 22844, 23447, 22855, 23460, 22847, + 23451, 22835, 23438, 22834, 23437, 22833, 23436, 22838, 23441, 22837, + 23440, 22920, 23552, 22910, 23543, 23017, 23680, 23037, 23706, 23009, + 23672, 23008, 23671, 23007, 23670, 23011, 23674, 23010, 23673, 23034, + 23703, 23033, 23702, 23032, 23701, 23036, 23705, 23035, 23704, 23146, + 23909, 23153, 23916, 23150, 23913, 23149, 23912, 23148, 23911, 23152, + 23915, 23151, 23914, 23201, 23979, 23199, 23977, 23203, 23981, 23207, + 23987, 22980, 23640, 22981, 23641, 23204, 23982, 18447, 18439, 18452, + 18444, 18448, 18440, 18450, 18442, 18213, 18205, 18216, 18208, 18214, + 18206, 18218, 18210, 18470, 18467, 18472, 18469, 18471, 18468, 40951, + 40951, 18253, 18250, 18255, 18252, 18254, 18251, 40951, 40951, 18485, + 18477, 18490, 18482, 18486, 18478, 18488, 18480, 18237, 18229, 18240, + 18232, 18238, 18230, 18242, 18234, 18511, 18502, 18514, 18505, 18513, + 18504, 18512, 18503, 18265, 18260, 18268, 18263, 18267, 18262, 18266, + 18261, 18572, 18569, 18574, 18571, 18573, 18570, 40951, 40951, 18299, + 18296, 18301, 18298, 18300, 18297, 40951, 40951, 18531, 18522, 18534, + 18525, 18533, 18524, 18532, 18523, 40951, 18311, 40951, 18314, 40951, + 18313, 40951, 18312, 18552, 18544, 18557, 18549, 18553, 18545, 18555, + 18547, 18283, 18275, 18286, 18278, 18284, 18276, 18288, 18280, 18437, + 18457, 18474, 18473, 18497, 18495, 18517, 18518, 18576, 18575, 18537, + 18538, 18564, 18562, 40951, 40951, 18454, 18446, 18453, 18445, 18449, + 18441, 18451, 18443, 18220, 18212, 18217, 18209, 18215, 18207, 18219, + 18211, 18492, 18484, 18491, 18483, 18487, 18479, 18489, 18481, 18244, + 18236, 18241, 18233, 18239, 18231, 18243, 18235, 18559, 18551, 18558, + 18550, 18554, 18546, 18556, 18548, 18290, 18282, 18287, 18279, 18285, + 18277, 18289, 18281, 18436, 18461, 18438, 18459, 18458, 40951, 18455, + 18456, 18222, 18226, 18223, 18224, 18221, 18396, 18424, 18425, 18429, + 18345, 18498, 18499, 18496, 40951, 18493, 18494, 18257, 18256, 18247, + 18246, 18245, 18428, 18427, 18426, 18516, 18520, 18509, 18508, 40951, + 40951, 18515, 18507, 18269, 18273, 18270, 18271, 40951, 18353, 18352, + 18351, 18536, 18540, 18529, 18528, 18584, 18583, 18535, 18527, 18316, + 18320, 18317, 18318, 18306, 18347, 18346, 18623, 40951, 40951, 18565, + 18566, 18563, 40951, 18560, 18561, 18303, 18302, 18293, 18292, 18291, + 18421, 18350, 40951, 16764, 16761, 16766, 16763, 37019, 17510, 33761, + 17439, 31718, 36992, 18978, 40757, 40756, 40758, 24189, 32047, 20456, + 29361, 17438, 16765, 16762, 20402, 11250, 11224, 24112, 31977, 33636, + 33634, 24065, 31942, 11223, 11218, 10529, 11217, 4995, 37548, 30526, + 37694, 20425, 20459, 24496, 31062, 24188, 32046, 31593, 24187, 32045, + 28962, 31296, 31297, 31679, 11232, 37562, 31885, 31879, 31893, 6007, + 33635, 33637, 31902, 11254, 20799, 30878, 37745, 6293, 6008, 2529, 20458, + 17514, 24120, 31988, 11255, 31742, 17353, 37393, 31884, 3854, 3896, + 25158, 31890, 8045, 37706, 8451, 34742, 20817, 17476, 36999, 31738, + 17503, 17463, 37695, 17504, 11196, 37573, 38824, 26974, 39366, 17636, + 20819, 20821, 20820, 40951, 24186, 32044, 17456, 31592, 20713, 7, 20712, + 6, 28957, 29359, 34713, 34701, 40951, 40951, 34708, 34707, 34710, 34709, + 34700, 34714, 34704, 34706, 34699, 34703, 34705, 34702, 34543, 34545, + 34542, 34541, 34538, 34537, 34540, 34539, 34533, 34544, 34534, 34536, + 34532, 34531, 34535, 40951, 24016, 24017, 24025, 24031, 24015, 24018, + 24021, 24022, 24023, 24024, 24026, 24014, 24028, 40951, 40951, 40951, + 17350, 8058, 8697, 17520, 25104, 27586, 28890, 31320, 32367, 39370, + 29093, 11190, 17351, 22555, 37582, 11372, 17901, 31321, 18673, 2540, + 20465, 6126, 25105, 34096, 36765, 20703, 37664, 29411, 25663, 32236, + 22737, 3762, 34076, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 8354, 8408, 8368, + 8420, 8066, 8082, 8347, 8405, 8413, 8081, 8065, 8427, 8213, 8204, 8207, + 8210, 8205, 8357, 8206, 8208, 8209, 8397, 8194, 8067, 8434, 8446, 8359, + 8365, 8412, 8358, 8346, 8404, 8069, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 0, + 100, 11266, 10553, 6130, 6009, 5260, 17352, 32555, 10555, 32552, 32544, + 3944, 11267, 31482, 31483, 32545, 3945, 32546, 32553, 25335, 11268, + 29454, 33992, 32548, 11264, 11269, 32549, 3946, 11270, 31655, 31863, + 32658, 36660, 37535, 38817, 11271, 30872, 30880, 20816, 3947, 37687, + 21610, 961, 32541, 3943, 16820, 32551, 32542, 32543, 37671, 32547, 32554, + 346, 3653, 17904, 10542, 20710, 32224, 17418, 11278, 11277, 11263, 11265, + 11279, 37680, 37681, 31889, 37682, 11276, 11272, 11273, 11274, 11275, + 31683, 37666, 31298, 2600, 37684, 34821, 38966, 38964, 38968, 38969, + 38974, 38962, 38973, 38972, 38960, 38967, 38959, 38961, 38970, 38958, + 38975, 17515, 32192, 32203, 32204, 32191, 32188, 32197, 32199, 32207, + 32208, 32200, 32206, 32202, 32185, 32193, 32189, 32195, 33823, 33827, + 33828, 33822, 33819, 33831, 33830, 33818, 33832, 33829, 33817, 33826, + 33821, 33824, 33820, 33825, 32196, 32190, 32201, 32205, 23774, 32198, + 32187, 32186, 32194, 38976, 37675, 37674, 40951, 40951, 40951, 40951, + 24190, 37896, 32049, 11308, 24094, 37767, 29392, 29365, 33964, 33979, + 24210, 32073, 24274, 32150, 24271, 37946, 32149, 11344, 24213, 32076, + 24221, 37909, 32056, 11322, 37768, 24219, 32082, 24204, 32068, 24102, + 24097, 11348, 37906, 37907, 11317, 11318, 32064, 11309, 972, 8029, 29394, + 24201, 977, 8031, 24240, 24234, 37923, 37920, 32110, 32104, 11359, 11356, + 32084, 37898, 24224, 24286, 37949, 32116, 11367, 24241, 32099, 24277, + 24101, 32092, 24275, 37913, 32090, 11349, 24099, 37770, 29405, 29378, + 33976, 33989, 24261, 32140, 24290, 32120, 37899, 11310, 24281, 37914, + 32096, 11350, 24200, 32063, 24272, 37950, 32151, 11346, 37955, 37951, + 37953, 37952, 37957, 37958, 32152, 29393, 33967, 37772, 31929, 11320, + 37007, 24220, 32083, 24096, 24206, 32066, 24095, 24285, 32115, 24104, + 17498, 8456, 31208, 36987, 36986, 16754, 20631, 28846, 16703, 29414, + 33803, 8464, 11016, 33796, 16768, 28805, 28793, 28797, 27590, 27597, + 11191, 10998, 32663, 2528, 32161, 4996, 34323, 8703, 17509, 31682, 20704, + 31919, 957, 26848, 34099, 11002, 11017, 31067, 29419, 25119, 25126, + 20792, 37747, 20777, 11221, 37572, 8470, 34735, 38954, 8032, 8023, 974, + 36988, 3654, 31773, 31681, 11192, 17355, 17726, 20446, 37297, 31891, + 20813, 33756, 39374, 29424, 27596, 2533, 29415, 1054, 1057, 29049, 362, + 29416, 364, 37563, 359, 16806, 17724, 11008, 1058, 17725, 1055, 20594, + 8057, 16804, 32159, 32160, 8649, 16821, 16807, 34488, 10558, 16790, + 26859, 31744, 29426, 20470, 29427, 34524, 24390, 18137, 24392, 18140, + 24381, 18130, 28542, 28541, 3652, 29425, 29429, 29428, 29054, 29051, + 24389, 18136, 29053, 29050, 24391, 18138, 29055, 29052, 31656, 34561, + 31665, 34570, 31664, 34569, 11019, 11022, 34549, 34721, 29412, 29413, + 34555, 34727, 29047, 29048, 34554, 34726, 28295, 28296, 28297, 34196, + 34285, 34198, 34287, 34131, 34138, 6804, 6750, 6809, 6542, 6555, 6805, + 6531, 6814, 6556, 34456, 34449, 34470, 34404, 32006, 24138, 11285, 37776, + 2534, 27605, 37579, 17499, 37566, 11248, 11021, 29423, 11024, 29046, + 31666, 34571, 29409, 8465, 29410, 8466, 30556, 20593, 28298, 20217, + 20800, 39395, 28891, 29364, 31925, 32002, 28802, 28803, 28804, 28798, + 10837, 11193, 34490, 11000, 4372, 24152, 31964, 24110, 31975, 31892, + 9946, 9945, 11242, 11241, 11220, 11245, 31476, 16791, 24395, 18146, + 38867, 38866, 24379, 18141, 16789, 16788, 16786, 16787, 11020, 11023, + 29420, 29421, 34197, 34286, 24380, 18129, 31663, 34568, 29417, 11014, + 29418, 11015, 38823, 27582, 37775, 11288, 16704, 16706, 33804, 16709, + 16708, 33805, 16707, 16705, 8467, 8468, 33797, 8469, 33798, 40669, 10838, + 16701, 20440, 37759, 11289, 31680, 31315, 39140, 24061, 31937, 24149, + 31946, 4355, 4352, 37483, 37479, 31882, 34229, 2524, 32565, 32561, 36658, + 31601, 38878, 31479, 37678, 39131, 20438, 37478, 37482, 4351, 4354, + 37472, 4349, 17524, 33863, 37760, 30548, 16817, 39379, 21612, 24159, + 32027, 16816, 3650, 10524, 360, 34831, 37499, 11009, 8475, 33788, 8651, + 8652, 1002, 1040, 1026, 1014, 1015, 1031, 1012, 982, 987, 1037, 1042, + 1028, 1027, 1022, 1029, 1010, 1034, 1023, 1030, 985, 994, 993, 1016, + 1019, 995, 1048, 1021, 1045, 991, 1020, 1018, 1046, 998, 1017, 1033, 992, + 999, 1008, 986, 1047, 1032, 983, 1013, 1044, 989, 1038, 1007, 984, 996, + 1009, 1050, 1049, 988, 990, 1051, 1039, 1036, 1025, 1024, 997, 1043, + 1000, 1035, 1005, 1004, 1041, 1001, 1006, 1003, 29430, 31924, 32844, + 3546, 38839, 20776, 8473, 10924, 16760, 8455, 39283, 16782, 365, 19928, + 6586, 6808, 4936, 37746, 28178, 20462, 30544, 30545, 31220, 31221, 10919, + 33880, 1011, 10549, 31667, 29278, 31669, 8052, 24155, 24156, 24154, + 31971, 31972, 31970, 24117, 24124, 24116, 31985, 31992, 31984, 24059, + 24057, 24058, 9948, 31935, 31933, 31934, 20787, 20406, 37828, 37867, + 34574, 34572, 37486, 4358, 4359, 31753, 24158, 32008, 20415, 20416, + 20417, 20418, 10571, 10569, 10573, 10562, 10566, 10574, 10563, 10567, + 10572, 10561, 10565, 10560, 10564, 10570, 10568, 34164, 31864, 17382, + 38834, 27419, 27411, 27418, 27412, 27416, 27417, 27415, 27414, 27413, + 11539, 17640, 37474, 4362, 37456, 4361, 37487, 4365, 39292, 3651, 34516, + 17468, 8, 16702, 10550, 3885, 3847, 3928, 3824, 3886, 3848, 3888, 367, + 34514, 37305, 20439, 3864, 3867, 3869, 3861, 11246, 3907, 3771, 31615, + 31612, 31613, 31614, 29932, 34816, 34824, 34825, 34805, 34803, 34806, + 34817, 34788, 34789, 34810, 34812, 34811, 34809, 34790, 34822, 34823, + 34792, 34801, 34800, 34799, 34798, 34814, 34828, 34804, 34791, 34802, + 34826, 34807, 34808, 34819, 34818, 34820, 34829, 34793, 3951, 30531, + 34815, 34796, 34827, 34795, 34794, 34797, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 29944, 29939, + 29942, 29943, 29936, 29937, 29935, 29934, 29941, 29938, 29940, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 6552, 6549, 6548, 6545, 6544, 6547, 6546, 6553, 6551, 6773, 6753, 6798, + 6786, 6768, 6756, 6772, 6770, 6752, 6799, 6787, 31154, 31152, 31151, + 31148, 31147, 31150, 31149, 31155, 31153, 31146, 31137, 31145, 31143, + 31138, 31139, 31141, 31142, 31136, 31140, 31144, 10859, 10871, 10868, + 10853, 10850, 10865, 10862, 10877, 10856, 29450, 29443, 29451, 29449, + 29444, 29445, 29446, 29448, 29442, 29453, 29452, 31182, 31183, 31184, + 31185, 31186, 31187, 31188, 31189, 31190, 31191, 31192, 31193, 31194, + 31195, 31196, 31197, 31198, 31199, 31200, 31201, 31202, 31203, 31204, + 31205, 31206, 31207, 6696, 6697, 6698, 6699, 6700, 6701, 6702, 6703, + 6704, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, 6713, 6714, 6715, + 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6724, 6725, 6726, 6727, + 6728, 6729, 6730, 6731, 6732, 6733, 6734, 6735, 6736, 6737, 6738, 6739, + 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6550, 28971, 28969, + 28967, 28972, 28973, 28975, 28976, 28970, 28974, 28968, 11212, 11210, + 11209, 11206, 11205, 11208, 11207, 11213, 11211, 11204, 28966, 4484, + 4430, 4500, 4416, 4493, 4429, 4492, 4428, 4491, 4427, 4490, 4426, 4481, + 4400, 4394, 4423, 4480, 4398, 4392, 4422, 4499, 4528, 4522, 4415, 4498, + 4526, 4520, 4413, 4507, 4544, 4521, 4393, 4541, 4399, 4527, 4419, 4506, + 4543, 4519, 4391, 4540, 4397, 4525, 4418, 4479, 4434, 4512, 4402, 4396, + 4515, 4437, 4421, 4497, 4433, 4511, 4529, 4523, 4514, 4436, 4414, 4505, + 4435, 4513, 4542, 4518, 4395, 4539, 4439, 4517, 4432, 4510, 4401, 4524, + 4516, 4438, 4417, 4483, 4425, 4482, 4424, 4390, 4386, 4407, 4404, 4382, + 4406, 4403, 4381, 4534, 4531, 4385, 4533, 4530, 4384, 4546, 4537, 4389, + 4545, 4536, 4388, 4408, 4405, 4380, 4535, 4532, 4383, 4547, 4538, 4387, + 4441, 4440, 4442, 4443, 4472, 4466, 4476, 4488, 4495, 4509, 4478, 4409, + 4411, 4431, 4420, 4489, 4496, 4410, 4412, 37805, 25259, 25258, 25261, + 25169, 25251, 25264, 25260, 17531, 24111, 24134, 24147, 24075, 24135, + 24090, 24091, 31950, 24410, 26976, 10536, 37853, 31967, 31735, 31736, + 31727, 31728, 31729, 31730, 31731, 31732, 31733, 31734, 3905, 39270, + 39279, 39273, 34342, 34351, 34341, 34348, 34350, 34340, 3902, 39267, + 3898, 39255, 3935, 39302, 3879, 39251, 3930, 39299, 3929, 39298, 3887, + 39259, 3892, 39258, 3891, 39257, 3826, 39214, 3825, 39213, 3852, 39240, + 3851, 39239, 3850, 39238, 3816, 39203, 39204, 17460, 25266, 39189, 11195, + 6525, 4999, 3767, 6517, 6526, 6518, 6523, 6522, 6524, 24073, 31948, + 20804, 20808, 37809, 25163, 37833, 37869, 25213, 25193, 37814, 25170, + 3859, 3858, 3931, 3932, 39155, 34345, 34352, 34347, 34344, 39282, 39300, + 37791, 37792, 22734, 39280, 39276, 39277, 39281, 39196, 39194, 39195, + 39197, 37831, 37886, 25204, 39247, 3874, 39246, 3873, 25227, 3909, 8046, + 37740, 33870, 8458, 3918, 39289, 24420, 37024, 34575, 2525, 10578, 8476, + 30552, 3924, 39291, 2750, 2756, 2757, 32374, 37742, 20434, 39264, 3916, + 32818, 31887, 3857, 3884, 39237, 39296, 39261, 39212, 33767, 6123, 31751, + 3764, 5259, 981, 30652, 6480, 8684, 8683, 34491, 17428, 102, 19104, + 31286, 40667, 37551, 37552, 37557, 37554, 37556, 37555, 37553, 37550, + 39151, 39224, 39268, 3904, 39287, 17453, 22738, 27408, 17434, 11535, + 25571, 20931, 32444, 37959, 29280, 31581, 2523, 36647, 17723, 5997, + 24295, 38880, 25108, 32537, 32372, 6003, 2601, 31474, 39161, 39177, + 39180, 39156, 39164, 39174, 3787, 3803, 3806, 3782, 3790, 3800, 3917, + 39227, 39208, 3815, 39269, 3836, 3821, 39198, 20435, 31740, 16657, 3532, + 3533, 28302, 28301, 28303, 39149, 11540, 37756, 31781, 31782, 31783, + 31784, 31785, 31786, 31787, 31788, 3934, 31780, 31213, 31318, 39152, + 10841, 10842, 10843, 10844, 10845, 10846, 39191, 39193, 3768, 3770, + 28175, 28176, 10881, 10884, 10883, 10882, 39218, 3832, 19105, 979, 8692, + 34485, 32532, 345, 17475, 17719, 34486, 2536, 17473, 30859, 37001, 37000, + 39125, 20285, 11281, 11282, 20791, 25570, 25569, 25568, 38840, 20426, + 26985, 26962, 26975, 25737, 11003, 37758, 8675, 17637, 29090, 6133, + 31022, 20932, 38870, 6483, 3875, 32665, 32577, 31746, 32660, 33875, 3469, + 34418, 39215, 39216, 3829, 3830, 33871, 34577, 31756, 3911, 37023, 37670, + 37669, 39209, 8693, 10923, 30551, 31459, 6065, 19927, 6538, 6134, 29352, + 366, 3925, 39294, 3855, 39235, 11381, 19785, 24062, 34461, 17422, 3922, + 31860, 31861, 2532, 19711, 31293, 32026, 24185, 20814, 3780, 32823, 6515, + 6125, 20390, 17720, 17721, 25666, 28194, 37741, 17511, 17470, 17436, + 32528, 34163, 33765, 20469, 31305, 36766, 20829, 19687, 17639, 9941, + 39219, 3912, 37798, 3915, 25245, 39262, 39228, 36659, 36646, 226, 16779, + 31769, 31761, 38872, 39377, 25244, 31295, 37868, 39250, 3877, 6299, + 19709, 28294, 19745, 2760, 19701, 30861, 19792, 30535, 19747, 23224, + 32670, 30858, 25572, 34489, 17507, 17505, 19730, 17501, 3833, 39223, + 34084, 34518, 6811, 30533, 3781, 30860, 19749, 31471, 32669, 19700, + 30534, 16656, 16651, 16649, 33759, 16650, 19723, 37690, 33762, 36652, + 30532, 19775, 33757, 3831, 39220, 16652, 6800, 19774, 33873, 37295, + 19708, 34083, 19768, 2749, 16655, 19725, 8689, 32668, 29038, 25243, + 37865, 25225, 37883, 3942, 39254, 39217, 3818, 19727, 24417, 26982, + 19791, 19758, 19759, 19719, 19720, 19742, 19741, 9953, 19728, 19734, + 19704, 32221, 17474, 32220, 26969, 26972, 26963, 26964, 26970, 26973, + 19738, 19751, 19737, 19750, 24409, 24407, 26968, 26971, 10906, 10904, + 10903, 10900, 10899, 10902, 10901, 10907, 10905, 10898, 10917, 10914, + 10913, 10910, 10909, 10912, 10911, 10918, 10916, 10908, 10896, 10893, + 10892, 10889, 10888, 10891, 10890, 10897, 10895, 10887, 19787, 19790, + 19746, 19716, 19764, 19752, 19771, 11373, 19755, 37545, 19777, 10539, + 19715, 3893, 37015, 37018, 3894, 19702, 19703, 34479, 19714, 32042, + 24175, 2613, 17522, 19743, 19782, 29432, 9947, 29434, 6588, 39306, 3948, + 3950, 3949, 19705, 19707, 19706, 36653, 19776, 39145, 19788, 30546, + 11214, 36998, 39293, 31299, 30542, 30541, 24109, 31974, 30654, 31871, + 34732, 38821, 26434, 25134, 26255, 34446, 34447, 39207, 980, 16711, + 25241, 37826, 24093, 31965, 17530, 22726, 22725, 24040, 24039, 24089, + 25149, 25138, 37777, 25267, 39199, 39200, 39201, 39275, 39278, 26435, + 26429, 26438, 26432, 26437, 26431, 26436, 26430, 26439, 26433, 37927, + 11363, 976, 8025, 31928, 25139, 25144, 25137, 25141, 25146, 25136, 25140, + 25145, 25142, 25147, 25148, 4925, 4670, 4798, 4671, 4862, 4735, 4799, + 4672, 4894, 4767, 4831, 4704, 4863, 4736, 4800, 4673, 4910, 4783, 4847, + 4720, 4879, 4752, 4816, 4689, 4895, 4768, 4832, 4705, 4864, 4737, 4801, + 4674, 4918, 4791, 4855, 4728, 4887, 4760, 4824, 4697, 4903, 4776, 4840, + 4713, 4872, 4745, 4809, 4682, 4911, 4784, 4848, 4721, 4880, 4753, 4817, + 4690, 4896, 4769, 4833, 4706, 4865, 4738, 4802, 4675, 4922, 4795, 4859, + 4732, 4891, 4764, 4828, 4701, 4907, 4780, 4844, 4717, 4876, 4749, 4813, + 4686, 4915, 4788, 4852, 4725, 4884, 4757, 4821, 4694, 4900, 4773, 4837, + 4710, 4869, 4742, 4806, 4679, 4919, 4792, 4856, 4729, 4888, 4761, 4825, + 4698, 4904, 4777, 4841, 4714, 4873, 4746, 4810, 4683, 4912, 4785, 4849, + 4722, 4881, 4754, 4818, 4691, 4897, 4770, 4834, 4707, 4866, 4739, 4803, + 4676, 4924, 4797, 4861, 4734, 4893, 4766, 4830, 4703, 4909, 4782, 4846, + 4719, 4878, 4751, 4815, 4688, 4917, 4790, 4854, 4727, 4886, 4759, 4823, + 4696, 4902, 4775, 4839, 4712, 4871, 4744, 4808, 4681, 4921, 4794, 4858, + 4731, 4890, 4763, 4827, 4700, 4906, 4779, 4843, 4716, 4875, 4748, 4812, + 4685, 4914, 4787, 4851, 4724, 4883, 4756, 4820, 4693, 4899, 4772, 4836, + 4709, 4868, 4741, 4805, 4678, 4923, 4796, 4860, 4733, 4892, 4765, 4829, + 4702, 4908, 4781, 4845, 4718, 4877, 4750, 4814, 4687, 4916, 4789, 4853, + 4726, 4885, 4758, 4822, 4695, 4901, 4774, 4838, 4711, 4870, 4743, 4807, + 4680, 4920, 4793, 4857, 4730, 4889, 4762, 4826, 4699, 4905, 4778, 4842, + 4715, 4874, 4747, 4811, 4684, 4913, 4786, 4850, 4723, 4882, 4755, 4819, + 4692, 4898, 4771, 4835, 4708, 4867, 4740, 4804, 4677, 32146, 32145, + 24276, 32091, 24100, 32147, 24278, 32093, 11319, 37908, 37945, 11343, + 24280, 32095, 24260, 32139, 32148, 32065, 37910, 11323, 32078, 32077, + 32141, 32143, 32142, 24225, 32085, 24279, 32094, 24202, 32062, 24222, + 32057, 29391, 29371, 29397, 29369, 33968, 33981, 29396, 29368, 33965, + 33980, 32165, 17420, 33966, 29366, 17421, 32166, 29367, 29395, 39134, + 2516, 2515, 2518, 2519, 32043, 24174, 37453, 4360, 37455, 37454, 25223, + 25187, 973, 8022, 32052, 24191, 32831, 32070, 24207, 32061, 24098, 37948, + 24052, 24051, 37765, 37764, 24053, 37766, 24050, 37763, 24239, 32109, + 37922, 11358, 24233, 32103, 37919, 11355, 24238, 32108, 37921, 11357, + 24232, 32102, 37918, 11354, 24235, 37917, 32107, 11352, 24237, 24231, + 32105, 32100, 24236, 24230, 32106, 32101, 37916, 11353, 31940, 16796, + 37299, 24195, 32054, 32053, 24376, 24198, 18125, 34548, 24197, 34718, + 24148, 31945, 37774, 11287, 37565, 40680, 40681, 24140, 32011, 24142, + 32013, 40675, 40671, 40676, 40672, 24121, 31989, 24119, 31986, 24118, + 31987, 24046, 31921, 24047, 31927, 11227, 11252, 24055, 31931, 11202, + 38854, 26857, 31926, 26858, 958, 5, 34100, 34101, 37667, 31877, 959, + 31878, 29930, 29929, 26856, 26855, 26850, 26849, 26854, 26852, 26853, + 26851, 31900, 16758, 16757, 16755, 16756, 6527, 6815, 6802, 6806, 6803, + 6528, 6520, 6529, 37762, 6810, 6533, 6748, 6816, 6519, 6521, 34410, + 34405, 34476, 34462, 34463, 37700, 37544, 37542, 32369, 37541, 32003, + 24126, 38818, 4373, 4374, 3941, 37306, 37307, 39232, 3840, 24145, 32016, + 24063, 31941, 20628, 37233, 20705, 11280, 34353, 20630, 32851, 16797, + 16798, 20471, 18009, 36990, 11300, 11301, 3819, 3860, 39192, 3769, 16810, + 16813, 16809, 16812, 16808, 16811, 32238, 31872, 33927, 31873, 3757, + 3756, 11234, 37561, 24162, 32029, 37308, 27598, 28792, 28790, 28791, + 28800, 28799, 28795, 28796, 37702, 37701, 28794, 28000, 34573, 31737, + 17446, 20786, 20778, 6820, 978, 24492, 24493, 24494, 20779, 31739, 20783, + 20780, 20784, 20782, 20785, 20781, 20929, 22727, 40677, 40678, 40679, + 31573, 31578, 31576, 31572, 31575, 31574, 31577, 27591, 27592, 27594, + 27593, 31569, 31570, 38869, 28293, 28292, 32576, 33848, 28288, 28289, + 6749, 28290, 6543, 31571, 27595, 28291, 20801, 32048, 40673, 372, 20797, + 37751, 37752, 20796, 20795, 37753, 37749, 20794, 37748, 20793, 37750, + 20798, 8038, 8044, 11235, 11236, 8039, 25122, 25130, 11225, 11226, 37698, + 37699, 33787, 33786, 25127, 25124, 25132, 25123, 25131, 25121, 25125, + 25120, 33838, 25129, 25128, 40674, 40670, 16802, 20472, 37559, 37560, + 37301, 37300, 33631, 8477, 16803, 363, 1056, 16793, 31579, 16794, 11215, + 37693, 37009, 16801, 16805, 24393, 18144, 24394, 18145, 24385, 18131, + 24388, 18134, 24386, 18132, 24387, 18133, 24384, 18135, 24378, 18127, + 24377, 18126, 24375, 18123, 24373, 18121, 24372, 18120, 24371, 18124, + 24374, 18122, 33772, 33770, 33773, 33771, 11262, 11261, 11258, 11257, + 33630, 33629, 33628, 33627, 11228, 11230, 11229, 18139, 18128, 24382, + 18142, 24383, 18143, 33846, 22735, 33847, 22736, 16799, 31659, 34564, + 31660, 34565, 31662, 34567, 31658, 34563, 31661, 34566, 31657, 34562, + 11231, 11240, 34559, 34731, 34557, 34729, 34558, 34730, 34556, 34728, + 34551, 34723, 34552, 34724, 34550, 34722, 34553, 34725, 34231, 34318, + 8033, 8035, 8034, 8036, 34546, 34717, 34547, 34716, 34720, 34719, 16710, + 31477, 37539, 17496, 29363, 32835, 32836, 32832, 31300, 38820, 11249, + 38819, 11247, 25133, 32837, 32834, 32833, 11216, 11244, 11238, 31881, + 11018, 38836, 38835, 11286, 31066, 31065, 37564, 37567, 37558, 37571, + 37570, 11260, 11259, 37568, 22724, 11243, 39304, 28801, 29380, 29407, + 33978, 33991, 24103, 24229, 37912, 11325, 29377, 29404, 33975, 33988, + 24105, 37769, 32074, 32075, 24211, 24212, 34346, 34339, 34349, 34343, + 10833, 10834, 10832, 10831, 11198, 3846, 39233, 3939, 39305, 3880, 39252, + 39230, 3837, 20401, 3841, 3863, 39244, 3866, 39248, 3899, 3900, 39265, + 3839, 39231, 3936, 39301, 24049, 37002, 24048, 25143, 24268, 24267, + 24269, 24270, 24205, 24215, 24214, 24263, 24265, 24264, 24199, 39133, + 16795, 31874, 24192, 32060, 32059, 24294, 32155, 31875, 32050, 37298, + 24194, 24193, 32051, 11339, 32830, 32828, 39245, 3901, 39266, 3890, + 39256, 19735, 19748, 19712, 19710, 19713, 33774, 2611, 33775, 2610, 3648, + 32829, 24245, 37931, 32124, 11328, 24107, 37773, 29402, 29375, 33973, + 33986, 24257, 37942, 32136, 11340, 8030, 971, 24256, 37933, 32135, 11330, + 40951, 40951, 29403, 29376, 33974, 33987, 24247, 37941, 32126, 11338, + 20422, 38849, 24246, 37932, 32125, 11329, 24258, 37943, 32137, 11341, + 24228, 37911, 32088, 11327, 968, 969, 967, 970, 31865, 31866, 29275, + 29276, 17502, 32089, 40951, 34830, 37013, 37017, 37014, 37016, 3853, + 3933, 3895, 3827, 11332, 11333, 37935, 37936, 24250, 32129, 24249, 32128, + 3772, 3773, 3774, 3775, 3776, 3778, 3777, 3779, 31912, 31913, 31910, + 31911, 31907, 31909, 31906, 31908, 37956, 37761, 30876, 30875, 30877, + 2755, 6818, 6532, 3906, 3817, 37668, 20400, 3940, 3870, 3862, 3865, 3868, + 29281, 37471, 4348, 24405, 32222, 39222, 32223, 34303, 37744, 18671, + 31585, 31584, 31583, 31582, 37538, 31686, 2531, 20463, 31458, 29058, + 39249, 3820, 37580, 9943, 19685, 40759, 22562, 1053, 97, 38957, 31596, + 24074, 31949, 34492, 34493, 24266, 37947, 32144, 11345, 16815, 16814, + 32666, 32364, 32362, 32363, 32361, 32365, 32366, 16800, 37755, 32657, + 11283, 31219, 31888, 19929, 17909, 17911, 17945, 17919, 17915, 17946, + 17953, 17916, 17952, 17925, 17921, 17920, 17914, 17956, 17927, 17928, + 17929, 17930, 17932, 17934, 17938, 17942, 17955, 17917, 17954, 17931, + 17933, 17935, 17944, 17913, 17937, 17948, 17947, 17949, 17939, 17951, + 17940, 17941, 17950, 17923, 17910, 17922, 17918, 17924, 17936, 17943, + 17926, 17912, 17957, 17959, 17993, 17967, 17963, 17994, 18001, 17964, + 18000, 17973, 17969, 17968, 17962, 18004, 17975, 17976, 17977, 17978, + 17980, 17982, 17986, 17990, 18003, 17965, 18002, 17979, 17981, 17983, + 17992, 17961, 17985, 17996, 17995, 17997, 17987, 17999, 17988, 17989, + 17998, 17971, 17958, 17970, 17966, 17972, 17984, 17991, 17974, 17960, + 22957, 23603, 22960, 23051, 23067, 23337, 23828, 22900, 23522, 22946, + 23582, 23214, 23995, 22780, 22979, 23118, 23119, 23948, 23191, 23965, + 23947, 22905, 23529, 23893, 23450, 23869, 23684, 23261, 24020, 27718, + 23094, 23218, 8485, 8579, 8535, 8629, 8499, 8593, 8496, 8590, 8540, 8634, + 8530, 8624, 8534, 8628, 8501, 8595, 8532, 8626, 8538, 8632, 8504, 8598, + 8509, 8603, 8541, 8635, 8542, 8636, 8505, 8599, 8510, 8604, 8537, 8631, + 8539, 8633, 8531, 8625, 8533, 8627, 8543, 8637, 8507, 8601, 8503, 8597, + 8536, 8630, 8526, 8620, 8493, 8587, 8521, 8615, 8489, 8583, 8494, 8588, + 8495, 8589, 8490, 8584, 8519, 8613, 8529, 8623, 8491, 8585, 8517, 8611, + 8520, 8614, 8484, 8578, 8492, 8586, 8512, 8606, 8513, 8607, 8508, 8602, + 8515, 8609, 8514, 8608, 8511, 8605, 8518, 8612, 8516, 8610, 8524, 8618, + 8522, 8616, 8523, 8617, 8525, 8619, 8639, 8640, 8641, 8643, 8644, 8638, + 8642, 8487, 8581, 8488, 8582, 8483, 8482, 8481, 8486, 8580, 40951, 40951, + 40951, 40951, 40951, 8577, 8574, 8575, 8576, 8572, 8573, 8645, 17768, + 17794, 17779, 17800, 17801, 17799, 17789, 17790, 17802, 17783, 17792, + 17795, 17797, 17803, 17785, 17788, 17793, 17787, 17791, 17804, 17784, + 17782, 17778, 17798, 17786, 17774, 17777, 17781, 17776, 17775, 17796, + 17780, 17769, 17773, 17771, 17806, 17770, 17772, 40951, 17805, 40951, + 40951, 40951, 40951, 40951, 17767, 40951, 40951, 37248, 37259, 37260, + 37253, 37255, 37238, 37277, 37249, 37252, 37250, 37251, 37287, 37276, + 37256, 37242, 37258, 37261, 37237, 37246, 37262, 37275, 37257, 37243, + 37282, 37247, 37288, 37270, 37236, 37244, 37278, 37279, 37280, 37241, + 37245, 37281, 37290, 37272, 37273, 37254, 37240, 37235, 37263, 37265, + 37264, 37266, 37267, 37274, 37268, 37283, 37284, 37285, 37269, 37239, + 37271, 37286, 37289, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 37291, 37292, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 37234, 17250, 17067, 17154, + 17194, 17170, 16857, 17232, 16895, 17085, 17076, 16977, 17310, 16925, + 16910, 17241, 17201, 16886, 17101, 17114, 16962, 16966, 16965, 16964, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 17184, + 17190, 17188, 17185, 17187, 17186, 17189, 40951, 16876, 16882, 16880, + 16877, 16879, 16878, 16881, 40951, 17300, 17306, 17304, 17301, 17303, + 17302, 17305, 40951, 16869, 16875, 16873, 16870, 16872, 16871, 16874, + 40951, 17136, 17142, 17140, 17137, 17139, 17138, 17141, 40951, 17045, + 17051, 17049, 17046, 17048, 17047, 17050, 40951, 17277, 17283, 17281, + 17278, 17280, 17279, 17282, 40951, 16988, 16994, 16992, 16989, 16991, + 16990, 16993, 40951, 8091, 8129, 8126, 8093, 8123, 8124, 8130, 8097, + 8098, 8099, 8106, 8128, 8100, 8094, 8122, 8119, 8121, 8125, 8109, 8108, + 8127, 8095, 8131, 8105, 8092, 8118, 8114, 8116, 8103, 8117, 8090, 8102, + 31923, 31922, 24125, 31993, 24067, 31938, 31760, 31759, 11201, 24128, + 32001, 31768, 24108, 31973, 11542, 31064, 17495, 31883, 20461, 11200, + 11324, 37895, 11203, 11251, 20810, 31023, 20457, 37304, 24088, 31963, + 37303, 37302, 24157, 32007, 37480, 37484, 4353, 4356, 24113, 31978, + 24066, 31943, 37696, 30525, 34406, 17464, 31898, 38852, 32158, 39365, + 37672, 31758, 31770, 37683, 10530, 10531, 37673, 37469, 37708, 37020, + 34506, 38855, 39348, 6002, 11219, 31897, 11222, 10537, 11239, 20811, + 20812, 25159, 25160, 11237, 11197, 37569, 26959, 31063, 31717, 8648, + 8685, 8686, 37392, 26958, 26960, 24123, 31991, 24122, 31990, 37461, + 37465, 4339, 4343, 29931, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 7916, 7869, 7919, + 7918, 7917, 7912, 7840, 7937, 7951, 7950, 7873, 7920, 7933, 7932, 7902, + 7901, 7900, 7899, 7929, 7938, 7928, 7927, 7888, 7887, 7891, 7915, 40951, + 7872, 7935, 7909, 7874, 7907, 7867, 7943, 7942, 7881, 7911, 7910, 7921, + 7871, 7875, 7896, 7838, 7880, 7931, 7930, 7843, 7926, 7854, 7949, 7948, + 7947, 7946, 7904, 7934, 7914, 7879, 7952, 7842, 7841, 7903, 7908, 7885, + 7884, 7883, 7939, 7870, 7945, 7944, 7858, 7922, 7890, 7857, 7856, 7882, + 7866, 7923, 7941, 7940, 7868, 7850, 7898, 7897, 7853, 7851, 7906, 7905, + 7913, 7844, 7860, 7852, 7862, 7848, 7878, 7877, 7876, 7846, 7889, 7865, + 7839, 7886, 7847, 7864, 7855, 7924, 7925, 7849, 7895, 7845, 7893, 7861, + 7894, 7863, 7936, 7892, 7859, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 21128, 21110, 21044, 21164, + 21152, 21096, 21196, 21111, 21124, 21107, 21059, 21063, 21048, 21033, + 21099, 21185, 21131, 21102, 21136, 21213, 21170, 21141, 21093, 21198, + 21042, 21153, 21029, 21133, 21004, 21122, 21058, 21056, 21151, 21079, + 21081, 21061, 21011, 21210, 21036, 21144, 21098, 21180, 21103, 21031, + 21169, 21120, 21142, 21211, 21129, 21194, 21054, 21159, 21045, 21113, + 21197, 21160, 21019, 21178, 21020, 21172, 21090, 21087, 21050, 21088, + 21021, 21139, 21150, 21043, 21006, 21181, 21126, 21183, 21149, 21123, + 21193, 21104, 21173, 21038, 21204, 21049, 21032, 21078, 21027, 21171, + 21203, 21070, 21028, 21065, 21047, 21086, 21165, 21067, 21035, 21051, + 21134, 21100, 21114, 21188, 21177, 21109, 21215, 21068, 21015, 21161, + 21046, 21206, 21182, 21041, 21064, 21166, 21002, 21175, 21168, 21192, + 21082, 21026, 21176, 21007, 21143, 21162, 21101, 21127, 21157, 21076, + 21132, 21005, 21135, 21055, 21022, 21115, 21116, 21154, 21003, 21118, + 21189, 21130, 21016, 21174, 21034, 21083, 21187, 21097, 21012, 21202, + 21030, 21205, 21155, 21095, 21167, 21199, 21023, 21137, 21008, 21156, + 21146, 21145, 21077, 21018, 21025, 21009, 21119, 21201, 21037, 21209, + 21040, 21200, 21080, 21112, 21085, 21121, 21163, 21158, 21138, 21014, + 21212, 21066, 21105, 21184, 21108, 21179, 21106, 21208, 21073, 21057, + 21091, 21074, 21094, 21017, 21186, 21089, 21069, 21147, 21024, 21084, + 21071, 21010, 21148, 21039, 21207, 21092, 21214, 21117, 21013, 21062, + 21075, 21191, 21053, 21140, 21125, 21060, 21190, 21052, 21195, 21072, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 20495, 20493, 20494, 20492, + 20507, 20503, 20502, 20499, 20500, 20501, 20497, 20496, 20504, 20498, + 20508, 20506, 20592, 20491, 20590, 11007, 20828, 20591, 20490, 20510, + 24045, 31920, 24064, 31939, 24060, 31936, 24139, 32010, 24054, 31930, + 31598, 17902, 24136, 32005, 24141, 32012, 24144, 32015, 24143, 32014, + 39132, 31880, 11233, 25157, 31599, 19581, 19574, 19571, 19577, 19576, + 19579, 19578, 19582, 19580, 20588, 20587, 20509, 20586, 19240, 19239, + 39138, 38827, 38830, 38828, 38831, 38829, 6801, 20584, 19575, 19573, + 19572, 38826, 25816, 31216, 20589, 20585, 40951, 20305, 20291, 20307, + 20385, 20309, 20387, 20306, 20384, 20308, 20386, 20346, 20336, 20348, + 20338, 20350, 20340, 20347, 20337, 20349, 20339, 20310, 20371, 20312, + 20373, 20314, 20375, 20311, 20372, 20313, 20374, 20366, 20331, 20368, + 20333, 20304, 20370, 20335, 20367, 20332, 20369, 20334, 20326, 20328, + 20330, 20327, 20329, 20341, 20321, 20356, 20343, 20315, 20358, 20345, + 20324, 20360, 20342, 20322, 20357, 20344, 20323, 20359, 20351, 20353, + 20355, 20352, 20354, 20298, 20380, 20300, 20382, 20299, 20381, 20361, + 20363, 20365, 20362, 20364, 20294, 20376, 20378, 20377, 20379, 20325, + 20383, 20301, 20302, 40951, 40951, 8288, 8287, 21453, 21452, 20389, + 20388, 20290, 21450, 21380, 21309, 21382, 21443, 21384, 21445, 21381, + 21442, 21383, 21444, 21405, 21395, 21407, 21397, 21409, 21399, 21406, + 21396, 21408, 21398, 21385, 21430, 21387, 21432, 21389, 21434, 21386, + 21431, 21388, 21433, 21420, 21390, 21422, 21392, 21367, 21424, 21394, + 21421, 21391, 21423, 21393, 21347, 21349, 21351, 21348, 21350, 21400, + 21324, 21410, 21402, 21318, 21412, 21404, 21327, 21414, 21401, 21325, + 21411, 21403, 21326, 21413, 21342, 21328, 21345, 21343, 21344, 21372, + 21439, 21374, 21441, 21373, 21440, 21415, 21417, 21419, 21416, 21418, + 21368, 21435, 21437, 21436, 21438, 21346, 21429, 21362, 21363, 21425, + 21427, 21426, 21428, 21449, 21451, 21448, 21446, 21447, 40951, 40951, + 40951, 40951, 40951, 4320, 4328, 4327, 4325, 4324, 4331, 4296, 4316, + 4284, 4312, 4326, 4322, 4329, 4333, 4309, 4315, 4319, 4330, 4308, 4314, + 4318, 4264, 4300, 4276, 4281, 4265, 4282, 4267, 4307, 4269, 4277, 4270, + 4278, 4283, 4289, 4274, 4295, 4332, 4297, 4286, 4292, 4303, 4299, 40951, + 19493, 19537, 19494, 19499, 19500, 19502, 19529, 19540, 19515, 19516, + 19518, 19520, 19527, 19523, 19519, 19526, 19495, 19505, 19539, 19506, + 19530, 19542, 19487, 19482, 19536, 19481, 19492, 19528, 19513, 19566, + 19477, 19480, 19563, 19564, 19484, 19483, 19550, 19549, 19567, 19546, + 19547, 19568, 19555, 19569, 19545, 19544, 19548, 19559, 19485, 19565, + 19486, 19570, 19538, 19501, 19504, 19503, 19517, 19524, 19521, 19522, + 19525, 19496, 19498, 19497, 19491, 19512, 19510, 19507, 19508, 19511, + 19509, 19489, 19490, 19532, 19533, 19535, 19534, 19531, 19514, 19543, + 19552, 19554, 19553, 19488, 19541, 19551, 19556, 19557, 19558, 19561, + 19560, 19562, 19478, 19479, 40951, 20484, 20487, 20486, 20482, 20480, + 20476, 20483, 20478, 20474, 20477, 20485, 20481, 20475, 20488, 20489, + 20479, 4321, 4310, 4323, 4287, 4280, 4279, 4306, 4302, 4294, 4271, 4290, + 4275, 4293, 4298, 4266, 4268, 4273, 4305, 4301, 4291, 4262, 4263, 4261, + 4260, 4285, 4317, 4311, 4259, 4288, 4313, 4304, 4272, 7983, 7986, 7987, + 7985, 7973, 7959, 7965, 7954, 7964, 7977, 7966, 7962, 7955, 7963, 7960, + 7989, 7953, 7972, 7968, 7981, 7988, 7958, 7967, 7976, 7975, 7982, 7980, + 7969, 7971, 7984, 7979, 7974, 7956, 7961, 7970, 7990, 7957, 7978, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 20505, 21365, + 21377, 21378, 21366, 21376, 21352, 21354, 21356, 21353, 21355, 21379, + 21357, 21359, 21361, 21358, 21360, 31078, 31082, 31094, 31088, 31080, + 31086, 31090, 31096, 31071, 31069, 31076, 31092, 31084, 31074, 31079, + 31083, 31095, 31089, 31081, 31087, 31091, 31097, 31072, 31070, 31077, + 31093, 31085, 31075, 31073, 31135, 31134, 40951, 31133, 31129, 31127, + 31108, 31106, 31126, 31118, 31103, 31112, 31128, 31111, 31105, 31131, + 31130, 31110, 31102, 31125, 31123, 31132, 31120, 31113, 31121, 31104, + 31099, 31109, 31116, 31100, 31122, 31124, 31101, 31114, 31098, 31107, + 31115, 31119, 31117, 6620, 6608, 6630, 6609, 6774, 6788, 6776, 6758, + 6755, 6771, 6769, 6751, 31215, 6789, 6795, 6794, 6791, 6790, 6793, 6792, + 6797, 6796, 6775, 6777, 6783, 6782, 6779, 6778, 6568, 6572, 6584, 6578, + 6570, 6576, 6580, 6561, 6559, 6557, 6566, 6582, 6574, 6564, 6569, 6573, + 6585, 6579, 6571, 6577, 6581, 6562, 6560, 6558, 6567, 6583, 6575, 6565, + 6695, 6694, 6563, 22560, 6643, 6639, 6637, 6605, 6603, 6635, 6626, 6600, + 6618, 6638, 6616, 6602, 6641, 6640, 6614, 6598, 6629, 6634, 6607, 6631, + 6619, 6632, 6601, 6594, 6610, 6625, 6615, 6604, 6628, 6599, 6636, 6591, + 6642, 6622, 6595, 6593, 6606, 6596, 6611, 6612, 6624, 6613, 6623, 6633, + 6627, 6597, 6621, 6589, 6617, 6781, 6780, 6785, 6784, 6757, 6759, 6765, + 6764, 6761, 6760, 6763, 6762, 6767, 6766, 6754, 20575, 20578, 20579, + 20517, 20580, 20576, 20577, 20516, 20582, 20583, 20581, 20549, 34191, + 34157, 34159, 24491, 6689, 6691, 6693, 6690, 6692, 6652, 6654, 6656, + 6653, 6655, 6672, 6674, 6676, 6673, 6675, 6677, 6679, 6681, 6678, 6680, + 6662, 6664, 6666, 6663, 6665, 6647, 6649, 6651, 6648, 6650, 6657, 6659, + 6661, 6658, 6660, 6686, 6688, 6687, 6667, 6669, 6671, 6668, 6670, 6682, + 6684, 6683, 6685, 34155, 34116, 34118, 34117, 34119, 34194, 34195, 34358, + 34158, 34151, 34284, 34288, 34203, 34202, 34201, 34166, 34167, 34171, + 34170, 34204, 34169, 34205, 34208, 34206, 34207, 34172, 34173, 34215, + 34216, 34217, 34214, 34213, 34324, 34325, 34332, 34326, 34327, 34142, + 34146, 34147, 34337, 34277, 34278, 34179, 34291, 34292, 34123, 34300, + 34298, 34299, 34126, 34186, 34185, 34125, 34187, 34180, 34297, 34295, + 34181, 34296, 34294, 34128, 34301, 34127, 34184, 34302, 34182, 34183, + 34241, 34242, 34245, 34244, 34243, 34251, 34252, 34253, 34248, 34249, + 34250, 34356, 34355, 34357, 34319, 34320, 34322, 34321, 34317, 34316, + 34338, 20573, 20574, 20556, 20558, 20565, 20564, 20571, 20569, 20560, + 20567, 20559, 20562, 20555, 20557, 20566, 20563, 20572, 20570, 20561, + 20568, 20550, 20554, 20553, 20552, 20551, 34189, 34141, 34121, 34124, + 34289, 34305, 34143, 34145, 34144, 34199, 34152, 34156, 34153, 34154, + 34133, 34293, 34276, 34258, 34240, 34200, 34222, 34246, 34176, 34130, + 34219, 34306, 34279, 34259, 34260, 34273, 34223, 34192, 34218, 34270, + 34174, 34336, 34261, 34274, 34150, 34225, 34165, 34280, 34262, 34255, + 34134, 34209, 34257, 34136, 34239, 34212, 34256, 34135, 34238, 34211, + 34235, 34236, 34290, 34221, 34272, 34175, 34313, 34314, 34315, 34310, + 34281, 34263, 34275, 34311, 34282, 34264, 34266, 34227, 34267, 34312, + 34283, 34265, 34268, 34228, 34269, 34220, 34237, 34120, 34129, 34139, + 34140, 34137, 34132, 34148, 34177, 34178, 34188, 34193, 34224, 34210, + 34226, 34232, 34233, 34230, 34234, 34247, 34254, 34271, 34307, 34308, + 34304, 34309, 34333, 34334, 34354, 34122, 34114, 20548, 20533, 20521, + 20540, 20539, 20546, 20544, 20535, 20542, 20534, 20537, 20532, 20520, + 20541, 20538, 20547, 20545, 20536, 20543, 20522, 20530, 20528, 20527, + 20524, 20523, 20526, 20525, 20531, 20529, 20518, 20519, 34168, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 20267, 20270, 20231, + 20281, 20278, 20226, 20264, 20243, 20260, 20277, 20255, 20261, 20236, + 20238, 20248, 20282, 20235, 20279, 20219, 20225, 20223, 20242, 20262, + 20256, 20244, 20241, 20249, 20240, 20265, 20268, 20247, 20233, 20257, + 20239, 20254, 20234, 20269, 20252, 20250, 20229, 20228, 20246, 20224, + 20227, 20237, 20253, 20251, 20271, 20258, 20266, 20263, 20275, 20230, + 20273, 20221, 20272, 20274, 20276, 20232, 20280, 20245, 20259, 20220, + 20222, 39884, 39891, 39883, 39890, 39888, 39889, 39886, 39887, 40659, + 40660, 40657, 40658, 40656, 40654, 40655, 40663, 40664, 40661, 40662, + 40666, 40665, 40531, 39551, 39552, 39545, 39550, 39548, 39549, 39546, + 39547, 39555, 39556, 39553, 39554, 39536, 39534, 39535, 39559, 39560, + 39557, 39558, 39544, 39542, 39543, 39540, 39541, 39533, 39539, 39538, + 39537, 39565, 39566, 39561, 39564, 39563, 39562, 40263, 40264, 40258, + 40262, 40261, 40259, 40260, 40267, 40268, 40265, 40266, 40252, 40250, + 40251, 40271, 40272, 40269, 40270, 40256, 40257, 40249, 40255, 40254, + 40253, 40277, 40278, 40273, 40276, 40275, 40274, 39519, 39520, 39513, + 39518, 39516, 39517, 39514, 39515, 39523, 39524, 39521, 39522, 39504, + 39502, 39503, 39527, 39528, 39525, 39526, 39512, 39510, 39511, 39508, + 39509, 39501, 39507, 39506, 39505, 39531, 39532, 39529, 39530, 40066, + 40067, 40061, 40065, 40064, 40062, 40063, 40070, 40071, 40068, 40069, + 40074, 40075, 40072, 40073, 40080, 40081, 40076, 40079, 40078, 40077, + 40086, 40087, 40082, 40085, 40084, 40083, 39809, 39810, 39804, 39808, + 39807, 39805, 39806, 39813, 39814, 39811, 39812, 39798, 39796, 39797, + 39817, 39818, 39815, 39816, 39802, 39803, 39795, 39801, 39800, 39799, + 39823, 39819, 39822, 39821, 39820, 40045, 40046, 40040, 40044, 40043, + 40041, 40042, 40049, 40050, 40047, 40048, 40033, 40034, 40031, 40032, + 40053, 40054, 40051, 40052, 40060, 40059, 40038, 40039, 40030, 40037, + 40036, 40035, 40057, 40058, 40055, 40056, 39690, 39691, 39688, 39689, + 39686, 39687, 39684, 39685, 39683, 39681, 39682, 39700, 39701, 39696, + 39699, 39698, 39697, 39694, 39695, 39692, 39693, 40509, 40510, 40503, + 40508, 40506, 40507, 40504, 40505, 40513, 40514, 40511, 40512, 40517, + 40518, 40515, 40516, 40502, 40501, 40523, 40524, 40519, 40522, 40521, + 40520, 40529, 40530, 40525, 40528, 40527, 40526, 39668, 39669, 39663, + 39667, 39666, 39664, 39665, 39675, 39676, 39673, 39674, 39657, 39656, + 39679, 39680, 39677, 39678, 39672, 39670, 39671, 39661, 39662, 39655, + 39660, 39659, 39658, 40488, 40489, 40483, 40487, 40486, 40484, 40485, + 40495, 40496, 40493, 40494, 40476, 40477, 40474, 40475, 40499, 40500, + 40497, 40498, 40492, 40490, 40491, 40481, 40482, 40473, 40480, 40479, + 40478, 39642, 39643, 39637, 39641, 39640, 39638, 39639, 39649, 39650, + 39647, 39648, 39631, 39629, 39630, 39653, 39654, 39651, 39652, 39646, + 39644, 39645, 39635, 39636, 39628, 39634, 39633, 39632, 40092, 40093, + 40088, 40091, 40090, 40089, 40099, 40100, 40097, 40098, 40103, 40104, + 40101, 40102, 40096, 40094, 40095, 40109, 40110, 40105, 40108, 40107, + 40106, 39839, 39840, 39833, 39838, 39836, 39837, 39834, 39835, 39843, + 39844, 39841, 39842, 39828, 39827, 39825, 39826, 39824, 39832, 39830, + 39831, 39829, 40237, 40238, 40232, 40236, 40235, 40233, 40234, 40241, + 40239, 40240, 40226, 40224, 40225, 40247, 40248, 40245, 40246, 40244, + 40242, 40243, 40230, 40231, 40223, 40229, 40228, 40227, 39777, 39778, + 39772, 39776, 39775, 39773, 39774, 39787, 39788, 39785, 39786, 39766, + 39764, 39765, 39784, 39782, 39783, 39781, 39779, 39780, 39770, 39771, + 39763, 39769, 39768, 39767, 39793, 39794, 39789, 39792, 39791, 39790, + 39992, 39993, 39986, 39991, 39989, 39990, 39987, 39988, 39996, 39997, + 39994, 39995, 39976, 39977, 39974, 39975, 40000, 40001, 39998, 39999, + 39985, 39983, 39984, 39981, 39982, 39973, 39980, 39979, 39978, 40006, + 40007, 40002, 40005, 40004, 40003, 39746, 39747, 39740, 39745, 39743, + 39744, 39741, 39742, 39750, 39751, 39748, 39749, 39733, 39734, 39731, + 39732, 39758, 39759, 39756, 39757, 39754, 39755, 39752, 39753, 39738, + 39739, 39730, 39737, 39736, 39735, 39959, 39960, 39954, 39958, 39957, + 39955, 39956, 39963, 39964, 39961, 39962, 39948, 39946, 39947, 39971, + 39972, 39969, 39970, 39967, 39968, 39965, 39966, 39952, 39953, 39945, + 39951, 39950, 39949, 39706, 39707, 39702, 39705, 39703, 39704, 39720, + 39721, 39718, 39719, 39711, 39712, 39709, 39710, 39728, 39729, 39726, + 39727, 39724, 39725, 39722, 39723, 39716, 39717, 39708, 39715, 39714, + 39713, 40029, 40028, 40022, 40023, 40020, 40021, 40011, 40009, 40010, + 40026, 40027, 40024, 40025, 40019, 40017, 40018, 40015, 40016, 40008, + 40014, 40013, 40012, 39858, 39859, 39852, 39857, 39855, 39856, 39853, + 39854, 39862, 39863, 39860, 39861, 39847, 39848, 39845, 39846, 39866, + 39867, 39864, 39865, 39851, 39849, 39850, 40119, 40117, 40118, 40122, + 40123, 40120, 40121, 40112, 40113, 40111, 40126, 40127, 40124, 40125, + 40116, 40114, 40115, 39762, 39761, 39760, 39877, 39878, 39875, 39876, + 39870, 39871, 39868, 39869, 39881, 39882, 39879, 39880, 39874, 39872, + 39873, 40543, 40544, 40541, 40542, 40534, 40532, 40533, 40540, 40538, + 40539, 40537, 40535, 40536, 40606, 40607, 40601, 40605, 40604, 40602, + 40603, 40642, 40643, 40640, 40641, 40595, 40593, 40594, 40646, 40647, + 40644, 40645, 40639, 40637, 40638, 40599, 40600, 40592, 40598, 40597, + 40596, 40652, 40653, 40648, 40651, 40650, 40649, 39612, 39613, 39606, + 39611, 39609, 39610, 39607, 39608, 39616, 39617, 39614, 39615, 39597, + 39595, 39596, 39620, 39621, 39618, 39619, 39605, 39603, 39604, 39601, + 39602, 39594, 39600, 39599, 39598, 39626, 39627, 39622, 39625, 39624, + 39623, 40620, 40621, 40614, 40619, 40617, 40618, 40615, 40616, 40624, + 40625, 40622, 40623, 40613, 40611, 40612, 40610, 40608, 40609, 40630, + 40626, 40629, 40628, 40627, 40635, 40636, 40631, 40634, 40633, 40632, + 40209, 40210, 40204, 40208, 40207, 40205, 40206, 40213, 40214, 40211, + 40212, 40197, 40196, 40203, 40202, 40222, 40221, 40201, 40195, 40200, + 40199, 40198, 40219, 40220, 40215, 40218, 40217, 40216, 40454, 40455, + 40449, 40453, 40452, 40450, 40451, 40461, 40462, 40459, 40460, 40443, + 40441, 40442, 40465, 40466, 40463, 40464, 40458, 40456, 40457, 40447, + 40448, 40440, 40446, 40445, 40444, 40471, 40472, 40467, 40470, 40469, + 40468, 40390, 40391, 40385, 40389, 40388, 40386, 40387, 40397, 40398, + 40395, 40396, 40401, 40402, 40399, 40400, 40394, 40392, 40393, 40405, + 40406, 40403, 40404, 40411, 40412, 40407, 40410, 40409, 40408, 40576, + 40577, 40574, 40575, 40568, 40566, 40567, 40584, 40585, 40582, 40583, + 40580, 40581, 40578, 40579, 40572, 40573, 40565, 40571, 40570, 40569, + 40590, 40591, 40586, 40589, 40588, 40587, 39578, 39579, 39576, 39577, + 39570, 39571, 39568, 39569, 39586, 39587, 39584, 39585, 39582, 39583, + 39580, 39581, 39575, 39567, 39574, 39573, 39572, 39592, 39593, 39588, + 39591, 39590, 39589, 40358, 40357, 40337, 40336, 40349, 40350, 40347, + 40348, 40345, 40346, 40343, 40344, 40341, 40342, 40335, 40340, 40339, + 40338, 40355, 40356, 40351, 40354, 40353, 40352, 40158, 40159, 40156, + 40157, 40155, 40153, 40154, 40162, 40163, 40160, 40161, 40168, 40169, + 40164, 40167, 40166, 40165, 40174, 40175, 40170, 40173, 40172, 40171, + 40424, 40425, 40422, 40423, 40416, 40414, 40415, 40432, 40433, 40430, + 40431, 40428, 40429, 40426, 40427, 40420, 40421, 40413, 40419, 40418, + 40417, 40438, 40439, 40434, 40437, 40436, 40435, 40373, 40374, 40371, + 40372, 40362, 40360, 40361, 40377, 40378, 40375, 40376, 40370, 40368, + 40369, 40366, 40367, 40359, 40365, 40364, 40363, 40383, 40384, 40379, + 40382, 40381, 40380, 39933, 39934, 39927, 39932, 39930, 39931, 39928, + 39929, 39920, 39921, 39918, 39919, 39937, 39938, 39935, 39936, 39925, + 39926, 39917, 39924, 39923, 39922, 39943, 39944, 39939, 39942, 39941, + 39940, 40295, 40296, 40289, 40294, 40292, 40293, 40290, 40291, 40282, + 40283, 40280, 40281, 40299, 40300, 40297, 40298, 40287, 40288, 40279, + 40286, 40285, 40284, 40305, 40306, 40301, 40304, 40303, 40302, 39907, + 39908, 39901, 39906, 39904, 39905, 39902, 39903, 39895, 39893, 39894, + 39911, 39912, 39909, 39910, 39899, 39900, 39892, 39898, 39897, 39896, + 39915, 39916, 39913, 39914, 40141, 40142, 40135, 40140, 40138, 40139, + 40136, 40137, 40130, 40129, 40145, 40146, 40143, 40144, 40134, 40128, + 40133, 40132, 40131, 40151, 40152, 40147, 40150, 40149, 40148, 40189, + 40190, 40183, 40188, 40186, 40187, 40184, 40185, 40179, 40177, 40178, + 40193, 40194, 40191, 40192, 40181, 40182, 40176, 40180, 40551, 40552, + 40545, 40550, 40548, 40549, 40546, 40547, 40564, 40563, 40555, 40556, + 40553, 40554, 40561, 40562, 40557, 40560, 40559, 40558, 40323, 40324, + 40317, 40322, 40320, 40321, 40318, 40319, 40310, 40311, 40308, 40309, + 40327, 40328, 40325, 40326, 40315, 40316, 40307, 40314, 40313, 40312, + 40333, 40334, 40329, 40332, 40331, 40330, 40951, 40951, 40951, 39498, + 39471, 39469, 39476, 39450, 39486, 39457, 39459, 39475, 39462, 39473, + 39446, 39474, 39492, 39480, 39461, 39487, 39460, 39493, 39451, 39454, + 39447, 39456, 39477, 39488, 39500, 39465, 39496, 39481, 39464, 39491, + 39489, 39485, 39490, 39497, 39468, 39478, 39467, 39458, 39466, 39499, + 39455, 39482, 39472, 39449, 39448, 39453, 39463, 39483, 39494, 39484, + 39452, 39495, 39479, 39470, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 25094, 25083, 25082, 25066, 25064, 25063, 25077, + 25081, 25080, 25096, 25075, 25074, 25065, 25062, 25061, 25098, 25071, + 25097, 25085, 25088, 25089, 25070, 25079, 25100, 25078, 25095, 25099, + 25084, 25087, 25076, 25090, 25091, 25072, 25073, 25101, 25092, 25067, + 25068, 25069, 25093, 25057, 25060, 25058, 25056, 25059, 25055, 25102, + 25103, 38225, 38226, 38062, 38211, 38212, 38185, 37988, 37995, 38098, + 38071, 38105, 38045, 38171, 38199, 38023, 38016, 37974, 37967, 38089, + 38192, 37981, 38124, 38009, 38002, 38037, 38030, 38164, 38178, 38143, + 38206, 38084, 38130, 38051, 38112, 38157, 38150, 38234, 38235, 38066, + 38067, 38220, 38221, 38187, 37990, 37997, 38100, 38077, 38107, 38048, + 38173, 38201, 38025, 38018, 37976, 37969, 38093, 38194, 37983, 38126, + 38011, 38004, 38039, 38032, 38166, 38180, 38145, 38208, 38085, 38135, + 38056, 38114, 38159, 38152, 38232, 38233, 38137, 38064, 38065, 38218, + 38219, 38186, 37989, 37996, 38099, 38075, 38076, 38106, 38047, 38172, + 38200, 38024, 38017, 37975, 37968, 38092, 38193, 37982, 38125, 38010, + 38003, 38038, 38031, 38165, 38179, 38144, 38207, 38081, 38082, 38134, + 38055, 38113, 38158, 38151, 38229, 38230, 38060, 38215, 38216, 38183, + 37986, 37993, 38096, 38074, 38103, 38043, 38169, 38197, 38021, 38014, + 37972, 37965, 38091, 38190, 37979, 38122, 38007, 38000, 38035, 38028, + 38162, 38176, 38141, 38204, 38080, 38133, 38054, 38110, 38155, 38148, + 38236, 38237, 38068, 38069, 38222, 38223, 38188, 37991, 37998, 38101, + 38078, 38108, 38049, 38174, 38202, 38026, 38019, 37977, 37970, 38094, + 38195, 37984, 38127, 38012, 38005, 38040, 38033, 38167, 38181, 38146, + 38209, 38086, 38136, 38057, 38115, 38160, 38153, 38228, 38231, 38139, + 38058, 38059, 38214, 38217, 38182, 37985, 37992, 38095, 38073, 38102, + 38041, 38042, 38168, 38196, 38020, 38013, 37971, 37964, 38090, 38189, + 37978, 38116, 38006, 37999, 38034, 38027, 38161, 38175, 38140, 38203, + 38079, 38132, 38053, 38109, 38154, 38147, 38224, 38227, 38138, 38061, + 38063, 38210, 38213, 38184, 37987, 37994, 38097, 38070, 38072, 38104, + 38044, 38046, 38170, 38198, 38022, 38015, 37973, 37966, 38087, 38191, + 37980, 38123, 38008, 38001, 38036, 38029, 38163, 38177, 38142, 38205, + 38083, 38129, 38131, 38050, 38052, 38111, 38156, 38149, 38128, 38088, + 37961, 37962, 37963, 38119, 38120, 38117, 38241, 38244, 38247, 38246, + 38250, 38242, 38249, 38240, 38238, 38245, 38248, 38239, 38243, 38257, + 38259, 38256, 38255, 38252, 38251, 38254, 38253, 38260, 38258, 38121, + 38118, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 10283, 10484, 10172, 10331, 10122, 10425, 10227, 10386, 10168, + 10327, 10248, 10411, 10156, 10315, 10115, 10412, 10276, 10477, 10224, + 10383, 10124, 10427, 10225, 10384, 10164, 10323, 10157, 10316, 10221, + 10380, 10278, 10479, 10123, 10426, 10267, 10445, 10264, 10442, 10265, + 10443, 10247, 10410, 10154, 10313, 10169, 10328, 10298, 8145, 8137, 8088, + 8138, 33776, 8112, 8101, 8115, 8111, 8120, 8113, 8110, 8107, 8143, 8132, + 10297, 10301, 10178, 10337, 10176, 10335, 10288, 10489, 10166, 10325, + 10174, 10333, 10128, 10453, 10136, 10462, 10132, 10457, 10131, 10456, + 10134, 10460, 10212, 10371, 10262, 10440, 10170, 10329, 10165, 10324, + 27730, 27767, 8096, 8104, 3413, 2779, 3416, 2781, 3412, 3388, 3405, 3415, + 2808, 3414, 2784, 3385, 3391, 3390, 2782, 2788, 3404, 2807, 2801, 2787, + 3400, 2795, 3395, 3401, 3394, 3398, 2777, 2773, 2805, 2804, 2799, 3407, + 3397, 3408, 3409, 2806, 2771, 3381, 2800, 2803, 3384, 3410, 3382, 2768, + 3393, 2786, 2793, 2810, 3387, 3392, 2772, 2796, 2797, 2798, 3396, 3383, + 2770, 2769, 3411, 2809, 2785, 3386, 2783, 2774, 2790, 3389, 2789, 2792, + 3406, 2780, 2794, 2791, 3403, 2778, 3402, 2802, 3399, 2767, 2775, 2776, + 2764, 2763, 3417, 3419, 2766, 2765, 3418, 3420, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 27727, 27723, 27726, 27722, 27728, + 27724, 27729, 27725, 27782, 27797, 27825, 27804, 27787, 27781, 27796, + 27824, 27803, 27786, 27783, 27798, 27826, 27808, 27788, 27774, 27773, + 27775, 27819, 27838, 27835, 27837, 27836, 27816, 27986, 27987, 22867, + 23470, 22868, 23471, 22907, 23532, 23133, 23894, 23132, 23851, 22809, + 23392, 22810, 23393, 23274, 23282, 22783, 23348, 22784, 23349, 22785, + 23350, 22781, 23346, 22782, 23347, 22786, 23351, 23077, 23775, 22948, + 23584, 22945, 23581, 22949, 23585, 22795, 23369, 22967, 23608, 23000, + 23683, 23001, 23685, 23047, 23723, 23052, 23728, 23050, 23726, 23053, + 23729, 23059, 23740, 23058, 23737, 23074, 23765, 23079, 23778, 23174, + 23944, 23183, 23956, 23178, 23951, 23127, 23845, 23128, 23846, 23182, + 23955, 22869, 23488, 22936, 23571, 22811, 23394, 27995, 23430, 23629, + 23643, 23666, 23777, 23259, 23889, 23941, 22929, 23560, 22930, 23561, + 22931, 23117, 23857, 23122, 23887, 22932, 23563, 22933, 23564, 22934, + 23565, 27802, 27771, 27848, 23100, 23801, 23120, 23613, 23290, 22993, + 23657, 22805, 23382, 23379, 23526, 22789, 23354, 22876, 23495, 23179, + 23952, 23180, 23953, 23181, 23954, 22884, 23506, 22952, 23589, 22996, + 23662, 23072, 23762, 23096, 23799, 22903, 23076, 23103, 22955, 23099, + 23281, 23121, 23124, 22939, 22812, 22796, 23371, 23045, 23721, 23170, + 23932, 22889, 23511, 22890, 23512, 22891, 23513, 23042, 23712, 22778, + 23343, 22803, 23097, 23220, 22816, 23396, 23093, 23793, 23080, 23092, + 23792, 40951, 40951, 22808, 23388, 40951, 23423, 40951, 23420, 22982, + 23642, 23102, 23819, 22971, 23618, 22972, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27703, 27706, 27715, + 23075, 23766, 23230, 27698, 27927, 23264, 23228, 23229, 23226, 23227, + 23225, 34770, 34772, 34782, 34774, 34771, 34773, 34781, 34762, 34761, + 34758, 34757, 34780, 34756, 34755, 34760, 34759, 34750, 34749, 34744, + 34743, 34752, 34751, 34746, 34745, 34768, 34764, 34763, 34754, 34753, + 34767, 34748, 34766, 34747, 34769, 34765, 34784, 34786, 34787, 34785, + 34783, 34775, 34776, 34777, 34778, 34779, 40951, 40951, 40951, 29388, + 29387, 29390, 29385, 29386, 29389, 29382, 29381, 29384, 29383, 40951, + 40951, 40951, 40951, 40951, 40951, 31352, 31351, 31340, 31341, 31328, + 31330, 31362, 31343, 31350, 31349, 31333, 31344, 31354, 31353, 31359, + 31364, 31346, 31345, 31332, 31367, 31355, 31356, 31334, 31369, 31366, + 31363, 31335, 31336, 31361, 31325, 31370, 31372, 31357, 31371, 31365, + 31368, 31360, 31339, 31358, 31377, 31378, 31348, 31347, 31331, 31342, + 31326, 31338, 31337, 31327, 31376, 31379, 31329, 31375, 31380, 31374, + 31373, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32496, + 32498, 32445, 32446, 32466, 32467, 32462, 32463, 32457, 32458, 32459, + 32460, 32489, 32490, 32447, 32464, 32465, 32448, 32486, 32485, 32482, + 32481, 32470, 32480, 32479, 32484, 32483, 32472, 32454, 32453, 32450, + 32449, 32471, 32456, 32455, 32452, 32451, 32473, 32488, 32487, 32478, + 32477, 32492, 32494, 32493, 32469, 32461, 32474, 32475, 32476, 32491, + 32468, 32511, 32512, 32523, 32524, 32515, 32516, 32517, 32518, 32519, + 32520, 32525, 32526, 32513, 32521, 32522, 32514, 32497, 32495, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32500, 32499, 32507, + 32509, 32506, 32505, 32502, 32501, 32504, 32503, 32510, 32508, 40951, + 40951, 40951, 40951, 40951, 40951, 8163, 8165, 8162, 8161, 8158, 8157, + 8160, 8159, 8166, 8164, 8154, 8155, 8150, 8151, 8152, 8153, 8149, 8156, + 10770, 10764, 10777, 10766, 10765, 10763, 10778, 10667, 10665, 10670, + 10771, 10821, 10676, 10788, 21585, 21587, 21584, 21583, 21580, 21579, + 21582, 21581, 21588, 21586, 21549, 21548, 21559, 21545, 21553, 21552, + 21566, 21546, 21555, 21543, 21547, 21551, 21550, 21561, 21558, 21556, + 21562, 21565, 21560, 21564, 21554, 21544, 21563, 21557, 21567, 21541, + 21568, 21542, 21577, 21574, 21576, 21575, 21578, 21573, 21571, 21572, + 21569, 21570, 31838, 31835, 31827, 31843, 31834, 31829, 31840, 31832, + 31831, 31833, 31837, 31825, 31842, 31841, 31839, 31845, 31844, 31836, + 31830, 31826, 31828, 31824, 31846, 31853, 31855, 31848, 31851, 31854, + 31852, 31850, 31849, 31821, 31820, 31823, 31822, 31856, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 31847, + 19228, 19231, 19232, 19229, 19186, 19187, 19195, 19189, 19191, 19194, + 19188, 19184, 19190, 19192, 19185, 19151, 19153, 19154, 19167, 19174, + 19181, 19216, 19133, 19140, 19214, 19218, 19164, 19237, 19220, 40951, + 40951, 40951, 20903, 20901, 20905, 20904, 20875, 20843, 20842, 20844, + 20884, 20863, 20849, 20850, 20882, 20876, 20883, 20845, 20846, 20847, + 20859, 20860, 20848, 20857, 20858, 20873, 20852, 20874, 20851, 20871, + 20872, 20838, 20839, 20854, 20869, 20870, 20840, 20841, 20853, 20861, + 20862, 20855, 20856, 20879, 20881, 20864, 20865, 20878, 20880, 20867, + 20868, 20866, 20877, 20902, 20908, 20910, 20911, 20912, 20906, 20907, + 20909, 20914, 20913, 20834, 20835, 20836, 20899, 20837, 20885, 20888, + 20897, 20890, 20895, 20893, 20891, 20889, 20886, 20887, 20892, 20900, + 40951, 20898, 20921, 20923, 20920, 20919, 20916, 20915, 20918, 20917, + 20924, 20922, 40951, 40951, 40951, 40951, 20894, 20896, 28607, 28605, + 28600, 28597, 28603, 28693, 28671, 28623, 28635, 28631, 28630, 28633, + 28632, 28625, 28624, 28622, 28735, 28737, 28734, 28733, 28730, 28729, + 28732, 28731, 28738, 28736, 28634, 28627, 28626, 28629, 28628, 40951, + 6245, 6263, 6265, 6262, 6246, 6264, 6254, 6253, 6250, 6249, 6225, 6226, + 6248, 6247, 6252, 6251, 6227, 6229, 6228, 6256, 6255, 6242, 6241, 6230, + 6231, 6240, 6236, 6235, 6234, 6239, 6238, 6232, 6233, 6237, 6261, 6259, + 6258, 6260, 6243, 6244, 6257, 6270, 6273, 6274, 6279, 6277, 6276, 6275, + 6271, 6272, 6278, 6213, 6211, 6210, 6212, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 6219, 6218, 6215, 6207, 6217, 6223, + 6214, 6221, 6224, 6222, 6220, 6216, 6209, 6208, 40951, 40951, 6286, 6288, + 6285, 6284, 6281, 6280, 6283, 6282, 6289, 6287, 40951, 40951, 6266, 6268, + 6267, 6269, 28577, 28570, 28569, 28574, 28573, 28567, 28566, 28565, + 28563, 28562, 28564, 28568, 28579, 28572, 28571, 28576, 28670, 28580, + 28581, 28578, 28667, 28668, 28669, 28708, 28710, 28709, 28552, 28704, + 28695, 28696, 28616, 28617, 35254, 35230, 35253, 35229, 35252, 35228, + 35267, 35243, 35261, 35237, 35256, 35232, 35255, 35231, 35272, 35248, + 35262, 35238, 35265, 35241, 35260, 35236, 35259, 35235, 35263, 35239, + 35264, 35240, 35258, 35234, 35257, 35233, 35266, 35242, 35270, 35246, + 35274, 35250, 35271, 35247, 35269, 35245, 35273, 35249, 35268, 35244, + 35275, 35251, 35276, 35288, 35296, 35293, 35292, 35298, 35299, 35277, + 35297, 35294, 35295, 35287, 35291, 35290, 35289, 35286, 35283, 35284, + 35285, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 35279, 35280, 35282, 35281, 35278, + 27037, 27038, 26996, 27013, 27024, 27023, 27000, 26999, 27012, 27018, + 27019, 27051, 27053, 27043, 27045, 27044, 26991, 26988, 26990, 27040, + 27041, 27055, 27056, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 17212, 17210, 17209, 17208, 17207, 17211, 40951, + 40951, 16906, 16904, 16903, 16902, 16901, 16905, 40951, 40951, 16921, + 16919, 16918, 16917, 16916, 16920, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 16862, 16868, 16866, 16863, 16865, 16864, + 16867, 40951, 16847, 16853, 16851, 16848, 16850, 16849, 16852, 40951, + 23362, 23338, 23368, 23363, 23459, 23622, 23811, 23611, 23602, 23605, + 23633, 23647, 23473, 23366, 23367, 23717, 23567, 23860, 23859, 23861, + 23862, 23821, 23258, 23764, 23421, 23744, 23422, 23808, 23809, 23365, + 23931, 23897, 23940, 23883, 23936, 23385, 23386, 23387, 23972, 23969, + 23970, 23971, 23983, 27685, 27908, 27917, 27918, 27975, 23802, 23570, + 23718, 23942, 23566, 18401, 23428, 23892, 23865, 27972, 27821, 27845, + 40951, 40951, 40951, 40951, 6468, 6469, 6470, 6471, 6472, 6473, 6431, + 6467, 6432, 6433, 6434, 6435, 6436, 6396, 6397, 6398, 6399, 6400, 6401, + 6437, 6438, 6439, 6440, 6441, 6442, 6443, 6444, 6445, 6446, 6447, 6402, + 6395, 6403, 6404, 6405, 6406, 6407, 6408, 6449, 6450, 6451, 6452, 6453, + 6454, 6410, 6409, 6411, 6412, 6413, 6414, 6415, 6389, 6428, 6390, 6429, + 6391, 6430, 6392, 6393, 6394, 6388, 6416, 6417, 6418, 6419, 6420, 6421, + 6422, 6423, 6424, 6425, 6426, 6427, 6455, 6456, 6457, 6458, 6459, 6460, + 6461, 27005, 27017, 27027, 27029, 27014, 27008, 26995, 27020, 27007, + 27010, 27022, 27033, 27035, 27031, 27036, 27025, 27016, 27034, 27002, + 27003, 27032, 26994, 27004, 26998, 27001, 26997, 26993, 27006, 27028, + 27030, 27015, 27009, 27021, 27011, 27026, 27047, 27050, 27042, 27049, + 27048, 27052, 27046, 27054, 26992, 27039, 26989, 40951, 40951, 27063, + 27065, 27062, 27061, 27058, 27057, 27060, 27059, 27066, 27064, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 19425, 19423, 19456, 19457, 19458, 19434, 19435, 19467, 19469, + 19402, 19400, 19399, 19403, 19409, 19410, 19412, 19411, 19416, 19413, + 19414, 19418, 19386, 19384, 40951, 40951, 40951, 40951, 19282, 19283, + 19351, 19352, 19371, 19365, 19366, 19369, 19368, 19367, 19326, 19311, + 19350, 19317, 19321, 19320, 19334, 19333, 19254, 19279, 19272, 19357, + 19270, 19277, 19307, 19300, 19306, 19361, 19302, 19305, 19304, 19345, + 19338, 19355, 19356, 19344, 19342, 19341, 19346, 19348, 19293, 19292, + 19378, 19379, 19243, 19242, 19358, 19297, 19295, 40951, 40951, 40951, + 40951, 7582, 7583, 7584, 7585, 7586, 7587, 7588, 7589, 7590, 7591, 7592, + 7593, 7594, 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, + 7605, 7606, 7607, 7608, 7609, 7610, 7611, 7612, 7613, 7614, 7615, 7616, + 7617, 7618, 7619, 7620, 7621, 7622, 7623, 7624, 7625, 7626, 7627, 7628, + 7629, 7630, 7631, 7632, 7633, 7634, 7635, 7636, 7637, 7638, 7639, 7640, + 7641, 7642, 7643, 7644, 7645, 7646, 7647, 7648, 7649, 7650, 7651, 7652, + 7653, 7654, 7655, 7656, 7657, 7658, 7659, 7660, 7661, 7662, 7663, 7664, + 7665, 7666, 7667, 7668, 7669, 7670, 7671, 7672, 7673, 7674, 7675, 7676, + 7677, 7678, 7679, 7680, 7681, 7682, 7683, 7684, 7685, 7686, 7687, 7688, + 7689, 7690, 7691, 7692, 7693, 7694, 7695, 7696, 7697, 7698, 7699, 7700, + 7701, 7702, 7703, 7704, 7705, 7706, 7707, 7708, 7709, 7710, 7711, 7712, + 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, 7721, 7722, 7723, 7724, + 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, 7733, 7734, 7735, 7736, + 7737, 7738, 7739, 7740, 7741, 7742, 7743, 7744, 7745, 7746, 7747, 7748, + 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756, 7757, 7758, 7759, 7760, + 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, 7769, 7770, 7771, 7772, + 7773, 7774, 7775, 7776, 7777, 7778, 7779, 7780, 7781, 7782, 7783, 7784, + 7785, 7786, 7787, 7788, 7789, 7790, 7791, 7792, 7793, 7794, 7795, 7796, + 7797, 7798, 7799, 7800, 7801, 7802, 7803, 7804, 7805, 7806, 7807, 7808, + 7809, 7810, 7811, 7812, 7813, 7814, 7815, 7816, 7817, 7818, 7819, 7820, + 7821, 7822, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, + 7833, 7834, 7835, 7836, 7837, 7380, 7381, 7382, 7383, 7384, 7385, 7386, + 7387, 7388, 7389, 7390, 7391, 7392, 7393, 7394, 7395, 7396, 7397, 7398, + 7399, 7400, 7401, 7402, 7403, 7404, 7405, 7406, 7407, 7408, 7409, 7410, + 7411, 7412, 7413, 7414, 7415, 7416, 7417, 7418, 7419, 7420, 7421, 7422, + 7423, 7424, 7425, 7426, 7427, 7428, 7429, 7430, 7431, 7432, 7433, 7434, + 7435, 7436, 7437, 7438, 7439, 7440, 7441, 7442, 7443, 7444, 7445, 7446, + 7447, 7448, 7449, 7450, 7451, 7452, 7453, 7454, 7455, 7456, 7457, 7458, + 7459, 7460, 7461, 7462, 7463, 7464, 7465, 7466, 7467, 7468, 7469, 7470, + 7471, 7472, 7473, 7474, 7475, 7366, 7367, 7368, 7369, 7370, 7371, 7372, + 7373, 7374, 7375, 7376, 7377, 7378, 7379, 40951, 40951, 7476, 7477, 7478, + 7479, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, 7490, + 7491, 7492, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, 7502, + 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510, 7511, 7512, 7513, 7514, + 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, 7526, + 7527, 7528, 7529, 7530, 7531, 7532, 7533, 7534, 7535, 7536, 7537, 7538, + 7539, 7540, 7541, 7542, 7543, 7544, 7545, 7546, 7547, 7548, 7549, 7550, + 7551, 7552, 7553, 7554, 7555, 7556, 7557, 7558, 7559, 7560, 7561, 7562, + 7563, 7564, 7565, 7566, 7567, 7568, 7569, 7570, 7571, 7572, 7573, 7574, + 7575, 7576, 7577, 7578, 7579, 7580, 7581, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 24005, 24008, 24009, 24006, 24007, 24011, + 24012, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 2506, 2507, 2505, 2508, 2504, 40951, 40951, 40951, + 40951, 40951, 19891, 19918, 19896, 19827, 19883, 19886, 19888, 19887, + 19882, 19889, 19885, 19884, 19828, 19861, 19862, 19859, 19860, 19825, + 19826, 19824, 19832, 19874, 19872, 19849, 19881, 19854, 40951, 19866, + 19892, 19840, 19835, 19876, 40951, 19878, 40951, 19852, 19856, 40951, + 19842, 19838, 40951, 19870, 19847, 19864, 19858, 19868, 19880, 19831, + 19834, 19837, 19893, 1158, 1157, 1188, 1186, 1187, 1189, 1417, 1415, + 1416, 1418, 1183, 1181, 1182, 1184, 1547, 1545, 1546, 1548, 1527, 1525, + 1526, 1528, 1542, 1540, 1541, 1543, 1560, 1558, 1559, 1561, 1422, 1420, + 1421, 1423, 1224, 1222, 1223, 1225, 1395, 1393, 1394, 1396, 1504, 1502, + 1503, 1505, 1509, 1507, 1508, 1510, 1214, 1213, 1205, 1204, 1228, 1227, + 1217, 1216, 1324, 1323, 1452, 1451, 1347, 1345, 1346, 1348, 1258, 1256, + 1257, 1259, 1270, 1268, 1269, 1271, 1386, 1384, 1385, 1387, 1400, 1399, + 1456, 1454, 1455, 1457, 1300, 1299, 1295, 1293, 1294, 1296, 1304, 1302, + 1303, 1305, 1584, 1583, 1582, 1581, 2368, 2367, 2376, 2375, 2372, 2371, + 2370, 2369, 2380, 2379, 2366, 2374, 2373, 2382, 2378, 2377, 2381, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 1390, 1388, 1389, 1391, 1553, 1552, + 1605, 1604, 1602, 1601, 1551, 1563, 1562, 1351, 1350, 1354, 1353, 1618, + 1616, 1617, 1619, 1554, 1555, 2058, 2057, 2060, 2059, 2079, 2078, 2087, + 2086, 2077, 2076, 2083, 2082, 2063, 2061, 2062, 2112, 2110, 2111, 1238, + 1236, 1237, 1239, 2069, 2067, 2073, 2056, 2081, 1660, 1651, 1656, 1663, + 1658, 1669, 2021, 2009, 2016, 2029, 2032, 2037, 2039, 2044, 2041, 2050, + 1738, 1744, 1723, 1718, 1777, 1773, 1779, 1928, 1921, 1933, 1941, 1900, + 1904, 1682, 1674, 1678, 1684, 2002, 1997, 2114, 1631, 1627, 1712, 1708, + 1696, 1701, 1692, 1699, 1694, 1703, 1882, 1878, 1880, 1884, 1753, 1755, + 1763, 1761, 1758, 1769, 1751, 1772, 1805, 1797, 1809, 1815, 1789, 1818, + 1836, 1825, 1830, 1840, 1819, 1841, 1857, 1847, 1869, 1862, 1865, 1872, + 1733, 1729, 1730, 1731, 2097, 2090, 2099, 2105, 2054, 2109, 2038, 1895, + 1643, 1949, 1952, 1956, 1950, 1953, 1955, 2085, 2084, 2071, 2075, 2055, + 2080, 1667, 1666, 1661, 1665, 1657, 1668, 2035, 2034, 2027, 2033, 2031, + 2036, 2048, 2047, 2042, 2046, 2040, 2049, 1693, 1702, 1879, 1883, 1752, + 1756, 1767, 1750, 1771, 1813, 1788, 1817, 1820, 1838, 1870, 1867, 1860, + 1866, 1864, 1871, 1642, 2107, 2094, 2103, 2093, 2053, 2108, 2068, 2066, + 2070, 2072, 2064, 1659, 1650, 1655, 1662, 1652, 2020, 2008, 2015, 2028, + 2010, 2043, 1737, 1743, 1722, 1717, 1776, 1778, 1927, 1920, 1932, 1940, + 1899, 1907, 1903, 1681, 1673, 1677, 1683, 2001, 2113, 1630, 1626, 1711, + 1707, 1695, 1700, 1691, 1698, 1881, 1877, 1754, 1762, 1760, 1757, 1768, + 1804, 1796, 1808, 1814, 1798, 1835, 1824, 1829, 1839, 1856, 1846, 1868, + 1861, 1848, 1732, 1728, 1734, 2096, 2089, 2098, 2104, 2091, 2074, 2065, + 1664, 1653, 2030, 2011, 2045, 2051, 1942, 1924, 1979, 1959, 1759, 1770, + 1816, 1863, 1849, 2106, 2092, 1957, 1951, 1954, 2000, 2004, 1633, 1635, + 1710, 1714, 1944, 1948, 1981, 1989, 1720, 1725, 1746, 1748, 1775, 1781, + 1906, 1911, 1680, 1688, 1970, 1965, 1984, 1978, 1987, 1946, 1909, 1686, + 1999, 2003, 1632, 1634, 1709, 1713, 1943, 1947, 1980, 1988, 1719, 1724, + 1745, 1747, 1774, 1780, 1905, 1910, 1679, 1687, 1968, 1963, 1982, 1976, + 1986, 1945, 1908, 1685, 1969, 1964, 1983, 1977, 1923, 1958, 1996, 1929, + 1922, 1934, 1971, 1966, 1985, 1998, 2115, 1644, 1645, 30650, 30651, 1892, + 1887, 1891, 1888, 1889, 1890, 1918, 1637, 1639, 1638, 1636, 1886, 1917, + 1640, 1991, 1893, 2018, 2007, 2006, 2005, 2013, 2023, 2025, 2024, 1740, + 1739, 1716, 1715, 1919, 1926, 1925, 1936, 1935, 1937, 1939, 1938, 1897, + 1896, 1902, 1961, 1960, 1967, 1973, 1972, 1975, 1974, 1671, 1676, 1675, + 1993, 1992, 1994, 1995, 1629, 1624, 1623, 1622, 1704, 1706, 1705, 1690, + 1689, 1875, 1873, 1794, 1795, 1792, 1799, 1800, 1807, 1806, 1811, 1810, + 1821, 1822, 1823, 1833, 1831, 1826, 1827, 40951, 40951, 1832, 1726, 1727, + 1844, 1843, 1854, 1853, 1852, 1859, 1858, 2101, 2100, 1654, 2019, 2017, + 2014, 2012, 2026, 2022, 1742, 1735, 1741, 1930, 1898, 1962, 1672, 1803, + 1812, 2088, 2095, 2102, 1837, 1876, 1845, 1874, 1793, 1625, 1766, 1850, + 1828, 1801, 1765, 1802, 1851, 1736, 1721, 1834, 1697, 1649, 1764, 1628, + 1901, 1931, 1855, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 1913, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 1914, 1885, 1646, 1647, 1842, 1912, 1894, 1641, 2052, 1915, + 1916, 1749, 32167, 1670, 1990, 1648, 38261, 38372, 38440, 38451, 38462, + 38473, 38484, 38495, 38506, 38262, 38273, 38284, 38295, 38306, 38317, + 38328, 31622, 31627, 31628, 31621, 31652, 31623, 31654, 31630, 31644, + 31626, 40951, 40951, 40951, 40951, 40951, 40951, 8360, 8362, 8192, 8193, + 8371, 8373, 8086, 8361, 8363, 8429, 8430, 8372, 8374, 8087, 8140, 8141, + 31653, 31624, 31625, 31639, 31651, 31636, 31648, 31634, 31646, 31638, + 31650, 31631, 31640, 31632, 31641, 31635, 31647, 31633, 31645, 31629, + 31642, 32661, 39263, 31637, 31649, 10541, 6129, 39141, 11256, 10540, + 6128, 39139, 33800, 33809, 33844, 40951, 33834, 33801, 33845, 33807, + 33808, 33811, 33815, 33810, 33814, 33812, 33816, 33843, 33791, 33793, + 33842, 33840, 33813, 33839, 33806, 40951, 33833, 33802, 33841, 33799, + 40951, 40951, 40951, 40951, 1092, 2387, 1074, 2389, 1104, 40951, 1089, + 1090, 1070, 1071, 1101, 1102, 2294, 2295, 2362, 2363, 1289, 1153, 1152, + 1143, 1142, 1571, 1570, 1146, 1145, 1588, 1586, 1587, 1589, 1163, 1162, + 1178, 1176, 1177, 1179, 1514, 1513, 1523, 1521, 1522, 1516, 1535, 1533, + 1534, 1536, 1320, 1318, 1319, 1321, 1286, 1284, 1285, 1287, 1358, 1356, + 1357, 1359, 1202, 1201, 1531, 1530, 1449, 1448, 1613, 1612, 1478, 1476, + 1477, 1479, 1484, 1482, 1483, 1485, 1465, 1463, 1464, 1466, 1210, 1208, + 1209, 1211, 1497, 1495, 1496, 1498, 1609, 1607, 1608, 1610, 1121, 1119, + 1120, 1122, 1265, 1263, 1264, 1266, 1249, 1247, 1248, 1250, 1431, 1429, + 1430, 1432, 1334, 1332, 1333, 1335, 1370, 1368, 1369, 1371, 1379, 1377, + 1378, 1380, 1410, 1408, 1409, 1411, 1308, 1306, 1307, 1309, 1575, 1574, + 1161, 1160, 1598, 1596, 1597, 1599, 1787, 1786, 1783, 1782, 1785, 1784, + 1791, 1790, 40951, 40951, 40755, 40951, 17625, 17629, 17597, 17613, + 17599, 17611, 17610, 17540, 17603, 17612, 17600, 17535, 17628, 17633, + 17607, 17620, 17622, 17619, 17618, 17615, 17614, 17617, 17616, 17623, + 17621, 17536, 17606, 17542, 17624, 17627, 17630, 17534, 17543, 17544, + 17545, 17546, 17547, 17548, 17549, 17550, 17551, 17552, 17553, 17554, + 17555, 17556, 17557, 17558, 17559, 17560, 17561, 17562, 17563, 17564, + 17565, 17566, 17567, 17568, 17541, 17605, 17604, 17533, 17595, 17626, + 17569, 17570, 17571, 17572, 17573, 17574, 17575, 17576, 17577, 17578, + 17579, 17580, 17581, 17582, 17583, 17584, 17585, 17586, 17587, 17588, + 17589, 17590, 17591, 17592, 17593, 17594, 17539, 17635, 17602, 17632, + 17538, 17601, 19098, 19091, 19093, 19097, 19089, 19081, 19036, 19038, + 19040, 19037, 19039, 19032, 19034, 19033, 19035, 19090, 19082, 19084, + 19086, 19083, 19085, 19057, 19059, 19061, 19058, 19060, 19041, 19043, + 19045, 19042, 19044, 19072, 19074, 19076, 19073, 19075, 19047, 19049, + 19051, 19048, 19050, 19052, 19054, 19056, 19053, 19055, 19062, 19064, + 19066, 19063, 19065, 19077, 19079, 19078, 19067, 19069, 19071, 19068, + 19070, 19080, 19046, 19088, 19087, 19031, 18981, 18998, 18982, 18983, + 18984, 18985, 19019, 19000, 18989, 18994, 18993, 18992, 18996, 18990, + 18991, 18995, 19017, 18987, 18999, 18988, 19002, 19001, 19016, 19011, + 18997, 19010, 18980, 19018, 18986, 19025, 40951, 40951, 40951, 19026, + 19027, 19005, 19006, 19013, 19012, 40951, 40951, 19004, 19003, 19028, + 19022, 19023, 19029, 40951, 40951, 19008, 19030, 19021, 19020, 19024, + 19009, 40951, 40951, 19014, 19007, 19015, 40951, 40951, 40951, 17537, + 17598, 17596, 17609, 17634, 17608, 17631, 40951, 19095, 19092, 19096, + 19094, 19101, 19099, 19100, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 20788, 20790, 20789, 29933, 31859, 40951, + 40951, 24968, 24994, 24987, 25020, 24978, 24969, 24998, 24964, 24976, + 25006, 25009, 25003, 40951, 24992, 25019, 25023, 25002, 25017, 25024, + 25031, 25034, 24979, 25030, 24977, 24980, 24963, 24986, 24989, 25012, + 25013, 24971, 25028, 24993, 24974, 25010, 24972, 25029, 24988, 24996, + 40951, 25021, 24985, 25005, 24970, 24981, 24995, 24966, 25000, 24975, + 25007, 25008, 24965, 24991, 24967, 25018, 25011, 25025, 24999, 25001, + 40951, 24973, 25027, 40951, 24984, 24983, 24997, 25033, 25026, 25036, + 25004, 24982, 25014, 25022, 24990, 25015, 25016, 25032, 25035, 40951, + 40951, 25046, 25047, 25049, 25048, 25037, 25038, 25045, 25039, 25040, + 25050, 25041, 25042, 25043, 25044, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24852, + 24851, 24853, 24840, 24841, 24842, 24843, 24844, 24845, 24846, 24848, + 24847, 24850, 24849, 24858, 24855, 24856, 24854, 24857, 24957, 24958, + 24860, 24859, 24861, 24960, 24959, 24863, 24864, 24865, 24862, 24866, + 24869, 24868, 24870, 24871, 24872, 24961, 24873, 24874, 24867, 24877, + 24878, 24876, 24875, 24879, 24880, 24881, 24882, 24883, 24884, 24887, + 24888, 24889, 24886, 24890, 24885, 24891, 24892, 24893, 24894, 24895, + 24896, 24897, 24898, 24899, 24900, 24902, 24901, 24903, 24904, 24906, + 24907, 24908, 24905, 24909, 24910, 24911, 24912, 24914, 24913, 24915, + 24916, 24962, 24917, 24918, 24920, 24921, 24922, 24919, 24923, 24924, + 24925, 24926, 24927, 24955, 24935, 24936, 24937, 24938, 24939, 24940, + 24941, 24942, 24943, 24944, 24945, 24946, 24947, 24948, 24949, 24950, + 24951, 24952, 24953, 24954, 24928, 24929, 24930, 24931, 24932, 24933, + 24934, 24956, 40951, 40951, 40951, 40951, 40951, 112, 113, 159, 40951, + 40951, 40951, 40951, 156, 151, 146, 126, 121, 139, 134, 114, 129, 154, + 149, 144, 124, 119, 140, 135, 115, 130, 157, 152, 147, 127, 122, 142, + 137, 117, 132, 158, 153, 148, 128, 123, 143, 138, 118, 133, 155, 150, + 145, 125, 120, 141, 136, 116, 131, 40951, 40951, 40951, 111, 107, 109, + 110, 108, 103, 104, 105, 106, 18166, 18163, 18167, 18153, 18148, 18154, + 18157, 18149, 18159, 18168, 18151, 18161, 18155, 18164, 18158, 18160, + 18170, 18152, 18162, 18156, 18165, 18169, 18150, 18171, 18183, 18192, + 18182, 18178, 18191, 18173, 18179, 18195, 18199, 18200, 18181, 18184, + 18190, 18189, 18197, 18198, 18180, 18187, 18193, 18188, 18177, 18196, + 18185, 18172, 18174, 18194, 18186, 18175, 18176, 18418, 18419, 18617, + 18613, 18654, 18619, 18349, 18423, 18616, 18612, 18355, 18354, 18415, + 18397, 18413, 18422, 18417, 18203, 18202, 18656, 18615, 18657, 18420, + 18611, 18393, 29360, 40951, 32211, 32214, 32209, 32212, 32183, 32213, + 32181, 32184, 32210, 32182, 32215, 32180, 2526, 40951, 40951, 40951, + 18610, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 31405, 31406, + 31417, 31386, 31389, 31420, 31398, 31397, 31418, 31425, 31384, 31411, + 31390, 31403, 31404, 31413, 31402, 31383, 31387, 31394, 31392, 31414, + 31391, 31382, 31412, 31399, 31400, 31385, 31388, 31410, 31424, 31395, + 31419, 31381, 31407, 31426, 31408, 31409, 31401, 31423, 31422, 31396, + 31415, 31416, 31421, 31393, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 25283, 25285, 25278, 25279, 25290, 25289, + 25292, 25300, 25302, 25281, 25293, 25276, 25296, 25294, 25274, 25287, + 25275, 25288, 25299, 25295, 25277, 25297, 25298, 25280, 25282, 25284, + 25286, 25291, 25301, 40951, 40951, 40951, 6039, 6050, 6041, 6012, 6035, + 6051, 6013, 6040, 6057, 6055, 6015, 6056, 6042, 6030, 6025, 6026, 6024, + 6010, 6033, 6023, 6058, 6020, 6032, 6049, 6029, 6053, 6043, 6038, 6047, + 6048, 6021, 6034, 6045, 6046, 6027, 6028, 6022, 6054, 6011, 6031, 6036, + 6052, 6016, 6017, 6018, 6019, 6014, 6044, 6037, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 8571, 8569, 8567, 8566, 8563, 8562, 8565, 8564, 8570, 8568, + 8561, 8560, 8558, 8549, 8547, 8556, 8554, 8545, 8551, 8552, 8559, 8557, + 8548, 8546, 8555, 8553, 8544, 8550, 40951, 40951, 40951, 40951, 30215, + 30209, 30195, 30210, 30182, 30212, 30214, 30211, 30206, 30202, 30194, + 30190, 30191, 30192, 30184, 30216, 30205, 30198, 30196, 30186, 30183, + 30207, 30200, 30188, 30204, 30193, 30189, 30187, 30208, 30203, 30201, + 30185, 30220, 30218, 30219, 30217, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 30213, 30199, 30197, 18013, 18029, 18037, + 18031, 18012, 18022, 18016, 18015, 18024, 18034, 18038, 18035, 18032, + 18020, 18036, 18027, 18021, 18019, 18023, 18033, 18025, 18026, 18028, + 18017, 18014, 18030, 18018, 40951, 40951, 40951, 40951, 40951, 30287, + 30286, 30283, 30256, 30257, 30279, 30254, 30280, 30255, 30259, 30288, + 30281, 30262, 30263, 30270, 30264, 30282, 30267, 30269, 30290, 30253, + 30266, 30265, 30277, 30274, 30284, 30285, 30258, 30289, 30268, 30271, + 30272, 30273, 30276, 30261, 30278, 30275, 30260, 8386, 8384, 8382, 8383, + 8385, 40951, 40951, 40951, 40951, 40951, 37709, 37712, 37716, 37720, + 37713, 37717, 37735, 37731, 37718, 37728, 37730, 37719, 37725, 37721, + 37736, 37714, 37733, 37732, 37724, 37710, 37734, 37723, 37711, 37722, + 37727, 37715, 37729, 37737, 37738, 37726, 40951, 37739, 30296, 30338, + 30339, 30319, 30320, 30317, 30318, 30316, 30331, 30308, 30309, 30313, + 30314, 30303, 30306, 30307, 30312, 30335, 30300, 30332, 30321, 30322, + 30325, 30326, 30327, 30336, 30310, 30311, 30323, 30324, 30334, 30330, + 30337, 30328, 30329, 30333, 40951, 40951, 40951, 40951, 30297, 30298, + 30299, 30315, 30304, 30305, 30301, 30302, 30340, 30295, 30292, 30293, + 30291, 30294, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 10594, 10593, 10589, 10590, 10591, 10592, + 10600, 10599, 10595, 10596, 10597, 10598, 10617, 10602, 10616, 10613, + 10618, 10611, 10608, 10604, 10609, 10607, 10610, 10615, 10614, 10584, + 10612, 10583, 10603, 10579, 10606, 10580, 10605, 10587, 10585, 10586, + 10581, 10582, 10601, 10588, 10634, 10633, 10629, 10630, 10631, 10632, + 10640, 10639, 10635, 10636, 10637, 10638, 10657, 10642, 10656, 10653, + 10658, 10651, 10648, 10644, 10649, 10647, 10650, 10655, 10654, 10624, + 10652, 10623, 10643, 10619, 10646, 10620, 10645, 10627, 10625, 10626, + 10621, 10622, 10641, 10628, 32797, 32803, 32814, 32811, 32801, 32800, + 32799, 32778, 32806, 32784, 32813, 32809, 32812, 32815, 32802, 32810, + 32789, 32808, 32805, 32783, 32788, 32790, 32787, 32782, 32776, 32773, + 32795, 32804, 32793, 32777, 32798, 32816, 32780, 32774, 32786, 32817, + 32794, 32791, 32792, 32775, 32771, 32796, 32772, 32781, 32770, 32779, + 32785, 32807, 30729, 30747, 30753, 30751, 30754, 30735, 30732, 30752, + 30737, 30736, 30733, 30731, 30749, 30748, 30739, 30734, 30742, 30738, + 30743, 30744, 30750, 30755, 30728, 30745, 30756, 30740, 30757, 30730, + 30746, 30741, 40951, 40951, 30764, 30766, 30763, 30762, 30759, 30758, + 30761, 30760, 30767, 30765, 40951, 40951, 40951, 40951, 40951, 40951, + 30656, 30657, 30658, 30659, 30684, 30677, 30663, 30660, 30666, 30676, + 30675, 30690, 30669, 30664, 30668, 30685, 30686, 30687, 30670, 30671, + 30688, 30665, 30681, 30680, 30674, 30662, 30673, 30661, 30672, 30678, + 30691, 30689, 30667, 30679, 30683, 30682, 40951, 40951, 40951, 40951, + 30692, 30693, 30694, 30695, 30720, 30713, 30699, 30696, 30702, 30712, + 30711, 30726, 30705, 30700, 30704, 30721, 30722, 30723, 30706, 30707, + 30724, 30701, 30717, 30716, 30710, 30698, 30709, 30697, 30708, 30714, + 30727, 30725, 30703, 30715, 30719, 30718, 40951, 40951, 40951, 40951, + 16693, 16684, 16673, 16672, 16675, 16664, 16674, 16671, 16670, 16685, + 16661, 16660, 16686, 16694, 16687, 16677, 16663, 16662, 16688, 16667, + 16666, 16665, 16695, 16689, 16690, 16669, 16668, 16679, 16678, 16681, + 16680, 16696, 16691, 16692, 16697, 16683, 16682, 16659, 16658, 16676, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 6071, 6120, 6087, + 6083, 6085, 6110, 6084, 6108, 6105, 6074, 6107, 6109, 6088, 6099, 6095, + 6090, 6118, 6082, 6073, 6091, 6094, 6096, 6121, 6112, 6070, 6076, 6077, + 6079, 6113, 6111, 6116, 6080, 6100, 6092, 6119, 6104, 6115, 6081, 6075, + 6098, 6086, 6117, 6102, 6114, 6103, 6101, 6089, 6078, 6072, 6106, 6097, + 6093, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 6122, 38912, 38881, 38882, 38892, 38891, 38894, 38893, + 38884, 38883, 38901, 38909, 40951, 38900, 38899, 38885, 38886, 38902, + 38910, 38888, 38887, 38903, 38890, 38889, 38913, 38904, 38911, 38905, + 40951, 38896, 38895, 38898, 38897, 38914, 38906, 38907, 40951, 38915, + 38908, 40951, 38947, 38916, 38917, 38927, 38926, 38929, 38928, 38919, + 38918, 38936, 38944, 40951, 38935, 38934, 38920, 38921, 38937, 38945, + 38923, 38922, 38938, 38925, 38924, 38948, 38939, 38946, 38940, 40951, + 38931, 38930, 38933, 38932, 38949, 38941, 38942, 40951, 38950, 38943, + 40951, 40951, 40951, 37427, 37428, 37441, 37401, 37430, 37429, 37432, + 37408, 37431, 37424, 37423, 37442, 37398, 37404, 37397, 37403, 37411, + 37410, 37445, 37399, 37434, 37426, 37425, 37402, 37409, 37405, 37421, + 37413, 37443, 37420, 37419, 37418, 37415, 37414, 37436, 37435, 37446, + 37444, 37438, 37407, 37437, 37406, 37447, 37400, 37440, 37439, 37396, + 37417, 37416, 37433, 37412, 37422, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24819, 24820, 24821, + 24822, 24823, 24824, 24825, 24826, 24827, 24758, 24759, 24760, 24761, + 24762, 24771, 24763, 24764, 24765, 24766, 24767, 24768, 24769, 24770, + 24772, 24773, 24774, 24775, 24839, 24776, 24777, 24778, 24779, 24780, + 24781, 24782, 24783, 24784, 24785, 24786, 24787, 24788, 24789, 24790, + 24791, 24792, 24793, 24794, 24795, 24796, 24797, 24798, 24799, 24800, + 24801, 24802, 24803, 24804, 24805, 24806, 24807, 24808, 24809, 24810, + 24811, 24812, 24813, 24814, 24815, 24816, 24817, 24818, 24499, 24835, + 24828, 24500, 24829, 24830, 24831, 24832, 24501, 24836, 24837, 24833, + 24834, 24838, 24505, 24506, 24507, 24508, 24509, 24510, 24511, 24512, + 24502, 24503, 24504, 24516, 24517, 24518, 24513, 24514, 24515, 24519, + 24520, 24521, 24522, 24523, 24524, 24527, 24528, 24529, 24530, 24531, + 24532, 24533, 24534, 24535, 24536, 24537, 24538, 24539, 24540, 24541, + 24542, 24543, 24544, 24545, 24546, 24547, 24548, 24549, 24550, 24551, + 24552, 24553, 24554, 24555, 24556, 24557, 24558, 24559, 24560, 24561, + 24562, 24563, 24564, 24565, 24566, 24567, 24568, 24569, 24570, 24571, + 24572, 24573, 24574, 24575, 24576, 24525, 24526, 24577, 24578, 24579, + 24580, 24581, 24582, 24583, 24584, 24585, 24586, 24587, 24588, 24589, + 24590, 24591, 24592, 24593, 24594, 24595, 24596, 24597, 24598, 24599, + 24600, 24601, 24602, 24603, 24604, 24605, 24606, 24607, 24608, 24609, + 24641, 24642, 24643, 24644, 24645, 24646, 24647, 24648, 24649, 24610, + 24611, 24612, 24613, 24614, 24615, 24616, 24617, 24618, 24619, 24620, + 24621, 24622, 24623, 24624, 24625, 24626, 24627, 24628, 24629, 24630, + 24631, 24632, 24633, 24634, 24635, 24636, 24637, 24638, 24639, 24640, + 24656, 24657, 24658, 24659, 24660, 24661, 24662, 24663, 24664, 24665, + 24666, 24667, 24668, 24669, 24670, 24671, 24672, 24673, 24674, 24675, + 24650, 24651, 24652, 24653, 24654, 24655, 24676, 24677, 24678, 24679, + 24680, 24681, 24682, 24683, 24718, 24719, 24720, 24721, 24722, 24723, + 24724, 24725, 24726, 24727, 24684, 24685, 24686, 24687, 24688, 24689, + 24690, 24691, 24692, 24693, 24694, 24695, 24696, 24697, 24698, 24699, + 24700, 24701, 24702, 24703, 24709, 24710, 24711, 24712, 24713, 24714, + 24715, 24716, 24717, 24704, 24705, 24706, 24707, 24708, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24739, 24734, 24737, + 24736, 24740, 24735, 24733, 24738, 24732, 24728, 24731, 24730, 24729, + 24746, 24741, 24745, 24742, 24743, 24744, 24747, 24749, 24748, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24750, + 24751, 24752, 24753, 24754, 24755, 24756, 24757, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 27868, 27989, 27988, 27853, 27869, 27857, 40951, 27887, 27889, + 27888, 27883, 27882, 27880, 27881, 27942, 27875, 27899, 27939, 27863, + 27903, 27864, 27907, 27870, 27909, 27886, 27923, 27924, 27922, 27866, + 27920, 27925, 27926, 27967, 27968, 27931, 27867, 27876, 27982, 27964, + 27965, 27938, 27937, 27872, 27952, 27955, 27956, 27953, 27951, 27979, + 40951, 27874, 27794, 27841, 27689, 27772, 27801, 27686, 27843, 27944, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 10008, + 10009, 10010, 10011, 10012, 10002, 40951, 40951, 10003, 40951, 9958, + 9959, 9960, 9961, 9962, 9963, 9964, 9965, 9966, 9967, 9968, 9969, 9970, + 9971, 9972, 9973, 9974, 9975, 9976, 9977, 9978, 9979, 9980, 9981, 9982, + 9983, 9984, 9985, 9986, 9987, 9988, 9989, 9990, 9991, 9992, 9993, 9994, + 9995, 9996, 9997, 9998, 9999, 10000, 10001, 40951, 10006, 10007, 40951, + 40951, 40951, 10004, 40951, 40951, 10005, 20610, 20621, 20612, 20606, + 20618, 20623, 20613, 20619, 20604, 20617, 20620, 20607, 20625, 20622, + 20614, 20611, 20624, 20615, 20608, 20609, 20616, 20605, 40951, 20626, + 20601, 20597, 20600, 20598, 20596, 20602, 20603, 20599, 31035, 31046, + 31037, 31031, 31043, 31048, 31038, 31044, 31029, 31042, 31045, 31032, + 31050, 31028, 31047, 31039, 31036, 31049, 31040, 31033, 31034, 31041, + 31030, 31027, 31051, 31058, 31053, 31054, 31057, 31056, 31055, 31052, + 28806, 28821, 28811, 28832, 28823, 28817, 28813, 28829, 28834, 28824, + 28830, 28815, 28809, 28828, 28810, 28831, 28807, 28818, 28814, 28836, + 28812, 28833, 28825, 28822, 28835, 28826, 28819, 28820, 28808, 28827, + 28816, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 28841, + 28838, 28839, 28844, 28845, 28843, 28840, 28837, 28842, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 19658, 19674, 19666, 19665, + 19671, 19676, 19660, 19672, 19661, 19670, 19673, 19663, 19678, 19675, + 19667, 19659, 19677, 19668, 19664, 40951, 19669, 19662, 40951, 40951, + 40951, 40951, 40951, 19679, 19683, 19682, 19681, 19680, 31429, 31448, + 31444, 31431, 31432, 31440, 31441, 31433, 31439, 31442, 31445, 31447, + 31450, 31446, 31437, 31430, 31449, 31435, 31434, 31443, 31436, 31438, + 31455, 31454, 31451, 31456, 31452, 31453, 40951, 40951, 40951, 31457, + 25309, 25315, 25319, 25317, 25311, 25327, 25320, 25328, 25321, 25305, + 25322, 25313, 25323, 25325, 25308, 25303, 25326, 25318, 25324, 25307, + 25304, 25310, 25312, 25306, 25314, 25316, 40951, 40951, 40951, 40951, + 40951, 25329, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 27402, 27403, 27404, 27405, + 27401, 27400, 27376, 27377, 27398, 27397, 27380, 27381, 27382, 27383, + 27378, 27379, 27396, 27393, 27392, 27384, 27385, 27386, 27394, 27399, + 27387, 27388, 27389, 27390, 27391, 27395, 27406, 27407, 27298, 27319, + 27320, 27321, 27318, 27317, 27310, 27314, 27313, 27303, 27304, 27316, + 27312, 27308, 27307, 27305, 27299, 27306, 27309, 27315, 27300, 27301, + 27302, 27311, 40951, 40951, 40951, 40951, 27286, 27291, 27323, 27322, + 27372, 27364, 27358, 27335, 27329, 27352, 27346, 27324, 27341, 27370, + 27368, 27362, 27339, 27333, 27356, 27350, 40951, 40951, 27373, 27365, + 27359, 27336, 27330, 27353, 27347, 27326, 27343, 27375, 27367, 27361, + 27338, 27332, 27355, 27349, 27328, 27345, 27371, 27369, 27363, 27340, + 27334, 27357, 27351, 27325, 27342, 27374, 27366, 27360, 27337, 27331, + 27354, 27348, 27327, 27344, 27290, 27296, 27295, 27289, 27288, 27293, + 27292, 27287, 27297, 27294, 21668, 21690, 21692, 21688, 40951, 21689, + 21691, 40951, 40951, 40951, 40951, 40951, 21693, 21684, 21687, 21686, + 21634, 21632, 21656, 21655, 40951, 21654, 21653, 21662, 40951, 21642, + 21638, 21637, 21645, 21644, 21641, 21640, 21639, 21647, 21646, 21643, + 21658, 21657, 21652, 21651, 21664, 21666, 21665, 21663, 21660, 21648, + 21649, 21650, 21667, 21661, 21633, 21635, 21636, 21659, 40951, 40951, + 21682, 21683, 21685, 40951, 40951, 40951, 40951, 21694, 21631, 21630, + 21629, 21628, 21670, 21669, 21671, 21672, 21695, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 21676, 21681, 21674, 21673, 21680, 21678, + 21677, 21675, 21679, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 30405, 30401, 30406, 30411, 30402, 30409, 30395, 30403, 30407, 30399, + 30394, 30390, 30408, 30391, 30393, 30392, 30410, 30383, 30384, 30386, + 30389, 30387, 30388, 30398, 30400, 30385, 30404, 30397, 30396, 30413, + 30412, 30414, 30226, 30244, 30225, 30235, 30247, 30248, 30237, 30241, + 30239, 30232, 30236, 30228, 30243, 30227, 30249, 30238, 30240, 30221, + 30222, 30245, 30224, 30246, 30223, 30231, 30233, 30229, 30242, 30230, + 30234, 30252, 30251, 30250, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 25610, 25614, 25613, 25618, + 25617, 25615, 25639, 25642, 25658, 25622, 25621, 25620, 25619, 25640, + 25632, 25638, 25624, 25634, 25623, 25636, 25616, 25631, 25645, 25641, + 25628, 25612, 25611, 25644, 25643, 25629, 25626, 25635, 25625, 25637, + 25630, 25627, 25633, 25660, 25659, 40951, 40951, 40951, 40951, 25646, + 25650, 25649, 25648, 25647, 25657, 25655, 25653, 25652, 25651, 25656, + 25654, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 2542, 2543, 2549, 2545, 2548, 2544, 2546, 2547, 2585, 2586, 2575, 2576, + 2577, 2578, 2573, 2574, 2590, 2563, 2562, 2561, 2552, 2550, 2551, 2587, + 2589, 2572, 2570, 2582, 2581, 2571, 2593, 2588, 2580, 2579, 2557, 2556, + 2555, 2560, 2559, 2558, 2592, 2553, 2568, 2569, 2595, 2594, 2591, 2567, + 2584, 2565, 2583, 2564, 2566, 2554, 40951, 40951, 40951, 2596, 37309, + 33835, 22667, 22662, 22668, 22663, 20752, 20763, 20754, 20748, 20760, + 20765, 20755, 20761, 20746, 20759, 20762, 20749, 20767, 20764, 20756, + 20753, 20766, 20757, 20750, 20751, 20758, 20747, 40951, 40951, 20772, + 20769, 20770, 20775, 20771, 20768, 20773, 20774, 20721, 20735, 20726, + 20722, 20732, 20725, 20727, 20733, 20719, 20731, 20734, 20723, 20724, + 20736, 20728, 20737, 20729, 20730, 20720, 40951, 40951, 40951, 40951, + 40951, 20742, 20739, 20740, 20745, 20741, 20738, 20743, 20744, 31690, + 31704, 31695, 31691, 31701, 31694, 31696, 31702, 31700, 31703, 31692, + 31693, 31705, 31697, 31707, 31698, 31699, 31706, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 31715, 31716, 31688, 31689, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 31712, 31709, 31710, 31714, 31711, 31708, 31713, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30415, 30457, 30458, + 30447, 30483, 30476, 30450, 30451, 30485, 30428, 30468, 30416, 30461, + 30430, 30470, 30418, 30462, 30429, 30469, 30417, 30446, 30482, 30436, + 30475, 30425, 30465, 30419, 30463, 30452, 30486, 30431, 30471, 30420, + 30441, 30444, 30432, 30421, 30459, 30439, 30478, 30437, 30477, 30440, + 30479, 30467, 30438, 30460, 30445, 30453, 30448, 30443, 30481, 30433, + 30472, 30449, 30484, 30454, 30487, 30434, 30473, 30422, 30426, 30423, + 30427, 30466, 30442, 30480, 30435, 30474, 30424, 30464, 30455, 30456, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 30074, 30077, 30100, 30075, 30089, + 30085, 30091, 30101, 30076, 30079, 30121, 30102, 30103, 30092, 30093, + 30104, 30122, 30123, 30105, 30106, 30078, 30118, 30094, 30095, 30080, + 30082, 30086, 30114, 30116, 30110, 30112, 30115, 30107, 30081, 30108, + 30124, 30087, 30088, 30096, 30083, 30097, 30090, 30117, 30120, 30111, + 30113, 30109, 30098, 30099, 30084, 30119, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30125, + 30128, 30151, 30126, 30140, 30136, 30142, 30152, 30127, 30130, 30172, + 30153, 30154, 30143, 30144, 30155, 30173, 30174, 30156, 30157, 30129, + 30169, 30145, 30146, 30131, 30133, 30137, 30165, 30167, 30161, 30163, + 30166, 30158, 30132, 30159, 30175, 30138, 30139, 30147, 30134, 30148, + 30141, 30168, 30171, 30162, 30164, 30160, 30149, 30150, 30135, 30170, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30178, 30177, 30181, + 30176, 30179, 30180, 19611, 19598, 19606, 19590, 19589, 19603, 19599, + 19602, 19587, 19600, 19584, 19583, 19592, 19591, 19610, 19597, 19596, + 19588, 19601, 19604, 19605, 19595, 19608, 19585, 19609, 19586, 19593, + 19594, 19607, 19618, 19620, 19622, 19619, 19621, 19613, 19612, 19617, + 19614, 19616, 19615, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 19629, 19631, 19628, 19627, 19624, 19623, 19626, 19625, 19632, + 19630, 40951, 40951, 40951, 40951, 40951, 40951, 17713, 17715, 17712, + 17711, 17708, 17707, 17710, 17709, 17716, 17714, 17699, 17700, 17701, + 17698, 17702, 17696, 17673, 17657, 17665, 17663, 17656, 17662, 17668, + 17670, 17664, 17660, 17658, 17671, 17672, 17669, 17667, 17654, 17659, + 17655, 17666, 17661, 17652, 17653, 40951, 40951, 40951, 17697, 17651, + 17649, 17648, 17650, 17704, 17703, 17695, 17679, 17687, 17685, 17678, + 17684, 17690, 17692, 17686, 17682, 17680, 17693, 17694, 17691, 17689, + 17676, 17681, 17677, 17688, 17683, 17674, 17675, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 17705, 17706, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 32268, 32266, 32265, 32262, 32261, + 32264, 32263, 32269, 32267, 32260, 32259, 32257, 32248, 32246, 32255, + 32253, 32244, 32250, 32251, 32258, 32256, 32247, 32245, 32254, 32252, + 32243, 32249, 32240, 32241, 32239, 32242, 40951, 39408, 39442, 39422, + 39421, 39427, 39426, 39404, 39403, 39402, 39413, 39434, 39407, 39438, + 39441, 39440, 39437, 39445, 39424, 39423, 39425, 39406, 39428, 39439, + 39409, 39433, 39444, 39429, 39430, 39417, 39415, 39414, 39416, 39418, + 39405, 39420, 39443, 39431, 39432, 39411, 39412, 39435, 39410, 40951, + 39400, 39399, 39401, 40951, 40951, 39419, 39436, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 1195, 1490, 1326, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 1061, 2351, 2352, 2349, 30345, 30353, 30367, 30354, 30358, 30364, 30355, + 30370, 30359, 30365, 30363, 30366, 30357, 30372, 30368, 30347, 30348, + 30360, 30346, 30344, 30371, 30361, 30349, 30350, 30356, 30362, 30369, + 30351, 30352, 30379, 30377, 30374, 30382, 30381, 30378, 30376, 30375, + 30380, 30343, 30373, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 33894, 33908, 33896, 33905, 33912, 33900, 33906, 33904, 33907, + 33897, 33914, 33910, 33901, 33895, 33913, 33902, 33899, 33903, 33911, + 33909, 33898, 33893, 33888, 33886, 33889, 33887, 33892, 33891, 33883, + 33882, 33884, 33890, 33885, 33915, 33918, 33917, 33916, 33921, 33922, + 33919, 33923, 33920, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 30492, 30504, 30502, 30507, 30496, + 30501, 30500, 30503, 30494, 30509, 30505, 30497, 30508, 30498, 30493, + 30499, 30506, 30495, 30491, 30490, 30489, 30488, 30513, 30510, 30511, + 30512, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 6491, + 6485, 6499, 6493, 6488, 6496, 6502, 6484, 6494, 6497, 6495, 6498, 6489, + 6504, 6500, 6486, 6492, 6503, 6490, 6487, 6501, 6509, 6506, 6507, 6511, + 6508, 6505, 6510, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 16720, 16731, 16722, 16716, 16728, 16733, 16723, 16729, + 16714, 16727, 16730, 16717, 16735, 16732, 16724, 16721, 16734, 16725, + 16718, 16719, 16726, 16715, 16736, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 4635, 4640, 4638, 4637, 4639, 4562, 4563, + 4581, 4582, 4579, 4580, 4574, 4575, 4576, 4577, 4608, 4564, 4555, 4565, + 4601, 4600, 4597, 4596, 4585, 4595, 4594, 4599, 4598, 4587, 4571, 4570, + 4567, 4566, 4586, 4573, 4572, 4569, 4568, 4588, 4603, 4602, 4593, 4592, + 4605, 4607, 4606, 4584, 4578, 4589, 4590, 4591, 4604, 4583, 4556, 4561, + 4560, 4645, 4644, 4654, 4655, 4648, 4649, 4650, 4651, 4652, 4653, 4656, + 4646, 4641, 4647, 4657, 4659, 4658, 4633, 4632, 4631, 4634, 4630, 40951, + 40951, 40951, 40951, 4626, 4624, 4621, 4612, 4614, 4619, 4617, 4609, + 4615, 4625, 4623, 4622, 4611, 4613, 4620, 4618, 4610, 4616, 4627, 4628, + 4666, 4668, 4665, 4664, 4661, 4660, 4663, 4662, 4669, 4667, 4636, 4558, + 4559, 4642, 4643, 4557, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 4629, 20979, 20981, 20983, 20939, 20940, 20949, 20950, + 20947, 20948, 20977, 20941, 20978, 20942, 20967, 20966, 20963, 20962, + 20951, 20961, 20960, 20965, 20964, 20953, 20944, 20943, 20936, 20934, + 20935, 20970, 20952, 20946, 20945, 20938, 20937, 20954, 20969, 20968, + 20959, 20958, 20974, 20976, 20971, 20973, 20975, 20955, 20956, 20957, + 20972, 20989, 20994, 20995, 20992, 20993, 20996, 20990, 20997, 20991, + 20982, 20980, 21000, 21001, 20998, 20984, 20985, 20987, 20986, 20988, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 20999, 40951, 40951, 33946, 33947, 33936, 33937, 33938, 33939, 33932, + 33933, 33943, 33935, 33948, 33944, 33949, 33945, 33940, 33942, 33941, + 33934, 33950, 33929, 33951, 33953, 33952, 33930, 33931, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 33960, 33962, 33959, 33958, 33955, + 33954, 33957, 33956, 33963, 33961, 40951, 40951, 40951, 40951, 40951, + 40951, 6174, 6176, 6175, 6170, 6172, 6173, 6171, 6159, 6158, 6155, 6154, + 6140, 6153, 6152, 6157, 6156, 6142, 6145, 6144, 6137, 6136, 6141, 6147, + 6146, 6139, 6138, 6143, 6163, 6162, 6151, 6150, 6165, 6148, 6149, 6166, + 6161, 6169, 6167, 6164, 6178, 6186, 6187, 6182, 6183, 6184, 6180, 6188, + 6181, 6189, 6206, 6205, 6190, 6204, 40951, 6199, 6201, 6198, 6197, 6194, + 6193, 6196, 6195, 6202, 6200, 6177, 6192, 6191, 6203, 6160, 6179, 6185, + 6168, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25371, + 25373, 25375, 25372, 25374, 25363, 25362, 25359, 25358, 25357, 25356, + 25361, 25360, 25342, 25351, 25350, 25345, 25344, 25341, 25353, 25352, + 25347, 25346, 25343, 25365, 25364, 25355, 25354, 25368, 25349, 25367, + 25370, 25369, 25366, 25348, 25378, 25379, 25377, 25376, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32741, 32744, 32748, + 32687, 32688, 32706, 32707, 32704, 32705, 32699, 32700, 32701, 32702, + 32733, 32689, 32734, 32690, 32726, 32725, 32722, 32721, 32710, 32720, + 32719, 32724, 32723, 32712, 32696, 32695, 32692, 32691, 32711, 32698, + 32697, 32694, 32693, 32713, 32728, 32727, 32718, 32717, 32730, 32732, + 32731, 32709, 32708, 32703, 32714, 32715, 32716, 32729, 32752, 32761, + 32762, 32755, 32756, 32757, 32758, 32759, 32760, 32763, 32753, 32764, + 32754, 32747, 32743, 32745, 32746, 32767, 32673, 32672, 32765, 32738, + 32735, 32742, 32750, 32684, 32749, 32751, 32739, 32680, 32682, 32679, + 32678, 32675, 32674, 32677, 32676, 32683, 32681, 32685, 32740, 32686, + 32766, 32736, 32737, 40951, 33649, 33647, 33646, 33643, 33642, 33645, + 33644, 33650, 33648, 33661, 33660, 33659, 33655, 33654, 33658, 33657, + 33653, 33656, 33651, 33652, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 22374, 22375, 22401, 22403, 22400, + 22376, 22402, 22377, 22391, 22390, 22366, 22364, 22365, 22384, 22389, + 22388, 22373, 22372, 40951, 22386, 22379, 22378, 22369, 22368, 22385, + 22381, 22380, 22371, 22367, 22370, 22387, 22393, 22392, 22363, 22361, + 22362, 22395, 22399, 22397, 22383, 22398, 22360, 22394, 22382, 22411, + 22414, 22415, 22418, 22416, 22412, 22417, 22413, 22408, 22407, 22406, + 22404, 22358, 22357, 22419, 22409, 22356, 22420, 22405, 22396, 22359, + 22410, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 28281, 28283, 28284, 28282, 28272, 28271, 28270, + 40951, 28269, 40951, 28268, 28267, 28260, 28259, 40951, 28254, 28262, + 28261, 28250, 28248, 28249, 28253, 28264, 28263, 28252, 28251, 28255, + 28274, 28273, 28266, 40951, 28265, 28277, 28280, 28258, 28276, 28279, + 28278, 28275, 28257, 28256, 28285, 40951, 40951, 40951, 40951, 40951, + 40951, 22435, 22436, 22447, 22448, 22445, 22446, 22466, 22437, 22467, + 22438, 22456, 22455, 22426, 22424, 22425, 22449, 22454, 22453, 22434, + 22433, 22432, 22451, 22442, 22441, 22429, 22427, 22439, 22428, 22450, + 22444, 22443, 22431, 22430, 22452, 22458, 22457, 22423, 22421, 22422, + 22463, 22465, 22440, 22462, 22464, 22459, 22460, 22461, 22470, 22471, + 22476, 22477, 22474, 22475, 22478, 22472, 22479, 22473, 22468, 22469, + 40951, 40951, 40951, 40951, 40951, 22486, 22488, 22485, 22484, 22481, + 22480, 22483, 22482, 22489, 22487, 40951, 40951, 40951, 40951, 40951, + 40951, 18095, 18096, 18099, 18102, 40951, 18052, 18053, 18066, 18067, + 18064, 18065, 18047, 18049, 40951, 40951, 18090, 18054, 40951, 40951, + 18089, 18055, 18086, 18085, 18082, 18081, 18070, 18080, 18079, 18084, + 18083, 18072, 18061, 18060, 18057, 18056, 18071, 18063, 18062, 18059, + 18058, 18073, 40951, 18088, 18087, 18078, 18077, 18092, 18094, 18093, + 40951, 18069, 18068, 40951, 18051, 18074, 18075, 18076, 18091, 40951, + 8077, 18097, 18098, 18104, 18113, 18114, 18107, 18108, 18109, 18110, + 40951, 40951, 18116, 18105, 40951, 40951, 18115, 18106, 18101, 40951, + 40951, 18117, 40951, 40951, 40951, 40951, 40951, 40951, 18103, 40951, + 40951, 40951, 40951, 40951, 18100, 18046, 18045, 18048, 18050, 18111, + 18112, 40951, 40951, 8257, 8258, 8256, 8255, 8254, 8253, 8252, 40951, + 40951, 40951, 8263, 8260, 8261, 8259, 8262, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 37588, 37589, 37612, + 37613, 37610, 37611, 37605, 37606, 37607, 37608, 40951, 37634, 40951, + 40951, 37590, 40951, 37633, 37591, 37630, 37629, 37626, 37625, 37614, + 37624, 37623, 37628, 37627, 37616, 37602, 37601, 37593, 37592, 37615, + 37604, 37603, 37595, 37594, 37617, 37632, 37631, 37622, 37621, 37636, + 37637, 37600, 37598, 37609, 37618, 37619, 37620, 37635, 37597, 37599, + 37596, 40951, 37639, 37650, 37659, 37660, 37653, 37654, 37655, 37656, + 37657, 37658, 40951, 37662, 40951, 40951, 37651, 40951, 37661, 37652, + 37583, 37644, 40951, 37640, 37647, 37646, 37641, 37584, 37638, 37587, + 37645, 37586, 37585, 40951, 37642, 37643, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 37649, 37648, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 29204, 29205, 29218, 29219, 29216, + 29217, 29200, 29201, 29202, 29203, 29244, 29206, 29245, 29207, 29232, + 29231, 29228, 29227, 29193, 29192, 29226, 29225, 29230, 29229, 29195, + 29194, 29213, 29212, 29209, 29208, 29197, 29215, 29214, 29211, 29210, + 29198, 29196, 29238, 29237, 29224, 29223, 29236, 29235, 29243, 29240, + 29239, 29234, 29233, 29242, 29220, 29221, 29222, 29241, 29258, 29267, + 29268, 29261, 29262, 29263, 29264, 29265, 29266, 29269, 29259, 29270, + 29260, 29253, 29246, 29249, 29254, 29247, 29248, 29251, 29274, 29255, + 29180, 29178, 29273, 29191, 29271, 29187, 29189, 29186, 29185, 29182, + 29181, 29184, 29183, 29190, 29188, 29179, 29257, 40951, 29272, 29256, + 29199, 29250, 29252, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 37311, 37312, 37313, 37331, 37332, 37329, 37330, + 37324, 37325, 37326, 37327, 37357, 37314, 37358, 37315, 37349, 37348, + 37345, 37344, 37333, 37343, 37342, 37347, 37346, 37335, 37321, 37320, + 37317, 37316, 37334, 37323, 37322, 37319, 37318, 37336, 37351, 37350, + 37341, 37340, 37354, 37356, 37355, 37353, 37328, 37337, 37338, 37339, + 37352, 37367, 37376, 37377, 37370, 37371, 37372, 37373, 37374, 37375, + 37378, 37365, 37368, 37379, 37366, 37369, 37359, 37362, 37364, 37363, + 37360, 37361, 37390, 37310, 37391, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 37386, 37388, 37385, 37384, 37381, 37380, 37383, + 37382, 37389, 37387, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32855, + 32857, 32878, 32879, 32876, 32877, 32871, 32872, 32873, 32874, 32904, + 32858, 32905, 32859, 32896, 32895, 32892, 32891, 32880, 32890, 32889, + 32894, 32893, 32882, 32865, 32864, 32868, 32867, 32881, 32866, 32861, + 32870, 32869, 32883, 32898, 32897, 32888, 32887, 32901, 32903, 32902, + 32900, 32875, 32884, 32885, 32886, 32899, 32933, 32940, 32941, 32938, + 32939, 32936, 32937, 40951, 40951, 32942, 32934, 32943, 32935, 32926, + 32928, 32930, 32929, 32927, 32925, 32945, 32944, 32924, 32923, 32906, + 32907, 32908, 32854, 32921, 32920, 32918, 32916, 32917, 32909, 32910, + 32919, 32922, 32914, 32915, 32913, 32912, 32911, 32860, 32862, 32863, + 32856, 32931, 32932, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27636, 27637, 27655, + 27656, 27653, 27654, 27648, 27649, 27650, 27651, 27682, 27638, 27683, + 27639, 27675, 27674, 27671, 27670, 27659, 27669, 27668, 27673, 27672, + 27661, 27645, 27644, 27641, 27640, 27660, 27647, 27646, 27643, 27642, + 27662, 27677, 27676, 27667, 27666, 27679, 27681, 27680, 27658, 27652, + 27663, 27664, 27665, 27678, 27657, 27611, 27620, 27621, 27614, 27615, + 27616, 27617, 27618, 27619, 27622, 27612, 27623, 27613, 27607, 27610, + 27609, 27606, 27625, 27624, 27684, 27608, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27632, 27634, 27631, + 27630, 27627, 27626, 27629, 27628, 27635, 27633, 40951, 40951, 40951, + 40951, 40951, 40951, 28164, 28159, 28004, 28165, 28163, 28161, 28160, + 28021, 28022, 28155, 28157, 28156, 28166, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 35301, 35303, 35318, 35319, 35316, + 35317, 35343, 35304, 35344, 35305, 35333, 35332, 35329, 35328, 35320, + 35327, 35326, 35331, 35330, 35322, 35313, 35312, 35307, 35306, 35321, + 35315, 35314, 35309, 35308, 35323, 35335, 35334, 35325, 35324, 35340, + 35342, 35311, 35339, 35341, 35336, 35337, 35338, 35310, 35346, 35348, + 35349, 35354, 35355, 35352, 35353, 35356, 35350, 35357, 35351, 35347, + 35345, 35302, 35300, 40951, 40951, 40951, 40951, 40951, 40951, 35364, + 35366, 35363, 35362, 35359, 35358, 35361, 35360, 35367, 35365, 40951, + 40951, 40951, 40951, 40951, 40951, 28773, 28775, 28772, 28771, 28768, + 28767, 28770, 28769, 28776, 28774, 28723, 28725, 28722, 28721, 28718, + 28717, 28720, 28719, 28726, 28724, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 191, 190, 178, 181, 175, 167, 193, 192, 183, 195, + 189, 184, 174, 196, 177, 197, 180, 194, 164, 171, 170, 187, 166, 186, + 182, 188, 165, 40951, 40951, 162, 163, 161, 203, 204, 210, 211, 208, 209, + 212, 207, 213, 205, 206, 200, 40951, 40951, 40951, 40951, 222, 224, 221, + 220, 217, 216, 219, 218, 225, 223, 215, 214, 198, 199, 201, 202, 185, + 173, 172, 169, 168, 179, 176, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 11026, + 11027, 11042, 11043, 11040, 11041, 11068, 11028, 11069, 11029, 11060, + 11059, 11056, 11055, 11044, 11054, 11053, 11058, 11057, 11046, 11037, + 11036, 11031, 11030, 11045, 11039, 11038, 11033, 11032, 11047, 11062, + 11061, 11052, 11051, 11065, 11067, 11035, 11064, 11066, 11048, 11049, + 11050, 11063, 11034, 11072, 11077, 11078, 11075, 11076, 11070, 11071, + 11079, 11073, 11080, 11074, 11083, 11085, 11084, 11082, 11081, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 39052, + 39041, 39070, 39060, 39062, 39063, 39069, 39059, 39045, 39054, 39042, + 39072, 39068, 39047, 39061, 39058, 39046, 39055, 39064, 39053, 39071, + 39044, 39043, 39066, 39067, 39050, 39049, 39048, 39051, 39056, 39057, + 39065, 39084, 39073, 39102, 39092, 39094, 39095, 39101, 39091, 39077, + 39086, 39074, 39104, 39100, 39079, 39093, 39090, 39078, 39087, 39096, + 39085, 39103, 39076, 39075, 39098, 39099, 39082, 39081, 39080, 39083, + 39088, 39089, 39097, 39111, 39113, 39110, 39109, 39106, 39105, 39108, + 39107, 39114, 39112, 39123, 39122, 39121, 39117, 39116, 39120, 39119, + 39115, 39118, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 39124, 10942, 10943, 10950, 10951, 10948, + 10949, 10977, 40951, 40951, 10978, 40951, 40951, 10968, 10967, 10966, + 10965, 10954, 10964, 10963, 10972, 40951, 10956, 10938, 40951, 10945, + 10944, 10955, 10939, 10937, 10947, 10946, 10957, 10970, 10969, 10962, + 10961, 10973, 10941, 10940, 10974, 10953, 10975, 10958, 10959, 10960, + 10971, 10952, 10976, 10983, 10987, 10988, 10985, 10986, 10989, 40951, + 10984, 10990, 40951, 40951, 10982, 10980, 10979, 10991, 10996, 10993, + 10997, 10992, 10981, 10926, 10994, 10995, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 10933, 10935, 10932, 10931, 10928, + 10927, 10930, 10929, 10936, 10934, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 28895, 28896, 28911, 28912, 28909, + 28910, 28892, 28893, 40951, 40951, 28937, 28897, 28938, 28898, 28931, + 28930, 28927, 28926, 28915, 28925, 28924, 28929, 28928, 28917, 28906, + 28905, 28900, 28899, 28916, 28908, 28907, 28902, 28901, 28918, 28933, + 28932, 28923, 28922, 28935, 28936, 28904, 28914, 28894, 28919, 28920, + 28921, 28934, 28913, 28903, 28947, 28952, 28953, 28950, 28951, 28945, + 28946, 40951, 40951, 28954, 28948, 28955, 28949, 28941, 28943, 28942, + 28940, 28939, 28956, 28944, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40732, 40753, 40750, 40749, 40752, 40748, 40747, 40745, 40746, + 40751, 40744, 40700, 40699, 40719, 40718, 40701, 40717, 40716, 40726, + 40703, 40711, 40710, 40693, 40692, 40702, 40713, 40712, 40697, 40696, + 40704, 40721, 40720, 40715, 40714, 40728, 40709, 40708, 40695, 40694, + 40722, 40723, 40724, 40731, 40729, 40727, 40730, 40705, 40706, 40707, + 40725, 40698, 40691, 40741, 40738, 40739, 40740, 40737, 40742, 40688, + 40687, 40685, 40684, 40686, 40690, 40683, 40736, 40734, 40733, 40735, + 40689, 40682, 40743, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 34052, 34073, 34071, 34070, 34072, 34068, 34069, 34066, 34067, + 34065, 34064, 34074, 34019, 34018, 34038, 34037, 34020, 34036, 34035, + 34040, 34039, 34022, 34030, 34029, 34013, 34012, 34021, 34032, 34031, + 34016, 34014, 34023, 34042, 34041, 34034, 34033, 34048, 34028, 34027, + 34015, 34043, 34044, 34045, 34051, 34049, 34047, 34050, 34024, 34025, + 34026, 34046, 34017, 34057, 34059, 33996, 33995, 33993, 33994, 34004, + 34005, 34000, 34003, 33999, 34002, 34007, 34008, 34006, 33998, 33997, + 34001, 34060, 34058, 34075, 34061, 34056, 34055, 34054, 34053, 34011, + 34010, 34009, 34062, 34063, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 5617, 5618, 5615, 5616, + 5613, 5614, 5623, 5624, 5621, 5622, 5619, 5620, 5786, 5787, 5788, 5785, + 31247, 31245, 31254, 31255, 31251, 31259, 31258, 31240, 31253, 31252, + 31244, 31257, 31250, 31243, 31249, 31248, 31241, 31246, 31256, 31235, + 31242, 31260, 31261, 31236, 31262, 31238, 31239, 31237, 31231, 31228, + 31232, 31230, 31226, 31229, 31233, 31227, 31234, 31268, 31267, 31278, + 31269, 31270, 31279, 31275, 31274, 31276, 31277, 31271, 31225, 31272, + 31273, 31264, 31263, 31223, 31265, 31266, 31224, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 10668, 10669, 10758, 10759, 10760, 10761, + 10768, 10767, 10769, 10773, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 34658, 34677, 34684, 34681, + 34671, 34668, 34663, 34685, 34653, 34670, 34679, 34660, 34657, 34666, + 34683, 34661, 34680, 34667, 34672, 34678, 34682, 34655, 34654, 34659, + 34675, 34669, 34665, 34664, 34673, 34656, 34674, 34676, 34662, 34686, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 34693, 34695, 34692, 34691, 34688, 34687, + 34690, 34689, 34696, 34694, 40951, 40951, 40951, 40951, 40951, 40951, + 3661, 3662, 3675, 3676, 3673, 3674, 3657, 3658, 3659, 40951, 3701, 3663, + 3702, 3664, 3693, 3692, 3689, 3688, 3677, 3687, 3686, 3691, 3690, 3679, + 3670, 3669, 3666, 3665, 3678, 3672, 3671, 3668, 3667, 3680, 3695, 3694, + 3685, 3684, 3698, 3700, 3699, 3697, 3660, 3681, 3682, 3683, 3696, 3729, + 3734, 3735, 3732, 3733, 3726, 3727, 3728, 40951, 3736, 3730, 3737, 3731, + 3721, 3723, 3725, 3724, 3722, 3739, 3738, 3750, 3751, 3752, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 3746, 3748, 3745, + 3744, 3741, 3740, 3743, 3742, 3749, 3747, 3720, 3718, 3715, 3706, 3708, + 3713, 3711, 3703, 3709, 3719, 3717, 3716, 3705, 3707, 3714, 3712, 3704, + 3710, 3753, 40951, 40951, 40951, 25735, 25736, 25681, 25680, 25690, + 25675, 25679, 25678, 25692, 25676, 25672, 25671, 25674, 25677, 25683, + 25682, 25689, 25694, 25670, 25669, 25673, 25696, 25686, 25687, 25688, + 25697, 25695, 25693, 25684, 25685, 25691, 25698, 40951, 40951, 25713, + 25712, 25721, 25707, 25711, 25710, 25723, 25708, 25704, 25703, 25706, + 25709, 25715, 25714, 25720, 25725, 25702, 25701, 25705, 25727, 25718, + 25719, 40951, 25728, 25726, 25724, 25716, 25717, 25722, 25729, 25730, + 25732, 25734, 25731, 25733, 25700, 25699, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25747, + 25748, 25757, 25758, 25755, 25756, 25784, 40951, 25749, 25785, 40951, + 25750, 25763, 25762, 25776, 25775, 25764, 25774, 25773, 25741, 25740, + 25766, 25743, 25742, 25752, 25751, 25765, 25746, 25744, 25754, 25753, + 25767, 25778, 25777, 25772, 25771, 25780, 25783, 25781, 25760, 25782, + 25768, 25769, 25770, 25779, 25759, 25761, 25739, 25745, 25794, 25799, + 25800, 25797, 25798, 25793, 40951, 40951, 40951, 25801, 40951, 25795, + 25802, 40951, 25796, 25792, 25791, 25790, 25788, 25789, 25803, 25787, + 25786, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25810, + 25812, 25809, 25808, 25805, 25804, 25807, 25806, 25813, 25811, 40951, + 40951, 40951, 40951, 40951, 40951, 18770, 18771, 18784, 18785, 18782, + 18783, 40951, 18801, 18772, 40951, 18800, 18773, 18807, 18806, 18789, + 18788, 18803, 18797, 18796, 18781, 18780, 18787, 18793, 18792, 18777, + 18776, 18769, 18791, 18790, 18779, 18778, 18786, 18795, 18794, 18775, + 18774, 18768, 18799, 18798, 18802, 18804, 18805, 18810, 18815, 18816, + 18813, 18814, 40951, 18818, 18811, 40951, 18817, 18812, 18809, 18808, + 18819, 18830, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 18826, + 18828, 18825, 18824, 18821, 18820, 18823, 18822, 18829, 18827, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25433, + 25431, 25425, 25436, 25428, 25435, 25439, 25430, 25427, 25429, 25432, + 25426, 25441, 25437, 25434, 25440, 25438, 25442, 25448, 25445, 25447, + 25444, 25446, 25443, 25424, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 21513, 21517, 21515, 21516, 21454, 21455, 21474, 21475, 21468, + 21469, 21470, 21471, 21472, 21473, 21499, 21456, 21500, 40951, 21490, + 21489, 21488, 21487, 21476, 21486, 21485, 21459, 21458, 21478, 21465, + 21464, 21461, 21460, 21477, 21467, 21466, 21463, 21462, 21479, 21492, + 21491, 21484, 21483, 21495, 21498, 21496, 21494, 21497, 21480, 21481, + 21482, 21493, 21457, 21519, 21518, 21526, 21527, 21524, 21525, 21521, + 40951, 40951, 40951, 21522, 21520, 21523, 21512, 21540, 21529, 21528, + 21507, 21510, 21506, 21508, 21504, 21503, 21511, 21502, 21505, 21509, + 21501, 21536, 21538, 21535, 21534, 21531, 21530, 21533, 21532, 21539, + 21537, 21514, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25086, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 35424, 35420, 35414, 35423, 35416, 35425, 35429, + 35431, 35426, 35421, 35422, 35427, 35415, 35432, 35430, 35417, 35428, + 35418, 35419, 35433, 35434, 35498, 35481, 35479, 35488, 35487, 35483, + 35489, 35485, 35484, 35491, 35492, 35493, 35490, 35482, 35495, 35413, + 35400, 35471, 35478, 35768, 35769, 35398, 35373, 35499, 35767, 35435, + 35500, 35486, 35494, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 35476, 8820, 8828, 8826, 8822, + 8827, 8823, 8821, 8824, 8825, 8889, 8829, 8838, 8837, 8831, 8830, 8842, + 8832, 8833, 8841, 8835, 8836, 8843, 8844, 8849, 8846, 8845, 8847, 8848, + 8851, 8853, 8855, 8854, 8856, 8862, 8859, 8860, 8864, 8857, 8858, 8863, + 8861, 8866, 8865, 8867, 8869, 8870, 8874, 8873, 8871, 8872, 8875, 8888, + 8876, 8877, 8878, 8887, 8879, 8883, 8884, 8882, 8880, 8881, 8886, 8885, + 8890, 8891, 8902, 8893, 8897, 8898, 8899, 8900, 8901, 8903, 8906, 8905, + 8904, 8907, 8909, 8910, 8911, 8912, 8913, 8914, 8915, 8916, 8917, 8918, + 8919, 8920, 8921, 8922, 8923, 8924, 8925, 8926, 8941, 8927, 8928, 8938, + 8932, 8929, 8930, 8931, 8939, 8936, 8940, 8937, 8933, 8935, 8948, 8944, + 8945, 8946, 8949, 8960, 8950, 8953, 8954, 8956, 8957, 8958, 8961, 8964, + 8962, 8963, 8965, 8966, 8968, 8969, 8998, 9005, 8999, 9000, 9001, 9002, + 9003, 9004, 9006, 9008, 9007, 9009, 9012, 9013, 9016, 9010, 9011, 9017, + 9062, 9061, 9063, 9018, 9021, 9022, 9023, 9019, 9020, 9024, 9027, 9025, + 9028, 9030, 9039, 9040, 9041, 9042, 9060, 9043, 9044, 9057, 9058, 9056, + 9045, 9046, 9047, 9048, 9049, 9050, 9051, 9054, 9055, 9064, 9154, 9065, + 9066, 9068, 9067, 9074, 9069, 9071, 9073, 9077, 9076, 9078, 9079, 9086, + 9080, 9081, 9085, 9089, 9090, 9087, 9088, 9096, 9093, 9097, 9098, 9099, + 9100, 9101, 9103, 9104, 9106, 9105, 9108, 9107, 9110, 9109, 9111, 9112, + 9114, 9115, 9119, 9121, 9125, 9126, 9139, 9131, 9132, 9127, 9128, 9129, + 9133, 9137, 9134, 9135, 9136, 9140, 9141, 9143, 9144, 9145, 9146, 9147, + 9148, 9158, 9149, 9150, 9153, 9152, 9151, 9155, 9156, 9157, 9159, 9160, + 9163, 9164, 9165, 9166, 9167, 9169, 9168, 9185, 9176, 9177, 9170, 9171, + 9173, 9174, 9172, 9175, 9184, 9178, 9183, 9181, 9180, 9182, 9187, 9206, + 9188, 9189, 9190, 9193, 9192, 9194, 9195, 9197, 9198, 9199, 9207, 9200, + 9201, 9202, 9205, 9204, 9203, 9208, 9209, 9211, 9212, 9213, 9214, 9216, + 9220, 9217, 9221, 9219, 9218, 9222, 9223, 9224, 9225, 9229, 9228, 9226, + 9227, 9230, 9231, 9233, 9252, 9254, 9234, 9236, 9235, 9237, 9238, 9241, + 9242, 9240, 9239, 9243, 9244, 9245, 9246, 9249, 9247, 9248, 9250, 9251, + 9255, 9256, 9253, 9257, 9258, 9259, 9260, 9262, 9265, 9264, 9266, 9267, + 9269, 9270, 9271, 9275, 9274, 9272, 9273, 9276, 9280, 9279, 9278, 9281, + 9282, 9284, 9285, 9289, 9286, 9287, 9292, 9290, 9293, 9294, 9296, 9295, + 9297, 9298, 9320, 9319, 9322, 9323, 9304, 9305, 9302, 9303, 9301, 9299, + 9307, 9306, 9308, 9310, 9316, 9317, 9314, 9315, 9324, 9325, 9326, 9344, + 9329, 9330, 9331, 9327, 9328, 9332, 9333, 9334, 9335, 9336, 9338, 9339, + 9340, 9341, 9342, 9367, 9345, 9348, 9346, 9347, 9353, 9354, 9351, 9352, + 9349, 9350, 9355, 9356, 9364, 9357, 9358, 9365, 9362, 9363, 9366, 9359, + 9360, 9361, 9368, 9369, 9370, 9371, 9372, 9373, 9374, 9376, 9377, 9375, + 9378, 9379, 9420, 9421, 9382, 9383, 9380, 9381, 9386, 9387, 9385, 9391, + 9388, 9390, 9389, 9395, 9396, 9394, 9392, 9393, 9397, 9398, 9399, 9400, + 9401, 9402, 9403, 9422, 9408, 9404, 9405, 9406, 9407, 9409, 9411, 9410, + 9412, 9413, 9415, 9414, 9416, 9418, 9417, 9423, 9424, 9427, 9428, 9425, + 9426, 9502, 9497, 9498, 9499, 9500, 9501, 9503, 9506, 9504, 9505, 9507, + 9549, 9508, 9538, 9539, 9515, 9517, 9534, 9518, 9540, 9522, 9520, 9521, + 9523, 9524, 9525, 9526, 9535, 9536, 9529, 9530, 9532, 9541, 9509, 9510, + 9514, 9512, 9550, 9542, 9544, 9543, 9545, 9551, 9552, 9546, 9547, 9548, + 9553, 9554, 9555, 9558, 9559, 9560, 9556, 9557, 9561, 9562, 9564, 9566, + 9567, 9585, 9583, 9584, 9587, 9586, 9568, 9575, 9573, 9574, 9569, 9570, + 9576, 9577, 9578, 9579, 9580, 9582, 9588, 9597, 9589, 9591, 9592, 9590, + 9593, 9595, 9594, 9596, 9599, 9601, 9600, 9602, 9603, 9636, 9638, 9604, + 9606, 9605, 9608, 9611, 9609, 9610, 9614, 9618, 9621, 9620, 9623, 9626, + 9624, 9625, 9629, 9631, 9637, 9639, 9640, 9644, 9651, 9649, 9647, 9648, + 9650, 9653, 9652, 9645, 9646, 9654, 9657, 9663, 9658, 9661, 9656, 9655, + 9664, 9662, 9659, 9660, 9665, 9666, 9667, 9668, 9669, 9670, 9671, 9673, + 9676, 9677, 9680, 9681, 9682, 9678, 9679, 9674, 9675, 9683, 9684, 9685, + 9686, 9687, 9688, 9690, 9691, 9692, 9693, 9694, 9698, 9695, 9719, 9712, + 9713, 9718, 9701, 9700, 9714, 9717, 9715, 9704, 9703, 9706, 9708, 9709, + 9710, 9711, 9707, 9720, 9696, 9721, 9722, 9723, 9724, 9725, 9726, 9734, + 9732, 9730, 9733, 9729, 9731, 9727, 9728, 9735, 9737, 9738, 9739, 9746, + 9741, 9742, 9750, 9751, 9747, 9749, 9748, 9752, 9754, 9753, 9755, 9766, + 9757, 9756, 9765, 9763, 9758, 9759, 9760, 9762, 9761, 9764, 9770, 9767, + 9769, 9768, 9771, 9772, 9773, 9774, 9777, 9778, 9780, 9781, 9782, 9783, + 9785, 9784, 9786, 9793, 9787, 9788, 9794, 9789, 9790, 9791, 9792, 9795, + 9799, 9796, 9797, 9798, 9800, 9801, 9802, 9803, 9809, 9807, 9805, 9806, + 9804, 9808, 9810, 9812, 9829, 9830, 9813, 9818, 9820, 9814, 9817, 9815, + 9816, 9821, 9827, 9828, 9822, 9825, 9826, 9831, 9837, 9836, 9832, 9834, + 9833, 9918, 9919, 9838, 9839, 9844, 9845, 9842, 9843, 9846, 9840, 9841, + 9847, 9850, 9851, 9853, 9854, 9855, 9859, 9856, 9857, 9858, 9848, 9849, + 9860, 9862, 9861, 9863, 9865, 9866, 9867, 9873, 9872, 9868, 9869, 9870, + 9905, 9874, 9875, 9876, 9877, 9878, 9897, 9880, 9881, 9883, 9882, 9884, + 9885, 9901, 9886, 9888, 9887, 9900, 9890, 9898, 9902, 9893, 9892, 9899, + 9894, 9896, 9895, 9903, 9904, 9906, 9910, 9908, 9909, 9907, 9913, 9912, + 9911, 9917, 9914, 9915, 9916, 9920, 9922, 9921, 9925, 9923, 9940, 9926, + 9929, 9931, 9927, 9928, 9932, 9930, 9933, 9935, 9937, 9938, 9939, 9343, + 8839, 8850, 8868, 8934, 8943, 8959, 8967, 9059, 9052, 9070, 9072, 9162, + 9186, 9232, 9261, 9263, 9277, 9283, 9318, 9291, 9321, 9300, 9309, 9313, + 9384, 9513, 9516, 9531, 9563, 9581, 9598, 9607, 9635, 9633, 9612, 9643, + 9672, 9689, 9699, 9819, 9852, 9835, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 8804, 8799, 8736, 8722, + 8783, 8779, 8711, 8756, 8803, 8744, 8728, 8789, 8778, 8710, 8755, 8742, + 8726, 8785, 8775, 8705, 8752, 8765, 8809, 8801, 8738, 8724, 8787, 8777, + 8707, 8754, 8766, 8810, 8802, 8739, 8725, 8811, 8793, 8794, 8740, 8716, + 8782, 8774, 8704, 8751, 8769, 8812, 8795, 8796, 8741, 8717, 8780, 8781, + 8764, 8807, 8790, 8791, 8731, 8721, 8797, 8798, 8732, 8735, 8733, 8734, + 8788, 8773, 8771, 8772, 8708, 8709, 8747, 8749, 8750, 8748, 8805, 8800, + 8737, 8723, 8784, 8763, 8806, 8792, 8729, 8730, 8719, 8720, 8746, 8745, + 8759, 8808, 8768, 8814, 8718, 8767, 8813, 8760, 8761, 8757, 8758, 8762, + 8770, 8712, 8715, 8714, 8713, 8743, 8727, 8786, 8776, 8706, 8753, 40951, + 8819, 8818, 8817, 8815, 8816, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 8840, 8834, 8852, 8892, 8894, 8895, + 8896, 8908, 8947, 8942, 8952, 8951, 8955, 8972, 8970, 8971, 8973, 8974, + 8992, 8978, 8975, 8976, 8977, 8994, 8995, 8993, 8982, 8981, 8979, 8980, + 8985, 8983, 8984, 8986, 8987, 8988, 8989, 8996, 8997, 8991, 8990, 9015, + 9014, 9026, 9029, 9034, 9038, 9031, 9035, 9036, 9032, 9033, 9037, 9053, + 9075, 9082, 9083, 9084, 9091, 9092, 9095, 9094, 9102, 9113, 9116, 9117, + 9118, 9120, 9122, 9123, 9124, 9130, 9138, 9142, 9161, 9179, 9191, 9196, + 9210, 9215, 9268, 9288, 9311, 9312, 9419, 9439, 9429, 9430, 9438, 9434, + 9435, 9436, 9433, 9432, 9431, 9437, 9441, 9440, 9447, 9448, 9442, 9443, + 9444, 9449, 9445, 9446, 9450, 9451, 9452, 9453, 9454, 9455, 9462, 9456, + 9461, 9460, 9458, 9457, 9459, 9463, 9464, 9469, 9470, 9465, 9466, 9467, + 9468, 9496, 9492, 9471, 9479, 9480, 9478, 9477, 9481, 9472, 9473, 9475, + 9476, 9474, 9493, 9482, 9489, 9491, 9486, 9487, 9490, 9485, 9488, 9484, + 9483, 9494, 9495, 9511, 9537, 9519, 9527, 9528, 9533, 9565, 9572, 9571, + 9632, 9613, 9615, 9634, 9616, 9617, 9619, 9622, 9627, 9628, 9630, 9697, + 9716, 9702, 9705, 9736, 9740, 9743, 9744, 9745, 9776, 9775, 9779, 9811, + 9823, 9824, 9864, 9871, 9879, 9891, 9889, 9924, 9934, 9936, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 10013, 10014, 10015, 10016, 10017, 10018, 10019, 10020, 10023, 10024, + 10021, 10022, 10025, 10026, 10027, 10028, 10029, 10030, 10031, 10032, + 10033, 10034, 10035, 10036, 10037, 10038, 10039, 10040, 10041, 10042, + 10043, 10044, 10045, 10046, 10047, 10048, 10049, 10050, 10051, 10052, + 10053, 10054, 10055, 10056, 10057, 10058, 10059, 10079, 10080, 10081, + 10082, 10083, 10084, 10085, 10086, 10087, 10062, 10063, 10064, 10065, + 10066, 10060, 10061, 10067, 10068, 10069, 10088, 10089, 10090, 10091, + 10092, 10093, 10094, 10095, 10096, 10097, 10070, 10071, 10072, 10073, + 10074, 10075, 10076, 10077, 10078, 10098, 10099, 10100, 10101, 10102, + 10103, 10104, 10105, 10106, 10107, 10108, 10109, 10110, 10111, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 11548, 11549, 11550, 11551, 11546, 11547, 11543, 11544, + 11545, 11552, 11553, 11554, 11559, 11560, 11561, 11562, 11555, 11556, + 11563, 11564, 11557, 11558, 11565, 11566, 11593, 11594, 11595, 11596, + 11597, 11598, 11599, 11600, 11601, 11602, 11583, 11584, 11581, 11582, + 11585, 11586, 11587, 11588, 11589, 11590, 11591, 11567, 11568, 11575, + 11569, 11570, 11571, 11572, 11576, 11573, 11574, 11577, 11578, 11579, + 11580, 11603, 11604, 11605, 11606, 11607, 11608, 11609, 11610, 11611, + 11612, 11613, 11614, 11615, 11616, 11617, 11618, 11619, 11620, 11621, + 11622, 11592, 11662, 11663, 11664, 11665, 11660, 11661, 11666, 11667, + 11668, 11669, 11674, 11670, 11671, 11672, 11673, 11675, 11676, 11677, + 11678, 11679, 11680, 11681, 11682, 11683, 11684, 11685, 11686, 11687, + 11688, 11689, 11690, 11691, 11692, 11693, 11694, 11695, 11696, 11697, + 11700, 11701, 11702, 11703, 11704, 11705, 11706, 11698, 11699, 11707, + 11780, 11781, 11782, 11783, 11784, 11785, 11786, 11787, 11788, 11789, + 11771, 11772, 11773, 11774, 11775, 11776, 11777, 11769, 11770, 11778, + 11779, 11712, 11708, 11709, 11713, 11714, 11710, 11711, 11715, 11716, + 11717, 11718, 11719, 11724, 11725, 11726, 11727, 11728, 11729, 11720, + 11721, 11730, 11722, 11723, 11731, 11732, 11733, 11734, 11735, 11736, + 11737, 11738, 11739, 11740, 11741, 11746, 11742, 11743, 11747, 11744, + 11745, 11748, 11749, 11750, 11751, 11752, 11762, 11763, 11764, 11765, + 11766, 11767, 11768, 11753, 11754, 11755, 11756, 11757, 11758, 11759, + 11760, 11761, 11794, 11795, 11796, 11797, 11798, 11799, 11800, 11790, + 11791, 11792, 11793, 11826, 11827, 11828, 11829, 11830, 11831, 11822, + 11823, 11824, 11825, 11832, 11833, 11801, 11802, 11805, 11806, 11807, + 11808, 11809, 11810, 11811, 11803, 11804, 11812, 11815, 11816, 11817, + 11818, 11813, 11814, 11819, 11820, 11821, 11837, 11838, 11839, 11840, + 11841, 11842, 11843, 11844, 11845, 11846, 11849, 11850, 11851, 11847, + 11848, 11852, 11853, 11854, 11855, 11856, 11857, 11893, 11891, 11892, + 11894, 11895, 11896, 11897, 11898, 11899, 11900, 11901, 11864, 11858, + 11859, 11865, 11866, 11867, 11868, 11869, 11860, 11861, 11862, 11863, + 11870, 11877, 11878, 11879, 11880, 11881, 11871, 11872, 11873, 11874, + 11875, 11876, 11882, 11883, 11888, 11884, 11885, 11886, 11887, 11889, + 11890, 11908, 11909, 11910, 11911, 11912, 11906, 11907, 11903, 11904, + 11905, 11913, 11914, 11917, 11915, 11916, 11918, 11919, 11920, 11921, + 11922, 11923, 11924, 11925, 11950, 11951, 11954, 11955, 11956, 11957, + 11958, 11952, 11953, 11959, 11960, 11961, 11930, 11931, 11932, 11933, + 11934, 11935, 11926, 11927, 11928, 11929, 11936, 11937, 11942, 11943, + 11944, 11938, 11939, 11945, 11940, 11941, 11946, 11947, 11948, 11949, + 11962, 11963, 11964, 11965, 11966, 11969, 11970, 11971, 11972, 11973, + 11967, 11968, 11974, 11975, 11983, 11984, 11985, 11986, 11979, 11980, + 11987, 11988, 11989, 11981, 11982, 11990, 11991, 11992, 11993, 11994, + 11995, 11996, 11997, 12638, 12639, 12640, 12641, 12642, 12643, 12644, + 12645, 12009, 12005, 12006, 12010, 12011, 12012, 12007, 12008, 12013, + 12014, 12016, 12017, 12018, 12021, 12019, 12020, 12022, 12023, 12024, + 12025, 12026, 12027, 12037, 12038, 12045, 12028, 12029, 12030, 12031, + 12032, 12033, 12034, 12035, 12036, 12046, 12047, 12039, 12040, 12041, + 12042, 12043, 12044, 12048, 12049, 12056, 12057, 12050, 12051, 12058, + 12052, 12053, 12059, 12060, 12061, 12054, 12055, 12062, 12068, 12066, + 12067, 12069, 12063, 12064, 12065, 12070, 12071, 12072, 12073, 12074, + 12075, 12076, 12077, 12078, 12079, 12080, 12081, 12138, 12139, 12140, + 12141, 12142, 12143, 12144, 12145, 12146, 12101, 12102, 12103, 12104, + 12105, 12106, 12107, 12108, 12098, 12099, 12100, 12109, 12126, 12127, + 12128, 12129, 12130, 12124, 12125, 12131, 12132, 12133, 12134, 12118, + 12119, 12120, 12110, 12111, 12112, 12113, 12114, 12115, 12121, 12116, + 12117, 12122, 12123, 12135, 12136, 12137, 12149, 12150, 12151, 12152, + 12147, 12148, 12153, 12154, 12155, 12156, 12159, 12160, 12161, 12162, + 12163, 12164, 12165, 12157, 12158, 12166, 12167, 12168, 12186, 12187, + 12188, 12189, 12190, 12191, 12192, 12193, 12194, 12169, 12170, 12171, + 12172, 12175, 12176, 12177, 12178, 12179, 12180, 12173, 12174, 12181, + 12184, 12185, 12182, 12183, 12202, 12203, 12206, 12207, 12208, 12204, + 12205, 12195, 12196, 12197, 12198, 12199, 12200, 12201, 12209, 12210, + 12211, 12212, 12213, 12214, 12215, 12218, 12219, 12220, 12221, 12222, + 12223, 12224, 12225, 12216, 12217, 12226, 12227, 12234, 12235, 12236, + 12228, 12229, 12230, 12231, 12237, 12238, 12239, 12232, 12233, 12245, + 12246, 12249, 12250, 12247, 12248, 12251, 12252, 12240, 12241, 12242, + 12243, 12244, 12253, 12254, 12255, 12260, 12261, 12262, 12263, 12264, + 12265, 12266, 12267, 12268, 12269, 12256, 12257, 12258, 12259, 12271, + 12272, 12275, 12273, 12274, 12276, 12277, 12278, 12279, 12280, 12281, + 12282, 12283, 12646, 12647, 12648, 12649, 12650, 12651, 12652, 12289, + 12287, 12288, 12284, 12285, 12286, 12290, 12291, 12292, 12293, 12294, + 12295, 12296, 12297, 12300, 12301, 12302, 12303, 12304, 12298, 12299, + 12305, 12306, 12307, 12308, 12309, 12310, 12311, 12312, 12313, 12314, + 12315, 12316, 12317, 12322, 12318, 12319, 12323, 12324, 12325, 12320, + 12321, 12326, 12327, 12328, 12334, 12335, 12336, 12337, 12329, 12330, + 12331, 12338, 12339, 12332, 12333, 12340, 12341, 12345, 12346, 12347, + 12348, 12349, 12350, 12342, 12343, 12344, 12351, 12352, 12353, 12356, + 12357, 12358, 12359, 12360, 12354, 12355, 12361, 12362, 12363, 12364, + 12365, 12366, 12367, 12368, 12369, 12370, 12371, 12380, 12381, 12372, + 12373, 12382, 12383, 12384, 12374, 12375, 12376, 12377, 12378, 12379, + 12389, 12385, 12386, 12390, 12391, 12392, 12393, 12387, 12388, 12394, + 12395, 12396, 12406, 12407, 12408, 12409, 12410, 12411, 12412, 12413, + 12414, 12415, 12401, 12402, 12397, 12398, 12399, 12400, 12403, 12404, + 12405, 12420, 12421, 12422, 12423, 12424, 12417, 12418, 12419, 12425, + 12426, 12427, 12454, 12455, 12456, 12457, 12458, 12459, 12460, 12461, + 12462, 12463, 12432, 12433, 12434, 12428, 12429, 12435, 12436, 12437, + 12438, 12439, 12430, 12431, 12442, 12443, 12440, 12441, 12444, 12445, + 12446, 12447, 12448, 12449, 12450, 12451, 12452, 12453, 12467, 12468, + 12469, 12470, 12471, 12472, 12473, 12474, 12475, 12476, 12477, 12478, + 12479, 12480, 12481, 12482, 12464, 12465, 12466, 12483, 12484, 12493, + 12485, 12486, 12487, 12488, 12490, 12491, 12492, 12494, 12495, 12496, + 12497, 12498, 12499, 12500, 12501, 12502, 12503, 12504, 12505, 12506, + 12507, 12508, 12509, 12510, 12511, 12512, 12513, 12520, 12521, 12514, + 12515, 12522, 12523, 12524, 12525, 12516, 12517, 12518, 12519, 12526, + 12527, 12528, 12529, 12534, 12530, 12531, 12535, 12536, 12537, 12532, + 12533, 12538, 12539, 12540, 12541, 12547, 12548, 12543, 12544, 12549, + 12550, 12551, 12552, 12553, 12545, 12546, 12554, 12555, 12562, 12563, + 12564, 12556, 12557, 12565, 12566, 12558, 12559, 12560, 12561, 12567, + 12570, 12571, 12572, 12573, 12568, 12569, 12574, 12583, 12584, 12585, + 12576, 12577, 12578, 12586, 12579, 12580, 12587, 12581, 12582, 12588, + 12589, 12590, 12591, 12592, 12593, 12594, 12595, 12596, 12609, 12597, + 12598, 12599, 12600, 12601, 12602, 12603, 12604, 12605, 12606, 12607, + 12608, 12610, 12611, 12612, 12613, 12633, 12634, 12635, 12636, 12637, + 12614, 12615, 12616, 12617, 12618, 12619, 12620, 12621, 12622, 12623, + 12624, 12625, 12626, 12627, 12628, 12629, 12630, 12631, 12632, 11626, + 11627, 11628, 11629, 11630, 11631, 11623, 11624, 11625, 11632, 11633, + 11637, 11638, 11639, 11640, 11641, 11642, 11643, 11644, 11645, 11646, + 11647, 11648, 11649, 11650, 11651, 11652, 11653, 11654, 11655, 11656, + 11634, 11635, 11636, 12489, 12542, 11978, 12003, 12000, 12002, 11999, + 12270, 11659, 11836, 12004, 12001, 11998, 11658, 11835, 11657, 11834, + 12097, 11902, 11976, 12015, 11977, 12416, 12575, 12093, 12084, 12088, + 12095, 12091, 12085, 12090, 12087, 12094, 12083, 12089, 12096, 12092, + 12086, 12082, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 12653, 12654, 12655, 12656, 12657, 12658, 12659, 12660, + 12661, 12662, 12663, 12664, 12665, 12666, 12667, 12668, 12669, 12670, + 12671, 12672, 12673, 12674, 12675, 12676, 12677, 12678, 12679, 12680, + 12681, 12682, 12683, 12684, 12685, 12686, 12687, 12688, 12689, 12690, + 12691, 12692, 12693, 12694, 12695, 12696, 12697, 12698, 12699, 12700, + 12701, 12702, 12703, 12704, 12705, 12706, 12707, 12708, 12709, 12710, + 12711, 12712, 12713, 12714, 12715, 12716, 12717, 12718, 12719, 12720, + 12721, 12722, 12723, 12724, 12725, 12726, 12727, 12728, 12729, 12730, + 12731, 12732, 12733, 12734, 12735, 12736, 12737, 12738, 12739, 12740, + 12741, 12742, 12743, 12744, 12745, 12746, 12747, 12748, 12749, 12750, + 12751, 12752, 12753, 12754, 12755, 12756, 12757, 12758, 12759, 12760, + 12761, 12762, 12763, 12764, 12765, 12766, 12767, 12768, 12769, 12770, + 12771, 12772, 12773, 12774, 12775, 12776, 12777, 12778, 12779, 12780, + 12781, 12782, 12783, 12784, 12785, 12786, 12787, 12788, 12789, 12790, + 12791, 12792, 12793, 12794, 12795, 12796, 12797, 12798, 12799, 12800, + 12801, 12802, 12803, 12804, 12805, 12806, 12807, 12808, 12809, 12810, + 12811, 12812, 12813, 12814, 12815, 12816, 12817, 12818, 12819, 12820, + 12821, 12822, 12823, 12824, 12825, 12826, 12827, 12828, 12829, 12830, + 12831, 12832, 12833, 12834, 12835, 12836, 12837, 12838, 12839, 12840, + 12841, 12842, 12843, 12844, 12845, 12846, 12847, 12848, 12849, 12850, + 12851, 12852, 12853, 12854, 12855, 12856, 12857, 12858, 12859, 12860, + 12861, 12862, 12863, 12864, 12865, 12866, 12867, 12868, 12869, 12870, + 12871, 12872, 12873, 12874, 12875, 12876, 12877, 12878, 12879, 12880, + 12881, 12882, 12883, 12884, 12885, 12886, 12887, 12888, 12889, 12890, + 12891, 12892, 12893, 12894, 12895, 12896, 12897, 12898, 12899, 12900, + 12901, 12902, 12903, 12904, 12905, 12906, 12907, 12908, 12909, 12910, + 12911, 12912, 12913, 12914, 12915, 12916, 12917, 12918, 12919, 12920, + 12921, 12922, 12923, 12924, 12925, 12926, 12927, 12928, 12929, 12930, + 12931, 12932, 12933, 12934, 12935, 12936, 12937, 12938, 12939, 12940, + 12941, 12942, 12943, 12944, 12945, 12946, 12947, 12948, 12949, 12950, + 12951, 12952, 12953, 12954, 12955, 12956, 12957, 12958, 12959, 12960, + 12961, 12962, 12963, 12964, 12965, 12966, 12967, 12968, 12969, 12970, + 12971, 12972, 12973, 12974, 12975, 12976, 12977, 12978, 12979, 12980, + 12981, 12982, 12983, 12984, 12985, 12986, 12987, 12988, 12989, 12990, + 12991, 12992, 12993, 12994, 12995, 12996, 12997, 12998, 12999, 13000, + 13001, 13002, 13003, 13004, 13005, 13006, 13007, 13008, 13009, 13010, + 13011, 13012, 13013, 13014, 13015, 13016, 13017, 13018, 13019, 13020, + 13021, 13022, 13023, 13024, 13025, 13026, 13027, 13028, 13029, 13030, + 13031, 13032, 13033, 13034, 13035, 13036, 13037, 13038, 13039, 13040, + 13041, 13042, 13043, 13044, 13045, 13046, 13047, 13048, 13049, 13050, + 13051, 13052, 13053, 13054, 13055, 13056, 13057, 13058, 13059, 13060, + 13061, 13062, 13063, 13064, 13065, 13066, 13067, 13068, 13069, 13070, + 13071, 13072, 13073, 13074, 13075, 13076, 13077, 13078, 13079, 13080, + 13081, 13082, 13083, 13084, 13085, 13086, 13087, 13088, 13089, 13090, + 13091, 13092, 13093, 13094, 13095, 13096, 13097, 13098, 13099, 13100, + 13101, 13102, 13103, 13104, 13105, 13106, 13107, 13108, 13109, 13110, + 13111, 13112, 13113, 13114, 13115, 13116, 13117, 13118, 13119, 13120, + 13121, 13122, 13123, 13124, 13125, 13126, 13127, 13128, 13129, 13130, + 13131, 13132, 13133, 13134, 13135, 13136, 13137, 13138, 13139, 13140, + 13141, 13142, 13143, 13144, 13145, 13146, 13147, 13148, 13149, 13150, + 13151, 13152, 13153, 13154, 13155, 13156, 13157, 13158, 13159, 13160, + 13161, 13162, 13163, 13164, 13165, 13166, 13167, 13168, 13169, 13170, + 13171, 13172, 13173, 13174, 13175, 13176, 13177, 13178, 13179, 13180, + 13181, 13182, 13183, 13184, 13185, 13186, 13187, 13188, 13189, 13190, + 13191, 13192, 13193, 13194, 13195, 13196, 13197, 13198, 13199, 13200, + 13201, 13202, 13203, 13204, 13205, 13206, 13207, 13208, 13209, 13210, + 13211, 13212, 13213, 13214, 13215, 13216, 13217, 13218, 13219, 13220, + 13221, 13222, 13223, 13224, 13225, 13226, 13227, 13228, 13229, 13230, + 13231, 13232, 13233, 13234, 13235, 13236, 13237, 13238, 13239, 13240, + 13241, 13242, 13243, 13244, 13245, 13246, 13247, 13248, 13249, 13250, + 13251, 13252, 13253, 13254, 13255, 13256, 13257, 13258, 13259, 13260, + 13261, 13262, 13263, 13264, 13265, 13266, 13267, 13268, 13269, 13270, + 13271, 13272, 13273, 13274, 13275, 13276, 13277, 13278, 13279, 13280, + 13281, 13282, 13283, 13284, 13285, 13286, 13287, 13288, 13289, 13290, + 13291, 13292, 13293, 13294, 13295, 13296, 13297, 13298, 13299, 13300, + 13301, 13302, 13303, 13304, 13305, 13306, 13307, 13308, 13309, 13310, + 13311, 13312, 13313, 13314, 13315, 13316, 13317, 13318, 13319, 13320, + 13321, 13322, 13323, 13324, 13325, 13326, 13327, 13328, 13329, 13330, + 13331, 13332, 13333, 13334, 13335, 13336, 13337, 13338, 13339, 13340, + 13341, 13342, 13343, 13344, 13345, 13346, 13347, 13348, 13349, 13350, + 13351, 13352, 13353, 13354, 13355, 13356, 13357, 13358, 13359, 13360, + 13361, 13362, 13363, 13364, 13365, 13366, 13367, 13368, 13369, 13370, + 13371, 13372, 13373, 13374, 13375, 13376, 13377, 13378, 13379, 13380, + 13381, 13382, 13383, 13384, 13385, 13386, 13387, 13388, 13389, 13390, + 13391, 13392, 13393, 13394, 13395, 13396, 13397, 13398, 13399, 13400, + 13401, 13402, 13403, 13404, 13405, 13406, 13407, 13408, 13409, 13410, + 13411, 13412, 13413, 13414, 13415, 13416, 13417, 13418, 13419, 13420, + 13421, 13422, 13423, 13424, 13425, 13426, 13427, 13428, 13429, 13430, + 13431, 13432, 13433, 13434, 13435, 13436, 13437, 13438, 13439, 13440, + 13441, 13442, 13443, 13444, 13445, 13446, 13447, 13448, 13449, 13450, + 13451, 13452, 13453, 13454, 13455, 13456, 13457, 13458, 13459, 13460, + 13461, 13462, 13463, 13464, 13465, 13466, 13467, 13468, 13469, 13470, + 13471, 13472, 13473, 13474, 13475, 13476, 13477, 13478, 13479, 13480, + 13481, 13482, 13483, 13484, 13485, 13486, 13487, 13488, 13489, 13490, + 13491, 13492, 13493, 13494, 13495, 13496, 13497, 13498, 13499, 13500, + 13501, 13502, 13503, 13504, 13505, 13506, 13507, 13508, 13509, 13510, + 13511, 13512, 13513, 13514, 13515, 13516, 13517, 13518, 13519, 13520, + 13521, 13522, 13523, 13524, 13525, 13526, 13527, 13528, 13529, 13530, + 13531, 13532, 13533, 13534, 13535, 13536, 13537, 13538, 13539, 13540, + 13541, 13542, 13543, 13544, 13545, 13546, 13547, 13548, 13549, 13550, + 13551, 13552, 13553, 13554, 13555, 13556, 13557, 13558, 13559, 13560, + 13561, 13562, 13563, 13564, 13565, 13566, 13567, 13568, 13569, 13570, + 13571, 13572, 13573, 13574, 13575, 13576, 13577, 13578, 13579, 13580, + 13581, 13582, 13583, 13584, 13585, 13586, 13587, 13588, 13589, 13590, + 13591, 13592, 13593, 13594, 13595, 13596, 13597, 13598, 13599, 13600, + 13601, 13602, 13603, 13604, 13605, 13606, 13607, 13608, 13609, 13610, + 13611, 13612, 13613, 13614, 13615, 13616, 13617, 13618, 13619, 13620, + 13621, 13622, 13623, 13624, 13625, 13626, 13627, 13628, 13629, 13630, + 13631, 13632, 13633, 13634, 13635, 13636, 13637, 13638, 13639, 13640, + 13641, 13642, 13643, 13644, 13645, 13646, 13647, 13648, 13649, 13650, + 13651, 13652, 13653, 13654, 13655, 13656, 13657, 13658, 13659, 13660, + 13661, 13662, 13663, 13664, 13665, 13666, 13667, 13668, 13669, 13670, + 13671, 13672, 13673, 13674, 13675, 13676, 13677, 13678, 13679, 13680, + 13681, 13682, 13683, 13684, 13685, 13686, 13687, 13688, 13689, 13690, + 13691, 13692, 13693, 13694, 13695, 13696, 13697, 13698, 13699, 13700, + 13701, 13702, 13703, 13704, 13705, 13706, 13707, 13708, 13709, 13710, + 13711, 13712, 13713, 13714, 13715, 13716, 13717, 13718, 13719, 13720, + 13721, 13722, 13723, 13724, 13725, 13726, 13727, 13728, 13729, 13730, + 13731, 13732, 13733, 13734, 13735, 13736, 13737, 13738, 13739, 13740, + 13741, 13742, 13743, 13744, 13745, 13746, 13747, 13748, 13749, 13750, + 13751, 13752, 13753, 13754, 13755, 13756, 13757, 13758, 13759, 13760, + 13761, 13762, 13763, 13764, 13765, 13766, 13767, 13768, 13769, 13770, + 13771, 13772, 13773, 13774, 13775, 13776, 13777, 13778, 13779, 13780, + 13781, 13782, 13783, 13784, 13785, 13786, 13787, 13788, 13789, 13790, + 13791, 13792, 13793, 13794, 13795, 13796, 13797, 13798, 13799, 13800, + 13801, 13802, 13803, 13804, 13805, 13806, 13807, 13808, 13809, 13810, + 13811, 13812, 13813, 13814, 13815, 13816, 13817, 13818, 13819, 13820, + 13821, 13822, 13823, 13824, 13825, 13826, 13827, 13828, 13829, 13830, + 13831, 13832, 13833, 13834, 13835, 13836, 13837, 13838, 13839, 13840, + 13841, 13842, 13843, 13844, 13845, 13846, 13847, 13848, 13849, 13850, + 13851, 13852, 13853, 13854, 13855, 13856, 13857, 13858, 13859, 13860, + 13861, 13862, 13863, 13864, 13865, 13866, 13867, 13868, 13869, 13870, + 13871, 13872, 13873, 13874, 13875, 13876, 13877, 13878, 13879, 13880, + 13881, 13882, 13883, 13884, 13885, 13886, 13887, 13888, 13889, 13890, + 13891, 13892, 13893, 13894, 13895, 13896, 13897, 13898, 13899, 13900, + 13901, 13902, 13903, 13904, 13905, 13906, 13907, 13908, 13909, 13910, + 13911, 13912, 13913, 13914, 13915, 13916, 13917, 13918, 13919, 13920, + 13921, 13922, 13923, 13924, 13925, 13926, 13927, 13928, 13929, 13930, + 13931, 13932, 13933, 13934, 13935, 13936, 13937, 13938, 13939, 13940, + 13941, 13942, 13943, 13944, 13945, 13946, 13947, 13948, 13949, 13950, + 13951, 13952, 13953, 13954, 13955, 13956, 13957, 13958, 13959, 13960, + 13961, 13962, 13963, 13964, 13965, 13966, 13967, 13968, 13969, 13970, + 13971, 13972, 13973, 13974, 13975, 13976, 13977, 13978, 13979, 13980, + 13981, 13982, 13983, 13984, 13985, 13986, 13987, 13988, 13989, 13990, + 13991, 13992, 13993, 13994, 13995, 13996, 13997, 13998, 13999, 14000, + 14001, 14002, 14003, 14004, 14005, 14006, 14007, 14008, 14009, 14010, + 14011, 14012, 14013, 14014, 14015, 14016, 14017, 14018, 14019, 14020, + 14021, 14022, 14023, 14024, 14025, 14026, 14027, 14028, 14029, 14030, + 14031, 14032, 14033, 14034, 14035, 14036, 14037, 14038, 14039, 14040, + 14041, 14042, 14043, 14044, 14045, 14046, 14047, 14048, 14049, 14050, + 14051, 14052, 14053, 14054, 14055, 14056, 14057, 14058, 14059, 14060, + 14061, 14062, 14063, 14064, 14065, 14066, 14067, 14068, 14069, 14070, + 14071, 14072, 14073, 14074, 14075, 14076, 14077, 14078, 14079, 14080, + 14081, 14082, 14083, 14084, 14085, 14086, 14087, 14088, 14089, 14090, + 14091, 14092, 14093, 14094, 14095, 14096, 14097, 14098, 14099, 14100, + 14101, 14102, 14103, 14104, 14105, 14106, 14107, 14108, 14109, 14110, + 14111, 14112, 14113, 14114, 14115, 14116, 14117, 14118, 14119, 14120, + 14121, 14122, 14123, 14124, 14125, 14126, 14127, 14128, 14129, 14130, + 14131, 14132, 14133, 14134, 14135, 14136, 14137, 14138, 14139, 14140, + 14141, 14142, 14143, 14144, 14145, 14146, 14147, 14148, 14149, 14150, + 14151, 14152, 14153, 14154, 14155, 14156, 14157, 14158, 14159, 14160, + 14161, 14162, 14163, 14164, 14165, 14166, 14167, 14168, 14169, 14170, + 14171, 14172, 14173, 14174, 14175, 14176, 14177, 14178, 14179, 14180, + 14181, 14182, 14183, 14184, 14185, 14186, 14187, 14188, 14189, 14190, + 14191, 14192, 14193, 14194, 14195, 14196, 14197, 14198, 14199, 14200, + 14201, 14202, 14203, 14204, 14205, 14206, 14207, 14208, 14209, 14210, + 14211, 14212, 14213, 14214, 14215, 14216, 14217, 14218, 14219, 14220, + 14221, 14222, 14223, 14224, 14225, 14226, 14227, 14228, 14229, 14230, + 14231, 14232, 14233, 14234, 14235, 14236, 14237, 14238, 14239, 14240, + 14241, 14242, 14243, 14244, 14245, 14246, 14247, 14248, 14249, 14250, + 14251, 14252, 14253, 14254, 14255, 14256, 14257, 14258, 14259, 14260, + 14261, 14262, 14263, 14264, 14265, 14266, 14267, 14268, 14269, 14270, + 14271, 14272, 14273, 14274, 14275, 14276, 14277, 14278, 14279, 14280, + 14281, 14282, 14283, 14284, 14285, 14286, 14287, 14288, 14289, 14290, + 14291, 14292, 14293, 14294, 14295, 14296, 14297, 14298, 14299, 14300, + 14301, 14302, 14303, 14304, 14305, 14306, 14307, 14308, 14309, 14310, + 14311, 14312, 14313, 14314, 14315, 14316, 14317, 14318, 14319, 14320, + 14321, 14322, 14323, 14324, 14325, 14326, 14327, 14328, 14329, 14330, + 14331, 14332, 14333, 14334, 14335, 14336, 14337, 14338, 14339, 14340, + 14341, 14342, 14343, 14344, 14345, 14346, 14347, 14348, 14349, 14350, + 14351, 14352, 14353, 14354, 14355, 14356, 14357, 14358, 14359, 14360, + 14361, 14362, 14363, 14364, 14365, 14366, 14367, 14368, 14369, 14370, + 14371, 14372, 14373, 14374, 14375, 14376, 14377, 14378, 14379, 14380, + 14381, 14382, 14383, 14384, 14385, 14386, 14387, 14388, 14389, 14390, + 14391, 14392, 14393, 14394, 14395, 14396, 14397, 14398, 14399, 14400, + 14401, 14402, 14403, 14404, 14405, 14406, 14407, 14408, 14409, 14410, + 14411, 14412, 14413, 14414, 14415, 14416, 14417, 14418, 14419, 14420, + 14421, 14422, 14423, 14424, 14425, 14426, 14427, 14428, 14429, 14430, + 14431, 14432, 14433, 14434, 14435, 14436, 14437, 14438, 14439, 14440, + 14441, 14442, 14443, 14444, 14445, 14446, 14447, 14448, 14449, 14450, + 14451, 14452, 14453, 14454, 14455, 14456, 14457, 14458, 14459, 14460, + 14461, 14462, 14463, 14464, 14465, 14466, 14467, 14468, 14469, 14470, + 14471, 14472, 14473, 14474, 14475, 14476, 14477, 14478, 14479, 14480, + 14481, 14482, 14483, 14484, 14485, 14486, 14487, 14488, 14489, 14490, + 14491, 14492, 14493, 14494, 14495, 14496, 14497, 14498, 14499, 14500, + 14501, 14502, 14503, 14504, 14505, 14506, 14507, 14508, 14509, 14510, + 14511, 14512, 14513, 14514, 14515, 14516, 14517, 14518, 14519, 14520, + 14521, 14522, 14523, 14524, 14525, 14526, 14527, 14528, 14529, 14530, + 14531, 14532, 14533, 14534, 14535, 14536, 14537, 14538, 14539, 14540, + 14541, 14542, 14543, 14544, 14545, 14546, 14547, 14548, 14549, 14550, + 14551, 14552, 14553, 14554, 14555, 14556, 14557, 14558, 14559, 14560, + 14561, 14562, 14563, 14564, 14565, 14566, 14567, 14568, 14569, 14570, + 14571, 14572, 14573, 14574, 14575, 14576, 14577, 14578, 14579, 14580, + 14581, 14582, 14583, 14584, 14585, 14586, 14587, 14588, 14589, 14590, + 14591, 14592, 14593, 14594, 14595, 14596, 14597, 14598, 14599, 14600, + 14601, 14602, 14603, 14604, 14605, 14606, 14607, 14608, 14609, 14610, + 14611, 14612, 14613, 14614, 14615, 14616, 14617, 14618, 14619, 14620, + 14621, 14622, 14623, 14624, 14625, 14626, 14627, 14628, 14629, 14630, + 14631, 14632, 14633, 14634, 14635, 14636, 14637, 14638, 14639, 14640, + 14641, 14642, 14643, 14644, 14645, 14646, 14647, 14648, 14649, 14650, + 14651, 14652, 14653, 14654, 14655, 14656, 14657, 14658, 14659, 14660, + 14661, 14662, 14663, 14664, 14665, 14666, 14667, 14668, 14669, 14670, + 14671, 14672, 14673, 14674, 14675, 14676, 14677, 14678, 14679, 14680, + 14681, 14682, 14683, 14684, 14685, 14686, 14687, 14688, 14689, 14690, + 14691, 14692, 14693, 14694, 14695, 14696, 14697, 14698, 14699, 14700, + 14701, 14702, 14703, 14704, 14705, 14706, 14707, 14708, 14709, 14710, + 14711, 14712, 14713, 14714, 14715, 14716, 14717, 14718, 14719, 14720, + 14721, 14722, 14723, 14724, 14725, 14726, 14727, 14728, 14729, 14730, + 14731, 14732, 14733, 14734, 14735, 14736, 14737, 14738, 14739, 14740, + 14741, 14742, 14743, 14744, 14745, 14746, 14747, 14748, 14749, 14750, + 14751, 14752, 14753, 14754, 14755, 14756, 14757, 14758, 14759, 14760, + 14761, 14762, 14763, 14764, 14765, 14766, 14767, 14768, 14769, 14770, + 14771, 14772, 14773, 14774, 14775, 14776, 14777, 14778, 14779, 14780, + 14781, 14782, 14783, 14784, 14785, 14786, 14787, 14788, 14789, 14790, + 14791, 14792, 14793, 14794, 14795, 14796, 14797, 14798, 14799, 14800, + 14801, 14802, 14803, 14804, 14805, 14806, 14807, 14808, 14809, 14810, + 14811, 14812, 14813, 14814, 14815, 14816, 14817, 14818, 14819, 14820, + 14821, 14822, 14823, 14824, 14825, 14826, 14827, 14828, 14829, 14830, + 14831, 14832, 14833, 14834, 14835, 14836, 14837, 14838, 14839, 14840, + 14841, 14842, 14843, 14844, 14845, 14846, 14847, 14848, 14849, 14850, + 14851, 14852, 14853, 14854, 14855, 14856, 14857, 14858, 14859, 14860, + 14861, 14862, 14863, 14864, 14865, 14866, 14867, 14868, 14869, 14870, + 14871, 14872, 14873, 14874, 14875, 14876, 14877, 14878, 14879, 14880, + 14881, 14882, 14883, 14884, 14885, 14886, 14887, 14888, 14889, 14890, + 14891, 14892, 14893, 14894, 14895, 14896, 14897, 14898, 14899, 14900, + 14901, 14902, 14903, 14904, 14905, 14906, 14907, 14908, 14909, 14910, + 14911, 14912, 14913, 14914, 14915, 14916, 14917, 14918, 14919, 14920, + 14921, 14922, 14923, 14924, 14925, 14926, 14927, 14928, 14929, 14930, + 14931, 14932, 14933, 14934, 14935, 14936, 14937, 14938, 14939, 14940, + 14941, 14942, 14943, 14944, 14945, 14946, 14947, 14948, 14949, 14950, + 14951, 14952, 14953, 14954, 14955, 14956, 14957, 14958, 14959, 14960, + 14961, 14962, 14963, 14964, 14965, 14966, 14967, 14968, 14969, 14970, + 14971, 14972, 14973, 14974, 14975, 14976, 14977, 14978, 14979, 14980, + 14981, 14982, 14983, 14984, 14985, 14986, 14987, 14988, 14989, 14990, + 14991, 14992, 14993, 14994, 14995, 14996, 14997, 14998, 14999, 15000, + 15001, 15002, 15003, 15004, 15005, 15006, 15007, 15008, 15009, 15010, + 15011, 15012, 15013, 15014, 15015, 15016, 15017, 15018, 15019, 15020, + 15021, 15022, 15023, 15024, 15025, 15026, 15027, 15028, 15029, 15030, + 15031, 15032, 15033, 15034, 15035, 15036, 15037, 15038, 15039, 15040, + 15041, 15042, 15043, 15044, 15045, 15046, 15047, 15048, 15049, 15050, + 15051, 15052, 15053, 15054, 15055, 15056, 15057, 15058, 15059, 15060, + 15061, 15062, 15063, 15064, 15065, 15066, 15067, 15068, 15069, 15070, + 15071, 15072, 15073, 15074, 15075, 15076, 15077, 15078, 15079, 15080, + 15081, 15082, 15083, 15084, 15085, 15086, 15087, 15088, 15089, 15090, + 15091, 15092, 15093, 15094, 15095, 15096, 15097, 15098, 15099, 15100, + 15101, 15102, 15103, 15104, 15105, 15106, 15107, 15108, 15109, 15110, + 15111, 15112, 15113, 15114, 15115, 15116, 15117, 15118, 15119, 15120, + 15121, 15122, 15123, 15124, 15125, 15126, 15127, 15128, 15129, 15130, + 15131, 15132, 15133, 15134, 15135, 15136, 15137, 15138, 15139, 15140, + 15141, 15142, 15143, 15144, 15145, 15146, 15147, 15148, 15149, 15150, + 15151, 15152, 15153, 15154, 15155, 15156, 15157, 15158, 15159, 15160, + 15161, 15162, 15163, 15164, 15165, 15166, 15167, 15168, 15169, 15170, + 15171, 15172, 15173, 15174, 15175, 15176, 15177, 15178, 15179, 15180, + 15181, 15182, 15183, 15184, 15185, 15186, 15187, 15188, 15189, 15190, + 15191, 15192, 15193, 15194, 15195, 15196, 15197, 15198, 15199, 15200, + 15201, 15202, 15203, 15204, 15205, 15206, 15207, 15208, 15209, 15210, + 15211, 15212, 15213, 15214, 15215, 15216, 15217, 15218, 15219, 15220, + 15221, 15222, 15223, 15224, 15225, 15226, 15227, 15228, 15229, 15230, + 15231, 15232, 15233, 15234, 15235, 15236, 15237, 15238, 15239, 15240, + 15241, 15242, 15243, 15244, 15245, 15246, 15247, 15248, 15249, 15250, + 15251, 15252, 15253, 15254, 15255, 15256, 15257, 15258, 15259, 15260, + 15261, 15262, 15263, 15264, 15265, 15266, 15267, 15268, 15269, 15270, + 15271, 15272, 15273, 15274, 15275, 15276, 15277, 15278, 15279, 15280, + 15281, 15282, 15283, 15284, 15285, 15286, 15287, 15288, 15289, 15290, + 15291, 15292, 15293, 15294, 15295, 15296, 15297, 15298, 15299, 15300, + 15301, 15302, 15303, 15304, 15305, 15306, 15307, 15308, 15309, 15310, + 15311, 15312, 15313, 15314, 15315, 15316, 15317, 15318, 15319, 15320, + 15321, 15322, 15323, 15324, 15325, 15326, 15327, 15328, 15329, 15330, + 15331, 15332, 15333, 15334, 15335, 15336, 15337, 15338, 15339, 15340, + 15341, 15342, 15343, 15344, 15345, 15346, 15347, 15348, 15349, 15350, + 15351, 15352, 15353, 15354, 15355, 15356, 15357, 15358, 15359, 15360, + 15361, 15362, 15363, 15364, 15365, 15366, 15367, 15368, 15369, 15370, + 15371, 15372, 15373, 15374, 15375, 15376, 15377, 15378, 15379, 15380, + 15381, 15382, 15383, 15384, 15385, 15386, 15387, 15388, 15389, 15390, + 15391, 15392, 15393, 15394, 15395, 15396, 15397, 15398, 15399, 15400, + 15401, 15402, 15403, 15404, 15405, 15406, 15407, 15408, 15409, 15410, + 15411, 15412, 15413, 15414, 15415, 15416, 15417, 15418, 15419, 15420, + 15421, 15422, 15423, 15424, 15425, 15426, 15427, 15428, 15429, 15430, + 15431, 15432, 15433, 15434, 15435, 15436, 15437, 15438, 15439, 15440, + 15441, 15442, 15443, 15444, 15445, 15446, 15447, 15448, 15449, 15450, + 15451, 15452, 15453, 15454, 15455, 15456, 15457, 15458, 15459, 15460, + 15461, 15462, 15463, 15464, 15465, 15466, 15467, 15468, 15469, 15470, + 15471, 15472, 15473, 15474, 15475, 15476, 15477, 15478, 15479, 15480, + 15481, 15482, 15483, 15484, 15485, 15486, 15487, 15488, 15489, 15490, + 15491, 15492, 15493, 15494, 15495, 15496, 15497, 15498, 15499, 15500, + 15501, 15502, 15503, 15504, 15505, 15506, 15507, 15508, 15509, 15510, + 15511, 15512, 15513, 15514, 15515, 15516, 15517, 15518, 15519, 15520, + 15521, 15522, 15523, 15524, 15525, 15526, 15527, 15528, 15529, 15530, + 15531, 15532, 15533, 15534, 15535, 15536, 15537, 15538, 15539, 15540, + 15541, 15542, 15543, 15544, 15545, 15546, 15547, 15548, 15549, 15550, + 15551, 15552, 15553, 15554, 15555, 15556, 15557, 15558, 15559, 15560, 15561, 15562, 15563, 15564, 15565, 15566, 15567, 15568, 15569, 15570, - 15571, 15572, 15573, 15574, 15575, 15576, 15577, 15578, 15553, 15554, - 15552, 15555, 15556, 15557, 15558, 15559, 15560, 15728, 15729, 15730, - 15731, 15732, 15727, 15739, 15740, 15741, 15742, 15733, 15734, 15735, - 15736, 15737, 15738, 15632, 15633, 15634, 15635, 15625, 15626, 15627, - 15628, 15629, 15630, 15631, 15619, 15620, 15621, 15622, 15623, 15624, - 15636, 15637, 15638, 15639, 15613, 15614, 15615, 15616, 15617, 15618, - 15706, 15707, 15708, 15709, 15710, 15711, 15712, 15713, 15714, 15715, - 15723, 15724, 15725, 15726, 15716, 15717, 15718, 15719, 15720, 15721, - 15722, 15611, 15612, 15836, 16849, 16848, 16850, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 15847, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 15840, 15839, 15841, 35762, 35762, 16898, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 16904, 16903, 16905, 16909, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 24744, 24745, 24746, 24747, 24748, 24749, - 24750, 24751, 24752, 24753, 24754, 24755, 24756, 24757, 24758, 24759, - 24760, 24761, 24762, 24763, 24764, 24765, 24766, 24767, 24768, 24769, - 24770, 24771, 24772, 24773, 24774, 24775, 24776, 24777, 24778, 24779, - 24780, 24781, 24782, 24783, 24784, 24785, 24786, 24787, 24788, 24789, - 24790, 24791, 24792, 24793, 24794, 24795, 24796, 24797, 24798, 24799, - 24800, 24801, 24802, 24803, 24804, 24805, 24806, 24807, 24808, 24809, - 24810, 24811, 24812, 24813, 24814, 24815, 24816, 24817, 24818, 24819, - 24820, 24821, 24822, 24823, 24824, 24825, 24826, 24827, 24828, 24829, - 24830, 24831, 24832, 24833, 24834, 24835, 24836, 24837, 24838, 24839, - 24840, 24841, 24842, 24843, 24844, 24845, 24846, 24847, 24848, 24849, - 24850, 24851, 24852, 24853, 24854, 24855, 24856, 24857, 24858, 24859, - 24860, 24861, 24862, 24863, 24864, 24865, 24866, 24867, 24868, 24869, - 24870, 24871, 24872, 24873, 24874, 24875, 24876, 24877, 24878, 24879, - 24880, 24881, 24882, 24883, 24884, 24885, 24886, 24887, 24888, 24889, - 24890, 24891, 24892, 24893, 24894, 24895, 24896, 24897, 24898, 24899, - 24900, 24901, 24902, 24903, 24904, 24905, 24906, 24907, 24908, 24909, - 24910, 24911, 24912, 24913, 24914, 24915, 24916, 24917, 24918, 24919, - 24920, 24921, 24922, 24923, 24924, 24925, 24926, 24927, 24928, 24929, - 24930, 24931, 24932, 24933, 24934, 24935, 24936, 24937, 24938, 24939, - 24940, 24941, 24942, 24943, 24944, 24945, 24946, 24947, 24948, 24949, - 24950, 24951, 24952, 24953, 24954, 24955, 24956, 24957, 24958, 24959, - 24960, 24961, 24962, 24963, 24964, 24965, 24966, 24967, 24968, 24969, - 24970, 24971, 24972, 24973, 24974, 24975, 24976, 24977, 24978, 24979, - 24980, 24981, 24982, 24983, 24984, 24985, 24986, 24987, 24988, 24989, - 24990, 24991, 24992, 24993, 24994, 24995, 24996, 24997, 24998, 24999, - 25000, 25001, 25002, 25003, 25004, 25005, 25006, 25007, 25008, 25009, - 25010, 25011, 25012, 25013, 25014, 25015, 25016, 25017, 25018, 25019, - 25020, 25021, 25022, 25023, 25024, 25025, 25026, 25027, 25028, 25029, - 25030, 25031, 25032, 25033, 25034, 25035, 25036, 25037, 25038, 25039, - 25040, 25041, 25042, 25043, 25044, 25045, 25046, 25047, 25048, 25049, - 25050, 25051, 25052, 25053, 25054, 25055, 25056, 25057, 25058, 25059, - 25060, 25061, 25062, 25063, 25064, 25065, 25066, 25067, 25068, 25069, - 25070, 25071, 25072, 25073, 25074, 25075, 25076, 25077, 25078, 25079, - 25080, 25081, 25082, 25083, 25084, 25085, 25086, 25087, 25088, 25089, - 25090, 25091, 25092, 25093, 25094, 25095, 25096, 25097, 25098, 25099, - 25100, 25101, 25102, 25103, 25104, 25105, 25106, 25107, 25108, 25109, - 25110, 25111, 25112, 25113, 25114, 25115, 25116, 25117, 25118, 25119, - 25120, 25121, 25122, 25123, 25124, 25125, 25126, 25127, 25128, 25129, - 25130, 25131, 25132, 25133, 25134, 25135, 25136, 25137, 25138, 25139, - 35762, 35762, 35762, 35762, 11196, 11194, 11143, 11176, 11103, 11116, - 11120, 11201, 11097, 11184, 11105, 11147, 11146, 11098, 11104, 11118, - 11150, 11179, 11172, 11099, 11119, 11173, 11197, 11123, 11151, 11124, - 11129, 11107, 11152, 11125, 11130, 11110, 11153, 11127, 11132, 11108, - 11109, 11161, 11162, 11128, 11133, 11114, 11165, 11126, 11131, 11111, - 11154, 11115, 11112, 11113, 11159, 11160, 11157, 11158, 11178, 11177, - 11186, 11192, 11189, 11164, 11163, 11117, 11106, 11155, 11156, 11095, - 11170, 11140, 11138, 11096, 11198, 11100, 11199, 11175, 11183, 11101, - 11167, 11148, 11166, 11121, 11200, 11180, 11102, 11195, 11181, 11122, - 11149, 11182, 11174, 11139, 11142, 11141, 11191, 11187, 11193, 11190, - 11188, 11137, 11136, 11135, 11134, 11145, 11144, 11168, 11171, 11169, - 11185, 35762, 35762, 35762, 35762, 35762, 11080, 11093, 11092, 11087, - 11094, 11074, 11067, 11066, 11063, 11065, 11068, 11069, 11064, 35762, - 35762, 35762, 11076, 11072, 11075, 11070, 11079, 11078, 11071, 11077, - 11073, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 11081, 11085, - 11088, 11083, 11091, 11090, 11084, 11089, 11086, 11082, 35762, 35762, - 11205, 11202, 11203, 11204, 27932, 27931, 27933, 27934, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35578, 35583, 35617, 35580, 35585, 35613, 35607, 35601, 35624, 35603, - 35597, 35621, 35579, 35584, 35618, 35581, 35586, 35614, 35608, 35602, - 35625, 35604, 35598, 35622, 35619, 35605, 35612, 35599, 35600, 35623, - 35606, 35582, 35628, 35593, 35610, 35620, 35631, 35630, 35596, 35629, - 35590, 35589, 35627, 35611, 35609, 35588, 35762, 35762, 35633, 35634, - 35635, 35626, 35576, 35591, 35594, 35595, 35573, 35574, 35592, 35615, - 35616, 35577, 35632, 35575, 35587, 35572, 35754, 35755, 35752, 35753, - 35756, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35652, 35653, 35669, 35641, 35650, 35749, 35704, 35705, 35672, 35673, - 35706, 35636, 35670, 35750, 35648, 35645, 35647, 35646, 35644, 35746, - 35745, 35748, 35747, 35742, 35741, 35744, 35743, 35642, 35676, 35638, - 35649, 35637, 35674, 35687, 35688, 35686, 35685, 35639, 35683, 35684, - 35682, 35680, 35681, 35679, 35678, 35689, 35691, 35692, 35690, 35654, - 35677, 35643, 35651, 35751, 35693, 35698, 35695, 35699, 35696, 35701, - 35702, 35697, 35694, 35700, 35675, 35703, 35736, 35733, 35724, 35734, - 35735, 35725, 35713, 35712, 35740, 35710, 35709, 35707, 35711, 35708, - 35727, 35732, 35726, 35730, 35731, 35728, 35729, 35656, 35660, 35659, - 35658, 35657, 35739, 35737, 35738, 35663, 35668, 35667, 35666, 35665, - 35664, 35714, 35723, 35716, 35721, 35715, 35719, 35720, 35717, 35718, - 35722, 35655, 35662, 35640, 35661, 35671, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 4893, 4765, 4886, 4868, 4869, - 4937, 4938, 4818, 4915, 4875, 4949, 4950, 4841, 4720, 4771, 4918, 4819, - 4723, 4724, 4913, 4925, 4864, 4801, 4892, 4744, 4941, 4813, 4827, 4823, - 4916, 4878, 4907, 4871, 4940, 4726, 4728, 4828, 4894, 4879, 4936, 4718, - 4896, 4910, 4912, 4866, 4920, 4848, 4766, 4928, 4919, 4838, 4719, 4778, - 4809, 4930, 4816, 4884, 4888, 4831, 4742, 4895, 4873, 4876, 4815, 4953, - 4883, 4832, 4931, 4908, 4807, 4814, 4865, 4870, 4882, 4834, 4881, 4839, - 4885, 4822, 4826, 4952, 4725, 4721, 4951, 4840, 4774, 4745, 4863, 4939, - 4880, 4889, 4872, 4717, 4849, 4877, 4874, 4770, 4842, 4716, 4932, 4773, - 4911, 4914, 4743, 4772, 4897, 4954, 4934, 4890, 4929, 4933, 4887, 4935, - 4891, 4804, 4730, 4769, 4867, 4922, 4923, 4921, 4924, 4817, 4767, 4942, - 4943, 4906, 4830, 4763, 4835, 4836, 4837, 4727, 4729, 4762, 4927, 4917, - 4833, 4843, 4847, 4846, 4845, 4844, 4803, 4800, 4799, 4757, 4759, 4760, - 4758, 4926, 4731, 4811, 4750, 4714, 4708, 4709, 4712, 4713, 4711, 4710, - 4715, 4856, 4851, 4852, 4850, 4861, 4860, 4859, 4858, 4862, 4854, 4812, - 4722, 4777, 4776, 4775, 4857, 4855, 4853, 4805, 4806, 4768, 4810, 4808, - 4779, 4786, 4782, 4790, 4785, 4794, 4784, 4783, 4780, 4781, 4788, 4789, - 4796, 4793, 4791, 4740, 4741, 4739, 4787, 4795, 4948, 4753, 4751, 4754, - 4756, 4755, 4752, 4944, 4946, 4945, 4947, 4797, 4798, 4747, 4746, 4748, - 4749, 4902, 4905, 4903, 4904, 4898, 4901, 4899, 4900, 4761, 4764, 4909, - 4738, 4732, 4737, 4735, 4734, 4733, 4736, 4820, 4824, 4821, 4825, 4829, - 4802, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 23803, 23680, 23696, 23788, 23675, 23800, 23738, 23789, 23787, - 23676, 23672, 23799, 23666, 23784, 23785, 23786, 23691, 23692, 23629, - 23630, 23625, 23624, 23755, 23826, 23823, 23698, 23697, 23804, 23805, - 23699, 23708, 23709, 23710, 23669, 23701, 23702, 23703, 23682, 23683, - 35762, 35762, 23746, 23679, 23681, 23707, 23706, 23751, 23750, 23802, - 23801, 23778, 23779, 23665, 23671, 23767, 23768, 23782, 23783, 23747, - 23849, 23721, 23781, 23690, 23807, 23825, 23809, 23754, 23847, 23770, - 23670, 23810, 23811, 23834, 23835, 23838, 23839, 23836, 23837, 23830, - 23831, 23832, 23833, 23744, 23745, 23840, 23841, 23769, 23845, 23752, - 23749, 23633, 23634, 23628, 23848, 23720, 23780, 23689, 23806, 23824, - 23808, 23753, 23654, 23655, 23658, 23659, 23660, 23693, 23694, 23695, - 23637, 23644, 23645, 23646, 23647, 23648, 23622, 23687, 23623, 23688, - 23621, 23685, 23620, 23684, 23635, 23653, 23661, 23652, 23638, 23639, - 23636, 23663, 23718, 23717, 23642, 23664, 23649, 23656, 23662, 23641, - 23657, 23790, 23813, 23852, 23777, 23740, 23700, 23668, 23677, 23714, - 23713, 23829, 23842, 23851, 23843, 23844, 23756, 23759, 23760, 23761, - 23762, 23763, 23764, 23765, 23766, 23757, 23758, 23722, 23748, 23686, - 23678, 23640, 23643, 23650, 23651, 23772, 23771, 23719, 23712, 23711, - 23850, 23673, 23674, 23739, 23735, 23626, 23793, 23795, 23741, 23743, - 23796, 23798, 23704, 23705, 23737, 23736, 23627, 23794, 23742, 23797, - 23821, 23820, 23822, 23819, 23815, 23816, 23817, 23818, 23667, 23715, - 23716, 23812, 23846, 23774, 23632, 23791, 23631, 23827, 23775, 23776, - 23792, 23828, 23773, 23723, 23726, 23730, 23733, 23732, 23731, 23727, - 23728, 23724, 23725, 23729, 23814, 23734, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 14240, 14228, 14251, - 14252, 14234, 14253, 14254, 14255, 14256, 14241, 14242, 14243, 14244, - 14245, 14246, 14247, 14248, 14249, 14250, 14229, 14230, 14231, 14232, - 14233, 14235, 14236, 14237, 14238, 14239, 13960, 13968, 13981, 13989, - 13995, 13996, 13961, 13962, 13963, 13964, 13965, 13966, 13967, 13969, - 13970, 13971, 13972, 13973, 13974, 13975, 13976, 13977, 13978, 13979, - 13980, 13982, 13983, 13984, 13985, 13986, 13987, 13988, 13990, 13991, - 13992, 13993, 13994, 7968, 7967, 7969, 14020, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 17140, 17141, - 17138, 17136, 17127, 17126, 17133, 17131, 17122, 17129, 17139, 17124, - 17137, 17135, 17128, 17125, 17134, 17132, 17123, 17130, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 22166, 22167, 22164, 22162, 22153, 22152, 22159, 22157, 22148, 22155, - 22165, 22150, 22163, 22161, 22154, 22151, 22160, 22158, 22149, 22156, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 23494, 10576, 10577, 10571, 10570, 10569, 31810, 31830, - 31851, 31799, 31844, 31808, 31794, 31853, 31798, 31812, 31818, 31841, - 31842, 31856, 31862, 31809, 31840, 31870, 31828, 31795, 31858, 31860, - 31827, 31873, 31807, 31822, 31819, 31800, 31814, 31797, 31855, 31848, - 31802, 31845, 31834, 31868, 31857, 31831, 31859, 31847, 31861, 31836, - 31824, 31867, 31837, 31823, 31854, 31863, 31833, 31869, 31806, 31850, - 31826, 31872, 31816, 31801, 31838, 31835, 31849, 31793, 31821, 31820, - 31871, 31865, 31843, 31813, 31811, 31817, 31825, 31864, 31866, 31839, - 31805, 31803, 31832, 31796, 31804, 31852, 31815, 31846, 31829, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 8366, 8364, 8363, - 8360, 8359, 8362, 8361, 8367, 8365, 8357, 8355, 8354, 8351, 8350, 8353, - 8352, 8358, 8356, 16051, 16050, 16049, 16048, 16047, 30395, 30394, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 21146, 21148, 21176, 21139, 21152, 21184, - 21155, 21185, 21157, 21186, 21159, 21161, 21178, 21180, 21163, 21166, - 21187, 21170, 21172, 21142, 21174, 21188, 21189, 21182, 21190, 21150, - 21194, 21196, 21229, 21191, 21200, 21203, 21205, 21237, 21207, 21238, - 21209, 21211, 21231, 21233, 21213, 21216, 21239, 21220, 21222, 21224, - 21227, 21240, 21241, 21235, 21242, 21198, 21634, 21636, 21666, 21640, - 21642, 21674, 21645, 21675, 21647, 21676, 21649, 21651, 21668, 21670, - 21653, 21656, 21677, 21660, 21662, 21630, 21664, 21678, 21679, 21672, - 21680, 21638, 21582, 21584, 21617, 21578, 21588, 21591, 21593, 35762, - 21595, 21625, 21597, 21599, 21619, 21621, 21601, 21604, 21626, 21608, - 21610, 21612, 21615, 21627, 21628, 21623, 21629, 21586, 21299, 21301, - 21331, 21305, 21307, 21339, 21310, 21340, 21312, 21341, 21314, 21316, - 21333, 21335, 21318, 21321, 21342, 21325, 21327, 21295, 21329, 21343, - 21344, 21337, 21345, 21303, 21353, 21355, 21390, 21359, 21361, 21364, - 21366, 21398, 21368, 21399, 21370, 21372, 21392, 21394, 21374, 21377, - 21400, 21381, 21383, 21385, 21388, 21401, 21402, 21396, 21403, 21357, - 22106, 35762, 22107, 22108, 35762, 35762, 22109, 35762, 35762, 22110, - 22111, 35762, 35762, 22112, 22113, 22114, 22115, 35762, 22116, 22117, - 22118, 22119, 22120, 22121, 22122, 22123, 22124, 22125, 22126, 22127, - 35762, 22128, 35762, 22129, 22130, 22131, 22132, 22133, 22134, 22135, - 35762, 22136, 22137, 22138, 22139, 22140, 22141, 22142, 22143, 22144, - 22145, 22146, 21243, 21244, 21245, 21246, 21247, 21248, 21249, 21250, - 21251, 21252, 21253, 21254, 21255, 21256, 21257, 21258, 21259, 21260, - 21261, 21262, 21263, 21264, 21265, 21266, 21267, 21268, 21269, 21270, - 21271, 21272, 21273, 21274, 21275, 21276, 21277, 21278, 21279, 21280, - 21281, 21282, 21283, 21284, 21285, 21286, 21287, 21288, 21289, 21290, - 21291, 21292, 21293, 21294, 21530, 21531, 35762, 21532, 21533, 21534, - 21535, 35762, 35762, 21536, 21537, 21538, 21539, 21540, 21541, 21542, - 21543, 35762, 21544, 21545, 21546, 21547, 21548, 21549, 21550, 35762, - 21551, 21552, 21553, 21554, 21555, 21556, 21557, 21558, 21559, 21560, - 21561, 21562, 21563, 21564, 21565, 21566, 21567, 21568, 21569, 21570, - 21571, 21572, 21573, 21574, 21575, 21576, 21475, 21476, 35762, 21477, - 21478, 21479, 21480, 35762, 21481, 21482, 21483, 21484, 21485, 35762, - 21486, 35762, 35762, 35762, 21487, 21488, 21489, 21490, 21491, 21492, - 21493, 35762, 21494, 21495, 21496, 21497, 21498, 21499, 21500, 21501, - 21502, 21503, 21504, 21505, 21506, 21507, 21508, 21509, 21510, 21511, - 21512, 21513, 21514, 21515, 21516, 21517, 21518, 21519, 21413, 21414, - 21415, 21416, 21417, 21418, 21419, 21420, 21421, 21422, 21423, 21424, - 21425, 21426, 21427, 21428, 21429, 21430, 21431, 21432, 21433, 21434, - 21435, 21436, 21437, 21438, 21439, 21440, 21441, 21442, 21443, 21444, - 21445, 21446, 21447, 21448, 21449, 21450, 21451, 21452, 21453, 21454, - 21455, 21456, 21457, 21458, 21459, 21460, 21461, 21462, 21463, 21464, - 22044, 22045, 22046, 22047, 22048, 22049, 22050, 22051, 22052, 22053, - 22054, 22055, 22056, 22057, 22058, 22059, 22060, 22061, 22062, 22063, - 22064, 22065, 22066, 22067, 22068, 22069, 22070, 22071, 22072, 22073, - 22074, 22075, 22076, 22077, 22078, 22079, 22080, 22081, 22082, 22083, - 22084, 22085, 22086, 22087, 22088, 22089, 22090, 22091, 22092, 22093, - 22094, 22095, 21876, 21878, 21908, 21882, 21884, 21916, 21887, 21917, - 21889, 21918, 21891, 21893, 21910, 21912, 21895, 21898, 21919, 21902, - 21904, 21872, 21906, 21920, 21921, 21914, 21922, 21880, 21930, 21932, - 21967, 21936, 21938, 21941, 21943, 21975, 21945, 21976, 21947, 21949, - 21969, 21971, 21951, 21954, 21977, 21958, 21960, 21962, 21965, 21978, - 21979, 21973, 21980, 21934, 21992, 21993, 21994, 21995, 21996, 21997, - 21998, 21999, 22000, 22001, 22002, 22003, 22004, 22005, 22006, 22007, - 22008, 22009, 22010, 22011, 22012, 22013, 22014, 22015, 22016, 22017, - 22018, 22019, 22020, 22021, 22022, 22023, 22024, 22025, 22026, 22027, - 22028, 22029, 22030, 22031, 22032, 22033, 22034, 22035, 22036, 22037, - 22038, 22039, 22040, 22041, 22042, 22043, 21766, 21768, 21798, 21772, - 21774, 21806, 21777, 21807, 21779, 21808, 21781, 21783, 21800, 21802, - 21785, 21788, 21809, 21792, 21794, 21762, 21796, 21810, 21811, 21804, - 21812, 21770, 21820, 21822, 21857, 21826, 21828, 21831, 21833, 21865, - 21835, 21866, 21837, 21839, 21859, 21861, 21841, 21844, 21867, 21848, - 21850, 21852, 21855, 21868, 21869, 21863, 21870, 21824, 21689, 21690, - 21691, 21692, 21693, 21694, 21695, 21696, 21697, 21698, 21699, 21700, - 21701, 21702, 21703, 21704, 21705, 21706, 21707, 21708, 21709, 21710, - 21711, 21712, 21713, 21714, 21715, 21716, 21717, 21718, 21719, 21720, - 21721, 21722, 21723, 21724, 21725, 21726, 21727, 21728, 21729, 21730, - 21731, 21732, 21733, 21734, 21735, 21736, 21737, 21738, 21739, 21740, - 21579, 21580, 35762, 35762, 21147, 21149, 21156, 21141, 21153, 21151, - 21154, 21143, 21158, 21160, 21162, 21179, 21181, 21183, 21164, 21169, - 21171, 21144, 21173, 21145, 21175, 21167, 21177, 21168, 21165, 21407, - 21195, 21197, 21206, 21193, 21201, 21199, 21202, 21225, 21208, 21210, - 21212, 21232, 21234, 21236, 21214, 21219, 21221, 21204, 21223, 21226, - 21228, 21217, 21230, 21218, 21215, 21409, 21405, 21412, 21406, 21408, - 21411, 21410, 21635, 21637, 21646, 21641, 21643, 21639, 21644, 21631, - 21648, 21650, 21652, 21669, 21671, 21673, 21654, 21659, 21661, 21632, - 21663, 21633, 21665, 21657, 21667, 21658, 21655, 21683, 21583, 21585, - 21594, 21581, 21589, 21587, 21590, 21613, 21596, 21598, 21600, 21620, - 21622, 21624, 21602, 21607, 21609, 21592, 21611, 21614, 21616, 21605, - 21618, 21606, 21603, 21685, 21681, 21688, 21682, 21684, 21687, 21686, - 21300, 21302, 21311, 21306, 21308, 21304, 21309, 21296, 21313, 21315, - 21317, 21334, 21336, 21338, 21319, 21324, 21326, 21297, 21328, 21298, - 21330, 21322, 21332, 21323, 21320, 21348, 21354, 21356, 21367, 21360, - 21362, 21358, 21363, 21386, 21369, 21371, 21373, 21393, 21395, 21397, - 21375, 21380, 21382, 21365, 21384, 21387, 21389, 21378, 21391, 21379, - 21376, 21350, 21346, 21404, 21347, 21349, 21352, 21351, 21877, 21879, - 21888, 21883, 21885, 21881, 21886, 21873, 21890, 21892, 21894, 21911, - 21913, 21915, 21896, 21901, 21903, 21874, 21905, 21875, 21907, 21899, - 21909, 21900, 21897, 21925, 21931, 21933, 21944, 21937, 21939, 21935, - 21940, 21963, 21946, 21948, 21950, 21970, 21972, 21974, 21952, 21957, - 21959, 21942, 21961, 21964, 21966, 21955, 21968, 21956, 21953, 21927, - 21923, 21981, 21924, 21926, 21929, 21928, 21767, 21769, 21778, 21773, - 21775, 21771, 21776, 21763, 21780, 21782, 21784, 21801, 21803, 21805, - 21786, 21791, 21793, 21764, 21795, 21765, 21797, 21789, 21799, 21790, - 21787, 21815, 21821, 21823, 21834, 21827, 21829, 21825, 21830, 21853, - 21836, 21838, 21840, 21860, 21862, 21864, 21842, 21847, 21849, 21832, - 21851, 21854, 21856, 21845, 21858, 21846, 21843, 21817, 21813, 21871, - 21814, 21816, 21819, 21818, 21140, 21192, 35762, 35762, 21471, 21473, - 21470, 21469, 21466, 21465, 21468, 21467, 21474, 21472, 21526, 21528, - 21525, 21524, 21521, 21520, 21523, 21522, 21529, 21527, 22102, 22104, - 22101, 22100, 22097, 22096, 22099, 22098, 22105, 22103, 21988, 21990, - 21987, 21986, 21983, 21982, 21985, 21984, 21991, 21989, 21747, 21749, - 21746, 21745, 21742, 21741, 21744, 21743, 21750, 21748, 28184, 28140, - 28165, 28381, 28336, 28122, 28185, 28149, 28298, 28250, 28226, 28187, - 28190, 28147, 28191, 28141, 28192, 28214, 28209, 28247, 28188, 28194, - 28203, 28204, 28195, 28201, 28205, 28143, 28277, 28186, 28215, 28144, - 28224, 28193, 28223, 28210, 28249, 28248, 28189, 28208, 28218, 28219, - 28222, 28221, 28289, 28197, 28198, 28199, 28271, 28239, 28202, 28206, - 28200, 28196, 28270, 28231, 28269, 28268, 28228, 28227, 28230, 28220, - 28217, 28216, 28266, 28265, 28267, 28238, 28314, 28318, 28317, 28315, - 28316, 28159, 28305, 28335, 28307, 28320, 28312, 28321, 28313, 28322, - 28311, 28169, 28170, 28334, 28380, 28308, 28309, 28310, 28306, 28332, - 28319, 28330, 28323, 28331, 28329, 28327, 28324, 28325, 28326, 28328, - 28156, 28162, 28160, 28161, 28365, 28364, 28172, 28164, 28175, 28178, - 28173, 28176, 28174, 28177, 28182, 28179, 28139, 28374, 28379, 28376, - 28378, 28353, 28355, 28333, 28363, 28356, 28360, 28354, 28362, 28361, - 28359, 28121, 28211, 28146, 28338, 28124, 28347, 28213, 28212, 28339, - 28252, 28255, 28254, 28253, 28262, 28302, 28151, 28375, 28133, 28258, - 28261, 28256, 28257, 28350, 28260, 28349, 28132, 28131, 28259, 28150, - 28348, 28130, 28225, 28145, 28340, 28357, 28123, 28207, 28142, 28278, - 28358, 28134, 28286, 28282, 28284, 28155, 28377, 28135, 28279, 28281, - 28280, 28285, 28283, 28373, 28251, 28148, 28180, 28367, 28368, 28366, - 28168, 28346, 28126, 28125, 28275, 28351, 28273, 28154, 28263, 28274, - 28372, 28272, 28276, 28264, 28152, 28181, 28171, 28352, 28137, 28138, - 28136, 28153, 28157, 28158, 28370, 28371, 28369, 28337, 28240, 28342, - 28243, 28244, 28245, 28242, 28246, 28241, 28235, 28236, 28237, 28234, - 28232, 28163, 28233, 28229, 28166, 28167, 28344, 28345, 28341, 28343, - 28128, 28129, 28127, 28287, 28292, 28295, 28296, 28297, 28303, 28288, - 28290, 28291, 28299, 28294, 28300, 28301, 28293, 28183, 28304, 28695, - 28694, 28693, 28715, 28714, 28713, 28670, 28669, 28668, 28054, 28053, - 28052, 28656, 28655, 28654, 28666, 28667, 28662, 28665, 28661, 28664, - 28663, 28111, 28114, 28110, 28113, 28112, 28660, 28530, 28531, 28532, - 28533, 28528, 28529, 28534, 28574, 28479, 28592, 28591, 28589, 28590, - 28593, 28568, 28571, 28569, 28570, 28567, 28598, 28597, 28595, 28596, - 28544, 28543, 28542, 28548, 28547, 28546, 28545, 28566, 28565, 28564, - 28541, 28540, 28539, 28614, 28613, 28612, 28588, 28587, 28586, 28711, - 28710, 28709, 28708, 28707, 28706, 28712, 28705, 28704, 28703, 28448, - 28447, 28445, 28446, 28452, 28451, 28449, 28450, 28440, 28439, 28437, - 28438, 28444, 28443, 28441, 28442, 28502, 28501, 28499, 28500, 28503, - 28517, 28520, 28518, 28519, 28476, 28507, 28506, 28505, 28504, 28462, - 28475, 28474, 28473, 28463, 28461, 28460, 28459, 28526, 28525, 28524, - 28523, 28522, 28521, 28698, 28697, 28696, 28701, 28700, 28699, 28702, - 28561, 28560, 28558, 28559, 28552, 28551, 28549, 28550, 28557, 28556, - 28579, 28578, 28575, 28580, 28585, 28582, 28581, 28601, 28600, 28599, - 28604, 28603, 28602, 28555, 28563, 28562, 28651, 28648, 28647, 28594, - 28554, 28577, 28584, 28609, 28653, 28650, 28646, 28553, 28576, 28583, - 28608, 28652, 28649, 28645, 28607, 28606, 28605, 28466, 28465, 28483, - 28481, 28482, 28480, 28492, 28490, 28491, 28489, 28509, 28508, 28640, - 28637, 28643, 28468, 28467, 28484, 28485, 28487, 28486, 28496, 28494, - 28495, 28493, 28511, 28510, 28641, 28638, 28644, 28472, 28471, 28469, - 28470, 28464, 28488, 28497, 28512, 28513, 28514, 28639, 28636, 28642, - 28498, 28538, 28536, 28537, 28535, 28458, 28456, 28454, 28457, 28455, - 28453, 28611, 28610, 28516, 28515, 28573, 28572, 28478, 28477, 28070, - 28069, 28065, 28068, 28072, 28071, 28066, 28067, 28064, 28073, 28383, - 28390, 28388, 28387, 28385, 28386, 28384, 28389, 28103, 28101, 28102, - 28079, 28078, 28077, 28062, 28060, 28061, 28063, 28118, 28117, 28116, - 28097, 28098, 28094, 28075, 28074, 28093, 28095, 28092, 28096, 28076, - 28091, 28089, 28090, 28086, 28088, 28087, 28080, 28082, 28081, 28084, - 28083, 28085, 28057, 28056, 28055, 28680, 28679, 28681, 28100, 28617, - 28616, 28619, 28618, 28047, 28049, 28046, 28048, 28051, 28050, 28412, - 28410, 28411, 28417, 28419, 28418, 28414, 28416, 28415, 28431, 28430, - 28429, 28423, 28424, 28425, 28426, 28427, 28428, 28420, 28422, 28421, - 28432, 28433, 28434, 28401, 28399, 28400, 28413, 28436, 28435, 28688, - 28687, 28685, 28686, 28684, 28689, 28683, 28682, 28672, 28677, 28675, - 28676, 28673, 28674, 28678, 28615, 28527, 28620, 28382, 28099, 28658, - 28657, 28120, 28115, 28659, 28691, 28690, 28692, 28716, 28391, 28392, - 28393, 28394, 28395, 28396, 28397, 28398, 28109, 28409, 28408, 28403, - 28406, 28405, 28402, 28404, 28407, 28059, 28119, 28671, 28058, 28717, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 28104, 28105, 28106, 28107, 28108, - 35762, 28628, 28629, 28630, 28631, 28632, 28633, 28634, 28635, 28621, - 28622, 28623, 28624, 28625, 28626, 28627, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 18911, 19184, 18676, 19189, - 18663, 19034, 19294, 19186, 19280, 19249, 18656, 18890, 18891, 19284, - 18651, 18703, 18677, 19028, 18827, 19008, 18888, 19278, 19165, 19253, - 18899, 18828, 18972, 19124, 19254, 18794, 19201, 35762, 35762, 35762, - 35762, 35762, 35762, 18818, 19021, 19066, 19170, 19208, 19244, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 7924, 7926, 7936, 7930, 7912, 7937, 7944, 35762, 7943, - 7917, 7914, 7913, 7911, 7948, 7932, 7931, 7933, 7947, 7934, 7935, 7919, - 7922, 7946, 7928, 7945, 35762, 35762, 7920, 7923, 7927, 7921, 7939, 7938, - 7940, 35762, 7942, 7918, 35762, 7941, 7916, 7925, 7915, 7929, 35762, - 35762, 35762, 35762, 35762, 23081, 23050, 23078, 23076, 23052, 23074, - 23071, 23072, 23073, 23080, 23057, 23058, 23082, 23061, 23059, 23054, - 23067, 23083, 23056, 23079, 23066, 23075, 23065, 23068, 23053, 23070, - 23051, 23064, 23048, 23077, 23049, 23062, 23060, 10214, 10192, 10212, - 10199, 10195, 10209, 10206, 10207, 10208, 10213, 10197, 10215, 10211, - 10198, 10216, 10196, 10201, 10205, 10210, 10204, 10202, 10203, 10200, - 10191, 10194, 10193, 23055, 23069, 23063, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 7841, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 25172, 25156, 25147, 25158, 25165, 25155, 25160, 25151, - 25177, 25181, 25183, 25186, 25150, 25145, 25180, 25170, 25154, 25153, - 25184, 25146, 25157, 25178, 25164, 25182, 25185, 25152, 25174, 25159, - 25149, 25169, 25148, 25166, 25171, 25173, 25179, 25163, 25161, 25162, - 25187, 25188, 25167, 25168, 25175, 25176, 25189, 35762, 35762, 35762, - 25198, 25202, 25201, 25204, 25203, 25200, 25199, 25194, 25192, 25191, - 25195, 25193, 25196, 25197, 35762, 35762, 25211, 25213, 25210, 25209, - 25206, 25205, 25208, 25207, 25214, 25212, 35762, 35762, 35762, 35762, - 25190, 25144, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 32479, 32462, 32482, 32472, 32476, 32473, 32478, 32466, - 32465, 32481, 32469, 32484, 32483, 32475, 32474, 32480, 32477, 32463, - 32457, 32464, 32458, 32486, 32467, 32459, 32468, 32460, 32485, 32471, - 32461, 32470, 32487, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 33793, 33792, 33824, 33825, 33826, 33828, 33817, 33820, 33806, 33809, - 33821, 33813, 33810, 33827, 33823, 33822, 33830, 33835, 33834, 33833, - 33819, 33798, 33797, 33832, 33831, 33818, 33829, 33801, 33803, 33807, - 33814, 33805, 33812, 33811, 33800, 33795, 33796, 33804, 33799, 33802, - 33794, 33808, 33815, 33816, 33839, 33840, 33837, 33838, 33847, 33849, - 33846, 33845, 33842, 33841, 33844, 33843, 33850, 33848, 35762, 35762, - 35762, 35762, 35762, 33836, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 24159, 24162, 24161, 24163, 24160, 24142, 24146, 24144, - 24143, 24145, 24154, 24157, 24155, 24158, 24156, 24164, 24165, 24166, - 24167, 24168, 24147, 24149, 24152, 24153, 24148, 24151, 24150, 24172, - 24169, 24173, 24171, 24170, 24180, 24182, 24179, 24178, 24175, 24174, - 24177, 24176, 24183, 24181, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 12689, 12695, 12693, 12690, 12692, 12691, - 12694, 35762, 12644, 12688, 12686, 12685, 35762, 12632, 12631, 35762, - 12643, 12642, 12641, 12628, 12627, 12640, 12639, 12638, 12637, 12636, - 12635, 12630, 12629, 12634, 12633, 35762, 22398, 22399, 22400, 22466, - 22487, 22475, 22440, 22573, 22401, 22402, 22406, 22525, 22510, 22515, - 22439, 22591, 22542, 22461, 22445, 22536, 22405, 22403, 22404, 22451, - 22495, 22548, 22586, 22411, 22407, 22415, 22552, 22490, 22501, 22528, - 22416, 22413, 22412, 22565, 22503, 22563, 22541, 22532, 22530, 22531, - 22592, 22571, 22410, 22425, 22417, 22564, 22509, 22534, 22470, 22593, - 22421, 22422, 22423, 22479, 22471, 22455, 22556, 22507, 22408, 22414, - 22409, 22482, 22577, 22578, 22418, 22419, 22420, 22493, 22448, 22498, - 22465, 22426, 22424, 22427, 22551, 22508, 22557, 22459, 22569, 22428, - 22432, 22436, 22505, 22477, 22545, 22526, 22429, 22433, 22434, 22476, - 22468, 22533, 22486, 22594, 22497, 22435, 22430, 22431, 22513, 22566, - 22572, 22444, 22584, 22437, 22496, 22446, 22543, 22483, 22522, 22453, - 22535, 22485, 22452, 22590, 22442, 22488, 22438, 22484, 22512, 22538, - 22550, 22520, 22555, 22523, 22481, 22499, 22580, 22549, 22516, 22559, - 22587, 22562, 22560, 22583, 22456, 22570, 22463, 22491, 22447, 22478, - 22454, 22502, 22458, 22537, 22457, 22517, 22443, 22581, 22472, 22567, - 22568, 22585, 22558, 22500, 22540, 22529, 22492, 22467, 22441, 22506, - 22514, 22553, 22519, 22449, 22544, 22489, 22504, 22473, 22474, 22576, - 22518, 22521, 22524, 22588, 22511, 22460, 22462, 22547, 22589, 22539, - 22527, 22579, 22582, 22554, 22574, 22480, 22546, 22469, 22561, 22450, - 22575, 22494, 22464, 35762, 35762, 22602, 22600, 22599, 22596, 22595, - 22598, 22597, 22603, 22601, 22393, 22392, 22396, 22394, 22391, 22395, - 22397, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 27, 9, 24, 14, 29, 21, 34, 28, 37, 39, 35, 40, 41, 10, 30, - 32, 18, 15, 31, 42, 13, 25, 36, 23, 12, 20, 33, 19, 38, 17, 11, 26, 16, - 22, 62, 44, 59, 49, 64, 56, 69, 63, 72, 74, 70, 75, 76, 45, 65, 67, 53, - 50, 66, 77, 48, 60, 71, 58, 47, 55, 68, 54, 73, 52, 46, 61, 51, 57, 85, - 86, 79, 84, 43, 78, 83, 82, 35762, 35762, 35762, 35762, 93, 95, 92, 91, - 88, 87, 90, 89, 96, 94, 35762, 35762, 35762, 35762, 80, 81, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 16234, 16220, 16215, 16195, 16190, 16208, 16203, 16183, 16198, - 16223, 16218, 16213, 16193, 16188, 16209, 16204, 16184, 16199, 16235, - 16221, 16216, 16196, 16191, 16211, 16206, 16186, 16201, 16236, 16222, - 16217, 16197, 16192, 16212, 16207, 16187, 16202, 16224, 16219, 16214, - 16194, 16189, 16210, 16205, 16185, 16200, 16181, 16182, 16172, 16179, - 16180, 16232, 16230, 16229, 16226, 16225, 16228, 16227, 16233, 16231, - 16238, 16174, 16173, 16175, 16237, 16178, 16177, 16176, 16171, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 26069, 26064, 26059, 26039, 26034, - 26052, 26047, 26027, 26042, 26067, 26062, 26057, 26037, 26032, 26053, - 26048, 26028, 26043, 26070, 26065, 26060, 26040, 26035, 26055, 26050, - 26030, 26045, 26071, 26066, 26061, 26041, 26036, 26056, 26051, 26031, - 26046, 26068, 26063, 26058, 26038, 26033, 26054, 26049, 26029, 26044, - 26026, 26019, 26021, 26011, 26013, 26014, 26016, 26023, 26022, 26017, - 26012, 26015, 26020, 26018, 26025, 26024, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 2231, 2240, 2237, 2135, 35762, 2247, 2235, 2239, 2227, 2243, 2234, 2183, - 2238, 2245, 2210, 2232, 2241, 2211, 2246, 2242, 2209, 2230, 2229, 2233, - 2228, 2134, 2236, 2244, 2105, 2107, 2106, 2108, 35762, 2147, 2145, 35762, - 2142, 35762, 35762, 2141, 35762, 2149, 2144, 2152, 2146, 2151, 2139, - 2154, 2148, 2140, 2153, 35762, 2138, 2137, 2136, 2143, 35762, 2155, - 35762, 2150, 35762, 35762, 35762, 35762, 35762, 35762, 2219, 35762, - 35762, 35762, 35762, 2220, 35762, 2221, 35762, 2224, 35762, 2223, 2216, - 2226, 35762, 2217, 2225, 35762, 2215, 35762, 35762, 2218, 35762, 2214, - 35762, 2222, 35762, 2212, 35762, 2213, 35762, 2202, 2199, 35762, 2196, - 35762, 35762, 2195, 2190, 2204, 2198, 35762, 2200, 2206, 2193, 2208, - 2203, 2194, 2207, 35762, 2192, 2191, 2189, 2197, 35762, 2188, 2201, 2205, - 2186, 35762, 2187, 35762, 2160, 2172, 2170, 2180, 2163, 2182, 2168, 2162, - 2166, 2175, 35762, 2178, 2171, 2177, 2157, 2161, 2173, 2158, 2181, 2174, - 2156, 2167, 2165, 2159, 2164, 2179, 2169, 2176, 35762, 35762, 35762, - 35762, 35762, 2122, 2119, 2130, 35762, 2133, 2117, 2121, 2115, 2125, - 35762, 2128, 2120, 2127, 2110, 2132, 2123, 2111, 2131, 2124, 2109, 2116, - 2114, 2112, 2113, 2129, 2118, 2126, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 2184, 2185, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 20707, 20723, 20738, 20714, 20742, 20741, 20739, - 20719, 20736, 20733, 20712, 20709, 20728, 20725, 20705, 20716, 20720, - 20737, 20734, 20713, 20710, 20729, 20726, 20706, 20717, 20718, 20735, - 20732, 20711, 20708, 20727, 20724, 20704, 20715, 20722, 20721, 20702, - 20745, 20731, 20730, 20743, 20740, 20744, 20703, 35762, 35762, 35762, - 35762, 10829, 10780, 10781, 10782, 10783, 10784, 10785, 10786, 10787, - 10788, 10789, 10790, 10791, 10792, 10793, 10794, 10795, 10796, 10797, - 10798, 10799, 10800, 10801, 10802, 10803, 10804, 10805, 10806, 10807, - 10808, 10809, 10810, 10811, 10812, 10813, 10814, 10815, 10816, 10817, - 10818, 10819, 10820, 10821, 10822, 10823, 10824, 10825, 10826, 10827, - 10828, 10879, 10830, 10831, 10832, 10833, 10834, 10835, 10836, 10837, - 10838, 10839, 10840, 10841, 10842, 10843, 10844, 10845, 10846, 10847, - 10848, 10849, 10850, 10851, 10852, 10853, 10854, 10855, 10856, 10857, - 10858, 10859, 10860, 10861, 10862, 10863, 10864, 10865, 10866, 10867, - 10868, 10869, 10870, 10871, 10872, 10873, 10874, 10875, 10876, 10877, - 10878, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 26691, 26761, 26736, 26732, 26695, 26700, 26720, - 26716, 26712, 26765, 26728, 26769, 26704, 26724, 26708, 35762, 35762, - 26762, 26737, 26733, 26696, 26701, 26721, 26717, 26713, 26766, 26729, - 26770, 26705, 26725, 26709, 26692, 35762, 26763, 26738, 26734, 26697, - 26702, 26722, 26718, 26714, 26767, 26730, 26771, 26706, 26726, 26710, - 26690, 35762, 26760, 26735, 26731, 26694, 26699, 26719, 26715, 26711, - 26764, 26727, 26768, 26703, 26723, 26707, 26693, 26698, 26742, 26739, - 26753, 26754, 26755, 26756, 26757, 26758, 26759, 26743, 26744, 26745, - 26746, 26747, 26748, 26749, 26750, 26751, 26752, 26740, 26741, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 10565, - 10564, 10549, 10561, 10558, 10543, 10540, 10555, 10552, 10567, 10546, - 10606, 10585, 6512, 6229, 6253, 26362, 26363, 26364, 26365, 26366, 26367, - 26368, 26369, 26370, 26371, 26372, 26373, 26374, 26375, 26376, 26377, - 26378, 26379, 26380, 26381, 26382, 26383, 26384, 26385, 26386, 26387, - 32455, 6343, 6344, 6240, 6511, 8343, 29496, 29497, 29498, 29499, 29500, + 15571, 15572, 15573, 15574, 15575, 15576, 15577, 15578, 15579, 15580, + 15581, 15582, 15583, 15584, 15585, 15586, 15587, 15588, 15589, 15590, + 15591, 15592, 15593, 15594, 15595, 15596, 15597, 15598, 15599, 15600, + 15601, 15602, 15603, 15604, 15605, 15606, 15607, 15608, 15609, 15610, + 15611, 15612, 15613, 15614, 15615, 15616, 15617, 15618, 15619, 15620, + 15621, 15622, 15623, 15624, 15625, 15626, 15627, 15628, 15880, 15881, + 15882, 15883, 15884, 15885, 15886, 15887, 15888, 15889, 15890, 15891, + 15892, 15893, 15894, 15895, 15896, 15897, 15898, 15899, 15900, 15901, + 15902, 15903, 15904, 15905, 15906, 15907, 15908, 15909, 15910, 15911, + 15912, 15913, 15914, 15915, 15916, 15917, 15918, 15919, 15920, 15921, + 15922, 15923, 15924, 15925, 15926, 15927, 15928, 15929, 15930, 15931, + 15932, 15933, 15934, 15935, 15936, 15937, 15938, 15939, 15940, 15941, + 15942, 15943, 15944, 15945, 15946, 15947, 15948, 15949, 15950, 15951, + 15952, 15953, 15954, 15955, 15956, 15957, 15958, 15959, 15960, 15961, + 15962, 15963, 15964, 15965, 15966, 15967, 15968, 15969, 15970, 15971, + 15972, 15973, 15974, 15975, 15976, 15977, 15978, 15979, 15980, 15981, + 15982, 15983, 15984, 15985, 15986, 15987, 15988, 15989, 15990, 15991, + 15992, 15993, 15994, 15995, 15996, 15997, 15998, 15999, 16000, 16001, + 16002, 16003, 16004, 16005, 16006, 16007, 16008, 16009, 16010, 16011, + 16012, 16013, 16014, 16015, 16016, 16017, 16018, 16019, 16020, 16021, + 16022, 16023, 16024, 16025, 16026, 16027, 16028, 16029, 16030, 16031, + 16032, 16033, 16034, 16035, 16036, 16037, 16038, 16039, 16040, 16041, + 16042, 16043, 16044, 16045, 16046, 16047, 16048, 16049, 16050, 16051, + 16052, 16053, 16054, 16055, 16056, 16057, 16058, 16059, 16060, 16061, + 16062, 16063, 16064, 16065, 16066, 16067, 16068, 16069, 16070, 16071, + 16072, 16073, 16074, 16075, 16076, 16077, 16078, 16079, 16080, 16081, + 16082, 16083, 16084, 16085, 16086, 16087, 16088, 16089, 16090, 16091, + 16092, 16093, 16094, 16095, 16096, 16097, 16098, 16099, 16100, 16101, + 16102, 16103, 16104, 16105, 16106, 16107, 16108, 16109, 16110, 16111, + 16112, 16113, 16114, 16115, 16116, 16117, 16118, 16119, 16120, 16121, + 16122, 16123, 16124, 16125, 16126, 16127, 16128, 16129, 16130, 16131, + 16132, 16133, 16134, 16135, 16136, 16137, 16138, 16139, 16140, 16141, + 16142, 16143, 16144, 16145, 16146, 16147, 16148, 16149, 16150, 16151, + 16152, 16153, 16154, 16155, 16156, 16157, 16158, 16159, 16160, 16161, + 16162, 16163, 16164, 16165, 16166, 16167, 16168, 16169, 16170, 16171, + 16172, 16173, 16174, 16175, 16176, 16177, 16178, 16179, 16180, 16181, + 16182, 16183, 16184, 16185, 16186, 16187, 16188, 16189, 16190, 16191, + 16192, 16193, 16194, 16195, 16196, 16197, 16198, 16199, 16200, 16201, + 16202, 16203, 16204, 16205, 16206, 16207, 16208, 16209, 16210, 16211, + 16212, 16213, 16214, 16215, 16216, 16217, 16218, 16219, 16220, 16221, + 16222, 16223, 16224, 16225, 16226, 16227, 16228, 16229, 16230, 16231, + 16232, 16233, 16234, 16235, 16236, 16237, 16238, 16239, 16240, 16241, + 16242, 16243, 16244, 16245, 16246, 16247, 16248, 16249, 16250, 16251, + 16252, 16253, 16254, 16255, 16256, 16257, 16258, 16259, 16260, 16261, + 16262, 16263, 16264, 16265, 16266, 16267, 16268, 16269, 16270, 16271, + 16272, 16273, 16274, 16275, 16276, 16277, 16278, 16279, 16280, 16281, + 16282, 16283, 16284, 16285, 16286, 16287, 16288, 16289, 16290, 16291, + 16292, 16293, 16294, 16295, 16296, 16297, 16298, 16299, 16300, 16301, + 16302, 16303, 16304, 16305, 16306, 16307, 16308, 16309, 16310, 16311, + 16312, 16313, 16314, 16315, 16316, 16317, 16318, 16319, 16320, 16321, + 16322, 16323, 16324, 16325, 16326, 16327, 16328, 16329, 16330, 16331, + 16332, 16333, 16334, 16335, 16336, 16337, 16338, 16339, 16340, 16341, + 16342, 16343, 16344, 16345, 16346, 16347, 16348, 16349, 16350, 16351, + 16352, 16353, 16354, 16355, 16356, 16357, 16358, 16359, 16360, 16361, + 16362, 16363, 16364, 16365, 16366, 16367, 16368, 16369, 16370, 16371, + 16372, 16373, 16374, 16375, 16376, 16377, 16378, 16379, 16380, 16381, + 16382, 16383, 16384, 16385, 16386, 16387, 16388, 16389, 16390, 16391, + 16392, 16393, 16394, 16395, 16396, 16397, 16398, 16399, 16400, 16401, + 16402, 16403, 16404, 16405, 16406, 16407, 16408, 16409, 16410, 16411, + 16412, 16413, 16414, 16415, 16416, 16417, 16418, 16419, 16420, 16421, + 16422, 16423, 16424, 16425, 16426, 16427, 16428, 16429, 16430, 16431, + 16432, 16433, 16434, 16435, 16436, 16437, 16438, 16439, 16440, 16441, + 16442, 16443, 16444, 16445, 16446, 16447, 16448, 16449, 16450, 16451, + 16452, 16453, 16454, 16455, 16456, 16457, 16458, 16459, 16460, 16461, + 16462, 16463, 16464, 16465, 16466, 16467, 16468, 16469, 16470, 16471, + 16472, 16473, 16474, 16475, 16476, 16477, 16478, 16479, 16480, 16481, + 16482, 16483, 16484, 16485, 16486, 16487, 16488, 16489, 16490, 16491, + 16492, 16493, 16494, 16495, 16496, 16497, 16498, 16499, 16500, 16501, + 16502, 16503, 16504, 16505, 16506, 16507, 16508, 16509, 16510, 16511, + 16512, 16513, 16514, 16515, 16516, 16517, 16518, 16519, 16520, 16521, + 16522, 16523, 16524, 16525, 16526, 16527, 16528, 16529, 16530, 16531, + 16532, 16533, 16534, 16535, 16536, 16537, 16538, 16539, 16540, 16541, + 16542, 16543, 16544, 16545, 16546, 16547, 16548, 16549, 16550, 16551, + 16552, 16553, 16554, 16555, 16556, 16557, 16558, 16559, 16560, 16561, + 16562, 16563, 16564, 16565, 16566, 16567, 16568, 16569, 16570, 16571, + 16572, 16573, 16574, 16575, 16576, 16577, 16578, 16579, 16580, 16581, + 16582, 16583, 16584, 16585, 16586, 16587, 16588, 16589, 16590, 16591, + 16592, 16593, 16594, 16595, 16596, 16597, 16598, 16599, 16600, 16601, + 16602, 16603, 16604, 16605, 16606, 16607, 16608, 16609, 16610, 16611, + 16612, 16613, 16614, 16615, 16616, 16617, 16618, 16619, 16620, 16621, + 16622, 16623, 16624, 16625, 16626, 16627, 16628, 16629, 16630, 16631, + 16632, 16633, 16634, 16635, 16636, 16637, 16638, 16639, 16640, 16641, + 16642, 16643, 16644, 16645, 16646, 16647, 15640, 15641, 15642, 15643, + 15644, 15645, 15646, 15647, 15648, 15649, 15650, 15651, 15652, 15653, + 15654, 15655, 15656, 15657, 15658, 15659, 15660, 15661, 15662, 15663, + 15664, 15665, 15666, 15667, 15668, 15669, 15670, 15671, 15672, 15673, + 15674, 15675, 15676, 15677, 15678, 15679, 15680, 15681, 15682, 15683, + 15684, 15685, 15686, 15687, 15688, 15689, 15690, 15691, 15692, 15693, + 15694, 15695, 15696, 15697, 15698, 15699, 15700, 15701, 15702, 15703, + 15704, 15705, 15706, 15707, 15708, 15709, 15710, 15711, 15712, 15713, + 15714, 15715, 15716, 15717, 15718, 15719, 15720, 15721, 15722, 15723, + 15724, 15725, 15726, 15727, 15728, 15729, 15730, 15731, 15732, 15733, + 15734, 15735, 15736, 15737, 15738, 15739, 15740, 15741, 15742, 15743, + 15744, 15745, 15746, 15747, 15748, 15749, 15750, 15751, 15752, 15753, + 15754, 15755, 15756, 15757, 15758, 15759, 15760, 15761, 15762, 15763, + 15764, 15765, 15766, 15767, 15768, 15769, 15770, 15771, 15772, 15773, + 15774, 15775, 15776, 15777, 15778, 15779, 15780, 15781, 15782, 15783, + 15784, 15785, 15786, 15787, 15788, 15789, 15790, 15791, 15792, 15793, + 15794, 15795, 15796, 15797, 15798, 15799, 15800, 15801, 15802, 15803, + 15804, 15805, 15806, 15807, 15808, 15809, 15810, 15811, 15812, 15813, + 15814, 15815, 15816, 15817, 15818, 15819, 15820, 15821, 15822, 15823, + 15824, 15825, 15826, 15827, 15828, 15829, 15830, 15831, 15832, 15833, + 15834, 15835, 15836, 15837, 15838, 15839, 15840, 15841, 15842, 15843, + 15844, 15845, 15846, 15847, 15848, 15849, 15850, 15851, 15852, 15853, + 15854, 15855, 15856, 15857, 15858, 15859, 15860, 15861, 15862, 15863, + 15864, 15865, 15866, 15867, 15868, 15869, 15870, 15871, 15872, 15873, + 15874, 15875, 15876, 15877, 15878, 15879, 15629, 15630, 15631, 15632, + 15633, 15634, 15635, 15636, 15637, 15638, 15639, 40951, 40951, 40951, + 40951, 40951, 445, 446, 447, 448, 449, 450, 451, 452, 453, 434, 435, 436, + 437, 438, 439, 440, 441, 442, 443, 444, 375, 376, 377, 378, 379, 380, + 373, 374, 381, 382, 383, 425, 426, 427, 428, 429, 430, 431, 432, 433, + 423, 424, 391, 387, 388, 392, 393, 394, 389, 390, 384, 385, 386, 395, + 396, 397, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 402, 403, + 404, 405, 406, 407, 398, 399, 400, 401, 408, 409, 410, 464, 465, 466, + 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, + 481, 482, 483, 415, 416, 417, 418, 419, 420, 421, 411, 412, 413, 414, + 422, 495, 496, 497, 498, 499, 500, 501, 484, 485, 486, 487, 492, 493, + 494, 502, 488, 489, 490, 491, 503, 504, 505, 506, 507, 510, 511, 512, + 513, 508, 509, 514, 515, 516, 517, 520, 521, 522, 523, 524, 518, 519, + 525, 526, 527, 528, 531, 532, 533, 534, 535, 529, 530, 536, 537, 538, + 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, + 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, + 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, + 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, + 595, 596, 597, 598, 599, 607, 608, 600, 601, 602, 609, 610, 611, 612, + 603, 604, 613, 605, 606, 618, 619, 620, 621, 622, 614, 615, 616, 617, + 623, 624, 625, 651, 652, 653, 654, 655, 656, 657, 649, 650, 658, 659, + 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, + 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, + 699, 700, 662, 663, 664, 665, 666, 667, 668, 660, 661, 669, 670, 701, + 702, 703, 704, 705, 706, 707, 708, 709, 710, 640, 641, 642, 643, 644, + 645, 646, 647, 648, 638, 639, 630, 631, 632, 633, 626, 627, 634, 635, + 636, 637, 628, 629, 713, 714, 715, 716, 717, 718, 719, 720, 721, 711, + 712, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 724, 725, 726, + 727, 728, 729, 730, 731, 732, 722, 723, 751, 752, 748, 749, 750, 753, + 754, 755, 744, 745, 746, 747, 756, 757, 758, 815, 816, 817, 818, 819, + 820, 821, 822, 823, 824, 735, 736, 737, 738, 739, 740, 741, 742, 743, + 733, 734, 763, 764, 765, 766, 759, 760, 767, 768, 769, 761, 762, 770, + 796, 794, 795, 797, 798, 799, 800, 801, 802, 803, 804, 777, 773, 774, + 778, 771, 772, 779, 780, 775, 776, 781, 782, 783, 785, 786, 787, 784, + 788, 789, 790, 791, 792, 793, 856, 857, 858, 859, 860, 861, 862, 863, + 864, 865, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 866, + 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, + 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, + 895, 836, 837, 840, 841, 842, 843, 844, 845, 838, 839, 846, 847, 896, + 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, + 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, + 925, 848, 849, 850, 851, 852, 853, 854, 855, 927, 928, 929, 930, 931, + 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, + 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 926, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 18950, 18940, 18939, 18936, 18935, 18921, 18934, + 18933, 18938, 18937, 18943, 18928, 18927, 18924, 18923, 18948, 18930, + 18929, 18926, 18925, 18922, 18942, 18941, 18932, 18931, 18945, 18949, + 18946, 18944, 18947, 18953, 18960, 18961, 18956, 18957, 18962, 18963, + 18954, 18958, 18959, 18955, 18964, 18920, 18919, 18917, 18951, 18918, + 18952, 18971, 18973, 18970, 18969, 18966, 18965, 18968, 18967, 18974, + 18972, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 2847, + 2813, 2872, 2873, 2843, 2880, 2891, 2862, 2879, 2874, 2875, 2826, 2892, + 2849, 2828, 2833, 2840, 2886, 2858, 2816, 2852, 2887, 2848, 2823, 2824, + 2856, 2830, 2871, 2812, 2870, 2839, 2863, 2894, 2811, 2857, 2888, 2842, + 2841, 2837, 2814, 2869, 2846, 2876, 2829, 2866, 2877, 2893, 2821, 2883, + 2890, 2831, 2817, 2845, 2819, 2838, 2881, 2822, 2896, 2898, 2818, 2884, + 2834, 2878, 2855, 2882, 2860, 2867, 2851, 2895, 2850, 2832, 2864, 2835, + 2861, 2889, 2885, 2868, 2853, 2825, 2859, 2815, 2854, 2897, 2820, 2865, + 2844, 2836, 2930, 2946, 2944, 2943, 2909, 2915, 2904, 2950, 2914, 2906, + 2936, 2949, 2908, 2934, 2935, 2899, 2939, 2947, 2941, 2942, 2921, 2918, + 2933, 2902, 2900, 2901, 2907, 2937, 2954, 2928, 2926, 2951, 2940, 2948, + 2920, 2924, 2925, 2922, 2945, 2917, 2923, 2932, 2938, 2919, 2952, 2916, + 2953, 2903, 2913, 2912, 2910, 2927, 2931, 2911, 2905, 2929, 3006, 3026, + 3047, 3043, 3005, 2994, 3007, 2955, 2983, 2957, 3027, 3023, 2980, 3028, + 2998, 2977, 2960, 2956, 2962, 3046, 3025, 2988, 3018, 2986, 3048, 2967, + 2968, 3032, 2999, 3036, 3014, 3040, 3035, 3001, 3042, 2992, 2974, 3024, + 3002, 2970, 2985, 2990, 3021, 3037, 3004, 3051, 2995, 3020, 3017, 3050, + 3041, 2982, 3052, 3010, 2969, 3039, 3015, 3012, 2964, 3000, 2976, 2987, + 2972, 2966, 3011, 3009, 3044, 3003, 3019, 3022, 2965, 3016, 2996, 2973, + 3049, 2997, 3033, 3031, 2984, 2975, 2979, 2961, 2981, 2963, 2978, 3045, + 3013, 2991, 2989, 3034, 2959, 2958, 3008, 2993, 2971, 3029, 3030, 3038, + 3080, 3164, 3113, 3087, 3114, 3073, 3112, 3118, 3100, 3127, 3163, 3109, + 3143, 3110, 3057, 3156, 3141, 3116, 3148, 3061, 3165, 3063, 3150, 3085, + 3095, 3076, 3082, 3152, 3169, 3111, 3058, 3106, 3158, 3056, 3108, 3053, + 3096, 3090, 3068, 3097, 3092, 3091, 3133, 3089, 3086, 3074, 3119, 3078, + 3066, 3125, 3154, 3153, 3166, 3059, 3140, 3157, 3105, 3084, 3120, 3060, + 3136, 3131, 3126, 3071, 3099, 3088, 3103, 3167, 3135, 3168, 3098, 3122, + 3147, 3064, 3102, 3107, 3159, 3081, 3065, 3121, 3155, 3077, 3101, 3069, + 3104, 3117, 3115, 3055, 3062, 3137, 3161, 3160, 3128, 3139, 3070, 3083, + 3075, 3149, 3094, 3145, 3142, 3067, 3130, 3146, 3123, 3132, 3129, 3144, + 3134, 3093, 3072, 3138, 3162, 3124, 3079, 3151, 3054, 3223, 3300, 3210, + 3197, 3307, 3200, 3263, 3289, 3279, 3249, 3226, 3274, 3294, 3235, 3190, + 3310, 3282, 3228, 3264, 3299, 3192, 3202, 3251, 3241, 3207, 3238, 3242, + 3250, 3281, 3248, 3267, 3290, 3231, 3214, 3184, 3237, 3245, 3208, 3201, + 3229, 3225, 3291, 3283, 3277, 3222, 3230, 3315, 3183, 3304, 3311, 3271, + 3303, 3187, 3288, 3297, 3305, 3308, 3196, 3273, 3293, 3181, 3243, 3284, + 3213, 3211, 3256, 3260, 3180, 3302, 3191, 3323, 3254, 3316, 3212, 3224, + 3269, 3319, 3199, 3173, 3179, 3240, 3189, 3206, 3236, 3185, 3175, 3252, + 3265, 3313, 3227, 3255, 3257, 3272, 3216, 3174, 3259, 3217, 3182, 3172, + 3220, 3275, 3239, 3193, 3270, 3253, 3312, 3232, 3261, 3171, 3178, 3246, + 3324, 3301, 3326, 3325, 3198, 3262, 3292, 3295, 3221, 3287, 3314, 3233, + 3320, 3317, 3318, 3322, 3321, 3188, 3266, 3247, 3276, 3309, 3177, 3306, + 3204, 3215, 3280, 3278, 3234, 3244, 3285, 3286, 3170, 3258, 3268, 3203, + 3195, 3218, 3205, 3209, 3296, 3194, 3219, 3298, 3176, 3186, 3331, 3380, + 3333, 3378, 3358, 3371, 3352, 3335, 3361, 3360, 3340, 3370, 3350, 3346, + 3337, 3368, 3363, 3369, 3366, 3329, 3328, 3348, 3347, 3345, 3373, 3365, + 3374, 3349, 3354, 3351, 3375, 3355, 3362, 3353, 3357, 3327, 3343, 3344, + 3364, 3356, 3379, 3376, 3336, 3334, 3332, 3338, 3359, 3341, 3342, 3339, + 3372, 3330, 3367, 3377, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 28219, 28211, 28232, 28209, 28233, 28220, 28235, 28215, 28206, 28223, + 28221, 28231, 28205, 28213, 28208, 28210, 28216, 28214, 28212, 28225, + 28228, 28217, 28227, 28234, 28226, 28207, 28230, 28229, 28222, 28224, + 28218, 40951, 28244, 28246, 28243, 28242, 28239, 28238, 28241, 28240, + 28247, 28245, 40951, 40951, 40951, 40951, 28237, 28236, 35842, 35839, + 35840, 35841, 35794, 35791, 35792, 35793, 35846, 35843, 35844, 35845, + 35834, 35831, 35832, 35833, 35838, 35835, 35836, 35837, 35830, 35827, + 35828, 35829, 35790, 35787, 35788, 35789, 35822, 35819, 35820, 35821, + 35795, 35800, 35811, 35812, 35823, 35826, 35824, 35825, 35818, 35815, + 35816, 35817, 35806, 35803, 35804, 35805, 35857, 35856, 35855, 35807, + 35814, 35864, 35862, 35859, 35809, 35858, 35860, 35802, 35810, 35799, + 35801, 35798, 35849, 35853, 35861, 35808, 35813, 35851, 35848, 35854, + 35797, 35847, 35863, 35796, 35852, 35850, 35865, 40951, 35872, 35874, + 35871, 35870, 35867, 35866, 35869, 35868, 35875, 35873, 40951, 40951, + 40951, 40951, 40951, 40951, 3441, 3446, 3462, 3464, 3454, 3452, 3444, + 3438, 3445, 3458, 3453, 3449, 3460, 3443, 3439, 3461, 3448, 3459, 3463, + 3457, 3451, 3465, 3450, 3466, 3455, 3456, 3447, 3442, 3440, 3467, 40951, + 40951, 3434, 3436, 3437, 3435, 3433, 3468, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 30994, 30995, 31000, 31001, + 30988, 30989, 31004, 31005, 30996, 30997, 30986, 30987, 31006, 31007, + 30990, 30991, 31002, 31003, 31008, 31009, 30998, 30999, 30992, 30993, + 31010, 31011, 30984, 30985, 30930, 30921, 30927, 30918, 30923, 30929, + 30922, 30926, 30932, 30916, 30928, 30914, 30919, 30917, 30925, 30920, + 30924, 30933, 30931, 30915, 30938, 30937, 30935, 30934, 30936, 30940, + 30939, 30970, 30971, 30949, 30969, 30967, 30975, 30977, 30976, 30978, + 30968, 30960, 30973, 30958, 30980, 30953, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 31018, 31020, 31017, 31016, + 31013, 31012, 31015, 31014, 31021, 31019, 40951, 30945, 30942, 30944, + 30947, 30941, 30943, 30946, 40951, 30972, 30979, 30957, 30965, 30981, + 30956, 30963, 30974, 30962, 30983, 30964, 30959, 30966, 30982, 30961, + 30955, 30948, 30951, 30952, 30954, 30950, 40951, 40951, 40951, 40951, + 40951, 30902, 30909, 30901, 30900, 30910, 30898, 30895, 30913, 30905, + 30903, 30911, 30897, 30896, 30906, 30912, 30908, 30904, 30899, 30907, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 22525, 22522, 22527, 22521, + 22510, 22509, 22506, 22505, 22498, 22504, 22503, 22508, 22507, 22499, + 22495, 22494, 22491, 22490, 22497, 22496, 22493, 22492, 22500, 22512, + 22511, 22502, 22501, 22517, 22520, 22518, 22516, 22519, 22514, 22513, + 22515, 22528, 22534, 22531, 22532, 22533, 22529, 22535, 22530, 22526, + 22524, 22523, 22537, 22536, 22544, 22546, 22543, 22542, 22539, 22538, + 22541, 22540, 22547, 22545, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 26886, 26890, 26893, 26894, 26864, 26896, 26872, 26887, + 26891, 26882, 26881, 26883, 26871, 26863, 26884, 26880, 26877, 26878, + 26892, 26874, 26885, 26888, 26870, 26868, 26895, 26879, 26876, 26866, + 26889, 26875, 26865, 26873, 26944, 26948, 26951, 26952, 26922, 26954, + 26930, 26945, 26949, 26940, 26939, 26941, 26929, 26921, 26942, 26938, + 26935, 26936, 26950, 26932, 26943, 26946, 26928, 26926, 26953, 26937, + 26934, 26924, 26947, 26933, 26923, 26931, 26908, 26902, 26900, 26898, + 26905, 26904, 26907, 26906, 26910, 26909, 26913, 26915, 26912, 26911, + 26916, 26917, 26919, 26920, 26914, 26918, 26903, 26901, 26899, 26897, + 26957, 26955, 26956, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 27502, 27428, 27490, 27501, 27506, 27505, + 27425, 27507, 27482, 27481, 27479, 27436, 27485, 27486, 27478, 27435, + 27444, 27452, 27488, 27424, 27449, 27448, 27443, 27442, 27441, 27440, + 27466, 27434, 27465, 27433, 27508, 27439, 27489, 27500, 27499, 27447, + 27446, 27423, 27504, 27510, 27438, 27437, 27475, 27431, 27451, 27450, + 27474, 27430, 27483, 27487, 27459, 27462, 27463, 27497, 27495, 27476, + 27432, 27484, 27464, 27498, 27496, 27494, 27492, 27422, 27493, 27491, + 27509, 27426, 27503, 27427, 27461, 27429, 27480, 27477, 27460, 40951, + 40951, 40951, 40951, 27514, 27445, 27513, 27512, 27511, 27519, 27525, + 27524, 27520, 27521, 27546, 27550, 27567, 27566, 27528, 27531, 27532, + 27548, 27535, 27536, 27537, 27538, 27539, 27542, 27544, 27545, 27541, + 27552, 27553, 27554, 27555, 27560, 27556, 27557, 27561, 27563, 27522, + 27523, 27530, 27565, 27529, 27564, 27526, 27534, 27527, 27551, 27568, + 27569, 27558, 27562, 27549, 27547, 27570, 27543, 27533, 27540, 27559, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27518, 27516, 27517, + 27515, 27467, 27468, 27469, 27470, 27471, 27472, 27473, 27453, 27454, + 27455, 27456, 27457, 27458, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 36644, 29851, + 30073, 30072, 22168, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 38877, 38876, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 35876, 35877, 35878, 35879, 35880, 35881, 35882, 35883, 35884, 35885, + 35886, 35887, 35888, 35889, 35890, 35891, 35892, 35893, 35894, 35895, + 35896, 35897, 35898, 35899, 35900, 35901, 35902, 35903, 35904, 35905, + 35906, 35907, 35908, 35909, 35910, 35911, 35912, 35913, 35914, 35915, + 35916, 35917, 35918, 35919, 35920, 35921, 35922, 35923, 35924, 35925, + 35926, 35927, 35928, 35929, 35930, 35931, 35932, 35933, 35934, 35935, + 35936, 35937, 35938, 35939, 35940, 35941, 35942, 35943, 35944, 35945, + 35946, 35947, 35948, 35949, 35950, 35951, 35952, 35953, 35954, 35955, + 35956, 35957, 35958, 35959, 35960, 35961, 35962, 35963, 35964, 35965, + 35966, 35967, 35968, 35969, 35970, 35971, 35972, 35973, 35974, 35975, + 35976, 35977, 35978, 35979, 35980, 35981, 35982, 35983, 35984, 35985, + 35986, 35987, 35988, 35989, 35990, 35991, 35992, 35993, 35994, 35995, + 35996, 35997, 35998, 35999, 36000, 36001, 36002, 36003, 36004, 36005, + 36006, 36007, 36008, 36009, 36010, 36011, 36012, 36013, 36014, 36015, + 36016, 36017, 36018, 36019, 36020, 36021, 36022, 36023, 36024, 36025, + 36026, 36027, 36028, 36029, 36030, 36031, 36032, 36033, 36034, 36035, + 36036, 36037, 36038, 36039, 36040, 36041, 36042, 36043, 36044, 36045, + 36046, 36047, 36048, 36049, 36050, 36051, 36052, 36053, 36054, 36055, + 36056, 36057, 36058, 36059, 36060, 36061, 36062, 36063, 36064, 36065, + 36066, 36067, 36068, 36069, 36070, 36071, 36072, 36073, 36074, 36075, + 36076, 36077, 36078, 36079, 36080, 36081, 36082, 36083, 36084, 36085, + 36086, 36087, 36088, 36089, 36090, 36091, 36092, 36093, 36094, 36095, + 36096, 36097, 36098, 36099, 36100, 36101, 36102, 36103, 36104, 36105, + 36106, 36107, 36108, 36109, 36110, 36111, 36112, 36113, 36114, 36115, + 36116, 36117, 36118, 36119, 36120, 36121, 36122, 36123, 36124, 36125, + 36126, 36127, 36128, 36129, 36130, 36131, 36132, 36133, 36134, 36135, + 36136, 36137, 36138, 36139, 36140, 36141, 36142, 36143, 36144, 36145, + 36146, 36147, 36148, 36149, 36150, 36151, 36152, 36153, 36154, 36155, + 36156, 36157, 36158, 36159, 36160, 36161, 36162, 36163, 36164, 36165, + 36166, 36167, 36168, 36169, 36170, 36171, 36172, 36173, 36174, 36175, + 36176, 36177, 36178, 36179, 36180, 36181, 36182, 36183, 36184, 36185, + 36186, 36187, 36188, 36189, 36190, 36191, 36192, 36193, 36194, 36195, + 36196, 36197, 36198, 36199, 36200, 36201, 36202, 36203, 36204, 36205, + 36206, 36207, 36208, 36209, 36210, 36211, 36212, 36213, 36214, 36215, + 36216, 36217, 36218, 36219, 36220, 36221, 36222, 36223, 36224, 36225, + 36226, 36227, 36228, 36229, 36230, 36231, 36232, 36233, 36234, 36235, + 36236, 36237, 36238, 36239, 36240, 36241, 36242, 36243, 36244, 36245, + 36246, 36247, 36248, 36249, 36250, 36251, 36252, 36253, 36254, 36255, + 36256, 36257, 36258, 36259, 36260, 36261, 36262, 36263, 36264, 36265, + 36266, 36267, 36268, 36269, 36270, 36271, 36272, 36273, 36274, 36275, + 36276, 36277, 36278, 36279, 36280, 36281, 36282, 36283, 36284, 36285, + 36286, 36287, 36288, 36289, 36290, 36291, 36292, 36293, 36294, 36295, + 36296, 36297, 36298, 36299, 36300, 36301, 36302, 36303, 36304, 36305, + 36306, 36307, 36308, 36309, 36310, 36311, 36312, 36313, 36314, 36315, + 36316, 36317, 36318, 36319, 36320, 36321, 36322, 36323, 36324, 36325, + 36326, 36327, 36328, 36329, 36330, 36331, 36332, 36333, 36334, 36335, + 36336, 36337, 36338, 36339, 36340, 36341, 36342, 36343, 36344, 36345, + 36346, 36347, 36348, 36349, 36350, 36351, 36352, 36353, 36354, 36355, + 36356, 36357, 36358, 36359, 36360, 36361, 36362, 36363, 36364, 36365, + 36366, 36367, 36368, 36369, 36370, 36371, 36372, 36373, 36374, 36375, + 36376, 36377, 36378, 36379, 36380, 36381, 36382, 36383, 36384, 36385, + 36386, 36387, 36388, 36389, 36390, 36391, 36392, 36393, 36394, 36395, + 36396, 36397, 36398, 36399, 36400, 36401, 36402, 36403, 36404, 36405, + 36406, 36407, 36408, 36409, 36410, 36411, 36412, 36413, 36414, 36415, + 36416, 36417, 36418, 36419, 36420, 36421, 36422, 36423, 36424, 36425, + 36426, 36427, 36428, 36429, 36430, 36431, 36432, 36433, 36434, 36435, + 36436, 36437, 36438, 36439, 36440, 36441, 36442, 36443, 36444, 36445, + 36446, 36447, 36448, 36449, 36450, 36451, 36452, 36453, 36454, 36455, + 36456, 36457, 36458, 36459, 36460, 36461, 36462, 36463, 36464, 36465, + 36466, 36467, 36468, 36469, 36470, 36471, 36472, 36473, 36474, 36475, + 36476, 36477, 36478, 36479, 36480, 36481, 36482, 36483, 36484, 36485, + 36486, 36487, 36488, 36489, 36490, 36491, 36492, 36493, 36494, 36495, + 36496, 36497, 36498, 36499, 36500, 36501, 36502, 36503, 36504, 36505, + 36506, 36507, 36508, 36509, 36510, 36511, 36512, 36513, 36514, 36515, + 36516, 36517, 36518, 36519, 36520, 36521, 36522, 36523, 36524, 36525, + 36526, 36527, 36528, 36529, 36530, 36531, 36532, 36533, 36534, 36535, + 36536, 36537, 36538, 36539, 36540, 36541, 36542, 36543, 36544, 36545, + 36546, 36547, 36548, 36549, 36550, 36551, 36552, 36553, 36554, 36555, + 36556, 36557, 36558, 36559, 36560, 36561, 36562, 36563, 36564, 36565, + 36566, 36567, 36568, 36569, 36570, 36571, 36572, 36573, 36574, 36575, + 36576, 36577, 36578, 36579, 36580, 36581, 36582, 36583, 36584, 36585, + 36586, 36587, 36588, 36589, 36590, 36591, 36592, 36593, 36594, 36595, + 36596, 36597, 36598, 36599, 36600, 36601, 36602, 36603, 36604, 36605, + 36606, 36607, 36608, 36609, 36610, 36611, 36612, 36613, 36614, 36615, + 36616, 36617, 36618, 36619, 36620, 36621, 36622, 36623, 36624, 36625, + 36626, 36627, 36628, 36629, 36630, 36631, 36632, 36633, 36634, 36635, + 36636, 36637, 36638, 36639, 36640, 36641, 36642, 36643, 21697, 21698, + 21699, 21700, 21701, 21702, 21703, 21704, 21705, 21706, 21707, 21708, + 21709, 21710, 21711, 21712, 21713, 21714, 21715, 21716, 21717, 21718, + 21719, 21720, 21721, 21722, 21723, 21724, 21725, 21726, 21727, 21728, + 21729, 21730, 21731, 21732, 21733, 21734, 21735, 21736, 21737, 21738, + 21739, 21740, 21741, 21742, 21743, 21744, 21745, 21746, 21747, 21748, + 21749, 21750, 21751, 21752, 21753, 21754, 21755, 21756, 21757, 21758, + 21759, 21760, 21761, 21762, 21763, 21764, 21765, 21766, 21767, 21768, + 21769, 21770, 21771, 21772, 21773, 21774, 21775, 21776, 21777, 21778, + 21779, 21780, 21781, 21782, 21783, 21784, 21785, 21786, 21787, 21788, + 21789, 21790, 21791, 21792, 21793, 21794, 21795, 21796, 21797, 21798, + 21799, 21800, 21801, 21802, 21803, 21804, 21805, 21806, 21807, 21808, + 21809, 21810, 21811, 21812, 21813, 21814, 21815, 21816, 21817, 21818, + 21819, 21820, 21821, 21822, 21823, 21824, 21825, 21826, 21827, 21828, + 21829, 21830, 21831, 21832, 21833, 21834, 21835, 21836, 21837, 21838, + 21839, 21840, 21841, 21842, 21843, 21844, 21845, 21846, 21847, 21848, + 21849, 21850, 21851, 21852, 21853, 21854, 21855, 21856, 21857, 21858, + 21859, 21860, 21861, 21862, 21863, 21864, 21865, 21866, 21867, 21868, + 21869, 21870, 21871, 21872, 21873, 21874, 21875, 21876, 21877, 21878, + 21879, 21880, 21881, 21882, 21883, 21884, 21885, 21886, 21887, 21888, + 21889, 21890, 21891, 21892, 21893, 21894, 21895, 21896, 21897, 21898, + 21899, 21900, 21901, 21902, 21903, 21904, 21905, 21906, 21907, 21908, + 21909, 21910, 21911, 21912, 21913, 21914, 21915, 21916, 21917, 21918, + 21919, 21920, 21921, 21922, 21923, 21924, 21925, 21926, 21927, 21928, + 21929, 21930, 21931, 21932, 21933, 21934, 21935, 21936, 21937, 21938, + 21939, 21940, 21941, 21942, 21943, 21944, 21945, 21946, 21947, 21948, + 21949, 21950, 21951, 21952, 21959, 21960, 21961, 21962, 21963, 21964, + 21965, 21966, 21967, 21968, 21969, 21970, 21971, 21972, 21973, 21974, + 21975, 21976, 21977, 21978, 21979, 21980, 21981, 21982, 21983, 21984, + 21985, 21986, 21987, 21988, 21989, 21990, 21991, 21992, 21993, 21994, + 21995, 21996, 21997, 21998, 21999, 22000, 22001, 22002, 22003, 22004, + 22005, 22006, 22007, 22008, 22009, 22010, 22011, 22012, 22013, 22014, + 22015, 22016, 22017, 22018, 22019, 22020, 22021, 22022, 22023, 22024, + 22025, 22026, 22027, 22028, 22029, 22030, 22031, 22032, 22033, 22034, + 22035, 22036, 22037, 22038, 22039, 22040, 22041, 22042, 22043, 22044, + 22045, 22046, 22047, 22048, 22049, 22050, 22051, 22052, 22053, 22054, + 22055, 22056, 22057, 22058, 22059, 22060, 22061, 22062, 22063, 22064, + 22065, 22066, 22067, 22068, 22069, 22070, 22071, 22072, 22073, 22074, + 22075, 22076, 22077, 22078, 22079, 22080, 22081, 22082, 22083, 22084, + 22085, 22086, 22087, 22088, 22089, 22090, 22091, 22092, 22093, 22094, + 22095, 22096, 22097, 22098, 22099, 22100, 22101, 22102, 22103, 22104, + 22105, 22106, 22107, 22108, 22109, 22110, 22111, 22112, 22113, 22114, + 22115, 22116, 22117, 22118, 22119, 22120, 22121, 22122, 22123, 22124, + 22125, 22126, 22127, 22128, 22129, 22130, 22131, 22132, 22133, 22134, + 22135, 22136, 22137, 22138, 22139, 22140, 22141, 22142, 22143, 22144, + 22145, 22146, 22147, 22148, 22149, 22150, 22151, 22152, 22153, 22154, + 22155, 22156, 22157, 22158, 22159, 22160, 22161, 22162, 22163, 22164, + 22165, 22166, 21953, 21954, 21955, 21956, 21957, 21958, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 22167, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 21336, 21337, 21338, 21339, 40951, 21340, 21341, 21329, 21330, 21331, + 21332, 21333, 40951, 21334, 21335, 40951, 21317, 20293, 19932, 19933, + 19934, 19931, 20213, 20214, 20215, 20216, 20205, 20206, 20207, 20208, + 20209, 20200, 20201, 20202, 20203, 20204, 20210, 20211, 20212, 19971, + 19975, 19976, 19977, 19978, 19979, 19980, 19981, 19982, 19972, 19973, + 19974, 19987, 19988, 19989, 19990, 19991, 19992, 19993, 19994, 20001, + 20002, 20003, 20004, 20005, 20006, 20007, 19995, 19996, 19997, 19998, + 19999, 20000, 19984, 19985, 19986, 19983, 20096, 20097, 20098, 20099, + 20100, 20101, 20102, 20103, 20112, 20113, 20114, 20115, 20116, 20117, + 20104, 20105, 20106, 20107, 20108, 20109, 20110, 20111, 20118, 20119, + 20120, 20121, 20122, 20123, 20124, 20125, 20126, 20127, 20128, 20129, + 20158, 20159, 20160, 20161, 20151, 20152, 20153, 20154, 20155, 20156, + 20157, 20147, 20148, 20149, 20150, 20146, 20130, 20131, 20132, 20133, + 20134, 20135, 20136, 20137, 20138, 20140, 20141, 20142, 20143, 20144, + 20145, 20139, 20050, 20051, 20052, 20053, 20054, 20055, 20056, 20057, + 20058, 20043, 20044, 20045, 20046, 20047, 20048, 20049, 20042, 20064, + 20065, 20066, 20036, 20037, 20038, 20039, 20040, 20041, 20035, 20059, + 20060, 20061, 20062, 20063, 19943, 19946, 19947, 19948, 19949, 19950, + 19951, 19952, 19953, 19944, 19945, 19964, 19965, 19966, 19967, 19968, + 19969, 19970, 19954, 19955, 19956, 19957, 19958, 19959, 19960, 19961, + 19962, 19963, 19935, 19936, 19937, 19938, 19939, 19940, 19941, 19942, + 20017, 20018, 20019, 20020, 20021, 20022, 20023, 20024, 20025, 20026, + 20027, 20028, 20029, 20030, 20031, 20032, 20033, 20034, 20009, 20010, + 20008, 20011, 20012, 20013, 20014, 20015, 20016, 20184, 20185, 20186, + 20187, 20188, 20183, 20195, 20196, 20197, 20198, 20189, 20190, 20191, + 20192, 20193, 20194, 20088, 20089, 20090, 20091, 20081, 20082, 20083, + 20084, 20085, 20086, 20087, 20075, 20076, 20077, 20078, 20079, 20080, + 20092, 20093, 20094, 20095, 20069, 20070, 20071, 20072, 20073, 20074, + 20162, 20163, 20164, 20165, 20166, 20167, 20168, 20169, 20170, 20171, + 20179, 20180, 20181, 20182, 20172, 20173, 20174, 20175, 20176, 20177, + 20178, 20067, 20068, 20292, 21315, 21314, 21316, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 20303, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 20296, 20295, 20297, 40951, 40951, 21364, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 21370, 21369, 21371, 21375, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 29455, 29456, 29457, 29458, 29459, 29460, + 29461, 29462, 29463, 29464, 29465, 29466, 29467, 29468, 29469, 29470, + 29471, 29472, 29473, 29474, 29475, 29476, 29477, 29478, 29479, 29480, + 29481, 29482, 29483, 29484, 29485, 29486, 29487, 29488, 29489, 29490, + 29491, 29492, 29493, 29494, 29495, 29496, 29497, 29498, 29499, 29500, 29501, 29502, 29503, 29504, 29505, 29506, 29507, 29508, 29509, 29510, 29511, 29512, 29513, 29514, 29515, 29516, 29517, 29518, 29519, 29520, - 29521, 29490, 29526, 29541, 29542, 29531, 29553, 24272, 24273, 24274, - 24275, 24276, 24277, 24278, 24279, 24280, 24281, 24282, 24283, 24284, - 24285, 24286, 24287, 24288, 24289, 24290, 24291, 24292, 24293, 24294, - 24295, 24296, 24297, 26969, 26970, 26971, 6239, 6238, 6287, 24303, 24304, - 24305, 24306, 24307, 24308, 24309, 24310, 24311, 24312, 24313, 24314, - 24315, 24316, 24317, 24318, 24319, 24320, 24321, 24322, 24323, 24324, - 24325, 24326, 24327, 24328, 8387, 24332, 24334, 24335, 24331, 24333, - 29232, 29479, 29478, 29485, 29554, 29527, 29528, 29530, 29540, 29548, - 29551, 29543, 29533, 29545, 29483, 29547, 29484, 29534, 29544, 29536, - 29529, 29495, 29489, 29488, 29487, 29524, 29535, 29549, 29550, 21137, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 26997, 26998, 26999, 27000, - 27001, 27002, 27003, 27004, 27005, 27006, 27007, 27008, 27009, 27010, - 27011, 27012, 27013, 27014, 27015, 27016, 27017, 27018, 27019, 27020, - 27021, 27022, 29270, 29491, 29493, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 29457, 29452, - 29444, 29492, 29438, 29450, 29473, 29449, 29437, 29462, 29470, 29461, - 29442, 29453, 29454, 29460, 29441, 29471, 29468, 29475, 29451, 29446, - 29465, 29455, 29458, 29435, 29436, 29477, 29448, 29439, 29443, 29459, - 29474, 29456, 29469, 29472, 29445, 29466, 29464, 29463, 29467, 29440, - 29447, 29476, 35762, 35762, 35762, 35762, 32452, 32446, 32447, 32449, - 32453, 32450, 32454, 32448, 32451, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 6291, 6289, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 27401, 27402, - 27399, 27403, 27398, 27400, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 10219, 13162, - 7734, 24578, 29724, 29723, 6522, 29725, 26962, 4630, 33941, 33767, 22902, - 11218, 11216, 11217, 13610, 24383, 33853, 13131, 33854, 13203, 33852, - 18158, 33851, 8377, 24382, 13130, 18157, 13202, 29648, 13611, 27918, - 32014, 3808, 34096, 34100, 34097, 34098, 7747, 7746, 7745, 7744, 13161, - 34162, 15972, 31673, 4706, 6175, 27734, 13062, 10243, 26230, 5823, 15969, - 32566, 6173, 27389, 15937, 29726, 3985, 11212, 11213, 15762, 13180, - 20989, 13101, 19445, 23855, 32421, 2525, 13723, 22390, 33944, 30811, - 19779, 3406, 26676, 26994, 14263, 26496, 26493, 6174, 29593, 14654, - 28875, 22182, 26811, 27117, 27118, 8177, 9648, 29581, 29170, 4628, 13193, - 27405, 10228, 26088, 29766, 13201, 13137, 28969, 27862, 16002, 10975, - 8176, 6212, 5698, 20592, 9650, 15943, 27935, 3629, 26809, 8175, 13167, - 31675, 27701, 34164, 7751, 32533, 3520, 7693, 2589, 13168, 4093, 26799, - 27108, 34184, 3740, 16359, 6213, 13107, 13127, 13126, 2729, 26420, 8156, - 30810, 8388, 26675, 16364, 5760, 34161, 23499, 27706, 13648, 15234, 4095, - 22901, 27061, 23619, 29597, 19778, 8168, 3510, 3511, 13122, 98, 5758, - 13113, 27348, 13135, 22894, 23520, 6215, 15233, 2506, 32435, 6520, 32420, - 7689, 26525, 33689, 10611, 28881, 3739, 13322, 4098, 13153, 23854, 23602, - 27700, 14279, 23618, 32538, 33694, 23853, 27530, 31789, 28856, 3412, - 6176, 28963, 27531, 29764, 29196, 32535, 15967, 360, 27408, 29769, 33963, - 13647, 26952, 26953, 8379, 33768, 13141, 16004, 29954, 28960, 4964, 3516, - 4705, 15979, 6521, 10520, 7690, 10351, 10352, 24253, 29579, 15978, 15977, - 25236, 16361, 13032, 15980, 3401, 2522, 15973, 20528, 8171, 27705, 10269, - 13097, 16357, 16360, 13031, 34065, 3855, 33950, 33949, 27390, 3870, - 18037, 2600, 4102, 359, 12426, 12427, 12428, 12429, 12430, 26976, 23515, - 26091, 33942, 8370, 32318, 19676, 26978, 5765, 11054, 8391, 34118, 28957, - 28958, 15966, 26981, 13615, 27940, 23497, 27352, 6181, 10716, 26667, - 4252, 12395, 25234, 29191, 4643, 954, 15942, 18039, 13134, 32534, 3986, - 32564, 15201, 2588, 13197, 3741, 26497, 18034, 26821, 11056, 2598, 10778, - 23517, 8371, 32319, 26979, 5766, 11055, 29195, 15968, 23498, 10777, - 26669, 13200, 14652, 34183, 3515, 26267, 26668, 26486, 6180, 13059, - 13057, 11211, 24730, 23518, 32422, 34107, 34024, 34050, 34071, 13138, - 33951, 26087, 32046, 32047, 7688, 25781, 8393, 34173, 13058, 24573, - 29951, 16461, 11062, 18029, 3745, 34172, 26925, 14658, 26816, 20983, - 2519, 15833, 34174, 34171, 13166, 4701, 4700, 4250, 13510, 20896, 34169, - 13105, 20902, 32581, 32582, 26796, 34170, 4631, 26510, 20899, 20900, - 25760, 25758, 2587, 8159, 26878, 16366, 16365, 14521, 2590, 13045, 356, - 16131, 28858, 16247, 14278, 10227, 20434, 24184, 13093, 14525, 3410, - 29949, 26672, 18026, 20527, 27328, 13326, 18021, 4094, 8369, 33961, 3517, - 4637, 32575, 29171, 14276, 15236, 3988, 14265, 34208, 26924, 15235, - 27104, 15237, 10528, 12383, 952, 4249, 28870, 7756, 29192, 11058, 10232, - 26670, 13148, 10696, 29182, 32021, 34035, 15989, 23319, 9646, 15266, - 8376, 3403, 3405, 3404, 3402, 23318, 5994, 27730, 26520, 4632, 22904, - 13154, 25792, 11209, 13118, 25779, 26095, 26097, 4961, 31676, 5703, 5993, - 5995, 3408, 7694, 26927, 27396, 26488, 29591, 32504, 3999, 19777, 24727, - 24728, 7740, 25773, 14264, 3987, 25796, 4000, 24255, 27727, 22741, 31681, - 26098, 13104, 27614, 26928, 6000, 26073, 16356, 26487, 13060, 16163, - 12464, 7737, 7738, 25783, 25782, 26805, 26802, 24568, 22918, 22919, - 33686, 22920, 24648, 967, 4962, 4963, 33688, 31688, 26955, 33690, 13121, - 26800, 26892, 32568, 7724, 7725, 7721, 963, 20529, 15828, 29177, 29176, - 29178, 29179, 3509, 12381, 19566, 27215, 20492, 7739, 17144, 20489, - 25786, 3523, 3525, 3998, 20432, 26957, 2592, 12459, 25762, 29019, 32445, - 24647, 17159, 16250, 16251, 16254, 16253, 16252, 13123, 12382, 34187, - 14648, 25141, 15981, 26681, 22893, 31687, 8398, 28852, 16367, 32506, - 3880, 34083, 18148, 18130, 18138, 18131, 28887, 28886, 32658, 10981, - 32656, 10982, 20561, 32707, 6236, 8385, 8384, 24720, 24722, 29838, 34048, - 15281, 5831, 26089, 11049, 17142, 23504, 29859, 22604, 4096, 7705, 7715, - 7717, 7701, 7699, 7709, 7707, 7697, 7703, 7711, 7695, 7713, 7706, 7716, - 7718, 7702, 7700, 7710, 7708, 7698, 7704, 7712, 7696, 7714, 27173, 27174, - 27175, 4696, 4697, 27336, 3997, 5697, 20986, 3882, 24646, 15941, 20897, - 28873, 10229, 29187, 29188, 16462, 20901, 19490, 31682, 27156, 34102, - 3895, 31686, 7691, 2593, 29563, 12463, 13160, 26500, 20431, 3852, 20554, - 20540, 20552, 20553, 20558, 19545, 32555, 26967, 27089, 27097, 27098, - 27102, 27099, 26968, 34025, 28043, 28042, 28039, 28038, 3830, 3868, - 28045, 28044, 28041, 28040, 3898, 3802, 3815, 10353, 17146, 32037, 26875, - 26822, 3818, 34039, 28970, 31670, 34167, 25770, 32570, 32033, 32490, - 25585, 15229, 27711, 26876, 13103, 25793, 10702, 10703, 10704, 13190, - 13192, 13191, 3814, 13164, 25780, 5704, 5705, 13119, 12431, 12432, 12433, - 24724, 24725, 24726, 12442, 12435, 12436, 10701, 26094, 26099, 33957, - 29190, 29189, 10354, 22905, 22168, 26080, 7723, 5695, 16165, 10246, 8151, - 25757, 27347, 26096, 29589, 10226, 20433, 29180, 32029, 32028, 32030, - 32031, 19517, 27176, 32583, 32035, 19533, 27189, 19454, 27121, 23501, - 19802, 19801, 2734, 2740, 2735, 2739, 2732, 19799, 2733, 34178, 23514, - 32489, 29577, 28855, 23519, 14268, 14270, 13084, 28945, 28947, 28946, - 28948, 28944, 28943, 34165, 28949, 13064, 27060, 28942, 28952, 28955, - 24380, 13056, 32619, 13067, 26498, 8157, 8158, 18022, 13094, 18023, - 18024, 13081, 13082, 13083, 10613, 34179, 953, 26814, 8397, 26522, 13088, - 10612, 13199, 950, 13110, 33959, 28872, 32419, 14273, 20591, 13072, - 15988, 13074, 13065, 2514, 13155, 28871, 10697, 13091, 13069, 14272, - 5767, 28940, 28941, 5768, 18025, 26813, 8396, 33958, 28876, 28877, 32716, - 13087, 13077, 13071, 26519, 27735, 15239, 29175, 15200, 26517, 26516, - 26512, 26514, 24689, 29073, 24663, 29061, 32553, 32562, 32552, 32561, - 24688, 29072, 24662, 29060, 15317, 15310, 15314, 15307, 24690, 29074, - 24664, 29062, 15318, 15311, 15315, 15308, 15939, 15940, 29015, 29016, - 19669, 32750, 27291, 11044, 27721, 15312, 19781, 15289, 15244, 29767, - 27608, 27609, 27610, 15333, 27611, 15301, 33677, 33674, 5996, 27070, - 27346, 15474, 29580, 26960, 15831, 15832, 32497, 22740, 19788, 29578, - 32493, 32494, 4699, 25767, 32532, 4702, 22906, 361, 13124, 26794, 25765, - 31674, 25766, 2521, 25763, 26993, 10251, 2505, 32491, 23496, 23511, - 29765, 23512, 160, 27917, 27404, 29181, 15963, 33668, 8160, 26795, 32503, - 11050, 24644, 28956, 24649, 26926, 11048, 26807, 24653, 3735, 24642, - 3734, 23513, 26529, 24645, 6178, 22605, 34175, 27065, 2591, 32488, 33940, - 27939, 3507, 3508, 26428, 9651, 2604, 19491, 32500, 26889, 6345, 4251, - 13511, 8368, 28869, 27920, 3527, 3636, 26686, 25233, 27919, 29599, 26100, - 15935, 15991, 12396, 35762, 35762, 35762, 35762, 34168, 26772, 33965, - 27329, 14649, 27913, 25267, 23509, 26959, 23506, 32654, 32657, 32655, - 28884, 24697, 227, 228, 35762, 35762, 35762, 27613, 25764, 10538, 26425, - 27709, 23507, 5700, 28874, 13159, 28861, 2523, 26666, 27349, 35762, - 35762, 35762, 295, 243, 344, 342, 339, 237, 235, 236, 233, 234, 332, 334, + 29521, 29522, 29523, 29524, 29525, 29526, 29527, 29528, 29529, 29530, + 29531, 29532, 29533, 29534, 29535, 29536, 29537, 29538, 29539, 29540, + 29541, 29542, 29543, 29544, 29545, 29546, 29547, 29548, 29549, 29550, + 29551, 29552, 29553, 29554, 29555, 29556, 29557, 29558, 29559, 29560, + 29561, 29562, 29563, 29564, 29565, 29566, 29567, 29568, 29569, 29570, + 29571, 29572, 29573, 29574, 29575, 29576, 29577, 29578, 29579, 29580, + 29581, 29582, 29583, 29584, 29585, 29586, 29587, 29588, 29589, 29590, + 29591, 29592, 29593, 29594, 29595, 29596, 29597, 29598, 29599, 29600, + 29601, 29602, 29603, 29604, 29605, 29606, 29607, 29608, 29609, 29610, + 29611, 29612, 29613, 29614, 29615, 29616, 29617, 29618, 29619, 29620, + 29621, 29622, 29623, 29624, 29625, 29626, 29627, 29628, 29629, 29630, + 29631, 29632, 29633, 29634, 29635, 29636, 29637, 29638, 29639, 29640, + 29641, 29642, 29643, 29644, 29645, 29646, 29647, 29648, 29649, 29650, + 29651, 29652, 29653, 29654, 29655, 29656, 29657, 29658, 29659, 29660, + 29661, 29662, 29663, 29664, 29665, 29666, 29667, 29668, 29669, 29670, + 29671, 29672, 29673, 29674, 29675, 29676, 29677, 29678, 29679, 29680, + 29681, 29682, 29683, 29684, 29685, 29686, 29687, 29688, 29689, 29690, + 29691, 29692, 29693, 29694, 29695, 29696, 29697, 29698, 29699, 29700, + 29701, 29702, 29703, 29704, 29705, 29706, 29707, 29708, 29709, 29710, + 29711, 29712, 29713, 29714, 29715, 29716, 29717, 29718, 29719, 29720, + 29721, 29722, 29723, 29724, 29725, 29726, 29727, 29728, 29729, 29730, + 29731, 29732, 29733, 29734, 29735, 29736, 29737, 29738, 29739, 29740, + 29741, 29742, 29743, 29744, 29745, 29746, 29747, 29748, 29749, 29750, + 29751, 29752, 29753, 29754, 29755, 29756, 29757, 29758, 29759, 29760, + 29761, 29762, 29763, 29764, 29765, 29766, 29767, 29768, 29769, 29770, + 29771, 29772, 29773, 29774, 29775, 29776, 29777, 29778, 29779, 29780, + 29781, 29782, 29783, 29784, 29785, 29786, 29787, 29788, 29789, 29790, + 29791, 29792, 29793, 29794, 29795, 29796, 29797, 29798, 29799, 29800, + 29801, 29802, 29803, 29804, 29805, 29806, 29807, 29808, 29809, 29810, + 29811, 29812, 29813, 29814, 29815, 29816, 29817, 29818, 29819, 29820, + 29821, 29822, 29823, 29824, 29825, 29826, 29827, 29828, 29829, 29830, + 29831, 29832, 29833, 29834, 29835, 29836, 29837, 29838, 29839, 29840, + 29841, 29842, 29843, 29844, 29845, 29846, 29847, 29848, 29849, 29850, + 40951, 40951, 40951, 40951, 11516, 11514, 11463, 11496, 11423, 11436, + 11440, 11521, 11417, 11504, 11425, 11467, 11466, 11418, 11424, 11438, + 11470, 11499, 11492, 11419, 11439, 11493, 11517, 11443, 11471, 11444, + 11449, 11427, 11472, 11445, 11450, 11430, 11473, 11447, 11452, 11428, + 11429, 11481, 11482, 11448, 11453, 11434, 11485, 11446, 11451, 11431, + 11474, 11435, 11432, 11433, 11479, 11480, 11477, 11478, 11498, 11497, + 11506, 11512, 11509, 11484, 11483, 11437, 11426, 11475, 11476, 11415, + 11490, 11460, 11458, 11416, 11518, 11420, 11519, 11495, 11503, 11421, + 11487, 11468, 11486, 11441, 11520, 11500, 11422, 11515, 11501, 11442, + 11469, 11502, 11494, 11459, 11462, 11461, 11511, 11507, 11513, 11510, + 11508, 11457, 11456, 11455, 11454, 11465, 11464, 11488, 11491, 11489, + 11505, 40951, 40951, 40951, 40951, 40951, 11400, 11413, 11412, 11407, + 11414, 11394, 11387, 11386, 11383, 11385, 11388, 11389, 11384, 40951, + 40951, 40951, 11396, 11392, 11395, 11390, 11399, 11398, 11391, 11397, + 11393, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 11401, 11405, + 11408, 11403, 11411, 11410, 11404, 11409, 11406, 11402, 40951, 40951, + 11525, 11522, 11523, 11524, 32839, 32838, 32840, 32841, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 37793, 32041, 24171, 32038, 11290, 25171, 32009, 25229, 965, 20428, + 38848, 24129, 27577, 31994, 24164, 32031, 29926, 31588, 31779, 20424, + 38822, 25118, 25117, 25116, 25115, 25113, 25114, 4485, 4486, 4494, 4508, + 4378, 4377, 32579, 32587, 32580, 32591, 32584, 32588, 32581, 32593, + 32586, 32590, 32583, 32592, 32585, 32589, 32582, 37832, 37800, 37802, + 37887, 37849, 37845, 37881, 37851, 25255, 25202, 25222, 25257, 25205, + 25247, 25249, 25228, 34085, 34086, 30653, 10840, 10576, 10575, 34097, + 34098, 24177, 32018, 17483, 17484, 350, 349, 356, 355, 358, 357, 353, + 354, 351, 352, 24167, 37785, 32034, 11296, 37462, 37458, 37466, 4340, + 4338, 4344, 24160, 37780, 32028, 11292, 28300, 24170, 37788, 32037, + 11299, 16653, 16654, 3842, 3845, 3844, 3843, 3876, 24183, 37778, 32024, + 11302, 24184, 37779, 32025, 11303, 24173, 37782, 32040, 11294, 34330, + 34331, 34329, 34328, 34521, 34523, 34522, 34520, 38832, 20407, 39241, + 39242, 37705, 34162, 34161, 34160, 34115, 20802, 24043, 20803, 38837, + 20405, 24178, 32019, 24179, 32020, 17469, 24169, 37787, 32036, 11298, + 20427, 38847, 38825, 24172, 32039, 24166, 32033, 24168, 32035, 24077, + 31952, 37841, 37877, 37840, 37876, 25191, 25211, 25190, 25210, 25188, + 25208, 25189, 25209, 37844, 37880, 25201, 25221, 37842, 37878, 25200, + 25220, 37835, 37871, 25195, 25215, 37838, 37874, 25198, 25218, 37839, + 37875, 25199, 25219, 37834, 37870, 25194, 25214, 37836, 37872, 25196, + 25216, 37837, 37873, 25197, 25217, 37843, 37879, 25192, 25212, 30832, + 30833, 30834, 30835, 30836, 30837, 30838, 30839, 30840, 30841, 30842, + 30843, 30844, 30845, 30846, 30847, 30848, 30849, 30850, 30851, 30852, + 30853, 30854, 30855, 30856, 30857, 30868, 30870, 30867, 30866, 30863, + 30862, 30865, 30864, 30871, 30869, 40951, 40951, 40951, 40951, 40951, + 40951, 4137, 4081, 3952, 4167, 4038, 3979, 4138, 4019, 4082, 4128, 4054, + 4113, 3995, 4010, 4098, 3964, 4171, 4039, 4069, 3980, 4139, 4020, 4083, + 3953, 4134, 4062, 4121, 4003, 4160, 4016, 4106, 3972, 4047, 4075, 3988, + 4146, 4028, 4091, 3958, 4129, 4055, 4114, 3996, 4153, 4011, 4099, 3965, + 4172, 4040, 4070, 3981, 4140, 4021, 4084, 4066, 4125, 4007, 4164, 4035, + 4110, 3976, 4179, 4051, 4078, 3992, 4150, 4032, 4095, 3961, 4059, 4118, + 4000, 4157, 4103, 3969, 4176, 4044, 3985, 4143, 4025, 4088, 4135, 4063, + 4122, 4004, 4161, 4017, 4107, 3973, 4168, 4048, 4076, 3989, 4147, 4029, + 4092, 3959, 4130, 4056, 4115, 3997, 4154, 4012, 4100, 3966, 4173, 4041, + 4071, 3982, 4141, 4022, 4085, 3954, 4068, 4127, 4009, 4166, 4037, 4112, + 3978, 4181, 4053, 4080, 3994, 4152, 4034, 4097, 3963, 4133, 4061, 4120, + 4002, 4159, 4015, 4105, 3971, 4178, 4046, 4074, 3987, 4145, 4027, 4090, + 3957, 4065, 4124, 4006, 4163, 4109, 3975, 4170, 4050, 3991, 4149, 4031, + 4094, 4131, 4058, 4117, 3999, 4156, 4013, 4102, 3968, 4175, 4043, 4072, + 3984, 4142, 4024, 4087, 3955, 4067, 4126, 4008, 4165, 4036, 4111, 3977, + 4180, 4052, 4079, 3993, 4151, 4033, 4096, 3962, 4132, 4060, 4119, 4001, + 4158, 4014, 4104, 3970, 4177, 4045, 4073, 3986, 4144, 4026, 4089, 3956, + 4136, 4064, 4123, 4005, 4162, 4018, 4108, 3974, 4169, 4049, 4077, 3990, + 4148, 4030, 4093, 3960, 4057, 4116, 3998, 4155, 4101, 3967, 4174, 4042, + 3983, 4023, 4086, 37470, 4347, 37467, 4345, 37468, 4346, 37463, 4341, + 37464, 4342, 37457, 4334, 4335, 4336, 4337, 28184, 37459, 37460, 11293, + 24161, 33837, 37783, 11295, 17356, 17357, 17358, 31947, 25162, 17359, + 37807, 25164, 19784, 39303, 37481, 17638, 4379, 4376, 24081, 31956, + 24080, 31955, 20404, 24078, 31953, 20403, 25165, 37810, 38838, 4504, + 4502, 4503, 4501, 22718, 22717, 22722, 22716, 22694, 22673, 22674, 22714, + 22680, 22698, 22721, 22696, 22719, 22720, 22702, 22699, 22679, 22678, + 22681, 22697, 22682, 22670, 22691, 22715, 22671, 22672, 22669, 22695, + 22700, 22686, 22677, 22701, 22703, 22676, 22693, 22685, 22683, 22684, + 22675, 22723, 22692, 22687, 22690, 22688, 22689, 22711, 22709, 22710, + 22713, 22708, 22705, 22712, 22707, 22706, 22704, 32594, 32626, 32595, + 32642, 32611, 32627, 32596, 32650, 32619, 32635, 32604, 32643, 32612, + 32628, 32597, 32654, 32623, 32639, 32608, 32647, 32616, 32632, 32601, + 32651, 32620, 32636, 32605, 32644, 32613, 32629, 32598, 32656, 32625, + 32641, 32610, 32649, 32618, 32634, 32603, 32653, 32622, 32638, 32607, + 32646, 32615, 32631, 32600, 32655, 32624, 32640, 32609, 32648, 32617, + 32633, 32602, 32652, 32621, 32637, 32606, 32645, 32614, 32630, 32599, + 37827, 37799, 37801, 37866, 37848, 37846, 37847, 37850, 25254, 25252, + 25253, 25256, 25207, 25246, 25248, 25242, 31957, 31997, 24132, 24082, + 25167, 25262, 37890, 37812, 24083, 24133, 31998, 31958, 37813, 37891, + 25263, 25168, 20429, 21627, 30342, 3882, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40767, 40772, 40806, 40769, 40774, 40802, 40796, 40790, + 40813, 40792, 40786, 40810, 40768, 40773, 40807, 40770, 40775, 40803, + 40797, 40791, 40814, 40793, 40787, 40811, 40808, 40794, 40801, 40788, + 40789, 40812, 40795, 40771, 40817, 40782, 40799, 40809, 40820, 40819, + 40785, 40818, 40779, 40778, 40816, 40800, 40798, 40777, 40951, 40951, + 40822, 40823, 40824, 40815, 40765, 40780, 40783, 40784, 40762, 40763, + 40781, 40804, 40805, 40766, 40821, 40764, 40776, 40761, 40943, 40944, + 40941, 40942, 40945, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40841, 40842, 40858, 40830, 40839, 40938, 40893, 40894, + 40861, 40862, 40895, 40825, 40859, 40939, 40837, 40834, 40836, 40835, + 40833, 40935, 40934, 40937, 40936, 40931, 40930, 40933, 40932, 40831, + 40865, 40827, 40838, 40826, 40863, 40876, 40877, 40875, 40874, 40828, + 40872, 40873, 40871, 40869, 40870, 40868, 40867, 40878, 40880, 40881, + 40879, 40843, 40866, 40832, 40840, 40940, 40882, 40887, 40884, 40888, + 40885, 40890, 40891, 40886, 40883, 40889, 40864, 40892, 40925, 40922, + 40913, 40923, 40924, 40914, 40902, 40901, 40929, 40899, 40898, 40896, + 40900, 40897, 40916, 40921, 40915, 40919, 40920, 40917, 40918, 40845, + 40849, 40848, 40847, 40846, 40928, 40926, 40927, 40852, 40857, 40856, + 40855, 40854, 40853, 40903, 40912, 40905, 40910, 40904, 40908, 40909, + 40906, 40907, 40911, 40844, 40851, 40829, 40850, 40860, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 5194, 5066, 5187, + 5169, 5170, 5238, 5239, 5119, 5216, 5176, 5250, 5251, 5142, 5021, 5072, + 5219, 5120, 5024, 5025, 5214, 5226, 5165, 5102, 5193, 5045, 5242, 5114, + 5128, 5124, 5217, 5179, 5208, 5172, 5241, 5027, 5029, 5129, 5195, 5180, + 5237, 5019, 5197, 5211, 5213, 5167, 5221, 5149, 5067, 5229, 5220, 5139, + 5020, 5079, 5110, 5231, 5117, 5185, 5189, 5132, 5043, 5196, 5174, 5177, + 5116, 5254, 5184, 5133, 5232, 5209, 5108, 5115, 5166, 5171, 5183, 5135, + 5182, 5140, 5186, 5123, 5127, 5253, 5026, 5022, 5252, 5141, 5075, 5046, + 5164, 5240, 5181, 5190, 5173, 5018, 5150, 5178, 5175, 5071, 5143, 5017, + 5233, 5074, 5212, 5215, 5044, 5073, 5198, 5255, 5235, 5191, 5230, 5234, + 5188, 5236, 5192, 5105, 5031, 5070, 5168, 5223, 5224, 5222, 5225, 5118, + 5068, 5243, 5244, 5207, 5131, 5064, 5136, 5137, 5138, 5028, 5030, 5063, + 5228, 5218, 5134, 5144, 5148, 5147, 5146, 5145, 5104, 5101, 5100, 5058, + 5060, 5061, 5059, 5227, 5032, 5112, 5051, 5015, 5009, 5010, 5013, 5014, + 5012, 5011, 5016, 5157, 5152, 5153, 5151, 5162, 5161, 5160, 5159, 5163, + 5155, 5113, 5023, 5078, 5077, 5076, 5158, 5156, 5154, 5106, 5107, 5069, + 5111, 5109, 5080, 5087, 5083, 5091, 5086, 5095, 5085, 5084, 5081, 5082, + 5089, 5090, 5097, 5094, 5092, 5041, 5042, 5040, 5088, 5096, 5249, 5054, + 5052, 5055, 5057, 5056, 5053, 5245, 5247, 5246, 5248, 5098, 5099, 5048, + 5047, 5049, 5050, 5203, 5206, 5204, 5205, 5199, 5202, 5200, 5201, 5062, + 5065, 5210, 5039, 5033, 5038, 5036, 5035, 5034, 5037, 5121, 5125, 5122, + 5126, 5130, 5103, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 28489, 28366, 28382, 28474, 28361, 28486, 28424, 28475, + 28473, 28362, 28358, 28485, 28352, 28470, 28471, 28472, 28377, 28378, + 28315, 28316, 28311, 28310, 28441, 28512, 28509, 28384, 28383, 28490, + 28491, 28385, 28394, 28395, 28396, 28355, 28387, 28388, 28389, 28368, + 28369, 40951, 40951, 28432, 28365, 28367, 28393, 28392, 28437, 28436, + 28488, 28487, 28464, 28465, 28351, 28357, 28453, 28454, 28468, 28469, + 28433, 28535, 28407, 28467, 28376, 28493, 28511, 28495, 28440, 28533, + 28456, 28356, 28496, 28497, 28520, 28521, 28524, 28525, 28522, 28523, + 28516, 28517, 28518, 28519, 28430, 28431, 28526, 28527, 28455, 28531, + 28438, 28435, 28319, 28320, 28314, 28534, 28406, 28466, 28375, 28492, + 28510, 28494, 28439, 28340, 28341, 28344, 28345, 28346, 28379, 28380, + 28381, 28323, 28330, 28331, 28332, 28333, 28334, 28308, 28373, 28309, + 28374, 28307, 28371, 28306, 28370, 28321, 28339, 28347, 28338, 28324, + 28325, 28322, 28349, 28404, 28403, 28328, 28350, 28335, 28342, 28348, + 28327, 28343, 28476, 28499, 28538, 28463, 28426, 28386, 28354, 28363, + 28400, 28399, 28515, 28528, 28537, 28529, 28530, 28442, 28445, 28446, + 28447, 28448, 28449, 28450, 28451, 28452, 28443, 28444, 28408, 28434, + 28372, 28364, 28326, 28329, 28336, 28337, 28458, 28457, 28405, 28398, + 28397, 28536, 28359, 28360, 28425, 28421, 28312, 28479, 28481, 28427, + 28429, 28482, 28484, 28390, 28391, 28423, 28422, 28313, 28480, 28428, + 28483, 28507, 28506, 28508, 28505, 28501, 28502, 28503, 28504, 28353, + 28401, 28402, 28498, 28532, 28460, 28318, 28477, 28317, 28513, 28461, + 28462, 28478, 28514, 28459, 28409, 28412, 28416, 28419, 28418, 28417, + 28413, 28414, 28410, 28411, 28415, 28500, 28420, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 18636, 18624, + 18647, 18648, 18630, 18649, 18650, 18651, 18652, 18637, 18638, 18639, + 18640, 18641, 18642, 18643, 18644, 18645, 18646, 18625, 18626, 18627, + 18628, 18629, 18631, 18632, 18633, 18634, 18635, 18356, 18364, 18377, + 18385, 18391, 18392, 18357, 18358, 18359, 18360, 18361, 18362, 18363, + 18365, 18366, 18367, 18368, 18369, 18370, 18371, 18372, 18373, 18374, + 18375, 18376, 18378, 18379, 18380, 18381, 18382, 18383, 18384, 18386, + 18387, 18388, 18389, 18390, 8271, 8270, 8272, 18416, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 21607, + 21608, 21605, 21603, 21594, 21593, 21600, 21598, 21589, 21596, 21606, + 21591, 21604, 21602, 21595, 21592, 21601, 21599, 21590, 21597, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 26844, 26845, 26842, 26840, 26831, 26830, 26837, 26835, 26826, + 26833, 26843, 26828, 26841, 26839, 26832, 26829, 26838, 26836, 26827, + 26834, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 28177, 10885, 10886, 10880, 10879, 10878, 36785, + 36805, 36826, 36774, 36819, 36783, 36769, 36828, 36773, 36787, 36793, + 36816, 36817, 36831, 36837, 36784, 36815, 36845, 36803, 36770, 36833, + 36835, 36802, 36848, 36782, 36797, 36794, 36775, 36789, 36772, 36830, + 36823, 36777, 36820, 36809, 36843, 36832, 36806, 36834, 36822, 36836, + 36811, 36799, 36842, 36812, 36798, 36829, 36838, 36808, 36844, 36781, + 36825, 36801, 36847, 36791, 36776, 36813, 36810, 36824, 36768, 36796, + 36795, 36846, 36840, 36818, 36788, 36786, 36792, 36800, 36839, 36841, + 36814, 36780, 36778, 36807, 36771, 36779, 36827, 36790, 36821, 36804, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 8669, + 8667, 8666, 8663, 8662, 8665, 8664, 8670, 8668, 8660, 8658, 8657, 8654, + 8653, 8656, 8655, 8661, 8659, 20515, 20514, 20513, 20512, 20511, 35370, + 35369, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 25824, 25826, 25854, 25817, + 25830, 25862, 25833, 25863, 25835, 25864, 25837, 25839, 25856, 25858, + 25841, 25844, 25865, 25848, 25850, 25820, 25852, 25866, 25867, 25860, + 25868, 25828, 25872, 25874, 25907, 25869, 25878, 25881, 25883, 25915, + 25885, 25916, 25887, 25889, 25909, 25911, 25891, 25894, 25917, 25898, + 25900, 25902, 25905, 25918, 25919, 25913, 25920, 25876, 26312, 26314, + 26344, 26318, 26320, 26352, 26323, 26353, 26325, 26354, 26327, 26329, + 26346, 26348, 26331, 26334, 26355, 26338, 26340, 26308, 26342, 26356, + 26357, 26350, 26358, 26316, 26260, 26262, 26295, 26256, 26266, 26269, + 26271, 40951, 26273, 26303, 26275, 26277, 26297, 26299, 26279, 26282, + 26304, 26286, 26288, 26290, 26293, 26305, 26306, 26301, 26307, 26264, + 25977, 25979, 26009, 25983, 25985, 26017, 25988, 26018, 25990, 26019, + 25992, 25994, 26011, 26013, 25996, 25999, 26020, 26003, 26005, 25973, + 26007, 26021, 26022, 26015, 26023, 25981, 26031, 26033, 26068, 26037, + 26039, 26042, 26044, 26076, 26046, 26077, 26048, 26050, 26070, 26072, + 26052, 26055, 26078, 26059, 26061, 26063, 26066, 26079, 26080, 26074, + 26081, 26035, 26784, 40951, 26785, 26786, 40951, 40951, 26787, 40951, + 40951, 26788, 26789, 40951, 40951, 26790, 26791, 26792, 26793, 40951, + 26794, 26795, 26796, 26797, 26798, 26799, 26800, 26801, 26802, 26803, + 26804, 26805, 40951, 26806, 40951, 26807, 26808, 26809, 26810, 26811, + 26812, 26813, 40951, 26814, 26815, 26816, 26817, 26818, 26819, 26820, + 26821, 26822, 26823, 26824, 25921, 25922, 25923, 25924, 25925, 25926, + 25927, 25928, 25929, 25930, 25931, 25932, 25933, 25934, 25935, 25936, + 25937, 25938, 25939, 25940, 25941, 25942, 25943, 25944, 25945, 25946, + 25947, 25948, 25949, 25950, 25951, 25952, 25953, 25954, 25955, 25956, + 25957, 25958, 25959, 25960, 25961, 25962, 25963, 25964, 25965, 25966, + 25967, 25968, 25969, 25970, 25971, 25972, 26208, 26209, 40951, 26210, + 26211, 26212, 26213, 40951, 40951, 26214, 26215, 26216, 26217, 26218, + 26219, 26220, 26221, 40951, 26222, 26223, 26224, 26225, 26226, 26227, + 26228, 40951, 26229, 26230, 26231, 26232, 26233, 26234, 26235, 26236, + 26237, 26238, 26239, 26240, 26241, 26242, 26243, 26244, 26245, 26246, + 26247, 26248, 26249, 26250, 26251, 26252, 26253, 26254, 26153, 26154, + 40951, 26155, 26156, 26157, 26158, 40951, 26159, 26160, 26161, 26162, + 26163, 40951, 26164, 40951, 40951, 40951, 26165, 26166, 26167, 26168, + 26169, 26170, 26171, 40951, 26172, 26173, 26174, 26175, 26176, 26177, + 26178, 26179, 26180, 26181, 26182, 26183, 26184, 26185, 26186, 26187, + 26188, 26189, 26190, 26191, 26192, 26193, 26194, 26195, 26196, 26197, + 26091, 26092, 26093, 26094, 26095, 26096, 26097, 26098, 26099, 26100, + 26101, 26102, 26103, 26104, 26105, 26106, 26107, 26108, 26109, 26110, + 26111, 26112, 26113, 26114, 26115, 26116, 26117, 26118, 26119, 26120, + 26121, 26122, 26123, 26124, 26125, 26126, 26127, 26128, 26129, 26130, + 26131, 26132, 26133, 26134, 26135, 26136, 26137, 26138, 26139, 26140, + 26141, 26142, 26722, 26723, 26724, 26725, 26726, 26727, 26728, 26729, + 26730, 26731, 26732, 26733, 26734, 26735, 26736, 26737, 26738, 26739, + 26740, 26741, 26742, 26743, 26744, 26745, 26746, 26747, 26748, 26749, + 26750, 26751, 26752, 26753, 26754, 26755, 26756, 26757, 26758, 26759, + 26760, 26761, 26762, 26763, 26764, 26765, 26766, 26767, 26768, 26769, + 26770, 26771, 26772, 26773, 26554, 26556, 26586, 26560, 26562, 26594, + 26565, 26595, 26567, 26596, 26569, 26571, 26588, 26590, 26573, 26576, + 26597, 26580, 26582, 26550, 26584, 26598, 26599, 26592, 26600, 26558, + 26608, 26610, 26645, 26614, 26616, 26619, 26621, 26653, 26623, 26654, + 26625, 26627, 26647, 26649, 26629, 26632, 26655, 26636, 26638, 26640, + 26643, 26656, 26657, 26651, 26658, 26612, 26670, 26671, 26672, 26673, + 26674, 26675, 26676, 26677, 26678, 26679, 26680, 26681, 26682, 26683, + 26684, 26685, 26686, 26687, 26688, 26689, 26690, 26691, 26692, 26693, + 26694, 26695, 26696, 26697, 26698, 26699, 26700, 26701, 26702, 26703, + 26704, 26705, 26706, 26707, 26708, 26709, 26710, 26711, 26712, 26713, + 26714, 26715, 26716, 26717, 26718, 26719, 26720, 26721, 26444, 26446, + 26476, 26450, 26452, 26484, 26455, 26485, 26457, 26486, 26459, 26461, + 26478, 26480, 26463, 26466, 26487, 26470, 26472, 26440, 26474, 26488, + 26489, 26482, 26490, 26448, 26498, 26500, 26535, 26504, 26506, 26509, + 26511, 26543, 26513, 26544, 26515, 26517, 26537, 26539, 26519, 26522, + 26545, 26526, 26528, 26530, 26533, 26546, 26547, 26541, 26548, 26502, + 26367, 26368, 26369, 26370, 26371, 26372, 26373, 26374, 26375, 26376, + 26377, 26378, 26379, 26380, 26381, 26382, 26383, 26384, 26385, 26386, + 26387, 26388, 26389, 26390, 26391, 26392, 26393, 26394, 26395, 26396, + 26397, 26398, 26399, 26400, 26401, 26402, 26403, 26404, 26405, 26406, + 26407, 26408, 26409, 26410, 26411, 26412, 26413, 26414, 26415, 26416, + 26417, 26418, 26257, 26258, 40951, 40951, 25825, 25827, 25834, 25819, + 25831, 25829, 25832, 25821, 25836, 25838, 25840, 25857, 25859, 25861, + 25842, 25847, 25849, 25822, 25851, 25823, 25853, 25845, 25855, 25846, + 25843, 26085, 25873, 25875, 25884, 25871, 25879, 25877, 25880, 25903, + 25886, 25888, 25890, 25910, 25912, 25914, 25892, 25897, 25899, 25882, + 25901, 25904, 25906, 25895, 25908, 25896, 25893, 26087, 26083, 26090, + 26084, 26086, 26089, 26088, 26313, 26315, 26324, 26319, 26321, 26317, + 26322, 26309, 26326, 26328, 26330, 26347, 26349, 26351, 26332, 26337, + 26339, 26310, 26341, 26311, 26343, 26335, 26345, 26336, 26333, 26361, + 26261, 26263, 26272, 26259, 26267, 26265, 26268, 26291, 26274, 26276, + 26278, 26298, 26300, 26302, 26280, 26285, 26287, 26270, 26289, 26292, + 26294, 26283, 26296, 26284, 26281, 26363, 26359, 26366, 26360, 26362, + 26365, 26364, 25978, 25980, 25989, 25984, 25986, 25982, 25987, 25974, + 25991, 25993, 25995, 26012, 26014, 26016, 25997, 26002, 26004, 25975, + 26006, 25976, 26008, 26000, 26010, 26001, 25998, 26026, 26032, 26034, + 26045, 26038, 26040, 26036, 26041, 26064, 26047, 26049, 26051, 26071, + 26073, 26075, 26053, 26058, 26060, 26043, 26062, 26065, 26067, 26056, + 26069, 26057, 26054, 26028, 26024, 26082, 26025, 26027, 26030, 26029, + 26555, 26557, 26566, 26561, 26563, 26559, 26564, 26551, 26568, 26570, + 26572, 26589, 26591, 26593, 26574, 26579, 26581, 26552, 26583, 26553, + 26585, 26577, 26587, 26578, 26575, 26603, 26609, 26611, 26622, 26615, + 26617, 26613, 26618, 26641, 26624, 26626, 26628, 26648, 26650, 26652, + 26630, 26635, 26637, 26620, 26639, 26642, 26644, 26633, 26646, 26634, + 26631, 26605, 26601, 26659, 26602, 26604, 26607, 26606, 26445, 26447, + 26456, 26451, 26453, 26449, 26454, 26441, 26458, 26460, 26462, 26479, + 26481, 26483, 26464, 26469, 26471, 26442, 26473, 26443, 26475, 26467, + 26477, 26468, 26465, 26493, 26499, 26501, 26512, 26505, 26507, 26503, + 26508, 26531, 26514, 26516, 26518, 26538, 26540, 26542, 26520, 26525, + 26527, 26510, 26529, 26532, 26534, 26523, 26536, 26524, 26521, 26495, + 26491, 26549, 26492, 26494, 26497, 26496, 25818, 25870, 40951, 40951, + 26149, 26151, 26148, 26147, 26144, 26143, 26146, 26145, 26152, 26150, + 26204, 26206, 26203, 26202, 26199, 26198, 26201, 26200, 26207, 26205, + 26780, 26782, 26779, 26778, 26775, 26774, 26777, 26776, 26783, 26781, + 26666, 26668, 26665, 26664, 26661, 26660, 26663, 26662, 26669, 26667, + 26425, 26427, 26424, 26423, 26420, 26419, 26422, 26421, 26428, 26426, + 33092, 33048, 33073, 33289, 33244, 33030, 33093, 33057, 33206, 33158, + 33134, 33095, 33098, 33055, 33099, 33049, 33100, 33122, 33117, 33155, + 33096, 33102, 33111, 33112, 33103, 33109, 33113, 33051, 33185, 33094, + 33123, 33052, 33132, 33101, 33131, 33118, 33157, 33156, 33097, 33116, + 33126, 33127, 33130, 33129, 33197, 33105, 33106, 33107, 33179, 33147, + 33110, 33114, 33108, 33104, 33178, 33139, 33177, 33176, 33136, 33135, + 33138, 33128, 33125, 33124, 33174, 33173, 33175, 33146, 33222, 33226, + 33225, 33223, 33224, 33067, 33213, 33243, 33215, 33228, 33220, 33229, + 33221, 33230, 33219, 33077, 33078, 33242, 33283, 33216, 33217, 33218, + 33214, 33240, 33227, 33238, 33231, 33239, 33237, 33235, 33232, 33233, + 33234, 33236, 33064, 33070, 33068, 33069, 33273, 33272, 33080, 33072, + 33083, 33086, 33081, 33084, 33082, 33085, 33090, 33087, 33047, 33282, + 33288, 33285, 33287, 33261, 33263, 33241, 33271, 33264, 33268, 33262, + 33270, 33269, 33267, 33029, 33119, 33054, 33246, 33032, 33255, 33121, + 33120, 33247, 33160, 33163, 33162, 33161, 33170, 33210, 33059, 33284, + 33041, 33166, 33169, 33164, 33165, 33258, 33168, 33257, 33040, 33039, + 33167, 33058, 33256, 33038, 33133, 33053, 33248, 33265, 33031, 33115, + 33050, 33186, 33266, 33042, 33194, 33190, 33192, 33063, 33286, 33043, + 33187, 33189, 33188, 33193, 33191, 33281, 33159, 33056, 33088, 33275, + 33276, 33274, 33076, 33254, 33034, 33033, 33183, 33259, 33181, 33062, + 33171, 33182, 33280, 33180, 33184, 33172, 33060, 33089, 33079, 33260, + 33045, 33046, 33044, 33061, 33065, 33066, 33278, 33279, 33277, 33245, + 33148, 33250, 33151, 33152, 33153, 33150, 33154, 33149, 33143, 33144, + 33145, 33142, 33140, 33071, 33141, 33137, 33074, 33075, 33252, 33253, + 33249, 33251, 33036, 33037, 33035, 33195, 33200, 33203, 33204, 33205, + 33211, 33196, 33198, 33199, 33207, 33202, 33208, 33209, 33201, 33091, + 33212, 33603, 33602, 33601, 33623, 33622, 33621, 33578, 33577, 33576, + 32962, 32961, 32960, 33564, 33563, 33562, 33574, 33575, 33570, 33573, + 33569, 33572, 33571, 33019, 33022, 33018, 33021, 33020, 33568, 33438, + 33439, 33440, 33441, 33436, 33437, 33442, 33482, 33387, 33500, 33499, + 33497, 33498, 33501, 33476, 33479, 33477, 33478, 33475, 33506, 33505, + 33503, 33504, 33452, 33451, 33450, 33456, 33455, 33454, 33453, 33474, + 33473, 33472, 33449, 33448, 33447, 33522, 33521, 33520, 33496, 33495, + 33494, 33619, 33618, 33617, 33616, 33615, 33614, 33620, 33613, 33612, + 33611, 33356, 33355, 33353, 33354, 33360, 33359, 33357, 33358, 33348, + 33347, 33345, 33346, 33352, 33351, 33349, 33350, 33410, 33409, 33407, + 33408, 33411, 33425, 33428, 33426, 33427, 33384, 33415, 33414, 33413, + 33412, 33370, 33383, 33382, 33381, 33371, 33369, 33368, 33367, 33434, + 33433, 33432, 33431, 33430, 33429, 33606, 33605, 33604, 33609, 33608, + 33607, 33610, 33469, 33468, 33466, 33467, 33460, 33459, 33457, 33458, + 33465, 33464, 33487, 33486, 33483, 33488, 33493, 33490, 33489, 33509, + 33508, 33507, 33512, 33511, 33510, 33463, 33471, 33470, 33559, 33556, + 33555, 33502, 33462, 33485, 33492, 33517, 33561, 33558, 33554, 33461, + 33484, 33491, 33516, 33560, 33557, 33553, 33515, 33514, 33513, 33374, + 33373, 33391, 33389, 33390, 33388, 33400, 33398, 33399, 33397, 33417, + 33416, 33548, 33545, 33551, 33376, 33375, 33392, 33393, 33395, 33394, + 33404, 33402, 33403, 33401, 33419, 33418, 33549, 33546, 33552, 33380, + 33379, 33377, 33378, 33372, 33396, 33405, 33420, 33421, 33422, 33547, + 33544, 33550, 33406, 33446, 33444, 33445, 33443, 33366, 33364, 33362, + 33365, 33363, 33361, 33519, 33518, 33424, 33423, 33481, 33480, 33386, + 33385, 32978, 32977, 32973, 32976, 32980, 32979, 32974, 32975, 32972, + 32981, 33291, 33298, 33296, 33295, 33293, 33294, 33292, 33297, 33011, + 33009, 33010, 32987, 32986, 32985, 32970, 32968, 32969, 32971, 33026, + 33025, 33024, 33005, 33006, 33002, 32983, 32982, 33001, 33003, 33000, + 33004, 32984, 32999, 32997, 32998, 32994, 32996, 32995, 32988, 32990, + 32989, 32992, 32991, 32993, 32965, 32964, 32963, 33588, 33587, 33589, + 33008, 33525, 33524, 33526, 33527, 32955, 32957, 32954, 32956, 32959, + 32958, 33320, 33318, 33319, 33325, 33327, 33326, 33322, 33324, 33323, + 33339, 33338, 33337, 33331, 33332, 33333, 33334, 33335, 33336, 33328, + 33330, 33329, 33340, 33341, 33342, 33309, 33307, 33308, 33321, 33344, + 33343, 33596, 33595, 33593, 33594, 33592, 33597, 33591, 33590, 33580, + 33585, 33583, 33584, 33581, 33582, 33586, 33523, 33435, 33528, 33290, + 33007, 33566, 33565, 33028, 33023, 33567, 33599, 33598, 33600, 33624, + 33299, 33300, 33301, 33302, 33303, 33304, 33305, 33306, 33017, 33317, + 33316, 33311, 33314, 33313, 33310, 33312, 33315, 32967, 33027, 33579, + 32966, 33625, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 33012, 33013, 33014, + 33015, 33016, 40951, 33536, 33537, 33538, 33539, 33540, 33541, 33542, + 33543, 33529, 33530, 33531, 33532, 33533, 33534, 33535, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 23497, 23771, + 23262, 23776, 23249, 23621, 23882, 23773, 23868, 23837, 23242, 23476, + 23477, 23872, 23237, 23289, 23263, 23614, 23413, 23594, 23474, 23866, + 23752, 23841, 23485, 23414, 23558, 23711, 23842, 23380, 23788, 40951, + 40951, 40951, 40951, 40951, 40951, 23404, 23607, 23653, 23757, 23796, + 23832, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 8227, 8229, 8239, 8233, 8215, 8240, 8247, + 40951, 8246, 8220, 8217, 8216, 8214, 8251, 8235, 8234, 8236, 8250, 8237, + 8238, 8222, 8225, 8249, 8231, 8248, 40951, 40951, 8223, 8226, 8230, 8224, + 8242, 8241, 8243, 40951, 8245, 8221, 40951, 8244, 8219, 8228, 8218, 8232, + 40951, 40951, 40951, 40951, 40951, 27764, 27733, 27761, 27759, 27735, + 27757, 27754, 27755, 27756, 27763, 27740, 27741, 27765, 27744, 27742, + 27737, 27750, 27766, 27739, 27762, 27749, 27758, 27748, 27751, 27736, + 27753, 27734, 27747, 27731, 27760, 27732, 27745, 27743, 10520, 10498, + 10518, 10505, 10501, 10515, 10512, 10513, 10514, 10519, 10503, 10521, + 10517, 10504, 10522, 10502, 10507, 10511, 10516, 10510, 10508, 10509, + 10506, 10497, 10500, 10499, 27738, 27752, 27746, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 8144, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 29882, 29866, 29857, 29868, 29873, 29865, 29870, + 29861, 29887, 29891, 29893, 29896, 29860, 29855, 29890, 29880, 29864, + 29863, 29894, 29856, 29867, 29888, 29876, 29892, 29895, 29862, 29884, + 29869, 29859, 29879, 29858, 29874, 29881, 29883, 29889, 29875, 29871, + 29872, 29897, 29898, 29877, 29878, 29885, 29886, 29899, 40951, 40951, + 40951, 29908, 29912, 29911, 29914, 29913, 29910, 29909, 29905, 29902, + 29901, 29904, 29903, 29906, 29907, 40951, 40951, 29922, 29924, 29921, + 29920, 29917, 29916, 29919, 29918, 29925, 29923, 40951, 40951, 40951, + 40951, 29900, 29915, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 37522, 37505, 37525, 37515, 37519, 37516, 37521, + 37507, 37506, 37524, 37512, 37527, 37526, 37518, 37517, 37523, 37520, + 37508, 37500, 37509, 37501, 37529, 37510, 37502, 37511, 37503, 37528, + 37514, 37504, 37513, 37530, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 38979, 38978, 39010, 39011, 39012, 39014, 39003, 39006, 38992, + 38995, 39007, 38999, 38996, 39013, 39009, 39008, 39016, 39021, 39020, + 39019, 39005, 38984, 38983, 39018, 39017, 39004, 39015, 38987, 38989, + 38993, 39000, 38991, 38998, 38997, 38986, 38981, 38982, 38990, 38985, + 38988, 38980, 38994, 39001, 39002, 39025, 39026, 39023, 39024, 39033, + 39035, 39032, 39031, 39028, 39027, 39030, 39029, 39036, 39034, 40951, + 40951, 40951, 40951, 40951, 39022, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 28864, 28867, 28866, 28868, 28865, 28847, 28851, + 28849, 28848, 28850, 28859, 28862, 28860, 28863, 28861, 28869, 28870, + 28871, 28872, 28873, 28852, 28854, 28857, 28858, 28853, 28856, 28855, + 28877, 28874, 28878, 28876, 28875, 28885, 28887, 28884, 28883, 28880, + 28879, 28882, 28881, 28888, 28886, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30046, + 30049, 30048, 30047, 30050, 30051, 30028, 30030, 30029, 30031, 30032, + 30033, 30040, 30041, 30045, 30042, 30043, 30044, 30052, 30056, 30053, + 30055, 30054, 30057, 30034, 30039, 30038, 30036, 30035, 30037, 30060, + 30058, 30059, 30068, 30070, 30067, 30066, 30063, 30062, 30065, 30064, + 30071, 30069, 40951, 40951, 40951, 40951, 30061, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 17006, 17012, 17010, 17007, 17009, 17008, 17011, 40951, 16961, + 17005, 17003, 17002, 40951, 16949, 16948, 40951, 16960, 16959, 16958, + 16945, 16944, 16957, 16956, 16955, 16954, 16953, 16952, 16947, 16946, + 16951, 16950, 40951, 27076, 27077, 27078, 27144, 27165, 27153, 27118, + 27253, 27079, 27080, 27084, 27203, 27188, 27193, 27117, 27271, 27220, + 27142, 27123, 27214, 27083, 27081, 27082, 27129, 27173, 27226, 27266, + 27089, 27085, 27093, 27230, 27168, 27178, 27205, 27094, 27091, 27090, + 27243, 27181, 27241, 27219, 27210, 27208, 27209, 27272, 27251, 27088, + 27103, 27095, 27242, 27187, 27212, 27147, 27273, 27099, 27100, 27101, + 27157, 27149, 27133, 27234, 27185, 27086, 27092, 27087, 27160, 27257, + 27258, 27096, 27097, 27098, 27171, 27126, 27176, 27143, 27104, 27102, + 27105, 27229, 27186, 27235, 27137, 27249, 27106, 27110, 27114, 27183, + 27155, 27223, 27204, 27107, 27111, 27112, 27154, 27146, 27211, 27164, + 27274, 27175, 27113, 27108, 27109, 27191, 27244, 27252, 27122, 27264, + 27115, 27174, 27124, 27221, 27161, 27199, 27131, 27213, 27163, 27130, + 27270, 27120, 27166, 27116, 27162, 27190, 27217, 27228, 27198, 27233, + 27200, 27159, 27179, 27260, 27227, 27194, 27237, 27267, 27240, 27238, + 27263, 27134, 27250, 27140, 27169, 27125, 27156, 27132, 27180, 27136, + 27216, 27135, 27195, 27121, 27261, 27151, 27246, 27247, 27265, 27236, + 27177, 27215, 27207, 27170, 27145, 27119, 27184, 27192, 27231, 27197, + 27127, 27222, 27167, 27182, 27148, 27150, 27256, 27196, 27202, 27201, + 27268, 27189, 27138, 27139, 27225, 27269, 27218, 27206, 27259, 27262, + 27232, 27254, 27158, 27224, 27152, 27239, 27128, 27255, 27172, 27141, + 40951, 40951, 27282, 27280, 27279, 27276, 27275, 27278, 27277, 27283, + 27281, 27071, 27070, 27074, 27072, 27069, 27073, 27075, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27, 9, 24, + 14, 29, 21, 34, 28, 37, 39, 35, 40, 41, 10, 30, 32, 18, 15, 31, 42, 13, + 25, 36, 23, 12, 20, 33, 19, 38, 17, 11, 26, 16, 22, 62, 44, 59, 49, 64, + 56, 69, 63, 72, 74, 70, 75, 76, 45, 65, 67, 53, 50, 66, 77, 48, 60, 71, + 58, 47, 55, 68, 54, 73, 52, 46, 61, 51, 57, 85, 86, 79, 84, 43, 78, 83, + 82, 40951, 40951, 40951, 40951, 93, 95, 92, 91, 88, 87, 90, 89, 96, 94, + 40951, 40951, 40951, 40951, 80, 81, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 20698, 20684, + 20679, 20659, 20654, 20672, 20667, 20647, 20662, 20687, 20682, 20677, + 20657, 20652, 20673, 20668, 20648, 20663, 20699, 20685, 20680, 20660, + 20655, 20675, 20670, 20650, 20665, 20700, 20686, 20681, 20661, 20656, + 20676, 20671, 20651, 20666, 20688, 20683, 20678, 20658, 20653, 20674, + 20669, 20649, 20664, 20645, 20646, 20636, 20643, 20644, 20696, 20694, + 20693, 20690, 20689, 20692, 20691, 20697, 20695, 20702, 20638, 20637, + 20639, 20701, 20642, 20641, 20640, 20635, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 30827, 30822, 30817, 30797, 30792, 30810, 30805, 30785, + 30800, 30825, 30820, 30815, 30795, 30790, 30811, 30806, 30786, 30801, + 30828, 30823, 30818, 30798, 30793, 30813, 30808, 30788, 30803, 30829, + 30824, 30819, 30799, 30794, 30814, 30809, 30789, 30804, 30826, 30821, + 30816, 30796, 30791, 30812, 30807, 30787, 30802, 30784, 30777, 30779, + 30769, 30771, 30772, 30774, 30781, 30780, 30775, 30770, 30773, 30778, + 30776, 30783, 30782, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 2246, 2254, 2252, 2150, + 40951, 2262, 2250, 2258, 2242, 2257, 2249, 2198, 2253, 2260, 2225, 2247, + 2255, 2226, 2261, 2256, 2224, 2245, 2244, 2248, 2243, 2149, 2251, 2259, + 2120, 2122, 2121, 2123, 40951, 2162, 2160, 40951, 2157, 40951, 40951, + 2156, 40951, 2164, 2159, 2167, 2161, 2166, 2154, 2169, 2163, 2155, 2168, + 40951, 2153, 2152, 2151, 2158, 40951, 2170, 40951, 2165, 40951, 40951, + 40951, 40951, 40951, 40951, 2234, 40951, 40951, 40951, 40951, 2236, + 40951, 2235, 40951, 2239, 40951, 2238, 2231, 2241, 40951, 2232, 2240, + 40951, 2230, 40951, 40951, 2233, 40951, 2229, 40951, 2237, 40951, 2227, + 40951, 2228, 40951, 2216, 2214, 40951, 2211, 40951, 40951, 2210, 2205, + 2218, 2213, 40951, 2215, 2221, 2208, 2223, 2217, 2209, 2222, 40951, 2207, + 2206, 2204, 2212, 40951, 2203, 2219, 2220, 2201, 40951, 2202, 40951, + 2175, 2187, 2185, 2195, 2178, 2197, 2183, 2177, 2181, 2190, 40951, 2193, + 2186, 2192, 2172, 2176, 2188, 2173, 2196, 2189, 2171, 2182, 2180, 2174, + 2179, 2194, 2184, 2191, 40951, 40951, 40951, 40951, 40951, 2136, 2134, + 2145, 40951, 2148, 2132, 2140, 2130, 2139, 40951, 2143, 2135, 2142, 2125, + 2147, 2137, 2126, 2146, 2138, 2124, 2131, 2129, 2127, 2128, 2144, 2133, + 2141, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 2199, 2200, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25385, + 25401, 25416, 25392, 25420, 25419, 25417, 25397, 25414, 25411, 25390, + 25387, 25406, 25403, 25383, 25394, 25398, 25415, 25412, 25391, 25388, + 25407, 25404, 25384, 25395, 25396, 25413, 25410, 25389, 25386, 25405, + 25402, 25382, 25393, 25400, 25399, 25380, 25423, 25409, 25408, 25421, + 25418, 25422, 25381, 40951, 40951, 40951, 40951, 11138, 11089, 11090, + 11091, 11092, 11093, 11094, 11095, 11096, 11097, 11098, 11099, 11100, + 11101, 11102, 11103, 11104, 11105, 11106, 11107, 11108, 11109, 11110, + 11111, 11112, 11113, 11114, 11115, 11116, 11117, 11118, 11119, 11120, + 11121, 11122, 11123, 11124, 11125, 11126, 11127, 11128, 11129, 11130, + 11131, 11132, 11133, 11134, 11135, 11136, 11137, 11188, 11139, 11140, + 11141, 11142, 11143, 11144, 11145, 11146, 11147, 11148, 11149, 11150, + 11151, 11152, 11153, 11154, 11155, 11156, 11157, 11158, 11159, 11160, + 11161, 11162, 11163, 11164, 11165, 11166, 11167, 11168, 11169, 11170, + 11171, 11172, 11173, 11174, 11175, 11176, 11177, 11178, 11179, 11180, + 11181, 11182, 11183, 11184, 11185, 11186, 11187, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 31485, + 31555, 31530, 31526, 31489, 31494, 31514, 31510, 31506, 31559, 31522, + 31563, 31498, 31518, 31502, 40951, 40951, 31556, 31531, 31527, 31490, + 31495, 31515, 31511, 31507, 31560, 31523, 31564, 31499, 31519, 31503, + 31486, 40951, 31557, 31532, 31528, 31491, 31496, 31516, 31512, 31508, + 31561, 31524, 31565, 31500, 31520, 31504, 31484, 40951, 31554, 31529, + 31525, 31488, 31493, 31513, 31509, 31505, 31558, 31521, 31562, 31497, + 31517, 31501, 31487, 31492, 31536, 31533, 31547, 31548, 31549, 31550, + 31551, 31552, 31553, 31537, 31538, 31539, 31540, 31541, 31542, 31543, + 31544, 31545, 31546, 31534, 31535, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 10874, 10873, 10858, 10870, 10867, + 10852, 10849, 10864, 10861, 10876, 10855, 10915, 10894, 6813, 6530, 6554, + 31156, 31157, 31158, 31159, 31160, 31161, 31162, 31163, 31164, 31165, + 31166, 31167, 31168, 31169, 31170, 31171, 31172, 31173, 31174, 31175, + 31176, 31177, 31178, 31179, 31180, 31181, 37498, 6644, 6645, 6541, 6812, + 8646, 34420, 34421, 34422, 34423, 34424, 34425, 34426, 34427, 34428, + 34429, 34430, 34431, 34432, 34433, 34434, 34435, 34436, 34437, 34438, + 34439, 34440, 34441, 34442, 34443, 34444, 34445, 34414, 34450, 34465, + 34466, 34455, 34477, 28977, 28978, 28979, 28980, 28981, 28982, 28983, + 28984, 28985, 28986, 28987, 28988, 28989, 28990, 28991, 28992, 28993, + 28994, 28995, 28996, 28997, 28998, 28999, 29000, 29001, 29002, 31764, + 31765, 31766, 6540, 6539, 6587, 29008, 29009, 29010, 29011, 29012, 29013, + 29014, 29015, 29016, 29017, 29018, 29019, 29020, 29021, 29022, 29023, + 29024, 29025, 29026, 29027, 29028, 29029, 29030, 29031, 29032, 29033, + 8690, 29040, 29043, 29044, 29039, 29041, 34149, 34403, 34402, 34409, + 34478, 34451, 34452, 34454, 34464, 34472, 34475, 34467, 34457, 34469, + 34407, 34471, 34408, 34458, 34468, 34460, 34453, 34419, 34413, 34412, + 34411, 34448, 34459, 34473, 34474, 25815, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 31794, 31795, 31796, 31797, 31798, 31799, 31800, 31801, + 31802, 31803, 31804, 31805, 31806, 31807, 31808, 31809, 31810, 31811, + 31812, 31813, 31814, 31815, 31816, 31817, 31818, 31819, 34190, 34415, + 34417, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 34381, 34376, 34368, 34416, 34362, 34374, + 34397, 34373, 34361, 34386, 34394, 34385, 34366, 34377, 34378, 34384, + 34365, 34395, 34392, 34399, 34375, 34370, 34389, 34379, 34382, 34359, + 34360, 34401, 34372, 34363, 34367, 34383, 34398, 34380, 34393, 34396, + 34369, 34390, 34388, 34387, 34391, 34364, 34371, 34400, 40951, 40951, + 40951, 40951, 37495, 37489, 37490, 37492, 37496, 37493, 37497, 37491, + 37494, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 6592, 6590, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 32230, 32231, 32228, 32232, 32227, 32229, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 10525, 17488, 8037, 29287, 34652, 34651, + 6823, 34697, 31757, 4931, 39127, 38952, 27585, 11538, 11536, 11537, + 18006, 29092, 39039, 17455, 39040, 17529, 39038, 22740, 39037, 8680, + 29091, 17454, 22739, 17528, 34576, 18007, 32825, 36989, 3828, 39284, + 39288, 39285, 39286, 8050, 8049, 8048, 8047, 17487, 39351, 20436, 36648, + 5007, 6476, 32563, 17383, 10551, 31024, 6124, 20433, 37663, 6474, 32218, + 20393, 34698, 4242, 11532, 11533, 20218, 17506, 25667, 17423, 24034, + 28299, 37448, 2541, 18119, 27068, 39130, 35786, 24399, 3426, 31470, + 31791, 18659, 31290, 31287, 6475, 34517, 19108, 33783, 26860, 31606, + 31914, 31915, 8480, 9952, 34505, 34081, 4929, 17519, 32234, 10534, 30882, + 34738, 17527, 17461, 33878, 32769, 20466, 11284, 8479, 6513, 5999, 25270, + 9955, 20399, 32842, 3649, 31604, 8478, 17493, 36650, 32530, 39353, 8054, + 37576, 3540, 7996, 2605, 17494, 4366, 31594, 31905, 39373, 3760, 20825, + 6514, 17429, 17451, 17450, 2748, 31214, 8459, 35785, 8691, 31469, 20830, + 6061, 39350, 28182, 32535, 18044, 19689, 4368, 27584, 31858, 28305, + 34525, 24398, 8471, 3530, 3531, 17444, 98, 6059, 17435, 32176, 17459, + 27574, 28204, 6516, 19688, 2522, 37473, 6821, 37395, 7992, 31319, 38874, + 10920, 33789, 3759, 17718, 4371, 17477, 28540, 28286, 32529, 18675, + 28304, 37581, 38879, 28539, 32359, 36764, 33764, 3432, 6477, 33872, + 32360, 34736, 34112, 37578, 20431, 370, 32237, 34741, 39148, 18043, + 31747, 31748, 8682, 38953, 17465, 20468, 34929, 33869, 5265, 3536, 5006, + 20443, 6822, 10828, 7993, 10659, 10660, 28958, 34503, 20442, 20441, + 29948, 20827, 17349, 20444, 3421, 2538, 20437, 25154, 8474, 32534, 10577, + 17419, 20823, 20826, 17348, 39253, 3881, 39136, 39135, 32219, 3897, + 22564, 2616, 4375, 369, 16743, 16744, 16745, 16746, 16747, 31772, 28199, + 30885, 39128, 8673, 37293, 24296, 31774, 6066, 11374, 8694, 39307, 33866, + 33867, 20430, 31777, 18011, 32848, 28180, 32217, 6482, 11025, 31461, + 4553, 16712, 29946, 34106, 4944, 964, 20398, 22566, 17458, 37577, 4243, + 37688, 19657, 2604, 17523, 3761, 31291, 22561, 31616, 11376, 2614, 11087, + 28201, 8674, 37294, 31775, 6067, 11375, 34111, 20432, 28181, 11086, + 31463, 17526, 19106, 39372, 3535, 31061, 31462, 31280, 6481, 17380, + 17378, 11531, 29441, 28202, 37449, 39295, 39210, 39236, 39260, 17462, + 39137, 30881, 37021, 37022, 7991, 30538, 8696, 39362, 17379, 29282, + 34926, 20927, 11382, 22556, 3765, 39361, 31720, 19112, 31611, 25661, + 2535, 20289, 39363, 39360, 17492, 5002, 5001, 4551, 17906, 25574, 39358, + 17427, 25580, 37703, 37704, 31591, 39359, 4932, 31304, 25577, 25578, + 30517, 30515, 2603, 8462, 31673, 20833, 20832, 18975, 2606, 17367, 348, + 20595, 33766, 20711, 18674, 10533, 25054, 28889, 17415, 18979, 3430, + 34924, 31466, 22553, 25153, 32156, 17722, 22548, 4367, 8672, 39146, 3537, + 4938, 37697, 34082, 18672, 19691, 4245, 18661, 39397, 31719, 19690, + 31901, 19692, 10836, 16700, 962, 4550, 33778, 8059, 34107, 11378, 10538, + 31464, 17472, 11005, 34095, 36996, 39221, 20453, 28002, 9950, 19721, + 8679, 3423, 3425, 3424, 3422, 28001, 6295, 32559, 31314, 4933, 27587, + 17478, 30549, 11529, 17440, 30536, 30889, 30891, 5262, 36651, 6004, 6294, + 6296, 3428, 7997, 31722, 32225, 31282, 34515, 37547, 4256, 24397, 29438, + 29439, 8043, 30530, 18660, 4244, 30553, 4257, 28960, 32556, 27421, 36656, + 30892, 17426, 32443, 31723, 6301, 30831, 20822, 31281, 17381, 20627, + 16781, 8040, 8041, 30540, 30539, 31600, 31597, 29277, 27601, 27602, + 38871, 27603, 29357, 966, 5263, 5264, 38873, 36663, 31750, 38875, 17443, + 31595, 31687, 37691, 8027, 8028, 8024, 975, 25155, 20284, 34090, 34089, + 34091, 34092, 3529, 16698, 24165, 32032, 25112, 8042, 21611, 25109, + 30543, 3543, 3545, 4255, 25052, 31752, 2608, 16776, 30519, 33928, 37488, + 29356, 21626, 20714, 20715, 20718, 20717, 20716, 17447, 16699, 39376, + 19102, 29852, 20445, 31475, 27573, 36662, 8701, 33760, 20831, 37549, + 3908, 39271, 22730, 22657, 22665, 22658, 33795, 33794, 37786, 11297, + 37790, 11291, 25224, 37882, 6537, 8688, 8687, 29431, 29433, 34813, 39234, + 19736, 6132, 30883, 11369, 21609, 28188, 34834, 27284, 4369, 8008, 8018, + 8020, 8004, 8002, 8012, 8010, 8000, 8006, 8014, 7998, 8016, 8009, 8019, + 8021, 8005, 8003, 8013, 8011, 8001, 8007, 8015, 7999, 8017, 31979, 31980, + 31981, 4997, 4998, 32164, 4254, 5998, 25664, 3910, 29355, 20397, 25575, + 33781, 10535, 34102, 34103, 20928, 25579, 24086, 36657, 31960, 39290, + 3923, 36661, 7994, 2609, 34487, 16780, 17486, 31294, 25051, 3878, 25186, + 25172, 25184, 25185, 25206, 24146, 37679, 31762, 31886, 31894, 31895, + 31899, 31896, 31763, 39211, 32951, 32950, 32947, 32946, 3856, 3883, + 32953, 32952, 32949, 32948, 3926, 3822, 3835, 10661, 21613, 37012, 31670, + 31617, 3838, 39225, 33879, 36645, 39356, 30527, 37692, 37008, 37533, + 30341, 19655, 32540, 31671, 17425, 30550, 11011, 11012, 11013, 17516, + 17518, 17517, 3834, 17490, 30537, 6005, 6006, 17441, 16748, 16749, 16750, + 29435, 29436, 29437, 16759, 16752, 16753, 11010, 30888, 30893, 39142, + 34105, 34104, 10662, 27588, 26846, 30874, 8026, 5996, 20629, 10552, 8454, + 30514, 32175, 30890, 34513, 10532, 25053, 34093, 37004, 37003, 37005, + 37006, 24114, 31982, 37707, 37010, 24131, 31996, 24044, 31918, 28185, + 24422, 24421, 2753, 2759, 2754, 2758, 2751, 24419, 2752, 39367, 28198, + 37532, 34500, 33763, 28203, 18664, 18667, 17405, 33854, 33856, 33855, + 33857, 33853, 33852, 39354, 33858, 17385, 31857, 33851, 33861, 33864, + 29089, 17360, 37743, 17388, 31292, 8460, 8461, 22549, 17416, 22550, + 22551, 17402, 17403, 17404, 10922, 39368, 963, 31609, 8700, 31316, 17410, + 10921, 17525, 960, 17431, 39144, 33780, 37394, 18669, 25269, 17393, + 20452, 17395, 17386, 2530, 17479, 33779, 11006, 17413, 17390, 18668, + 6068, 33850, 33849, 6069, 22552, 31608, 8699, 39143, 33785, 33784, 37894, + 17409, 17398, 17392, 31313, 32564, 19694, 34088, 19654, 31311, 31310, + 31306, 31308, 29400, 33984, 29373, 33971, 37677, 37686, 37676, 37685, + 29399, 33983, 29372, 33970, 19772, 19765, 19769, 19762, 29401, 33985, + 29374, 33972, 19773, 19766, 19770, 19763, 20395, 20396, 33924, 33925, + 24289, 37928, 32119, 11364, 32550, 19767, 24401, 19744, 19699, 34739, + 32437, 32438, 32439, 19789, 32440, 19756, 38862, 38859, 6297, 31867, + 32174, 19930, 34504, 31755, 20287, 20288, 37540, 27420, 24408, 34501, + 37536, 37537, 5000, 30524, 37575, 5003, 27589, 371, 17448, 31589, 30523, + 36649, 30522, 2537, 30520, 31790, 10557, 2521, 37534, 28179, 28195, + 34737, 28196, 160, 32824, 32233, 34094, 20423, 38853, 8463, 31590, 37546, + 11370, 29353, 33865, 29358, 31721, 11368, 31602, 29362, 3755, 29351, + 3754, 28197, 31323, 29354, 6479, 27285, 39364, 31862, 2607, 37531, 39126, + 32847, 3527, 3528, 31222, 9954, 2620, 24087, 37543, 31684, 6646, 4552, + 17907, 8671, 33777, 32827, 3547, 3656, 31480, 29945, 32826, 34527, 30894, + 20391, 20455, 16713, 40951, 40951, 40951, 40951, 39357, 31566, 39150, + 32157, 19103, 32820, 29979, 28193, 31754, 28190, 37784, 37781, 37789, + 33792, 29408, 227, 228, 40951, 40951, 40951, 32442, 30521, 10847, 31218, + 32538, 28191, 6001, 33782, 17482, 33769, 2539, 31460, 32177, 40951, + 40951, 40951, 295, 243, 344, 342, 339, 237, 235, 236, 233, 234, 332, 334, 335, 321, 288, 250, 281, 282, 283, 265, 309, 286, 336, 337, 307, 308, 270, 323, 276, 277, 259, 300, 256, 278, 319, 257, 258, 255, 310, 317, 338, 329, 279, 238, 318, 311, 316, 333, 298, 299, 297, 301, 302, 303, - 230, 231, 284, 312, 241, 304, 305, 240, 246, 326, 327, 296, 247, 248, + 230, 231, 284, 312, 240, 304, 305, 241, 246, 326, 327, 296, 247, 248, 249, 232, 343, 322, 328, 271, 340, 289, 254, 331, 253, 325, 252, 330, 313, 280, 324, 341, 274, 242, 291, 251, 290, 239, 314, 315, 320, 294, 268, 266, 267, 293, 292, 260, 261, 262, 263, 264, 229, 244, 245, 306, - 275, 287, 269, 285, 273, 272, 20525, 25235, 20594, 35762, 35762, 35762, - 35762, 15228, 20771, 13645, 26946, 25897, 3823, 3899, 3860, 3803, 3885, - 22283, 3994, 15329, 33678, 13042, 34005, 27397, 3893, 3886, 19796, 22308, - 3995, 15330, 33679, 13043, 34084, 34086, 29411, 3891, 3909, 3847, 34019, - 34020, 10527, 3892, 3910, 3848, 34055, 32018, 19798, 22309, 3996, 33683, - 33680, 13044, 32016, 19791, 22301, 3990, 15302, 33675, 13039, 19784, - 22289, 3993, 15277, 33673, 13041, 19792, 22300, 3991, 15306, 33676, - 13040, 19782, 22306, 3992, 15271, 33672, 19794, 22303, 32036, 22302, - 19786, 22288, 13182, 22287, 27071, 19783, 15276, 22299, 15305, 28850, - 22305, 15269, 33671, 15267, 19793, 15324, 15323, 6506, 24298, 6516, - 24299, 24576, 35762, 35762, 35762, 35762, 35762, 35762, 18137, 18150, - 18133, 18146, 18128, 18149, 18132, 18139, 18151, 18134, 18147, 18129, - 35762, 35762, 35762, 35762, 15274, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 19590, 32726, 27244, 10996, 19597, 32723, 27251, 10993, 19586, 32722, - 27240, 10992, 35762, 35762, 35762, 35762, 19589, 32725, 27243, 10995, - 19599, 32727, 27253, 10997, 15285, 15326, 15299, 15263, 15284, 15325, - 15298, 15262, 19633, 32760, 27304, 11013, 19632, 32759, 27303, 11012, - 19631, 32756, 27302, 11009, 19635, 32762, 27306, 11015, 19634, 32761, - 27305, 11014, 19663, 32737, 27270, 11031, 19671, 32752, 27293, 11046, - 19673, 32748, 27326, 11042, 19623, 32746, 27284, 11040, 19624, 32747, - 27285, 11041, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 19672, 32751, 27294, 11045, 24695, 24669, 29067, 29079, 19509, 32647, - 35762, 35762, 35762, 35762, 35762, 35762, 34121, 34136, 34126, 34131, - 34146, 34141, 34151, 34156, 34123, 34138, 34128, 34133, 34148, 34143, - 34153, 34158, 34122, 34137, 34127, 34132, 34147, 34142, 34152, 34157, - 34119, 34134, 34124, 34129, 34144, 34139, 34149, 34154, 34120, 34135, - 34125, 34130, 34145, 34140, 34150, 34155, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 19639, 32766, 27310, 11020, 19653, 32778, - 27325, 11025, 19598, 32724, 27252, 10994, 15240, 15243, 15242, 15241, - 19607, 27259, 19642, 27295, 19664, 27290, 19668, 27286, 19606, 27258, - 19662, 27269, 33969, 33968, 35762, 35762, 2501, 2498, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 6234, 6235, 6233, 19481, 19479, - 19480, 19482, 19478, 10985, 10987, 10986, 10988, 26673, 34040, 4639, - 26674, 35571, 23320, 13085, 24574, 32019, 13068, 27406, 15990, 28718, - 4960, 26975, 19561, 27210, 14659, 14655, 16358, 13066, 7748, 24254, - 27351, 11059, 20652, 13092, 28959, 13076, 14271, 14269, 13086, 27751, - 28950, 13073, 27938, 26823, 4627, 26265, 27760, 26877, 20898, 23505, - 27942, 26507, 16464, 13115, 22921, 34186, 33943, 14657, 10692, 34160, - 11060, 7692, 32565, 29194, 13614, 27340, 13133, 27733, 32020, 4247, - 21060, 9645, 18038, 28972, 13158, 8392, 2581, 9652, 2599, 26808, 5763, - 2602, 14266, 27764, 29598, 12333, 13609, 26494, 18027, 26266, 11206, - 13174, 30393, 6211, 4097, 9638, 7752, 4638, 26684, 26873, 9653, 27612, - 5699, 19446, 20987, 23500, 2603, 28951, 34207, 28953, 13078, 13090, - 26079, 13195, 24577, 10616, 13095, 13080, 27702, 18036, 13643, 15936, - 13143, 8399, 20486, 27707, 32541, 32633, 11221, 11207, 3450, 27861, - 26090, 13187, 4704, 10522, 13644, 20487, 27106, 27941, 29556, 13512, - 35565, 15827, 27698, 29952, 8378, 16750, 20658, 26491, 15938, 26423, - 26954, 20590, 23503, 22896, 2601, 29768, 20895, 11051, 28882, 26072, - 25798, 28860, 13147, 26085, 3518, 3746, 27729, 14280, 26890, 12422, - 12423, 12425, 12424, 4253, 19780, 13165, 32423, 29761, 29762, 27541, - 11214, 23508, 20984, 22183, 22184, 5999, 9640, 27544, 3635, 13321, 25772, - 13102, 33956, 4703, 22147, 16003, 4641, 32531, 29564, 18031, 10521, - 13070, 101, 6177, 25759, 3514, 26515, 26508, 26518, 26509, 20662, 13108, - 33339, 22728, 12420, 13507, 35757, 4625, 25797, 3738, 27704, 13612, 8374, - 28967, 26995, 13128, 16467, 31792, 26528, 11208, 8154, 2583, 13125, - 32425, 4634, 20661, 20593, 20524, 29193, 2742, 27542, 31680, 4640, 3411, - 27350, 3409, 29197, 26982, 24256, 24357, 24368, 24371, 24360, 24350, - 24365, 33980, 3771, 24351, 33977, 33993, 33996, 33972, 33985, 33990, - 3768, 3784, 3787, 3763, 3776, 3781, 24358, 24369, 24372, 24361, 24356, - 24366, 33981, 3772, 24352, 33999, 34002, 34003, 33998, 34000, 34001, - 3790, 3793, 3794, 3789, 3791, 3792, 24375, 24378, 24379, 24374, 24376, - 24377, 33982, 3773, 24353, 33978, 33994, 33997, 33973, 33983, 33991, - 3769, 3785, 3788, 3764, 3774, 3782, 24359, 24370, 24373, 24362, 24354, - 24367, 33984, 3775, 24355, 33974, 3765, 24363, 33975, 3766, 24364, 33987, - 33988, 33986, 3778, 3779, 3777, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 34202, 34205, 34201, 34203, - 34200, 34199, 34204, 34195, 34198, 34194, 34196, 34193, 34192, 34197, - 35762, 35762, 2743, 25771, 4633, 27936, 32022, 19795, 14267, 26678, - 11057, 99, 29583, 34191, 8395, 35762, 35762, 35762, 35479, 18030, 26274, - 4001, 20660, 26679, 24347, 20990, 13156, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 27335, 5834, 26980, 2586, 10695, 3407, 22900, 1, - 20509, 8373, 5761, 27710, 18040, 15983, 22916, 34163, 26792, 27756, - 18032, 4707, 23516, 32424, 15231, 26687, 27345, 22917, 16009, 20530, - 14653, 13163, 14523, 17228, 13157, 34180, 3521, 7750, 26810, 34182, - 13109, 20526, 8347, 12434, 24348, 16000, 16460, 34166, 35762, 13646, 946, - 20595, 26530, 26825, 26824, 26513, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 23502, 10880, 3989, 3524, 25761, 13144, 30809, - 13186, 31679, 26812, 3519, 16459, 13509, 26495, 35762, 35762, 35762, - 35762, 22389, 27546, 13075, 13079, 13089, 10890, 3743, 4642, 27912, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 14656, 27283, 19622, - 26231, 26232, 16170, 15238, 19667, 27289, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 3925, 3955, 3926, 3970, 3941, 3956, 3927, 3978, - 3948, 3964, 3934, 3971, 3942, 3957, 3928, 3982, 3952, 3967, 3938, 3975, - 3961, 3931, 3979, 3949, 3965, 3935, 3972, 3943, 3958, 3929, 3984, 3954, - 3969, 3940, 3977, 3947, 3963, 3933, 3981, 3951, 3937, 3974, 3945, 3960, - 3930, 3983, 3953, 3968, 3939, 3976, 3946, 3962, 3932, 3980, 3950, 3966, - 3936, 3973, 3944, 3959, 20541, 20542, 20549, 20551, 20546, 20574, 20575, - 20571, 20573, 20569, 20572, 20565, 20568, 20566, 20570, 20567, 20545, - 20548, 20543, 20547, 20544, 20550, 32692, 32693, 32700, 32702, 32697, - 32682, 32683, 32679, 32681, 32677, 32680, 32673, 32676, 32674, 32678, - 32675, 32696, 32699, 32694, 32698, 32695, 32701, 32664, 19447, 32661, - 19449, 19530, 32715, 27195, 20587, 33658, 33659, 33660, 33661, 33662, - 33663, 15949, 15950, 15951, 15952, 15953, 15954, 19448, 19450, 27120, - 27119, 32663, 15948, 32690, 32713, 32666, 32714, 32712, 27160, 27190, - 27147, 27191, 27170, 19489, 27155, 32671, 20537, 16342, 32668, 32669, - 35762, 19488, 5997, 16338, 15278, 32685, 32709, 32662, 19451, 32686, - 32710, 20563, 20556, 4173, 4177, 4169, 4172, 4174, 4179, 4170, 4167, - 4176, 4178, 4180, 4175, 4168, 4171, 4184, 4207, 2504, 16339, 19487, - 27154, 16340, 19577, 27229, 11002, 32719, 19486, 27153, 33766, 27162, - 24301, 24300, 24302, 34043, 19532, 22897, 27188, 24330, 29584, 29585, - 29587, 29588, 29586, 34109, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 27743, 27745, 27742, 27741, 27738, 27737, 27740, 27739, 27746, - 27744, 35762, 35762, 35762, 35762, 35762, 35762, 6553, 6554, 6555, 6556, - 6557, 6558, 6559, 6560, 6561, 6562, 6563, 6564, 6565, 6566, 6567, 6568, - 6569, 6570, 6571, 6572, 6573, 6574, 6575, 6576, 6577, 6578, 6579, 6580, - 6581, 6582, 6583, 6584, 6585, 6586, 6587, 6588, 6589, 6590, 6591, 6592, - 6593, 6594, 6595, 6596, 6597, 6598, 6599, 6600, 6601, 6602, 6603, 6604, - 6605, 6606, 6607, 6608, 6609, 6610, 6611, 6612, 6613, 6614, 6615, 6616, - 6617, 6618, 6619, 6620, 6621, 6622, 6623, 6624, 6625, 6626, 6627, 6628, - 6629, 6630, 6631, 6632, 6633, 6634, 6635, 6636, 6637, 6638, 6639, 6640, - 6641, 6642, 6643, 6644, 6645, 6646, 6647, 6648, 6649, 6650, 6651, 6652, - 6653, 6654, 6655, 6656, 6657, 6658, 6659, 6660, 6661, 6662, 6663, 6664, - 6665, 6666, 6667, 6668, 6669, 6670, 6671, 6672, 6673, 6674, 6675, 6676, - 6677, 6678, 6679, 6680, 6681, 6682, 6683, 6684, 6685, 6686, 6687, 6688, - 6689, 6690, 6691, 6692, 6693, 6694, 6695, 6696, 6697, 6698, 6699, 6700, - 6701, 6702, 6703, 6704, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, - 6713, 6714, 6715, 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6724, - 6725, 6726, 6727, 6728, 6729, 6730, 6731, 6732, 6733, 6734, 6735, 6736, - 6737, 6738, 6739, 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6748, - 6749, 6750, 6751, 6752, 6753, 6754, 6755, 6756, 6757, 6758, 6759, 6760, - 6761, 6762, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 6771, 6772, - 6773, 6774, 6775, 6776, 6777, 6778, 6779, 6780, 6781, 6782, 6783, 6784, - 6785, 6786, 6787, 6788, 6789, 6790, 6791, 6792, 6793, 6794, 6795, 6796, - 6797, 6798, 6799, 6800, 6801, 6802, 6803, 6804, 6805, 6806, 6807, 6808, - 6809, 6810, 6811, 6812, 6813, 6814, 6815, 6816, 6817, 6818, 6819, 6820, - 6821, 6822, 6823, 6824, 6825, 6826, 6827, 6828, 6829, 6830, 6831, 6832, - 6833, 6834, 6835, 6836, 6837, 6838, 6839, 6840, 6841, 6842, 6843, 6844, - 6845, 6846, 6847, 6848, 6849, 6850, 6851, 6852, 6853, 6854, 6855, 6856, - 6857, 6858, 6859, 6860, 6861, 6862, 6863, 6864, 6865, 6866, 6867, 6868, - 6869, 6870, 6871, 6872, 6873, 6874, 6875, 6876, 6877, 6878, 6879, 6880, - 6881, 6882, 6883, 6884, 6885, 6886, 6887, 6888, 6889, 6890, 6891, 6892, - 6893, 6894, 6895, 6896, 6897, 6898, 6899, 6900, 6901, 6902, 6903, 6904, - 6905, 6906, 6907, 6908, 6909, 6910, 6911, 6912, 6913, 6914, 6915, 6916, - 6917, 6918, 6919, 6920, 6921, 6922, 6923, 6924, 6925, 6926, 6927, 6928, - 6929, 6930, 6931, 6932, 6933, 6934, 6935, 6936, 6937, 6938, 6939, 6940, - 6941, 6942, 6943, 6944, 6945, 6946, 6947, 6948, 6949, 6950, 6951, 6952, - 6953, 6954, 6955, 6956, 6957, 6958, 6959, 6960, 6961, 6962, 6963, 6964, - 6965, 6966, 6967, 6968, 6969, 6970, 6971, 6972, 6973, 6974, 6975, 6976, - 6977, 6978, 6979, 6980, 6981, 6982, 6983, 6984, 6985, 6986, 6987, 6988, - 6989, 6990, 6991, 6992, 6993, 6994, 6995, 6996, 6997, 6998, 6999, 7000, - 7001, 7002, 7003, 7004, 7005, 7006, 7007, 7008, 7009, 7010, 7011, 7012, - 7013, 7014, 7015, 7016, 7017, 7018, 7019, 7020, 7021, 7022, 7023, 7024, - 7025, 7026, 7027, 7028, 7029, 7030, 7031, 7032, 7033, 7034, 7035, 7036, - 7037, 7038, 7039, 7040, 7041, 7042, 7043, 7044, 7045, 7046, 7047, 7048, - 7049, 7050, 7051, 7052, 7053, 7054, 7055, 7056, 7057, 7058, 7059, 7060, - 7061, 7062, 7063, 7064, 6537, 6538, 6539, 6540, 6541, 6542, 6543, 6544, - 6545, 6546, 6547, 6548, 6549, 6550, 6551, 6552, 6523, 6524, 6525, 6526, - 6527, 6528, 6529, 6530, 6531, 6532, 6533, 6534, 6535, 6536, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 19444, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 30045, 29974, 30037, 30046, - 29962, 30035, 29956, 29955, 30032, 30040, 29957, 30036, 29960, 29977, - 30048, 30044, 29969, 29971, 29968, 29967, 29964, 29963, 29966, 29965, - 29972, 29970, 29961, 30043, 30030, 29973, 29976, 30038, 29959, 29978, - 29979, 29980, 29981, 29982, 29983, 29984, 29985, 29986, 29987, 29988, - 29989, 29990, 29991, 29992, 29993, 29994, 29995, 29996, 29997, 29998, - 29999, 30000, 30001, 30002, 30003, 30033, 30042, 30041, 29958, 30034, - 29975, 30004, 30005, 30006, 30007, 30008, 30009, 30010, 30011, 30012, - 30013, 30014, 30015, 30016, 30017, 30018, 30019, 30020, 30021, 30022, - 30023, 30024, 30025, 30026, 30027, 30028, 30029, 30031, 30049, 30039, - 30047, 5694, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 33161, 33172, 33183, 33195, 33206, 33217, 33228, 33239, 33250, 33258, - 33259, 33260, 33261, 33263, 33264, 33265, 33266, 33267, 33268, 33269, - 33270, 33271, 33272, 33274, 33275, 33276, 33277, 33278, 33279, 33280, - 33281, 33282, 33283, 33285, 33286, 33287, 33288, 33289, 33290, 33291, - 33292, 33293, 33294, 33296, 33297, 33298, 33299, 33300, 33301, 33302, - 33303, 33304, 33305, 33307, 33308, 33309, 33310, 33311, 33312, 33313, - 33314, 33315, 33316, 33318, 33319, 33320, 33321, 33322, 33323, 33324, - 33325, 33326, 33327, 33329, 33330, 33331, 33332, 33333, 33334, 33335, - 33336, 33337, 33338, 33085, 33086, 33087, 33088, 33089, 33090, 33091, - 33092, 33093, 33094, 33096, 33097, 33098, 33099, 33100, 33101, 33102, - 33103, 33104, 33105, 33107, 33108, 33109, 33110, 33111, 33112, 33113, - 33114, 33115, 33116, 33118, 33119, 33120, 33121, 33122, 33123, 33124, - 33125, 33126, 33127, 33129, 33130, 33131, 33132, 33133, 33134, 33135, - 33136, 33137, 33138, 33140, 33141, 33142, 33143, 33144, 33145, 33146, - 33147, 33148, 33149, 33151, 33152, 33153, 33154, 33155, 33156, 33157, - 33158, 33159, 33160, 33162, 33163, 33164, 33165, 33166, 33167, 33168, - 33169, 33170, 33171, 33173, 33174, 33175, 33176, 33177, 33178, 33179, - 33180, 33181, 33182, 33184, 33185, 33186, 33187, 33188, 33189, 33190, - 33191, 33192, 33193, 33196, 33197, 33198, 33199, 33200, 33201, 33202, - 33203, 33204, 33205, 33207, 33208, 33209, 33210, 33211, 33212, 33213, - 33214, 33215, 33216, 33218, 33219, 33220, 33221, 33222, 33223, 33224, - 33225, 33226, 33227, 33229, 33230, 33231, 33232, 33233, 33234, 33235, - 33236, 33237, 33238, 33240, 33241, 33242, 33243, 33244, 33245, 33246, - 33247, 33248, 33249, 33251, 33252, 33253, 33254, 33255, 33256, 33257, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 25143, 25142, 29575, 29166, - 29576, 29601, 12457, 13030, 12455, 12468, 12461, 12460, 3, 2, 347, 3522, - 2596, 4957, 5989, 15959, 15985, 29953, 19875, 24468, 12458, 20659, 25216, - 12466, 19877, 33665, 33770, 13171, 13320, 5762, 8375, 27914, 20491, - 28965, 27915, 20490, 27945, 10237, 11210, 10519, 10238, 10518, 10239, - 10517, 10240, 10515, 10241, 24336, 24258, 29858, 29857, 12456, 13029, - 5692, 4965, 12454, 12467, 12421, 29632, 29602, 12502, 12501, 16242, - 13120, 13318, 16243, 14274, 14522, 16244, 26992, 27539, 16245, 32630, - 32782, 29168, 10253, 10250, 26093, 26092, 15830, 15984, 4626, 4956, - 24641, 24260, 16169, 16168, 24570, 24575, 29572, 29559, 12453, 12505, - 5991, 15961, 15987, 5990, 15960, 15986, 19878, 33666, 33771, 26416, - 26415, 26793, 26417, 26418, 26773, 27072, 27079, 27107, 28731, 28732, - 29557, 28730, 28733, 29558, 10516, 10242, 26881, 26882, 26929, 26880, - 26883, 26930, 27754, 29600, 5693, 10222, 22730, 24084, 29571, 29574, - 29169, 12452, 12450, 13063, 29573, 29167, 28725, 29950, 28724, 27728, - 8169, 10221, 29595, 29560, 25790, 26010, 26879, 26931, 1048, 1049, 24259, - 27944, 18311, 18929, 10220, 2297, 350, 29936, 16751, 18043, 18044, 18086, - 18052, 32125, 14921, 14922, 14899, 14920, 13314, 13315, 13316, 24082, - 13317, 29657, 35760, 35759, 35761, 20655, 27343, 20653, 27341, 26489, - 20656, 27344, 25215, 24083, 34189, 20654, 27342, 13319, 26490, 33962, - 22891, 22892, 19628, 27299, 34696, 23890, 33340, 33451, 33519, 33530, - 33541, 33552, 33563, 33574, 33585, 33341, 33352, 33363, 33374, 33385, - 33396, 33407, 26848, 4955, 4248, 35758, 9338, 9337, 22191, 22189, 22249, - 22247, 15743, 4792, 33418, 33429, 33440, 33452, 33463, 33474, 33485, - 33496, 33507, 33515, 33516, 33517, 33518, 33520, 33521, 33522, 33523, - 33524, 33525, 33526, 33527, 33528, 33529, 33531, 33532, 33533, 33534, - 33535, 33536, 33537, 33538, 33539, 33540, 33542, 33543, 33544, 33545, - 33546, 33547, 33548, 33549, 33550, 33551, 33553, 33554, 33555, 33556, - 33557, 33558, 33559, 33560, 33561, 33562, 33564, 33565, 33566, 33567, - 33568, 33569, 33570, 33571, 33572, 33573, 33575, 33576, 33577, 33578, - 33579, 33580, 33581, 33582, 33583, 33584, 33586, 33587, 33588, 33589, - 33590, 33591, 33592, 33593, 33594, 33595, 33342, 33343, 33344, 33345, - 33346, 33347, 33348, 33349, 33350, 33351, 33353, 33354, 33355, 33356, - 33357, 33358, 33359, 33360, 33361, 33362, 33364, 33365, 33366, 33367, - 33368, 33369, 33370, 33371, 33372, 33373, 33375, 33376, 33377, 33378, - 33379, 33380, 33381, 33382, 33383, 33384, 33386, 33387, 33388, 33389, - 33390, 33391, 33392, 33393, 33394, 33395, 33397, 33398, 33399, 33400, - 33401, 33402, 33403, 33404, 33405, 33406, 33408, 33409, 33410, 33411, - 33412, 33413, 33414, 33415, 33416, 33417, 33419, 33420, 33421, 33422, - 33423, 33424, 33425, 33426, 33427, 33428, 33430, 33431, 33432, 33433, - 33434, 33435, 33436, 33437, 33438, 33439, 33441, 33442, 33443, 33444, - 33445, 33446, 33447, 33448, 33449, 33450, 33453, 33454, 33455, 33456, - 33457, 33458, 33459, 33460, 33461, 33462, 33464, 33465, 33466, 33467, - 33468, 33469, 33470, 33471, 33472, 33473, 33475, 33476, 33477, 33478, - 33479, 33480, 33481, 33482, 33483, 33484, 33486, 33487, 33488, 33489, - 33490, 33491, 33492, 33493, 33494, 33495, 33497, 33498, 33499, 33500, - 33501, 33502, 33503, 33504, 33505, 33506, 33508, 33509, 33510, 33511, - 33512, 33513, 33514, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 17147, 17148, 17155, 17157, 17154, 17153, 17150, 17149, - 17152, 17151, 17158, 17156, 18176, 18735, 18335, 18960, 18570, 19331, - 18270, 18870, 18271, 18871, 18272, 18872, 18444, 19110, 18445, 19111, - 18446, 19112, 18509, 19207, 18254, 18853, 18250, 18849, 18955, 19078, - 18186, 18745, 18187, 18746, 18275, 18876, 18276, 18877, 18260, 18859, - 18261, 18860, 18954, 18956, 18340, 18962, 18341, 18963, 18358, 18990, - 18388, 19030, 18394, 19051, 18484, 19169, 18574, 19335, 18575, 19336, - 18571, 19332, 18572, 19333, 18754, 19127, 19128, 19286, 19287, 19216, - 19217, 18944, 18945, 2267, 2270, 2268, 2272, 2274, 2271, 2273, 2269, - 2275, 10448, 10443, 10442, 10449, 10444, 10445, 10447, 10446, 3594, 3593, - 3595, 14488, 14487, 14486, 14485, 14490, 14489, 25862, 25861, 3539, - 30399, 30407, 30416, 30408, 30410, 30405, 30409, 30404, 30420, 30419, - 30422, 30414, 30401, 30421, 30403, 30402, 30415, 30406, 30418, 30412, - 30413, 30411, 30417, 30400, 30538, 30545, 30546, 30541, 30542, 30547, - 30548, 30539, 30543, 30544, 30540, 30604, 30611, 30612, 30607, 30608, - 30613, 30614, 30605, 30609, 30610, 30606, 30715, 30722, 30723, 30718, - 30719, 30724, 30725, 30716, 30720, 30721, 30717, 30615, 30622, 30623, - 30618, 30619, 30624, 30625, 30616, 30620, 30621, 30617, 30693, 30700, - 30701, 30696, 30697, 30702, 30703, 30694, 30698, 30699, 30695, 30593, - 30600, 30601, 30596, 30597, 30602, 30603, 30594, 30598, 30599, 30595, - 30704, 30711, 30712, 30707, 30708, 30713, 30714, 30705, 30709, 30710, - 30706, 30626, 30633, 30634, 30629, 30630, 30635, 30636, 30627, 30631, - 30632, 30628, 30759, 30766, 30767, 30762, 30763, 30768, 30769, 30760, - 30764, 30765, 30761, 30748, 30755, 30756, 30751, 30752, 30757, 30758, - 30749, 30753, 30754, 30750, 30781, 30788, 30789, 30784, 30785, 30790, - 30791, 30782, 30786, 30787, 30783, 30648, 30655, 30656, 30651, 30652, - 30657, 30658, 30649, 30653, 30654, 30650, 30571, 30578, 30579, 30574, - 30575, 30580, 30581, 30572, 30576, 30577, 30573, 30770, 30777, 30778, - 30773, 30774, 30779, 30780, 30771, 30775, 30776, 30772, 30549, 30556, - 30557, 30552, 30553, 30558, 30559, 30550, 30554, 30555, 30551, 30560, - 30567, 30568, 30563, 30564, 30569, 30570, 30561, 30565, 30566, 30562, - 30637, 30644, 30645, 30640, 30641, 30646, 30647, 30638, 30642, 30643, - 30639, 30582, 30589, 30590, 30585, 30586, 30591, 30592, 30583, 30587, - 30588, 30584, 30737, 30744, 30745, 30740, 30741, 30746, 30747, 30738, - 30742, 30743, 30739, 30659, 30667, 30668, 30662, 30663, 30669, 30670, - 30660, 30664, 30665, 30661, 30671, 30678, 30679, 30674, 30675, 30680, - 30681, 30672, 30676, 30677, 30673, 30682, 30689, 30690, 30685, 30686, - 30691, 30692, 30683, 30687, 30688, 30684, 30726, 30733, 30734, 30729, - 30730, 30735, 30736, 30727, 30731, 30732, 30728, 30526, 30527, 30534, - 30535, 30530, 30531, 30536, 30537, 30528, 30532, 30533, 30529, 30666, - 28756, 28754, 28755, 13411, 17710, 17708, 17711, 17709, 17700, 17706, - 17704, 17707, 17705, 17701, 17721, 17717, 17722, 17718, 17702, 17719, - 17715, 17720, 17716, 17703, 17732, 17712, 17714, 17713, 17728, 17731, - 17729, 17724, 17730, 17725, 17726, 17727, 17733, 17723, 17872, 17749, - 17750, 17751, 17748, 17867, 17859, 15860, 15862, 15864, 15861, 15863, - 16853, 16855, 16857, 16854, 16856, 16846, 16845, 16844, 16847, 23097, - 23102, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, + 275, 287, 269, 285, 273, 272, 25151, 29947, 25272, 40951, 40951, 40951, + 40951, 19684, 25449, 18041, 31741, 30655, 3849, 3927, 3889, 3823, 3913, + 26961, 4251, 19786, 38863, 17364, 39190, 32226, 3921, 3914, 24416, 26986, + 4252, 19783, 38864, 17365, 39272, 39274, 34335, 3919, 3937, 3871, 39205, + 39206, 10835, 3920, 3938, 3872, 39243, 36993, 24418, 26987, 4253, 38868, + 38865, 17366, 36991, 24411, 26979, 4247, 19757, 38860, 17361, 24404, + 26967, 4250, 19732, 38858, 17363, 24412, 26978, 4248, 19761, 38861, + 17362, 24402, 26984, 4249, 19726, 38857, 24414, 26981, 37011, 26980, + 24406, 26966, 17508, 26965, 31868, 24403, 19731, 26977, 19760, 33758, + 26983, 19724, 38856, 19722, 24413, 19779, 19778, 6807, 29003, 6817, + 29004, 29285, 40951, 40951, 40951, 40951, 40951, 40951, 22664, 22732, + 22660, 22728, 22655, 22731, 22659, 22666, 22733, 22661, 22729, 22656, + 40951, 40951, 40951, 40951, 19729, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 24209, 37904, 32072, 11315, 24216, 37901, 32079, 11312, 24203, 37900, + 32069, 11311, 40951, 40951, 40951, 40951, 24208, 37903, 32071, 11314, + 24218, 37905, 32081, 11316, 19740, 19781, 19754, 19718, 19739, 19780, + 19753, 19717, 24253, 37938, 32132, 11335, 24252, 37937, 32131, 11334, + 24251, 37934, 32130, 11331, 24255, 37940, 32134, 11337, 24254, 37939, + 32133, 11336, 24283, 37915, 32098, 11351, 24291, 37930, 32121, 11366, + 24293, 37926, 32154, 11362, 24243, 37924, 32112, 11360, 24244, 37925, + 32113, 11361, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 24292, 37929, 32122, 11365, 29406, 29379, 33977, 33990, 24106, 37771, + 40951, 40951, 40951, 40951, 40951, 40951, 39310, 39325, 39315, 39320, + 39335, 39330, 39340, 39345, 39312, 39327, 39317, 39322, 39337, 39332, + 39342, 39347, 39311, 39326, 39316, 39321, 39336, 39331, 39341, 39346, + 39308, 39323, 39313, 39318, 39333, 39328, 39338, 39343, 39309, 39324, + 39314, 39319, 39334, 39329, 39339, 39344, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 24259, 37944, 32138, 11342, 24273, 37954, + 32153, 11347, 24217, 37902, 32080, 11313, 19695, 19698, 19697, 19696, + 24227, 32087, 24262, 32123, 24284, 32118, 24288, 32114, 24226, 32086, + 24282, 32097, 39154, 39153, 40951, 40951, 2517, 2514, 32067, 11326, + 29034, 29035, 29042, 29036, 29398, 29370, 33969, 33982, 40951, 40951, + 40951, 40951, 24223, 32058, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 6535, 6536, 6534, 24071, 24069, + 24070, 24072, 24068, 11304, 11306, 11305, 11307, 31467, 39226, 4940, + 31468, 40760, 28003, 17406, 29283, 36994, 17389, 32235, 20454, 33626, + 5261, 31771, 24180, 32021, 19113, 19109, 20824, 17387, 8051, 28959, + 32179, 11379, 25330, 17414, 33868, 17397, 18666, 18665, 17408, 32659, + 33859, 17394, 32845, 31618, 4928, 31059, 32667, 31672, 25576, 28189, + 32850, 31301, 20930, 17437, 27604, 39375, 39129, 19111, 11001, 39349, + 11380, 7995, 37689, 34110, 18010, 32168, 17457, 32562, 36995, 4548, + 25738, 9949, 22565, 33881, 17485, 8695, 2597, 9956, 2615, 31603, 6064, + 2618, 18662, 32671, 34526, 16648, 18005, 31288, 22554, 31060, 11526, + 17500, 35368, 6512, 4370, 9942, 8055, 4939, 31478, 31668, 9957, 32441, + 6000, 24035, 25665, 28183, 2619, 33860, 39396, 33862, 17399, 17412, + 30873, 17521, 29286, 10925, 17417, 17401, 32531, 22563, 18040, 20392, + 17466, 8702, 25106, 32536, 37665, 37757, 11541, 11527, 3470, 32768, + 30884, 17513, 5005, 10830, 18039, 25107, 31903, 32849, 34480, 17908, + 40754, 20283, 32527, 34927, 8681, 21216, 25336, 31285, 20394, 31217, + 31749, 25268, 28187, 27576, 2617, 34740, 25573, 11371, 33790, 30830, + 30555, 33768, 17471, 30879, 3538, 3766, 32558, 18676, 31685, 16739, + 16740, 16742, 16741, 4554, 24400, 17491, 37450, 34733, 34734, 32370, + 11534, 28192, 25662, 26861, 26862, 6300, 9944, 32373, 3655, 17717, 30529, + 17424, 38977, 5004, 26825, 20467, 4942, 37574, 34502, 22558, 10829, + 17391, 101, 6478, 30516, 3534, 31309, 31303, 31312, 31302, 25340, 17430, + 38517, 27409, 16737, 17903, 40946, 4926, 30554, 3758, 32533, 18008, 8677, + 33876, 31792, 17452, 20933, 36767, 31322, 11528, 8457, 2599, 17449, + 37452, 4935, 25339, 25271, 25150, 34109, 2761, 32371, 36655, 4941, 3431, + 32178, 3429, 34113, 31778, 28961, 29066, 29077, 29080, 29069, 29059, + 29074, 39165, 3791, 29060, 39162, 39178, 39181, 39157, 39170, 39175, + 3788, 3804, 3807, 3783, 3796, 3801, 29067, 29078, 29081, 29070, 29065, + 29075, 39166, 3792, 29061, 39184, 39187, 39188, 39183, 39185, 39186, + 3810, 3813, 3814, 3809, 3811, 3812, 29084, 29087, 29088, 29083, 29085, + 29086, 39167, 3793, 29062, 39163, 39179, 39182, 39158, 39168, 39176, + 3789, 3805, 3808, 3784, 3794, 3802, 29068, 29079, 29082, 29071, 29063, + 29076, 39169, 3795, 29064, 39159, 3785, 29072, 39160, 3786, 29073, 39172, + 39173, 39171, 3798, 3799, 3797, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 39391, 39394, 39390, 39392, + 39389, 39388, 39393, 39384, 39387, 39383, 39385, 39382, 39381, 39386, + 40951, 40951, 2762, 30528, 4934, 32843, 36997, 24415, 18663, 31472, + 11377, 99, 34507, 39380, 8698, 40951, 40951, 40951, 40668, 22557, 31068, + 4258, 25338, 31473, 29056, 25668, 17480, 19656, 40951, 40951, 40951, + 40951, 40951, 32846, 32163, 6135, 31776, 2602, 11004, 3427, 27583, 1, + 25135, 8676, 6062, 32539, 22567, 20447, 27599, 39352, 31586, 32664, + 22559, 5008, 28200, 37451, 19686, 31481, 32173, 27600, 20473, 25156, + 19107, 17489, 18977, 21696, 17481, 39369, 3541, 8053, 31605, 39371, + 17432, 25152, 8650, 16751, 29057, 20464, 20926, 39355, 24033, 18042, 956, + 25273, 31324, 31620, 31619, 31307, 17445, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 28186, 11189, 4246, 3544, 30518, 17467, 35784, + 17512, 36654, 31607, 3539, 20925, 17905, 31289, 32216, 40951, 40951, + 34108, 27067, 32375, 17396, 17400, 17411, 11199, 3763, 4943, 32819, + 17407, 40951, 40951, 40951, 40951, 40951, 40951, 19110, 32111, 24242, + 31025, 31026, 20634, 19693, 24287, 32117, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 4182, 4212, 4183, 4227, 4198, 4216, 4184, 4235, + 4205, 4213, 4191, 4228, 4199, 4217, 4185, 4239, 4209, 4224, 4195, 4232, + 4221, 4188, 4236, 4206, 4214, 4192, 4229, 4200, 4218, 4186, 4241, 4211, + 4226, 4197, 4234, 4204, 4223, 4190, 4238, 4208, 4194, 4231, 4202, 4220, + 4187, 4240, 4210, 4225, 4196, 4233, 4203, 4222, 4189, 4237, 4207, 4215, + 4193, 4230, 4201, 4219, 25173, 25174, 25181, 25183, 25178, 25239, 25240, + 25236, 25238, 25234, 25237, 25230, 25233, 25231, 25235, 25232, 25177, + 25180, 25175, 25179, 25176, 25182, 37854, 37855, 37862, 37864, 37859, + 37824, 37825, 37821, 37823, 37819, 37822, 37815, 37818, 37816, 37820, + 37817, 37858, 37861, 37856, 37860, 37857, 37863, 37797, 24036, 37794, + 24041, 24127, 37893, 32004, 25265, 38841, 38842, 38843, 38844, 38845, + 38846, 20409, 20410, 20411, 20412, 20413, 20414, 24037, 24042, 31917, + 31916, 37796, 20408, 37852, 37889, 37804, 37892, 37888, 31966, 32000, + 31944, 31999, 31976, 24085, 31959, 37811, 25166, 20809, 37806, 37808, + 40951, 24084, 6298, 20805, 19733, 37829, 37884, 37795, 24038, 37830, + 37885, 25226, 25203, 4454, 4458, 4446, 4452, 4455, 4460, 4447, 4449, + 4457, 4459, 4461, 4456, 4450, 4451, 4477, 4487, 2520, 20806, 24079, + 31954, 20807, 24196, 32055, 11321, 37897, 24076, 31951, 38951, 31968, + 29006, 29005, 29007, 39229, 24130, 27578, 31995, 29037, 34508, 34509, + 34511, 34512, 34510, 39297, 39202, 31767, 3903, 24137, 24092, 4453, 4474, + 4469, 4448, 4464, 4463, 4471, 4462, 4467, 4473, 4444, 4468, 4465, 4475, + 4445, 4470, 37476, 31962, 4364, 24151, 37803, 25250, 27579, 27580, 37475, + 31961, 4363, 24150, 37485, 4350, 4357, 37477, 32572, 32574, 32571, 32570, + 32567, 32566, 32569, 32568, 32575, 32573, 40951, 40951, 40951, 40951, + 40951, 40951, 6854, 6855, 6856, 6857, 6858, 6859, 6860, 6861, 6862, 6863, + 6864, 6865, 6866, 6867, 6868, 6869, 6870, 6871, 6872, 6873, 6874, 6875, + 6876, 6877, 6878, 6879, 6880, 6881, 6882, 6883, 6884, 6885, 6886, 6887, + 6888, 6889, 6890, 6891, 6892, 6893, 6894, 6895, 6896, 6897, 6898, 6899, + 6900, 6901, 6902, 6903, 6904, 6905, 6906, 6907, 6908, 6909, 6910, 6911, + 6912, 6913, 6914, 6915, 6916, 6917, 6918, 6919, 6920, 6921, 6922, 6923, + 6924, 6925, 6926, 6927, 6928, 6929, 6930, 6931, 6932, 6933, 6934, 6935, + 6936, 6937, 6938, 6939, 6940, 6941, 6942, 6943, 6944, 6945, 6946, 6947, + 6948, 6949, 6950, 6951, 6952, 6953, 6954, 6955, 6956, 6957, 6958, 6959, + 6960, 6961, 6962, 6963, 6964, 6965, 6966, 6967, 6968, 6969, 6970, 6971, + 6972, 6973, 6974, 6975, 6976, 6977, 6978, 6979, 6980, 6981, 6982, 6983, + 6984, 6985, 6986, 6987, 6988, 6989, 6990, 6991, 6992, 6993, 6994, 6995, + 6996, 6997, 6998, 6999, 7000, 7001, 7002, 7003, 7004, 7005, 7006, 7007, + 7008, 7009, 7010, 7011, 7012, 7013, 7014, 7015, 7016, 7017, 7018, 7019, + 7020, 7021, 7022, 7023, 7024, 7025, 7026, 7027, 7028, 7029, 7030, 7031, + 7032, 7033, 7034, 7035, 7036, 7037, 7038, 7039, 7040, 7041, 7042, 7043, + 7044, 7045, 7046, 7047, 7048, 7049, 7050, 7051, 7052, 7053, 7054, 7055, + 7056, 7057, 7058, 7059, 7060, 7061, 7062, 7063, 7064, 7065, 7066, 7067, + 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7076, 7077, 7078, 7079, + 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, + 7092, 7093, 7094, 7095, 7096, 7097, 7098, 7099, 7100, 7101, 7102, 7103, + 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7113, 7114, 7115, + 7116, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, + 7128, 7129, 7130, 7131, 7132, 7133, 7134, 7135, 7136, 7137, 7138, 7139, + 7140, 7141, 7142, 7143, 7144, 7145, 7146, 7147, 7148, 7149, 7150, 7151, + 7152, 7153, 7154, 7155, 7156, 7157, 7158, 7159, 7160, 7161, 7162, 7163, + 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7175, + 7176, 7177, 7178, 7179, 7180, 7181, 7182, 7183, 7184, 7185, 7186, 7187, + 7188, 7189, 7190, 7191, 7192, 7193, 7194, 7195, 7196, 7197, 7198, 7199, + 7200, 7201, 7202, 7203, 7204, 7205, 7206, 7207, 7208, 7209, 7210, 7211, + 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, 7220, 7221, 7222, 7223, + 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, 7232, 7233, 7234, 7235, + 7236, 7237, 7238, 7239, 7240, 7241, 7242, 7243, 7244, 7245, 7246, 7247, + 7248, 7249, 7250, 7251, 7252, 7253, 7254, 7255, 7256, 7257, 7258, 7259, + 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, 7268, 7269, 7270, 7271, + 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, 7280, 7281, 7282, 7283, + 7284, 7285, 7286, 7287, 7288, 7289, 7290, 7291, 7292, 7293, 7294, 7295, + 7296, 7297, 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305, 7306, 7307, + 7308, 7309, 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, + 7320, 7321, 7322, 7323, 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, + 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, + 7344, 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7355, + 7356, 7357, 7358, 7359, 7360, 7361, 7362, 7363, 7364, 7365, 6838, 6839, + 6840, 6841, 6842, 6843, 6844, 6845, 6846, 6847, 6848, 6849, 6850, 6851, + 6852, 6853, 6824, 6825, 6826, 6827, 6828, 6829, 6830, 6831, 6832, 6833, + 6834, 6835, 6836, 6837, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 24032, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 35020, 34949, 35012, 35021, 34937, 35010, 34931, 34930, 35007, + 35015, 34932, 35011, 34935, 34952, 35023, 35019, 34944, 34946, 34943, + 34942, 34939, 34938, 34941, 34940, 34947, 34945, 34936, 35018, 35005, + 34948, 34951, 35013, 34934, 34953, 34954, 34955, 34956, 34957, 34958, + 34959, 34960, 34961, 34962, 34963, 34964, 34965, 34966, 34967, 34968, + 34969, 34970, 34971, 34972, 34973, 34974, 34975, 34976, 34977, 34978, + 35008, 35017, 35016, 34933, 35009, 34950, 34979, 34980, 34981, 34982, + 34983, 34984, 34985, 34986, 34987, 34988, 34989, 34990, 34991, 34992, + 34993, 34994, 34995, 34996, 34997, 34998, 34999, 35000, 35001, 35002, + 35003, 35004, 35006, 35024, 35014, 35022, 5995, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 38339, 38350, 38361, 38373, 38384, + 38395, 38406, 38417, 38428, 38436, 38437, 38438, 38439, 38441, 38442, + 38443, 38444, 38445, 38446, 38447, 38448, 38449, 38450, 38452, 38453, + 38454, 38455, 38456, 38457, 38458, 38459, 38460, 38461, 38463, 38464, + 38465, 38466, 38467, 38468, 38469, 38470, 38471, 38472, 38474, 38475, + 38476, 38477, 38478, 38479, 38480, 38481, 38482, 38483, 38485, 38486, + 38487, 38488, 38489, 38490, 38491, 38492, 38493, 38494, 38496, 38497, + 38498, 38499, 38500, 38501, 38502, 38503, 38504, 38505, 38507, 38508, + 38509, 38510, 38511, 38512, 38513, 38514, 38515, 38516, 38263, 38264, + 38265, 38266, 38267, 38268, 38269, 38270, 38271, 38272, 38274, 38275, + 38276, 38277, 38278, 38279, 38280, 38281, 38282, 38283, 38285, 38286, + 38287, 38288, 38289, 38290, 38291, 38292, 38293, 38294, 38296, 38297, + 38298, 38299, 38300, 38301, 38302, 38303, 38304, 38305, 38307, 38308, + 38309, 38310, 38311, 38312, 38313, 38314, 38315, 38316, 38318, 38319, + 38320, 38321, 38322, 38323, 38324, 38325, 38326, 38327, 38329, 38330, + 38331, 38332, 38333, 38334, 38335, 38336, 38337, 38338, 38340, 38341, + 38342, 38343, 38344, 38345, 38346, 38347, 38348, 38349, 38351, 38352, + 38353, 38354, 38355, 38356, 38357, 38358, 38359, 38360, 38362, 38363, + 38364, 38365, 38366, 38367, 38368, 38369, 38370, 38371, 38374, 38375, + 38376, 38377, 38378, 38379, 38380, 38381, 38382, 38383, 38385, 38386, + 38387, 38388, 38389, 38390, 38391, 38392, 38393, 38394, 38396, 38397, + 38398, 38399, 38400, 38401, 38402, 38403, 38404, 38405, 38407, 38408, + 38409, 38410, 38411, 38412, 38413, 38414, 38415, 38416, 38418, 38419, + 38420, 38421, 38422, 38423, 38424, 38425, 38426, 38427, 38429, 38430, + 38431, 38432, 38433, 38434, 38435, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 29854, 29853, 34498, 34077, 34499, 34529, 16774, 17347, 16772, + 16785, 16778, 16777, 3, 2, 347, 3542, 2612, 5258, 6290, 20419, 20449, + 34928, 24495, 29177, 16775, 25337, 29928, 16783, 24497, 38850, 38955, + 17497, 17647, 6063, 8678, 32821, 25111, 33874, 32822, 25110, 32853, + 10543, 11530, 10827, 10544, 10826, 10545, 10825, 10546, 10823, 10547, + 29045, 28963, 34833, 34832, 16773, 17346, 5993, 5266, 16771, 16784, + 16738, 34560, 34530, 16819, 16818, 20706, 17442, 17645, 20707, 18670, + 18976, 20708, 31789, 32368, 20709, 37754, 37960, 34079, 10559, 10556, + 30887, 30886, 20286, 20448, 4927, 5257, 29350, 28965, 20633, 20632, + 29279, 29284, 34495, 34483, 16770, 16822, 6292, 20421, 20451, 6291, + 20420, 20450, 24498, 38851, 38956, 31210, 31209, 31587, 31211, 31212, + 31567, 31869, 31876, 31904, 33639, 33640, 34481, 33638, 33641, 34482, + 10824, 10548, 31676, 31677, 31724, 31675, 31678, 31725, 32662, 34528, + 5994, 10528, 27410, 28789, 34494, 34497, 34080, 16769, 16767, 17384, + 34496, 34078, 33633, 34925, 33632, 32557, 8472, 10527, 34519, 34484, + 30547, 30768, 31674, 31726, 1059, 1060, 28964, 32852, 22893, 23515, + 10526, 2313, 361, 34911, 21230, 22570, 22571, 22613, 22579, 37100, 19375, + 19376, 19353, 19374, 17641, 17642, 17643, 28787, 17644, 34585, 40949, + 40948, 40950, 25333, 32171, 25331, 32169, 31283, 25334, 32172, 29927, + 28788, 39378, 25332, 32170, 17646, 31284, 39147, 27571, 27572, 24248, + 32127, 39885, 28575, 38518, 38629, 38697, 38708, 38719, 38730, 38741, + 38752, 38763, 38519, 38530, 38541, 38552, 38563, 38574, 38585, 31643, + 5256, 4549, 40947, 9642, 9641, 9337, 2827, 26869, 26867, 26927, 26925, + 20199, 5093, 27245, 27248, 38596, 38607, 38618, 38630, 38641, 38652, + 38663, 38674, 38685, 38693, 38694, 38695, 38696, 38698, 38699, 38700, + 38701, 38702, 38703, 38704, 38705, 38706, 38707, 38709, 38710, 38711, + 38712, 38713, 38714, 38715, 38716, 38717, 38718, 38720, 38721, 38722, + 38723, 38724, 38725, 38726, 38727, 38728, 38729, 38731, 38732, 38733, + 38734, 38735, 38736, 38737, 38738, 38739, 38740, 38742, 38743, 38744, + 38745, 38746, 38747, 38748, 38749, 38750, 38751, 38753, 38754, 38755, + 38756, 38757, 38758, 38759, 38760, 38761, 38762, 38764, 38765, 38766, + 38767, 38768, 38769, 38770, 38771, 38772, 38773, 38520, 38521, 38522, + 38523, 38524, 38525, 38526, 38527, 38528, 38529, 38531, 38532, 38533, + 38534, 38535, 38536, 38537, 38538, 38539, 38540, 38542, 38543, 38544, + 38545, 38546, 38547, 38548, 38549, 38550, 38551, 38553, 38554, 38555, + 38556, 38557, 38558, 38559, 38560, 38561, 38562, 38564, 38565, 38566, + 38567, 38568, 38569, 38570, 38571, 38572, 38573, 38575, 38576, 38577, + 38578, 38579, 38580, 38581, 38582, 38583, 38584, 38586, 38587, 38588, + 38589, 38590, 38591, 38592, 38593, 38594, 38595, 38597, 38598, 38599, + 38600, 38601, 38602, 38603, 38604, 38605, 38606, 38608, 38609, 38610, + 38611, 38612, 38613, 38614, 38615, 38616, 38617, 38619, 38620, 38621, + 38622, 38623, 38624, 38625, 38626, 38627, 38628, 38631, 38632, 38633, + 38634, 38635, 38636, 38637, 38638, 38639, 38640, 38642, 38643, 38644, + 38645, 38646, 38647, 38648, 38649, 38650, 38651, 38653, 38654, 38655, + 38656, 38657, 38658, 38659, 38660, 38661, 38662, 38664, 38665, 38666, + 38667, 38668, 38669, 38670, 38671, 38672, 38673, 38675, 38676, 38677, + 38678, 38679, 38680, 38681, 38682, 38683, 38684, 38686, 38687, 38688, + 38689, 38690, 38691, 38692, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 21614, + 21615, 21622, 21624, 21621, 21620, 21617, 21616, 21619, 21618, 21625, + 21623, 22758, 23321, 22917, 23546, 23156, 23919, 22852, 23456, 22853, + 23457, 22854, 23458, 23028, 23697, 23029, 23698, 23030, 23699, 23095, + 23795, 22836, 23439, 22832, 23435, 23541, 23665, 22768, 23331, 22769, + 23332, 22857, 23462, 22858, 23463, 22842, 23445, 22843, 23446, 23540, + 23542, 22922, 23548, 22923, 23549, 22940, 23576, 22970, 23616, 22978, + 23638, 23068, 23756, 23160, 23923, 23161, 23924, 23157, 23920, 23158, + 23921, 23340, 23714, 23715, 23874, 23875, 23804, 23805, 23530, 23531, + 2283, 2286, 2284, 2288, 2290, 2287, 2289, 2285, 2291, 10756, 10751, + 10750, 10757, 10752, 10753, 10755, 10754, 3614, 3613, 3615, 18884, 18883, + 18882, 18881, 18886, 18885, 30619, 30618, 3559, 35374, 35382, 35391, + 35383, 35385, 35380, 35384, 35379, 35395, 35394, 35397, 35389, 35376, + 35396, 35378, 35377, 35390, 35381, 35393, 35387, 35388, 35386, 35392, + 35375, 35513, 35520, 35521, 35516, 35517, 35522, 35523, 35514, 35518, + 35519, 35515, 35579, 35586, 35587, 35582, 35583, 35588, 35589, 35580, + 35584, 35585, 35581, 35690, 35697, 35698, 35693, 35694, 35699, 35700, + 35691, 35695, 35696, 35692, 35590, 35597, 35598, 35593, 35594, 35599, + 35600, 35591, 35595, 35596, 35592, 35668, 35675, 35676, 35671, 35672, + 35677, 35678, 35669, 35673, 35674, 35670, 35568, 35575, 35576, 35571, + 35572, 35577, 35578, 35569, 35573, 35574, 35570, 35679, 35686, 35687, + 35682, 35683, 35688, 35689, 35680, 35684, 35685, 35681, 35601, 35608, + 35609, 35604, 35605, 35610, 35611, 35602, 35606, 35607, 35603, 35734, + 35741, 35742, 35737, 35738, 35743, 35744, 35735, 35739, 35740, 35736, + 35723, 35730, 35731, 35726, 35727, 35732, 35733, 35724, 35728, 35729, + 35725, 35756, 35763, 35764, 35759, 35760, 35765, 35766, 35757, 35761, + 35762, 35758, 35623, 35630, 35631, 35626, 35627, 35632, 35633, 35624, + 35628, 35629, 35625, 35546, 35553, 35554, 35549, 35550, 35555, 35556, + 35547, 35551, 35552, 35548, 35745, 35752, 35753, 35748, 35749, 35754, + 35755, 35746, 35750, 35751, 35747, 35524, 35531, 35532, 35527, 35528, + 35533, 35534, 35525, 35529, 35530, 35526, 35535, 35542, 35543, 35538, + 35539, 35544, 35545, 35536, 35540, 35541, 35537, 35612, 35619, 35620, + 35615, 35616, 35621, 35622, 35613, 35617, 35618, 35614, 35557, 35564, + 35565, 35560, 35561, 35566, 35567, 35558, 35562, 35563, 35559, 35712, + 35719, 35720, 35715, 35716, 35721, 35722, 35713, 35717, 35718, 35714, + 35634, 35642, 35643, 35637, 35638, 35644, 35645, 35635, 35639, 35640, + 35636, 35646, 35653, 35654, 35649, 35650, 35655, 35656, 35647, 35651, + 35652, 35648, 35657, 35664, 35665, 35660, 35661, 35666, 35667, 35658, + 35662, 35663, 35659, 35701, 35708, 35709, 35704, 35705, 35710, 35711, + 35702, 35706, 35707, 35703, 35501, 35502, 35509, 35510, 35505, 35506, + 35511, 35512, 35503, 35507, 35508, 35504, 35641, 33664, 33662, 33663, + 17807, 22179, 22177, 22180, 22178, 22169, 22175, 22173, 22176, 22174, + 22170, 22190, 22186, 22191, 22187, 22171, 22188, 22184, 22189, 22185, + 22172, 22201, 22181, 22183, 22182, 22197, 22200, 22198, 22193, 22199, + 22194, 22195, 22196, 22202, 22192, 22341, 22218, 22219, 22220, 22217, + 22336, 22328, 20316, 20318, 20320, 20317, 20319, 21319, 21321, 21323, + 21320, 21322, 21312, 21311, 21310, 21313, 27780, 27785, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, }; static const unsigned int aliases_start = 0xf0000; -static const unsigned int aliases_end = 0xf01d9; +static const unsigned int aliases_end = 0xf01dd; static const unsigned int name_aliases[] = { 0x0000, 0x0000, @@ -17397,12 +18751,16 @@ static const unsigned int name_aliases[] = { 0xFEFF, 0x122D4, 0x122D5, + 0x12327, + 0x1680B, 0x16E56, 0x16E57, 0x16E76, 0x16E77, 0x1B001, 0x1D0C5, + 0x1E899, + 0x1E89A, 0xE0100, 0xE0101, 0xE0102, diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 0bb5e12d7c3dd9..d86741e1dfc18c 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -62,11 +62,10 @@ pass */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -396,6 +395,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 754a368f77e940..1063e54217b746 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -5,10 +5,10 @@ * See the xxlimited module for an extension module template. */ +// Test the limited C API version 3.5 #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x03050000 +# define Py_LIMITED_API 0x03050000 #endif #include "Python.h" @@ -297,6 +297,10 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, +#ifdef Py_GIL_DISABLED + // These definitions are in the limited API, but not until 3.13. + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif {0, NULL} }; diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index 1e4e0ea3743ce3..a46bf8f0e64ee2 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -384,6 +384,7 @@ xx_exec(PyObject *m) static struct PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 560f43e5b3a643..9c548f44558d41 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -288,6 +288,7 @@ xxsubtype_exec(PyObject* m) static struct PyModuleDef_Slot xxsubtype_slots[] = { {Py_mod_exec, xxsubtype_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index fe9a6d8d4150ab..78dcce73cdaade 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -267,12 +267,12 @@ newcompobject(PyTypeObject *type) self->eof = 0; self->is_initialised = 0; self->zdict = NULL; - self->unused_data = PyBytes_FromStringAndSize("", 0); + self->unused_data = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (self->unused_data == NULL) { Py_DECREF(self); return NULL; } - self->unconsumed_tail = PyBytes_FromStringAndSize("", 0); + self->unconsumed_tail = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (self->unconsumed_tail == NULL) { Py_DECREF(self); return NULL; @@ -489,8 +489,8 @@ zlib_decompress_impl(PyObject *module, Py_buffer *data, int wbits, Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; case Z_MEM_ERROR: @@ -915,8 +915,8 @@ zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; default: @@ -1293,8 +1293,8 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; default: @@ -1495,8 +1495,8 @@ decompress_buf(ZlibDecompressor *self, Py_ssize_t max_length) err = inflate(&self->zst, Z_SYNC_FLUSH); Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; default: @@ -2106,6 +2106,7 @@ zlib_exec(PyObject *mod) static PyModuleDef_Slot zlib_slots[] = { {Py_mod_exec, zlib_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Objects/abstract.c b/Objects/abstract.c index 07d4b89fe188c8..f6647874d732f6 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -862,7 +862,7 @@ PyObject_Format(PyObject *obj, PyObject *format_spec) /* If no format_spec is provided, use an empty string */ if (format_spec == NULL) { - empty = PyUnicode_New(0, 0); + empty = Py_GetConstant(Py_CONSTANT_EMPTY_STR); format_spec = empty; } @@ -1000,20 +1000,6 @@ binary_op(PyObject *v, PyObject *w, const int op_slot, const char *op_name) PyObject *result = BINARY_OP1(v, w, op_slot, op_name); if (result == Py_NotImplemented) { Py_DECREF(result); - - if (op_slot == NB_SLOT(nb_rshift) && - PyCFunction_CheckExact(v) && - strcmp(((PyCFunctionObject *)v)->m_ml->ml_name, "print") == 0) - { - PyErr_Format(PyExc_TypeError, - "unsupported operand type(s) for %.100s: " - "'%.100s' and '%.100s'. Did you mean \"print(, " - "file=)\"?", - op_name, - Py_TYPE(v)->tp_name, - Py_TYPE(w)->tp_name); - return NULL; - } return binop_type_error(v, w, op_name); } return result; @@ -1390,7 +1376,7 @@ _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs) } UNARY_FUNC(PyNumber_Negative, nb_negative, __neg__, "unary -") -UNARY_FUNC(PyNumber_Positive, nb_positive, __pow__, "unary +") +UNARY_FUNC(PyNumber_Positive, nb_positive, __pos__, "unary +") UNARY_FUNC(PyNumber_Invert, nb_invert, __invert__, "unary ~") UNARY_FUNC(PyNumber_Absolute, nb_absolute, __abs__, "abs()") @@ -1521,7 +1507,6 @@ PyNumber_Long(PyObject *o) { PyObject *result; PyNumberMethods *m; - PyObject *trunc_func; Py_buffer view; if (o == NULL) { @@ -1563,37 +1548,6 @@ PyNumber_Long(PyObject *o) if (m && m->nb_index) { return PyNumber_Index(o); } - trunc_func = _PyObject_LookupSpecial(o, &_Py_ID(__trunc__)); - if (trunc_func) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "The delegation of int() to __trunc__ is deprecated.", 1)) { - Py_DECREF(trunc_func); - return NULL; - } - result = _PyObject_CallNoArgs(trunc_func); - Py_DECREF(trunc_func); - if (result == NULL || PyLong_CheckExact(result)) { - return result; - } - if (PyLong_Check(result)) { - Py_SETREF(result, _PyLong_Copy((PyLongObject *)result)); - return result; - } - /* __trunc__ is specified to return an Integral type, - but int() needs to return an int. */ - if (!PyIndex_Check(result)) { - PyErr_Format( - PyExc_TypeError, - "__trunc__ returned non-Integral (type %.200s)", - Py_TYPE(result)->tp_name); - Py_DECREF(result); - return NULL; - } - Py_SETREF(result, PyNumber_Index(result)); - return result; - } - if (PyErr_Occurred()) - return NULL; if (PyUnicode_Check(o)) /* The below check is done in PyLong_FromUnicodeObject(). */ @@ -2173,7 +2127,7 @@ PySequence_Fast(PyObject *v, const char *m) PY_ITERSEARCH_COUNT: -1 if error, else # of times obj appears in seq. PY_ITERSEARCH_INDEX: 0-based index of first occurrence of obj in seq; set ValueError and return -1 if none found; also return -1 on error. - Py_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on error. + PY_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on error. */ Py_ssize_t _PySequence_IterSearch(PyObject *seq, PyObject *obj, int operation) @@ -2190,7 +2144,15 @@ _PySequence_IterSearch(PyObject *seq, PyObject *obj, int operation) it = PyObject_GetIter(seq); if (it == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - type_error("argument of type '%.200s' is not iterable", seq); + if (operation == PY_ITERSEARCH_CONTAINS) { + type_error( + "argument of type '%.200s' is not a container or iterable", + seq + ); + } + else { + type_error("argument of type '%.200s' is not iterable", seq); + } } return -1; } @@ -2905,7 +2867,50 @@ PyAIter_Check(PyObject *obj) tp->tp_as_async->am_anext != &_PyObject_NextNotImplemented); } +static int +iternext(PyObject *iter, PyObject **item) +{ + iternextfunc tp_iternext = Py_TYPE(iter)->tp_iternext; + if ((*item = tp_iternext(iter))) { + return 1; + } + + PyThreadState *tstate = _PyThreadState_GET(); + /* When the iterator is exhausted it must return NULL; + * a StopIteration exception may or may not be set. */ + if (!_PyErr_Occurred(tstate)) { + return 0; + } + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + _PyErr_Clear(tstate); + return 0; + } + + /* Error case: an exception (different than StopIteration) is set. */ + return -1; +} + +/* Return 1 and set 'item' to the next item of 'iter' on success. + * Return 0 and set 'item' to NULL when there are no remaining values. + * Return -1, set 'item' to NULL and set an exception on error. + */ +int +PyIter_NextItem(PyObject *iter, PyObject **item) +{ + assert(iter != NULL); + assert(item != NULL); + + if (Py_TYPE(iter)->tp_iternext == NULL) { + *item = NULL; + PyErr_Format(PyExc_TypeError, "expected an iterator, got '%T'", iter); + return -1; + } + + return iternext(iter, item); +} + /* Return next item. + * * If an error occurs, return NULL. PyErr_Occurred() will be true. * If the iteration terminates normally, return NULL and clear the * PyExc_StopIteration exception (if it was set). PyErr_Occurred() @@ -2915,17 +2920,9 @@ PyAIter_Check(PyObject *obj) PyObject * PyIter_Next(PyObject *iter) { - PyObject *result; - result = (*Py_TYPE(iter)->tp_iternext)(iter); - if (result == NULL) { - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyErr_Occurred(tstate) - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - { - _PyErr_Clear(tstate); - } - } - return result; + PyObject *item; + (void)iternext(iter, &item); + return item; } PySendResult diff --git a/Objects/boolobject.c b/Objects/boolobject.c index fb48dcbeca7850..a88a8ad0cfd560 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -71,8 +71,8 @@ static PyObject * bool_invert(PyObject *v) { if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Bitwise inversion '~' on bool is deprecated. This " - "returns the bitwise inversion of the underlying int " + "Bitwise inversion '~' on bool is deprecated and will be removed in " + "Python 3.16. This returns the bitwise inversion of the underlying int " "object and is usually not what you expect from negating " "a bool. Use the 'not' operator for boolean negation or " "~int(x) if you really want the bitwise inversion of the " diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index acc59b926448ca..fd2a85a3fe0a61 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -42,15 +42,16 @@ _getbytevalue(PyObject* arg, int *value) } static int -bytearray_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags) +bytearray_getbuffer(PyObject *self, Py_buffer *view, int flags) { - void *ptr; + PyByteArrayObject *obj = _PyByteArray_CAST(self); if (view == NULL) { PyErr_SetString(PyExc_BufferError, "bytearray_getbuffer: view==NULL argument is obsolete"); return -1; } - ptr = (void *) PyByteArray_AS_STRING(obj); + + void *ptr = (void *) PyByteArray_AS_STRING(obj); /* cannot fail if view != NULL and readonly == 0 */ (void)PyBuffer_FillInfo(view, (PyObject*)obj, ptr, Py_SIZE(obj), 0, flags); obj->ob_exports++; @@ -58,8 +59,9 @@ bytearray_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags) } static void -bytearray_releasebuffer(PyByteArrayObject *obj, Py_buffer *view) +bytearray_releasebuffer(PyObject *self, Py_buffer *view) { + PyByteArrayObject *obj = _PyByteArray_CAST(self); obj->ob_exports--; assert(obj->ob_exports >= 0); } @@ -286,46 +288,53 @@ PyByteArray_Concat(PyObject *a, PyObject *b) /* Functions stuffed into the type object */ static Py_ssize_t -bytearray_length(PyByteArrayObject *self) +bytearray_length(PyObject *op) { + PyByteArrayObject *self = _PyByteArray_CAST(op); return Py_SIZE(self); } static PyObject * -bytearray_iconcat(PyByteArrayObject *self, PyObject *other) +bytearray_iconcat(PyObject *op, PyObject *other) { - Py_ssize_t size; - Py_buffer vo; + PyByteArrayObject *self = _PyByteArray_CAST(op); + Py_buffer vo; if (PyObject_GetBuffer(other, &vo, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", Py_TYPE(other)->tp_name, Py_TYPE(self)->tp_name); return NULL; } - size = Py_SIZE(self); + Py_ssize_t size = Py_SIZE(self); if (size > PY_SSIZE_T_MAX - vo.len) { PyBuffer_Release(&vo); return PyErr_NoMemory(); } + if (PyByteArray_Resize((PyObject *)self, size + vo.len) < 0) { PyBuffer_Release(&vo); return NULL; } + memcpy(PyByteArray_AS_STRING(self) + size, vo.buf, vo.len); PyBuffer_Release(&vo); return Py_NewRef(self); } static PyObject * -bytearray_repeat(PyByteArrayObject *self, Py_ssize_t count) +bytearray_repeat(PyObject *op, Py_ssize_t count) { - if (count < 0) + PyByteArrayObject *self = _PyByteArray_CAST(op); + if (count < 0) { count = 0; + } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) + if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { return PyErr_NoMemory(); + } Py_ssize_t size = mysize * count; + PyByteArrayObject* result = (PyByteArrayObject *)PyByteArray_FromStringAndSize(NULL, size); const char* buf = PyByteArray_AS_STRING(self); if (result != NULL && size != 0) { @@ -335,20 +344,24 @@ bytearray_repeat(PyByteArrayObject *self, Py_ssize_t count) } static PyObject * -bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count) +bytearray_irepeat(PyObject *op, Py_ssize_t count) { - if (count < 0) + PyByteArrayObject *self = _PyByteArray_CAST(op); + if (count < 0) { count = 0; + } else if (count == 1) { return Py_NewRef(self); } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) + if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { return PyErr_NoMemory(); + } const Py_ssize_t size = mysize * count; - if (PyByteArray_Resize((PyObject *)self, size) < 0) + if (PyByteArray_Resize((PyObject *)self, size) < 0) { return NULL; + } char* buf = PyByteArray_AS_STRING(self); _PyBytes_Repeat(buf, size, buf, mysize); @@ -357,8 +370,9 @@ bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count) } static PyObject * -bytearray_getitem(PyByteArrayObject *self, Py_ssize_t i) +bytearray_getitem(PyObject *op, Py_ssize_t i) { + PyByteArrayObject *self = _PyByteArray_CAST(op); if (i < 0 || i >= Py_SIZE(self)) { PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); return NULL; @@ -367,8 +381,9 @@ bytearray_getitem(PyByteArrayObject *self, Py_ssize_t i) } static PyObject * -bytearray_subscript(PyByteArrayObject *self, PyObject *index) +bytearray_subscript(PyObject *op, PyObject *index) { + PyByteArrayObject *self = _PyByteArray_CAST(op); if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -559,12 +574,13 @@ bytearray_setslice(PyByteArrayObject *self, Py_ssize_t lo, Py_ssize_t hi, } static int -bytearray_setitem(PyByteArrayObject *self, Py_ssize_t i, PyObject *value) +bytearray_setitem(PyObject *op, Py_ssize_t i, PyObject *value) { - int ival = -1; + PyByteArrayObject *self = _PyByteArray_CAST(op); // GH-91153: We need to do this *before* the size check, in case value has a // nasty __index__ method that changes the size of the bytearray: + int ival = -1; if (value && !_getbytevalue(value, &ival)) { return -1; } @@ -588,11 +604,11 @@ bytearray_setitem(PyByteArrayObject *self, Py_ssize_t i, PyObject *value) } static int -bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *values) +bytearray_ass_subscript(PyObject *op, PyObject *index, PyObject *values) { - Py_ssize_t start, stop, step, slicelen, needed; - char *buf, *bytes; - buf = PyByteArray_AS_STRING(self); + PyByteArrayObject *self = _PyByteArray_CAST(op); + Py_ssize_t start, stop, step, slicelen; + char *buf = PyByteArray_AS_STRING(self); if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -645,6 +661,8 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu return -1; } + char *bytes; + Py_ssize_t needed; if (values == NULL) { bytes = NULL; needed = 0; @@ -661,7 +679,7 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu values = PyByteArray_FromObject(values); if (values == NULL) return -1; - err = bytearray_ass_subscript(self, index, values); + err = bytearray_ass_subscript((PyObject*)self, index, values); Py_DECREF(values); return err; } @@ -670,10 +688,14 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu bytes = PyByteArray_AS_STRING(values); needed = Py_SIZE(values); } + /* Make sure b[5:2] = ... inserts before 5, not before 2. */ if ((step < 0 && start < stop) || (step > 0 && start > stop)) + { stop = start; + } + if (step == 1) { return bytearray_setslice_linear(self, start, stop, bytes, needed); } @@ -785,7 +807,7 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, if (encoded == NULL) return -1; assert(PyBytes_Check(encoded)); - new = bytearray_iconcat(self, encoded); + new = bytearray_iconcat((PyObject*)self, encoded); Py_DECREF(encoded); if (new == NULL) return -1; @@ -926,8 +948,9 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, /* Mostly copied from string_repr, but without the "smart quote" functionality. */ static PyObject * -bytearray_repr(PyByteArrayObject *self) +bytearray_repr(PyObject *op) { + PyByteArrayObject *self = _PyByteArray_CAST(op); const char *className = _PyType_Name(Py_TYPE(self)); const char *quote_prefix = "(b"; const char *quote_postfix = ")"; @@ -1021,7 +1044,7 @@ bytearray_str(PyObject *op) return NULL; } } - return bytearray_repr((PyByteArrayObject*)op); + return bytearray_repr(op); } static PyObject * @@ -1080,8 +1103,9 @@ bytearray_richcompare(PyObject *self, PyObject *other, int op) } static void -bytearray_dealloc(PyByteArrayObject *self) +bytearray_dealloc(PyObject *op) { + PyByteArrayObject *self = _PyByteArray_CAST(op); if (self->ob_exports > 0) { PyErr_SetString(PyExc_SystemError, "deallocated bytearray object has exported buffers"); @@ -1121,16 +1145,44 @@ bytearray_dealloc(PyByteArrayObject *self) #include "stringlib/transmogrify.h" +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +bytearray.find + + sub: object + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Return -1 on failure. +[clinic start generated code]*/ + static PyObject * -bytearray_find(PyByteArrayObject *self, PyObject *args) +bytearray_find_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=413e1cab2ae87da0 input=793dfad803e2952f]*/ { - return _Py_bytes_find(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_find(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytearray.count = bytearray.find + +Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end]. +[clinic start generated code]*/ + static PyObject * -bytearray_count(PyByteArrayObject *self, PyObject *args) +bytearray_count_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=a21ee2692e4f1233 input=4deb529db38deda8]*/ { - return _Py_bytes_count(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_count(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } /*[clinic input] @@ -1162,40 +1214,111 @@ bytearray_copy_impl(PyByteArrayObject *self) PyByteArray_GET_SIZE(self)); } +/*[clinic input] +bytearray.index = bytearray.find + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ + static PyObject * -bytearray_index(PyByteArrayObject *self, PyObject *args) +bytearray_index_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=067a1e78efc672a7 input=8cbaf6836dbd2a9a]*/ { - return _Py_bytes_index(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_index(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytearray.rfind = bytearray.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Return -1 on failure. +[clinic start generated code]*/ + static PyObject * -bytearray_rfind(PyByteArrayObject *self, PyObject *args) +bytearray_rfind_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=51bf886f932b283c input=eaa107468a158423]*/ { - return _Py_bytes_rfind(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_rfind(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytearray.rindex = bytearray.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ + static PyObject * -bytearray_rindex(PyByteArrayObject *self, PyObject *args) +bytearray_rindex_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=38e1cf66bafb08b9 input=81cf49d0af4d5bd0]*/ { - return _Py_bytes_rindex(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_rindex(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } static int bytearray_contains(PyObject *self, PyObject *arg) { - return _Py_bytes_contains(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), arg); + return _Py_bytes_contains(PyByteArray_AS_STRING(self), + PyByteArray_GET_SIZE(self), + arg); } +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +bytearray.startswith + + prefix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytearray. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytearray. + / + +Return True if the bytearray starts with the specified prefix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytearray_startswith(PyByteArrayObject *self, PyObject *args) +bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=a3d9b6d44d3662a6 input=76385e0b376b45c1]*/ { - return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subobj, start, end); } +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +bytearray.endswith + + suffix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytearray. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytearray. + / + +Return True if the bytearray ends with the specified suffix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytearray_endswith(PyByteArrayObject *self, PyObject *args) +bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=e75ea8c227954caa input=9b8baa879aa3d74b]*/ { - return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subobj, start, end); } /*[clinic input] @@ -1729,6 +1852,10 @@ bytearray_extend(PyByteArrayObject *self, PyObject *iterable_of_ints) while ((item = PyIter_Next(it)) != NULL) { if (! _getbytevalue(item, &value)) { + if (PyErr_ExceptionMatches(PyExc_TypeError) && PyUnicode_Check(iterable_of_ints)) { + PyErr_Format(PyExc_TypeError, + "expected iterable of integers; got: 'str'"); + } Py_DECREF(item); Py_DECREF(it); Py_DECREF(bytearray_obj); @@ -2161,31 +2288,30 @@ bytearray_sizeof_impl(PyByteArrayObject *self) } static PySequenceMethods bytearray_as_sequence = { - (lenfunc)bytearray_length, /* sq_length */ - (binaryfunc)PyByteArray_Concat, /* sq_concat */ - (ssizeargfunc)bytearray_repeat, /* sq_repeat */ - (ssizeargfunc)bytearray_getitem, /* sq_item */ + bytearray_length, /* sq_length */ + PyByteArray_Concat, /* sq_concat */ + bytearray_repeat, /* sq_repeat */ + bytearray_getitem, /* sq_item */ 0, /* sq_slice */ - (ssizeobjargproc)bytearray_setitem, /* sq_ass_item */ + bytearray_setitem, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)bytearray_contains, /* sq_contains */ - (binaryfunc)bytearray_iconcat, /* sq_inplace_concat */ - (ssizeargfunc)bytearray_irepeat, /* sq_inplace_repeat */ + bytearray_contains, /* sq_contains */ + bytearray_iconcat, /* sq_inplace_concat */ + bytearray_irepeat, /* sq_inplace_repeat */ }; static PyMappingMethods bytearray_as_mapping = { - (lenfunc)bytearray_length, - (binaryfunc)bytearray_subscript, - (objobjargproc)bytearray_ass_subscript, + bytearray_length, + bytearray_subscript, + bytearray_ass_subscript, }; static PyBufferProcs bytearray_as_buffer = { - (getbufferproc)bytearray_getbuffer, - (releasebufferproc)bytearray_releasebuffer, + bytearray_getbuffer, + bytearray_releasebuffer, }; -static PyMethodDef -bytearray_methods[] = { +static PyMethodDef bytearray_methods[] = { {"__alloc__", (PyCFunction)bytearray_alloc, METH_NOARGS, alloc_doc}, BYTEARRAY_REDUCE_METHODDEF BYTEARRAY_REDUCE_EX_METHODDEF @@ -2196,18 +2322,15 @@ bytearray_methods[] = { STRINGLIB_CENTER_METHODDEF BYTEARRAY_CLEAR_METHODDEF BYTEARRAY_COPY_METHODDEF - {"count", (PyCFunction)bytearray_count, METH_VARARGS, - _Py_count__doc__}, + BYTEARRAY_COUNT_METHODDEF BYTEARRAY_DECODE_METHODDEF - {"endswith", (PyCFunction)bytearray_endswith, METH_VARARGS, - _Py_endswith__doc__}, + BYTEARRAY_ENDSWITH_METHODDEF STRINGLIB_EXPANDTABS_METHODDEF BYTEARRAY_EXTEND_METHODDEF - {"find", (PyCFunction)bytearray_find, METH_VARARGS, - _Py_find__doc__}, + BYTEARRAY_FIND_METHODDEF BYTEARRAY_FROMHEX_METHODDEF BYTEARRAY_HEX_METHODDEF - {"index", (PyCFunction)bytearray_index, METH_VARARGS, _Py_index__doc__}, + BYTEARRAY_INDEX_METHODDEF BYTEARRAY_INSERT_METHODDEF {"isalnum", stringlib_isalnum, METH_NOARGS, _Py_isalnum__doc__}, @@ -2237,16 +2360,15 @@ bytearray_methods[] = { BYTEARRAY_REMOVEPREFIX_METHODDEF BYTEARRAY_REMOVESUFFIX_METHODDEF BYTEARRAY_REVERSE_METHODDEF - {"rfind", (PyCFunction)bytearray_rfind, METH_VARARGS, _Py_rfind__doc__}, - {"rindex", (PyCFunction)bytearray_rindex, METH_VARARGS, _Py_rindex__doc__}, + BYTEARRAY_RFIND_METHODDEF + BYTEARRAY_RINDEX_METHODDEF STRINGLIB_RJUST_METHODDEF BYTEARRAY_RPARTITION_METHODDEF BYTEARRAY_RSPLIT_METHODDEF BYTEARRAY_RSTRIP_METHODDEF BYTEARRAY_SPLIT_METHODDEF BYTEARRAY_SPLITLINES_METHODDEF - {"startswith", (PyCFunction)bytearray_startswith, METH_VARARGS , - _Py_startswith__doc__}, + BYTEARRAY_STARTSWITH_METHODDEF BYTEARRAY_STRIP_METHODDEF {"swapcase", stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, @@ -2294,12 +2416,12 @@ PyTypeObject PyByteArray_Type = { "bytearray", sizeof(PyByteArrayObject), 0, - (destructor)bytearray_dealloc, /* tp_dealloc */ + bytearray_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)bytearray_repr, /* tp_repr */ + bytearray_repr, /* tp_repr */ &bytearray_as_number, /* tp_as_number */ &bytearray_as_sequence, /* tp_as_sequence */ &bytearray_as_mapping, /* tp_as_mapping */ @@ -2314,7 +2436,7 @@ PyTypeObject PyByteArray_Type = { bytearray_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ - (richcmpfunc)bytearray_richcompare, /* tp_richcompare */ + bytearray_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ bytearray_iter, /* tp_iter */ 0, /* tp_iternext */ @@ -2329,7 +2451,7 @@ PyTypeObject PyByteArray_Type = { (initproc)bytearray___init__, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ PyType_GenericNew, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; /*********************** Bytearray Iterator ****************************/ diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index c1bc6383df30ce..c239ae18a593e3 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -92,57 +92,6 @@ _Py_bytes_isalnum(const char *cptr, Py_ssize_t len) } -PyDoc_STRVAR_shared(_Py_isascii__doc__, -"B.isascii() -> bool\n\ -\n\ -Return True if B is empty or all characters in B are ASCII,\n\ -False otherwise."); - -// Optimization is copied from ascii_decode in unicodeobject.c -/* Mask to quickly check whether a C 'size_t' contains a - non-ASCII, UTF8-encoded char. */ -#if (SIZEOF_SIZE_T == 8) -# define ASCII_CHAR_MASK 0x8080808080808080ULL -#elif (SIZEOF_SIZE_T == 4) -# define ASCII_CHAR_MASK 0x80808080U -#else -# error C 'size_t' size should be either 4 or 8! -#endif - -PyObject* -_Py_bytes_isascii(const char *cptr, Py_ssize_t len) -{ - const char *p = cptr; - const char *end = p + len; - - while (p < end) { - /* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h - for an explanation. */ - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { - /* Help allocation */ - const char *_p = p; - while (_p + SIZEOF_SIZE_T <= end) { - size_t value = *(const size_t *) _p; - if (value & ASCII_CHAR_MASK) { - Py_RETURN_FALSE; - } - _p += SIZEOF_SIZE_T; - } - p = _p; - if (_p == end) - break; - } - if ((unsigned char)*p & 0x80) { - Py_RETURN_FALSE; - } - p++; - } - Py_RETURN_TRUE; -} - -#undef ASCII_CHAR_MASK - - PyDoc_STRVAR_shared(_Py_isdigit__doc__, "B.isdigit() -> bool\n\ \n\ @@ -438,6 +387,7 @@ _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to) #include "stringlib/fastsearch.h" #include "stringlib/count.h" #include "stringlib/find.h" +#include "stringlib/find_max_char.h" /* Wraps stringlib_parse_args_finds() and additionally checks the first @@ -453,31 +403,21 @@ stringlib_parse_args_finds(). */ Py_LOCAL_INLINE(int) -parse_args_finds_byte(const char *function_name, PyObject *args, - PyObject **subobj, char *byte, - Py_ssize_t *start, Py_ssize_t *end) +parse_args_finds_byte(const char *function_name, PyObject **subobj, char *byte) { - PyObject *tmp_subobj; - Py_ssize_t ival; - - if(!stringlib_parse_args_finds(function_name, args, &tmp_subobj, - start, end)) - return 0; - - if (PyObject_CheckBuffer(tmp_subobj)) { - *subobj = tmp_subobj; + if (PyObject_CheckBuffer(*subobj)) { return 1; } - if (!_PyIndex_Check(tmp_subobj)) { + if (!_PyIndex_Check(*subobj)) { PyErr_Format(PyExc_TypeError, "argument should be integer or bytes-like object, " "not '%.200s'", - Py_TYPE(tmp_subobj)->tp_name); + Py_TYPE(*subobj)->tp_name); return 0; } - ival = PyNumber_AsSsize_t(tmp_subobj, NULL); + Py_ssize_t ival = PyNumber_AsSsize_t(*subobj, NULL); if (ival == -1 && PyErr_Occurred()) { return 0; } @@ -492,35 +432,40 @@ parse_args_finds_byte(const char *function_name, PyObject *args, } /* helper macro to fixup start/end slice values */ -#define ADJUST_INDICES(start, end, len) \ - if (end > len) \ - end = len; \ - else if (end < 0) { \ - end += len; \ - if (end < 0) \ - end = 0; \ - } \ - if (start < 0) { \ - start += len; \ - if (start < 0) \ - start = 0; \ - } +#define ADJUST_INDICES(start, end, len) \ + do { \ + if (end > len) { \ + end = len; \ + } \ + else if (end < 0) { \ + end += len; \ + if (end < 0) { \ + end = 0; \ + } \ + } \ + if (start < 0) { \ + start += len; \ + if (start < 0) { \ + start = 0; \ + } \ + } \ + } while (0) Py_LOCAL_INLINE(Py_ssize_t) find_internal(const char *str, Py_ssize_t len, - const char *function_name, PyObject *args, int dir) + const char *function_name, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end, + int dir) { - PyObject *subobj; char byte; Py_buffer subbuf; const char *sub; Py_ssize_t sub_len; - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; Py_ssize_t res; - if (!parse_args_finds_byte(function_name, args, - &subobj, &byte, &start, &end)) + if (!parse_args_finds_byte(function_name, &subobj, &byte)) { return -2; + } if (subobj) { if (PyObject_GetBuffer(subobj, &subbuf, PyBUF_SIMPLE) != 0) @@ -566,37 +511,21 @@ find_internal(const char *str, Py_ssize_t len, return res; } -PyDoc_STRVAR_shared(_Py_find__doc__, -"B.find(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - PyObject * -_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "find", args, +1); + Py_ssize_t result = find_internal(str, len, "find", sub, start, end, +1); if (result == -2) return NULL; return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_index__doc__, -"B.index(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the subsection is not found."); - PyObject * -_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "index", args, +1); + Py_ssize_t result = find_internal(str, len, "index", sub, start, end, +1); if (result == -2) return NULL; if (result == -1) { @@ -607,37 +536,21 @@ _Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_rfind__doc__, -"B.rfind(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - PyObject * -_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "rfind", args, -1); + Py_ssize_t result = find_internal(str, len, "rfind", sub, start, end, -1); if (result == -2) return NULL; return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_rindex__doc__, -"B.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raise ValueError when the subsection is not found."); - PyObject * -_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "rindex", args, -1); + Py_ssize_t result = find_internal(str, len, "rindex", sub, start, end, -1); if (result == -2) return NULL; if (result == -1) { @@ -648,28 +561,20 @@ _Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_count__doc__, -"B.count(sub[, start[, end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of subsection sub in\n\ -bytes B[start:end]. Optional arguments start and end are interpreted\n\ -as in slice notation."); - PyObject * -_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *sub_obj, + Py_ssize_t start, Py_ssize_t end) { - PyObject *sub_obj; const char *sub; Py_ssize_t sub_len; char byte; - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; Py_buffer vsub; PyObject *count_obj; - if (!parse_args_finds_byte("count", args, - &sub_obj, &byte, &start, &end)) + if (!parse_args_finds_byte("count", &sub_obj, &byte)) { return NULL; + } if (sub_obj) { if (PyObject_GetBuffer(sub_obj, &vsub, PyBUF_SIMPLE) != 0) @@ -771,66 +676,65 @@ tailmatch(const char *str, Py_ssize_t len, PyObject *substr, static PyObject * _Py_bytes_tailmatch(const char *str, Py_ssize_t len, - const char *function_name, PyObject *args, + const char *function_name, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end, int direction) { - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj = NULL; - int result; - - if (!stringlib_parse_args_finds(function_name, args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = tailmatch(str, len, PyTuple_GET_ITEM(subobj, i), - start, end, direction); - if (result == -1) + PyObject *item = PyTuple_GET_ITEM(subobj, i); + int result = tailmatch(str, len, item, start, end, direction); + if (result < 0) { return NULL; + } else if (result) { Py_RETURN_TRUE; } } Py_RETURN_FALSE; } - result = tailmatch(str, len, subobj, start, end, direction); + int result = tailmatch(str, len, subobj, start, end, direction); if (result == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) + if (PyErr_ExceptionMatches(PyExc_TypeError)) { PyErr_Format(PyExc_TypeError, "%s first arg must be bytes or a tuple of bytes, " "not %s", function_name, Py_TYPE(subobj)->tp_name); + } return NULL; } - else - return PyBool_FromLong(result); + return PyBool_FromLong(result); } -PyDoc_STRVAR_shared(_Py_startswith__doc__, -"B.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if B starts with the specified prefix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -prefix can also be a tuple of bytes to try."); +PyObject * +_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +{ + return _Py_bytes_tailmatch(str, len, "startswith", subobj, start, end, -1); +} PyObject * -_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) { - return _Py_bytes_tailmatch(str, len, "startswith", args, -1); + return _Py_bytes_tailmatch(str, len, "endswith", subobj, start, end, +1); } -PyDoc_STRVAR_shared(_Py_endswith__doc__, -"B.endswith(suffix[, start[, end]]) -> bool\n\ +PyDoc_STRVAR_shared(_Py_isascii__doc__, +"B.isascii() -> bool\n\ \n\ -Return True if B ends with the specified suffix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -suffix can also be a tuple of bytes to try."); +Return True if B is empty or all characters in B are ASCII,\n\ +False otherwise."); -PyObject * -_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args) +PyObject* +_Py_bytes_isascii(const char *cptr, Py_ssize_t len) { - return _Py_bytes_tailmatch(str, len, "endswith", args, +1); + const char *p = cptr; + const char *end = p + len; + Py_ssize_t max_char = stringlib_find_max_char(cptr, end); + if (max_char > 127) { + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 26227dd251122d..dcc1aba76abbed 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -52,25 +52,25 @@ static inline PyObject* bytes_get_empty(void) /* - For PyBytes_FromString(), the parameter `str' points to a null-terminated - string containing exactly `size' bytes. + For PyBytes_FromString(), the parameter 'str' points to a null-terminated + string containing exactly 'size' bytes. - For PyBytes_FromStringAndSize(), the parameter `str' is - either NULL or else points to a string containing at least `size' bytes. - For PyBytes_FromStringAndSize(), the string in the `str' parameter does + For PyBytes_FromStringAndSize(), the parameter 'str' is + either NULL or else points to a string containing at least 'size' bytes. + For PyBytes_FromStringAndSize(), the string in the 'str' parameter does not have to be null-terminated. (Therefore it is safe to construct a - substring by calling `PyBytes_FromStringAndSize(origstring, substrlen)'.) - If `str' is NULL then PyBytes_FromStringAndSize() will allocate `size+1' + substring by calling 'PyBytes_FromStringAndSize(origstring, substrlen)'.) + If 'str' is NULL then PyBytes_FromStringAndSize() will allocate 'size+1' bytes (setting the last byte to the null terminating character) and you can - fill in the data yourself. If `str' is non-NULL then the resulting + fill in the data yourself. If 'str' is non-NULL then the resulting PyBytes object must be treated as immutable and you must not fill in nor alter the data yourself, since the strings may be shared. - The PyObject member `op->ob_size', which denotes the number of "extra + The PyObject member 'op->ob_size', which denotes the number of "extra items" in a variable-size object, will contain the number of bytes allocated for string data, not counting the null terminating character. - It is therefore equal to the `size' parameter (for - PyBytes_FromStringAndSize()) or the length of the string in the `str' + It is therefore equal to the 'size' parameter (for + PyBytes_FromStringAndSize()) or the length of the string in the 'str' parameter (for PyBytes_FromString()). */ static PyObject * @@ -477,21 +477,32 @@ formatlong(PyObject *v, int flags, int prec, int type) static int byte_converter(PyObject *arg, char *p) { - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or " + "a single byte, not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + return 0; + } *p = PyBytes_AS_STRING(arg)[0]; return 1; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or " + "a single byte, not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + return 0; + } *p = PyByteArray_AS_STRING(arg)[0]; return 1; } - else { + else if (PyIndex_Check(arg)) { int overflow; long ival = PyLong_AsLongAndOverflow(arg, &overflow); if (ival == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - goto onError; - } return 0; } if (!(0 <= ival && ival <= 255)) { @@ -503,9 +514,9 @@ byte_converter(PyObject *arg, char *p) *p = (char)ival; return 1; } - onError: - PyErr_SetString(PyExc_TypeError, - "%c requires an integer in range(256) or a single byte"); + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or a single byte, not %T", + arg); return 0; } @@ -1394,8 +1405,9 @@ bytes_str(PyObject *op) } static Py_ssize_t -bytes_length(PyBytesObject *a) +bytes_length(PyObject *self) { + PyBytesObject *a = _PyBytes_CAST(self); return Py_SIZE(a); } @@ -1445,11 +1457,9 @@ bytes_concat(PyObject *a, PyObject *b) } static PyObject * -bytes_repeat(PyBytesObject *a, Py_ssize_t n) +bytes_repeat(PyObject *self, Py_ssize_t n) { - Py_ssize_t size; - PyBytesObject *op; - size_t nbytes; + PyBytesObject *a = _PyBytes_CAST(self); if (n < 0) n = 0; /* watch out for overflows: the size can overflow int, @@ -1460,17 +1470,17 @@ bytes_repeat(PyBytesObject *a, Py_ssize_t n) "repeated bytes are too long"); return NULL; } - size = Py_SIZE(a) * n; + Py_ssize_t size = Py_SIZE(a) * n; if (size == Py_SIZE(a) && PyBytes_CheckExact(a)) { return Py_NewRef(a); } - nbytes = (size_t)size; + size_t nbytes = (size_t)size; if (nbytes + PyBytesObject_SIZE <= nbytes) { PyErr_SetString(PyExc_OverflowError, "repeated bytes are too long"); return NULL; } - op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + nbytes); + PyBytesObject *op = PyObject_Malloc(PyBytesObject_SIZE + nbytes); if (op == NULL) { return PyErr_NoMemory(); } @@ -1493,8 +1503,9 @@ bytes_contains(PyObject *self, PyObject *arg) } static PyObject * -bytes_item(PyBytesObject *a, Py_ssize_t i) +bytes_item(PyObject *self, Py_ssize_t i) { + PyBytesObject *a = _PyBytes_CAST(self); if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -1520,21 +1531,17 @@ bytes_compare_eq(PyBytesObject *a, PyBytesObject *b) } static PyObject* -bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) +bytes_richcompare(PyObject *aa, PyObject *bb, int op) { - int c; - Py_ssize_t len_a, len_b; - Py_ssize_t min_len; - /* Make sure both arguments are strings. */ - if (!(PyBytes_Check(a) && PyBytes_Check(b))) { + if (!(PyBytes_Check(aa) && PyBytes_Check(bb))) { if (_Py_GetConfig()->bytes_warning && (op == Py_EQ || op == Py_NE)) { - if (PyUnicode_Check(a) || PyUnicode_Check(b)) { + if (PyUnicode_Check(aa) || PyUnicode_Check(bb)) { if (PyErr_WarnEx(PyExc_BytesWarning, "Comparison between bytes and string", 1)) return NULL; } - if (PyLong_Check(a) || PyLong_Check(b)) { + if (PyLong_Check(aa) || PyLong_Check(bb)) { if (PyErr_WarnEx(PyExc_BytesWarning, "Comparison between bytes and int", 1)) return NULL; @@ -1542,7 +1549,10 @@ bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) } Py_RETURN_NOTIMPLEMENTED; } - else if (a == b) { + + PyBytesObject *a = _PyBytes_CAST(aa); + PyBytesObject *b = _PyBytes_CAST(bb); + if (a == b) { switch (op) { case Py_EQ: case Py_LE: @@ -1564,38 +1574,43 @@ bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) return PyBool_FromLong(eq); } else { - len_a = Py_SIZE(a); - len_b = Py_SIZE(b); - min_len = Py_MIN(len_a, len_b); + Py_ssize_t len_a = Py_SIZE(a); + Py_ssize_t len_b = Py_SIZE(b); + Py_ssize_t min_len = Py_MIN(len_a, len_b); + int c; if (min_len > 0) { c = Py_CHARMASK(*a->ob_sval) - Py_CHARMASK(*b->ob_sval); if (c == 0) c = memcmp(a->ob_sval, b->ob_sval, min_len); } - else + else { c = 0; - if (c != 0) + } + if (c != 0) { Py_RETURN_RICHCOMPARE(c, 0, op); + } Py_RETURN_RICHCOMPARE(len_a, len_b, op); } } static Py_hash_t -bytes_hash(PyBytesObject *a) +bytes_hash(PyObject *self) { + PyBytesObject *a = _PyBytes_CAST(self); _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS if (a->ob_shash == -1) { /* Can't fail */ - a->ob_shash = _Py_HashBytes(a->ob_sval, Py_SIZE(a)); + a->ob_shash = Py_HashBuffer(a->ob_sval, Py_SIZE(a)); } return a->ob_shash; _Py_COMP_DIAG_POP } static PyObject* -bytes_subscript(PyBytesObject* self, PyObject* item) +bytes_subscript(PyObject *op, PyObject* item) { + PyBytesObject *self = _PyBytes_CAST(op); if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -1623,7 +1638,7 @@ bytes_subscript(PyBytesObject* self, PyObject* item) &stop, step); if (slicelength <= 0) { - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else if (start == 0 && step == 1 && slicelength == PyBytes_GET_SIZE(self) && @@ -1659,31 +1674,32 @@ bytes_subscript(PyBytesObject* self, PyObject* item) } static int -bytes_buffer_getbuffer(PyBytesObject *self, Py_buffer *view, int flags) +bytes_buffer_getbuffer(PyObject *op, Py_buffer *view, int flags) { + PyBytesObject *self = _PyBytes_CAST(op); return PyBuffer_FillInfo(view, (PyObject*)self, (void *)self->ob_sval, Py_SIZE(self), 1, flags); } static PySequenceMethods bytes_as_sequence = { - (lenfunc)bytes_length, /*sq_length*/ - (binaryfunc)bytes_concat, /*sq_concat*/ - (ssizeargfunc)bytes_repeat, /*sq_repeat*/ - (ssizeargfunc)bytes_item, /*sq_item*/ + bytes_length, /*sq_length*/ + bytes_concat, /*sq_concat*/ + bytes_repeat, /*sq_repeat*/ + bytes_item, /*sq_item*/ 0, /*sq_slice*/ 0, /*sq_ass_item*/ 0, /*sq_ass_slice*/ - (objobjproc)bytes_contains /*sq_contains*/ + bytes_contains /*sq_contains*/ }; static PyMappingMethods bytes_as_mapping = { - (lenfunc)bytes_length, - (binaryfunc)bytes_subscript, + bytes_length, + bytes_subscript, 0, }; static PyBufferProcs bytes_as_buffer = { - (getbufferproc)bytes_buffer_getbuffer, + bytes_buffer_getbuffer, NULL, }; @@ -1856,37 +1872,95 @@ bytes_join(PyBytesObject *self, PyObject *iterable_of_bytes) } PyObject * -_PyBytes_Join(PyObject *sep, PyObject *x) +PyBytes_Join(PyObject *sep, PyObject *iterable) { - assert(sep != NULL && PyBytes_Check(sep)); - assert(x != NULL); - return bytes_join((PyBytesObject*)sep, x); + if (sep == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + if (!PyBytes_Check(sep)) { + PyErr_Format(PyExc_TypeError, + "sep: expected bytes, got %T", sep); + return NULL; + } + + return stringlib_bytes_join(sep, iterable); } +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +bytes.find + + sub: object + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Return -1 on failure. +[clinic start generated code]*/ + static PyObject * -bytes_find(PyBytesObject *self, PyObject *args) +bytes_find_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=d5961a1c77b472a1 input=3171e62a8ae7f240]*/ { - return _Py_bytes_find(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_find(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytes.index = bytes.find + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ + static PyObject * -bytes_index(PyBytesObject *self, PyObject *args) +bytes_index_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=0da25cc74683ba42 input=aa34ad71ba0bafe3]*/ { - return _Py_bytes_index(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_index(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytes.rfind = bytes.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Return -1 on failure. +[clinic start generated code]*/ static PyObject * -bytes_rfind(PyBytesObject *self, PyObject *args) +bytes_rfind_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=51b60fa4ad011c09 input=864c3e7f3010b33c]*/ { - return _Py_bytes_rfind(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_rfind(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytes.rindex = bytes.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ static PyObject * -bytes_rindex(PyBytesObject *self, PyObject *args) +bytes_rindex_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=42bf674e0a0aabf6 input=21051fc5cfeacf2c]*/ { - return _Py_bytes_rindex(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_rindex(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } @@ -2023,10 +2097,19 @@ bytes_rstrip_impl(PyBytesObject *self, PyObject *bytes) } +/*[clinic input] +bytes.count = bytes.find + +Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end]. +[clinic start generated code]*/ + static PyObject * -bytes_count(PyBytesObject *self, PyObject *args) +bytes_count_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=9848140b9be17d0f input=b6e4a5ed515e1e59]*/ { - return _Py_bytes_count(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_count(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } @@ -2285,16 +2368,52 @@ bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) return PyBytes_FromStringAndSize(self_start, self_len); } +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +bytes.startswith + + prefix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return True if the bytes starts with the specified prefix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytes_startswith(PyBytesObject *self, PyObject *args) +bytes_startswith_impl(PyBytesObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=b1e8da1cbd528e8c input=8a4165df8adfa6c9]*/ { - return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subobj, start, end); } +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +bytes.endswith + + suffix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return True if the bytes ends with the specified suffix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytes_endswith(PyBytesObject *self, PyObject *args) +bytes_endswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=038b633111f3629d input=b5c3407a2a5c9aac]*/ { - return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subobj, start, end); } @@ -2488,17 +2607,14 @@ bytes_methods[] = { {"capitalize", stringlib_capitalize, METH_NOARGS, _Py_capitalize__doc__}, STRINGLIB_CENTER_METHODDEF - {"count", (PyCFunction)bytes_count, METH_VARARGS, - _Py_count__doc__}, + BYTES_COUNT_METHODDEF BYTES_DECODE_METHODDEF - {"endswith", (PyCFunction)bytes_endswith, METH_VARARGS, - _Py_endswith__doc__}, + BYTES_ENDSWITH_METHODDEF STRINGLIB_EXPANDTABS_METHODDEF - {"find", (PyCFunction)bytes_find, METH_VARARGS, - _Py_find__doc__}, + BYTES_FIND_METHODDEF BYTES_FROMHEX_METHODDEF BYTES_HEX_METHODDEF - {"index", (PyCFunction)bytes_index, METH_VARARGS, _Py_index__doc__}, + BYTES_INDEX_METHODDEF {"isalnum", stringlib_isalnum, METH_NOARGS, _Py_isalnum__doc__}, {"isalpha", stringlib_isalpha, METH_NOARGS, @@ -2524,16 +2640,15 @@ bytes_methods[] = { BYTES_REPLACE_METHODDEF BYTES_REMOVEPREFIX_METHODDEF BYTES_REMOVESUFFIX_METHODDEF - {"rfind", (PyCFunction)bytes_rfind, METH_VARARGS, _Py_rfind__doc__}, - {"rindex", (PyCFunction)bytes_rindex, METH_VARARGS, _Py_rindex__doc__}, + BYTES_RFIND_METHODDEF + BYTES_RINDEX_METHODDEF STRINGLIB_RJUST_METHODDEF BYTES_RPARTITION_METHODDEF BYTES_RSPLIT_METHODDEF BYTES_RSTRIP_METHODDEF BYTES_SPLIT_METHODDEF BYTES_SPLITLINES_METHODDEF - {"startswith", (PyCFunction)bytes_startswith, METH_VARARGS, - _Py_startswith__doc__}, + BYTES_STARTSWITH_METHODDEF BYTES_STRIP_METHODDEF {"swapcase", stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, @@ -2933,11 +3048,11 @@ PyTypeObject PyBytes_Type = { 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)bytes_repr, /* tp_repr */ + bytes_repr, /* tp_repr */ &bytes_as_number, /* tp_as_number */ &bytes_as_sequence, /* tp_as_sequence */ &bytes_as_mapping, /* tp_as_mapping */ - (hashfunc)bytes_hash, /* tp_hash */ + bytes_hash, /* tp_hash */ 0, /* tp_call */ bytes_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -2964,7 +3079,7 @@ PyTypeObject PyBytes_Type = { 0, /* tp_init */ bytes_alloc, /* tp_alloc */ bytes_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; void @@ -3025,11 +3140,9 @@ PyBytes_ConcatAndDel(PyObject **pv, PyObject *w) /* The following function breaks the notion that bytes are immutable: - it changes the size of a bytes object. We get away with this only if there - is only one module referencing the object. You can also think of it + it changes the size of a bytes object. You can think of it as creating a new bytes object and destroying the old one, only - more efficiently. In any case, don't use this if the bytes object may - already be known to some other part of the code... + more efficiently. Note that if there's not enough memory to resize the bytes object, the original bytes object at *pv is deallocated, *pv is set to NULL, an "out of memory" exception is set, and -1 is returned. Else (on success) 0 is @@ -3045,28 +3158,40 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) PyBytesObject *sv; v = *pv; if (!PyBytes_Check(v) || newsize < 0) { - goto error; + *pv = 0; + Py_DECREF(v); + PyErr_BadInternalCall(); + return -1; } - if (Py_SIZE(v) == newsize) { + Py_ssize_t oldsize = PyBytes_GET_SIZE(v); + if (oldsize == newsize) { /* return early if newsize equals to v->ob_size */ return 0; } - if (Py_SIZE(v) == 0) { - if (newsize == 0) { - return 0; - } + if (oldsize == 0) { *pv = _PyBytes_FromSize(newsize, 0); Py_DECREF(v); return (*pv == NULL) ? -1 : 0; } - if (Py_REFCNT(v) != 1) { - goto error; - } if (newsize == 0) { *pv = bytes_get_empty(); Py_DECREF(v); return 0; } + if (Py_REFCNT(v) != 1) { + if (oldsize < newsize) { + *pv = _PyBytes_FromSize(newsize, 0); + if (*pv) { + memcpy(PyBytes_AS_STRING(*pv), PyBytes_AS_STRING(v), oldsize); + } + } + else { + *pv = PyBytes_FromStringAndSize(PyBytes_AS_STRING(v), newsize); + } + Py_DECREF(v); + return (*pv == NULL) ? -1 : 0; + } + #ifdef Py_TRACE_REFS _Py_ForgetReference(v); #endif @@ -3074,7 +3199,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) PyObject_Realloc(v, PyBytesObject_SIZE + newsize); if (*pv == NULL) { #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif PyObject_Free(v); PyErr_NoMemory(); @@ -3089,11 +3214,6 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS sv->ob_shash = -1; /* invalidate cached hash value */ _Py_COMP_DIAG_POP return 0; -error: - *pv = 0; - Py_DECREF(v); - PyErr_BadInternalCall(); - return -1; } diff --git a/Objects/capsule.c b/Objects/capsule.c index 555979dab2b789..28965e0f21b7a0 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -317,10 +317,14 @@ static int capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) { // Capsule object is only tracked by the GC - // if _PyCapsule_SetTraverse() is called - assert(capsule->traverse_func != NULL); + // if _PyCapsule_SetTraverse() is called, but + // this can still be manually triggered by gc.get_referents() + + if (capsule->traverse_func != NULL) { + return capsule->traverse_func((PyObject*)capsule, visit, arg); + } - return capsule->traverse_func((PyObject*)capsule, visit, arg); + return 0; } diff --git a/Objects/cellobject.c b/Objects/cellobject.c index f1a43be38b2b58..590c8a80857699 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -1,9 +1,12 @@ /* Cell object implementation */ #include "Python.h" +#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" +#define _PyCell_CAST(op) _Py_CAST(PyCellObject*, (op)) + PyObject * PyCell_New(PyObject *obj) { @@ -56,8 +59,7 @@ PyCell_Get(PyObject *op) PyErr_BadInternalCall(); return NULL; } - PyObject *value = PyCell_GET(op); - return Py_XNewRef(value); + return PyCell_GetRef((PyCellObject *)op); } int @@ -67,15 +69,14 @@ PyCell_Set(PyObject *op, PyObject *value) PyErr_BadInternalCall(); return -1; } - PyObject *old_value = PyCell_GET(op); - PyCell_SET(op, Py_XNewRef(value)); - Py_XDECREF(old_value); + PyCell_SetTakeRef((PyCellObject *)op, Py_XNewRef(value)); return 0; } static void -cell_dealloc(PyCellObject *op) +cell_dealloc(PyObject *self) { + PyCellObject *op = _PyCell_CAST(self); _PyObject_GC_UNTRACK(op); Py_XDECREF(op->ob_ref); PyObject_GC_Del(op); @@ -102,10 +103,12 @@ cell_richcompare(PyObject *a, PyObject *b, int op) } static PyObject * -cell_repr(PyCellObject *op) +cell_repr(PyObject *self) { - if (op->ob_ref == NULL) + PyCellObject *op = _PyCell_CAST(self); + if (op->ob_ref == NULL) { return PyUnicode_FromFormat("", op); + } return PyUnicode_FromFormat("", op, Py_TYPE(op->ob_ref)->tp_name, @@ -113,24 +116,26 @@ cell_repr(PyCellObject *op) } static int -cell_traverse(PyCellObject *op, visitproc visit, void *arg) +cell_traverse(PyObject *self, visitproc visit, void *arg) { + PyCellObject *op = _PyCell_CAST(self); Py_VISIT(op->ob_ref); return 0; } static int -cell_clear(PyCellObject *op) +cell_clear(PyObject *self) { + PyCellObject *op = _PyCell_CAST(self); Py_CLEAR(op->ob_ref); return 0; } static PyObject * -cell_get_contents(PyCellObject *op, void *closure) +cell_get_contents(PyObject *self, void *closure) { - if (op->ob_ref == NULL) - { + PyCellObject *op = _PyCell_CAST(self); + if (op->ob_ref == NULL) { PyErr_SetString(PyExc_ValueError, "Cell is empty"); return NULL; } @@ -138,15 +143,15 @@ cell_get_contents(PyCellObject *op, void *closure) } static int -cell_set_contents(PyCellObject *op, PyObject *obj, void *Py_UNUSED(ignored)) +cell_set_contents(PyObject *self, PyObject *obj, void *Py_UNUSED(ignored)) { + PyCellObject *op = _PyCell_CAST(self); Py_XSETREF(op->ob_ref, Py_XNewRef(obj)); return 0; } static PyGetSetDef cell_getsetlist[] = { - {"cell_contents", (getter)cell_get_contents, - (setter)cell_set_contents, NULL}, + {"cell_contents", cell_get_contents, cell_set_contents, NULL}, {NULL} /* sentinel */ }; @@ -155,12 +160,12 @@ PyTypeObject PyCell_Type = { "cell", sizeof(PyCellObject), 0, - (destructor)cell_dealloc, /* tp_dealloc */ + cell_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)cell_repr, /* tp_repr */ + cell_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -172,8 +177,8 @@ PyTypeObject PyCell_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ cell_new_doc, /* tp_doc */ - (traverseproc)cell_traverse, /* tp_traverse */ - (inquiry)cell_clear, /* tp_clear */ + cell_traverse, /* tp_traverse */ + cell_clear, /* tp_clear */ cell_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -188,6 +193,6 @@ PyTypeObject PyCell_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - (newfunc)cell_new, /* tp_new */ + cell_new, /* tp_new */ 0, /* tp_free */ }; diff --git a/Objects/classobject.c b/Objects/classobject.c index d7e520f556d9a0..775894ad5a7166 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -10,6 +10,7 @@ #include "clinic/classobject.c.h" +#define _PyMethodObject_CAST(op) _Py_CAST(PyMethodObject*, (op)) #define TP_DESCR_GET(t) ((t)->tp_descr_get) /*[clinic input] @@ -166,13 +167,14 @@ static PyMemberDef method_memberlist[] = { should only be used for the class, not for instances */ static PyObject * -method_get_doc(PyMethodObject *im, void *context) +method_get_doc(PyObject *self, void *context) { + PyMethodObject *im = _PyMethodObject_CAST(self); return PyObject_GetAttr(im->im_func, &_Py_ID(__doc__)); } static PyGetSetDef method_getset[] = { - {"__doc__", (getter)method_get_doc, NULL, NULL}, + {"__doc__", method_get_doc, NULL, NULL}, {0} }; @@ -188,15 +190,18 @@ method_getattro(PyObject *obj, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); } if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, obj, (PyObject *)Py_TYPE(obj)); + if (f != NULL) { + PyObject *res = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } @@ -232,8 +237,9 @@ method_new_impl(PyTypeObject *type, PyObject *function, PyObject *instance) } static void -method_dealloc(PyMethodObject *im) +method_dealloc(PyObject *self) { + PyMethodObject *im = _PyMethodObject_CAST(self); _PyObject_GC_UNTRACK(im); if (im->im_weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)im); @@ -271,8 +277,9 @@ method_richcompare(PyObject *self, PyObject *other, int op) } static PyObject * -method_repr(PyMethodObject *a) +method_repr(PyObject *op) { + PyMethodObject *a = _PyMethodObject_CAST(op); PyObject *self = a->im_self; PyObject *func = a->im_func; PyObject *funcname, *result; @@ -298,22 +305,26 @@ method_repr(PyMethodObject *a) } static Py_hash_t -method_hash(PyMethodObject *a) +method_hash(PyObject *self) { - Py_hash_t x, y; - x = _Py_HashPointer(a->im_self); - y = PyObject_Hash(a->im_func); - if (y == -1) + PyMethodObject *a = _PyMethodObject_CAST(self); + Py_hash_t x = PyObject_GenericHash(a->im_self); + Py_hash_t y = PyObject_Hash(a->im_func); + if (y == -1) { return -1; + } + x = x ^ y; - if (x == -1) + if (x == -1) { x = -2; + } return x; } static int -method_traverse(PyMethodObject *im, visitproc visit, void *arg) +method_traverse(PyObject *self, visitproc visit, void *arg) { + PyMethodObject *im = _PyMethodObject_CAST(self); Py_VISIT(im->im_func); Py_VISIT(im->im_self); return 0; @@ -330,17 +341,17 @@ PyTypeObject PyMethod_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "method", .tp_basicsize = sizeof(PyMethodObject), - .tp_dealloc = (destructor)method_dealloc, + .tp_dealloc = method_dealloc, .tp_vectorcall_offset = offsetof(PyMethodObject, vectorcall), - .tp_repr = (reprfunc)method_repr, - .tp_hash = (hashfunc)method_hash, + .tp_repr = method_repr, + .tp_hash = method_hash, .tp_call = PyVectorcall_Call, .tp_getattro = method_getattro, .tp_setattro = PyObject_GenericSetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL, .tp_doc = method_new__doc__, - .tp_traverse = (traverseproc)method_traverse, + .tp_traverse = method_traverse, .tp_richcompare = method_richcompare, .tp_weaklistoffset = offsetof(PyMethodObject, im_weakreflist), .tp_methods = method_methods, @@ -396,7 +407,7 @@ instancemethod_get_doc(PyObject *self, void *context) } static PyGetSetDef instancemethod_getset[] = { - {"__doc__", (getter)instancemethod_get_doc, NULL, NULL}, + {"__doc__", instancemethod_get_doc, NULL, NULL}, {0} }; @@ -410,14 +421,17 @@ instancemethod_getattro(PyObject *self, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, self, (PyObject *)Py_TYPE(self)); + if (f != NULL) { + PyObject *res = f(descr, self, (PyObject *)Py_TYPE(self)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } @@ -531,7 +545,7 @@ PyTypeObject PyInstanceMethod_Type = { .tp_name = "instancemethod", .tp_basicsize = sizeof(PyInstanceMethodObject), .tp_dealloc = instancemethod_dealloc, - .tp_repr = (reprfunc)instancemethod_repr, + .tp_repr = instancemethod_repr, .tp_call = instancemethod_call, .tp_getattro = instancemethod_getattro, .tp_setattro = PyObject_GenericSetAttr, diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index d95245067e2608..c748c53e1c0a75 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -101,6 +101,106 @@ bytearray___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(bytearray_find__doc__, +"find($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTEARRAY_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(bytearray_find), METH_FASTCALL, bytearray_find__doc__}, + +static PyObject * +bytearray_find_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytearray_find(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_find_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of subsection \'sub\' in bytes B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTEARRAY_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(bytearray_count), METH_FASTCALL, bytearray_count__doc__}, + +static PyObject * +bytearray_count_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_count(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_count_impl(self, sub, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_clear__doc__, "clear($self, /)\n" "--\n" @@ -137,6 +237,261 @@ bytearray_copy(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) return bytearray_copy_impl(self); } +PyDoc_STRVAR(bytearray_index__doc__, +"index($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTEARRAY_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(bytearray_index), METH_FASTCALL, bytearray_index__doc__}, + +static PyObject * +bytearray_index_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_index(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_index_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_rfind__doc__, +"rfind($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTEARRAY_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(bytearray_rfind), METH_FASTCALL, bytearray_rfind__doc__}, + +static PyObject * +bytearray_rfind_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_rfind(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_rfind_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_rindex__doc__, +"rindex($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTEARRAY_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(bytearray_rindex), METH_FASTCALL, bytearray_rindex__doc__}, + +static PyObject * +bytearray_rindex_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_rindex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_rindex_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytearray starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytearray.\n" +" end\n" +" Optional stop position. Default: end of the bytearray."); + +#define BYTEARRAY_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(bytearray_startswith), METH_FASTCALL, bytearray_startswith__doc__}, + +static PyObject * +bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_startswith(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytearray ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytearray.\n" +" end\n" +" Optional stop position. Default: end of the bytearray."); + +#define BYTEARRAY_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(bytearray_endswith), METH_FASTCALL, bytearray_endswith__doc__}, + +static PyObject * +bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_endswith(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_removeprefix__doc__, "removeprefix($self, prefix, /)\n" "--\n" @@ -1261,4 +1616,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=0797a5e03cda2a16 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5f861b02e3fa278b input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 1e45be3e7aefb3..0b4b37501735c1 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -294,6 +294,210 @@ PyDoc_STRVAR(bytes_join__doc__, #define BYTES_JOIN_METHODDEF \ {"join", (PyCFunction)bytes_join, METH_O, bytes_join__doc__}, +PyDoc_STRVAR(bytes_find__doc__, +"find($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTES_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(bytes_find), METH_FASTCALL, bytes_find__doc__}, + +static PyObject * +bytes_find_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_find(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_find_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_index__doc__, +"index($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTES_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(bytes_index), METH_FASTCALL, bytes_index__doc__}, + +static PyObject * +bytes_index_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_index(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_index_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_rfind__doc__, +"rfind($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTES_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(bytes_rfind), METH_FASTCALL, bytes_rfind__doc__}, + +static PyObject * +bytes_rfind_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_rfind(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_rfind_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_rindex__doc__, +"rindex($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTES_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(bytes_rindex), METH_FASTCALL, bytes_rindex__doc__}, + +static PyObject * +bytes_rindex_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_rindex(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_rindex_impl(self, sub, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytes_strip__doc__, "strip($self, bytes=None, /)\n" "--\n" @@ -396,6 +600,55 @@ bytes_rstrip(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(bytes_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of subsection \'sub\' in bytes B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(bytes_count), METH_FASTCALL, bytes_count__doc__}, + +static PyObject * +bytes_count_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_count(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_count_impl(self, sub, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytes_translate__doc__, "translate($self, table, /, delete=b\'\')\n" "--\n" @@ -652,6 +905,108 @@ bytes_removesuffix(PyBytesObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(bytes_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytes starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(bytes_startswith), METH_FASTCALL, bytes_startswith__doc__}, + +static PyObject * +bytes_startswith_impl(PyBytesObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytes_startswith(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytes ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(bytes_endswith), METH_FASTCALL, bytes_endswith__doc__}, + +static PyObject * +bytes_endswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_endswith(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytes_decode__doc__, "decode($self, /, encoding=\'utf-8\', errors=\'strict\')\n" "--\n" @@ -1029,4 +1384,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8a49dbbd78914a6f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d6801c6001e57f91 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/complexobject.c.h b/Objects/clinic/complexobject.c.h index 49b50304021f7b..58fd4e26871b4d 100644 --- a/Objects/clinic/complexobject.c.h +++ b/Objects/clinic/complexobject.c.h @@ -94,9 +94,12 @@ PyDoc_STRVAR(complex_new__doc__, "complex(real=0, imag=0)\n" "--\n" "\n" -"Create a complex number from a real part and an optional imaginary part.\n" +"Create a complex number from a string or numbers.\n" "\n" -"This is equivalent to (real + imag*1j) where imag defaults to 0."); +"If a string is given, parse it as a complex number.\n" +"If a single number is given, convert it to a complex number.\n" +"If the \'real\' or \'imag\' arguments are given, create a complex number\n" +"with the specified real and imaginary components."); static PyObject * complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i); @@ -157,4 +160,13 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=04e6261649967b30 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(complex_from_number__doc__, +"from_number($type, number, /)\n" +"--\n" +"\n" +"Convert number to a complex floating-point number."); + +#define COMPLEX_FROM_NUMBER_METHODDEF \ + {"from_number", (PyCFunction)complex_from_number, METH_O|METH_CLASS, complex_from_number__doc__}, +/*[clinic end generated code: output=188438cc9ae167f7 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/descrobject.c.h b/Objects/clinic/descrobject.c.h index 02fb440d9c83af..d79be80d3ec165 100644 --- a/Objects/clinic/descrobject.c.h +++ b/Objects/clinic/descrobject.c.h @@ -8,6 +8,12 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(mappingproxy_new__doc__, +"mappingproxy(mapping)\n" +"--\n" +"\n" +"Read-only proxy of a mapping."); + static PyObject * mappingproxy_new_impl(PyTypeObject *type, PyObject *mapping); @@ -167,4 +173,4 @@ property_init(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a4664ccf3da10f5a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=050e331316a04207 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h index daaef211b1db49..fb46c4c64334f9 100644 --- a/Objects/clinic/dictobject.c.h +++ b/Objects/clinic/dictobject.c.h @@ -66,21 +66,6 @@ PyDoc_STRVAR(dict___contains____doc__, #define DICT___CONTAINS___METHODDEF \ {"__contains__", (PyCFunction)dict___contains__, METH_O|METH_COEXIST, dict___contains____doc__}, -static PyObject * -dict___contains___impl(PyDictObject *self, PyObject *key); - -static PyObject * -dict___contains__(PyDictObject *self, PyObject *key) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = dict___contains___impl(self, key); - Py_END_CRITICAL_SECTION(); - - return return_value; -} - PyDoc_STRVAR(dict_get__doc__, "get($self, key, default=None, /)\n" "--\n" @@ -327,4 +312,4 @@ dict_values(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { return dict_values_impl(self); } -/*[clinic end generated code: output=c8fda06bac5b05f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f3dd5f3fb8122aef input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index 10f6149cc88c22..dd29135590a6a6 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -197,7 +197,7 @@ PyDoc_STRVAR(float_new__doc__, "float(x=0, /)\n" "--\n" "\n" -"Convert a string or number to a floating point number, if possible."); +"Convert a string or number to a floating-point number, if possible."); static PyObject * float_new_impl(PyTypeObject *type, PyObject *x); @@ -227,6 +227,15 @@ float_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(float_from_number__doc__, +"from_number($type, number, /)\n" +"--\n" +"\n" +"Convert real number to a floating-point number."); + +#define FLOAT_FROM_NUMBER_METHODDEF \ + {"from_number", (PyCFunction)float_from_number, METH_O|METH_CLASS, float_from_number__doc__}, + PyDoc_STRVAR(float___getnewargs____doc__, "__getnewargs__($self, /)\n" "--\n" @@ -256,7 +265,7 @@ PyDoc_STRVAR(float___getformat____doc__, "It exists mainly to be used in Python\'s test suite.\n" "\n" "This function returns whichever of \'unknown\', \'IEEE, big-endian\' or \'IEEE,\n" -"little-endian\' best describes the format of floating point numbers used by the\n" +"little-endian\' best describes the format of floating-point numbers used by the\n" "C type named by typestr."); #define FLOAT___GETFORMAT___METHODDEF \ @@ -318,4 +327,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=c79743c8551c30d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=366cea9463cc5bf6 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index a61550a49b66fc..588e021fb71fd3 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -125,29 +125,14 @@ list_append(PyListObject *self, PyObject *object) return return_value; } -PyDoc_STRVAR(py_list_extend__doc__, +PyDoc_STRVAR(list_extend__doc__, "extend($self, iterable, /)\n" "--\n" "\n" "Extend list by appending elements from the iterable."); -#define PY_LIST_EXTEND_METHODDEF \ - {"extend", (PyCFunction)py_list_extend, METH_O, py_list_extend__doc__}, - -static PyObject * -py_list_extend_impl(PyListObject *self, PyObject *iterable); - -static PyObject * -py_list_extend(PyListObject *self, PyObject *iterable) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION2(self, iterable); - return_value = py_list_extend_impl(self, iterable); - Py_END_CRITICAL_SECTION2(); - - return return_value; -} +#define LIST_EXTEND_METHODDEF \ + {"extend", (PyCFunction)list_extend, METH_O, list_extend__doc__}, PyDoc_STRVAR(list_pop__doc__, "pop($self, index=-1, /)\n" @@ -268,7 +253,9 @@ list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = list_sort_impl(self, keyfunc, reverse); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -452,4 +439,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=26dfb2c9846348f9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=854957a1d4a89bbd input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index 4a3d71c6111af5..90375b9a082cca 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -116,7 +116,7 @@ int___format__(PyObject *self, PyObject *arg) } PyDoc_STRVAR(int___round____doc__, -"__round__($self, ndigits=, /)\n" +"__round__($self, ndigits=None, /)\n" "--\n" "\n" "Rounding an Integral returns itself.\n" @@ -133,7 +133,7 @@ static PyObject * int___round__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *o_ndigits = NULL; + PyObject *o_ndigits = Py_None; if (!_PyArg_CheckPositional("__round__", nargs, 0, 1)) { goto exit; @@ -267,7 +267,7 @@ PyDoc_STRVAR(int_to_bytes__doc__, " the most significant byte is at the beginning of the byte array. If\n" " byteorder is \'little\', the most significant byte is at the end of the\n" " byte array. To request the native byte order of the host system, use\n" -" `sys.byteorder\' as the byte order value. Default is to use \'big\'.\n" +" sys.byteorder as the byte order value. Default is to use \'big\'.\n" " signed\n" " Determines whether two\'s complement is used to represent the integer.\n" " If signed is False and a negative integer is given, an OverflowError\n" @@ -380,7 +380,7 @@ PyDoc_STRVAR(int_from_bytes__doc__, " the most significant byte is at the beginning of the byte array. If\n" " byteorder is \'little\', the most significant byte is at the end of the\n" " byte array. To request the native byte order of the host system, use\n" -" `sys.byteorder\' as the byte order value. Default is to use \'big\'.\n" +" sys.byteorder as the byte order value. Default is to use \'big\'.\n" " signed\n" " Indicates whether two\'s complement is used to represent the integer."); @@ -476,4 +476,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=7e6e57246e55911f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a53f5ba9a6c16737 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/setobject.c.h b/Objects/clinic/setobject.c.h index f3c96995ede60d..d6e381a9975050 100644 --- a/Objects/clinic/setobject.c.h +++ b/Objects/clinic/setobject.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(set_pop__doc__, @@ -21,7 +22,13 @@ set_pop_impl(PySetObject *so); static PyObject * set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set_pop_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_pop_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set_update__doc__, @@ -34,28 +41,22 @@ PyDoc_STRVAR(set_update__doc__, {"update", _PyCFunction_CAST(set_update), METH_FASTCALL, set_update__doc__}, static PyObject * -set_update_impl(PySetObject *so, PyObject *args); +set_update_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args); static PyObject * set_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("update", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_update_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_update_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -74,7 +75,13 @@ set_copy_impl(PySetObject *so); static PyObject * set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set_copy_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_copy_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(frozenset_copy__doc__, @@ -92,7 +99,13 @@ frozenset_copy_impl(PySetObject *so); static PyObject * frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return frozenset_copy_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = frozenset_copy_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set_clear__doc__, @@ -110,7 +123,13 @@ set_clear_impl(PySetObject *so); static PyObject * set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set_clear_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_clear_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set_union__doc__, @@ -123,28 +142,22 @@ PyDoc_STRVAR(set_union__doc__, {"union", _PyCFunction_CAST(set_union), METH_FASTCALL, set_union__doc__}, static PyObject * -set_union_impl(PySetObject *so, PyObject *args); +set_union_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args); static PyObject * set_union(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("union", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_union_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_union_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -158,28 +171,23 @@ PyDoc_STRVAR(set_intersection_multi__doc__, {"intersection", _PyCFunction_CAST(set_intersection_multi), METH_FASTCALL, set_intersection_multi__doc__}, static PyObject * -set_intersection_multi_impl(PySetObject *so, PyObject *args); +set_intersection_multi_impl(PySetObject *so, Py_ssize_t nargs, + PyObject *const *args); static PyObject * set_intersection_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("intersection", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_intersection_multi_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_intersection_multi_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -193,28 +201,23 @@ PyDoc_STRVAR(set_intersection_update_multi__doc__, {"intersection_update", _PyCFunction_CAST(set_intersection_update_multi), METH_FASTCALL, set_intersection_update_multi__doc__}, static PyObject * -set_intersection_update_multi_impl(PySetObject *so, PyObject *args); +set_intersection_update_multi_impl(PySetObject *so, Py_ssize_t nargs, + PyObject *const *args); static PyObject * set_intersection_update_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("intersection_update", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_intersection_update_multi_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_intersection_update_multi_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -227,6 +230,21 @@ PyDoc_STRVAR(set_isdisjoint__doc__, #define SET_ISDISJOINT_METHODDEF \ {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, set_isdisjoint__doc__}, +static PyObject * +set_isdisjoint_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_isdisjoint(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_isdisjoint_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_difference_update__doc__, "difference_update($self, /, *others)\n" "--\n" @@ -237,28 +255,23 @@ PyDoc_STRVAR(set_difference_update__doc__, {"difference_update", _PyCFunction_CAST(set_difference_update), METH_FASTCALL, set_difference_update__doc__}, static PyObject * -set_difference_update_impl(PySetObject *so, PyObject *args); +set_difference_update_impl(PySetObject *so, Py_ssize_t nargs, + PyObject *const *args); static PyObject * set_difference_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("difference_update", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_difference_update_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_difference_update_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -272,28 +285,23 @@ PyDoc_STRVAR(set_difference_multi__doc__, {"difference", _PyCFunction_CAST(set_difference_multi), METH_FASTCALL, set_difference_multi__doc__}, static PyObject * -set_difference_multi_impl(PySetObject *so, PyObject *args); +set_difference_multi_impl(PySetObject *so, Py_ssize_t nargs, + PyObject *const *args); static PyObject * set_difference_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("difference", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_difference_multi_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_difference_multi_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -315,6 +323,21 @@ PyDoc_STRVAR(set_symmetric_difference__doc__, #define SET_SYMMETRIC_DIFFERENCE_METHODDEF \ {"symmetric_difference", (PyCFunction)set_symmetric_difference, METH_O, set_symmetric_difference__doc__}, +static PyObject * +set_symmetric_difference_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_symmetric_difference(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_symmetric_difference_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_issubset__doc__, "issubset($self, other, /)\n" "--\n" @@ -324,6 +347,21 @@ PyDoc_STRVAR(set_issubset__doc__, #define SET_ISSUBSET_METHODDEF \ {"issubset", (PyCFunction)set_issubset, METH_O, set_issubset__doc__}, +static PyObject * +set_issubset_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_issubset(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_issubset_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_issuperset__doc__, "issuperset($self, other, /)\n" "--\n" @@ -333,6 +371,21 @@ PyDoc_STRVAR(set_issuperset__doc__, #define SET_ISSUPERSET_METHODDEF \ {"issuperset", (PyCFunction)set_issuperset, METH_O, set_issuperset__doc__}, +static PyObject * +set_issuperset_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_issuperset(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_issuperset_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_add__doc__, "add($self, object, /)\n" "--\n" @@ -344,6 +397,21 @@ PyDoc_STRVAR(set_add__doc__, #define SET_ADD_METHODDEF \ {"add", (PyCFunction)set_add, METH_O, set_add__doc__}, +static PyObject * +set_add_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_add(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_add_impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set___contains____doc__, "__contains__($self, object, /)\n" "--\n" @@ -353,6 +421,21 @@ PyDoc_STRVAR(set___contains____doc__, #define SET___CONTAINS___METHODDEF \ {"__contains__", (PyCFunction)set___contains__, METH_O|METH_COEXIST, set___contains____doc__}, +static PyObject * +set___contains___impl(PySetObject *so, PyObject *key); + +static PyObject * +set___contains__(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___contains___impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set_remove__doc__, "remove($self, object, /)\n" "--\n" @@ -364,6 +447,21 @@ PyDoc_STRVAR(set_remove__doc__, #define SET_REMOVE_METHODDEF \ {"remove", (PyCFunction)set_remove, METH_O, set_remove__doc__}, +static PyObject * +set_remove_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_remove(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_remove_impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set_discard__doc__, "discard($self, object, /)\n" "--\n" @@ -376,6 +474,21 @@ PyDoc_STRVAR(set_discard__doc__, #define SET_DISCARD_METHODDEF \ {"discard", (PyCFunction)set_discard, METH_O, set_discard__doc__}, +static PyObject * +set_discard_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_discard(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_discard_impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set___reduce____doc__, "__reduce__($self, /)\n" "--\n" @@ -391,7 +504,13 @@ set___reduce___impl(PySetObject *so); static PyObject * set___reduce__(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set___reduce___impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___reduce___impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set___sizeof____doc__, @@ -409,6 +528,12 @@ set___sizeof___impl(PySetObject *so); static PyObject * set___sizeof__(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set___sizeof___impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___sizeof___impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } -/*[clinic end generated code: output=34a30591148da884 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d4b41191b2c602f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index 2bb0a98a2ed64c..0ba4ff48bc8804 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -9,16 +9,16 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywordsWithVararg() PyDoc_STRVAR(typevar_new__doc__, -"typevar(name, *constraints, bound=None, covariant=False,\n" -" contravariant=False, infer_variance=False)\n" +"typevar(name, *constraints, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a TypeVar."); static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance); + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance); static PyObject * typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -26,14 +26,14 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -42,20 +42,21 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevar", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[6]; + PyObject *argsbuf[7]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; PyObject *constraints = NULL; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; @@ -80,7 +81,13 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[3]) { - covariant = PyObject_IsTrue(fastargs[3]); + default_value = fastargs[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + covariant = PyObject_IsTrue(fastargs[4]); if (covariant < 0) { goto exit; } @@ -88,8 +95,8 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[4]) { - contravariant = PyObject_IsTrue(fastargs[4]); + if (fastargs[5]) { + contravariant = PyObject_IsTrue(fastargs[5]); if (contravariant < 0) { goto exit; } @@ -97,12 +104,12 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[5]); + infer_variance = PyObject_IsTrue(fastargs[6]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: - return_value = typevar_new_impl(type, name, constraints, bound, covariant, contravariant, infer_variance); + return_value = typevar_new_impl(type, name, constraints, bound, default_value, covariant, contravariant, infer_variance); exit: Py_XDECREF(constraints); @@ -117,6 +124,36 @@ PyDoc_STRVAR(typevar_typing_subst__doc__, #define TYPEVAR_TYPING_SUBST_METHODDEF \ {"__typing_subst__", (PyCFunction)typevar_typing_subst, METH_O, typevar_typing_subst__doc__}, +PyDoc_STRVAR(typevar_typing_prepare_subst__doc__, +"__typing_prepare_subst__($self, alias, args, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF \ + {"__typing_prepare_subst__", _PyCFunction_CAST(typevar_typing_prepare_subst), METH_FASTCALL, typevar_typing_prepare_subst__doc__}, + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args); + +static PyObject * +typevar_typing_prepare_subst(typevarobject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *alias; + PyObject *__clinic_args; + + if (!_PyArg_CheckPositional("__typing_prepare_subst__", nargs, 2, 2)) { + goto exit; + } + alias = args[0]; + __clinic_args = args[1]; + return_value = typevar_typing_prepare_subst_impl(self, alias, __clinic_args); + +exit: + return return_value; +} + PyDoc_STRVAR(typevar_reduce__doc__, "__reduce__($self, /)\n" "--\n" @@ -134,6 +171,23 @@ typevar_reduce(typevarobject *self, PyObject *Py_UNUSED(ignored)) return typevar_reduce_impl(self); } +PyDoc_STRVAR(typevar_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevar_has_default, METH_NOARGS, typevar_has_default__doc__}, + +static PyObject * +typevar_has_default_impl(typevarobject *self); + +static PyObject * +typevar_has_default(typevarobject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevar_has_default_impl(self); +} + PyDoc_STRVAR(paramspecargs_new__doc__, "paramspecargs(origin)\n" "--\n" @@ -243,15 +297,16 @@ paramspeckwargs_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(paramspec_new__doc__, -"paramspec(name, *, bound=None, covariant=False, contravariant=False,\n" -" infer_variance=False)\n" +"paramspec(name, *, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a ParamSpec object."); static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance); + PyObject *default_value, int covariant, int contravariant, + int infer_variance); static PyObject * paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -259,14 +314,14 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -275,19 +330,20 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "paramspec", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; @@ -311,7 +367,13 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - covariant = PyObject_IsTrue(fastargs[2]); + default_value = fastargs[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + covariant = PyObject_IsTrue(fastargs[3]); if (covariant < 0) { goto exit; } @@ -319,8 +381,8 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[3]) { - contravariant = PyObject_IsTrue(fastargs[3]); + if (fastargs[4]) { + contravariant = PyObject_IsTrue(fastargs[4]); if (contravariant < 0) { goto exit; } @@ -328,12 +390,12 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[4]); + infer_variance = PyObject_IsTrue(fastargs[5]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: - return_value = paramspec_new_impl(type, name, bound, covariant, contravariant, infer_variance); + return_value = paramspec_new_impl(type, name, bound, default_value, covariant, contravariant, infer_variance); exit: return return_value; @@ -394,14 +456,32 @@ paramspec_reduce(paramspecobject *self, PyObject *Py_UNUSED(ignored)) return paramspec_reduce_impl(self); } +PyDoc_STRVAR(paramspec_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define PARAMSPEC_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)paramspec_has_default, METH_NOARGS, paramspec_has_default__doc__}, + +static PyObject * +paramspec_has_default_impl(paramspecobject *self); + +static PyObject * +paramspec_has_default(paramspecobject *self, PyObject *Py_UNUSED(ignored)) +{ + return paramspec_has_default_impl(self); +} + PyDoc_STRVAR(typevartuple__doc__, -"typevartuple(name)\n" +"typevartuple(name, *, default=typing.NoDefault)\n" "--\n" "\n" "Create a new TypeVarTuple with the given name."); static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name); +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value); static PyObject * typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -409,14 +489,14 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), }, + .ob_item = { &_Py_ID(name), &_Py_ID(default), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -425,17 +505,19 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", NULL}; + static const char * const _keywords[] = {"name", "default", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevartuple", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; + PyObject *argsbuf[2]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *default_value = &_Py_NoDefaultStruct; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { @@ -446,7 +528,12 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } name = fastargs[0]; - return_value = typevartuple_impl(type, name); + if (!noptargs) { + goto skip_optional_kwonly; + } + default_value = fastargs[1]; +skip_optional_kwonly: + return_value = typevartuple_impl(type, name, default_value); exit: return return_value; @@ -507,6 +594,23 @@ typevartuple_reduce(typevartupleobject *self, PyObject *Py_UNUSED(ignored)) return typevartuple_reduce_impl(self); } +PyDoc_STRVAR(typevartuple_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVARTUPLE_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevartuple_has_default, METH_NOARGS, typevartuple_has_default__doc__}, + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self); + +static PyObject * +typevartuple_has_default(typevartupleobject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevartuple_has_default_impl(self); +} + PyDoc_STRVAR(typealias_reduce__doc__, "__reduce__($self, /)\n" "--\n" @@ -591,4 +695,4 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5a582d9d89ad787b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73b39e550e4e336c input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 3e5167d9242fe4..78e14b0021d006 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -136,6 +136,61 @@ unicode_center(PyObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(unicode_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation."); + +#define UNICODE_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(unicode_count), METH_FASTCALL, unicode_count__doc__}, + +static Py_ssize_t +unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_count(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("count", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_count_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_encode__doc__, "encode($self, /, encoding=\'utf-8\', errors=\'strict\')\n" "--\n" @@ -301,6 +356,118 @@ unicode_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(unicode_find__doc__, +"find($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Return -1 on failure."); + +#define UNICODE_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(unicode_find), METH_FASTCALL, unicode_find__doc__}, + +static Py_ssize_t +unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_find(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("find", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_find_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_index__doc__, +"index($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Raises ValueError when the substring is not found."); + +#define UNICODE_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(unicode_index), METH_FASTCALL, unicode_index__doc__}, + +static Py_ssize_t +unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_index(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("index", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_index_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_isascii__doc__, "isascii($self, /)\n" "--\n" @@ -892,6 +1059,118 @@ unicode_removesuffix(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(unicode_rfind__doc__, +"rfind($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Return -1 on failure."); + +#define UNICODE_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(unicode_rfind), METH_FASTCALL, unicode_rfind__doc__}, + +static Py_ssize_t +unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_rfind(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("rfind", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_rfind_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_rindex__doc__, +"rindex($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Raises ValueError when the substring is not found."); + +#define UNICODE_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(unicode_rindex), METH_FASTCALL, unicode_rindex__doc__}, + +static Py_ssize_t +unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_rindex(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("rindex", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_rindex_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_rjust__doc__, "rjust($self, width, fillchar=\' \', /)\n" "--\n" @@ -1369,6 +1648,108 @@ unicode_zfill(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(unicode_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the string starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A string or a tuple of strings to try.\n" +" start\n" +" Optional start position. Default: start of the string.\n" +" end\n" +" Optional stop position. Default: end of the string."); + +#define UNICODE_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(unicode_startswith), METH_FASTCALL, unicode_startswith__doc__}, + +static PyObject * +unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_startswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = unicode_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the string ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A string or a tuple of strings to try.\n" +" start\n" +" Optional start position. Default: start of the string.\n" +" end\n" +" Optional stop position. Default: end of the string."); + +#define UNICODE_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(unicode_endswith), METH_FASTCALL, unicode_endswith__doc__}, + +static PyObject * +unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_endswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = unicode_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode___format____doc__, "__format__($self, format_spec, /)\n" "--\n" @@ -1507,4 +1888,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=1aab29bab5201c78 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9fee62bd337f809b input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index dc46b773c26528..775ea7aca824c4 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -5,12 +5,16 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE +#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs +#include "pycore_object.h" // _PyObject_SetDeferredRefcount #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches #include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_uniqueid.h" // _PyObject_AssignUniqueId() #include "clinic/codeobject.c.h" static const char * @@ -99,10 +103,14 @@ PyCode_ClearWatcher(int watcher_id) * generic helpers ******************/ -/* all_name_chars(s): true iff s matches [a-zA-Z0-9_]* */ static int -all_name_chars(PyObject *o) +should_intern_string(PyObject *o) { +#ifdef Py_GIL_DISABLED + // The free-threaded build interns (and immortalizes) all string constants + return 1; +#else + // compute if s matches [a-zA-Z0-9_] const unsigned char *s, *e; if (!PyUnicode_IS_ASCII(o)) @@ -115,11 +123,17 @@ all_name_chars(PyObject *o) return 0; } return 1; +#endif } +#ifdef Py_GIL_DISABLED +static PyObject *intern_one_constant(PyObject *op); +#endif + static int intern_strings(PyObject *tuple) { + PyInterpreterState *interp = _PyInterpreterState_GET(); Py_ssize_t i; for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { @@ -129,21 +143,24 @@ intern_strings(PyObject *tuple) "non-string found in code slot"); return -1; } - PyUnicode_InternInPlace(&_PyTuple_ITEMS(tuple)[i]); + _PyUnicode_InternImmortal(interp, &_PyTuple_ITEMS(tuple)[i]); } return 0; } -/* Intern selected string constants */ +/* Intern constants. In the default build, this interns selected string + constants. In the free-threaded build, this also interns non-string + constants. */ static int -intern_string_constants(PyObject *tuple, int *modified) +intern_constants(PyObject *tuple, int *modified) { + PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { PyObject *v = PyTuple_GET_ITEM(tuple, i); if (PyUnicode_CheckExact(v)) { - if (all_name_chars(v)) { + if (should_intern_string(v)) { PyObject *w = v; - PyUnicode_InternInPlace(&v); + _PyUnicode_InternMortal(interp, &v); if (w != v) { PyTuple_SET_ITEM(tuple, i, v); if (modified) { @@ -153,7 +170,7 @@ intern_string_constants(PyObject *tuple, int *modified) } } else if (PyTuple_CheckExact(v)) { - if (intern_string_constants(v, NULL) < 0) { + if (intern_constants(v, NULL) < 0) { return -1; } } @@ -164,7 +181,7 @@ intern_string_constants(PyObject *tuple, int *modified) return -1; } int tmp_modified = 0; - if (intern_string_constants(tmp, &tmp_modified) < 0) { + if (intern_constants(tmp, &tmp_modified) < 0) { Py_DECREF(tmp); return -1; } @@ -183,6 +200,56 @@ intern_string_constants(PyObject *tuple, int *modified) } Py_DECREF(tmp); } +#ifdef Py_GIL_DISABLED + else if (PySlice_Check(v)) { + PySliceObject *slice = (PySliceObject *)v; + PyObject *tmp = PyTuple_New(3); + if (tmp == NULL) { + return -1; + } + PyTuple_SET_ITEM(tmp, 0, Py_NewRef(slice->start)); + PyTuple_SET_ITEM(tmp, 1, Py_NewRef(slice->stop)); + PyTuple_SET_ITEM(tmp, 2, Py_NewRef(slice->step)); + int tmp_modified = 0; + if (intern_constants(tmp, &tmp_modified) < 0) { + Py_DECREF(tmp); + return -1; + } + if (tmp_modified) { + v = PySlice_New(PyTuple_GET_ITEM(tmp, 0), + PyTuple_GET_ITEM(tmp, 1), + PyTuple_GET_ITEM(tmp, 2)); + if (v == NULL) { + Py_DECREF(tmp); + return -1; + } + PyTuple_SET_ITEM(tuple, i, v); + Py_DECREF(slice); + if (modified) { + *modified = 1; + } + } + Py_DECREF(tmp); + } + + // Intern non-string constants in the free-threaded build + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (!_Py_IsImmortal(v) && !PyCode_Check(v) && + !PyUnicode_CheckExact(v) && !tstate->suppress_co_const_immortalization) + { + PyObject *interned = intern_one_constant(v); + if (interned == NULL) { + return -1; + } + else if (interned != v) { + PyTuple_SET_ITEM(tuple, i, interned); + Py_SETREF(v, interned); + if (modified) { + *modified = 1; + } + } + } +#endif } return 0; } @@ -386,9 +453,13 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) con->stacksize = 1; } + PyInterpreterState *interp = _PyInterpreterState_GET(); co->co_filename = Py_NewRef(con->filename); co->co_name = Py_NewRef(con->name); co->co_qualname = Py_NewRef(con->qualname); + _PyUnicode_InternMortal(interp, &co->co_filename); + _PyUnicode_InternMortal(interp, &co->co_name); + _PyUnicode_InternMortal(interp, &co->co_qualname); co->co_flags = con->flags; co->co_firstlineno = con->firstlineno; @@ -414,11 +485,16 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - PyInterpreterState *interp = _PyInterpreterState_GET(); - co->co_version = interp->next_func_version; - if (interp->next_func_version != 0) { - interp->next_func_version++; - } +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->func_state.mutex); +#endif + co->co_version = interp->func_state.next_version; + if (interp->func_state.next_version != 0) { + interp->func_state.next_version++; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->func_state.mutex); +#endif co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; /* not set */ @@ -490,7 +566,7 @@ get_line_delta(const uint8_t *ptr) static PyObject * remove_column_info(PyObject *locations) { - int offset = 0; + Py_ssize_t offset = 0; const uint8_t *data = (const uint8_t *)PyBytes_AS_STRING(locations); PyObject *res = PyBytes_FromStringAndSize(NULL, 32); if (res == NULL) { @@ -530,18 +606,41 @@ remove_column_info(PyObject *locations) return res; } -/* The caller is responsible for ensuring that the given data is valid. */ - -PyCodeObject * -_PyCode_New(struct _PyCodeConstructor *con) +static int +intern_code_constants(struct _PyCodeConstructor *con) { +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _py_code_state *state = &interp->code_state; + PyMutex_Lock(&state->mutex); +#endif if (intern_strings(con->names) < 0) { - return NULL; + goto error; } - if (intern_string_constants(con->consts, NULL) < 0) { - return NULL; + if (intern_constants(con->consts, NULL) < 0) { + goto error; } if (intern_strings(con->localsplusnames) < 0) { + goto error; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return 0; + +error: +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return -1; +} + +/* The caller is responsible for ensuring that the given data is valid. */ + +PyCodeObject * +_PyCode_New(struct _PyCodeConstructor *con) +{ + if (intern_code_constants(con) < 0) { return NULL; } @@ -557,13 +656,22 @@ _PyCode_New(struct _PyCodeConstructor *con) } Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT); - PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); + PyCodeObject *co; +#ifdef Py_GIL_DISABLED + co = PyObject_GC_NewVar(PyCodeObject, &PyCode_Type, size); +#else + co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); +#endif if (co == NULL) { Py_XDECREF(replacement_locations); PyErr_NoMemory(); return NULL; } init_code(co, con); +#ifdef Py_GIL_DISABLED + co->_co_unique_id = _PyObject_AssignUniqueId((PyObject *)co); + _PyObject_GC_TRACK(co); +#endif Py_XDECREF(replacement_locations); return co; } @@ -750,7 +858,7 @@ PyUnstable_Code_New(int argcount, int kwonlyargcount, static const uint8_t assert0[6] = { RESUME, RESUME_AT_FUNC_START, - LOAD_ASSERTION_ERROR, 0, + LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR, RAISE_VARARGS, 1 }; @@ -1163,8 +1271,9 @@ typedef struct { static void -lineiter_dealloc(lineiterator *li) +lineiter_dealloc(PyObject *self) { + lineiterator *li = (lineiterator*)self; Py_DECREF(li->li_code); Py_TYPE(li)->tp_free(li); } @@ -1178,8 +1287,9 @@ _source_offset_converter(int *value) { } static PyObject * -lineiter_next(lineiterator *li) +lineiter_next(PyObject *self) { + lineiterator *li = (lineiterator*)self; PyCodeAddressRange *bounds = &li->li_line; if (!_PyLineTable_NextAddressRange(bounds)) { return NULL; @@ -1203,7 +1313,7 @@ PyTypeObject _PyLineIterator = { sizeof(lineiterator), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)lineiter_dealloc, /* tp_dealloc */ + lineiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1225,7 +1335,7 @@ PyTypeObject _PyLineIterator = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)lineiter_next, /* tp_iternext */ + lineiter_next, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1237,7 +1347,7 @@ PyTypeObject _PyLineIterator = { 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; static lineiterator * @@ -1264,15 +1374,17 @@ typedef struct { } positionsiterator; static void -positionsiter_dealloc(positionsiterator* pi) +positionsiter_dealloc(PyObject *self) { + positionsiterator *pi = (positionsiterator*)self; Py_DECREF(pi->pi_code); Py_TYPE(pi)->tp_free(pi); } static PyObject* -positionsiter_next(positionsiterator* pi) +positionsiter_next(PyObject *self) { + positionsiterator *pi = (positionsiterator*)self; if (pi->pi_offset >= pi->pi_range.ar_end) { assert(pi->pi_offset == pi->pi_range.ar_end); if (at_end(&pi->pi_range)) { @@ -1294,7 +1406,7 @@ PyTypeObject _PyPositionsIterator = { sizeof(positionsiterator), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)positionsiter_dealloc, /* tp_dealloc */ + positionsiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1316,7 +1428,7 @@ PyTypeObject _PyPositionsIterator = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)positionsiter_next, /* tp_iternext */ + positionsiter_next, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1328,12 +1440,13 @@ PyTypeObject _PyPositionsIterator = { 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; static PyObject* -code_positionsiterator(PyCodeObject* code, PyObject* Py_UNUSED(args)) +code_positionsiterator(PyObject *self, PyObject* Py_UNUSED(args)) { + PyCodeObject *code = (PyCodeObject*)self; positionsiterator* pi = (positionsiterator*)PyType_GenericAlloc(&_PyPositionsIterator, 0); if (pi == NULL) { return NULL; @@ -1486,47 +1599,39 @@ PyCode_GetFreevars(PyCodeObject *code) return _PyCode_GetFreevars(code); } +#ifdef _Py_TIER2 + static void clear_executors(PyCodeObject *co) { + assert(co->co_executors); for (int i = 0; i < co->co_executors->size; i++) { - Py_CLEAR(co->co_executors->executors[i]); + if (co->co_executors->executors[i]) { + _Py_ExecutorDetach(co->co_executors->executors[i]); + assert(co->co_executors->executors[i] == NULL); + } } PyMem_Free(co->co_executors); co->co_executors = NULL; } void -_PyCode_Clear_Executors(PyCodeObject *code) { - int code_len = (int)Py_SIZE(code); - for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - uint8_t opcode = instr->op.code; - uint8_t oparg = instr->op.arg; - if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *exec = code->co_executors->executors[oparg]; - assert(exec->vm_data.opcode != ENTER_EXECUTOR); - instr->op.code = exec->vm_data.opcode; - instr->op.arg = exec->vm_data.oparg; - } - } +_PyCode_Clear_Executors(PyCodeObject *code) +{ clear_executors(code); } +#endif + static void deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { Py_ssize_t len = Py_SIZE(code); for (int i = 0; i < len; i++) { - int opcode = _Py_GetBaseOpcode(code, i); - if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *exec = code->co_executors->executors[instructions[i].op.arg]; - opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; - instructions[i].op.arg = exec->vm_data.oparg; - } - assert(opcode != ENTER_EXECUTOR); - int caches = _PyOpcode_Caches[opcode]; - instructions[i].op.code = opcode; + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code, i); + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); + int caches = _PyOpcode_Caches[inst.op.code]; + instructions[i] = inst; for (int j = 1; j <= caches; j++) { instructions[i+j].cache = 0; } @@ -1718,6 +1823,11 @@ code_dealloc(PyCodeObject *co) } Py_SET_REFCNT(co, 0); +#ifdef Py_GIL_DISABLED + PyObject_GC_UnTrack(co); +#endif + + _PyFunction_ClearCodeByVersion(co->co_version); if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyCodeObjectExtra *co_extra = co->co_extra; @@ -1732,9 +1842,11 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } +#ifdef _Py_TIER2 if (co->co_executors != NULL) { clear_executors(co); } +#endif Py_XDECREF(co->co_consts); Py_XDECREF(co->co_names); @@ -1745,6 +1857,9 @@ code_dealloc(PyCodeObject *co) Py_XDECREF(co->co_qualname); Py_XDECREF(co->co_linetable); Py_XDECREF(co->co_exceptiontable); +#ifdef Py_GIL_DISABLED + assert(co->_co_unique_id == -1); +#endif if (co->_co_cached != NULL) { Py_XDECREF(co->_co_cached->_co_code); Py_XDECREF(co->_co_cached->_co_cellvars); @@ -1759,9 +1874,20 @@ code_dealloc(PyCodeObject *co) PyObject_Free(co); } +#ifdef Py_GIL_DISABLED +static int +code_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyCodeObject *co = (PyCodeObject*)self; + Py_VISIT(co->co_consts); + return 0; +} +#endif + static PyObject * -code_repr(PyCodeObject *co) +code_repr(PyObject *self) { + PyCodeObject *co = (PyCodeObject*)self; int lineno; if (co->co_firstlineno != 0) lineno = co->co_firstlineno; @@ -1812,33 +1938,12 @@ code_richcompare(PyObject *self, PyObject *other, int op) goto unequal; } for (int i = 0; i < Py_SIZE(co); i++) { - _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; - _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; - uint8_t co_code = _Py_GetBaseOpcode(co, i); - uint8_t co_arg = co_instr.op.arg; - uint8_t cp_code = _Py_GetBaseOpcode(cp, i); - uint8_t cp_arg = cp_instr.op.arg; - - if (co_code == ENTER_EXECUTOR) { - const int exec_index = co_arg; - _PyExecutorObject *exec = co->co_executors->executors[exec_index]; - co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; - co_arg = exec->vm_data.oparg; - } - assert(co_code != ENTER_EXECUTOR); - - if (cp_code == ENTER_EXECUTOR) { - const int exec_index = cp_arg; - _PyExecutorObject *exec = cp->co_executors->executors[exec_index]; - cp_code = _PyOpcode_Deopt[exec->vm_data.opcode]; - cp_arg = exec->vm_data.oparg; - } - assert(cp_code != ENTER_EXECUTOR); - - if (co_code != cp_code || co_arg != cp_arg) { + _Py_CODEUNIT co_instr = _Py_GetBaseCodeUnit(co, i); + _Py_CODEUNIT cp_instr = _Py_GetBaseCodeUnit(cp, i); + if (co_instr.cache != cp_instr.cache) { goto unequal; } - i += _PyOpcode_Caches[co_code]; + i += _PyOpcode_Caches[co_instr.op.code]; } /* compare constants */ @@ -1889,12 +1994,13 @@ code_richcompare(PyObject *self, PyObject *other, int op) } static Py_hash_t -code_hash(PyCodeObject *co) +code_hash(PyObject *self) { + PyCodeObject *co = (PyCodeObject*)self; Py_uhash_t uhash = 20221211; #define SCRAMBLE_IN(H) do { \ uhash ^= (Py_uhash_t)(H); \ - uhash *= _PyHASH_MULTIPLIER; \ + uhash *= PyHASH_MULTIPLIER; \ } while (0) #define SCRAMBLE_IN_HASH(EXPR) do { \ Py_hash_t h = PyObject_Hash(EXPR); \ @@ -1917,22 +2023,10 @@ code_hash(PyCodeObject *co) SCRAMBLE_IN(co->co_firstlineno); SCRAMBLE_IN(Py_SIZE(co)); for (int i = 0; i < Py_SIZE(co); i++) { - _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; - uint8_t co_code = co_instr.op.code; - uint8_t co_arg = co_instr.op.arg; - if (co_code == ENTER_EXECUTOR) { - _PyExecutorObject *exec = co->co_executors->executors[co_arg]; - assert(exec != NULL); - assert(exec->vm_data.opcode != ENTER_EXECUTOR); - co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; - co_arg = exec->vm_data.oparg; - } - else { - co_code = _Py_GetBaseOpcode(co, i); - } - SCRAMBLE_IN(co_code); - SCRAMBLE_IN(co_arg); - i += _PyOpcode_Caches[co_code]; + _Py_CODEUNIT co_instr = _Py_GetBaseCodeUnit(co, i); + SCRAMBLE_IN(co_instr.op.code); + SCRAMBLE_IN(co_instr.op.arg); + i += _PyOpcode_Caches[co_instr.op.code]; } if ((Py_hash_t)uhash == -1) { return -2; @@ -1963,8 +2057,9 @@ static PyMemberDef code_memberlist[] = { static PyObject * -code_getlnotab(PyCodeObject *code, void *closure) +code_getlnotab(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; if (PyErr_WarnEx(PyExc_DeprecationWarning, "co_lnotab is deprecated, use co_lines instead.", 1) < 0) { @@ -1974,51 +2069,57 @@ code_getlnotab(PyCodeObject *code, void *closure) } static PyObject * -code_getvarnames(PyCodeObject *code, void *closure) +code_getvarnames(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return _PyCode_GetVarnames(code); } static PyObject * -code_getcellvars(PyCodeObject *code, void *closure) +code_getcellvars(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return _PyCode_GetCellvars(code); } static PyObject * -code_getfreevars(PyCodeObject *code, void *closure) +code_getfreevars(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return _PyCode_GetFreevars(code); } static PyObject * -code_getcodeadaptive(PyCodeObject *code, void *closure) +code_getcodeadaptive(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return PyBytes_FromStringAndSize(code->co_code_adaptive, _PyCode_NBYTES(code)); } static PyObject * -code_getcode(PyCodeObject *code, void *closure) +code_getcode(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return _PyCode_GetCode(code); } static PyGetSetDef code_getsetlist[] = { - {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, - {"_co_code_adaptive", (getter)code_getcodeadaptive, NULL, NULL}, + {"co_lnotab", code_getlnotab, NULL, NULL}, + {"_co_code_adaptive", code_getcodeadaptive, NULL, NULL}, // The following old names are kept for backward compatibility. - {"co_varnames", (getter)code_getvarnames, NULL, NULL}, - {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, - {"co_freevars", (getter)code_getfreevars, NULL, NULL}, - {"co_code", (getter)code_getcode, NULL, NULL}, + {"co_varnames", code_getvarnames, NULL, NULL}, + {"co_cellvars", code_getcellvars, NULL, NULL}, + {"co_freevars", code_getfreevars, NULL, NULL}, + {"co_code", code_getcode, NULL, NULL}, {0} }; static PyObject * -code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) +code_sizeof(PyObject *self, PyObject *Py_UNUSED(args)) { + PyCodeObject *co = (PyCodeObject*)self; size_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co)); _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; if (co_extra != NULL) { @@ -2029,8 +2130,9 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) } static PyObject * -code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) +code_linesiterator(PyObject *self, PyObject *Py_UNUSED(args)) { + PyCodeObject *code = (PyCodeObject*)self; return (PyObject *)new_linesiterator(code); } @@ -2172,12 +2274,13 @@ code__varname_from_oparg_impl(PyCodeObject *self, int oparg) /* XXX code objects need to participate in GC? */ static struct PyMethodDef code_methods[] = { - {"__sizeof__", (PyCFunction)code_sizeof, METH_NOARGS}, - {"co_lines", (PyCFunction)code_linesiterator, METH_NOARGS}, - {"co_positions", (PyCFunction)code_positionsiterator, METH_NOARGS}, + {"__sizeof__", code_sizeof, METH_NOARGS}, + {"co_lines", code_linesiterator, METH_NOARGS}, + {"co_positions", code_positionsiterator, METH_NOARGS}, CODE_REPLACE_METHODDEF CODE__VARNAME_FROM_OPARG_METHODDEF - {"__replace__", _PyCFunction_CAST(code_replace), METH_FASTCALL|METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(code_replace), METH_FASTCALL|METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {NULL, NULL} /* sentinel */ }; @@ -2192,19 +2295,27 @@ PyTypeObject PyCode_Type = { 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)code_repr, /* tp_repr */ + code_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)code_hash, /* tp_hash */ + code_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ +#ifdef Py_GIL_DISABLED + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ +#else Py_TPFLAGS_DEFAULT, /* tp_flags */ +#endif code_new__doc__, /* tp_doc */ +#ifdef Py_GIL_DISABLED + code_traverse, /* tp_traverse */ +#else 0, /* tp_traverse */ +#endif 0, /* tp_clear */ code_richcompare, /* tp_richcompare */ offsetof(PyCodeObject, co_weakreflist), /* tp_weaklistoffset */ @@ -2237,6 +2348,7 @@ _PyCode_ConstantKey(PyObject *op) if (op == Py_None || op == Py_Ellipsis || PyLong_CheckExact(op) || PyUnicode_CheckExact(op) + || PySlice_Check(op) /* code_richcompare() uses _PyCode_ConstantKey() internally */ || PyCode_Check(op)) { @@ -2357,48 +2469,182 @@ _PyCode_ConstantKey(PyObject *op) return key; } -void -_PyStaticCode_Fini(PyCodeObject *co) +#ifdef Py_GIL_DISABLED +static PyObject * +intern_one_constant(PyObject *op) { - deopt_code(co, _PyCode_CODE(co)); - if (co->co_executors != NULL) { - clear_executors(co); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_hashtable_t *consts = interp->code_state.constants; + + assert(!PyUnicode_CheckExact(op)); // strings are interned separately + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts, op); + if (entry == NULL) { + if (_Py_hashtable_set(consts, op, op) != 0) { + return NULL; + } + +#ifdef Py_REF_DEBUG + Py_ssize_t refcnt = Py_REFCNT(op); + if (refcnt != 1) { + // Adjust the reftotal to account for the fact that we only + // restore a single reference in _PyCode_Fini. + _Py_AddRefTotal(_PyThreadState_GET(), -(refcnt - 1)); + } +#endif + + _Py_SetImmortal(op); + return op; } - PyMem_Free(co->co_extra); - if (co->_co_cached != NULL) { - Py_CLEAR(co->_co_cached->_co_code); - Py_CLEAR(co->_co_cached->_co_cellvars); - Py_CLEAR(co->_co_cached->_co_freevars); - Py_CLEAR(co->_co_cached->_co_varnames); - PyMem_Free(co->_co_cached); - co->_co_cached = NULL; + + assert(_Py_IsImmortal(entry->value)); + return (PyObject *)entry->value; +} + +static int +compare_constants(const void *key1, const void *key2) { + PyObject *op1 = (PyObject *)key1; + PyObject *op2 = (PyObject *)key2; + if (op1 == op2) { + return 1; } - co->co_extra = NULL; - if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)co); - co->co_weakreflist = NULL; + if (Py_TYPE(op1) != Py_TYPE(op2)) { + return 0; } - free_monitoring_data(co->_co_monitoring); - co->_co_monitoring = NULL; + // We compare container contents by identity because we have already + // internalized the items. + if (PyTuple_CheckExact(op1)) { + Py_ssize_t size = PyTuple_GET_SIZE(op1); + if (size != PyTuple_GET_SIZE(op2)) { + return 0; + } + for (Py_ssize_t i = 0; i < size; i++) { + if (PyTuple_GET_ITEM(op1, i) != PyTuple_GET_ITEM(op2, i)) { + return 0; + } + } + return 1; + } + else if (PyFrozenSet_CheckExact(op1)) { + if (PySet_GET_SIZE(op1) != PySet_GET_SIZE(op2)) { + return 0; + } + Py_ssize_t pos1 = 0, pos2 = 0; + PyObject *obj1, *obj2; + Py_hash_t hash1, hash2; + while ((_PySet_NextEntry(op1, &pos1, &obj1, &hash1)) && + (_PySet_NextEntry(op2, &pos2, &obj2, &hash2))) + { + if (obj1 != obj2) { + return 0; + } + } + return 1; + } + else if (PySlice_Check(op1)) { + PySliceObject *s1 = (PySliceObject *)op1; + PySliceObject *s2 = (PySliceObject *)op2; + return (s1->start == s2->start && + s1->stop == s2->stop && + s1->step == s2->step); + } + else if (PyBytes_CheckExact(op1) || PyLong_CheckExact(op1)) { + return PyObject_RichCompareBool(op1, op2, Py_EQ); + } + else if (PyFloat_CheckExact(op1)) { + // Ensure that, for example, +0.0 and -0.0 are distinct + double f1 = PyFloat_AS_DOUBLE(op1); + double f2 = PyFloat_AS_DOUBLE(op2); + return memcmp(&f1, &f2, sizeof(double)) == 0; + } + else if (PyComplex_CheckExact(op1)) { + Py_complex c1 = ((PyComplexObject *)op1)->cval; + Py_complex c2 = ((PyComplexObject *)op2)->cval; + return memcmp(&c1, &c2, sizeof(Py_complex)) == 0; + } + _Py_FatalErrorFormat("unexpected type in compare_constants: %s", + Py_TYPE(op1)->tp_name); + return 0; } -int -_PyStaticCode_Init(PyCodeObject *co) +static Py_uhash_t +hash_const(const void *key) { - int res = intern_strings(co->co_names); - if (res < 0) { - return -1; + PyObject *op = (PyObject *)key; + if (PySlice_Check(op)) { + PySliceObject *s = (PySliceObject *)op; + PyObject *data[3] = { s->start, s->stop, s->step }; + return Py_HashBuffer(&data, sizeof(data)); } - res = intern_string_constants(co->co_consts, NULL); - if (res < 0) { - return -1; + else if (PyTuple_CheckExact(op)) { + Py_ssize_t size = PyTuple_GET_SIZE(op); + PyObject **data = _PyTuple_ITEMS(op); + return Py_HashBuffer(data, sizeof(PyObject *) * size); } - res = intern_strings(co->co_localsplusnames); - if (res < 0) { - return -1; + Py_hash_t h = PyObject_Hash(op); + if (h == -1) { + // This should never happen: all the constants we support have + // infallible hash functions. + Py_FatalError("code: hash failed"); + } + return (Py_uhash_t)h; +} + +static int +clear_containers(_Py_hashtable_t *ht, const void *key, const void *value, + void *user_data) +{ + // First clear containers to avoid recursive deallocation later on in + // destroy_key. + PyObject *op = (PyObject *)key; + if (PyTuple_CheckExact(op)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(op); i++) { + Py_CLEAR(_PyTuple_ITEMS(op)[i]); + } + } + else if (PySlice_Check(op)) { + PySliceObject *slice = (PySliceObject *)op; + Py_SETREF(slice->start, Py_None); + Py_SETREF(slice->stop, Py_None); + Py_SETREF(slice->step, Py_None); + } + else if (PyFrozenSet_CheckExact(op)) { + _PySet_ClearInternal((PySetObject *)op); } - _PyCode_Quicken(co); return 0; } -#define MAX_CODE_UNITS_PER_LOC_ENTRY 8 +static void +destroy_key(void *key) +{ + _Py_ClearImmortal(key); +} +#endif + +PyStatus +_PyCode_Init(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + struct _py_code_state *state = &interp->code_state; + state->constants = _Py_hashtable_new_full(&hash_const, &compare_constants, + &destroy_key, NULL, NULL); + if (state->constants == NULL) { + return _PyStatus_NO_MEMORY(); + } +#endif + return _PyStatus_OK(); +} + +void +_PyCode_Fini(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + // Free interned constants + struct _py_code_state *state = &interp->code_state; + if (state->constants) { + _Py_hashtable_foreach(state->constants, &clear_containers, NULL); + _Py_hashtable_destroy(state->constants); + state->constants = NULL; + } +#endif +} diff --git a/Objects/complexobject.c b/Objects/complexobject.c index d8b0e84da5df4a..787235c63a6be1 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -88,8 +88,7 @@ _Py_c_quot(Py_complex a, Py_complex b) * numerators and denominator by whichever of {b.real, b.imag} has * larger magnitude. The earliest reference I found was to CACM * Algorithm 116 (Complex Division, Robert L. Smith, Stanford - * University). As usual, though, we're still ignoring all IEEE - * endcases. + * University). */ Py_complex r; /* the result */ const double abs_breal = b.real < 0 ? -b.real : b.real; @@ -120,6 +119,28 @@ _Py_c_quot(Py_complex a, Py_complex b) /* At least one of b.real or b.imag is a NaN */ r.real = r.imag = Py_NAN; } + + /* Recover infinities and zeros that computed as nan+nanj. See e.g. + the C11, Annex G.5.2, routine _Cdivd(). */ + if (isnan(r.real) && isnan(r.imag)) { + if ((isinf(a.real) || isinf(a.imag)) + && isfinite(b.real) && isfinite(b.imag)) + { + const double x = copysign(isinf(a.real) ? 1.0 : 0.0, a.real); + const double y = copysign(isinf(a.imag) ? 1.0 : 0.0, a.imag); + r.real = Py_INFINITY * (x*b.real + y*b.imag); + r.imag = Py_INFINITY * (y*b.real - x*b.imag); + } + else if ((isinf(abs_breal) || isinf(abs_bimag)) + && isfinite(a.real) && isfinite(a.imag)) + { + const double x = copysign(isinf(b.real) ? 1.0 : 0.0, b.real); + const double y = copysign(isinf(b.imag) ? 1.0 : 0.0, b.imag); + r.real = 0.0 * (a.real*x + a.imag*y); + r.imag = 0.0 * (a.imag*x - a.real*y); + } + } + return r; } #ifdef _M_ARM64 @@ -152,6 +173,8 @@ _Py_c_pow(Py_complex a, Py_complex b) } r.real = len*cos(phase); r.imag = len*sin(phase); + + _Py_ADJUST_ERANGE2(r.real, r.imag); } return r; } @@ -188,16 +211,16 @@ _Py_c_abs(Py_complex z) /* sets errno = ERANGE on overflow; otherwise errno = 0 */ double result; - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { + if (!isfinite(z.real) || !isfinite(z.imag)) { /* C99 rules: if either the real or the imaginary part is an infinity, return infinity, even if the other part is a NaN. */ - if (Py_IS_INFINITY(z.real)) { + if (isinf(z.real)) { result = fabs(z.real); errno = 0; return result; } - if (Py_IS_INFINITY(z.imag)) { + if (isinf(z.imag)) { result = fabs(z.imag); errno = 0; return result; @@ -207,7 +230,7 @@ _Py_c_abs(Py_complex z) return Py_NAN; } result = hypot(z.real, z.imag); - if (!Py_IS_FINITE(result)) + if (!isfinite(result)) errno = ERANGE; else errno = 0; @@ -523,7 +546,7 @@ complex_div(PyObject *v, PyObject *w) errno = 0; quot = _Py_c_quot(a, b); if (errno == EDOM) { - PyErr_SetString(PyExc_ZeroDivisionError, "complex division by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } return PyComplex_FromCComplex(quot); @@ -546,15 +569,15 @@ complex_pow(PyObject *v, PyObject *w, PyObject *z) // a faster and more accurate algorithm. if (b.imag == 0.0 && b.real == floor(b.real) && fabs(b.real) <= 100.0) { p = c_powi(a, (long)b.real); + _Py_ADJUST_ERANGE2(p.real, p.imag); } else { p = _Py_c_pow(a, b); } - _Py_ADJUST_ERANGE2(p.real, p.imag); if (errno == EDOM) { PyErr_SetString(PyExc_ZeroDivisionError, - "0.0 to a negative or complex power"); + "zero to a negative or complex power"); return NULL; } else if (errno == ERANGE) { @@ -736,22 +759,6 @@ complex___complex___impl(PyComplexObject *self) } -static PyMethodDef complex_methods[] = { - COMPLEX_CONJUGATE_METHODDEF - COMPLEX___COMPLEX___METHODDEF - COMPLEX___GETNEWARGS___METHODDEF - COMPLEX___FORMAT___METHODDEF - {NULL, NULL} /* sentinel */ -}; - -static PyMemberDef complex_members[] = { - {"real", Py_T_DOUBLE, offsetof(PyComplexObject, cval.real), Py_READONLY, - "the real part of a complex number"}, - {"imag", Py_T_DOUBLE, offsetof(PyComplexObject, cval.imag), Py_READONLY, - "the imaginary part of a complex number"}, - {0}, -}; - static PyObject * complex_from_string_inner(const char *s, Py_ssize_t len, void *type) { @@ -894,8 +901,8 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) } else { PyErr_Format(PyExc_TypeError, - "complex() argument must be a string or a number, not '%.200s'", - Py_TYPE(v)->tp_name); + "complex() argument must be a string or a number, not %T", + v); return NULL; } @@ -905,20 +912,94 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) return result; } +/* The constructor should only accept a string as a positional argument, + * not as by the 'real' keyword. But Argument Clinic does not allow + * to distinguish between argument passed positionally and by keyword. + * So the constructor must be split into two parts: actual_complex_new() + * handles the case of no arguments and one positional argument, and calls + * complex_new(), implemented with Argument Clinic, to handle the remaining + * cases: 'real' and 'imag' arguments. This separation is well suited + * for different constructor roles: converting a string or number to a complex + * number and constructing a complex number from real and imaginary parts. + */ +static PyObject * +actual_complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *res = NULL; + PyNumberMethods *nbr; + + if (PyTuple_GET_SIZE(args) > 1 || (kwargs != NULL && PyDict_GET_SIZE(kwargs))) { + return complex_new(type, args, kwargs); + } + if (!PyTuple_GET_SIZE(args)) { + return complex_subtype_from_doubles(type, 0, 0); + } + + PyObject *arg = PyTuple_GET_ITEM(args, 0); + /* Special-case for a single argument when type(arg) is complex. */ + if (PyComplex_CheckExact(arg) && type == &PyComplex_Type) { + /* Note that we can't know whether it's safe to return + a complex *subclass* instance as-is, hence the restriction + to exact complexes here. If either the input or the + output is a complex subclass, it will be handled below + as a non-orthogonal vector. */ + return Py_NewRef(arg); + } + if (PyUnicode_Check(arg)) { + return complex_subtype_from_string(type, arg); + } + PyObject *tmp = try_complex_special_method(arg); + if (tmp) { + Py_complex c = ((PyComplexObject*)tmp)->cval; + res = complex_subtype_from_doubles(type, c.real, c.imag); + Py_DECREF(tmp); + } + else if (PyErr_Occurred()) { + return NULL; + } + else if (PyComplex_Check(arg)) { + /* Note that if arg is of a complex subtype, we're only + retaining its real & imag parts here, and the return + value is (properly) of the builtin complex type. */ + Py_complex c = ((PyComplexObject*)arg)->cval; + res = complex_subtype_from_doubles(type, c.real, c.imag); + } + else if ((nbr = Py_TYPE(arg)->tp_as_number) != NULL && + (nbr->nb_float != NULL || nbr->nb_index != NULL)) + { + /* The argument really is entirely real, and contributes + nothing in the imaginary direction. + Just treat it as a double. */ + double r = PyFloat_AsDouble(arg); + if (r != -1.0 || !PyErr_Occurred()) { + res = complex_subtype_from_doubles(type, r, 0); + } + } + else { + PyErr_Format(PyExc_TypeError, + "complex() argument must be a string or a number, not %T", + arg); + } + return res; +} + /*[clinic input] @classmethod complex.__new__ as complex_new real as r: object(c_default="NULL") = 0 imag as i: object(c_default="NULL") = 0 -Create a complex number from a real part and an optional imaginary part. +Create a complex number from a string or numbers. -This is equivalent to (real + imag*1j) where imag defaults to 0. +If a string is given, parse it as a complex number. +If a single number is given, convert it to a complex number. +If the 'real' or 'imag' arguments are given, create a complex number +with the specified real and imaginary components. [clinic start generated code]*/ static PyObject * complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) -/*[clinic end generated code: output=b6c7dd577b537dc1 input=f4c667f2596d4fd1]*/ +/*[clinic end generated code: output=b6c7dd577b537dc1 input=ff4268dc540958a4]*/ { PyObject *tmp; PyNumberMethods *nbr, *nbi = NULL; @@ -930,32 +1011,10 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) if (r == NULL) { r = _PyLong_GetZero(); } + PyObject *orig_r = r; - /* Special-case for a single argument when type(arg) is complex. */ - if (PyComplex_CheckExact(r) && i == NULL && - type == &PyComplex_Type) { - /* Note that we can't know whether it's safe to return - a complex *subclass* instance as-is, hence the restriction - to exact complexes here. If either the input or the - output is a complex subclass, it will be handled below - as a non-orthogonal vector. */ - return Py_NewRef(r); - } - if (PyUnicode_Check(r)) { - if (i != NULL) { - PyErr_SetString(PyExc_TypeError, - "complex() can't take second arg" - " if first is a string"); - return NULL; - } - return complex_subtype_from_string(type, r); - } - if (i != NULL && PyUnicode_Check(i)) { - PyErr_SetString(PyExc_TypeError, - "complex() second arg can't be a string"); - return NULL; - } - + /* DEPRECATED: The call of try_complex_special_method() for the "real" + * part will be dropped after the end of the deprecation period. */ tmp = try_complex_special_method(r); if (tmp) { r = tmp; @@ -970,9 +1029,8 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) (nbr->nb_float == NULL && nbr->nb_index == NULL && !PyComplex_Check(r))) { PyErr_Format(PyExc_TypeError, - "complex() first argument must be a string or a number, " - "not '%.200s'", - Py_TYPE(r)->tp_name); + "complex() argument 'real' must be a real number, not %T", + r); if (own_r) { Py_DECREF(r); } @@ -984,9 +1042,8 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) (nbi->nb_float == NULL && nbi->nb_index == NULL && !PyComplex_Check(i))) { PyErr_Format(PyExc_TypeError, - "complex() second argument must be a number, " - "not '%.200s'", - Py_TYPE(i)->tp_name); + "complex() argument 'imag' must be a real number, not %T", + i); if (own_r) { Py_DECREF(r); } @@ -998,6 +1055,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) both be treated as numbers, and the constructor should return a complex number equal to (real + imag*1j). + The following is DEPRECATED: Note that we do NOT assume the input to already be in canonical form; the "real" and "imag" parts might themselves be complex numbers, which slightly complicates the code below. */ @@ -1008,19 +1066,27 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) cr = ((PyComplexObject*)r)->cval; cr_is_complex = 1; if (own_r) { + /* r was a newly created complex number, rather + than the original "real" argument. */ Py_DECREF(r); } + nbr = Py_TYPE(orig_r)->tp_as_number; + if (nbr == NULL || + (nbr->nb_float == NULL && nbr->nb_index == NULL)) + { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "complex() argument 'real' must be a real number, not %T", + orig_r)) { + return NULL; + } + } } else { /* The "real" part really is entirely real, and contributes nothing in the imaginary direction. Just treat it as a double. */ tmp = PyNumber_Float(r); - if (own_r) { - /* r was a newly created complex number, rather - than the original "real" argument. */ - Py_DECREF(r); - } + assert(!own_r); if (tmp == NULL) return NULL; assert(PyFloat_Check(tmp)); @@ -1032,6 +1098,11 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) ci.real = cr.imag; } else if (PyComplex_Check(i)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "complex() argument 'imag' must be a real number, not %T", + i)) { + return NULL; + } ci = ((PyComplexObject*)i)->cval; ci_is_complex = 1; } else { @@ -1057,6 +1128,52 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) return complex_subtype_from_doubles(type, cr.real, ci.real); } +/*[clinic input] +@classmethod +complex.from_number + + number: object + / + +Convert number to a complex floating-point number. +[clinic start generated code]*/ + +static PyObject * +complex_from_number(PyTypeObject *type, PyObject *number) +/*[clinic end generated code: output=658a7a5fb0de074d input=3f8bdd3a2bc3facd]*/ +{ + if (PyComplex_CheckExact(number) && type == &PyComplex_Type) { + Py_INCREF(number); + return number; + } + Py_complex cv = PyComplex_AsCComplex(number); + if (cv.real == -1.0 && PyErr_Occurred()) { + return NULL; + } + PyObject *result = PyComplex_FromCComplex(cv); + if (type != &PyComplex_Type && result != NULL) { + Py_SETREF(result, PyObject_CallOneArg((PyObject *)type, result)); + } + return result; +} + +static PyMethodDef complex_methods[] = { + COMPLEX_FROM_NUMBER_METHODDEF + COMPLEX_CONJUGATE_METHODDEF + COMPLEX___COMPLEX___METHODDEF + COMPLEX___GETNEWARGS___METHODDEF + COMPLEX___FORMAT___METHODDEF + {NULL, NULL} /* sentinel */ +}; + +static PyMemberDef complex_members[] = { + {"real", Py_T_DOUBLE, offsetof(PyComplexObject, cval.real), Py_READONLY, + "the real part of a complex number"}, + {"imag", Py_T_DOUBLE, offsetof(PyComplexObject, cval.imag), Py_READONLY, + "the imaginary part of a complex number"}, + {0}, +}; + static PyNumberMethods complex_as_number = { (binaryfunc)complex_add, /* nb_add */ (binaryfunc)complex_sub, /* nb_subtract */ @@ -1131,6 +1248,6 @@ PyTypeObject PyComplex_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - complex_new, /* tp_new */ - PyObject_Del, /* tp_free */ + actual_complex_new, /* tp_new */ + PyObject_Free, /* tp_free */ }; diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 571948229054d0..d280bf7adfc8e6 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -397,7 +397,7 @@ method_vectorcall_FASTCALL( if (method_check_args(func, args, nargs, kwnames)) { return NULL; } - _PyCFunctionFast meth = (_PyCFunctionFast) + PyCFunctionFast meth = (PyCFunctionFast) method_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -416,7 +416,7 @@ method_vectorcall_FASTCALL_KEYWORDS( if (method_check_args(func, args, nargs, NULL)) { return NULL; } - _PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords) + PyCFunctionFastWithKeywords meth = (PyCFunctionFastWithKeywords) method_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -913,6 +913,7 @@ descr_new(PyTypeObject *descrtype, PyTypeObject *type, const char *name) descr = (PyDescrObject *)PyType_GenericAlloc(descrtype, 0); if (descr != NULL) { + _PyObject_SetDeferredRefcount((PyObject *)descr); descr->d_type = (PyTypeObject*)Py_XNewRef(type); descr->d_name = PyUnicode_InternFromString(name); if (descr->d_name == NULL) { @@ -1169,8 +1170,8 @@ mappingproxy_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) static PyMethodDef mappingproxy_methods[] = { {"get", _PyCFunction_CAST(mappingproxy_get), METH_FASTCALL, - PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." - " d defaults to None.")}, + PyDoc_STR("get($self, key, default=None, /)\n--\n\n" + "Return the value for key if key is in the mapping, else default.")}, {"keys", mappingproxy_keys, METH_NOARGS, PyDoc_STR("D.keys() -> a set-like object providing a view on D's keys")}, {"values", mappingproxy_values, METH_NOARGS, @@ -1258,11 +1259,12 @@ mappingproxy.__new__ as mappingproxy_new mapping: object +Read-only proxy of a mapping. [clinic start generated code]*/ static PyObject * mappingproxy_new_impl(PyTypeObject *type, PyObject *mapping) -/*[clinic end generated code: output=65f27f02d5b68fa7 input=d2d620d4f598d4f8]*/ +/*[clinic end generated code: output=65f27f02d5b68fa7 input=c156df096ef7590c]*/ { mappingproxyobject *mappingproxy; @@ -1350,7 +1352,7 @@ wrapper_hash(PyObject *self) { wrapperobject *wp = (wrapperobject *)self; Py_hash_t x, y; - x = _Py_HashPointer(wp->self); + x = PyObject_GenericHash(wp->self); y = _Py_HashPointer(wp->descr); x = x ^ y; if (x == -1) @@ -1523,22 +1525,34 @@ class property(object): self.__doc__ = doc except AttributeError: # read-only or dict-less class pass + self.__name = None + + def __set_name__(self, owner, name): + self.__name = name + + @property + def __name__(self): + return self.__name if self.__name is not None else self.fget.__name__ + + @__name__.setter + def __name__(self, value): + self.__name = value def __get__(self, inst, type=None): if inst is None: return self if self.__get is None: - raise AttributeError, "property has no getter" + raise AttributeError("property has no getter") return self.__get(inst) def __set__(self, inst, value): if self.__set is None: - raise AttributeError, "property has no setter" + raise AttributeError("property has no setter") return self.__set(inst, value) def __delete__(self, inst): if self.__del is None: - raise AttributeError, "property has no deleter" + raise AttributeError("property has no deleter") return self.__del(inst) */ @@ -1632,6 +1646,20 @@ property_dealloc(PyObject *self) Py_TYPE(self)->tp_free(self); } +static int +property_name(propertyobject *prop, PyObject **name) +{ + if (prop->prop_name != NULL) { + *name = Py_NewRef(prop->prop_name); + return 1; + } + if (prop->prop_get == NULL) { + *name = NULL; + return 0; + } + return PyObject_GetOptionalAttr(prop->prop_get, &_Py_ID(__name__), name); +} + static PyObject * property_descr_get(PyObject *self, PyObject *obj, PyObject *type) { @@ -1641,11 +1669,15 @@ property_descr_get(PyObject *self, PyObject *obj, PyObject *type) propertyobject *gs = (propertyobject *)self; if (gs->prop_get == NULL) { + PyObject *propname; + if (property_name(gs, &propname) < 0) { + return NULL; + } PyObject *qualname = PyType_GetQualName(Py_TYPE(obj)); - if (gs->prop_name != NULL && qualname != NULL) { + if (propname != NULL && qualname != NULL) { PyErr_Format(PyExc_AttributeError, "property %R of %R object has no getter", - gs->prop_name, + propname, qualname); } else if (qualname != NULL) { @@ -1656,6 +1688,7 @@ property_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyErr_SetString(PyExc_AttributeError, "property has no getter"); } + Py_XDECREF(propname); Py_XDECREF(qualname); return NULL; } @@ -1677,16 +1710,20 @@ property_descr_set(PyObject *self, PyObject *obj, PyObject *value) } if (func == NULL) { + PyObject *propname; + if (property_name(gs, &propname) < 0) { + return -1; + } PyObject *qualname = NULL; if (obj != NULL) { qualname = PyType_GetQualName(Py_TYPE(obj)); } - if (gs->prop_name != NULL && qualname != NULL) { + if (propname != NULL && qualname != NULL) { PyErr_Format(PyExc_AttributeError, value == NULL ? "property %R of %R object has no deleter" : "property %R of %R object has no setter", - gs->prop_name, + propname, qualname); } else if (qualname != NULL) { @@ -1702,6 +1739,7 @@ property_descr_set(PyObject *self, PyObject *obj, PyObject *value) "property has no deleter" : "property has no setter"); } + Py_XDECREF(propname); Py_XDECREF(qualname); return -1; } @@ -1734,15 +1772,12 @@ property_copy(PyObject *old, PyObject *get, PyObject *set, PyObject *del) return NULL; if (get == NULL || get == Py_None) { - Py_XDECREF(get); get = pold->prop_get ? pold->prop_get : Py_None; } if (set == NULL || set == Py_None) { - Py_XDECREF(set); set = pold->prop_set ? pold->prop_set : Py_None; } if (del == NULL || del == Py_None) { - Py_XDECREF(del); del = pold->prop_del ? pold->prop_del : Py_None; } if (pold->getter_doc && get != Py_None) { @@ -1828,22 +1863,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, /* if no docstring given and the getter has one, use that one */ else if (fget != NULL) { int rc = PyObject_GetOptionalAttr(fget, &_Py_ID(__doc__), &prop_doc); - if (rc <= 0) { + if (rc < 0) { return rc; } - if (!Py_IS_TYPE(self, &PyProperty_Type) && - prop_doc != NULL && prop_doc != Py_None) { - // This oddity preserves the long existing behavior of surfacing - // an AttributeError when using a dict-less (__slots__) property - // subclass as a decorator on a getter method with a docstring. - // See PropertySubclassTest.test_slots_docstring_copy_exception. - int err = PyObject_SetAttr( - (PyObject *)self, &_Py_ID(__doc__), prop_doc); - if (err < 0) { - Py_DECREF(prop_doc); // release our new reference. - return -1; - } - } if (prop_doc == Py_None) { prop_doc = NULL; Py_DECREF(Py_None); @@ -1871,7 +1893,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, Py_DECREF(prop_doc); if (err < 0) { assert(PyErr_Occurred()); - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (!self->getter_doc && + PyErr_ExceptionMatches(PyExc_AttributeError)) + { PyErr_Clear(); // https://github.com/python/cpython/issues/98963#issuecomment-1574413319 // Python silently dropped this doc assignment through 3.11. @@ -1890,6 +1914,28 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, return 0; } +static PyObject * +property_get__name__(propertyobject *prop, void *Py_UNUSED(ignored)) +{ + PyObject *name; + if (property_name(prop, &name) < 0) { + return NULL; + } + if (name == NULL) { + PyErr_SetString(PyExc_AttributeError, + "'property' object has no attribute '__name__'"); + } + return name; +} + +static int +property_set__name__(propertyobject *prop, PyObject *value, + void *Py_UNUSED(ignored)) +{ + Py_XSETREF(prop->prop_name, Py_XNewRef(value)); + return 0; +} + static PyObject * property_get___isabstractmethod__(propertyobject *prop, void *closure) { @@ -1920,6 +1966,7 @@ property_get___isabstractmethod__(propertyobject *prop, void *closure) } static PyGetSetDef property_getsetlist[] = { + {"__name__", (getter)property_get__name__, (setter)property_set__name__}, {"__isabstractmethod__", (getter)property_get___isabstractmethod__, NULL, NULL, @@ -1972,7 +2019,7 @@ PyTypeObject PyDictProxy_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MAPPING, /* tp_flags */ - 0, /* tp_doc */ + mappingproxy_new__doc__, /* tp_doc */ mappingproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ mappingproxy_richcompare, /* tp_richcompare */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 9b1defa5cbc609..f28a92657834be 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -21,6 +21,7 @@ As of Python 3.6, this is compact and ordered. Basic idea is described here: | dk_log2_size | | dk_log2_index_bytes | | dk_kind | +| dk_version | | dk_usable | | dk_nentries | +---------------------+ @@ -33,7 +34,7 @@ As of Python 3.6, this is compact and ordered. Basic idea is described here: dk_indices is actual hashtable. It holds index in entries, or DKIX_EMPTY(-1) or DKIX_DUMMY(-2). -Size of indices is dk_size. Type of each index in indices is vary on dk_size: +Size of indices is dk_size. Type of each index in indices varies with dk_size: * int8 for dk_size <= 128 * int16 for 256 <= dk_size <= 2**15 @@ -55,7 +56,7 @@ The DictObject can be in one of two forms. Either: A combined table: ma_values == NULL, dk_refcnt == 1. - Values are stored in the me_value field of the PyDictKeysObject. + Values are stored in the me_value field of the PyDictKeyEntry. Or: A split table: ma_values != NULL, dk_refcnt >= 1 @@ -80,6 +81,8 @@ DK_ENTRIES(keys)[index] if index >= 0): Active upon key insertion. Dummy slots cannot be made Unused again else the probe sequence in case of collision would have no way to know they were once active. + In free-threaded builds dummy slots are not re-used to allow lock-free + lookups to proceed safely. 4. Pending. index >= 0, key != NULL, and value == NULL (split only) Not yet inserted in split-table. @@ -113,19 +116,20 @@ As a consequence of this, split keys have a maximum size of 16. #define PyDict_MINSIZE 8 #include "Python.h" -#include "pycore_bitutils.h" // _Py_bit_length -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_code.h" // stats -#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION -#include "pycore_dict.h" // export _PyDict_SizeOf() -#include "pycore_freelist.h" // _PyFreeListState_GET() -#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() -#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "stringlib/eq.h" // unicode_eq() +#include "pycore_bitutils.h" // _Py_bit_length +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_code.h" // stats +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // export _PyDict_SizeOf() +#include "pycore_freelist.h" // _PyFreeListState_GET() +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "stringlib/eq.h" // unicode_eq() #include @@ -150,13 +154,135 @@ ASSERT_DICT_LOCKED(PyObject *op) _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); } #define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + ASSERT_DICT_LOCKED(op); \ + } +#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); \ + } -#else +#define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp) +#define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]); +#define STORE_INDEX(keys, size, idx, value) _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)keys->dk_indices)[idx], (int##size##_t)value); +#define ASSERT_OWNED_OR_SHARED(mp) \ + assert(_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp)); + +#define LOCK_KEYS_IF_SPLIT(keys, kind) \ + if (kind == DICT_KEYS_SPLIT) { \ + LOCK_KEYS(keys); \ + } + +#define UNLOCK_KEYS_IF_SPLIT(keys, kind) \ + if (kind == DICT_KEYS_SPLIT) { \ + UNLOCK_KEYS(keys); \ + } + +static inline Py_ssize_t +load_keys_nentries(PyDictObject *mp) +{ + PyDictKeysObject *keys = _Py_atomic_load_ptr(&mp->ma_keys); + return _Py_atomic_load_ssize(&keys->dk_nentries); +} + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_keys, keys); +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) +{ + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_values, values); +} + +#define LOCK_KEYS(keys) PyMutex_LockFlags(&keys->dk_mutex, _Py_LOCK_DONT_DETACH) +#define UNLOCK_KEYS(keys) PyMutex_Unlock(&keys->dk_mutex) + +#define ASSERT_KEYS_LOCKED(keys) assert(PyMutex_IsLocked(&keys->dk_mutex)) +#define LOAD_SHARED_KEY(key) _Py_atomic_load_ptr_acquire(&key) +#define STORE_SHARED_KEY(key, value) _Py_atomic_store_ptr_release(&key, value) +// Inc refs the keys object, giving the previous value +#define INCREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, 1) +// Dec refs the keys object, giving the previous value +#define DECREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, -1) +#define LOAD_KEYS_NENTRIES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) + +#define INCREF_KEYS_FT(dk) dictkeys_incref(dk) +#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(_PyInterpreterState_GET(), dk, shared) + +static inline void split_keys_entry_added(PyDictKeysObject *keys) +{ + ASSERT_KEYS_LOCKED(keys); + + // We increase before we decrease so we never get too small of a value + // when we're racing with reads + _Py_atomic_store_ssize_relaxed(&keys->dk_nentries, keys->dk_nentries + 1); + _Py_atomic_store_ssize_release(&keys->dk_usable, keys->dk_usable - 1); +} + +#else /* Py_GIL_DISABLED */ #define ASSERT_DICT_LOCKED(op) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) +#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op) +#define LOCK_KEYS(keys) +#define UNLOCK_KEYS(keys) +#define ASSERT_KEYS_LOCKED(keys) +#define LOAD_SHARED_KEY(key) key +#define STORE_SHARED_KEY(key, value) key = value +#define INCREF_KEYS(dk) dk->dk_refcnt++ +#define DECREF_KEYS(dk) dk->dk_refcnt-- +#define LOAD_KEYS_NENTRIES(keys) keys->dk_nentries +#define INCREF_KEYS_FT(dk) +#define DECREF_KEYS_FT(dk, shared) +#define LOCK_KEYS_IF_SPLIT(keys, kind) +#define UNLOCK_KEYS_IF_SPLIT(keys, kind) +#define IS_DICT_SHARED(mp) (false) +#define SET_DICT_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) ((const int##size##_t*)(keys->dk_indices))[idx] +#define STORE_INDEX(keys, size, idx, value) ((int##size##_t*)(keys->dk_indices))[idx] = (int##size##_t)value + +static inline void split_keys_entry_added(PyDictKeysObject *keys) +{ + keys->dk_usable--; + keys->dk_nentries++; +} + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + mp->ma_keys = keys; +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) +{ + mp->ma_values = values; +} + +static inline Py_ssize_t +load_keys_nentries(PyDictObject *mp) +{ + return mp->ma_keys->dk_nentries; +} + #endif +#define STORE_KEY(ep, key) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_key, key) +#define STORE_VALUE(ep, value) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, value) +#define STORE_SPLIT_VALUE(mp, idx, value) FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_values->values[idx], value) +#define STORE_HASH(ep, hash) FT_ATOMIC_STORE_SSIZE_RELAXED(ep->me_hash, hash) +#define STORE_KEYS_USABLE(keys, usable) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_usable, usable) +#define STORE_KEYS_NENTRIES(keys, nentries) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_nentries, nentries) +#define STORE_USED(mp, used) FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, used) + #define PERTURB_SHIFT 5 /* @@ -256,81 +382,43 @@ static int dictresize(PyInterpreterState *interp, PyDictObject *mp, static PyObject* dict_iter(PyObject *dict); -static int -contains_lock_held(PyDictObject *mp, PyObject *key); -static int -contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash); static int setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value); static int dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, PyObject **result, int incref_result); -#include "clinic/dictobject.c.h" - - -#ifdef WITH_FREELISTS -static struct _Py_dict_freelist * -get_dict_state(void) -{ - _PyFreeListState *state = _PyFreeListState_GET(); - return &state->dicts; -} +#ifndef NDEBUG +static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); #endif +#include "clinic/dictobject.c.h" -void -_PyDict_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *state = &freelist_state->dicts; - while (state->numfree > 0) { - PyDictObject *op = state->free_list[--state->numfree]; - assert(PyDict_CheckExact(op)); - PyObject_GC_Del(op); - } - while (state->keys_numfree > 0) { - PyMem_Free(state->keys_free_list[--state->keys_numfree]); - } - if (is_finalization) { - state->numfree = -1; - state->keys_numfree = -1; - } -#endif -} - -void -_PyDict_Fini(PyInterpreterState *Py_UNUSED(interp)) -{ - // With Py_GIL_DISABLED: - // the freelists for the current thread state have already been cleared. -#ifndef Py_GIL_DISABLED - _PyFreeListState *state = _PyFreeListState_GET(); - _PyDict_ClearFreeList(state, 1); -#endif -} static inline Py_hash_t unicode_get_hash(PyObject *o) { assert(PyUnicode_CheckExact(o)); - return _PyASCIIObject_CAST(o)->hash; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(o)->hash); } /* Print summary info about the state of the optimized allocator */ void _PyDict_DebugMallocStats(FILE *out) { -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *state = get_dict_state(); _PyDebugAllocatorStats(out, "free PyDictObject", - state->numfree, sizeof(PyDictObject)); -#endif + _Py_FREELIST_SIZE(dicts), + sizeof(PyDictObject)); + _PyDebugAllocatorStats(out, "free PyDictKeysObject", + _Py_FREELIST_SIZE(dictkeys), + sizeof(PyDictKeysObject)); } #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyDictKeysObject *keys); +#define _Py_DICT_IMMORTAL_INITIAL_REFCNT PY_SSIZE_T_MIN + +static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); /* PyDictKeysObject has refcounts like PyObject does, so we have the following two functions to mirror what Py_INCREF() and Py_DECREF() do. @@ -342,26 +430,28 @@ static void free_keys_object(PyDictKeysObject *keys); static inline void dictkeys_incref(PyDictKeysObject *dk) { - if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); return; } #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif - dk->dk_refcnt++; + INCREF_KEYS(dk); } static inline void -dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) +dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) { - if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); return; } - assert(dk->dk_refcnt > 0); + assert(FT_ATOMIC_LOAD_SSIZE(dk->dk_refcnt) > 0); #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif - if (--dk->dk_refcnt == 0) { + if (DECREF_KEYS(dk) == 1) { if (DK_IS_UNICODE(dk)) { PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dk); Py_ssize_t i, n; @@ -378,7 +468,7 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) Py_XDECREF(entries[i].me_value); } } - free_keys_object(dk); + free_keys_object(dk, use_qsbr); } } @@ -390,22 +480,18 @@ dictkeys_get_index(const PyDictKeysObject *keys, Py_ssize_t i) Py_ssize_t ix; if (log2size < 8) { - const int8_t *indices = (const int8_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 8, i); } else if (log2size < 16) { - const int16_t *indices = (const int16_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 16, i); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - const int64_t *indices = (const int64_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 64, i); } #endif else { - const int32_t *indices = (const int32_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 32, i); } assert(ix >= DKIX_DUMMY); return ix; @@ -421,25 +507,21 @@ dictkeys_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix) assert(keys->dk_version == 0); if (log2size < 8) { - int8_t *indices = (int8_t*)(keys->dk_indices); assert(ix <= 0x7f); - indices[i] = (char)ix; + STORE_INDEX(keys, 8, i, ix); } else if (log2size < 16) { - int16_t *indices = (int16_t*)(keys->dk_indices); assert(ix <= 0x7fff); - indices[i] = (int16_t)ix; + STORE_INDEX(keys, 16, i, ix); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - int64_t *indices = (int64_t*)(keys->dk_indices); - indices[i] = ix; + STORE_INDEX(keys, 64, i, ix); } #endif else { - int32_t *indices = (int32_t*)(keys->dk_indices); assert(ix <= 0x7fffffff); - indices[i] = (int32_t)ix; + STORE_INDEX(keys, 32, i, ix); } } @@ -508,10 +590,13 @@ estimate_log2_keysize(Py_ssize_t n) * (which cannot fail and thus can do no allocation). */ static PyDictKeysObject empty_keys_struct = { - _Py_IMMORTAL_REFCNT, /* dk_refcnt */ + _Py_DICT_IMMORTAL_INITIAL_REFCNT, /* dk_refcnt */ 0, /* dk_log2_size */ 0, /* dk_log2_index_bytes */ DICT_KEYS_UNICODE, /* dk_kind */ +#ifdef Py_GIL_DISABLED + {0}, /* dk_mutex */ +#endif 1, /* dk_version */ 0, /* dk_usable (immutable) */ 0, /* dk_nentries */ @@ -534,8 +619,9 @@ static inline int get_index_from_order(PyDictObject *mp, Py_ssize_t i) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); - assert(i < (((char *)mp->ma_values)[-2])); - return ((char *)mp->ma_values)[-3-i]; + assert(i < mp->ma_values->size); + uint8_t *array = get_insertion_order_array(mp->ma_values); + return array[i]; } #ifdef DEBUG_PYDICT @@ -558,6 +644,8 @@ dump_entries(PyDictKeysObject *dk) int _PyDict_CheckConsistency(PyObject *op, int check_content) { + ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); + #define CHECK(expr) \ do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) @@ -569,10 +657,15 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) int splitted = _PyDict_HasSplitTable(mp); Py_ssize_t usable = USABLE_FRACTION(DK_SIZE(keys)); + // In the free-threaded build, shared keys may be concurrently modified, + // so use atomic loads. + Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable); + Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries); + CHECK(0 <= mp->ma_used && mp->ma_used <= usable); - CHECK(0 <= keys->dk_usable && keys->dk_usable <= usable); - CHECK(0 <= keys->dk_nentries && keys->dk_nentries <= usable); - CHECK(keys->dk_usable + keys->dk_nentries <= usable); + CHECK(0 <= dk_usable && dk_usable <= usable); + CHECK(0 <= dk_nentries && dk_nentries <= usable); + CHECK(dk_usable + dk_nentries <= usable); if (!splitted) { /* combined table */ @@ -582,9 +675,14 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) else { CHECK(keys->dk_kind == DICT_KEYS_SPLIT); CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + if (mp->ma_values->embedded) { + CHECK(mp->ma_values->embedded == 1); + CHECK(mp->ma_values->valid == 1); + } } if (check_content) { + LOCK_KEYS_IF_SPLIT(keys, keys->dk_kind); for (Py_ssize_t i=0; i < DK_SIZE(keys); i++) { Py_ssize_t ix = dictkeys_get_index(keys, i); CHECK(DKIX_DUMMY <= ix && ix <= usable); @@ -640,6 +738,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) CHECK(mp->ma_values->values[index] != NULL); } } + UNLOCK_KEYS_IF_SPLIT(keys, keys->dk_kind); } return 1; @@ -650,7 +749,6 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) { - PyDictKeysObject *dk; Py_ssize_t usable; int log2_bytes; size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry); @@ -673,15 +771,11 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) log2_bytes = log2_size + 2; } -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *state = get_dict_state(); - if (log2_size == PyDict_LOG_MINSIZE && unicode && state->keys_numfree > 0) { - dk = state->keys_free_list[--state->keys_numfree]; - OBJECT_STAT_INC(from_freelist); + PyDictKeysObject *dk = NULL; + if (log2_size == PyDict_LOG_MINSIZE && unicode) { + dk = _Py_FREELIST_POP_MEM(dictkeys); } - else -#endif - { + if (dk == NULL) { dk = PyMem_Malloc(sizeof(PyDictKeysObject) + ((size_t)1 << log2_bytes) + entry_size * usable); @@ -691,12 +785,15 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } } #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; dk->dk_log2_index_bytes = log2_bytes; dk->dk_kind = unicode ? DICT_KEYS_UNICODE : DICT_KEYS_GENERAL; +#ifdef Py_GIL_DISABLED + dk->dk_mutex = (PyMutex){0}; +#endif dk->dk_nentries = 0; dk->dk_usable = usable; dk->dk_version = 0; @@ -706,43 +803,60 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } static void -free_keys_object(PyDictKeysObject *keys) -{ -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *state = get_dict_state(); - if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE - && state->keys_numfree < PyDict_MAXFREELIST - && state->keys_numfree >= 0 - && DK_IS_UNICODE(keys)) { - state->keys_free_list[state->keys_numfree++] = keys; - OBJECT_STAT_INC(to_freelist); +free_keys_object(PyDictKeysObject *keys, bool use_qsbr) +{ +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeDelayed(keys); return; } #endif - PyMem_Free(keys); + if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) { + _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free); + } + else { + PyMem_Free(keys); + } +} + +static size_t +values_size_from_count(size_t count) +{ + assert(count >= 1); + size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *)); + assert(suffix_size < 128); + assert(suffix_size % sizeof(PyObject *) == 0); + return (count + 1) * sizeof(PyObject *) + suffix_size; } +#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) + static inline PyDictValues* new_values(size_t size) { - assert(size >= 1); - size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *)); - assert(prefix_size < 256); - size_t n = prefix_size + size * sizeof(PyObject *); - uint8_t *mem = PyMem_Malloc(n); - if (mem == NULL) { + size_t n = values_size_from_count(size); + PyDictValues *res = (PyDictValues *)PyMem_Malloc(n); + if (res == NULL) { return NULL; } - assert(prefix_size % sizeof(PyObject *) == 0); - mem[prefix_size-1] = (uint8_t)prefix_size; - return (PyDictValues*)(mem + prefix_size); + res->embedded = 0; + res->size = 0; + assert(size < 256); + res->capacity = (uint8_t)size; + return res; } static inline void -free_values(PyDictValues *values) +free_values(PyDictValues *values, bool use_qsbr) { - int prefix_size = ((uint8_t *)values)[-1]; - PyMem_Free(((char *)values)-prefix_size); + assert(values->embedded == 0); +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeDelayed(values); + return; + } +#endif + PyMem_Free(values); } /* Consumes a reference to the keys object */ @@ -751,54 +865,36 @@ new_dict(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { - PyDictObject *mp; assert(keys != NULL); -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *state = get_dict_state(); - if (state->numfree > 0) { - mp = state->free_list[--state->numfree]; - assert (mp != NULL); - assert (Py_IS_TYPE(mp, &PyDict_Type)); - OBJECT_STAT_INC(from_freelist); - _Py_NewReference((PyObject *)mp); - } - else -#endif - { + PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts); + if (mp == NULL) { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); if (free_values_on_failure) { - free_values(values); + free_values(values, false); } return NULL; } } + assert(Py_IS_TYPE(mp, &PyDict_Type)); mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; - mp->ma_version_tag = DICT_NEXT_VERSION(interp); + mp->_ma_watcher_tag = 0; ASSERT_CONSISTENT(mp); return (PyObject *)mp; } -static inline size_t -shared_keys_usable_size(PyDictKeysObject *keys) -{ - return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; -} - -/* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(interp, keys); return PyErr_NoMemory(); } - ((char *)values)[-2] = 0; + dictkeys_incref(keys); for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } @@ -860,7 +956,7 @@ clone_combined_dict_keys(PyDictObject *orig) we have it now; calling dictkeys_incref would be an error as keys->dk_refcnt is already set to 1 (after memcpy). */ #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif return keys; } @@ -897,7 +993,7 @@ lookdict_index(PyDictKeysObject *k, Py_hash_t hash, Py_ssize_t index) static inline Py_ALWAYS_INLINE Py_ssize_t do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, - Py_ssize_t (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) + int (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) { void *ep0 = _DK_ENTRIES(dk); size_t mask = DK_MASK(dk); @@ -907,7 +1003,7 @@ do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, for (;;) { ix = dictkeys_get_index(dk, i); if (ix >= 0) { - Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + int cmp = check_lookup(mp, dk, ep0, ix, key, hash); if (cmp < 0) { return cmp; } else if (cmp) { @@ -923,7 +1019,7 @@ do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, // Manual loop unrolling ix = dictkeys_get_index(dk, i); if (ix >= 0) { - Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + int cmp = check_lookup(mp, dk, ep0, ix, key, hash); if (cmp < 0) { return cmp; } else if (cmp) { @@ -939,7 +1035,7 @@ do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, Py_UNREACHABLE(); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline int compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { @@ -947,7 +1043,6 @@ compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, assert(ep->me_key != NULL); assert(PyUnicode_CheckExact(ep->me_key)); assert(!PyUnicode_CheckExact(key)); - // TODO: Thread safety if (unicode_get_hash(ep->me_key) == hash) { PyObject *startkey = ep->me_key; @@ -975,15 +1070,16 @@ unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key return do_lookup(mp, dk, key, hash, compare_unicode_generic); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline int compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + PyObject *ep_key = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key); + assert(ep_key != NULL); + assert(PyUnicode_CheckExact(ep_key)); + if (ep_key == key || + (unicode_get_hash(ep_key) == hash && unicode_eq(ep_key, key))) { return 1; } return 0; @@ -995,7 +1091,7 @@ unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) return do_lookup(NULL, dk, key, hash, compare_unicode_unicode); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline int compare_generic(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { @@ -1053,6 +1149,14 @@ _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key) return unicodekeys_lookup_unicode(dk, key, hash); } +#ifdef Py_GIL_DISABLED + +static Py_ssize_t +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, + Py_hash_t hash); + +#endif + /* The basic lookup function used by all operations. This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. @@ -1075,16 +1179,40 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu DictKeysKind kind; Py_ssize_t ix; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); start: dk = mp->ma_keys; kind = dk->dk_kind; if (kind != DICT_KEYS_GENERAL) { if (PyUnicode_CheckExact(key)) { +#ifdef Py_GIL_DISABLED + if (kind == DICT_KEYS_SPLIT) { + // A split dictionaries keys can be mutated by other + // dictionaries but if we have a unicode key we can avoid + // locking the shared keys. + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + LOCK_KEYS(dk); + ix = unicodekeys_lookup_unicode(dk, key, hash); + UNLOCK_KEYS(dk); + } + } + else { + ix = unicodekeys_lookup_unicode(dk, key, hash); + } +#else ix = unicodekeys_lookup_unicode(dk, key, hash); +#endif } else { + INCREF_KEYS_FT(dk); + LOCK_KEYS_IF_SPLIT(dk, kind); + ix = unicodekeys_lookup_generic(mp, dk, key, hash); + + UNLOCK_KEYS_IF_SPLIT(dk, kind); + DECREF_KEYS_FT(dk, IS_DICT_SHARED(mp)); if (ix == DKIX_KEY_CHANGED) { goto start; } @@ -1118,132 +1246,537 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu return ix; } -int -_PyDict_HasOnlyStringKeys(PyObject *dict) +#ifdef Py_GIL_DISABLED +static inline void +ensure_shared_on_read(PyDictObject *mp) { - Py_ssize_t pos = 0; - PyObject *key, *value; - assert(PyDict_Check(dict)); - /* Shortcut */ - if (((PyDictObject *)dict)->ma_keys->dk_kind != DICT_KEYS_GENERAL) - return 1; - while (PyDict_Next(dict, &pos, &key, &value)) - if (!PyUnicode_Check(key)) - return 0; - return 1; + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // The first time we access a dict from a non-owning thread we mark it + // as shared. This ensures that a concurrent resize operation will + // delay freeing the old keys or values using QSBR, which is necessary + // to safely allow concurrent reads without locking... + Py_BEGIN_CRITICAL_SECTION(mp); + if (!IS_DICT_SHARED(mp)) { + SET_DICT_SHARED(mp); + } + Py_END_CRITICAL_SECTION(); + } } +#endif -#define MAINTAIN_TRACKING(mp, key, value) \ - do { \ - if (!_PyObject_GC_IS_TRACKED(mp)) { \ - if (_PyObject_GC_MAY_BE_TRACKED(key) || \ - _PyObject_GC_MAY_BE_TRACKED(value)) { \ - _PyObject_GC_TRACK(mp); \ - } \ - } \ - } while(0) +static inline void +ensure_shared_on_resize(PyDictObject *mp) +{ +#ifdef Py_GIL_DISABLED + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // We are writing to the dict from another thread that owns + // it and we haven't marked it as shared which will ensure + // that when we re-size ma_keys or ma_values that we will + // free using QSBR. We need to lock the dictionary to + // contend with writes from the owning thread, mark it as + // shared, and then we can continue with lock-free reads. + // Technically this is a little heavy handed, we could just + // free the individual old keys / old-values using qsbr + SET_DICT_SHARED(mp); + } +#endif +} -void -_PyDict_MaybeUntrack(PyObject *op) +#ifdef Py_GIL_DISABLED + +static inline Py_ALWAYS_INLINE int +compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { - PyDictObject *mp; - PyObject *value; - Py_ssize_t i, numentries; + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(ep->me_key)); + assert(!PyUnicode_CheckExact(key)); - if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) - return; + if (startkey != NULL) { + if (!_Py_TryIncrefCompare(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } - mp = (PyDictObject *) op; - numentries = mp->ma_keys->dk_nentries; - if (_PyDict_HasSplitTable(mp)) { - for (i = 0; i < numentries; i++) { - if ((value = mp->ma_values->values[i]) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value)) { - return; + if (unicode_get_hash(startkey) == hash) { + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; } - } - } - else { - if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++) { - if ((value = ep0[i].me_value) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value)) - return; + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; } } else { - PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++) { - if ((value = ep0[i].me_value) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value) || - _PyObject_GC_MAY_BE_TRACKED(ep0[i].me_key)) - return; - } + Py_DECREF(startkey); } } - _PyObject_GC_UNTRACK(op); + return 0; } -/* Internal function to find slot for an item from its hash - when it is known that the key is not present in the dict. - - The dict must be combined. */ +// Search non-Unicode key from Unicode table static Py_ssize_t -find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) +unicodekeys_lookup_generic_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) { - assert(keys != NULL); + return do_lookup(mp, dk, key, hash, compare_unicode_generic_threadsafe); +} - const size_t mask = DK_MASK(keys); - size_t i = hash & mask; - Py_ssize_t ix = dictkeys_get_index(keys, i); - for (size_t perturb = hash; ix >= 0;) { - perturb >>= PERTURB_SHIFT; - i = (i*5 + perturb + 1) & mask; - ix = dictkeys_get_index(keys, i); +static inline Py_ALWAYS_INLINE int +compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(startkey)); + if (startkey == key) { + return 1; } - return i; + if (startkey != NULL) { + if (_Py_IsImmortal(startkey)) { + return unicode_get_hash(startkey) == hash && unicode_eq(startkey, key); + } + else { + if (!_Py_TryIncrefCompare(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + if (unicode_get_hash(startkey) == hash && unicode_eq(startkey, key)) { + Py_DECREF(startkey); + return 1; + } + Py_DECREF(startkey); + } + } + return 0; } -static int -insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) +static Py_ssize_t _Py_HOT_FUNCTION +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) { - return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); + return do_lookup(NULL, dk, key, hash, compare_unicode_unicode_threadsafe); } -static Py_ssize_t -insert_into_dictkeys(PyDictKeysObject *keys, PyObject *name) +static inline Py_ALWAYS_INLINE int +compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { - assert(PyUnicode_CheckExact(name)); - Py_hash_t hash = unicode_get_hash(name); - if (hash == -1) { - hash = PyUnicode_Type.tp_hash(name); - if (hash == -1) { - PyErr_Clear(); - return DKIX_EMPTY; - } + PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + if (startkey == key) { + return 1; } - Py_ssize_t ix = unicodekeys_lookup_unicode(keys, name, hash); - if (ix == DKIX_EMPTY) { - if (keys->dk_usable <= 0) { - return DKIX_EMPTY; + Py_ssize_t ep_hash = _Py_atomic_load_ssize_relaxed(&ep->me_hash); + if (ep_hash == hash) { + if (startkey == NULL || !_Py_TryIncrefCompare(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; } - /* Insert into new slot. */ - keys->dk_version = 0; - Py_ssize_t hashpos = find_empty_slot(keys, hash); - ix = keys->dk_nentries; - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(keys)[ix]; - dictkeys_set_index(keys, hashpos, ix); - assert(ep->me_key == NULL); - ep->me_key = Py_NewRef(name); - keys->dk_usable--; - keys->dk_nentries++; - } - assert (ix < SHARED_KEYS_MAX_SIZE); - return ix; + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + return 0; +} + +static Py_ssize_t +dictkeys_generic_lookup_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_generic_threadsafe); +} + +Py_ssize_t +_Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) +{ + PyDictKeysObject *dk; + DictKeysKind kind; + Py_ssize_t ix; + PyObject *value; + + ensure_shared_on_read(mp); + + dk = _Py_atomic_load_ptr(&mp->ma_keys); + kind = dk->dk_kind; + + if (kind != DICT_KEYS_GENERAL) { + if (PyUnicode_CheckExact(key)) { + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + } + else { + ix = unicodekeys_lookup_generic_threadsafe(mp, dk, key, hash); + } + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + + if (ix >= 0) { + if (kind == DICT_KEYS_SPLIT) { + PyDictValues *values = _Py_atomic_load_ptr(&mp->ma_values); + if (values == NULL) + goto read_failed; + + uint8_t capacity = _Py_atomic_load_uint8_relaxed(&values->capacity); + if (ix >= (Py_ssize_t)capacity) + goto read_failed; + + value = _Py_TryXGetRef(&values->values[ix]); + if (value == NULL) + goto read_failed; + + if (values != _Py_atomic_load_ptr(&mp->ma_values)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = _Py_TryXGetRef(&DK_UNICODE_ENTRIES(dk)[ix].me_value); + if (value == NULL) { + goto read_failed; + } + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + } + else { + value = NULL; + } + } + else { + ix = dictkeys_generic_lookup_threadsafe(mp, dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + if (ix >= 0) { + value = _Py_TryXGetRef(&DK_ENTRIES(dk)[ix].me_value); + if (value == NULL) + goto read_failed; + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = NULL; + } + } + + *value_addr = value; + return ix; + +read_failed: + // In addition to the normal races of the dict being modified the _Py_TryXGetRef + // can all fail if they don't yet have a shared ref count. That can happen here + // or in the *_lookup_* helper. In that case we need to take the lock to avoid + // mutation and do a normal incref which will make them shared. + Py_BEGIN_CRITICAL_SECTION(mp); + ix = _Py_dict_lookup(mp, key, hash, &value); + *value_addr = value; + if (value != NULL) { + assert(ix >= 0); + _Py_NewRefWithLock(value); + } + Py_END_CRITICAL_SECTION(); + return ix; +} + +Py_ssize_t +_Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr) +{ + PyDictKeysObject *dk = _Py_atomic_load_ptr(&mp->ma_keys); + if (dk->dk_kind == DICT_KEYS_UNICODE && PyUnicode_CheckExact(key)) { + Py_ssize_t ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + if (ix == DKIX_EMPTY) { + *value_addr = PyStackRef_NULL; + return ix; + } + else if (ix >= 0) { + PyObject **addr_of_value = &DK_UNICODE_ENTRIES(dk)[ix].me_value; + PyObject *value = _Py_atomic_load_ptr(addr_of_value); + if (value == NULL) { + *value_addr = PyStackRef_NULL; + return DKIX_EMPTY; + } + if (_Py_IsImmortal(value) || _PyObject_HasDeferredRefcount(value)) { + *value_addr = (_PyStackRef){ .bits = (uintptr_t)value | Py_TAG_DEFERRED }; + return ix; + } + if (_Py_TryIncrefCompare(addr_of_value, value)) { + *value_addr = PyStackRef_FromPyObjectSteal(value); + return ix; + } + } + } + + PyObject *obj; + Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &obj); + if (ix >= 0 && obj != NULL) { + *value_addr = PyStackRef_FromPyObjectSteal(obj); + } + else { + *value_addr = PyStackRef_NULL; + } + return ix; +} + +#else // Py_GIL_DISABLED + +Py_ssize_t +_Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) +{ + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, value_addr); + Py_XNewRef(*value_addr); + return ix; +} + +Py_ssize_t +_Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr) +{ + PyObject *val; + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &val); + if (val == NULL) { + *value_addr = PyStackRef_NULL; + } + else { + *value_addr = PyStackRef_FromPyObjectNew(val); + } + return ix; +} + +#endif + +int +_PyDict_HasOnlyStringKeys(PyObject *dict) +{ + Py_ssize_t pos = 0; + PyObject *key, *value; + assert(PyDict_Check(dict)); + /* Shortcut */ + if (((PyDictObject *)dict)->ma_keys->dk_kind != DICT_KEYS_GENERAL) + return 1; + while (PyDict_Next(dict, &pos, &key, &value)) + if (!PyUnicode_Check(key)) + return 0; + return 1; +} + +#define MAINTAIN_TRACKING(mp, key, value) \ + do { \ + if (!_PyObject_GC_IS_TRACKED(mp)) { \ + if (_PyObject_GC_MAY_BE_TRACKED(key) || \ + _PyObject_GC_MAY_BE_TRACKED(value)) { \ + _PyObject_GC_TRACK(mp); \ + } \ + } \ + } while(0) + +void +_PyDict_MaybeUntrack(PyObject *op) +{ + PyDictObject *mp; + PyObject *value; + Py_ssize_t i, numentries; + + ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); + + if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) + return; + + mp = (PyDictObject *) op; + ASSERT_CONSISTENT(mp); + numentries = mp->ma_keys->dk_nentries; + if (_PyDict_HasSplitTable(mp)) { + for (i = 0; i < numentries; i++) { + if ((value = mp->ma_values->values[i]) == NULL) + continue; + if (_PyObject_GC_MAY_BE_TRACKED(value)) { + return; + } + } + } + else { + if (DK_IS_UNICODE(mp->ma_keys)) { + PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(mp->ma_keys); + for (i = 0; i < numentries; i++) { + if ((value = ep0[i].me_value) == NULL) + continue; + if (_PyObject_GC_MAY_BE_TRACKED(value)) + return; + } + } + else { + PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); + for (i = 0; i < numentries; i++) { + if ((value = ep0[i].me_value) == NULL) + continue; + if (_PyObject_GC_MAY_BE_TRACKED(value) || + _PyObject_GC_MAY_BE_TRACKED(ep0[i].me_key)) + return; + } + } + } + _PyObject_GC_UNTRACK(op); +} + +void +_PyDict_EnablePerThreadRefcounting(PyObject *op) +{ + assert(PyDict_Check(op)); +#ifdef Py_GIL_DISABLED + Py_ssize_t id = _PyObject_AssignUniqueId(op); + if ((uint64_t)id >= (uint64_t)DICT_UNIQUE_ID_MAX) { + _PyObject_ReleaseUniqueId(id); + return; + } + + PyDictObject *mp = (PyDictObject *)op; + assert((mp->_ma_watcher_tag >> DICT_UNIQUE_ID_SHIFT) == 0); + // Plus 1 so that _ma_watcher_tag=0 represents an unassigned id + mp->_ma_watcher_tag += ((uint64_t)id + 1) << DICT_UNIQUE_ID_SHIFT; +#endif +} + +static inline int +is_unusable_slot(Py_ssize_t ix) +{ +#ifdef Py_GIL_DISABLED + return ix >= 0 || ix == DKIX_DUMMY; +#else + return ix >= 0; +#endif +} + +/* Internal function to find slot for an item from its hash + when it is known that the key is not present in the dict. + */ +static Py_ssize_t +find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) +{ + assert(keys != NULL); + + const size_t mask = DK_MASK(keys); + size_t i = hash & mask; + Py_ssize_t ix = dictkeys_get_index(keys, i); + for (size_t perturb = hash; is_unusable_slot(ix);) { + perturb >>= PERTURB_SHIFT; + i = (i*5 + perturb + 1) & mask; + ix = dictkeys_get_index(keys, i); + } + return i; +} + +static int +insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) +{ + return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); +} + +static inline int +insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, + Py_hash_t hash, PyObject *key, PyObject *value) +{ + if (mp->ma_keys->dk_usable <= 0) { + /* Need to resize. */ + if (insertion_resize(interp, mp, 1) < 0) { + return -1; + } + } + + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); + mp->ma_keys->dk_version = 0; + + Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); + dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); + + if (DK_IS_UNICODE(mp->ma_keys)) { + PyDictUnicodeEntry *ep; + ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); + } + else { + PyDictKeyEntry *ep; + ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); + STORE_HASH(ep, hash); + } + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1); + STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1); + assert(mp->ma_keys->dk_usable >= 0); + return 0; +} + +static Py_ssize_t +insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash) +{ + assert(PyUnicode_CheckExact(key)); + Py_ssize_t ix; + + +#ifdef Py_GIL_DISABLED + ix = unicodekeys_lookup_unicode_threadsafe(keys, key, hash); + if (ix >= 0) { + return ix; + } +#endif + + LOCK_KEYS(keys); + ix = unicodekeys_lookup_unicode(keys, key, hash); + if (ix == DKIX_EMPTY && keys->dk_usable > 0) { + // Insert into new slot + keys->dk_version = 0; + Py_ssize_t hashpos = find_empty_slot(keys, hash); + ix = keys->dk_nentries; + dictkeys_set_index(keys, hashpos, ix); + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(keys)[ix]; + STORE_SHARED_KEY(ep->me_key, Py_NewRef(key)); + split_keys_entry_added(keys); + } + assert (ix < SHARED_KEYS_MAX_SIZE); + UNLOCK_KEYS(keys); + return ix; +} + +static void +insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix) +{ + assert(PyUnicode_CheckExact(key)); + ASSERT_DICT_LOCKED(mp); + MAINTAIN_TRACKING(mp, key, value); + PyObject *old_value = mp->ma_values->values[ix]; + if (old_value == NULL) { + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); + STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); + _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); + STORE_USED(mp, mp->ma_used + 1); + } + else { + _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); + STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + Py_DECREF(old_value); + } + ASSERT_CONSISTENT(mp); } /* @@ -1266,6 +1799,21 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); } + if (_PyDict_HasSplitTable(mp)) { + Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (ix != DKIX_EMPTY) { + insert_split_value(interp, mp, key, value, ix); + Py_DECREF(key); + Py_DECREF(value); + return 0; + } + + /* No space in shared keys. Resize and continue below. */ + if (insertion_resize(interp, mp, 1) < 0) { + goto Fail; + } + } + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); if (ix == DKIX_ERROR) goto Fail; @@ -1273,70 +1821,29 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, MAINTAIN_TRACKING(mp, key, value); if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); + assert(!_PyDict_HasSplitTable(mp)); /* Insert into new slot. */ - mp->ma_keys->dk_version = 0; assert(old_value == NULL); - if (mp->ma_keys->dk_usable <= 0) { - /* Need to resize. */ - if (insertion_resize(interp, mp, 1) < 0) - goto Fail; - } - - Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); - dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - - if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep; - ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - if (mp->ma_values) { - Py_ssize_t index = mp->ma_keys->dk_nentries; - _PyDictValues_AddToInsertionOrder(mp->ma_values, index); - assert (mp->ma_values->values[index] == NULL); - mp->ma_values->values[index] = value; - } - else { - ep->me_value = value; - } + if (insert_combined_dict(interp, mp, hash, key, value) < 0) { + goto Fail; } - else { - PyDictKeyEntry *ep; - ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - ep->me_hash = hash; - ep->me_value = value; - } - mp->ma_used++; - mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; - assert(mp->ma_keys->dk_usable >= 0); + STORE_USED(mp, mp->ma_used + 1); ASSERT_CONSISTENT(mp); return 0; } if (old_value != value) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_MODIFIED, mp, key, value); - if (_PyDict_HasSplitTable(mp)) { - mp->ma_values->values[ix] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); - mp->ma_used++; - } + _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); + assert(old_value != NULL); + assert(!_PyDict_HasSplitTable(mp)); + if (DK_IS_UNICODE(mp->ma_keys)) { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); } else { - assert(old_value != NULL); - if (DK_IS_UNICODE(mp->ma_keys)) { - DK_UNICODE_ENTRIES(mp->ma_keys)[ix].me_value = value; - } - else { - DK_ENTRIES(mp->ma_keys)[ix].me_value = value; - } + PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); } - mp->ma_version_tag = new_version; } Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ ASSERT_CONSISTENT(mp); @@ -1349,7 +1856,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, return -1; } -// Same to insertdict but specialized for ma_keys = Py_EMPTY_KEYS. +// Same as insertdict but specialized for ma_keys == Py_EMPTY_KEYS. // Consumes key and value references. static int insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, @@ -1358,9 +1865,6 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, assert(mp->ma_keys == Py_EMPTY_KEYS); ASSERT_DICT_LOCKED(mp); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); - int unicode = PyUnicode_CheckExact(key); PyDictKeysObject *newkeys = new_keys_object( interp, PyDict_LOG_MINSIZE, unicode); @@ -1369,29 +1873,35 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, Py_DECREF(value); return -1; } + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); + /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ - mp->ma_keys = newkeys; - mp->ma_values = NULL; + assert(mp->ma_values == NULL); MAINTAIN_TRACKING(mp, key, value); size_t hashpos = (size_t)hash & (PyDict_MINSIZE-1); - dictkeys_set_index(mp->ma_keys, hashpos, 0); + dictkeys_set_index(newkeys, hashpos, 0); if (unicode) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(newkeys); ep->me_key = key; - ep->me_value = value; + STORE_VALUE(ep, value); } else { - PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *ep = DK_ENTRIES(newkeys); ep->me_key = key; ep->me_hash = hash; - ep->me_value = value; - } - mp->ma_used++; - mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; + STORE_VALUE(ep, value); + } + STORE_USED(mp, mp->ma_used + 1); + newkeys->dk_usable--; + newkeys->dk_nentries++; + // We store the keys last so no one can see them in a partially inconsistent + // state so that we don't need to switch the keys to being shared yet for + // the case where we're inserting from the non-owner thread. We don't use + // set_keys here because the transition from empty to non-empty is safe + // as the empty keys will never be freed. + FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_keys, newkeys); return 0; } @@ -1436,7 +1946,7 @@ actually be smaller than the old one. If a table is split (its keys and hashes are shared, its values are not), then the values are temporarily copied into the table, it is resized as a combined table, then the me_value slots in the old table are NULLed out. -After resizing a table is always combined. +After resizing, a table is always combined. This function supports: - Unicode split -> Unicode combined or Generic @@ -1447,7 +1957,7 @@ static int dictresize(PyInterpreterState *interp, PyDictObject *mp, uint8_t log2_newsize, int unicode) { - PyDictKeysObject *oldkeys; + PyDictKeysObject *oldkeys, *newkeys; PyDictValues *oldvalues; ASSERT_DICT_LOCKED(mp); @@ -1465,30 +1975,31 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, unicode = 0; } + ensure_shared_on_resize(mp); /* NOTE: Current odict checks mp->ma_keys to detect resize happen. * So we can't reuse oldkeys even if oldkeys->dk_size == newsize. * TODO: Try reusing oldkeys when reimplement odict. */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(interp, log2_newsize, unicode); - if (mp->ma_keys == NULL) { - mp->ma_keys = oldkeys; + newkeys = new_keys_object(interp, log2_newsize, unicode); + if (newkeys == NULL) { return -1; } // New table must be large enough. - assert(mp->ma_keys->dk_usable >= mp->ma_used); + assert(newkeys->dk_usable >= mp->ma_used); Py_ssize_t numentries = mp->ma_used; if (oldvalues != NULL) { - PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); + LOCK_KEYS(oldkeys); + PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); /* Convert split table into new combined table. * We must incref keys; we can transfer values. */ - if (mp->ma_keys->dk_kind == DICT_KEYS_GENERAL) { + if (newkeys->dk_kind == DICT_KEYS_GENERAL) { // split -> generic - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); @@ -1498,10 +2009,10 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_hash = unicode_get_hash(ep->me_key); newentries[i].me_value = oldvalues->values[index]; } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } else { // split -> combined unicode - PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys); for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); @@ -1510,18 +2021,27 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_key = Py_NewRef(ep->me_key); newentries[i].me_value = oldvalues->values[index]; } - build_indices_unicode(mp->ma_keys, newentries, numentries); + build_indices_unicode(newkeys, newentries, numentries); + } + UNLOCK_KEYS(oldkeys); + set_keys(mp, newkeys); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + set_values(mp, NULL); + if (oldvalues->embedded) { + assert(oldvalues->embedded == 1); + assert(oldvalues->valid == 1); + FT_ATOMIC_STORE_UINT8(oldvalues->valid, 0); + } + else { + free_values(oldvalues, IS_DICT_SHARED(mp)); } - dictkeys_decref(interp, oldkeys); - mp->ma_values = NULL; - free_values(oldvalues); } else { // oldkeys is combined. if (oldkeys->dk_kind == DICT_KEYS_GENERAL) { // generic -> generic - assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); + assert(newkeys->dk_kind == DICT_KEYS_GENERAL); PyDictKeyEntry *oldentries = DK_ENTRIES(oldkeys); - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); if (oldkeys->dk_nentries == numentries) { memcpy(newentries, oldentries, numentries * sizeof(PyDictKeyEntry)); } @@ -1533,12 +2053,12 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i] = *ep++; } } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } else { // oldkeys is combined unicode PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); if (unicode) { // combined unicode -> combined unicode - PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys); if (oldkeys->dk_nentries == numentries && mp->ma_keys->dk_kind == DICT_KEYS_UNICODE) { memcpy(newentries, oldentries, numentries * sizeof(PyDictUnicodeEntry)); } @@ -1550,10 +2070,10 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i] = *ep++; } } - build_indices_unicode(mp->ma_keys, newentries, numentries); + build_indices_unicode(newkeys, newentries, numentries); } else { // combined unicode -> generic - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); PyDictUnicodeEntry *ep = oldentries; for (Py_ssize_t i = 0; i < numentries; i++) { while (ep->me_value == NULL) @@ -1563,22 +2083,24 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_value = ep->me_value; ep++; } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } } + set_keys(mp, newkeys); + if (oldkeys != Py_EMPTY_KEYS) { #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); - free_keys_object(oldkeys); + free_keys_object(oldkeys, IS_DICT_SHARED(mp)); } } - mp->ma_keys->dk_usable -= numentries; - mp->ma_keys->dk_nentries = numentries; + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - numentries); + STORE_KEYS_NENTRIES(mp->ma_keys, numentries); ASSERT_CONSISTENT(mp); return 0; } @@ -1675,13 +2197,10 @@ dict_getitem(PyObject *op, PyObject *key, const char *warnmsg) } PyDictObject *mp = (PyDictObject *)op; - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - PyErr_FormatUnraisable(warnmsg); - return NULL; - } + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + PyErr_FormatUnraisable(warnmsg); + return NULL; } PyThreadState *tstate = _PyThreadState_GET(); @@ -1695,9 +2214,13 @@ dict_getitem(PyObject *op, PyObject *key, const char *warnmsg) PyObject *value; Py_ssize_t ix; (void)ix; - PyObject *exc = _PyErr_GetRaisedException(tstate); +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif /* Ignore any exception raised by the lookup */ PyObject *exc2 = _PyErr_Occurred(tstate); @@ -1726,12 +2249,9 @@ _PyDict_LookupIndex(PyDictObject *mp, PyObject *key) assert(PyDict_CheckExact((PyObject*)mp)); assert(PyUnicode_CheckExact(key)); - Py_hash_t hash = unicode_get_hash(key); + Py_hash_t hash = _PyObject_HashFast(key); if (hash == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - return -1; - } + return -1; } return _Py_dict_lookup(mp, key, hash, &value); @@ -1753,11 +2273,68 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } +/* Gets an item and provides a new reference if the value is present. + * Returns 1 if the key is present, 0 if the key is missing, and -1 if an + * exception occurred. +*/ +int +_PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, + Py_hash_t hash, PyObject **result) +{ + PyObject *value; + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } + *result = Py_NewRef(value); + return 1; // key is present +} + +/* Gets an item and provides a new reference if the value is present. + * Returns 1 if the key is present, 0 if the key is missing, and -1 if an + * exception occurred. +*/ +int +_PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result) +{ + PyObject *value; +#ifdef Py_GIL_DISABLED + Py_ssize_t ix = _Py_dict_lookup_threadsafe(op, key, hash, &value); +#else + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); +#endif + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } +#ifdef Py_GIL_DISABLED + *result = value; +#else + *result = Py_NewRef(value); +#endif + return 1; // key is present +} int PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) @@ -1767,20 +2344,30 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) *result = NULL; return -1; } - PyDictObject*mp = (PyDictObject *)op; - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) - { - hash = PyObject_Hash(key); - if (hash == -1) { - *result = NULL; - return -1; - } + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + *result = NULL; + return -1; + } + + return _PyDict_GetItemRef_KnownHash((PyDictObject *)op, key, hash, result); +} + +int +_PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result) +{ + ASSERT_DICT_LOCKED(op); + assert(PyUnicode_CheckExact(key)); + + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + *result = NULL; + return -1; } PyObject *value; - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); assert(ix >= 0 || value == NULL); if (ix == DKIX_ERROR) { *result = NULL; @@ -1794,7 +2381,6 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) return 1; // key is present } - /* Variant of PyDict_GetItem() that doesn't suppress exceptions. This returns NULL *with* an exception set if an exception occurred. It returns NULL *without* an exception set if the key wasn't present. @@ -1811,15 +2397,17 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) PyErr_BadInternalCall(); return NULL; } - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) - { - hash = PyObject_Hash(key); - if (hash == -1) { - return NULL; - } + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } @@ -1828,7 +2416,7 @@ PyObject * _PyDict_GetItemWithError(PyObject *dp, PyObject *kv) { assert(PyUnicode_CheckExact(kv)); - Py_hash_t hash = kv->ob_type->tp_hash(kv); + Py_hash_t hash = Py_TYPE(kv)->tp_hash(kv); if (hash == -1) { return NULL; } @@ -1860,7 +2448,6 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) return rv; } - /* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. * @@ -1870,6 +2457,8 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) * Raise an exception and return NULL if an error occurred (ex: computing the * key hash failed, key comparison failed, ...). Return NULL if the key doesn't * exist. Return the value if the key exists. + * + * Returns a new reference. */ PyObject * _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) @@ -1878,25 +2467,84 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) Py_hash_t hash; PyObject *value; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } /* namespace 1: globals */ - ix = _Py_dict_lookup(globals, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(globals, key, hash, &value); if (ix == DKIX_ERROR) return NULL; if (ix != DKIX_EMPTY && value != NULL) return value; /* namespace 2: builtins */ - ix = _Py_dict_lookup(builtins, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(builtins, key, hash, &value); assert(ix >= 0 || value == NULL); return value; } +void +_PyDict_LoadGlobalStackRef(PyDictObject *globals, PyDictObject *builtins, PyObject *key, _PyStackRef *res) +{ + Py_ssize_t ix; + Py_hash_t hash; + + hash = _PyObject_HashFast(key); + if (hash == -1) { + *res = PyStackRef_NULL; + return; + } + + /* namespace 1: globals */ + ix = _Py_dict_lookup_threadsafe_stackref(globals, key, hash, res); + if (ix == DKIX_ERROR) { + return; + } + if (ix != DKIX_EMPTY && !PyStackRef_IsNull(*res)) { + return; + } + + /* namespace 2: builtins */ + ix = _Py_dict_lookup_threadsafe_stackref(builtins, key, hash, res); + assert(ix >= 0 || PyStackRef_IsNull(*res)); +} + +PyObject * +_PyDict_LoadBuiltinsFromGlobals(PyObject *globals) +{ + if (!PyDict_Check(globals)) { + PyErr_BadInternalCall(); + return NULL; + } + + PyDictObject *mp = (PyDictObject *)globals; + PyObject *key = &_Py_ID(__builtins__); + Py_hash_t hash = unicode_get_hash(key); + + // Use the stackref variant to avoid reference count contention on the + // builtins module in the free threading build. It's important not to + // make any escaping calls between the lookup and the `PyStackRef_CLOSE()` + // because the `ref` is not visible to the GC. + _PyStackRef ref; + Py_ssize_t ix = _Py_dict_lookup_threadsafe_stackref(mp, key, hash, &ref); + if (ix == DKIX_ERROR) { + return NULL; + } + if (PyStackRef_IsNull(ref)) { + return Py_NewRef(PyEval_GetBuiltins()); + } + PyObject *builtins = PyStackRef_AsPyObjectBorrow(ref); + if (PyModule_Check(builtins)) { + builtins = _PyModule_GetDict(builtins); + assert(builtins != NULL); + } + _Py_INCREF_BUILTINS(builtins); + PyStackRef_CLOSE(ref); + return builtins; +} + /* Consumes references to key and value */ static int setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) @@ -1906,14 +2554,11 @@ setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) assert(key); assert(value); assert(PyDict_Check(mp)); - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - Py_DECREF(key); - Py_DECREF(value); - return -1; - } + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + Py_DECREF(key); + Py_DECREF(value); + return -1; } PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -1963,13 +2608,23 @@ setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) Py_NewRef(key), Py_NewRef(value)); } - + +int +_PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, PyObject *value, + Py_hash_t hash) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (mp->ma_keys == Py_EMPTY_KEYS) { + return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + } + /* insertdict() handles any resizing that might be necessary */ + return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); +} + int _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, - Py_hash_t hash) + Py_hash_t hash) { - PyDictObject *mp; - if (!PyDict_Check(op)) { PyErr_BadInternalCall(); return -1; @@ -1977,21 +2632,10 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(key); assert(value); assert(hash != -1); - mp = (PyDictObject *)op; int res; - PyInterpreterState *interp = _PyInterpreterState_GET(); - - Py_BEGIN_CRITICAL_SECTION(mp); - - if (mp->ma_keys == Py_EMPTY_KEYS) { - res = insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); - } - else { - /* insertdict() handles any resizing that might be necessary */ - res = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); - } - + Py_BEGIN_CRITICAL_SECTION(op); + res = _PyDict_SetItem_KnownHash_LockHeld((PyDictObject *)op, key, value, hash); Py_END_CRITICAL_SECTION(); return res; } @@ -1999,22 +2643,24 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, static void delete_index_from_values(PyDictValues *values, Py_ssize_t ix) { - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; + uint8_t *array = get_insertion_order_array(values); + int size = values->size; + assert(size <= values->capacity); int i; - for (i = 1; size_ptr[-i] != ix; i++) { - assert(i <= size); + for (i = 0; array[i] != ix; i++) { + assert(i < size); } - assert(i <= size); + assert(i < size); + size--; for (; i < size; i++) { - size_ptr[-i] = size_ptr[-i-1]; + array[i] = array[i+1]; } - *size_ptr = size -1; + values->size = size; } -static int +static void delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, - PyObject *old_value, uint64_t new_version) + PyObject *old_value) { PyObject *old_key; @@ -2023,11 +2669,10 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); - mp->ma_used--; - mp->ma_version_tag = new_version; - if (mp->ma_values) { + STORE_USED(mp, mp->ma_used - 1); + if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); - mp->ma_values->values[ix] = NULL; + STORE_SPLIT_VALUE(mp, ix, NULL); assert(ix < SHARED_KEYS_MAX_SIZE); /* Update order */ delete_index_from_values(mp->ma_values, ix); @@ -2039,33 +2684,30 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; - ep->me_hash = 0; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); + STORE_HASH(ep, 0); } Py_DECREF(old_key); } Py_DECREF(old_value); ASSERT_CONSISTENT(mp); - return 0; } int PyDict_DelItem(PyObject *op, PyObject *key) { - Py_hash_t hash; assert(key); - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return _PyDict_DelItem_KnownHash(op, key, hash); @@ -2097,9 +2739,9 @@ delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) } PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - return delitem_common(mp, hash, ix, old_value, new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, old_value); + return 0; } int @@ -2114,9 +2756,10 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) static int delitemif_lock_held(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) + int (*predicate)(PyObject *value, void *arg), + void *arg) { - Py_ssize_t hashpos, ix; + Py_ssize_t ix; PyDictObject *mp; Py_hash_t hash; PyObject *old_value; @@ -2124,35 +2767,28 @@ delitemif_lock_held(PyObject *op, PyObject *key, ASSERT_DICT_LOCKED(op); - if (!PyDict_Check(op)) { - PyErr_BadInternalCall(); - return -1; - } assert(key); hash = PyObject_Hash(key); if (hash == -1) return -1; mp = (PyDictObject *)op; ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) + if (ix == DKIX_ERROR) { return -1; + } if (ix == DKIX_EMPTY || old_value == NULL) { - _PyErr_SetKeyError(key); - return -1; + return 0; } - res = predicate(old_value); + res = predicate(old_value, arg); if (res == -1) return -1; - hashpos = lookdict_index(mp->ma_keys, hash, ix); - assert(hashpos >= 0); - if (res > 0) { PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - return delitem_common(mp, hashpos, ix, old_value, new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, old_value); + return 1; } else { return 0; } @@ -2164,11 +2800,13 @@ delitemif_lock_held(PyObject *op, PyObject *key, */ int _PyDict_DelItemIf(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) + int (*predicate)(PyObject *value, void *arg), + void *arg) { + assert(PyDict_Check(op)); int res; Py_BEGIN_CRITICAL_SECTION(op); - res = delitemif_lock_held(op, key, predicate); + res = delitemif_lock_held(op, key, predicate, arg); Py_END_CRITICAL_SECTION(); return res; } @@ -2193,24 +2831,29 @@ clear_lock_held(PyObject *op) } /* Empty the dict... */ PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); - dictkeys_incref(Py_EMPTY_KEYS); - mp->ma_keys = Py_EMPTY_KEYS; - mp->ma_values = NULL; - mp->ma_used = 0; - mp->ma_version_tag = new_version; - /* ...then clear the keys and values */ - if (oldvalues != NULL) { - n = oldkeys->dk_nentries; - for (i = 0; i < n; i++) - Py_CLEAR(oldvalues->values[i]); - free_values(oldvalues); - dictkeys_decref(interp, oldkeys); + _PyDict_NotifyEvent(interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); + // We don't inc ref empty keys because they're immortal + ensure_shared_on_resize(mp); + STORE_USED(mp, 0); + if (oldvalues == NULL) { + set_keys(mp, Py_EMPTY_KEYS); + assert(oldkeys->dk_refcnt == 1); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); } else { - assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys); + n = oldkeys->dk_nentries; + for (i = 0; i < n; i++) { + Py_CLEAR(oldvalues->values[i]); + } + if (oldvalues->embedded) { + oldvalues->size = 0; + } + else { + set_values(mp, NULL); + set_keys(mp, Py_EMPTY_KEYS); + free_values(oldvalues, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, oldkeys, false); + } } ASSERT_CONSISTENT(mp); } @@ -2240,18 +2883,15 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, if (!PyDict_Check(op)) return 0; - ASSERT_DICT_LOCKED(op); - mp = (PyDictObject *)op; i = *ppos; - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); if (i < 0 || i >= mp->ma_used) return 0; int index = get_index_from_order(mp, i); value = mp->ma_values->values[index]; - - key = DK_UNICODE_ENTRIES(mp->ma_keys)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(mp->ma_keys)[index].me_key); hash = unicode_get_hash(key); assert(value != NULL); } @@ -2315,11 +2955,7 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, int PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { - int res; - Py_BEGIN_CRITICAL_SECTION(op); - res = _PyDict_Next(op, ppos, pkey, pvalue, NULL); - Py_END_CRITICAL_SECTION(); - return res; + return _PyDict_Next(op, ppos, pkey, pvalue, NULL); } @@ -2357,9 +2993,8 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, assert(old_value != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, Py_NewRef(old_value)); ASSERT_CONSISTENT(mp); if (result) { @@ -2392,15 +3027,12 @@ pop_lock_held(PyObject *op, PyObject *key, PyObject **result) return 0; } - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - if (result) { - *result = NULL; - } - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + if (result) { + *result = NULL; } + return -1; } return _PyDict_Pop_KnownHash(dict, key, hash, result); } @@ -2489,8 +3121,9 @@ dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, return NULL; } - while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(iterable); + while (_PySet_NextEntryRef(iterable, &pos, &key, &hash)) { + if (insertdict(interp, mp, key, hash, Py_NewRef(value))) { Py_DECREF(mp); return NULL; } @@ -2548,7 +3181,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) goto dict_iter_exit; } } -dict_iter_exit: +dict_iter_exit:; Py_END_CRITICAL_SECTION(); } else { while ((key = PyIter_Next(it)) != NULL) { @@ -2593,26 +3226,22 @@ dict_dealloc(PyObject *self) PyObject_GC_UnTrack(mp); Py_TRASHCAN_BEGIN(mp, dict_dealloc) if (values != NULL) { - for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { - Py_XDECREF(values->values[i]); + if (values->embedded == 0) { + for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { + Py_XDECREF(values->values[i]); + } + free_values(values, false); } - free_values(values); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *state = get_dict_state(); - if (state->numfree < PyDict_MAXFREELIST && state->numfree >=0 && - Py_IS_TYPE(mp, &PyDict_Type)) { - state->free_list[state->numfree++] = mp; - OBJECT_STAT_INC(to_freelist); + if (Py_IS_TYPE(mp, &PyDict_Type)) { + _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free); } - else -#endif - { + else { Py_TYPE(mp)->tp_free((PyObject *)mp); } Py_TRASHCAN_END @@ -2623,14 +3252,12 @@ static PyObject * dict_repr_lock_held(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; - Py_ssize_t i; PyObject *key = NULL, *value = NULL; - _PyUnicodeWriter writer; - int first; + ASSERT_DICT_LOCKED(mp); - i = Py_ReprEnter((PyObject *)mp); - if (i != 0) { - return i > 0 ? PyUnicode_FromString("{...}") : NULL; + int res = Py_ReprEnter((PyObject *)mp); + if (res != 0) { + return (res > 0 ? PyUnicode_FromString("{...}") : NULL); } if (mp->ma_used == 0) { @@ -2638,66 +3265,70 @@ dict_repr_lock_held(PyObject *self) return PyUnicode_FromString("{}"); } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; - /* "{" + "1: 2" + ", 3: 4" * (len - 1) + "}" */ - writer.min_length = 1 + 4 + (2 + 4) * (mp->ma_used - 1) + 1; + // "{" + "1: 2" + ", 3: 4" * (len - 1) + "}" + Py_ssize_t prealloc = 1 + 4 + 6 * (mp->ma_used - 1) + 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { + goto error; + } - if (_PyUnicodeWriter_WriteChar(&writer, '{') < 0) + if (PyUnicodeWriter_WriteChar(writer, '{') < 0) { goto error; + } /* Do repr() on each key+value pair, and insert ": " between them. Note that repr may mutate the dict. */ - i = 0; - first = 1; + Py_ssize_t i = 0; + int first = 1; while (_PyDict_Next((PyObject *)mp, &i, &key, &value, NULL)) { - PyObject *s; - int res; - - /* Prevent repr from deleting key or value during key format. */ + // Prevent repr from deleting key or value during key format. Py_INCREF(key); Py_INCREF(value); if (!first) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) + // Write ", " + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { + goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; + } } first = 0; - s = PyObject_Repr(key); - if (s == NULL) - goto error; - res = _PyUnicodeWriter_WriteStr(&writer, s); - Py_DECREF(s); - if (res < 0) + // Write repr(key) + if (PyUnicodeWriter_WriteRepr(writer, key) < 0) { goto error; + } - if (_PyUnicodeWriter_WriteASCIIString(&writer, ": ", 2) < 0) + // Write ": " + if (PyUnicodeWriter_WriteChar(writer, ':') < 0) { goto error; - - s = PyObject_Repr(value); - if (s == NULL) + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; - res = _PyUnicodeWriter_WriteStr(&writer, s); - Py_DECREF(s); - if (res < 0) + } + + // Write repr(value) + if (PyUnicodeWriter_WriteRepr(writer, value) < 0) { goto error; + } Py_CLEAR(key); Py_CLEAR(value); } - writer.overallocate = 0; - if (_PyUnicodeWriter_WriteChar(&writer, '}') < 0) + if (PyUnicodeWriter_WriteChar(writer, '}') < 0) { goto error; + } Py_ReprLeave((PyObject *)mp); - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: Py_ReprLeave((PyObject *)mp); - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); Py_XDECREF(key); Py_XDECREF(value); return NULL; @@ -2716,24 +3347,22 @@ dict_repr(PyObject *self) static Py_ssize_t dict_length(PyObject *self) { - PyDictObject *mp = (PyDictObject *)self; - return _Py_atomic_load_ssize_relaxed(&mp->ma_used); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)self)->ma_used); } static PyObject * -dict_subscript_lock_held(PyObject *self, PyObject *key) +dict_subscript(PyObject *self, PyObject *key) { PyDictObject *mp = (PyDictObject *)self; Py_ssize_t ix; Py_hash_t hash; PyObject *value; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } - ix = _Py_dict_lookup(mp, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) { @@ -2753,17 +3382,7 @@ dict_subscript_lock_held(PyObject *self, PyObject *key) _PyErr_SetKeyError(key); return NULL; } - return Py_NewRef(value); -} - -static PyObject * -dict_subscript(PyObject *self, PyObject *key) -{ - PyObject *res; - Py_BEGIN_CRITICAL_SECTION(self); - res = dict_subscript_lock_held(self, key); - Py_END_CRITICAL_SECTION(); - return res; + return value; } static int @@ -3119,6 +3738,9 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) static int dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *other, int override) { + ASSERT_DICT_LOCKED(mp); + ASSERT_DICT_LOCKED(other); + if (other == mp || other->ma_used == 0) /* a.update(a) or a.update({}); nothing to do */ return 0; @@ -3131,25 +3753,21 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe PyDictKeysObject *okeys = other->ma_keys; // If other is clean, combined, and just allocated, just clone it. - if (other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); + if (mp->ma_values == NULL && + other->ma_values == NULL && + other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used) + ) { + _PyDict_NotifyEvent(interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) return -1; - dictkeys_decref(interp, mp->ma_keys); + ensure_shared_on_resize(mp); + dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; - if (mp->ma_values != NULL) { - free_values(mp->ma_values); - mp->ma_values = NULL; - } - - mp->ma_used = other->ma_used; - mp->ma_version_tag = new_version; + STORE_USED(mp, other->ma_used); ASSERT_CONSISTENT(mp); if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { @@ -3187,7 +3805,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe Py_NewRef(key), hash, Py_NewRef(value)); } else { - err = contains_known_hash_lock_held(mp, key, hash); + err = _PyDict_Contains_KnownHash((PyObject *)mp, key, hash); if (err == 0) { err = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); @@ -3270,7 +3888,7 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) for (key = PyIter_Next(iter); key; key = PyIter_Next(iter)) { if (override != 1) { - status = contains_lock_held(mp, key); + status = PyDict_Contains(a, key); if (status != 0) { if (status > 0) { if (override == 0) { @@ -3351,6 +3969,27 @@ dict_copy_impl(PyDictObject *self) return PyDict_Copy((PyObject *)self); } +/* Copies the values, but does not change the reference + * counts of the objects in the array. + * Return NULL, but does *not* set an exception on failure */ +static PyDictValues * +copy_values(PyDictValues *values) +{ + PyDictValues *newvalues = new_values(values->capacity); + if (newvalues == NULL) { + return NULL; + } + newvalues->size = values->size; + uint8_t *values_order = get_insertion_order_array(values); + uint8_t *new_values_order = get_insertion_order_array(newvalues); + memcpy(new_values_order, values_order, values->capacity); + for (int i = 0; i < values->capacity; i++) { + newvalues->values[i] = values->values[i]; + } + assert(newvalues->embedded == 0); + return newvalues; +} + static PyObject * copy_lock_held(PyObject *o) { @@ -3368,26 +4007,23 @@ copy_lock_held(PyObject *o) if (_PyDict_HasSplitTable(mp)) { PyDictObject *split_copy; - size_t size = shared_keys_usable_size(mp->ma_keys); - PyDictValues *newvalues = new_values(size); - if (newvalues == NULL) + PyDictValues *newvalues = copy_values(mp->ma_values); + if (newvalues == NULL) { return PyErr_NoMemory(); + } split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); if (split_copy == NULL) { - free_values(newvalues); + free_values(newvalues, false); return NULL; } - size_t prefix_size = ((uint8_t *)newvalues)[-1]; - memcpy(((char *)newvalues)-prefix_size, ((char *)mp->ma_values)-prefix_size, prefix_size-1); + for (size_t i = 0; i < newvalues->capacity; i++) { + Py_XINCREF(newvalues->values[i]); + } split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; - split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); + split_copy->_ma_watcher_tag = 0; dictkeys_incref(mp->ma_keys); - for (size_t i = 0; i < size; i++) { - PyObject *value = mp->ma_values->values[i]; - split_copy->ma_values->values[i] = Py_XNewRef(value); - } if (_PyObject_GC_IS_TRACKED(mp)) _PyObject_GC_TRACK(split_copy); return (PyObject *)split_copy; @@ -3465,7 +4101,7 @@ PyDict_Size(PyObject *mp) PyErr_BadInternalCall(); return -1; } - return ((PyDictObject *)mp)->ma_used; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)mp)->ma_used); } /* Return 1 if dicts equal, 0 if not, -1 if error. @@ -3484,7 +4120,7 @@ dict_equal_lock_held(PyDictObject *a, PyDictObject *b) /* can't be equal if # of entries differ */ return 0; /* Same # of entries -- check all of 'em. Exit early on any diff. */ - for (i = 0; i < a->ma_keys->dk_nentries; i++) { + for (i = 0; i < LOAD_KEYS_NENTRIES(a->ma_keys); i++) { PyObject *key, *aval; Py_hash_t hash; if (DK_IS_UNICODE(a->ma_keys)) { @@ -3494,7 +4130,7 @@ dict_equal_lock_held(PyDictObject *a, PyDictObject *b) continue; } hash = unicode_get_hash(key); - if (a->ma_values) + if (_PyDict_HasSplitTable(a)) aval = a->ma_values->values[i]; else aval = ep->me_value; @@ -3568,7 +4204,6 @@ dict_richcompare(PyObject *v, PyObject *w, int op) /*[clinic input] @coexist -@critical_section dict.__contains__ key: object @@ -3578,25 +4213,17 @@ True if the dictionary has the specified key, else False. [clinic start generated code]*/ static PyObject * -dict___contains___impl(PyDictObject *self, PyObject *key) -/*[clinic end generated code: output=1b314e6da7687dae input=bc76ec9c157cb81b]*/ +dict___contains__(PyDictObject *self, PyObject *key) +/*[clinic end generated code: output=a3d03db709ed6e6b input=fe1cb42ad831e820]*/ { - register PyDictObject *mp = self; - Py_hash_t hash; - Py_ssize_t ix; - PyObject *value; - - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; - } - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) + int contains = PyDict_Contains((PyObject *)self, key); + if (contains < 0) { return NULL; - if (ix == DKIX_EMPTY || value == NULL) - Py_RETURN_FALSE; - Py_RETURN_TRUE; + } + if (contains) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; } /*[clinic input] @@ -3618,18 +4245,17 @@ dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) Py_hash_t hash; Py_ssize_t ix; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } - ix = _Py_dict_lookup(self, key, hash, &val); + ix = _Py_dict_lookup_threadsafe(self, key, hash, &val); if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || val == NULL) { - val = default_value; + val = Py_NewRef(default_value); } - return Py_NewRef(val); + return val; } static int @@ -3651,14 +4277,12 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu return -1; } - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - if (result) { - *result = NULL; - } - return -1; + hash = _PyObject_HashFast(key); + if (hash == -1) { + if (result) { + *result = NULL; } + return -1; } if (mp->ma_keys == Py_EMPTY_KEYS) { @@ -3684,6 +4308,29 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } } + if (_PyDict_HasSplitTable(mp)) { + Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (ix != DKIX_EMPTY) { + PyObject *value = mp->ma_values->values[ix]; + int already_present = value != NULL; + if (!already_present) { + insert_split_value(interp, mp, key, default_value, ix); + value = default_value; + } + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return already_present; + } + + /* No space in shared keys. Resize and continue below. */ + if (insertion_resize(interp, mp, 1) < 0) { + goto error; + } + } + + assert(!_PyDict_HasSplitTable(mp)); + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); if (ix == DKIX_ERROR) { if (result) { @@ -3693,46 +4340,19 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, default_value); - mp->ma_keys->dk_version = 0; + assert(!_PyDict_HasSplitTable(mp)); value = default_value; - if (mp->ma_keys->dk_usable <= 0) { - if (insertion_resize(interp, mp, 1) < 0) { - if (result) { - *result = NULL; - } - return -1; - } - } - Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); - dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - if (DK_IS_UNICODE(mp->ma_keys)) { - assert(PyUnicode_CheckExact(key)); - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = Py_NewRef(key); - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t index = (int)mp->ma_keys->dk_nentries; - assert(index < SHARED_KEYS_MAX_SIZE); - assert(mp->ma_values->values[index] == NULL); - mp->ma_values->values[index] = Py_NewRef(value); - _PyDictValues_AddToInsertionOrder(mp->ma_values, index); - } - else { - ep->me_value = Py_NewRef(value); + + if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { + Py_DECREF(key); + Py_DECREF(value); + if (result) { + *result = NULL; } } - else { - PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = Py_NewRef(key); - ep->me_hash = hash; - ep->me_value = Py_NewRef(value); - } + MAINTAIN_TRACKING(mp, key, value); - mp->ma_used++; - mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; + STORE_USED(mp, mp->ma_used + 1); assert(mp->ma_keys->dk_usable >= 0); ASSERT_CONSISTENT(mp); if (result) { @@ -3740,29 +4360,19 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } return 0; } - else if (value == NULL) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, default_value); - value = default_value; - assert(_PyDict_HasSplitTable(mp)); - assert(mp->ma_values->values[ix] == NULL); - MAINTAIN_TRACKING(mp, key, value); - mp->ma_values->values[ix] = Py_NewRef(value); - _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); - mp->ma_used++; - mp->ma_version_tag = new_version; - ASSERT_CONSISTENT(mp); - if (result) { - *result = incref_result ? Py_NewRef(value) : value; - } - return 0; - } + assert(value != NULL); ASSERT_CONSISTENT(mp); if (result) { *result = incref_result ? Py_NewRef(value) : value; } return 1; + +error: + if (result) { + *result = NULL; + } + return -1; } int @@ -3860,9 +4470,10 @@ dict_popitem_impl(PyDictObject *self) { Py_ssize_t i, j; PyObject *res; - uint64_t new_version; PyInterpreterState *interp = _PyInterpreterState_GET(); + ASSERT_DICT_LOCKED(self); + /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which * could empty the dict, so if we checked the size first and that @@ -3881,8 +4492,8 @@ dict_popitem_impl(PyDictObject *self) return NULL; } /* Convert split table to combined table */ - if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { - if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1)) { + if (_PyDict_HasSplitTable(self)) { + if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { Py_DECREF(res); return NULL; } @@ -3901,8 +4512,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, self, key, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3917,8 +4527,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, self, key, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3934,9 +4543,8 @@ dict_popitem_impl(PyDictObject *self) PyTuple_SET_ITEM(res, 0, key); PyTuple_SET_ITEM(res, 1, value); /* We can't dk_usable++ since there is DKIX_DUMMY in indices */ - self->ma_keys->dk_nentries = i; - self->ma_used--; - self->ma_version_tag = new_version; + STORE_KEYS_NENTRIES(self->ma_keys, i); + STORE_USED(self, self->ma_used - 1); ASSERT_CONSISTENT(self); return res; } @@ -3949,9 +4557,11 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) Py_ssize_t i, n = keys->dk_nentries; if (DK_IS_UNICODE(keys)) { - if (mp->ma_values != NULL) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); + if (_PyDict_HasSplitTable(mp)) { + if (!mp->ma_values->embedded) { + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); + } } } else { @@ -3986,7 +4596,7 @@ static Py_ssize_t sizeof_lock_held(PyDictObject *mp) { size_t res = _PyObject_SIZE(Py_TYPE(mp)); - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { res += shared_keys_usable_size(mp->ma_keys) * sizeof(PyObject*); } /* If the dictionary is split, the keys portion is accounted-for @@ -4063,8 +4673,8 @@ PyDoc_STRVAR(getitem__doc__, "__getitem__($self, key, /)\n--\n\nReturn self[key]."); PyDoc_STRVAR(update__doc__, -"D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n\ -If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n\ +"D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.\n\ +If E is present and has a .keys() method, then does: for k in E.keys(): D[k] = E[k]\n\ If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n\ In either case, this is followed by: for k in F: D[k] = F[k]"); @@ -4092,49 +4702,17 @@ static PyMethodDef mapp_methods[] = { {NULL, NULL} /* sentinel */ }; -static int -contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash) +/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ +int +PyDict_Contains(PyObject *op, PyObject *key) { - Py_ssize_t ix; - PyObject *value; + Py_hash_t hash = _PyObject_HashFast(key); - ASSERT_DICT_LOCKED(mp); - - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) + if (hash == -1) { return -1; - return (ix != DKIX_EMPTY && value != NULL); -} - -static int -contains_lock_held(PyDictObject *mp, PyObject *key) -{ - Py_hash_t hash; - Py_ssize_t ix; - PyObject *value; - - ASSERT_DICT_LOCKED(mp); - - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; } - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) - return -1; - return (ix != DKIX_EMPTY && value != NULL); -} -/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ -int -PyDict_Contains(PyObject *op, PyObject *key) -{ - int res; - Py_BEGIN_CRITICAL_SECTION(op); - res = contains_lock_held((PyDictObject *)op, key); - Py_END_CRITICAL_SECTION(); - return res; + return _PyDict_Contains_KnownHash(op, key, hash); } int @@ -4157,10 +4735,20 @@ _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyObject *value; Py_ssize_t ix; +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return -1; - return (ix != DKIX_EMPTY && value != NULL); + if (ix != DKIX_EMPTY && value != NULL) { +#ifdef Py_GIL_DISABLED + Py_DECREF(value); +#endif + return 1; + } + return 0; } int @@ -4207,8 +4795,7 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDictObject *d = (PyDictObject *)self; d->ma_used = 0; - d->ma_version_tag = DICT_NEXT_VERSION( - _PyInterpreterState_GET()); + d->_ma_watcher_tag = 0; dictkeys_incref(Py_EMPTY_KEYS); d->ma_keys = Py_EMPTY_KEYS; d->ma_values = NULL; @@ -4378,7 +4965,8 @@ PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) kv = PyUnicode_FromString(key); if (kv == NULL) return -1; - PyUnicode_InternInPlace(&kv); /* XXX Should we really? */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &kv); /* XXX Should we really? */ err = PyDict_SetItem(v, kv, item); Py_DECREF(kv); return err; @@ -4420,22 +5008,24 @@ typedef struct { static PyObject * dictiter_new(PyDictObject *dict, PyTypeObject *itertype) { + Py_ssize_t used; dictiterobject *di; di = PyObject_GC_New(dictiterobject, itertype); if (di == NULL) { return NULL; } di->di_dict = (PyDictObject*)Py_NewRef(dict); - di->di_used = dict->ma_used; - di->len = dict->ma_used; + used = FT_ATOMIC_LOAD_SSIZE_RELAXED(dict->ma_used); + di->di_used = used; + di->len = used; if (itertype == &PyDictRevIterKey_Type || itertype == &PyDictRevIterItem_Type || itertype == &PyDictRevIterValue_Type) { - if (dict->ma_values) { - di->di_pos = dict->ma_used - 1; + if (_PyDict_HasSplitTable(dict)) { + di->di_pos = used - 1; } else { - di->di_pos = dict->ma_keys->dk_nentries - 1; + di->di_pos = load_keys_nentries(dict) - 1; } } else { @@ -4481,8 +5071,8 @@ dictiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { dictiterobject *di = (dictiterobject *)self; Py_ssize_t len = 0; - if (di->di_dict != NULL && di->di_used == di->di_dict->ma_used) - len = di->len; + if (di->di_dict != NULL && di->di_used == FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_dict->ma_used)) + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->len); return PyLong_FromSize_t(len); } @@ -4502,6 +5092,14 @@ static PyMethodDef dictiter_methods[] = { {NULL, NULL} /* sentinel */ }; +#ifdef Py_GIL_DISABLED + +static int +dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value); + +#else /* Py_GIL_DISABLED */ + static PyObject* dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) { @@ -4523,11 +5121,11 @@ dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) i = di->di_pos; k = d->ma_keys; assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(k)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key); assert(d->ma_values->values[index] != NULL); } else { @@ -4569,6 +5167,8 @@ dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) return NULL; } +#endif /* Py_GIL_DISABLED */ + static PyObject* dictiter_iternextkey(PyObject *self) { @@ -4579,9 +5179,13 @@ dictiter_iternextkey(PyObject *self) return NULL; PyObject *value; - Py_BEGIN_CRITICAL_SECTION(d); +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, &value, NULL) < 0) { + value = NULL; + } +#else value = dictiter_iternextkey_lock_held(d, self); - Py_END_CRITICAL_SECTION(); +#endif return value; } @@ -4619,6 +5223,8 @@ PyTypeObject PyDictIterKey_Type = { 0, }; +#ifndef Py_GIL_DISABLED + static PyObject * dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) { @@ -4638,7 +5244,7 @@ dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) i = di->di_pos; assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); @@ -4684,6 +5290,8 @@ dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) return NULL; } +#endif /* Py_GIL_DISABLED */ + static PyObject * dictiter_iternextvalue(PyObject *self) { @@ -4694,9 +5302,13 @@ dictiter_iternextvalue(PyObject *self) return NULL; PyObject *value; - Py_BEGIN_CRITICAL_SECTION(d); +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, NULL, &value) < 0) { + value = NULL; + } +#else value = dictiter_iternextvalue_lock_held(d, self); - Py_END_CRITICAL_SECTION(); +#endif return value; } @@ -4734,29 +5346,32 @@ PyTypeObject PyDictIterValue_Type = { 0, }; -static PyObject * -dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self) +static int +dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value) { dictiterobject *di = (dictiterobject *)self; - PyObject *key, *value, *result; + PyObject *key, *value; Py_ssize_t i; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); di->di_used = -1; /* Make this state sticky */ - return NULL; + return -1; } - i = di->di_pos; + i = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_pos); + assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(d->ma_keys)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(d->ma_keys)[index].me_key); value = d->ma_values->values[index]; assert(value != NULL); } @@ -4793,34 +5408,178 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self) } di->di_pos = i+1; di->len--; - result = di->di_result; - if (Py_REFCNT(result) == 1) { - PyObject *oldkey = PyTuple_GET_ITEM(result, 0); - PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); - PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(value)); - Py_INCREF(result); - Py_DECREF(oldkey); - Py_DECREF(oldvalue); - // bpo-42536: The GC may have untracked this result tuple. Since we're - // recycling it, make sure it's tracked again: - if (!_PyObject_GC_IS_TRACKED(result)) { - _PyObject_GC_TRACK(result); + if (out_key != NULL) { + *out_key = Py_NewRef(key); + } + if (out_value != NULL) { + *out_value = Py_NewRef(value); + } + return 0; + +fail: + di->di_dict = NULL; + Py_DECREF(d); + return -1; +} + +#ifdef Py_GIL_DISABLED + +// Grabs the key and/or value from the provided locations and if successful +// returns them with an increased reference count. If either one is unsuccessful +// nothing is incref'd and returns -1. +static int +acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, + PyObject **out_key, PyObject **out_value) +{ + if (out_key) { + *out_key = _Py_TryXGetRef(key_loc); + if (*out_key == NULL) { + return -1; + } + } + + if (out_value) { + if (!_Py_TryIncrefCompare(value_loc, value)) { + if (out_key) { + Py_DECREF(*out_key); + } + return -1; + } + *out_value = value; + } + + return 0; +} + +static int +dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value) +{ + int res; + dictiterobject *di = (dictiterobject *)self; + Py_ssize_t i; + PyDictKeysObject *k; + + assert (PyDict_Check(d)); + + if (di->di_used != _Py_atomic_load_ssize_relaxed(&d->ma_used)) { + PyErr_SetString(PyExc_RuntimeError, + "dictionary changed size during iteration"); + di->di_used = -1; /* Make this state sticky */ + return -1; + } + + ensure_shared_on_read(d); + + i = _Py_atomic_load_ssize_relaxed(&di->di_pos); + k = _Py_atomic_load_ptr_relaxed(&d->ma_keys); + assert(i >= 0); + if (_PyDict_HasSplitTable(d)) { + PyDictValues *values = _Py_atomic_load_ptr_relaxed(&d->ma_values); + if (values == NULL) { + goto concurrent_modification; + } + + Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size); + if (i >= used) { + goto fail; + } + + // We're racing against writes to the order from delete_index_from_values, but + // single threaded can suffer from concurrent modification to those as well and + // can have either duplicated or skipped attributes, so we strive to do no better + // here. + int index = get_index_from_order(d, i); + PyObject *value = _Py_atomic_load_ptr(&values->values[index]); + if (acquire_key_value(&DK_UNICODE_ENTRIES(k)[index].me_key, value, + &values->values[index], out_key, out_value) < 0) { + goto try_locked; } } else { - result = PyTuple_New(2); - if (result == NULL) - return NULL; - PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(value)); + Py_ssize_t n = _Py_atomic_load_ssize_relaxed(&k->dk_nentries); + if (DK_IS_UNICODE(k)) { + PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i]; + PyObject *value; + while (i < n && + (value = _Py_atomic_load_ptr(&entry_ptr->me_value)) == NULL) { + entry_ptr++; + i++; + } + if (i >= n) + goto fail; + + if (acquire_key_value(&entry_ptr->me_key, value, + &entry_ptr->me_value, out_key, out_value) < 0) { + goto try_locked; + } + } + else { + PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; + PyObject *value; + while (i < n && + (value = _Py_atomic_load_ptr(&entry_ptr->me_value)) == NULL) { + entry_ptr++; + i++; + } + + if (i >= n) + goto fail; + + if (acquire_key_value(&entry_ptr->me_key, value, + &entry_ptr->me_value, out_key, out_value) < 0) { + goto try_locked; + } + } + } + // We found an element (key), but did not expect it + Py_ssize_t len; + if ((len = _Py_atomic_load_ssize_relaxed(&di->len)) == 0) { + goto concurrent_modification; } - return result; + + _Py_atomic_store_ssize_relaxed(&di->di_pos, i + 1); + _Py_atomic_store_ssize_relaxed(&di->len, len - 1); + return 0; + +concurrent_modification: + PyErr_SetString(PyExc_RuntimeError, + "dictionary keys changed during iteration"); fail: di->di_dict = NULL; Py_DECREF(d); - return NULL; + return -1; + +try_locked: + Py_BEGIN_CRITICAL_SECTION(d); + res = dictiter_iternextitem_lock_held(d, self, out_key, out_value); + Py_END_CRITICAL_SECTION(); + return res; +} + +#endif + +static bool +has_unique_reference(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return (_Py_IsOwnedByCurrentThread(op) && + op->ob_ref_local == 1 && + _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); +#else + return Py_REFCNT(op) == 1; +#endif +} + +static bool +acquire_iter_result(PyObject *result) +{ + if (has_unique_reference(result)) { + Py_INCREF(result); + return true; + } + return false; } static PyObject * @@ -4832,11 +5591,37 @@ dictiter_iternextitem(PyObject *self) if (d == NULL) return NULL; - PyObject *item; - Py_BEGIN_CRITICAL_SECTION(d); - item = dictiter_iternextitem_lock_held(d, self); - Py_END_CRITICAL_SECTION(); - return item; + PyObject *key, *value; +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, &key, &value) == 0) { +#else + if (dictiter_iternextitem_lock_held(d, self, &key, &value) == 0) { + +#endif + PyObject *result = di->di_result; + if (acquire_iter_result(result)) { + PyObject *oldkey = PyTuple_GET_ITEM(result, 0); + PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, key); + PyTuple_SET_ITEM(result, 1, value); + Py_DECREF(oldkey); + Py_DECREF(oldvalue); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result == NULL) + return NULL; + PyTuple_SET_ITEM(result, 0, key); + PyTuple_SET_ITEM(result, 1, value); + } + return result; + } + return NULL; } PyTypeObject PyDictIterItem_Type = { @@ -4876,7 +5661,7 @@ PyTypeObject PyDictIterItem_Type = { /* dictreviter */ static PyObject * -dictreviter_iter_PyDict_Next(PyDictObject *d, PyObject *self) +dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; @@ -4897,9 +5682,9 @@ dictreviter_iter_PyDict_Next(PyDictObject *d, PyObject *self) if (i < 0) { goto fail; } - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(k)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key); value = d->ma_values->values[index]; assert (value != NULL); } @@ -4983,7 +5768,7 @@ dictreviter_iternext(PyObject *self) PyObject *value; Py_BEGIN_CRITICAL_SECTION(d); - value = dictreviter_iter_PyDict_Next(d, self); + value = dictreviter_iter_lock_held(d, self); Py_END_CRITICAL_SECTION(); return value; @@ -5085,7 +5870,7 @@ dictview_len(PyObject *self) _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_ssize_t len = 0; if (dv->dv_dict != NULL) - len = dv->dv_dict->ma_used; + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(dv->dv_dict->ma_used); return len; } @@ -5886,9 +6671,10 @@ dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) /* Returns NULL if cannot allocate a new PyDictKeysObject, but does not set an error */ PyDictKeysObject * -_PyDict_NewKeysForClass(void) +_PyDict_NewKeysForClass(PyHeapTypeObject *cls) { PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *keys = new_keys_object( interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { @@ -5900,65 +6686,60 @@ _PyDict_NewKeysForClass(void) keys->dk_usable = SHARED_KEYS_MAX_SIZE; keys->dk_kind = DICT_KEYS_SPLIT; } + if (cls->ht_type.tp_dict) { + PyObject *attrs = PyDict_GetItem(cls->ht_type.tp_dict, &_Py_ID(__static_attributes__)); + if (attrs != NULL && PyTuple_Check(attrs)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(attrs); i++) { + PyObject *key = PyTuple_GET_ITEM(attrs, i); + Py_hash_t hash; + if (PyUnicode_CheckExact(key) && (hash = unicode_get_hash(key)) != -1) { + if (insert_split_key(keys, key, hash) == DKIX_EMPTY) { + break; + } + } + } + } + } return keys; } -#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) - -int +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); + OBJECT_STAT_INC(inline_values); +#ifdef Py_GIL_DISABLED + Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable); + if (usable > 1) { + LOCK_KEYS(keys); + if (keys->dk_usable > 1) { + _Py_atomic_store_ssize(&keys->dk_usable, keys->dk_usable - 1); + } + UNLOCK_KEYS(keys); + } +#else if (keys->dk_usable > 1) { keys->dk_usable--; } +#endif size_t size = shared_keys_usable_size(keys); - PyDictValues *values = new_values(size); - if (values == NULL) { - PyErr_NoMemory(); - return -1; - } - assert(((uint8_t *)values)[-1] >= (size + 2)); - ((uint8_t *)values)[-2] = 0; + PyDictValues *values = _PyObject_InlineValues(obj); + assert(size < 256); + values->capacity = (uint8_t)size; + values->size = 0; + values->embedded = 1; + values->valid = 1; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values); - return 0; -} - -int -_PyObject_InitializeDict(PyObject *obj) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyTypeObject *tp = Py_TYPE(obj); - if (tp->tp_dictoffset == 0) { - return 0; - } - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - OBJECT_STAT_INC(new_values); - return _PyObject_InitInlineValues(obj, tp); - } - PyObject *dict; - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - } - else { - dict = PyDict_New(); - } - if (dict == NULL) { - return -1; - } - PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - *dictptr = dict; - return 0; + _PyObject_ManagedDictPointer(obj)->dict = NULL; } -static PyObject * +static PyDictObject * make_dict_from_instance_attributes(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values) { @@ -5973,127 +6754,242 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(interp, keys, values, used, 0); + PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); if (track && res) { _PyObject_GC_TRACK(res); } return res; } -PyObject * -_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) +PyDictObject * +_PyObject_MaterializeManagedDict_LockHeld(PyObject *obj) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj); + OBJECT_STAT_INC(dict_materialized_on_request); - return make_dict_from_instance_attributes(interp, keys, values); + + PyDictValues *values = _PyObject_InlineValues(obj); + PyDictObject *dict; + if (values->valid) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); + dict = make_dict_from_instance_attributes(interp, keys, values); + } + else { + dict = (PyDictObject *)PyDict_New(); + } + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + dict); + return dict; } -static bool -has_unique_reference(PyObject *op) +PyDictObject * +_PyObject_MaterializeManagedDict(PyObject *obj) { + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + return dict; + } + + Py_BEGIN_CRITICAL_SECTION(obj); + #ifdef Py_GIL_DISABLED - return (_Py_IsOwnedByCurrentThread(op) && - op->ob_ref_local == 1 && - _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); -#else - return Py_REFCNT(op) == 1; + dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + // We raced with another thread creating the dict + goto exit; + } +#endif + dict = _PyObject_MaterializeManagedDict_LockHeld(obj); + +#ifdef Py_GIL_DISABLED +exit: #endif + Py_END_CRITICAL_SECTION(); + return dict; } -// Return true if the dict was dematerialized, false otherwise. -bool -_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) +int +_PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value) { - assert(_PyObject_DictOrValuesPointer(obj) == dorv); - assert(!_PyDictOrValues_IsValues(*dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); - if (dict == NULL) { - return false; - } - // It's likely that this dict still shares its keys (if it was materialized - // on request and not heavily modified): - if (!PyDict_CheckExact(dict)) { - return false; - } - assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); - if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || - !has_unique_reference((PyObject *)dict)) - { - return false; + if (value == NULL) { + Py_hash_t hash = _PyObject_HashFast(name); + if (hash == -1) { + return -1; + } + return delitem_knownhash_lock_held((PyObject *)dict, name, hash); + } else { + return setitem_lock_held(dict, name, value); } - assert(dict->ma_values); - // We have an opportunity to do something *really* cool: dematerialize it! - _PyDictKeys_DecRef(dict->ma_keys); - _PyDictOrValues_SetValues(dorv, dict->ma_values); - OBJECT_STAT_INC(dict_dematerialized); - // Don't try this at home, kids: - dict->ma_keys = NULL; - dict->ma_values = NULL; - Py_DECREF(dict); - return true; } -int -_PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, +// Called with either the object's lock or the dict's lock held +// depending on whether or not a dict has been materialized for +// the object. +static int +store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) { - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); - assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); Py_ssize_t ix = DKIX_EMPTY; + PyDictObject *dict = _PyObject_GetManagedDict(obj); + assert(dict == NULL || ((PyDictObject *)dict)->ma_values == values); if (PyUnicode_CheckExact(name)) { - ix = insert_into_dictkeys(keys, name); - } - if (ix == DKIX_EMPTY) { + Py_hash_t hash = unicode_get_hash(name); + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(name); + assert(hash != -1); + } + + ix = insert_split_key(keys, name, hash); + #ifdef Py_STATS - if (PyUnicode_CheckExact(name)) { - if (shared_keys_usable_size(keys) == SHARED_KEYS_MAX_SIZE) { - OBJECT_STAT_INC(dict_materialized_too_big); + if (ix == DKIX_EMPTY) { + if (PyUnicode_CheckExact(name)) { + if (shared_keys_usable_size(keys) == SHARED_KEYS_MAX_SIZE) { + OBJECT_STAT_INC(dict_materialized_too_big); + } + else { + OBJECT_STAT_INC(dict_materialized_new_key); + } } else { - OBJECT_STAT_INC(dict_materialized_new_key); + OBJECT_STAT_INC(dict_materialized_str_subclass); } } - else { - OBJECT_STAT_INC(dict_materialized_str_subclass); - } #endif - PyObject *dict = make_dict_from_instance_attributes( - interp, keys, values); + } + + if (ix == DKIX_EMPTY) { + int res; if (dict == NULL) { - return -1; - } - _PyObject_DictOrValuesPointer(obj)->dict = dict; - if (value == NULL) { - return PyDict_DelItem(dict, name); - } - else { - return PyDict_SetItem(dict, name, value); + // Make the dict but don't publish it in the object + // so that no one else will see it. + dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values); + if (dict == NULL || + _PyDict_SetItem_LockHeld(dict, name, value) < 0) { + Py_XDECREF(dict); + return -1; + } + + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); + return 0; } + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); + + res = _PyDict_SetItem_LockHeld(dict, name, value); + return res; } + PyObject *old_value = values->values[ix]; - values->values[ix] = Py_XNewRef(value); + if (old_value == NULL && value == NULL) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + Py_TYPE(obj)->tp_name, name); + return -1; + } + + if (dict) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDict_WatchEvent event = (old_value == NULL ? PyDict_EVENT_ADDED : + value == NULL ? PyDict_EVENT_DELETED : + PyDict_EVENT_MODIFIED); + _PyDict_NotifyEvent(interp, event, dict, name, value); + } + + FT_ATOMIC_STORE_PTR_RELEASE(values->values[ix], Py_XNewRef(value)); + if (old_value == NULL) { - if (value == NULL) { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - Py_TYPE(obj)->tp_name, name); - return -1; - } _PyDictValues_AddToInsertionOrder(values, ix); + if (dict) { + assert(dict->ma_values == values); + STORE_USED(dict, dict->ma_used + 1); + } } else { if (value == NULL) { delete_index_from_values(values, ix); + if (dict) { + assert(dict->ma_values == values); + STORE_USED(dict, dict->ma_used - 1); + } } Py_DECREF(old_value); } return 0; } +static inline int +store_instance_attr_dict(PyObject *obj, PyDictObject *dict, PyObject *name, PyObject *value) +{ + PyDictValues *values = _PyObject_InlineValues(obj); + int res; + Py_BEGIN_CRITICAL_SECTION(dict); + if (dict->ma_values == values) { + res = store_instance_attr_lock_held(obj, values, name, value); + } + else { + res = _PyDict_SetItem_LockHeld(dict, name, value); + } + Py_END_CRITICAL_SECTION(); + return res; +} + +int +_PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value) +{ + PyDictValues *values = _PyObject_InlineValues(obj); + if (!FT_ATOMIC_LOAD_UINT8(values->valid)) { + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + dict = (PyDictObject *)PyObject_GenericGetDict(obj, NULL); + if (dict == NULL) { + return -1; + } + int res = store_instance_attr_dict(obj, dict, name, value); + Py_DECREF(dict); + return res; + } + return store_instance_attr_dict(obj, dict, name, value); + } + +#ifdef Py_GIL_DISABLED + // We have a valid inline values, at least for now... There are two potential + // races with having the values become invalid. One is the dictionary + // being detached from the object. The other is if someone is inserting + // into the dictionary directly and therefore causing it to resize. + // + // If we haven't materialized the dictionary yet we lock on the object, which + // will also be used to prevent the dictionary from being materialized while + // we're doing the insertion. If we race and the dictionary gets created + // then we'll need to release the object lock and lock the dictionary to + // prevent resizing. + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + int res; + Py_BEGIN_CRITICAL_SECTION(obj); + dict = _PyObject_GetManagedDict(obj); + + if (dict == NULL) { + res = store_instance_attr_lock_held(obj, values, name, value); + } + Py_END_CRITICAL_SECTION(); + + if (dict == NULL) { + return res; + } + } + return store_instance_attr_dict(obj, dict, name, value); +#else + return store_instance_attr_lock_held(obj, values, name, value); +#endif +} + /* Sanity check for managed dicts */ #if 0 #define CHECK(val) assert(val); if (!(val)) { return 0; } @@ -6103,9 +6999,9 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) { PyTypeObject *tp = Py_TYPE(obj); CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + if (_PyManagedDictPointer_IsValues(*managed_dict)) { + PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict); int size = ((uint8_t *)values)[-2]; int count = 0; PyDictKeysObject *keys = CACHED_KEYS(tp); @@ -6117,27 +7013,87 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) CHECK(size == count); } else { - if (dorv_ptr->dict != NULL) { - CHECK(PyDict_Check(dorv_ptr->dict)); + if (managed_dict->dict != NULL) { + CHECK(PyDict_Check(managed_dict->dict)); } } return 1; } #endif -PyObject * -_PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name) +// Attempts to get an instance attribute from the inline values. Returns true +// if successful, or false if the caller needs to lookup in the dictionary. +bool +_PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr) { assert(PyUnicode_CheckExact(name)); + PyDictValues *values = _PyObject_InlineValues(obj); + if (!FT_ATOMIC_LOAD_UINT8(values->valid)) { + return false; + } + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name); if (ix == DKIX_EMPTY) { - return NULL; + *attr = NULL; + return true; + } + +#ifdef Py_GIL_DISABLED + PyObject *value = _Py_atomic_load_ptr_acquire(&values->values[ix]); + if (value == NULL || _Py_TryIncrefCompare(&values->values[ix], value)) { + *attr = value; + return true; + } + + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + // No dict, lock the object to prevent one from being + // materialized... + bool success = false; + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + // Still no dict, we can read from the values + assert(values->valid); + value = values->values[ix]; + *attr = _Py_XNewRefWithLock(value); + success = true; + } + + Py_END_CRITICAL_SECTION(); + + if (success) { + return true; + } + } + + // We have a dictionary, we'll need to lock it to prevent + // the values from being resized. + assert(dict != NULL); + + bool success; + Py_BEGIN_CRITICAL_SECTION(dict); + + if (dict->ma_values == values && FT_ATOMIC_LOAD_UINT8(values->valid)) { + value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); + *attr = _Py_XNewRefWithLock(value); + success = true; + } else { + // Caller needs to lookup from the dictionary + success = false; } + + Py_END_CRITICAL_SECTION(); + + return success; +#else PyObject *value = values->values[ix]; - return Py_XNewRef(value); + *attr = Py_XNewRef(value); + return true; +#endif } int @@ -6147,122 +7103,252 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) if (tp->tp_dictoffset == 0) { return 1; } - PyObject *dict; - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { + PyDictObject *dict; + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (FT_ATOMIC_LOAD_UINT8(values->valid)) { PyDictKeysObject *keys = CACHED_KEYS(tp); for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) { + if (FT_ATOMIC_LOAD_PTR_RELAXED(values->values[i]) != NULL) { return 0; } } return 1; } - dict = _PyDictOrValues_GetDict(dorv); + dict = _PyObject_GetManagedDict(obj); + } + else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + dict = _PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - dict = *dictptr; + dict = (PyDictObject *)*dictptr; } if (dict == NULL) { return 1; } - return ((PyDictObject *)dict)->ma_used == 0; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)dict)->ma_used) == 0; } -void -_PyObject_FreeInstanceAttributes(PyObject *self) +int +PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) { - PyTypeObject *tp = Py_TYPE(self); - assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - if (!_PyDictOrValues_IsValues(dorv)) { - return; + PyTypeObject *tp = Py_TYPE(obj); + if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + return 0; } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_XDECREF(values->values[i]); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_VISIT(values->values[i]); + } + return 0; + } + } + Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict); + return 0; +} + +static void +set_dict_inline_values(PyObject *obj, PyDictObject *new_dict) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + + PyDictValues *values = _PyObject_InlineValues(obj); + + Py_XINCREF(new_dict); + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict); + + if (values->valid) { + FT_ATOMIC_STORE_UINT8(values->valid, 0); + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_CLEAR(values->values[i]); + } } - free_values(values); } int -PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) +_PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict) { + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + int err = 0; PyTypeObject *tp = Py_TYPE(obj); - if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - return 0; - } - assert(tp->tp_dictoffset); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_VISIT(values->values[i]); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { +#ifdef Py_GIL_DISABLED + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_ManagedDictPointer(obj)->dict; + if (dict == NULL) { + set_dict_inline_values(obj, (PyDictObject *)new_dict); + } + + Py_END_CRITICAL_SECTION(); + + if (dict == NULL) { + return 0; + } +#else + set_dict_inline_values(obj, (PyDictObject *)new_dict); + return 0; +#endif + } + + Py_BEGIN_CRITICAL_SECTION2(dict, obj); + + // We've locked dict, but the actual dict could have changed + // since we locked it. + dict = _PyObject_ManagedDictPointer(obj)->dict; + err = _PyDict_DetachFromObject(dict, obj); + if (err == 0) { + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)Py_XNewRef(new_dict)); + } + Py_END_CRITICAL_SECTION2(); + + if (err == 0) { + Py_XDECREF(dict); } } else { - PyObject *dict = _PyDictOrValues_GetDict(dorv); - Py_VISIT(dict); + PyDictObject *dict; + + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_ManagedDictPointer(obj)->dict; + + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)Py_XNewRef(new_dict)); + + Py_END_CRITICAL_SECTION(); + + Py_XDECREF(dict); } - return 0; + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + return err; } void PyObject_ClearManagedDict(PyObject *obj) { - PyTypeObject *tp = Py_TYPE(obj); - if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - return; + if (_PyObject_SetManagedDict(obj, NULL) < 0) { + PyErr_WriteUnraisable(NULL); } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_CLEAR(values->values[i]); +} + +int +_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) +{ + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj); + assert(_PyObject_ManagedDictPointer(obj)->dict == mp); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + + if (FT_ATOMIC_LOAD_PTR_RELAXED(mp->ma_values) != _PyObject_InlineValues(obj)) { + return 0; + } + + // We could be called with an unlocked dict when the caller knows the + // values are already detached, so we assert after inline values check. + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + assert(mp->ma_values->embedded == 1); + assert(mp->ma_values->valid == 1); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + + PyDictValues *values = copy_values(mp->ma_values); + + if (values == NULL) { + /* Out of memory. Clear the dict */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *oldkeys = mp->ma_keys; + set_keys(mp, Py_EMPTY_KEYS); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + STORE_USED(mp, 0); + PyErr_NoMemory(); + return -1; + } + mp->ma_values = values; + + FT_ATOMIC_STORE_UINT8(_PyObject_InlineValues(obj)->valid, 0); + + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + ASSERT_CONSISTENT(mp); + return 0; +} + +static inline PyObject * +ensure_managed_dict(PyObject *obj) +{ + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + PyTypeObject *tp = Py_TYPE(obj); + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(obj)->valid)) { + dict = _PyObject_MaterializeManagedDict(obj); + } + else { +#ifdef Py_GIL_DISABLED + // Check again that we're not racing with someone else creating the dict + Py_BEGIN_CRITICAL_SECTION(obj); + dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + goto done; + } +#endif + dict = (PyDictObject *)new_dict_with_shared_keys(_PyInterpreterState_GET(), + CACHED_KEYS(tp)); + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); + +#ifdef Py_GIL_DISABLED +done: + Py_END_CRITICAL_SECTION(); +#endif } - dorv_ptr->dict = NULL; - free_values(values); } - else { - PyObject *dict = dorv_ptr->dict; - if (dict) { - dorv_ptr->dict = NULL; - Py_DECREF(dict); + return (PyObject *)dict; +} + +static inline PyObject * +ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr) +{ + PyDictKeysObject *cached; + + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr); + if (dict == NULL) { +#ifdef Py_GIL_DISABLED + Py_BEGIN_CRITICAL_SECTION(obj); + dict = *dictptr; + if (dict != NULL) { + goto done; } +#endif + PyTypeObject *tp = Py_TYPE(obj); + if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); + dict = new_dict_with_shared_keys(interp, cached); + } + else { + dict = PyDict_New(); + } + FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, dict); +#ifdef Py_GIL_DISABLED +done: + Py_END_CRITICAL_SECTION(); +#endif } + return dict; } PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { - PyObject *dict; - PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - OBJECT_STAT_INC(dict_materialized_on_request); - dict = make_dict_from_instance_attributes( - interp, CACHED_KEYS(tp), values); - if (dict != NULL) { - dorv_ptr->dict = dict; - } - } - else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); - if (dict == NULL) { - dictkeys_incref(CACHED_KEYS(tp)); - OBJECT_STAT_INC(dict_materialized_on_request); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - dorv_ptr->dict = dict; - } - } + return Py_XNewRef(ensure_managed_dict(obj)); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -6271,64 +7357,28 @@ PyObject_GenericGetDict(PyObject *obj, void *context) "This object has no __dict__"); return NULL; } - dict = *dictptr; - if (dict == NULL) { - PyTypeObject *tp = Py_TYPE(obj); - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys( - interp, CACHED_KEYS(tp)); - } - else { - *dictptr = dict = PyDict_New(); - } - } + + return Py_XNewRef(ensure_nonmanaged_dict(obj, dictptr)); } - return Py_XNewRef(dict); } int -_PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, +_PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *key, PyObject *value) { PyObject *dict; int res; - PyDictKeysObject *cached; - PyInterpreterState *interp = _PyInterpreterState_GET(); assert(dictptr != NULL); - if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { - assert(dictptr != NULL); - dict = *dictptr; - if (dict == NULL) { - assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)); - dictkeys_incref(cached); - dict = new_dict_with_shared_keys(interp, cached); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } - else { - res = PyDict_SetItem(dict, key, value); - } - } else { - dict = *dictptr; - if (dict == NULL) { - dict = PyDict_New(); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } else { - res = PyDict_SetItem(dict, key, value); - } + dict = ensure_nonmanaged_dict(obj, dictptr); + if (dict == NULL) { + return -1; } + + Py_BEGIN_CRITICAL_SECTION(dict); + res = _PyDict_SetItem_LockHeld((PyDictObject *)dict, key, value); ASSERT_CONSISTENT(dict); + Py_END_CRITICAL_SECTION(); return res; } @@ -6336,7 +7386,7 @@ void _PyDictKeys_DecRef(PyDictKeysObject *keys) { PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, @@ -6378,7 +7428,7 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->ma_version_tag |= (1LL << watcher_id); + ((PyDictObject*)dict)->_ma_watcher_tag |= (1LL << watcher_id); return 0; } @@ -6393,7 +7443,7 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->ma_version_tag &= ~(1LL << watcher_id); + ((PyDictObject*)dict)->_ma_watcher_tag &= ~(1LL << watcher_id); return 0; } @@ -6461,3 +7511,24 @@ _PyDict_SendEvent(int watcher_bits, watcher_bits >>= 1; } } + +#ifndef NDEBUG +static int +_PyObject_InlineValuesConsistencyCheck(PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + return 1; + } + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + return 1; + } + if (dict->ma_values == _PyObject_InlineValues(obj) || + _PyObject_InlineValues(obj)->valid == 0) { + return 1; + } + assert(0); + return 0; +} +#endif diff --git a/Objects/exception_handling_notes.txt b/Objects/exception_handling_notes.txt deleted file mode 100644 index 387ef935ce739e..00000000000000 --- a/Objects/exception_handling_notes.txt +++ /dev/null @@ -1,182 +0,0 @@ -Description of exception handling in Python 3.11 ------------------------------------------------- - -Python 3.11 uses what is known as "zero-cost" exception handling. -Prior to 3.11, exceptions were handled by a runtime stack of "blocks". - -In zero-cost exception handling, the cost of supporting exceptions is minimized. -In the common case (where no exception is raised) the cost is reduced -to zero (or close to zero). -The cost of raising an exception is increased, but not by much. - -The following code: - -def f(): - try: - g(0) - except: - return "fail" - -compiles as follows in 3.10: - - 2 0 SETUP_FINALLY 7 (to 16) - - 3 2 LOAD_GLOBAL 0 (g) - 4 LOAD_CONST 1 (0) - 6 CALL_NO_KW 1 - 8 POP_TOP - 10 POP_BLOCK - 12 LOAD_CONST 0 (None) - 14 RETURN_VALUE - - 4 >> 16 POP_TOP - 18 POP_TOP - 20 POP_TOP - - 5 22 POP_EXCEPT - 24 LOAD_CONST 3 ('fail') - 26 RETURN_VALUE - -Note the explicit instructions to push and pop from the "block" stack: -SETUP_FINALLY and POP_BLOCK. - -In 3.11, the SETUP_FINALLY and POP_BLOCK are eliminated, replaced with -a table to determine where to jump to when an exception is raised. - - 1 0 RESUME 0 - - 2 2 NOP - - 3 4 LOAD_GLOBAL 1 (g + NULL) - 16 LOAD_CONST 1 (0) - 18 PRECALL 1 - 22 CALL 1 - 32 POP_TOP - 34 LOAD_CONST 0 (None) - 36 RETURN_VALUE - >> 38 PUSH_EXC_INFO - - 4 40 POP_TOP - - 5 42 POP_EXCEPT - 44 LOAD_CONST 2 ('fail') - 46 RETURN_VALUE - >> 48 COPY 3 - 50 POP_EXCEPT - 52 RERAISE 1 -ExceptionTable: - 4 to 32 -> 38 [0] - 38 to 40 -> 48 [1] lasti - -(Note this code is from 3.11, later versions may have slightly different bytecode.) - -If an instruction raises an exception then its offset is used to find the target to jump to. -For example, the CALL at offset 22, falls into the range 4 to 32. -So, if g() raises an exception, then control jumps to offset 38. - - -Unwinding ---------- - -When an exception is raised, the current instruction offset is used to find following: -target to jump to, stack depth, and 'lasti', which determines whether the instruction -offset of the raising instruction should be pushed. - -This information is stored in the exception table, described below. - -If there is no relevant entry, the exception bubbles up to the caller. - -If there is an entry, then: - 1. pop values from the stack until it matches the stack depth for the handler. - 2. if 'lasti' is true, then push the offset that the exception was raised at. - 3. push the exception to the stack. - 4. jump to the target offset and resume execution. - - -Format of the exception table ------------------------------ - -Conceptually, the exception table consists of a sequence of 5-tuples: - 1. start-offset (inclusive) - 2. end-offset (exclusive) - 3. target - 4. stack-depth - 5. push-lasti (boolean) - -All offsets and lengths are in instructions, not bytes. - -We want the format to be compact, but quickly searchable. -For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. -For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. -Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. - -It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: - start, size, target, depth, push-lasti - -Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each instruction takes 2 bytes. -It also happens that depth is generally quite small. - -So, we need to encode: - start (up to 30 bits) - size (up to 30 bits) - target (up to 30 bits) - depth (up to ~8 bits) - lasti (1 bit) - -We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. -Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. -Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. -The 8 bits of a bit are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. - -In addition, we will combine depth and lasti into a single value, ((depth<<1)+lasti), before encoding. - -For example, the exception entry: - start: 20 - end: 28 - target: 100 - depth: 3 - lasti: False - -is encoded first by converting to the more compact four value form: - start: 20 - size: 8 - target: 100 - depth<<1+lasti: 6 - -which is then encoded as: - 148 (MSB + 20 for start) - 8 (size) - 65 (Extend bit + 1) - 36 (Remainder of target, 100 == (1<<6)+36) - 6 - -for a total of five bytes. - - - -Script to parse the exception table ------------------------------------ - -def parse_varint(iterator): - b = next(iterator) - val = b & 63 - while b&64: - val <<= 6 - b = next(iterator) - val |= b&63 - return val - -def parse_exception_table(code): - iterator = iter(code.co_exceptiontable) - try: - while True: - start = parse_varint(iterator)*2 - length = parse_varint(iterator)*2 - end = start + length - 2 # Present as inclusive, not exclusive - target = parse_varint(iterator)*2 - dl = parse_varint(iterator) - depth = dl >> 1 - lasti = bool(dl&1) - yield start, end, target, depth, lasti - except StopIteration: - return diff --git a/Objects/exceptions.c b/Objects/exceptions.c index cff55d05163b6b..6fbe0f197eaebf 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -78,6 +78,40 @@ BaseException_init(PyBaseExceptionObject *self, PyObject *args, PyObject *kwds) return 0; } + +static PyObject * +BaseException_vectorcall(PyObject *type_obj, PyObject * const*args, + size_t nargsf, PyObject *kwnames) +{ + PyTypeObject *type = _PyType_CAST(type_obj); + if (!_PyArg_NoKwnames(type->tp_name, kwnames)) { + return NULL; + } + + PyBaseExceptionObject *self; + self = (PyBaseExceptionObject *)type->tp_alloc(type, 0); + if (!self) { + return NULL; + } + + // The dict is created on the fly in PyObject_GenericSetAttr() + self->dict = NULL; + self->notes = NULL; + self->traceback = NULL; + self->cause = NULL; + self->context = NULL; + self->suppress_context = 0; + + self->args = _PyTuple_FromArray(args, PyVectorcall_NARGS(nargsf)); + if (!self->args) { + Py_DECREF(self); + return NULL; + } + + return (PyObject *)self; +} + + static int BaseException_clear(PyBaseExceptionObject *self) { @@ -120,7 +154,7 @@ BaseException_str(PyBaseExceptionObject *self) { switch (PyTuple_GET_SIZE(self->args)) { case 0: - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); case 1: return PyObject_Str(PyTuple_GET_ITEM(self->args, 0)); default: @@ -486,6 +520,7 @@ static PyTypeObject _PyExc_BaseException = { (initproc)BaseException_init, /* tp_init */ 0, /* tp_alloc */ BaseException_new, /* tp_new */ + .tp_vectorcall = BaseException_vectorcall, }; /* the CPython API expects exceptions to be (PyObject *) - both a hold-over from the previous implementation and also allowing Python objects to be used @@ -510,10 +545,10 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \ }; \ PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME -#define MiddlingExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCDOC) \ -static PyTypeObject _PyExc_ ## EXCNAME = { \ +#define MiddlingExtendsExceptionEx(EXCBASE, EXCNAME, PYEXCNAME, EXCSTORE, EXCDOC) \ +PyTypeObject _PyExc_ ## EXCNAME = { \ PyVarObject_HEAD_INIT(NULL, 0) \ - # EXCNAME, \ + # PYEXCNAME, \ sizeof(Py ## EXCSTORE ## Object), \ 0, (destructor)EXCSTORE ## _dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, \ @@ -522,8 +557,12 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \ (inquiry)EXCSTORE ## _clear, 0, 0, 0, 0, 0, 0, 0, &_ ## EXCBASE, \ 0, 0, 0, offsetof(Py ## EXCSTORE ## Object, dict), \ (initproc)EXCSTORE ## _init, 0, 0, \ -}; \ -PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME +}; + +#define MiddlingExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCDOC) \ + static MiddlingExtendsExceptionEx( \ + EXCBASE, EXCNAME, EXCNAME, EXCSTORE, EXCDOC); \ + PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME #define ComplexExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCNEW, \ EXCMETHODS, EXCMEMBERS, EXCGETSET, \ @@ -2177,6 +2216,10 @@ SimpleExtendsException(PyExc_Exception, RuntimeError, SimpleExtendsException(PyExc_RuntimeError, RecursionError, "Recursion limit exceeded."); +// PythonFinalizationError extends RuntimeError +SimpleExtendsException(PyExc_RuntimeError, PythonFinalizationError, + "Operation blocked during Python finalization."); + /* * NotImplementedError extends RuntimeError */ @@ -2569,8 +2612,8 @@ MiddlingExtendsException(PyExc_IndentationError, TabError, SyntaxError, /* * IncompleteInputError extends SyntaxError */ -MiddlingExtendsException(PyExc_SyntaxError, IncompleteInputError, SyntaxError, - "incomplete input."); +MiddlingExtendsExceptionEx(PyExc_SyntaxError, IncompleteInputError, _IncompleteInputError, + SyntaxError, "incomplete input."); /* * LookupError extends Exception @@ -2951,46 +2994,55 @@ UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * UnicodeEncodeError_str(PyObject *self) { - PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; PyObject *result = NULL; PyObject *reason_str = NULL; PyObject *encoding_str = NULL; - if (!uself->object) + if (exc->object == NULL) { /* Not properly initialized. */ - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } /* Get reason and encoding as strings, which they might not be if they've been modified after we were constructed. */ - reason_str = PyObject_Str(uself->reason); - if (reason_str == NULL) + reason_str = PyObject_Str(exc->reason); + if (reason_str == NULL) { goto done; - encoding_str = PyObject_Str(uself->encoding); - if (encoding_str == NULL) + } + encoding_str = PyObject_Str(exc->encoding); + if (encoding_str == NULL) { goto done; + } - if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) { - Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start); + Py_ssize_t len = PyUnicode_GET_LENGTH(exc->object); + Py_ssize_t start = exc->start, end = exc->end; + + if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) { + Py_UCS4 badchar = PyUnicode_ReadChar(exc->object, start); const char *fmt; - if (badchar <= 0xff) + if (badchar <= 0xff) { fmt = "'%U' codec can't encode character '\\x%02x' in position %zd: %U"; - else if (badchar <= 0xffff) + } + else if (badchar <= 0xffff) { fmt = "'%U' codec can't encode character '\\u%04x' in position %zd: %U"; - else + } + else { fmt = "'%U' codec can't encode character '\\U%08x' in position %zd: %U"; + } result = PyUnicode_FromFormat( fmt, encoding_str, (int)badchar, - uself->start, + start, reason_str); } else { result = PyUnicode_FromFormat( "'%U' codec can't encode characters in position %zd-%zd: %U", encoding_str, - uself->start, - uself->end-1, + start, + end - 1, reason_str); } done: @@ -3064,41 +3116,46 @@ UnicodeDecodeError_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * UnicodeDecodeError_str(PyObject *self) { - PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; PyObject *result = NULL; PyObject *reason_str = NULL; PyObject *encoding_str = NULL; - if (!uself->object) + if (exc->object == NULL) { /* Not properly initialized. */ - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } /* Get reason and encoding as strings, which they might not be if they've been modified after we were constructed. */ - reason_str = PyObject_Str(uself->reason); - if (reason_str == NULL) + reason_str = PyObject_Str(exc->reason); + if (reason_str == NULL) { goto done; - encoding_str = PyObject_Str(uself->encoding); - if (encoding_str == NULL) + } + encoding_str = PyObject_Str(exc->encoding); + if (encoding_str == NULL) { goto done; + } - if (uself->start < PyBytes_GET_SIZE(uself->object) && uself->end == uself->start+1) { - int byte = (int)(PyBytes_AS_STRING(((PyUnicodeErrorObject *)self)->object)[uself->start]&0xff); + Py_ssize_t len = PyBytes_GET_SIZE(exc->object); + Py_ssize_t start = exc->start, end = exc->end; + + if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) { + int badbyte = (int)(PyBytes_AS_STRING(exc->object)[start] & 0xff); result = PyUnicode_FromFormat( "'%U' codec can't decode byte 0x%02x in position %zd: %U", encoding_str, - byte, - uself->start, + badbyte, + start, reason_str); } else { result = PyUnicode_FromFormat( "'%U' codec can't decode bytes in position %zd-%zd: %U", encoding_str, - uself->start, - uself->end-1, - reason_str - ); + start, + end - 1, + reason_str); } done: Py_XDECREF(reason_str); @@ -3161,42 +3218,49 @@ UnicodeTranslateError_init(PyUnicodeErrorObject *self, PyObject *args, static PyObject * UnicodeTranslateError_str(PyObject *self) { - PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; PyObject *result = NULL; PyObject *reason_str = NULL; - if (!uself->object) + if (exc->object == NULL) { /* Not properly initialized. */ - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } /* Get reason as a string, which it might not be if it's been modified after we were constructed. */ - reason_str = PyObject_Str(uself->reason); - if (reason_str == NULL) + reason_str = PyObject_Str(exc->reason); + if (reason_str == NULL) { goto done; + } - if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) { - Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start); + Py_ssize_t len = PyUnicode_GET_LENGTH(exc->object); + Py_ssize_t start = exc->start, end = exc->end; + + if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) { + Py_UCS4 badchar = PyUnicode_ReadChar(exc->object, start); const char *fmt; - if (badchar <= 0xff) + if (badchar <= 0xff) { fmt = "can't translate character '\\x%02x' in position %zd: %U"; - else if (badchar <= 0xffff) + } + else if (badchar <= 0xffff) { fmt = "can't translate character '\\u%04x' in position %zd: %U"; - else + } + else { fmt = "can't translate character '\\U%08x' in position %zd: %U"; + } result = PyUnicode_FromFormat( fmt, (int)badchar, - uself->start, - reason_str - ); - } else { + start, + reason_str); + } + else { result = PyUnicode_FromFormat( "can't translate characters in position %zd-%zd: %U", - uself->start, - uself->end-1, - reason_str - ); + start, + end - 1, + reason_str); } done: Py_XDECREF(reason_str); @@ -3244,7 +3308,7 @@ SimpleExtendsException(PyExc_Exception, ArithmeticError, * FloatingPointError extends ArithmeticError */ SimpleExtendsException(PyExc_ArithmeticError, FloatingPointError, - "Floating point operation failed."); + "Floating-point operation failed."); /* @@ -3344,8 +3408,9 @@ _PyErr_NoMemory(PyThreadState *tstate) } static void -MemoryError_dealloc(PyBaseExceptionObject *self) +MemoryError_dealloc(PyObject *obj) { + PyBaseExceptionObject *self = (PyBaseExceptionObject *)obj; _PyObject_GC_UNTRACK(self); BaseException_clear(self); @@ -3404,7 +3469,7 @@ PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), - 0, (destructor)MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, + 0, MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, PyDoc_STR("Out of memory."), (traverseproc)BaseException_traverse, @@ -3539,7 +3604,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning, #undef EOPNOTSUPP #undef EPROTONOSUPPORT #undef EPROTOTYPE -#undef ETIMEDOUT #undef EWOULDBLOCK #if defined(WSAEALREADY) && !defined(EALREADY) @@ -3560,9 +3624,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning, #if defined(WSAESHUTDOWN) && !defined(ESHUTDOWN) #define ESHUTDOWN WSAESHUTDOWN #endif -#if defined(WSAETIMEDOUT) && !defined(ETIMEDOUT) -#define ETIMEDOUT WSAETIMEDOUT -#endif #if defined(WSAEWOULDBLOCK) && !defined(EWOULDBLOCK) #define EWOULDBLOCK WSAEWOULDBLOCK #endif @@ -3640,11 +3701,12 @@ static struct static_exception static_exceptions[] = { // Level 4: Other subclasses ITEM(IndentationError), // base: SyntaxError(Exception) - ITEM(IncompleteInputError), // base: SyntaxError(Exception) + {&_PyExc_IncompleteInputError, "_IncompleteInputError"}, // base: SyntaxError(Exception) ITEM(IndexError), // base: LookupError(Exception) ITEM(KeyError), // base: LookupError(Exception) ITEM(ModuleNotFoundError), // base: ImportError(Exception) ITEM(NotImplementedError), // base: RuntimeError(Exception) + ITEM(PythonFinalizationError), // base: RuntimeError(Exception) ITEM(RecursionError), // base: RuntimeError(Exception) ITEM(UnboundLocalError), // base: NameError(Exception) ITEM(UnicodeError), // base: ValueError(Exception) @@ -3674,6 +3736,11 @@ _PyExc_InitTypes(PyInterpreterState *interp) if (_PyStaticType_InitBuiltin(interp, exc) < 0) { return -1; } + if (exc->tp_new == BaseException_new + && exc->tp_init == (initproc)BaseException_init) + { + exc->tp_vectorcall = BaseException_vectorcall; + } } return 0; } @@ -3684,7 +3751,7 @@ _PyExc_FiniTypes(PyInterpreterState *interp) { for (Py_ssize_t i=Py_ARRAY_LENGTH(static_exceptions) - 1; i >= 0; i--) { PyTypeObject *exc = static_exceptions[i].exc; - _PyStaticType_Dealloc(interp, exc); + _PyStaticType_FiniBuiltin(interp, exc); } } @@ -3747,6 +3814,9 @@ _PyExc_InitState(PyInterpreterState *interp) #endif ADD_ERRNO(ProcessLookupError, ESRCH); ADD_ERRNO(TimeoutError, ETIMEDOUT); +#ifdef WSAETIMEDOUT + ADD_ERRNO(TimeoutError, WSAETIMEDOUT); +#endif return _PyStatus_OK(); diff --git a/Objects/fileobject.c b/Objects/fileobject.c index e30ab952dff571..c377d1bb28b56f 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -80,13 +80,7 @@ PyFile_GetLine(PyObject *f, int n) "EOF when reading a line"); } else if (s[len-1] == '\n') { - if (Py_REFCNT(result) == 1) - _PyBytes_Resize(&result, len-1); - else { - PyObject *v; - v = PyBytes_FromStringAndSize(s, len-1); - Py_SETREF(result, v); - } + (void) _PyBytes_Resize(&result, len-1); } } if (n < 0 && result != NULL && PyUnicode_Check(result)) { @@ -468,7 +462,7 @@ PyTypeObject PyStdPrinter_Type = { 0, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ 0, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 9b322c52d4daea..7e14a8ad959590 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -7,8 +7,8 @@ #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_dtoa.h" // _Py_dg_dtoa() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_interp.h" // _PyInterpreterState.float_state #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats() @@ -26,16 +26,6 @@ class float "PyObject *" "&PyFloat_Type" #include "clinic/floatobject.c.h" -#ifdef WITH_FREELISTS -static struct _Py_float_state * -get_float_state(void) -{ - _PyFreeListState *state = _PyFreeListState_GET(); - assert(state != NULL); - return &state->floats; -} -#endif - double PyFloat_GetMax(void) @@ -98,10 +88,18 @@ PyFloat_GetInfo(void) return NULL; } -#define SetIntFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyLong_FromLong(flag)) -#define SetDblFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyFloat_FromDouble(flag)) +#define SetFlag(CALL) \ + do { \ + PyObject *flag = (CALL); \ + if (flag == NULL) { \ + Py_CLEAR(floatinfo); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(floatinfo, pos++, flag); \ + } while (0) + +#define SetIntFlag(FLAG) SetFlag(PyLong_FromLong((FLAG))) +#define SetDblFlag(FLAG) SetFlag(PyFloat_FromDouble((FLAG))) SetDblFlag(DBL_MAX); SetIntFlag(DBL_MAX_EXP); @@ -116,39 +114,61 @@ PyFloat_GetInfo(void) SetIntFlag(FLT_ROUNDS); #undef SetIntFlag #undef SetDblFlag +#undef SetFlag - if (PyErr_Occurred()) { - Py_CLEAR(floatinfo); - return NULL; - } return floatinfo; } PyObject * PyFloat_FromDouble(double fval) { - PyFloatObject *op; -#ifdef WITH_FREELISTS - struct _Py_float_state *state = get_float_state(); - op = state->free_list; - if (op != NULL) { - state->free_list = (PyFloatObject *) Py_TYPE(op); - state->numfree--; - OBJECT_STAT_INC(from_freelist); - } - else -#endif - { + PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats); + if (op == NULL) { op = PyObject_Malloc(sizeof(PyFloatObject)); if (!op) { return PyErr_NoMemory(); } + _PyObject_Init((PyObject*)op, &PyFloat_Type); } - _PyObject_Init((PyObject*)op, &PyFloat_Type); op->ob_fval = fval; return (PyObject *) op; } +#ifdef Py_GIL_DISABLED + +PyObject *_PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value) +{ + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + return PyFloat_FromDouble(value); +} + +#else // Py_GIL_DISABLED + +PyObject *_PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value) +{ + PyObject *left_o = PyStackRef_AsPyObjectSteal(left); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + if (Py_REFCNT(left_o) == 1) { + ((PyFloatObject *)left_o)->ob_fval = value; + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + return left_o; + } + else if (Py_REFCNT(right_o) == 1) { + ((PyFloatObject *)right_o)->ob_fval = value; + _Py_DECREF_NO_DEALLOC(left_o); + return right_o; + } + else { + PyObject *result = PyFloat_FromDouble(value); + _Py_DECREF_NO_DEALLOC(left_o); + _Py_DECREF_NO_DEALLOC(right_o); + return result; + } +} + +#endif // Py_GIL_DISABLED + static PyObject * float_from_string_inner(const char *s, Py_ssize_t len, void *obj) { @@ -243,35 +263,17 @@ void _PyFloat_ExactDealloc(PyObject *obj) { assert(PyFloat_CheckExact(obj)); - PyFloatObject *op = (PyFloatObject *)obj; -#ifdef WITH_FREELISTS - struct _Py_float_state *state = get_float_state(); - if (state->numfree >= PyFloat_MAXFREELIST || state->numfree < 0) { - PyObject_Free(op); - return; - } - state->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)state->free_list); - state->free_list = op; - OBJECT_STAT_INC(to_freelist); -#else - PyObject_Free(op); -#endif + _Py_FREELIST_FREE(floats, obj, PyObject_Free); } static void float_dealloc(PyObject *op) { assert(PyFloat_Check(op)); -#ifdef WITH_FREELISTS - if (PyFloat_CheckExact(op)) { + if (PyFloat_CheckExact(op)) _PyFloat_ExactDealloc(op); - } else -#endif - { Py_TYPE(op)->tp_free(op); - } } double @@ -413,7 +415,7 @@ float_richcompare(PyObject *v, PyObject *w, int op) if (PyFloat_Check(w)) j = PyFloat_AS_DOUBLE(w); - else if (!Py_IS_FINITE(i)) { + else if (!isfinite(i)) { if (PyLong_Check(w)) /* If i is an infinity, its magnitude exceeds any * finite integer, so it doesn't matter which int we @@ -427,7 +429,6 @@ float_richcompare(PyObject *v, PyObject *w, int op) else if (PyLong_Check(w)) { int vsign = i == 0.0 ? 0 : i < 0.0 ? -1 : 1; int wsign = _PyLong_Sign(w); - size_t nbits; int exponent; if (vsign != wsign) { @@ -440,20 +441,22 @@ float_richcompare(PyObject *v, PyObject *w, int op) } /* The signs are the same. */ /* Convert w to a double if it fits. In particular, 0 fits. */ - nbits = _PyLong_NumBits(w); - if (nbits == (size_t)-1 && PyErr_Occurred()) { - /* This long is so large that size_t isn't big enough - * to hold the # of bits. Replace with little doubles + int64_t nbits64 = _PyLong_NumBits(w); + assert(nbits64 >= 0); + assert(!PyErr_Occurred()); + if (nbits64 > DBL_MAX_EXP) { + /* This Python integer is larger than any finite C double. + * Replace with little doubles * that give the same outcome -- w is so large that * its magnitude must exceed the magnitude of any * finite float. */ - PyErr_Clear(); i = (double)vsign; assert(wsign != 0); j = wsign * 2.0; goto Compare; } + int nbits = (int)nbits64; if (nbits <= 48) { j = PyLong_AsDouble(w); /* It's impossible that <= 48 bits overflowed. */ @@ -477,12 +480,12 @@ float_richcompare(PyObject *v, PyObject *w, int op) /* exponent is the # of bits in v before the radix point; * we know that nbits (the # of bits in w) > 48 at this point */ - if (exponent < 0 || (size_t)exponent < nbits) { + if (exponent < nbits) { i = 1.0; j = 2.0; goto Compare; } - if ((size_t)exponent > nbits) { + if (exponent > nbits) { i = 2.0; j = 1.0; goto Compare; @@ -618,7 +621,7 @@ float_div(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(w, b); if (b == 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "float division by zero"); + "division by zero"); return NULL; } a = a / b; @@ -634,7 +637,7 @@ float_rem(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "float modulo by zero"); + "division by zero"); return NULL; } mod = fmod(vx, wx); @@ -699,7 +702,7 @@ float_divmod(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(v, vx); CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { - PyErr_SetString(PyExc_ZeroDivisionError, "float divmod()"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } _float_div_mod(vx, wx, &floordiv, &mod); @@ -714,7 +717,7 @@ float_floor_div(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(v, vx); CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { - PyErr_SetString(PyExc_ZeroDivisionError, "float floor division by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } _float_div_mod(vx, wx, &floordiv, &mod); @@ -744,13 +747,13 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) if (iw == 0) { /* v**0 is 1, even 0**0 */ return PyFloat_FromDouble(1.0); } - if (Py_IS_NAN(iv)) { /* nan**w = nan, unless w == 0 */ + if (isnan(iv)) { /* nan**w = nan, unless w == 0 */ return PyFloat_FromDouble(iv); } - if (Py_IS_NAN(iw)) { /* v**nan = nan, unless v == 1; 1**nan = 1 */ + if (isnan(iw)) { /* v**nan = nan, unless v == 1; 1**nan = 1 */ return PyFloat_FromDouble(iv == 1.0 ? 1.0 : iw); } - if (Py_IS_INFINITY(iw)) { + if (isinf(iw)) { /* v**inf is: 0.0 if abs(v) < 1; 1.0 if abs(v) == 1; inf if * abs(v) > 1 (including case where v infinite) * @@ -765,7 +768,7 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) else return PyFloat_FromDouble(0.0); } - if (Py_IS_INFINITY(iv)) { + if (isinf(iv)) { /* (+-inf)**w is: inf for w positive, 0 for w negative; in * both cases, we need to add the appropriate sign if w is * an odd integer. @@ -783,8 +786,7 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) int iw_is_odd = DOUBLE_IS_ODD_INTEGER(iw); if (iw < 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "0.0 cannot be raised to a " - "negative power"); + "zero to a negative power"); return NULL; } /* use correct sign if iw is odd */ @@ -880,7 +882,7 @@ float_is_integer_impl(PyObject *self) if (x == -1.0 && PyErr_Occurred()) return NULL; - if (!Py_IS_FINITE(x)) + if (!isfinite(x)) Py_RETURN_FALSE; errno = 0; o = (floor(x) == x) ? Py_True : Py_False; @@ -1016,7 +1018,7 @@ double_round(double x, int ndigits) { } y = (x*pow1)*pow2; /* if y overflows, then rounded value is exactly x */ - if (!Py_IS_FINITE(y)) + if (!isfinite(y)) return PyFloat_FromDouble(x); } else { @@ -1036,7 +1038,7 @@ double_round(double x, int ndigits) { z *= pow1; /* if computation resulted in overflow, raise OverflowError */ - if (!Py_IS_FINITE(z)) { + if (!isfinite(z)) { PyErr_SetString(PyExc_OverflowError, "overflow occurred during round"); return NULL; @@ -1084,7 +1086,7 @@ float___round___impl(PyObject *self, PyObject *o_ndigits) return NULL; /* nans and infinities round to themselves */ - if (!Py_IS_FINITE(x)) + if (!isfinite(x)) return PyFloat_FromDouble(x); /* Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x @@ -1138,69 +1140,39 @@ char_from_hex(int x) return Py_hexdigits[x]; } +/* This table maps characters to their hexadecimal values, only + * works with encodings whose lower half is ASCII (like UTF-8). + * '0' maps to 0, ..., '9' maps to 9. + * 'a' and 'A' map to 10, ..., 'f' and 'F' map to 15. + * All other indices map to -1. + */ +static const int +_CHAR_TO_HEX[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +}; + +/* Convert a character to its hexadecimal value, or -1 if it's not a + * valid hexadecimal character, only works with encodings whose lower + * half is ASCII (like UTF-8). + */ static int -hex_from_char(char c) { - int x; - switch(c) { - case '0': - x = 0; - break; - case '1': - x = 1; - break; - case '2': - x = 2; - break; - case '3': - x = 3; - break; - case '4': - x = 4; - break; - case '5': - x = 5; - break; - case '6': - x = 6; - break; - case '7': - x = 7; - break; - case '8': - x = 8; - break; - case '9': - x = 9; - break; - case 'a': - case 'A': - x = 10; - break; - case 'b': - case 'B': - x = 11; - break; - case 'c': - case 'C': - x = 12; - break; - case 'd': - case 'D': - x = 13; - break; - case 'e': - case 'E': - x = 14; - break; - case 'f': - case 'F': - x = 15; - break; - default: - x = -1; - break; - } - return x; +hex_from_char(unsigned char c) { + return _CHAR_TO_HEX[c]; } /* convert a float to a hexadecimal string */ @@ -1232,7 +1204,7 @@ float_hex_impl(PyObject *self) CONVERT_TO_DOUBLE(self, x); - if (Py_IS_NAN(x) || Py_IS_INFINITY(x)) + if (isnan(x) || isinf(x)) return float_repr((PyFloatObject *)self); if (x == 0.0) { @@ -1565,12 +1537,12 @@ float_as_integer_ratio_impl(PyObject *self) CONVERT_TO_DOUBLE(self, self_double); - if (Py_IS_INFINITY(self_double)) { + if (isinf(self_double)) { PyErr_SetString(PyExc_OverflowError, "cannot convert Infinity to integer ratio"); return NULL; } - if (Py_IS_NAN(self_double)) { + if (isnan(self_double)) { PyErr_SetString(PyExc_ValueError, "cannot convert NaN to integer ratio"); return NULL; @@ -1628,12 +1600,12 @@ float.__new__ as float_new x: object(c_default="NULL") = 0 / -Convert a string or number to a floating point number, if possible. +Convert a string or number to a floating-point number, if possible. [clinic start generated code]*/ static PyObject * float_new_impl(PyTypeObject *type, PyObject *x) -/*[clinic end generated code: output=ccf1e8dc460ba6ba input=f43661b7de03e9d8]*/ +/*[clinic end generated code: output=ccf1e8dc460ba6ba input=55909f888aa0c8a6]*/ { if (type != &PyFloat_Type) { if (x == NULL) { @@ -1695,6 +1667,36 @@ float_vectorcall(PyObject *type, PyObject * const*args, } +/*[clinic input] +@classmethod +float.from_number + + number: object + / + +Convert real number to a floating-point number. +[clinic start generated code]*/ + +static PyObject * +float_from_number(PyTypeObject *type, PyObject *number) +/*[clinic end generated code: output=bbcf05529fe907a3 input=1f8424d9bc11866a]*/ +{ + if (PyFloat_CheckExact(number) && type == &PyFloat_Type) { + Py_INCREF(number); + return number; + } + double x = PyFloat_AsDouble(number); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + PyObject *result = PyFloat_FromDouble(x); + if (type != &PyFloat_Type && result != NULL) { + Py_SETREF(result, PyObject_CallOneArg((PyObject *)type, result)); + } + return result; +} + + /*[clinic input] float.__getnewargs__ [clinic start generated code]*/ @@ -1729,13 +1731,13 @@ You probably don't want to use this function. It exists mainly to be used in Python's test suite. This function returns whichever of 'unknown', 'IEEE, big-endian' or 'IEEE, -little-endian' best describes the format of floating point numbers used by the +little-endian' best describes the format of floating-point numbers used by the C type named by typestr. [clinic start generated code]*/ static PyObject * float___getformat___impl(PyTypeObject *type, const char *typestr) -/*[clinic end generated code: output=2bfb987228cc9628 input=d5a52600f835ad67]*/ +/*[clinic end generated code: output=2bfb987228cc9628 input=90d5e246409a246e]*/ { float_format_type r; @@ -1808,6 +1810,7 @@ float___format___impl(PyObject *self, PyObject *format_spec) } static PyMethodDef float_methods[] = { + FLOAT_FROM_NUMBER_METHODDEF FLOAT_CONJUGATE_METHODDEF FLOAT___TRUNC___METHODDEF FLOAT___FLOOR___METHODDEF @@ -1921,7 +1924,7 @@ _init_global_state(void) float_format_type detected_double_format, detected_float_format; /* We attempt to determine if this machine is using IEEE - floating point formats by peering at the bits of some + floating-point formats by peering at the bits of some carefully chosen values. If it looks like we are on an IEEE platform, the float packing/unpacking routines can just copy bits, if not they resort to arithmetic & shifts @@ -1989,27 +1992,6 @@ _PyFloat_InitTypes(PyInterpreterState *interp) return _PyStatus_OK(); } -void -_PyFloat_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_float_state *state = &freelist_state->floats; - PyFloatObject *f = state->free_list; - while (f != NULL) { - PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); - PyObject_Free(f); - f = next; - } - state->free_list = NULL; - if (is_finalization) { - state->numfree = -1; - } - else { - state->numfree = 0; - } -#endif -} - void _PyFloat_FiniType(PyInterpreterState *interp) { @@ -2020,12 +2002,10 @@ _PyFloat_FiniType(PyInterpreterState *interp) void _PyFloat_DebugMallocStats(FILE *out) { -#ifdef WITH_FREELISTS - struct _Py_float_state *state = get_float_state(); _PyDebugAllocatorStats(out, "free PyFloatObject", - state->numfree, sizeof(PyFloatObject)); -#endif + _Py_FREELIST_SIZE(floats), + sizeof(PyFloatObject)); } @@ -2055,12 +2035,12 @@ PyFloat_Pack2(double x, char *data, int le) e = 0; bits = 0; } - else if (Py_IS_INFINITY(x)) { + else if (isinf(x)) { sign = (x < 0.0); e = 0x1f; bits = 0; } - else if (Py_IS_NAN(x)) { + else if (isnan(x)) { /* There are 2046 distinct half-precision NaNs (1022 signaling and 1024 quiet), but there are only two quiet NaNs that don't arise by quieting a signaling NaN; we get those by setting the topmost bit @@ -2229,7 +2209,7 @@ PyFloat_Pack4(double x, char *data, int le) float y = (float)x; int i, incr = 1; - if (Py_IS_INFINITY(y) && !Py_IS_INFINITY(x)) + if (isinf(y) && !isinf(x)) goto Overflow; unsigned char s[sizeof(float)]; @@ -2410,7 +2390,7 @@ PyFloat_Unpack2(const char *data, int le) if (e == 0x1f) { if (f == 0) { /* Infinity */ - return sign ? -Py_HUGE_VAL : Py_HUGE_VAL; + return sign ? -Py_INFINITY : Py_INFINITY; } else { /* NaN */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index a914c61aac2fd5..55394afa523213 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1,10 +1,12 @@ /* Frame object implementation */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() +#include "pycore_ceval.h" // _PyEval_SetOpcodeTrace() #include "pycore_code.h" // CO_FAST_LOCAL, etc. +#include "pycore_dict.h" // _PyDict_LoadBuiltinsFromGlobals() #include "pycore_function.h" // _PyFunction_FromConstructor() #include "pycore_moduleobject.h" // _PyModule_GetDict() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches @@ -16,12 +18,841 @@ #define OFF(x) offsetof(PyFrameObject, x) + +// Returns borrowed reference or NULL +static PyObject * +framelocalsproxy_getval(_PyInterpreterFrame *frame, PyCodeObject *co, int i) +{ + _PyStackRef *fast = _PyFrame_GetLocalsArray(frame); + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *value = PyStackRef_AsPyObjectBorrow(fast[i]); + PyObject *cell = NULL; + + if (value == NULL) { + return NULL; + } + + if (kind == CO_FAST_FREE || kind & CO_FAST_CELL) { + // The cell was set when the frame was created from + // the function's closure. + assert(PyCell_Check(value)); + cell = value; + } + + if (cell != NULL) { + value = PyCell_GET(cell); + } + + if (value == NULL) { + return NULL; + } + + return value; +} + +static int +framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) +{ + /* + * Returns -2 (!) if an error occurred; exception will be set. + * Returns the fast locals index of the key on success: + * - if read == true, returns the index if the value is not NULL + * - if read == false, returns the index if the value is not hidden + * Otherwise returns -1. + */ + + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + + // Ensure that the key is hashable. + Py_hash_t key_hash = PyObject_Hash(key); + if (key_hash == -1) { + return -2; + } + bool found = false; + + // We do 2 loops here because it's highly possible the key is interned + // and we can do a pointer comparison. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (name == key) { + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + found = true; + } + } + if (found) { + // This is an attempt to read an unset local variable or + // write to a variable that is hidden from regular write operations + return -1; + } + // This is unlikely, but we need to make sure. This means the key + // is not interned. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + Py_hash_t name_hash = PyObject_Hash(name); + assert(name_hash != -1); // keys are exact unicode + if (name_hash != key_hash) { + continue; + } + int same = PyObject_RichCompareBool(name, key, Py_EQ); + if (same < 0) { + return -2; + } + if (same) { + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + + return -1; +} + +static PyObject * +framelocalsproxy_getitem(PyObject *self, PyObject *key) +{ + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); + + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i == -2) { + return NULL; + } + if (i >= 0) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + assert(value != NULL); + return Py_NewRef(value); + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + if (extra != NULL) { + PyObject *value = PyDict_GetItem(extra, key); + if (value != NULL) { + return Py_NewRef(value); + } + } + + PyErr_Format(PyExc_KeyError, "local variable '%R' is not defined", key); + return NULL; +} + +static int +framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) +{ + /* Merge locals into fast locals */ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + _PyStackRef *fast = _PyFrame_GetLocalsArray(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + + int i = framelocalsproxy_getkeyindex(frame, key, false); + if (i == -2) { + return -1; + } + if (i >= 0) { + if (value == NULL) { + PyErr_SetString(PyExc_ValueError, "cannot remove local variables from FrameLocalsProxy"); + return -1; + } + + _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); + + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + _PyStackRef oldvalue = fast[i]; + PyObject *cell = NULL; + if (kind == CO_FAST_FREE) { + // The cell was set when the frame was created from + // the function's closure. + assert(oldvalue.bits != 0 && PyCell_Check(PyStackRef_AsPyObjectBorrow(oldvalue))); + cell = PyStackRef_AsPyObjectBorrow(oldvalue); + } else if (kind & CO_FAST_CELL && oldvalue.bits != 0) { + PyObject *as_obj = PyStackRef_AsPyObjectBorrow(oldvalue); + if (PyCell_Check(as_obj)) { + cell = as_obj; + } + } + if (cell != NULL) { + PyObject *oldvalue_o = PyCell_GET(cell); + if (value != oldvalue_o) { + PyCell_SET(cell, Py_XNewRef(value)); + Py_XDECREF(oldvalue_o); + } + } else if (value != PyStackRef_AsPyObjectBorrow(oldvalue)) { + PyStackRef_XCLOSE(fast[i]); + fast[i] = PyStackRef_FromPyObjectNew(value); + } + return 0; + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + + if (extra == NULL) { + if (value == NULL) { + _PyErr_SetKeyError(key); + return -1; + } + extra = PyDict_New(); + if (extra == NULL) { + return -1; + } + frame->f_extra_locals = extra; + } + + assert(PyDict_Check(extra)); + + if (value == NULL) { + return PyDict_DelItem(extra, key); + } else { + return PyDict_SetItem(extra, key, value); + } +} + +static int +framelocalsproxy_merge(PyObject* self, PyObject* other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + return -1; + } + + PyObject *keys = PyMapping_Keys(other); + if (keys == NULL) { + return -1; + } + + PyObject *iter = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iter == NULL) { + return -1; + } + + PyObject *key = NULL; + PyObject *value = NULL; + + while ((key = PyIter_Next(iter)) != NULL) { + value = PyObject_GetItem(other, key); + if (value == NULL) { + Py_DECREF(key); + Py_DECREF(iter); + return -1; + } + + if (framelocalsproxy_setitem(self, key, value) < 0) { + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(iter); + return -1; + } + + Py_DECREF(key); + Py_DECREF(value); + } + + Py_DECREF(iter); + + return 0; +} + +static PyObject * +framelocalsproxy_keys(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *names = PyList_New(0); + if (names == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *val = framelocalsproxy_getval(frame->f_frame, co, i); + if (val) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (PyList_Append(names, name) < 0) { + Py_DECREF(names); + return NULL; + } + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + assert(PyDict_Check(frame->f_extra_locals)); + + Py_ssize_t i = 0; + PyObject *key = NULL; + PyObject *value = NULL; + + while (PyDict_Next(frame->f_extra_locals, &i, &key, &value)) { + if (PyList_Append(names, key) < 0) { + Py_DECREF(names); + return NULL; + } + } + } + + return names; +} + +static void +framelocalsproxy_dealloc(PyObject *self) +{ + PyObject_GC_UnTrack(self); + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (PyTuple_GET_SIZE(args) != 1) { + PyErr_Format(PyExc_TypeError, + "FrameLocalsProxy expected 1 argument, got %zd", + PyTuple_GET_SIZE(args)); + return NULL; + } + PyObject *item = PyTuple_GET_ITEM(args, 0); + + if (!PyFrame_Check(item)) { + PyErr_Format(PyExc_TypeError, "expect frame, not %T", item); + return NULL; + } + PyFrameObject *frame = (PyFrameObject*)item; + + if (kwds != NULL && PyDict_Size(kwds) != 0) { + PyErr_SetString(PyExc_TypeError, + "FrameLocalsProxy takes no keyword arguments"); + return 0; + } + + PyFrameLocalsProxyObject *self = (PyFrameLocalsProxyObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + + ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)Py_NewRef(frame); + + return (PyObject *)self; +} + +static int +framelocalsproxy_tp_clear(PyObject *self) +{ + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static int +framelocalsproxy_visit(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static PyObject * +framelocalsproxy_iter(PyObject *self) +{ + PyObject* keys = framelocalsproxy_keys(self, NULL); + if (keys == NULL) { + return NULL; + } + + PyObject* iter = PyObject_GetIter(keys); + Py_XDECREF(keys); + + return iter; +} + +static PyObject * +framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) +{ + if (PyFrameLocalsProxy_Check(other)) { + bool result = ((PyFrameLocalsProxyObject*)self)->frame == ((PyFrameLocalsProxyObject*)other)->frame; + if (op == Py_EQ) { + return PyBool_FromLong(result); + } else if (op == Py_NE) { + return PyBool_FromLong(!result); + } + } else if (PyDict_Check(other)) { + PyObject *dct = PyDict_New(); + if (dct == NULL) { + return NULL; + } + + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + return NULL; + } + + PyObject *result = PyObject_RichCompare(dct, other, op); + Py_DECREF(dct); + return result; + } + + Py_RETURN_NOTIMPLEMENTED; +} + +static PyObject * +framelocalsproxy_repr(PyObject *self) +{ + int i = Py_ReprEnter(self); + if (i != 0) { + return i > 0 ? PyUnicode_FromString("{...}") : NULL; + } + + PyObject *dct = PyDict_New(); + if (dct == NULL) { + Py_ReprLeave(self); + return NULL; + } + + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + Py_ReprLeave(self); + return NULL; + } + + PyObject *repr = PyObject_Repr(dct); + Py_DECREF(dct); + + Py_ReprLeave(self); + + return repr; +} + +static PyObject* +framelocalsproxy_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + PyObject *result = PyDict_New(); + if (result == NULL) { + return NULL; + } + + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + if (PyDict_Update(result, other) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_inplace_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + if (framelocalsproxy_merge(self, other) < 0) { + Py_RETURN_NOTIMPLEMENTED; + } + + return Py_NewRef(self); +} + +static PyObject* +framelocalsproxy_values(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *values = PyList_New(0); + if (values == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + if (value) { + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } + } + } + + return values; +} + +static PyObject * +framelocalsproxy_items(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *items = PyList_New(0); + if (items == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + + if (value) { + PyObject *pair = PyTuple_Pack(2, name, value); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + + Py_DECREF(pair); + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyObject *pair = PyTuple_Pack(2, key, value); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + + Py_DECREF(pair); + } + } + + return items; +} + +static Py_ssize_t +framelocalsproxy_length(PyObject *self) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + Py_ssize_t size = 0; + + if (frame->f_extra_locals != NULL) { + assert(PyDict_Check(frame->f_extra_locals)); + size += PyDict_Size(frame->f_extra_locals); + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + size++; + } + } + return size; +} + +static int +framelocalsproxy_contains(PyObject *self, PyObject *key) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i == -2) { + return -1; + } + if (i >= 0) { + return 1; + } + + PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; + if (extra != NULL) { + return PyDict_Contains(extra, key); + } + + return 0; +} + +static PyObject* framelocalsproxy___contains__(PyObject *self, PyObject *key) +{ + int result = framelocalsproxy_contains(self, key); + if (result < 0) { + return NULL; + } + return PyBool_FromLong(result); +} + +static PyObject* +framelocalsproxy_update(PyObject *self, PyObject *other) +{ + if (framelocalsproxy_merge(self, other) < 0) { + PyErr_SetString(PyExc_TypeError, "update() argument must be dict or another FrameLocalsProxy"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* +framelocalsproxy_get(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "get expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "setdefault expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + if (framelocalsproxy_setitem(self, key, default_value) < 0) { + return NULL; + } + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_pop(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (!_PyArg_CheckPositional("pop", nargs, 1, 2)) { + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = NULL; + + if (nargs == 2) { + default_value = args[1]; + } + + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + + int i = framelocalsproxy_getkeyindex(frame, key, false); + if (i == -2) { + return NULL; + } + + if (i >= 0) { + PyErr_SetString(PyExc_ValueError, "cannot remove local variables from FrameLocalsProxy"); + return NULL; + } + + PyObject *result = NULL; + + if (frame->f_extra_locals == NULL) { + if (default_value != NULL) { + return Py_XNewRef(default_value); + } else { + _PyErr_SetKeyError(key); + return NULL; + } + } + + if (PyDict_Pop(frame->f_extra_locals, key, &result) < 0) { + return NULL; + } + + if (result == NULL) { + if (default_value != NULL) { + return Py_XNewRef(default_value); + } else { + _PyErr_SetKeyError(key); + return NULL; + } + } + + return result; +} + +static PyObject* +framelocalsproxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject* result = PyDict_New(); + + if (result == NULL) { + return NULL; + } + + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_reversed(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyObject *result = framelocalsproxy_keys(self, NULL); + + if (result == NULL) { + return NULL; + } + + if (PyList_Reverse(result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + +static PyNumberMethods framelocalsproxy_as_number = { + .nb_or = framelocalsproxy_or, + .nb_inplace_or = framelocalsproxy_inplace_or, +}; + +static PySequenceMethods framelocalsproxy_as_sequence = { + .sq_contains = framelocalsproxy_contains, +}; + +static PyMappingMethods framelocalsproxy_as_mapping = { + framelocalsproxy_length, // mp_length + framelocalsproxy_getitem, // mp_subscript + framelocalsproxy_setitem, // mp_ass_subscript +}; + +static PyMethodDef framelocalsproxy_methods[] = { + {"__contains__", framelocalsproxy___contains__, METH_O | METH_COEXIST, + NULL}, + {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, + NULL}, + {"update", framelocalsproxy_update, METH_O, + NULL}, + {"__reversed__", _PyCFunction_CAST(framelocalsproxy_reversed), METH_NOARGS, + NULL}, + {"copy", _PyCFunction_CAST(framelocalsproxy_copy), METH_NOARGS, + NULL}, + {"keys", _PyCFunction_CAST(framelocalsproxy_keys), METH_NOARGS, + NULL}, + {"values", _PyCFunction_CAST(framelocalsproxy_values), METH_NOARGS, + NULL}, + {"items", _PyCFunction_CAST(framelocalsproxy_items), METH_NOARGS, + NULL}, + {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, + NULL}, + {"pop", _PyCFunction_CAST(framelocalsproxy_pop), METH_FASTCALL, + NULL}, + {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, + NULL}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyFrameLocalsProxy_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "FrameLocalsProxy", + .tp_basicsize = sizeof(PyFrameLocalsProxyObject), + .tp_dealloc = (destructor)framelocalsproxy_dealloc, + .tp_repr = &framelocalsproxy_repr, + .tp_as_number = &framelocalsproxy_as_number, + .tp_as_sequence = &framelocalsproxy_as_sequence, + .tp_as_mapping = &framelocalsproxy_as_mapping, + .tp_getattro = PyObject_GenericGetAttr, + .tp_setattro = PyObject_GenericSetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MAPPING, + .tp_traverse = framelocalsproxy_visit, + .tp_clear = framelocalsproxy_tp_clear, + .tp_richcompare = framelocalsproxy_richcompare, + .tp_iter = framelocalsproxy_iter, + .tp_methods = framelocalsproxy_methods, + .tp_alloc = PyType_GenericAlloc, + .tp_new = framelocalsproxy_new, + .tp_free = PyObject_GC_Del, +}; + +PyObject * +_PyFrameLocalsProxy_New(PyFrameObject *frame) +{ + PyObject* args = PyTuple_Pack(1, frame); + if (args == NULL) { + return NULL; + } + + PyObject* proxy = (PyObject*)framelocalsproxy_new(&PyFrameLocalsProxy_Type, args, NULL); + Py_DECREF(args); + return proxy; +} + static PyMemberDef frame_memberlist[] = { {"f_trace_lines", Py_T_BOOL, OFF(f_trace_lines), 0}, {NULL} /* Sentinel */ }; - static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { @@ -30,23 +861,43 @@ frame_getlocals(PyFrameObject *f, void *closure) return NULL; } assert(!_PyFrame_IsIncomplete(f->f_frame)); - PyObject *locals = _PyFrame_GetLocals(f->f_frame, 1); - if (locals) { - f->f_fast_as_locals = 1; + + PyCodeObject *co = _PyFrame_GetCode(f->f_frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(f->f_frame)) { + if (f->f_frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + f->f_frame->f_locals = PyDict_New(); + if (f->f_frame->f_locals == NULL) { + return NULL; + } + } + return Py_NewRef(f->f_frame->f_locals); } - return locals; + + return _PyFrameLocalsProxy_New(f); } int PyFrame_GetLineNumber(PyFrameObject *f) { assert(f != NULL); - if (f->f_lineno != 0) { - return f->f_lineno; + if (f->f_lineno == -1) { + // We should calculate it once. If we can't get the line number, + // set f->f_lineno to 0. + f->f_lineno = PyUnstable_InterpreterFrame_GetLine(f->f_frame); + if (f->f_lineno < 0) { + f->f_lineno = 0; + return -1; + } } - else { - return PyUnstable_InterpreterFrame_GetLine(f->f_frame); + + if (f->f_lineno > 0) { + return f->f_lineno; } + return PyUnstable_InterpreterFrame_GetLine(f->f_frame); } static PyObject * @@ -291,9 +1142,7 @@ mark_stacks(PyCodeObject *code_obj, int len) if (co_code == NULL) { return NULL; } - _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code); int64_t *stacks = PyMem_New(int64_t, len+1); - int i, j, opcode; if (stacks == NULL) { PyErr_NoMemory(); @@ -304,31 +1153,29 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[i] = UNINITIALIZED; } stacks[0] = EMPTY_STACK; - if (code_obj->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) - { - // Generators get sent None while starting: - stacks[0] = push_value(stacks[0], Object); - } int todo = 1; while (todo) { todo = 0; /* Scan instructions */ - for (i = 0; i < len;) { + for (int i = 0; i < len;) { + int j; int64_t next_stack = stacks[i]; - opcode = _Py_GetBaseOpcode(code_obj, i); + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code_obj, i); + int opcode = inst.op.code; int oparg = 0; while (opcode == EXTENDED_ARG) { - oparg = (oparg << 8) | code[i].op.arg; + oparg = (oparg << 8) | inst.op.arg; i++; - opcode = _Py_GetBaseOpcode(code_obj, i); + inst = _Py_GetBaseCodeUnit(code_obj, i); + opcode = inst.op.code; stacks[i] = next_stack; } + oparg = (oparg << 8) | inst.op.arg; int next_i = i + _PyOpcode_Caches[opcode] + 1; if (next_stack == UNINITIALIZED) { i = next_i; continue; } - oparg = (oparg << 8) | code[i].op.arg; switch (opcode) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: @@ -336,7 +1183,7 @@ mark_stacks(PyCodeObject *code_obj, int len) case POP_JUMP_IF_NOT_NONE: { int64_t target_stack; - int j = next_i + oparg; + j = next_i + oparg; assert(j < len); next_stack = pop_value(next_stack); target_stack = next_stack; @@ -401,8 +1248,6 @@ mark_stacks(PyCodeObject *code_obj, int len) assert(pop_value(next_stack) == EMPTY_STACK); assert(top_of_stack(next_stack) == Object); break; - case RETURN_CONST: - break; case RAISE_VARARGS: break; case RERAISE: @@ -592,25 +1437,11 @@ first_line_not_before(int *lines, int len, int line) return result; } -static bool -frame_is_cleared(PyFrameObject *frame) -{ - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - if (frame->f_frame->stacktop == 0) { - return true; - } - if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); - return gen->gi_frame_state == FRAME_CLEARED; - } - return false; -} - static bool frame_is_suspended(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame->f_frame); return FRAME_STATE_SUSPENDED(gen->gi_frame_state); } return false; @@ -783,7 +1614,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore for (int i = 0; i < code->co_nlocalsplus; i++) { // Counting every unbound local is overly-cautious, but a full flow // analysis (like we do in the compiler) is probably too expensive: - unbound += f->f_frame->localsplus[i] == NULL; + unbound += PyStackRef_IsNull(f->f_frame->localsplus[i]); } if (unbound) { const char *e = "assigning None to %d unbound local%s"; @@ -794,8 +1625,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore // Do this in a second pass to avoid writing a bunch of Nones when // warnings are being treated as errors and the previous bit raises: for (int i = 0; i < code->co_nlocalsplus; i++) { - if (f->f_frame->localsplus[i] == NULL) { - f->f_frame->localsplus[i] = Py_NewRef(Py_None); + if (PyStackRef_IsNull(f->f_frame->localsplus[i])) { + f->f_frame->localsplus[i] = PyStackRef_None; unbound--; } } @@ -808,14 +1639,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore while (start_stack > best_stack) { if (top_of_stack(start_stack) == Except) { /* Pop exception stack as well as the evaluation stack */ - PyObject *exc = _PyFrame_StackPop(f->f_frame); + PyObject *exc = PyStackRef_AsPyObjectBorrow(_PyFrame_StackPop(f->f_frame)); assert(PyExceptionInstance_Check(exc) || exc == Py_None); PyThreadState *tstate = _PyThreadState_GET(); Py_XSETREF(tstate->exc_info->exc_value, exc == Py_None ? NULL : exc); } else { - PyObject *v = _PyFrame_StackPop(f->f_frame); - Py_XDECREF(v); + PyStackRef_XCLOSE(_PyFrame_StackPop(f->f_frame)); } start_stack = pop_value(start_stack); } @@ -875,8 +1705,6 @@ frame_dealloc(PyFrameObject *f) } Py_TRASHCAN_BEGIN(f, frame_dealloc); - PyObject *co = NULL; - /* GH-106092: If f->f_frame was on the stack and we reached the maximum * nesting depth for deallocations, the trashcan may have delayed this * deallocation until after f->f_frame is freed. Avoid dereferencing @@ -885,20 +1713,21 @@ frame_dealloc(PyFrameObject *f) /* Kill all local variables including specials, if we own them */ if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { - /* Don't clear code object until the end */ - co = frame->f_executable; - frame->f_executable = NULL; - Py_CLEAR(frame->f_funcobj); + PyStackRef_CLEAR(frame->f_executable); + PyStackRef_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); - PyObject **locals = _PyFrame_GetLocalsArray(frame); - for (int i = 0; i < frame->stacktop; i++) { - Py_CLEAR(locals[i]); + _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); + _PyStackRef *sp = frame->stackpointer; + while (sp > locals) { + sp--; + PyStackRef_CLEAR(*sp); } } Py_CLEAR(f->f_back); Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); + Py_CLEAR(f->f_locals_cache); PyObject_GC_Del(f); - Py_XDECREF(co); Py_TRASHCAN_END; } @@ -907,6 +1736,8 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) { Py_VISIT(f->f_back); Py_VISIT(f->f_trace); + Py_VISIT(f->f_extra_locals); + Py_VISIT(f->f_locals_cache); if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) { return 0; } @@ -918,14 +1749,18 @@ static int frame_tp_clear(PyFrameObject *f) { Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); + Py_CLEAR(f->f_locals_cache); /* locals and stack */ - PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame); - assert(f->f_frame->stacktop >= 0); - for (int i = 0; i < f->f_frame->stacktop; i++) { - Py_CLEAR(locals[i]); - } - f->f_frame->stacktop = 0; + _PyStackRef *locals = _PyFrame_GetLocalsArray(f->f_frame); + _PyStackRef *sp = f->f_frame->stackpointer; + assert(sp >= locals); + while (sp > locals) { + sp--; + PyStackRef_CLEAR(*sp); + } + f->f_frame->stackpointer = locals; Py_CLEAR(f->f_frame->f_locals); return 0; } @@ -934,7 +1769,7 @@ static PyObject * frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(f->f_frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(f->f_frame); if (gen->gi_frame_state == FRAME_EXECUTING) { goto running; } @@ -962,7 +1797,7 @@ frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(clear__doc__, -"F.clear(): clear most references held by the frame"); +"F.clear(): clear all references held by the frame"); static PyObject * frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) @@ -1035,9 +1870,8 @@ static void init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) { PyCodeObject *code = (PyCodeObject *)func->func_code; - _PyFrame_Initialize(frame, (PyFunctionObject*)Py_NewRef(func), - Py_XNewRef(locals), code, 0); - frame->previous = NULL; + _PyFrame_Initialize(frame, PyStackRef_FromPyObjectNew(func), + Py_XNewRef(locals), code, 0, NULL); } PyFrameObject* @@ -1053,8 +1887,9 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace = NULL; f->f_trace_lines = 1; f->f_trace_opcodes = 0; - f->f_fast_as_locals = 0; f->f_lineno = 0; + f->f_extra_locals = NULL; + f->f_locals_cache = NULL; return f; } @@ -1063,7 +1898,7 @@ PyFrameObject* PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) { - PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref + PyObject *builtins = _PyDict_LoadBuiltinsFromGlobals(globals); if (builtins == NULL) { return NULL; } @@ -1078,6 +1913,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, .fc_closure = NULL }; PyFunctionObject *func = _PyFunction_FromConstructor(&desc); + _Py_DECREF_BUILTINS(builtins); if (func == NULL) { return NULL; } @@ -1097,32 +1933,6 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, return f; } -static int -_PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) -{ - // This only works when opcode is a non-quickened form: - assert(_PyOpcode_Deopt[opcode] == opcode); - int check_oparg = 0; - for (_Py_CODEUNIT *instruction = _PyCode_CODE(_PyFrame_GetCode(frame)); - instruction < frame->instr_ptr; instruction++) - { - int check_opcode = _PyOpcode_Deopt[instruction->op.code]; - check_oparg |= instruction->op.arg; - if (check_opcode == opcode && check_oparg == oparg) { - return 1; - } - if (check_opcode == EXTENDED_ARG) { - check_oparg <<= 8; - } - else { - check_oparg = 0; - } - instruction += _PyOpcode_Caches[check_opcode]; - } - return 0; -} - - // Initialize frame free variables if needed static void frame_init_get_vars(_PyInterpreterFrame *frame) @@ -1132,18 +1942,19 @@ frame_init_get_vars(_PyInterpreterFrame *frame) PyCodeObject *co = _PyFrame_GetCode(frame); int lasti = _PyInterpreterFrame_LASTI(frame); if (!(lasti < 0 && _PyCode_CODE(co)->op.code == COPY_FREE_VARS - && PyFunction_Check(frame->f_funcobj))) + && PyStackRef_FunctionCheck(frame->f_funcobj))) { /* Free vars are initialized */ return; } /* Free vars have not been initialized -- Do that */ - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - int offset = PyCode_GetFirstFree(co); + PyFunctionObject *func = _PyFrame_GetFunction(frame); + PyObject *closure = func->func_closure; + int offset = PyUnstable_Code_GetFirstFree(co); for (int i = 0; i < co->co_nfreevars; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } // COPY_FREE_VARS doesn't have inline CACHEs, either: frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)); @@ -1168,136 +1979,79 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, return 0; } - PyObject *value = frame->localsplus[i]; - if (frame->stacktop) { + PyObject *value = NULL; + if (frame->stackpointer == NULL || frame->stackpointer > frame->localsplus + i) { + value = PyStackRef_AsPyObjectBorrow(frame->localsplus[i]); if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. assert(value != NULL && PyCell_Check(value)); value = PyCell_GET(value); } else if (kind & CO_FAST_CELL) { - // Note that no *_DEREF ops can happen before MAKE_CELL - // executes. So there's no need to duplicate the work - // that MAKE_CELL would otherwise do later, if it hasn't - // run yet. if (value != NULL) { - if (PyCell_Check(value) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. + if (PyCell_Check(value)) { + assert(!_PyFrame_IsIncomplete(frame)); value = PyCell_GET(value); } // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), // with the initial value set when the frame was created... - // (unlikely) ...or it was set to some initial value by - // an earlier call to PyFrame_LocalsToFast(). + // (unlikely) ...or it was set via the f_locals proxy. } } } - else { - assert(value == NULL); - } *pvalue = value; return 1; } -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden) +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame) { - /* Merge fast locals into f->f_locals */ - PyObject *locals = frame->f_locals; - if (locals == NULL) { - locals = frame->f_locals = PyDict_New(); - if (locals == NULL) { - return NULL; - } - } - PyObject *hidden = NULL; - - /* If include_hidden, "hidden" fast locals (from inlined comprehensions in - module/class scopes) will be included in the returned dict, but not in - frame->f_locals; the returned dict will be a modified copy. Non-hidden - locals will still be updated in frame->f_locals. */ - if (include_hidden) { - hidden = PyDict_New(); - if (hidden == NULL) { - return NULL; - } - } - - frame_init_get_vars(frame); + /* + * This function returns if there are hidden locals introduced by PEP 709, + * which are the isolated fast locals for inline comprehensions + */ + PyCodeObject* co = _PyFrame_GetCode(frame); - PyCodeObject *co = _PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *value; // borrowed reference - if (!frame_get_var(frame, co, i, &value)) { - continue; - } - - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + if (kind & CO_FAST_HIDDEN) { - if (include_hidden && value != NULL) { - if (PyObject_SetItem(hidden, name, value) != 0) { - goto error; - } - } - continue; - } - if (value == NULL) { - if (PyObject_DelItem(locals, name) != 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Clear(); - } - else { - goto error; - } - } - } - else { - if (PyObject_SetItem(locals, name, value) != 0) { - goto error; - } - } - } + PyObject* value = framelocalsproxy_getval(frame, co, i); - if (include_hidden && PyDict_Size(hidden)) { - PyObject *innerlocals = PyDict_New(); - if (innerlocals == NULL) { - goto error; - } - if (PyDict_Merge(innerlocals, locals, 1) != 0) { - Py_DECREF(innerlocals); - goto error; - } - if (PyDict_Merge(innerlocals, hidden, 1) != 0) { - Py_DECREF(innerlocals); - goto error; + if (value != NULL) { + return true; + } } - locals = innerlocals; - } - else { - Py_INCREF(locals); } - Py_CLEAR(hidden); - - return locals; - error: - Py_XDECREF(hidden); - return NULL; + return false; } -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame) { - PyObject *locals = _PyFrame_GetLocals(frame, 0); - if (locals == NULL) { - return -1; + // We should try to avoid creating the FrameObject if possible. + // So we check if the frame is a module or class level scope + PyCodeObject *co = _PyFrame_GetCode(frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(frame)) { + if (frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + frame->f_locals = PyDict_New(); + if (frame->f_locals == NULL) { + return NULL; + } + } + return Py_NewRef(frame->f_locals); } - Py_DECREF(locals); - return 0; + + PyFrameObject* f = _PyFrame_GetFrameObject(frame); + + return _PyFrameLocalsProxy_New(f); } @@ -1351,112 +2105,25 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) int PyFrame_FastToLocalsWithError(PyFrameObject *f) { - if (f == NULL) { - PyErr_BadInternalCall(); - return -1; - } - assert(!_PyFrame_IsIncomplete(f->f_frame)); - int err = _PyFrame_FastToLocalsWithError(f->f_frame); - if (err == 0) { - f->f_fast_as_locals = 1; - } - return err; + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. + return 0; } void PyFrame_FastToLocals(PyFrameObject *f) { - int res; - assert(!_PyFrame_IsIncomplete(f->f_frame)); - assert(!PyErr_Occurred()); - - res = PyFrame_FastToLocalsWithError(f); - if (res < 0) - PyErr_Clear(); -} - -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) -{ - /* Merge locals into fast locals */ - PyObject *locals; - PyObject **fast; - PyCodeObject *co; - locals = frame->f_locals; - if (locals == NULL) { - return; - } - fast = _PyFrame_GetLocalsArray(frame); - co = _PyFrame_GetCode(frame); - - PyObject *exc = PyErr_GetRaisedException(); - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - /* Same test as in PyFrame_FastToLocals() above. */ - if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) { - continue; - } - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = PyObject_GetItem(locals, name); - /* We only care about NULLs if clear is true. */ - if (value == NULL) { - PyErr_Clear(); - if (!clear) { - continue; - } - } - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set when the frame was created from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } - else if (kind & CO_FAST_CELL && oldvalue != NULL) { - /* Same test as in PyFrame_FastToLocals() above. */ - if (PyCell_Check(oldvalue) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. - cell = oldvalue; - } - // (unlikely) Otherwise, it must have been set to some - // initial value by an earlier call to PyFrame_LocalsToFast(). - } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); - } - } - else if (value != oldvalue) { - if (value == NULL) { - // Probably can't delete this, since the compiler's flow - // analysis may have already "proven" that it exists here: - const char *e = "assigning None to unbound local %R"; - if (PyErr_WarnFormat(PyExc_RuntimeWarning, 0, e, name)) { - // It's okay if frame_obj is NULL, just try anyways: - PyErr_WriteUnraisable((PyObject *)frame->frame_obj); - } - value = Py_NewRef(Py_None); - } - Py_XSETREF(fast[i], Py_NewRef(value)); - } - Py_XDECREF(value); - } - PyErr_SetRaisedException(exc); + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. + return; } void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { - assert(!_PyFrame_IsIncomplete(f->f_frame)); - if (f && f->f_fast_as_locals && !frame_is_cleared(f)) { - _PyFrame_LocalsToFast(f->f_frame, clear); - f->f_fast_as_locals = 0; - } + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. + return; } int @@ -1534,24 +2201,6 @@ PyFrame_GetGenerator(PyFrameObject *frame) if (frame->f_frame->owner != FRAME_OWNED_BY_GENERATOR) { return NULL; } - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame->f_frame); return Py_NewRef(gen); } - -PyObject* -_PyEval_BuiltinsFromGlobals(PyThreadState *tstate, PyObject *globals) -{ - PyObject *builtins = PyDict_GetItemWithError(globals, &_Py_ID(__builtins__)); - if (builtins) { - if (PyModule_Check(builtins)) { - builtins = _PyModule_GetDict(builtins); - assert(builtins != NULL); - } - return builtins; - } - if (PyErr_Occurred()) { - return NULL; - } - - return _PyEval_GetBuiltins(tstate); -} diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 08b2823d8cf024..1f2387f68440aa 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -2,7 +2,8 @@ /* Function object implementation */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() +#include "pycore_dict.h" // _Py_INCREF_DICT() +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -111,11 +112,14 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) Py_XDECREF(module); return NULL; } - op->func_globals = Py_NewRef(constr->fc_globals); - op->func_builtins = Py_NewRef(constr->fc_builtins); + _Py_INCREF_DICT(constr->fc_globals); + op->func_globals = constr->fc_globals; + _Py_INCREF_BUILTINS(constr->fc_builtins); + op->func_builtins = constr->fc_builtins; op->func_name = Py_NewRef(constr->fc_name); op->func_qualname = Py_NewRef(constr->fc_qualname); - op->func_code = Py_NewRef(constr->fc_code); + _Py_INCREF_CODE((PyCodeObject *)constr->fc_code); + op->func_code = constr->fc_code; op->func_defaults = Py_XNewRef(constr->fc_defaults); op->func_kwdefaults = Py_XNewRef(constr->fc_kwdefaults); op->func_closure = Py_XNewRef(constr->fc_closure); @@ -124,9 +128,13 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + op->func_version = FUNC_VERSION_UNSET; + // NOTE: functions created via FrameConstructor do not use deferred + // reference counting because they are typically not part of cycles + // nor accessed by multiple threads. _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return op; @@ -137,11 +145,10 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname { assert(globals != NULL); assert(PyDict_Check(globals)); - Py_INCREF(globals); + _Py_INCREF_DICT(globals); - PyThreadState *tstate = _PyThreadState_GET(); - - PyCodeObject *code_obj = (PyCodeObject *)Py_NewRef(code); + PyCodeObject *code_obj = (PyCodeObject *)code; + _Py_INCREF_CODE(code_obj); assert(code_obj->co_name != NULL); PyObject *name = Py_NewRef(code_obj->co_name); @@ -155,7 +162,8 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname PyObject *consts = code_obj->co_consts; assert(PyTuple_Check(consts)); PyObject *doc; - if (PyTuple_Size(consts) >= 1) { + if (code_obj->co_flags & CO_HAS_DOCSTRING) { + assert(PyTuple_Size(consts) >= 1); doc = PyTuple_GetItem(consts, 0); if (!PyUnicode_Check(doc)) { doc = Py_None; @@ -173,11 +181,10 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname goto error; } - builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref + builtins = _PyDict_LoadBuiltinsFromGlobals(globals); if (builtins == NULL) { goto error; } - Py_INCREF(builtins); PyFunctionObject *op = PyObject_GC_New(PyFunctionObject, &PyFunction_Type); if (op == NULL) { @@ -199,9 +206,16 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + op->func_version = FUNC_VERSION_UNSET; + if ((code_obj->co_flags & CO_NESTED) == 0) { + // Use deferred reference counting for top-level functions, but not + // nested functions because they are more likely to capture variables, + // which makes prompt deallocation more important. + _PyObject_SetDeferredRefcount((PyObject *)op); + } _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return (PyObject *)op; @@ -218,92 +232,165 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname } /* -Function versions ------------------ +(This is purely internal documentation. There are no public APIs here.) + +Function (and code) versions +---------------------------- + +The Tier 1 specializer generates CALL variants that can be invalidated +by changes to critical function attributes: + +- __code__ +- __defaults__ +- __kwdefaults__ +- __closure__ -Function versions are used to detect when a function object has been -updated, invalidating inline cache data used by the `CALL` bytecode -(notably `CALL_PY_EXACT_ARGS` and a few other `CALL` specializations). +For this purpose function objects have a 32-bit func_version member +that the specializer writes to the specialized instruction's inline +cache and which is checked by a guard on the specialized instructions. -They are also used by the Tier 2 superblock creation code to find -the function being called (and from there the code object). +The MAKE_FUNCTION bytecode sets func_version from the code object's +co_version field. The latter is initialized from a counter in the +interpreter state (interp->func_state.next_version) and never changes. +When this counter overflows, it remains zero and the specializer loses +the ability to specialize calls to new functions. -How does a function's `func_version` field get initialized? +The func_version is reset to zero when any of the critical attributes +is modified; after this point the specializer will no longer specialize +calls to this function, and the guard will always fail. -- `PyFunction_New` and friends initialize it to 0. -- The `MAKE_FUNCTION` instruction sets it from the code's `co_version`. -- It is reset to 0 when various attributes like `__code__` are set. -- A new version is allocated by `_PyFunction_GetVersionForCurrentState` - when the specializer needs a version and the version is 0. +The function and code version cache +----------------------------------- -The latter allocates versions using a counter in the interpreter state; -when the counter wraps around to 0, no more versions are allocated. -There is one other special case: functions with a non-standard -`vectorcall` field are not given a version. +The Tier 2 optimizer now has a problem, since it needs to find the +function and code objects given only the version number from the inline +cache. Our solution is to maintain a cache mapping version numbers to +function and code objects. To limit the cache size we could hash +the version number, but for now we simply use it modulo the table size. -When the function version is 0, the `CALL` bytecode is not specialized. +There are some corner cases (e.g. generator expressions) where we will +be unable to find the function object in the cache but we can still +find the code object. For this reason the cache stores both the +function object and the code object. -Code object versions --------------------- +The cache doesn't contain strong references; cache entries are +invalidated whenever the function or code object is deallocated. -So where to code objects get their `co_version`? -There is a per-interpreter counter, `next_func_version`. -This is initialized to 1 when the interpreter is created. +Invariants +---------- + +These should hold at any time except when one of the cache-mutating +functions is running. + +- For any slot s at index i: + - s->func == NULL or s->func->func_version % FUNC_VERSION_CACHE_SIZE == i + - s->code == NULL or s->code->co_version % FUNC_VERSION_CACHE_SIZE == i + if s->func != NULL, then s->func->func_code == s->code -Code objects get a new `co_version` allocated from this counter upon -creation. Since code objects are nominally immutable, `co_version` can -not be invalidated. The only way it can be 0 is when 2**32 or more -code objects have been created during the process's lifetime. -(The counter isn't reset by `fork()`, extending the lifetime.) */ +static inline struct _func_version_cache_item * +get_cache_item(PyInterpreterState *interp, uint32_t version) +{ + return interp->func_state.func_version_cache + + (version % FUNC_VERSION_CACHE_SIZE); +} + void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) { + assert(func->func_version == FUNC_VERSION_UNSET); + assert(version >= FUNC_VERSION_FIRST_VALID); + // This should only be called from MAKE_FUNCTION. No code is specialized + // based on the version, so we do not need to stop the world to set it. + func->func_version = version; +#ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); - if (func->func_version != 0) { - PyFunctionObject **slot = - interp->func_state.func_version_cache - + (func->func_version % FUNC_VERSION_CACHE_SIZE); - if (*slot == func) { - *slot = NULL; - } + struct _func_version_cache_item *slot = get_cache_item(interp, version); + slot->func = func; + slot->code = func->func_code; +#endif +} + +static void +func_clear_version(PyInterpreterState *interp, PyFunctionObject *func) +{ + if (func->func_version < FUNC_VERSION_FIRST_VALID) { + // Version was never set or has already been cleared. + return; } - func->func_version = version; - if (version != 0) { - interp->func_state.func_version_cache[ - version % FUNC_VERSION_CACHE_SIZE] = func; +#ifndef Py_GIL_DISABLED + struct _func_version_cache_item *slot = + get_cache_item(interp, func->func_version); + if (slot->func == func) { + slot->func = NULL; + // Leave slot->code alone, there may be use for it. } +#endif + func->func_version = FUNC_VERSION_CLEARED; +} + +// Called when any of the critical function attributes are changed +static void +_PyFunction_ClearVersion(PyFunctionObject *func) +{ + if (func->func_version < FUNC_VERSION_FIRST_VALID) { + // Version was never set or has already been cleared. + return; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + func_clear_version(interp, func); + _PyEval_StartTheWorld(interp); +} + +void +_PyFunction_ClearCodeByVersion(uint32_t version) +{ +#ifndef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _func_version_cache_item *slot = get_cache_item(interp, version); + if (slot->code) { + assert(PyCode_Check(slot->code)); + PyCodeObject *code = (PyCodeObject *)slot->code; + if (code->co_version == version) { + slot->code = NULL; + slot->func = NULL; + } + } +#endif } PyFunctionObject * -_PyFunction_LookupByVersion(uint32_t version) +_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code) { +#ifdef Py_GIL_DISABLED + return NULL; +#else PyInterpreterState *interp = _PyInterpreterState_GET(); - PyFunctionObject *func = interp->func_state.func_version_cache[ - version % FUNC_VERSION_CACHE_SIZE]; - if (func != NULL && func->func_version == version) { - return func; + struct _func_version_cache_item *slot = get_cache_item(interp, version); + if (slot->code) { + assert(PyCode_Check(slot->code)); + PyCodeObject *code = (PyCodeObject *)slot->code; + if (code->co_version == version) { + *p_code = slot->code; + } + } + else { + *p_code = NULL; + } + if (slot->func && slot->func->func_version == version) { + assert(slot->func->func_code == slot->code); + return slot->func; } return NULL; +#endif } uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) { - if (func->func_version != 0) { - return func->func_version; - } - if (func->vectorcall != _PyFunction_Vectorcall) { - return 0; - } - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->func_state.next_version == 0) { - return 0; - } - uint32_t v = interp->func_state.next_version++; - _PyFunction_SetVersion(func, v); - return v; + return func->func_version; } PyObject * @@ -370,7 +457,7 @@ PyFunction_SetDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, (PyFunctionObject *) op, defaults); - _PyFunction_SetVersion((PyFunctionObject *)op, 0); + _PyFunction_ClearVersion((PyFunctionObject *)op); Py_XSETREF(((PyFunctionObject *)op)->func_defaults, defaults); return 0; } @@ -379,7 +466,7 @@ void PyFunction_SetVectorcall(PyFunctionObject *func, vectorcallfunc vectorcall) { assert(func != NULL); - _PyFunction_SetVersion(func, 0); + _PyFunction_ClearVersion(func); func->vectorcall = vectorcall; } @@ -412,7 +499,7 @@ PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, (PyFunctionObject *) op, defaults); - _PyFunction_SetVersion((PyFunctionObject *)op, 0); + _PyFunction_ClearVersion((PyFunctionObject *)op); Py_XSETREF(((PyFunctionObject *)op)->func_kwdefaults, defaults); return 0; } @@ -445,7 +532,7 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure) Py_TYPE(closure)->tp_name); return -1; } - _PyFunction_SetVersion((PyFunctionObject *)op, 0); + _PyFunction_ClearVersion((PyFunctionObject *)op); Py_XSETREF(((PyFunctionObject *)op)->func_closure, closure); return 0; } @@ -454,7 +541,22 @@ static PyObject * func_get_annotation_dict(PyFunctionObject *op) { if (op->func_annotations == NULL) { - return NULL; + if (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate)) { + Py_RETURN_NONE; + } + PyObject *one = _PyLong_GetOne(); + PyObject *ann_dict = _PyObject_CallOneArg(op->func_annotate, one); + if (ann_dict == NULL) { + return NULL; + } + if (!PyDict_Check(ann_dict)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(ann_dict)->tp_name); + Py_DECREF(ann_dict); + return NULL; + } + Py_XSETREF(op->func_annotations, ann_dict); + return ann_dict; } if (PyTuple_CheckExact(op->func_annotations)) { PyObject *ann_tuple = op->func_annotations; @@ -471,6 +573,7 @@ func_get_annotation_dict(PyFunctionObject *op) PyTuple_GET_ITEM(ann_tuple, i + 1)); if (err < 0) { + Py_DECREF(ann_dict); return NULL; } } @@ -507,8 +610,9 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) "non-dict annotations"); return -1; } - _PyFunction_SetVersion((PyFunctionObject *)op, 0); - Py_XSETREF(((PyFunctionObject *)op)->func_annotations, annotations); + PyFunctionObject *func = (PyFunctionObject *)op; + Py_XSETREF(func->func_annotations, annotations); + Py_CLEAR(func->func_annotate); return 0; } @@ -526,8 +630,9 @@ static PyMemberDef func_memberlist[] = { }; static PyObject * -func_get_code(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_code(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (PySys_Audit("object.__getattr__", "Os", op, "__code__") < 0) { return NULL; } @@ -536,10 +641,9 @@ func_get_code(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { - Py_ssize_t nclosure; - int nfree; + PyFunctionObject *op = _PyFunction_CAST(self); /* Not legal to del f.func_code or to set it to anything * other than a code object. */ @@ -554,9 +658,9 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) return -1; } - nfree = ((PyCodeObject *)value)->co_nfreevars; - nclosure = (op->func_closure == NULL ? 0 : - PyTuple_GET_SIZE(op->func_closure)); + int nfree = ((PyCodeObject *)value)->co_nfreevars; + Py_ssize_t nclosure = (op->func_closure == NULL ? 0 : + PyTuple_GET_SIZE(op->func_closure)); if (nclosure != nfree) { PyErr_Format(PyExc_ValueError, "%U() requires a code object with %zd free vars," @@ -580,20 +684,22 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) } handle_func_event(PyFunction_EVENT_MODIFY_CODE, op, value); - _PyFunction_SetVersion(op, 0); + _PyFunction_ClearVersion(op); Py_XSETREF(op->func_code, Py_NewRef(value)); return 0; } static PyObject * -func_get_name(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_name(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); return Py_NewRef(op->func_name); } static int -func_set_name(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_name(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); /* Not legal to del f.func_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -606,14 +712,16 @@ func_set_name(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -func_get_qualname(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); return Py_NewRef(op->func_qualname); } static int -func_set_qualname(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); /* Not legal to del f.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -626,8 +734,9 @@ func_set_qualname(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } static PyObject * -func_get_defaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_defaults(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (PySys_Audit("object.__getattr__", "Os", op, "__defaults__") < 0) { return NULL; } @@ -638,10 +747,11 @@ func_get_defaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_defaults(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { /* Legal to del f.func_defaults. * Can only set func_defaults to NULL or a tuple. */ + PyFunctionObject *op = _PyFunction_CAST(self); if (value == Py_None) value = NULL; if (value != NULL && !PyTuple_Check(value)) { @@ -660,14 +770,15 @@ func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, op, value); - _PyFunction_SetVersion(op, 0); + _PyFunction_ClearVersion(op); Py_XSETREF(op->func_defaults, Py_XNewRef(value)); return 0; } static PyObject * -func_get_kwdefaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_kwdefaults(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (PySys_Audit("object.__getattr__", "Os", op, "__kwdefaults__") < 0) { return NULL; @@ -679,8 +790,9 @@ func_get_kwdefaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_kwdefaults(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (value == Py_None) value = NULL; /* Legal to del f.func_kwdefaults. @@ -701,15 +813,52 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignor } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, op, value); - _PyFunction_SetVersion(op, 0); + _PyFunction_ClearVersion(op); Py_XSETREF(op->func_kwdefaults, Py_XNewRef(value)); return 0; } static PyObject * -func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_annotate(PyObject *self, void *Py_UNUSED(ignored)) { - if (op->func_annotations == NULL) { + PyFunctionObject *op = _PyFunction_CAST(self); + if (op->func_annotate == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(op->func_annotate); +} + +static int +func_set_annotate(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) +{ + PyFunctionObject *op = _PyFunction_CAST(self); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "__annotate__ cannot be deleted"); + return -1; + } + if (Py_IsNone(value)) { + Py_XSETREF(op->func_annotate, value); + return 0; + } + else if (PyCallable_Check(value)) { + Py_XSETREF(op->func_annotate, Py_XNewRef(value)); + Py_CLEAR(op->func_annotations); + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "__annotate__ must be callable or None"); + return -1; + } +} + +static PyObject * +func_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFunctionObject *op = _PyFunction_CAST(self); + if (op->func_annotations == NULL && + (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate))) { op->func_annotations = PyDict_New(); if (op->func_annotations == NULL) return NULL; @@ -719,8 +868,9 @@ func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_annotations(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (value == Py_None) value = NULL; /* Legal to del f.func_annotations. @@ -731,14 +881,15 @@ func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(igno "__annotations__ must be set to a dict object"); return -1; } - _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_annotations, Py_XNewRef(value)); + Py_CLEAR(op->func_annotate); return 0; } static PyObject * -func_get_type_params(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_type_params(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (op->func_typeparams == NULL) { return PyTuple_New(0); } @@ -748,10 +899,11 @@ func_get_type_params(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_type_params(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_type_params(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { /* Not legal to del f.__type_params__ or to set it to anything * other than a tuple object. */ + PyFunctionObject *op = _PyFunction_CAST(self); if (value == NULL || !PyTuple_Check(value)) { PyErr_SetString(PyExc_TypeError, "__type_params__ must be set to a tuple"); @@ -773,18 +925,15 @@ _Py_set_function_type_params(PyThreadState *Py_UNUSED(ignored), PyObject *func, } static PyGetSetDef func_getsetlist[] = { - {"__code__", (getter)func_get_code, (setter)func_set_code}, - {"__defaults__", (getter)func_get_defaults, - (setter)func_set_defaults}, - {"__kwdefaults__", (getter)func_get_kwdefaults, - (setter)func_set_kwdefaults}, - {"__annotations__", (getter)func_get_annotations, - (setter)func_set_annotations}, + {"__code__", func_get_code, func_set_code}, + {"__defaults__", func_get_defaults, func_set_defaults}, + {"__kwdefaults__", func_get_kwdefaults, func_set_kwdefaults}, + {"__annotations__", func_get_annotations, func_set_annotations}, + {"__annotate__", func_get_annotate, func_set_annotate}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, - {"__name__", (getter)func_get_name, (setter)func_set_name}, - {"__qualname__", (getter)func_get_qualname, (setter)func_set_qualname}, - {"__type_params__", (getter)func_get_type_params, - (setter)func_set_type_params}, + {"__name__", func_get_name, func_set_name}, + {"__qualname__", func_get_qualname, func_set_qualname}, + {"__type_params__", func_get_type_params, func_set_type_params}, {NULL} /* Sentinel */ }; @@ -904,11 +1053,20 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, } static int -func_clear(PyFunctionObject *op) +func_clear(PyObject *self) { - _PyFunction_SetVersion(op, 0); - Py_CLEAR(op->func_globals); - Py_CLEAR(op->func_builtins); + PyFunctionObject *op = _PyFunction_CAST(self); + func_clear_version(_PyInterpreterState_GET(), op); + PyObject *globals = op->func_globals; + op->func_globals = NULL; + if (globals != NULL) { + _Py_DECREF_DICT(globals); + } + PyObject *builtins = op->func_builtins; + op->func_builtins = NULL; + if (builtins != NULL) { + _Py_DECREF_BUILTINS(builtins); + } Py_CLEAR(op->func_module); Py_CLEAR(op->func_defaults); Py_CLEAR(op->func_kwdefaults); @@ -916,6 +1074,7 @@ func_clear(PyFunctionObject *op) Py_CLEAR(op->func_dict); Py_CLEAR(op->func_closure); Py_CLEAR(op->func_annotations); + Py_CLEAR(op->func_annotate); Py_CLEAR(op->func_typeparams); // Don't Py_CLEAR(op->func_code), since code is always required // to be non-NULL. Similarly, name and qualname shouldn't be NULL. @@ -928,8 +1087,9 @@ func_clear(PyFunctionObject *op) } static void -func_dealloc(PyFunctionObject *op) +func_dealloc(PyObject *self) { + PyFunctionObject *op = _PyFunction_CAST(self); assert(Py_REFCNT(op) == 0); Py_SET_REFCNT(op, 1); handle_func_event(PyFunction_EVENT_DESTROY, op, NULL); @@ -942,25 +1102,26 @@ func_dealloc(PyFunctionObject *op) if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); } - _PyFunction_SetVersion(op, 0); - (void)func_clear(op); + (void)func_clear((PyObject*)op); // These aren't cleared by func_clear(). - Py_DECREF(op->func_code); + _Py_DECREF_CODE((PyCodeObject *)op->func_code); Py_DECREF(op->func_name); Py_DECREF(op->func_qualname); PyObject_GC_Del(op); } static PyObject* -func_repr(PyFunctionObject *op) +func_repr(PyObject *self) { + PyFunctionObject *op = _PyFunction_CAST(self); return PyUnicode_FromFormat("", op->func_qualname, op); } static int -func_traverse(PyFunctionObject *f, visitproc visit, void *arg) +func_traverse(PyObject *self, visitproc visit, void *arg) { + PyFunctionObject *f = _PyFunction_CAST(self); Py_VISIT(f->func_code); Py_VISIT(f->func_globals); Py_VISIT(f->func_builtins); @@ -972,6 +1133,7 @@ func_traverse(PyFunctionObject *f, visitproc visit, void *arg) Py_VISIT(f->func_dict); Py_VISIT(f->func_closure); Py_VISIT(f->func_annotations); + Py_VISIT(f->func_annotate); Py_VISIT(f->func_typeparams); Py_VISIT(f->func_qualname); return 0; @@ -992,12 +1154,12 @@ PyTypeObject PyFunction_Type = { "function", sizeof(PyFunctionObject), 0, - (destructor)func_dealloc, /* tp_dealloc */ + func_dealloc, /* tp_dealloc */ offsetof(PyFunctionObject, vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)func_repr, /* tp_repr */ + func_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1011,8 +1173,8 @@ PyTypeObject PyFunction_Type = { Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_METHOD_DESCRIPTOR, /* tp_flags */ func_new__doc__, /* tp_doc */ - (traverseproc)func_traverse, /* tp_traverse */ - (inquiry)func_clear, /* tp_clear */ + func_traverse, /* tp_traverse */ + func_clear, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyFunctionObject, func_weakreflist), /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -1058,12 +1220,75 @@ functools_wraps(PyObject *wrapper, PyObject *wrapped) COPY_ATTR(__name__); COPY_ATTR(__qualname__); COPY_ATTR(__doc__); - COPY_ATTR(__annotations__); return 0; #undef COPY_ATTR } +// Used for wrapping __annotations__ and __annotate__ on classmethod +// and staticmethod objects. +static PyObject * +descriptor_get_wrapped_attribute(PyObject *wrapped, PyObject *obj, PyObject *name) +{ + PyObject *dict = PyObject_GenericGetDict(obj, NULL); + if (dict == NULL) { + return NULL; + } + PyObject *res; + if (PyDict_GetItemRef(dict, name, &res) < 0) { + Py_DECREF(dict); + return NULL; + } + if (res != NULL) { + Py_DECREF(dict); + return res; + } + res = PyObject_GetAttr(wrapped, name); + if (res == NULL) { + Py_DECREF(dict); + return NULL; + } + if (PyDict_SetItem(dict, name, res) < 0) { + Py_DECREF(dict); + Py_DECREF(res); + return NULL; + } + Py_DECREF(dict); + return res; +} + +static int +descriptor_set_wrapped_attribute(PyObject *oobj, PyObject *name, PyObject *value, + char *type_name) +{ + PyObject *dict = PyObject_GenericGetDict(oobj, NULL); + if (dict == NULL) { + return -1; + } + if (value == NULL) { + if (PyDict_DelItem(dict, name) < 0) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + PyErr_Format(PyExc_AttributeError, + "'%.200s' object has no attribute '%U'", + type_name, name); + Py_DECREF(dict); + return -1; + } + else { + Py_DECREF(dict); + return -1; + } + } + Py_DECREF(dict); + return 0; + } + else { + Py_DECREF(dict); + return PyDict_SetItem(dict, name, value); + } +} + /* Class method object */ @@ -1091,9 +1316,14 @@ typedef struct { PyObject *cm_dict; } classmethod; +#define _PyClassMethod_CAST(cm) \ + (assert(PyObject_TypeCheck((cm), &PyClassMethod_Type)), \ + _Py_CAST(classmethod*, cm)) + static void -cm_dealloc(classmethod *cm) +cm_dealloc(PyObject *self) { + classmethod *cm = _PyClassMethod_CAST(self); _PyObject_GC_UNTRACK((PyObject *)cm); Py_XDECREF(cm->cm_callable); Py_XDECREF(cm->cm_dict); @@ -1101,16 +1331,18 @@ cm_dealloc(classmethod *cm) } static int -cm_traverse(classmethod *cm, visitproc visit, void *arg) +cm_traverse(PyObject *self, visitproc visit, void *arg) { + classmethod *cm = _PyClassMethod_CAST(self); Py_VISIT(cm->cm_callable); Py_VISIT(cm->cm_dict); return 0; } static int -cm_clear(classmethod *cm) +cm_clear(PyObject *self) { + classmethod *cm = _PyClassMethod_CAST(self); Py_CLEAR(cm->cm_callable); Py_CLEAR(cm->cm_dict); return 0; @@ -1157,8 +1389,9 @@ static PyMemberDef cm_memberlist[] = { }; static PyObject * -cm_get___isabstractmethod__(classmethod *cm, void *closure) +cm_get___isabstractmethod__(PyObject *self, void *closure) { + classmethod *cm = _PyClassMethod_CAST(self); int res = _PyObject_IsAbstract(cm->cm_callable); if (res == -1) { return NULL; @@ -1169,16 +1402,45 @@ cm_get___isabstractmethod__(classmethod *cm, void *closure) Py_RETURN_FALSE; } +static PyObject * +cm_get___annotations__(PyObject *self, void *closure) +{ + classmethod *cm = _PyClassMethod_CAST(self); + return descriptor_get_wrapped_attribute(cm->cm_callable, self, &_Py_ID(__annotations__)); +} + +static int +cm_set___annotations__(PyObject *self, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(self, &_Py_ID(__annotations__), value, "classmethod"); +} + +static PyObject * +cm_get___annotate__(PyObject *self, void *closure) +{ + classmethod *cm = _PyClassMethod_CAST(self); + return descriptor_get_wrapped_attribute(cm->cm_callable, self, &_Py_ID(__annotate__)); +} + +static int +cm_set___annotate__(PyObject *self, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(self, &_Py_ID(__annotate__), value, "classmethod"); +} + + static PyGetSetDef cm_getsetlist[] = { - {"__isabstractmethod__", - (getter)cm_get___isabstractmethod__, NULL, NULL, NULL}, + {"__isabstractmethod__", cm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", cm_get___annotations__, cm_set___annotations__, NULL, NULL}, + {"__annotate__", cm_get___annotate__, cm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; static PyObject* -cm_repr(classmethod *cm) +cm_repr(PyObject *self) { + classmethod *cm = _PyClassMethod_CAST(self); return PyUnicode_FromFormat("", cm->cm_callable); } @@ -1210,12 +1472,12 @@ PyTypeObject PyClassMethod_Type = { "classmethod", sizeof(classmethod), 0, - (destructor)cm_dealloc, /* tp_dealloc */ + cm_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)cm_repr, /* tp_repr */ + cm_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1227,14 +1489,14 @@ PyTypeObject PyClassMethod_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, classmethod_doc, /* tp_doc */ - (traverseproc)cm_traverse, /* tp_traverse */ - (inquiry)cm_clear, /* tp_clear */ + cm_traverse, /* tp_traverse */ + cm_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ 0, /* tp_methods */ - cm_memberlist, /* tp_members */ + cm_memberlist, /* tp_members */ cm_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ @@ -1283,9 +1545,14 @@ typedef struct { PyObject *sm_dict; } staticmethod; +#define _PyStaticMethod_CAST(cm) \ + (assert(PyObject_TypeCheck((cm), &PyStaticMethod_Type)), \ + _Py_CAST(staticmethod*, cm)) + static void -sm_dealloc(staticmethod *sm) +sm_dealloc(PyObject *self) { + staticmethod *sm = _PyStaticMethod_CAST(self); _PyObject_GC_UNTRACK((PyObject *)sm); Py_XDECREF(sm->sm_callable); Py_XDECREF(sm->sm_dict); @@ -1293,16 +1560,18 @@ sm_dealloc(staticmethod *sm) } static int -sm_traverse(staticmethod *sm, visitproc visit, void *arg) +sm_traverse(PyObject *self, visitproc visit, void *arg) { + staticmethod *sm = _PyStaticMethod_CAST(self); Py_VISIT(sm->sm_callable); Py_VISIT(sm->sm_dict); return 0; } static int -sm_clear(staticmethod *sm) +sm_clear(PyObject *self) { + staticmethod *sm = _PyStaticMethod_CAST(self); Py_CLEAR(sm->sm_callable); Py_CLEAR(sm->sm_dict); return 0; @@ -1353,8 +1622,9 @@ static PyMemberDef sm_memberlist[] = { }; static PyObject * -sm_get___isabstractmethod__(staticmethod *sm, void *closure) +sm_get___isabstractmethod__(PyObject *self, void *closure) { + staticmethod *sm = _PyStaticMethod_CAST(self); int res = _PyObject_IsAbstract(sm->sm_callable); if (res == -1) { return NULL; @@ -1365,16 +1635,44 @@ sm_get___isabstractmethod__(staticmethod *sm, void *closure) Py_RETURN_FALSE; } +static PyObject * +sm_get___annotations__(PyObject *self, void *closure) +{ + staticmethod *sm = _PyStaticMethod_CAST(self); + return descriptor_get_wrapped_attribute(sm->sm_callable, self, &_Py_ID(__annotations__)); +} + +static int +sm_set___annotations__(PyObject *self, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(self, &_Py_ID(__annotations__), value, "staticmethod"); +} + +static PyObject * +sm_get___annotate__(PyObject *self, void *closure) +{ + staticmethod *sm = _PyStaticMethod_CAST(self); + return descriptor_get_wrapped_attribute(sm->sm_callable, self, &_Py_ID(__annotate__)); +} + +static int +sm_set___annotate__(PyObject *self, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(self, &_Py_ID(__annotate__), value, "staticmethod"); +} + static PyGetSetDef sm_getsetlist[] = { - {"__isabstractmethod__", - (getter)sm_get___isabstractmethod__, NULL, NULL, NULL}, + {"__isabstractmethod__", sm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", sm_get___annotations__, sm_set___annotations__, NULL, NULL}, + {"__annotate__", sm_get___annotate__, sm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; static PyObject* -sm_repr(staticmethod *sm) +sm_repr(PyObject *self) { + staticmethod *sm = _PyStaticMethod_CAST(self); return PyUnicode_FromFormat("", sm->sm_callable); } @@ -1404,12 +1702,12 @@ PyTypeObject PyStaticMethod_Type = { "staticmethod", sizeof(staticmethod), 0, - (destructor)sm_dealloc, /* tp_dealloc */ + sm_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)sm_repr, /* tp_repr */ + sm_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1421,14 +1719,14 @@ PyTypeObject PyStaticMethod_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, staticmethod_doc, /* tp_doc */ - (traverseproc)sm_traverse, /* tp_traverse */ - (inquiry)sm_clear, /* tp_clear */ + sm_traverse, /* tp_traverse */ + sm_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ 0, /* tp_methods */ - sm_memberlist, /* tp_members */ + sm_memberlist, /* tp_members */ sm_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index c045d495e85526..64b4e2645cbaee 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -4,6 +4,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" +#include "pycore_typevarobject.h" // _Py_typing_type_repr #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check @@ -51,96 +52,29 @@ ga_traverse(PyObject *self, visitproc visit, void *arg) } static int -ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) -{ - PyObject *qualname = NULL; - PyObject *module = NULL; - PyObject *r = NULL; - int rc; - - if (p == Py_Ellipsis) { - // The Ellipsis object - r = PyUnicode_FromString("..."); - goto done; - } - - if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && - (rc = PyObject_HasAttrWithError(p, &_Py_ID(__args__))) > 0) - { - // It looks like a GenericAlias - goto use_repr; - } - if (rc < 0) { - goto done; - } - - if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { - goto done; - } - if (qualname == NULL) { - goto use_repr; - } - if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { - goto done; - } - if (module == NULL || module == Py_None) { - goto use_repr; - } - - // Looks like a class - if (PyUnicode_Check(module) && - _PyUnicode_EqualToASCIIString(module, "builtins")) - { - // builtins don't need a module name - r = PyObject_Str(qualname); - goto done; - } - else { - r = PyUnicode_FromFormat("%S.%S", module, qualname); - goto done; - } - -use_repr: - r = PyObject_Repr(p); - -done: - Py_XDECREF(qualname); - Py_XDECREF(module); - if (r == NULL) { - // error if any of the above PyObject_Repr/PyUnicode_From* fail - rc = -1; - } - else { - rc = _PyUnicodeWriter_WriteStr(writer, r); - Py_DECREF(r); - } - return rc; -} - -static int -ga_repr_items_list(_PyUnicodeWriter *writer, PyObject *p) +ga_repr_items_list(PyUnicodeWriter *writer, PyObject *p) { assert(PyList_CheckExact(p)); Py_ssize_t len = PyList_GET_SIZE(p); - if (_PyUnicodeWriter_WriteASCIIString(writer, "[", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { return -1; } for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { return -1; } } PyObject *item = PyList_GET_ITEM(p, i); - if (ga_repr_item(writer, item) < 0) { + if (_Py_typing_type_repr(writer, item) < 0) { return -1; } } - if (_PyUnicodeWriter_WriteASCIIString(writer, "]", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { return -1; } @@ -153,49 +87,55 @@ ga_repr(PyObject *self) gaobject *alias = (gaobject *)self; Py_ssize_t len = PyTuple_GET_SIZE(alias->args); - _PyUnicodeWriter writer; - _PyUnicodeWriter_Init(&writer); + // Estimation based on the shortest format: "int[int, int, int]" + Py_ssize_t estimate = (len <= PY_SSIZE_T_MAX / 5) ? len * 5 : len; + estimate = 3 + 1 + estimate + 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(estimate); + if (writer == NULL) { + return NULL; + } if (alias->starred) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, "*", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, '*') < 0) { goto error; } } - if (ga_repr_item(&writer, alias->origin) < 0) { + if (_Py_typing_type_repr(writer, alias->origin) < 0) { goto error; } - if (_PyUnicodeWriter_WriteASCIIString(&writer, "[", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { goto error; } for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { goto error; } } PyObject *p = PyTuple_GET_ITEM(alias->args, i); if (PyList_CheckExact(p)) { // Looks like we are working with ParamSpec's list of type args: - if (ga_repr_items_list(&writer, p) < 0) { + if (ga_repr_items_list(writer, p) < 0) { goto error; } } - else if (ga_repr_item(&writer, p) < 0) { + else if (_Py_typing_type_repr(writer, p) < 0) { goto error; } } if (len == 0) { // for something like tuple[()] we should print a "()" - if (_PyUnicodeWriter_WriteASCIIString(&writer, "()", 2) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, "()", 2) < 0) { goto error; } } - if (_PyUnicodeWriter_WriteASCIIString(&writer, "]", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { goto error; } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); + error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } @@ -537,6 +477,8 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } PyDoc_STRVAR(genericalias__doc__, +"GenericAlias(origin, args, /)\n" +"--\n\n" "Represent a PEP 585 generic type\n" "\n" "E.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,)."); @@ -559,6 +501,10 @@ ga_getitem(PyObject *self, PyObject *item) } PyObject *res = Py_GenericAlias(alias->origin, newargs); + if (res == NULL) { + Py_DECREF(newargs); + return NULL; + } ((gaobject *)res)->starred = alias->starred; Py_DECREF(newargs); diff --git a/Objects/genobject.c b/Objects/genobject.c index 59ab7abf6180bd..19c2c4e3331a89 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -6,19 +6,31 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() -#include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_utils.h" // RESUME_AFTER_YIELD_FROM +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pystats.h" -static PyObject *gen_close(PyGenObject *, PyObject *); -static PyObject *async_gen_asend_new(PyAsyncGenObject *, PyObject *); -static PyObject *async_gen_athrow_new(PyAsyncGenObject *, PyObject *); +// Forward declarations +static PyObject* gen_close(PyObject *, PyObject *); +static PyObject* async_gen_asend_new(PyAsyncGenObject *, PyObject *); +static PyObject* async_gen_athrow_new(PyAsyncGenObject *, PyObject *); + + +#define _PyGen_CAST(op) \ + _Py_CAST(PyGenObject*, (op)) +#define _PyCoroObject_CAST(op) \ + (assert(PyCoro_CheckExact(op)), \ + _Py_CAST(PyCoroObject*, (op))) +#define _PyAsyncGenObject_CAST(op) \ + _Py_CAST(PyAsyncGenObject*, (op)) + static const char *NON_INIT_CORO_MSG = "can't send non-None value to a " "just-started coroutine"; @@ -29,8 +41,7 @@ static const char *ASYNC_GEN_IGNORED_EXIT_MSG = /* Returns a borrowed reference */ static inline PyCodeObject * _PyGen_GetCode(PyGenObject *gen) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); - return _PyFrame_GetCode(frame); + return _PyFrame_GetCode(&gen->gi_iframe); } PyCodeObject * @@ -42,12 +53,13 @@ PyGen_GetCode(PyGenObject *gen) { } static int -gen_traverse(PyGenObject *gen, visitproc visit, void *arg) +gen_traverse(PyObject *self, visitproc visit, void *arg) { + PyGenObject *gen = _PyGen_CAST(self); Py_VISIT(gen->gi_name); Py_VISIT(gen->gi_qualname); if (gen->gi_frame_state != FRAME_CLEARED) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); + _PyInterpreterFrame *frame = &gen->gi_iframe; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame->owner == FRAME_OWNED_BY_GENERATOR); int err = _PyFrame_Traverse(frame, visit, arg); @@ -55,6 +67,11 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg) return err; } } + else { + // We still need to visit the code object when the frame is cleared to + // ensure that it's kept alive if the reference is deferred. + _Py_VISIT_STACKREF(gen->gi_iframe.f_executable); + } /* No need to visit cr_origin, because it's just tuples/str/int, so can't participate in a reference cycle. */ Py_VISIT(gen->gi_exc_state.exc_value); @@ -102,7 +119,7 @@ _PyGen_Finalize(PyObject *self) _PyErr_WarnUnawaitedCoroutine((PyObject *)gen); } else { - PyObject *res = gen_close(gen, NULL); + PyObject *res = gen_close((PyObject*)gen, NULL); if (res == NULL) { if (PyErr_Occurred()) { PyErr_WriteUnraisable(self); @@ -118,9 +135,9 @@ _PyGen_Finalize(PyObject *self) } static void -gen_dealloc(PyGenObject *gen) +gen_dealloc(PyObject *self) { - PyObject *self = (PyObject *) gen; + PyGenObject *gen = _PyGen_CAST(self); _PyObject_GC_UNTRACK(gen); @@ -139,18 +156,18 @@ gen_dealloc(PyGenObject *gen) and GC_Del. */ Py_CLEAR(((PyAsyncGenObject*)gen)->ag_origin_or_finalizer); } + if (PyCoro_CheckExact(gen)) { + Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); + } if (gen->gi_frame_state != FRAME_CLEARED) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; gen->gi_frame_state = FRAME_CLEARED; frame->previous = NULL; _PyFrame_ClearExceptCode(frame); _PyErr_ClearExcState(&gen->gi_exc_state); } assert(gen->gi_exc_state.exc_value == NULL); - if (_PyGen_GetCode(gen)->co_flags & CO_COROUTINE) { - Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); - } - Py_DECREF(_PyGen_GetCode(gen)); + PyStackRef_CLEAR(gen->gi_iframe.f_executable); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); @@ -162,7 +179,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc, int closing) { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; *presult = NULL; if (gen->gi_frame_state == FRAME_CREATED && arg && arg != Py_None) { @@ -212,7 +229,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, /* Push arg onto the frame's value stack */ PyObject *arg_obj = arg ? arg : Py_None; - _PyFrame_StackPush(frame, Py_NewRef(arg_obj)); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectNew(arg_obj)); _PyErr_StackItem *prev_exc_info = tstate->exc_info; gen->gi_exc_state.previous_item = prev_exc_info; @@ -257,8 +274,9 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, } static PySendResult -PyGen_am_send(PyGenObject *gen, PyObject *arg, PyObject **result) +PyGen_am_send(PyObject *self, PyObject *arg, PyObject **result) { + PyGenObject *gen = _PyGen_CAST(self); return gen_send_ex2(gen, arg, result, 0, 0); } @@ -287,9 +305,9 @@ PyDoc_STRVAR(send_doc, return next yielded value or raise StopIteration."); static PyObject * -gen_send(PyGenObject *gen, PyObject *arg) +gen_send(PyObject *gen, PyObject *arg) { - return gen_send_ex(gen, arg, 0, 0); + return gen_send_ex((PyGenObject*)gen, arg, 0, 0); } PyDoc_STRVAR(close_doc, @@ -306,7 +324,7 @@ gen_close_iter(PyObject *yf) PyObject *retval = NULL; if (PyGen_CheckExact(yf) || PyCoro_CheckExact(yf)) { - retval = gen_close((PyGenObject *)yf, NULL); + retval = gen_close((PyObject *)yf, NULL); if (retval == NULL) return -1; } @@ -329,10 +347,11 @@ gen_close_iter(PyObject *yf) static inline bool is_resume(_Py_CODEUNIT *instr) { + uint8_t code = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code); return ( - instr->op.code == RESUME || - instr->op.code == RESUME_CHECK || - instr->op.code == INSTRUMENTED_RESUME + code == RESUME || + code == RESUME_CHECK || + code == INSTRUMENTED_RESUME ); } @@ -340,20 +359,19 @@ PyObject * _PyGen_yf(PyGenObject *gen) { if (gen->gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - assert(is_resume(frame->instr_ptr)); - assert((frame->instr_ptr->op.arg & RESUME_OPARG_LOCATION_MASK) >= RESUME_AFTER_YIELD_FROM); - return Py_NewRef(_PyFrame_StackPeek(frame)); + _PyInterpreterFrame *frame = &gen->gi_iframe; + // GH-122390: These asserts are wrong in the presence of ENTER_EXECUTOR! + // assert(is_resume(frame->instr_ptr)); + // assert((frame->instr_ptr->op.arg & RESUME_OPARG_LOCATION_MASK) >= RESUME_AFTER_YIELD_FROM); + return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame)); } return NULL; } static PyObject * -gen_close(PyGenObject *gen, PyObject *args) +gen_close(PyObject *self, PyObject *args) { - PyObject *retval; - int err = 0; - + PyGenObject *gen = _PyGen_CAST(self); if (gen->gi_frame_state == FRAME_CREATED) { gen->gi_frame_state = FRAME_COMPLETED; @@ -362,7 +380,9 @@ gen_close(PyGenObject *gen, PyObject *args) if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { Py_RETURN_NONE; } + PyObject *yf = _PyGen_yf(gen); + int err = 0; if (yf) { PyFrameState state = gen->gi_frame_state; gen->gi_frame_state = FRAME_EXECUTING; @@ -370,7 +390,7 @@ gen_close(PyGenObject *gen, PyObject *args) gen->gi_frame_state = state; Py_DECREF(yf); } - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; if (is_resume(frame->instr_ptr)) { /* We can safely ignore the outermost try block * as it is automatically generated to handle @@ -380,13 +400,15 @@ gen_close(PyGenObject *gen, PyObject *args) // RESUME after YIELD_VALUE and exception depth is 1 assert((oparg & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START); gen->gi_frame_state = FRAME_COMPLETED; + _PyFrame_ClearLocals(&gen->gi_iframe); Py_RETURN_NONE; } } if (err == 0) { PyErr_SetNone(PyExc_GeneratorExit); } - retval = gen_send_ex(gen, Py_None, 1, 1); + + PyObject *retval = gen_send_ex(gen, Py_None, 1, 1); if (retval) { const char *msg = "generator ignored GeneratorExit"; if (PyCoro_CheckExact(gen)) { @@ -399,10 +421,12 @@ gen_close(PyGenObject *gen, PyObject *args) return NULL; } assert(PyErr_Occurred()); + if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) { PyErr_Clear(); /* ignore this error */ Py_RETURN_NONE; } + /* if the generator returned a value while closing, StopIteration was * raised in gen_send_ex() above; retrieve and return the value here */ if (_PyGen_FetchStopIterationValue(&retval) == 0) { @@ -428,7 +452,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, PyObject *yf = _PyGen_yf(gen); if (yf) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; PyObject *ret; int err; if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && @@ -579,10 +603,12 @@ gen_throw(PyGenObject *gen, PyObject *const *args, Py_ssize_t nargs) static PyObject * -gen_iternext(PyGenObject *gen) +gen_iternext(PyObject *self) { + assert(PyGen_CheckExact(self) || PyCoro_CheckExact(self)); + PyGenObject *gen = _PyGen_CAST(self); + PyObject *result; - assert(PyGen_CheckExact(gen) || PyCoro_CheckExact(gen)); if (gen_send_ex2(gen, NULL, &result, 0, 0) == PYGEN_RETURN) { if (result != Py_None) { _PyGen_SetStopIterationValue(result); @@ -656,21 +682,24 @@ _PyGen_FetchStopIterationValue(PyObject **pvalue) } static PyObject * -gen_repr(PyGenObject *gen) +gen_repr(PyObject *self) { + PyGenObject *gen = _PyGen_CAST(self); return PyUnicode_FromFormat("", gen->gi_qualname, gen); } static PyObject * -gen_get_name(PyGenObject *op, void *Py_UNUSED(ignored)) +gen_get_name(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); return Py_NewRef(op->gi_name); } static int -gen_set_name(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) +gen_set_name(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); /* Not legal to del gen.gi_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -683,14 +712,16 @@ gen_set_name(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -gen_get_qualname(PyGenObject *op, void *Py_UNUSED(ignored)) +gen_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); return Py_NewRef(op->gi_qualname); } static int -gen_set_qualname(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) +gen_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); /* Not legal to del gen.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -703,18 +734,20 @@ gen_set_qualname(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -gen_getyieldfrom(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getyieldfrom(PyObject *gen, void *Py_UNUSED(ignored)) { - PyObject *yf = _PyGen_yf(gen); - if (yf == NULL) + PyObject *yf = _PyGen_yf(_PyGen_CAST(gen)); + if (yf == NULL) { Py_RETURN_NONE; + } return yf; } static PyObject * -gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getrunning(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); if (gen->gi_frame_state == FRAME_EXECUTING) { Py_RETURN_TRUE; } @@ -722,8 +755,9 @@ gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) } static PyObject * -gen_getsuspended(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); return PyBool_FromLong(FRAME_STATE_SUSPENDED(gen->gi_frame_state)); } @@ -736,12 +770,13 @@ _gen_getframe(PyGenObject *gen, const char *const name) if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { Py_RETURN_NONE; } - return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject((_PyInterpreterFrame *)gen->gi_iframe)); + return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject(&gen->gi_iframe)); } static PyObject * -gen_getframe(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getframe(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); return _gen_getframe(gen, "gi_frame"); } @@ -755,22 +790,23 @@ _gen_getcode(PyGenObject *gen, const char *const name) } static PyObject * -gen_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getcode(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); return _gen_getcode(gen, "gi_code"); } static PyGetSetDef gen_getsetlist[] = { - {"__name__", (getter)gen_get_name, (setter)gen_set_name, + {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the generator")}, - {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the generator")}, - {"gi_yieldfrom", (getter)gen_getyieldfrom, NULL, + {"gi_yieldfrom", gen_getyieldfrom, NULL, PyDoc_STR("object being iterated by yield from, or None")}, - {"gi_running", (getter)gen_getrunning, NULL, NULL}, - {"gi_frame", (getter)gen_getframe, NULL, NULL}, - {"gi_suspended", (getter)gen_getsuspended, NULL, NULL}, - {"gi_code", (getter)gen_getcode, NULL, NULL}, + {"gi_running", gen_getrunning, NULL, NULL}, + {"gi_frame", gen_getframe, NULL, NULL}, + {"gi_suspended", gen_getsuspended, NULL, NULL}, + {"gi_code", gen_getcode, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -792,10 +828,11 @@ PyDoc_STRVAR(sizeof__doc__, "gen.__sizeof__() -> size of gen in memory, in bytes"); static PyMethodDef gen_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, send_doc}, - {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, - {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, + {"send", gen_send, METH_O, send_doc}, + {"throw", _PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, + {"close", gen_close, METH_NOARGS, close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; @@ -803,23 +840,22 @@ static PyAsyncMethods gen_as_async = { 0, /* am_await */ 0, /* am_aiter */ 0, /* am_anext */ - (sendfunc)PyGen_am_send, /* am_send */ + PyGen_am_send, /* am_send */ }; PyTypeObject PyGen_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "generator", /* tp_name */ - offsetof(PyGenObject, gi_iframe) + - offsetof(_PyInterpreterFrame, localsplus), /* tp_basicsize */ + offsetof(PyGenObject, gi_iframe.localsplus), /* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ - (destructor)gen_dealloc, /* tp_dealloc */ + gen_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ &gen_as_async, /* tp_as_async */ - (reprfunc)gen_repr, /* tp_repr */ + gen_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -831,12 +867,12 @@ PyTypeObject PyGen_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gen_traverse, /* tp_traverse */ + gen_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyGenObject, gi_weakreflist), /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)gen_iternext, /* tp_iternext */ + gen_iternext, /* tp_iternext */ gen_methods, /* tp_methods */ gen_memberlist, /* tp_members */ gen_getsetlist, /* tp_getset */ @@ -895,17 +931,18 @@ _Py_MakeCoro(PyFunctionObject *func) return make_gen(&PyGen_Type, func); } if (coro_flags == CO_ASYNC_GENERATOR) { - PyAsyncGenObject *o; - o = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); - if (o == NULL) { + PyAsyncGenObject *ag; + ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); + if (ag == NULL) { return NULL; } - o->ag_origin_or_finalizer = NULL; - o->ag_closed = 0; - o->ag_hooks_inited = 0; - o->ag_running_async = 0; - return (PyObject*)o; + ag->ag_origin_or_finalizer = NULL; + ag->ag_closed = 0; + ag->ag_hooks_inited = 0; + ag->ag_running_async = 0; + return (PyObject*)ag; } + assert (coro_flags == CO_COROUTINE); PyObject *coro = make_gen(&PyCoro_Type, func); if (!coro) { @@ -945,7 +982,7 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, /* Copy the frame */ assert(f->f_frame->frame_obj == NULL); assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; _PyFrame_Copy((_PyInterpreterFrame *)f->_f_frame_data, frame); gen->gi_frame_state = FRAME_CREATED; assert(frame->frame_obj == f); @@ -987,6 +1024,11 @@ typedef struct { PyCoroObject *cw_coroutine; } PyCoroWrapper; +#define _PyCoroWrapper_CAST(op) \ + (assert(Py_IS_TYPE((op), &_PyCoroWrapper_Type)), \ + _Py_CAST(PyCoroWrapper*, (op))) + + static int gen_is_coroutine(PyObject *o) { @@ -1043,20 +1085,21 @@ _PyCoro_GetAwaitableIter(PyObject *o) } PyErr_Format(PyExc_TypeError, - "object %.100s can't be used in 'await' expression", + "'%.100s' object can't be awaited", ot->tp_name); return NULL; } static PyObject * -coro_repr(PyCoroObject *coro) +coro_repr(PyObject *self) { + PyCoroObject *coro = _PyCoroObject_CAST(self); return PyUnicode_FromFormat("", coro->cr_qualname, coro); } static PyObject * -coro_await(PyCoroObject *coro) +coro_await(PyObject *coro) { PyCoroWrapper *cw = PyObject_GC_New(PyCoroWrapper, &_PyCoroWrapper_Type); if (cw == NULL) { @@ -1068,7 +1111,7 @@ coro_await(PyCoroObject *coro) } static PyObject * -coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored)) +coro_get_cr_await(PyObject *coro, void *Py_UNUSED(ignored)) { PyObject *yf = _PyGen_yf((PyGenObject *) coro); if (yf == NULL) @@ -1077,8 +1120,9 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored)) } static PyObject * -cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { + PyCoroObject *coro = _PyCoroObject_CAST(self); if (FRAME_STATE_SUSPENDED(coro->cr_frame_state)) { Py_RETURN_TRUE; } @@ -1086,8 +1130,9 @@ cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored)) } static PyObject * -cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getrunning(PyObject *self, void *Py_UNUSED(ignored)) { + PyCoroObject *coro = _PyCoroObject_CAST(self); if (coro->cr_frame_state == FRAME_EXECUTING) { Py_RETURN_TRUE; } @@ -1095,29 +1140,29 @@ cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) } static PyObject * -cr_getframe(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getframe(PyObject *coro, void *Py_UNUSED(ignored)) { - return _gen_getframe((PyGenObject *)coro, "cr_frame"); + return _gen_getframe(_PyGen_CAST(coro), "cr_frame"); } static PyObject * -cr_getcode(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getcode(PyObject *coro, void *Py_UNUSED(ignored)) { - return _gen_getcode((PyGenObject *)coro, "cr_code"); + return _gen_getcode(_PyGen_CAST(coro), "cr_code"); } static PyGetSetDef coro_getsetlist[] = { - {"__name__", (getter)gen_get_name, (setter)gen_set_name, + {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the coroutine")}, - {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the coroutine")}, - {"cr_await", (getter)coro_get_cr_await, NULL, + {"cr_await", coro_get_cr_await, NULL, PyDoc_STR("object being awaited on, or None")}, - {"cr_running", (getter)cr_getrunning, NULL, NULL}, - {"cr_frame", (getter)cr_getframe, NULL, NULL}, - {"cr_code", (getter)cr_getcode, NULL, NULL}, - {"cr_suspended", (getter)cr_getsuspended, NULL, NULL}, + {"cr_running", cr_getrunning, NULL, NULL}, + {"cr_frame", cr_getframe, NULL, NULL}, + {"cr_code", cr_getcode, NULL, NULL}, + {"cr_suspended", cr_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1144,33 +1189,33 @@ PyDoc_STRVAR(coro_close_doc, "close() -> raise GeneratorExit inside coroutine."); static PyMethodDef coro_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, coro_send_doc}, + {"send", gen_send, METH_O, coro_send_doc}, {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, - {"close",(PyCFunction)gen_close, METH_NOARGS, coro_close_doc}, + {"close", gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; static PyAsyncMethods coro_as_async = { - (unaryfunc)coro_await, /* am_await */ + coro_await, /* am_await */ 0, /* am_aiter */ 0, /* am_anext */ - (sendfunc)PyGen_am_send, /* am_send */ + PyGen_am_send, /* am_send */ }; PyTypeObject PyCoro_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "coroutine", /* tp_name */ - offsetof(PyCoroObject, cr_iframe) + - offsetof(_PyInterpreterFrame, localsplus), /* tp_basicsize */ + offsetof(PyCoroObject, cr_iframe.localsplus),/* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ - (destructor)gen_dealloc, /* tp_dealloc */ + gen_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ &coro_as_async, /* tp_as_async */ - (reprfunc)coro_repr, /* tp_repr */ + coro_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1182,7 +1227,7 @@ PyTypeObject PyCoro_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gen_traverse, /* tp_traverse */ + gen_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyCoroObject, cr_weakreflist), /* tp_weaklistoffset */ @@ -1212,49 +1257,55 @@ PyTypeObject PyCoro_Type = { }; static void -coro_wrapper_dealloc(PyCoroWrapper *cw) +coro_wrapper_dealloc(PyObject *self) { + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); _PyObject_GC_UNTRACK((PyObject *)cw); Py_CLEAR(cw->cw_coroutine); PyObject_GC_Del(cw); } static PyObject * -coro_wrapper_iternext(PyCoroWrapper *cw) +coro_wrapper_iternext(PyObject *self) { - return gen_iternext((PyGenObject *)cw->cw_coroutine); + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); + return gen_iternext((PyObject *)cw->cw_coroutine); } static PyObject * -coro_wrapper_send(PyCoroWrapper *cw, PyObject *arg) +coro_wrapper_send(PyObject *self, PyObject *arg) { - return gen_send((PyGenObject *)cw->cw_coroutine, arg); + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); + return gen_send((PyObject *)cw->cw_coroutine, arg); } static PyObject * -coro_wrapper_throw(PyCoroWrapper *cw, PyObject *const *args, Py_ssize_t nargs) +coro_wrapper_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); return gen_throw((PyGenObject *)cw->cw_coroutine, args, nargs); } static PyObject * -coro_wrapper_close(PyCoroWrapper *cw, PyObject *args) +coro_wrapper_close(PyObject *self, PyObject *args) { - return gen_close((PyGenObject *)cw->cw_coroutine, args); + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); + return gen_close((PyObject *)cw->cw_coroutine, args); } static int -coro_wrapper_traverse(PyCoroWrapper *cw, visitproc visit, void *arg) +coro_wrapper_traverse(PyObject *self, visitproc visit, void *arg) { + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); Py_VISIT((PyObject *)cw->cw_coroutine); return 0; } static PyMethodDef coro_wrapper_methods[] = { - {"send",(PyCFunction)coro_wrapper_send, METH_O, coro_send_doc}, - {"throw",_PyCFunction_CAST(coro_wrapper_throw), - METH_FASTCALL, coro_throw_doc}, - {"close",(PyCFunction)coro_wrapper_close, METH_NOARGS, coro_close_doc}, + {"send", coro_wrapper_send, METH_O, coro_send_doc}, + {"throw", _PyCFunction_CAST(coro_wrapper_throw), METH_FASTCALL, + coro_throw_doc}, + {"close", coro_wrapper_close, METH_NOARGS, coro_close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -1263,7 +1314,7 @@ PyTypeObject _PyCoroWrapper_Type = { "coroutine_wrapper", sizeof(PyCoroWrapper), /* tp_basicsize */ 0, /* tp_itemsize */ - (destructor)coro_wrapper_dealloc, /* destructor tp_dealloc */ + coro_wrapper_dealloc, /* destructor tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1280,12 +1331,12 @@ PyTypeObject _PyCoroWrapper_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "A wrapper object implementing __await__ for coroutines.", - (traverseproc)coro_wrapper_traverse, /* tp_traverse */ + coro_wrapper_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)coro_wrapper_iternext, /* tp_iternext */ + coro_wrapper_iternext, /* tp_iternext */ coro_wrapper_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1379,6 +1430,9 @@ typedef struct PyAsyncGenASend { AwaitableState ags_state; } PyAsyncGenASend; +#define _PyAsyncGenASend_CAST(op) \ + _Py_CAST(PyAsyncGenASend*, (op)) + typedef struct PyAsyncGenAThrow { PyObject_HEAD @@ -1400,19 +1454,24 @@ typedef struct _PyAsyncGenWrappedValue { #define _PyAsyncGenWrappedValue_CheckExact(o) \ Py_IS_TYPE(o, &_PyAsyncGenWrappedValue_Type) +#define _PyAsyncGenWrappedValue_CAST(op) \ + (assert(_PyAsyncGenWrappedValue_CheckExact(op)), \ + _Py_CAST(_PyAsyncGenWrappedValue*, (op))) static int -async_gen_traverse(PyAsyncGenObject *gen, visitproc visit, void *arg) +async_gen_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(gen->ag_origin_or_finalizer); - return gen_traverse((PyGenObject*)gen, visit, arg); + PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); + Py_VISIT(ag->ag_origin_or_finalizer); + return gen_traverse((PyObject*)ag, visit, arg); } static PyObject * -async_gen_repr(PyAsyncGenObject *o) +async_gen_repr(PyObject *self) { + PyAsyncGenObject *o = _PyAsyncGenObject_CAST(self); return PyUnicode_FromFormat("", o->ag_qualname, o); } @@ -1456,12 +1515,13 @@ async_gen_init_hooks(PyAsyncGenObject *o) static PyObject * -async_gen_anext(PyAsyncGenObject *o) +async_gen_anext(PyObject *self) { - if (async_gen_init_hooks(o)) { + PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); + if (async_gen_init_hooks(ag)) { return NULL; } - return async_gen_asend_new(o, NULL); + return async_gen_asend_new(ag, NULL); } @@ -1502,20 +1562,21 @@ async_gen_athrow(PyAsyncGenObject *o, PyObject *args) } static PyObject * -ag_getframe(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) +ag_getframe(PyObject *ag, void *Py_UNUSED(ignored)) { return _gen_getframe((PyGenObject *)ag, "ag_frame"); } static PyObject * -ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) +ag_getcode(PyObject *gen, void *Py_UNUSED(ignored)) { - return _gen_getcode(gen, "ag_code"); + return _gen_getcode((PyGenObject*)gen, "ag_code"); } static PyObject * -ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) +ag_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { + PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); if (FRAME_STATE_SUSPENDED(ag->ag_frame_state)) { Py_RETURN_TRUE; } @@ -1523,15 +1584,15 @@ ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) } static PyGetSetDef async_gen_getsetlist[] = { - {"__name__", (getter)gen_get_name, (setter)gen_set_name, + {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the async generator")}, - {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the async generator")}, - {"ag_await", (getter)coro_get_cr_await, NULL, + {"ag_await", coro_get_cr_await, NULL, PyDoc_STR("object being awaited on, or None")}, - {"ag_frame", (getter)ag_getframe, NULL, NULL}, - {"ag_code", (getter)ag_getcode, NULL, NULL}, - {"ag_suspended", (getter)ag_getsuspended, NULL, NULL}, + {"ag_frame", ag_getframe, NULL, NULL}, + {"ag_code", ag_getcode, NULL, NULL}, + {"ag_suspended", ag_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1569,24 +1630,23 @@ static PyMethodDef async_gen_methods[] = { static PyAsyncMethods async_gen_as_async = { 0, /* am_await */ PyObject_SelfIter, /* am_aiter */ - (unaryfunc)async_gen_anext, /* am_anext */ - (sendfunc)PyGen_am_send, /* am_send */ + async_gen_anext, /* am_anext */ + PyGen_am_send, /* am_send */ }; PyTypeObject PyAsyncGen_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "async_generator", /* tp_name */ - offsetof(PyAsyncGenObject, ag_iframe) + - offsetof(_PyInterpreterFrame, localsplus), /* tp_basicsize */ + offsetof(PyAsyncGenObject, ag_iframe.localsplus), /* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ - (destructor)gen_dealloc, /* tp_dealloc */ + gen_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ &async_gen_as_async, /* tp_as_async */ - (reprfunc)async_gen_repr, /* tp_repr */ + async_gen_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1598,7 +1658,7 @@ PyTypeObject PyAsyncGen_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_traverse, /* tp_traverse */ + async_gen_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyAsyncGenObject, ag_weakreflist), /* tp_weaklistoffset */ @@ -1628,58 +1688,21 @@ PyTypeObject PyAsyncGen_Type = { }; -#ifdef WITH_FREELISTS -static struct _Py_async_gen_state * -get_async_gen_state(void) -{ - _PyFreeListState *state = _PyFreeListState_GET(); - return &state->async_gens; -} -#endif - - PyObject * PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) { - PyAsyncGenObject *o; - o = (PyAsyncGenObject *)gen_new_with_qualname( - &PyAsyncGen_Type, f, name, qualname); - if (o == NULL) { + PyAsyncGenObject *ag; + ag = (PyAsyncGenObject *)gen_new_with_qualname(&PyAsyncGen_Type, f, + name, qualname); + if (ag == NULL) { return NULL; } - o->ag_origin_or_finalizer = NULL; - o->ag_closed = 0; - o->ag_hooks_inited = 0; - o->ag_running_async = 0; - return (PyObject*)o; -} - - -void -_PyAsyncGen_ClearFreeLists(_PyFreeListState *freelist_state, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = &freelist_state->async_gens; - - while (state->value_numfree > 0) { - _PyAsyncGenWrappedValue *o; - o = state->value_freelist[--state->value_numfree]; - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - PyObject_GC_Del(o); - } - while (state->asend_numfree > 0) { - PyAsyncGenASend *o; - o = state->asend_freelist[--state->asend_numfree]; - assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type)); - PyObject_GC_Del(o); - } - - if (is_finalization) { - state->value_numfree = -1; - state->asend_numfree = -1; - } -#endif + ag->ag_origin_or_finalizer = NULL; + ag->ag_closed = 0; + ag->ag_hooks_inited = 0; + ag->ag_running_async = 0; + return (PyObject*)ag; } static PyObject * @@ -1716,43 +1739,38 @@ async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) static void -async_gen_asend_dealloc(PyAsyncGenASend *o) +async_gen_asend_dealloc(PyObject *self) { - if (PyObject_CallFinalizerFromDealloc((PyObject *)o)) { + assert(PyAsyncGenASend_CheckExact(self)); + PyAsyncGenASend *ags = _PyAsyncGenASend_CAST(self); + + if (PyObject_CallFinalizerFromDealloc(self)) { return; } - _PyObject_GC_UNTRACK((PyObject *)o); - Py_CLEAR(o->ags_gen); - Py_CLEAR(o->ags_sendval); -#ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = get_async_gen_state(); - if (state->asend_numfree >= 0 && state->asend_numfree < _PyAsyncGen_MAXFREELIST) { - assert(PyAsyncGenASend_CheckExact(o)); - _PyGC_CLEAR_FINALIZED((PyObject *)o); - state->asend_freelist[state->asend_numfree++] = o; - } - else -#endif - { - PyObject_GC_Del(o); - } + _PyObject_GC_UNTRACK(self); + Py_CLEAR(ags->ags_gen); + Py_CLEAR(ags->ags_sendval); + + _PyGC_CLEAR_FINALIZED(self); + + _Py_FREELIST_FREE(async_gen_asends, self, PyObject_GC_Del); } static int -async_gen_asend_traverse(PyAsyncGenASend *o, visitproc visit, void *arg) +async_gen_asend_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(o->ags_gen); - Py_VISIT(o->ags_sendval); + PyAsyncGenASend *ags = _PyAsyncGenASend_CAST(self); + Py_VISIT(ags->ags_gen); + Py_VISIT(ags->ags_sendval); return 0; } static PyObject * -async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) +async_gen_asend_send(PyObject *self, PyObject *arg) { - PyObject *result; - + PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); if (o->ags_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( PyExc_RuntimeError, @@ -1762,6 +1780,7 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) if (o->ags_state == AWAITABLE_STATE_INIT) { if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, "anext(): asynchronous generator is already running"); @@ -1775,7 +1794,7 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) } o->ags_gen->ag_running_async = 1; - result = gen_send((PyGenObject*)o->ags_gen, arg); + PyObject *result = gen_send((PyObject*)o->ags_gen, arg); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { @@ -1787,16 +1806,16 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) static PyObject * -async_gen_asend_iternext(PyAsyncGenASend *o) +async_gen_asend_iternext(PyObject *ags) { - return async_gen_asend_send(o, NULL); + return async_gen_asend_send(ags, NULL); } static PyObject * -async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t nargs) +async_gen_asend_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *result; + PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); if (o->ags_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( @@ -1805,10 +1824,24 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg return NULL; } - result = gen_throw((PyGenObject*)o->ags_gen, args, nargs); + if (o->ags_state == AWAITABLE_STATE_INIT) { + if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; + PyErr_SetString( + PyExc_RuntimeError, + "anext(): asynchronous generator is already running"); + return NULL; + } + + o->ags_state = AWAITABLE_STATE_ITER; + o->ags_gen->ag_running_async = 1; + } + + PyObject *result = gen_throw((PyGenObject*)o->ags_gen, args, nargs); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { + o->ags_gen->ag_running_async = 0; o->ags_state = AWAITABLE_STATE_CLOSED; } @@ -1817,24 +1850,43 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg static PyObject * -async_gen_asend_close(PyAsyncGenASend *o, PyObject *args) +async_gen_asend_close(PyObject *self, PyObject *args) { - o->ags_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); + if (o->ags_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + + PyObject *result = async_gen_asend_throw(self, &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } + + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; } static void -async_gen_asend_finalize(PyAsyncGenASend *o) +async_gen_asend_finalize(PyObject *self) { - if (o->ags_state == AWAITABLE_STATE_INIT) { - _PyErr_WarnUnawaitedAgenMethod(o->ags_gen, &_Py_ID(asend)); + PyAsyncGenASend *ags = _PyAsyncGenASend_CAST(self); + if (ags->ags_state == AWAITABLE_STATE_INIT) { + _PyErr_WarnUnawaitedAgenMethod(ags->ags_gen, &_Py_ID(asend)); } } static PyMethodDef async_gen_asend_methods[] = { - {"send", (PyCFunction)async_gen_asend_send, METH_O, send_doc}, + {"send", async_gen_asend_send, METH_O, send_doc}, {"throw", _PyCFunction_CAST(async_gen_asend_throw), METH_FASTCALL, throw_doc}, - {"close", (PyCFunction)async_gen_asend_close, METH_NOARGS, close_doc}, + {"close", async_gen_asend_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -1853,7 +1905,7 @@ PyTypeObject _PyAsyncGenASend_Type = { sizeof(PyAsyncGenASend), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)async_gen_asend_dealloc, /* tp_dealloc */ + async_gen_asend_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1870,12 +1922,12 @@ PyTypeObject _PyAsyncGenASend_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_asend_traverse, /* tp_traverse */ + async_gen_asend_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)async_gen_asend_iternext, /* tp_iternext */ + async_gen_asend_iternext, /* tp_iternext */ async_gen_asend_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1887,38 +1939,27 @@ PyTypeObject _PyAsyncGenASend_Type = { 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ - .tp_finalize = (destructor)async_gen_asend_finalize, + .tp_finalize = async_gen_asend_finalize, }; static PyObject * async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { - PyAsyncGenASend *o; -#ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = get_async_gen_state(); - if (state->asend_numfree > 0) { - state->asend_numfree--; - o = state->asend_freelist[state->asend_numfree]; - _Py_NewReference((PyObject *)o); - } - else -#endif - { - o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); - if (o == NULL) { + PyAsyncGenASend *ags = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends); + if (ags == NULL) { + ags = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); + if (ags == NULL) { return NULL; } } - o->ags_gen = (PyAsyncGenObject*)Py_NewRef(gen); - - o->ags_sendval = Py_XNewRef(sendval); + ags->ags_gen = (PyAsyncGenObject*)Py_NewRef(gen); + ags->ags_sendval = Py_XNewRef(sendval); + ags->ags_state = AWAITABLE_STATE_INIT; - o->ags_state = AWAITABLE_STATE_INIT; - - _PyObject_GC_TRACK((PyObject*)o); - return (PyObject*)o; + _PyObject_GC_TRACK((PyObject*)ags); + return (PyObject*)ags; } @@ -1926,30 +1967,20 @@ async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) static void -async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) +async_gen_wrapped_val_dealloc(PyObject *self) { - _PyObject_GC_UNTRACK((PyObject *)o); - Py_CLEAR(o->agw_val); -#ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = get_async_gen_state(); - if (state->value_numfree >= 0 && state->value_numfree < _PyAsyncGen_MAXFREELIST) { - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - state->value_freelist[state->value_numfree++] = o; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - PyObject_GC_Del(o); - } + _PyAsyncGenWrappedValue *agw = _PyAsyncGenWrappedValue_CAST(self); + _PyObject_GC_UNTRACK(self); + Py_CLEAR(agw->agw_val); + _Py_FREELIST_FREE(async_gens, self, PyObject_GC_Del); } static int -async_gen_wrapped_val_traverse(_PyAsyncGenWrappedValue *o, - visitproc visit, void *arg) +async_gen_wrapped_val_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(o->agw_val); + _PyAsyncGenWrappedValue *agw = _PyAsyncGenWrappedValue_CAST(self); + Py_VISIT(agw->agw_val); return 0; } @@ -1960,7 +1991,7 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { sizeof(_PyAsyncGenWrappedValue), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)async_gen_wrapped_val_dealloc, /* tp_dealloc */ + async_gen_wrapped_val_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1977,7 +2008,7 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_wrapped_val_traverse, /* tp_traverse */ + async_gen_wrapped_val_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -2000,27 +2031,17 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { PyObject * _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) { - _PyAsyncGenWrappedValue *o; assert(val); -#ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = get_async_gen_state(); - if (state->value_numfree > 0) { - state->value_numfree--; - o = state->value_freelist[state->value_numfree]; - OBJECT_STAT_INC(from_freelist); - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - _Py_NewReference((PyObject*)o); - } - else -#endif - { + _PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens); + if (o == NULL) { o = PyObject_GC_New(_PyAsyncGenWrappedValue, &_PyAsyncGenWrappedValue_Type); if (o == NULL) { return NULL; } } + assert(_PyAsyncGenWrappedValue_CheckExact(o)); o->agw_val = Py_NewRef(val); _PyObject_GC_TRACK((PyObject*)o); return (PyObject*)o; @@ -2029,34 +2050,40 @@ _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) /* ---------- Async Generator AThrow awaitable ------------ */ +#define _PyAsyncGenAThrow_CAST(op) \ + (assert(Py_IS_TYPE((op), &_PyAsyncGenAThrow_Type)), \ + _Py_CAST(PyAsyncGenAThrow*, (op))) static void -async_gen_athrow_dealloc(PyAsyncGenAThrow *o) +async_gen_athrow_dealloc(PyObject *self) { - if (PyObject_CallFinalizerFromDealloc((PyObject *)o)) { + PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); + if (PyObject_CallFinalizerFromDealloc(self)) { return; } - _PyObject_GC_UNTRACK((PyObject *)o); - Py_CLEAR(o->agt_gen); - Py_CLEAR(o->agt_args); - PyObject_GC_Del(o); + _PyObject_GC_UNTRACK(self); + Py_CLEAR(agt->agt_gen); + Py_CLEAR(agt->agt_args); + PyObject_GC_Del(self); } static int -async_gen_athrow_traverse(PyAsyncGenAThrow *o, visitproc visit, void *arg) +async_gen_athrow_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(o->agt_gen); - Py_VISIT(o->agt_args); + PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); + Py_VISIT(agt->agt_gen); + Py_VISIT(agt->agt_args); return 0; } static PyObject * -async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) +async_gen_athrow_send(PyObject *self, PyObject *arg) { - PyGenObject *gen = (PyGenObject*)o->agt_gen; + PyAsyncGenAThrow *o = _PyAsyncGenAThrow_CAST(self); + PyGenObject *gen = _PyGen_CAST(o->agt_gen); PyObject *retval; if (o->agt_state == AWAITABLE_STATE_CLOSED) { @@ -2139,7 +2166,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) assert(o->agt_state == AWAITABLE_STATE_ITER); - retval = gen_send((PyGenObject *)gen, arg); + retval = gen_send((PyObject *)gen, arg); if (o->agt_args) { return async_gen_unwrap_value(o->agt_gen, retval); } else { @@ -2186,9 +2213,9 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) static PyObject * -async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t nargs) +async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *retval; + PyAsyncGenAThrow *o = _PyAsyncGenAThrow_CAST(self); if (o->agt_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( @@ -2197,10 +2224,36 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na return NULL; } - retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); + if (o->agt_state == AWAITABLE_STATE_INIT) { + if (o->agt_gen->ag_running_async) { + o->agt_state = AWAITABLE_STATE_CLOSED; + if (o->agt_args == NULL) { + PyErr_SetString( + PyExc_RuntimeError, + "aclose(): asynchronous generator is already running"); + } + else { + PyErr_SetString( + PyExc_RuntimeError, + "athrow(): asynchronous generator is already running"); + } + return NULL; + } + + o->agt_state = AWAITABLE_STATE_ITER; + o->agt_gen->ag_running_async = 1; + } + + PyObject *retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); if (o->agt_args) { - return async_gen_unwrap_value(o->agt_gen, retval); - } else { + retval = async_gen_unwrap_value(o->agt_gen, retval); + if (retval == NULL) { + o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; + } + return retval; + } + else { /* aclose() mode */ if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { o->agt_gen->ag_running_async = 0; @@ -2209,6 +2262,10 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; } + if (retval == NULL) { + o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; + } if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { @@ -2226,17 +2283,35 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na static PyObject * -async_gen_athrow_iternext(PyAsyncGenAThrow *o) +async_gen_athrow_iternext(PyObject *agt) { - return async_gen_athrow_send(o, Py_None); + return async_gen_athrow_send(agt, Py_None); } static PyObject * -async_gen_athrow_close(PyAsyncGenAThrow *o, PyObject *args) +async_gen_athrow_close(PyObject *self, PyObject *args) { - o->agt_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); + if (agt->agt_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + PyObject *result = async_gen_athrow_throw((PyObject*)agt, + &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } else { + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; + } } @@ -2250,10 +2325,10 @@ async_gen_athrow_finalize(PyAsyncGenAThrow *o) } static PyMethodDef async_gen_athrow_methods[] = { - {"send", (PyCFunction)async_gen_athrow_send, METH_O, send_doc}, + {"send", async_gen_athrow_send, METH_O, send_doc}, {"throw", _PyCFunction_CAST(async_gen_athrow_throw), METH_FASTCALL, throw_doc}, - {"close", (PyCFunction)async_gen_athrow_close, METH_NOARGS, close_doc}, + {"close", async_gen_athrow_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -2272,7 +2347,7 @@ PyTypeObject _PyAsyncGenAThrow_Type = { sizeof(PyAsyncGenAThrow), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)async_gen_athrow_dealloc, /* tp_dealloc */ + async_gen_athrow_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -2289,12 +2364,12 @@ PyTypeObject _PyAsyncGenAThrow_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_athrow_traverse, /* tp_traverse */ + async_gen_athrow_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)async_gen_athrow_iternext, /* tp_iternext */ + async_gen_athrow_iternext, /* tp_iternext */ async_gen_athrow_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c deleted file mode 100644 index 16e27b64c0c9c2..00000000000000 --- a/Objects/interpreteridobject.c +++ /dev/null @@ -1,294 +0,0 @@ -/* InterpreterID object */ - -#include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_interp.h" // _PyInterpreterState_LookUpID() -#include "interpreteridobject.h" - - -typedef struct interpid { - PyObject_HEAD - int64_t id; -} interpid; - -static interpid * -newinterpid(PyTypeObject *cls, int64_t id, int force) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp == NULL) { - if (force) { - PyErr_Clear(); - } - else { - return NULL; - } - } - - if (interp != NULL) { - if (_PyInterpreterState_IDIncref(interp) < 0) { - return NULL; - } - } - - interpid *self = PyObject_New(interpid, cls); - if (self == NULL) { - if (interp != NULL) { - _PyInterpreterState_IDDecref(interp); - } - return NULL; - } - self->id = id; - - return self; -} - -static int -interp_id_converter(PyObject *arg, void *ptr) -{ - int64_t id; - if (PyObject_TypeCheck(arg, &PyInterpreterID_Type)) { - id = ((interpid *)arg)->id; - } - else if (_PyIndex_Check(arg)) { - id = PyLong_AsLongLong(arg); - if (id == -1 && PyErr_Occurred()) { - return 0; - } - if (id < 0) { - PyErr_Format(PyExc_ValueError, - "interpreter ID must be a non-negative int, got %R", arg); - return 0; - } - } - else { - PyErr_Format(PyExc_TypeError, - "interpreter ID must be an int, got %.100s", - Py_TYPE(arg)->tp_name); - return 0; - } - *(int64_t *)ptr = id; - return 1; -} - -static PyObject * -interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", "force", NULL}; - int64_t id; - int force = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&|$p:InterpreterID.__init__", kwlist, - interp_id_converter, &id, &force)) { - return NULL; - } - - return (PyObject *)newinterpid(cls, id, force); -} - -static void -interpid_dealloc(PyObject *v) -{ - int64_t id = ((interpid *)v)->id; - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp != NULL) { - _PyInterpreterState_IDDecref(interp); - } - else { - // already deleted - PyErr_Clear(); - } - Py_TYPE(v)->tp_free(v); -} - -static PyObject * -interpid_repr(PyObject *self) -{ - PyTypeObject *type = Py_TYPE(self); - const char *name = _PyType_Name(type); - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%s(%" PRId64 ")", name, id->id); -} - -static PyObject * -interpid_str(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%" PRId64 "", id->id); -} - -static PyObject * -interpid_int(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyLong_FromLongLong(id->id); -} - -static PyNumberMethods interpid_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - 0, /* nb_bool */ - 0, /* nb_invert */ - 0, /* nb_lshift */ - 0, /* nb_rshift */ - 0, /* nb_and */ - 0, /* nb_xor */ - 0, /* nb_or */ - (unaryfunc)interpid_int, /* nb_int */ - 0, /* nb_reserved */ - 0, /* nb_float */ - - 0, /* nb_inplace_add */ - 0, /* nb_inplace_subtract */ - 0, /* nb_inplace_multiply */ - 0, /* nb_inplace_remainder */ - 0, /* nb_inplace_power */ - 0, /* nb_inplace_lshift */ - 0, /* nb_inplace_rshift */ - 0, /* nb_inplace_and */ - 0, /* nb_inplace_xor */ - 0, /* nb_inplace_or */ - - 0, /* nb_floor_divide */ - 0, /* nb_true_divide */ - 0, /* nb_inplace_floor_divide */ - 0, /* nb_inplace_true_divide */ - - (unaryfunc)interpid_int, /* nb_index */ -}; - -static Py_hash_t -interpid_hash(PyObject *self) -{ - interpid *id = (interpid *)self; - PyObject *obj = PyLong_FromLongLong(id->id); - if (obj == NULL) { - return -1; - } - Py_hash_t hash = PyObject_Hash(obj); - Py_DECREF(obj); - return hash; -} - -static PyObject * -interpid_richcompare(PyObject *self, PyObject *other, int op) -{ - if (op != Py_EQ && op != Py_NE) { - Py_RETURN_NOTIMPLEMENTED; - } - - if (!PyObject_TypeCheck(self, &PyInterpreterID_Type)) { - Py_RETURN_NOTIMPLEMENTED; - } - - interpid *id = (interpid *)self; - int equal; - if (PyObject_TypeCheck(other, &PyInterpreterID_Type)) { - interpid *otherid = (interpid *)other; - equal = (id->id == otherid->id); - } - else if (PyLong_CheckExact(other)) { - /* Fast path */ - int overflow; - long long otherid = PyLong_AsLongLongAndOverflow(other, &overflow); - if (otherid == -1 && PyErr_Occurred()) { - return NULL; - } - equal = !overflow && (otherid >= 0) && (id->id == otherid); - } - else if (PyNumber_Check(other)) { - PyObject *pyid = PyLong_FromLongLong(id->id); - if (pyid == NULL) { - return NULL; - } - PyObject *res = PyObject_RichCompare(pyid, other, op); - Py_DECREF(pyid); - return res; - } - else { - Py_RETURN_NOTIMPLEMENTED; - } - - if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - -PyDoc_STRVAR(interpid_doc, -"A interpreter ID identifies a interpreter and may be used as an int."); - -PyTypeObject PyInterpreterID_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "InterpreterID", /* tp_name */ - sizeof(interpid), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)interpid_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)interpid_repr, /* tp_repr */ - &interpid_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - interpid_hash, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)interpid_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - interpid_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - interpid_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - interpid_new, /* tp_new */ -}; - -PyObject *PyInterpreterID_New(int64_t id) -{ - return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0); -} - -PyObject * -PyInterpreterState_GetIDObject(PyInterpreterState *interp) -{ - if (_PyInterpreterState_IDInitref(interp) != 0) { - return NULL; - }; - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0); -} - -PyInterpreterState * -PyInterpreterID_LookUp(PyObject *requested_id) -{ - int64_t id; - if (!interp_id_converter(requested_id, &id)) { - return NULL; - } - return _PyInterpreterState_LookUpID(id); -} diff --git a/Objects/listobject.c b/Objects/listobject.c index 7fdb91eab890b5..930aefde325a7c 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3,12 +3,16 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_dict.h" // _PyDictViewObject +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list -#include "pycore_list.h" // struct _Py_list_state, _PyListIterObject +#include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_setobject.h" // _PySet_NextEntry() #include /*[clinic input] @@ -20,16 +24,50 @@ class list "PyListObject *" "&PyList_Type" _Py_DECLARE_STR(list_err, "list index out of range"); -#ifdef WITH_FREELISTS -static struct _Py_list_state * -get_list_state(void) +#ifdef Py_GIL_DISABLED +typedef struct { + Py_ssize_t allocated; + PyObject *ob_item[]; +} _PyListArray; + +static _PyListArray * +list_allocate_array(size_t capacity) +{ + if (capacity > PY_SSIZE_T_MAX/sizeof(PyObject*) - 1) { + return NULL; + } + _PyListArray *array = PyMem_Malloc(sizeof(_PyListArray) + capacity * sizeof(PyObject *)); + if (array == NULL) { + return NULL; + } + array->allocated = capacity; + return array; +} + +static Py_ssize_t +list_capacity(PyObject **items) { - _PyFreeListState *state = _PyFreeListState_GET(); - assert(state != NULL); - return &state->lists; + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + return array->allocated; } #endif +static void +free_list_items(PyObject** items, bool use_qsbr) +{ +#ifdef Py_GIL_DISABLED + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + if (use_qsbr) { + _PyMem_FreeDelayed(array); + } + else { + PyMem_Free(array); + } +#else + PyMem_Free(items); +#endif +} + /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -46,8 +84,7 @@ get_list_state(void) static int list_resize(PyListObject *self, Py_ssize_t newsize) { - PyObject **items; - size_t new_allocated, num_allocated_bytes; + size_t new_allocated, target_bytes; Py_ssize_t allocated = self->allocated; /* Bypass realloc() when a previous overallocation is large enough @@ -79,9 +116,37 @@ list_resize(PyListObject *self, Py_ssize_t newsize) if (newsize == 0) new_allocated = 0; + +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(new_allocated); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + PyObject **old_items = self->ob_item; + if (self->ob_item) { + if (new_allocated < (size_t)allocated) { + target_bytes = new_allocated * sizeof(PyObject*); + } + else { + target_bytes = allocated * sizeof(PyObject*); + } + memcpy(array->ob_item, self->ob_item, target_bytes); + } + if (new_allocated > (size_t)allocated) { + memset(array->ob_item + allocated, 0, sizeof(PyObject *) * (new_allocated - allocated)); + } + _Py_atomic_store_ptr_release(&self->ob_item, &array->ob_item); + self->allocated = new_allocated; + Py_SET_SIZE(self, newsize); + if (old_items != NULL) { + free_list_items(old_items, _PyObject_GC_IS_SHARED(self)); + } +#else + PyObject **items; if (new_allocated <= (size_t)PY_SSIZE_T_MAX / sizeof(PyObject *)) { - num_allocated_bytes = new_allocated * sizeof(PyObject *); - items = (PyObject **)PyMem_Realloc(self->ob_item, num_allocated_bytes); + target_bytes = new_allocated * sizeof(PyObject *); + items = (PyObject **)PyMem_Realloc(self->ob_item, target_bytes); } else { // integer overflow @@ -94,12 +159,14 @@ list_resize(PyListObject *self, Py_ssize_t newsize) self->ob_item = items; Py_SET_SIZE(self, newsize); self->allocated = new_allocated; +#endif return 0; } static int list_preallocate_exact(PyListObject *self, Py_ssize_t size) { + PyObject **items; assert(self->ob_item == NULL); assert(size > 0); @@ -109,65 +176,46 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) * allocated size up to the nearest even number. */ size = (size + 1) & ~(size_t)1; - PyObject **items = PyMem_New(PyObject*, size); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + items = array->ob_item; + memset(items, 0, size * sizeof(PyObject *)); +#else + items = PyMem_New(PyObject*, size); if (items == NULL) { PyErr_NoMemory(); return -1; } - self->ob_item = items; +#endif + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, items); self->allocated = size; return 0; } -void -_PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_list_state *state = &freelist_state->lists; - while (state->numfree > 0) { - PyListObject *op = state->free_list[--state->numfree]; - assert(PyList_CheckExact(op)); - PyObject_GC_Del(op); - } - if (is_finalization) { - state->numfree = -1; - } -#endif -} - /* Print summary info about the state of the optimized allocator */ void _PyList_DebugMallocStats(FILE *out) { -#ifdef WITH_FREELISTS - struct _Py_list_state *state = get_list_state(); _PyDebugAllocatorStats(out, "free PyListObject", - state->numfree, sizeof(PyListObject)); -#endif + _Py_FREELIST_SIZE(lists), + sizeof(PyListObject)); } PyObject * PyList_New(Py_ssize_t size) { - PyListObject *op; - if (size < 0) { PyErr_BadInternalCall(); return NULL; } -#ifdef WITH_FREELISTS - struct _Py_list_state *state = get_list_state(); - if (PyList_MAXFREELIST && state->numfree > 0) { - state->numfree--; - op = state->free_list[state->numfree]; - OBJECT_STAT_INC(from_freelist); - _Py_NewReference((PyObject *)op); - } - else -#endif - { + PyListObject *op = _Py_FREELIST_POP(PyListObject, lists); + if (op == NULL) { op = PyObject_GC_New(PyListObject, &PyList_Type); if (op == NULL) { return NULL; @@ -177,7 +225,17 @@ PyList_New(Py_ssize_t size) op->ob_item = NULL; } else { +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + memset(&array->ob_item, 0, size * sizeof(PyObject *)); + op->ob_item = array->ob_item; +#else op->ob_item = (PyObject **) PyMem_Calloc(size, sizeof(PyObject *)); +#endif if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); @@ -198,11 +256,20 @@ list_new_prealloc(Py_ssize_t size) return NULL; } assert(op->ob_item == NULL); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + op->ob_item = array->ob_item; +#else op->ob_item = PyMem_New(PyObject *, size); if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); } +#endif op->allocated = size; return (PyObject *) op; } @@ -232,6 +299,67 @@ valid_index(Py_ssize_t i, Py_ssize_t limit) return (size_t) i < (size_t) limit; } +#ifdef Py_GIL_DISABLED + +static PyObject * +list_item_impl(PyListObject *self, Py_ssize_t idx) +{ + PyObject *item = NULL; + Py_BEGIN_CRITICAL_SECTION(self); + if (!_PyObject_GC_IS_SHARED(self)) { + _PyObject_GC_SET_SHARED(self); + } + Py_ssize_t size = Py_SIZE(self); + if (!valid_index(idx, size)) { + goto exit; + } +#ifdef Py_GIL_DISABLED + item = _Py_NewRefWithLock(self->ob_item[idx]); +#else + item = Py_NewRef(self->ob_item[idx]); +#endif +exit: + Py_END_CRITICAL_SECTION(); + return item; +} + +static inline PyObject* +list_get_item_ref(PyListObject *op, Py_ssize_t i) +{ + if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { + return list_item_impl(op, i); + } + // Need atomic operation for the getting size. + Py_ssize_t size = PyList_GET_SIZE(op); + if (!valid_index(i, size)) { + return NULL; + } + PyObject **ob_item = _Py_atomic_load_ptr(&op->ob_item); + if (ob_item == NULL) { + return NULL; + } + Py_ssize_t cap = list_capacity(ob_item); + assert(cap != -1 && cap >= size); + if (!valid_index(i, cap)) { + return NULL; + } + PyObject *item = _Py_TryXGetRef(&ob_item[i]); + if (item == NULL) { + return list_item_impl(op, i); + } + return item; +} +#else +static inline PyObject* +list_get_item_ref(PyListObject *op, Py_ssize_t i) +{ + if (!valid_index(i, Py_SIZE(op))) { + return NULL; + } + return Py_NewRef(PyList_GET_ITEM(op, i)); +} +#endif + PyObject * PyList_GetItem(PyObject *op, Py_ssize_t i) { @@ -254,21 +382,13 @@ PyList_GetItemRef(PyObject *op, Py_ssize_t i) PyErr_SetString(PyExc_TypeError, "expected a list"); return NULL; } - if (!valid_index(i, Py_SIZE(op))) { + PyObject *item = list_get_item_ref((PyListObject *)op, i); + if (item == NULL) { _Py_DECLARE_STR(list_err, "list index out of range"); PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); return NULL; } - return Py_NewRef(PyList_GET_ITEM(op, i)); -} - -static inline PyObject* -list_get_item_ref(PyListObject *op, Py_ssize_t i) -{ - if (!valid_index(i, Py_SIZE(op))) { - return NULL; - } - return Py_NewRef(PyList_GET_ITEM(op, i)); + return item; } int @@ -294,7 +414,7 @@ PyList_SetItem(PyObject *op, Py_ssize_t i, p = self->ob_item + i; Py_XSETREF(*p, newitem); ret = 0; -end: +end:; Py_END_CRITICAL_SECTION(); return ret; } @@ -352,7 +472,7 @@ _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem) Py_DECREF(newitem); return -1; } - PyList_SET_ITEM(self, len, newitem); + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], newitem); return 0; } @@ -388,18 +508,13 @@ list_dealloc(PyObject *self) while (--i >= 0) { Py_XDECREF(op->ob_item[i]); } - PyMem_Free(op->ob_item); + free_list_items(op->ob_item, false); } -#ifdef WITH_FREELISTS - struct _Py_list_state *state = get_list_state(); - if (state->numfree < PyList_MAXFREELIST && state->numfree >= 0 && PyList_CheckExact(op)) { - state->free_list[state->numfree++] = op; - OBJECT_STAT_INC(to_freelist); + if (PyList_CheckExact(op)) { + _Py_FREELIST_FREE(lists, op, PyObject_GC_Del); } - else -#endif - { - Py_TYPE(op)->tp_free((PyObject *)op); + else { + PyObject_GC_Del(op); } Py_TRASHCAN_END } @@ -407,49 +522,48 @@ list_dealloc(PyObject *self) static PyObject * list_repr_impl(PyListObject *v) { - PyObject *s; - _PyUnicodeWriter writer; - Py_ssize_t i = Py_ReprEnter((PyObject*)v); - if (i != 0) { - return i > 0 ? PyUnicode_FromString("[...]") : NULL; + int res = Py_ReprEnter((PyObject*)v); + if (res != 0) { + return (res > 0 ? PyUnicode_FromString("[...]") : NULL); } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; /* "[" + "1" + ", 2" * (len - 1) + "]" */ - writer.min_length = 1 + 1 + (2 + 1) * (Py_SIZE(v) - 1) + 1; + Py_ssize_t prealloc = 1 + 1 + (2 + 1) * (Py_SIZE(v) - 1) + 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { + goto error; + } - if (_PyUnicodeWriter_WriteChar(&writer, '[') < 0) + if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { goto error; + } /* Do repr() on each element. Note that this may mutate the list, so must refetch the list size on each iteration. */ - for (i = 0; i < Py_SIZE(v); ++i) { + for (Py_ssize_t i = 0; i < Py_SIZE(v); ++i) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { + goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; + } } - s = PyObject_Repr(v->ob_item[i]); - if (s == NULL) - goto error; - - if (_PyUnicodeWriter_WriteStr(&writer, s) < 0) { - Py_DECREF(s); + if (PyUnicodeWriter_WriteRepr(writer, v->ob_item[i]) < 0) { goto error; } - Py_DECREF(s); } - writer.overallocate = 0; - if (_PyUnicodeWriter_WriteChar(&writer, ']') < 0) + if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { goto error; + } Py_ReprLeave((PyObject *)v); - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); Py_ReprLeave((PyObject *)v); return NULL; } @@ -501,11 +615,21 @@ list_item(PyObject *aa, Py_ssize_t i) PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); return NULL; } - return Py_NewRef(a->ob_item[i]); + PyObject *item; +#ifdef Py_GIL_DISABLED + item = list_get_item_ref(a, i); + if (item == NULL) { + PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); + return NULL; + } +#else + item = Py_NewRef(a->ob_item[i]); +#endif + return item; } static PyObject * -list_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) +list_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { PyListObject *np; PyObject **src, **dest; @@ -549,26 +673,18 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) else if (ihigh > Py_SIZE(a)) { ihigh = Py_SIZE(a); } - ret = list_slice((PyListObject *)a, ilow, ihigh); + ret = list_slice_lock_held((PyListObject *)a, ilow, ihigh); Py_END_CRITICAL_SECTION(); return ret; } static PyObject * -list_concat(PyObject *aa, PyObject *bb) +list_concat_lock_held(PyListObject *a, PyListObject *b) { - PyListObject *a = (PyListObject *)aa; Py_ssize_t size; Py_ssize_t i; PyObject **src, **dest; PyListObject *np; - if (!PyList_Check(bb)) { - PyErr_Format(PyExc_TypeError, - "can only concatenate list (not \"%.200s\") to list", - Py_TYPE(bb)->tp_name); - return NULL; - } -#define b ((PyListObject *)bb) assert((size_t)Py_SIZE(a) + (size_t)Py_SIZE(b) < PY_SSIZE_T_MAX); size = Py_SIZE(a) + Py_SIZE(b); if (size == 0) { @@ -592,13 +708,29 @@ list_concat(PyObject *aa, PyObject *bb) } Py_SET_SIZE(np, size); return (PyObject *)np; -#undef b } static PyObject * -list_repeat(PyObject *aa, Py_ssize_t n) +list_concat(PyObject *aa, PyObject *bb) { + if (!PyList_Check(bb)) { + PyErr_Format(PyExc_TypeError, + "can only concatenate list (not \"%.200s\") to list", + Py_TYPE(bb)->tp_name); + return NULL; + } PyListObject *a = (PyListObject *)aa; + PyListObject *b = (PyListObject *)bb; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(a, b); + ret = list_concat_lock_held(a, b); + Py_END_CRITICAL_SECTION2(); + return ret; +} + +static PyObject * +list_repeat_lock_held(PyListObject *a, Py_ssize_t n) +{ const Py_ssize_t input_size = Py_SIZE(a); if (input_size == 0 || n <= 0) return PyList_New(0); @@ -637,8 +769,19 @@ list_repeat(PyObject *aa, Py_ssize_t n) return (PyObject *) np; } +static PyObject * +list_repeat(PyObject *aa, Py_ssize_t n) +{ + PyObject *ret; + PyListObject *a = (PyListObject *)aa; + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_repeat_lock_held(a, n); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void -list_clear(PyListObject *a) +list_clear_impl(PyListObject *a, bool is_resize) { PyObject **items = a->ob_item; if (items == NULL) { @@ -649,21 +792,31 @@ list_clear(PyListObject *a) this list, we make it empty first. */ Py_ssize_t i = Py_SIZE(a); Py_SET_SIZE(a, 0); - a->ob_item = NULL; + FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item, NULL); a->allocated = 0; while (--i >= 0) { Py_XDECREF(items[i]); } - PyMem_Free(items); - +#ifdef Py_GIL_DISABLED + bool use_qsbr = is_resize && _PyObject_GC_IS_SHARED(a); +#else + bool use_qsbr = false; +#endif + free_list_items(items, use_qsbr); // Note that there is no guarantee that the list is actually empty // at this point, because XDECREF may have populated it indirectly again! } +static void +list_clear(PyListObject *a) +{ + list_clear_impl(a, true); +} + static int list_clear_slot(PyObject *self) { - list_clear((PyListObject *)self); + list_clear_impl((PyListObject *)self, false); return 0; } @@ -674,7 +827,7 @@ list_clear_slot(PyObject *self) * guaranteed the call cannot fail. */ static int -list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) +list_ass_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) { /* Because [X]DECREF can recursively invoke list operations on this list, we must postpone all [X]DECREF activity until @@ -697,15 +850,6 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) if (v == NULL) n = 0; else { - if (a == b) { - /* Special case "a[i:j] = a" -- copy b first */ - v = list_slice(b, 0, Py_SIZE(b)); - if (v == NULL) - return result; - result = list_ass_slice(a, ilow, ihigh, v); - Py_DECREF(v); - return result; - } v_as_SF = PySequence_Fast(v, "can only assign an iterable"); if(v_as_SF == NULL) goto Error; @@ -779,6 +923,34 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) #undef b } +static int +list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) +{ + int ret; + if (a == (PyListObject *)v) { + Py_BEGIN_CRITICAL_SECTION(a); + Py_ssize_t n = PyList_GET_SIZE(a); + PyObject *copy = list_slice_lock_held(a, 0, n); + if (copy == NULL) { + return -1; + } + ret = list_ass_slice_lock_held(a, ilow, ihigh, copy); + Py_DECREF(copy); + Py_END_CRITICAL_SECTION(); + } + else if (v != NULL && PyList_CheckExact(v)) { + Py_BEGIN_CRITICAL_SECTION2(a, v); + ret = list_ass_slice_lock_held(a, ilow, ihigh, v); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_ass_slice_lock_held(a, ilow, ihigh, v); + Py_END_CRITICAL_SECTION(); + } + return ret; +} + int PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) { @@ -789,27 +961,28 @@ PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) return list_ass_slice((PyListObject *)a, ilow, ihigh, v); } -static PyObject * -list_inplace_repeat(PyObject *_self, Py_ssize_t n) +static int +list_inplace_repeat_lock_held(PyListObject *self, Py_ssize_t n) { - PyListObject *self = (PyListObject *)_self; Py_ssize_t input_size = PyList_GET_SIZE(self); if (input_size == 0 || n == 1) { - return Py_NewRef(self); + return 0; } if (n < 1) { list_clear(self); - return Py_NewRef(self); + return 0; } if (input_size > PY_SSIZE_T_MAX / n) { - return PyErr_NoMemory(); + PyErr_NoMemory(); + return -1; } Py_ssize_t output_size = input_size * n; - if (list_resize(self, output_size) < 0) - return NULL; + if (list_resize(self, output_size) < 0) { + return -1; + } PyObject **items = self->ob_item; for (Py_ssize_t j = 0; j < input_size; j++) { @@ -817,25 +990,59 @@ list_inplace_repeat(PyObject *_self, Py_ssize_t n) } _Py_memory_repeat((char *)items, sizeof(PyObject *)*output_size, sizeof(PyObject *)*input_size); + return 0; +} - return Py_NewRef(self); +static PyObject * +list_inplace_repeat(PyObject *_self, Py_ssize_t n) +{ + PyObject *ret; + PyListObject *self = (PyListObject *) _self; + Py_BEGIN_CRITICAL_SECTION(self); + if (list_inplace_repeat_lock_held(self, n) < 0) { + ret = NULL; + } + else { + ret = Py_NewRef(self); + } + Py_END_CRITICAL_SECTION(); + return ret; } static int -list_ass_item(PyObject *aa, Py_ssize_t i, PyObject *v) +list_ass_item_lock_held(PyListObject *a, Py_ssize_t i, PyObject *v) { - PyListObject *a = (PyListObject *)aa; if (!valid_index(i, Py_SIZE(a))) { PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); return -1; } - if (v == NULL) - return list_ass_slice(a, i, i+1, v); - Py_SETREF(a->ob_item[i], Py_NewRef(v)); + PyObject *tmp = a->ob_item[i]; + if (v == NULL) { + Py_ssize_t size = Py_SIZE(a); + for (Py_ssize_t idx = i; idx < size - 1; idx++) { + FT_ATOMIC_STORE_PTR_RELAXED(a->ob_item[idx], a->ob_item[idx + 1]); + } + Py_SET_SIZE(a, size - 1); + } + else { + FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item[i], Py_NewRef(v)); + } + Py_DECREF(tmp); return 0; } +static int +list_ass_item(PyObject *aa, Py_ssize_t i, PyObject *v) +{ + int ret; + PyListObject *a = (PyListObject *)aa; + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_ass_item_lock_held(a, i, v); + Py_END_CRITICAL_SECTION(); + return ret; +} + /*[clinic input] @critical_section list.insert @@ -883,7 +1090,7 @@ static PyObject * list_copy_impl(PyListObject *self) /*[clinic end generated code: output=ec6b72d6209d418e input=81c54b0c7bb4f73d]*/ { - return list_slice(self, 0, Py_SIZE(self)); + return list_slice_lock_held(self, 0, Py_SIZE(self)); } /*[clinic input] @@ -939,13 +1146,13 @@ list_extend_fast(PyListObject *self, PyObject *iterable) PyObject **dest = self->ob_item + m; for (Py_ssize_t i = 0; i < n; i++) { PyObject *o = src[i]; - dest[i] = Py_NewRef(o); + FT_ATOMIC_STORE_PTR_RELEASE(dest[i], Py_NewRef(o)); } return 0; } static int -list_extend_iter(PyListObject *self, PyObject *iterable) +list_extend_iter_lock_held(PyListObject *self, PyObject *iterable) { PyObject *it = PyObject_GetIter(iterable); if (it == NULL) { @@ -996,7 +1203,7 @@ list_extend_iter(PyListObject *self, PyObject *iterable) if (Py_SIZE(self) < self->allocated) { Py_ssize_t len = Py_SIZE(self); - PyList_SET_ITEM(self, len, item); // steals item ref + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], item); // steals item ref Py_SET_SIZE(self, len + 1); } else { @@ -1019,45 +1226,151 @@ list_extend_iter(PyListObject *self, PyObject *iterable) return -1; } - static int -list_extend(PyListObject *self, PyObject *iterable) +list_extend_lock_held(PyListObject *self, PyObject *iterable) { - // Special cases: - // 1) lists and tuples which can use PySequence_Fast ops - // 2) extending self to self requires making a copy first - if (PyList_CheckExact(iterable) - || PyTuple_CheckExact(iterable) - || (PyObject *)self == iterable) - { - iterable = PySequence_Fast(iterable, "argument must be iterable"); - if (!iterable) { - return -1; - } + PyObject *seq = PySequence_Fast(iterable, "argument must be iterable"); + if (!seq) { + return -1; + } - int res = list_extend_fast(self, iterable); - Py_DECREF(iterable); - return res; + int res = list_extend_fast(self, seq); + Py_DECREF(seq); + return res; +} + +static int +list_extend_set(PyListObject *self, PySetObject *other) +{ + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PySet_GET_SIZE(other); + if (list_resize(self, m + n) < 0) { + return -1; } - else { - return list_extend_iter(self, iterable); + /* populate the end of self with iterable's items */ + Py_ssize_t setpos = 0; + Py_hash_t hash; + PyObject *key; + PyObject **dest = self->ob_item + m; + while (_PySet_NextEntryRef((PyObject *)other, &setpos, &key, &hash)) { + FT_ATOMIC_STORE_PTR_RELEASE(*dest, key); + dest++; } + Py_SET_SIZE(self, m + n); + return 0; } +static int +list_extend_dict(PyListObject *self, PyDictObject *dict, int which_item) +{ + // which_item: 0 for keys and 1 for values + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PyDict_GET_SIZE(dict); + if (list_resize(self, m + n) < 0) { + return -1; + } -PyObject * -_PyList_Extend(PyListObject *self, PyObject *iterable) + PyObject **dest = self->ob_item + m; + Py_ssize_t pos = 0; + PyObject *keyvalue[2]; + while (_PyDict_Next((PyObject *)dict, &pos, &keyvalue[0], &keyvalue[1], NULL)) { + PyObject *obj = keyvalue[which_item]; + Py_INCREF(obj); + FT_ATOMIC_STORE_PTR_RELEASE(*dest, obj); + dest++; + } + + Py_SET_SIZE(self, m + n); + return 0; +} + +static int +list_extend_dictitems(PyListObject *self, PyDictObject *dict) { - if (list_extend(self, iterable) < 0) { - return NULL; + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PyDict_GET_SIZE(dict); + if (list_resize(self, m + n) < 0) { + return -1; } - Py_RETURN_NONE; + + PyObject **dest = self->ob_item + m; + Py_ssize_t pos = 0; + Py_ssize_t i = 0; + PyObject *key, *value; + while (_PyDict_Next((PyObject *)dict, &pos, &key, &value, NULL)) { + PyObject *item = PyTuple_Pack(2, key, value); + if (item == NULL) { + Py_SET_SIZE(self, m + i); + return -1; + } + FT_ATOMIC_STORE_PTR_RELEASE(*dest, item); + dest++; + i++; + } + + Py_SET_SIZE(self, m + n); + return 0; } +static int +_list_extend(PyListObject *self, PyObject *iterable) +{ + // Special case: + // lists and tuples which can use PySequence_Fast ops + int res = -1; + if ((PyObject *)self == iterable) { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_inplace_repeat_lock_held(self, 2); + Py_END_CRITICAL_SECTION(); + } + else if (PyList_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_lock_held(self, iterable); + Py_END_CRITICAL_SECTION2(); + } + else if (PyTuple_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_extend_lock_held(self, iterable); + Py_END_CRITICAL_SECTION(); + } + else if (PyAnySet_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_set(self, (PySetObject *)iterable); + Py_END_CRITICAL_SECTION2(); + } + else if (PyDict_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_dict(self, (PyDictObject *)iterable, 0 /*keys*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictKeys_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dict(self, dict, 0 /*keys*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictValues_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dict(self, dict, 1 /*values*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictItems_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dictitems(self, dict); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_extend_iter_lock_held(self, iterable); + Py_END_CRITICAL_SECTION(); + } + return res; +} /*[clinic input] -@critical_section self iterable -list.extend as py_list_extend +list.extend as list_extend iterable: object / @@ -1066,12 +1379,20 @@ Extend list by appending elements from the iterable. [clinic start generated code]*/ static PyObject * -py_list_extend_impl(PyListObject *self, PyObject *iterable) -/*[clinic end generated code: output=a2f115ceace2c845 input=1d42175414e1a5f3]*/ +list_extend(PyListObject *self, PyObject *iterable) +/*[clinic end generated code: output=630fb3bca0c8e789 input=979da7597a515791]*/ { - return _PyList_Extend(self, iterable); + if (_list_extend(self, iterable) < 0) { + return NULL; + } + Py_RETURN_NONE; } +PyObject * +_PyList_Extend(PyListObject *self, PyObject *iterable) +{ + return list_extend(self, iterable); +} int PyList_Extend(PyObject *self, PyObject *iterable) @@ -1080,7 +1401,7 @@ PyList_Extend(PyObject *self, PyObject *iterable) PyErr_BadInternalCall(); return -1; } - return list_extend((PyListObject*)self, iterable); + return _list_extend((PyListObject*)self, iterable); } @@ -1100,7 +1421,7 @@ static PyObject * list_inplace_concat(PyObject *_self, PyObject *other) { PyListObject *self = (PyListObject *)_self; - if (list_extend(self, other) < 0) { + if (_list_extend(self, other) < 0) { return NULL; } return Py_NewRef(self); @@ -1276,6 +1597,15 @@ sortslice_advance(sortslice *slice, Py_ssize_t n) /* Avoid malloc for small temp arrays. */ #define MERGESTATE_TEMP_SIZE 256 +/* The largest value of minrun. This must be a power of 2, and >= 1, so that + * the compute_minrun() algorithm guarantees to return a result no larger than + * this, + */ +#define MAX_MINRUN 64 +#if ((MAX_MINRUN) < 1) || ((MAX_MINRUN) & ((MAX_MINRUN) - 1)) +#error "MAX_MINRUN must be a power of 2, and >= 1" +#endif + /* One MergeState exists on the stack per invocation of mergesort. It's just * a convenient way to pass state around among the helper functions. */ @@ -1333,123 +1663,248 @@ struct s_MergeState { int (*tuple_elem_compare)(PyObject *, PyObject *, MergeState *); }; -/* binarysort is the best method for sorting small arrays: it does - few compares, but can do data movement quadratic in the number of - elements. - [lo, hi) is a contiguous slice of a list, and is sorted via - binary insertion. This sort is stable. - On entry, must have lo <= start <= hi, and that [lo, start) is already - sorted (pass start == lo if you don't know!). - If islt() complains return -1, else 0. +/* binarysort is the best method for sorting small arrays: it does few + compares, but can do data movement quadratic in the number of elements. + ss->keys is viewed as an array of n kays, a[:n]. a[:ok] is already sorted. + Pass ok = 0 (or 1) if you don't know. + It's sorted in-place, by a stable binary insertion sort. If ss->values + isn't NULL, it's permuted in lockstap with ss->keys. + On entry, must have n >= 1, and 0 <= ok <= n <= MAX_MINRUN. + Return -1 if comparison raises an exception, else 0. Even in case of error, the output slice will be some permutation of the input (nothing is lost or duplicated). */ static int -binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) +binarysort(MergeState *ms, const sortslice *ss, Py_ssize_t n, Py_ssize_t ok) { - Py_ssize_t k; - PyObject **l, **p, **r; + Py_ssize_t k; /* for IFLT macro expansion */ + PyObject ** const a = ss->keys; + PyObject ** const v = ss->values; + const bool has_values = v != NULL; PyObject *pivot; - - assert(lo.keys <= start && start <= hi); - /* assert [lo, start) is sorted */ - if (lo.keys == start) - ++start; - for (; start < hi; ++start) { - /* set l to where *start belongs */ - l = lo.keys; - r = start; - pivot = *r; - /* Invariants: - * pivot >= all in [lo, l). - * pivot < all in [r, start). - * The second is vacuously true at the start. + Py_ssize_t M; + + assert(0 <= ok && ok <= n && 1 <= n && n <= MAX_MINRUN); + /* assert a[:ok] is sorted */ + if (! ok) + ++ok; + /* Regular insertion sort has average- and worst-case O(n**2) cost + for both # of comparisons and number of bytes moved. But its branches + are highly predictable, and it loves sorted input (n-1 compares and no + data movement). This is significant in cases like sortperf.py's %sort, + where an out-of-order element near the start of a run is moved into + place slowly but then the remaining elements up to length minrun are + generally at worst one slot away from their correct position (so only + need 1 or 2 commpares to resolve). If comparisons are very fast (such + as for a list of Python floats), the simple inner loop leaves it + very competitive with binary insertion, despite that it does + significantly more compares overall on random data. + + Binary insertion sort has worst, average, and best case O(n log n) + cost for # of comparisons, but worst and average case O(n**2) cost + for data movement. The more expensive comparisons, the more important + the comparison advantage. But its branches are less predictable the + more "randomish" the data, and that's so significant its worst case + in real life is random input rather than reverse-ordered (which does + about twice the data movement than random input does). + + Note that the number of bytes moved doesn't seem to matter. MAX_MINRUN + of 64 is so small that the key and value pointers all fit in a corner + of L1 cache, and moving things around in that is very fast. */ +#if 0 // ordinary insertion sort. + PyObject * vpivot = NULL; + for (; ok < n; ++ok) { + pivot = a[ok]; + if (has_values) + vpivot = v[ok]; + for (M = ok - 1; M >= 0; --M) { + k = ISLT(pivot, a[M]); + if (k < 0) { + a[M + 1] = pivot; + if (has_values) + v[M + 1] = vpivot; + goto fail; + } + else if (k) { + a[M + 1] = a[M]; + if (has_values) + v[M + 1] = v[M]; + } + else + break; + } + a[M + 1] = pivot; + if (has_values) + v[M + 1] = vpivot; + } +#else // binary insertion sort + Py_ssize_t L, R; + for (; ok < n; ++ok) { + /* set L to where a[ok] belongs */ + L = 0; + R = ok; + pivot = a[ok]; + /* Slice invariants. vacuously true at the start: + * all a[0:L] <= pivot + * all a[L:R] unknown + * all a[R:ok] > pivot */ - assert(l < r); + assert(L < R); do { - p = l + ((r - l) >> 1); - IFLT(pivot, *p) - r = p; + /* don't do silly ;-) things to prevent overflow when finding + the midpoint; L and R are very far from filling a Py_ssize_t */ + M = (L + R) >> 1; +#if 1 // straightforward, but highly unpredictable branch on random data + IFLT(pivot, a[M]) + R = M; else - l = p+1; - } while (l < r); - assert(l == r); - /* The invariants still hold, so pivot >= all in [lo, l) and - pivot < all in [l, start), so pivot belongs at l. Note - that if there are elements equal to pivot, l points to the - first slot after them -- that's why this sort is stable. - Slide over to make room. - Caution: using memmove is much slower under MSVC 5; - we're not usually moving many slots. */ - for (p = start; p > l; --p) - *p = *(p-1); - *l = pivot; - if (lo.values != NULL) { - Py_ssize_t offset = lo.values - lo.keys; - p = start + offset; - pivot = *p; - l += offset; - for (p = start + offset; p > l; --p) - *p = *(p-1); - *l = pivot; + L = M + 1; +#else + /* Try to get compiler to generate conditional move instructions + instead. Works fine, but leaving it disabled for now because + it's not yielding consistently faster sorts. Needs more + investigation. More computation in the inner loop adds its own + costs, which can be significant when compares are fast. */ + k = ISLT(pivot, a[M]); + if (k < 0) + goto fail; + Py_ssize_t Mp1 = M + 1; + R = k ? M : R; + L = k ? L : Mp1; +#endif + } while (L < R); + assert(L == R); + /* a[:L] holds all elements from a[:ok] <= pivot now, so pivot belongs + at index L. Slide a[L:ok] to the right a slot to make room for it. + Caution: using memmove is much slower under MSVC 5; we're not + usually moving many slots. Years later: under Visual Studio 2022, + memmove seems just slightly slower than doing it "by hand". */ + for (M = ok; M > L; --M) + a[M] = a[M - 1]; + a[L] = pivot; + if (has_values) { + pivot = v[ok]; + for (M = ok; M > L; --M) + v[M] = v[M - 1]; + v[L] = pivot; } } +#endif // pick binary or regular insertion sort return 0; fail: return -1; } -/* -Return the length of the run beginning at lo, in the slice [lo, hi). lo < hi -is required on entry. "A run" is the longest ascending sequence, with - - lo[0] <= lo[1] <= lo[2] <= ... - -or the longest descending sequence, with - - lo[0] > lo[1] > lo[2] > ... +static void +sortslice_reverse(sortslice *s, Py_ssize_t n) +{ + reverse_slice(s->keys, &s->keys[n]); + if (s->values != NULL) + reverse_slice(s->values, &s->values[n]); +} -Boolean *descending is set to 0 in the former case, or to 1 in the latter. -For its intended use in a stable mergesort, the strictness of the defn of -"descending" is needed so that the caller can safely reverse a descending -sequence without violating stability (strict > ensures there are no equal -elements to get out of order). +/* +Return the length of the run beginning at slo->keys, spanning no more than +nremaining elements. The run beginning there may be ascending or descending, +but the function permutes it in place, if needed, so that it's always ascending +upon return. Returns -1 in case of error. */ static Py_ssize_t -count_run(MergeState *ms, PyObject **lo, PyObject **hi, int *descending) +count_run(MergeState *ms, sortslice *slo, Py_ssize_t nremaining) { - Py_ssize_t k; + Py_ssize_t k; /* used by IFLT macro expansion */ Py_ssize_t n; + PyObject ** const lo = slo->keys; - assert(lo < hi); - *descending = 0; - ++lo; - if (lo == hi) - return 1; - - n = 2; - IFLT(*lo, *(lo-1)) { - *descending = 1; - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) - ; - else - break; - } + /* In general, as things go on we've established that the slice starts + with a monotone run of n elements, starting at lo. */ + + /* We're n elements into the slice, and the most recent neq+1 elements are + * all equal. This reverses them in-place, and resets neq for reuse. + */ +#define REVERSE_LAST_NEQ \ + if (neq) { \ + sortslice slice = *slo; \ + ++neq; \ + sortslice_advance(&slice, n - neq); \ + sortslice_reverse(&slice, neq); \ + neq = 0; \ + } + + /* Sticking to only __lt__ compares is confusing and error-prone. But in + * this routine, almost all uses of IFLT can be captured by tiny macros + * giving mnemonic names to the intent. Note that inline functions don't + * work for this (IFLT expands to code including `goto fail`). + */ +#define IF_NEXT_LARGER IFLT(lo[n-1], lo[n]) +#define IF_NEXT_SMALLER IFLT(lo[n], lo[n-1]) + + assert(nremaining); + /* try ascending run first */ + for (n = 1; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; } - else { - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) + if (n == nremaining) + return n; + /* lo[n] is strictly less */ + /* If n is 1 now, then the first compare established it's a descending + * run, so fall through to the descending case. But if n > 1, there are + * n elements in an ascending run terminated by the strictly less lo[n]. + * If the first key < lo[n-1], *somewhere* along the way the sequence + * increased, so we're done (there is no descending run). + * Else first key >= lo[n-1], which implies that the entire ascending run + * consists of equal elements. In that case, this is a descending run, + * and we reverse the all-equal prefix in-place. + */ + if (n > 1) { + IFLT(lo[0], lo[n-1]) + return n; + sortslice_reverse(slo, n); + } + ++n; /* in all cases it's been established that lo[n] has been resolved */ + + /* Finish descending run. All-squal subruns are reversed in-place on the + * fly. Their original order will be restored at the end by the whole-slice + * reversal. + */ + Py_ssize_t neq = 0; + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER { + /* This ends the most recent run of equal elements, but still in + * the "descending" direction. + */ + REVERSE_LAST_NEQ + } + else { + IF_NEXT_LARGER /* descending run is over */ break; + else /* not x < y and not y < x implies x == y */ + ++neq; } } + REVERSE_LAST_NEQ + sortslice_reverse(slo, n); /* transform to ascending run */ + + /* And after reversing, it's possible this can be extended by a + * naturally increasing suffix; e.g., [3, 2, 3, 4, 1] makes an + * ascending run from the first 4 elements. + */ + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; + } return n; fail: return -1; + +#undef REVERSE_LAST_NEQ +#undef IF_NEXT_SMALLER +#undef IF_NEXT_LARGER } /* @@ -2147,10 +2602,10 @@ merge_force_collapse(MergeState *ms) /* Compute a good value for the minimum run length; natural runs shorter * than this are boosted artificially via binary insertion. * - * If n < 64, return n (it's too small to bother with fancy stuff). - * Else if n is an exact power of 2, return 32. - * Else return an int k, 32 <= k <= 64, such that n/k is close to, but - * strictly less than, an exact power of 2. + * If n < MAX_MINRUN return n (it's too small to bother with fancy stuff). + * Else if n is an exact power of 2, return MAX_MINRUN / 2. + * Else return an int k, MAX_MINRUN / 2 <= k <= MAX_MINRUN, such that n/k is + * close to, but strictly less than, an exact power of 2. * * See listsort.txt for more info. */ @@ -2160,21 +2615,13 @@ merge_compute_minrun(Py_ssize_t n) Py_ssize_t r = 0; /* becomes 1 if any 1 bits are shifted off */ assert(n >= 0); - while (n >= 64) { + while (n >= MAX_MINRUN) { r |= n & 1; n >>= 1; } return n + r; } -static void -reverse_sortslice(sortslice *s, Py_ssize_t n) -{ - reverse_slice(s->keys, &s->keys[n]); - if (s->values != NULL) - reverse_slice(s->values, &s->values[n]); -} - /* Here we define custom comparison functions to optimize for the cases one commonly * encounters in practice: homogeneous lists, often of one of the basic types. */ @@ -2343,6 +2790,7 @@ unsafe_tuple_compare(PyObject *v, PyObject *w, MergeState *ms) * duplicated). */ /*[clinic input] +@critical_section list.sort * @@ -2362,7 +2810,7 @@ The reverse flag can be set to sort in descending order. static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=a74c4cd3ec6b5c08]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=667bf25d0e3a3676]*/ { MergeState ms; Py_ssize_t nremaining; @@ -2389,7 +2837,7 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) saved_ob_item = self->ob_item; saved_allocated = self->allocated; Py_SET_SIZE(self, 0); - self->ob_item = NULL; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, NULL); self->allocated = -1; /* any operation will reset it to >= 0 */ if (keyfunc == NULL) { @@ -2541,20 +2989,17 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) */ minrun = merge_compute_minrun(nremaining); do { - int descending; Py_ssize_t n; /* Identify next run. */ - n = count_run(&ms, lo.keys, lo.keys + nremaining, &descending); + n = count_run(&ms, &lo, nremaining); if (n < 0) goto fail; - if (descending) - reverse_sortslice(&lo, n); /* If short, extend to min(minrun, nremaining). */ if (n < minrun) { const Py_ssize_t force = nremaining <= minrun ? nremaining : minrun; - if (binarysort(&ms, lo, lo.keys + force, lo.keys + n) < 0) + if (binarysort(&ms, &lo, force, n) < 0) goto fail; n = force; } @@ -2609,15 +3054,20 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) final_ob_item = self->ob_item; i = Py_SIZE(self); Py_SET_SIZE(self, saved_ob_size); - self->ob_item = saved_ob_item; - self->allocated = saved_allocated; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, saved_ob_item); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, saved_allocated); if (final_ob_item != NULL) { /* we cannot use list_clear() for this because it does not guarantee that the list is really empty when it returns */ while (--i >= 0) { Py_XDECREF(final_ob_item[i]); } - PyMem_Free(final_ob_item); +#ifdef Py_GIL_DISABLED + bool use_qsbr = _PyObject_GC_IS_SHARED(self); +#else + bool use_qsbr = false; +#endif + free_list_items(final_ob_item, use_qsbr); } return Py_XNewRef(result); } @@ -2631,7 +3081,9 @@ PyList_Sort(PyObject *v) PyErr_BadInternalCall(); return -1; } + Py_BEGIN_CRITICAL_SECTION(v); v = list_sort_impl((PyListObject *)v, NULL, 0); + Py_END_CRITICAL_SECTION(); if (v == NULL) return -1; Py_DECREF(v); @@ -2687,7 +3139,7 @@ PyList_AsTuple(PyObject *v) } PyObject * -_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n) +_PyList_FromStackRefSteal(const _PyStackRef *src, Py_ssize_t n) { if (n == 0) { return PyList_New(0); @@ -2696,13 +3148,15 @@ _PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n) PyListObject *list = (PyListObject *)PyList_New(n); if (list == NULL) { for (Py_ssize_t i = 0; i < n; i++) { - Py_DECREF(src[i]); + PyStackRef_CLOSE(src[i]); } return NULL; } PyObject **dst = list->ob_item; - memcpy(dst, src, n * sizeof(PyObject *)); + for (Py_ssize_t i = 0; i < n; i++) { + dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + } return (PyObject *)list; } @@ -2748,7 +3202,7 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, else if (cmp < 0) return NULL; } - PyErr_Format(PyExc_ValueError, "%R is not in list", value); + PyErr_SetString(PyExc_ValueError, "list.index(x): x not in list"); return NULL; } @@ -2811,15 +3265,14 @@ list_remove_impl(PyListObject *self, PyObject *value) int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) { - if (list_ass_slice(self, i, i+1, - (PyObject *)NULL) == 0) + if (list_ass_slice_lock_held(self, i, i+1, NULL) == 0) Py_RETURN_NONE; return NULL; } else if (cmp < 0) return NULL; } - PyErr_Format(PyExc_ValueError, "%R is not in list", value); + PyErr_SetString(PyExc_ValueError, "list.remove(x): x not in list"); return NULL; } @@ -2887,7 +3340,14 @@ list_richcompare_impl(PyObject *v, PyObject *w, int op) } /* Compare the final item again using the proper operator */ - return PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + PyObject *vitem = vl->ob_item[i]; + PyObject *witem = wl->ob_item[i]; + Py_INCREF(vitem); + Py_INCREF(witem); + PyObject *result = PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + Py_DECREF(vitem); + Py_DECREF(witem); + return result; } static PyObject * @@ -2927,7 +3387,7 @@ list___init___impl(PyListObject *self, PyObject *iterable) list_clear(self); } if (iterable != NULL) { - if (list_extend(self, iterable) < 0) { + if (_list_extend(self, iterable) < 0) { return -1; } } @@ -2971,7 +3431,8 @@ list___sizeof___impl(PyListObject *self) /*[clinic end generated code: output=3417541f95f9a53e input=b8030a5d5ce8a187]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)self->allocated * sizeof(void*); + Py_ssize_t allocated = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->allocated); + res += (size_t)allocated * sizeof(void*); return PyLong_FromSize_t(res); } @@ -2987,7 +3448,7 @@ static PyMethodDef list_methods[] = { LIST_COPY_METHODDEF LIST_APPEND_METHODDEF LIST_INSERT_METHODDEF - PY_LIST_EXTEND_METHODDEF + LIST_EXTEND_METHODDEF LIST_POP_METHODDEF LIST_REMOVE_METHODDEF LIST_INDEX_METHODDEF @@ -3011,6 +3472,45 @@ static PySequenceMethods list_as_sequence = { list_inplace_repeat, /* sq_inplace_repeat */ }; +static inline PyObject * +list_slice_step_lock_held(PyListObject *a, Py_ssize_t start, Py_ssize_t step, Py_ssize_t len) +{ + PyListObject *np = (PyListObject *)list_new_prealloc(len); + if (np == NULL) { + return NULL; + } + size_t cur; + Py_ssize_t i; + PyObject **src = a->ob_item; + PyObject **dest = np->ob_item; + for (cur = start, i = 0; i < len; + cur += (size_t)step, i++) { + PyObject *v = src[cur]; + dest[i] = Py_NewRef(v); + } + Py_SET_SIZE(np, len); + return (PyObject *)np; +} + +static PyObject * +list_slice_wrap(PyListObject *aa, Py_ssize_t start, Py_ssize_t stop, Py_ssize_t step) +{ + PyObject *res = NULL; + Py_BEGIN_CRITICAL_SECTION(aa); + Py_ssize_t len = PySlice_AdjustIndices(Py_SIZE(aa), &start, &stop, step); + if (len <= 0) { + res = PyList_New(0); + } + else if (step == 1) { + res = list_slice_lock_held(aa, start, stop); + } + else { + res = list_slice_step_lock_held(aa, start, step, len); + } + Py_END_CRITICAL_SECTION(); + return res; +} + static PyObject * list_subscript(PyObject* _self, PyObject* item) { @@ -3025,38 +3525,11 @@ list_subscript(PyObject* _self, PyObject* item) return list_item((PyObject *)self, i); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength, i; - size_t cur; - PyObject* result; - PyObject* it; - PyObject **src, **dest; - + Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } - slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, - step); - - if (slicelength <= 0) { - return PyList_New(0); - } - else if (step == 1) { - return list_slice(self, start, stop); - } - else { - result = list_new_prealloc(slicelength); - if (!result) return NULL; - - src = self->ob_item; - dest = ((PyListObject *)result)->ob_item; - for (cur = start, i = 0; i < slicelength; - cur += (size_t)step, i++) { - it = Py_NewRef(src[cur]); - dest[i] = it; - } - Py_SET_SIZE(result, slicelength); - return result; - } + return list_slice_wrap(self, start, stop, step); } else { PyErr_Format(PyExc_TypeError, @@ -3066,6 +3539,23 @@ list_subscript(PyObject* _self, PyObject* item) } } +static Py_ssize_t +adjust_slice_indexes(PyListObject *lst, + Py_ssize_t *start, Py_ssize_t *stop, + Py_ssize_t step) +{ + Py_ssize_t slicelength = PySlice_AdjustIndices(Py_SIZE(lst), start, stop, + step); + + /* Make sure s[5:2] = [..] inserts at the right place: + before 5, not before 2. */ + if ((step < 0 && *start < *stop) || + (step > 0 && *start > *stop)) + *stop = *start; + + return slicelength; +} + static int list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) { @@ -3079,22 +3569,11 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) return list_ass_item((PyObject *)self, i, value); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength; + Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } - slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, - step); - - if (step == 1) - return list_ass_slice(self, start, stop, value); - - /* Make sure s[5:2] = [..] inserts at the right place: - before 5, not before 2. */ - if ((step < 0 && start < stop) || - (step > 0 && start > stop)) - stop = start; if (value == NULL) { /* delete slice */ @@ -3103,6 +3582,12 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) Py_ssize_t i; int res; + Py_ssize_t slicelength = adjust_slice_indexes(self, &start, &stop, + step); + + if (step == 1) + return list_ass_slice(self, start, stop, value); + if (slicelength <= 0) return 0; @@ -3167,8 +3652,10 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) /* protect against a[::-1] = a */ if (self == (PyListObject*)value) { - seq = list_slice((PyListObject*)value, 0, - PyList_GET_SIZE(value)); + Py_BEGIN_CRITICAL_SECTION(value); + seq = list_slice_lock_held((PyListObject*)value, 0, + Py_SIZE(value)); + Py_END_CRITICAL_SECTION(); } else { seq = PySequence_Fast(value, @@ -3178,6 +3665,15 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) if (!seq) return -1; + Py_ssize_t slicelength = adjust_slice_indexes(self, &start, &stop, + step); + + if (step == 1) { + int res = list_ass_slice(self, start, stop, seq); + Py_DECREF(seq); + return res; + } + if (PySequence_Fast_GET_SIZE(seq) != slicelength) { PyErr_Format(PyExc_ValueError, "attempt to assign sequence of " @@ -3373,33 +3869,34 @@ static PyObject * listiter_next(PyObject *self) { _PyListIterObject *it = (_PyListIterObject *)self; - PyListObject *seq; - PyObject *item; - - assert(it != NULL); - seq = it->it_seq; - if (seq == NULL) + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index < 0) { return NULL; - assert(PyList_Check(seq)); - - if (it->it_index < PyList_GET_SIZE(seq)) { - item = PyList_GET_ITEM(seq, it->it_index); - ++it->it_index; - return Py_NewRef(item); } - it->it_seq = NULL; - Py_DECREF(seq); - return NULL; + PyObject *item = list_get_item_ref(it->it_seq, index); + if (item == NULL) { + // out-of-bounds + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1); +#ifndef Py_GIL_DISABLED + PyListObject *seq = it->it_seq; + it->it_seq = NULL; + Py_DECREF(seq); +#endif + return NULL; + } + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index + 1); + return item; } static PyObject * listiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + assert(self != NULL); _PyListIterObject *it = (_PyListIterObject *)self; - Py_ssize_t len; - if (it->it_seq) { - len = PyList_GET_SIZE(it->it_seq) - it->it_index; + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index >= 0) { + Py_ssize_t len = PyList_GET_SIZE(it->it_seq) - index; if (len >= 0) return PyLong_FromSsize_t(len); } @@ -3420,8 +3917,8 @@ listiter_setstate(PyObject *self, PyObject *state) if (index == -1 && PyErr_Occurred()) return NULL; if (it->it_seq != NULL) { - if (index < 0) - index = 0; + if (index < -1) + index = -1; else if (index > PyList_GET_SIZE(it->it_seq)) index = PyList_GET_SIZE(it->it_seq); /* iterator exhausted */ it->it_index = index; @@ -3526,26 +4023,24 @@ static PyObject * listreviter_next(PyObject *self) { listreviterobject *it = (listreviterobject *)self; - PyObject *item; - Py_ssize_t index; - PyListObject *seq; - assert(it != NULL); - seq = it->it_seq; - if (seq == NULL) { + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index < 0) { return NULL; } - assert(PyList_Check(seq)); - index = it->it_index; - if (index>=0 && index < PyList_GET_SIZE(seq)) { - item = PyList_GET_ITEM(seq, index); - it->it_index--; - return Py_NewRef(item); + PyListObject *seq = it->it_seq; + assert(PyList_Check(seq)); + PyObject *item = list_get_item_ref(seq, index); + if (item != NULL) { + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index - 1); + return item; } - it->it_index = -1; + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1); +#ifndef Py_GIL_DISABLED it->it_seq = NULL; Py_DECREF(seq); +#endif return NULL; } @@ -3553,7 +4048,8 @@ static PyObject * listreviter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { listreviterobject *it = (listreviterobject *)self; - Py_ssize_t len = it->it_index + 1; + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + Py_ssize_t len = index + 1; if (it->it_seq == NULL || PyList_GET_SIZE(it->it_seq) < len) len = 0; return PyLong_FromSsize_t(len); @@ -3588,6 +4084,7 @@ static PyObject * listiter_reduce_general(void *_it, int forward) { PyObject *list; + PyObject *iter; /* _PyEval_GetBuiltin can invoke arbitrary code, * call must be before access of iterator pointers. @@ -3595,29 +4092,21 @@ listiter_reduce_general(void *_it, int forward) /* the objects are not the same, index is of different types! */ if (forward) { - PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); - if (!iter) { - return NULL; - } + iter = _PyEval_GetBuiltin(&_Py_ID(iter)); _PyListIterObject *it = (_PyListIterObject *)_it; - if (it->it_seq) { + if (it->it_index >= 0) { return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } - Py_DECREF(iter); } else { - PyObject *reversed = _PyEval_GetBuiltin(&_Py_ID(reversed)); - if (!reversed) { - return NULL; - } + iter = _PyEval_GetBuiltin(&_Py_ID(reversed)); listreviterobject *it = (listreviterobject *)_it; - if (it->it_seq) { - return Py_BuildValue("N(O)n", reversed, it->it_seq, it->it_index); + if (it->it_index >= 0) { + return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } - Py_DECREF(reversed); } /* empty iterator, create an empty list */ list = PyList_New(0); if (list == NULL) return NULL; - return Py_BuildValue("N(N)", _PyEval_GetBuiltin(&_Py_ID(iter)), list); + return Py_BuildValue("N(N)", iter, list); } diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 32a59e510f0754..f387d9c116e502 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -212,24 +212,43 @@ A detailed description of timsort follows. Runs ---- -count_run() returns the # of elements in the next run. A run is either -"ascending", which means non-decreasing: +count_run() returns the # of elements in the next run, and, if it's a +descending run, reverses it in-place. A run is either "ascending", which +means non-decreasing: a0 <= a1 <= a2 <= ... -or "descending", which means strictly decreasing: +or "descending", which means non-increasing: - a0 > a1 > a2 > ... + a0 >= a1 >= a2 >= ... Note that a run is always at least 2 long, unless we start at the array's -last element. - -The definition of descending is strict, because the main routine reverses -a descending run in-place, transforming a descending run into an ascending -run. Reversal is done via the obvious fast "swap elements starting at each -end, and converge at the middle" method, and that can violate stability if -the slice contains any equal elements. Using a strict definition of -descending ensures that a descending run contains distinct elements. +last element. If all elements in the array are equal, it can be viewed as +both ascending and descending. Upon return, the run count_run() identifies +is always ascending. + +Reversal is done via the obvious fast "swap elements starting at each +end, and converge at the middle" method. That can violate stability if +the slice contains any equal elements. For that reason, for a long time +the code used strict inequality (">" rather than ">=") in its definition +of descending. + +Removing that restriction required some complication: when processing a +descending run, all-equal sub-runs of elements are reversed in-place, on the +fly. Their original relative order is restored "by magic" via the final +"reverse the entire run" step. + +This makes processing descending runs a little more costly. We only use +`__lt__` comparisons, so that `x == y` has to be deduced from +`not x < y and not y < x`. But so long as a run remains strictly decreasing, +only one of those compares needs to be done per loop iteration. So the primsry +extra cost is paid only when there are equal elements, and they get some +compensating benefit by not needing to end the descending run. + +There's one more trick added since the original: after reversing a descending +run, it's possible that it can be extended by an adjacent ascending run. For +example, given [3, 2, 1, 3, 4, 5, 0], the 3-element descending prefix is +reversed in-place, and then extended by [3, 4, 5]. If an array is random, it's very unlikely we'll see long runs. If a natural run contains less than minrun elements (see next section), the main loop @@ -251,9 +270,9 @@ result. This has two primary good effects: Computing minrun ---------------- -If N < 64, minrun is N. IOW, binary insertion sort is used for the whole -array then; it's hard to beat that given the overheads of trying something -fancier (see note BINSORT). +If N < MAX_MINRUN, minrun is N. IOW, binary insertion sort is used for the +whole array then; it's hard to beat that given the overheads of trying +something fancier (see note BINSORT). When N is a power of 2, testing on random data showed that minrun values of 16, 32, 64 and 128 worked about equally well. At 256 the data-movement cost @@ -291,12 +310,13 @@ place, and r < minrun is small compared to N), or q a little larger than a power of 2 regardless of r (then we've got a case similar to "2112", again leaving too little work for the last merge to do). -Instead we pick a minrun in range(32, 65) such that N/minrun is exactly a -power of 2, or if that isn't possible, is close to, but strictly less than, -a power of 2. This is easier to do than it may sound: take the first 6 -bits of N, and add 1 if any of the remaining bits are set. In fact, that -rule covers every case in this section, including small N and exact powers -of 2; merge_compute_minrun() is a deceptively simple function. +Instead we pick a minrun in range(MAX_MINRUN / 2, MAX_MINRUN + 1) such that +N/minrun is exactly a power of 2, or if that isn't possible, is close to, but +strictly less than, a power of 2. This is easier to do than it may sound: +take the first log2(MAX_MINRUN) bits of N, and add 1 if any of the remaining +bits are set. In fact, that rule covers every case in this section, including +small N and exact powers of 2; merge_compute_minrun() is a deceptively simple +function. The Merge Pattern diff --git a/Objects/longobject.c b/Objects/longobject.c index e655ba19e8f1c1..4e948940485730 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -31,6 +31,13 @@ class int "PyObject *" "&PyLong_Type" /* If defined, use algorithms from the _pylong.py module */ #define WITH_PYLONG_MODULE 1 +// Forward declarations +static PyLongObject* long_neg(PyLongObject *v); +static PyLongObject *x_divrem(PyLongObject *, PyLongObject *, PyLongObject **); +static PyObject* long_long(PyObject *v); +static PyObject* long_lshift_int64(PyLongObject *a, int64_t shiftby); + + static inline void _Py_DECREF_INT(PyLongObject *op) { @@ -133,8 +140,16 @@ long_normalize(PyLongObject *v) /* Allocate a new int object with size digits. Return NULL and set exception if we run out of memory. */ -#define MAX_LONG_DIGITS \ +#if SIZEOF_SIZE_T < 8 +# define MAX_LONG_DIGITS \ ((PY_SSIZE_T_MAX - offsetof(PyLongObject, long_value.ob_digit))/sizeof(digit)) +#else +/* Guarantee that the number of bits fits in int64_t. + This is more than an exbibyte, that is more than many of modern + architectures support in principle. + -1 is added to avoid overflow in _PyLong_Frexp(). */ +# define MAX_LONG_DIGITS ((INT64_MAX-1) / PyLong_SHIFT) +#endif PyLongObject * _PyLong_New(Py_ssize_t size) @@ -258,17 +273,17 @@ _PyLong_FromLarge(stwodigits ival) } /* Create a new int object from a C word-sized int */ -static inline PyObject * +static inline PyLongObject * _PyLong_FromSTwoDigits(stwodigits x) { if (IS_SMALL_INT(x)) { - return get_small_int((sdigit)x); + return (PyLongObject*)get_small_int((sdigit)x); } assert(x != 0); if (is_medium_int(x)) { - return _PyLong_FromMedium((sdigit)x); + return (PyLongObject*)_PyLong_FromMedium((sdigit)x); } - return _PyLong_FromLarge(x); + return (PyLongObject*)_PyLong_FromLarge(x); } /* If a freshly-allocated int is already shared, it must @@ -284,7 +299,7 @@ _PyLong_Negate(PyLongObject **x_p) return; } - *x_p = (PyLongObject *)_PyLong_FromSTwoDigits(-medium_value(x)); + *x_p = _PyLong_FromSTwoDigits(-medium_value(x)); Py_DECREF(x); } @@ -401,12 +416,12 @@ PyLong_FromDouble(double dval) double frac; int i, ndig, expo, neg; neg = 0; - if (Py_IS_INFINITY(dval)) { + if (isinf(dval)) { PyErr_SetString(PyExc_OverflowError, "cannot convert float infinity to integer"); return NULL; } - if (Py_IS_NAN(dval)) { + if (isnan(dval)) { PyErr_SetString(PyExc_ValueError, "cannot convert float NaN to integer"); return NULL; @@ -483,11 +498,18 @@ PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) do_decref = 1; } if (_PyLong_IsCompact(v)) { -#if SIZEOF_LONG < SIZEOF_VOID_P - intptr_t tmp = _PyLong_CompactValue(v); - res = (long)tmp; - if (res != tmp) { - *overflow = tmp < 0 ? -1 : 1; +#if SIZEOF_LONG < SIZEOF_SIZE_T + Py_ssize_t tmp = _PyLong_CompactValue(v); + if (tmp < LONG_MIN) { + *overflow = -1; + res = -1; + } + else if (tmp > LONG_MAX) { + *overflow = 1; + res = -1; + } + else { + res = (long)tmp; } #else res = _PyLong_CompactValue(v); @@ -632,14 +654,15 @@ PyLong_AsUnsignedLong(PyObject *vv) v = (PyLongObject *)vv; if (_PyLong_IsNonNegativeCompact(v)) { -#if SIZEOF_LONG < SIZEOF_VOID_P - intptr_t tmp = _PyLong_CompactValue(v); +#if SIZEOF_LONG < SIZEOF_SIZE_T + size_t tmp = (size_t)_PyLong_CompactValue(v); unsigned long res = (unsigned long)tmp; if (res != tmp) { goto overflow; } + return res; #else - return _PyLong_CompactValue(v); + return (unsigned long)(size_t)_PyLong_CompactValue(v); #endif } if (_PyLong_IsNegative(v)) { @@ -685,7 +708,7 @@ PyLong_AsSize_t(PyObject *vv) v = (PyLongObject *)vv; if (_PyLong_IsNonNegativeCompact(v)) { - return _PyLong_CompactValue(v); + return (size_t)_PyLong_CompactValue(v); } if (_PyLong_IsNegative(v)) { PyErr_SetString(PyExc_OverflowError, @@ -722,7 +745,11 @@ _PyLong_AsUnsignedLongMask(PyObject *vv) } v = (PyLongObject *)vv; if (_PyLong_IsCompact(v)) { - return (unsigned long)_PyLong_CompactValue(v); +#if SIZEOF_LONG < SIZEOF_SIZE_T + return (unsigned long)(size_t)_PyLong_CompactValue(v); +#else + return (unsigned long)(long)_PyLong_CompactValue(v); +#endif } i = _PyLong_DigitCount(v); int sign = _PyLong_NonCompactSign(v); @@ -770,6 +797,18 @@ _PyLong_Sign(PyObject *vv) return _PyLong_NonCompactSign(v); } +int +PyLong_GetSign(PyObject *vv, int *sign) +{ + if (!PyLong_Check(vv)) { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } + + *sign = _PyLong_Sign(vv); + return 0; +} + static int bit_length_digit(digit x) { @@ -780,11 +819,11 @@ bit_length_digit(digit x) return _Py_bit_length((unsigned long)x); } -size_t +int64_t _PyLong_NumBits(PyObject *vv) { PyLongObject *v = (PyLongObject *)vv; - size_t result = 0; + int64_t result = 0; Py_ssize_t ndigits; int msd_bits; @@ -794,20 +833,12 @@ _PyLong_NumBits(PyObject *vv) assert(ndigits == 0 || v->long_value.ob_digit[ndigits - 1] != 0); if (ndigits > 0) { digit msd = v->long_value.ob_digit[ndigits - 1]; - if ((size_t)(ndigits - 1) > SIZE_MAX / (size_t)PyLong_SHIFT) - goto Overflow; - result = (size_t)(ndigits - 1) * (size_t)PyLong_SHIFT; + assert(ndigits <= INT64_MAX / PyLong_SHIFT); + result = (int64_t)(ndigits - 1) * PyLong_SHIFT; msd_bits = bit_length_digit(msd); - if (SIZE_MAX - msd_bits < result) - goto Overflow; result += msd_bits; } return result; - - Overflow: - PyErr_SetString(PyExc_OverflowError, "int has too many bits " - "to express in a platform size_t"); - return (size_t)-1; } PyObject * @@ -928,7 +959,8 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, int _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, - int little_endian, int is_signed) + int little_endian, int is_signed, + int with_exceptions) { Py_ssize_t i; /* index into v->long_value.ob_digit */ Py_ssize_t ndigits; /* number of digits */ @@ -945,8 +977,10 @@ _PyLong_AsByteArray(PyLongObject* v, ndigits = _PyLong_DigitCount(v); if (_PyLong_IsNegative(v)) { if (!is_signed) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative int to unsigned"); + if (with_exceptions) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative int to unsigned"); + } return -1; } do_twos_comp = 1; @@ -967,7 +1001,12 @@ _PyLong_AsByteArray(PyLongObject* v, /* Copy over all the Python digits. It's crucial that every Python digit except for the MSD contribute exactly PyLong_SHIFT bits to the total, so first assert that the int is - normalized. */ + normalized. + NOTE: PyLong_AsNativeBytes() assumes that this function will fill in 'n' + bytes even if it eventually fails to convert the whole number. Make sure + you account for that if you are changing this algorithm to return without + doing that. + */ assert(ndigits == 0 || v->long_value.ob_digit[ndigits - 1] != 0); j = 0; accum = 0; @@ -1052,11 +1091,262 @@ _PyLong_AsByteArray(PyLongObject* v, return 0; Overflow: - PyErr_SetString(PyExc_OverflowError, "int too big to convert"); + if (with_exceptions) { + PyErr_SetString(PyExc_OverflowError, "int too big to convert"); + } return -1; } +// Refactored out for readability, not reuse +static inline int +_fits_in_n_bits(Py_ssize_t v, Py_ssize_t n) +{ + if (n >= (Py_ssize_t)sizeof(Py_ssize_t) * 8) { + return 1; + } + // If all bits above n are the same, we fit. + // (Use n-1 if we require the sign bit to be consistent.) + Py_ssize_t v_extended = v >> ((int)n - 1); + return v_extended == 0 || v_extended == -1; +} + +static inline int +_resolve_endianness(int *endianness) +{ + if (*endianness == -1 || (*endianness & 2)) { + *endianness = PY_LITTLE_ENDIAN; + } else { + *endianness &= 1; + } + assert(*endianness == 0 || *endianness == 1); + return 0; +} + +Py_ssize_t +PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) +{ + PyLongObject *v; + union { + Py_ssize_t v; + unsigned char b[sizeof(Py_ssize_t)]; + } cv; + int do_decref = 0; + Py_ssize_t res = 0; + + if (vv == NULL || n < 0) { + PyErr_BadInternalCall(); + return -1; + } + + int little_endian = flags; + if (_resolve_endianness(&little_endian) < 0) { + return -1; + } + + if (PyLong_Check(vv)) { + v = (PyLongObject *)vv; + } + else if (flags != -1 && (flags & Py_ASNATIVEBYTES_ALLOW_INDEX)) { + v = (PyLongObject *)_PyNumber_Index(vv); + if (v == NULL) { + return -1; + } + do_decref = 1; + } + else { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } + + if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) + && _PyLong_IsNegative(v)) { + PyErr_SetString(PyExc_ValueError, "Cannot convert negative int"); + if (do_decref) { + Py_DECREF(v); + } + return -1; + } + + if (_PyLong_IsCompact(v)) { + res = 0; + cv.v = _PyLong_CompactValue(v); + /* Most paths result in res = sizeof(compact value). Only the case + * where 0 < n < sizeof(compact value) do we need to check and adjust + * our return value. */ + res = sizeof(cv.b); + if (n <= 0) { + // nothing to do! + } + else if (n <= (Py_ssize_t)sizeof(cv.b)) { +#if PY_LITTLE_ENDIAN + if (little_endian) { + memcpy(buffer, cv.b, n); + } + else { + for (Py_ssize_t i = 0; i < n; ++i) { + ((unsigned char*)buffer)[n - i - 1] = cv.b[i]; + } + } +#else + if (little_endian) { + for (Py_ssize_t i = 0; i < n; ++i) { + ((unsigned char*)buffer)[i] = cv.b[sizeof(cv.b) - i - 1]; + } + } + else { + memcpy(buffer, &cv.b[sizeof(cv.b) - n], n); + } +#endif + + /* If we fit, return the requested number of bytes */ + if (_fits_in_n_bits(cv.v, n * 8)) { + res = n; + } else if (cv.v > 0 && _fits_in_n_bits(cv.v, n * 8 + 1)) { + /* Positive values with the MSB set do not require an + * additional bit when the caller's intent is to treat them + * as unsigned. */ + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + res = n; + } else { + res = n + 1; + } + } + } + else { + unsigned char fill = cv.v < 0 ? 0xFF : 0x00; +#if PY_LITTLE_ENDIAN + if (little_endian) { + memcpy(buffer, cv.b, sizeof(cv.b)); + memset((char *)buffer + sizeof(cv.b), fill, n - sizeof(cv.b)); + } + else { + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = 0; i < n - (int)sizeof(cv.b); ++i) { + *b++ = fill; + } + for (Py_ssize_t i = sizeof(cv.b); i > 0; --i) { + *b++ = cv.b[i - 1]; + } + } +#else + if (little_endian) { + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = sizeof(cv.b); i > 0; --i) { + *b++ = cv.b[i - 1]; + } + for (Py_ssize_t i = 0; i < n - (int)sizeof(cv.b); ++i) { + *b++ = fill; + } + } + else { + memset(buffer, fill, n - sizeof(cv.b)); + memcpy((char *)buffer + n - sizeof(cv.b), cv.b, sizeof(cv.b)); + } +#endif + } + } + else { + if (n > 0) { + _PyLong_AsByteArray(v, buffer, (size_t)n, little_endian, 1, 0); + } + + /* Calculates the number of bits required for the *absolute* value + * of v. This does not take sign into account, only magnitude. */ + int64_t nb = _PyLong_NumBits((PyObject *)v); + assert(nb >= 0); + /* Normally this would be ((nb - 1) / 8) + 1 to avoid rounding up + * multiples of 8 to the next byte, but we add an implied bit for + * the sign and it cancels out. */ + res = (Py_ssize_t)(nb / 8) + 1; + + /* Two edge cases exist that are best handled after extracting the + * bits. These may result in us reporting overflow when the value + * actually fits. + */ + if (n > 0 && res == n + 1 && nb % 8 == 0) { + if (_PyLong_IsNegative(v)) { + /* Values of 0x80...00 from negative values that use every + * available bit in the buffer do not require an additional + * bit to store the sign. */ + int is_edge_case = 1; + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = 0; i < n && is_edge_case; ++i, ++b) { + if (i == 0) { + is_edge_case = (*b == (little_endian ? 0 : 0x80)); + } else if (i < n - 1) { + is_edge_case = (*b == 0); + } else { + is_edge_case = (*b == (little_endian ? 0x80 : 0)); + } + } + if (is_edge_case) { + res = n; + } + } + else { + /* Positive values with the MSB set do not require an + * additional bit when the caller's intent is to treat them + * as unsigned. */ + unsigned char *b = (unsigned char *)buffer; + if (b[little_endian ? n - 1 : 0] & 0x80) { + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + res = n; + } else { + res = n + 1; + } + } + } + } + } + + if (do_decref) { + Py_DECREF(v); + } + + return res; +} + + +PyObject * +PyLong_FromNativeBytes(const void* buffer, size_t n, int flags) +{ + if (!buffer) { + PyErr_BadInternalCall(); + return NULL; + } + + int little_endian = flags; + if (_resolve_endianness(&little_endian) < 0) { + return NULL; + } + + return _PyLong_FromByteArray( + (const unsigned char *)buffer, + n, + little_endian, + (flags == -1 || !(flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) ? 1 : 0 + ); +} + + +PyObject * +PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n, int flags) +{ + if (!buffer) { + PyErr_BadInternalCall(); + return NULL; + } + + int little_endian = flags; + if (_resolve_endianness(&little_endian) < 0) { + return NULL; + } + + return _PyLong_FromByteArray((const unsigned char *)buffer, n, little_endian, 0); +} + + /* Create a new int object from a C pointer */ PyObject * @@ -1231,7 +1521,7 @@ PyLong_AsLongLong(PyObject *vv) } else { res = _PyLong_AsByteArray((PyLongObject *)v, (unsigned char *)&bytes, - SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 1); + SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 1, 1); } if (do_decref) { Py_DECREF(v); @@ -1266,11 +1556,22 @@ PyLong_AsUnsignedLongLong(PyObject *vv) v = (PyLongObject*)vv; if (_PyLong_IsNonNegativeCompact(v)) { res = 0; - bytes = _PyLong_CompactValue(v); +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + size_t tmp = (size_t)_PyLong_CompactValue(v); + bytes = (unsigned long long)tmp; + if (bytes != tmp) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert " + "to C unsigned long long"); + res = -1; + } +#else + bytes = (unsigned long long)(size_t)_PyLong_CompactValue(v); +#endif } else { res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes, - SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 0); + SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 0, 1); } /* Plan 9 can't handle long long in ? : expressions */ @@ -1297,7 +1598,11 @@ _PyLong_AsUnsignedLongLongMask(PyObject *vv) } v = (PyLongObject *)vv; if (_PyLong_IsCompact(v)) { - return (unsigned long long)(signed long long)_PyLong_CompactValue(v); +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + return (unsigned long long)(size_t)_PyLong_CompactValue(v); +#else + return (unsigned long long)(long long)_PyLong_CompactValue(v); +#endif } i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); @@ -1369,7 +1674,22 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) do_decref = 1; } if (_PyLong_IsCompact(v)) { +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + Py_ssize_t tmp = _PyLong_CompactValue(v); + if (tmp < LLONG_MIN) { + *overflow = -1; + res = -1; + } + else if (tmp > LLONG_MAX) { + *overflow = 1; + res = -1; + } + else { + res = (long long)tmp; + } +#else res = _PyLong_CompactValue(v); +#endif } else { i = _PyLong_DigitCount(v); @@ -1765,7 +2085,9 @@ long_to_decimal_string_internal(PyObject *aa, digit *pout, *pin, rem, tenpow; int negative; int d; - int kind; + + // writer or bytes_writer can be used, but not both at the same time. + assert(writer == NULL || bytes_writer == NULL); a = (PyLongObject *)aa; if (a == NULL || !PyLong_Check(a)) { @@ -1878,7 +2200,6 @@ long_to_decimal_string_internal(PyObject *aa, Py_DECREF(scratch); return -1; } - kind = writer->kind; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, strlen); @@ -1893,7 +2214,6 @@ long_to_decimal_string_internal(PyObject *aa, Py_DECREF(scratch); return -1; } - kind = PyUnicode_KIND(str); } #define WRITE_DIGITS(p) \ @@ -1941,19 +2261,23 @@ long_to_decimal_string_internal(PyObject *aa, WRITE_DIGITS(p); assert(p == *bytes_str); } - else if (kind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *p; - WRITE_UNICODE_DIGITS(Py_UCS1); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *p; - WRITE_UNICODE_DIGITS(Py_UCS2); - } else { - Py_UCS4 *p; - assert (kind == PyUnicode_4BYTE_KIND); - WRITE_UNICODE_DIGITS(Py_UCS4); + int kind = writer ? writer->kind : PyUnicode_KIND(str); + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_UNICODE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_UNICODE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_UNICODE_DIGITS(Py_UCS4); + } } + #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS @@ -1994,11 +2318,12 @@ long_format_binary(PyObject *aa, int base, int alternate, PyObject *v = NULL; Py_ssize_t sz; Py_ssize_t size_a; - int kind; int negative; int bits; assert(base == 2 || base == 8 || base == 16); + // writer or bytes_writer can be used, but not both at the same time. + assert(writer == NULL || bytes_writer == NULL); if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); return -1; @@ -2046,7 +2371,6 @@ long_format_binary(PyObject *aa, int base, int alternate, if (writer) { if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) return -1; - kind = writer->kind; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); @@ -2057,7 +2381,6 @@ long_format_binary(PyObject *aa, int base, int alternate, v = PyUnicode_New(sz, 'x'); if (v == NULL) return -1; - kind = PyUnicode_KIND(v); } #define WRITE_DIGITS(p) \ @@ -2118,19 +2441,23 @@ long_format_binary(PyObject *aa, int base, int alternate, WRITE_DIGITS(p); assert(p == *bytes_str); } - else if (kind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *p; - WRITE_UNICODE_DIGITS(Py_UCS1); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *p; - WRITE_UNICODE_DIGITS(Py_UCS2); - } else { - Py_UCS4 *p; - assert (kind == PyUnicode_4BYTE_KIND); - WRITE_UNICODE_DIGITS(Py_UCS4); + int kind = writer ? writer->kind : PyUnicode_KIND(v); + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_UNICODE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_UNICODE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_UNICODE_DIGITS(Py_UCS4); + } } + #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS @@ -2299,8 +2626,6 @@ long_from_binary_base(const char *start, const char *end, Py_ssize_t digits, int return 0; } -static PyObject *long_neg(PyLongObject *v); - #ifdef WITH_PYLONG_MODULE /* asymptotically faster str-to-long conversion for base 10, using _pylong.py */ static int @@ -2829,11 +3154,6 @@ PyLong_FromUnicodeObject(PyObject *u, int base) return NULL; } -/* forward */ -static PyLongObject *x_divrem - (PyLongObject *, PyLongObject *, PyLongObject **); -static PyObject *long_long(PyObject *v); - /* Int division with remainder, top-level routine */ static int @@ -2844,8 +3164,7 @@ long_divrem(PyLongObject *a, PyLongObject *b, PyLongObject *z; if (size_b == 0) { - PyErr_SetString(PyExc_ZeroDivisionError, - "integer division or modulo by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return -1; } if (size_a < size_b || @@ -2908,7 +3227,7 @@ long_rem(PyLongObject *a, PyLongObject *b, PyLongObject **prem) if (size_b == 0) { PyErr_SetString(PyExc_ZeroDivisionError, - "integer modulo by zero"); + "division by zero"); return -1; } if (size_a < size_b || @@ -3090,9 +3409,11 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) #endif double -_PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) +_PyLong_Frexp(PyLongObject *a, int64_t *e) { - Py_ssize_t a_size, a_bits, shift_digits, shift_bits, x_size; + Py_ssize_t a_size, shift_digits, x_size; + int shift_bits; + int64_t a_bits; /* See below for why x_digits is always large enough. */ digit rem; digit x_digits[2 + (DBL_MANT_DIG + 1) / PyLong_SHIFT] = {0,}; @@ -3108,14 +3429,7 @@ _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) *e = 0; return 0.0; } - a_bits = bit_length_digit(a->long_value.ob_digit[a_size-1]); - /* The following is an overflow-free version of the check - "if ((a_size - 1) * PyLong_SHIFT + a_bits > PY_SSIZE_T_MAX) ..." */ - if (a_size >= (PY_SSIZE_T_MAX - 1) / PyLong_SHIFT + 1 && - (a_size > (PY_SSIZE_T_MAX - 1) / PyLong_SHIFT + 1 || - a_bits > (PY_SSIZE_T_MAX - 1) % PyLong_SHIFT + 1)) - goto overflow; - a_bits = (a_size - 1) * PyLong_SHIFT + a_bits; + a_bits = _PyLong_NumBits((PyObject *)a); /* Shift the first DBL_MANT_DIG + 2 bits of a into x_digits[0:x_size] (shifting left if a_bits <= DBL_MANT_DIG + 2). @@ -3143,19 +3457,19 @@ _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) in both cases. */ if (a_bits <= DBL_MANT_DIG + 2) { - shift_digits = (DBL_MANT_DIG + 2 - a_bits) / PyLong_SHIFT; - shift_bits = (DBL_MANT_DIG + 2 - a_bits) % PyLong_SHIFT; + shift_digits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) / PyLong_SHIFT; + shift_bits = (DBL_MANT_DIG + 2 - (int)a_bits) % PyLong_SHIFT; x_size = shift_digits; rem = v_lshift(x_digits + x_size, a->long_value.ob_digit, a_size, - (int)shift_bits); + shift_bits); x_size += a_size; x_digits[x_size++] = rem; } else { - shift_digits = (a_bits - DBL_MANT_DIG - 2) / PyLong_SHIFT; - shift_bits = (a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT; + shift_digits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) / PyLong_SHIFT); + shift_bits = (int)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); rem = v_rshift(x_digits, a->long_value.ob_digit + shift_digits, - a_size - shift_digits, (int)shift_bits); + a_size - shift_digits, shift_bits); x_size = a_size - shift_digits; /* For correct rounding below, we need the least significant bit of x to be 'sticky' for this shift: if any of the bits @@ -3181,21 +3495,13 @@ _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) /* Rescale; make correction if result is 1.0. */ dx /= 4.0 * EXP2_DBL_MANT_DIG; if (dx == 1.0) { - if (a_bits == PY_SSIZE_T_MAX) - goto overflow; + assert(a_bits < INT64_MAX); dx = 0.5; a_bits += 1; } *e = a_bits; return _PyLong_IsNegative(a) ? -dx : dx; - - overflow: - /* exponent > PY_SSIZE_T_MAX */ - PyErr_SetString(PyExc_OverflowError, - "huge integer: number of bits overflows a Py_ssize_t"); - *e = 0; - return -1.0; } /* Get a C double from an int object. Rounds to the nearest double, @@ -3204,7 +3510,7 @@ _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) double PyLong_AsDouble(PyObject *v) { - Py_ssize_t exponent; + int64_t exponent; double x; if (v == NULL) { @@ -3223,7 +3529,9 @@ PyLong_AsDouble(PyObject *v) return (double)medium_value((PyLongObject *)v); } x = _PyLong_Frexp((PyLongObject *)v, &exponent); - if ((x == -1.0 && PyErr_Occurred()) || exponent > DBL_MAX_EXP) { + assert(exponent >= 0); + assert(!PyErr_Occurred()); + if (exponent > DBL_MAX_EXP) { PyErr_SetString(PyExc_OverflowError, "int too large to convert to float"); return -1.0; @@ -3292,14 +3600,15 @@ long_dealloc(PyObject *self) } static Py_hash_t -long_hash(PyLongObject *v) +long_hash(PyObject *obj) { + PyLongObject *v = (PyLongObject *)obj; Py_uhash_t x; Py_ssize_t i; int sign; if (_PyLong_IsCompact(v)) { - x = _PyLong_CompactValue(v); + x = (Py_uhash_t)_PyLong_CompactValue(v); if (x == (Py_uhash_t)-1) { x = (Py_uhash_t)-2; } @@ -3437,11 +3746,12 @@ x_sub(PyLongObject *a, PyLongObject *b) return maybe_small_long(long_normalize(z)); } -PyObject * -_PyLong_Add(PyLongObject *a, PyLongObject *b) +static PyLongObject * +long_add(PyLongObject *a, PyLongObject *b) { if (_PyLong_BothAreCompact(a, b)) { - return _PyLong_FromSTwoDigits(medium_value(a) + medium_value(b)); + stwodigits z = medium_value(a) + medium_value(b); + return _PyLong_FromSTwoDigits(z); } PyLongObject *z; @@ -3466,24 +3776,31 @@ _PyLong_Add(PyLongObject *a, PyLongObject *b) else z = x_add(a, b); } - return (PyObject *)z; + return z; +} + +PyObject * +_PyLong_Add(PyLongObject *a, PyLongObject *b) +{ + return (PyObject*)long_add(a, b); } static PyObject * -long_add(PyLongObject *a, PyLongObject *b) +long_add_method(PyObject *a, PyObject *b) { CHECK_BINOP(a, b); - return _PyLong_Add(a, b); + return (PyObject*)long_add((PyLongObject*)a, (PyLongObject*)b); } -PyObject * -_PyLong_Subtract(PyLongObject *a, PyLongObject *b) -{ - PyLongObject *z; +static PyLongObject * +long_sub(PyLongObject *a, PyLongObject *b) +{ if (_PyLong_BothAreCompact(a, b)) { return _PyLong_FromSTwoDigits(medium_value(a) - medium_value(b)); } + + PyLongObject *z; if (_PyLong_IsNegative(a)) { if (_PyLong_IsNegative(b)) { z = x_sub(b, a); @@ -3502,16 +3819,23 @@ _PyLong_Subtract(PyLongObject *a, PyLongObject *b) else z = x_sub(a, b); } - return (PyObject *)z; + return z; +} + +PyObject * +_PyLong_Subtract(PyLongObject *a, PyLongObject *b) +{ + return (PyObject*)long_sub(a, b); } static PyObject * -long_sub(PyLongObject *a, PyLongObject *b) +long_sub_method(PyObject *a, PyObject *b) { CHECK_BINOP(a, b); - return _PyLong_Subtract(a, b); + return (PyObject*)long_sub((PyLongObject*)a, (PyLongObject*)b); } + /* Grade school multiplication, ignoring the signs. * Returns the absolute value of the product, or NULL if error. */ @@ -3530,7 +3854,7 @@ x_mul(PyLongObject *a, PyLongObject *b) memset(z->long_value.ob_digit, 0, _PyLong_DigitCount(z) * sizeof(digit)); if (a == b) { /* Efficient squaring per HAC, Algorithm 14.16: - * http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf + * https://cacr.uwaterloo.ca/hac/about/chap14.pdf * Gives slightly less than a 2x speedup when a == b, * via exploiting that each entry in the multiplication * pyramid appears twice (except for the size_a squares). @@ -3927,32 +4251,35 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) return NULL; } -PyObject * -_PyLong_Multiply(PyLongObject *a, PyLongObject *b) -{ - PyLongObject *z; +static PyLongObject* +long_mul(PyLongObject *a, PyLongObject *b) +{ /* fast path for single-digit multiplication */ if (_PyLong_BothAreCompact(a, b)) { stwodigits v = medium_value(a) * medium_value(b); return _PyLong_FromSTwoDigits(v); } - z = k_mul(a, b); + PyLongObject *z = k_mul(a, b); /* Negate if exactly one of the inputs is negative. */ if (!_PyLong_SameSign(a, b) && z) { _PyLong_Negate(&z); - if (z == NULL) - return NULL; } - return (PyObject *)z; + return z; +} + +PyObject * +_PyLong_Multiply(PyLongObject *a, PyLongObject *b) +{ + return (PyObject*)long_mul(a, b); } static PyObject * -long_mul(PyLongObject *a, PyLongObject *b) +long_mul_method(PyObject *a, PyObject *b) { CHECK_BINOP(a, b); - return _PyLong_Multiply(a, b); + return (PyObject*)long_mul((PyLongObject*)a, (PyLongObject*)b); } /* Fast modulo division for single-digit longs. */ @@ -4107,13 +4434,13 @@ l_divmod(PyLongObject *v, PyLongObject *w, if ((_PyLong_IsNegative(mod) && _PyLong_IsPositive(w)) || (_PyLong_IsPositive(mod) && _PyLong_IsNegative(w))) { PyLongObject *temp; - temp = (PyLongObject *) long_add(mod, w); + temp = long_add(mod, w); Py_SETREF(mod, temp); if (mod == NULL) { Py_DECREF(div); return -1; } - temp = (PyLongObject *) long_sub(div, (PyLongObject *)_PyLong_GetOne()); + temp = long_sub(div, (PyLongObject *)_PyLong_GetOne()); if (temp == NULL) { Py_DECREF(mod); Py_DECREF(div); @@ -4154,7 +4481,7 @@ l_mod(PyLongObject *v, PyLongObject *w, PyLongObject **pmod) if ((_PyLong_IsNegative(mod) && _PyLong_IsPositive(w)) || (_PyLong_IsPositive(mod) && _PyLong_IsNegative(w))) { PyLongObject *temp; - temp = (PyLongObject *) long_add(mod, w); + temp = long_add(mod, w); Py_SETREF(mod, temp); if (mod == NULL) return -1; @@ -4501,21 +4828,12 @@ long_divmod(PyObject *a, PyObject *b) static PyLongObject * long_invmod(PyLongObject *a, PyLongObject *n) { - PyLongObject *b, *c; - /* Should only ever be called for positive n */ assert(_PyLong_IsPositive(n)); - b = (PyLongObject *)PyLong_FromLong(1L); - if (b == NULL) { - return NULL; - } - c = (PyLongObject *)PyLong_FromLong(0L); - if (c == NULL) { - Py_DECREF(b); - return NULL; - } Py_INCREF(a); + PyLongObject *b = (PyLongObject *)Py_NewRef(_PyLong_GetOne()); + PyLongObject *c = (PyLongObject *)Py_NewRef(_PyLong_GetZero()); Py_INCREF(n); /* references now owned: a, b, c, n */ @@ -4532,7 +4850,7 @@ long_invmod(PyLongObject *a, PyLongObject *n) if (t == NULL) { goto Error; } - s = (PyLongObject *)long_sub(b, t); + s = long_sub(b, t); Py_DECREF(t); if (s == NULL) { goto Error; @@ -4738,7 +5056,7 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } else if (i <= HUGE_EXP_CUTOFF / PyLong_SHIFT ) { /* Left-to-right binary exponentiation (HAC Algorithm 14.79) */ - /* http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf */ + /* https://cacr.uwaterloo.ca/hac/about/chap14.pdf */ /* Find the first significant exponent bit. Search right to left * because we're primarily trying to cut overhead for small powers. @@ -4827,7 +5145,7 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } if (negativeOutput && !_PyLong_IsZero(z)) { - temp = (PyLongObject *)long_sub(z, c); + temp = long_sub(z, c); if (temp == NULL) goto Error; Py_SETREF(z, temp); @@ -4850,13 +5168,15 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } static PyObject * -long_invert(PyLongObject *v) +long_invert(PyObject *self) { + PyLongObject *v = _PyLong_CAST(self); + /* Implement ~x as -(x+1) */ - PyLongObject *x; if (_PyLong_IsCompact(v)) - return _PyLong_FromSTwoDigits(~medium_value(v)); - x = (PyLongObject *) long_add(v, (PyLongObject *)_PyLong_GetOne()); + return (PyObject*)_PyLong_FromSTwoDigits(~medium_value(v)); + + PyLongObject *x = long_add(v, (PyLongObject *)_PyLong_GetOne()); if (x == NULL) return NULL; _PyLong_Negate(&x); @@ -4865,64 +5185,45 @@ long_invert(PyLongObject *v) return (PyObject *)x; } -static PyObject * +static PyLongObject * long_neg(PyLongObject *v) { - PyLongObject *z; - if (_PyLong_IsCompact(v)) + if (_PyLong_IsCompact(v)) { return _PyLong_FromSTwoDigits(-medium_value(v)); - z = (PyLongObject *)_PyLong_Copy(v); - if (z != NULL) + } + + PyLongObject *z = (PyLongObject *)_PyLong_Copy(v); + if (z != NULL) { _PyLong_FlipSign(z); - return (PyObject *)z; + } + return z; } static PyObject * +long_neg_method(PyObject *v) +{ + return (PyObject*)long_neg(_PyLong_CAST(v)); +} + +static PyLongObject* long_abs(PyLongObject *v) { if (_PyLong_IsNegative(v)) return long_neg(v); else - return long_long((PyObject *)v); + return (PyLongObject*)long_long((PyObject *)v); } -static int -long_bool(PyLongObject *v) +static PyObject * +long_abs_method(PyObject *v) { - return !_PyLong_IsZero(v); + return (PyObject*)long_abs(_PyLong_CAST(v)); } -/* wordshift, remshift = divmod(shiftby, PyLong_SHIFT) */ static int -divmod_shift(PyObject *shiftby, Py_ssize_t *wordshift, digit *remshift) -{ - assert(PyLong_Check(shiftby)); - assert(!_PyLong_IsNegative((PyLongObject *)shiftby)); - Py_ssize_t lshiftby = PyLong_AsSsize_t((PyObject *)shiftby); - if (lshiftby >= 0) { - *wordshift = lshiftby / PyLong_SHIFT; - *remshift = lshiftby % PyLong_SHIFT; - return 0; - } - /* PyLong_Check(shiftby) is true and shiftby is not negative, so it must - be that PyLong_AsSsize_t raised an OverflowError. */ - assert(PyErr_ExceptionMatches(PyExc_OverflowError)); - PyErr_Clear(); - PyLongObject *wordshift_obj = divrem1((PyLongObject *)shiftby, PyLong_SHIFT, remshift); - if (wordshift_obj == NULL) { - return -1; - } - *wordshift = PyLong_AsSsize_t((PyObject *)wordshift_obj); - Py_DECREF(wordshift_obj); - if (*wordshift >= 0 && *wordshift < PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(digit)) { - return 0; - } - PyErr_Clear(); - /* Clip the value. With such large wordshift the right shift - returns 0 and the left shift raises an error in _PyLong_New(). */ - *wordshift = PY_SSIZE_T_MAX / sizeof(digit); - *remshift = 0; - return 0; +long_bool(PyObject *v) +{ + return !_PyLong_IsZero(_PyLong_CAST(v)); } /* Inner function for both long_rshift and _PyLong_Rshift, shifting an @@ -4948,7 +5249,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) m = medium_value(a); shift = wordshift == 0 ? remshift : PyLong_SHIFT; x = m < 0 ? ~(~m >> shift) : m >> shift; - return _PyLong_FromSTwoDigits(x); + return (PyObject*)_PyLong_FromSTwoDigits(x); } a_negative = _PyLong_IsNegative(a); @@ -5018,8 +5319,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) static PyObject * long_rshift(PyObject *a, PyObject *b) { - Py_ssize_t wordshift; - digit remshift; + int64_t shiftby; CHECK_BINOP(a, b); @@ -5030,24 +5330,45 @@ long_rshift(PyObject *a, PyObject *b) if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) - return NULL; - return long_rshift1((PyLongObject *)a, wordshift, remshift); + if (PyLong_AsInt64(b, &shiftby) < 0) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { + return NULL; + } + PyErr_Clear(); + if (_PyLong_IsNegative((PyLongObject *)a)) { + return PyLong_FromLong(-1); + } + else { + return PyLong_FromLong(0); + } + } + return _PyLong_Rshift(a, shiftby); } /* Return a >> shiftby. */ PyObject * -_PyLong_Rshift(PyObject *a, size_t shiftby) +_PyLong_Rshift(PyObject *a, int64_t shiftby) { Py_ssize_t wordshift; digit remshift; assert(PyLong_Check(a)); + assert(shiftby >= 0); if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - wordshift = shiftby / PyLong_SHIFT; - remshift = shiftby % PyLong_SHIFT; +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { + if (_PyLong_IsNegative((PyLongObject *)a)) { + return PyLong_FromLong(-1); + } + else { + return PyLong_FromLong(0); + } + } +#endif + wordshift = (Py_ssize_t)(shiftby / PyLong_SHIFT); + remshift = (digit)(shiftby % PyLong_SHIFT); return long_rshift1((PyLongObject *)a, wordshift, remshift); } @@ -5062,7 +5383,7 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) stwodigits m = medium_value(a); // bypass undefined shift operator behavior stwodigits x = m < 0 ? -(-m << remshift) : m << remshift; - return _PyLong_FromSTwoDigits(x); + return (PyObject*)_PyLong_FromSTwoDigits(x); } oldsize = _PyLong_DigitCount(a); @@ -5092,42 +5413,61 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) return (PyObject *) maybe_small_long(z); } + static PyObject * -long_lshift(PyObject *a, PyObject *b) +long_lshift_method(PyObject *aa, PyObject *bb) { - Py_ssize_t wordshift; - digit remshift; - - CHECK_BINOP(a, b); + CHECK_BINOP(aa, bb); + PyLongObject *a = (PyLongObject*)aa; + PyLongObject *b = (PyLongObject*)bb; - if (_PyLong_IsNegative((PyLongObject *)b)) { + if (_PyLong_IsNegative(b)) { PyErr_SetString(PyExc_ValueError, "negative shift count"); return NULL; } - if (_PyLong_IsZero((PyLongObject *)a)) { + if (_PyLong_IsZero(a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) + + int64_t shiftby; + if (PyLong_AsInt64(bb, &shiftby) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + } return NULL; - return long_lshift1((PyLongObject *)a, wordshift, remshift); + } + return long_lshift_int64(a, shiftby); } /* Return a << shiftby. */ -PyObject * -_PyLong_Lshift(PyObject *a, size_t shiftby) +static PyObject * +long_lshift_int64(PyLongObject *a, int64_t shiftby) { - Py_ssize_t wordshift; - digit remshift; + assert(shiftby >= 0); - assert(PyLong_Check(a)); - if (_PyLong_IsZero((PyLongObject *)a)) { + if (_PyLong_IsZero(a)) { return PyLong_FromLong(0); } - wordshift = shiftby / PyLong_SHIFT; - remshift = shiftby % PyLong_SHIFT; - return long_lshift1((PyLongObject *)a, wordshift, remshift); +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + return NULL; + } +#endif + Py_ssize_t wordshift = (Py_ssize_t)(shiftby / PyLong_SHIFT); + digit remshift = (digit)(shiftby % PyLong_SHIFT); + return long_lshift1(a, wordshift, remshift); } +PyObject * +_PyLong_Lshift(PyObject *a, int64_t shiftby) +{ + return long_lshift_int64(_PyLong_CAST(a), shiftby); +} + + /* Compute two's complement of digit vector a[0:m], writing result to z[0:m]. The digit vector a need not be normalized, but should not be entirely zero. a and z may point to the same digit vector. */ @@ -5275,7 +5615,7 @@ long_and(PyObject *a, PyObject *b) PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; if (_PyLong_IsCompact(x) && _PyLong_IsCompact(y)) { - return _PyLong_FromSTwoDigits(medium_value(x) & medium_value(y)); + return (PyObject*)_PyLong_FromSTwoDigits(medium_value(x) & medium_value(y)); } return long_bitwise(x, '&', y); } @@ -5287,7 +5627,7 @@ long_xor(PyObject *a, PyObject *b) PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; if (_PyLong_IsCompact(x) && _PyLong_IsCompact(y)) { - return _PyLong_FromSTwoDigits(medium_value(x) ^ medium_value(y)); + return (PyObject*)_PyLong_FromSTwoDigits(medium_value(x) ^ medium_value(y)); } return long_bitwise(x, '^', y); } @@ -5299,7 +5639,7 @@ long_or(PyObject *a, PyObject *b) PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; if (_PyLong_IsCompact(x) && _PyLong_IsCompact(y)) { - return _PyLong_FromSTwoDigits(medium_value(x) | medium_value(y)); + return (PyObject*)_PyLong_FromSTwoDigits(medium_value(x) | medium_value(y)); } return long_bitwise(x, '|', y); } @@ -5333,10 +5673,10 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) } /* Initial reduction: make sure that 0 <= b <= a. */ - a = (PyLongObject *)long_abs(a); + a = long_abs(a); if (a == NULL) return NULL; - b = (PyLongObject *)long_abs(b); + b = long_abs(b); if (b == NULL) { Py_DECREF(a); return NULL; @@ -5718,12 +6058,11 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) /* compare twice the remainder with the divisor, to see if we need to adjust the quotient and remainder */ - PyObject *one = _PyLong_GetOne(); // borrowed reference - twice_rem = long_lshift((PyObject *)rem, one); + twice_rem = long_lshift_int64(rem, 1); if (twice_rem == NULL) goto error; if (quo_is_neg) { - temp = long_neg((PyLongObject*)twice_rem); + temp = (PyObject*)long_neg((PyLongObject*)twice_rem); Py_SETREF(twice_rem, temp); if (twice_rem == NULL) goto error; @@ -5734,18 +6073,19 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) quo_is_odd = (quo->long_value.ob_digit[0] & 1) != 0; if ((_PyLong_IsNegative((PyLongObject *)b) ? cmp < 0 : cmp > 0) || (cmp == 0 && quo_is_odd)) { /* fix up quotient */ + PyObject *one = _PyLong_GetOne(); // borrowed reference if (quo_is_neg) - temp = long_sub(quo, (PyLongObject *)one); + temp = (PyObject*)long_sub(quo, (PyLongObject *)one); else - temp = long_add(quo, (PyLongObject *)one); + temp = (PyObject*)long_add(quo, (PyLongObject *)one); Py_SETREF(quo, (PyLongObject *)temp); if (quo == NULL) goto error; /* and remainder */ if (quo_is_neg) - temp = long_add(rem, (PyLongObject *)b); + temp = (PyObject*)long_add(rem, (PyLongObject *)b); else - temp = long_sub(rem, (PyLongObject *)b); + temp = (PyObject*)long_sub(rem, (PyLongObject *)b); Py_SETREF(rem, (PyLongObject *)temp); if (rem == NULL) goto error; @@ -5769,7 +6109,7 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) /*[clinic input] int.__round__ - ndigits as o_ndigits: object = NULL + ndigits as o_ndigits: object = None / Rounding an Integral returns itself. @@ -5779,10 +6119,8 @@ Rounding with an ndigits argument also returns an integer. static PyObject * int___round___impl(PyObject *self, PyObject *o_ndigits) -/*[clinic end generated code: output=954fda6b18875998 input=1614cf23ec9e18c3]*/ +/*[clinic end generated code: output=954fda6b18875998 input=30c2aec788263144]*/ { - PyObject *temp, *result, *ndigits; - /* To round an integer m to the nearest 10**n (n positive), we make use of * the divmod_near operation, defined by: * @@ -5797,10 +6135,10 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) * * m - divmod_near(m, 10**n)[1]. */ - if (o_ndigits == NULL) + if (o_ndigits == Py_None) return long_long(self); - ndigits = _PyNumber_Index(o_ndigits); + PyObject *ndigits = _PyNumber_Index(o_ndigits); if (ndigits == NULL) return NULL; @@ -5811,12 +6149,12 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) } /* result = self - divmod_near(self, 10 ** -ndigits)[1] */ - temp = long_neg((PyLongObject*)ndigits); + PyObject *temp = (PyObject*)long_neg((PyLongObject*)ndigits); Py_SETREF(ndigits, temp); if (ndigits == NULL) return NULL; - result = PyLong_FromLong(10L); + PyObject *result = PyLong_FromLong(10); if (result == NULL) { Py_DECREF(ndigits); return NULL; @@ -5833,8 +6171,8 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) if (result == NULL) return NULL; - temp = long_sub((PyLongObject *)self, - (PyLongObject *)PyTuple_GET_ITEM(result, 1)); + temp = (PyObject*)long_sub((PyLongObject*)self, + (PyLongObject*)PyTuple_GET_ITEM(result, 1)); Py_SETREF(result, temp); return result; @@ -5871,51 +6209,10 @@ static PyObject * int_bit_length_impl(PyObject *self) /*[clinic end generated code: output=fc1977c9353d6a59 input=e4eb7a587e849a32]*/ { - PyLongObject *result, *x, *y; - Py_ssize_t ndigits; - int msd_bits; - digit msd; - - assert(self != NULL); - assert(PyLong_Check(self)); - - ndigits = _PyLong_DigitCount((PyLongObject *)self); - if (ndigits == 0) - return PyLong_FromLong(0); - - msd = ((PyLongObject *)self)->long_value.ob_digit[ndigits-1]; - msd_bits = bit_length_digit(msd); - - if (ndigits <= PY_SSIZE_T_MAX/PyLong_SHIFT) - return PyLong_FromSsize_t((ndigits-1)*PyLong_SHIFT + msd_bits); - - /* expression above may overflow; use Python integers instead */ - result = (PyLongObject *)PyLong_FromSsize_t(ndigits - 1); - if (result == NULL) - return NULL; - x = (PyLongObject *)PyLong_FromLong(PyLong_SHIFT); - if (x == NULL) - goto error; - y = (PyLongObject *)long_mul(result, x); - Py_DECREF(x); - if (y == NULL) - goto error; - Py_SETREF(result, y); - - x = (PyLongObject *)PyLong_FromLong((long)msd_bits); - if (x == NULL) - goto error; - y = (PyLongObject *)long_add(result, x); - Py_DECREF(x); - if (y == NULL) - goto error; - Py_SETREF(result, y); - - return (PyObject *)result; - - error: - Py_DECREF(result); - return NULL; + int64_t nbits = _PyLong_NumBits(self); + assert(nbits >= 0); + assert(!PyErr_Occurred()); + return PyLong_FromInt64(nbits); } static int @@ -5949,40 +6246,13 @@ int_bit_count_impl(PyObject *self) PyLongObject *z = (PyLongObject *)self; Py_ssize_t ndigits = _PyLong_DigitCount(z); - Py_ssize_t bit_count = 0; + int64_t bit_count = 0; - /* Each digit has up to PyLong_SHIFT ones, so the accumulated bit count - from the first PY_SSIZE_T_MAX/PyLong_SHIFT digits can't overflow a - Py_ssize_t. */ - Py_ssize_t ndigits_fast = Py_MIN(ndigits, PY_SSIZE_T_MAX/PyLong_SHIFT); - for (Py_ssize_t i = 0; i < ndigits_fast; i++) { + for (Py_ssize_t i = 0; i < ndigits; i++) { bit_count += popcount_digit(z->long_value.ob_digit[i]); } - PyObject *result = PyLong_FromSsize_t(bit_count); - if (result == NULL) { - return NULL; - } - - /* Use Python integers if bit_count would overflow. */ - for (Py_ssize_t i = ndigits_fast; i < ndigits; i++) { - PyObject *x = PyLong_FromLong(popcount_digit(z->long_value.ob_digit[i])); - if (x == NULL) { - goto error; - } - PyObject *y = long_add((PyLongObject *)result, (PyLongObject *)x); - Py_DECREF(x); - if (y == NULL) { - goto error; - } - Py_SETREF(result, y); - } - - return result; - - error: - Py_DECREF(result); - return NULL; + return PyLong_FromInt64(bit_count); } /*[clinic input] @@ -6026,7 +6296,7 @@ int.to_bytes the most significant byte is at the beginning of the byte array. If byteorder is 'little', the most significant byte is at the end of the byte array. To request the native byte order of the host system, use - `sys.byteorder' as the byte order value. Default is to use 'big'. + sys.byteorder as the byte order value. Default is to use 'big'. * signed as is_signed: bool = False Determines whether two's complement is used to represent the integer. @@ -6039,7 +6309,7 @@ Return an array of bytes representing an integer. static PyObject * int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=89c801df114050a3 input=d42ecfb545039d71]*/ +/*[clinic end generated code: output=89c801df114050a3 input=a0103d0e9ad85c2b]*/ { int little_endian; PyObject *bytes; @@ -6068,7 +6338,7 @@ int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, if (_PyLong_AsByteArray((PyLongObject *)self, (unsigned char *)PyBytes_AS_STRING(bytes), - length, little_endian, is_signed) < 0) { + length, little_endian, is_signed, 1) < 0) { Py_DECREF(bytes); return NULL; } @@ -6090,7 +6360,7 @@ int.from_bytes the most significant byte is at the beginning of the byte array. If byteorder is 'little', the most significant byte is at the end of the byte array. To request the native byte order of the host system, use - `sys.byteorder' as the byte order value. Default is to use 'big'. + sys.byteorder as the byte order value. Default is to use 'big'. * signed as is_signed: bool = False Indicates whether two's complement is used to represent the integer. @@ -6101,7 +6371,7 @@ Return the integer represented by the given array of bytes. static PyObject * int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=efc5d68e31f9314f input=33326dccdd655553]*/ +/*[clinic end generated code: output=efc5d68e31f9314f input=2ff527997fe7b0c5]*/ { int little_endian; PyObject *long_obj, *bytes; @@ -6223,7 +6493,7 @@ PyDoc_STRVAR(long_doc, int(x, base=10) -> integer\n\ \n\ Convert a number or string to an integer, or return 0 if no arguments\n\ -are given. If x is a number, return x.__int__(). For floating point\n\ +are given. If x is a number, return x.__int__(). For floating-point\n\ numbers, this truncates towards zero.\n\ \n\ If x is not a number or if base is given, then x must be a string,\n\ @@ -6235,18 +6505,18 @@ Base 0 means to interpret the base from the string as an integer literal.\n\ 4"); static PyNumberMethods long_as_number = { - (binaryfunc)long_add, /*nb_add*/ - (binaryfunc)long_sub, /*nb_subtract*/ - (binaryfunc)long_mul, /*nb_multiply*/ + long_add_method, /*nb_add*/ + long_sub_method, /*nb_subtract*/ + long_mul_method, /*nb_multiply*/ long_mod, /*nb_remainder*/ long_divmod, /*nb_divmod*/ long_pow, /*nb_power*/ - (unaryfunc)long_neg, /*nb_negative*/ + long_neg_method, /*nb_negative*/ long_long, /*tp_positive*/ - (unaryfunc)long_abs, /*tp_absolute*/ - (inquiry)long_bool, /*tp_bool*/ - (unaryfunc)long_invert, /*nb_invert*/ - long_lshift, /*nb_lshift*/ + long_abs_method, /*tp_absolute*/ + long_bool, /*tp_bool*/ + long_invert, /*nb_invert*/ + long_lshift_method, /*nb_lshift*/ long_rshift, /*nb_rshift*/ long_and, /*nb_and*/ long_xor, /*nb_xor*/ @@ -6285,7 +6555,7 @@ PyTypeObject PyLong_Type = { &long_as_number, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)long_hash, /* tp_hash */ + long_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -6396,12 +6666,78 @@ _PyLong_FiniTypes(PyInterpreterState *interp) int PyUnstable_Long_IsCompact(const PyLongObject* op) { - return _PyLong_IsCompact(op); + return _PyLong_IsCompact((PyLongObject*)op); } #undef PyUnstable_Long_CompactValue Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) { - return _PyLong_CompactValue(op); + return _PyLong_CompactValue((PyLongObject*)op); +} + +PyObject* PyLong_FromInt32(int32_t value) +{ return PyLong_FromNativeBytes(&value, sizeof(value), -1); } + +PyObject* PyLong_FromUInt32(uint32_t value) +{ return PyLong_FromUnsignedNativeBytes(&value, sizeof(value), -1); } + +PyObject* PyLong_FromInt64(int64_t value) +{ return PyLong_FromNativeBytes(&value, sizeof(value), -1); } + +PyObject* PyLong_FromUInt64(uint64_t value) +{ return PyLong_FromUnsignedNativeBytes(&value, sizeof(value), -1); } + +#define LONG_TO_INT(obj, value, type_name) \ + do { \ + int flags = (Py_ASNATIVEBYTES_NATIVE_ENDIAN \ + | Py_ASNATIVEBYTES_ALLOW_INDEX); \ + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, value, sizeof(*value), flags); \ + if (bytes < 0) { \ + return -1; \ + } \ + if ((size_t)bytes > sizeof(*value)) { \ + PyErr_SetString(PyExc_OverflowError, \ + "Python int too large to convert to " type_name); \ + return -1; \ + } \ + return 0; \ + } while (0) + +int PyLong_AsInt32(PyObject *obj, int32_t *value) +{ + LONG_TO_INT(obj, value, "C int32_t"); +} + +int PyLong_AsInt64(PyObject *obj, int64_t *value) +{ + LONG_TO_INT(obj, value, "C int64_t"); +} + +#define LONG_TO_UINT(obj, value, type_name) \ + do { \ + int flags = (Py_ASNATIVEBYTES_NATIVE_ENDIAN \ + | Py_ASNATIVEBYTES_UNSIGNED_BUFFER \ + | Py_ASNATIVEBYTES_REJECT_NEGATIVE \ + | Py_ASNATIVEBYTES_ALLOW_INDEX); \ + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, value, sizeof(*value), flags); \ + if (bytes < 0) { \ + return -1; \ + } \ + if ((size_t)bytes > sizeof(*value)) { \ + PyErr_SetString(PyExc_OverflowError, \ + "Python int too large to convert to " type_name); \ + return -1; \ + } \ + return 0; \ + } while (0) + +int PyLong_AsUInt32(PyObject *obj, uint32_t *value) +{ + LONG_TO_UINT(obj, value, "C uint32_t"); +} + +int PyLong_AsUInt64(PyObject *obj, uint64_t *value) +{ + LONG_TO_UINT(obj, value, "C uint64_t"); } diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 6a38952fdc1f3b..d4672e8198cb24 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -109,8 +109,6 @@ mbuf_release(_PyManagedBufferObject *self) if (self->flags&_Py_MANAGED_BUFFER_RELEASED) return; - /* NOTE: at this point self->exports can still be > 0 if this function - is called from mbuf_clear() to break up a reference cycle. */ self->flags |= _Py_MANAGED_BUFFER_RELEASED; /* PyBuffer_Release() decrements master->obj and sets it to NULL. */ @@ -268,7 +266,7 @@ PyTypeObject _PyManagedBuffer_Type = { /* Assumptions: ndim >= 1. The macro tests for a corner case that should perhaps be explicitly forbidden in the PEP. */ #define HAVE_SUBOFFSETS_IN_LAST_DIM(view) \ - (view->suboffsets && view->suboffsets[dest->ndim-1] >= 0) + (view->suboffsets && view->suboffsets[view->ndim-1] >= 0) static inline int last_dim_is_contiguous(const Py_buffer *dest, const Py_buffer *src) @@ -1096,32 +1094,19 @@ PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char orde /* Inform the managed buffer that this particular memoryview will not access the underlying buffer again. If no other memoryviews are registered with the managed buffer, the underlying buffer is released instantly and - marked as inaccessible for both the memoryview and the managed buffer. - - This function fails if the memoryview itself has exported buffers. */ -static int + marked as inaccessible for both the memoryview and the managed buffer. */ +static void _memory_release(PyMemoryViewObject *self) { + assert(self->exports == 0); if (self->flags & _Py_MEMORYVIEW_RELEASED) - return 0; + return; - if (self->exports == 0) { - self->flags |= _Py_MEMORYVIEW_RELEASED; - assert(self->mbuf->exports > 0); - if (--self->mbuf->exports == 0) - mbuf_release(self->mbuf); - return 0; + self->flags |= _Py_MEMORYVIEW_RELEASED; + assert(self->mbuf->exports > 0); + if (--self->mbuf->exports == 0) { + mbuf_release(self->mbuf); } - if (self->exports > 0) { - PyErr_Format(PyExc_BufferError, - "memoryview has %zd exported buffer%s", self->exports, - self->exports==1 ? "" : "s"); - return -1; - } - - PyErr_SetString(PyExc_SystemError, - "_memory_release(): negative export count"); - return -1; } /*[clinic input] @@ -1134,9 +1119,21 @@ static PyObject * memoryview_release_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=d0b7e3ba95b7fcb9 input=bc71d1d51f4a52f0]*/ { - if (_memory_release(self) < 0) + if (self->exports == 0) { + _memory_release(self); + Py_RETURN_NONE; + } + + if (self->exports > 0) { + PyErr_Format(PyExc_BufferError, + "memoryview has %zd exported buffer%s", self->exports, + self->exports==1 ? "" : "s"); return NULL; - Py_RETURN_NONE; + } + + PyErr_SetString(PyExc_SystemError, + "memoryview: negative export count"); + return NULL; } static void @@ -1145,7 +1142,7 @@ memory_dealloc(PyObject *_self) PyMemoryViewObject *self = (PyMemoryViewObject *)_self; assert(self->exports == 0); _PyObject_GC_UNTRACK(self); - (void)_memory_release(self); + _memory_release(self); Py_CLEAR(self->mbuf); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); @@ -1164,8 +1161,10 @@ static int memory_clear(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - (void)_memory_release(self); - Py_CLEAR(self->mbuf); + if (self->exports == 0) { + _memory_release(self); + Py_CLEAR(self->mbuf); + } return 0; } @@ -3087,7 +3086,7 @@ memory_hash(PyObject *_self) } /* Can't fail */ - self->hash = _Py_HashBytes(mem, view->len); + self->hash = Py_HashBuffer(mem, view->len); if (mem != view->buf) PyMem_Free(mem); @@ -3255,6 +3254,9 @@ PyDoc_STRVAR(memory_f_contiguous_doc, "A bool indicating whether the memory is Fortran contiguous."); PyDoc_STRVAR(memory_contiguous_doc, "A bool indicating whether the memory is contiguous."); +PyDoc_STRVAR(memory_exit_doc, + "__exit__($self, /, *exc_info)\n--\n\n" + "Release the underlying buffer exposed by the memoryview object."); static PyGetSetDef memory_getsetlist[] = { @@ -3283,7 +3285,8 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_TOREADONLY_METHODDEF MEMORYVIEW__FROM_FLAGS_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, - {"__exit__", memory_exit, METH_VARARGS, NULL}, + {"__exit__", memory_exit, METH_VARARGS, memory_exit_doc}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} }; diff --git a/Objects/methodobject.c b/Objects/methodobject.c index b40b2821c3880d..345da4607423cf 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -156,8 +156,9 @@ PyCMethod_GetClass(PyObject *op) /* Methods (the standard built-in methods, that is) */ static void -meth_dealloc(PyCFunctionObject *m) +meth_dealloc(PyObject *self) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); // The Py_TRASHCAN mechanism requires that we be able to // call PyObject_GC_UnTrack twice on an object. PyObject_GC_UnTrack(m); @@ -175,8 +176,9 @@ meth_dealloc(PyCFunctionObject *m) } static PyObject * -meth_reduce(PyCFunctionObject *m, PyObject *Py_UNUSED(ignored)) +meth_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); if (m->m_self == NULL || PyModule_Check(m->m_self)) return PyUnicode_FromString(m->m_ml->ml_name); @@ -185,32 +187,35 @@ meth_reduce(PyCFunctionObject *m, PyObject *Py_UNUSED(ignored)) } static PyMethodDef meth_methods[] = { - {"__reduce__", (PyCFunction)meth_reduce, METH_NOARGS, NULL}, + {"__reduce__", meth_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; static PyObject * -meth_get__text_signature__(PyCFunctionObject *m, void *closure) +meth_get__text_signature__(PyObject *self, void *closure) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); return _PyType_GetTextSignatureFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc, m->m_ml->ml_flags); } static PyObject * -meth_get__doc__(PyCFunctionObject *m, void *closure) +meth_get__doc__(PyObject *self, void *closure) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); return _PyType_GetDocFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc); } static PyObject * -meth_get__name__(PyCFunctionObject *m, void *closure) +meth_get__name__(PyObject *self, void *closure) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); return PyUnicode_FromString(m->m_ml->ml_name); } static PyObject * -meth_get__qualname__(PyCFunctionObject *m, void *closure) +meth_get__qualname__(PyObject *self, void *closure) { /* If __self__ is a module or NULL, return m.__name__ (e.g. len.__qualname__ == 'len') @@ -220,14 +225,15 @@ meth_get__qualname__(PyCFunctionObject *m, void *closure) Otherwise return type(m.__self__).__qualname__ + '.' + m.__name__ (e.g. [].append.__qualname__ == 'list.append') */ - PyObject *type, *type_qualname, *res; - if (m->m_self == NULL || PyModule_Check(m->m_self)) + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); + if (m->m_self == NULL || PyModule_Check(m->m_self)) { return PyUnicode_FromString(m->m_ml->ml_name); + } - type = PyType_Check(m->m_self) ? m->m_self : (PyObject*)Py_TYPE(m->m_self); + PyObject *type = PyType_Check(m->m_self) ? m->m_self : (PyObject*)Py_TYPE(m->m_self); - type_qualname = PyObject_GetAttr(type, &_Py_ID(__qualname__)); + PyObject *type_qualname = PyObject_GetAttr(type, &_Py_ID(__qualname__)); if (type_qualname == NULL) return NULL; @@ -238,14 +244,15 @@ meth_get__qualname__(PyCFunctionObject *m, void *closure) return NULL; } - res = PyUnicode_FromFormat("%S.%s", type_qualname, m->m_ml->ml_name); + PyObject *res = PyUnicode_FromFormat("%S.%s", type_qualname, m->m_ml->ml_name); Py_DECREF(type_qualname); return res; } static int -meth_traverse(PyCFunctionObject *m, visitproc visit, void *arg) +meth_traverse(PyObject *self, visitproc visit, void *arg) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); Py_VISIT(PyCFunction_GET_CLASS(m)); Py_VISIT(m->m_self); Py_VISIT(m->m_module); @@ -253,22 +260,22 @@ meth_traverse(PyCFunctionObject *m, visitproc visit, void *arg) } static PyObject * -meth_get__self__(PyCFunctionObject *m, void *closure) +meth_get__self__(PyObject *meth, void *closure) { - PyObject *self; - - self = PyCFunction_GET_SELF(m); - if (self == NULL) + PyCFunctionObject *m = _PyCFunctionObject_CAST(meth); + PyObject *self = PyCFunction_GET_SELF(m); + if (self == NULL) { self = Py_None; + } return Py_NewRef(self); } -static PyGetSetDef meth_getsets [] = { - {"__doc__", (getter)meth_get__doc__, NULL, NULL}, - {"__name__", (getter)meth_get__name__, NULL, NULL}, - {"__qualname__", (getter)meth_get__qualname__, NULL, NULL}, - {"__self__", (getter)meth_get__self__, NULL, NULL}, - {"__text_signature__", (getter)meth_get__text_signature__, NULL, NULL}, +static PyGetSetDef meth_getsets[] = { + {"__doc__", meth_get__doc__, NULL, NULL}, + {"__name__", meth_get__name__, NULL, NULL}, + {"__qualname__", meth_get__qualname__, NULL, NULL}, + {"__self__", meth_get__self__, NULL, NULL}, + {"__text_signature__", meth_get__text_signature__, NULL, NULL}, {0} }; @@ -280,15 +287,18 @@ static PyMemberDef meth_members[] = { }; static PyObject * -meth_repr(PyCFunctionObject *m) +meth_repr(PyObject *self) { - if (m->m_self == NULL || PyModule_Check(m->m_self)) + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); + if (m->m_self == NULL || PyModule_Check(m->m_self)) { return PyUnicode_FromFormat("", - m->m_ml->ml_name); + m->m_ml->ml_name); + } + return PyUnicode_FromFormat("", - m->m_ml->ml_name, - Py_TYPE(m->m_self)->tp_name, - m->m_self); + m->m_ml->ml_name, + Py_TYPE(m->m_self)->tp_name, + m->m_self); } static PyObject * @@ -317,14 +327,15 @@ meth_richcompare(PyObject *self, PyObject *other, int op) } static Py_hash_t -meth_hash(PyCFunctionObject *a) +meth_hash(PyObject *self) { - Py_hash_t x, y; - x = _Py_HashPointer(a->m_self); - y = _Py_HashPointer((void*)(a->m_ml->ml_meth)); + PyCFunctionObject *a = _PyCFunctionObject_CAST(self); + Py_hash_t x = PyObject_GenericHash(a->m_self); + Py_hash_t y = _Py_HashPointer((void*)(a->m_ml->ml_meth)); x ^= y; - if (x == -1) + if (x == -1) { x = -2; + } return x; } @@ -334,16 +345,16 @@ PyTypeObject PyCFunction_Type = { "builtin_function_or_method", sizeof(PyCFunctionObject), 0, - (destructor)meth_dealloc, /* tp_dealloc */ + meth_dealloc, /* tp_dealloc */ offsetof(PyCFunctionObject, vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)meth_repr, /* tp_repr */ + meth_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)meth_hash, /* tp_hash */ + meth_hash, /* tp_hash */ cfunction_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -352,7 +363,7 @@ PyTypeObject PyCFunction_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)meth_traverse, /* tp_traverse */ + meth_traverse, /* tp_traverse */ 0, /* tp_clear */ meth_richcompare, /* tp_richcompare */ offsetof(PyCFunctionObject, m_weakreflist), /* tp_weaklistoffset */ @@ -417,7 +428,7 @@ cfunction_vectorcall_FASTCALL( return NULL; } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - _PyCFunctionFast meth = (_PyCFunctionFast) + PyCFunctionFast meth = (PyCFunctionFast) cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -433,7 +444,7 @@ cfunction_vectorcall_FASTCALL_KEYWORDS( { PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - _PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords) + PyCFunctionFastWithKeywords meth = (PyCFunctionFastWithKeywords) cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -552,4 +563,3 @@ cfunction_call(PyObject *func, PyObject *args, PyObject *kwargs) } return _Py_CheckFunctionResult(tstate, func, result, NULL); } - diff --git a/Objects/mimalloc/alloc.c b/Objects/mimalloc/alloc.c index f96c6f0b37f873..44c84cf1931717 100644 --- a/Objects/mimalloc/alloc.c +++ b/Objects/mimalloc/alloc.c @@ -26,6 +26,15 @@ terms of the MIT license. A copy of the license can be found in the file // Allocation // ------------------------------------------------------ +#if (MI_DEBUG>0) +static inline void mi_debug_fill(mi_page_t* page, mi_block_t* block, int c, size_t size) { + size_t offset = (size_t)page->debug_offset; + if (offset < size) { + memset((char*)block + offset, c, size - offset); + } +} +#endif + // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { @@ -65,7 +74,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if (!zero && !mi_page_is_huge(page)) { - memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); + mi_debug_fill(page, block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); } #elif (MI_SECURE!=0) if (!zero) { block->next = 0; } // don't leak internal data @@ -228,7 +237,7 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL? { - // Suspicous: decoded value a in block is in the same page (or NULL) -- maybe a double free? + // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free? // (continue in separate function to improve code generation) is_double_free = mi_check_is_double_freex(page, block); } @@ -426,7 +435,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading if (segment->kind != MI_SEGMENT_HUGE) { // not for huge segments as we just reset the content - memset(block, MI_DEBUG_FREED, mi_usable_size(block)); + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_usable_size(block)); } #endif @@ -480,7 +489,7 @@ static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block mi_check_padding(page, block); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if (!mi_page_is_huge(page)) { // huge page content may be already decommitted - memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_page_block_size(page)); } #endif mi_block_set_next(page, block, page->local_free); @@ -575,7 +584,7 @@ void mi_free(void* p) mi_attr_noexcept mi_check_padding(page, block); mi_stat_free(page, block); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif mi_track_free_size(p, mi_page_usable_size_of(page,block)); // faster then mi_usable_size as we already know the page and that p is unaligned mi_block_set_next(page, block, page->local_free); @@ -600,7 +609,10 @@ bool _mi_free_delayed_block(mi_block_t* block) { // get segment and page const mi_segment_t* const segment = _mi_ptr_segment(block); mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); +#ifndef Py_GIL_DISABLED + // The GC traverses heaps of other threads, which can trigger this assert. mi_assert_internal(_mi_thread_id() == segment->thread_id); +#endif mi_page_t* const page = _mi_segment_page_of(segment, block); // Clear the no-delayed flag so delayed freeing is used again for this page. diff --git a/Objects/mimalloc/arena.c b/Objects/mimalloc/arena.c index f8883603860dce..5db5d950c43b68 100644 --- a/Objects/mimalloc/arena.c +++ b/Objects/mimalloc/arena.c @@ -269,7 +269,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t ar return p; } -// allocate in a speficic arena +// allocate in a specific arena static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_node, int numa_node, size_t size, size_t alignment, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld ) { @@ -493,7 +493,7 @@ static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, size_t bitidx = startidx; bool all_purged = false; while (bitidx < endidx) { - // count consequetive ones in the purge mask + // count consecutive ones in the purge mask size_t count = 0; while (bitidx + count < endidx && (purge & ((size_t)1 << (bitidx + count))) != 0) { count++; @@ -530,7 +530,7 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi if (purge != 0) { size_t bitidx = 0; while (bitidx < MI_BITMAP_FIELD_BITS) { - // find consequetive range of ones in the purge mask + // find consecutive range of ones in the purge mask size_t bitlen = 0; while (bitidx + bitlen < MI_BITMAP_FIELD_BITS && (purge & ((size_t)1 << (bitidx + bitlen))) != 0) { bitlen++; diff --git a/Objects/mimalloc/bitmap.c b/Objects/mimalloc/bitmap.c index ec3c755822dac1..31830756f58c7c 100644 --- a/Objects/mimalloc/bitmap.c +++ b/Objects/mimalloc/bitmap.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ---------------------------------------------------------------------------- Concurrent bitmap that can set/reset sequences of bits atomically, -represeted as an array of fields where each field is a machine word (`size_t`) +represented as an array of fields where each field is a machine word (`size_t`) There are two api's; the standard one cannot have sequences that cross between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). @@ -108,7 +108,7 @@ bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fiel return false; } -// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled +// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fulfilled bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_pred_fun_t pred_fun, void* pred_arg, diff --git a/Objects/mimalloc/bitmap.h b/Objects/mimalloc/bitmap.h index 9ba15d5d6f09ea..d60f6dd3cb8e27 100644 --- a/Objects/mimalloc/bitmap.h +++ b/Objects/mimalloc/bitmap.h @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ---------------------------------------------------------------------------- Concurrent bitmap that can set/reset sequences of bits atomically, -represeted as an array of fields where each field is a machine word (`size_t`) +represented as an array of fields where each field is a machine word (`size_t`) There are two api's; the standard one cannot have sequences that cross between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). @@ -72,7 +72,7 @@ bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_ // For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); -// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled +// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fulfilled typedef bool (mi_cdecl *mi_bitmap_pred_fun_t)(mi_bitmap_index_t bitmap_idx, void* pred_arg); bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_pred_fun_t pred_fun, void* pred_arg, mi_bitmap_index_t* bitmap_idx); diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index 154dad0b128480..d92dc768e5ec28 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -98,7 +98,10 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t if (mi_page_all_free(page)) { // no more used blocks, free the page. // note: this will free retired pages as well. - _mi_page_free(page, pq, collect >= MI_FORCE); + bool freed = _PyMem_mi_page_maybe_free(page, pq, collect >= MI_FORCE); + if (!freed && collect == MI_ABANDON) { + _mi_page_abandon(page, pq); + } } else if (collect == MI_ABANDON) { // still used blocks but the thread is done; abandon the page @@ -153,6 +156,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) // collect retired pages _mi_heap_collect_retired(heap, force); + // free pages that were delayed with QSBR + _PyMem_mi_heap_collect_qsbr(heap); + // collect all pages owned by this thread mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL ); @@ -440,7 +446,7 @@ void mi_heap_delete(mi_heap_t* heap) if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (!mi_heap_is_backing(heap)) { - // tranfer still used pages to the backing heap + // transfer still used pages to the backing heap mi_heap_absorb(heap->tld->heap_backing, heap); } else { diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c index 5897f0512f8ef9..81b241063ff40f 100644 --- a/Objects/mimalloc/init.c +++ b/Objects/mimalloc/init.c @@ -13,27 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array -const mi_page_t _mi_page_empty = { - 0, false, false, false, 0, - 0, // capacity - 0, // reserved capacity - { 0 }, // flags - false, // is_zero - 0, // retire_expire - NULL, // free - 0, // used - 0, // xblock_size - NULL, // local_free - #if (MI_PADDING || MI_ENCODE_FREELIST) - { 0, 0 }, - #endif - MI_ATOMIC_VAR_INIT(0), // xthread_free - MI_ATOMIC_VAR_INIT(0), // xheap - NULL, NULL - #if MI_INTPTR_SIZE==8 - , { 0 } // padding - #endif -}; +const mi_page_t _mi_page_empty; #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) @@ -122,6 +102,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { MI_BIN_FULL, 0, // page retired min/max NULL, // next false, + 0, 0 }; @@ -579,7 +560,7 @@ void mi_process_init(void) mi_attr_noexcept { _mi_verbose_message("secure level: %d\n", MI_SECURE); _mi_verbose_message("mem tracking: %s\n", MI_TRACK_TOOL); #if MI_TSAN - _mi_verbose_message("thread santizer enabled\n"); + _mi_verbose_message("thread sanitizer enabled\n"); #endif mi_thread_init(); diff --git a/Objects/mimalloc/options.c b/Objects/mimalloc/options.c index 345b560e3e7f4c..2a8f481d569e1e 100644 --- a/Objects/mimalloc/options.c +++ b/Objects/mimalloc/options.c @@ -269,7 +269,7 @@ static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop // (recursively) invoke malloc again to allocate space for the thread local // variables on demand. This is why we use a _mi_preloading test on such // platforms. However, C code generator may move the initial thread local address -// load before the `if` and we therefore split it out in a separate funcion. +// load before the `if` and we therefore split it out in a separate function. static mi_decl_thread bool recurse = false; static mi_decl_noinline bool mi_recurse_enter_prim(void) { diff --git a/Objects/mimalloc/os.c b/Objects/mimalloc/os.c index f3bc7184c41c5b..c9103168c12507 100644 --- a/Objects/mimalloc/os.c +++ b/Objects/mimalloc/os.c @@ -115,8 +115,12 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); - init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + mi_heap_t* heap = mi_prim_get_default_heap(); + // gh-123022: default heap may not be initialized in CPython in background threads + if (mi_heap_is_initialized(heap)) { + uintptr_t r = _mi_heap_random_next(heap); + init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + } #endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); @@ -553,8 +557,12 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { // Initialize the start address after the 32TiB area start = ((uintptr_t)32 << 40); // 32TiB virtual start address #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); - start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + mi_heap_t* heap = mi_prim_get_default_heap(); + // gh-123022: default heap may not be initialized in CPython in background threads + if (mi_heap_is_initialized(heap)) { + uintptr_t r = _mi_heap_random_next(heap); + start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + } #endif } end = start + size; diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c index 8f0ce920156e04..ff7444cce10923 100644 --- a/Objects/mimalloc/page.c +++ b/Objects/mimalloc/page.c @@ -225,6 +225,9 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // and the local free list if (page->local_free != NULL) { + // any previous QSBR goals are no longer valid because we reused the page + _PyMem_mi_page_clear_qsbr(page); + if mi_likely(page->free == NULL) { // usual case page->free = page->local_free; @@ -267,6 +270,7 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) { // TODO: push on full queue immediately if it is full? mi_page_queue_t* pq = mi_page_queue(heap, mi_page_block_size(page)); mi_page_queue_push(heap, pq, page); + _PyMem_mi_page_reclaimed(page); mi_assert_expensive(_mi_page_is_valid(page)); } @@ -383,6 +387,13 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { mi_heap_t* pheap = mi_page_heap(page); +#ifdef Py_GIL_DISABLED + if (page->qsbr_node.next != NULL) { + // remove from QSBR queue, but keep the goal + llist_remove(&page->qsbr_node); + } +#endif + // remove from our page list mi_segments_tld_t* segments_tld = &pheap->tld->segments; mi_page_queue_remove(pq, page); @@ -417,6 +428,11 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { mi_heap_t* heap = mi_page_heap(page); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); + mi_assert_internal(page->qsbr_node.next == NULL); +#endif + // remove from the page list // (no need to do _mi_heap_delayed_free first as all blocks are already free) mi_segments_tld_t* segments_tld = &heap->tld->segments; @@ -444,6 +460,9 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { mi_page_set_has_aligned(page, false); + // any previous QSBR goals are no longer valid because we reused the page + _PyMem_mi_page_clear_qsbr(page); + // don't retire too often.. // (or we end up retiring and re-allocating most of the time) // NOTE: refine this more: we should not retire if this @@ -462,10 +481,10 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if (index < heap->page_retired_min) heap->page_retired_min = index; if (index > heap->page_retired_max) heap->page_retired_max = index; mi_assert_internal(mi_page_all_free(page)); - return; // dont't free after all + return; // don't free after all } } - _mi_page_free(page, pq, false); + _PyMem_mi_page_maybe_free(page, pq, false); } // free retired pages: we don't need to look at the entire queues @@ -480,7 +499,10 @@ void _mi_heap_collect_retired(mi_heap_t* heap, bool force) { if (mi_page_all_free(page)) { page->retire_expire--; if (force || page->retire_expire == 0) { - _mi_page_free(pq->first, pq, force); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); +#endif + _PyMem_mi_page_maybe_free(page, pq, force); } else { // keep retired, update min/max @@ -661,6 +683,8 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi // set fields mi_page_set_heap(page, heap); page->tag = heap->tag; + page->use_qsbr = heap->page_use_qsbr; + page->debug_offset = heap->debug_offset; page->xblock_size = (block_size < MI_HUGE_BLOCK_SIZE ? (uint32_t)block_size : MI_HUGE_BLOCK_SIZE); // initialize before _mi_segment_page_start size_t page_size; const void* page_start = _mi_segment_page_start(segment, page, &page_size); @@ -690,6 +714,10 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_assert_internal(page->xthread_free == 0); mi_assert_internal(page->next == NULL); mi_assert_internal(page->prev == NULL); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); + mi_assert_internal(page->qsbr_node.next == NULL); +#endif mi_assert_internal(page->retire_expire == 0); mi_assert_internal(!mi_page_has_aligned(page)); #if (MI_PADDING || MI_ENCODE_FREELIST) @@ -749,6 +777,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p mi_heap_stat_counter_increase(heap, searches, count); if (page == NULL) { + _PyMem_mi_heap_collect_qsbr(heap); // some pages might be safe to free now _mi_heap_collect_retired(heap, false); // perhaps make a page available? page = mi_page_fresh(heap, pq); if (page == NULL && first_try) { @@ -759,6 +788,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p else { mi_assert(pq->first == page); page->retire_expire = 0; + _PyMem_mi_page_clear_qsbr(page); } mi_assert_internal(page == NULL || mi_page_immediate_available(page)); return page; @@ -784,6 +814,7 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) { if (mi_page_immediate_available(page)) { page->retire_expire = 0; + _PyMem_mi_page_clear_qsbr(page); return page; // fast path } } @@ -877,6 +908,7 @@ static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size, size_t huge_alignme return NULL; } else { + _PyMem_mi_heap_collect_qsbr(heap); return mi_large_huge_page_alloc(heap,size,huge_alignment); } } diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c index ec8447ab40d70c..1598ebabf0a4da 100644 --- a/Objects/mimalloc/prim/unix/prim.c +++ b/Objects/mimalloc/prim/unix/prim.c @@ -27,6 +27,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // mmap #include // sysconf +#include // open, close, read, access #if defined(__linux__) #include @@ -50,7 +51,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__FreeBSD__) && !defined(__sun) +#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__sun) && !defined(__NetBSD__) #define MI_HAS_SYSCALL_H #include #endif @@ -76,7 +77,7 @@ static int mi_prim_access(const char *fpath, int mode) { return syscall(SYS_access,fpath,mode); } -#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__FreeBSD__) && !defined(__sun) // avoid unused warnings +#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__sun) && !defined(__NetBSD__) // avoid unused warnings static int mi_prim_open(const char* fpath, int open_flags) { return open(fpath,open_flags); @@ -738,7 +739,7 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { #endif bool _mi_prim_random_buf(void* buf, size_t buf_len) { #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 - // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf + // We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf // may fail silently on macOS. See PR #390, and return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); #else diff --git a/Objects/mimalloc/prim/windows/etw.h b/Objects/mimalloc/prim/windows/etw.h index 4e0a092a10f4ba..e9ec35fc9ed7eb 100644 --- a/Objects/mimalloc/prim/windows/etw.h +++ b/Objects/mimalloc/prim/windows/etw.h @@ -136,7 +136,7 @@ extern "C" { // - MCGEN_EVENTSETINFORMATION // - MCGEN_EVENTWRITETRANSFER // -// If the the macro is undefined, the MC implementation will default to the +// If the macro is undefined, the MC implementation will default to the // corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is // undefined, the EventRegister[MyProviderName] macro will use EventRegister // in user mode and will use EtwRegister in kernel mode. diff --git a/Objects/mimalloc/segment-map.c b/Objects/mimalloc/segment-map.c index 3cd2127e56c1a7..5141f2e1b1dffa 100644 --- a/Objects/mimalloc/segment-map.c +++ b/Objects/mimalloc/segment-map.c @@ -84,7 +84,7 @@ static mi_segment_t* _mi_segment_of(const void* p) { // TODO: maintain max/min allocated range for efficiency for more efficient rejection of invalid pointers? // search downwards for the first segment in case it is an interior pointer - // could be slow but searches in MI_INTPTR_SIZE * MI_SEGMENT_SIZE (512MiB) steps trough + // could be slow but searches in MI_INTPTR_SIZE * MI_SEGMENT_SIZE (512MiB) steps through // valid huge objects // note: we could maintain a lowest index to speed up the path for invalid pointers? size_t lobitidx; diff --git a/Objects/mimalloc/segment.c b/Objects/mimalloc/segment.c index 584233b8b57bb4..9b092b9b734d4c 100644 --- a/Objects/mimalloc/segment.c +++ b/Objects/mimalloc/segment.c @@ -718,7 +718,7 @@ static mi_page_t* mi_segment_span_allocate(mi_segment_t* segment, size_t slice_i // set slice back pointers for the first MI_MAX_SLICE_OFFSET entries size_t extra = slice_count-1; if (extra > MI_MAX_SLICE_OFFSET) extra = MI_MAX_SLICE_OFFSET; - if (slice_index + extra >= segment->slice_entries) extra = segment->slice_entries - slice_index - 1; // huge objects may have more slices than avaiable entries in the segment->slices + if (slice_index + extra >= segment->slice_entries) extra = segment->slice_entries - slice_index - 1; // huge objects may have more slices than available entries in the segment->slices mi_slice_t* slice_next = slice + 1; for (size_t i = 1; i <= extra; i++, slice_next++) { @@ -814,6 +814,9 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment const size_t extra = align_offset - info_size; // recalculate due to potential guard pages *psegment_slices = mi_segment_calculate_slices(required + extra, ppre_size, pinfo_slices); + + // mi_page_t.slice_count type is uint32_t + if (*psegment_slices > (size_t)UINT32_MAX) return NULL; } const size_t segment_size = (*psegment_slices) * MI_SEGMENT_SLICE_SIZE; @@ -865,6 +868,9 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi size_t pre_size; size_t segment_slices = mi_segment_calculate_slices(required, &pre_size, &info_slices); + // mi_page_t.slice_count type is uint32_t + if (segment_slices > (size_t)UINT32_MAX) return NULL; + // Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little) const bool eager_delay = (// !_mi_os_has_overcommit() && // never delay on overcommit systems _mi_current_thread_count() > 1 && // do not delay for the first N threads @@ -982,6 +988,10 @@ static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld mi_assert_internal(mi_page_all_free(page)); mi_segment_t* segment = _mi_ptr_segment(page); mi_assert_internal(segment->used > 0); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); + mi_assert_internal(page->qsbr_node.next == NULL); +#endif size_t inuse = page->capacity * mi_page_block_size(page); _mi_stat_decrease(&tld->stats->page_committed, inuse); @@ -1270,10 +1280,13 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s // ensure used count is up to date and collect potential concurrent frees mi_page_t* const page = mi_slice_to_page(slice); _mi_page_free_collect(page, false); - if (mi_page_all_free(page)) { + if (mi_page_all_free(page) && _PyMem_mi_page_is_safe_to_free(page)) { // if this page is all free now, free it without adding to any queues (yet) mi_assert_internal(page->next == NULL && page->prev==NULL); _mi_stat_decrease(&tld->stats->pages_abandoned, 1); +#ifdef Py_GIL_DISABLED + page->qsbr_goal = 0; +#endif segment->abandoned--; slice = mi_segment_page_clear(page, tld); // re-assign slice due to coalesce! mi_assert_internal(!mi_slice_is_used(slice)); @@ -1344,15 +1357,18 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, mi_page_set_heap(page, target_heap); _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set) _mi_page_free_collect(page, false); // ensure used count is up to date - if (mi_page_all_free(page)) { + if (mi_page_all_free(page) && _PyMem_mi_page_is_safe_to_free(page)) { // if everything free by now, free the page +#ifdef Py_GIL_DISABLED + page->qsbr_goal = 0; +#endif slice = mi_segment_page_clear(page, tld); // set slice again due to coalesceing } else { // otherwise reclaim it into the heap _mi_page_reclaim(target_heap, page); if (requested_block_size == page->xblock_size && mi_page_has_any_available(page) && - heap == target_heap) { + requested_block_size <= MI_MEDIUM_OBJ_SIZE_MAX && heap == target_heap) { if (right_page_reclaimed != NULL) { *right_page_reclaimed = true; } } } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 3a1c516658dce7..535b0d068f064f 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -3,13 +3,21 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dict.h" // _PyDict_EnablePerThreadRefcounting() +#include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_interp.h" // PyInterpreterState.importlib +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "osdefs.h" // MAXPATHLEN + + +#define _PyModule_CAST(op) \ + (assert(PyModule_Check(op)), _Py_CAST(PyModuleObject*, (op))) static PyMemberDef module_members[] = { @@ -88,21 +96,31 @@ new_module_notrack(PyTypeObject *mt) m->md_weaklist = NULL; m->md_name = NULL; m->md_dict = PyDict_New(); - if (m->md_dict != NULL) { - return m; + if (m->md_dict == NULL) { + Py_DECREF(m); + return NULL; } - Py_DECREF(m); - return NULL; + return m; +} + +static void +track_module(PyModuleObject *m) +{ + _PyDict_EnablePerThreadRefcounting(m->md_dict); + PyObject_GC_Track(m->md_dict); + + _PyObject_SetDeferredRefcount((PyObject *)m); + PyObject_GC_Track(m); } static PyObject * new_module(PyTypeObject *mt, PyObject *args, PyObject *kws) { - PyObject *m = (PyObject *)new_module_notrack(mt); + PyModuleObject *m = new_module_notrack(mt); if (m != NULL) { - PyObject_GC_Track(m); + track_module(m); } - return m; + return (PyObject *)m; } PyObject * @@ -113,7 +131,7 @@ PyModule_NewObject(PyObject *name) return NULL; if (module_init_dict(m, m->md_dict, name, NULL) != 0) goto fail; - PyObject_GC_Track(m); + track_module(m); return (PyObject *)m; fail: @@ -171,6 +189,7 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) if (func == NULL) { return -1; } + _PyObject_SetDeferredRefcount(func); if (PyObject_SetAttrString(module, fdef->ml_name, func) != 0) { Py_DECREF(func); return -1; @@ -211,7 +230,9 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) return NULL; } name = _PyImport_ResolveNameWithPackageContext(name); - if ((m = (PyModuleObject*)PyModule_New(name)) == NULL) + + m = (PyModuleObject*)PyModule_New(name); + if (m == NULL) return NULL; if (module->m_size > 0) { @@ -237,6 +258,9 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) } } m->md_def = module; +#ifdef Py_GIL_DISABLED + m->md_gil = Py_MOD_GIL_USED; +#endif return (PyObject*)m; } @@ -249,6 +273,8 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio PyObject *m = NULL; int has_multiple_interpreters_slot = 0; void *multiple_interpreters = (void *)0; + int has_gil_slot = 0; + void *gil_slot = Py_MOD_GIL_USED; int has_execution_slots = 0; const char *name; int ret; @@ -303,6 +329,17 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio multiple_interpreters = cur_slot->value; has_multiple_interpreters_slot = 1; break; + case Py_mod_gil: + if (has_gil_slot) { + PyErr_Format( + PyExc_SystemError, + "module %s has more than one 'gil' slot", + name); + goto error; + } + gil_slot = cur_slot->value; + has_gil_slot = 1; + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -362,6 +399,11 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio if (PyModule_Check(m)) { ((PyModuleObject*)m)->md_state = NULL; ((PyModuleObject*)m)->md_def = def; +#ifdef Py_GIL_DISABLED + ((PyModuleObject*)m)->md_gil = gil_slot; +#else + (void)gil_slot; +#endif } else { if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) { PyErr_Format( @@ -403,6 +445,19 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio return NULL; } +#ifdef Py_GIL_DISABLED +int +PyUnstable_Module_SetGIL(PyObject *module, void *gil) +{ + if (!PyModule_Check(module)) { + PyErr_BadInternalCall(); + return -1; + } + ((PyModuleObject *)module)->md_gil = gil; + return 0; +} +#endif + int PyModule_ExecDef(PyObject *module, PyModuleDef *def) { @@ -458,6 +513,7 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) } break; case Py_mod_multiple_interpreters: + case Py_mod_gil: /* handled in PyModule_FromDefAndSpec2 */ break; default: @@ -705,35 +761,30 @@ static int module___init___impl(PyModuleObject *self, PyObject *name, PyObject *doc) /*[clinic end generated code: output=e7e721c26ce7aad7 input=57f9e177401e5e1e]*/ { - PyObject *dict = self->md_dict; - if (dict == NULL) { - dict = PyDict_New(); - if (dict == NULL) - return -1; - self->md_dict = dict; - } - if (module_init_dict(self, dict, name, doc) < 0) - return -1; - return 0; + return module_init_dict(self, self->md_dict, name, doc); } static void -module_dealloc(PyModuleObject *m) +module_dealloc(PyObject *self) { - int verbose = _Py_GetConfig()->verbose; + PyModuleObject *m = _PyModule_CAST(self); PyObject_GC_UnTrack(m); + + int verbose = _Py_GetConfig()->verbose; if (verbose && m->md_name) { PySys_FormatStderr("# destroy %U\n", m->md_name); } if (m->md_weaklist != NULL) PyObject_ClearWeakRefs((PyObject *) m); + /* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_free && (m->md_def->m_size <= 0 || m->md_state != NULL)) { m->md_def->m_free(m); } + Py_XDECREF(m->md_dict); Py_XDECREF(m->md_name); if (m->md_state != NULL) @@ -742,8 +793,9 @@ module_dealloc(PyModuleObject *m) } static PyObject * -module_repr(PyModuleObject *m) +module_repr(PyObject *self) { + PyModuleObject *m = _PyModule_CAST(self); PyInterpreterState *interp = _PyInterpreterState_GET(); return _PyImport_ImportlibModuleRepr(interp, (PyObject *)m); } @@ -784,11 +836,104 @@ _PyModuleSpec_IsUninitializedSubmodule(PyObject *spec, PyObject *name) return rc; } +int +_PyModuleSpec_GetFileOrigin(PyObject *spec, PyObject **p_origin) +{ + PyObject *has_location = NULL; + int rc = PyObject_GetOptionalAttr(spec, &_Py_ID(has_location), &has_location); + if (rc <= 0) { + return rc; + } + // If origin is not a location, or doesn't exist, or is not a str, we could consider falling + // back to module.__file__. But the cases in which module.__file__ is not __spec__.origin + // are cases in which we probably shouldn't be guessing. + rc = PyObject_IsTrue(has_location); + Py_DECREF(has_location); + if (rc <= 0) { + return rc; + } + // has_location is true, so origin is a location + PyObject *origin = NULL; + rc = PyObject_GetOptionalAttr(spec, &_Py_ID(origin), &origin); + if (rc <= 0) { + return rc; + } + assert(origin != NULL); + if (!PyUnicode_Check(origin)) { + Py_DECREF(origin); + return 0; + } + *p_origin = origin; + return 1; +} + +int +_PyModule_IsPossiblyShadowing(PyObject *origin) +{ + // origin must be a unicode subtype + // Returns 1 if the module at origin could be shadowing a module of the + // same name later in the module search path. The condition we check is basically: + // root = os.path.dirname(origin.removesuffix(os.sep + "__init__.py")) + // return not sys.flags.safe_path and root == (sys.path[0] or os.getcwd()) + // Returns 0 otherwise (or if we aren't sure) + // Returns -1 if an error occurred that should be propagated + if (origin == NULL) { + return 0; + } + + // not sys.flags.safe_path + const PyConfig *config = _Py_GetConfig(); + if (config->safe_path) { + return 0; + } + + // root = os.path.dirname(origin.removesuffix(os.sep + "__init__.py")) + wchar_t root[MAXPATHLEN + 1]; + Py_ssize_t size = PyUnicode_AsWideChar(origin, root, MAXPATHLEN); + if (size < 0) { + return -1; + } + assert(size <= MAXPATHLEN); + root[size] = L'\0'; + + wchar_t *sep = wcsrchr(root, SEP); + if (sep == NULL) { + return 0; + } + // If it's a package then we need to look one directory further up + if (wcscmp(sep + 1, L"__init__.py") == 0) { + *sep = L'\0'; + sep = wcsrchr(root, SEP); + if (sep == NULL) { + return 0; + } + } + *sep = L'\0'; + + // sys.path[0] or os.getcwd() + wchar_t *sys_path_0 = config->sys_path_0; + if (!sys_path_0) { + return 0; + } + + wchar_t sys_path_0_buf[MAXPATHLEN]; + if (sys_path_0[0] == L'\0') { + // if sys.path[0] == "", treat it as if it were the current directory + if (!_Py_wgetcwd(sys_path_0_buf, MAXPATHLEN)) { + return -1; + } + sys_path_0 = sys_path_0_buf; + } + + int result = wcscmp(sys_path_0, root) == 0; + return result; +} + PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { // When suppress=1, this function suppresses AttributeError. - PyObject *attr, *mod_name, *getattr, *origin; + PyObject *attr, *mod_name, *getattr; attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr) { return attr; @@ -819,81 +964,131 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) Py_DECREF(getattr); return result; } + + // The attribute was not found. We make a best effort attempt at a useful error message, + // but only if we're not suppressing AttributeError. + if (suppress == 1) { + return NULL; + } if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name) < 0) { return NULL; } - if (mod_name && PyUnicode_Check(mod_name)) { - PyObject *spec; - if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__spec__), &spec) < 0) { - Py_DECREF(mod_name); - return NULL; + if (!mod_name || !PyUnicode_Check(mod_name)) { + Py_XDECREF(mod_name); + PyErr_Format(PyExc_AttributeError, + "module has no attribute '%U'", name); + return NULL; + } + PyObject *spec; + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__spec__), &spec) < 0) { + Py_DECREF(mod_name); + return NULL; + } + if (spec == NULL) { + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U'", + mod_name, name); + Py_DECREF(mod_name); + return NULL; + } + + PyObject *origin = NULL; + if (_PyModuleSpec_GetFileOrigin(spec, &origin) < 0) { + goto done; + } + + int is_possibly_shadowing = _PyModule_IsPossiblyShadowing(origin); + if (is_possibly_shadowing < 0) { + goto done; + } + int is_possibly_shadowing_stdlib = 0; + if (is_possibly_shadowing) { + PyObject *stdlib_modules = PySys_GetObject("stdlib_module_names"); + if (stdlib_modules && PyAnySet_Check(stdlib_modules)) { + is_possibly_shadowing_stdlib = PySet_Contains(stdlib_modules, mod_name); + if (is_possibly_shadowing_stdlib < 0) { + goto done; + } + } + } + + if (is_possibly_shadowing_stdlib) { + assert(origin); + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U' " + "(consider renaming '%U' since it has the same " + "name as the standard library module named '%U' " + "and prevents importing that standard library module)", + mod_name, name, origin, mod_name); + } + else { + int rc = _PyModuleSpec_IsInitializing(spec); + if (rc < 0) { + goto done; } - if (suppress != 1) { - int rc = _PyModuleSpec_IsInitializing(spec); + else if (rc > 0) { + if (is_possibly_shadowing) { + assert(origin); + // For non-stdlib modules, only mention the possibility of + // shadowing if the module is being initialized. + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U' " + "(consider renaming '%U' if it has the same name " + "as a library you intended to import)", + mod_name, name, origin); + } + else if (origin) { + PyErr_Format(PyExc_AttributeError, + "partially initialized " + "module '%U' from '%U' has no attribute '%U' " + "(most likely due to a circular import)", + mod_name, origin, name); + } + else { + PyErr_Format(PyExc_AttributeError, + "partially initialized " + "module '%U' has no attribute '%U' " + "(most likely due to a circular import)", + mod_name, name); + } + } + else { + assert(rc == 0); + rc = _PyModuleSpec_IsUninitializedSubmodule(spec, name); if (rc > 0) { - int valid_spec = PyObject_GetOptionalAttr(spec, &_Py_ID(origin), &origin); - if (valid_spec == -1) { - Py_XDECREF(spec); - Py_DECREF(mod_name); - return NULL; - } - if (valid_spec == 1 && !PyUnicode_Check(origin)) { - valid_spec = 0; - Py_DECREF(origin); - } - if (valid_spec == 1) { - PyErr_Format(PyExc_AttributeError, - "partially initialized " - "module '%U' from '%U' has no attribute '%U' " - "(most likely due to a circular import)", - mod_name, origin, name); - Py_DECREF(origin); - } - else { - PyErr_Format(PyExc_AttributeError, - "partially initialized " - "module '%U' has no attribute '%U' " - "(most likely due to a circular import)", - mod_name, name); - } + PyErr_Format(PyExc_AttributeError, + "cannot access submodule '%U' of module '%U' " + "(most likely due to a circular import)", + name, mod_name); } else if (rc == 0) { - rc = _PyModuleSpec_IsUninitializedSubmodule(spec, name); - if (rc > 0) { - PyErr_Format(PyExc_AttributeError, - "cannot access submodule '%U' of module '%U' " - "(most likely due to a circular import)", - name, mod_name); - } - else if (rc == 0) { - PyErr_Format(PyExc_AttributeError, - "module '%U' has no attribute '%U'", - mod_name, name); - } + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U'", + mod_name, name); } } - Py_XDECREF(spec); - Py_DECREF(mod_name); - return NULL; - } - Py_XDECREF(mod_name); - if (suppress != 1) { - PyErr_Format(PyExc_AttributeError, - "module has no attribute '%U'", name); } + +done: + Py_XDECREF(origin); + Py_DECREF(spec); + Py_DECREF(mod_name); return NULL; } PyObject* -_Py_module_getattro(PyModuleObject *m, PyObject *name) +_Py_module_getattro(PyObject *self, PyObject *name) { + PyModuleObject *m = _PyModule_CAST(self); return _Py_module_getattro_impl(m, name, 0); } static int -module_traverse(PyModuleObject *m, visitproc visit, void *arg) +module_traverse(PyObject *self, visitproc visit, void *arg) { + PyModuleObject *m = _PyModule_CAST(self); + /* bpo-39824: Don't call m_traverse() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_traverse && (m->md_def->m_size <= 0 || m->md_state != NULL)) @@ -902,13 +1097,16 @@ module_traverse(PyModuleObject *m, visitproc visit, void *arg) if (res) return res; } + Py_VISIT(m->md_dict); return 0; } static int -module_clear(PyModuleObject *m) +module_clear(PyObject *self) { + PyModuleObject *m = _PyModule_CAST(self); + /* bpo-39824: Don't call m_clear() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_clear && (m->md_def->m_size <= 0 || m->md_state != NULL)) @@ -958,7 +1156,7 @@ static PyMethodDef module_methods[] = { }; static PyObject * -module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +module_get_dict(PyModuleObject *m) { PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); if (dict == NULL) { @@ -969,10 +1167,103 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) Py_DECREF(dict); return NULL; } + return dict; +} + +static PyObject * +module_get_annotate(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyModuleObject *m = _PyModule_CAST(self); + + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } + + PyObject *annotate; + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) == 0) { + annotate = Py_None; + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate) == -1) { + Py_CLEAR(annotate); + } + } + Py_DECREF(dict); + return annotate; +} + +static int +module_set_annotate(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) +{ + PyModuleObject *m = _PyModule_CAST(self); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } + + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return -1; + } + + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + Py_DECREF(dict); + return -1; + } + + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), value) == -1) { + Py_DECREF(dict); + return -1; + } + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + return -1; + } + } + Py_DECREF(dict); + return 0; +} + +static PyObject * +module_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyModuleObject *m = _PyModule_CAST(self); + + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } PyObject *annotations; if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) == 0) { - annotations = PyDict_New(); + PyObject *annotate; + int annotate_result = PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate); + if (annotate_result < 0) { + Py_DECREF(dict); + return NULL; + } + if (annotate_result == 1 && PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + Py_DECREF(annotate); + Py_DECREF(dict); + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotate); + Py_DECREF(annotations); + Py_DECREF(dict); + return NULL; + } + } + else { + annotations = PyDict_New(); + } + Py_XDECREF(annotate); if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); @@ -986,38 +1277,43 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) } static int -module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) +module_set_annotations(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { - int ret = -1; - PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); + PyModuleObject *m = _PyModule_CAST(self); + + PyObject *dict = module_get_dict(m); if (dict == NULL) { return -1; } - if (!PyDict_Check(dict)) { - PyErr_Format(PyExc_TypeError, ".__dict__ is not a dictionary"); - goto exit; - } + int ret = -1; if (value != NULL) { /* set */ ret = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); } else { /* delete */ - ret = PyDict_DelItem(dict, &_Py_ID(__annotations__)); - if (ret < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_SetString(PyExc_AttributeError, "__annotations__"); + ret = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + if (ret == 0) { + PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__annotations__)); + ret = -1; } + else if (ret > 0) { + ret = 0; + } + } + if (ret == 0 && PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + ret = -1; } -exit: Py_DECREF(dict); return ret; } static PyGetSetDef module_getsets[] = { - {"__annotations__", (getter)module_get_annotations, (setter)module_set_annotations}, + {"__annotations__", module_get_annotations, module_set_annotations}, + {"__annotate__", module_get_annotate, module_set_annotate}, {NULL} }; @@ -1026,26 +1322,26 @@ PyTypeObject PyModule_Type = { "module", /* tp_name */ sizeof(PyModuleObject), /* tp_basicsize */ 0, /* tp_itemsize */ - (destructor)module_dealloc, /* tp_dealloc */ + module_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)module_repr, /* tp_repr */ + module_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - (getattrofunc)_Py_module_getattro, /* tp_getattro */ + _Py_module_getattro, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ module___init____doc__, /* tp_doc */ - (traverseproc)module_traverse, /* tp_traverse */ - (inquiry)module_clear, /* tp_clear */ + module_traverse, /* tp_traverse */ + module_clear, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyModuleObject, md_weaklist), /* tp_weaklistoffset */ 0, /* tp_iter */ diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index b975bcfeea2cdf..5b7547103a2b3f 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -43,10 +43,28 @@ namespace_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int namespace_init(_PyNamespaceObject *ns, PyObject *args, PyObject *kwds) { - if (PyTuple_GET_SIZE(args) != 0) { - PyErr_Format(PyExc_TypeError, "no positional arguments expected"); + PyObject *arg = NULL; + if (!PyArg_UnpackTuple(args, _PyType_Name(Py_TYPE(ns)), 0, 1, &arg)) { return -1; } + if (arg != NULL) { + PyObject *dict; + if (PyDict_CheckExact(arg)) { + dict = Py_NewRef(arg); + } + else { + dict = PyObject_CallOneArg((PyObject *)&PyDict_Type, arg); + if (dict == NULL) { + return -1; + } + } + int err = (!PyArg_ValidateKeywordArguments(dict) || + PyDict_Update(ns->ns_dict, dict) < 0); + Py_DECREF(dict); + if (err) { + return -1; + } + } if (kwds == NULL) { return 0; } @@ -219,15 +237,17 @@ namespace_replace(PyObject *self, PyObject *args, PyObject *kwargs) static PyMethodDef namespace_methods[] = { {"__reduce__", (PyCFunction)namespace_reduce, METH_NOARGS, namespace_reduce__doc__}, - {"__replace__", _PyCFunction_CAST(namespace_replace), METH_VARARGS|METH_KEYWORDS, NULL}, + {"__replace__", _PyCFunction_CAST(namespace_replace), METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\n" + "Return a copy of the namespace object with new values for the specified attributes.")}, {NULL, NULL} // sentinel }; PyDoc_STRVAR(namespace_doc, -"A simple attribute-based namespace.\n\ -\n\ -SimpleNamespace(**kwargs)"); +"SimpleNamespace(mapping_or_iterable=(), /, **kwargs)\n\ +--\n\n\ +A simple attribute-based namespace."); PyTypeObject _PyNamespace_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) diff --git a/Objects/object.c b/Objects/object.c index 275aa6713c8c21..7cc74a8dc0d8eb 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -6,14 +6,19 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_context.h" // _PyContextTokenMissing_Type +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition +#include "pycore_object_state.h" // struct _reftracer_runtime_state +#include "pycore_long.h" // _PyLong_GetZero() #include "pycore_optimizer.h" // _PyUOpExecutor_Type, _PyUOpOptimizer_Type, ... #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() @@ -23,7 +28,6 @@ #include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_unionobject.h" // _PyUnion_Type -#include "interpreteridobject.h" // _PyInterpreterID_Type #ifdef Py_LIMITED_API // Prevent recursive call _Py_IncRef() <=> Py_INCREF() @@ -74,21 +78,16 @@ get_legacy_reftotal(void) interp->object_state.reftotal static inline void -reftotal_increment(PyInterpreterState *interp) +reftotal_add(PyThreadState *tstate, Py_ssize_t n) { - REFTOTAL(interp)++; -} - -static inline void -reftotal_decrement(PyInterpreterState *interp) -{ - REFTOTAL(interp)--; -} - -static inline void -reftotal_add(PyInterpreterState *interp, Py_ssize_t n) -{ - REFTOTAL(interp) += n; +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + // relaxed store to avoid data race with read in get_reftotal() + Py_ssize_t reftotal = tstate_impl->reftotal + n; + _Py_atomic_store_ssize_relaxed(&tstate_impl->reftotal, reftotal); +#else + REFTOTAL(tstate->interp) += n; +#endif } static inline Py_ssize_t get_global_reftotal(_PyRuntimeState *); @@ -118,7 +117,15 @@ get_reftotal(PyInterpreterState *interp) { /* For a single interpreter, we ignore the legacy _Py_RefTotal, since we can't determine which interpreter updated it. */ - return REFTOTAL(interp); + Py_ssize_t total = REFTOTAL(interp); +#ifdef Py_GIL_DISABLED + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + /* This may race with other threads modifications to their reftotal */ + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)p; + total += _Py_atomic_load_ssize_relaxed(&tstate_impl->reftotal); + } +#endif + return total; } static inline Py_ssize_t @@ -130,7 +137,7 @@ get_global_reftotal(_PyRuntimeState *runtime) HEAD_LOCK(&_PyRuntime); PyInterpreterState *interp = PyInterpreterState_Head(); for (; interp != NULL; interp = PyInterpreterState_Next(interp)) { - total += REFTOTAL(interp); + total += get_reftotal(interp); } HEAD_UNLOCK(&_PyRuntime); @@ -164,6 +171,48 @@ _PyDebug_PrintTotalRefs(void) { #define REFCHAIN(interp) interp->object_state.refchain #define REFCHAIN_VALUE ((void*)(uintptr_t)1) +static inline int +has_own_refchain(PyInterpreterState *interp) +{ + if (interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC) { + return (_Py_IsMainInterpreter(interp) + || _PyInterpreterState_Main() == NULL); + } + return 1; +} + +static int +refchain_init(PyInterpreterState *interp) +{ + if (!has_own_refchain(interp)) { + // Legacy subinterpreters share a refchain with the main interpreter. + REFCHAIN(interp) = REFCHAIN(_PyInterpreterState_Main()); + return 0; + } + _Py_hashtable_allocator_t alloc = { + // Don't use default PyMem_Malloc() and PyMem_Free() which + // require the caller to hold the GIL. + .malloc = PyMem_RawMalloc, + .free = PyMem_RawFree, + }; + REFCHAIN(interp) = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct, + NULL, NULL, &alloc); + if (REFCHAIN(interp) == NULL) { + return -1; + } + return 0; +} + +static void +refchain_fini(PyInterpreterState *interp) +{ + if (has_own_refchain(interp) && REFCHAIN(interp) != NULL) { + _Py_hashtable_destroy(REFCHAIN(interp)); + } + REFCHAIN(interp) = NULL; +} + bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj) { @@ -223,32 +272,32 @@ _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op) void _Py_INCREF_IncRefTotal(void) { - reftotal_increment(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), 1); } /* This is used strictly by Py_DECREF(). */ void _Py_DECREF_DecRefTotal(void) { - reftotal_decrement(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), -1); } void -_Py_IncRefTotal(PyInterpreterState *interp) +_Py_IncRefTotal(PyThreadState *tstate) { - reftotal_increment(interp); + reftotal_add(tstate, 1); } void -_Py_DecRefTotal(PyInterpreterState *interp) +_Py_DecRefTotal(PyThreadState *tstate) { - reftotal_decrement(interp); + reftotal_add(tstate, -1); } void -_Py_AddRefTotal(PyInterpreterState *interp, Py_ssize_t n) +_Py_AddRefTotal(PyThreadState *tstate, Py_ssize_t n) { - reftotal_add(interp, n); + reftotal_add(tstate, n); } /* This includes the legacy total @@ -268,7 +317,10 @@ _Py_GetLegacyRefTotal(void) Py_ssize_t _PyInterpreterState_GetRefTotal(PyInterpreterState *interp) { - return get_reftotal(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t total = get_reftotal(interp); + HEAD_UNLOCK(&_PyRuntime); + return total; } #endif /* Py_REF_DEBUG */ @@ -299,13 +351,13 @@ _Py_DecRef(PyObject *o) #ifdef Py_GIL_DISABLED # ifdef Py_REF_DEBUG -static inline int -is_shared_refcnt_dead(Py_ssize_t shared) +static int +is_dead(PyObject *o) { # if SIZEOF_SIZE_T == 8 - return shared == (Py_ssize_t)0xDDDDDDDDDDDDDDDD; + return (uintptr_t)o->ob_type == 0xDDDDDDDDDDDDDDDD; # else - return shared == (Py_ssize_t)0xDDDDDDDD; + return (uintptr_t)o->ob_type == 0xDDDDDDDD; # endif } # endif @@ -335,8 +387,8 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) } #ifdef Py_REF_DEBUG - if ((_Py_REF_IS_MERGED(new_shared) && new_shared < 0) || - is_shared_refcnt_dead(shared)) + if ((new_shared < 0 && _Py_REF_IS_MERGED(new_shared)) || + (should_queue && is_dead(o))) { _Py_NegativeRefcount(filename, lineno, o); } @@ -346,7 +398,7 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) if (should_queue) { #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif _Py_brc_queue_object(o); } @@ -367,14 +419,18 @@ _Py_MergeZeroLocalRefcount(PyObject *op) { assert(op->ob_ref_local == 0); - _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); - Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); if (shared == 0) { // Fast-path: shared refcount is zero (including flags) _Py_Dealloc(op); return; } + // gh-121794: This must be before the store to `ob_ref_shared` (gh-119999), + // but should outside the fast-path to maintain the invariant that + // a zero `ob_tid` implies a merged refcount. + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + // Slow-path: atomically set the flags (low two bits) to _Py_REF_MERGED. Py_ssize_t new_shared; do { @@ -393,24 +449,27 @@ Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) { assert(!_Py_IsImmortal(op)); + +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyThreadState_GET(), extra); +#endif + + // gh-119999: Write to ob_ref_local and ob_tid before merging the refcount. + Py_ssize_t local = (Py_ssize_t)op->ob_ref_local; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + Py_ssize_t refcnt; Py_ssize_t new_shared; Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); do { refcnt = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); - refcnt += (Py_ssize_t)op->ob_ref_local; + refcnt += local; refcnt += extra; new_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, &shared, new_shared)); - -#ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyInterpreterState_GET(), extra); -#endif - - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); - _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); return refcnt; } #endif /* Py_GIL_DISABLED */ @@ -520,6 +579,7 @@ int PyObject_Print(PyObject *op, FILE *fp, int flags) { int ret = 0; + int write_error = 0; if (PyErr_CheckSignals()) return -1; #ifdef USE_STACKCHECK @@ -558,14 +618,20 @@ PyObject_Print(PyObject *op, FILE *fp, int flags) ret = -1; } else { - fwrite(t, 1, len, fp); + /* Versions of Android and OpenBSD from before 2023 fail to + set the `ferror` indicator when writing to a read-only + stream, so we need to check the return value. + (https://github.com/openbsd/src/commit/fc99cf9338942ecd9adc94ea08bf6188f0428c15) */ + if (fwrite(t, 1, len, fp) != (size_t)len) { + write_error = 1; + } } Py_DECREF(s); } } } if (ret == 0) { - if (ferror(fp)) { + if (write_error || ferror(fp)) { PyErr_SetFromErrno(PyExc_OSError); clearerr(fp); ret = -1; @@ -793,20 +859,53 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } +static void +clear_freelist(struct _Py_freelist *freelist, int is_finalization, + freefunc dofree) +{ + void *ptr; + while ((ptr = _PyFreeList_PopNoStats(freelist)) != NULL) { + dofree(ptr); + } + assert(freelist->size == 0 || freelist->size == -1); + assert(freelist->freelist == NULL); + if (is_finalization) { + freelist->size = -1; + } +} + +static void +free_object(void *obj) +{ + PyObject *op = (PyObject *)obj; + PyTypeObject *tp = Py_TYPE(op); + tp->tp_free(op); + Py_DECREF(tp); +} + void -_PyObject_ClearFreeLists(_PyFreeListState *state, int is_finalization) +_PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) { // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() - _PyFloat_ClearFreeList(state, is_finalization); - _PyTuple_ClearFreeList(state, is_finalization); - _PyList_ClearFreeList(state, is_finalization); - _PyDict_ClearFreeList(state, is_finalization); - _PyContext_ClearFreeList(state, is_finalization); - _PyAsyncGen_ClearFreeLists(state, is_finalization); - // Only be cleared if is_finalization is true. - _PyObjectStackChunk_ClearFreeList(state, is_finalization); - _PySlice_ClearFreeList(state, is_finalization); + clear_freelist(&freelists->floats, is_finalization, free_object); + for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { + clear_freelist(&freelists->tuples[i], is_finalization, free_object); + } + clear_freelist(&freelists->lists, is_finalization, free_object); + clear_freelist(&freelists->dicts, is_finalization, free_object); + clear_freelist(&freelists->dictkeys, is_finalization, PyMem_Free); + clear_freelist(&freelists->slices, is_finalization, free_object); + clear_freelist(&freelists->contexts, is_finalization, free_object); + clear_freelist(&freelists->async_gens, is_finalization, free_object); + clear_freelist(&freelists->async_gen_asends, is_finalization, free_object); + clear_freelist(&freelists->futureiters, is_finalization, free_object); + if (is_finalization) { + // Only clear object stack chunks during finalization. We use object + // stacks during GC, so emptying the free-list is counterproductive. + clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree); + } + clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); } /* @@ -1114,18 +1213,7 @@ _PyObject_GetAttrId(PyObject *v, _Py_Identifier *name) } int -_PyObject_SetAttrId(PyObject *v, _Py_Identifier *name, PyObject *w) -{ - int result; - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) - return -1; - result = PyObject_SetAttr(v, oname, w); - return result; -} - -static inline int -set_attribute_error_context(PyObject* v, PyObject* name) +_PyObject_SetAttributeErrorContext(PyObject* v, PyObject* name) { assert(PyErr_Occurred()); if (!PyErr_ExceptionMatches(PyExc_AttributeError)){ @@ -1180,7 +1268,7 @@ PyObject_GetAttr(PyObject *v, PyObject *name) } if (result == NULL) { - set_attribute_error_context(v, name); + _PyObject_SetAttributeErrorContext(v, name); } return result; } @@ -1209,9 +1297,9 @@ PyObject_GetOptionalAttr(PyObject *v, PyObject *name, PyObject **result) return 0; } if (tp->tp_getattro == _Py_type_getattro) { - int supress_missing_attribute_exception = 0; - *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &supress_missing_attribute_exception); - if (supress_missing_attribute_exception) { + int suppress_missing_attribute_exception = 0; + *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &suppress_missing_attribute_exception); + if (suppress_missing_attribute_exception) { // return 0 without having to clear the exception return 0; } @@ -1315,7 +1403,8 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) } Py_INCREF(name); - PyUnicode_InternInPlace(&name); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &name); if (tp->tp_setattro != NULL) { err = (*tp->tp_setattro)(v, name, value); Py_DECREF(name); @@ -1397,16 +1486,15 @@ _PyObject_GetDictPtr(PyObject *obj) if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return _PyObject_ComputedDictPointer(obj); } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, _PyDictOrValues_GetValues(*dorv_ptr)); + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + dict = _PyObject_MaterializeManagedDict(obj); if (dict == NULL) { PyErr_Clear(); return NULL; } - dorv_ptr->dict = dict; } - return &dorv_ptr->dict; + return (PyObject **)&_PyObject_ManagedDictPointer(obj)->dict; } PyObject * @@ -1459,13 +1547,13 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } - PyObject *descr = _PyType_Lookup(tp, name); + PyObject *descr = _PyType_LookupRef(tp, name); descrgetfunc f = NULL; if (descr != NULL) { - Py_INCREF(descr); if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { meth_found = 1; - } else { + } + else { f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { *method = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1474,22 +1562,18 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } } - PyObject *dict; - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); - if (attr != NULL) { - *method = attr; - Py_XDECREF(descr); - return 0; - } - dict = NULL; - } - else { - dict = dorv_ptr->dict; + PyObject *dict, *attr; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(obj, name, &attr)) { + if (attr != NULL) { + *method = attr; + Py_XDECREF(descr); + return 0; } + dict = NULL; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -1532,7 +1616,7 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); return 0; } @@ -1566,11 +1650,10 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, goto done; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); f = NULL; if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { res = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1582,29 +1665,24 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } } if (dict == NULL) { - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - if (PyUnicode_CheckExact(name)) { - res = _PyObject_GetInstanceAttribute(obj, values, name); - if (res != NULL) { - goto done; - } - } - else { - dict = _PyObject_MakeDictFromInstanceAttributes(obj, values); - if (dict == NULL) { - res = NULL; - goto done; - } - dorv_ptr->dict = dict; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + if (PyUnicode_CheckExact(name) && + _PyObject_TryGetInstanceAttribute(obj, name, &res)) { + if (res != NULL) { + goto done; } } else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); + dict = (PyObject *)_PyObject_MaterializeManagedDict(obj); + if (dict == NULL) { + res = NULL; + goto done; + } } } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); + } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); if (dictptr) { @@ -1649,7 +1727,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1672,6 +1750,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, descrsetfunc f; int res = -1; + assert(!PyType_IsSubtype(tp, &PyType_Type)); if (!PyUnicode_Check(name)){ PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", @@ -1685,10 +1764,9 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, Py_INCREF(name); Py_INCREF(tp); - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_set; if (f != NULL) { res = f(descr, obj, value); @@ -1698,22 +1776,15 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, if (dict == NULL) { PyObject **dictptr; + + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + res = _PyObject_StoreInstanceAttribute(obj, name, value); + goto error_check; + } + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } - dictptr = &dorv_ptr->dict; - if (*dictptr == NULL) { - if (_PyObject_InitInlineValues(obj, tp) < 0) { - goto done; - } - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + dictptr = (PyObject **)&managed_dict->dict; } else { dictptr = _PyObject_ComputedDictPointer(obj); @@ -1731,7 +1802,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); } - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } else { PyErr_Format(PyExc_AttributeError, @@ -1741,7 +1812,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, goto done; } else { - res = _PyObjectDict_SetItem(tp, dictptr, name, value); + res = _PyObjectDict_SetItem(tp, obj, dictptr, name, value); } } else { @@ -1754,17 +1825,10 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } error_check: if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - if (PyType_IsSubtype(tp, &PyType_Type)) { - PyErr_Format(PyExc_AttributeError, - "type object '%.50s' has no attribute '%U'", - ((PyTypeObject*)obj)->tp_name, name); - } - else { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); - } - set_attribute_error_context(obj, name); + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1784,9 +1848,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { - if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj))) - { + if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && + _PyObject_GetManagedDict(obj) == NULL + ) { /* Was unable to convert to dict */ PyErr_NoMemory(); } @@ -1806,7 +1870,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } + Py_BEGIN_CRITICAL_SECTION(obj); Py_XSETREF(*dictptr, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -2014,6 +2080,11 @@ static PyNumberMethods none_as_number = { 0, /* nb_index */ }; +PyDoc_STRVAR(none_doc, +"NoneType()\n" +"--\n\n" +"The type of the None singleton."); + PyTypeObject _PyNone_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "NoneType", @@ -2035,7 +2106,7 @@ PyTypeObject _PyNone_Type = { 0, /*tp_setattro */ 0, /*tp_as_buffer */ Py_TPFLAGS_DEFAULT, /*tp_flags */ - 0, /*tp_doc */ + none_doc, /*tp_doc */ 0, /*tp_traverse */ 0, /*tp_clear */ _Py_BaseObject_RichCompare, /*tp_richcompare */ @@ -2100,19 +2171,20 @@ notimplemented_dealloc(PyObject *notimplemented) static int notimplemented_bool(PyObject *v) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "NotImplemented should not be used in a boolean context", - 1) < 0) - { - return -1; - } - return 1; + PyErr_SetString(PyExc_TypeError, + "NotImplemented should not be used in a boolean context"); + return -1; } static PyNumberMethods notimplemented_as_number = { .nb_bool = notimplemented_bool, }; +PyDoc_STRVAR(notimplemented_doc, +"NotImplementedType()\n" +"--\n\n" +"The type of the NotImplemented singleton."); + PyTypeObject _PyNotImplemented_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "NotImplementedType", @@ -2134,7 +2206,7 @@ PyTypeObject _PyNotImplemented_Type = { 0, /*tp_setattro */ 0, /*tp_as_buffer */ Py_TPFLAGS_DEFAULT, /*tp_flags */ - 0, /*tp_doc */ + notimplemented_doc, /*tp_doc */ 0, /*tp_traverse */ 0, /*tp_clear */ 0, /*tp_richcompare */ @@ -2161,16 +2233,7 @@ PyStatus _PyObject_InitState(PyInterpreterState *interp) { #ifdef Py_TRACE_REFS - _Py_hashtable_allocator_t alloc = { - // Don't use default PyMem_Malloc() and PyMem_Free() which - // require the caller to hold the GIL. - .malloc = PyMem_RawMalloc, - .free = PyMem_RawFree, - }; - REFCHAIN(interp) = _Py_hashtable_new_full( - _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct, - NULL, NULL, &alloc); - if (REFCHAIN(interp) == NULL) { + if (refchain_init(interp) < 0) { return _PyStatus_NO_MEMORY(); } #endif @@ -2181,8 +2244,7 @@ void _PyObject_FiniState(PyInterpreterState *interp) { #ifdef Py_TRACE_REFS - _Py_hashtable_destroy(REFCHAIN(interp)); - REFCHAIN(interp) = NULL; + refchain_fini(interp); #endif } @@ -2234,12 +2296,12 @@ static PyTypeObject* static_types[] = { &PyFilter_Type, &PyFloat_Type, &PyFrame_Type, + &PyFrameLocalsProxy_Type, &PyFrozenSet_Type, &PyFunction_Type, &PyGen_Type, &PyGetSetDescr_Type, &PyInstanceMethod_Type, - &PyInterpreterID_Type, &PyListIter_Type, &PyListRevIter_Type, &PyList_Type, @@ -2281,9 +2343,11 @@ static PyTypeObject* static_types[] = { &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, +#ifdef _Py_TIER2 &_PyCounterExecutor_Type, &_PyCounterOptimizer_Type, &_PyDefaultOptimizer_Type, +#endif &_Py_GenericAliasIterType, &_PyHamtItems_Type, &_PyHamtKeys_Type, @@ -2292,6 +2356,7 @@ static PyTypeObject* static_types[] = { &_PyHamt_BitmapNode_Type, &_PyHamt_CollisionNode_Type, &_PyHamt_Type, + &_PyInstructionSequence_Type, &_PyLegacyEventHandler_Type, &_PyLineIterator, &_PyManagedBuffer_Type, @@ -2303,12 +2368,15 @@ static PyTypeObject* static_types[] = { &_PyPositionsIterator, &_PyUnicodeASCIIIter_Type, &_PyUnion_Type, +#ifdef _Py_TIER2 &_PyUOpExecutor_Type, &_PyUOpOptimizer_Type, +#endif &_PyWeakref_CallableProxyType, &_PyWeakref_ProxyType, &_PyWeakref_RefType, &_PyTypeAlias_Type, + &_PyNoDefault_Type, // subclasses: _PyTypes_FiniTypes() deallocates them before their base // class @@ -2337,6 +2405,14 @@ _PyTypes_InitTypes(PyInterpreterState *interp) } } + // Cache __reduce__ from PyBaseObject_Type object + PyObject *baseobj_dict = _PyType_GetDict(&PyBaseObject_Type); + PyObject *baseobj_reduce = PyDict_GetItemWithError(baseobj_dict, &_Py_ID(__reduce__)); + if (baseobj_reduce == NULL && PyErr_Occurred()) { + return _PyStatus_ERR("Can't get __reduce__ from base object"); + } + _Py_INTERP_CACHED_OBJECT(interp, objreduce) = baseobj_reduce; + // Must be after static types are initialized if (_Py_initialize_generic(interp) < 0) { return _PyStatus_ERR("Can't initialize generic types"); @@ -2359,7 +2435,7 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(interp, type); + _PyStaticType_FiniBuiltin(interp, type); } } @@ -2367,16 +2443,13 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) static inline void new_reference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 #if !defined(Py_GIL_DISABLED) op->ob_refcnt = 1; #else op->ob_tid = _Py_ThreadId(); op->_padding = 0; - op->ob_mutex = (struct _PyMutex){ 0 }; + op->ob_mutex = (PyMutex){ 0 }; op->ob_gc_bits = 0; op->ob_ref_local = 1; op->ob_ref_shared = 0; @@ -2384,13 +2457,18 @@ new_reference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_CREATE, data); + } } void _Py_NewReference(PyObject *op) { #ifdef Py_REF_DEBUG - reftotal_increment(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif new_reference(op); } @@ -2402,14 +2480,55 @@ _Py_NewReferenceNoTotal(PyObject *op) } void -_Py_ResurrectReference(PyObject *op) +_Py_SetImmortalUntracked(PyObject *op) +{ +#ifdef Py_DEBUG + // For strings, use _PyUnicode_InternImmortal instead. + if (PyUnicode_CheckExact(op)) { + assert(PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL + || PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC); + } +#endif +#ifdef Py_GIL_DISABLED + op->ob_tid = _Py_UNOWNED_TID; + op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + op->ob_ref_shared = 0; +#else + op->ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT; +#endif +} + +void +_Py_SetImmortal(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); + if (PyObject_IS_GC(op) && _PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); } + _Py_SetImmortalUntracked(op); +} + +void +_PyObject_SetDeferredRefcount(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + assert(PyType_IS_GC(Py_TYPE(op))); + assert(_Py_IsOwnedByCurrentThread(op)); + assert(op->ob_ref_shared == 0); + _PyObject_SET_GC_BITS(op, _PyGC_BITS_DEFERRED); + op->ob_ref_shared = _Py_REF_SHARED(_Py_REF_DEFERRED, 0); +#endif +} + +void +_Py_ResurrectReference(PyObject *op) +{ #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + if (_PyRuntime.ref_tracer.tracer_func != NULL) { + void* data = _PyRuntime.ref_tracer.tracer_data; + _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_CREATE, data); + } } @@ -2673,33 +2792,30 @@ Py_ReprLeave(PyObject *obj) /* Trashcan support. */ -#define _PyTrash_UNWIND_LEVEL 50 - /* Add op to the gcstate->trash_delete_later list. Called when the current * call-stack depth gets large. op must be a currently untracked gc'ed * object, with refcount 0. Py_DECREF must already have been called on it. */ -static void -_PyTrash_thread_deposit_object(struct _py_trashcan *trash, PyObject *op) +void +_PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op) { _PyObject_ASSERT(op, _PyObject_IS_GC(op)); _PyObject_ASSERT(op, !_PyObject_GC_IS_TRACKED(op)); _PyObject_ASSERT(op, Py_REFCNT(op) == 0); #ifdef Py_GIL_DISABLED - _PyObject_ASSERT(op, op->ob_tid == 0); - op->ob_tid = (uintptr_t)trash->delete_later; + op->ob_tid = (uintptr_t)tstate->delete_later; #else - _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)trash->delete_later); + _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)tstate->delete_later); #endif - trash->delete_later = op; + tstate->delete_later = op; } /* Deallocate all the objects in the gcstate->trash_delete_later list. * Called when the call-stack unwinds again. */ -static void -_PyTrash_thread_destroy_chain(struct _py_trashcan *trash) +void +_PyTrash_thread_destroy_chain(PyThreadState *tstate) { - /* We need to increase trash_delete_nesting here, otherwise, + /* We need to increase c_recursion_remaining here, otherwise, _PyTrash_thread_destroy_chain will be called recursively and then possibly crash. An example that may crash without increase: @@ -2710,17 +2826,18 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) tups = [(tup,) for tup in tups] del tups */ - assert(trash->delete_nesting == 0); - ++trash->delete_nesting; - while (trash->delete_later) { - PyObject *op = trash->delete_later; + assert(tstate->c_recursion_remaining > Py_TRASHCAN_HEADROOM); + tstate->c_recursion_remaining--; + while (tstate->delete_later) { + PyObject *op = tstate->delete_later; destructor dealloc = Py_TYPE(op)->tp_dealloc; #ifdef Py_GIL_DISABLED - trash->delete_later = (PyObject*) op->ob_tid; + tstate->delete_later = (PyObject*) op->ob_tid; op->ob_tid = 0; + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, _Py_REF_MERGED); #else - trash->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); + tstate->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); #endif /* Call the deallocator directly. This used to try to @@ -2731,92 +2848,10 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) */ _PyObject_ASSERT(op, Py_REFCNT(op) == 0); (*dealloc)(op); - assert(trash->delete_nesting == 1); - } - --trash->delete_nesting; -} - - -static struct _py_trashcan * -_PyTrash_get_state(PyThreadState *tstate) -{ - if (tstate != NULL) { - return &tstate->trash; - } - // The current thread must be finalizing. - // Fall back to using thread-local state. - // XXX Use thread-local variable syntax? - assert(PyThread_tss_is_created(&_PyRuntime.trashTSSkey)); - struct _py_trashcan *trash = - (struct _py_trashcan *)PyThread_tss_get(&_PyRuntime.trashTSSkey); - if (trash == NULL) { - trash = PyMem_RawMalloc(sizeof(struct _py_trashcan)); - if (trash == NULL) { - Py_FatalError("Out of memory"); - } - PyThread_tss_set(&_PyRuntime.trashTSSkey, (void *)trash); - } - return trash; -} - -static void -_PyTrash_clear_state(PyThreadState *tstate) -{ - if (tstate != NULL) { - assert(tstate->trash.delete_later == NULL); - return; - } - if (PyThread_tss_is_created(&_PyRuntime.trashTSSkey)) { - struct _py_trashcan *trash = - (struct _py_trashcan *)PyThread_tss_get(&_PyRuntime.trashTSSkey); - if (trash != NULL) { - PyThread_tss_set(&_PyRuntime.trashTSSkey, (void *)NULL); - PyMem_RawFree(trash); - } - } -} - - -int -_PyTrash_begin(PyThreadState *tstate, PyObject *op) -{ - // XXX Make sure the GIL is held. - struct _py_trashcan *trash = _PyTrash_get_state(tstate); - if (trash->delete_nesting >= _PyTrash_UNWIND_LEVEL) { - /* Store the object (to be deallocated later) and jump past - * Py_TRASHCAN_END, skipping the body of the deallocator */ - _PyTrash_thread_deposit_object(trash, op); - return 1; - } - ++trash->delete_nesting; - return 0; -} - - -void -_PyTrash_end(PyThreadState *tstate) -{ - // XXX Make sure the GIL is held. - struct _py_trashcan *trash = _PyTrash_get_state(tstate); - --trash->delete_nesting; - if (trash->delete_nesting <= 0) { - if (trash->delete_later != NULL) { - _PyTrash_thread_destroy_chain(trash); - } - _PyTrash_clear_state(tstate); } + tstate->c_recursion_remaining++; } - -/* bpo-40170: It's only be used in Py_TRASHCAN_BEGIN macro to hide - implementation details. */ -int -_PyTrash_cond(PyObject *op, destructor dealloc) -{ - return Py_TYPE(op)->tp_dealloc == dealloc; -} - - void _Py_NO_RETURN _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, const char *file, int line, const char *function) @@ -2883,6 +2918,12 @@ _Py_Dealloc(PyObject *op) Py_INCREF(type); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_DESTROY, data); + } + #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif @@ -2970,3 +3011,87 @@ _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) { Py_SET_REFCNT(ob, refcnt); } + +int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { + assert(PyGILState_Check()); + _PyRuntime.ref_tracer.tracer_func = tracer; + _PyRuntime.ref_tracer.tracer_data = data; + return 0; +} + +PyRefTracer PyRefTracer_GetTracer(void** data) { + assert(PyGILState_Check()); + if (data != NULL) { + *data = _PyRuntime.ref_tracer.tracer_data; + } + return _PyRuntime.ref_tracer.tracer_func; +} + + + +static PyObject* constants[] = { + &_Py_NoneStruct, // Py_CONSTANT_NONE + (PyObject*)(&_Py_FalseStruct), // Py_CONSTANT_FALSE + (PyObject*)(&_Py_TrueStruct), // Py_CONSTANT_TRUE + &_Py_EllipsisObject, // Py_CONSTANT_ELLIPSIS + &_Py_NotImplementedStruct, // Py_CONSTANT_NOT_IMPLEMENTED + NULL, // Py_CONSTANT_ZERO + NULL, // Py_CONSTANT_ONE + NULL, // Py_CONSTANT_EMPTY_STR + NULL, // Py_CONSTANT_EMPTY_BYTES + NULL, // Py_CONSTANT_EMPTY_TUPLE +}; + +void +_Py_GetConstant_Init(void) +{ + constants[Py_CONSTANT_ZERO] = _PyLong_GetZero(); + constants[Py_CONSTANT_ONE] = _PyLong_GetOne(); + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_New(0, 0); + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize(NULL, 0); + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); +#ifndef NDEBUG + for (size_t i=0; i < Py_ARRAY_LENGTH(constants); i++) { + assert(constants[i] != NULL); + assert(_Py_IsImmortal(constants[i])); + } +#endif +} + +PyObject* +Py_GetConstant(unsigned int constant_id) +{ + if (constant_id < Py_ARRAY_LENGTH(constants)) { + return constants[constant_id]; + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + + +PyObject* +Py_GetConstantBorrowed(unsigned int constant_id) +{ + // All constants are immortal + return Py_GetConstant(constant_id); +} + + +// Py_TYPE() implementation for the stable ABI +#undef Py_TYPE +PyTypeObject* +Py_TYPE(PyObject *ob) +{ + return _Py_TYPE(ob); +} + + +// Py_REFCNT() implementation for the stable ABI +#undef Py_REFCNT +Py_ssize_t +Py_REFCNT(PyObject *ob) +{ + return _Py_REFCNT(ob); +} diff --git a/Objects/object_layout.md b/Objects/object_layout.md index 4f379bed8d77e2..4a781668636324 100644 --- a/Objects/object_layout.md +++ b/Objects/object_layout.md @@ -16,7 +16,49 @@ Since the introduction of the cycle GC, there has also been a pre-header. Before 3.11, this pre-header was two words in size. It should be considered opaque to all code except the cycle GC. -## 3.11 pre-header +### 3.13 + +In 3.13, the values array is embedded into the object, so there is no +need for a values pointer (it is just a fixed offset into the object). +So the pre-header is these two fields: + +* weakreflist +* dict_pointer + +If the object has no physical dictionary, then the ``dict_pointer`` +is set to `NULL`. + +In 3.13 only objects with no additional data could have inline values. +That is, instances of classes with `tp_basicsize == sizeof(PyObject)`. +In 3.14, any object whose class has `tp_itemsize == 0` can have inline values. +In both versions, the inline values starts `tp_basicsize` bytes after the object. + +
    + 3.12 + +### 3.12 + +In 3.12, the pointer to the list of weak references is added to the +pre-header. In order to make space for it, the ``dict`` and ``values`` +pointers are combined into a single tagged pointer: + +* weakreflist +* dict_or_values + +If the object has no physical dictionary, then the ``dict_or_values`` +has its low bit set to one, and points to the values array. +If the object has a physical dictionary, then the ``dict_or_values`` +has its low bit set to zero, and points to the dictionary. + +The untagged form is chosen for the dictionary pointer, rather than +the values pointer, to enable the (legacy) C-API function +`_PyObject_GetDictPtr(PyObject *obj)` to work. +
    + +
    + 3.11 + +### 3.11 In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``. The reason for moving the ``__dict__`` to the pre-header is that it allows @@ -33,27 +75,49 @@ The values pointer refers to the ``PyDictValues`` array which holds the values of the objects's attributes. Should the dictionary be needed, then ``values`` is set to ``NULL`` and the ``dict`` field points to the dictionary. +
    -## 3.12 pre-header +## Layout of a "normal" Python object -In 3.12, the pointer to the list of weak references is added to the -pre-header. In order to make space for it, the ``dict`` and ``values`` -pointers are combined into a single tagged pointer: +A "normal" Python object is one that doesn't inherit from a builtin +class, doesn't have slots. + +### 3.13 + +In 3.13 the values are embedded into the object, as follows: * weakreflist * dict_or_values +* GC 1 +* GC 2 +* ob_refcnt +* ob_type +* Inlined values: + * Flags + * values 0 + * values 1 + * ... + * Insertion order bytes -If the object has no physical dictionary, then the ``dict_or_values`` -has its low bit set to one, and points to the values array. -If the object has a physical dictionary, then the ``dict_or_values`` -has its low bit set to zero, and points to the dictionary. +This has all the advantages of the layout used in 3.12, plus: +* Access to values is even faster as there is one less load +* Fast access is mostly maintained when the `__dict__` is materialized -The untagged form is chosen for the dictionary pointer, rather than -the values pointer, to enable the (legacy) C-API function -`_PyObject_GetDictPtr(PyObject *obj)` to work. +![Layout of "normal" object in 3.13](./object_layout_313.png) + +For objects with opaque parts defined by a C extension, +the layout is much the same as for 3.12 +![Layout of "full" object in 3.13](./object_layout_full_313.png) -## Layout of a "normal" Python object in 3.12: + +
    + 3.12 + +### 3.12: + +In 3.12, the header and pre-header form the entire object for "normal" +Python objects: * weakreflist * dict_or_values @@ -62,9 +126,6 @@ the values pointer, to enable the (legacy) C-API function * ob_refcnt * ob_type -For a "normal" Python object, one that doesn't inherit from a builtin -class or have slots, the header and pre-header form the entire object. - ![Layout of "normal" object in 3.12](./object_layout_312.png) There are several advantages to this layout: @@ -79,4 +140,6 @@ The full layout object, with an opaque part defined by a C extension, and `__slots__` looks like this: ![Layout of "full" object in 3.12](./object_layout_full_312.png) +
    + diff --git a/Objects/object_layout_312.gv b/Objects/object_layout_312.gv index c0068d78568524..731a25332b3ace 100644 --- a/Objects/object_layout_312.gv +++ b/Objects/object_layout_312.gv @@ -20,6 +20,7 @@ digraph ideal { shape = none label = < + diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png index 396dab183b3e9b..a63d095ea0b19e 100644 Binary files a/Objects/object_layout_312.png and b/Objects/object_layout_312.png differ diff --git a/Objects/object_layout_313.gv b/Objects/object_layout_313.gv new file mode 100644 index 00000000000000..d85352876f273b --- /dev/null +++ b/Objects/object_layout_313.gv @@ -0,0 +1,45 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = <
    values
    Insertion order
    values[0]
    values[1]
    ...
    + + + + + + + + + + + + +
    object
    weakrefs
    dict pointer
    GC info 0
    GC info 1
    refcount
    __class__
    values flags
    values[0]
    values[1]
    ...
    Insertion order
    > + + ] + + class [ + shape = none + label = < + + + + + +
    class
    ...
    dict_offset
    ...
    cached_keys
    > + ] + + keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"] + NULL [ label = " NULL"; shape="plain"] + object:w -> NULL + object:h -> class:head + object:dv -> NULL + class:k -> keys + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_313.png b/Objects/object_layout_313.png new file mode 100644 index 00000000000000..f7059c286c84e6 Binary files /dev/null and b/Objects/object_layout_313.png differ diff --git a/Objects/object_layout_full_313.gv b/Objects/object_layout_full_313.gv new file mode 100644 index 00000000000000..551312d51a7012 --- /dev/null +++ b/Objects/object_layout_full_313.gv @@ -0,0 +1,25 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = < + + + + + + + + + + + +
    object
    weakrefs
    dict pointer
    GC info 0
    GC info 1
    refcount
    __class__
    opaque (extension) data
    ...
    __slot__ 0
    ...
    > + ] + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_full_313.png b/Objects/object_layout_full_313.png new file mode 100644 index 00000000000000..7352ec69e5d8db Binary files /dev/null and b/Objects/object_layout_full_313.png differ diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 6a12c3dca38b36..dfeccfa4dd7658 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -12,6 +12,12 @@ #include // malloc() #include #ifdef WITH_MIMALLOC +// Forward declarations of functions used in our mimalloc modifications +static void _PyMem_mi_page_clear_qsbr(mi_page_t *page); +static bool _PyMem_mi_page_is_safe_to_free(mi_page_t *page); +static bool _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force); +static void _PyMem_mi_page_reclaimed(mi_page_t *page); +static void _PyMem_mi_heap_collect_qsbr(mi_heap_t *heap); # include "pycore_mimalloc.h" # include "mimalloc/static.c" # include "mimalloc/internal.h" // for stats @@ -86,6 +92,113 @@ _PyMem_RawFree(void *Py_UNUSED(ctx), void *ptr) #ifdef WITH_MIMALLOC +static void +_PyMem_mi_page_clear_qsbr(mi_page_t *page) +{ +#ifdef Py_GIL_DISABLED + // Clear the QSBR goal and remove the page from the QSBR linked list. + page->qsbr_goal = 0; + if (page->qsbr_node.next != NULL) { + llist_remove(&page->qsbr_node); + } +#endif +} + +// Check if an empty, newly reclaimed page is safe to free now. +static bool +_PyMem_mi_page_is_safe_to_free(mi_page_t *page) +{ + assert(mi_page_all_free(page)); +#ifdef Py_GIL_DISABLED + assert(page->qsbr_node.next == NULL); + if (page->use_qsbr && page->qsbr_goal != 0) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (tstate == NULL) { + return false; + } + return _Py_qbsr_goal_reached(tstate->qsbr, page->qsbr_goal); + } +#endif + return true; + +} + +static bool +_PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) +{ +#ifdef Py_GIL_DISABLED + assert(mi_page_all_free(page)); + if (page->use_qsbr) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + if (page->qsbr_goal != 0 && _Py_qbsr_goal_reached(tstate->qsbr, page->qsbr_goal)) { + _PyMem_mi_page_clear_qsbr(page); + _mi_page_free(page, pq, force); + return true; + } + + _PyMem_mi_page_clear_qsbr(page); + page->retire_expire = 0; + page->qsbr_goal = _Py_qsbr_deferred_advance(tstate->qsbr); + llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); + return false; + } +#endif + _mi_page_free(page, pq, force); + return true; +} + +static void +_PyMem_mi_page_reclaimed(mi_page_t *page) +{ +#ifdef Py_GIL_DISABLED + assert(page->qsbr_node.next == NULL); + if (page->qsbr_goal != 0) { + if (mi_page_all_free(page)) { + assert(page->qsbr_node.next == NULL); + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + page->retire_expire = 0; + llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); + } + else { + page->qsbr_goal = 0; + } + } +#endif +} + +static void +_PyMem_mi_heap_collect_qsbr(mi_heap_t *heap) +{ +#ifdef Py_GIL_DISABLED + if (!heap->page_use_qsbr) { + return; + } + + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct llist_node *head = &tstate->mimalloc.page_list; + if (llist_empty(head)) { + return; + } + + struct llist_node *node; + llist_for_each_safe(node, head) { + mi_page_t *page = llist_data(node, mi_page_t, qsbr_node); + if (!mi_page_all_free(page)) { + // We allocated from this page some point after the delayed free + _PyMem_mi_page_clear_qsbr(page); + continue; + } + + if (!_Py_qsbr_poll(tstate->qsbr, page->qsbr_goal)) { + return; + } + + _PyMem_mi_page_clear_qsbr(page); + _mi_page_free(page, mi_page_queue_of(page), false); + } +#endif +} + void * _PyMem_MiMalloc(void *ctx, size_t size) { @@ -273,8 +386,16 @@ _PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr, ) { #ifdef MS_WINDOWS + /* Unlike free(), VirtualFree() does not special-case NULL to noop. */ + if (ptr == NULL) { + return; + } VirtualFree(ptr, 0, MEM_RELEASE); #elif defined(ARENAS_USE_MMAP) + /* Unlike free(), munmap() does not special-case NULL to noop. */ + if (ptr == NULL) { + return; + } munmap(ptr, size); #else free(ptr); @@ -948,6 +1069,224 @@ _PyMem_Strdup(const char *str) return copy; } +/***********************************************/ +/* Delayed freeing support for Py_GIL_DISABLED */ +/***********************************************/ + +// So that sizeof(struct _mem_work_chunk) is 4096 bytes on 64-bit platforms. +#define WORK_ITEMS_PER_CHUNK 254 + +// A pointer to be freed once the QSBR read sequence reaches qsbr_goal. +struct _mem_work_item { + uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free + uint64_t qsbr_goal; +}; + +// A fixed-size buffer of pointers to be freed +struct _mem_work_chunk { + // Linked list node of chunks in queue + struct llist_node node; + + Py_ssize_t rd_idx; // index of next item to read + Py_ssize_t wr_idx; // index of next item to write + struct _mem_work_item array[WORK_ITEMS_PER_CHUNK]; +}; + +static void +free_work_item(uintptr_t ptr) +{ + if (ptr & 0x01) { + PyObject_Free((char *)(ptr - 1)); + } + else { + PyMem_Free((void *)ptr); + } +} + +static void +free_delayed(uintptr_t ptr) +{ +#ifndef Py_GIL_DISABLED + free_work_item(ptr); +#else + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyInterpreterState_GetFinalizing(interp) != NULL || + interp->stoptheworld.world_stopped) + { + // Free immediately during interpreter shutdown or if the world is + // stopped. + free_work_item(ptr); + return; + } + + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct llist_node *head = &tstate->mem_free_queue; + + struct _mem_work_chunk *buf = NULL; + if (!llist_empty(head)) { + // Try to re-use the last buffer + buf = llist_data(head->prev, struct _mem_work_chunk, node); + if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + // already full + buf = NULL; + } + } + + if (buf == NULL) { + buf = PyMem_Calloc(1, sizeof(*buf)); + if (buf != NULL) { + llist_insert_tail(head, &buf->node); + } + } + + if (buf == NULL) { + // failed to allocate a buffer, free immediately + _PyEval_StopTheWorld(tstate->base.interp); + free_work_item(ptr); + _PyEval_StartTheWorld(tstate->base.interp); + return; + } + + assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK); + uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr); + buf->array[buf->wr_idx].ptr = ptr; + buf->array[buf->wr_idx].qsbr_goal = seq; + buf->wr_idx++; + + if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + _PyMem_ProcessDelayed((PyThreadState *)tstate); + } +#endif +} + +void +_PyMem_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed((uintptr_t)ptr); +} + +void +_PyObject_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed(((uintptr_t)ptr)|0x01); +} + +static struct _mem_work_chunk * +work_queue_first(struct llist_node *head) +{ + return llist_data(head->next, struct _mem_work_chunk, node); +} + +static void +process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, + bool keep_empty) +{ + while (!llist_empty(head)) { + struct _mem_work_chunk *buf = work_queue_first(head); + + while (buf->rd_idx < buf->wr_idx) { + struct _mem_work_item *item = &buf->array[buf->rd_idx]; + if (!_Py_qsbr_poll(qsbr, item->qsbr_goal)) { + return; + } + + free_work_item(item->ptr); + buf->rd_idx++; + } + + assert(buf->rd_idx == buf->wr_idx); + if (keep_empty && buf->node.next == head) { + // Keep the last buffer in the queue to reduce re-allocations + buf->rd_idx = buf->wr_idx = 0; + return; + } + + llist_remove(&buf->node); + PyMem_Free(buf); + } +} + +static void +process_interp_queue(struct _Py_mem_interp_free_queue *queue, + struct _qsbr_thread_state *qsbr) +{ + if (!_Py_atomic_load_int_relaxed(&queue->has_work)) { + return; + } + + // Try to acquire the lock, but don't block if it's already held. + if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) { + process_queue(&queue->head, qsbr, false); + + int more_work = !llist_empty(&queue->head); + _Py_atomic_store_int_relaxed(&queue->has_work, more_work); + + PyMutex_Unlock(&queue->mutex); + } +} + +void +_PyMem_ProcessDelayed(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + + // Process thread-local work + process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true); + + // Process shared interpreter work + process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr); +} + +void +_PyMem_AbandonDelayed(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + struct llist_node *queue = &((_PyThreadStateImpl *)tstate)->mem_free_queue; + + if (llist_empty(queue)) { + return; + } + + // Check if the queue contains one empty buffer + struct _mem_work_chunk *buf = work_queue_first(queue); + if (buf->rd_idx == buf->wr_idx) { + llist_remove(&buf->node); + PyMem_Free(buf); + assert(llist_empty(queue)); + return; + } + + // Merge the thread's work queue into the interpreter's work queue. + PyMutex_Lock(&interp->mem_free_queue.mutex); + llist_concat(&interp->mem_free_queue.head, queue); + _Py_atomic_store_int_relaxed(&interp->mem_free_queue.has_work, 1); + PyMutex_Unlock(&interp->mem_free_queue.mutex); + + assert(llist_empty(queue)); // the thread's queue is now empty +} + +void +_PyMem_FiniDelayed(PyInterpreterState *interp) +{ + struct llist_node *head = &interp->mem_free_queue.head; + while (!llist_empty(head)) { + struct _mem_work_chunk *buf = work_queue_first(head); + + while (buf->rd_idx < buf->wr_idx) { + // Free the remaining items immediately. There should be no other + // threads accessing the memory at this point during shutdown. + struct _mem_work_item *item = &buf->array[buf->rd_idx]; + free_work_item(item->ptr); + buf->rd_idx++; + } + + llist_remove(&buf->node); + PyMem_Free(buf); + } +} /**************************/ /* the "object" allocator */ @@ -1018,7 +1357,7 @@ static int running_on_valgrind = -1; typedef struct _obmalloc_state OMState; /* obmalloc state for main interpreter and shared by all interpreters without - * their own obmalloc state. By not explicitly initalizing this structure, it + * their own obmalloc state. By not explicitly initializing this structure, it * will be allocated in the BSS which is a small performance win. The radix * tree arrays are fairly large but are sparsely used. */ static struct _obmalloc_state obmalloc_state_main; @@ -1138,6 +1477,8 @@ _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp) { #ifdef WITH_MIMALLOC if (_PyMem_MimallocEnabled()) { + Py_ssize_t leaked = _PyInterpreterState_GetAllocatedBlocks(interp); + interp->runtime->obmalloc.interpreter_leaks += leaked; return; } #endif @@ -2269,6 +2610,33 @@ write_size_t(void *p, size_t n) } } +static void +fill_mem_debug(debug_alloc_api_t *api, void *data, int c, size_t nbytes, + bool is_alloc) +{ +#ifdef Py_GIL_DISABLED + if (api->api_id == 'o') { + // Don't overwrite the first few bytes of a PyObject allocation in the + // free-threaded build + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + size_t debug_offset; + if (is_alloc) { + debug_offset = tstate->mimalloc.current_object_heap->debug_offset; + } + else { + char *alloc = (char *)data - 2*SST; // start of the allocation + debug_offset = _mi_ptr_page(alloc)->debug_offset; + } + debug_offset -= 2*SST; // account for pymalloc extra bytes + if (debug_offset < nbytes) { + memset((char *)data + debug_offset, c, nbytes - debug_offset); + } + return; + } +#endif + memset(data, c, nbytes); +} + /* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: @@ -2345,7 +2713,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); if (nbytes > 0 && !use_calloc) { - memset(data, PYMEM_CLEANBYTE, nbytes); + fill_mem_debug(api, data, PYMEM_CLEANBYTE, nbytes, true); } /* at tail, write pad (SST bytes) and serialno (SST bytes) */ @@ -2393,8 +2761,9 @@ _PyMem_DebugRawFree(void *ctx, void *p) _PyMem_DebugCheckAddress(__func__, api->api_id, p); nbytes = read_size_t(q); - nbytes += PYMEM_DEBUG_EXTRA_BYTES; - memset(q, PYMEM_DEADBYTE, nbytes); + nbytes += PYMEM_DEBUG_EXTRA_BYTES - 2*SST; + memset(q, PYMEM_DEADBYTE, 2*SST); + fill_mem_debug(api, p, PYMEM_DEADBYTE, nbytes, false); api->alloc.free(api->alloc.ctx, q); } @@ -2414,7 +2783,6 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) size_t total; /* 2 * SST + nbytes + 2 * SST */ size_t original_nbytes; #define ERASED_SIZE 64 - uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ _PyMem_DebugCheckAddress(__func__, api->api_id, p); @@ -2431,9 +2799,11 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) #ifdef PYMEM_DEBUG_SERIALNO size_t block_serialno = read_size_t(tail + SST); #endif +#ifndef Py_GIL_DISABLED /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. */ + uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(save, data, original_nbytes); memset(data - 2 * SST, PYMEM_DEADBYTE, @@ -2446,6 +2816,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) memset(tail - ERASED_SIZE, PYMEM_DEADBYTE, ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST); } +#endif /* Resize and add decorations. */ r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total); @@ -2473,6 +2844,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) write_size_t(tail + SST, block_serialno); #endif +#ifndef Py_GIL_DISABLED /* Restore saved bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(data, save, Py_MIN(nbytes, original_nbytes)); @@ -2485,6 +2857,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) Py_MIN(nbytes - i, ERASED_SIZE)); } } +#endif if (r == NULL) { return NULL; diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 421bc52992d735..e151023dd764bf 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -200,7 +200,7 @@ Here are some ways to address this challenge: Adding the checks to the concrete API would help make any interpreter switch to OrderedDict less painful for extension modules. However, this won't work. The equivalent C API call to `dict.__setitem__(obj, k, v)` -is 'PyDict_SetItem(obj, k, v)`. This illustrates how subclasses in C call +is `PyDict_SetItem(obj, k, v)`. This illustrates how subclasses in C call the base class's methods, since there is no equivalent of super() in the C API. Calling into Python for parent class API would work, but some extension modules already rely on this feature of the concrete API. @@ -276,7 +276,7 @@ tp_dictoffset (__dict__) - - tp_init __init__ object_init dict_init tp_alloc - PyType_GenericAlloc (repeated) tp_new __new__ object_new dict_new -tp_free - PyObject_Del PyObject_GC_Del +tp_free - PyObject_Free PyObject_GC_Del ================= ================ =================== ================ Relevant Methods @@ -535,8 +535,12 @@ _odict_get_index_raw(PyODictObject *od, PyObject *key, Py_hash_t hash) PyObject *value = NULL; PyDictKeysObject *keys = ((PyDictObject *)od)->ma_keys; Py_ssize_t ix; - +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe((PyDictObject *)od, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup((PyDictObject *)od, key, hash, &value); +#endif if (ix == DKIX_EMPTY) { return keys->dk_nentries; /* index of new entry */ } @@ -792,6 +796,7 @@ _odict_clear_nodes(PyODictObject *od) _odictnode_DEALLOC(node); node = next; } + od->od_state++; } /* There isn't any memory management of nodes past this point. */ @@ -802,24 +807,40 @@ _odict_keys_equal(PyODictObject *a, PyODictObject *b) { _ODictNode *node_a, *node_b; + // keep operands' state to detect undesired mutations + const size_t state_a = a->od_state; + const size_t state_b = b->od_state; + node_a = _odict_FIRST(a); node_b = _odict_FIRST(b); while (1) { - if (node_a == NULL && node_b == NULL) + if (node_a == NULL && node_b == NULL) { /* success: hit the end of each at the same time */ return 1; - else if (node_a == NULL || node_b == NULL) + } + else if (node_a == NULL || node_b == NULL) { /* unequal length */ return 0; + } else { - int res = PyObject_RichCompareBool( - (PyObject *)_odictnode_KEY(node_a), - (PyObject *)_odictnode_KEY(node_b), - Py_EQ); - if (res < 0) + PyObject *key_a = Py_NewRef(_odictnode_KEY(node_a)); + PyObject *key_b = Py_NewRef(_odictnode_KEY(node_b)); + int res = PyObject_RichCompareBool(key_a, key_b, Py_EQ); + Py_DECREF(key_a); + Py_DECREF(key_b); + if (res < 0) { return res; - else if (res == 0) + } + else if (a->od_state != state_a || b->od_state != state_b) { + PyErr_SetString(PyExc_RuntimeError, + "OrderedDict mutated during iteration"); + return -1; + } + else if (res == 0) { + // This check comes after the check on the state + // in order for the exception to be set correctly. return 0; + } /* otherwise it must match, so move on to the next one */ node_a = _odictnode_NEXT(node_a); diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index ce9eef69ad75a8..2942ab624edf72 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -83,7 +83,7 @@ range_from_array(PyTypeObject *type, PyObject *const *args, Py_ssize_t num_args) switch (num_args) { case 3: step = args[2]; - /* fallthrough */ + _Py_FALLTHROUGH; case 2: /* Convert borrowed refs to owned refs */ start = PyNumber_Index(args[0]); @@ -143,14 +143,14 @@ range_new(PyTypeObject *type, PyObject *args, PyObject *kw) static PyObject * -range_vectorcall(PyTypeObject *type, PyObject *const *args, +range_vectorcall(PyObject *rangetype, PyObject *const *args, size_t nargsf, PyObject *kwnames) { Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (!_PyArg_NoKwnames("range", kwnames)) { return NULL; } - return range_from_array(type, args, nargs); + return range_from_array((PyTypeObject *)rangetype, args, nargs); } PyDoc_STRVAR(range_doc, @@ -655,7 +655,7 @@ range_index(rangeobject *r, PyObject *ob) } /* object is not in the range */ - PyErr_Format(PyExc_ValueError, "%R is not in range", ob); + PyErr_SetString(PyExc_ValueError, "range.index(x): x not in range"); return NULL; } @@ -751,7 +751,7 @@ PyDoc_STRVAR(index_doc, static PyMethodDef range_methods[] = { {"__reversed__", range_reverse, METH_NOARGS, reverse_doc}, - {"__reduce__", (PyCFunction)range_reduce, METH_VARARGS}, + {"__reduce__", (PyCFunction)range_reduce, METH_NOARGS}, {"count", (PyCFunction)range_count, METH_O, count_doc}, {"index", (PyCFunction)range_index, METH_O, index_doc}, {NULL, NULL} /* sentinel */ @@ -803,7 +803,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_init */ 0, /* tp_alloc */ range_new, /* tp_new */ - .tp_vectorcall = (vectorcallfunc)range_vectorcall + .tp_vectorcall = range_vectorcall }; /*********************** range Iterator **************************/ @@ -899,7 +899,7 @@ PyTypeObject PyRangeIter_Type = { sizeof(_PyRangeIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)PyObject_Del, /* tp_dealloc */ + (destructor)PyObject_Free, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ diff --git a/Objects/setobject.c b/Objects/setobject.c index 6a4c8c45f0836d..66d7fc730c555c 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2,7 +2,7 @@ /* set object implementation Written and maintained by Raymond D. Hettinger - Derived from Lib/sets.py and Objects/dictobject.c. + Derived from Objects/dictobject.c. The basic lookup function used by all operations. This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. @@ -37,8 +37,11 @@ #include "pycore_dict.h" // _PyDict_Contains_KnownHash() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED() #include "pycore_pyerrors.h" // _PyErr_SetKeyError() #include "pycore_setobject.h" // _PySet_NextEntry() definition + +#include "stringlib/eq.h" // unicode_eq() #include // offsetof() #include "clinic/setobject.c.h" @@ -95,7 +98,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) return entry; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && _PyUnicode_EQ(startkey, key)) + && unicode_eq(startkey, key)) return entry; table = so->table; Py_INCREF(startkey); @@ -130,6 +133,8 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) int probes; int cmp; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + /* Pre-increment is necessary to prevent arbitrary code in the rich comparison from deallocating the key just before the insertion. */ Py_INCREF(key); @@ -154,7 +159,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) goto found_active; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && _PyUnicode_EQ(startkey, key)) + && unicode_eq(startkey, key)) goto found_active; table = so->table; Py_INCREF(startkey); @@ -181,14 +186,14 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) found_unused_or_dummy: if (freeslot == NULL) goto found_unused; - so->used++; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used + 1); freeslot->key = key; freeslot->hash = hash; return 0; found_unused: so->fill++; - so->used++; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used + 1); entry->key = key; entry->hash = hash; if ((size_t)so->fill*5 < mask*3) @@ -354,7 +359,7 @@ set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) old_key = entry->key; entry->key = dummy; entry->hash = -1; - so->used--; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used - 1); Py_DECREF(old_key); return DISCARD_FOUND; } @@ -362,13 +367,9 @@ set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) static int set_add_key(PySetObject *so, PyObject *key) { - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return set_add_entry(so, key, hash); } @@ -376,13 +377,9 @@ set_add_key(PySetObject *so, PyObject *key) static int set_contains_key(PySetObject *so, PyObject *key) { - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return set_contains_entry(so, key, hash); } @@ -390,13 +387,9 @@ set_contains_key(PySetObject *so, PyObject *key) static int set_discard_key(PySetObject *so, PyObject *key) { - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return set_discard_entry(so, key, hash); } @@ -406,15 +399,16 @@ set_empty_to_minsize(PySetObject *so) { memset(so->smalltable, 0, sizeof(so->smalltable)); so->fill = 0; - so->used = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, 0); so->mask = PySet_MINSIZE - 1; so->table = so->smalltable; so->hash = -1; } static int -set_clear_internal(PySetObject *so) +set_clear_internal(PyObject *self) { + PySetObject *so = _PySet_CAST(self); setentry *entry; setentry *table = so->table; Py_ssize_t fill = so->fill; @@ -499,8 +493,9 @@ set_next(PySetObject *so, Py_ssize_t *pos_ptr, setentry **entry_ptr) } static void -set_dealloc(PySetObject *so) +set_dealloc(PyObject *self) { + PySetObject *so = _PySet_CAST(self); setentry *entry; Py_ssize_t used = so->used; @@ -523,7 +518,7 @@ set_dealloc(PySetObject *so) } static PyObject * -set_repr(PySetObject *so) +set_repr_lock_held(PySetObject *so) { PyObject *result=NULL, *keys, *listrepr, *tmp; int status = Py_ReprEnter((PyObject*)so); @@ -567,14 +562,26 @@ set_repr(PySetObject *so) return result; } +static PyObject * +set_repr(PyObject *self) +{ + PySetObject *so = _PySet_CAST(self); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(so); + result = set_repr_lock_held(so); + Py_END_CRITICAL_SECTION(); + return result; +} + static Py_ssize_t -set_len(PyObject *so) +set_len(PyObject *self) { - return ((PySetObject *)so)->used; + PySetObject *so = _PySet_CAST(self); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(so->used); } static int -set_merge(PySetObject *so, PyObject *otherset) +set_merge_lock_held(PySetObject *so, PyObject *otherset) { PySetObject *other; PyObject *key; @@ -583,9 +590,10 @@ set_merge(PySetObject *so, PyObject *otherset) setentry *other_entry; assert (PyAnySet_Check(so)); - assert (PyAnySet_Check(otherset)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(otherset); - other = (PySetObject*)otherset; + other = _PySet_CAST(otherset); if (other == so || other->used == 0) /* a.update(a) or a.update(set()); nothing to do */ return 0; @@ -612,7 +620,7 @@ set_merge(PySetObject *so, PyObject *otherset) } } so->fill = other->fill; - so->used = other->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, other->used); return 0; } @@ -621,7 +629,7 @@ set_merge(PySetObject *so, PyObject *otherset) setentry *newtable = so->table; size_t newmask = (size_t)so->mask; so->fill = other->used; - so->used = other->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, other->used); for (i = other->mask + 1; i > 0 ; i--, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { @@ -645,6 +653,7 @@ set_merge(PySetObject *so, PyObject *otherset) } /*[clinic input] +@critical_section set.pop so: setobject @@ -655,7 +664,7 @@ Raises KeyError if the set is empty. static PyObject * set_pop_impl(PySetObject *so) -/*[clinic end generated code: output=4d65180f1271871b input=4a3f5552e660a260]*/ +/*[clinic end generated code: output=4d65180f1271871b input=9296c84921125060]*/ { /* Make sure the search finger is in bounds */ setentry *entry = so->table + (so->finger & so->mask); @@ -674,14 +683,15 @@ set_pop_impl(PySetObject *so) key = entry->key; entry->key = dummy; entry->hash = -1; - so->used--; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used - 1); so->finger = entry - so->table + 1; /* next place to start */ return key; } static int -set_traverse(PySetObject *so, visitproc visit, void *arg) +set_traverse(PyObject *self, visitproc visit, void *arg) { + PySetObject *so = _PySet_CAST(self); Py_ssize_t pos = 0; setentry *entry; @@ -705,18 +715,19 @@ _shuffle_bits(Py_uhash_t h) large primes with "interesting bit patterns" and that passed tests for good collision statistics on a variety of problematic datasets including powersets and graph structures (such as David Eppstein's - graph recipes in Lib/test/test_set.py) */ + graph recipes in Lib/test/test_set.py). + + This hash algorithm can be used on either a frozenset or a set. + When it is used on a set, it computes the hash value of the equivalent + frozenset without creating a new frozenset object. */ static Py_hash_t -frozenset_hash(PyObject *self) +frozenset_hash_impl(PyObject *self) { - PySetObject *so = (PySetObject *)self; + PySetObject *so = _PySet_CAST(self); Py_uhash_t hash = 0; setentry *entry; - if (so->hash != -1) - return so->hash; - /* Xor-in shuffled bits from every entry's hash field because xor is commutative and a frozenset hash should be independent of order. @@ -749,6 +760,20 @@ frozenset_hash(PyObject *self) if (hash == (Py_uhash_t)-1) hash = 590923713UL; + return (Py_hash_t)hash; +} + +static Py_hash_t +frozenset_hash(PyObject *self) +{ + PySetObject *so = _PySet_CAST(self); + Py_uhash_t hash; + + if (so->hash != -1) { + return so->hash; + } + + hash = frozenset_hash_impl(self); so->hash = hash; return hash; } @@ -764,8 +789,9 @@ typedef struct { } setiterobject; static void -setiter_dealloc(setiterobject *si) +setiter_dealloc(PyObject *self) { + setiterobject *si = (setiterobject*)self; /* bpo-31095: UnTrack is needed before calling any callbacks */ _PyObject_GC_UNTRACK(si); Py_XDECREF(si->si_set); @@ -773,8 +799,9 @@ setiter_dealloc(setiterobject *si) } static int -setiter_traverse(setiterobject *si, visitproc visit, void *arg) +setiter_traverse(PyObject *self, visitproc visit, void *arg) { + setiterobject *si = (setiterobject*)self; Py_VISIT(si->si_set); return 0; } @@ -790,8 +817,6 @@ setiter_len(setiterobject *si, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); -static PyObject *setiter_iternext(setiterobject *si); - static PyObject * setiter_reduce(setiterobject *si, PyObject *Py_UNUSED(ignored)) { @@ -816,9 +841,10 @@ static PyMethodDef setiter_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyObject *setiter_iternext(setiterobject *si) +static PyObject *setiter_iternext(PyObject *self) { - PyObject *key; + setiterobject *si = (setiterobject*)self; + PyObject *key = NULL; Py_ssize_t i, mask; setentry *entry; PySetObject *so = si->si_set; @@ -827,30 +853,35 @@ static PyObject *setiter_iternext(setiterobject *si) return NULL; assert (PyAnySet_Check(so)); - if (si->si_used != so->used) { + Py_ssize_t so_used = FT_ATOMIC_LOAD_SSIZE(so->used); + Py_ssize_t si_used = FT_ATOMIC_LOAD_SSIZE(si->si_used); + if (si_used != so_used) { PyErr_SetString(PyExc_RuntimeError, "Set changed size during iteration"); si->si_used = -1; /* Make this state sticky */ return NULL; } + Py_BEGIN_CRITICAL_SECTION(so); i = si->si_pos; assert(i>=0); entry = so->table; mask = so->mask; - while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) + while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) { i++; + } + if (i <= mask) { + key = Py_NewRef(entry[i].key); + } + Py_END_CRITICAL_SECTION(); si->si_pos = i+1; - if (i > mask) - goto fail; + if (key == NULL) { + si->si_set = NULL; + Py_DECREF(so); + return NULL; + } si->len--; - key = entry[i].key; - return Py_NewRef(key); - -fail: - si->si_set = NULL; - Py_DECREF(so); - return NULL; + return key; } PyTypeObject PySetIter_Type = { @@ -859,7 +890,7 @@ PyTypeObject PySetIter_Type = { sizeof(setiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)setiter_dealloc, /* tp_dealloc */ + setiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -876,71 +907,73 @@ PyTypeObject PySetIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)setiter_traverse, /* tp_traverse */ + setiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)setiter_iternext, /* tp_iternext */ + setiter_iternext, /* tp_iternext */ setiter_methods, /* tp_methods */ 0, }; static PyObject * -set_iter(PySetObject *so) +set_iter(PyObject *so) { + Py_ssize_t size = set_len(so); setiterobject *si = PyObject_GC_New(setiterobject, &PySetIter_Type); if (si == NULL) return NULL; si->si_set = (PySetObject*)Py_NewRef(so); - si->si_used = so->used; + si->si_used = size; si->si_pos = 0; - si->len = so->used; + si->len = size; _PyObject_GC_TRACK(si); return (PyObject *)si; } static int -set_update_internal(PySetObject *so, PyObject *other) +set_update_dict_lock_held(PySetObject *so, PyObject *other) { - PyObject *key, *it; - - if (PyAnySet_Check(other)) - return set_merge(so, other); + assert(PyDict_CheckExact(other)); - if (PyDict_CheckExact(other)) { - PyObject *value; - Py_ssize_t pos = 0; - Py_hash_t hash; - Py_ssize_t dictsize = PyDict_GET_SIZE(other); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(other); - /* Do one big resize at the start, rather than - * incrementally resizing as we insert new keys. Expect - * that there will be no (or few) overlapping keys. - */ - if (dictsize < 0) + /* Do one big resize at the start, rather than + * incrementally resizing as we insert new keys. Expect + * that there will be no (or few) overlapping keys. + */ + Py_ssize_t dictsize = PyDict_GET_SIZE(other); + if ((so->fill + dictsize)*5 >= so->mask*3) { + if (set_table_resize(so, (so->used + dictsize)*2) != 0) { return -1; - if ((so->fill + dictsize)*5 >= so->mask*3) { - if (set_table_resize(so, (so->used + dictsize)*2) != 0) - return -1; } - int err = 0; - Py_BEGIN_CRITICAL_SECTION(other); - while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - if (set_add_entry(so, key, hash)) { - err = -1; - goto exit; - } + } + + Py_ssize_t pos = 0; + PyObject *key; + PyObject *value; + Py_hash_t hash; + while (_PyDict_Next(other, &pos, &key, &value, &hash)) { + if (set_add_entry(so, key, hash)) { + return -1; } -exit: - Py_END_CRITICAL_SECTION(); - return err; } + return 0; +} - it = PyObject_GetIter(other); - if (it == NULL) +static int +set_update_iterable_lock_held(PySetObject *so, PyObject *other) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + + PyObject *it = PyObject_GetIter(other); + if (it == NULL) { return -1; + } + PyObject *key; while ((key = PyIter_Next(it)) != NULL) { if (set_add_key(so, key)) { Py_DECREF(it); @@ -955,23 +988,85 @@ set_update_internal(PySetObject *so, PyObject *other) return 0; } +static int +set_update_lock_held(PySetObject *so, PyObject *other) +{ + if (PyAnySet_Check(other)) { + return set_merge_lock_held(so, other); + } + else if (PyDict_CheckExact(other)) { + return set_update_dict_lock_held(so, other); + } + return set_update_iterable_lock_held(so, other); +} + +// set_update for a `so` that is only visible to the current thread +static int +set_update_local(PySetObject *so, PyObject *other) +{ + assert(Py_REFCNT(so) == 1); + if (PyAnySet_Check(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(other); + rv = set_merge_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } + else if (PyDict_CheckExact(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(other); + rv = set_update_dict_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } + return set_update_iterable_lock_held(so, other); +} + +static int +set_update_internal(PySetObject *so, PyObject *other) +{ + if (PyAnySet_Check(other)) { + if (Py_Is((PyObject *)so, other)) { + return 0; + } + int rv; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_merge_lock_held(so, other); + Py_END_CRITICAL_SECTION2(); + return rv; + } + else if (PyDict_CheckExact(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_update_dict_lock_held(so, other); + Py_END_CRITICAL_SECTION2(); + return rv; + } + else { + int rv; + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_update_iterable_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } +} + /*[clinic input] set.update so: setobject *others as args: object - / Update the set, adding elements from all others. [clinic start generated code]*/ static PyObject * -set_update_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=34f6371704974c8a input=eb47c4fbaeb3286e]*/ +set_update_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=050e2a21f8d7d16a input=df4fe486e38cd337]*/ { Py_ssize_t i; - for (i=0 ; iweakreflist = NULL; if (iterable != NULL) { - if (set_update_internal(so, iterable)) { + if (set_update_local(so, iterable)) { Py_DECREF(so); return NULL; } @@ -1099,7 +1194,9 @@ set_swap_bodies(PySetObject *a, PySetObject *b) Py_hash_t h; t = a->fill; a->fill = b->fill; b->fill = t; - t = a->used; a->used = b->used; b->used = t; + t = a->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(a->used, b->used); + FT_ATOMIC_STORE_SSIZE_RELAXED(b->used, t); t = a->mask; a->mask = b->mask; b->mask = t; u = a->table; @@ -1126,6 +1223,7 @@ set_swap_bodies(PySetObject *a, PySetObject *b) } /*[clinic input] +@critical_section set.copy so: setobject @@ -1134,12 +1232,22 @@ Return a shallow copy of a set. static PyObject * set_copy_impl(PySetObject *so) -/*[clinic end generated code: output=c9223a1e1cc6b041 input=2b80b288d47b8cf1]*/ +/*[clinic end generated code: output=c9223a1e1cc6b041 input=c169a4fbb8209257]*/ { - return make_new_set_basetype(Py_TYPE(so), (PyObject *)so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + PyObject *copy = make_new_set_basetype(Py_TYPE(so), NULL); + if (copy == NULL) { + return NULL; + } + if (set_merge_lock_held((PySetObject *)copy, (PyObject *)so) < 0) { + Py_DECREF(copy); + return NULL; + } + return copy; } /*[clinic input] +@critical_section frozenset.copy so: setobject @@ -1148,15 +1256,16 @@ Return a shallow copy of a set. static PyObject * frozenset_copy_impl(PySetObject *so) -/*[clinic end generated code: output=b356263526af9e70 input=3dc65577d344eff7]*/ +/*[clinic end generated code: output=b356263526af9e70 input=fbf5bef131268dd7]*/ { if (PyFrozenSet_CheckExact(so)) { return Py_NewRef(so); } - return set_copy(so, NULL); + return set_copy_impl(so); } /*[clinic input] +@critical_section set.clear so: setobject @@ -1165,9 +1274,9 @@ Remove all elements from this set. static PyObject * set_clear_impl(PySetObject *so) -/*[clinic end generated code: output=4e71d5a83904161a input=74ac19794da81a39]*/ +/*[clinic end generated code: output=4e71d5a83904161a input=c6f831b366111950]*/ { - set_clear_internal(so); + set_clear_internal((PyObject*)so); Py_RETURN_NONE; } @@ -1175,14 +1284,13 @@ set_clear_impl(PySetObject *so) set.union so: setobject *others as args: object - / Return a new set with elements from the set and all others. [clinic start generated code]*/ static PyObject * -set_union_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=2c83d05a446a1477 input=2e2024fa1e40ac84]*/ +set_union_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=f68ec24d5c19d404 input=ddf088706e9577b2]*/ { PySetObject *result; PyObject *other; @@ -1192,11 +1300,11 @@ set_union_impl(PySetObject *so, PyObject *args) if (result == NULL) return NULL; - for (i=0 ; ikey); + Py_hash_t hash = entry->hash; + int rv = set_discard_entry(so, key, hash); + if (rv < 0) { + Py_DECREF(key); + return -1; + } + if (rv == DISCARD_NOTFOUND) { + if (set_add_entry(so, key, hash)) { + Py_DECREF(key); + return -1; + } + } + Py_DECREF(key); + } + return 0; } /*[clinic input] @@ -1739,58 +1926,41 @@ static PyObject * set_symmetric_difference_update(PySetObject *so, PyObject *other) /*[clinic end generated code: output=fbb049c0806028de input=a50acf0365e1f0a5]*/ { - PySetObject *otherset; - PyObject *key; - Py_ssize_t pos = 0; - Py_hash_t hash; - setentry *entry; - int rv; - - if ((PyObject *)so == other) + if (Py_Is((PyObject *)so, other)) { return set_clear(so, NULL); + } + int rv; if (PyDict_CheckExact(other)) { - PyObject *res; - - Py_BEGIN_CRITICAL_SECTION(other); - res = set_symmetric_difference_update_dict(so, other); - Py_END_CRITICAL_SECTION(); - - return res; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_symmetric_difference_update_dict(so, other); + Py_END_CRITICAL_SECTION2(); } - - if (PyAnySet_Check(other)) { - otherset = (PySetObject *)Py_NewRef(other); - } else { - otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); - if (otherset == NULL) - return NULL; + else if (PyAnySet_Check(other)) { + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_symmetric_difference_update_set(so, (PySetObject *)other); + Py_END_CRITICAL_SECTION2(); } - - while (set_next(otherset, &pos, &entry)) { - key = entry->key; - hash = entry->hash; - Py_INCREF(key); - rv = set_discard_entry(so, key, hash); - if (rv < 0) { - Py_DECREF(otherset); - Py_DECREF(key); + else { + PySetObject *otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); + if (otherset == NULL) { return NULL; } - if (rv == DISCARD_NOTFOUND) { - if (set_add_entry(so, key, hash)) { - Py_DECREF(otherset); - Py_DECREF(key); - return NULL; - } - } - Py_DECREF(key); + + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_symmetric_difference_update_set(so, otherset); + Py_END_CRITICAL_SECTION(); + + Py_DECREF(otherset); + } + if (rv < 0) { + return NULL; } - Py_DECREF(otherset); Py_RETURN_NONE; } /*[clinic input] +@critical_section so other set.symmetric_difference so: setobject other: object @@ -1800,39 +1970,42 @@ Return a new set with elements in either the set or other but not both. [clinic start generated code]*/ static PyObject * -set_symmetric_difference(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=f95364211b88775a input=f18af370ad72ebac]*/ +set_symmetric_difference_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=270ee0b5d42b0797 input=624f6e7bbdf70db1]*/ { - PyObject *rv; - PySetObject *otherset; - - otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); - if (otherset == NULL) + PySetObject *result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); + if (result == NULL) { return NULL; - rv = set_symmetric_difference_update(otherset, (PyObject *)so); - if (rv == NULL) { - Py_DECREF(otherset); + } + if (set_update_lock_held(result, other) < 0) { + Py_DECREF(result); return NULL; } - Py_DECREF(rv); - return (PyObject *)otherset; + if (set_symmetric_difference_update_set(result, so) < 0) { + Py_DECREF(result); + return NULL; + } + return (PyObject *)result; } static PyObject * -set_xor(PySetObject *so, PyObject *other) +set_xor(PyObject *self, PyObject *other) { - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + if (!PyAnySet_Check(self) || !PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); return set_symmetric_difference(so, other); } static PyObject * -set_ixor(PySetObject *so, PyObject *other) +set_ixor(PyObject *self, PyObject *other) { PyObject *result; if (!PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); + result = set_symmetric_difference_update(so, other); if (result == NULL) return NULL; @@ -1841,6 +2014,7 @@ set_ixor(PySetObject *so, PyObject *other) } /*[clinic input] +@critical_section so other set.issubset so: setobject other: object @@ -1850,8 +2024,8 @@ Report whether another set contains this set. [clinic start generated code]*/ static PyObject * -set_issubset(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=78aef1f377aedef1 input=37fbc579b609db0c]*/ +set_issubset_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=b2b59d5f314555ce input=f2a4fd0f2537758b]*/ { setentry *entry; Py_ssize_t pos = 0; @@ -1885,6 +2059,7 @@ set_issubset(PySetObject *so, PyObject *other) } /*[clinic input] +@critical_section so other set.issuperset so: setobject other: object @@ -1894,8 +2069,8 @@ Report whether this set contains another set. [clinic start generated code]*/ static PyObject * -set_issuperset(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=7d2b71dd714a7ec7 input=fd5dab052f2e9bb3]*/ +set_issuperset_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=ecf00ce552c09461 input=5f2e1f262e6e4ccc]*/ { if (PyAnySet_Check(other)) { return set_issubset((PySetObject *)other, (PyObject *)so); @@ -1925,8 +2100,9 @@ set_issuperset(PySetObject *so, PyObject *other) } static PyObject * -set_richcompare(PySetObject *v, PyObject *w, int op) +set_richcompare(PyObject *self, PyObject *w, int op) { + PySetObject *v = _PySet_CAST(self); PyObject *r1; int r2; @@ -1943,7 +2119,7 @@ set_richcompare(PySetObject *v, PyObject *w, int op) Py_RETURN_FALSE; return set_issubset(v, w); case Py_NE: - r1 = set_richcompare(v, w, Py_EQ); + r1 = set_richcompare((PyObject*)v, w, Py_EQ); if (r1 == NULL) return NULL; r2 = PyObject_IsTrue(r1); @@ -1968,6 +2144,7 @@ set_richcompare(PySetObject *v, PyObject *w, int op) } /*[clinic input] +@critical_section set.add so: setobject object as key: object @@ -1979,8 +2156,8 @@ This has no effect if the element is already present. [clinic start generated code]*/ static PyObject * -set_add(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=cd9c2d5c2069c2ba input=96f1efe029e47972]*/ +set_add_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=4cc4a937f1425c96 input=03baf62cb0e66514]*/ { if (set_add_key(so, key)) return NULL; @@ -1988,9 +2165,8 @@ set_add(PySetObject *so, PyObject *key) } static int -set_contains(PySetObject *so, PyObject *key) +set_contains_lock_held(PySetObject *so, PyObject *key) { - PyObject *tmpkey; int rv; rv = set_contains_key(so, key); @@ -1998,16 +2174,34 @@ set_contains(PySetObject *so, PyObject *key) if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return -1; PyErr_Clear(); - tmpkey = make_new_set(&PyFrozenSet_Type, key); - if (tmpkey == NULL) - return -1; - rv = set_contains_key(so, tmpkey); - Py_DECREF(tmpkey); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(key); + hash = frozenset_hash_impl(key); + Py_END_CRITICAL_SECTION(); + rv = set_contains_entry(so, key, hash); } return rv; } +int +_PySet_Contains(PySetObject *so, PyObject *key) +{ + int rv; + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_contains_lock_held(so, key); + Py_END_CRITICAL_SECTION(); + return rv; +} + +static int +set_contains(PyObject *self, PyObject *key) +{ + PySetObject *so = _PySet_CAST(self); + return _PySet_Contains(so, key); +} + /*[clinic input] +@critical_section @coexist set.__contains__ so: setobject @@ -2018,18 +2212,19 @@ x.__contains__(y) <==> y in x. [clinic start generated code]*/ static PyObject * -set___contains__(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=b5948bc5c590d3ca input=cf4c72db704e4cf0]*/ +set___contains___impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=b44863d034b3c70e input=4a7d568459617f24]*/ { long result; - result = set_contains(so, key); + result = set_contains_lock_held(so, key); if (result < 0) return NULL; return PyBool_FromLong(result); } /*[clinic input] +@critical_section set.remove so: setobject object as key: object @@ -2041,10 +2236,9 @@ If the element is not a member, raise a KeyError. [clinic start generated code]*/ static PyObject * -set_remove(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=08ae496d0cd2b8c1 input=10132515dfe8ebd7]*/ +set_remove_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=0b9134a2a2200363 input=893e1cb1df98227a]*/ { - PyObject *tmpkey; int rv; rv = set_discard_key(so, key); @@ -2052,11 +2246,11 @@ set_remove(PySetObject *so, PyObject *key) if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return NULL; PyErr_Clear(); - tmpkey = make_new_set(&PyFrozenSet_Type, key); - if (tmpkey == NULL) - return NULL; - rv = set_discard_key(so, tmpkey); - Py_DECREF(tmpkey); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(key); + hash = frozenset_hash_impl(key); + Py_END_CRITICAL_SECTION(); + rv = set_discard_entry(so, key, hash); if (rv < 0) return NULL; } @@ -2069,6 +2263,7 @@ set_remove(PySetObject *so, PyObject *key) } /*[clinic input] +@critical_section set.discard so: setobject object as key: object @@ -2081,10 +2276,9 @@ an exception when an element is missing from the set. [clinic start generated code]*/ static PyObject * -set_discard(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=9181b60d7bb7d480 input=82a689eba94d5ad9]*/ +set_discard_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=eec3b687bf32759e input=861cb7fb69b4def0]*/ { - PyObject *tmpkey; int rv; rv = set_discard_key(so, key); @@ -2092,11 +2286,11 @@ set_discard(PySetObject *so, PyObject *key) if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return NULL; PyErr_Clear(); - tmpkey = make_new_set(&PyFrozenSet_Type, key); - if (tmpkey == NULL) - return NULL; - rv = set_discard_key(so, tmpkey); - Py_DECREF(tmpkey); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(key); + hash = frozenset_hash_impl(key); + Py_END_CRITICAL_SECTION(); + rv = set_discard_entry(so, key, hash); if (rv < 0) return NULL; } @@ -2104,6 +2298,7 @@ set_discard(PySetObject *so, PyObject *key) } /*[clinic input] +@critical_section set.__reduce__ so: setobject @@ -2112,7 +2307,7 @@ Return state information for pickling. static PyObject * set___reduce___impl(PySetObject *so) -/*[clinic end generated code: output=9af7d0e029df87ee input=531375e87a24a449]*/ +/*[clinic end generated code: output=9af7d0e029df87ee input=59405a4249e82f71]*/ { PyObject *keys=NULL, *args=NULL, *result=NULL, *state=NULL; @@ -2134,6 +2329,7 @@ set___reduce___impl(PySetObject *so) } /*[clinic input] +@critical_section set.__sizeof__ so: setobject @@ -2142,7 +2338,7 @@ S.__sizeof__() -> size of S in memory, in bytes. static PyObject * set___sizeof___impl(PySetObject *so) -/*[clinic end generated code: output=4bfa3df7bd38ed88 input=0f214fc2225319fc]*/ +/*[clinic end generated code: output=4bfa3df7bd38ed88 input=09e1a09f168eaa23]*/ { size_t res = _PyObject_SIZE(Py_TYPE(so)); if (so->table != so->smalltable) { @@ -2152,17 +2348,29 @@ set___sizeof___impl(PySetObject *so) } static int -set_init(PySetObject *self, PyObject *args, PyObject *kwds) +set_init(PyObject *so, PyObject *args, PyObject *kwds) { + PySetObject *self = _PySet_CAST(so); PyObject *iterable = NULL; - if (!_PyArg_NoKeywords("set", kwds)) + if (!_PyArg_NoKeywords("set", kwds)) return -1; if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; + + if (Py_REFCNT(self) == 1 && self->fill == 0) { + self->hash = -1; + if (iterable == NULL) { + return 0; + } + return set_update_local(self, iterable); + } + Py_BEGIN_CRITICAL_SECTION(self); if (self->fill) - set_clear_internal(self); + set_clear_internal((PyObject*)self); self->hash = -1; + Py_END_CRITICAL_SECTION(); + if (iterable == NULL) return 0; return set_update_internal(self, iterable); @@ -2198,7 +2406,7 @@ static PySequenceMethods set_as_sequence = { 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)set_contains, /* sq_contains */ + set_contains, /* sq_contains */ }; /* set object ********************************************************/ @@ -2230,7 +2438,7 @@ static PyMethodDef set_methods[] = { static PyNumberMethods set_as_number = { 0, /*nb_add*/ - (binaryfunc)set_sub, /*nb_subtract*/ + set_sub, /*nb_subtract*/ 0, /*nb_multiply*/ 0, /*nb_remainder*/ 0, /*nb_divmod*/ @@ -2242,22 +2450,22 @@ static PyNumberMethods set_as_number = { 0, /*nb_invert*/ 0, /*nb_lshift*/ 0, /*nb_rshift*/ - (binaryfunc)set_and, /*nb_and*/ - (binaryfunc)set_xor, /*nb_xor*/ - (binaryfunc)set_or, /*nb_or*/ + set_and, /*nb_and*/ + set_xor, /*nb_xor*/ + set_or, /*nb_or*/ 0, /*nb_int*/ 0, /*nb_reserved*/ 0, /*nb_float*/ 0, /*nb_inplace_add*/ - (binaryfunc)set_isub, /*nb_inplace_subtract*/ + set_isub, /*nb_inplace_subtract*/ 0, /*nb_inplace_multiply*/ 0, /*nb_inplace_remainder*/ 0, /*nb_inplace_power*/ 0, /*nb_inplace_lshift*/ 0, /*nb_inplace_rshift*/ - (binaryfunc)set_iand, /*nb_inplace_and*/ - (binaryfunc)set_ixor, /*nb_inplace_xor*/ - (binaryfunc)set_ior, /*nb_inplace_or*/ + set_iand, /*nb_inplace_and*/ + set_ixor, /*nb_inplace_xor*/ + set_ior, /*nb_inplace_or*/ }; PyDoc_STRVAR(set_doc, @@ -2272,12 +2480,12 @@ PyTypeObject PySet_Type = { sizeof(PySetObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)set_dealloc, /* tp_dealloc */ + set_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)set_repr, /* tp_repr */ + set_repr, /* tp_repr */ &set_as_number, /* tp_as_number */ &set_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -2289,13 +2497,13 @@ PyTypeObject PySet_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | - _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ + _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ set_doc, /* tp_doc */ - (traverseproc)set_traverse, /* tp_traverse */ - (inquiry)set_clear_internal, /* tp_clear */ - (richcmpfunc)set_richcompare, /* tp_richcompare */ + set_traverse, /* tp_traverse */ + set_clear_internal, /* tp_clear */ + set_richcompare, /* tp_richcompare */ offsetof(PySetObject, weakreflist), /* tp_weaklistoffset */ - (getiterfunc)set_iter, /* tp_iter */ + set_iter, /* tp_iter */ 0, /* tp_iternext */ set_methods, /* tp_methods */ 0, /* tp_members */ @@ -2305,7 +2513,7 @@ PyTypeObject PySet_Type = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - (initproc)set_init, /* tp_init */ + set_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ set_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ @@ -2333,7 +2541,7 @@ static PyMethodDef frozenset_methods[] = { static PyNumberMethods frozenset_as_number = { 0, /*nb_add*/ - (binaryfunc)set_sub, /*nb_subtract*/ + set_sub, /*nb_subtract*/ 0, /*nb_multiply*/ 0, /*nb_remainder*/ 0, /*nb_divmod*/ @@ -2345,9 +2553,9 @@ static PyNumberMethods frozenset_as_number = { 0, /*nb_invert*/ 0, /*nb_lshift*/ 0, /*nb_rshift*/ - (binaryfunc)set_and, /*nb_and*/ - (binaryfunc)set_xor, /*nb_xor*/ - (binaryfunc)set_or, /*nb_or*/ + set_and, /*nb_and*/ + set_xor, /*nb_xor*/ + set_or, /*nb_or*/ }; PyDoc_STRVAR(frozenset_doc, @@ -2362,12 +2570,12 @@ PyTypeObject PyFrozenSet_Type = { sizeof(PySetObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)set_dealloc, /* tp_dealloc */ + set_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)set_repr, /* tp_repr */ + set_repr, /* tp_repr */ &frozenset_as_number, /* tp_as_number */ &set_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -2379,13 +2587,13 @@ PyTypeObject PyFrozenSet_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | - _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ + _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ frozenset_doc, /* tp_doc */ - (traverseproc)set_traverse, /* tp_traverse */ - (inquiry)set_clear_internal, /* tp_clear */ - (richcmpfunc)set_richcompare, /* tp_richcompare */ + set_traverse, /* tp_traverse */ + set_clear_internal, /* tp_clear */ + set_richcompare, /* tp_richcompare */ offsetof(PySetObject, weakreflist), /* tp_weaklistoffset */ - (getiterfunc)set_iter, /* tp_iter */ + set_iter, /* tp_iter */ 0, /* tp_iternext */ frozenset_methods, /* tp_methods */ 0, /* tp_members */ @@ -2424,7 +2632,7 @@ PySet_Size(PyObject *anyset) PyErr_BadInternalCall(); return -1; } - return PySet_GET_SIZE(anyset); + return set_len(anyset); } int @@ -2434,7 +2642,14 @@ PySet_Clear(PyObject *set) PyErr_BadInternalCall(); return -1; } - return set_clear_internal((PySetObject *)set); + (void)set_clear((PySetObject *)set, NULL); + return 0; +} + +void +_PySet_ClearInternal(PySetObject *so) +{ + (void)set_clear_internal((PyObject*)so); } int @@ -2444,7 +2659,12 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_contains_key((PySetObject *)anyset, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_contains_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2454,7 +2674,12 @@ PySet_Discard(PyObject *set, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_discard_key((PySetObject *)set, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(set); + rv = set_discard_key((PySetObject *)set, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2465,7 +2690,12 @@ PySet_Add(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_add_key((PySetObject *)anyset, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_add_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2484,6 +2714,23 @@ _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash return 1; } +int +_PySet_NextEntryRef(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash) +{ + setentry *entry; + + if (!PyAnySet_Check(set)) { + PyErr_BadInternalCall(); + return -1; + } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(set); + if (set_next((PySetObject *)set, pos, &entry) == 0) + return 0; + *key = Py_NewRef(entry->key); + *hash = entry->hash; + return 1; +} + PyObject * PySet_Pop(PyObject *set) { diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 9880c123c80f95..4fef0af93fe095 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -15,6 +15,7 @@ this type and there is exactly one in existence. #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -57,6 +58,11 @@ static PyMethodDef ellipsis_methods[] = { {NULL, NULL} }; +PyDoc_STRVAR(ellipsis_doc, +"ellipsis()\n" +"--\n\n" +"The type of the Ellipsis singleton."); + PyTypeObject PyEllipsis_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "ellipsis", /* tp_name */ @@ -78,7 +84,7 @@ PyTypeObject PyEllipsis_Type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ + ellipsis_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ @@ -103,20 +109,6 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type); /* Slice object implementation */ -void _PySlice_ClearFreeList(_PyFreeListState *state, int is_finalization) -{ - if (!is_finalization) { - return; - } -#ifdef WITH_FREELISTS - PySliceObject *obj = state->slices.slice_cache; - if (obj != NULL) { - state->slices.slice_cache = NULL; - PyObject_GC_Del(obj); - } -#endif -} - /* start, stop, and step are python objects with None indicating no index is present. */ @@ -125,17 +117,8 @@ static PySliceObject * _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) { assert(start != NULL && stop != NULL && step != NULL); - PySliceObject *obj; -#ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); - if (state->slices.slice_cache != NULL) { - obj = state->slices.slice_cache; - state->slices.slice_cache = NULL; - _Py_NewReference((PyObject *)obj); - } - else -#endif - { + PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices); + if (obj == NULL) { obj = PyObject_GC_New(PySliceObject, &PySlice_Type); if (obj == NULL) { goto error; @@ -360,20 +343,11 @@ Create a slice object. This is used for extended slicing (e.g. a[0:10:2])."); static void slice_dealloc(PySliceObject *r) { - _PyObject_GC_UNTRACK(r); + PyObject_GC_UnTrack(r); Py_DECREF(r->step); Py_DECREF(r->start); Py_DECREF(r->stop); -#ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); - if (state->slices.slice_cache == NULL) { - state->slices.slice_cache = r; - } - else -#endif - { - PyObject_GC_Del(r); - } + _Py_FREELIST_FREE(slices, r, PyObject_GC_Del); } static PyObject * diff --git a/Objects/stringlib/clinic/transmogrify.h.h b/Objects/stringlib/clinic/transmogrify.h.h index 3a985ab5c7a9f5..cef7a9496fa874 100644 --- a/Objects/stringlib/clinic/transmogrify.h.h +++ b/Objects/stringlib/clinic/transmogrify.h.h @@ -113,10 +113,24 @@ stringlib_ljust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "ljust(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "ljust(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -169,10 +183,24 @@ stringlib_rjust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "rjust(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "rjust(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -225,10 +253,24 @@ stringlib_center(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "center(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "center(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -279,4 +321,4 @@ stringlib_zfill(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=b409bdf9ab68d5a6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06dd79019356b6bb input=a9049054013a1b77]*/ diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index f98e71c3fc6ecf..440410d0aef17d 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -331,7 +331,7 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, case _Py_ERROR_REPLACE: memset(p, '?', endpos - startpos); p += (endpos - startpos); - /* fall through */ + _Py_FALLTHROUGH; case _Py_ERROR_IGNORE: i += (endpos - startpos - 1); break; @@ -379,7 +379,7 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, } startpos = k; assert(startpos < endpos); - /* fall through */ + _Py_FALLTHROUGH; default: rep = unicode_encode_call_errorhandler( errors, &error_handler_obj, "utf-8", "surrogates not allowed", diff --git a/Objects/stringlib/eq.h b/Objects/stringlib/eq.h index 2eac4baf5ca9ce..821b692f26b830 100644 --- a/Objects/stringlib/eq.h +++ b/Objects/stringlib/eq.h @@ -4,14 +4,19 @@ * unicode_eq() is called when the hash of two unicode objects is equal. */ Py_LOCAL_INLINE(int) -unicode_eq(PyObject *a, PyObject *b) +unicode_eq(PyObject *str1, PyObject *str2) { - if (PyUnicode_GET_LENGTH(a) != PyUnicode_GET_LENGTH(b)) + Py_ssize_t len = PyUnicode_GET_LENGTH(str1); + if (PyUnicode_GET_LENGTH(str2) != len) { return 0; - if (PyUnicode_GET_LENGTH(a) == 0) - return 1; - if (PyUnicode_KIND(a) != PyUnicode_KIND(b)) + } + + int kind = PyUnicode_KIND(str1); + if (PyUnicode_KIND(str2) != kind) { return 0; - return memcmp(PyUnicode_1BYTE_DATA(a), PyUnicode_1BYTE_DATA(b), - PyUnicode_GET_LENGTH(a) * PyUnicode_KIND(a)) == 0; + } + + const void *data1 = PyUnicode_DATA(str1); + const void *data2 = PyUnicode_DATA(str2); + return (memcmp(data1, data2, len * kind) == 0); } diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 257b7bd6788ad2..05e700b06258f0 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -256,7 +256,7 @@ STRINGLIB(_factorize)(const STRINGLIB_CHAR *needle, The local period of the cut is the minimal length of a string w such that (left endswith w or w endswith left) - and (right startswith w or w startswith left). + and (right startswith w or w startswith right). The Critical Factorization Theorem says that this maximal local period is the global period of the string. @@ -337,21 +337,20 @@ STRINGLIB(_preprocess)(const STRINGLIB_CHAR *needle, Py_ssize_t len_needle, if (p->is_periodic) { assert(p->cut <= len_needle/2); assert(p->cut < p->period); - p->gap = 0; // unused } else { // A lower bound on the period p->period = Py_MAX(p->cut, len_needle - p->cut) + 1; - // The gap between the last character and the previous - // occurrence of an equivalent character (modulo TABLE_SIZE) - p->gap = len_needle; - STRINGLIB_CHAR last = needle[len_needle - 1] & TABLE_MASK; - for (Py_ssize_t i = len_needle - 2; i >= 0; i--) { - STRINGLIB_CHAR x = needle[i] & TABLE_MASK; - if (x == last) { - p->gap = len_needle - 1 - i; - break; - } + } + // The gap between the last character and the previous + // occurrence of an equivalent character (modulo TABLE_SIZE) + p->gap = len_needle; + STRINGLIB_CHAR last = needle[len_needle - 1] & TABLE_MASK; + for (Py_ssize_t i = len_needle - 2; i >= 0; i--) { + STRINGLIB_CHAR x = needle[i] & TABLE_MASK; + if (x == last) { + p->gap = len_needle - 1 - i; + break; } } // Fill up a compressed Boyer-Moore "Bad Character" table @@ -383,6 +382,8 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, const STRINGLIB_CHAR *window; LOG("===== Two-way: \"%s\" in \"%s\". =====\n", needle, haystack); + Py_ssize_t gap = p->gap; + Py_ssize_t gap_jump_end = Py_MIN(len_needle, cut + gap); if (p->is_periodic) { LOG("Needle is periodic.\n"); Py_ssize_t memory = 0; @@ -408,8 +409,16 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, Py_ssize_t i = Py_MAX(cut, memory); for (; i < len_needle; i++) { if (needle[i] != window[i]) { - LOG("Right half does not match.\n"); - window_last += i - cut + 1; + if (i < gap_jump_end) { + LOG("Early right half mismatch: jump by gap.\n"); + assert(gap >= i - cut + 1); + window_last += gap; + } + else { + LOG("Late right half mismatch: jump by n (>gap)\n"); + assert(i - cut + 1 > gap); + window_last += i - cut + 1; + } memory = 0; goto periodicwindowloop; } @@ -442,10 +451,8 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, } } else { - Py_ssize_t gap = p->gap; period = Py_MAX(gap, period); LOG("Needle is not periodic.\n"); - Py_ssize_t gap_jump_end = Py_MIN(len_needle, cut + gap); windowloop: while (window_last < haystack_end) { for (;;) { @@ -463,19 +470,19 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, window = window_last - len_needle + 1; assert((window[len_needle - 1] & TABLE_MASK) == (needle[len_needle - 1] & TABLE_MASK)); - for (Py_ssize_t i = cut; i < gap_jump_end; i++) { - if (needle[i] != window[i]) { - LOG("Early right half mismatch: jump by gap.\n"); - assert(gap >= i - cut + 1); - window_last += gap; - goto windowloop; - } - } - for (Py_ssize_t i = gap_jump_end; i < len_needle; i++) { + Py_ssize_t i = cut; + for (; i < len_needle; i++) { if (needle[i] != window[i]) { - LOG("Late right half mismatch.\n"); - assert(i - cut + 1 > gap); - window_last += i - cut + 1; + if (i < gap_jump_end) { + LOG("Early right half mismatch: jump by gap.\n"); + assert(gap >= i - cut + 1); + window_last += gap; + } + else { + LOG("Late right half mismatch: jump by n (>gap)\n"); + assert(i - cut + 1 > gap); + window_last += i - cut + 1; + } goto windowloop; } } @@ -746,6 +753,22 @@ STRINGLIB(count_char)(const STRINGLIB_CHAR *s, Py_ssize_t n, } +static inline Py_ssize_t +STRINGLIB(count_char_no_maxcount)(const STRINGLIB_CHAR *s, Py_ssize_t n, + const STRINGLIB_CHAR p0) +/* A specialized function of count_char that does not cut off at a maximum. + As a result, the compiler is able to vectorize the loop. */ +{ + Py_ssize_t count = 0; + for (Py_ssize_t i = 0; i < n; i++) { + if (s[i] == p0) { + count++; + } + } + return count; +} + + Py_LOCAL_INLINE(Py_ssize_t) FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, const STRINGLIB_CHAR* p, Py_ssize_t m, @@ -766,6 +789,9 @@ FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, else if (mode == FAST_RSEARCH) return STRINGLIB(rfind_char)(s, n, p[0]); else { + if (maxcount == PY_SSIZE_T_MAX) { + return STRINGLIB(count_char_no_maxcount)(s, n, p[0]); + } return STRINGLIB(count_char)(s, n, p[0], maxcount); } } diff --git a/Objects/stringlib/find.h b/Objects/stringlib/find.h index 509b9297396be8..c385718a5b2692 100644 --- a/Objects/stringlib/find.h +++ b/Objects/stringlib/find.h @@ -70,50 +70,3 @@ STRINGLIB(contains_obj)(PyObject* str, PyObject* sub) } #endif /* STRINGLIB_WANT_CONTAINS_OBJ */ - -/* -This function is a helper for the "find" family (find, rfind, index, -rindex) and for count, startswith and endswith, because they all have -the same behaviour for the arguments. - -It does not touch the variables received until it knows everything -is ok. -*/ - -#define FORMAT_BUFFER_SIZE 50 - -Py_LOCAL_INLINE(int) -STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args, - PyObject **subobj, - Py_ssize_t *start, Py_ssize_t *end) -{ - PyObject *tmp_subobj; - Py_ssize_t tmp_start = 0; - Py_ssize_t tmp_end = PY_SSIZE_T_MAX; - PyObject *obj_start=Py_None, *obj_end=Py_None; - char format[FORMAT_BUFFER_SIZE] = "O|OO:"; - size_t len = strlen(format); - - strncpy(format + len, function_name, FORMAT_BUFFER_SIZE - len - 1); - format[FORMAT_BUFFER_SIZE - 1] = '\0'; - - if (!PyArg_ParseTuple(args, format, &tmp_subobj, &obj_start, &obj_end)) - return 0; - - /* To support None in "start" and "end" arguments, meaning - the same as if they were not passed. - */ - if (obj_start != Py_None) - if (!_PyEval_SliceIndex(obj_start, &tmp_start)) - return 0; - if (obj_end != Py_None) - if (!_PyEval_SliceIndex(obj_end, &tmp_end)) - return 0; - - *start = tmp_start; - *end = tmp_end; - *subobj = tmp_subobj; - return 1; -} - -#undef FORMAT_BUFFER_SIZE diff --git a/Objects/stringlib/find_max_char.h b/Objects/stringlib/find_max_char.h index b9ffdfc2e352ce..7ab3fc88b331b1 100644 --- a/Objects/stringlib/find_max_char.h +++ b/Objects/stringlib/find_max_char.h @@ -1,6 +1,7 @@ /* Finding the optimal width of unicode characters in a buffer */ -#if !STRINGLIB_IS_UNICODE +/* find_max_char for one-byte will work for bytes objects as well. */ +#if !STRINGLIB_IS_UNICODE && STRINGLIB_SIZEOF_CHAR > 1 # error "find_max_char.h is specific to Unicode" #endif @@ -20,19 +21,20 @@ Py_LOCAL_INLINE(Py_UCS4) STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) { const unsigned char *p = (const unsigned char *) begin; + const unsigned char *_end = (const unsigned char *)end; - while (p < end) { + while (p < _end) { if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { /* Help register allocation */ const unsigned char *_p = p; - while (_p + SIZEOF_SIZE_T <= end) { + while (_p + SIZEOF_SIZE_T <= _end) { size_t value = *(const size_t *) _p; if (value & UCS1_ASCII_CHAR_MASK) return 255; _p += SIZEOF_SIZE_T; } p = _p; - if (p == end) + if (p == _end) break; } if (*p++ & 0x80) diff --git a/Objects/stringlib/repr.h b/Objects/stringlib/repr.h new file mode 100644 index 00000000000000..87b1a8ba629dc6 --- /dev/null +++ b/Objects/stringlib/repr.h @@ -0,0 +1,95 @@ +/* stringlib: repr() implementation */ + +#ifndef STRINGLIB_FASTSEARCH_H +#error must include "stringlib/fastsearch.h" before including this module +#endif + + +static void +STRINGLIB(repr)(PyObject *unicode, Py_UCS4 quote, + STRINGLIB_CHAR *odata) +{ + Py_ssize_t isize = PyUnicode_GET_LENGTH(unicode); + const void *idata = PyUnicode_DATA(unicode); + int ikind = PyUnicode_KIND(unicode); + + *odata++ = quote; + for (Py_ssize_t i = 0; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); + + /* Escape quotes and backslashes */ + if ((ch == quote) || (ch == '\\')) { + *odata++ = '\\'; + *odata++ = ch; + continue; + } + + /* Map special whitespace to '\t', \n', '\r' */ + if (ch == '\t') { + *odata++ = '\\'; + *odata++ = 't'; + } + else if (ch == '\n') { + *odata++ = '\\'; + *odata++ = 'n'; + } + else if (ch == '\r') { + *odata++ = '\\'; + *odata++ = 'r'; + } + + /* Map non-printable US ASCII to '\xhh' */ + else if (ch < ' ' || ch == 0x7F) { + *odata++ = '\\'; + *odata++ = 'x'; + *odata++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *odata++ = Py_hexdigits[ch & 0x000F]; + } + + /* Copy ASCII characters as-is */ + else if (ch < 0x7F) { + *odata++ = ch; + } + + /* Non-ASCII characters */ + else { + /* Map Unicode whitespace and control characters + (categories Z* and C* except ASCII space) + */ + if (!Py_UNICODE_ISPRINTABLE(ch)) { + *odata++ = '\\'; + /* Map 8-bit characters to '\xhh' */ + if (ch <= 0xff) { + *odata++ = 'x'; + *odata++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *odata++ = Py_hexdigits[ch & 0x000F]; + } + /* Map 16-bit characters to '\uxxxx' */ + else if (ch <= 0xffff) { + *odata++ = 'u'; + *odata++ = Py_hexdigits[(ch >> 12) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 8) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 4) & 0xF]; + *odata++ = Py_hexdigits[ch & 0xF]; + } + /* Map 21-bit characters to '\U00xxxxxx' */ + else { + *odata++ = 'U'; + *odata++ = Py_hexdigits[(ch >> 28) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 24) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 20) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 16) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 12) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 8) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 4) & 0xF]; + *odata++ = Py_hexdigits[ch & 0xF]; + } + } + /* Copy characters as-is */ + else { + *odata++ = ch; + } + } + } + *odata = quote; +} diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index 91c71ab5736c25..44b269ba8ceb55 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -73,7 +73,7 @@ Py_LOCAL_INLINE(PyObject *) SubString_new_object_or_empty(SubString *str) { if (str->str == NULL) { - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } return SubString_new_object(str); } @@ -531,7 +531,7 @@ render_field(PyObject *fieldobj, SubString *format_spec, _PyUnicodeWriter *write format_spec->start, format_spec->end); else - format_spec_object = PyUnicode_New(0, 0); + format_spec_object = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (format_spec_object == NULL) goto done; diff --git a/Objects/structseq.c b/Objects/structseq.c index 581d6ad240885a..56a7851b98788d 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -41,12 +41,20 @@ get_type_attr_as_size(PyTypeObject *tp, PyObject *name) get_type_attr_as_size(tp, &_Py_ID(n_sequence_fields)) #define REAL_SIZE_TP(tp) \ get_type_attr_as_size(tp, &_Py_ID(n_fields)) -#define REAL_SIZE(op) REAL_SIZE_TP(Py_TYPE(op)) +#define REAL_SIZE(op) get_real_size((PyObject *)op) #define UNNAMED_FIELDS_TP(tp) \ get_type_attr_as_size(tp, &_Py_ID(n_unnamed_fields)) #define UNNAMED_FIELDS(op) UNNAMED_FIELDS_TP(Py_TYPE(op)) +static Py_ssize_t +get_real_size(PyObject *op) +{ + // Compute the real size from the visible size (i.e., Py_SIZE()) and the + // number of non-sequence fields accounted for in tp_basicsize. + Py_ssize_t hidden = Py_TYPE(op)->tp_basicsize - offsetof(PyStructSequence, ob_item); + return Py_SIZE(op) + hidden / sizeof(PyObject *); +} PyObject * PyStructSequence_New(PyTypeObject *type) @@ -120,6 +128,9 @@ structseq_dealloc(PyStructSequence *obj) PyObject_GC_UnTrack(obj); PyTypeObject *tp = Py_TYPE(obj); + // gh-122527: We can't use REAL_SIZE_TP() or any macros that access the + // type's dictionary here, because the dictionary may have already been + // cleared by the garbage collector. size = REAL_SIZE(obj); for (i = 0; i < size; ++i) { Py_XDECREF(obj->ob_item[i]); @@ -255,83 +266,66 @@ static PyObject * structseq_repr(PyStructSequence *obj) { PyTypeObject *typ = Py_TYPE(obj); - _PyUnicodeWriter writer; - /* Write "typename(" */ - PyObject *type_name = PyUnicode_DecodeUTF8(typ->tp_name, - strlen(typ->tp_name), - NULL); - if (type_name == NULL) { + // count 5 characters per item: "x=1, " + Py_ssize_t type_name_len = strlen(typ->tp_name); + Py_ssize_t prealloc = (type_name_len + 1 + + VISIBLE_SIZE(obj) * 5 + 1); + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { return NULL; } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; - /* count 5 characters per item: "x=1, " */ - writer.min_length = (PyUnicode_GET_LENGTH(type_name) + 1 - + VISIBLE_SIZE(obj) * 5 + 1); - - if (_PyUnicodeWriter_WriteStr(&writer, type_name) < 0) { - Py_DECREF(type_name); + // Write "typename(" + if (PyUnicodeWriter_WriteUTF8(writer, typ->tp_name, type_name_len) < 0) { goto error; } - Py_DECREF(type_name); - - if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) { + if (PyUnicodeWriter_WriteChar(writer, '(') < 0) { goto error; } for (Py_ssize_t i=0; i < VISIBLE_SIZE(obj); i++) { if (i > 0) { - /* Write ", " */ - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + // Write ", " + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { + goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; } } - /* Write "name=repr" */ + // Write name const char *name_utf8 = typ->tp_members[i].name; if (name_utf8 == NULL) { - PyErr_Format(PyExc_SystemError, "In structseq_repr(), member %zd name is NULL" + PyErr_Format(PyExc_SystemError, + "In structseq_repr(), member %zd name is NULL" " for type %.500s", i, typ->tp_name); goto error; } - - PyObject *name = PyUnicode_DecodeUTF8(name_utf8, strlen(name_utf8), NULL); - if (name == NULL) { - goto error; - } - if (_PyUnicodeWriter_WriteStr(&writer, name) < 0) { - Py_DECREF(name); + if (PyUnicodeWriter_WriteUTF8(writer, name_utf8, -1) < 0) { goto error; } - Py_DECREF(name); - if (_PyUnicodeWriter_WriteChar(&writer, '=') < 0) { + // Write "=" + repr(value) + if (PyUnicodeWriter_WriteChar(writer, '=') < 0) { goto error; } - PyObject *value = PyStructSequence_GetItem((PyObject*)obj, i); assert(value != NULL); - PyObject *repr = PyObject_Repr(value); - if (repr == NULL) { + if (PyUnicodeWriter_WriteRepr(writer, value) < 0) { goto error; } - if (_PyUnicodeWriter_WriteStr(&writer, repr) < 0) { - Py_DECREF(repr); - goto error; - } - Py_DECREF(repr); } - if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) { + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { goto error; } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } @@ -453,7 +447,9 @@ structseq_replace(PyStructSequence *self, PyObject *args, PyObject *kwargs) static PyMethodDef structseq_methods[] = { {"__reduce__", (PyCFunction)structseq_reduce, METH_NOARGS, NULL}, - {"__replace__", _PyCFunction_CAST(structseq_replace), METH_VARARGS | METH_KEYWORDS, NULL}, + {"__replace__", _PyCFunction_CAST(structseq_replace), METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\n" + "Return a copy of the structure with new values for the specified fields.")}, {NULL, NULL} // sentinel }; @@ -563,10 +559,14 @@ initialize_members(PyStructSequence_Desc *desc, static void initialize_static_fields(PyTypeObject *type, PyStructSequence_Desc *desc, - PyMemberDef *tp_members, unsigned long tp_flags) + PyMemberDef *tp_members, Py_ssize_t n_members, + unsigned long tp_flags) { type->tp_name = desc->name; - type->tp_basicsize = sizeof(PyStructSequence) - sizeof(PyObject *); + // Account for hidden members in tp_basicsize because they are not + // included in the variable size. + Py_ssize_t n_hidden = n_members - desc->n_in_sequence; + type->tp_basicsize = sizeof(PyStructSequence) + (n_hidden - 1) * sizeof(PyObject *); type->tp_itemsize = sizeof(PyObject *); type->tp_dealloc = (destructor)structseq_dealloc; type->tp_repr = (reprfunc)structseq_repr; @@ -603,6 +603,9 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, PyStructSequence_Desc *desc, unsigned long tp_flags) { + if (Py_TYPE(type) == NULL) { + Py_SET_TYPE(type, &PyType_Type); + } Py_ssize_t n_unnamed_members; Py_ssize_t n_members = count_members(desc, &n_unnamed_members); PyMemberDef *members = NULL; @@ -616,9 +619,9 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, if (members == NULL) { goto error; } - initialize_static_fields(type, desc, members, tp_flags); + initialize_static_fields(type, desc, members, n_members, tp_flags); - _Py_SetImmortal(type); + _Py_SetImmortal((PyObject *)type); } #ifndef NDEBUG else { @@ -679,7 +682,7 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) if (members == NULL) { return -1; } - initialize_static_fields(type, desc, members, 0); + initialize_static_fields(type, desc, members, n_members, 0); if (initialize_static_type(type, desc, n_members, n_unnamed_members) < 0) { PyMem_Free(members); return -1; @@ -713,7 +716,7 @@ _PyStructSequence_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) return; } - _PyStaticType_Dealloc(interp, type); + _PyStaticType_FiniBuiltin(interp, type); if (_Py_IsMainInterpreter(interp)) { // Undo _PyStructSequence_InitBuiltinWithFlags(). @@ -755,7 +758,8 @@ _PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags) /* The name in this PyType_Spec is statically allocated so it is */ /* expected that it'll outlive the PyType_Spec */ spec.name = desc->name; - spec.basicsize = sizeof(PyStructSequence) - sizeof(PyObject *); + Py_ssize_t hidden = n_members - desc->n_in_sequence; + spec.basicsize = (int)(sizeof(PyStructSequence) + (hidden - 1) * sizeof(PyObject *)); spec.itemsize = sizeof(PyObject *); spec.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | tp_flags; spec.slots = slots; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 7d73c3fb0f7f2c..f3132e0933ac30 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_freelist.h" // _Py_FREELIST_PUSH(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_modsupport.h" // _PyArg_NoKwnames() @@ -17,7 +18,6 @@ class tuple "PyTupleObject *" "&PyTuple_Type" #include "clinic/tupleobject.c.h" -static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t); static inline int maybe_freelist_push(PyTupleObject *); @@ -38,22 +38,20 @@ tuple_alloc(Py_ssize_t size) PyErr_BadInternalCall(); return NULL; } -#ifdef Py_DEBUG assert(size != 0); // The empty tuple is statically allocated. -#endif - - PyTupleObject *op = maybe_freelist_pop(size); - if (op == NULL) { - /* Check for overflow */ - if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) - - sizeof(PyObject *))) / sizeof(PyObject *)) { - return (PyTupleObject *)PyErr_NoMemory(); + Py_ssize_t index = size - 1; + if (index < PyTuple_MAXSAVESIZE) { + PyTupleObject *op = _Py_FREELIST_POP(PyTupleObject, tuples[index]); + if (op != NULL) { + return op; } - op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size); - if (op == NULL) - return NULL; } - return op; + /* Check for overflow */ + if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) - + sizeof(PyObject *))) / sizeof(PyObject *)) { + return (PyTupleObject *)PyErr_NoMemory(); + } + return PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size); } // The empty tuple singleton is not tracked by the GC. @@ -183,8 +181,9 @@ PyTuple_Pack(Py_ssize_t n, ...) /* Methods */ static void -tupledealloc(PyTupleObject *op) +tuple_dealloc(PyObject *self) { + PyTupleObject *op = _PyTuple_CAST(self); if (Py_SIZE(op) == 0) { /* The empty tuple is statically allocated. */ if (op == &_Py_SINGLETON(tuple_empty)) { @@ -201,7 +200,7 @@ tupledealloc(PyTupleObject *op) } PyObject_GC_UnTrack(op); - Py_TRASHCAN_BEGIN(op, tupledealloc) + Py_TRASHCAN_BEGIN(op, tuple_dealloc) Py_ssize_t i = Py_SIZE(op); while (--i >= 0) { @@ -216,73 +215,71 @@ tupledealloc(PyTupleObject *op) } static PyObject * -tuplerepr(PyTupleObject *v) +tuple_repr(PyObject *self) { - Py_ssize_t i, n; - _PyUnicodeWriter writer; - - n = Py_SIZE(v); - if (n == 0) + PyTupleObject *v = _PyTuple_CAST(self); + Py_ssize_t n = PyTuple_GET_SIZE(v); + if (n == 0) { return PyUnicode_FromString("()"); + } /* While not mutable, it is still possible to end up with a cycle in a tuple through an object that stores itself within a tuple (and thus infinitely asks for the repr of itself). This should only be possible within a type. */ - i = Py_ReprEnter((PyObject *)v); - if (i != 0) { - return i > 0 ? PyUnicode_FromString("(...)") : NULL; + int res = Py_ReprEnter((PyObject *)v); + if (res != 0) { + return res > 0 ? PyUnicode_FromString("(...)") : NULL; } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; - if (Py_SIZE(v) > 1) { - /* "(" + "1" + ", 2" * (len - 1) + ")" */ - writer.min_length = 1 + 1 + (2 + 1) * (Py_SIZE(v) - 1) + 1; + Py_ssize_t prealloc; + if (n > 1) { + // "(" + "1" + ", 2" * (len - 1) + ")" + prealloc = 1 + 1 + (2 + 1) * (n - 1) + 1; } else { - /* "(1,)" */ - writer.min_length = 4; + // "(1,)" + prealloc = 4; + } + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { + goto error; } - if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) + if (PyUnicodeWriter_WriteChar(writer, '(') < 0) { goto error; + } /* Do repr() on each element. */ - for (i = 0; i < n; ++i) { - PyObject *s; - + for (Py_ssize_t i = 0; i < n; ++i) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { + goto error; + } } - s = PyObject_Repr(v->ob_item[i]); - if (s == NULL) - goto error; - - if (_PyUnicodeWriter_WriteStr(&writer, s) < 0) { - Py_DECREF(s); + if (PyUnicodeWriter_WriteRepr(writer, v->ob_item[i]) < 0) { goto error; } - Py_DECREF(s); } - writer.overallocate = 0; - if (n > 1) { - if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) + if (n == 1) { + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { goto error; + } } - else { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ",)", 2) < 0) - goto error; + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { + goto error; } Py_ReprLeave((PyObject *)v); - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); Py_ReprLeave((PyObject *)v); return NULL; } @@ -318,13 +315,14 @@ tuplerepr(PyTupleObject *v) /* Tests have shown that it's not worth to cache the hash value, see https://bugs.python.org/issue9685 */ static Py_hash_t -tuplehash(PyTupleObject *v) +tuple_hash(PyObject *op) { - Py_ssize_t i, len = Py_SIZE(v); + PyTupleObject *v = _PyTuple_CAST(op); + Py_ssize_t len = Py_SIZE(v); PyObject **item = v->ob_item; Py_uhash_t acc = _PyHASH_XXPRIME_5; - for (i = 0; i < len; i++) { + for (Py_ssize_t i = 0; i < len; i++) { Py_uhash_t lane = PyObject_Hash(item[i]); if (lane == (Py_uhash_t)-1) { return -1; @@ -344,25 +342,27 @@ tuplehash(PyTupleObject *v) } static Py_ssize_t -tuplelength(PyTupleObject *a) +tuple_length(PyObject *self) { + PyTupleObject *a = _PyTuple_CAST(self); return Py_SIZE(a); } static int -tuplecontains(PyTupleObject *a, PyObject *el) +tuple_contains(PyObject *self, PyObject *el) { - Py_ssize_t i; - int cmp; - - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i) + PyTupleObject *a = _PyTuple_CAST(self); + int cmp = 0; + for (Py_ssize_t i = 0; cmp == 0 && i < Py_SIZE(a); ++i) { cmp = PyObject_RichCompareBool(PyTuple_GET_ITEM(a, i), el, Py_EQ); + } return cmp; } static PyObject * -tupleitem(PyTupleObject *a, Py_ssize_t i) +tuple_item(PyObject *op, Py_ssize_t i) { + PyTupleObject *a = _PyTuple_CAST(op); if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, "tuple index out of range"); return NULL; @@ -390,6 +390,27 @@ _PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) return (PyObject *)tuple; } +PyObject * +_PyTuple_FromStackRefSteal(const _PyStackRef *src, Py_ssize_t n) +{ + if (n == 0) { + return tuple_get_empty(); + } + PyTupleObject *tuple = tuple_alloc(n); + if (tuple == NULL) { + for (Py_ssize_t i = 0; i < n; i++) { + PyStackRef_CLOSE(src[i]); + } + return NULL; + } + PyObject **dst = tuple->ob_item; + for (Py_ssize_t i = 0; i < n; i++) { + dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + } + _PyObject_GC_TRACK(tuple); + return (PyObject *)tuple; +} + PyObject * _PyTuple_FromArraySteal(PyObject *const *src, Py_ssize_t n) { @@ -413,7 +434,7 @@ _PyTuple_FromArraySteal(PyObject *const *src, Py_ssize_t n) } static PyObject * -tupleslice(PyTupleObject *a, Py_ssize_t ilow, +tuple_slice(PyTupleObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { if (ilow < 0) @@ -435,16 +456,13 @@ PyTuple_GetSlice(PyObject *op, Py_ssize_t i, Py_ssize_t j) PyErr_BadInternalCall(); return NULL; } - return tupleslice((PyTupleObject *)op, i, j); + return tuple_slice((PyTupleObject *)op, i, j); } static PyObject * -tupleconcat(PyTupleObject *a, PyObject *bb) +tuple_concat(PyObject *aa, PyObject *bb) { - Py_ssize_t size; - Py_ssize_t i; - PyObject **src, **dest; - PyTupleObject *np; + PyTupleObject *a = _PyTuple_CAST(aa); if (Py_SIZE(a) == 0 && PyTuple_CheckExact(bb)) { return Py_NewRef(bb); } @@ -460,34 +478,38 @@ tupleconcat(PyTupleObject *a, PyObject *bb) return Py_NewRef(a); } assert((size_t)Py_SIZE(a) + (size_t)Py_SIZE(b) < PY_SSIZE_T_MAX); - size = Py_SIZE(a) + Py_SIZE(b); + Py_ssize_t size = Py_SIZE(a) + Py_SIZE(b); if (size == 0) { return tuple_get_empty(); } - np = tuple_alloc(size); + PyTupleObject *np = tuple_alloc(size); if (np == NULL) { return NULL; } - src = a->ob_item; - dest = np->ob_item; - for (i = 0; i < Py_SIZE(a); i++) { + + PyObject **src = a->ob_item; + PyObject **dest = np->ob_item; + for (Py_ssize_t i = 0; i < Py_SIZE(a); i++) { PyObject *v = src[i]; dest[i] = Py_NewRef(v); } + src = b->ob_item; dest = np->ob_item + Py_SIZE(a); - for (i = 0; i < Py_SIZE(b); i++) { + for (Py_ssize_t i = 0; i < Py_SIZE(b); i++) { PyObject *v = src[i]; dest[i] = Py_NewRef(v); } + _PyObject_GC_TRACK(np); return (PyObject *)np; } static PyObject * -tuplerepeat(PyTupleObject *a, Py_ssize_t n) +tuple_repeat(PyObject *self, Py_ssize_t n) { + PyTupleObject *a = _PyTuple_CAST(self); const Py_ssize_t input_size = Py_SIZE(a); if (input_size == 0 || n == 1) { if (PyTuple_CheckExact(a)) { @@ -602,17 +624,17 @@ tuple_count(PyTupleObject *self, PyObject *value) } static int -tupletraverse(PyTupleObject *o, visitproc visit, void *arg) +tuple_traverse(PyObject *self, visitproc visit, void *arg) { - Py_ssize_t i; - - for (i = Py_SIZE(o); --i >= 0; ) + PyTupleObject *o = _PyTuple_CAST(self); + for (Py_ssize_t i = Py_SIZE(o); --i >= 0; ) { Py_VISIT(o->ob_item[i]); + } return 0; } static PyObject * -tuplerichcompare(PyObject *v, PyObject *w, int op) +tuple_richcompare(PyObject *v, PyObject *w, int op) { PyTupleObject *vt, *wt; Py_ssize_t i; @@ -751,26 +773,27 @@ tuple_subtype_new(PyTypeObject *type, PyObject *iterable) } static PySequenceMethods tuple_as_sequence = { - (lenfunc)tuplelength, /* sq_length */ - (binaryfunc)tupleconcat, /* sq_concat */ - (ssizeargfunc)tuplerepeat, /* sq_repeat */ - (ssizeargfunc)tupleitem, /* sq_item */ + tuple_length, /* sq_length */ + tuple_concat, /* sq_concat */ + tuple_repeat, /* sq_repeat */ + tuple_item, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)tuplecontains, /* sq_contains */ + tuple_contains, /* sq_contains */ }; static PyObject* -tuplesubscript(PyTupleObject* self, PyObject* item) +tuple_subscript(PyObject *op, PyObject* item) { + PyTupleObject *self = _PyTuple_CAST(op); if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) i += PyTuple_GET_SIZE(self); - return tupleitem(self, i); + return tuple_item(op, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, i; @@ -824,7 +847,7 @@ static PyObject * tuple___getnewargs___impl(PyTupleObject *self) /*[clinic end generated code: output=25e06e3ee56027e2 input=1aeb4b286a21639a]*/ { - return Py_BuildValue("(N)", tupleslice(self, 0, Py_SIZE(self))); + return Py_BuildValue("(N)", tuple_slice(self, 0, Py_SIZE(self))); } static PyMethodDef tuple_methods[] = { @@ -836,8 +859,8 @@ static PyMethodDef tuple_methods[] = { }; static PyMappingMethods tuple_as_mapping = { - (lenfunc)tuplelength, - (binaryfunc)tuplesubscript, + tuple_length, + tuple_subscript, 0 }; @@ -848,16 +871,16 @@ PyTypeObject PyTuple_Type = { "tuple", sizeof(PyTupleObject) - sizeof(PyObject *), sizeof(PyObject *), - (destructor)tupledealloc, /* tp_dealloc */ + tuple_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)tuplerepr, /* tp_repr */ + tuple_repr, /* tp_repr */ 0, /* tp_as_number */ &tuple_as_sequence, /* tp_as_sequence */ &tuple_as_mapping, /* tp_as_mapping */ - (hashfunc)tuplehash, /* tp_hash */ + tuple_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -867,9 +890,9 @@ PyTypeObject PyTuple_Type = { Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TUPLE_SUBCLASS | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_SEQUENCE, /* tp_flags */ tuple_new__doc__, /* tp_doc */ - (traverseproc)tupletraverse, /* tp_traverse */ + tuple_traverse, /* tp_traverse */ 0, /* tp_clear */ - tuplerichcompare, /* tp_richcompare */ + tuple_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ tuple_iter, /* tp_iter */ 0, /* tp_iternext */ @@ -946,7 +969,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) if (sv == NULL) { *pv = NULL; #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif PyObject_GC_Del(v); return -1; @@ -961,16 +984,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) return 0; } - -static void maybe_freelist_clear(_PyFreeListState *, int); - - -void -_PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization) -{ - maybe_freelist_clear(state, is_finalization); -} - /*********************** Tuple Iterator **************************/ @@ -990,8 +1003,9 @@ tupleiter_traverse(_PyTupleIterObject *it, visitproc visit, void *arg) } static PyObject * -tupleiter_next(_PyTupleIterObject *it) +tupleiter_next(PyObject *obj) { + _PyTupleIterObject *it = (_PyTupleIterObject *)obj; PyTupleObject *seq; PyObject *item; @@ -1092,7 +1106,7 @@ PyTypeObject PyTupleIter_Type = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)tupleiter_next, /* tp_iternext */ + tupleiter_next, /* tp_iternext */ tupleiter_methods, /* tp_methods */ 0, }; @@ -1120,102 +1134,29 @@ tuple_iter(PyObject *seq) * freelists * *************/ -#define STATE (state->tuples) -#define FREELIST_FINALIZED (STATE.numfree[0] < 0) - -static inline PyTupleObject * -maybe_freelist_pop(Py_ssize_t size) -{ -#ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); - if (size == 0) { - return NULL; - } - assert(size > 0); - if (size < PyTuple_MAXSAVESIZE) { - Py_ssize_t index = size - 1; - PyTupleObject *op = STATE.free_list[index]; - if (op != NULL) { - /* op is the head of a linked list, with the first item - pointing to the next node. Here we pop off the old head. */ - STATE.free_list[index] = (PyTupleObject *) op->ob_item[0]; - STATE.numfree[index]--; - /* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */ -#ifdef Py_TRACE_REFS - /* maybe_freelist_push() ensures these were already set. */ - // XXX Can we drop these? See commit 68055ce6fe01 (GvR, Dec 1998). - Py_SET_SIZE(op, size); - Py_SET_TYPE(op, &PyTuple_Type); -#endif - _Py_NewReference((PyObject *)op); - /* END inlined _PyObject_InitVar() */ - OBJECT_STAT_INC(from_freelist); - return op; - } - } -#endif - return NULL; -} - static inline int maybe_freelist_push(PyTupleObject *op) { -#ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); - if (Py_SIZE(op) == 0) { + if (!Py_IS_TYPE(op, &PyTuple_Type)) { return 0; } Py_ssize_t index = Py_SIZE(op) - 1; - if (index < PyTuple_NFREELISTS - && STATE.numfree[index] < PyTuple_MAXFREELIST - && STATE.numfree[index] >= 0 - && Py_IS_TYPE(op, &PyTuple_Type)) - { - /* op is the head of a linked list, with the first item - pointing to the next node. Here we set op as the new head. */ - op->ob_item[0] = (PyObject *) STATE.free_list[index]; - STATE.free_list[index] = op; - STATE.numfree[index]++; - OBJECT_STAT_INC(to_freelist); - return 1; + if (index < PyTuple_MAXSAVESIZE) { + return _Py_FREELIST_PUSH(tuples[index], op, Py_tuple_MAXFREELIST); } -#endif return 0; } -static void -maybe_freelist_clear(_PyFreeListState *state, int fini) -{ -#ifdef WITH_FREELISTS - for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) { - PyTupleObject *p = STATE.free_list[i]; - STATE.free_list[i] = NULL; - STATE.numfree[i] = fini ? -1 : 0; - while (p) { - PyTupleObject *q = p; - p = (PyTupleObject *)(p->ob_item[0]); - PyObject_GC_Del(q); - } - } -#endif -} - /* Print summary info about the state of the optimized allocator */ void _PyTuple_DebugMallocStats(FILE *out) { -#ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); - for (int i = 0; i < PyTuple_NFREELISTS; i++) { + for (int i = 0; i < PyTuple_MAXSAVESIZE; i++) { int len = i + 1; char buf[128]; PyOS_snprintf(buf, sizeof(buf), "free %d-sized PyTupleObject", len); - _PyDebugAllocatorStats(out, buf, STATE.numfree[i], + _PyDebugAllocatorStats(out, buf, _Py_FREELIST_SIZE(tuples[i]), _PyObject_VAR_SIZE(&PyTuple_Type, len)); } -#endif } - -#undef STATE -#undef FREELIST_FINALIZED diff --git a/Objects/typeobject.c b/Objects/typeobject.c index fcd54a2b040e9d..e87763a54d4ae8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,7 +6,8 @@ #include "pycore_code.h" // CO_FAST_FREE #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_frame.h" // _PyInterpreterFrame -#include "pycore_long.h" // _PyLong_IsNegative() +#include "pycore_lock.h" // _PySeqLock_* +#include "pycore_long.h" // _PyLong_IsNegative(), _PyLong_GetOne() #include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetDef() @@ -42,7 +43,8 @@ class object "PyObject *" "&PyBaseObject_Type" & ((1 << MCACHE_SIZE_EXP) - 1)) #define MCACHE_HASH_METHOD(type, name) \ - MCACHE_HASH((type)->tp_version_tag, ((Py_ssize_t)(name)) >> 3) + MCACHE_HASH(FT_ATOMIC_LOAD_UINT32_RELAXED((type)->tp_version_tag), \ + ((Py_ssize_t)(name)) >> 3) #define MCACHE_CACHEABLE_NAME(name) \ PyUnicode_CheckExact(name) && \ PyUnicode_IS_READY(name) && \ @@ -52,6 +54,36 @@ class object "PyObject *" "&PyBaseObject_Type" #define NEXT_VERSION_TAG(interp) \ (interp)->types.next_version_tag +#ifdef Py_GIL_DISABLED + +// There's a global lock for mutation of types. This avoids having to take +// additional locks while doing various subclass processing which may result +// in odd behaviors w.r.t. running with the GIL as the outer type lock could +// be released and reacquired during a subclass update if there's contention +// on the subclass lock. +#define TYPE_LOCK &PyInterpreterState_Get()->types.mutex +#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUT(TYPE_LOCK) +#define END_TYPE_LOCK() Py_END_CRITICAL_SECTION() + +#define BEGIN_TYPE_DICT_LOCK(d) \ + Py_BEGIN_CRITICAL_SECTION2_MUT(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) + +#define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2() + +#define ASSERT_TYPE_LOCK_HELD() \ + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) + +#else + +#define BEGIN_TYPE_LOCK() +#define END_TYPE_LOCK() +#define BEGIN_TYPE_DICT_LOCK(d) +#define END_TYPE_DICT_LOCK() +#define ASSERT_TYPE_LOCK_HELD() + +#endif + + typedef struct PySlot_Offset { short subslot_offset; short slot_offset; @@ -86,88 +118,227 @@ type_from_ref(PyObject *ref) /* helpers for for static builtin types */ +#ifndef NDEBUG static inline int -static_builtin_index_is_set(PyTypeObject *self) +managed_static_type_index_is_set(PyTypeObject *self) { return self->tp_subclasses != NULL; } +#endif static inline size_t -static_builtin_index_get(PyTypeObject *self) +managed_static_type_index_get(PyTypeObject *self) { - assert(static_builtin_index_is_set(self)); + assert(managed_static_type_index_is_set(self)); /* We store a 1-based index so 0 can mean "not initialized". */ return (size_t)self->tp_subclasses - 1; } static inline void -static_builtin_index_set(PyTypeObject *self, size_t index) +managed_static_type_index_set(PyTypeObject *self, size_t index) { - assert(index < _Py_MAX_STATIC_BUILTIN_TYPES); + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); /* We store a 1-based index so 0 can mean "not initialized". */ self->tp_subclasses = (PyObject *)(index + 1); } static inline void -static_builtin_index_clear(PyTypeObject *self) +managed_static_type_index_clear(PyTypeObject *self) { self->tp_subclasses = NULL; } -static inline static_builtin_state * -static_builtin_state_get(PyInterpreterState *interp, PyTypeObject *self) +static PyTypeObject * +static_ext_type_lookup(PyInterpreterState *interp, size_t index, + int64_t *p_interp_count) +{ + assert(interp->runtime == &_PyRuntime); + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + + size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + int64_t interp_count = + _PyRuntime.types.managed_static.types[full_index].interp_count; + assert((interp_count == 0) == + (_PyRuntime.types.managed_static.types[full_index].type == NULL)); + *p_interp_count = interp_count; + + PyTypeObject *type = interp->types.for_extensions.initialized[index].type; + if (type == NULL) { + return NULL; + } + assert(!interp->types.for_extensions.initialized[index].isbuiltin); + assert(type == _PyRuntime.types.managed_static.types[full_index].type); + assert(managed_static_type_index_is_set(type)); + return type; +} + +static managed_static_type_state * +managed_static_type_state_get(PyInterpreterState *interp, PyTypeObject *self) { - return &(interp->types.builtins[static_builtin_index_get(self)]); + // It's probably a builtin type. + size_t index = managed_static_type_index_get(self); + managed_static_type_state *state = + &(interp->types.builtins.initialized[index]); + if (state->type == self) { + return state; + } + if (index > _Py_MAX_MANAGED_STATIC_EXT_TYPES) { + return state; + } + return &(interp->types.for_extensions.initialized[index]); } /* For static types we store some state in an array on each interpreter. */ -static_builtin_state * +managed_static_type_state * _PyStaticType_GetState(PyInterpreterState *interp, PyTypeObject *self) { assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - return static_builtin_state_get(interp, self); + return managed_static_type_state_get(interp, self); } /* Set the type's per-interpreter state. */ static void -static_builtin_state_init(PyInterpreterState *interp, PyTypeObject *self) +managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int initial) { - if (!static_builtin_index_is_set(self)) { - static_builtin_index_set(self, interp->types.num_builtins_initialized); + assert(interp->runtime == &_PyRuntime); + + size_t index; + if (initial) { + assert(!managed_static_type_index_is_set(self)); + if (isbuiltin) { + index = interp->types.builtins.num_initialized; + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + } + else { + PyMutex_Lock(&interp->types.mutex); + index = interp->types.for_extensions.next_index; + interp->types.for_extensions.next_index++; + PyMutex_Unlock(&interp->types.mutex); + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + } + managed_static_type_index_set(self, index); + } + else { + index = managed_static_type_index_get(self); + if (isbuiltin) { + assert(index == interp->types.builtins.num_initialized); + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + } + else { + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + } + } + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + + assert((initial == 1) == + (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); + (void)_Py_atomic_add_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count, 1); + + if (initial) { + assert(_PyRuntime.types.managed_static.types[full_index].type == NULL); + _PyRuntime.types.managed_static.types[full_index].type = self; + } + else { + assert(_PyRuntime.types.managed_static.types[full_index].type == self); } - static_builtin_state *state = static_builtin_state_get(interp, self); - /* It should only be called once for each builtin type. */ + managed_static_type_state *state = isbuiltin + ? &(interp->types.builtins.initialized[index]) + : &(interp->types.for_extensions.initialized[index]); + + /* It should only be called once for each builtin type per interpreter. */ assert(state->type == NULL); state->type = self; + state->isbuiltin = isbuiltin; /* state->tp_subclasses is left NULL until init_subclasses() sets it. */ /* state->tp_weaklist is left NULL until insert_head() or insert_after() (in weakrefobject.c) sets it. */ - interp->types.num_builtins_initialized++; + if (isbuiltin) { + interp->types.builtins.num_initialized++; + } + else { + interp->types.for_extensions.num_initialized++; + } } /* Reset the type's per-interpreter state. - This basically undoes what static_builtin_state_init() did. */ + This basically undoes what managed_static_type_state_init() did. */ static void -static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) +managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int final) { - static_builtin_state *state = static_builtin_state_get(interp, self); + size_t index = managed_static_type_index_get(self); + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + + managed_static_type_state *state = isbuiltin + ? &(interp->types.builtins.initialized[index]) + : &(interp->types.for_extensions.initialized[index]); + assert(state != NULL); + + assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); + assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); assert(state->type != NULL); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. - if (_Py_IsMainInterpreter(interp)) { - static_builtin_index_clear(self); + (void)_Py_atomic_add_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count, -1); + if (final) { + assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); + _PyRuntime.types.managed_static.types[full_index].type = NULL; + + managed_static_type_index_clear(self); + } + + if (isbuiltin) { + assert(interp->types.builtins.num_initialized > 0); + interp->types.builtins.num_initialized--; + } + else { + PyMutex_Lock(&interp->types.mutex); + assert(interp->types.for_extensions.num_initialized > 0); + interp->types.for_extensions.num_initialized--; + if (interp->types.for_extensions.num_initialized == 0) { + interp->types.for_extensions.next_index = 0; + } + PyMutex_Unlock(&interp->types.mutex); + } +} + + + +PyObject * +_PyStaticType_GetBuiltins(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + Py_ssize_t count = (Py_ssize_t)interp->types.builtins.num_initialized; + assert(count <= _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + + PyObject *results = PyList_New(count); + if (results == NULL) { + return NULL; + } + for (Py_ssize_t i = 0; i < count; i++) { + PyTypeObject *cls = interp->types.builtins.initialized[i].type; + assert(cls != NULL); + assert(interp->types.builtins.initialized[i].isbuiltin); + PyList_SET_ITEM(results, i, Py_NewRef((PyObject *)cls)); } - assert(interp->types.num_builtins_initialized > 0); - interp->types.num_builtins_initialized--; + return results; } -// Also see _PyStaticType_InitBuiltin() and _PyStaticType_Dealloc(). + +// Also see _PyStaticType_InitBuiltin() and _PyStaticType_FiniBuiltin(). /* end static builtin helpers */ @@ -177,7 +348,7 @@ start_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); assert(!state->readying); state->readying = 1; @@ -192,7 +363,7 @@ stop_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); assert(state->readying); state->readying = 0; @@ -207,7 +378,7 @@ is_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); return state->readying; } @@ -222,7 +393,7 @@ lookup_tp_dict(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_dict; } @@ -248,7 +419,7 @@ set_tp_dict(PyTypeObject *self, PyObject *dict) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); state->tp_dict = dict; return; @@ -261,7 +432,7 @@ clear_tp_dict(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); Py_CLEAR(state->tp_dict); return; @@ -279,18 +450,24 @@ lookup_tp_bases(PyTypeObject *self) PyObject * _PyType_GetBases(PyTypeObject *self) { - /* It returns a borrowed reference. */ - return lookup_tp_bases(self); + PyObject *res; + + BEGIN_TYPE_LOCK(); + res = lookup_tp_bases(self); + Py_INCREF(res); + END_TYPE_LOCK(); + + return res; } static inline void -set_tp_bases(PyTypeObject *self, PyObject *bases) +set_tp_bases(PyTypeObject *self, PyObject *bases, int initial) { assert(PyTuple_CheckExact(bases)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { // XXX tp_bases can probably be statically allocated for each // static builtin type. - assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(initial); assert(self->tp_bases == NULL); if (PyTuple_GET_SIZE(bases) == 0) { assert(self->tp_base == NULL); @@ -307,10 +484,10 @@ set_tp_bases(PyTypeObject *self, PyObject *bases) } static inline void -clear_tp_bases(PyTypeObject *self) +clear_tp_bases(PyTypeObject *self, int final) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (final) { if (self->tp_bases != NULL) { if (PyTuple_GET_SIZE(self->tp_bases) == 0) { Py_CLEAR(self->tp_bases); @@ -330,24 +507,40 @@ clear_tp_bases(PyTypeObject *self) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { + ASSERT_TYPE_LOCK_HELD(); return self->tp_mro; } PyObject * _PyType_GetMRO(PyTypeObject *self) { - /* It returns a borrowed reference. */ - return lookup_tp_mro(self); +#ifdef Py_GIL_DISABLED + PyObject *mro = _Py_atomic_load_ptr_relaxed(&self->tp_mro); + if (mro == NULL) { + return NULL; + } + if (_Py_TryIncrefCompare(&self->tp_mro, mro)) { + return mro; + } + + BEGIN_TYPE_LOCK(); + mro = lookup_tp_mro(self); + Py_XINCREF(mro); + END_TYPE_LOCK(); + return mro; +#else + return Py_XNewRef(lookup_tp_mro(self)); +#endif } static inline void -set_tp_mro(PyTypeObject *self, PyObject *mro) +set_tp_mro(PyTypeObject *self, PyObject *mro, int initial) { assert(PyTuple_CheckExact(mro)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { // XXX tp_mro can probably be statically allocated for each // static builtin type. - assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(initial); assert(self->tp_mro == NULL); /* Other checks are done via set_tp_bases. */ _Py_SetImmortal(mro); @@ -356,10 +549,10 @@ set_tp_mro(PyTypeObject *self, PyObject *mro) } static inline void -clear_tp_mro(PyTypeObject *self) +clear_tp_mro(PyTypeObject *self, int final) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (final) { if (self->tp_mro != NULL) { if (PyTuple_GET_SIZE(self->tp_mro) == 0) { Py_CLEAR(self->tp_mro); @@ -385,7 +578,7 @@ init_tp_subclasses(PyTypeObject *self) } if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); state->tp_subclasses = subclasses; return subclasses; } @@ -401,7 +594,7 @@ clear_tp_subclasses(PyTypeObject *self) has no subclass. */ if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); Py_CLEAR(state->tp_subclasses); return; } @@ -413,7 +606,7 @@ lookup_tp_subclasses(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_subclasses; } @@ -646,9 +839,15 @@ type_cache_clear(struct type_cache *cache, PyObject *value) { for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) { struct type_cache_entry *entry = &cache->hashtable[i]; +#ifdef Py_GIL_DISABLED + _PySeqLock_LockWrite(&entry->sequence); +#endif entry->version = 0; Py_XSETREF(entry->name, _Py_XNewRef(value)); entry->value = NULL; +#ifdef Py_GIL_DISABLED + _PySeqLock_UnlockWrite(&entry->sequence); +#endif } } @@ -662,7 +861,7 @@ _PyType_InitCache(PyInterpreterState *interp) assert(entry->name == NULL); entry->version = 0; - // Set to None so _PyType_Lookup() can use Py_SETREF(), + // Set to None so _PyType_LookupRef() can use Py_SETREF(), // rather than using slower Py_XSETREF(). entry->name = Py_None; entry->value = NULL; @@ -674,7 +873,7 @@ static unsigned int _PyType_ClearCache(PyInterpreterState *interp) { struct type_cache *cache = &interp->types.type_cache; - // Set to None, rather than NULL, so _PyType_Lookup() can + // Set to None, rather than NULL, so _PyType_LookupRef() can // use Py_SETREF() rather than using slower Py_XSETREF(). type_cache_clear(cache, Py_None); @@ -696,10 +895,14 @@ _PyTypes_Fini(PyInterpreterState *interp) struct type_cache *cache = &interp->types.type_cache; type_cache_clear(cache, NULL); - assert(interp->types.num_builtins_initialized == 0); - // All the static builtin types should have been finalized already. - for (size_t i = 0; i < _Py_MAX_STATIC_BUILTIN_TYPES; i++) { - assert(interp->types.builtins[i].type == NULL); + // All the managed static types should have been finalized already. + assert(interp->types.for_extensions.num_initialized == 0); + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + assert(interp->types.for_extensions.initialized[i].type == NULL); + } + assert(interp->types.builtins.num_initialized == 0); + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + assert(interp->types.builtins.initialized[i].type == NULL); } } @@ -709,7 +912,8 @@ PyType_AddWatcher(PyType_WatchCallback callback) { PyInterpreterState *interp = _PyInterpreterState_GET(); - for (int i = 0; i < TYPE_MAX_WATCHERS; i++) { + // start at 1, 0 is reserved for cpython optimizer + for (int i = 1; i < TYPE_MAX_WATCHERS; i++) { if (!interp->type_watchers[i]) { interp->type_watchers[i] = callback; return i; @@ -760,8 +964,10 @@ PyType_Watch(int watcher_id, PyObject* obj) return -1; } // ensure we will get a callback on the next modification + BEGIN_TYPE_LOCK(); assign_version_tag(interp, type); type->tp_watched |= (1 << watcher_id); + END_TYPE_LOCK(); return 0; } @@ -781,8 +987,40 @@ PyType_Unwatch(int watcher_id, PyObject* obj) return 0; } -void -PyType_Modified(PyTypeObject *type) +static void +set_version_unlocked(PyTypeObject *tp, unsigned int version) +{ + ASSERT_TYPE_LOCK_HELD(); +#ifndef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + // lookup the old version and set to null + if (tp->tp_version_tag != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (tp->tp_version_tag % TYPE_VERSION_CACHE_SIZE); + *slot = NULL; + } + if (version) { + tp->tp_versions_used++; + } +#else + if (version) { + _Py_atomic_add_uint16(&tp->tp_versions_used, 1); + } +#endif + FT_ATOMIC_STORE_UINT32_RELAXED(tp->tp_version_tag, version); +#ifndef Py_GIL_DISABLED + if (version != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + *slot = tp; + } +#endif +} + +static void +type_modified_unlocked(PyTypeObject *type) { /* Invalidate any cached data for the specified type and all subclasses. This function is called after the base @@ -790,18 +1028,20 @@ PyType_Modified(PyTypeObject *type) Invariants: - - before Py_TPFLAGS_VALID_VERSION_TAG can be set on a type, + - before tp_version_tag can be set on a type, it must first be set on all super types. - This function clears the Py_TPFLAGS_VALID_VERSION_TAG of a + This function clears the tp_version_tag of a type (so it must first clear it on all subclasses). The - tp_version_tag value is meaningless unless this flag is set. + tp_version_tag value is meaningless when equal to zero. We don't assign new version tags eagerly, but only as needed. */ - if (!_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + if (type->tp_version_tag == 0) { return; } + // Cannot modify static builtin types. + assert((type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) == 0); PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses != NULL) { @@ -814,7 +1054,7 @@ PyType_Modified(PyTypeObject *type) if (subclass == NULL) { continue; } - PyType_Modified(subclass); + type_modified_unlocked(subclass); Py_DECREF(subclass); } } @@ -839,8 +1079,7 @@ PyType_Modified(PyTypeObject *type) } } - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + set_version_unlocked(type, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -848,6 +1087,22 @@ PyType_Modified(PyTypeObject *type) } } +void +PyType_Modified(PyTypeObject *type) +{ + // Quick check without the lock held + if (type->tp_version_tag == 0) { + return; + } + + BEGIN_TYPE_LOCK(); + type_modified_unlocked(type); + END_TYPE_LOCK(); +} + +static int +is_subtype_with_mro(PyObject *a_mro, PyTypeObject *a, PyTypeObject *b); + static void type_mro_modified(PyTypeObject *type, PyObject *bases) { /* @@ -866,6 +1121,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { int custom = !Py_IS_TYPE(type, &PyType_Type); int unbound; + ASSERT_TYPE_LOCK_HELD(); if (custom) { PyObject *mro_meth, *type_mro_meth; mro_meth = lookup_maybe_method( @@ -891,7 +1147,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { PyObject *b = PyTuple_GET_ITEM(bases, i); PyTypeObject *cls = _PyType_CAST(b); - if (!PyType_IsSubtype(type, cls)) { + if (!is_subtype_with_mro(lookup_tp_mro(type), type, cls)) { goto clear; } } @@ -899,8 +1155,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + set_version_unlocked(type, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -908,17 +1163,61 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { } } +/* +The Tier 2 interpreter requires looking up the type object by the type version, so it can install +watchers to understand when they change. + +So we add a global cache from type version to borrowed references of type objects. + +This is similar to func_version_cache. +*/ + +void +_PyType_SetVersion(PyTypeObject *tp, unsigned int version) +{ + + BEGIN_TYPE_LOCK(); + set_version_unlocked(tp, version); + END_TYPE_LOCK(); +} + +PyTypeObject * +_PyType_LookupByVersion(unsigned int version) +{ +#ifdef Py_GIL_DISABLED + return NULL; +#else + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + if (*slot && (*slot)->tp_version_tag == version) { + return *slot; + } + return NULL; +#endif +} + +unsigned int +_PyType_GetVersionForCurrentState(PyTypeObject *tp) +{ + return tp->tp_version_tag; +} + + + #define MAX_VERSIONS_PER_CLASS 1000 static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { - /* Ensure that the tp_version_tag is valid and set - Py_TPFLAGS_VALID_VERSION_TAG. To respect the invariant, this - must first be done on all super classes. Return 0 if this - cannot be done, 1 if Py_TPFLAGS_VALID_VERSION_TAG. + ASSERT_TYPE_LOCK_HELD(); + + /* Ensure that the tp_version_tag is valid. + * To respect the invariant, this must first be done on all super classes. + * Return 0 if this cannot be done, 1 if tp_version_tag is set. */ - if (_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + if (type->tp_version_tag != 0) { return 1; } if (!_PyType_HasFeature(type, Py_TPFLAGS_READY)) { @@ -927,14 +1226,22 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) if (type->tp_versions_used >= MAX_VERSIONS_PER_CLASS) { return 0; } - type->tp_versions_used++; + + PyObject *bases = lookup_tp_bases(type); + Py_ssize_t n = PyTuple_GET_SIZE(bases); + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *b = PyTuple_GET_ITEM(bases, i); + if (!assign_version_tag(interp, _PyType_CAST(b))) { + return 0; + } + } if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + set_version_unlocked(type, NEXT_GLOBAL_VERSION_TAG++); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -943,25 +1250,20 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_VERSION_TAG(interp)++; + set_version_unlocked(type, NEXT_VERSION_TAG(interp)++); assert (type->tp_version_tag != 0); } - - PyObject *bases = lookup_tp_bases(type); - Py_ssize_t n = PyTuple_GET_SIZE(bases); - for (Py_ssize_t i = 0; i < n; i++) { - PyObject *b = PyTuple_GET_ITEM(bases, i); - if (!assign_version_tag(interp, _PyType_CAST(b))) - return 0; - } - type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; return 1; } int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return assign_version_tag(interp, type); + int assigned; + BEGIN_TYPE_LOCK(); + assigned = assign_version_tag(interp, type); + END_TYPE_LOCK(); + return assigned; } @@ -1092,10 +1394,9 @@ type_set_qualname(PyTypeObject *type, PyObject *value, void *context) } static PyObject * -type_module(PyTypeObject *type, void *context) +type_module(PyTypeObject *type) { PyObject *mod; - if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { PyObject *dict = lookup_tp_dict(type); if (PyDict_GetItemRef(dict, &_Py_ID(__module__), &mod) == 0) { @@ -1107,8 +1408,10 @@ type_module(PyTypeObject *type, void *context) if (s != NULL) { mod = PyUnicode_FromStringAndSize( type->tp_name, (Py_ssize_t)(s - type->tp_name)); - if (mod != NULL) - PyUnicode_InternInPlace(&mod); + if (mod != NULL) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &mod); + } } else { mod = &_Py_ID(builtins); @@ -1117,6 +1420,12 @@ type_module(PyTypeObject *type, void *context) return mod; } +static PyObject * +type_get_module(PyTypeObject *type, void *context) +{ + return type_module(type); +} + static int type_set_module(PyTypeObject *type, PyObject *value, void *context) { @@ -1126,9 +1435,53 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); PyObject *dict = lookup_tp_dict(type); + if (PyDict_Pop(dict, &_Py_ID(__firstlineno__), NULL) < 0) { + return -1; + } return PyDict_SetItem(dict, &_Py_ID(__module__), value); } + +PyObject * +_PyType_GetFullyQualifiedName(PyTypeObject *type, char sep) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + return PyUnicode_FromString(type->tp_name); + } + + PyObject *qualname = type_qualname(type, NULL); + if (qualname == NULL) { + return NULL; + } + + PyObject *module = type_module(type); + if (module == NULL) { + Py_DECREF(qualname); + return NULL; + } + + PyObject *result; + if (PyUnicode_Check(module) + && !_PyUnicode_Equal(module, &_Py_ID(builtins)) + && !_PyUnicode_Equal(module, &_Py_ID(__main__))) + { + result = PyUnicode_FromFormat("%U%c%U", module, sep, qualname); + } + else { + result = Py_NewRef(qualname); + } + Py_DECREF(module); + Py_DECREF(qualname); + return result; +} + +PyObject * +PyType_GetFullyQualifiedName(PyTypeObject *type) +{ + return _PyType_GetFullyQualifiedName(type, '.'); +} + + static PyObject * type_abstractmethods(PyTypeObject *type, void *context) { @@ -1164,40 +1517,49 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) } else { abstract = 0; - res = PyDict_DelItem(dict, &_Py_ID(__abstractmethods__)); - if (res && PyErr_ExceptionMatches(PyExc_KeyError)) { + res = PyDict_Pop(dict, &_Py_ID(__abstractmethods__), NULL); + if (res == 0) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); return -1; } } - if (res == 0) { - PyType_Modified(type); - if (abstract) - type->tp_flags |= Py_TPFLAGS_IS_ABSTRACT; - else - type->tp_flags &= ~Py_TPFLAGS_IS_ABSTRACT; + if (res < 0) { + return -1; } - return res; + + PyType_Modified(type); + if (abstract) + type->tp_flags |= Py_TPFLAGS_IS_ABSTRACT; + else + type->tp_flags &= ~Py_TPFLAGS_IS_ABSTRACT; + return 0; } static PyObject * type_get_bases(PyTypeObject *type, void *context) { - PyObject *bases = lookup_tp_bases(type); + PyObject *bases = _PyType_GetBases(type); if (bases == NULL) { Py_RETURN_NONE; } - return Py_NewRef(bases); + return bases; } static PyObject * type_get_mro(PyTypeObject *type, void *context) { - PyObject *mro = lookup_tp_mro(type); + PyObject *mro; + + BEGIN_TYPE_LOCK(); + mro = lookup_tp_mro(type); if (mro == NULL) { - Py_RETURN_NONE; + mro = Py_None; + } else { + Py_INCREF(mro); } - return Py_NewRef(mro); + + END_TYPE_LOCK(); + return mro; } static PyTypeObject *best_base(PyObject *); @@ -1219,6 +1581,8 @@ static int recurse_down_subclasses(PyTypeObject *type, PyObject *name, static int mro_hierarchy(PyTypeObject *type, PyObject *temp) { + ASSERT_TYPE_LOCK_HELD(); + PyObject *old_mro; int res = mro_internal(type, &old_mro); if (res <= 0) { @@ -1244,7 +1608,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_XDECREF(tuple); if (res < 0) { - set_tp_mro(type, old_mro); + set_tp_mro(type, old_mro, 0); Py_DECREF(new_mro); return -1; } @@ -1282,7 +1646,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) } static int -type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) +type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) { // Check arguments if (!check_set_special_type_attr(type, new_bases, "__bases__")) { @@ -1313,7 +1677,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) } PyTypeObject *base = (PyTypeObject*)ob; - if (PyType_IsSubtype(base, type) || + if (is_subtype_with_mro(lookup_tp_mro(base), base, type) || /* In case of reentering here again through a custom mro() the above check is not enough since it relies on base->tp_mro which would gonna be updated inside @@ -1345,7 +1709,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; - set_tp_bases(type, Py_NewRef(new_bases)); + set_tp_bases(type, Py_NewRef(new_bases), 0); type->tp_base = (PyTypeObject *)Py_NewRef(new_base); PyObject *temp = PyList_New(0); @@ -1393,7 +1757,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) "", 2, 3, &cls, &new_mro, &old_mro); /* Do not rollback if cls has a newer version of MRO. */ if (lookup_tp_mro(cls) == new_mro) { - set_tp_mro(cls, Py_XNewRef(old_mro)); + set_tp_mro(cls, Py_XNewRef(old_mro), 0); Py_DECREF(new_mro); } } @@ -1403,7 +1767,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) if (lookup_tp_bases(type) == new_bases) { assert(type->tp_base == new_base); - set_tp_bases(type, old_bases); + set_tp_bases(type, old_bases, 0); type->tp_base = old_base; Py_DECREF(new_bases); @@ -1418,6 +1782,16 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) return -1; } +static int +type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) +{ + int res; + BEGIN_TYPE_LOCK(); + res = type_set_bases_unlocked(type, new_bases, context); + END_TYPE_LOCK(); + return res; +} + static PyObject * type_dict(PyTypeObject *type, void *context) { @@ -1465,76 +1839,188 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context) } static PyObject * -type_get_annotations(PyTypeObject *type, void *context) +type_get_annotate(PyTypeObject *type, void *Py_UNUSED(ignored)) { if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotations__'", type->tp_name); + PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotate__'", type->tp_name); return NULL; } - PyObject *annotations; - PyObject *dict = lookup_tp_dict(type); - if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) { + PyObject *annotate; + PyObject *dict = PyType_GetDict(type); + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) < 0) { + Py_DECREF(dict); return NULL; } - if (annotations) { - descrgetfunc get = Py_TYPE(annotations)->tp_descr_get; + if (annotate) { + descrgetfunc get = Py_TYPE(annotate)->tp_descr_get; if (get) { - Py_SETREF(annotations, get(annotations, NULL, (PyObject *)type)); + Py_SETREF(annotate, get(annotate, NULL, (PyObject *)type)); } } else { - annotations = PyDict_New(); - if (annotations) { - int result = PyDict_SetItem( - dict, &_Py_ID(__annotations__), annotations); - if (result) { - Py_CLEAR(annotations); - } else { - PyType_Modified(type); - } + annotate = Py_None; + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate); + if (result < 0) { + Py_DECREF(dict); + return NULL; } } - return annotations; + Py_DECREF(dict); + return annotate; } static int -type_set_annotations(PyTypeObject *type, PyObject *value, void *context) +type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) { + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } if (_PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)) { PyErr_Format(PyExc_TypeError, - "cannot set '__annotations__' attribute of immutable type '%s'", + "cannot set '__annotate__' attribute of immutable type '%s'", type->tp_name); return -1; } - int result; - PyObject *dict = lookup_tp_dict(type); - if (value != NULL) { - /* set */ - result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); - } else { - /* delete */ - result = PyDict_DelItem(dict, &_Py_ID(__annotations__)); - if (result < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_SetString(PyExc_AttributeError, "__annotations__"); - } + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + return -1; } - if (result == 0) { - PyType_Modified(type); + PyObject *dict = PyType_GetDict(type); + assert(PyDict_Check(dict)); + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), value); + if (result < 0) { + Py_DECREF(dict); + return -1; } - return result; + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + PyType_Modified(type); + return -1; + } + } + Py_DECREF(dict); + PyType_Modified(type); + return 0; } static PyObject * -type_get_type_params(PyTypeObject *type, void *context) +type_get_annotations(PyTypeObject *type, void *context) { - PyObject *params; - if (PyDict_GetItemRef(lookup_tp_dict(type), &_Py_ID(__type_params__), ¶ms) == 0) { - return PyTuple_New(0); + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotations__'", type->tp_name); + return NULL; } - return params; + + PyObject *annotations; + PyObject *dict = PyType_GetDict(type); + if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) { + Py_DECREF(dict); + return NULL; + } + if (annotations) { + descrgetfunc get = Py_TYPE(annotations)->tp_descr_get; + if (get) { + Py_SETREF(annotations, get(annotations, NULL, (PyObject *)type)); + } + } + else { + PyObject *annotate = type_get_annotate(type, NULL); + if (annotate == NULL) { + Py_DECREF(dict); + return NULL; + } + if (PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + Py_DECREF(dict); + Py_DECREF(annotate); + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotations); + Py_DECREF(annotate); + Py_DECREF(dict); + return NULL; + } + } + else { + annotations = PyDict_New(); + } + Py_DECREF(annotate); + if (annotations) { + int result = PyDict_SetItem( + dict, &_Py_ID(__annotations__), annotations); + if (result) { + Py_CLEAR(annotations); + } else { + PyType_Modified(type); + } + } + } + Py_DECREF(dict); + return annotations; +} + +static int +type_set_annotations(PyTypeObject *type, PyObject *value, void *context) +{ + if (_PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)) { + PyErr_Format(PyExc_TypeError, + "cannot set '__annotations__' attribute of immutable type '%s'", + type->tp_name); + return -1; + } + + int result; + PyObject *dict = PyType_GetDict(type); + if (value != NULL) { + /* set */ + result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); + } else { + /* delete */ + result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + if (result == 0) { + PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); + return -1; + } + } + if (result < 0) { + Py_DECREF(dict); + return -1; + } + else if (result == 0) { + if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + PyType_Modified(type); + Py_DECREF(dict); + return -1; + } + } + PyType_Modified(type); + Py_DECREF(dict); + return 0; +} + +static PyObject * +type_get_type_params(PyTypeObject *type, void *context) +{ + if (type == &PyType_Type) { + return PyTuple_New(0); + } + + PyObject *params; + if (PyDict_GetItemRef(lookup_tp_dict(type), &_Py_ID(__type_params__), ¶ms) == 0) { + return PyTuple_New(0); + } + return params; } static int @@ -1592,13 +2078,14 @@ static PyGetSetDef type_getsets[] = { {"__qualname__", (getter)type_qualname, (setter)type_set_qualname, NULL}, {"__bases__", (getter)type_get_bases, (setter)type_set_bases, NULL}, {"__mro__", (getter)type_get_mro, NULL, NULL}, - {"__module__", (getter)type_module, (setter)type_set_module, NULL}, + {"__module__", (getter)type_get_module, (setter)type_set_module, NULL}, {"__abstractmethods__", (getter)type_abstractmethods, (setter)type_set_abstractmethods, NULL}, {"__dict__", (getter)type_dict, NULL, NULL}, {"__doc__", (getter)type_get_doc, (setter)type_set_doc, NULL}, {"__text_signature__", (getter)type_get_text_signature, NULL, NULL}, {"__annotations__", (getter)type_get_annotations, (setter)type_set_annotations, NULL}, + {"__annotate__", (getter)type_get_annotate, (setter)type_set_annotate, NULL}, {"__type_params__", (getter)type_get_type_params, (setter)type_set_type_params, NULL}, {0} }; @@ -1613,28 +2100,31 @@ type_repr(PyObject *self) return PyUnicode_FromFormat("", type); } - PyObject *mod, *name, *rtn; - - mod = type_module(type, NULL); - if (mod == NULL) + PyObject *mod = type_module(type); + if (mod == NULL) { PyErr_Clear(); + } else if (!PyUnicode_Check(mod)) { - Py_SETREF(mod, NULL); + Py_CLEAR(mod); } - name = type_qualname(type, NULL); + + PyObject *name = type_qualname(type, NULL); if (name == NULL) { Py_XDECREF(mod); return NULL; } - if (mod != NULL && !_PyUnicode_Equal(mod, &_Py_ID(builtins))) - rtn = PyUnicode_FromFormat("", mod, name); - else - rtn = PyUnicode_FromFormat("", type->tp_name); - + PyObject *result; + if (mod != NULL && !_PyUnicode_Equal(mod, &_Py_ID(builtins))) { + result = PyUnicode_FromFormat("", mod, name); + } + else { + result = PyUnicode_FromFormat("", type->tp_name); + } Py_XDECREF(mod); Py_DECREF(name); - return rtn; + + return result; } static PyObject * @@ -1706,7 +2196,7 @@ type_call(PyObject *self, PyObject *args, PyObject *kwds) PyObject * _PyType_NewManagedObject(PyTypeObject *type) { - assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(type->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(_PyType_IS_GC(type)); assert(type->tp_new == PyBaseObject_Type.tp_new); assert(type->tp_alloc == PyType_GenericAlloc); @@ -1715,11 +2205,6 @@ _PyType_NewManagedObject(PyTypeObject *type) if (obj == NULL) { return PyErr_NoMemory(); } - _PyObject_DictOrValuesPointer(obj)->dict = NULL; - if (_PyObject_InitInlineValues(obj, type)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -1733,9 +2218,13 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) * flag to indicate when that is safe) it does not seem worth the memory * savings. An example type that doesn't need the +1 is a subclass of * tuple. See GH-100659 and GH-81381. */ - const size_t size = _PyObject_VAR_SIZE(type, nitems+1); + size_t size = _PyObject_VAR_SIZE(type, nitems+1); const size_t presize = _PyType_PreHeaderSize(type); + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + assert(type->tp_itemsize == 0); + size += _PyInlineValuesSize(type); + } char *alloc = _PyObject_MallocWithType(type, size + presize); if (alloc == NULL) { return PyErr_NoMemory(); @@ -1744,6 +2233,8 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) if (presize) { ((PyObject **)alloc)[0] = NULL; ((PyObject **)alloc)[1] = NULL; + } + if (PyType_IS_GC(type)) { _PyObject_GC_Link(obj); } memset(obj, '\0', size); @@ -1754,6 +2245,9 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) else { _PyObject_InitVar((PyVarObject *)obj, type, nitems); } + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(obj, type); + } return obj; } @@ -1903,6 +2397,10 @@ subtype_clear(PyObject *self) if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { PyObject_ClearManagedDict(self); } + else { + assert((base->tp_flags & Py_TPFLAGS_INLINE_VALUES) == + (type->tp_flags & Py_TPFLAGS_INLINE_VALUES)); + } } else if (type->tp_dictoffset != base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -1971,7 +2469,7 @@ subtype_dealloc(PyObject *self) reference counting. Only decref if the base type is not already a heap allocated type. Otherwise, basedealloc should have decref'd it already */ if (type_needs_decref) { - Py_DECREF(type); + _Py_DECREF_TYPE(type); } /* Done */ @@ -2030,15 +2528,7 @@ subtype_dealloc(PyObject *self) finalizers since they might rely on part of the object being finalized that has already been destroyed. */ if (type->tp_weaklistoffset && !base->tp_weaklistoffset) { - /* Modeled after GET_WEAKREFS_LISTPTR(). - - This is never triggered for static types so we can avoid the - (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). */ - PyWeakReference **list = \ - _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(self); - while (*list) { - _PyWeakref_ClearRef(*list); - } + _PyWeakref_ClearWeakRefsNoCallbacks(self); } } @@ -2053,14 +2543,7 @@ subtype_dealloc(PyObject *self) /* If we added a dict, DECREF it, or free inline values. */ if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(self); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - _PyObject_FreeInstanceAttributes(self); - } - else { - Py_XDECREF(_PyDictOrValues_GetDict(*dorv_ptr)); - } - dorv_ptr->values = NULL; + PyObject_ClearManagedDict(self); } else if (type->tp_dictoffset && !base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -2096,7 +2579,7 @@ subtype_dealloc(PyObject *self) reference counting. Only decref if the base type is not already a heap allocated type. Otherwise, basedealloc should have decref'd it already */ if (type_needs_decref) { - Py_DECREF(type); + _Py_DECREF_TYPE(type); } endlabel: @@ -2156,27 +2639,35 @@ type_is_subtype_base_chain(PyTypeObject *a, PyTypeObject *b) return (b == &PyBaseObject_Type); } -int -PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) +static int +is_subtype_with_mro(PyObject *a_mro, PyTypeObject *a, PyTypeObject *b) { - PyObject *mro; - - mro = lookup_tp_mro(a); - if (mro != NULL) { + int res; + if (a_mro != NULL) { /* Deal with multiple inheritance without recursion by walking the MRO tuple */ Py_ssize_t i, n; - assert(PyTuple_Check(mro)); - n = PyTuple_GET_SIZE(mro); + assert(PyTuple_Check(a_mro)); + n = PyTuple_GET_SIZE(a_mro); + res = 0; for (i = 0; i < n; i++) { - if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) - return 1; + if (PyTuple_GET_ITEM(a_mro, i) == (PyObject *)b) { + res = 1; + break; + } } - return 0; } - else + else { /* a is not completely initialized yet; follow tp_base */ - return type_is_subtype_base_chain(a, b); + res = type_is_subtype_base_chain(a, b); + } + return res; +} + +int +PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) +{ + return is_subtype_with_mro(a->tp_mro, a, b); } /* Routines to do a method lookup in the type without looking in the @@ -2186,7 +2677,7 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) Variants: - _PyObject_LookupSpecial() returns NULL without raising an exception - when the _PyType_Lookup() call fails; + when the _PyType_LookupRef() call fails; - lookup_maybe_method() and lookup_method() are internal routines similar to _PyObject_LookupSpecial(), but can return unbound PyFunction @@ -2199,30 +2690,48 @@ _PyObject_LookupSpecial(PyObject *self, PyObject *attr) { PyObject *res; - res = _PyType_Lookup(Py_TYPE(self), attr); + res = _PyType_LookupRef(Py_TYPE(self), attr); if (res != NULL) { descrgetfunc f; - if ((f = Py_TYPE(res)->tp_descr_get) == NULL) - Py_INCREF(res); - else - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if ((f = Py_TYPE(res)->tp_descr_get) != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); + } } return res; } +/* Steals a reference to self */ PyObject * -_PyObject_LookupSpecialId(PyObject *self, _Py_Identifier *attrid) +_PyObject_LookupSpecialMethod(PyObject *self, PyObject *attr, PyObject **self_or_null) { - PyObject *attr = _PyUnicode_FromId(attrid); /* borrowed */ - if (attr == NULL) + PyObject *res; + + res = _PyType_LookupRef(Py_TYPE(self), attr); + if (res == NULL) { + Py_DECREF(self); + *self_or_null = NULL; return NULL; - return _PyObject_LookupSpecial(self, attr); + } + + if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { + /* Avoid temporary PyMethodObject */ + *self_or_null = self; + } + else { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (f != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); + } + *self_or_null = NULL; + Py_DECREF(self); + } + return res; } static PyObject * lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) { - PyObject *res = _PyType_Lookup(Py_TYPE(self), attr); + PyObject *res = _PyType_LookupRef(Py_TYPE(self), attr); if (res == NULL) { return NULL; } @@ -2230,16 +2739,12 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { /* Avoid temporary PyMethodObject */ *unbound = 1; - Py_INCREF(res); } else { *unbound = 0; descrgetfunc f = Py_TYPE(res)->tp_descr_get; - if (f == NULL) { - Py_INCREF(res); - } - else { - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if (f != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); } } return res; @@ -2538,8 +3043,10 @@ pmerge(PyObject *acc, PyObject **to_merge, Py_ssize_t to_merge_size) } static PyObject * -mro_implementation(PyTypeObject *type) +mro_implementation_unlocked(PyTypeObject *type) { + ASSERT_TYPE_LOCK_HELD(); + if (!_PyType_IsReady(type)) { if (PyType_Ready(type) < 0) return NULL; @@ -2619,6 +3126,16 @@ mro_implementation(PyTypeObject *type) return result; } +static PyObject * +mro_implementation(PyTypeObject *type) +{ + PyObject *mro; + BEGIN_TYPE_LOCK(); + mro = mro_implementation_unlocked(type); + END_TYPE_LOCK(); + return mro; +} + /*[clinic input] type.mro @@ -2657,7 +3174,7 @@ mro_check(PyTypeObject *type, PyObject *mro) } PyTypeObject *base = (PyTypeObject*)obj; - if (!PyType_IsSubtype(solid, solid_base(base))) { + if (!is_subtype_with_mro(lookup_tp_mro(solid), solid, solid_base(base))) { PyErr_Format( PyExc_TypeError, "mro() returned base with unsuitable layout ('%.500s')", @@ -2688,6 +3205,9 @@ mro_invoke(PyTypeObject *type) { PyObject *mro_result; PyObject *new_mro; + + ASSERT_TYPE_LOCK_HELD(); + const int custom = !Py_IS_TYPE(type, &PyType_Type); if (custom) { @@ -2700,7 +3220,7 @@ mro_invoke(PyTypeObject *type) Py_DECREF(mro_meth); } else { - mro_result = mro_implementation(type); + mro_result = mro_implementation_unlocked(type); } if (mro_result == NULL) return NULL; @@ -2747,8 +3267,10 @@ mro_invoke(PyTypeObject *type) - Returns -1 in case of an error. */ static int -mro_internal(PyTypeObject *type, PyObject **p_old_mro) +mro_internal_unlocked(PyTypeObject *type, int initial, PyObject **p_old_mro) { + ASSERT_TYPE_LOCK_HELD(); + PyObject *new_mro, *old_mro; int reent; @@ -2768,7 +3290,7 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) return 0; } - set_tp_mro(type, new_mro); + set_tp_mro(type, new_mro, initial); type_mro_modified(type, new_mro); /* corner case: the super class might have been hidden @@ -2782,7 +3304,7 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) else { /* For static builtin types, this is only called during init before the method cache has been populated. */ - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + assert(type->tp_version_tag); } if (p_old_mro != NULL) @@ -2793,6 +3315,16 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) return 1; } +static int +mro_internal(PyTypeObject *type, PyObject **p_old_mro) +{ + int res; + BEGIN_TYPE_LOCK(); + res = mro_internal_unlocked(type, 0, p_old_mro); + END_TYPE_LOCK(); + return res; +} + /* Calculate the best base amongst multiple base classes. This is the first one that's on the path to the "solid base". */ @@ -2969,19 +3501,26 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) return func(descr, obj, value); } /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */ - dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return -1; - } if (value != NULL && !PyDict_Check(value)) { PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, " "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } - Py_XSETREF(*dictptr, Py_XNewRef(value)); + + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + return _PyObject_SetManagedDict(obj, value); + } + else { + dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return -1; + } + Py_CLEAR(*dictptr); + *dictptr = Py_XNewRef(value); + } return 0; } @@ -3076,7 +3615,7 @@ type_init(PyObject *cls, PyObject *args, PyObject *kwds) unsigned long PyType_GetFlags(PyTypeObject *type) { - return type->tp_flags; + return FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags); } @@ -3376,7 +3915,7 @@ type_new_alloc(type_new_ctx *ctx) type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; - set_tp_bases(type, Py_NewRef(ctx->bases)); + set_tp_bases(type, Py_NewRef(ctx->bases), 1); type->tp_base = (PyTypeObject *)Py_NewRef(ctx->base); type->tp_dealloc = subtype_dealloc; @@ -3390,6 +3929,11 @@ type_new_alloc(type_new_ctx *ctx) et->ht_name = Py_NewRef(ctx->name); et->ht_module = NULL; et->_ht_tpname = NULL; + et->ht_token = NULL; + +#ifdef Py_GIL_DISABLED + et->unique_id = _PyObject_AssignUniqueId((PyObject *)et); +#endif return type; } @@ -4093,10 +4637,71 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type) return 1; } -static PyObject * -_PyType_FromMetaclass_impl( +static int +check_immutable_bases(const char *type_name, PyObject *bases, int skip_first) +{ + Py_ssize_t i = 0; + if (skip_first) { + // When testing the MRO, skip the type itself + i = 1; + } + for (; itype != Py_T_PYSSIZET) { + PyErr_Format( + PyExc_SystemError, + "type of %s must be Py_T_PYSSIZET", + memb->name); + return -1; + } + if (memb->flags == Py_READONLY) { + *dest = memb->offset; + return 0; + } + else if (memb->flags == (Py_READONLY | Py_RELATIVE_OFFSET)) { + *dest = memb->offset + type_data_offset; + return 0; + } + PyErr_Format( + PyExc_SystemError, + "flags for %s must be Py_READONLY or (Py_READONLY | Py_RELATIVE_OFFSET)", + memb->name); + return -1; +} + +PyObject * +PyType_FromMetaclass( PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in, int _allow_tp_new) + PyType_Spec *spec, PyObject *bases_in) { /* Invariant: A non-NULL value in one of these means this function holds * a strong reference or owns allocated memory. @@ -4118,10 +4723,11 @@ _PyType_FromMetaclass_impl( const PyType_Slot *slot; Py_ssize_t nmembers = 0; - Py_ssize_t weaklistoffset, dictoffset, vectorcalloffset; + const PyMemberDef *weaklistoffset_member = NULL; + const PyMemberDef *dictoffset_member = NULL; + const PyMemberDef *vectorcalloffset_member = NULL; char *res_start; - nmembers = weaklistoffset = dictoffset = vectorcalloffset = 0; for (slot = spec->slots; slot->slot; slot++) { if (slot->slot < 0 || (size_t)slot->slot >= Py_ARRAY_LENGTH(pyslot_offsets)) { @@ -4138,24 +4744,6 @@ _PyType_FromMetaclass_impl( } for (const PyMemberDef *memb = slot->pfunc; memb->name != NULL; memb++) { nmembers++; - if (strcmp(memb->name, "__weaklistoffset__") == 0) { - // The PyMemberDef must be a Py_ssize_t and readonly - assert(memb->type == Py_T_PYSSIZET); - assert(memb->flags == Py_READONLY); - weaklistoffset = memb->offset; - } - if (strcmp(memb->name, "__dictoffset__") == 0) { - // The PyMemberDef must be a Py_ssize_t and readonly - assert(memb->type == Py_T_PYSSIZET); - assert(memb->flags == Py_READONLY); - dictoffset = memb->offset; - } - if (strcmp(memb->name, "__vectorcalloffset__") == 0) { - // The PyMemberDef must be a Py_ssize_t and readonly - assert(memb->type == Py_T_PYSSIZET); - assert(memb->flags == Py_READONLY); - vectorcalloffset = memb->offset; - } if (memb->flags & Py_RELATIVE_OFFSET) { if (spec->basicsize > 0) { PyErr_SetString( @@ -4170,6 +4758,15 @@ _PyType_FromMetaclass_impl( goto finally; } } + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + weaklistoffset_member = memb; + } + if (strcmp(memb->name, "__dictoffset__") == 0) { + dictoffset_member = memb; + } + if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + vectorcalloffset_member = memb; + } } break; case Py_tp_doc: @@ -4249,23 +4846,8 @@ _PyType_FromMetaclass_impl( * and only heap types can be mutable.) */ if (spec->flags & Py_TPFLAGS_IMMUTABLETYPE) { - for (int i=0; iname, - b->tp_name)) - { - goto finally; - } - } + if (check_immutable_bases(spec->name, bases, 0) < 0) { + goto finally; } } @@ -4285,21 +4867,10 @@ _PyType_FromMetaclass_impl( goto finally; } if (metaclass->tp_new && metaclass->tp_new != PyType_Type.tp_new) { - if (_allow_tp_new) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "Type %s uses PyType_Spec with a metaclass that has custom " - "tp_new. This is deprecated and will no longer be allowed in " - "Python 3.14.", spec->name) < 0) { - goto finally; - } - } - else { - PyErr_SetString( - PyExc_TypeError, - "Metaclasses with custom tp_new are not supported."); - goto finally; - } + PyErr_SetString( + PyExc_TypeError, + "Metaclasses with custom tp_new are not supported."); + goto finally; } /* Calculate best base, and check that all bases are type objects */ @@ -4337,6 +4908,24 @@ _PyType_FromMetaclass_impl( Py_ssize_t itemsize = spec->itemsize; + /* Compute special offsets */ + + Py_ssize_t weaklistoffset = 0; + if (special_offset_from_member(weaklistoffset_member, type_data_offset, + &weaklistoffset) < 0) { + goto finally; + } + Py_ssize_t dictoffset = 0; + if (special_offset_from_member(dictoffset_member, type_data_offset, + &dictoffset) < 0) { + goto finally; + } + Py_ssize_t vectorcalloffset = 0; + if (special_offset_from_member(vectorcalloffset_member, type_data_offset, + &vectorcalloffset) < 0) { + goto finally; + } + /* Allocate the new type * * Between here and PyType_Ready, we should limit: @@ -4368,7 +4957,7 @@ _PyType_FromMetaclass_impl( /* Set slots we have prepared */ type->tp_base = (PyTypeObject *)Py_NewRef(base); - set_tp_bases(type, bases); + set_tp_bases(type, bases, 1); bases = NULL; // We give our reference to bases to the type type->tp_doc = tp_doc; @@ -4414,6 +5003,11 @@ _PyType_FromMetaclass_impl( } } break; + case Py_tp_token: + { + res->ht_token = slot->pfunc == Py_TP_USE_SPEC ? spec : slot->pfunc; + } + break; default: { /* Copy other slots directly */ @@ -4445,6 +5039,11 @@ _PyType_FromMetaclass_impl( type->tp_weaklistoffset = weaklistoffset; type->tp_dictoffset = dictoffset; +#ifdef Py_GIL_DISABLED + // Assign a unique id to enable per-thread refcounting + res->unique_id = _PyObject_AssignUniqueId((PyObject *)res); +#endif + /* Ready the type (which includes inheritance). * * After this call we should generally only touch up what's @@ -4523,29 +5122,22 @@ _PyType_FromMetaclass_impl( return (PyObject*)res; } -PyObject * -PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in) -{ - return _PyType_FromMetaclass_impl(metaclass, module, spec, bases_in, 0); -} - PyObject * PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { - return _PyType_FromMetaclass_impl(NULL, module, spec, bases, 1); + return PyType_FromMetaclass(NULL, module, spec, bases); } PyObject * PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { - return _PyType_FromMetaclass_impl(NULL, NULL, spec, bases, 1); + return PyType_FromMetaclass(NULL, NULL, spec, bases); } PyObject * PyType_FromSpec(PyType_Spec *spec) { - return _PyType_FromMetaclass_impl(NULL, NULL, spec, NULL, 1); + return PyType_FromMetaclass(NULL, NULL, spec, NULL); } PyObject * @@ -4561,9 +5153,9 @@ PyType_GetQualName(PyTypeObject *type) } PyObject * -_PyType_GetModuleName(PyTypeObject *type) +PyType_GetModuleName(PyTypeObject *type) { - return type_module(type, NULL); + return type_module(type); } void * @@ -4576,8 +5168,15 @@ PyType_GetSlot(PyTypeObject *type, int slot) PyErr_BadInternalCall(); return NULL; } + int slot_offset = pyslot_offsets[slot].slot_offset; - parent_slot = *(void**)((char*)type + pyslot_offsets[slot].slot_offset); + if (slot_offset >= (int)sizeof(PyTypeObject)) { + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + return NULL; + } + } + + parent_slot = *(void**)((char*)type + slot_offset); if (parent_slot == NULL) { return NULL; } @@ -4631,18 +5230,36 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + // type_ready_mro() ensures that no heap type is + // contained in a static type MRO. + return NULL; + } + else { + PyHeapTypeObject *ht = (PyHeapTypeObject*)type; + PyObject *module = ht->ht_module; + if (module && _PyModule_GetDef(module) == def) { + return module; + } + } + + PyObject *res = NULL; + BEGIN_TYPE_LOCK(); + PyObject *mro = lookup_tp_mro(type); // The type must be ready assert(mro != NULL); assert(PyTuple_Check(mro)); - // mro_invoke() ensures that the type MRO cannot be empty, so we don't have - // to check i < PyTuple_GET_SIZE(mro) at the first loop iteration. + // mro_invoke() ensures that the type MRO cannot be empty. assert(PyTuple_GET_SIZE(mro) >= 1); + // Also, the first item in the MRO is the type itself, which + // we already checked above. We skip it in the loop. + assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)type); Py_ssize_t n = PyTuple_GET_SIZE(mro); - for (Py_ssize_t i = 0; i < n; i++) { + for (Py_ssize_t i = 1; i < n; i++) { PyObject *super = PyTuple_GET_ITEM(mro, i); - if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { + if (!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { // Static types in the MRO need to be skipped continue; } @@ -4650,17 +5267,107 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) PyHeapTypeObject *ht = (PyHeapTypeObject*)super; PyObject *module = ht->ht_module; if (module && _PyModule_GetDef(module) == def) { - return module; + res = module; + break; } } + END_TYPE_LOCK(); - PyErr_Format( - PyExc_TypeError, - "PyType_GetModuleByDef: No superclass of '%s' has the given module", - type->tp_name); - return NULL; + if (res == NULL) { + PyErr_Format( + PyExc_TypeError, + "PyType_GetModuleByDef: No superclass of '%s' has the given module", + type->tp_name); + } + return res; } + +static PyTypeObject * +get_base_by_token_recursive(PyObject *bases, void *token) +{ + assert(bases != NULL); + PyTypeObject *res = NULL; + Py_ssize_t n = PyTuple_GET_SIZE(bases); + for (Py_ssize_t i = 0; i < n; i++) { + PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); + if (!_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE)) { + continue; + } + if (((PyHeapTypeObject*)base)->ht_token == token) { + res = base; + break; + } + base = get_base_by_token_recursive(lookup_tp_bases(base), token); + if (base != NULL) { + res = base; + break; + } + } + return res; // Prefer to return recursively from one place +} + +int +PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) +{ + if (result != NULL) { + *result = NULL; + } + + if (token == NULL) { + PyErr_Format(PyExc_SystemError, + "PyType_GetBaseByToken called with token=NULL"); + return -1; + } + if (!PyType_Check(type)) { + PyErr_Format(PyExc_TypeError, + "expected a type, got a '%T' object", type); + return -1; + } + + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + // No static type has a heaptype superclass, + // which is ensured by type_ready_mro(). + return 0; + } + if (((PyHeapTypeObject*)type)->ht_token == token) { +found: + if (result != NULL) { + *result = (PyTypeObject *)Py_NewRef(type); + } + return 1; + } + + PyObject *mro = type->tp_mro; // No lookup, following PyType_IsSubtype() + if (mro == NULL) { + PyTypeObject *base; + base = get_base_by_token_recursive(lookup_tp_bases(type), token); + if (base != NULL) { + // Copying the given type can cause a slowdown, + // unlike the overwrite below. + type = base; + goto found; + } + return 0; + } + // mro_invoke() ensures that the type MRO cannot be empty. + assert(PyTuple_GET_SIZE(mro) >= 1); + // Also, the first item in the MRO is the type itself, which + // we already checked above. We skip it in the loop. + assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)type); + Py_ssize_t n = PyTuple_GET_SIZE(mro); + for (Py_ssize_t i = 1; i < n; i++) { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(mro, i); + if (_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE) + && ((PyHeapTypeObject*)base)->ht_token == token) { + type = base; + goto found; + } + } + return 0; +} + + void * PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) { @@ -4696,15 +5403,12 @@ PyObject_GetItemData(PyObject *obj) static PyObject * find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) { - Py_hash_t hash; - if (!PyUnicode_CheckExact(name) || - (hash = _PyASCIIObject_CAST(name)->hash) == -1) - { - hash = PyObject_Hash(name); - if (hash == -1) { - *error = -1; - return NULL; - } + ASSERT_TYPE_LOCK_HELD(); + + Py_hash_t hash = _PyObject_HashFast(name); + if (hash == -1) { + *error = -1; + return NULL; } /* Look in tp_dict of types in MRO */ @@ -4732,14 +5436,13 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) PyObject *base = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); - res = _PyDict_GetItem_KnownHash(dict, name, hash); - if (res != NULL) { - break; - } - if (PyErr_Occurred()) { + if (_PyDict_GetItemRef_KnownHash((PyDictObject *)dict, name, hash, &res) < 0) { *error = -1; goto done; } + if (res != NULL) { + break; + } } *error = 0; done: @@ -4765,10 +5468,68 @@ is_dunder_name(PyObject *name) return 0; } +static PyObject * +update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) +{ + _Py_atomic_store_uint32_relaxed(&entry->version, version_tag); + _Py_atomic_store_ptr_relaxed(&entry->value, value); /* borrowed */ + assert(_PyASCIIObject_CAST(name)->hash != -1); + OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); + // We're releasing this under the lock for simplicity sake because it's always a + // exact unicode object or Py_None so it's safe to do so. + PyObject *old_name = entry->name; + _Py_atomic_store_ptr_relaxed(&entry->name, Py_NewRef(name)); + return old_name; +} + +#if Py_GIL_DISABLED + +static void +update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, + unsigned int version_tag, PyObject *value) +{ + _PySeqLock_LockWrite(&entry->sequence); + + // update the entry + if (entry->name == name && + entry->value == value && + entry->version == version_tag) { + // We raced with another update, bail and restore previous sequence. + _PySeqLock_AbandonWrite(&entry->sequence); + return; + } + + PyObject *old_value = update_cache(entry, name, version_tag, value); + + // Then update sequence to the next valid value + _PySeqLock_UnlockWrite(&entry->sequence); + + Py_DECREF(old_value); +} + +#endif + +void +_PyTypes_AfterFork(void) +{ +#ifdef Py_GIL_DISABLED + struct type_cache *cache = get_type_cache(); + for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) { + struct type_cache_entry *entry = &cache->hashtable[i]; + if (_PySeqLock_AfterFork(&entry->sequence)) { + // Entry was in the process of updating while forking, clear it... + entry->value = NULL; + Py_SETREF(entry->name, Py_None); + entry->version = 0; + } + } +#endif +} + /* Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception! */ PyObject * -_PyType_Lookup(PyTypeObject *type, PyObject *name) +_PyType_LookupRef(PyTypeObject *type, PyObject *name) { PyObject *res; int error; @@ -4777,20 +5538,63 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) unsigned int h = MCACHE_HASH_METHOD(type, name); struct type_cache *cache = get_type_cache(); struct type_cache_entry *entry = &cache->hashtable[h]; +#ifdef Py_GIL_DISABLED + // synchronize-with other writing threads by doing an acquire load on the sequence + while (1) { + uint32_t sequence = _PySeqLock_BeginRead(&entry->sequence); + uint32_t entry_version = _Py_atomic_load_uint32_relaxed(&entry->version); + uint32_t type_version = _Py_atomic_load_uint32_acquire(&type->tp_version_tag); + if (entry_version == type_version && + _Py_atomic_load_ptr_relaxed(&entry->name) == name) { + OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); + OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); + PyObject *value = _Py_atomic_load_ptr_relaxed(&entry->value); + // If the sequence is still valid then we're done + if (value == NULL || _Py_TryIncref(value)) { + if (_PySeqLock_EndRead(&entry->sequence, sequence)) { + return value; + } + Py_XDECREF(value); + } + else { + // If we can't incref the object we need to fallback to locking + break; + } + } + else { + // cache miss + break; + } + } +#else if (entry->version == type->tp_version_tag && entry->name == name) { - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + assert(type->tp_version_tag); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); + Py_XINCREF(entry->value); return entry->value; } +#endif OBJECT_STAT_INC_COND(type_cache_misses, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_misses, is_dunder_name(name)); /* We may end up clearing live exceptions below, so make sure it's ours. */ assert(!PyErr_Occurred()); + // We need to atomically do the lookup and capture the version before + // anyone else can modify our mro or mutate the type. + + int has_version = 0; + int version = 0; + BEGIN_TYPE_LOCK(); res = find_name_in_mro(type, name, &error); + if (MCACHE_CACHEABLE_NAME(name)) { + has_version = assign_version_tag(interp, type); + version = type->tp_version_tag; + } + END_TYPE_LOCK(); + /* Only put NULL results into cache if there was no error. */ if (error) { /* It's not ideal to clear the error condition, @@ -4807,31 +5611,73 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) return NULL; } - if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(interp, type)) { - h = MCACHE_HASH_METHOD(type, name); - struct type_cache_entry *entry = &cache->hashtable[h]; - entry->version = type->tp_version_tag; - entry->value = res; /* borrowed */ - assert(_PyASCIIObject_CAST(name)->hash != -1); - OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); - Py_SETREF(entry->name, Py_NewRef(name)); + if (has_version) { +#if Py_GIL_DISABLED + update_cache_gil_disabled(entry, name, version, res); +#else + PyObject *old_value = update_cache(entry, name, version, res); + Py_DECREF(old_value); +#endif } return res; } PyObject * -_PyType_LookupId(PyTypeObject *type, _Py_Identifier *name) +_PyType_Lookup(PyTypeObject *type, PyObject *name) { - PyObject *oname; - oname = _PyUnicode_FromId(name); /* borrowed */ - if (oname == NULL) - return NULL; - return _PyType_Lookup(type, oname); + PyObject *res = _PyType_LookupRef(type, name); + Py_XDECREF(res); + return res; +} + +static void +set_flags(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + ASSERT_TYPE_LOCK_HELD(); + self->tp_flags = (self->tp_flags & ~mask) | flags; +} + +void +_PyType_SetFlags(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + BEGIN_TYPE_LOCK(); + set_flags(self, mask, flags); + END_TYPE_LOCK(); +} + +static void +set_flags_recursive(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + if (PyType_HasFeature(self, Py_TPFLAGS_IMMUTABLETYPE) || + (self->tp_flags & mask) == flags) + { + return; + } + + set_flags(self, mask, flags); + + PyObject *children = _PyType_GetSubclasses(self); + if (children == NULL) { + return; + } + + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(children); i++) { + PyObject *child = PyList_GET_ITEM(children, i); + set_flags_recursive((PyTypeObject *)child, mask, flags); + } + Py_DECREF(children); +} + +void +_PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + BEGIN_TYPE_LOCK(); + set_flags_recursive(self, mask, flags); + END_TYPE_LOCK(); } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. The argument suppress_missing_attribute is used to provide a fast path for hasattr. The possible values are: @@ -4867,10 +5713,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin meta_get = NULL; /* Look for the attribute in the metatype */ - meta_attribute = _PyType_Lookup(metatype, name); + meta_attribute = _PyType_LookupRef(metatype, name); if (meta_attribute != NULL) { - Py_INCREF(meta_attribute); meta_get = Py_TYPE(meta_attribute)->tp_descr_get; if (meta_get != NULL && PyDescr_IsData(meta_attribute)) { @@ -4887,10 +5732,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin /* No data descriptor found on metatype. Look in tp_dict of this * type and its bases */ - attribute = _PyType_Lookup(type, name); + attribute = _PyType_LookupRef(type, name); if (attribute != NULL) { /* Implement descriptor functionality, if any */ - Py_INCREF(attribute); descrgetfunc local_get = Py_TYPE(attribute)->tp_descr_get; Py_XDECREF(meta_attribute); @@ -4935,13 +5779,49 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. */ PyObject * _Py_type_getattro(PyObject *type, PyObject *name) { return _Py_type_getattro_impl((PyTypeObject *)type, name, NULL); } +static int +type_update_dict(PyTypeObject *type, PyDictObject *dict, PyObject *name, + PyObject *value, PyObject **old_value) +{ + // We don't want any re-entrancy between when we update the dict + // and call type_modified_unlocked, including running the destructor + // of the current value as it can observe the cache in an inconsistent + // state. Because we have an exact unicode and our dict has exact + // unicodes we know that this will all complete without releasing + // the locks. + if (_PyDict_GetItemRef_Unicode_LockHeld(dict, name, old_value) < 0) { + return -1; + } + + /* Clear the VALID_VERSION flag of 'type' and all its + subclasses. This could possibly be unified with the + update_subclasses() recursion in update_slot(), but carefully: + they each have their own conditions on which to stop + recursing into subclasses. */ + type_modified_unlocked(type); + + if (_PyDict_SetItem_LockHeld(dict, name, value) < 0) { + PyErr_Format(PyExc_AttributeError, + "type object '%.50s' has no attribute '%U'", + ((PyTypeObject*)type)->tp_name, name); + _PyObject_SetAttributeErrorContext((PyObject *)type, name); + return -1; + } + + if (is_dunder_name(name)) { + return update_slot(type, name); + } + + return 0; +} + static int type_setattro(PyObject *self, PyObject *name, PyObject *value) { @@ -4954,45 +5834,70 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) name, type->tp_name); return -1; } - if (PyUnicode_Check(name)) { - if (PyUnicode_CheckExact(name)) { - Py_INCREF(name); - } - else { - name = _PyUnicode_Copy(name); - if (name == NULL) - return -1; - } - /* bpo-40521: Interned strings are shared by all subinterpreters */ + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "attribute name must be string, not '%.200s'", + Py_TYPE(name)->tp_name); + return -1; + } + + if (PyUnicode_CheckExact(name)) { + Py_INCREF(name); + } + else { + name = _PyUnicode_Copy(name); + if (name == NULL) + return -1; + } + if (!PyUnicode_CHECK_INTERNED(name)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &name); if (!PyUnicode_CHECK_INTERNED(name)) { - PyUnicode_InternInPlace(&name); - if (!PyUnicode_CHECK_INTERNED(name)) { - PyErr_SetString(PyExc_MemoryError, - "Out of memory interning an attribute name"); - Py_DECREF(name); - return -1; - } + PyErr_SetString(PyExc_MemoryError, + "Out of memory interning an attribute name"); + Py_DECREF(name); + return -1; } } - else { - /* Will fail in _PyObject_GenericSetAttrWithDict. */ - Py_INCREF(name); + + PyTypeObject *metatype = Py_TYPE(type); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); + + PyObject *old_value = NULL; + PyObject *descr = _PyType_LookupRef(metatype, name); + if (descr != NULL) { + descrsetfunc f = Py_TYPE(descr)->tp_descr_set; + if (f != NULL) { + res = f(descr, (PyObject *)type, value); + goto done; + } } - res = _PyObject_GenericSetAttrWithDict((PyObject *)type, name, value, NULL); - if (res == 0) { - /* Clear the VALID_VERSION flag of 'type' and all its - subclasses. This could possibly be unified with the - update_subclasses() recursion in update_slot(), but carefully: - they each have their own conditions on which to stop - recursing into subclasses. */ - PyType_Modified(type); - if (is_dunder_name(name)) { - res = update_slot(type, name); + PyObject *dict = type->tp_dict; + if (dict == NULL) { + // We don't just do PyType_Ready because we could already be readying + BEGIN_TYPE_LOCK(); + dict = type->tp_dict; + if (dict == NULL) { + dict = type->tp_dict = PyDict_New(); + } + END_TYPE_LOCK(); + if (dict == NULL) { + res = -1; + goto done; } - assert(_PyType_CheckConsistency(type)); } + + BEGIN_TYPE_DICT_LOCK(dict); + res = type_update_dict(type, (PyDictObject *)dict, name, value, &old_value); + assert(_PyType_CheckConsistency(type)); + END_TYPE_DICT_LOCK(); + +done: Py_DECREF(name); + Py_XDECREF(descr); + Py_XDECREF(old_value); return res; } @@ -5010,7 +5915,7 @@ type_dealloc_common(PyTypeObject *type) static void -clear_static_tp_subclasses(PyTypeObject *type) +clear_static_tp_subclasses(PyTypeObject *type, int isbuiltin) { PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses == NULL) { @@ -5047,47 +5952,74 @@ clear_static_tp_subclasses(PyTypeObject *type) continue; } // All static builtin subtypes should have been finalized already. - assert(!(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + assert(!isbuiltin || !(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); Py_DECREF(subclass); } +#else + (void)isbuiltin; #endif clear_tp_subclasses(type); } static void -clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) +clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type, + int isbuiltin, int final) { - if (_Py_IsMainInterpreter(interp)) { + if (final) { Py_CLEAR(type->tp_cache); } clear_tp_dict(type); - clear_tp_bases(type); - clear_tp_mro(type); - clear_static_tp_subclasses(type); + clear_tp_bases(type, final); + clear_tp_mro(type, final); + clear_static_tp_subclasses(type, isbuiltin); } -void -_PyStaticType_Dealloc(PyInterpreterState *interp, PyTypeObject *type) + +static void +fini_static_type(PyInterpreterState *interp, PyTypeObject *type, + int isbuiltin, int final) { assert(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); assert(_Py_IsImmortal((PyObject *)type)); type_dealloc_common(type); - clear_static_type_objects(interp, type); + clear_static_type_objects(interp, type, isbuiltin, final); - if (_Py_IsMainInterpreter(interp)) { + if (final) { type->tp_flags &= ~Py_TPFLAGS_READY; - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; + _PyType_SetVersion(type, 0); } _PyStaticType_ClearWeakRefs(interp, type); - static_builtin_state_clear(interp, type); + managed_static_type_state_clear(interp, type, isbuiltin, final); /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ } +void +_PyTypes_FiniExtTypes(PyInterpreterState *interp) +{ + for (size_t i = _Py_MAX_MANAGED_STATIC_EXT_TYPES; i > 0; i--) { + if (interp->types.for_extensions.num_initialized == 0) { + break; + } + int64_t count = 0; + PyTypeObject *type = static_ext_type_lookup(interp, i-1, &count); + if (type == NULL) { + continue; + } + int final = (count == 1); + fini_static_type(interp, type, 0, final); + } +} + +void +_PyStaticType_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) +{ + fini_static_type(interp, type, 1, _Py_IsMainInterpreter(interp)); +} + static void type_dealloc(PyObject *self) @@ -5098,7 +6030,6 @@ type_dealloc(PyObject *self) _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); _PyObject_GC_UNTRACK(type); - type_dealloc_common(type); // PyObject_ClearWeakRefs() raises an exception if Py_REFCNT() != 0 @@ -5126,6 +6057,10 @@ type_dealloc(PyObject *self) } Py_XDECREF(et->ht_module); PyMem_Free(et->_ht_tpname); +#ifdef Py_GIL_DISABLED + assert(et->unique_id == -1); +#endif + et->ht_token = NULL; Py_TYPE(type)->tp_free((PyObject *)type); } @@ -5323,7 +6258,7 @@ type_clear(PyObject *self) the dict, so that other objects caught in a reference cycle don't start calling destroyed methods. - Otherwise, the we need to clear tp_mro, which is + Otherwise, we need to clear tp_mro, which is part of a hard cycle (its first element is the class itself) that won't be broken otherwise (it's a tuple and tuples don't have a tp_clear handler). @@ -5529,15 +6464,19 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } comma_w_quotes_sep = PyUnicode_FromString("', '"); + if (!comma_w_quotes_sep) { + Py_DECREF(sorted_methods); + return NULL; + } joined = PyUnicode_Join(comma_w_quotes_sep, sorted_methods); - method_count = PyObject_Length(sorted_methods); - Py_DECREF(sorted_methods); + Py_DECREF(comma_w_quotes_sep); if (joined == NULL) { - Py_DECREF(comma_w_quotes_sep); + Py_DECREF(sorted_methods); return NULL; } + method_count = PyObject_Length(sorted_methods); + Py_DECREF(sorted_methods); if (method_count == -1) { - Py_DECREF(comma_w_quotes_sep); Py_DECREF(joined); return NULL; } @@ -5549,17 +6488,12 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) method_count > 1 ? "s" : "", joined); Py_DECREF(joined); - Py_DECREF(comma_w_quotes_sep); return NULL; } PyObject *obj = type->tp_alloc(type, 0); if (obj == NULL) { return NULL; } - if (_PyObject_InitializeDict(obj)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -5576,7 +6510,7 @@ object_repr(PyObject *self) PyObject *mod, *name, *rtn; type = Py_TYPE(self); - mod = type_module(type, NULL); + mod = type_module(type); if (mod == NULL) PyErr_Clear(); else if (!PyUnicode_Check(mod)) { @@ -5743,6 +6677,11 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* !same_slots_added(newbase, oldbase))) { goto differs; } + if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) != + ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES))) + { + goto differs; + } /* The above does not check for the preheader */ if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == ((newto->tp_flags & Py_TPFLAGS_PREHEADER))) @@ -5759,29 +6698,13 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* return 0; } + + static int -object_set_class(PyObject *self, PyObject *value, void *closure) +object_set_class_world_stopped(PyObject *self, PyTypeObject *newto) { PyTypeObject *oldto = Py_TYPE(self); - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "can't delete __class__ attribute"); - return -1; - } - if (!PyType_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__class__ must be set to a class, not '%s' object", - Py_TYPE(value)->tp_name); - return -1; - } - PyTypeObject *newto = (PyTypeObject *)value; - - if (PySys_Audit("object.__setattr__", "OsO", - self, "__class__", value) < 0) { - return -1; - } - /* In versions of CPython prior to 3.5, the code in compatible_for_assignment was not set up to correctly check for memory layout / slot / etc. compatibility for non-HEAPTYPE classes, so we just @@ -5844,23 +6767,28 @@ object_set_class(PyObject *self, PyObject *value, void *closure) if (compatible_for_assignment(oldto, newto, "__class__")) { /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ - assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags & Py_TPFLAGS_PREHEADER)); - _PyObject_GetDictPtr(self); - if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self))) - { - /* Was unable to convert to dict */ - PyErr_NoMemory(); - return -1; + if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_GetManagedDict(self); + if (dict == NULL) { + dict = _PyObject_MaterializeManagedDict_LockHeld(self); + if (dict == NULL) { + return -1; + } + } + + assert(_PyObject_GetManagedDict(self) == dict); + + if (_PyDict_DetachFromObject(dict, self) < 0) { + return -1; + } + } if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_INCREF(newto); } + Py_SET_TYPE(self, newto); - if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) - Py_DECREF(oldto); - RARE_EVENT_INC(set_class); return 0; } else { @@ -5868,6 +6796,48 @@ object_set_class(PyObject *self, PyObject *value, void *closure) } } +static int +object_set_class(PyObject *self, PyObject *value, void *closure) +{ + + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "can't delete __class__ attribute"); + return -1; + } + if (!PyType_Check(value)) { + PyErr_Format(PyExc_TypeError, + "__class__ must be set to a class, not '%s' object", + Py_TYPE(value)->tp_name); + return -1; + } + PyTypeObject *newto = (PyTypeObject *)value; + + if (PySys_Audit("object.__setattr__", "OsO", + self, "__class__", value) < 0) { + return -1; + } + +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); +#endif + PyTypeObject *oldto = Py_TYPE(self); + int res = object_set_class_world_stopped(self, newto); +#ifdef Py_GIL_DISABLED + _PyEval_StartTheWorld(interp); +#endif + if (res == 0) { + if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) { + Py_DECREF(oldto); + } + + RARE_EVENT_INC(set_class); + return 0; + } + return res; +} + static PyGetSetDef object_getsets[] = { {"__class__", object_get_class, object_set_class, PyDoc_STR("the object's class")}, @@ -6039,7 +7009,7 @@ object_getstate_default(PyObject *obj, int required) iterate over it */ if (slotnames_size != PyList_GET_SIZE(slotnames)) { PyErr_Format(PyExc_RuntimeError, - "__slotsname__ changed size during iteration"); + "__slotnames__ changed size during iteration"); goto error; } @@ -6317,6 +7287,7 @@ reduce_newobj(PyObject *obj) } else { /* args == NULL */ + Py_DECREF(copyreg); Py_DECREF(kwargs); PyErr_BadInternalCall(); return NULL; @@ -6403,18 +7374,7 @@ static PyObject * object___reduce_ex___impl(PyObject *self, int protocol) /*[clinic end generated code: output=2e157766f6b50094 input=f326b43fb8a4c5ff]*/ { -#define objreduce \ - (_Py_INTERP_CACHED_OBJECT(_PyInterpreterState_GET(), objreduce)) - PyObject *reduce, *res; - - if (objreduce == NULL) { - PyObject *dict = lookup_tp_dict(&PyBaseObject_Type); - objreduce = PyDict_GetItemWithError(dict, &_Py_ID(__reduce__)); - if (objreduce == NULL && PyErr_Occurred()) { - return NULL; - } - } - + PyObject *reduce; if (PyObject_GetOptionalAttr(self, &_Py_ID(__reduce__), &reduce) < 0) { return NULL; } @@ -6428,10 +7388,12 @@ object___reduce_ex___impl(PyObject *self, int protocol) Py_DECREF(reduce); return NULL; } - override = (clsreduce != objreduce); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + override = (clsreduce != _Py_INTERP_CACHED_OBJECT(interp, objreduce)); Py_DECREF(clsreduce); if (override) { - res = _PyObject_CallNoArgs(reduce); + PyObject *res = _PyObject_CallNoArgs(reduce); Py_DECREF(reduce); return res; } @@ -6440,7 +7402,6 @@ object___reduce_ex___impl(PyObject *self, int protocol) } return _common_reduce(self, protocol); -#undef objreduce } static PyObject * @@ -6509,8 +7470,11 @@ object___sizeof___impl(PyObject *self) res = 0; isize = Py_TYPE(self)->tp_itemsize; - if (isize > 0) - res = Py_SIZE(self) * isize; + if (isize > 0) { + /* This assumes that ob_size is valid if tp_itemsize is not 0, + which isn't true for PyLongObject. */ + res = _PyVarObject_CAST(self)->ob_size * isize; + } res += Py_TYPE(self)->tp_basicsize; return PyLong_FromSsize_t(res); @@ -6574,7 +7538,7 @@ static PyMethodDef object_methods[] = { OBJECT___REDUCE_EX___METHODDEF OBJECT___REDUCE___METHODDEF OBJECT___GETSTATE___METHODDEF - {"__subclasshook__", object_subclasshook, METH_CLASS | METH_VARARGS, + {"__subclasshook__", object_subclasshook, METH_CLASS | METH_O, object_subclasshook_doc}, {"__init_subclass__", object_init_subclass, METH_CLASS | METH_NOARGS, object_init_subclass_doc}, @@ -6590,12 +7554,6 @@ PyDoc_STRVAR(object_doc, "When called, it accepts no arguments and returns a new featureless\n" "instance that has no instance attributes and cannot be given any.\n"); -static Py_hash_t -object_hash(PyObject *obj) -{ - return _Py_HashPointer(obj); -} - PyTypeObject PyBaseObject_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "object", /* tp_name */ @@ -6610,7 +7568,7 @@ PyTypeObject PyBaseObject_Type = { 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - object_hash, /* tp_hash */ + PyObject_GenericHash, /* tp_hash */ 0, /* tp_call */ object_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -6635,7 +7593,7 @@ PyTypeObject PyBaseObject_Type = { object_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ object_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; @@ -6799,28 +7757,29 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) #undef COPYVAL /* Setup fast subclass flags */ - if (PyType_IsSubtype(base, (PyTypeObject*)PyExc_BaseException)) { + PyObject *mro = lookup_tp_mro(base); + if (is_subtype_with_mro(mro, base, (PyTypeObject*)PyExc_BaseException)) { type->tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyType_Type)) { + else if (is_subtype_with_mro(mro, base, &PyType_Type)) { type->tp_flags |= Py_TPFLAGS_TYPE_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyLong_Type)) { + else if (is_subtype_with_mro(mro, base, &PyLong_Type)) { type->tp_flags |= Py_TPFLAGS_LONG_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyBytes_Type)) { + else if (is_subtype_with_mro(mro, base, &PyBytes_Type)) { type->tp_flags |= Py_TPFLAGS_BYTES_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyUnicode_Type)) { + else if (is_subtype_with_mro(mro, base, &PyUnicode_Type)) { type->tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyTuple_Type)) { + else if (is_subtype_with_mro(mro, base, &PyTuple_Type)) { type->tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyList_Type)) { + else if (is_subtype_with_mro(mro, base, &PyList_Type)) { type->tp_flags |= Py_TPFLAGS_LIST_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyDict_Type)) { + else if (is_subtype_with_mro(mro, base, &PyDict_Type)) { type->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS; } @@ -7042,7 +8001,7 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) return 0; } -static int add_operators(PyTypeObject *); +static int add_operators(PyTypeObject *type); static int add_tp_new_wrapper(PyTypeObject *type); #define COLLECTION_FLAGS (Py_TPFLAGS_SEQUENCE | Py_TPFLAGS_MAPPING) @@ -7125,10 +8084,10 @@ type_ready_set_type(PyTypeObject *type) } static int -type_ready_set_bases(PyTypeObject *type) +type_ready_set_bases(PyTypeObject *type, int initial) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (!initial) { assert(lookup_tp_bases(type) != NULL); return 0; } @@ -7147,7 +8106,7 @@ type_ready_set_bases(PyTypeObject *type) if (bases == NULL) { return -1; } - set_tp_bases(type, bases); + set_tp_bases(type, bases, 1); } return 0; } @@ -7257,10 +8216,12 @@ type_ready_preheader(PyTypeObject *type) } static int -type_ready_mro(PyTypeObject *type) +type_ready_mro(PyTypeObject *type, int initial) { + ASSERT_TYPE_LOCK_HELD(); + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (!initial) { assert(lookup_tp_mro(type) != NULL); return 0; } @@ -7268,7 +8229,7 @@ type_ready_mro(PyTypeObject *type) } /* Calculate method resolution order */ - if (mro_internal(type, NULL) < 0) { + if (mro_internal_unlocked(type, initial, NULL) < 0) { return -1; } PyObject *mro = lookup_tp_mro(type); @@ -7332,6 +8293,8 @@ inherit_patma_flags(PyTypeObject *type, PyTypeObject *base) { static int type_ready_inherit(PyTypeObject *type) { + ASSERT_TYPE_LOCK_HELD(); + /* Inherit special flags from dominant base */ PyTypeObject *base = type->tp_base; if (base != NULL) { @@ -7357,7 +8320,7 @@ type_ready_inherit(PyTypeObject *type) /* Sanity check for tp_free. */ if (_PyType_IS_GC(type) && (type->tp_flags & Py_TPFLAGS_BASETYPE) && - (type->tp_free == NULL || type->tp_free == PyObject_Del)) + (type->tp_free == NULL || type->tp_free == PyObject_Free)) { /* This base class needs to call tp_free, but doesn't have * one, or its tp_free is for non-gc'ed objects. @@ -7421,7 +8384,7 @@ type_ready_add_subclasses(PyTypeObject *type) // Set tp_new and the "__new__" key in the type dictionary. // Use the Py_TPFLAGS_DISALLOW_INSTANTIATION flag. static int -type_ready_set_new(PyTypeObject *type, int rerunbuiltin) +type_ready_set_new(PyTypeObject *type, int initial) { PyTypeObject *base = type->tp_base; /* The condition below could use some explanation. @@ -7443,7 +8406,7 @@ type_ready_set_new(PyTypeObject *type, int rerunbuiltin) if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { if (type->tp_new != NULL) { - if (!rerunbuiltin || base == NULL || type->tp_new != base->tp_new) { + if (initial || base == NULL || type->tp_new != base->tp_new) { // If "__new__" key does not exists in the type dictionary, // set it to tp_new_wrapper(). if (add_tp_new_wrapper(type) < 0) { @@ -7478,12 +8441,15 @@ type_ready_managed_dict(PyTypeObject *type) } PyHeapTypeObject* et = (PyHeapTypeObject*)type; if (et->ht_cached_keys == NULL) { - et->ht_cached_keys = _PyDict_NewKeysForClass(); + et->ht_cached_keys = _PyDict_NewKeysForClass(et); if (et->ht_cached_keys == NULL) { PyErr_NoMemory(); return -1; } } + if (type->tp_itemsize == 0) { + type->tp_flags |= Py_TPFLAGS_INLINE_VALUES; + } return 0; } @@ -7522,8 +8488,10 @@ type_ready_post_checks(PyTypeObject *type) static int -type_ready(PyTypeObject *type, int rerunbuiltin) +type_ready(PyTypeObject *type, int initial) { + ASSERT_TYPE_LOCK_HELD(); + _PyObject_ASSERT((PyObject *)type, !is_readying(type)); start_readying(type); @@ -7550,19 +8518,19 @@ type_ready(PyTypeObject *type, int rerunbuiltin) if (type_ready_set_type(type) < 0) { goto error; } - if (type_ready_set_bases(type) < 0) { + if (type_ready_set_bases(type, initial) < 0) { goto error; } - if (type_ready_mro(type) < 0) { + if (type_ready_mro(type, initial) < 0) { goto error; } - if (type_ready_set_new(type, rerunbuiltin) < 0) { + if (type_ready_set_new(type, initial) < 0) { goto error; } if (type_ready_fill_dict(type) < 0) { goto error; } - if (!rerunbuiltin) { + if (initial) { if (type_ready_inherit(type) < 0) { goto error; } @@ -7576,7 +8544,7 @@ type_ready(PyTypeObject *type, int rerunbuiltin) if (type_ready_add_subclasses(type) < 0) { goto error; } - if (!rerunbuiltin) { + if (initial) { if (type_ready_managed_dict(type) < 0) { goto error; } @@ -7586,7 +8554,7 @@ type_ready(PyTypeObject *type, int rerunbuiltin) } /* All done -- set the ready flag */ - type->tp_flags = type->tp_flags | Py_TPFLAGS_READY; + type->tp_flags |= Py_TPFLAGS_READY; stop_readying(type); assert(_PyType_CheckConsistency(type)); @@ -7609,45 +8577,72 @@ PyType_Ready(PyTypeObject *type) /* Historically, all static types were immutable. See bpo-43908 */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + /* Static types must be immortal */ + _Py_SetImmortalUntracked((PyObject *)type); } - return type_ready(type, 0); + int res; + BEGIN_TYPE_LOCK(); + if (!(type->tp_flags & Py_TPFLAGS_READY)) { + res = type_ready(type, 1); + } else { + res = 0; + assert(_PyType_CheckConsistency(type)); + } + END_TYPE_LOCK(); + return res; } -int -_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) + +static int +init_static_type(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int initial) { assert(_Py_IsImmortal((PyObject *)self)); assert(!(self->tp_flags & Py_TPFLAGS_HEAPTYPE)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); - int ismain = _Py_IsMainInterpreter(interp); if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { - assert(ismain); + assert(initial); self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); - self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; - self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; + _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); } else { - assert(!ismain); + assert(!initial); assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - assert(self->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG); + assert(self->tp_version_tag != 0); } - static_builtin_state_init(interp, self); + managed_static_type_state_init(interp, self, isbuiltin, initial); - int res = type_ready(self, !ismain); + int res; + BEGIN_TYPE_LOCK(); + res = type_ready(self, initial); + END_TYPE_LOCK(); if (res < 0) { - static_builtin_state_clear(interp, self); + _PyStaticType_ClearWeakRefs(interp, self); + managed_static_type_state_clear(interp, self, isbuiltin, initial); } return res; } +int +_PyStaticType_InitForExtension(PyInterpreterState *interp, PyTypeObject *self) +{ + return init_static_type(interp, self, 0, ((self->tp_flags & Py_TPFLAGS_READY) == 0)); +} + +int +_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) +{ + return init_static_type(interp, self, 1, _Py_IsMainInterpreter(interp)); +} + static int add_subclass(PyTypeObject *base, PyTypeObject *type) @@ -7784,6 +8779,27 @@ check_num_args(PyObject *ob, int n) return 0; } +static Py_ssize_t +check_pow_args(PyObject *ob) +{ + // Returns the argument count on success or `-1` on error. + int min = 1; + int max = 2; + if (!PyTuple_CheckExact(ob)) { + PyErr_SetString(PyExc_SystemError, + "PyArg_UnpackTuple() argument list is not a tuple"); + return -1; + } + Py_ssize_t size = PyTuple_GET_SIZE(ob); + if (size >= min && size <= max) { + return size; + } + PyErr_Format( + PyExc_TypeError, + "expected %d or %d arguments, got %zd", min, max, PyTuple_GET_SIZE(ob)); + return -1; +} + /* Generic wrappers for overloadable 'operators' such as __getitem__ */ /* There's a wrapper *function* for each distinct function typedef used @@ -7865,8 +8881,15 @@ wrap_ternaryfunc(PyObject *self, PyObject *args, void *wrapped) /* Note: This wrapper only works for __pow__() */ - if (!PyArg_UnpackTuple(args, "", 1, 2, &other, &third)) + Py_ssize_t size = check_pow_args(args); + if (size == -1) { return NULL; + } + other = PyTuple_GET_ITEM(args, 0); + if (size == 2) { + third = PyTuple_GET_ITEM(args, 1); + } + return (*func)(self, other, third); } @@ -7877,10 +8900,17 @@ wrap_ternaryfunc_r(PyObject *self, PyObject *args, void *wrapped) PyObject *other; PyObject *third = Py_None; - /* Note: This wrapper only works for __pow__() */ + /* Note: This wrapper only works for __rpow__() */ - if (!PyArg_UnpackTuple(args, "", 1, 2, &other, &third)) + Py_ssize_t size = check_pow_args(args); + if (size == -1) { return NULL; + } + other = PyTuple_GET_ITEM(args, 0); + if (size == 2) { + third = PyTuple_GET_ITEM(args, 1); + } + return (*func)(other, self, third); } @@ -7901,8 +8931,9 @@ wrap_indexargfunc(PyObject *self, PyObject *args, void *wrapped) PyObject* o; Py_ssize_t i; - if (!PyArg_UnpackTuple(args, "", 1, 1, &o)) + if (!check_num_args(args, 1)) return NULL; + o = PyTuple_GET_ITEM(args, 0); i = PyNumber_AsSsize_t(o, PyExc_OverflowError); if (i == -1 && PyErr_Occurred()) return NULL; @@ -7958,7 +8989,7 @@ wrap_sq_setitem(PyObject *self, PyObject *args, void *wrapped) int res; PyObject *arg, *value; - if (!PyArg_UnpackTuple(args, "", 2, 2, &arg, &value)) + if (!PyArg_UnpackTuple(args, "__setitem__", 2, 2, &arg, &value)) return NULL; i = getindex(self, arg); if (i == -1 && PyErr_Occurred()) @@ -8014,7 +9045,7 @@ wrap_objobjargproc(PyObject *self, PyObject *args, void *wrapped) int res; PyObject *key, *value; - if (!PyArg_UnpackTuple(args, "", 2, 2, &key, &value)) + if (!PyArg_UnpackTuple(args, "__setitem__", 2, 2, &key, &value)) return NULL; res = (*func)(self, key, value); if (res == -1 && PyErr_Occurred()) @@ -8043,9 +9074,12 @@ wrap_delitem(PyObject *self, PyObject *args, void *wrapped) https://mail.python.org/pipermail/python-dev/2003-April/034535.html */ static int -hackcheck(PyObject *self, setattrofunc func, const char *what) +hackcheck_unlocked(PyObject *self, setattrofunc func, const char *what) { PyTypeObject *type = Py_TYPE(self); + + ASSERT_TYPE_LOCK_HELD(); + PyObject *mro = lookup_tp_mro(type); if (!mro) { /* Probably ok not to check the call in this case. */ @@ -8087,6 +9121,20 @@ hackcheck(PyObject *self, setattrofunc func, const char *what) return 1; } +static int +hackcheck(PyObject *self, setattrofunc func, const char *what) +{ + if (!PyType_Check(self)) { + return 1; + } + + int res; + BEGIN_TYPE_LOCK(); + res = hackcheck_unlocked(self, func, what); + END_TYPE_LOCK(); + return res; +} + static PyObject * wrap_setattr(PyObject *self, PyObject *args, void *wrapped) { @@ -8094,7 +9142,7 @@ wrap_setattr(PyObject *self, PyObject *args, void *wrapped) int res; PyObject *name, *value; - if (!PyArg_UnpackTuple(args, "", 2, 2, &name, &value)) + if (!PyArg_UnpackTuple(args, "__setattr__", 2, 2, &name, &value)) return NULL; if (!hackcheck(self, func, "__setattr__")) return NULL; @@ -8204,7 +9252,7 @@ wrap_descr_get(PyObject *self, PyObject *args, void *wrapped) PyObject *obj; PyObject *type = NULL; - if (!PyArg_UnpackTuple(args, "", 1, 2, &obj, &type)) + if (!PyArg_UnpackTuple(args, "__get__", 1, 2, &obj, &type)) return NULL; if (obj == Py_None) obj = NULL; @@ -8225,7 +9273,7 @@ wrap_descr_set(PyObject *self, PyObject *args, void *wrapped) PyObject *obj, *value; int ret; - if (!PyArg_UnpackTuple(args, "", 2, 2, &obj, &value)) + if (!PyArg_UnpackTuple(args, "__set__", 2, 2, &obj, &value)) return NULL; ret = (*func)(self, obj, value); if (ret < 0) @@ -8254,7 +9302,7 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) { PyObject *arg = NULL; - if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { + if (!PyArg_UnpackTuple(args, "__buffer__", 1, 1, &arg)) { return NULL; } Py_ssize_t flags = PyNumber_AsSsize_t(arg, PyExc_OverflowError); @@ -8275,7 +9323,7 @@ static PyObject * wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) { PyObject *arg = NULL; - if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { + if (!PyArg_UnpackTuple(args, "__release_buffer__", 1, 1, &arg)) { return NULL; } if (!PyMemoryView_Check(arg)) { @@ -8901,33 +9949,33 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when the attribute is present. So we use - _PyType_Lookup and create the method only when needed, with + _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattr = _PyType_Lookup(tp, &_Py_ID(__getattr__)); + getattr = _PyType_LookupRef(tp, &_Py_ID(__getattr__)); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ tp->tp_getattro = _Py_slot_tp_getattro; return _Py_slot_tp_getattro(self, name); } - Py_INCREF(getattr); /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when self has the default __getattribute__ - method. So we use _PyType_Lookup and create the method only when + method. So we use _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattribute = _PyType_Lookup(tp, &_Py_ID(__getattribute__)); + getattribute = _PyType_LookupRef(tp, &_Py_ID(__getattribute__)); if (getattribute == NULL || (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == (void *)PyObject_GenericGetAttr)) { + Py_XDECREF(getattribute); res = _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); /* if res == NULL with no exception set, then it must be an AttributeError suppressed by us. */ if (res == NULL && !PyErr_Occurred()) { res = call_attribute(self, getattr, name); } - } else { - Py_INCREF(getattribute); + } + else { res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -9034,7 +10082,7 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyTypeObject *tp = Py_TYPE(self); PyObject *get; - get = _PyType_Lookup(tp, &_Py_ID(__get__)); + get = _PyType_LookupRef(tp, &_Py_ID(__get__)); if (get == NULL) { /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -9046,7 +10094,9 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) if (type == NULL) type = Py_None; PyObject *stack[3] = {self, obj, type}; - return PyObject_Vectorcall(get, stack, 3, NULL); + PyObject *res = PyObject_Vectorcall(get, stack, 3, NULL); + Py_DECREF(get); + return res; } static int @@ -9255,13 +10305,13 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) return -1; } -static int -releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +static releasebufferproc +releasebuffer_maybe_call_super_unlocked(PyObject *self, Py_buffer *buffer) { PyTypeObject *self_type = Py_TYPE(self); PyObject *mro = lookup_tp_mro(self_type); if (mro == NULL) { - return -1; + return NULL; } assert(PyTuple_Check(mro)); @@ -9275,9 +10325,8 @@ releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) } i++; /* skip self_type */ if (i >= n) - return -1; + return NULL; - releasebufferproc base_releasebuffer = NULL; for (; i < n; i++) { PyObject *obj = PyTuple_GET_ITEM(mro, i); if (!PyType_Check(obj)) { @@ -9287,15 +10336,25 @@ releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) if (base_type->tp_as_buffer != NULL && base_type->tp_as_buffer->bf_releasebuffer != NULL && base_type->tp_as_buffer->bf_releasebuffer != slot_bf_releasebuffer) { - base_releasebuffer = base_type->tp_as_buffer->bf_releasebuffer; - break; + return base_type->tp_as_buffer->bf_releasebuffer; } } + return NULL; +} + +static void +releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +{ + releasebufferproc base_releasebuffer; + + BEGIN_TYPE_LOCK(); + base_releasebuffer = releasebuffer_maybe_call_super_unlocked(self, buffer); + END_TYPE_LOCK(); + if (base_releasebuffer != NULL) { base_releasebuffer(self, buffer); } - return 0; } static void @@ -9372,11 +10431,7 @@ static void slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) { releasebuffer_call_python(self, buffer); - if (releasebuffer_maybe_call_super(self, buffer) < 0) { - if (PyErr_Occurred()) { - PyErr_WriteUnraisable(self); - } - } + releasebuffer_maybe_call_super(self, buffer); } static PyObject * @@ -9512,7 +10567,8 @@ static pytype_slotdef slotdefs[] = { TPSLOT(__getattribute__, tp_getattro, _Py_slot_tp_getattr_hook, wrap_binaryfunc, "__getattribute__($self, name, /)\n--\n\nReturn getattr(self, name)."), - TPSLOT(__getattr__, tp_getattro, _Py_slot_tp_getattr_hook, NULL, ""), + TPSLOT(__getattr__, tp_getattro, _Py_slot_tp_getattr_hook, NULL, + "__getattr__($self, name, /)\n--\n\nImplement getattr(self, name)."), TPSLOT(__setattr__, tp_setattro, slot_tp_setattro, wrap_setattr, "__setattr__($self, name, value, /)\n--\n\nImplement setattr(self, name, value)."), TPSLOT(__delattr__, tp_setattro, slot_tp_setattro, wrap_delattr, @@ -9547,7 +10603,9 @@ static pytype_slotdef slotdefs[] = { TPSLOT(__new__, tp_new, slot_tp_new, NULL, "__new__(type, /, *args, **kwargs)\n--\n\n" "Create and return new object. See help(type) for accurate signature."), - TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""), + TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, + "__del__($self, /)\n--\n\n" + "Called when the instance is about to be destroyed."), BUFSLOT(__buffer__, bf_getbuffer, slot_bf_getbuffer, wrap_buffer, "__buffer__($self, flags, /)\n--\n\n" @@ -9831,6 +10889,8 @@ resolve_slotdups(PyTypeObject *type, PyObject *name) static pytype_slotdef * update_one_slot(PyTypeObject *type, pytype_slotdef *p) { + ASSERT_TYPE_LOCK_HELD(); + PyObject *descr; PyWrapperDescrObject *d; @@ -9879,7 +10939,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) d = (PyWrapperDescrObject *)descr; if ((specific == NULL || specific == d->d_wrapped) && d->d_base->wrapper == p->wrapper && - PyType_IsSubtype(type, PyDescr_TYPE(d))) + is_subtype_with_mro(lookup_tp_mro(type), type, PyDescr_TYPE(d))) { specific = d->d_wrapped; } @@ -9931,6 +10991,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) type->tp_flags &= ~Py_TPFLAGS_HAVE_VECTORCALL; } } + Py_DECREF(descr); } while ((++p)->offset == offset); if (specific && !use_generic) *ptr = specific; @@ -9944,6 +11005,8 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) static int update_slots_callback(PyTypeObject *type, void *data) { + ASSERT_TYPE_LOCK_HELD(); + pytype_slotdef **pp = (pytype_slotdef **)data; for (; *pp; pp++) { update_one_slot(type, *pp); @@ -9960,6 +11023,7 @@ update_slot(PyTypeObject *type, PyObject *name) pytype_slotdef **pp; int offset; + ASSERT_TYPE_LOCK_HELD(); assert(PyUnicode_CheckExact(name)); assert(PyUnicode_CHECK_INTERNED(name)); @@ -9993,10 +11057,17 @@ update_slot(PyTypeObject *type, PyObject *name) static void fixup_slot_dispatchers(PyTypeObject *type) { + // This lock isn't strictly necessary because the type has not been + // exposed to anyone else yet, but update_ont_slot calls find_name_in_mro + // where we'd like to assert that the type is locked. + BEGIN_TYPE_LOCK(); + assert(!PyErr_Occurred()); for (pytype_slotdef *p = slotdefs; p->name; ) { p = update_one_slot(type, p); } + + END_TYPE_LOCK(); } static void @@ -10004,6 +11075,8 @@ update_all_slots(PyTypeObject* type) { pytype_slotdef *p; + ASSERT_TYPE_LOCK_HELD(); + /* Clear the VALID_VERSION flag of 'type' and all its subclasses. */ PyType_Modified(type); @@ -10014,6 +11087,24 @@ update_all_slots(PyTypeObject* type) } +PyObject * +_PyType_GetSlotWrapperNames(void) +{ + size_t len = Py_ARRAY_LENGTH(slotdefs) - 1; + PyObject *names = PyList_New(len); + if (names == NULL) { + return NULL; + } + assert(slotdefs[len].name == NULL); + for (size_t i = 0; i < len; i++) { + pytype_slotdef *slotdef = &slotdefs[i]; + assert(slotdef->name != NULL); + PyList_SET_ITEM(names, i, Py_NewRef(slotdef->name_strobj)); + } + return names; +} + + /* Call __set_name__ on all attributes (including descriptors) in a newly generated type */ static int @@ -10147,6 +11238,26 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, return 0; } +static int +slot_inherited(PyTypeObject *type, pytype_slotdef *slotdef, void **slot) +{ + void **slot_base = slotptr(type->tp_base, slotdef->offset); + if (slot_base == NULL || *slot != *slot_base) { + return 0; + } + + /* Some slots are inherited in pairs. */ + if (slot == (void *)&type->tp_hash) { + return (type->tp_richcompare == type->tp_base->tp_richcompare); + } + else if (slot == (void *)&type->tp_richcompare) { + return (type->tp_hash == type->tp_base->tp_hash); + } + + /* It must be inherited (see type_ready_inherit()). */ + return 1; +} + /* This function is called by PyType_Ready() to populate the type's dictionary with method descriptors for function slots. For each function slot (like tp_repr) that's defined in the type, one or more @@ -10191,6 +11302,13 @@ add_operators(PyTypeObject *type) ptr = slotptr(type, p->offset); if (!ptr || !*ptr) continue; + /* Also ignore when the type slot has been inherited. */ + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN + && type->tp_base != NULL + && slot_inherited(type, p, ptr)) + { + continue; + } int r = PyDict_Contains(dict, p->name_strobj); if (r > 0) continue; @@ -10219,6 +11337,30 @@ add_operators(PyTypeObject *type) } +int +PyType_Freeze(PyTypeObject *type) +{ + // gh-121654: Check the __mro__ instead of __bases__ + PyObject *mro = type_get_mro(type, NULL); + if (!PyTuple_Check(mro)) { + Py_DECREF(mro); + PyErr_SetString(PyExc_TypeError, "unable to get the type MRO"); + return -1; + } + + int check = check_immutable_bases(type->tp_name, mro, 1); + Py_DECREF(mro); + if (check < 0) { + return -1; + } + + type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + PyType_Modified(type); + + return 0; +} + + /* Cooperative 'super' */ typedef struct { @@ -10276,7 +11418,15 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * PyObject *mro, *res; Py_ssize_t i, n; + BEGIN_TYPE_LOCK(); mro = lookup_tp_mro(su_obj_type); + /* keep a strong reference to mro because su_obj_type->tp_mro can be + replaced during PyDict_GetItemRef(dict, name, &res) and because + another thread can modify it after we end the critical section + below */ + Py_XINCREF(mro); + END_TYPE_LOCK(); + if (mro == NULL) return NULL; @@ -10289,12 +11439,11 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * break; } i++; /* skip su->type (if any) */ - if (i >= n) + if (i >= n) { + Py_DECREF(mro); return NULL; + } - /* keep a strong reference to mro because su_obj_type->tp_mro can be - replaced during PyDict_GetItemRef(dict, name, &res) */ - Py_INCREF(mro); do { PyObject *obj = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(obj)); @@ -10502,7 +11651,7 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, } assert(_PyFrame_GetCode(cframe)->co_nlocalsplus > 0); - PyObject *firstarg = _PyFrame_GetLocalsArray(cframe)[0]; + PyObject *firstarg = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[0]); // The first argument might be a cell. if (firstarg != NULL && (_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { // "firstarg" is a cell here unless (very unlikely) super() @@ -10524,13 +11673,13 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, // Look for __class__ in the free vars. PyTypeObject *type = NULL; - int i = PyCode_GetFirstFree(co); + int i = PyUnstable_Code_GetFirstFree(co); for (; i < co->co_nlocalsplus; i++) { assert((_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_FREE) != 0); PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); assert(PyUnicode_Check(name)); if (_PyUnicode_Equal(name, &_Py_ID(__class__))) { - PyObject *cell = _PyFrame_GetLocalsArray(cframe)[i]; + PyObject *cell = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[i]); if (cell == NULL || !PyCell_Check(cell)) { PyErr_SetString(PyExc_RuntimeError, "super(): bad __class__ cell"); diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index 896daa7d8066b7..642160fe0bd8bc 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -80,3 +80,5 @@ {offsetof(PyAsyncMethods, am_anext), offsetof(PyTypeObject, tp_as_async)}, {-1, offsetof(PyTypeObject, tp_finalize)}, {offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)}, +{-1, offsetof(PyTypeObject, tp_vectorcall)}, +{-1, offsetof(PyHeapTypeObject, ht_token)}, diff --git a/Objects/typeslots.py b/Objects/typeslots.py index 8ab05f91be12b0..c7f8a33bb1e74e 100755 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -13,7 +13,11 @@ def generate_typeslots(out=sys.stdout): continue member = m.group(1) - if member.startswith("tp_"): + if member == "tp_token": + # The heap type structure (ht_*) is an implementation detail; + # the public slot for it has a familiar `tp_` prefix + member = '{-1, offsetof(PyHeapTypeObject, ht_token)}' + elif member.startswith("tp_"): member = f'{{-1, offsetof(PyTypeObject, {member})}}' elif member.startswith("am_"): member = (f'{{offsetof(PyAsyncMethods, {member}),'+ diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 7f80c9c61b8abc..91cc37c9a72636 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -23,6 +23,8 @@ typedef struct { PyObject *evaluate_bound; PyObject *constraints; PyObject *evaluate_constraints; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -31,12 +33,16 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *name; + PyObject *default_value; + PyObject *evaluate_default; } typevartupleobject; typedef struct { PyObject_HEAD PyObject *name; PyObject *bound; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -53,6 +59,262 @@ typedef struct { #include "clinic/typevarobject.c.h" +/* NoDefault is a marker object to indicate that a parameter has no default. */ + +static PyObject * +NoDefault_repr(PyObject *op) +{ + return PyUnicode_FromString("typing.NoDefault"); +} + +static PyObject * +NoDefault_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("NoDefault"); +} + +static PyMethodDef nodefault_methods[] = { + {"__reduce__", NoDefault_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static PyObject * +nodefault_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) { + PyErr_SetString(PyExc_TypeError, "NoDefaultType takes no arguments"); + return NULL; + } + return &_Py_NoDefaultStruct; +} + +static void +nodefault_dealloc(PyObject *nodefault) +{ + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref NoDefault out of existence. Instead, + * since NoDefault is an immortal object, re-set the reference count. + */ + _Py_SetImmortal(nodefault); +} + +PyDoc_STRVAR(nodefault_doc, +"NoDefaultType()\n" +"--\n\n" +"The type of the NoDefault singleton."); + +PyTypeObject _PyNoDefault_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "NoDefaultType", + .tp_dealloc = nodefault_dealloc, + .tp_repr = NoDefault_repr, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = nodefault_doc, + .tp_methods = nodefault_methods, + .tp_new = nodefault_new, +}; + +PyObject _Py_NoDefaultStruct = _PyObject_HEAD_INIT(&_PyNoDefault_Type); + +typedef struct { + PyObject_HEAD + PyObject *value; +} constevaluatorobject; + +static void +constevaluator_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + constevaluatorobject *ce = (constevaluatorobject *)self; + + _PyObject_GC_UNTRACK(self); + + Py_XDECREF(ce->value); + + Py_TYPE(self)->tp_free(self); + Py_DECREF(tp); +} + +static int +constevaluator_traverse(PyObject *self, visitproc visit, void *arg) +{ + constevaluatorobject *ce = (constevaluatorobject *)self; + Py_VISIT(ce->value); + return 0; +} + +static int +constevaluator_clear(PyObject *self) +{ + Py_CLEAR(((constevaluatorobject *)self)->value); + return 0; +} + +static PyObject * +constevaluator_repr(PyObject *self) +{ + PyObject *value = ((constevaluatorobject *)self)->value; + return PyUnicode_FromFormat("", value); +} + +static PyObject * +constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (!_PyArg_NoKeywords("constevaluator.__call__", kwargs)) { + return NULL; + } + int format; + if (!PyArg_ParseTuple(args, "i:constevaluator.__call__", &format)) { + return NULL; + } + PyObject *value = ((constevaluatorobject *)self)->value; + if (format == 3) { // STRING + PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 + if (writer == NULL) { + return NULL; + } + if (PyTuple_Check(value)) { + if (PyUnicodeWriter_WriteChar(writer, '(') < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(value); i++) { + PyObject *item = PyTuple_GET_ITEM(value, i); + if (i > 0) { + if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + if (_Py_typing_type_repr(writer, item) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + else { + if (_Py_typing_type_repr(writer, value) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + return PyUnicodeWriter_Finish(writer); + } + return Py_NewRef(value); +} + +static PyObject * +constevaluator_alloc(PyObject *value) +{ + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.constevaluator_type; + assert(tp != NULL); + constevaluatorobject *ce = PyObject_GC_New(constevaluatorobject, tp); + if (ce == NULL) { + return NULL; + } + ce->value = Py_NewRef(value); + _PyObject_GC_TRACK(ce); + return (PyObject *)ce; + +} + +PyDoc_STRVAR(constevaluator_doc, +"_ConstEvaluator()\n" +"--\n\n" +"Internal type for implementing evaluation functions."); + +static PyType_Slot constevaluator_slots[] = { + {Py_tp_doc, (void *)constevaluator_doc}, + {Py_tp_dealloc, constevaluator_dealloc}, + {Py_tp_traverse, constevaluator_traverse}, + {Py_tp_clear, constevaluator_clear}, + {Py_tp_repr, constevaluator_repr}, + {Py_tp_call, constevaluator_call}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +PyType_Spec constevaluator_spec = { + .name = "_typing._ConstEvaluator", + .basicsize = sizeof(constevaluatorobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .slots = constevaluator_slots, +}; + +int +_Py_typing_type_repr(PyUnicodeWriter *writer, PyObject *p) +{ + PyObject *qualname = NULL; + PyObject *module = NULL; + PyObject *r = NULL; + int rc; + + if (p == Py_Ellipsis) { + // The Ellipsis object + r = PyUnicode_FromString("..."); + goto exit; + } + + if (p == (PyObject *)&_PyNone_Type) { + return PyUnicodeWriter_WriteUTF8(writer, "None", 4); + } + + if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && + (rc = PyObject_HasAttrWithError(p, &_Py_ID(__args__))) > 0) + { + // It looks like a GenericAlias + goto use_repr; + } + if (rc < 0) { + goto exit; + } + + if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { + goto exit; + } + if (qualname == NULL) { + goto use_repr; + } + if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { + goto exit; + } + if (module == NULL || module == Py_None) { + goto use_repr; + } + + // Looks like a class + if (PyUnicode_Check(module) && + _PyUnicode_EqualToASCIIString(module, "builtins")) + { + // builtins don't need a module name + r = PyObject_Str(qualname); + goto exit; + } + else { + r = PyUnicode_FromFormat("%S.%S", module, qualname); + goto exit; + } + +use_repr: + r = PyObject_Repr(p); +exit: + Py_XDECREF(qualname); + Py_XDECREF(module); + if (r == NULL) { + return -1; + } + rc = PyUnicodeWriter_WriteStr(writer, r); + Py_DECREF(r); + return rc; +} + + static PyObject * call_typing_func_object(const char *name, PyObject **args, size_t nargs) { @@ -111,10 +373,10 @@ caller(void) if (f == NULL) { Py_RETURN_NONE; } - if (f == NULL || f->f_funcobj == NULL) { + if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { Py_RETURN_NONE; } - PyObject *r = PyFunction_GetModule(f->f_funcobj); + PyObject *r = PyFunction_GetModule(PyStackRef_AsPyObjectBorrow(f->f_funcobj)); if (!r) { PyErr_Clear(); Py_RETURN_NONE; @@ -200,6 +462,8 @@ typevar_dealloc(PyObject *self) Py_XDECREF(tv->evaluate_bound); Py_XDECREF(tv->constraints); Py_XDECREF(tv->evaluate_constraints); + Py_XDECREF(tv->default_value); + Py_XDECREF(tv->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -216,6 +480,8 @@ typevar_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(tv->evaluate_bound); Py_VISIT(tv->constraints); Py_VISIT(tv->evaluate_constraints); + Py_VISIT(tv->default_value); + Py_VISIT(tv->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -227,6 +493,8 @@ typevar_clear(typevarobject *self) Py_CLEAR(self->evaluate_bound); Py_CLEAR(self->constraints); Py_CLEAR(self->evaluate_constraints); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -266,6 +534,20 @@ typevar_bound(typevarobject *self, void *Py_UNUSED(ignored)) return bound; } +static PyObject * +typevar_default(typevarobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + static PyObject * typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) { @@ -280,15 +562,56 @@ typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) return constraints; } +static PyObject * +typevar_evaluate_bound(typevarobject *self, void *Py_UNUSED(ignored)) +{ + if (self->evaluate_bound != NULL) { + return Py_NewRef(self->evaluate_bound); + } + if (self->bound != NULL) { + return constevaluator_alloc(self->bound); + } + Py_RETURN_NONE; +} + +static PyObject * +typevar_evaluate_constraints(typevarobject *self, void *Py_UNUSED(ignored)) +{ + if (self->evaluate_constraints != NULL) { + return Py_NewRef(self->evaluate_constraints); + } + if (self->constraints != NULL) { + return constevaluator_alloc(self->constraints); + } + Py_RETURN_NONE; +} + +static PyObject * +typevar_evaluate_default(typevarobject *self, void *Py_UNUSED(ignored)) +{ + if (self->evaluate_default != NULL) { + return Py_NewRef(self->evaluate_default); + } + if (self->default_value != NULL) { + return constevaluator_alloc(self->default_value); + } + Py_RETURN_NONE; +} + static PyGetSetDef typevar_getset[] = { {"__bound__", (getter)typevar_bound, NULL, NULL, NULL}, {"__constraints__", (getter)typevar_constraints, NULL, NULL, NULL}, + {"__default__", (getter)typevar_default, NULL, NULL, NULL}, + {"evaluate_bound", (getter)typevar_evaluate_bound, NULL, NULL, NULL}, + {"evaluate_constraints", (getter)typevar_evaluate_constraints, NULL, NULL, NULL}, + {"evaluate_default", (getter)typevar_evaluate_default, NULL, NULL, NULL}, {0} }; static typevarobject * typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, PyObject *constraints, PyObject *evaluate_constraints, + PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { @@ -305,6 +628,8 @@ typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, tv->evaluate_bound = Py_XNewRef(evaluate_bound); tv->constraints = Py_XNewRef(constraints); tv->evaluate_constraints = Py_XNewRef(evaluate_constraints); + tv->default_value = Py_XNewRef(default_value); + tv->evaluate_default = NULL; tv->covariant = covariant; tv->contravariant = contravariant; @@ -328,6 +653,7 @@ typevar.__new__ as typevar_new name: object(subclass_of="&PyUnicode_Type") *constraints: object bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -337,9 +663,9 @@ Create a TypeVar. static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance) -/*[clinic end generated code: output=1d200450ee99226d input=41ae33a916bfe76f]*/ + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance) +/*[clinic end generated code: output=d2b248ff074eaab6 input=836f97f631d7293a]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, @@ -386,6 +712,7 @@ typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, PyObject *tv = (PyObject *)typevar_alloc(name, bound, NULL, constraints, NULL, + default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); @@ -410,6 +737,66 @@ typevar_typing_subst(typevarobject *self, PyObject *arg) return result; } +/*[clinic input] +typevar.__typing_prepare_subst__ as typevar_typing_prepare_subst + + alias: object + args: object + / + +[clinic start generated code]*/ + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args) +/*[clinic end generated code: output=82c3f4691e0ded22 input=201a750415d14ffb]*/ +{ + PyObject *params = PyObject_GetAttrString(alias, "__parameters__"); + if (params == NULL) { + return NULL; + } + Py_ssize_t i = PySequence_Index(params, (PyObject *)self); + if (i == -1) { + Py_DECREF(params); + return NULL; + } + Py_ssize_t args_len = PySequence_Length(args); + if (args_len == -1) { + Py_DECREF(params); + return NULL; + } + if (i < args_len) { + // We already have a value for our TypeVar + Py_DECREF(params); + return Py_NewRef(args); + } + else if (i == args_len) { + // If the TypeVar has a default, use it. + PyObject *dflt = typevar_default(self, NULL); + if (dflt == NULL) { + Py_DECREF(params); + return NULL; + } + if (dflt != &_Py_NoDefaultStruct) { + PyObject *new_args = PyTuple_Pack(1, dflt); + Py_DECREF(dflt); + if (new_args == NULL) { + Py_DECREF(params); + return NULL; + } + PyObject *result = PySequence_Concat(args, new_args); + Py_DECREF(params); + Py_DECREF(new_args); + return result; + } + } + Py_DECREF(params); + PyErr_Format(PyExc_TypeError, + "Too few arguments for %S; actual %d, expected at least %d", + alias, args_len, i + 1); + return NULL; +} + /*[clinic input] typevar.__reduce__ as typevar_reduce @@ -422,6 +809,23 @@ typevar_reduce_impl(typevarobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevar.has_default as typevar_has_default + +[clinic start generated code]*/ + +static PyObject * +typevar_has_default_impl(typevarobject *self) +/*[clinic end generated code: output=76bf0b8dc98b97dd input=31024aa030761cf6]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevar_mro_entries(PyObject *self, PyObject *args) { @@ -432,7 +836,9 @@ typevar_mro_entries(PyObject *self, PyObject *args) static PyMethodDef typevar_methods[] = { TYPEVAR_TYPING_SUBST_METHODDEF + TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF TYPEVAR_REDUCE_METHODDEF + TYPEVAR_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevar_mro_entries, METH_O}, {0} }; @@ -457,12 +863,18 @@ variables::\n\ class StrOrBytesSequence[A: (str, bytes)]:\n\ ...\n\ \n\ +Type variables can also have defaults:\n\ +\n\ + class IntDefault[T = int]:\n\ + ...\n\ +\n\ However, if desired, reusable type variables can also be constructed\n\ manually, like so::\n\ \n\ T = TypeVar('T') # Can be anything\n\ S = TypeVar('S', bound=str) # Can be any subtype of str\n\ A = TypeVar('A', str, bytes) # Must be exactly str or bytes\n\ + D = TypeVar('D', default=int) # Defaults to int\n\ \n\ Type variables exist primarily for the benefit of static type\n\ checkers. They serve as the parameters for generic types as well\n\ @@ -739,6 +1151,8 @@ paramspec_dealloc(PyObject *self) Py_DECREF(ps->name); Py_XDECREF(ps->bound); + Py_XDECREF(ps->default_value); + Py_XDECREF(ps->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -752,6 +1166,8 @@ paramspec_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); paramspecobject *ps = (paramspecobject *)self; Py_VISIT(ps->bound); + Py_VISIT(ps->default_value); + Py_VISIT(ps->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -760,6 +1176,8 @@ static int paramspec_clear(paramspecobject *self) { Py_CLEAR(self->bound); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -800,14 +1218,42 @@ paramspec_kwargs(PyObject *self, void *unused) return (PyObject *)paramspecattr_new(tp, self); } +static PyObject * +paramspec_default(paramspecobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + +static PyObject * +paramspec_evaluate_default(paramspecobject *self, void *unused) +{ + if (self->evaluate_default != NULL) { + return Py_NewRef(self->evaluate_default); + } + if (self->default_value != NULL) { + return constevaluator_alloc(self->default_value); + } + Py_RETURN_NONE; +} + static PyGetSetDef paramspec_getset[] = { {"args", (getter)paramspec_args, NULL, PyDoc_STR("Represents positional arguments."), NULL}, {"kwargs", (getter)paramspec_kwargs, NULL, PyDoc_STR("Represents keyword arguments."), NULL}, + {"__default__", (getter)paramspec_default, NULL, "The default value for this ParamSpec.", NULL}, + {"evaluate_default", (getter)paramspec_evaluate_default, NULL, NULL, NULL}, {0}, }; static paramspecobject * -paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, +paramspec_alloc(PyObject *name, PyObject *bound, PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspec_type; @@ -820,6 +1266,8 @@ paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, ps->covariant = covariant; ps->contravariant = contravariant; ps->infer_variance = infer_variance; + ps->default_value = Py_XNewRef(default_value); + ps->evaluate_default = NULL; _PyObject_GC_TRACK(ps); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)ps, "__module__", module) < 0) { @@ -837,6 +1285,7 @@ paramspec.__new__ as paramspec_new name: object(subclass_of="&PyUnicode_Type") * bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -846,8 +1295,9 @@ Create a ParamSpec object. static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance) -/*[clinic end generated code: output=fd2daab79cba62da input=57c49c581979b952]*/ + PyObject *default_value, int covariant, int contravariant, + int infer_variance) +/*[clinic end generated code: output=47ca9d63fa5a094d input=495e1565bc067ab9]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, "Bivariant types are not supported."); @@ -869,7 +1319,7 @@ paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, return NULL; } PyObject *ps = (PyObject *)paramspec_alloc( - name, bound, covariant, contravariant, infer_variance, module); + name, bound, default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); Py_DECREF(module); return ps; @@ -925,6 +1375,22 @@ paramspec_reduce_impl(paramspecobject *self) return Py_NewRef(self->name); } +/*[clinic input] +paramspec.has_default as paramspec_has_default + +[clinic start generated code]*/ + +static PyObject * +paramspec_has_default_impl(paramspecobject *self) +/*[clinic end generated code: output=daaae7467a6a4368 input=2112e97eeb76cd59]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * paramspec_mro_entries(PyObject *self, PyObject *args) { @@ -936,6 +1402,7 @@ paramspec_mro_entries(PyObject *self, PyObject *args) static PyMethodDef paramspec_methods[] = { PARAMSPEC_TYPING_SUBST_METHODDEF PARAMSPEC_TYPING_PREPARE_SUBST_METHODDEF + PARAMSPEC_HAS_DEFAULT_METHODDEF PARAMSPEC_REDUCE_METHODDEF {"__mro_entries__", paramspec_mro_entries, METH_O}, {0} @@ -950,10 +1417,17 @@ where the use of '**' creates a parameter specification::\n\ \n\ type IntFunc[**P] = Callable[P, int]\n\ \n\ +The following syntax creates a parameter specification that defaults\n\ +to a callable accepting two positional-only arguments of types int\n\ +and str:\n\ +\n\ + type IntFuncDefault[**P = (int, str)] = Callable[P, int]\n\ +\n\ For compatibility with Python 3.11 and earlier, ParamSpec objects\n\ can also be created as follows::\n\ \n\ P = ParamSpec('P')\n\ + DefaultP = ParamSpec('DefaultP', default=(int, str))\n\ \n\ Parameter specification variables exist primarily for the benefit of\n\ static type checkers. They are used to forward the parameter types of\n\ @@ -1021,6 +1495,8 @@ typevartuple_dealloc(PyObject *self) typevartupleobject *tvt = (typevartupleobject *)self; Py_DECREF(tvt->name); + Py_XDECREF(tvt->default_value); + Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -1060,7 +1536,7 @@ static PyMemberDef typevartuple_members[] = { }; static typevartupleobject * -typevartuple_alloc(PyObject *name, PyObject *module) +typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; typevartupleobject *tvt = PyObject_GC_New(typevartupleobject, tp); @@ -1068,6 +1544,8 @@ typevartuple_alloc(PyObject *name, PyObject *module) return NULL; } tvt->name = Py_NewRef(name); + tvt->default_value = Py_XNewRef(default_value); + tvt->evaluate_default = NULL; _PyObject_GC_TRACK(tvt); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)tvt, "__module__", module) < 0) { @@ -1083,19 +1561,22 @@ typevartuple_alloc(PyObject *name, PyObject *module) typevartuple.__new__ name: object(subclass_of="&PyUnicode_Type") + * + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault Create a new TypeVarTuple with the given name. [clinic start generated code]*/ static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name) -/*[clinic end generated code: output=09d417a28f976202 input=00d28abcf1fc96bb]*/ +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value) +/*[clinic end generated code: output=9d6b76dfe95aae51 input=e149739929a866d0]*/ { PyObject *module = caller(); if (module == NULL) { return NULL; } - PyObject *result = (PyObject *)typevartuple_alloc(name, module); + PyObject *result = (PyObject *)typevartuple_alloc(name, module, default_value); Py_DECREF(module); return result; } @@ -1148,6 +1629,23 @@ typevartuple_reduce_impl(typevartupleobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevartuple.has_default as typevartuple_has_default + +[clinic start generated code]*/ + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self) +/*[clinic end generated code: output=4895f602f56a5e29 input=9ef3250ddb2c1851]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevartuple_mro_entries(PyObject *self, PyObject *args) { @@ -1160,6 +1658,8 @@ static int typevartuple_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); + Py_VISIT(((typevartupleobject *)self)->default_value); + Py_VISIT(((typevartupleobject *)self)->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -1167,14 +1667,49 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) static int typevartuple_clear(PyObject *self) { + Py_CLEAR(((typevartupleobject *)self)->default_value); + Py_CLEAR(((typevartupleobject *)self)->evaluate_default); PyObject_ClearManagedDict(self); return 0; } +static PyObject * +typevartuple_default(typevartupleobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + +static PyObject * +typevartuple_evaluate_default(typevartupleobject *self, void *unused) +{ + if (self->evaluate_default != NULL) { + return Py_NewRef(self->evaluate_default); + } + if (self->default_value != NULL) { + return constevaluator_alloc(self->default_value); + } + Py_RETURN_NONE; +} + +static PyGetSetDef typevartuple_getset[] = { + {"__default__", (getter)typevartuple_default, NULL, "The default value for this TypeVarTuple.", NULL}, + {"evaluate_default", (getter)typevartuple_evaluate_default, NULL, NULL, NULL}, + {0}, +}; + static PyMethodDef typevartuple_methods[] = { TYPEVARTUPLE_TYPING_SUBST_METHODDEF TYPEVARTUPLE_TYPING_PREPARE_SUBST_METHODDEF TYPEVARTUPLE_REDUCE_METHODDEF + TYPEVARTUPLE_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevartuple_mro_entries, METH_O}, {0} }; @@ -1190,10 +1725,15 @@ where a single '*' indicates a type variable tuple::\n\ def move_first_element_to_last[T, *Ts](tup: tuple[T, *Ts]) -> tuple[*Ts, T]:\n\ return (*tup[1:], tup[0])\n\ \n\ +Type variables tuples can have default values:\n\ +\n\ + type AliasWithDefault[*Ts = (str, int)] = tuple[*Ts]\n\ +\n\ For compatibility with Python 3.11 and earlier, TypeVarTuple objects\n\ can also be created as follows::\n\ \n\ Ts = TypeVarTuple('Ts') # Can be given any name\n\ + DefaultTs = TypeVarTuple('Ts', default=(str, int))\n\ \n\ Just as a TypeVar (type variable) is a placeholder for a single type,\n\ a TypeVarTuple is a placeholder for an *arbitrary* number of types. For\n\ @@ -1218,6 +1758,7 @@ PyType_Slot typevartuple_slots[] = { {Py_tp_doc, (void *)typevartuple_doc}, {Py_tp_members, typevartuple_members}, {Py_tp_methods, typevartuple_methods}, + {Py_tp_getset, typevartuple_getset}, {Py_tp_new, typevartuple}, {Py_tp_iter, typevartuple_iter}, {Py_tp_repr, typevartuple_repr}, @@ -1241,21 +1782,39 @@ PyObject * _Py_make_typevar(PyObject *name, PyObject *evaluate_bound, PyObject *evaluate_constraints) { return (PyObject *)typevar_alloc(name, NULL, evaluate_bound, NULL, evaluate_constraints, - false, false, true, NULL); + NULL, false, false, true, NULL); } PyObject * _Py_make_paramspec(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)paramspec_alloc(v, NULL, false, false, true, NULL); + return (PyObject *)paramspec_alloc(v, NULL, NULL, false, false, true, NULL); } PyObject * _Py_make_typevartuple(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)typevartuple_alloc(v, NULL); + return (PyObject *)typevartuple_alloc(v, NULL, NULL); +} + +static PyObject * +get_type_param_default(PyThreadState *ts, PyObject *typeparam) { + // Does not modify refcount of existing objects. + if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { + return typevar_default((typevarobject *)typeparam, NULL); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { + return paramspec_default((paramspecobject *)typeparam, NULL); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { + return typevartuple_default((typevartupleobject *)typeparam, NULL); + } + else { + PyErr_Format(PyExc_TypeError, "Expected a type param, got %R", typeparam); + return NULL; + } } static void @@ -1306,6 +1865,17 @@ typealias_value(PyObject *self, void *unused) return typealias_get_value(ta); } +static PyObject * +typealias_evaluate_value(PyObject *self, void *unused) +{ + typealiasobject *ta = (typealiasobject *)self; + if (ta->compute_value != NULL) { + return Py_NewRef(ta->compute_value); + } + assert(ta->value != NULL); + return constevaluator_alloc(ta->value); +} + static PyObject * typealias_parameters(PyObject *self, void *unused) { @@ -1349,10 +1919,70 @@ static PyGetSetDef typealias_getset[] = { {"__parameters__", typealias_parameters, (setter)NULL, NULL, NULL}, {"__type_params__", typealias_type_params, (setter)NULL, NULL, NULL}, {"__value__", typealias_value, (setter)NULL, NULL, NULL}, + {"evaluate_value", typealias_evaluate_value, (setter)NULL, NULL, NULL}, {"__module__", typealias_module, (setter)NULL, NULL, NULL}, {0} }; +static PyObject * +typealias_check_type_params(PyObject *type_params, int *err) { + // Can return type_params or NULL without exception set. + // Does not change the reference count of type_params, + // sets `*err` to 1 when error happens and sets an exception, + // otherwise `*err` is set to 0. + *err = 0; + if (type_params == NULL) { + return NULL; + } + + assert(PyTuple_Check(type_params)); + Py_ssize_t length = PyTuple_GET_SIZE(type_params); + if (!length) { // 0-length tuples are the same as `NULL`. + return NULL; + } + + PyThreadState *ts = _PyThreadState_GET(); + int default_seen = 0; + for (Py_ssize_t index = 0; index < length; index++) { + PyObject *type_param = PyTuple_GET_ITEM(type_params, index); + PyObject *dflt = get_type_param_default(ts, type_param); + if (dflt == NULL) { + *err = 1; + return NULL; + } + if (dflt == &_Py_NoDefaultStruct) { + if (default_seen) { + *err = 1; + PyErr_Format(PyExc_TypeError, + "non-default type parameter '%R' " + "follows default type parameter", + type_param); + return NULL; + } + } else { + default_seen = 1; + Py_DECREF(dflt); + } + } + + return type_params; +} + +static PyObject * +typelias_convert_type_params(PyObject *type_params) +{ + if ( + type_params == NULL + || Py_IsNone(type_params) + || (PyTuple_Check(type_params) && PyTuple_GET_SIZE(type_params) == 0) + ) { + return NULL; + } + else { + return type_params; + } +} + static typealiasobject * typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, PyObject *value, PyObject *module) @@ -1362,7 +1992,7 @@ typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, return NULL; } ta->name = Py_NewRef(name); - ta->type_params = Py_IsNone(type_params) ? NULL : Py_XNewRef(type_params); + ta->type_params = Py_XNewRef(type_params); ta->compute_value = Py_XNewRef(compute_value); ta->value = Py_XNewRef(value); ta->module = Py_XNewRef(module); @@ -1440,11 +2070,18 @@ typealias_new_impl(PyTypeObject *type, PyObject *name, PyObject *value, PyErr_SetString(PyExc_TypeError, "type_params must be a tuple"); return NULL; } + + int err = 0; + PyObject *checked_params = typealias_check_type_params(type_params, &err); + if (err) { + return NULL; + } + PyObject *module = caller(); if (module == NULL) { return NULL; } - PyObject *ta = (PyObject *)typealias_alloc(name, type_params, NULL, value, + PyObject *ta = (PyObject *)typealias_alloc(name, checked_params, NULL, value, module); Py_DECREF(module); return ta; @@ -1510,7 +2147,7 @@ _Py_make_typealias(PyThreadState* unused, PyObject *args) assert(PyTuple_GET_SIZE(args) == 3); PyObject *name = PyTuple_GET_ITEM(args, 0); assert(PyUnicode_Check(name)); - PyObject *type_params = PyTuple_GET_ITEM(args, 1); + PyObject *type_params = typelias_convert_type_params(PyTuple_GET_ITEM(args, 1)); PyObject *compute_value = PyTuple_GET_ITEM(args, 2); assert(PyFunction_Check(compute_value)); return (PyObject *)typealias_alloc(name, type_params, compute_value, NULL, NULL); @@ -1674,6 +2311,7 @@ int _Py_initialize_generic(PyInterpreterState *interp) MAKE_TYPE(paramspec); MAKE_TYPE(paramspecargs); MAKE_TYPE(paramspeckwargs); + MAKE_TYPE(constevaluator); #undef MAKE_TYPE return 0; } @@ -1686,4 +2324,26 @@ void _Py_clear_generic_types(PyInterpreterState *interp) Py_CLEAR(interp->cached_objects.paramspec_type); Py_CLEAR(interp->cached_objects.paramspecargs_type); Py_CLEAR(interp->cached_objects.paramspeckwargs_type); + Py_CLEAR(interp->cached_objects.constevaluator_type); +} + +PyObject * +_Py_set_typeparam_default(PyThreadState *ts, PyObject *typeparam, PyObject *evaluate_default) +{ + if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { + Py_XSETREF(((typevarobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { + Py_XSETREF(((paramspecobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { + Py_XSETREF(((typevartupleobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else { + PyErr_Format(PyExc_TypeError, "Expected a type param, got %R", typeparam); + return NULL; + } } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0a569a950e88e2..9cd9781e412524 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -44,7 +44,9 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_bytesobject.h" // _PyBytes_Repeat() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_codecs.h" // _PyCodec_Lookup() +#include "pycore_critical_section.h" // Py_*_CRITICAL_SECTION_SEQUENCE_FAST #include "pycore_format.h" // F_LJUST +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.fs_codec #include "pycore_long.h" // _PyLong_FormatWriter() @@ -176,10 +178,7 @@ NOTE: In the interpreter's initialization phase, some globals are currently *_to++ = (to_type) *_iter++; \ } while (0) -#define LATIN1(ch) \ - (ch < 128 \ - ? (PyObject*)&_Py_SINGLETON(strings).ascii[ch] \ - : (PyObject*)&_Py_SINGLETON(strings).latin1[ch - 128]) +#define LATIN1 _Py_LATIN1_CHR #ifdef MS_WINDOWS /* On Windows, overallocate by 50% is the best factor */ @@ -201,6 +200,11 @@ static PyObject * unicode_decode_utf8(const char *s, Py_ssize_t size, _Py_error_handler error_handler, const char *errors, Py_ssize_t *consumed); +static int +unicode_decode_utf8_writer(_PyUnicodeWriter *writer, + const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed); #ifdef Py_DEBUG static inline int unicode_is_finalizing(void); static int unicode_is_singleton(PyObject *unicode); @@ -214,18 +218,20 @@ static inline PyObject* unicode_get_empty(void) return &_Py_STR(empty); } -/* This dictionary holds all interned unicode strings. Note that references - to strings in this dictionary are *not* counted in the string's ob_refcnt. - When the interned string reaches a refcnt of 0 the string deallocation - function will delete the reference from this dictionary. -*/ +/* This dictionary holds per-interpreter interned strings. + * See InternalDocs/string_interning.md for details. + */ static inline PyObject *get_interned_dict(PyInterpreterState *interp) { return _Py_INTERP_CACHED_OBJECT(interp, interned_strings); } +/* This hashtable holds statically allocated interned strings. + * See InternalDocs/string_interning.md for details. + */ #define INTERNED_STRINGS _PyRuntime.cached_objects.interned_strings +/* Get number of all interned strings for the current interpreter. */ Py_ssize_t _PyUnicode_InternedSize(void) { @@ -233,8 +239,29 @@ _PyUnicode_InternedSize(void) return _Py_hashtable_len(INTERNED_STRINGS) + PyDict_GET_SIZE(dict); } +/* Get number of immortal interned strings for the current interpreter. */ +Py_ssize_t +_PyUnicode_InternedSize_Immortal(void) +{ + PyObject *dict = get_interned_dict(_PyInterpreterState_GET()); + PyObject *key, *value; + Py_ssize_t pos = 0; + Py_ssize_t count = 0; + + // It's tempting to keep a count and avoid a loop here. But, this function + // is intended for refleak tests. It spends extra work to report the true + // value, to help detect bugs in optimizations. + + while (PyDict_Next(dict, &pos, &key, &value)) { + assert(PyUnicode_CHECK_INTERNED(key) != SSTATE_INTERNED_IMMORTAL_STATIC); + if (PyUnicode_CHECK_INTERNED(key) == SSTATE_INTERNED_IMMORTAL) { + count++; + } + } + return _Py_hashtable_len(INTERNED_STRINGS) + count; +} + static Py_hash_t unicode_hash(PyObject *); -static int unicode_compare_eq(PyObject *, PyObject *); static Py_uhash_t hashtable_unicode_hash(const void *key) @@ -248,35 +275,45 @@ hashtable_unicode_compare(const void *key1, const void *key2) PyObject *obj1 = (PyObject *)key1; PyObject *obj2 = (PyObject *)key2; if (obj1 != NULL && obj2 != NULL) { - return unicode_compare_eq(obj1, obj2); + return unicode_eq(obj1, obj2); } else { return obj1 == obj2; } } +/* Return true if this interpreter should share the main interpreter's + intern_dict. That's important for interpreters which load basic + single-phase init extension modules (m_size == -1). There could be interned + immortal strings that are shared between interpreters, due to the + PyDict_Update(mdict, m_copy) call in import_find_extension(). + + It's not safe to deallocate those strings until all interpreters that + potentially use them are freed. By storing them in the main interpreter, we + ensure they get freed after all other interpreters are freed. +*/ +static bool +has_shared_intern_dict(PyInterpreterState *interp) +{ + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + return interp != main_interp && interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC; +} + static int init_interned_dict(PyInterpreterState *interp) { - if (_Py_IsMainInterpreter(interp)) { - assert(INTERNED_STRINGS == NULL); - _Py_hashtable_allocator_t hashtable_alloc = {PyMem_RawMalloc, PyMem_RawFree}; - INTERNED_STRINGS = _Py_hashtable_new_full( - hashtable_unicode_hash, - hashtable_unicode_compare, - NULL, - NULL, - &hashtable_alloc - ); - if (INTERNED_STRINGS == NULL) { + assert(get_interned_dict(interp) == NULL); + PyObject *interned; + if (has_shared_intern_dict(interp)) { + interned = get_interned_dict(_PyInterpreterState_Main()); + Py_INCREF(interned); + } + else { + interned = PyDict_New(); + if (interned == NULL) { return -1; } } - assert(get_interned_dict(interp) == NULL); - PyObject *interned = interned = PyDict_New(); - if (interned == NULL) { - return -1; - } _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = interned; return 0; } @@ -286,11 +323,62 @@ clear_interned_dict(PyInterpreterState *interp) { PyObject *interned = get_interned_dict(interp); if (interned != NULL) { - PyDict_Clear(interned); + if (!has_shared_intern_dict(interp)) { + // only clear if the dict belongs to this interpreter + PyDict_Clear(interned); + } Py_DECREF(interned); _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = NULL; } - if (_Py_IsMainInterpreter(interp) && INTERNED_STRINGS != NULL) { +} + +static PyStatus +init_global_interned_strings(PyInterpreterState *interp) +{ + assert(INTERNED_STRINGS == NULL); + _Py_hashtable_allocator_t hashtable_alloc = {PyMem_RawMalloc, PyMem_RawFree}; + + INTERNED_STRINGS = _Py_hashtable_new_full( + hashtable_unicode_hash, + hashtable_unicode_compare, + // Objects stored here are immortal and statically allocated, + // so we don't need key_destroy_func & value_destroy_func: + NULL, + NULL, + &hashtable_alloc + ); + if (INTERNED_STRINGS == NULL) { + PyErr_Clear(); + return _PyStatus_ERR("failed to create global interned dict"); + } + + /* Intern statically allocated string identifiers, deepfreeze strings, + * and one-byte latin-1 strings. + * This must be done before any module initialization so that statically + * allocated string identifiers are used instead of heap allocated strings. + * Deepfreeze uses the interned identifiers if present to save space + * else generates them and they are interned to speed up dict lookups. + */ + _PyUnicode_InitStaticStrings(interp); + + for (int i = 0; i < 256; i++) { + PyObject *s = LATIN1(i); + _PyUnicode_InternStatic(interp, &s); + assert(s == LATIN1(i)); + } +#ifdef Py_DEBUG + assert(_PyUnicode_CheckConsistency(&_Py_STR(empty), 1)); + + for (int i = 0; i < 256; i++) { + assert(_PyUnicode_CheckConsistency(LATIN1(i), 1)); + } +#endif + return _PyStatus_OK(); +} + +static void clear_global_interned_strings(void) +{ + if (INTERNED_STRINGS != NULL) { _Py_hashtable_destroy(INTERNED_STRINGS); INTERNED_STRINGS = NULL; } @@ -505,7 +593,7 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) /* Disable checks during Python finalization. For example, it allows to call _PyObject_Dump() during finalization for debugging purpose. */ - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; } @@ -623,6 +711,41 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) } CHECK(PyUnicode_READ(kind, data, ascii->length) == 0); } + + /* Check interning state */ +#ifdef Py_DEBUG + // Note that we do not check `_Py_IsImmortal(op)`, since stable ABI + // extensions can make immortal strings mortal (but with a high enough + // refcount). + // The other way is extremely unlikely (worth a potential failed assertion + // in a debug build), so we do check `!_Py_IsImmortal(op)`. + switch (PyUnicode_CHECK_INTERNED(op)) { + case SSTATE_NOT_INTERNED: + if (ascii->state.statically_allocated) { + // This state is for two exceptions: + // - strings are currently checked before they're interned + // - the 256 one-latin1-character strings + // are static but use SSTATE_NOT_INTERNED + } + else { + CHECK(!_Py_IsImmortal(op)); + } + break; + case SSTATE_INTERNED_MORTAL: + CHECK(!ascii->state.statically_allocated); + CHECK(!_Py_IsImmortal(op)); + break; + case SSTATE_INTERNED_IMMORTAL: + CHECK(!ascii->state.statically_allocated); + break; + case SSTATE_INTERNED_IMMORTAL_STATIC: + CHECK(ascii->state.statically_allocated); + break; + default: + Py_UNREACHABLE(); + } +#endif + return 1; #undef CHECK @@ -893,6 +1016,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -903,6 +1027,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -913,6 +1038,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -1276,46 +1402,6 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) return obj; } -#if SIZEOF_WCHAR_T == 2 -/* Helper function to convert a 16-bits wchar_t representation to UCS4, this - will decode surrogate pairs, the other conversions are implemented as macros - for efficiency. - - This function assumes that unicode can hold one more code point than wstr - characters for a terminating null character. */ -static void -unicode_convert_wchar_to_ucs4(const wchar_t *begin, const wchar_t *end, - PyObject *unicode) -{ - const wchar_t *iter; - Py_UCS4 *ucs4_out; - - assert(unicode != NULL); - assert(_PyUnicode_CHECK(unicode)); - assert(_PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); - ucs4_out = PyUnicode_4BYTE_DATA(unicode); - - for (iter = begin; iter < end; ) { - assert(ucs4_out < (PyUnicode_4BYTE_DATA(unicode) + - _PyUnicode_GET_LENGTH(unicode))); - if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) - && (iter+1) < end - && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) - { - *ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); - iter += 2; - } - else { - *ucs4_out++ = *iter; - iter++; - } - } - assert(ucs4_out == (PyUnicode_4BYTE_DATA(unicode) + - _PyUnicode_GET_LENGTH(unicode))); - -} -#endif - static int unicode_check_modifiable(PyObject *unicode) { @@ -1579,16 +1665,74 @@ unicode_dealloc(PyObject *unicode) _Py_FatalRefcountError("deallocating an Unicode singleton"); } #endif - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref an immortal string out of existence. Since - * the string is an immortal object, just re-set the reference count. - */ - if (PyUnicode_CHECK_INTERNED(unicode) - || _PyUnicode_STATE(unicode).statically_allocated) - { + if (_PyUnicode_STATE(unicode).statically_allocated) { + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref an immortal string out of existence. Since + * the string is an immortal object, just re-set the reference count. + */ +#ifdef Py_DEBUG + Py_UNREACHABLE(); +#endif _Py_SetImmortal(unicode); return; } + switch (_PyUnicode_STATE(unicode).interned) { + case SSTATE_NOT_INTERNED: + break; + case SSTATE_INTERNED_MORTAL: + /* Remove the object from the intern dict. + * Before doing so, we set the refcount to 2: the key and value + * in the interned_dict. + */ + assert(Py_REFCNT(unicode) == 0); + Py_SET_REFCNT(unicode, 2); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_IncRefTotal(_PyThreadState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *interned = get_interned_dict(interp); + assert(interned != NULL); + PyObject *popped; + int r = PyDict_Pop(interned, unicode, &popped); + if (r == -1) { + PyErr_WriteUnraisable(unicode); + // We don't know what happened to the string. It's probably + // best to leak it: + // - if it was popped, there are no more references to it + // so it can't cause trouble (except wasted memory) + // - if it wasn't popped, it'll remain interned + _Py_SetImmortal(unicode); + _PyUnicode_STATE(unicode).interned = SSTATE_INTERNED_IMMORTAL; + return; + } + if (r == 0) { + // The interned string was not found in the interned_dict. +#ifdef Py_DEBUG + Py_UNREACHABLE(); +#endif + _Py_SetImmortal(unicode); + return; + } + // Successfully popped. + assert(popped == unicode); + // Only our `popped` reference should be left; remove it too. + assert(Py_REFCNT(unicode) == 1); + Py_SET_REFCNT(unicode, 0); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_DecRefTotal(_PyThreadState_GET()); +#endif + break; + default: + // As with `statically_allocated` above. +#ifdef Py_REF_DEBUG + Py_UNREACHABLE(); +#endif + _Py_SetImmortal(unicode); + return; + } if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); } @@ -1624,7 +1768,7 @@ unicode_modifiable(PyObject *unicode) assert(_PyUnicode_CHECK(unicode)); if (Py_REFCNT(unicode) != 1) return 0; - if (_PyUnicode_HASH(unicode) != -1) + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(unicode)) != -1) return 0; if (PyUnicode_CHECK_INTERNED(unicode)) return 0; @@ -1751,7 +1895,6 @@ static PyObject* get_latin1_char(Py_UCS1 ch) { PyObject *o = LATIN1(ch); - assert(_Py_IsImmortal(o)); return o; } @@ -1781,6 +1924,62 @@ unicode_char(Py_UCS4 ch) return unicode; } + +static inline void +unicode_write_widechar(int kind, void *data, + const wchar_t *u, Py_ssize_t size, + Py_ssize_t num_surrogates) +{ + switch (kind) { + case PyUnicode_1BYTE_KIND: + _PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, u, u + size, data); + break; + + case PyUnicode_2BYTE_KIND: +#if SIZEOF_WCHAR_T == 2 + memcpy(data, u, size * 2); +#else + _PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, u, u + size, data); +#endif + break; + + case PyUnicode_4BYTE_KIND: + { +#if SIZEOF_WCHAR_T == 2 + // Convert a 16-bits wchar_t representation to UCS4, this will decode + // surrogate pairs. + const wchar_t *end = u + size; + Py_UCS4 *ucs4_out = (Py_UCS4*)data; +# ifndef NDEBUG + Py_UCS4 *ucs4_end = (Py_UCS4*)data + (size - num_surrogates); +# endif + for (const wchar_t *iter = u; iter < end; ) { + assert(ucs4_out < ucs4_end); + if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) + && (iter+1) < end + && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) + { + *ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); + iter += 2; + } + else { + *ucs4_out++ = *iter; + iter++; + } + } + assert(ucs4_out == ucs4_end); +#else + assert(num_surrogates == 0); + memcpy(data, u, size * 4); +#endif + break; + } + default: + Py_UNREACHABLE(); + } +} + + PyObject * PyUnicode_FromWideChar(const wchar_t *u, Py_ssize_t size) { @@ -1833,36 +2032,63 @@ PyUnicode_FromWideChar(const wchar_t *u, Py_ssize_t size) if (!unicode) return NULL; - switch (PyUnicode_KIND(unicode)) { - case PyUnicode_1BYTE_KIND: - _PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, - u, u + size, PyUnicode_1BYTE_DATA(unicode)); - break; - case PyUnicode_2BYTE_KIND: -#if Py_UNICODE_SIZE == 2 - memcpy(PyUnicode_2BYTE_DATA(unicode), u, size * 2); -#else - _PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, - u, u + size, PyUnicode_2BYTE_DATA(unicode)); -#endif - break; - case PyUnicode_4BYTE_KIND: -#if SIZEOF_WCHAR_T == 2 - /* This is the only case which has to process surrogates, thus - a simple copy loop is not enough and we need a function. */ - unicode_convert_wchar_to_ucs4(u, u + size, unicode); -#else - assert(num_surrogates == 0); - memcpy(PyUnicode_4BYTE_DATA(unicode), u, size * 4); + unicode_write_widechar(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode), + u, size, num_surrogates); + + return unicode_result(unicode); +} + + +int +PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *pub_writer, + const wchar_t *str, + Py_ssize_t size) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + if (size < 0) { + size = wcslen(str); + } + + if (size == 0) { + return 0; + } + +#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION + /* Oracle Solaris uses non-Unicode internal wchar_t form for + non-Unicode locales and hence needs conversion to UCS-4 first. */ + if (_Py_LocaleUsesNonUnicodeWchar()) { + wchar_t* converted = _Py_DecodeNonUnicodeWchar(str, size); + if (!converted) { + return -1; + } + + int res = PyUnicodeWriter_WriteUCS4(pub_writer, converted, size); + PyMem_Free(converted); + return res; + } #endif - break; - default: - Py_UNREACHABLE(); + + Py_UCS4 maxchar = 0; + Py_ssize_t num_surrogates; + if (find_maxchar_surrogates(str, str + size, + &maxchar, &num_surrogates) == -1) { + return -1; } - return unicode_result(unicode); + if (_PyUnicodeWriter_Prepare(writer, size - num_surrogates, maxchar) < 0) { + return -1; + } + + int kind = writer->kind; + void *data = (Py_UCS1*)writer->data + writer->pos * kind; + unicode_write_widechar(kind, data, str, size, num_surrogates); + + writer->pos += size - num_surrogates; + return 0; } + PyObject * PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) { @@ -1933,7 +2159,7 @@ _PyUnicode_FromId(_Py_Identifier *id) if (!obj) { goto end; } - PyUnicode_InternInPlace(&obj); + _PyUnicode_InternImmortal(interp, &obj); if (index >= ids->size) { // Overallocate to reduce the number of realloc @@ -2088,6 +2314,51 @@ _PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size) return res; } + +int +PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *pub_writer, + Py_UCS4 *str, + Py_ssize_t size) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)pub_writer; + + if (size < 0) { + PyErr_SetString(PyExc_ValueError, + "size must be positive"); + return -1; + } + + if (size == 0) { + return 0; + } + + Py_UCS4 max_char = ucs4lib_find_max_char(str, str + size); + + if (_PyUnicodeWriter_Prepare(writer, size, max_char) < 0) { + return -1; + } + + int kind = writer->kind; + void *data = (Py_UCS1*)writer->data + writer->pos * kind; + if (kind == PyUnicode_1BYTE_KIND) { + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS1, + str, str + size, + data); + } + else if (kind == PyUnicode_2BYTE_KIND) { + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS2, + str, str + size, + data); + } + else { + memcpy(data, str, size * sizeof(Py_UCS4)); + } + writer->pos += size; + + return 0; +} + + PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, Py_ssize_t size) { @@ -2376,14 +2647,12 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, } static int -unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, +unicode_fromformat_write_utf8(_PyUnicodeWriter *writer, const char *str, Py_ssize_t width, Py_ssize_t precision, int flags) { /* UTF-8 */ + Py_ssize_t *pconsumed = NULL; Py_ssize_t length; - PyObject *unicode; - int res; - if (precision == -1) { length = strlen(str); } @@ -2392,12 +2661,28 @@ unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, while (length < precision && str[length]) { length++; } + if (length == precision) { + /* The input string is not NUL-terminated. If it ends with an + * incomplete UTF-8 sequence, truncate the string just before it. + * Incomplete sequences in the middle and sequences which cannot + * be valid prefixes are still treated as errors and replaced + * with \xfffd. */ + pconsumed = &length; + } + } + + if (width < 0) { + return unicode_decode_utf8_writer(writer, str, length, + _Py_ERROR_REPLACE, "replace", pconsumed); } - unicode = PyUnicode_DecodeUTF8Stateful(str, length, "replace", NULL); + + PyObject *unicode = PyUnicode_DecodeUTF8Stateful(str, length, + "replace", pconsumed); if (unicode == NULL) return -1; - res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); + int res = unicode_fromformat_write_str(writer, unicode, + width, -1, flags); Py_DECREF(unicode); return res; } @@ -2406,11 +2691,7 @@ static int unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t width, Py_ssize_t precision, int flags) { - /* UTF-8 */ Py_ssize_t length; - PyObject *unicode; - int res; - if (precision == -1) { length = wcslen(str); } @@ -2420,11 +2701,17 @@ unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, length++; } } - unicode = PyUnicode_FromWideChar(str, length); + + if (width < 0) { + return PyUnicodeWriter_WriteWideChar((PyUnicodeWriter*)writer, + str, length); + } + + PyObject *unicode = PyUnicode_FromWideChar(str, length); if (unicode == NULL) return -1; - res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); + int res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); Py_DECREF(unicode); return res; } @@ -2434,11 +2721,6 @@ unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, #define F_SIZE 3 #define F_PTRDIFF 4 #define F_INTMAX 5 -static const char * const formats[] = {"%d", "%ld", "%lld", "%zd", "%td", "%jd"}; -static const char * const formats_o[] = {"%o", "%lo", "%llo", "%zo", "%to", "%jo"}; -static const char * const formats_u[] = {"%u", "%lu", "%llu", "%zu", "%tu", "%ju"}; -static const char * const formats_x[] = {"%x", "%lx", "%llx", "%zx", "%tx", "%jx"}; -static const char * const formats_X[] = {"%X", "%lX", "%llX", "%zX", "%tX", "%jX"}; static const char* unicode_fromformat_arg(_PyUnicodeWriter *writer, @@ -2468,6 +2750,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, switch (*f++) { case '-': flags |= F_LJUST; continue; case '0': flags |= F_ZERO; continue; + case '#': flags |= F_ALT; continue; } f--; break; @@ -2579,47 +2862,44 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, case 'd': case 'i': case 'o': case 'u': case 'x': case 'X': { - /* used by sprintf */ char buffer[MAX_INTMAX_CHARS]; - const char *fmt = NULL; - switch (*f) { - case 'o': fmt = formats_o[sizemod]; break; - case 'u': fmt = formats_u[sizemod]; break; - case 'x': fmt = formats_x[sizemod]; break; - case 'X': fmt = formats_X[sizemod]; break; - default: fmt = formats[sizemod]; break; - } - int issigned = (*f == 'd' || *f == 'i'); + + // Fill buffer using sprinf, with one of many possible format + // strings, like "%llX" for `long long` in hexadecimal. + // The type/size is in `sizemod`; the format is in `*f`. + + // Use macros with nested switches to keep the sprintf format strings + // as compile-time literals, avoiding warnings and maybe allowing + // optimizations. + + // `SPRINT` macro does one sprintf + // Example usage: SPRINT("l", "X", unsigned long) expands to + // sprintf(buffer, "%" "l" "X", va_arg(*vargs, unsigned long)) + #define SPRINT(SIZE_SPEC, FMT_CHAR, TYPE) \ + sprintf(buffer, "%" SIZE_SPEC FMT_CHAR, va_arg(*vargs, TYPE)) + + // One inner switch to handle all format variants + #define DO_SPRINTS(SIZE_SPEC, SIGNED_TYPE, UNSIGNED_TYPE) \ + switch (*f) { \ + case 'o': len = SPRINT(SIZE_SPEC, "o", UNSIGNED_TYPE); break; \ + case 'u': len = SPRINT(SIZE_SPEC, "u", UNSIGNED_TYPE); break; \ + case 'x': len = SPRINT(SIZE_SPEC, "x", UNSIGNED_TYPE); break; \ + case 'X': len = SPRINT(SIZE_SPEC, "X", UNSIGNED_TYPE); break; \ + default: len = SPRINT(SIZE_SPEC, "d", SIGNED_TYPE); break; \ + } + + // Outer switch to handle all the sizes/types switch (sizemod) { - case F_LONG: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, long)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned long)); - break; - case F_LONGLONG: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, long long)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned long long)); - break; - case F_SIZE: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, Py_ssize_t)) : - sprintf(buffer, fmt, va_arg(*vargs, size_t)); - break; - case F_PTRDIFF: - len = sprintf(buffer, fmt, va_arg(*vargs, ptrdiff_t)); - break; - case F_INTMAX: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, intmax_t)) : - sprintf(buffer, fmt, va_arg(*vargs, uintmax_t)); - break; - default: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, int)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned int)); - break; + case F_LONG: DO_SPRINTS("l", long, unsigned long); break; + case F_LONGLONG: DO_SPRINTS("ll", long long, unsigned long long); break; + case F_SIZE: DO_SPRINTS("z", Py_ssize_t, size_t); break; + case F_PTRDIFF: DO_SPRINTS("t", ptrdiff_t, ptrdiff_t); break; + case F_INTMAX: DO_SPRINTS("j", intmax_t, uintmax_t); break; + default: DO_SPRINTS("", int, unsigned int); break; } + #undef SPRINT + #undef DO_SPRINTS + assert(len >= 0); int sign = (buffer[0] == '-'); @@ -2698,7 +2978,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, else { /* UTF-8 */ const char *s = va_arg(*vargs, const char*); - if (unicode_fromformat_write_cstr(writer, s, width, precision, flags) < 0) + if (unicode_fromformat_write_utf8(writer, s, width, precision, flags) < 0) return NULL; } break; @@ -2737,7 +3017,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, } else { assert(str != NULL); - if (unicode_fromformat_write_cstr(writer, str, width, precision, flags) < 0) + if (unicode_fromformat_write_utf8(writer, str, width, precision, flags) < 0) return NULL; } break; @@ -2791,71 +3071,137 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, break; } - default: - invalid_format: - PyErr_Format(PyExc_SystemError, "invalid format string: %s", p); - return NULL; - } - + case 'T': + { + PyObject *obj = va_arg(*vargs, PyObject *); + PyTypeObject *type = (PyTypeObject *)Py_NewRef(Py_TYPE(obj)); + + PyObject *type_name; + if (flags & F_ALT) { + type_name = _PyType_GetFullyQualifiedName(type, ':'); + } + else { + type_name = PyType_GetFullyQualifiedName(type); + } + Py_DECREF(type); + if (!type_name) { + return NULL; + } + + if (unicode_fromformat_write_str(writer, type_name, + width, precision, flags) == -1) { + Py_DECREF(type_name); + return NULL; + } + Py_DECREF(type_name); + break; + } + + case 'N': + { + PyObject *type_raw = va_arg(*vargs, PyObject *); + assert(type_raw != NULL); + + if (!PyType_Check(type_raw)) { + PyErr_SetString(PyExc_TypeError, "%N argument must be a type"); + return NULL; + } + PyTypeObject *type = (PyTypeObject*)type_raw; + + PyObject *type_name; + if (flags & F_ALT) { + type_name = _PyType_GetFullyQualifiedName(type, ':'); + } + else { + type_name = PyType_GetFullyQualifiedName(type); + } + if (!type_name) { + return NULL; + } + if (unicode_fromformat_write_str(writer, type_name, + width, precision, flags) == -1) { + Py_DECREF(type_name); + return NULL; + } + Py_DECREF(type_name); + break; + } + + default: + invalid_format: + PyErr_Format(PyExc_SystemError, "invalid format string: %s", p); + return NULL; + } + f++; return f; } -PyObject * -PyUnicode_FromFormatV(const char *format, va_list vargs) +static int +unicode_from_format(_PyUnicodeWriter *writer, const char *format, va_list vargs) { - va_list vargs2; - const char *f; - _PyUnicodeWriter writer; - - _PyUnicodeWriter_Init(&writer); - writer.min_length = strlen(format) + 100; - writer.overallocate = 1; + Py_ssize_t len = strlen(format); + writer->min_length += len + 100; + writer->overallocate = 1; // Copy varags to be able to pass a reference to a subfunction. + va_list vargs2; va_copy(vargs2, vargs); - for (f = format; *f; ) { + // _PyUnicodeWriter_WriteASCIIString() below requires the format string + // to be encoded to ASCII. + int is_ascii = (ucs1lib_find_max_char((Py_UCS1*)format, (Py_UCS1*)format + len) < 128); + if (!is_ascii) { + Py_ssize_t i; + for (i=0; i < len && (unsigned char)format[i] <= 127; i++); + PyErr_Format(PyExc_ValueError, + "PyUnicode_FromFormatV() expects an ASCII-encoded format " + "string, got a non-ASCII byte: 0x%02x", + (unsigned char)format[i]); + goto fail; + } + + for (const char *f = format; *f; ) { if (*f == '%') { - f = unicode_fromformat_arg(&writer, f, &vargs2); + f = unicode_fromformat_arg(writer, f, &vargs2); if (f == NULL) goto fail; } else { - const char *p; - Py_ssize_t len; - - p = f; - do - { - if ((unsigned char)*p > 127) { - PyErr_Format(PyExc_ValueError, - "PyUnicode_FromFormatV() expects an ASCII-encoded format " - "string, got a non-ASCII byte: 0x%02x", - (unsigned char)*p); - goto fail; - } - p++; + const char *p = strchr(f, '%'); + if (p != NULL) { + len = p - f; + } + else { + len = strlen(f); + writer->overallocate = 0; } - while (*p != '\0' && *p != '%'); - len = p - f; - - if (*p == '\0') - writer.overallocate = 0; - if (_PyUnicodeWriter_WriteASCIIString(&writer, f, len) < 0) + if (_PyUnicodeWriter_WriteASCIIString(writer, f, len) < 0) { goto fail; - - f = p; + } + f += len; } } va_end(vargs2); - return _PyUnicodeWriter_Finish(&writer); + return 0; fail: va_end(vargs2); - _PyUnicodeWriter_Dealloc(&writer); - return NULL; + return -1; +} + +PyObject * +PyUnicode_FromFormatV(const char *format, va_list vargs) +{ + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + + if (unicode_from_format(&writer, format, vargs) < 0) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } PyObject * @@ -2870,6 +3216,23 @@ PyUnicode_FromFormat(const char *format, ...) return ret; } +int +PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) +{ + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + + va_list vargs; + va_start(vargs, format); + int res = unicode_from_format(_writer, format, vargs); + va_end(vargs); + + if (res < 0) { + _writer->pos = old_pos; + } + return res; +} + static Py_ssize_t unicode_get_widechar_size(PyObject *unicode) { @@ -4631,8 +4994,9 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) const char *p = start; #if SIZEOF_SIZE_T <= SIZEOF_VOID_P - assert(_Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)); - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { + if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T) + && _Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)) + { /* Fast path, see in STRINGLIB(utf8_decode) for an explanation. */ /* Help allocation */ @@ -4679,46 +5043,14 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) return p - start; } -static PyObject * -unicode_decode_utf8(const char *s, Py_ssize_t size, - _Py_error_handler error_handler, const char *errors, - Py_ssize_t *consumed) -{ - if (size == 0) { - if (consumed) - *consumed = 0; - _Py_RETURN_UNICODE_EMPTY(); - } - - /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 && (unsigned char)s[0] < 128) { - if (consumed) { - *consumed = 1; - } - return get_latin1_char((unsigned char)s[0]); - } - - const char *starts = s; - const char *end = s + size; - - // fast path: try ASCII string. - PyObject *u = PyUnicode_New(size, 127); - if (u == NULL) { - return NULL; - } - s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); - if (s == end) { - if (consumed) { - *consumed = size; - } - return u; - } - - // Use _PyUnicodeWriter after fast path is failed. - _PyUnicodeWriter writer; - _PyUnicodeWriter_InitWithBuffer(&writer, u); - writer.pos = s - starts; +static int +unicode_decode_utf8_impl(_PyUnicodeWriter *writer, + const char *starts, const char *s, const char *end, + _Py_error_handler error_handler, + const char *errors, + Py_ssize_t *consumed) +{ Py_ssize_t startinpos, endinpos; const char *errmsg = ""; PyObject *error_handler_obj = NULL; @@ -4726,18 +5058,18 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, while (s < end) { Py_UCS4 ch; - int kind = writer.kind; + int kind = writer->kind; if (kind == PyUnicode_1BYTE_KIND) { - if (PyUnicode_IS_ASCII(writer.buffer)) - ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos); + if (PyUnicode_IS_ASCII(writer->buffer)) + ch = asciilib_utf8_decode(&s, end, writer->data, &writer->pos); else - ch = ucs1lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs1lib_utf8_decode(&s, end, writer->data, &writer->pos); } else if (kind == PyUnicode_2BYTE_KIND) { - ch = ucs2lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs2lib_utf8_decode(&s, end, writer->data, &writer->pos); } else { assert(kind == PyUnicode_4BYTE_KIND); - ch = ucs4lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs4lib_utf8_decode(&s, end, writer->data, &writer->pos); } switch (ch) { @@ -4760,7 +5092,7 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, /* Truncated surrogate code in range D800-DFFF */ goto End; } - /* fall through */ + _Py_FALLTHROUGH; case 3: case 4: errmsg = "invalid continuation byte"; @@ -4768,7 +5100,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, endinpos = startinpos + ch - 1; break; default: - if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) + // ch doesn't fit into kind, so change the buffer kind to write + // the character + if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) goto onError; continue; } @@ -4782,7 +5116,7 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, break; case _Py_ERROR_REPLACE: - if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0) + if (_PyUnicodeWriter_WriteCharInline(writer, 0xfffd) < 0) goto onError; s += (endinpos - startinpos); break; @@ -4791,13 +5125,13 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, { Py_ssize_t i; - if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + if (_PyUnicodeWriter_PrepareKind(writer, PyUnicode_2BYTE_KIND) < 0) goto onError; for (i=startinpos; ikind, writer->data, writer->pos, ch + 0xdc00); - writer.pos++; + writer->pos++; } s += (endinpos - startinpos); break; @@ -4808,8 +5142,13 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, errors, &error_handler_obj, "utf-8", errmsg, &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) + writer)) { goto onError; + } + + if (_PyUnicodeWriter_Prepare(writer, end - s, 127) < 0) { + return -1; + } } } @@ -4819,13 +5158,106 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return 0; onError: Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - _PyUnicodeWriter_Dealloc(&writer); - return NULL; + return -1; +} + + +static PyObject * +unicode_decode_utf8(const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed) +{ + if (size == 0) { + if (consumed) { + *consumed = 0; + } + _Py_RETURN_UNICODE_EMPTY(); + } + + /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if (size == 1 && (unsigned char)s[0] < 128) { + if (consumed) { + *consumed = 1; + } + return get_latin1_char((unsigned char)s[0]); + } + + // fast path: try ASCII string. + const char *starts = s; + const char *end = s + size; + PyObject *u = PyUnicode_New(size, 127); + if (u == NULL) { + return NULL; + } + Py_ssize_t decoded = ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); + if (decoded == size) { + if (consumed) { + *consumed = size; + } + return u; + } + s += decoded; + size -= decoded; + + // Use _PyUnicodeWriter after fast path is failed. + _PyUnicodeWriter writer; + _PyUnicodeWriter_InitWithBuffer(&writer, u); + writer.pos = decoded; + + if (unicode_decode_utf8_impl(&writer, starts, s, end, + error_handler, errors, + consumed) < 0) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); +} + + +// Used by PyUnicodeWriter_WriteUTF8() implementation +static int +unicode_decode_utf8_writer(_PyUnicodeWriter *writer, + const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed) +{ + if (size == 0) { + if (consumed) { + *consumed = 0; + } + return 0; + } + + // fast path: try ASCII string. + if (_PyUnicodeWriter_Prepare(writer, size, 127) < 0) { + return -1; + } + + const char *starts = s; + const char *end = s + size; + Py_ssize_t decoded = 0; + Py_UCS1 *dest = (Py_UCS1*)writer->data + writer->pos * writer->kind; + if (writer->kind == PyUnicode_1BYTE_KIND) { + decoded = ascii_decode(s, end, dest); + writer->pos += decoded; + + if (decoded == size) { + if (consumed) { + *consumed = size; + } + return 0; + } + s += decoded; + size -= decoded; + } + + return unicode_decode_utf8_impl(writer, starts, s, end, + error_handler, errors, consumed); } @@ -4980,7 +5412,7 @@ _Py_DecodeUTF8_surrogateescape(const char *arg, Py_ssize_t arglen, } -/* UTF-8 encoder using the surrogateescape error handler . +/* UTF-8 encoder. On success, return 0 and write the newly allocated character string (use PyMem_Free() to free the memory) into *str. @@ -5884,7 +6316,7 @@ _PyUnicode_GetNameCAPI(void) ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( PyUnicodeData_CAPSULE_NAME, 1); - // It's fine if we overwite the value here. It's always the same value. + // It's fine if we overwrite the value here. It's always the same value. _Py_atomic_store_ptr(&interp->unicode.ucnhash_capi, ucnhash_capi); } return ucnhash_capi; @@ -6695,7 +7127,7 @@ unicode_encode_ucs1(PyObject *unicode, case _Py_ERROR_REPLACE: memset(str, '?', collend - collstart); str += (collend - collstart); - /* fall through */ + _Py_FALLTHROUGH; case _Py_ERROR_IGNORE: pos = collend; break; @@ -6734,7 +7166,7 @@ unicode_encode_ucs1(PyObject *unicode, break; collstart = pos; assert(collstart != collend); - /* fall through */ + _Py_FALLTHROUGH; default: rep = unicode_encode_call_errorhandler(errors, &error_handler_obj, @@ -7795,8 +8227,12 @@ charmap_decode_mapping(const char *s, if (key == NULL) goto onError; - item = PyObject_GetItem(mapping, key); + int rc = PyMapping_GetOptionalItem(mapping, key, &item); Py_DECREF(key); + if (rc == 0) { + /* No mapping found means: mapping is undefined. */ + goto Undefined; + } if (item == NULL) { if (PyErr_ExceptionMatches(PyExc_LookupError)) { /* No mapping found means: mapping is undefined. */ @@ -7810,7 +8246,7 @@ charmap_decode_mapping(const char *s, if (item == Py_None) goto Undefined; if (PyLong_Check(item)) { - long value = PyLong_AS_LONG(item); + long value = PyLong_AsLong(item); if (value == 0xFFFE) goto Undefined; if (value < 0 || value > MAX_UNICODE) { @@ -7960,7 +8396,7 @@ PyUnicode_BuildEncodingMap(PyObject* string) int count2 = 0, count3 = 0; int kind; const void *data; - Py_ssize_t length; + int length; Py_UCS4 ch; if (!PyUnicode_Check(string) || !PyUnicode_GET_LENGTH(string)) { @@ -7969,8 +8405,7 @@ PyUnicode_BuildEncodingMap(PyObject* string) } kind = PyUnicode_KIND(string); data = PyUnicode_DATA(string); - length = PyUnicode_GET_LENGTH(string); - length = Py_MIN(length, 256); + length = (int)Py_MIN(PyUnicode_GET_LENGTH(string), 256); memset(level1, 0xFF, sizeof level1); memset(level2, 0xFF, sizeof level2); @@ -8095,19 +8530,25 @@ encoding_map_lookup(Py_UCS4 c, PyObject *mapping) return i; } -/* Lookup the character ch in the mapping. If the character - can't be found, Py_None is returned (or NULL, if another - error occurred). */ +/* Lookup the character in the mapping. + On success, return PyLong, PyBytes or None (if the character can't be found). + If the result is PyLong, put its value in replace. + On error, return NULL. + */ static PyObject * -charmapencode_lookup(Py_UCS4 c, PyObject *mapping) +charmapencode_lookup(Py_UCS4 c, PyObject *mapping, unsigned char *replace) { PyObject *w = PyLong_FromLong((long)c); PyObject *x; if (w == NULL) return NULL; - x = PyObject_GetItem(mapping, w); + int rc = PyMapping_GetOptionalItem(mapping, w, &x); Py_DECREF(w); + if (rc == 0) { + /* No mapping found means: mapping is undefined. */ + Py_RETURN_NONE; + } if (x == NULL) { if (PyErr_ExceptionMatches(PyExc_LookupError)) { /* No mapping found means: mapping is undefined. */ @@ -8119,13 +8560,14 @@ charmapencode_lookup(Py_UCS4 c, PyObject *mapping) else if (x == Py_None) return x; else if (PyLong_Check(x)) { - long value = PyLong_AS_LONG(x); + long value = PyLong_AsLong(x); if (value < 0 || value > 255) { PyErr_SetString(PyExc_TypeError, "character mapping must be in range(256)"); Py_DECREF(x); return NULL; } + *replace = (unsigned char)value; return x; } else if (PyBytes_Check(x)) @@ -8166,6 +8608,7 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, PyObject **outobj, Py_ssize_t *outpos) { PyObject *rep; + unsigned char replace; char *outstart; Py_ssize_t outsize = PyBytes_GET_SIZE(*outobj); @@ -8182,7 +8625,7 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, return enc_SUCCESS; } - rep = charmapencode_lookup(c, mapping); + rep = charmapencode_lookup(c, mapping, &replace); if (rep==NULL) return enc_EXCEPTION; else if (rep==Py_None) { @@ -8197,7 +8640,7 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, return enc_EXCEPTION; } outstart = PyBytes_AS_STRING(*outobj); - outstart[(*outpos)++] = (char)PyLong_AS_LONG(rep); + outstart[(*outpos)++] = (char)replace; } else { const char *repchars = PyBytes_AS_STRING(rep); @@ -8246,6 +8689,7 @@ charmap_encoding_error( /* find all unencodable characters */ while (collendpos < size) { PyObject *rep; + unsigned char replace; if (Py_IS_TYPE(mapping, &EncodingMapType)) { ch = PyUnicode_READ_CHAR(unicode, collendpos); val = encoding_map_lookup(ch, mapping); @@ -8256,7 +8700,7 @@ charmap_encoding_error( } ch = PyUnicode_READ_CHAR(unicode, collendpos); - rep = charmapencode_lookup(ch, mapping); + rep = charmapencode_lookup(ch, mapping, &replace); if (rep==NULL) return -1; else if (rep!=Py_None) { @@ -8287,7 +8731,7 @@ charmap_encoding_error( return -1; } } - /* fall through */ + _Py_FALLTHROUGH; case _Py_ERROR_IGNORE: *inpos = collendpos; break; @@ -8521,17 +8965,24 @@ unicode_translate_call_errorhandler(const char *errors, /* Lookup the character ch in the mapping and put the result in result, which must be decrefed by the caller. + The result can be PyLong, PyUnicode, None or NULL. + If the result is PyLong, put its value in replace. Return 0 on success, -1 on error */ static int -charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) +charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result, Py_UCS4 *replace) { PyObject *w = PyLong_FromLong((long)c); PyObject *x; if (w == NULL) return -1; - x = PyObject_GetItem(mapping, w); + int rc = PyMapping_GetOptionalItem(mapping, w, &x); Py_DECREF(w); + if (rc == 0) { + /* No mapping found means: use 1:1 mapping. */ + *result = NULL; + return 0; + } if (x == NULL) { if (PyErr_ExceptionMatches(PyExc_LookupError)) { /* No mapping found means: use 1:1 mapping. */ @@ -8546,7 +8997,7 @@ charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) return 0; } else if (PyLong_Check(x)) { - long value = PyLong_AS_LONG(x); + long value = PyLong_AsLong(x); if (value < 0 || value > MAX_UNICODE) { PyErr_Format(PyExc_ValueError, "character mapping must be in range(0x%x)", @@ -8555,6 +9006,7 @@ charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) return -1; } *result = x; + *replace = (Py_UCS4)value; return 0; } else if (PyUnicode_Check(x)) { @@ -8578,8 +9030,9 @@ charmaptranslate_output(Py_UCS4 ch, PyObject *mapping, _PyUnicodeWriter *writer) { PyObject *item; + Py_UCS4 replace; - if (charmaptranslate_lookup(ch, mapping, &item)) + if (charmaptranslate_lookup(ch, mapping, &item, &replace)) return -1; if (item == NULL) { @@ -8596,10 +9049,7 @@ charmaptranslate_output(Py_UCS4 ch, PyObject *mapping, } if (PyLong_Check(item)) { - long ch = (Py_UCS4)PyLong_AS_LONG(item); - /* PyLong_AS_LONG() cannot fail, charmaptranslate_lookup() already - used it */ - if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) { + if (_PyUnicodeWriter_WriteCharInline(writer, replace) < 0) { Py_DECREF(item); return -1; } @@ -8626,9 +9076,10 @@ unicode_fast_translate_lookup(PyObject *mapping, Py_UCS1 ch, Py_UCS1 *translate) { PyObject *item = NULL; + Py_UCS4 replace; int ret = 0; - if (charmaptranslate_lookup(ch, mapping, &item)) { + if (charmaptranslate_lookup(ch, mapping, &item, &replace)) { return -1; } @@ -8642,10 +9093,7 @@ unicode_fast_translate_lookup(PyObject *mapping, Py_UCS1 ch, return 1; } else if (PyLong_Check(item)) { - long replace = PyLong_AS_LONG(item); - /* PyLong_AS_LONG() cannot fail, charmaptranslate_lookup() already - used it */ - if (127 < replace) { + if (replace > 127) { /* invalid character or character outside ASCII: skip the fast translate */ goto exit; @@ -8653,8 +9101,6 @@ unicode_fast_translate_lookup(PyObject *mapping, Py_UCS1 ch, translate[ch] = (Py_UCS1)replace; } else if (PyUnicode_Check(item)) { - Py_UCS4 replace; - if (PyUnicode_GET_LENGTH(item) != 1) goto exit; @@ -8807,8 +9253,9 @@ _PyUnicode_TranslateCharmap(PyObject *input, /* find all untranslatable characters */ while (collend < size) { PyObject *x; + Py_UCS4 replace; ch = PyUnicode_READ(kind, data, collend); - if (charmaptranslate_lookup(ch, mapping, &x)) + if (charmaptranslate_lookup(ch, mapping, &x, &replace)) goto onError; Py_XDECREF(x); if (x != Py_None) @@ -8903,19 +9350,24 @@ _PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode) /* --- Helpers ------------------------------------------------------------ */ /* helper macro to fixup start/end slice values */ -#define ADJUST_INDICES(start, end, len) \ - if (end > len) \ - end = len; \ - else if (end < 0) { \ - end += len; \ - if (end < 0) \ - end = 0; \ - } \ - if (start < 0) { \ - start += len; \ - if (start < 0) \ - start = 0; \ - } +#define ADJUST_INDICES(start, end, len) \ + do { \ + if (end > len) { \ + end = len; \ + } \ + else if (end < 0) { \ + end += len; \ + if (end < 0) { \ + end = 0; \ + } \ + } \ + if (start < 0) { \ + start += len; \ + if (start < 0) { \ + start = 0; \ + } \ + } \ + } while (0) static Py_ssize_t any_find_slice(PyObject* s1, PyObject* s2, @@ -9136,99 +9588,30 @@ _PyUnicode_InsertThousandsGrouping( return count; } -static Py_ssize_t -unicode_count_impl(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end) +Py_ssize_t +PyUnicode_Count(PyObject *str, + PyObject *substr, + Py_ssize_t start, + Py_ssize_t end) { - assert(PyUnicode_Check(str)); - assert(PyUnicode_Check(substr)); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) + return -1; - Py_ssize_t result; - int kind1, kind2; - const void *buf1 = NULL, *buf2 = NULL; - Py_ssize_t len1, len2; + return unicode_count_impl(str, substr, start, end); +} - kind1 = PyUnicode_KIND(str); - kind2 = PyUnicode_KIND(substr); - if (kind1 < kind2) - return 0; +Py_ssize_t +PyUnicode_Find(PyObject *str, + PyObject *substr, + Py_ssize_t start, + Py_ssize_t end, + int direction) +{ + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) + return -2; - len1 = PyUnicode_GET_LENGTH(str); - len2 = PyUnicode_GET_LENGTH(substr); - ADJUST_INDICES(start, end, len1); - if (end - start < len2) - return 0; - - buf1 = PyUnicode_DATA(str); - buf2 = PyUnicode_DATA(substr); - if (kind2 != kind1) { - buf2 = unicode_askind(kind2, buf2, len2, kind1); - if (!buf2) - goto onError; - } - - // We don't reuse `anylib_count` here because of the explicit casts. - switch (kind1) { - case PyUnicode_1BYTE_KIND: - result = ucs1lib_count( - ((const Py_UCS1*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_2BYTE_KIND: - result = ucs2lib_count( - ((const Py_UCS2*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_4BYTE_KIND: - result = ucs4lib_count( - ((const Py_UCS4*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - default: - Py_UNREACHABLE(); - } - - assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); - if (kind2 != kind1) - PyMem_Free((void *)buf2); - - return result; - onError: - assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); - if (kind2 != kind1) - PyMem_Free((void *)buf2); - return -1; -} - -Py_ssize_t -PyUnicode_Count(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end) -{ - if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) - return -1; - - return unicode_count_impl(str, substr, start, end); -} - -Py_ssize_t -PyUnicode_Find(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end, - int direction) -{ - if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) - return -2; - - return any_find_slice(str, substr, start, end, direction); -} + return any_find_slice(str, substr, start, end, direction); +} Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, @@ -9571,13 +9954,14 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) return NULL; } - /* NOTE: the following code can't call back into Python code, - * so we are sure that fseq won't be mutated. - */ + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(seq); items = PySequence_Fast_ITEMS(fseq); seqlen = PySequence_Fast_GET_SIZE(fseq); res = _PyUnicode_JoinArray(separator, items, seqlen); + + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); + Py_DECREF(fseq); return res; } @@ -10611,26 +10995,6 @@ unicode_compare(PyObject *str1, PyObject *str2) #undef COMPARE } -static int -unicode_compare_eq(PyObject *str1, PyObject *str2) -{ - int kind; - const void *data1, *data2; - Py_ssize_t len; - int cmp; - - len = PyUnicode_GET_LENGTH(str1); - if (PyUnicode_GET_LENGTH(str2) != len) - return 0; - kind = PyUnicode_KIND(str1); - if (PyUnicode_KIND(str2) != kind) - return 0; - data1 = PyUnicode_DATA(str1); - data2 = PyUnicode_DATA(str2); - - cmp = memcmp(data1, data2, len * kind); - return (cmp == 0); -} int _PyUnicode_Equal(PyObject *str1, PyObject *str2) @@ -10640,7 +11004,25 @@ _PyUnicode_Equal(PyObject *str1, PyObject *str2) if (str1 == str2) { return 1; } - return unicode_compare_eq(str1, str2); + return unicode_eq(str1, str2); +} + + +int +PyUnicode_Equal(PyObject *str1, PyObject *str2) +{ + if (!PyUnicode_Check(str1)) { + PyErr_Format(PyExc_TypeError, + "first argument must be str, not %T", str1); + return -1; + } + if (!PyUnicode_Check(str2)) { + PyErr_Format(PyExc_TypeError, + "second argument must be str, not %T", str2); + return -1; + } + + return _PyUnicode_Equal(str1, str2); } @@ -10826,16 +11208,19 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) if (left == right_uni) return 1; - if (PyUnicode_CHECK_INTERNED(left)) + assert(PyUnicode_CHECK_INTERNED(right_uni)); + if (PyUnicode_CHECK_INTERNED(left)) { return 0; + } - assert(_PyUnicode_HASH(right_uni) != -1); - Py_hash_t hash = _PyUnicode_HASH(left); - if (hash != -1 && hash != _PyUnicode_HASH(right_uni)) { + Py_hash_t right_hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(right_uni)); + assert(right_hash != -1); + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(left)); + if (hash != -1 && hash != right_hash) { return 0; } - return unicode_compare_eq(left, right_uni); + return unicode_eq(left, right_uni); } PyObject * @@ -10863,7 +11248,7 @@ PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) } } else if (op == Py_EQ || op == Py_NE) { - result = unicode_compare_eq(left, right); + result = unicode_eq(left, right); result ^= (op == Py_NE); return PyBool_FromLong(result); } @@ -10873,12 +11258,6 @@ PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) } } -int -_PyUnicode_EQ(PyObject *aa, PyObject *bb) -{ - return unicode_eq(aa, bb); -} - int PyUnicode_Contains(PyObject *str, PyObject *substr) { @@ -11073,47 +11452,87 @@ PyUnicode_AppendAndDel(PyObject **pleft, PyObject *right) Py_XDECREF(right); } -/* -Wraps asciilib_parse_args_finds() and additionally ensures that the -first argument is a unicode object. -*/ +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +str.count as unicode_count -> Py_ssize_t -static inline int -parse_args_finds_unicode(const char * function_name, PyObject *args, - PyObject **substring, - Py_ssize_t *start, Py_ssize_t *end) -{ - if (asciilib_parse_args_finds(function_name, args, substring, start, end)) { - if (ensure_unicode(*substring) < 0) - return 0; - return 1; - } - return 0; -} + self as str: self + sub as substr: unicode + start: slice_index(accept={int, NoneType}, c_default='0') = None + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + / -PyDoc_STRVAR(count__doc__, - "S.count(sub[, start[, end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of substring sub in\n\ -string S[start:end]. Optional arguments start and end are\n\ -interpreted as in slice notation."); +Return the number of non-overlapping occurrences of substring sub in string S[start:end]. -static PyObject * -unicode_count(PyObject *self, PyObject *args) +Optional arguments start and end are interpreted as in slice notation. +[clinic start generated code]*/ + +static Py_ssize_t +unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=8fcc3aef0b18edbf input=6f168ffd94be8785]*/ { - PyObject *substring = NULL; /* initialize to fix a compiler warning */ - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; + assert(PyUnicode_Check(str)); + assert(PyUnicode_Check(substr)); + Py_ssize_t result; + int kind1, kind2; + const void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; - if (!parse_args_finds_unicode("count", args, &substring, &start, &end)) - return NULL; + kind1 = PyUnicode_KIND(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) + return 0; - result = unicode_count_impl(self, substring, start, end); - if (result == -1) - return NULL; + len1 = PyUnicode_GET_LENGTH(str); + len2 = PyUnicode_GET_LENGTH(substr); + ADJUST_INDICES(start, end, len1); + if (end - start < len2) + return 0; + + buf1 = PyUnicode_DATA(str); + buf2 = PyUnicode_DATA(substr); + if (kind2 != kind1) { + buf2 = unicode_askind(kind2, buf2, len2, kind1); + if (!buf2) + goto onError; + } + + // We don't reuse `anylib_count` here because of the explicit casts. + switch (kind1) { + case PyUnicode_1BYTE_KIND: + result = ucs1lib_count( + ((const Py_UCS1*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_count( + ((const Py_UCS2*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_count( + ((const Py_UCS4*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + default: + Py_UNREACHABLE(); + } + + assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); + if (kind2 != kind1) + PyMem_Free((void *)buf2); - return PyLong_FromSsize_t(result); + return result; + onError: + assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); + if (kind2 != kind1) + PyMem_Free((void *)buf2); + return -1; } /*[clinic input] @@ -11224,33 +11643,25 @@ unicode_expandtabs_impl(PyObject *self, int tabsize) return NULL; } -PyDoc_STRVAR(find__doc__, - "S.find(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - -static PyObject * -unicode_find(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; - - if (!parse_args_finds_unicode("find", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.find as unicode_find = str.count - result = any_find_slice(self, substring, start, end, 1); +Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Return -1 on failure. +[clinic start generated code]*/ - return PyLong_FromSsize_t(result); +static Py_ssize_t +unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=51dbe6255712e278 input=4a89d2d68ef57256]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, 1); + if (result < 0) { + return -1; + } + return result; } static PyObject * @@ -11284,47 +11695,39 @@ unicode_hash(PyObject *self) #ifdef Py_DEBUG assert(_Py_HashSecret_Initialized); #endif - if (_PyUnicode_HASH(self) != -1) - return _PyUnicode_HASH(self); - - x = _Py_HashBytes(PyUnicode_DATA(self), + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(self)); + if (hash != -1) { + return hash; + } + x = Py_HashBuffer(PyUnicode_DATA(self), PyUnicode_GET_LENGTH(self) * PyUnicode_KIND(self)); - _PyUnicode_HASH(self) = x; + + FT_ATOMIC_STORE_SSIZE_RELAXED(_PyUnicode_HASH(self), x); return x; } -PyDoc_STRVAR(index__doc__, - "S.index(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the substring is not found."); - -static PyObject * -unicode_index(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - Py_ssize_t result; - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - - if (!parse_args_finds_unicode("index", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.index as unicode_index = str.count - result = any_find_slice(self, substring, start, end, 1); +Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Raises ValueError when the substring is not found. +[clinic start generated code]*/ - if (result < 0) { +static Py_ssize_t +unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=77558288837cdf40 input=d986aeac0be14a1c]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, 1); + if (result == -1) { PyErr_SetString(PyExc_ValueError, "substring not found"); - return NULL; } - - return PyLong_FromSsize_t(result); + else if (result < 0) { + return -1; + } + return result; } /*[clinic input] @@ -12245,24 +12648,17 @@ unicode_removesuffix_impl(PyObject *self, PyObject *suffix) static PyObject * unicode_repr(PyObject *unicode) { - PyObject *repr; - Py_ssize_t isize; - Py_ssize_t osize, squote, dquote, i, o; - Py_UCS4 max, quote; - int ikind, okind, unchanged; - const void *idata; - void *odata; - - isize = PyUnicode_GET_LENGTH(unicode); - idata = PyUnicode_DATA(unicode); + Py_ssize_t isize = PyUnicode_GET_LENGTH(unicode); + const void *idata = PyUnicode_DATA(unicode); /* Compute length of output, quote characters, and maximum character */ - osize = 0; - max = 127; - squote = dquote = 0; - ikind = PyUnicode_KIND(unicode); - for (i = 0; i < isize; i++) { + Py_ssize_t osize = 0; + Py_UCS4 maxch = 127; + Py_ssize_t squote = 0; + Py_ssize_t dquote = 0; + int ikind = PyUnicode_KIND(unicode); + for (Py_ssize_t i = 0; i < isize; i++) { Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); Py_ssize_t incr = 1; switch (ch) { @@ -12278,7 +12674,7 @@ unicode_repr(PyObject *unicode) else if (ch < 0x7f) ; else if (Py_UNICODE_ISPRINTABLE(ch)) - max = ch > max ? ch : max; + maxch = (ch > maxch) ? ch : maxch; else if (ch < 0x100) incr = 4; /* \xHH */ else if (ch < 0x10000) @@ -12294,10 +12690,10 @@ unicode_repr(PyObject *unicode) osize += incr; } - quote = '\''; - unchanged = (osize == isize); + Py_UCS4 quote = '\''; + int changed = (osize != isize); if (squote) { - unchanged = 0; + changed = 1; if (dquote) /* Both squote and dquote present. Use squote, and escape them */ @@ -12307,164 +12703,82 @@ unicode_repr(PyObject *unicode) } osize += 2; /* quotes */ - repr = PyUnicode_New(osize, max); + PyObject *repr = PyUnicode_New(osize, maxch); if (repr == NULL) return NULL; - okind = PyUnicode_KIND(repr); - odata = PyUnicode_DATA(repr); + int okind = PyUnicode_KIND(repr); + void *odata = PyUnicode_DATA(repr); + + if (!changed) { + PyUnicode_WRITE(okind, odata, 0, quote); - PyUnicode_WRITE(okind, odata, 0, quote); - PyUnicode_WRITE(okind, odata, osize-1, quote); - if (unchanged) { _PyUnicode_FastCopyCharacters(repr, 1, unicode, 0, isize); + + PyUnicode_WRITE(okind, odata, osize-1, quote); } else { - for (i = 0, o = 1; i < isize; i++) { - Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); - - /* Escape quotes and backslashes */ - if ((ch == quote) || (ch == '\\')) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, ch); - continue; - } - - /* Map special whitespace to '\t', \n', '\r' */ - if (ch == '\t') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 't'); - } - else if (ch == '\n') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'n'); - } - else if (ch == '\r') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'r'); - } - - /* Map non-printable US ASCII to '\xhh' */ - else if (ch < ' ' || ch == 0x7F) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } - - /* Copy ASCII characters as-is */ - else if (ch < 0x7F) { - PyUnicode_WRITE(okind, odata, o++, ch); - } - - /* Non-ASCII characters */ - else { - /* Map Unicode whitespace and control characters - (categories Z* and C* except ASCII space) - */ - if (!Py_UNICODE_ISPRINTABLE(ch)) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - /* Map 8-bit characters to '\xhh' */ - if (ch <= 0xff) { - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } - /* Map 16-bit characters to '\uxxxx' */ - else if (ch <= 0xffff) { - PyUnicode_WRITE(okind, odata, o++, 'u'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - /* Map 21-bit characters to '\U00xxxxxx' */ - else { - PyUnicode_WRITE(okind, odata, o++, 'U'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 28) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 24) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 20) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 16) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - } - /* Copy characters as-is */ - else { - PyUnicode_WRITE(okind, odata, o++, ch); - } - } + switch (okind) { + case PyUnicode_1BYTE_KIND: + ucs1lib_repr(unicode, quote, odata); + break; + case PyUnicode_2BYTE_KIND: + ucs2lib_repr(unicode, quote, odata); + break; + default: + assert(okind == PyUnicode_4BYTE_KIND); + ucs4lib_repr(unicode, quote, odata); } } - /* Closing quote already added at the beginning */ + assert(_PyUnicode_CheckConsistency(repr, 1)); return repr; } -PyDoc_STRVAR(rfind__doc__, - "S.rfind(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); +/*[clinic input] +str.rfind as unicode_rfind = str.count -static PyObject * -unicode_rfind(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; +Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (!parse_args_finds_unicode("rfind", args, &substring, &start, &end)) - return NULL; - - result = any_find_slice(self, substring, start, end, -1); - - if (result == -2) - return NULL; - - return PyLong_FromSsize_t(result); -} - -PyDoc_STRVAR(rindex__doc__, - "S.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the substring is not found."); +Optional arguments start and end are interpreted as in slice notation. +Return -1 on failure. +[clinic start generated code]*/ -static PyObject * -unicode_rindex(PyObject *self, PyObject *args) +static Py_ssize_t +unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=880b29f01dd014c8 input=898361fb71f59294]*/ { - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; + Py_ssize_t result = any_find_slice(str, substr, start, end, -1); + if (result < 0) { + return -1; + } + return result; +} - if (!parse_args_finds_unicode("rindex", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.rindex as unicode_rindex = str.count - result = any_find_slice(self, substring, start, end, -1); +Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Raises ValueError when the substring is not found. +[clinic start generated code]*/ - if (result < 0) { +static Py_ssize_t +unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=5f3aef124c867fe1 input=35943dead6c1ea9d]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, -1); + if (result == -1) { PyErr_SetString(PyExc_ValueError, "substring not found"); - return NULL; } - - return PyLong_FromSsize_t(result); + else if (result < 0) { + return -1; + } + return result; } /*[clinic input] @@ -12963,30 +13277,30 @@ unicode_zfill_impl(PyObject *self, Py_ssize_t width) return u; } -PyDoc_STRVAR(startswith__doc__, - "S.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if S starts with the specified prefix, False otherwise.\n\ -With optional start, test S beginning at that position.\n\ -With optional end, stop comparing S at that position.\n\ -prefix can also be a tuple of strings to try."); +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +str.startswith as unicode_startswith + + prefix as subobj: object + A string or a tuple of strings to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the string. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the string. + / + +Return True if the string starts with the specified prefix, False otherwise. +[clinic start generated code]*/ static PyObject * -unicode_startswith(PyObject *self, - PyObject *args) +unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=4bd7cfd0803051d4 input=5f918b5f5f89d856]*/ { - PyObject *subobj; - PyObject *substring; - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - int result; - - if (!asciilib_parse_args_finds("startswith", args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); + PyObject *substring = PyTuple_GET_ITEM(subobj, i); if (!PyUnicode_Check(substring)) { PyErr_Format(PyExc_TypeError, "tuple for startswith must only contain str, " @@ -12994,9 +13308,10 @@ unicode_startswith(PyObject *self, Py_TYPE(substring)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, -1); - if (result == -1) + int result = tailmatch(self, substring, start, end, -1); + if (result < 0) { return NULL; + } if (result) { Py_RETURN_TRUE; } @@ -13010,37 +13325,38 @@ unicode_startswith(PyObject *self, "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, -1); - if (result == -1) + int result = tailmatch(self, subobj, start, end, -1); + if (result < 0) { return NULL; + } return PyBool_FromLong(result); } -PyDoc_STRVAR(endswith__doc__, - "S.endswith(suffix[, start[, end]]) -> bool\n\ -\n\ -Return True if S ends with the specified suffix, False otherwise.\n\ -With optional start, test S beginning at that position.\n\ -With optional end, stop comparing S at that position.\n\ -suffix can also be a tuple of strings to try."); +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +str.endswith as unicode_endswith + + suffix as subobj: object + A string or a tuple of strings to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the string. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the string. + / + +Return True if the string ends with the specified suffix, False otherwise. +[clinic start generated code]*/ static PyObject * -unicode_endswith(PyObject *self, - PyObject *args) +unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=cce6f8ceb0102ca9 input=00fbdc774a7d4d71]*/ { - PyObject *subobj; - PyObject *substring; - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - int result; - - if (!asciilib_parse_args_finds("endswith", args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); + PyObject *substring = PyTuple_GET_ITEM(subobj, i); if (!PyUnicode_Check(substring)) { PyErr_Format(PyExc_TypeError, "tuple for endswith must only contain str, " @@ -13048,9 +13364,10 @@ unicode_endswith(PyObject *self, Py_TYPE(substring)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, +1); - if (result == -1) + int result = tailmatch(self, substring, start, end, +1); + if (result < 0) { return NULL; + } if (result) { Py_RETURN_TRUE; } @@ -13063,12 +13380,14 @@ unicode_endswith(PyObject *self, "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, +1); - if (result == -1) + int result = tailmatch(self, subobj, start, end, +1); + if (result < 0) { return NULL; + } return PyBool_FromLong(result); } + static inline void _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) { @@ -13092,6 +13411,7 @@ _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) } } + void _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) { @@ -13100,12 +13420,54 @@ _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) /* ASCII is the bare minimum */ writer->min_char = 127; - /* use a value smaller than PyUnicode_1BYTE_KIND() so + /* use a kind value smaller than PyUnicode_1BYTE_KIND so _PyUnicodeWriter_PrepareKind() will copy the buffer. */ - writer->kind = 0; - assert(writer->kind <= PyUnicode_1BYTE_KIND); + assert(writer->kind == 0); + assert(writer->kind < PyUnicode_1BYTE_KIND); } + +PyUnicodeWriter* +PyUnicodeWriter_Create(Py_ssize_t length) +{ + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length must be positive"); + return NULL; + } + + const size_t size = sizeof(_PyUnicodeWriter); + PyUnicodeWriter *pub_writer; + pub_writer = _Py_FREELIST_POP_MEM(unicode_writers); + if (pub_writer == NULL) { + pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == NULL) { + return (PyUnicodeWriter *)PyErr_NoMemory(); + } + } + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + _PyUnicodeWriter_Init(writer); + if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { + PyUnicodeWriter_Discard(pub_writer); + return NULL; + } + writer->overallocate = 1; + + return pub_writer; +} + + +void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +{ + if (writer == NULL) { + return; + } + _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); + _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); +} + + // Initialize _PyUnicodeWriter with initial buffer static inline void _PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) @@ -13116,6 +13478,7 @@ _PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) writer->min_length = writer->size; } + int _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar) @@ -13123,6 +13486,7 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t newlen; PyObject *newbuffer; + assert(length >= 0); assert(maxchar <= MAX_UNICODE); /* ensure that the _PyUnicodeWriter_Prepare macro was used */ @@ -13231,9 +13595,23 @@ _PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, Py_UCS4 ch) return _PyUnicodeWriter_WriteCharInline(writer, ch); } +int +PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) +{ + if (ch > MAX_UNICODE) { + PyErr_SetString(PyExc_ValueError, + "character must be in range(0x110000)"); + return -1; + } + + return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); +} + int _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) { + assert(PyUnicode_Check(str)); + Py_UCS4 maxchar; Py_ssize_t len; @@ -13259,31 +13637,68 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) return 0; } +int +PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +{ + if (Py_TYPE(obj) == &PyLong_Type) { + return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + } + + PyObject *str = PyObject_Str(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + + +int +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +{ + if (Py_TYPE(obj) == &PyLong_Type) { + return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + } + + PyObject *repr = PyObject_Repr(obj); + if (repr == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, repr); + Py_DECREF(repr); + return res; +} + + int _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) { - Py_UCS4 maxchar; - Py_ssize_t len; - assert(0 <= start); assert(end <= PyUnicode_GET_LENGTH(str)); assert(start <= end); - if (end == 0) - return 0; - if (start == 0 && end == PyUnicode_GET_LENGTH(str)) return _PyUnicodeWriter_WriteStr(writer, str); - if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) + Py_ssize_t len = end - start; + if (len == 0) { + return 0; + } + + Py_UCS4 maxchar; + if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) { maxchar = _PyUnicode_FindMaxChar(str, start, end); - else + } + else { maxchar = writer->maxchar; - len = end - start; - - if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) + } + if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) { return -1; + } _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, str, start, len); @@ -13291,6 +13706,29 @@ _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, return 0; } + +int +PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + if (!PyUnicode_Check(str)) { + PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + return -1; + } + if (start < 0 || start > end) { + PyErr_Format(PyExc_ValueError, "invalid start argument"); + return -1; + } + if (end > PyUnicode_GET_LENGTH(str)) { + PyErr_Format(PyExc_ValueError, "invalid end argument"); + return -1; + } + + return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, + start, end); +} + + int _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, const char *ascii, Py_ssize_t len) @@ -13351,6 +13789,51 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, return 0; } +int +PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size) +{ + if (size < 0) { + size = strlen(str); + } + + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + int res = unicode_decode_utf8_writer(_writer, str, size, + _Py_ERROR_STRICT, NULL, NULL); + if (res < 0) { + _writer->pos = old_pos; + } + return res; +} + + +int +PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, + const char *string, + Py_ssize_t length, + const char *errors, + Py_ssize_t *consumed) +{ + if (length < 0) { + length = strlen(string); + } + + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + int res = unicode_decode_utf8_writer(_writer, string, length, + _Py_ERROR_UNKNOWN, errors, consumed); + if (res < 0) { + _writer->pos = old_pos; + if (consumed) { + *consumed = 0; + } + } + return res; +} + + int _PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, const char *str, Py_ssize_t len) @@ -13397,6 +13880,17 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) return unicode_result(str); } + +PyObject* +PyUnicodeWriter_Finish(PyUnicodeWriter *writer) +{ + PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); + assert(((_PyUnicodeWriter*)writer)->buffer == NULL); + _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); + return str; +} + + void _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) { @@ -13413,7 +13907,7 @@ Return a formatted version of the string, using substitutions from args and kwar The substitutions are identified by braces ('{' and '}')."); PyDoc_STRVAR(format_map__doc__, - "format_map($self, /, mapping)\n\ + "format_map($self, mapping, /)\n\ --\n\ \n\ Return a formatted version of the string, using substitutions from mapping.\n\ @@ -13500,16 +13994,16 @@ static PyMethodDef unicode_methods[] = { UNICODE_CASEFOLD_METHODDEF UNICODE_TITLE_METHODDEF UNICODE_CENTER_METHODDEF - {"count", (PyCFunction) unicode_count, METH_VARARGS, count__doc__}, + UNICODE_COUNT_METHODDEF UNICODE_EXPANDTABS_METHODDEF - {"find", (PyCFunction) unicode_find, METH_VARARGS, find__doc__}, + UNICODE_FIND_METHODDEF UNICODE_PARTITION_METHODDEF - {"index", (PyCFunction) unicode_index, METH_VARARGS, index__doc__}, + UNICODE_INDEX_METHODDEF UNICODE_LJUST_METHODDEF UNICODE_LOWER_METHODDEF UNICODE_LSTRIP_METHODDEF - {"rfind", (PyCFunction) unicode_rfind, METH_VARARGS, rfind__doc__}, - {"rindex", (PyCFunction) unicode_rindex, METH_VARARGS, rindex__doc__}, + UNICODE_RFIND_METHODDEF + UNICODE_RINDEX_METHODDEF UNICODE_RJUST_METHODDEF UNICODE_RSTRIP_METHODDEF UNICODE_RPARTITION_METHODDEF @@ -13518,8 +14012,8 @@ static PyMethodDef unicode_methods[] = { UNICODE_SWAPCASE_METHODDEF UNICODE_TRANSLATE_METHODDEF UNICODE_UPPER_METHODDEF - {"startswith", (PyCFunction) unicode_startswith, METH_VARARGS, startswith__doc__}, - {"endswith", (PyCFunction) unicode_endswith, METH_VARARGS, endswith__doc__}, + UNICODE_STARTSWITH_METHODDEF + UNICODE_ENDSWITH_METHODDEF UNICODE_REMOVEPREFIX_METHODDEF UNICODE_REMOVESUFFIX_METHODDEF UNICODE_ISASCII_METHODDEF @@ -13980,14 +14474,21 @@ formatchar(PyObject *v) if (PyUnicode_GET_LENGTH(v) == 1) { return PyUnicode_READ_CHAR(v, 0); } - goto onError; + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + return (Py_UCS4) -1; } else { int overflow; long x = PyLong_AsLongAndOverflow(v, &overflow); if (x == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - goto onError; + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, not %T", + v); + return (Py_UCS4) -1; } return (Py_UCS4) -1; } @@ -14001,11 +14502,6 @@ formatchar(PyObject *v) return (Py_UCS4) x; } - - onError: - PyErr_SetString(PyExc_TypeError, - "%c requires int or char"); - return (Py_UCS4) -1; } /* Parse options of an argument: flags, width, precision. @@ -14621,6 +15117,65 @@ unicode_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, return unicode; } +static const char * +arg_as_utf8(PyObject *obj, const char *name) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "str() argument '%s' must be str, not %T", + name, obj); + return NULL; + } + return _PyUnicode_AsUTF8NoNUL(obj); +} + +static PyObject * +unicode_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + assert(Py_Is(_PyType_CAST(type), &PyUnicode_Type)); + + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) != 0) { + // Fallback to unicode_new() + PyObject *tuple = _PyTuple_FromArray(args, nargs); + if (tuple == NULL) { + return NULL; + } + PyObject *dict = _PyStack_AsDict(args + nargs, kwnames); + if (dict == NULL) { + Py_DECREF(tuple); + return NULL; + } + PyObject *ret = unicode_new(_PyType_CAST(type), tuple, dict); + Py_DECREF(tuple); + Py_DECREF(dict); + return ret; + } + if (!_PyArg_CheckPositional("str", nargs, 0, 3)) { + return NULL; + } + if (nargs == 0) { + return unicode_get_empty(); + } + PyObject *object = args[0]; + if (nargs == 1) { + return PyObject_Str(object); + } + const char *encoding = arg_as_utf8(args[1], "encoding"); + if (encoding == NULL) { + return NULL; + } + const char *errors = NULL; + if (nargs == 3) { + errors = arg_as_utf8(args[2], "errors"); + if (errors == NULL) { + return NULL; + } + } + return PyUnicode_FromEncodedObject(object, encoding, errors); +} + static PyObject * unicode_subtype_new(PyTypeObject *type, PyObject *unicode) { @@ -14761,7 +15316,8 @@ PyTypeObject PyUnicode_Type = { 0, /* tp_init */ 0, /* tp_alloc */ unicode_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ + .tp_vectorcall = unicode_vectorcall, }; /* Initialize the Unicode implementation */ @@ -14804,30 +15360,19 @@ _PyUnicode_InitState(PyInterpreterState *interp) PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *interp) { - // Initialize the global interned dict + if (_Py_IsMainInterpreter(interp)) { + PyStatus status = init_global_interned_strings(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + assert(INTERNED_STRINGS); + if (init_interned_dict(interp)) { PyErr_Clear(); return _PyStatus_ERR("failed to create interned dict"); } - if (_Py_IsMainInterpreter(interp)) { - /* Intern statically allocated string identifiers and deepfreeze strings. - * This must be done before any module initialization so that statically - * allocated string identifiers are used instead of heap allocated strings. - * Deepfreeze uses the interned identifiers if present to save space - * else generates them and they are interned to speed up dict lookups. - */ - _PyUnicode_InitStaticStrings(interp); - -#ifdef Py_DEBUG - assert(_PyUnicode_CheckConsistency(&_Py_STR(empty), 1)); - - for (int i = 0; i < 256; i++) { - assert(_PyUnicode_CheckConsistency(LATIN1(i), 1)); - } -#endif - } - return _PyStatus_OK(); } @@ -14850,106 +15395,250 @@ _PyUnicode_InitTypes(PyInterpreterState *interp) return _PyStatus_ERR("Can't initialize unicode types"); } +static /* non-null */ PyObject* +intern_static(PyInterpreterState *interp, PyObject *s /* stolen */) +{ + // Note that this steals a reference to `s`, but in many cases that + // stolen ref is returned, requiring no decref/incref. + + assert(s != NULL); + assert(_PyUnicode_CHECK(s)); + assert(_PyUnicode_STATE(s).statically_allocated); + assert(!PyUnicode_CHECK_INTERNED(s)); + +#ifdef Py_DEBUG + /* We must not add process-global interned string if there's already a + * per-interpreter interned_dict, which might contain duplicates. + */ + PyObject *interned = get_interned_dict(interp); + assert(interned == NULL); +#endif + + /* Look in the global cache first. */ + PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); + /* We should only init each string once */ + assert(r == NULL); + /* but just in case (for the non-debug build), handle this */ + if (r != NULL && r != s) { + assert(_PyUnicode_STATE(r).interned == SSTATE_INTERNED_IMMORTAL_STATIC); + assert(_PyUnicode_CHECK(r)); + Py_DECREF(s); + return Py_NewRef(r); + } + + if (_Py_hashtable_set(INTERNED_STRINGS, s, s) < -1) { + Py_FatalError("failed to intern static string"); + } + + _PyUnicode_STATE(s).interned = SSTATE_INTERNED_IMMORTAL_STATIC; + return s; +} void -_PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) +_PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **p) { - PyObject *s = *p; + // This should only be called as part of runtime initialization + assert(!Py_IsInitialized()); + + *p = intern_static(interp, *p); + assert(*p); +} + +static void +immortalize_interned(PyObject *s) +{ + assert(PyUnicode_CHECK_INTERNED(s) == SSTATE_INTERNED_MORTAL); + assert(!_Py_IsImmortal(s)); +#ifdef Py_REF_DEBUG + /* The reference count value should be excluded from the RefTotal. + The decrements to these objects will not be registered so they + need to be accounted for in here. */ + for (Py_ssize_t i = 0; i < Py_REFCNT(s); i++) { + _Py_DecRefTotal(_PyThreadState_GET()); + } +#endif + _PyUnicode_STATE(s).interned = SSTATE_INTERNED_IMMORTAL; + _Py_SetImmortal(s); +} + +static /* non-null */ PyObject* +intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, + bool immortalize) +{ + // Note that this steals a reference to `s`, but in many cases that + // stolen ref is returned, requiring no decref/incref. + #ifdef Py_DEBUG assert(s != NULL); assert(_PyUnicode_CHECK(s)); #else if (s == NULL || !PyUnicode_Check(s)) { - return; + return s; } #endif /* If it's a subclass, we don't really know what putting it in the interned dict might do. */ if (!PyUnicode_CheckExact(s)) { - return; + return s; } - if (PyUnicode_CHECK_INTERNED(s)) { - return; + /* Is it already interned? */ + switch (PyUnicode_CHECK_INTERNED(s)) { + case SSTATE_NOT_INTERNED: + // no, go on + break; + case SSTATE_INTERNED_MORTAL: + // yes but we might need to make it immortal + if (immortalize) { + immortalize_interned(s); + } + return s; + default: + // all done + return s; } - /* Look in the global cache first. */ - PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); - if (r != NULL && r != s) { - Py_SETREF(*p, Py_NewRef(r)); - return; + /* Statically allocated strings must be already interned. */ + assert(!_PyUnicode_STATE(s).statically_allocated); + +#if Py_GIL_DISABLED + /* In the free-threaded build, all interned strings are immortal */ + immortalize = 1; +#endif + + /* If it's already immortal, intern it as such */ + if (_Py_IsImmortal(s)) { + immortalize = 1; } - /* Handle statically allocated strings. */ - if (_PyUnicode_STATE(s).statically_allocated) { - assert(_Py_IsImmortal(s)); - if (_Py_hashtable_set(INTERNED_STRINGS, s, s) == 0) { - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL_STATIC; + /* if it's a short string, get the singleton */ + if (PyUnicode_GET_LENGTH(s) == 1 && + PyUnicode_KIND(s) == PyUnicode_1BYTE_KIND) { + PyObject *r = LATIN1(*(unsigned char*)PyUnicode_DATA(s)); + assert(PyUnicode_CHECK_INTERNED(r)); + Py_DECREF(s); + return r; + } +#ifdef Py_DEBUG + assert(!unicode_is_singleton(s)); +#endif + + /* Look in the global cache now. */ + { + PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); + if (r != NULL) { + assert(_PyUnicode_STATE(r).statically_allocated); + assert(r != s); // r must be statically_allocated; s is not + Py_DECREF(s); + return Py_NewRef(r); } - return; } - /* Look in the per-interpreter cache. */ + /* Do a setdefault on the per-interpreter cache. */ PyObject *interned = get_interned_dict(interp); assert(interned != NULL); PyObject *t; - int res = PyDict_SetDefaultRef(interned, s, s, &t); - if (res < 0) { - PyErr_Clear(); - return; - } - else if (res == 1) { - // value was already present (not inserted) - Py_SETREF(*p, t); - return; + { + int res = PyDict_SetDefaultRef(interned, s, s, &t); + if (res < 0) { + PyErr_Clear(); + return s; + } + else if (res == 1) { + // value was already present (not inserted) + Py_DECREF(s); + if (immortalize && + PyUnicode_CHECK_INTERNED(t) == SSTATE_INTERNED_MORTAL) { + immortalize_interned(t); + } + return t; + } + else { + // value was newly inserted + assert (s == t); + Py_DECREF(t); + } } - Py_DECREF(t); - if (_Py_IsImmortal(s)) { - // XXX Restrict this to the main interpreter? - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL_STATIC; - return; - } + /* NOT_INTERNED -> INTERNED_MORTAL */ + assert(_PyUnicode_STATE(s).interned == SSTATE_NOT_INTERNED); + + if (!_Py_IsImmortal(s)) { + /* The two references in interned dict (key and value) are not counted. + unicode_dealloc() and _PyUnicode_ClearInterned() take care of this. */ + Py_SET_REFCNT(s, Py_REFCNT(s) - 2); #ifdef Py_REF_DEBUG - /* The reference count value excluding the 2 references from the - interned dictionary should be excluded from the RefTotal. The - decrements to these objects will not be registered so they - need to be accounted for in here. */ - for (Py_ssize_t i = 0; i < Py_REFCNT(s) - 2; i++) { - _Py_DecRefTotal(_PyInterpreterState_GET()); + /* let's be pedantic with the ref total */ + _Py_DecRefTotal(_PyThreadState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); +#endif + } + _PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL; + + /* INTERNED_MORTAL -> INTERNED_IMMORTAL (if needed) */ + +#ifdef Py_DEBUG + if (_Py_IsImmortal(s)) { + assert(immortalize); } #endif - _Py_SetImmortal(s); - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL; + if (immortalize) { + immortalize_interned(s); + } + + return s; +} + +void +_PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **p) +{ + *p = intern_common(interp, *p, 1); + assert(*p); +} + +void +_PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **p) +{ + *p = intern_common(interp, *p, 0); + assert(*p); +} + + +void +_PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) +{ + _PyUnicode_InternImmortal(interp, p); + return; } void PyUnicode_InternInPlace(PyObject **p) { PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUnicode_InternInPlace(interp, p); + _PyUnicode_InternMortal(interp, p); } -// Function kept for the stable ABI. +// Public-looking name kept for the stable ABI; user should not call this: PyAPI_FUNC(void) PyUnicode_InternImmortal(PyObject **); void PyUnicode_InternImmortal(PyObject **p) { - PyUnicode_InternInPlace(p); - // Leak a reference on purpose - Py_INCREF(*p); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, p); } PyObject * PyUnicode_InternFromString(const char *cp) { PyObject *s = PyUnicode_FromString(cp); - if (s == NULL) + if (s == NULL) { return NULL; - PyUnicode_InternInPlace(&s); + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &s); return s; } @@ -14963,20 +15652,13 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } assert(PyDict_CheckExact(interned)); - /* TODO: - * Currently, the runtime is not able to guarantee that it can exit without - * allocations that carry over to a future initialization of Python within - * the same process. i.e: - * ./python -X showrefcount -c 'import itertools' - * [237 refs, 237 blocks] - * - * Therefore, this should remain disabled for until there is a strict guarantee - * that no memory will be left after `Py_Finalize`. - */ -#ifdef Py_DEBUG - /* For all non-singleton interned strings, restore the two valid references - to that instance from within the intern string dictionary and let the - normal reference counting process clean up these instances. */ + if (has_shared_intern_dict(interp)) { + // the dict doesn't belong to this interpreter, skip the debug + // checks on it and just clear the pointer to it + clear_interned_dict(interp); + return; + } + #ifdef INTERNED_STATS fprintf(stderr, "releasing %zd interned strings\n", PyDict_GET_SIZE(interned)); @@ -14990,10 +15672,16 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) int shared = 0; switch (PyUnicode_CHECK_INTERNED(s)) { case SSTATE_INTERNED_IMMORTAL: + /* Make immortal interned strings mortal again. */ // Skip the Immortal Instance check and restore // the two references (key and value) ignored // by PyUnicode_InternInPlace(). _Py_SetMortal(s, 2); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_IncRefTotal(_PyThreadState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); +#endif #ifdef INTERNED_STATS total_length += PyUnicode_GET_LENGTH(s); #endif @@ -15009,9 +15697,17 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } break; case SSTATE_INTERNED_MORTAL: - /* fall through */ + // Restore 2 references held by the interned dict; these will + // be decref'd by clear_interned_dict's PyDict_Clear. + Py_SET_REFCNT(s, Py_REFCNT(s) + 2); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_IncRefTotal(_PyThreadState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + break; case SSTATE_NOT_INTERNED: - /* fall through */ + _Py_FALLTHROUGH; default: Py_UNREACHABLE(); } @@ -15030,8 +15726,10 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) for (Py_ssize_t i=0; i < ids->size; i++) { Py_XINCREF(ids->array[i]); } -#endif /* Py_DEBUG */ clear_interned_dict(interp); + if (_Py_IsMainInterpreter(interp)) { + clear_global_interned_strings(); + } } @@ -15241,7 +15939,7 @@ encode_wstr_utf8(wchar_t *wstr, char **str, const char *name) int res; res = _Py_EncodeUTF8Ex(wstr, str, NULL, NULL, 1, _Py_ERROR_STRICT); if (res == -2) { - PyErr_Format(PyExc_RuntimeWarning, "cannot decode %s", name); + PyErr_Format(PyExc_RuntimeWarning, "cannot encode %s", name); return -1; } if (res < 0) { @@ -15394,7 +16092,11 @@ init_fs_encoding(PyThreadState *tstate) PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate) { - PyStatus status = init_fs_encoding(tstate); + PyStatus status = _PyCodec_InitRegistry(tstate->interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = init_fs_encoding(tstate); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -15454,9 +16156,9 @@ unicode_is_finalizing(void) void _PyUnicode_FiniTypes(PyInterpreterState *interp) { - _PyStaticType_Dealloc(interp, &EncodingMapType); - _PyStaticType_Dealloc(interp, &PyFieldNameIter_Type); - _PyStaticType_Dealloc(interp, &PyFormatterIter_Type); + _PyStaticType_FiniBuiltin(interp, &EncodingMapType); + _PyStaticType_FiniBuiltin(interp, &PyFieldNameIter_Type); + _PyStaticType_FiniBuiltin(interp, &PyFormatterIter_Type); } @@ -15465,8 +16167,10 @@ _PyUnicode_Fini(PyInterpreterState *interp) { struct _Py_unicode_state *state = &interp->unicode; - // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() - assert(get_interned_dict(interp) == NULL); + if (!has_shared_intern_dict(interp)) { + // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() + assert(get_interned_dict(interp) == NULL); + } _PyUnicode_FiniEncodings(&state->fs_codec); @@ -15490,6 +16194,7 @@ static PyMethodDef _string_methods[] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Objects/unicodetype_db.h b/Objects/unicodetype_db.h index e6dbeffbe2aa3e..5be810dd67426a 100644 --- a/Objects/unicodetype_db.h +++ b/Objects/unicodetype_db.h @@ -1,4 +1,4 @@ -/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ /* a list of unique character type descriptors */ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { @@ -50,6 +50,7 @@ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { {0, 211, 0, 0, 0, 10113}, {0, 209, 0, 0, 0, 10113}, {163, 0, 163, 0, 0, 9993}, + {42561, 0, 42561, 0, 0, 9993}, {0, 213, 0, 0, 0, 10113}, {130, 0, 130, 0, 0, 9993}, {0, 214, 0, 0, 0, 10113}, @@ -84,6 +85,7 @@ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { {42319, 0, 42319, 0, 0, 9993}, {42315, 0, 42315, 0, 0, 9993}, {-207, 0, -207, 0, 0, 9993}, + {42343, 0, 42343, 0, 0, 9993}, {42280, 0, 42280, 0, 0, 9993}, {42308, 0, 42308, 0, 0, 9993}, {-209, 0, -209, 0, 0, 9993}, @@ -405,6 +407,8 @@ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { {0, -48, 0, 0, 0, 10113}, {0, -42307, 0, 0, 0, 10113}, {0, -35384, 0, 0, 0, 10113}, + {0, -42343, 0, 0, 0, 10113}, + {0, -42561, 0, 0, 0, 10113}, {-928, 0, -928, 0, 0, 9993}, {16778124, 17826698, 16778124, 0, 0, 26377}, {16778127, 17826701, 16778127, 0, 0, 26377}, @@ -1751,613 +1755,1218 @@ const Py_UCS4 _PyUnicode_ExtendedCase[] = { }; /* type indexes */ -#define SHIFT 7 +#define SHIFT 6 static const unsigned short index1[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 34, 35, 36, 37, - 38, 39, 34, 34, 34, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 64, 64, 64, 65, 66, 64, - 64, 64, 64, 67, 68, 64, 64, 64, 64, 64, 64, 69, 70, 71, 72, 73, 74, 75, - 76, 64, 77, 78, 79, 80, 81, 82, 83, 64, 64, 84, 85, 34, 34, 34, 34, 34, - 34, 86, 34, 34, 34, 34, 34, 87, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 88, 89, 90, 91, 92, 93, 34, 94, 34, 34, - 34, 95, 96, 34, 34, 34, 34, 34, 97, 34, 34, 34, 98, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 99, 100, 101, 34, 34, 34, 34, 34, 34, 102, 103, 34, - 34, 34, 34, 34, 34, 34, 34, 104, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 105, 34, 34, 34, 93, 34, 34, 34, 34, 34, 34, 34, 34, 106, 34, 34, 34, 34, - 107, 108, 34, 34, 34, 34, 34, 109, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 93, 34, 34, 34, 34, 34, 34, 110, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 111, 112, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 113, 34, 34, 34, 34, 114, 34, 34, 115, 116, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 117, 34, 34, 34, - 34, 34, 34, 34, 34, 118, 34, 34, 119, 120, 121, 122, 123, 124, 125, 126, - 127, 128, 129, 130, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 131, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 133, 134, - 135, 136, 137, 138, 139, 34, 140, 141, 142, 143, 144, 145, 146, 147, 148, - 149, 132, 150, 151, 152, 153, 154, 155, 156, 34, 34, 157, 158, 159, 160, - 161, 162, 163, 164, 165, 166, 167, 168, 169, 132, 170, 171, 172, 173, - 174, 175, 176, 177, 178, 179, 180, 132, 181, 182, 132, 183, 184, 185, - 186, 132, 187, 188, 189, 190, 191, 192, 193, 132, 194, 195, 196, 197, - 132, 198, 199, 200, 34, 34, 34, 34, 34, 34, 34, 201, 202, 34, 203, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 204, 34, 34, 34, 34, 34, 34, 34, 34, 205, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 34, 34, 34, 34, 206, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 34, 34, 34, 34, 207, 208, 209, 210, 132, 132, 132, 132, - 211, 212, 213, 214, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 215, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 216, 217, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 218, 34, 34, 219, 34, 34, 220, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 221, 222, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 223, 224, 64, - 225, 226, 227, 228, 229, 230, 132, 231, 232, 233, 234, 235, 236, 237, - 238, 64, 64, 64, 64, 239, 240, 132, 132, 132, 132, 132, 132, 132, 132, - 241, 132, 242, 243, 244, 132, 132, 245, 132, 132, 132, 246, 132, 132, - 132, 132, 132, 247, 34, 248, 249, 132, 132, 132, 132, 132, 250, 251, 252, - 132, 253, 254, 132, 132, 255, 256, 257, 258, 259, 132, 64, 260, 64, 64, - 64, 64, 64, 261, 262, 263, 264, 265, 64, 64, 266, 267, 64, 268, 132, 132, - 132, 132, 132, 132, 132, 132, 269, 270, 271, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 86, 272, 34, 273, 274, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 275, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 276, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 277, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 109, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 278, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 279, 34, 280, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 281, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 282, 34, 34, 34, 34, 283, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 34, 275, 34, 34, 284, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 285, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 286, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 287, 132, 288, 289, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 26, 26, 26, 26, 26, 68, 69, + 70, 71, 72, 73, 74, 75, 26, 26, 26, 26, 26, 26, 26, 26, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + 98, 99, 100, 101, 102, 103, 104, 105, 12, 106, 106, 107, 106, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 119, 119, 119, 119, + 119, 119, 119, 119, 120, 121, 122, 123, 119, 119, 119, 119, 119, 119, + 119, 119, 119, 124, 125, 119, 119, 119, 119, 119, 119, 119, 119, 119, + 119, 119, 119, 119, 119, 126, 127, 119, 128, 129, 106, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 119, 119, 119, 139, 140, 141, 142, 143, + 144, 26, 145, 146, 147, 148, 149, 119, 119, 119, 119, 119, 150, 26, 151, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 152, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 153, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 119, 154, 155, 156, 157, 153, + 158, 26, 159, 160, 26, 26, 26, 161, 162, 26, 26, 26, 26, 26, 26, 26, 163, + 26, 164, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 165, 26, 26, 26, 26, + 26, 26, 26, 166, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 167, 26, 168, 169, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 170, 26, 171, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 172, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 173, 26, 26, 26, 26, + 26, 26, 26, 160, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 174, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 175, 176, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 177, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 160, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 178, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 179, 26, 158, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 180, 26, 26, 26, 26, 26, 26, 26, 26, 26, 159, 26, 26, 26, 26, 26, 181, + 182, 26, 183, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 184, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 185, 186, 26, 26, 26, 26, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 211, 212, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 26, 214, 215, 216, 26, 217, 26, 218, 219, + 220, 221, 222, 26, 223, 26, 26, 224, 225, 226, 227, 228, 229, 26, 230, + 231, 232, 233, 234, 235, 236, 26, 237, 238, 239, 240, 241, 213, 213, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 26, 26, + 26, 26, 256, 257, 258, 213, 259, 260, 261, 262, 263, 213, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 213, 26, 273, 274, 275, 276, 277, 278, 213, + 213, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, + 213, 213, 306, 307, 308, 309, 310, 311, 312, 313, 213, 213, 314, 213, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 213, 213, 326, + 327, 328, 329, 213, 330, 331, 332, 213, 213, 213, 213, 333, 334, 335, + 336, 337, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 218, + 213, 338, 339, 26, 26, 26, 340, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 341, 342, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 343, 344, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 237, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 313, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 345, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 26, + 26, 26, 26, 26, 26, 26, 324, 346, 347, 348, 349, 350, 351, 213, 213, 213, + 213, 213, 213, 352, 213, 213, 213, 353, 354, 213, 26, 355, 356, 357, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 358, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 359, 273, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 360, 26, 26, 26, 26, 361, 362, 26, 26, 26, 26, 26, + 363, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 364, 365, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 119, 119, 119, 366, 119, 119, 119, 119, 119, 119, 138, 213, + 367, 368, 119, 369, 119, 119, 119, 370, 371, 372, 373, 374, 119, 375, + 213, 376, 119, 377, 213, 213, 378, 379, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, 390, 391, 392, 393, 119, 119, 119, 119, 119, 119, + 119, 119, 394, 395, 396, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 397, 213, 213, 213, 398, 399, + 400, 213, 401, 402, 213, 213, 213, 213, 403, 404, 213, 213, 213, 213, + 213, 213, 213, 405, 213, 213, 213, 406, 213, 213, 213, 213, 213, 213, + 213, 407, 26, 26, 26, 408, 409, 410, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 411, 412, 213, 413, 213, 213, 213, 414, 415, 416, + 417, 213, 213, 213, 213, 418, 119, 419, 420, 421, 422, 423, 424, 425, + 426, 213, 213, 119, 119, 119, 427, 119, 119, 119, 119, 119, 119, 119, + 119, 119, 119, 119, 428, 119, 429, 119, 430, 431, 432, 433, 434, 119, + 119, 119, 119, 119, 435, 436, 437, 119, 119, 438, 366, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 439, + 440, 26, 441, 181, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 152, 26, 442, 26, 26, 26, 26, 443, 444, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 445, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 446, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 165, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 177, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 447, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 448, 26, 26, 26, 449, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 450, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 451, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 452, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 26, 445, 26, + 26, 26, 26, 26, 452, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 453, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 454, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 455, 456, 213, 213, 12, 12, 12, 457, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, }; static const unsigned short index2[] = { @@ -2382,1586 +2991,1316 @@ static const unsigned short index2[] = { 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 34, 29, 30, 29, 30, 29, 30, 35, 36, 37, 29, 30, 29, 30, 38, 29, 30, 39, 39, 29, 30, 19, 40, 41, 42, 29, 30, 39, 43, 44, 45, 46, 29, - 30, 47, 19, 45, 48, 49, 50, 29, 30, 29, 30, 29, 30, 51, 29, 30, 51, 19, - 19, 29, 30, 51, 29, 30, 52, 52, 29, 30, 29, 30, 53, 29, 30, 19, 54, 29, - 30, 19, 55, 54, 54, 54, 54, 56, 57, 58, 56, 57, 58, 56, 57, 58, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 59, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 60, 56, 57, - 58, 29, 30, 61, 62, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 47, 48, 45, 49, 50, 51, 29, 30, 29, 30, 29, 30, 52, 29, 30, 52, 19, + 19, 29, 30, 52, 29, 30, 53, 53, 29, 30, 29, 30, 54, 29, 30, 19, 55, 29, + 30, 19, 56, 55, 55, 55, 55, 57, 58, 59, 57, 58, 59, 57, 58, 59, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 60, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 61, 57, 58, + 59, 29, 30, 62, 63, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 63, 19, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 19, 19, 19, 19, 19, 64, - 29, 30, 65, 66, 67, 67, 29, 30, 68, 69, 70, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 71, 72, 73, 74, 75, 19, 76, 76, 19, 77, 19, 78, 79, 19, 19, - 19, 76, 80, 19, 81, 19, 82, 83, 19, 84, 85, 83, 86, 87, 19, 19, 85, 19, - 88, 89, 19, 19, 90, 19, 19, 19, 19, 19, 19, 19, 91, 19, 19, 92, 19, 93, - 92, 19, 19, 19, 94, 92, 95, 96, 96, 97, 19, 19, 19, 19, 19, 98, 19, 54, - 19, 19, 19, 19, 19, 19, 19, 19, 99, 100, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 102, 102, 102, 102, 102, 102, 102, 101, 101, 5, 5, 5, 5, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 101, 101, 101, 101, 101, 5, 5, 5, 5, 5, 5, 5, - 102, 5, 102, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 24, 24, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 64, 19, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 19, 19, 19, 19, 19, 65, + 29, 30, 66, 67, 68, 68, 29, 30, 69, 70, 71, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 72, 73, 74, 75, 76, 19, 77, 77, 19, 78, 19, 79, 80, 19, 19, + 19, 77, 81, 19, 82, 83, 84, 85, 19, 86, 87, 85, 88, 89, 19, 19, 87, 19, + 90, 91, 19, 19, 92, 19, 19, 19, 19, 19, 19, 19, 93, 19, 19, 94, 19, 95, + 94, 19, 19, 19, 96, 94, 97, 98, 98, 99, 19, 19, 19, 19, 19, 100, 19, 55, + 19, 19, 19, 19, 19, 19, 19, 19, 101, 102, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 104, 104, 104, 104, 104, 104, 104, 103, 103, 5, 5, 5, 5, 104, + 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 103, 103, 103, 103, 103, 5, 5, 5, 5, 5, 5, 5, + 104, 5, 104, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 103, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 105, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 29, 30, 29, 30, 102, 5, 29, 30, 0, 0, 104, 49, 49, 49, 4, 105, 0, - 0, 0, 0, 5, 5, 106, 24, 107, 107, 107, 0, 108, 0, 109, 109, 110, 16, 16, + 24, 24, 29, 30, 29, 30, 104, 5, 29, 30, 0, 0, 106, 50, 50, 50, 4, 107, 0, + 0, 0, 0, 5, 5, 108, 24, 109, 109, 109, 0, 110, 0, 111, 111, 112, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 0, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 111, 112, 112, 112, 113, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 114, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 115, 116, 116, 117, 118, 119, 120, 120, 120, 121, 122, - 123, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 124, 125, 126, 127, 128, 129, 4, 29, 30, 130, - 29, 30, 19, 63, 63, 63, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 113, 114, 114, 114, 115, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 116, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 117, 118, 118, 119, 120, 121, 122, 122, 122, 123, 124, + 125, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 126, 127, 128, 129, 130, 131, 4, 29, 30, 132, + 29, 30, 19, 64, 64, 64, 133, 133, 133, 133, 133, 133, 133, 133, 133, 133, + 133, 133, 133, 133, 133, 133, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 4, 24, 24, 24, 24, 24, 5, 5, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 133, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 134, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 135, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 136, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 0, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 0, 0, 102, 4, 4, 4, 4, 4, 5, 19, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 137, 19, 4, 4, 0, 0, 4, 4, 4, 0, 24, 24, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 0, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 0, 0, 104, 4, 4, 4, 4, 4, 5, 19, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 139, 19, 4, 4, 0, 0, 4, 4, 4, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 24, 4, 24, 24, 4, 24, 24, 4, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 54, 54, - 54, 54, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 20, 20, 20, 20, 4, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, + 55, 55, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 20, 20, 20, 20, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, - 20, 4, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 24, + 20, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 4, 4, 4, 4, 54, 54, 24, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 4, 54, 24, 24, 24, 24, 24, 24, 24, 20, 4, 24, 24, 24, 24, 24, 24, - 102, 102, 24, 24, 4, 24, 24, 24, 24, 54, 54, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 54, 54, 54, 4, 4, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 0, 20, 54, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, + 12, 13, 14, 15, 4, 4, 4, 4, 55, 55, 24, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 4, 55, 24, 24, 24, 24, 24, 24, 24, 20, 4, 24, 24, 24, 24, 24, 24, + 104, 104, 24, 24, 4, 24, 24, 24, 24, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 55, 55, 55, 4, 4, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 20, 55, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 102, 102, 4, 4, 4, 4, 102, 0, 0, 24, 4, - 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 24, 24, 24, 24, 102, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 102, 24, 24, 24, 102, 24, 24, 24, 24, 24, 0, 0, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 0, 0, - 4, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 5, 54, 54, 54, 54, 54, 54, 0, 20, 20, 0, 0, 0, 0, 0, 0, - 24, 24, 24, 24, 24, 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 104, 104, 4, 4, 4, 4, 104, 0, 0, 24, 4, + 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 24, 24, 24, 24, 104, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 104, 24, 24, 24, 104, 24, 24, 24, 24, 24, 0, 0, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 0, 0, + 4, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 5, 55, 55, 55, 55, 55, 55, 0, 20, 20, 0, 0, 0, 0, 0, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 20, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 17, 24, 54, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, - 17, 24, 17, 17, 54, 24, 24, 24, 24, 24, 24, 24, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 24, 24, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 102, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 17, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 0, 0, 0, 54, 54, 54, 54, 0, 0, 24, - 54, 17, 17, 17, 24, 24, 24, 24, 0, 0, 17, 17, 0, 0, 17, 17, 24, 54, 0, 0, - 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 54, 54, 0, 54, 54, 54, 24, 24, 0, 0, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, 4, 4, 26, 26, 26, 26, 26, 26, 4, - 4, 54, 4, 24, 0, 0, 24, 24, 17, 0, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, - 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, - 54, 54, 0, 54, 54, 0, 0, 24, 0, 17, 17, 17, 24, 24, 0, 0, 0, 0, 24, 24, - 0, 0, 24, 24, 24, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 0, - 54, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 24, 54, - 54, 54, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 0, 0, 24, 54, 17, 17, - 17, 24, 24, 24, 24, 24, 0, 24, 24, 17, 0, 17, 17, 24, 0, 0, 54, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 24, 24, 0, 0, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 4, 4, 0, 0, 0, 0, 0, 0, 0, 54, 24, 24, 24, 24, 24, - 24, 0, 24, 17, 17, 0, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, - 54, 54, 0, 0, 24, 54, 17, 24, 17, 24, 24, 24, 24, 0, 0, 17, 17, 0, 0, 17, - 17, 24, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 0, 0, 0, 0, 54, 54, 0, 54, 54, - 54, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 54, 26, 26, 26, - 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 54, 0, 54, 54, 54, 54, 54, - 54, 0, 0, 0, 54, 54, 54, 0, 54, 54, 54, 54, 0, 0, 0, 54, 54, 0, 54, 0, - 54, 54, 0, 0, 0, 54, 54, 0, 0, 0, 54, 54, 54, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 17, 17, 24, 17, 17, 0, 0, 0, - 17, 17, 17, 0, 17, 17, 17, 24, 0, 0, 54, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 24, 17, 24, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, + 17, 24, 17, 17, 55, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 24, 24, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 104, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 17, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 0, 0, 55, 55, 55, 55, 0, 0, 24, + 55, 17, 17, 17, 24, 24, 24, 24, 0, 0, 17, 17, 0, 0, 17, 17, 24, 55, 0, 0, + 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 55, 55, 0, 55, 55, 55, 24, 24, 0, 0, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 4, 4, 26, 26, 26, 26, 26, 26, 4, + 4, 55, 4, 24, 0, 0, 24, 24, 17, 0, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, + 55, 55, 0, 55, 55, 0, 0, 24, 0, 17, 17, 17, 24, 24, 0, 0, 0, 0, 24, 24, + 0, 0, 24, 24, 24, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 0, + 55, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 24, 55, + 55, 55, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 0, 0, 24, 55, 17, 17, + 17, 24, 24, 24, 24, 24, 0, 24, 24, 17, 0, 17, 17, 24, 0, 0, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 4, 4, 0, 0, 0, 0, 0, 0, 0, 55, 24, 24, 24, 24, 24, + 24, 0, 24, 17, 17, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, + 55, 55, 0, 0, 24, 55, 17, 24, 17, 24, 24, 24, 24, 0, 0, 17, 17, 0, 0, 17, + 17, 24, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 0, 0, 0, 0, 55, 55, 0, 55, 55, + 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 55, 26, 26, 26, + 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 55, 0, 55, 55, 55, 55, 55, + 55, 0, 0, 0, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 0, 55, 55, 0, 55, 0, + 55, 55, 0, 0, 0, 55, 55, 0, 0, 0, 55, 55, 55, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 17, 17, 24, 17, 17, 0, 0, 0, + 17, 17, 17, 0, 17, 17, 17, 24, 0, 0, 55, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, - 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 24, 17, 17, 17, 24, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 24, 54, + 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 24, 17, 17, 17, 24, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 55, 24, 24, 24, 17, 17, 17, 17, 0, 24, 24, 24, 0, 24, 24, 24, 24, 0, 0, 0, 0, - 0, 0, 0, 24, 24, 0, 54, 54, 54, 0, 0, 54, 0, 0, 54, 54, 24, 24, 0, 0, 6, + 0, 0, 0, 24, 24, 0, 55, 55, 55, 0, 0, 55, 0, 0, 55, 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 4, 26, 26, 26, 26, - 26, 26, 26, 4, 54, 24, 17, 17, 4, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 54, 54, 54, 54, 54, 0, 0, 24, 54, 17, 24, 17, 17, 17, 17, 17, 0, + 26, 26, 26, 4, 55, 24, 17, 17, 4, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 55, 0, 0, 24, 55, 17, 24, 17, 17, 17, 17, 17, 0, 24, 17, 17, 0, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 17, 17, 0, 0, 0, 0, - 0, 0, 54, 54, 0, 54, 54, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 0, 54, 54, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 17, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 24, 54, 17, 17, 17, 24, 24, 24, 24, 0, 17, 17, 17, 0, 17, 17, 17, 24, - 54, 4, 0, 0, 0, 0, 54, 54, 54, 17, 26, 26, 26, 26, 26, 26, 26, 54, 54, - 54, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 4, 54, 54, 54, 54, 54, 54, 0, 24, 17, 17, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 24, 0, 0, 0, 0, 17, 17, 17, 24, 24, + 0, 0, 55, 55, 0, 55, 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 0, 55, 55, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 17, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 24, 24, 55, 17, 17, 17, 24, 24, 24, 24, 0, 17, 17, 17, 0, 17, 17, 17, 24, + 55, 4, 0, 0, 0, 0, 55, 55, 55, 17, 26, 26, 26, 26, 26, 26, 26, 55, 55, + 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 4, 55, 55, 55, 55, 55, 55, 0, 24, 17, 17, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 24, 0, 0, 0, 0, 17, 17, 17, 24, 24, 24, 0, 24, 0, 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 17, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 54, 138, 24, 24, - 24, 24, 24, 24, 24, 0, 0, 0, 0, 4, 54, 54, 54, 54, 54, 54, 102, 24, 24, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 55, 140, 24, 24, + 24, 24, 24, 24, 24, 0, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 104, 24, 24, 24, 24, 24, 24, 24, 24, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 0, 54, 0, 54, 54, 54, 54, 54, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 24, 54, 138, 24, 24, 24, 24, 24, 24, 24, 24, 24, 54, 0, 0, 54, 54, - 54, 54, 54, 0, 102, 0, 24, 24, 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 0, 0, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 0, 55, 0, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 24, 55, 140, 24, 24, 24, 24, 24, 24, 24, 24, 24, 55, 0, 0, 55, 55, + 55, 55, 55, 0, 104, 0, 24, 24, 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 0, 0, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 4, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 4, 24, 4, 24, 4, 24, 4, 4, 4, 4, 17, 17, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 17, 24, 24, 24, 24, 24, 4, 24, 24, 54, 54, 54, 54, 54, + 26, 26, 26, 26, 4, 24, 4, 24, 4, 24, 4, 4, 4, 4, 17, 17, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 17, 24, 24, 24, 24, 24, 4, 24, 24, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 4, 4, 4, 4, 4, 4, 4, 4, 24, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 24, - 24, 24, 24, 17, 24, 24, 24, 24, 24, 24, 17, 24, 24, 17, 17, 24, 24, 54, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, 4, 4, 54, 54, 54, 54, 54, - 54, 17, 17, 24, 24, 54, 54, 54, 54, 24, 24, 24, 54, 17, 17, 17, 54, 54, - 17, 17, 17, 17, 17, 17, 17, 54, 54, 54, 24, 24, 24, 24, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 17, 24, 24, 17, 17, 17, 17, - 17, 17, 24, 54, 17, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 17, 17, 24, - 4, 4, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, - 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, - 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 0, 139, 0, 0, 0, - 0, 0, 139, 0, 0, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, - 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, - 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, - 140, 140, 140, 140, 4, 101, 140, 140, 140, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 0, 54, 54, 54, 54, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 0, 54, 0, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 24, 24, 24, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 141, 142, 143, 144, 145, 146, 147, 148, 149, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, - 0, 0, 0, 0, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, - 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, - 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, - 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, - 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - 232, 233, 234, 235, 0, 0, 236, 237, 238, 239, 240, 241, 0, 0, 4, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 1, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, - 242, 242, 242, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 17, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 0, 24, 24, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 24, 24, 17, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, - 17, 17, 17, 17, 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 4, 4, 4, 102, 4, 4, 4, 4, 54, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 20, 24, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 243, 243, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 24, 54, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 24, 24, 24, 17, 17, 17, 17, 24, 24, - 17, 17, 17, 0, 0, 0, 0, 17, 17, 24, 17, 17, 17, 17, 17, 17, 24, 24, 24, - 0, 0, 0, 0, 4, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 141, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 24, 17, 17, 24, 0, 0, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 17, 24, 17, 24, 24, 24, 24, 24, 24, 24, 0, - 24, 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, 17, 17, - 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 24, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, - 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 102, 4, 4, 4, 4, 4, 4, 0, 0, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 17, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 24, 24, 24, 24, 24, 17, 24, - 17, 17, 17, 17, 17, 24, 17, 17, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 24, 24, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, 24, 24, 24, 54, 54, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 24, - 24, 17, 17, 17, 24, 17, 24, 24, 24, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, - 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 17, 17, 17, 17, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, - 17, 24, 24, 0, 0, 0, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 0, 0, 54, 54, 54, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 102, 102, 102, 102, 102, 102, 4, 4, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 0, 0, 0, 0, 0, 0, 0, 253, 253, - 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, - 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, - 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 0, 0, - 253, 253, 253, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, - 24, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, - 24, 24, 24, 24, 24, 54, 54, 54, 54, 24, 54, 54, 54, 54, 54, 54, 24, 54, - 54, 17, 24, 24, 54, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 24, + 24, 24, 24, 17, 24, 24, 24, 24, 24, 24, 17, 24, 24, 17, 17, 24, 24, 55, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, + 55, 17, 17, 24, 24, 55, 55, 55, 55, 24, 24, 24, 55, 17, 17, 17, 55, 55, + 17, 17, 17, 17, 17, 17, 17, 55, 55, 55, 24, 24, 24, 24, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 17, 24, 24, 17, 17, 17, 17, + 17, 17, 24, 55, 17, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 17, 17, 24, + 4, 4, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 0, 141, 0, 0, 0, + 0, 0, 141, 0, 0, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, + 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, + 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, + 142, 142, 142, 142, 4, 103, 142, 142, 142, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, + 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, + 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 143, 144, 145, 146, 147, + 148, 149, 150, 151, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 152, 153, 154, 155, 156, 157, 158, + 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, + 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, + 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, + 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 233, 234, 235, 236, 237, 0, 0, 238, 239, 240, 241, + 242, 243, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 1, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 4, 4, 4, 244, 244, 244, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 24, 24, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, + 17, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 0, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 17, 24, 24, 24, 24, 24, + 24, 24, 17, 17, 17, 17, 17, 17, 17, 17, 24, 17, 17, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 4, 4, 4, 104, 4, 4, 4, 4, 55, 24, 0, 0, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, + 24, 20, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 245, 245, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 24, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 24, 24, 24, + 17, 17, 17, 17, 24, 24, 17, 17, 17, 0, 0, 0, 0, 17, 17, 24, 17, 17, 17, + 17, 17, 17, 24, 24, 24, 0, 0, 0, 0, 4, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 143, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 24, 24, 17, 17, 24, 0, 0, 4, 4, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 24, 17, 24, 24, + 24, 24, 24, 24, 24, 0, 24, 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 24, + 24, 17, 17, 17, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, + 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 104, 4, 4, 4, + 4, 4, 4, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, + 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 24, 24, + 24, 24, 24, 17, 24, 17, 17, 17, 17, 17, 24, 17, 17, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 17, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, 24, 24, + 24, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 24, 17, 24, 24, 17, 17, 17, 24, 17, 24, 24, 24, 17, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 17, 17, 17, 17, 17, 24, 24, 24, + 24, 24, 24, 24, 24, 17, 17, 24, 24, 0, 0, 0, 4, 4, 4, 4, 4, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 0, 0, 0, 55, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 104, 104, + 104, 104, 104, 4, 4, 246, 247, 248, 249, 250, 251, 252, 253, 254, 29, 30, + 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 0, 0, 255, 255, 255, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 24, 24, 24, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 17, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 24, 55, 55, 55, + 55, 55, 55, 24, 55, 55, 17, 24, 24, 55, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 101, 254, 19, 19, 19, 255, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 256, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 257, 258, 259, 260, 261, 262, 19, - 19, 263, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 103, 256, 19, 19, 19, 257, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 258, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 264, 264, 264, 264, 264, 264, 264, - 264, 265, 265, 265, 265, 265, 265, 265, 265, 264, 264, 264, 264, 264, - 264, 0, 0, 265, 265, 265, 265, 265, 265, 0, 0, 264, 264, 264, 264, 264, - 264, 264, 264, 265, 265, 265, 265, 265, 265, 265, 265, 264, 264, 264, - 264, 264, 264, 264, 264, 265, 265, 265, 265, 265, 265, 265, 265, 264, - 264, 264, 264, 264, 264, 0, 0, 265, 265, 265, 265, 265, 265, 0, 0, 266, - 264, 267, 264, 268, 264, 269, 264, 0, 265, 0, 265, 0, 265, 0, 265, 264, - 264, 264, 264, 264, 264, 264, 264, 265, 265, 265, 265, 265, 265, 265, - 265, 270, 270, 271, 271, 271, 271, 272, 272, 273, 273, 274, 274, 275, - 275, 0, 0, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, - 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, 321, 322, 323, 264, 264, 324, 325, 326, 0, 327, - 328, 265, 265, 329, 329, 330, 5, 331, 5, 5, 5, 332, 333, 334, 0, 335, - 336, 337, 337, 337, 337, 338, 5, 5, 5, 264, 264, 339, 340, 0, 0, 341, - 342, 265, 265, 343, 343, 0, 5, 5, 5, 264, 264, 344, 345, 346, 126, 347, - 348, 265, 265, 349, 349, 130, 5, 5, 5, 0, 0, 350, 351, 352, 0, 353, 354, - 355, 355, 356, 356, 357, 5, 5, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 20, - 358, 358, 20, 20, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 5, 4, 4, 5, 2, 2, 20, 20, 20, 20, 20, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 20, 20, 20, 20, 20, 0, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 359, 101, 0, 0, 360, 361, 362, 363, - 364, 365, 4, 4, 4, 4, 4, 101, 359, 25, 21, 22, 360, 361, 362, 363, 364, - 365, 4, 4, 4, 4, 4, 0, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 5, 5, 5, 5, 24, 5, 5, 5, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 120, 4, 4, 4, 4, - 120, 4, 4, 19, 120, 120, 120, 19, 19, 120, 120, 120, 19, 4, 120, 4, 4, - 366, 120, 120, 120, 120, 120, 4, 4, 4, 4, 4, 4, 120, 4, 367, 4, 120, 4, - 368, 369, 120, 120, 366, 19, 120, 120, 370, 120, 19, 54, 54, 54, 54, 19, - 4, 4, 19, 19, 120, 120, 4, 4, 4, 4, 4, 120, 19, 19, 19, 19, 4, 4, 4, 4, - 371, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, - 372, 372, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, - 373, 373, 373, 373, 242, 242, 242, 29, 30, 242, 242, 242, 242, 26, 4, 4, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 25, 21, 22, 360, 361, 362, 363, 364, - 365, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 374, 374, 374, 374, - 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, - 374, 374, 374, 374, 374, 374, 374, 374, 375, 375, 375, 375, 375, 375, - 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - 375, 375, 375, 375, 375, 375, 359, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 359, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 25, 21, 22, 360, 361, - 362, 363, 364, 365, 26, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 25, - 21, 22, 360, 361, 362, 363, 364, 365, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 259, 260, 261, 262, + 263, 264, 19, 19, 265, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 266, 266, 266, 266, 266, 266, 266, 266, 267, 267, 267, 267, + 267, 267, 267, 267, 266, 266, 266, 266, 266, 266, 0, 0, 267, 267, 267, + 267, 267, 267, 0, 0, 266, 266, 266, 266, 266, 266, 266, 266, 267, 267, + 267, 267, 267, 267, 267, 267, 266, 266, 266, 266, 266, 266, 266, 266, + 267, 267, 267, 267, 267, 267, 267, 267, 266, 266, 266, 266, 266, 266, 0, + 0, 267, 267, 267, 267, 267, 267, 0, 0, 268, 266, 269, 266, 270, 266, 271, + 266, 0, 267, 0, 267, 0, 267, 0, 267, 266, 266, 266, 266, 266, 266, 266, + 266, 267, 267, 267, 267, 267, 267, 267, 267, 272, 272, 273, 273, 273, + 273, 274, 274, 275, 275, 276, 276, 277, 277, 0, 0, 278, 279, 280, 281, + 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, + 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, + 324, 325, 266, 266, 326, 327, 328, 0, 329, 330, 267, 267, 331, 331, 332, + 5, 333, 5, 5, 5, 334, 335, 336, 0, 337, 338, 339, 339, 339, 339, 340, 5, + 5, 5, 266, 266, 341, 342, 0, 0, 343, 344, 267, 267, 345, 345, 0, 5, 5, 5, + 266, 266, 346, 347, 348, 128, 349, 350, 267, 267, 351, 351, 132, 5, 5, 5, + 0, 0, 352, 353, 354, 0, 355, 356, 357, 357, 358, 358, 359, 5, 5, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 20, 360, 360, 20, 20, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 5, 2, 2, 20, 20, 20, 20, 20, + 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 1, 20, 20, 20, 20, 20, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 361, + 103, 0, 0, 362, 363, 364, 365, 366, 367, 4, 4, 4, 4, 4, 103, 361, 25, 21, + 22, 362, 363, 364, 365, 366, 367, 4, 4, 4, 4, 4, 0, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 5, 5, 5, 5, 24, 5, 5, 5, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 122, 4, 4, 4, 4, 122, 4, 4, 19, 122, 122, 122, 19, 19, 122, 122, + 122, 19, 4, 122, 4, 4, 368, 122, 122, 122, 122, 122, 4, 4, 4, 4, 4, 4, + 122, 4, 369, 4, 122, 4, 370, 371, 122, 122, 368, 19, 122, 122, 372, 122, + 19, 55, 55, 55, 55, 19, 4, 4, 19, 19, 122, 122, 4, 4, 4, 4, 4, 122, 19, + 19, 19, 19, 4, 4, 4, 4, 373, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, + 374, 374, 374, 374, 374, 374, 375, 375, 375, 375, 375, 375, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 244, 244, 244, 29, 30, 244, 244, + 244, 244, 26, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 376, 376, 376, 376, 376, + 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, + 376, 376, 376, 376, 376, 376, 376, 377, 377, 377, 377, 377, 377, 377, + 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, + 377, 377, 377, 377, 377, 361, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 25, + 21, 22, 362, 363, 364, 365, 366, 367, 26, 361, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 25, 21, + 22, 362, 363, 364, 365, 366, 367, 26, 25, 21, 22, 362, 363, 364, 365, + 366, 367, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 29, 30, 376, 377, 378, 379, 380, - 29, 30, 29, 30, 29, 30, 381, 382, 383, 384, 19, 29, 30, 19, 29, 30, 19, - 19, 19, 19, 19, 101, 101, 385, 385, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 19, 4, 4, 4, 4, 4, 4, 29, 30, 29, 30, 24, 24, 24, 29, 30, 0, 0, 0, 0, - 0, 4, 4, 4, 4, 26, 4, 4, 386, 386, 386, 386, 386, 386, 386, 386, 386, - 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, - 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, - 386, 0, 386, 0, 0, 0, 0, 0, 386, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 102, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, - 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 24, 24, 24, 24, 24, 24, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 29, 30, 378, + 379, 380, 381, 382, 29, 30, 29, 30, 29, 30, 383, 384, 385, 386, 19, 29, + 30, 19, 29, 30, 19, 19, 19, 19, 19, 103, 103, 387, 387, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 4, 4, 4, 4, + 4, 4, 29, 30, 29, 30, 24, 24, 24, 29, 30, 0, 0, 0, 0, 0, 4, 4, 4, 4, 26, + 4, 4, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, + 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, + 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 0, 388, 0, 0, 0, + 0, 0, 388, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 104, 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 387, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 389, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 4, 4, 4, 4, 102, 54, 242, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 24, 24, 24, 24, 17, 17, 4, 102, 102, - 102, 102, 102, 4, 4, 242, 242, 242, 102, 54, 4, 4, 4, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 24, 24, 5, 5, 102, 102, 54, - 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 102, 102, 102, 54, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 4, 4, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, - 26, 26, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, + 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 4, 4, 4, 4, 104, 55, 244, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 24, 24, 24, 24, 17, 17, 4, 104, + 104, 104, 104, 104, 4, 4, 244, 244, 244, 104, 55, 4, 4, 4, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 5, 5, 104, 104, + 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 17, 104, 104, 104, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 4, 4, 26, 26, 26, + 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 388, 54, 54, 388, 54, 54, 54, 388, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 388, 54, 388, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 388, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 388, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, - 54, 388, 388, 388, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 388, 388, 388, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 388, 388, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, - 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 388, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 102, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 4, 4, 4, + 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, 26, 26, 4, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 390, 55, 55, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 55, 55, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 390, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 390, 55, 390, 390, 390, 55, 55, 55, 55, 55, 55, 390, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 390, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 390, 390, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 104, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 102, 102, - 102, 102, 102, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, - 4, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 54, - 24, 5, 5, 5, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 102, 29, 30, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 104, 104, 104, 104, 104, 104, 4, 4, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 104, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 101, 101, 24, 24, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 24, 24, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 5, 5, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 19, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 55, 24, 5, 5, 5, 4, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 4, 104, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 103, + 103, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 24, 24, 4, 4, 4, 4, + 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 104, 104, 104, 104, 104, 104, 104, 104, 104, + 5, 5, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 19, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 101, - 19, 19, 19, 19, 19, 19, 19, 19, 29, 30, 29, 30, 389, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 102, 5, 5, 29, 30, 390, 19, 54, 29, 30, 29, 30, 391, - 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 392, 393, 394, 395, 392, 19, 396, 397, 398, 399, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 400, 401, 402, 29, - 30, 29, 30, 0, 0, 0, 0, 0, 29, 30, 0, 19, 0, 19, 29, 30, 29, 30, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 101, - 101, 29, 30, 54, 101, 101, 19, 54, 54, 54, 54, 54, 54, 54, 24, 54, 54, - 54, 24, 54, 54, 54, 54, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 24, 24, 17, 4, 4, - 4, 4, 24, 0, 0, 0, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 54, 54, 54, 54, 54, 54, 4, 4, 4, 54, 4, 54, - 54, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 24, 24, - 24, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 17, 24, 24, - 24, 24, 17, 17, 24, 24, 17, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 0, 102, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 54, 54, - 54, 54, 54, 24, 102, 54, 54, 54, 54, 54, 54, 54, 54, 54, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, - 24, 24, 24, 24, 17, 17, 24, 24, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 54, 24, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 0, 0, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 4, 4, 4, 4, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 54, 54, 54, 54, 54, 54, 4, 4, 4, - 54, 17, 24, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 54, 24, 24, 24, 54, 54, 24, 24, 54, 54, 54, 54, 54, 24, 24, 54, 24, - 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 102, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 24, - 24, 17, 17, 4, 4, 54, 102, 102, 17, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, - 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, - 54, 54, 54, 54, 54, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 30, 29, 30, 29, 30, 29, 30, 103, 19, 19, 19, 19, 19, 19, 19, 19, 29, 30, + 29, 30, 391, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 104, 5, 5, 29, 30, + 392, 19, 55, 29, 30, 29, 30, 393, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 394, 395, 396, 397, 394, 19, + 398, 399, 400, 401, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 402, 403, 404, 29, 30, 29, 30, 405, 29, 30, 0, 0, 29, 30, 0, + 19, 0, 19, 29, 30, 29, 30, 29, 30, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 103, 103, 29, 30, 55, 103, 103, 19, + 55, 55, 55, 55, 55, 55, 55, 24, 55, 55, 55, 24, 55, 55, 55, 55, 24, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 17, 17, 24, 24, 17, 4, 4, 4, 4, 24, 0, 0, 0, 26, 26, 26, + 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 17, + 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 24, 24, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 55, 55, 55, 55, 55, 55, 4, 4, 4, 55, 4, 55, 55, 24, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, + 24, 24, 24, 24, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 24, 24, 24, 17, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 24, 17, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, + 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 104, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 55, 55, 55, 55, 55, 24, 104, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 17, 17, + 24, 24, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 24, 55, + 55, 55, 55, 55, 55, 55, 55, 24, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 0, 0, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 104, 55, 55, 55, 55, 55, 55, 4, 4, 4, 55, 17, 24, 17, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 55, 24, 24, 24, 55, + 55, 24, 24, 55, 55, 55, 55, 55, 24, 24, 55, 24, 55, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 104, 4, 4, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 24, 24, 17, 17, 4, 4, 55, + 104, 104, 17, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, + 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 403, 19, 19, 19, 19, 19, 19, 19, 5, 101, 101, 101, - 101, 19, 19, 19, 19, 19, 19, 19, 19, 19, 101, 5, 5, 0, 0, 0, 0, 404, 405, - 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, - 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, - 476, 477, 478, 479, 480, 481, 482, 483, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 24, 17, 17, 24, 17, 17, 4, - 17, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 388, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 388, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, - 485, 486, 487, 488, 489, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, - 492, 493, 494, 495, 0, 0, 0, 0, 0, 54, 24, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 54, 54, 0, 54, 0, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 496, 496, 496, - 496, 496, 496, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 496, 496, 4, 4, 4, - 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, - 4, 5, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 17, 4, 4, 5, 0, - 4, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, - 0, 0, 0, 0, 496, 54, 496, 54, 496, 0, 496, 54, 496, 54, 496, 54, 496, 54, - 496, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 20, 0, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 5, 4, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 5, 4, 4, 4, 4, 4, 4, 16, 16, 16, 16, 16, 16, 16, + 407, 19, 19, 19, 19, 19, 19, 19, 5, 103, 103, 103, 103, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 103, 5, 5, 0, 0, 0, 0, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, + 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, + 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 17, 17, 24, 17, 17, 24, 17, 17, 4, 17, 24, 0, 0, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 488, 489, 490, 491, 492, 493, 494, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 495, 496, 497, 498, 499, 0, 0, 0, 0, 0, 55, 24, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 0, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 500, 500, 500, 500, 500, 500, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 500, 500, 4, 4, 4, 4, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 17, 4, 4, 5, 0, 4, 5, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 0, 0, 0, 0, 500, 55, 500, + 55, 500, 0, 500, 55, 500, 55, 500, 55, 500, 55, 500, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 20, 0, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 5, 4, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 5, 4, 4, 4, 4, 4, 4, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 4, 4, 4, 5, 17, 5, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 497, 497, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 0, 0, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, - 54, 0, 0, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 0, 0, 0, 4, 4, 4, 5, - 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, - 20, 4, 4, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 4, 4, 4, 0, 0, 0, - 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 16, 16, 16, 4, 4, 4, 5, 17, 5, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 501, 501, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, + 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 0, 0, 0, 4, + 4, 4, 5, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 20, 20, 20, 4, 4, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 0, 0, 0, 0, 4, 4, 4, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 26, 26, 26, 26, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 4, 4, 4, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 0, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 242, 54, - 54, 54, 54, 54, 54, 54, 54, 242, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, - 24, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 4, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 4, 242, 242, 242, 242, 242, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 498, 498, 498, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, - 498, 0, 0, 0, 0, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 500, 500, - 500, 500, 500, 500, 500, 500, 500, 500, 500, 0, 500, 500, 500, 500, 500, - 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 0, 500, 500, 500, 500, - 500, 500, 500, 0, 500, 500, 0, 501, 501, 501, 501, 501, 501, 501, 501, - 501, 501, 501, 0, 501, 501, 501, 501, 501, 501, 501, 501, 501, 501, 501, - 501, 501, 501, 501, 0, 501, 501, 501, 501, 501, 501, 501, 0, 501, 501, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 101, 102, 102, 101, 101, 101, 0, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 0, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 54, 54, 54, 54, 54, 54, 0, 0, 54, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 54, 54, 0, 0, 0, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 4, 26, 26, 26, 26, - 26, 26, 26, 26, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 26, 26, 26, 26, 26, 26, 26, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 26, 26, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 26, 26, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 0, 0, 0, 0, 26, 26, - 26, 26, 26, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 26, 26, 26, 26, 26, 26, 0, 0, 0, 4, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 26, 26, 54, 54, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 24, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 24, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 244, 55, 55, 55, 55, 55, 55, 55, 55, 244, 0, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 4, 244, 244, 244, 244, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 0, 0, 0, 0, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 4, 504, 504, 504, 504, 504, 504, 504, 504, 504, + 504, 504, 0, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, + 504, 504, 504, 0, 504, 504, 504, 504, 504, 504, 504, 0, 504, 504, 0, 505, + 505, 505, 505, 505, 505, 505, 505, 505, 505, 505, 0, 505, 505, 505, 505, + 505, 505, 505, 505, 505, 505, 505, 505, 505, 505, 505, 0, 505, 505, 505, + 505, 505, 505, 505, 0, 505, 505, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 104, 104, 103, 103, 103, 0, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 0, 0, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 4, 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 4, 4, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 55, 55, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, + 26, 26, 26, 26, 26, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 26, 26, 55, 55, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 54, 24, 24, 24, 0, 24, 24, 0, 0, 0, 0, 0, 24, 24, 24, 24, - 54, 54, 54, 54, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 24, 24, 24, 0, 0, 0, 0, 24, 25, 21, 22, 360, 26, 26, 26, 26, - 26, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 26, 26, 4, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 0, 0, 0, 0, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 26, 26, 26, - 26, 26, 26, 26, 26, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, - 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, - 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, - 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, - 108, 108, 108, 108, 108, 108, 108, 108, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 0, 0, 0, 0, 0, 0, - 0, 26, 26, 26, 26, 26, 26, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 24, 24, 4, 0, 0, 54, 54, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 26, 26, 26, 26, 4, 4, 4, 4, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, - 24, 24, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 24, - 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, - 4, 4, 4, 0, 0, 0, 0, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 24, 54, 54, 24, 24, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 17, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 17, 17, 24, - 24, 4, 4, 20, 4, 4, 4, 4, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 17, 24, 24, 24, - 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, - 54, 17, 17, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 4, 4, 54, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 24, 24, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, - 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 54, 54, 54, 54, 4, 4, 4, - 4, 24, 24, 24, 24, 4, 17, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 4, - 54, 4, 4, 4, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 17, 17, 24, 17, 24, 24, 4, 4, 4, - 4, 4, 4, 24, 54, 54, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 0, 54, 0, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 4, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 17, 17, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, - 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 0, 24, 24, 54, 17, - 17, 24, 17, 17, 17, 17, 0, 0, 17, 17, 0, 0, 17, 17, 17, 0, 0, 54, 0, 0, - 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 17, 17, 0, 0, 24, 24, - 24, 24, 24, 24, 24, 0, 0, 0, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 24, 24, 24, 17, - 24, 54, 54, 54, 54, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, - 4, 0, 4, 24, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 24, 24, 17, 24, 17, 17, 17, - 17, 24, 24, 17, 24, 24, 54, 54, 4, 54, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 0, 0, - 17, 17, 17, 17, 24, 24, 17, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 54, 54, 54, 54, 24, 24, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, - 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 24, 17, 24, 24, 4, 4, 4, 54, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, - 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 17, 24, 54, 4, 0, 0, 0, 0, 0, 0, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 24, 24, 24, 17, 17, 24, 24, 24, 24, 17, 24, 24, 24, + 26, 26, 55, 24, 24, 24, 0, 24, 24, 0, 0, 0, 0, 0, 24, 24, 24, 24, 55, 55, + 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 24, 24, 24, 0, 0, 0, 0, 24, 25, 21, 22, 362, 26, 26, 26, 26, 26, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 4, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 0, 0, 0, + 0, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, + 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, + 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 55, + 55, 104, 55, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 0, 0, 0, 24, 24, 24, 24, 24, 4, 104, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 21, 22, + 362, 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 24, 24, 4, 0, 0, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 55, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 26, 26, 26, 26, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 25, 21, 22, 362, + 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 55, 55, 24, 24, 55, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 24, 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, + 17, 24, 24, 24, 24, 17, 17, 24, 24, 4, 4, 20, 4, 4, 4, 4, 24, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 20, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, + 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, + 24, 24, 24, 24, 17, 24, 24, 24, 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 4, 4, 4, 4, 55, 17, 17, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, + 4, 4, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 17, 17, 55, 55, 55, 55, 4, 4, 4, 4, 24, 24, 24, 24, 4, 17, 24, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 55, 4, 55, 4, 4, 4, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, + 17, 17, 24, 17, 24, 24, 4, 4, 4, 4, 4, 4, 24, 55, 55, 24, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, + 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 17, 17, 24, 24, 24, 24, 24, 24, + 24, 24, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 24, 24, 17, 17, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, + 55, 55, 0, 24, 24, 55, 17, 17, 24, 17, 17, 17, 17, 0, 0, 17, 17, 0, 0, + 17, 17, 17, 0, 0, 55, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 17, 17, 0, 0, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 24, 24, 24, + 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 55, 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 17, 17, 17, 24, 24, 24, 24, + 24, 24, 0, 17, 0, 0, 17, 0, 17, 17, 17, 17, 0, 17, 17, 24, 17, 24, 55, + 24, 55, 4, 4, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, + 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 24, 24, 24, 17, 24, 55, 55, + 55, 55, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 0, 4, + 24, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 17, 24, 17, 17, 17, 17, 24, + 24, 17, 24, 24, 55, 55, 4, 55, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 0, 0, 17, 17, + 17, 17, 24, 24, 17, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, + 24, 24, 24, 24, 24, 24, 17, 17, 24, 17, 24, 24, 4, 4, 4, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 24, 17, + 17, 24, 24, 24, 24, 24, 24, 17, 24, 55, 4, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 24, 17, 24, 17, 17, 24, 24, 24, 24, 17, 24, 24, 24, 24, 24, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 4, 4, 4, - 4, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, - 17, 17, 17, 0, 17, 17, 0, 0, 24, 24, 17, 24, 54, 17, 54, 17, 24, 4, 4, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 0, 0, 24, 24, - 17, 17, 17, 17, 24, 54, 4, 54, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 17, 54, 24, 24, 24, - 24, 4, 4, 4, 4, 4, 4, 4, 4, 24, 0, 0, 0, 0, 0, 0, 0, 0, 54, 24, 24, 24, - 24, 24, 24, 17, 17, 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, 4, 4, 4, - 54, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, - 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 17, 24, - 54, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 0, 0, 0, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 0, 17, 24, 24, 24, 24, 24, 24, 24, 17, 24, - 24, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 24, 24, 24, 0, 0, 0, 24, 0, 24, 24, 0, 24, 24, 24, 24, 24, 24, - 24, 54, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, 17, 17, 0, 24, 24, 0, - 17, 17, 24, 17, 24, 54, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 17, 17, 4, 4, - 0, 0, 0, 0, 0, 0, 0, 24, 24, 54, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 17, 17, 24, 24, 24, 24, 24, 0, 0, 0, 17, 17, 24, 17, 24, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 0, 4, 4, 4, 4, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 24, - 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, - 0, 0, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 24, 24, 24, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 102, 102, 102, 102, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 26, 26, 26, 26, 26, 26, 26, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 18, 18, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 0, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 17, 17, 17, 17, 17, 17, 0, 17, 17, 0, 0, 24, 24, 17, 24, + 55, 17, 55, 17, 24, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, + 24, 24, 24, 24, 0, 0, 24, 24, 17, 17, 17, 17, 24, 55, 4, 55, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, + 24, 24, 24, 17, 55, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 24, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 24, 24, 24, 24, 24, 24, 17, 17, 24, 24, 24, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 17, 24, 24, 4, 4, 4, 55, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 17, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 17, 24, 55, 4, + 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 0, 0, 0, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 0, 17, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, 17, + 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 24, 24, 24, 24, 24, 24, 0, 0, 0, 24, 0, 24, 24, 0, 24, 24, + 24, 24, 24, 24, 24, 55, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 17, + 17, 0, 24, 24, 0, 17, 17, 24, 17, 24, 55, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 17, 17, 4, 4, 0, + 0, 0, 0, 0, 0, 0, 24, 24, 55, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 17, 17, 24, 24, 24, 24, 24, 0, 0, 0, 17, 17, 24, 17, 24, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 4, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 0, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 24, 55, 55, 55, 55, 55, 55, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 17, 17, 17, 24, 24, 24, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 24, 24, + 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 104, + 104, 104, 104, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 0, 26, 26, 26, 26, 26, 26, 26, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 104, 104, 104, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 104, 4, 4, 4, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, + 18, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 0, 0, 24, 54, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 24, 55, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 102, 4, 102, 24, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 102, 102, 102, 0, 102, - 102, 102, 102, 102, 102, 102, 0, 102, 102, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 0, 0, 4, 24, 24, 4, 20, 20, 20, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 104, 104, 104, 104, 104, 104, 104, + 104, 104, 104, 104, 104, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 104, 104, 4, 104, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, + 104, 104, 104, 0, 104, 104, 104, 104, 104, 104, 104, 0, 104, 104, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 0, 0, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 4, 24, 24, 4, 20, 20, 20, 20, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 24, 24, 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 24, 24, 24, 4, 4, 4, 17, 17, - 17, 17, 17, 17, 20, 20, 20, 20, 20, 20, 20, 20, 24, 24, 24, 24, 24, 24, - 24, 24, 4, 4, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, - 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 24, 24, 24, 4, 4, 4, 17, 17, 17, + 17, 17, 17, 20, 20, 20, 20, 20, 20, 20, 20, 24, 24, 24, 24, 24, 24, 24, + 24, 4, 4, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 24, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 24, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 0, 0, 0, 0, 0, 0, 0, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, + 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, + 0, 0, 0, 0, 0, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 120, 0, 120, 120, 0, 0, 120, 0, 0, 120, 120, 0, 0, 120, 120, 120, - 120, 0, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 0, 19, 0, + 19, 122, 0, 122, 122, 0, 0, 122, 0, 0, 122, 122, 0, 0, 122, 122, 122, + 122, 0, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 0, 19, 0, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, + 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 120, 120, 0, 120, 120, 120, 120, 0, 0, 120, 120, 120, - 120, 120, 120, 120, 120, 0, 120, 120, 120, 120, 120, 120, 120, 0, 19, 19, + 19, 19, 19, 19, 19, 122, 122, 0, 122, 122, 122, 122, 0, 0, 122, 122, 122, + 122, 122, 122, 122, 122, 0, 122, 122, 122, 122, 122, 122, 122, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 120, 120, 0, 120, 120, 120, 120, 0, 120, 120, - 120, 120, 120, 0, 120, 0, 0, 0, 120, 120, 120, 120, 120, 120, 120, 0, 19, + 19, 19, 19, 19, 19, 19, 122, 122, 0, 122, 122, 122, 122, 0, 122, 122, + 122, 122, 122, 0, 122, 0, 0, 0, 122, 122, 122, 122, 122, 122, 122, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 0, 0, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 4, + 19, 19, 19, 0, 0, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, 19, 19, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 4, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, 19, 19, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 4, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, 19, 19, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 4, 19, 19, 19, 19, 19, + 4, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, 19, 19, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 4, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 4, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, - 19, 19, 120, 19, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, + 19, 19, 122, 19, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, @@ -3973,297 +4312,188 @@ static const unsigned short index2[] = { 4, 4, 4, 4, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 54, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 0, 0, - 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, - 0, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 102, - 102, 102, 102, 102, 102, 102, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 0, 0, 0, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 24, 24, 24, 24, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 24, 24, 24, 24, 24, 24, 24, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 502, 502, 502, - 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - 502, 502, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, - 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, - 503, 503, 503, 503, 503, 503, 503, 503, 24, 24, 24, 24, 24, 24, 24, 102, - 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 55, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 0, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, + 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 0, 24, 24, 24, 24, + 24, 0, 0, 0, 0, 0, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 24, 24, 24, 24, 24, 24, + 24, 104, 104, 104, 104, 104, 104, 104, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 0, 0, 0, 0, 55, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, + 24, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 104, 24, 24, 24, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 24, 24, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, + 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 0, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, + 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, + 506, 506, 506, 506, 506, 506, 506, 506, 506, 507, 507, 507, 507, 507, + 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, + 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, + 507, 24, 24, 24, 24, 24, 24, 24, 104, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 4, 26, 26, 26, 4, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, + 26, 4, 26, 26, 26, 4, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 0, - 0, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 0, - 54, 0, 54, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 54, 0, 54, 0, 54, 0, 54, 54, - 54, 0, 54, 54, 0, 54, 0, 0, 54, 0, 54, 0, 54, 0, 54, 0, 54, 0, 54, 54, 0, - 54, 0, 0, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 0, 54, 54, 54, 54, 0, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, - 0, 0, 0, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 359, 359, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, 4, 4, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 0, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 0, 55, 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 0, 55, 0, 0, 0, 0, 0, 0, 55, 0, + 0, 0, 0, 55, 0, 55, 0, 55, 0, 55, 55, 55, 0, 55, 55, 0, 55, 0, 0, 55, 0, + 55, 0, 55, 0, 55, 0, 55, 0, 55, 55, 0, 55, 0, 0, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 361, 361, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 4, 4, 4, 4, 4, 4, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 4, 4, 4, 4, 4, 4, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 504, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, + 4, 4, 4, 4, 4, 4, 4, 4, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 4, 4, 4, 4, 4, 4, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 4, 4, 4, 4, 4, 4, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 508, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, + 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, - 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 4, + 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, - 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 390, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, @@ -4273,19 +4503,8 @@ static const unsigned short index2[] = { 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; /* Returns the numeric value as double for Unicode characters @@ -4348,6 +4567,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1018A: case 0x104A0: case 0x10D30: + case 0x10D40: case 0x11066: case 0x110F0: case 0x11136: @@ -4357,17 +4577,23 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D0: case 0x11650: case 0x116C0: + case 0x116D0: + case 0x116DA: case 0x11730: case 0x118E0: case 0x11950: + case 0x11BF0: case 0x11C50: case 0x11D50: case 0x11DA0: case 0x11F50: + case 0x16130: case 0x16A60: case 0x16AC0: case 0x16B50: + case 0x16D70: case 0x16E80: + case 0x1CCF0: case 0x1D2C0: case 0x1D2E0: case 0x1D7CE: @@ -4378,6 +4604,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E140: case 0x1E2F0: case 0x1E4F0: + case 0x1E5F1: case 0x1E950: case 0x1F100: case 0x1F101: @@ -4473,6 +4700,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10BA9: case 0x10CFA: case 0x10D31: + case 0x10D41: case 0x10E60: case 0x10F1D: case 0x10F51: @@ -4488,9 +4716,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D1: case 0x11651: case 0x116C1: + case 0x116D1: + case 0x116DB: case 0x11731: case 0x118E1: case 0x11951: + case 0x11BF1: case 0x11C51: case 0x11C5A: case 0x11D51: @@ -4502,11 +4733,14 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12434: case 0x1244F: case 0x12458: + case 0x16131: case 0x16A61: case 0x16AC1: case 0x16B51: + case 0x16D71: case 0x16E81: case 0x16E94: + case 0x1CCF1: case 0x1D2C1: case 0x1D2E1: case 0x1D360: @@ -4520,6 +4754,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E141: case 0x1E2F1: case 0x1E4F1: + case 0x1E5F2: case 0x1E8C7: case 0x1E951: case 0x1EC71: @@ -4959,6 +5194,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10B79: case 0x10BAA: case 0x10D32: + case 0x10D42: case 0x10E61: case 0x10F1E: case 0x10FC6: @@ -4973,9 +5209,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D2: case 0x11652: case 0x116C2: + case 0x116D2: + case 0x116DC: case 0x11732: case 0x118E2: case 0x11952: + case 0x11BF2: case 0x11C52: case 0x11C5B: case 0x11D52: @@ -4991,11 +5230,14 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12450: case 0x12456: case 0x12459: + case 0x16132: case 0x16A62: case 0x16AC2: case 0x16B52: + case 0x16D72: case 0x16E82: case 0x16E95: + case 0x1CCF2: case 0x1D2C2: case 0x1D2E2: case 0x1D361: @@ -5008,6 +5250,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E142: case 0x1E2F2: case 0x1E4F2: + case 0x1E5F3: case 0x1E8C8: case 0x1E952: case 0x1EC72: @@ -5189,6 +5432,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10B7A: case 0x10BAB: case 0x10D33: + case 0x10D43: case 0x10E62: case 0x10F1F: case 0x10FC7: @@ -5203,9 +5447,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D3: case 0x11653: case 0x116C3: + case 0x116D3: + case 0x116DD: case 0x11733: case 0x118E3: case 0x11953: + case 0x11BF3: case 0x11C53: case 0x11C5C: case 0x11D53: @@ -5226,11 +5473,14 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1244B: case 0x12451: case 0x12457: + case 0x16133: case 0x16A63: case 0x16AC3: case 0x16B53: + case 0x16D73: case 0x16E83: case 0x16E96: + case 0x1CCF3: case 0x1D2C3: case 0x1D2E3: case 0x1D362: @@ -5243,6 +5493,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E143: case 0x1E2F3: case 0x1E4F3: + case 0x1E5F4: case 0x1E8C9: case 0x1E953: case 0x1EC73: @@ -5415,6 +5666,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10B7B: case 0x10BAC: case 0x10D34: + case 0x10D44: case 0x10E63: case 0x10F20: case 0x10FC8: @@ -5429,9 +5681,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D4: case 0x11654: case 0x116C4: + case 0x116D4: + case 0x116DE: case 0x11734: case 0x118E4: case 0x11954: + case 0x11BF4: case 0x11C54: case 0x11C5D: case 0x11D54: @@ -5453,10 +5708,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12452: case 0x12453: case 0x12469: + case 0x16134: case 0x16A64: case 0x16AC4: case 0x16B54: + case 0x16D74: case 0x16E84: + case 0x1CCF4: case 0x1D2C4: case 0x1D2E4: case 0x1D363: @@ -5469,6 +5727,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E144: case 0x1E2F4: case 0x1E4F4: + case 0x1E5F5: case 0x1E8CA: case 0x1E954: case 0x1EC74: @@ -5618,6 +5877,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10AEC: case 0x10CFB: case 0x10D35: + case 0x10D45: case 0x10E64: case 0x10F21: case 0x11056: @@ -5631,9 +5891,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D5: case 0x11655: case 0x116C5: + case 0x116D5: + case 0x116DF: case 0x11735: case 0x118E5: case 0x11955: + case 0x11BF5: case 0x11C55: case 0x11C5E: case 0x11D55: @@ -5651,10 +5914,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12454: case 0x12455: case 0x1246A: + case 0x16135: case 0x16A65: case 0x16AC5: case 0x16B55: + case 0x16D75: case 0x16E85: + case 0x1CCF5: case 0x1D2C5: case 0x1D2E5: case 0x1D364: @@ -5668,6 +5934,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E145: case 0x1E2F5: case 0x1E4F5: + case 0x1E5F6: case 0x1E8CB: case 0x1E955: case 0x1EC75: @@ -5818,6 +6085,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x104A6: case 0x109C5: case 0x10D36: + case 0x10D46: case 0x10E65: case 0x11057: case 0x1106C: @@ -5830,9 +6098,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D6: case 0x11656: case 0x116C6: + case 0x116D6: + case 0x116E0: case 0x11736: case 0x118E6: case 0x11956: + case 0x11BF6: case 0x11C56: case 0x11C5F: case 0x11D56: @@ -5846,10 +6117,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12440: case 0x1244E: case 0x1246B: + case 0x16136: case 0x16A66: case 0x16AC6: case 0x16B56: + case 0x16D76: case 0x16E86: + case 0x1CCF6: case 0x1D2C6: case 0x1D2E6: case 0x1D365: @@ -5861,6 +6135,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E146: case 0x1E2F6: case 0x1E4F6: + case 0x1E5F7: case 0x1E8CC: case 0x1E956: case 0x1EC76: @@ -5971,6 +6246,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x104A7: case 0x109C6: case 0x10D37: + case 0x10D47: case 0x10E66: case 0x11058: case 0x1106D: @@ -5983,9 +6259,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D7: case 0x11657: case 0x116C7: + case 0x116D7: + case 0x116E1: case 0x11737: case 0x118E7: case 0x11957: + case 0x11BF7: case 0x11C57: case 0x11C60: case 0x11D57: @@ -6000,10 +6279,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12442: case 0x12443: case 0x1246C: + case 0x16137: case 0x16A67: case 0x16AC7: case 0x16B57: + case 0x16D77: case 0x16E87: + case 0x1CCF7: case 0x1D2C7: case 0x1D2E7: case 0x1D366: @@ -6015,6 +6297,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E147: case 0x1E2F7: case 0x1E4F7: + case 0x1E5F8: case 0x1E8CD: case 0x1E957: case 0x1EC77: @@ -6125,6 +6408,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x104A8: case 0x109C7: case 0x10D38: + case 0x10D48: case 0x10E67: case 0x11059: case 0x1106E: @@ -6137,9 +6421,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D8: case 0x11658: case 0x116C8: + case 0x116D8: + case 0x116E2: case 0x11738: case 0x118E8: case 0x11958: + case 0x11BF8: case 0x11C58: case 0x11C61: case 0x11D58: @@ -6153,10 +6440,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12444: case 0x12445: case 0x1246D: + case 0x16138: case 0x16A68: case 0x16AC8: case 0x16B58: + case 0x16D78: case 0x16E88: + case 0x1CCF8: case 0x1D2C8: case 0x1D2E8: case 0x1D367: @@ -6168,6 +6458,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E148: case 0x1E2F8: case 0x1E4F8: + case 0x1E5F9: case 0x1E8CE: case 0x1E958: case 0x1EC78: @@ -6275,6 +6566,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x104A9: case 0x109C8: case 0x10D39: + case 0x10D49: case 0x10E68: case 0x1105A: case 0x1106F: @@ -6287,9 +6579,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D9: case 0x11659: case 0x116C9: + case 0x116D9: + case 0x116E3: case 0x11739: case 0x118E9: case 0x11959: + case 0x11BF9: case 0x11C59: case 0x11C62: case 0x11D59: @@ -6305,10 +6600,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12448: case 0x12449: case 0x1246E: + case 0x16139: case 0x16A69: case 0x16AC9: case 0x16B59: + case 0x16D79: case 0x16E89: + case 0x1CCF9: case 0x1D2C9: case 0x1D2E9: case 0x1D368: @@ -6320,6 +6618,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E149: case 0x1E2F9: case 0x1E4F9: + case 0x1E5FA: case 0x1E8CF: case 0x1E959: case 0x1EC79: diff --git a/Objects/unionobject.c b/Objects/unionobject.c index bf5605686f8df7..6e65a653a95c46 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -1,11 +1,10 @@ // types.UnionType -- used to represent e.g. Union[int, str], int | str #include "Python.h" #include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK -#include "pycore_typevarobject.h" // _PyTypeAlias_Type +#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_typing_type_repr #include "pycore_unionobject.h" - static PyObject *make_union(PyObject *); @@ -81,7 +80,7 @@ is_same(PyObject *left, PyObject *right) static int contains(PyObject **items, Py_ssize_t size, PyObject *obj) { - for (int i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < size; i++) { int is_duplicate = is_same(items[i], obj); if (is_duplicate) { // -1 or 1 return is_duplicate; @@ -97,7 +96,7 @@ merge(PyObject **items1, Py_ssize_t size1, PyObject *tuple = NULL; Py_ssize_t pos = 0; - for (int i = 0; i < size2; i++) { + for (Py_ssize_t i = 0; i < size2; i++) { PyObject *arg = items2[i]; int is_duplicate = contains(items1, size1, arg); if (is_duplicate < 0) { @@ -181,87 +180,32 @@ _Py_union_type_or(PyObject* self, PyObject* other) return new_union; } -static int -union_repr_item(_PyUnicodeWriter *writer, PyObject *p) -{ - PyObject *qualname = NULL; - PyObject *module = NULL; - PyObject *r = NULL; - int rc; - - if (p == (PyObject *)&_PyNone_Type) { - return _PyUnicodeWriter_WriteASCIIString(writer, "None", 4); - } - - if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && - (rc = PyObject_HasAttrWithError(p, &_Py_ID(__args__))) > 0) - { - // It looks like a GenericAlias - goto use_repr; - } - if (rc < 0) { - goto exit; - } - - if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { - goto exit; - } - if (qualname == NULL) { - goto use_repr; - } - if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { - goto exit; - } - if (module == NULL || module == Py_None) { - goto use_repr; - } - - // Looks like a class - if (PyUnicode_Check(module) && - _PyUnicode_EqualToASCIIString(module, "builtins")) - { - // builtins don't need a module name - r = PyObject_Str(qualname); - goto exit; - } - else { - r = PyUnicode_FromFormat("%S.%S", module, qualname); - goto exit; - } - -use_repr: - r = PyObject_Repr(p); -exit: - Py_XDECREF(qualname); - Py_XDECREF(module); - if (r == NULL) { - return -1; - } - rc = _PyUnicodeWriter_WriteStr(writer, r); - Py_DECREF(r); - return rc; -} - static PyObject * union_repr(PyObject *self) { unionobject *alias = (unionobject *)self; Py_ssize_t len = PyTuple_GET_SIZE(alias->args); - _PyUnicodeWriter writer; - _PyUnicodeWriter_Init(&writer); - for (Py_ssize_t i = 0; i < len; i++) { - if (i > 0 && _PyUnicodeWriter_WriteASCIIString(&writer, " | ", 3) < 0) { + // Shortest type name "int" (3 chars) + " | " (3 chars) separator + Py_ssize_t estimate = (len <= PY_SSIZE_T_MAX / 6) ? len * 6 : len; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(estimate); + if (writer == NULL) { + return NULL; + } + + for (Py_ssize_t i = 0; i < len; i++) { + if (i > 0 && PyUnicodeWriter_WriteUTF8(writer, " | ", 3) < 0) { goto error; } PyObject *p = PyTuple_GET_ITEM(alias->args, i); - if (union_repr_item(&writer, p) < 0) { + if (_Py_typing_type_repr(writer, p) < 0) { goto error; } } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); + error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index df74be6aba7244..9e3da1c3394d5b 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1,24 +1,58 @@ #include "Python.h" +#include "pycore_critical_section.h" +#include "pycore_lock.h" #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GET_WEAKREFS_LISTPTR() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pystate.h" #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#ifdef Py_GIL_DISABLED +/* + * Thread-safety for free-threaded builds + * ====================================== + * + * In free-threaded builds we need to protect mutable state of: + * + * - The weakref (wr_object, hash, wr_callback) + * - The referenced object (its head-of-list pointer) + * - The linked list of weakrefs + * + * For now we've chosen to address this in a straightforward way: + * + * - The weakref's hash is protected using the weakref's per-object lock. + * - The other mutable is protected by a striped lock keyed on the referenced + * object's address. + * - The striped lock must be locked using `_Py_LOCK_DONT_DETACH` in order to + * support atomic deletion from WeakValueDictionaries. As a result, we must + * be careful not to perform any operations that could suspend while the + * lock is held. + * + * Since the world is stopped when the GC runs, it is free to clear weakrefs + * without acquiring any locks. + */ +#endif #define GET_WEAKREFS_LISTPTR(o) \ ((PyWeakReference **) _PyObject_GET_WEAKREFS_LISTPTR(o)) Py_ssize_t -_PyWeakref_GetWeakrefCount(PyWeakReference *head) +_PyWeakref_GetWeakrefCount(PyObject *obj) { - Py_ssize_t count = 0; + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + return 0; + } + LOCK_WEAKREFS(obj); + Py_ssize_t count = 0; + PyWeakReference *head = *GET_WEAKREFS_LISTPTR(obj); while (head != NULL) { ++count; head = head->wr_next; } + UNLOCK_WEAKREFS(obj); return count; } @@ -33,54 +67,57 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) self->wr_next = NULL; self->wr_callback = Py_XNewRef(callback); self->vectorcall = weakref_vectorcall; +#ifdef Py_GIL_DISABLED + self->weakrefs_lock = &WEAKREF_LIST_LOCK(ob); + _PyObject_SetMaybeWeakref(ob); + _PyObject_SetMaybeWeakref((PyObject *)self); +#endif } -static PyWeakReference * -new_weakref(PyObject *ob, PyObject *callback) -{ - PyWeakReference *result; - - result = PyObject_GC_New(PyWeakReference, &_PyWeakref_RefType); - if (result) { - init_weakref(result, ob, callback); - PyObject_GC_Track(result); - } - return result; -} - - -/* This function clears the passed-in reference and removes it from the - * list of weak references for the referent. This is the only code that - * removes an item from the doubly-linked list of weak references for an - * object; it is also responsible for clearing the callback slot. - */ +// Clear the weakref and steal its callback into `callback`, if provided. static void -clear_weakref(PyWeakReference *self) +clear_weakref_lock_held(PyWeakReference *self, PyObject **callback) { - PyObject *callback = self->wr_callback; - if (self->wr_object != Py_None) { PyWeakReference **list = GET_WEAKREFS_LISTPTR(self->wr_object); - - if (*list == self) - /* If 'self' is the end of the list (and thus self->wr_next == NULL) - then the weakref list itself (and thus the value of *list) will - end up being set to NULL. */ - *list = self->wr_next; - self->wr_object = Py_None; - if (self->wr_prev != NULL) + if (*list == self) { + /* If 'self' is the end of the list (and thus self->wr_next == + NULL) then the weakref list itself (and thus the value of *list) + will end up being set to NULL. */ + FT_ATOMIC_STORE_PTR(*list, self->wr_next); + } + FT_ATOMIC_STORE_PTR(self->wr_object, Py_None); + if (self->wr_prev != NULL) { self->wr_prev->wr_next = self->wr_next; - if (self->wr_next != NULL) + } + if (self->wr_next != NULL) { self->wr_next->wr_prev = self->wr_prev; + } self->wr_prev = NULL; self->wr_next = NULL; } if (callback != NULL) { - Py_DECREF(callback); + *callback = self->wr_callback; self->wr_callback = NULL; } } +// Clear the weakref and its callback +static void +clear_weakref(PyObject *op) +{ + PyWeakReference *self = _PyWeakref_CAST(op); + PyObject *callback = NULL; + + // self->wr_object may be Py_None if the GC cleared the weakref, so lock + // using the pointer in the weakref. + LOCK_WEAKREFS_FOR_WR(self); + clear_weakref_lock_held(self, &callback); + UNLOCK_WEAKREFS_FOR_WR(self); + Py_XDECREF(callback); +} + + /* Cyclic gc uses this to *just* clear the passed-in reference, leaving * the callback intact and uncalled. It must be possible to call self's * tp_dealloc() after calling this, so self has to be left in a sane enough @@ -95,38 +132,38 @@ clear_weakref(PyWeakReference *self) void _PyWeakref_ClearRef(PyWeakReference *self) { - PyObject *callback; - assert(self != NULL); assert(PyWeakref_Check(self)); - /* Preserve and restore the callback around clear_weakref. */ - callback = self->wr_callback; - self->wr_callback = NULL; - clear_weakref(self); - self->wr_callback = callback; + clear_weakref_lock_held(self, NULL); } static void weakref_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); - clear_weakref((PyWeakReference *) self); + clear_weakref(self); Py_TYPE(self)->tp_free(self); } static int -gc_traverse(PyWeakReference *self, visitproc visit, void *arg) +gc_traverse(PyObject *op, visitproc visit, void *arg) { + PyWeakReference *self = _PyWeakref_CAST(op); Py_VISIT(self->wr_callback); return 0; } static int -gc_clear(PyWeakReference *self) +gc_clear(PyObject *op) { - clear_weakref(self); + PyWeakReference *self = _PyWeakref_CAST(op); + PyObject *callback; + // The world is stopped during GC in free-threaded builds. It's safe to + // call this without holding the lock. + clear_weakref_lock_held(self, &callback); + Py_XDECREF(callback); return 0; } @@ -150,7 +187,7 @@ weakref_vectorcall(PyObject *self, PyObject *const *args, } static Py_hash_t -weakref_hash(PyWeakReference *self) +weakref_hash_lock_held(PyWeakReference *self) { if (self->hash != -1) return self->hash; @@ -164,6 +201,16 @@ weakref_hash(PyWeakReference *self) return self->hash; } +static Py_hash_t +weakref_hash(PyObject *op) +{ + PyWeakReference *self = _PyWeakref_CAST(op); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(self); + hash = weakref_hash_lock_held(self); + Py_END_CRITICAL_SECTION(); + return hash; +} static PyObject * weakref_repr(PyObject *self) @@ -177,13 +224,13 @@ weakref_repr(PyObject *self) PyObject *repr; if (name == NULL || !PyUnicode_Check(name)) { repr = PyUnicode_FromFormat( - "", - self, Py_TYPE(obj)->tp_name, obj); + "", + self, obj, obj); } else { repr = PyUnicode_FromFormat( - "", - self, Py_TYPE(obj)->tp_name, obj, name); + "", + self, obj, obj, name); } Py_DECREF(obj); Py_XDECREF(name); @@ -276,6 +323,135 @@ insert_head(PyWeakReference *newref, PyWeakReference **list) *list = newref; } +/* See if we can reuse either the basic ref or proxy in list instead of + * creating a new weakref + */ +static PyWeakReference * +try_reuse_basic_ref(PyWeakReference *list, PyTypeObject *type, + PyObject *callback) +{ + if (callback != NULL) { + return NULL; + } + + PyWeakReference *ref, *proxy; + get_basic_refs(list, &ref, &proxy); + + PyWeakReference *cand = NULL; + if (type == &_PyWeakref_RefType) { + cand = ref; + } + if ((type == &_PyWeakref_ProxyType) || + (type == &_PyWeakref_CallableProxyType)) { + cand = proxy; + } + + if (cand != NULL && _Py_TryIncref((PyObject *) cand)) { + return cand; + } + return NULL; +} + +static int +is_basic_ref(PyWeakReference *ref) +{ + return (ref->wr_callback == NULL) && PyWeakref_CheckRefExact(ref); +} + +static int +is_basic_proxy(PyWeakReference *proxy) +{ + return (proxy->wr_callback == NULL) && PyWeakref_CheckProxy(proxy); +} + +static int +is_basic_ref_or_proxy(PyWeakReference *wr) +{ + return is_basic_ref(wr) || is_basic_proxy(wr); +} + +/* Insert `newref` in the appropriate position in `list` */ +static void +insert_weakref(PyWeakReference *newref, PyWeakReference **list) +{ + PyWeakReference *ref, *proxy; + get_basic_refs(*list, &ref, &proxy); + + PyWeakReference *prev; + if (is_basic_ref(newref)) { + prev = NULL; + } + else if (is_basic_proxy(newref)) { + prev = ref; + } + else { + prev = (proxy == NULL) ? ref : proxy; + } + + if (prev == NULL) { + insert_head(newref, list); + } + else { + insert_after(newref, prev); + } +} + +static PyWeakReference * +allocate_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) +{ + PyWeakReference *newref = (PyWeakReference *) type->tp_alloc(type, 0); + if (newref == NULL) { + return NULL; + } + init_weakref(newref, obj, callback); + return newref; +} + +static PyWeakReference * +get_or_create_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) +{ + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + PyErr_Format(PyExc_TypeError, + "cannot create weak reference to '%s' object", + Py_TYPE(obj)->tp_name); + return NULL; + } + if (callback == Py_None) + callback = NULL; + + PyWeakReference **list = GET_WEAKREFS_LISTPTR(obj); + if ((type == &_PyWeakref_RefType) || + (type == &_PyWeakref_ProxyType) || + (type == &_PyWeakref_CallableProxyType)) + { + LOCK_WEAKREFS(obj); + PyWeakReference *basic_ref = try_reuse_basic_ref(*list, type, callback); + if (basic_ref != NULL) { + UNLOCK_WEAKREFS(obj); + return basic_ref; + } + PyWeakReference *newref = allocate_weakref(type, obj, callback); + if (newref == NULL) { + UNLOCK_WEAKREFS(obj); + return NULL; + } + insert_weakref(newref, list); + UNLOCK_WEAKREFS(obj); + return newref; + } + else { + // We may not be able to safely allocate inside the lock + PyWeakReference *newref = allocate_weakref(type, obj, callback); + if (newref == NULL) { + return NULL; + } + LOCK_WEAKREFS(obj); + insert_weakref(newref, list); + UNLOCK_WEAKREFS(obj); + return newref; + } +} + static int parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, PyObject **obp, PyObject **callbackp) @@ -286,54 +462,11 @@ parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, static PyObject * weakref___new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - PyWeakReference *self = NULL; PyObject *ob, *callback = NULL; - if (parse_weakref_init_args("__new__", args, kwargs, &ob, &callback)) { - PyWeakReference *ref, *proxy; - PyWeakReference **list; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - if (callback == Py_None) - callback = NULL; - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL && type == &_PyWeakref_RefType) { - if (ref != NULL) { - /* We can re-use an existing reference. */ - return Py_NewRef(ref); - } - } - /* We have to create a new reference. */ - /* Note: the tp_alloc() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - self = (PyWeakReference *) (type->tp_alloc(type, 0)); - if (self != NULL) { - init_weakref(self, ob, callback); - if (callback == NULL && type == &_PyWeakref_RefType) { - insert_head(self, list); - } - else { - PyWeakReference *prev; - - get_basic_refs(*list, &ref, &proxy); - prev = (proxy == NULL) ? ref : proxy; - if (prev == NULL) - insert_head(self, list); - else - insert_after(self, prev); - } - } + return (PyObject *)get_or_create_weakref(type, ob, callback); } - return (PyObject *)self; + return NULL; } static int @@ -371,11 +504,11 @@ _PyWeakref_RefType = { .tp_vectorcall_offset = offsetof(PyWeakReference, vectorcall), .tp_call = PyVectorcall_Call, .tp_repr = weakref_repr, - .tp_hash = (hashfunc)weakref_hash, + .tp_hash = weakref_hash, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_BASETYPE, - .tp_traverse = (traverseproc)gc_traverse, - .tp_clear = (inquiry)gc_clear, + .tp_traverse = gc_traverse, + .tp_clear = gc_clear, .tp_richcompare = weakref_richcompare, .tp_methods = weakref_methods, .tp_members = weakref_members, @@ -471,10 +604,18 @@ static PyObject * proxy_repr(PyObject *proxy) { PyObject *obj = _PyWeakref_GET_REF(proxy); - PyObject *repr = PyUnicode_FromFormat( - "", - proxy, Py_TYPE(obj)->tp_name, obj); - Py_DECREF(obj); + PyObject *repr; + if (obj != NULL) { + repr = PyUnicode_FromFormat( + "", + proxy, obj, obj); + Py_DECREF(obj); + } + else { + repr = PyUnicode_FromFormat( + "", + proxy); + } return repr; } @@ -551,11 +692,9 @@ proxy_bool(PyObject *proxy) } static void -proxy_dealloc(PyWeakReference *self) +proxy_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); - if (self->wr_callback != NULL) - PyObject_GC_UnTrack((PyObject *)self); clear_weakref(self); PyObject_GC_Del(self); } @@ -716,7 +855,7 @@ _PyWeakref_ProxyType = { sizeof(PyWeakReference), 0, /* methods */ - (destructor)proxy_dealloc, /* tp_dealloc */ + proxy_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -734,8 +873,8 @@ _PyWeakref_ProxyType = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gc_traverse, /* tp_traverse */ - (inquiry)gc_clear, /* tp_clear */ + gc_traverse, /* tp_traverse */ + gc_clear, /* tp_clear */ proxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ proxy_iter, /* tp_iter */ @@ -751,7 +890,7 @@ _PyWeakref_CallableProxyType = { sizeof(PyWeakReference), 0, /* methods */ - (destructor)proxy_dealloc, /* tp_dealloc */ + proxy_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -768,135 +907,29 @@ _PyWeakref_CallableProxyType = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gc_traverse, /* tp_traverse */ - (inquiry)gc_clear, /* tp_clear */ + gc_traverse, /* tp_traverse */ + gc_clear, /* tp_clear */ proxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ proxy_iter, /* tp_iter */ proxy_iternext, /* tp_iternext */ }; - - PyObject * PyWeakref_NewRef(PyObject *ob, PyObject *callback) { - PyWeakReference *result = NULL; - PyWeakReference **list; - PyWeakReference *ref, *proxy; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == Py_None) - callback = NULL; - if (callback == NULL) - /* return existing weak reference if it exists */ - result = ref; - if (result != NULL) - Py_INCREF(result); - else { - /* Note: new_weakref() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - result = new_weakref(ob, callback); - if (result != NULL) { - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL) { - if (ref == NULL) - insert_head(result, list); - else { - /* Someone else added a ref without a callback - during GC. Return that one instead of this one - to avoid violating the invariants of the list - of weakrefs for ob. */ - Py_SETREF(result, (PyWeakReference*)Py_NewRef(ref)); - } - } - else { - PyWeakReference *prev; - - prev = (proxy == NULL) ? ref : proxy; - if (prev == NULL) - insert_head(result, list); - else - insert_after(result, prev); - } - } - } - return (PyObject *) result; + return (PyObject *)get_or_create_weakref(&_PyWeakref_RefType, ob, + callback); } - PyObject * PyWeakref_NewProxy(PyObject *ob, PyObject *callback) { - PyWeakReference *result = NULL; - PyWeakReference **list; - PyWeakReference *ref, *proxy; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == Py_None) - callback = NULL; - if (callback == NULL) - /* attempt to return an existing weak reference if it exists */ - result = proxy; - if (result != NULL) - Py_INCREF(result); - else { - /* Note: new_weakref() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - result = new_weakref(ob, callback); - if (result != NULL) { - PyWeakReference *prev; - - if (PyCallable_Check(ob)) { - Py_SET_TYPE(result, &_PyWeakref_CallableProxyType); - } - else { - Py_SET_TYPE(result, &_PyWeakref_ProxyType); - } - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL) { - if (proxy != NULL) { - /* Someone else added a proxy without a callback - during GC. Return that one instead of this one - to avoid violating the invariants of the list - of weakrefs for ob. */ - Py_SETREF(result, (PyWeakReference*)Py_NewRef(proxy)); - goto skip_insert; - } - prev = ref; - } - else - prev = (proxy == NULL) ? ref : proxy; - - if (prev == NULL) - insert_head(result, list); - else - insert_after(result, prev); - skip_insert: - ; - } + PyTypeObject *type = &_PyWeakref_ProxyType; + if (PyCallable_Check(ob)) { + type = &_PyWeakref_CallableProxyType; } - return (PyObject *) result; + return (PyObject *)get_or_create_weakref(type, ob, callback); } @@ -965,67 +998,82 @@ PyObject_ClearWeakRefs(PyObject *object) PyErr_BadInternalCall(); return; } + list = GET_WEAKREFS_LISTPTR(object); - /* Remove the callback-less basic and proxy references */ - if (*list != NULL && (*list)->wr_callback == NULL) { - clear_weakref(*list); - if (*list != NULL && (*list)->wr_callback == NULL) - clear_weakref(*list); + if (FT_ATOMIC_LOAD_PTR(*list) == NULL) { + // Fast path for the common case + return; } - if (*list != NULL) { - PyWeakReference *current = *list; - Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); - PyObject *exc = PyErr_GetRaisedException(); - - if (count == 1) { - PyObject *callback = current->wr_callback; - - current->wr_callback = NULL; - clear_weakref(current); - if (callback != NULL) { - if (Py_REFCNT((PyObject *)current) > 0) { - handle_callback(current, callback); - } - Py_DECREF(callback); - } + + /* Remove the callback-less basic and proxy references, which always appear + at the head of the list. + */ + for (int done = 0; !done;) { + LOCK_WEAKREFS(object); + if (*list != NULL && is_basic_ref_or_proxy(*list)) { + PyObject *callback; + clear_weakref_lock_held(*list, &callback); + assert(callback == NULL); } - else { - PyObject *tuple; - Py_ssize_t i = 0; - - tuple = PyTuple_New(count * 2); - if (tuple == NULL) { - _PyErr_ChainExceptions1(exc); - return; - } + done = (*list == NULL) || !is_basic_ref_or_proxy(*list); + UNLOCK_WEAKREFS(object); + } - for (i = 0; i < count; ++i) { - PyWeakReference *next = current->wr_next; - - if (Py_REFCNT((PyObject *)current) > 0) { - PyTuple_SET_ITEM(tuple, i * 2, Py_NewRef(current)); - PyTuple_SET_ITEM(tuple, i * 2 + 1, current->wr_callback); - } - else { - Py_DECREF(current->wr_callback); - } - current->wr_callback = NULL; - clear_weakref(current); - current = next; - } - for (i = 0; i < count; ++i) { - PyObject *callback = PyTuple_GET_ITEM(tuple, i * 2 + 1); - - /* The tuple may have slots left to NULL */ - if (callback != NULL) { - PyObject *item = PyTuple_GET_ITEM(tuple, i * 2); - handle_callback((PyWeakReference *)item, callback); - } + /* Deal with non-canonical (subtypes or refs with callbacks) references. */ + Py_ssize_t num_weakrefs = _PyWeakref_GetWeakrefCount(object); + if (num_weakrefs == 0) { + return; + } + + PyObject *exc = PyErr_GetRaisedException(); + PyObject *tuple = PyTuple_New(num_weakrefs * 2); + if (tuple == NULL) { + _PyWeakref_ClearWeakRefsNoCallbacks(object); + PyErr_WriteUnraisable(NULL); + PyErr_SetRaisedException(exc); + return; + } + + Py_ssize_t num_items = 0; + for (int done = 0; !done;) { + PyObject *callback = NULL; + LOCK_WEAKREFS(object); + PyWeakReference *cur = *list; + if (cur != NULL) { + clear_weakref_lock_held(cur, &callback); + if (_Py_TryIncref((PyObject *) cur)) { + assert(num_items / 2 < num_weakrefs); + PyTuple_SET_ITEM(tuple, num_items, (PyObject *) cur); + PyTuple_SET_ITEM(tuple, num_items + 1, callback); + num_items += 2; + callback = NULL; } - Py_DECREF(tuple); } - assert(!PyErr_Occurred()); - PyErr_SetRaisedException(exc); + done = (*list == NULL); + UNLOCK_WEAKREFS(object); + + Py_XDECREF(callback); + } + + for (Py_ssize_t i = 0; i < num_items; i += 2) { + PyObject *callback = PyTuple_GET_ITEM(tuple, i + 1); + if (callback != NULL) { + PyObject *weakref = PyTuple_GET_ITEM(tuple, i); + handle_callback((PyWeakReference *)weakref, callback); + } + } + + Py_DECREF(tuple); + + assert(!PyErr_Occurred()); + PyErr_SetRaisedException(exc); +} + +void +PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *obj) +{ + if (_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + _PyWeakref_ClearWeakRefsNoCallbacks(obj); } } @@ -1038,12 +1086,32 @@ PyObject_ClearWeakRefs(PyObject *object) void _PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type) { - static_builtin_state *state = _PyStaticType_GetState(interp, type); + managed_static_type_state *state = _PyStaticType_GetState(interp, type); PyObject **list = _PyStaticType_GET_WEAKREFS_LISTPTR(state); - while (*list != NULL) { - /* Note that clear_weakref() pops the first ref off the type's - weaklist before clearing its wr_object and wr_callback. - That is how we're able to loop over the list. */ - clear_weakref((PyWeakReference *)*list); + // This is safe to do without holding the lock in free-threaded builds; + // there is only one thread running and no new threads can be created. + while (*list) { + _PyWeakref_ClearRef((PyWeakReference *)*list); } } + +void +_PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj) +{ + /* Modeled after GET_WEAKREFS_LISTPTR(). + + This is never triggered for static types so we can avoid the + (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). */ + PyWeakReference **list = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(obj); + LOCK_WEAKREFS(obj); + while (*list) { + _PyWeakref_ClearRef(*list); + } + UNLOCK_WEAKREFS(obj); +} + +int +_PyWeakref_IsDead(PyObject *weakref) +{ + return _PyWeakref_IS_DEAD(weakref); +} diff --git a/PC/_testconsole.c b/PC/_testconsole.c index 1dc0d230c4d7c3..0dcea866f65d35 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -1,17 +1,16 @@ /* Testing module for multi-phase initialization of extension modules (PEP 489) */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" #ifdef MS_WINDOWS -#include "pycore_fileutils.h" // _Py_get_osfhandle() -#include "pycore_runtime.h" // _Py_ID() - #define WIN32_LEAN_AND_MEAN #include #include @@ -32,6 +31,7 @@ static int execfunc(PyObject *m) PyModuleDef_Slot testconsole_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -57,20 +57,24 @@ module _testconsole _testconsole.write_input file: object - s: PyBytesObject + s: Py_buffer Writes UTF-16-LE encoded bytes to the console as if typed by a user. [clinic start generated code]*/ static PyObject * -_testconsole_write_input_impl(PyObject *module, PyObject *file, - PyBytesObject *s) -/*[clinic end generated code: output=48f9563db34aedb3 input=4c774f2d05770bc6]*/ +_testconsole_write_input_impl(PyObject *module, PyObject *file, Py_buffer *s) +/*[clinic end generated code: output=58631a8985426ad3 input=68062f1bb2e52206]*/ { INPUT_RECORD *rec = NULL; - PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( - &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); + PyObject *mod = PyImport_ImportModule("_io"); + if (mod == NULL) { + return NULL; + } + + PyTypeObject *winconsoleio_type = (PyTypeObject *)PyObject_GetAttrString(mod, "_WindowsConsoleIO"); + Py_DECREF(mod); if (winconsoleio_type == NULL) { return NULL; } @@ -81,8 +85,8 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, return NULL; } - const wchar_t *p = (const wchar_t *)PyBytes_AS_STRING(s); - DWORD size = (DWORD)PyBytes_GET_SIZE(s) / sizeof(wchar_t); + const wchar_t *p = (const wchar_t *)s->buf; + DWORD size = (DWORD)s->len / sizeof(wchar_t); rec = (INPUT_RECORD*)PyMem_Calloc(size, sizeof(INPUT_RECORD)); if (!rec) @@ -96,9 +100,11 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, prec->Event.KeyEvent.uChar.UnicodeChar = *p; } - HANDLE hInput = _Py_get_osfhandle(((winconsoleio*)file)->fd); - if (hInput == INVALID_HANDLE_VALUE) + HANDLE hInput = (HANDLE)_get_osfhandle(((winconsoleio*)file)->fd); + if (hInput == INVALID_HANDLE_VALUE) { + PyErr_SetFromErrno(PyExc_OSError); goto error; + } DWORD total = 0; while (total < size) { diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index 5ab6dcb032550b..3a59e78f171526 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -55,12 +55,26 @@ _query_thread(LPVOID param) IWbemLocator *locator = NULL; IWbemServices *services = NULL; IEnumWbemClassObject* enumerator = NULL; + HRESULT hr = S_OK; BSTR bstrQuery = NULL; struct _query_data *data = (struct _query_data*)param; - HRESULT hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); + // gh-125315: Copy the query string first, so that if the main thread gives + // up on waiting we aren't left with a dangling pointer (and a likely crash) + bstrQuery = SysAllocString(data->query); + if (!bstrQuery) { + hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY); + } + + if (SUCCEEDED(hr)) { + hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); + } + if (FAILED(hr)) { CloseHandle(data->writePipe); + if (bstrQuery) { + SysFreeString(bstrQuery); + } return (DWORD)hr; } @@ -101,12 +115,6 @@ _query_thread(LPVOID param) NULL, EOAC_NONE ); } - if (SUCCEEDED(hr)) { - bstrQuery = SysAllocString(data->query); - if (!bstrQuery) { - hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY); - } - } if (SUCCEEDED(hr)) { hr = services->ExecQuery( bstr_t("WQL"), @@ -279,9 +287,11 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) // a timeout. The initEvent will be set after COM initialization, it will // take a longer time when first initialized. The connectEvent will be set // after connected to WMI. - err = wait_event(data.initEvent, 1000); if (!err) { - err = wait_event(data.connectEvent, 100); + err = wait_event(data.initEvent, 1000); + if (!err) { + err = wait_event(data.connectEvent, 100); + } } while (!err) { @@ -305,28 +315,33 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) CloseHandle(data.readPipe); } - // Allow the thread some time to clean up - switch (WaitForSingleObject(hThread, 100)) { - case WAIT_OBJECT_0: - // Thread ended cleanly - if (!GetExitCodeThread(hThread, (LPDWORD)&err)) { - err = GetLastError(); - } - break; - case WAIT_TIMEOUT: - // Probably stuck - there's not much we can do, unfortunately - if (err == 0 || err == ERROR_BROKEN_PIPE) { - err = WAIT_TIMEOUT; + if (hThread) { + // Allow the thread some time to clean up + int thread_err; + switch (WaitForSingleObject(hThread, 100)) { + case WAIT_OBJECT_0: + // Thread ended cleanly + if (!GetExitCodeThread(hThread, (LPDWORD)&thread_err)) { + thread_err = GetLastError(); + } + break; + case WAIT_TIMEOUT: + // Probably stuck - there's not much we can do, unfortunately + thread_err = WAIT_TIMEOUT; + break; + default: + thread_err = GetLastError(); + break; } - break; - default: + // An error on our side is more likely to be relevant than one from + // the thread, but if we don't have one on our side we'll take theirs. if (err == 0 || err == ERROR_BROKEN_PIPE) { - err = GetLastError(); + err = thread_err; } - break; + + CloseHandle(hThread); } - CloseHandle(hThread); CloseHandle(data.initEvent); CloseHandle(data.connectEvent); hThread = NULL; @@ -355,12 +370,18 @@ static PyMethodDef wmi_functions[] = { { NULL, NULL, 0, NULL } }; +static PyModuleDef_Slot wmi_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef wmi_def = { PyModuleDef_HEAD_INIT, "_wmi", - NULL, // doc - 0, // m_size - wmi_functions + NULL, // doc + 0, // m_size + wmi_functions, // m_methods + wmi_slots, // m_slots }; extern "C" { diff --git a/PC/clinic/_testconsole.c.h b/PC/clinic/_testconsole.c.h index 2c71c11c438b5b..4c11e545499ac5 100644 --- a/PC/clinic/_testconsole.c.h +++ b/PC/clinic/_testconsole.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() - #if defined(MS_WINDOWS) PyDoc_STRVAR(_testconsole_write_input__doc__, @@ -17,58 +11,30 @@ PyDoc_STRVAR(_testconsole_write_input__doc__, "Writes UTF-16-LE encoded bytes to the console as if typed by a user."); #define _TESTCONSOLE_WRITE_INPUT_METHODDEF \ - {"write_input", _PyCFunction_CAST(_testconsole_write_input), METH_FASTCALL|METH_KEYWORDS, _testconsole_write_input__doc__}, + {"write_input", (PyCFunction)(void(*)(void))_testconsole_write_input, METH_VARARGS|METH_KEYWORDS, _testconsole_write_input__doc__}, static PyObject * -_testconsole_write_input_impl(PyObject *module, PyObject *file, - PyBytesObject *s); +_testconsole_write_input_impl(PyObject *module, PyObject *file, Py_buffer *s); static PyObject * -_testconsole_write_input(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testconsole_write_input(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(file), &_Py_ID(s), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"file", "s", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "write_input", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; + static char *_keywords[] = {"file", "s", NULL}; PyObject *file; - PyBytesObject *s; + Py_buffer s = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - file = args[0]; - if (!PyBytes_Check(args[1])) { - _PyArg_BadArgument("write_input", "argument 's'", "bytes", args[1]); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Oy*:write_input", _keywords, + &file, &s)) goto exit; - } - s = (PyBytesObject *)args[1]; - return_value = _testconsole_write_input_impl(module, file, s); + return_value = _testconsole_write_input_impl(module, file, &s); exit: + /* Cleanup for s */ + if (s.obj) { + PyBuffer_Release(&s); + } + return return_value; } @@ -83,48 +49,21 @@ PyDoc_STRVAR(_testconsole_read_output__doc__, "Reads a str from the console as written to stdout."); #define _TESTCONSOLE_READ_OUTPUT_METHODDEF \ - {"read_output", _PyCFunction_CAST(_testconsole_read_output), METH_FASTCALL|METH_KEYWORDS, _testconsole_read_output__doc__}, + {"read_output", (PyCFunction)(void(*)(void))_testconsole_read_output, METH_VARARGS|METH_KEYWORDS, _testconsole_read_output__doc__}, static PyObject * _testconsole_read_output_impl(PyObject *module, PyObject *file); static PyObject * -_testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testconsole_read_output(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(file), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"file", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "read_output", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"file", NULL}; PyObject *file; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:read_output", _keywords, + &file)) goto exit; - } - file = args[0]; return_value = _testconsole_read_output_impl(module, file); exit: @@ -140,4 +79,4 @@ _testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef _TESTCONSOLE_READ_OUTPUT_METHODDEF #define _TESTCONSOLE_READ_OUTPUT_METHODDEF #endif /* !defined(_TESTCONSOLE_READ_OUTPUT_METHODDEF) */ -/*[clinic end generated code: output=08a1c844b3657272 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d60ce07157e3741a input=a9049054013a1b77]*/ diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index e3f7ea43f38211..a77d0855af293f 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -355,10 +355,24 @@ msvcrt_putch(PyObject *module, PyObject *arg) PyObject *return_value = NULL; char char_value; - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "putch(): argument must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + goto exit; + } char_value = PyBytes_AS_STRING(arg)[0]; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "putch(): argument must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + goto exit; + } char_value = PyByteArray_AS_STRING(arg)[0]; } else { @@ -396,7 +410,10 @@ msvcrt_putwch(PyObject *module, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("putwch", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "putwch(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } unicode_char = PyUnicode_READ_CHAR(arg, 0); @@ -430,10 +447,24 @@ msvcrt_ungetch(PyObject *module, PyObject *arg) PyObject *return_value = NULL; char char_value; - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "ungetch(): argument must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + goto exit; + } char_value = PyBytes_AS_STRING(arg)[0]; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "ungetch(): argument must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + goto exit; + } char_value = PyByteArray_AS_STRING(arg)[0]; } else { @@ -471,7 +502,10 @@ msvcrt_ungetwch(PyObject *module, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("ungetwch", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "ungetwch(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } unicode_char = PyUnicode_READ_CHAR(arg, 0); @@ -697,4 +731,4 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_GETERRORMODE_METHODDEF #define MSVCRT_GETERRORMODE_METHODDEF #endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ -/*[clinic end generated code: output=de9687b46212c2ed input=a9049054013a1b77]*/ +/*[clinic end generated code: output=692c6f52bb9193ce input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index 5eff2f5b2310bb..b744f711b0d636 100644 --- a/PC/config.c +++ b/PC/config.c @@ -35,9 +35,9 @@ extern PyObject* PyInit__codecs(void); extern PyObject* PyInit__weakref(void); /* XXX: These two should really be extracted to standalone extensions. */ extern PyObject* PyInit_xxsubtype(void); -extern PyObject* PyInit__xxsubinterpreters(void); -extern PyObject* PyInit__xxinterpchannels(void); -extern PyObject* PyInit__xxinterpqueues(void); +extern PyObject* PyInit__interpreters(void); +extern PyObject* PyInit__interpchannels(void); +extern PyObject* PyInit__interpqueues(void); extern PyObject* PyInit__random(void); extern PyObject* PyInit_itertools(void); extern PyObject* PyInit__collections(void); @@ -139,9 +139,9 @@ struct _inittab _PyImport_Inittab[] = { {"_json", PyInit__json}, {"xxsubtype", PyInit_xxsubtype}, - {"_xxsubinterpreters", PyInit__xxsubinterpreters}, - {"_xxinterpchannels", PyInit__xxinterpchannels}, - {"_xxinterpqueues", PyInit__xxinterpqueues}, + {"_interpreters", PyInit__interpreters}, + {"_interpchannels", PyInit__interpchannels}, + {"_interpqueues", PyInit__interpqueues}, #ifdef _Py_HAVE_ZLIB {"zlib", PyInit_zlib}, #endif diff --git a/PC/errmap.h b/PC/errmap.h index a7489ab75c6561..a064ecb80b1ed9 100644 --- a/PC/errmap.h +++ b/PC/errmap.h @@ -129,6 +129,9 @@ winerror_to_errno(int winerror) case ERROR_NO_UNICODE_TRANSLATION: // 1113 return EILSEQ; + case WAIT_TIMEOUT: // 258 + return ETIMEDOUT; + case ERROR_INVALID_FUNCTION: // 1 case ERROR_INVALID_ACCESS: // 12 case ERROR_INVALID_DATA: // 13 diff --git a/PC/launcher.c b/PC/launcher.c index 8e60ab9303cb95..47fafbc3bf6bad 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1271,6 +1271,7 @@ static PYC_MAGIC magic_values[] = { { 3450, 3499, L"3.11" }, { 3500, 3549, L"3.12" }, { 3550, 3599, L"3.13" }, + { 3600, 3649, L"3.14" }, { 0 } }; diff --git a/PC/launcher2.c b/PC/launcher2.c index e426eccd700044..befcbe30600f2c 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -853,7 +853,7 @@ searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) } wchar_t filename[MAXLEN]; - if (wcsncpy_s(filename, MAXLEN, command, lastDot)) { + if (wcsncpy_s(filename, MAXLEN, command, commandLength)) { return RC_BAD_VIRTUAL_PATH; } @@ -1594,6 +1594,7 @@ _registryReadLegacyEnvironment(const SearchInfo *search, HKEY root, EnvironmentI int count = swprintf_s(realTag, tagLength + 4, L"%s-32", env->tag); if (count == -1) { + debug(L"# Failed to generate 32bit tag\n"); free(realTag); return RC_INTERNAL_ERROR; } @@ -1749,10 +1750,18 @@ appxSearch(const SearchInfo *search, EnvironmentInfo **result, const wchar_t *pa exeName = search->windowed ? L"pythonw.exe" : L"python.exe"; } - if (FAILED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, buffer)) || - !join(buffer, MAXLEN, L"Microsoft\\WindowsApps") || + // Failure to get LocalAppData may just mean we're running as a user who + // doesn't have a profile directory. + // In this case, return "not found", but don't fail. + // Chances are they can't launch Store installs anyway. + if (FAILED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, buffer))) { + return RC_NO_PYTHON; + } + + if (!join(buffer, MAXLEN, L"Microsoft\\WindowsApps") || !join(buffer, MAXLEN, packageFamilyName) || !join(buffer, MAXLEN, exeName)) { + debug(L"# Failed to construct App Execution Alias path\n"); return RC_INTERNAL_ERROR; } @@ -1953,6 +1962,8 @@ struct AppxSearchInfo { struct AppxSearchInfo APPX_SEARCH[] = { // Releases made through the Store + { L"PythonSoftwareFoundation.Python.3.14_qbz5n2kfra8p0", L"3.14", 10 }, + { L"PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0", L"3.13", 10 }, { L"PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0", L"3.12", 10 }, { L"PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0", L"3.11", 10 }, { L"PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0", L"3.10", 10 }, @@ -1960,8 +1971,10 @@ struct AppxSearchInfo APPX_SEARCH[] = { { L"PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0", L"3.8", 10 }, // Side-loadable releases. Note that the publisher ID changes whenever we - // renew our code-signing certificate, so the newer ID has a higher - // priority (lower sortKey) + // change our code signing certificate subject, so the newer IDs have higher + // priorities (lower sortKey) + { L"PythonSoftwareFoundation.Python.3.14_3847v3x7pw1km", L"3.14", 11 }, + { L"PythonSoftwareFoundation.Python.3.13_3847v3x7pw1km", L"3.13", 11 }, { L"PythonSoftwareFoundation.Python.3.12_3847v3x7pw1km", L"3.12", 11 }, { L"PythonSoftwareFoundation.Python.3.11_3847v3x7pw1km", L"3.11", 11 }, { L"PythonSoftwareFoundation.Python.3.11_hd69rhyc2wevp", L"3.11", 12 }, @@ -1982,6 +1995,7 @@ collectEnvironments(const SearchInfo *search, EnvironmentInfo **result) EnvironmentInfo *env = NULL; if (!result) { + debug(L"# collectEnvironments() was passed a NULL result\n"); return RC_INTERNAL_ERROR; } *result = NULL; @@ -2042,7 +2056,9 @@ struct StoreSearchInfo { struct StoreSearchInfo STORE_SEARCH[] = { - { L"3", /* 3.11 */ L"9NRWMJP3717K" }, + { L"3", /* 3.13 */ L"9PNRBTZXMB4Z" }, + { L"3.14", L"9NTRHQCBBPR8" }, + { L"3.13", L"9PNRBTZXMB4Z" }, { L"3.12", L"9NCVDN91XZQP" }, { L"3.11", L"9NRWMJP3717K" }, { L"3.10", L"9PJPW5LDXLZ5" }, @@ -2276,6 +2292,7 @@ int selectEnvironment(const SearchInfo *search, EnvironmentInfo *root, EnvironmentInfo **best) { if (!best) { + debug(L"# selectEnvironment() was passed a NULL best\n"); return RC_INTERNAL_ERROR; } if (!root) { @@ -2693,6 +2710,11 @@ process(int argc, wchar_t ** argv) DWORD len = GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", NULL, 0); if (len > 1) { wchar_t *limitToCompany = allocSearchInfoBuffer(&search, len); + if (!limitToCompany) { + exitCode = RC_NO_MEMORY; + winerror(0, L"Failed to allocate internal buffer"); + goto abort; + } search.limitToCompany = limitToCompany; if (0 == GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", limitToCompany, len)) { exitCode = RC_INTERNAL_ERROR; diff --git a/PC/layout/__main__.py b/PC/layout/__main__.py index f7aa1e6d261f4a..05a059eee7c1d7 100644 --- a/PC/layout/__main__.py +++ b/PC/layout/__main__.py @@ -1,7 +1,7 @@ import sys try: - import layout + import layout # noqa: F401 except ImportError: # Failed to import our package, which likely means we were started directly # Add the additional search path needed to locate our module. diff --git a/PC/layout/main.py b/PC/layout/main.py index d176b272f1c19d..0350ed7af3f9b5 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -121,7 +121,7 @@ def get_tcltk_lib(ns): def get_layout(ns): - def in_build(f, dest="", new_name=None): + def in_build(f, dest="", new_name=None, no_lib=False): n, _, x = f.rpartition(".") n = new_name or n src = ns.build / f @@ -136,7 +136,7 @@ def in_build(f, dest="", new_name=None): pdb = src.with_suffix(".pdb") if pdb.is_file(): yield dest + n + ".pdb", pdb - if ns.include_dev: + if ns.include_dev and not no_lib: lib = src.with_suffix(".lib") if lib.is_file(): yield "libs/" + n + ".lib", lib @@ -202,7 +202,9 @@ def in_build(f, dest="", new_name=None): yield "LICENSE.txt", ns.build / "LICENSE.txt" - for dest, src in rglob(ns.build, "*.pyd"): + dest = "" if ns.flat_dlls else "DLLs/" + + for _, src in rglob(ns.build, "*.pyd"): if ns.include_freethreaded: if not src.match("*.cp*t-win*.pyd"): continue @@ -217,14 +219,14 @@ def in_build(f, dest="", new_name=None): continue if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: continue - yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + yield from in_build(src.name, dest=dest, no_lib=True) - for dest, src in rglob(ns.build, "*.dll"): + for _, src in rglob(ns.build, "*.dll"): if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: continue if src in EXCLUDE_FROM_DLLS: continue - yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + yield from in_build(src.name, dest=dest, no_lib=True) if ns.zip_lib: zip_name = PYTHON_ZIP_NAME @@ -599,6 +601,15 @@ def main(): ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) ns.build = ns.build or Path(sys.executable).parent ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") + if ns.copy and not ns.copy.is_absolute(): + ns.copy = (Path.cwd() / ns.copy).resolve() + if not ns.temp: + # Put temp on a Dev Drive for speed if we're copying to one. + # If not, the regular temp dir will have to do. + if ns.copy and getattr(os.path, "isdevdrive", lambda d: False)(ns.copy): + ns.temp = ns.copy.with_name(ns.copy.name + "_temp") + else: + ns.temp = Path(tempfile.mkdtemp()) if not ns.source.is_absolute(): ns.source = (Path.cwd() / ns.source).resolve() if not ns.build.is_absolute(): @@ -617,21 +628,11 @@ def main(): else: ns.arch = "amd64" - if ns.copy and not ns.copy.is_absolute(): - ns.copy = (Path.cwd() / ns.copy).resolve() if ns.zip and not ns.zip.is_absolute(): ns.zip = (Path.cwd() / ns.zip).resolve() if ns.catalog and not ns.catalog.is_absolute(): ns.catalog = (Path.cwd() / ns.catalog).resolve() - if not ns.temp: - # Put temp on a Dev Drive for speed if we're copying to one. - # If not, the regular temp dir will have to do. - if ns.copy and getattr(os.path, "isdevdrive", lambda d: False)(ns.copy): - ns.temp = ns.copy.with_name(ns.copy.name + "_temp") - else: - ns.temp = Path(tempfile.mkdtemp()) - configure_logger(ns) log_info( diff --git a/PC/layout/support/appxmanifest.py b/PC/layout/support/appxmanifest.py index 1fb03380278f43..53977beb8af834 100644 --- a/PC/layout/support/appxmanifest.py +++ b/PC/layout/support/appxmanifest.py @@ -209,7 +209,7 @@ class PACKAGE_ID(ctypes.Structure): result = ctypes.create_unicode_buffer(256) result_len = ctypes.c_uint32(256) r = ctypes.windll.kernel32.PackageFamilyNameFromId( - pid, ctypes.byref(result_len), result + ctypes.byref(pid), ctypes.byref(result_len), result ) if r: raise OSError(r, "failed to get package family name") diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 5ff703217b421f..b170e06b47dd59 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -656,6 +656,7 @@ exec_module(PyObject* m) static PyModuleDef_Slot msvcrt_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index 8bbf877a5bb5ed..010f5fe5646630 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -95,7 +95,12 @@ WIN32 is still required for the locale module. #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ /* Define to 1 if you want to disable the GIL */ -#undef Py_GIL_DISABLED +/* Uncomment the definition for free-threaded builds, or define it manually + * when compiling extension modules. Note that we test with #ifdef, so + * defining as 0 will still disable the GIL. */ +#ifndef Py_GIL_DISABLED +/* #define Py_GIL_DISABLED 1 */ +#endif /* Compiler specific defines */ @@ -164,9 +169,9 @@ WIN32 is still required for the locale module. #endif /* MS_WIN64 */ /* set the version macros for the windows headers */ -/* Python 3.9+ requires Windows 8 or greater */ -#define Py_WINVER 0x0602 /* _WIN32_WINNT_WIN8 */ -#define Py_NTDDI NTDDI_WIN8 +/* Python 3.13+ requires Windows 10 or greater */ +#define Py_WINVER 0x0A00 /* _WIN32_WINNT_WIN10 */ +#define Py_NTDDI NTDDI_WIN10 /* We only set these values when building Python - we don't want to force these values on extensions, as that will affect the prototypes and @@ -311,19 +316,19 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ file in their Makefile */ # if defined(Py_GIL_DISABLED) # if defined(_DEBUG) -# pragma comment(lib,"python313t_d.lib") +# pragma comment(lib,"python314t_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3t.lib") # else -# pragma comment(lib,"python313t.lib") +# pragma comment(lib,"python314t.lib") # endif /* _DEBUG */ # else /* Py_GIL_DISABLED */ # if defined(_DEBUG) -# pragma comment(lib,"python313_d.lib") +# pragma comment(lib,"python314_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3.lib") # else -# pragma comment(lib,"python313.lib") +# pragma comment(lib,"python314.lib") # endif /* _DEBUG */ # endif /* Py_GIL_DISABLED */ # endif /* _MSC_VER */ @@ -526,9 +531,6 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if you want to compile in mimalloc memory allocator. */ #define WITH_MIMALLOC 1 -/* Define if you want to compile in object freelists optimization */ -#define WITH_FREELISTS 1 - /* Define if you have clock. */ /* #define HAVE_CLOCK */ diff --git a/PC/python3dll.c b/PC/python3dll.c index aa6bfe2c4022db..8657ddb9fa5155 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -55,6 +55,8 @@ EXPORT_FUNC(Py_GenericAlias) EXPORT_FUNC(Py_GetArgcArgv) EXPORT_FUNC(Py_GetBuildInfo) EXPORT_FUNC(Py_GetCompiler) +EXPORT_FUNC(Py_GetConstant) +EXPORT_FUNC(Py_GetConstantBorrowed) EXPORT_FUNC(Py_GetCopyright) EXPORT_FUNC(Py_GetExecPrefix) EXPORT_FUNC(Py_GetPath) @@ -79,12 +81,14 @@ EXPORT_FUNC(Py_Main) EXPORT_FUNC(Py_MakePendingCalls) EXPORT_FUNC(Py_NewInterpreter) EXPORT_FUNC(Py_NewRef) +EXPORT_FUNC(Py_REFCNT) EXPORT_FUNC(Py_ReprEnter) EXPORT_FUNC(Py_ReprLeave) EXPORT_FUNC(Py_SetPath) EXPORT_FUNC(Py_SetProgramName) EXPORT_FUNC(Py_SetPythonHome) EXPORT_FUNC(Py_SetRecursionLimit) +EXPORT_FUNC(Py_TYPE) EXPORT_FUNC(Py_VaBuildValue) EXPORT_FUNC(Py_XNewRef) EXPORT_FUNC(PyAIter_Check) @@ -251,6 +255,9 @@ EXPORT_FUNC(PyEval_EvalFrame) EXPORT_FUNC(PyEval_EvalFrameEx) EXPORT_FUNC(PyEval_GetBuiltins) EXPORT_FUNC(PyEval_GetFrame) +EXPORT_FUNC(PyEval_GetFrameBuiltins) +EXPORT_FUNC(PyEval_GetFrameGlobals) +EXPORT_FUNC(PyEval_GetFrameLocals) EXPORT_FUNC(PyEval_GetFuncDesc) EXPORT_FUNC(PyEval_GetFuncName) EXPORT_FUNC(PyEval_GetGlobals) @@ -320,6 +327,7 @@ EXPORT_FUNC(PyInterpreterState_GetID) EXPORT_FUNC(PyInterpreterState_New) EXPORT_FUNC(PyIter_Check) EXPORT_FUNC(PyIter_Next) +EXPORT_FUNC(PyIter_NextItem) EXPORT_FUNC(PyIter_Send) EXPORT_FUNC(PyList_Append) EXPORT_FUNC(PyList_AsTuple) @@ -335,23 +343,31 @@ EXPORT_FUNC(PyList_Size) EXPORT_FUNC(PyList_Sort) EXPORT_FUNC(PyLong_AsDouble) EXPORT_FUNC(PyLong_AsInt) +EXPORT_FUNC(PyLong_AsInt32) +EXPORT_FUNC(PyLong_AsInt64) EXPORT_FUNC(PyLong_AsLong) EXPORT_FUNC(PyLong_AsLongAndOverflow) EXPORT_FUNC(PyLong_AsLongLong) EXPORT_FUNC(PyLong_AsLongLongAndOverflow) EXPORT_FUNC(PyLong_AsSize_t) EXPORT_FUNC(PyLong_AsSsize_t) +EXPORT_FUNC(PyLong_AsUInt32) +EXPORT_FUNC(PyLong_AsUInt64) EXPORT_FUNC(PyLong_AsUnsignedLong) EXPORT_FUNC(PyLong_AsUnsignedLongLong) EXPORT_FUNC(PyLong_AsUnsignedLongLongMask) EXPORT_FUNC(PyLong_AsUnsignedLongMask) EXPORT_FUNC(PyLong_AsVoidPtr) EXPORT_FUNC(PyLong_FromDouble) +EXPORT_FUNC(PyLong_FromInt32) +EXPORT_FUNC(PyLong_FromInt64) EXPORT_FUNC(PyLong_FromLong) EXPORT_FUNC(PyLong_FromLongLong) EXPORT_FUNC(PyLong_FromSize_t) EXPORT_FUNC(PyLong_FromSsize_t) EXPORT_FUNC(PyLong_FromString) +EXPORT_FUNC(PyLong_FromUInt32) +EXPORT_FUNC(PyLong_FromUInt64) EXPORT_FUNC(PyLong_FromUnsignedLong) EXPORT_FUNC(PyLong_FromUnsignedLongLong) EXPORT_FUNC(PyLong_FromVoidPtr) @@ -630,14 +646,19 @@ EXPORT_FUNC(PyTuple_Pack) EXPORT_FUNC(PyTuple_SetItem) EXPORT_FUNC(PyTuple_Size) EXPORT_FUNC(PyType_ClearCache) +EXPORT_FUNC(PyType_Freeze) EXPORT_FUNC(PyType_FromMetaclass) EXPORT_FUNC(PyType_FromModuleAndSpec) EXPORT_FUNC(PyType_FromSpec) EXPORT_FUNC(PyType_FromSpecWithBases) EXPORT_FUNC(PyType_GenericAlloc) EXPORT_FUNC(PyType_GenericNew) +EXPORT_FUNC(PyType_GetBaseByToken) EXPORT_FUNC(PyType_GetFlags) +EXPORT_FUNC(PyType_GetFullyQualifiedName) EXPORT_FUNC(PyType_GetModule) +EXPORT_FUNC(PyType_GetModuleByDef) +EXPORT_FUNC(PyType_GetModuleName) EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) EXPORT_FUNC(PyType_GetQualName) @@ -697,6 +718,7 @@ EXPORT_FUNC(PyUnicode_DecodeUTF8Stateful) EXPORT_FUNC(PyUnicode_EncodeCodePage) EXPORT_FUNC(PyUnicode_EncodeFSDefault) EXPORT_FUNC(PyUnicode_EncodeLocale) +EXPORT_FUNC(PyUnicode_Equal) EXPORT_FUNC(PyUnicode_EqualToUTF8) EXPORT_FUNC(PyUnicode_EqualToUTF8AndSize) EXPORT_FUNC(PyUnicode_Find) @@ -831,7 +853,6 @@ EXPORT_DATA(PyExc_FutureWarning) EXPORT_DATA(PyExc_GeneratorExit) EXPORT_DATA(PyExc_ImportError) EXPORT_DATA(PyExc_ImportWarning) -EXPORT_DATA(PyExc_IncompleteInputError) EXPORT_DATA(PyExc_IndentationError) EXPORT_DATA(PyExc_IndexError) EXPORT_DATA(PyExc_InterruptedError) diff --git a/PC/venvlauncher.c b/PC/venvlauncher.c index fe97d32e93b5f6..b1c8d0763d8c76 100644 --- a/PC/venvlauncher.c +++ b/PC/venvlauncher.c @@ -484,8 +484,8 @@ process(int argc, wchar_t ** argv) // We do not update argv[0] to point at the target runtime, and so we do not // pass through our original argv[0] in an environment variable. - //exitCode = smuggle_path(); - //if (exitCode) return exitCode; + exitCode = smuggle_path(); + if (exitCode) return exitCode; exitCode = launch(home_path, GetCommandLineW()); return exitCode; diff --git a/PC/winreg.c b/PC/winreg.c index 77b80217ac0ab1..efdf8addc06186 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -200,7 +200,7 @@ PyHKEY_hashFunc(PyObject *ob) /* Just use the address. XXX - should we use the handle value? */ - return _Py_HashPointer(ob); + return PyObject_GenericHash(ob); } @@ -2179,6 +2179,7 @@ exec_module(PyObject *m) static PyModuleDef_Slot winreg_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/winsound.c b/PC/winsound.c index 7e4ebd90f50c2e..094c77ae34d678 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -35,11 +35,10 @@ winsound.PlaySound(None, 0) */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -247,6 +246,7 @@ exec_module(PyObject *module) static PyModuleDef_Slot sound_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PCbuild/_ctypes_test.vcxproj b/PCbuild/_ctypes_test.vcxproj index 97354739c09834..f15b80852e362a 100644 --- a/PCbuild/_ctypes_test.vcxproj +++ b/PCbuild/_ctypes_test.vcxproj @@ -87,6 +87,7 @@ + @@ -94,6 +95,7 @@ + @@ -109,4 +111,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_ctypes_test.vcxproj.filters b/PCbuild/_ctypes_test.vcxproj.filters index 5174196c52e4d0..618cfb32115e99 100644 --- a/PCbuild/_ctypes_test.vcxproj.filters +++ b/PCbuild/_ctypes_test.vcxproj.filters @@ -15,6 +15,9 @@ Header Files + + Header Files + @@ -26,4 +29,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj index 490d7df87eb1c6..ee7421484b5312 100644 --- a/PCbuild/_decimal.vcxproj +++ b/PCbuild/_decimal.vcxproj @@ -93,51 +93,55 @@ - _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + BUILD_LIBMPDEC;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) CONFIG_32;PPRO;MASM;%(PreprocessorDefinitions) CONFIG_32;ANSI;%(PreprocessorDefinitions) CONFIG_64;ANSI;%(PreprocessorDefinitions) CONFIG_64;MASM;%(PreprocessorDefinitions) - ..\Modules\_decimal;..\Modules\_decimal\libmpdec;%(AdditionalIncludeDirectories) + ..\Modules\_decimal;..\Modules\_decimal\windows;$(mpdecimalDir)\libmpdec;%(AdditionalIncludeDirectories) - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + true true true diff --git a/PCbuild/_decimal.vcxproj.filters b/PCbuild/_decimal.vcxproj.filters index 0cbd3d0736c241..e4bdb64ec1fb9f 100644 --- a/PCbuild/_decimal.vcxproj.filters +++ b/PCbuild/_decimal.vcxproj.filters @@ -21,49 +21,55 @@ Header Files - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + + Header Files\libmpdec + + + Header Files\libmpdec + + Header Files\libmpdec @@ -71,46 +77,46 @@ Source Files - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec @@ -120,8 +126,8 @@ - + Source Files\libmpdec - \ No newline at end of file + diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 49f529ebbc2f9b..a3c2d32c454e04 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -142,7 +142,6 @@ - @@ -195,6 +194,7 @@ + @@ -223,6 +223,8 @@ + + @@ -235,9 +237,11 @@ + + @@ -253,6 +257,7 @@ + @@ -263,6 +268,7 @@ + @@ -474,25 +480,6 @@
    - - - - -$(IntDir)\deepfreeze_mappings.txt - - - - - - - - diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 5b1bd7552b4cd9..91b1d75fb8df5e 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -94,6 +94,12 @@ Source Files + + Source Files + + + Source Files + Source Files @@ -229,6 +235,10 @@ Source Files + + + Source Files + Source Files @@ -241,9 +251,6 @@ Source Files - - Source Files - Source Files @@ -310,6 +317,9 @@ Source Files + + Python + Source Files @@ -376,6 +386,9 @@ Source Files + + Source Files + Source Files @@ -454,6 +467,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 6911aacab29b97..c41235eac356af 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -96,11 +96,8 @@ - - - @@ -120,13 +117,16 @@ - - + + + + + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 6059959bb9a040..0a00df655deefc 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -30,9 +30,6 @@ Source Files - - Source Files - Source Files @@ -90,24 +87,33 @@ Source Files - - Source Files - Source Files Source Files - + Source Files - + + Source Files + + Source Files Source Files + + Source Files + + + Source Files + + + Source Files + diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index a825cac9138674..87db569423de2a 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -108,6 +108,12 @@ false + + + _Py_JIT;%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) + + diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters index abfeeb39630daf..27429ea5833077 100644 --- a/PCbuild/_testinternalcapi.vcxproj.filters +++ b/PCbuild/_testinternalcapi.vcxproj.filters @@ -27,4 +27,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj new file mode 100644 index 00000000000000..846e027e10c7fa --- /dev/null +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -0,0 +1,132 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933} + _testlimitedcapi + Win32Proj + false + + + + + DynamicLibrary + NotSet + + + + $(PyStdlibPydExt) + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + {885d4898-d08d-4091-9c40-c700cfe3fc5a} + false + + + + + + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters new file mode 100644 index 00000000000000..57be2e2fc5b950 --- /dev/null +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -0,0 +1,38 @@ + + + + + {a76a90d8-8e8b-4c36-8f58-8bd46abe9f5e} + + + {071b2ff4-e5a1-4e79-b0c5-cf46b0094a80} + + + + + + + + + + + + + + + + + + + + + + + + + + + Resource Files + + + diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 83b50db4467033..abe649553756a7 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -8,7 +8,7 @@ echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). echo. echo.After the flags recognized by this script, up to 9 arguments to be passed echo.directly to MSBuild may be passed. If the argument contains an '=', the -echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`). +echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v141"`). echo.Alternatively you can put extra flags for MSBuild in a file named echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file echo.will be picked automatically by MSBuild. Flags put in this file does not @@ -36,7 +36,9 @@ echo. overrides -c and -d echo. --disable-gil Enable experimental support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. -echo. --experimental-jit Enable the experimental just-in-time compiler. +echo. --experimental-jit Enable the experimental just-in-time compiler. +echo. --experimental-jit-off Ditto but off by default (PYTHON_JIT=1 enables). +echo. --experimental-jit-interpreter Enable the experimental Tier 2 interpreter. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -66,6 +68,7 @@ set verbose=/nologo /v:m /clp:summary set kill= set do_pgo= set pgo_job=-m test --pgo +set UseTIER2= :CheckOpts if "%~1"=="-h" goto Usage @@ -86,7 +89,10 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts -if "%~1"=="--experimental-jit" (set UseJIT=true) & shift & goto CheckOpts +if "%~1"=="--experimental-jit" (set UseJIT=true) & (set UseTIER2=1) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-off" (set UseJIT=true) & (set UseTIER2=3) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter" (set UseTIER2=4) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -179,6 +185,7 @@ echo on /p:DisableGil=%UseDisableGil%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^ /p:UseJIT=%UseJIT%^ + /p:UseTIER2=%UseTIER2%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/env.bat b/PCbuild/env.bat index 2820e304582cff..cf4638b7aa63a7 100644 --- a/PCbuild/env.bat +++ b/PCbuild/env.bat @@ -4,8 +4,8 @@ rem command window. However, most builds of Python will ignore the version rem of the tools on PATH and use PlatformToolset instead. Ideally, both sets of rem tools should be the same version to avoid potential conflicts. rem -rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v100" (or -rem 'v110', 'v120' or 'v140') to the build script. +rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v141" (or +rem 'v142', 'v143') to the build script. echo Build environments: x86, amd64, x86_amd64 echo. @@ -20,8 +20,7 @@ call "%VSTOOLS%" %_ARGS% exit /B 0 :skip_vswhere -if not defined VSTOOLS set VSTOOLS=%VS140COMNTOOLS% -if not defined VSTOOLS set VSTOOLS=%VS120COMNTOOLS% -if not defined VSTOOLS set VSTOOLS=%VS110COMNTOOLS% -if not defined VSTOOLS set VSTOOLS=%VS100COMNTOOLS% +if not defined VSTOOLS set VSTOOLS=%VS143COMNTOOLS% +if not defined VSTOOLS set VSTOOLS=%VS142COMNTOOLS% +if not defined VSTOOLS set VSTOOLS=%VS141COMNTOOLS% call "%VSTOOLS%..\..\VC\vcvarsall.bat" %_ARGS% diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat index ce7e71efa31f6c..82dd34beede6ee 100644 --- a/PCbuild/find_msbuild.bat +++ b/PCbuild/find_msbuild.bat @@ -39,16 +39,6 @@ @if defined MSBUILD @if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio installation) & goto :found :skip_vswhere -@rem VS 2015 and earlier register MSBuild separately, so we can find it. -@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul -@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( - @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") -) -@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found - - -@exit /b 1 - :found @pushd %MSBUILD% >nul 2>nul @if not ERRORLEVEL 1 @( diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index d3f62c93869003..d65d080ca71a90 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -29,6 +29,9 @@ :begin_search @set PYTHON= +@rem If PYTHON_FOR_BUILD is set, use that +@if NOT "%PYTHON_FOR_BUILD%"=="" @(set PYTHON="%PYTHON_FOR_BUILD%") && (set _Py_Python_Source=found as PYTHON_FOR_BUILD) && goto :found + @rem If there is an active virtual env, use that one @if NOT "%VIRTUAL_ENV%"=="" (set PYTHON="%VIRTUAL_ENV%\Scripts\python.exe") & (set _Py_Python_Source=found in virtual env) & goto :found @@ -36,13 +39,15 @@ @if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%_Py_D%\..\externals) @rem If we have Python in externals, use that one -@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" @rem If HOST_PYTHON is recent enough, use that -@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 9)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.11 3.10 3.9) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@rem It is fine to add new versions to this list when they have released, +@rem but we do not use prerelease builds here. +@for %%p in (3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 60ce12b725e233..dfacd1d1e788d4 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -45,7 +45,7 @@ if "%ORG%"=="" (set ORG=python) call "%PCBUILD%\find_python.bat" "%PYTHON%" if NOT DEFINED PYTHON ( - where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1 + where /Q git || echo Python 3.10 or later could not be found or installed, and git.exe is not on your PATH && exit /B 1 ) echo.Fetching external libraries... @@ -53,10 +53,11 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 -set libraries=%libraries% sqlite-3.45.1.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.15 +set libraries=%libraries% mpdecimal-4.0.0 +set libraries=%libraries% sqlite-3.45.3.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 set libraries=%libraries% xz-5.2.5 set libraries=%libraries% zlib-1.3.1 @@ -76,8 +77,8 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.13 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.1 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.15 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index b7b78be768d7ec..a2a637a3044373 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -77,7 +77,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index a0b5fbd1304b41..d10e1c46a91480 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30028.174 +# Visual Studio Version 17 +VisualStudioVersion = 17.8.34525.116 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" ProjectSection(SolutionItems) = preProject @@ -159,6 +159,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_wmi", "_wmi.vcxproj", "{54 EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testclinic_limited", "_testclinic_limited.vcxproj", "{01FDF29A-40A1-46DF-84F5-85EBBD2A2410}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testlimitedcapi", "_testlimitedcapi.vcxproj", "{7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM = Debug|ARM @@ -1648,6 +1650,38 @@ Global {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|Win32.Build.0 = Release|Win32 {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|x64.ActiveCfg = Release|x64 {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|x64.Build.0 = Release|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM.ActiveCfg = Debug|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM.Build.0 = Debug|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM64.Build.0 = Debug|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|Win32.ActiveCfg = Debug|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|Win32.Build.0 = Debug|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|x64.ActiveCfg = Debug|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|x64.Build.0 = Debug|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM.ActiveCfg = Release|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM.Build.0 = Release|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM64.ActiveCfg = Release|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM64.Build.0 = Release|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|Win32.ActiveCfg = Release|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|Win32.Build.0 = Release|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|x64.ActiveCfg = Release|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 9c85e5efa4af4a..c65341179376ea 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -217,11 +217,6 @@ public override bool Execute() { - - - $(VCInstallDir)\redist\ - - <_RedistFiles Include="$(VCInstallDir)\Redist\MSVC\*\*.*" /> diff --git a/PCbuild/python.props b/PCbuild/python.props index e21f1f60464bc8..6e90178f4ea8ab 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -6,7 +6,7 @@ Release - - + @@ -674,7 +703,7 @@ $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) - $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + $(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1')) - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index ceaa21217267cf..740790cc5e1119 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -48,6 +48,9 @@ Include + + Include + Include @@ -72,6 +75,9 @@ Include + + Include + Include @@ -114,6 +120,9 @@ Include + + Include + Include @@ -207,6 +216,9 @@ Include + + Include + Include @@ -330,9 +342,6 @@ Include - - Include - Modules @@ -348,6 +357,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -375,6 +387,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -396,6 +411,9 @@ Include\cpython + + Include + Include @@ -408,9 +426,6 @@ Include - - Include - Include\cpython @@ -423,6 +438,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -492,9 +510,6 @@ Include - - Include\cpython - Include\cpython @@ -504,9 +519,6 @@ Include\cpython - - Include\cpython - Include\cpython @@ -543,6 +555,9 @@ Include\internal + + Include\internal + Include\internal @@ -561,6 +576,9 @@ Include\internal + + Include\internal + Include\internal @@ -591,6 +609,9 @@ Include\internal + + Include\internal + Include\internal @@ -627,6 +648,9 @@ Include\internal + + Include\internal + Include\internal @@ -666,6 +690,9 @@ Include\internal + + Include\internal + Include\internal @@ -702,6 +729,9 @@ Include\internal + + Include\internal + Include\internal @@ -723,6 +753,9 @@ Include\internal + + Include\internal + Include\internal @@ -750,6 +783,9 @@ Include\internal + + Include\internal + Include\internal @@ -783,6 +819,9 @@ Include\internal + + Include\internal + Include\internal @@ -813,6 +852,9 @@ Include\internal + + Include\internal + Include\internal\mimalloc @@ -875,15 +917,6 @@ Modules - - Modules - - - Modules - - - Modules - Modules @@ -1265,6 +1298,9 @@ Python + + Python + Python @@ -1340,9 +1376,15 @@ Python + + Python + Source Files + + Source Files + Source Files @@ -1376,6 +1418,9 @@ Python + + Python + Python @@ -1385,6 +1430,9 @@ Python + + Python + Python @@ -1418,6 +1466,9 @@ Python + + Python + Python @@ -1448,6 +1499,9 @@ Python + + Python + Python @@ -1463,9 +1517,6 @@ Objects - - Objects - Modules @@ -1532,13 +1583,13 @@ Parser - + Modules - + Modules - + Modules diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 387565515fa0b0..693fcee5f90ce2 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -3,7 +3,7 @@ Quick Start Guide 1. Install Microsoft Visual Studio 2017 or later with Python workload and Python native development component. -1a. Optionally install Python 3.6 or later. If not installed, +1a. Optionally install Python 3.10 or later. If not installed, get_externals.bat (via build.bat) will download and use Python via NuGet. 2. Run "build.bat" to build Python in 32-bit Release configuration. @@ -57,7 +57,7 @@ Building Python using the build.bat script In this directory you can find build.bat, a script designed to make building Python on Windows simpler. This script will use the env.bat -script to detect either Visual Studio 2017 or 2015, either of +script to detect either Visual Studio 2017 or later, either of which may be used to build Python. Currently Visual Studio 2017 is officially supported. @@ -131,29 +131,31 @@ xxlimited_35 The following sub-projects are for individual modules of the standard library which are implemented in C; each one builds a DLL (renamed to .pyd) of the same name as the project: -_asyncio -_ctypes -_ctypes_test -_zoneinfo -_decimal -_elementtree -_hashlib -_multiprocessing -_overlapped -_socket -_testbuffer -_testcapi -_testclinic -_testclinic_limited -_testconsole -_testimportmultiple -_testmultiphase -_testsinglephase -_tkinter -pyexpat -select -unicodedata -winsound + * _asyncio + * _ctypes + * _ctypes_test + * _zoneinfo + * _decimal + * _elementtree + * _hashlib + * _multiprocessing + * _overlapped + * _socket + * _testbuffer + * _testcapi + * _testlimitedcapi + * _testinternalcapi + * _testclinic + * _testclinic_limited + * _testconsole + * _testimportmultiple + * _testmultiphase + * _testsinglephase + * _tkinter + * pyexpat + * select + * unicodedata + * winsound The following Python-controlled sub-projects wrap external projects. Note that these external libraries are not necessary for a working @@ -170,7 +172,7 @@ _lzma Homepage: https://tukaani.org/xz/ _ssl - Python wrapper for version 3.0 of the OpenSSL secure sockets + Python wrapper for version 3.0.15 of the OpenSSL secure sockets library, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. @@ -189,11 +191,11 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.45.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.45.3, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter - Wraps version 8.6.6 of the Tk windowing system, which is downloaded + Wraps version 8.6.15 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. @@ -226,12 +228,18 @@ directory. This script extracts all the external sub-projects from and https://github.com/python/cpython-bin-deps via a Python script called "get_external.py", located in this directory. -If Python 3.6 or later is not available via the "py.exe" launcher, the -path or command to use for Python can be provided in the PYTHON_FOR_BUILD -environment variable, or get_externals.bat will download the latest -version of NuGet and use it to download the latest "pythonx86" package -for use with get_external.py. Everything downloaded by these scripts is -stored in ..\externals (relative to this directory). +Everything downloaded by these scripts is stored in ..\externals +(relative to this directory), or the path specified by the EXTERNALS_DIR +environment variable. + +The path or command to use for Python can be provided with the +PYTHON_FOR_BUILD environment variable. If this is not set, an active +virtual environment will be used. If none is active, and HOST_PYTHON is +set to a recent enough version or "py.exe" is able to find a recent +enough version, those will be used. If all else fails, a copy of Python +will be downloaded from NuGet and extracted to the externals directory. +This will then be used for later builds (see PCbuild/find_python.bat +for the full logic). It is also possible to download sources from each project's homepage, though you may have to change folder names or pass the names to MSBuild @@ -303,6 +311,8 @@ _testclinic_limited extension, the file Modules/_testclinic_limited.c: * In PCbuild/, copy _testclinic.vcxproj to _testclinic_limited.vcxproj, replace RootNamespace value with `_testclinic_limited`, replace `_asyncio.c` with `_testclinic_limited.c`. +* In PCbuild/, copy _testclinic.vcxproj.filters to + _testclinic_limited.vcxproj.filters, edit the list of files in the new file. * Open Visual Studio, open PCbuild\pcbuild.sln solution, add the PCbuild\_testclinic_limited.vcxproj project to the solution ("add existing project). diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index a90620d6ca8b7d..416241d9d0df10 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -31,11 +31,17 @@ <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> + <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> + <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> + <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> + <_SbomOutputs Include="$(PySourcePath)Misc\externals.spdx.json;$(PySourcePath)Misc\sbom.spdx.json"> + json + - @@ -79,7 +85,31 @@ - + + + + + + + + + + + + + + - + + + + + @@ -124,9 +161,7 @@ Condition="($(Platform) == 'Win32' or $(Platform) == 'x64') and $(Configuration) != 'PGInstrument' and $(Configuration) != 'PGUpdate'"> - diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 293f99ae135faa..c436215780fda0 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -38,18 +38,18 @@ set regrtestargs=--fast-ci set exe= :CheckOpts -if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts -if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts -if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts +if "%~1"=="-O" (set dashO=-O) & shift & goto CheckOpts +if "%~1"=="-q" (set qmode=yes) & shift & goto CheckOpts +if "%~1"=="-d" (set suffix=_d) & shift & goto CheckOpts rem HACK: Need some way to infer the version number in this script -if "%1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts -if "%1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts -if "%1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts -if "%1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts -if "%1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts -if "%1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts -if "%1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts -if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts +if "%~1"=="--disable-gil" (set pyname=python3.14t) & shift & goto CheckOpts +if "%~1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts +if "%~1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%~1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%~1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts +if "%~1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts +if "%~1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts +if NOT "%~1"=="" (set regrtestargs=%regrtestargs% %~1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%amd64 set exe=%prefix%\%pyname%%suffix%.exe diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 8ddf01d5dd1dca..b4cb401609d409 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 8.6.13.1 + 8.6.15.0 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) @@ -37,9 +37,6 @@ Debug $(BuildDirTop)_$(TclMachine) $(BuildDirTop)_VC13 - $(BuildDirTop)_VC12 - $(BuildDirTop)_VC11 - $(BuildDirTop)_VC10 ', - '', - deepfreezemappings, - PCBUILD_PROJECT, - ) - outfile.writelines(lines) - with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - [deepfreezemappingsfile, ], - PCBUILD_PROJECT, - ) - outfile.writelines(lines) - with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - deepfreezerules, - PCBUILD_PROJECT, - ) - outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): lines = infile.readlines() @@ -729,17 +665,6 @@ def regen_pcbuild(modules): PCBUILD_FILTERS, ) outfile.writelines(lines) - print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}') - with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - corelines, - PCBUILD_FILTERS, - ) - outfile.writelines(lines) ####################################### diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index 33d1b323fc1753..b5b6de0e7dc2dc 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -370,9 +370,14 @@ def generate_static_strings_initializer(identifiers, strings): # This use of _Py_ID() is ignored by iter_global_strings() # since iter_files() ignores .h files. printer.write(f'string = &_Py_ID({i});') + printer.write(f'_PyUnicode_InternStatic(interp, &string);') printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') - printer.write(f'_PyUnicode_InternInPlace(interp, &string);') - # XXX What about "strings"? + printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);') + for value, name in sorted(strings.items()): + printer.write(f'string = &_Py_STR({name});') + printer.write(f'_PyUnicode_InternStatic(interp, &string);') + printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') + printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);') printer.write(END) printer.write(after) @@ -414,15 +419,31 @@ def generate_global_object_finalizers(generated_immortal_objects): def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': identifiers = set(IDENTIFIERS) strings = {} + # Note that we store strings as they appear in C source, so the checks here + # can be defeated, e.g.: + # - "a" and "\0x61" won't be reported as duplicate. + # - "\n" appears as 2 characters. + # Probably not worth adding a C string parser. for name, string, *_ in iter_global_strings(): if string is None: if name not in IGNORED: identifiers.add(name) else: + if len(string) == 1 and ord(string) < 256: + # Give a nice message for common mistakes. + # To cover tricky cases (like "\n") we also generate C asserts. + raise ValueError( + 'do not use &_Py_ID or &_Py_STR for one-character latin-1 ' + + f'strings, use _Py_LATIN1_CHR instead: {string!r}') if string not in strings: strings[string] = name elif name != strings[string]: raise ValueError(f'string mismatch for {name!r} ({string!r} != {strings[name]!r}') + overlap = identifiers & set(strings.keys()) + if overlap: + raise ValueError( + 'do not use both _Py_ID and _Py_DECLARE_STR for the same string: ' + + repr(overlap)) return identifiers, strings diff --git a/Tools/build/generate_re_casefix.py b/Tools/build/generate_re_casefix.py index b57ac07426c27c..6cebfbd025c58c 100755 --- a/Tools/build/generate_re_casefix.py +++ b/Tools/build/generate_re_casefix.py @@ -23,9 +23,9 @@ def update_file(file, content): # Maps the code of lowercased character to codes of different lowercased # characters which have the same uppercase. -_EXTRA_CASES = { +_EXTRA_CASES = {{ %s -} +}} """ def uname(i): diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 442487f2d2546b..020f874cffeaef 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -4,14 +4,13 @@ import hashlib import json import glob -import pathlib +from pathlib import Path, PurePosixPath, PureWindowsPath import subprocess import sys +import urllib.request import typing -import zipfile -from urllib.request import urlopen -CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent +CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent # Before adding a new entry to this list, double check that # the license expression is a valid SPDX license expression: @@ -53,13 +52,14 @@ class PackageFiles(typing.NamedTuple): # values to 'exclude' if we create new files within tracked # directories that aren't sourced from third-party packages. PACKAGE_TO_FILES = { - # NOTE: pip's entry in this structure is automatically generated in - # the 'discover_pip_sbom_package()' function below. "mpdecimal": PackageFiles( include=["Modules/_decimal/libmpdec/**"] ), "expat": PackageFiles( - include=["Modules/expat/**"] + include=["Modules/expat/**"], + exclude=[ + "Modules/expat/expat_config.h", + ] ), "macholib": PackageFiles( include=["Lib/ctypes/macholib/**"], @@ -69,9 +69,6 @@ class PackageFiles(typing.NamedTuple): "Lib/ctypes/macholib/fetch_macholib.bat", ], ), - "libb2": PackageFiles( - include=["Modules/_blake2/impl/**"] - ), "hacl-star": PackageFiles( include=["Modules/_hacl/**"], exclude=[ @@ -96,6 +93,19 @@ def error_if(value: bool, error_message: str) -> None: sys.exit(1) +def is_root_directory_git_index() -> bool: + """Checks if the root directory is a git index""" + try: + subprocess.check_call( + ["git", "-C", str(CPYTHON_ROOT_DIR), "rev-parse"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + return False + return True + + def filter_gitignored_paths(paths: list[str]) -> list[str]: """ Filter out paths excluded by the gitignore file. @@ -108,6 +118,10 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: '.gitignore:9:*.a Tools/lib.a' """ + # No paths means no filtering to be done. + if not paths: + return [] + # Filter out files in gitignore. # Non-matching files show up as '::' git_check_ignore_proc = subprocess.run( @@ -119,284 +133,53 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: # 1 means matches, 0 means no matches. assert git_check_ignore_proc.returncode in (0, 1) + # Paths may or may not be quoted, Windows quotes paths. + git_check_ignore_re = re.compile(r"^::\s+(\"([^\"]+)\"|(.+))\Z") + # Return the list of paths sorted git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines() - return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) - - -def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | None = None) -> tuple[str, str] | None: - """ - Fetches the SHA256 checksum and download location from PyPI. - If we're given a filename then we match with that, otherwise we use wheels. - """ - # Get pip's download location from PyPI. Check that the checksum is correct too. - try: - raw_text = urlopen(f"https://pypi.org/pypi/{project}/{version}/json").read() - release_metadata = json.loads(raw_text) - url: dict[str, typing.Any] - - # Look for a matching artifact filename and then check - # its remote checksum to the local one. - for url in release_metadata["urls"]: - # pip can only use Python-only dependencies, so there's - # no risk of picking the 'incorrect' wheel here. - if ( - (filename is None and url["packagetype"] == "bdist_wheel") - or (filename is not None and url["filename"] == filename) - ): - break - else: - raise ValueError(f"No matching filename on PyPI for '{filename}'") - - # Successfully found the download URL for the matching artifact. - download_url = url["url"] - checksum_sha256 = url["digests"]["sha256"] - return download_url, checksum_sha256 - - except (OSError, ValueError) as e: - # Fail if we're running in CI where we should have an internet connection. - error_if( - "CI" in os.environ, - f"Couldn't fetch metadata for project '{project}' from PyPI: {e}" - ) - return None - - -def find_ensurepip_pip_wheel() -> pathlib.Path | None: - """Try to find the pip wheel bundled in ensurepip. If missing return None""" - - ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled" + git_check_not_ignored = [] + for line in git_check_ignore_lines: + if match := git_check_ignore_re.fullmatch(line): + git_check_not_ignored.append(match.group(2) or match.group(3)) + return sorted(git_check_not_ignored) - pip_wheels = [] - try: - for wheel_filename in os.listdir(ensurepip_bundled_dir): - if wheel_filename.startswith("pip-"): - pip_wheels.append(wheel_filename) - else: - print(f"Unexpected wheel in ensurepip: '{wheel_filename}'") - sys.exit(1) - - # Ignore this error, likely caused by downstream distributors - # deleting the 'ensurepip/_bundled' directory. - except FileNotFoundError: - pass - - if len(pip_wheels) == 0: - return None - elif len(pip_wheels) > 1: - print("Multiple pip wheels detected in 'Lib/ensurepip/_bundled'") - sys.exit(1) - # Otherwise return the one pip wheel. - return ensurepip_bundled_dir / pip_wheels[0] - -def maybe_remove_pip_and_deps_from_sbom(sbom_data: dict[str, typing.Any]) -> None: +def get_externals() -> list[str]: """ - Removes pip and its dependencies from the SBOM data - if the pip wheel is removed from ensurepip. This is done - by redistributors of Python and pip. + Parses 'PCbuild/get_externals.bat' for external libraries. + Returns a list of (git tag, name, version) tuples. """ - - # If there's a wheel we don't remove anything. - if find_ensurepip_pip_wheel() is not None: - return - - # Otherwise we traverse the relationships - # to find dependent packages to remove. - sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") - sbom_spdx_ids_to_remove = {sbom_pip_spdx_id} - - # Find all package SPDXIDs that pip depends on. - for sbom_relationship in sbom_data["relationships"]: - if ( - sbom_relationship["relationshipType"] == "DEPENDS_ON" - and sbom_relationship["spdxElementId"] == sbom_pip_spdx_id - ): - sbom_spdx_ids_to_remove.add(sbom_relationship["relatedSpdxElement"]) - - # Remove all the packages and relationships. - sbom_data["packages"] = [ - sbom_package for sbom_package in sbom_data["packages"] - if sbom_package["SPDXID"] not in sbom_spdx_ids_to_remove - ] - sbom_data["relationships"] = [ - sbom_relationship for sbom_relationship in sbom_data["relationships"] - if sbom_relationship["relatedSpdxElement"] not in sbom_spdx_ids_to_remove - ] - - -def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: - """pip is a part of a packaging ecosystem (Python, surprise!) so it's actually - automatable to discover the metadata we need like the version and checksums - so let's do that on behalf of our friends at the PyPA. This function also - discovers vendored packages within pip and fetches their metadata. - """ - global PACKAGE_TO_FILES - - pip_wheel_filepath = find_ensurepip_pip_wheel() - if pip_wheel_filepath is None: - return # There's no pip wheel, nothing to discover. - - # Add the wheel filename to the list of files so the SBOM file - # and relationship generator can work its magic on the wheel too. - PACKAGE_TO_FILES["pip"] = PackageFiles( - include=[str(pip_wheel_filepath.relative_to(CPYTHON_ROOT_DIR))] - ) - - # Wheel filename format puts the version right after the project name. - pip_version = pip_wheel_filepath.name.split("-")[1] - pip_checksum_sha256 = hashlib.sha256( - pip_wheel_filepath.read_bytes() - ).hexdigest() - - pip_metadata = fetch_package_metadata_from_pypi( - project="pip", - version=pip_version, - filename=pip_wheel_filepath.name, - ) - # We couldn't fetch any metadata from PyPI, - # so we give up on verifying if we're not in CI. - if pip_metadata is None: - return - - pip_download_url, pip_actual_sha256 = pip_metadata - if pip_actual_sha256 != pip_checksum_sha256: - raise ValueError("Unexpected") - - # Parse 'pip/_vendor/vendor.txt' from the wheel for sub-dependencies. - with zipfile.ZipFile(pip_wheel_filepath) as whl: - vendor_txt_data = whl.read("pip/_vendor/vendor.txt").decode() - - # With this version regex we're assuming that pip isn't using pre-releases. - # If any version doesn't match we get a failure below, so we're safe doing this. - version_pin_re = re.compile(r"^([a-zA-Z0-9_.-]+)==([0-9.]*[0-9])$") - sbom_pip_dependency_spdx_ids = set() - for line in vendor_txt_data.splitlines(): - line = line.partition("#")[0].strip() # Strip comments and whitespace. - if not line: # Skip empty lines. - continue - - # Non-empty lines we must be able to match. - match = version_pin_re.match(line) - error_if(match is None, f"Couldn't parse line from pip vendor.txt: '{line}'") - assert match is not None # Make mypy happy. - - # Parse out and normalize the project name. - project_name, project_version = match.groups() - project_name = project_name.lower() - - # At this point if pip's metadata fetch succeeded we should - # expect this request to also succeed. - project_metadata = ( - fetch_package_metadata_from_pypi(project_name, project_version) - ) - assert project_metadata is not None - project_download_url, project_checksum_sha256 = project_metadata - - # Update our SBOM data with what we received from PyPI. - # Don't overwrite any existing values. - sbom_project_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{project_name}") - sbom_pip_dependency_spdx_ids.add(sbom_project_spdx_id) - for package in sbom_data["packages"]: - if package["SPDXID"] != sbom_project_spdx_id: - continue - - # Only thing missing from this blob is the `licenseConcluded`, - # that needs to be triaged by human maintainers if the list changes. - package.update({ - "SPDXID": sbom_project_spdx_id, - "name": project_name, - "versionInfo": project_version, - "downloadLocation": project_download_url, - "checksums": [ - {"algorithm": "SHA256", "checksumValue": project_checksum_sha256} - ], - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": f"pkg:pypi/{project_name}@{project_version}", - "referenceType": "purl", - }, - ], - "primaryPackagePurpose": "SOURCE" - }) - break - - PACKAGE_TO_FILES[project_name] = PackageFiles(include=None) - - # Remove pip from the existing SBOM packages if it's there - # and then overwrite its entry with our own generated one. - sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") - sbom_data["packages"] = [ - sbom_package - for sbom_package in sbom_data["packages"] - if sbom_package["name"] != "pip" - ] - sbom_data["packages"].append( - { - "SPDXID": sbom_pip_spdx_id, - "name": "pip", - "versionInfo": pip_version, - "originator": "Organization: Python Packaging Authority", - "licenseConcluded": "NOASSERTION", - "downloadLocation": pip_download_url, - "checksums": [ - {"algorithm": "SHA256", "checksumValue": pip_checksum_sha256} - ], - "externalRefs": [ - { - "referenceCategory": "SECURITY", - "referenceLocator": f"cpe:2.3:a:pypa:pip:{pip_version}:*:*:*:*:*:*:*", - "referenceType": "cpe23Type", - }, - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": f"pkg:pypi/pip@{pip_version}", - "referenceType": "purl", - }, - ], - "primaryPackagePurpose": "SOURCE", - } + get_externals_bat_path = CPYTHON_ROOT_DIR / "PCbuild/get_externals.bat" + externals = re.findall( + r"set\s+libraries\s*=\s*%libraries%\s+([a-zA-Z0-9.-]+)\s", + get_externals_bat_path.read_text() ) - for sbom_dep_spdx_id in sorted(sbom_pip_dependency_spdx_ids): - sbom_data["relationships"].append({ - "spdxElementId": sbom_pip_spdx_id, - "relatedSpdxElement": sbom_dep_spdx_id, - "relationshipType": "DEPENDS_ON" - }) + return externals -def main() -> None: - sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" - sbom_data = json.loads(sbom_path.read_bytes()) - - # Check if pip should be removed if the wheel is missing. - # We can't reset the SBOM relationship data until checking this. - maybe_remove_pip_and_deps_from_sbom(sbom_data) +def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: + """Make a bunch of assertions about the SBOM package data to ensure it's consistent.""" - # We regenerate all of this information. Package information - # should be preserved though since that is edited by humans. - sbom_data["files"] = [] - sbom_data["relationships"] = [] - - # Insert pip's SBOM metadata from the wheel. - discover_pip_sbom_package(sbom_data) - - # Ensure all packages in this tool are represented also in the SBOM file. - actual_names = {package["name"] for package in sbom_data["packages"]} - expected_names = set(PACKAGE_TO_FILES) - error_if( - actual_names != expected_names, - f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", - ) - - # Make a bunch of assertions about the SBOM data to ensure it's consistent. for package in sbom_data["packages"]: # Properties and ID must be properly formed. error_if( "name" not in package, "Package is missing the 'name' field" ) + + # Verify that the checksum matches the expected value + # and that the download URL is valid. + if "checksums" not in package or "CI" in os.environ: + download_location = package["downloadLocation"] + resp = urllib.request.urlopen(download_location) + error_if(resp.status != 200, f"Couldn't access URL: {download_location}'") + + package["checksums"] = [{ + "algorithm": "SHA256", + "checksumValue": hashlib.sha256(resp.read()).hexdigest() + }] + missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys()) error_if( bool(missing_required_keys), @@ -421,6 +204,20 @@ def main() -> None: ), ) + # HACL* specifies its expected rev in a refresh script. + if package["name"] == "hacl-star": + hacl_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/_hacl/refresh.sh").read_text() + hacl_expected_rev_match = re.search( + r"expected_hacl_star_rev=([0-9a-f]{40})", + hacl_refresh_sh + ) + hacl_expected_rev = hacl_expected_rev_match and hacl_expected_rev_match.group(1) + + error_if( + hacl_expected_rev != version, + "HACL* SBOM version doesn't match value in 'Modules/_hacl/refresh.sh'" + ) + # License must be on the approved list for SPDX. license_concluded = package["licenseConcluded"] error_if( @@ -428,6 +225,26 @@ def main() -> None: f"License identifier must be 'NOASSERTION'" ) + +def create_source_sbom() -> None: + sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" + sbom_data = json.loads(sbom_path.read_bytes()) + + # We regenerate all of this information. Package information + # should be preserved though since that is edited by humans. + sbom_data["files"] = [] + sbom_data["relationships"] = [] + + # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(PACKAGE_TO_FILES) + error_if( + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", + ) + + check_sbom_packages(sbom_data) + # We call 'sorted()' here a lot to avoid filesystem scan order issues. for name, files in sorted(PACKAGE_TO_FILES.items()): package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}") @@ -442,12 +259,20 @@ def main() -> None: ) for path in paths: + + # Normalize the filename from any combination of slashes. + path = str(PurePosixPath(PureWindowsPath(path))) + # Skip directories and excluded files if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude: continue # SPDX requires SHA1 to be used for files, but we provide SHA256 too. data = (CPYTHON_ROOT_DIR / path).read_bytes() + # We normalize line-endings for consistent checksums. + # This is a rudimentary check for binary files. + if b"\x00" not in data: + data = data.replace(b"\r\n", b"\n") checksum_sha1 = hashlib.sha1(data).hexdigest() checksum_sha256 = hashlib.sha256(data).hexdigest() @@ -472,5 +297,68 @@ def main() -> None: sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) +def create_externals_sbom() -> None: + sbom_path = CPYTHON_ROOT_DIR / "Misc/externals.spdx.json" + sbom_data = json.loads(sbom_path.read_bytes()) + + externals = get_externals() + externals_name_to_version = {} + externals_name_to_git_tag = {} + for git_tag in externals: + name, _, version = git_tag.rpartition("-") + externals_name_to_version[name] = version + externals_name_to_git_tag[name] = git_tag + + # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(externals_name_to_version) + error_if( + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", + ) + + # Set the versionInfo and downloadLocation fields for all packages. + for package in sbom_data["packages"]: + package_version = externals_name_to_version[package["name"]] + + # Update the version information in all the locations. + package["versionInfo"] = package_version + for external_ref in package["externalRefs"]: + if external_ref["referenceType"] != "cpe23Type": + continue + # Version is the fifth field of a CPE. + cpe23ref = external_ref["referenceLocator"] + external_ref["referenceLocator"] = re.sub( + r"\A(cpe(?::[^:]+){4}):[^:]+:", + fr"\1:{package_version}:", + cpe23ref + ) + + download_location = ( + f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz" + ) + download_location_changed = download_location != package["downloadLocation"] + package["downloadLocation"] = download_location + + # If the download URL has changed we want one to get recalulated. + if download_location_changed: + package.pop("checksums", None) + + check_sbom_packages(sbom_data) + + # Update the SBOM on disk + sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) + + +def main() -> None: + # Don't regenerate the SBOM if we're not a git repository. + if not is_root_directory_git_index(): + print("Skipping SBOM generation due to not being a git repository") + return + + create_source_sbom() + create_externals_sbom() + + if __name__ == "__main__": main() diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 5dce4e042d1eb4..f9fd29509f3225 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -32,11 +32,10 @@ '_testconsole', '_testimportmultiple', '_testinternalcapi', + '_testlimitedcapi', '_testmultiphase', '_testsinglephase', - '_xxsubinterpreters', - '_xxinterpchannels', - '_xxinterpqueues', + '_testexternalinspection', '_xxtestfuzz', 'idlelib.idle_test', 'test', diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh index e34a36c1a573e5..e1ecefddeb8732 100755 --- a/Tools/build/regen-configure.sh +++ b/Tools/build/regen-configure.sh @@ -2,15 +2,13 @@ set -e -x -# The check_generated_files job of .github/workflows/build.yml must kept in +# The check_autoconf_regen job of .github/workflows/build.yml must kept in # sync with this script. Use the same container image than the job so the job # doesn't need to run autoreconf in a container. -IMAGE="ubuntu:22.04" -DEPENDENCIES="autotools-dev autoconf autoconf-archive pkg-config" +IMAGE="ghcr.io/python/autoconf:2024.10.16.11360930377" AUTORECONF="autoreconf -ivf -Werror" WORK_DIR="/src" -SHELL_CMD="apt-get update && apt-get -yq install $DEPENDENCIES && cd $WORK_DIR && $AUTORECONF" abs_srcdir=$(cd $(dirname $0)/../..; pwd) @@ -28,4 +26,4 @@ if command -v selinuxenabled >/dev/null && selinuxenabled; then PATH_OPT=":Z" fi -"$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE" /usr/bin/bash -c "$SHELL_CMD" +"$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE" diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 83146622c74f94..2874c436c1ebc9 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -25,6 +25,8 @@ import csv SCRIPT_NAME = 'Tools/build/stable_abi.py' +DEFAULT_MANIFEST_PATH = ( + Path(__file__).parent / '../../Misc/stable_abi.toml').resolve() MISSING = object() EXCLUDED_HEADERS = { @@ -225,9 +227,9 @@ def sort_key(item): key=sort_key): write(f'EXPORT_DATA({item.name})') -REST_ROLES = { - 'function': 'function', - 'data': 'var', +ITEM_KIND_TO_DOC_ROLE = { + 'function': 'func', + 'data': 'data', 'struct': 'type', 'macro': 'macro', # 'const': 'const', # all undocumented @@ -236,22 +238,28 @@ def sort_key(item): @generator("doc_list", 'Doc/data/stable_abi.dat') def gen_doc_annotations(manifest, args, outfile): - """Generate/check the stable ABI list for documentation annotations""" + """Generate/check the stable ABI list for documentation annotations + + See ``StableABIEntry`` in ``Doc/tools/extensions/c_annotations.py`` + for a description of each field. + """ writer = csv.DictWriter( outfile, ['role', 'name', 'added', 'ifdef_note', 'struct_abi_kind'], lineterminator='\n') writer.writeheader() - for item in manifest.select(REST_ROLES.keys(), include_abi_only=False): + kinds = set(ITEM_KIND_TO_DOC_ROLE) + for item in manifest.select(kinds, include_abi_only=False): if item.ifdef: ifdef_note = manifest.contents[item.ifdef].doc else: ifdef_note = None row = { - 'role': REST_ROLES[item.kind], + 'role': ITEM_KIND_TO_DOC_ROLE[item.kind], 'name': item.name, 'added': item.added, - 'ifdef_note': ifdef_note} + 'ifdef_note': ifdef_note, + } rows = [row] if item.kind == 'struct': row['struct_abi_kind'] = item.struct_abi_kind @@ -259,7 +267,8 @@ def gen_doc_annotations(manifest, args, outfile): rows.append({ 'role': 'member', 'name': f'{item.name}.{member_name}', - 'added': item.added}) + 'added': item.added, + }) writer.writerows(rows) @generator("ctypes_test", 'Lib/test/test_stable_abi_ctypes.py') @@ -275,7 +284,10 @@ def gen_ctypes_test(manifest, args, outfile): import sys import unittest from test.support.import_helper import import_module - from _testcapi import get_feature_macros + try: + from _testcapi import get_feature_macros + except ImportError: + raise unittest.SkipTest("requires _testcapi") feature_macros = get_feature_macros() @@ -601,7 +613,7 @@ def check_private_names(manifest): if name.startswith('_') and not item.abi_only: raise ValueError( f'`{name}` is private (underscore-prefixed) and should be ' - + 'removed from the stable ABI list or or marked `abi_only`') + + 'removed from the stable ABI list or marked `abi_only`') def check_dump(manifest, filename): """Check that manifest.dump() corresponds to the data. @@ -631,8 +643,9 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( - "file", type=Path, metavar='FILE', - help="file with the stable abi manifest", + "file", type=Path, metavar='FILE', nargs='?', + default=DEFAULT_MANIFEST_PATH, + help=f"file with the stable abi manifest (default: {DEFAULT_MANIFEST_PATH})", ) parser.add_argument( "--generate", action='store_true', @@ -674,7 +687,7 @@ def main(): if args.list: for gen in generators: - print(f'{gen.arg_name}: {base_path / gen.default_path}') + print(f'{gen.arg_name}: {(base_path / gen.default_path).resolve()}') sys.exit(0) run_all_generators = args.generate_all @@ -725,8 +738,10 @@ def main(): if not results: if args.generate: - parser.error('No file specified. Use --help for usage.') - parser.error('No check specified. Use --help for usage.') + parser.error('No file specified. Use --generate-all to regenerate ' + + 'all files, or --help for usage.') + parser.error('No check specified. Use --all to check all files, ' + + 'or --help for usage.') failed_results = [name for name, result in results.items() if not result] diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 781f9a4c8206c8..0c47470a0ecb7a 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -7,17 +7,10 @@ set here=%~dp0 set rt_opts=-q -d set regrtest_args= set arm32_ssh= +set cmdline_args=%* +set cmdline_args=%cmdline_args:,=#COMMA#% -:CheckOpts -if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts -if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts -if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts -if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts +call:CheckOpts %cmdline_args% if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto NativeExecution if "%arm32_ssh%"=="true" goto :Arm32Ssh @@ -49,3 +42,16 @@ echo The test worker should have the SSH agent running. echo Also a key must be created with ssh-keygen and added to both the buildbot worker machine echo and the ARM32 worker device: see https://docs.microsoft.com/en-us/windows/iot-core/connect-your-device/ssh exit /b 127 + +:CheckOpts +set arg="%~1" +if %arg%=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts +if %arg%=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts +if %arg%=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts +if NOT %arg%=="" (set regrtest_args=%regrtest_args% %arg:#COMMA#=,%) & shift & goto CheckOpts +goto:eof diff --git a/Tools/c-analyzer/README b/Tools/c-analyzer/README index 86bf1e77e0bfea..41ea13280347a7 100644 --- a/Tools/c-analyzer/README +++ b/Tools/c-analyzer/README @@ -11,9 +11,8 @@ falls into one of several categories: * module state * Python runtime state -The ignored-globals.txt file is organized similarly. Of the different -categories, the last two are problematic and generally should not exist -in the codebase. +Of the different categories, the last two are problematic and +generally should not exist in the codebase. Globals that hold module state (i.e. in Modules/*.c) cause problems when multiple interpreters are in use. For more info, see PEP 3121, @@ -42,4 +41,3 @@ You can also use the more generic tool: If it reports any globals then they should be resolved. If the globals are runtime state then they should be folded into _PyRuntimeState. -Otherwise they should be added to ignored-globals.txt. diff --git a/Tools/c-analyzer/TODO b/Tools/c-analyzer/TODO index 3d599538510bd9..e81ceb29c64bf0 100644 --- a/Tools/c-analyzer/TODO +++ b/Tools/c-analyzer/TODO @@ -562,8 +562,6 @@ Objects/unicodeobject.c:static_strings static _Py_Iden # PyTypeObject (311) Modules/_abc.c:_abc_data_type static PyTypeObject _abc_data_type -Modules/_blake2/blake2b_impl.c:PyBlake2_BLAKE2bType PyTypeObject PyBlake2_BLAKE2bType -Modules/_blake2/blake2s_impl.c:PyBlake2_BLAKE2sType PyTypeObject PyBlake2_BLAKE2sType Modules/_collectionsmodule.c:defdict_type static PyTypeObject defdict_type Modules/_collectionsmodule.c:deque_type static PyTypeObject deque_type Modules/_collectionsmodule.c:dequeiter_type static PyTypeObject dequeiter_type diff --git a/Tools/c-analyzer/c_parser/parser/_regexes.py b/Tools/c-analyzer/c_parser/parser/_regexes.py index 5695daff67d6bd..c1a8ab3ad2f15d 100644 --- a/Tools/c-analyzer/c_parser/parser/_regexes.py +++ b/Tools/c-analyzer/c_parser/parser/_regexes.py @@ -72,6 +72,7 @@ def _ind(text, level=1, edges='both'): long | float | double | + _Complex | void | struct | @@ -121,6 +122,16 @@ def _ind(text, level=1, edges='both'): | (?: signed | unsigned ) # implies int | + (?: + (?: (?: float | double | long\s+double ) \s+ )? + _Complex + ) + | + (?: + _Complex + (?: \s+ (?: float | double | long\s+double ) )? + ) + | (?: (?: (?: signed | unsigned ) \s+ )? (?: (?: long | short ) \s+ )? diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 6ece70c77fd55c..d20cd19f6e6d5e 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -7,6 +7,7 @@ FILES_WITHOUT_INTERNAL_CAPI = frozenset(( # Modules/ '_testcapimodule.c', + '_testlimitedcapi.c', '_testclinic_limited.c', 'xxlimited.c', 'xxlimited_35.c', @@ -15,8 +16,8 @@ # C files in the fhe following directories must not be built with # Py_BUILD_CORE. DIRS_WITHOUT_INTERNAL_CAPI = frozenset(( - # Modules/_testcapi/ - '_testcapi', + '_testcapi', # Modules/_testcapi/ + '_testlimitedcapi', # Modules/_testlimitedcapi/ )) TOOL = 'gcc' @@ -148,7 +149,7 @@ def _iter_top_include_lines(lines, topfile, cwd, raw): partial = 0 # depth files = [topfile] - # We start at 1 in case there are source lines (including blank onces) + # We start at 1 in case there are source lines (including blank ones) # before the first marker line. Also, we already verified in # _parse_marker_line() that the preprocessor reported lno as 1. lno = 1 diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 444063d2148934..3a73f65f8ff7b3 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -83,9 +83,11 @@ def clean_lines(text): Python/frozen_modules/*.h Python/generated_cases.c.h Python/executor_cases.c.h +Python/optimizer_cases.c.h # not actually source Python/bytecodes.c +Python/optimizer_bytecodes.c # mimalloc Objects/mimalloc/*.c @@ -102,6 +104,7 @@ def clean_lines(text): # The problem with xmlparse.c is that something # has gone wrong where # we handle "maybe inline actual" # in Tools/c-analyzer/c_parser/parser/_global.py. +Modules/expat/internal.h Modules/expat/xmlparse.c ''') @@ -117,11 +120,14 @@ def clean_lines(text): Modules/_decimal/**/*.c Modules/_decimal/libmpdec Modules/_elementtree.c Modules/expat Modules/_hacl/*.c Modules/_hacl/include +Modules/_hacl/*.c Modules/_hacl/ Modules/_hacl/*.h Modules/_hacl/include +Modules/_hacl/*.h Modules/_hacl/ Modules/md5module.c Modules/_hacl/include Modules/sha1module.c Modules/_hacl/include Modules/sha2module.c Modules/_hacl/include Modules/sha3module.c Modules/_hacl/include +Modules/blake2module.c Modules/_hacl/include Objects/stringlib/*.h Objects # possible system-installed headers, just in case @@ -164,6 +170,7 @@ def clean_lines(text): Objects/stringlib/find.h Objects/stringlib/fastsearch.h Objects/stringlib/partition.h Objects/stringlib/fastsearch.h Objects/stringlib/replace.h Objects/stringlib/fastsearch.h +Objects/stringlib/repr.h Objects/stringlib/fastsearch.h Objects/stringlib/split.h Objects/stringlib/fastsearch.h # @end=tsv@ @@ -318,9 +325,11 @@ def clean_lines(text): _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), _abs('Objects/typeobject.c'): (35_000, 200), _abs('Python/compile.c'): (20_000, 500), + _abs('Python/optimizer.c'): (100_000, 5_000), _abs('Python/parking_lot.c'): (40_000, 1000), _abs('Python/pylifecycle.c'): (500_000, 5000), _abs('Python/pystate.c'): (500_000, 5000), + _abs('Python/initconfig.c'): (50_000, 500), # Generated files: _abs('Include/internal/pycore_opcode.h'): (10_000, 1000), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 1d9576d083d8dc..badd7b79102310 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -43,6 +43,7 @@ Objects/enumobject.c - PyReversed_Type - Objects/fileobject.c - PyStdPrinter_Type - Objects/floatobject.c - PyFloat_Type - Objects/frameobject.c - PyFrame_Type - +Objects/frameobject.c - PyFrameLocalsProxy_Type - Objects/funcobject.c - PyClassMethod_Type - Objects/funcobject.c - PyFunction_Type - Objects/funcobject.c - PyStaticMethod_Type - @@ -54,7 +55,6 @@ Objects/genobject.c - _PyAsyncGenASend_Type - Objects/genobject.c - _PyAsyncGenAThrow_Type - Objects/genobject.c - _PyAsyncGenWrappedValue_Type - Objects/genobject.c - _PyCoroWrapper_Type - -Objects/interpreteridobject.c - PyInterpreterID_Type - Objects/iterobject.c - PyCallIter_Type - Objects/iterobject.c - PySeqIter_Type - Objects/iterobject.c - _PyAnextAwaitable_Type - @@ -93,6 +93,7 @@ Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - Objects/typevarobject.c - _PyTypeAlias_Type - +Objects/typevarobject.c - _PyNoDefault_Type - Objects/unicodeobject.c - PyUnicodeIter_Type - Objects/unicodeobject.c - PyUnicode_Type - Objects/weakrefobject.c - _PyWeakref_CallableProxyType - @@ -104,6 +105,7 @@ Python/bltinmodule.c - PyZip_Type - Python/context.c - PyContextToken_Type - Python/context.c - PyContextVar_Type - Python/context.c - PyContext_Type - +Python/instruction_sequence.c - _PyInstructionSequence_Type - Python/traceback.c - PyTraceBack_Type - ##----------------------- @@ -189,6 +191,7 @@ Objects/exceptions.c - _PyExc_ProcessLookupError - Objects/exceptions.c - _PyExc_TimeoutError - Objects/exceptions.c - _PyExc_EOFError - Objects/exceptions.c - _PyExc_RuntimeError - +Objects/exceptions.c - _PyExc_PythonFinalizationError - Objects/exceptions.c - _PyExc_RecursionError - Objects/exceptions.c - _PyExc_NotImplementedError - Objects/exceptions.c - _PyExc_NameError - @@ -254,6 +257,7 @@ Objects/exceptions.c - PyExc_ProcessLookupError - Objects/exceptions.c - PyExc_TimeoutError - Objects/exceptions.c - PyExc_EOFError - Objects/exceptions.c - PyExc_RuntimeError - +Objects/exceptions.c - PyExc_PythonFinalizationError - Objects/exceptions.c - PyExc_RecursionError - Objects/exceptions.c - PyExc_NotImplementedError - Objects/exceptions.c - PyExc_NameError - @@ -292,14 +296,18 @@ Objects/exceptions.c - PyExc_UnicodeWarning - Objects/exceptions.c - PyExc_BytesWarning - Objects/exceptions.c - PyExc_ResourceWarning - Objects/exceptions.c - PyExc_EncodingWarning - -Python/crossinterp.c - _PyExc_InterpreterError - -Python/crossinterp.c - _PyExc_InterpreterNotFoundError - -Python/crossinterp.c - PyExc_InterpreterError - -Python/crossinterp.c - PyExc_InterpreterNotFoundError - +Python/crossinterp_exceptions.h - _PyExc_InterpreterError - +Python/crossinterp_exceptions.h - _PyExc_InterpreterNotFoundError - +Python/crossinterp_exceptions.h - PyExc_InterpreterError - +Python/crossinterp_exceptions.h - PyExc_InterpreterNotFoundError - ##----------------------- ## singletons +Modules/_datetimemodule.c - zero_delta - +Modules/_datetimemodule.c - utc_timezone - +Modules/_datetimemodule.c - capi - +Modules/_datetimemodule.c - _globals - Objects/boolobject.c - _Py_FalseStruct - Objects/boolobject.c - _Py_TrueStruct - Objects/dictobject.c - empty_keys_struct - @@ -308,6 +316,7 @@ Objects/object.c - _Py_NotImplementedStruct - Objects/setobject.c - _dummy_struct - Objects/setobject.c - _PySet_Dummy - Objects/sliceobject.c - _Py_EllipsisObject - +Objects/typevarobject.c - _Py_NoDefaultStruct - Python/instrumentation.c - _PyInstrumentation_DISABLE - Python/instrumentation.c - _PyInstrumentation_MISSING - @@ -349,38 +358,6 @@ Modules/_testclinic.c - TestClass - ##----------------------- ## static types -Modules/_ctypes/_ctypes.c - PyCArrayType_Type - -Modules/_ctypes/_ctypes.c - PyCArray_Type - -Modules/_ctypes/_ctypes.c - PyCData_Type - -Modules/_ctypes/_ctypes.c - PyCFuncPtrType_Type - -Modules/_ctypes/_ctypes.c - PyCFuncPtr_Type - -Modules/_ctypes/_ctypes.c - PyCPointerType_Type - -Modules/_ctypes/_ctypes.c - PyCPointer_Type - -Modules/_ctypes/_ctypes.c - PyCSimpleType_Type - -Modules/_ctypes/_ctypes.c - PyCStructType_Type - -Modules/_ctypes/_ctypes.c - Simple_Type - -Modules/_ctypes/_ctypes.c - Struct_Type - -Modules/_ctypes/_ctypes.c - UnionType_Type - -Modules/_ctypes/_ctypes.c - Union_Type - -Modules/_ctypes/callproc.c - PyCArg_Type - -Modules/_ctypes/ctypes.h - PyCArg_Type - -Modules/_ctypes/ctypes.h - PyCArrayType_Type - -Modules/_ctypes/ctypes.h - PyCArray_Type - -Modules/_ctypes/ctypes.h - PyCData_Type - -Modules/_ctypes/ctypes.h - PyCFuncPtrType_Type - -Modules/_ctypes/ctypes.h - PyCFuncPtr_Type - -Modules/_ctypes/ctypes.h - PyCPointerType_Type - -Modules/_ctypes/ctypes.h - PyCPointer_Type - -Modules/_ctypes/ctypes.h - PyCSimpleType_Type - -Modules/_ctypes/ctypes.h - PyCStgDict_Type - -Modules/_ctypes/ctypes.h - PyCStructType_Type - -Modules/_ctypes/ctypes.h - PyExc_ArgError - -Modules/_ctypes/ctypes.h - _ctypes_conversion_encoding - -Modules/_ctypes/ctypes.h - _ctypes_conversion_errors - -Modules/_ctypes/ctypes.h - _ctypes_ptrtype_cache - -Modules/_ctypes/ctypes.h - basespec_string - -Modules/_ctypes/stgdict.c - PyCStgDict_Type - -Modules/_cursesmodule.c - PyCursesWindow_Type - Modules/_datetimemodule.c - PyDateTime_DateTimeType - Modules/_datetimemodule.c - PyDateTime_DateType - Modules/_datetimemodule.c - PyDateTime_DeltaType - @@ -393,6 +370,7 @@ Modules/xxmodule.c - Str_Type - Modules/xxmodule.c - Xxo_Type - Modules/xxsubtype.c - spamdict_type - Modules/xxsubtype.c - spamlist_type - +Modules/_testcapi/monitoring.c - PyCodeLike_Type - ##----------------------- ## non-static types - initialized once @@ -404,32 +382,14 @@ Modules/_tkinter.c - Tktt_Type - Modules/xxlimited_35.c - Xxo_Type - ## exception types -Modules/_ctypes/_ctypes.c - PyExc_ArgError - -Modules/_cursesmodule.c - PyCursesError - Modules/_tkinter.c - Tkinter_TclError - Modules/xxlimited_35.c - ErrorObject - Modules/xxmodule.c - ErrorObject - -##----------------------- -## cached - initialized once - -## manually cached PyUnicodeOjbect -Modules/_ctypes/callproc.c _ctypes_get_errobj error_object_name - -Modules/_ctypes/_ctypes.c CreateSwappedType suffix - - ##----------------------- ## other -## initialized once -Modules/_ctypes/_ctypes.c - _unpickle - -Modules/_ctypes/_ctypes.c PyCArrayType_from_ctype cache - -Modules/_cursesmodule.c - ModDict - -Modules/_datetimemodule.c datetime_strptime module - - ## state -Modules/_ctypes/_ctypes.c - _ctypes_ptrtype_cache - -Modules/_ctypes/_ctypes.c - global_state - -Modules/_ctypes/ctypes.h - global_state - Modules/_datetimemodule.c - _datetime_global_state - Modules/_tkinter.c - tcl_lock - Modules/_tkinter.c - excInCmd - @@ -447,10 +407,11 @@ Modules/_tkinter.c - trbInCmd - Include/datetime.h - PyDateTimeAPI - Modules/_ctypes/cfield.c _ctypes_get_fielddesc initialized - Modules/_ctypes/malloc_closure.c - _pagesize - -Modules/_cursesmodule.c - initialised - -Modules/_cursesmodule.c - initialised_setupterm - -Modules/_cursesmodule.c - initialisedcolors - -Modules/_cursesmodule.c - screen_encoding - +Modules/_cursesmodule.c - curses_module_loaded - +Modules/_cursesmodule.c - curses_initscr_called - +Modules/_cursesmodule.c - curses_setupterm_called - +Modules/_cursesmodule.c - curses_start_color_called - +Modules/_cursesmodule.c - curses_screen_encoding - Modules/_elementtree.c - expat_capi - Modules/readline.c - libedit_append_replace_history_offset - Modules/readline.c - using_libedit_emulation - @@ -480,3 +441,4 @@ Modules/readline.c - sigwinch_ohandler - Modules/readline.c - completed_input_string - Modules/rotatingtree.c - random_stream - Modules/rotatingtree.c - random_value - +Modules/rotatingtree.c - random_mutex - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c75aff8c1723c1..686f3935d91bda 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -90,6 +90,7 @@ Python/initconfig.c - _Py_StandardStreamErrors - # Internal constant list Python/initconfig.c - PYCONFIG_SPEC - +Python/initconfig.c - PYPRECONFIG_SPEC - ##----------------------- @@ -164,8 +165,8 @@ Python/pylifecycle.c _Py_FatalErrorFormat reentrant - Python/pylifecycle.c fatal_error reentrant - # explicitly protected, internal-only -Modules/_xxinterpchannelsmodule.c - _globals - -Modules/_xxinterpqueuesmodule.c - _globals - +Modules/_interpchannelsmodule.c - _globals - +Modules/_interpqueuesmodule.c - _globals - # set once during module init Modules/_decimal/_decimal.c - minalloc_is_set - @@ -198,6 +199,7 @@ Python/pystate.c - _Py_tss_tstate - Include/internal/pycore_blocks_output_buffer.h - BUFFER_BLOCK_SIZE - Modules/_csv.c - quote_styles - +Modules/_ctypes/_ctypes.c - _ctypesmodule - Modules/_ctypes/cfield.c - ffi_type_double - Modules/_ctypes/cfield.c - ffi_type_float - Modules/_ctypes/cfield.c - ffi_type_longdouble - @@ -216,6 +218,7 @@ Modules/_datetimemodule.c - max_fold_seconds - Modules/_datetimemodule.c datetime_isoformat specs - Modules/_datetimemodule.c parse_hh_mm_ss_ff correction - Modules/_datetimemodule.c time_isoformat specs - +Modules/_datetimemodule.c - capi_types - Modules/_decimal/_decimal.c - cond_map_template - Modules/_decimal/_decimal.c - dec_signal_string - Modules/_decimal/_decimal.c - dflt_ctx - @@ -245,11 +248,11 @@ Modules/_struct.c - bigendian_table - Modules/_struct.c - lilendian_table - Modules/_struct.c - native_table - Modules/_tkinter.c - state_key - -Modules/_xxinterpchannelsmodule.c - _channelid_end_recv - -Modules/_xxinterpchannelsmodule.c - _channelid_end_send - +Modules/_interpchannelsmodule.c - _channelid_end_recv - +Modules/_interpchannelsmodule.c - _channelid_end_send - Modules/_zoneinfo.c - DAYS_BEFORE_MONTH - Modules/_zoneinfo.c - DAYS_IN_MONTH - -Modules/_xxsubinterpretersmodule.c - no_exception - +Modules/_interpretersmodule.c - no_exception - Modules/arraymodule.c - descriptors - Modules/arraymodule.c - emptybuf - Modules/cjkcodecs/_codecs_cn.c - _mapping_list - @@ -342,9 +345,9 @@ Python/ast_opt.c fold_unaryop ops - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - Python/codecs.c - Py_hexdigits - +Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - -Python/codecs.c _PyCodecRegistry_Init methods - -Python/compile.c - NO_LABEL - +Python/codecs.c _PyCodec_InitRegistry methods - Python/compile.c - NO_LOCATION - Python/dynload_shlib.c - _PyImport_DynLoadFiletab - Python/dynload_stub.c - _PyImport_DynLoadFiletab - @@ -361,8 +364,11 @@ Python/import.c - _PyImport_Inittab - Python/import.c - _PySys_ImplCacheTag - Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - +Python/lock.c - TIME_TO_BE_FAIR_NS - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - +Python/perf_jit_trampoline.c - _Py_perfmap_jit_callbacks - +Python/perf_jit_trampoline.c - perf_jit_map_state - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - @@ -382,12 +388,13 @@ Python/optimizer.c - _PyCounterOptimizer_Type - Python/optimizer.c - _PyUOpExecutor_Type - Python/optimizer.c - _PyUOpOptimizer_Type - Python/optimizer.c - _PyOptimizer_Default - +Python/optimizer.c - _ColdExit_Type - +Python/optimizer.c - Py_FatalErrorExecutor - +Python/optimizer.c - EMPTY_FILTER - ##----------------------- ## test code -Modules/_ctypes/_ctypes_test.c - _ctypes_test_slots - -Modules/_ctypes/_ctypes_test.c - _ctypes_testmodule - Modules/_ctypes/_ctypes_test.c - _xxx_lib - Modules/_ctypes/_ctypes_test.c - an_integer - Modules/_ctypes/_ctypes_test.c - bottom - @@ -395,7 +402,6 @@ Modules/_ctypes/_ctypes_test.c - last_tf_arg_s - Modules/_ctypes/_ctypes_test.c - last_tf_arg_u - Modules/_ctypes/_ctypes_test.c - last_tfrsuv_arg - Modules/_ctypes/_ctypes_test.c - left - -Modules/_ctypes/_ctypes_test.c - module_methods - Modules/_ctypes/_ctypes_test.c - my_eggs - Modules/_ctypes/_ctypes_test.c - my_spams - Modules/_ctypes/_ctypes_test.c - right - @@ -438,7 +444,6 @@ Modules/_testcapi/heaptype.c - _testcapimodule - Modules/_testcapi/mem.c - FmData - Modules/_testcapi/mem.c - FmHook - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - -Modules/_testcapi/unicode.c - _testcapimodule - Modules/_testcapi/watchers.c - g_dict_watch_events - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - @@ -449,6 +454,9 @@ Modules/_testcapi/watchers.c - num_code_object_destroyed_events - Modules/_testcapi/watchers.c - pyfunc_watchers - Modules/_testcapi/watchers.c - func_watcher_ids - Modules/_testcapi/watchers.c - func_watcher_callbacks - +Modules/_testcapi/watchers.c - context_watcher_ids - +Modules/_testcapi/watchers.c - context_switches - +Modules/_testcapi/watchers.c add_context_watcher callbacks - Modules/_testcapimodule.c - BasicStaticTypes - Modules/_testcapimodule.c - num_basic_static_types_used - Modules/_testcapimodule.c - ContainerNoGC_members - @@ -586,6 +594,7 @@ Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - Modules/_testmultiphase.c - testexport_methods - Modules/_testmultiphase.c - uninitialized_def - Modules/_testsinglephase.c - global_state - +Modules/_testsinglephase.c - static_module_circular - Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - Modules/_xxtestfuzz/fuzzer.c - RE_FLAG_DEBUG - @@ -620,8 +629,6 @@ Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput ELEMENTTREE_PARSEWHOLE_INITI Include/internal/pycore_importdl.h - _PyImport_DynLoadFiletab - Include/py_curses.h - PyCurses_API - Include/pydecimal.h - _decimal_api - -Modules/_blake2/blake2module.c - blake2b_type_spec - -Modules/_blake2/blake2module.c - blake2s_type_spec - Modules/_io/fileio.c - _Py_open_cloexec_works - Modules/_io/_iomodule.h - PyIOBase_Type - Modules/_io/_iomodule.h - PyRawIOBase_Type - @@ -734,6 +741,11 @@ Modules/expat/xmlrole.c - error - ## other Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - -Python/optimizer_analysis.c - _Py_PartitionRootNode_Type - -Python/optimizer_analysis.c - _Py_UOpsAbstractInterpContext_Type - Modules/clinic/md5module.c.h _md5_md5 _keywords - +Modules/clinic/grpmodule.c.h grp_getgrgid _keywords - +Modules/clinic/grpmodule.c.h grp_getgrnam _keywords - +Objects/object.c - constants static PyObject*[] + + +## False positives +Python/specialize.c - _Py_InitCleanup - diff --git a/Tools/cases_generator/README.md b/Tools/cases_generator/README.md index 7fec8a882336cd..fb512c4646b851 100644 --- a/Tools/cases_generator/README.md +++ b/Tools/cases_generator/README.md @@ -13,6 +13,9 @@ What's currently here: - `parser.py` helper for interactions with `parsing.py` - `tierN_generator.py`: a couple of driver scripts to read `Python/bytecodes.c` and write `Python/generated_cases.c.h` (and several other files) +- `optimizer_generator.py`: reads `Python/bytecodes.c` and + `Python/optimizer_bytecodes.c` and writes + `Python/optimizer_cases.c.h` - `stack.py`: code to handle generalized stack effects - `cwriter.py`: code which understands tokens and how to format C code; main class: `CWriter` diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index b80fa66e2a159a..a725ec10d4e52a 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1,79 +1,92 @@ from dataclasses import dataclass, field +import itertools import lexer import parser +import re from typing import Optional - @dataclass class Properties: - escapes: bool - infallible: bool + escaping_calls: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] + error_with_pop: bool + error_without_pop: bool deopts: bool oparg: bool jumps: bool eval_breaker: bool - ends_with_eval_breaker: bool needs_this: bool always_exits: bool stores_sp: bool - tier_one_only: bool uses_co_consts: bool uses_co_names: bool uses_locals: bool has_free: bool - + side_exit: bool pure: bool - passthrough: bool - guard: bool + tier: int | None = None + oparg_and_1: bool = False + const_oparg: int = -1 + needs_prev: bool = False def dump(self, indent: str) -> None: - print(indent, end="") - text = ", ".join([f"{key}: {value}" for (key, value) in self.__dict__.items()]) + simple_properties = self.__dict__.copy() + del simple_properties["escaping_calls"] + text = "escaping_calls:\n" + for tkns in self.escaping_calls.values(): + text += f"{indent} {tkns}\n" + text += ", ".join([f"{key}: {value}" for (key, value) in simple_properties.items()]) print(indent, text, sep="") @staticmethod def from_list(properties: list["Properties"]) -> "Properties": + escaping_calls: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] = {} + for p in properties: + escaping_calls.update(p.escaping_calls) return Properties( - escapes=any(p.escapes for p in properties), - infallible=all(p.infallible for p in properties), + escaping_calls=escaping_calls, + error_with_pop=any(p.error_with_pop for p in properties), + error_without_pop=any(p.error_without_pop for p in properties), deopts=any(p.deopts for p in properties), oparg=any(p.oparg for p in properties), jumps=any(p.jumps for p in properties), eval_breaker=any(p.eval_breaker for p in properties), - ends_with_eval_breaker=any(p.ends_with_eval_breaker for p in properties), needs_this=any(p.needs_this for p in properties), always_exits=any(p.always_exits for p in properties), stores_sp=any(p.stores_sp for p in properties), - tier_one_only=any(p.tier_one_only for p in properties), uses_co_consts=any(p.uses_co_consts for p in properties), uses_co_names=any(p.uses_co_names for p in properties), uses_locals=any(p.uses_locals for p in properties), has_free=any(p.has_free for p in properties), + side_exit=any(p.side_exit for p in properties), pure=all(p.pure for p in properties), - passthrough=all(p.passthrough for p in properties), - guard=all(p.guard for p in properties), + needs_prev=any(p.needs_prev for p in properties), ) + @property + def infallible(self) -> bool: + return not self.error_with_pop and not self.error_without_pop + + @property + def escapes(self) -> bool: + return bool(self.escaping_calls) SKIP_PROPERTIES = Properties( - escapes=False, - infallible=True, + escaping_calls={}, + error_with_pop=False, + error_without_pop=False, deopts=False, oparg=False, jumps=False, eval_breaker=False, - ends_with_eval_breaker=False, needs_this=False, always_exits=False, stores_sp=False, - tier_one_only=False, uses_co_consts=False, uses_co_names=False, uses_locals=False, has_free=False, - pure=False, - passthrough=False, - guard=False, + side_exit=False, + pure=True, ) @@ -91,6 +104,20 @@ def properties(self) -> Properties: return SKIP_PROPERTIES +class Flush: + @property + def properties(self) -> Properties: + return SKIP_PROPERTIES + + @property + def name(self) -> str: + return "flush" + + @property + def size(self) -> int: + return 0 + + @dataclass class StackItem: name: str @@ -98,18 +125,19 @@ class StackItem: condition: str | None size: str peek: bool = False - type_prop: None | tuple[str, None | str] = field( - default_factory=lambda: None, init=True, compare=False, hash=False - ) + used: bool = False def __str__(self) -> str: cond = f" if ({self.condition})" if self.condition else "" - size = f"[{self.size}]" if self.size != "1" else "" + size = f"[{self.size}]" if self.size else "" type = "" if self.type is None else f"{self.type} " return f"{type}{self.name}{size}{cond} {self.peek}" def is_array(self) -> bool: - return self.type == "PyObject **" + return self.size != "" + + def get_size(self) -> str: + return self.size if self.size else "1" @dataclass @@ -137,10 +165,16 @@ class Uop: annotations: list[str] stack: StackEffect caches: list[CacheEntry] + deferred_refs: dict[lexer.Token, str | None] + output_stores: list[lexer.Token] body: list[lexer.Token] properties: Properties _size: int = -1 implicitly_created: bool = False + replicated = 0 + replicates: "Uop | None" = None + # Size of the instruction(s), only set for uops containing the INSTRUCTION_SIZE macro + instruction_size: int | None = None def dump(self, indent: str) -> None: print( @@ -155,20 +189,25 @@ def size(self) -> int: self._size = sum(c.size for c in self.caches) return self._size - def is_viable(self) -> bool: + def why_not_viable(self) -> str | None: if self.name == "_SAVE_RETURN_OFFSET": - return True # Adjusts next_instr, but only in tier 1 code - if self.properties.needs_this: - return False + return None # Adjusts next_instr, but only in tier 1 code if "INSTRUMENTED" in self.name: - return False + return "is instrumented" if "replaced" in self.annotations: - return False + return "is replaced" if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"): - return False + return "has tier 1 control flow" + if self.properties.needs_this: + return "uses the 'this_instr' variable" if len([c for c in self.caches if c.name != "unused"]) > 1: - return False - return True + return "has unused cache entries" + if self.properties.error_with_pop and self.properties.error_without_pop: + return "has both popping and not-popping errors" + return None + + def is_viable(self) -> bool: + return self.why_not_viable() is None def is_super(self) -> bool: for tkn in self.body: @@ -177,11 +216,12 @@ def is_super(self) -> bool: return False -Part = Uop | Skip +Part = Uop | Skip | Flush @dataclass class Instruction: + where: lexer.Token name: str parts: list[Part] _properties: Properties | None @@ -219,7 +259,9 @@ def is_super(self) -> bool: @dataclass class PseudoInstruction: name: str + stack: StackEffect targets: list[Instruction] + as_sequence: bool flags: list[str] opcode: int = -1 @@ -271,20 +313,58 @@ def override_error( ) -def convert_stack_item(item: parser.StackEffect) -> StackItem: - return StackItem( - item.name, item.type, item.cond, (item.size or "1"), type_prop=item.type_prop - ) +def convert_stack_item( + item: parser.StackEffect, replace_op_arg_1: str | None +) -> StackItem: + cond = item.cond + if replace_op_arg_1 and OPARG_AND_1.match(item.cond): + cond = replace_op_arg_1 + return StackItem(item.name, item.type, cond, item.size) -def analyze_stack(op: parser.InstDef) -> StackEffect: +def analyze_stack( + op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None +) -> StackEffect: inputs: list[StackItem] = [ - convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect) + convert_stack_item(i, replace_op_arg_1) + for i in op.inputs + if isinstance(i, parser.StackEffect) + ] + outputs: list[StackItem] = [ + convert_stack_item(i, replace_op_arg_1) for i in op.outputs ] - outputs: list[StackItem] = [convert_stack_item(i) for i in op.outputs] - for input, output in zip(inputs, outputs): - if input.name == output.name: - input.peek = output.peek = True + # Mark variables with matching names at the base of the stack as "peek" + modified = False + input_names: dict[str, lexer.Token] = { i.name : i.first_token for i in op.inputs if i.name != "unused" } + for input, output in itertools.zip_longest(inputs, outputs): + if output is None: + pass + elif input is None: + if output.name in input_names: + raise analysis_error( + f"Reuse of variable '{output.name}' at different stack location", + input_names[output.name]) + elif input.name == output.name: + if not modified: + input.peek = output.peek = True + else: + modified = True + if output.name in input_names: + raise analysis_error( + f"Reuse of variable '{output.name}' at different stack location", + input_names[output.name]) + if isinstance(op, parser.InstDef): + output_names = [out.name for out in outputs] + for input in inputs: + if ( + variable_used(op, input.name) + or variable_used(op, "DECREF_INPUTS") + or (not input.peek and input.name in output_names) + ): + input.used = True + for output in outputs: + if variable_used(op, output.name): + output.used = True return StackEffect(inputs, outputs) @@ -300,17 +380,135 @@ def analyze_caches(inputs: list[parser.InputEffect]) -> list[CacheEntry]: return [CacheEntry(i.name, int(i.size)) for i in caches] +def find_assignment_target(node: parser.InstDef, idx: int) -> list[lexer.Token]: + """Find the tokens that make up the left-hand side of an assignment""" + offset = 0 + for tkn in reversed(node.block.tokens[: idx]): + if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + return node.block.tokens[idx - offset : idx] + offset += 1 + return [] + + +def find_stores_outputs(node: parser.InstDef) -> list[lexer.Token]: + res: list[lexer.Token] = [] + outnames = { out.name for out in node.outputs } + innames = { out.name for out in node.inputs } + for idx, tkn in enumerate(node.block.tokens): + if tkn.kind == "AND": + name = node.block.tokens[idx+1] + if name.text in outnames: + res.append(name) + if tkn.kind != "EQUALS": + continue + lhs = find_assignment_target(node, idx) + assert lhs + while lhs and lhs[0].kind == "COMMENT": + lhs = lhs[1:] + if len(lhs) != 1 or lhs[0].kind != "IDENTIFIER": + continue + name = lhs[0] + if name.text in innames: + raise analysis_error(f"Cannot assign to input variable '{name.text}'", name) + if name.text in outnames: + res.append(name) + return res + +def analyze_deferred_refs(node: parser.InstDef) -> dict[lexer.Token, str | None]: + """Look for PyStackRef_FromPyObjectNew() calls""" + + def in_frame_push(idx: int) -> bool: + for tkn in reversed(node.block.tokens[: idx - 1]): + if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + return False + if tkn.kind == "IDENTIFIER" and tkn.text == "_PyFrame_PushUnchecked": + return True + return False + + refs: dict[lexer.Token, str | None] = {} + for idx, tkn in enumerate(node.block.tokens): + if tkn.kind != "IDENTIFIER" or tkn.text != "PyStackRef_FromPyObjectNew": + continue + + if idx == 0 or node.block.tokens[idx - 1].kind != "EQUALS": + if in_frame_push(idx): + # PyStackRef_FromPyObjectNew() is called in _PyFrame_PushUnchecked() + refs[tkn] = None + continue + raise analysis_error("Expected '=' before PyStackRef_FromPyObjectNew", tkn) + + lhs = find_assignment_target(node, idx - 1) + if len(lhs) == 0: + raise analysis_error( + "PyStackRef_FromPyObjectNew() must be assigned to an output", tkn + ) + + if lhs[0].kind == "TIMES" or any( + t.kind == "ARROW" or t.kind == "LBRACKET" for t in lhs[1:] + ): + # Don't handle: *ptr = ..., ptr->field = ..., or ptr[field] = ... + # Assume that they are visible to the GC. + refs[tkn] = None + continue + + if len(lhs) != 1 or lhs[0].kind != "IDENTIFIER": + raise analysis_error( + "PyStackRef_FromPyObjectNew() must be assigned to an output", tkn + ) + + name = lhs[0].text + match = ( + any(var.name == name for var in node.inputs) + or any(var.name == name for var in node.outputs) + ) + if not match: + raise analysis_error( + f"PyStackRef_FromPyObjectNew() must be assigned to an input or output, not '{name}'", + tkn, + ) + + refs[tkn] = name + + return refs + + def variable_used(node: parser.InstDef, name: str) -> bool: """Determine whether a variable with a given name is used in a node.""" return any( - token.kind == "IDENTIFIER" and token.text == name for token in node.tokens + token.kind == "IDENTIFIER" and token.text == name for token in node.block.tokens + ) + + +def oparg_used(node: parser.InstDef) -> bool: + """Determine whether `oparg` is used in a node.""" + return any( + token.kind == "IDENTIFIER" and token.text == "oparg" for token in node.tokens ) -def is_infallible(op: parser.InstDef) -> bool: - return not ( +def tier_variable(node: parser.InstDef) -> int | None: + """Determine whether a tier variable is used in a node.""" + for token in node.tokens: + if token.kind == "ANNOTATION": + if token.text == "specializing": + return 1 + if re.fullmatch(r"tier\d", token.text): + return int(token.text[-1]) + return None + + +def has_error_with_pop(op: parser.InstDef) -> bool: + return ( variable_used(op, "ERROR_IF") - or variable_used(op, "error") + or variable_used(op, "pop_1_error") + or variable_used(op, "exception_unwind") + or variable_used(op, "resume_with_error") + ) + + +def has_error_without_pop(op: parser.InstDef) -> bool: + return ( + variable_used(op, "ERROR_NO_POP") or variable_used(op, "pop_1_error") or variable_used(op, "exception_unwind") or variable_used(op, "resume_with_error") @@ -318,78 +516,204 @@ def is_infallible(op: parser.InstDef) -> bool: NON_ESCAPING_FUNCTIONS = ( - "Py_INCREF", - "_PyDictOrValues_IsValues", - "_PyObject_DictOrValuesPointer", - "_PyDictOrValues_GetValues", - "_PyObject_MakeInstanceAttributesFromDict", - "Py_DECREF", - "_Py_DECREF_SPECIALIZED", - "DECREF_INPUTS_AND_REUSE_FLOAT", - "PyUnicode_Append", - "_PyLong_IsZero", - "Py_SIZE", - "Py_TYPE", + "PyCFunction_GET_FLAGS", + "PyCFunction_GET_FUNCTION", + "PyCFunction_GET_SELF", + "PyCell_GetRef", + "PyCell_New", + "PyCell_SwapTakeRef", + "PyExceptionInstance_Class", + "PyException_GetCause", + "PyException_GetContext", + "PyException_GetTraceback", + "PyFloat_AS_DOUBLE", + "PyFloat_FromDouble", + "PyFunction_GET_CODE", + "PyFunction_GET_GLOBALS", "PyList_GET_ITEM", - "PyTuple_GET_ITEM", "PyList_GET_SIZE", + "PyList_SET_ITEM", + "PyLong_AsLong", + "PyLong_FromLong", + "PyLong_FromSsize_t", + "PySlice_New", + "PyStackRef_AsPyObjectBorrow", + "PyStackRef_AsPyObjectNew", + "PyStackRef_AsPyObjectSteal", + "PyStackRef_CLEAR", + "PyStackRef_CLOSE", + "PyStackRef_CLOSE_SPECIALIZED", + "PyStackRef_DUP", + "PyStackRef_False", + "PyStackRef_FromPyObjectImmortal", + "PyStackRef_FromPyObjectNew", + "PyStackRef_FromPyObjectSteal", + "PyStackRef_Is", + "PyStackRef_IsNull", + "PyStackRef_None", + "PyStackRef_TYPE", + "PyStackRef_True", + "PyTuple_GET_ITEM", "PyTuple_GET_SIZE", + "PyType_HasFeature", + "PyUnicode_Append", + "PyUnicode_Concat", + "PyUnicode_GET_LENGTH", + "PyUnicode_READ_CHAR", "Py_ARRAY_LENGTH", + "Py_CLEAR", + "Py_DECREF", + "Py_FatalError", + "Py_INCREF", + "Py_IS_TYPE", + "Py_NewRef", + "Py_REFCNT", + "Py_SIZE", + "Py_TYPE", + "Py_UNREACHABLE", "Py_Unicode_GET_LENGTH", - "PyUnicode_READ_CHAR", - "_Py_SINGLETON", - "PyUnicode_GET_LENGTH", + "Py_XDECREF", + "_PyCode_CODE", + "_PyDictValues_AddToInsertionOrder", + "_PyErr_Occurred", + "_PyEval_FrameClearAndPop", + "_PyFloat_FromDouble_ConsumeInputs", + "_PyFrame_GetCode", + "_PyFrame_IsIncomplete", + "_PyFrame_PushUnchecked", + "_PyFrame_SetStackPointer", + "_PyFrame_StackPush", + "_PyFunction_SetVersion", + "_PyGen_GetGeneratorFromFrame", + "_PyInterpreterState_GET", + "_PyList_AppendTakeRef", + "_PyList_FromStackRefSteal", + "_PyList_ITEMS", + "_PyLong_Add", + "_PyLong_CompactValue", + "_PyLong_DigitCount", "_PyLong_IsCompact", "_PyLong_IsNonNegativeCompact", - "_PyLong_CompactValue", - "_Py_NewRef", - "_Py_IsImmortal", - "_Py_STR", - "_PyLong_Add", + "_PyLong_IsZero", "_PyLong_Multiply", "_PyLong_Subtract", - "Py_NewRef", - "_PyList_ITEMS", + "_PyManagedDictPointer_IsValues", + "_PyObject_GC_IS_TRACKED", + "_PyObject_GC_MAY_BE_TRACKED", + "_PyObject_GC_TRACK", + "_PyObject_GetManagedDict", + "_PyObject_InlineValues", + "_PyObject_ManagedDictPointer", + "_PyThreadState_HasStackSpace", + "_PyTuple_FromArraySteal", + "_PyTuple_FromStackRefSteal", "_PyTuple_ITEMS", - "_PyList_AppendTakeRef", + "_PyType_HasFeature", + "_PyType_NewManagedObject", + "_PyUnicode_Equal", + "_PyUnicode_JoinArray", + "_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY", + "_Py_DECREF_NO_DEALLOC", + "_Py_DECREF_SPECIALIZED", + "_Py_EnterRecursiveCallTstateUnchecked", + "_Py_ID", + "_Py_IsImmortal", + "_Py_LeaveRecursiveCallPy", + "_Py_LeaveRecursiveCallTstate", + "_Py_NewRef", + "_Py_SINGLETON", + "_Py_STR", "_Py_atomic_load_uintptr_relaxed", - "_PyFrame_GetCode", - "_PyThreadState_HasStackSpace", + "_Py_set_eval_breaker_bit", + "advance_backoff_counter", + "assert", + "backoff_counter_triggers", + "initial_temperature_backoff_counter", + "maybe_lltrace_resume_frame", + "restart_backoff_counter", ) -ESCAPING_FUNCTIONS = ( - "import_name", - "import_from", -) - - -def makes_escaping_api_call(instr: parser.InstDef) -> bool: - if "CALL_INTRINSIC" in instr.name: - return True - tkns = iter(instr.tokens) - for tkn in tkns: - if tkn.kind != lexer.IDENTIFIER: - continue +def find_stmt_start(node: parser.InstDef, idx: int) -> lexer.Token: + assert idx < len(node.block.tokens) + while True: + tkn = node.block.tokens[idx-1] + if tkn.kind in {"SEMI", "LBRACE", "RBRACE", "CMACRO"}: + break + idx -= 1 + assert idx > 0 + while node.block.tokens[idx].kind == "COMMENT": + idx += 1 + return node.block.tokens[idx] + + +def find_stmt_end(node: parser.InstDef, idx: int) -> lexer.Token: + assert idx < len(node.block.tokens) + while True: + idx += 1 + tkn = node.block.tokens[idx] + if tkn.kind == "SEMI": + return node.block.tokens[idx+1] + +def check_escaping_calls(instr: parser.InstDef, escapes: dict[lexer.Token, tuple[lexer.Token, lexer.Token]]) -> None: + calls = {escapes[t][0] for t in escapes} + in_if = 0 + tkn_iter = iter(instr.block.tokens) + for tkn in tkn_iter: + if tkn.kind == "IF": + next(tkn_iter) + in_if = 1 + if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF"): + next(tkn_iter) + in_if = 1 + elif tkn.kind == "LPAREN" and in_if: + in_if += 1 + elif tkn.kind == "RPAREN": + if in_if: + in_if -= 1 + elif tkn in calls and in_if: + raise analysis_error(f"Escaping call '{tkn.text} in condition", tkn) + +def find_escaping_api_calls(instr: parser.InstDef) -> dict[lexer.Token, tuple[lexer.Token, lexer.Token]]: + result: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] = {} + tokens = instr.block.tokens + for idx, tkn in enumerate(tokens): try: - next_tkn = next(tkns) - except StopIteration: - return False + next_tkn = tokens[idx+1] + except IndexError: + break + if tkn.kind == "SWITCH": + raise analysis_error(f"switch statements are not supported due to their complex flow control. Sorry.", tkn) if next_tkn.kind != lexer.LPAREN: continue - if tkn.text in ESCAPING_FUNCTIONS: - return True - if not tkn.text.startswith("Py") and not tkn.text.startswith("_Py"): - continue - if tkn.text.endswith("Check"): - continue - if tkn.text.startswith("Py_Is"): + if tkn.kind == lexer.IDENTIFIER: + if tkn.text.upper() == tkn.text: + # simple macro + continue + #if not tkn.text.startswith(("Py", "_Py", "monitor")): + # continue + if tkn.text.startswith(("sym_", "optimize_")): + # Optimize functions + continue + if tkn.text.endswith("Check"): + continue + if tkn.text.startswith("Py_Is"): + continue + if tkn.text.endswith("CheckExact"): + continue + if tkn.text in NON_ESCAPING_FUNCTIONS: + continue + elif tkn.kind == "RPAREN": + prev = tokens[idx-1] + if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": + #cast + continue + elif tkn.kind != "RBRACKET": continue - if tkn.text.endswith("CheckExact"): - continue - if tkn.text in NON_ESCAPING_FUNCTIONS: - continue - return True - return False + start = find_stmt_start(instr, idx) + end = find_stmt_end(instr, idx) + result[start] = tkn, end + check_escaping_calls(instr, result) + return result EXITS = { @@ -401,10 +725,6 @@ def makes_escaping_api_call(instr: parser.InstDef) -> bool: } -def eval_breaker_at_end(op: parser.InstDef) -> bool: - return op.tokens[-5].text == "CHECK_EVAL_BREAKER" - - def always_exits(op: parser.InstDef) -> bool: depth = 0 tkn_iter = iter(op.tokens) @@ -426,7 +746,7 @@ def always_exits(op: parser.InstDef) -> bool: if tkn.text == "DEOPT_IF" or tkn.text == "ERROR_IF": next(tkn_iter) # '(' t = next(tkn_iter) - if t.text == "true": + if t.text in ("true", "1"): return True return False @@ -445,48 +765,135 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: ) +OPARG_AND_1 = re.compile("\\(*oparg *& *1") + + +def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: + for effect in op.inputs: + if isinstance(effect, parser.CacheEffect): + continue + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + for effect in op.outputs: + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + return False + + def compute_properties(op: parser.InstDef) -> Properties: + escaping_calls = find_escaping_api_calls(op) has_free = ( variable_used(op, "PyCell_New") - or variable_used(op, "PyCell_GET") - or variable_used(op, "PyCell_SET") + or variable_used(op, "PyCell_GetRef") + or variable_used(op, "PyCell_SetTakeRef") + or variable_used(op, "PyCell_SwapTakeRef") ) - infallible = is_infallible(op) - deopts = variable_used(op, "DEOPT_IF") - passthrough = stack_effect_only_peeks(op) and infallible + deopts_if = variable_used(op, "DEOPT_IF") + exits_if = variable_used(op, "EXIT_IF") + if deopts_if and exits_if: + tkn = op.tokens[0] + raise lexer.make_syntax_error( + "Op cannot contain both EXIT_IF and DEOPT_IF", + tkn.filename, + tkn.line, + tkn.column, + op.name, + ) + error_with_pop = has_error_with_pop(op) + error_without_pop = has_error_without_pop(op) return Properties( - escapes=makes_escaping_api_call(op), - infallible=infallible, - deopts=deopts, - oparg=variable_used(op, "oparg"), + escaping_calls=escaping_calls, + error_with_pop=error_with_pop, + error_without_pop=error_without_pop, + deopts=deopts_if, + side_exit=exits_if, + oparg=oparg_used(op), jumps=variable_used(op, "JUMPBY"), - eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"), - ends_with_eval_breaker=eval_breaker_at_end(op), + eval_breaker="CHECK_PERIODIC" in op.name, needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), stores_sp=variable_used(op, "SYNC_SP"), - tier_one_only=variable_used(op, "TIER_ONE_ONLY"), uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), uses_co_names=variable_used(op, "FRAME_CO_NAMES"), uses_locals=(variable_used(op, "GETLOCAL") or variable_used(op, "SETLOCAL")) and not has_free, has_free=has_free, pure="pure" in op.annotations, - passthrough=passthrough, - guard=passthrough and deopts, + tier=tier_variable(op), + needs_prev=variable_used(op, "prev_instr"), ) -def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect]) -> Uop: - return Uop( +def make_uop( + name: str, + op: parser.InstDef, + inputs: list[parser.InputEffect], + uops: dict[str, Uop], +) -> Uop: + result = Uop( name=name, context=op.context, annotations=op.annotations, stack=analyze_stack(op), caches=analyze_caches(inputs), + deferred_refs=analyze_deferred_refs(op), + output_stores=find_stores_outputs(op), body=op.block.tokens, properties=compute_properties(op), ) + if effect_depends_on_oparg_1(op) and "split" in op.annotations: + result.properties.oparg_and_1 = True + for bit in ("0", "1"): + name_x = name + "_" + bit + properties = compute_properties(op) + if properties.oparg: + # May not need oparg anymore + properties.oparg = any( + token.text == "oparg" for token in op.block.tokens + ) + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op, bit), + caches=analyze_caches(inputs), + deferred_refs=analyze_deferred_refs(op), + output_stores=find_stores_outputs(op), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + for anno in op.annotations: + if anno.startswith("replicate"): + result.replicated = int(anno[10:-1]) + break + else: + return result + for oparg in range(result.replicated): + name_x = name + "_" + str(oparg) + properties = compute_properties(op) + properties.oparg = False + properties.const_oparg = oparg + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(inputs), + deferred_refs=analyze_deferred_refs(op), + output_stores=find_stores_outputs(op), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + + return result def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: @@ -496,13 +903,16 @@ def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: raise override_error( op.name, op.context, uops[op.name].context, op.tokens[0] ) - uops[op.name] = make_uop(op.name, op, op.inputs) + uops[op.name] = make_uop(op.name, op, op.inputs, uops) def add_instruction( - name: str, parts: list[Part], instructions: dict[str, Instruction] + where: lexer.Token, + name: str, + parts: list[Part], + instructions: dict[str, Instruction], ) -> None: - instructions[name] = Instruction(name, parts, None) + instructions[name] = Instruction(where, name, parts, None) def desugar_inst( @@ -523,32 +933,37 @@ def desugar_inst( uop_index = len(parts) # Place holder for the uop. parts.append(Skip(0)) - uop = make_uop("_" + inst.name, inst, op_inputs) + uop = make_uop("_" + inst.name, inst, op_inputs, uops) uop.implicitly_created = True uops[inst.name] = uop if uop_index < 0: parts.append(uop) else: parts[uop_index] = uop - add_instruction(name, parts, instructions) + add_instruction(inst.first_token, name, parts, instructions) def add_macro( macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] ) -> None: - parts: list[Uop | Skip] = [] + parts: list[Part] = [] for part in macro.uops: match part: case parser.OpName(): - if part.name not in uops: - analysis_error(f"No Uop named {part.name}", macro.tokens[0]) - parts.append(uops[part.name]) + if part.name == "flush": + parts.append(Flush()) + else: + if part.name not in uops: + raise analysis_error( + f"No Uop named {part.name}", macro.tokens[0] + ) + parts.append(uops[part.name]) case parser.CacheEffect(): parts.append(Skip(part.size)) case _: assert False assert parts - add_instruction(macro.name, parts, instructions) + add_instruction(macro.first_token, macro.name, parts, instructions) def add_family( @@ -575,7 +990,9 @@ def add_pseudo( ) -> None: pseudos[pseudo.name] = PseudoInstruction( pseudo.name, + analyze_stack(pseudo), [instructions[target] for target in pseudo.targets], + pseudo.as_sequence, pseudo.flags, ) @@ -603,15 +1020,10 @@ def assign_opcodes( instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 instmap["INSTRUMENTED_LINE"] = 254 + instmap["ENTER_EXECUTOR"] = 255 instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] - # Special case: this instruction is implemented in ceval.c - # rather than bytecodes.c, so we need to add it explicitly - # here (at least until we add something to bytecodes.c to - # declare external instructions). - instrumented.append("INSTRUMENTED_LINE") - specialized: set[str] = set() no_arg: list[str] = [] has_arg: list[str] = [] @@ -669,6 +1081,35 @@ def add_instruction(name: str) -> None: return instmap, len(no_arg), min_instrumented +def get_instruction_size_for_uop(instructions: dict[str, Instruction], uop: Uop) -> int | None: + """Return the size of the instruction that contains the given uop or + `None` if the uop does not contains the `INSTRUCTION_SIZE` macro. + + If there is more than one instruction that contains the uop, + ensure that they all have the same size. + """ + for tkn in uop.body: + if tkn.text == "INSTRUCTION_SIZE": + break + else: + return None + + size = None + for inst in instructions.values(): + if uop in inst.parts: + if size is None: + size = inst.size + if size != inst.size: + raise analysis_error( + "All instructions containing a uop with the `INSTRUCTION_SIZE` macro " + f"must have the same size: {size} != {inst.size}", + tkn + ) + if size is None: + raise analysis_error(f"No instruction containing the uop '{uop.name}' was found", tkn) + return size + + def analyze_forest(forest: list[parser.AstNode]) -> Analysis: instructions: dict[str, Instruction] = {} uops: dict[str, Uop] = {} @@ -712,6 +1153,8 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: continue if target.text in instructions: instructions[target.text].is_target = True + for uop in uops.values(): + uop.instruction_size = get_instruction_size_for_uop(instructions, uop) # Special case BINARY_OP_INPLACE_ADD_UNICODE # BINARY_OP_INPLACE_ADD_UNICODE is not a normal family member, # as it is the wrong size, but we need it to maintain an diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 069f0177a74018..8cba9127fa3071 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -18,8 +18,9 @@ def __init__(self, out: TextIO, indent: int, line_directives: bool): def set_position(self, tkn: Token) -> None: if self.last_token is not None: - if self.last_token.line < tkn.line: + if self.last_token.end_line < tkn.line: self.out.write("\n") + if self.last_token.line < tkn.line: if self.line_directives: self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') self.out.write(" " * self.indents[-1]) @@ -91,6 +92,8 @@ def emit_token(self, tkn: Token) -> None: self.maybe_dedent(tkn.text) self.set_position(tkn) self.emit_text(tkn.text) + if tkn.kind == "CMACRO": + self.newline = True self.maybe_indent(tkn.text) def emit_str(self, txt: str) -> None: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 2fc2ab115321cf..dad2557e97a948 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -4,28 +4,66 @@ from analyzer import ( Instruction, Uop, - analyze_files, Properties, - Skip, + StackItem, + analysis_error, ) from cwriter import CWriter -from typing import Callable, Mapping, TextIO, Iterator +from typing import Callable, TextIO, Iterator, Iterable from lexer import Token -from stack import StackOffset, Stack +from stack import Storage, StackError +# Set this to true for voluminous output showing state of stack and locals +PRINT_STACKS = False -ROOT = Path(__file__).parent.parent.parent -DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute().as_posix() +class TokenIterator: + + look_ahead: Token | None + iterator: Iterator[Token] + + def __init__(self, tkns: Iterable[Token]): + self.iterator = iter(tkns) + self.look_ahead = None + + def __iter__(self) -> "TokenIterator": + return self + + def __next__(self) -> Token: + if self.look_ahead is None: + return next(self.iterator) + else: + res = self.look_ahead + self.look_ahead = None + return res + + def peek(self) -> Token | None: + if self.look_ahead is None: + for tkn in self.iterator: + self.look_ahead = tkn + break + return self.look_ahead + +ROOT = Path(__file__).parent.parent.parent.resolve() +DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").as_posix() def root_relative_path(filename: str) -> str: try: - return Path(filename).absolute().relative_to(ROOT).as_posix() + return Path(filename).resolve().relative_to(ROOT).as_posix() except ValueError: # Not relative to root, just return original path. return filename +def type_and_null(var: StackItem) -> tuple[str, str]: + if var.type: + return var.type, "NULL" + elif var.is_array(): + return "_PyStackRef *", "NULL" + else: + return "_PyStackRef", "PyStackRef_NULL" + + def write_header( generator: str, sources: list[str], outfile: TextIO, comment: str = "//" ) -> None: @@ -38,153 +76,468 @@ def write_header( ) -def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: +def emit_to(out: CWriter, tkn_iter: TokenIterator, end: str) -> Token: parens = 0 for tkn in tkn_iter: if tkn.kind == end and parens == 0: - return + return tkn if tkn.kind == "LPAREN": parens += 1 if tkn.kind == "RPAREN": parens -= 1 out.emit(tkn) + raise analysis_error(f"Expecting {end}. Reached end of file", tkn) -def replace_deopt( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - unused: Stack, - inst: Instruction | None, -) -> None: - out.emit_at("DEOPT_IF", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon - out.emit(", ") - assert inst is not None - assert inst.family is not None - out.emit(inst.family.name) - out.emit(");\n") - - -def replace_error( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - out.emit_at("if ", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "COMMA") - label = next(tkn_iter).text - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - out.emit(") ") - c_offset = stack.peek_offset.to_c() - try: - offset = -int(c_offset) - close = ";\n" - except ValueError: - offset = None - out.emit(f"{{ stack_pointer += {c_offset}; ") - close = "; }\n" - out.emit("goto ") - if offset: - out.emit(f"pop_{offset}_") - out.emit(label) - out.emit(close) - - -def replace_decrefs( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - out.emit_at("", tkn) - for var in uop.stack.inputs: - if var.name == "unused" or var.name == "null" or var.peek: - continue - if var.size != "1": - out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"Py_DECREF({var.name}[_i]);\n") - out.emit("}\n") - elif var.condition: - out.emit(f"Py_XDECREF({var.name});\n") - else: - out.emit(f"Py_DECREF({var.name});\n") +ReplacementFunctionType = Callable[ + [Token, TokenIterator, Uop, Storage, Instruction | None], bool +] +def always_true(tkn: Token | None) -> bool: + if tkn is None: + return False + return tkn.text in {"true", "1"} -def replace_sync_sp( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - stack.flush(out) - - -def replace_check_eval_breaker( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - if not uop.properties.ends_with_eval_breaker: - out.emit_at("CHECK_EVAL_BREAKER();", tkn) +class Emitter: + out: CWriter + _replacers: dict[str, ReplacementFunctionType] -REPLACEMENT_FUNCTIONS = { - "DEOPT_IF": replace_deopt, - "ERROR_IF": replace_error, - "DECREF_INPUTS": replace_decrefs, - "CHECK_EVAL_BREAKER": replace_check_eval_breaker, - "SYNC_SP": replace_sync_sp, -} + def __init__(self, out: CWriter): + self._replacers = { + "EXIT_IF": self.exit_if, + "DEOPT_IF": self.deopt_if, + "ERROR_IF": self.error_if, + "ERROR_NO_POP": self.error_no_pop, + "DECREF_INPUTS": self.decref_inputs, + "DEAD": self.kill, + "INPUTS_DEAD": self.kill_inputs, + "SYNC_SP": self.sync_sp, + "SAVE_STACK": self.save_stack, + "RELOAD_STACK": self.reload_stack, + "PyStackRef_CLOSE": self.stackref_close, + "PyStackRef_CLOSE_SPECIALIZED": self.stackref_close, + "PyStackRef_AsPyObjectSteal": self.stackref_steal, + "DISPATCH": self.dispatch, + "INSTRUCTION_SIZE": self.instruction_size, + } + self.out = out -ReplacementFunctionType = Callable[ - [CWriter, Token, Iterator[Token], Uop, Stack, Instruction | None], None -] + def dispatch( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.emit(tkn) + return False + def deopt_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit_at("DEOPT_IF", tkn) + lparen = next(tkn_iter) + self.emit(lparen) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() + emit_to(self.out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + self.out.emit(", ") + assert inst is not None + assert inst.family is not None + self.out.emit(inst.family.name) + self.out.emit(");\n") + return not always_true(first_tkn) -def emit_tokens( - out: CWriter, - uop: Uop, - stack: Stack, - inst: Instruction | None, - replacement_functions: Mapping[ - str, ReplacementFunctionType - ] = REPLACEMENT_FUNCTIONS, -) -> None: - tkns = uop.body[1:-1] - if not tkns: - return - tkn_iter = iter(tkns) - out.start_line() - for tkn in tkn_iter: - if tkn.kind == "IDENTIFIER" and tkn.text in replacement_functions: - replacement_functions[tkn.text](out, tkn, tkn_iter, uop, stack, inst) + exit_if = deopt_if + + def error_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + lparen = next(tkn_iter) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() + unconditional = always_true(first_tkn) + if unconditional: + next(tkn_iter) + comma = next(tkn_iter) + if comma.kind != "COMMA": + raise analysis_error(f"Expected comma, got '{comma.text}'", comma) + self.out.start_line() + else: + self.out.emit_at("if ", tkn) + self.emit(lparen) + emit_to(self.out, tkn_iter, "COMMA") + self.out.emit(") ") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + storage.clear_inputs("at ERROR_IF") + c_offset = storage.stack.peek_offset() + try: + offset = -int(c_offset) + except ValueError: + offset = -1 + if offset > 0: + self.out.emit(f"goto pop_{offset}_") + self.out.emit(label) + self.out.emit(";\n") + elif offset == 0: + self.out.emit("goto ") + self.out.emit(label) + self.out.emit(";\n") else: - out.emit(tkn) + self.out.emit("{\n") + storage.copy().flush(self.out) + self.out.emit("goto ") + self.out.emit(label) + self.out.emit(";\n") + self.out.emit("}\n") + return not unconditional + + def error_no_pop( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + self.out.emit_at("goto error;", tkn) + return False + + def decref_inputs( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + self.out.emit_at("", tkn) + for var in uop.stack.inputs: + if var.name == "unused" or var.name == "null" or var.peek: + continue + if var.size: + if var.size == "1": + self.out.emit(f"PyStackRef_CLOSE({var.name}[0]);\n") + else: + self.out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + self.out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n") + self.out.emit("}\n") + elif var.condition: + if var.condition == "1": + self.out.emit(f"PyStackRef_CLOSE({var.name});\n") + elif var.condition != "0": + self.out.emit(f"PyStackRef_XCLOSE({var.name});\n") + else: + self.out.emit(f"PyStackRef_CLOSE({var.name});\n") + for input in storage.inputs: + input.defined = False + return True + + def kill_inputs( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + for var in storage.inputs: + var.defined = False + return True + + def kill( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + name_tkn = next(tkn_iter) + name = name_tkn.text + next(tkn_iter) + next(tkn_iter) + for var in storage.inputs: + if var.name == name: + var.defined = False + break + else: + raise analysis_error(f"'{name}' is not a live input-only variable", name_tkn) + return True + + def stackref_close( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit(tkn) + tkn = next(tkn_iter) + assert tkn.kind == "LPAREN" + self.out.emit(tkn) + name = next(tkn_iter) + self.out.emit(name) + if name.kind == "IDENTIFIER": + for var in storage.inputs: + if var.name == name.text: + var.defined = False + rparen = emit_to(self.out, tkn_iter, "RPAREN") + self.emit(rparen) + return True + + stackref_steal = stackref_close + + def sync_sp( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + storage.clear_inputs("when syncing stack") + storage.flush(self.out) + self._print_storage(storage) + return True + + def emit_save(self, storage: Storage) -> None: + storage.save(self.out) + self._print_storage(storage) + + def save_stack( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + self.emit_save(storage) + return True + + def emit_reload(self, storage: Storage) -> None: + storage.reload(self.out) + self._print_storage(storage) + + def reload_stack( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + self.emit_reload(storage) + return True + + def instruction_size(self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + """Replace the INSTRUCTION_SIZE macro with the size of the current instruction.""" + if uop.instruction_size is None: + raise analysis_error("The INSTRUCTION_SIZE macro requires uop.instruction_size to be set", tkn) + self.out.emit(f" {uop.instruction_size} ") + return True + + def _print_storage(self, storage: Storage) -> None: + if PRINT_STACKS: + self.out.start_line() + self.emit(storage.as_comment()) + self.out.start_line() + + def _emit_if( + self, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> tuple[bool, Token, Storage]: + """Returns (reachable?, closing '}', stack).""" + tkn = next(tkn_iter) + assert tkn.kind == "LPAREN" + self.out.emit(tkn) + rparen = emit_to(self.out, tkn_iter, "RPAREN") + self.emit(rparen) + if_storage = storage.copy() + reachable, rbrace, if_storage = self._emit_block(tkn_iter, uop, if_storage, inst, True) + try: + maybe_else = tkn_iter.peek() + if maybe_else and maybe_else.kind == "ELSE": + self._print_storage(storage) + self.emit(rbrace) + self.emit(next(tkn_iter)) + maybe_if = tkn_iter.peek() + if maybe_if and maybe_if.kind == "IF": + #Emit extra braces around the if to get scoping right + self.emit(" {\n") + self.emit(next(tkn_iter)) + else_reachable, rbrace, else_storage = self._emit_if(tkn_iter, uop, storage, inst) + self.out.start_line() + self.emit("}\n") + else: + else_reachable, rbrace, else_storage = self._emit_block(tkn_iter, uop, storage, inst, True) + if not reachable: + # Discard the if storage + reachable = else_reachable + storage = else_storage + elif not else_reachable: + # Discard the else storage + storage = if_storage + reachable = True + else: + if PRINT_STACKS: + self.emit("/* Merge */\n") + else_storage.merge(if_storage, self.out) + storage = else_storage + self._print_storage(storage) + else: + if reachable: + if PRINT_STACKS: + self.emit("/* Merge */\n") + if_storage.merge(storage, self.out) + storage = if_storage + self._print_storage(storage) + else: + # Discard the if storage + reachable = True + except StackError as ex: + self._print_storage(if_storage) + raise analysis_error(ex.args[0], rbrace) # from None + return reachable, rbrace, storage + + def _emit_block( + self, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + emit_first_brace: bool + ) -> tuple[bool, Token, Storage]: + """ Returns (reachable?, closing '}', stack).""" + braces = 1 + out_stores = set(uop.output_stores) + tkn = next(tkn_iter) + reload: Token | None = None + try: + reachable = True + line : int = -1 + if tkn.kind != "LBRACE": + raise analysis_error(f"PEP 7: expected '{{', found: {tkn.text}", tkn) + escaping_calls = uop.properties.escaping_calls + if emit_first_brace: + self.emit(tkn) + self._print_storage(storage) + for tkn in tkn_iter: + if PRINT_STACKS and tkn.line != line: + self.out.start_line() + self.emit(storage.as_comment()) + self.out.start_line() + line = tkn.line + if tkn in escaping_calls: + if tkn != reload: + self.emit_save(storage) + _, reload = escaping_calls[tkn] + elif tkn == reload: + self.emit_reload(storage) + if tkn.kind == "LBRACE": + self.out.emit(tkn) + braces += 1 + elif tkn.kind == "RBRACE": + self._print_storage(storage) + braces -= 1 + if braces == 0: + return reachable, tkn, storage + self.out.emit(tkn) + elif tkn.kind == "GOTO": + reachable = False; + self.out.emit(tkn) + elif tkn.kind == "IDENTIFIER": + if tkn.text in self._replacers: + if not self._replacers[tkn.text](tkn, tkn_iter, uop, storage, inst): + reachable = False + else: + if tkn in out_stores: + for out in storage.outputs: + if out.name == tkn.text: + out.defined = True + out.in_memory = False + break + if tkn.text.startswith("DISPATCH"): + self._print_storage(storage) + reachable = False + self.out.emit(tkn) + elif tkn.kind == "IF": + self.out.emit(tkn) + if_reachable, rbrace, storage = self._emit_if(tkn_iter, uop, storage, inst) + if reachable: + reachable = if_reachable + self.out.emit(rbrace) + else: + self.out.emit(tkn) + except StackError as ex: + raise analysis_error(ex.args[0], tkn) from None + raise analysis_error("Expecting closing brace. Reached end of file", tkn) + + + def emit_tokens( + self, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> Storage: + tkn_iter = TokenIterator(uop.body) + self.out.start_line() + _, rbrace, storage = self._emit_block(tkn_iter, uop, storage, inst, False) + try: + self._print_storage(storage) + storage.push_outputs() + self._print_storage(storage) + except StackError as ex: + raise analysis_error(ex.args[0], rbrace) + return storage + + def emit(self, txt: str | Token) -> None: + self.out.emit(txt) def cflags(p: Properties) -> str: @@ -205,14 +558,18 @@ def cflags(p: Properties) -> str: flags.append("HAS_EVAL_BREAK_FLAG") if p.deopts: flags.append("HAS_DEOPT_FLAG") + if p.side_exit: + flags.append("HAS_EXIT_FLAG") if not p.infallible: flags.append("HAS_ERROR_FLAG") + if p.error_without_pop: + flags.append("HAS_ERROR_NO_POP_FLAG") if p.escapes: flags.append("HAS_ESCAPES_FLAG") if p.pure: flags.append("HAS_PURE_FLAG") - if p.passthrough: - flags.append("HAS_PASSTHROUGH_FLAG") + if p.oparg_and_1: + flags.append("HAS_OPARG_AND_1_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index e87aff43762b11..203286834e3e3f 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -74,7 +74,7 @@ We update it as the need arises. ### Syntax Each op definition has a kind, a name, a stack and instruction stream effect, -and a piece of C code describing its semantics:: +and a piece of C code describing its semantics: ``` file: @@ -109,10 +109,7 @@ and a piece of C code describing its semantics:: NAME [":" type] [ "if" "(" C-expression ")" ] type: - NAME ["*"] | type_prop - - type_prop: - "&" "(" NAME ["+" NAME] ")" + NAME ["*"] stream: NAME "/" size @@ -127,7 +124,13 @@ and a piece of C code describing its semantics:: "family" "(" NAME ")" = "{" NAME ("," NAME)+ [","] "}" ";" pseudo: - "pseudo" "(" NAME ")" = "{" NAME ("," NAME)+ [","] "}" ";" + "pseudo" "(" NAME "," stack_effect ["," "(" flags ")"]")" = "{" NAME ("," NAME)+ [","] "}" ";" + + flags: + flag ("|" flag)* + + flag: + HAS_ARG | HAS_DEOPT | etc.. ``` The following definitions may occur: @@ -142,26 +145,7 @@ The following definitions may occur: The optional `type` in an `object` is the C type. It defaults to `PyObject *`. The objects before the "--" are the objects on top of the stack at the start of the instruction. Those after the "--" are the objects on top of the stack at the -end of the instruction. When prefixed by a `&`, the `type` production rule follows the -`type_prop` production rule. This indicates the type of the value is of that specific type -after the operation. In this case, the type may also contain 64-bit refinement information -that is fetched from a previously defined operand in the instruction header, such as -a type version tag. This follows the format `type + refinement`. The list of possible types -and their refinements are below. They obey the following predicates: - - -* `PYLONG_TYPE`: `Py_TYPE(val) == &PyLong_Type` -* `PYFLOAT_TYPE`: `Py_TYPE(val) == &PyFloat_Type` -* `PYUNICODE_TYPE`: `Py_TYPE(val) == &PYUNICODE_TYPE` -* `NULL_TYPE`: `val == NULL` -* `GUARD_TYPE_VERSION_TYPE`: `type->tp_version_tag == auxillary` -* `GUARD_DORV_VALUES_TYPE`: `_PyDictOrValues_IsValues(obj)` -* `GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_TYPE`: - `_PyDictOrValues_IsValues(obj) || _PyObject_MakeInstanceAttributesFromDict(obj, dorv)` -* `GUARD_KEYS_VERSION_TYPE`: `owner_heap_type->ht_cached_keys->dk_version == auxillary` -* `PYMETHOD_TYPE`: `Py_TYPE(val) == &PyMethod_Type` -* `PYFUNCTION_TYPE_VERSION_TYPE`: - `PyFunction_Check(callable) && func->func_version == auxillary && code->co_argcount == oparg + (self_or_null != NULL)` +end of the instruction. An `inst` without `stack_effect` is a transitional form to allow the original C code @@ -190,18 +174,21 @@ list of annotations and their meanings are as follows: * `override`. For external use by other interpreter definitions to override the current instruction definition. * `pure`. This instruction has no side effects. +* 'tierN'. This instruction only used by tier N interpreter. ### Special functions/macros -The C code may include special functions that are understood by the tools as +The C code may include special functions and macros that are understood by the tools as part of the DSL. -Those functions include: +Those include: * `DEOPT_IF(cond, instruction)`. Deoptimize if `cond` is met. * `ERROR_IF(cond, label)`. Jump to error handler at `label` if `cond` is true. * `DECREF_INPUTS()`. Generate `Py_DECREF()` calls for the input stack effects. * `SYNC_SP()`. Synchronizes the physical stack pointer with the stack effects. +* `INSTRUCTION_SIZE`. Replaced with the size of the instruction which is equal +to `1 + INLINE_CACHE_ENTRIES`. Note that the use of `DECREF_INPUTS()` is optional -- manual calls to `Py_DECREF()` or other approaches are also acceptable @@ -260,7 +247,8 @@ The same is true for all members of a pseudo instruction ## Examples -(Another source of examples can be found in the [tests](test_generator.py).) +(Another source of examples can be found in the +[tests](https://github.com/python/cpython/blob/main/Lib/test/test_generated_cases.py).) Some examples: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 4f8d01c5492f51..37f96398ff175f 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -79,7 +79,7 @@ def choice(*opts: str) -> str: opmap = {pattern.replace("\\", "") or "\\": op for op, pattern in operators.items()} # Macros -macro = r"# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)" +macro = r"#.*\n" CMACRO = "CMACRO" id_re = r"[a-zA-Z_][0-9a-zA-Z_]*" @@ -222,6 +222,10 @@ def choice(*opts: str) -> str: "register", "replaced", "pure", + "split", + "replicate", + "tier1", + "tier2", } __all__ = [] @@ -238,7 +242,7 @@ def make_syntax_error( return SyntaxError(message, (filename, line, column, line_text)) -@dataclass(slots=True) +@dataclass(slots=True, frozen=True) class Token: filename: str kind: str @@ -329,6 +333,9 @@ def tokenize(src: str, line: int = 1, filename: str = "") -> Iterator[Token]: line += newlines else: begin = line, start - linestart + if kind == CMACRO: + linestart = end + line += 1 if kind != "\n": yield Token( filename, kind, text, begin, (line, start - linestart + len(text)) diff --git a/Tools/cases_generator/opcode_id_generator.py b/Tools/cases_generator/opcode_id_generator.py index 5a3009a5c04c27..888ae0f3861eb6 100644 --- a/Tools/cases_generator/opcode_id_generator.py +++ b/Tools/cases_generator/opcode_id_generator.py @@ -4,12 +4,9 @@ """ import argparse -import os.path -import sys from analyzer import ( Analysis, - Instruction, analyze_files, ) from generators_common import ( @@ -40,6 +37,7 @@ def write_define(name: str, op: int) -> None: out.emit("\n") write_define("HAVE_ARGUMENT", analysis.have_arg) + write_define("MIN_SPECIALIZED_OPCODE", analysis.opmap["RESUME"]+1) write_define("MIN_INSTRUMENTED_OPCODE", analysis.min_instrumented) diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 3e9fa3e26daa53..2ad7604af9cc0d 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -4,14 +4,12 @@ """ import argparse -import os.path -import sys from analyzer import ( Analysis, Instruction, + PseudoInstruction, analyze_files, - Skip, Uop, ) from generators_common import ( @@ -19,7 +17,6 @@ ROOT, write_header, cflags, - StackOffset, ) from cwriter import CWriter from typing import TextIO @@ -50,8 +47,11 @@ "DEOPT", "ERROR", "ESCAPES", + "EXIT", "PURE", "PASSTHROUGH", + "OPARG_AND_1", + "ERROR_NO_POP", ] @@ -91,12 +91,19 @@ def emit_stack_effect_function( def generate_stack_effect_functions(analysis: Analysis, out: CWriter) -> None: popped_data: list[tuple[str, str]] = [] pushed_data: list[tuple[str, str]] = [] - for inst in analysis.instructions.values(): + + def add(inst: Instruction | PseudoInstruction) -> None: stack = get_stack_effect(inst) popped = (-stack.base_offset).to_c() pushed = (stack.top_offset - stack.base_offset).to_c() popped_data.append((inst.name, popped)) pushed_data.append((inst.name, pushed)) + + for inst in analysis.instructions.values(): + add(inst) + for pseudo in analysis.pseudos.values(): + add(pseudo) + emit_stack_effect_function(out, "popped", sorted(popped_data)) emit_stack_effect_function(out, "pushed", sorted(pushed_data)) @@ -145,7 +152,6 @@ def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: if inst.family is not None: deopt = inst.family.name deopts.append((inst.name, deopt)) - deopts.append(("INSTRUMENTED_LINE", "INSTRUMENTED_LINE")) for name, deopt in sorted(deopts): out.emit(f"[{name}] = {deopt},\n") out.emit("};\n\n") @@ -173,7 +179,6 @@ def generate_name_table(analysis: Analysis, out: CWriter) -> None: out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit(f"const char *_PyOpcode_OpName[{table_size}] = {{\n") names = list(analysis.instructions) + list(analysis.pseudos) - names.append("INSTRUMENTED_LINE") for name in sorted(names): out.emit(f'[{name}] = "{name}",\n') out.emit("};\n") @@ -282,7 +287,7 @@ def is_viable_expansion(inst: Instruction) -> bool: continue if "replaced" in part.annotations: continue - if part.properties.tier_one_only or not part.is_viable(): + if part.properties.tier == 1 or not part.is_viable(): return False return True @@ -300,6 +305,7 @@ def generate_pseudo_targets(analysis: Analysis, out: CWriter) -> None: table_size = len(analysis.pseudos) max_targets = max(len(pseudo.targets) for pseudo in analysis.pseudos.values()) out.emit("struct pseudo_targets {\n") + out.emit(f"uint8_t as_sequence;\n") out.emit(f"uint8_t targets[{max_targets + 1}];\n") out.emit("};\n") out.emit( @@ -310,10 +316,11 @@ def generate_pseudo_targets(analysis: Analysis, out: CWriter) -> None: f"const struct pseudo_targets _PyOpcode_PseudoTargets[{table_size}] = {{\n" ) for pseudo in analysis.pseudos.values(): + as_sequence = "1" if pseudo.as_sequence else "0" targets = ["0"] * (max_targets + 1) for i, target in enumerate(pseudo.targets): targets[i] = target.name - out.emit(f"[{pseudo.name}-256] = {{ {{ {', '.join(targets)} }} }},\n") + out.emit(f"[{pseudo.name}-256] = {{ {as_sequence}, {{ {', '.join(targets)} }} }},\n") out.emit("};\n\n") out.emit("#endif // NEED_OPCODE_METADATA\n") out.emit("static inline bool\n") diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py new file mode 100644 index 00000000000000..7a1dfe1b85bf1a --- /dev/null +++ b/Tools/cases_generator/optimizer_generator.py @@ -0,0 +1,248 @@ +"""Generate the cases for the tier 2 optimizer. +Reads the instruction definitions from bytecodes.c and optimizer_bytecodes.c +Writes the cases to optimizer_cases.c.h, which is #included in Python/optimizer_analysis.c. +""" + +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + analyze_files, + StackItem, + analysis_error, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + Emitter, + TokenIterator, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import Local, Stack, StackError, Storage + +DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" +DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_posix() + + +def validate_uop(override: Uop, uop: Uop) -> None: + # To do + pass + + +def type_name(var: StackItem) -> str: + if var.is_array(): + return f"_Py_UopsSymbol **" + if var.type: + return var.type + return f"_Py_UopsSymbol *" + + +def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: + variables = {"unused"} + if not skip_inputs: + for var in reversed(uop.stack.inputs): + if var.used and var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = NULL;\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + for var in uop.stack.outputs: + if var.peek: + continue + if var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = NULL;\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + + +def decref_inputs( + out: CWriter, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + + +def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: + for var in reversed(uop.stack.inputs): + stack.pop(var) + top_offset = stack.top_offset.copy() + for var in uop.stack.outputs: + if var.is_array() and not var.peek and not var.name == "unused": + c_offset = top_offset.to_c() + out.emit(f"{var.name} = &stack_pointer[{c_offset}];\n") + top_offset.push(var) + for var in uop.stack.outputs: + local = Local.undefined(var) + stack.push(local) + if var.name != "unused" and not var.peek: + local.defined = True + if var.is_array(): + if var.size == "1": + out.emit(f"{var.name}[0] = sym_new_not_null(ctx);\n") + else: + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") + out.emit("}\n") + elif var.name == "null": + out.emit(f"{var.name} = sym_new_null(ctx);\n") + else: + out.emit(f"{var.name} = sym_new_not_null(ctx);\n") + + +class OptimizerEmitter(Emitter): + + def emit_save(self, storage: Storage) -> None: + storage.flush(self.out) + + def emit_reload(self, storage: Storage) -> None: + pass + + +def write_uop( + override: Uop | None, + uop: Uop, + out: CWriter, + stack: Stack, + debug: bool, + skip_inputs: bool, +) -> None: + locals: dict[str, Local] = {} + prototype = override if override else uop + try: + out.start_line() + if override: + code_list, storage = Storage.for_uop(stack, prototype, extract_bits=False) + for code in code_list: + out.emit(code) + if debug: + args = [] + for input in prototype.stack.inputs: + if not input.peek or override: + args.append(input.name) + out.emit(f'DEBUG_PRINTF({", ".join(args)});\n') + if override: + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = cast = "PyObject *" + else: + type = f"uint{cache.size*16}_t " + cast = f"uint{cache.size*16}_t" + out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") + if override: + emitter = OptimizerEmitter(out) + # No reference management of inputs needed. + for var in storage.inputs: # type: ignore[possibly-undefined] + var.defined = False + storage = emitter.emit_tokens(override, storage, None) + out.start_line() + storage.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=False) + else: + emit_default(out, uop, stack) + out.start_line() + stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=False) + except StackError as ex: + raise analysis_error(ex.args[0], prototype.body[0]) # from None + + +SKIPS = ("_EXTENDED_ARG",) + + +def generate_abstract_interpreter( + filenames: list[str], + abstract: Analysis, + base: Analysis, + outfile: TextIO, + debug: bool, +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 2, False) + out.emit("\n") + base_uop_names = set([uop.name for uop in base.uops.values()]) + for abstract_uop_name in abstract.uops: + assert ( + abstract_uop_name in base_uop_names + ), f"All abstract uops should override base uops, but {abstract_uop_name} is not." + + for uop in base.uops.values(): + override: Uop | None = None + if uop.name in abstract.uops: + override = abstract.uops[uop.name] + validate_uop(override, uop) + if uop.properties.tier == 1: + continue + if uop.replicates: + continue + if uop.is_super(): + continue + if not uop.is_viable(): + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + continue + out.emit(f"case {uop.name}: {{\n") + if override: + declare_variables(override, out, skip_inputs=False) + else: + declare_variables(uop, out, skip_inputs=True) + stack = Stack() + write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) + out.start_line() + out.emit("break;\n") + out.emit("}") + out.emit("\n\n") + + +def generate_tier2_abstract_from_files( + filenames: list[str], outfilename: str, debug: bool = False +) -> None: + assert len(filenames) == 2, "Need a base file and an abstract cases file." + base = analyze_files([filenames[0]]) + abstract = analyze_files([filenames[1]]) + with open(outfilename, "w") as outfile: + generate_abstract_interpreter(filenames, abstract, base, outfile, debug) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the tier 2 interpreter.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + + +arg_parser.add_argument("input", nargs="*", help="Abstract interpreter definition file") + +arg_parser.add_argument( + "base", nargs="*", help="The base instruction definition file(s)" +) + +arg_parser.add_argument("-d", "--debug", help="Insert debug calls", action="store_true") + +if __name__ == "__main__": + args = arg_parser.parse_args() + if not args.input: + args.base.append(DEFAULT_INPUT) + args.input.append(DEFAULT_ABSTRACT_INPUT) + else: + args.base.append(args.input[-1]) + args.input.pop() + abstract = analyze_files(args.input) + base = analyze_files(args.base) + with open(args.output, "w") as outfile: + generate_abstract_interpreter(args.input, abstract, base, outfile, args.debug) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 2b77d14d21143f..db672ad5501f15 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -1,4 +1,4 @@ -from parsing import ( +from parsing import ( # noqa: F401 InstDef, Macro, Pseudo, diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 307919cb37ce1e..de31d9b232f9df 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -60,6 +60,12 @@ def tokens(self) -> list[lx.Token]: end = context.end return tokens[begin:end] + @property + def first_token(self) -> lx.Token: + context = self.context + assert context is not None + return context.owner.tokens[context.begin] + @dataclass class Block(Node): @@ -75,11 +81,6 @@ class StackEffect(Node): size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond - # Optional `(type, refinement)` - type_prop: None | tuple[str, None | str] = field( - default_factory=lambda: None, init=True, compare=False, hash=False - ) - def __repr__(self) -> str: items = [self.name, self.type, self.cond, self.size] while items and items[-1] == "": @@ -143,8 +144,11 @@ class Family(Node): @dataclass class Pseudo(Node): name: str + inputs: list[InputEffect] + outputs: list[OutputEffect] flags: list[str] # instr flags to set on the pseudo instruction targets: list[str] # opcodes this can be replaced by + as_sequence: bool AstNode = InstDef | Macro | Pseudo | Family @@ -184,7 +188,13 @@ def inst_header(self) -> InstHeader | None: # | annotation* op(NAME, (inputs -- outputs)) annotations = [] while anno := self.expect(lx.ANNOTATION): - annotations.append(anno.text) + if anno.text == "replicate": + self.require(lx.LPAREN) + times = self.require(lx.NUMBER) + self.require(lx.RPAREN) + annotations.append(f"replicate({times.text})") + else: + annotations.append(anno.text) tkn = self.expect(lx.INST) if not tkn: tkn = self.expect(lx.OP) @@ -260,25 +270,14 @@ def cache_effect(self) -> CacheEffect | None: @contextual def stack_effect(self) -> StackEffect | None: - # IDENTIFIER [':' [IDENTIFIER [TIMES]] ['&' '(' IDENTIFIER ['+' IDENTIFIER] ')']] ['if' '(' expression ')'] + # IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')'] # | IDENTIFIER '[' expression ']' if tkn := self.expect(lx.IDENTIFIER): type_text = "" - type_prop = None if self.expect(lx.COLON): - if i := self.expect(lx.IDENTIFIER): - type_text = i.text.strip() - if self.expect(lx.TIMES): - type_text += " *" - if self.expect(lx.AND): - consumed_bracket = self.expect(lx.LPAREN) is not None - type_prop_text = self.require(lx.IDENTIFIER).text.strip() - refinement = None - if self.expect(lx.PLUS): - refinement = self.require(lx.IDENTIFIER).text.strip() - type_prop = (type_prop_text, refinement) - if consumed_bracket: - self.require(lx.RPAREN) + type_text = self.require(lx.IDENTIFIER).text.strip() + if self.expect(lx.TIMES): + type_text += " *" cond_text = "" if self.expect(lx.IF): self.require(lx.LPAREN) @@ -293,9 +292,8 @@ def stack_effect(self) -> StackEffect | None: if not (size := self.expression()): raise self.make_syntax_error("Expected expression") self.require(lx.RBRACKET) - type_text = "PyObject **" size_text = size.text.strip() - return StackEffect(tkn.text, type_text, cond_text, size_text, type_prop) + return StackEffect(tkn.text, type_text, cond_text, size_text) return None @contextual @@ -419,19 +417,29 @@ def pseudo_def(self) -> Pseudo | None: if self.expect(lx.LPAREN): if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.COMMA): - flags = self.flags() - else: - flags = [] - if self.expect(lx.RPAREN): - if self.expect(lx.EQUALS): - if not self.expect(lx.LBRACE): - raise self.make_syntax_error("Expected {") - if members := self.members(): - if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Pseudo(tkn.text, flags, members) + inp, outp = self.io_effect() + if self.expect(lx.COMMA): + flags = self.flags() + else: + flags = [] + if self.expect(lx.RPAREN): + if self.expect(lx.EQUALS): + if self.expect(lx.LBRACE): + as_sequence = False + closing = lx.RBRACE + elif self.expect(lx.LBRACKET): + as_sequence = True + closing = lx.RBRACKET + else: + raise self.make_syntax_error("Expected { or [") + if members := self.members(allow_sequence=True): + if self.expect(closing) and self.expect(lx.SEMI): + return Pseudo( + tkn.text, inp, outp, flags, members, as_sequence + ) return None - def members(self) -> list[str] | None: + def members(self, allow_sequence : bool=False) -> list[str] | None: here = self.getpos() if tkn := self.expect(lx.IDENTIFIER): members = [tkn.text] @@ -441,8 +449,10 @@ def members(self) -> list[str] | None: else: break peek = self.peek() - if not peek or peek.kind != lx.RBRACE: - raise self.make_syntax_error("Expected comma or right paren") + kinds = [lx.RBRACE, lx.RBRACKET] if allow_sequence else [lx.RBRACE] + if not peek or peek.kind not in kinds: + raise self.make_syntax_error( + f"Expected comma or right paren{'/bracket' if allow_sequence else ''}") return members self.setpos(here) return None diff --git a/Tools/cases_generator/py_metadata_generator.py b/Tools/cases_generator/py_metadata_generator.py index 0dbcd599f9d4d9..3ec06faf338488 100644 --- a/Tools/cases_generator/py_metadata_generator.py +++ b/Tools/cases_generator/py_metadata_generator.py @@ -12,14 +12,12 @@ from generators_common import ( DEFAULT_INPUT, ROOT, - root_relative_path, write_header, ) from cwriter import CWriter from typing import TextIO - DEFAULT_OUTPUT = ROOT / "Lib/_opcode_metadata.py" diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index f62ece43c1be7f..a954bed4df073c 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -1,7 +1,8 @@ import re -from analyzer import StackItem, Instruction, Uop +from analyzer import StackItem, StackEffect, Instruction, Uop, PseudoInstruction from dataclasses import dataclass from cwriter import CWriter +from typing import Iterator UNUSED = {"unused"} @@ -23,13 +24,87 @@ def maybe_parenthesize(sym: str) -> str: def var_size(var: StackItem) -> str: if var.condition: - # Special case simplification - if var.condition == "oparg & 1" and var.size == "1": + # Special case simplifications + if var.condition == "0": + return "0" + elif var.condition == "1": + return var.get_size() + elif var.condition == "oparg & 1" and not var.size: return f"({var.condition})" else: - return f"(({var.condition}) ? {var.size} : 0)" - else: + return f"(({var.condition}) ? {var.get_size()} : 0)" + elif var.size: return var.size + else: + return "1" + + +@dataclass +class Local: + item: StackItem + cached: bool + in_memory: bool + defined: bool + + def __repr__(self) -> str: + return f"Local('{self.item.name}', mem={self.in_memory}, defined={self.defined}, array={self.is_array()})" + + def compact_str(self) -> str: + mtag = "M" if self.in_memory else "" + dtag = "D" if self.defined else "" + atag = "A" if self.is_array() else "" + return f"'{self.item.name}'{mtag}{dtag}{atag}" + + @staticmethod + def unused(defn: StackItem) -> "Local": + return Local(defn, False, defn.is_array(), False) + + @staticmethod + def undefined(defn: StackItem) -> "Local": + array = defn.is_array() + return Local(defn, not array, array, False) + + @staticmethod + def redefinition(var: StackItem, prev: "Local") -> "Local": + assert var.is_array() == prev.is_array() + return Local(var, prev.cached, prev.in_memory, True) + + @staticmethod + def from_memory(defn: StackItem) -> "Local": + return Local(defn, True, True, True) + + def copy(self) -> "Local": + return Local( + self.item, + self.cached, + self.in_memory, + self.defined + ) + + @property + def size(self) -> str: + return self.item.size + + @property + def name(self) -> str: + return self.item.name + + @property + def condition(self) -> str | None: + return self.item.condition + + def is_array(self) -> bool: + return self.item.is_array() + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Local): + return NotImplemented + return ( + self.item is other.item + and self.cached is other.cached + and self.in_memory is other.in_memory + and self.defined is other.defined + ) @dataclass @@ -43,6 +118,9 @@ class StackOffset: def empty() -> "StackOffset": return StackOffset([], []) + def copy(self) -> "StackOffset": + return StackOffset(self.popped[:], self.pushed[:]) + def pop(self, item: StackItem) -> None: self.popped.append(var_size(item)) @@ -57,7 +135,11 @@ def __neg__(self) -> "StackOffset": def simplify(self) -> None: "Remove matching values from both the popped and pushed list" - if not self.popped or not self.pushed: + if not self.popped: + self.pushed.sort() + return + if not self.pushed: + self.popped.sort() return # Sort the list so the lexically largest element is last. popped = sorted(self.popped) @@ -78,6 +160,8 @@ def simplify(self) -> None: popped.append(pop) self.popped.extend(popped) self.pushed.extend(pushed) + self.pushed.sort() + self.popped.sort() def to_c(self) -> str: self.simplify() @@ -103,106 +187,436 @@ def to_c(self) -> str: res = "-" + res[3:] return res + def as_int(self) -> int | None: + self.simplify() + int_offset = 0 + for item in self.popped: + try: + int_offset -= int(item) + except ValueError: + return None + for item in self.pushed: + try: + int_offset += int(item) + except ValueError: + return None + return int_offset + def clear(self) -> None: self.popped = [] self.pushed = [] + def __bool__(self) -> bool: + self.simplify() + return bool(self.popped) or bool(self.pushed) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, StackOffset): + return NotImplemented + return self.to_c() == other.to_c() + -class SizeMismatch(Exception): +class StackError(Exception): pass +def array_or_scalar(var: StackItem | Local) -> str: + return "array" if var.is_array() else "scalar" class Stack: def __init__(self) -> None: self.top_offset = StackOffset.empty() self.base_offset = StackOffset.empty() - self.peek_offset = StackOffset.empty() - self.variables: list[StackItem] = [] + self.variables: list[Local] = [] self.defined: set[str] = set() - def pop(self, var: StackItem) -> str: + def pop(self, var: StackItem, extract_bits: bool = True) -> tuple[str, Local]: self.top_offset.pop(var) - if not var.peek: - self.peek_offset.pop(var) indirect = "&" if var.is_array() else "" if self.variables: popped = self.variables.pop() - if popped.size != var.size: - raise SizeMismatch( - f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " - f"Expected {var.size} got {popped.size}" + if var.is_array() ^ popped.is_array(): + raise StackError( + f"Array mismatch when popping '{popped.name}' from stack to assign to '{var.name}'. " + f"Expected {array_or_scalar(var)} got {array_or_scalar(popped)}" ) - if popped.name == var.name: - return "" - elif popped.name in UNUSED: - self.defined.add(var.name) - return ( - f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" + if popped.size != var.size: + raise StackError( + f"Size mismatch when popping '{popped.name}' from stack to assign to '{var.name}'. " + f"Expected {var_size(var)} got {var_size(popped.item)}" ) - elif var.name in UNUSED: - return "" + if var.name in UNUSED: + if popped.name not in UNUSED and popped.name in self.defined: + raise StackError( + f"Value is declared unused, but is already cached by prior operation as '{popped.name}'" + ) + return "", popped + if not var.used: + return "", popped + self.defined.add(var.name) + if popped.defined: + if popped.name == var.name: + return "", popped + else: + defn = f"{var.name} = {popped.name};\n" else: - self.defined.add(var.name) - return f"{var.name} = {popped.name};\n" + if var.is_array(): + defn = f"{var.name} = &stack_pointer[{self.top_offset.to_c()}];\n" + else: + defn = f"{var.name} = stack_pointer[{self.top_offset.to_c()}];\n" + popped.in_memory = True + return defn, Local.redefinition(var, popped) + self.base_offset.pop(var) - if var.name in UNUSED: - return "" - else: - self.defined.add(var.name) + if var.name in UNUSED or not var.used: + return "", Local.unused(var) + self.defined.add(var.name) cast = f"({var.type})" if (not indirect and var.type) else "" - assign = ( - f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];" - ) + bits = ".bits" if cast and extract_bits else "" + assign = f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}]{bits};" if var.condition: - return f"if ({var.condition}) {{ {assign} }}\n" - return f"{assign}\n" + if var.condition == "1": + assign = f"{assign}\n" + elif var.condition == "0": + return "", Local.unused(var) + else: + assign = f"if ({var.condition}) {{ {assign} }}\n" + else: + assign = f"{assign}\n" + return assign, Local.from_memory(var) - def push(self, var: StackItem) -> str: + def push(self, var: Local) -> None: + assert(var not in self.variables) self.variables.append(var) - if var.is_array() and var.name not in self.defined and var.name not in UNUSED: - c_offset = self.top_offset.to_c() - self.top_offset.push(var) + self.top_offset.push(var.item) + if var.item.used: self.defined.add(var.name) - return f"{var.name} = &stack_pointer[{c_offset}];\n" - else: - self.top_offset.push(var) - return "" - def flush(self, out: CWriter) -> None: - out.start_line() - for var in self.variables: - if not var.peek: - cast = "(PyObject *)" if var.type else "" - if var.name not in UNUSED and not var.is_array(): - if var.condition: - out.emit(f"if ({var.condition}) ") - out.emit( - f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n" - ) - self.base_offset.push(var) - if self.base_offset.to_c() != self.top_offset.to_c(): - print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) - assert False - number = self.base_offset.to_c() + @staticmethod + def _do_emit( + out: CWriter, + var: StackItem, + base_offset: StackOffset, + cast_type: str = "uintptr_t", + extract_bits: bool = True, + ) -> None: + cast = f"({cast_type})" if var.type else "" + bits = ".bits" if cast and extract_bits else "" + if var.condition == "0": + return + if var.condition and var.condition != "1": + out.emit(f"if ({var.condition}) ") + out.emit(f"stack_pointer[{base_offset.to_c()}]{bits} = {cast}{var.name};\n") + + def _adjust_stack_pointer(self, out: CWriter, number: str) -> None: if number != "0": + out.start_line() out.emit(f"stack_pointer += {number};\n") - self.variables = [] - self.base_offset.clear() + out.emit("assert(WITHIN_STACK_BOUNDS());\n") + + def flush( + self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = True + ) -> None: + out.start_line() + var_offset = self.base_offset.copy() + for var in self.variables: + if ( + var.defined and + not var.in_memory + ): + Stack._do_emit(out, var.item, var_offset, cast_type, extract_bits) + var.in_memory = True + var_offset.push(var.item) + number = self.top_offset.to_c() + self._adjust_stack_pointer(out, number) + self.base_offset -= self.top_offset self.top_offset.clear() - self.peek_offset.clear() out.start_line() + def is_flushed(self) -> bool: + return not self.variables and not self.base_offset and not self.top_offset + + def peek_offset(self) -> str: + return self.top_offset.to_c() + def as_comment(self) -> str: - return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" + variables = ", ".join([v.compact_str() for v in self.variables]) + return ( + f"/* Variables: {variables}. base: {self.base_offset.to_c()}. top: {self.top_offset.to_c()} */" + ) + def copy(self) -> "Stack": + other = Stack() + other.top_offset = self.top_offset.copy() + other.base_offset = self.base_offset.copy() + other.variables = [var.copy() for var in self.variables] + other.defined = set(self.defined) + return other -def get_stack_effect(inst: Instruction) -> Stack: + def __eq__(self, other: object) -> bool: + if not isinstance(other, Stack): + return NotImplemented + return ( + self.top_offset == other.top_offset + and self.base_offset == other.base_offset + and self.variables == other.variables + ) + + def align(self, other: "Stack", out: CWriter) -> None: + if len(self.variables) != len(other.variables): + raise StackError("Cannot align stacks: differing variables") + if self.top_offset == other.top_offset: + return + diff = self.top_offset - other.top_offset + try: + self.top_offset -= diff + self.base_offset -= diff + self._adjust_stack_pointer(out, diff.to_c()) + except ValueError: + raise StackError("Cannot align stacks: cannot adjust stack pointer") + + def merge(self, other: "Stack", out: CWriter) -> None: + if len(self.variables) != len(other.variables): + raise StackError("Cannot merge stacks: differing variables") + for self_var, other_var in zip(self.variables, other.variables): + if self_var.name != other_var.name: + raise StackError(f"Mismatched variables on stack: {self_var.name} and {other_var.name}") + self_var.defined = self_var.defined and other_var.defined + self_var.in_memory = self_var.in_memory and other_var.in_memory + self.align(other, out) + + +def get_stack_effect(inst: Instruction | PseudoInstruction) -> Stack: stack = Stack() - for uop in inst.parts: - if not isinstance(uop, Uop): - continue - for var in reversed(uop.stack.inputs): - stack.pop(var) - for i, var in enumerate(uop.stack.outputs): - stack.push(var) + + def stacks(inst: Instruction | PseudoInstruction) -> Iterator[StackEffect]: + if isinstance(inst, Instruction): + for uop in inst.parts: + if isinstance(uop, Uop): + yield uop.stack + else: + assert isinstance(inst, PseudoInstruction) + yield inst.stack + + for s in stacks(inst): + locals: dict[str, Local] = {} + for var in reversed(s.inputs): + _, local = stack.pop(var) + if var.name != "unused": + locals[local.name] = local + for var in s.outputs: + if var.name in locals: + local = locals[var.name] + else: + local = Local.unused(var) + stack.push(local) return stack + +@dataclass +class Storage: + + stack: Stack + inputs: list[Local] + outputs: list[Local] + peeks: list[Local] + spilled: int = 0 + + @staticmethod + def needs_defining(var: Local) -> bool: + return ( + not var.defined and + not var.is_array() and + var.name != "unused" + ) + + @staticmethod + def is_live(var: Local) -> bool: + return ( + var.defined and + var.name != "unused" + ) + + def first_input_not_cleared(self) -> str: + for input in self.inputs: + if input.defined: + return input.name + return "" + + def clear_inputs(self, reason:str) -> None: + while self.inputs: + tos = self.inputs.pop() + if self.is_live(tos) and not tos.is_array(): + raise StackError( + f"Input '{tos.name}' is still live {reason}" + ) + self.stack.pop(tos.item) + + def clear_dead_inputs(self) -> None: + live = "" + while self.inputs: + tos = self.inputs[-1] + if self.is_live(tos): + live = tos.name + break + self.inputs.pop() + self.stack.pop(tos.item) + for var in self.inputs: + if not var.defined and not var.is_array() and var.name != "unused": + raise StackError( + f"Input '{var.name}' is not live, but '{live}' is" + ) + + def _push_defined_outputs(self) -> None: + defined_output = "" + for output in self.outputs: + if output.defined and not output.in_memory: + defined_output = output.name + if not defined_output: + return + self.clear_inputs(f"when output '{defined_output}' is defined") + undefined = "" + for out in self.outputs: + if out.defined: + if undefined: + f"Locals not defined in stack order. " + f"Expected '{undefined}' to be defined before '{out.name}'" + else: + undefined = out.name + while self.outputs and not self.needs_defining(self.outputs[0]): + out = self.outputs.pop(0) + self.stack.push(out) + + def locals_cached(self) -> bool: + for out in self.outputs: + if out.defined: + return True + return False + + def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = True) -> None: + self.clear_dead_inputs() + self._push_defined_outputs() + self.stack.flush(out, cast_type, extract_bits) + + def save(self, out: CWriter) -> None: + assert self.spilled >= 0 + if self.spilled == 0: + self.flush(out) + out.start_line() + out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + self.spilled += 1 + + def reload(self, out: CWriter) -> None: + if self.spilled == 0: + raise StackError("Cannot reload stack as it hasn't been saved") + assert self.spilled > 0 + self.spilled -= 1 + if self.spilled == 0: + out.start_line() + out.emit("stack_pointer = _PyFrame_GetStackPointer(frame);\n") + + @staticmethod + def for_uop(stack: Stack, uop: Uop, extract_bits: bool = True) -> tuple[list[str], "Storage"]: + code_list: list[str] = [] + inputs: list[Local] = [] + peeks: list[Local] = [] + for input in reversed(uop.stack.inputs): + code, local = stack.pop(input, extract_bits) + code_list.append(code) + if input.peek: + peeks.append(local) + else: + inputs.append(local) + inputs.reverse() + peeks.reverse() + for peek in peeks: + stack.push(peek) + top_offset = stack.top_offset.copy() + for ouput in uop.stack.outputs: + if ouput.is_array() and ouput.used and not ouput.peek: + c_offset = top_offset.to_c() + top_offset.push(ouput) + code_list.append(f"{ouput.name} = &stack_pointer[{c_offset}];\n") + else: + top_offset.push(ouput) + for var in inputs: + stack.push(var) + outputs = [ Local.undefined(var) for var in uop.stack.outputs if not var.peek ] + return code_list, Storage(stack, inputs, outputs, peeks) + + @staticmethod + def copy_list(arg: list[Local]) -> list[Local]: + return [ l.copy() for l in arg ] + + def copy(self) -> "Storage": + new_stack = self.stack.copy() + variables = { var.name: var for var in new_stack.variables } + inputs = [ variables[var.name] for var in self.inputs] + assert [v.name for v in inputs] == [v.name for v in self.inputs], (inputs, self.inputs) + return Storage( + new_stack, inputs, + self.copy_list(self.outputs), self.copy_list(self.peeks) + ) + + def sanity_check(self) -> None: + names: set[str] = set() + for var in self.inputs: + if var.name in names: + raise StackError(f"Duplicate name {var.name}") + names.add(var.name) + names = set() + for var in self.outputs: + if var.name in names: + raise StackError(f"Duplicate name {var.name}") + names.add(var.name) + names = set() + for var in self.stack.variables: + if var.name in names: + raise StackError(f"Duplicate name {var.name}") + names.add(var.name) + + def is_flushed(self) -> bool: + for var in self.outputs: + if var.defined and not var.in_memory: + return False + return self.stack.is_flushed() + + def merge(self, other: "Storage", out: CWriter) -> None: + self.sanity_check() + if len(self.inputs) != len(other.inputs): + self.clear_dead_inputs() + other.clear_dead_inputs() + if len(self.inputs) != len(other.inputs): + diff = self.inputs[-1] if len(self.inputs) > len(other.inputs) else other.inputs[-1] + raise StackError(f"Unmergeable inputs. Differing state of '{diff.name}'") + for var, other_var in zip(self.inputs, other.inputs): + if var.defined != other_var.defined: + raise StackError(f"'{var.name}' is cleared on some paths, but not all") + if len(self.outputs) != len(other.outputs): + self._push_defined_outputs() + other._push_defined_outputs() + if len(self.outputs) != len(other.outputs): + var = self.outputs[0] if len(self.outputs) > len(other.outputs) else other.outputs[0] + raise StackError(f"'{var.name}' is set on some paths, but not all") + self.stack.merge(other.stack, out) + self.sanity_check() + + def push_outputs(self) -> None: + if self.spilled: + raise StackError(f"Unbalanced stack spills") + self.clear_inputs("at the end of the micro-op") + if self.inputs: + raise StackError(f"Input variable '{self.inputs[-1].name}' is still live") + self._push_defined_outputs() + if self.outputs: + for out in self.outputs: + if self.needs_defining(out): + raise StackError(f"Output variable '{self.outputs[0].name}' is not defined") + self.stack.push(out) + self.outputs = [] + + def as_comment(self) -> str: + stack_comment = self.stack.as_comment() + next_line = "\n " + inputs = ", ".join([var.compact_str() for var in self.inputs]) + outputs = ", ".join([var.compact_str() for var in self.outputs]) + peeks = ", ".join([var.name for var in self.peeks]) + return f"{stack_comment[:-2]}{next_line}inputs: {inputs}{next_line}outputs: {outputs}{next_line}peeks: {peeks} */" diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 44a699c92bbd22..c5097b7584724c 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -14,7 +14,6 @@ ROOT, ) from cwriter import CWriter -from typing import TextIO DEFAULT_OUTPUT = ROOT / "Python/opcode_targets.h" @@ -31,6 +30,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: out.emit(target) out.emit("};\n") + arg_parser = argparse.ArgumentParser( description="Generate the file with dispatch targets.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index aba36ec74e5766..fcdd3bdacd0e7a 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -4,8 +4,6 @@ """ import argparse -import os.path -import sys from analyzer import ( Analysis, @@ -14,19 +12,20 @@ Part, analyze_files, Skip, - StackItem, + Flush, analysis_error, + StackItem, ) from generators_common import ( DEFAULT_INPUT, ROOT, write_header, - emit_tokens, + type_and_null, + Emitter, ) from cwriter import CWriter -from typing import TextIO, Iterator -from lexer import Token -from stack import StackOffset, Stack, SizeMismatch +from typing import TextIO +from stack import Local, Stack, StackError, get_stack_effect, Storage DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h" @@ -35,47 +34,63 @@ FOOTER = "#undef TIER_ONE\n" +def declare_variable(var: StackItem, out: CWriter) -> None: + type, null = type_and_null(var) + space = " " if type[-1].isalnum() else "" + if var.condition: + out.emit(f"{type}{space}{var.name} = {null};\n") + else: + out.emit(f"{type}{space}{var.name};\n") + + def declare_variables(inst: Instruction, out: CWriter) -> None: - variables = {"unused"} - for uop in inst.parts: - if isinstance(uop, Uop): - for var in reversed(uop.stack.inputs): - if var.name not in variables: - type = var.type if var.type else "PyObject *" - variables.add(var.name) - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") - for var in uop.stack.outputs: - if var.name not in variables: - variables.add(var.name) - type = var.type if var.type else "PyObject *" - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") + try: + stack = get_stack_effect(inst) + except StackError as ex: + raise analysis_error(ex.args[0], inst.where) from None + required = set(stack.defined) + required.discard("unused") + for part in inst.parts: + if not isinstance(part, Uop): + continue + for var in part.stack.inputs: + if var.name in required: + required.remove(var.name) + declare_variable(var, out) + for var in part.stack.outputs: + if var.name in required: + required.remove(var.name) + declare_variable(var, out) def write_uop( - uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool -) -> int: + uop: Part, + emitter: Emitter, + offset: int, + stack: Stack, + inst: Instruction, + braces: bool, +) -> tuple[int, Stack]: # out.emit(stack.as_comment() + "\n") if isinstance(uop, Skip): entries = "entries" if uop.size > 1 else "entry" - out.emit(f"/* Skip {uop.size} cache {entries} */\n") - return offset + uop.size + emitter.emit(f"/* Skip {uop.size} cache {entries} */\n") + return (offset + uop.size), stack + if isinstance(uop, Flush): + emitter.emit(f"// flush\n") + stack.flush(emitter.out) + return offset, stack try: - out.start_line() - if braces: - out.emit(f"// {uop.name}\n") - for var in reversed(uop.stack.inputs): - out.emit(stack.pop(var)) + locals: dict[str, Local] = {} + emitter.out.start_line() if braces: - out.emit("{\n") - if not uop.properties.stores_sp: - for i, var in enumerate(uop.stack.outputs): - out.emit(stack.push(var)) + emitter.out.emit(f"// {uop.name}\n") + emitter.emit("{\n") + code_list, storage = Storage.for_uop(stack, uop) + emitter._print_storage(storage) + for code in code_list: + emitter.emit(code) + for cache in uop.caches: if cache.name != "unused": if cache.size == 4: @@ -84,20 +99,20 @@ def write_uop( else: type = f"uint{cache.size*16}_t " reader = f"read_u{cache.size*16}" - out.emit( + emitter.emit( f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" ) + if inst.family is None: + emitter.emit(f"(void){cache.name};\n") offset += cache.size - emit_tokens(out, uop, stack, inst) - if uop.properties.stores_sp: - for i, var in enumerate(uop.stack.outputs): - out.emit(stack.push(var)) + + storage = emitter.emit_tokens(uop, storage, inst) if braces: - out.start_line() - out.emit("}\n") - # out.emit(stack.as_comment() + "\n") - return offset - except SizeMismatch as ex: + emitter.out.start_line() + emitter.emit("}\n") + # emitter.emit(stack.as_comment() + "\n") + return offset, storage.stack + except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) @@ -105,7 +120,7 @@ def uses_this(inst: Instruction) -> bool: if inst.properties.needs_this: return True for uop in inst.parts: - if isinstance(uop, Skip): + if not isinstance(uop, Uop): continue for cache in uop.caches: if cache.name != "unused": @@ -126,13 +141,18 @@ def generate_tier1( """ ) out = CWriter(outfile, 2, lines) + emitter = Emitter(out) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): needs_this = uses_this(inst) out.emit("\n") out.emit(f"TARGET({name}) {{\n") + unused_guard = "(void)this_instr;\n" if inst.family is None else "" + if inst.properties.needs_prev: + out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") if needs_this and not inst.is_target: - out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") + out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") + out.emit(unused_guard) else: out.emit(f"frame->instr_ptr = next_instr;\n") out.emit(f"next_instr += {inst.size};\n") @@ -140,7 +160,8 @@ def generate_tier1( if inst.is_target: out.emit(f"PREDICTED({name});\n") if needs_this: - out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n") + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") + out.emit(unused_guard) if inst.family is not None: out.emit( f"static_assert({inst.family.size} == {inst.size-1}" @@ -152,12 +173,11 @@ def generate_tier1( for part in inst.parts: # Only emit braces if more than one uop insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 - offset = write_uop(part, out, offset, stack, inst, insert_braces) + offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) out.start_line() + + stack.flush(out) if not inst.parts[-1].properties.always_exits: - stack.flush(out) - if inst.parts[-1].properties.ends_with_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();\n") out.emit("DISPATCH();\n") out.start_line() out.emit("}") diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 7897b89b2752a7..ce761495cca435 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -4,115 +4,183 @@ """ import argparse -import os.path -import sys from analyzer import ( Analysis, Instruction, Uop, - Part, analyze_files, - Skip, StackItem, analysis_error, ) from generators_common import ( DEFAULT_INPUT, ROOT, - write_header, - emit_tokens, emit_to, - REPLACEMENT_FUNCTIONS, + write_header, + type_and_null, + Emitter, + TokenIterator, + always_true, ) from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token -from stack import StackOffset, Stack, SizeMismatch +from stack import Local, Stack, StackError, Storage DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h" +def declare_variable( + var: StackItem, uop: Uop, required: set[str], out: CWriter +) -> None: + if not var.used or var.name not in required: + return + required.remove(var.name) + type, null = type_and_null(var) + space = " " if type[-1].isalnum() else "" + if var.condition: + out.emit(f"{type}{space}{var.name} = {null};\n") + if uop.replicates: + # Replicas may not use all their conditional variables + # So avoid a compiler warning with a fake use + out.emit(f"(void){var.name};\n") + else: + out.emit(f"{type}{space}{var.name};\n") + + def declare_variables(uop: Uop, out: CWriter) -> None: - variables = {"unused"} + stack = Stack() for var in reversed(uop.stack.inputs): - if var.name not in variables: - type = var.type if var.type else "PyObject *" - variables.add(var.name) - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") + stack.pop(var) for var in uop.stack.outputs: - if var.name not in variables: - variables.add(var.name) - type = var.type if var.type else "PyObject *" - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") - - -def tier2_replace_error( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - out.emit_at("if ", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "COMMA") - label = next(tkn_iter).text - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - out.emit(") ") - c_offset = stack.peek_offset.to_c() - try: - offset = -int(c_offset) - close = ";\n" - except ValueError: - offset = None - out.emit(f"{{ stack_pointer += {c_offset}; ") - close = "; }\n" - out.emit("goto ") - if offset: - out.emit(f"pop_{offset}_") - out.emit(label + "_tier_two") - out.emit(close) - - -def tier2_replace_deopt( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - unused: Stack, - inst: Instruction | None, -) -> None: - out.emit_at("if ", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon - out.emit(") goto deoptimize;\n") + stack.push(Local.undefined(var)) + required = set(stack.defined) + required.discard("unused") + for var in reversed(uop.stack.inputs): + declare_variable(var, uop, required, out) + for var in uop.stack.outputs: + declare_variable(var, uop, required, out) -TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() -TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error -TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt +class Tier2Emitter(Emitter): + def __init__(self, out: CWriter): + super().__init__(out) + self._replacers["oparg"] = self.oparg -def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: + def error_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit_at("if ", tkn) + lparen = next(tkn_iter) + self.emit(lparen) + assert lparen.kind == "LPAREN" + first_tkn = next(tkn_iter) + self.out.emit(first_tkn) + emit_to(self.out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + self.emit(") JUMP_TO_ERROR();\n") + return not always_true(first_tkn) + + + def error_no_pop( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + self.out.emit_at("JUMP_TO_ERROR();", tkn) + return False + + def deopt_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit_at("if ", tkn) + lparen = next(tkn_iter) + self.emit(lparen) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() + emit_to(self.out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + self.emit(") {\n") + self.emit("UOP_STAT_INC(uopcode, miss);\n") + self.emit("JUMP_TO_JUMP_TARGET();\n") + self.emit("}\n") + return not always_true(first_tkn) + + def exit_if( # type: ignore[override] + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit_at("if ", tkn) + lparen = next(tkn_iter) + self.emit(lparen) + first_tkn = tkn_iter.peek() + emit_to(self.out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + self.emit(") {\n") + self.emit("UOP_STAT_INC(uopcode, miss);\n") + self.emit("JUMP_TO_JUMP_TARGET();\n") + self.emit("}\n") + return not always_true(first_tkn) + + def oparg( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if not uop.name.endswith("_0") and not uop.name.endswith("_1"): + self.emit(tkn) + return True + amp = next(tkn_iter) + if amp.text != "&": + self.emit(tkn) + self.emit(amp) + return True + one = next(tkn_iter) + assert one.text == "1" + self.out.emit_at(uop.name[-1], tkn) + return True + + +def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: + locals: dict[str, Local] = {} try: - out.start_line() + emitter.out.start_line() if uop.properties.oparg: - out.emit("oparg = CURRENT_OPARG();\n") - for var in reversed(uop.stack.inputs): - out.emit(stack.pop(var)) - if not uop.properties.stores_sp: - for i, var in enumerate(uop.stack.outputs): - out.emit(stack.push(var)) + emitter.emit("oparg = CURRENT_OPARG();\n") + assert uop.properties.const_oparg < 0 + elif uop.properties.const_oparg >= 0: + emitter.emit(f"oparg = {uop.properties.const_oparg};\n") + emitter.emit(f"assert(oparg == CURRENT_OPARG());\n") + code_list, storage = Storage.for_uop(stack, uop) + for code in code_list: + emitter.emit(code) for cache in uop.caches: if cache.name != "unused": if cache.size == 4: @@ -120,14 +188,11 @@ def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: else: type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" - out.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") - emit_tokens(out, uop, stack, None, TIER2_REPLACEMENT_FUNCTIONS) - if uop.properties.stores_sp: - for i, var in enumerate(uop.stack.outputs): - out.emit(stack.push(var)) - except SizeMismatch as ex: - raise analysis_error(ex.args[0], uop.body[0]) - + emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") + storage = emitter.emit_tokens(uop, storage, None) + except StackError as ex: + raise analysis_error(ex.args[0], uop.body[0]) from None + return storage.stack SKIPS = ("_EXTENDED_ARG",) @@ -145,24 +210,29 @@ def generate_tier2( """ ) out = CWriter(outfile, 2, lines) + emitter = Tier2Emitter(out) out.emit("\n") for name, uop in analysis.uops.items(): - if uop.properties.tier_one_only: + if uop.properties.tier == 1: + continue + if uop.properties.oparg_and_1: + out.emit(f"/* {uop.name} is split on (oparg & 1) */\n\n") continue if uop.is_super(): continue - if not uop.is_viable(): - out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + why_not_viable = uop.why_not_viable() + if why_not_viable is not None: + out.emit( + f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n" + ) continue out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) stack = Stack() - write_uop(uop, out, stack) + stack = write_uop(uop, emitter, stack) out.start_line() if not uop.properties.always_exits: stack.flush(out) - if uop.properties.ends_with_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();\n") out.emit("break;\n") out.start_line() out.emit("}") diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py index 633249f1c6b1fe..aae89faaa928e1 100644 --- a/Tools/cases_generator/uop_id_generator.py +++ b/Tools/cases_generator/uop_id_generator.py @@ -4,12 +4,9 @@ """ import argparse -import os.path -import sys from analyzer import ( Analysis, - Instruction, analyze_files, ) from generators_common import ( @@ -38,15 +35,17 @@ def generate_uop_ids( next_id += 1 PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP"} - for uop in analysis.uops.values(): - if uop.name in PRE_DEFINED: + uops = [(uop.name, uop) for uop in analysis.uops.values()] + # Sort so that _BASE comes immediately before _BASE_0, etc. + for name, uop in sorted(uops): + if name in PRE_DEFINED: continue - if uop.properties.tier_one_only: + if uop.properties.tier == 1: continue - if uop.implicitly_created and not distinct_namespace: - out.emit(f"#define {uop.name} {uop.name[1:]}\n") + if uop.implicitly_created and not distinct_namespace and not uop.replicated: + out.emit(f"#define {name} {name[1:]}\n") else: - out.emit(f"#define {uop.name} {next_id}\n") + out.emit(f"#define {name} {next_id}\n") next_id += 1 out.emit(f"#define MAX_UOP_ID {next_id-1}\n") diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 9083ecc48bdf5b..7b3325ada4a49f 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -15,28 +15,50 @@ write_header, cflags, ) +from stack import Stack from cwriter import CWriter from typing import TextIO - DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h" def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") + out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): - if uop.is_viable() and not uop.properties.tier_one_only: + if uop.is_viable() and uop.properties.tier != 1: out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") + out.emit("};\n\n") + out.emit("const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = {\n") + for uop in analysis.uops.values(): + if uop.replicated: + out.emit(f"[{uop.name}] = {uop.replicated},\n") + out.emit("};\n\n") out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {\n") for uop in sorted(analysis.uops.values(), key=lambda t: t.name): - if uop.is_viable() and not uop.properties.tier_one_only: + if uop.is_viable() and uop.properties.tier != 1: out.emit(f'[{uop.name}] = "{uop.name}",\n') out.emit("};\n") + out.emit("int _PyUop_num_popped(int opcode, int oparg)\n{\n") + out.emit("switch(opcode) {\n") + for uop in analysis.uops.values(): + if uop.is_viable() and uop.properties.tier != 1: + stack = Stack() + for var in reversed(uop.stack.inputs): + stack.pop(var) + popped = (-stack.base_offset).to_c() + out.emit(f"case {uop.name}:\n") + out.emit(f" return {popped};\n") + out.emit("default:\n") + out.emit(" return -1;\n") + out.emit("}\n") + out.emit("}\n\n") out.emit("#endif // NEED_OPCODE_METADATA\n\n") diff --git a/Tools/ccbench/ccbench.py b/Tools/ccbench/ccbench.py deleted file mode 100644 index d52701a82948da..00000000000000 --- a/Tools/ccbench/ccbench.py +++ /dev/null @@ -1,606 +0,0 @@ -# This file should be kept compatible with both Python 2.6 and Python >= 3.0. - -from __future__ import division -from __future__ import print_function - -""" -ccbench, a Python concurrency benchmark. -""" - -import time -import os -import sys -import itertools -import threading -import subprocess -import socket -from optparse import OptionParser, SUPPRESS_HELP -import platform - -# Compatibility -try: - xrange -except NameError: - xrange = range - -try: - map = itertools.imap -except AttributeError: - pass - - -THROUGHPUT_DURATION = 2.0 - -LATENCY_PING_INTERVAL = 0.1 -LATENCY_DURATION = 2.0 - -BANDWIDTH_PACKET_SIZE = 1024 -BANDWIDTH_DURATION = 2.0 - - -def task_pidigits(): - """Pi calculation (Python)""" - _map = map - _count = itertools.count - _islice = itertools.islice - - def calc_ndigits(n): - # From http://shootout.alioth.debian.org/ - def gen_x(): - return _map(lambda k: (k, 4*k + 2, 0, 2*k + 1), _count(1)) - - def compose(a, b): - aq, ar, as_, at = a - bq, br, bs, bt = b - return (aq * bq, - aq * br + ar * bt, - as_ * bq + at * bs, - as_ * br + at * bt) - - def extract(z, j): - q, r, s, t = z - return (q*j + r) // (s*j + t) - - def pi_digits(): - z = (1, 0, 0, 1) - x = gen_x() - while 1: - y = extract(z, 3) - while y != extract(z, 4): - z = compose(z, next(x)) - y = extract(z, 3) - z = compose((10, -10*y, 0, 1), z) - yield y - - return list(_islice(pi_digits(), n)) - - return calc_ndigits, (50, ) - -def task_regex(): - """regular expression (C)""" - # XXX this task gives horrendous latency results. - import re - # Taken from the `inspect` module - pat = re.compile(r'^(\s*def\s)|(.*(? return the previous one. - if end_event: - return niters, duration - niters += step - duration = t2 - start_time - if duration >= min_duration: - end_event.append(None) - return niters, duration - if t2 - t1 < 0.01: - # Minimize interference of measurement on overall runtime - step = step * 3 // 2 - elif do_yield: - # OS scheduling of Python threads is sometimes so bad that we - # have to force thread switching ourselves, otherwise we get - # completely useless results. - _sleep(0.0001) - t1 = t2 - - -def run_throughput_test(func, args, nthreads): - assert nthreads >= 1 - - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - end_event = [] - - if nthreads == 1: - # Pure single-threaded performance, without any switching or - # synchronization overhead. - start_time = time.time() - results.append(loop(start_time, THROUGHPUT_DURATION, - end_event, do_yield=False)) - return results - - started = False - ready_cond = threading.Condition() - start_cond = threading.Condition() - ready = [] - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - results.append(loop(start_time, THROUGHPUT_DURATION, - end_event, do_yield=True)) - - threads = [] - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # We don't want measurements to include thread startup overhead, - # so we arrange for timing to start after all threads are ready. - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - with start_cond: - start_time = time.time() - started = True - start_cond.notify(nthreads) - for t in threads: - t.join() - - return results - -def run_throughput_tests(max_threads): - for task in throughput_tasks: - print(task.__doc__) - print() - func, args = task() - nthreads = 1 - baseline_speed = None - while nthreads <= max_threads: - results = run_throughput_test(func, args, nthreads) - # Taking the max duration rather than average gives pessimistic - # results rather than optimistic. - speed = sum(r[0] for r in results) / max(r[1] for r in results) - print("threads=%d: %d" % (nthreads, speed), end="") - if baseline_speed is None: - print(" iterations/s.") - baseline_speed = speed - else: - print(" ( %d %%)" % (speed / baseline_speed * 100)) - nthreads += 1 - print() - - -LAT_END = "END" - -def _sendto(sock, s, addr): - sock.sendto(s.encode('ascii'), addr) - -def _recv(sock, n): - return sock.recv(n).decode('ascii') - -def latency_client(addr, nb_pings, interval): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - _time = time.time - _sleep = time.sleep - def _ping(): - _sendto(sock, "%r\n" % _time(), addr) - # The first ping signals the parent process that we are ready. - _ping() - # We give the parent a bit of time to notice. - _sleep(1.0) - for i in range(nb_pings): - _sleep(interval) - _ping() - _sendto(sock, LAT_END + "\n", addr) - finally: - sock.close() - -def run_latency_client(**kwargs): - cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] - cmd_line.extend(['--latclient', repr(kwargs)]) - return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - -def run_latency_test(func, args, nthreads): - # Create a listening socket to receive the pings. We use UDP which should - # be painlessly cross-platform. - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 0)) - addr = sock.getsockname() - - interval = LATENCY_PING_INTERVAL - duration = LATENCY_DURATION - nb_pings = int(duration / interval) - - results = [] - threads = [] - end_event = [] - start_cond = threading.Condition() - started = False - if nthreads > 0: - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - ready = [] - ready_cond = threading.Condition() - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - loop(start_time, duration * 1.5, end_event, do_yield=False) - - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # Wait for threads to be ready - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - - # Run the client and wait for the first ping(s) to arrive before - # unblocking the background threads. - chunks = [] - process = run_latency_client(addr=sock.getsockname(), - nb_pings=nb_pings, interval=interval) - s = _recv(sock, 4096) - _time = time.time - - with start_cond: - start_time = _time() - started = True - start_cond.notify(nthreads) - - while LAT_END not in s: - s = _recv(sock, 4096) - t = _time() - chunks.append((t, s)) - - # Tell the background threads to stop. - end_event.append(None) - for t in threads: - t.join() - process.wait() - sock.close() - - for recv_time, chunk in chunks: - # NOTE: it is assumed that a line sent by a client wasn't received - # in two chunks because the lines are very small. - for line in chunk.splitlines(): - line = line.strip() - if line and line != LAT_END: - send_time = eval(line) - assert isinstance(send_time, float) - results.append((send_time, recv_time)) - - return results - -def run_latency_tests(max_threads): - for task in latency_tasks: - print("Background CPU task:", task.__doc__) - print() - func, args = task() - nthreads = 0 - while nthreads <= max_threads: - results = run_latency_test(func, args, nthreads) - n = len(results) - # We print out milliseconds - lats = [1000 * (t2 - t1) for (t1, t2) in results] - #print(list(map(int, lats))) - avg = sum(lats) / n - dev = (sum((x - avg) ** 2 for x in lats) / n) ** 0.5 - print("CPU threads=%d: %d ms. (std dev: %d ms.)" % (nthreads, avg, dev), end="") - print() - #print(" [... from %d samples]" % n) - nthreads += 1 - print() - - -BW_END = "END" - -def bandwidth_client(addr, packet_size, duration): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 0)) - local_addr = sock.getsockname() - _time = time.time - _sleep = time.sleep - def _send_chunk(msg): - _sendto(sock, ("%r#%s\n" % (local_addr, msg)).rjust(packet_size), addr) - # We give the parent some time to be ready. - _sleep(1.0) - try: - start_time = _time() - end_time = start_time + duration * 2.0 - i = 0 - while _time() < end_time: - _send_chunk(str(i)) - s = _recv(sock, packet_size) - assert len(s) == packet_size - i += 1 - _send_chunk(BW_END) - finally: - sock.close() - -def run_bandwidth_client(**kwargs): - cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] - cmd_line.extend(['--bwclient', repr(kwargs)]) - return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - -def run_bandwidth_test(func, args, nthreads): - # Create a listening socket to receive the packets. We use UDP which should - # be painlessly cross-platform. - with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: - sock.bind(("127.0.0.1", 0)) - addr = sock.getsockname() - - duration = BANDWIDTH_DURATION - packet_size = BANDWIDTH_PACKET_SIZE - - results = [] - threads = [] - end_event = [] - start_cond = threading.Condition() - started = False - if nthreads > 0: - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - ready = [] - ready_cond = threading.Condition() - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - loop(start_time, duration * 1.5, end_event, do_yield=False) - - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # Wait for threads to be ready - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - - # Run the client and wait for the first packet to arrive before - # unblocking the background threads. - process = run_bandwidth_client(addr=addr, - packet_size=packet_size, - duration=duration) - _time = time.time - # This will also wait for the parent to be ready - s = _recv(sock, packet_size) - remote_addr = eval(s.partition('#')[0]) - - with start_cond: - start_time = _time() - started = True - start_cond.notify(nthreads) - - n = 0 - first_time = None - while not end_event and BW_END not in s: - _sendto(sock, s, remote_addr) - s = _recv(sock, packet_size) - if first_time is None: - first_time = _time() - n += 1 - end_time = _time() - - end_event.append(None) - for t in threads: - t.join() - process.kill() - - return (n - 1) / (end_time - first_time) - -def run_bandwidth_tests(max_threads): - for task in bandwidth_tasks: - print("Background CPU task:", task.__doc__) - print() - func, args = task() - nthreads = 0 - baseline_speed = None - while nthreads <= max_threads: - results = run_bandwidth_test(func, args, nthreads) - speed = results - #speed = len(results) * 1.0 / results[-1][0] - print("CPU threads=%d: %.1f" % (nthreads, speed), end="") - if baseline_speed is None: - print(" packets/s.") - baseline_speed = speed - else: - print(" ( %d %%)" % (speed / baseline_speed * 100)) - nthreads += 1 - print() - - -def main(): - usage = "usage: %prog [-h|--help] [options]" - parser = OptionParser(usage=usage) - parser.add_option("-t", "--throughput", - action="store_true", dest="throughput", default=False, - help="run throughput tests") - parser.add_option("-l", "--latency", - action="store_true", dest="latency", default=False, - help="run latency tests") - parser.add_option("-b", "--bandwidth", - action="store_true", dest="bandwidth", default=False, - help="run I/O bandwidth tests") - parser.add_option("-i", "--interval", - action="store", type="int", dest="check_interval", default=None, - help="sys.setcheckinterval() value " - "(Python 3.8 and older)") - parser.add_option("-I", "--switch-interval", - action="store", type="float", dest="switch_interval", default=None, - help="sys.setswitchinterval() value " - "(Python 3.2 and newer)") - parser.add_option("-n", "--num-threads", - action="store", type="int", dest="nthreads", default=4, - help="max number of threads in tests") - - # Hidden option to run the pinging and bandwidth clients - parser.add_option("", "--latclient", - action="store", dest="latclient", default=None, - help=SUPPRESS_HELP) - parser.add_option("", "--bwclient", - action="store", dest="bwclient", default=None, - help=SUPPRESS_HELP) - - options, args = parser.parse_args() - if args: - parser.error("unexpected arguments") - - if options.latclient: - kwargs = eval(options.latclient) - latency_client(**kwargs) - return - - if options.bwclient: - kwargs = eval(options.bwclient) - bandwidth_client(**kwargs) - return - - if not options.throughput and not options.latency and not options.bandwidth: - options.throughput = options.latency = options.bandwidth = True - if options.check_interval: - sys.setcheckinterval(options.check_interval) - if options.switch_interval: - sys.setswitchinterval(options.switch_interval) - - print("== %s %s (%s) ==" % ( - platform.python_implementation(), - platform.python_version(), - platform.python_build()[0], - )) - # Processor identification often has repeated spaces - cpu = ' '.join(platform.processor().split()) - print("== %s %s on '%s' ==" % ( - platform.machine(), - platform.system(), - cpu, - )) - print() - - if options.throughput: - print("--- Throughput ---") - print() - run_throughput_tests(options.nthreads) - - if options.latency: - print("--- Latency ---") - print() - run_latency_tests(options.nthreads) - - if options.bandwidth: - print("--- I/O bandwidth ---") - print() - run_bandwidth_tests(options.nthreads) - -if __name__ == "__main__": - main() diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index db57d17899af93..362715d264620e 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4,6324 +4,7 @@ # Copyright 2012-2013 by Larry Hastings. # Licensed to the PSF under a contributor agreement. # -from __future__ import annotations - -import abc -import argparse -import ast -import builtins as bltns -import collections -import contextlib -import copy -import dataclasses as dc -import enum -import functools -import inspect -import io -import itertools -import os -import pprint -import re -import shlex -import sys -import textwrap - -from collections.abc import ( - Callable, - Iterable, - Iterator, - Sequence, -) -from operator import attrgetter -from types import FunctionType, NoneType -from typing import ( - TYPE_CHECKING, - Any, - Final, - Literal, - NamedTuple, - NoReturn, - Protocol, - TypeVar, - cast, - overload, -) - - -# Local imports. -import libclinic -import libclinic.cpp -from libclinic import ClinicError - - -# TODO: -# -# soon: -# -# * allow mixing any two of {positional-only, positional-or-keyword, -# keyword-only} -# * dict constructor uses positional-only and keyword-only -# * max and min use positional only with an optional group -# and keyword-only -# - - -# match '#define Py_LIMITED_API' -LIMITED_CAPI_REGEX = re.compile(r'#define +Py_LIMITED_API') - - -class Sentinels(enum.Enum): - unspecified = "unspecified" - unknown = "unknown" - - def __repr__(self) -> str: - return f"<{self.value.capitalize()}>" - - -unspecified: Final = Sentinels.unspecified -unknown: Final = Sentinels.unknown - - -# This one needs to be a distinct class, unlike the other two -class Null: - def __repr__(self) -> str: - return '' - - -NULL = Null() - -TemplateDict = dict[str, str] - - -@overload -def warn_or_fail( - *args: object, - fail: Literal[True], - filename: str | None = None, - line_number: int | None = None, -) -> NoReturn: ... - -@overload -def warn_or_fail( - *args: object, - fail: Literal[False] = False, - filename: str | None = None, - line_number: int | None = None, -) -> None: ... - -def warn_or_fail( - *args: object, - fail: bool = False, - filename: str | None = None, - line_number: int | None = None, -) -> None: - joined = " ".join([str(a) for a in args]) - if clinic: - if filename is None: - filename = clinic.filename - if getattr(clinic, 'block_parser', None) and (line_number is None): - line_number = clinic.block_parser.line_number - error = ClinicError(joined, filename=filename, lineno=line_number) - if fail: - raise error - else: - print(error.report(warn_only=True)) - - -def warn( - *args: object, - filename: str | None = None, - line_number: int | None = None, -) -> None: - return warn_or_fail(*args, filename=filename, line_number=line_number, fail=False) - -def fail( - *args: object, - filename: str | None = None, - line_number: int | None = None, -) -> NoReturn: - warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) - - -is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match - -def is_legal_py_identifier(s: str) -> bool: - return all(is_legal_c_identifier(field) for field in s.split('.')) - -# identifiers that are okay in Python but aren't a good idea in C. -# so if they're used Argument Clinic will add "_value" to the end -# of the name in C. -c_keywords = set(""" -asm auto break case char const continue default do double -else enum extern float for goto if inline int long -register return short signed sizeof static struct switch -typedef typeof union unsigned void volatile while -""".strip().split()) - -def ensure_legal_c_identifier(s: str) -> str: - # for now, just complain if what we're given isn't legal - if not is_legal_c_identifier(s): - fail("Illegal C identifier:", s) - # but if we picked a C keyword, pick something else - if s in c_keywords: - return s + "_value" - return s - - -def linear_format(s: str, **kwargs: str) -> str: - """ - Perform str.format-like substitution, except: - * The strings substituted must be on lines by - themselves. (This line is the "source line".) - * If the substitution text is empty, the source line - is removed in the output. - * If the field is not recognized, the original line - is passed unmodified through to the output. - * If the substitution text is not empty: - * Each line of the substituted text is indented - by the indent of the source line. - * A newline will be added to the end. - """ - lines = [] - for line in s.split('\n'): - indent, curly, trailing = line.partition('{') - if not curly: - lines.extend([line, "\n"]) - continue - - name, curly, trailing = trailing.partition('}') - if not curly or name not in kwargs: - lines.extend([line, "\n"]) - continue - - if trailing: - fail(f"Text found after {{{name}}} block marker! " - "It must be on a line by itself.") - if indent.strip(): - fail(f"Non-whitespace characters found before {{{name}}} block marker! " - "It must be on a line by itself.") - - value = kwargs[name] - if not value: - continue - - stripped = [line.rstrip() for line in value.split("\n")] - value = textwrap.indent("\n".join(stripped), indent) - lines.extend([value, "\n"]) - - return "".join(lines[:-1]) - - -class CRenderData: - def __init__(self) -> None: - - # The C statements to declare variables. - # Should be full lines with \n eol characters. - self.declarations: list[str] = [] - - # The C statements required to initialize the variables before the parse call. - # Should be full lines with \n eol characters. - self.initializers: list[str] = [] - - # The C statements needed to dynamically modify the values - # parsed by the parse call, before calling the impl. - self.modifications: list[str] = [] - - # The entries for the "keywords" array for PyArg_ParseTuple. - # Should be individual strings representing the names. - self.keywords: list[str] = [] - - # The "format units" for PyArg_ParseTuple. - # Should be individual strings that will get - self.format_units: list[str] = [] - - # The varargs arguments for PyArg_ParseTuple. - self.parse_arguments: list[str] = [] - - # The parameter declarations for the impl function. - self.impl_parameters: list[str] = [] - - # The arguments to the impl function at the time it's called. - self.impl_arguments: list[str] = [] - - # For return converters: the name of the variable that - # should receive the value returned by the impl. - self.return_value = "return_value" - - # For return converters: the code to convert the return - # value from the parse function. This is also where - # you should check the _return_value for errors, and - # "goto exit" if there are any. - self.return_conversion: list[str] = [] - self.converter_retval = "_return_value" - - # The C statements required to do some operations - # after the end of parsing but before cleaning up. - # These operations may be, for example, memory deallocations which - # can only be done without any error happening during argument parsing. - self.post_parsing: list[str] = [] - - # The C statements required to clean up after the impl call. - self.cleanup: list[str] = [] - - # The C statements to generate critical sections (per-object locking). - self.lock: list[str] = [] - self.unlock: list[str] = [] - - -class Language(metaclass=abc.ABCMeta): - - start_line = "" - body_prefix = "" - stop_line = "" - checksum_line = "" - - def __init__(self, filename: str) -> None: - ... - - @abc.abstractmethod - def render( - self, - clinic: Clinic | None, - signatures: Iterable[Module | Class | Function] - ) -> str: - ... - - def parse_line(self, line: str) -> None: - ... - - def validate(self) -> None: - def assert_only_one( - attr: str, - *additional_fields: str - ) -> None: - """ - Ensures that the string found at getattr(self, attr) - contains exactly one formatter replacement string for - each valid field. The list of valid fields is - ['dsl_name'] extended by additional_fields. - - e.g. - self.fmt = "{dsl_name} {a} {b}" - - # this passes - self.assert_only_one('fmt', 'a', 'b') - - # this fails, the format string has a {b} in it - self.assert_only_one('fmt', 'a') - - # this fails, the format string doesn't have a {c} in it - self.assert_only_one('fmt', 'a', 'b', 'c') - - # this fails, the format string has two {a}s in it, - # it must contain exactly one - self.fmt2 = '{dsl_name} {a} {a}' - self.assert_only_one('fmt2', 'a') - - """ - fields = ['dsl_name'] - fields.extend(additional_fields) - line: str = getattr(self, attr) - fcf = libclinic.FormatCounterFormatter() - fcf.format(line) - def local_fail(should_be_there_but_isnt: bool) -> None: - if should_be_there_but_isnt: - fail("{} {} must contain {{{}}} exactly once!".format( - self.__class__.__name__, attr, name)) - else: - fail("{} {} must not contain {{{}}}!".format( - self.__class__.__name__, attr, name)) - - for name, count in fcf.counts.items(): - if name in fields: - if count > 1: - local_fail(True) - else: - local_fail(False) - for name in fields: - if fcf.counts.get(name) != 1: - local_fail(True) - - assert_only_one('start_line') - assert_only_one('stop_line') - - field = "arguments" if "{arguments}" in self.checksum_line else "checksum" - assert_only_one('checksum_line', field) - - - -class PythonLanguage(Language): - - language = 'Python' - start_line = "#/*[{dsl_name} input]" - body_prefix = "#" - stop_line = "#[{dsl_name} start generated code]*/" - checksum_line = "#/*[{dsl_name} end generated code: {arguments}]*/" - - -ParamTuple = tuple["Parameter", ...] - - -def permute_left_option_groups( - l: Sequence[Iterable[Parameter]] -) -> Iterator[ParamTuple]: - """ - Given [(1,), (2,), (3,)], should yield: - () - (3,) - (2, 3) - (1, 2, 3) - """ - yield tuple() - accumulator: list[Parameter] = [] - for group in reversed(l): - accumulator = list(group) + accumulator - yield tuple(accumulator) - - -def permute_right_option_groups( - l: Sequence[Iterable[Parameter]] -) -> Iterator[ParamTuple]: - """ - Given [(1,), (2,), (3,)], should yield: - () - (1,) - (1, 2) - (1, 2, 3) - """ - yield tuple() - accumulator: list[Parameter] = [] - for group in l: - accumulator.extend(group) - yield tuple(accumulator) - - -def permute_optional_groups( - left: Sequence[Iterable[Parameter]], - required: Iterable[Parameter], - right: Sequence[Iterable[Parameter]] -) -> tuple[ParamTuple, ...]: - """ - Generator function that computes the set of acceptable - argument lists for the provided iterables of - argument groups. (Actually it generates a tuple of tuples.) - - Algorithm: prefer left options over right options. - - If required is empty, left must also be empty. - """ - required = tuple(required) - if not required: - if left: - raise ValueError("required is empty but left is not") - - accumulator: list[ParamTuple] = [] - counts = set() - for r in permute_right_option_groups(right): - for l in permute_left_option_groups(left): - t = l + required + r - if len(t) in counts: - continue - counts.add(len(t)) - accumulator.append(t) - - accumulator.sort(key=len) - return tuple(accumulator) - - -def declare_parser( - f: Function, - *, - hasformat: bool = False, - clinic: Clinic, - limited_capi: bool, -) -> str: - """ - Generates the code template for a static local PyArg_Parser variable, - with an initializer. For core code (incl. builtin modules) the - kwtuple field is also statically initialized. Otherwise - it is initialized at runtime. - """ - if hasformat: - fname = '' - format_ = '.format = "{format_units}:{name}",' - else: - fname = '.fname = "{name}",' - format_ = '' - - num_keywords = len([ - p for p in f.parameters.values() - if not p.is_positional_only() and not p.is_vararg() - ]) - if limited_capi: - declarations = """ - #define KWTUPLE NULL - """ - elif num_keywords == 0: - declarations = """ - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - """ - else: - declarations = """ - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS %d - static struct {{ - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - }} _kwtuple = {{ - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = {{ {keywords_py} }}, - }}; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - """ % num_keywords - - condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' - clinic.add_include('pycore_gc.h', 'PyGC_Head', condition=condition) - clinic.add_include('pycore_runtime.h', '_Py_ID()', condition=condition) - - declarations += """ - static const char * const _keywords[] = {{{keywords_c} NULL}}; - static _PyArg_Parser _parser = {{ - .keywords = _keywords, - %s - .kwtuple = KWTUPLE, - }}; - #undef KWTUPLE - """ % (format_ or fname) - return libclinic.normalize_snippet(declarations) - - -class CLanguage(Language): - - body_prefix = "#" - language = 'C' - start_line = "/*[{dsl_name} input]" - body_prefix = "" - stop_line = "[{dsl_name} start generated code]*/" - checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" - - NO_VARARG: Final[str] = "PY_SSIZE_T_MAX" - - PARSER_PROTOTYPE_KEYWORD: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) - PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = libclinic.normalize_snippet(""" - static int - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) - PARSER_PROTOTYPE_VARARGS: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args) - """) - PARSER_PROTOTYPE_FASTCALL: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) - """) - PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - PARSER_PROTOTYPE_DEF_CLASS: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - PARSER_PROTOTYPE_NOARGS: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) - """) - PARSER_PROTOTYPE_GETTER: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, void *Py_UNUSED(context)) - """) - PARSER_PROTOTYPE_SETTER: Final[str] = libclinic.normalize_snippet(""" - static int - {c_basename}({self_type}{self_name}, PyObject *value, void *Py_UNUSED(context)) - """) - METH_O_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({impl_parameters}) - """) - DOCSTRING_PROTOTYPE_VAR: Final[str] = libclinic.normalize_snippet(""" - PyDoc_VAR({c_basename}__doc__); - """) - DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" - PyDoc_STRVAR({c_basename}__doc__, - {docstring}); - """) - GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" - PyDoc_STRVAR({getset_basename}__doc__, - {docstring}); - #define {getset_basename}_HAS_DOCSTR - """) - IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" - static {impl_return_type} - {c_basename}_impl({impl_parameters}) - """) - METHODDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #define {methoddef_name} \ - {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, - """) - GETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #if defined({getset_basename}_HAS_DOCSTR) - # define {getset_basename}_DOCSTR {getset_basename}__doc__ - #else - # define {getset_basename}_DOCSTR NULL - #endif - #if defined({getset_name}_GETSETDEF) - # undef {getset_name}_GETSETDEF - # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, - #else - # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, NULL, {getset_basename}_DOCSTR}}, - #endif - """) - SETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #if defined({getset_name}_HAS_DOCSTR) - # define {getset_basename}_DOCSTR {getset_basename}__doc__ - #else - # define {getset_basename}_DOCSTR NULL - #endif - #if defined({getset_name}_GETSETDEF) - # undef {getset_name}_GETSETDEF - # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, - #else - # define {getset_name}_GETSETDEF {{"{name}", NULL, (setter){getset_basename}_set, NULL}}, - #endif - """) - METHODDEF_PROTOTYPE_IFNDEF: Final[str] = libclinic.normalize_snippet(""" - #ifndef {methoddef_name} - #define {methoddef_name} - #endif /* !defined({methoddef_name}) */ - """) - COMPILER_DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" - // Emit compiler warnings when we get to Python {major}.{minor}. - #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 - # error {message} - #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 - # ifdef _MSC_VER - # pragma message ({message}) - # else - # warning {message} - # endif - #endif - """ - DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" - if ({condition}) {{{{{errcheck} - if (PyErr_WarnEx(PyExc_DeprecationWarning, - {message}, 1)) - {{{{ - goto exit; - }}}} - }}}} - """ - - def __init__(self, filename: str) -> None: - super().__init__(filename) - self.cpp = libclinic.cpp.Monitor(filename) - - def parse_line(self, line: str) -> None: - self.cpp.writeline(line) - - def render( - self, - clinic: Clinic | None, - signatures: Iterable[Module | Class | Function] - ) -> str: - function = None - for o in signatures: - if isinstance(o, Function): - if function: - fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) - function = o - return self.render_function(clinic, function) - - def compiler_deprecated_warning( - self, - func: Function, - parameters: list[Parameter], - ) -> str | None: - minversion: VersionTuple | None = None - for p in parameters: - for version in p.deprecated_positional, p.deprecated_keyword: - if version and (not minversion or minversion > version): - minversion = version - if not minversion: - return None - - # Format the preprocessor warning and error messages. - assert isinstance(self.cpp.filename, str) - message = f"Update the clinic input of {func.full_name!r}." - code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( - major=minversion[0], - minor=minversion[1], - message=libclinic.c_repr(message), - ) - return libclinic.normalize_snippet(code) - - def deprecate_positional_use( - self, - func: Function, - params: dict[int, Parameter], - ) -> str: - assert len(params) > 0 - first_pos = next(iter(params)) - last_pos = next(reversed(params)) - - # Format the deprecation message. - if len(params) == 1: - condition = f"nargs == {first_pos+1}" - amount = f"{first_pos+1} " if first_pos else "" - pl = "s" - else: - condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" - amount = f"more than {first_pos} " if first_pos else "" - pl = "s" if first_pos != 1 else "" - message = ( - f"Passing {amount}positional argument{pl} to " - f"{func.fulldisplayname}() is deprecated." - ) - - for (major, minor), group in itertools.groupby( - params.values(), key=attrgetter("deprecated_positional") - ): - names = [repr(p.name) for p in group] - pstr = libclinic.pprint_words(names) - if len(names) == 1: - message += ( - f" Parameter {pstr} will become a keyword-only parameter " - f"in Python {major}.{minor}." - ) - else: - message += ( - f" Parameters {pstr} will become keyword-only parameters " - f"in Python {major}.{minor}." - ) - - # Append deprecation warning to docstring. - docstring = textwrap.fill(f"Note: {message}") - func.docstring += f"\n\n{docstring}\n" - # Format and return the code block. - code = self.DEPRECATION_WARNING_PROTOTYPE.format( - condition=condition, - errcheck="", - message=libclinic.wrapped_c_string_literal(message, width=64, - subsequent_indent=20), - ) - return libclinic.normalize_snippet(code, indent=4) - - def deprecate_keyword_use( - self, - func: Function, - params: dict[int, Parameter], - argname_fmt: str | None, - *, - fastcall: bool, - limited_capi: bool, - clinic: Clinic, - ) -> str: - assert len(params) > 0 - last_param = next(reversed(params.values())) - - # Format the deprecation message. - containscheck = "" - conditions = [] - for i, p in params.items(): - if p.is_optional(): - if argname_fmt: - conditions.append(f"nargs < {i+1} && {argname_fmt % i}") - elif fastcall: - conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))") - containscheck = "PySequence_Contains" - clinic.add_include('pycore_runtime.h', '_Py_ID()') - else: - conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") - containscheck = "PyDict_Contains" - clinic.add_include('pycore_runtime.h', '_Py_ID()') - else: - conditions = [f"nargs < {i+1}"] - condition = ") || (".join(conditions) - if len(conditions) > 1: - condition = f"(({condition}))" - if last_param.is_optional(): - if fastcall: - if limited_capi: - condition = f"kwnames && PyTuple_Size(kwnames) && {condition}" - else: - condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" - else: - if limited_capi: - condition = f"kwargs && PyDict_Size(kwargs) && {condition}" - else: - condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" - names = [repr(p.name) for p in params.values()] - pstr = libclinic.pprint_words(names) - pl = 's' if len(params) != 1 else '' - message = ( - f"Passing keyword argument{pl} {pstr} to " - f"{func.fulldisplayname}() is deprecated." - ) - - for (major, minor), group in itertools.groupby( - params.values(), key=attrgetter("deprecated_keyword") - ): - names = [repr(p.name) for p in group] - pstr = libclinic.pprint_words(names) - pl = 's' if len(names) != 1 else '' - message += ( - f" Parameter{pl} {pstr} will become positional-only " - f"in Python {major}.{minor}." - ) - - if containscheck: - errcheck = f""" - if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail - goto exit; - }}}}""" - else: - errcheck = "" - if argname_fmt: - # Append deprecation warning to docstring. - docstring = textwrap.fill(f"Note: {message}") - func.docstring += f"\n\n{docstring}\n" - # Format and return the code block. - code = self.DEPRECATION_WARNING_PROTOTYPE.format( - condition=condition, - errcheck=errcheck, - message=libclinic.wrapped_c_string_literal(message, width=64, - subsequent_indent=20), - ) - return libclinic.normalize_snippet(code, indent=4) - - def output_templates( - self, - f: Function, - clinic: Clinic - ) -> dict[str, str]: - parameters = list(f.parameters.values()) - assert parameters - first_param = parameters.pop(0) - assert isinstance(first_param.converter, self_converter) - requires_defining_class = False - if parameters and isinstance(parameters[0].converter, defining_class_converter): - requires_defining_class = True - del parameters[0] - converters = [p.converter for p in parameters] - - if f.critical_section: - clinic.add_include('pycore_critical_section.h', 'Py_BEGIN_CRITICAL_SECTION()') - has_option_groups = parameters and (parameters[0].group or parameters[-1].group) - simple_return = (f.return_converter.type == 'PyObject *' - and not f.critical_section) - new_or_init = f.kind.new_or_init - - vararg: int | str = self.NO_VARARG - pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 - for i, p in enumerate(parameters, 1): - if p.is_keyword_only(): - assert not p.is_positional_only() - if not p.is_optional(): - min_kw_only = i - max_pos - elif p.is_vararg(): - if vararg != self.NO_VARARG: - fail("Too many var args") - pseudo_args += 1 - vararg = i - 1 - else: - if vararg == self.NO_VARARG: - max_pos = i - if p.is_positional_only(): - pos_only = i - if not p.is_optional(): - min_pos = i - - meth_o = (len(parameters) == 1 and - parameters[0].is_positional_only() and - not converters[0].is_optional() and - not requires_defining_class and - not new_or_init) - - # we have to set these things before we're done: - # - # docstring_prototype - # docstring_definition - # impl_prototype - # methoddef_define - # parser_prototype - # parser_definition - # impl_definition - # cpp_if - # cpp_endif - # methoddef_ifndef - - return_value_declaration = "PyObject *return_value = NULL;" - methoddef_define = self.METHODDEF_PROTOTYPE_DEFINE - if new_or_init and not f.docstring: - docstring_prototype = docstring_definition = '' - elif f.kind is GETTER: - methoddef_define = self.GETTERDEF_PROTOTYPE_DEFINE - if f.docstring: - docstring_prototype = '' - docstring_definition = self.GETSET_DOCSTRING_PROTOTYPE_STRVAR - else: - docstring_prototype = docstring_definition = '' - elif f.kind is SETTER: - if f.docstring: - fail("docstrings are only supported for @getter, not @setter") - return_value_declaration = "int {return_value};" - methoddef_define = self.SETTERDEF_PROTOTYPE_DEFINE - docstring_prototype = docstring_definition = '' - else: - docstring_prototype = self.DOCSTRING_PROTOTYPE_VAR - docstring_definition = self.DOCSTRING_PROTOTYPE_STRVAR - impl_definition = self.IMPL_DEFINITION_PROTOTYPE - impl_prototype = parser_prototype = parser_definition = None - - # parser_body_fields remembers the fields passed in to the - # previous call to parser_body. this is used for an awful hack. - parser_body_fields: tuple[str, ...] = () - def parser_body( - prototype: str, - *fields: str, - declarations: str = '' - ) -> str: - nonlocal parser_body_fields - lines = [] - lines.append(prototype) - parser_body_fields = fields - - preamble = libclinic.normalize_snippet(""" - {{ - {return_value_declaration} - {parser_declarations} - {declarations} - {initializers} - """) + "\n" - finale = libclinic.normalize_snippet(""" - {modifications} - {lock} - {return_value} = {c_basename}_impl({impl_arguments}); - {unlock} - {return_conversion} - {post_parsing} - - {exit_label} - {cleanup} - return return_value; - }} - """) - for field in preamble, *fields, finale: - lines.append(field) - return linear_format("\n".join(lines), parser_declarations=declarations) - - fastcall = not new_or_init - limited_capi = clinic.limited_capi - if limited_capi and (pseudo_args or - (any(p.is_optional() for p in parameters) and - any(p.is_keyword_only() and not p.is_optional() for p in parameters)) or - any(c.broken_limited_capi for c in converters)): - warn(f"Function {f.full_name} cannot use limited C API") - limited_capi = False - - parsearg: str | None - if f.kind in {GETTER, SETTER} and parameters: - fail(f"@{f.kind.name.lower()} method cannot define parameters") - - if not parameters: - parser_code: list[str] | None - if f.kind is GETTER: - flags = "" # This should end up unused - parser_prototype = self.PARSER_PROTOTYPE_GETTER - parser_code = [] - elif f.kind is SETTER: - flags = "" - parser_prototype = self.PARSER_PROTOTYPE_SETTER - parser_code = [] - elif not requires_defining_class: - # no parameters, METH_NOARGS - flags = "METH_NOARGS" - parser_prototype = self.PARSER_PROTOTYPE_NOARGS - parser_code = [] - else: - assert fastcall - - flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - return_error = ('return NULL;' if simple_return - else 'goto exit;') - parser_code = [libclinic.normalize_snippet(""" - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) {{ - PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); - %s - }} - """ % return_error, indent=4)] - - if simple_return: - parser_definition = '\n'.join([ - parser_prototype, - '{{', - *parser_code, - ' return {c_basename}_impl({impl_arguments});', - '}}']) - else: - parser_definition = parser_body(parser_prototype, *parser_code) - - elif meth_o: - flags = "METH_O" - - if (isinstance(converters[0], object_converter) and - converters[0].format_unit == 'O'): - meth_o_prototype = self.METH_O_PROTOTYPE - - if simple_return: - # maps perfectly to METH_O, doesn't need a return converter. - # so we skip making a parse function - # and call directly into the impl function. - impl_prototype = parser_prototype = parser_definition = '' - impl_definition = meth_o_prototype - else: - # SLIGHT HACK - # use impl_parameters for the parser here! - parser_prototype = meth_o_prototype - parser_definition = parser_body(parser_prototype) - - else: - argname = 'arg' - if parameters[0].name == argname: - argname += '_' - parser_prototype = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *%s) - """ % argname) - - displayname = parameters[0].get_displayname(0) - parsearg = converters[0].parse_arg(argname, displayname, limited_capi=limited_capi) - if parsearg is None: - parsearg = """ - if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ - goto exit; - }} - """ % argname - parser_definition = parser_body(parser_prototype, - libclinic.normalize_snippet(parsearg, indent=4)) - - elif has_option_groups: - # positional parameters with option groups - # (we have to generate lots of PyArg_ParseTuple calls - # in a big switch statement) - - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') - - elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: - if fastcall: - # positional-only, but no option groups - # we only need one call to _PyArg_ParseStack - - flags = "METH_FASTCALL" - parser_prototype = self.PARSER_PROTOTYPE_FASTCALL - nargs = 'nargs' - argname_fmt = 'args[%d]' - else: - # positional-only, but no option groups - # we only need one call to PyArg_ParseTuple - - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - if limited_capi: - nargs = 'PyTuple_Size(args)' - argname_fmt = 'PyTuple_GetItem(args, %d)' - else: - nargs = 'PyTuple_GET_SIZE(args)' - argname_fmt = 'PyTuple_GET_ITEM(args, %d)' - - left_args = f"{nargs} - {max_pos}" - max_args = self.NO_VARARG if (vararg != self.NO_VARARG) else max_pos - if limited_capi: - parser_code = [] - if nargs != 'nargs': - nargs_def = f'Py_ssize_t nargs = {nargs};' - parser_code.append(libclinic.normalize_snippet(nargs_def, indent=4)) - nargs = 'nargs' - if min_pos == max_args: - pl = '' if min_pos == 1 else 's' - parser_code.append(libclinic.normalize_snippet(f""" - if ({nargs} != {min_pos}) {{{{ - PyErr_Format(PyExc_TypeError, "{{name}} expected {min_pos} argument{pl}, got %zd", {nargs}); - goto exit; - }}}} - """, - indent=4)) - else: - if min_pos: - pl = '' if min_pos == 1 else 's' - parser_code.append(libclinic.normalize_snippet(f""" - if ({nargs} < {min_pos}) {{{{ - PyErr_Format(PyExc_TypeError, "{{name}} expected at least {min_pos} argument{pl}, got %zd", {nargs}); - goto exit; - }}}} - """, - indent=4)) - if max_args != self.NO_VARARG: - pl = '' if max_args == 1 else 's' - parser_code.append(libclinic.normalize_snippet(f""" - if ({nargs} > {max_args}) {{{{ - PyErr_Format(PyExc_TypeError, "{{name}} expected at most {max_args} argument{pl}, got %zd", {nargs}); - goto exit; - }}}} - """, - indent=4)) - else: - clinic.add_include('pycore_modsupport.h', - '_PyArg_CheckPositional()') - parser_code = [libclinic.normalize_snippet(f""" - if (!_PyArg_CheckPositional("{{name}}", {nargs}, {min_pos}, {max_args})) {{{{ - goto exit; - }}}} - """, indent=4)] - - has_optional = False - for i, p in enumerate(parameters): - if p.is_vararg(): - if fastcall: - parser_code.append(libclinic.normalize_snippet(""" - %s = PyTuple_New(%s); - if (!%s) {{ - goto exit; - }} - for (Py_ssize_t i = 0; i < %s; ++i) {{ - PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); - }} - """ % ( - p.converter.parser_name, - left_args, - p.converter.parser_name, - left_args, - p.converter.parser_name, - max_pos - ), indent=4)) - else: - parser_code.append(libclinic.normalize_snippet(""" - %s = PyTuple_GetSlice(%d, -1); - """ % ( - p.converter.parser_name, - max_pos - ), indent=4)) - continue - - displayname = p.get_displayname(i+1) - argname = argname_fmt % i - parsearg = p.converter.parse_arg(argname, displayname, limited_capi=limited_capi) - if parsearg is None: - parser_code = None - break - if has_optional or p.is_optional(): - has_optional = True - parser_code.append(libclinic.normalize_snippet(""" - if (%s < %d) {{ - goto skip_optional; - }} - """, indent=4) % (nargs, i + 1)) - parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) - - if parser_code is not None: - if has_optional: - parser_code.append("skip_optional:") - else: - if limited_capi: - fastcall = False - if fastcall: - clinic.add_include('pycore_modsupport.h', - '_PyArg_ParseStack()') - parser_code = [libclinic.normalize_snippet(""" - if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - else: - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - parser_code = [libclinic.normalize_snippet(""" - if (!PyArg_ParseTuple(args, "{format_units}:{name}", - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - parser_definition = parser_body(parser_prototype, *parser_code) - - else: - deprecated_positionals: dict[int, Parameter] = {} - deprecated_keywords: dict[int, Parameter] = {} - for i, p in enumerate(parameters): - if p.deprecated_positional: - deprecated_positionals[i] = p - if p.deprecated_keyword: - deprecated_keywords[i] = p - - has_optional_kw = ( - max(pos_only, min_pos) + min_kw_only - < len(converters) - int(vararg != self.NO_VARARG) - ) - - if limited_capi: - parser_code = None - fastcall = False - else: - if vararg == self.NO_VARARG: - clinic.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywords()') - args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only - ) - nargs = "nargs" - else: - clinic.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywordsWithVararg()') - args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only, - vararg - ) - nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" - - if fastcall: - flags = "METH_FASTCALL|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS - argname_fmt = 'args[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [libclinic.normalize_snippet(""" - args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); - if (!args) {{ - goto exit; - }} - """ % args_declaration, indent=4)] - else: - # positional-or-keyword arguments - flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - argname_fmt = 'fastargs[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - declarations += "\nPyObject * const *fastargs;" - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [libclinic.normalize_snippet(""" - fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); - if (!fastargs) {{ - goto exit; - }} - """ % args_declaration, indent=4)] - - if requires_defining_class: - flags = 'METH_METHOD|' + flags - parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - - if parser_code is not None: - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt, - clinic=clinic, - fastcall=fastcall, - limited_capi=limited_capi) - parser_code.append(code) - - add_label: str | None = None - for i, p in enumerate(parameters): - if isinstance(p.converter, defining_class_converter): - raise ValueError("defining_class should be the first " - "parameter (after self)") - displayname = p.get_displayname(i+1) - parsearg = p.converter.parse_arg(argname_fmt % i, displayname, limited_capi=limited_capi) - if parsearg is None: - parser_code = None - break - if add_label and (i == pos_only or i == max_pos): - parser_code.append("%s:" % add_label) - add_label = None - if not p.is_optional(): - parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) - elif i < pos_only: - add_label = 'skip_optional_posonly' - parser_code.append(libclinic.normalize_snippet(""" - if (nargs < %d) {{ - goto %s; - }} - """ % (i + 1, add_label), indent=4)) - if has_optional_kw: - parser_code.append(libclinic.normalize_snippet(""" - noptargs--; - """, indent=4)) - parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) - else: - if i < max_pos: - label = 'skip_optional_pos' - first_opt = max(min_pos, pos_only) - else: - label = 'skip_optional_kwonly' - first_opt = max_pos + min_kw_only - if vararg != self.NO_VARARG: - first_opt += 1 - if i == first_opt: - add_label = label - parser_code.append(libclinic.normalize_snippet(""" - if (!noptargs) {{ - goto %s; - }} - """ % add_label, indent=4)) - if i + 1 == len(parameters): - parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) - else: - add_label = label - parser_code.append(libclinic.normalize_snippet(""" - if (%s) {{ - """ % (argname_fmt % i), indent=4)) - parser_code.append(libclinic.normalize_snippet(parsearg, indent=8)) - parser_code.append(libclinic.normalize_snippet(""" - if (!--noptargs) {{ - goto %s; - }} - }} - """ % add_label, indent=4)) - - if parser_code is not None: - if add_label: - parser_code.append("%s:" % add_label) - else: - declarations = declare_parser(f, clinic=clinic, - hasformat=True, - limited_capi=limited_capi) - if limited_capi: - # positional-or-keyword arguments - assert not fastcall - flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - parser_code = [libclinic.normalize_snippet(""" - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, - {parse_arguments})) - goto exit; - """, indent=4)] - declarations = "static char *_keywords[] = {{{keywords_c} NULL}};" - if deprecated_positionals or deprecated_keywords: - declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" - - elif fastcall: - clinic.add_include('pycore_modsupport.h', - '_PyArg_ParseStackAndKeywords()') - parser_code = [libclinic.normalize_snippet(""" - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - else: - clinic.add_include('pycore_modsupport.h', - '_PyArg_ParseTupleAndKeywordsFast()') - parser_code = [libclinic.normalize_snippet(""" - if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - if deprecated_positionals or deprecated_keywords: - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, None, - clinic=clinic, - fastcall=fastcall, - limited_capi=limited_capi) - parser_code.append(code) - - if deprecated_positionals: - code = self.deprecate_positional_use(f, deprecated_positionals) - # Insert the deprecation code before parameter parsing. - parser_code.insert(0, code) - - assert parser_prototype is not None - parser_definition = parser_body(parser_prototype, *parser_code, - declarations=declarations) - - - # Copy includes from parameters to Clinic after parse_arg() has been - # called above. - for converter in converters: - for include in converter.includes: - clinic.add_include(include.filename, include.reason, - condition=include.condition) - - if new_or_init: - methoddef_define = '' - - if f.kind is METHOD_NEW: - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - else: - return_value_declaration = "int return_value = -1;" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD___INIT__ - - fields = list(parser_body_fields) - parses_positional = 'METH_NOARGS' not in flags - parses_keywords = 'METH_KEYWORDS' in flags - if parses_keywords: - assert parses_positional - - if requires_defining_class: - raise ValueError("Slot methods cannot access their defining class.") - - if not parses_keywords: - declarations = '{base_type_ptr}' - clinic.add_include('pycore_modsupport.h', - '_PyArg_NoKeywords()') - fields.insert(0, libclinic.normalize_snippet(""" - if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ - goto exit; - }} - """, indent=4)) - if not parses_positional: - clinic.add_include('pycore_modsupport.h', - '_PyArg_NoPositional()') - fields.insert(0, libclinic.normalize_snippet(""" - if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ - goto exit; - }} - """, indent=4)) - - parser_definition = parser_body(parser_prototype, *fields, - declarations=declarations) - - - methoddef_cast_end = "" - if flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): - methoddef_cast = "(PyCFunction)" - elif f.kind is GETTER: - methoddef_cast = "" # This should end up unused - elif limited_capi: - methoddef_cast = "(PyCFunction)(void(*)(void))" - else: - methoddef_cast = "_PyCFunction_CAST(" - methoddef_cast_end = ")" - - if f.methoddef_flags: - flags += '|' + f.methoddef_flags - - methoddef_define = methoddef_define.replace('{methoddef_flags}', flags) - methoddef_define = methoddef_define.replace('{methoddef_cast}', methoddef_cast) - methoddef_define = methoddef_define.replace('{methoddef_cast_end}', methoddef_cast_end) - - methoddef_ifndef = '' - conditional = self.cpp.condition() - if not conditional: - cpp_if = cpp_endif = '' - else: - cpp_if = "#if " + conditional - cpp_endif = "#endif /* " + conditional + " */" - - if methoddef_define and f.full_name not in clinic.ifndef_symbols: - clinic.ifndef_symbols.add(f.full_name) - methoddef_ifndef = self.METHODDEF_PROTOTYPE_IFNDEF - - # add ';' to the end of parser_prototype and impl_prototype - # (they mustn't be None, but they could be an empty string.) - assert parser_prototype is not None - if parser_prototype: - assert not parser_prototype.endswith(';') - parser_prototype += ';' - - if impl_prototype is None: - impl_prototype = impl_definition - if impl_prototype: - impl_prototype += ";" - - parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration) - - compiler_warning = self.compiler_deprecated_warning(f, parameters) - if compiler_warning: - parser_definition = compiler_warning + "\n\n" + parser_definition - - d = { - "docstring_prototype" : docstring_prototype, - "docstring_definition" : docstring_definition, - "impl_prototype" : impl_prototype, - "methoddef_define" : methoddef_define, - "parser_prototype" : parser_prototype, - "parser_definition" : parser_definition, - "impl_definition" : impl_definition, - "cpp_if" : cpp_if, - "cpp_endif" : cpp_endif, - "methoddef_ifndef" : methoddef_ifndef, - } - - # make sure we didn't forget to assign something, - # and wrap each non-empty value in \n's - d2 = {} - for name, value in d.items(): - assert value is not None, "got a None value for template " + repr(name) - if value: - value = '\n' + value + '\n' - d2[name] = value - return d2 - - @staticmethod - def group_to_variable_name(group: int) -> str: - adjective = "left_" if group < 0 else "right_" - return "group_" + adjective + str(abs(group)) - - def render_option_group_parsing( - self, - f: Function, - template_dict: TemplateDict, - limited_capi: bool, - ) -> None: - # positional only, grouped, optional arguments! - # can be optional on the left or right. - # here's an example: - # - # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] - # - # Here group D are required, and all other groups are optional. - # (Group D's "group" is actually None.) - # We can figure out which sets of arguments we have based on - # how many arguments are in the tuple. - # - # Note that you need to count up on both sides. For example, - # you could have groups C+D, or C+D+E, or C+D+E+F. - # - # What if the number of arguments leads us to an ambiguous result? - # Clinic prefers groups on the left. So in the above example, - # five arguments would map to B+C, not C+D. - - out = [] - parameters = list(f.parameters.values()) - if isinstance(parameters[0].converter, self_converter): - del parameters[0] - - group: list[Parameter] | None = None - left = [] - right = [] - required: list[Parameter] = [] - last: int | Literal[Sentinels.unspecified] = unspecified - - for p in parameters: - group_id = p.group - if group_id != last: - last = group_id - group = [] - if group_id < 0: - left.append(group) - elif group_id == 0: - group = required - else: - right.append(group) - assert group is not None - group.append(p) - - count_min = sys.maxsize - count_max = -1 - - if limited_capi: - nargs = 'PyTuple_Size(args)' - else: - nargs = 'PyTuple_GET_SIZE(args)' - out.append(f"switch ({nargs}) {{\n") - for subset in permute_optional_groups(left, required, right): - count = len(subset) - count_min = min(count_min, count) - count_max = max(count_max, count) - - if count == 0: - out.append(""" case 0: - break; -""") - continue - - group_ids = {p.group for p in subset} # eliminate duplicates - d: dict[str, str | int] = {} - d['count'] = count - d['name'] = f.name - d['format_units'] = "".join(p.converter.format_unit for p in subset) - - parse_arguments: list[str] = [] - for p in subset: - p.converter.parse_argument(parse_arguments) - d['parse_arguments'] = ", ".join(parse_arguments) - - group_ids.discard(0) - lines = "\n".join([ - self.group_to_variable_name(g) + " = 1;" - for g in group_ids - ]) - - s = """\ - case {count}: - if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ - goto exit; - }} - {group_booleans} - break; -""" - s = linear_format(s, group_booleans=lines) - s = s.format_map(d) - out.append(s) - - out.append(" default:\n") - s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' - out.append(s.format(f.full_name, count_min, count_max)) - out.append(' goto exit;\n') - out.append("}") - - template_dict['option_group_parsing'] = libclinic.format_escape("".join(out)) - - def render_function( - self, - clinic: Clinic | None, - f: Function | None - ) -> str: - if f is None or clinic is None: - return "" - - data = CRenderData() - - assert f.parameters, "We should always have a 'self' at this point!" - parameters = f.render_parameters - converters = [p.converter for p in parameters] - - templates = self.output_templates(f, clinic) - - f_self = parameters[0] - selfless = parameters[1:] - assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!" - - if f.critical_section: - match len(f.target_critical_section): - case 0: - lock = 'Py_BEGIN_CRITICAL_SECTION({self_name});' - unlock = 'Py_END_CRITICAL_SECTION();' - case 1: - lock = 'Py_BEGIN_CRITICAL_SECTION({target_critical_section});' - unlock = 'Py_END_CRITICAL_SECTION();' - case _: - lock = 'Py_BEGIN_CRITICAL_SECTION2({target_critical_section});' - unlock = 'Py_END_CRITICAL_SECTION2();' - data.lock.append(lock) - data.unlock.append(unlock) - - last_group = 0 - first_optional = len(selfless) - positional = selfless and selfless[-1].is_positional_only() - has_option_groups = False - - # offset i by -1 because first_optional needs to ignore self - for i, p in enumerate(parameters, -1): - c = p.converter - - if (i != -1) and (p.default is not unspecified): - first_optional = min(first_optional, i) - - if p.is_vararg(): - data.cleanup.append(f"Py_XDECREF({c.parser_name});") - - # insert group variable - group = p.group - if last_group != group: - last_group = group - if group: - group_name = self.group_to_variable_name(group) - data.impl_arguments.append(group_name) - data.declarations.append("int " + group_name + " = 0;") - data.impl_parameters.append("int " + group_name) - has_option_groups = True - - c.render(p, data) - - if has_option_groups and (not positional): - fail("You cannot use optional groups ('[' and ']') " - "unless all parameters are positional-only ('/').") - - # HACK - # when we're METH_O, but have a custom return converter, - # we use "impl_parameters" for the parsing function - # because that works better. but that means we must - # suppress actually declaring the impl's parameters - # as variables in the parsing function. but since it's - # METH_O, we have exactly one anyway, so we know exactly - # where it is. - if ("METH_O" in templates['methoddef_define'] and - '{impl_parameters}' in templates['parser_prototype']): - data.declarations.pop(0) - - full_name = f.full_name - template_dict = {'full_name': full_name} - template_dict['name'] = f.displayname - if f.kind in {GETTER, SETTER}: - template_dict['getset_name'] = f.c_basename.upper() - template_dict['getset_basename'] = f.c_basename - if f.kind is GETTER: - template_dict['c_basename'] = f.c_basename + "_get" - elif f.kind is SETTER: - template_dict['c_basename'] = f.c_basename + "_set" - # Implicitly add the setter value parameter. - data.impl_parameters.append("PyObject *value") - data.impl_arguments.append("value") - else: - template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" - template_dict['c_basename'] = f.c_basename - - template_dict['docstring'] = libclinic.docstring_for_c_string(f.docstring) - template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' - template_dict['target_critical_section'] = ', '.join(f.target_critical_section) - for converter in converters: - converter.set_template_dict(template_dict) - - f.return_converter.render(f, data) - if f.kind is SETTER: - # All setters return an int. - template_dict['impl_return_type'] = 'int' - else: - template_dict['impl_return_type'] = f.return_converter.type - - template_dict['declarations'] = libclinic.format_escape("\n".join(data.declarations)) - template_dict['initializers'] = "\n\n".join(data.initializers) - template_dict['modifications'] = '\n\n'.join(data.modifications) - template_dict['keywords_c'] = ' '.join('"' + k + '",' - for k in data.keywords) - keywords = [k for k in data.keywords if k] - template_dict['keywords_py'] = ' '.join('&_Py_ID(' + k + '),' - for k in keywords) - template_dict['format_units'] = ''.join(data.format_units) - template_dict['parse_arguments'] = ', '.join(data.parse_arguments) - if data.parse_arguments: - template_dict['parse_arguments_comma'] = ','; - else: - template_dict['parse_arguments_comma'] = ''; - template_dict['impl_parameters'] = ", ".join(data.impl_parameters) - template_dict['impl_arguments'] = ", ".join(data.impl_arguments) - - template_dict['return_conversion'] = libclinic.format_escape("".join(data.return_conversion).rstrip()) - template_dict['post_parsing'] = libclinic.format_escape("".join(data.post_parsing).rstrip()) - template_dict['cleanup'] = libclinic.format_escape("".join(data.cleanup)) - - template_dict['return_value'] = data.return_value - template_dict['lock'] = "\n".join(data.lock) - template_dict['unlock'] = "\n".join(data.unlock) - - # used by unpack tuple code generator - unpack_min = first_optional - unpack_max = len(selfless) - template_dict['unpack_min'] = str(unpack_min) - template_dict['unpack_max'] = str(unpack_max) - - if has_option_groups: - self.render_option_group_parsing(f, template_dict, - limited_capi=clinic.limited_capi) - - # buffers, not destination - for name, destination in clinic.destination_buffers.items(): - template = templates[name] - if has_option_groups: - template = linear_format(template, - option_group_parsing=template_dict['option_group_parsing']) - template = linear_format(template, - declarations=template_dict['declarations'], - return_conversion=template_dict['return_conversion'], - initializers=template_dict['initializers'], - modifications=template_dict['modifications'], - post_parsing=template_dict['post_parsing'], - cleanup=template_dict['cleanup'], - lock=template_dict['lock'], - unlock=template_dict['unlock'], - ) - - # Only generate the "exit:" label - # if we have any gotos - need_exit_label = "goto exit;" in template - template = linear_format(template, - exit_label="exit:" if need_exit_label else '' - ) - - s = template.format_map(template_dict) - - # mild hack: - # reflow long impl declarations - if name in {"impl_prototype", "impl_definition"}: - s = libclinic.wrap_declarations(s) - - if clinic.line_prefix: - s = libclinic.indent_all_lines(s, clinic.line_prefix) - if clinic.line_suffix: - s = libclinic.suffix_all_lines(s, clinic.line_suffix) - - destination.append(s) - - return clinic.get_destination('block').dump() - - -@dc.dataclass(slots=True, repr=False) -class Block: - r""" - Represents a single block of text embedded in - another file. If dsl_name is None, the block represents - verbatim text, raw original text from the file, in - which case "input" will be the only non-false member. - If dsl_name is not None, the block represents a Clinic - block. - - input is always str, with embedded \n characters. - input represents the original text from the file; - if it's a Clinic block, it is the original text with - the body_prefix and redundant leading whitespace removed. - - dsl_name is either str or None. If str, it's the text - found on the start line of the block between the square - brackets. - - signatures is a list. - It may only contain clinic.Module, clinic.Class, and - clinic.Function objects. At the moment it should - contain at most one of each. - - output is either str or None. If str, it's the output - from this block, with embedded '\n' characters. - - indent is a str. It's the leading whitespace - that was found on every line of input. (If body_prefix is - not empty, this is the indent *after* removing the - body_prefix.) - - "indent" is different from the concept of "preindent" - (which is not stored as state on Block objects). - "preindent" is the whitespace that - was found in front of every line of input *before* the - "body_prefix" (see the Language object). If body_prefix - is empty, preindent must always be empty too. - - To illustrate the difference between "indent" and "preindent": - - Assume that '_' represents whitespace. - If the block processed was in a Python file, and looked like this: - ____#/*[python] - ____#__for a in range(20): - ____#____print(a) - ____#[python]*/ - "preindent" would be "____" and "indent" would be "__". - - """ - input: str - dsl_name: str | None = None - signatures: list[Module | Class | Function] = dc.field(default_factory=list) - output: Any = None # TODO: Very dynamic; probably untypeable in its current form? - indent: str = '' - - def __repr__(self) -> str: - dsl_name = self.dsl_name or "text" - def summarize(s: object) -> str: - s = repr(s) - if len(s) > 30: - return s[:26] + "..." + s[0] - return s - parts = ( - repr(dsl_name), - f"input={summarize(self.input)}", - f"output={summarize(self.output)}" - ) - return f"" - - -class BlockParser: - """ - Block-oriented parser for Argument Clinic. - Iterator, yields Block objects. - """ - - def __init__( - self, - input: str, - language: Language, - *, - verify: bool = True - ) -> None: - """ - "input" should be a str object - with embedded \n characters. - - "language" should be a Language object. - """ - language.validate() - - self.input = collections.deque(reversed(input.splitlines(keepends=True))) - self.block_start_line_number = self.line_number = 0 - - self.language = language - before, _, after = language.start_line.partition('{dsl_name}') - assert _ == '{dsl_name}' - self.find_start_re = libclinic.create_regex(before, after, - whole_line=False) - self.start_re = libclinic.create_regex(before, after) - self.verify = verify - self.last_checksum_re: re.Pattern[str] | None = None - self.last_dsl_name: str | None = None - self.dsl_name: str | None = None - self.first_block = True - - def __iter__(self) -> BlockParser: - return self - - def __next__(self) -> Block: - while True: - if not self.input: - raise StopIteration - - if self.dsl_name: - return_value = self.parse_clinic_block(self.dsl_name) - self.dsl_name = None - self.first_block = False - return return_value - block = self.parse_verbatim_block() - if self.first_block and not block.input: - continue - self.first_block = False - return block - - - def is_start_line(self, line: str) -> str | None: - match = self.start_re.match(line.lstrip()) - return match.group(1) if match else None - - def _line(self, lookahead: bool = False) -> str: - self.line_number += 1 - line = self.input.pop() - if not lookahead: - self.language.parse_line(line) - return line - - def parse_verbatim_block(self) -> Block: - lines = [] - self.block_start_line_number = self.line_number - - while self.input: - line = self._line() - dsl_name = self.is_start_line(line) - if dsl_name: - self.dsl_name = dsl_name - break - lines.append(line) - - return Block("".join(lines)) - - def parse_clinic_block(self, dsl_name: str) -> Block: - in_lines = [] - self.block_start_line_number = self.line_number + 1 - stop_line = self.language.stop_line.format(dsl_name=dsl_name) - body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) - - def is_stop_line(line: str) -> bool: - # make sure to recognize stop line even if it - # doesn't end with EOL (it could be the very end of the file) - if line.startswith(stop_line): - remainder = line.removeprefix(stop_line) - if remainder and not remainder.isspace(): - fail(f"Garbage after stop line: {remainder!r}") - return True - else: - # gh-92256: don't allow incorrectly formatted stop lines - if line.lstrip().startswith(stop_line): - fail(f"Whitespace is not allowed before the stop line: {line!r}") - return False - - # consume body of program - while self.input: - line = self._line() - if is_stop_line(line) or self.is_start_line(line): - break - if body_prefix: - line = line.lstrip() - assert line.startswith(body_prefix) - line = line.removeprefix(body_prefix) - in_lines.append(line) - - # consume output and checksum line, if present. - if self.last_dsl_name == dsl_name: - checksum_re = self.last_checksum_re - else: - before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') - assert _ == '{arguments}' - checksum_re = libclinic.create_regex(before, after, word=False) - self.last_dsl_name = dsl_name - self.last_checksum_re = checksum_re - assert checksum_re is not None - - # scan forward for checksum line - out_lines = [] - arguments = None - while self.input: - line = self._line(lookahead=True) - match = checksum_re.match(line.lstrip()) - arguments = match.group(1) if match else None - if arguments: - break - out_lines.append(line) - if self.is_start_line(line): - break - - output: str | None - output = "".join(out_lines) - if arguments: - d = {} - for field in shlex.split(arguments): - name, equals, value = field.partition('=') - if not equals: - fail(f"Mangled Argument Clinic marker line: {line!r}") - d[name.strip()] = value.strip() - - if self.verify: - if 'input' in d: - checksum = d['output'] - else: - checksum = d['checksum'] - - computed = libclinic.compute_checksum(output, len(checksum)) - if checksum != computed: - fail("Checksum mismatch! " - f"Expected {checksum!r}, computed {computed!r}. " - "Suggested fix: remove all generated code including " - "the end marker, or use the '-f' option.") - else: - # put back output - output_lines = output.splitlines(keepends=True) - self.line_number -= len(output_lines) - self.input.extend(reversed(output_lines)) - output = None - - return Block("".join(in_lines), dsl_name, output=output) - - -@dc.dataclass(slots=True, frozen=True) -class Include: - """ - An include like: #include "pycore_long.h" // _Py_ID() - """ - # Example: "pycore_long.h". - filename: str - - # Example: "_Py_ID()". - reason: str - - # None means unconditional include. - # Example: "#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)". - condition: str | None - - def sort_key(self) -> tuple[str, str]: - # order: '#if' comes before 'NO_CONDITION' - return (self.condition or 'NO_CONDITION', self.filename) - - -@dc.dataclass(slots=True) -class BlockPrinter: - language: Language - f: io.StringIO = dc.field(default_factory=io.StringIO) - - # '#include "header.h" // reason': column of '//' comment - INCLUDE_COMMENT_COLUMN: Final[int] = 35 - - def print_block( - self, - block: Block, - *, - core_includes: bool = False, - limited_capi: bool, - header_includes: dict[str, Include], - ) -> None: - input = block.input - output = block.output - dsl_name = block.dsl_name - write = self.f.write - - assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name) - - if not dsl_name: - write(input) - return - - write(self.language.start_line.format(dsl_name=dsl_name)) - write("\n") - - body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) - if not body_prefix: - write(input) - else: - for line in input.split('\n'): - write(body_prefix) - write(line) - write("\n") - - write(self.language.stop_line.format(dsl_name=dsl_name)) - write("\n") - - output = '' - if core_includes and header_includes: - # Emit optional "#include" directives for C headers - output += '\n' - - current_condition: str | None = None - includes = sorted(header_includes.values(), key=Include.sort_key) - for include in includes: - if include.condition != current_condition: - if current_condition: - output += '#endif\n' - current_condition = include.condition - if include.condition: - output += f'{include.condition}\n' - - if current_condition: - line = f'# include "{include.filename}"' - else: - line = f'#include "{include.filename}"' - if include.reason: - comment = f'// {include.reason}\n' - line = line.ljust(self.INCLUDE_COMMENT_COLUMN - 1) + comment - output += line - - if current_condition: - output += '#endif\n' - - input = ''.join(block.input) - output += ''.join(block.output) - if output: - if not output.endswith('\n'): - output += '\n' - write(output) - - arguments = "output={output} input={input}".format( - output=libclinic.compute_checksum(output, 16), - input=libclinic.compute_checksum(input, 16) - ) - write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) - write("\n") - - def write(self, text: str) -> None: - self.f.write(text) - - -class BufferSeries: - """ - Behaves like a "defaultlist". - When you ask for an index that doesn't exist yet, - the object grows the list until that item exists. - So o[n] will always work. - - Supports negative indices for actual items. - e.g. o[-1] is an element immediately preceding o[0]. - """ - - def __init__(self) -> None: - self._start = 0 - self._array: list[list[str]] = [] - - def __getitem__(self, i: int) -> list[str]: - i -= self._start - if i < 0: - self._start += i - prefix: list[list[str]] = [[] for x in range(-i)] - self._array = prefix + self._array - i = 0 - while i >= len(self._array): - self._array.append([]) - return self._array[i] - - def clear(self) -> None: - for ta in self._array: - ta.clear() - - def dump(self) -> str: - texts = ["".join(ta) for ta in self._array] - self.clear() - return "".join(texts) - - -@dc.dataclass(slots=True, repr=False) -class Destination: - name: str - type: str - clinic: Clinic - buffers: BufferSeries = dc.field(init=False, default_factory=BufferSeries) - filename: str = dc.field(init=False) # set in __post_init__ - - args: dc.InitVar[tuple[str, ...]] = () - - def __post_init__(self, args: tuple[str, ...]) -> None: - valid_types = ('buffer', 'file', 'suppress') - if self.type not in valid_types: - fail( - f"Invalid destination type {self.type!r} for {self.name}, " - f"must be {', '.join(valid_types)}" - ) - extra_arguments = 1 if self.type == "file" else 0 - if len(args) < extra_arguments: - fail(f"Not enough arguments for destination " - f"{self.name!r} new {self.type!r}") - if len(args) > extra_arguments: - fail(f"Too many arguments for destination {self.name!r} new {self.type!r}") - if self.type =='file': - d = {} - filename = self.clinic.filename - d['path'] = filename - dirname, basename = os.path.split(filename) - if not dirname: - dirname = '.' - d['dirname'] = dirname - d['basename'] = basename - d['basename_root'], d['basename_extension'] = os.path.splitext(filename) - self.filename = args[0].format_map(d) - - def __repr__(self) -> str: - if self.type == 'file': - type_repr = f"type='file' file={self.filename!r}" - else: - type_repr = f"type={self.type!r}" - return f"" - - def clear(self) -> None: - if self.type != 'buffer': - fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'") - self.buffers.clear() - - def dump(self) -> str: - return self.buffers.dump() - - -# "extensions" maps the file extension ("c", "py") to Language classes. -LangDict = dict[str, Callable[[str], Language]] -extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } -extensions['py'] = PythonLanguage - - -ClassDict = dict[str, "Class"] -DestinationDict = dict[str, Destination] -ModuleDict = dict[str, "Module"] - - -class Parser(Protocol): - def __init__(self, clinic: Clinic) -> None: ... - def parse(self, block: Block) -> None: ... - - -clinic: Clinic | None = None -class Clinic: - - presets_text = """ -preset block -everything block -methoddef_ifndef buffer 1 -docstring_prototype suppress -parser_prototype suppress -cpp_if suppress -cpp_endif suppress - -preset original -everything block -methoddef_ifndef buffer 1 -docstring_prototype suppress -parser_prototype suppress -cpp_if suppress -cpp_endif suppress - -preset file -everything file -methoddef_ifndef file 1 -docstring_prototype suppress -parser_prototype suppress -impl_definition block - -preset buffer -everything buffer -methoddef_ifndef buffer 1 -impl_definition block -docstring_prototype suppress -impl_prototype suppress -parser_prototype suppress - -preset partial-buffer -everything buffer -methoddef_ifndef buffer 1 -docstring_prototype block -impl_prototype suppress -methoddef_define block -parser_prototype block -impl_definition block - -""" - - def __init__( - self, - language: CLanguage, - printer: BlockPrinter | None = None, - *, - filename: str, - limited_capi: bool, - verify: bool = True, - ) -> None: - # maps strings to Parser objects. - # (instantiated from the "parsers" global.) - self.parsers: dict[str, Parser] = {} - self.language: CLanguage = language - if printer: - fail("Custom printers are broken right now") - self.printer = printer or BlockPrinter(language) - self.verify = verify - self.limited_capi = limited_capi - self.filename = filename - self.modules: ModuleDict = {} - self.classes: ClassDict = {} - self.functions: list[Function] = [] - # dict: include name => Include instance - self.includes: dict[str, Include] = {} - - self.line_prefix = self.line_suffix = '' - - self.destinations: DestinationDict = {} - self.add_destination("block", "buffer") - self.add_destination("suppress", "suppress") - self.add_destination("buffer", "buffer") - if filename: - self.add_destination("file", "file", "{dirname}/clinic/{basename}.h") - - d = self.get_destination_buffer - self.destination_buffers = { - 'cpp_if': d('file'), - 'docstring_prototype': d('suppress'), - 'docstring_definition': d('file'), - 'methoddef_define': d('file'), - 'impl_prototype': d('file'), - 'parser_prototype': d('suppress'), - 'parser_definition': d('file'), - 'cpp_endif': d('file'), - 'methoddef_ifndef': d('file', 1), - 'impl_definition': d('block'), - } - - DestBufferType = dict[str, list[str]] - DestBufferList = list[DestBufferType] - - self.destination_buffers_stack: DestBufferList = [] - self.ifndef_symbols: set[str] = set() - - self.presets: dict[str, dict[Any, Any]] = {} - preset = None - for line in self.presets_text.strip().split('\n'): - line = line.strip() - if not line: - continue - name, value, *options = line.split() - if name == 'preset': - self.presets[value] = preset = {} - continue - - if len(options): - index = int(options[0]) - else: - index = 0 - buffer = self.get_destination_buffer(value, index) - - if name == 'everything': - for name in self.destination_buffers: - preset[name] = buffer - continue - - assert name in self.destination_buffers - preset[name] = buffer - - global clinic - clinic = self - - def add_include(self, name: str, reason: str, - *, condition: str | None = None) -> None: - try: - existing = self.includes[name] - except KeyError: - pass - else: - if existing.condition and not condition: - # If the previous include has a condition and the new one is - # unconditional, override the include. - pass - else: - # Already included, do nothing. Only mention a single reason, - # no need to list all of them. - return - - self.includes[name] = Include(name, reason, condition) - - def add_destination( - self, - name: str, - type: str, - *args: str - ) -> None: - if name in self.destinations: - fail(f"Destination already exists: {name!r}") - self.destinations[name] = Destination(name, type, self, args) - - def get_destination(self, name: str) -> Destination: - d = self.destinations.get(name) - if not d: - fail(f"Destination does not exist: {name!r}") - return d - - def get_destination_buffer( - self, - name: str, - item: int = 0 - ) -> list[str]: - d = self.get_destination(name) - return d.buffers[item] - - def parse(self, input: str) -> str: - printer = self.printer - self.block_parser = BlockParser(input, self.language, verify=self.verify) - for block in self.block_parser: - dsl_name = block.dsl_name - if dsl_name: - if dsl_name not in self.parsers: - assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." - self.parsers[dsl_name] = parsers[dsl_name](self) - parser = self.parsers[dsl_name] - parser.parse(block) - printer.print_block(block, - limited_capi=self.limited_capi, - header_includes=self.includes) - - # these are destinations not buffers - for name, destination in self.destinations.items(): - if destination.type == 'suppress': - continue - output = destination.dump() - - if output: - block = Block("", dsl_name="clinic", output=output) - - if destination.type == 'buffer': - block.input = "dump " + name + "\n" - warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") - printer.write("\n") - printer.print_block(block, - limited_capi=self.limited_capi, - header_includes=self.includes) - continue - - if destination.type == 'file': - try: - dirname = os.path.dirname(destination.filename) - try: - os.makedirs(dirname) - except FileExistsError: - if not os.path.isdir(dirname): - fail(f"Can't write to destination " - f"{destination.filename!r}; " - f"can't make directory {dirname!r}!") - if self.verify: - with open(destination.filename) as f: - parser_2 = BlockParser(f.read(), language=self.language) - blocks = list(parser_2) - if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): - fail(f"Modified destination file " - f"{destination.filename!r}; not overwriting!") - except FileNotFoundError: - pass - - block.input = 'preserve\n' - printer_2 = BlockPrinter(self.language) - printer_2.print_block(block, - core_includes=True, - limited_capi=self.limited_capi, - header_includes=self.includes) - libclinic.write_file(destination.filename, - printer_2.f.getvalue()) - continue - - return printer.f.getvalue() - - def _module_and_class( - self, fields: Sequence[str] - ) -> tuple[Module | Clinic, Class | None]: - """ - fields should be an iterable of field names. - returns a tuple of (module, class). - the module object could actually be self (a clinic object). - this function is only ever used to find the parent of where - a new class/module should go. - """ - parent: Clinic | Module | Class = self - module: Clinic | Module = self - cls: Class | None = None - - for idx, field in enumerate(fields): - if not isinstance(parent, Class): - if field in parent.modules: - parent = module = parent.modules[field] - continue - if field in parent.classes: - parent = cls = parent.classes[field] - else: - fullname = ".".join(fields[idx:]) - fail(f"Parent class or module {fullname!r} does not exist.") - - return module, cls - - def __repr__(self) -> str: - return "" - - -def parse_file( - filename: str, - *, - limited_capi: bool, - output: str | None = None, - verify: bool = True, -) -> None: - if not output: - output = filename - - extension = os.path.splitext(filename)[1][1:] - if not extension: - fail(f"Can't extract file type for file {filename!r}") - - try: - language = extensions[extension](filename) - except KeyError: - fail(f"Can't identify file type for file {filename!r}") - - with open(filename, encoding="utf-8") as f: - raw = f.read() - - # exit quickly if there are no clinic markers in the file - find_start_re = BlockParser("", language).find_start_re - if not find_start_re.search(raw): - return - - if LIMITED_CAPI_REGEX.search(raw): - limited_capi = True - - assert isinstance(language, CLanguage) - clinic = Clinic(language, - verify=verify, - filename=filename, - limited_capi=limited_capi) - cooked = clinic.parse(raw) - - libclinic.write_file(output, cooked) - - -class PythonParser: - def __init__(self, clinic: Clinic) -> None: - pass - - def parse(self, block: Block) -> None: - with contextlib.redirect_stdout(io.StringIO()) as s: - exec(block.input) - block.output = s.getvalue() - - -@dc.dataclass(repr=False) -class Module: - name: str - module: Module | Clinic - - def __post_init__(self) -> None: - self.parent = self.module - self.modules: ModuleDict = {} - self.classes: ClassDict = {} - self.functions: list[Function] = [] - - def __repr__(self) -> str: - return "" - - -@dc.dataclass(repr=False) -class Class: - name: str - module: Module | Clinic - cls: Class | None - typedef: str - type_object: str - - def __post_init__(self) -> None: - self.parent = self.cls or self.module - self.classes: ClassDict = {} - self.functions: list[Function] = [] - - def __repr__(self) -> str: - return "" - - -unsupported_special_methods: set[str] = set(""" - -__abs__ -__add__ -__and__ -__call__ -__delitem__ -__divmod__ -__eq__ -__float__ -__floordiv__ -__ge__ -__getattr__ -__getattribute__ -__getitem__ -__gt__ -__hash__ -__iadd__ -__iand__ -__ifloordiv__ -__ilshift__ -__imatmul__ -__imod__ -__imul__ -__index__ -__int__ -__invert__ -__ior__ -__ipow__ -__irshift__ -__isub__ -__iter__ -__itruediv__ -__ixor__ -__le__ -__len__ -__lshift__ -__lt__ -__matmul__ -__mod__ -__mul__ -__neg__ -__next__ -__or__ -__pos__ -__pow__ -__radd__ -__rand__ -__rdivmod__ -__repr__ -__rfloordiv__ -__rlshift__ -__rmatmul__ -__rmod__ -__rmul__ -__ror__ -__rpow__ -__rrshift__ -__rshift__ -__rsub__ -__rtruediv__ -__rxor__ -__setattr__ -__setitem__ -__str__ -__sub__ -__truediv__ -__xor__ - -""".strip().split()) - - -class FunctionKind(enum.Enum): - INVALID = enum.auto() - CALLABLE = enum.auto() - STATIC_METHOD = enum.auto() - CLASS_METHOD = enum.auto() - METHOD_INIT = enum.auto() - METHOD_NEW = enum.auto() - GETTER = enum.auto() - SETTER = enum.auto() - - @functools.cached_property - def new_or_init(self) -> bool: - return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} - - def __repr__(self) -> str: - return f"" - - -INVALID: Final = FunctionKind.INVALID -CALLABLE: Final = FunctionKind.CALLABLE -STATIC_METHOD: Final = FunctionKind.STATIC_METHOD -CLASS_METHOD: Final = FunctionKind.CLASS_METHOD -METHOD_INIT: Final = FunctionKind.METHOD_INIT -METHOD_NEW: Final = FunctionKind.METHOD_NEW -GETTER: Final = FunctionKind.GETTER -SETTER: Final = FunctionKind.SETTER - -ParamDict = dict[str, "Parameter"] -ReturnConverterType = Callable[..., "CReturnConverter"] - - -@dc.dataclass(repr=False) -class Function: - """ - Mutable duck type for inspect.Function. - - docstring - a str containing - * embedded line breaks - * text outdented to the left margin - * no trailing whitespace. - It will always be true that - (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) - """ - parameters: ParamDict = dc.field(default_factory=dict) - _: dc.KW_ONLY - name: str - module: Module | Clinic - cls: Class | None - c_basename: str - full_name: str - return_converter: CReturnConverter - kind: FunctionKind - coexist: bool - return_annotation: object = inspect.Signature.empty - docstring: str = '' - # docstring_only means "don't generate a machine-readable - # signature, just a normal docstring". it's True for - # functions with optional groups because we can't represent - # those accurately with inspect.Signature in 3.4. - docstring_only: bool = False - critical_section: bool = False - target_critical_section: list[str] = dc.field(default_factory=list) - - def __post_init__(self) -> None: - self.parent = self.cls or self.module - self.self_converter: self_converter | None = None - self.__render_parameters__: list[Parameter] | None = None - - @functools.cached_property - def displayname(self) -> str: - """Pretty-printable name.""" - if self.kind.new_or_init: - assert isinstance(self.cls, Class) - return self.cls.name - else: - return self.name - - @functools.cached_property - def fulldisplayname(self) -> str: - parent: Class | Module | Clinic | None - if self.kind.new_or_init: - parent = getattr(self.cls, "parent", None) - else: - parent = self.parent - name = self.displayname - while isinstance(parent, (Module, Class)): - name = f"{parent.name}.{name}" - parent = parent.parent - return name - - @property - def render_parameters(self) -> list[Parameter]: - if not self.__render_parameters__: - l: list[Parameter] = [] - self.__render_parameters__ = l - for p in self.parameters.values(): - p = p.copy() - p.converter.pre_render() - l.append(p) - return self.__render_parameters__ - - @property - def methoddef_flags(self) -> str | None: - if self.kind.new_or_init: - return None - flags = [] - match self.kind: - case FunctionKind.CLASS_METHOD: - flags.append('METH_CLASS') - case FunctionKind.STATIC_METHOD: - flags.append('METH_STATIC') - case _ as kind: - acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER, FunctionKind.SETTER} - assert kind in acceptable_kinds, f"unknown kind: {kind!r}" - if self.coexist: - flags.append('METH_COEXIST') - return '|'.join(flags) - - def __repr__(self) -> str: - return f'' - - def copy(self, **overrides: Any) -> Function: - f = dc.replace(self, **overrides) - f.parameters = { - name: value.copy(function=f) - for name, value in f.parameters.items() - } - return f - - -VersionTuple = tuple[int, int] - - -@dc.dataclass(repr=False, slots=True) -class Parameter: - """ - Mutable duck type of inspect.Parameter. - """ - name: str - kind: inspect._ParameterKind - _: dc.KW_ONLY - default: object = inspect.Parameter.empty - function: Function - converter: CConverter - annotation: object = inspect.Parameter.empty - docstring: str = '' - group: int = 0 - # (`None` signifies that there is no deprecation) - deprecated_positional: VersionTuple | None = None - deprecated_keyword: VersionTuple | None = None - right_bracket_count: int = dc.field(init=False, default=0) - - def __repr__(self) -> str: - return f'' - - def is_keyword_only(self) -> bool: - return self.kind == inspect.Parameter.KEYWORD_ONLY - - def is_positional_only(self) -> bool: - return self.kind == inspect.Parameter.POSITIONAL_ONLY - - def is_vararg(self) -> bool: - return self.kind == inspect.Parameter.VAR_POSITIONAL - - def is_optional(self) -> bool: - return not self.is_vararg() and (self.default is not unspecified) - - def copy( - self, - /, - *, - converter: CConverter | None = None, - function: Function | None = None, - **overrides: Any - ) -> Parameter: - function = function or self.function - if not converter: - converter = copy.copy(self.converter) - converter.function = function - return dc.replace(self, **overrides, function=function, converter=converter) - - def get_displayname(self, i: int) -> str: - if i == 0: - return 'argument' - if not self.is_positional_only(): - return f'argument {self.name!r}' - else: - return f'argument {i}' - - def render_docstring(self) -> str: - lines = [f" {self.name}"] - lines.extend(f" {line}" for line in self.docstring.split("\n")) - return "\n".join(lines).rstrip() - - -CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) - -def add_c_converter( - f: CConverterClassT, - name: str | None = None -) -> CConverterClassT: - if not name: - name = f.__name__ - if not name.endswith('_converter'): - return f - name = name.removesuffix('_converter') - converters[name] = f - return f - -def add_default_legacy_c_converter(cls: CConverterClassT) -> CConverterClassT: - # automatically add converter for default format unit - # (but without stomping on the existing one if it's already - # set, in case you subclass) - if ((cls.format_unit not in ('O&', '')) and - (cls.format_unit not in legacy_converters)): - legacy_converters[cls.format_unit] = cls - return cls - -def add_legacy_c_converter( - format_unit: str, - **kwargs: Any -) -> Callable[[CConverterClassT], CConverterClassT]: - """ - Adds a legacy converter. - """ - def closure(f: CConverterClassT) -> CConverterClassT: - added_f: Callable[..., CConverter] - if not kwargs: - added_f = f - else: - added_f = functools.partial(f, **kwargs) - if format_unit: - legacy_converters[format_unit] = added_f - return f - return closure - -class CConverterAutoRegister(type): - def __init__( - cls, name: str, bases: tuple[type[object], ...], classdict: dict[str, Any] - ) -> None: - converter_cls = cast(type["CConverter"], cls) - add_c_converter(converter_cls) - add_default_legacy_c_converter(converter_cls) - -class CConverter(metaclass=CConverterAutoRegister): - """ - For the init function, self, name, function, and default - must be keyword-or-positional parameters. All other - parameters must be keyword-only. - """ - - # The C name to use for this variable. - name: str - - # The Python name to use for this variable. - py_name: str - - # The C type to use for this variable. - # 'type' should be a Python string specifying the type, e.g. "int". - # If this is a pointer type, the type string should end with ' *'. - type: str | None = None - - # The Python default value for this parameter, as a Python value. - # Or the magic value "unspecified" if there is no default. - # Or the magic value "unknown" if this value is a cannot be evaluated - # at Argument-Clinic-preprocessing time (but is presumed to be valid - # at runtime). - default: object = unspecified - - # If not None, default must be isinstance() of this type. - # (You can also specify a tuple of types.) - default_type: bltns.type[object] | tuple[bltns.type[object], ...] | None = None - - # "default" converted into a C value, as a string. - # Or None if there is no default. - c_default: str | None = None - - # "default" converted into a Python value, as a string. - # Or None if there is no default. - py_default: str | None = None - - # The default value used to initialize the C variable when - # there is no default, but not specifying a default may - # result in an "uninitialized variable" warning. This can - # easily happen when using option groups--although - # properly-written code won't actually use the variable, - # the variable does get passed in to the _impl. (Ah, if - # only dataflow analysis could inline the static function!) - # - # This value is specified as a string. - # Every non-abstract subclass should supply a valid value. - c_ignored_default: str = 'NULL' - - # If true, wrap with Py_UNUSED. - unused = False - - # The C converter *function* to be used, if any. - # (If this is not None, format_unit must be 'O&'.) - converter: str | None = None - - # Should Argument Clinic add a '&' before the name of - # the variable when passing it into the _impl function? - impl_by_reference = False - - # Should Argument Clinic add a '&' before the name of - # the variable when passing it into PyArg_ParseTuple (AndKeywords)? - parse_by_reference = True - - ############################################################# - ############################################################# - ## You shouldn't need to read anything below this point to ## - ## write your own converter functions. ## - ############################################################# - ############################################################# - - # The "format unit" to specify for this variable when - # parsing arguments using PyArg_ParseTuple (AndKeywords). - # Custom converters should always use the default value of 'O&'. - format_unit = 'O&' - - # What encoding do we want for this variable? Only used - # by format units starting with 'e'. - encoding: str | None = None - - # Should this object be required to be a subclass of a specific type? - # If not None, should be a string representing a pointer to a - # PyTypeObject (e.g. "&PyUnicode_Type"). - # Only used by the 'O!' format unit (and the "object" converter). - subclass_of: str | None = None - - # See also the 'length_name' property. - # Only used by format units ending with '#'. - length = False - - # Should we show this parameter in the generated - # __text_signature__? This is *almost* always True. - # (It's only False for __new__, __init__, and METH_STATIC functions.) - show_in_signature = True - - # Overrides the name used in a text signature. - # The name used for a "self" parameter must be one of - # self, type, or module; however users can set their own. - # This lets the self_converter overrule the user-settable - # name, *just* for the text signature. - # Only set by self_converter. - signature_name: str | None = None - - broken_limited_capi: bool = False - - # keep in sync with self_converter.__init__! - def __init__(self, - # Positional args: - name: str, - py_name: str, - function: Function, - default: object = unspecified, - *, # Keyword only args: - c_default: str | None = None, - py_default: str | None = None, - annotation: str | Literal[Sentinels.unspecified] = unspecified, - unused: bool = False, - **kwargs: Any - ) -> None: - self.name = ensure_legal_c_identifier(name) - self.py_name = py_name - self.unused = unused - self.includes: list[Include] = [] - - if default is not unspecified: - if (self.default_type - and default is not unknown - and not isinstance(default, self.default_type) - ): - if isinstance(self.default_type, type): - types_str = self.default_type.__name__ - else: - names = [cls.__name__ for cls in self.default_type] - types_str = ', '.join(names) - cls_name = self.__class__.__name__ - fail(f"{cls_name}: default value {default!r} for field " - f"{name!r} is not of type {types_str!r}") - self.default = default - - if c_default: - self.c_default = c_default - if py_default: - self.py_default = py_default - - if annotation is not unspecified: - fail("The 'annotation' parameter is not currently permitted.") - - # Make sure not to set self.function until after converter_init() has been called. - # This prevents you from caching information - # about the function in converter_init(). - # (That breaks if we get cloned.) - self.converter_init(**kwargs) - self.function = function - - # Add a custom __getattr__ method to improve the error message - # if somebody tries to access self.function in converter_init(). - # - # mypy will assume arbitrary access is okay for a class with a __getattr__ method, - # and that's not what we want, - # so put it inside an `if not TYPE_CHECKING` block - if not TYPE_CHECKING: - def __getattr__(self, attr): - if attr == "function": - fail( - f"{self.__class__.__name__!r} object has no attribute 'function'.\n" - f"Note: accessing self.function inside converter_init is disallowed!" - ) - return super().__getattr__(attr) - # this branch is just here for coverage reporting - else: # pragma: no cover - pass - - def converter_init(self) -> None: - pass - - def is_optional(self) -> bool: - return (self.default is not unspecified) - - def _render_self(self, parameter: Parameter, data: CRenderData) -> None: - self.parameter = parameter - name = self.parser_name - - # impl_arguments - s = ("&" if self.impl_by_reference else "") + name - data.impl_arguments.append(s) - if self.length: - data.impl_arguments.append(self.length_name) - - # impl_parameters - data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) - if self.length: - data.impl_parameters.append(f"Py_ssize_t {self.length_name}") - - def _render_non_self( - self, - parameter: Parameter, - data: CRenderData - ) -> None: - self.parameter = parameter - name = self.name - - # declarations - d = self.declaration(in_parser=True) - data.declarations.append(d) - - # initializers - initializers = self.initialize() - if initializers: - data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip()) - - # modifications - modifications = self.modify() - if modifications: - data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) - - # keywords - if parameter.is_vararg(): - pass - elif parameter.is_positional_only(): - data.keywords.append('') - else: - data.keywords.append(parameter.name) - - # format_units - if self.is_optional() and '|' not in data.format_units: - data.format_units.append('|') - if parameter.is_keyword_only() and '$' not in data.format_units: - data.format_units.append('$') - data.format_units.append(self.format_unit) - - # parse_arguments - self.parse_argument(data.parse_arguments) - - # post_parsing - if post_parsing := self.post_parsing(): - data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') - - # cleanup - cleanup = self.cleanup() - if cleanup: - data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") - - def render(self, parameter: Parameter, data: CRenderData) -> None: - """ - parameter is a clinic.Parameter instance. - data is a CRenderData instance. - """ - self._render_self(parameter, data) - self._render_non_self(parameter, data) - - @functools.cached_property - def length_name(self) -> str: - """Computes the name of the associated "length" variable.""" - assert self.length is not None - return self.parser_name + "_length" - - # Why is this one broken out separately? - # For "positional-only" function parsing, - # which generates a bunch of PyArg_ParseTuple calls. - def parse_argument(self, args: list[str]) -> None: - assert not (self.converter and self.encoding) - if self.format_unit == 'O&': - assert self.converter - args.append(self.converter) - - if self.encoding: - args.append(libclinic.c_repr(self.encoding)) - elif self.subclass_of: - args.append(self.subclass_of) - - s = ("&" if self.parse_by_reference else "") + self.parser_name - args.append(s) - - if self.length: - args.append(f"&{self.length_name}") - - # - # All the functions after here are intended as extension points. - # - - def simple_declaration( - self, - by_reference: bool = False, - *, - in_parser: bool = False - ) -> str: - """ - Computes the basic declaration of the variable. - Used in computing the prototype declaration and the - variable declaration. - """ - assert isinstance(self.type, str) - prototype = [self.type] - if by_reference or not self.type.endswith('*'): - prototype.append(" ") - if by_reference: - prototype.append('*') - if in_parser: - name = self.parser_name - else: - name = self.name - if self.unused: - name = f"Py_UNUSED({name})" - prototype.append(name) - return "".join(prototype) - - def declaration(self, *, in_parser: bool = False) -> str: - """ - The C statement to declare this variable. - """ - declaration = [self.simple_declaration(in_parser=True)] - default = self.c_default - if not default and self.parameter.group: - default = self.c_ignored_default - if default: - declaration.append(" = ") - declaration.append(default) - declaration.append(";") - if self.length: - declaration.append('\n') - declaration.append(f"Py_ssize_t {self.length_name};") - return "".join(declaration) - - def initialize(self) -> str: - """ - The C statements required to set up this variable before parsing. - Returns a string containing this code indented at column 0. - If no initialization is necessary, returns an empty string. - """ - return "" - - def modify(self) -> str: - """ - The C statements required to modify this variable after parsing. - Returns a string containing this code indented at column 0. - If no modification is necessary, returns an empty string. - """ - return "" - - def post_parsing(self) -> str: - """ - The C statements required to do some operations after the end of parsing but before cleaning up. - Return a string containing this code indented at column 0. - If no operation is necessary, return an empty string. - """ - return "" - - def cleanup(self) -> str: - """ - The C statements required to clean up after this variable. - Returns a string containing this code indented at column 0. - If no cleanup is necessary, returns an empty string. - """ - return "" - - def pre_render(self) -> None: - """ - A second initialization function, like converter_init, - called just before rendering. - You are permitted to examine self.function here. - """ - pass - - def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, expected_literal: bool = True) -> str: - assert '"' not in expected - if limited_capi: - if expected_literal: - return (f'PyErr_Format(PyExc_TypeError, ' - f'"{{{{name}}}}() {displayname} must be {expected}, not %.50s", ' - f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') - else: - return (f'PyErr_Format(PyExc_TypeError, ' - f'"{{{{name}}}}() {displayname} must be %.50s, not %.50s", ' - f'"{expected}", ' - f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') - else: - if expected_literal: - expected = f'"{expected}"' - self.add_include('pycore_modsupport.h', '_PyArg_BadArgument()') - return f'_PyArg_BadArgument("{{{{name}}}}", "{displayname}", {expected}, {{argname}});' - - def format_code(self, fmt: str, *, - argname: str, - bad_argument: str | None = None, - bad_argument2: str | None = None, - **kwargs: Any) -> str: - if '{bad_argument}' in fmt: - if not bad_argument: - raise TypeError("required 'bad_argument' argument") - fmt = fmt.replace('{bad_argument}', bad_argument) - if '{bad_argument2}' in fmt: - if not bad_argument2: - raise TypeError("required 'bad_argument2' argument") - fmt = fmt.replace('{bad_argument2}', bad_argument2) - return fmt.format(argname=argname, paramname=self.parser_name, **kwargs) - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'O&': - return self.format_code(""" - if (!{converter}({argname}, &{paramname})) {{{{ - goto exit; - }}}} - """, - argname=argname, - converter=self.converter) - if self.format_unit == 'O!': - cast = '(%s)' % self.type if self.type != 'PyObject *' else '' - if self.subclass_of in type_checks: - typecheck, typename = type_checks[self.subclass_of] - return self.format_code(""" - if (!{typecheck}({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = {cast}{argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, typename, limited_capi=limited_capi), - typecheck=typecheck, typename=typename, cast=cast) - return self.format_code(""" - if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = {cast}{argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, '({subclass_of})->tp_name', - expected_literal=False, limited_capi=limited_capi), - subclass_of=self.subclass_of, cast=cast) - if self.format_unit == 'O': - cast = '(%s)' % self.type if self.type != 'PyObject *' else '' - return self.format_code(""" - {paramname} = {cast}{argname}; - """, - argname=argname, cast=cast) - return None - - def set_template_dict(self, template_dict: TemplateDict) -> None: - pass - - @property - def parser_name(self) -> str: - if self.name in libclinic.CLINIC_PREFIXED_ARGS: # bpo-39741 - return libclinic.CLINIC_PREFIX + self.name - else: - return self.name - - def add_include(self, name: str, reason: str, - *, condition: str | None = None) -> None: - include = Include(name, reason, condition) - self.includes.append(include) - -type_checks = { - '&PyLong_Type': ('PyLong_Check', 'int'), - '&PyTuple_Type': ('PyTuple_Check', 'tuple'), - '&PyList_Type': ('PyList_Check', 'list'), - '&PySet_Type': ('PySet_Check', 'set'), - '&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'), - '&PyDict_Type': ('PyDict_Check', 'dict'), - '&PyUnicode_Type': ('PyUnicode_Check', 'str'), - '&PyBytes_Type': ('PyBytes_Check', 'bytes'), - '&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'), -} - - -ConverterType = Callable[..., CConverter] -ConverterDict = dict[str, ConverterType] - -# maps strings to callables. -# these callables must be of the form: -# def foo(name, default, *, ...) -# The callable may have any number of keyword-only parameters. -# The callable must return a CConverter object. -# The callable should not call builtins.print. -converters: ConverterDict = {} - -# maps strings to callables. -# these callables follow the same rules as those for "converters" above. -# note however that they will never be called with keyword-only parameters. -legacy_converters: ConverterDict = {} - -# maps strings to callables. -# these callables must be of the form: -# def foo(*, ...) -# The callable may have any number of keyword-only parameters. -# The callable must return a CReturnConverter object. -# The callable should not call builtins.print. -ReturnConverterDict = dict[str, ReturnConverterType] -return_converters: ReturnConverterDict = {} - -TypeSet = set[bltns.type[object]] - - -class bool_converter(CConverter): - type = 'int' - default_type = bool - format_unit = 'p' - c_ignored_default = '0' - - def converter_init(self, *, accept: TypeSet = {object}) -> None: - if accept == {int}: - self.format_unit = 'i' - elif accept != {object}: - fail(f"bool_converter: illegal 'accept' argument {accept!r}") - if self.default is not unspecified and self.default is not unknown: - self.default = bool(self.default) - self.c_default = str(int(self.default)) - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'i': - return self.format_code(""" - {paramname} = PyLong_AsInt({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - elif self.format_unit == 'p': - return self.format_code(""" - {paramname} = PyObject_IsTrue({argname}); - if ({paramname} < 0) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class defining_class_converter(CConverter): - """ - A special-case converter: - this is the default converter used for the defining class. - """ - type = 'PyTypeObject *' - format_unit = '' - show_in_signature = False - - def converter_init(self, *, type: str | None = None) -> None: - self.specified_type = type - - def render(self, parameter: Parameter, data: CRenderData) -> None: - self._render_self(parameter, data) - - def set_template_dict(self, template_dict: TemplateDict) -> None: - template_dict['defining_class_name'] = self.name - - -class char_converter(CConverter): - type = 'char' - default_type = (bytes, bytearray) - format_unit = 'c' - c_ignored_default = "'\0'" - - def converter_init(self) -> None: - if isinstance(self.default, self.default_type): - if len(self.default) != 1: - fail(f"char_converter: illegal default value {self.default!r}") - - self.c_default = repr(bytes(self.default))[1:] - if self.c_default == '"\'"': - self.c_default = r"'\''" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'c': - return self.format_code(""" - if (PyBytes_Check({argname}) && PyBytes_GET_SIZE({argname}) == 1) {{{{ - {paramname} = PyBytes_AS_STRING({argname})[0]; - }}}} - else if (PyByteArray_Check({argname}) && PyByteArray_GET_SIZE({argname}) == 1) {{{{ - {paramname} = PyByteArray_AS_STRING({argname})[0]; - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'a byte string of length 1', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -@add_legacy_c_converter('B', bitwise=True) -class unsigned_char_converter(CConverter): - type = 'unsigned char' - default_type = int - format_unit = 'b' - c_ignored_default = "'\0'" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'B' - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'b': - return self.format_code(""" - {{{{ - long ival = PyLong_AsLong({argname}); - if (ival == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else if (ival < 0) {{{{ - PyErr_SetString(PyExc_OverflowError, - "unsigned byte integer is less than minimum"); - goto exit; - }}}} - else if (ival > UCHAR_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "unsigned byte integer is greater than maximum"); - goto exit; - }}}} - else {{{{ - {paramname} = (unsigned char) ival; - }}}} - }}}} - """, - argname=argname) - elif self.format_unit == 'B': - return self.format_code(""" - {{{{ - unsigned long ival = PyLong_AsUnsignedLongMask({argname}); - if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else {{{{ - {paramname} = (unsigned char) ival; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class byte_converter(unsigned_char_converter): pass - -class short_converter(CConverter): - type = 'short' - default_type = int - format_unit = 'h' - c_ignored_default = "0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'h': - return self.format_code(""" - {{{{ - long ival = PyLong_AsLong({argname}); - if (ival == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else if (ival < SHRT_MIN) {{{{ - PyErr_SetString(PyExc_OverflowError, - "signed short integer is less than minimum"); - goto exit; - }}}} - else if (ival > SHRT_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "signed short integer is greater than maximum"); - goto exit; - }}}} - else {{{{ - {paramname} = (short) ival; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_short_converter(CConverter): - type = 'unsigned short' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'H' - else: - self.converter = '_PyLong_UnsignedShort_Converter' - self.add_include('pycore_long.h', - '_PyLong_UnsignedShort_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'H': - return self.format_code(""" - {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {{{{ - unsigned long uval = PyLong_AsUnsignedLong({argname}); - if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - if (uval > USHRT_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "Python int too large for C unsigned short"); - goto exit; - }}}} - {paramname} = (unsigned short) uval; - }}}} - """, - argname=argname) - -@add_legacy_c_converter('C', accept={str}) -class int_converter(CConverter): - type = 'int' - default_type = int - format_unit = 'i' - c_ignored_default = "0" - - def converter_init( - self, *, accept: TypeSet = {int}, type: str | None = None - ) -> None: - if accept == {str}: - self.format_unit = 'C' - elif accept != {int}: - fail(f"int_converter: illegal 'accept' argument {accept!r}") - if type is not None: - self.type = type - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'i': - return self.format_code(""" - {paramname} = PyLong_AsInt({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - elif self.format_unit == 'C': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - if (PyUnicode_GET_LENGTH({argname}) != 1) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyUnicode_READ_CHAR({argname}, 0); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'a unicode character', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_int_converter(CConverter): - type = 'unsigned int' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'I' - else: - self.converter = '_PyLong_UnsignedInt_Converter' - self.add_include('pycore_long.h', - '_PyLong_UnsignedInt_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'I': - return self.format_code(""" - {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {{{{ - unsigned long uval = PyLong_AsUnsignedLong({argname}); - if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - if (uval > UINT_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "Python int too large for C unsigned int"); - goto exit; - }}}} - {paramname} = (unsigned int) uval; - }}}} - """, - argname=argname) - -class long_converter(CConverter): - type = 'long' - default_type = int - format_unit = 'l' - c_ignored_default = "0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'l': - return self.format_code(""" - {paramname} = PyLong_AsLong({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_long_converter(CConverter): - type = 'unsigned long' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'k' - else: - self.converter = '_PyLong_UnsignedLong_Converter' - self.add_include('pycore_long.h', - '_PyLong_UnsignedLong_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'k': - return self.format_code(""" - if (!PyLong_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {paramname} = PyLong_AsUnsignedLong({argname}); - if ({paramname} == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - -class long_long_converter(CConverter): - type = 'long long' - default_type = int - format_unit = 'L' - c_ignored_default = "0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'L': - return self.format_code(""" - {paramname} = PyLong_AsLongLong({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_long_long_converter(CConverter): - type = 'unsigned long long' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'K' - else: - self.converter = '_PyLong_UnsignedLongLong_Converter' - self.add_include('pycore_long.h', - '_PyLong_UnsignedLongLong_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'K': - return self.format_code(""" - if (!PyLong_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {paramname} = PyLong_AsUnsignedLongLong({argname}); - if ({paramname} == (unsigned long long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - -class Py_ssize_t_converter(CConverter): - type = 'Py_ssize_t' - c_ignored_default = "0" - - def converter_init(self, *, accept: TypeSet = {int}) -> None: - if accept == {int}: - self.format_unit = 'n' - self.default_type = int - self.add_include('pycore_abstract.h', '_PyNumber_Index()') - elif accept == {int, NoneType}: - self.converter = '_Py_convert_optional_to_ssize_t' - self.add_include('pycore_abstract.h', - '_Py_convert_optional_to_ssize_t()') - else: - fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'n': - if limited_capi: - PyNumber_Index = 'PyNumber_Index' - else: - PyNumber_Index = '_PyNumber_Index' - return self.format_code(""" - {{{{ - Py_ssize_t ival = -1; - PyObject *iobj = {PyNumber_Index}({argname}); - if (iobj != NULL) {{{{ - ival = PyLong_AsSsize_t(iobj); - Py_DECREF(iobj); - }}}} - if (ival == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - {paramname} = ival; - }}}} - """, - argname=argname, - PyNumber_Index=PyNumber_Index) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return self.format_code(""" - if ({argname} != Py_None) {{{{ - if (PyIndex_Check({argname})) {{{{ - {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'integer or None', limited_capi=limited_capi), - ) - - -class slice_index_converter(CConverter): - type = 'Py_ssize_t' - - def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: - if accept == {int}: - self.converter = '_PyEval_SliceIndexNotNone' - self.nullable = False - elif accept == {int, NoneType}: - self.converter = '_PyEval_SliceIndex' - self.nullable = True - else: - fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - if self.nullable: - return self.format_code(""" - if (!Py_IsNone({argname})) {{{{ - if (PyIndex_Check({argname})) {{{{ - {paramname} = PyNumber_AsSsize_t({argname}, NULL); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - return 0; - }}}} - }}}} - else {{{{ - PyErr_SetString(PyExc_TypeError, - "slice indices must be integers or " - "None or have an __index__ method"); - goto exit; - }}}} - }}}} - """, - argname=argname) - else: - return self.format_code(""" - if (PyIndex_Check({argname})) {{{{ - {paramname} = PyNumber_AsSsize_t({argname}, NULL); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - PyErr_SetString(PyExc_TypeError, - "slice indices must be integers or " - "have an __index__ method"); - goto exit; - }}}} - """, - argname=argname) - -class size_t_converter(CConverter): - type = 'size_t' - converter = '_PyLong_Size_t_Converter' - c_ignored_default = "0" - - def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: - self.add_include('pycore_long.h', - '_PyLong_Size_t_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'n': - return self.format_code(""" - {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {paramname} = PyLong_AsSize_t({argname}); - if ({paramname} == (size_t)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - - -class fildes_converter(CConverter): - type = 'int' - converter = '_PyLong_FileDescriptor_Converter' - - def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: - self.add_include('pycore_fileutils.h', - '_PyLong_FileDescriptor_Converter()') - - def _parse_arg(self, argname: str, displayname: str) -> str | None: - return self.format_code(""" - {paramname} = PyObject_AsFileDescriptor({argname}); - if ({paramname} == -1) {{{{ - goto exit; - }}}} - """, - argname=argname) - - -class float_converter(CConverter): - type = 'float' - default_type = float - format_unit = 'f' - c_ignored_default = "0.0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'f': - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); - }}}} - else - {{{{ - {paramname} = (float) PyFloat_AsDouble({argname}); - if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class double_converter(CConverter): - type = 'double' - default_type = float - format_unit = 'd' - c_ignored_default = "0.0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'd': - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = PyFloat_AS_DOUBLE({argname}); - }}}} - else - {{{{ - {paramname} = PyFloat_AsDouble({argname}); - if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -class Py_complex_converter(CConverter): - type = 'Py_complex' - default_type = complex - format_unit = 'D' - c_ignored_default = "{0.0, 0.0}" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'D': - return self.format_code(""" - {paramname} = PyComplex_AsCComplex({argname}); - if (PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -class object_converter(CConverter): - type = 'PyObject *' - format_unit = 'O' - - def converter_init( - self, *, - converter: str | None = None, - type: str | None = None, - subclass_of: str | None = None - ) -> None: - if converter: - if subclass_of: - fail("object: Cannot pass in both 'converter' and 'subclass_of'") - self.format_unit = 'O&' - self.converter = converter - elif subclass_of: - self.format_unit = 'O!' - self.subclass_of = subclass_of - - if type is not None: - self.type = type - - -# -# We define three conventions for buffer types in the 'accept' argument: -# -# buffer : any object supporting the buffer interface -# rwbuffer: any object supporting the buffer interface, but must be writeable -# robuffer: any object supporting the buffer interface, but must not be writeable -# - -class buffer: pass -class rwbuffer: pass -class robuffer: pass - -StrConverterKeyType = tuple[frozenset[type[object]], bool, bool] - -def str_converter_key( - types: TypeSet, encoding: bool | str | None, zeroes: bool -) -> StrConverterKeyType: - return (frozenset(types), bool(encoding), bool(zeroes)) - -str_converter_argument_map: dict[StrConverterKeyType, str] = {} - -class str_converter(CConverter): - type = 'const char *' - default_type = (str, Null, NoneType) - format_unit = 's' - - def converter_init( - self, - *, - accept: TypeSet = {str}, - encoding: str | None = None, - zeroes: bool = False - ) -> None: - - key = str_converter_key(accept, encoding, zeroes) - format_unit = str_converter_argument_map.get(key) - if not format_unit: - fail("str_converter: illegal combination of arguments", key) - - self.format_unit = format_unit - self.length = bool(zeroes) - if encoding: - if self.default not in (Null, None, unspecified): - fail("str_converter: Argument Clinic doesn't support default values for encoded strings") - self.encoding = encoding - self.type = 'char *' - # sorry, clinic can't support preallocated buffers - # for es# and et# - self.c_default = "NULL" - if NoneType in accept and self.c_default == "Py_None": - self.c_default = "NULL" - - def post_parsing(self) -> str: - if self.encoding: - name = self.name - return f"PyMem_FREE({name});\n" - else: - return "" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 's': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - Py_ssize_t {length_name}; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - if (strlen({paramname}) != (size_t){length_name}) {{{{ - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - length_name=self.length_name) - if self.format_unit == 'z': - return self.format_code(""" - if ({argname} == Py_None) {{{{ - {paramname} = NULL; - }}}} - else if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t {length_name}; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - if (strlen({paramname}) != (size_t){length_name}) {{{{ - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - }}}} - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), - length_name=self.length_name) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -# -# This is the fourth or fifth rewrite of registering all the -# string converter format units. Previous approaches hid -# bugs--generally mismatches between the semantics of the format -# unit and the arguments necessary to represent those semantics -# properly. Hopefully with this approach we'll get it 100% right. -# -# The r() function (short for "register") both registers the -# mapping from arguments to format unit *and* registers the -# legacy C converter for that format unit. -# -ConverterKeywordDict = dict[str, TypeSet | bool] - -def r(format_unit: str, - *, - accept: TypeSet, - encoding: bool = False, - zeroes: bool = False -) -> None: - if not encoding and format_unit != 's': - # add the legacy c converters here too. - # - # note: add_legacy_c_converter can't work for - # es, es#, et, or et# - # because of their extra encoding argument - # - # also don't add the converter for 's' because - # the metaclass for CConverter adds it for us. - kwargs: ConverterKeywordDict = {} - if accept != {str}: - kwargs['accept'] = accept - if zeroes: - kwargs['zeroes'] = True - added_f = functools.partial(str_converter, **kwargs) - legacy_converters[format_unit] = added_f - - d = str_converter_argument_map - key = str_converter_key(accept, encoding, zeroes) - if key in d: - sys.exit("Duplicate keys specified for str_converter_argument_map!") - d[key] = format_unit - -r('es', encoding=True, accept={str}) -r('es#', encoding=True, zeroes=True, accept={str}) -r('et', encoding=True, accept={bytes, bytearray, str}) -r('et#', encoding=True, zeroes=True, accept={bytes, bytearray, str}) -r('s', accept={str}) -r('s#', zeroes=True, accept={robuffer, str}) -r('y', accept={robuffer}) -r('y#', zeroes=True, accept={robuffer}) -r('z', accept={str, NoneType}) -r('z#', zeroes=True, accept={robuffer, str, NoneType}) -del r - - -class PyBytesObject_converter(CConverter): - type = 'PyBytesObject *' - format_unit = 'S' - # accept = {bytes} - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'S': - return self.format_code(""" - if (!PyBytes_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = ({type}){argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'bytes', limited_capi=limited_capi), - type=self.type) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class PyByteArrayObject_converter(CConverter): - type = 'PyByteArrayObject *' - format_unit = 'Y' - # accept = {bytearray} - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'Y': - return self.format_code(""" - if (!PyByteArray_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = ({type}){argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'bytearray', limited_capi=limited_capi), - type=self.type) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unicode_converter(CConverter): - type = 'PyObject *' - default_type = (str, Null, NoneType) - format_unit = 'U' - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'U': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = {argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -@add_legacy_c_converter('u') -@add_legacy_c_converter('u#', zeroes=True) -@add_legacy_c_converter('Z', accept={str, NoneType}) -@add_legacy_c_converter('Z#', accept={str, NoneType}, zeroes=True) -class Py_UNICODE_converter(CConverter): - type = 'const wchar_t *' - default_type = (str, Null, NoneType) - - def converter_init( - self, *, - accept: TypeSet = {str}, - zeroes: bool = False - ) -> None: - format_unit = 'Z' if accept=={str, NoneType} else 'u' - if zeroes: - format_unit += '#' - self.length = True - self.format_unit = format_unit - else: - self.accept = accept - if accept == {str}: - self.converter = '_PyUnicode_WideCharString_Converter' - elif accept == {str, NoneType}: - self.converter = '_PyUnicode_WideCharString_Opt_Converter' - else: - fail(f"Py_UNICODE_converter: illegal 'accept' argument {accept!r}") - self.c_default = "NULL" - - def cleanup(self) -> str: - if self.length: - return "" - else: - return f"""PyMem_Free((void *){self.parser_name});\n""" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if not self.length: - if self.accept == {str}: - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyUnicode_AsWideCharString({argname}, NULL); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - ) - elif self.accept == {str, NoneType}: - return self.format_code(""" - if ({argname} == Py_None) {{{{ - {paramname} = NULL; - }}}} - else if (PyUnicode_Check({argname})) {{{{ - {paramname} = PyUnicode_AsWideCharString({argname}, NULL); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -@add_legacy_c_converter('s*', accept={str, buffer}) -@add_legacy_c_converter('z*', accept={str, buffer, NoneType}) -@add_legacy_c_converter('w*', accept={rwbuffer}) -class Py_buffer_converter(CConverter): - type = 'Py_buffer' - format_unit = 'y*' - impl_by_reference = True - c_ignored_default = "{NULL, NULL}" - - def converter_init(self, *, accept: TypeSet = {buffer}) -> None: - if self.default not in (unspecified, None): - fail("The only legal default value for Py_buffer is None.") - - self.c_default = self.c_ignored_default - - if accept == {str, buffer, NoneType}: - format_unit = 'z*' - elif accept == {str, buffer}: - format_unit = 's*' - elif accept == {buffer}: - format_unit = 'y*' - elif accept == {rwbuffer}: - format_unit = 'w*' - else: - fail("Py_buffer_converter: illegal combination of arguments") - - self.format_unit = format_unit - - def cleanup(self) -> str: - name = self.name - return "".join(["if (", name, ".obj) {\n PyBuffer_Release(&", name, ");\n}\n"]) - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - # PyBUF_SIMPLE guarantees that the format units of the buffers are C-contiguous. - if self.format_unit == 'y*': - return self.format_code(""" - if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), - ) - elif self.format_unit == 's*': - return self.format_code(""" - if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t len; - const char *ptr = PyUnicode_AsUTF8AndSize({argname}, &len); - if (ptr == NULL) {{{{ - goto exit; - }}}} - if (PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) {{{{ - goto exit; - }}}} - }}}} - else {{{{ /* any bytes-like object */ - if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ - goto exit; - }}}} - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), - ) - elif self.format_unit == 'w*': - return self.format_code(""" - if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_WRITABLE) < 0) {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'read-write bytes-like object', limited_capi=limited_capi), - bad_argument2=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -def correct_name_for_self( - f: Function -) -> tuple[str, str]: - if f.kind in {CALLABLE, METHOD_INIT, GETTER, SETTER}: - if f.cls: - return "PyObject *", "self" - return "PyObject *", "module" - if f.kind is STATIC_METHOD: - return "void *", "null" - if f.kind in (CLASS_METHOD, METHOD_NEW): - return "PyTypeObject *", "type" - raise AssertionError(f"Unhandled type of function f: {f.kind!r}") - -def required_type_for_self_for_parser( - f: Function -) -> str | None: - type, _ = correct_name_for_self(f) - if f.kind in (METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD): - return type - return None - - -class self_converter(CConverter): - """ - A special-case converter: - this is the default converter used for "self". - """ - type: str | None = None - format_unit = '' - - def converter_init(self, *, type: str | None = None) -> None: - self.specified_type = type - - def pre_render(self) -> None: - f = self.function - default_type, default_name = correct_name_for_self(f) - self.signature_name = default_name - self.type = self.specified_type or self.type or default_type - - kind = self.function.kind - - if kind is STATIC_METHOD or kind.new_or_init: - self.show_in_signature = False - - # tp_new (METHOD_NEW) functions are of type newfunc: - # typedef PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *); - # - # tp_init (METHOD_INIT) functions are of type initproc: - # typedef int (*initproc)(PyObject *, PyObject *, PyObject *); - # - # All other functions generated by Argument Clinic are stored in - # PyMethodDef structures, in the ml_meth slot, which is of type PyCFunction: - # typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); - # However! We habitually cast these functions to PyCFunction, - # since functions that accept keyword arguments don't fit this signature - # but are stored there anyway. So strict type equality isn't important - # for these functions. - # - # So: - # - # * The name of the first parameter to the impl and the parsing function will always - # be self.name. - # - # * The type of the first parameter to the impl will always be of self.type. - # - # * If the function is neither tp_new (METHOD_NEW) nor tp_init (METHOD_INIT): - # * The type of the first parameter to the parsing function is also self.type. - # This means that if you step into the parsing function, your "self" parameter - # is of the correct type, which may make debugging more pleasant. - # - # * Else if the function is tp_new (METHOD_NEW): - # * The type of the first parameter to the parsing function is "PyTypeObject *", - # so the type signature of the function call is an exact match. - # * If self.type != "PyTypeObject *", we cast the first parameter to self.type - # in the impl call. - # - # * Else if the function is tp_init (METHOD_INIT): - # * The type of the first parameter to the parsing function is "PyObject *", - # so the type signature of the function call is an exact match. - # * If self.type != "PyObject *", we cast the first parameter to self.type - # in the impl call. - - @property - def parser_type(self) -> str: - assert self.type is not None - return required_type_for_self_for_parser(self.function) or self.type - - def render(self, parameter: Parameter, data: CRenderData) -> None: - """ - parameter is a clinic.Parameter instance. - data is a CRenderData instance. - """ - if self.function.kind is STATIC_METHOD: - return - - self._render_self(parameter, data) - - if self.type != self.parser_type: - # insert cast to impl_argument[0], aka self. - # we know we're in the first slot in all the CRenderData lists, - # because we render parameters in order, and self is always first. - assert len(data.impl_arguments) == 1 - assert data.impl_arguments[0] == self.name - assert self.type is not None - data.impl_arguments[0] = '(' + self.type + ")" + data.impl_arguments[0] - - def set_template_dict(self, template_dict: TemplateDict) -> None: - template_dict['self_name'] = self.name - template_dict['self_type'] = self.parser_type - kind = self.function.kind - cls = self.function.cls - - if kind.new_or_init and cls and cls.typedef: - if kind is METHOD_NEW: - type_check = ( - '({0} == base_tp || {0}->tp_init == base_tp->tp_init)' - ).format(self.name) - else: - type_check = ('(Py_IS_TYPE({0}, base_tp) ||\n ' - ' Py_TYPE({0})->tp_new == base_tp->tp_new)' - ).format(self.name) - - line = f'{type_check} &&\n ' - template_dict['self_type_check'] = line - - type_object = cls.type_object - type_ptr = f'PyTypeObject *base_tp = {type_object};' - template_dict['base_type_ptr'] = type_ptr - - -def add_c_return_converter( - f: ReturnConverterType, - name: str | None = None -) -> ReturnConverterType: - if not name: - name = f.__name__ - if not name.endswith('_return_converter'): - return f - name = name.removesuffix('_return_converter') - return_converters[name] = f - return f - - -class CReturnConverterAutoRegister(type): - def __init__( - cls: ReturnConverterType, - name: str, - bases: tuple[type[object], ...], - classdict: dict[str, Any] - ) -> None: - add_c_return_converter(cls) - - -class CReturnConverter(metaclass=CReturnConverterAutoRegister): - - # The C type to use for this variable. - # 'type' should be a Python string specifying the type, e.g. "int". - # If this is a pointer type, the type string should end with ' *'. - type = 'PyObject *' - - # The Python default value for this parameter, as a Python value. - # Or the magic value "unspecified" if there is no default. - default: object = None - - def __init__( - self, - *, - py_default: str | None = None, - **kwargs: Any - ) -> None: - self.py_default = py_default - try: - self.return_converter_init(**kwargs) - except TypeError as e: - s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) - sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) - - def return_converter_init(self) -> None: ... - - def declare(self, data: CRenderData) -> None: - line: list[str] = [] - add = line.append - add(self.type) - if not self.type.endswith('*'): - add(' ') - add(data.converter_retval + ';') - data.declarations.append(''.join(line)) - data.return_value = data.converter_retval - - def err_occurred_if( - self, - expr: str, - data: CRenderData - ) -> None: - line = f'if (({expr}) && PyErr_Occurred()) {{\n goto exit;\n}}\n' - data.return_conversion.append(line) - - def err_occurred_if_null_pointer( - self, - variable: str, - data: CRenderData - ) -> None: - line = f'if ({variable} == NULL) {{\n goto exit;\n}}\n' - data.return_conversion.append(line) - - def render( - self, - function: Function, - data: CRenderData - ) -> None: ... - - -add_c_return_converter(CReturnConverter, 'object') - - -class bool_return_converter(CReturnConverter): - type = 'int' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: - self.declare(data) - self.err_occurred_if(f"{data.converter_retval} == -1", data) - data.return_conversion.append( - f'return_value = PyBool_FromLong((long){data.converter_retval});\n' - ) - - -class long_return_converter(CReturnConverter): - type = 'long' - conversion_fn = 'PyLong_FromLong' - cast = '' - unsigned_cast = '' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: - self.declare(data) - self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) - data.return_conversion.append( - f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' - ) - - -class int_return_converter(long_return_converter): - type = 'int' - cast = '(long)' - - -class init_return_converter(long_return_converter): - """ - Special return converter for __init__ functions. - """ - type = 'int' - cast = '(long)' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: ... - - -class unsigned_long_return_converter(long_return_converter): - type = 'unsigned long' - conversion_fn = 'PyLong_FromUnsignedLong' - unsigned_cast = '(unsigned long)' - - -class unsigned_int_return_converter(unsigned_long_return_converter): - type = 'unsigned int' - cast = '(unsigned long)' - unsigned_cast = '(unsigned int)' - - -class Py_ssize_t_return_converter(long_return_converter): - type = 'Py_ssize_t' - conversion_fn = 'PyLong_FromSsize_t' - - -class size_t_return_converter(long_return_converter): - type = 'size_t' - conversion_fn = 'PyLong_FromSize_t' - unsigned_cast = '(size_t)' - - -class double_return_converter(CReturnConverter): - type = 'double' - cast = '' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: - self.declare(data) - self.err_occurred_if(f"{data.converter_retval} == -1.0", data) - data.return_conversion.append( - f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' - ) - - -class float_return_converter(double_return_converter): - type = 'float' - cast = '(double)' - - -def eval_ast_expr( - node: ast.expr, - globals: dict[str, Any], - *, - filename: str = '-' -) -> Any: - """ - Takes an ast.Expr node. Compiles it into a function object, - then calls the function object with 0 arguments. - Returns the result of that function call. - - globals represents the globals dict the expression - should see. (There's no equivalent for "locals" here.) - """ - - if isinstance(node, ast.Expr): - node = node.value - - expr = ast.Expression(node) - co = compile(expr, filename, 'eval') - fn = FunctionType(co, globals) - return fn() - - -class IndentStack: - def __init__(self) -> None: - self.indents: list[int] = [] - self.margin: str | None = None - - def _ensure(self) -> None: - if not self.indents: - fail('IndentStack expected indents, but none are defined.') - - def measure(self, line: str) -> int: - """ - Returns the length of the line's margin. - """ - if '\t' in line: - fail('Tab characters are illegal in the Argument Clinic DSL.') - stripped = line.lstrip() - if not len(stripped): - # we can't tell anything from an empty line - # so just pretend it's indented like our current indent - self._ensure() - return self.indents[-1] - return len(line) - len(stripped) - - def infer(self, line: str) -> int: - """ - Infer what is now the current margin based on this line. - Returns: - 1 if we have indented (or this is the first margin) - 0 if the margin has not changed - -N if we have dedented N times - """ - indent = self.measure(line) - margin = ' ' * indent - if not self.indents: - self.indents.append(indent) - self.margin = margin - return 1 - current = self.indents[-1] - if indent == current: - return 0 - if indent > current: - self.indents.append(indent) - self.margin = margin - return 1 - # indent < current - if indent not in self.indents: - fail("Illegal outdent.") - outdent_count = 0 - while indent != current: - self.indents.pop() - current = self.indents[-1] - outdent_count -= 1 - self.margin = margin - return outdent_count - - @property - def depth(self) -> int: - """ - Returns how many margins are currently defined. - """ - return len(self.indents) - - def dedent(self, line: str) -> str: - """ - Dedents a line by the currently defined margin. - """ - assert self.margin is not None, "Cannot call .dedent() before calling .infer()" - margin = self.margin - indent = self.indents[-1] - if not line.startswith(margin): - fail('Cannot dedent; line does not start with the previous margin.') - return line[indent:] - - -StateKeeper = Callable[[str], None] -ConverterArgs = dict[str, Any] - -class ParamState(enum.IntEnum): - """Parameter parsing state. - - [ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] <- line - 01 2 3 4 5 6 <- state transitions - """ - # Before we've seen anything. - # Legal transitions: to LEFT_SQUARE_BEFORE or REQUIRED - START = 0 - - # Left square backets before required params. - LEFT_SQUARE_BEFORE = 1 - - # In a group, before required params. - GROUP_BEFORE = 2 - - # Required params, positional-or-keyword or positional-only (we - # don't know yet). Renumber left groups! - REQUIRED = 3 - - # Positional-or-keyword or positional-only params that now must have - # default values. - OPTIONAL = 4 - - # In a group, after required params. - GROUP_AFTER = 5 - - # Right square brackets after required params. - RIGHT_SQUARE_AFTER = 6 - - -class FunctionNames(NamedTuple): - full_name: str - c_basename: str - - -class DSLParser: - function: Function | None - state: StateKeeper - keyword_only: bool - positional_only: bool - deprecated_positional: VersionTuple | None - deprecated_keyword: VersionTuple | None - group: int - parameter_state: ParamState - indent: IndentStack - kind: FunctionKind - coexist: bool - forced_text_signature: str | None - parameter_continuation: str - preserve_output: bool - critical_section: bool - target_critical_section: list[str] - from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') - - def __init__(self, clinic: Clinic) -> None: - self.clinic = clinic - - self.directives = {} - for name in dir(self): - # functions that start with directive_ are added to directives - _, s, key = name.partition("directive_") - if s: - self.directives[key] = getattr(self, name) - - # functions that start with at_ are too, with an @ in front - _, s, key = name.partition("at_") - if s: - self.directives['@' + key] = getattr(self, name) - - self.reset() - - def reset(self) -> None: - self.function = None - self.state = self.state_dsl_start - self.keyword_only = False - self.positional_only = False - self.deprecated_positional = None - self.deprecated_keyword = None - self.group = 0 - self.parameter_state: ParamState = ParamState.START - self.indent = IndentStack() - self.kind = CALLABLE - self.coexist = False - self.forced_text_signature = None - self.parameter_continuation = '' - self.preserve_output = False - self.critical_section = False - self.target_critical_section = [] - - def directive_module(self, name: str) -> None: - fields = name.split('.')[:-1] - module, cls = self.clinic._module_and_class(fields) - if cls: - fail("Can't nest a module inside a class!") - - if name in module.modules: - fail(f"Already defined module {name!r}!") - - m = Module(name, module) - module.modules[name] = m - self.block.signatures.append(m) - - def directive_class( - self, - name: str, - typedef: str, - type_object: str - ) -> None: - fields = name.split('.') - name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - parent = cls or module - if name in parent.classes: - fail(f"Already defined class {name!r}!") - - c = Class(name, module, cls, typedef, type_object) - parent.classes[name] = c - self.block.signatures.append(c) - - def directive_set(self, name: str, value: str) -> None: - if name not in ("line_prefix", "line_suffix"): - fail(f"unknown variable {name!r}") - - value = value.format_map({ - 'block comment start': '/*', - 'block comment end': '*/', - }) - - self.clinic.__dict__[name] = value - - def directive_destination( - self, - name: str, - command: str, - *args: str - ) -> None: - match command: - case "new": - self.clinic.add_destination(name, *args) - case "clear": - self.clinic.get_destination(name).clear() - case _: - fail(f"unknown destination command {command!r}") - - - def directive_output( - self, - command_or_name: str, - destination: str = '' - ) -> None: - fd = self.clinic.destination_buffers - - if command_or_name == "preset": - preset = self.clinic.presets.get(destination) - if not preset: - fail(f"Unknown preset {destination!r}!") - fd.update(preset) - return - - if command_or_name == "push": - self.clinic.destination_buffers_stack.append(fd.copy()) - return - - if command_or_name == "pop": - if not self.clinic.destination_buffers_stack: - fail("Can't 'output pop', stack is empty!") - previous_fd = self.clinic.destination_buffers_stack.pop() - fd.update(previous_fd) - return - - # secret command for debugging! - if command_or_name == "print": - self.block.output.append(pprint.pformat(fd)) - self.block.output.append('\n') - return - - d = self.clinic.get_destination_buffer(destination) - - if command_or_name == "everything": - for name in list(fd): - fd[name] = d - return - - if command_or_name not in fd: - allowed = ["preset", "push", "pop", "print", "everything"] - allowed.extend(fd) - fail(f"Invalid command or destination name {command_or_name!r}. " - "Must be one of:\n -", - "\n - ".join([repr(word) for word in allowed])) - fd[command_or_name] = d - - def directive_dump(self, name: str) -> None: - self.block.output.append(self.clinic.get_destination(name).dump()) - - def directive_printout(self, *args: str) -> None: - self.block.output.append(' '.join(args)) - self.block.output.append('\n') - - def directive_preserve(self) -> None: - if self.preserve_output: - fail("Can't have 'preserve' twice in one block!") - self.preserve_output = True - - def at_classmethod(self) -> None: - if self.kind is not CALLABLE: - fail("Can't set @classmethod, function is not a normal callable") - self.kind = CLASS_METHOD - - def at_critical_section(self, *args: str) -> None: - if len(args) > 2: - fail("Up to 2 critical section variables are supported") - self.target_critical_section.extend(args) - self.critical_section = True - - def at_getter(self) -> None: - match self.kind: - case FunctionKind.GETTER: - fail("Cannot apply @getter twice to the same function!") - case FunctionKind.SETTER: - fail("Cannot apply both @getter and @setter to the same function!") - case _: - self.kind = FunctionKind.GETTER - - def at_setter(self) -> None: - match self.kind: - case FunctionKind.SETTER: - fail("Cannot apply @setter twice to the same function!") - case FunctionKind.GETTER: - fail("Cannot apply both @getter and @setter to the same function!") - case _: - self.kind = FunctionKind.SETTER - - def at_staticmethod(self) -> None: - if self.kind is not CALLABLE: - fail("Can't set @staticmethod, function is not a normal callable") - self.kind = STATIC_METHOD - - def at_coexist(self) -> None: - if self.coexist: - fail("Called @coexist twice!") - self.coexist = True - - def at_text_signature(self, text_signature: str) -> None: - if self.forced_text_signature: - fail("Called @text_signature twice!") - self.forced_text_signature = text_signature - - def parse(self, block: Block) -> None: - self.reset() - self.block = block - self.saved_output = self.block.output - block.output = [] - block_start = self.clinic.block_parser.line_number - lines = block.input.split('\n') - for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): - if '\t' in line: - fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', - line_number=block_start) - try: - self.state(line) - except ClinicError as exc: - exc.lineno = line_number - raise - - self.do_post_block_processing_cleanup(line_number) - block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) - - if self.preserve_output: - if block.output: - fail("'preserve' only works for blocks that don't produce any output!") - block.output = self.saved_output - - def in_docstring(self) -> bool: - """Return true if we are processing a docstring.""" - return self.state in { - self.state_parameter_docstring, - self.state_function_docstring, - } - - def valid_line(self, line: str) -> bool: - # ignore comment-only lines - if line.lstrip().startswith('#'): - return False - - # Ignore empty lines too - # (but not in docstring sections!) - if not self.in_docstring() and not line.strip(): - return False - - return True - - def next( - self, - state: StateKeeper, - line: str | None = None - ) -> None: - self.state = state - if line is not None: - self.state(line) - - def state_dsl_start(self, line: str) -> None: - if not self.valid_line(line): - return - - # is it a directive? - fields = shlex.split(line) - directive_name = fields[0] - directive = self.directives.get(directive_name, None) - if directive: - try: - directive(*fields[1:]) - except TypeError as e: - fail(str(e)) - return - - self.next(self.state_modulename_name, line) - - @staticmethod - def parse_function_names(line: str) -> FunctionNames: - left, as_, right = line.partition(' as ') - full_name = left.strip() - c_basename = right.strip() - if as_ and not c_basename: - fail("No C basename provided after 'as' keyword") - if not c_basename: - fields = full_name.split(".") - if fields[-1] == '__new__': - fields.pop() - c_basename = "_".join(fields) - if not is_legal_py_identifier(full_name): - fail(f"Illegal function name: {full_name!r}") - if not is_legal_c_identifier(c_basename): - fail(f"Illegal C basename: {c_basename!r}") - return FunctionNames(full_name=full_name, c_basename=c_basename) - - def update_function_kind(self, fullname: str) -> None: - fields = fullname.split('.') - name = fields.pop() - _, cls = self.clinic._module_and_class(fields) - if name in unsupported_special_methods: - fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") - - if name == '__new__': - if (self.kind is CLASS_METHOD) and cls: - self.kind = METHOD_NEW - else: - fail("'__new__' must be a class method!") - elif name == '__init__': - if (self.kind is CALLABLE) and cls: - self.kind = METHOD_INIT - else: - fail( - "'__init__' must be a normal method; " - f"got '{self.kind}'!" - ) - - def state_modulename_name(self, line: str) -> None: - # looking for declaration, which establishes the leftmost column - # line should be - # modulename.fnname [as c_basename] [-> return annotation] - # square brackets denote optional syntax. - # - # alternatively: - # modulename.fnname [as c_basename] = modulename.existing_fn_name - # clones the parameters and return converter from that - # function. you can't modify them. you must enter a - # new docstring. - # - # (but we might find a directive first!) - # - # this line is permitted to start with whitespace. - # we'll call this number of spaces F (for "function"). - - assert self.valid_line(line) - self.indent.infer(line) - - # are we cloning? - before, equals, existing = line.rpartition('=') - if equals: - full_name, c_basename = self.parse_function_names(before) - existing = existing.strip() - if is_legal_py_identifier(existing): - # we're cloning! - fields = [x.strip() for x in existing.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - for existing_function in (cls or module).functions: - if existing_function.name == function_name: - break - else: - print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) - print(f"{(cls or module).functions=}", file=sys.stderr) - fail(f"Couldn't find existing function {existing!r}!") - - fields = [x.strip() for x in full_name.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - self.update_function_kind(full_name) - overrides: dict[str, Any] = { - "name": function_name, - "full_name": full_name, - "module": module, - "cls": cls, - "c_basename": c_basename, - "docstring": "", - } - if not (existing_function.kind is self.kind and - existing_function.coexist == self.coexist): - # Allow __new__ or __init__ methods. - if existing_function.kind.new_or_init: - overrides["kind"] = self.kind - # Future enhancement: allow custom return converters - overrides["return_converter"] = CReturnConverter() - else: - fail("'kind' of function and cloned function don't match! " - "(@classmethod/@staticmethod/@coexist)") - function = existing_function.copy(**overrides) - self.function = function - self.block.signatures.append(function) - (cls or module).functions.append(function) - self.next(self.state_function_docstring) - return - - line, _, returns = line.partition('->') - returns = returns.strip() - full_name, c_basename = self.parse_function_names(line) - - return_converter = None - if returns: - if self.kind in {GETTER, SETTER}: - fail(f"@{self.kind.name.lower()} method cannot define a return type") - ast_input = f"def x() -> {returns}: pass" - try: - module_node = ast.parse(ast_input) - except SyntaxError: - fail(f"Badly formed annotation for {full_name!r}: {returns!r}") - function_node = module_node.body[0] - assert isinstance(function_node, ast.FunctionDef) - try: - name, legacy, kwargs = self.parse_converter(function_node.returns) - if legacy: - fail(f"Legacy converter {name!r} not allowed as a return converter") - if name not in return_converters: - fail(f"No available return converter called {name!r}") - return_converter = return_converters[name](**kwargs) - except ValueError: - fail(f"Badly formed annotation for {full_name!r}: {returns!r}") - - fields = [x.strip() for x in full_name.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - if self.kind in {GETTER, SETTER}: - if not cls: - fail("@getter and @setter must be methods") - - self.update_function_kind(full_name) - if self.kind is METHOD_INIT and not return_converter: - return_converter = init_return_converter() - - if not return_converter: - return_converter = CReturnConverter() - - self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, - return_converter=return_converter, kind=self.kind, coexist=self.coexist, - critical_section=self.critical_section, - target_critical_section=self.target_critical_section) - self.block.signatures.append(self.function) - - # insert a self converter automatically - type, name = correct_name_for_self(self.function) - kwargs = {} - if cls and type == "PyObject *": - kwargs['type'] = cls.typedef - sc = self.function.self_converter = self_converter(name, name, self.function, **kwargs) - p_self = Parameter(name, inspect.Parameter.POSITIONAL_ONLY, - function=self.function, converter=sc) - self.function.parameters[name] = p_self - - (cls or module).functions.append(self.function) - self.next(self.state_parameters_start) - - # Now entering the parameters section. The rules, formally stated: - # - # * All lines must be indented with spaces only. - # * The first line must be a parameter declaration. - # * The first line must be indented. - # * This first line establishes the indent for parameters. - # * We'll call this number of spaces P (for "parameter"). - # * Thenceforth: - # * Lines indented with P spaces specify a parameter. - # * Lines indented with > P spaces are docstrings for the previous - # parameter. - # * We'll call this number of spaces D (for "docstring"). - # * All subsequent lines indented with >= D spaces are stored as - # part of the per-parameter docstring. - # * All lines will have the first D spaces of the indent stripped - # before they are stored. - # * It's illegal to have a line starting with a number of spaces X - # such that P < X < D. - # * A line with < P spaces is the first line of the function - # docstring, which ends processing for parameters and per-parameter - # docstrings. - # * The first line of the function docstring must be at the same - # indent as the function declaration. - # * It's illegal to have any line in the parameters section starting - # with X spaces such that F < X < P. (As before, F is the indent - # of the function declaration.) - # - # Also, currently Argument Clinic places the following restrictions on groups: - # * Each group must contain at least one parameter. - # * Each group may contain at most one group, which must be the furthest - # thing in the group from the required parameters. (The nested group - # must be the first in the group when it's before the required - # parameters, and the last thing in the group when after the required - # parameters.) - # * There may be at most one (top-level) group to the left or right of - # the required parameters. - # * You must specify a slash, and it must be after all parameters. - # (In other words: either all parameters are positional-only, - # or none are.) - # - # Said another way: - # * Each group must contain at least one parameter. - # * All left square brackets before the required parameters must be - # consecutive. (You can't have a left square bracket followed - # by a parameter, then another left square bracket. You can't - # have a left square bracket, a parameter, a right square bracket, - # and then a left square bracket.) - # * All right square brackets after the required parameters must be - # consecutive. - # - # These rules are enforced with a single state variable: - # "parameter_state". (Previously the code was a miasma of ifs and - # separate boolean state variables.) The states are defined in the - # ParamState class. - - def state_parameters_start(self, line: str) -> None: - if not self.valid_line(line): - return - - # if this line is not indented, we have no parameters - if not self.indent.infer(line): - return self.next(self.state_function_docstring, line) - - self.parameter_continuation = '' - return self.next(self.state_parameter, line) - - - def to_required(self) -> None: - """ - Transition to the "required" parameter state. - """ - if self.parameter_state is not ParamState.REQUIRED: - self.parameter_state = ParamState.REQUIRED - assert self.function is not None - for p in self.function.parameters.values(): - p.group = -p.group - - def state_parameter(self, line: str) -> None: - assert isinstance(self.function, Function) - - if not self.valid_line(line): - return - - if self.parameter_continuation: - line = self.parameter_continuation + ' ' + line.lstrip() - self.parameter_continuation = '' - - assert self.indent.depth == 2 - indent = self.indent.infer(line) - if indent == -1: - # we outdented, must be to definition column - return self.next(self.state_function_docstring, line) - - if indent == 1: - # we indented, must be to new parameter docstring column - return self.next(self.state_parameter_docstring_start, line) - - line = line.rstrip() - if line.endswith('\\'): - self.parameter_continuation = line[:-1] - return - - line = line.lstrip() - version: VersionTuple | None = None - match = self.from_version_re.fullmatch(line) - if match: - line = match[1] - version = self.parse_version(match[2]) - - func = self.function - match line: - case '*': - self.parse_star(func, version) - case '[': - self.parse_opening_square_bracket(func) - case ']': - self.parse_closing_square_bracket(func) - case '/': - self.parse_slash(func, version) - case param: - self.parse_parameter(param) - - def parse_parameter(self, line: str) -> None: - assert self.function is not None - - match self.parameter_state: - case ParamState.START | ParamState.REQUIRED: - self.to_required() - case ParamState.LEFT_SQUARE_BEFORE: - self.parameter_state = ParamState.GROUP_BEFORE - case ParamState.GROUP_BEFORE: - if not self.group: - self.to_required() - case ParamState.GROUP_AFTER | ParamState.OPTIONAL: - pass - case st: - fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.a)") - - # handle "as" for parameters too - c_name = None - name, have_as_token, trailing = line.partition(' as ') - if have_as_token: - name = name.strip() - if ' ' not in name: - fields = trailing.strip().split(' ') - if not fields: - fail("Invalid 'as' clause!") - c_name = fields[0] - if c_name.endswith(':'): - name += ':' - c_name = c_name[:-1] - fields[0] = name - line = ' '.join(fields) - - default: str | None - base, equals, default = line.rpartition('=') - if not equals: - base = default - default = None - - module = None - try: - ast_input = f"def x({base}): pass" - module = ast.parse(ast_input) - except SyntaxError: - try: - # the last = was probably inside a function call, like - # c: int(accept={str}) - # so assume there was no actual default value. - default = None - ast_input = f"def x({line}): pass" - module = ast.parse(ast_input) - except SyntaxError: - pass - if not module: - fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", - repr(line)) - - function = module.body[0] - assert isinstance(function, ast.FunctionDef) - function_args = function.args - - if len(function_args.args) > 1: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (comma?): {line!r}") - if function_args.defaults or function_args.kw_defaults: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (default value?): {line!r}") - if function_args.kwarg: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (**kwargs?): {line!r}") - - if function_args.vararg: - is_vararg = True - parameter = function_args.vararg - else: - is_vararg = False - parameter = function_args.args[0] - - parameter_name = parameter.arg - name, legacy, kwargs = self.parse_converter(parameter.annotation) - - value: object - if not default: - if self.parameter_state is ParamState.OPTIONAL: - fail(f"Can't have a parameter without a default ({parameter_name!r}) " - "after a parameter with a default!") - if is_vararg: - value = NULL - kwargs.setdefault('c_default', "NULL") - else: - value = unspecified - if 'py_default' in kwargs: - fail("You can't specify py_default without specifying a default value!") - else: - if is_vararg: - fail("Vararg can't take a default value!") - - if self.parameter_state is ParamState.REQUIRED: - self.parameter_state = ParamState.OPTIONAL - default = default.strip() - bad = False - ast_input = f"x = {default}" - try: - module = ast.parse(ast_input) - - if 'c_default' not in kwargs: - # we can only represent very simple data values in C. - # detect whether default is okay, via a denylist - # of disallowed ast nodes. - class DetectBadNodes(ast.NodeVisitor): - bad = False - def bad_node(self, node: ast.AST) -> None: - self.bad = True - - # inline function call - visit_Call = bad_node - # inline if statement ("x = 3 if y else z") - visit_IfExp = bad_node - - # comprehensions and generator expressions - visit_ListComp = visit_SetComp = bad_node - visit_DictComp = visit_GeneratorExp = bad_node - - # literals for advanced types - visit_Dict = visit_Set = bad_node - visit_List = visit_Tuple = bad_node - - # "starred": "a = [1, 2, 3]; *a" - visit_Starred = bad_node - - denylist = DetectBadNodes() - denylist.visit(module) - bad = denylist.bad - else: - # if they specify a c_default, we can be more lenient about the default value. - # but at least make an attempt at ensuring it's a valid expression. - try: - value = eval(default) - except NameError: - pass # probably a named constant - except Exception as e: - fail("Malformed expression given as default value " - f"{default!r} caused {e!r}") - else: - if value is unspecified: - fail("'unspecified' is not a legal default value!") - if bad: - fail(f"Unsupported expression as default value: {default!r}") - - assignment = module.body[0] - assert isinstance(assignment, ast.Assign) - expr = assignment.value - # mild hack: explicitly support NULL as a default value - c_default: str | None - if isinstance(expr, ast.Name) and expr.id == 'NULL': - value = NULL - py_default = '' - c_default = "NULL" - elif (isinstance(expr, ast.BinOp) or - (isinstance(expr, ast.UnaryOp) and - not (isinstance(expr.operand, ast.Constant) and - type(expr.operand.value) in {int, float, complex}) - )): - c_default = kwargs.get("c_default") - if not (isinstance(c_default, str) and c_default): - fail(f"When you specify an expression ({default!r}) " - f"as your default value, " - f"you MUST specify a valid c_default.", - ast.dump(expr)) - py_default = default - value = unknown - elif isinstance(expr, ast.Attribute): - a = [] - n: ast.expr | ast.Attribute = expr - while isinstance(n, ast.Attribute): - a.append(n.attr) - n = n.value - if not isinstance(n, ast.Name): - fail(f"Unsupported default value {default!r} " - "(looked like a Python constant)") - a.append(n.id) - py_default = ".".join(reversed(a)) - - c_default = kwargs.get("c_default") - if not (isinstance(c_default, str) and c_default): - fail(f"When you specify a named constant ({py_default!r}) " - "as your default value, " - "you MUST specify a valid c_default.") - - try: - value = eval(py_default) - except NameError: - value = unknown - else: - value = ast.literal_eval(expr) - py_default = repr(value) - if isinstance(value, (bool, NoneType)): - c_default = "Py_" + py_default - elif isinstance(value, str): - c_default = libclinic.c_repr(value) - else: - c_default = py_default - - except SyntaxError as e: - fail(f"Syntax error: {e.text!r}") - except (ValueError, AttributeError): - value = unknown - c_default = kwargs.get("c_default") - py_default = default - if not (isinstance(c_default, str) and c_default): - fail("When you specify a named constant " - f"({py_default!r}) as your default value, " - "you MUST specify a valid c_default.") - - kwargs.setdefault('c_default', c_default) - kwargs.setdefault('py_default', py_default) - - dict = legacy_converters if legacy else converters - legacy_str = "legacy " if legacy else "" - if name not in dict: - fail(f'{name!r} is not a valid {legacy_str}converter') - # if you use a c_name for the parameter, we just give that name to the converter - # but the parameter object gets the python name - converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) - - kind: inspect._ParameterKind - if is_vararg: - kind = inspect.Parameter.VAR_POSITIONAL - elif self.keyword_only: - kind = inspect.Parameter.KEYWORD_ONLY - else: - kind = inspect.Parameter.POSITIONAL_OR_KEYWORD - - if isinstance(converter, self_converter): - if len(self.function.parameters) == 1: - if self.parameter_state is not ParamState.REQUIRED: - fail("A 'self' parameter cannot be marked optional.") - if value is not unspecified: - fail("A 'self' parameter cannot have a default value.") - if self.group: - fail("A 'self' parameter cannot be in an optional group.") - kind = inspect.Parameter.POSITIONAL_ONLY - self.parameter_state = ParamState.START - self.function.parameters.clear() - else: - fail("A 'self' parameter, if specified, must be the " - "very first thing in the parameter block.") - - if isinstance(converter, defining_class_converter): - _lp = len(self.function.parameters) - if _lp == 1: - if self.parameter_state is not ParamState.REQUIRED: - fail("A 'defining_class' parameter cannot be marked optional.") - if value is not unspecified: - fail("A 'defining_class' parameter cannot have a default value.") - if self.group: - fail("A 'defining_class' parameter cannot be in an optional group.") - else: - fail("A 'defining_class' parameter, if specified, must either " - "be the first thing in the parameter block, or come just " - "after 'self'.") - - - p = Parameter(parameter_name, kind, function=self.function, - converter=converter, default=value, group=self.group, - deprecated_positional=self.deprecated_positional) - - names = [k.name for k in self.function.parameters.values()] - if parameter_name in names[1:]: - fail(f"You can't have two parameters named {parameter_name!r}!") - elif names and parameter_name == names[0] and c_name is None: - fail(f"Parameter {parameter_name!r} requires a custom C name") - - key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name - self.function.parameters[key] = p - - @staticmethod - def parse_converter( - annotation: ast.expr | None - ) -> tuple[str, bool, ConverterArgs]: - match annotation: - case ast.Constant(value=str() as value): - return value, True, {} - case ast.Name(name): - return name, False, {} - case ast.Call(func=ast.Name(name)): - symbols = globals() - kwargs: ConverterArgs = {} - for node in annotation.keywords: - if not isinstance(node.arg, str): - fail("Cannot use a kwarg splat in a function-call annotation") - kwargs[node.arg] = eval_ast_expr(node.value, symbols) - return name, False, kwargs - case _: - fail( - "Annotations must be either a name, a function call, or a string." - ) - - def parse_version(self, thenceforth: str) -> VersionTuple: - """Parse Python version in `[from ...]` marker.""" - assert isinstance(self.function, Function) - - try: - major, minor = thenceforth.split(".") - return int(major), int(minor) - except ValueError: - fail( - f"Function {self.function.name!r}: expected format '[from major.minor]' " - f"where 'major' and 'minor' are integers; got {thenceforth!r}" - ) - - def parse_star(self, function: Function, version: VersionTuple | None) -> None: - """Parse keyword-only parameter marker '*'. - - The 'version' parameter signifies the future version from which - the marker will take effect (None means it is already in effect). - """ - if version is None: - if self.keyword_only: - fail(f"Function {function.name!r} uses '*' more than once.") - self.check_previous_star() - self.check_remaining_star() - self.keyword_only = True - else: - if self.keyword_only: - fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") - if self.deprecated_positional: - if self.deprecated_positional == version: - fail(f"Function {function.name!r} uses '* [from " - f"{version[0]}.{version[1]}]' more than once.") - if self.deprecated_positional < version: - fail(f"Function {function.name!r}: '* [from " - f"{version[0]}.{version[1]}]' must precede '* [from " - f"{self.deprecated_positional[0]}.{self.deprecated_positional[1]}]'") - self.deprecated_positional = version - - def parse_opening_square_bracket(self, function: Function) -> None: - """Parse opening parameter group symbol '['.""" - match self.parameter_state: - case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: - self.parameter_state = ParamState.LEFT_SQUARE_BEFORE - case ParamState.REQUIRED | ParamState.GROUP_AFTER: - self.parameter_state = ParamState.GROUP_AFTER - case st: - fail(f"Function {function.name!r} " - f"has an unsupported group configuration. " - f"(Unexpected state {st}.b)") - self.group += 1 - function.docstring_only = True - - def parse_closing_square_bracket(self, function: Function) -> None: - """Parse closing parameter group symbol ']'.""" - if not self.group: - fail(f"Function {function.name!r} has a ']' without a matching '['.") - if not any(p.group == self.group for p in function.parameters.values()): - fail(f"Function {function.name!r} has an empty group. " - "All groups must contain at least one parameter.") - self.group -= 1 - match self.parameter_state: - case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: - self.parameter_state = ParamState.GROUP_BEFORE - case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: - self.parameter_state = ParamState.RIGHT_SQUARE_AFTER - case st: - fail(f"Function {function.name!r} " - f"has an unsupported group configuration. " - f"(Unexpected state {st}.c)") - - def parse_slash(self, function: Function, version: VersionTuple | None) -> None: - """Parse positional-only parameter marker '/'. - - The 'version' parameter signifies the future version from which - the marker will take effect (None means it is already in effect). - """ - if version is None: - if self.deprecated_keyword: - fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") - if self.deprecated_positional: - fail(f"Function {function.name!r}: '/' must precede '* [from ...]'") - if self.keyword_only: - fail(f"Function {function.name!r}: '/' must precede '*'") - if self.positional_only: - fail(f"Function {function.name!r} uses '/' more than once.") - else: - if self.deprecated_keyword: - if self.deprecated_keyword == version: - fail(f"Function {function.name!r} uses '/ [from " - f"{version[0]}.{version[1]}]' more than once.") - if self.deprecated_keyword > version: - fail(f"Function {function.name!r}: '/ [from " - f"{version[0]}.{version[1]}]' must precede '/ [from " - f"{self.deprecated_keyword[0]}.{self.deprecated_keyword[1]}]'") - if self.deprecated_positional: - fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") - if self.keyword_only: - fail(f"Function {function.name!r}: '/ [from ...]' must precede '*'") - self.positional_only = True - self.deprecated_keyword = version - if version is not None: - found = False - for p in reversed(function.parameters.values()): - found = p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD - break - if not found: - fail(f"Function {function.name!r} specifies '/ [from ...]' " - f"without preceding parameters.") - # REQUIRED and OPTIONAL are allowed here, that allows positional-only - # without option groups to work (and have default values!) - allowed = { - ParamState.REQUIRED, - ParamState.OPTIONAL, - ParamState.RIGHT_SQUARE_AFTER, - ParamState.GROUP_BEFORE, - } - if (self.parameter_state not in allowed) or self.group: - fail(f"Function {function.name!r} has an unsupported group configuration. " - f"(Unexpected state {self.parameter_state}.d)") - # fixup preceding parameters - for p in function.parameters.values(): - if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: - if version is None: - p.kind = inspect.Parameter.POSITIONAL_ONLY - elif p.deprecated_keyword is None: - p.deprecated_keyword = version - - def state_parameter_docstring_start(self, line: str) -> None: - assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" - self.parameter_docstring_indent = len(self.indent.margin) - assert self.indent.depth == 3 - return self.next(self.state_parameter_docstring, line) - - def docstring_append(self, obj: Function | Parameter, line: str) -> None: - """Add a rstripped line to the current docstring.""" - # gh-80282: We filter out non-ASCII characters from the docstring, - # since historically, some compilers may balk on non-ASCII input. - # If you're using Argument Clinic in an external project, - # you may not need to support the same array of platforms as CPython, - # so you may be able to remove this restriction. - matches = re.finditer(r'[^\x00-\x7F]', line) - if offending := ", ".join([repr(m[0]) for m in matches]): - warn("Non-ascii characters are not allowed in docstrings:", - offending) - - docstring = obj.docstring - if docstring: - docstring += "\n" - if stripped := line.rstrip(): - docstring += self.indent.dedent(stripped) - obj.docstring = docstring - - # every line of the docstring must start with at least F spaces, - # where F > P. - # these F spaces will be stripped. - def state_parameter_docstring(self, line: str) -> None: - if not self.valid_line(line): - return - - indent = self.indent.measure(line) - if indent < self.parameter_docstring_indent: - self.indent.infer(line) - assert self.indent.depth < 3 - if self.indent.depth == 2: - # back to a parameter - return self.next(self.state_parameter, line) - assert self.indent.depth == 1 - return self.next(self.state_function_docstring, line) - - assert self.function and self.function.parameters - last_param = next(reversed(self.function.parameters.values())) - self.docstring_append(last_param, line) - - # the final stanza of the DSL is the docstring. - def state_function_docstring(self, line: str) -> None: - assert self.function is not None - - if self.group: - fail(f"Function {self.function.name!r} has a ']' without a matching '['.") - - if not self.valid_line(line): - return - - self.docstring_append(self.function, line) - - def format_docstring_signature( - self, f: Function, parameters: list[Parameter] - ) -> str: - lines = [] - lines.append(f.displayname) - if self.forced_text_signature: - lines.append(self.forced_text_signature) - elif f.kind in {GETTER, SETTER}: - # @getter and @setter do not need signatures like a method or a function. - return '' - else: - lines.append('(') - - # populate "right_bracket_count" field for every parameter - assert parameters, "We should always have a self parameter. " + repr(f) - assert isinstance(parameters[0].converter, self_converter) - # self is always positional-only. - assert parameters[0].is_positional_only() - assert parameters[0].right_bracket_count == 0 - positional_only = True - for p in parameters[1:]: - if not p.is_positional_only(): - positional_only = False - else: - assert positional_only - if positional_only: - p.right_bracket_count = abs(p.group) - else: - # don't put any right brackets around non-positional-only parameters, ever. - p.right_bracket_count = 0 - - right_bracket_count = 0 - - def fix_right_bracket_count(desired: int) -> str: - nonlocal right_bracket_count - s = '' - while right_bracket_count < desired: - s += '[' - right_bracket_count += 1 - while right_bracket_count > desired: - s += ']' - right_bracket_count -= 1 - return s - - need_slash = False - added_slash = False - need_a_trailing_slash = False - - # we only need a trailing slash: - # * if this is not a "docstring_only" signature - # * and if the last *shown* parameter is - # positional only - if not f.docstring_only: - for p in reversed(parameters): - if not p.converter.show_in_signature: - continue - if p.is_positional_only(): - need_a_trailing_slash = True - break - - - added_star = False - - first_parameter = True - last_p = parameters[-1] - line_length = len(''.join(lines)) - indent = " " * line_length - def add_parameter(text: str) -> None: - nonlocal line_length - nonlocal first_parameter - if first_parameter: - s = text - first_parameter = False - else: - s = ' ' + text - if line_length + len(s) >= 72: - lines.extend(["\n", indent]) - line_length = len(indent) - s = text - line_length += len(s) - lines.append(s) - - for p in parameters: - if not p.converter.show_in_signature: - continue - assert p.name - - is_self = isinstance(p.converter, self_converter) - if is_self and f.docstring_only: - # this isn't a real machine-parsable signature, - # so let's not print the "self" parameter - continue - - if p.is_positional_only(): - need_slash = not f.docstring_only - elif need_slash and not (added_slash or p.is_positional_only()): - added_slash = True - add_parameter('/,') - - if p.is_keyword_only() and not added_star: - added_star = True - add_parameter('*,') - - p_lines = [fix_right_bracket_count(p.right_bracket_count)] - - if isinstance(p.converter, self_converter): - # annotate first parameter as being a "self". - # - # if inspect.Signature gets this function, - # and it's already bound, the self parameter - # will be stripped off. - # - # if it's not bound, it should be marked - # as positional-only. - # - # note: we don't print "self" for __init__, - # because this isn't actually the signature - # for __init__. (it can't be, __init__ doesn't - # have a docstring.) if this is an __init__ - # (or __new__), then this signature is for - # calling the class to construct a new instance. - p_lines.append('$') - - if p.is_vararg(): - p_lines.append("*") - - name = p.converter.signature_name or p.name - p_lines.append(name) - - if not p.is_vararg() and p.converter.is_optional(): - p_lines.append('=') - value = p.converter.py_default - if not value: - value = repr(p.converter.default) - p_lines.append(value) - - if (p != last_p) or need_a_trailing_slash: - p_lines.append(',') - - p_output = "".join(p_lines) - add_parameter(p_output) - - lines.append(fix_right_bracket_count(0)) - if need_a_trailing_slash: - add_parameter('/') - lines.append(')') - - # PEP 8 says: - # - # The Python standard library will not use function annotations - # as that would result in a premature commitment to a particular - # annotation style. Instead, the annotations are left for users - # to discover and experiment with useful annotation styles. - # - # therefore this is commented out: - # - # if f.return_converter.py_default: - # lines.append(' -> ') - # lines.append(f.return_converter.py_default) - - if not f.docstring_only: - lines.append("\n" + libclinic.SIG_END_MARKER + "\n") - - signature_line = "".join(lines) - - # now fix up the places where the brackets look wrong - return signature_line.replace(', ]', ',] ') - - @staticmethod - def format_docstring_parameters(params: list[Parameter]) -> str: - """Create substitution text for {parameters}""" - return "".join(p.render_docstring() + "\n" for p in params if p.docstring) - - def format_docstring(self) -> str: - assert self.function is not None - f = self.function - # For the following special cases, it does not make sense to render a docstring. - if f.kind in {METHOD_INIT, METHOD_NEW, GETTER, SETTER} and not f.docstring: - return f.docstring - - # Enforce the summary line! - # The first line of a docstring should be a summary of the function. - # It should fit on one line (80 columns? 79 maybe?) and be a paragraph - # by itself. - # - # Argument Clinic enforces the following rule: - # * either the docstring is empty, - # * or it must have a summary line. - # - # Guido said Clinic should enforce this: - # http://mail.python.org/pipermail/python-dev/2013-June/127110.html - - lines = f.docstring.split('\n') - if len(lines) >= 2: - if lines[1]: - fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" - "Every non-blank function docstring must start with " - "a single line summary followed by an empty line.") - elif len(lines) == 1: - # the docstring is only one line right now--the summary line. - # add an empty line after the summary line so we have space - # between it and the {parameters} we're about to add. - lines.append('') - - parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 - if parameters_marker_count > 1: - fail('You may not specify {parameters} more than once in a docstring!') - - # insert signature at front and params after the summary line - if not parameters_marker_count: - lines.insert(2, '{parameters}') - lines.insert(0, '{signature}') - - # finalize docstring - params = f.render_parameters - parameters = self.format_docstring_parameters(params) - signature = self.format_docstring_signature(f, params) - docstring = "\n".join(lines) - return linear_format(docstring, - signature=signature, - parameters=parameters).rstrip() - - def check_remaining_star(self, lineno: int | None = None) -> None: - assert isinstance(self.function, Function) - - if self.keyword_only: - symbol = '*' - elif self.deprecated_positional: - symbol = '* [from ...]' - else: - return - - for p in reversed(self.function.parameters.values()): - if self.keyword_only: - if p.kind == inspect.Parameter.KEYWORD_ONLY: - return - elif self.deprecated_positional: - if p.deprecated_positional == self.deprecated_positional: - return - break - - fail(f"Function {self.function.name!r} specifies {symbol!r} " - f"without following parameters.", line_number=lineno) - - def check_previous_star(self, lineno: int | None = None) -> None: - assert isinstance(self.function, Function) - - for p in self.function.parameters.values(): - if p.kind == inspect.Parameter.VAR_POSITIONAL: - fail(f"Function {self.function.name!r} uses '*' more than once.") - - - def do_post_block_processing_cleanup(self, lineno: int) -> None: - """ - Called when processing the block is done. - """ - if not self.function: - return - - self.check_remaining_star(lineno) - self.function.docstring = self.format_docstring() - - - - -# maps strings to callables. -# the callable should return an object -# that implements the clinic parser -# interface (__init__ and parse). -# -# example parsers: -# "clinic", handles the Clinic DSL -# "python", handles running Python code -# -parsers: dict[str, Callable[[Clinic], Parser]] = { - 'clinic': DSLParser, - 'python': PythonParser, -} - - -def create_cli() -> argparse.ArgumentParser: - cmdline = argparse.ArgumentParser( - prog="clinic.py", - description="""Preprocessor for CPython C files. - -The purpose of the Argument Clinic is automating all the boilerplate involved -with writing argument parsing code for builtins and providing introspection -signatures ("docstrings") for CPython builtins. - -For more information see https://devguide.python.org/development-tools/clinic/""") - cmdline.add_argument("-f", "--force", action='store_true', - help="force output regeneration") - cmdline.add_argument("-o", "--output", type=str, - help="redirect file output to OUTPUT") - cmdline.add_argument("-v", "--verbose", action='store_true', - help="enable verbose mode") - cmdline.add_argument("--converters", action='store_true', - help=("print a list of all supported converters " - "and return converters")) - cmdline.add_argument("--make", action='store_true', - help="walk --srcdir to run over all relevant files") - cmdline.add_argument("--srcdir", type=str, default=os.curdir, - help="the directory tree to walk in --make mode") - cmdline.add_argument("--exclude", type=str, action="append", - help=("a file to exclude in --make mode; " - "can be given multiple times")) - cmdline.add_argument("--limited", dest="limited_capi", action='store_true', - help="use the Limited C API") - cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", - help="the list of files to process") - return cmdline - - -def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: - if ns.converters: - if ns.filename: - parser.error( - "can't specify --converters and a filename at the same time" - ) - converters: list[tuple[str, str]] = [] - return_converters: list[tuple[str, str]] = [] - ignored = set(""" - add_c_converter - add_c_return_converter - add_default_legacy_c_converter - add_legacy_c_converter - """.strip().split()) - module = globals() - for name in module: - for suffix, ids in ( - ("_return_converter", return_converters), - ("_converter", converters), - ): - if name in ignored: - continue - if name.endswith(suffix): - ids.append((name, name.removesuffix(suffix))) - break - print() - - print("Legacy converters:") - legacy = sorted(legacy_converters) - print(' ' + ' '.join(c for c in legacy if c[0].isupper())) - print(' ' + ' '.join(c for c in legacy if c[0].islower())) - print() - - for title, attribute, ids in ( - ("Converters", 'converter_init', converters), - ("Return converters", 'return_converter_init', return_converters), - ): - print(title + ":") - longest = -1 - for name, short_name in ids: - longest = max(longest, len(short_name)) - for name, short_name in sorted(ids, key=lambda x: x[1].lower()): - cls = module[name] - callable = getattr(cls, attribute, None) - if not callable: - continue - signature = inspect.signature(callable) - parameters = [] - for parameter_name, parameter in signature.parameters.items(): - if parameter.kind == inspect.Parameter.KEYWORD_ONLY: - if parameter.default != inspect.Parameter.empty: - s = f'{parameter_name}={parameter.default!r}' - else: - s = parameter_name - parameters.append(s) - print(' {}({})'.format(short_name, ', '.join(parameters))) - print() - print("All converters also accept (c_default=None, py_default=None, annotation=None).") - print("All return converters also accept (py_default=None).") - return - - if ns.make: - if ns.output or ns.filename: - parser.error("can't use -o or filenames with --make") - if not ns.srcdir: - parser.error("--srcdir must not be empty with --make") - if ns.exclude: - excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude] - excludes = [os.path.normpath(f) for f in excludes] - else: - excludes = [] - for root, dirs, files in os.walk(ns.srcdir): - for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): - if rcs_dir in dirs: - dirs.remove(rcs_dir) - for filename in files: - # handle .c, .cpp and .h files - if not filename.endswith(('.c', '.cpp', '.h')): - continue - path = os.path.join(root, filename) - path = os.path.normpath(path) - if path in excludes: - continue - if ns.verbose: - print(path) - parse_file(path, - verify=not ns.force, limited_capi=ns.limited_capi) - return - - if not ns.filename: - parser.error("no input files") - - if ns.output and len(ns.filename) > 1: - parser.error("can't use -o with multiple filenames") - - for filename in ns.filename: - if ns.verbose: - print(filename) - parse_file(filename, output=ns.output, - verify=not ns.force, limited_capi=ns.limited_capi) - - -def main(argv: list[str] | None = None) -> NoReturn: - parser = create_cli() - args = parser.parse_args(argv) - try: - run_clinic(parser, args) - except ClinicError as exc: - sys.stderr.write(exc.report()) - sys.exit(1) - else: - sys.exit(0) +from libclinic.cli import main if __name__ == "__main__": diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 1b300b55acc21e..7c5cede2396677 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -2,6 +2,8 @@ from .errors import ( ClinicError, + warn, + fail, ) from .formatting import ( SIG_END_MARKER, @@ -9,16 +11,28 @@ docstring_for_c_string, format_escape, indent_all_lines, + linear_format, normalize_snippet, pprint_words, suffix_all_lines, wrap_declarations, wrapped_c_string_literal, ) +from .identifiers import ( + ensure_legal_c_identifier, + is_legal_c_identifier, + is_legal_py_identifier, +) from .utils import ( FormatCounterFormatter, + NULL, + Null, + Sentinels, + VersionTuple, compute_checksum, create_regex, + unknown, + unspecified, write_file, ) @@ -26,6 +40,8 @@ __all__ = [ # Error handling "ClinicError", + "warn", + "fail", # Formatting helpers "SIG_END_MARKER", @@ -33,16 +49,28 @@ "docstring_for_c_string", "format_escape", "indent_all_lines", + "linear_format", "normalize_snippet", "pprint_words", "suffix_all_lines", "wrap_declarations", "wrapped_c_string_literal", + # Identifier helpers + "ensure_legal_c_identifier", + "is_legal_c_identifier", + "is_legal_py_identifier", + # Utility functions "FormatCounterFormatter", + "NULL", + "Null", + "Sentinels", + "VersionTuple", "compute_checksum", "create_regex", + "unknown", + "unspecified", "write_file", ] diff --git a/Tools/clinic/libclinic/app.py b/Tools/clinic/libclinic/app.py new file mode 100644 index 00000000000000..632bed3ce53dde --- /dev/null +++ b/Tools/clinic/libclinic/app.py @@ -0,0 +1,271 @@ +from __future__ import annotations +import os + +from collections.abc import Callable, Sequence +from typing import Any, TYPE_CHECKING + + +import libclinic +from libclinic import fail, warn +from libclinic.function import Class +from libclinic.block_parser import Block, BlockParser +from libclinic.codegen import BlockPrinter, Destination, CodeGen +from libclinic.parser import Parser, PythonParser +from libclinic.dsl_parser import DSLParser +if TYPE_CHECKING: + from libclinic.clanguage import CLanguage + from libclinic.function import ( + Module, Function, ClassDict, ModuleDict) + from libclinic.codegen import DestinationDict + + +# maps strings to callables. +# the callable should return an object +# that implements the clinic parser +# interface (__init__ and parse). +# +# example parsers: +# "clinic", handles the Clinic DSL +# "python", handles running Python code +# +parsers: dict[str, Callable[[Clinic], Parser]] = { + 'clinic': DSLParser, + 'python': PythonParser, +} + + +class Clinic: + + presets_text = """ +preset block +everything block +methoddef_ifndef buffer 1 +docstring_prototype suppress +parser_prototype suppress +cpp_if suppress +cpp_endif suppress + +preset original +everything block +methoddef_ifndef buffer 1 +docstring_prototype suppress +parser_prototype suppress +cpp_if suppress +cpp_endif suppress + +preset file +everything file +methoddef_ifndef file 1 +docstring_prototype suppress +parser_prototype suppress +impl_definition block + +preset buffer +everything buffer +methoddef_ifndef buffer 1 +impl_definition block +docstring_prototype suppress +impl_prototype suppress +parser_prototype suppress + +preset partial-buffer +everything buffer +methoddef_ifndef buffer 1 +docstring_prototype block +impl_prototype suppress +methoddef_define block +parser_prototype block +impl_definition block + +""" + + def __init__( + self, + language: CLanguage, + printer: BlockPrinter | None = None, + *, + filename: str, + limited_capi: bool, + verify: bool = True, + ) -> None: + # maps strings to Parser objects. + # (instantiated from the "parsers" global.) + self.parsers: dict[str, Parser] = {} + self.language: CLanguage = language + if printer: + fail("Custom printers are broken right now") + self.printer = printer or BlockPrinter(language) + self.verify = verify + self.limited_capi = limited_capi + self.filename = filename + self.modules: ModuleDict = {} + self.classes: ClassDict = {} + self.functions: list[Function] = [] + self.codegen = CodeGen(self.limited_capi) + + self.line_prefix = self.line_suffix = '' + + self.destinations: DestinationDict = {} + self.add_destination("block", "buffer") + self.add_destination("suppress", "suppress") + self.add_destination("buffer", "buffer") + if filename: + self.add_destination("file", "file", "{dirname}/clinic/{basename}.h") + + d = self.get_destination_buffer + self.destination_buffers = { + 'cpp_if': d('file'), + 'docstring_prototype': d('suppress'), + 'docstring_definition': d('file'), + 'methoddef_define': d('file'), + 'impl_prototype': d('file'), + 'parser_prototype': d('suppress'), + 'parser_definition': d('file'), + 'cpp_endif': d('file'), + 'methoddef_ifndef': d('file', 1), + 'impl_definition': d('block'), + } + + DestBufferType = dict[str, list[str]] + DestBufferList = list[DestBufferType] + + self.destination_buffers_stack: DestBufferList = [] + + self.presets: dict[str, dict[Any, Any]] = {} + preset = None + for line in self.presets_text.strip().split('\n'): + line = line.strip() + if not line: + continue + name, value, *options = line.split() + if name == 'preset': + self.presets[value] = preset = {} + continue + + if len(options): + index = int(options[0]) + else: + index = 0 + buffer = self.get_destination_buffer(value, index) + + if name == 'everything': + for name in self.destination_buffers: + preset[name] = buffer + continue + + assert name in self.destination_buffers + preset[name] = buffer + + def add_destination( + self, + name: str, + type: str, + *args: str + ) -> None: + if name in self.destinations: + fail(f"Destination already exists: {name!r}") + self.destinations[name] = Destination(name, type, self, args) + + def get_destination(self, name: str) -> Destination: + d = self.destinations.get(name) + if not d: + fail(f"Destination does not exist: {name!r}") + return d + + def get_destination_buffer( + self, + name: str, + item: int = 0 + ) -> list[str]: + d = self.get_destination(name) + return d.buffers[item] + + def parse(self, input: str) -> str: + printer = self.printer + self.block_parser = BlockParser(input, self.language, verify=self.verify) + for block in self.block_parser: + dsl_name = block.dsl_name + if dsl_name: + if dsl_name not in self.parsers: + assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." + self.parsers[dsl_name] = parsers[dsl_name](self) + parser = self.parsers[dsl_name] + parser.parse(block) + printer.print_block(block) + + # these are destinations not buffers + for name, destination in self.destinations.items(): + if destination.type == 'suppress': + continue + output = destination.dump() + + if output: + block = Block("", dsl_name="clinic", output=output) + + if destination.type == 'buffer': + block.input = "dump " + name + "\n" + warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") + printer.write("\n") + printer.print_block(block) + continue + + if destination.type == 'file': + try: + dirname = os.path.dirname(destination.filename) + try: + os.makedirs(dirname) + except FileExistsError: + if not os.path.isdir(dirname): + fail(f"Can't write to destination " + f"{destination.filename!r}; " + f"can't make directory {dirname!r}!") + if self.verify: + with open(destination.filename) as f: + parser_2 = BlockParser(f.read(), language=self.language) + blocks = list(parser_2) + if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): + fail(f"Modified destination file " + f"{destination.filename!r}; not overwriting!") + except FileNotFoundError: + pass + + block.input = 'preserve\n' + includes = self.codegen.get_includes() + + printer_2 = BlockPrinter(self.language) + printer_2.print_block(block, header_includes=includes) + libclinic.write_file(destination.filename, + printer_2.f.getvalue()) + continue + + return printer.f.getvalue() + + def _module_and_class( + self, fields: Sequence[str] + ) -> tuple[Module | Clinic, Class | None]: + """ + fields should be an iterable of field names. + returns a tuple of (module, class). + the module object could actually be self (a clinic object). + this function is only ever used to find the parent of where + a new class/module should go. + """ + parent: Clinic | Module | Class = self + module: Clinic | Module = self + cls: Class | None = None + + for idx, field in enumerate(fields): + if not isinstance(parent, Class): + if field in parent.modules: + parent = module = parent.modules[field] + continue + if field in parent.classes: + parent = cls = parent.classes[field] + else: + fullname = ".".join(fields[idx:]) + fail(f"Parent class or module {fullname!r} does not exist.") + + return module, cls + + def __repr__(self) -> str: + return "" diff --git a/Tools/clinic/libclinic/block_parser.py b/Tools/clinic/libclinic/block_parser.py new file mode 100644 index 00000000000000..4c0198b53592a9 --- /dev/null +++ b/Tools/clinic/libclinic/block_parser.py @@ -0,0 +1,256 @@ +from __future__ import annotations +import collections +import dataclasses as dc +import re +import shlex +from typing import Any + +import libclinic +from libclinic import fail, ClinicError +from libclinic.language import Language +from libclinic.function import ( + Module, Class, Function) + + +@dc.dataclass(slots=True, repr=False) +class Block: + r""" + Represents a single block of text embedded in + another file. If dsl_name is None, the block represents + verbatim text, raw original text from the file, in + which case "input" will be the only non-false member. + If dsl_name is not None, the block represents a Clinic + block. + + input is always str, with embedded \n characters. + input represents the original text from the file; + if it's a Clinic block, it is the original text with + the body_prefix and redundant leading whitespace removed. + + dsl_name is either str or None. If str, it's the text + found on the start line of the block between the square + brackets. + + signatures is a list. + It may only contain clinic.Module, clinic.Class, and + clinic.Function objects. At the moment it should + contain at most one of each. + + output is either str or None. If str, it's the output + from this block, with embedded '\n' characters. + + indent is a str. It's the leading whitespace + that was found on every line of input. (If body_prefix is + not empty, this is the indent *after* removing the + body_prefix.) + + "indent" is different from the concept of "preindent" + (which is not stored as state on Block objects). + "preindent" is the whitespace that + was found in front of every line of input *before* the + "body_prefix" (see the Language object). If body_prefix + is empty, preindent must always be empty too. + + To illustrate the difference between "indent" and "preindent": + + Assume that '_' represents whitespace. + If the block processed was in a Python file, and looked like this: + ____#/*[python] + ____#__for a in range(20): + ____#____print(a) + ____#[python]*/ + "preindent" would be "____" and "indent" would be "__". + + """ + input: str + dsl_name: str | None = None + signatures: list[Module | Class | Function] = dc.field(default_factory=list) + output: Any = None # TODO: Very dynamic; probably untypeable in its current form? + indent: str = '' + + def __repr__(self) -> str: + dsl_name = self.dsl_name or "text" + def summarize(s: object) -> str: + s = repr(s) + if len(s) > 30: + return s[:26] + "..." + s[0] + return s + parts = ( + repr(dsl_name), + f"input={summarize(self.input)}", + f"output={summarize(self.output)}" + ) + return f"" + + +class BlockParser: + """ + Block-oriented parser for Argument Clinic. + Iterator, yields Block objects. + """ + + def __init__( + self, + input: str, + language: Language, + *, + verify: bool = True + ) -> None: + """ + "input" should be a str object + with embedded \n characters. + + "language" should be a Language object. + """ + language.validate() + + self.input = collections.deque(reversed(input.splitlines(keepends=True))) + self.block_start_line_number = self.line_number = 0 + + self.language = language + before, _, after = language.start_line.partition('{dsl_name}') + assert _ == '{dsl_name}' + self.find_start_re = libclinic.create_regex(before, after, + whole_line=False) + self.start_re = libclinic.create_regex(before, after) + self.verify = verify + self.last_checksum_re: re.Pattern[str] | None = None + self.last_dsl_name: str | None = None + self.dsl_name: str | None = None + self.first_block = True + + def __iter__(self) -> BlockParser: + return self + + def __next__(self) -> Block: + while True: + if not self.input: + raise StopIteration + + if self.dsl_name: + try: + return_value = self.parse_clinic_block(self.dsl_name) + except ClinicError as exc: + exc.filename = self.language.filename + exc.lineno = self.line_number + raise + self.dsl_name = None + self.first_block = False + return return_value + block = self.parse_verbatim_block() + if self.first_block and not block.input: + continue + self.first_block = False + return block + + + def is_start_line(self, line: str) -> str | None: + match = self.start_re.match(line.lstrip()) + return match.group(1) if match else None + + def _line(self, lookahead: bool = False) -> str: + self.line_number += 1 + line = self.input.pop() + if not lookahead: + self.language.parse_line(line) + return line + + def parse_verbatim_block(self) -> Block: + lines = [] + self.block_start_line_number = self.line_number + + while self.input: + line = self._line() + dsl_name = self.is_start_line(line) + if dsl_name: + self.dsl_name = dsl_name + break + lines.append(line) + + return Block("".join(lines)) + + def parse_clinic_block(self, dsl_name: str) -> Block: + in_lines = [] + self.block_start_line_number = self.line_number + 1 + stop_line = self.language.stop_line.format(dsl_name=dsl_name) + body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) + + def is_stop_line(line: str) -> bool: + # make sure to recognize stop line even if it + # doesn't end with EOL (it could be the very end of the file) + if line.startswith(stop_line): + remainder = line.removeprefix(stop_line) + if remainder and not remainder.isspace(): + fail(f"Garbage after stop line: {remainder!r}") + return True + else: + # gh-92256: don't allow incorrectly formatted stop lines + if line.lstrip().startswith(stop_line): + fail(f"Whitespace is not allowed before the stop line: {line!r}") + return False + + # consume body of program + while self.input: + line = self._line() + if is_stop_line(line) or self.is_start_line(line): + break + if body_prefix: + line = line.lstrip() + assert line.startswith(body_prefix) + line = line.removeprefix(body_prefix) + in_lines.append(line) + + # consume output and checksum line, if present. + if self.last_dsl_name == dsl_name: + checksum_re = self.last_checksum_re + else: + before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') + assert _ == '{arguments}' + checksum_re = libclinic.create_regex(before, after, word=False) + self.last_dsl_name = dsl_name + self.last_checksum_re = checksum_re + assert checksum_re is not None + + # scan forward for checksum line + out_lines = [] + arguments = None + while self.input: + line = self._line(lookahead=True) + match = checksum_re.match(line.lstrip()) + arguments = match.group(1) if match else None + if arguments: + break + out_lines.append(line) + if self.is_start_line(line): + break + + output: str | None + output = "".join(out_lines) + if arguments: + d = {} + for field in shlex.split(arguments): + name, equals, value = field.partition('=') + if not equals: + fail(f"Mangled Argument Clinic marker line: {line!r}") + d[name.strip()] = value.strip() + + if self.verify: + if 'input' in d: + checksum = d['output'] + else: + checksum = d['checksum'] + + computed = libclinic.compute_checksum(output, len(checksum)) + if checksum != computed: + fail("Checksum mismatch! " + f"Expected {checksum!r}, computed {computed!r}. " + "Suggested fix: remove all generated code including " + "the end marker, or use the '-f' option.") + else: + # put back output + output_lines = output.splitlines(keepends=True) + self.line_number -= len(output_lines) + self.input.extend(reversed(output_lines)) + output = None + + return Block("".join(in_lines), dsl_name, output=output) diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py new file mode 100644 index 00000000000000..32aba81ab8a850 --- /dev/null +++ b/Tools/clinic/libclinic/clanguage.py @@ -0,0 +1,548 @@ +from __future__ import annotations +import itertools +import sys +import textwrap +from typing import TYPE_CHECKING, Literal, Final +from operator import attrgetter +from collections.abc import Iterable + +import libclinic +from libclinic import ( + unspecified, fail, Sentinels, VersionTuple) +from libclinic.codegen import CRenderData, TemplateDict, CodeGen +from libclinic.language import Language +from libclinic.function import ( + Module, Class, Function, Parameter, + permute_optional_groups, + GETTER, SETTER, METHOD_INIT) +from libclinic.converters import defining_class_converter, self_converter +from libclinic.parse_args import ParseArgsCodeGen +if TYPE_CHECKING: + from libclinic.app import Clinic + + +def c_id(name: str) -> str: + if len(name) == 1 and ord(name) < 256: + if name.isalnum(): + return f"_Py_LATIN1_CHR('{name}')" + else: + return f'_Py_LATIN1_CHR({ord(name)})' + else: + return f'&_Py_ID({name})' + + +class CLanguage(Language): + + body_prefix = "#" + language = 'C' + start_line = "/*[{dsl_name} input]" + body_prefix = "" + stop_line = "[{dsl_name} start generated code]*/" + checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" + + COMPILER_DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" + // Emit compiler warnings when we get to Python {major}.{minor}. + #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 + # error {message} + #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 + # ifdef _MSC_VER + # pragma message ({message}) + # else + # warning {message} + # endif + #endif + """ + DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" + if ({condition}) {{{{{errcheck} + if (PyErr_WarnEx(PyExc_DeprecationWarning, + {message}, 1)) + {{{{ + goto exit; + }}}} + }}}} + """ + + def __init__(self, filename: str) -> None: + super().__init__(filename) + self.cpp = libclinic.cpp.Monitor(filename) + + def parse_line(self, line: str) -> None: + self.cpp.writeline(line) + + def render( + self, + clinic: Clinic, + signatures: Iterable[Module | Class | Function] + ) -> str: + function = None + for o in signatures: + if isinstance(o, Function): + if function: + fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) + function = o + return self.render_function(clinic, function) + + def compiler_deprecated_warning( + self, + func: Function, + parameters: list[Parameter], + ) -> str | None: + minversion: VersionTuple | None = None + for p in parameters: + for version in p.deprecated_positional, p.deprecated_keyword: + if version and (not minversion or minversion > version): + minversion = version + if not minversion: + return None + + # Format the preprocessor warning and error messages. + assert isinstance(self.cpp.filename, str) + message = f"Update the clinic input of {func.full_name!r}." + code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( + major=minversion[0], + minor=minversion[1], + message=libclinic.c_repr(message), + ) + return libclinic.normalize_snippet(code) + + def deprecate_positional_use( + self, + func: Function, + params: dict[int, Parameter], + ) -> str: + assert len(params) > 0 + first_pos = next(iter(params)) + last_pos = next(reversed(params)) + + # Format the deprecation message. + if len(params) == 1: + condition = f"nargs == {first_pos+1}" + amount = f"{first_pos+1} " if first_pos else "" + pl = "s" + else: + condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" + amount = f"more than {first_pos} " if first_pos else "" + pl = "s" if first_pos != 1 else "" + message = ( + f"Passing {amount}positional argument{pl} to " + f"{func.fulldisplayname}() is deprecated." + ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_positional") + ): + names = [repr(p.name) for p in group] + pstr = libclinic.pprint_words(names) + if len(names) == 1: + message += ( + f" Parameter {pstr} will become a keyword-only parameter " + f"in Python {major}.{minor}." + ) + else: + message += ( + f" Parameters {pstr} will become keyword-only parameters " + f"in Python {major}.{minor}." + ) + + # Append deprecation warning to docstring. + docstring = textwrap.fill(f"Note: {message}") + func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. + code = self.DEPRECATION_WARNING_PROTOTYPE.format( + condition=condition, + errcheck="", + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), + ) + return libclinic.normalize_snippet(code, indent=4) + + def deprecate_keyword_use( + self, + func: Function, + params: dict[int, Parameter], + argname_fmt: str | None = None, + *, + fastcall: bool, + codegen: CodeGen, + ) -> str: + assert len(params) > 0 + last_param = next(reversed(params.values())) + limited_capi = codegen.limited_capi + + # Format the deprecation message. + containscheck = "" + conditions = [] + for i, p in params.items(): + if p.is_optional(): + if argname_fmt: + conditions.append(f"nargs < {i+1} && {argname_fmt % i}") + elif fastcall: + conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, {c_id(p.name)})") + containscheck = "PySequence_Contains" + codegen.add_include('pycore_runtime.h', '_Py_ID()') + else: + conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, {c_id(p.name)})") + containscheck = "PyDict_Contains" + codegen.add_include('pycore_runtime.h', '_Py_ID()') + else: + conditions = [f"nargs < {i+1}"] + condition = ") || (".join(conditions) + if len(conditions) > 1: + condition = f"(({condition}))" + if last_param.is_optional(): + if fastcall: + if limited_capi: + condition = f"kwnames && PyTuple_Size(kwnames) && {condition}" + else: + condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" + else: + if limited_capi: + condition = f"kwargs && PyDict_Size(kwargs) && {condition}" + else: + condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" + names = [repr(p.name) for p in params.values()] + pstr = libclinic.pprint_words(names) + pl = 's' if len(params) != 1 else '' + message = ( + f"Passing keyword argument{pl} {pstr} to " + f"{func.fulldisplayname}() is deprecated." + ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_keyword") + ): + names = [repr(p.name) for p in group] + pstr = libclinic.pprint_words(names) + pl = 's' if len(names) != 1 else '' + message += ( + f" Parameter{pl} {pstr} will become positional-only " + f"in Python {major}.{minor}." + ) + + if containscheck: + errcheck = f""" + if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail + goto exit; + }}}}""" + else: + errcheck = "" + if argname_fmt: + # Append deprecation warning to docstring. + docstring = textwrap.fill(f"Note: {message}") + func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. + code = self.DEPRECATION_WARNING_PROTOTYPE.format( + condition=condition, + errcheck=errcheck, + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), + ) + return libclinic.normalize_snippet(code, indent=4) + + def output_templates( + self, + f: Function, + codegen: CodeGen, + ) -> dict[str, str]: + args = ParseArgsCodeGen(f, codegen) + return args.parse_args(self) + + @staticmethod + def group_to_variable_name(group: int) -> str: + adjective = "left_" if group < 0 else "right_" + return "group_" + adjective + str(abs(group)) + + def render_option_group_parsing( + self, + f: Function, + template_dict: TemplateDict, + limited_capi: bool, + ) -> None: + # positional only, grouped, optional arguments! + # can be optional on the left or right. + # here's an example: + # + # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] + # + # Here group D are required, and all other groups are optional. + # (Group D's "group" is actually None.) + # We can figure out which sets of arguments we have based on + # how many arguments are in the tuple. + # + # Note that you need to count up on both sides. For example, + # you could have groups C+D, or C+D+E, or C+D+E+F. + # + # What if the number of arguments leads us to an ambiguous result? + # Clinic prefers groups on the left. So in the above example, + # five arguments would map to B+C, not C+D. + + out = [] + parameters = list(f.parameters.values()) + if isinstance(parameters[0].converter, self_converter): + del parameters[0] + + group: list[Parameter] | None = None + left = [] + right = [] + required: list[Parameter] = [] + last: int | Literal[Sentinels.unspecified] = unspecified + + for p in parameters: + group_id = p.group + if group_id != last: + last = group_id + group = [] + if group_id < 0: + left.append(group) + elif group_id == 0: + group = required + else: + right.append(group) + assert group is not None + group.append(p) + + count_min = sys.maxsize + count_max = -1 + + if limited_capi: + nargs = 'PyTuple_Size(args)' + else: + nargs = 'PyTuple_GET_SIZE(args)' + out.append(f"switch ({nargs}) {{\n") + for subset in permute_optional_groups(left, required, right): + count = len(subset) + count_min = min(count_min, count) + count_max = max(count_max, count) + + if count == 0: + out.append(""" case 0: + break; +""") + continue + + group_ids = {p.group for p in subset} # eliminate duplicates + d: dict[str, str | int] = {} + d['count'] = count + d['name'] = f.name + d['format_units'] = "".join(p.converter.format_unit for p in subset) + + parse_arguments: list[str] = [] + for p in subset: + p.converter.parse_argument(parse_arguments) + d['parse_arguments'] = ", ".join(parse_arguments) + + group_ids.discard(0) + lines = "\n".join([ + self.group_to_variable_name(g) + " = 1;" + for g in group_ids + ]) + + s = """\ + case {count}: + if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ + goto exit; + }} + {group_booleans} + break; +""" + s = libclinic.linear_format(s, group_booleans=lines) + s = s.format_map(d) + out.append(s) + + out.append(" default:\n") + s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' + out.append(s.format(f.full_name, count_min, count_max)) + out.append(' goto exit;\n') + out.append("}") + + template_dict['option_group_parsing'] = libclinic.format_escape("".join(out)) + + def render_function( + self, + clinic: Clinic, + f: Function | None + ) -> str: + if f is None: + return "" + + codegen = clinic.codegen + data = CRenderData() + + assert f.parameters, "We should always have a 'self' at this point!" + parameters = f.render_parameters + converters = [p.converter for p in parameters] + + templates = self.output_templates(f, codegen) + + f_self = parameters[0] + selfless = parameters[1:] + assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!" + + if f.critical_section: + match len(f.target_critical_section): + case 0: + lock = 'Py_BEGIN_CRITICAL_SECTION({self_name});' + unlock = 'Py_END_CRITICAL_SECTION();' + case 1: + lock = 'Py_BEGIN_CRITICAL_SECTION({target_critical_section});' + unlock = 'Py_END_CRITICAL_SECTION();' + case _: + lock = 'Py_BEGIN_CRITICAL_SECTION2({target_critical_section});' + unlock = 'Py_END_CRITICAL_SECTION2();' + data.lock.append(lock) + data.unlock.append(unlock) + + last_group = 0 + first_optional = len(selfless) + positional = selfless and selfless[-1].is_positional_only() + has_option_groups = False + requires_defining_class = (len(selfless) + and isinstance(selfless[0].converter, + defining_class_converter)) + pass_vararg_directly = (all(p.is_positional_only() or p.is_vararg() + for p in selfless) + and not requires_defining_class) + + # offset i by -1 because first_optional needs to ignore self + for i, p in enumerate(parameters, -1): + c = p.converter + + if (i != -1) and (p.default is not unspecified): + first_optional = min(first_optional, i) + + if p.is_vararg() and not pass_vararg_directly: + data.cleanup.append(f"Py_XDECREF({c.parser_name});") + + # insert group variable + group = p.group + if last_group != group: + last_group = group + if group: + group_name = self.group_to_variable_name(group) + data.impl_arguments.append(group_name) + data.declarations.append("int " + group_name + " = 0;") + data.impl_parameters.append("int " + group_name) + has_option_groups = True + + if p.is_vararg() and pass_vararg_directly: + data.impl_arguments.append('nvararg') + data.impl_parameters.append('Py_ssize_t nargs') + p.converter.type = 'PyObject *const *' + + c.render(p, data) + + if has_option_groups and (not positional): + fail("You cannot use optional groups ('[' and ']') " + "unless all parameters are positional-only ('/').") + + # HACK + # when we're METH_O, but have a custom return converter, + # we use "impl_parameters" for the parsing function + # because that works better. but that means we must + # suppress actually declaring the impl's parameters + # as variables in the parsing function. but since it's + # METH_O, we have exactly one anyway, so we know exactly + # where it is. + if ("METH_O" in templates['methoddef_define'] and + '{impl_parameters}' in templates['parser_prototype']): + data.declarations.pop(0) + + full_name = f.full_name + template_dict = {'full_name': full_name} + template_dict['name'] = f.displayname + if f.kind in {GETTER, SETTER}: + template_dict['getset_name'] = f.c_basename.upper() + template_dict['getset_basename'] = f.c_basename + if f.kind is GETTER: + template_dict['c_basename'] = f.c_basename + "_get" + elif f.kind is SETTER: + template_dict['c_basename'] = f.c_basename + "_set" + # Implicitly add the setter value parameter. + data.impl_parameters.append("PyObject *value") + data.impl_arguments.append("value") + else: + template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" + template_dict['c_basename'] = f.c_basename + + template_dict['docstring'] = libclinic.docstring_for_c_string(f.docstring) + template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' + template_dict['target_critical_section'] = ', '.join(f.target_critical_section) + for converter in converters: + converter.set_template_dict(template_dict) + + if f.kind not in {SETTER, METHOD_INIT}: + f.return_converter.render(f, data) + template_dict['impl_return_type'] = f.return_converter.type + + template_dict['declarations'] = libclinic.format_escape("\n".join(data.declarations)) + template_dict['initializers'] = "\n\n".join(data.initializers) + template_dict['modifications'] = '\n\n'.join(data.modifications) + template_dict['keywords_c'] = ' '.join('"' + k + '",' + for k in data.keywords) + keywords = [k for k in data.keywords if k] + template_dict['keywords_py'] = ' '.join(c_id(k) + ',' + for k in keywords) + template_dict['format_units'] = ''.join(data.format_units) + template_dict['parse_arguments'] = ', '.join(data.parse_arguments) + if data.parse_arguments: + template_dict['parse_arguments_comma'] = ','; + else: + template_dict['parse_arguments_comma'] = ''; + template_dict['impl_parameters'] = ", ".join(data.impl_parameters) + template_dict['impl_arguments'] = ", ".join(data.impl_arguments) + + template_dict['return_conversion'] = libclinic.format_escape("".join(data.return_conversion).rstrip()) + template_dict['post_parsing'] = libclinic.format_escape("".join(data.post_parsing).rstrip()) + template_dict['cleanup'] = libclinic.format_escape("".join(data.cleanup)) + + template_dict['return_value'] = data.return_value + template_dict['lock'] = "\n".join(data.lock) + template_dict['unlock'] = "\n".join(data.unlock) + + # used by unpack tuple code generator + unpack_min = first_optional + unpack_max = len(selfless) + template_dict['unpack_min'] = str(unpack_min) + template_dict['unpack_max'] = str(unpack_max) + + if has_option_groups: + self.render_option_group_parsing(f, template_dict, + limited_capi=codegen.limited_capi) + + # buffers, not destination + for name, destination in clinic.destination_buffers.items(): + template = templates[name] + if has_option_groups: + template = libclinic.linear_format(template, + option_group_parsing=template_dict['option_group_parsing']) + template = libclinic.linear_format(template, + declarations=template_dict['declarations'], + return_conversion=template_dict['return_conversion'], + initializers=template_dict['initializers'], + modifications=template_dict['modifications'], + post_parsing=template_dict['post_parsing'], + cleanup=template_dict['cleanup'], + lock=template_dict['lock'], + unlock=template_dict['unlock'], + ) + + # Only generate the "exit:" label + # if we have any gotos + label = "exit:" if "goto exit;" in template else "" + template = libclinic.linear_format(template, exit_label=label) + + s = template.format_map(template_dict) + + # mild hack: + # reflow long impl declarations + if name in {"impl_prototype", "impl_definition"}: + s = libclinic.wrap_declarations(s) + + if clinic.line_prefix: + s = libclinic.indent_all_lines(s, clinic.line_prefix) + if clinic.line_suffix: + s = libclinic.suffix_all_lines(s, clinic.line_suffix) + + destination.append(s) + + return clinic.get_destination('block').dump() diff --git a/Tools/clinic/libclinic/cli.py b/Tools/clinic/libclinic/cli.py new file mode 100644 index 00000000000000..f36c6d04efd383 --- /dev/null +++ b/Tools/clinic/libclinic/cli.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import argparse +import inspect +import os +import re +import sys +from collections.abc import Callable +from typing import NoReturn + + +# Local imports. +import libclinic +import libclinic.cpp +from libclinic import ClinicError +from libclinic.language import Language, PythonLanguage +from libclinic.block_parser import BlockParser +from libclinic.converter import ( + ConverterType, converters, legacy_converters) +from libclinic.return_converters import ( + return_converters, ReturnConverterType) +from libclinic.clanguage import CLanguage +from libclinic.app import Clinic + + +# TODO: +# +# soon: +# +# * allow mixing any two of {positional-only, positional-or-keyword, +# keyword-only} +# * dict constructor uses positional-only and keyword-only +# * max and min use positional only with an optional group +# and keyword-only +# + + +# Match '#define Py_LIMITED_API'. +# Match '# define Py_LIMITED_API 0x030d0000' (without the version). +LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API') + + +# "extensions" maps the file extension ("c", "py") to Language classes. +LangDict = dict[str, Callable[[str], Language]] +extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } +extensions['py'] = PythonLanguage + + +def parse_file( + filename: str, + *, + limited_capi: bool, + output: str | None = None, + verify: bool = True, +) -> None: + if not output: + output = filename + + extension = os.path.splitext(filename)[1][1:] + if not extension: + raise ClinicError(f"Can't extract file type for file {filename!r}") + + try: + language = extensions[extension](filename) + except KeyError: + raise ClinicError(f"Can't identify file type for file {filename!r}") + + with open(filename, encoding="utf-8") as f: + raw = f.read() + + # exit quickly if there are no clinic markers in the file + find_start_re = BlockParser("", language).find_start_re + if not find_start_re.search(raw): + return + + if LIMITED_CAPI_REGEX.search(raw): + limited_capi = True + + assert isinstance(language, CLanguage) + clinic = Clinic(language, + verify=verify, + filename=filename, + limited_capi=limited_capi) + cooked = clinic.parse(raw) + + libclinic.write_file(output, cooked) + + +def create_cli() -> argparse.ArgumentParser: + cmdline = argparse.ArgumentParser( + prog="clinic.py", + description="""Preprocessor for CPython C files. + +The purpose of the Argument Clinic is automating all the boilerplate involved +with writing argument parsing code for builtins and providing introspection +signatures ("docstrings") for CPython builtins. + +For more information see https://devguide.python.org/development-tools/clinic/""") + cmdline.add_argument("-f", "--force", action='store_true', + help="force output regeneration") + cmdline.add_argument("-o", "--output", type=str, + help="redirect file output to OUTPUT") + cmdline.add_argument("-v", "--verbose", action='store_true', + help="enable verbose mode") + cmdline.add_argument("--converters", action='store_true', + help=("print a list of all supported converters " + "and return converters")) + cmdline.add_argument("--make", action='store_true', + help="walk --srcdir to run over all relevant files") + cmdline.add_argument("--srcdir", type=str, default=os.curdir, + help="the directory tree to walk in --make mode") + cmdline.add_argument("--exclude", type=str, action="append", + help=("a file to exclude in --make mode; " + "can be given multiple times")) + cmdline.add_argument("--limited", dest="limited_capi", action='store_true', + help="use the Limited C API") + cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", + help="the list of files to process") + return cmdline + + +def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: + if ns.converters: + if ns.filename: + parser.error( + "can't specify --converters and a filename at the same time" + ) + AnyConverterType = ConverterType | ReturnConverterType + converter_list: list[tuple[str, AnyConverterType]] = [] + return_converter_list: list[tuple[str, AnyConverterType]] = [] + + for name, converter in converters.items(): + converter_list.append(( + name, + converter, + )) + for name, return_converter in return_converters.items(): + return_converter_list.append(( + name, + return_converter + )) + + print() + + print("Legacy converters:") + legacy = sorted(legacy_converters) + print(' ' + ' '.join(c for c in legacy if c[0].isupper())) + print(' ' + ' '.join(c for c in legacy if c[0].islower())) + print() + + for title, attribute, ids in ( + ("Converters", 'converter_init', converter_list), + ("Return converters", 'return_converter_init', return_converter_list), + ): + print(title + ":") + + ids.sort(key=lambda item: item[0].lower()) + longest = -1 + for name, _ in ids: + longest = max(longest, len(name)) + + for name, cls in ids: + callable = getattr(cls, attribute, None) + if not callable: + continue + signature = inspect.signature(callable) + parameters = [] + for parameter_name, parameter in signature.parameters.items(): + if parameter.kind == inspect.Parameter.KEYWORD_ONLY: + if parameter.default != inspect.Parameter.empty: + s = f'{parameter_name}={parameter.default!r}' + else: + s = parameter_name + parameters.append(s) + print(' {}({})'.format(name, ', '.join(parameters))) + print() + print("All converters also accept (c_default=None, py_default=None, annotation=None).") + print("All return converters also accept (py_default=None).") + return + + if ns.make: + if ns.output or ns.filename: + parser.error("can't use -o or filenames with --make") + if not ns.srcdir: + parser.error("--srcdir must not be empty with --make") + if ns.exclude: + excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude] + excludes = [os.path.normpath(f) for f in excludes] + else: + excludes = [] + for root, dirs, files in os.walk(ns.srcdir): + for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): + if rcs_dir in dirs: + dirs.remove(rcs_dir) + for filename in files: + # handle .c, .cpp and .h files + if not filename.endswith(('.c', '.cpp', '.h')): + continue + path = os.path.join(root, filename) + path = os.path.normpath(path) + if path in excludes: + continue + if ns.verbose: + print(path) + parse_file(path, + verify=not ns.force, limited_capi=ns.limited_capi) + return + + if not ns.filename: + parser.error("no input files") + + if ns.output and len(ns.filename) > 1: + parser.error("can't use -o with multiple filenames") + + for filename in ns.filename: + if ns.verbose: + print(filename) + parse_file(filename, output=ns.output, + verify=not ns.force, limited_capi=ns.limited_capi) + + +def main(argv: list[str] | None = None) -> NoReturn: + parser = create_cli() + args = parser.parse_args(argv) + try: + run_clinic(parser, args) + except ClinicError as exc: + sys.stderr.write(exc.report()) + sys.exit(1) + else: + sys.exit(0) diff --git a/Tools/clinic/libclinic/codegen.py b/Tools/clinic/libclinic/codegen.py new file mode 100644 index 00000000000000..b2f1db6f8ef8da --- /dev/null +++ b/Tools/clinic/libclinic/codegen.py @@ -0,0 +1,302 @@ +from __future__ import annotations +import dataclasses as dc +import io +import os +from typing import Final, TYPE_CHECKING + +import libclinic +from libclinic import fail +from libclinic.language import Language +from libclinic.block_parser import Block +if TYPE_CHECKING: + from libclinic.app import Clinic + + +TemplateDict = dict[str, str] + + +class CRenderData: + def __init__(self) -> None: + + # The C statements to declare variables. + # Should be full lines with \n eol characters. + self.declarations: list[str] = [] + + # The C statements required to initialize the variables before the parse call. + # Should be full lines with \n eol characters. + self.initializers: list[str] = [] + + # The C statements needed to dynamically modify the values + # parsed by the parse call, before calling the impl. + self.modifications: list[str] = [] + + # The entries for the "keywords" array for PyArg_ParseTuple. + # Should be individual strings representing the names. + self.keywords: list[str] = [] + + # The "format units" for PyArg_ParseTuple. + # Should be individual strings that will get + self.format_units: list[str] = [] + + # The varargs arguments for PyArg_ParseTuple. + self.parse_arguments: list[str] = [] + + # The parameter declarations for the impl function. + self.impl_parameters: list[str] = [] + + # The arguments to the impl function at the time it's called. + self.impl_arguments: list[str] = [] + + # For return converters: the name of the variable that + # should receive the value returned by the impl. + self.return_value = "return_value" + + # For return converters: the code to convert the return + # value from the parse function. This is also where + # you should check the _return_value for errors, and + # "goto exit" if there are any. + self.return_conversion: list[str] = [] + self.converter_retval = "_return_value" + + # The C statements required to do some operations + # after the end of parsing but before cleaning up. + # These operations may be, for example, memory deallocations which + # can only be done without any error happening during argument parsing. + self.post_parsing: list[str] = [] + + # The C statements required to clean up after the impl call. + self.cleanup: list[str] = [] + + # The C statements to generate critical sections (per-object locking). + self.lock: list[str] = [] + self.unlock: list[str] = [] + + +@dc.dataclass(slots=True, frozen=True) +class Include: + """ + An include like: #include "pycore_long.h" // _Py_ID() + """ + # Example: "pycore_long.h". + filename: str + + # Example: "_Py_ID()". + reason: str + + # None means unconditional include. + # Example: "#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)". + condition: str | None + + def sort_key(self) -> tuple[str, str]: + # order: '#if' comes before 'NO_CONDITION' + return (self.condition or 'NO_CONDITION', self.filename) + + +@dc.dataclass(slots=True) +class BlockPrinter: + language: Language + f: io.StringIO = dc.field(default_factory=io.StringIO) + + # '#include "header.h" // reason': column of '//' comment + INCLUDE_COMMENT_COLUMN: Final[int] = 35 + + def print_block( + self, + block: Block, + *, + header_includes: list[Include] | None = None, + ) -> None: + input = block.input + output = block.output + dsl_name = block.dsl_name + write = self.f.write + + assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name) + + if not dsl_name: + write(input) + return + + write(self.language.start_line.format(dsl_name=dsl_name)) + write("\n") + + body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) + if not body_prefix: + write(input) + else: + for line in input.split('\n'): + write(body_prefix) + write(line) + write("\n") + + write(self.language.stop_line.format(dsl_name=dsl_name)) + write("\n") + + output = '' + if header_includes: + # Emit optional "#include" directives for C headers + output += '\n' + + current_condition: str | None = None + for include in header_includes: + if include.condition != current_condition: + if current_condition: + output += '#endif\n' + current_condition = include.condition + if include.condition: + output += f'{include.condition}\n' + + if current_condition: + line = f'# include "{include.filename}"' + else: + line = f'#include "{include.filename}"' + if include.reason: + comment = f'// {include.reason}\n' + line = line.ljust(self.INCLUDE_COMMENT_COLUMN - 1) + comment + output += line + + if current_condition: + output += '#endif\n' + + input = ''.join(block.input) + output += ''.join(block.output) + if output: + if not output.endswith('\n'): + output += '\n' + write(output) + + arguments = "output={output} input={input}".format( + output=libclinic.compute_checksum(output, 16), + input=libclinic.compute_checksum(input, 16) + ) + write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) + write("\n") + + def write(self, text: str) -> None: + self.f.write(text) + + +class BufferSeries: + """ + Behaves like a "defaultlist". + When you ask for an index that doesn't exist yet, + the object grows the list until that item exists. + So o[n] will always work. + + Supports negative indices for actual items. + e.g. o[-1] is an element immediately preceding o[0]. + """ + + def __init__(self) -> None: + self._start = 0 + self._array: list[list[str]] = [] + + def __getitem__(self, i: int) -> list[str]: + i -= self._start + if i < 0: + self._start += i + prefix: list[list[str]] = [[] for x in range(-i)] + self._array = prefix + self._array + i = 0 + while i >= len(self._array): + self._array.append([]) + return self._array[i] + + def clear(self) -> None: + for ta in self._array: + ta.clear() + + def dump(self) -> str: + texts = ["".join(ta) for ta in self._array] + self.clear() + return "".join(texts) + + +@dc.dataclass(slots=True, repr=False) +class Destination: + name: str + type: str + clinic: Clinic + buffers: BufferSeries = dc.field(init=False, default_factory=BufferSeries) + filename: str = dc.field(init=False) # set in __post_init__ + + args: dc.InitVar[tuple[str, ...]] = () + + def __post_init__(self, args: tuple[str, ...]) -> None: + valid_types = ('buffer', 'file', 'suppress') + if self.type not in valid_types: + fail( + f"Invalid destination type {self.type!r} for {self.name}, " + f"must be {', '.join(valid_types)}" + ) + extra_arguments = 1 if self.type == "file" else 0 + if len(args) < extra_arguments: + fail(f"Not enough arguments for destination " + f"{self.name!r} new {self.type!r}") + if len(args) > extra_arguments: + fail(f"Too many arguments for destination {self.name!r} new {self.type!r}") + if self.type =='file': + d = {} + filename = self.clinic.filename + d['path'] = filename + dirname, basename = os.path.split(filename) + if not dirname: + dirname = '.' + d['dirname'] = dirname + d['basename'] = basename + d['basename_root'], d['basename_extension'] = os.path.splitext(filename) + self.filename = args[0].format_map(d) + + def __repr__(self) -> str: + if self.type == 'file': + type_repr = f"type='file' file={self.filename!r}" + else: + type_repr = f"type={self.type!r}" + return f"" + + def clear(self) -> None: + if self.type != 'buffer': + fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'") + self.buffers.clear() + + def dump(self) -> str: + return self.buffers.dump() + + +DestinationDict = dict[str, Destination] + + +class CodeGen: + def __init__(self, limited_capi: bool) -> None: + self.limited_capi = limited_capi + self._ifndef_symbols: set[str] = set() + # dict: include name => Include instance + self._includes: dict[str, Include] = {} + + def add_ifndef_symbol(self, name: str) -> bool: + if name in self._ifndef_symbols: + return False + self._ifndef_symbols.add(name) + return True + + def add_include(self, name: str, reason: str, + *, condition: str | None = None) -> None: + try: + existing = self._includes[name] + except KeyError: + pass + else: + if existing.condition and not condition: + # If the previous include has a condition and the new one is + # unconditional, override the include. + pass + else: + # Already included, do nothing. Only mention a single reason, + # no need to list all of them. + return + + self._includes[name] = Include(name, reason, condition) + + def get_includes(self) -> list[Include]: + return sorted(self._includes.values(), + key=Include.sort_key) diff --git a/Tools/clinic/libclinic/converter.py b/Tools/clinic/libclinic/converter.py new file mode 100644 index 00000000000000..86853bb4fba253 --- /dev/null +++ b/Tools/clinic/libclinic/converter.py @@ -0,0 +1,552 @@ +from __future__ import annotations +import builtins as bltns +import functools +from typing import Any, TypeVar, Literal, TYPE_CHECKING, cast +from collections.abc import Callable + +import libclinic +from libclinic import fail +from libclinic import Sentinels, unspecified, unknown +from libclinic.codegen import CRenderData, Include, TemplateDict +from libclinic.function import Function, Parameter + + +CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) + + +type_checks = { + '&PyLong_Type': ('PyLong_Check', 'int'), + '&PyTuple_Type': ('PyTuple_Check', 'tuple'), + '&PyList_Type': ('PyList_Check', 'list'), + '&PySet_Type': ('PySet_Check', 'set'), + '&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'), + '&PyDict_Type': ('PyDict_Check', 'dict'), + '&PyUnicode_Type': ('PyUnicode_Check', 'str'), + '&PyBytes_Type': ('PyBytes_Check', 'bytes'), + '&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'), +} + + +def add_c_converter( + f: CConverterClassT, + name: str | None = None +) -> CConverterClassT: + if not name: + name = f.__name__ + if not name.endswith('_converter'): + return f + name = name.removesuffix('_converter') + converters[name] = f + return f + + +def add_default_legacy_c_converter(cls: CConverterClassT) -> CConverterClassT: + # automatically add converter for default format unit + # (but without stomping on the existing one if it's already + # set, in case you subclass) + if ((cls.format_unit not in ('O&', '')) and + (cls.format_unit not in legacy_converters)): + legacy_converters[cls.format_unit] = cls + return cls + + +class CConverterAutoRegister(type): + def __init__( + cls, name: str, bases: tuple[type[object], ...], classdict: dict[str, Any] + ) -> None: + converter_cls = cast(type["CConverter"], cls) + add_c_converter(converter_cls) + add_default_legacy_c_converter(converter_cls) + +class CConverter(metaclass=CConverterAutoRegister): + """ + For the init function, self, name, function, and default + must be keyword-or-positional parameters. All other + parameters must be keyword-only. + """ + + # The C name to use for this variable. + name: str + + # The Python name to use for this variable. + py_name: str + + # The C type to use for this variable. + # 'type' should be a Python string specifying the type, e.g. "int". + # If this is a pointer type, the type string should end with ' *'. + type: str | None = None + + # The Python default value for this parameter, as a Python value. + # Or the magic value "unspecified" if there is no default. + # Or the magic value "unknown" if this value is a cannot be evaluated + # at Argument-Clinic-preprocessing time (but is presumed to be valid + # at runtime). + default: object = unspecified + + # If not None, default must be isinstance() of this type. + # (You can also specify a tuple of types.) + default_type: bltns.type[object] | tuple[bltns.type[object], ...] | None = None + + # "default" converted into a C value, as a string. + # Or None if there is no default. + c_default: str | None = None + + # "default" converted into a Python value, as a string. + # Or None if there is no default. + py_default: str | None = None + + # The default value used to initialize the C variable when + # there is no default, but not specifying a default may + # result in an "uninitialized variable" warning. This can + # easily happen when using option groups--although + # properly-written code won't actually use the variable, + # the variable does get passed in to the _impl. (Ah, if + # only dataflow analysis could inline the static function!) + # + # This value is specified as a string. + # Every non-abstract subclass should supply a valid value. + c_ignored_default: str = 'NULL' + + # If true, wrap with Py_UNUSED. + unused = False + + # The C converter *function* to be used, if any. + # (If this is not None, format_unit must be 'O&'.) + converter: str | None = None + + # Should Argument Clinic add a '&' before the name of + # the variable when passing it into the _impl function? + impl_by_reference = False + + # Should Argument Clinic add a '&' before the name of + # the variable when passing it into PyArg_ParseTuple (AndKeywords)? + parse_by_reference = True + + ############################################################# + ############################################################# + ## You shouldn't need to read anything below this point to ## + ## write your own converter functions. ## + ############################################################# + ############################################################# + + # The "format unit" to specify for this variable when + # parsing arguments using PyArg_ParseTuple (AndKeywords). + # Custom converters should always use the default value of 'O&'. + format_unit = 'O&' + + # What encoding do we want for this variable? Only used + # by format units starting with 'e'. + encoding: str | None = None + + # Should this object be required to be a subclass of a specific type? + # If not None, should be a string representing a pointer to a + # PyTypeObject (e.g. "&PyUnicode_Type"). + # Only used by the 'O!' format unit (and the "object" converter). + subclass_of: str | None = None + + # See also the 'length_name' property. + # Only used by format units ending with '#'. + length = False + + # Should we show this parameter in the generated + # __text_signature__? This is *almost* always True. + # (It's only False for __new__, __init__, and METH_STATIC functions.) + show_in_signature = True + + # Overrides the name used in a text signature. + # The name used for a "self" parameter must be one of + # self, type, or module; however users can set their own. + # This lets the self_converter overrule the user-settable + # name, *just* for the text signature. + # Only set by self_converter. + signature_name: str | None = None + + broken_limited_capi: bool = False + + # keep in sync with self_converter.__init__! + def __init__(self, + # Positional args: + name: str, + py_name: str, + function: Function, + default: object = unspecified, + *, # Keyword only args: + c_default: str | None = None, + py_default: str | None = None, + annotation: str | Literal[Sentinels.unspecified] = unspecified, + unused: bool = False, + **kwargs: Any + ) -> None: + self.name = libclinic.ensure_legal_c_identifier(name) + self.py_name = py_name + self.unused = unused + self._includes: list[Include] = [] + + if default is not unspecified: + if (self.default_type + and default is not unknown + and not isinstance(default, self.default_type) + ): + if isinstance(self.default_type, type): + types_str = self.default_type.__name__ + else: + names = [cls.__name__ for cls in self.default_type] + types_str = ', '.join(names) + cls_name = self.__class__.__name__ + fail(f"{cls_name}: default value {default!r} for field " + f"{name!r} is not of type {types_str!r}") + self.default = default + + if c_default: + self.c_default = c_default + if py_default: + self.py_default = py_default + + if annotation is not unspecified: + fail("The 'annotation' parameter is not currently permitted.") + + # Make sure not to set self.function until after converter_init() has been called. + # This prevents you from caching information + # about the function in converter_init(). + # (That breaks if we get cloned.) + self.converter_init(**kwargs) + self.function = function + + # Add a custom __getattr__ method to improve the error message + # if somebody tries to access self.function in converter_init(). + # + # mypy will assume arbitrary access is okay for a class with a __getattr__ method, + # and that's not what we want, + # so put it inside an `if not TYPE_CHECKING` block + if not TYPE_CHECKING: + def __getattr__(self, attr): + if attr == "function": + fail( + f"{self.__class__.__name__!r} object has no attribute 'function'.\n" + f"Note: accessing self.function inside converter_init is disallowed!" + ) + return super().__getattr__(attr) + # this branch is just here for coverage reporting + else: # pragma: no cover + pass + + def converter_init(self) -> None: + pass + + def is_optional(self) -> bool: + return (self.default is not unspecified) + + def _render_self(self, parameter: Parameter, data: CRenderData) -> None: + self.parameter = parameter + name = self.parser_name + + # impl_arguments + s = ("&" if self.impl_by_reference else "") + name + data.impl_arguments.append(s) + if self.length: + data.impl_arguments.append(self.length_name) + + # impl_parameters + data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) + if self.length: + data.impl_parameters.append(f"Py_ssize_t {self.length_name}") + + def _render_non_self( + self, + parameter: Parameter, + data: CRenderData + ) -> None: + self.parameter = parameter + name = self.name + + # declarations + d = self.declaration(in_parser=True) + data.declarations.append(d) + + # initializers + initializers = self.initialize() + if initializers: + data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip()) + + # modifications + modifications = self.modify() + if modifications: + data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) + + # keywords + if parameter.is_vararg(): + pass + elif parameter.is_positional_only(): + data.keywords.append('') + else: + data.keywords.append(parameter.name) + + # format_units + if self.is_optional() and '|' not in data.format_units: + data.format_units.append('|') + if parameter.is_keyword_only() and '$' not in data.format_units: + data.format_units.append('$') + data.format_units.append(self.format_unit) + + # parse_arguments + self.parse_argument(data.parse_arguments) + + # post_parsing + if post_parsing := self.post_parsing(): + data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') + + # cleanup + cleanup = self.cleanup() + if cleanup: + data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") + + def render(self, parameter: Parameter, data: CRenderData) -> None: + """ + parameter is a clinic.Parameter instance. + data is a CRenderData instance. + """ + self._render_self(parameter, data) + self._render_non_self(parameter, data) + + @functools.cached_property + def length_name(self) -> str: + """Computes the name of the associated "length" variable.""" + assert self.length is not None + return self.parser_name + "_length" + + # Why is this one broken out separately? + # For "positional-only" function parsing, + # which generates a bunch of PyArg_ParseTuple calls. + def parse_argument(self, args: list[str]) -> None: + assert not (self.converter and self.encoding) + if self.format_unit == 'O&': + assert self.converter + args.append(self.converter) + + if self.encoding: + args.append(libclinic.c_repr(self.encoding)) + elif self.subclass_of: + args.append(self.subclass_of) + + s = ("&" if self.parse_by_reference else "") + self.parser_name + args.append(s) + + if self.length: + args.append(f"&{self.length_name}") + + # + # All the functions after here are intended as extension points. + # + + def simple_declaration( + self, + by_reference: bool = False, + *, + in_parser: bool = False + ) -> str: + """ + Computes the basic declaration of the variable. + Used in computing the prototype declaration and the + variable declaration. + """ + assert isinstance(self.type, str) + prototype = [self.type] + if by_reference or not self.type.endswith('*'): + prototype.append(" ") + if by_reference: + prototype.append('*') + if in_parser: + name = self.parser_name + else: + name = self.name + if self.unused: + name = f"Py_UNUSED({name})" + prototype.append(name) + return "".join(prototype) + + def declaration(self, *, in_parser: bool = False) -> str: + """ + The C statement to declare this variable. + """ + declaration = [self.simple_declaration(in_parser=True)] + default = self.c_default + if not default and self.parameter.group: + default = self.c_ignored_default + if default: + declaration.append(" = ") + declaration.append(default) + declaration.append(";") + if self.length: + declaration.append('\n') + declaration.append(f"Py_ssize_t {self.length_name};") + return "".join(declaration) + + def initialize(self) -> str: + """ + The C statements required to set up this variable before parsing. + Returns a string containing this code indented at column 0. + If no initialization is necessary, returns an empty string. + """ + return "" + + def modify(self) -> str: + """ + The C statements required to modify this variable after parsing. + Returns a string containing this code indented at column 0. + If no modification is necessary, returns an empty string. + """ + return "" + + def post_parsing(self) -> str: + """ + The C statements required to do some operations after the end of parsing but before cleaning up. + Return a string containing this code indented at column 0. + If no operation is necessary, return an empty string. + """ + return "" + + def cleanup(self) -> str: + """ + The C statements required to clean up after this variable. + Returns a string containing this code indented at column 0. + If no cleanup is necessary, returns an empty string. + """ + return "" + + def pre_render(self) -> None: + """ + A second initialization function, like converter_init, + called just before rendering. + You are permitted to examine self.function here. + """ + pass + + def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, expected_literal: bool = True) -> str: + assert '"' not in expected + if limited_capi: + if expected_literal: + return (f'PyErr_Format(PyExc_TypeError, ' + f'"{{{{name}}}}() {displayname} must be {expected}, not %T", ' + f'{{argname}});') + else: + return (f'PyErr_Format(PyExc_TypeError, ' + f'"{{{{name}}}}() {displayname} must be %s, not %T", ' + f'"{expected}", {{argname}});') + else: + if expected_literal: + expected = f'"{expected}"' + self.add_include('pycore_modsupport.h', '_PyArg_BadArgument()') + return f'_PyArg_BadArgument("{{{{name}}}}", "{displayname}", {expected}, {{argname}});' + + def format_code(self, fmt: str, *, + argname: str, + bad_argument: str | None = None, + bad_argument2: str | None = None, + **kwargs: Any) -> str: + if '{bad_argument}' in fmt: + if not bad_argument: + raise TypeError("required 'bad_argument' argument") + fmt = fmt.replace('{bad_argument}', bad_argument) + if '{bad_argument2}' in fmt: + if not bad_argument2: + raise TypeError("required 'bad_argument2' argument") + fmt = fmt.replace('{bad_argument2}', bad_argument2) + return fmt.format(argname=argname, paramname=self.parser_name, **kwargs) + + def use_converter(self) -> None: + """Method called when self.converter is used to parse an argument.""" + pass + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'O&': + self.use_converter() + return self.format_code(""" + if (!{converter}({argname}, &{paramname})) {{{{ + goto exit; + }}}} + """, + argname=argname, + converter=self.converter) + if self.format_unit == 'O!': + cast = '(%s)' % self.type if self.type != 'PyObject *' else '' + if self.subclass_of in type_checks: + typecheck, typename = type_checks[self.subclass_of] + return self.format_code(""" + if (!{typecheck}({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = {cast}{argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, typename, limited_capi=limited_capi), + typecheck=typecheck, typename=typename, cast=cast) + return self.format_code(""" + if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = {cast}{argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, '({subclass_of})->tp_name', + expected_literal=False, limited_capi=limited_capi), + subclass_of=self.subclass_of, cast=cast) + if self.format_unit == 'O': + cast = '(%s)' % self.type if self.type != 'PyObject *' else '' + return self.format_code(""" + {paramname} = {cast}{argname}; + """, + argname=argname, cast=cast) + return None + + def set_template_dict(self, template_dict: TemplateDict) -> None: + pass + + @property + def parser_name(self) -> str: + if self.name in libclinic.CLINIC_PREFIXED_ARGS: # bpo-39741 + return libclinic.CLINIC_PREFIX + self.name + else: + return self.name + + def add_include(self, name: str, reason: str, + *, condition: str | None = None) -> None: + include = Include(name, reason, condition) + self._includes.append(include) + + def get_includes(self) -> list[Include]: + return self._includes + + +ConverterType = Callable[..., CConverter] +ConverterDict = dict[str, ConverterType] + +# maps strings to callables. +# these callables must be of the form: +# def foo(name, default, *, ...) +# The callable may have any number of keyword-only parameters. +# The callable must return a CConverter object. +# The callable should not call builtins.print. +converters: ConverterDict = {} + +# maps strings to callables. +# these callables follow the same rules as those for "converters" above. +# note however that they will never be called with keyword-only parameters. +legacy_converters: ConverterDict = {} + + +def add_legacy_c_converter( + format_unit: str, + **kwargs: Any +) -> Callable[[CConverterClassT], CConverterClassT]: + def closure(f: CConverterClassT) -> CConverterClassT: + added_f: Callable[..., CConverter] + if not kwargs: + added_f = f + else: + added_f = functools.partial(f, **kwargs) + if format_unit: + legacy_converters[format_unit] = added_f + return f + return closure diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py new file mode 100644 index 00000000000000..bd5c2a2b73b94a --- /dev/null +++ b/Tools/clinic/libclinic/converters.py @@ -0,0 +1,1230 @@ +import builtins as bltns +import functools +import sys +from types import NoneType +from typing import Any + +from libclinic import fail, Null, unspecified, unknown +from libclinic.function import ( + Function, Parameter, + CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, + GETTER, SETTER) +from libclinic.codegen import CRenderData, TemplateDict +from libclinic.converter import ( + CConverter, legacy_converters, add_legacy_c_converter) + + +TypeSet = set[bltns.type[object]] + + +class bool_converter(CConverter): + type = 'int' + default_type = bool + format_unit = 'p' + c_ignored_default = '0' + + def converter_init(self, *, accept: TypeSet = {object}) -> None: + if accept == {int}: + self.format_unit = 'i' + elif accept != {object}: + fail(f"bool_converter: illegal 'accept' argument {accept!r}") + if self.default is not unspecified and self.default is not unknown: + self.default = bool(self.default) + self.c_default = str(int(self.default)) + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'i': + return self.format_code(""" + {paramname} = PyLong_AsInt({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + elif self.format_unit == 'p': + return self.format_code(""" + {paramname} = PyObject_IsTrue({argname}); + if ({paramname} < 0) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class defining_class_converter(CConverter): + """ + A special-case converter: + this is the default converter used for the defining class. + """ + type = 'PyTypeObject *' + format_unit = '' + show_in_signature = False + + def converter_init(self, *, type: str | None = None) -> None: + self.specified_type = type + + def render(self, parameter: Parameter, data: CRenderData) -> None: + self._render_self(parameter, data) + + def set_template_dict(self, template_dict: TemplateDict) -> None: + template_dict['defining_class_name'] = self.name + + +class char_converter(CConverter): + type = 'char' + default_type = (bytes, bytearray) + format_unit = 'c' + c_ignored_default = "'\0'" + + def converter_init(self) -> None: + if isinstance(self.default, self.default_type): + if len(self.default) != 1: + fail(f"char_converter: illegal default value {self.default!r}") + + self.c_default = repr(bytes(self.default))[1:] + if self.c_default == '"\'"': + self.c_default = r"'\''" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'c': + return self.format_code(""" + if (PyBytes_Check({argname})) {{{{ + if (PyBytes_GET_SIZE({argname}) != 1) {{{{ + PyErr_Format(PyExc_TypeError, + "{{name}}(): {displayname} must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE({argname})); + goto exit; + }}}} + {paramname} = PyBytes_AS_STRING({argname})[0]; + }}}} + else if (PyByteArray_Check({argname})) {{{{ + if (PyByteArray_GET_SIZE({argname}) != 1) {{{{ + PyErr_Format(PyExc_TypeError, + "{{name}}(): {displayname} must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE({argname})); + goto exit; + }}}} + {paramname} = PyByteArray_AS_STRING({argname})[0]; + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + displayname=displayname, + bad_argument=self.bad_argument(displayname, 'a byte string of length 1', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +@add_legacy_c_converter('B', bitwise=True) +class unsigned_char_converter(CConverter): + type = 'unsigned char' + default_type = int + format_unit = 'b' + c_ignored_default = "'\0'" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'B' + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'b': + return self.format_code(""" + {{{{ + long ival = PyLong_AsLong({argname}); + if (ival == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + else if (ival < 0) {{{{ + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + }}}} + else if (ival > UCHAR_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + }}}} + else {{{{ + {paramname} = (unsigned char) ival; + }}}} + }}}} + """, + argname=argname) + elif self.format_unit == 'B': + return self.format_code(""" + {{{{ + unsigned long ival = PyLong_AsUnsignedLongMask({argname}); + if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + else {{{{ + {paramname} = (unsigned char) ival; + }}}} + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class byte_converter(unsigned_char_converter): + pass + + +class short_converter(CConverter): + type = 'short' + default_type = int + format_unit = 'h' + c_ignored_default = "0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'h': + return self.format_code(""" + {{{{ + long ival = PyLong_AsLong({argname}); + if (ival == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + else if (ival < SHRT_MIN) {{{{ + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + }}}} + else if (ival > SHRT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + }}}} + else {{{{ + {paramname} = (short) ival; + }}}} + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_short_converter(CConverter): + type = 'unsigned short' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'H' + else: + self.converter = '_PyLong_UnsignedShort_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedShort_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedShort_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'H': + return self.format_code(""" + {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); + if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {{{{ + unsigned long uval = PyLong_AsUnsignedLong({argname}); + if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + if (uval > USHRT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C unsigned short"); + goto exit; + }}}} + {paramname} = (unsigned short) uval; + }}}} + """, + argname=argname) + + +@add_legacy_c_converter('C', accept={str}) +class int_converter(CConverter): + type = 'int' + default_type = int + format_unit = 'i' + c_ignored_default = "0" + + def converter_init( + self, *, accept: TypeSet = {int}, type: str | None = None + ) -> None: + if accept == {str}: + self.format_unit = 'C' + elif accept != {int}: + fail(f"int_converter: illegal 'accept' argument {accept!r}") + if type is not None: + self.type = type + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'i': + return self.format_code(""" + {paramname} = PyLong_AsInt({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + elif self.format_unit == 'C': + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + if (PyUnicode_GET_LENGTH({argname}) != 1) {{{{ + PyErr_Format(PyExc_TypeError, + "{{name}}(): {displayname} must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH({argname})); + goto exit; + }}}} + {paramname} = PyUnicode_READ_CHAR({argname}, 0); + """, + argname=argname, + displayname=displayname, + bad_argument=self.bad_argument(displayname, 'a unicode character', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_int_converter(CConverter): + type = 'unsigned int' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'I' + else: + self.converter = '_PyLong_UnsignedInt_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedInt_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedInt_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'I': + return self.format_code(""" + {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); + if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {{{{ + unsigned long uval = PyLong_AsUnsignedLong({argname}); + if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + if (uval > UINT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C unsigned int"); + goto exit; + }}}} + {paramname} = (unsigned int) uval; + }}}} + """, + argname=argname) + + +class long_converter(CConverter): + type = 'long' + default_type = int + format_unit = 'l' + c_ignored_default = "0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'l': + return self.format_code(""" + {paramname} = PyLong_AsLong({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_long_converter(CConverter): + type = 'unsigned long' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'k' + else: + self.converter = '_PyLong_UnsignedLong_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedLong_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedLong_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'k': + return self.format_code(""" + if (!PyLong_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyLong_AsUnsignedLongMask({argname}); + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), + ) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsUnsignedLong({argname}); + if ({paramname} == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class long_long_converter(CConverter): + type = 'long long' + default_type = int + format_unit = 'L' + c_ignored_default = "0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'L': + return self.format_code(""" + {paramname} = PyLong_AsLongLong({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_long_long_converter(CConverter): + type = 'unsigned long long' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'K' + else: + self.converter = '_PyLong_UnsignedLongLong_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedLongLong_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedLongLong_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'K': + return self.format_code(""" + if (!PyLong_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyLong_AsUnsignedLongLongMask({argname}); + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), + ) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsUnsignedLongLong({argname}); + if ({paramname} == (unsigned long long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class Py_ssize_t_converter(CConverter): + type = 'Py_ssize_t' + c_ignored_default = "0" + + def converter_init(self, *, accept: TypeSet = {int}) -> None: + if accept == {int}: + self.format_unit = 'n' + self.default_type = int + elif accept == {int, NoneType}: + self.converter = '_Py_convert_optional_to_ssize_t' + else: + fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") + + def use_converter(self) -> None: + if self.converter == '_Py_convert_optional_to_ssize_t': + self.add_include('pycore_abstract.h', + '_Py_convert_optional_to_ssize_t()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'n': + if limited_capi: + PyNumber_Index = 'PyNumber_Index' + else: + PyNumber_Index = '_PyNumber_Index' + self.add_include('pycore_abstract.h', '_PyNumber_Index()') + return self.format_code(""" + {{{{ + Py_ssize_t ival = -1; + PyObject *iobj = {PyNumber_Index}({argname}); + if (iobj != NULL) {{{{ + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + }}}} + if (ival == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + {paramname} = ival; + }}}} + """, + argname=argname, + PyNumber_Index=PyNumber_Index) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + return self.format_code(""" + if ({argname} != Py_None) {{{{ + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'integer or None', limited_capi=limited_capi), + ) + + +class slice_index_converter(CConverter): + type = 'Py_ssize_t' + + def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: + if accept == {int}: + self.converter = '_PyEval_SliceIndexNotNone' + self.nullable = False + elif accept == {int, NoneType}: + self.converter = '_PyEval_SliceIndex' + self.nullable = True + else: + fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + if self.nullable: + return self.format_code(""" + if (!Py_IsNone({argname})) {{{{ + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, NULL); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + return 0; + }}}} + }}}} + else {{{{ + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "None or have an __index__ method"); + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, NULL); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + goto exit; + }}}} + """, + argname=argname) + + +class size_t_converter(CConverter): + type = 'size_t' + converter = '_PyLong_Size_t_Converter' + c_ignored_default = "0" + + def use_converter(self) -> None: + self.add_include('pycore_long.h', + '_PyLong_Size_t_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'n': + return self.format_code(""" + {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsSize_t({argname}); + if ({paramname} == (size_t)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class fildes_converter(CConverter): + type = 'int' + converter = '_PyLong_FileDescriptor_Converter' + + def use_converter(self) -> None: + self.add_include('pycore_fileutils.h', + '_PyLong_FileDescriptor_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + return self.format_code(""" + {paramname} = PyObject_AsFileDescriptor({argname}); + if ({paramname} < 0) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class float_converter(CConverter): + type = 'float' + default_type = float + format_unit = 'f' + c_ignored_default = "0.0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'f': + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); + }}}} + else + {{{{ + {paramname} = (float) PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + {paramname} = (float) PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class double_converter(CConverter): + type = 'double' + default_type = float + format_unit = 'd' + c_ignored_default = "0.0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'd': + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = PyFloat_AS_DOUBLE({argname}); + }}}} + else + {{{{ + {paramname} = PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + {paramname} = PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class Py_complex_converter(CConverter): + type = 'Py_complex' + default_type = complex + format_unit = 'D' + c_ignored_default = "{0.0, 0.0}" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'D': + return self.format_code(""" + {paramname} = PyComplex_AsCComplex({argname}); + if (PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class object_converter(CConverter): + type = 'PyObject *' + format_unit = 'O' + + def converter_init( + self, *, + converter: str | None = None, + type: str | None = None, + subclass_of: str | None = None + ) -> None: + if converter: + if subclass_of: + fail("object: Cannot pass in both 'converter' and 'subclass_of'") + self.format_unit = 'O&' + self.converter = converter + elif subclass_of: + self.format_unit = 'O!' + self.subclass_of = subclass_of + + if type is not None: + self.type = type + + +# +# We define three conventions for buffer types in the 'accept' argument: +# +# buffer : any object supporting the buffer interface +# rwbuffer: any object supporting the buffer interface, but must be writeable +# robuffer: any object supporting the buffer interface, but must not be writeable +# + +class buffer: + pass +class rwbuffer: + pass +class robuffer: + pass + + +StrConverterKeyType = tuple[frozenset[type[object]], bool, bool] + +def str_converter_key( + types: TypeSet, encoding: bool | str | None, zeroes: bool +) -> StrConverterKeyType: + return (frozenset(types), bool(encoding), bool(zeroes)) + +str_converter_argument_map: dict[StrConverterKeyType, str] = {} + + +class str_converter(CConverter): + type = 'const char *' + default_type = (str, Null, NoneType) + format_unit = 's' + + def converter_init( + self, + *, + accept: TypeSet = {str}, + encoding: str | None = None, + zeroes: bool = False + ) -> None: + + key = str_converter_key(accept, encoding, zeroes) + format_unit = str_converter_argument_map.get(key) + if not format_unit: + fail("str_converter: illegal combination of arguments", key) + + self.format_unit = format_unit + self.length = bool(zeroes) + if encoding: + if self.default not in (Null, None, unspecified): + fail("str_converter: Argument Clinic doesn't support default values for encoded strings") + self.encoding = encoding + self.type = 'char *' + # sorry, clinic can't support preallocated buffers + # for es# and et# + self.c_default = "NULL" + if NoneType in accept and self.c_default == "Py_None": + self.c_default = "NULL" + + def post_parsing(self) -> str: + if self.encoding: + name = self.name + return f"PyMem_FREE({name});\n" + else: + return "" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 's': + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + if (strlen({paramname}) != (size_t){length_name}) {{{{ + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + length_name=self.length_name) + if self.format_unit == 'z': + return self.format_code(""" + if ({argname} == Py_None) {{{{ + {paramname} = NULL; + }}}} + else if (PyUnicode_Check({argname})) {{{{ + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + if (strlen({paramname}) != (size_t){length_name}) {{{{ + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), + length_name=self.length_name) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + +# +# This is the fourth or fifth rewrite of registering all the +# string converter format units. Previous approaches hid +# bugs--generally mismatches between the semantics of the format +# unit and the arguments necessary to represent those semantics +# properly. Hopefully with this approach we'll get it 100% right. +# +# The r() function (short for "register") both registers the +# mapping from arguments to format unit *and* registers the +# legacy C converter for that format unit. +# +def r(format_unit: str, + *, + accept: TypeSet, + encoding: bool = False, + zeroes: bool = False +) -> None: + if not encoding and format_unit != 's': + # add the legacy c converters here too. + # + # note: add_legacy_c_converter can't work for + # es, es#, et, or et# + # because of their extra encoding argument + # + # also don't add the converter for 's' because + # the metaclass for CConverter adds it for us. + kwargs: dict[str, Any] = {} + if accept != {str}: + kwargs['accept'] = accept + if zeroes: + kwargs['zeroes'] = True + added_f = functools.partial(str_converter, **kwargs) + legacy_converters[format_unit] = added_f + + d = str_converter_argument_map + key = str_converter_key(accept, encoding, zeroes) + if key in d: + sys.exit("Duplicate keys specified for str_converter_argument_map!") + d[key] = format_unit + +r('es', encoding=True, accept={str}) +r('es#', encoding=True, zeroes=True, accept={str}) +r('et', encoding=True, accept={bytes, bytearray, str}) +r('et#', encoding=True, zeroes=True, accept={bytes, bytearray, str}) +r('s', accept={str}) +r('s#', zeroes=True, accept={robuffer, str}) +r('y', accept={robuffer}) +r('y#', zeroes=True, accept={robuffer}) +r('z', accept={str, NoneType}) +r('z#', zeroes=True, accept={robuffer, str, NoneType}) +del r + + +class PyBytesObject_converter(CConverter): + type = 'PyBytesObject *' + format_unit = 'S' + # accept = {bytes} + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'S': + return self.format_code(""" + if (!PyBytes_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = ({type}){argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'bytes', limited_capi=limited_capi), + type=self.type) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class PyByteArrayObject_converter(CConverter): + type = 'PyByteArrayObject *' + format_unit = 'Y' + # accept = {bytearray} + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'Y': + return self.format_code(""" + if (!PyByteArray_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = ({type}){argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'bytearray', limited_capi=limited_capi), + type=self.type) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unicode_converter(CConverter): + type = 'PyObject *' + default_type = (str, Null, NoneType) + format_unit = 'U' + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'U': + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = {argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +@add_legacy_c_converter('u') +@add_legacy_c_converter('u#', zeroes=True) +@add_legacy_c_converter('Z', accept={str, NoneType}) +@add_legacy_c_converter('Z#', accept={str, NoneType}, zeroes=True) +class Py_UNICODE_converter(CConverter): + type = 'const wchar_t *' + default_type = (str, Null, NoneType) + + def converter_init( + self, *, + accept: TypeSet = {str}, + zeroes: bool = False + ) -> None: + format_unit = 'Z' if accept=={str, NoneType} else 'u' + if zeroes: + format_unit += '#' + self.length = True + self.format_unit = format_unit + else: + self.accept = accept + if accept == {str}: + self.converter = '_PyUnicode_WideCharString_Converter' + elif accept == {str, NoneType}: + self.converter = '_PyUnicode_WideCharString_Opt_Converter' + else: + fail(f"Py_UNICODE_converter: illegal 'accept' argument {accept!r}") + self.c_default = "NULL" + + def cleanup(self) -> str: + if self.length: + return "" + else: + return f"""PyMem_Free((void *){self.parser_name});\n""" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if not self.length: + if self.accept == {str}: + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyUnicode_AsWideCharString({argname}, NULL); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + ) + elif self.accept == {str, NoneType}: + return self.format_code(""" + if ({argname} == Py_None) {{{{ + {paramname} = NULL; + }}}} + else if (PyUnicode_Check({argname})) {{{{ + {paramname} = PyUnicode_AsWideCharString({argname}, NULL); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +@add_legacy_c_converter('s*', accept={str, buffer}) +@add_legacy_c_converter('z*', accept={str, buffer, NoneType}) +@add_legacy_c_converter('w*', accept={rwbuffer}) +class Py_buffer_converter(CConverter): + type = 'Py_buffer' + format_unit = 'y*' + impl_by_reference = True + c_ignored_default = "{NULL, NULL}" + + def converter_init(self, *, accept: TypeSet = {buffer}) -> None: + if self.default not in (unspecified, None): + fail("The only legal default value for Py_buffer is None.") + + self.c_default = self.c_ignored_default + + if accept == {str, buffer, NoneType}: + format_unit = 'z*' + elif accept == {str, buffer}: + format_unit = 's*' + elif accept == {buffer}: + format_unit = 'y*' + elif accept == {rwbuffer}: + format_unit = 'w*' + else: + fail("Py_buffer_converter: illegal combination of arguments") + + self.format_unit = format_unit + + def cleanup(self) -> str: + name = self.name + return "".join(["if (", name, ".obj) {\n PyBuffer_Release(&", name, ");\n}\n"]) + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + # PyBUF_SIMPLE guarantees that the format units of the buffers are C-contiguous. + if self.format_unit == 'y*': + return self.format_code(""" + if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + elif self.format_unit == 's*': + return self.format_code(""" + if (PyUnicode_Check({argname})) {{{{ + Py_ssize_t len; + const char *ptr = PyUnicode_AsUTF8AndSize({argname}, &len); + if (ptr == NULL) {{{{ + goto exit; + }}}} + if (PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) {{{{ + goto exit; + }}}} + }}}} + else {{{{ /* any bytes-like object */ + if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + elif self.format_unit == 'w*': + return self.format_code(""" + if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_WRITABLE) < 0) {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'read-write bytes-like object', limited_capi=limited_capi), + bad_argument2=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +def correct_name_for_self( + f: Function +) -> tuple[str, str]: + if f.kind in {CALLABLE, METHOD_INIT, GETTER, SETTER}: + if f.cls: + return "PyObject *", "self" + return "PyObject *", "module" + if f.kind is STATIC_METHOD: + return "void *", "null" + if f.kind in (CLASS_METHOD, METHOD_NEW): + return "PyTypeObject *", "type" + raise AssertionError(f"Unhandled type of function f: {f.kind!r}") + + +class self_converter(CConverter): + """ + A special-case converter: + this is the default converter used for "self". + """ + type: str | None = None + format_unit = '' + + def converter_init(self, *, type: str | None = None) -> None: + self.specified_type = type + + def pre_render(self) -> None: + f = self.function + default_type, default_name = correct_name_for_self(f) + self.signature_name = default_name + self.type = self.specified_type or self.type or default_type + + kind = self.function.kind + + if kind is STATIC_METHOD or kind.new_or_init: + self.show_in_signature = False + + # tp_new (METHOD_NEW) functions are of type newfunc: + # typedef PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *); + # + # tp_init (METHOD_INIT) functions are of type initproc: + # typedef int (*initproc)(PyObject *, PyObject *, PyObject *); + # + # All other functions generated by Argument Clinic are stored in + # PyMethodDef structures, in the ml_meth slot, which is of type PyCFunction: + # typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); + # However! We habitually cast these functions to PyCFunction, + # since functions that accept keyword arguments don't fit this signature + # but are stored there anyway. So strict type equality isn't important + # for these functions. + # + # So: + # + # * The name of the first parameter to the impl and the parsing function will always + # be self.name. + # + # * The type of the first parameter to the impl will always be of self.type. + # + # * If the function is neither tp_new (METHOD_NEW) nor tp_init (METHOD_INIT): + # * The type of the first parameter to the parsing function is also self.type. + # This means that if you step into the parsing function, your "self" parameter + # is of the correct type, which may make debugging more pleasant. + # + # * Else if the function is tp_new (METHOD_NEW): + # * The type of the first parameter to the parsing function is "PyTypeObject *", + # so the type signature of the function call is an exact match. + # * If self.type != "PyTypeObject *", we cast the first parameter to self.type + # in the impl call. + # + # * Else if the function is tp_init (METHOD_INIT): + # * The type of the first parameter to the parsing function is "PyObject *", + # so the type signature of the function call is an exact match. + # * If self.type != "PyObject *", we cast the first parameter to self.type + # in the impl call. + + @property + def parser_type(self) -> str: + assert self.type is not None + if self.function.kind in {METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD}: + tp, _ = correct_name_for_self(self.function) + return tp + return self.type + + def render(self, parameter: Parameter, data: CRenderData) -> None: + """ + parameter is a clinic.Parameter instance. + data is a CRenderData instance. + """ + if self.function.kind is STATIC_METHOD: + return + + self._render_self(parameter, data) + + if self.type != self.parser_type: + # insert cast to impl_argument[0], aka self. + # we know we're in the first slot in all the CRenderData lists, + # because we render parameters in order, and self is always first. + assert len(data.impl_arguments) == 1 + assert data.impl_arguments[0] == self.name + assert self.type is not None + data.impl_arguments[0] = '(' + self.type + ")" + data.impl_arguments[0] + + def set_template_dict(self, template_dict: TemplateDict) -> None: + template_dict['self_name'] = self.name + template_dict['self_type'] = self.parser_type + kind = self.function.kind + cls = self.function.cls + + if kind.new_or_init and cls and cls.typedef: + if kind is METHOD_NEW: + type_check = ( + '({0} == base_tp || {0}->tp_init == base_tp->tp_init)' + ).format(self.name) + else: + type_check = ('(Py_IS_TYPE({0}, base_tp) ||\n ' + ' Py_TYPE({0})->tp_new == base_tp->tp_new)' + ).format(self.name) + + line = f'{type_check} &&\n ' + template_dict['self_type_check'] = line + + type_object = cls.type_object + type_ptr = f'PyTypeObject *base_tp = {type_object};' + template_dict['base_type_ptr'] = type_ptr diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py new file mode 100644 index 00000000000000..5ca3bd5cb6c3f1 --- /dev/null +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -0,0 +1,1601 @@ +from __future__ import annotations +import ast +import enum +import inspect +import pprint +import re +import shlex +import sys +from collections.abc import Callable +from types import FunctionType, NoneType +from typing import TYPE_CHECKING, Any, NamedTuple + +import libclinic +from libclinic import ( + ClinicError, VersionTuple, + fail, warn, unspecified, unknown, NULL) +from libclinic.function import ( + Module, Class, Function, Parameter, + FunctionKind, + CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, + GETTER, SETTER) +from libclinic.converter import ( + converters, legacy_converters) +from libclinic.converters import ( + self_converter, defining_class_converter, + correct_name_for_self) +from libclinic.return_converters import ( + CReturnConverter, return_converters, + int_return_converter) +from libclinic.parser import create_parser_namespace +if TYPE_CHECKING: + from libclinic.block_parser import Block + from libclinic.app import Clinic + + +unsupported_special_methods: set[str] = set(""" + +__abs__ +__add__ +__and__ +__call__ +__delitem__ +__divmod__ +__eq__ +__float__ +__floordiv__ +__ge__ +__getattr__ +__getattribute__ +__getitem__ +__gt__ +__hash__ +__iadd__ +__iand__ +__ifloordiv__ +__ilshift__ +__imatmul__ +__imod__ +__imul__ +__index__ +__int__ +__invert__ +__ior__ +__ipow__ +__irshift__ +__isub__ +__iter__ +__itruediv__ +__ixor__ +__le__ +__len__ +__lshift__ +__lt__ +__matmul__ +__mod__ +__mul__ +__neg__ +__next__ +__or__ +__pos__ +__pow__ +__radd__ +__rand__ +__rdivmod__ +__repr__ +__rfloordiv__ +__rlshift__ +__rmatmul__ +__rmod__ +__rmul__ +__ror__ +__rpow__ +__rrshift__ +__rshift__ +__rsub__ +__rtruediv__ +__rxor__ +__setattr__ +__setitem__ +__str__ +__sub__ +__truediv__ +__xor__ + +""".strip().split()) + + +StateKeeper = Callable[[str], None] +ConverterArgs = dict[str, Any] + + +class ParamState(enum.IntEnum): + """Parameter parsing state. + + [ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] <- line + 01 2 3 4 5 6 <- state transitions + """ + # Before we've seen anything. + # Legal transitions: to LEFT_SQUARE_BEFORE or REQUIRED + START = 0 + + # Left square brackets before required params. + LEFT_SQUARE_BEFORE = 1 + + # In a group, before required params. + GROUP_BEFORE = 2 + + # Required params, positional-or-keyword or positional-only (we + # don't know yet). Renumber left groups! + REQUIRED = 3 + + # Positional-or-keyword or positional-only params that now must have + # default values. + OPTIONAL = 4 + + # In a group, after required params. + GROUP_AFTER = 5 + + # Right square brackets after required params. + RIGHT_SQUARE_AFTER = 6 + + +class FunctionNames(NamedTuple): + full_name: str + c_basename: str + + +def eval_ast_expr( + node: ast.expr, + *, + filename: str = '-' +) -> Any: + """ + Takes an ast.Expr node. Compiles it into a function object, + then calls the function object with 0 arguments. + Returns the result of that function call. + + globals represents the globals dict the expression + should see. (There's no equivalent for "locals" here.) + """ + + if isinstance(node, ast.Expr): + node = node.value + + expr = ast.Expression(node) + namespace = create_parser_namespace() + co = compile(expr, filename, 'eval') + fn = FunctionType(co, namespace) + return fn() + + +class IndentStack: + def __init__(self) -> None: + self.indents: list[int] = [] + self.margin: str | None = None + + def _ensure(self) -> None: + if not self.indents: + fail('IndentStack expected indents, but none are defined.') + + def measure(self, line: str) -> int: + """ + Returns the length of the line's margin. + """ + if '\t' in line: + fail('Tab characters are illegal in the Argument Clinic DSL.') + stripped = line.lstrip() + if not len(stripped): + # we can't tell anything from an empty line + # so just pretend it's indented like our current indent + self._ensure() + return self.indents[-1] + return len(line) - len(stripped) + + def infer(self, line: str) -> int: + """ + Infer what is now the current margin based on this line. + Returns: + 1 if we have indented (or this is the first margin) + 0 if the margin has not changed + -N if we have dedented N times + """ + indent = self.measure(line) + margin = ' ' * indent + if not self.indents: + self.indents.append(indent) + self.margin = margin + return 1 + current = self.indents[-1] + if indent == current: + return 0 + if indent > current: + self.indents.append(indent) + self.margin = margin + return 1 + # indent < current + if indent not in self.indents: + fail("Illegal outdent.") + outdent_count = 0 + while indent != current: + self.indents.pop() + current = self.indents[-1] + outdent_count -= 1 + self.margin = margin + return outdent_count + + @property + def depth(self) -> int: + """ + Returns how many margins are currently defined. + """ + return len(self.indents) + + def dedent(self, line: str) -> str: + """ + Dedents a line by the currently defined margin. + """ + assert self.margin is not None, "Cannot call .dedent() before calling .infer()" + margin = self.margin + indent = self.indents[-1] + if not line.startswith(margin): + fail('Cannot dedent; line does not start with the previous margin.') + return line[indent:] + + +class DSLParser: + function: Function | None + state: StateKeeper + keyword_only: bool + positional_only: bool + deprecated_positional: VersionTuple | None + deprecated_keyword: VersionTuple | None + group: int + parameter_state: ParamState + indent: IndentStack + kind: FunctionKind + coexist: bool + forced_text_signature: str | None + parameter_continuation: str + preserve_output: bool + critical_section: bool + target_critical_section: list[str] + from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') + + def __init__(self, clinic: Clinic) -> None: + self.clinic = clinic + + self.directives = {} + for name in dir(self): + # functions that start with directive_ are added to directives + _, s, key = name.partition("directive_") + if s: + self.directives[key] = getattr(self, name) + + # functions that start with at_ are too, with an @ in front + _, s, key = name.partition("at_") + if s: + self.directives['@' + key] = getattr(self, name) + + self.reset() + + def reset(self) -> None: + self.function = None + self.state = self.state_dsl_start + self.keyword_only = False + self.positional_only = False + self.deprecated_positional = None + self.deprecated_keyword = None + self.group = 0 + self.parameter_state: ParamState = ParamState.START + self.indent = IndentStack() + self.kind = CALLABLE + self.coexist = False + self.forced_text_signature = None + self.parameter_continuation = '' + self.preserve_output = False + self.critical_section = False + self.target_critical_section = [] + + def directive_module(self, name: str) -> None: + fields = name.split('.')[:-1] + module, cls = self.clinic._module_and_class(fields) + if cls: + fail("Can't nest a module inside a class!") + + if name in module.modules: + fail(f"Already defined module {name!r}!") + + m = Module(name, module) + module.modules[name] = m + self.block.signatures.append(m) + + def directive_class( + self, + name: str, + typedef: str, + type_object: str + ) -> None: + fields = name.split('.') + name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + parent = cls or module + if name in parent.classes: + fail(f"Already defined class {name!r}!") + + c = Class(name, module, cls, typedef, type_object) + parent.classes[name] = c + self.block.signatures.append(c) + + def directive_set(self, name: str, value: str) -> None: + if name not in ("line_prefix", "line_suffix"): + fail(f"unknown variable {name!r}") + + value = value.format_map({ + 'block comment start': '/*', + 'block comment end': '*/', + }) + + self.clinic.__dict__[name] = value + + def directive_destination( + self, + name: str, + command: str, + *args: str + ) -> None: + match command: + case "new": + self.clinic.add_destination(name, *args) + case "clear": + self.clinic.get_destination(name).clear() + case _: + fail(f"unknown destination command {command!r}") + + + def directive_output( + self, + command_or_name: str, + destination: str = '' + ) -> None: + fd = self.clinic.destination_buffers + + if command_or_name == "preset": + preset = self.clinic.presets.get(destination) + if not preset: + fail(f"Unknown preset {destination!r}!") + fd.update(preset) + return + + if command_or_name == "push": + self.clinic.destination_buffers_stack.append(fd.copy()) + return + + if command_or_name == "pop": + if not self.clinic.destination_buffers_stack: + fail("Can't 'output pop', stack is empty!") + previous_fd = self.clinic.destination_buffers_stack.pop() + fd.update(previous_fd) + return + + # secret command for debugging! + if command_or_name == "print": + self.block.output.append(pprint.pformat(fd)) + self.block.output.append('\n') + return + + d = self.clinic.get_destination_buffer(destination) + + if command_or_name == "everything": + for name in list(fd): + fd[name] = d + return + + if command_or_name not in fd: + allowed = ["preset", "push", "pop", "print", "everything"] + allowed.extend(fd) + fail(f"Invalid command or destination name {command_or_name!r}. " + "Must be one of:\n -", + "\n - ".join([repr(word) for word in allowed])) + fd[command_or_name] = d + + def directive_dump(self, name: str) -> None: + self.block.output.append(self.clinic.get_destination(name).dump()) + + def directive_printout(self, *args: str) -> None: + self.block.output.append(' '.join(args)) + self.block.output.append('\n') + + def directive_preserve(self) -> None: + if self.preserve_output: + fail("Can't have 'preserve' twice in one block!") + self.preserve_output = True + + def at_classmethod(self) -> None: + if self.kind is not CALLABLE: + fail("Can't set @classmethod, function is not a normal callable") + self.kind = CLASS_METHOD + + def at_critical_section(self, *args: str) -> None: + if len(args) > 2: + fail("Up to 2 critical section variables are supported") + self.target_critical_section.extend(args) + self.critical_section = True + + def at_getter(self) -> None: + match self.kind: + case FunctionKind.GETTER: + fail("Cannot apply @getter twice to the same function!") + case FunctionKind.SETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.GETTER + + def at_setter(self) -> None: + match self.kind: + case FunctionKind.SETTER: + fail("Cannot apply @setter twice to the same function!") + case FunctionKind.GETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.SETTER + + def at_staticmethod(self) -> None: + if self.kind is not CALLABLE: + fail("Can't set @staticmethod, function is not a normal callable") + self.kind = STATIC_METHOD + + def at_coexist(self) -> None: + if self.coexist: + fail("Called @coexist twice!") + self.coexist = True + + def at_text_signature(self, text_signature: str) -> None: + if self.forced_text_signature: + fail("Called @text_signature twice!") + self.forced_text_signature = text_signature + + def parse(self, block: Block) -> None: + self.reset() + self.block = block + self.saved_output = self.block.output + block.output = [] + block_start = self.clinic.block_parser.line_number + lines = block.input.split('\n') + for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): + if '\t' in line: + fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', + line_number=block_start) + try: + self.state(line) + except ClinicError as exc: + exc.lineno = line_number + exc.filename = self.clinic.filename + raise + + self.do_post_block_processing_cleanup(line_number) + block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) + + if self.preserve_output: + if block.output: + fail("'preserve' only works for blocks that don't produce any output!", + line_number=line_number) + block.output = self.saved_output + + def in_docstring(self) -> bool: + """Return true if we are processing a docstring.""" + return self.state in { + self.state_parameter_docstring, + self.state_function_docstring, + } + + def valid_line(self, line: str) -> bool: + # ignore comment-only lines + if line.lstrip().startswith('#'): + return False + + # Ignore empty lines too + # (but not in docstring sections!) + if not self.in_docstring() and not line.strip(): + return False + + return True + + def next( + self, + state: StateKeeper, + line: str | None = None + ) -> None: + self.state = state + if line is not None: + self.state(line) + + def state_dsl_start(self, line: str) -> None: + if not self.valid_line(line): + return + + # is it a directive? + fields = shlex.split(line) + directive_name = fields[0] + directive = self.directives.get(directive_name, None) + if directive: + try: + directive(*fields[1:]) + except TypeError as e: + fail(str(e)) + return + + self.next(self.state_modulename_name, line) + + def parse_function_names(self, line: str) -> FunctionNames: + left, as_, right = line.partition(' as ') + full_name = left.strip() + c_basename = right.strip() + if as_ and not c_basename: + fail("No C basename provided after 'as' keyword") + if not c_basename: + fields = full_name.split(".") + if fields[-1] == '__new__': + fields.pop() + c_basename = "_".join(fields) + if not libclinic.is_legal_py_identifier(full_name): + fail(f"Illegal function name: {full_name!r}") + if not libclinic.is_legal_c_identifier(c_basename): + fail(f"Illegal C basename: {c_basename!r}") + names = FunctionNames(full_name=full_name, c_basename=c_basename) + self.normalize_function_kind(names.full_name) + return names + + def normalize_function_kind(self, fullname: str) -> None: + # Fetch the method name and possibly class. + fields = fullname.split('.') + name = fields.pop() + _, cls = self.clinic._module_and_class(fields) + + # Check special method requirements. + if name in unsupported_special_methods: + fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") + if name == '__init__' and (self.kind is not CALLABLE or not cls): + fail(f"{name!r} must be a normal method; got '{self.kind}'!") + if name == '__new__' and (self.kind is not CLASS_METHOD or not cls): + fail("'__new__' must be a class method!") + if self.kind in {GETTER, SETTER} and not cls: + fail("@getter and @setter must be methods") + + # Normalise self.kind. + if name == '__new__': + self.kind = METHOD_NEW + elif name == '__init__': + self.kind = METHOD_INIT + + def resolve_return_converter( + self, full_name: str, forced_converter: str + ) -> CReturnConverter: + if forced_converter: + if self.kind in {GETTER, SETTER}: + fail(f"@{self.kind.name.lower()} method cannot define a return type") + if self.kind is METHOD_INIT: + fail("__init__ methods cannot define a return type") + ast_input = f"def x() -> {forced_converter}: pass" + try: + module_node = ast.parse(ast_input) + except SyntaxError: + fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") + function_node = module_node.body[0] + assert isinstance(function_node, ast.FunctionDef) + try: + name, legacy, kwargs = self.parse_converter(function_node.returns) + if legacy: + fail(f"Legacy converter {name!r} not allowed as a return converter") + if name not in return_converters: + fail(f"No available return converter called {name!r}") + return return_converters[name](**kwargs) + except ValueError: + fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") + + if self.kind in {METHOD_INIT, SETTER}: + return int_return_converter() + return CReturnConverter() + + def parse_cloned_function(self, names: FunctionNames, existing: str) -> None: + full_name, c_basename = names + fields = [x.strip() for x in existing.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + parent = cls or module + + for existing_function in parent.functions: + if existing_function.name == function_name: + break + else: + print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) + print(f"{(cls or module).functions=}", file=sys.stderr) + fail(f"Couldn't find existing function {existing!r}!") + + fields = [x.strip() for x in full_name.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + overrides: dict[str, Any] = { + "name": function_name, + "full_name": full_name, + "module": module, + "cls": cls, + "c_basename": c_basename, + "docstring": "", + } + if not (existing_function.kind is self.kind and + existing_function.coexist == self.coexist): + # Allow __new__ or __init__ methods. + if existing_function.kind.new_or_init: + overrides["kind"] = self.kind + # Future enhancement: allow custom return converters + overrides["return_converter"] = CReturnConverter() + else: + fail("'kind' of function and cloned function don't match! " + "(@classmethod/@staticmethod/@coexist)") + function = existing_function.copy(**overrides) + self.function = function + self.block.signatures.append(function) + (cls or module).functions.append(function) + self.next(self.state_function_docstring) + + def state_modulename_name(self, line: str) -> None: + # looking for declaration, which establishes the leftmost column + # line should be + # modulename.fnname [as c_basename] [-> return annotation] + # square brackets denote optional syntax. + # + # alternatively: + # modulename.fnname [as c_basename] = modulename.existing_fn_name + # clones the parameters and return converter from that + # function. you can't modify them. you must enter a + # new docstring. + # + # (but we might find a directive first!) + # + # this line is permitted to start with whitespace. + # we'll call this number of spaces F (for "function"). + + assert self.valid_line(line) + self.indent.infer(line) + + # are we cloning? + before, equals, existing = line.rpartition('=') + if equals: + existing = existing.strip() + if libclinic.is_legal_py_identifier(existing): + if self.forced_text_signature: + fail("Cannot use @text_signature when cloning a function") + # we're cloning! + names = self.parse_function_names(before) + return self.parse_cloned_function(names, existing) + + line, _, returns = line.partition('->') + returns = returns.strip() + full_name, c_basename = self.parse_function_names(line) + return_converter = self.resolve_return_converter(full_name, returns) + + fields = [x.strip() for x in full_name.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + func = Function( + name=function_name, + full_name=full_name, + module=module, + cls=cls, + c_basename=c_basename, + return_converter=return_converter, + kind=self.kind, + coexist=self.coexist, + critical_section=self.critical_section, + target_critical_section=self.target_critical_section, + forced_text_signature=self.forced_text_signature + ) + self.add_function(func) + + self.next(self.state_parameters_start) + + def add_function(self, func: Function) -> None: + # Insert a self converter automatically. + tp, name = correct_name_for_self(func) + if func.cls and tp == "PyObject *": + func.self_converter = self_converter(name, name, func, + type=func.cls.typedef) + else: + func.self_converter = self_converter(name, name, func) + func.parameters[name] = Parameter( + name, + inspect.Parameter.POSITIONAL_ONLY, + function=func, + converter=func.self_converter + ) + + self.block.signatures.append(func) + self.function = func + (func.cls or func.module).functions.append(func) + + # Now entering the parameters section. The rules, formally stated: + # + # * All lines must be indented with spaces only. + # * The first line must be a parameter declaration. + # * The first line must be indented. + # * This first line establishes the indent for parameters. + # * We'll call this number of spaces P (for "parameter"). + # * Thenceforth: + # * Lines indented with P spaces specify a parameter. + # * Lines indented with > P spaces are docstrings for the previous + # parameter. + # * We'll call this number of spaces D (for "docstring"). + # * All subsequent lines indented with >= D spaces are stored as + # part of the per-parameter docstring. + # * All lines will have the first D spaces of the indent stripped + # before they are stored. + # * It's illegal to have a line starting with a number of spaces X + # such that P < X < D. + # * A line with < P spaces is the first line of the function + # docstring, which ends processing for parameters and per-parameter + # docstrings. + # * The first line of the function docstring must be at the same + # indent as the function declaration. + # * It's illegal to have any line in the parameters section starting + # with X spaces such that F < X < P. (As before, F is the indent + # of the function declaration.) + # + # Also, currently Argument Clinic places the following restrictions on groups: + # * Each group must contain at least one parameter. + # * Each group may contain at most one group, which must be the furthest + # thing in the group from the required parameters. (The nested group + # must be the first in the group when it's before the required + # parameters, and the last thing in the group when after the required + # parameters.) + # * There may be at most one (top-level) group to the left or right of + # the required parameters. + # * You must specify a slash, and it must be after all parameters. + # (In other words: either all parameters are positional-only, + # or none are.) + # + # Said another way: + # * Each group must contain at least one parameter. + # * All left square brackets before the required parameters must be + # consecutive. (You can't have a left square bracket followed + # by a parameter, then another left square bracket. You can't + # have a left square bracket, a parameter, a right square bracket, + # and then a left square bracket.) + # * All right square brackets after the required parameters must be + # consecutive. + # + # These rules are enforced with a single state variable: + # "parameter_state". (Previously the code was a miasma of ifs and + # separate boolean state variables.) The states are defined in the + # ParamState class. + + def state_parameters_start(self, line: str) -> None: + if not self.valid_line(line): + return + + # if this line is not indented, we have no parameters + if not self.indent.infer(line): + return self.next(self.state_function_docstring, line) + + assert self.function is not None + if self.function.kind in {GETTER, SETTER}: + getset = self.function.kind.name.lower() + fail(f"@{getset} methods cannot define parameters") + + self.parameter_continuation = '' + return self.next(self.state_parameter, line) + + + def to_required(self) -> None: + """ + Transition to the "required" parameter state. + """ + if self.parameter_state is not ParamState.REQUIRED: + self.parameter_state = ParamState.REQUIRED + assert self.function is not None + for p in self.function.parameters.values(): + p.group = -p.group + + def state_parameter(self, line: str) -> None: + assert isinstance(self.function, Function) + + if not self.valid_line(line): + return + + if self.parameter_continuation: + line = self.parameter_continuation + ' ' + line.lstrip() + self.parameter_continuation = '' + + assert self.indent.depth == 2 + indent = self.indent.infer(line) + if indent == -1: + # we outdented, must be to definition column + return self.next(self.state_function_docstring, line) + + if indent == 1: + # we indented, must be to new parameter docstring column + return self.next(self.state_parameter_docstring_start, line) + + line = line.rstrip() + if line.endswith('\\'): + self.parameter_continuation = line[:-1] + return + + line = line.lstrip() + version: VersionTuple | None = None + match = self.from_version_re.fullmatch(line) + if match: + line = match[1] + version = self.parse_version(match[2]) + + func = self.function + match line: + case '*': + self.parse_star(func, version) + case '[': + self.parse_opening_square_bracket(func) + case ']': + self.parse_closing_square_bracket(func) + case '/': + self.parse_slash(func, version) + case param: + self.parse_parameter(param) + + def parse_parameter(self, line: str) -> None: + assert self.function is not None + + match self.parameter_state: + case ParamState.START | ParamState.REQUIRED: + self.to_required() + case ParamState.LEFT_SQUARE_BEFORE: + self.parameter_state = ParamState.GROUP_BEFORE + case ParamState.GROUP_BEFORE: + if not self.group: + self.to_required() + case ParamState.GROUP_AFTER | ParamState.OPTIONAL: + pass + case st: + fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.a)") + + # handle "as" for parameters too + c_name = None + name, have_as_token, trailing = line.partition(' as ') + if have_as_token: + name = name.strip() + if ' ' not in name: + fields = trailing.strip().split(' ') + if not fields: + fail("Invalid 'as' clause!") + c_name = fields[0] + if c_name.endswith(':'): + name += ':' + c_name = c_name[:-1] + fields[0] = name + line = ' '.join(fields) + + default: str | None + base, equals, default = line.rpartition('=') + if not equals: + base = default + default = None + + module = None + try: + ast_input = f"def x({base}): pass" + module = ast.parse(ast_input) + except SyntaxError: + try: + # the last = was probably inside a function call, like + # c: int(accept={str}) + # so assume there was no actual default value. + default = None + ast_input = f"def x({line}): pass" + module = ast.parse(ast_input) + except SyntaxError: + pass + if not module: + fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", + repr(line)) + + function = module.body[0] + assert isinstance(function, ast.FunctionDef) + function_args = function.args + + if len(function_args.args) > 1: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (comma?): {line!r}") + if function_args.defaults or function_args.kw_defaults: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (default value?): {line!r}") + if function_args.kwarg: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (**kwargs?): {line!r}") + + if function_args.vararg: + self.check_previous_star() + self.check_remaining_star() + is_vararg = True + parameter = function_args.vararg + else: + is_vararg = False + parameter = function_args.args[0] + + parameter_name = parameter.arg + name, legacy, kwargs = self.parse_converter(parameter.annotation) + + value: object + if not default: + if self.parameter_state is ParamState.OPTIONAL: + fail(f"Can't have a parameter without a default ({parameter_name!r}) " + "after a parameter with a default!") + if is_vararg: + value = NULL + kwargs.setdefault('c_default', "NULL") + else: + value = unspecified + if 'py_default' in kwargs: + fail("You can't specify py_default without specifying a default value!") + else: + if is_vararg: + fail("Vararg can't take a default value!") + + if self.parameter_state is ParamState.REQUIRED: + self.parameter_state = ParamState.OPTIONAL + default = default.strip() + bad = False + ast_input = f"x = {default}" + try: + module = ast.parse(ast_input) + + if 'c_default' not in kwargs: + # we can only represent very simple data values in C. + # detect whether default is okay, via a denylist + # of disallowed ast nodes. + class DetectBadNodes(ast.NodeVisitor): + bad = False + def bad_node(self, node: ast.AST) -> None: + self.bad = True + + # inline function call + visit_Call = bad_node + # inline if statement ("x = 3 if y else z") + visit_IfExp = bad_node + + # comprehensions and generator expressions + visit_ListComp = visit_SetComp = bad_node + visit_DictComp = visit_GeneratorExp = bad_node + + # literals for advanced types + visit_Dict = visit_Set = bad_node + visit_List = visit_Tuple = bad_node + + # "starred": "a = [1, 2, 3]; *a" + visit_Starred = bad_node + + denylist = DetectBadNodes() + denylist.visit(module) + bad = denylist.bad + else: + # if they specify a c_default, we can be more lenient about the default value. + # but at least make an attempt at ensuring it's a valid expression. + try: + value = eval(default) + except NameError: + pass # probably a named constant + except Exception as e: + fail("Malformed expression given as default value " + f"{default!r} caused {e!r}") + else: + if value is unspecified: + fail("'unspecified' is not a legal default value!") + if bad: + fail(f"Unsupported expression as default value: {default!r}") + + assignment = module.body[0] + assert isinstance(assignment, ast.Assign) + expr = assignment.value + # mild hack: explicitly support NULL as a default value + c_default: str | None + if isinstance(expr, ast.Name) and expr.id == 'NULL': + value = NULL + py_default = '' + c_default = "NULL" + elif (isinstance(expr, ast.BinOp) or + (isinstance(expr, ast.UnaryOp) and + not (isinstance(expr.operand, ast.Constant) and + type(expr.operand.value) in {int, float, complex}) + )): + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail(f"When you specify an expression ({default!r}) " + f"as your default value, " + f"you MUST specify a valid c_default.", + ast.dump(expr)) + py_default = default + value = unknown + elif isinstance(expr, ast.Attribute): + a = [] + n: ast.expr | ast.Attribute = expr + while isinstance(n, ast.Attribute): + a.append(n.attr) + n = n.value + if not isinstance(n, ast.Name): + fail(f"Unsupported default value {default!r} " + "(looked like a Python constant)") + a.append(n.id) + py_default = ".".join(reversed(a)) + + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail(f"When you specify a named constant ({py_default!r}) " + "as your default value, " + "you MUST specify a valid c_default.") + + try: + value = eval(py_default) + except NameError: + value = unknown + else: + value = ast.literal_eval(expr) + py_default = repr(value) + if isinstance(value, (bool, NoneType)): + c_default = "Py_" + py_default + elif isinstance(value, str): + c_default = libclinic.c_repr(value) + else: + c_default = py_default + + except SyntaxError as e: + fail(f"Syntax error: {e.text!r}") + except (ValueError, AttributeError): + value = unknown + c_default = kwargs.get("c_default") + py_default = default + if not (isinstance(c_default, str) and c_default): + fail("When you specify a named constant " + f"({py_default!r}) as your default value, " + "you MUST specify a valid c_default.") + + kwargs.setdefault('c_default', c_default) + kwargs.setdefault('py_default', py_default) + + dict = legacy_converters if legacy else converters + legacy_str = "legacy " if legacy else "" + if name not in dict: + fail(f'{name!r} is not a valid {legacy_str}converter') + # if you use a c_name for the parameter, we just give that name to the converter + # but the parameter object gets the python name + converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) + + kind: inspect._ParameterKind + if is_vararg: + kind = inspect.Parameter.VAR_POSITIONAL + elif self.keyword_only: + kind = inspect.Parameter.KEYWORD_ONLY + else: + kind = inspect.Parameter.POSITIONAL_OR_KEYWORD + + if isinstance(converter, self_converter): + if len(self.function.parameters) == 1: + if self.parameter_state is not ParamState.REQUIRED: + fail("A 'self' parameter cannot be marked optional.") + if value is not unspecified: + fail("A 'self' parameter cannot have a default value.") + if self.group: + fail("A 'self' parameter cannot be in an optional group.") + kind = inspect.Parameter.POSITIONAL_ONLY + self.parameter_state = ParamState.START + self.function.parameters.clear() + else: + fail("A 'self' parameter, if specified, must be the " + "very first thing in the parameter block.") + + if isinstance(converter, defining_class_converter): + _lp = len(self.function.parameters) + if _lp == 1: + if self.parameter_state is not ParamState.REQUIRED: + fail("A 'defining_class' parameter cannot be marked optional.") + if value is not unspecified: + fail("A 'defining_class' parameter cannot have a default value.") + if self.group: + fail("A 'defining_class' parameter cannot be in an optional group.") + if self.function.cls is None: + fail("A 'defining_class' parameter cannot be defined at module level.") + kind = inspect.Parameter.POSITIONAL_ONLY + else: + fail("A 'defining_class' parameter, if specified, must either " + "be the first thing in the parameter block, or come just " + "after 'self'.") + + + p = Parameter(parameter_name, kind, function=self.function, + converter=converter, default=value, group=self.group, + deprecated_positional=self.deprecated_positional) + + names = [k.name for k in self.function.parameters.values()] + if parameter_name in names[1:]: + fail(f"You can't have two parameters named {parameter_name!r}!") + elif names and parameter_name == names[0] and c_name is None: + fail(f"Parameter {parameter_name!r} requires a custom C name") + + key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name + self.function.parameters[key] = p + + if is_vararg: + self.keyword_only = True + + @staticmethod + def parse_converter( + annotation: ast.expr | None + ) -> tuple[str, bool, ConverterArgs]: + match annotation: + case ast.Constant(value=str() as value): + return value, True, {} + case ast.Name(name): + return name, False, {} + case ast.Call(func=ast.Name(name)): + kwargs: ConverterArgs = {} + for node in annotation.keywords: + if not isinstance(node.arg, str): + fail("Cannot use a kwarg splat in a function-call annotation") + kwargs[node.arg] = eval_ast_expr(node.value) + return name, False, kwargs + case _: + fail( + "Annotations must be either a name, a function call, or a string." + ) + + def parse_version(self, thenceforth: str) -> VersionTuple: + """Parse Python version in `[from ...]` marker.""" + assert isinstance(self.function, Function) + + try: + major, minor = thenceforth.split(".") + return int(major), int(minor) + except ValueError: + fail( + f"Function {self.function.name!r}: expected format '[from major.minor]' " + f"where 'major' and 'minor' are integers; got {thenceforth!r}" + ) + + def parse_star(self, function: Function, version: VersionTuple | None) -> None: + """Parse keyword-only parameter marker '*'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + self.check_previous_star() + self.check_remaining_star() + self.keyword_only = True + else: + if self.keyword_only: + fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") + if self.deprecated_positional: + if self.deprecated_positional == version: + fail(f"Function {function.name!r} uses '* [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_positional < version: + fail(f"Function {function.name!r}: '* [from " + f"{version[0]}.{version[1]}]' must precede '* [from " + f"{self.deprecated_positional[0]}.{self.deprecated_positional[1]}]'") + self.deprecated_positional = version + + def parse_opening_square_bracket(self, function: Function) -> None: + """Parse opening parameter group symbol '['.""" + match self.parameter_state: + case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: + self.parameter_state = ParamState.LEFT_SQUARE_BEFORE + case ParamState.REQUIRED | ParamState.GROUP_AFTER: + self.parameter_state = ParamState.GROUP_AFTER + case st: + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " + f"(Unexpected state {st}.b)") + self.group += 1 + function.docstring_only = True + + def parse_closing_square_bracket(self, function: Function) -> None: + """Parse closing parameter group symbol ']'.""" + if not self.group: + fail(f"Function {function.name!r} has a ']' without a matching '['.") + if not any(p.group == self.group for p in function.parameters.values()): + fail(f"Function {function.name!r} has an empty group. " + "All groups must contain at least one parameter.") + self.group -= 1 + match self.parameter_state: + case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: + self.parameter_state = ParamState.GROUP_BEFORE + case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: + self.parameter_state = ParamState.RIGHT_SQUARE_AFTER + case st: + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " + f"(Unexpected state {st}.c)") + + def parse_slash(self, function: Function, version: VersionTuple | None) -> None: + """Parse positional-only parameter marker '/'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + if self.deprecated_keyword: + fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/' must precede '*'") + if self.positional_only: + fail(f"Function {function.name!r} uses '/' more than once.") + else: + if self.deprecated_keyword: + if self.deprecated_keyword == version: + fail(f"Function {function.name!r} uses '/ [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_keyword > version: + fail(f"Function {function.name!r}: '/ [from " + f"{version[0]}.{version[1]}]' must precede '/ [from " + f"{self.deprecated_keyword[0]}.{self.deprecated_keyword[1]}]'") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '*'") + self.positional_only = True + self.deprecated_keyword = version + if version is not None: + found = False + for p in reversed(function.parameters.values()): + found = p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + break + if not found: + fail(f"Function {function.name!r} specifies '/ [from ...]' " + f"without preceding parameters.") + # REQUIRED and OPTIONAL are allowed here, that allows positional-only + # without option groups to work (and have default values!) + allowed = { + ParamState.REQUIRED, + ParamState.OPTIONAL, + ParamState.RIGHT_SQUARE_AFTER, + ParamState.GROUP_BEFORE, + } + if (self.parameter_state not in allowed) or self.group: + fail(f"Function {function.name!r} has an unsupported group configuration. " + f"(Unexpected state {self.parameter_state}.d)") + # fixup preceding parameters + for p in function.parameters.values(): + if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: + if version is None: + p.kind = inspect.Parameter.POSITIONAL_ONLY + elif p.deprecated_keyword is None: + p.deprecated_keyword = version + + def state_parameter_docstring_start(self, line: str) -> None: + assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" + self.parameter_docstring_indent = len(self.indent.margin) + assert self.indent.depth == 3 + return self.next(self.state_parameter_docstring, line) + + def docstring_append(self, obj: Function | Parameter, line: str) -> None: + """Add a rstripped line to the current docstring.""" + # gh-80282: We filter out non-ASCII characters from the docstring, + # since historically, some compilers may balk on non-ASCII input. + # If you're using Argument Clinic in an external project, + # you may not need to support the same array of platforms as CPython, + # so you may be able to remove this restriction. + matches = re.finditer(r'[^\x00-\x7F]', line) + if offending := ", ".join([repr(m[0]) for m in matches]): + warn("Non-ascii characters are not allowed in docstrings:", + offending) + + docstring = obj.docstring + if docstring: + docstring += "\n" + if stripped := line.rstrip(): + docstring += self.indent.dedent(stripped) + obj.docstring = docstring + + # every line of the docstring must start with at least F spaces, + # where F > P. + # these F spaces will be stripped. + def state_parameter_docstring(self, line: str) -> None: + if not self.valid_line(line): + return + + indent = self.indent.measure(line) + if indent < self.parameter_docstring_indent: + self.indent.infer(line) + assert self.indent.depth < 3 + if self.indent.depth == 2: + # back to a parameter + return self.next(self.state_parameter, line) + assert self.indent.depth == 1 + return self.next(self.state_function_docstring, line) + + assert self.function and self.function.parameters + last_param = next(reversed(self.function.parameters.values())) + self.docstring_append(last_param, line) + + # the final stanza of the DSL is the docstring. + def state_function_docstring(self, line: str) -> None: + assert self.function is not None + + if self.group: + fail(f"Function {self.function.name!r} has a ']' without a matching '['.") + + if not self.valid_line(line): + return + + self.docstring_append(self.function, line) + + @staticmethod + def format_docstring_signature( + f: Function, parameters: list[Parameter] + ) -> str: + lines = [] + lines.append(f.displayname) + if f.forced_text_signature: + lines.append(f.forced_text_signature) + elif f.kind in {GETTER, SETTER}: + # @getter and @setter do not need signatures like a method or a function. + return '' + else: + lines.append('(') + + # populate "right_bracket_count" field for every parameter + assert parameters, "We should always have a self parameter. " + repr(f) + assert isinstance(parameters[0].converter, self_converter) + # self is always positional-only. + assert parameters[0].is_positional_only() + assert parameters[0].right_bracket_count == 0 + positional_only = True + for p in parameters[1:]: + if not p.is_positional_only(): + positional_only = False + else: + assert positional_only + if positional_only: + p.right_bracket_count = abs(p.group) + else: + # don't put any right brackets around non-positional-only parameters, ever. + p.right_bracket_count = 0 + + right_bracket_count = 0 + + def fix_right_bracket_count(desired: int) -> str: + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + need_slash = False + added_slash = False + need_a_trailing_slash = False + + # we only need a trailing slash: + # * if this is not a "docstring_only" signature + # * and if the last *shown* parameter is + # positional only + if not f.docstring_only: + for p in reversed(parameters): + if not p.converter.show_in_signature: + continue + if p.is_positional_only(): + need_a_trailing_slash = True + break + + + added_star = False + + first_parameter = True + last_p = parameters[-1] + line_length = len(''.join(lines)) + indent = " " * line_length + def add_parameter(text: str) -> None: + nonlocal line_length + nonlocal first_parameter + if first_parameter: + s = text + first_parameter = False + else: + s = ' ' + text + if line_length + len(s) >= 72: + lines.extend(["\n", indent]) + line_length = len(indent) + s = text + line_length += len(s) + lines.append(s) + + for p in parameters: + if not p.converter.show_in_signature: + continue + assert p.name + + is_self = isinstance(p.converter, self_converter) + if is_self and f.docstring_only: + # this isn't a real machine-parsable signature, + # so let's not print the "self" parameter + continue + + if p.is_positional_only(): + need_slash = not f.docstring_only + elif need_slash and not (added_slash or p.is_positional_only()): + added_slash = True + add_parameter('/,') + + if p.is_keyword_only() and not added_star: + added_star = True + add_parameter('*,') + + p_lines = [fix_right_bracket_count(p.right_bracket_count)] + + if isinstance(p.converter, self_converter): + # annotate first parameter as being a "self". + # + # if inspect.Signature gets this function, + # and it's already bound, the self parameter + # will be stripped off. + # + # if it's not bound, it should be marked + # as positional-only. + # + # note: we don't print "self" for __init__, + # because this isn't actually the signature + # for __init__. (it can't be, __init__ doesn't + # have a docstring.) if this is an __init__ + # (or __new__), then this signature is for + # calling the class to construct a new instance. + p_lines.append('$') + + if p.is_vararg(): + p_lines.append("*") + added_star = True + + name = p.converter.signature_name or p.name + p_lines.append(name) + + if not p.is_vararg() and p.converter.is_optional(): + p_lines.append('=') + value = p.converter.py_default + if not value: + value = repr(p.converter.default) + p_lines.append(value) + + if (p != last_p) or need_a_trailing_slash: + p_lines.append(',') + + p_output = "".join(p_lines) + add_parameter(p_output) + + lines.append(fix_right_bracket_count(0)) + if need_a_trailing_slash: + add_parameter('/') + lines.append(')') + + # PEP 8 says: + # + # The Python standard library will not use function annotations + # as that would result in a premature commitment to a particular + # annotation style. Instead, the annotations are left for users + # to discover and experiment with useful annotation styles. + # + # therefore this is commented out: + # + # if f.return_converter.py_default: + # lines.append(' -> ') + # lines.append(f.return_converter.py_default) + + if not f.docstring_only: + lines.append("\n" + libclinic.SIG_END_MARKER + "\n") + + signature_line = "".join(lines) + + # now fix up the places where the brackets look wrong + return signature_line.replace(', ]', ',] ') + + @staticmethod + def format_docstring_parameters(params: list[Parameter]) -> str: + """Create substitution text for {parameters}""" + return "".join(p.render_docstring() + "\n" for p in params if p.docstring) + + def format_docstring(self) -> str: + assert self.function is not None + f = self.function + # For the following special cases, it does not make sense to render a docstring. + if f.kind in {METHOD_INIT, METHOD_NEW, GETTER, SETTER} and not f.docstring: + return f.docstring + + # Enforce the summary line! + # The first line of a docstring should be a summary of the function. + # It should fit on one line (80 columns? 79 maybe?) and be a paragraph + # by itself. + # + # Argument Clinic enforces the following rule: + # * either the docstring is empty, + # * or it must have a summary line. + # + # Guido said Clinic should enforce this: + # http://mail.python.org/pipermail/python-dev/2013-June/127110.html + + lines = f.docstring.split('\n') + if len(lines) >= 2: + if lines[1]: + fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" + "Every non-blank function docstring must start with " + "a single line summary followed by an empty line.") + elif len(lines) == 1: + # the docstring is only one line right now--the summary line. + # add an empty line after the summary line so we have space + # between it and the {parameters} we're about to add. + lines.append('') + + parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 + if parameters_marker_count > 1: + fail('You may not specify {parameters} more than once in a docstring!') + + # insert signature at front and params after the summary line + if not parameters_marker_count: + lines.insert(2, '{parameters}') + lines.insert(0, '{signature}') + + # finalize docstring + params = f.render_parameters + parameters = self.format_docstring_parameters(params) + signature = self.format_docstring_signature(f, params) + docstring = "\n".join(lines) + return libclinic.linear_format(docstring, + signature=signature, + parameters=parameters).rstrip() + + def check_remaining_star(self, lineno: int | None = None) -> None: + assert isinstance(self.function, Function) + + if self.keyword_only: + symbol = '*' + elif self.deprecated_positional: + symbol = '* [from ...]' + else: + return + + for p in reversed(self.function.parameters.values()): + if self.keyword_only: + if (p.kind == inspect.Parameter.KEYWORD_ONLY or + p.kind == inspect.Parameter.VAR_POSITIONAL): + return + elif self.deprecated_positional: + if p.deprecated_positional == self.deprecated_positional: + return + break + + fail(f"Function {self.function.name!r} specifies {symbol!r} " + f"without following parameters.", line_number=lineno) + + def check_previous_star(self) -> None: + assert isinstance(self.function, Function) + + if self.keyword_only: + fail(f"Function {self.function.name!r} uses '*' more than once.") + + + def do_post_block_processing_cleanup(self, lineno: int) -> None: + """ + Called when processing the block is done. + """ + if not self.function: + return + + self.check_remaining_star(lineno) + try: + self.function.docstring = self.format_docstring() + except ClinicError as exc: + exc.lineno = lineno + exc.filename = self.clinic.filename + raise diff --git a/Tools/clinic/libclinic/errors.py b/Tools/clinic/libclinic/errors.py index afb21b02386fe7..f06bdfbd864b2c 100644 --- a/Tools/clinic/libclinic/errors.py +++ b/Tools/clinic/libclinic/errors.py @@ -1,4 +1,5 @@ import dataclasses as dc +from typing import Literal, NoReturn, overload @dc.dataclass @@ -24,3 +25,48 @@ def report(self, *, warn_only: bool = False) -> str: class ParseError(ClinicError): pass + + +@overload +def warn_or_fail( + *args: object, + fail: Literal[True], + filename: str | None = None, + line_number: int | None = None, +) -> NoReturn: ... + +@overload +def warn_or_fail( + *args: object, + fail: Literal[False] = False, + filename: str | None = None, + line_number: int | None = None, +) -> None: ... + +def warn_or_fail( + *args: object, + fail: bool = False, + filename: str | None = None, + line_number: int | None = None, +) -> None: + joined = " ".join([str(a) for a in args]) + error = ClinicError(joined, filename=filename, lineno=line_number) + if fail: + raise error + else: + print(error.report(warn_only=True)) + + +def warn( + *args: object, + filename: str | None = None, + line_number: int | None = None, +) -> None: + return warn_or_fail(*args, filename=filename, line_number=line_number, fail=False) + +def fail( + *args: object, + filename: str | None = None, + line_number: int | None = None, +) -> NoReturn: + warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) diff --git a/Tools/clinic/libclinic/formatting.py b/Tools/clinic/libclinic/formatting.py index 8b3ad7ba566bc8..873ece6210017a 100644 --- a/Tools/clinic/libclinic/formatting.py +++ b/Tools/clinic/libclinic/formatting.py @@ -4,6 +4,8 @@ import textwrap from typing import Final +from libclinic import ClinicError + SIG_END_MARKER: Final = "--" @@ -171,3 +173,51 @@ def wrap_declarations(text: str, length: int = 78) -> str: lines.append(line.rstrip()) prefix = spaces return "\n".join(lines) + + +def linear_format(text: str, **kwargs: str) -> str: + """ + Perform str.format-like substitution, except: + * The strings substituted must be on lines by + themselves. (This line is the "source line".) + * If the substitution text is empty, the source line + is removed in the output. + * If the field is not recognized, the original line + is passed unmodified through to the output. + * If the substitution text is not empty: + * Each line of the substituted text is indented + by the indent of the source line. + * A newline will be added to the end. + """ + lines = [] + for line in text.split("\n"): + indent, curly, trailing = line.partition("{") + if not curly: + lines.extend([line, "\n"]) + continue + + name, curly, trailing = trailing.partition("}") + if not curly or name not in kwargs: + lines.extend([line, "\n"]) + continue + + if trailing: + raise ClinicError( + f"Text found after '{{{name}}}' block marker! " + "It must be on a line by itself." + ) + if indent.strip(): + raise ClinicError( + f"Non-whitespace characters found before '{{{name}}}' block marker! " + "It must be on a line by itself." + ) + + value = kwargs[name] + if not value: + continue + + stripped = [line.rstrip() for line in value.split("\n")] + value = textwrap.indent("\n".join(stripped), indent) + lines.extend([value, "\n"]) + + return "".join(lines[:-1]) diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py new file mode 100644 index 00000000000000..93901263e44c04 --- /dev/null +++ b/Tools/clinic/libclinic/function.py @@ -0,0 +1,310 @@ +from __future__ import annotations +import dataclasses as dc +import copy +import enum +import functools +import inspect +from collections.abc import Iterable, Iterator, Sequence +from typing import Final, Any, TYPE_CHECKING +if TYPE_CHECKING: + from libclinic.converter import CConverter + from libclinic.converters import self_converter + from libclinic.return_converters import CReturnConverter + from libclinic.app import Clinic + +from libclinic import VersionTuple, unspecified + + +ClassDict = dict[str, "Class"] +ModuleDict = dict[str, "Module"] +ParamDict = dict[str, "Parameter"] + + +@dc.dataclass(repr=False) +class Module: + name: str + module: Module | Clinic + + def __post_init__(self) -> None: + self.parent = self.module + self.modules: ModuleDict = {} + self.classes: ClassDict = {} + self.functions: list[Function] = [] + + def __repr__(self) -> str: + return "" + + +@dc.dataclass(repr=False) +class Class: + name: str + module: Module | Clinic + cls: Class | None + typedef: str + type_object: str + + def __post_init__(self) -> None: + self.parent = self.cls or self.module + self.classes: ClassDict = {} + self.functions: list[Function] = [] + + def __repr__(self) -> str: + return "" + + +class FunctionKind(enum.Enum): + CALLABLE = enum.auto() + STATIC_METHOD = enum.auto() + CLASS_METHOD = enum.auto() + METHOD_INIT = enum.auto() + METHOD_NEW = enum.auto() + GETTER = enum.auto() + SETTER = enum.auto() + + @functools.cached_property + def new_or_init(self) -> bool: + return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} + + def __repr__(self) -> str: + return f"" + + +CALLABLE: Final = FunctionKind.CALLABLE +STATIC_METHOD: Final = FunctionKind.STATIC_METHOD +CLASS_METHOD: Final = FunctionKind.CLASS_METHOD +METHOD_INIT: Final = FunctionKind.METHOD_INIT +METHOD_NEW: Final = FunctionKind.METHOD_NEW +GETTER: Final = FunctionKind.GETTER +SETTER: Final = FunctionKind.SETTER + + +@dc.dataclass(repr=False) +class Function: + """ + Mutable duck type for inspect.Function. + + docstring - a str containing + * embedded line breaks + * text outdented to the left margin + * no trailing whitespace. + It will always be true that + (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) + """ + parameters: ParamDict = dc.field(default_factory=dict) + _: dc.KW_ONLY + name: str + module: Module | Clinic + cls: Class | None + c_basename: str + full_name: str + return_converter: CReturnConverter + kind: FunctionKind + coexist: bool + return_annotation: object = inspect.Signature.empty + docstring: str = '' + # docstring_only means "don't generate a machine-readable + # signature, just a normal docstring". it's True for + # functions with optional groups because we can't represent + # those accurately with inspect.Signature in 3.4. + docstring_only: bool = False + forced_text_signature: str | None = None + critical_section: bool = False + target_critical_section: list[str] = dc.field(default_factory=list) + + def __post_init__(self) -> None: + self.parent = self.cls or self.module + self.self_converter: self_converter | None = None + self.__render_parameters__: list[Parameter] | None = None + + @functools.cached_property + def displayname(self) -> str: + """Pretty-printable name.""" + if self.kind.new_or_init: + assert isinstance(self.cls, Class) + return self.cls.name + else: + return self.name + + @functools.cached_property + def fulldisplayname(self) -> str: + parent: Class | Module | Clinic | None + if self.kind.new_or_init: + parent = getattr(self.cls, "parent", None) + else: + parent = self.parent + name = self.displayname + while isinstance(parent, (Module, Class)): + name = f"{parent.name}.{name}" + parent = parent.parent + return name + + @property + def render_parameters(self) -> list[Parameter]: + if not self.__render_parameters__: + l: list[Parameter] = [] + self.__render_parameters__ = l + for p in self.parameters.values(): + p = p.copy() + p.converter.pre_render() + l.append(p) + return self.__render_parameters__ + + @property + def methoddef_flags(self) -> str | None: + if self.kind.new_or_init: + return None + flags = [] + match self.kind: + case FunctionKind.CLASS_METHOD: + flags.append('METH_CLASS') + case FunctionKind.STATIC_METHOD: + flags.append('METH_STATIC') + case _ as kind: + acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER, FunctionKind.SETTER} + assert kind in acceptable_kinds, f"unknown kind: {kind!r}" + if self.coexist: + flags.append('METH_COEXIST') + return '|'.join(flags) + + def __repr__(self) -> str: + return f'' + + def copy(self, **overrides: Any) -> Function: + f = dc.replace(self, **overrides) + f.parameters = { + name: value.copy(function=f) + for name, value in f.parameters.items() + } + return f + + +@dc.dataclass(repr=False, slots=True) +class Parameter: + """ + Mutable duck type of inspect.Parameter. + """ + name: str + kind: inspect._ParameterKind + _: dc.KW_ONLY + default: object = inspect.Parameter.empty + function: Function + converter: CConverter + annotation: object = inspect.Parameter.empty + docstring: str = '' + group: int = 0 + # (`None` signifies that there is no deprecation) + deprecated_positional: VersionTuple | None = None + deprecated_keyword: VersionTuple | None = None + right_bracket_count: int = dc.field(init=False, default=0) + + def __repr__(self) -> str: + return f'' + + def is_keyword_only(self) -> bool: + return self.kind == inspect.Parameter.KEYWORD_ONLY + + def is_positional_only(self) -> bool: + return self.kind == inspect.Parameter.POSITIONAL_ONLY + + def is_vararg(self) -> bool: + return self.kind == inspect.Parameter.VAR_POSITIONAL + + def is_optional(self) -> bool: + return not self.is_vararg() and (self.default is not unspecified) + + def copy( + self, + /, + *, + converter: CConverter | None = None, + function: Function | None = None, + **overrides: Any + ) -> Parameter: + function = function or self.function + if not converter: + converter = copy.copy(self.converter) + converter.function = function + return dc.replace(self, **overrides, function=function, converter=converter) + + def get_displayname(self, i: int) -> str: + if i == 0: + return 'argument' + if not self.is_positional_only(): + return f'argument {self.name!r}' + else: + return f'argument {i}' + + def render_docstring(self) -> str: + lines = [f" {self.name}"] + lines.extend(f" {line}" for line in self.docstring.split("\n")) + return "\n".join(lines).rstrip() + + +ParamTuple = tuple["Parameter", ...] + + +def permute_left_option_groups( + l: Sequence[Iterable[Parameter]] +) -> Iterator[ParamTuple]: + """ + Given [(1,), (2,), (3,)], should yield: + () + (3,) + (2, 3) + (1, 2, 3) + """ + yield tuple() + accumulator: list[Parameter] = [] + for group in reversed(l): + accumulator = list(group) + accumulator + yield tuple(accumulator) + + +def permute_right_option_groups( + l: Sequence[Iterable[Parameter]] +) -> Iterator[ParamTuple]: + """ + Given [(1,), (2,), (3,)], should yield: + () + (1,) + (1, 2) + (1, 2, 3) + """ + yield tuple() + accumulator: list[Parameter] = [] + for group in l: + accumulator.extend(group) + yield tuple(accumulator) + + +def permute_optional_groups( + left: Sequence[Iterable[Parameter]], + required: Iterable[Parameter], + right: Sequence[Iterable[Parameter]] +) -> tuple[ParamTuple, ...]: + """ + Generator function that computes the set of acceptable + argument lists for the provided iterables of + argument groups. (Actually it generates a tuple of tuples.) + + Algorithm: prefer left options over right options. + + If required is empty, left must also be empty. + """ + required = tuple(required) + if not required: + if left: + raise ValueError("required is empty but left is not") + + accumulator: list[ParamTuple] = [] + counts = set() + for r in permute_right_option_groups(right): + for l in permute_left_option_groups(left): + t = l + required + r + if len(t) in counts: + continue + counts.add(len(t)) + accumulator.append(t) + + accumulator.sort(key=len) + return tuple(accumulator) diff --git a/Tools/clinic/libclinic/identifiers.py b/Tools/clinic/libclinic/identifiers.py new file mode 100644 index 00000000000000..d3b80bbcef3b2b --- /dev/null +++ b/Tools/clinic/libclinic/identifiers.py @@ -0,0 +1,31 @@ +import re +from .errors import ClinicError + + +is_legal_c_identifier = re.compile("^[A-Za-z_][A-Za-z0-9_]*$").match + + +def is_legal_py_identifier(identifier: str) -> bool: + return all(is_legal_c_identifier(field) for field in identifier.split(".")) + + +# Identifiers that are okay in Python but aren't a good idea in C. +# So if they're used Argument Clinic will add "_value" to the end +# of the name in C. +_c_keywords = frozenset(""" +asm auto break case char const continue default do double +else enum extern float for goto if inline int long +register return short signed sizeof static struct switch +typedef typeof union unsigned void volatile while +""".strip().split() +) + + +def ensure_legal_c_identifier(identifier: str) -> str: + # For now, just complain if what we're given isn't legal. + if not is_legal_c_identifier(identifier): + raise ClinicError(f"Illegal C identifier: {identifier}") + # But if we picked a C keyword, pick something else. + if identifier in _c_keywords: + return identifier + "_value" + return identifier diff --git a/Tools/clinic/libclinic/language.py b/Tools/clinic/libclinic/language.py new file mode 100644 index 00000000000000..15975430c16022 --- /dev/null +++ b/Tools/clinic/libclinic/language.py @@ -0,0 +1,103 @@ +from __future__ import annotations +import abc +import typing +from collections.abc import ( + Iterable, +) + +import libclinic +from libclinic import fail +from libclinic.function import ( + Module, Class, Function) + +if typing.TYPE_CHECKING: + from libclinic.app import Clinic + + +class Language(metaclass=abc.ABCMeta): + + start_line = "" + body_prefix = "" + stop_line = "" + checksum_line = "" + + def __init__(self, filename: str) -> None: + self.filename = filename + + @abc.abstractmethod + def render( + self, + clinic: Clinic, + signatures: Iterable[Module | Class | Function] + ) -> str: + ... + + def parse_line(self, line: str) -> None: + ... + + def validate(self) -> None: + def assert_only_one( + attr: str, + *additional_fields: str + ) -> None: + """ + Ensures that the string found at getattr(self, attr) + contains exactly one formatter replacement string for + each valid field. The list of valid fields is + ['dsl_name'] extended by additional_fields. + + e.g. + self.fmt = "{dsl_name} {a} {b}" + + # this passes + self.assert_only_one('fmt', 'a', 'b') + + # this fails, the format string has a {b} in it + self.assert_only_one('fmt', 'a') + + # this fails, the format string doesn't have a {c} in it + self.assert_only_one('fmt', 'a', 'b', 'c') + + # this fails, the format string has two {a}s in it, + # it must contain exactly one + self.fmt2 = '{dsl_name} {a} {a}' + self.assert_only_one('fmt2', 'a') + + """ + fields = ['dsl_name'] + fields.extend(additional_fields) + line: str = getattr(self, attr) + fcf = libclinic.FormatCounterFormatter() + fcf.format(line) + def local_fail(should_be_there_but_isnt: bool) -> None: + if should_be_there_but_isnt: + fail("{} {} must contain {{{}}} exactly once!".format( + self.__class__.__name__, attr, name)) + else: + fail("{} {} must not contain {{{}}}!".format( + self.__class__.__name__, attr, name)) + + for name, count in fcf.counts.items(): + if name in fields: + if count > 1: + local_fail(True) + else: + local_fail(False) + for name in fields: + if fcf.counts.get(name) != 1: + local_fail(True) + + assert_only_one('start_line') + assert_only_one('stop_line') + + field = "arguments" if "{arguments}" in self.checksum_line else "checksum" + assert_only_one('checksum_line', field) + + +class PythonLanguage(Language): + + language = 'Python' + start_line = "#/*[{dsl_name} input]" + body_prefix = "#" + stop_line = "#[{dsl_name} start generated code]*/" + checksum_line = "#/*[{dsl_name} end generated code: {arguments}]*/" diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py new file mode 100644 index 00000000000000..559d4fbdd09c57 --- /dev/null +++ b/Tools/clinic/libclinic/parse_args.py @@ -0,0 +1,935 @@ +from __future__ import annotations +from typing import TYPE_CHECKING, Final + +import libclinic +from libclinic import fail, warn +from libclinic.function import ( + Function, Parameter, + GETTER, SETTER, METHOD_NEW) +from libclinic.converter import CConverter +from libclinic.converters import ( + defining_class_converter, object_converter, self_converter) +if TYPE_CHECKING: + from libclinic.clanguage import CLanguage + from libclinic.codegen import CodeGen + + +def declare_parser( + f: Function, + *, + hasformat: bool = False, + codegen: CodeGen, +) -> str: + """ + Generates the code template for a static local PyArg_Parser variable, + with an initializer. For core code (incl. builtin modules) the + kwtuple field is also statically initialized. Otherwise + it is initialized at runtime. + """ + limited_capi = codegen.limited_capi + if hasformat: + fname = '' + format_ = '.format = "{format_units}:{name}",' + else: + fname = '.fname = "{name}",' + format_ = '' + + num_keywords = len([ + p for p in f.parameters.values() + if not p.is_positional_only() and not p.is_vararg() + ]) + + condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' + if limited_capi: + declarations = """ + #define KWTUPLE NULL + """ + elif num_keywords == 0: + declarations = """ + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + """ + + codegen.add_include('pycore_runtime.h', '_Py_SINGLETON()', + condition=condition) + else: + # XXX Why do we not statically allocate the tuple + # for non-builtin modules? + declarations = """ + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS %d + static struct {{ + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + }} _kwtuple = {{ + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = {{ {keywords_py} }}, + }}; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + """ % num_keywords + + codegen.add_include('pycore_gc.h', 'PyGC_Head', + condition=condition) + codegen.add_include('pycore_runtime.h', '_Py_ID()', + condition=condition) + + declarations += """ + static const char * const _keywords[] = {{{keywords_c} NULL}}; + static _PyArg_Parser _parser = {{ + .keywords = _keywords, + %s + .kwtuple = KWTUPLE, + }}; + #undef KWTUPLE + """ % (format_ or fname) + return libclinic.normalize_snippet(declarations) + + +NO_VARARG: Final[str] = "PY_SSIZE_T_MAX" +PARSER_PROTOTYPE_KEYWORD: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) +""") +PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = libclinic.normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) +""") +PARSER_PROTOTYPE_VARARGS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args) +""") +PARSER_PROTOTYPE_FASTCALL: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) +""") +PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +""") +PARSER_PROTOTYPE_DEF_CLASS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +""") +PARSER_PROTOTYPE_NOARGS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) +""") +PARSER_PROTOTYPE_GETTER: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, void *Py_UNUSED(context)) +""") +PARSER_PROTOTYPE_SETTER: Final[str] = libclinic.normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *value, void *Py_UNUSED(context)) +""") +METH_O_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({impl_parameters}) +""") +DOCSTRING_PROTOTYPE_VAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_VAR({c_basename}__doc__); +""") +DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_STRVAR({c_basename}__doc__, + {docstring}); +""") +GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_STRVAR({getset_basename}__doc__, + {docstring}); + #define {getset_basename}_HAS_DOCSTR +""") +IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" + static {impl_return_type} + {c_basename}_impl({impl_parameters}) +""") +METHODDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #define {methoddef_name} \ + {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, +""") +GETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #if defined({getset_basename}_HAS_DOCSTR) + # define {getset_basename}_DOCSTR {getset_basename}__doc__ + #else + # define {getset_basename}_DOCSTR NULL + #endif + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, NULL, {getset_basename}_DOCSTR}}, + #endif +""") +SETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #if defined({getset_name}_HAS_DOCSTR) + # define {getset_basename}_DOCSTR {getset_basename}__doc__ + #else + # define {getset_basename}_DOCSTR NULL + #endif + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", NULL, (setter){getset_basename}_set, NULL}}, + #endif +""") +METHODDEF_PROTOTYPE_IFNDEF: Final[str] = libclinic.normalize_snippet(""" + #ifndef {methoddef_name} + #define {methoddef_name} + #endif /* !defined({methoddef_name}) */ +""") + + +class ParseArgsCodeGen: + func: Function + codegen: CodeGen + limited_capi: bool = False + + # Function parameters + parameters: list[Parameter] + converters: list[CConverter] + + # Is 'defining_class' used for the first parameter? + requires_defining_class: bool + + # Use METH_FASTCALL calling convention? + fastcall: bool + + # Declaration of the return variable (ex: "int return_value;") + return_value_declaration: str + + # Calling convention (ex: "METH_NOARGS") + flags: str + + # Variables declarations + declarations: str + + pos_only: int = 0 + min_pos: int = 0 + max_pos: int = 0 + min_kw_only: int = 0 + pseudo_args: int = 0 + vararg: int | str = NO_VARARG + + docstring_prototype: str + docstring_definition: str + impl_prototype: str | None + impl_definition: str + methoddef_define: str + parser_prototype: str + parser_definition: str + cpp_if: str + cpp_endif: str + methoddef_ifndef: str + + parser_body_fields: tuple[str, ...] + + def __init__(self, func: Function, codegen: CodeGen) -> None: + self.func = func + self.codegen = codegen + + self.parameters = list(self.func.parameters.values()) + first_param = self.parameters.pop(0) + if not isinstance(first_param.converter, self_converter): + raise ValueError("the first parameter must use self_converter") + + self.requires_defining_class = False + if self.parameters and isinstance(self.parameters[0].converter, defining_class_converter): + self.requires_defining_class = True + del self.parameters[0] + self.converters = [p.converter for p in self.parameters] + + if self.func.critical_section: + self.codegen.add_include('pycore_critical_section.h', + 'Py_BEGIN_CRITICAL_SECTION()') + self.fastcall = not self.is_new_or_init() + + self.pos_only = 0 + self.min_pos = 0 + self.max_pos = 0 + self.min_kw_only = 0 + self.pseudo_args = 0 + for i, p in enumerate(self.parameters, 1): + if p.is_keyword_only(): + assert not p.is_positional_only() + if not p.is_optional(): + self.min_kw_only = i - self.max_pos - int(self.vararg != NO_VARARG) + elif p.is_vararg(): + self.pseudo_args += 1 + self.vararg = i - 1 + else: + if self.vararg == NO_VARARG: + self.max_pos = i + if p.is_positional_only(): + self.pos_only = i + if not p.is_optional(): + self.min_pos = i + + def is_new_or_init(self) -> bool: + return self.func.kind.new_or_init + + def has_option_groups(self) -> bool: + return (bool(self.parameters + and (self.parameters[0].group or self.parameters[-1].group))) + + def use_meth_o(self) -> bool: + return (len(self.parameters) == 1 + and self.parameters[0].is_positional_only() + and not self.converters[0].is_optional() + and not self.requires_defining_class + and not self.is_new_or_init()) + + def use_simple_return(self) -> bool: + return (self.func.return_converter.type == 'PyObject *' + and not self.func.critical_section) + + def select_prototypes(self) -> None: + self.docstring_prototype = '' + self.docstring_definition = '' + self.methoddef_define = METHODDEF_PROTOTYPE_DEFINE + self.return_value_declaration = "PyObject *return_value = NULL;" + + if self.is_new_or_init() and not self.func.docstring: + pass + elif self.func.kind is GETTER: + self.methoddef_define = GETTERDEF_PROTOTYPE_DEFINE + if self.func.docstring: + self.docstring_definition = GETSET_DOCSTRING_PROTOTYPE_STRVAR + elif self.func.kind is SETTER: + if self.func.docstring: + fail("docstrings are only supported for @getter, not @setter") + self.return_value_declaration = "int {return_value};" + self.methoddef_define = SETTERDEF_PROTOTYPE_DEFINE + else: + self.docstring_prototype = DOCSTRING_PROTOTYPE_VAR + self.docstring_definition = DOCSTRING_PROTOTYPE_STRVAR + + def init_limited_capi(self) -> None: + self.limited_capi = self.codegen.limited_capi + if self.limited_capi and (self.pseudo_args or + (any(p.is_optional() for p in self.parameters) and + any(p.is_keyword_only() and not p.is_optional() for p in self.parameters)) or + any(c.broken_limited_capi for c in self.converters)): + warn(f"Function {self.func.full_name} cannot use limited C API") + self.limited_capi = False + + def parser_body( + self, + *fields: str, + declarations: str = '' + ) -> None: + lines = [self.parser_prototype] + self.parser_body_fields = fields + + preamble = libclinic.normalize_snippet(""" + {{ + {return_value_declaration} + {parser_declarations} + {declarations} + {initializers} + """) + "\n" + finale = libclinic.normalize_snippet(""" + {modifications} + {lock} + {return_value} = {c_basename}_impl({impl_arguments}); + {unlock} + {return_conversion} + {post_parsing} + + {exit_label} + {cleanup} + return return_value; + }} + """) + for field in preamble, *fields, finale: + lines.append(field) + code = libclinic.linear_format("\n".join(lines), + parser_declarations=self.declarations) + self.parser_definition = code + + def parse_no_args(self) -> None: + parser_code: list[str] | None + simple_return = self.use_simple_return() + if self.func.kind is GETTER: + self.parser_prototype = PARSER_PROTOTYPE_GETTER + parser_code = [] + elif self.func.kind is SETTER: + self.parser_prototype = PARSER_PROTOTYPE_SETTER + parser_code = [] + elif not self.requires_defining_class: + # no self.parameters, METH_NOARGS + self.flags = "METH_NOARGS" + self.parser_prototype = PARSER_PROTOTYPE_NOARGS + parser_code = [] + else: + assert self.fastcall + + self.flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_DEF_CLASS + return_error = ('return NULL;' if simple_return + else 'goto exit;') + parser_code = [libclinic.normalize_snippet(""" + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) {{ + PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); + %s + }} + """ % return_error, indent=4)] + + if simple_return: + self.parser_definition = '\n'.join([ + self.parser_prototype, + '{{', + *parser_code, + ' return {c_basename}_impl({impl_arguments});', + '}}']) + else: + self.parser_body(*parser_code) + + def parse_one_arg(self) -> None: + self.flags = "METH_O" + + if (isinstance(self.converters[0], object_converter) and + self.converters[0].format_unit == 'O'): + meth_o_prototype = METH_O_PROTOTYPE + + if self.use_simple_return(): + # maps perfectly to METH_O, doesn't need a return converter. + # so we skip making a parse function + # and call directly into the impl function. + self.impl_prototype = '' + self.impl_definition = meth_o_prototype + else: + # SLIGHT HACK + # use impl_parameters for the parser here! + self.parser_prototype = meth_o_prototype + self.parser_body() + + else: + argname = 'arg' + if self.parameters[0].name == argname: + argname += '_' + self.parser_prototype = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *%s) + """ % argname) + + displayname = self.parameters[0].get_displayname(0) + parsearg: str | None + parsearg = self.converters[0].parse_arg(argname, displayname, + limited_capi=self.limited_capi) + if parsearg is None: + self.converters[0].use_converter() + parsearg = """ + if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ + goto exit; + }} + """ % argname + + parser_code = libclinic.normalize_snippet(parsearg, indent=4) + self.parser_body(parser_code) + + def parse_option_groups(self) -> None: + # positional parameters with option groups + # (we have to generate lots of PyArg_ParseTuple calls + # in a big switch statement) + + self.flags = "METH_VARARGS" + self.parser_prototype = PARSER_PROTOTYPE_VARARGS + parser_code = ' {option_group_parsing}' + self.parser_body(parser_code) + + def parse_pos_only(self) -> None: + if self.fastcall: + # positional-only, but no option groups + # we only need one call to _PyArg_ParseStack + + self.flags = "METH_FASTCALL" + self.parser_prototype = PARSER_PROTOTYPE_FASTCALL + nargs = 'nargs' + argname_fmt = 'args[%d]' + else: + # positional-only, but no option groups + # we only need one call to PyArg_ParseTuple + + self.flags = "METH_VARARGS" + self.parser_prototype = PARSER_PROTOTYPE_VARARGS + if self.limited_capi: + nargs = 'PyTuple_Size(args)' + argname_fmt = 'PyTuple_GetItem(args, %d)' + else: + nargs = 'PyTuple_GET_SIZE(args)' + argname_fmt = 'PyTuple_GET_ITEM(args, %d)' + + if self.vararg != NO_VARARG: + self.declarations = f"Py_ssize_t nvararg = {nargs} - {self.max_pos};" + else: + self.declarations = "" + + max_args = NO_VARARG if (self.vararg != NO_VARARG) else self.max_pos + if self.limited_capi: + parser_code = [] + if nargs != 'nargs': + nargs_def = f'Py_ssize_t nargs = {nargs};' + parser_code.append(libclinic.normalize_snippet(nargs_def, indent=4)) + nargs = 'nargs' + if self.min_pos == max_args: + pl = '' if self.min_pos == 1 else 's' + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} != {self.min_pos}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected {self.min_pos} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + else: + if self.min_pos: + pl = '' if self.min_pos == 1 else 's' + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} < {self.min_pos}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected at least {self.min_pos} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + if max_args != NO_VARARG: + pl = '' if max_args == 1 else 's' + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} > {max_args}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected at most {max_args} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + else: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_CheckPositional()') + parser_code = [libclinic.normalize_snippet(f""" + if (!_PyArg_CheckPositional("{{name}}", {nargs}, {self.min_pos}, {max_args})) {{{{ + goto exit; + }}}} + """, indent=4)] + + has_optional = False + use_parser_code = True + for i, p in enumerate(self.parameters): + if p.is_vararg(): + var = p.converter.parser_name + if self.fastcall: + code = f"{var} = args + {self.vararg};" + else: + code = f"{var} = _PyTuple_CAST(args)->ob_item;" + formatted_code = libclinic.normalize_snippet(code, indent=4) + parser_code.append(formatted_code) + continue + + displayname = p.get_displayname(i+1) + argname = argname_fmt % i + parsearg: str | None + parsearg = p.converter.parse_arg(argname, displayname, limited_capi=self.limited_capi) + if parsearg is None: + use_parser_code = False + parser_code = [] + break + if has_optional or p.is_optional(): + has_optional = True + parser_code.append(libclinic.normalize_snippet(""" + if (%s < %d) {{ + goto skip_optional; + }} + """, indent=4) % (nargs, i + 1)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + + if use_parser_code: + if has_optional: + parser_code.append("skip_optional:") + else: + for parameter in self.parameters: + parameter.converter.use_converter() + + if self.limited_capi: + self.fastcall = False + if self.fastcall: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_ParseStack()') + parser_code = [libclinic.normalize_snippet(""" + if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + else: + self.flags = "METH_VARARGS" + self.parser_prototype = PARSER_PROTOTYPE_VARARGS + parser_code = [libclinic.normalize_snippet(""" + if (!PyArg_ParseTuple(args, "{format_units}:{name}", + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + self.parser_body(*parser_code, declarations=self.declarations) + + def parse_general(self, clang: CLanguage) -> None: + parsearg: str | None + deprecated_positionals: dict[int, Parameter] = {} + deprecated_keywords: dict[int, Parameter] = {} + for i, p in enumerate(self.parameters): + if p.deprecated_positional: + deprecated_positionals[i] = p + if p.deprecated_keyword: + deprecated_keywords[i] = p + + has_optional_kw = ( + max(self.pos_only, self.min_pos) + self.min_kw_only + < len(self.converters) - int(self.vararg != NO_VARARG) + ) + + use_parser_code = True + if self.limited_capi: + parser_code = [] + use_parser_code = False + self.fastcall = False + else: + if self.vararg == NO_VARARG: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywords()') + args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( + self.min_pos, + self.max_pos, + self.min_kw_only + ) + nargs = "nargs" + else: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywordsWithVararg()') + args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( + self.min_pos, + self.max_pos, + self.min_kw_only, + self.vararg + ) + nargs = f"Py_MIN(nargs, {self.max_pos})" if self.max_pos else "0" + + if self.fastcall: + self.flags = "METH_FASTCALL|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_FASTCALL_KEYWORDS + argname_fmt = 'args[%d]' + self.declarations = declare_parser(self.func, codegen=self.codegen) + self.declarations += "\nPyObject *argsbuf[%s];" % len(self.converters) + if has_optional_kw: + self.declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, self.min_pos + self.min_kw_only) + parser_code = [libclinic.normalize_snippet(""" + args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); + if (!args) {{ + goto exit; + }} + """ % args_declaration, indent=4)] + else: + # positional-or-keyword arguments + self.flags = "METH_VARARGS|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD + argname_fmt = 'fastargs[%d]' + self.declarations = declare_parser(self.func, codegen=self.codegen) + self.declarations += "\nPyObject *argsbuf[%s];" % len(self.converters) + self.declarations += "\nPyObject * const *fastargs;" + self.declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if has_optional_kw: + self.declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, self.min_pos + self.min_kw_only) + parser_code = [libclinic.normalize_snippet(""" + fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); + if (!fastargs) {{ + goto exit; + }} + """ % args_declaration, indent=4)] + + if self.requires_defining_class: + self.flags = 'METH_METHOD|' + self.flags + self.parser_prototype = PARSER_PROTOTYPE_DEF_CLASS + + if use_parser_code: + if deprecated_keywords: + code = clang.deprecate_keyword_use(self.func, deprecated_keywords, + argname_fmt, + codegen=self.codegen, + fastcall=self.fastcall) + parser_code.append(code) + + add_label: str | None = None + for i, p in enumerate(self.parameters): + if isinstance(p.converter, defining_class_converter): + raise ValueError("defining_class should be the first " + "parameter (after clang)") + displayname = p.get_displayname(i+1) + parsearg = p.converter.parse_arg(argname_fmt % i, displayname, limited_capi=self.limited_capi) + if parsearg is None: + parser_code = [] + use_parser_code = False + break + if add_label and (i == self.pos_only or i == self.max_pos): + parser_code.append("%s:" % add_label) + add_label = None + if not p.is_optional(): + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + elif i < self.pos_only: + add_label = 'skip_optional_posonly' + parser_code.append(libclinic.normalize_snippet(""" + if (nargs < %d) {{ + goto %s; + }} + """ % (i + 1, add_label), indent=4)) + if has_optional_kw: + parser_code.append(libclinic.normalize_snippet(""" + noptargs--; + """, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + else: + if i < self.max_pos: + label = 'skip_optional_pos' + first_opt = max(self.min_pos, self.pos_only) + else: + label = 'skip_optional_kwonly' + first_opt = self.max_pos + self.min_kw_only + if self.vararg != NO_VARARG: + first_opt += 1 + if i == first_opt: + add_label = label + parser_code.append(libclinic.normalize_snippet(""" + if (!noptargs) {{ + goto %s; + }} + """ % add_label, indent=4)) + if i + 1 == len(self.parameters): + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + else: + add_label = label + parser_code.append(libclinic.normalize_snippet(""" + if (%s) {{ + """ % (argname_fmt % i), indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=8)) + parser_code.append(libclinic.normalize_snippet(""" + if (!--noptargs) {{ + goto %s; + }} + }} + """ % add_label, indent=4)) + + if use_parser_code: + if add_label: + parser_code.append("%s:" % add_label) + else: + for parameter in self.parameters: + parameter.converter.use_converter() + + self.declarations = declare_parser(self.func, codegen=self.codegen, + hasformat=True) + if self.limited_capi: + # positional-or-keyword arguments + assert not self.fastcall + self.flags = "METH_VARARGS|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD + parser_code = [libclinic.normalize_snippet(""" + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, + {parse_arguments})) + goto exit; + """, indent=4)] + self.declarations = "static char *_keywords[] = {{{keywords_c} NULL}};" + if deprecated_positionals or deprecated_keywords: + self.declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" + + elif self.fastcall: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_ParseStackAndKeywords()') + parser_code = [libclinic.normalize_snippet(""" + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + else: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_ParseTupleAndKeywordsFast()') + parser_code = [libclinic.normalize_snippet(""" + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + if deprecated_positionals or deprecated_keywords: + self.declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if deprecated_keywords: + code = clang.deprecate_keyword_use(self.func, deprecated_keywords, + codegen=self.codegen, + fastcall=self.fastcall) + parser_code.append(code) + + if deprecated_positionals: + code = clang.deprecate_positional_use(self.func, deprecated_positionals) + # Insert the deprecation code before parameter parsing. + parser_code.insert(0, code) + + assert self.parser_prototype is not None + self.parser_body(*parser_code, declarations=self.declarations) + + def copy_includes(self) -> None: + # Copy includes from parameters to Clinic after parse_arg() + # has been called above. + for converter in self.converters: + for include in converter.get_includes(): + self.codegen.add_include( + include.filename, + include.reason, + condition=include.condition) + + def handle_new_or_init(self) -> None: + self.methoddef_define = '' + + if self.func.kind is METHOD_NEW: + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD + else: + self.return_value_declaration = "int return_value = -1;" + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD___INIT__ + + fields: list[str] = list(self.parser_body_fields) + parses_positional = 'METH_NOARGS' not in self.flags + parses_keywords = 'METH_KEYWORDS' in self.flags + if parses_keywords: + assert parses_positional + + if self.requires_defining_class: + raise ValueError("Slot methods cannot access their defining class.") + + if not parses_keywords: + self.declarations = '{base_type_ptr}' + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_NoKeywords()') + fields.insert(0, libclinic.normalize_snippet(""" + if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ + goto exit; + }} + """, indent=4)) + if not parses_positional: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_NoPositional()') + fields.insert(0, libclinic.normalize_snippet(""" + if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ + goto exit; + }} + """, indent=4)) + + self.parser_body(*fields, declarations=self.declarations) + + def process_methoddef(self, clang: CLanguage) -> None: + methoddef_cast_end = "" + if self.flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): + methoddef_cast = "(PyCFunction)" + elif self.func.kind is GETTER: + methoddef_cast = "" # This should end up unused + elif self.limited_capi: + methoddef_cast = "(PyCFunction)(void(*)(void))" + else: + methoddef_cast = "_PyCFunction_CAST(" + methoddef_cast_end = ")" + + if self.func.methoddef_flags: + self.flags += '|' + self.func.methoddef_flags + + self.methoddef_define = self.methoddef_define.replace('{methoddef_flags}', self.flags) + self.methoddef_define = self.methoddef_define.replace('{methoddef_cast}', methoddef_cast) + self.methoddef_define = self.methoddef_define.replace('{methoddef_cast_end}', methoddef_cast_end) + + self.methoddef_ifndef = '' + conditional = clang.cpp.condition() + if not conditional: + self.cpp_if = self.cpp_endif = '' + else: + self.cpp_if = "#if " + conditional + self.cpp_endif = "#endif /* " + conditional + " */" + + if self.methoddef_define and self.codegen.add_ifndef_symbol(self.func.full_name): + self.methoddef_ifndef = METHODDEF_PROTOTYPE_IFNDEF + + def finalize(self, clang: CLanguage) -> None: + # add ';' to the end of self.parser_prototype and self.impl_prototype + # (they mustn't be None, but they could be an empty string.) + assert self.parser_prototype is not None + if self.parser_prototype: + assert not self.parser_prototype.endswith(';') + self.parser_prototype += ';' + + if self.impl_prototype is None: + self.impl_prototype = self.impl_definition + if self.impl_prototype: + self.impl_prototype += ";" + + self.parser_definition = self.parser_definition.replace("{return_value_declaration}", self.return_value_declaration) + + compiler_warning = clang.compiler_deprecated_warning(self.func, self.parameters) + if compiler_warning: + self.parser_definition = compiler_warning + "\n\n" + self.parser_definition + + def create_template_dict(self) -> dict[str, str]: + d = { + "docstring_prototype" : self.docstring_prototype, + "docstring_definition" : self.docstring_definition, + "impl_prototype" : self.impl_prototype, + "methoddef_define" : self.methoddef_define, + "parser_prototype" : self.parser_prototype, + "parser_definition" : self.parser_definition, + "impl_definition" : self.impl_definition, + "cpp_if" : self.cpp_if, + "cpp_endif" : self.cpp_endif, + "methoddef_ifndef" : self.methoddef_ifndef, + } + + # make sure we didn't forget to assign something, + # and wrap each non-empty value in \n's + d2 = {} + for name, value in d.items(): + assert value is not None, "got a None value for template " + repr(name) + if value: + value = '\n' + value + '\n' + d2[name] = value + return d2 + + def parse_args(self, clang: CLanguage) -> dict[str, str]: + self.select_prototypes() + self.init_limited_capi() + + self.flags = "" + self.declarations = "" + self.parser_prototype = "" + self.parser_definition = "" + self.impl_prototype = None + self.impl_definition = IMPL_DEFINITION_PROTOTYPE + + # parser_body_fields remembers the fields passed in to the + # previous call to parser_body. this is used for an awful hack. + self.parser_body_fields: tuple[str, ...] = () + + if not self.parameters: + self.parse_no_args() + elif self.use_meth_o(): + self.parse_one_arg() + elif self.has_option_groups(): + self.parse_option_groups() + elif (not self.requires_defining_class + and self.pos_only == len(self.parameters) - self.pseudo_args): + self.parse_pos_only() + else: + self.parse_general(clang) + + self.copy_includes() + if self.is_new_or_init(): + self.handle_new_or_init() + self.process_methoddef(clang) + self.finalize(clang) + + return self.create_template_dict() diff --git a/Tools/clinic/libclinic/parser.py b/Tools/clinic/libclinic/parser.py new file mode 100644 index 00000000000000..8135dd9ceb3937 --- /dev/null +++ b/Tools/clinic/libclinic/parser.py @@ -0,0 +1,53 @@ +from __future__ import annotations +import contextlib +import functools +import io +from types import NoneType +from typing import Any, Protocol, TYPE_CHECKING + +from libclinic import unspecified +from libclinic.block_parser import Block +from libclinic.converter import CConverter, converters +from libclinic.converters import buffer, robuffer, rwbuffer +from libclinic.return_converters import CReturnConverter, return_converters +if TYPE_CHECKING: + from libclinic.app import Clinic + + +class Parser(Protocol): + def __init__(self, clinic: Clinic) -> None: ... + def parse(self, block: Block) -> None: ... + + +@functools.cache +def _create_parser_base_namespace() -> dict[str, Any]: + ns = dict( + CConverter=CConverter, + CReturnConverter=CReturnConverter, + buffer=buffer, + robuffer=robuffer, + rwbuffer=rwbuffer, + unspecified=unspecified, + NoneType=NoneType, + ) + for name, converter in converters.items(): + ns[f'{name}_converter'] = converter + for name, return_converter in return_converters.items(): + ns[f'{name}_return_converter'] = return_converter + return ns + + +def create_parser_namespace() -> dict[str, Any]: + base_namespace = _create_parser_base_namespace() + return base_namespace.copy() + + +class PythonParser: + def __init__(self, clinic: Clinic) -> None: + pass + + def parse(self, block: Block) -> None: + namespace = create_parser_namespace() + with contextlib.redirect_stdout(io.StringIO()) as s: + exec(block.input, namespace) + block.output = s.getvalue() diff --git a/Tools/clinic/libclinic/return_converters.py b/Tools/clinic/libclinic/return_converters.py new file mode 100644 index 00000000000000..b41e053bae5f3a --- /dev/null +++ b/Tools/clinic/libclinic/return_converters.py @@ -0,0 +1,173 @@ +import sys +from collections.abc import Callable +from libclinic.codegen import CRenderData +from libclinic.function import Function +from typing import Any + + +ReturnConverterType = Callable[..., "CReturnConverter"] + + +# maps strings to callables. +# these callables must be of the form: +# def foo(*, ...) +# The callable may have any number of keyword-only parameters. +# The callable must return a CReturnConverter object. +# The callable should not call builtins.print. +ReturnConverterDict = dict[str, ReturnConverterType] +return_converters: ReturnConverterDict = {} + + +def add_c_return_converter( + f: ReturnConverterType, + name: str | None = None +) -> ReturnConverterType: + if not name: + name = f.__name__ + if not name.endswith('_return_converter'): + return f + name = name.removesuffix('_return_converter') + return_converters[name] = f + return f + + +class CReturnConverterAutoRegister(type): + def __init__( + cls: ReturnConverterType, + name: str, + bases: tuple[type[object], ...], + classdict: dict[str, Any] + ) -> None: + add_c_return_converter(cls) + + +class CReturnConverter(metaclass=CReturnConverterAutoRegister): + + # The C type to use for this variable. + # 'type' should be a Python string specifying the type, e.g. "int". + # If this is a pointer type, the type string should end with ' *'. + type = 'PyObject *' + + # The Python default value for this parameter, as a Python value. + # Or the magic value "unspecified" if there is no default. + default: object = None + + def __init__( + self, + *, + py_default: str | None = None, + **kwargs: Any + ) -> None: + self.py_default = py_default + try: + self.return_converter_init(**kwargs) + except TypeError as e: + s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) + sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) + + def return_converter_init(self) -> None: ... + + def declare(self, data: CRenderData) -> None: + line: list[str] = [] + add = line.append + add(self.type) + if not self.type.endswith('*'): + add(' ') + add(data.converter_retval + ';') + data.declarations.append(''.join(line)) + data.return_value = data.converter_retval + + def err_occurred_if( + self, + expr: str, + data: CRenderData + ) -> None: + line = f'if (({expr}) && PyErr_Occurred()) {{\n goto exit;\n}}\n' + data.return_conversion.append(line) + + def err_occurred_if_null_pointer( + self, + variable: str, + data: CRenderData + ) -> None: + line = f'if ({variable} == NULL) {{\n goto exit;\n}}\n' + data.return_conversion.append(line) + + def render( + self, + function: Function, + data: CRenderData + ) -> None: ... + + +add_c_return_converter(CReturnConverter, 'object') + + +class bool_return_converter(CReturnConverter): + type = 'int' + + def render(self, function: Function, data: CRenderData) -> None: + self.declare(data) + self.err_occurred_if(f"{data.converter_retval} == -1", data) + data.return_conversion.append( + f'return_value = PyBool_FromLong((long){data.converter_retval});\n' + ) + + +class long_return_converter(CReturnConverter): + type = 'long' + conversion_fn = 'PyLong_FromLong' + cast = '' + unsigned_cast = '' + + def render(self, function: Function, data: CRenderData) -> None: + self.declare(data) + self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) + data.return_conversion.append( + f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' + ) + + +class int_return_converter(long_return_converter): + type = 'int' + cast = '(long)' + + +class unsigned_long_return_converter(long_return_converter): + type = 'unsigned long' + conversion_fn = 'PyLong_FromUnsignedLong' + unsigned_cast = '(unsigned long)' + + +class unsigned_int_return_converter(unsigned_long_return_converter): + type = 'unsigned int' + cast = '(unsigned long)' + unsigned_cast = '(unsigned int)' + + +class Py_ssize_t_return_converter(long_return_converter): + type = 'Py_ssize_t' + conversion_fn = 'PyLong_FromSsize_t' + + +class size_t_return_converter(long_return_converter): + type = 'size_t' + conversion_fn = 'PyLong_FromSize_t' + unsigned_cast = '(size_t)' + + +class double_return_converter(CReturnConverter): + type = 'double' + cast = '' + + def render(self, function: Function, data: CRenderData) -> None: + self.declare(data) + self.err_occurred_if(f"{data.converter_retval} == -1.0", data) + data.return_conversion.append( + f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' + ) + + +class float_return_converter(double_return_converter): + type = 'float' + cast = '(double)' diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py index d2d09387a73d1e..17e8f35be73bf4 100644 --- a/Tools/clinic/libclinic/utils.py +++ b/Tools/clinic/libclinic/utils.py @@ -1,9 +1,10 @@ import collections +import enum import hashlib import os import re import string -from typing import Literal +from typing import Literal, Final def write_file(filename: str, new_contents: str) -> None: @@ -66,3 +67,27 @@ def get_value( ) -> Literal[""]: self.counts[key] += 1 return "" + + +VersionTuple = tuple[int, int] + + +class Sentinels(enum.Enum): + unspecified = "unspecified" + unknown = "unknown" + + def __repr__(self) -> str: + return f"<{self.value.capitalize()}>" + + +unspecified: Final = Sentinels.unspecified +unknown: Final = Sentinels.unknown + + +# This one needs to be a distinct class, unlike the other two +class Null: + def __repr__(self) -> str: + return '' + + +NULL = Null() diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py new file mode 100644 index 00000000000000..767aeae9349070 --- /dev/null +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -0,0 +1,324 @@ +# This script runs a set of small benchmarks to help identify scaling +# bottlenecks in the free-threaded interpreter. The benchmarks consist +# of patterns that ought to scale well, but haven't in the past. This is +# typically due to reference count contention or lock contention. +# +# This is not intended to be a general multithreading benchmark suite, nor +# are the benchmarks intended to be representative of real-world workloads. +# +# On Linux, to avoid confounding hardware effects, the script attempts to: +# * Use a single CPU socket (to avoid NUMA effects) +# * Use distinct physical cores (to avoid hyperthreading/SMT effects) +# * Use "performance" cores (Intel, ARM) on CPUs that have performance and +# efficiency cores +# +# It also helps to disable dynamic frequency scaling (i.e., "Turbo Boost") +# +# Intel: +# > echo "1" | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo +# +# AMD: +# > echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost +# + +import math +import os +import queue +import sys +import threading +import time + +# The iterations in individual benchmarks are scaled by this factor. +WORK_SCALE = 100 + +ALL_BENCHMARKS = {} + +threads = [] +in_queues = [] +out_queues = [] + + +def register_benchmark(func): + ALL_BENCHMARKS[func.__name__] = func + return func + +@register_benchmark +def object_cfunction(): + accu = 0 + tab = [1] * 100 + for i in range(1000 * WORK_SCALE): + tab.pop(0) + tab.append(i) + accu += tab[50] + return accu + +@register_benchmark +def cmodule_function(): + for i in range(1000 * WORK_SCALE): + math.floor(i * i) + +@register_benchmark +def mult_constant(): + x = 1.0 + for i in range(3000 * WORK_SCALE): + x *= 1.01 + +def simple_gen(): + for i in range(10): + yield i + +@register_benchmark +def generator(): + accu = 0 + for i in range(100 * WORK_SCALE): + for v in simple_gen(): + accu += v + return accu + +class Counter: + def __init__(self): + self.i = 0 + + def next_number(self): + self.i += 1 + return self.i + +@register_benchmark +def pymethod(): + c = Counter() + for i in range(1000 * WORK_SCALE): + c.next_number() + return c.i + +def next_number(i): + return i + 1 + +@register_benchmark +def pyfunction(): + accu = 0 + for i in range(1000 * WORK_SCALE): + accu = next_number(i) + return accu + +def double(x): + return x + x + +module = sys.modules[__name__] + +@register_benchmark +def module_function(): + total = 0 + for i in range(1000 * WORK_SCALE): + total += module.double(i) + return total + +class MyObject: + pass + +@register_benchmark +def load_string_const(): + accu = 0 + for i in range(1000 * WORK_SCALE): + if i == 'a string': + accu += 7 + else: + accu += 1 + return accu + +@register_benchmark +def load_tuple_const(): + accu = 0 + for i in range(1000 * WORK_SCALE): + if i == (1, 2): + accu += 7 + else: + accu += 1 + return accu + +@register_benchmark +def create_pyobject(): + for i in range(1000 * WORK_SCALE): + o = MyObject() + +@register_benchmark +def create_closure(): + for i in range(1000 * WORK_SCALE): + def foo(x): + return x + foo(i) + +@register_benchmark +def create_dict(): + for i in range(1000 * WORK_SCALE): + d = { + "key": "value", + } + +thread_local = threading.local() + +@register_benchmark +def thread_local_read(): + tmp = thread_local + tmp.x = 10 + for i in range(500 * WORK_SCALE): + _ = tmp.x + _ = tmp.x + _ = tmp.x + _ = tmp.x + _ = tmp.x + + +def bench_one_thread(func): + t0 = time.perf_counter_ns() + func() + t1 = time.perf_counter_ns() + return t1 - t0 + + +def bench_parallel(func): + t0 = time.perf_counter_ns() + for inq in in_queues: + inq.put(func) + for outq in out_queues: + outq.get() + t1 = time.perf_counter_ns() + return t1 - t0 + + +def benchmark(func): + delta_one_thread = bench_one_thread(func) + delta_many_threads = bench_parallel(func) + + speedup = delta_one_thread * len(threads) / delta_many_threads + if speedup >= 1: + factor = speedup + direction = "faster" + else: + factor = 1 / speedup + direction = "slower" + + use_color = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + color = reset_color = "" + if use_color: + if speedup <= 1.1: + color = "\x1b[31m" # red + elif speedup < len(threads)/2: + color = "\x1b[33m" # yellow + reset_color = "\x1b[0m" + + print(f"{color}{func.__name__:<18} {round(factor, 1):>4}x {direction}{reset_color}") + +def determine_num_threads_and_affinity(): + if sys.platform != "linux": + return [None] * os.cpu_count() + + # Try to use `lscpu -p` on Linux + import subprocess + try: + output = subprocess.check_output(["lscpu", "-p=cpu,node,core,MAXMHZ"], + text=True, env={"LC_NUMERIC": "C"}) + except (FileNotFoundError, subprocess.CalledProcessError): + return [None] * os.cpu_count() + + table = [] + for line in output.splitlines(): + if line.startswith("#"): + continue + cpu, node, core, maxhz = line.split(",") + if maxhz == "": + maxhz = "0" + table.append((int(cpu), int(node), int(core), float(maxhz))) + + cpus = [] + cores = set() + max_mhz_all = max(row[3] for row in table) + for cpu, node, core, maxmhz in table: + # Choose only CPUs on the same node, unique cores, and try to avoid + # "efficiency" cores. + if node == 0 and core not in cores and maxmhz == max_mhz_all: + cpus.append(cpu) + cores.add(core) + return cpus + + +def thread_run(cpu, in_queue, out_queue): + if cpu is not None and hasattr(os, "sched_setaffinity"): + # Set the affinity for the current thread + os.sched_setaffinity(0, (cpu,)) + + while True: + func = in_queue.get() + if func is None: + break + func() + out_queue.put(None) + + +def initialize_threads(opts): + if opts.threads == -1: + cpus = determine_num_threads_and_affinity() + else: + cpus = [None] * opts.threads # don't set affinity + + print(f"Running benchmarks with {len(cpus)} threads") + for cpu in cpus: + inq = queue.Queue() + outq = queue.Queue() + in_queues.append(inq) + out_queues.append(outq) + t = threading.Thread(target=thread_run, args=(cpu, inq, outq), daemon=True) + threads.append(t) + t.start() + + +def main(opts): + global WORK_SCALE + if not hasattr(sys, "_is_gil_enabled") or sys._is_gil_enabled(): + sys.stderr.write("expected to be run with the GIL disabled\n") + + benchmark_names = opts.benchmarks + if benchmark_names: + for name in benchmark_names: + if name not in ALL_BENCHMARKS: + sys.stderr.write(f"Unknown benchmark: {name}\n") + sys.exit(1) + else: + benchmark_names = ALL_BENCHMARKS.keys() + + WORK_SCALE = opts.scale + + if not opts.baseline_only: + initialize_threads(opts) + + do_bench = not opts.baseline_only and not opts.parallel_only + for name in benchmark_names: + func = ALL_BENCHMARKS[name] + if do_bench: + benchmark(func) + continue + + if opts.parallel_only: + delta_ns = bench_parallel(func) + else: + delta_ns = bench_one_thread(func) + + time_ms = delta_ns / 1_000_000 + print(f"{func.__name__:<18} {time_ms:.1f} ms") + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("-t", "--threads", type=int, default=-1, + help="number of threads to use") + parser.add_argument("--scale", type=int, default=100, + help="work scale factor for the benchmark (default=100)") + parser.add_argument("--baseline-only", default=False, action="store_true", + help="only run the baseline benchmarks (single thread)") + parser.add_argument("--parallel-only", default=False, action="store_true", + help="only run the parallel benchmark (many threads)") + parser.add_argument("benchmarks", nargs="*", + help="benchmarks to run") + options = parser.parse_args() + main(options) diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 483f28b46dfec7..946af4be1a7589 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -66,7 +66,6 @@ def _type_unsigned_short_ptr(): def _type_unsigned_int_ptr(): return gdb.lookup_type('unsigned int').pointer() - def _sizeof_void_p(): return gdb.lookup_type('void').pointer().sizeof @@ -79,6 +78,7 @@ def _managed_dict_offset(): return -3 * _sizeof_void_p() +Py_TPFLAGS_INLINE_VALUES = (1 << 2) Py_TPFLAGS_MANAGED_DICT = (1 << 4) Py_TPFLAGS_HEAPTYPE = (1 << 9) Py_TPFLAGS_LONG_SUBCLASS = (1 << 24) @@ -97,6 +97,8 @@ def _managed_dict_offset(): hexdigits = "0123456789abcdef" +USED_TAGS = 0b11 + ENCODING = locale.getpreferredencoding() FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)' @@ -155,7 +157,12 @@ class PyObjectPtr(object): _typename = 'PyObject' def __init__(self, gdbval, cast_to=None): - if cast_to: + # Clear the tagged pointer + if gdbval.type.name == '_PyStackRef': + if cast_to is None: + cast_to = gdb.lookup_type('PyObject').pointer() + self._gdbval = gdb.Value(int(gdbval['bits']) & ~USED_TAGS).cast(cast_to) + elif cast_to: self._gdbval = gdbval.cast(cast_to) else: self._gdbval = gdbval @@ -252,7 +259,7 @@ def proxyval(self, visited): Derived classes will override this. - For example, a PyIntObject* with ob_ival 42 in the inferior process + For example, a PyLongObjectPtr* with long_value 42 in the inferior process should result in an int(42) in this process. visited: a set of all gdb.Value pyobject pointers already visited @@ -493,11 +500,12 @@ def get_keys_values(self): has_values = int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT if not has_values: return None - ptr = self._gdbval.cast(_type_char_ptr()) + _managed_dict_offset() - char_ptr = ptr.cast(_type_char_ptr().pointer()).dereference() - if (int(char_ptr) & 1) == 0: + obj_ptr = self._gdbval.cast(_type_char_ptr()) + dict_ptr_ptr = obj_ptr + _managed_dict_offset() + dict_ptr = dict_ptr_ptr.cast(_type_char_ptr().pointer()).dereference() + if int(dict_ptr): return None - char_ptr += 1 + char_ptr = obj_ptr + typeobj.field('tp_basicsize') values_ptr = char_ptr.cast(gdb.lookup_type("PyDictValues").pointer()) values = values_ptr['values'] return PyKeysValuesPair(self.get_cached_keys(), values) @@ -863,7 +871,7 @@ class PyLongObjectPtr(PyObjectPtr): def proxyval(self, visited): ''' - Python's Include/longobjrep.h has this declaration: + Python's Include/longinterpr.h has this declaration: typedef struct _PyLongValue { uintptr_t lv_tag; /* Number of digits, sign and flags */ @@ -872,14 +880,18 @@ def proxyval(self, visited): struct _longobject { PyObject_HEAD - _PyLongValue long_value; + _PyLongValue long_value; }; with this description: The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + The sign of the value is stored in the lower 2 bits of lv_tag. + - 0: Positive + - 1: Zero + - 2: Negative + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. where SHIFT can be either: #define PyLong_SHIFT 30 @@ -1043,7 +1055,7 @@ def iter_locals(self): obj_ptr_ptr = gdb.lookup_type("PyObject").pointer().pointer() - localsplus = self._gdbval["localsplus"].cast(obj_ptr_ptr) + localsplus = self._gdbval["localsplus"] for i in safe_range(self.co_nlocals): pyop_value = PyObjectPtr.from_pyobject_ptr(localsplus[i]) @@ -1572,7 +1584,10 @@ def to_string (self): return stringify(proxyval) def pretty_printer_lookup(gdbval): - type = gdbval.type.unqualified() + type = gdbval.type.strip_typedefs().unqualified() + if type.code == gdb.TYPE_CODE_UNION and type.tag == '_PyStackRef': + return PyObjectPtrPrinter(gdbval) + if type.code != gdb.TYPE_CODE_PTR: return None @@ -1753,8 +1768,11 @@ def is_waiting_for_gil(self): return (name == 'take_gil') def is_gc_collect(self): - '''Is this frame gc_collect_main() within the garbage-collector?''' - return self._gdbframe.name() in ('collect', 'gc_collect_main') + '''Is this frame a collector within the garbage-collector?''' + return self._gdbframe.name() in ( + 'collect', 'gc_collect_full', 'gc_collect_main', + 'gc_collect_young', 'gc_collect_increment', + ) def get_pyop(self): try: diff --git a/Tools/iobench/iobench.py b/Tools/iobench/iobench.py deleted file mode 100644 index 4017149ec91630..00000000000000 --- a/Tools/iobench/iobench.py +++ /dev/null @@ -1,568 +0,0 @@ -import itertools -import os -import platform -import re -import sys -import time -from optparse import OptionParser - -out = sys.stdout - -TEXT_ENCODING = 'utf8' -NEWLINES = 'lf' - - -def text_open(fn, mode, encoding=None): - try: - return open(fn, mode, encoding=encoding or TEXT_ENCODING) - except TypeError: - return open(fn, mode) - - -def get_file_sizes(): - for s in ['20 KiB', '400 KiB', '10 MiB']: - size, unit = s.split() - size = int(size) * {'KiB': 1024, 'MiB': 1024 ** 2}[unit] - yield s.replace(' ', ''), size - - -def get_binary_files(): - return ((name + ".bin", size) for name, size in get_file_sizes()) - - -def get_text_files(): - return ((f"{name}-{TEXT_ENCODING}-{NEWLINES}.txt", size) - for name, size in get_file_sizes()) - - -def with_open_mode(mode): - def decorate(f): - f.file_open_mode = mode - return f - return decorate - - -def with_sizes(*sizes): - def decorate(f): - f.file_sizes = sizes - return f - return decorate - - -# Here begin the tests - -@with_open_mode("r") -@with_sizes("medium") -def read_bytewise(f): - """ read one unit at a time """ - f.seek(0) - while f.read(1): - pass - - -@with_open_mode("r") -@with_sizes("medium") -def read_small_chunks(f): - """ read 20 units at a time """ - f.seek(0) - while f.read(20): - pass - - -@with_open_mode("r") -@with_sizes("medium") -def read_big_chunks(f): - """ read 4096 units at a time """ - f.seek(0) - while f.read(4096): - pass - - -@with_open_mode("r") -@with_sizes("small", "medium", "large") -def read_whole_file(f): - """ read whole contents at once """ - f.seek(0) - while f.read(): - pass - - -@with_open_mode("rt") -@with_sizes("medium") -def read_lines(f): - """ read one line at a time """ - f.seek(0) - for line in f: - pass - - -@with_open_mode("r") -@with_sizes("medium") -def seek_forward_bytewise(f): - """ seek forward one unit at a time """ - f.seek(0, 2) - size = f.tell() - f.seek(0, 0) - for i in range(0, size - 1): - f.seek(i, 0) - - -@with_open_mode("r") -@with_sizes("medium") -def seek_forward_blockwise(f): - """ seek forward 1000 units at a time """ - f.seek(0, 2) - size = f.tell() - f.seek(0, 0) - for i in range(0, size - 1, 1000): - f.seek(i, 0) - - -@with_open_mode("rb") -@with_sizes("medium") -def read_seek_bytewise(f): - """ alternate read & seek one unit """ - f.seek(0) - while f.read(1): - f.seek(1, 1) - - -@with_open_mode("rb") -@with_sizes("medium") -def read_seek_blockwise(f): - """ alternate read & seek 1000 units """ - f.seek(0) - while f.read(1000): - f.seek(1000, 1) - - -@with_open_mode("w") -@with_sizes("small") -def write_bytewise(f, source): - """ write one unit at a time """ - for i in range(0, len(source)): - f.write(source[i:i+1]) - - -@with_open_mode("w") -@with_sizes("medium") -def write_small_chunks(f, source): - """ write 20 units at a time """ - for i in range(0, len(source), 20): - f.write(source[i:i+20]) - - -@with_open_mode("w") -@with_sizes("medium") -def write_medium_chunks(f, source): - """ write 4096 units at a time """ - for i in range(0, len(source), 4096): - f.write(source[i:i+4096]) - - -@with_open_mode("w") -@with_sizes("large") -def write_large_chunks(f, source): - """ write 1e6 units at a time """ - for i in range(0, len(source), 1000000): - f.write(source[i:i+1000000]) - - -@with_open_mode("w+") -@with_sizes("small") -def modify_bytewise(f, source): - """ modify one unit at a time """ - f.seek(0) - for i in range(0, len(source)): - f.write(source[i:i+1]) - - -@with_open_mode("w+") -@with_sizes("medium") -def modify_small_chunks(f, source): - """ modify 20 units at a time """ - f.seek(0) - for i in range(0, len(source), 20): - f.write(source[i:i+20]) - - -@with_open_mode("w+") -@with_sizes("medium") -def modify_medium_chunks(f, source): - """ modify 4096 units at a time """ - f.seek(0) - for i in range(0, len(source), 4096): - f.write(source[i:i+4096]) - - -@with_open_mode("wb+") -@with_sizes("medium") -def modify_seek_forward_bytewise(f, source): - """ alternate write & seek one unit """ - f.seek(0) - for i in range(0, len(source), 2): - f.write(source[i:i+1]) - f.seek(i+2) - - -@with_open_mode("wb+") -@with_sizes("medium") -def modify_seek_forward_blockwise(f, source): - """ alternate write & seek 1000 units """ - f.seek(0) - for i in range(0, len(source), 2000): - f.write(source[i:i+1000]) - f.seek(i+2000) - - -# XXX the 2 following tests don't work with py3k's text IO -@with_open_mode("wb+") -@with_sizes("medium") -def read_modify_bytewise(f, source): - """ alternate read & write one unit """ - f.seek(0) - for i in range(0, len(source), 2): - f.read(1) - f.write(source[i+1:i+2]) - - -@with_open_mode("wb+") -@with_sizes("medium") -def read_modify_blockwise(f, source): - """ alternate read & write 1000 units """ - f.seek(0) - for i in range(0, len(source), 2000): - f.read(1000) - f.write(source[i+1000:i+2000]) - - -read_tests = [ - read_bytewise, read_small_chunks, read_lines, read_big_chunks, - None, read_whole_file, None, - seek_forward_bytewise, seek_forward_blockwise, - read_seek_bytewise, read_seek_blockwise, -] - -write_tests = [ - write_bytewise, write_small_chunks, write_medium_chunks, write_large_chunks, -] - -modify_tests = [ - modify_bytewise, modify_small_chunks, modify_medium_chunks, - None, - modify_seek_forward_bytewise, modify_seek_forward_blockwise, - read_modify_bytewise, read_modify_blockwise, -] - - -def run_during(duration, func): - _t = time.time - n = 0 - start = os.times() - start_timestamp = _t() - real_start = start[4] or start_timestamp - while True: - func() - n += 1 - if _t() - start_timestamp > duration: - break - end = os.times() - real = (end[4] if start[4] else time.time()) - real_start - return n, real, sum(end[0:2]) - sum(start[0:2]) - - -def warm_cache(filename): - with open(filename, "rb") as f: - f.read() - - -def run_all_tests(options): - def print_label(filename, func): - name = re.split(r'[-.]', filename)[0] - out.write( - f"[{name.center(7)}] {func.__doc__.strip()}... ".ljust(52)) - out.flush() - - def print_results(size, n, real, cpu): - bw = n * float(size) / 1024 ** 2 / real - bw = ("%4d MiB/s" if bw > 100 else "%.3g MiB/s") % bw - out.write(bw.rjust(12) + "\n") - if cpu < 0.90 * real: - out.write(" warning: test above used only " - f"{cpu / real:%} CPU, " - "result may be flawed!\n") - - def run_one_test(name, size, open_func, test_func, *args): - mode = test_func.file_open_mode - print_label(name, test_func) - if "w" not in mode or "+" in mode: - warm_cache(name) - with open_func(name) as f: - n, real, cpu = run_during(1.5, lambda: test_func(f, *args)) - print_results(size, n, real, cpu) - - def run_test_family(tests, mode_filter, files, open_func, *make_args): - for test_func in tests: - if test_func is None: - out.write("\n") - continue - if mode_filter in test_func.file_open_mode: - continue - for s in test_func.file_sizes: - name, size = files[size_names[s]] - #name += file_ext - args = tuple(f(name, size) for f in make_args) - run_one_test(name, size, - open_func, test_func, *args) - - size_names = { - "small": 0, - "medium": 1, - "large": 2, - } - - print(f"Python {sys.version}") - print("Unicode: PEP 393") - print(platform.platform()) - binary_files = list(get_binary_files()) - text_files = list(get_text_files()) - if "b" in options: - print("Binary unit = one byte") - if "t" in options: - print(f"Text unit = one character ({TEXT_ENCODING}-decoded)") - - # Binary reads - if "b" in options and "r" in options: - print("\n** Binary input **\n") - run_test_family(read_tests, "t", binary_files, lambda fn: open(fn, "rb")) - - # Text reads - if "t" in options and "r" in options: - print("\n** Text input **\n") - run_test_family(read_tests, "b", text_files, lambda fn: text_open(fn, "r")) - - # Binary writes - if "b" in options and "w" in options: - print("\n** Binary append **\n") - - def make_test_source(name, size): - with open(name, "rb") as f: - return f.read() - run_test_family(write_tests, "t", binary_files, - lambda fn: open(os.devnull, "wb"), make_test_source) - - # Text writes - if "t" in options and "w" in options: - print("\n** Text append **\n") - - def make_test_source(name, size): - with text_open(name, "r") as f: - return f.read() - run_test_family(write_tests, "b", text_files, - lambda fn: text_open(os.devnull, "w"), make_test_source) - - # Binary overwrites - if "b" in options and "w" in options: - print("\n** Binary overwrite **\n") - - def make_test_source(name, size): - with open(name, "rb") as f: - return f.read() - run_test_family(modify_tests, "t", binary_files, - lambda fn: open(fn, "r+b"), make_test_source) - - # Text overwrites - if "t" in options and "w" in options: - print("\n** Text overwrite **\n") - - def make_test_source(name, size): - with text_open(name, "r") as f: - return f.read() - run_test_family(modify_tests, "b", text_files, - lambda fn: text_open(fn, "r+"), make_test_source) - - -def prepare_files(): - print("Preparing files...") - # Binary files - for name, size in get_binary_files(): - if os.path.isfile(name) and os.path.getsize(name) == size: - continue - with open(name, "wb") as f: - f.write(os.urandom(size)) - # Text files - chunk = [] - with text_open(__file__, "r", encoding='utf8') as f: - for line in f: - if line.startswith("# "): - break - else: - raise RuntimeError( - f"Couldn't find chunk marker in {__file__} !") - if NEWLINES == "all": - it = itertools.cycle(["\n", "\r", "\r\n"]) - else: - it = itertools.repeat( - {"cr": "\r", "lf": "\n", "crlf": "\r\n"}[NEWLINES]) - chunk = "".join(line.replace("\n", next(it)) for line in f) - if isinstance(chunk, bytes): - chunk = chunk.decode('utf8') - chunk = chunk.encode(TEXT_ENCODING) - for name, size in get_text_files(): - if os.path.isfile(name) and os.path.getsize(name) == size: - continue - head = chunk * (size // len(chunk)) - tail = chunk[:size % len(chunk)] - # Adjust tail to end on a character boundary - while True: - try: - tail.decode(TEXT_ENCODING) - break - except UnicodeDecodeError: - tail = tail[:-1] - with open(name, "wb") as f: - f.write(head) - f.write(tail) - - -def main(): - global TEXT_ENCODING, NEWLINES - - usage = "usage: %prog [-h|--help] [options]" - parser = OptionParser(usage=usage) - parser.add_option("-b", "--binary", - action="store_true", dest="binary", default=False, - help="run binary I/O tests") - parser.add_option("-t", "--text", - action="store_true", dest="text", default=False, - help="run text I/O tests") - parser.add_option("-r", "--read", - action="store_true", dest="read", default=False, - help="run read tests") - parser.add_option("-w", "--write", - action="store_true", dest="write", default=False, - help="run write & modify tests") - parser.add_option("-E", "--encoding", - action="store", dest="encoding", default=None, - help=f"encoding for text tests (default: {TEXT_ENCODING})") - parser.add_option("-N", "--newlines", - action="store", dest="newlines", default='lf', - help="line endings for text tests " - "(one of: {lf (default), cr, crlf, all})") - parser.add_option("-m", "--io-module", - action="store", dest="io_module", default=None, - help="io module to test (default: builtin open())") - options, args = parser.parse_args() - if args: - parser.error("unexpected arguments") - NEWLINES = options.newlines.lower() - if NEWLINES not in ('lf', 'cr', 'crlf', 'all'): - parser.error(f"invalid 'newlines' option: {NEWLINES!r}") - - test_options = "" - if options.read: - test_options += "r" - if options.write: - test_options += "w" - elif not options.read: - test_options += "rw" - if options.text: - test_options += "t" - if options.binary: - test_options += "b" - elif not options.text: - test_options += "tb" - - if options.encoding: - TEXT_ENCODING = options.encoding - - if options.io_module: - globals()['open'] = __import__(options.io_module, {}, {}, ['open']).open - - prepare_files() - run_all_tests(test_options) - - -if __name__ == "__main__": - main() - - -# -- This part to exercise text reading. Don't change anything! -- -# - -""" -1. -Gáttir allar, -áðr gangi fram, -um skoðask skyli, -um skyggnast skyli, -því at óvíst er at vita, -hvar óvinir -sitja á fleti fyrir. - -2. -Gefendr heilir! -Gestr er inn kominn, -hvar skal sitja sjá? -Mjök er bráðr, -sá er á bröndum skal -síns of freista frama. - -3. -Elds er þörf, -þeims inn er kominn -ok á kné kalinn; -matar ok váða -er manni þörf, -þeim er hefr um fjall farit. - -4. -Vatns er þörf, -þeim er til verðar kemr, -þerru ok þjóðlaðar, -góðs of æðis, -ef sér geta mætti, -orðs ok endrþögu. - -5. -Vits er þörf, -þeim er víða ratar; -dælt er heima hvat; -at augabragði verðr, -sá er ekki kann -ok með snotrum sitr. - -6. -At hyggjandi sinni -skyli-t maðr hræsinn vera, -heldr gætinn at geði; -þá er horskr ok þögull -kemr heimisgarða til, -sjaldan verðr víti vörum, -því at óbrigðra vin -fær maðr aldregi -en mannvit mikit. - -7. -Inn vari gestr, -er til verðar kemr, -þunnu hljóði þegir, -eyrum hlýðir, -en augum skoðar; -svá nýsisk fróðra hverr fyrir. - -8. -Hinn er sæll, -er sér of getr -lof ok líknstafi; -ódælla er við þat, -er maðr eiga skal -annars brjóstum í. -""" - -""" -C'est revenir tard, je le sens, sur un sujet trop rebattu et déjà presque oublié. Mon état, qui ne me permet plus aucun travail suivi, mon aversion pour le genre polémique, ont causé ma lenteur à écrire et ma répugnance à publier. J'aurais même tout à fait supprimé ces Lettres, ou plutôt je lie les aurais point écrites, s'il n'eût été question que de moi : Mais ma patrie ne m'est pas tellement devenue étrangère que je puisse voir tranquillement opprimer ses citoyens, surtout lorsqu'ils n'ont compromis leurs droits qu'en défendant ma cause. Je serais le dernier des hommes si dans une telle occasion j'écoutais un sentiment qui n'est plus ni douceur ni patience, mais faiblesse et lâcheté, dans celui qu'il empêche de remplir son devoir. -Rien de moins important pour le public, j'en conviens, que la matière de ces lettres. La constitution d'une petite République, le sort d'un petit particulier, l'exposé de quelques injustices, la réfutation de quelques sophismes ; tout cela n'a rien en soi d'assez considérable pour mériter beaucoup de lecteurs : mais si mes sujets sont petits mes objets sont grands, et dignes de l'attention de tout honnête homme. Laissons Genève à sa place, et Rousseau dans sa dépression ; mais la religion, mais la liberté, la justice ! voilà, qui que vous soyez, ce qui n'est pas au-dessous de vous. -Qu'on ne cherche pas même ici dans le style le dédommagement de l'aridité de la matière. Ceux que quelques traits heureux de ma plume ont si fort irrités trouveront de quoi s'apaiser dans ces lettres, L'honneur de défendre un opprimé eût enflammé mon coeur si j'avais parlé pour un autre. Réduit au triste emploi de me défendre moi-même, j'ai dû me borner à raisonner ; m'échauffer eût été m'avilir. J'aurai donc trouvé grâce en ce point devant ceux qui s'imaginent qu'il est essentiel à la vérité d'être dite froidement ; opinion que pourtant j'ai peine à comprendre. Lorsqu'une vive persuasion nous anime, le moyen d'employer un langage glacé ? Quand Archimède tout transporté courait nu dans les rues de Syracuse, en avait-il moins trouvé la vérité parce qu'il se passionnait pour elle ? Tout au contraire, celui qui la sent ne peut s'abstenir de l'adorer ; celui qui demeure froid ne l'a pas vue. -Quoi qu'il en soit, je prie les lecteurs de vouloir bien mettre à part mon beau style, et d'examiner seulement si je raisonne bien ou mal ; car enfin, de cela seul qu'un auteur s'exprime en bons termes, je ne vois pas comment il peut s'ensuivre que cet auteur ne sait ce qu'il dit. -""" diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 04a6c0780bf972..801c64e4059ccc 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -1,39 +1,52 @@ The JIT Compiler ================ -This version of CPython can be built with an experimental just-in-time compiler. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first. +This version of CPython can be built with an experimental just-in-time compiler[^pep-744]. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first. ## Installing LLVM The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 16 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-16`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 19 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 16 on Ubuntu/Debian: +Install LLVM 19 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 16 +sudo ./llvm.sh 19 +``` + +Install LLVM 19 on Fedora Linux 40 or newer: + +```sh +sudo dnf install 'clang(major) = 19' 'llvm(major) = 19' ``` ### macOS -Install LLVM 16 with [Homebrew](https://brew.sh): +Install LLVM 19 with [Homebrew](https://brew.sh): ```sh -brew install llvm@16 +brew install llvm@19 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. ### Windows -Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Install LLVM 19 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** + +Alternatively, you can use [chocolatey](https://chocolatey.org): + +```sh +choco install llvm --version=19.1.0 +``` + ## Building @@ -41,6 +54,10 @@ For `PCbuild`-based builds, pass the new `--experimental-jit` option to `build.b For all other builds, pass the new `--enable-experimental-jit` option to `configure`. -Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform. +Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform. + +The JIT can also be enabled or disabled using the `PYTHON_JIT` environment variable, even on builds where it is enabled or disabled by default. More details about configuring CPython with the JIT and optional values for `--enable-experimental-jit` can be found [here](https://docs.python.org/dev/whatsnew/3.13.html#experimental-jit-compiler). + +[^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 603bbef59ba2e6..925b56ac669aa0 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -1,4 +1,5 @@ """Utilities for invoking LLVM tools.""" + import asyncio import functools import os @@ -7,8 +8,8 @@ import subprocess import typing -_LLVM_VERSION = 16 -_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+") +_LLVM_VERSION = 19 +_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 8eeb78e6cd69ee..228fc389584dd7 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -1,20 +1,41 @@ """Schema for the JSON produced by llvm-readobj --elf-output-style=JSON.""" + import typing HoleKind: typing.TypeAlias = typing.Literal[ + "ARM64_RELOC_BRANCH26", "ARM64_RELOC_GOT_LOAD_PAGE21", "ARM64_RELOC_GOT_LOAD_PAGEOFF12", + "ARM64_RELOC_PAGE21", + "ARM64_RELOC_PAGEOFF12", "ARM64_RELOC_UNSIGNED", - "IMAGE_REL_AMD64_ADDR64", + "IMAGE_REL_AMD64_REL32", + "IMAGE_REL_ARM64_BRANCH26", + "IMAGE_REL_ARM64_PAGEBASE_REL21", + "IMAGE_REL_ARM64_PAGEOFFSET_12A", + "IMAGE_REL_ARM64_PAGEOFFSET_12L", "IMAGE_REL_I386_DIR32", + "IMAGE_REL_I386_REL32", "R_AARCH64_ABS64", + "R_AARCH64_ADR_GOT_PAGE", + "R_AARCH64_ADR_PREL_PG_HI21", "R_AARCH64_CALL26", "R_AARCH64_JUMP26", + "R_AARCH64_ADD_ABS_LO12_NC", + "R_AARCH64_LD64_GOT_LO12_NC", "R_AARCH64_MOVW_UABS_G0_NC", "R_AARCH64_MOVW_UABS_G1_NC", "R_AARCH64_MOVW_UABS_G2_NC", "R_AARCH64_MOVW_UABS_G3", "R_X86_64_64", + "R_X86_64_GOTPCREL", + "R_X86_64_GOTPCRELX", + "R_X86_64_PC32", + "R_X86_64_REX_GOTPCRELX", + "X86_64_RELOC_BRANCH", + "X86_64_RELOC_GOT", + "X86_64_RELOC_GOT_LOAD", + "X86_64_RELOC_SIGNED", "X86_64_RELOC_UNSIGNED", ] @@ -51,12 +72,12 @@ class _COFFSymbol(typing.TypedDict): class _ELFSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int class _MachOSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int @@ -82,7 +103,7 @@ class ELFSection(typing.TypedDict): Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] SectionData: dict[typing.Literal["Bytes"], list[int]] Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]] - Type: dict[typing.Literal["Value"], str] + Type: dict[typing.Literal["Name"], str] class MachOSection(typing.TypedDict): diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 71c678e04fbfd5..2cd051b0a77b8d 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -1,7 +1,9 @@ """Core data structures for compiled code templates.""" + import dataclasses import enum import sys +import typing import _schema @@ -27,16 +29,86 @@ class HoleValue(enum.Enum): GOT = enum.auto() # The current uop's oparg (exposed as _JIT_OPARG): OPARG = enum.auto() - # The current uop's operand (exposed as _JIT_OPERAND): + # The current uop's operand on 64-bit platforms (exposed as _JIT_OPERAND): OPERAND = enum.auto() + # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): + OPERAND_HI = enum.auto() + OPERAND_LO = enum.auto() # The current uop's target (exposed as _JIT_TARGET): TARGET = enum.auto() - # The base address of the machine code for the first uop (exposed as _JIT_TOP): - TOP = enum.auto() + # The base address of the machine code for the jump target (exposed as _JIT_JUMP_TARGET): + JUMP_TARGET = enum.auto() + # The base address of the machine code for the error jump target (exposed as _JIT_ERROR_TARGET): + ERROR_TARGET = enum.auto() # A hardcoded value of zero (used for symbol lookups): ZERO = enum.auto() +# Map relocation types to our JIT's patch functions. "r" suffixes indicate that +# the patch function is relative. "x" suffixes indicate that they are "relaxing" +# (see comments in jit.c for more info): +_PATCH_FUNCS = { + # aarch64-apple-darwin: + "ARM64_RELOC_BRANCH26": "patch_aarch64_26r", + "ARM64_RELOC_GOT_LOAD_PAGE21": "patch_aarch64_21rx", + "ARM64_RELOC_GOT_LOAD_PAGEOFF12": "patch_aarch64_12x", + "ARM64_RELOC_PAGE21": "patch_aarch64_21r", + "ARM64_RELOC_PAGEOFF12": "patch_aarch64_12", + "ARM64_RELOC_UNSIGNED": "patch_64", + # x86_64-pc-windows-msvc: + "IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx", + # aarch64-pc-windows-msvc: + "IMAGE_REL_ARM64_BRANCH26": "patch_aarch64_26r", + "IMAGE_REL_ARM64_PAGEBASE_REL21": "patch_aarch64_21rx", + "IMAGE_REL_ARM64_PAGEOFFSET_12A": "patch_aarch64_12", + "IMAGE_REL_ARM64_PAGEOFFSET_12L": "patch_aarch64_12x", + # i686-pc-windows-msvc: + "IMAGE_REL_I386_DIR32": "patch_32", + "IMAGE_REL_I386_REL32": "patch_x86_64_32rx", + # aarch64-unknown-linux-gnu: + "R_AARCH64_ABS64": "patch_64", + "R_AARCH64_ADD_ABS_LO12_NC": "patch_aarch64_12", + "R_AARCH64_ADR_GOT_PAGE": "patch_aarch64_21rx", + "R_AARCH64_ADR_PREL_PG_HI21": "patch_aarch64_21r", + "R_AARCH64_CALL26": "patch_aarch64_26r", + "R_AARCH64_JUMP26": "patch_aarch64_26r", + "R_AARCH64_LD64_GOT_LO12_NC": "patch_aarch64_12x", + "R_AARCH64_MOVW_UABS_G0_NC": "patch_aarch64_16a", + "R_AARCH64_MOVW_UABS_G1_NC": "patch_aarch64_16b", + "R_AARCH64_MOVW_UABS_G2_NC": "patch_aarch64_16c", + "R_AARCH64_MOVW_UABS_G3": "patch_aarch64_16d", + # x86_64-unknown-linux-gnu: + "R_X86_64_64": "patch_64", + "R_X86_64_GOTPCREL": "patch_32r", + "R_X86_64_GOTPCRELX": "patch_x86_64_32rx", + "R_X86_64_PC32": "patch_32r", + "R_X86_64_REX_GOTPCRELX": "patch_x86_64_32rx", + # x86_64-apple-darwin: + "X86_64_RELOC_BRANCH": "patch_32r", + "X86_64_RELOC_GOT": "patch_x86_64_32rx", + "X86_64_RELOC_GOT_LOAD": "patch_x86_64_32rx", + "X86_64_RELOC_SIGNED": "patch_32r", + "X86_64_RELOC_UNSIGNED": "patch_64", +} +# Translate HoleValues to C expressions: +_HOLE_EXPRS = { + HoleValue.CODE: "(uintptr_t)code", + HoleValue.CONTINUE: "(uintptr_t)code + sizeof(code_body)", + HoleValue.DATA: "(uintptr_t)data", + HoleValue.EXECUTOR: "(uintptr_t)executor", + # These should all have been turned into DATA values by process_relocations: + # HoleValue.GOT: "", + HoleValue.OPARG: "instruction->oparg", + HoleValue.OPERAND: "instruction->operand", + HoleValue.OPERAND_HI: "(instruction->operand >> 32)", + HoleValue.OPERAND_LO: "(instruction->operand & UINT32_MAX)", + HoleValue.TARGET: "instruction->target", + HoleValue.JUMP_TARGET: "state->instruction_starts[instruction->jump_target]", + HoleValue.ERROR_TARGET: "state->instruction_starts[instruction->error_target]", + HoleValue.ZERO: "", +} + + @dataclasses.dataclass class Hole: """ @@ -53,19 +125,57 @@ class Hole: symbol: str | None # ...plus this addend: addend: int + need_state: bool = False + func: str = dataclasses.field(init=False) # Convenience method: replace = dataclasses.replace - def as_c(self) -> str: - """Dump this hole as an initialization of a C Hole struct.""" - parts = [ - f"{self.offset:#x}", - f"HoleKind_{self.kind}", - f"HoleValue_{self.value.name}", - f"&{self.symbol}" if self.symbol else "NULL", - _format_addend(self.addend), - ] - return f"{{{', '.join(parts)}}}" + def __post_init__(self) -> None: + self.func = _PATCH_FUNCS[self.kind] + + def fold(self, other: typing.Self, body: bytes) -> typing.Self | None: + """Combine two holes into a single hole, if possible.""" + instruction_a = int.from_bytes( + body[self.offset : self.offset + 4], byteorder=sys.byteorder + ) + instruction_b = int.from_bytes( + body[other.offset : other.offset + 4], byteorder=sys.byteorder + ) + reg_a = instruction_a & 0b11111 + reg_b1 = instruction_b & 0b11111 + reg_b2 = (instruction_b >> 5) & 0b11111 + + if ( + self.offset + 4 == other.offset + and self.value == other.value + and self.symbol == other.symbol + and self.addend == other.addend + and self.func == "patch_aarch64_21rx" + and other.func == "patch_aarch64_12x" + and reg_a == reg_b1 == reg_b2 + ): + # These can *only* be properly relaxed when they appear together and + # patch the same value: + folded = self.replace() + folded.func = "patch_aarch64_33rx" + return folded + return None + + def as_c(self, where: str) -> str: + """Dump this hole as a call to a patch_* function.""" + location = f"{where} + {self.offset:#x}" + value = _HOLE_EXPRS[self.value] + if self.symbol: + if value: + value += " + " + value += f"(uintptr_t)&{self.symbol}" + if _signed(self.addend) or not value: + if value: + value += " + " + value += f"{_signed(self.addend):#x}" + if self.need_state: + return f"{self.func}({location}, {value}, state);" + return f"{self.func}({location}, {value});" @dataclasses.dataclass @@ -87,42 +197,55 @@ def pad(self, alignment: int) -> None: self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") self.body.extend([0] * padding) - def emit_aarch64_trampoline(self, hole: Hole) -> None: - """Even with the large code model, AArch64 Linux insists on 28-bit jumps.""" - base = len(self.body) - where = slice(hole.offset, hole.offset + 4) - instruction = int.from_bytes(self.body[where], sys.byteorder) - instruction &= 0xFC000000 - instruction |= ((base - hole.offset) >> 2) & 0x03FFFFFF - self.body[where] = instruction.to_bytes(4, sys.byteorder) - self.disassembly += [ - f"{base + 4 * 0: x}: d2800008 mov x8, #0x0", - f"{base + 4 * 0:016x}: R_AARCH64_MOVW_UABS_G0_NC {hole.symbol}", - f"{base + 4 * 1:x}: f2a00008 movk x8, #0x0, lsl #16", - f"{base + 4 * 1:016x}: R_AARCH64_MOVW_UABS_G1_NC {hole.symbol}", - f"{base + 4 * 2:x}: f2c00008 movk x8, #0x0, lsl #32", - f"{base + 4 * 2:016x}: R_AARCH64_MOVW_UABS_G2_NC {hole.symbol}", - f"{base + 4 * 3:x}: f2e00008 movk x8, #0x0, lsl #48", - f"{base + 4 * 3:016x}: R_AARCH64_MOVW_UABS_G3 {hole.symbol}", - f"{base + 4 * 4:x}: d61f0100 br x8", - ] - for code in [ - 0xD2800008.to_bytes(4, sys.byteorder), - 0xF2A00008.to_bytes(4, sys.byteorder), - 0xF2C00008.to_bytes(4, sys.byteorder), - 0xF2E00008.to_bytes(4, sys.byteorder), - 0xD61F0100.to_bytes(4, sys.byteorder), - ]: - self.body.extend(code) - for i, kind in enumerate( - [ - "R_AARCH64_MOVW_UABS_G0_NC", - "R_AARCH64_MOVW_UABS_G1_NC", - "R_AARCH64_MOVW_UABS_G2_NC", - "R_AARCH64_MOVW_UABS_G3", - ] - ): - self.holes.append(hole.replace(offset=base + 4 * i, kind=kind)) + def remove_jump(self, *, alignment: int = 1) -> None: + """Remove a zero-length continuation jump, if it exists.""" + hole = max(self.holes, key=lambda hole: hole.offset) + match hole: + case Hole( + offset=offset, + kind="IMAGE_REL_AMD64_REL32", + value=HoleValue.GOT, + symbol="_JIT_CONTINUE", + addend=-4, + ) as hole: + # jmp qword ptr [rip] + jump = b"\x48\xFF\x25\x00\x00\x00\x00" + offset -= 3 + case Hole( + offset=offset, + kind="IMAGE_REL_I386_REL32" | "X86_64_RELOC_BRANCH", + value=HoleValue.CONTINUE, + symbol=None, + addend=-4, + ) as hole: + # jmp 5 + jump = b"\xE9\x00\x00\x00\x00" + offset -= 1 + case Hole( + offset=offset, + kind="R_AARCH64_JUMP26", + value=HoleValue.CONTINUE, + symbol=None, + addend=0, + ) as hole: + # b #4 + jump = b"\x00\x00\x00\x14" + case Hole( + offset=offset, + kind="R_X86_64_GOTPCRELX", + value=HoleValue.GOT, + symbol="_JIT_CONTINUE", + addend=addend, + ) as hole: + assert _signed(addend) == -4 + # jmp qword ptr [rip] + jump = b"\xFF\x25\x00\x00\x00\x00" + offset -= 2 + case _: + return + if self.body[offset:] == jump and offset % alignment == 0: + self.body = self.body[:offset] + self.holes.remove(hole) @dataclasses.dataclass @@ -139,13 +262,36 @@ class StencilGroup: default_factory=dict, init=False ) _got: dict[str, int] = dataclasses.field(default_factory=dict, init=False) + _trampolines: set[int] = dataclasses.field(default_factory=set, init=False) - def process_relocations(self, *, alignment: int = 1) -> None: + def process_relocations( + self, + known_symbols: dict[str, int], + *, + alignment: int = 1, + ) -> None: """Fix up all GOT and internal relocations for this stencil group.""" + for hole in self.code.holes.copy(): + if ( + hole.kind + in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"} + and hole.value is HoleValue.ZERO + ): + hole.func = "patch_aarch64_trampoline" + hole.need_state = True + assert hole.symbol is not None + if hole.symbol in known_symbols: + ordinal = known_symbols[hole.symbol] + else: + ordinal = len(known_symbols) + known_symbols[hole.symbol] = ordinal + self._trampolines.add(ordinal) + hole.addend = ordinal + hole.symbol = None + self.code.remove_jump(alignment=alignment) self.code.pad(alignment) self.data.pad(8) for stencil in [self.code, self.data]: - holes = [] for hole in stencil.holes: if hole.value is HoleValue.GOT: assert hole.symbol is not None @@ -157,14 +303,12 @@ def process_relocations(self, *, alignment: int = 1) -> None: hole.addend += addend hole.symbol = None elif ( - hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + hole.kind in {"IMAGE_REL_AMD64_REL32"} and hole.value is HoleValue.ZERO ): - self.code.emit_aarch64_trampoline(hole) - continue - holes.append(hole) - stencil.holes[:] = holes - self.code.pad(alignment) + raise ValueError( + f"Add PyAPI_FUNC(...) or PyAPI_DATA(...) to declaration of {hole.symbol}!" + ) self._emit_global_offset_table() self.code.holes.sort(key=lambda hole: hole.offset) self.data.holes.sort(key=lambda hole: hole.offset) @@ -188,8 +332,9 @@ def _emit_global_offset_table(self) -> None: if value_part and not symbol and not addend: addend_part = "" else: + signed = "+" if symbol is not None else "" addend_part = f"&{symbol}" if symbol else "" - addend_part += _format_addend(addend, signed=symbol is not None) + addend_part += f"{_signed(addend):{signed}#x}" if value_part: value_part += "+" self.data.disassembly.append( @@ -197,6 +342,21 @@ def _emit_global_offset_table(self) -> None: ) self.data.body.extend([0] * 8) + def _get_trampoline_mask(self) -> str: + bitmask: int = 0 + trampoline_mask: list[str] = [] + for ordinal in self._trampolines: + bitmask |= 1 << ordinal + while bitmask: + word = bitmask & ((1 << 32) - 1) + trampoline_mask.append(f"{word:#04x}") + bitmask >>= 32 + return "{" + (", ".join(trampoline_mask) or "0") + "}" + + def as_c(self, opname: str) -> str: + """Dump this hole as a StencilGroup initializer.""" + return f"{{emit_{opname}, {len(self.code.body)}, {len(self.data.body)}, {self._get_trampoline_mask()}}}" + def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: """ @@ -213,8 +373,8 @@ def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: return HoleValue.ZERO, symbol -def _format_addend(addend: int, signed: bool = False) -> str: - addend %= 1 << 64 - if addend & (1 << 63): - addend -= 1 << 64 - return f"{addend:{'+#x' if signed else '#x'}}" +def _signed(value: int) -> int: + value %= 1 << 64 + if value & (1 << 63): + value -= 1 << 64 + return value diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 51b091eb246413..634208da3c8157 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -1,4 +1,5 @@ """Target-specific code generation, parsing, and processing.""" + import asyncio import dataclasses import hashlib @@ -25,7 +26,6 @@ PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" - _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation @@ -37,16 +37,17 @@ class _Target(typing.Generic[_S, _R]): triple: str _: dataclasses.KW_ONLY alignment: int = 1 + args: typing.Sequence[str] = () prefix: str = "" + stable: bool = False debug: bool = False - force: bool = False verbose: bool = False + known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) def _compute_digest(self, out: pathlib.Path) -> str: hasher = hashlib.sha256() hasher.update(self.triple.encode()) - hasher.update(self.alignment.to_bytes()) - hasher.update(self.prefix.encode()) + hasher.update(self.debug.to_bytes()) # These dependencies are also reflected in _JITSources in regen.targets: hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) hasher.update((out / "pyconfig.h").read_bytes()) @@ -89,7 +90,9 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: if group.data.body: line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" group.data.disassembly.append(line) - group.process_relocations() + group.process_relocations( + known_symbols=self.known_symbols, alignment=self.alignment + ) return group def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: @@ -106,7 +109,7 @@ async def _compile( o = tempdir / f"{opname}.o" args = [ f"--target={self.triple}", - "-DPy_BUILD_CORE", + "-DPy_BUILD_CORE_MODULE", "-D_DEBUG" if self.debug else "-DNDEBUG", f"-D_JIT_OPCODE={opname}", "-D_PyJIT_ACTIVE", @@ -116,60 +119,91 @@ async def _compile( f"-I{CPYTHON / 'Include' / 'internal'}", f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", f"-I{CPYTHON / 'Python'}", + f"-I{CPYTHON / 'Tools' / 'jit'}", "-O3", "-c", + # This debug info isn't necessary, and bloats out the JIT'ed code. + # We *may* be able to re-enable this, process it, and JIT it for a + # nicer debugging experience... but that needs a lot more research: "-fno-asynchronous-unwind-tables", - # SET_FUNCTION_ATTRIBUTE on 32-bit Windows debug builds: - "-fno-jump-tables", - # Position-independent code adds indirection to every load and jump: - "-fno-pic", - # Don't make calls to weird stack-smashing canaries: + # Don't call built-in functions that we can't find or patch: + "-fno-builtin", + # Emit relaxable 64-bit calls/jumps, so we don't have to worry about + # about emitting in-range trampolines for out-of-range targets. + # We can probably remove this and emit trampolines in the future: + "-fno-plt", + # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", - # We have three options for code model: - # - "small": the default, assumes that code and data reside in the - # lowest 2GB of memory (128MB on aarch64) - # - "medium": assumes that code resides in the lowest 2GB of memory, - # and makes no assumptions about data (not available on aarch64) - # - "large": makes no assumptions about either code or data - "-mcmodel=large", + "-std=c11", "-o", f"{o}", - "-std=c11", f"{c}", + *self.args, ] await _llvm.run("clang", args, echo=self.verbose) return await self._parse(o) async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() - opnames = sorted(re.findall(r"\n {8}case (\w+): \{\n", generated_cases)) + cases_and_opnames = sorted( + re.findall( + r"\n {8}(case (\w+): \{\n.*?\n {8}\})", generated_cases, flags=re.DOTALL + ) + ) tasks = [] with tempfile.TemporaryDirectory() as tempdir: work = pathlib.Path(tempdir).resolve() async with asyncio.TaskGroup() as group: - for opname in opnames: - coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) + coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) + tasks.append(group.create_task(coro, name="trampoline")) + template = TOOLS_JIT_TEMPLATE_C.read_text() + for case, opname in cases_and_opnames: + # Write out a copy of the template with *only* this case + # inserted. This is about twice as fast as #include'ing all + # of executor_cases.c.h each time we compile (since the C + # compiler wastes a bunch of time parsing the dead code for + # all of the other cases): + c = work / f"{opname}.c" + c.write_text(template.replace("CASE", case)) + coro = self._compile(opname, c, work) tasks.append(group.create_task(coro, name=opname)) return {task.get_name(): task.result() for task in tasks} - def build(self, out: pathlib.Path, *, comment: str = "") -> None: + def build( + self, out: pathlib.Path, *, comment: str = "", force: bool = False + ) -> None: """Build jit_stencils.h in the given directory.""" + if not self.stable: + warning = f"JIT support for {self.triple} is still experimental!" + request = "Please report any issues you encounter.".center(len(warning)) + outline = "=" * len(warning) + print("\n".join(["", outline, warning, request, outline, ""])) digest = f"// {self._compute_digest(out)}\n" jit_stencils = out / "jit_stencils.h" if ( - not self.force + not force and jit_stencils.exists() and jit_stencils.read_text().startswith(digest) ): return stencil_groups = asyncio.run(self._build_stencils()) - with jit_stencils.open("w") as file: - file.write(digest) - if comment: - file.write(f"// {comment}\n") - file.write("") - for line in _writer.dump(stencil_groups): - file.write(f"{line}\n") + jit_stencils_new = out / "jit_stencils.h.new" + try: + with jit_stencils_new.open("w") as file: + file.write(digest) + if comment: + file.write(f"// {comment}\n") + file.write("\n") + for line in _writer.dump(stencil_groups, self.known_symbols): + file.write(f"{line}\n") + try: + jit_stencils_new.replace(jit_stencils) + except FileNotFoundError: + # another process probably already moved the file + if not jit_stencils.is_file(): + raise + finally: + jit_stencils_new.unlink(missing_ok=True) class _COFF( @@ -200,12 +234,21 @@ def _handle_section( offset = base + symbol["Value"] name = symbol["Name"] name = name.removeprefix(self.prefix) - group.symbols[name] = value, offset + if name not in group.symbols: + group.symbols[name] = value, offset for wrapped_relocation in section["Relocations"]: relocation = wrapped_relocation["Relocation"] hole = self._handle_relocation(base, relocation, stencil.body) stencil.holes.append(hole) + def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None]: + if name.startswith("__imp_"): + name = name.removeprefix("__imp_") + name = name.removeprefix(self.prefix) + return _stencils.HoleValue.GOT, name + name = name.removeprefix(self.prefix) + return _stencils.symbol_to_value(name) + def _handle_relocation( self, base: int, relocation: _schema.COFFRelocation, raw: bytes ) -> _stencils.Hole: @@ -213,21 +256,36 @@ def _handle_relocation( case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_AMD64_ADDR64" as kind}, + "Type": {"Name": "IMAGE_REL_I386_DIR32" as kind}, }: offset += base - s = s.removeprefix(self.prefix) - value, symbol = _stencils.symbol_to_value(s) - addend = int.from_bytes(raw[offset : offset + 8], "little") + value, symbol = self._unwrap_dllimport(s) + addend = int.from_bytes(raw[offset : offset + 4], "little") case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + "Type": { + "Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind + }, }: offset += base - s = s.removeprefix(self.prefix) - value, symbol = _stencils.symbol_to_value(s) - addend = int.from_bytes(raw[offset : offset + 4], "little") + value, symbol = self._unwrap_dllimport(s) + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Symbol": s, + "Type": { + "Name": "IMAGE_REL_ARM64_BRANCH26" + | "IMAGE_REL_ARM64_PAGEBASE_REL21" + | "IMAGE_REL_ARM64_PAGEOFFSET_12A" + | "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind + }, + }: + offset += base + value, symbol = self._unwrap_dllimport(s) + addend = 0 case _: raise NotImplementedError(relocation) return _stencils.Hole(offset, kind, value, symbol, addend) @@ -239,7 +297,7 @@ class _ELF( def _handle_section( self, section: _schema.ELFSection, group: _stencils.StencilGroup ) -> None: - section_type = section["Type"]["Value"] + section_type = section["Type"]["Name"] flags = {flag["Name"] for flag in section["Flags"]["Flags"]} if section_type == "SHT_RELA": assert "SHF_INFO_LINK" in flags, flags @@ -267,7 +325,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = len(stencil.body) + symbol["Value"] - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset stencil.body.extend(section["SectionData"]["Bytes"]) @@ -276,6 +334,7 @@ def _handle_section( assert section_type in { "SHT_GROUP", "SHT_LLVM_ADDRSIG", + "SHT_NOTE", "SHT_NULL", "SHT_STRTAB", "SHT_SYMTAB", @@ -284,12 +343,28 @@ def _handle_section( def _handle_relocation( self, base: int, relocation: _schema.ELFRelocation, raw: bytes ) -> _stencils.Hole: + symbol: str | None match relocation: case { "Addend": addend, "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": { + "Name": "R_AARCH64_ADR_GOT_PAGE" + | "R_AARCH64_LD64_GOT_LO12_NC" + | "R_X86_64_GOTPCREL" + | "R_X86_64_GOTPCRELX" + | "R_X86_64_REX_GOTPCRELX" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + case { + "Addend": addend, + "Offset": offset, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -310,6 +385,8 @@ def _handle_section( flags = {flag["Name"] for flag in section["Attributes"]["Flags"]} name = section["Name"]["Value"] name = name.removeprefix(self.prefix) + if "Debug" in flags: + return if "SomeInstructions" in flags: value = _stencils.HoleValue.CODE stencil = group.code @@ -330,7 +407,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = symbol["Value"] - start_address - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset assert "Relocations" in section @@ -346,9 +423,9 @@ def _handle_relocation( match relocation: case { "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "ARM64_RELOC_GOT_LOAD_PAGE21" + "Name": "ARM64_RELOC_GOT_LOAD_PAGE21" | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind }, }: @@ -358,12 +435,38 @@ def _handle_relocation( addend = 0 case { "Offset": offset, - "Section": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Section": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_SIGNED" as kind}, + } | { + "Offset": offset, + "Symbol": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Section": {"Name": s}, + "Type": {"Name": kind}, } | { "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -371,24 +474,40 @@ def _handle_relocation( addend = 0 case _: raise NotImplementedError(relocation) - # Turn Clang's weird __bzero calls into normal bzero calls: - if symbol == "__bzero": - symbol = "bzero" return _stencils.Hole(offset, kind, value, symbol, addend) def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" + target: _COFF | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): - return _MachO(host, alignment=8, prefix="_") - if re.fullmatch(r"aarch64-.*-linux-gnu", host): - return _ELF(host, alignment=8) - if re.fullmatch(r"i686-pc-windows-msvc", host): - return _COFF(host, prefix="_") - if re.fullmatch(r"x86_64-apple-darwin.*", host): - return _MachO(host, prefix="_") - if re.fullmatch(r"x86_64-pc-windows-msvc", host): - return _COFF(host) - if re.fullmatch(r"x86_64-.*-linux-gnu", host): - return _ELF(host) - raise ValueError(host) + target = _MachO(host, alignment=8, prefix="_") + elif re.fullmatch(r"aarch64-pc-windows-msvc", host): + args = ["-fms-runtime-lib=dll"] + target = _COFF(host, alignment=8, args=args) + elif re.fullmatch(r"aarch64-.*-linux-gnu", host): + args = [ + "-fpic", + # On aarch64 Linux, intrinsics were being emitted and this flag + # was required to disable them. + "-mno-outline-atomics", + ] + target = _ELF(host, alignment=8, args=args) + elif re.fullmatch(r"i686-pc-windows-msvc", host): + args = [ + "-DPy_NO_ENABLE_SHARED", + # __attribute__((preserve_none)) is not supported + "-Wno-ignored-attributes", + ] + target = _COFF(host, args=args, prefix="_") + elif re.fullmatch(r"x86_64-apple-darwin.*", host): + target = _MachO(host, prefix="_") + elif re.fullmatch(r"x86_64-pc-windows-msvc", host): + args = ["-fms-runtime-lib=dll"] + target = _COFF(host, args=args) + elif re.fullmatch(r"x86_64-.*-linux-gnu", host): + args = ["-fpic"] + target = _ELF(host, args=args) + else: + raise ValueError(host) + return target diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 8a2a42e75cfb9b..f33d8ef322f073 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -1,95 +1,82 @@ """Utilities for writing StencilGroups out to a C header file.""" + +import itertools import typing +import math -import _schema import _stencils -def _dump_header() -> typing.Iterator[str]: - yield "typedef enum {" - for kind in typing.get_args(_schema.HoleKind): - yield f" HoleKind_{kind}," - yield "} HoleKind;" - yield "" - yield "typedef enum {" - for value in _stencils.HoleValue: - yield f" HoleValue_{value.name}," - yield "} HoleValue;" - yield "" - yield "typedef struct {" - yield " const uint64_t offset;" - yield " const HoleKind kind;" - yield " const HoleValue value;" - yield " const void *symbol;" - yield " const uint64_t addend;" - yield "} Hole;" +def _dump_footer( + groups: dict[str, _stencils.StencilGroup], symbols: dict[str, int] +) -> typing.Iterator[str]: + symbol_mask_size = max(math.ceil(len(symbols) / 32), 1) + yield f'static_assert(SYMBOL_MASK_WORDS >= {symbol_mask_size}, "SYMBOL_MASK_WORDS too small");' yield "" yield "typedef struct {" - yield " const size_t body_size;" - yield " const unsigned char * const body;" - yield " const size_t holes_size;" - yield " const Hole * const holes;" - yield "} Stencil;" - yield "" - yield "typedef struct {" - yield " const Stencil code;" - yield " const Stencil data;" + yield " void (*emit)(" + yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," + yield " const _PyUOpInstruction *instruction, jit_state *state);" + yield " size_t code_size;" + yield " size_t data_size;" + yield " symbol_mask trampoline_mask;" yield "} StencilGroup;" yield "" - - -def _dump_footer(opnames: typing.Iterable[str]) -> typing.Iterator[str]: - yield "#define INIT_STENCIL(STENCIL) { \\" - yield " .body_size = Py_ARRAY_LENGTH(STENCIL##_body) - 1, \\" - yield " .body = STENCIL##_body, \\" - yield " .holes_size = Py_ARRAY_LENGTH(STENCIL##_holes) - 1, \\" - yield " .holes = STENCIL##_holes, \\" - yield "}" - yield "" - yield "#define INIT_STENCIL_GROUP(OP) { \\" - yield " .code = INIT_STENCIL(OP##_code), \\" - yield " .data = INIT_STENCIL(OP##_data), \\" - yield "}" + yield f"static const StencilGroup trampoline = {groups['trampoline'].as_c('trampoline')};" yield "" - yield "static const StencilGroup stencil_groups[512] = {" - for opname in opnames: - yield f" [{opname}] = INIT_STENCIL_GROUP({opname})," + yield "static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {" + for opname, group in sorted(groups.items()): + if opname == "trampoline": + continue + yield f" [{opname}] = {group.as_c(opname)}," yield "};" yield "" - yield "#define GET_PATCHES() { \\" - for value in _stencils.HoleValue: - yield f" [HoleValue_{value.name}] = (uint64_t)0xBADBADBADBADBADB, \\" - yield "}" + yield f"static const void * const symbols_map[{max(len(symbols), 1)}] = {{" + if symbols: + for symbol, ordinal in symbols.items(): + yield f" [{ordinal}] = &{symbol}," + else: + yield " 0" + yield "};" def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: - yield f"// {opname}" + yield "void" + yield f"emit_{opname}(" + yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," + yield " const _PyUOpInstruction *instruction, jit_state *state)" + yield "{" for part, stencil in [("code", group.code), ("data", group.data)]: for line in stencil.disassembly: - yield f"// {line}" + yield f" // {line}" if stencil.body: - size = len(stencil.body) + 1 - yield f"static const unsigned char {opname}_{part}_body[{size}] = {{" + yield f" const unsigned char {part}_body[{len(stencil.body)}] = {{" for i in range(0, len(stencil.body), 8): row = " ".join(f"{byte:#04x}," for byte in stencil.body[i : i + 8]) - yield f" {row}" - yield "};" - else: - yield f"static const unsigned char {opname}_{part}_body[1];" - if stencil.holes: - size = len(stencil.holes) + 1 - yield f"static const Hole {opname}_{part}_holes[{size}] = {{" - for hole in stencil.holes: - yield f" {hole.as_c()}," - yield "};" - else: - yield f"static const Hole {opname}_{part}_holes[1];" + yield f" {row}" + yield " };" + # Data is written first (so relaxations in the code work properly): + for part, stencil in [("data", group.data), ("code", group.code)]: + if stencil.body: + yield f" memcpy({part}, {part}_body, sizeof({part}_body));" + skip = False + stencil.holes.sort(key=lambda hole: hole.offset) + for hole, pair in itertools.zip_longest(stencil.holes, stencil.holes[1:]): + if skip: + skip = False + continue + if pair and (folded := hole.fold(pair, stencil.body)): + skip = True + hole = folded + yield f" {hole.as_c(part)}" + yield "}" yield "" -def dump(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: +def dump( + groups: dict[str, _stencils.StencilGroup], symbols: dict[str, int] +) -> typing.Iterator[str]: """Yield a JIT compiler line-by-line as a C header file.""" - yield from _dump_header() - for opname, group in groups.items(): + for opname, group in sorted(groups.items()): yield from _dump_stencil(opname, group) - yield from _dump_footer(groups) + yield from _dump_footer(groups, symbols) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 4d4ace14ebf26c..4a23c6f0afa74a 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -1,4 +1,5 @@ """Build an experimental just-in-time compiler for CPython.""" + import argparse import pathlib import shlex @@ -23,6 +24,5 @@ ) args = parser.parse_args() args.target.debug = args.debug - args.target.force = args.force args.target.verbose = args.verbose - args.target.build(pathlib.Path.cwd(), comment=comment) + args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force) diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt new file mode 100644 index 00000000000000..e379e39def0eaf --- /dev/null +++ b/Tools/jit/ignore-tests-emulated-linux.txt @@ -0,0 +1,84 @@ +test_multiprocessing_fork +test_strftime_y2k +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_run +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_subprocess +test.test_asyncio.test_unix_events.TestFork.test_fork_signal_handling +test.test_cmd_line.CmdLineTest.test_no_std_streams +test.test_cmd_line.CmdLineTest.test_no_stdin +test.test_concurrent_futures.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecutorTest.test_ressources_gced_in_workers +test.test_external_inspection.TestGetStackTrace.test_remote_stack_trace +test.test_external_inspection.TestGetStackTrace.test_self_trace +test.test_faulthandler.FaultHandlerTests.test_enable_fd +test.test_faulthandler.FaultHandlerTests.test_enable_file +test.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_logging.ConfigDictTest.test_111615 +test.test_logging.ConfigDictTest.test_config_queue_handler +test.test_logging.ConfigDictTest.test_multiprocessing_queues +test.test_logging.ConfigDictTest.test_config_queue_handler_multiprocessing_context +test.test_os.ForkTests.test_fork_warns_when_non_python_thread_exists +test.test_os.TimerfdTests.test_timerfd_initval +test.test_os.TimerfdTests.test_timerfd_interval +test.test_os.TimerfdTests.test_timerfd_TFD_TIMER_ABSTIME +test.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.PosixPathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PosixPathTest.test_is_mount_root +test.test_posix.TestPosixSpawn.test_close_file +test.test_posix.TestPosixSpawnP.test_close_file +test.test_posixpath.PosixPathTest.test_ismount +test.test_signal.StressTest.test_stress_modifying_handlers +test.test_socket.BasicCANTest.testFilter +test.test_socket.BasicCANTest.testLoopback +test.test_socket.LinuxKernelCryptoAPI.test_aead_aes_gcm +test.test_socket.LinuxKernelCryptoAPI.test_aes_cbc +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_args_0 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_executable +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCase.test_empty_env +test.test_subprocess.ProcessTestCase.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCase.test_one_environment_variable +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCaseNoPoll.test_empty_env +test.test_subprocess.ProcessTestCaseNoPoll.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCaseNoPoll.test_one_environment_variable +test.test_venv.BasicTest.test_zippath_from_non_installed_posix diff --git a/Tools/jit/jit.h b/Tools/jit/jit.h new file mode 100644 index 00000000000000..47da64cb12bd24 --- /dev/null +++ b/Tools/jit/jit.h @@ -0,0 +1,4 @@ +// To use preserve_none in JIT builds, we need to declare a separate function +// pointer with __attribute__((preserve_none)), since this attribute may not be +// supported by the compiler used to build the rest of the interpreter. +typedef jit_func __attribute__((preserve_none)) jit_func_preserve_none; diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 12303a550d8879..57c1006ab423e9 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -1,7 +1,9 @@ #include "Python.h" +#include "pycore_backoff.h" #include "pycore_call.h" #include "pycore_ceval.h" +#include "pycore_cell.h" #include "pycore_dict.h" #include "pycore_emscripten_signal.h" #include "pycore_intrinsics.h" @@ -9,12 +11,18 @@ #include "pycore_long.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_range.h" #include "pycore_setobject.h" #include "pycore_sliceobject.h" +#include "pycore_descrobject.h" +#include "pycore_stackref.h" #include "ceval_macros.h" +#include "jit.h" + #undef CURRENT_OPARG #define CURRENT_OPARG() (_oparg) @@ -38,61 +46,87 @@ goto LABEL ## _tier_two; \ } while (0) +#undef GOTO_TIER_TWO +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + __attribute__((musttail)) \ + return ((jit_func_preserve_none)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \ +} while (0) + +#undef GOTO_TIER_ONE +#define GOTO_TIER_ONE(TARGET) \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + return TARGET; \ +} while (0) + #undef LOAD_IP #define LOAD_IP(UNUSED) \ do { \ } while (0) #define PATCH_VALUE(TYPE, NAME, ALIAS) \ - extern void ALIAS; \ - TYPE NAME = (TYPE)(uint64_t)&ALIAS; + PyAPI_DATA(void) ALIAS; \ + TYPE NAME = (TYPE)(uintptr_t)&ALIAS; #define PATCH_JUMP(ALIAS) \ - extern void ALIAS; \ +do { \ + PyAPI_DATA(void) ALIAS; \ __attribute__((musttail)) \ - return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); + return ((jit_func_preserve_none)&ALIAS)(frame, stack_pointer, tstate); \ +} while (0) + +#undef JUMP_TO_JUMP_TARGET +#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET) + +#undef JUMP_TO_ERROR +#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET) + +#undef WITHIN_STACK_BOUNDS +#define WITHIN_STACK_BOUNDS() 1 -_Py_CODEUNIT * -_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) +#define TIER_TWO 2 + +__attribute__((preserve_none)) _Py_CODEUNIT * +_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { // Locals that the instruction implementations expect to exist: PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) int oparg; - int opcode = _JIT_OPCODE; - _PyUOpInstruction *next_uop; + int uopcode = _JIT_OPCODE; + _Py_CODEUNIT *next_instr; // Other stuff we need handy: PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) +#if SIZEOF_VOID_P == 8 PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) +#else + assert(SIZEOF_VOID_P == 4); + PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI) + PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO) + uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo; +#endif PATCH_VALUE(uint32_t, _target, _JIT_TARGET) - // The actual instruction definitions (only one will be used): - if (opcode == _JUMP_TO_TOP) { - CHECK_EVAL_BREAKER(); - PATCH_JUMP(_JIT_TOP); - } - switch (opcode) { -#include "executor_cases.c.h" + + OPT_STAT_INC(uops_executed); + UOP_STAT_INC(uopcode, execution_count); + + switch (uopcode) { + // The actual instruction definition gets inserted here: + CASE default: Py_UNREACHABLE(); } PATCH_JUMP(_JIT_CONTINUE); // Labels that the instruction implementations expect to exist: -unbound_local_error_tier_two: - _PyEval_FormatExcCheckArg( - tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)); - goto error_tier_two; -pop_4_error_tier_two: - STACK_SHRINK(1); -pop_3_error_tier_two: - STACK_SHRINK(1); -pop_2_error_tier_two: - STACK_SHRINK(1); -pop_1_error_tier_two: - STACK_SHRINK(1); + error_tier_two: - _PyFrame_SetStackPointer(frame, stack_pointer); - return NULL; -deoptimize: - _PyFrame_SetStackPointer(frame, stack_pointer); - return _PyCode_CODE(_PyFrame_GetCode(frame)) + _target; + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(NULL); +exit_to_tier1: + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); +exit_to_tier1_dynamic: + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(frame->instr_ptr); } diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c new file mode 100644 index 00000000000000..f0cffa2f049d26 --- /dev/null +++ b/Tools/jit/trampoline.c @@ -0,0 +1,24 @@ +#include "Python.h" + +#include "pycore_ceval.h" +#include "pycore_frame.h" +#include "pycore_jit.h" + +#include "jit.h" + +_Py_CODEUNIT * +_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) +{ + // This is subtle. The actual trace will return to us once it exits, so we + // need to make sure that we stay alive until then. If our trace side-exits + // into another trace, and this trace is then invalidated, the code for + // *this function* will be freed and we'll crash upon return: + PyAPI_DATA(void) _JIT_EXECUTOR; + PyObject *executor = (PyObject *)(uintptr_t)&_JIT_EXECUTOR; + Py_INCREF(executor); + // Note that this is *not* a tail call: + PyAPI_DATA(void) _JIT_CONTINUE; + _Py_CODEUNIT *target = ((jit_func_preserve_none)&_JIT_CONTINUE)(frame, stack_pointer, tstate); + Py_SETREF(tstate->previous_executor, executor); + return target; +} diff --git a/Tools/msi/README.txt b/Tools/msi/README.txt index c25ada8397cd95..8ae156450d5240 100644 --- a/Tools/msi/README.txt +++ b/Tools/msi/README.txt @@ -9,7 +9,7 @@ script: For an official release, the installer should be built with the Tools/msi/buildrelease.bat script and environment variables: - set PYTHON= + set PYTHON= set SPHINXBUILD= set PATH=;%PATH% @@ -61,24 +61,20 @@ the initial download size by separating them into their own MSIs. Building the Installer ====================== -Before building the installer, download extra build dependencies using -Tools\msi\get_externals.bat. (Note that this is in addition to the +Before building the installer, download the extra build dependencies +using Tools\msi\get_externals.bat. (Note that this is in addition to the similarly named file in PCbuild.) -One of the dependencies used in builds is WiX, a toolset that lets developers -create installers for Windows Installer, the Windows installation engine. WiX -has a dependency on the Microsoft .NET Framework Version 3.5 (which may not be -configured on recent versions of Windows, such as Windows 10). If you are -building on a recent Windows version, use the Control Panel (Programs | Programs -and Features | Turn Windows Features on or off) and ensure that the entry -".NET Framework 3.5 (includes .NET 2.0 and 3.0)" is enabled. - -For Python 3.11.x and above, enable "Microsoft .NET Framework 4.8 Advanced Services" -instead of "Microsoft .NET Framework Version 3.5" available for Windows 10 and above. -Also make sure "MSVC v143 - VS 2022 C++ ARM64 build tools" are selected under -"Desktop Development with C++" in "Visual Studio installer" even if you are not -building on ARM64 along with other x64 related v143 build tools. This is because for -3.11.x and above we have upgraded to Wix-3.14. +One of the dependencies used in the build process is WiX, a toolset that +allows developers to create installers for Windows Installer, the +Windows installation engine. If you're not using the copy of WiX +installed by Tools\msi\get_externals.bat, you'll need to set the +"WixInstallPath" environment variable before building. + +Additionally, ensure that "MSVC v14x - VS 20xx C++ ARM64/ARM64EC build tools" +is selected under "Desktop Development with C++" in the "Visual Studio Installer", +even if you're not building on ARM64. This is required because we've upgraded +to WiX 3.14, which requires these tools for Python 3.10 and later versions. For testing, the installer should be built with the Tools/msi/build.bat script: @@ -107,7 +103,7 @@ be available alongside. This takes longer, but is easier to share. For an official release, the installer should be built with the Tools/msi/buildrelease.bat script: - set PYTHON= + set PYTHON= set SPHINXBUILD= set PATH=;%PATH% diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index 839f6204d9e0b2..77fb4779208853 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -127,7 +127,7 @@ if "%1" EQU "x86" ( set OUTDIR_PLAT=amd64 set OBJDIR_PLAT=x64 ) else if "%1" EQU "ARM64" ( - set BUILD=%Py_OutDir%amd64\ + set BUILD=%Py_OutDir%arm64\ set PGO=%~2 set BUILD_PLAT=ARM64 set OUTDIR_PLAT=arm64 diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 1540f050159a54..49f681d3e11d2e 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -88,6 +88,7 @@ Select Customize to review current options.
    Install Python [ShortVersion] for &all users for &all users (requires admin privileges) Use admin privi&leges when installing py.exe + Python Launcher is already installed &Precompile standard library Download debugging &symbols Download debu&g binaries (requires VS 2017 or later) @@ -122,7 +123,7 @@ Feel free to post at <a href="https://discuss.python.org/c/users/7">discus You must restart your computer to complete the rollback of the software. &Restart Unable to install [WixBundleName] due to an existing install. Use Programs and Features to modify, repair or remove [WixBundleName]. - At least Windows 8.1 or Windows Server 2012 are required to install [WixBundleName] + At least Windows 10 or Windows Server 2016 are required to install [WixBundleName] Visit <a href="https://www.python.org/downloads/">python.org</a> to download an earlier version of Python. Disable path length limit diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index 3a17ffbaa0b655..1e0df5084ff079 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -213,6 +213,7 @@ static struct { LPCWSTR regName; LPCWSTR variableName; } OPTIONAL_FEATURES[] = { { L"Shortcuts", L"Shortcuts" }, // Include_launcher and AssociateFiles are handled separately and so do // not need to be included in this list. + { L"freethreaded", L"Include_freethreaded" }, { nullptr, nullptr } }; @@ -442,6 +443,14 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { ThemeControlElevates(_theme, ID_INSTALL_BUTTON, elevated); ThemeControlElevates(_theme, ID_INSTALL_SIMPLE_BUTTON, elevated); ThemeControlElevates(_theme, ID_INSTALL_UPGRADE_BUTTON, elevated); + + LONGLONG blockedLauncher; + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { + LOC_STRING *pLocString = nullptr; + if (SUCCEEDED(LocGetString(_wixLoc, L"#(loc.ShortInstallLauncherBlockedLabel)", &pLocString)) && pLocString) { + ThemeSetTextControl(_theme, ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, pLocString->wzText); + } + } } void Custom1Page_Show() { @@ -456,11 +465,11 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; LPCWSTR locKey = L"#(loc.Include_launcherHelp)"; - LONGLONG detectedLauncher; + LONGLONG blockedLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher)) && detectedLauncher) { + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherRemove)"; - } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &detectedLauncher)) && detectedLauncher) { + } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherUpgrade)"; } @@ -718,25 +727,67 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { __in DWORD64 /*dw64Version*/, __in BOOTSTRAPPER_RELATED_OPERATION operation ) { - if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation && - (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1) || - CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_JustForMe", -1))) { - auto hr = LoadAssociateFilesStateFromKey(_engine, fPerMachine ? HKEY_LOCAL_MACHINE : HKEY_CURRENT_USER); - if (hr == S_OK) { - _engine->SetVariableNumeric(L"AssociateFiles", 1); - } else if (hr == S_FALSE) { - _engine->SetVariableNumeric(L"AssociateFiles", 0); - } else if (FAILED(hr)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); + // Only check launcher_AllUsers because we'll find the same packages + // twice if we check launcher_JustForMe as well. + if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected existing launcher install"); + + LONGLONG blockedLauncher, detectedLauncher; + if (FAILED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher))) { + blockedLauncher = 0; } - LONGLONG includeLauncher; - if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) - || includeLauncher == -1) { - _engine->SetVariableNumeric(L"Include_launcher", 1); - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0); + // Get the prior DetectedLauncher value so we can see if we've + // detected more than one, and then update the stored variable + // (we use the original value later on via the local). + if (FAILED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher))) { + detectedLauncher = 0; + } + if (!detectedLauncher) { + _engine->SetVariableNumeric(L"DetectedLauncher", 1); + } + + if (blockedLauncher) { + // Nothing else to do, we're already blocking + } + else if (BOOTSTRAPPER_RELATED_OPERATION_DOWNGRADE == operation) { + // Found a higher version, so we can't install ours. + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Higher version launcher has been detected."); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Launcher will not be installed"); + _engine->SetVariableNumeric(L"BlockedLauncher", 1); + } + else if (detectedLauncher) { + if (!blockedLauncher) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Multiple launcher installs have been detected."); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "No launcher will be installed or upgraded until one has been removed."); + _engine->SetVariableNumeric(L"BlockedLauncher", 1); + } + } + else if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation) { + // Found an older version, so let's run the equivalent as an upgrade + // This overrides "unknown" all users options, but will leave alone + // any that have already been set/detected. + // User can deselect the option to include the launcher, but cannot + // change it from the current per user/machine setting. + LONGLONG includeLauncher, includeLauncherAllUsers; + if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))) { + includeLauncher = -1; + } + if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &includeLauncherAllUsers))) { + includeLauncherAllUsers = -1; + } + + if (includeLauncher < 0) { + _engine->SetVariableNumeric(L"Include_launcher", 1); + } + if (includeLauncherAllUsers < 0) { + _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0); + } else if (includeLauncherAllUsers != fPerMachine ? 1 : 0) { + // Requested AllUsers option is inconsistent, so block + _engine->SetVariableNumeric(L"BlockedLauncher", 1); + } + _engine->SetVariableNumeric(L"DetectedOldLauncher", 1); } - _engine->SetVariableNumeric(L"DetectedOldLauncher", 1); } return CheckCanceled() ? IDCANCEL : IDNOACTION; } @@ -784,48 +835,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { __in LPCWSTR wzPackageId, __in HRESULT hrStatus, __in BOOTSTRAPPER_PACKAGE_STATE state - ) { - if (FAILED(hrStatus)) { - return; - } - - BOOL detectedLauncher = FALSE; - HKEY hkey = HKEY_LOCAL_MACHINE; - if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1)) { - if (BOOTSTRAPPER_PACKAGE_STATE_PRESENT == state || BOOTSTRAPPER_PACKAGE_STATE_OBSOLETE == state) { - detectedLauncher = TRUE; - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 1); - } - } else if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_JustForMe", -1)) { - if (BOOTSTRAPPER_PACKAGE_STATE_PRESENT == state || BOOTSTRAPPER_PACKAGE_STATE_OBSOLETE == state) { - detectedLauncher = TRUE; - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 0); - } - } - - LONGLONG includeLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) - && includeLauncher != -1) { - detectedLauncher = FALSE; - } - - if (detectedLauncher) { - /* When we detect the current version of the launcher. */ - _engine->SetVariableNumeric(L"Include_launcher", 1); - _engine->SetVariableNumeric(L"DetectedLauncher", 1); - _engine->SetVariableString(L"Include_launcherState", L"disable"); - _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable"); - - auto hr = LoadAssociateFilesStateFromKey(_engine, hkey); - if (hr == S_OK) { - _engine->SetVariableNumeric(L"AssociateFiles", 1); - } else if (hr == S_FALSE) { - _engine->SetVariableNumeric(L"AssociateFiles", 0); - } else if (FAILED(hr)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); - } - } - } + ) { } virtual STDMETHODIMP_(void) OnDetectComplete(__in HRESULT hrStatus) { @@ -835,19 +845,67 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } if (SUCCEEDED(hrStatus)) { - LONGLONG includeLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) - && includeLauncher == -1) { - if (BOOTSTRAPPER_ACTION_LAYOUT == _command.action || - (BOOTSTRAPPER_ACTION_INSTALL == _command.action && !_upgrading)) { - // When installing/downloading, we want to include the launcher - // by default. - _engine->SetVariableNumeric(L"Include_launcher", 1); - } else { - // Any other action, if we didn't detect the MSI then we want to - // keep it excluded - _engine->SetVariableNumeric(L"Include_launcher", 0); - _engine->SetVariableNumeric(L"AssociateFiles", 0); + // Update launcher install states + // If we didn't detect any existing installs, Include_launcher and + // InstallLauncherAllUsers will both be -1, so we will set to their + // defaults and leave the options enabled. + // Otherwise, if we detected an existing install, we disable the + // options so they remain fixed. + // The code in OnDetectRelatedMsiPackage is responsible for figuring + // out whether existing installs are compatible with the settings in + // place during detection. + LONGLONG blockedLauncher; + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) + && blockedLauncher) { + _engine->SetVariableNumeric(L"Include_launcher", 0); + _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 0); + _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable"); + _engine->SetVariableString(L"Include_launcherState", L"disable"); + } + else { + LONGLONG includeLauncher, includeLauncherAllUsers, associateFiles; + + if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))) { + includeLauncher = -1; + } + if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &includeLauncherAllUsers))) { + includeLauncherAllUsers = -1; + } + if (FAILED(BalGetNumericVariable(L"AssociateFiles", &associateFiles))) { + associateFiles = -1; + } + + if (includeLauncherAllUsers < 0) { + includeLauncherAllUsers = 0; + _engine->SetVariableNumeric(L"InstallLauncherAllUsers", includeLauncherAllUsers); + } + + if (includeLauncher < 0) { + if (BOOTSTRAPPER_ACTION_LAYOUT == _command.action || + (BOOTSTRAPPER_ACTION_INSTALL == _command.action && !_upgrading)) { + // When installing/downloading, we include the launcher + // (though downloads should ignore this setting anyway) + _engine->SetVariableNumeric(L"Include_launcher", 1); + } else { + // Any other action, we should have detected an existing + // install (e.g. on remove/modify), so if we didn't, we + // assume it's not selected. + _engine->SetVariableNumeric(L"Include_launcher", 0); + _engine->SetVariableNumeric(L"AssociateFiles", 0); + } + } + + if (associateFiles < 0) { + auto hr = LoadAssociateFilesStateFromKey( + _engine, + includeLauncherAllUsers ? HKEY_LOCAL_MACHINE : HKEY_CURRENT_USER + ); + if (FAILED(hr)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); + } else if (hr == S_OK) { + associateFiles = 1; + } + _engine->SetVariableNumeric(L"AssociateFiles", associateFiles); } } } @@ -2614,7 +2672,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { /*Elevate when installing for all users*/ L"InstallAllUsers or " /*Elevate when installing the launcher for all users and it was not detected*/ - L"(Include_launcher and InstallLauncherAllUsers and not DetectedLauncher)" + L"(Include_launcher and InstallLauncherAllUsers and not BlockedLauncher)" L")", L"" }; @@ -3028,11 +3086,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; if (IsWindowsServer()) { - if (IsWindowsVersionOrGreater(6, 2, 0)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows Server 2012 or later"); + if (IsWindowsVersionOrGreater(10, 0, 0)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows Server 2016 or later"); return; + } else if (IsWindowsVersionOrGreater(6, 2, 0)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2012"); } else if (IsWindowsVersionOrGreater(6, 1, 1)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected Windows Server 2008 R2"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); } else if (IsWindowsVersionOrGreater(6, 1, 0)) { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); } else if (IsWindowsVersionOrGreater(6, 0, 0)) { @@ -3040,14 +3100,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } else { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2003 or earlier"); } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows Server 2012 or later is required to continue installation"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows Server 2016 or later is required to continue installation"); } else { if (IsWindows10OrGreater()) { BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows 10 or later"); return; } else if (IsWindows8Point1OrGreater()) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows 8.1"); - return; + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8.1"); } else if (IsWindows8OrGreater()) { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8"); } else if (IsWindows7OrGreater()) { @@ -3057,7 +3116,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } else { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows XP or earlier"); } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows 8.1 or later is required to continue installation"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows 10 or later is required to continue installation"); } LocGetString(_wixLoc, L"#(loc.FailureOldOS)", &pLocString); diff --git a/Tools/msi/bundle/bootstrap/pythonba.vcxproj b/Tools/msi/bundle/bootstrap/pythonba.vcxproj index bb383bfcaa240e..3970e857894ba5 100644 --- a/Tools/msi/bundle/bootstrap/pythonba.vcxproj +++ b/Tools/msi/bundle/bootstrap/pythonba.vcxproj @@ -24,7 +24,6 @@ v143 v142 v141 - v140 {7A09B132-B3EE-499B-A700-A4B2157FEA3D} PythonBA @@ -51,8 +50,6 @@ comctl32.lib;gdiplus.lib;msimg32.lib;shlwapi.lib;wininet.lib;dutil.lib;balutil.lib;version.lib;uxtheme.lib;%(AdditionalDependencies) $(WixInstallPath)sdk\vs2017\lib\x86 - $(WixInstallPath)sdk\vs2015\lib\x86 - $(WixInstallPath)sdk\vs2013\lib\x86 pythonba.def true diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs index 9b4f072152d5c0..abfeb88784890c 100644 --- a/Tools/msi/bundle/bundle.wxs +++ b/Tools/msi/bundle/bundle.wxs @@ -28,10 +28,11 @@ - + - + + @@ -91,10 +92,11 @@ + - + diff --git a/Tools/msi/bundle/packagegroups/launcher.wxs b/Tools/msi/bundle/packagegroups/launcher.wxs index a6922758f31f14..080598a0a486ef 100644 --- a/Tools/msi/bundle/packagegroups/launcher.wxs +++ b/Tools/msi/bundle/packagegroups/launcher.wxs @@ -11,7 +11,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> @@ -25,7 +25,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index adaf63c69d5ade..49ecb3429ad8f3 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -48,6 +48,12 @@ + + + + + + @@ -69,8 +75,14 @@ - - + + + + + + + + @@ -147,12 +159,6 @@ - - - - - - diff --git a/Tools/msi/pip/pip.wxs b/Tools/msi/pip/pip.wxs index 1d8083cad91a56..627c4710a9fdfa 100644 --- a/Tools/msi/pip/pip.wxs +++ b/Tools/msi/pip/pip.wxs @@ -25,8 +25,8 @@ - - + + (&DefaultFeature=3) AND NOT (!DefaultFeature=3) diff --git a/Tools/msi/purge.py b/Tools/msi/purge.py index e25219a6caf9d4..4a13d368d7c744 100644 --- a/Tools/msi/purge.py +++ b/Tools/msi/purge.py @@ -51,14 +51,6 @@ "test_pdb.msi", "tools.msi", "ucrt.msi", - "Windows6.0-KB2999226-x64.msu", - "Windows6.0-KB2999226-x86.msu", - "Windows6.1-KB2999226-x64.msu", - "Windows6.1-KB2999226-x86.msu", - "Windows8.1-KB2999226-x64.msu", - "Windows8.1-KB2999226-x86.msu", - "Windows8-RT-KB2999226-x64.msu", - "Windows8-RT-KB2999226-x86.msu", ] PATHS = [ "python-{}.exe".format(m.group(0)), diff --git a/Tools/msi/test/test_files.wxs b/Tools/msi/test/test_files.wxs index bb9b258692a62f..6862a5899db2ba 100644 --- a/Tools/msi/test/test_files.wxs +++ b/Tools/msi/test/test_files.wxs @@ -1,6 +1,6 @@ - + diff --git a/Tools/msi/wix.props b/Tools/msi/wix.props index d8ced317d0ce81..707c8aeacb648d 100644 --- a/Tools/msi/wix.props +++ b/Tools/msi/wix.props @@ -5,8 +5,6 @@ $(MSBuildThisFileDirectory)\Wix\ $(ExternalsDir)\windows-installer\wix-314\ - $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows Installer XML\3.10@InstallRoot) - $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows Installer XML\3.10@InstallRoot) $(WixInstallPath)\Wix.targets \ No newline at end of file diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index af1f0584bb5403..0dcf6ef844a048 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -1,15 +1,10 @@ #!/usr/bin/env python3 """Check proposed changes for common issues.""" -import re import sys -import shutil import os.path import subprocess import sysconfig -import reindent -import untabify - def get_python_source_dir(): src_dir = sysconfig.get_config_var('abs_srcdir') @@ -18,13 +13,6 @@ def get_python_source_dir(): return os.path.abspath(src_dir) -# Excluded directories which are copies of external libraries: -# don't check their coding style -EXCLUDE_DIRS = [ - os.path.join('Modules', '_decimal', 'libmpdec'), - os.path.join('Modules', 'expat'), - os.path.join('Modules', 'zlib'), - ] SRCDIR = get_python_source_dir() @@ -155,62 +143,7 @@ def changed_files(base_branch=None): else: sys.exit('need a git checkout to get modified files') - filenames2 = [] - for filename in filenames: - # Normalize the path to be able to match using .startswith() - filename = os.path.normpath(filename) - if any(filename.startswith(path) for path in EXCLUDE_DIRS): - # Exclude the file - continue - filenames2.append(filename) - - return filenames2 - - -def report_modified_files(file_paths): - count = len(file_paths) - if count == 0: - return n_files_str(count) - else: - lines = [f"{n_files_str(count)}:"] - for path in file_paths: - lines.append(f" {path}") - return "\n".join(lines) - - -#: Python files that have tabs by design: -_PYTHON_FILES_WITH_TABS = frozenset({ - 'Tools/c-analyzer/cpython/_parser.py', -}) - - -@status("Fixing Python file whitespace", info=report_modified_files) -def normalize_whitespace(file_paths): - """Make sure that the whitespace for .py files have been normalized.""" - reindent.makebackup = False # No need to create backups. - fixed = [ - path for path in file_paths - if ( - path.endswith('.py') - and path not in _PYTHON_FILES_WITH_TABS - and reindent.check(os.path.join(SRCDIR, path)) - ) - ] - return fixed - - -@status("Fixing C file whitespace", info=report_modified_files) -def normalize_c_whitespace(file_paths): - """Report if any C files """ - fixed = [] - for path in file_paths: - abspath = os.path.join(SRCDIR, path) - with open(abspath, 'r') as f: - if '\t' not in f.read(): - continue - untabify.process(abspath, 8, verbose=False) - fixed.append(path) - return fixed + return list(map(os.path.normpath, filenames)) @status("Docs modified", modal=True) @@ -250,40 +183,14 @@ def regenerated_pyconfig_h_in(file_paths): return "not needed" -def ci(pull_request): - if pull_request == 'false': - print('Not a pull request; skipping') - return - base_branch = get_base_branch() - file_paths = changed_files(base_branch) - python_files = [fn for fn in file_paths if fn.endswith('.py')] - c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] - fixed = [] - fixed.extend(normalize_whitespace(python_files)) - fixed.extend(normalize_c_whitespace(c_files)) - if not fixed: - print('No whitespace issues found') - else: - count = len(fixed) - print(f'Please fix the {n_files_str(count)} with whitespace issues') - print('(on Unix you can run `make patchcheck` to make the fixes)') - sys.exit(1) - - def main(): base_branch = get_base_branch() file_paths = changed_files(base_branch) - python_files = [fn for fn in file_paths if fn.endswith('.py')] - c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] - doc_files = [fn for fn in file_paths if fn.startswith('Doc') and - fn.endswith(('.rst', '.inc'))] + has_doc_files = any(fn for fn in file_paths if fn.startswith('Doc') and + fn.endswith(('.rst', '.inc'))) misc_files = {p for p in file_paths if p.startswith('Misc')} - # PEP 8 whitespace rules enforcement. - normalize_whitespace(python_files) - # C rules enforcement. - normalize_c_whitespace(c_files) # Docs updated. - docs_modified(doc_files) + docs_modified(has_doc_files) # Misc/ACKS changed. credit_given(misc_files) # Misc/NEWS changed. @@ -294,19 +201,14 @@ def main(): regenerated_pyconfig_h_in(file_paths) # Test suite run and passed. - if python_files or c_files: - end = " and check for refleaks?" if c_files else "?" - print() - print("Did you run the test suite" + end) + has_c_files = any(fn for fn in file_paths if fn.endswith(('.c', '.h'))) + has_python_files = any(fn for fn in file_paths if fn.endswith('.py')) + print() + if has_c_files: + print("Did you run the test suite and check for refleaks?") + elif has_python_files: + print("Did you run the test suite?") if __name__ == '__main__': - import argparse - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--ci', - help='Perform pass/fail checks') - args = parser.parse_args() - if args.ci: - ci(args.ci) - else: - main() + main() diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index b081240ffff017..1587d53d59472e 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -108,7 +108,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * clear_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) _PyPegen_clear_memo_statistics(); #endif Py_RETURN_NONE; @@ -117,7 +117,7 @@ clear_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) static PyObject * get_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) return _PyPegen_get_memo_statistics(); #else Py_RETURN_NONE; @@ -128,7 +128,7 @@ get_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) static PyObject * dump_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) PyObject *list = _PyPegen_get_memo_statistics(); if (list == NULL) { return NULL; diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py index 262c8a6db68f6e..0b0b4b291c2b0e 100755 --- a/Tools/peg_generator/pegen/__main__.py +++ b/Tools/peg_generator/pegen/__main__.py @@ -107,7 +107,10 @@ def generate_python_code( help="Suppress code emission for rule actions", ) -python_parser = subparsers.add_parser("python", help="Generate Python code") +python_parser = subparsers.add_parser( + "python", + help="Generate Python code, needs grammar definition with Python actions", +) python_parser.set_defaults(func=generate_python_code) python_parser.add_argument("grammar_filename", help="Grammar description") python_parser.add_argument( diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 7cdd5debe9a225..a4951d05e80ebd 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -3,7 +3,7 @@ import re from dataclasses import dataclass, field from enum import Enum -from typing import IO, Any, Dict, List, Optional, Set, Text, Tuple +from typing import IO, Any, Callable, Dict, List, Optional, Set, Text, Tuple from pegen import grammar from pegen.grammar import ( @@ -130,7 +130,7 @@ def __init__( self.gen = parser_generator self.exact_tokens = exact_tokens self.non_exact_tokens = non_exact_tokens - self.cache: Dict[Any, FunctionCall] = {} + self.cache: Dict[str, str] = {} self.cleanup_statements: List[str] = [] def keyword_helper(self, keyword: str) -> FunctionCall: @@ -206,21 +206,6 @@ def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall: comment=f"token='{val}'", ) - def visit_Rhs(self, node: Rhs) -> FunctionCall: - if node in self.cache: - return self.cache[node] - if node.can_be_inlined: - self.cache[node] = self.generate_call(node.alts[0].items[0]) - else: - name = self.gen.artifical_rule_from_rhs(node) - self.cache[node] = FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - comment=f"{node}", - ) - return self.cache[node] - def visit_NamedItem(self, node: NamedItem) -> FunctionCall: call = self.generate_call(node.item) if node.name: @@ -253,7 +238,7 @@ def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall: else: return FunctionCall( function=f"_PyPegen_lookahead", - arguments=[positive, call.function, *call.arguments], + arguments=[positive, f"(void *(*)(Parser *)) {call.function}", *call.arguments], return_type="int", ) @@ -302,44 +287,62 @@ def visit_Opt(self, node: Opt) -> FunctionCall: comment=f"{node}", ) - def visit_Repeat0(self, node: Repeat0) -> FunctionCall: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, False) - self.cache[node] = FunctionCall( + def _generate_artificial_rule_call( + self, + node: Any, + prefix: str, + rule_generation_func: Callable[[], str], + return_type: Optional[str] = None, + ) -> FunctionCall: + node_str = f"{node}" + key = f"{prefix}_{node_str}" + if key in self.cache: + name = self.cache[key] + else: + name = rule_generation_func() + self.cache[key] = name + + return FunctionCall( assigned_variable=f"{name}_var", function=f"{name}_rule", arguments=["p"], - return_type="asdl_seq *", - comment=f"{node}", + return_type=return_type, + comment=node_str, + ) + + def visit_Rhs(self, node: Rhs) -> FunctionCall: + if node.can_be_inlined: + return self.generate_call(node.alts[0].items[0]) + + return self._generate_artificial_rule_call( + node, + "rhs", + lambda: self.gen.artificial_rule_from_rhs(node), + ) + + def visit_Repeat0(self, node: Repeat0) -> FunctionCall: + return self._generate_artificial_rule_call( + node, + "repeat0", + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), + "asdl_seq *", ) - return self.cache[node] def visit_Repeat1(self, node: Repeat1) -> FunctionCall: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, True) - self.cache[node] = FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - return_type="asdl_seq *", - comment=f"{node}", + return self._generate_artificial_rule_call( + node, + "repeat1", + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), + "asdl_seq *", ) - return self.cache[node] def visit_Gather(self, node: Gather) -> FunctionCall: - if node in self.cache: - return self.cache[node] - name = self.gen.artifical_rule_from_gather(node) - self.cache[node] = FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - return_type="asdl_seq *", - comment=f"{node}", + return self._generate_artificial_rule_call( + node, + "gather", + lambda: self.gen.artificial_rule_from_gather(node), + "asdl_seq *", ) - return self.cache[node] def visit_Group(self, node: Group) -> FunctionCall: return self.generate_call(node.rhs) @@ -645,7 +648,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.print("}") self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);") self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);") - self.print("for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") + self.print("for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") self.print("PyMem_Free(_children);") if memoize and node.name: self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py index bf31fe53263645..2e3a607f7209b0 100644 --- a/Tools/peg_generator/pegen/grammar_parser.py +++ b/Tools/peg_generator/pegen/grammar_parser.py @@ -575,7 +575,7 @@ def target_atoms(self) -> Optional[str]: @memoize def target_atom(self) -> Optional[str]: - # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | "?" | ":" | !"}" !"]" OP + # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | FSTRING_START | FSTRING_MIDDLE | FSTRING_END | "?" | ":" | !"}" !"]" OP mark = self._mark() cut = False if ( @@ -625,6 +625,21 @@ def target_atom(self) -> Optional[str]: ): return string . string self._reset(mark) + if ( + (fstring_start := self.fstring_start()) + ): + return fstring_start . string + self._reset(mark) + if ( + (fstring_middle := self.fstring_middle()) + ): + return fstring_middle . string + self._reset(mark) + if ( + (fstring_end := self.fstring_end()) + ): + return fstring_end . string + self._reset(mark) if ( (literal := self.expect("?")) ): diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram index f22c334ca371b2..f484c4781823bc 100644 --- a/Tools/peg_generator/pegen/metagrammar.gram +++ b/Tools/peg_generator/pegen/metagrammar.gram @@ -126,6 +126,9 @@ target_atom[str]: | NAME { name.string } | NUMBER { number.string } | STRING { string.string } + | FSTRING_START { fstring_start.string } + | FSTRING_MIDDLE { fstring_middle.string } + | FSTRING_END { fstring_end.string } | "?" { "?" } | ":" { ":" } | !"}" !"]" OP { op.string } diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index ed0aec9db2443f..692eb9ed2417d7 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -205,6 +205,36 @@ def string(self) -> Optional[tokenize.TokenInfo]: return self._tokenizer.getnext() return None + @memoize + def fstring_start(self) -> Optional[tokenize.TokenInfo]: + FSTRING_START = getattr(token, "FSTRING_START") + if not FSTRING_START: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_START: + return self._tokenizer.getnext() + return None + + @memoize + def fstring_middle(self) -> Optional[tokenize.TokenInfo]: + FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE") + if not FSTRING_MIDDLE: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_MIDDLE: + return self._tokenizer.getnext() + return None + + @memoize + def fstring_end(self) -> Optional[tokenize.TokenInfo]: + FSTRING_END = getattr(token, "FSTRING_END") + if not FSTRING_END: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_END: + return self._tokenizer.getnext() + return None + @memoize def op(self) -> Optional[tokenize.TokenInfo]: tok = self._tokenizer.peek() diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index ad28f6c27dcb37..b42b12c8aa0dee 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -41,7 +41,7 @@ class RuleCollectorVisitor(GrammarVisitor): - """Visitor that invokes a provieded callmaker visitor with just the NamedItem nodes""" + """Visitor that invokes a provided callmaker visitor with just the NamedItem nodes""" def __init__(self, rules: Dict[str, Rule], callmakervisitor: GrammarVisitor) -> None: self.rulses = rules @@ -167,7 +167,7 @@ def keyword_type(self) -> int: self.keyword_counter += 1 return self.keyword_counter - def artifical_rule_from_rhs(self, rhs: Rhs) -> str: + def artificial_rule_from_rhs(self, rhs: Rhs) -> str: self.counter += 1 name = f"_tmp_{self.counter}" # TODO: Pick a nicer name. self.all_rules[name] = Rule(name, None, rhs) @@ -183,9 +183,7 @@ def artificial_rule_from_repeat(self, node: Plain, is_repeat1: bool) -> str: self.all_rules[name] = Rule(name, None, Rhs([Alt([NamedItem(None, node)])])) return name - def artifical_rule_from_gather(self, node: Gather) -> str: - self.counter += 1 - name = f"_gather_{self.counter}" + def artificial_rule_from_gather(self, node: Gather) -> str: self.counter += 1 extra_function_name = f"_loop0_{self.counter}" extra_function_alt = Alt( @@ -197,6 +195,8 @@ def artifical_rule_from_gather(self, node: Gather) -> str: None, Rhs([extra_function_alt]), ) + self.counter += 1 + name = f"_gather_{self.counter}" alt = Alt( [NamedItem("elem", node.node), NamedItem("seq", NameLeaf(extra_function_name))], ) diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 4a2883eb4ee202..4bb26480ebc0af 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -1,6 +1,6 @@ import os.path import token -from typing import IO, Any, Dict, Optional, Sequence, Set, Text, Tuple +from typing import IO, Any, Callable, Dict, Optional, Sequence, Set, Text, Tuple from pegen import grammar from pegen.grammar import ( @@ -93,13 +93,14 @@ def visit_Forced(self, node: Forced) -> bool: class PythonCallMakerVisitor(GrammarVisitor): def __init__(self, parser_generator: ParserGenerator): self.gen = parser_generator - self.cache: Dict[Any, Any] = {} + self.cache: Dict[str, Tuple[str, str]] = {} def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: name = node.value if name == "SOFT_KEYWORD": return "soft_keyword", "self.soft_keyword()" - if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT"): + if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT", + "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"): name = name.lower() return name, f"self.{name}()" if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"): @@ -110,16 +111,6 @@ def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: return "literal", f"self.expect({node.value})" - def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str]: - if node in self.cache: - return self.cache[node] - if len(node.alts) == 1 and len(node.alts[0].items) == 1: - self.cache[node] = self.visit(node.alts[0].items[0]) - else: - name = self.gen.artifical_rule_from_rhs(node) - self.cache[node] = name, f"self.{name}()" - return self.cache[node] - def visit_NamedItem(self, node: NamedItem) -> Tuple[Optional[str], str]: name, call = self.visit(node.item) if node.name: @@ -151,26 +142,57 @@ def visit_Opt(self, node: Opt) -> Tuple[str, str]: else: return "opt", f"{call}," + def _generate_artificial_rule_call( + self, + node: Any, + prefix: str, + call_by_name_func: Callable[[str], str], + rule_generation_func: Callable[[], str], + ) -> Tuple[str, str]: + node_str = f"{node}" + key = f"{prefix}_{node_str}" + if key in self.cache: + return self.cache[key] + + name = rule_generation_func() + call = call_by_name_func(name) + self.cache[key] = name, call + return self.cache[key] + + def visit_Rhs(self, node: Rhs) -> Tuple[str, str]: + if len(node.alts) == 1 and len(node.alts[0].items) == 1: + return self.visit(node.alts[0].items[0]) + + return self._generate_artificial_rule_call( + node, + "rhs", + lambda name: f"self.{name}()", + lambda: self.gen.artificial_rule_from_rhs(node), + ) + def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, False) - self.cache[node] = name, f"self.{name}()," # Also a trailing comma! - return self.cache[node] + return self._generate_artificial_rule_call( + node, + "repeat0", + lambda name: f"self.{name}(),", # Also a trailing comma! + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), + ) def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, True) - self.cache[node] = name, f"self.{name}()" # But no trailing comma here! - return self.cache[node] + return self._generate_artificial_rule_call( + node, + "repeat1", + lambda name: f"self.{name}()", # But no trailing comma here! + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), + ) def visit_Gather(self, node: Gather) -> Tuple[str, str]: - if node in self.cache: - return self.cache[node] - name = self.gen.artifical_rule_from_gather(node) - self.cache[node] = name, f"self.{name}()" # No trailing comma here either! - return self.cache[node] + return self._generate_artificial_rule_call( + node, + "gather", + lambda name: f"self.{name}()", # No trailing comma here either! + lambda: self.gen.artificial_rule_from_gather(node), + ) def visit_Group(self, node: Group) -> Tuple[Optional[str], str]: return self.visit(node.rhs) diff --git a/Tools/peg_generator/pegen/sccutils.py b/Tools/peg_generator/pegen/sccutils.py index 1f0586bb2f7d6d..da4c9331625dd9 100644 --- a/Tools/peg_generator/pegen/sccutils.py +++ b/Tools/peg_generator/pegen/sccutils.py @@ -18,7 +18,7 @@ def strongly_connected_components( exactly once; vertices not part of a SCC are returned as singleton sets. - From http://code.activestate.com/recipes/578507/. + From https://code.activestate.com/recipes/578507-strongly-connected-components-of-a-directed-graph/. """ identified: Set[str] = set() stack: List[str] = [] @@ -81,7 +81,7 @@ def topsort( {B, C} {A} - From http://code.activestate.com/recipes/577413/. + From https://code.activestate.com/recipes/577413-topological-sort/history/1/. """ # TODO: Use a faster algorithm? for k, v in data.items(): diff --git a/Tools/peg_generator/pegen/validator.py b/Tools/peg_generator/pegen/validator.py index c48a01eedf5d5c..4699d5712d9522 100644 --- a/Tools/peg_generator/pegen/validator.py +++ b/Tools/peg_generator/pegen/validator.py @@ -34,6 +34,18 @@ def check_intersection(self, first_alt: Alt, second_alt: Alt) -> None: ) +class RaiseRuleValidator(GrammarValidator): + def visit_Alt(self, node: Alt) -> None: + if self.rulename and self.rulename.startswith('invalid'): + # raising is allowed in invalid rules + return + if node.action and 'RAISE_SYNTAX_ERROR' in node.action: + raise ValidationError( + f"In {self.rulename!r} there is an alternative that contains " + f"RAISE_SYNTAX_ERROR; this is only allowed in invalid_ rules" + ) + + def validate_grammar(the_grammar: grammar.Grammar) -> None: for validator_cls in GrammarValidator.__subclasses__(): validator = validator_cls(the_grammar) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index c0a63b40ff4155..57f0b982b00f5d 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.8.0 +mypy==1.12 # needed for peg_generator: -types-psutil==5.9.5.20240106 -types-setuptools==69.0.0.20240125 +types-psutil==6.0.0.20240901 +types-setuptools==74.0.0.20240831 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 064731a236ee86..66898885c0a412 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.97.4 +hypothesis==6.111.2 diff --git a/Tools/scripts/sortperf.py b/Tools/scripts/sortperf.py new file mode 100644 index 00000000000000..1978a6c83dbe2b --- /dev/null +++ b/Tools/scripts/sortperf.py @@ -0,0 +1,197 @@ +""" +List sort performance test. + +To install `pyperf` you would need to: + + python3 -m pip install pyperf + +To run: + + python3 Tools/scripts/sortperf + +Options: + + * `benchmark` name to run + * `--rnd-seed` to set random seed + * `--size` to set the sorted list size + +Based on https://github.com/python/cpython/blob/963904335e579bfe39101adf3fd6a0cf705975ff/Lib/test/sortperf.py +""" + +from __future__ import annotations + +import argparse +import time +import random + + +# =============== +# Data generation +# =============== + +def _random_data(size: int, rand: random.Random) -> list[float]: + result = [rand.random() for _ in range(size)] + # Shuffle it a bit... + for i in range(10): + i = rand.randrange(size) + temp = result[:i] + del result[:i] + temp.reverse() + result.extend(temp) + del temp + assert len(result) == size + return result + + +def list_sort(size: int, rand: random.Random) -> list[float]: + return _random_data(size, rand) + + +def list_sort_descending(size: int, rand: random.Random) -> list[float]: + return list(reversed(list_sort_ascending(size, rand))) + + +def list_sort_ascending(size: int, rand: random.Random) -> list[float]: + return sorted(_random_data(size, rand)) + + +def list_sort_ascending_exchanged(size: int, rand: random.Random) -> list[float]: + result = list_sort_ascending(size, rand) + # Do 3 random exchanges. + for _ in range(3): + i1 = rand.randrange(size) + i2 = rand.randrange(size) + result[i1], result[i2] = result[i2], result[i1] + return result + + +def list_sort_ascending_random(size: int, rand: random.Random) -> list[float]: + assert size >= 10, "This benchmark requires size to be >= 10" + result = list_sort_ascending(size, rand) + # Replace the last 10 with random floats. + result[-10:] = [rand.random() for _ in range(10)] + return result + + +def list_sort_ascending_one_percent(size: int, rand: random.Random) -> list[float]: + result = list_sort_ascending(size, rand) + # Replace 1% of the elements at random. + for _ in range(size // 100): + result[rand.randrange(size)] = rand.random() + return result + + +def list_sort_duplicates(size: int, rand: random.Random) -> list[float]: + assert size >= 4 + result = list_sort_ascending(4, rand) + # Arrange for lots of duplicates. + result = result * (size // 4) + # Force the elements to be distinct objects, else timings can be + # artificially low. + return list(map(abs, result)) + + +def list_sort_equal(size: int, rand: random.Random) -> list[float]: + # All equal. Again, force the elements to be distinct objects. + return list(map(abs, [-0.519012] * size)) + + +def list_sort_worst_case(size: int, rand: random.Random) -> list[float]: + # This one looks like [3, 2, 1, 0, 0, 1, 2, 3]. It was a bad case + # for an older implementation of quicksort, which used the median + # of the first, last and middle elements as the pivot. + half = size // 2 + result = list(range(half - 1, -1, -1)) + result.extend(range(half)) + # Force to float, so that the timings are comparable. This is + # significantly faster if we leave them as ints. + return list(map(float, result)) + + +# ========= +# Benchmark +# ========= + +class Benchmark: + def __init__(self, name: str, size: int, seed: int) -> None: + self._name = name + self._size = size + self._seed = seed + self._random = random.Random(self._seed) + + def run(self, loops: int) -> float: + all_data = self._prepare_data(loops) + start = time.perf_counter() + + for data in all_data: + data.sort() # Benching this method! + + return time.perf_counter() - start + + def _prepare_data(self, loops: int) -> list[float]: + bench = BENCHMARKS[self._name] + data = bench(self._size, self._random) + return [data.copy() for _ in range(loops)] + + +def add_cmdline_args(cmd: list[str], args) -> None: + if args.benchmark: + cmd.append(args.benchmark) + cmd.append(f"--size={args.size}") + cmd.append(f"--rng-seed={args.rng_seed}") + + +def add_parser_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "benchmark", + choices=BENCHMARKS, + nargs="?", + help="Can be any of: {0}".format(", ".join(BENCHMARKS)), + ) + parser.add_argument( + "--size", + type=int, + default=DEFAULT_SIZE, + help=f"Size of the lists to sort (default: {DEFAULT_SIZE})", + ) + parser.add_argument( + "--rng-seed", + type=int, + default=DEFAULT_RANDOM_SEED, + help=f"Random number generator seed (default: {DEFAULT_RANDOM_SEED})", + ) + + +DEFAULT_SIZE = 1 << 14 +DEFAULT_RANDOM_SEED = 0 +BENCHMARKS = { + "list_sort": list_sort, + "list_sort_descending": list_sort_descending, + "list_sort_ascending": list_sort_ascending, + "list_sort_ascending_exchanged": list_sort_ascending_exchanged, + "list_sort_ascending_random": list_sort_ascending_random, + "list_sort_ascending_one_percent": list_sort_ascending_one_percent, + "list_sort_duplicates": list_sort_duplicates, + "list_sort_equal": list_sort_equal, + "list_sort_worst_case": list_sort_worst_case, +} + +if __name__ == "__main__": + # This needs `pyperf` 3rd party library: + import pyperf + + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) + add_parser_args(runner.argparser) + args = runner.parse_args() + + runner.metadata["description"] = "Test `list.sort()` with different data" + runner.metadata["list_sort_size"] = args.size + runner.metadata["list_sort_random_seed"] = args.rng_seed + + if args.benchmark: + benchmarks = (args.benchmark,) + else: + benchmarks = sorted(BENCHMARKS) + for bench in benchmarks: + benchmark = Benchmark(bench, args.size, args.rng_seed) + runner.bench_time_func(bench, benchmark.run) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 9b7e7b999ea7c7..5793e5c649d6b3 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -11,6 +11,7 @@ import argparse import collections from collections.abc import KeysView +from dataclasses import dataclass from datetime import date import enum import functools @@ -21,6 +22,7 @@ from pathlib import Path import re import sys +import textwrap from typing import Any, Callable, TextIO, TypeAlias @@ -100,6 +102,10 @@ def load_raw_data(input: Path) -> RawData: file=sys.stderr, ) continue + # Hack to handle older data files where some uops + # are missing an underscore prefix in their name + if key.startswith("uops[") and key[5:6] != "_": + key = "uops[_" + key[5:] stats[key.strip()] += int(value) stats["__nfiles__"] += 1 @@ -108,13 +114,71 @@ def load_raw_data(input: Path) -> RawData: return data else: - raise ValueError(f"{input:r} is not a file or directory path") + raise ValueError(f"{input} is not a file or directory path") def save_raw_data(data: RawData, json_output: TextIO): json.dump(data, json_output) +@dataclass(frozen=True) +class Doc: + text: str + doc: str + + def markdown(self) -> str: + return textwrap.dedent( + f""" + {self.text} +
    + + + {self.doc} +
    + """ + ) + + +class Count(int): + def markdown(self) -> str: + return format(self, ",d") + + +@dataclass(frozen=True) +class Ratio: + num: int + den: int | None = None + percentage: bool = True + + def __float__(self): + if self.den == 0: + return 0.0 + elif self.den is None: + return self.num + else: + return self.num / self.den + + def markdown(self) -> str: + if self.den is None: + return "" + elif self.den == 0: + if self.num != 0: + return f"{self.num:,} / 0 !!" + return "" + elif self.percentage: + return f"{self.num / self.den:,.01%}" + else: + return f"{self.num / self.den:,.02f}" + + +class DiffRatio(Ratio): + def __init__(self, base: int | str, head: int | str): + if isinstance(base, str) or isinstance(head, str): + super().__init__(0, 0) + else: + super().__init__(head - base, base) + + class OpcodeStats: """ Manages the data related to specific set of opcodes, e.g. tier1 (with prefix @@ -330,16 +394,22 @@ def get_call_stats(self) -> dict[str, int]: return result def get_object_stats(self) -> dict[str, tuple[int, int]]: - total_materializations = self._data.get("Object new values", 0) + total_materializations = self._data.get("Object inline values", 0) total_allocations = self._data.get("Object allocations", 0) + self._data.get( "Object allocations from freelist", 0 ) - total_increfs = self._data.get( - "Object interpreter increfs", 0 - ) + self._data.get("Object increfs", 0) - total_decrefs = self._data.get( - "Object interpreter decrefs", 0 - ) + self._data.get("Object decrefs", 0) + total_increfs = ( + self._data.get("Object interpreter mortal increfs", 0) + + self._data.get("Object mortal increfs", 0) + + self._data.get("Object interpreter immortal increfs", 0) + + self._data.get("Object immortal increfs", 0) + ) + total_decrefs = ( + self._data.get("Object interpreter mortal decrefs", 0) + + self._data.get("Object mortal decrefs", 0) + + self._data.get("Object interpreter immortal decrefs", 0) + + self._data.get("Object immortal decrefs", 0) + ) result = {} for key, value in self._data.items(): @@ -387,19 +457,92 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: inner_loop = self._data["Optimization inner loop"] recursive_call = self._data["Optimization recursive call"] low_confidence = self._data["Optimization low confidence"] + executors_invalidated = self._data["Executors invalidated"] return { - "Optimization attempts": (attempts, None), - "Traces created": (created, attempts), - "Trace stack overflow": (trace_stack_overflow, attempts), - "Trace stack underflow": (trace_stack_underflow, attempts), - "Trace too long": (trace_too_long, attempts), - "Trace too short": (trace_too_short, attempts), - "Inner loop found": (inner_loop, attempts), - "Recursive call": (recursive_call, attempts), - "Low confidence": (low_confidence, attempts), - "Traces executed": (executed, None), - "Uops executed": (uops, executed), + Doc( + "Optimization attempts", + "The number of times a potential trace is identified. Specifically, this " + "occurs in the JUMP BACKWARD instruction when the counter reaches a " + "threshold.", + ): (attempts, None), + Doc( + "Traces created", "The number of traces that were successfully created." + ): (created, attempts), + Doc( + "Trace stack overflow", + "A trace is truncated because it would require more than 5 stack frames.", + ): (trace_stack_overflow, attempts), + Doc( + "Trace stack underflow", + "A potential trace is abandoned because it pops more frames than it pushes.", + ): (trace_stack_underflow, attempts), + Doc( + "Trace too long", + "A trace is truncated because it is longer than the instruction buffer.", + ): (trace_too_long, attempts), + Doc( + "Trace too short", + "A potential trace is abandoced because it it too short.", + ): (trace_too_short, attempts), + Doc( + "Inner loop found", "A trace is truncated because it has an inner loop" + ): (inner_loop, attempts), + Doc( + "Recursive call", + "A trace is truncated because it has a recursive call.", + ): (recursive_call, attempts), + Doc( + "Low confidence", + "A trace is abandoned because the likelihood of the jump to top being taken " + "is too low.", + ): (low_confidence, attempts), + Doc( + "Executors invalidated", + "The number of executors that were invalidated due to watched " + "dictionary changes.", + ): (executors_invalidated, created), + Doc("Traces executed", "The number of traces that were executed"): ( + executed, + None, + ), + Doc( + "Uops executed", + "The total number of uops (micro-operations) that were executed", + ): ( + uops, + executed, + ), + } + + def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]: + attempts = self._data["Optimization optimizer attempts"] + successes = self._data["Optimization optimizer successes"] + no_memory = self._data["Optimization optimizer failure no memory"] + builtins_changed = self._data["Optimizer remove globals builtins changed"] + incorrect_keys = self._data["Optimizer remove globals incorrect keys"] + + return { + Doc( + "Optimizer attempts", + "The number of times the trace optimizer (_Py_uop_analyze_and_optimize) was run.", + ): (attempts, None), + Doc( + "Optimizer successes", + "The number of traces that were successfully optimized.", + ): (successes, attempts), + Doc( + "Optimizer no memory", + "The number of optimizations that failed due to no memory.", + ): (no_memory, attempts), + Doc( + "Remove globals builtins changed", + "The builtins changed during optimization", + ): (builtins_changed, attempts), + Doc( + "Remove globals incorrect keys", + "The keys in the globals dictionary aren't what was expected", + ): (incorrect_keys, attempts), } def get_histogram(self, prefix: str) -> list[tuple[int, int]]: @@ -415,52 +558,12 @@ def get_histogram(self, prefix: str) -> list[tuple[int, int]]: def get_rare_events(self) -> list[tuple[str, int]]: prefix = "Rare event " return [ - (key[len(prefix) + 1:-1], val) + (key[len(prefix) + 1 : -1].replace("_", " "), val) for key, val in self._data.items() if key.startswith(prefix) ] -class Count(int): - def markdown(self) -> str: - return format(self, ",d") - - -class Ratio: - def __init__(self, num: int, den: int | None, percentage: bool = True): - self.num = num - self.den = den - self.percentage = percentage - - def __float__(self): - if self.den == 0: - return 0.0 - elif self.den is None: - return self.num - else: - return self.num / self.den - - def markdown(self) -> str: - if self.den is None: - return "" - elif self.den == 0: - if self.num != 0: - return f"{self.num:,} / 0 !!" - return "" - elif self.percentage: - return f"{self.num / self.den:,.01%}" - else: - return f"{self.num / self.den:,.02f}" - - -class DiffRatio(Ratio): - def __init__(self, base: int | str, head: int | str): - if isinstance(base, str) or isinstance(head, str): - super().__init__(0, 0) - else: - super().__init__(head - base, base) - - class JoinMode(enum.Enum): # Join using the first column as a key SIMPLE = 0 @@ -568,13 +671,16 @@ def __init__( title: str = "", summary: str = "", part_iter=None, + *, comparative: bool = True, + doc: str = "", ): self.title = title if not summary: self.summary = title.lower() else: self.summary = summary + self.doc = textwrap.dedent(doc) if part_iter is None: part_iter = [] if isinstance(part_iter, list): @@ -620,7 +726,7 @@ def calc(stats: Stats) -> Rows: def execution_count_section() -> Section: return Section( "Execution counts", - "execution counts for all instructions", + "Execution counts for Tier 1 instructions.", [ Table( ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), @@ -628,12 +734,17 @@ def execution_count_section() -> Section: join_mode=JoinMode.CHANGE_ONE_COLUMN, ) ], + doc=""" + The "miss ratio" column shows the percentage of times the instruction + executed that it deoptimized. When this happens, the base unspecialized + instruction is not counted. + """, ) -def pair_count_section() -> Section: +def pair_count_section(prefix: str, title=None) -> Section: def calc_pair_count_table(stats: Stats) -> Rows: - opcode_stats = stats.get_opcode_stats("opcode") + opcode_stats = stats.get_opcode_stats(prefix) pair_counts = opcode_stats.get_pair_counts() total = opcode_stats.get_total_execution_count() @@ -655,7 +766,7 @@ def calc_pair_count_table(stats: Stats) -> Rows: return Section( "Pair counts", - "Pair counts for top 100 pairs", + f"Pair counts for top 100 {title if title else prefix} pairs", [ Table( ("Pair", "Count:", "Self:", "Cumulative:"), @@ -663,6 +774,10 @@ def calc_pair_count_table(stats: Stats) -> Rows: ) ], comparative=False, + doc=""" + Pairs of specialized operations that deoptimize and are then followed by + the corresponding unspecialized instruction are not counted as pairs. + """, ) @@ -705,22 +820,33 @@ def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = Non return Section( "Predecessor/Successor Pairs", - "Top 5 predecessors and successors of each opcode", + "Top 5 predecessors and successors of each Tier 1 opcode.", iter_pre_succ_pairs_tables, comparative=False, + doc=""" + This does not include the unspecialized instructions that occur after a + specialized instruction deoptimizes. + """, ) def specialization_section() -> Section: def calc_specialization_table(opcode: str) -> RowCalculator: def calc(stats: Stats) -> Rows: + DOCS = { + "deferred": 'Lists the number of "deferred" (i.e. not specialized) instructions executed.', + "hit": "Specialized instructions that complete.", + "miss": "Specialized instructions that deopt.", + "deopt": "Specialized instructions that deopt.", + } + opcode_stats = stats.get_opcode_stats("opcode") total = opcode_stats.get_specialization_total(opcode) specialization_counts = opcode_stats.get_specialization_counts(opcode) return [ ( - f"{label:>12}", + Doc(label, DOCS[label]), Count(count), Ratio(count, total), ) @@ -790,7 +916,7 @@ def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = Non JoinMode.CHANGE, ), Table( - ("", "Count:", "Ratio:"), + ("Success", "Count:", "Ratio:"), calc_specialization_success_failure_table(opcode), JoinMode.CHANGE, ), @@ -804,7 +930,7 @@ def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = Non return Section( "Specialization stats", - "specialization stats by family", + "Specialization stats by family", iter_specialization_tables, ) @@ -822,19 +948,35 @@ def calc_specialization_effectiveness_table(stats: Stats) -> Rows: ) = opcode_stats.get_specialized_total_counts() return [ - ("Basic", Count(basic), Ratio(basic, total)), ( - "Not specialized", + Doc( + "Basic", + "Instructions that are not and cannot be specialized, e.g. `LOAD_FAST`.", + ), + Count(basic), + Ratio(basic, total), + ), + ( + Doc( + "Not specialized", + "Instructions that could be specialized but aren't, e.g. `LOAD_ATTR`, `BINARY_SLICE`.", + ), Count(not_specialized), Ratio(not_specialized, total), ), ( - "Specialized hits", + Doc( + "Specialized hits", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that complete.", + ), Count(specialized_hits), Ratio(specialized_hits, total), ), ( - "Specialized misses", + Doc( + "Specialized misses", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that deopt.", + ), Count(specialized_misses), Ratio(specialized_misses, total), ), @@ -879,7 +1021,7 @@ def calc_misses_by_table(stats: Stats) -> Rows: ), Section( "Deferred by instruction", - "", + "Breakdown of deferred (not specialized) instruction counts by family", [ Table( ("Name", "Count:", "Ratio:"), @@ -890,7 +1032,7 @@ def calc_misses_by_table(stats: Stats) -> Rows: ), Section( "Misses by instruction", - "", + "Breakdown of misses (specialized deopts) instruction counts by family", [ Table( ("Name", "Count:", "Ratio:"), @@ -900,6 +1042,10 @@ def calc_misses_by_table(stats: Stats) -> Rows: ], ), ], + doc=""" + All entries are execution counts. Should add up to the total number of + Tier 1 instructions executed. + """, ) @@ -922,6 +1068,13 @@ def calc_call_stats_table(stats: Stats) -> Rows: JoinMode.CHANGE, ) ], + doc=""" + This shows what fraction of calls to Python functions are inlined (i.e. + not having a call at the C level) and for those that are not, where the + call comes from. The various categories overlap. + + Also includes the count of frame objects created. + """, ) @@ -935,7 +1088,7 @@ def calc_object_stats_table(stats: Stats) -> Rows: return Section( "Object stats", - "allocations, frees and dict materializatons", + "Allocations, frees and dict materializatons", [ Table( ("", "Count:", "Ratio:"), @@ -943,6 +1096,15 @@ def calc_object_stats_table(stats: Stats) -> Rows: JoinMode.CHANGE, ) ], + doc=""" + Below, "allocations" means "allocations that are not from a freelist". + Total allocations = "Allocations from freelist" + "Allocations". + + "Inline values" is the number of values arrays inlined into objects. + + The cache hit/miss numbers are for the MRO cache, split into dunder and + other names. + """, ) @@ -969,6 +1131,9 @@ def calc_gc_stats(stats: Stats) -> Rows: calc_gc_stats, ) ], + doc=""" + Collected/visits gives some measure of efficiency. + """, ) @@ -985,6 +1150,14 @@ def calc_optimization_table(stats: Stats) -> Rows: for label, (value, den) in optimization_stats.items() ] + def calc_optimizer_table(stats: Stats) -> Rows: + optimizer_stats = stats.get_optimizer_stats() + + return [ + (label, Count(value), Ratio(value, den)) + for label, (value, den) in optimizer_stats.items() + ] + def calc_histogram_table(key: str, den: str) -> RowCalculator: def calc(stats: Stats) -> Rows: histogram = stats.get_histogram(key) @@ -1019,6 +1192,17 @@ def calc_unsupported_opcodes_table(stats: Stats) -> Rows: reverse=True, ) + def calc_error_in_opcodes_table(stats: Stats) -> Rows: + error_in_opcodes = stats.get_opcode_stats("error_in_opcode") + return sorted( + [ + (opcode, Count(count)) + for opcode, count in error_in_opcodes.get_opcode_counts().items() + ], + key=itemgetter(1), + reverse=True, + ) + def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None): if not base_stats.get_optimization_stats() or ( head_stats is not None and not head_stats.get_optimization_stats() @@ -1026,6 +1210,7 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) return yield Table(("", "Count:", "Ratio:"), calc_optimization_table, JoinMode.CHANGE) + yield Table(("", "Count:", "Ratio:"), calc_optimizer_table, JoinMode.CHANGE) for name, den in [ ("Trace length", "Optimization traces created"), ("Optimized trace length", "Optimization traces created"), @@ -1053,6 +1238,7 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) ) ], ) + yield pair_count_section(prefix="uop", title="Non-JIT uop") yield Section( "Unsupported opcodes", "", @@ -1064,6 +1250,11 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) ) ], ) + yield Section( + "Optimizer errored out with opcode", + "Optimization stopped after encountering this opcode", + [Table(("Opcode", "Count:"), calc_error_in_opcodes_table, JoinMode.CHANGE)], + ) return Section( "Optimization (Tier 2) stats", @@ -1074,7 +1265,19 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) def rare_event_section() -> Section: def calc_rare_event_table(stats: Stats) -> Table: - return [(x, Count(y)) for x, y in stats.get_rare_events()] + DOCS = { + "set class": "Setting an object's class, `obj.__class__ = ...`", + "set bases": "Setting the bases of a class, `cls.__bases__ = ...`", + "set eval frame func": ( + "Setting the PEP 523 frame eval function " + "`_PyInterpreterState_SetFrameEvalFunc()`" + ), + "builtin dict": "Modifying the builtins, `__builtins__.__dict__[var] = ...`", + "func modification": "Modifying a function, e.g. `func.__defaults__ = ...`, etc.", + "watched dict modification": "A watched dict has been modified", + "watched globals modification": "A watched `globals()` dict has been modified", + } + return [(Doc(x, DOCS[x]), Count(y)) for x, y in stats.get_rare_events()] return Section( "Rare events", @@ -1096,7 +1299,7 @@ def calc_rows(stats: Stats) -> Rows: LAYOUT = [ execution_count_section(), - pair_count_section(), + pair_count_section("opcode"), pre_succ_pairs_section(), specialization_section(), specialization_effectiveness_section(), @@ -1134,6 +1337,9 @@ def to_markdown(x): print("
    ", file=out) print("", obj.summary, "", file=out) print(file=out) + if obj.doc: + print(obj.doc, file=out) + if head_stats is not None and obj.comparative is False: print("Not included in comparative output.\n") else: @@ -1149,24 +1355,36 @@ def to_markdown(x): if len(rows) == 0: return - width = len(header) - header_line = "|" - under_line = "|" + alignments = [] for item in header: - under = "---" + if item.endswith(":"): + alignments.append("right") + else: + alignments.append("left") + + print("", file=out) + print("", file=out) + print("", file=out) + for item, align in zip(header, alignments): if item.endswith(":"): item = item[:-1] - under += ":" - header_line += item + " | " - under_line += under + "|" - print(header_line, file=out) - print(under_line, file=out) + print(f'', file=out) + print("", file=out) + print("", file=out) + + print("", file=out) for row in rows: - if len(row) != width: + if len(row) != len(header): raise ValueError( "Wrong number of elements in row '" + str(row) + "'" ) - print("|", " | ".join(to_markdown(i) for i in row), "|", file=out) + print("", file=out) + for col, align in zip(row, alignments): + print(f'', file=out) + print("", file=out) + print("", file=out) + + print("
    {item}
    {to_markdown(col)}
    ", file=out) print(file=out) case list(): diff --git a/Tools/ssl/make_ssl_data.py b/Tools/ssl/make_ssl_data.py index 98608716576792..d24e02210d489c 100755 --- a/Tools/ssl/make_ssl_data.py +++ b/Tools/ssl/make_ssl_data.py @@ -15,7 +15,6 @@ import operator import os import re -import sys parser = argparse.ArgumentParser( diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index baa16102068aa0..eae0e0c5e8761f 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -43,13 +43,14 @@ log = logging.getLogger("multissl") OPENSSL_OLD_VERSIONS = [ + "1.1.1w", ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1w", - "3.0.13", - "3.1.5", - "3.2.1", + "3.0.15", + "3.1.7", + "3.2.3", + "3.3.2", ] LIBRESSL_OLD_VERSIONS = [ @@ -397,6 +398,7 @@ def run_python_tests(self, tests, network=True): class BuildOpenSSL(AbstractBuilder): library = "OpenSSL" url_templates = ( + "https://github.com/openssl/openssl/releases/download/openssl-{v}/openssl-{v}.tar.gz", "https://www.openssl.org/source/openssl-{v}.tar.gz", "https://www.openssl.org/source/old/{s}/openssl-{v}.tar.gz" ) @@ -439,6 +441,7 @@ def short_version(self): parsed = parsed[:2] return ".".join(str(i) for i in parsed) + class BuildLibreSSL(AbstractBuilder): library = "LibreSSL" url_templates = ( diff --git a/Tools/stringbench/README b/Tools/stringbench/README deleted file mode 100644 index a271f12632afff..00000000000000 --- a/Tools/stringbench/README +++ /dev/null @@ -1,68 +0,0 @@ -stringbench is a set of performance tests comparing byte string -operations with unicode operations. The two string implementations -are loosely based on each other and sometimes the algorithm for one is -faster than the other. - -These test set was started at the Need For Speed sprint in Reykjavik -to identify which string methods could be sped up quickly and to -identify obvious places for improvement. - -Here is an example of a benchmark - - -@bench('"Andrew".startswith("A")', 'startswith single character', 1000) -def startswith_single(STR): - s1 = STR("Andrew") - s2 = STR("A") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -The bench decorator takes three parameters. The first is a short -description of how the code works. In most cases this is Python code -snippet. It is not the code which is actually run because the real -code is hand-optimized to focus on the method being tested. - -The second parameter is a group title. All benchmarks with the same -group title are listed together. This lets you compare different -implementations of the same algorithm, such as "t in s" -vs. "s.find(t)". - -The last is a count. Each benchmark loops over the algorithm either -100 or 1000 times, depending on the algorithm performance. The output -time is the time per benchmark call so the reader needs a way to know -how to scale the performance. - -These parameters become function attributes. - - -Here is an example of the output - - -========== count newlines -38.54 41.60 92.7 ...text.with.2000.newlines.count("\n") (*100) -========== early match, single character -1.14 1.18 96.8 ("A"*1000).find("A") (*1000) -0.44 0.41 105.6 "A" in "A"*1000 (*1000) -1.15 1.17 98.1 ("A"*1000).index("A") (*1000) - -The first column is the run time in milliseconds for byte strings. -The second is the run time for unicode strings. The third is a -percentage; byte time / unicode time. It's the percentage by which -unicode is faster than byte strings. - -The last column contains the code snippet and the repeat count for the -internal benchmark loop. - -The times are computed with 'timeit.py' which repeats the test more -and more times until the total time takes over 0.2 seconds, returning -the best time for a single iteration. - -The final line of the output is the cumulative time for byte and -unicode strings, and the overall performance of unicode relative to -bytes. For example - -4079.83 5432.25 75.1 TOTAL - -However, this has no meaning as it evenly weights every test. - diff --git a/Tools/stringbench/stringbench.py b/Tools/stringbench/stringbench.py deleted file mode 100644 index 5d2b4146378626..00000000000000 --- a/Tools/stringbench/stringbench.py +++ /dev/null @@ -1,1482 +0,0 @@ - -# Various microbenchmarks comparing unicode and byte string performance -# Please keep this file both 2.x and 3.x compatible! - -import timeit -import itertools -import operator -import re -import sys -import datetime -import optparse - -VERSION = '2.0' - -def p(*args): - sys.stdout.write(' '.join(str(s) for s in args) + '\n') - -if sys.version_info >= (3,): - BYTES = bytes_from_str = lambda x: x.encode('ascii') - UNICODE = unicode_from_str = lambda x: x -else: - BYTES = bytes_from_str = lambda x: x - UNICODE = unicode_from_str = lambda x: x.decode('ascii') - -class UnsupportedType(TypeError): - pass - - -p('stringbench v%s' % VERSION) -p(sys.version) -p(datetime.datetime.now()) - -REPEAT = 1 -REPEAT = 3 -#REPEAT = 7 - -if __name__ != "__main__": - raise SystemExit("Must run as main program") - -parser = optparse.OptionParser() -parser.add_option("-R", "--skip-re", dest="skip_re", - action="store_true", - help="skip regular expression tests") -parser.add_option("-8", "--8-bit", dest="bytes_only", - action="store_true", - help="only do 8-bit string benchmarks") -parser.add_option("-u", "--unicode", dest="unicode_only", - action="store_true", - help="only do Unicode string benchmarks") - - -_RANGE_1000 = list(range(1000)) -_RANGE_100 = list(range(100)) -_RANGE_10 = list(range(10)) - -dups = {} -def bench(s, group, repeat_count): - def blah(f): - if f.__name__ in dups: - raise AssertionError("Multiple functions with same name: %r" % - (f.__name__,)) - dups[f.__name__] = 1 - f.comment = s - f.is_bench = True - f.group = group - f.repeat_count = repeat_count - return f - return blah - -def uses_re(f): - f.uses_re = True - -####### 'in' comparisons - -@bench('"A" in "A"*1000', "early match, single character", 1000) -def in_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - for x in _RANGE_1000: - s2 in s1 - -@bench('"B" in "A"*1000', "no match, single character", 1000) -def in_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - for x in _RANGE_1000: - s2 in s1 - - -@bench('"AB" in "AB"*1000', "early match, two characters", 1000) -def in_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - for x in _RANGE_1000: - s2 in s1 - -@bench('"BC" in "AB"*1000', "no match, two characters", 1000) -def in_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - for x in _RANGE_1000: - s2 in s1 - -@bench('"BC" in ("AB"*300+"C")', "late match, two characters", 1000) -def in_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - for x in _RANGE_1000: - s2 in s1 - -@bench('s="ABC"*33; (s+"E") in ((s+"D")*300+s+"E")', - "late match, 100 characters", 100) -def in_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*300 + m+e - s2 = m+e - for x in _RANGE_100: - s2 in s1 - -# Try with regex -@uses_re -@bench('s="ABC"*33; re.compile(s+"D").search((s+"D")*300+s+"E")', - "late match, 100 characters", 100) -def re_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*300 + m+e - s2 = m+e - pat = re.compile(s2) - search = pat.search - for x in _RANGE_100: - search(s1) - - -#### same tests as 'in' but use 'find' - -@bench('("A"*1000).find("A")', "early match, single character", 1000) -def find_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("A"*1000).find("B")', "no match, single character", 1000) -def find_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - - -@bench('("AB"*1000).find("AB")', "early match, two characters", 1000) -def find_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*1000).find("BC")', "no match, two characters", 1000) -def find_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*1000).find("CA")', "no match, two characters", 1000) -def find_test_no_match_two_character_bis(STR): - s1 = STR("AB" * 1000) - s2 = STR("CA") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*300+"C").find("BC")', "late match, two characters", 1000) -def find_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*300+"CA").find("CA")', "late match, two characters", 1000) -def find_test_slow_match_two_characters_bis(STR): - s1 = STR("AB" * 300+"CA") - s2 = STR("CA") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").find(s+"E")', - "late match, 100 characters", 100) -def find_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_find = s1.find - for x in _RANGE_100: - s1_find(s2) - -@bench('s="ABC"*33; ((s+"D")*500+"E"+s).find("E"+s)', - "late match, 100 characters", 100) -def find_test_slow_match_100_characters_bis(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + e+m - s2 = e+m - s1_find = s1.find - for x in _RANGE_100: - s1_find(s2) - - -#### Same tests for 'rfind' - -@bench('("A"*1000).rfind("A")', "early match, single character", 1000) -def rfind_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("A"*1000).rfind("B")', "no match, single character", 1000) -def rfind_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - - -@bench('("AB"*1000).rfind("AB")', "early match, two characters", 1000) -def rfind_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("AB"*1000).rfind("BC")', "no match, two characters", 1000) -def rfind_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("AB"*1000).rfind("CA")', "no match, two characters", 1000) -def rfind_test_no_match_two_character_bis(STR): - s1 = STR("AB" * 1000) - s2 = STR("CA") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("C"+"AB"*300).rfind("CA")', "late match, two characters", 1000) -def rfind_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("BC"+"AB"*300).rfind("BC")', "late match, two characters", 1000) -def rfind_test_slow_match_two_characters_bis(STR): - s1 = STR("BC" + "AB" * 300) - s2 = STR("BC") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rfind("E"+s)', - "late match, 100 characters", 100) -def rfind_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e+m + (d+m)*500 - s2 = e+m - s1_rfind = s1.rfind - for x in _RANGE_100: - s1_rfind(s2) - -@bench('s="ABC"*33; (s+"E"+("D"+s)*500).rfind(s+"E")', - "late match, 100 characters", 100) -def rfind_test_slow_match_100_characters_bis(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = m+e + (d+m)*500 - s2 = m+e - s1_rfind = s1.rfind - for x in _RANGE_100: - s1_rfind(s2) - - -#### Now with index. -# Skip the ones which fail because that would include exception overhead. - -@bench('("A"*1000).index("A")', "early match, single character", 1000) -def index_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('("AB"*1000).index("AB")', "early match, two characters", 1000) -def index_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('("AB"*300+"C").index("BC")', "late match, two characters", 1000) -def index_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").index(s+"E")', - "late match, 100 characters", 100) -def index_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_index = s1.index - for x in _RANGE_100: - s1_index(s2) - - -#### Same for rindex - -@bench('("A"*1000).rindex("A")', "early match, single character", 1000) -def rindex_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('("AB"*1000).rindex("AB")', "early match, two characters", 1000) -def rindex_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('("C"+"AB"*300).rindex("CA")', "late match, two characters", 1000) -def rindex_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rindex("E"+s)', - "late match, 100 characters", 100) -def rindex_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rindex = s1.rindex - for x in _RANGE_100: - s1_rindex(s2) - - -#### Same for partition - -@bench('("A"*1000).partition("A")', "early match, single character", 1000) -def partition_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("A"*1000).partition("B")', "no match, single character", 1000) -def partition_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - - -@bench('("AB"*1000).partition("AB")', "early match, two characters", 1000) -def partition_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("AB"*1000).partition("BC")', "no match, two characters", 1000) -def partition_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("AB"*300+"C").partition("BC")', "late match, two characters", 1000) -def partition_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").partition(s+"E")', - "late match, 100 characters", 100) -def partition_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_partition = s1.partition - for x in _RANGE_100: - s1_partition(s2) - - -#### Same for rpartition - -@bench('("A"*1000).rpartition("A")', "early match, single character", 1000) -def rpartition_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("A"*1000).rpartition("B")', "no match, single character", 1000) -def rpartition_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - - -@bench('("AB"*1000).rpartition("AB")', "early match, two characters", 1000) -def rpartition_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("AB"*1000).rpartition("BC")', "no match, two characters", 1000) -def rpartition_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("C"+"AB"*300).rpartition("CA")', "late match, two characters", 1000) -def rpartition_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rpartition("E"+s)', - "late match, 100 characters", 100) -def rpartition_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rpartition = s1.rpartition - for x in _RANGE_100: - s1_rpartition(s2) - - -#### Same for split(s, 1) - -@bench('("A"*1000).split("A", 1)', "early match, single character", 1000) -def split_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("A"*1000).split("B", 1)', "no match, single character", 1000) -def split_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - - -@bench('("AB"*1000).split("AB", 1)', "early match, two characters", 1000) -def split_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("AB"*1000).split("BC", 1)', "no match, two characters", 1000) -def split_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("AB"*300+"C").split("BC", 1)', "late match, two characters", 1000) -def split_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").split(s+"E", 1)', - "late match, 100 characters", 100) -def split_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_split = s1.split - for x in _RANGE_100: - s1_split(s2, 1) - - -#### Same for rsplit(s, 1) - -@bench('("A"*1000).rsplit("A", 1)', "early match, single character", 1000) -def rsplit_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("A"*1000).rsplit("B", 1)', "no match, single character", 1000) -def rsplit_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - - -@bench('("AB"*1000).rsplit("AB", 1)', "early match, two characters", 1000) -def rsplit_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("AB"*1000).rsplit("BC", 1)', "no match, two characters", 1000) -def rsplit_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("C"+"AB"*300).rsplit("CA", 1)', "late match, two characters", 1000) -def rsplit_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rsplit("E"+s, 1)', - "late match, 100 characters", 100) -def rsplit_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rsplit = s1.rsplit - for x in _RANGE_100: - s1_rsplit(s2, 1) - - -#### Benchmark the operator-based methods - -@bench('"A"*10', "repeat 1 character 10 times", 1000) -def repeat_single_10_times(STR): - s = STR("A") - for x in _RANGE_1000: - s * 10 - -@bench('"A"*1000', "repeat 1 character 1000 times", 1000) -def repeat_single_1000_times(STR): - s = STR("A") - for x in _RANGE_1000: - s * 1000 - -@bench('"ABCDE"*10', "repeat 5 characters 10 times", 1000) -def repeat_5_10_times(STR): - s = STR("ABCDE") - for x in _RANGE_1000: - s * 10 - -@bench('"ABCDE"*1000', "repeat 5 characters 1000 times", 1000) -def repeat_5_1000_times(STR): - s = STR("ABCDE") - for x in _RANGE_1000: - s * 1000 - -# + for concat - -@bench('"Andrew"+"Dalke"', "concat two strings", 1000) -def concat_two_strings(STR): - s1 = STR("Andrew") - s2 = STR("Dalke") - for x in _RANGE_1000: - s1+s2 - -@bench('s1+s2+s3+s4+...+s20', "concat 20 strings of words length 4 to 15", - 1000) -def concat_many_strings(STR): - s1=STR('TIXSGYNREDCVBHJ') - s2=STR('PUMTLXBZVDO') - s3=STR('FVZNJ') - s4=STR('OGDXUW') - s5=STR('WEIMRNCOYVGHKB') - s6=STR('FCQTNMXPUZH') - s7=STR('TICZJYRLBNVUEAK') - s8=STR('REYB') - s9=STR('PWUOQ') - s10=STR('EQHCMKBS') - s11=STR('AEVDFOH') - s12=STR('IFHVD') - s13=STR('JGTCNLXWOHQ') - s14=STR('ITSKEPYLROZAWXF') - s15=STR('THEK') - s16=STR('GHPZFBUYCKMNJIT') - s17=STR('JMUZ') - s18=STR('WLZQMTB') - s19=STR('KPADCBW') - s20=STR('TNJHZQAGBU') - for x in _RANGE_1000: - (s1 + s2+ s3+ s4+ s5+ s6+ s7+ s8+ s9+s10+ - s11+s12+s13+s14+s15+s16+s17+s18+s19+s20) - - -#### Benchmark join - -def get_bytes_yielding_seq(STR, arg): - if STR is BYTES and sys.version_info >= (3,): - raise UnsupportedType - return STR(arg) - -@bench('"A".join("")', - "join empty string, with 1 character sep", 100) -def join_empty_single(STR): - sep = STR("A") - s2 = get_bytes_yielding_seq(STR, "") - sep_join = sep.join - for x in _RANGE_100: - sep_join(s2) - -@bench('"ABCDE".join("")', - "join empty string, with 5 character sep", 100) -def join_empty_5(STR): - sep = STR("ABCDE") - s2 = get_bytes_yielding_seq(STR, "") - sep_join = sep.join - for x in _RANGE_100: - sep_join(s2) - -@bench('"A".join("ABC..Z")', - "join string with 26 characters, with 1 character sep", 1000) -def join_alphabet_single(STR): - sep = STR("A") - s2 = get_bytes_yielding_seq(STR, "ABCDEFGHIJKLMnOPQRSTUVWXYZ") - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join("ABC..Z")', - "join string with 26 characters, with 5 character sep", 1000) -def join_alphabet_5(STR): - sep = STR("ABCDE") - s2 = get_bytes_yielding_seq(STR, "ABCDEFGHIJKLMnOPQRSTUVWXYZ") - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"A".join(list("ABC..Z"))', - "join list of 26 characters, with 1 character sep", 1000) -def join_alphabet_list_single(STR): - sep = STR("A") - s2 = [STR(x) for x in "ABCDEFGHIJKLMnOPQRSTUVWXYZ"] - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join(list("ABC..Z"))', - "join list of 26 characters, with 5 character sep", 1000) -def join_alphabet_list_five(STR): - sep = STR("ABCDE") - s2 = [STR(x) for x in "ABCDEFGHIJKLMnOPQRSTUVWXYZ"] - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"A".join(["Bob"]*100)', - "join list of 100 words, with 1 character sep", 1000) -def join_100_words_single(STR): - sep = STR("A") - s2 = [STR("Bob")]*100 - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join(["Bob"]*100))', - "join list of 100 words, with 5 character sep", 1000) -def join_100_words_5(STR): - sep = STR("ABCDE") - s2 = [STR("Bob")]*100 - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -#### split tests - -@bench('("Here are some words. "*2).split()', "split whitespace (small)", 1000) -def whitespace_split(STR): - s = STR("Here are some words. "*2) - s_split = s.split - for x in _RANGE_1000: - s_split() - -@bench('("Here are some words. "*2).rsplit()', "split whitespace (small)", 1000) -def whitespace_rsplit(STR): - s = STR("Here are some words. "*2) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit() - -@bench('("Here are some words. "*2).split(None, 1)', - "split 1 whitespace", 1000) -def whitespace_split_1(STR): - s = STR("Here are some words. "*2) - s_split = s.split - N = None - for x in _RANGE_1000: - s_split(N, 1) - -@bench('("Here are some words. "*2).rsplit(None, 1)', - "split 1 whitespace", 1000) -def whitespace_rsplit_1(STR): - s = STR("Here are some words. "*2) - s_rsplit = s.rsplit - N = None - for x in _RANGE_1000: - s_rsplit(N, 1) - -@bench('("Here are some words. "*2).partition(" ")', - "split 1 whitespace", 1000) -def whitespace_partition(STR): - sep = STR(" ") - s = STR("Here are some words. "*2) - s_partition = s.partition - for x in _RANGE_1000: - s_partition(sep) - -@bench('("Here are some words. "*2).rpartition(" ")', - "split 1 whitespace", 1000) -def whitespace_rpartition(STR): - sep = STR(" ") - s = STR("Here are some words. "*2) - s_rpartition = s.rpartition - for x in _RANGE_1000: - s_rpartition(sep) - -human_text = """\ -Python is a dynamic object-oriented programming language that can be -used for many kinds of software development. It offers strong support -for integration with other languages and tools, comes with extensive -standard libraries, and can be learned in a few days. Many Python -programmers report substantial productivity gains and feel the language -encourages the development of higher quality, more maintainable code. - -Python runs on Windows, Linux/Unix, Mac OS X, Amiga, Palm -Handhelds, and Nokia mobile phones. Python has also been ported to the -Java and .NET virtual machines. - -Python is distributed under an OSI-approved open source license that -makes it free to use, even for commercial products. -"""*25 -human_text_bytes = bytes_from_str(human_text) -human_text_unicode = unicode_from_str(human_text) -def _get_human_text(STR): - if STR is UNICODE: - return human_text_unicode - if STR is BYTES: - return human_text_bytes - raise AssertionError - -@bench('human_text.split()', "split whitespace (huge)", 10) -def whitespace_split_huge(STR): - s = _get_human_text(STR) - s_split = s.split - for x in _RANGE_10: - s_split() - -@bench('human_text.rsplit()', "split whitespace (huge)", 10) -def whitespace_rsplit_huge(STR): - s = _get_human_text(STR) - s_rsplit = s.rsplit - for x in _RANGE_10: - s_rsplit() - - - -@bench('"this\\nis\\na\\ntest\\n".split("\\n")', "split newlines", 1000) -def newlines_split(STR): - s = STR("this\nis\na\ntest\n") - s_split = s.split - nl = STR("\n") - for x in _RANGE_1000: - s_split(nl) - - -@bench('"this\\nis\\na\\ntest\\n".rsplit("\\n")', "split newlines", 1000) -def newlines_rsplit(STR): - s = STR("this\nis\na\ntest\n") - s_rsplit = s.rsplit - nl = STR("\n") - for x in _RANGE_1000: - s_rsplit(nl) - -@bench('"this\\nis\\na\\ntest\\n".splitlines()', "split newlines", 1000) -def newlines_splitlines(STR): - s = STR("this\nis\na\ntest\n") - s_splitlines = s.splitlines - for x in _RANGE_1000: - s_splitlines() - -## split text with 2000 newlines - -def _make_2000_lines(): - import random - r = random.Random(100) - chars = list(map(chr, range(32, 128))) - i = 0 - while i < len(chars): - chars[i] = " " - i += r.randrange(9) - s = "".join(chars) - s = s*4 - words = [] - for i in range(2000): - start = r.randrange(96) - n = r.randint(5, 65) - words.append(s[start:start+n]) - return "\n".join(words)+"\n" - -_text_with_2000_lines = _make_2000_lines() -_text_with_2000_lines_bytes = bytes_from_str(_text_with_2000_lines) -_text_with_2000_lines_unicode = unicode_from_str(_text_with_2000_lines) -def _get_2000_lines(STR): - if STR is UNICODE: - return _text_with_2000_lines_unicode - if STR is BYTES: - return _text_with_2000_lines_bytes - raise AssertionError - - -@bench('"...text...".split("\\n")', "split 2000 newlines", 10) -def newlines_split_2000(STR): - s = _get_2000_lines(STR) - s_split = s.split - nl = STR("\n") - for x in _RANGE_10: - s_split(nl) - -@bench('"...text...".rsplit("\\n")', "split 2000 newlines", 10) -def newlines_rsplit_2000(STR): - s = _get_2000_lines(STR) - s_rsplit = s.rsplit - nl = STR("\n") - for x in _RANGE_10: - s_rsplit(nl) - -@bench('"...text...".splitlines()', "split 2000 newlines", 10) -def newlines_splitlines_2000(STR): - s = _get_2000_lines(STR) - s_splitlines = s.splitlines - for x in _RANGE_10: - s_splitlines() - - -## split text on "--" characters -@bench( - '"this--is--a--test--of--the--emergency--broadcast--system".split("--")', - "split on multicharacter separator (small)", 1000) -def split_multichar_sep_small(STR): - s = STR("this--is--a--test--of--the--emergency--broadcast--system") - s_split = s.split - pat = STR("--") - for x in _RANGE_1000: - s_split(pat) -@bench( - '"this--is--a--test--of--the--emergency--broadcast--system".rsplit("--")', - "split on multicharacter separator (small)", 1000) -def rsplit_multichar_sep_small(STR): - s = STR("this--is--a--test--of--the--emergency--broadcast--system") - s_rsplit = s.rsplit - pat = STR("--") - for x in _RANGE_1000: - s_rsplit(pat) - -## split dna text on "ACTAT" characters -@bench('dna.split("ACTAT")', - "split on multicharacter separator (dna)", 10) -def split_multichar_sep_dna(STR): - s = _get_dna(STR) - s_split = s.split - pat = STR("ACTAT") - for x in _RANGE_10: - s_split(pat) - -@bench('dna.rsplit("ACTAT")', - "split on multicharacter separator (dna)", 10) -def rsplit_multichar_sep_dna(STR): - s = _get_dna(STR) - s_rsplit = s.rsplit - pat = STR("ACTAT") - for x in _RANGE_10: - s_rsplit(pat) - - - -## split with limits - -GFF3_example = "\t".join([ - "I", "Genomic_canonical", "region", "357208", "396183", ".", "+", ".", - "ID=Sequence:R119;note=Clone R119%3B Genbank AF063007;Name=R119"]) - -@bench('GFF3_example.split("\\t")', "tab split", 1000) -def tab_split_no_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_split = s.split - for x in _RANGE_1000: - s_split(sep) - -@bench('GFF3_example.split("\\t", 8)', "tab split", 1000) -def tab_split_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_split = s.split - for x in _RANGE_1000: - s_split(sep, 8) - -@bench('GFF3_example.rsplit("\\t")', "tab split", 1000) -def tab_rsplit_no_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit(sep) - -@bench('GFF3_example.rsplit("\\t", 8)', "tab split", 1000) -def tab_rsplit_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit(sep, 8) - -#### Count characters - -@bench('...text.with.2000.newlines.count("\\n")', - "count newlines", 10) -def count_newlines(STR): - s = _get_2000_lines(STR) - s_count = s.count - nl = STR("\n") - for x in _RANGE_10: - s_count(nl) - -# Orchid sequences concatenated, from Biopython -_dna = """ -CGTAACAAGGTTTCCGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGGGTT -AATCTGGAGGATCTGTTTACTTTGGTCACCCATGAGCATTTGCTGTTGAAGTGACCTAGAATTGCCATCG -AGCCTCCTTGGGAGCTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGCAGTTTTGCTCCAAGTCGTT -TGACACATAATTGGTGAAGGGGGTGGCATCCTTCCCTGACCCTCCCCCAACTATTTTTTTAACAACTCTC -AGCAACGGAGACTCAGTCTTCGGCAAATGCGATAAATGGTGTGAATTGCAGAATCCCGTGCACCATCGAG -TCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCTGCCTGGGCATTGCGAGTCATAT -CTCTCCCTTAACGAGGCTGTCCATACATACTGTTCAGCCGGTGCGGATGTGAGTTTGGCCCCTTGTTCTT -TGGTACGGGGGGTCTAAGAGCTGCATGGGCTTTTGATGGTCCTAAATACGGCAAGAGGTGGACGAACTAT -GCTACAACAAAATTGTTGTGCAGAGGCCCCGGGTTGTCGTATTAGATGGGCCACCGTAATCTGAAGACCC -TTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGGCCATTTGGTTGCGACCCCAGGTCAG -GTGAGCAACAGCTGTCGTAACAAGGTTTCCGTAGGGTGAACTGCGGAAGGATCATTGTTGAGATCACATA -ATAATTGATCGAGTTAATCTGGAGGATCTGTTTACTTGGGTCACCCATGGGCATTTGCTGTTGAAGTGAC -CTAGATTTGCCATCGAGCCTCCTTGGGAGCATCCTTGTTGGCGATATCTAAACCCTCAATTTTTCCCCCA -ATCAAATTACACAAAATTGGTGGAGGGGGTGGCATTCTTCCCTTACCCTCCCCCAAATATTTTTTTAACA -ACTCTCAGCAACGGATATCTCAGCTCTTGCATCGATGAAGAACCCACCGAAATGCGATAAATGGTGTGAA -TTGCAGAATCCCGTGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACG -CCTGCCTGGGCATTGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACATACTGTTCAGCCGGTGCG -GATGTGAGTTTGGCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGATGCATGGGCTTTTGATGGTCCTAA -ATACGGCAAGAGGTGGACGAACTATGCTACAACAAAATTGTTGTGCAAAGGCCCCGGGTTGTCGTATAAG -ATGGGCCACCGATATCTGAAGACCCTTTTGGACCCCATTGGAGCCCATCAACCCATGTCAGTTGATGGCC -ATTCGTAACAAGGTTTCCGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGA -GTTAATCTGGAGGATCTGTTTACTTGGGTCACCCATGGGCATTTGCTGTTGAAGTGACCTAGATTTGCCA -TCGAGCCTCCTTGGGAGCTTTCTTGTTGGCGATATCTAAACCCTTGCCCGGCAGAGTTTTGGGAATCCCG -TGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCTGCCTGGGCAT -TGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACACACCTGTTCAGCCGGTGCGGATGTGAGTTTG -GCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGCTGCATGGGCTTTTGATGGTCCTAAATACGGCAAGAG -GTGGACGAACTATGCTACAACAAAATTGTTGTGCAAAGGCCCCGGGTTGTCGTATTAGATGGGCCACCAT -AATCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGGCCATTTGGTTGC -GACCCAGTCAGGTGAGGGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGAG -TTAATCTGGAGGATCTGTTTACTTTGGTCACCCATGGGCATTTGCTGTTGAAGTGACCTAGATTTGCCAT -CGAGCCTCCTTGGGAGCTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGGAGTTTGGCGCCAAGTCA -TATGACACATAATTGGTGAAGGGGGTGGCATCCTGCCCTGACCCTCCCCAAATTATTTTTTTAACAACTC -TCAGCAACGGATATCTCGGCTCTTGCATCGATGAAGAACGCAGCGAAATGCGATAAATGGTGTGAATTGC -AGAATCCCGTGAACCATCGAGTCTTTGGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCT -GCCTGGGCATTGGGAATCATATCTCTCCCCTAACGAGGCTATCCAAACATACTGTTCATCCGGTGCGGAT -GTGAGTTTGGCCCCTTGTTCTTTGGTACCGGGGGTCTAAGAGCTGCATGGGCATTTGATGGTCCTCAAAA -CGGCAAGAGGTGGACGAACTATGCCACAACAAAATTGTTGTCCCAAGGCCCCGGGTTGTCGTATTAGATG -GGCCACCGTAACCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGACCA -TTTGTTGCGACCCCAGTCAGCTGAGCAACCCGCTGAGTGGAAGGTCATTGCCGATATCACATAATAATTG -ATCGAGTTAATCTGGAGGATCTGTTTACTTGGTCACCCATGAGCATTTGCTGTTGAAGTGACCTAGATTT -GCCATCGAGCCTCCTTGGGAGTTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGGAGTTGTGCGCCA -AGTCATATGACACATAATTGGTGAAGGGGGTGGCATCCTGCCCTGACCCTCCCCAAATTATTTTTTTAAC -AACTCTCAGCAACGGATATCTCGGCTCTTGCATCGATGAAGAACGCAGCGAAATGCGATAAATGGTGTGA -ATTGCAGAATCCCGTGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCAC -GCCTGCCTGGGCATTGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACATACTGTTCATCCGGTGC -GGATGTGAGTTTGGCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGCTGCATGGGCATTTGATGGTCCTC -AAAACGGCAAGAGGTGGACGAACTATGCTACAACCAAATTGTTGTCCCAAGGCCCCGGGTTGTCGTATTA -GATGGGCCACCGTAACCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATG -ACCATGTGTTGCGACCCCAGTCAGCTGAGCAACGCGCTGAGCGTAACAAGGTTTCCGTAGGTGGACCTCC -GGGAGGATCATTGTTGAGATCACATAATAATTGATCGAGGTAATCTGGAGGATCTGCATATTTTGGTCAC -""" -_dna = "".join(_dna.splitlines()) -_dna = _dna * 25 -_dna_bytes = bytes_from_str(_dna) -_dna_unicode = unicode_from_str(_dna) - -def _get_dna(STR): - if STR is UNICODE: - return _dna_unicode - if STR is BYTES: - return _dna_bytes - raise AssertionError - -@bench('dna.count("AACT")', "count AACT substrings in DNA example", 10) -def count_aact(STR): - seq = _get_dna(STR) - seq_count = seq.count - needle = STR("AACT") - for x in _RANGE_10: - seq_count(needle) - -##### startswith and endswith - -@bench('"Andrew".startswith("A")', 'startswith single character', 1000) -def startswith_single(STR): - s1 = STR("Andrew") - s2 = STR("A") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -@bench('"Andrew".startswith("Andrew")', 'startswith multiple characters', - 1000) -def startswith_multiple(STR): - s1 = STR("Andrew") - s2 = STR("Andrew") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -@bench('"Andrew".startswith("Anders")', - 'startswith multiple characters - not!', 1000) -def startswith_multiple_not(STR): - s1 = STR("Andrew") - s2 = STR("Anders") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - - -# endswith - -@bench('"Andrew".endswith("w")', 'endswith single character', 1000) -def endswith_single(STR): - s1 = STR("Andrew") - s2 = STR("w") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -@bench('"Andrew".endswith("Andrew")', 'endswith multiple characters', 1000) -def endswith_multiple(STR): - s1 = STR("Andrew") - s2 = STR("Andrew") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -@bench('"Andrew".endswith("Anders")', - 'endswith multiple characters - not!', 1000) -def endswith_multiple_not(STR): - s1 = STR("Andrew") - s2 = STR("Anders") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -#### Strip - -@bench('"Hello!\\n".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_right(STR): - s = STR("Hello!\n") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"Hello!\\n".rstrip()', 'strip terminal newline', 1000) -def terminal_newline_rstrip(STR): - s = STR("Hello!\n") - s_rstrip = s.rstrip - for x in _RANGE_1000: - s_rstrip() - -@bench('"\\nHello!".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_left(STR): - s = STR("\nHello!") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"\\nHello!\\n".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_both(STR): - s = STR("\nHello!\n") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"\\nHello!".rstrip()', 'strip terminal newline', 1000) -def terminal_newline_lstrip(STR): - s = STR("\nHello!") - s_lstrip = s.lstrip - for x in _RANGE_1000: - s_lstrip() - -@bench('s="Hello!\\n"; s[:-1] if s[-1]=="\\n" else s', - 'strip terminal newline', 1000) -def terminal_newline_if_else(STR): - s = STR("Hello!\n") - NL = STR("\n") - for x in _RANGE_1000: - s[:-1] if (s[-1] == NL) else s - - -# Strip multiple spaces or tabs - -@bench('"Hello\\t \\t".strip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_strip(STR): - s = STR("Hello\t \t!") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"Hello\\t \\t".rstrip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_rstrip(STR): - s = STR("Hello!\t \t") - s_rstrip = s.rstrip - for x in _RANGE_1000: - s_rstrip() - -@bench('"\\t \\tHello".rstrip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_lstrip(STR): - s = STR("\t \tHello!") - s_lstrip = s.lstrip - for x in _RANGE_1000: - s_lstrip() - - -#### replace -@bench('"This is a test".replace(" ", "\\t")', 'replace single character', - 1000) -def replace_single_character(STR): - s = STR("This is a test!") - from_str = STR(" ") - to_str = STR("\t") - s_replace = s.replace - for x in _RANGE_1000: - s_replace(from_str, to_str) - -@uses_re -@bench('re.sub(" ", "\\t", "This is a test"', 'replace single character', - 1000) -def replace_single_character_re(STR): - s = STR("This is a test!") - pat = re.compile(STR(" ")) - to_str = STR("\t") - pat_sub = pat.sub - for x in _RANGE_1000: - pat_sub(to_str, s) - -@bench('"...text.with.2000.lines...replace("\\n", " ")', - 'replace single character, big string', 10) -def replace_single_character_big(STR): - s = _get_2000_lines(STR) - from_str = STR("\n") - to_str = STR(" ") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str) - -@uses_re -@bench('re.sub("\\n", " ", "...text.with.2000.lines...")', - 'replace single character, big string', 10) -def replace_single_character_big_re(STR): - s = _get_2000_lines(STR) - pat = re.compile(STR("\n")) - to_str = STR(" ") - pat_sub = pat.sub - for x in _RANGE_10: - pat_sub(to_str, s) - - -@bench('dna.replace("ATC", "ATT")', - 'replace multiple characters, dna', 10) -def replace_multiple_characters_dna(STR): - seq = _get_dna(STR) - from_str = STR("ATC") - to_str = STR("ATT") - seq_replace = seq.replace - for x in _RANGE_10: - seq_replace(from_str, to_str) - -# This increases the character count -@bench('"...text.with.2000.newlines...replace("\\n", "\\r\\n")', - 'replace and expand multiple characters, big string', 10) -def replace_multiple_character_big(STR): - s = _get_2000_lines(STR) - from_str = STR("\n") - to_str = STR("\r\n") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str) - - -# This decreases the character count -@bench('"When shall we three meet again?".replace("ee", "")', - 'replace/remove multiple characters', 1000) -def replace_multiple_character_remove(STR): - s = STR("When shall we three meet again?") - from_str = STR("ee") - to_str = STR("") - s_replace = s.replace - for x in _RANGE_1000: - s_replace(from_str, to_str) - - -big_s = "A" + ("Z"*128*1024) -big_s_bytes = bytes_from_str(big_s) -big_s_unicode = unicode_from_str(big_s) -def _get_big_s(STR): - if STR is UNICODE: return big_s_unicode - if STR is BYTES: return big_s_bytes - raise AssertionError - -# The older replace implementation counted all matches in -# the string even when it only needed to make one replacement. -@bench('("A" + ("Z"*128*1024)).replace("A", "BB", 1)', - 'quick replace single character match', 10) -def quick_replace_single_match(STR): - s = _get_big_s(STR) - from_str = STR("A") - to_str = STR("BB") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str, 1) - -@bench('("A" + ("Z"*128*1024)).replace("AZZ", "BBZZ", 1)', - 'quick replace multiple character match', 10) -def quick_replace_multiple_match(STR): - s = _get_big_s(STR) - from_str = STR("AZZ") - to_str = STR("BBZZ") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str, 1) - - -#### - -# CCP does a lot of this, for internationalisation of ingame messages. -_format = "The %(thing)s is %(place)s the %(location)s." -_format_dict = { "thing":"THING", "place":"PLACE", "location":"LOCATION", } -_format_bytes = bytes_from_str(_format) -_format_unicode = unicode_from_str(_format) -_format_dict_bytes = dict((bytes_from_str(k), bytes_from_str(v)) for (k,v) in _format_dict.items()) -_format_dict_unicode = dict((unicode_from_str(k), unicode_from_str(v)) for (k,v) in _format_dict.items()) - -def _get_format(STR): - if STR is UNICODE: - return _format_unicode - if STR is BYTES: - if sys.version_info >= (3,): - raise UnsupportedType - return _format_bytes - raise AssertionError - -def _get_format_dict(STR): - if STR is UNICODE: - return _format_dict_unicode - if STR is BYTES: - if sys.version_info >= (3,): - raise UnsupportedType - return _format_dict_bytes - raise AssertionError - -# Formatting. -@bench('"The %(k1)s is %(k2)s the %(k3)s."%{"k1":"x","k2":"y","k3":"z",}', - 'formatting a string type with a dict', 1000) -def format_with_dict(STR): - s = _get_format(STR) - d = _get_format_dict(STR) - for x in _RANGE_1000: - s % d - - -#### Upper- and lower- case conversion - -@bench('("Where in the world is Carmen San Deigo?"*10).lower()', - "case conversion -- rare", 1000) -def lower_conversion_rare(STR): - s = STR("Where in the world is Carmen San Deigo?"*10) - s_lower = s.lower - for x in _RANGE_1000: - s_lower() - -@bench('("WHERE IN THE WORLD IS CARMEN SAN DEIGO?"*10).lower()', - "case conversion -- dense", 1000) -def lower_conversion_dense(STR): - s = STR("WHERE IN THE WORLD IS CARMEN SAN DEIGO?"*10) - s_lower = s.lower - for x in _RANGE_1000: - s_lower() - - -@bench('("wHERE IN THE WORLD IS cARMEN sAN dEIGO?"*10).upper()', - "case conversion -- rare", 1000) -def upper_conversion_rare(STR): - s = STR("Where in the world is Carmen San Deigo?"*10) - s_upper = s.upper - for x in _RANGE_1000: - s_upper() - -@bench('("where in the world is carmen san deigo?"*10).upper()', - "case conversion -- dense", 1000) -def upper_conversion_dense(STR): - s = STR("where in the world is carmen san deigo?"*10) - s_upper = s.upper - for x in _RANGE_1000: - s_upper() - - -# end of benchmarks - -################# - -class BenchTimer(timeit.Timer): - def best(self, repeat=1): - for i in range(1, 10): - number = 10**i - x = self.timeit(number) - if x > 0.02: - break - times = [x] - for i in range(1, repeat): - times.append(self.timeit(number)) - return min(times) / number - -def main(): - (options, test_names) = parser.parse_args() - if options.bytes_only and options.unicode_only: - raise SystemExit("Only one of --8-bit and --unicode are allowed") - - bench_functions = [] - for (k,v) in globals().items(): - if hasattr(v, "is_bench"): - if test_names: - for name in test_names: - if name in v.group: - break - else: - # Not selected, ignore - continue - if options.skip_re and hasattr(v, "uses_re"): - continue - - bench_functions.append( (v.group, k, v) ) - bench_functions.sort() - - p("bytes\tunicode") - p("(in ms)\t(in ms)\t%\tcomment") - - bytes_total = uni_total = 0.0 - - for title, group in itertools.groupby(bench_functions, - operator.itemgetter(0)): - # Flush buffer before each group - sys.stdout.flush() - p("="*10, title) - for (_, k, v) in group: - if hasattr(v, "is_bench"): - bytes_time = 0.0 - bytes_time_s = " - " - if not options.unicode_only: - try: - bytes_time = BenchTimer("__main__.%s(__main__.BYTES)" % (k,), - "import __main__").best(REPEAT) - bytes_time_s = "%.2f" % (1000 * bytes_time) - bytes_total += bytes_time - except UnsupportedType: - bytes_time_s = "N/A" - uni_time = 0.0 - uni_time_s = " - " - if not options.bytes_only: - try: - uni_time = BenchTimer("__main__.%s(__main__.UNICODE)" % (k,), - "import __main__").best(REPEAT) - uni_time_s = "%.2f" % (1000 * uni_time) - uni_total += uni_time - except UnsupportedType: - uni_time_s = "N/A" - try: - average = bytes_time/uni_time - except (TypeError, ZeroDivisionError): - average = 0.0 - p("%s\t%s\t%.1f\t%s (*%d)" % ( - bytes_time_s, uni_time_s, 100.*average, - v.comment, v.repeat_count)) - - if bytes_total == uni_total == 0.0: - p("That was zippy!") - else: - try: - ratio = bytes_total/uni_total - except ZeroDivisionError: - ratio = 0.0 - p("%.2f\t%.2f\t%.1f\t%s" % ( - 1000*bytes_total, 1000*uni_total, 100.*ratio, - "TOTAL")) - -if __name__ == "__main__": - main() diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt new file mode 100644 index 00000000000000..e5eb665ae212de --- /dev/null +++ b/Tools/tsan/suppressions_free_threading.txt @@ -0,0 +1,48 @@ +# This file contains suppressions for the free-threaded build. It contains the +# suppressions for the default build and additional suppressions needed only in +# the free-threaded build. +# +# reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions + +## Default build suppresssions + +race:get_allocator_unlocked +race:set_allocator_unlocked + +## Free-threaded suppressions + + +# These entries are for warnings that trigger in a library function, as called +# by a CPython function. + +# https://gist.github.com/swtaarrs/08dfe7883b4c975c31ecb39388987a67 +race:free_threadstate + + +# These warnings trigger directly in a CPython function. + +race_top:assign_version_tag +race_top:new_reference +race_top:_multiprocessing_SemLock_acquire_impl +race_top:list_get_item_ref +race_top:_Py_slot_tp_getattr_hook +race_top:add_threadstate +race_top:dump_traceback +race_top:fatal_error +race_top:_multiprocessing_SemLock_release_impl +race_top:_PyFrame_GetCode +race_top:_PyFrame_Initialize +race_top:PyInterpreterState_ThreadHead +race_top:_PyObject_TryGetInstanceAttribute +race_top:PyThreadState_Next +race_top:PyUnstable_InterpreterFrame_GetLine +race_top:tstate_delete_common +race_top:tstate_is_freed +race_top:type_modified_unlocked +race_top:write_thread_id +race_top:PyThreadState_Clear +# Only seen on macOS, sample: https://gist.github.com/aisk/dda53f5d494a4556c35dde1fce03259c +race_top:set_default_allocator_unlocked + +# https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 +thread:pthread_create diff --git a/Tools/tsan/supressions.txt b/Tools/tsan/supressions.txt new file mode 100644 index 00000000000000..22ba9d6ba2ab4d --- /dev/null +++ b/Tools/tsan/supressions.txt @@ -0,0 +1,7 @@ +# This file contains suppressions for the default (with GIL) build. +# reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions +race:get_allocator_unlocked +race:set_allocator_unlocked + +# https://gist.github.com/mpage/daaf32b39180c1989572957b943eb665 +thread:pthread_create diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index a5ac09eb96f19a..c94de7f9377b74 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -44,7 +44,7 @@ # * Doc/library/stdtypes.rst, and # * Doc/library/unicodedata.rst # * Doc/reference/lexical_analysis.rst (two occurrences) -UNIDATA_VERSION = "15.1.0" +UNIDATA_VERSION = "16.0.0" UNICODE_DATA = "UnicodeData%s.txt" COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt" EASTASIAN_WIDTH = "EastAsianWidth%s.txt" diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 23b38c8e93638a..bc3e4ba8bd5b76 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -1,6 +1,7 @@ # Python WebAssembly (WASM) build -**WARNING: WASM support is work-in-progress! Lots of features are not working yet.** +**WASI support is [tier 2](https://peps.python.org/pep-0011/#tier-2).** +**Emscripten is NOT officially supported as of Python 3.13.** This directory contains configuration and helpers to facilitate cross compilation of CPython to WebAssembly (WASM). Python supports Emscripten @@ -274,7 +275,7 @@ Node builds use ``NODERAWFS``. ### Hosting Python WASM builds The simple REPL terminal uses SharedArrayBuffer. For security reasons -browsers only provide the feature in secure environents with cross-origin +browsers only provide the feature in secure environments with cross-origin isolation. The webserver must send cross-origin headers and correct MIME types for the JavaScript and WASM files. Otherwise the terminal will fail to load with an error message like ``Browsers disable shared array buffer``. @@ -298,66 +299,7 @@ AddType application/wasm wasm ## WASI (wasm32-wasi) -**NOTE**: The instructions below assume a Unix-based OS due to cross-compilation for CPython being set up for `./configure`. - -### Prerequisites - -Developing for WASI requires two additional tools to be installed beyond the typical tools required to build CPython: - -1. The [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+ -2. A WASI host/runtime ([wasmtime](https://wasmtime.dev) 14+ is recommended and what the instructions below assume) - -All of this is provided in the [devcontainer](https://devguide.python.org/getting-started/setup-building/#contribute-using-github-codespaces) if you don't want to install these tools locally. - -### Building - -Building for WASI requires doing a cross-build where you have a "build" Python to help produce a WASI build of CPython (technically it's a "host x host" cross-build because the build Python is also the target Python while the host build is the WASI build; yes, it's confusing terminology). In the end you should have a build Python in `cross-build/build` and a WASI build in `cross-build/wasm32-wasi`. - -The easiest way to do a build is to use the `wasi.py` script. You can either have it perform the entire build process from start to finish in one step, or you can do it in discrete steps that mirror running `configure` and `make` for each of the two builds of Python you end up producing (which are beneficial when you only need to do a specific step after getting a complete build, e.g. editing some code and you just need to run `make` for the WASI build). The script is designed to self-document what actions it is performing on your behalf, both as a way to check its work but also for educaitonal purposes. - -The discrete steps for building via `wasi.py` are: -```shell -python Tools/wasm/wasi.py configure-build-python -python Tools/wasm/wasi.py make-build-python -python Tools/wasm/wasi.py configure-host -python Tools/wasm/wasi.py make-host -``` - -To do it all in a single command, run: -```shell -python Tools/wasm/wasi.py build -``` - -That will: - -1. Run `configure` for the build Python (same as `wasi.py configure-build-python`) -2. Run `make` for the build Python (`wasi.py make-build-python`) -3. Run `configure` for the WASI build (`wasi.py configure-host`) -4. Run `make` for the WASI build (`wasi.py make-host`) - -See the `--help` for the various options available for each of the subcommands which controls things like the location of the WASI SDK, the command to use with the WASI host/runtime, etc. Also note that you can use `--` as a separator for any of the `configure`-related commands -- including `build` itself -- to pass arguments to the underlying `configure` call. For example, if you want a pydebug build that also caches the results from `configure`, you can do: -```shell -python Tools/wasm/wasi.py build -- -C --with-pydebug -``` - -The `wasi.py` script is able to infer details from the build Python, and so you only technically need to specify `--with-pydebug` once via `configure-build-python` as this will lead to `configure-host` detecting its use if you use the discrete steps: -```shell -python Tools/wasm/wasi.py configure-build-python -- -C --with-pydebug -python Tools/wasm/wasi.py make-build-python -python Tools/wasm/wasi.py configure-host -- -C -python Tools/wasm/wasi.py make-host -``` - - -### Running - -If you used `wasi.py` to do your build then there will be a `cross-build/wasm32-wasi/python.sh` file which you can use to run the `python.wasm` file (see the output from the `configure-host` subcommand): -```shell -cross-build/wasm32-wasi/python.sh --version -``` - -While you _can_ run `python.wasm` directly, Python will fail to start up without certain things being set (e.g. `PYTHONPATH` for `sysconfig` data). As such, the `python.sh` file records these details for you. - +See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). ## Detecting WebAssembly builds diff --git a/Tools/wasm/Setup.local.example b/Tools/wasm/Setup.local.example deleted file mode 100644 index 7b2fb13f6ceef2..00000000000000 --- a/Tools/wasm/Setup.local.example +++ /dev/null @@ -1,13 +0,0 @@ -# Module/Setup.local with reduced stdlib -*disabled* -_asyncio -_bz2 -_decimal -_pickle -pyexpat _elementtree -_sha3 _blake2 -_zoneinfo -xxsubtype - -# cjk codecs -#_multibytecodec _codecs_cn _codecs_hk _codecs_iso2022 _codecs_jp _codecs_kr _codecs_tw diff --git a/Tools/wasm/build_wasi.sh b/Tools/wasm/build_wasi.sh deleted file mode 100755 index 436306222ce1d0..00000000000000 --- a/Tools/wasm/build_wasi.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/bash - -set -e -x - -# Quick check to avoid running configure just to fail in the end. -if [ -f Programs/python.o ]; then - echo "Can't do an out-of-tree build w/ an in-place build pre-existing (i.e., found Programs/python.o)" >&2 - exit 1 -fi - -if [ ! -f Modules/Setup.local ]; then - touch Modules/Setup.local -fi - -# TODO: check if `make` and `pkgconfig` are installed -# TODO: detect if wasmtime is installed - -# Create the "build" Python. -mkdir -p builddir/build -pushd builddir/build -../../configure -C -make -s -j 4 all -export PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` -popd - -# Create the "host"/WASI Python. -export CONFIG_SITE="$(pwd)/Tools/wasm/config.site-wasm32-wasi" -export HOSTRUNNER="wasmtime run --mapdir /::$(pwd) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/builddir/wasi/python.wasm --" - -mkdir -p builddir/wasi -pushd builddir/wasi -../../Tools/wasm/wasi-env \ - ../../configure \ - -C \ - --host=wasm32-unknown-wasi \ - --build=$(../../config.guess) \ - --with-build-python=../build/python -make -s -j 4 all -# Create a helper script for executing the host/WASI Python. -printf "#!/bin/sh\nexec $HOSTRUNNER \"\$@\"\n" > run_wasi.sh -chmod 755 run_wasi.sh -./run_wasi.sh --version -popd - diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/config.site-wasm32-wasi index 5e98775400f6ea..c5d8b3e205db26 100644 --- a/Tools/wasm/config.site-wasm32-wasi +++ b/Tools/wasm/config.site-wasm32-wasi @@ -40,3 +40,20 @@ ac_cv_header_netpacket_packet_h=no # Disable int-conversion for wask-sdk as it triggers an error from version 17. ac_cv_disable_int_conversion=yes + +# preadv(), readv(), pwritev(), and writev() under wasmtime's WASI 0.2 support +# do not use more than the first buffer provided, failing under test_posix. +# Since wasmtime will not be changing this behaviour, disable the functions. +# https://github.com/bytecodealliance/wasmtime/issues/7830 +ac_cv_func_preadv=no +ac_cv_func_readv=no +ac_cv_func_pwritev=no +ac_cv_func_writev=no + +# WASI SDK 22 added multiple stubs which we don't implement. +# https://github.com/python/cpython/issues/120371 +ac_cv_func_chmod=no +ac_cv_func_fchmod=no +ac_cv_func_fchmodat=no +ac_cv_func_statvfs=no +ac_cv_func_fstatvfs=no diff --git a/Tools/wasm/emscripten/node_pre.js b/Tools/wasm/emscripten/node_pre.js new file mode 100644 index 00000000000000..3490d3ca591ef6 --- /dev/null +++ b/Tools/wasm/emscripten/node_pre.js @@ -0,0 +1,9 @@ +// If process is undefined, we're not running in the node runtime let it go I +// guess? +if (typeof process !== "undefined") { + const nodeVersion = Number(process.versions.node.split('.',1)[0]); + if (nodeVersion < 18) { + process.stderr.write(`Node version must be >= 18, got version ${process.version}\n`); + process.exit(1); + } +} diff --git a/Tools/wasm/python.html b/Tools/wasm/python.html index 17ffa0ea8bfeff..81a035a5c4cd93 100644 --- a/Tools/wasm/python.html +++ b/Tools/wasm/python.html @@ -35,11 +35,12 @@

    Simple REPL for Python WASM

    -
    +
    + +
    +
    The simple REPL provides a limited Python experience in the browser. diff --git a/Tools/wasm/python.worker.js b/Tools/wasm/python.worker.js index 1b794608fffe7b..4ce4e16fc0fa19 100644 --- a/Tools/wasm/python.worker.js +++ b/Tools/wasm/python.worker.js @@ -19,18 +19,18 @@ class StdinBuffer { } stdin = () => { - if (this.numberOfCharacters + 1 === this.readIndex) { + while (this.numberOfCharacters + 1 === this.readIndex) { if (!this.sentNull) { // Must return null once to indicate we're done for now. this.sentNull = true return null } this.sentNull = false + // Prompt will reset this.readIndex to 1 this.prompt() } const char = this.buffer[this.readIndex] this.readIndex += 1 - // How do I send an EOF?? return char } } @@ -71,7 +71,11 @@ var Module = { onmessage = (event) => { if (event.data.type === 'run') { - // TODO: Set up files from event.data.files + if (event.data.files) { + for (const [filename, contents] of Object.entries(event.data.files)) { + Module.FS.writeFile(filename, contents) + } + } const ret = callMain(event.data.args) postMessage({ type: 'finished', diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 48908b02e60b96..4c5078a1f675e2 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -1,6 +1,8 @@ #!/bin/sh set -e +# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py . + # function usage() { echo "wasi-env - Run command with WASI-SDK" @@ -55,7 +57,6 @@ if command -v ccache >/dev/null 2>&1; then CXX="ccache ${CXX}" fi -LDSHARED="${WASI_SDK_PATH}/bin/wasm-ld" AR="${WASI_SDK_PATH}/bin/llvm-ar" RANLIB="${WASI_SDK_PATH}/bin/ranlib" @@ -72,5 +73,5 @@ export CFLAGS LDFLAGS export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR export PATH -# no exec, it makes arvg[0] path absolute. +# no exec, it makes argv[0] path absolute. "$@" diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 46ecae74a9ecea..050e3723feb247 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -20,12 +20,13 @@ CROSS_BUILD_DIR = CHECKOUT / "cross-build" BUILD_DIR = CROSS_BUILD_DIR / "build" -HOST_TRIPLE = "wasm32-wasi" -HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8") +WASMTIME_VAR_NAME = "WASMTIME" +WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" + def updated_env(updates={}): """Create a new dict representing the environment to use. @@ -60,12 +61,17 @@ def subdir(working_dir, *, clean_ok=False): def decorator(func): @functools.wraps(func) def wrapper(context): + nonlocal working_dir + + if callable(working_dir): + working_dir = working_dir(context) try: tput_output = subprocess.check_output(["tput", "cols"], encoding="utf-8") - terminal_width = int(tput_output.strip()) except subprocess.CalledProcessError: terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) print("⎯" * terminal_width) print("📁", working_dir) if (clean_ok and getattr(context, "clean", False) and @@ -165,7 +171,7 @@ def wasi_sdk_env(context): wasi_sdk_path = context.wasi_sdk_path sysroot = wasi_sdk_path / "share" / "wasi-sysroot" env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", - "LDSHARED": "wasm-ld", "AR": "llvm-ar", "RANLIB": "ranlib"} + "AR": "llvm-ar", "RANLIB": "ranlib"} for env_var, binary_name in list(env.items()): env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) @@ -190,7 +196,7 @@ def wasi_sdk_env(context): return env -@subdir(HOST_DIR, clean_ok=True) +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): @@ -215,18 +221,27 @@ def configure_wasi_python(context, working_dir): # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. - host_runner = context.host_runner.format(GUEST_DIR="/", - HOST_DIR=CHECKOUT, - ENV_VAR_NAME="PYTHONPATH", - ENV_VAR_VALUE=f"/{sysconfig_data}", - PYTHON_WASM=working_dir / "python.wasm") + args = {"GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data}", + "PYTHON_WASM": working_dir / "python.wasm"} + # Check dynamically for wasmtime in case it was specified manually via + # `--host-runner`. + if WASMTIME_HOST_RUNNER_VAR in context.host_runner: + if wasmtime := shutil.which("wasmtime"): + args[WASMTIME_VAR_NAME] = wasmtime + else: + raise FileNotFoundError("wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime") + host_runner = context.host_runner.format_map(args) env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} build_python = os.fsdecode(build_python_path()) # The path to `configure` MUST be relative, else `python.wasm` is unable # to find the stdlib due to Python not recognizing that it's being # executed from within a checkout. configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), - f"--host={HOST_TRIPLE}", + f"--host={context.host_triple}", f"--build={build_platform()}", f"--with-build-python={build_python}"] if pydebug: @@ -246,7 +261,7 @@ def configure_wasi_python(context, working_dir): sys.stdout.flush() -@subdir(HOST_DIR) +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" call(["make", "--jobs", str(cpu_count()), "all"], @@ -277,14 +292,15 @@ def clean_contents(context): def main(): - default_host_runner = (f"{shutil.which('wasmtime')} run " + default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " # Make sure the stack size will work for a pydebug # build. - # The 8388608 value comes from `ulimit -s` under Linux - # which equates to 8291 KiB. - "--wasm max-wasm-stack=8388608 " - # Enable thread support. - "--wasm threads=y --wasi threads=y " + # Use 16 MiB stack. + "--wasm max-wasm-stack=16777216 " + # Use WASI 0.2 primitives. + "--wasi preview2 " + # Enable thread support; causes use of preview1. + #"--wasm threads=y --wasi threads=y " # Map the checkout to / to load the stdlib from /Lib. "--dir {HOST_DIR}::{GUEST_DIR} " # Set PYTHONPATH to the sysconfig data. @@ -329,6 +345,9 @@ def main(): help="Command template for running the WASI host " "(default designed for wasmtime 14 or newer: " f"`{default_host_runner}`)") + for subcommand in build, configure_host, make_host: + subcommand.add_argument("--host-triple", action="store", default="wasm32-wasi", + help="The target triple for the WASI host build") context = parser.parse_args() diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index c0b9999a5dad03..bcb80212362b71 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -329,8 +329,10 @@ def _check_wasi() -> None: # workaround for https://github.com/python/cpython/issues/95952 "HOSTRUNNER": ( "wasmtime run " - "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib " - "--mapdir /::{srcdir} --" + "--wasm max-wasm-stack=16777216 " + "--wasi preview2 " + "--dir {srcdir}::/ " + "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib" ), "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]], }, diff --git a/aclocal.m4 b/aclocal.m4 index 09ae5d1aa8a608..b082a5b1bc5e07 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -41,32 +41,81 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun # If neither value is found, the user is instructed to specify the # ordering. # +# Early versions of this macro (i.e., before serial 12) would not work +# when interprocedural optimization (via link-time optimization) was +# enabled. This would happen when, say, the GCC/clang "-flto" flag, or the +# ICC "-ipo" flag was used, for example. The problem was that under +# these conditions, the compiler did not allocate for and write the special +# float value in the data segment of the object file, since doing so might +# not prove optimal once more context was available. Thus, the special value +# (in platform-dependent binary form) could not be found in the object file, +# and the macro would fail. +# +# The solution to the above problem was to: +# +# 1) Compile and link a whole test program rather than just compile an +# object file. This ensures that we reach the point where even an +# interprocedural optimizing compiler writes values to the data segment. +# +# 2) Add code that requires the compiler to write the special value to +# the data segment, as opposed to "optimizing away" the variable's +# allocation. This could be done via compiler keywords or options, but +# it's tricky to make this work for all versions of all compilers with +# all optimization settings. The chosen solution was to make the exit +# code of the test program depend on the storing of the special value +# in memory (in the data segment). Because the exit code can be +# verified, any compiler that aspires to be correct will produce a +# program binary that contains the value, which the macro can then find. +# +# How does the exit code depend on the special value residing in memory? +# Memory, unlike variables and registers, can be addressed indirectly at run +# time. The exit code of this test program is a result of indirectly reading +# and writing to the memory region where the special value is supposed to +# reside. The actual memory addresses used and the values to be written are +# derived from the the program input ("argv") and are therefore not known at +# compile or link time. The compiler has no choice but to defer the +# computation to run time, and to prepare by allocating and populating the +# data segment with the special value. For further details, refer to the +# source code of the test program. +# +# Note that the test program is never meant to be run. It only exists to host +# a double float value in a given platform's binary format. Thus, error +# handling is not included. +# # LICENSE # -# Copyright (c) 2008 Daniel Amelang +# Copyright (c) 2008, 2023 Daniel Amelang # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 11 +#serial 12 AC_DEFUN([AX_C_FLOAT_WORDS_BIGENDIAN], [AC_CACHE_CHECK(whether float word ordering is bigendian, ax_cv_c_float_words_bigendian, [ ax_cv_c_float_words_bigendian=unknown -AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ +AC_LINK_IFELSE([AC_LANG_SOURCE([[ + +#include + +static double m[] = {9.090423496703681e+223, 0.0}; -double d = 90904234967036810337470478905505011476211692735615632014797120844053488865816695273723469097858056257517020191247487429516932130503560650002327564517570778480236724525140520121371739201496540132640109977779420565776568942592.0; +int main (int argc, char *argv[]) +{ + m[atoi (argv[1])] += atof (argv[2]); + return m[atoi (argv[3])] > 0.0; +} ]])], [ -if grep noonsees conftest.$ac_objext >/dev/null ; then +if grep noonsees conftest$EXEEXT >/dev/null ; then ax_cv_c_float_words_bigendian=yes fi -if grep seesnoon conftest.$ac_objext >/dev/null ; then +if grep seesnoon conftest$EXEEXT >/dev/null ; then if test "$ax_cv_c_float_words_bigendian" = unknown; then ax_cv_c_float_words_bigendian=no else @@ -150,6 +199,80 @@ AS_VAR_IF(CACHEVAR,yes, AS_VAR_POPDEF([CACHEVAR])dnl ])dnl AX_CHECK_COMPILE_FLAGS +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_define.html +# =========================================================================== +# +# SYNOPSIS +# +# AC_CHECK_DEFINE([symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# AX_CHECK_DEFINE([includes],[symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# +# DESCRIPTION +# +# Complements AC_CHECK_FUNC but it does not check for a function but for a +# define to exist. Consider a usage like: +# +# AC_CHECK_DEFINE(__STRICT_ANSI__, CFLAGS="$CFLAGS -D_XOPEN_SOURCE=500") +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +AU_ALIAS([AC_CHECK_DEFINED], [AC_CHECK_DEFINE]) +AC_DEFUN([AC_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$1])dnl +AC_CACHE_CHECK([for $1 defined], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ + #ifdef $1 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$2], [$3])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AU_ALIAS([AX_CHECK_DEFINED], [AX_CHECK_DEFINE]) +AC_DEFUN([AX_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$2_$1])dnl +AC_CACHE_CHECK([for $2 defined in $1], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <$1>]], [[ + #ifdef $2 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AC_DEFUN([AX_CHECK_FUNC], +[AS_VAR_PUSHDEF([ac_var], [ac_cv_func_$2])dnl +AC_CACHE_CHECK([for $2], ac_var, +dnl AC_LANG_FUNC_LINK_TRY +[AC_LINK_IFELSE([AC_LANG_PROGRAM([$1 + #undef $2 + char $2 ();],[ + char (*f) () = $2; + return f != $2; ])], + [AS_VAR_SET(ac_var, yes)], + [AS_VAR_SET(ac_var, no)])]) +AS_IF([test AS_VAR_GET(ac_var) = yes], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +])# AC_CHECK_FUNC + # =========================================================================== # https://www.gnu.org/software/autoconf-archive/ax_check_openssl.html # =========================================================================== diff --git a/config.sub b/config.sub index d74fb6deac942a..1bb6a05dc11026 100755 --- a/config.sub +++ b/config.sub @@ -1,14 +1,15 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2021 Free Software Foundation, Inc. +# Copyright 1992-2024 Free Software Foundation, Inc. # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2021-08-14' +# Patched 2024-02-03 to include support for arm64_32 and iOS/tvOS/watchOS simulators +timestamp='2024-01-01' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or +# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -76,13 +77,13 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2021 Free Software Foundation, Inc. +Copyright 1992-2024 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -130,7 +131,7 @@ IFS=$saved_IFS # Separate into logical components for further validation case $1 in *-*-*-*-*) - echo Invalid configuration \`"$1"\': more than four components >&2 + echo "Invalid configuration '$1': more than four components" >&2 exit 1 ;; *-*-*-*) @@ -145,7 +146,8 @@ case $1 in nto-qnx* | linux-* | uclinux-uclibc* \ | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova*) + | storm-chaos* | os2-emx* | rtmk-nova* | managarm-* \ + | windows-* ) basic_machine=$field1 basic_os=$maybe_os ;; @@ -943,7 +945,7 @@ $basic_machine EOF IFS=$saved_IFS ;; - # We use `pc' rather than `unknown' + # We use 'pc' rather than 'unknown' # because (1) that's what they normally are, and # (2) the word "unknown" tends to confuse beginning users. i*86 | x86_64) @@ -1020,6 +1022,11 @@ case $cpu-$vendor in ;; # Here we normalize CPU types with a missing or matching vendor + armh-unknown | armh-alt) + cpu=armv7l + vendor=alt + basic_os=${basic_os:-linux-gnueabihf} + ;; dpx20-unknown | dpx20-bull) cpu=rs6000 vendor=bull @@ -1070,7 +1077,7 @@ case $cpu-$vendor in pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) cpu=i586 ;; - pentiumpro-* | p6-* | 6x86-* | athlon-* | athalon_*-*) + pentiumpro-* | p6-* | 6x86-* | athlon-* | athlon_*-*) cpu=i686 ;; pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) @@ -1121,7 +1128,7 @@ case $cpu-$vendor in xscale-* | xscalee[bl]-*) cpu=`echo "$cpu" | sed 's/^xscale/arm/'` ;; - arm64-*) + arm64-* | aarch64le-* | arm64_32-*) cpu=aarch64 ;; @@ -1175,7 +1182,7 @@ case $cpu-$vendor in case $cpu in 1750a | 580 \ | a29k \ - | aarch64 | aarch64_be \ + | aarch64 | aarch64_be | aarch64c | arm64ec \ | abacus \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ @@ -1194,50 +1201,29 @@ case $cpu-$vendor in | d10v | d30v | dlx | dsp16xx \ | e2k | elxsi | epiphany \ | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | javascript \ | h8300 | h8500 \ | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ | i370 | i*86 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ + | kvx \ | le32 | le64 \ | lm32 \ - | loongarch32 | loongarch64 | loongarchx32 \ + | loongarch32 | loongarch64 \ | m32c | m32r | m32rle \ | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ | m88110 | m88k | maxq | mb | mcore | mep | metag \ | microblaze | microblazeel \ - | mips | mipsbe | mipseb | mipsel | mipsle \ - | mips16 \ - | mips64 | mips64eb | mips64el \ - | mips64octeon | mips64octeonel \ - | mips64orion | mips64orionel \ - | mips64r5900 | mips64r5900el \ - | mips64vr | mips64vrel \ - | mips64vr4100 | mips64vr4100el \ - | mips64vr4300 | mips64vr4300el \ - | mips64vr5000 | mips64vr5000el \ - | mips64vr5900 | mips64vr5900el \ - | mipsisa32 | mipsisa32el \ - | mipsisa32r2 | mipsisa32r2el \ - | mipsisa32r3 | mipsisa32r3el \ - | mipsisa32r5 | mipsisa32r5el \ - | mipsisa32r6 | mipsisa32r6el \ - | mipsisa64 | mipsisa64el \ - | mipsisa64r2 | mipsisa64r2el \ - | mipsisa64r3 | mipsisa64r3el \ - | mipsisa64r5 | mipsisa64r5el \ - | mipsisa64r6 | mipsisa64r6el \ - | mipsisa64sb1 | mipsisa64sb1el \ - | mipsisa64sr71k | mipsisa64sr71kel \ - | mipsr5900 | mipsr5900el \ - | mipstx39 | mipstx39el \ + | mips* \ | mmix \ | mn10200 | mn10300 \ | moxie \ | mt \ | msp430 \ + | nanomips* \ | nds32 | nds32le | nds32be \ | nfp \ | nios | nios2 | nios2eb | nios2el \ @@ -1269,6 +1255,7 @@ case $cpu-$vendor in | ubicom32 \ | v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \ | vax \ + | vc4 \ | visium \ | w65 \ | wasm32 | wasm64 \ @@ -1280,7 +1267,7 @@ case $cpu-$vendor in ;; *) - echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + echo "Invalid configuration '$1': machine '$cpu-$vendor' not recognized" 1>&2 exit 1 ;; esac @@ -1301,11 +1288,12 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if test x$basic_os != x +if test x"$basic_os" != x then -# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just +# First recognize some ad-hoc cases, or perhaps split kernel-os, or else just # set os. +obj= case $basic_os in gnu/linux*) kernel=linux @@ -1336,6 +1324,10 @@ EOF kernel=linux os=`echo "$basic_os" | sed -e 's|linux|gnu|'` ;; + managarm*) + kernel=managarm + os=`echo "$basic_os" | sed -e 's|managarm|mlibc|'` + ;; *) kernel= os=$basic_os @@ -1501,10 +1493,16 @@ case $os in os=eabi ;; *) - os=elf + os= + obj=elf ;; esac ;; + aout* | coff* | elf* | pe*) + # These are machine code file formats, not OSes + obj=$os + os= + ;; *) # No normalization, but not necessarily accepted, that comes below. ;; @@ -1523,12 +1521,15 @@ else # system, and we'll never get to this point. kernel= +obj= case $cpu-$vendor in score-*) - os=elf + os= + obj=elf ;; spu-*) - os=elf + os= + obj=elf ;; *-acorn) os=riscix1.2 @@ -1538,28 +1539,35 @@ case $cpu-$vendor in os=gnu ;; arm*-semi) - os=aout + os= + obj=aout ;; c4x-* | tic4x-*) - os=coff + os= + obj=coff ;; c8051-*) - os=elf + os= + obj=elf ;; clipper-intergraph) os=clix ;; hexagon-*) - os=elf + os= + obj=elf ;; tic54x-*) - os=coff + os= + obj=coff ;; tic55x-*) - os=coff + os= + obj=coff ;; tic6x-*) - os=coff + os= + obj=coff ;; # This must come before the *-dec entry. pdp10-*) @@ -1581,19 +1589,24 @@ case $cpu-$vendor in os=sunos3 ;; m68*-cisco) - os=aout + os= + obj=aout ;; mep-*) - os=elf + os= + obj=elf ;; mips*-cisco) - os=elf + os= + obj=elf ;; - mips*-*) - os=elf + mips*-*|nanomips*-*) + os= + obj=elf ;; or32-*) - os=coff + os= + obj=coff ;; *-tti) # must be before sparc entry or we get the wrong os. os=sysv3 @@ -1602,7 +1615,8 @@ case $cpu-$vendor in os=sunos4.1.1 ;; pru-*) - os=elf + os= + obj=elf ;; *-be) os=beos @@ -1683,10 +1697,12 @@ case $cpu-$vendor in os=uxpv ;; *-rom68k) - os=coff + os= + obj=coff ;; *-*bug) - os=coff + os= + obj=coff ;; *-apple) os=macos @@ -1704,10 +1720,11 @@ esac fi -# Now, validate our (potentially fixed-up) OS. +# Now, validate our (potentially fixed-up) individual pieces (OS, OBJ). + case $os in # Sometimes we do "kernel-libc", so those need to count as OSes. - musl* | newlib* | relibc* | uclibc*) + llvm* | musl* | newlib* | relibc* | uclibc*) ;; # Likewise for "kernel-abi" eabi* | gnueabi*) @@ -1715,6 +1732,9 @@ case $os in # VxWorks passes extra cpu info in the 4th filed. simlinux | simwindows | spe) ;; + # See `case $cpu-$os` validation below + ghcjs) + ;; # Now accept the basic system types. # The portable systems comes first. # Each alternative MUST end in a * to match a version number. @@ -1723,7 +1743,7 @@ case $os in | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ | sym* | plan9* | psp* | sim* | xray* | os68k* | v88r* \ | hiux* | abug | nacl* | netware* | windows* \ - | os9* | macos* | osx* | ios* \ + | os9* | macos* | osx* | ios* | tvos* | watchos* \ | mpw* | magic* | mmixware* | mon960* | lnews* \ | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ | aos* | aros* | cloudabi* | sortix* | twizzler* \ @@ -1732,11 +1752,11 @@ case $os in | mirbsd* | netbsd* | dicos* | openedition* | ose* \ | bitrig* | openbsd* | secbsd* | solidbsd* | libertybsd* | os108* \ | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \ - | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ - | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | bosx* | nextstep* | cxux* | oabi* \ + | ptx* | ecoff* | winnt* | domain* | vsta* \ | udi* | lites* | ieee* | go32* | aux* | hcos* \ | chorusrdb* | cegcc* | glidix* | serenity* \ - | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | cygwin* | msys* | moss* | proelf* | rtems* \ | midipix* | mingw32* | mingw64* | mint* \ | uxpv* | beos* | mpeix* | udk* | moxiebox* \ | interix* | uwin* | mks* | rhapsody* | darwin* \ @@ -1748,49 +1768,119 @@ case $os in | skyos* | haiku* | rdos* | toppers* | drops* | es* \ | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ - | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx* | zephyr*) + | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx* | zephyr* \ + | fiwix* | mlibc* | cos* | mbr* | ironclad* ) ;; # This one is extra strict with allowed versions sco3.2v2 | sco3.2v[4-9]* | sco5v6*) # Don't forget version if it is 3.2v4 or newer. ;; + # This refers to builds using the UEFI calling convention + # (which depends on the architecture) and PE file format. + # Note that this is both a different calling convention and + # different file format than that of GNU-EFI + # (x86_64-w64-mingw32). + uefi) + ;; none) ;; + kernel* | msvc* ) + # Restricted further below + ;; + '') + if test x"$obj" = x + then + echo "Invalid configuration '$1': Blank OS only allowed with explicit machine code file format" 1>&2 + fi + ;; *) - echo Invalid configuration \`"$1"\': OS \`"$os"\' not recognized 1>&2 + echo "Invalid configuration '$1': OS '$os' not recognized" 1>&2 + exit 1 + ;; +esac + +case $obj in + aout* | coff* | elf* | pe*) + ;; + '') + # empty is fine + ;; + *) + echo "Invalid configuration '$1': Machine code format '$obj' not recognized" 1>&2 + exit 1 + ;; +esac + +# Here we handle the constraint that a (synthetic) cpu and os are +# valid only in combination with each other and nowhere else. +case $cpu-$os in + # The "javascript-unknown-ghcjs" triple is used by GHC; we + # accept it here in order to tolerate that, but reject any + # variations. + javascript-ghcjs) + ;; + javascript-* | *-ghcjs) + echo "Invalid configuration '$1': cpu '$cpu' is not valid with os '$os$obj'" 1>&2 exit 1 ;; esac # As a final step for OS-related things, validate the OS-kernel combination # (given a valid OS), if there is a kernel. -case $kernel-$os in - linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* \ - | linux-musl* | linux-relibc* | linux-uclibc* ) +case $kernel-$os-$obj in + linux-gnu*- | linux-android*- | linux-dietlibc*- | linux-llvm*- \ + | linux-mlibc*- | linux-musl*- | linux-newlib*- \ + | linux-relibc*- | linux-uclibc*- ) + ;; + uclinux-uclibc*- ) ;; - uclinux-uclibc* ) + managarm-mlibc*- | managarm-kernel*- ) ;; - -dietlibc* | -newlib* | -musl* | -relibc* | -uclibc* ) + windows*-msvc*-) + ;; + -dietlibc*- | -llvm*- | -mlibc*- | -musl*- | -newlib*- | -relibc*- \ + | -uclibc*- ) # These are just libc implementations, not actual OSes, and thus # require a kernel. - echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 + echo "Invalid configuration '$1': libc '$os' needs explicit kernel." 1>&2 exit 1 ;; - kfreebsd*-gnu* | kopensolaris*-gnu*) + -kernel*- ) + echo "Invalid configuration '$1': '$os' needs explicit kernel." 1>&2 + exit 1 ;; - vxworks-simlinux | vxworks-simwindows | vxworks-spe) + *-kernel*- ) + echo "Invalid configuration '$1': '$kernel' does not support '$os'." 1>&2 + exit 1 ;; - nto-qnx*) + *-msvc*- ) + echo "Invalid configuration '$1': '$os' needs 'windows'." 1>&2 + exit 1 ;; - os2-emx) + kfreebsd*-gnu*- | kopensolaris*-gnu*-) ;; - *-eabi* | *-gnueabi*) + vxworks-simlinux- | vxworks-simwindows- | vxworks-spe-) ;; - -*) + nto-qnx*-) + ;; + os2-emx-) + ;; + *-eabi*- | *-gnueabi*-) + ;; + ios*-simulator- | tvos*-simulator- | watchos*-simulator- ) + ;; + none--*) + # None (no kernel, i.e. freestanding / bare metal), + # can be paired with an machine code file format + ;; + -*-) # Blank kernel with real OS is always fine. ;; - *-*) - echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 + --*) + # Blank kernel and OS with real machine code file format is always fine. + ;; + *-*-*) + echo "Invalid configuration '$1': Kernel '$kernel' not known to work with OS '$os'." 1>&2 exit 1 ;; esac @@ -1873,7 +1963,7 @@ case $vendor in ;; esac -echo "$cpu-$vendor-${kernel:+$kernel-}$os" +echo "$cpu-$vendor${kernel:+-$kernel}${os:+-$os}${obj:+-$obj}" exit # Local variables: diff --git a/configure b/configure index 705a778cafced3..1097747e055179 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for python 3.13. +# Generated by GNU Autoconf 2.71 for python 3.14. # # Report bugs to . # @@ -611,8 +611,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='python' PACKAGE_TARNAME='python' -PACKAGE_VERSION='3.13' -PACKAGE_STRING='python 3.13' +PACKAGE_VERSION='3.14' +PACKAGE_STRING='python 3.14' PACKAGE_BUGREPORT='https://github.com/python/cpython/issues/' PACKAGE_URL='' @@ -661,6 +661,10 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE +MODULE__TESTEXTERNALINSPECTION_FALSE +MODULE__TESTEXTERNALINSPECTION_TRUE +MODULE__TESTSINGLEPHASE_FALSE +MODULE__TESTSINGLEPHASE_TRUE MODULE__TESTMULTIPHASE_FALSE MODULE__TESTMULTIPHASE_TRUE MODULE__TESTIMPORTMULTIPLE_FALSE @@ -669,6 +673,8 @@ MODULE__TESTBUFFER_FALSE MODULE__TESTBUFFER_TRUE MODULE__TESTINTERNALCAPI_FALSE MODULE__TESTINTERNALCAPI_TRUE +MODULE__TESTLIMITEDCAPI_FALSE +MODULE__TESTLIMITEDCAPI_TRUE MODULE__TESTCLINIC_LIMITED_FALSE MODULE__TESTCLINIC_LIMITED_TRUE MODULE__TESTCLINIC_FALSE @@ -707,6 +713,11 @@ MODULE__CURSES_FALSE MODULE__CURSES_TRUE MODULE__CTYPES_FALSE MODULE__CTYPES_TRUE +LIBHACL_SIMD256_OBJS +LIBHACL_SIMD256_FLAGS +LIBHACL_SIMD128_OBJS +LIBHACL_SIMD128_FLAGS +LIBHACL_CFLAGS MODULE__BLAKE2_FALSE MODULE__BLAKE2_TRUE MODULE__SHA3_FALSE @@ -769,12 +780,12 @@ MODULE__MULTIPROCESSING_FALSE MODULE__MULTIPROCESSING_TRUE MODULE__ZONEINFO_FALSE MODULE__ZONEINFO_TRUE -MODULE__XXINTERPQUEUES_FALSE -MODULE__XXINTERPQUEUES_TRUE -MODULE__XXINTERPCHANNELS_FALSE -MODULE__XXINTERPCHANNELS_TRUE -MODULE__XXSUBINTERPRETERS_FALSE -MODULE__XXSUBINTERPRETERS_TRUE +MODULE__INTERPQUEUES_FALSE +MODULE__INTERPQUEUES_TRUE +MODULE__INTERPCHANNELS_FALSE +MODULE__INTERPCHANNELS_TRUE +MODULE__INTERPRETERS_FALSE +MODULE__INTERPRETERS_TRUE MODULE__TYPING_FALSE MODULE__TYPING_TRUE MODULE__STRUCT_FALSE @@ -789,8 +800,6 @@ MODULE__POSIXSUBPROCESS_FALSE MODULE__POSIXSUBPROCESS_TRUE MODULE__PICKLE_FALSE MODULE__PICKLE_TRUE -MODULE__OPCODE_FALSE -MODULE__OPCODE_TRUE MODULE__LSPROF_FALSE MODULE__LSPROF_TRUE MODULE__JSON_FALSE @@ -813,8 +822,6 @@ MODULE__IO_FALSE MODULE__IO_TRUE MODULE_BUILDTYPE TEST_MODULES -LIBB2_LIBS -LIBB2_CFLAGS OPENSSL_LDFLAGS OPENSSL_LIBS OPENSSL_INCLUDES @@ -835,6 +842,7 @@ PY_ENABLE_SHARED PLATLIBDIR BINLIBDEST LIBPYTHON +MODULE_DEPS_SHARED EXT_SUFFIX ALT_SOABI SOABI @@ -862,7 +870,7 @@ DTRACE_OBJS DTRACE_HEADERS DFLAGS DTRACE -WITH_MIMALLOC +INSTALL_MIMALLOC MIMALLOC_HEADERS GDBM_LIBS GDBM_CFLAGS @@ -873,6 +881,7 @@ TCLTK_CFLAGS LIBSQLITE3_LIBS LIBSQLITE3_CFLAGS LIBMPDEC_INTERNAL +LIBMPDEC_LIBS LIBMPDEC_CFLAGS MODULE__CTYPES_MALLOC_CLOSURE LIBFFI_LIBS @@ -922,6 +931,7 @@ DEF_MAKE_RULE DEF_MAKE_ALL_RULE JIT_STENCILS_H REGEN_JIT_COMMAND +ABI_THREAD ABIFLAGS LN MKDIR_P @@ -969,10 +979,12 @@ LDFLAGS CFLAGS CC HAS_XCRUN +IPHONEOS_DEPLOYMENT_TARGET EXPORT_MACOSX_DEPLOYMENT_TARGET CONFIGURE_MACOSX_DEPLOYMENT_TARGET _PYTHON_HOST_PLATFORM -MACHDEP +APP_STORE_COMPLIANCE_PATCH +INSTALLTARGETS FRAMEWORKINSTALLAPPSPREFIX FRAMEWORKUNIXTOOLSPREFIX FRAMEWORKPYTHONW @@ -980,6 +992,8 @@ FRAMEWORKALTINSTALLLAST FRAMEWORKALTINSTALLFIRST FRAMEWORKINSTALLLAST FRAMEWORKINSTALLFIRST +RESSRCDIR +PYTHONFRAMEWORKINSTALLNAMEPREFIX PYTHONFRAMEWORKINSTALLDIR PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKDIR @@ -989,6 +1003,9 @@ LIPO_INTEL64_FLAGS LIPO_32BIT_FLAGS ARCH_RUN_32BIT UNIVERSALSDK +host_exec_prefix +host_prefix +MACHDEP PKG_CONFIG_LIBDIR PKG_CONFIG_PATH PKG_CONFIG @@ -1064,6 +1081,7 @@ enable_universalsdk with_universal_archs with_framework_name enable_framework +with_app_store_compliance with_emscripten_target enable_wasm_dynamic_linking enable_wasm_pthreads @@ -1081,6 +1099,8 @@ enable_optimizations with_lto enable_bolt with_strict_overflow +enable_safety +enable_slower_safety with_dsymutil with_address_sanitizer with_memory_sanitizer @@ -1098,7 +1118,6 @@ enable_ipv6 with_doc_strings with_mimalloc with_pymalloc -with_freelists with_c_locale_coercion with_valgrind with_dtrace @@ -1137,6 +1156,8 @@ LIBUUID_CFLAGS LIBUUID_LIBS LIBFFI_CFLAGS LIBFFI_LIBS +LIBMPDEC_CFLAGS +LIBMPDEC_LIBS LIBSQLITE3_CFLAGS LIBSQLITE3_LIBS TCLTK_CFLAGS @@ -1158,9 +1179,7 @@ LIBEDIT_LIBS CURSES_CFLAGS CURSES_LIBS PANEL_CFLAGS -PANEL_LIBS -LIBB2_CFLAGS -LIBB2_LIBS' +PANEL_LIBS' # Initialize some variables set by options. @@ -1709,7 +1728,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures python 3.13 to adapt to many kinds of systems. +\`configure' configures python 3.14 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1775,7 +1794,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of python 3.13:";; + short | recursive ) echo "Configuration of python 3.14:";; esac cat <<\_ACEOF @@ -1804,13 +1823,17 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) - --enable-experimental-jit + --enable-experimental-jit[=no|yes|yes-off|interpreter] build the experimental just-in-time compiler (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer (default is no) + --enable-safety enable usage of the security compiler options with + no performance overhead + --enable-slower-safety enable usage of the security compiler options with + performance overhead --enable-loadable-sqlite-extensions support loadable extensions in the sqlite3 module, see Doc/library/sqlite3.rst (default is no) @@ -1824,9 +1847,9 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-build-python=python3.13 + --with-build-python=python3.14 path to build python binary for cross compiling - (default: _bootstrap_python or python3.13) + (default: _bootstrap_python or python3.14) --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) @@ -1841,6 +1864,10 @@ Optional Packages: specify the name for the python framework on macOS only valid when --enable-framework is set. see Mac/README.rst (default is 'Python') + --with-app-store-compliance=[PATCH-FILE] + Enable any patches required for compiliance with app + stores. Optional PATCH-FILE specifies the custom + patch to apply. --with-emscripten-target=[browser|node] Emscripten platform --with-suffix=SUFFIX set executable suffix to SUFFIX (default is empty, @@ -1878,8 +1905,9 @@ Optional Packages: --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) - --with-system-libmpdec build _decimal module using an installed libmpdec - library, see Doc/library/decimal.rst (default is no) + --with-system-libmpdec build _decimal module using an installed mpdecimal + library, see Doc/library/decimal.rst (default is + yes) --with-decimal-contextvar build _decimal module using a coroutine-local rather than a thread-local context (default is yes) @@ -1891,7 +1919,6 @@ Optional Packages: --with-mimalloc build with mimalloc memory allocator (default is yes if C11 stdatomic.h is available.) --with-pymalloc enable specialized mallocs (default is yes) - --with-freelists enable object freelists (default is yes) --with-c-locale-coercion enable C locale coercion to a UTF-8 based locale (default is yes) @@ -1958,6 +1985,10 @@ Some influential environment variables: LIBFFI_CFLAGS C compiler flags for LIBFFI, overriding pkg-config LIBFFI_LIBS linker flags for LIBFFI, overriding pkg-config + LIBMPDEC_CFLAGS + C compiler flags for LIBMPDEC, overriding pkg-config + LIBMPDEC_LIBS + linker flags for LIBMPDEC, overriding pkg-config LIBSQLITE3_CFLAGS C compiler flags for LIBSQLITE3, overriding pkg-config LIBSQLITE3_LIBS @@ -1992,9 +2023,6 @@ Some influential environment variables: PANEL_CFLAGS C compiler flags for PANEL, overriding pkg-config PANEL_LIBS linker flags for PANEL, overriding pkg-config - LIBB2_CFLAGS - C compiler flags for LIBB2, overriding pkg-config - LIBB2_LIBS linker flags for LIBB2, overriding pkg-config Use these variables to override the choices made by `configure' or to help it to find libraries and programs with nonstandard names/locations. @@ -2063,7 +2091,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -python configure 3.13 +python configure 3.14 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -2720,7 +2748,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by python $as_me 3.13, which was +It was created by python $as_me 3.14, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -3682,7 +3710,7 @@ fi fi ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_build_python" >&5 printf "%s\n" "$with_build_python" >&6; } @@ -3807,7 +3835,7 @@ rm confdefs.h mv confdefs.h.new confdefs.h -VERSION=3.13 +VERSION=3.14 # Version number of Python's own shared library file. @@ -4004,6 +4032,150 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then as_fn_error $? "pkg-config is required" "$LINENO" 5] fi +# Set name for machine-dependent library files + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking MACHDEP" >&5 +printf %s "checking MACHDEP... " >&6; } +if test -z "$MACHDEP" +then + # avoid using uname for cross builds + if test "$cross_compiling" = yes; then + # ac_sys_system and ac_sys_release are used for setting + # a lot of different things including 'define_xopen_source' + # in the case statement below. + case "$host" in + *-*-linux-android*) + ac_sys_system=Linux-android + ;; + *-*-linux*) + ac_sys_system=Linux + ;; + *-*-cygwin*) + ac_sys_system=Cygwin + ;; + *-apple-ios*) + ac_sys_system=iOS + ;; + *-*-vxworks*) + ac_sys_system=VxWorks + ;; + *-*-emscripten) + ac_sys_system=Emscripten + ;; + *-*-wasi) + ac_sys_system=WASI + ;; + *) + # for now, limit cross builds to known configurations + MACHDEP="unknown" + as_fn_error $? "cross build not supported for $host" "$LINENO" 5 + esac + ac_sys_release= + else + ac_sys_system=`uname -s` + if test "$ac_sys_system" = "AIX" \ + -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then + ac_sys_release=`uname -v` + else + ac_sys_release=`uname -r` + fi + fi + ac_md_system=`echo $ac_sys_system | + tr -d '/ ' | tr '[A-Z]' '[a-z]'` + ac_md_release=`echo $ac_sys_release | + tr -d '/ ' | sed 's/^[A-Z]\.//' | sed 's/\..*//'` + MACHDEP="$ac_md_system$ac_md_release" + + case $MACHDEP in + aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; + linux*) MACHDEP="linux";; + cygwin*) MACHDEP="cygwin";; + darwin*) MACHDEP="darwin";; + '') MACHDEP="unknown";; + esac + + if test "$ac_sys_system" = "SunOS"; then + # For Solaris, there isn't an OS version specific macro defined + # in most compilers, so we define one here. + SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\(0-9\)$!.0\1!g' | tr -d '.'` + +printf "%s\n" "#define Py_SUNOS_VERSION $SUNOS_VERSION" >>confdefs.h + + fi +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: \"$MACHDEP\"" >&5 +printf "%s\n" "\"$MACHDEP\"" >&6; } + + +if test -z "$host_prefix"; then + case $ac_sys_system in #( + Emscripten) : + host_prefix=/ ;; #( + *) : + host_prefix='${prefix}' + ;; +esac +fi + + +if test -z "$host_exec_prefix"; then + case $ac_sys_system in #( + Emscripten) : + host_exec_prefix=$host_prefix ;; #( + *) : + host_exec_prefix='${exec_prefix}' + ;; +esac +fi + + +# On cross-compile builds, configure will look for a host-specific compiler by +# prepending the user-provided host triple to the required binary name. +# +# On iOS, this results in binaries like "arm64-apple-ios13.0-simulator-gcc", +# which isn't a binary that exists, and isn't very convenient, as it contains the +# iOS version. As the default cross-compiler name won't exist, configure falls +# back to gcc, which *definitely* won't work. We're providing wrapper scripts for +# these tools; the binary names of these scripts are better defaults than "gcc". +# This only requires that the user put the platform scripts folder (e.g., +# "iOS/Resources/bin") in their path, rather than defining platform-specific +# names/paths for AR, CC, CPP, and CXX explicitly; and if the user forgets to +# either put the platform scripts folder in the path, or specify CC etc, +# configure will fail. +if test -z "$AR"; then + case "$host" in + aarch64-apple-ios*-simulator) AR=arm64-apple-ios-simulator-ar ;; + aarch64-apple-ios*) AR=arm64-apple-ios-ar ;; + x86_64-apple-ios*-simulator) AR=x86_64-apple-ios-simulator-ar ;; + *) + esac +fi +if test -z "$CC"; then + case "$host" in + aarch64-apple-ios*-simulator) CC=arm64-apple-ios-simulator-clang ;; + aarch64-apple-ios*) CC=arm64-apple-ios-clang ;; + x86_64-apple-ios*-simulator) CC=x86_64-apple-ios-simulator-clang ;; + *) + esac +fi +if test -z "$CPP"; then + case "$host" in + aarch64-apple-ios*-simulator) CPP=arm64-apple-ios-simulator-cpp ;; + aarch64-apple-ios*) CPP=arm64-apple-ios-cpp ;; + x86_64-apple-ios*-simulator) CPP=x86_64-apple-ios-simulator-cpp ;; + *) + esac +fi +if test -z "$CXX"; then + case "$host" in + aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang++ ;; + aarch64-apple-ios*) CXX=arm64-apple-ios-clang++ ;; + x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang++ ;; + *) + esac +fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-universalsdk" >&5 printf %s "checking for --enable-universalsdk... " >&6; } # Check whether --enable-universalsdk was given. @@ -4119,109 +4291,154 @@ then : enableval=$enable_framework; case $enableval in yes) - enableval=/Library/Frameworks + case $ac_sys_system in + Darwin) enableval=/Library/Frameworks ;; + iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + *) as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 + esac esac + case $enableval in no) - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - if test "x${prefix}" = "xNONE"; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= + case $ac_sys_system in + iOS) as_fn_error $? "iOS builds must use --enable-framework" "$LINENO" 5 ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE"; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac ;; *) PYTHONFRAMEWORKPREFIX="${enableval}" PYTHONFRAMEWORKINSTALLDIR=$PYTHONFRAMEWORKPREFIX/$PYTHONFRAMEWORKDIR - FRAMEWORKINSTALLFIRST="frameworkinstallstructure" - FRAMEWORKALTINSTALLFIRST="frameworkinstallstructure " - FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" - FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" - FRAMEWORKPYTHONW="frameworkpythonw" - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + case $ac_sys_system in #( + Darwin) : + FRAMEWORKINSTALLFIRST="frameworkinstallversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" + FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" + FRAMEWORKPYTHONW="frameworkpythonw" + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi + case "${enableval}" in + /System*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + if test "${prefix}" = "NONE" ; then + # See below + FRAMEWORKUNIXTOOLSPREFIX="/usr" + fi + ;; + + /Library*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + + */Library/Frameworks) + MDIR="`dirname "${enableval}"`" + MDIR="`dirname "${MDIR}"`" + FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" + + if test "${prefix}" = "NONE"; then + # User hasn't specified the + # --prefix option, but wants to install + # the framework in a non-default location, + # ensure that the compatibility links get + # installed relative to that prefix as well + # instead of in /usr/local. + FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" + fi + ;; - case "${enableval}" in - /System*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "${prefix}" = "NONE" ; then - # See below - FRAMEWORKUNIXTOOLSPREFIX="/usr" - fi - ;; + *) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + esac - /Library*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; + prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION + PYTHONFRAMEWORKINSTALLNAMEPREFIX=${prefix} + RESSRCDIR=Mac/Resources/framework - */Library/Frameworks) - MDIR="`dirname "${enableval}"`" - MDIR="`dirname "${MDIR}"`" - FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" - - if test "${prefix}" = "NONE"; then - # User hasn't specified the - # --prefix option, but wants to install - # the framework in a non-default location, - # ensure that the compatibility links get - # installed relative to that prefix as well - # instead of in /usr/local. - FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" - fi - ;; + # Add files for Mac specific code to the list of output + # files: + ac_config_files="$ac_config_files Mac/Makefile" - *) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; - esac + ac_config_files="$ac_config_files Mac/PythonLauncher/Makefile" - prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION + ac_config_files="$ac_config_files Mac/Resources/framework/Info.plist" - # Add files for Mac specific code to the list of output - # files: - ac_config_files="$ac_config_files Mac/Makefile" + ac_config_files="$ac_config_files Mac/Resources/app/Info.plist" - ac_config_files="$ac_config_files Mac/PythonLauncher/Makefile" + ;; + iOS) : + FRAMEWORKINSTALLFIRST="frameworkinstallunversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallunversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKALTINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKPYTHONW= + INSTALLTARGETS="libinstall inclinstall sharedinstall" - ac_config_files="$ac_config_files Mac/Resources/framework/Info.plist" + prefix=$PYTHONFRAMEWORKPREFIX + PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" + RESSRCDIR=iOS/Resources - ac_config_files="$ac_config_files Mac/Resources/app/Info.plist" + ac_config_files="$ac_config_files iOS/Resources/Info.plist" - esac + ;; + *) + as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 + ;; + esac + esac else $as_nop - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= - + case $ac_sys_system in + iOS) as_fn_error $? "iOS builds must use --enable-framework" "$LINENO" 5 ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac fi @@ -4239,79 +4456,58 @@ fi -printf "%s\n" "#define _PYTHONFRAMEWORK \"${PYTHONFRAMEWORK}\"" >>confdefs.h -# Set name for machine-dependent library files -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking MACHDEP" >&5 -printf %s "checking MACHDEP... " >&6; } -if test -z "$MACHDEP" -then - # avoid using uname for cross builds - if test "$cross_compiling" = yes; then - # ac_sys_system and ac_sys_release are used for setting - # a lot of different things including 'define_xopen_source' - # in the case statement below. - case "$host" in - *-*-linux-android*) - ac_sys_system=Linux-android - ;; - *-*-linux*) - ac_sys_system=Linux - ;; - *-*-cygwin*) - ac_sys_system=Cygwin - ;; - *-*-vxworks*) - ac_sys_system=VxWorks - ;; - *-*-emscripten) - ac_sys_system=Emscripten - ;; - *-*-wasi) - ac_sys_system=WASI - ;; - *) - # for now, limit cross builds to known configurations - MACHDEP="unknown" - as_fn_error $? "cross build not supported for $host" "$LINENO" 5 - esac - ac_sys_release= - else - ac_sys_system=`uname -s` - if test "$ac_sys_system" = "AIX" \ - -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then - ac_sys_release=`uname -v` - else - ac_sys_release=`uname -r` - fi - fi - ac_md_system=`echo $ac_sys_system | - tr -d '/ ' | tr '[A-Z]' '[a-z]'` - ac_md_release=`echo $ac_sys_release | - tr -d '/ ' | sed 's/^[A-Z]\.//' | sed 's/\..*//'` - MACHDEP="$ac_md_system$ac_md_release" +printf "%s\n" "#define _PYTHONFRAMEWORK \"${PYTHONFRAMEWORK}\"" >>confdefs.h - case $MACHDEP in - aix*) MACHDEP="aix";; - linux*) MACHDEP="linux";; - cygwin*) MACHDEP="cygwin";; - darwin*) MACHDEP="darwin";; - '') MACHDEP="unknown";; + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-app-store-compliance" >&5 +printf %s "checking for --with-app-store-compliance... " >&6; } + +# Check whether --with-app_store_compliance was given. +if test ${with_app_store_compliance+y} +then : + withval=$with_app_store_compliance; + case "$withval" in + yes) + case $ac_sys_system in + Darwin|iOS) + # iOS is able to share the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + ;; + *) as_fn_error $? "no default app store compliance patch available for $ac_sys_system" "$LINENO" 5 ;; + esac + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying default app store compliance patch" >&5 +printf "%s\n" "applying default app store compliance patch" >&6; } + ;; + *) + APP_STORE_COMPLIANCE_PATCH="${withval}" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying custom app store compliance patch" >&5 +printf "%s\n" "applying custom app store compliance patch" >&6; } + ;; esac - if test "$ac_sys_system" = "SunOS"; then - # For Solaris, there isn't an OS version specific macro defined - # in most compilers, so we define one here. - SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\(0-9\)$!.0\1!g' | tr -d '.'` +else $as_nop -printf "%s\n" "#define Py_SUNOS_VERSION $SUNOS_VERSION" >>confdefs.h + case $ac_sys_system in + iOS) + # Always apply the compliance patch on iOS; we can use the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying default app store compliance patch" >&5 +printf "%s\n" "applying default app store compliance patch" >&6; } + ;; + *) + # No default app compliance patching on any other platform + APP_STORE_COMPLIANCE_PATCH= + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not patching for app store compliance" >&5 +printf "%s\n" "not patching for app store compliance" >&6; } + ;; + esac - fi fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: \"$MACHDEP\"" >&5 -printf "%s\n" "\"$MACHDEP\"" >&6; } + + if test "$cross_compiling" = yes; then @@ -4319,27 +4515,49 @@ if test "$cross_compiling" = yes; then *-*-linux*) case "$host_cpu" in arm*) - _host_cpu=arm + _host_ident=arm ;; *) - _host_cpu=$host_cpu + _host_ident=$host_cpu esac ;; *-*-cygwin*) - _host_cpu= + _host_ident= + ;; + *-apple-ios*) + _host_os=`echo $host | cut -d '-' -f3` + _host_device=`echo $host | cut -d '-' -f4` + _host_device=${_host_device:=os} + + # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking iOS deployment target" >&5 +printf %s "checking iOS deployment target... " >&6; } + IPHONEOS_DEPLOYMENT_TARGET=$(echo ${_host_os} | cut -c4-) + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=13.0} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $IPHONEOS_DEPLOYMENT_TARGET" >&5 +printf "%s\n" "$IPHONEOS_DEPLOYMENT_TARGET" >&6; } + + case "$host_cpu" in + aarch64) + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + ;; + *) + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + ;; + esac ;; *-*-vxworks*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; wasm32-*-* | wasm64-*-*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; *) # for now, limit cross builds to known configurations MACHDEP="unknown" as_fn_error $? "cross build not supported for $host" "$LINENO" 5 esac - _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + _PYTHON_HOST_PLATFORM="$MACHDEP${_host_ident:+-$_host_ident}" fi # Some systems cannot stand _XOPEN_SOURCE being defined at all; they @@ -4406,6 +4624,9 @@ printf "%s\n" "#define _BSD_SOURCE 1" >>confdefs.h define_xopen_source=no;; Darwin/[12][0-9].*) define_xopen_source=no;; + # On iOS, defining _POSIX_C_SOURCE also disables platform specific features. + iOS/*) + define_xopen_source=no;; # On QNX 6.3.2, defining _XOPEN_SOURCE prevents netdb.h from # defining NI_NUMERICHOST. QNX/6.3.2) @@ -4468,6 +4689,9 @@ fi CONFIGURE_MACOSX_DEPLOYMENT_TARGET= EXPORT_MACOSX_DEPLOYMENT_TARGET='#' +# Record the value of IPHONEOS_DEPLOYMENT_TARGET enforced by the selected host triple. + + # checks for alternative programs # compiler flags are generated in two sets, BASECFLAGS and OPT. OPT is just @@ -4500,6 +4724,16 @@ case $host in #( ;; esac +case $ac_sys_system in #( + iOS) : + + as_fn_append CFLAGS " -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}" + as_fn_append LDFLAGS " -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}" + ;; #( + *) : + ;; +esac + if test "$ac_sys_system" = "Darwin" then # Extract the first word of "xcrun", so it can be a program name with args. @@ -5967,7 +6201,9 @@ then : else $as_nop cat > conftest.c <conftest.out 2>/dev/null; then ac_cv_cc_name=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` + if test $(expr "//$CC" : '.*/\(.*\)') = "mpicc"; then + ac_cv_cc_name="mpicc" + fi else ac_cv_cc_name="unknown" fi @@ -6169,24 +6408,52 @@ fi - -preset_cxx="$CXX" -if test -z "$CXX" -then - case "$CC" in - gcc) if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}g++", so it can be a program name with args. -set dummy ${ac_tool_prefix}g++; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_path_CXX+y} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for GCC compatible compiler" >&5 +printf %s "checking for GCC compatible compiler... " >&6; } +if test ${ac_cv_gcc_compat+y} then : printf %s "(cached) " >&6 else $as_nop - case $CXX in - [\\/]* | ?:[\\/]*) - ac_cv_path_CXX="$CXX" # Let the user override the test with a path. - ;; + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #if !defined(__GNUC__) + #error "not GCC compatible" + #else + /* GCC compatible! */ + #endif + +_ACEOF +if ac_fn_c_try_cpp "$LINENO" +then : + ac_cv_gcc_compat=yes +else $as_nop + ac_cv_gcc_compat=no +fi +rm -f conftest.err conftest.i conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_compat" >&5 +printf "%s\n" "$ac_cv_gcc_compat" >&6; } + + + +preset_cxx="$CXX" +if test -z "$CXX" +then + case "$CC" in + gcc) if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}g++", so it can be a program name with args. +set dummy ${ac_tool_prefix}g++; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_path_CXX+y} +then : + printf %s "(cached) " >&6 +else $as_nop + case $CXX in + [\\/]* | ?:[\\/]*) + ac_cv_path_CXX="$CXX" # Let the user override the test with a path. + ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in notfound @@ -6762,6 +7029,8 @@ printf %s "checking for multiarch... " >&6; } case $ac_sys_system in #( Darwin*) : MULTIARCH="" ;; #( + iOS) : + MULTIARCH="" ;; #( FreeBSD*) : MULTIARCH="" ;; #( *) : @@ -6769,8 +7038,6 @@ case $ac_sys_system in #( ;; esac -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $MULTIARCH" >&5 -printf "%s\n" "$MULTIARCH" >&6; } if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then if test x$PLATFORM_TRIPLET != x$MULTIARCH; then @@ -6780,6 +7047,16 @@ elif test x$PLATFORM_TRIPLET != x && test x$MULTIARCH = x; then MULTIARCH=$PLATFORM_TRIPLET fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $MULTIARCH" >&5 +printf "%s\n" "$MULTIARCH" >&6; } + +case $ac_sys_system in #( + iOS) : + SOABI_PLATFORM=`echo "$PLATFORM_TRIPLET" | cut -d '-' -f2` ;; #( + *) : + SOABI_PLATFORM=$PLATFORM_TRIPLET + ;; +esac if test x$MULTIARCH != x; then MULTIARCH_CPPFLAGS="-DMULTIARCH=\\\"$MULTIARCH\\\"" @@ -6793,14 +7070,14 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=1 ;; #( x86_64-apple-darwin*/clang) : PY_SUPPORT_TIER=1 ;; #( + aarch64-apple-darwin*/clang) : + PY_SUPPORT_TIER=1 ;; #( i686-pc-windows-msvc/msvc) : PY_SUPPORT_TIER=1 ;; #( x86_64-pc-windows-msvc/msvc) : PY_SUPPORT_TIER=1 ;; #( - aarch64-apple-darwin*/clang) : - PY_SUPPORT_TIER=2 ;; #( - aarch64-*-linux-gnu/gcc) : + aarch64-*-linux-gnu/gcc) : PY_SUPPORT_TIER=2 ;; #( aarch64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( @@ -6821,8 +7098,17 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( x86_64-*-freebsd*/clang) : PY_SUPPORT_TIER=3 ;; #( + aarch64-apple-ios*-simulator/clang) : + PY_SUPPORT_TIER=3 ;; #( + aarch64-apple-ios*/clang) : + PY_SUPPORT_TIER=3 ;; #( + aarch64-*-linux-android/clang) : + PY_SUPPORT_TIER=3 ;; #( + x86_64-*-linux-android/clang) : + PY_SUPPORT_TIER=3 ;; #( *) : - PY_SUPPORT_TIER=0 + + PY_SUPPORT_TIER=0 ;; esac @@ -6906,6 +7192,9 @@ printf "%s\n" "$ANDROID_API_LEVEL" >&6; } printf "%s\n" "#define ANDROID_API_LEVEL $ANDROID_API_LEVEL" >>confdefs.h + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the Android arm ABI" >&5 printf %s "checking for the Android arm ABI... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_arm_arch" >&5 @@ -7059,6 +7348,9 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $EXEEXT" >&5 printf "%s\n" "$EXEEXT" >&6; } +# Make sure we keep EXEEXT and ac_exeext sync'ed. +ac_exeext=$EXEEXT + # Test whether we're running on a non-case-sensitive system, in which # case we give a warning if no ext is given @@ -7271,17 +7563,25 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking LDLIBRARY" >&5 printf %s "checking LDLIBRARY... " >&6; } -# MacOSX framework builds need more magic. LDLIBRARY is the dynamic +# Apple framework builds need more magic. LDLIBRARY is the dynamic # library that we build, but we do not want to link against it (we # will find it with a -framework option). For this reason there is an # extra variable BLDLIBRARY against which Python and the extension # modules are linked, BLDLIBRARY. This is normally the same as -# LDLIBRARY, but empty for MacOSX framework builds. +# LDLIBRARY, but empty for MacOSX framework builds. iOS does the same, +# but uses a non-versioned framework layout. if test "$enable_framework" then - LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' - RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} + case $ac_sys_system in + Darwin) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)';; + iOS) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)';; + *) + as_fn_error $? "Unknown platform for framework build" "$LINENO" 5;; + esac BLDLIBRARY='' + RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} else BLDLIBRARY='$(LDLIBRARY)' fi @@ -7294,66 +7594,80 @@ printf "%s\n" "#define Py_ENABLE_SHARED 1" >>confdefs.h case $ac_sys_system in CYGWIN*) - LDLIBRARY='libpython$(LDVERSION).dll.a' - DLLLIBRARY='libpython$(LDVERSION).dll' - ;; + LDLIBRARY='libpython$(LDVERSION).dll.a' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + DLLLIBRARY='libpython$(LDVERSION).dll' + ;; SunOS*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + INSTSONAME="$LDLIBRARY".$SOVERSION + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; Linux*|GNU*|NetBSD*|FreeBSD*|DragonFly*|OpenBSD*|VxWorks*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + + # The Android Gradle plugin will only package libraries whose names end + # with ".so". + if test "$ac_sys_system" != "Linux-android"; then + INSTSONAME="$LDLIBRARY".$SOVERSION + fi + + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; hp*|HP*) - case `uname -m` in - ia64) - LDLIBRARY='libpython$(LDVERSION).so' - ;; - *) - LDLIBRARY='libpython$(LDVERSION).sl' - ;; - esac - BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} - ;; + case `uname -m` in + ia64) + LDLIBRARY='libpython$(LDVERSION).so' + ;; + *) + LDLIBRARY='libpython$(LDVERSION).sl' + ;; + esac + BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} + ;; Darwin*) - LDLIBRARY='libpython$(LDVERSION).dylib' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).dylib' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} + ;; + iOS) + LDLIBRARY='libpython$(LDVERSION).dylib' + ;; AIX*) - LDLIBRARY='libpython$(LDVERSION).so' - RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).so' + RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} + ;; esac else # shared is disabled PY_ENABLE_SHARED=0 case $ac_sys_system in CYGWIN*) - BLDLIBRARY='$(LIBRARY)' - LDLIBRARY='libpython$(LDVERSION).dll.a' - ;; + BLDLIBRARY='$(LIBRARY)' + LDLIBRARY='libpython$(LDVERSION).dll.a' + ;; esac fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 +printf "%s\n" "$LDLIBRARY" >&6; } if test "$cross_compiling" = yes; then - RUNSHARED= + RUNSHARED= fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 +printf %s "checking HOSTRUNNER... " >&6; } if test -z "$HOSTRUNNER" then @@ -7469,76 +7783,19 @@ else fi HOSTRUNNER="$NODE" - # bigint for ctypes c_longlong, c_longdouble - # no longer available in Node 16 - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bigint" >&5 -printf %s "checking for node --experimental-wasm-bigint... " >&6; } -if test ${ac_cv_tool_node_wasm_bigint+y} -then : - printf %s "(cached) " >&6 -else $as_nop - - if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then - ac_cv_tool_node_wasm_bigint=yes - else - ac_cv_tool_node_wasm_bigint=no - fi - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bigint" >&5 -printf "%s\n" "$ac_cv_tool_node_wasm_bigint" >&6; } - if test "x$ac_cv_tool_node_wasm_bigint" = xyes -then : - - as_fn_append HOSTRUNNER " --experimental-wasm-bigint" - -fi - - if test "x$enable_wasm_pthreads" = xyes -then : - - as_fn_append HOSTRUNNER " --experimental-wasm-threads" - # no longer available in Node 16 - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bulk-memory" >&5 -printf %s "checking for node --experimental-wasm-bulk-memory... " >&6; } -if test ${ac_cv_tool_node_wasm_bulk_memory+y} -then : - printf %s "(cached) " >&6 -else $as_nop - - if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then - ac_cv_tool_node_wasm_bulk_memory=yes - else - ac_cv_tool_node_wasm_bulk_memory=no - fi - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bulk_memory" >&5 -printf "%s\n" "$ac_cv_tool_node_wasm_bulk_memory" >&6; } - if test "x$ac_cv_tool_node_wasm_bulk_memory" = xyes -then : - - as_fn_append HOSTRUNNER " --experimental-wasm-bulk-memory" - -fi - -fi - if test "x$host_cpu" = xwasm64 then : as_fn_append HOSTRUNNER " --experimental-wasm-memory64" fi ;; #( WASI/*) : - HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --mapdir /::$(srcdir) --' ;; #( + HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2 --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/' ;; #( *) : HOSTRUNNER='' ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 -printf %s "checking HOSTRUNNER... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $HOSTRUNNER" >&5 printf "%s\n" "$HOSTRUNNER" >&6; } @@ -7546,9 +7803,6 @@ if test -n "$HOSTRUNNER"; then PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD" fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 -printf "%s\n" "$LDLIBRARY" >&6; } - # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable case $ac_sys_system/$ac_sys_emscripten_target in #( Emscripten/browser*) : @@ -7866,7 +8120,9 @@ fi # For calculating the .so ABI tag. + ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -7896,6 +8152,7 @@ printf "%s\n" "#define Py_GIL_DISABLED 1" >>confdefs.h # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -7948,6 +8205,10 @@ printf "%s\n" "#define Py_TRACE_REFS 1" >>confdefs.h fi +if test "$disable_gil" = "yes" -a "$with_trace_refs" = "yes"; +then + as_fn_error $? "--disable-gil cannot be used with --with-trace-refs" "$LINENO" 5 +fi # Check for --enable-pystats { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-pystats" >&5 @@ -8012,11 +8273,25 @@ else $as_nop enable_experimental_jit=no fi -if test "x$enable_experimental_jit" = xno +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;; +esac +if ${tier2_flags:+false} : +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " $tier2_flags" +fi +if ${jit_flags:+false} : then : else $as_nop - as_fn_append CFLAGS_NODIST " -D_Py_JIT" + as_fn_append CFLAGS_NODIST " $jit_flags" REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue @@ -8026,8 +8301,8 @@ fi fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 -printf "%s\n" "$enable_experimental_jit" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5 +printf "%s\n" "$tier2_flags $jit_flags" >&6; } # Enable optimization flags @@ -8067,13 +8342,13 @@ if test "$Py_OPT" = 'true' ; then *gcc*) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fno-semantic-interposition" >&5 printf %s "checking whether C compiler accepts -fno-semantic-interposition... " >&6; } -if test ${ax_cv_check_cflags___fno_semantic_interposition+y} +if test ${ax_cv_check_cflags__Werror__fno_semantic_interposition+y} then : printf %s "(cached) " >&6 else $as_nop ax_check_save_flags=$CFLAGS - CFLAGS="$CFLAGS -fno-semantic-interposition" + CFLAGS="$CFLAGS -Werror -fno-semantic-interposition" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -8087,16 +8362,16 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ax_cv_check_cflags___fno_semantic_interposition=yes + ax_cv_check_cflags__Werror__fno_semantic_interposition=yes else $as_nop - ax_cv_check_cflags___fno_semantic_interposition=no + ax_cv_check_cflags__Werror__fno_semantic_interposition=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___fno_semantic_interposition" >&5 -printf "%s\n" "$ax_cv_check_cflags___fno_semantic_interposition" >&6; } -if test "x$ax_cv_check_cflags___fno_semantic_interposition" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fno_semantic_interposition" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__fno_semantic_interposition" >&6; } +if test "x$ax_cv_check_cflags__Werror__fno_semantic_interposition" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -fno-semantic-interposition" @@ -8652,9 +8927,9 @@ case "$CC_BASENAME" in *clang*) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=" ${LLVM_PROFDATA} merge -output=\"\$(shell pwd)/code.profclangd\" \"\$(shell pwd)\"/*.profclangr " + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test $LLVM_PROF_FOUND = not-found then LLVM_PROF_ERR=yes @@ -8668,9 +8943,9 @@ case "$CC_BASENAME" in case $ac_sys_system in Darwin*) PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=" ${LLVM_PROFDATA} merge -output=\"\$(shell pwd)/code.profclangd\" \"\$(shell pwd)\"/*.profclangr " + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test "${LLVM_PROF_FOUND}" = "not-found" then LLVM_PROF_ERR=yes @@ -9177,6 +9452,11 @@ then : PYDEBUG_CFLAGS="-Og" fi +# gh-120688: WASI uses -O3 in debug mode to support more recursive calls +if test "$ac_sys_system" = "WASI"; then + PYDEBUG_CFLAGS="-O3" +fi + # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line @@ -9268,6 +9548,7 @@ fi then : wasm_debug=yes fi + as_fn_append LDFLAGS_NODIST " --pre-js=\$(srcdir)/Tools/wasm/emscripten/node_pre.js" as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sNODERAWFS" as_fn_append LINKFORSHARED " -sEXIT_RUNTIME" WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" @@ -9323,7 +9604,7 @@ then : fi - as_fn_append LDFLAGS_NODIST " -z stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760" + as_fn_append LDFLAGS_NODIST " -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040" ;; #( *) : @@ -9363,22 +9644,39 @@ else $as_nop BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS" fi -case $GCC in -yes) - CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" +# Enable flags that warn and protect for potential security vulnerabilities. +# These flags should be enabled by default for all builds. +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-safety" >&5 +printf %s "checking for --enable-safety... " >&6; } +# Check whether --enable-safety was given. +if test ${enable_safety+y} +then : + enableval=$enable_safety; if test "x$disable_safety" = xyes +then : + enable_safety=no +else $as_nop + enable_safety=yes +fi +else $as_nop + enable_safety=no +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_safety" >&5 +printf "%s\n" "$enable_safety" >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can add -Wextra" >&5 -printf %s "checking if we can add -Wextra... " >&6; } -if test ${ac_cv_enable_extra_warning+y} +if test "$enable_safety" = "yes" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fstack-protector-strong" >&5 +printf %s "checking whether C compiler accepts -fstack-protector-strong... " >&6; } +if test ${ax_cv_check_cflags__Werror__fstack_protector_strong+y} then : printf %s "(cached) " >&6 else $as_nop - py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wextra -Werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -fstack-protector-strong" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -9391,40 +9689,35 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_enable_extra_warning=yes + ax_cv_check_cflags__Werror__fstack_protector_strong=yes else $as_nop - ac_cv_enable_extra_warning=no + ax_cv_check_cflags__Werror__fstack_protector_strong=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$py_cflags - + CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_extra_warning" >&5 -printf "%s\n" "$ac_cv_enable_extra_warning" >&6; } - - - if test "x$ac_cv_enable_extra_warning" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fstack_protector_strong" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__fstack_protector_strong" >&6; } +if test "x$ax_cv_check_cflags__Werror__fstack_protector_strong" = xyes then : - CFLAGS_NODIST="$CFLAGS_NODIST -Wextra" + CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -fstack-protector-strong not supported" >&5 +printf "%s\n" "$as_me: WARNING: -fstack-protector-strong not supported" >&2;} fi - # Python doesn't violate C99 aliasing rules, but older versions of - # GCC produce warnings for legal Python code. Enable - # -fno-strict-aliasing on versions of GCC that support but produce - # warnings. See Issue3326 - ac_save_cc="$CC" - CC="$CC -fno-strict-aliasing" - save_CFLAGS="$CFLAGS" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts and needs -fno-strict-aliasing" >&5 -printf %s "checking whether $CC accepts and needs -fno-strict-aliasing... " >&6; } -if test ${ac_cv_no_strict_aliasing+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wtrampolines" >&5 +printf %s "checking whether C compiler accepts -Wtrampolines... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wtrampolines+y} then : printf %s "(cached) " >&6 else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wtrampolines" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - int main (void) { @@ -9432,71 +9725,36 @@ main (void) ; return 0; } - -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : - - CC="$ac_save_cc -fstrict-aliasing" - CFLAGS="$CFLAGS -Werror -Wstrict-aliasing" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - void f(int **x) {} -int -main (void) -{ -double *x; f((int **) &x); - ; - return 0; -} - _ACEOF if ac_fn_c_try_compile "$LINENO" then : - - ac_cv_no_strict_aliasing=no - -else $as_nop - - ac_cv_no_strict_aliasing=yes - -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ax_cv_check_cflags__Werror__Wtrampolines=yes else $as_nop - - ac_cv_no_strict_aliasing=no - + ax_cv_check_cflags__Werror__Wtrampolines=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_no_strict_aliasing" >&5 -printf "%s\n" "$ac_cv_no_strict_aliasing" >&6; } - CFLAGS="$save_CFLAGS" - CC="$ac_save_cc" - if test "x$ac_cv_no_strict_aliasing" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wtrampolines" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wtrampolines" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wtrampolines" = xyes then : - BASECFLAGS="$BASECFLAGS -fno-strict-aliasing" + CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} fi - # ICC doesn't recognize the option, but only emits a warning - ## XXX does it emit an unused result warning and can it be disabled? - case "$CC_BASENAME" in #( - *icc*) : - ac_cv_disable_unused_result_warning=no - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can disable $CC unused-result warning" >&5 -printf %s "checking if we can disable $CC unused-result warning... " >&6; } -if test ${ac_cv_disable_unused_result_warning+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wimplicit-fallthrough" >&5 +printf %s "checking whether C compiler accepts -Wimplicit-fallthrough... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wimplicit_fallthrough+y} then : printf %s "(cached) " >&6 else $as_nop - py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wunused-result -Werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wimplicit-fallthrough" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -9509,39 +9767,33 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_disable_unused_result_warning=yes + ax_cv_check_cflags__Werror__Wimplicit_fallthrough=yes else $as_nop - ac_cv_disable_unused_result_warning=no + ax_cv_check_cflags__Werror__Wimplicit_fallthrough=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$py_cflags - + CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_result_warning" >&5 -printf "%s\n" "$ac_cv_disable_unused_result_warning" >&6; } - - ;; #( - *) : - ;; -esac - if test "x$ac_cv_disable_unused_result_warning" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" = xyes then : - BASECFLAGS="$BASECFLAGS -Wno-unused-result" - CFLAGS_NODIST="$CFLAGS_NODIST -Wno-unused-result" + CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wimplicit-fallthrough not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wimplicit-fallthrough not supported" >&2;} fi - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can disable $CC unused-parameter warning" >&5 -printf %s "checking if we can disable $CC unused-parameter warning... " >&6; } -if test ${ac_cv_disable_unused_parameter_warning+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Werror=format-security" >&5 +printf %s "checking whether C compiler accepts -Werror=format-security... " >&6; } +if test ${ax_cv_check_cflags__Werror__Werror_format_security+y} then : printf %s "(cached) " >&6 else $as_nop - py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wunused-parameter -Werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Werror=format-security" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -9554,35 +9806,33 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_disable_unused_parameter_warning=yes + ax_cv_check_cflags__Werror__Werror_format_security=yes else $as_nop - ac_cv_disable_unused_parameter_warning=no + ax_cv_check_cflags__Werror__Werror_format_security=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$py_cflags - + CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_parameter_warning" >&5 -printf "%s\n" "$ac_cv_disable_unused_parameter_warning" >&6; } - - - if test "x$ac_cv_disable_unused_parameter_warning" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Werror_format_security" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Werror_format_security" >&6; } +if test "x$ax_cv_check_cflags__Werror__Werror_format_security" = xyes then : - CFLAGS_NODIST="$CFLAGS_NODIST -Wno-unused-parameter" + CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Werror=format-security not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;} fi - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can disable $CC int-conversion warning" >&5 -printf %s "checking if we can disable $CC int-conversion warning... " >&6; } -if test ${ac_cv_disable_int_conversion_warning+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wbidi-chars=any" >&5 +printf %s "checking whether C compiler accepts -Wbidi-chars=any... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wbidi_chars_any+y} then : printf %s "(cached) " >&6 else $as_nop - py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wint-conversion -Werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wbidi-chars=any" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -9595,35 +9845,33 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_disable_int_conversion_warning=yes + ax_cv_check_cflags__Werror__Wbidi_chars_any=yes else $as_nop - ac_cv_disable_int_conversion_warning=no + ax_cv_check_cflags__Werror__Wbidi_chars_any=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$py_cflags - + CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_int_conversion_warning" >&5 -printf "%s\n" "$ac_cv_disable_int_conversion_warning" >&6; } - - - if test "x$ac_cv_disable_int_conversion" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wbidi_chars_any" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wbidi_chars_any" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wbidi_chars_any" = xyes then : - CFLAGS_NODIST="$CFLAGS_NODIST -Wno-int-conversion" + CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wbidi-chars=any not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wbidi-chars=any not supported" >&2;} fi - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can disable $CC missing-field-initializers warning" >&5 -printf %s "checking if we can disable $CC missing-field-initializers warning... " >&6; } -if test ${ac_cv_disable_missing_field_initializers_warning+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wall" >&5 +printf %s "checking whether C compiler accepts -Wall... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wall+y} then : printf %s "(cached) " >&6 else $as_nop - py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wmissing-field-initializers -Werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wall" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -9636,35 +9884,55 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_disable_missing_field_initializers_warning=yes + ax_cv_check_cflags__Werror__Wall=yes else $as_nop - ac_cv_disable_missing_field_initializers_warning=no + ax_cv_check_cflags__Werror__Wall=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$py_cflags - + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wall" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wall" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wall" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wall" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wall not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wall not supported" >&2;} fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_missing_field_initializers_warning" >&5 -printf "%s\n" "$ac_cv_disable_missing_field_initializers_warning" >&6; } +fi - if test "x$ac_cv_disable_missing_field_initializers_warning" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-slower-safety" >&5 +printf %s "checking for --enable-slower-safety... " >&6; } +# Check whether --enable-slower-safety was given. +if test ${enable_slower_safety+y} then : - CFLAGS_NODIST="$CFLAGS_NODIST -Wno-missing-field-initializers" + enableval=$enable_slower_safety; if test "x$disable_slower_safety" = xyes +then : + enable_slower_safety=no +else $as_nop + enable_slower_safety=yes +fi +else $as_nop + enable_slower_safety=no fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_slower_safety" >&5 +printf "%s\n" "$enable_slower_safety" >&6; } - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can enable $CC sign-compare warning" >&5 -printf %s "checking if we can enable $CC sign-compare warning... " >&6; } -if test ${ac_cv_enable_sign_compare_warning+y} +if test "$enable_slower_safety" = "yes" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -D_FORTIFY_SOURCE=3" >&5 +printf %s "checking whether C compiler accepts -D_FORTIFY_SOURCE=3... " >&6; } +if test ${ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3+y} then : printf %s "(cached) " >&6 else $as_nop - py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wsign-compare -Werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -D_FORTIFY_SOURCE=3" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -9677,34 +9945,40 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_enable_sign_compare_warning=yes + ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3=yes else $as_nop - ac_cv_enable_sign_compare_warning=no + ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$py_cflags - + CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_sign_compare_warning" >&5 -printf "%s\n" "$ac_cv_enable_sign_compare_warning" >&6; } - - - if test "x$ac_cv_enable_sign_compare_warning" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" >&6; } +if test "x$ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" = xyes then : - BASECFLAGS="$BASECFLAGS -Wsign-compare" + CFLAGS_NODIST="$CFLAGS_NODIST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&5 +printf "%s\n" "$as_me: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&2;} fi +fi + +case $GCC in +yes) + CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can enable $CC unreachable-code warning" >&5 -printf %s "checking if we can enable $CC unreachable-code warning... " >&6; } -if test ${ac_cv_enable_unreachable_code_warning+y} + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can add -Wextra" >&5 +printf %s "checking if we can add -Wextra... " >&6; } +if test ${ac_cv_enable_extra_warning+y} then : printf %s "(cached) " >&6 else $as_nop py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wunreachable-code -Werror" + as_fn_append CFLAGS " -Wextra -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9718,48 +9992,40 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_enable_unreachable_code_warning=yes + ac_cv_enable_extra_warning=yes else $as_nop - ac_cv_enable_unreachable_code_warning=no + ac_cv_enable_extra_warning=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_unreachable_code_warning" >&5 -printf "%s\n" "$ac_cv_enable_unreachable_code_warning" >&6; } - - - # Don't enable unreachable code warning in debug mode, since it usually - # results in non-standard code paths. - # Issue #24324: Unfortunately, the unreachable code warning does not work - # correctly on gcc and has been silently removed from the compiler. - # It is supported on clang but on OS X systems gcc may be an alias - # for clang. Try to determine if the compiler is not really gcc and, - # if so, only then enable the warning. - if test $ac_cv_enable_unreachable_code_warning = yes && \ - test "$Py_DEBUG" != "true" && \ - test -z "`$CC --version 2>/dev/null | grep 'Free Software Foundation'`" - then - BASECFLAGS="$BASECFLAGS -Wunreachable-code" - else - ac_cv_enable_unreachable_code_warning=no - fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_extra_warning" >&5 +printf "%s\n" "$ac_cv_enable_extra_warning" >&6; } + if test "x$ac_cv_enable_extra_warning" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wextra" +fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can enable $CC strict-prototypes warning" >&5 -printf %s "checking if we can enable $CC strict-prototypes warning... " >&6; } -if test ${ac_cv_enable_strict_prototypes_warning+y} + # Python doesn't violate C99 aliasing rules, but older versions of + # GCC produce warnings for legal Python code. Enable + # -fno-strict-aliasing on versions of GCC that support but produce + # warnings. See Issue3326 + ac_save_cc="$CC" + CC="$CC -fno-strict-aliasing" + save_CFLAGS="$CFLAGS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts and needs -fno-strict-aliasing" >&5 +printf %s "checking whether $CC accepts and needs -fno-strict-aliasing... " >&6; } +if test ${ac_cv_no_strict_aliasing+y} then : printf %s "(cached) " >&6 else $as_nop - - py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wstrict-prototypes -Werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + int main (void) { @@ -9767,37 +10033,72 @@ main (void) ; return 0; } + _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_enable_strict_prototypes_warning=yes + + CC="$ac_save_cc -fstrict-aliasing" + CFLAGS="$CFLAGS -Werror -Wstrict-aliasing" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + void f(int **x) {} +int +main (void) +{ +double *x; f((int **) &x); + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + + ac_cv_no_strict_aliasing=no + else $as_nop - ac_cv_enable_strict_prototypes_warning=no + + ac_cv_no_strict_aliasing=yes + fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$py_cflags -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_strict_prototypes_warning" >&5 -printf "%s\n" "$ac_cv_enable_strict_prototypes_warning" >&6; } +else $as_nop + ac_cv_no_strict_aliasing=no - if test "x$ac_cv_enable_strict_prototypes_warning" = xyes +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_no_strict_aliasing" >&5 +printf "%s\n" "$ac_cv_no_strict_aliasing" >&6; } + CFLAGS="$save_CFLAGS" + CC="$ac_save_cc" + if test "x$ac_cv_no_strict_aliasing" = xyes then : - CFLAGS_NODIST="$CFLAGS_NODIST -Wstrict-prototypes" + BASECFLAGS="$BASECFLAGS -fno-strict-aliasing" fi - ac_save_cc="$CC" - CC="$CC -Werror=implicit-function-declaration" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can make implicit function declaration an error in $CC" >&5 -printf %s "checking if we can make implicit function declaration an error in $CC... " >&6; } -if test ${ac_cv_enable_implicit_function_declaration_error+y} + # ICC doesn't recognize the option, but only emits a warning + ## XXX does it emit an unused result warning and can it be disabled? + case "$CC_BASENAME" in #( + *icc*) : + ac_cv_disable_unused_result_warning=no + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can disable $CC unused-result warning" >&5 +printf %s "checking if we can disable $CC unused-result warning... " >&6; } +if test ${ac_cv_disable_unused_result_warning+y} then : printf %s "(cached) " >&6 else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ + py_cflags=$CFLAGS + as_fn_append CFLAGS " -Wunused-result -Werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ int main (void) @@ -9806,40 +10107,43 @@ main (void) ; return 0; } - _ACEOF if ac_fn_c_try_compile "$LINENO" then : - - ac_cv_enable_implicit_function_declaration_error=yes - + ac_cv_disable_unused_result_warning=yes else $as_nop - - ac_cv_enable_implicit_function_declaration_error=no - + ac_cv_disable_unused_result_warning=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$py_cflags + fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_implicit_function_declaration_error" >&5 -printf "%s\n" "$ac_cv_enable_implicit_function_declaration_error" >&6; } - CC="$ac_save_cc" +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_result_warning" >&5 +printf "%s\n" "$ac_cv_disable_unused_result_warning" >&6; } - if test "x$ac_cv_enable_implicit_function_declaration_error" = xyes + ;; #( + *) : + ;; +esac + if test "x$ac_cv_disable_unused_result_warning" = xyes then : - CFLAGS_NODIST="$CFLAGS_NODIST -Werror=implicit-function-declaration" + BASECFLAGS="$BASECFLAGS -Wno-unused-result" + CFLAGS_NODIST="$CFLAGS_NODIST -Wno-unused-result" fi - ac_save_cc="$CC" - CC="$CC -fvisibility=hidden" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can use visibility in $CC" >&5 -printf %s "checking if we can use visibility in $CC... " >&6; } -if test ${ac_cv_enable_visibility+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can disable $CC unused-parameter warning" >&5 +printf %s "checking if we can disable $CC unused-parameter warning... " >&6; } +if test ${ac_cv_disable_unused_parameter_warning+y} then : printf %s "(cached) " >&6 else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ + py_cflags=$CFLAGS + as_fn_append CFLAGS " -Wunused-parameter -Werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ int main (void) @@ -9848,22 +10152,319 @@ main (void) ; return 0; } - _ACEOF if ac_fn_c_try_compile "$LINENO" then : - - ac_cv_enable_visibility=yes - + ac_cv_disable_unused_parameter_warning=yes else $as_nop - - ac_cv_enable_visibility=no - + ac_cv_disable_unused_parameter_warning=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$py_cflags + fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_visibility" >&5 -printf "%s\n" "$ac_cv_enable_visibility" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_parameter_warning" >&5 +printf "%s\n" "$ac_cv_disable_unused_parameter_warning" >&6; } + + + if test "x$ac_cv_disable_unused_parameter_warning" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wno-unused-parameter" +fi + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can disable $CC int-conversion warning" >&5 +printf %s "checking if we can disable $CC int-conversion warning... " >&6; } +if test ${ac_cv_disable_int_conversion_warning+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + py_cflags=$CFLAGS + as_fn_append CFLAGS " -Wint-conversion -Werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_disable_int_conversion_warning=yes +else $as_nop + ac_cv_disable_int_conversion_warning=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$py_cflags + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_int_conversion_warning" >&5 +printf "%s\n" "$ac_cv_disable_int_conversion_warning" >&6; } + + + if test "x$ac_cv_disable_int_conversion" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wno-int-conversion" +fi + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can disable $CC missing-field-initializers warning" >&5 +printf %s "checking if we can disable $CC missing-field-initializers warning... " >&6; } +if test ${ac_cv_disable_missing_field_initializers_warning+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + py_cflags=$CFLAGS + as_fn_append CFLAGS " -Wmissing-field-initializers -Werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_disable_missing_field_initializers_warning=yes +else $as_nop + ac_cv_disable_missing_field_initializers_warning=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$py_cflags + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_missing_field_initializers_warning" >&5 +printf "%s\n" "$ac_cv_disable_missing_field_initializers_warning" >&6; } + + + if test "x$ac_cv_disable_missing_field_initializers_warning" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wno-missing-field-initializers" +fi + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can enable $CC sign-compare warning" >&5 +printf %s "checking if we can enable $CC sign-compare warning... " >&6; } +if test ${ac_cv_enable_sign_compare_warning+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + py_cflags=$CFLAGS + as_fn_append CFLAGS " -Wsign-compare -Werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_enable_sign_compare_warning=yes +else $as_nop + ac_cv_enable_sign_compare_warning=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$py_cflags + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_sign_compare_warning" >&5 +printf "%s\n" "$ac_cv_enable_sign_compare_warning" >&6; } + + + if test "x$ac_cv_enable_sign_compare_warning" = xyes +then : + BASECFLAGS="$BASECFLAGS -Wsign-compare" +fi + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can enable $CC unreachable-code warning" >&5 +printf %s "checking if we can enable $CC unreachable-code warning... " >&6; } +if test ${ac_cv_enable_unreachable_code_warning+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + py_cflags=$CFLAGS + as_fn_append CFLAGS " -Wunreachable-code -Werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_enable_unreachable_code_warning=yes +else $as_nop + ac_cv_enable_unreachable_code_warning=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$py_cflags + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_unreachable_code_warning" >&5 +printf "%s\n" "$ac_cv_enable_unreachable_code_warning" >&6; } + + + # Don't enable unreachable code warning in debug mode, since it usually + # results in non-standard code paths. + # Issue #24324: Unfortunately, the unreachable code warning does not work + # correctly on gcc and has been silently removed from the compiler. + # It is supported on clang but on OS X systems gcc may be an alias + # for clang. Try to determine if the compiler is not really gcc and, + # if so, only then enable the warning. + if test $ac_cv_enable_unreachable_code_warning = yes && \ + test "$Py_DEBUG" != "true" && \ + test -z "`$CC --version 2>/dev/null | grep 'Free Software Foundation'`" + then + BASECFLAGS="$BASECFLAGS -Wunreachable-code" + else + ac_cv_enable_unreachable_code_warning=no + fi + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can enable $CC strict-prototypes warning" >&5 +printf %s "checking if we can enable $CC strict-prototypes warning... " >&6; } +if test ${ac_cv_enable_strict_prototypes_warning+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + py_cflags=$CFLAGS + as_fn_append CFLAGS " -Wstrict-prototypes -Werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_enable_strict_prototypes_warning=yes +else $as_nop + ac_cv_enable_strict_prototypes_warning=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$py_cflags + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_strict_prototypes_warning" >&5 +printf "%s\n" "$ac_cv_enable_strict_prototypes_warning" >&6; } + + + if test "x$ac_cv_enable_strict_prototypes_warning" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wstrict-prototypes" +fi + + ac_save_cc="$CC" + CC="$CC -Werror=implicit-function-declaration" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can make implicit function declaration an error in $CC" >&5 +printf %s "checking if we can make implicit function declaration an error in $CC... " >&6; } +if test ${ac_cv_enable_implicit_function_declaration_error+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main (void) +{ + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + + ac_cv_enable_implicit_function_declaration_error=yes + +else $as_nop + + ac_cv_enable_implicit_function_declaration_error=no + +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_implicit_function_declaration_error" >&5 +printf "%s\n" "$ac_cv_enable_implicit_function_declaration_error" >&6; } + CC="$ac_save_cc" + + if test "x$ac_cv_enable_implicit_function_declaration_error" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Werror=implicit-function-declaration" +fi + + ac_save_cc="$CC" + CC="$CC -fvisibility=hidden" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we can use visibility in $CC" >&5 +printf %s "checking if we can use visibility in $CC... " >&6; } +if test ${ac_cv_enable_visibility+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main (void) +{ + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + + ac_cv_enable_visibility=yes + +else $as_nop + + ac_cv_enable_visibility=no + +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_visibility" >&5 +printf "%s\n" "$ac_cv_enable_visibility" >&6; } CC="$ac_save_cc" if test "x$ac_cv_enable_visibility" = xyes @@ -10465,6 +11066,12 @@ if test "x$ac_cv_header_linux_memfd_h" = xyes then : printf "%s\n" "#define HAVE_LINUX_MEMFD_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "linux/netfilter_ipv4.h" "ac_cv_header_linux_netfilter_ipv4_h" "$ac_includes_default" +if test "x$ac_cv_header_linux_netfilter_ipv4_h" = xyes +then : + printf "%s\n" "#define HAVE_LINUX_NETFILTER_IPV4_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "linux/random.h" "ac_cv_header_linux_random_h" "$ac_includes_default" if test "x$ac_cv_header_linux_random_h" = xyes @@ -11183,42 +11790,22 @@ then : fi -# checks for typedefs - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_t in time.h" >&5 -printf %s "checking for clock_t in time.h... " >&6; } -if test ${ac_cv_clock_t_time_h+y} -then : - printf %s "(cached) " >&6 -else $as_nop - - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "clock_t" >/dev/null 2>&1 +# Check for clock_t in time.h. +ac_fn_c_check_type "$LINENO" "clock_t" "ac_cv_type_clock_t" "#include +" +if test "x$ac_cv_type_clock_t" = xyes then : - ac_cv_clock_t_time_h=yes -else $as_nop - ac_cv_clock_t_time_h=no -fi -rm -rf conftest* +printf "%s\n" "#define HAVE_CLOCK_T 1" >>confdefs.h -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_clock_t_time_h" >&5 -printf "%s\n" "$ac_cv_clock_t_time_h" >&6; } -if test "x$ac_cv_clock_t_time_h" = xno -then : +else $as_nop printf "%s\n" "#define clock_t long" >>confdefs.h - fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for makedev" >&5 printf %s "checking for makedev... " >&6; } if test ${ac_cv_func_makedev+y} @@ -11459,12 +12046,16 @@ then : printf "%s\n" "#define HAVE_SSIZE_T 1" >>confdefs.h + fi ac_fn_c_check_type "$LINENO" "__uint128_t" "ac_cv_type___uint128_t" "$ac_includes_default" if test "x$ac_cv_type___uint128_t" = xyes then : +printf "%s\n" "#define HAVE___UINT128_T 1" >>confdefs.h + + printf "%s\n" "#define HAVE_GCC_UINT128_T 1" >>confdefs.h fi @@ -12666,7 +13257,12 @@ then BLDSHARED="$LDSHARED" fi ;; - Emscripten|WASI) + iOS/*) + LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + BLDSHARED="$LDSHARED" + ;; + Emscripten*|WASI*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; Linux*|GNU*|QNX*|VxWorks*|Haiku*) @@ -12754,7 +13350,6 @@ then then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; - Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; Emscripten*|WASI*) if test "x$enable_wasm_dynamic_linking" = xyes @@ -12795,30 +13390,34 @@ then Linux-android*) LINKFORSHARED="-pie -Xlinker -export-dynamic";; Linux*|GNU*) LINKFORSHARED="-Xlinker -export-dynamic";; # -u libsys_s pulls in all symbols in libsys - Darwin/*) + Darwin/*|iOS/*) LINKFORSHARED="$extra_undefs -framework CoreFoundation" # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - stack_size="1000000" # 16 MB - if test "$with_ubsan" = "yes" - then - # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" # 64 MB - fi - - LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + stack_size="1000000" # 16 MB + if test "$with_ubsan" = "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi printf "%s\n" "#define THREAD_STACK_SIZE 0x$stack_size" >>confdefs.h - if test "$enable_framework" - then - LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + if test $ac_sys_system = "Darwin"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + + if test "$enable_framework"; then + LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + fi + LINKFORSHARED="$LINKFORSHARED" + elif test $ac_sys_system = "iOS"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)' fi - LINKFORSHARED="$LINKFORSHARED";; + ;; OpenUNIX*|UnixWare*) LINKFORSHARED="-Wl,-Bexport";; SCO_SV*) LINKFORSHARED="-Wl,-Bexport";; ReliantUNIX*) LINKFORSHARED="-W1 -Blargedynsym";; @@ -13447,6 +14046,14 @@ done fi +# gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. +# This restriction inhibits the proper generation of time-based UUIDs. +if test "$ac_sys_system" = "NetBSD"; then + have_uuid=missing + printf "%s\n" "#define HAVE_UUID_H 0" >>confdefs.h + +fi + if test "x$have_uuid" = xmissing then : have_uuid=no @@ -13609,6 +14216,51 @@ printf "%s\n" "$AIX_BUILDDATE" >&6; } *) ;; esac +# check for _Complex C type +# +# Note that despite most compilers define __STDC_IEC_559_COMPLEX__ - almost +# none properly support C11+ Annex G (where pure imaginary types +# represented by _Imaginary are mandatory). This is a bug (see e.g. +# llvm/llvm-project#60269), so we don't rely on presence +# of __STDC_IEC_559_COMPLEX__. +if test "$cross_compiling" = yes +then : + ac_cv_c_complex_supported=no +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#define test(type, out) \ +{ \ + type complex z = 1 + 2*I; z = z*z; \ + (out) = (out) || creal(z) != -3 || cimag(z) != 4; \ +} +int main(void) +{ + int res = 0; + test(float, res); + test(double, res); + test(long double, res); + return res; +} +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + ac_cv_c_complex_supported=yes +else $as_nop + ac_cv_c_complex_supported=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + +if test "$ac_cv_c_complex_supported" = "yes"; then + +printf "%s\n" "#define Py_HAVE_C_COMPLEX 1" >>confdefs.h + +fi + # check for systems that require aligned memory access { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking aligned memory access is required" >&5 printf %s "checking aligned memory access is required... " >&6; } @@ -13618,7 +14270,14 @@ then : else $as_nop if test "$cross_compiling" = yes then : + +# "yes" changes the hash function to FNV, which causes problems with Numba +# (https://github.com/numba/numba/blob/0.59.0/numba/cpython/hashing.py#L470). +if test "$ac_sys_system" = "Linux-android"; then + ac_cv_aligned_required=no +else ac_cv_aligned_required=yes +fi else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -14207,6 +14866,10 @@ then : ctypes_malloc_closure=yes ;; #( + iOS) : + + ctypes_malloc_closure=yes + ;; #( sunos5) : as_fn_append LIBFFI_LIBS " -mimpure-text" ;; #( @@ -14361,6 +15024,83 @@ LIBS=$save_LIBS +fi + +# Check for libffi with real complex double support. +# This is a workaround, since FFI_TARGET_HAS_COMPLEX_TYPE was defined in libffi v3.2.1, +# but real support was provided only in libffi v3.3.0. +# See https://github.com/python/cpython/issues/125206 for more details. +# +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking libffi has complex type support" >&5 +printf %s "checking libffi has complex type support... " >&6; } +if test ${ac_cv_ffi_complex_double_supported+y} +then : + printf %s "(cached) " >&6 +else $as_nop + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$LIBFFI_CFLAGS $CPPFLAGS" + LDFLAGS="$LIBFFI_LIBS $LDFLAGS" + LIBS="$LIBFFI_LIBS $LIBS" +if test "$cross_compiling" = yes +then : + ac_cv_ffi_complex_double_supported=no +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include +int z_is_expected(double complex z) +{ + const double complex expected = CMPLX(1.25, -0.5); + return z == expected; +} +int main(void) +{ + double complex z = 1.25 - 0.5 * I; + ffi_type *args[1] = {&ffi_type_complex_double}; + void *values[1] = {&z}; + ffi_cif cif; + if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, 1, + &ffi_type_sint, args) != FFI_OK) + { + return 2; + } + ffi_arg rc; + ffi_call(&cif, FFI_FN(z_is_expected), &rc, values); + return !rc; +} + +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + ac_cv_ffi_complex_double_supported=yes +else $as_nop + ac_cv_ffi_complex_double_supported=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_ffi_complex_double_supported" >&5 +printf "%s\n" "$ac_cv_ffi_complex_double_supported" >&6; } +if test "$ac_cv_ffi_complex_double_supported" = "yes"; then + +printf "%s\n" "#define Py_FFI_SUPPORT_C_COMPLEX 1" >>confdefs.h + fi # Check for use of the system libmpdec library @@ -14372,36 +15112,163 @@ if test ${with_system_libmpdec+y} then : withval=$with_system_libmpdec; else $as_nop - with_system_libmpdec="no" + with_system_libmpdec="yes" fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 printf "%s\n" "$with_system_libmpdec" >&6; } + + if test "x$with_system_libmpdec" = xyes then : - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LDFLAGS=${LIBMPDEC_LDFLAGS-"-lmpdec"} - LIBMPDEC_INTERNAL= +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec >= 2.5.0" >&5 +printf %s "checking for libmpdec >= 2.5.0... " >&6; } -else $as_nop +if test -n "$LIBMPDEC_CFLAGS"; then + pkg_cv_LIBMPDEC_CFLAGS="$LIBMPDEC_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBMPDEC_CFLAGS=`$PKG_CONFIG --cflags "libmpdec >= 2.5.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LIBMPDEC_LIBS"; then + pkg_cv_LIBMPDEC_LIBS="$LIBMPDEC_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBMPDEC_LIBS=`$PKG_CONFIG --libs "libmpdec >= 2.5.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` + else + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LIBMPDEC_PKG_ERRORS" >&5 + + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL= +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL= +else + LIBMPDEC_CFLAGS=$pkg_cv_LIBMPDEC_CFLAGS + LIBMPDEC_LIBS=$pkg_cv_LIBMPDEC_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } +fi +else $as_nop LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no +fi - if test "x$with_pydebug" = xyes +if test "x$with_system_libmpdec" = xyes then : + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS - as_fn_append LIBMPDEC_CFLAGS " -DTEST_COVERAGE" -fi + CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" + LIBS="$LIBMPDEC_LIBS $LIBS" + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif +int +main (void) +{ +const char *x = mpd_version(); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + have_mpdec=yes +else $as_nop + have_mpdec=no fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 +printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} +fi +if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&5 +printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&2;} + LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no +fi +# Disable forced inlining in debug builds, see GH-94847 +if test "x$with_pydebug" = xyes +then : + as_fn_append LIBMPDEC_CFLAGS " -DTEST_COVERAGE" +fi # Check whether _decimal should use a coroutine-local or thread-local context { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-decimal-contextvar" >&5 @@ -14426,51 +15293,53 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_decimal_contextvar" >&5 printf "%s\n" "$with_decimal_contextvar" >&6; } -# Check for libmpdec machine flavor -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for decimal libmpdec machine" >&5 +if test "x$with_system_libmpdec" = xno +then : + # Check for libmpdec machine flavor + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for decimal libmpdec machine" >&5 printf %s "checking for decimal libmpdec machine... " >&6; } -case $ac_sys_system in #( + case $ac_sys_system in #( Darwin*) : libmpdec_system=Darwin ;; #( SunOS*) : libmpdec_system=sunos ;; #( *) : libmpdec_system=other - ;; + ;; esac -libmpdec_machine=unknown -if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal -elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi -elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - case $CC in #( + libmpdec_machine=unknown + if test "$libmpdec_system" = Darwin; then + # universal here means: build libmpdec with the same arch options + # the python interpreter was built with + libmpdec_machine=universal + elif test $ac_cv_sizeof_size_t -eq 8; then + if test "$ac_cv_gcc_asm_for_x64" = yes; then + libmpdec_machine=x64 + elif test "$ac_cv_type___uint128_t" = yes; then + libmpdec_machine=uint128 + else + libmpdec_machine=ansi64 + fi + elif test $ac_cv_sizeof_size_t -eq 4; then + if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then + case $CC in #( *gcc*) : libmpdec_machine=ppro ;; #( *clang*) : libmpdec_machine=ppro ;; #( *) : libmpdec_machine=ansi32 - ;; + ;; esac - else - libmpdec_machine=ansi32 - fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libmpdec_machine" >&5 + else + libmpdec_machine=ansi32 + fi + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libmpdec_machine" >&5 printf "%s\n" "$libmpdec_machine" >&6; } -case $libmpdec_machine in #( + case $libmpdec_machine in #( x64) : as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_64=1 -DASM=1" ;; #( uint128) : @@ -14487,8 +15356,9 @@ case $libmpdec_machine in #( as_fn_append LIBMPDEC_CFLAGS " -DUNIVERSAL=1" ;; #( *) : as_fn_error $? "_decimal: unsupported architecture" "$LINENO" 5 - ;; + ;; esac +fi if test "$have_ipa_pure_const_bug" = yes; then # Some versions of gcc miscompile inline asm: @@ -14507,6 +15377,9 @@ fi + + + if test "$ac_sys_system" = "Emscripten" -a -z "$LIBSQLITE3_CFLAGS" -a -z "$LIBSQLITE3_LIBS" then : @@ -16053,24 +16926,47 @@ else # (e.g. gnu pth with pthread emulation) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS in unistd.h" >&5 printf %s "checking for _POSIX_THREADS in unistd.h... " >&6; } - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS defined in unistd.h" >&5 +printf %s "checking for _POSIX_THREADS defined in unistd.h... " >&6; } +if test ${ac_cv_defined__POSIX_THREADS_unistd_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _POSIX_THREADS -yes -#endif +int +main (void) +{ + #ifdef _POSIX_THREADS + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined__POSIX_THREADS_unistd_h=yes +else $as_nop + ac_cv_defined__POSIX_THREADS_unistd_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__POSIX_THREADS_unistd_h" >&5 +printf "%s\n" "$ac_cv_defined__POSIX_THREADS_unistd_h" >&6; } +if test $ac_cv_defined__POSIX_THREADS_unistd_h != "no" then : unistd_defines_pthreads=yes else $as_nop unistd_defines_pthreads=no fi -rm -rf conftest* - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $unistd_defines_pthreads" >&5 printf "%s\n" "$unistd_defines_pthreads" >&6; } @@ -16571,65 +17467,135 @@ ipv6lib=none ipv6trylibc=no if test "$ipv6" = yes -a "$cross_compiling" = no; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 stack type" >&5 -printf %s "checking ipv6 stack type... " >&6; } for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta; do case $i in inria) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for IPV6_INRIA_VERSION defined in netinet/in.h" >&5 +printf %s "checking for IPV6_INRIA_VERSION defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef IPV6_INRIA_VERSION -yes -#endif +int +main (void) +{ + + #ifdef IPV6_INRIA_VERSION + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=yes +else $as_nop + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&6; } +if test $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h != "no" then : ipv6type=$i fi -rm -rf conftest* - ;; kame) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __KAME__ defined in netinet/in.h" >&5 +printf %s "checking for __KAME__ defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined___KAME___netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef __KAME__ -yes -#endif +int +main (void) +{ + + #ifdef __KAME__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes + ac_cv_defined___KAME___netinet_in_h=yes +else $as_nop + ac_cv_defined___KAME___netinet_in_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___KAME___netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined___KAME___netinet_in_h" >&6; } +if test $ac_cv_defined___KAME___netinet_in_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-glibc) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __GLIBC__ defined in features.h" >&5 +printf %s "checking for __GLIBC__ defined in features.h... " >&6; } +if test ${ac_cv_defined___GLIBC___features_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -#endif +int +main (void) +{ + + #ifdef __GLIBC__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6trylibc=yes + ac_cv_defined___GLIBC___features_h=yes +else $as_nop + ac_cv_defined___GLIBC___features_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___GLIBC___features_h" >&5 +printf "%s\n" "$ac_cv_defined___GLIBC___features_h" >&6; } +if test $ac_cv_defined___GLIBC___features_h != "no" +then : + ipv6type=$i + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-inet6) if test -d /usr/inet6; then @@ -16648,83 +17614,159 @@ rm -rf conftest* fi ;; toshiba) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _TOSHIBA_INET6 defined in sys/param.h" >&5 +printf %s "checking for _TOSHIBA_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__TOSHIBA_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _TOSHIBA_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _TOSHIBA_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ac_cv_defined__TOSHIBA_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__TOSHIBA_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__TOSHIBA_INET6_sys_param_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; v6d) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h" >&5 +printf %s "checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h... " >&6; } +if test ${ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef __V6D__ -yes -#endif +int +main (void) +{ + + #ifdef __V6D__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=yes +else $as_nop + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&5 +printf "%s\n" "$ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&6; } +if test $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h != "no" +then : + ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" fi -rm -rf conftest* - ;; zeta) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _ZETA_MINAMI_INET6 defined in sys/param.h" >&5 +printf %s "checking for _ZETA_MINAMI_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _ZETA_MINAMI_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _ZETA_MINAMI_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; esac if test "$ipv6type" != "unknown"; then break fi done + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 stack type" >&5 +printf %s "checking ipv6 stack type... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ipv6type" >&5 printf "%s\n" "$ipv6type" >&6; } fi if test "$ipv6" = "yes" -a "$ipv6lib" != "none"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 library" >&5 +printf %s "checking ipv6 library... " >&6; } if test -d $ipv6libdir -a -f $ipv6libdir/lib$ipv6lib.a; then LIBS="-L$ipv6libdir -l$ipv6lib $LIBS" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: using lib$ipv6lib" >&5 -printf "%s\n" "$as_me: using lib$ipv6lib" >&6;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: lib$ipv6lib" >&5 +printf "%s\n" "lib$ipv6lib" >&6; } else if test "x$ipv6trylibc" = xyes then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: using libc" >&5 -printf "%s\n" "$as_me: using libc" >&6;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: libc" >&5 +printf "%s\n" "libc" >&6; } else $as_nop @@ -16956,6 +17998,7 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_mimalloc" >&5 printf "%s\n" "$with_mimalloc" >&6; } +INSTALL_MIMALLOC=$with_mimalloc @@ -16991,31 +18034,6 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_pymalloc" >&5 printf "%s\n" "$with_pymalloc" >&6; } -# Check whether objects such as float, tuple and dict are using -# freelists to optimization memory allocation. -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-freelists" >&5 -printf %s "checking for --with-freelists... " >&6; } - -# Check whether --with-freelists was given. -if test ${with_freelists+y} -then : - withval=$with_freelists; -fi - - -if test -z "$with_freelists" -then - with_freelists="yes" -fi -if test "$with_freelists" != "no" -then - -printf "%s\n" "#define WITH_FREELISTS 1" >>confdefs.h - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_freelists" >&5 -printf "%s\n" "$with_freelists" >&6; } - # Check for --with-c-locale-coercion { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-c-locale-coercion" >&5 printf %s "checking for --with-c-locale-coercion... " >&6; } @@ -17248,6 +18266,31 @@ else printf "%s\n" "$MACHDEP_OBJS" >&6; } fi +if test "$ac_sys_system" = "Linux-android"; then + # When these functions are used in an unprivileged process, they crash rather + # than returning an error. + blocked_funcs="chroot initgroups setegid seteuid setgid sethostname + setregid setresgid setresuid setreuid setuid" + + # These functions are unimplemented and always return an error + # (https://android.googlesource.com/platform/system/sepolicy/+/refs/heads/android13-release/public/domain.te#1044) + blocked_funcs="$blocked_funcs sem_open sem_unlink" + + # Before API level 23, when fchmodat is called with the unimplemented flag + # AT_SYMLINK_NOFOLLOW, instead of returning ENOTSUP as it should, it actually + # follows the symlink. + if test "$ANDROID_API_LEVEL" -lt 23; then + blocked_funcs="$blocked_funcs fchmodat" + fi + + for name in $blocked_funcs; do + as_func_var=`printf "%s\n" "ac_cv_func_$name" | $as_tr_sh` + + eval "$as_func_var=no" + + done +fi + # checks for library functions ac_fn_c_check_func "$LINENO" "accept4" "ac_cv_func_accept4" if test "x$ac_cv_func_accept4" = xyes @@ -17458,12 +18501,6 @@ if test "x$ac_cv_func_getegid" = xyes then : printf "%s\n" "#define HAVE_GETEGID 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" -if test "x$ac_cv_func_getentropy" = xyes -then : - printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "geteuid" "ac_cv_func_geteuid" if test "x$ac_cv_func_geteuid" = xyes @@ -17506,12 +18543,6 @@ if test "x$ac_cv_func_getgrouplist" = xyes then : printf "%s\n" "#define HAVE_GETGROUPLIST 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "getgroups" "ac_cv_func_getgroups" -if test "x$ac_cv_func_getgroups" = xyes -then : - printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "gethostname" "ac_cv_func_gethostname" if test "x$ac_cv_func_gethostname" = xyes @@ -17878,6 +18909,12 @@ if test "x$ac_cv_func_preadv2" = xyes then : printf "%s\n" "#define HAVE_PREADV2 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "process_vm_readv" "ac_cv_func_process_vm_readv" +if test "x$ac_cv_func_process_vm_readv" = xyes +then : + printf "%s\n" "#define HAVE_PROCESS_VM_READV 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pthread_cond_timedwait_relative_np" "ac_cv_func_pthread_cond_timedwait_relative_np" if test "x$ac_cv_func_pthread_cond_timedwait_relative_np" = xyes @@ -18238,12 +19275,6 @@ if test "x$ac_cv_func_sysconf" = xyes then : printf "%s\n" "#define HAVE_SYSCONF 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" -if test "x$ac_cv_func_system" = xyes -then : - printf "%s\n" "#define HAVE_SYSTEM 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "tcgetpgrp" "ac_cv_func_tcgetpgrp" if test "x$ac_cv_func_tcgetpgrp" = xyes @@ -18409,14 +19440,40 @@ then : fi -# Force lchmod off for Linux. Linux disallows changing the mode of symbolic -# links. Some libc implementations have a stub lchmod implementation that always -# returns an error. -if test "$MACHDEP" != linux; then - ac_fn_c_check_func "$LINENO" "lchmod" "ac_cv_func_lchmod" -if test "x$ac_cv_func_lchmod" = xyes +# Force lchmod off for Linux. Linux disallows changing the mode of symbolic +# links. Some libc implementations have a stub lchmod implementation that always +# returns an error. +if test "$MACHDEP" != linux; then + ac_fn_c_check_func "$LINENO" "lchmod" "ac_cv_func_lchmod" +if test "x$ac_cv_func_lchmod" = xyes +then : + printf "%s\n" "#define HAVE_LCHMOD 1" >>confdefs.h + +fi + +fi + +# iOS defines some system methods that can be linked (so they are +# found by configure), but either raise a compilation error (because the +# header definition prevents usage - autoconf doesn't use the headers), or +# raise an error if used at runtime. Force these symbols off. +if test "$ac_sys_system" != "iOS" ; then + ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" +if test "x$ac_cv_func_getentropy" = xyes +then : + printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "getgroups" "ac_cv_func_getgroups" +if test "x$ac_cv_func_getgroups" = xyes +then : + printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" +if test "x$ac_cv_func_system" = xyes then : - printf "%s\n" "#define HAVE_LCHMOD 1" >>confdefs.h + printf "%s\n" "#define HAVE_SYSTEM 1" >>confdefs.h fi @@ -19421,10 +20478,89 @@ fi printf "%s\n" "$ac_cv_lib_resolv_inet_aton" >&6; } if test "x$ac_cv_lib_resolv_inet_aton" = xyes then : - printf "%s\n" "#define HAVE_LIBRESOLV 1" >>confdefs.h + SOCKET_LIBS="-lresolv" +fi + + +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for hstrerror in -lc" >&5 +printf %s "checking for hstrerror in -lc... " >&6; } +if test ${ac_cv_lib_c_hstrerror+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_check_lib_save_LIBS=$LIBS +LIBS="-lc $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ - LIBS="-lresolv $LIBS" +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char hstrerror (); +int +main (void) +{ +return hstrerror (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_c_hstrerror=yes +else $as_nop + ac_cv_lib_c_hstrerror=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_c_hstrerror" >&5 +printf "%s\n" "$ac_cv_lib_c_hstrerror" >&6; } +if test "x$ac_cv_lib_c_hstrerror" = xyes +then : + $ac_cv_prog_TRUE +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for hstrerror in -lresolv" >&5 +printf %s "checking for hstrerror in -lresolv... " >&6; } +if test ${ac_cv_lib_resolv_hstrerror+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_check_lib_save_LIBS=$LIBS +LIBS="-lresolv $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char hstrerror (); +int +main (void) +{ +return hstrerror (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_resolv_hstrerror=yes +else $as_nop + ac_cv_lib_resolv_hstrerror=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_resolv_hstrerror" >&5 +printf "%s\n" "$ac_cv_lib_resolv_hstrerror" >&6; } +if test "x$ac_cv_lib_resolv_hstrerror" = xyes +then : + SOCKET_LIBS="-lresolv" fi @@ -21717,6 +22853,11 @@ fi done +# On Android and iOS, clock_settime can be linked (so it is found by +# configure), but when used in an unprivileged process, it crashes rather than +# returning an error. Force the symbol off. +if test "$ac_sys_system" != "Linux-android" && test "$ac_sys_system" != "iOS" +then for ac_func in clock_settime do : @@ -21727,7 +22868,7 @@ then : else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 printf %s "checking for clock_settime in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_settime+y} then : @@ -21765,7 +22906,7 @@ printf "%s\n" "$ac_cv_lib_rt_clock_settime" >&6; } if test "x$ac_cv_lib_rt_clock_settime" = xyes then : - printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h + printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h fi @@ -21774,7 +22915,12 @@ fi fi done +fi +# On Android before API level 23, clock_nanosleep returns the wrong value when +# interrupted by a signal (https://issuetracker.google.com/issues/216495770). +if ! { test "$ac_sys_system" = "Linux-android" && + test "$ANDROID_API_LEVEL" -lt 23; }; then for ac_func in clock_nanosleep do : @@ -21785,7 +22931,7 @@ then : else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 printf %s "checking for clock_nanosleep in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_nanosleep+y} then : @@ -21823,7 +22969,7 @@ printf "%s\n" "$ac_cv_lib_rt_clock_nanosleep" >&6; } if test "x$ac_cv_lib_rt_clock_nanosleep" = xyes then : - printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h + printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h fi @@ -21832,6 +22978,7 @@ fi fi done +fi for ac_func in nanosleep @@ -21995,7 +23142,9 @@ else $as_nop if test "$cross_compiling" = yes then : -if test "${enable_ipv6+set}" = set; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_buggy_getaddrinfo="no" +elif test "${enable_ipv6+set}" = set; then ac_cv_buggy_getaddrinfo="no -- configured with --(en|dis)able-ipv6" else ac_cv_buggy_getaddrinfo=yes @@ -23040,18 +24189,26 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -double d = 90904234967036810337470478905505011476211692735615632014797120844053488865816695273723469097858056257517020191247487429516932130503560650002327564517570778480236724525140520121371739201496540132640109977779420565776568942592.0; +#include + +static double m[] = {9.090423496703681e+223, 0.0}; + +int main (int argc, char *argv[]) +{ + m[atoi (argv[1])] += atof (argv[2]); + return m[atoi (argv[3])] > 0.0; +} _ACEOF -if ac_fn_c_try_compile "$LINENO" +if ac_fn_c_try_link "$LINENO" then : -if grep noonsees conftest.$ac_objext >/dev/null ; then +if grep noonsees conftest$EXEEXT >/dev/null ; then ax_cv_c_float_words_bigendian=yes fi -if grep seesnoon conftest.$ac_objext >/dev/null ; then +if grep seesnoon conftest$EXEEXT >/dev/null ; then if test "$ax_cv_c_float_words_bigendian" = unknown; then ax_cv_c_float_words_bigendian=no else @@ -23061,7 +24218,8 @@ fi fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_c_float_words_bigendian" >&5 printf "%s\n" "$ax_cv_c_float_words_bigendian" >&6; } @@ -23098,7 +24256,7 @@ else # While Python doesn't currently have full support for these platforms # (see e.g., issue 1762561), we can at least make sure that float <-> string # conversions work. - # FLOAT_WORDS_BIGENDIAN doesnt actually detect this case, but if it's not big + # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, but if it's not big # or little, then it must be this? printf "%s\n" "#define DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 1" >>confdefs.h @@ -23898,7 +25056,7 @@ printf %s "checking ABIFLAGS... " >&6; } printf "%s\n" "$ABIFLAGS" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking SOABI" >&5 printf %s "checking SOABI... " >&6; } -SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} +SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${SOABI_PLATFORM:+-$SOABI_PLATFORM} { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $SOABI" >&5 printf "%s\n" "$SOABI" >&6; } @@ -23907,7 +25065,7 @@ printf "%s\n" "$SOABI" >&6; } if test "$Py_DEBUG" = 'true'; then # Similar to SOABI but remove "d" flag from ABIFLAGS - ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} + ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${SOABI_PLATFORM:+-$SOABI_PLATFORM} printf "%s\n" "#define ALT_SOABI \"${ALT_SOABI}\"" >>confdefs.h @@ -23922,21 +25080,33 @@ LDVERSION='$(VERSION)$(ABIFLAGS)' { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDVERSION" >&5 printf "%s\n" "$LDVERSION" >&6; } -# On Android and Cygwin the shared libraries must be linked with libpython. +# Configure the flags and dependencies used when compiling shared modules. +# Do not rename LIBPYTHON - it's accessed via sysconfig by package build +# systems (e.g. Meson) to decide whether to link extension modules against +# libpython. + +MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' +LIBPYTHON='' + +# On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -else - LIBPYTHON='' + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" + LIBPYTHON="\$(BLDLIBRARY)" +fi + +# On iOS the shared libraries must be linked with the Python framework +if test "$ac_sys_system" = "iOS"; then + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) PLATLIBDIR="lib" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-platlibdir" >&5 @@ -23955,7 +25125,7 @@ then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } @@ -23969,9 +25139,9 @@ fi if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi @@ -24923,9 +26093,60 @@ then : printf "%s\n" "#define HAVE_RL_COMPDISP_FUNC_T 1" >>confdefs.h + +fi + + + # Some editline versions declare rl_startup_hook as taking no args, others + # declare it as taking 2. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if rl_startup_hook takes arguments" >&5 +printf %s "checking if rl_startup_hook takes arguments... " >&6; } +if test ${ac_cv_readline_rl_startup_hook_takes_args+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include /* Must be first for Gnu Readline */ + #ifdef WITH_EDITLINE + # include + #else + # include + # include + #endif + + extern int test_hook_func(const char *text, int state); +int +main (void) +{ +rl_startup_hook=test_hook_func; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_readline_rl_startup_hook_takes_args=yes +else $as_nop + ac_cv_readline_rl_startup_hook_takes_args=no + fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_startup_hook_takes_args" >&5 +printf "%s\n" "$ac_cv_readline_rl_startup_hook_takes_args" >&6; } + if test "x$ac_cv_readline_rl_startup_hook_takes_args" = xyes +then : +printf "%s\n" "#define Py_RL_STARTUP_HOOK_TAKES_ARGS 1" >>confdefs.h + + +fi + CFLAGS=$save_CFLAGS @@ -25167,285 +26388,175 @@ printf "%s\n" "$ac_cv_stat_tv_nsec" >&6; } if test "$ac_cv_stat_tv_nsec" = yes then -printf "%s\n" "#define HAVE_STAT_TV_NSEC 1" >>confdefs.h - -fi - -# Look for BSD style subsecond timestamps in struct stat -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for tv_nsec2 in struct stat" >&5 -printf %s "checking for tv_nsec2 in struct stat... " >&6; } -if test ${ac_cv_stat_tv_nsec2+y} -then : - printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main (void) -{ - -struct stat st; -st.st_mtimespec.tv_nsec = 1; - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : - ac_cv_stat_tv_nsec2=yes -else $as_nop - ac_cv_stat_tv_nsec2=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_stat_tv_nsec2" >&5 -printf "%s\n" "$ac_cv_stat_tv_nsec2" >&6; } -if test "$ac_cv_stat_tv_nsec2" = yes -then - -printf "%s\n" "#define HAVE_STAT_TV_NSEC2 1" >>confdefs.h - -fi - -have_curses=no -have_panel=no - - -ac_fn_c_check_header_compile "$LINENO" "curses.h" "ac_cv_header_curses_h" "$ac_includes_default" -if test "x$ac_cv_header_curses_h" = xyes -then : - printf "%s\n" "#define HAVE_CURSES_H 1" >>confdefs.h - -fi -ac_fn_c_check_header_compile "$LINENO" "ncurses.h" "ac_cv_header_ncurses_h" "$ac_includes_default" -if test "x$ac_cv_header_ncurses_h" = xyes -then : - printf "%s\n" "#define HAVE_NCURSES_H 1" >>confdefs.h - -fi - - -if test "x$ac_cv_header_ncurses_h" = xyes -then : - - if test "$ac_sys_system" != "Darwin"; then - -pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncursesw" >&5 -printf %s "checking for ncursesw... " >&6; } - -if test -n "$CURSES_CFLAGS"; then - pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncursesw" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi -if test -n "$CURSES_LIBS"; then - pkg_cv_CURSES_LIBS="$CURSES_LIBS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncursesw" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi - - - -if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then - _pkg_short_errors_supported=yes -else - _pkg_short_errors_supported=no -fi - if test $_pkg_short_errors_supported = yes; then - CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncursesw" 2>&1` - else - CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncursesw" 2>&1` - fi - # Put the nasty error message in config.log where it belongs - echo "$CURSES_PKG_ERRORS" >&5 - - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - +printf "%s\n" "#define HAVE_STAT_TV_NSEC 1" >>confdefs.h + +fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncursesw" >&5 -printf %s "checking for initscr in -lncursesw... " >&6; } -if test ${ac_cv_lib_ncursesw_initscr+y} +# Look for BSD style subsecond timestamps in struct stat +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for tv_nsec2 in struct stat" >&5 +printf %s "checking for tv_nsec2 in struct stat... " >&6; } +if test ${ac_cv_stat_tv_nsec2+y} then : printf %s "(cached) " >&6 else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncursesw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); +#include int main (void) { -return initscr (); + +struct stat st; +st.st_mtimespec.tv_nsec = 1; + ; return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_ncursesw_initscr=yes + ac_cv_stat_tv_nsec2=yes else $as_nop - ac_cv_lib_ncursesw_initscr=no + ac_cv_stat_tv_nsec2=no fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncursesw_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncursesw_initscr" >&6; } -if test "x$ac_cv_lib_ncursesw_initscr" = xyes -then : - - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_stat_tv_nsec2" >&5 +printf "%s\n" "$ac_cv_stat_tv_nsec2" >&6; } +if test "$ac_cv_stat_tv_nsec2" = yes +then - have_curses=ncursesw - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncursesw"} +printf "%s\n" "#define HAVE_STAT_TV_NSEC2 1" >>confdefs.h fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether year with century should be normalized for strftime" >&5 +printf %s "checking whether year with century should be normalized for strftime... " >&6; } +if test ${ac_cv_normalize_century+y} +then : + printf %s "(cached) " >&6 +else $as_nop -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS +if test "$cross_compiling" = yes +then : + ac_cv_normalize_century=yes +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +int main(void) +{ + char year[5]; + struct tm date = { + .tm_year = -1801, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { + return 1; + } + return 0; +} -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + ac_cv_normalize_century=yes +else $as_nop + ac_cv_normalize_century=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_normalize_century" >&5 +printf "%s\n" "$ac_cv_normalize_century" >&6; } +if test "$ac_cv_normalize_century" = yes +then +printf "%s\n" "#define Py_NORMALIZE_CENTURY 1" >>confdefs.h + +fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncursesw" >&5 -printf %s "checking for initscr in -lncursesw... " >&6; } -if test ${ac_cv_lib_ncursesw_initscr+y} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C99-specific strftime specifiers are supported" >&5 +printf %s "checking whether C99-specific strftime specifiers are supported... " >&6; } +if test ${ac_cv_strftime_c99_support+y} then : printf %s "(cached) " >&6 else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncursesw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + +if test "$cross_compiling" = yes +then : + ac_cv_strftime_c99_support=no +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) +#include +#include + +int main(void) { -return initscr (); - ; - return 0; + char full_date[11]; + struct tm date = { + .tm_year = 0, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(full_date, sizeof(full_date), "%F", &date) && !strcmp(full_date, "1900-01-01")) { + return 0; + } + return 1; } + _ACEOF -if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_run "$LINENO" then : - ac_cv_lib_ncursesw_initscr=yes + ac_cv_strftime_c99_support=yes else $as_nop - ac_cv_lib_ncursesw_initscr=no + ac_cv_strftime_c99_support=no fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncursesw_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncursesw_initscr" >&6; } -if test "x$ac_cv_lib_ncursesw_initscr" = xyes -then : - - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - - have_curses=ncursesw - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncursesw"} fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_strftime_c99_support" >&5 +printf "%s\n" "$ac_cv_strftime_c99_support" >&6; } +if test "$ac_cv_strftime_c99_support" = yes +then +printf "%s\n" "#define Py_STRFTIME_C99_SUPPORT 1" >>confdefs.h -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - +fi -else - CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS - CURSES_LIBS=$pkg_cv_CURSES_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } +have_curses=no +have_panel=no - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - have_curses=ncursesw -fi - fi +# Check for ncursesw/panelw first. If that fails, try ncurses/panel. - if test "x$have_curses" = xno -then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses" >&5 -printf %s "checking for ncurses... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncursesw" >&5 +printf %s "checking for ncursesw... " >&6; } if test -n "$CURSES_CFLAGS"; then pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncurses" 2>/dev/null` + pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncursesw" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -25457,215 +26568,53 @@ if test -n "$CURSES_LIBS"; then pkg_cv_CURSES_LIBS="$CURSES_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncurses" 2>/dev/null` + pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncursesw" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes fi - else - pkg_failed=untried -fi - - - -if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then - _pkg_short_errors_supported=yes -else - _pkg_short_errors_supported=no -fi - if test $_pkg_short_errors_supported = yes; then - CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` - else - CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` - fi - # Put the nasty error message in config.log where it belongs - echo "$CURSES_PKG_ERRORS" >&5 - - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncurses" >&5 -printf %s "checking for initscr in -lncurses... " >&6; } -if test ${ac_cv_lib_ncurses_initscr+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncurses $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) -{ -return initscr (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ncurses_initscr=yes -else $as_nop - ac_cv_lib_ncurses_initscr=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncurses_initscr" >&6; } -if test "x$ac_cv_lib_ncurses_initscr" = xyes -then : - - have_curses=ncurses - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncurses"} - -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncurses" >&5 -printf %s "checking for initscr in -lncurses... " >&6; } -if test ${ac_cv_lib_ncurses_initscr+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncurses $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) -{ -return initscr (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ncurses_initscr=yes -else $as_nop - ac_cv_lib_ncurses_initscr=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncurses_initscr" >&6; } -if test "x$ac_cv_lib_ncurses_initscr" = xyes -then : - - have_curses=ncurses - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncurses"} - + else + pkg_failed=untried fi -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncursesw" 2>&1` + else + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncursesw" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 + have_curses=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_curses=no else CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS CURSES_LIBS=$pkg_cv_CURSES_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_curses=ncurses - -fi - -fi - - -fi -CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') - -if test "$have_curses" != no -a "$ac_sys_system" = "Darwin"; then - - as_fn_append CURSES_CFLAGS " -D_XOPEN_SOURCE_EXTENDED=1" - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - -fi - - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking curses module flags" >&5 -printf %s "checking curses module flags... " >&6; } -if test "x$have_curses" = xno -then : - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -else $as_nop - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_curses (CFLAGS: $CURSES_CFLAGS, LIBS: $CURSES_LIBS)" >&5 -printf "%s\n" "$have_curses (CFLAGS: $CURSES_CFLAGS, LIBS: $CURSES_LIBS)" >&6; } - -fi - -ac_fn_c_check_header_compile "$LINENO" "panel.h" "ac_cv_header_panel_h" "$ac_includes_default" -if test "x$ac_cv_header_panel_h" = xyes -then : - printf "%s\n" "#define HAVE_PANEL_H 1" >>confdefs.h - -fi - - -if test "x$ac_cv_header_panel_h" = xyes -then : - - - if test "$ac_sys_system" != "Darwin"; then - if test "x$have_curses" = xncursesw -then : +printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h + have_curses=yes pkg_failed=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panelw" >&5 @@ -25725,143 +26674,101 @@ fi # Put the nasty error message in config.log where it belongs echo "$PANEL_PKG_ERRORS" >&5 + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanelw" >&5 -printf %s "checking for update_panels in -lpanelw... " >&6; } -if test ${ac_cv_lib_panelw_update_panels+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanelw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +printf "%s\n" "#define HAVE_PANELW 1" >>confdefs.h -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char update_panels (); -int -main (void) -{ -return update_panels (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_panelw_update_panels=yes -else $as_nop - ac_cv_lib_panelw_update_panels=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS + have_panel=yes fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panelw_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panelw_update_panels" >&6; } -if test "x$ac_cv_lib_panelw_update_panels" = xyes -then : - - have_panel=panelw - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanelw"} - fi -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +if test "x$have_curses" = xno +then : - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanelw" >&5 -printf %s "checking for update_panels in -lpanelw... " >&6; } -if test ${ac_cv_lib_panelw_update_panels+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanelw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses" >&5 +printf %s "checking for ncurses... " >&6; } -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char update_panels (); -int -main (void) -{ -return update_panels (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_panelw_update_panels=yes -else $as_nop - ac_cv_lib_panelw_update_panels=no +if test -n "$CURSES_CFLAGS"; then + pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncurses" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS + else + pkg_failed=untried fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panelw_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panelw_update_panels" >&6; } -if test "x$ac_cv_lib_panelw_update_panels" = xyes -then : - - have_panel=panelw - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanelw"} - +if test -n "$CURSES_LIBS"; then + pkg_cv_CURSES_LIBS="$CURSES_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncurses" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried fi -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` + else + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 + have_curses=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_curses=no else - PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS - PANEL_LIBS=$pkg_cv_PANEL_LIBS + CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS + CURSES_LIBS=$pkg_cv_CURSES_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_panel=panelw - -fi - -fi - fi - - if test "x$have_curses" = xncurses -then : +printf "%s\n" "#define HAVE_NCURSES 1" >>confdefs.h + have_curses=yes pkg_failed=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panel" >&5 @@ -25921,83 +26828,165 @@ fi # Put the nasty error message in config.log where it belongs echo "$PANEL_PKG_ERRORS" >&5 + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +printf "%s\n" "#define HAVE_PANEL 1" >>confdefs.h + + have_panel=yes +fi +fi + + +fi + +save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + # Make sure we've got the header defines. + as_fn_append CPPFLAGS " $CURSES_CFLAGS $PANEL_CFLAGS" + ac_fn_c_check_header_compile "$LINENO" "ncursesw/curses.h" "ac_cv_header_ncursesw_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncursesw/ncurses.h" "ac_cv_header_ncursesw_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncursesw/panel.h" "ac_cv_header_ncursesw_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_PANEL_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/curses.h" "ac_cv_header_ncurses_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/ncurses.h" "ac_cv_header_ncurses_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/panel.h" "ac_cv_header_ncurses_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_PANEL_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "curses.h" "ac_cv_header_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses.h" "ac_cv_header_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_H 1" >>confdefs.h - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS +fi +ac_fn_c_check_header_compile "$LINENO" "panel.h" "ac_cv_header_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_PANEL_H 1" >>confdefs.h + +fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanel" >&5 -printf %s "checking for update_panels in -lpanel... " >&6; } -if test ${ac_cv_lib_panel_update_panels+y} + # Check that we're able to link with crucial curses/panel functions. This + # also serves as a fallback in case pkg-config failed. + as_fn_append LIBS " $CURSES_LIBS $PANEL_LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing initscr" >&5 +printf %s "checking for library containing initscr... " >&6; } +if test ${ac_cv_search_initscr+y} then : printf %s "(cached) " >&6 else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanel $LIBS" + ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ -char update_panels (); +char initscr (); int main (void) { -return update_panels (); +return initscr (); ; return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +for ac_lib in '' ncursesw ncurses +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_panel_update_panels=yes -else $as_nop - ac_cv_lib_panel_update_panels=no + ac_cv_search_initscr=$ac_res fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS + conftest$ac_exeext + if test ${ac_cv_search_initscr+y} +then : + break fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panel_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panel_update_panels" >&6; } -if test "x$ac_cv_lib_panel_update_panels" = xyes +done +if test ${ac_cv_search_initscr+y} then : - have_panel=panel - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanel"} - +else $as_nop + ac_cv_search_initscr=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_initscr" >&5 +printf "%s\n" "$ac_cv_search_initscr" >&6; } +ac_res=$ac_cv_search_initscr +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + if test "x$have_curses" = xno +then : + have_curses=yes + CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"} +fi +else $as_nop + have_curses=no fi - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanel" >&5 -printf %s "checking for update_panels in -lpanel... " >&6; } -if test ${ac_cv_lib_panel_update_panels+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing update_panels" >&5 +printf %s "checking for library containing update_panels... " >&6; } +if test ${ac_cv_search_update_panels+y} then : printf %s "(cached) " >&6 else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanel $LIBS" + ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -26013,76 +27002,82 @@ return update_panels (); return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +for ac_lib in '' panelw panel +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_panel_update_panels=yes -else $as_nop - ac_cv_lib_panel_update_panels=no + ac_cv_search_update_panels=$ac_res fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS + conftest$ac_exeext + if test ${ac_cv_search_update_panels+y} +then : + break fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panel_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panel_update_panels" >&6; } -if test "x$ac_cv_lib_panel_update_panels" = xyes +done +if test ${ac_cv_search_update_panels+y} then : - have_panel=panel - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanel"} - +else $as_nop + ac_cv_search_update_panels=no fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -else - PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS - PANEL_LIBS=$pkg_cv_PANEL_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - - have_panel=panel - +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS fi - +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_update_panels" >&5 +printf "%s\n" "$ac_cv_search_update_panels" >&6; } +ac_res=$ac_cv_search_update_panels +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + if test "x$have_panel" = xno +then : + have_panel=yes + PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"} +fi +else $as_nop + have_panel=no fi -fi -PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking panel flags" >&5 -printf %s "checking panel flags... " >&6; } -if test "x$have_panel" = xno + +if test "have_curses" != "no" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -else $as_nop +if test "x$ac_sys_system" = xDarwin +then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)" >&5 -printf "%s\n" "$have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)" >&6; } -fi + as_fn_append CURSES_CFLAGS " -D_XOPEN_SOURCE_EXTENDED=1" -# first curses header check -ac_save_cppflags="$CPPFLAGS" -if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I/usr/include/ncursesw" fi +PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') + # On Solaris, term.h requires curses.h ac_fn_c_check_header_compile "$LINENO" "term.h" "ac_cv_header_term_h" " -#ifdef HAVE_CURSES_H -#include +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include #endif " @@ -26102,7 +27097,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26132,10 +27142,6 @@ printf "%s\n" "#define MVWDELCH_IS_EXPRESSION 1" >>confdefs.h fi -# Issue #25720: ncurses has introduced the NCURSES_OPAQUE symbol making opaque -# structs since version 5.7. If the macro is defined as zero before including -# [n]curses.h, ncurses will expose fields of the structs regardless of the -# configuration. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether WINDOW has _flags" >&5 printf %s "checking whether WINDOW has _flags... " >&6; } if test ${ac_cv_window_has_flags+y} @@ -26145,8 +27151,20 @@ else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define NCURSES_OPAQUE 0 - #include +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif int main (void) @@ -26190,7 +27208,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26233,7 +27266,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26276,7 +27324,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26319,7 +27382,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26362,7 +27440,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26405,7 +27498,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26448,7 +27556,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26491,7 +27614,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26534,7 +27672,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26577,7 +27730,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26620,7 +27788,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26655,27 +27838,38 @@ fi CPPFLAGS=$ac_save_cppflags +fi +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for device files" >&5 printf "%s\n" "$as_me: checking for device files" >&6;} -if test "x$cross_compiling" = xyes; then - if test "${ac_cv_file__dev_ptmx+set}" != set; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no +else + if test "x$cross_compiling" = xyes; then + if test "${ac_cv_file__dev_ptmx+set}" != set; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 printf %s "checking for /dev/ptmx... " >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 printf "%s\n" "not set" >&6; } - as_fn_error $? "set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 - fi - if test "${ac_cv_file__dev_ptc+set}" != set; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 + as_fn_error $? "set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + fi + if test "${ac_cv_file__dev_ptc+set}" != set; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 printf %s "checking for /dev/ptc... " >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 printf "%s\n" "not set" >&6; } - as_fn_error $? "set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + as_fn_error $? "set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + fi fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 printf %s "checking for /dev/ptmx... " >&6; } if test ${ac_cv_file__dev_ptmx+y} then : @@ -26696,12 +27890,12 @@ then : fi -if test "x$ac_cv_file__dev_ptmx" = xyes; then + if test "x$ac_cv_file__dev_ptmx" = xyes; then printf "%s\n" "#define HAVE_DEV_PTMX 1" >>confdefs.h -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 printf %s "checking for /dev/ptc... " >&6; } if test ${ac_cv_file__dev_ptc+y} then : @@ -26722,10 +27916,11 @@ then : fi -if test "x$ac_cv_file__dev_ptc" = xyes; then + if test "x$ac_cv_file__dev_ptc" = xyes; then printf "%s\n" "#define HAVE_DEV_PTC 1" >>confdefs.h + fi fi if test $ac_sys_system = Darwin @@ -26745,6 +27940,9 @@ ac_fn_c_check_type "$LINENO" "socklen_t" "ac_cv_type_socklen_t" " if test "x$ac_cv_type_socklen_t" = xyes then : +printf "%s\n" "#define HAVE_SOCKLEN_T 1" >>confdefs.h + + else $as_nop printf "%s\n" "#define socklen_t int" >>confdefs.h @@ -26893,7 +28091,6 @@ done SRCDIRS="\ Modules \ - Modules/_blake2 \ Modules/_ctypes \ Modules/_decimal \ Modules/_decimal/libmpdec \ @@ -26904,6 +28101,7 @@ SRCDIRS="\ Modules/_sre \ Modules/_testcapi \ Modules/_testinternalcapi \ + Modules/_testlimitedcapi \ Modules/_xxtestfuzz \ Modules/cjkcodecs \ Modules/expat \ @@ -26915,8 +28113,7 @@ SRCDIRS="\ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for build directories" >&5 printf %s "checking for build directories... " >&6; } for dir in $SRCDIRS; do @@ -27083,6 +28280,8 @@ else $as_nop with_ensurepip=no ;; #( WASI) : with_ensurepip=no ;; #( + iOS) : + with_ensurepip=no ;; #( *) : with_ensurepip=upgrade ;; @@ -27836,9 +29035,6 @@ fi # builtin hash modules default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" - -printf "%s\n" "#define PY_BUILTIN_HASHLIB_HASHES /**/" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-builtin-hashlib-hashes" >&5 printf %s "checking for --with-builtin-hashlib-hashes... " >&6; } @@ -27863,6 +29059,7 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_builtin_hashlib_hashes" >&5 printf "%s\n" "$with_builtin_hashlib_hashes" >&6; } + printf "%s\n" "#define PY_BUILTIN_HASHLIB_HASHES \"$with_builtin_hashlib_hashes\"" >>confdefs.h @@ -27879,95 +29076,13 @@ for builtin_hash in $with_builtin_hashlib_hashes; do sha3) : with_builtin_sha3=yes ;; #( blake2) : - with_builtin_blake2=yes - ;; #( - *) : - ;; -esac -done -IFS=$as_save_IFS - -if test "x$with_builtin_blake2" = xyes -then : - - -pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libb2" >&5 -printf %s "checking for libb2... " >&6; } - -if test -n "$LIBB2_CFLAGS"; then - pkg_cv_LIBB2_CFLAGS="$LIBB2_CFLAGS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libb2\""; } >&5 - ($PKG_CONFIG --exists --print-errors "libb2") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_LIBB2_CFLAGS=`$PKG_CONFIG --cflags "libb2" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi -if test -n "$LIBB2_LIBS"; then - pkg_cv_LIBB2_LIBS="$LIBB2_LIBS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libb2\""; } >&5 - ($PKG_CONFIG --exists --print-errors "libb2") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_LIBB2_LIBS=`$PKG_CONFIG --libs "libb2" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi - - - -if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then - _pkg_short_errors_supported=yes -else - _pkg_short_errors_supported=no -fi - if test $_pkg_short_errors_supported = yes; then - LIBB2_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libb2" 2>&1` - else - LIBB2_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libb2" 2>&1` - fi - # Put the nasty error message in config.log where it belongs - echo "$LIBB2_PKG_ERRORS" >&5 - - have_libb2=no -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - have_libb2=no -else - LIBB2_CFLAGS=$pkg_cv_LIBB2_CFLAGS - LIBB2_LIBS=$pkg_cv_LIBB2_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - - have_libb2=yes - -printf "%s\n" "#define HAVE_LIBB2 1" >>confdefs.h - - -fi - -fi + with_builtin_blake2=yes + ;; #( + *) : + ;; +esac +done +IFS=$as_save_IFS # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. @@ -28022,10 +29137,6 @@ printf %s "checking whether libatomic is needed by ... " >&6; } if test ${ac_cv_libatomic_needed+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes -then : - ac_cv_libatomic_needed=no else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -28067,16 +29178,14 @@ int main() } _ACEOF -if ac_fn_c_try_run "$LINENO" +if ac_fn_c_try_link "$LINENO" then : ac_cv_libatomic_needed=no else $as_nop ac_cv_libatomic_needed=yes fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libatomic_needed" >&5 printf "%s\n" "$ac_cv_libatomic_needed" >&6; } @@ -28108,6 +29217,27 @@ case $ac_sys_system in #( ;; #( Darwin) : ;; #( + iOS) : + + + + py_cv_module__curses=n/a + py_cv_module__curses_panel=n/a + py_cv_module__gdbm=n/a + py_cv_module__multiprocessing=n/a + py_cv_module__posixshmem=n/a + py_cv_module__posixsubprocess=n/a + py_cv_module__scproxy=n/a + py_cv_module__tkinter=n/a + py_cv_module_grp=n/a + py_cv_module_nis=n/a + py_cv_module_readline=n/a + py_cv_module_pwd=n/a + py_cv_module_spwd=n/a + py_cv_module_syslog=n/a + py_cv_module_=n/a + + ;; #( CYGWIN*) : @@ -28123,7 +29253,7 @@ case $ac_sys_system in #( py_cv_module__scproxy=n/a ;; #( - Emscripten|WASI) : + Emscripten) : @@ -28136,18 +29266,15 @@ case $ac_sys_system in #( py_cv_module__posixsubprocess=n/a py_cv_module__scproxy=n/a py_cv_module__tkinter=n/a - py_cv_module__xxsubinterpreters=n/a - py_cv_module__xxinterpchannels=n/a - py_cv_module__xxinterpqueues=n/a + py_cv_module__interpreters=n/a + py_cv_module__interpchannels=n/a + py_cv_module__interpqueues=n/a py_cv_module_grp=n/a py_cv_module_pwd=n/a py_cv_module_resource=n/a py_cv_module_syslog=n/a py_cv_module_=n/a - case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser*) : - py_cv_module_fcntl=n/a @@ -28155,24 +29282,43 @@ case $ac_sys_system in #( py_cv_module_termios=n/a py_cv_module_=n/a - ;; #( - Emscripten/node*) : - ;; #( - WASI/*) : + ;; #( + WASI) : + + + + py_cv_module__curses=n/a + py_cv_module__curses_panel=n/a + py_cv_module__dbm=n/a + py_cv_module__gdbm=n/a + py_cv_module__multiprocessing=n/a + py_cv_module__posixshmem=n/a + py_cv_module__posixsubprocess=n/a + py_cv_module__scproxy=n/a + py_cv_module__tkinter=n/a + py_cv_module__interpreters=n/a + py_cv_module__interpchannels=n/a + py_cv_module__interpqueues=n/a + py_cv_module_grp=n/a + py_cv_module_pwd=n/a + py_cv_module_resource=n/a + py_cv_module_syslog=n/a + py_cv_module_=n/a py_cv_module__ctypes_test=n/a + py_cv_module__testexternalinspection=n/a + py_cv_module__testimportmultiple=n/a + py_cv_module__testmultiphase=n/a + py_cv_module__testsinglephase=n/a py_cv_module_fcntl=n/a py_cv_module_mmap=n/a py_cv_module_termios=n/a + py_cv_module_xxlimited=n/a + py_cv_module_xxlimited_35=n/a py_cv_module_=n/a - - ;; #( - *) : - ;; -esac ;; #( *) : @@ -28418,28 +29564,6 @@ then : -fi - - - if test "$py_cv_module__opcode" != "n/a" -then : - py_cv_module__opcode=yes -fi - if test "$py_cv_module__opcode" = yes; then - MODULE__OPCODE_TRUE= - MODULE__OPCODE_FALSE='#' -else - MODULE__OPCODE_TRUE='#' - MODULE__OPCODE_FALSE= -fi - - as_fn_append MODULE_BLOCK "MODULE__OPCODE_STATE=$py_cv_module__opcode$as_nl" - if test "x$py_cv_module__opcode" = xyes -then : - - - - fi @@ -28597,20 +29721,20 @@ then : fi - if test "$py_cv_module__xxsubinterpreters" != "n/a" + if test "$py_cv_module__interpreters" != "n/a" then : - py_cv_module__xxsubinterpreters=yes + py_cv_module__interpreters=yes fi - if test "$py_cv_module__xxsubinterpreters" = yes; then - MODULE__XXSUBINTERPRETERS_TRUE= - MODULE__XXSUBINTERPRETERS_FALSE='#' + if test "$py_cv_module__interpreters" = yes; then + MODULE__INTERPRETERS_TRUE= + MODULE__INTERPRETERS_FALSE='#' else - MODULE__XXSUBINTERPRETERS_TRUE='#' - MODULE__XXSUBINTERPRETERS_FALSE= + MODULE__INTERPRETERS_TRUE='#' + MODULE__INTERPRETERS_FALSE= fi - as_fn_append MODULE_BLOCK "MODULE__XXSUBINTERPRETERS_STATE=$py_cv_module__xxsubinterpreters$as_nl" - if test "x$py_cv_module__xxsubinterpreters" = xyes + as_fn_append MODULE_BLOCK "MODULE__INTERPRETERS_STATE=$py_cv_module__interpreters$as_nl" + if test "x$py_cv_module__interpreters" = xyes then : @@ -28619,20 +29743,20 @@ then : fi - if test "$py_cv_module__xxinterpchannels" != "n/a" + if test "$py_cv_module__interpchannels" != "n/a" then : - py_cv_module__xxinterpchannels=yes + py_cv_module__interpchannels=yes fi - if test "$py_cv_module__xxinterpchannels" = yes; then - MODULE__XXINTERPCHANNELS_TRUE= - MODULE__XXINTERPCHANNELS_FALSE='#' + if test "$py_cv_module__interpchannels" = yes; then + MODULE__INTERPCHANNELS_TRUE= + MODULE__INTERPCHANNELS_FALSE='#' else - MODULE__XXINTERPCHANNELS_TRUE='#' - MODULE__XXINTERPCHANNELS_FALSE= + MODULE__INTERPCHANNELS_TRUE='#' + MODULE__INTERPCHANNELS_FALSE= fi - as_fn_append MODULE_BLOCK "MODULE__XXINTERPCHANNELS_STATE=$py_cv_module__xxinterpchannels$as_nl" - if test "x$py_cv_module__xxinterpchannels" = xyes + as_fn_append MODULE_BLOCK "MODULE__INTERPCHANNELS_STATE=$py_cv_module__interpchannels$as_nl" + if test "x$py_cv_module__interpchannels" = xyes then : @@ -28641,20 +29765,20 @@ then : fi - if test "$py_cv_module__xxinterpqueues" != "n/a" + if test "$py_cv_module__interpqueues" != "n/a" then : - py_cv_module__xxinterpqueues=yes + py_cv_module__interpqueues=yes fi - if test "$py_cv_module__xxinterpqueues" = yes; then - MODULE__XXINTERPQUEUES_TRUE= - MODULE__XXINTERPQUEUES_FALSE='#' + if test "$py_cv_module__interpqueues" = yes; then + MODULE__INTERPQUEUES_TRUE= + MODULE__INTERPQUEUES_FALSE='#' else - MODULE__XXINTERPQUEUES_TRUE='#' - MODULE__XXINTERPQUEUES_FALSE= + MODULE__INTERPQUEUES_TRUE='#' + MODULE__INTERPQUEUES_FALSE= fi - as_fn_append MODULE_BLOCK "MODULE__XXINTERPQUEUES_STATE=$py_cv_module__xxinterpqueues$as_nl" - if test "x$py_cv_module__xxinterpqueues" = xyes + as_fn_append MODULE_BLOCK "MODULE__INTERPQUEUES_STATE=$py_cv_module__interpqueues$as_nl" + if test "x$py_cv_module__interpqueues" = xyes then : @@ -28952,7 +30076,7 @@ fi then : - + as_fn_append MODULE_BLOCK "MODULE__SOCKET_LDFLAGS=$SOCKET_LIBS$as_nl" fi if test "$py_cv_module__socket" = yes; then @@ -29473,7 +30597,7 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" + fi @@ -29511,7 +30635,7 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" + fi @@ -29549,7 +30673,7 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" + fi @@ -29625,8 +30749,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBB2_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=$LIBB2_LIBS$as_nl" + + fi if test "$py_cv_module__blake2" = yes; then @@ -29641,6 +30765,153 @@ fi printf "%s\n" "$py_cv_module__blake2" >&6; } +LIBHACL_CFLAGS='-I$(srcdir)/Modules/_hacl -I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED)' +case "$ac_sys_system" in + Linux*) + if test "$ac_cv_func_explicit_bzero" = "no"; then + LIBHACL_CFLAGS="$LIBHACL_CFLAGS -DLINUX_NO_EXPLICIT_BZERO" + fi + ;; +esac + + +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2" >&5 +printf %s "checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2... " >&6; } +if test ${ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -msse -msse2 -msse3 -msse4.1 -msse4.2" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2=yes +else $as_nop + ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2" >&6; } +if test "x$ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2" = xyes +then : + + LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2" + + +printf "%s\n" "#define HACL_CAN_COMPILE_SIMD128 1" >>confdefs.h + + + # macOS universal2 builds *support* the -msse etc flags because they're + # available on x86_64. However, performance of the HACL SIMD128 implementation + # isn't great, so it's disabled on ARM64. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD128 implementation" >&5 +printf %s "checking for HACL* SIMD128 implementation... " >&6; } + if test "$UNIVERSAL_ARCHS" == "universal2"; then + LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 +printf "%s\n" "universal2" >&6; } + else + LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: standard" >&5 +printf "%s\n" "standard" >&6; } + fi + + +else $as_nop + : +fi + +fi + + + +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +# +# Although AVX support is not guaranteed on Android +# (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a +# runtime CPUID check. +if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mavx2" >&5 +printf %s "checking whether C compiler accepts -mavx2... " >&6; } +if test ${ax_cv_check_cflags__Werror__mavx2+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -mavx2" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__mavx2=yes +else $as_nop + ax_cv_check_cflags__Werror__mavx2=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__mavx2" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__mavx2" >&6; } +if test "x$ax_cv_check_cflags__Werror__mavx2" = xyes +then : + + LIBHACL_SIMD256_FLAGS="-mavx2" + +printf "%s\n" "#define HACL_CAN_COMPILE_SIMD256 1" >>confdefs.h + + + # macOS universal2 builds *support* the -mavx2 compiler flag because it's + # available on x86_64; but the HACL SIMD256 build then fails because the + # implementation requires symbols that aren't available on ARM64. Use a + # wrapped implementation if we're building for universal2. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD256 implementation" >&5 +printf %s "checking for HACL* SIMD256 implementation... " >&6; } + if test "$UNIVERSAL_ARCHS" == "universal2"; then + LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 +printf "%s\n" "universal2" >&6; } + else + LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: standard" >&5 +printf "%s\n" "standard" >&6; } + fi + +else $as_nop + : +fi + +fi + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _ctypes" >&5 printf %s "checking for stdlib extension module _ctypes... " >&6; } @@ -29687,7 +30958,7 @@ then : if true then : - if test "$have_curses" != "no" + if test "$have_curses" = "yes" then : py_cv_module__curses=yes else $as_nop @@ -29726,7 +30997,7 @@ then : if true then : - if test "$have_panel" != "no" + if test "$have_curses" = "yes" && test "$have_panel" = "yes" then : py_cv_module__curses_panel=yes else $as_nop @@ -29765,7 +31036,7 @@ then : if true then : - if true + if test "$have_mpdec" = "yes" then : py_cv_module__decimal=yes else $as_nop @@ -29781,7 +31052,7 @@ fi then : as_fn_append MODULE_BLOCK "MODULE__DECIMAL_CFLAGS=$LIBMPDEC_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__DECIMAL_LDFLAGS=$LIBMPDEC_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__DECIMAL_LDFLAGS=$LIBMPDEC_LIBS$as_nl" fi if test "$py_cv_module__decimal" = yes; then @@ -30353,6 +31624,44 @@ fi printf "%s\n" "$py_cv_module__testclinic_limited" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testlimitedcapi" >&5 +printf %s "checking for stdlib extension module _testlimitedcapi... " >&6; } + if test "$py_cv_module__testlimitedcapi" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if true +then : + py_cv_module__testlimitedcapi=yes +else $as_nop + py_cv_module__testlimitedcapi=missing +fi +else $as_nop + py_cv_module__testlimitedcapi=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTLIMITEDCAPI_STATE=$py_cv_module__testlimitedcapi$as_nl" + if test "x$py_cv_module__testlimitedcapi" = xyes +then : + + + + +fi + if test "$py_cv_module__testlimitedcapi" = yes; then + MODULE__TESTLIMITEDCAPI_TRUE= + MODULE__TESTLIMITEDCAPI_FALSE='#' +else + MODULE__TESTLIMITEDCAPI_TRUE='#' + MODULE__TESTLIMITEDCAPI_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testlimitedcapi" >&5 +printf "%s\n" "$py_cv_module__testlimitedcapi" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testinternalcapi" >&5 printf %s "checking for stdlib extension module _testinternalcapi... " >&6; } if test "$py_cv_module__testinternalcapi" != "n/a" @@ -30505,6 +31814,82 @@ fi printf "%s\n" "$py_cv_module__testmultiphase" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testsinglephase" >&5 +printf %s "checking for stdlib extension module _testsinglephase... " >&6; } + if test "$py_cv_module__testsinglephase" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if test "$ac_cv_func_dlopen" = yes +then : + py_cv_module__testsinglephase=yes +else $as_nop + py_cv_module__testsinglephase=missing +fi +else $as_nop + py_cv_module__testsinglephase=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTSINGLEPHASE_STATE=$py_cv_module__testsinglephase$as_nl" + if test "x$py_cv_module__testsinglephase" = xyes +then : + + + + +fi + if test "$py_cv_module__testsinglephase" = yes; then + MODULE__TESTSINGLEPHASE_TRUE= + MODULE__TESTSINGLEPHASE_FALSE='#' +else + MODULE__TESTSINGLEPHASE_TRUE='#' + MODULE__TESTSINGLEPHASE_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testsinglephase" >&5 +printf "%s\n" "$py_cv_module__testsinglephase" >&6; } + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testexternalinspection" >&5 +printf %s "checking for stdlib extension module _testexternalinspection... " >&6; } + if test "$py_cv_module__testexternalinspection" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if true +then : + py_cv_module__testexternalinspection=yes +else $as_nop + py_cv_module__testexternalinspection=missing +fi +else $as_nop + py_cv_module__testexternalinspection=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTEXTERNALINSPECTION_STATE=$py_cv_module__testexternalinspection$as_nl" + if test "x$py_cv_module__testexternalinspection" = xyes +then : + + + + +fi + if test "$py_cv_module__testexternalinspection" = yes; then + MODULE__TESTEXTERNALINSPECTION_TRUE= + MODULE__TESTEXTERNALINSPECTION_FALSE='#' +else + MODULE__TESTEXTERNALINSPECTION_TRUE='#' + MODULE__TESTEXTERNALINSPECTION_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testexternalinspection" >&5 +printf "%s\n" "$py_cv_module__testexternalinspection" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxsubtype" >&5 printf %s "checking for stdlib extension module xxsubtype... " >&6; } if test "$py_cv_module_xxsubtype" != "n/a" @@ -30603,8 +31988,8 @@ fi if test "x$py_cv_module__ctypes_test" = xyes then : - - as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBM$as_nl" + as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_CFLAGS=$LIBFFI_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBFFI_LIBS $LIBM$as_nl" fi if test "$py_cv_module__ctypes_test" = yes; then @@ -30625,7 +32010,7 @@ printf %s "checking for stdlib extension module xxlimited... " >&6; } if test "$py_cv_module_xxlimited" != "n/a" then : - if true + if test "$TEST_MODULES" = yes then : if test "$ac_cv_func_dlopen" = yes then : @@ -30663,7 +32048,7 @@ printf %s "checking for stdlib extension module xxlimited_35... " >&6; } if test "$py_cv_module_xxlimited_35" != "n/a" then : - if true + if test "$TEST_MODULES" = yes then : if test "$ac_cv_func_dlopen" = yes then : @@ -30857,10 +32242,6 @@ if test -z "${MODULE__LSPROF_TRUE}" && test -z "${MODULE__LSPROF_FALSE}"; then as_fn_error $? "conditional \"MODULE__LSPROF\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__OPCODE_TRUE}" && test -z "${MODULE__OPCODE_FALSE}"; then - as_fn_error $? "conditional \"MODULE__OPCODE\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi if test -z "${MODULE__PICKLE_TRUE}" && test -z "${MODULE__PICKLE_FALSE}"; then as_fn_error $? "conditional \"MODULE__PICKLE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -30889,16 +32270,16 @@ if test -z "${MODULE__TYPING_TRUE}" && test -z "${MODULE__TYPING_FALSE}"; then as_fn_error $? "conditional \"MODULE__TYPING\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__XXSUBINTERPRETERS_TRUE}" && test -z "${MODULE__XXSUBINTERPRETERS_FALSE}"; then - as_fn_error $? "conditional \"MODULE__XXSUBINTERPRETERS\" was never defined. +if test -z "${MODULE__INTERPRETERS_TRUE}" && test -z "${MODULE__INTERPRETERS_FALSE}"; then + as_fn_error $? "conditional \"MODULE__INTERPRETERS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__XXINTERPCHANNELS_TRUE}" && test -z "${MODULE__XXINTERPCHANNELS_FALSE}"; then - as_fn_error $? "conditional \"MODULE__XXINTERPCHANNELS\" was never defined. +if test -z "${MODULE__INTERPCHANNELS_TRUE}" && test -z "${MODULE__INTERPCHANNELS_FALSE}"; then + as_fn_error $? "conditional \"MODULE__INTERPCHANNELS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__XXINTERPQUEUES_TRUE}" && test -z "${MODULE__XXINTERPQUEUES_FALSE}"; then - as_fn_error $? "conditional \"MODULE__XXINTERPQUEUES\" was never defined. +if test -z "${MODULE__INTERPQUEUES_TRUE}" && test -z "${MODULE__INTERPQUEUES_FALSE}"; then + as_fn_error $? "conditional \"MODULE__INTERPQUEUES\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE__ZONEINFO_TRUE}" && test -z "${MODULE__ZONEINFO_FALSE}"; then @@ -31101,6 +32482,10 @@ if test -z "${MODULE__TESTCLINIC_LIMITED_TRUE}" && test -z "${MODULE__TESTCLINIC as_fn_error $? "conditional \"MODULE__TESTCLINIC_LIMITED\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTLIMITEDCAPI_TRUE}" && test -z "${MODULE__TESTLIMITEDCAPI_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTLIMITEDCAPI\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__TESTINTERNALCAPI_TRUE}" && test -z "${MODULE__TESTINTERNALCAPI_FALSE}"; then as_fn_error $? "conditional \"MODULE__TESTINTERNALCAPI\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -31117,6 +32502,14 @@ if test -z "${MODULE__TESTMULTIPHASE_TRUE}" && test -z "${MODULE__TESTMULTIPHASE as_fn_error $? "conditional \"MODULE__TESTMULTIPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTSINGLEPHASE_TRUE}" && test -z "${MODULE__TESTSINGLEPHASE_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTSINGLEPHASE\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${MODULE__TESTEXTERNALINSPECTION_TRUE}" && test -z "${MODULE__TESTEXTERNALINSPECTION_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTEXTERNALINSPECTION\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then as_fn_error $? "conditional \"MODULE_XXSUBTYPE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -31527,7 +32920,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by python $as_me 3.13, which was +This file was extended by python $as_me 3.14, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -31591,7 +32984,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -python config.status 3.13 +python config.status 3.14 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" @@ -31719,6 +33112,7 @@ do "Mac/PythonLauncher/Makefile") CONFIG_FILES="$CONFIG_FILES Mac/PythonLauncher/Makefile" ;; "Mac/Resources/framework/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/framework/Info.plist" ;; "Mac/Resources/app/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/app/Info.plist" ;; + "iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES iOS/Resources/Info.plist" ;; "Makefile.pre") CONFIG_FILES="$CONFIG_FILES Makefile.pre" ;; "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;; diff --git a/configure.ac b/configure.ac index dee7ed552b370f..6d514705e91ce5 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ dnl to regenerate the configure script. dnl # Set VERSION so we only need to edit in one place (i.e., here) -m4_define([PYTHON_VERSION], [3.13]) +m4_define([PYTHON_VERSION], [3.14]) AC_PREREQ([2.71]) @@ -164,7 +164,7 @@ AC_ARG_WITH([build-python], dnl Build Python interpreter is used for regeneration and freezing. ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python AC_MSG_RESULT([$with_build_python]) ], [ AS_VAR_IF([cross_compiling], [yes], @@ -307,6 +307,143 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then AC_MSG_ERROR([pkg-config is required])] fi +# Set name for machine-dependent library files +AC_ARG_VAR([MACHDEP], [name for machine-dependent library files]) +AC_MSG_CHECKING([MACHDEP]) +if test -z "$MACHDEP" +then + # avoid using uname for cross builds + if test "$cross_compiling" = yes; then + # ac_sys_system and ac_sys_release are used for setting + # a lot of different things including 'define_xopen_source' + # in the case statement below. + case "$host" in + *-*-linux-android*) + ac_sys_system=Linux-android + ;; + *-*-linux*) + ac_sys_system=Linux + ;; + *-*-cygwin*) + ac_sys_system=Cygwin + ;; + *-apple-ios*) + ac_sys_system=iOS + ;; + *-*-vxworks*) + ac_sys_system=VxWorks + ;; + *-*-emscripten) + ac_sys_system=Emscripten + ;; + *-*-wasi) + ac_sys_system=WASI + ;; + *) + # for now, limit cross builds to known configurations + MACHDEP="unknown" + AC_MSG_ERROR([cross build not supported for $host]) + esac + ac_sys_release= + else + ac_sys_system=`uname -s` + if test "$ac_sys_system" = "AIX" \ + -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then + ac_sys_release=`uname -v` + else + ac_sys_release=`uname -r` + fi + fi + ac_md_system=`echo $ac_sys_system | + tr -d '[/ ]' | tr '[[A-Z]]' '[[a-z]]'` + ac_md_release=`echo $ac_sys_release | + tr -d '[/ ]' | sed 's/^[[A-Z]]\.//' | sed 's/\..*//'` + MACHDEP="$ac_md_system$ac_md_release" + + case $MACHDEP in + aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; + linux*) MACHDEP="linux";; + cygwin*) MACHDEP="cygwin";; + darwin*) MACHDEP="darwin";; + '') MACHDEP="unknown";; + esac + + if test "$ac_sys_system" = "SunOS"; then + # For Solaris, there isn't an OS version specific macro defined + # in most compilers, so we define one here. + SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\([0-9]\)$!.0\1!g' | tr -d '.'` + AC_DEFINE_UNQUOTED([Py_SUNOS_VERSION], [$SUNOS_VERSION], + [The version of SunOS/Solaris as reported by `uname -r' without the dot.]) + fi +fi +AC_MSG_RESULT(["$MACHDEP"]) + +dnl For cross compilation, we distinguish between "prefix" (where we install the +dnl files) and "host_prefix" (where we expect to find the files at runtime) + +if test -z "$host_prefix"; then + AS_CASE([$ac_sys_system], + [Emscripten], [host_prefix=/], + [host_prefix='${prefix}'] + ) +fi +AC_SUBST([host_prefix]) + +if test -z "$host_exec_prefix"; then + AS_CASE([$ac_sys_system], + [Emscripten], [host_exec_prefix=$host_prefix], + [host_exec_prefix='${exec_prefix}'] + ) +fi +AC_SUBST([host_exec_prefix]) + +# On cross-compile builds, configure will look for a host-specific compiler by +# prepending the user-provided host triple to the required binary name. +# +# On iOS, this results in binaries like "arm64-apple-ios13.0-simulator-gcc", +# which isn't a binary that exists, and isn't very convenient, as it contains the +# iOS version. As the default cross-compiler name won't exist, configure falls +# back to gcc, which *definitely* won't work. We're providing wrapper scripts for +# these tools; the binary names of these scripts are better defaults than "gcc". +# This only requires that the user put the platform scripts folder (e.g., +# "iOS/Resources/bin") in their path, rather than defining platform-specific +# names/paths for AR, CC, CPP, and CXX explicitly; and if the user forgets to +# either put the platform scripts folder in the path, or specify CC etc, +# configure will fail. +if test -z "$AR"; then + case "$host" in + aarch64-apple-ios*-simulator) AR=arm64-apple-ios-simulator-ar ;; + aarch64-apple-ios*) AR=arm64-apple-ios-ar ;; + x86_64-apple-ios*-simulator) AR=x86_64-apple-ios-simulator-ar ;; + *) + esac +fi +if test -z "$CC"; then + case "$host" in + aarch64-apple-ios*-simulator) CC=arm64-apple-ios-simulator-clang ;; + aarch64-apple-ios*) CC=arm64-apple-ios-clang ;; + x86_64-apple-ios*-simulator) CC=x86_64-apple-ios-simulator-clang ;; + *) + esac +fi +if test -z "$CPP"; then + case "$host" in + aarch64-apple-ios*-simulator) CPP=arm64-apple-ios-simulator-cpp ;; + aarch64-apple-ios*) CPP=arm64-apple-ios-cpp ;; + x86_64-apple-ios*-simulator) CPP=x86_64-apple-ios-simulator-cpp ;; + *) + esac +fi +if test -z "$CXX"; then + case "$host" in + aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang++ ;; + aarch64-apple-ios*) CXX=arm64-apple-ios-clang++ ;; + x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang++ ;; + *) + esac +fi + AC_MSG_CHECKING([for --enable-universalsdk]) AC_ARG_ENABLE([universalsdk], AS_HELP_STRING([--enable-universalsdk@<:@=SDKDIR@:>@], @@ -416,109 +553,155 @@ AC_ARG_ENABLE([framework], [ case $enableval in yes) - enableval=/Library/Frameworks + case $ac_sys_system in + Darwin) enableval=/Library/Frameworks ;; + iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + *) AC_MSG_ERROR([Unknown platform for framework build]) + esac esac + case $enableval in no) - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - if test "x${prefix}" = "xNONE"; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= + case $ac_sys_system in + iOS) AC_MSG_ERROR([iOS builds must use --enable-framework]) ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE"; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac ;; *) PYTHONFRAMEWORKPREFIX="${enableval}" PYTHONFRAMEWORKINSTALLDIR=$PYTHONFRAMEWORKPREFIX/$PYTHONFRAMEWORKDIR - FRAMEWORKINSTALLFIRST="frameworkinstallstructure" - FRAMEWORKALTINSTALLFIRST="frameworkinstallstructure " - FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" - FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" - FRAMEWORKPYTHONW="frameworkpythonw" - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - - case "${enableval}" in - /System*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "${prefix}" = "NONE" ; then - # See below - FRAMEWORKUNIXTOOLSPREFIX="/usr" - fi - ;; + case $ac_sys_system in #( + Darwin) : + FRAMEWORKINSTALLFIRST="frameworkinstallversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" + FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" + FRAMEWORKPYTHONW="frameworkpythonw" + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi - /Library*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; + case "${enableval}" in + /System*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + if test "${prefix}" = "NONE" ; then + # See below + FRAMEWORKUNIXTOOLSPREFIX="/usr" + fi + ;; + + /Library*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + + */Library/Frameworks) + MDIR="`dirname "${enableval}"`" + MDIR="`dirname "${MDIR}"`" + FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" + + if test "${prefix}" = "NONE"; then + # User hasn't specified the + # --prefix option, but wants to install + # the framework in a non-default location, + # ensure that the compatibility links get + # installed relative to that prefix as well + # instead of in /usr/local. + FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" + fi + ;; - */Library/Frameworks) - MDIR="`dirname "${enableval}"`" - MDIR="`dirname "${MDIR}"`" - FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" - - if test "${prefix}" = "NONE"; then - # User hasn't specified the - # --prefix option, but wants to install - # the framework in a non-default location, - # ensure that the compatibility links get - # installed relative to that prefix as well - # instead of in /usr/local. - FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" - fi - ;; + *) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + esac - *) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; + prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION + PYTHONFRAMEWORKINSTALLNAMEPREFIX=${prefix} + RESSRCDIR=Mac/Resources/framework + + # Add files for Mac specific code to the list of output + # files: + AC_CONFIG_FILES([Mac/Makefile]) + AC_CONFIG_FILES([Mac/PythonLauncher/Makefile]) + AC_CONFIG_FILES([Mac/Resources/framework/Info.plist]) + AC_CONFIG_FILES([Mac/Resources/app/Info.plist]) + ;; + iOS) : + FRAMEWORKINSTALLFIRST="frameworkinstallunversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallunversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKALTINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKPYTHONW= + INSTALLTARGETS="libinstall inclinstall sharedinstall" + + prefix=$PYTHONFRAMEWORKPREFIX + PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" + RESSRCDIR=iOS/Resources + + AC_CONFIG_FILES([iOS/Resources/Info.plist]) + ;; + *) + AC_MSG_ERROR([Unknown platform for framework build]) + ;; + esac esac - - prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION - - # Add files for Mac specific code to the list of output - # files: - AC_CONFIG_FILES([Mac/Makefile]) - AC_CONFIG_FILES([Mac/PythonLauncher/Makefile]) - AC_CONFIG_FILES([Mac/Resources/framework/Info.plist]) - AC_CONFIG_FILES([Mac/Resources/app/Info.plist]) - esac ],[ - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= - + case $ac_sys_system in + iOS) AC_MSG_ERROR([iOS builds must use --enable-framework]) ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac ]) AC_SUBST([PYTHONFRAMEWORK]) AC_SUBST([PYTHONFRAMEWORKIDENTIFIER]) AC_SUBST([PYTHONFRAMEWORKDIR]) AC_SUBST([PYTHONFRAMEWORKPREFIX]) AC_SUBST([PYTHONFRAMEWORKINSTALLDIR]) +AC_SUBST([PYTHONFRAMEWORKINSTALLNAMEPREFIX]) +AC_SUBST([RESSRCDIR]) AC_SUBST([FRAMEWORKINSTALLFIRST]) AC_SUBST([FRAMEWORKINSTALLLAST]) AC_SUBST([FRAMEWORKALTINSTALLFIRST]) @@ -526,77 +709,51 @@ AC_SUBST([FRAMEWORKALTINSTALLLAST]) AC_SUBST([FRAMEWORKPYTHONW]) AC_SUBST([FRAMEWORKUNIXTOOLSPREFIX]) AC_SUBST([FRAMEWORKINSTALLAPPSPREFIX]) +AC_SUBST([INSTALLTARGETS]) AC_DEFINE_UNQUOTED([_PYTHONFRAMEWORK], ["${PYTHONFRAMEWORK}"], [framework name]) -# Set name for machine-dependent library files -AC_ARG_VAR([MACHDEP], [name for machine-dependent library files]) -AC_MSG_CHECKING([MACHDEP]) -if test -z "$MACHDEP" -then - # avoid using uname for cross builds - if test "$cross_compiling" = yes; then - # ac_sys_system and ac_sys_release are used for setting - # a lot of different things including 'define_xopen_source' - # in the case statement below. - case "$host" in - *-*-linux-android*) - ac_sys_system=Linux-android - ;; - *-*-linux*) - ac_sys_system=Linux - ;; - *-*-cygwin*) - ac_sys_system=Cygwin - ;; - *-*-vxworks*) - ac_sys_system=VxWorks - ;; - *-*-emscripten) - ac_sys_system=Emscripten - ;; - *-*-wasi) - ac_sys_system=WASI - ;; - *) - # for now, limit cross builds to known configurations - MACHDEP="unknown" - AC_MSG_ERROR([cross build not supported for $host]) - esac - ac_sys_release= - else - ac_sys_system=`uname -s` - if test "$ac_sys_system" = "AIX" \ - -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then - ac_sys_release=`uname -v` - else - ac_sys_release=`uname -r` - fi - fi - ac_md_system=`echo $ac_sys_system | - tr -d '[/ ]' | tr '[[A-Z]]' '[[a-z]]'` - ac_md_release=`echo $ac_sys_release | - tr -d '[/ ]' | sed 's/^[[A-Z]]\.//' | sed 's/\..*//'` - MACHDEP="$ac_md_system$ac_md_release" - - case $MACHDEP in - aix*) MACHDEP="aix";; - linux*) MACHDEP="linux";; - cygwin*) MACHDEP="cygwin";; - darwin*) MACHDEP="darwin";; - '') MACHDEP="unknown";; +dnl quadrigraphs "@<:@" and "@:>@" produce "[" and "]" in the output +AC_MSG_CHECKING([for --with-app-store-compliance]) +AC_ARG_WITH( + [app_store_compliance], + [AS_HELP_STRING( + [--with-app-store-compliance=@<:@PATCH-FILE@:>@], + [Enable any patches required for compiliance with app stores. + Optional PATCH-FILE specifies the custom patch to apply.] + )],[ + case "$withval" in + yes) + case $ac_sys_system in + Darwin|iOS) + # iOS is able to share the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + ;; + *) AC_MSG_ERROR([no default app store compliance patch available for $ac_sys_system]) ;; + esac + AC_MSG_RESULT([applying default app store compliance patch]) + ;; + *) + APP_STORE_COMPLIANCE_PATCH="${withval}" + AC_MSG_RESULT([applying custom app store compliance patch]) + ;; esac - - if test "$ac_sys_system" = "SunOS"; then - # For Solaris, there isn't an OS version specific macro defined - # in most compilers, so we define one here. - SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\([0-9]\)$!.0\1!g' | tr -d '.'` - AC_DEFINE_UNQUOTED([Py_SUNOS_VERSION], [$SUNOS_VERSION], - [The version of SunOS/Solaris as reported by `uname -r' without the dot.]) - fi -fi -AC_MSG_RESULT(["$MACHDEP"]) + ],[ + case $ac_sys_system in + iOS) + # Always apply the compliance patch on iOS; we can use the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + AC_MSG_RESULT([applying default app store compliance patch]) + ;; + *) + # No default app compliance patching on any other platform + APP_STORE_COMPLIANCE_PATCH= + AC_MSG_RESULT([not patching for app store compliance]) + ;; + esac +]) +AC_SUBST([APP_STORE_COMPLIANCE_PATCH]) AC_SUBST([_PYTHON_HOST_PLATFORM]) if test "$cross_compiling" = yes; then @@ -604,27 +761,47 @@ if test "$cross_compiling" = yes; then *-*-linux*) case "$host_cpu" in arm*) - _host_cpu=arm + _host_ident=arm ;; *) - _host_cpu=$host_cpu + _host_ident=$host_cpu esac ;; *-*-cygwin*) - _host_cpu= + _host_ident= + ;; + *-apple-ios*) + _host_os=`echo $host | cut -d '-' -f3` + _host_device=`echo $host | cut -d '-' -f4` + _host_device=${_host_device:=os} + + # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + AC_MSG_CHECKING([iOS deployment target]) + IPHONEOS_DEPLOYMENT_TARGET=$(echo ${_host_os} | cut -c4-) + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=13.0} + AC_MSG_RESULT([$IPHONEOS_DEPLOYMENT_TARGET]) + + case "$host_cpu" in + aarch64) + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + ;; + *) + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + ;; + esac ;; *-*-vxworks*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; wasm32-*-* | wasm64-*-*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; *) # for now, limit cross builds to known configurations MACHDEP="unknown" AC_MSG_ERROR([cross build not supported for $host]) esac - _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + _PYTHON_HOST_PLATFORM="$MACHDEP${_host_ident:+-$_host_ident}" fi # Some systems cannot stand _XOPEN_SOURCE being defined at all; they @@ -690,6 +867,9 @@ case $ac_sys_system/$ac_sys_release in define_xopen_source=no;; Darwin/@<:@[12]@:>@@<:@0-9@:>@.*) define_xopen_source=no;; + # On iOS, defining _POSIX_C_SOURCE also disables platform specific features. + iOS/*) + define_xopen_source=no;; # On QNX 6.3.2, defining _XOPEN_SOURCE prevents netdb.h from # defining NI_NUMERICHOST. QNX/6.3.2) @@ -748,6 +928,9 @@ AC_SUBST([EXPORT_MACOSX_DEPLOYMENT_TARGET]) CONFIGURE_MACOSX_DEPLOYMENT_TARGET= EXPORT_MACOSX_DEPLOYMENT_TARGET='#' +# Record the value of IPHONEOS_DEPLOYMENT_TARGET enforced by the selected host triple. +AC_SUBST([IPHONEOS_DEPLOYMENT_TARGET]) + # checks for alternative programs # compiler flags are generated in two sets, BASECFLAGS and OPT. OPT is just @@ -780,6 +963,14 @@ AS_CASE([$host], ], ) +dnl Add the compiler flag for the iOS minimum supported OS version. +AS_CASE([$ac_sys_system], + [iOS], [ + AS_VAR_APPEND([CFLAGS], [" -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}"]) + AS_VAR_APPEND([LDFLAGS], [" -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}"]) + ], +) + if test "$ac_sys_system" = "Darwin" then dnl look for SDKROOT @@ -863,7 +1054,9 @@ dnl check for GCC last, other compilers set __GNUC__, too. dnl msvc is listed for completeness. AC_CACHE_CHECK([for CC compiler name], [ac_cv_cc_name], [ cat > conftest.c <conftest.out 2>/dev/null; then ac_cv_cc_name=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` + if test $(expr "//$CC" : '.*/\(.*\)') = "mpicc"; then + ac_cv_cc_name="mpicc" + fi else ac_cv_cc_name="unknown" fi @@ -891,6 +1087,18 @@ rm -f conftest.c conftest.out # _POSIX_SOURCE, _POSIX_1_SOURCE, and more AC_USE_SYSTEM_EXTENSIONS +AC_CACHE_CHECK([for GCC compatible compiler], + [ac_cv_gcc_compat], + [AC_PREPROC_IFELSE([AC_LANG_SOURCE([ + #if !defined(__GNUC__) + #error "not GCC compatible" + #else + /* GCC compatible! */ + #endif + ], [])], + [ac_cv_gcc_compat=yes], + [ac_cv_gcc_compat=no])]) + AC_SUBST([CXX]) preset_cxx="$CXX" @@ -935,14 +1143,22 @@ else fi rm -f conftest.out +dnl On some platforms, using a true "triplet" for MULTIARCH would be redundant. +dnl For example, `arm64-apple-darwin` is redundant, because there isn't a +dnl non-Apple Darwin. Including the CPU architecture can also be potentially +dnl redundant - on macOS, for example, it's possible to do a single compile +dnl pass that includes multiple architectures, so it would be misleading for +dnl MULTIARCH (and thus the sysconfigdata module name) to include a single CPU +dnl architecture. PLATFORM_TRIPLET will be a pair or single value for these +dnl platforms. AC_MSG_CHECKING([for multiarch]) AS_CASE([$ac_sys_system], [Darwin*], [MULTIARCH=""], + [iOS], [MULTIARCH=""], [FreeBSD*], [MULTIARCH=""], [MULTIARCH=$($CC --print-multiarch 2>/dev/null)] ) AC_SUBST([MULTIARCH]) -AC_MSG_RESULT([$MULTIARCH]) if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then if test x$PLATFORM_TRIPLET != x$MULTIARCH; then @@ -952,6 +1168,17 @@ elif test x$PLATFORM_TRIPLET != x && test x$MULTIARCH = x; then MULTIARCH=$PLATFORM_TRIPLET fi AC_SUBST([PLATFORM_TRIPLET]) +AC_MSG_RESULT([$MULTIARCH]) + +dnl Even if we *do* include the CPU architecture in the MULTIARCH value, some +dnl platforms don't need the CPU architecture in the SOABI tag. These platforms +dnl will have multiple sysconfig modules (one for each CPU architecture), but +dnl use a single "fat" binary at runtime. SOABI_PLATFORM is the component of +dnl the PLATFORM_TRIPLET that will be used in binary module extensions. +AS_CASE([$ac_sys_system], + [iOS], [SOABI_PLATFORM=`echo "$PLATFORM_TRIPLET" | cut -d '-' -f2`], + [SOABI_PLATFORM=$PLATFORM_TRIPLET] +) if test x$MULTIARCH != x; then MULTIARCH_CPPFLAGS="-DMULTIARCH=\\\"$MULTIARCH\\\"" @@ -966,10 +1193,10 @@ AC_MSG_CHECKING([for PEP 11 support tier]) AS_CASE([$host/$ac_cv_cc_name], [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version + [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC - [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=2], dnl macOS on M1, any version [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc @@ -981,6 +1208,11 @@ AS_CASE([$host/$ac_cv_cc_name], [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 + [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 + [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 + [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 + [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [PY_SUPPORT_TIER=0] ) @@ -1025,6 +1257,9 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then AC_DEFINE_UNQUOTED([ANDROID_API_LEVEL], [$ANDROID_API_LEVEL], [The Android API level.]) + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + AC_MSG_CHECKING([for the Android arm ABI]) AC_MSG_RESULT([$_arm_arch]) if test "$_arm_arch" = 7; then @@ -1123,6 +1358,9 @@ AC_ARG_WITH([suffix], ]) AC_MSG_RESULT([$EXEEXT]) +# Make sure we keep EXEEXT and ac_exeext sync'ed. +AS_VAR_SET([ac_exeext], [$EXEEXT]) + # Test whether we're running on a non-case-sensitive system, in which # case we give a warning if no ext is given AC_SUBST([BUILDEXEEXT]) @@ -1294,17 +1532,25 @@ fi AC_MSG_CHECKING([LDLIBRARY]) -# MacOSX framework builds need more magic. LDLIBRARY is the dynamic +# Apple framework builds need more magic. LDLIBRARY is the dynamic # library that we build, but we do not want to link against it (we # will find it with a -framework option). For this reason there is an # extra variable BLDLIBRARY against which Python and the extension # modules are linked, BLDLIBRARY. This is normally the same as -# LDLIBRARY, but empty for MacOSX framework builds. +# LDLIBRARY, but empty for MacOSX framework builds. iOS does the same, +# but uses a non-versioned framework layout. if test "$enable_framework" then - LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' - RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} + case $ac_sys_system in + Darwin) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)';; + iOS) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)';; + *) + AC_MSG_ERROR([Unknown platform for framework build]);; + esac BLDLIBRARY='' + RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} else BLDLIBRARY='$(LDLIBRARY)' fi @@ -1316,66 +1562,78 @@ if test $enable_shared = "yes"; then [Defined if Python is built as a shared library.]) case $ac_sys_system in CYGWIN*) - LDLIBRARY='libpython$(LDVERSION).dll.a' - DLLLIBRARY='libpython$(LDVERSION).dll' - ;; + LDLIBRARY='libpython$(LDVERSION).dll.a' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + DLLLIBRARY='libpython$(LDVERSION).dll' + ;; SunOS*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + INSTSONAME="$LDLIBRARY".$SOVERSION + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; Linux*|GNU*|NetBSD*|FreeBSD*|DragonFly*|OpenBSD*|VxWorks*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + + # The Android Gradle plugin will only package libraries whose names end + # with ".so". + if test "$ac_sys_system" != "Linux-android"; then + INSTSONAME="$LDLIBRARY".$SOVERSION + fi + + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; hp*|HP*) - case `uname -m` in - ia64) - LDLIBRARY='libpython$(LDVERSION).so' - ;; - *) - LDLIBRARY='libpython$(LDVERSION).sl' - ;; - esac - BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} - ;; + case `uname -m` in + ia64) + LDLIBRARY='libpython$(LDVERSION).so' + ;; + *) + LDLIBRARY='libpython$(LDVERSION).sl' + ;; + esac + BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} + ;; Darwin*) - LDLIBRARY='libpython$(LDVERSION).dylib' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).dylib' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} + ;; + iOS) + LDLIBRARY='libpython$(LDVERSION).dylib' + ;; AIX*) - LDLIBRARY='libpython$(LDVERSION).so' - RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).so' + RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} + ;; esac else # shared is disabled PY_ENABLE_SHARED=0 case $ac_sys_system in CYGWIN*) - BLDLIBRARY='$(LIBRARY)' - LDLIBRARY='libpython$(LDVERSION).dll.a' - ;; + BLDLIBRARY='$(LIBRARY)' + LDLIBRARY='libpython$(LDVERSION).dll.a' + ;; esac fi +AC_MSG_RESULT([$LDLIBRARY]) if test "$cross_compiling" = yes; then - RUNSHARED= + RUNSHARED= fi +AC_MSG_CHECKING([HOSTRUNNER]) AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then @@ -1383,45 +1641,16 @@ then [Emscripten/node*], [ AC_PATH_TOOL([NODE], [node], [node]) HOSTRUNNER="$NODE" - # bigint for ctypes c_longlong, c_longdouble - # no longer available in Node 16 - AC_CACHE_CHECK([for node --experimental-wasm-bigint], [ac_cv_tool_node_wasm_bigint], [ - if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then - ac_cv_tool_node_wasm_bigint=yes - else - ac_cv_tool_node_wasm_bigint=no - fi - ]) - AS_VAR_IF([ac_cv_tool_node_wasm_bigint], [yes], [ - AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bigint"]) - ]) - - AS_VAR_IF([enable_wasm_pthreads], [yes], [ - AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-threads"]) - # no longer available in Node 16 - AC_CACHE_CHECK([for node --experimental-wasm-bulk-memory], [ac_cv_tool_node_wasm_bulk_memory], [ - if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then - ac_cv_tool_node_wasm_bulk_memory=yes - else - ac_cv_tool_node_wasm_bulk_memory=no - fi - ]) - AS_VAR_IF([ac_cv_tool_node_wasm_bulk_memory], [yes], [ - AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bulk-memory"]) - ]) - ]) - AS_VAR_IF([host_cpu], [wasm64], [AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-memory64"])]) ], dnl TODO: support other WASI runtimes - dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the + dnl wasmtime starts the process with "/" as CWD. For OOT builds add the dnl directory containing _sysconfigdata to PYTHONPATH. - [WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --mapdir /::$(srcdir) --'], + [WASI/*], [HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2 --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/'], [HOSTRUNNER=''] ) fi AC_SUBST([HOSTRUNNER]) -AC_MSG_CHECKING([HOSTRUNNER]) AC_MSG_RESULT([$HOSTRUNNER]) if test -n "$HOSTRUNNER"; then @@ -1429,8 +1658,6 @@ if test -n "$HOSTRUNNER"; then PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD" fi -AC_MSG_RESULT([$LDLIBRARY]) - # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], [Emscripten/browser*], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js'], @@ -1491,7 +1718,9 @@ fi # For calculating the .so ABI tag. AC_SUBST([ABIFLAGS]) +AC_SUBST([ABI_THREAD]) ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -1508,6 +1737,7 @@ then [Define if you want to disable the GIL]) # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -1541,6 +1771,10 @@ then [Define if you want to enable tracing references for debugging purpose]) fi +if test "$disable_gil" = "yes" -a "$with_trace_refs" = "yes"; +then + AC_MSG_ERROR([--disable-gil cannot be used with --with-trace-refs]) +fi # Check for --enable-pystats AC_MSG_CHECKING([for --enable-pystats]) @@ -1581,14 +1815,27 @@ fi # Check for --enable-experimental-jit: AC_MSG_CHECKING([for --enable-experimental-jit]) AC_ARG_ENABLE([experimental-jit], - [AS_HELP_STRING([--enable-experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@], [build the experimental just-in-time compiler (default is no)])], [], [enable_experimental_jit=no]) -AS_VAR_IF([enable_experimental_jit], - [no], +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) AC_MSG_ERROR( + [invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;; +esac +AS_VAR_IF([tier2_flags], + [], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $tier2_flags"])]) +AS_VAR_IF([jit_flags], [], - [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) @@ -1598,7 +1845,7 @@ AS_VAR_IF([enable_experimental_jit], [])]) AC_SUBST([REGEN_JIT_COMMAND]) AC_SUBST([JIT_STENCILS_H]) -AC_MSG_RESULT([$enable_experimental_jit]) +AC_MSG_RESULT([$tier2_flags $jit_flags]) # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) @@ -1632,7 +1879,7 @@ if test "$Py_OPT" = 'true' ; then AX_CHECK_COMPILE_FLAG([-fno-semantic-interposition],[ CFLAGS_NODIST="$CFLAGS_NODIST -fno-semantic-interposition" LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" - ]) + ], [], [-Werror]) ;; esac elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then @@ -1844,9 +2091,13 @@ case "$CC_BASENAME" in *clang*) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=m4_normalize(" + ${LLVM_PROFDATA} merge + -output=\"\$(shell pwd)/code.profclangd\" + \"\$(shell pwd)\"/*.profclangr + ") + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test $LLVM_PROF_FOUND = not-found then LLVM_PROF_ERR=yes @@ -1860,9 +2111,13 @@ case "$CC_BASENAME" in case $ac_sys_system in Darwin*) PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=m4_normalize(" + ${LLVM_PROFDATA} merge + -output=\"\$(shell pwd)/code.profclangd\" + \"\$(shell pwd)\"/*.profclangr + ") + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test "${LLVM_PROF_FOUND}" = "not-found" then LLVM_PROF_ERR=yes @@ -2078,6 +2333,11 @@ PYDEBUG_CFLAGS="-O0" AS_VAR_IF([ac_cv_cc_supports_og], [yes], [PYDEBUG_CFLAGS="-Og"]) +# gh-120688: WASI uses -O3 in debug mode to support more recursive calls +if test "$ac_sys_system" = "WASI"; then + PYDEBUG_CFLAGS="-O3" +fi + # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line AC_SUBST([OPT]) @@ -2155,6 +2415,7 @@ AS_CASE([$ac_sys_system], ], [node*], [ AS_VAR_IF([ac_sys_emscripten_target], [node-debug], [wasm_debug=yes]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" --pre-js=\$(srcdir)/Tools/wasm/emscripten/node_pre.js"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sNODERAWFS"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXIT_RUNTIME"]) WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" @@ -2192,9 +2453,10 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--max-memory=10485760"]) ]) - dnl increase initial memory and stack size, move stack first + dnl gh-117645: Set the memory size to 40 MiB, the stack size to 16 MiB, + dnl and move the stack first. dnl https://github.com/WebAssembly/wasi-libc/issues/233 - AS_VAR_APPEND([LDFLAGS_NODIST], [" -z stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040"]) ] ) @@ -2220,7 +2482,7 @@ AC_DEFUN([PY_CHECK_CC_WARNING], [ AS_VAR_PUSHDEF([py_var], [ac_cv_$1_]m4_normalize($2)[_warning]) AC_CACHE_CHECK([m4_ifblank([$3], [if we can $1 $CC $2 warning], [$3])], [py_var], [ AS_VAR_COPY([py_cflags], [CFLAGS]) - AS_VAR_APPEND([CFLAGS], ["-W$2 -Werror"]) + AS_VAR_APPEND([CFLAGS], [" -W$2 -Werror"]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])], [AS_VAR_SET([py_var], [yes])], [AS_VAR_SET([py_var], [no])]) @@ -2234,6 +2496,36 @@ AS_VAR_IF([with_strict_overflow], [yes], [BASECFLAGS="$BASECFLAGS $STRICT_OVERFLOW_CFLAGS"], [BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS"]) +# Enable flags that warn and protect for potential security vulnerabilities. +# These flags should be enabled by default for all builds. + +AC_MSG_CHECKING([for --enable-safety]) +AC_ARG_ENABLE([safety], + [AS_HELP_STRING([--enable-safety], [enable usage of the security compiler options with no performance overhead])], + [AS_VAR_IF([disable_safety], [yes], [enable_safety=no], [enable_safety=yes])], [enable_safety=no]) +AC_MSG_RESULT([$enable_safety]) + +if test "$enable_safety" = "yes" +then + AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wtrampolines], [CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"], [AC_MSG_WARN([-Wimplicit-fallthrough not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Werror=format-security], [CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"], [AC_MSG_WARN([-Werror=format-security not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wbidi-chars=any], [CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"], [AC_MSG_WARN([-Wbidi-chars=any not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wall], [CFLAGS_NODIST="$CFLAGS_NODIST -Wall"], [AC_MSG_WARN([-Wall not supported])], [-Werror]) +fi + +AC_MSG_CHECKING([for --enable-slower-safety]) +AC_ARG_ENABLE([slower-safety], + [AS_HELP_STRING([--enable-slower-safety], [enable usage of the security compiler options with performance overhead])], + [AS_VAR_IF([disable_slower_safety], [yes], [enable_slower_safety=no], [enable_slower_safety=yes])], [enable_slower_safety=no]) +AC_MSG_RESULT([$enable_slower_safety]) + +if test "$enable_slower_safety" = "yes" +then + AX_CHECK_COMPILE_FLAG([-D_FORTIFY_SOURCE=3], [CFLAGS_NODIST="$CFLAGS_NODIST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3"], [AC_MSG_WARN([-D_FORTIFY_SOURCE=3 not supported])], [-Werror]) +fi + case $GCC in yes) CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -2716,7 +3008,7 @@ AC_DEFINE([STDC_HEADERS], [1], AC_CHECK_HEADERS([ \ alloca.h asm/types.h bluetooth.h conio.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ - linux/random.h linux/soundcard.h \ + linux/netfilter_ipv4.h linux/random.h linux/soundcard.h \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ @@ -2783,15 +3075,11 @@ AC_CHECK_HEADERS( #endif ]) -# checks for typedefs -AC_CACHE_CHECK([for clock_t in time.h], [ac_cv_clock_t_time_h], [ - AC_EGREP_HEADER([clock_t], [time.h], [ac_cv_clock_t_time_h=yes], [ac_cv_clock_t_time_h=no]) -]) -dnl checks for "no" -AS_VAR_IF([ac_cv_clock_t_time_h], [no], [ - AC_DEFINE([clock_t], [long], - [Define to 'long' if doesn't define.]) -]) +# Check for clock_t in time.h. +AC_CHECK_TYPES([clock_t], [], + [AC_DEFINE([clock_t], [long], + [Define to 'long' if does not define clock_t.])], + [@%:@include ]) AC_CACHE_CHECK([for makedev], [ac_cv_func_makedev], [ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -2866,12 +3154,10 @@ AC_DEFINE_UNQUOTED([RETSIGTYPE],[void],[assume C89 semantics that RETSIGTYPE is AC_TYPE_SIZE_T AC_TYPE_UID_T -AC_CHECK_TYPE([ssize_t], - AC_DEFINE([HAVE_SSIZE_T], [1], - [Define if your compiler provides ssize_t]), [], []) -AC_CHECK_TYPE([__uint128_t], - AC_DEFINE([HAVE_GCC_UINT128_T], [1], - [Define if your compiler provides __uint128_t]), [], []) +AC_CHECK_TYPES([ssize_t]) +AC_CHECK_TYPES([__uint128_t], + [AC_DEFINE([HAVE_GCC_UINT128_T], [1], + [Define if your compiler provides __uint128_t])]) # Sizes and alignments of various common basic types # ANSI C requires sizeof(char) == 1, so no need to check it @@ -3206,7 +3492,12 @@ then BLDSHARED="$LDSHARED" fi ;; - Emscripten|WASI) + iOS/*) + LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + BLDSHARED="$LDSHARED" + ;; + Emscripten*|WASI*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; Linux*|GNU*|QNX*|VxWorks*|Haiku*) @@ -3252,7 +3543,7 @@ then LDCXXSHARED='$(CXX) -Wl,-G,-Bexport';; WASI*) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ - dnl not iplemented yet + dnl not implemented yet ]);; CYGWIN*) LDSHARED="gcc -shared -Wl,--enable-auto-image-base" @@ -3290,7 +3581,6 @@ then then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; - Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; Emscripten*|WASI*) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ @@ -3326,30 +3616,34 @@ then Linux-android*) LINKFORSHARED="-pie -Xlinker -export-dynamic";; Linux*|GNU*) LINKFORSHARED="-Xlinker -export-dynamic";; # -u libsys_s pulls in all symbols in libsys - Darwin/*) + Darwin/*|iOS/*) LINKFORSHARED="$extra_undefs -framework CoreFoundation" # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - stack_size="1000000" # 16 MB - if test "$with_ubsan" = "yes" - then - # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" # 64 MB - fi + stack_size="1000000" # 16 MB + if test "$with_ubsan" = "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi - LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + AC_DEFINE_UNQUOTED([THREAD_STACK_SIZE], + [0x$stack_size], + [Custom thread stack size depending on chosen sanitizer runtimes.]) - AC_DEFINE_UNQUOTED([THREAD_STACK_SIZE], - [0x$stack_size], - [Custom thread stack size depending on chosen sanitizer runtimes.]) + if test $ac_sys_system = "Darwin"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" - if test "$enable_framework" - then - LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + if test "$enable_framework"; then + LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + fi + LINKFORSHARED="$LINKFORSHARED" + elif test $ac_sys_system = "iOS"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)' fi - LINKFORSHARED="$LINKFORSHARED";; + ;; OpenUNIX*|UnixWare*) LINKFORSHARED="-Wl,-Bexport";; SCO_SV*) LINKFORSHARED="-Wl,-Bexport";; ReliantUNIX*) LINKFORSHARED="-W1 -Blargedynsym";; @@ -3422,7 +3716,7 @@ esac AC_MSG_RESULT([$SHLIBS]) dnl perf trampoline is Linux specific and requires an arch-specific -dnl trampoline in asssembly. +dnl trampoline in assembly. AC_MSG_CHECKING([perf trampoline]) AS_CASE([$PLATFORM_TRIPLET], [x86_64-linux-gnu], [perf_trampoline=yes], @@ -3503,6 +3797,13 @@ AS_VAR_IF([have_uuid], [missing], [ ]) ]) +# gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. +# This restriction inhibits the proper generation of time-based UUIDs. +if test "$ac_sys_system" = "NetBSD"; then + have_uuid=missing + AC_DEFINE([HAVE_UUID_H], [0]) +fi + AS_VAR_IF([have_uuid], [missing], [have_uuid=no]) # 'Real Time' functions on Solaris @@ -3543,6 +3844,35 @@ dnl The AIX_BUILDDATE is obtained from the kernel fileset - bos.mp64 *) ;; esac +# check for _Complex C type +# +# Note that despite most compilers define __STDC_IEC_559_COMPLEX__ - almost +# none properly support C11+ Annex G (where pure imaginary types +# represented by _Imaginary are mandatory). This is a bug (see e.g. +# llvm/llvm-project#60269), so we don't rely on presence +# of __STDC_IEC_559_COMPLEX__. +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#define test(type, out) \ +{ \ + type complex z = 1 + 2*I; z = z*z; \ + (out) = (out) || creal(z) != -3 || cimag(z) != 4; \ +} +int main(void) +{ + int res = 0; + test(float, res); + test(double, res); + test(long double, res); + return res; +}]])], [ac_cv_c_complex_supported=yes], +[ac_cv_c_complex_supported=no], +[ac_cv_c_complex_supported=no]) +if test "$ac_cv_c_complex_supported" = "yes"; then + AC_DEFINE([Py_HAVE_C_COMPLEX], [1], + [Defined if _Complex C type is available.]) +fi + # check for systems that require aligned memory access AC_CACHE_CHECK([aligned memory access is required], [ac_cv_aligned_required], [AC_RUN_IFELSE([AC_LANG_SOURCE([[ @@ -3560,7 +3890,14 @@ int main(void) }]])], [ac_cv_aligned_required=no], [ac_cv_aligned_required=yes], -[ac_cv_aligned_required=yes]) +[ +# "yes" changes the hash function to FNV, which causes problems with Numba +# (https://github.com/numba/numba/blob/0.59.0/numba/cpython/hashing.py#L470). +if test "$ac_sys_system" = "Linux-android"; then + ac_cv_aligned_required=no +else + ac_cv_aligned_required=yes +fi]) ]) if test "$ac_cv_aligned_required" = yes ; then AC_DEFINE([HAVE_ALIGNED_REQUIRED], [1], @@ -3723,6 +4060,9 @@ AS_VAR_IF([have_libffi], [yes], [ dnl when do we need USING_APPLE_OS_LIBFFI? ctypes_malloc_closure=yes ], + [iOS], [ + ctypes_malloc_closure=yes + ], [sunos5], [AS_VAR_APPEND([LIBFFI_LIBS], [" -mimpure-text"])] ) AS_VAR_IF([ctypes_malloc_closure], [yes], [ @@ -3744,35 +4084,105 @@ AS_VAR_IF([have_libffi], [yes], [ ]) ]) +# Check for libffi with real complex double support. +# This is a workaround, since FFI_TARGET_HAS_COMPLEX_TYPE was defined in libffi v3.2.1, +# but real support was provided only in libffi v3.3.0. +# See https://github.com/python/cpython/issues/125206 for more details. +# +AC_CACHE_CHECK([libffi has complex type support], [ac_cv_ffi_complex_double_supported], +[WITH_SAVE_ENV([ + CPPFLAGS="$LIBFFI_CFLAGS $CPPFLAGS" + LDFLAGS="$LIBFFI_LIBS $LDFLAGS" + LIBS="$LIBFFI_LIBS $LIBS" +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include +int z_is_expected(double complex z) +{ + const double complex expected = CMPLX(1.25, -0.5); + return z == expected; +} +int main(void) +{ + double complex z = 1.25 - 0.5 * I; + ffi_type *args[1] = {&ffi_type_complex_double}; + void *values[1] = {&z}; + ffi_cif cif; + if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, 1, + &ffi_type_sint, args) != FFI_OK) + { + return 2; + } + ffi_arg rc; + ffi_call(&cif, FFI_FN(z_is_expected), &rc, values); + return !rc; +} +]])], [ac_cv_ffi_complex_double_supported=yes], +[ac_cv_ffi_complex_double_supported=no], +[ac_cv_ffi_complex_double_supported=no]) +])]) +if test "$ac_cv_ffi_complex_double_supported" = "yes"; then + AC_DEFINE([Py_FFI_SUPPORT_C_COMPLEX], [1], + [Defined if _Complex C type can be used with libffi.]) +fi + # Check for use of the system libmpdec library AC_MSG_CHECKING([for --with-system-libmpdec]) AC_ARG_WITH( [system_libmpdec], [AS_HELP_STRING( [--with-system-libmpdec], - [build _decimal module using an installed libmpdec library, see Doc/library/decimal.rst (default is no)] + [build _decimal module using an installed mpdecimal library, see Doc/library/decimal.rst (default is yes)] )], [], - [with_system_libmpdec="no"]) + [with_system_libmpdec="yes"]) AC_MSG_RESULT([$with_system_libmpdec]) -AS_VAR_IF([with_system_libmpdec], [yes], [ - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LDFLAGS=${LIBMPDEC_LDFLAGS-"-lmpdec"} - LIBMPDEC_INTERNAL= -], [ - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - - dnl Disable forced inlining in debug builds, see GH-94847 - AS_VAR_IF([with_pydebug], [yes], [ - AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DTEST_COVERAGE"]) - ]) -]) - -AC_SUBST([LIBMPDEC_CFLAGS]) -AC_SUBST([LIBMPDEC_INTERNAL]) +AC_DEFUN([USE_BUNDLED_LIBMPDEC], + [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no]) + +AS_VAR_IF( + [with_system_libmpdec], [yes], + [PKG_CHECK_MODULES( + [LIBMPDEC], [libmpdec >= 2.5.0], [], + [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL=])], + [USE_BUNDLED_LIBMPDEC()]) + +AS_VAR_IF([with_system_libmpdec], [yes], + [WITH_SAVE_ENV([ + CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" + LIBS="$LIBMPDEC_LIBS $LIBS" + + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([ + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + ], [const char *x = mpd_version();])], + [have_mpdec=yes], + [have_mpdec=no]) + ])], + [AC_MSG_WARN([m4_normalize([ + the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; + consider using a system installed mpdecimal library.])])]) + +AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], + [AC_MSG_WARN([m4_normalize([ + no system libmpdecimal found; falling back to bundled libmpdecimal + (deprecated and scheduled for removal in Python 3.15)])]) + USE_BUNDLED_LIBMPDEC()]) + +# Disable forced inlining in debug builds, see GH-94847 +AS_VAR_IF( + [with_pydebug], [yes], + [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DTEST_COVERAGE"])]) # Check whether _decimal should use a coroutine-local or thread-local context AC_MSG_CHECKING([for --with-decimal-contextvar]) @@ -3793,50 +4203,52 @@ fi AC_MSG_RESULT([$with_decimal_contextvar]) -# Check for libmpdec machine flavor -AC_MSG_CHECKING([for decimal libmpdec machine]) -AS_CASE([$ac_sys_system], - [Darwin*], [libmpdec_system=Darwin], - [SunOS*], [libmpdec_system=sunos], - [libmpdec_system=other] -) - -libmpdec_machine=unknown -if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal -elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi -elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - AS_CASE([$CC], - [*gcc*], [libmpdec_machine=ppro], - [*clang*], [libmpdec_machine=ppro], - [libmpdec_machine=ansi32] - ) - else - libmpdec_machine=ansi32 - fi -fi -AC_MSG_RESULT([$libmpdec_machine]) - -AS_CASE([$libmpdec_machine], - [x64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DASM=1"])], - [uint128], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1"])], - [ansi64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1"])], - [ppro], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas"])], - [ansi32], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1"])], - [ansi-legacy], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1"])], - [universal], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DUNIVERSAL=1"])], - [AC_MSG_ERROR([_decimal: unsupported architecture])] -) +AS_VAR_IF( + [with_system_libmpdec], [no], + [# Check for libmpdec machine flavor + AC_MSG_CHECKING([for decimal libmpdec machine]) + AS_CASE([$ac_sys_system], + [Darwin*], [libmpdec_system=Darwin], + [SunOS*], [libmpdec_system=sunos], + [libmpdec_system=other] + ) + + libmpdec_machine=unknown + if test "$libmpdec_system" = Darwin; then + # universal here means: build libmpdec with the same arch options + # the python interpreter was built with + libmpdec_machine=universal + elif test $ac_cv_sizeof_size_t -eq 8; then + if test "$ac_cv_gcc_asm_for_x64" = yes; then + libmpdec_machine=x64 + elif test "$ac_cv_type___uint128_t" = yes; then + libmpdec_machine=uint128 + else + libmpdec_machine=ansi64 + fi + elif test $ac_cv_sizeof_size_t -eq 4; then + if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then + AS_CASE([$CC], + [*gcc*], [libmpdec_machine=ppro], + [*clang*], [libmpdec_machine=ppro], + [libmpdec_machine=ansi32] + ) + else + libmpdec_machine=ansi32 + fi + fi + AC_MSG_RESULT([$libmpdec_machine]) + + AS_CASE([$libmpdec_machine], + [x64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DASM=1"])], + [uint128], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1"])], + [ansi64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1"])], + [ppro], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas"])], + [ansi32], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1"])], + [ansi-legacy], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1"])], + [universal], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DUNIVERSAL=1"])], + [AC_MSG_ERROR([_decimal: unsupported architecture])] + )]) if test "$have_ipa_pure_const_bug" = yes; then # Some versions of gcc miscompile inline asm: @@ -3851,6 +4263,9 @@ if test "$have_glibc_memmove_bug" = yes; then AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -U_FORTIFY_SOURCE"]) fi +AC_SUBST([LIBMPDEC_CFLAGS]) +AC_SUBST([LIBMPDEC_INTERNAL]) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -4209,13 +4624,9 @@ else # define _POSIX_THREADS in unistd.h. Some apparently don't # (e.g. gnu pth with pthread emulation) AC_MSG_CHECKING([for _POSIX_THREADS in unistd.h]) - AC_EGREP_CPP([yes], - [ -#include -#ifdef _POSIX_THREADS -yes -#endif - ], unistd_defines_pthreads=yes, unistd_defines_pthreads=no) + AX_CHECK_DEFINE([unistd.h], [_POSIX_THREADS], + [unistd_defines_pthreads=yes], + [unistd_defines_pthreads=no]) AC_MSG_RESULT([$unistd_defines_pthreads]) AC_DEFINE([_REENTRANT]) @@ -4389,40 +4800,26 @@ ipv6lib=none ipv6trylibc=no if test "$ipv6" = yes -a "$cross_compiling" = no; then - AC_MSG_CHECKING([ipv6 stack type]) for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta; do case $i in inria) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include -#ifdef IPV6_INRIA_VERSION -yes -@%:@endif], - [ipv6type=$i]) + AX_CHECK_DEFINE([netinet/in.h], [IPV6_INRIA_VERSION], [ipv6type=$i]) ;; kame) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include -#ifdef __KAME__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes]) + AX_CHECK_DEFINE([netinet/in.h], [__KAME__], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes]) ;; linux-glibc) - dnl http://www.v6.linux.or.jp/ - AC_EGREP_CPP([yes], [ -#include -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -@%:@endif], - [ipv6type=$i; - ipv6trylibc=yes]) + dnl Advanced IPv6 support was added to glibc 2.1 in 1999. + AX_CHECK_DEFINE([features.h], [__GLIBC__], + [ipv6type=$i + ipv6trylibc=yes]) ;; linux-inet6) dnl http://www.v6.linux.or.jp/ @@ -4442,51 +4839,41 @@ yes fi ;; toshiba) - AC_EGREP_CPP([yes], [ -#include -#ifdef _TOSHIBA_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_TOSHIBA_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; v6d) - AC_EGREP_CPP([yes], [ -#include -#ifdef __V6D__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) + AX_CHECK_DEFINE([/usr/local/v6/include/sys/v6config.h], [__V6D__], + [ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) ;; zeta) - AC_EGREP_CPP([yes], [ -#include -#ifdef _ZETA_MINAMI_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_ZETA_MINAMI_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; esac if test "$ipv6type" != "unknown"; then break fi done + AC_MSG_CHECKING([ipv6 stack type]) AC_MSG_RESULT([$ipv6type]) fi if test "$ipv6" = "yes" -a "$ipv6lib" != "none"; then + AC_MSG_CHECKING([ipv6 library]) if test -d $ipv6libdir -a -f $ipv6libdir/lib$ipv6lib.a; then LIBS="-L$ipv6libdir -l$ipv6lib $LIBS" - AC_MSG_NOTICE([using lib$ipv6lib]) + AC_MSG_RESULT([lib$ipv6lib]) else AS_VAR_IF([ipv6trylibc], [yes], [ - AC_MSG_NOTICE([using libc]) + AC_MSG_RESULT([libc]) ], [ AC_MSG_ERROR([m4_normalize([ No $ipv6lib library found; cannot continue. @@ -4602,7 +4989,7 @@ elif test "$disable_gil" = "yes"; then fi AC_MSG_RESULT([$with_mimalloc]) -AC_SUBST([WITH_MIMALLOC]) +AC_SUBST([INSTALL_MIMALLOC], [$with_mimalloc]) AC_SUBST([MIMALLOC_HEADERS]) # Check for Python-specific malloc support @@ -4627,24 +5014,6 @@ then fi AC_MSG_RESULT([$with_pymalloc]) -# Check whether objects such as float, tuple and dict are using -# freelists to optimization memory allocation. -AC_MSG_CHECKING([for --with-freelists]) -AC_ARG_WITH( - [freelists], - [AS_HELP_STRING([--with-freelists], [enable object freelists (default is yes)])]) - -if test -z "$with_freelists" -then - with_freelists="yes" -fi -if test "$with_freelists" != "no" -then - AC_DEFINE([WITH_FREELISTS], [1], - [Define if you want to compile in object freelists optimization]) -fi -AC_MSG_RESULT([$with_freelists]) - # Check for --with-c-locale-coercion AC_MSG_CHECKING([for --with-c-locale-coercion]) AC_ARG_WITH( @@ -4782,14 +5151,38 @@ else AC_MSG_RESULT([$MACHDEP_OBJS]) fi +if test "$ac_sys_system" = "Linux-android"; then + # When these functions are used in an unprivileged process, they crash rather + # than returning an error. + blocked_funcs="chroot initgroups setegid seteuid setgid sethostname + setregid setresgid setresuid setreuid setuid" + + # These functions are unimplemented and always return an error + # (https://android.googlesource.com/platform/system/sepolicy/+/refs/heads/android13-release/public/domain.te#1044) + blocked_funcs="$blocked_funcs sem_open sem_unlink" + + # Before API level 23, when fchmodat is called with the unimplemented flag + # AT_SYMLINK_NOFOLLOW, instead of returning ENOTSUP as it should, it actually + # follows the symlink. + if test "$ANDROID_API_LEVEL" -lt 23; then + blocked_funcs="$blocked_funcs fchmodat" + fi + + for name in $blocked_funcs; do + AS_VAR_PUSHDEF([func_var], [ac_cv_func_$name]) + AS_VAR_SET([func_var], [no]) + AS_VAR_POPDEF([func_var]) + done +fi + # checks for library functions AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ - gai_strerror getegid getentropy geteuid getgid getgrent getgrgid getgrgid_r \ - getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \ + gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ + getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ @@ -4797,7 +5190,7 @@ AC_CHECK_FUNCS([ \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ - pread preadv preadv2 pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pread preadv preadv2 process_vm_readv pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ @@ -4806,7 +5199,7 @@ AC_CHECK_FUNCS([ \ setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ - sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ + sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ tmpnam tmpnam_r truncate ttyname umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) @@ -4818,6 +5211,14 @@ if test "$MACHDEP" != linux; then AC_CHECK_FUNCS([lchmod]) fi +# iOS defines some system methods that can be linked (so they are +# found by configure), but either raise a compilation error (because the +# header definition prevents usage - autoconf doesn't use the headers), or +# raise an error if used at runtime. Force these symbols off. +if test "$ac_sys_system" != "iOS" ; then + AC_CHECK_FUNCS([getentropy getgroups system]) +fi + AC_CHECK_DECL([dirfd], [AC_DEFINE([HAVE_DIRFD], [1], [Define if you have the 'dirfd' function or macro.])], @@ -4910,10 +5311,13 @@ AS_VAR_IF([ac_cv_broken_unsetenv], [yes], [ dnl check for true AC_CHECK_PROGS([TRUE], [true], [/bin/true]) -dnl On some systems (e.g. Solaris 9), hstrerror and inet_aton are in -lresolv +dnl On some systems (e.g. Solaris), hstrerror and inet_aton are in -lresolv dnl On others, they are in the C library, so we to take no action AC_CHECK_LIB([c], [inet_aton], [$ac_cv_prog_TRUE], - AC_CHECK_LIB([resolv], [inet_aton]) + AC_CHECK_LIB([resolv], [inet_aton], [SOCKET_LIBS="-lresolv"]) +) +AC_CHECK_LIB([c], [hstrerror], [$ac_cv_prog_TRUE], + AC_CHECK_LIB([resolv], [hstrerror], [SOCKET_LIBS="-lresolv"]) ) # On Tru64, chflags seems to be present, but calling it will @@ -5118,17 +5522,28 @@ AC_CHECK_FUNCS([clock_getres], [], [ ]) ]) -AC_CHECK_FUNCS([clock_settime], [], [ - AC_CHECK_LIB([rt], [clock_settime], [ - AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) - ]) -]) +# On Android and iOS, clock_settime can be linked (so it is found by +# configure), but when used in an unprivileged process, it crashes rather than +# returning an error. Force the symbol off. +if test "$ac_sys_system" != "Linux-android" && test "$ac_sys_system" != "iOS" +then + AC_CHECK_FUNCS([clock_settime], [], [ + AC_CHECK_LIB([rt], [clock_settime], [ + AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) + ]) + ]) +fi -AC_CHECK_FUNCS([clock_nanosleep], [], [ - AC_CHECK_LIB([rt], [clock_nanosleep], [ - AC_DEFINE([HAVE_CLOCK_NANOSLEEP], [1]) - ]) -]) +# On Android before API level 23, clock_nanosleep returns the wrong value when +# interrupted by a signal (https://issuetracker.google.com/issues/216495770). +if ! { test "$ac_sys_system" = "Linux-android" && + test "$ANDROID_API_LEVEL" -lt 23; }; then + AC_CHECK_FUNCS([clock_nanosleep], [], [ + AC_CHECK_LIB([rt], [clock_nanosleep], [ + AC_DEFINE([HAVE_CLOCK_NANOSLEEP], [1]) + ]) + ]) +fi AC_CHECK_FUNCS([nanosleep], [], [ AC_CHECK_LIB([rt], [nanosleep], [ @@ -5268,7 +5683,9 @@ int main(void) [ac_cv_buggy_getaddrinfo=no], [ac_cv_buggy_getaddrinfo=yes], [ -if test "${enable_ipv6+set}" = set; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_buggy_getaddrinfo="no" +elif test "${enable_ipv6+set}" = set; then ac_cv_buggy_getaddrinfo="no -- configured with --(en|dis)able-ipv6" else ac_cv_buggy_getaddrinfo=yes @@ -5545,7 +5962,7 @@ else # While Python doesn't currently have full support for these platforms # (see e.g., issue 1762561), we can at least make sure that float <-> string # conversions work. - # FLOAT_WORDS_BIGENDIAN doesnt actually detect this case, but if it's not big + # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, but if it's not big # or little, then it must be this? AC_DEFINE([DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754], [1], [Define if C doubles are 64-bit IEEE 754 binary format, stored @@ -5824,7 +6241,7 @@ AC_SUBST([SOABI]) AC_MSG_CHECKING([ABIFLAGS]) AC_MSG_RESULT([$ABIFLAGS]) AC_MSG_CHECKING([SOABI]) -SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} +SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${SOABI_PLATFORM:+-$SOABI_PLATFORM} AC_MSG_RESULT([$SOABI]) # Release build, debug build (Py_DEBUG), and trace refs build (Py_TRACE_REFS) @@ -5832,7 +6249,7 @@ AC_MSG_RESULT([$SOABI]) if test "$Py_DEBUG" = 'true'; then # Similar to SOABI but remove "d" flag from ABIFLAGS AC_SUBST([ALT_SOABI]) - ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} + ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${SOABI_PLATFORM:+-$SOABI_PLATFORM} AC_DEFINE_UNQUOTED([ALT_SOABI], ["${ALT_SOABI}"], [Alternative SOABI used in debug build to load C extensions built in release mode]) fi @@ -5844,21 +6261,33 @@ AC_MSG_CHECKING([LDVERSION]) LDVERSION='$(VERSION)$(ABIFLAGS)' AC_MSG_RESULT([$LDVERSION]) -# On Android and Cygwin the shared libraries must be linked with libpython. +# Configure the flags and dependencies used when compiling shared modules. +# Do not rename LIBPYTHON - it's accessed via sysconfig by package build +# systems (e.g. Meson) to decide whether to link extension modules against +# libpython. +AC_SUBST([MODULE_DEPS_SHARED]) AC_SUBST([LIBPYTHON]) +MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' +LIBPYTHON='' + +# On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -else - LIBPYTHON='' + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" + LIBPYTHON="\$(BLDLIBRARY)" +fi + +# On iOS the shared libraries must be linked with the Python framework +if test "$ac_sys_system" = "iOS"; then + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi AC_SUBST([BINLIBDEST]) -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) AC_SUBST([PLATLIBDIR]) PLATLIBDIR="lib" AC_MSG_CHECKING([for --with-platlibdir]) @@ -5877,7 +6306,7 @@ if test -n "$withval" -a "$withval" != yes -a "$withval" != no then AC_MSG_RESULT([yes]) PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else AC_MSG_RESULT([no]) fi], @@ -5887,9 +6316,9 @@ fi], dnl define LIBPL after ABIFLAGS and LDVERSION is defined. AC_SUBST([PY_ENABLE_SHARED]) if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi AC_SUBST([LIBPL]) @@ -6108,11 +6537,22 @@ AS_VAR_IF([with_readline], [no], [ ]) # in readline as well as newer editline (April 2023) - AC_CHECK_TYPE([rl_compdisp_func_t], - [AC_DEFINE([HAVE_RL_COMPDISP_FUNC_T], [1], - [Define if readline supports rl_compdisp_func_t])], - [], - [readline_includes]) + AC_CHECK_TYPES([rl_compdisp_func_t], [], [], [readline_includes]) + + # Some editline versions declare rl_startup_hook as taking no args, others + # declare it as taking 2. + AC_CACHE_CHECK([if rl_startup_hook takes arguments], [ac_cv_readline_rl_startup_hook_takes_args], [ + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([readline_includes] + [extern int test_hook_func(const char *text, int state);], + [rl_startup_hook=test_hook_func;])], + [ac_cv_readline_rl_startup_hook_takes_args=yes], + [ac_cv_readline_rl_startup_hook_takes_args=no] + ) + ]) + AS_VAR_IF([ac_cv_readline_rl_startup_hook_takes_args], [yes], [ + AC_DEFINE([Py_RL_STARTUP_HOOK_TAKES_ARGS], [1], [Define if rl_startup_hook takes arguments]) + ]) m4_undefine([readline_includes]) ])dnl WITH_SAVE_ENV() @@ -6272,55 +6712,144 @@ then [Define if you have struct stat.st_mtimensec]) fi -dnl check for ncurses/ncursesw and panel/panelw +AC_CACHE_CHECK([whether year with century should be normalized for strftime], [ac_cv_normalize_century], [ +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include + +int main(void) +{ + char year[5]; + struct tm date = { + .tm_year = -1801, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { + return 1; + } + return 0; +} +]])], +[ac_cv_normalize_century=yes], +[ac_cv_normalize_century=no], +[ac_cv_normalize_century=yes])]) +if test "$ac_cv_normalize_century" = yes +then + AC_DEFINE([Py_NORMALIZE_CENTURY], [1], + [Define if year with century should be normalized for strftime.]) +fi + +AC_CACHE_CHECK([whether C99-specific strftime specifiers are supported], [ac_cv_strftime_c99_support], [ +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include + +int main(void) +{ + char full_date[11]; + struct tm date = { + .tm_year = 0, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(full_date, sizeof(full_date), "%F", &date) && !strcmp(full_date, "1900-01-01")) { + return 0; + } + return 1; +} +]])], +[ac_cv_strftime_c99_support=yes], +[ac_cv_strftime_c99_support=no], +[ac_cv_strftime_c99_support=no])]) +if test "$ac_cv_strftime_c99_support" = yes +then + AC_DEFINE([Py_STRFTIME_C99_SUPPORT], [1], + [Define if C99-specific strftime specifiers are supported.]) +fi + +dnl check for ncursesw/ncurses and panelw/panel dnl NOTE: old curses is not detected. -dnl have_curses=[no, ncursesw, ncurses] -dnl have_panel=[no, panelw, panel] +dnl have_curses=[no, yes] +dnl have_panel=[no, yes] have_curses=no have_panel=no -AH_TEMPLATE([HAVE_NCURSESW], [Define to 1 if you have the `ncursesw' library.]) -AC_CHECK_HEADERS([curses.h ncurses.h]) +dnl PY_CHECK_CURSES(LIBCURSES, LIBPANEL) +dnl Sets 'have_curses' and 'have_panel'. +dnl For the PKG_CHECK_MODULES() calls, we can safely reuse the first variable +dnl here, since we're only calling the macro a second time if the first call +dnl fails. +AC_DEFUN([PY_CHECK_CURSES], [dnl +AS_VAR_PUSHDEF([curses_var], [m4_toupper([$1])]) +AS_VAR_PUSHDEF([panel_var], [m4_toupper([$2])]) +PKG_CHECK_MODULES([CURSES], [$1], + [AC_DEFINE([HAVE_]curses_var, [1], [Define if you have the '$1' library]) + AS_VAR_SET([have_curses], [yes]) + PKG_CHECK_MODULES([PANEL], [$2], + [AC_DEFINE([HAVE_]panel_var, [1], [Define if you have the '$2' library]) + AS_VAR_SET([have_panel], [yes])], + [AS_VAR_SET([have_panel], [no])])], + [AS_VAR_SET([have_curses], [no])]) +AS_VAR_POPDEF([curses_var]) +AS_VAR_POPDEF([panel_var])]) + +# Check for ncursesw/panelw first. If that fails, try ncurses/panel. +PY_CHECK_CURSES([ncursesw], [panelw]) +AS_VAR_IF([have_curses], [no], + [PY_CHECK_CURSES([ncurses], [panel])]) -AS_VAR_IF([ac_cv_header_ncurses_h], [yes], [ - if test "$ac_sys_system" != "Darwin"; then - dnl On macOS, there is no separate /usr/lib/libncursesw nor libpanelw. - PKG_CHECK_MODULES([CURSES], [ncursesw], [ - AC_DEFINE([HAVE_NCURSESW], [1]) - have_curses=ncursesw - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([ncursesw], [initscr], [ - AC_DEFINE([HAVE_NCURSESW], [1]) - have_curses=ncursesw - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncursesw"} - ]) - ]) - ]) - fi - - AS_VAR_IF([have_curses], [no], [ - PKG_CHECK_MODULES([CURSES], [ncurses], [ - have_curses=ncurses - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([ncurses], [initscr], [ - have_curses=ncurses - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncurses"} - ]) - ]) - ]) - ]) +WITH_SAVE_ENV([ + # Make sure we've got the header defines. + AS_VAR_APPEND([CPPFLAGS], [" $CURSES_CFLAGS $PANEL_CFLAGS"]) + AC_CHECK_HEADERS(m4_normalize([ + ncursesw/curses.h ncursesw/ncurses.h ncursesw/panel.h + ncurses/curses.h ncurses/ncurses.h ncurses/panel.h + curses.h ncurses.h panel.h + ])) -])dnl ac_cv_header_ncurses_h = yes + # Check that we're able to link with crucial curses/panel functions. This + # also serves as a fallback in case pkg-config failed. + AS_VAR_APPEND([LIBS], [" $CURSES_LIBS $PANEL_LIBS"]) + AC_SEARCH_LIBS([initscr], [ncursesw ncurses], + [AS_VAR_IF([have_curses], [no], + [AS_VAR_SET([have_curses], [yes]) + CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"}])], + [AS_VAR_SET([have_curses], [no])]) + AC_SEARCH_LIBS([update_panels], [panelw panel], + [AS_VAR_IF([have_panel], [no], + [AS_VAR_SET([have_panel], [yes]) + PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"}])], + [AS_VAR_SET([have_panel], [no])]) + +dnl Issue #25720: ncurses has introduced the NCURSES_OPAQUE symbol making opaque +dnl structs since version 5.7. If the macro is defined as zero before including +dnl [n]curses.h, ncurses will expose fields of the structs regardless of the +dnl configuration. +AC_DEFUN([_CURSES_INCLUDES],dnl +[ +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif +]) +AS_IF([test "have_curses" != "no"], [ dnl remove _XOPEN_SOURCE macro from curses cflags. pyconfig.h sets dnl the macro to 700. CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -if test "$have_curses" != no -a "$ac_sys_system" = "Darwin"; then +AS_VAR_IF([ac_sys_system], [Darwin], [ dnl On macOS, there is no separate /usr/lib/libncursesw nor libpanelw. dnl System-supplied ncurses combines libncurses/libpanel and supports wide dnl characters, so we can use it like ncursesw. @@ -6330,82 +6859,17 @@ if test "$have_curses" != no -a "$ac_sys_system" = "Darwin"; then dnl _XOPEN_SOURCE_EXTENDED here for ncurses wide char support. AS_VAR_APPEND([CURSES_CFLAGS], [" -D_XOPEN_SOURCE_EXTENDED=1"]) - AC_DEFINE([HAVE_NCURSESW], [1]) -fi - -dnl TODO: detect "curses" and special cases tinfo, terminfo, or termcap - -AC_MSG_CHECKING([curses module flags]) -AS_VAR_IF([have_curses], [no], [ - AC_MSG_RESULT([no]) -], [ - AC_MSG_RESULT([$have_curses (CFLAGS: $CURSES_CFLAGS, LIBS: $CURSES_LIBS)]) ]) -dnl check for ncurses' panel/panelw library -AC_CHECK_HEADERS([panel.h]) - -AS_VAR_IF([ac_cv_header_panel_h], [yes], [ - - if test "$ac_sys_system" != "Darwin"; then - dnl On macOS, there is no separate /usr/lib/libncursesw nor libpanelw. - AS_VAR_IF([have_curses], [ncursesw], [ - PKG_CHECK_MODULES([PANEL], [panelw], [ - have_panel=panelw - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([panelw], [update_panels], [ - have_panel=panelw - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanelw"} - ]) - ]) - ]) - ]) - fi - - AS_VAR_IF([have_curses], [ncurses], [ - PKG_CHECK_MODULES([PANEL], [panel], [ - have_panel=panel - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([panel], [update_panels], [ - have_panel=panel - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanel"} - ]) - ]) - ]) - ]) - -])dnl ac_cv_header_panel_h = yes - dnl pyconfig.h defines _XOPEN_SOURCE=700 PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -AC_MSG_CHECKING([panel flags]) -AS_VAR_IF([have_panel], [no], [ - AC_MSG_RESULT([no]) -], [ - AC_MSG_RESULT([$have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)]) -]) - -# first curses header check -ac_save_cppflags="$CPPFLAGS" -if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I/usr/include/ncursesw" -fi - # On Solaris, term.h requires curses.h -AC_CHECK_HEADERS([term.h], [], [], [ -#ifdef HAVE_CURSES_H -#include -#endif -]) +AC_CHECK_HEADERS([term.h], [], [], _CURSES_INCLUDES) # On HP/UX 11.0, mvwdelch is a block with a return statement AC_CACHE_CHECK([whether mvwdelch is an expression], [ac_cv_mvwdelch_is_expression], -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], [[ +AC_COMPILE_IFELSE([AC_LANG_PROGRAM(_CURSES_INCLUDES, [[ int rtn; rtn = mvwdelch(0,0,0); ]])], @@ -6418,15 +6882,8 @@ then [Define if mvwdelch in curses.h is an expression.]) fi -# Issue #25720: ncurses has introduced the NCURSES_OPAQUE symbol making opaque -# structs since version 5.7. If the macro is defined as zero before including -# [n]curses.h, ncurses will expose fields of the structs regardless of the -# configuration. AC_CACHE_CHECK([whether WINDOW has _flags], [ac_cv_window_has_flags], -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #define NCURSES_OPAQUE 0 - #include -]], [[ +AC_COMPILE_IFELSE([AC_LANG_PROGRAM(_CURSES_INCLUDES, [[ WINDOW *w; w->_flags = 0; ]])], @@ -6448,8 +6905,7 @@ AC_DEFUN([PY_CHECK_CURSES_FUNC], [for curses function $1], [py_var], [AC_COMPILE_IFELSE( - [AC_LANG_PROGRAM( - [@%:@include ], [ + [AC_LANG_PROGRAM(_CURSES_INCLUDES, [ #ifndef $1 void *x=$1 #endif @@ -6477,32 +6933,41 @@ PY_CHECK_CURSES_FUNC([has_key]) PY_CHECK_CURSES_FUNC([typeahead]) PY_CHECK_CURSES_FUNC([use_env]) CPPFLAGS=$ac_save_cppflags +])dnl have_curses != no +])dnl save env AC_MSG_NOTICE([checking for device files]) dnl NOTE: Inform user how to proceed with files when cross compiling. -if test "x$cross_compiling" = xyes; then - if test "${ac_cv_file__dev_ptmx+set}" != set; then - AC_MSG_CHECKING([for /dev/ptmx]) - AC_MSG_RESULT([not set]) - AC_MSG_ERROR([set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling]) - fi - if test "${ac_cv_file__dev_ptc+set}" != set; then - AC_MSG_CHECKING([for /dev/ptc]) - AC_MSG_RESULT([not set]) - AC_MSG_ERROR([set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling]) +dnl Some cross-compile builds are predictable; they won't ever +dnl have /dev/ptmx or /dev/ptc, so we can set them explicitly. +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no +else + if test "x$cross_compiling" = xyes; then + if test "${ac_cv_file__dev_ptmx+set}" != set; then + AC_MSG_CHECKING([for /dev/ptmx]) + AC_MSG_RESULT([not set]) + AC_MSG_ERROR([set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling]) + fi + if test "${ac_cv_file__dev_ptc+set}" != set; then + AC_MSG_CHECKING([for /dev/ptc]) + AC_MSG_RESULT([not set]) + AC_MSG_ERROR([set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling]) + fi fi -fi -AC_CHECK_FILE([/dev/ptmx], [], []) -if test "x$ac_cv_file__dev_ptmx" = xyes; then - AC_DEFINE([HAVE_DEV_PTMX], [1], - [Define to 1 if you have the /dev/ptmx device file.]) -fi -AC_CHECK_FILE([/dev/ptc], [], []) -if test "x$ac_cv_file__dev_ptc" = xyes; then - AC_DEFINE([HAVE_DEV_PTC], [1], - [Define to 1 if you have the /dev/ptc device file.]) + AC_CHECK_FILE([/dev/ptmx], [], []) + if test "x$ac_cv_file__dev_ptmx" = xyes; then + AC_DEFINE([HAVE_DEV_PTMX], [1], + [Define to 1 if you have the /dev/ptmx device file.]) + fi + AC_CHECK_FILE([/dev/ptc], [], []) + if test "x$ac_cv_file__dev_ptc" = xyes; then + AC_DEFINE([HAVE_DEV_PTC], [1], + [Define to 1 if you have the /dev/ptc device file.]) + fi fi if test $ac_sys_system = Darwin @@ -6510,12 +6975,9 @@ then LIBS="$LIBS -framework CoreFoundation" fi -AC_CHECK_TYPE( - [socklen_t], [], - [AC_DEFINE( - [socklen_t], [int], - [Define to `int' if does not define.] - )], [ +AC_CHECK_TYPES([socklen_t], [], + [AC_DEFINE([socklen_t], [int], + [Define to 'int' if does not define.])], [ #ifdef HAVE_SYS_TYPES_H #include #endif @@ -6612,7 +7074,6 @@ done AC_SUBST([SRCDIRS]) SRCDIRS="\ Modules \ - Modules/_blake2 \ Modules/_ctypes \ Modules/_decimal \ Modules/_decimal/libmpdec \ @@ -6623,6 +7084,7 @@ SRCDIRS="\ Modules/_sre \ Modules/_testcapi \ Modules/_testinternalcapi \ + Modules/_testlimitedcapi \ Modules/_xxtestfuzz \ Modules/cjkcodecs \ Modules/expat \ @@ -6634,8 +7096,7 @@ SRCDIRS="\ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" AC_MSG_CHECKING([for build directories]) for dir in $SRCDIRS; do if test ! -d $dir; then @@ -6736,6 +7197,7 @@ AC_ARG_WITH([ensurepip], AS_CASE([$ac_sys_system], [Emscripten], [with_ensurepip=no], [WASI], [with_ensurepip=no], + [iOS], [with_ensurepip=no], [with_ensurepip=upgrade] ) ]) @@ -7008,8 +7470,6 @@ AC_DEFINE([PY_SSL_DEFAULT_CIPHERS], [1]) # builtin hash modules default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" -AC_DEFINE([PY_BUILTIN_HASHLIB_HASHES], [], [enabled builtin hash modules] -) AC_MSG_CHECKING([for --with-builtin-hashlib-hashes]) AC_ARG_WITH( [builtin-hashlib-hashes], @@ -7026,7 +7486,8 @@ AC_ARG_WITH( AC_MSG_RESULT([$with_builtin_hashlib_hashes]) AC_DEFINE_UNQUOTED([PY_BUILTIN_HASHLIB_HASHES], - ["$with_builtin_hashlib_hashes"]) + ["$with_builtin_hashlib_hashes"], + [enabled builtin hash modules]) as_save_IFS=$IFS IFS=, @@ -7041,15 +7502,6 @@ for builtin_hash in $with_builtin_hashlib_hashes; do done IFS=$as_save_IFS -dnl libb2 for blake2. _blake2 module falls back to vendored copy. -AS_VAR_IF([with_builtin_blake2], [yes], [ - PKG_CHECK_MODULES([LIBB2], [libb2], [ - have_libb2=yes - AC_DEFINE([HAVE_LIBB2], [1], - [Define to 1 if you want to build _blake2 module with libb2]) - ], [have_libb2=no]) -]) - # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. AC_MSG_CHECKING([for --disable-test-modules]) @@ -7084,7 +7536,7 @@ CPPFLAGS="${BASECPPFLAGS} -I. -I${srcdir}/Include ${CPPFLAGS}" AC_CACHE_CHECK([whether libatomic is needed by ], [ac_cv_libatomic_needed], -[AC_RUN_IFELSE([AC_LANG_SOURCE([[ +[AC_LINK_IFELSE([AC_LANG_SOURCE([[ // pyatomic.h needs uint64_t and Py_ssize_t types #include // int64_t, intptr_t #ifdef HAVE_SYS_TYPES_H @@ -7121,9 +7573,8 @@ int main() return 0; // all good } ]])], - [ac_cv_libatomic_needed=no], dnl build succeeded - [ac_cv_libatomic_needed=yes], dnl build failed - [ac_cv_libatomic_needed=no]) dnl cross compilation + [ac_cv_libatomic_needed=no], dnl build and link succeeded + [ac_cv_libatomic_needed=yes]) dnl build and link failed ]) AS_VAR_IF([ac_cv_libatomic_needed], [yes], @@ -7145,15 +7596,36 @@ AS_CASE([$ac_sys_system], [VxWorks*], [PY_STDLIB_MOD_SET_NA([_scproxy], [termios], [grp])], dnl The _scproxy module is available on macOS [Darwin], [], + [iOS], [ + dnl subprocess and multiprocessing are not supported (no fork syscall). + dnl curses and tkinter user interface are not available. + dnl gdbm and nis aren't available + dnl Stub implementations are provided for pwd, grp etc APIs + PY_STDLIB_MOD_SET_NA( + [_curses], + [_curses_panel], + [_gdbm], + [_multiprocessing], + [_posixshmem], + [_posixsubprocess], + [_scproxy], + [_tkinter], + [grp], + [nis], + [readline], + [pwd], + [spwd], + [syslog], + ) + ], [CYGWIN*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [QNX*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [FreeBSD*], [PY_STDLIB_MOD_SET_NA([_scproxy])], - [Emscripten|WASI], [ + [Emscripten], [ dnl subprocess and multiprocessing are not supported (no fork syscall). dnl curses and tkinter user interface are not available. dnl dbm and gdbm aren't available, too. - dnl Emscripten and WASI provide only stubs for pwd, grp APIs. - dnl resource functions (get/setrusage) are stubs, too. + dnl pwd, grp APIs, and resource functions (get/setrusage) are stubs. PY_STDLIB_MOD_SET_NA( [_curses], [_curses_panel], @@ -7164,33 +7636,58 @@ AS_CASE([$ac_sys_system], [_posixsubprocess], [_scproxy], [_tkinter], - [_xxsubinterpreters], - [_xxinterpchannels], - [_xxinterpqueues], + [_interpreters], + [_interpchannels], + [_interpqueues], [grp], [pwd], [resource], [syslog], ) - AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [ - dnl These modules are not particularly useful in browsers. - PY_STDLIB_MOD_SET_NA( - [fcntl], - [readline], - [termios], - ) - ], - [Emscripten/node*], [], - [WASI/*], [ - dnl WASI SDK 15.0 does not support file locking, mmap, and more. - PY_STDLIB_MOD_SET_NA( - [_ctypes_test], - [fcntl], - [mmap], - [termios], - ) - ] + dnl fcntl, readline, and termios are not particularly useful in browsers. + PY_STDLIB_MOD_SET_NA( + [fcntl], + [readline], + [termios], + ) + ], + [WASI], [ + dnl subprocess and multiprocessing are not supported (no fork syscall). + dnl curses and tkinter user interface are not available. + dnl dbm and gdbm aren't available, too. + dnl pwd, grp APIs, and resource functions (get/setrusage) are stubs. + PY_STDLIB_MOD_SET_NA( + [_curses], + [_curses_panel], + [_dbm], + [_gdbm], + [_multiprocessing], + [_posixshmem], + [_posixsubprocess], + [_scproxy], + [_tkinter], + [_interpreters], + [_interpchannels], + [_interpqueues], + [grp], + [pwd], + [resource], + [syslog], + ) + dnl WASI SDK 15.0 does not support file locking, mmap, and more. + dnl Test modules that must be compiled as shared libraries are not supported + dnl (see Modules/Setup.stdlib.in). + PY_STDLIB_MOD_SET_NA( + [_ctypes_test], + [_testexternalinspection], + [_testimportmultiple], + [_testmultiphase], + [_testsinglephase], + [fcntl], + [mmap], + [termios], + [xxlimited], + [xxlimited_35], ) ], [PY_STDLIB_MOD_SET_NA([_scproxy])] @@ -7273,7 +7770,6 @@ PY_STDLIB_MOD_SIMPLE([_csv]) PY_STDLIB_MOD_SIMPLE([_heapq]) PY_STDLIB_MOD_SIMPLE([_json]) PY_STDLIB_MOD_SIMPLE([_lsprof]) -PY_STDLIB_MOD_SIMPLE([_opcode]) PY_STDLIB_MOD_SIMPLE([_pickle]) PY_STDLIB_MOD_SIMPLE([_posixsubprocess]) PY_STDLIB_MOD_SIMPLE([_queue]) @@ -7281,9 +7777,9 @@ PY_STDLIB_MOD_SIMPLE([_random]) PY_STDLIB_MOD_SIMPLE([select]) PY_STDLIB_MOD_SIMPLE([_struct]) PY_STDLIB_MOD_SIMPLE([_typing]) -PY_STDLIB_MOD_SIMPLE([_xxsubinterpreters]) -PY_STDLIB_MOD_SIMPLE([_xxinterpchannels]) -PY_STDLIB_MOD_SIMPLE([_xxinterpqueues]) +PY_STDLIB_MOD_SIMPLE([_interpreters]) +PY_STDLIB_MOD_SIMPLE([_interpchannels]) +PY_STDLIB_MOD_SIMPLE([_interpqueues]) PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules @@ -7311,7 +7807,7 @@ PY_STDLIB_MOD([mmap], PY_STDLIB_MOD([_socket], [], m4_flatten([test "$ac_cv_header_sys_socket_h" = "yes" -a "$ac_cv_header_sys_types_h" = "yes" - -a "$ac_cv_header_netinet_in_h" = "yes"])) + -a "$ac_cv_header_netinet_in_h" = "yes"]), [], [$SOCKET_LIBS]) dnl platform specific extensions PY_STDLIB_MOD([grp], [], @@ -7341,32 +7837,90 @@ PY_STDLIB_MOD_SIMPLE([unicodedata]) dnl By default we always compile these even when OpenSSL is available dnl (issue #14693). The modules are small. -PY_STDLIB_MOD([_md5], - [test "$with_builtin_md5" = yes], [], - [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) -PY_STDLIB_MOD([_sha1], - [test "$with_builtin_sha1" = yes], [], - [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) -PY_STDLIB_MOD([_sha2], - [test "$with_builtin_sha2" = yes], [], - [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) +PY_STDLIB_MOD([_md5], [test "$with_builtin_md5" = yes]) +PY_STDLIB_MOD([_sha1], [test "$with_builtin_sha1" = yes]) +PY_STDLIB_MOD([_sha2], [test "$with_builtin_sha2" = yes]) PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes]) -PY_STDLIB_MOD([_blake2], - [test "$with_builtin_blake2" = yes], [], - [$LIBB2_CFLAGS], [$LIBB2_LIBS]) +PY_STDLIB_MOD([_blake2], [test "$with_builtin_blake2" = yes]) + +LIBHACL_CFLAGS='-I$(srcdir)/Modules/_hacl -I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED)' +case "$ac_sys_system" in + Linux*) + if test "$ac_cv_func_explicit_bzero" = "no"; then + LIBHACL_CFLAGS="$LIBHACL_CFLAGS -DLINUX_NO_EXPLICIT_BZERO" + fi + ;; +esac +AC_SUBST([LIBHACL_CFLAGS]) + +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then + dnl This can be extended here to detect e.g. Power8, which HACL* should also support. + AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[ + [LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2"] + + AC_DEFINE([HACL_CAN_COMPILE_SIMD128], [1], [HACL* library can compile SIMD128 implementations]) + + # macOS universal2 builds *support* the -msse etc flags because they're + # available on x86_64. However, performance of the HACL SIMD128 implementation + # isn't great, so it's disabled on ARM64. + AC_MSG_CHECKING([for HACL* SIMD128 implementation]) + if test "$UNIVERSAL_ARCHS" == "universal2"; then + [LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o"] + AC_MSG_RESULT([universal2]) + else + [LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o"] + AC_MSG_RESULT([standard]) + fi + + ], [], [-Werror]) +fi +AC_SUBST([LIBHACL_SIMD128_FLAGS]) +AC_SUBST([LIBHACL_SIMD128_OBJS]) + +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +# +# Although AVX support is not guaranteed on Android +# (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a +# runtime CPUID check. +if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then + AX_CHECK_COMPILE_FLAG([-mavx2],[ + [LIBHACL_SIMD256_FLAGS="-mavx2"] + AC_DEFINE([HACL_CAN_COMPILE_SIMD256], [1], [HACL* library can compile SIMD256 implementations]) + + # macOS universal2 builds *support* the -mavx2 compiler flag because it's + # available on x86_64; but the HACL SIMD256 build then fails because the + # implementation requires symbols that aren't available on ARM64. Use a + # wrapped implementation if we're building for universal2. + AC_MSG_CHECKING([for HACL* SIMD256 implementation]) + if test "$UNIVERSAL_ARCHS" == "universal2"; then + [LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o"] + AC_MSG_RESULT([universal2]) + else + [LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o"] + AC_MSG_RESULT([standard]) + fi + ], [], [-Werror]) +fi +AC_SUBST([LIBHACL_SIMD256_FLAGS]) +AC_SUBST([LIBHACL_SIMD256_OBJS]) PY_STDLIB_MOD([_ctypes], [], [test "$have_libffi" = yes], [$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS], [$LIBFFI_LIBS]) PY_STDLIB_MOD([_curses], - [], [test "$have_curses" != "no"], + [], [test "$have_curses" = "yes"], [$CURSES_CFLAGS], [$CURSES_LIBS] ) PY_STDLIB_MOD([_curses_panel], - [], [test "$have_panel" != "no"], + [], [test "$have_curses" = "yes" && test "$have_panel" = "yes"], [$PANEL_CFLAGS $CURSES_CFLAGS], [$PANEL_LIBS $CURSES_LIBS] ) -PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LDFLAGS]) +PY_STDLIB_MOD([_decimal], + [], [test "$have_mpdec" = "yes"], + [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) PY_STDLIB_MOD([_dbm], [test -n "$with_dbmliborder"], [test "$have_dbm" != "no"], [$DBM_CFLAGS], [$DBM_LIBS]) @@ -7410,20 +7964,23 @@ PY_STDLIB_MOD([_testcapi], [], [], [$LIBATOMIC]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testlimitedcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testsinglephase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testexternalinspection], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes], - [], [$LIBM]) + [$LIBFFI_CFLAGS], [$LIBFFI_LIBS $LIBM]) dnl Limited API template modules. dnl Emscripten does not support shared libraries yet. -PY_STDLIB_MOD([xxlimited], [], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([xxlimited_35], [], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) diff --git a/iOS/README.rst b/iOS/README.rst new file mode 100644 index 00000000000000..4d7c344d5e9e17 --- /dev/null +++ b/iOS/README.rst @@ -0,0 +1,385 @@ +==================== +Python on iOS README +==================== + +:Authors: + Russell Keith-Magee (2023-11) + +This document provides a quick overview of some iOS specific features in the +Python distribution. + +These instructions are only needed if you're planning to compile Python for iOS +yourself. Most users should *not* need to do this. If you're looking to +experiment with writing an iOS app in Python, tools such as `BeeWare's Briefcase +`__ and `Kivy's Buildozer +`__ will provide a much more approachable +user experience. + +Compilers for building on iOS +============================= + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting an open the Platforms tab of the Xcode Settings panel. + +iOS specific arguments to configure +=================================== + +* ``--enable-framework[=DIR]`` + + This argument specifies the location where the Python.framework will be + installed. If ``DIR`` is not specified, the framework will be installed into + a subdirectory of the ``iOS/Frameworks`` folder. + + This argument *must* be provided when configuring iOS builds. iOS does not + support non-framework builds. + +* ``--with-framework-name=NAME`` + + Specify the name for the Python framework; defaults to ``Python``. + + .. admonition:: Use this option with care! + + Unless you know what you're doing, changing the name of the Python + framework on iOS is not advised. If you use this option, you won't be able + to run the ``make testios`` target without making significant manual + alterations, and you won't be able to use any binary packages unless you + compile them yourself using your own framework name. + +Building Python on iOS +====================== + +ABIs and Architectures +---------------------- + +iOS apps can be deployed on physical devices, and on the iOS simulator. Although +the API used on these devices is identical, the ABI is different - you need to +link against different libraries for an iOS device build (``iphoneos``) or an +iOS simulator build (``iphonesimulator``). + +Apple uses the ``XCframework`` format to allow specifying a single dependency +that supports multiple ABIs. An ``XCframework`` is a wrapper around multiple +ABI-specific frameworks that share a common API. + +iOS can also support different CPU architectures within each ABI. At present, +there is only a single supported architecture on physical devices - ARM64. +However, the *simulator* supports 2 architectures - ARM64 (for running on Apple +Silicon machines), and x86_64 (for running on older Intel-based machines). + +To support multiple CPU architectures on a single platform, Apple uses a "fat +binary" format - a single physical file that contains support for multiple +architectures. It is possible to compile and use a "thin" single architecture +version of a binary for testing purposes; however, the "thin" binary will not be +portable to machines using other architectures. + +Building a single-architecture framework +---------------------------------------- + +The Python build system will create a ``Python.framework`` that supports a +*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a +framework to contain non-library content, so the iOS build will produce a +``bin`` and ``lib`` folder in the same output folder as ``Python.framework``. +The ``lib`` folder will be needed at runtime to support the Python library. + +If you want to use Python in a real iOS project, you need to produce multiple +``Python.framework`` builds, one for each ABI and architecture. iOS builds of +Python *must* be constructed as framework builds. To support this, you must +provide the ``--enable-framework`` flag when configuring the build. The build +also requires the use of cross-compilation. The minimal commands for building +Python for the ARM64 iOS simulator will look something like:: + + $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" + $ ./configure \ + --enable-framework \ + --host=arm64-apple-ios-simulator \ + --build=arm64-apple-darwin \ + --with-build-python=/path/to/python.exe + $ make + $ make install + +In this invocation: + +* ``iOS/Resources/bin`` has been added to the path, providing some shims for the + compilers and linkers needed by the build. Xcode requires the use of ``xcrun`` + to invoke compiler tooling. However, if ``xcrun`` is pre-evaluated and the + result passed to ``configure``, these results can embed user- and + version-specific paths into the sysconfig data, which limits the portability + of the compiled Python. Alternatively, if ``xcrun`` is used *as* the compiler, + it requires that compiler variables like ``CC`` include spaces, which can + cause significant problems with many C configuration systems which assume that + ``CC`` will be a single executable. + + To work around this problem, the ``iOS/Resources/bin`` folder contains some + wrapper scripts that present as simple compilers and linkers, but wrap + underlying calls to ``xcrun``. This allows configure to use a ``CC`` + definition without spaces, and without user- or version-specific paths, while + retaining the ability to adapt to the local Xcode install. These scripts are + included in the ``bin`` directory of an iOS install. + + These scripts will, by default, use the currently active Xcode installation. + If you want to use a different Xcode installation, you can use + ``xcode-select`` to set a new default Xcode globally, or you can use the + ``DEVELOPER_DIR`` environment variable to specify an Xcode install. The + scripts will use the default ``iphoneos``/``iphonesimulator`` SDK version for + the select Xcode install; if you want to use a different SDK, you can set the + ``IOS_SDK_VERSION`` environment variable. (e.g, setting + ``IOS_SDK_VERSION=17.1`` would cause the scripts to use the ``iphoneos17.1`` + and ``iphonesimulator17.1`` SDKs, regardless of the Xcode default.) + + The path has also been cleared of any user customizations. A common source of + bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS + build. Resetting the path to a known "bare bones" value is the easiest way to + avoid these problems. + +* ``--host`` is the architecture and ABI that you want to build, in GNU compiler + triple format. This will be one of: + + - ``arm64-apple-ios`` for ARM64 iOS devices. + - ``arm64-apple-ios-simulator`` for the iOS simulator running on Apple + Silicon devices. + - ``x86_64-apple-ios-simulator`` for the iOS simulator running on Intel + devices. + +* ``--build`` is the GNU compiler triple for the machine that will be running + the compiler. This is one of: + + - ``arm64-apple-darwin`` for Apple Silicon devices. + - ``x86_64-apple-darwin`` for Intel devices. + +* ``/path/to/python.exe`` is the path to a Python binary on the machine that + will be running the compiler. This is needed because the Python compilation + process involves running some Python code. On a normal desktop build of + Python, you can compile a python interpreter and then use that interpreter to + run Python code. However, the binaries produced for iOS won't run on macOS, so + you need to provide an external Python interpreter. This interpreter must be + the same version as the Python that is being compiled. To be completely safe, + this should be the *exact* same commit hash. However, the longer a Python + release has been stable, the more likely it is that this constraint can be + relaxed - the same micro version will often be sufficient. + +* The ``install`` target for iOS builds is slightly different to other + platforms. On most platforms, ``make install`` will install the build into + the final runtime location. This won't be the case for iOS, as the final + runtime location will be on a physical device. + + However, you still need to run the ``install`` target for iOS builds, as it + performs some final framework assembly steps. The location specified with + ``--enable-framework`` will be the location where ``make install`` will + assemble the complete iOS framework. This completed framework can then + be copied and relocated as required. + +For a full CPython build, you also need to specify the paths to iOS builds of +the binary libraries that CPython depends on (XZ, BZip2, LibFFI and OpenSSL). +This can be done by defining the ``LIBLZMA_CFLAGS``, ``LIBLZMA_LIBS``, +``BZIP2_CFLAGS``, ``BZIP2_LIBS``, ``LIBFFI_CFLAGS``, and ``LIBFFI_LIBS`` +environment variables, and the ``--with-openssl`` configure option. Versions of +these libraries pre-compiled for iOS can be found in `this repository +`__. LibFFI is +especially important, as many parts of the standard library (including the +``platform``, ``sysconfig`` and ``webbrowser`` modules) require the use of the +``ctypes`` module at runtime. + +By default, Python will be compiled with an iOS deployment target (i.e., the +minimum supported iOS version) of 13.0. To specify a different deployment +target, provide the version number as part of the ``--host`` argument - for +example, ``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 +simulator build with a deployment target of 15.4. + +Merge thin frameworks into fat frameworks +----------------------------------------- + +Once you've built a ``Python.framework`` for each ABI and and architecture, you +must produce a "fat" framework for each ABI that contains all the architectures +for that ABI. + +The ``iphoneos`` build only needs to support a single architecture, so it can be +used without modification. + +If you only want to support a single simulator architecture, (e.g., only support +ARM64 simulators), you can use a single architecture ``Python.framework`` build. +However, if you want to create ``Python.xcframework`` that supports *all* +architectures, you'll need to merge the ``iphonesimulator`` builds for ARM64 and +x86_64 into a single "fat" framework. + +The "fat" framework can be constructed by performing a directory merge of the +content of the two "thin" ``Python.framework`` directories, plus the ``bin`` and +``lib`` folders for each thin framework. When performing this merge: + +* The pure Python standard library content is identical for each architecture, + except for a handful of platform-specific files (such as the ``sysconfig`` + module). Ensure that the "fat" framework has the union of all standard library + files. + +* Any binary files in the standard library, plus the main + ``libPython3.X.dylib``, can be merged using the ``lipo`` tool, provide by + Xcode:: + + $ lipo -create -output module.dylib path/to/x86_64/module.dylib path/to/arm64/module.dylib + +* The header files will be identical on both architectures, except for + ``pyconfig.h``. Copy all the headers from one platform (say, arm64), rename + ``pyconfig.h`` to ``pyconfig-arm64.h``, and copy the ``pyconfig.h`` for the + other architecture into the merged header folder as ``pyconfig-x86_64.h``. + Then copy the ``iOS/Resources/pyconfig.h`` file from the CPython sources into + the merged headers folder. This will allow the two Python architectures to + share a common ``pyconfig.h`` header file. + +At this point, you should have 2 Python.framework folders - one for ``iphoneos``, +and one for ``iphonesimulator`` that is a merge of x86+64 and ARM64 content. + +Merge frameworks into an XCframework +------------------------------------ + +Now that we have 2 (potentially fat) ABI-specific frameworks, we can merge those +frameworks into a single ``XCframework``. + +The initial skeleton of an ``XCframework`` is built using:: + + xcodebuild -create-xcframework -output Python.xcframework -framework path/to/iphoneos/Python.framework -framework path/to/iphonesimulator/Python.framework + +Then, copy the ``bin`` and ``lib`` folders into the architecture-specific slices of +the XCframework:: + + cp path/to/iphoneos/bin Python.xcframework/ios-arm64 + cp path/to/iphoneos/lib Python.xcframework/ios-arm64 + + cp path/to/iphonesimulator/bin Python.xcframework/ios-arm64_x86_64-simulator + cp path/to/iphonesimulator/lib Python.xcframework/ios-arm64_x86_64-simulator + +Note that the name of the architecture-specific slice for the simulator will +depend on the CPU architecture(s) that you build. + +You now have a Python.xcframework that can be used in a project. + +Testing Python on iOS +===================== + +The ``iOS/testbed`` folder that contains an Xcode project that is able to run +the iOS test suite. This project converts the Python test suite into a single +test case in Xcode's XCTest framework. The single XCTest passes if the test +suite passes. + +To run the test suite, configure a Python build for an iOS simulator (i.e., +``--host=arm64-apple-ios-simulator`` or ``--host=x86_64-apple-ios-simulator`` +), specifying a framework build (i.e. ``--enable-framework``). Ensure that your +``PATH`` has been configured to include the ``iOS/Resources/bin`` folder and +exclude any non-iOS tools, then run:: + + $ make all + $ make install + $ make testios + +This will: + +* Build an iOS framework for your chosen architecture; +* Finalize the single-platform framework; +* Make a clean copy of the testbed project; +* Install the Python iOS framework into the copy of the testbed project; and +* Run the test suite on an "iPhone SE (3rd generation)" simulator. + +While the test suite is running, Xcode does not display any console output. +After showing some Xcode build commands, the console output will print ``Testing +started``, and then appear to stop. It will remain in this state until the test +suite completes. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 +minutes to run; a couple of extra minutes is required to boot and prepare the +iOS simulator. + +On success, the test suite will exit and report successful completion of the +test suite. No output of the Python test suite will be displayed. + +On failure, the output of the Python test suite *will* be displayed. This will +show the details of the tests that failed. + +Debugging test failures +----------------------- + +The easiest way to diagnose a single test failure is to open the testbed project +in Xcode and run the tests from there using the "Product > Test" menu item. + +To test in Xcode, you must ensure the testbed project has a copy of a compiled +framework. If you've configured your build with the default install location of +``iOS/Frameworks``, you can copy from that location into the test project. To +test on an ARM64 simulator, run:: + + $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* + $ cp -r iOS/Frameworks/arm64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator + +To test on an x86-64 simulator, run:: + + $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* + $ cp -r iOS/Frameworks/x86_64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator + +To test on a physical device:: + + $ rm -rf iOS/testbed/Python.xcframework/ios-arm64/* + $ cp -r iOS/Frameworks/arm64-iphoneos/* iOS/testbed/Python.xcframework/ios-arm64 + +Alternatively, you can configure your build to install directly into the +testbed project. For a simulator, use:: + + --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator + +For a physical device, use:: + + --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64 + + +Testing on an iOS device +^^^^^^^^^^^^^^^^^^^^^^^^ + +To test on an iOS device, the app needs to be signed with known developer +credentials. To obtain these credentials, you must have an iOS Developer +account, and your Xcode install will need to be logged into your account (see +the Accounts tab of the Preferences dialog). + +Once the project is open, and you're signed into your Apple Developer account, +select the root node of the project tree (labeled "iOSTestbed"), then the +"Signing & Capabilities" tab in the details page. Select a development team +(this will likely be your own name), and plug in a physical device to your +macOS machine with a USB cable. You should then be able to select your physical +device from the list of targets in the pulldown in the Xcode titlebar. + +Running specific tests +^^^^^^^^^^^^^^^^^^^^^^ + +As the test suite is being executed on an iOS simulator, it is not possible to +pass in command line arguments to configure test suite operation. To work around +this limitation, the arguments that would normally be passed as command line +arguments are configured as a static string at the start of the XCTest method +``- (void)testPython`` in ``iOSTestbedTests.m``. To pass an argument to the test +suite, add a a string to the ``argv`` definition. These arguments will be passed +to the test suite as if they had been passed to ``python -m test`` at the +command line. + +Disabling automated breakpoints +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, Xcode will inserts an automatic breakpoint whenever a signal is +raised. The Python test suite raises many of these signals as part of normal +operation; unless you are trying to diagnose an issue with signals, the +automatic breakpoints can be inconvenient. However, they can be disabled by +creating a symbolic breakpoint that is triggered at the start of the test run. + +Select "Debug > Breakpoints > Create Symbolic Breakpoint" from the Xcode menu, and +populate the new brewpoint with the following details: + +* **Name**: IgnoreSignals +* **Symbol**: UIApplicationMain +* **Action**: Add debugger commands for: + - ``process handle SIGINT -n true -p true -s false`` + - ``process handle SIGUSR1 -n true -p true -s false`` + - ``process handle SIGUSR2 -n true -p true -s false`` + - ``process handle SIGXFSZ -n true -p true -s false`` +* Check the "Automatically continue after evaluating" box. + +All other details can be left blank. When the process executes the +``UIApplicationMain`` entry point, the breakpoint will trigger, run the debugger +commands to disable the automatic breakpoints, and automatically resume. diff --git a/iOS/Resources/Info.plist.in b/iOS/Resources/Info.plist.in new file mode 100644 index 00000000000000..c3e261ecd9eff7 --- /dev/null +++ b/iOS/Resources/Info.plist.in @@ -0,0 +1,34 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + Python + CFBundleGetInfoString + Python Runtime and Library + CFBundleIdentifier + @PYTHONFRAMEWORKIDENTIFIER@ + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Python + CFBundlePackageType + FMWK + CFBundleShortVersionString + @VERSION@ + CFBundleLongVersionString + %VERSION%, (c) 2001-2024 Python Software Foundation. + CFBundleSignature + ???? + CFBundleVersion + 1 + CFBundleSupportedPlatforms + + iPhoneOS + + MinimumOSVersion + @IPHONEOS_DEPLOYMENT_TARGET@ + + diff --git a/iOS/Resources/bin/arm64-apple-ios-ar b/iOS/Resources/bin/arm64-apple-ios-ar new file mode 100755 index 00000000000000..8122332b9c1de0 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-clang b/iOS/Resources/bin/arm64-apple-ios-clang new file mode 100755 index 00000000000000..4d525751eba798 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-clang++ b/iOS/Resources/bin/arm64-apple-ios-clang++ new file mode 100755 index 00000000000000..f24bec11268f7e --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-cpp b/iOS/Resources/bin/arm64-apple-ios-cpp new file mode 100755 index 00000000000000..891bb25bb4318c --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/iOS/Resources/bin/arm64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..74ed3bc6df1c2b --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/iOS/Resources/bin/arm64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..32574cad284441 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..ef37d05b512959 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..6aaf6fbe188c32 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..74ed3bc6df1c2b --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..bcbe91f6061e16 --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..86f03ea32bc2fd --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..e6a42d9b85dec7 --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E $@ diff --git a/iOS/Resources/dylib-Info-template.plist b/iOS/Resources/dylib-Info-template.plist new file mode 100644 index 00000000000000..f652e272f71c88 --- /dev/null +++ b/iOS/Resources/dylib-Info-template.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + + CFBundleIdentifier + + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSupportedPlatforms + + iPhoneOS + + MinimumOSVersion + 12.0 + CFBundleVersion + 1 + + diff --git a/iOS/Resources/pyconfig.h b/iOS/Resources/pyconfig.h new file mode 100644 index 00000000000000..4acff2c6051637 --- /dev/null +++ b/iOS/Resources/pyconfig.h @@ -0,0 +1,7 @@ +#ifdef __arm64__ +#include "pyconfig-arm64.h" +#endif + +#ifdef __x86_64__ +#include "pyconfig-x86_64.h" +#endif diff --git a/iOS/testbed/Python.xcframework/Info.plist b/iOS/testbed/Python.xcframework/Info.plist new file mode 100644 index 00000000000000..c6418de6e74a4e --- /dev/null +++ b/iOS/testbed/Python.xcframework/Info.plist @@ -0,0 +1,44 @@ + + + + + AvailableLibraries + + + BinaryPath + Python.framework/Python + LibraryIdentifier + ios-arm64 + LibraryPath + Python.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + + + BinaryPath + Python.framework/Python + LibraryIdentifier + ios-arm64_x86_64-simulator + LibraryPath + Python.framework + SupportedArchitectures + + arm64 + x86_64 + + SupportedPlatform + ios + SupportedPlatformVariant + simulator + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/iOS/testbed/Python.xcframework/ios-arm64/README b/iOS/testbed/Python.xcframework/ios-arm64/README new file mode 100644 index 00000000000000..c1b076d12cddb7 --- /dev/null +++ b/iOS/testbed/Python.xcframework/ios-arm64/README @@ -0,0 +1,4 @@ +This directory is intentionally empty. + +It should be used as a target for `--enable-framework` when compiling an iOS on-device +build for testing purposes. diff --git a/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README b/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README new file mode 100644 index 00000000000000..ae334e5d769d9d --- /dev/null +++ b/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README @@ -0,0 +1,4 @@ +This directory is intentionally empty. + +It should be used as a target for `--enable-framework` when compiling an iOS simulator +build for testing purposes (either x86_64 or ARM64). diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj new file mode 100644 index 00000000000000..d57cfc3dbe0304 --- /dev/null +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -0,0 +1,570 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 56; + objects = { + +/* Begin PBXBuildFile section */ + 607A66172B0EFA380010BFC8 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66162B0EFA380010BFC8 /* AppDelegate.m */; }; + 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 607A66212B0EFA390010BFC8 /* Assets.xcassets */; }; + 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */; }; + 607A66282B0EFA390010BFC8 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66272B0EFA390010BFC8 /* main.m */; }; + 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */; }; + 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; + 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; + 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */ = {isa = PBXBuildFile; fileRef = 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 607A662E2B0EFA3A0010BFC8 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 607A660A2B0EFA380010BFC8 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 607A66112B0EFA380010BFC8; + remoteInfo = iOSTestbed; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 607A664E2B0EFC080010BFC8 /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */, + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 0; + }; + 607A66522B0EFFE00010BFC8 /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */, + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 607A66122B0EFA380010BFC8 /* iOSTestbed.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = iOSTestbed.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 607A66152B0EFA380010BFC8 /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = ""; }; + 607A66162B0EFA380010BFC8 /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = ""; }; + 607A66212B0EFA390010BFC8 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + 607A66242B0EFA390010BFC8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; + 607A66272B0EFA390010BFC8 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; + 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = iOSTestbedTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = iOSTestbedTests.m; sourceTree = ""; }; + 607A664A2B0EFB310010BFC8 /* Python.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Python.xcframework; sourceTree = ""; }; + 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "dylib-Info-template.plist"; sourceTree = ""; }; + 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "iOSTestbed-Info.plist"; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 607A660F2B0EFA380010BFC8 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 607A662A2B0EFA3A0010BFC8 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 607A66092B0EFA380010BFC8 = { + isa = PBXGroup; + children = ( + 607A664A2B0EFB310010BFC8 /* Python.xcframework */, + 607A66142B0EFA380010BFC8 /* iOSTestbed */, + 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */, + 607A66132B0EFA380010BFC8 /* Products */, + 607A664F2B0EFFE00010BFC8 /* Frameworks */, + ); + sourceTree = ""; + }; + 607A66132B0EFA380010BFC8 /* Products */ = { + isa = PBXGroup; + children = ( + 607A66122B0EFA380010BFC8 /* iOSTestbed.app */, + 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */, + ); + name = Products; + sourceTree = ""; + }; + 607A66142B0EFA380010BFC8 /* iOSTestbed */ = { + isa = PBXGroup; + children = ( + 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */, + 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */, + 607A66152B0EFA380010BFC8 /* AppDelegate.h */, + 607A66162B0EFA380010BFC8 /* AppDelegate.m */, + 607A66212B0EFA390010BFC8 /* Assets.xcassets */, + 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */, + 607A66272B0EFA390010BFC8 /* main.m */, + ); + path = iOSTestbed; + sourceTree = ""; + }; + 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + isa = PBXGroup; + children = ( + 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */, + ); + path = iOSTestbedTests; + sourceTree = ""; + }; + 607A664F2B0EFFE00010BFC8 /* Frameworks */ = { + isa = PBXGroup; + children = ( + ); + name = Frameworks; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 607A66112B0EFA380010BFC8 /* iOSTestbed */ = { + isa = PBXNativeTarget; + buildConfigurationList = 607A66412B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbed" */; + buildPhases = ( + 607A660E2B0EFA380010BFC8 /* Sources */, + 607A660F2B0EFA380010BFC8 /* Frameworks */, + 607A66102B0EFA380010BFC8 /* Resources */, + 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */, + 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */, + 607A664E2B0EFC080010BFC8 /* Embed Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = iOSTestbed; + productName = iOSTestbed; + productReference = 607A66122B0EFA380010BFC8 /* iOSTestbed.app */; + productType = "com.apple.product-type.application"; + }; + 607A662C2B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 607A66442B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbedTests" */; + buildPhases = ( + 607A66292B0EFA3A0010BFC8 /* Sources */, + 607A662A2B0EFA3A0010BFC8 /* Frameworks */, + 607A662B2B0EFA3A0010BFC8 /* Resources */, + 607A66522B0EFFE00010BFC8 /* Embed Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + 607A662F2B0EFA3A0010BFC8 /* PBXTargetDependency */, + ); + name = iOSTestbedTests; + productName = iOSTestbedTests; + productReference = 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 607A660A2B0EFA380010BFC8 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastUpgradeCheck = 1500; + TargetAttributes = { + 607A66112B0EFA380010BFC8 = { + CreatedOnToolsVersion = 15.0.1; + }; + 607A662C2B0EFA3A0010BFC8 = { + CreatedOnToolsVersion = 15.0.1; + TestTargetID = 607A66112B0EFA380010BFC8; + }; + }; + }; + buildConfigurationList = 607A660D2B0EFA380010BFC8 /* Build configuration list for PBXProject "iOSTestbed" */; + compatibilityVersion = "Xcode 14.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 607A66092B0EFA380010BFC8; + productRefGroup = 607A66132B0EFA380010BFC8 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 607A66112B0EFA380010BFC8 /* iOSTestbed */, + 607A662C2B0EFA3A0010BFC8 /* iOSTestbedTests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 607A66102B0EFA380010BFC8 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */, + 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */, + 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 607A662B2B0EFA3A0010BFC8 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXShellScriptBuildPhase section */ + 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Install Target Specific Python Standard Library"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; + }; + 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Prepare Python Binary Modules"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + }; +/* End PBXShellScriptBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 607A660E2B0EFA380010BFC8 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66172B0EFA380010BFC8 /* AppDelegate.m in Sources */, + 607A66282B0EFA390010BFC8 /* main.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 607A66292B0EFA3A0010BFC8 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 607A662F2B0EFA3A0010BFC8 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 607A66112B0EFA380010BFC8 /* iOSTestbed */; + targetProxy = 607A662E2B0EFA3A0010BFC8 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin PBXVariantGroup section */ + 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 607A66242B0EFA390010BFC8 /* Base */, + ); + name = LaunchScreen.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 607A663F2B0EFA3A0010BFC8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + }; + name = Debug; + }; + 607A66402B0EFA3A0010BFC8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 607A66422B0EFA3A0010BFC8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + INFOPLIST_FILE = "iOSTestbed/iOSTestbed-Info.plist"; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; + INFOPLIST_KEY_UIMainStoryboardFile = Main; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 3.13.0a1; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbed; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 607A66432B0EFA3A0010BFC8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + INFOPLIST_FILE = "iOSTestbed/iOSTestbed-Info.plist"; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; + INFOPLIST_KEY_UIMainStoryboardFile = Main; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 3.13.0a1; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbed; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; + 607A66452B0EFA3A0010BFC8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 3HEZE76D99; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/iOSTestbed.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/iOSTestbed"; + }; + name = Debug; + }; + 607A66462B0EFA3A0010BFC8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 3HEZE76D99; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/iOSTestbed.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/iOSTestbed"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 607A660D2B0EFA380010BFC8 /* Build configuration list for PBXProject "iOSTestbed" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 607A663F2B0EFA3A0010BFC8 /* Debug */, + 607A66402B0EFA3A0010BFC8 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 607A66412B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbed" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 607A66422B0EFA3A0010BFC8 /* Debug */, + 607A66432B0EFA3A0010BFC8 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 607A66442B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbedTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 607A66452B0EFA3A0010BFC8 /* Debug */, + 607A66462B0EFA3A0010BFC8 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 607A660A2B0EFA380010BFC8 /* Project object */; +} diff --git a/iOS/testbed/iOSTestbed/AppDelegate.h b/iOS/testbed/iOSTestbed/AppDelegate.h new file mode 100644 index 00000000000000..f695b3b5efc08b --- /dev/null +++ b/iOS/testbed/iOSTestbed/AppDelegate.h @@ -0,0 +1,11 @@ +// +// AppDelegate.h +// iOSTestbed +// + +#import + +@interface AppDelegate : UIResponder + + +@end diff --git a/iOS/testbed/iOSTestbed/AppDelegate.m b/iOS/testbed/iOSTestbed/AppDelegate.m new file mode 100644 index 00000000000000..e5085399d0ca5f --- /dev/null +++ b/iOS/testbed/iOSTestbed/AppDelegate.m @@ -0,0 +1,19 @@ +// +// AppDelegate.m +// iOSTestbed +// + +#import "AppDelegate.h" + +@interface AppDelegate () + +@end + +@implementation AppDelegate + + +- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + return YES; +} + +@end diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json b/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000000000..eb878970081645 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json b/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000000000..13613e3ee1a934 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json b/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json new file mode 100644 index 00000000000000..73c00596a7fca3 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard b/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard new file mode 100644 index 00000000000000..5daafe73a866b7 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/iOS/testbed/iOSTestbed/dylib-Info-template.plist b/iOS/testbed/iOSTestbed/dylib-Info-template.plist new file mode 100644 index 00000000000000..f652e272f71c88 --- /dev/null +++ b/iOS/testbed/iOSTestbed/dylib-Info-template.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + + CFBundleIdentifier + + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSupportedPlatforms + + iPhoneOS + + MinimumOSVersion + 12.0 + CFBundleVersion + 1 + + diff --git a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist b/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist new file mode 100644 index 00000000000000..e2aa460b6fd5ee --- /dev/null +++ b/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist @@ -0,0 +1,54 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + ${PRODUCT_NAME} + CFBundleExecutable + ${EXECUTABLE_NAME} + CFBundleIdentifier + org.python.iOSTestbed + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + ${PRODUCT_NAME} + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UIRequiresFullScreen + + UILaunchStoryboardName + Launch Screen + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + MainModule + ios + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + + + diff --git a/iOS/testbed/iOSTestbed/main.m b/iOS/testbed/iOSTestbed/main.m new file mode 100644 index 00000000000000..e32bd78c9b42ee --- /dev/null +++ b/iOS/testbed/iOSTestbed/main.m @@ -0,0 +1,16 @@ +// +// main.m +// iOSTestbed +// + +#import +#import "AppDelegate.h" + +int main(int argc, char * argv[]) { + NSString * appDelegateClassName; + @autoreleasepool { + appDelegateClassName = NSStringFromClass([AppDelegate class]); + + return UIApplicationMain(argc, argv, nil, appDelegateClassName); + } +} diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m new file mode 100644 index 00000000000000..9bf502a808eb88 --- /dev/null +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -0,0 +1,113 @@ +#import +#import + +@interface iOSTestbedTests : XCTestCase + +@end + +@implementation iOSTestbedTests + + +- (void)testPython { + // Arguments to pass into the test suite runner. + // argv[0] must identify the process; any subsequent arg + // will be handled as if it were an argument to `python -m test` + const char *argv[] = { + "iOSTestbed", // argv[0] is the process that is running. + "-uall", // Enable all resources + "--single-process", // always run all tests sequentially in a single process + "--rerun", // Re-run failed tests in verbose mode + "-W", // Display test output on failure + // To run a subset of tests, add the test names below; e.g., + // "test_os", + // "test_sys", + }; + + // Start a Python interpreter. + int exit_code; + PyStatus status; + PyPreConfig preconfig; + PyConfig config; + NSString *python_home; + wchar_t *wtmp_str; + + NSString *resourcePath = [[NSBundle mainBundle] resourcePath]; + + // Generate an isolated Python configuration. + NSLog(@"Configuring isolated Python..."); + PyPreConfig_InitIsolatedConfig(&preconfig); + PyConfig_InitIsolatedConfig(&config); + + // Configure the Python interpreter: + // Enforce UTF-8 encoding for stderr, stdout, file-system encoding and locale. + // See https://docs.python.org/3/library/os.html#python-utf-8-mode. + preconfig.utf8_mode = 1; + // Don't buffer stdio. We want output to appears in the log immediately + config.buffered_stdio = 0; + // Don't write bytecode; we can't modify the app bundle + // after it has been signed. + config.write_bytecode = 0; + // Ensure that signal handlers are installed + config.install_signal_handlers = 1; + // Run the test module. + config.run_module = Py_DecodeLocale("test", NULL); + // For debugging - enable verbose mode. + // config.verbose = 1; + + NSLog(@"Pre-initializing Python runtime..."); + status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to pre-initialize Python interpreter: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + // Set the home for the Python interpreter + python_home = [NSString stringWithFormat:@"%@/python", resourcePath, nil]; + NSLog(@"PythonHome: %@", python_home); + wtmp_str = Py_DecodeLocale([python_home UTF8String], NULL); + status = PyConfig_SetString(&config, &config.home, wtmp_str); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to set PYTHONHOME: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + PyMem_RawFree(wtmp_str); + + // Read the site config + status = PyConfig_Read(&config); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to read site config: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + NSLog(@"Configure argc/argv..."); + status = PyConfig_SetBytesArgv(&config, sizeof(argv) / sizeof(char *), (char**) argv); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to configure argc/argv: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + NSLog(@"Initializing Python runtime..."); + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to initialize Python interpreter: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + // Start the test suite. Print a separator to differentiate Python startup logs from app logs + NSLog(@"---------------------------------------------------------------------------"); + + exit_code = Py_RunMain(); + XCTAssertEqual(exit_code, 0, @"Python test suite did not pass"); + + NSLog(@"---------------------------------------------------------------------------"); + + Py_Finalize(); +} + + +@end diff --git a/pyconfig.h.in b/pyconfig.h.in index 2b4bb1a2b52866..fcb8a965b1e476 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -54,6 +54,12 @@ /* Define if getpgrp() must be called as getpgrp(0). */ #undef GETPGRP_HAVE_ARG +/* HACL* library can compile SIMD128 implementations */ +#undef HACL_CAN_COMPILE_SIMD128 + +/* HACL* library can compile SIMD256 implementations */ +#undef HACL_CAN_COMPILE_SIMD256 + /* Define if you have the 'accept' function. */ #undef HAVE_ACCEPT @@ -157,6 +163,9 @@ /* Define to 1 if you have the `clock_settime' function. */ #undef HAVE_CLOCK_SETTIME +/* Define to 1 if the system has the type `clock_t'. */ +#undef HAVE_CLOCK_T + /* Define to 1 if you have the `closefrom' function. */ #undef HAVE_CLOSEFROM @@ -667,9 +676,6 @@ /* Define to 1 if you have the `lchown' function. */ #undef HAVE_LCHOWN -/* Define to 1 if you want to build _blake2 module with libb2 */ -#undef HAVE_LIBB2 - /* Define to 1 if you have the `db' library (-ldb). */ #undef HAVE_LIBDB @@ -685,9 +691,6 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LIBINTL_H -/* Define to 1 if you have the `resolv' library (-lresolv). */ -#undef HAVE_LIBRESOLV - /* Define to 1 if you have the `sendfile' library (-lsendfile). */ #undef HAVE_LIBSENDFILE @@ -733,6 +736,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_MEMFD_H +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_NETFILTER_IPV4_H + /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_NETLINK_H @@ -826,12 +832,33 @@ /* Define to 1 if you have the `nanosleep' function. */ #undef HAVE_NANOSLEEP -/* Define to 1 if you have the `ncursesw' library. */ +/* Define if you have the 'ncurses' library */ +#undef HAVE_NCURSES + +/* Define if you have the 'ncursesw' library */ #undef HAVE_NCURSESW +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSESW_CURSES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSESW_NCURSES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSESW_PANEL_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_CURSES_H + /* Define to 1 if you have the header file. */ #undef HAVE_NCURSES_H +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_NCURSES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_PANEL_H + /* Define to 1 if you have the header file. */ #undef HAVE_NDBM_H @@ -875,6 +902,12 @@ /* Define to 1 if you have the `openpty' function. */ #undef HAVE_OPENPTY +/* Define if you have the 'panel' library */ +#undef HAVE_PANEL + +/* Define if you have the 'panelw' library */ +#undef HAVE_PANELW + /* Define to 1 if you have the header file. */ #undef HAVE_PANEL_H @@ -933,6 +966,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_PROCESS_H +/* Define to 1 if you have the `process_vm_readv' function. */ +#undef HAVE_PROCESS_VM_READV + /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES @@ -1009,7 +1045,7 @@ /* Define if you can turn off readline's signal handling. */ #undef HAVE_RL_CATCH_SIGNAL -/* Define if readline supports rl_compdisp_func_t */ +/* Define to 1 if the system has the type `rl_compdisp_func_t'. */ #undef HAVE_RL_COMPDISP_FUNC_T /* Define if you have readline 2.2 */ @@ -1195,13 +1231,16 @@ /* Define if you have the 'socketpair' function. */ #undef HAVE_SOCKETPAIR +/* Define to 1 if the system has the type `socklen_t'. */ +#undef HAVE_SOCKLEN_T + /* Define to 1 if you have the header file. */ #undef HAVE_SPAWN_H /* Define to 1 if you have the `splice' function. */ #undef HAVE_SPLICE -/* Define if your compiler provides ssize_t */ +/* Define to 1 if the system has the type `ssize_t'. */ #undef HAVE_SSIZE_T /* Define to 1 if you have the `statvfs' function. */ @@ -1568,6 +1607,9 @@ /* Define to 1 if you have the `_getpty' function. */ #undef HAVE__GETPTY +/* Define to 1 if the system has the type `__uint128_t'. */ +#undef HAVE___UINT128_T + /* Define to 1 if `major', `minor', and `makedev' are declared in . */ #undef MAJOR_IN_MKDEV @@ -1640,6 +1682,9 @@ /* Defined if Python is built as a shared library. */ #undef Py_ENABLE_SHARED +/* Defined if _Complex C type can be used with libffi. */ +#undef Py_FFI_SUPPORT_C_COMPLEX + /* Define if you want to disable the GIL */ #undef Py_GIL_DISABLED @@ -1647,9 +1692,21 @@ SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM +/* Defined if _Complex C type is available. */ +#undef Py_HAVE_C_COMPLEX + +/* Define if year with century should be normalized for strftime. */ +#undef Py_NORMALIZE_CENTURY + +/* Define if rl_startup_hook takes arguments */ +#undef Py_RL_STARTUP_HOOK_TAKES_ARGS + /* Define if you want to enable internal statistics gathering. */ #undef Py_STATS +/* Define if C99-specific strftime specifiers are supported. */ +#undef Py_STRFTIME_C99_SUPPORT + /* The version of SunOS/Solaris as reported by `uname -r' without the dot. */ #undef Py_SUNOS_VERSION @@ -1847,9 +1904,6 @@ /* Define to build the readline module against libedit. */ #undef WITH_EDITLINE -/* Define if you want to compile in object freelists optimization */ -#undef WITH_FREELISTS - /* Define to 1 if libintl is needed for locale functions. */ #undef WITH_LIBINTL @@ -1932,7 +1986,7 @@ /* Define on FreeBSD to activate all library features */ #undef __BSD_VISIBLE -/* Define to 'long' if doesn't define. */ +/* Define to 'long' if does not define clock_t. */ #undef clock_t /* Define to empty if `const' does not conform to ANSI C. */ @@ -1956,7 +2010,7 @@ /* Define to `unsigned int' if does not define. */ #undef size_t -/* Define to `int' if does not define. */ +/* Define to 'int' if does not define. */ #undef socklen_t /* Define to `int' if doesn't define. */